From 2d21ac55c334faf3a56e5634905ed6987fc787d4 Mon Sep 17 00:00:00 2001
From: Apple <opensource@apple.com>
Date: Tue, 30 Oct 2007 03:59:30 +0000
Subject: [PATCH] xnu-1228.tar.gz

---
 APPLE_LICENSE                                 |   513 +-
 EXTERNAL_HEADERS/Makefile                     |     7 +-
 EXTERNAL_HEADERS/ar.h                         |    30 +-
 EXTERNAL_HEADERS/architecture/Makefile        |     3 +
 EXTERNAL_HEADERS/architecture/i386/asm_help.h |    30 +-
 EXTERNAL_HEADERS/architecture/i386/cpu.h      |    48 +-
 EXTERNAL_HEADERS/architecture/i386/desc.h     |    30 +-
 EXTERNAL_HEADERS/architecture/i386/io.h       |    30 +-
 EXTERNAL_HEADERS/architecture/i386/pio.h      |    30 +-
 EXTERNAL_HEADERS/architecture/i386/reg_help.h |    30 +-
 EXTERNAL_HEADERS/architecture/i386/sel.h      |    30 +-
 EXTERNAL_HEADERS/architecture/i386/table.h    |    30 +-
 EXTERNAL_HEADERS/architecture/i386/tss.h      |    30 +-
 EXTERNAL_HEADERS/architecture/ppc/asm_help.h  |    30 +-
 .../architecture/ppc/basic_regs.h             |    30 +-
 EXTERNAL_HEADERS/architecture/ppc/cframe.h    |    30 +-
 EXTERNAL_HEADERS/architecture/ppc/fp_regs.h   |    30 +-
 .../architecture/ppc/macro_help.h             |    30 +-
 .../architecture/ppc/pseudo_inst.h            |    30 +-
 EXTERNAL_HEADERS/architecture/ppc/reg_help.h  |    30 +-
 EXTERNAL_HEADERS/i386/_limits.h               |    30 +-
 EXTERNAL_HEADERS/i386/limits.h                |    30 +-
 EXTERNAL_HEADERS/mach-o/fat.h                 |    30 +-
 EXTERNAL_HEADERS/mach-o/kld.h                 |    30 +-
 EXTERNAL_HEADERS/mach-o/loader.h              |   674 +-
 EXTERNAL_HEADERS/mach-o/nlist.h               |    30 +-
 EXTERNAL_HEADERS/mach-o/reloc.h               |    30 +-
 EXTERNAL_HEADERS/machine/Makefile             |     2 +
 EXTERNAL_HEADERS/machine/limits.h             |    32 +-
 EXTERNAL_HEADERS/ppc/_limits.h                |    30 +-
 EXTERNAL_HEADERS/ppc/limits.h                 |    30 +-
 EXTERNAL_HEADERS/stdarg.h                     |   152 +-
 EXTERNAL_HEADERS/stdint.h                     |    24 +-
 Makefile                                      |    10 +-
 README                                        |   217 +-
 bsd/Makefile                                  |     8 +-
 bsd/bsm/audit.h                               |    19 +-
 bsd/bsm/audit_kernel.h                        |   105 +-
 bsd/bsm/audit_kevents.h                       |    66 +-
 bsd/bsm/audit_klib.h                          |    22 +-
 bsd/bsm/audit_record.h                        |    31 +-
 bsd/conf/MASTER                               |   250 +-
 bsd/conf/MASTER.i386                          |    33 +-
 bsd/conf/MASTER.ppc                           |    24 +-
 bsd/conf/Makefile                             |    98 +-
 bsd/conf/Makefile.i386                        |   280 +-
 bsd/conf/Makefile.ppc                         |   124 +
 bsd/conf/Makefile.template                    |    10 +-
 bsd/conf/compat_hdrs                          |     2 +
 bsd/conf/files                                |   291 +-
 bsd/conf/files.i386                           |    12 +-
 bsd/conf/files.ppc                            |    15 +-
 bsd/conf/param.c                              |    52 +-
 bsd/conf/tools/doconf/Makefile                |    16 +-
 bsd/conf/tools/doconf/doconf.csh              |    16 +-
 bsd/crypto/Makefile                           |     8 +-
 bsd/crypto/aes/Makefile                       |     6 +-
 bsd/crypto/aes/aes.h                          |   110 +-
 .../ppc => bsd/crypto/aes/i386}/Makefile      |    23 +-
 bsd/crypto/aes/i386/aes_modes.c               |   471 +
 bsd/crypto/aes/i386/aes_x86_v2.s              |  1298 ++
 bsd/crypto/aes/i386/aesopt.h                  |   719 +
 bsd/crypto/aes/i386/edefs.h                   |   130 +
 .../i386 => bsd/crypto/aes/ppc}/Makefile      |    23 +-
 bsd/crypto/aes/{ => ppc}/aescrypt.c           |     0
 bsd/crypto/aes/{ => ppc}/aeskey.c             |     0
 bsd/crypto/aes/{ => ppc}/aesopt.h             |     2 +-
 bsd/crypto/aes/{ => ppc}/aestab.c             |     0
 bsd/crypto/aes/{ => ppc}/aestab.h             |     0
 bsd/crypto/blowfish/bf_enc.c                  |    12 +-
 bsd/crypto/des/Makefile                       |     4 +
 bsd/crypto/des/des_ecb.c                      |     3 +-
 bsd/crypto/md5.c                              |   308 -
 bsd/crypto/md5.h                              |    76 -
 bsd/crypto/rc4/Makefile                       |     4 +
 bsd/crypto/sha1.c                             |   274 -
 bsd/crypto/sha1.h                             |    95 +-
 bsd/crypto/sha2/Makefile                      |     4 +
 bsd/crypto/sha2/sha2.c                        |    12 +-
 bsd/dev/busvar.h                              |    32 +-
 bsd/dev/chud/chud_bsd_callback.c              |    56 +-
 bsd/dev/chud/chud_process.c                   |    62 +-
 bsd/dev/dtrace/blist.c                        |   926 +
 bsd/dev/dtrace/blist.h                        |   100 +
 bsd/dev/dtrace/dtrace.c                       | 16547 ++++++++++++++++
 bsd/dev/dtrace/dtrace_alloc.c                 |   160 +
 bsd/dev/dtrace/dtrace_glue.c                  |  1600 ++
 bsd/dev/dtrace/dtrace_ptss.c                  |   309 +
 bsd/dev/dtrace/dtrace_subr.c                  |   156 +
 bsd/dev/dtrace/fasttrap.c                     |  2579 +++
 bsd/dev/dtrace/fbt.c                          |   483 +
 bsd/dev/dtrace/lockstat.c                     |   402 +
 bsd/dev/dtrace/profile_prvd.c                 |   840 +
 bsd/dev/dtrace/sdt.c                          |   673 +
 bsd/dev/dtrace/sdt_subr.c                     |   229 +
 bsd/dev/dtrace/systrace.c                     |  1131 ++
 bsd/dev/dtrace/systrace.h                     |    94 +
 bsd/dev/i386/conf.c                           |    47 +-
 bsd/dev/i386/cons.c                           |    34 +-
 bsd/dev/i386/cons.h                           |    30 +-
 bsd/dev/i386/dis_tables.c                     |  3305 +++
 bsd/dev/i386/dtrace_isa.c                     |   649 +
 bsd/dev/i386/dtrace_subr_x86.c                |   308 +
 bsd/dev/i386/fasttrap_isa.c                   |  2237 +++
 bsd/dev/i386/fasttrap_regset.h                |   122 +
 bsd/dev/i386/fbt_x86.c                        |   824 +
 bsd/dev/i386/instr_size.c                     |   138 +
 bsd/dev/i386/kern_machdep.c                   |    36 +-
 bsd/dev/i386/km.c                             |    30 +-
 bsd/dev/i386/lock_stubs.c                     |    30 +-
 bsd/dev/i386/mem.c                            |    30 +-
 bsd/dev/i386/memmove.c                        |    30 +-
 bsd/dev/i386/munge.s                          |    38 +-
 bsd/dev/i386/pci_device.h                     |    30 +-
 bsd/dev/i386/sdt_x86.c                        |    84 +
 bsd/dev/i386/sel.h                            |    30 +-
 bsd/dev/i386/sel_inline.h                     |    30 +-
 bsd/dev/i386/stubs.c                          |    72 +-
 bsd/dev/i386/sysctl.c                         |   227 +-
 bsd/dev/i386/systemcalls.c                    |   381 +-
 bsd/dev/i386/table_inline.h                   |    30 +-
 bsd/dev/i386/unix_signal.c                    |   347 +-
 bsd/dev/kmreg_com.h                           |    30 +-
 bsd/dev/ldd.h                                 |    30 +-
 bsd/dev/memdev.c                              |    39 +-
 bsd/dev/ppc/conf.c                            |   126 +-
 bsd/dev/ppc/cons.c                            |    83 +-
 bsd/dev/ppc/cons.h                            |    55 +-
 bsd/dev/ppc/dtrace_isa.c                      |   601 +
 bsd/dev/ppc/dtrace_subr_ppc.c                 |   193 +
 bsd/dev/ppc/fasttrap_isa.c                    |   722 +
 bsd/dev/ppc/fbt_ppc.c                         |   680 +
 bsd/dev/ppc/ffs.c                             |    30 +-
 bsd/dev/ppc/ffs.s                             |    30 +-
 bsd/dev/ppc/kern_machdep.c                    |    48 +-
 bsd/dev/ppc/km.c                              |    41 +-
 bsd/dev/ppc/machdep.c                         |    77 +-
 bsd/dev/ppc/mem.c                             |    44 +-
 bsd/dev/ppc/memmove.c                         |    34 +-
 bsd/dev/ppc/munge.s                           |    59 +-
 bsd/dev/ppc/nvram.c                           |    30 +-
 bsd/dev/ppc/ppc_init.c                        |    30 +-
 bsd/dev/ppc/sdt_ppc.c                         |    71 +
 bsd/dev/ppc/stubs.c                           |    35 +-
 bsd/dev/ppc/sysctl.c                          |    39 +-
 bsd/dev/ppc/systemcalls.c                     |   167 +-
 bsd/dev/ppc/unix_signal.c                     |   290 +-
 bsd/dev/ppc/xsumas.s                          |    30 +-
 .../include/WindowsTypesForMac.h              |    30 +-
 bsd/dev/random/YarrowCoreLib/include/yarrow.h |    30 +-
 .../YarrowCoreLib/include/yarrowUtils.h       |    30 +-
 bsd/dev/random/YarrowCoreLib/port/smf.c       |    34 +-
 .../random/YarrowCoreLib/src/assertverify.h   |    30 +-
 bsd/dev/random/YarrowCoreLib/src/comp.c       |    30 +-
 bsd/dev/random/YarrowCoreLib/src/comp.h       |    30 +-
 .../random/YarrowCoreLib/src/entropysources.h |    30 +-
 bsd/dev/random/YarrowCoreLib/src/macOnly.h    |    30 +-
 bsd/dev/random/YarrowCoreLib/src/prng.c       |   100 +-
 bsd/dev/random/YarrowCoreLib/src/prng.h       |    30 +-
 bsd/dev/random/YarrowCoreLib/src/prngpriv.h   |    32 +-
 bsd/dev/random/YarrowCoreLib/src/sha1mod.c    |    66 +-
 bsd/dev/random/YarrowCoreLib/src/sha1mod.h    |    43 +-
 bsd/dev/random/YarrowCoreLib/src/smf.h        |    34 +-
 .../random/YarrowCoreLib/src/userdefines.h    |    30 +-
 .../random/YarrowCoreLib/src/yarrowUtils.c    |    30 +-
 bsd/dev/random/randomdev.c                    |   136 +-
 bsd/dev/random/randomdev.h                    |    30 +-
 bsd/dev/unix_startup.c                        |   100 +-
 bsd/dev/vn/Makefile                           |     1 +
 bsd/dev/vn/shadow.c                           |    41 +-
 bsd/dev/vn/shadow.h                           |    30 +-
 bsd/dev/vn/vn.c                               |   237 +-
 bsd/hfs/MacOSStubs.c                          |    54 +-
 bsd/hfs/hfs.h                                 |   449 +-
 bsd/hfs/hfs_attrlist.c                        |  2133 +-
 bsd/hfs/hfs_attrlist.h                        |    68 +-
 bsd/hfs/hfs_btreeio.c                         |   159 +-
 bsd/hfs/hfs_btreeio.h                         |    57 +
 bsd/hfs/hfs_catalog.c                         |  1881 +-
 bsd/hfs/hfs_catalog.h                         |   149 +-
 bsd/hfs/hfs_chash.c                           |   247 +-
 bsd/hfs/hfs_cnode.c                           |   443 +-
 bsd/hfs/hfs_cnode.h                           |   111 +-
 bsd/hfs/hfs_dbg.h                             |    33 +-
 bsd/hfs/hfs_encodinghint.c                    |    45 +-
 bsd/hfs/hfs_encodings.c                       |   129 +-
 bsd/hfs/hfs_encodings.h                       |    40 +-
 bsd/hfs/hfs_endian.c                          |   120 +-
 bsd/hfs/hfs_endian.h                          |    30 +-
 bsd/hfs/hfs_format.h                          |   195 +-
 bsd/hfs/hfs_fsctl.h                           |    62 +-
 bsd/hfs/hfs_hotfiles.c                        |   190 +-
 bsd/hfs/hfs_hotfiles.h                        |    45 +-
 bsd/hfs/hfs_link.c                            |  1231 +-
 bsd/hfs/hfs_lookup.c                          |   329 +-
 bsd/hfs/hfs_macos_defs.h                      |    78 +-
 bsd/hfs/hfs_mount.h                           |    35 +-
 bsd/hfs/hfs_notification.c                    |    40 +-
 bsd/hfs/hfs_quota.c                           |    61 +-
 bsd/hfs/hfs_quota.h                           |    42 +-
 bsd/hfs/hfs_readwrite.c                       |  1816 +-
 bsd/hfs/hfs_search.c                          |   366 +-
 bsd/hfs/hfs_vfsops.c                          |  1708 +-
 bsd/hfs/hfs_vfsutils.c                        |   827 +-
 bsd/hfs/hfs_vnops.c                           |  1690 +-
 bsd/hfs/hfs_xattr.c                           |  1689 +-
 bsd/hfs/hfscommon/BTree/BTree.c               |   187 +-
 bsd/hfs/hfscommon/BTree/BTreeAllocate.c       |   116 +-
 bsd/hfs/hfscommon/BTree/BTreeMiscOps.c        |    98 +-
 bsd/hfs/hfscommon/BTree/BTreeNodeOps.c        |   227 +-
 bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c    |   147 +-
 bsd/hfs/hfscommon/BTree/BTreeScanner.c        |    37 +-
 bsd/hfs/hfscommon/BTree/BTreeTreeOps.c        |   229 +-
 bsd/hfs/hfscommon/Catalog/CatalogUtilities.c  |    60 +-
 bsd/hfs/hfscommon/Catalog/FileIDsServices.c   |    66 +-
 bsd/hfs/hfscommon/Misc/BTreeWrapper.c         |    54 +-
 bsd/hfs/hfscommon/Misc/FileExtentMapping.c    |   357 +-
 bsd/hfs/hfscommon/Misc/VolumeAllocation.c     |   402 +-
 .../hfscommon/Unicode/UCStringCompareData.h   |    36 +-
 bsd/hfs/hfscommon/Unicode/UnicodeWrappers.c   |   131 +-
 bsd/hfs/hfscommon/headers/BTreeScanner.h      |    30 +-
 bsd/hfs/hfscommon/headers/BTreesInternal.h    |    94 +-
 bsd/hfs/hfscommon/headers/BTreesPrivate.h     |   225 +-
 bsd/hfs/hfscommon/headers/CatalogPrivate.h    |    48 +-
 bsd/hfs/hfscommon/headers/FileMgrInternal.h   |   121 +-
 .../hfscommon/headers/HFSUnicodeWrappers.h    |    38 +-
 bsd/hfs/rangelist.c                           |    64 +-
 bsd/hfs/rangelist.h                           |    30 +-
 bsd/i386/Makefile                             |     6 +-
 bsd/i386/_param.h                             |    42 +
 bsd/i386/_structs.h                           |   105 +
 bsd/i386/_types.h                             |    34 +-
 bsd/i386/dis_tables.h                         |   116 +
 bsd/i386/disklabel.h                          |    30 +-
 bsd/i386/endian.h                             |    35 +-
 bsd/i386/exec.h                               |    30 +-
 bsd/i386/fasttrap_isa.h                       |   116 +
 bsd/i386/param.h                              |    40 +-
 bsd/i386/profile.h                            |    30 +-
 bsd/i386/psl.h                                |    30 +-
 bsd/i386/ptrace.h                             |    30 +-
 bsd/i386/reboot.h                             |    30 +-
 bsd/i386/reg.h                                |    30 +-
 bsd/i386/setjmp.h                             |    50 +-
 bsd/i386/signal.h                             |    64 +-
 bsd/i386/types.h                              |    38 +-
 bsd/i386/ucontext.h                           |    79 +-
 bsd/i386/vmparam.h                            |    33 +-
 bsd/isofs/cd9660/cd9660_bmap.c                |    30 +-
 bsd/isofs/cd9660/cd9660_lookup.c              |    45 +-
 bsd/isofs/cd9660/cd9660_mount.h               |    33 +-
 bsd/isofs/cd9660/cd9660_node.c                |    47 +-
 bsd/isofs/cd9660/cd9660_node.h                |    34 +-
 bsd/isofs/cd9660/cd9660_rrip.c                |    43 +-
 bsd/isofs/cd9660/cd9660_rrip.h                |    30 +-
 bsd/isofs/cd9660/cd9660_util.c                |   675 +-
 bsd/isofs/cd9660/cd9660_vfsops.c              |   135 +-
 bsd/isofs/cd9660/cd9660_vnops.c               |   144 +-
 bsd/isofs/cd9660/iso.h                        |    53 +-
 bsd/isofs/cd9660/iso_rrip.h                   |    30 +-
 bsd/kern/ast.h                                |    30 +-
 bsd/kern/bsd_init.c                           |   590 +-
 bsd/kern/bsd_stubs.c                          |   149 +-
 bsd/kern/imageboot.c                          |   154 +
 bsd/kern/init_sysent.c                        |   971 +-
 bsd/kern/kdebug.c                             |   146 +-
 bsd/kern/kern_acct.c                          |   128 +-
 bsd/kern/kern_aio.c                           |   182 +-
 bsd/kern/kern_audit.c                         |   894 +-
 bsd/kern/kern_authorization.c                 |   104 +-
 bsd/kern/kern_bsm_audit.c                     |   264 +-
 bsd/kern/kern_bsm_klib.c                      |   171 +-
 bsd/kern/kern_bsm_token.c                     |    95 +-
 bsd/kern/kern_clock.c                         |   222 +-
 bsd/kern/kern_control.c                       |   212 +-
 bsd/kern/kern_core.c                          |   157 +-
 bsd/kern/kern_credential.c                    |  2844 ++-
 bsd/kern/kern_descrip.c                       |  2735 ++-
 bsd/kern/kern_event.c                         |   329 +-
 bsd/kern/kern_exec.c                          |  2143 +-
 bsd/kern/kern_exit.c                          |  1322 +-
 bsd/kern/kern_fork.c                          |  1184 +-
 bsd/kern/kern_ktrace.c                        |   622 -
 bsd/kern/kern_lock.c                          |   466 -
 bsd/kern/kern_lockf.c                         |   718 +-
 bsd/kern/kern_malloc.c                        |   532 +-
 bsd/kern/kern_memorystatus.c                  |   116 +
 bsd/kern/kern_mib.c                           |   391 +-
 bsd/kern/kern_mman.c                          |   338 +-
 bsd/kern/kern_newsysctl.c                     |   445 +-
 bsd/kern/kern_panicinfo.c                     |    43 +-
 bsd/kern/kern_pcsamples.c                     |    49 +-
 bsd/kern/kern_physio.c                        |   161 +-
 bsd/kern/kern_proc.c                          |  2231 ++-
 bsd/kern/kern_prot.c                          |  1664 +-
 bsd/kern/kern_resource.c                      |   896 +-
 bsd/kern/kern_shutdown.c                      |   353 +-
 bsd/kern/kern_sig.c                           |  2337 +--
 bsd/kern/kern_subr.c                          |   127 +-
 bsd/kern/kern_symfile.c                       |   363 +-
 bsd/kern/kern_synch.c                         |   117 +-
 bsd/kern/kern_sysctl.c                        |  2148 +-
 bsd/kern/kern_time.c                          |   230 +-
 bsd/kern/kern_xxx.c                           |    58 +-
 bsd/kern/kpi_mbuf.c                           |   330 +-
 bsd/kern/kpi_mbuf_internal.h                  |    36 +
 bsd/kern/kpi_socket.c                         |   213 +-
 bsd/kern/kpi_socketfilter.c                   |    96 +-
 bsd/kern/mach_fat.c                           |    91 +-
 bsd/kern/mach_header.c                        |    40 +-
 bsd/kern/mach_header.h                        |    30 +-
 bsd/kern/mach_loader.c                        |   584 +-
 bsd/kern/mach_loader.h                        |    36 +-
 bsd/kern/mach_process.c                       |   236 +-
 bsd/kern/makesyscalls.sh                      |   228 +-
 bsd/kern/mcache.c                             |  1560 ++
 bsd/kern/md5c.c                               |   342 -
 bsd/kern/netboot.c                            |   170 +-
 bsd/kern/posix_sem.c                          |   244 +-
 bsd/kern/posix_shm.c                          |   338 +-
 bsd/kern/preload.h                            |    30 +-
 bsd/kern/proc_info.c                          |   444 +-
 bsd/kern/pthread_support.c                    |   195 +
 bsd/kern/pthread_synch.c                      |  2087 ++
 bsd/kern/qsort.c                              |    63 +-
 bsd/kern/socket_info.c                        |    42 +-
 bsd/kern/spl.c                                |    36 +-
 bsd/kern/subr_log.c                           |   266 +-
 bsd/kern/subr_prf.c                           |   173 +-
 bsd/kern/subr_prof.c                          |   284 +-
 bsd/kern/subr_sbuf.c                          |   584 +
 bsd/kern/subr_xxx.c                           |    70 +-
 bsd/kern/sys_domain.c                         |    35 +-
 bsd/kern/sys_generic.c                        |   956 +-
 bsd/kern/sys_pipe.c                           |   413 +-
 bsd/kern/sys_socket.c                         |   449 +-
 bsd/kern/syscalls.c                           |   316 +-
 bsd/kern/syscalls.master                      |   951 +-
 bsd/kern/sysctl_init.c                        |   978 +-
 bsd/kern/sysv_ipc.c                           |    39 +-
 bsd/kern/sysv_msg.c                           |   530 +-
 bsd/kern/sysv_sem.c                           |   433 +-
 bsd/kern/sysv_shm.c                           |   389 +-
 bsd/kern/tty.c                                |   938 +-
 bsd/kern/tty_compat.c                         |    30 +-
 bsd/kern/tty_conf.c                           |    99 +-
 bsd/kern/tty_ptmx.c                           |  1454 ++
 bsd/kern/tty_pty.c                            |   435 +-
 bsd/kern/tty_subr.c                           |   114 +-
 bsd/kern/tty_tb.c                             |   376 -
 bsd/kern/tty_tty.c                            |   194 +-
 bsd/kern/ubc_subr.c                           |  2243 ++-
 bsd/kern/uipc_domain.c                        |   227 +-
 bsd/kern/uipc_mbuf.c                          |  5837 ++++--
 bsd/kern/uipc_mbuf2.c                         |   196 +-
 bsd/kern/uipc_proto.c                         |   113 +-
 bsd/kern/uipc_socket.c                        |  2937 ++-
 bsd/kern/uipc_socket2.c                       |  1168 +-
 bsd/kern/uipc_syscalls.c                      |  2175 +-
 bsd/kern/uipc_usrreq.c                        |   806 +-
 bsd/libkern/bcd.c                             |    30 +-
 bsd/libkern/bcmp.c                            |    30 +-
 bsd/libkern/inet_ntop.c                       |    36 +-
 bsd/libkern/libkern.h                         |   112 +-
 bsd/libkern/locc.c                            |    30 +-
 bsd/libkern/random.c                          |    38 +-
 bsd/libkern/rindex.c                          |    30 +-
 bsd/libkern/scanc.c                           |    38 +-
 bsd/libkern/skpc.c                            |    37 +-
 bsd/libkern/strsep.c                          |    77 +
 bsd/libkern/strtol.c                          |    30 +-
 bsd/machine/Makefile                          |     8 +-
 bsd/machine/_limits.h                         |    34 +-
 bsd/machine/_param.h                          |    36 +
 bsd/machine/_structs.h                        |    36 +
 bsd/machine/_types.h                          |    34 +-
 bsd/machine/byte_order.h                      |    32 +-
 bsd/machine/cons.h                            |    36 +-
 bsd/machine/dis_tables.h                      |    39 +
 bsd/machine/disklabel.h                       |    36 +-
 bsd/machine/endian.h                          |    36 +-
 bsd/machine/exec.h                            |    47 +-
 bsd/machine/fasttrap_isa.h                    |    41 +
 bsd/machine/param.h                           |    36 +-
 bsd/machine/profile.h                         |    38 +-
 bsd/machine/psl.h                             |    36 +-
 bsd/machine/ptrace.h                          |    36 +-
 bsd/machine/reboot.h                          |    36 +-
 bsd/machine/reg.h                             |    36 +-
 bsd/machine/setjmp.h                          |    41 +-
 bsd/machine/signal.h                          |    36 +-
 bsd/machine/spl.h                             |    33 +-
 bsd/machine/types.h                           |    36 +-
 bsd/machine/ucontext.h                        |    34 +-
 bsd/machine/vmparam.h                         |    36 +-
 bsd/man/Makefile                              |     2 +
 bsd/man/man2/EV_SET.2                         |     1 +
 bsd/man/man2/FD_CLR.2                         |     1 +
 bsd/man/man2/FD_COPY.2                        |     1 +
 bsd/man/man2/FD_ISSET.2                       |     1 +
 bsd/man/man2/FD_SET.2                         |     1 +
 bsd/man/man2/FD_ZERO.2                        |     1 +
 bsd/man/man2/Makefile                         |   362 +-
 bsd/man/man2/_exit.2                          |    18 +-
 bsd/man/man2/accept.2                         |    90 +-
 bsd/man/man2/access.2                         |    73 +-
 bsd/man/man2/acct.2                           |     4 +-
 bsd/man/man2/adjtime.2                        |     2 +-
 bsd/man/man2/aio_cancel.2                     |    14 +-
 bsd/man/man2/aio_error.2                      |    14 +-
 bsd/man/man2/aio_read.2                       |    79 +-
 bsd/man/man2/aio_return.2                     |    28 +-
 bsd/man/man2/aio_suspend.2                    |    28 +-
 bsd/man/man2/aio_write.2                      |    54 +-
 bsd/man/man2/audit.2                          |    78 +
 bsd/man/man2/auditctl.2                       |    59 +
 bsd/man/man2/auditon.2                        |   195 +
 bsd/man/man2/bind.2                           |   111 +-
 bsd/man/man2/chdir.2                          |    63 +-
 bsd/man/man2/chflags.2                        |    19 +-
 bsd/man/man2/chmod.2                          |    90 +-
 bsd/man/man2/chown.2                          |    78 +-
 bsd/man/man2/close.2                          |    28 +-
 bsd/man/man2/connect.2                        |   153 +-
 bsd/man/man2/dup.2                            |    77 +-
 bsd/man/man2/execve.2                         |   119 +-
 bsd/man/man2/fcntl.2                          |    63 +-
 bsd/man/man2/fgetxattr.2                      |     1 +
 bsd/man/man2/fhopen.2                         |    64 +
 bsd/man/man2/flistxattr.2                     |     1 +
 bsd/man/man2/flock.2                          |     4 +-
 bsd/man/man2/fork.2                           |    45 +-
 bsd/man/man2/fremovexattr.2                   |     1 +
 bsd/man/man2/fsctl.2                          |     2 +-
 bsd/man/man2/fsetxattr.2                      |     1 +
 bsd/man/man2/fstat64.2                        |     1 +
 bsd/man/man2/fstatfs64.2                      |     1 +
 bsd/man/man2/fsync.2                          |    81 +-
 bsd/man/man2/getattrlist.2                    |  1287 +-
 bsd/man/man2/getaudit.2                       |    61 +
 bsd/man/man2/getauid.2                        |    55 +
 bsd/man/man2/getdirentries.2                  |     4 +-
 bsd/man/man2/getfh.2                          |    46 +-
 bsd/man/man2/getgid.2                         |    24 +-
 bsd/man/man2/getgroups.2                      |    52 +-
 bsd/man/man2/getitimer.2                      |    37 +-
 bsd/man/man2/getlcid.2                        |    87 +
 bsd/man/man2/getlogin.2                       |     8 +-
 bsd/man/man2/getpeername.2                    |    55 +-
 bsd/man/man2/getpgrp.2                        |    14 +-
 bsd/man/man2/getpid.2                         |    19 +-
 bsd/man/man2/getpriority.2                    |   108 +-
 bsd/man/man2/getrlimit.2                      |    82 +-
 bsd/man/man2/getrusage.2                      |    35 +-
 bsd/man/man2/getsid.2                         |    10 +-
 bsd/man/man2/getsockname.2                    |    59 +-
 bsd/man/man2/getsockopt.2                     |   132 +-
 bsd/man/man2/gettimeofday.2                   |    36 +-
 bsd/man/man2/getuid.2                         |    23 +-
 bsd/man/man2/getxattr.2                       |     4 +-
 bsd/man/man2/i386_set_ldt.2                   |     1 +
 bsd/man/man2/intro.2                          |     5 +-
 bsd/man/man2/ioctl.2                          |    34 +-
 bsd/man/man2/issetugid.2                      |    85 +-
 bsd/man/man2/kevent.2                         |     1 +
 bsd/man/man2/kill.2                           |    51 +-
 bsd/man/man2/kqueue.2                         |    35 +-
 bsd/man/man2/ktrace.2                         |   168 -
 bsd/man/man2/link.2                           |   131 +-
 bsd/man/man2/listen.2                         |    53 +-
 bsd/man/man2/listxattr.2                      |     4 +-
 bsd/man/man2/lseek.2                          |    30 +-
 bsd/man/man2/lstat64.2                        |     2 +
 bsd/man/man2/madvise.2                        |    93 +-
 bsd/man/man2/minherit.2                       |    36 +-
 bsd/man/man2/mkdir.2                          |    90 +-
 bsd/man/man2/mknod.2                          |   104 +-
 bsd/man/man2/mlock.2                          |   106 +-
 bsd/man/man2/mmap.2                           |   169 +-
 bsd/man/man2/mount.2                          |     2 +-
 bsd/man/man2/mprotect.2                       |    56 +-
 bsd/man/man2/msync.2                          |    36 +-
 bsd/man/man2/munmap.2                         |    41 +-
 bsd/man/man2/nfsclnt.2                        |    97 +
 bsd/man/man2/nfssvc.2                         |   209 +-
 bsd/man/man2/open.2                           |   269 +-
 bsd/man/man2/pathconf.2                       |    52 +-
 bsd/man/man2/pipe.2                           |    47 +-
 bsd/man/man2/poll.2                           |   112 +-
 bsd/man/man2/posix_madvise.2                  |     2 +-
 bsd/man/man2/posix_spawn.2                    |   349 +
 bsd/man/man2/profil.2                         |    10 +-
 bsd/man/man2/quotactl.2                       |     1 +
 bsd/man/man2/read.2                           |   168 +-
 bsd/man/man2/readlink.2                       |    69 +-
 bsd/man/man2/reboot.2                         |    20 +-
 bsd/man/man2/recv.2                           |   216 +-
 bsd/man/man2/removexattr.2                    |     4 +-
 bsd/man/man2/rename.2                         |   209 +-
 bsd/man/man2/rmdir.2                          |    55 +-
 bsd/man/man2/select.2                         |   109 +-
 bsd/man/man2/semctl.2                         |    33 +-
 bsd/man/man2/semget.2                         |    62 +-
 bsd/man/man2/semop.2                          |    80 +-
 bsd/man/man2/send.2                           |   181 +-
 bsd/man/man2/sendfile.2                       |   236 +
 bsd/man/man2/setaudit.2                       |    56 +
 bsd/man/man2/setauid.2                        |    54 +
 bsd/man/man2/setlcid.2                        |    90 +
 bsd/man/man2/setpgid.2                        |    84 +-
 bsd/man/man2/setsid.2                         |    15 +-
 bsd/man/man2/setuid.2                         |    52 +-
 bsd/man/man2/setxattr.2                       |     4 +-
 bsd/man/man2/shmat.2                          |    42 +-
 bsd/man/man2/shmctl.2                         |   121 +-
 bsd/man/man2/shmget.2                         |    55 +-
 bsd/man/man2/shutdown.2                       |    27 +-
 bsd/man/man2/sigaction.2                      |    11 +-
 bsd/man/man2/sigaltstack.2                    |    71 +-
 bsd/man/man2/sigpending.2                     |     4 +-
 bsd/man/man2/sigprocmask.2                    |    15 +-
 bsd/man/man2/sigsuspend.2                     |     8 +-
 bsd/man/man2/socket.2                         |    60 +-
 bsd/man/man2/socketpair.2                     |    63 +-
 bsd/man/man2/stat.2                           |   195 +-
 bsd/man/man2/stat64.2                         |     1 +
 bsd/man/man2/statfs.2                         |    91 +-
 bsd/man/man2/statfs64.2                       |     3 +
 bsd/man/man2/symlink.2                        |   129 +-
 bsd/man/man2/sync.2                           |     4 +-
 bsd/man/man2/truncate.2                       |   122 +-
 bsd/man/man2/umask.2                          |    26 +-
 bsd/man/man2/unlink.2                         |    67 +-
 bsd/man/man2/utimes.2                         |    46 +-
 bsd/man/man2/vfork.2                          |    29 +-
 bsd/man/man2/wait.2                           |   126 +-
 bsd/man/man2/write.2                          |   178 +-
 bsd/man/man3/Makefile                         |    62 +
 .../man3/posix_spawn_file_actions_addclose.3  |   154 +
 bsd/man/man3/posix_spawn_file_actions_init.3  |   115 +
 bsd/man/man3/posix_spawnattr_init.3           |   126 +
 bsd/man/man3/posix_spawnattr_setbinpref_np.3  |   152 +
 bsd/man/man3/posix_spawnattr_setflags.3       |   169 +
 bsd/man/man3/posix_spawnattr_setpgroup.3      |   114 +
 bsd/man/man3/posix_spawnattr_setsigdefault.3  |   114 +
 bsd/man/man3/posix_spawnattr_setsigmask.3     |   113 +
 .../man3/posix_spawnattr_setspecialport_np.3  |   125 +
 bsd/man/man3/queue.3                          |   977 +
 bsd/man/man4/Makefile                         |     2 +-
 bsd/man/man4/aio.4                            |    52 +
 bsd/man/man4/icmp.4                           |    24 +-
 bsd/man/man4/icmp6.4                          |    20 +
 bsd/man/man4/inet.4                           |     6 +-
 bsd/man/man4/ip.4                             |    17 +-
 bsd/man/man4/lo.4                             |     2 +-
 bsd/man/man4/netintro.4                       |     4 +-
 bsd/man/man4/random.4                         |     4 +-
 bsd/man/man4/route.4                          |     7 +-
 bsd/man/man4/tcp.4                            |     2 +-
 bsd/man/man4/tun.4                            |     4 +-
 bsd/man/man4/udp.4                            |     2 +-
 bsd/man/man5/acct.5                           |     4 +-
 bsd/man/man5/core.5                           |    15 +-
 bsd/man/man5/types.5                          |     6 +-
 bsd/man/man8/Makefile                         |    18 +
 bsd/man/{man2/sigreturn.2 => man8/sticky.8}   |   114 +-
 bsd/man/man9/Makefile                         |    24 +-
 bsd/man/man9/copy.9                           |    78 +-
 bsd/man/man9/copyin.9                         |     1 +
 bsd/man/man9/copyinstr.9                      |     1 +
 bsd/man/man9/copyout.9                        |     1 +
 bsd/man/man9/copystr.9                        |     1 +
 bsd/man/man9/fetch.9                          |    43 +-
 bsd/man/man9/fubyte.9                         |     1 +
 bsd/man/man9/fuibyte.9                        |     1 +
 bsd/man/man9/fuiword.9                        |     1 +
 bsd/man/man9/fulong.9                         |     1 +
 bsd/man/man9/fuulong.9                        |     1 +
 bsd/man/man9/fuword.9                         |     1 +
 bsd/man/man9/store.9                          |    46 +-
 bsd/man/man9/style.9                          |    80 +-
 bsd/man/man9/subyte.9                         |     1 +
 bsd/man/man9/suibyte.9                        |     1 +
 bsd/man/man9/suiword.9                        |     1 +
 bsd/man/man9/sulong.9                         |     1 +
 bsd/man/man9/suulong.9                        |     1 +
 bsd/man/man9/suword.9                         |     1 +
 bsd/miscfs/deadfs/dead_vnops.c                |   132 +-
 bsd/miscfs/devfs/Makefile                     |     5 +-
 bsd/miscfs/devfs/devfs.h                      |    62 +-
 bsd/miscfs/devfs/devfs_proto.h                |    43 +-
 bsd/miscfs/devfs/devfs_tree.c                 |   282 +-
 bsd/miscfs/devfs/devfs_vfsops.c               |   102 +-
 bsd/miscfs/devfs/devfs_vnops.c                |   196 +-
 bsd/miscfs/devfs/devfsdefs.h                  |    73 +-
 bsd/miscfs/devfs/reproto.sh                   |     4 +-
 bsd/miscfs/fdesc/Makefile                     |     4 +-
 bsd/miscfs/fdesc/fdesc.h                      |    38 +-
 bsd/miscfs/fdesc/fdesc_vfsops.c               |    88 +-
 bsd/miscfs/fdesc/fdesc_vnops.c                |   160 +-
 bsd/miscfs/fifofs/fifo.h                      |    30 +-
 bsd/miscfs/fifofs/fifo_vnops.c                |   137 +-
 bsd/miscfs/nullfs/null.h                      |    31 +-
 bsd/miscfs/nullfs/null_subr.c                 |    30 +-
 bsd/miscfs/nullfs/null_vfsops.c               |    30 +-
 bsd/miscfs/nullfs/null_vnops.c                |    30 +-
 bsd/miscfs/specfs/spec_vnops.c                |   354 +-
 bsd/miscfs/specfs/specdev.h                   |    31 +-
 bsd/miscfs/synthfs/synthfs.h                  |    30 +-
 bsd/miscfs/synthfs/synthfs_util.c             |   146 +-
 bsd/miscfs/synthfs/synthfs_vfsops.c           |    57 +-
 bsd/miscfs/synthfs/synthfs_vnops.c            |    34 +-
 bsd/miscfs/union/union.h                      |   146 +-
 bsd/miscfs/union/union_subr.c                 |  1040 +-
 bsd/miscfs/union/union_vfsops.c               |   101 +-
 bsd/miscfs/union/union_vnops.c                |   806 +-
 bsd/miscfs/volfs/volfs.h                      |    67 -
 bsd/miscfs/volfs/volfs_vfsops.c               |   424 -
 bsd/miscfs/volfs/volfs_vnops.c                |   978 -
 bsd/net/Makefile                              |     4 +-
 bsd/net/bpf.c                                 |   997 +-
 bsd/net/bpf.h                                 |   199 +-
 bsd/net/bpf_compat.h                          |    30 +-
 bsd/net/bpf_filter.c                          |    34 +-
 bsd/net/bpfdesc.h                             |    38 +-
 bsd/net/bridge.c                              |    34 +-
 bsd/net/bridge.h                              |    30 +-
 bsd/net/bsd_comp.c                            |    45 +-
 bsd/net/devtimer.c                            |    36 +-
 bsd/net/devtimer.h                            |    30 +-
 bsd/net/dlil.c                                |  2091 +-
 bsd/net/dlil.h                                |   596 +-
 bsd/net/dlil_pvt.h                            |    30 +-
 bsd/net/ether_at_pr_module.c                  |   317 +-
 bsd/net/ether_if_module.c                     |   252 +-
 bsd/net/ether_if_module.h                     |    41 +
 bsd/net/ether_inet6_pr_module.c               |    93 +-
 bsd/net/ether_inet_pr_module.c                |   212 +-
 bsd/net/etherdefs.h                           |    30 +-
 bsd/net/ethernet.h                            |    31 +-
 bsd/net/firewire.h                            |    30 +-
 bsd/net/ieee8023ad.h                          |    30 +-
 bsd/net/if.c                                  |   466 +-
 bsd/net/if.h                                  |    94 +-
 bsd/net/if_arp.h                              |    30 +-
 bsd/net/if_atm.h                              |    30 +-
 bsd/net/if_bond.c                             |   927 +-
 bsd/net/if_bond_var.h                         |    52 +-
 bsd/net/if_disc.c                             |    45 +-
 bsd/net/if_dl.h                               |    30 +-
 bsd/net/if_dummy.c                            |    56 +-
 bsd/net/if_ether.h                            |    32 +-
 bsd/net/if_ethersubr.c                        |    34 +-
 bsd/net/if_faith.c                            |   452 -
 bsd/net/if_fddisubr.c                         |    30 +-
 bsd/net/if_gif.c                              |   386 +-
 bsd/net/if_gif.h                              |    45 +-
 bsd/net/if_llc.h                              |    30 +-
 bsd/net/if_loop.c                             |   285 +-
 bsd/net/if_media.c                            |    30 +-
 bsd/net/if_media.h                            |    30 +-
 bsd/net/if_mib.c                              |    43 +-
 bsd/net/if_mib.h                              |    30 +-
 bsd/net/if_ppp.h                              |    30 +-
 bsd/net/if_pppvar.h                           |    30 +-
 bsd/net/if_stf.c                              |   437 +-
 bsd/net/if_types.h                            |    30 +-
 bsd/net/if_var.h                              |   204 +-
 bsd/net/if_vlan.c                             |   493 +-
 bsd/net/if_vlan_var.h                         |    32 +-
 bsd/net/init.c                                |    30 +-
 bsd/net/init.h                                |    33 +-
 bsd/net/kext_net.c                            |    30 +-
 bsd/net/kext_net.h                            |    36 +-
 bsd/net/kpi_interface.c                       |   249 +-
 bsd/net/kpi_interface.h                       |   139 +-
 bsd/net/kpi_interfacefilter.c                 |    30 +-
 bsd/net/kpi_interfacefilter.h                 |    45 +-
 bsd/net/kpi_protocol.c                        |   390 +-
 bsd/net/kpi_protocol.h                        |    98 +-
 bsd/net/lacp.h                                |    30 +-
 bsd/net/multicast_list.c                      |    30 +-
 bsd/net/multicast_list.h                      |    30 +-
 bsd/net/ndrv.c                                |   355 +-
 bsd/net/ndrv.h                                |    44 +-
 bsd/net/ndrv_var.h                            |    32 +-
 bsd/net/net_osdep.c                           |    35 +-
 bsd/net/net_osdep.h                           |    30 +-
 bsd/net/pfkeyv2.h                             |    35 +-
 bsd/net/ppp_comp.h                            |    30 +-
 bsd/net/ppp_deflate.c                         |    47 +-
 bsd/net/ppp_defs.h                            |    30 +-
 bsd/net/radix.c                               |   262 +-
 bsd/net/radix.h                               |    32 +-
 bsd/net/raw_cb.c                              |    51 +-
 bsd/net/raw_cb.h                              |    30 +-
 bsd/net/raw_usrreq.c                          |    71 +-
 bsd/net/route.c                               |   542 +-
 bsd/net/route.h                               |   167 +-
 bsd/net/rtsock.c                              |   137 +-
 bsd/net/rtsock_mip.c                          |    30 +-
 bsd/net/slip.h                                |    30 +-
 bsd/net/zlib.h                                |  1221 +-
 bsd/netat/Makefile                            |     1 +
 bsd/netat/adsp.c                              |    55 +-
 bsd/netat/adsp.h                              |    36 +-
 bsd/netat/adsp_CLDeny.c                       |    34 +-
 bsd/netat/adsp_CLListen.c                     |    36 +-
 bsd/netat/adsp_Close.c                        |    64 +-
 bsd/netat/adsp_Control.c                      |    49 +-
 bsd/netat/adsp_Init.c                         |    36 +-
 bsd/netat/adsp_InitGlobals.c                  |    30 +-
 bsd/netat/adsp_NewCID.c                       |    32 +-
 bsd/netat/adsp_Open.c                         |    41 +-
 bsd/netat/adsp_Options.c                      |    34 +-
 bsd/netat/adsp_Packet.c                       |    62 +-
 bsd/netat/adsp_Read.c                         |    74 +-
 bsd/netat/adsp_RxAttn.c                       |    33 +-
 bsd/netat/adsp_RxData.c                       |    36 +-
 bsd/netat/adsp_Status.c                       |    40 +-
 bsd/netat/adsp_Timer.c                        |    44 +-
 bsd/netat/adsp_TimerElem.c                    |    38 +-
 bsd/netat/adsp_Write.c                        |    39 +-
 bsd/netat/adsp_attention.c                    |    32 +-
 bsd/netat/adsp_internal.h                     |    80 +-
 bsd/netat/adsp_misc.c                         |    34 +-
 bsd/netat/adsp_reset.c                        |    42 +-
 bsd/netat/adsp_stream.c                       |    56 +-
 bsd/netat/appletalk.h                         |    30 +-
 bsd/netat/asp.h                               |    48 +-
 bsd/netat/asp_proto.c                         |   127 +-
 bsd/netat/at.c                                |   114 +-
 bsd/netat/at_aarp.h                           |    38 +-
 bsd/netat/at_config.h                         |    30 +-
 bsd/netat/at_ddp_brt.h                        |    34 +-
 bsd/netat/at_pat.h                            |    31 +-
 bsd/netat/at_pcb.c                            |    34 +-
 bsd/netat/at_pcb.h                            |    44 +-
 bsd/netat/at_proto.c                          |    72 +-
 bsd/netat/at_snmp.h                           |    36 +-
 bsd/netat/at_var.h                            |    72 +-
 bsd/netat/atp.h                               |    57 +-
 bsd/netat/atp_alloc.c                         |    42 +-
 bsd/netat/atp_misc.c                          |    45 +-
 bsd/netat/atp_open.c                          |    36 +-
 bsd/netat/atp_read.c                          |    40 +-
 bsd/netat/atp_write.c                         |    91 +-
 bsd/netat/aurp.h                              |    31 +-
 bsd/netat/aurp_aurpd.c                        |    34 +-
 bsd/netat/aurp_cfg.c                          |    32 +-
 bsd/netat/aurp_gdata.c                        |    32 +-
 bsd/netat/aurp_misc.c                         |    33 +-
 bsd/netat/aurp_open.c                         |    33 +-
 bsd/netat/aurp_rd.c                           |    33 +-
 bsd/netat/aurp_ri.c                           |    31 +-
 bsd/netat/aurp_rx.c                           |    33 +-
 bsd/netat/aurp_tickle.c                       |    33 +-
 bsd/netat/aurp_tx.c                           |    33 +-
 bsd/netat/aurp_zi.c                           |    33 +-
 bsd/netat/ddp.c                               |   104 +-
 bsd/netat/ddp.h                               |   121 +-
 bsd/netat/ddp_aarp.c                          |    82 +-
 bsd/netat/ddp_aep.c                           |    34 +-
 bsd/netat/ddp_brt.c                           |    35 +-
 bsd/netat/ddp_lap.c                           |   128 +-
 bsd/netat/ddp_nbp.c                           |   153 +-
 bsd/netat/ddp_proto.c                         |    33 +-
 bsd/netat/ddp_r_rtmp.c                        |   113 +-
 bsd/netat/ddp_r_zip.c                         |   113 +-
 bsd/netat/ddp_rtmp.c                          |    51 +-
 bsd/netat/ddp_rtmptable.c                     |    84 +-
 bsd/netat/ddp_sip.c                           |    41 +-
 bsd/netat/ddp_usrreq.c                        |    61 +-
 bsd/netat/debug.h                             |    51 +-
 bsd/netat/drv_dep.c                           |    79 +-
 bsd/netat/ep.h                                |    30 +-
 bsd/netat/lap.h                               |    30 +-
 bsd/netat/nbp.h                               |    40 +-
 bsd/netat/pap.h                               |    30 +-
 bsd/netat/routing_tables.h                    |    47 +-
 bsd/netat/rtmp.h                              |    47 +-
 bsd/netat/sys_dep.c                           |    69 +-
 bsd/netat/sys_glue.c                          |   251 +-
 bsd/netat/sysglue.h                           |    40 +-
 bsd/netat/zip.h                               |    30 +-
 bsd/netinet/bootp.h                           |    34 +-
 bsd/netinet/dhcp.h                            |    51 +-
 bsd/netinet/dhcp_options.c                    |   291 +-
 bsd/netinet/dhcp_options.h                    |   136 +-
 bsd/netinet/icmp6.h                           |    38 +-
 bsd/netinet/icmp_var.h                        |    54 +-
 bsd/netinet/if_atm.c                          |    36 +-
 bsd/netinet/if_atm.h                          |    30 +-
 bsd/netinet/if_ether.h                        |    30 +-
 bsd/netinet/if_fddi.h                         |    30 +-
 bsd/netinet/if_tun.h                          |    30 +-
 bsd/netinet/igmp.c                            |    75 +-
 bsd/netinet/igmp.h                            |    30 +-
 bsd/netinet/igmp_var.h                        |    32 +-
 bsd/netinet/in.c                              |   357 +-
 bsd/netinet/in.h                              |   105 +-
 bsd/netinet/in_arp.c                          |   149 +-
 bsd/netinet/in_arp.h                          |    30 +-
 bsd/netinet/in_bootp.c                        |   612 -
 bsd/netinet/in_cksum.c                        |   310 +-
 bsd/netinet/in_dhcp.c                         |   962 +
 bsd/netinet/in_dhcp.h                         |    49 +
 bsd/netinet/in_gif.c                          |    88 +-
 bsd/netinet/in_gif.h                          |    30 +-
 bsd/netinet/in_pcb.c                          |   495 +-
 bsd/netinet/in_pcb.h                          |   122 +-
 bsd/netinet/in_proto.c                        |    58 +-
 bsd/netinet/in_rmx.c                          |    91 +-
 bsd/netinet/in_systm.h                        |    30 +-
 bsd/netinet/in_var.h                          |   117 +-
 bsd/netinet/ip.h                              |    47 +-
 bsd/netinet/ip6.h                             |    30 +-
 bsd/netinet/ip_compat.h                       |    30 +-
 bsd/netinet/ip_divert.c                       |    95 +-
 bsd/netinet/ip_divert.h                       |     2 +-
 bsd/netinet/ip_dummynet.c                     |    98 +-
 bsd/netinet/ip_dummynet.h                     |    34 +-
 bsd/netinet/ip_ecn.c                          |    33 +-
 bsd/netinet/ip_ecn.h                          |    30 +-
 bsd/netinet/ip_encap.c                        |    95 +-
 bsd/netinet/ip_encap.h                        |    32 +-
 bsd/netinet/ip_flow.c                         |    58 +-
 bsd/netinet/ip_flow.h                         |    30 +-
 bsd/netinet/ip_fw.h                           |    37 +-
 bsd/netinet/ip_fw2.c                          |   161 +-
 bsd/netinet/ip_fw2.h                          |    39 +
 bsd/netinet/ip_fw2_compat.c                   |    53 +-
 bsd/netinet/ip_fw2_compat.h                   |     4 +-
 bsd/netinet/ip_icmp.c                         |   132 +-
 bsd/netinet/ip_icmp.h                         |    30 +-
 bsd/netinet/ip_id.c                           |    28 +-
 bsd/netinet/ip_input.c                        |   500 +-
 bsd/netinet/ip_mroute.c                       |   419 +-
 bsd/netinet/ip_mroute.h                       |   120 +-
 bsd/netinet/ip_output.c                       |   641 +-
 bsd/netinet/ip_var.h                          |   130 +-
 bsd/netinet/kpi_ipfilter.c                    |    36 +-
 bsd/netinet/kpi_ipfilter.h                    |    33 +-
 bsd/netinet/kpi_ipfilter_var.h                |    30 +-
 bsd/netinet/raw_ip.c                          |   150 +-
 bsd/netinet/tcp.h                             |    46 +-
 bsd/netinet/tcp_debug.c                       |    30 +-
 bsd/netinet/tcp_debug.h                       |    30 +-
 bsd/netinet/tcp_fsm.h                         |    30 +-
 bsd/netinet/tcp_input.c                       |   694 +-
 bsd/netinet/tcp_output.c                      |   547 +-
 bsd/netinet/tcp_sack.c                        |    34 +-
 bsd/netinet/tcp_seq.h                         |    32 +-
 bsd/netinet/tcp_subr.c                        |   359 +-
 bsd/netinet/tcp_timer.c                       |   452 +-
 bsd/netinet/tcp_timer.h                       |    54 +-
 bsd/netinet/tcp_usrreq.c                      |   289 +-
 bsd/netinet/tcp_var.h                         |   301 +-
 bsd/netinet/tcpip.h                           |    30 +-
 bsd/netinet/udp.h                             |    35 +-
 bsd/netinet/udp_usrreq.c                      |   373 +-
 bsd/netinet/udp_var.h                         |    62 +-
 bsd/netinet6/ah.h                             |     2 +-
 bsd/netinet6/ah6.h                            |     2 +-
 bsd/netinet6/ah_core.c                        |    90 +-
 bsd/netinet6/ah_input.c                       |   137 +-
 bsd/netinet6/ah_output.c                      |   111 +-
 bsd/netinet6/esp.h                            |     2 +-
 bsd/netinet6/esp6.h                           |     2 +-
 bsd/netinet6/esp_core.c                       |   251 +-
 bsd/netinet6/esp_input.c                      |   241 +-
 bsd/netinet6/esp_output.c                     |   343 +-
 bsd/netinet6/esp_rijndael.c                   |    51 +-
 bsd/netinet6/frag6.c                          |    17 -
 bsd/netinet6/icmp6.c                          |   186 +-
 bsd/netinet6/in6.c                            |   141 +-
 bsd/netinet6/in6.h                            |    63 +-
 bsd/netinet6/in6_cksum.c                      |    86 +-
 bsd/netinet6/in6_gif.c                        |    59 +-
 bsd/netinet6/in6_ifattach.c                   |    96 +-
 bsd/netinet6/in6_pcb.c                        |   104 +-
 bsd/netinet6/in6_prefix.c                     |    12 +-
 bsd/netinet6/in6_proto.c                      |    82 +-
 bsd/netinet6/in6_rmx.c                        |    71 +-
 bsd/netinet6/in6_src.c                        |    19 +-
 bsd/netinet6/in6_var.h                        |    14 +-
 bsd/netinet6/ip6_forward.c                    |    97 +-
 bsd/netinet6/ip6_fw.c                         |   100 +-
 bsd/netinet6/ip6_fw.h                         |    70 +-
 bsd/netinet6/ip6_input.c                      |   173 +-
 bsd/netinet6/ip6_mroute.c                     |   114 +-
 bsd/netinet6/ip6_output.c                     |   105 +-
 bsd/netinet6/ip6_var.h                        |    13 +-
 bsd/netinet6/ip6protosw.h                     |     2 +-
 bsd/netinet6/ipcomp.h                         |     3 +-
 bsd/netinet6/ipcomp6.h                        |     3 +-
 bsd/netinet6/ipcomp_core.c                    |    42 +-
 bsd/netinet6/ipcomp_input.c                   |    66 +-
 bsd/netinet6/ipcomp_output.c                  |    45 +-
 bsd/netinet6/ipsec.c                          |   912 +-
 bsd/netinet6/ipsec.h                          |    16 +-
 bsd/netinet6/ipsec6.h                         |     2 +-
 bsd/netinet6/mld6.c                           |    20 +-
 bsd/netinet6/nd6.c                            |   144 +-
 bsd/netinet6/nd6.h                            |    23 +-
 bsd/netinet6/nd6_nbr.c                        |    22 +-
 bsd/netinet6/nd6_rtr.c                        |   107 +-
 bsd/netinet6/raw_ip6.c                        |    59 +-
 bsd/netinet6/route6.c                         |    36 +-
 bsd/netinet6/scope6.c                         |    11 +-
 bsd/netinet6/udp6_usrreq.c                    |   146 +-
 bsd/netkey/key.c                              |  1779 +-
 bsd/netkey/key.h                              |    12 +-
 bsd/netkey/key_debug.c                        |     3 +-
 bsd/netkey/key_debug.h                        |     5 +-
 bsd/netkey/keydb.c                            |    61 +-
 bsd/netkey/keydb.h                            |    13 +-
 bsd/netkey/keysock.c                          |    67 +-
 bsd/netkey/keysock.h                          |     4 +
 bsd/nfs/Makefile                              |     4 +-
 bsd/nfs/krpc.h                                |    30 +-
 bsd/nfs/krpc_subr.c                           |    50 +-
 bsd/nfs/nfs.h                                 |  1102 +-
 bsd/nfs/nfs4_subs.c                           |   797 +
 bsd/nfs/nfs4_vnops.c                          |  2151 ++
 bsd/nfs/nfs_bio.c                             |  3800 ++--
 bsd/nfs/nfs_boot.c                            |    62 +-
 bsd/nfs/nfs_gss.c                             |  3255 +++
 bsd/nfs/nfs_gss.h                             |   173 +
 bsd/nfs/nfs_lock.c                            |   741 +-
 bsd/nfs/nfs_lock.h                            |    52 +-
 bsd/nfs/nfs_node.c                            |   650 +-
 bsd/nfs/nfs_serv.c                            |  5300 ++---
 bsd/nfs/nfs_socket.c                          |  4225 ++--
 bsd/nfs/nfs_srvcache.c                        |   191 +-
 bsd/nfs/nfs_subs.c                            |  4177 ++--
 bsd/nfs/nfs_syscalls.c                        |  1899 +-
 bsd/nfs/nfs_vfsops.c                          |  2204 +-
 bsd/nfs/nfs_vnops.c                           |  6974 ++++---
 bsd/nfs/nfsdiskless.h                         |    30 +-
 bsd/nfs/nfsm_subs.h                           |  1098 +-
 bsd/nfs/nfsmount.h                            |   232 +-
 bsd/nfs/nfsnode.h                             |   380 +-
 bsd/nfs/nfsproto.h                            |   813 +-
 bsd/nfs/nfsrtt.h                              |   129 -
 bsd/nfs/nfsrvcache.h                          |    35 +-
 bsd/nfs/rpcv2.h                               |   104 +-
 bsd/nfs/xdr_subs.h                            |    56 +-
 bsd/ppc/Makefile                              |     7 +-
 bsd/ppc/_param.h                              |    42 +
 bsd/ppc/_structs.h                            |   218 +
 bsd/ppc/_types.h                              |    34 +-
 bsd/ppc/decodePPC.h                           |   919 +
 bsd/ppc/disklabel.h                           |    30 +-
 bsd/ppc/endian.h                              |    66 +-
 bsd/ppc/exec.h                                |    30 +-
 bsd/ppc/fasttrap_isa.h                        |   106 +
 bsd/ppc/param.h                               |    40 +-
 bsd/ppc/profile.h                             |    32 +-
 bsd/ppc/psl.h                                 |    30 +-
 bsd/ppc/ptrace.h                              |    30 +-
 bsd/ppc/reboot.h                              |    30 +-
 bsd/ppc/reg.h                                 |    30 +-
 bsd/ppc/setjmp.h                              |    65 +-
 bsd/ppc/signal.h                              |    91 +-
 bsd/ppc/types.h                               |    36 +-
 bsd/ppc/ucontext.h                            |    40 +-
 bsd/ppc/vmparam.h                             |    36 +-
 bsd/sys/Makefile                              |    68 +-
 bsd/sys/_endian.h                             |   116 +-
 bsd/sys/_select.h                             |    52 +
 bsd/sys/_structs.h                            |   254 +
 bsd/sys/_types.h                              |   104 +-
 bsd/sys/acct.h                                |    32 +-
 bsd/sys/aio.h                                 |    82 +-
 bsd/sys/aio_kern.h                            |    32 +-
 bsd/sys/appleapiopts.h                        |    30 +-
 bsd/sys/attr.h                                |    75 +-
 bsd/sys/bsdtask_info.h                        |    43 +-
 bsd/sys/buf.h                                 |    56 +-
 bsd/sys/buf_internal.h                        |    65 +-
 bsd/sys/callout.h                             |    30 +-
 bsd/sys/cdefs.h                               |   238 +-
 bsd/sys/clist.h                               |    30 +-
 bsd/sys/codesign.h                            |    60 +
 bsd/sys/conf.h                                |    30 +-
 bsd/sys/dir.h                                 |    30 +-
 bsd/sys/dirent.h                              |    63 +-
 bsd/sys/dis_tables.h                          |    48 +
 bsd/sys/disk.h                                |    94 +-
 bsd/sys/disklabel.h                           |    30 +-
 bsd/sys/disktab.h                             |    30 +-
 bsd/sys/dkstat.h                              |    30 +-
 bsd/sys/dmap.h                                |    30 +-
 bsd/sys/domain.h                              |    34 +-
 bsd/sys/dtrace.h                              |  2397 +++
 bsd/sys/dtrace_glue.h                         |   503 +
 bsd/sys/dtrace_impl.h                         |  1343 ++
 bsd/sys/dtrace_ptss.h                         |   103 +
 bsd/sys/errno.h                               |    78 +-
 bsd/sys/ev.h                                  |    71 +-
 bsd/sys/event.h                               |    56 +-
 bsd/sys/eventvar.h                            |    35 +-
 bsd/sys/exec.h                                |    30 +-
 bsd/sys/fasttrap.h                            |   115 +
 bsd/sys/fasttrap_impl.h                       |   222 +
 bsd/sys/fasttrap_isa.h                        |    48 +
 bsd/sys/fbt.h                                 |    64 +
 bsd/sys/fcntl.h                               |   156 +-
 bsd/sys/file.h                                |    40 +-
 bsd/sys/file_internal.h                       |   125 +-
 bsd/sys/filedesc.h                            |    55 +-
 bsd/sys/filio.h                               |    30 +-
 bsd/sys/fsctl.h                               |    30 +-
 bsd/sys/fsevents.h                            |    86 +-
 bsd/sys/fslog.h                               |   127 +
 bsd/sys/gmon.h                                |   180 +-
 bsd/sys/imageboot.h                           |    34 +
 bsd/sys/imgact.h                              |    57 +-
 bsd/sys/ioccom.h                              |    30 +-
 bsd/sys/ioctl.h                               |    30 +-
 bsd/sys/ioctl_compat.h                        |    30 +-
 bsd/sys/ipc.h                                 |    51 +-
 bsd/sys/ipcs.h                                |    32 +-
 bsd/sys/kauth.h                               |   139 +-
 bsd/sys/kdebug.h                              |   138 +-
 bsd/sys/kern_control.h                        |    63 +-
 bsd/sys/kern_event.h                          |   108 +-
 bsd/sys/kern_memorystatus.h                   |    62 +
 bsd/sys/kernel.h                              |    30 +-
 bsd/sys/kernel_types.h                        |     2 -
 bsd/sys/kpi_mbuf.h                            |   422 +-
 bsd/sys/kpi_socket.h                          |   136 +-
 bsd/sys/kpi_socketfilter.h                    |   147 +-
 bsd/sys/ktrace.h                              |   217 -
 bsd/sys/lctx.h                                |    28 +
 bsd/sys/linker_set.h                          |   204 +-
 bsd/sys/loadable_fs.h                         |    30 +-
 bsd/sys/lock.h                                |   177 +-
 bsd/sys/lockf.h                               |    41 +-
 bsd/sys/lockstat.h                            |   231 +
 bsd/sys/mach_swapon.h                         |    30 +-
 bsd/sys/malloc.h                              |   177 +-
 bsd/sys/mbuf.h                                |   218 +-
 bsd/sys/mcache.h                              |   283 +
 bsd/sys/md5.h                                 |    79 +-
 bsd/sys/mman.h                                |    57 +-
 bsd/sys/mount.h                               |   181 +-
 bsd/sys/mount_internal.h                      |   125 +-
 bsd/sys/msg.h                                 |    96 +-
 bsd/sys/msgbuf.h                              |    44 +-
 bsd/sys/mtio.h                                |    30 +-
 bsd/sys/namei.h                               |    76 +-
 bsd/sys/netport.h                             |    30 +-
 bsd/sys/param.h                               |    32 +-
 bsd/sys/paths.h                               |    30 +-
 bsd/sys/pipe.h                                |    49 +-
 bsd/sys/poll.h                                |    32 +-
 bsd/sys/posix_sem.h                           |    79 +
 bsd/sys/posix_shm.h                           |    82 +
 bsd/sys/proc.h                                |    81 +-
 bsd/sys/proc_info.h                           |   102 +-
 bsd/sys/proc_internal.h                       |   513 +-
 bsd/sys/protosw.h                             |    37 +-
 bsd/sys/pthread_internal.h                    |   230 +
 bsd/sys/ptrace.h                              |    38 +-
 bsd/sys/ptrace_internal.h                     |    97 -
 bsd/sys/queue.h                               |   536 +-
 bsd/sys/quota.h                               |    41 +-
 bsd/sys/random.h                              |    30 +-
 bsd/sys/reboot.h                              |    38 +-
 bsd/sys/resource.h                            |   155 +-
 bsd/sys/resourcevar.h                         |    57 +-
 bsd/sys/sbuf.h                                |    85 +
 bsd/sys/sdt.h                                 |    41 +
 bsd/sys/sdt_impl.h                            |   134 +
 bsd/sys/select.h                              |   133 +-
 bsd/sys/sem.h                                 |    61 +-
 bsd/sys/sem_internal.h                        |    51 +-
 bsd/sys/semaphore.h                           |    42 +-
 bsd/sys/shm.h                                 |    59 +-
 bsd/sys/shm_internal.h                        |    50 +-
 bsd/sys/signal.h                              |   141 +-
 bsd/sys/signalvar.h                           |    61 +-
 bsd/sys/socket.h                              |   191 +-
 bsd/sys/socketvar.h                           |   472 +-
 bsd/sys/sockio.h                              |    58 +-
 bsd/sys/spawn.h                               |    64 +
 bsd/sys/spawn_internal.h                      |   220 +
 bsd/sys/stat.h                                |   237 +-
 bsd/sys/subr_prf.h                            |    40 +-
 bsd/sys/sys_domain.h                          |    34 +-
 bsd/sys/syscall.h                             |   215 +-
 bsd/sys/sysctl.h                              |   251 +-
 bsd/sys/sysent.h                              |    60 +-
 bsd/sys/syslimits.h                           |    56 +-
 bsd/sys/syslog.h                              |    42 +-
 bsd/sys/sysproto.h                            |   726 +-
 bsd/sys/systm.h                               |    48 +-
 bsd/sys/termios.h                             |   106 +-
 bsd/sys/time.h                                |   183 +-
 bsd/sys/timeb.h                               |    30 +-
 bsd/sys/times.h                               |    30 +-
 bsd/sys/tprintf.h                             |    30 +-
 bsd/sys/trace.h                               |    30 +-
 bsd/sys/tty.h                                 |    39 +-
 bsd/sys/ttychars.h                            |    30 +-
 bsd/sys/ttycom.h                              |    39 +-
 bsd/sys/ttydefaults.h                         |    30 +-
 bsd/sys/ttydev.h                              |    30 +-
 bsd/sys/types.h                               |   138 +-
 bsd/sys/ubc.h                                 |    67 +-
 bsd/sys/ubc_internal.h                        |   114 +-
 bsd/sys/ucontext.h                            |    65 +-
 bsd/sys/ucred.h                               |    67 +-
 bsd/sys/uio.h                                 |    40 +-
 bsd/sys/uio_internal.h                        |    30 +-
 bsd/sys/un.h                                  |    41 +-
 bsd/sys/unistd.h                              |    54 +-
 bsd/sys/unpcb.h                               |   135 +-
 bsd/sys/user.h                                |   145 +-
 bsd/sys/utfconv.h                             |   187 +-
 bsd/sys/utsname.h                             |    30 +-
 bsd/sys/ux_exception.h                        |    35 +-
 bsd/sys/vadvise.h                             |    30 +-
 bsd/sys/vcmd.h                                |    30 +-
 bsd/sys/version.h                             |    78 -
 bsd/sys/vfs_context.h                         |    10 +-
 bsd/sys/vlimit.h                              |    30 +-
 bsd/sys/vm.h                                  |    33 +-
 bsd/sys/vmmeter.h                             |   100 +-
 bsd/sys/vmparam.h                             |    30 +-
 bsd/sys/vnioctl.h                             |    32 +-
 bsd/sys/vnode.h                               |   135 +-
 bsd/sys/vnode_if.h                            |   136 +-
 bsd/sys/vnode_internal.h                      |   163 +-
 bsd/sys/vstat.h                               |    37 +-
 bsd/sys/wait.h                                |    82 +-
 bsd/sys/xattr.h                               |    33 +-
 bsd/ufs/ffs/ffs_alloc.c                       |    30 +-
 bsd/ufs/ffs/ffs_balloc.c                      |    30 +-
 bsd/ufs/ffs/ffs_extern.h                      |    40 +-
 bsd/ufs/ffs/ffs_inode.c                       |    33 +-
 bsd/ufs/ffs/ffs_subr.c                        |    30 +-
 bsd/ufs/ffs/ffs_tables.c                      |    30 +-
 bsd/ufs/ffs/ffs_vfsops.c                      |    50 +-
 bsd/ufs/ffs/ffs_vnops.c                       |    32 +-
 bsd/ufs/ffs/fs.h                              |    30 +-
 bsd/ufs/ufs/dinode.h                          |    30 +-
 bsd/ufs/ufs/dir.h                             |    44 +-
 bsd/ufs/ufs/inode.h                           |    30 +-
 bsd/ufs/ufs/quota.h                           |    44 +-
 bsd/ufs/ufs/ufs_attrlist.c                    |    30 +-
 bsd/ufs/ufs/ufs_bmap.c                        |   121 +-
 bsd/ufs/ufs/ufs_byte_order.c                  |    30 +-
 bsd/ufs/ufs/ufs_byte_order.h                  |    30 +-
 bsd/ufs/ufs/ufs_extern.h                      |    44 +-
 bsd/ufs/ufs/ufs_ihash.c                       |    35 +-
 bsd/ufs/ufs/ufs_inode.c                       |    30 +-
 bsd/ufs/ufs/ufs_lookup.c                      |    45 +-
 bsd/ufs/ufs/ufs_quota.c                       |    42 +-
 bsd/ufs/ufs/ufs_readwrite.c                   |    73 +-
 bsd/ufs/ufs/ufs_vfsops.c                      |    39 +-
 bsd/ufs/ufs/ufs_vnops.c                       |    62 +-
 bsd/ufs/ufs/ufsmount.h                        |    30 +-
 bsd/uxkern/ux_exception.c                     |   208 +-
 bsd/vfs/kpi_vfs.c                             |  1234 +-
 bsd/vfs/vfs_attrlist.c                        |   458 +-
 bsd/vfs/vfs_bio.c                             |   802 +-
 bsd/vfs/vfs_cache.c                           |   821 +-
 bsd/vfs/vfs_cluster.c                         |  2975 +--
 bsd/vfs/vfs_conf.c                            |   130 +-
 bsd/vfs/vfs_fsevents.c                        |  2286 ++-
 bsd/vfs/vfs_fslog.c                           |   460 +
 bsd/vfs/vfs_init.c                            |   114 +-
 bsd/vfs/vfs_journal.c                         |  1397 +-
 bsd/vfs/vfs_journal.h                         |    84 +-
 bsd/vfs/vfs_lookup.c                          |   530 +-
 bsd/vfs/vfs_quota.c                           |   136 +-
 bsd/vfs/vfs_subr.c                            |  1936 +-
 bsd/vfs/vfs_support.c                         |   172 +-
 bsd/vfs/vfs_support.h                         |    36 +-
 bsd/vfs/vfs_syscalls.c                        |  3693 ++--
 bsd/vfs/vfs_utfconv.c                         |   476 +-
 bsd/vfs/vfs_utfconvdata.h                     |    32 +-
 bsd/vfs/vfs_vnops.c                           |   538 +-
 bsd/vfs/vfs_xattr.c                           |  1088 +-
 bsd/vfs/vnode_if.c                            |   253 +-
 bsd/vfs/vnode_if.sh                           |     6 +-
 bsd/vm/dp_backing_file.c                      |    78 +-
 bsd/vm/vm_pager.h                             |    30 +-
 bsd/vm/vm_unix.c                              |  2395 +--
 bsd/vm/vnode_pager.c                          |   174 +-
 bsd/vm/vnode_pager.h                          |    36 +-
 config/BSDKernel.exports                      |    66 +-
 config/BSDKernel.i386.exports                 |    14 +
 config/BSDKernel.ppc.exports                  |    14 +
 config/DtraceIgnored.symbols                  |     1 +
 config/IOKit.exports                          |   304 +-
 config/IOKit.i386.exports                     |     6 +
 config/IOKit.ppc.exports                      |     8 +
 config/Libkern.exports                        |    37 +
 config/Libkern.i386.exports                   |     2 +
 config/MACFramework.exports                   |    19 +
 config/MACFramework.i386.exports              |     9 +
 config/MACFramework.ppc.exports               |     9 +
 config/Mach.i386.exports                      |     2 +
 config/Makefile                               |    67 +-
 config/MasterVersion                          |     2 +-
 .../PlugIns/MACFramework.kext/Info.plist      |    32 +
 config/System6.0.exports                      |   237 +-
 config/System6.0.i386.exports                 |     7 +
 config/System6.0.ppc.exports                  |     6 +
 config/Unsupported.exports                    |   104 +-
 config/Unsupported.i386.exports               |    17 +-
 config/Unsupported.ppc.exports                |     3 +
 config/compress-man-pages.pl                  |    95 +
 config/newvers.pl                             |     3 +
 config/version.c                              |    34 +-
 .../platform/drvAppleMacIO/AppleMacIO.cpp     |    30 +-
 .../Drivers/platform/drvAppleNMI/AppleNMI.cpp |    30 +-
 .../platform/drvAppleNVRAM/AppleNVRAM.cpp     |    30 +-
 .../platform/drvAppleNVRAM/AppleNVRAM.h       |    30 +-
 .../drvApplePlatformExpert/AppleCPU.cpp       |    30 +-
 .../drvApplePlatformExpert/AppleCPU.h         |    30 +-
 .../ApplePlatformExpert.cpp                   |    30 +-
 .../GenericInterruptController.cpp            |    30 +-
 .../GenericInterruptController.h              |    30 +-
 iokit/Families/IONVRAM/IONVRAMController.cpp  |    30 +-
 .../IOSystemManagement/IOWatchDogTimer.cpp    |    36 +-
 iokit/IOKit/IOBSD.h                           |    30 +-
 iokit/IOKit/IOBufferMemoryDescriptor.h        |    45 +-
 iokit/IOKit/IOCPU.h                           |    33 +-
 iokit/IOKit/IOCatalogue.h                     |    30 +-
 iokit/IOKit/IOCommand.h                       |    30 +-
 iokit/IOKit/IOCommandGate.h                   |    34 +-
 iokit/IOKit/IOCommandPool.h                   |    30 +-
 iokit/IOKit/IOCommandQueue.h                  |    50 +-
 iokit/IOKit/IOConditionLock.h                 |    30 +-
 iokit/IOKit/IODMACommand.h                    |   102 +-
 iokit/IOKit/IODMAController.h                 |    64 +
 iokit/IOKit/IODMAEventSource.h                |    81 +
 iokit/IOKit/IODataQueue.h                     |    30 +-
 iokit/IOKit/IODataQueueShared.h               |    55 +-
 iokit/IOKit/IODeviceMemory.h                  |    30 +-
 iokit/IOKit/IODeviceTreeSupport.h             |    32 +-
 iokit/IOKit/IOEventSource.h                   |    30 +-
 iokit/IOKit/IOFilterInterruptEventSource.h    |    30 +-
 iokit/IOKit/IOHibernatePrivate.h              |    71 +-
 iokit/IOKit/IOInterleavedMemoryDescriptor.h   |   166 +
 iokit/IOKit/IOInterruptController.h           |    30 +-
 iokit/IOKit/IOInterruptEventSource.h          |    30 +-
 iokit/IOKit/IOInterrupts.h                    |    30 +-
 iokit/IOKit/IOKitDebug.h                      |    37 +-
 iokit/IOKit/IOKitKeys.h                       |    36 +-
 iokit/IOKit/IOKitKeysPrivate.h                |    35 +-
 iokit/IOKit/IOKitServer.h                     |    33 +-
 iokit/IOKit/IOLib.h                           |    39 +-
 iokit/IOKit/IOLocks.h                         |    34 +-
 iokit/IOKit/IOLocksPrivate.h                  |    60 +
 iokit/IOKit/IOMapper.h                        |    37 +-
 iokit/IOKit/IOMemoryCursor.h                  |    30 +-
 iokit/IOKit/IOMemoryDescriptor.h              |   149 +-
 iokit/IOKit/IOMessage.h                       |    32 +-
 iokit/IOKit/IOMultiMemoryDescriptor.h         |    32 +-
 iokit/IOKit/IONVRAM.h                         |    30 +-
 iokit/IOKit/IONotifier.h                      |    30 +-
 iokit/IOKit/IOPMEventSource.h                 |    36 +-
 iokit/IOKit/IOPlatformExpert.h                |    46 +-
 iokit/IOKit/IOPolledInterface.h               |    34 +-
 iokit/IOKit/IORangeAllocator.h                |    30 +-
 iokit/IOKit/IORegistryEntry.h                 |    32 +-
 iokit/IOKit/IOReturn.h                        |    30 +-
 iokit/IOKit/IOService.h                       |  1577 +-
 iokit/IOKit/IOServicePM.h                     |   408 +-
 iokit/IOKit/IOSharedDataQueue.h               |   129 +
 iokit/IOKit/IOSharedLock.h                    |    30 +-
 iokit/IOKit/IOSyncer.h                        |    47 +-
 iokit/IOKit/IOTimeStamp.h                     |    30 +-
 iokit/IOKit/IOTimerEventSource.h              |    30 +-
 iokit/IOKit/IOTypes.h                         |    55 +-
 iokit/IOKit/IOUserClient.h                    |   117 +-
 iokit/IOKit/IOWorkLoop.h                      |    58 +-
 iokit/IOKit/Makefile                          |    11 +-
 iokit/IOKit/OSMessageNotification.h           |    66 +-
 iokit/IOKit/assert.h                          |    30 +-
 iokit/IOKit/i386/IOSharedLockImp.h            |    30 +-
 iokit/IOKit/machine/IOSharedLockImp.h         |    32 +-
 iokit/IOKit/nvram/IONVRAMController.h         |    30 +-
 iokit/IOKit/nvram/Makefile                    |     2 +
 iokit/IOKit/pci/IOPCIDevice.h                 |    33 +-
 iokit/IOKit/platform/AppleMacIO.h             |    30 +-
 iokit/IOKit/platform/AppleMacIODevice.h       |    30 +-
 iokit/IOKit/platform/AppleNMI.h               |    30 +-
 iokit/IOKit/platform/ApplePlatformExpert.h    |    30 +-
 iokit/IOKit/platform/Makefile                 |    12 +-
 iokit/IOKit/power/IOPwrController.h           |    30 +-
 iokit/IOKit/power/Makefile                    |     2 +
 iokit/IOKit/ppc/IODBDMA.h                     |    30 +-
 iokit/IOKit/ppc/IOSharedLockImp.h             |    30 +-
 iokit/IOKit/pwr_mgt/IOPM.h                    |   150 +-
 iokit/IOKit/pwr_mgt/IOPMDeprecated.h          |    30 +-
 iokit/IOKit/pwr_mgt/IOPMLibDefs.h             |    47 +-
 iokit/IOKit/pwr_mgt/IOPMPagingPlexus.h        |    42 +-
 iokit/IOKit/pwr_mgt/IOPMPowerSource.h         |    70 +-
 iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h     |    30 +-
 iokit/IOKit/pwr_mgt/IOPMPrivate.h             |    30 +-
 iokit/IOKit/pwr_mgt/IOPMchangeNoteList.h      |   135 -
 iokit/IOKit/pwr_mgt/IOPMinformee.h            |    58 +-
 iokit/IOKit/pwr_mgt/IOPMinformeeList.h        |    41 +-
 iokit/IOKit/pwr_mgt/IOPMlog.h                 |    41 +-
 iokit/IOKit/pwr_mgt/IOPMpmChild.h             |    35 -
 iokit/IOKit/pwr_mgt/IOPMpowerState.h          |    76 +-
 iokit/IOKit/pwr_mgt/IOPowerConnection.h       |    44 +-
 iokit/IOKit/pwr_mgt/Makefile                  |     4 +-
 iokit/IOKit/pwr_mgt/RootDomain.h              |    95 +-
 iokit/IOKit/rtc/IORTCController.h             |    30 +-
 iokit/IOKit/rtc/Makefile                      |     2 +
 iokit/IOKit/system.h                          |    36 +-
 .../IOKit/system_management/IOWatchDogTimer.h |    36 +-
 iokit/IOKit/system_management/Makefile        |     2 +
 iokit/Kernel/IOBufferMemoryDescriptor.cpp     |   290 +-
 iokit/Kernel/IOCPU.cpp                        |   311 +-
 iokit/Kernel/IOCatalogue.cpp                  |   493 +-
 iokit/Kernel/IOCommand.cpp                    |    30 +-
 iokit/Kernel/IOCommandGate.cpp                |    30 +-
 iokit/Kernel/IOCommandPool.cpp                |    30 +-
 iokit/Kernel/IOCommandQueue.cpp               |    30 +-
 iokit/Kernel/IOConditionLock.cpp              |    30 +-
 iokit/Kernel/IOCopyMapper.cpp                 |    36 +-
 iokit/Kernel/IOCopyMapper.h                   |    30 +-
 iokit/Kernel/IODMACommand.cpp                 |   276 +-
 iokit/Kernel/IODMAController.cpp              |   100 +
 iokit/Kernel/IODMAEventSource.cpp             |   166 +
 iokit/Kernel/IODataQueue.cpp                  |    38 +-
 iokit/Kernel/IODeviceMemory.cpp               |    30 +-
 iokit/Kernel/IODeviceTreeSupport.cpp          |    83 +-
 iokit/Kernel/IOEventSource.cpp                |    30 +-
 iokit/Kernel/IOFilterInterruptEventSource.cpp |    30 +-
 iokit/Kernel/IOHibernateIO.cpp                |   498 +-
 iokit/Kernel/IOHibernateInternal.h            |    31 +-
 iokit/Kernel/IOHibernateRestoreKernel.c       |    52 +-
 .../Kernel/IOInterleavedMemoryDescriptor.cpp  |   406 +
 iokit/Kernel/IOInterruptController.cpp        |    30 +-
 iokit/Kernel/IOInterruptEventSource.cpp       |    32 +-
 iokit/Kernel/IOKitDebug.cpp                   |    38 +-
 iokit/Kernel/IOKitKernelInternal.h            |    77 +-
 iokit/Kernel/IOLib.cpp                        |    88 +-
 iokit/Kernel/IOLocks.cpp                      |    73 +-
 iokit/Kernel/IOMapper.cpp                     |    33 +-
 iokit/Kernel/IOMemoryCursor.cpp               |    30 +-
 iokit/Kernel/IOMemoryDescriptor.cpp           |  1228 +-
 iokit/Kernel/IOMultiMemoryDescriptor.cpp      |    30 +-
 iokit/Kernel/IONVRAM.cpp                      |    70 +-
 iokit/Kernel/IOPMPowerSource.cpp              |   149 +-
 iokit/Kernel/IOPMPowerSourceList.cpp          |    30 +-
 iokit/Kernel/IOPMPowerStateQueue.cpp          |   107 +-
 iokit/Kernel/IOPMPowerStateQueue.h            |    48 +-
 iokit/Kernel/IOPMWorkArbiter.cpp              |   153 -
 iokit/Kernel/IOPMWorkArbiter.h                |    77 -
 iokit/Kernel/IOPMchangeNoteList.cpp           |   234 -
 iokit/Kernel/IOPMinformee.cpp                 |    54 +-
 iokit/Kernel/IOPMinformeeList.cpp             |    57 +-
 iokit/Kernel/IOPMpmChild.cpp                  |    36 -
 iokit/Kernel/IOPMrootDomain.cpp               |  1373 +-
 iokit/Kernel/IOPMrootDomainInternal.h         |    73 +
 iokit/Kernel/IOPlatformExpert.cpp             |   146 +-
 iokit/Kernel/IOPowerConnection.cpp            |    50 +-
 iokit/Kernel/IORangeAllocator.cpp             |    30 +-
 iokit/Kernel/IORegistryEntry.cpp              |   106 +-
 iokit/Kernel/IOService.cpp                    |   127 +-
 iokit/Kernel/IOServicePM.cpp                  |  7154 ++++---
 iokit/Kernel/IOServicePMPrivate.h             |   568 +
 iokit/Kernel/IOServicePrivate.h               |    36 +-
 iokit/Kernel/IOSharedDataQueue.cpp            |   217 +
 iokit/Kernel/IOStartIOKit.cpp                 |    75 +-
 iokit/Kernel/IOStringFuncs.c                  |    49 +-
 iokit/Kernel/IOSyncer.cpp                     |    30 +-
 iokit/Kernel/IOTimerEventSource.cpp           |    32 +-
 iokit/Kernel/IOUserClient.cpp                 |  1856 +-
 iokit/Kernel/IOWorkLoop.cpp                   |   142 +-
 iokit/Kernel/PMmisc.cpp                       |    34 +-
 iokit/Kernel/RootDomainUserClient.cpp         |   145 +-
 iokit/Kernel/RootDomainUserClient.h           |    40 +-
 iokit/Kernel/i386/IOAsmSupport.s              |    30 +-
 iokit/Kernel/i386/IOSharedLock.s              |    30 +-
 iokit/Kernel/ppc/IOAsmSupport.s               |    30 +-
 iokit/Kernel/ppc/IODBDMA.cpp                  |    30 +-
 iokit/Kernel/ppc/IOSharedLock.s               |    30 +-
 iokit/KernelConfigTables.cpp                  |    38 +-
 iokit/Makefile                                |     4 +-
 iokit/Tests/TestCollections.cpp               |    30 +-
 iokit/Tests/TestContainers.cpp                |    30 +-
 iokit/Tests/TestDevice.cpp                    |    30 +-
 iokit/Tests/Tests.cpp                         |    30 +-
 iokit/Tests/Tests.h                           |    30 +-
 iokit/User/Makefile                           |     6 +-
 iokit/bsddev/DINetBootHook.cpp                |    41 +-
 iokit/bsddev/DINetBootHook.h                  |     6 +-
 iokit/bsddev/IOKitBSDInit.cpp                 |   103 +-
 iokit/bsddev/IOKitBSDInit.h                   |    30 +-
 iokit/conf/MASTER                             |    36 +
 iokit/conf/MASTER.i386                        |    15 +-
 iokit/conf/MASTER.ppc                         |     5 +-
 iokit/conf/Makefile                           |    24 +-
 iokit/conf/Makefile.i386                      |     3 +-
 iokit/conf/Makefile.template                  |    24 +-
 iokit/conf/files                              |    20 +-
 iokit/conf/tools/doconf/Makefile              |    18 +-
 iokit/conf/tools/doconf/doconf.csh            |    16 +-
 .../architecture/i386/kernBootStruct.h        |    28 -
 iokit/include/assert.h                        |    28 -
 iokit/include/bsddev/EventShmemLock.h         |    28 -
 iokit/include/bsddev/Makefile                 |    41 -
 iokit/include/bsddev/ev_keymap.h              |    28 -
 iokit/include/bsddev/ev_types.h               |    29 -
 iokit/include/bsddev/event.h                  |    29 -
 iokit/include/bsddev/evio.h                   |    29 -
 iokit/include/bsddev/evsio.h                  |    36 -
 iokit/include/bsddev/i386/EventShmemLock.h    |    29 -
 iokit/include/bsddev/i386/event.h             |    28 -
 iokit/include/bsddev/i386/evio.h              |    29 -
 iokit/include/bsddev/i386/evsio.h             |    29 -
 iokit/include/bsddev/machine/EventShmemLock.h |    35 -
 iokit/include/bsddev/machine/Makefile         |    39 -
 iokit/include/bsddev/machine/event.h          |    35 -
 iokit/include/bsddev/machine/evio.h           |    35 -
 iokit/include/bsddev/machine/evsio.h          |    35 -
 iokit/include/bsddev/ppc/EventShmemLock.h     |    28 -
 iokit/include/bsddev/ppc/event.h              |    29 -
 iokit/include/bsddev/ppc/evio.h               |    29 -
 iokit/include/bsddev/ppc/evsio.h              |    29 -
 iokit/include/drivers/event_status_driver.h   |    40 -
 iokit/include/mach/mach.h                     |    27 -
 iokit/mach-o/mach_header.h                    |    30 +-
 kgmacros                                      |  3244 ++-
 libkern/Makefile                              |    10 +-
 libkern/c++/OSArray.cpp                       |    30 +-
 libkern/c++/OSBoolean.cpp                     |    30 +-
 libkern/c++/OSCPPDebug.cpp                    |    30 +-
 libkern/c++/OSCollection.cpp                  |    30 +-
 libkern/c++/OSCollectionIterator.cpp          |    30 +-
 libkern/c++/OSData.cpp                        |    30 +-
 libkern/c++/OSDictionary.cpp                  |    30 +-
 libkern/c++/OSIterator.cpp                    |    30 +-
 libkern/c++/OSMetaClass.cpp                   |    48 +-
 libkern/c++/OSNumber.cpp                      |    38 +-
 libkern/c++/OSObject.cpp                      |   114 +-
 libkern/c++/OSObjectAsm.s                     |    36 +-
 libkern/c++/OSOrderedSet.cpp                  |    30 +-
 libkern/c++/OSRuntime.cpp                     |    36 +-
 libkern/c++/OSSerialize.cpp                   |    34 +-
 libkern/c++/OSSet.cpp                         |    30 +-
 libkern/c++/OSString.cpp                      |    30 +-
 libkern/c++/OSSymbol.cpp                      |    91 +-
 libkern/c++/OSUnserialize.cpp                 |    48 +-
 libkern/c++/OSUnserialize.y                   |    30 +-
 libkern/c++/OSUnserializeXML.cpp              |  2333 ++-
 libkern/c++/OSUnserializeXML.y                |    34 +-
 .../TestSerialization/test1/test1_main.cpp    |    30 +-
 .../TestSerialization/test2/test2_main.cpp    |    30 +-
 libkern/conf/MASTER                           |    16 +-
 libkern/conf/MASTER.i386                      |    10 +-
 libkern/conf/MASTER.ppc                       |     5 +-
 libkern/conf/Makefile                         |    24 +-
 libkern/conf/Makefile.i386                    |     8 +-
 libkern/conf/Makefile.template                |    12 +-
 libkern/conf/files                            |    12 +
 libkern/conf/tools/doconf/Makefile            |    18 +-
 libkern/conf/tools/doconf/doconf.csh          |    16 +-
 libkern/crypto/md5.c                          |   364 +
 libkern/crypto/sha1.c                         |   373 +
 libkern/gen/OSAtomicOperations.c              |   171 +-
 libkern/gen/OSDebug.cpp                       |    88 +-
 libkern/i386/OSAtomic.c                       |    47 -
 libkern/i386/OSAtomic.s                       |    99 +-
 libkern/kmod/Makefile                         |    14 +-
 libkern/kmod/Makefile.kmod                    |    19 +-
 libkern/kmod/c_start.c                        |    30 +-
 libkern/kmod/c_stop.c                         |    30 +-
 libkern/kmod/cplus_start.c                    |    30 +-
 libkern/kmod/cplus_stop.c                     |    30 +-
 libkern/libkern/Makefile                      |    17 +-
 libkern/libkern/OSAtomic.h                    |   120 +-
 libkern/libkern/OSBase.h                      |    30 +-
 libkern/libkern/OSByteOrder.h                 |    76 +-
 libkern/libkern/OSCrossEndian.h               |    30 +-
 libkern/libkern/OSDebug.h                     |    40 +-
 libkern/libkern/OSMalloc.h                    |    36 +-
 libkern/libkern/OSReturn.h                    |    30 +-
 libkern/libkern/OSTypes.h                     |    42 +-
 libkern/libkern/_OSByteOrder.h                |   133 +
 libkern/libkern/c++/Makefile                  |     4 +
 libkern/libkern/c++/OSArray.h                 |    30 +-
 libkern/libkern/c++/OSBoolean.h               |    30 +-
 libkern/libkern/c++/OSCPPDebug.h              |    30 +-
 libkern/libkern/c++/OSCollection.h            |    30 +-
 libkern/libkern/c++/OSCollectionIterator.h    |    30 +-
 libkern/libkern/c++/OSContainers.h            |    30 +-
 libkern/libkern/c++/OSData.h                  |    30 +-
 libkern/libkern/c++/OSDictionary.h            |    30 +-
 libkern/libkern/c++/OSEndianTypes.h           |    30 +-
 libkern/libkern/c++/OSIterator.h              |    30 +-
 libkern/libkern/c++/OSLib.h                   |    43 +-
 libkern/libkern/c++/OSMetaClass.h             |    49 +-
 libkern/libkern/c++/OSNumber.h                |    30 +-
 libkern/libkern/c++/OSObject.h                |    30 +-
 libkern/libkern/c++/OSOrderedSet.h            |    32 +-
 libkern/libkern/c++/OSSerialize.h             |    30 +-
 libkern/libkern/c++/OSSet.h                   |    30 +-
 libkern/libkern/c++/OSString.h                |    30 +-
 libkern/libkern/c++/OSSymbol.h                |    30 +-
 libkern/libkern/c++/OSUnserialize.h           |    30 +-
 .../libkern/crypto}/Makefile                  |    18 +-
 libkern/libkern/crypto/md5.h                  |    79 +
 libkern/libkern/crypto/sha1.h                 |    69 +
 libkern/libkern/i386/Makefile                 |     3 +-
 libkern/libkern/i386/OSByteOrder.h            |    89 +-
 libkern/libkern/i386/_OSByteOrder.h           |    90 +
 libkern/libkern/locks.h                       |    36 +-
 libkern/libkern/machine/Makefile              |     4 +
 libkern/libkern/machine/OSByteOrder.h         |    30 +-
 libkern/libkern/ppc/OSByteOrder.h             |    34 +-
 libkern/libkern/sysctl.h                      |    36 +-
 libkern/libkern/version.h.template            |     7 +-
 libkern/libkern/zlib.h                        |   723 +
 libkern/mach-o/loader.h                       |    42 +-
 libkern/mach-o/mach_header.h                  |    30 +-
 libkern/ppc/OSAtomic.s                        |    30 +-
 libkern/ppc/bcmp.s                            |    30 +-
 libkern/ppc/memcmp.s                          |    30 +-
 libkern/ppc/strlen.s                          |    30 +-
 libkern/stdio/scanf.c                         |    15 +-
 libkern/uuid/Makefile                         |     4 +
 libkern/uuid/uuid.c                           |     2 +
 {bsd/net => libkern}/zlib.c                   |   644 +-
 libsa/Makefile                                |     2 +
 libsa/bootstrap.cpp                           |    32 +-
 libsa/bsearch.c                               |    36 +-
 libsa/c++rem3.c                               |    38 +-
 libsa/c++rem3.h                               |    28 +-
 libsa/catalogue.cpp                           |    76 +-
 libsa/conf/MASTER                             |    16 +
 libsa/conf/MASTER.i386                        |    10 +-
 libsa/conf/MASTER.ppc                         |     3 +-
 libsa/conf/Makefile                           |    23 +-
 libsa/conf/Makefile.template                  |    15 +-
 libsa/conf/files                              |     1 +
 libsa/conf/tools/doconf/Makefile              |    18 +-
 libsa/conf/tools/doconf/doconf.csh            |    16 +-
 libsa/dgraph.c                                |    88 +-
 libsa/dgraph.h                                |    53 +
 libsa/i386/setjmp.s                           |    30 +-
 libsa/kext.cpp                                |   186 +-
 libsa/kld_patch.c                             |   205 +-
 libsa/kld_patch.h                             |    30 +-
 libsa/kmod.cpp                                |  1231 --
 libsa/libsa/i386/setjmp.h                     |    30 +-
 libsa/libsa/kext.h                            |     3 +
 libsa/libsa/malloc.h                          |     1 -
 libsa/libsa/ppc/setjmp.h                      |    30 +-
 libsa/libsa/setjmp.h                          |    36 +-
 libsa/libsa/stdlib.h                          |    17 +-
 libsa/load.c                                  |    79 +-
 libsa/mach.c                                  |    30 +-
 libsa/mach_loader.h                           |    33 +-
 libsa/malloc.c                                |    85 +-
 libsa/misc.c                                  |    33 +-
 libsa/mkext.c                                 |    30 +-
 libsa/ppc/setjmp.s                            |    30 +-
 libsa/sort.c                                  |    32 +-
 libsa/strrchr.c                               |    30 +-
 libsa/strstr.c                                |    28 +-
 libsa/vers_rsrc.c                             |    52 +-
 libsyscall/BSDmakefile                        |   137 +
 libsyscall/GNUmakefile                        |     8 +
 libsyscall/Makefile                           |    55 +
 libsyscall/Makefile.inc                       |    52 +
 libsyscall/Makefile.xbs                       |   128 +
 libsyscall/create-syscalls.pl                 |   265 +
 libsyscall/custom/SYS.h                       |   395 +
 libsyscall/custom/__fork.s                    |   312 +
 libsyscall/custom/__getpid.s                  |   185 +
 libsyscall/custom/__gettimeofday.s            |    84 +
 libsyscall/custom/__lseek.s                   |    45 +
 libsyscall/custom/__pipe.s                    |    71 +
 libsyscall/custom/__ptrace.s                  |    74 +
 libsyscall/custom/__sigaltstack.s             |    45 +
 libsyscall/custom/__sigreturn.s               |    45 +
 libsyscall/custom/__syscall.s                 |    63 +
 libsyscall/custom/__vfork.s                   |   246 +
 libsyscall/custom/custom.s                    |   143 +
 libsyscall/fixdups.ed                         |     2 +
 libsyscall/include/Makefile.inc               |     1 +
 libsyscall/include/processor_facilities.h     |    36 +
 libsyscall/mach/Makefile.inc                  |    73 +
 libsyscall/mach/bootstrap_ports.c             |    72 +
 libsyscall/mach/brk.2                         |   150 +
 libsyscall/mach/clock.defs                    |    28 +
 libsyscall/mach/clock_priv.defs               |    28 +
 libsyscall/mach/clock_reply.defs              |    28 +
 libsyscall/mach/clock_sleep.c                 |    38 +
 libsyscall/mach/err_iokit.sub                 |   249 +
 libsyscall/mach/err_ipc.sub                   |    98 +
 libsyscall/mach/err_kern.sub                  |   182 +
 libsyscall/mach/err_mach_ipc.sub              |   119 +
 libsyscall/mach/err_server.sub                |   357 +
 libsyscall/mach/err_us.sub                    |    45 +
 libsyscall/mach/error_codes.c                 |   151 +
 libsyscall/mach/errorlib.h                    |   115 +
 libsyscall/mach/exc.defs                      |    29 +
 libsyscall/mach/exc_catcher.c                 |    67 +
 libsyscall/mach/exc_catcher_state.c           |    70 +
 libsyscall/mach/exc_catcher_state_identity.c  |    72 +
 libsyscall/mach/externs.h                     |    42 +
 libsyscall/mach/fprintf_stderr.c              |    62 +
 libsyscall/mach/headers/Makefile.inc          |    10 +
 libsyscall/mach/headers/errorlib.h            |   101 +
 libsyscall/mach/headers/mach.h                |   133 +
 libsyscall/mach/headers/mach_error.h          |    93 +
 libsyscall/mach/headers/mach_init.h           |   124 +
 libsyscall/mach/headers/mach_interface.h      |    54 +
 libsyscall/mach/headers/port_obj.h            |   103 +
 libsyscall/mach/headers/sync.h                |    35 +
 libsyscall/mach/headers/task.h                |    38 +
 libsyscall/mach/headers/thread_act.h          |    38 +
 libsyscall/mach/headers/vm_task.h             |    34 +
 libsyscall/mach/host_priv.defs                |    28 +
 libsyscall/mach/host_security.defs            |    28 +
 libsyscall/mach/i386/Makefile.inc             |     3 +
 libsyscall/mach/i386/mach_absolute_time.S     |    32 +
 libsyscall/mach/ledger.defs                   |    28 +
 libsyscall/mach/lock_set.defs                 |    28 +
 libsyscall/mach/mach_error.c                  |    87 +
 libsyscall/mach/mach_error_string.c           |   179 +
 libsyscall/mach/mach_host.defs                |    30 +
 libsyscall/mach/mach_init.c                   |   268 +
 libsyscall/mach/mach_init_libSystem.c         |    56 +
 libsyscall/mach/mach_init_ports.c             |   140 +
 libsyscall/mach/mach_msg.c                    |   619 +
 libsyscall/mach/mach_port.defs                |    28 +
 libsyscall/mach/mach_traps.s                  |    54 +
 libsyscall/mach/mach_vm.defs                  |    28 +
 libsyscall/mach/mig_allocate.c                |    68 +
 libsyscall/mach/mig_deallocate.c              |    65 +
 libsyscall/mach/mig_reply_setup.c             |    81 +
 libsyscall/mach/mig_strncpy.c                 |    91 +
 libsyscall/mach/ms_thread_switch.c            |    76 +
 libsyscall/mach/notify.defs                   |    28 +
 libsyscall/mach/panic.c                       |    88 +
 libsyscall/mach/port_obj.c                    |    56 +
 libsyscall/mach/ppc/Makefile.inc              |     3 +
 libsyscall/mach/ppc/mach_absolute_time.s      |    37 +
 libsyscall/mach/ppc64/Makefile.inc            |     4 +
 libsyscall/mach/processor.defs                |    28 +
 libsyscall/mach/processor_set.defs            |    28 +
 libsyscall/mach/sbrk.c                        |    78 +
 libsyscall/mach/semaphore.c                   |    86 +
 libsyscall/mach/servers/Makefile.inc          |    16 +
 libsyscall/mach/servers/key_defs.h            |   108 +
 libsyscall/mach/servers/ls_defs.h             |   238 +
 libsyscall/mach/servers/netname.defs          |    76 +
 libsyscall/mach/servers/netname_defs.h        |    71 +
 libsyscall/mach/servers/nm_defs.h             |    84 +
 libsyscall/mach/slot_name.c                   |    88 +
 libsyscall/mach/task.defs                     |    28 +
 libsyscall/mach/thread_act.defs               |    28 +
 libsyscall/mach/vm_map.defs                   |    40 +
 libsyscall/mach/x86_64/Makefile.inc           |     3 +
 libsyscall/mach/x86_64/mach_absolute_time.S   |    32 +
 makedefs/MakeInc.cmd                          |     7 +
 makedefs/MakeInc.def                          |   290 +-
 makedefs/MakeInc.dir                          |   339 +-
 makedefs/MakeInc.rule                         |   288 +-
 osfmk/Makefile                                |    23 +-
 .../UserNotification/KUNCUserNotifications.c  |    38 +-
 .../UserNotification/KUNCUserNotifications.h  |    38 +-
 osfmk/UserNotification/Makefile               |    10 +-
 osfmk/UserNotification/UNDReply.defs          |    38 +-
 osfmk/UserNotification/UNDRequest.defs        |    42 +-
 osfmk/UserNotification/UNDTypes.defs          |    38 +-
 osfmk/UserNotification/UNDTypes.h             |    36 +-
 osfmk/chud/chud_cpu.c                         |    86 +-
 osfmk/chud/chud_dtrace.h                      |    41 +
 osfmk/chud/chud_glue.c                        |    32 +-
 osfmk/chud/chud_memory.c                      |    65 +-
 osfmk/chud/chud_osfmk_callback.c              |   119 +-
 osfmk/chud/chud_thread.c                      |   212 +-
 osfmk/chud/chud_thread.h                      |    36 +
 osfmk/chud/chud_xnu.h                         |   117 +-
 osfmk/chud/chud_xnu_glue.h                    |    36 +-
 osfmk/chud/chud_xnu_private.h                 |    34 +-
 osfmk/chud/i386/chud_cpu_asm.h                |    32 +-
 osfmk/chud/i386/chud_cpu_asm.s                |    30 +-
 osfmk/chud/i386/chud_cpu_i386.c               |   144 +-
 osfmk/chud/i386/chud_osfmk_callback_i386.c    |   142 +-
 osfmk/chud/i386/chud_thread_i386.c            |   799 +-
 osfmk/chud/i386/chud_xnu_glue.h               |    32 +-
 osfmk/chud/i386/chud_xnu_private.h            |    44 +-
 osfmk/chud/ppc/chud_cpu_asm.h                 |    32 +-
 osfmk/chud/ppc/chud_cpu_asm.s                 |    31 +-
 osfmk/chud/ppc/chud_cpu_ppc.c                 |   129 +-
 osfmk/chud/ppc/chud_osfmk_callback_ppc.c      |    65 +-
 osfmk/chud/ppc/chud_spr.h                     |    32 +-
 osfmk/chud/ppc/chud_thread_ppc.c              |   186 +-
 osfmk/chud/ppc/chud_xnu_glue.h                |    32 +-
 osfmk/chud/ppc/chud_xnu_private.h             |    32 +-
 osfmk/conf/MASTER                             |    79 +-
 osfmk/conf/MASTER.i386                        |    15 +-
 osfmk/conf/MASTER.ppc                         |    10 +-
 osfmk/conf/Makefile                           |   261 +-
 osfmk/conf/Makefile.i386                      |   110 +-
 osfmk/conf/Makefile.ppc                       |    29 +-
 osfmk/conf/Makefile.template                  |    20 +-
 osfmk/conf/files                              |    43 +-
 osfmk/conf/files.i386                         |    50 +-
 osfmk/conf/files.ppc                          |     8 +-
 osfmk/conf/tools/doconf/Makefile              |    18 +-
 osfmk/conf/tools/doconf/doconf.csh            |    16 +-
 osfmk/console/i386/kdasm.s                    |    30 +-
 osfmk/console/i386/serial_console.c           |    49 +-
 osfmk/console/i386/text_console.c             |    63 +-
 osfmk/console/i386/text_console.h             |    46 +-
 osfmk/console/i386/video_scroll.c             |    30 +-
 osfmk/console/iso_font.c                      |    30 +-
 osfmk/console/panic_dialog.c                  |   156 +-
 osfmk/console/panic_ui/genimage.c             |     2 +-
 osfmk/console/ppc/serial_console.c            |    79 +-
 osfmk/console/ppc/video_scroll.s              |    30 +-
 osfmk/console/serial_general.c                |    55 +-
 osfmk/console/serial_protos.h                 |    52 +-
 osfmk/console/video_console.c                 |   887 +-
 osfmk/console/video_console.h                 |    36 +-
 osfmk/ddb/db_access.c                         |    30 +-
 osfmk/ddb/db_access.h                         |    30 +-
 osfmk/ddb/db_aout.c                           |    76 +-
 osfmk/ddb/db_aout.h                           |    38 +-
 osfmk/ddb/db_break.c                          |    54 +-
 osfmk/ddb/db_break.h                          |    40 +-
 osfmk/ddb/db_coff.h                           |    34 +-
 osfmk/ddb/db_command.c                        |   579 +-
 osfmk/ddb/db_command.h                        |    36 +-
 osfmk/ddb/db_cond.c                           |    35 +-
 osfmk/ddb/db_cond.h                           |    30 +-
 osfmk/ddb/db_examine.c                        |    80 +-
 osfmk/ddb/db_examine.h                        |    50 +-
 osfmk/ddb/db_expr.c                           |    37 +-
 osfmk/ddb/db_expr.h                           |    30 +-
 osfmk/ddb/db_ext_symtab.c                     |    30 +-
 osfmk/ddb/db_input.c                          |    59 +-
 osfmk/ddb/db_input.h                          |    30 +-
 osfmk/ddb/db_lex.c                            |    52 +-
 osfmk/ddb/db_lex.h                            |    46 +-
 osfmk/ddb/db_macro.c                          |    49 +-
 osfmk/ddb/db_macro.h                          |    30 +-
 osfmk/ddb/db_output.c                         |    34 +-
 osfmk/ddb/db_output.h                         |    30 +-
 osfmk/ddb/db_print.c                          |   209 +-
 osfmk/ddb/db_print.h                          |    36 +-
 osfmk/ddb/db_run.c                            |   111 +-
 osfmk/ddb/db_run.h                            |    60 +-
 osfmk/ddb/db_sym.c                            |   202 +-
 osfmk/ddb/db_sym.h                            |    42 +-
 osfmk/ddb/db_task_thread.c                    |    82 +-
 osfmk/ddb/db_task_thread.h                    |    30 +-
 osfmk/ddb/db_trap.c                           |    41 +-
 osfmk/ddb/db_trap.h                           |    30 +-
 osfmk/ddb/db_variables.c                      |   175 +-
 osfmk/ddb/db_variables.h                      |    36 +-
 osfmk/ddb/db_watch.c                          |    51 +-
 osfmk/ddb/db_watch.h                          |    48 +-
 osfmk/ddb/db_write_cmd.c                      |    39 +-
 osfmk/ddb/db_write_cmd.h                      |    30 +-
 osfmk/ddb/makedis.c                           |    30 +-
 osfmk/ddb/nlist.h                             |    30 +-
 osfmk/ddb/orig/db_print.c                     |    34 +-
 osfmk/ddb/stab.h                              |    30 +-
 osfmk/ddb/tr.c                                |    30 +-
 osfmk/ddb/tr.h                                |    30 +-
 osfmk/default_pager/Makefile                  |    12 +-
 osfmk/default_pager/default_pager.c           |    52 +-
 osfmk/default_pager/default_pager_alerts.defs |    31 +-
 osfmk/default_pager/default_pager_internal.h  |    45 +-
 osfmk/default_pager/default_pager_object.defs |    32 +-
 osfmk/default_pager/default_pager_types.defs  |    32 +-
 osfmk/default_pager/default_pager_types.h     |    30 +-
 osfmk/default_pager/diag.h                    |    32 +-
 osfmk/default_pager/dp_backing_store.c        |   368 +-
 osfmk/default_pager/dp_memory_object.c        |    37 +-
 osfmk/device/Makefile                         |     6 +-
 osfmk/device/device.defs                      |   285 +-
 osfmk/device/device_init.c                    |    30 +-
 osfmk/device/device_port.h                    |    30 +-
 osfmk/device/device_types.defs                |    32 +-
 osfmk/device/device_types.h                   |    61 +-
 osfmk/device/iokit_rpc.c                      |   103 +-
 osfmk/device/subrs.c                          |   241 +-
 osfmk/gssd/Makefile                           |    65 +
 osfmk/gssd/gssd_mach.defs                     |    92 +
 osfmk/gssd/gssd_mach_types.h                  |    58 +
 osfmk/i386/AT386/autoconf.c                   |    24 -
 osfmk/i386/AT386/conf.c                       |    30 +-
 osfmk/i386/AT386/config.h                     |    43 -
 osfmk/i386/AT386/cram.h                       |   157 -
 osfmk/i386/AT386/himem.c                      |   303 -
 osfmk/i386/AT386/himem.h                      |   130 -
 osfmk/i386/AT386/machdep.mk                   |    56 -
 osfmk/i386/AT386/misc_protos.h                |    63 -
 osfmk/i386/AT386/model_dep.c                  |   196 +-
 osfmk/i386/AT386/physmem_entries.h            |    83 -
 osfmk/i386/Diagnostics.c                      |   104 +-
 osfmk/i386/Diagnostics.h                      |    30 +-
 osfmk/i386/Makefile                           |     7 +-
 osfmk/i386/_setjmp.s                          |    30 +-
 osfmk/i386/acpi.c                             |   253 +-
 osfmk/i386/acpi.h                             |    30 +-
 osfmk/i386/acpi_wakeup.s                      |    30 +-
 osfmk/i386/apic.h                             |    30 +-
 osfmk/i386/arch_types.h                       |    30 +-
 osfmk/i386/asm.h                              |    34 +-
 osfmk/i386/asm64.h                            |    36 +-
 osfmk/i386/ast.h                              |    32 +-
 osfmk/i386/ast_check.c                        |    34 +-
 osfmk/i386/ast_types.h                        |    30 +-
 osfmk/i386/bcopy.s                            |    34 +-
 osfmk/i386/bsd_i386.c                         |   557 +-
 osfmk/i386/bzero.s                            |    30 +-
 osfmk/i386/commpage/atomic.s                  |   162 +-
 osfmk/i386/commpage/bcopy_scalar.s            |    38 +-
 .../commpage/{bcopy_sse3.s => bcopy_sse2.s}   |    44 +-
 .../commpage/{bcopy_sse4.s => bcopy_sse3x.s}  |    44 +-
 .../{bcopy_sse4_64.s => bcopy_sse3x_64.s}     |    42 +-
 osfmk/i386/commpage/bzero_scalar.s            |    38 +-
 .../commpage/{bzero_sse3.s => bzero_sse2.s}   |    44 +-
 .../{bzero_sse3_64.s => bzero_sse2_64.s}      |    42 +-
 osfmk/i386/commpage/cacheflush.s              |    40 +-
 osfmk/i386/commpage/commpage.c                |   190 +-
 osfmk/i386/commpage/commpage.h                |    54 +-
 osfmk/i386/commpage/commpage_asm.s            |   156 +-
 osfmk/i386/commpage/commpage_gettimeofday.s   |    79 +-
 .../commpage/commpage_mach_absolute_time.s    |   169 +-
 osfmk/i386/commpage/commpage_sigs.c           |   212 +-
 .../{longcopy_sse4.s => longcopy_sse3x.s}     |    42 +-
 ...longcopy_sse4_64.s => longcopy_sse3x_64.s} |    42 +-
 ...t_pattern_sse3.s => memset_pattern_sse2.s} |    44 +-
 ...ern_sse3_64.s => memset_pattern_sse2_64.s} |    42 +-
 osfmk/i386/commpage/pthreads.s                |    30 +-
 osfmk/i386/commpage/spinlocks.s               |    30 +-
 osfmk/i386/cpu.c                              |    59 +-
 osfmk/i386/cpu_affinity.h                     |    48 +
 osfmk/i386/cpu_capabilities.h                 |    80 +-
 osfmk/i386/cpu_data.h                         |   112 +-
 osfmk/i386/cpu_number.h                       |    30 +-
 osfmk/i386/cpu_threads.c                      |   623 +-
 osfmk/i386/cpu_threads.h                      |    62 +-
 osfmk/i386/cpu_topology.c                     |   258 +
 osfmk/i386/cpu_topology.h                     |   147 +
 osfmk/i386/cpuid.c                            |   829 +-
 osfmk/i386/cpuid.h                            |   231 +-
 osfmk/i386/cswitch.s                          |    32 +-
 osfmk/i386/db_disasm.c                        |    30 +-
 osfmk/i386/db_gcc_aout.c                      |    30 +-
 osfmk/i386/db_interface.c                     |    34 +-
 osfmk/i386/db_machdep.h                       |    60 +-
 osfmk/i386/db_trace.c                         |   178 +-
 osfmk/i386/eflags.h                           |    30 +-
 osfmk/i386/endian.h                           |    30 +-
 osfmk/i386/etimer.c                           |   180 +
 osfmk/i386/exec.h                             |    30 +-
 osfmk/i386/flipc_page.h                       |    30 +-
 osfmk/i386/fpu.c                              |   152 +-
 osfmk/i386/fpu.h                              |    41 +-
 osfmk/i386/gcc.s                              |    30 +-
 osfmk/i386/gdb_defs.h                         |    30 +-
 osfmk/i386/gdt.c                              |    72 +-
 osfmk/i386/genassym.c                         |   105 +-
 osfmk/i386/hi_res_clock.h                     |    41 -
 osfmk/i386/hi_res_clock_map.c                 |    59 -
 osfmk/i386/hibernate_i386.c                   |    32 +-
 osfmk/i386/hibernate_restore.s                |    34 +-
 osfmk/i386/hpet.c                             |   211 +-
 osfmk/i386/hpet.h                             |    64 +-
 osfmk/i386/hw_defs.h                          |    45 +
 osfmk/i386/hw_lock_types.h                    |    30 +-
 osfmk/i386/i386_init.c                        |    62 +-
 osfmk/i386/i386_lock.s                        |  1570 +-
 osfmk/i386/i386_lowmem.h                      |    30 +-
 osfmk/i386/i386_vm_init.c                     |    81 +-
 osfmk/i386/idt.s                              |    62 +-
 osfmk/i386/idt64.s                            |   123 +-
 osfmk/i386/intel_read_fault.h                 |    39 -
 osfmk/i386/io_emulate.c                       |   146 -
 osfmk/i386/io_emulate.h                       |    69 -
 osfmk/i386/io_map.c                           |    30 +-
 osfmk/i386/io_map_entries.h                   |    30 +-
 osfmk/i386/io_port.h                          |    30 +-
 osfmk/i386/iopb.c                             |   662 -
 osfmk/i386/iopb.h                             |    30 +-
 osfmk/i386/iopb_entries.h                     |    44 -
 osfmk/i386/ipl.h                              |    30 +-
 osfmk/i386/ktss.c                             |    35 +-
 osfmk/i386/ldt.c                              |    60 +-
 osfmk/i386/lock.h                             |    37 +-
 osfmk/i386/locks.h                            |    72 +-
 osfmk/i386/locks_i386.c                       |   667 +-
 osfmk/i386/locore.s                           |   539 +-
 osfmk/i386/loose_ends.c                       |   357 +-
 osfmk/i386/lowglobals.h                       |    48 +-
 osfmk/i386/lowmem_vectors.s                   |    47 +-
 osfmk/i386/machdep_call.c                     |    32 +-
 osfmk/i386/machdep_call.h                     |    41 +-
 osfmk/i386/machine_check.c                    |    95 +-
 osfmk/i386/machine_check.h                    |     2 +
 osfmk/i386/machine_cpu.h                      |    30 +-
 osfmk/i386/machine_routines.c                 |   238 +-
 osfmk/i386/machine_routines.h                 |    54 +-
 osfmk/i386/machine_routines_asm.s             |   207 +-
 osfmk/i386/machine_rpc.h                      |    30 +-
 osfmk/i386/machlimits.h                       |    30 +-
 osfmk/i386/machparam.h                        |    30 +-
 osfmk/i386/mcount.s                           |    30 +-
 osfmk/i386/misc_protos.h                      |    48 +-
 osfmk/i386/mp.c                               |   535 +-
 osfmk/i386/mp.h                               |    70 +-
 osfmk/i386/mp_desc.c                          |   295 +-
 osfmk/i386/mp_desc.h                          |    36 +-
 osfmk/i386/mp_events.h                        |    40 +-
 osfmk/i386/mp_slave_boot.h                    |    30 +-
 osfmk/i386/mp_slave_boot.s                    |    30 +-
 osfmk/i386/mtrr.c                             |    67 +-
 osfmk/i386/mtrr.h                             |    30 +-
 osfmk/i386/net_filter.c                       |  1554 --
 osfmk/i386/ntoh.h                             |    68 -
 osfmk/i386/ntoh.s                             |   113 -
 osfmk/i386/pcb.c                              |   655 +-
 osfmk/i386/perfmon.c                          |   290 +-
 osfmk/i386/perfmon.h                          |   152 +-
 osfmk/i386/phys.c                             |    58 +-
 osfmk/i386/pic.h                              |   200 -
 osfmk/i386/pio.h                              |    32 +-
 osfmk/i386/pit.h                              |   126 -
 osfmk/i386/pmCPU.c                            |   685 +-
 osfmk/i386/pmCPU.h                            |   160 +-
 osfmk/i386/pmap.c                             |  2862 +--
 osfmk/i386/pmap.h                             |    80 +-
 osfmk/i386/postcode.h                         |    35 +-
 osfmk/i386/proc_reg.h                         |    60 +-
 osfmk/i386/read_fault.c                       |   281 -
 osfmk/i386/rtclock.c                          |   186 +-
 osfmk/i386/rtclock.h                          |    34 +-
 osfmk/i386/sched_param.h                      |    30 +-
 osfmk/i386/seg.h                              |    44 +-
 osfmk/i386/serial_io.h                        |    42 +
 osfmk/i386/setjmp.h                           |    30 +-
 osfmk/i386/setjmp.s                           |    30 +-
 osfmk/i386/simple_lock.h                      |    30 +-
 osfmk/i386/stab.h                             |    30 +-
 osfmk/i386/start.s                            |    38 +-
 osfmk/i386/start64.s                          |    77 +-
 osfmk/i386/startup64.c                        |   161 +-
 osfmk/i386/task.h                             |    36 +-
 osfmk/i386/thread.h                           |    38 +-
 osfmk/i386/timer.h                            |    30 +-
 osfmk/i386/trap.c                             |   370 +-
 osfmk/i386/trap.h                             |    49 +-
 osfmk/i386/tsc.c                              |   181 +-
 osfmk/i386/tsc.h                              |    36 +-
 osfmk/i386/tss.h                              |    32 +-
 osfmk/i386/user_ldt.c                         |    38 +-
 osfmk/i386/user_ldt.h                         |    30 +-
 osfmk/i386/vm_tuning.h                        |    30 +-
 osfmk/i386/vmx.h                              |    43 +
 osfmk/i386/vmx/vmx_asm.h                      |   158 +
 osfmk/i386/vmx/vmx_cpu.c                      |   373 +
 osfmk/i386/vmx/vmx_cpu.h                      |    96 +
 osfmk/i386/vmx/vmx_shims.c                    |    58 +
 osfmk/i386/vmx/vmx_shims.h                    |    40 +
 osfmk/i386/xpr.h                              |    30 +-
 osfmk/ipc/ipc_entry.c                         |    30 +-
 osfmk/ipc/ipc_entry.h                         |    30 +-
 osfmk/ipc/ipc_hash.c                          |    43 +-
 osfmk/ipc/ipc_hash.h                          |    36 +-
 osfmk/ipc/ipc_init.c                          |    50 +-
 osfmk/ipc/ipc_init.h                          |    38 +-
 osfmk/ipc/ipc_kmsg.c                          |   161 +-
 osfmk/ipc/ipc_kmsg.h                          |    42 +-
 osfmk/ipc/ipc_labelh.c                        |   243 +
 osfmk/ipc/ipc_labelh.h                        |   113 +
 osfmk/ipc/ipc_machdep.h                       |    30 +-
 osfmk/ipc/ipc_mqueue.c                        |    85 +-
 osfmk/ipc/ipc_mqueue.h                        |    32 +-
 osfmk/ipc/ipc_notify.c                        |    42 +-
 osfmk/ipc/ipc_notify.h                        |    30 +-
 osfmk/ipc/ipc_object.c                        |    86 +-
 osfmk/ipc/ipc_object.h                        |    58 +-
 osfmk/ipc/ipc_port.c                          |    68 +-
 osfmk/ipc/ipc_port.h                          |    42 +-
 osfmk/ipc/ipc_print.h                         |    30 +-
 osfmk/ipc/ipc_pset.c                          |    30 +-
 osfmk/ipc/ipc_pset.h                          |    30 +-
 osfmk/ipc/ipc_right.c                         |   234 +-
 osfmk/ipc/ipc_right.h                         |    30 +-
 osfmk/ipc/ipc_space.c                         |    37 +-
 osfmk/ipc/ipc_space.h                         |    40 +-
 osfmk/ipc/ipc_splay.c                         |    30 +-
 osfmk/ipc/ipc_splay.h                         |    30 +-
 osfmk/ipc/ipc_table.c                         |    30 +-
 osfmk/ipc/ipc_table.h                         |    32 +-
 osfmk/ipc/ipc_types.h                         |    30 +-
 osfmk/ipc/mach_debug.c                        |   273 +-
 osfmk/ipc/mach_msg.c                          |   284 +-
 osfmk/ipc/mach_port.c                         |   217 +-
 osfmk/ipc/mig_log.c                           |    30 +-
 osfmk/ipc/port.h                              |    30 +-
 osfmk/kdp/Makefile                            |     5 +
 osfmk/kdp/kdp.c                               |   133 +-
 osfmk/kdp/kdp.h                               |    30 +-
 osfmk/kdp/kdp_callout.h                       |    46 +
 osfmk/kdp/kdp_core.h                          |    30 +-
 osfmk/kdp/kdp_en_debugger.h                   |    30 +-
 osfmk/kdp/kdp_internal.h                      |    35 +-
 osfmk/kdp/kdp_private.h                       |    30 +-
 osfmk/kdp/kdp_protocol.h                      |    30 +-
 osfmk/kdp/kdp_udp.c                           |   170 +-
 osfmk/kdp/kdp_udp.h                           |    30 +-
 osfmk/kdp/ml/i386/kdp_machdep.c               |   223 +-
 osfmk/kdp/ml/i386/kdp_vm.c                    |    42 +-
 osfmk/kdp/ml/ppc/kdp_asm.s                    |    30 +-
 osfmk/kdp/ml/ppc/kdp_machdep.c                |    96 +-
 osfmk/kdp/ml/ppc/kdp_misc.s                   |    30 +-
 osfmk/kdp/ml/ppc/kdp_vm.c                     |    71 +-
 osfmk/kern/Makefile                           |     3 +-
 osfmk/kern/affinity.c                         |   552 +
 osfmk/kern/affinity.h                         |    67 +
 osfmk/kern/assert.h                           |    44 +-
 osfmk/kern/ast.c                              |    38 +-
 osfmk/kern/ast.h                              |    36 +-
 osfmk/kern/bits.c                             |    30 +-
 osfmk/kern/bsd_kern.c                         |   150 +-
 osfmk/kern/call_entry.h                       |    30 +-
 osfmk/kern/clock.c                            |   313 +-
 osfmk/kern/clock.h                            |    60 +-
 osfmk/kern/clock_oldops.c                     |    38 +-
 osfmk/kern/counters.c                         |    60 +-
 osfmk/kern/counters.h                         |    50 +-
 osfmk/kern/cpu_data.h                         |    30 +-
 osfmk/kern/cpu_number.h                       |    30 +-
 osfmk/kern/debug.c                            |   317 +-
 osfmk/kern/debug.h                            |    59 +-
 osfmk/kern/etimer.h                           |    34 +-
 osfmk/kern/exception.c                        |   400 +-
 osfmk/kern/exception.h                        |    40 +-
 osfmk/kern/hibernate.c                        |   172 +-
 osfmk/kern/host.c                             |   124 +-
 osfmk/kern/host.h                             |    30 +-
 osfmk/kern/host_notify.c                      |    30 +-
 osfmk/kern/host_notify.h                      |    32 +-
 osfmk/kern/host_statistics.h                  |    38 +-
 osfmk/kern/ipc_clock.c                        |    30 +-
 osfmk/kern/ipc_host.c                         |   126 +-
 osfmk/kern/ipc_host.h                         |    42 +-
 osfmk/kern/ipc_kobject.c                      |    68 +-
 osfmk/kern/ipc_kobject.h                      |    42 +-
 osfmk/kern/ipc_mig.c                          |    98 +-
 osfmk/kern/ipc_mig.h                          |    38 +-
 osfmk/kern/ipc_sync.c                         |    30 +-
 osfmk/kern/ipc_sync.h                         |    30 +-
 osfmk/kern/ipc_tt.c                           |   159 +-
 osfmk/kern/ipc_tt.h                           |    30 +-
 osfmk/kern/kalloc.c                           |    74 +-
 osfmk/kern/kalloc.h                           |    32 +-
 osfmk/kern/kern_print.h                       |    47 +-
 osfmk/kern/kern_types.h                       |    44 +-
 osfmk/kern/kmod.c                             |   877 +-
 osfmk/kern/ledger.c                           |    38 +-
 osfmk/kern/ledger.h                           |    32 +-
 osfmk/kern/lock.h                             |    48 +-
 osfmk/kern/locks.c                            |   159 +-
 osfmk/kern/locks.h                            |    91 +-
 osfmk/kern/mach_clock.c                       |   211 +-
 osfmk/kern/mach_param.h                       |    38 +-
 osfmk/kern/machine.c                          |   186 +-
 osfmk/kern/machine.h                          |    30 +-
 osfmk/kern/macro_help.h                       |    30 +-
 osfmk/kern/misc_protos.h                      |    76 +-
 osfmk/kern/mk_sp.c                            |    64 +-
 osfmk/kern/mk_timer.c                         |    30 +-
 osfmk/kern/mk_timer.h                         |    32 +-
 osfmk/kern/norma_protos.h                     |    30 +-
 osfmk/kern/page_decrypt.c                     |    46 +-
 osfmk/kern/page_decrypt.h                     |    42 +-
 osfmk/kern/pms.h                              |    37 +-
 osfmk/kern/printf.c                           |   131 +-
 osfmk/kern/priority.c                         |    89 +-
 osfmk/kern/processor.c                        |   576 +-
 osfmk/kern/processor.h                        |   232 +-
 osfmk/kern/processor_data.c                   |    45 +-
 osfmk/kern/processor_data.h                   |    50 +-
 osfmk/kern/profile.c                          |   498 -
 osfmk/kern/profile.h                          |   187 -
 osfmk/kern/queue.c                            |    34 +-
 osfmk/kern/queue.h                            |    39 +-
 osfmk/kern/sched.h                            |    70 +-
 osfmk/kern/sched_average.c                    |   146 +-
 osfmk/kern/sched_prim.c                       |  2203 +-
 osfmk/kern/sched_prim.h                       |    74 +-
 osfmk/kern/security.c                         |   425 +
 osfmk/kern/simple_lock.h                      |    56 +-
 osfmk/kern/spl.c                              |    30 +-
 osfmk/kern/spl.h                              |    30 +-
 osfmk/kern/stack.c                            |   154 +-
 osfmk/kern/startup.c                          |   160 +-
 osfmk/kern/startup.h                          |    32 +-
 osfmk/kern/symbols.c                          |   228 +
 osfmk/kern/sync_lock.c                        |    32 +-
 osfmk/kern/sync_lock.h                        |    30 +-
 osfmk/kern/sync_sema.c                        |    41 +-
 osfmk/kern/sync_sema.h                        |    34 +-
 osfmk/kern/syscall_emulation.c                |    52 +-
 osfmk/kern/syscall_subr.c                     |    87 +-
 osfmk/kern/syscall_subr.h                     |    30 +-
 osfmk/kern/syscall_sw.c                       |    30 +-
 osfmk/kern/syscall_sw.h                       |    44 +-
 osfmk/kern/task.c                             |   563 +-
 osfmk/kern/task.h                             |   120 +-
 osfmk/kern/task_policy.c                      |    30 +-
 osfmk/kern/task_swap.c                        |    30 +-
 osfmk/kern/task_swap.h                        |    30 +-
 osfmk/kern/thread.c                           |   331 +-
 osfmk/kern/thread.h                           |   194 +-
 osfmk/kern/thread_act.c                       |    87 +-
 osfmk/kern/thread_call.c                      |    97 +-
 osfmk/kern/thread_call.h                      |    37 +-
 osfmk/kern/thread_policy.c                    |   109 +-
 osfmk/kern/timer.c                            |   122 +-
 osfmk/kern/timer.h                            |    82 +-
 osfmk/kern/timer_call.c                       |   105 +-
 osfmk/kern/timer_call.h                       |    30 +-
 osfmk/kern/wait_queue.c                       |   143 +-
 osfmk/kern/wait_queue.h                       |    59 +-
 osfmk/kern/xpr.c                              |    47 +-
 osfmk/kern/xpr.h                              |    30 +-
 osfmk/kern/zalloc.c                           |   346 +-
 osfmk/kern/zalloc.h                           |    41 +-
 osfmk/libsa/ctype.h                           |    30 +-
 osfmk/libsa/errno.h                           |    30 +-
 osfmk/libsa/float.h                           |    30 +-
 osfmk/libsa/i386/float.h                      |    30 +-
 osfmk/libsa/i386/math.h                       |    30 +-
 osfmk/libsa/i386/stdarg.h                     |    66 -
 osfmk/libsa/i386/types.h                      |    30 +-
 osfmk/libsa/i386/va_list.h                    |    79 -
 osfmk/libsa/ieeefloat.h                       |    30 +-
 osfmk/libsa/machine/stdarg.h                  |    35 -
 osfmk/libsa/machine/stdarg_apple.h            |    30 +-
 osfmk/libsa/machine/types.h                   |    36 +-
 osfmk/libsa/machine/va_list.h                 |    35 -
 osfmk/libsa/math.h                            |    30 +-
 osfmk/libsa/ppc/float.h                       |    30 +-
 osfmk/libsa/ppc/math.h                        |    30 +-
 osfmk/libsa/ppc/stdarg.h                      |    25 -
 osfmk/libsa/ppc/stdarg_apple.h                |    30 +-
 osfmk/libsa/ppc/types.h                       |    30 +-
 osfmk/libsa/ppc/va_list.h                     |   126 -
 osfmk/libsa/stdarg.h                          |    43 -
 osfmk/libsa/stdio.h                           |    39 +-
 osfmk/libsa/stdlib.h                          |    34 +-
 osfmk/libsa/string.h                          |    63 +-
 osfmk/libsa/sys/timers.h                      |    30 +-
 osfmk/libsa/types.h                           |    30 +-
 osfmk/libsa/va_list.h                         |    43 -
 osfmk/lockd/Makefile                          |    65 +
 osfmk/lockd/lockd_mach.defs                   |    76 +
 osfmk/lockd/lockd_mach_types.h                |    49 +
 osfmk/mach-o/loader.h                         |    42 +-
 osfmk/mach-o/mach_header.c                    |    41 +-
 osfmk/mach-o/mach_header.h                    |    30 +-
 osfmk/mach/Makefile                           |    38 +-
 osfmk/mach/Makefile.template                  |    13 +-
 osfmk/mach/alert.h                            |    30 +-
 osfmk/mach/audit_triggers.defs                |    24 +-
 osfmk/mach/boolean.h                          |    30 +-
 osfmk/mach/bootstrap.h                        |    30 +-
 osfmk/mach/clock.defs                         |    32 +-
 osfmk/mach/clock_priv.defs                    |    31 +-
 osfmk/mach/clock_reply.defs                   |    32 +-
 osfmk/mach/clock_types.defs                   |    32 +-
 osfmk/mach/clock_types.h                      |    30 +-
 osfmk/mach/error.h                            |    30 +-
 osfmk/mach/events_info.h                      |    30 +-
 osfmk/mach/exc.defs                           |    31 +-
 osfmk/mach/exception.h                        |    34 +-
 osfmk/mach/exception_types.h                  |    59 +-
 osfmk/mach/flipc_cb.h                         |    30 +-
 osfmk/mach/flipc_debug.h                      |    30 +-
 osfmk/mach/flipc_device.h                     |    30 +-
 osfmk/mach/flipc_locks.h                      |    30 +-
 osfmk/mach/flipc_types.h                      |    30 +-
 osfmk/mach/host_info.h                        |    47 +-
 osfmk/mach/host_notify.h                      |    30 +-
 osfmk/mach/host_notify_reply.defs             |    32 +-
 osfmk/mach/host_priv.defs                     |    32 +-
 osfmk/mach/host_reboot.h                      |    30 +-
 osfmk/mach/host_security.defs                 |    31 +-
 osfmk/mach/host_special_ports.h               |    38 +-
 osfmk/mach/i386/Makefile                      |     2 +-
 osfmk/mach/i386/_structs.h                    |   561 +
 osfmk/mach/i386/_types.h                      |   221 +
 osfmk/mach/i386/boolean.h                     |    32 +-
 osfmk/mach/i386/exception.h                   |    32 +-
 osfmk/mach/i386/flipc_dep.h                   |    30 +-
 osfmk/mach/i386/fp_reg.h                      |    32 +-
 osfmk/mach/i386/kern_return.h                 |    30 +-
 osfmk/mach/i386/machine_types.defs            |    32 +-
 osfmk/mach/i386/ndr_def.h                     |    30 +-
 osfmk/mach/i386/processor_info.h              |    30 +-
 osfmk/mach/i386/rpc.h                         |    30 +-
 osfmk/mach/i386/sdt_isa.h                     |   465 +
 osfmk/mach/i386/syscall_sw.h                  |    53 +-
 osfmk/mach/i386/thread_state.h                |    32 +-
 osfmk/mach/i386/thread_status.h               |   474 +-
 osfmk/mach/i386/vm_param.h                    |    42 +-
 osfmk/mach/i386/vm_types.h                    |    39 +-
 osfmk/mach/kern_return.h                      |    30 +-
 osfmk/mach/kmod.h                             |    54 +-
 osfmk/mach/ledger.defs                        |    31 +-
 osfmk/mach/lock_set.defs                      |    31 +-
 osfmk/mach/mach.h                             |    30 +-
 osfmk/mach/mach_exc.defs                      |   119 +
 osfmk/mach/mach_host.defs                     |    40 +-
 osfmk/mach/mach_interface.h                   |    31 +-
 osfmk/mach/mach_norma.defs                    |    32 +-
 osfmk/mach/mach_notify.defs                   |    34 +-
 osfmk/mach/mach_param.h                       |    30 +-
 osfmk/mach/mach_port.defs                     |    44 +-
 osfmk/mach/mach_syscalls.h                    |    30 +-
 osfmk/mach/mach_time.h                        |    30 +-
 osfmk/mach/mach_traps.h                       |    36 +-
 osfmk/mach/mach_types.defs                    |    47 +-
 osfmk/mach/mach_types.h                       |    37 +-
 osfmk/mach/mach_vm.defs                       |    53 +-
 osfmk/mach/machine.h                          |    74 +-
 osfmk/mach/machine/Makefile                   |     2 +-
 osfmk/mach/machine/asm.h                      |    34 +-
 osfmk/mach/machine/boolean.h                  |    34 +-
 osfmk/mach/machine/exception.h                |    34 +-
 osfmk/mach/machine/kern_return.h              |    34 +-
 osfmk/mach/machine/machine_types.defs         |    36 +-
 osfmk/mach/machine/ndr_def.h                  |    34 +-
 osfmk/mach/machine/processor_info.h           |    34 +-
 osfmk/mach/machine/rpc.h                      |    34 +-
 osfmk/mach/machine/sdt.h                      |   257 +
 osfmk/mach/machine/sdt_isa.h                  |    41 +
 osfmk/mach/machine/syscall_sw.h               |    34 +-
 osfmk/mach/machine/thread_state.h             |    34 +-
 osfmk/mach/machine/thread_status.h            |    34 +-
 osfmk/mach/machine/vm_param.h                 |    34 +-
 osfmk/mach/machine/vm_types.h                 |    34 +-
 osfmk/mach/memory_object.defs                 |    35 +-
 osfmk/mach/memory_object.h                    |    30 +-
 osfmk/mach/memory_object_control.defs         |    39 +-
 osfmk/mach/memory_object_default.defs         |    31 +-
 osfmk/mach/memory_object_name.defs            |    32 +-
 osfmk/mach/memory_object_types.h              |    60 +-
 osfmk/mach/message.h                          |    86 +-
 osfmk/mach/mig.h                              |    50 +-
 osfmk/mach/mig_errors.h                       |    44 +-
 osfmk/mach/mig_log.h                          |    30 +-
 osfmk/mach/mk_timer.h                         |    30 +-
 osfmk/mach/mk_traps.h                         |    30 +-
 osfmk/mach/msg_type.h                         |    34 +-
 osfmk/mach/ndr.h                              |    36 +-
 osfmk/mach/norma_special_ports.h              |    30 +-
 osfmk/mach/notify.defs                        |    32 +-
 osfmk/mach/notify.h                           |    30 +-
 osfmk/mach/policy.h                           |    30 +-
 osfmk/mach/port.h                             |    63 +-
 osfmk/mach/ppc/Makefile                       |     2 +-
 osfmk/mach/ppc/_structs.h                     |   392 +
 osfmk/mach/ppc/_types.h                       |    64 +-
 osfmk/mach/ppc/boolean.h                      |    30 +-
 osfmk/mach/ppc/exception.h                    |    32 +-
 osfmk/mach/ppc/kern_return.h                  |    30 +-
 osfmk/mach/ppc/machine_types.defs             |    32 +-
 osfmk/mach/ppc/ndr_def.h                      |    30 +-
 osfmk/mach/ppc/processor_info.h               |    30 +-
 osfmk/mach/ppc/rpc.h                          |    30 +-
 osfmk/mach/ppc/sdt_isa.h                      |   427 +
 osfmk/mach/ppc/syscall_sw.h                   |    32 +-
 osfmk/mach/ppc/thread_state.h                 |    30 +-
 osfmk/mach/ppc/thread_status.h                |    69 +-
 osfmk/mach/ppc/vm_param.h                     |    37 +-
 osfmk/mach/ppc/vm_types.h                     |    37 +-
 osfmk/mach/processor.defs                     |    32 +-
 osfmk/mach/processor_info.h                   |    32 +-
 osfmk/mach/processor_set.defs                 |    35 +-
 osfmk/mach/prof.defs                          |    34 +-
 osfmk/mach/prof_types.h                       |    30 +-
 osfmk/mach/rpc.h                              |    30 +-
 osfmk/mach/sdt.h                              |    32 +
 osfmk/mach/security.defs                      |   208 +
 osfmk/mach/semaphore.defs                     |    32 +-
 osfmk/mach/semaphore.h                        |    30 +-
 osfmk/mach/shared_memory_server.h             |    63 +-
 osfmk/mach/shared_region.h                    |    99 +
 osfmk/mach/std_types.defs                     |    32 +-
 osfmk/mach/std_types.h                        |    30 +-
 osfmk/mach/sync.defs                          |    32 +-
 osfmk/mach/sync_policy.h                      |    30 +-
 osfmk/mach/syscall_sw.h                       |    38 +-
 osfmk/mach/task.defs                          |    39 +-
 osfmk/mach/task_access.defs                   |    52 +
 osfmk/mach/task_info.h                        |    49 +-
 osfmk/mach/task_ledger.h                      |    30 +-
 osfmk/mach/task_policy.h                      |    30 +-
 osfmk/mach/task_special_ports.h               |    56 +-
 osfmk/mach/thread_act.defs                    |    42 +-
 osfmk/mach/thread_info.h                      |    30 +-
 osfmk/mach/thread_policy.h                    |    64 +-
 osfmk/mach/thread_special_ports.h             |    30 +-
 osfmk/mach/thread_status.h                    |    30 +-
 osfmk/mach/thread_switch.h                    |    30 +-
 osfmk/mach/time_value.h                       |    30 +-
 osfmk/mach/upl.defs                           |    31 +-
 osfmk/mach/vm_attributes.h                    |    30 +-
 osfmk/mach/vm_behavior.h                      |    30 +-
 osfmk/mach/vm_inherit.h                       |    30 +-
 osfmk/mach/vm_map.defs                        |    34 +-
 osfmk/mach/vm_param.h                         |    36 +-
 osfmk/mach/vm_prot.h                          |    36 +-
 osfmk/mach/vm_purgable.h                      |   115 +-
 osfmk/mach/vm_region.h                        |    55 +-
 osfmk/mach/vm_statistics.h                    |   107 +-
 osfmk/mach/vm_sync.h                          |    30 +-
 osfmk/mach/vm_types.h                         |    35 +-
 osfmk/mach_debug/hash_info.h                  |    30 +-
 osfmk/mach_debug/ipc_info.h                   |    30 +-
 osfmk/mach_debug/lockgroup_info.h             |    30 +-
 osfmk/mach_debug/mach_debug.h                 |    30 +-
 osfmk/mach_debug/mach_debug_types.defs        |    32 +-
 osfmk/mach_debug/mach_debug_types.h           |    30 +-
 osfmk/mach_debug/page_info.h                  |    30 +-
 osfmk/mach_debug/vm_info.h                    |    34 +-
 osfmk/mach_debug/zone_info.h                  |    30 +-
 osfmk/machine/asm.h                           |    36 +-
 osfmk/machine/ast.h                           |    36 +-
 osfmk/machine/ast_types.h                     |    36 +-
 osfmk/machine/commpage.h                      |    34 +-
 osfmk/machine/cpu_affinity.h                  |    45 +
 osfmk/machine/cpu_capabilities.h              |    36 +-
 osfmk/machine/cpu_data.h                      |    36 +-
 osfmk/machine/cpu_number.h                    |    36 +-
 osfmk/machine/db_machdep.h                    |    36 +-
 osfmk/machine/endian.h                        |    36 +-
 osfmk/machine/io_map_entries.h                |    36 +-
 osfmk/machine/lock.h                          |    36 +-
 osfmk/machine/locks.h                         |    36 +-
 osfmk/machine/machine_cpu.h                   |    36 +-
 osfmk/machine/machine_routines.h              |    36 +-
 osfmk/machine/machine_rpc.h                   |    36 +-
 osfmk/machine/machlimits.h                    |    36 +-
 osfmk/machine/machparam.h                     |    36 +-
 osfmk/machine/pmap.h                          |    36 +-
 osfmk/machine/sched_param.h                   |    36 +-
 osfmk/machine/setjmp.h                        |    36 +-
 osfmk/machine/simple_lock.h                   |    36 +-
 osfmk/machine/task.h                          |    36 +-
 osfmk/machine/thread.h                        |    36 +-
 osfmk/machine/timer.h                         |    36 +-
 osfmk/machine/trap.h                          |    36 +-
 osfmk/machine/vm_tuning.h                     |    36 +-
 osfmk/machine/xpr.h                           |    36 +-
 osfmk/man/mach_port_status.html               |    39 +-
 osfmk/ppc/AltiAssist.s                        |    35 +-
 osfmk/ppc/Diagnostics.c                       |    59 +-
 osfmk/ppc/Diagnostics.h                       |    32 +-
 osfmk/ppc/Emulate.s                           |    54 +-
 osfmk/ppc/Emulate64.s                         |    42 +-
 osfmk/ppc/Firmware.h                          |    30 +-
 osfmk/ppc/Firmware.s                          |    30 +-
 osfmk/ppc/FirmwareC.c                         |   471 +-
 osfmk/ppc/FirmwareCalls.h                     |    30 +-
 osfmk/ppc/Makefile                            |     1 -
 osfmk/ppc/PPCcalls.c                          |    30 +-
 osfmk/ppc/PPCcalls.h                          |    30 +-
 osfmk/ppc/Performance.h                       |    30 +-
 osfmk/ppc/Performance.s                       |    30 +-
 osfmk/ppc/PseudoKernel.c                      |    78 +-
 osfmk/ppc/PseudoKernel.h                      |    32 +-
 osfmk/ppc/_setjmp.s                           |    30 +-
 osfmk/ppc/aligned_data.s                      |    30 +-
 osfmk/ppc/asm.h                               |    78 +-
 osfmk/ppc/ast.h                               |    30 +-
 osfmk/ppc/ast_types.h                         |    30 +-
 osfmk/ppc/atomic_switch.h                     |    30 +-
 osfmk/ppc/atomic_switch.s                     |    30 +-
 osfmk/ppc/bat_init.c                          |    30 +-
 osfmk/ppc/bcopy.s                             |    30 +-
 osfmk/ppc/bcopytest.c                         |    15 +-
 osfmk/ppc/bits.s                              |    30 +-
 osfmk/ppc/boot.h                              |    30 +-
 osfmk/ppc/bzero.s                             |    51 +-
 osfmk/ppc/cache.s                             |    30 +-
 osfmk/ppc/commpage/atomic.s                   |    30 +-
 osfmk/ppc/commpage/bcopy_64.s                 |    31 +-
 osfmk/ppc/commpage/bcopy_970.s                |    31 +-
 osfmk/ppc/commpage/bcopy_g3.s                 |    31 +-
 osfmk/ppc/commpage/bcopy_g4.s                 |    31 +-
 osfmk/ppc/commpage/bigcopy_970.s              |    30 +-
 osfmk/ppc/commpage/bzero_128.s                |    31 +-
 osfmk/ppc/commpage/bzero_32.s                 |    31 +-
 osfmk/ppc/commpage/cacheflush.s               |    31 +-
 osfmk/ppc/commpage/commpage.c                 |    54 +-
 osfmk/ppc/commpage/commpage.h                 |    36 +-
 osfmk/ppc/commpage/commpage_asm.s             |    30 +-
 osfmk/ppc/commpage/gettimeofday.s             |    31 +-
 osfmk/ppc/commpage/mach_absolute_time.s       |    31 +-
 osfmk/ppc/commpage/memset_64.s                |    31 +-
 osfmk/ppc/commpage/memset_g3.s                |    31 +-
 osfmk/ppc/commpage/memset_g4.s                |    31 +-
 osfmk/ppc/commpage/memset_g5.s                |    31 +-
 osfmk/ppc/commpage/pthread.s                  |    30 +-
 osfmk/ppc/commpage/spinlocks.s                |    30 +-
 osfmk/ppc/conf.c                              |    36 +-
 osfmk/ppc/console_feed.c                      |    30 +-
 osfmk/ppc/console_feed_entries.h              |    30 +-
 osfmk/ppc/cpu.c                               |   139 +-
 osfmk/ppc/cpu_affinity.h                      |    52 +
 osfmk/ppc/cpu_capabilities.h                  |    61 +-
 osfmk/ppc/cpu_data.h                          |    30 +-
 osfmk/ppc/cpu_internal.h                      |    41 +-
 osfmk/ppc/cpu_number.h                        |    30 +-
 osfmk/ppc/cswtch.s                            |    30 +-
 osfmk/ppc/db_asm.s                            |    30 +-
 osfmk/ppc/db_disasm.c                         |    44 +-
 osfmk/ppc/db_interface.c                      |   106 +-
 osfmk/ppc/db_low_trace.c                      |   173 +-
 osfmk/ppc/db_low_trace.h                      |    97 +-
 osfmk/ppc/db_machdep.h                        |    38 +-
 osfmk/ppc/db_trace.c                          |  1004 +-
 osfmk/ppc/endian.h                            |    30 +-
 osfmk/{kern => ppc}/etimer.c                  |   111 +-
 osfmk/ppc/exception.h                         |    51 +-
 osfmk/ppc/fpu_protos.h                        |    30 +-
 osfmk/ppc/genassym.c                          |    67 +-
 osfmk/ppc/hardclock_entries.h                 |    30 +-
 osfmk/ppc/hexfont.h                           |    30 +-
 osfmk/ppc/hibernate_ppc.c                     |    44 +-
 osfmk/ppc/hibernate_restore.s                 |    30 +-
 osfmk/ppc/hw_counters.h                       |    30 +-
 osfmk/ppc/hw_exception.s                      |   151 +-
 osfmk/ppc/hw_lock.s                           |   195 +-
 osfmk/ppc/hw_lock_types.h                     |    30 +-
 osfmk/ppc/hw_perfmon.c                        |    72 +-
 osfmk/ppc/hw_perfmon.h                        |    30 +-
 osfmk/ppc/hw_perfmon_mmcr.h                   |    30 +-
 osfmk/ppc/hw_vm.s                             |    44 +-
 osfmk/ppc/instrumentation.h                   |    30 +-
 osfmk/ppc/interrupt.c                         |    82 +-
 osfmk/ppc/io_map.c                            |    34 +-
 osfmk/ppc/io_map_entries.h                    |    30 +-
 osfmk/ppc/lock.h                              |    30 +-
 osfmk/ppc/locks.h                             |    35 +-
 osfmk/ppc/locks_ppc.c                         |   502 +-
 osfmk/ppc/low_trace.h                         |    30 +-
 osfmk/ppc/lowglobals.h                        |    35 +-
 osfmk/ppc/lowmem_vectors.s                    |   747 +-
 osfmk/ppc/machine_cpu.h                       |    30 +-
 osfmk/ppc/machine_routines.c                  |   158 +-
 osfmk/ppc/machine_routines.h                  |    54 +-
 osfmk/ppc/machine_routines_asm.s              |   100 +-
 osfmk/ppc/machine_rpc.h                       |    30 +-
 osfmk/ppc/machlimits.h                        |    30 +-
 osfmk/ppc/machparam.h                         |    30 +-
 osfmk/ppc/mappings.c                          |   128 +-
 osfmk/ppc/mappings.h                          |    33 +-
 osfmk/ppc/mcount.s                            |    36 +-
 osfmk/ppc/mem.h                               |    30 +-
 osfmk/ppc/misc.c                              |    30 +-
 osfmk/ppc/misc_asm.s                          |    30 +-
 osfmk/ppc/misc_protos.h                       |    66 +-
 osfmk/ppc/model_dep.c                         |   489 +-
 osfmk/ppc/movc.s                              |    30 +-
 osfmk/ppc/mp.h                                |    30 +-
 osfmk/ppc/new_screen.h                        |    33 +-
 osfmk/ppc/pcb.c                               |   179 +-
 osfmk/ppc/pmap.c                              |   200 +-
 osfmk/ppc/pmap.h                              |    46 +-
 osfmk/{kern => ppc}/pms.c                     |   195 +-
 osfmk/ppc/pms.h                               |    29 -
 osfmk/ppc/pmsCPU.c                            |    45 +-
 osfmk/ppc/ppc_init.c                          |    55 +-
 osfmk/ppc/ppc_vm_init.c                       |    69 +-
 osfmk/ppc/proc_reg.h                          |    30 +-
 osfmk/ppc/rtclock.c                           |    48 +-
 osfmk/ppc/rtclock.h                           |    32 +-
 osfmk/ppc/savearea.c                          |    67 +-
 osfmk/ppc/savearea.h                          |    51 +-
 osfmk/ppc/savearea_asm.s                      |    30 +-
 osfmk/ppc/scc_8530.h                          |    30 +-
 osfmk/ppc/sched_param.h                       |    30 +-
 osfmk/ppc/screen_switch.h                     |    30 +-
 osfmk/ppc/serial_defs.h                       |    30 +-
 osfmk/ppc/serial_io.c                         |    95 +-
 osfmk/ppc/serial_io.h                         |    41 +-
 osfmk/ppc/setjmp.h                            |    30 +-
 osfmk/ppc/simple_lock.h                       |    30 +-
 osfmk/ppc/skiplists.s                         |    30 +-
 osfmk/ppc/spec_reg.h                          |    30 +-
 osfmk/ppc/start.s                             |    31 +-
 osfmk/ppc/status.c                            |   182 +-
 osfmk/ppc/task.h                              |    34 +-
 osfmk/ppc/testjump.c                          |    30 +-
 osfmk/ppc/thread.h                            |    40 +-
 osfmk/ppc/trap.c                              |   540 +-
 osfmk/ppc/trap.h                              |    37 +-
 osfmk/ppc/vm_tuning.h                         |    30 +-
 osfmk/ppc/vmachmon.c                          |   256 +-
 osfmk/ppc/vmachmon.h                          |    30 +-
 osfmk/ppc/vmachmon_asm.s                      |    30 +-
 osfmk/ppc/xpr.h                               |    30 +-
 osfmk/profiling/Makefile                      |     6 +
 osfmk/profiling/i386/profile-asm.s            |    30 +-
 osfmk/profiling/i386/profile-md.c             |    30 +-
 osfmk/profiling/i386/profile-md.h             |    30 +-
 osfmk/profiling/machine/profile-md.h          |    36 +-
 osfmk/profiling/ppc/profile-md.h              |    30 +-
 osfmk/profiling/profile-internal.h            |    30 +-
 osfmk/profiling/profile-kgmon.c               |    33 +-
 osfmk/profiling/profile-mk.c                  |    30 +-
 osfmk/profiling/profile-mk.h                  |    30 +-
 osfmk/sys/scsi.h                              |    30 +-
 osfmk/sys/sdi.h                               |    30 +-
 osfmk/sys/sdi_edt.h                           |    30 +-
 osfmk/sys/syslog.h                            |    30 +-
 osfmk/sys/time.h                              |    30 +-
 osfmk/sys/tm.h                                |    30 +-
 osfmk/sys/types.h                             |    30 +-
 osfmk/sys/varargs.h                           |    34 +-
 osfmk/vm/Makefile                             |     3 +-
 osfmk/vm/bsd_vm.c                             |   147 +-
 osfmk/vm/cpm.h                                |    32 +-
 osfmk/vm/device_vm.c                          |    37 +-
 osfmk/vm/memory_object.c                      |   408 +-
 osfmk/vm/memory_object.h                      |    45 +-
 osfmk/vm/pmap.h                               |    78 +-
 osfmk/vm/task_working_set.c                   |  2305 ---
 osfmk/vm/task_working_set.h                   |   255 -
 osfmk/vm/vm_apple_protect.c                   |   318 +-
 osfmk/vm/vm_debug.c                           |    46 +-
 osfmk/vm/vm_debug.h                           |    30 +-
 osfmk/vm/vm_external.c                        |    30 +-
 osfmk/vm/vm_external.h                        |    30 +-
 osfmk/vm/vm_fault.c                           |  3860 ++--
 osfmk/vm/vm_fault.h                           |    47 +-
 osfmk/vm/vm_init.c                            |    71 +-
 osfmk/vm/vm_init.h                            |    34 +-
 osfmk/vm/vm_kern.c                            |   203 +-
 osfmk/vm/vm_kern.h                            |    61 +-
 osfmk/vm/vm_map.c                             |  5334 ++---
 osfmk/vm/vm_map.h                             |   120 +-
 osfmk/vm/vm_object.c                          |  1756 +-
 osfmk/vm/vm_object.h                          |   201 +-
 osfmk/vm/vm_page.h                            |   282 +-
 osfmk/vm/vm_pageout.c                         |  4784 +++--
 osfmk/vm/vm_pageout.h                         |    52 +-
 osfmk/vm/vm_print.h                           |    36 +-
 osfmk/vm/vm_protos.h                          |   121 +-
 osfmk/vm/vm_purgeable.c                       |   563 +
 osfmk/vm/vm_purgeable_internal.h              |   121 +
 osfmk/vm/vm_resident.c                        |  2010 +-
 osfmk/vm/vm_shared_memory_server.c            |  2581 ---
 osfmk/vm/vm_shared_memory_server.h            |   310 -
 osfmk/vm/vm_shared_region.c                   |  1271 ++
 osfmk/vm/vm_shared_region.h                   |   165 +
 osfmk/vm/vm_user.c                            |   410 +-
 pexpert/Makefile                              |     5 +-
 pexpert/conf/MASTER                           |    15 +-
 pexpert/conf/MASTER.i386                      |    15 +-
 pexpert/conf/MASTER.ppc                       |     9 +-
 pexpert/conf/Makefile                         |    24 +-
 pexpert/conf/Makefile.template                |    12 +-
 pexpert/conf/files                            |     3 +-
 pexpert/conf/files.i386                       |     1 -
 pexpert/conf/tools/doconf/Makefile            |    18 +-
 pexpert/conf/tools/doconf/doconf.csh          |    16 +-
 pexpert/gen/bootargs.c                        |    53 +-
 pexpert/gen/device_tree.c                     |    51 +-
 pexpert/gen/pe_gen.c                          |    42 +-
 pexpert/i386/boot_images.h                    |    30 +-
 pexpert/i386/fakePPCDeviceTree.c              |   170 -
 pexpert/i386/fakePPCDeviceTree.h              |    42 -
 pexpert/i386/fakePPCStructs.h                 |    86 -
 pexpert/i386/kd.c                             |    30 +-
 pexpert/i386/pe_bootargs.c                    |    30 +-
 pexpert/i386/pe_identify_machine.c            |    30 +-
 pexpert/i386/pe_init.c                        |    94 +-
 pexpert/i386/pe_interrupt.c                   |    56 +-
 pexpert/i386/pe_kprintf.c                     |   114 +-
 pexpert/i386/pe_misc.s                        |    30 +-
 pexpert/i386/pe_serial.c                      |    36 +-
 pexpert/i386/pe_spl.c                         |    30 +-
 pexpert/pexpert/AppleBoot.h                   |   171 +
 pexpert/pexpert/Clut.h                        |   101 +
 pexpert/pexpert/Makefile                      |     5 +
 pexpert/pexpert/boot.h                        |    30 +-
 pexpert/pexpert/device_tree.h                 |    32 +-
 pexpert/pexpert/i386/boot.h                   |    30 +-
 pexpert/pexpert/i386/efi.h                    |    35 +-
 pexpert/pexpert/i386/fb_entries.h             |    30 +-
 pexpert/pexpert/i386/kd_entries.h             |    30 +-
 pexpert/pexpert/i386/kdsoft.h                 |    30 +-
 pexpert/pexpert/i386/protos.h                 |    48 +-
 pexpert/pexpert/machine/boot.h                |    37 +-
 pexpert/pexpert/machine/protos.h              |    37 +-
 pexpert/pexpert/pe_images.h                   |    34 +-
 pexpert/pexpert/pexpert.h                     |    57 +-
 pexpert/pexpert/ppc/boot.h                    |    40 +-
 pexpert/pexpert/ppc/interrupts.h              |    30 +-
 pexpert/pexpert/ppc/powermac.h                |    33 +-
 pexpert/pexpert/ppc/protos.h                  |    45 +-
 pexpert/pexpert/protos.h                      |    32 +-
 pexpert/ppc/pe_bootargs.c                     |    30 +-
 pexpert/ppc/pe_clock_speed.c                  |    31 +-
 pexpert/ppc/pe_clock_speed_asm.s              |    30 +-
 pexpert/ppc/pe_identify_machine.c             |    40 +-
 pexpert/ppc/pe_init.c                         |    92 +-
 pexpert/ppc/pe_kprintf.c                      |    74 +-
 pexpert/ppc/pe_misc.s                         |    30 +-
 security/Makefile                             |    53 +
 security/_label.h                             |    84 +
 security/conf/MASTER                          |    66 +
 security/conf/MASTER.i386                     |    24 +
 security/conf/MASTER.ppc                      |    29 +
 security/conf/Makefile                        |    64 +
 security/conf/Makefile.i386                   |     8 +
 security/conf/Makefile.ppc                    |     8 +
 security/conf/Makefile.template               |   105 +
 .../if_faith.h => security/conf/copyright.nai |    34 +-
 security/conf/files                           |    28 +
 security/conf/files.i386                      |     1 +
 security/conf/files.ppc                       |     1 +
 security/conf/kernelversion.major             |     1 +
 security/conf/kernelversion.minor             |     1 +
 security/conf/kernelversion.variant           |     1 +
 .../include => security/conf/tools}/Makefile  |    20 +-
 security/conf/tools/doconf/Makefile           |    49 +
 security/conf/tools/doconf/doconf.csh         |   321 +
 security/conf/tools/newvers/Makefile          |    47 +
 security/conf/tools/newvers/newvers.csh       |    33 +
 security/conf/version.major                   |     1 +
 security/conf/version.minor                   |     1 +
 security/conf/version.variant                 |     1 +
 security/mac.h                                |   170 +
 security/mac_alloc.c                          |   160 +
 security/mac_alloc.h                          |    76 +
 security/mac_audit.c                          |   398 +
 security/mac_base.c                           |  2279 +++
 security/mac_data.c                           |    89 +
 security/mac_data.h                           |   145 +
 security/mac_file.c                           |   230 +
 security/mac_framework.h                      |   526 +
 security/mac_inet.c                           |   308 +
 security/mac_internal.h                       |   433 +
 security/mac_iokit.c                          |    75 +
 security/mac_label.c                          |    81 +
 security/mac_mach_internal.h                  |   111 +
 security/mac_net.c                            |   532 +
 security/mac_pipe.c                           |   249 +
 security/mac_policy.h                         |  6254 ++++++
 security/mac_port.c                           |   282 +
 security/mac_posix_sem.c                      |   190 +
 security/mac_posix_shm.c                      |   205 +
 security/mac_process.c                        |   542 +
 security/mac_socket.c                         |   644 +
 security/mac_stub.c                           |   679 +
 security/mac_system.c                         |   181 +
 security/mac_sysv_msg.c                       |   237 +
 security/mac_sysv_sem.c                       |   167 +
 security/mac_sysv_shm.c                       |   185 +
 security/mac_task.c                           |   136 +
 security/mac_vfs.c                            |  1395 ++
 security/mac_vfs_subr.c                       |   197 +
 tools/cred_dump_backtraces.c                  |    80 +
 tools/cred_dump_creds.c                       |   117 +
 tools/lockstat/Makefile                       |    10 +
 tools/lockstat/lockstat.c                     |   457 +
 tools/tests/MMTest/MMtest.c                   |   516 +
 tools/tests/MMTest/MPMMtest.c                 |   773 +
 tools/tests/MMTest/Makefile                   |    21 +
 tools/tests/affinity/Makefile                 |    18 +
 tools/tests/affinity/pool.c                   |   504 +
 tools/tests/affinity/sets.c                   |   496 +
 tools/tests/affinity/tags.c                   |   152 +
 tools/tests/xnu_quick_test/README             |   178 +
 tools/tests/xnu_quick_test/helpers/arch.c     |    24 +
 tools/tests/xnu_quick_test/helpers/launch.c   |   118 +
 tools/tests/xnu_quick_test/helpers/sleep.c    |     6 +
 tools/tests/xnu_quick_test/main.c             |   527 +
 tools/tests/xnu_quick_test/makefile           |   112 +
 tools/tests/xnu_quick_test/memory_tests.c     |   255 +
 tools/tests/xnu_quick_test/misc.c             |   366 +
 tools/tests/xnu_quick_test/sema_tests.c       |   165 +
 .../xnu_quick_test/shared_memory_tests.c      |   137 +
 tools/tests/xnu_quick_test/socket_tests.c     |   529 +
 tools/tests/xnu_quick_test/tests.c            |  5128 +++++
 tools/tests/xnu_quick_test/tests.h            |   121 +
 tools/tests/xnu_quick_test/xattr_tests.c      |   164 +
 2708 files changed, 316181 insertions(+), 143170 deletions(-)
 rename {iokit/include/bsddev/ppc => bsd/crypto/aes/i386}/Makefile (55%)
 create mode 100644 bsd/crypto/aes/i386/aes_modes.c
 create mode 100644 bsd/crypto/aes/i386/aes_x86_v2.s
 create mode 100644 bsd/crypto/aes/i386/aesopt.h
 create mode 100644 bsd/crypto/aes/i386/edefs.h
 rename {iokit/include/bsddev/i386 => bsd/crypto/aes/ppc}/Makefile (55%)
 rename bsd/crypto/aes/{ => ppc}/aescrypt.c (100%)
 rename bsd/crypto/aes/{ => ppc}/aeskey.c (100%)
 rename bsd/crypto/aes/{ => ppc}/aesopt.h (99%)
 rename bsd/crypto/aes/{ => ppc}/aestab.c (100%)
 rename bsd/crypto/aes/{ => ppc}/aestab.h (100%)
 delete mode 100644 bsd/crypto/md5.c
 delete mode 100644 bsd/crypto/md5.h
 delete mode 100644 bsd/crypto/sha1.c
 create mode 100644 bsd/dev/dtrace/blist.c
 create mode 100644 bsd/dev/dtrace/blist.h
 create mode 100644 bsd/dev/dtrace/dtrace.c
 create mode 100644 bsd/dev/dtrace/dtrace_alloc.c
 create mode 100644 bsd/dev/dtrace/dtrace_glue.c
 create mode 100644 bsd/dev/dtrace/dtrace_ptss.c
 create mode 100644 bsd/dev/dtrace/dtrace_subr.c
 create mode 100644 bsd/dev/dtrace/fasttrap.c
 create mode 100644 bsd/dev/dtrace/fbt.c
 create mode 100644 bsd/dev/dtrace/lockstat.c
 create mode 100644 bsd/dev/dtrace/profile_prvd.c
 create mode 100644 bsd/dev/dtrace/sdt.c
 create mode 100644 bsd/dev/dtrace/sdt_subr.c
 create mode 100644 bsd/dev/dtrace/systrace.c
 create mode 100644 bsd/dev/dtrace/systrace.h
 create mode 100644 bsd/dev/i386/dis_tables.c
 create mode 100644 bsd/dev/i386/dtrace_isa.c
 create mode 100644 bsd/dev/i386/dtrace_subr_x86.c
 create mode 100644 bsd/dev/i386/fasttrap_isa.c
 create mode 100644 bsd/dev/i386/fasttrap_regset.h
 create mode 100644 bsd/dev/i386/fbt_x86.c
 create mode 100644 bsd/dev/i386/instr_size.c
 create mode 100644 bsd/dev/i386/sdt_x86.c
 create mode 100644 bsd/dev/ppc/dtrace_isa.c
 create mode 100644 bsd/dev/ppc/dtrace_subr_ppc.c
 create mode 100644 bsd/dev/ppc/fasttrap_isa.c
 create mode 100644 bsd/dev/ppc/fbt_ppc.c
 create mode 100644 bsd/dev/ppc/sdt_ppc.c
 create mode 100644 bsd/hfs/hfs_btreeio.h
 create mode 100644 bsd/i386/_param.h
 create mode 100644 bsd/i386/_structs.h
 create mode 100644 bsd/i386/dis_tables.h
 create mode 100644 bsd/i386/fasttrap_isa.h
 create mode 100644 bsd/kern/imageboot.c
 delete mode 100644 bsd/kern/kern_ktrace.c
 delete mode 100644 bsd/kern/kern_lock.c
 create mode 100644 bsd/kern/kern_memorystatus.c
 create mode 100644 bsd/kern/kpi_mbuf_internal.h
 create mode 100644 bsd/kern/mcache.c
 delete mode 100644 bsd/kern/md5c.c
 create mode 100644 bsd/kern/pthread_support.c
 create mode 100644 bsd/kern/pthread_synch.c
 create mode 100644 bsd/kern/subr_sbuf.c
 create mode 100644 bsd/kern/tty_ptmx.c
 delete mode 100644 bsd/kern/tty_tb.c
 create mode 100644 bsd/libkern/strsep.c
 create mode 100644 bsd/machine/_param.h
 create mode 100644 bsd/machine/_structs.h
 create mode 100644 bsd/machine/dis_tables.h
 create mode 100644 bsd/machine/fasttrap_isa.h
 create mode 100644 bsd/man/man2/EV_SET.2
 create mode 100644 bsd/man/man2/FD_CLR.2
 create mode 100644 bsd/man/man2/FD_COPY.2
 create mode 100644 bsd/man/man2/FD_ISSET.2
 create mode 100644 bsd/man/man2/FD_SET.2
 create mode 100644 bsd/man/man2/FD_ZERO.2
 create mode 100644 bsd/man/man2/audit.2
 create mode 100644 bsd/man/man2/auditctl.2
 create mode 100644 bsd/man/man2/auditon.2
 create mode 100644 bsd/man/man2/fgetxattr.2
 create mode 100644 bsd/man/man2/fhopen.2
 create mode 100644 bsd/man/man2/flistxattr.2
 create mode 100644 bsd/man/man2/fremovexattr.2
 create mode 100644 bsd/man/man2/fsetxattr.2
 create mode 100644 bsd/man/man2/fstat64.2
 create mode 100644 bsd/man/man2/fstatfs64.2
 create mode 100644 bsd/man/man2/getaudit.2
 create mode 100644 bsd/man/man2/getauid.2
 create mode 100644 bsd/man/man2/getlcid.2
 create mode 100644 bsd/man/man2/i386_set_ldt.2
 create mode 100644 bsd/man/man2/kevent.2
 delete mode 100644 bsd/man/man2/ktrace.2
 create mode 100644 bsd/man/man2/lstat64.2
 create mode 100644 bsd/man/man2/nfsclnt.2
 create mode 100644 bsd/man/man2/posix_spawn.2
 create mode 100644 bsd/man/man2/sendfile.2
 create mode 100644 bsd/man/man2/setaudit.2
 create mode 100644 bsd/man/man2/setauid.2
 create mode 100644 bsd/man/man2/setlcid.2
 create mode 100644 bsd/man/man2/stat64.2
 create mode 100644 bsd/man/man2/statfs64.2
 create mode 100644 bsd/man/man3/Makefile
 create mode 100644 bsd/man/man3/posix_spawn_file_actions_addclose.3
 create mode 100644 bsd/man/man3/posix_spawn_file_actions_init.3
 create mode 100644 bsd/man/man3/posix_spawnattr_init.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setbinpref_np.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setflags.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setpgroup.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setsigdefault.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setsigmask.3
 create mode 100644 bsd/man/man3/posix_spawnattr_setspecialport_np.3
 create mode 100644 bsd/man/man3/queue.3
 create mode 100644 bsd/man/man4/aio.4
 create mode 100644 bsd/man/man8/Makefile
 rename bsd/man/{man2/sigreturn.2 => man8/sticky.8} (53%)
 create mode 100644 bsd/man/man9/copyin.9
 create mode 100644 bsd/man/man9/copyinstr.9
 create mode 100644 bsd/man/man9/copyout.9
 create mode 100644 bsd/man/man9/copystr.9
 create mode 100644 bsd/man/man9/fubyte.9
 create mode 100644 bsd/man/man9/fuibyte.9
 create mode 100644 bsd/man/man9/fuiword.9
 create mode 100644 bsd/man/man9/fulong.9
 create mode 100644 bsd/man/man9/fuulong.9
 create mode 100644 bsd/man/man9/fuword.9
 create mode 100644 bsd/man/man9/subyte.9
 create mode 100644 bsd/man/man9/suibyte.9
 create mode 100644 bsd/man/man9/suiword.9
 create mode 100644 bsd/man/man9/sulong.9
 create mode 100644 bsd/man/man9/suulong.9
 create mode 100644 bsd/man/man9/suword.9
 delete mode 100644 bsd/miscfs/volfs/volfs.h
 delete mode 100644 bsd/miscfs/volfs/volfs_vfsops.c
 delete mode 100644 bsd/miscfs/volfs/volfs_vnops.c
 create mode 100644 bsd/net/ether_if_module.h
 delete mode 100644 bsd/net/if_faith.c
 delete mode 100644 bsd/netinet/in_bootp.c
 create mode 100644 bsd/netinet/in_dhcp.c
 create mode 100644 bsd/netinet/in_dhcp.h
 create mode 100644 bsd/nfs/nfs4_subs.c
 create mode 100644 bsd/nfs/nfs4_vnops.c
 create mode 100644 bsd/nfs/nfs_gss.c
 create mode 100644 bsd/nfs/nfs_gss.h
 delete mode 100644 bsd/nfs/nfsrtt.h
 create mode 100644 bsd/ppc/_param.h
 create mode 100644 bsd/ppc/_structs.h
 create mode 100644 bsd/ppc/decodePPC.h
 create mode 100644 bsd/ppc/fasttrap_isa.h
 create mode 100644 bsd/sys/_select.h
 create mode 100644 bsd/sys/_structs.h
 create mode 100644 bsd/sys/codesign.h
 create mode 100644 bsd/sys/dis_tables.h
 create mode 100644 bsd/sys/dtrace.h
 create mode 100644 bsd/sys/dtrace_glue.h
 create mode 100644 bsd/sys/dtrace_impl.h
 create mode 100644 bsd/sys/dtrace_ptss.h
 create mode 100644 bsd/sys/fasttrap.h
 create mode 100644 bsd/sys/fasttrap_impl.h
 create mode 100644 bsd/sys/fasttrap_isa.h
 create mode 100644 bsd/sys/fbt.h
 create mode 100644 bsd/sys/fslog.h
 create mode 100644 bsd/sys/imageboot.h
 create mode 100644 bsd/sys/kern_memorystatus.h
 delete mode 100644 bsd/sys/ktrace.h
 create mode 100644 bsd/sys/lctx.h
 create mode 100644 bsd/sys/lockstat.h
 create mode 100644 bsd/sys/mcache.h
 create mode 100644 bsd/sys/posix_sem.h
 create mode 100644 bsd/sys/posix_shm.h
 create mode 100644 bsd/sys/pthread_internal.h
 delete mode 100644 bsd/sys/ptrace_internal.h
 create mode 100644 bsd/sys/sbuf.h
 create mode 100644 bsd/sys/sdt.h
 create mode 100644 bsd/sys/sdt_impl.h
 create mode 100644 bsd/sys/spawn.h
 create mode 100644 bsd/sys/spawn_internal.h
 delete mode 100644 bsd/sys/version.h
 create mode 100644 bsd/vfs/vfs_fslog.c
 create mode 100644 config/DtraceIgnored.symbols
 create mode 100644 config/MACFramework.exports
 create mode 100644 config/MACFramework.i386.exports
 create mode 100644 config/MACFramework.ppc.exports
 create mode 100644 config/System.kext/PlugIns/MACFramework.kext/Info.plist
 create mode 100755 config/compress-man-pages.pl
 create mode 100644 iokit/IOKit/IODMAController.h
 create mode 100644 iokit/IOKit/IODMAEventSource.h
 create mode 100644 iokit/IOKit/IOInterleavedMemoryDescriptor.h
 create mode 100644 iokit/IOKit/IOLocksPrivate.h
 create mode 100644 iokit/IOKit/IOSharedDataQueue.h
 delete mode 100644 iokit/IOKit/pwr_mgt/IOPMchangeNoteList.h
 delete mode 100644 iokit/IOKit/pwr_mgt/IOPMpmChild.h
 create mode 100644 iokit/Kernel/IODMAController.cpp
 create mode 100644 iokit/Kernel/IODMAEventSource.cpp
 create mode 100644 iokit/Kernel/IOInterleavedMemoryDescriptor.cpp
 delete mode 100644 iokit/Kernel/IOPMWorkArbiter.cpp
 delete mode 100644 iokit/Kernel/IOPMWorkArbiter.h
 delete mode 100644 iokit/Kernel/IOPMchangeNoteList.cpp
 delete mode 100644 iokit/Kernel/IOPMpmChild.cpp
 create mode 100644 iokit/Kernel/IOPMrootDomainInternal.h
 create mode 100644 iokit/Kernel/IOServicePMPrivate.h
 create mode 100644 iokit/Kernel/IOSharedDataQueue.cpp
 delete mode 100644 iokit/include/architecture/i386/kernBootStruct.h
 delete mode 100644 iokit/include/assert.h
 delete mode 100644 iokit/include/bsddev/EventShmemLock.h
 delete mode 100644 iokit/include/bsddev/Makefile
 delete mode 100644 iokit/include/bsddev/ev_keymap.h
 delete mode 100644 iokit/include/bsddev/ev_types.h
 delete mode 100644 iokit/include/bsddev/event.h
 delete mode 100644 iokit/include/bsddev/evio.h
 delete mode 100644 iokit/include/bsddev/evsio.h
 delete mode 100644 iokit/include/bsddev/i386/EventShmemLock.h
 delete mode 100644 iokit/include/bsddev/i386/event.h
 delete mode 100644 iokit/include/bsddev/i386/evio.h
 delete mode 100644 iokit/include/bsddev/i386/evsio.h
 delete mode 100644 iokit/include/bsddev/machine/EventShmemLock.h
 delete mode 100644 iokit/include/bsddev/machine/Makefile
 delete mode 100644 iokit/include/bsddev/machine/event.h
 delete mode 100644 iokit/include/bsddev/machine/evio.h
 delete mode 100644 iokit/include/bsddev/machine/evsio.h
 delete mode 100644 iokit/include/bsddev/ppc/EventShmemLock.h
 delete mode 100644 iokit/include/bsddev/ppc/event.h
 delete mode 100644 iokit/include/bsddev/ppc/evio.h
 delete mode 100644 iokit/include/bsddev/ppc/evsio.h
 delete mode 100644 iokit/include/drivers/event_status_driver.h
 delete mode 100644 iokit/include/mach/mach.h
 create mode 100644 libkern/crypto/md5.c
 create mode 100644 libkern/crypto/sha1.c
 delete mode 100644 libkern/i386/OSAtomic.c
 create mode 100644 libkern/libkern/_OSByteOrder.h
 rename {iokit/include/drivers => libkern/libkern/crypto}/Makefile (67%)
 create mode 100644 libkern/libkern/crypto/md5.h
 create mode 100644 libkern/libkern/crypto/sha1.h
 create mode 100644 libkern/libkern/i386/_OSByteOrder.h
 create mode 100644 libkern/libkern/zlib.h
 rename {bsd/net => libkern}/zlib.c (93%)
 delete mode 100644 libsa/kmod.cpp
 create mode 100644 libsyscall/BSDmakefile
 create mode 100644 libsyscall/GNUmakefile
 create mode 100644 libsyscall/Makefile
 create mode 100644 libsyscall/Makefile.inc
 create mode 100644 libsyscall/Makefile.xbs
 create mode 100755 libsyscall/create-syscalls.pl
 create mode 100644 libsyscall/custom/SYS.h
 create mode 100644 libsyscall/custom/__fork.s
 create mode 100644 libsyscall/custom/__getpid.s
 create mode 100644 libsyscall/custom/__gettimeofday.s
 create mode 100644 libsyscall/custom/__lseek.s
 create mode 100644 libsyscall/custom/__pipe.s
 create mode 100644 libsyscall/custom/__ptrace.s
 create mode 100644 libsyscall/custom/__sigaltstack.s
 create mode 100644 libsyscall/custom/__sigreturn.s
 create mode 100644 libsyscall/custom/__syscall.s
 create mode 100644 libsyscall/custom/__vfork.s
 create mode 100644 libsyscall/custom/custom.s
 create mode 100644 libsyscall/fixdups.ed
 create mode 100644 libsyscall/include/Makefile.inc
 create mode 100644 libsyscall/include/processor_facilities.h
 create mode 100644 libsyscall/mach/Makefile.inc
 create mode 100644 libsyscall/mach/bootstrap_ports.c
 create mode 100644 libsyscall/mach/brk.2
 create mode 100644 libsyscall/mach/clock.defs
 create mode 100644 libsyscall/mach/clock_priv.defs
 create mode 100644 libsyscall/mach/clock_reply.defs
 create mode 100644 libsyscall/mach/clock_sleep.c
 create mode 100755 libsyscall/mach/err_iokit.sub
 create mode 100644 libsyscall/mach/err_ipc.sub
 create mode 100644 libsyscall/mach/err_kern.sub
 create mode 100644 libsyscall/mach/err_mach_ipc.sub
 create mode 100644 libsyscall/mach/err_server.sub
 create mode 100644 libsyscall/mach/err_us.sub
 create mode 100644 libsyscall/mach/error_codes.c
 create mode 100644 libsyscall/mach/errorlib.h
 create mode 100644 libsyscall/mach/exc.defs
 create mode 100644 libsyscall/mach/exc_catcher.c
 create mode 100644 libsyscall/mach/exc_catcher_state.c
 create mode 100644 libsyscall/mach/exc_catcher_state_identity.c
 create mode 100644 libsyscall/mach/externs.h
 create mode 100644 libsyscall/mach/fprintf_stderr.c
 create mode 100644 libsyscall/mach/headers/Makefile.inc
 create mode 100644 libsyscall/mach/headers/errorlib.h
 create mode 100644 libsyscall/mach/headers/mach.h
 create mode 100644 libsyscall/mach/headers/mach_error.h
 create mode 100644 libsyscall/mach/headers/mach_init.h
 create mode 100644 libsyscall/mach/headers/mach_interface.h
 create mode 100644 libsyscall/mach/headers/port_obj.h
 create mode 100644 libsyscall/mach/headers/sync.h
 create mode 100644 libsyscall/mach/headers/task.h
 create mode 100644 libsyscall/mach/headers/thread_act.h
 create mode 100644 libsyscall/mach/headers/vm_task.h
 create mode 100644 libsyscall/mach/host_priv.defs
 create mode 100644 libsyscall/mach/host_security.defs
 create mode 100644 libsyscall/mach/i386/Makefile.inc
 create mode 100644 libsyscall/mach/i386/mach_absolute_time.S
 create mode 100644 libsyscall/mach/ledger.defs
 create mode 100644 libsyscall/mach/lock_set.defs
 create mode 100644 libsyscall/mach/mach_error.c
 create mode 100644 libsyscall/mach/mach_error_string.c
 create mode 100644 libsyscall/mach/mach_host.defs
 create mode 100644 libsyscall/mach/mach_init.c
 create mode 100644 libsyscall/mach/mach_init_libSystem.c
 create mode 100644 libsyscall/mach/mach_init_ports.c
 create mode 100644 libsyscall/mach/mach_msg.c
 create mode 100644 libsyscall/mach/mach_port.defs
 create mode 100644 libsyscall/mach/mach_traps.s
 create mode 100644 libsyscall/mach/mach_vm.defs
 create mode 100644 libsyscall/mach/mig_allocate.c
 create mode 100644 libsyscall/mach/mig_deallocate.c
 create mode 100644 libsyscall/mach/mig_reply_setup.c
 create mode 100644 libsyscall/mach/mig_strncpy.c
 create mode 100644 libsyscall/mach/ms_thread_switch.c
 create mode 100644 libsyscall/mach/notify.defs
 create mode 100644 libsyscall/mach/panic.c
 create mode 100644 libsyscall/mach/port_obj.c
 create mode 100644 libsyscall/mach/ppc/Makefile.inc
 create mode 100644 libsyscall/mach/ppc/mach_absolute_time.s
 create mode 100644 libsyscall/mach/ppc64/Makefile.inc
 create mode 100644 libsyscall/mach/processor.defs
 create mode 100644 libsyscall/mach/processor_set.defs
 create mode 100644 libsyscall/mach/sbrk.c
 create mode 100644 libsyscall/mach/semaphore.c
 create mode 100644 libsyscall/mach/servers/Makefile.inc
 create mode 100644 libsyscall/mach/servers/key_defs.h
 create mode 100644 libsyscall/mach/servers/ls_defs.h
 create mode 100644 libsyscall/mach/servers/netname.defs
 create mode 100644 libsyscall/mach/servers/netname_defs.h
 create mode 100644 libsyscall/mach/servers/nm_defs.h
 create mode 100644 libsyscall/mach/slot_name.c
 create mode 100644 libsyscall/mach/task.defs
 create mode 100644 libsyscall/mach/thread_act.defs
 create mode 100644 libsyscall/mach/vm_map.defs
 create mode 100644 libsyscall/mach/x86_64/Makefile.inc
 create mode 100644 libsyscall/mach/x86_64/mach_absolute_time.S
 create mode 100644 osfmk/chud/chud_dtrace.h
 create mode 100644 osfmk/chud/chud_thread.h
 create mode 100644 osfmk/gssd/Makefile
 create mode 100644 osfmk/gssd/gssd_mach.defs
 create mode 100644 osfmk/gssd/gssd_mach_types.h
 delete mode 100644 osfmk/i386/AT386/autoconf.c
 delete mode 100644 osfmk/i386/AT386/config.h
 delete mode 100644 osfmk/i386/AT386/cram.h
 delete mode 100644 osfmk/i386/AT386/himem.c
 delete mode 100644 osfmk/i386/AT386/himem.h
 delete mode 100644 osfmk/i386/AT386/machdep.mk
 delete mode 100644 osfmk/i386/AT386/misc_protos.h
 delete mode 100644 osfmk/i386/AT386/physmem_entries.h
 rename osfmk/i386/commpage/{bcopy_sse3.s => bcopy_sse2.s} (91%)
 rename osfmk/i386/commpage/{bcopy_sse4.s => bcopy_sse3x.s} (94%)
 rename osfmk/i386/commpage/{bcopy_sse4_64.s => bcopy_sse3x_64.s} (94%)
 rename osfmk/i386/commpage/{bzero_sse3.s => bzero_sse2.s} (72%)
 rename osfmk/i386/commpage/{bzero_sse3_64.s => bzero_sse2_64.s} (74%)
 rename osfmk/i386/commpage/{longcopy_sse4.s => longcopy_sse3x.s} (84%)
 rename osfmk/i386/commpage/{longcopy_sse4_64.s => longcopy_sse3x_64.s} (83%)
 rename osfmk/i386/commpage/{memset_pattern_sse3.s => memset_pattern_sse2.s} (79%)
 rename osfmk/i386/commpage/{memset_pattern_sse3_64.s => memset_pattern_sse2_64.s} (80%)
 create mode 100644 osfmk/i386/cpu_affinity.h
 create mode 100644 osfmk/i386/cpu_topology.c
 create mode 100644 osfmk/i386/cpu_topology.h
 create mode 100644 osfmk/i386/etimer.c
 delete mode 100644 osfmk/i386/hi_res_clock.h
 delete mode 100644 osfmk/i386/hi_res_clock_map.c
 create mode 100644 osfmk/i386/hw_defs.h
 delete mode 100644 osfmk/i386/intel_read_fault.h
 delete mode 100644 osfmk/i386/io_emulate.c
 delete mode 100644 osfmk/i386/io_emulate.h
 delete mode 100644 osfmk/i386/iopb.c
 delete mode 100644 osfmk/i386/iopb_entries.h
 delete mode 100644 osfmk/i386/net_filter.c
 delete mode 100644 osfmk/i386/ntoh.h
 delete mode 100644 osfmk/i386/ntoh.s
 delete mode 100644 osfmk/i386/pic.h
 delete mode 100644 osfmk/i386/pit.h
 delete mode 100644 osfmk/i386/read_fault.c
 create mode 100644 osfmk/i386/serial_io.h
 create mode 100644 osfmk/i386/vmx.h
 create mode 100644 osfmk/i386/vmx/vmx_asm.h
 create mode 100644 osfmk/i386/vmx/vmx_cpu.c
 create mode 100644 osfmk/i386/vmx/vmx_cpu.h
 create mode 100644 osfmk/i386/vmx/vmx_shims.c
 create mode 100644 osfmk/i386/vmx/vmx_shims.h
 create mode 100644 osfmk/ipc/ipc_labelh.c
 create mode 100644 osfmk/ipc/ipc_labelh.h
 create mode 100644 osfmk/kdp/kdp_callout.h
 create mode 100644 osfmk/kern/affinity.c
 create mode 100644 osfmk/kern/affinity.h
 delete mode 100644 osfmk/kern/profile.c
 delete mode 100644 osfmk/kern/profile.h
 create mode 100644 osfmk/kern/security.c
 create mode 100644 osfmk/kern/symbols.c
 delete mode 100644 osfmk/libsa/i386/stdarg.h
 delete mode 100644 osfmk/libsa/i386/va_list.h
 delete mode 100644 osfmk/libsa/machine/stdarg.h
 delete mode 100644 osfmk/libsa/machine/va_list.h
 delete mode 100644 osfmk/libsa/ppc/stdarg.h
 delete mode 100644 osfmk/libsa/ppc/va_list.h
 delete mode 100644 osfmk/libsa/stdarg.h
 delete mode 100644 osfmk/libsa/va_list.h
 create mode 100644 osfmk/lockd/Makefile
 create mode 100644 osfmk/lockd/lockd_mach.defs
 create mode 100644 osfmk/lockd/lockd_mach_types.h
 create mode 100644 osfmk/mach/i386/_structs.h
 create mode 100644 osfmk/mach/i386/_types.h
 create mode 100644 osfmk/mach/i386/sdt_isa.h
 create mode 100644 osfmk/mach/mach_exc.defs
 create mode 100644 osfmk/mach/machine/sdt.h
 create mode 100644 osfmk/mach/machine/sdt_isa.h
 create mode 100644 osfmk/mach/ppc/_structs.h
 create mode 100644 osfmk/mach/ppc/sdt_isa.h
 create mode 100644 osfmk/mach/sdt.h
 create mode 100644 osfmk/mach/security.defs
 create mode 100644 osfmk/mach/shared_region.h
 create mode 100644 osfmk/mach/task_access.defs
 create mode 100644 osfmk/machine/cpu_affinity.h
 create mode 100644 osfmk/ppc/cpu_affinity.h
 rename osfmk/{kern => ppc}/etimer.c (57%)
 rename osfmk/{kern => ppc}/pms.c (86%)
 delete mode 100644 osfmk/ppc/pms.h
 delete mode 100644 osfmk/vm/task_working_set.c
 delete mode 100644 osfmk/vm/task_working_set.h
 create mode 100644 osfmk/vm/vm_purgeable.c
 create mode 100644 osfmk/vm/vm_purgeable_internal.h
 delete mode 100644 osfmk/vm/vm_shared_memory_server.c
 delete mode 100644 osfmk/vm/vm_shared_memory_server.h
 create mode 100644 osfmk/vm/vm_shared_region.c
 create mode 100644 osfmk/vm/vm_shared_region.h
 delete mode 100644 pexpert/i386/fakePPCDeviceTree.c
 delete mode 100644 pexpert/i386/fakePPCDeviceTree.h
 delete mode 100644 pexpert/i386/fakePPCStructs.h
 create mode 100644 pexpert/pexpert/AppleBoot.h
 create mode 100644 pexpert/pexpert/Clut.h
 create mode 100644 security/Makefile
 create mode 100644 security/_label.h
 create mode 100644 security/conf/MASTER
 create mode 100644 security/conf/MASTER.i386
 create mode 100644 security/conf/MASTER.ppc
 create mode 100644 security/conf/Makefile
 create mode 100644 security/conf/Makefile.i386
 create mode 100644 security/conf/Makefile.ppc
 create mode 100644 security/conf/Makefile.template
 rename bsd/net/if_faith.h => security/conf/copyright.nai (60%)
 create mode 100644 security/conf/files
 create mode 100644 security/conf/files.i386
 create mode 100644 security/conf/files.ppc
 create mode 100644 security/conf/kernelversion.major
 create mode 100644 security/conf/kernelversion.minor
 create mode 100644 security/conf/kernelversion.variant
 rename {iokit/include => security/conf/tools}/Makefile (54%)
 create mode 100644 security/conf/tools/doconf/Makefile
 create mode 100644 security/conf/tools/doconf/doconf.csh
 create mode 100644 security/conf/tools/newvers/Makefile
 create mode 100644 security/conf/tools/newvers/newvers.csh
 create mode 100644 security/conf/version.major
 create mode 100644 security/conf/version.minor
 create mode 100644 security/conf/version.variant
 create mode 100644 security/mac.h
 create mode 100644 security/mac_alloc.c
 create mode 100644 security/mac_alloc.h
 create mode 100644 security/mac_audit.c
 create mode 100644 security/mac_base.c
 create mode 100644 security/mac_data.c
 create mode 100644 security/mac_data.h
 create mode 100644 security/mac_file.c
 create mode 100644 security/mac_framework.h
 create mode 100644 security/mac_inet.c
 create mode 100644 security/mac_internal.h
 create mode 100644 security/mac_iokit.c
 create mode 100644 security/mac_label.c
 create mode 100644 security/mac_mach_internal.h
 create mode 100644 security/mac_net.c
 create mode 100644 security/mac_pipe.c
 create mode 100644 security/mac_policy.h
 create mode 100644 security/mac_port.c
 create mode 100644 security/mac_posix_sem.c
 create mode 100644 security/mac_posix_shm.c
 create mode 100644 security/mac_process.c
 create mode 100644 security/mac_socket.c
 create mode 100644 security/mac_stub.c
 create mode 100644 security/mac_system.c
 create mode 100644 security/mac_sysv_msg.c
 create mode 100644 security/mac_sysv_sem.c
 create mode 100644 security/mac_sysv_shm.c
 create mode 100644 security/mac_task.c
 create mode 100644 security/mac_vfs.c
 create mode 100644 security/mac_vfs_subr.c
 create mode 100644 tools/cred_dump_backtraces.c
 create mode 100644 tools/cred_dump_creds.c
 create mode 100644 tools/lockstat/Makefile
 create mode 100644 tools/lockstat/lockstat.c
 create mode 100644 tools/tests/MMTest/MMtest.c
 create mode 100644 tools/tests/MMTest/MPMMtest.c
 create mode 100644 tools/tests/MMTest/Makefile
 create mode 100644 tools/tests/affinity/Makefile
 create mode 100644 tools/tests/affinity/pool.c
 create mode 100644 tools/tests/affinity/sets.c
 create mode 100644 tools/tests/affinity/tags.c
 create mode 100644 tools/tests/xnu_quick_test/README
 create mode 100644 tools/tests/xnu_quick_test/helpers/arch.c
 create mode 100644 tools/tests/xnu_quick_test/helpers/launch.c
 create mode 100644 tools/tests/xnu_quick_test/helpers/sleep.c
 create mode 100644 tools/tests/xnu_quick_test/main.c
 create mode 100644 tools/tests/xnu_quick_test/makefile
 create mode 100644 tools/tests/xnu_quick_test/memory_tests.c
 create mode 100644 tools/tests/xnu_quick_test/misc.c
 create mode 100644 tools/tests/xnu_quick_test/sema_tests.c
 create mode 100644 tools/tests/xnu_quick_test/shared_memory_tests.c
 create mode 100644 tools/tests/xnu_quick_test/socket_tests.c
 create mode 100644 tools/tests/xnu_quick_test/tests.c
 create mode 100644 tools/tests/xnu_quick_test/tests.h
 create mode 100644 tools/tests/xnu_quick_test/xattr_tests.c

diff --git a/APPLE_LICENSE b/APPLE_LICENSE
index a0a84169d..fe81a60ca 100644
--- a/APPLE_LICENSE
+++ b/APPLE_LICENSE
@@ -1,26 +1,19 @@
 APPLE PUBLIC SOURCE LICENSE
-Version 1.1 - April 19,1999
+Version 2.0 - August 6, 2003
 
 Please read this License carefully before downloading this software.
-By downloading and using this software, you are agreeing to be bound
-by the terms of this License.  If you do not or cannot agree to the
-terms of this License, please do not download or use the software.
-
-1. General; Definitions.  This License applies to any program or other
-work which Apple Computer, Inc. ("Apple") publicly announces as
-subject to this Apple Public Source License and which contains a
-notice placed by Apple identifying such program or work as "Original
-Code" and stating that it is subject to the terms of this Apple Public
-Source License version 1.1 (or subsequent version thereof), as it may
-be revised from time to time by Apple ("License").  As used in this
-License:
-
-1.1 "Affected Original Code" means only those specific portions of
-Original Code that allegedly infringe upon any party's intellectual
-property rights or are otherwise the subject of a claim of
-infringement.
-
-1.2 "Applicable Patent Rights" mean: (a) in the case where Apple is
+By downloading or using this software, you are agreeing to be bound by
+the terms of this License. If you do not or cannot agree to the terms
+of this License, please do not download or use the software.
+
+1. General; Definitions. This License applies to any program or other
+work which Apple Computer, Inc. ("Apple") makes publicly available and
+which contains a notice placed by Apple identifying such program or
+work as "Original Code" and stating that it is subject to the terms of
+this Apple Public Source License version 2.0 ("License"). As used in
+this License:
+
+1.1 "Applicable Patent Rights" mean: (a) in the case where Apple is
 the grantor of rights, (i) claims of patents that are now or hereafter
 acquired, owned by or assigned to Apple and (ii) that cover subject
 matter contained in the Original Code, but only to the extent
@@ -30,22 +23,27 @@ rights, (i) claims of patents that are now or hereafter acquired,
 owned by or assigned to You and (ii) that cover subject matter in Your
 Modifications, taken alone or in combination with Original Code.
 
+1.2 "Contributor" means any person or entity that creates or
+contributes to the creation of Modifications.
+
 1.3 "Covered Code" means the Original Code, Modifications, the
 combination of Original Code and any Modifications, and/or any
 respective portions thereof.
 
-1.4 "Deploy" means to use, sublicense or distribute Covered Code other
-than for Your internal research and development (R&D), and includes
-without limitation, any and all internal use or distribution of
-Covered Code within Your business or organization except for R&D use,
-as well as direct or indirect sublicensing or distribution of Covered
-Code by You to any third party in any form or manner.
+1.4 "Externally Deploy" means: (a) to sublicense, distribute or
+otherwise make Covered Code available, directly or indirectly, to
+anyone other than You; and/or (b) to use Covered Code, alone or as
+part of a Larger Work, in any way to provide a service, including but
+not limited to delivery of content, through electronic communication
+with a client other than You.
 
 1.5 "Larger Work" means a work which combines Covered Code or portions
 thereof with code not governed by the terms of this License.
 
 1.6 "Modifications" mean any addition to, deletion from, and/or change
-to, the substance and/or structure of Covered Code.  When code is
+to, the substance and/or structure of the Original Code, any previous
+Modifications, the combination of Original Code and any previous
+Modifications, and/or any respective portions thereof. When code is
 released as a series of files, a Modification is: (a) any addition to
 or deletion from the contents of a file containing Covered Code;
 and/or (b) any new file or other representation of computer program
@@ -66,7 +64,7 @@ scripts used to control compilation and installation of an executable
 (object code).
 
 1.9 "You" or "Your" means an individual or a legal entity exercising
-rights under this License.  For legal entities, "You" or "Your"
+rights under this License. For legal entities, "You" or "Your"
 includes any entity which controls, is controlled by, or is under
 common control with, You, where "control" means (a) the power, direct
 or indirect, to cause the direction or management of such entity,
@@ -74,258 +72,253 @@ whether by contract or otherwise, or (b) ownership of fifty percent
 (50%) or more of the outstanding shares or beneficial ownership of
 such entity.
 
-2. Permitted Uses; Conditions & Restrictions.  Subject to the terms
+2. Permitted Uses; Conditions & Restrictions. Subject to the terms
 and conditions of this License, Apple hereby grants You, effective on
 the date You accept this License and download the Original Code, a
-world-wide, royalty-free, non- exclusive license, to the extent of
+world-wide, royalty-free, non-exclusive license, to the extent of
 Apple's Applicable Patent Rights and copyrights covering the Original
 Code, to do the following:
 
-2.1 You may use, copy, modify and distribute Original Code, with or
-without Modifications, solely for Your internal research and
-development, provided that You must in each instance:
-
-(a) retain and reproduce in all copies of Original Code the copyright
-and other proprietary notices and disclaimers of Apple as they appear
-in the Original Code, and keep intact all notices in the Original Code
-that refer to this License;
-
-(b) include a copy of this License with every copy of Source Code of
-Covered Code and documentation You distribute, and You may not offer
-or impose any terms on such Source Code that alter or restrict this
-License or the recipients' rights hereunder, except as permitted under
-Section 6; and
-
-(c) completely and accurately document all Modifications that you have
-made and the date of each such Modification, designate the version of
-the Original Code you used, prominently include a file carrying such
-information with the Modifications, and duplicate the notice in
-Exhibit A in each file of the Source Code of all such Modifications.
-
-2.2 You may Deploy Covered Code, provided that You must in each
-  instance:
-
-(a) satisfy all the conditions of Section 2.1 with respect to the
-Source Code of the Covered Code;
-
-(b) make all Your Deployed Modifications publicly available in Source
-Code form via electronic distribution (e.g. download from a web site)
-under the terms of this License and subject to the license grants set
-forth in Section 3 below, and any additional terms You may choose to
-offer under Section 6.  You must continue to make the Source Code of
-Your Deployed Modifications available for as long as you Deploy the
-Covered Code or twelve (12) months from the date of initial
-Deployment, whichever is longer;
-
-(c) if You Deploy Covered Code containing Modifications made by You,
-inform others of how to obtain those Modifications by filling out and
-submitting the information found at
-http://www.apple.com/publicsource/modifications.html, if available;
-and
-
-(d) if You Deploy Covered Code in object code, executable form only,
-include a prominent notice, in the code itself as well as in related
-documentation, stating that Source Code of the Covered Code is
-available under the terms of this License with information on how and
-where to obtain such Source Code.
-
-3. Your Grants.  In consideration of, and as a condition to, the
-licenses granted to You under this License:
-
-(a) You hereby grant to Apple and all third parties a non-exclusive,
-royalty-free license, under Your Applicable Patent Rights and other
-intellectual property rights owned or controlled by You, to use,
-reproduce, modify, distribute and Deploy Your Modifications of the
-same scope and extent as Apple's licenses under Sections 2.1 and 2.2;
-and
-
-(b) You hereby grant to Apple and its subsidiaries a non-exclusive,
-worldwide, royalty-free, perpetual and irrevocable license, under Your
-Applicable Patent Rights and other intellectual property rights owned
-or controlled by You, to use, reproduce, execute, compile, display,
-perform, modify or have modified (for Apple and/or its subsidiaries),
-sublicense and distribute Your Modifications, in any form, through
-multiple tiers of distribution.
-
-4. Larger Works.  You may create a Larger Work by combining Covered
+2.1 Unmodified Code. You may use, reproduce, display, perform,
+internally distribute within Your organization, and Externally Deploy
+verbatim, unmodified copies of the Original Code, for commercial or
+non-commercial purposes, provided that in each instance:
+
+(a) You must retain and reproduce in all copies of Original Code the
+copyright and other proprietary notices and disclaimers of Apple as
+they appear in the Original Code, and keep intact all notices in the
+Original Code that refer to this License; and
+
+(b) You must include a copy of this License with every copy of Source
+Code of Covered Code and documentation You distribute or Externally
+Deploy, and You may not offer or impose any terms on such Source Code
+that alter or restrict this License or the recipients' rights
+hereunder, except as permitted under Section 6.
+
+2.2 Modified Code. You may modify Covered Code and use, reproduce,
+display, perform, internally distribute within Your organization, and
+Externally Deploy Your Modifications and Covered Code, for commercial
+or non-commercial purposes, provided that in each instance You also
+meet all of these conditions:
+
+(a) You must satisfy all the conditions of Section 2.1 with respect to
+the Source Code of the Covered Code;
+
+(b) You must duplicate, to the extent it does not already exist, the
+notice in Exhibit A in each file of the Source Code of all Your
+Modifications, and cause the modified files to carry prominent notices
+stating that You changed the files and the date of any change; and
+
+(c) If You Externally Deploy Your Modifications, You must make
+Source Code of all Your Externally Deployed Modifications either
+available to those to whom You have Externally Deployed Your
+Modifications, or publicly available. Source Code of Your Externally
+Deployed Modifications must be released under the terms set forth in
+this License, including the license grants set forth in Section 3
+below, for as long as you Externally Deploy the Covered Code or twelve
+(12) months from the date of initial External Deployment, whichever is
+longer. You should preferably distribute the Source Code of Your
+Externally Deployed Modifications electronically (e.g. download from a
+web site).
+
+2.3 Distribution of Executable Versions. In addition, if You
+Externally Deploy Covered Code (Original Code and/or Modifications) in
+object code, executable form only, You must include a prominent
+notice, in the code itself as well as in related documentation,
+stating that Source Code of the Covered Code is available under the
+terms of this License with information on how and where to obtain such
+Source Code.
+
+2.4 Third Party Rights. You expressly acknowledge and agree that
+although Apple and each Contributor grants the licenses to their
+respective portions of the Covered Code set forth herein, no
+assurances are provided by Apple or any Contributor that the Covered
+Code does not infringe the patent or other intellectual property
+rights of any other entity. Apple and each Contributor disclaim any
+liability to You for claims brought by any other entity based on
+infringement of intellectual property rights or otherwise. As a
+condition to exercising the rights and licenses granted hereunder, You
+hereby assume sole responsibility to secure any other intellectual
+property rights needed, if any. For example, if a third party patent
+license is required to allow You to distribute the Covered Code, it is
+Your responsibility to acquire that license before distributing the
+Covered Code.
+
+3. Your Grants. In consideration of, and as a condition to, the
+licenses granted to You under this License, You hereby grant to any
+person or entity receiving or distributing Covered Code under this
+License a non-exclusive, royalty-free, perpetual, irrevocable license,
+under Your Applicable Patent Rights and other intellectual property
+rights (other than patent) owned or controlled by You, to use,
+reproduce, display, perform, modify, sublicense, distribute and
+Externally Deploy Your Modifications of the same scope and extent as
+Apple's licenses under Sections 2.1 and 2.2 above.
+
+4. Larger Works. You may create a Larger Work by combining Covered
 Code with other code not governed by the terms of this License and
-distribute the Larger Work as a single product.  In each such
-instance, You must make sure the requirements of this License are
-fulfilled for the Covered Code or any portion thereof.
+distribute the Larger Work as a single product. In each such instance,
+You must make sure the requirements of this License are fulfilled for
+the Covered Code or any portion thereof.
 
-5. Limitations on Patent License.  Except as expressly stated in
+5. Limitations on Patent License. Except as expressly stated in
 Section 2, no other patent rights, express or implied, are granted by
-Apple herein.  Modifications and/or Larger Works may require
-additional patent licenses from Apple which Apple may grant in its
-sole discretion.
-
-6. Additional Terms.  You may choose to offer, and to charge a fee
-for, warranty, support, indemnity or liability obligations and/or
-other rights consistent with the scope of the license granted herein
-("Additional Terms") to one or more recipients of Covered
-Code. However, You may do so only on Your own behalf and as Your sole
-responsibility, and not on behalf of Apple. You must obtain the
-recipient's agreement that any such Additional Terms are offered by
-You alone, and You hereby agree to indemnify, defend and hold Apple
-harmless for any liability incurred by or claims asserted against
-Apple by reason of any such Additional Terms.
-
-7. Versions of the License.  Apple may publish revised and/or new
-versions of this License from time to time.  Each version will be
-given a distinguishing version number.  Once Original Code has been
-published under a particular version of this License, You may continue
-to use it under the terms of that version. You may also choose to use
-such Original Code under the terms of any subsequent version of this
-License published by Apple.  No one other than Apple has the right to
+Apple herein. Modifications and/or Larger Works may require additional
+patent licenses from Apple which Apple may grant in its sole
+discretion.
+
+6. Additional Terms. You may choose to offer, and to charge a fee for,
+warranty, support, indemnity or liability obligations and/or other
+rights consistent with the scope of the license granted herein
+("Additional Terms") to one or more recipients of Covered Code.
+However, You may do so only on Your own behalf and as Your sole
+responsibility, and not on behalf of Apple or any Contributor. You
+must obtain the recipient's agreement that any such Additional Terms
+are offered by You alone, and You hereby agree to indemnify, defend
+and hold Apple and every Contributor harmless for any liability
+incurred by or claims asserted against Apple or such Contributor by
+reason of any such Additional Terms.
+
+7. Versions of the License. Apple may publish revised and/or new
+versions of this License from time to time. Each version will be given
+a distinguishing version number. Once Original Code has been published
+under a particular version of this License, You may continue to use it
+under the terms of that version. You may also choose to use such
+Original Code under the terms of any subsequent version of this
+License published by Apple. No one other than Apple has the right to
 modify the terms applicable to Covered Code created under this
 License.
 
-8. NO WARRANTY OR SUPPORT.  The Original Code may contain in whole or
-in part pre-release, untested, or not fully tested works.  The
-Original Code may contain errors that could cause failures or loss of
-data, and may be incomplete or contain inaccuracies.  You expressly
-acknowledge and agree that use of the Original Code, or any portion
-thereof, is at Your sole and entire risk.  THE ORIGINAL CODE IS
-PROVIDED "AS IS" AND WITHOUT WARRANTY, UPGRADES OR SUPPORT OF ANY KIND
-AND APPLE AND APPLE'S LICENSOR(S) (FOR THE PURPOSES OF SECTIONS 8 AND
-9, APPLE AND APPLE'S LICENSOR(S) ARE COLLECTIVELY REFERRED TO AS
-"APPLE") EXPRESSLY DISCLAIM ALL WARRANTIES AND/OR CONDITIONS, EXPRESS
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-AND/OR CONDITIONS OF MERCHANTABILITY OR SATISFACTORY QUALITY AND
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY
-RIGHTS.  APPLE DOES NOT WARRANT THAT THE FUNCTIONS CONTAINED IN THE
-ORIGINAL CODE WILL MEET YOUR REQUIREMENTS, OR THAT THE OPERATION OF
-THE ORIGINAL CODE WILL BE UNINTERRUPTED OR ERROR- FREE, OR THAT
-DEFECTS IN THE ORIGINAL CODE WILL BE CORRECTED.  NO ORAL OR WRITTEN
-INFORMATION OR ADVICE GIVEN BY APPLE OR AN APPLE AUTHORIZED
-REPRESENTATIVE SHALL CREATE A WARRANTY OR IN ANY WAY INCREASE THE
-SCOPE OF THIS WARRANTY.  You acknowledge that the Original Code is not
-intended for use in the operation of nuclear facilities, aircraft
-navigation, communication systems, or air traffic control machines in
-which case the failure of the Original Code could lead to death,
-personal injury, or severe physical or environmental damage.
-
-9. Liability.
-
-9.1 Infringement.  If any portion of, or functionality implemented by,
-the Original Code becomes the subject of a claim of infringement,
-Apple may, at its option: (a) attempt to procure the rights necessary
-for Apple and You to continue using the Affected Original Code; (b)
-modify the Affected Original Code so that it is no longer infringing;
-or (c) suspend Your rights to use, reproduce, modify, sublicense and
-distribute the Affected Original Code until a final determination of
-the claim is made by a court or governmental administrative agency of
-competent jurisdiction and Apple lifts the suspension as set forth
-below.  Such suspension of rights will be effective immediately upon
-Apple's posting of a notice to such effect on the Apple web site that
-is used for implementation of this License.  Upon such final
-determination being made, if Apple is legally able, without the
-payment of a fee or royalty, to resume use, reproduction,
-modification, sublicensing and distribution of the Affected Original
-Code, Apple will lift the suspension of rights to the Affected
-Original Code by posting a notice to such effect on the Apple web site
-that is used for implementation of this License.  If Apple suspends
-Your rights to Affected Original Code, nothing in this License shall
-be construed to restrict You, at Your option and subject to applicable
-law, from replacing the Affected Original Code with non-infringing
-code or independently negotiating for necessary rights from such third
-party.
-
-9.2 LIMITATION OF LIABILITY.  UNDER NO CIRCUMSTANCES SHALL APPLE BE
-LIABLE FOR ANY INCIDENTAL, SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES
-ARISING OUT OF OR RELATING TO THIS LICENSE OR YOUR USE OR INABILITY TO
-USE THE ORIGINAL CODE, OR ANY PORTION THEREOF, WHETHER UNDER A THEORY
-OF CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCTS LIABILITY
-OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES AND NOTWITHSTANDING THE FAILURE OF ESSENTIAL PURPOSE OF
-ANY REMEDY.  In no event shall Apple's total liability to You for all
-damages under this License exceed the amount of fifty dollars
-($50.00).
-
-10. Trademarks.  This License does not grant any rights to use the
-trademarks or trade names "Apple", "Apple Computer", "Mac OS X", "Mac
-OS X Server" or any other trademarks or trade names belonging to Apple
-(collectively "Apple Marks") and no Apple Marks may be used to endorse
-or promote products derived from the Original Code other than as
-permitted by and in strict compliance at all times with Apple's third
-party trademark usage guidelines which are posted at
-http://www.apple.com/legal/guidelinesfor3rdparties.html.
-
-11. Ownership.  Apple retains all rights, title and interest in and to
-the Original Code and any Modifications made by or on behalf of Apple
-("Apple Modifications"), and such Apple Modifications will not be
-automatically subject to this License.  Apple may, at its sole
-discretion, choose to license such Apple Modifications under this
-License, or on different terms from those contained in this License or
-may choose not to license them at all.  Apple's development, use,
-reproduction, modification, sublicensing and distribution of Covered
-Code will not be subject to this License.
+8. NO WARRANTY OR SUPPORT. The Covered Code may contain in whole or in
+part pre-release, untested, or not fully tested works. The Covered
+Code may contain errors that could cause failures or loss of data, and
+may be incomplete or contain inaccuracies. You expressly acknowledge
+and agree that use of the Covered Code, or any portion thereof, is at
+Your sole and entire risk. THE COVERED CODE IS PROVIDED "AS IS" AND
+WITHOUT WARRANTY, UPGRADES OR SUPPORT OF ANY KIND AND APPLE AND
+APPLE'S LICENSOR(S) (COLLECTIVELY REFERRED TO AS "APPLE" FOR THE
+PURPOSES OF SECTIONS 8 AND 9) AND ALL CONTRIBUTORS EXPRESSLY DISCLAIM
+ALL WARRANTIES AND/OR CONDITIONS, EXPRESS OR IMPLIED, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF
+MERCHANTABILITY, OF SATISFACTORY QUALITY, OF FITNESS FOR A PARTICULAR
+PURPOSE, OF ACCURACY, OF QUIET ENJOYMENT, AND NONINFRINGEMENT OF THIRD
+PARTY RIGHTS. APPLE AND EACH CONTRIBUTOR DOES NOT WARRANT AGAINST
+INTERFERENCE WITH YOUR ENJOYMENT OF THE COVERED CODE, THAT THE
+FUNCTIONS CONTAINED IN THE COVERED CODE WILL MEET YOUR REQUIREMENTS,
+THAT THE OPERATION OF THE COVERED CODE WILL BE UNINTERRUPTED OR
+ERROR-FREE, OR THAT DEFECTS IN THE COVERED CODE WILL BE CORRECTED. NO
+ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY APPLE, AN APPLE
+AUTHORIZED REPRESENTATIVE OR ANY CONTRIBUTOR SHALL CREATE A WARRANTY.
+You acknowledge that the Covered Code is not intended for use in the
+operation of nuclear facilities, aircraft navigation, communication
+systems, or air traffic control machines in which case the failure of
+the Covered Code could lead to death, personal injury, or severe
+physical or environmental damage.
+
+9. LIMITATION OF LIABILITY. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO
+EVENT SHALL APPLE OR ANY CONTRIBUTOR BE LIABLE FOR ANY INCIDENTAL,
+SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATING
+TO THIS LICENSE OR YOUR USE OR INABILITY TO USE THE COVERED CODE, OR
+ANY PORTION THEREOF, WHETHER UNDER A THEORY OF CONTRACT, WARRANTY,
+TORT (INCLUDING NEGLIGENCE), PRODUCTS LIABILITY OR OTHERWISE, EVEN IF
+APPLE OR SUCH CONTRIBUTOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES AND NOTWITHSTANDING THE FAILURE OF ESSENTIAL PURPOSE OF ANY
+REMEDY. SOME JURISDICTIONS DO NOT ALLOW THE LIMITATION OF LIABILITY OF
+INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY
+TO YOU. In no event shall Apple's total liability to You for all
+damages (other than as may be required by applicable law) under this
+License exceed the amount of fifty dollars ($50.00).
+
+10. Trademarks. This License does not grant any rights to use the
+trademarks or trade names "Apple", "Apple Computer", "Mac", "Mac OS",
+"QuickTime", "QuickTime Streaming Server" or any other trademarks,
+service marks, logos or trade names belonging to Apple (collectively
+"Apple Marks") or to any trademark, service mark, logo or trade name
+belonging to any Contributor. You agree not to use any Apple Marks in
+or as part of the name of products derived from the Original Code or
+to endorse or promote products derived from the Original Code other
+than as expressly permitted by and in strict compliance at all times
+with Apple's third party trademark usage guidelines which are posted
+at http://www.apple.com/legal/guidelinesfor3rdparties.html.
+
+11. Ownership. Subject to the licenses granted under this License,
+each Contributor retains all rights, title and interest in and to any
+Modifications made by such Contributor. Apple retains all rights,
+title and interest in and to the Original Code and any Modifications
+made by or on behalf of Apple ("Apple Modifications"), and such Apple
+Modifications will not be automatically subject to this License. Apple
+may, at its sole discretion, choose to license such Apple
+Modifications under this License, or on different terms from those
+contained in this License or may choose not to license them at all.
 
 12. Termination.
 
-12.1 Termination.  This License and the rights granted hereunder will
-   terminate:
+12.1 Termination. This License and the rights granted hereunder will
+terminate:
 
 (a) automatically without notice from Apple if You fail to comply with
 any term(s) of this License and fail to cure such breach within 30
-days of becoming aware of such breach; (b) immediately in the event of
-the circumstances described in Section 13.5(b); or (c) automatically
-without notice from Apple if You, at any time during the term of this
-License, commence an action for patent infringement against Apple.
-
-12.2 Effect of Termination.  Upon termination, You agree to
-immediately stop any further use, reproduction, modification,
-sublicensing and distribution of the Covered Code and to destroy all
-copies of the Covered Code that are in your possession or control.
-All sublicenses to the Covered Code which have been properly granted
-prior to termination shall survive any termination of this License.
-Provisions which, by their nature, should remain in effect beyond the
-termination of this License shall survive, including but not limited
-to Sections 3, 5, 8, 9, 10, 11, 12.2 and 13.  Neither party will be
-liable to the other for compensation, indemnity or damages of any sort
-solely as a result of terminating this License in accordance with its
-terms, and termination of this License will be without prejudice to
-any other right or remedy of either party.
-
-13.  Miscellaneous.
-
-13.1 Government End Users.  The Covered Code is a "commercial item" as
-defined in FAR 2.101.  Government software and technical data rights
-in the Covered Code include only those rights customarily provided to
-the public as defined in this License. This customary commercial
-license in technical data and software is provided in accordance with
-FAR 12.211 (Technical Data) and 12.212 (Computer Software) and, for
+days of becoming aware of such breach;
+
+(b) immediately in the event of the circumstances described in Section
+13.5(b); or
+
+(c) automatically without notice from Apple if You, at any time during
+the term of this License, commence an action for patent infringement
+against Apple; provided that Apple did not first commence
+an action for patent infringement against You in that instance.
+
+12.2 Effect of Termination. Upon termination, You agree to immediately
+stop any further use, reproduction, modification, sublicensing and
+distribution of the Covered Code. All sublicenses to the Covered Code
+which have been properly granted prior to termination shall survive
+any termination of this License. Provisions which, by their nature,
+should remain in effect beyond the termination of this License shall
+survive, including but not limited to Sections 3, 5, 8, 9, 10, 11,
+12.2 and 13. No party will be liable to any other for compensation,
+indemnity or damages of any sort solely as a result of terminating
+this License in accordance with its terms, and termination of this
+License will be without prejudice to any other right or remedy of
+any party.
+
+13. Miscellaneous.
+
+13.1 Government End Users. The Covered Code is a "commercial item" as
+defined in FAR 2.101. Government software and technical data rights in
+the Covered Code include only those rights customarily provided to the
+public as defined in this License. This customary commercial license
+in technical data and software is provided in accordance with FAR
+12.211 (Technical Data) and 12.212 (Computer Software) and, for
 Department of Defense purchases, DFAR 252.227-7015 (Technical Data --
 Commercial Items) and 227.7202-3 (Rights in Commercial Computer
-Software or Computer Software Documentation).  Accordingly, all U.S.
+Software or Computer Software Documentation). Accordingly, all U.S.
 Government End Users acquire Covered Code with only those rights set
 forth herein.
 
-13.2 Relationship of Parties.  This License will not be construed as
+13.2 Relationship of Parties. This License will not be construed as
 creating an agency, partnership, joint venture or any other form of
-legal association between You and Apple, and You will not represent to
-the contrary, whether expressly, by implication, appearance or
-otherwise.
+legal association between or among You, Apple or any Contributor, and
+You will not represent to the contrary, whether expressly, by
+implication, appearance or otherwise.
 
-13.3 Independent Development.  Nothing in this License will impair
+13.3 Independent Development. Nothing in this License will impair
 Apple's right to acquire, license, develop, have others develop for
 it, market and/or distribute technology or products that perform the
 same or similar functions as, or otherwise compete with,
 Modifications, Larger Works, technology or products that You may
 develop, produce, market or distribute.
 
-13.4 Waiver; Construction.  Failure by Apple to enforce any provision
-of this License will not be deemed a waiver of future enforcement of
-that or any other provision.  Any law or regulation which provides
-that the language of a contract shall be construed against the drafter
-will not apply to this License.
+13.4 Waiver; Construction. Failure by Apple or any Contributor to
+enforce any provision of this License will not be deemed a waiver of
+future enforcement of that or any other provision. Any law or
+regulation which provides that the language of a contract shall be
+construed against the drafter will not apply to this License.
 
-13.5 Severability.  (a) If for any reason a court of competent
+13.5 Severability. (a) If for any reason a court of competent
 jurisdiction finds any provision of this License, or portion thereof,
 to be unenforceable, that provision of the License will be enforced to
 the maximum extent permissible so as to effect the economic benefits
 and intent of the parties, and the remainder of this License will
-continue in full force and effect.  (b) Notwithstanding the foregoing,
+continue in full force and effect. (b) Notwithstanding the foregoing,
 if applicable law prohibits or restricts You from fully and/or
 specifically complying with Sections 2 and/or 3 or prevents the
 enforceability of either of those Sections, this License will
@@ -333,7 +326,7 @@ immediately terminate and You must immediately discontinue any use of
 the Covered Code and destroy all copies of it that are in your
 possession or control.
 
-13.6 Dispute Resolution.  Any litigation or other dispute resolution
+13.6 Dispute Resolution. Any litigation or other dispute resolution
 between You and Apple relating to this License shall take place in the
 Northern District of California, and You and Apple hereby consent to
 the personal jurisdiction of, and venue in, the state and federal
@@ -341,9 +334,9 @@ courts within that District with respect to this License. The
 application of the United Nations Convention on Contracts for the
 International Sale of Goods is expressly excluded.
 
-13.7 Entire Agreement; Governing Law.  This License constitutes the
+13.7 Entire Agreement; Governing Law. This License constitutes the
 entire agreement between the parties with respect to the subject
-matter hereof.  This License shall be governed by the laws of the
+matter hereof. This License shall be governed by the laws of the
 United States and the State of California, except that body of
 California law concerning conflicts of law.
 
@@ -355,18 +348,20 @@ connexes soient rediges en anglais.
 
 EXHIBIT A.
 
-"Portions Copyright (c) 1999-2000 Apple Computer, Inc.  All Rights
-Reserved.  This file contains Original Code and/or Modifications of
-Original Code as defined in and that are subject to the Apple Public
-Source License Version 1.1 (the "License").  You may not use this file
-except in compliance with the License.  Please obtain a copy of the
-License at http://www.apple.com/publicsource and read it before using
-this file.
+"Portions Copyright (c) 1999-2003 Apple Computer, Inc. All Rights
+Reserved.
+
+This file contains Original Code and/or Modifications of Original Code
+as defined in and that are subject to the Apple Public Source License
+Version 2.0 (the 'License'). You may not use this file except in
+compliance with the License. Please obtain a copy of the License at
+http://www.opensource.apple.com/apsl/ and read it before using this
+file.
 
 The Original Code and all software distributed under the License are
-distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE OR NON- INFRINGEMENT.  Please see the
-License for the specific language governing rights and limitations
-under the License."
+FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+Please see the License for the specific language governing rights and
+limitations under the License."
diff --git a/EXTERNAL_HEADERS/Makefile b/EXTERNAL_HEADERS/Makefile
index fceef154e..f90e7e8dc 100644
--- a/EXTERNAL_HEADERS/Makefile
+++ b/EXTERNAL_HEADERS/Makefile
@@ -20,7 +20,12 @@ INSTINC_SUBDIRS_I386 =	\
 	architecture	\
 	i386
 
+INSTINC_SUBDIRS_ARM =	\
+	architecture	\
+	arm
+
 EXPORT_FILES = \
+	AppleSecureBootEpoch.h \
 	ar.h		\
 	stdarg.h	\
 	stdbool.h	\
@@ -36,5 +41,3 @@ EXPORT_MI_DIR = .
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/EXTERNAL_HEADERS/ar.h b/EXTERNAL_HEADERS/ar.h
index 79ab2ade0..c80b50183 100644
--- a/EXTERNAL_HEADERS/ar.h
+++ b/EXTERNAL_HEADERS/ar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1991, 1993
diff --git a/EXTERNAL_HEADERS/architecture/Makefile b/EXTERNAL_HEADERS/architecture/Makefile
index 41a4fa6d4..ed5ca391b 100644
--- a/EXTERNAL_HEADERS/architecture/Makefile
+++ b/EXTERNAL_HEADERS/architecture/Makefile
@@ -15,6 +15,9 @@ INSTINC_SUBDIRS_PPC = 	\
 INSTINC_SUBDIRS_I386 =	\
 	i386
 
+INSTINC_SUBDIRS_ARM =	\
+	arm
+
 EXPORT_FILES = 
 
 INSTALL_MI_LIST = 
diff --git a/EXTERNAL_HEADERS/architecture/i386/asm_help.h b/EXTERNAL_HEADERS/architecture/i386/asm_help.h
index 0da8f189c..5d9ec0e0c 100644
--- a/EXTERNAL_HEADERS/architecture/i386/asm_help.h
+++ b/EXTERNAL_HEADERS/architecture/i386/asm_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991 NeXT Computer, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/i386/cpu.h b/EXTERNAL_HEADERS/architecture/i386/cpu.h
index 5db7029a3..0432ef9af 100644
--- a/EXTERNAL_HEADERS/architecture/i386/cpu.h
+++ b/EXTERNAL_HEADERS/architecture/i386/cpu.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
@@ -50,19 +56,3 @@ typedef struct _cr0 {
 			cd	:1,
 			pg	:1;
 } cr0_t;
-
-/*
- * Debugging register 6
- */
-
-typedef struct _dr6 {
-    unsigned int	b0	:1,
-    			b1	:1,
-			b2	:1,
-			b3	:1,
-				:9,
-			bd	:1,
-			bs	:1,
-			bt	:1,
-				:16;
-} dr6_t;
diff --git a/EXTERNAL_HEADERS/architecture/i386/desc.h b/EXTERNAL_HEADERS/architecture/i386/desc.h
index 4d3c62cac..ee7891783 100644
--- a/EXTERNAL_HEADERS/architecture/i386/desc.h
+++ b/EXTERNAL_HEADERS/architecture/i386/desc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/i386/io.h b/EXTERNAL_HEADERS/architecture/i386/io.h
index 2fe56c292..840faf1cd 100644
--- a/EXTERNAL_HEADERS/architecture/i386/io.h
+++ b/EXTERNAL_HEADERS/architecture/i386/io.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/i386/pio.h b/EXTERNAL_HEADERS/architecture/i386/pio.h
index 5fb0fafcf..1c9586d62 100644
--- a/EXTERNAL_HEADERS/architecture/i386/pio.h
+++ b/EXTERNAL_HEADERS/architecture/i386/pio.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/EXTERNAL_HEADERS/architecture/i386/reg_help.h b/EXTERNAL_HEADERS/architecture/i386/reg_help.h
index 89dc49a08..af56a36f7 100644
--- a/EXTERNAL_HEADERS/architecture/i386/reg_help.h
+++ b/EXTERNAL_HEADERS/architecture/i386/reg_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991 NeXT Computer, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/i386/sel.h b/EXTERNAL_HEADERS/architecture/i386/sel.h
index 8588273e5..9b35c3f5a 100644
--- a/EXTERNAL_HEADERS/architecture/i386/sel.h
+++ b/EXTERNAL_HEADERS/architecture/i386/sel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/i386/table.h b/EXTERNAL_HEADERS/architecture/i386/table.h
index cba52e414..11647535a 100644
--- a/EXTERNAL_HEADERS/architecture/i386/table.h
+++ b/EXTERNAL_HEADERS/architecture/i386/table.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/i386/tss.h b/EXTERNAL_HEADERS/architecture/i386/tss.h
index 68c902900..cfc1f9b40 100644
--- a/EXTERNAL_HEADERS/architecture/i386/tss.h
+++ b/EXTERNAL_HEADERS/architecture/i386/tss.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/ppc/asm_help.h b/EXTERNAL_HEADERS/architecture/ppc/asm_help.h
index 9d8181e21..0ff2171c4 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/asm_help.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/asm_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1996 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/ppc/basic_regs.h b/EXTERNAL_HEADERS/architecture/ppc/basic_regs.h
index f902cb084..b9dbdf699 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/basic_regs.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/basic_regs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1996 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/ppc/cframe.h b/EXTERNAL_HEADERS/architecture/ppc/cframe.h
index 3dc63034c..0db3fce7d 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/cframe.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/cframe.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/ppc/fp_regs.h b/EXTERNAL_HEADERS/architecture/ppc/fp_regs.h
index 51b78d404..ab48b8821 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/fp_regs.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/fp_regs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1996 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/ppc/macro_help.h b/EXTERNAL_HEADERS/architecture/ppc/macro_help.h
index 17152ca39..a149f8eb0 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/macro_help.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/macro_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1996 NeXT Software, Inc.
diff --git a/EXTERNAL_HEADERS/architecture/ppc/pseudo_inst.h b/EXTERNAL_HEADERS/architecture/ppc/pseudo_inst.h
index c2514014e..da4071e6b 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/pseudo_inst.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/pseudo_inst.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1996 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/architecture/ppc/reg_help.h b/EXTERNAL_HEADERS/architecture/ppc/reg_help.h
index d9d1b5ec9..6a0e2842e 100644
--- a/EXTERNAL_HEADERS/architecture/ppc/reg_help.h
+++ b/EXTERNAL_HEADERS/architecture/ppc/reg_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1996 NeXT Software, Inc.  All rights reserved.
  *
diff --git a/EXTERNAL_HEADERS/i386/_limits.h b/EXTERNAL_HEADERS/i386/_limits.h
index 3b9e7a6f7..5eec8cd1f 100644
--- a/EXTERNAL_HEADERS/i386/_limits.h
+++ b/EXTERNAL_HEADERS/i386/_limits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_I386__LIMITS_H_
 #define	_I386__LIMITS_H_
diff --git a/EXTERNAL_HEADERS/i386/limits.h b/EXTERNAL_HEADERS/i386/limits.h
index 5c6e31df1..50bdd3c69 100644
--- a/EXTERNAL_HEADERS/i386/limits.h
+++ b/EXTERNAL_HEADERS/i386/limits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1993
diff --git a/EXTERNAL_HEADERS/mach-o/fat.h b/EXTERNAL_HEADERS/mach-o/fat.h
index 1a3312ae1..6e724325f 100644
--- a/EXTERNAL_HEADERS/mach-o/fat.h
+++ b/EXTERNAL_HEADERS/mach-o/fat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * This header file describes the structures of the file format for "fat"
diff --git a/EXTERNAL_HEADERS/mach-o/kld.h b/EXTERNAL_HEADERS/mach-o/kld.h
index 23cba81b8..0fd1207e0 100644
--- a/EXTERNAL_HEADERS/mach-o/kld.h
+++ b/EXTERNAL_HEADERS/mach-o/kld.h
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.1 (the "License").  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON- INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACHO_KLD_H_
diff --git a/EXTERNAL_HEADERS/mach-o/loader.h b/EXTERNAL_HEADERS/mach-o/loader.h
index 5b917af6a..b8277bc62 100644
--- a/EXTERNAL_HEADERS/mach-o/loader.h
+++ b/EXTERNAL_HEADERS/mach-o/loader.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHO_LOADER_H_
 #define _MACHO_LOADER_H_
@@ -25,6 +31,7 @@
 /*
  * This file describes the format of mach object files.
  */
+#include <stdint.h>
 
 /*
  * <mach/machine.h> is needed here for the cpu_type_t and cpu_subtype_t types
@@ -43,10 +50,13 @@
  * states and the structures of those flavors for each machine.
  */
 #include <mach/machine/thread_status.h>
+#ifndef KERNEL
+#include <architecture/byte_order.h>
+#endif /* KERNEL */
 
 /*
- * The mach header appears at the very beginning of the object file; it
- * is the same for both 32-bit and 64-bit architectures.
+ * The 32-bit mach header appears at the very beginning of the object file for
+ * 32-bit architectures.
  */
 struct mach_header {
 	uint32_t	magic;		/* mach magic number identifier */
@@ -58,6 +68,10 @@ struct mach_header {
 	uint32_t	flags;		/* flags */
 };
 
+/* Constant for the magic field of the mach_header (32-bit architectures) */
+#define	MH_MAGIC	0xfeedface	/* the mach magic number */
+#define MH_CIGAM	0xcefaedfe	/* NXSwapInt(MH_MAGIC) */
+
 /*
  * The 64-bit mach header appears at the very beginning of object files for
  * 64-bit architectures.
@@ -73,18 +87,9 @@ struct mach_header_64 {
 	uint32_t	reserved;	/* reserved */
 };
 
-/* Constant for the magic field of the mach_header (32-bit architectures) */
-#define	MH_MAGIC	0xfeedface	/* the mach magic number */
-#define MH_CIGAM	0xcefaedfe
-
 /* Constant for the magic field of the mach_header_64 (64-bit architectures) */
-#define MH_MAGIC_64	0xfeedfacf	/* the 64-bit mach magic number */
-#define MH_CIGAM_64	0xcffaedfe	
-
-/* Constants for the cmd field of new load commands, the type */
-#define LC_SEGMENT_64	0x19	/* 64-bit segment of this file to be mapped */
-#define LC_ROUTINES_64	0x1a	/* 64-bit image routines */
-
+#define MH_MAGIC_64 0xfeedfacf /* the 64-bit mach magic number */
+#define MH_CIGAM_64 0xcffaedfe /* NXSwapInt(MH_MAGIC_64) */
 
 /*
  * The layout of the file depends on the filetype.  For all but the MH_OBJECT
@@ -97,10 +102,10 @@ struct mach_header_64 {
  * assembler and input (and possibly output) of the link editor (the .o
  * format).  All sections are in one unnamed segment with no segment padding. 
  * This format is used as an executable format when the file is so small the
- * segment padding greatly increases it's size.
+ * segment padding greatly increases its size.
  *
  * The file type MH_PRELOAD is an executable format intended for things that
- * not executed under the kernel (proms, stand alones, kernels, etc).  The
+ * are not executed under the kernel (proms, stand alones, kernels, etc).  The
  * format can be executed under the kernel but may demand paged it and not
  * preload it before execution.
  *
@@ -114,13 +119,17 @@ struct mach_header_64 {
 #define	MH_FVMLIB	0x3		/* fixed VM shared library file */
 #define	MH_CORE		0x4		/* core file */
 #define	MH_PRELOAD	0x5		/* preloaded executable file */
-#define	MH_DYLIB	0x6		/* dynamicly bound shared library file*/
+#define	MH_DYLIB	0x6		/* dynamically bound shared library */
 #define	MH_DYLINKER	0x7		/* dynamic link editor */
-#define	MH_BUNDLE	0x8		/* dynamicly bound bundle file */
+#define	MH_BUNDLE	0x8		/* dynamically bound bundle file */
+#define	MH_DYLIB_STUB	0x9		/* shared library stub for static */
+					/*  linking only, no section contents */
+#define	MH_DSYM		0xa		/* companion file with only debug */
+					/*  sections */
 
 /* Constants for the flags field of the mach_header */
 #define	MH_NOUNDEFS	0x1		/* the object file has no undefined
-					   references, can be executed */
+					   references */
 #define	MH_INCRLINK	0x2		/* the object file is the output of an
 					   incremental link against a base file
 					   and can't be link edited again */
@@ -130,12 +139,59 @@ struct mach_header_64 {
 #define MH_BINDATLOAD	0x8		/* the object file's undefined
 					   references are bound by the dynamic
 					   linker when loaded. */
-#define MH_PREBOUND	0x10		/* the file has it's dynamic undefined
+#define MH_PREBOUND	0x10		/* the file has its dynamic undefined
 					   references prebound. */
+#define MH_SPLIT_SEGS	0x20		/* the file has its read-only and
+					   read-write segments split */
+#define MH_LAZY_INIT	0x40		/* the shared library init routine is
+					   to be run lazily via catching memory
+					   faults to its writeable segments
+					   (obsolete) */
+#define MH_TWOLEVEL	0x80		/* the image is using two-level name
+					   space bindings */
+#define MH_FORCE_FLAT	0x100		/* the executable is forcing all images
+					   to use flat name space bindings */
+#define MH_NOMULTIDEFS	0x200		/* this umbrella guarantees no multiple
+					   defintions of symbols in its
+					   sub-images so the two-level namespace
+					   hints can always be used. */
+#define MH_NOFIXPREBINDING 0x400	/* do not have dyld notify the
+					   prebinding agent about this
+					   executable */
+#define MH_PREBINDABLE  0x800           /* the binary is not prebound but can
+					   have its prebinding redone. only used
+                                           when MH_PREBOUND is not set. */
+#define MH_ALLMODSBOUND 0x1000		/* indicates that this binary binds to
+                                           all two-level namespace modules of
+					   its dependent libraries. only used
+					   when MH_PREBINDABLE and MH_TWOLEVEL
+					   are both set. */ 
+#define MH_SUBSECTIONS_VIA_SYMBOLS 0x2000/* safe to divide up the sections into
+					    sub-sections via symbols for dead
+					    code stripping */
+#define MH_CANONICAL    0x4000		/* the binary has been canonicalized
+					   via the unprebind operation */
+#define MH_WEAK_DEFINES	0x8000		/* the final linked image contains
+					   external weak symbols */
+#define MH_BINDS_TO_WEAK 0x10000	/* the final linked image uses
+					   weak symbols */
+
 #define MH_ALLOW_STACK_EXECUTION 0x20000/* When this bit is set, all stacks 
 					   in the task will be given stack
 					   execution privilege.  Only used in
 					   MH_EXECUTE filetypes. */
+#define MH_ROOT_SAFE 0x40000           /* When this bit is set, the binary 
+					  declares it is safe for use in
+					  processes with uid zero */
+                                         
+#define MH_SETUID_SAFE 0x80000         /* When this bit is set, the binary 
+					  declares it is safe for use in
+					  processes when issetugid() is true */
+
+#define MH_NO_REEXPORTED_DYLIBS 0x100000 /* When this bit is set on a dylib, 
+					  the static linker does not need to
+					  examine dependent dylibs to see
+					  if any are re-exported */
 
 /*
  * The load commands directly follow the mach_header.  The total size of all
@@ -149,17 +205,27 @@ struct mach_header_64 {
  * pointer of the current load command.  The cmdsize for 32-bit architectures
  * MUST be a multiple of 4 bytes and for 64-bit architectures MUST be a multiple
  * of 8 bytes (these are forever the maximum alignment of any load commands).
- * sizeof(long) (this is forever the maximum alignment of any load commands).
  * The padded bytes must be zero.  All tables in the object file must also
  * follow these rules so the file can be memory mapped.  Otherwise the pointers
  * to these tables will not work well or at all on some machines.  With all
  * padding zeroed like objects will compare byte for byte.
  */
 struct load_command {
-	unsigned long cmd;		/* type of load command */
-	unsigned long cmdsize;		/* total size of command in bytes */
+	uint32_t cmd;		/* type of load command */
+	uint32_t cmdsize;	/* total size of command in bytes */
 };
 
+/*
+ * After MacOS X 10.1 when a new load command is added that is required to be
+ * understood by the dynamic linker for the image to execute properly the
+ * LC_REQ_DYLD bit will be or'ed into the load command constant.  If the dynamic
+ * linker sees such a load command it it does not understand will issue a
+ * "unknown load command required for execution" error and refuse to use the
+ * image.  Other load commands without this bit that are not understood will
+ * simply be ignored.
+ */
+#define LC_REQ_DYLD 0x80000000
+
 /* Constants for the cmd field of all load commands, the type */
 #define	LC_SEGMENT	0x1	/* segment of this file to be mapped */
 #define	LC_SYMTAB	0x2	/* link-edit stab symbol table info */
@@ -172,12 +238,34 @@ struct load_command {
 #define LC_FVMFILE	0x9	/* fixed VM file inclusion (internal use) */
 #define LC_PREPAGE      0xa     /* prepage command (internal use) */
 #define	LC_DYSYMTAB	0xb	/* dynamic link-edit symbol table info */
-#define	LC_LOAD_DYLIB	0xc	/* load a dynamicly linked shared library */
-#define	LC_ID_DYLIB	0xd	/* dynamicly linked shared lib identification */
+#define	LC_LOAD_DYLIB	0xc	/* load a dynamically linked shared library */
+#define	LC_ID_DYLIB	0xd	/* dynamically linked shared lib ident */
 #define LC_LOAD_DYLINKER 0xe	/* load a dynamic linker */
 #define LC_ID_DYLINKER	0xf	/* dynamic linker identification */
-#define	LC_PREBOUND_DYLIB 0x10	/* modules prebound for a dynamicly */
+#define	LC_PREBOUND_DYLIB 0x10	/* modules prebound for a dynamically */
 				/*  linked shared library */
+#define	LC_ROUTINES	0x11	/* image routines */
+#define	LC_SUB_FRAMEWORK 0x12	/* sub framework */
+#define	LC_SUB_UMBRELLA 0x13	/* sub umbrella */
+#define	LC_SUB_CLIENT	0x14	/* sub client */
+#define	LC_SUB_LIBRARY  0x15	/* sub library */
+#define	LC_TWOLEVEL_HINTS 0x16	/* two-level namespace lookup hints */
+#define	LC_PREBIND_CKSUM  0x17	/* prebind checksum */
+
+/*
+ * load a dynamically linked shared library that is allowed to be missing
+ * (all symbols are weak imported).
+ */
+#define	LC_LOAD_WEAK_DYLIB (0x18 | LC_REQ_DYLD)
+
+#define	LC_SEGMENT_64	0x19	/* 64-bit segment of this file to be
+				   mapped */
+#define	LC_ROUTINES_64	0x1a	/* 64-bit image routines */
+#define LC_UUID		0x1b	/* the uuid */
+#define LC_RPATH       (0x1c | LC_REQ_DYLD)    /* runpath additions */
+#define LC_CODE_SIGNATURE 0x1d	/* local of code signature */
+#define LC_SEGMENT_SPLIT_INFO 0x1e /* local of info to split segments */
+#define LC_REEXPORT_DYLIB (0x1f | LC_REQ_DYLD) /* load and re-export dylib */
 
 /*
  * A variable length string in a load command is represented by an lc_str
@@ -185,11 +273,13 @@ struct load_command {
  * the offset is from the start of the load command structure.  The size
  * of the string is reflected in the cmdsize field of the load command.
  * Once again any padded bytes to bring the cmdsize field to a multiple
- * of sizeof(long) must be zero.
+ * of 4 bytes must be zero.
  */
 union lc_str {
-	unsigned long	offset;	/* offset to the string */
+	uint32_t	offset;	/* offset to the string */
+#ifndef __LP64__
 	char		*ptr;	/* pointer to the string */
+#endif 
 };
 
 /*
@@ -204,18 +294,18 @@ union lc_str {
  * section structures directly follow the segment command and their size is
  * reflected in cmdsize.
  */
-struct segment_command {	/* for 32-bit architectures */
-	unsigned long	cmd;		/* LC_SEGMENT */
-	unsigned long	cmdsize;	/* includes sizeof section structs */
+struct segment_command { /* for 32-bit architectures */
+	uint32_t	cmd;		/* LC_SEGMENT */
+	uint32_t	cmdsize;	/* includes sizeof section structs */
 	char		segname[16];	/* segment name */
-	unsigned long	vmaddr;		/* memory address of this segment */
-	unsigned long	vmsize;		/* memory size of this segment */
-	unsigned long	fileoff;	/* file offset of this segment */
-	unsigned long	filesize;	/* amount to map from the file */
+	uint32_t	vmaddr;		/* memory address of this segment */
+	uint32_t	vmsize;		/* memory size of this segment */
+	uint32_t	fileoff;	/* file offset of this segment */
+	uint32_t	filesize;	/* amount to map from the file */
 	vm_prot_t	maxprot;	/* maximum VM protection */
 	vm_prot_t	initprot;	/* initial VM protection */
-	unsigned long	nsects;		/* number of sections in segment */
-	unsigned long	flags;		/* flags */
+	uint32_t	nsects;		/* number of sections in segment */
+	uint32_t	flags;		/* flags */
 };
 
 /*
@@ -224,7 +314,7 @@ struct segment_command {	/* for 32-bit architectures */
  * sections then section_64 structures directly follow the 64-bit segment
  * command and their size is reflected in cmdsize.
  */
-struct segment_command_64 {	/* for 64-bit architectures */
+struct segment_command_64 { /* for 64-bit architectures */
 	uint32_t	cmd;		/* LC_SEGMENT_64 */
 	uint32_t	cmdsize;	/* includes sizeof section_64 structs */
 	char		segname[16];	/* segment name */
@@ -238,7 +328,6 @@ struct segment_command_64 {	/* for 64-bit architectures */
 	uint32_t	flags;		/* flags */
 };
 
-
 /* Constants for the flags field of the segment_command */
 #define	SG_HIGHVM	0x1	/* the file contents for this segment is for
 				   the high part of the VM space, the low part
@@ -249,20 +338,25 @@ struct segment_command_64 {	/* for 64-bit architectures */
 #define	SG_NORELOC	0x4	/* this segment has nothing that was relocated
 				   in it and nothing relocated to it, that is
 				   it maybe safely replaced without relocation*/
+#define SG_PROTECTED_VERSION_1	0x8 /* This segment is protected.  If the
+				       segment starts at file offset 0, the
+				       first page of the segment is not
+				       protected.  All other pages of the
+				       segment are protected. */
 
 /*
  * A segment is made up of zero or more sections.  Non-MH_OBJECT files have
  * all of their segments with the proper sections in each, and padded to the
  * specified segment alignment when produced by the link editor.  The first
  * segment of a MH_EXECUTE and MH_FVMLIB format file contains the mach_header
- * and load commands of the object file before it's first section.  The zero
+ * and load commands of the object file before its first section.  The zero
  * fill sections are always last in their segment (in all formats).  This
  * allows the zeroed segment padding to be mapped into memory where zero fill
  * sections might be. The gigabyte zero fill sections, those with the section
  * type S_GB_ZEROFILL, can only be in a segment with sections of this type.
  * These segments are then placed after all other segments.
  *
- * The MH_OBJECT format has all of it's sections in one segment for
+ * The MH_OBJECT format has all of its sections in one segment for
  * compactness.  There is no padding to a specified segment boundary and the
  * mach_header and load commands are not part of the segment.
  *
@@ -277,18 +371,18 @@ struct segment_command_64 {	/* for 64-bit architectures */
  * fields of the section structure for mach object files is described in the
  * header file <reloc.h>.
  */
-struct section {		/* for 32-bit architectures */
+struct section { /* for 32-bit architectures */
 	char		sectname[16];	/* name of this section */
 	char		segname[16];	/* segment this section goes in */
-	unsigned long	addr;		/* memory address of this section */
-	unsigned long	size;		/* size in bytes of this section */
-	unsigned long	offset;		/* file offset of this section */
-	unsigned long	align;		/* section alignment (power of 2) */
-	unsigned long	reloff;		/* file offset of relocation entries */
-	unsigned long	nreloc;		/* number of relocation entries */
-	unsigned long	flags;		/* flags (section type and attributes)*/
-	unsigned long	reserved1;	/* reserved */
-	unsigned long	reserved2;	/* reserved */
+	uint32_t	addr;		/* memory address of this section */
+	uint32_t	size;		/* size in bytes of this section */
+	uint32_t	offset;		/* file offset of this section */
+	uint32_t	align;		/* section alignment (power of 2) */
+	uint32_t	reloff;		/* file offset of relocation entries */
+	uint32_t	nreloc;		/* number of relocation entries */
+	uint32_t	flags;		/* flags (section type and attributes)*/
+	uint32_t	reserved1;	/* reserved (for offset or index) */
+	uint32_t	reserved2;	/* reserved (for count or sizeof) */
 };
 
 struct section_64 { /* for 64-bit architectures */
@@ -306,7 +400,6 @@ struct section_64 { /* for 64-bit architectures */
 	uint32_t	reserved3;	/* reserved */
 };
 
-
 /*
  * The flags field of a section structure is separated into two parts a section
  * type and section attributes.  The section types are mutually exclusive (it
@@ -345,6 +438,17 @@ struct section_64 { /* for 64-bit architectures */
 						   the reserved2 field */
 #define	S_MOD_INIT_FUNC_POINTERS	0x9	/* section with only function
 						   pointers for initialization*/
+#define	S_MOD_TERM_FUNC_POINTERS	0xa	/* section with only function
+						   pointers for termination */
+#define	S_COALESCED			0xb	/* section contains symbols that
+						   are to be coalesced */
+#define	S_GB_ZEROFILL			0xc	/* zero fill on demand section
+						   (that can be larger than 4
+						   gigabytes) */
+#define	S_INTERPOSING			0xd	/* section with only pairs of
+						   function pointers for
+						   interposing */
+#define	S_16BYTE_LITERALS	0xe	/* section with only 16 byte literals */
 /*
  * Constants for the section attributes part of the flags field of a section
  * structure.
@@ -352,6 +456,28 @@ struct section_64 { /* for 64-bit architectures */
 #define SECTION_ATTRIBUTES_USR	 0xff000000	/* User setable attributes */
 #define S_ATTR_PURE_INSTRUCTIONS 0x80000000	/* section contains only true
 						   machine instructions */
+#define S_ATTR_NO_TOC 		 0x40000000	/* section contains coalesced
+						   symbols that are not to be
+						   in a ranlib table of
+						   contents */
+#define S_ATTR_STRIP_STATIC_SYMS 0x20000000	/* ok to strip static symbols
+						   in this section in files
+						   with the MH_DYLDLINK flag */
+#define S_ATTR_NO_DEAD_STRIP	 0x10000000	/* no dead stripping */
+#define S_ATTR_LIVE_SUPPORT	 0x08000000	/* blocks are live if they
+						   reference live blocks */
+#define S_ATTR_SELF_MODIFYING_CODE 0x04000000	/* Used with i386 code stubs
+						   written on by dyld */
+/*
+ * If a segment contains any sections marked with S_ATTR_DEBUG then all
+ * sections in that segment must have this attribute.  No section other than
+ * a section marked with this attribute may reference the contents of this
+ * section.  A section with this attribute may contain no symbols and must have
+ * a section type S_REGULAR.  The static linker will not copy section contents
+ * from sections with this attribute into its output file.  These sections
+ * generally contain DWARF debugging info.
+ */ 
+#define	S_ATTR_DEBUG		 0x02000000	/* a debug section */
 #define SECTION_ATTRIBUTES_SYS	 0x00ffff00	/* system setable attributes */
 #define S_ATTR_SOME_INSTRUCTIONS 0x00000400	/* section contains some
 						   machine instructions */
@@ -405,7 +531,7 @@ struct section_64 { /* for 64-bit architectures */
 #define SECT_OBJC_STRINGS "__selector_strs"	/* string table */
 #define SECT_OBJC_REFS "__selector_refs"	/* string table */
 
-#define	SEG_ICON	 "__ICON"	/* the NeXT icon segment */
+#define	SEG_ICON	 "__ICON"	/* the icon segment */
 #define	SECT_ICON_HEADER "__header"	/* the icon headers */
 #define	SECT_ICON_TIFF   "__tiff"	/* the icons in tiff format */
 
@@ -417,16 +543,20 @@ struct section_64 { /* for 64-bit architectures */
 
 #define SEG_UNIXSTACK	"__UNIXSTACK"	/* the unix stack segment */
 
+#define SEG_IMPORT	"__IMPORT"	/* the segment for the self (dyld) */
+					/* modifing code stubs that has read, */
+					/* write and execute permissions */
+
 /*
  * Fixed virtual memory shared libraries are identified by two things.  The
  * target pathname (the name of the library as found for execution), and the
  * minor version number.  The address of where the headers are loaded is in
- * header_addr.
+ * header_addr. (THIS IS OBSOLETE and no longer supported).
  */
 struct fvmlib {
 	union lc_str	name;		/* library's target pathname */
-	unsigned long	minor_version;	/* library's minor version number */
-	unsigned long	header_addr;	/* library's header address */
+	uint32_t	minor_version;	/* library's minor version number */
+	uint32_t	header_addr;	/* library's header address */
 };
 
 /*
@@ -434,10 +564,11 @@ struct fvmlib {
  * contains a fvmlib_command (cmd == LC_IDFVMLIB) to identify the library.
  * An object that uses a fixed virtual shared library also contains a
  * fvmlib_command (cmd == LC_LOADFVMLIB) for each library it uses.
+ * (THIS IS OBSOLETE and no longer supported).
  */
 struct fvmlib_command {
-	unsigned long	cmd;		/* LC_IDFVMLIB or LC_LOADFVMLIB */
-	unsigned long	cmdsize;	/* includes pathname string */
+	uint32_t	cmd;		/* LC_IDFVMLIB or LC_LOADFVMLIB */
+	uint32_t	cmdsize;	/* includes pathname string */
 	struct fvmlib	fvmlib;		/* the library identification */
 };
 
@@ -452,26 +583,99 @@ struct fvmlib_command {
  */
 struct dylib {
     union lc_str  name;			/* library's path name */
-    unsigned long timestamp;		/* library's build time stamp */
-    unsigned long current_version;	/* library's current version number */
-    unsigned long compatibility_version;/* library's compatibility vers number*/
+    uint32_t timestamp;			/* library's build time stamp */
+    uint32_t current_version;		/* library's current version number */
+    uint32_t compatibility_version;	/* library's compatibility vers number*/
 };
 
 /*
- * A dynamicly linked shared library (filetype == MH_DYLIB in the mach header)
+ * A dynamically linked shared library (filetype == MH_DYLIB in the mach header)
  * contains a dylib_command (cmd == LC_ID_DYLIB) to identify the library.
- * An object that uses a dynamicly linked shared library also contains a
- * dylib_command (cmd == LC_LOAD_DYLIB) for each library it uses.
+ * An object that uses a dynamically linked shared library also contains a
+ * dylib_command (cmd == LC_LOAD_DYLIB, LC_LOAD_WEAK_DYLIB, or
+ * LC_REEXPORT_DYLIB) for each library it uses.
  */
 struct dylib_command {
-	unsigned long	cmd;		/* LC_ID_DYLIB or LC_LOAD_DYLIB */
-	unsigned long	cmdsize;	/* includes pathname string */
+	uint32_t	cmd;		/* LC_ID_DYLIB, LC_LOAD_{,WEAK_}DYLIB,
+					   LC_REEXPORT_DYLIB */
+	uint32_t	cmdsize;	/* includes pathname string */
 	struct dylib	dylib;		/* the library identification */
 };
 
 /*
- * A program (filetype == MH_EXECUTE) or bundle (filetype == MH_BUNDLE) that is
- * prebound to it's dynamic libraries has one of these for each library that
+ * A dynamically linked shared library may be a subframework of an umbrella
+ * framework.  If so it will be linked with "-umbrella umbrella_name" where
+ * Where "umbrella_name" is the name of the umbrella framework. A subframework
+ * can only be linked against by its umbrella framework or other subframeworks
+ * that are part of the same umbrella framework.  Otherwise the static link
+ * editor produces an error and states to link against the umbrella framework.
+ * The name of the umbrella framework for subframeworks is recorded in the
+ * following structure.
+ */
+struct sub_framework_command {
+	uint32_t	cmd;		/* LC_SUB_FRAMEWORK */
+	uint32_t	cmdsize;	/* includes umbrella string */
+	union lc_str 	umbrella;	/* the umbrella framework name */
+};
+
+/*
+ * For dynamically linked shared libraries that are subframework of an umbrella
+ * framework they can allow clients other than the umbrella framework or other
+ * subframeworks in the same umbrella framework.  To do this the subframework
+ * is built with "-allowable_client client_name" and an LC_SUB_CLIENT load
+ * command is created for each -allowable_client flag.  The client_name is
+ * usually a framework name.  It can also be a name used for bundles clients
+ * where the bundle is built with "-client_name client_name".
+ */
+struct sub_client_command {
+	uint32_t	cmd;		/* LC_SUB_CLIENT */
+	uint32_t	cmdsize;	/* includes client string */
+	union lc_str 	client;		/* the client name */
+};
+
+/*
+ * A dynamically linked shared library may be a sub_umbrella of an umbrella
+ * framework.  If so it will be linked with "-sub_umbrella umbrella_name" where
+ * Where "umbrella_name" is the name of the sub_umbrella framework.  When
+ * staticly linking when -twolevel_namespace is in effect a twolevel namespace 
+ * umbrella framework will only cause its subframeworks and those frameworks
+ * listed as sub_umbrella frameworks to be implicited linked in.  Any other
+ * dependent dynamic libraries will not be linked it when -twolevel_namespace
+ * is in effect.  The primary library recorded by the static linker when
+ * resolving a symbol in these libraries will be the umbrella framework.
+ * Zero or more sub_umbrella frameworks may be use by an umbrella framework.
+ * The name of a sub_umbrella framework is recorded in the following structure.
+ */
+struct sub_umbrella_command {
+	uint32_t	cmd;		/* LC_SUB_UMBRELLA */
+	uint32_t	cmdsize;	/* includes sub_umbrella string */
+	union lc_str 	sub_umbrella;	/* the sub_umbrella framework name */
+};
+
+/*
+ * A dynamically linked shared library may be a sub_library of another shared
+ * library.  If so it will be linked with "-sub_library library_name" where
+ * Where "library_name" is the name of the sub_library shared library.  When
+ * staticly linking when -twolevel_namespace is in effect a twolevel namespace 
+ * shared library will only cause its subframeworks and those frameworks
+ * listed as sub_umbrella frameworks and libraries listed as sub_libraries to
+ * be implicited linked in.  Any other dependent dynamic libraries will not be
+ * linked it when -twolevel_namespace is in effect.  The primary library
+ * recorded by the static linker when resolving a symbol in these libraries
+ * will be the umbrella framework (or dynamic library). Zero or more sub_library
+ * shared libraries may be use by an umbrella framework or (or dynamic library).
+ * The name of a sub_library framework is recorded in the following structure.
+ * For example /usr/lib/libobjc_profile.A.dylib would be recorded as "libobjc".
+ */
+struct sub_library_command {
+	uint32_t	cmd;		/* LC_SUB_LIBRARY */
+	uint32_t	cmdsize;	/* includes sub_library string */
+	union lc_str 	sub_library;	/* the sub_library name */
+};
+
+/*
+ * A program (filetype == MH_EXECUTE) that is
+ * prebound to its dynamic libraries has one of these for each library that
  * the static linker used in prebinding.  It contains a bit vector for the
  * modules in the library.  The bits indicate which modules are bound (1) and
  * which are not (0) from the library.  The bit for module 0 is the low bit
@@ -479,10 +683,10 @@ struct dylib_command {
  * (linked_modules[N/8] >> N%8) & 1
  */
 struct prebound_dylib_command {
-	unsigned long	cmd;		/* LC_PREBOUND_DYLIB */
-	unsigned long	cmdsize;	/* includes strings */
+	uint32_t	cmd;		/* LC_PREBOUND_DYLIB */
+	uint32_t	cmdsize;	/* includes strings */
 	union lc_str	name;		/* library's path name */
-	unsigned long	nmodules;	/* number of modules in library */
+	uint32_t	nmodules;	/* number of modules in library */
 	union lc_str	linked_modules;	/* bit vector of linked modules */
 };
 
@@ -493,8 +697,8 @@ struct prebound_dylib_command {
  * A file can have at most one of these.
  */
 struct dylinker_command {
-	unsigned long	cmd;		/* LC_ID_DYLINKER or LC_LOAD_DYLINKER */
-	unsigned long	cmdsize;	/* includes pathname string */
+	uint32_t	cmd;		/* LC_ID_DYLINKER or LC_LOAD_DYLINKER */
+	uint32_t	cmdsize;	/* includes pathname string */
 	union lc_str    name;		/* dynamic linker's path name */
 };
 
@@ -503,13 +707,13 @@ struct dylinker_command {
  * use in the thread state primitives.  The machine specific data structures
  * follow the struct thread_command as follows.
  * Each flavor of machine specific data structure is preceded by an unsigned
- * long constant for the flavor of that data structure, an unsigned long
+ * long constant for the flavor of that data structure, an uint32_t
  * that is the count of longs of the size of the state data structure and then
  * the state data structure follows.  This triple may be repeated for many
  * flavors.  The constants for the flavors, counts and state data structure
  * definitions are expected to be in the header file <machine/thread_status.h>.
  * These machine specific data structures sizes must be multiples of
- * sizeof(long).  The cmdsize reflects the total size of the thread_command
+ * 4 bytes  The cmdsize reflects the total size of the thread_command
  * and all of the sizes of the constants for the flavors, counts and state
  * data structures.
  *
@@ -520,31 +724,70 @@ struct dylinker_command {
  * and environment variables are copied onto that stack.
  */
 struct thread_command {
-	unsigned long	cmd;		/* LC_THREAD or  LC_UNIXTHREAD */
-	unsigned long	cmdsize;	/* total size of this command */
-	/* unsigned long flavor		   flavor of thread state */
-	/* unsigned long count		   count of longs in thread state */
+	uint32_t	cmd;		/* LC_THREAD or  LC_UNIXTHREAD */
+	uint32_t	cmdsize;	/* total size of this command */
+	/* uint32_t flavor		   flavor of thread state */
+	/* uint32_t count		   count of longs in thread state */
 	/* struct XXX_thread_state state   thread state for this flavor */
 	/* ... */
 };
 
+/*
+ * The routines command contains the address of the dynamic shared library 
+ * initialization routine and an index into the module table for the module
+ * that defines the routine.  Before any modules are used from the library the
+ * dynamic linker fully binds the module that defines the initialization routine
+ * and then calls it.  This gets called before any module initialization
+ * routines (used for C++ static constructors) in the library.
+ */
+struct routines_command { /* for 32-bit architectures */
+	uint32_t	cmd;		/* LC_ROUTINES */
+	uint32_t	cmdsize;	/* total size of this command */
+	uint32_t	init_address;	/* address of initialization routine */
+	uint32_t	init_module;	/* index into the module table that */
+				        /*  the init routine is defined in */
+	uint32_t	reserved1;
+	uint32_t	reserved2;
+	uint32_t	reserved3;
+	uint32_t	reserved4;
+	uint32_t	reserved5;
+	uint32_t	reserved6;
+};
+
+/*
+ * The 64-bit routines command.  Same use as above.
+ */
+struct routines_command_64 { /* for 64-bit architectures */
+	uint32_t	cmd;		/* LC_ROUTINES_64 */
+	uint32_t	cmdsize;	/* total size of this command */
+	uint64_t	init_address;	/* address of initialization routine */
+	uint64_t	init_module;	/* index into the module table that */
+					/*  the init routine is defined in */
+	uint64_t	reserved1;
+	uint64_t	reserved2;
+	uint64_t	reserved3;
+	uint64_t	reserved4;
+	uint64_t	reserved5;
+	uint64_t	reserved6;
+};
+
 /*
  * The symtab_command contains the offsets and sizes of the link-edit 4.3BSD
  * "stab" style symbol table information as described in the header files
  * <nlist.h> and <stab.h>.
  */
 struct symtab_command {
-	unsigned long	cmd;		/* LC_SYMTAB */
-	unsigned long	cmdsize;	/* sizeof(struct symtab_command) */
-	unsigned long	symoff;		/* symbol table offset */
-	unsigned long	nsyms;		/* number of symbol table entries */
-	unsigned long	stroff;		/* string table offset */
-	unsigned long	strsize;	/* string table size in bytes */
+	uint32_t	cmd;		/* LC_SYMTAB */
+	uint32_t	cmdsize;	/* sizeof(struct symtab_command) */
+	uint32_t	symoff;		/* symbol table offset */
+	uint32_t	nsyms;		/* number of symbol table entries */
+	uint32_t	stroff;		/* string table offset */
+	uint32_t	strsize;	/* string table size in bytes */
 };
 
 /*
  * This is the second set of the symbolic information which is used to support
- * the data structures for the dynamicly link editor.
+ * the data structures for the dynamically link editor.
  *
  * The original set of symbolic information in the symtab_command which contains
  * the symbol and string tables must also be present when this load command is
@@ -552,7 +795,9 @@ struct symtab_command {
  * into three groups of symbols:
  *	local symbols (static and debugging symbols) - grouped by module
  *	defined external symbols - grouped by module (sorted by name if not lib)
- *	undefined external symbols (sorted by name)
+ *	undefined external symbols (sorted by name if MH_BINDATLOAD is not set,
+ *	     			    and in order the were seen by the static
+ *				    linker if MH_BINDATLOAD is set)
  * In this load command there are offsets and counts to each of the three groups
  * of symbols.
  *
@@ -563,7 +808,7 @@ struct symtab_command {
  *	reference symbol table
  *	indirect symbol table
  * The first three tables above (the table of contents, module table and
- * reference symbol table) are only present if the file is a dynamicly linked
+ * reference symbol table) are only present if the file is a dynamically linked
  * shared library.  For executable and object modules, which are files
  * containing only one module, the information that would be in these three
  * tables is determined as follows:
@@ -572,7 +817,7 @@ struct symtab_command {
  *		       file is part of the module.
  *	reference symbol table - is the defined and undefined external symbols
  *
- * For dynamicly linked shared library files this load command also contains
+ * For dynamically linked shared library files this load command also contains
  * offsets and sizes to the pool of relocation entries for all sections
  * separated into two groups:
  *	external relocation entries
@@ -581,8 +826,8 @@ struct symtab_command {
  * off the section structures.
  */
 struct dysymtab_command {
-    unsigned long cmd;		/* LC_DYSYMTAB */
-    unsigned long cmdsize;	/* sizeof(struct dysymtab_command) */
+    uint32_t cmd;	/* LC_DYSYMTAB */
+    uint32_t cmdsize;	/* sizeof(struct dysymtab_command) */
 
     /*
      * The symbols indicated by symoff and nsyms of the LC_SYMTAB load command
@@ -597,51 +842,51 @@ struct dysymtab_command {
      *
      * The last two groups are used by the dynamic binding process to do the
      * binding (indirectly through the module table and the reference symbol
-     * table when this is a dynamicly linked shared library file).
+     * table when this is a dynamically linked shared library file).
      */
-    unsigned long ilocalsym;	/* index to local symbols */
-    unsigned long nlocalsym;	/* number of local symbols */
+    uint32_t ilocalsym;	/* index to local symbols */
+    uint32_t nlocalsym;	/* number of local symbols */
 
-    unsigned long iextdefsym;	/* index to externally defined symbols */
-    unsigned long nextdefsym;	/* number of externally defined symbols */
+    uint32_t iextdefsym;/* index to externally defined symbols */
+    uint32_t nextdefsym;/* number of externally defined symbols */
 
-    unsigned long iundefsym;	/* index to undefined symbols */
-    unsigned long nundefsym;	/* number of undefined symbols */
+    uint32_t iundefsym;	/* index to undefined symbols */
+    uint32_t nundefsym;	/* number of undefined symbols */
 
     /*
      * For the for the dynamic binding process to find which module a symbol
      * is defined in the table of contents is used (analogous to the ranlib
      * structure in an archive) which maps defined external symbols to modules
-     * they are defined in.  This exists only in a dynamicly linked shared
+     * they are defined in.  This exists only in a dynamically linked shared
      * library file.  For executable and object modules the defined external
      * symbols are sorted by name and is use as the table of contents.
      */
-    unsigned long tocoff;	/* file offset to table of contents */
-    unsigned long ntoc;		/* number of entries in table of contents */
+    uint32_t tocoff;	/* file offset to table of contents */
+    uint32_t ntoc;	/* number of entries in table of contents */
 
     /*
      * To support dynamic binding of "modules" (whole object files) the symbol
      * table must reflect the modules that the file was created from.  This is
      * done by having a module table that has indexes and counts into the merged
      * tables for each module.  The module structure that these two entries
-     * refer to is described below.  This exists only in a dynamicly linked
+     * refer to is described below.  This exists only in a dynamically linked
      * shared library file.  For executable and object modules the file only
      * contains one module so everything in the file belongs to the module.
      */
-    unsigned long modtaboff;	/* file offset to module table */
-    unsigned long nmodtab;	/* number of module table entries */
+    uint32_t modtaboff;	/* file offset to module table */
+    uint32_t nmodtab;	/* number of module table entries */
 
     /*
      * To support dynamic module binding the module structure for each module
      * indicates the external references (defined and undefined) each module
      * makes.  For each module there is an offset and a count into the
      * reference symbol table for the symbols that the module references.
-     * This exists only in a dynamicly linked shared library file.  For
+     * This exists only in a dynamically linked shared library file.  For
      * executable and object modules the defined external symbols and the
      * undefined external symbols indicates the external references.
      */
-    unsigned long extrefsymoff;  /* offset to referenced symbol table */
-    unsigned long nextrefsyms;	 /* number of referenced symbol table entries */
+    uint32_t extrefsymoff;	/* offset to referenced symbol table */
+    uint32_t nextrefsyms;	/* number of referenced symbol table entries */
 
     /*
      * The sections that contain "symbol pointers" and "routine stubs" have
@@ -653,8 +898,8 @@ struct dysymtab_command {
      * the symbol table to the symbol that the pointer or stub is referring to.
      * The indirect symbol table is ordered to match the entries in the section.
      */
-    unsigned long indirectsymoff; /* file offset to the indirect symbol table */
-    unsigned long nindirectsyms;  /* number of indirect symbol table entries */
+    uint32_t indirectsymoff; /* file offset to the indirect symbol table */
+    uint32_t nindirectsyms;  /* number of indirect symbol table entries */
 
     /*
      * To support relocating an individual module in a library file quickly the
@@ -672,6 +917,8 @@ struct dysymtab_command {
      * this requires the r_address field to be something other than a section
      * offset to identify the item to be relocated.  In this case r_address is
      * set to the offset from the vmaddr of the first LC_SEGMENT command.
+     * For MH_SPLIT_SEGS images r_address is set to the the offset from the
+     * vmaddr of the first read-write LC_SEGMENT command.
      *
      * The relocation entries are grouped by module and the module table
      * entries have indexes and counts into them for the group of external
@@ -681,16 +928,16 @@ struct dysymtab_command {
      * remaining external relocation entries for them (for merged sections
      * remaining relocation entries must be local).
      */
-    unsigned long extreloff;	/* offset to external relocation entries */
-    unsigned long nextrel;	/* number of external relocation entries */
+    uint32_t extreloff;	/* offset to external relocation entries */
+    uint32_t nextrel;	/* number of external relocation entries */
 
     /*
      * All the local relocation entries are grouped together (they are not
      * grouped by their module since they are only used if the object is moved
      * from it staticly link edited address).
      */
-    unsigned long locreloff;	/* offset to local relocation entries */
-    unsigned long nlocrel;	/* number of local relocation entries */
+    uint32_t locreloff;	/* offset to local relocation entries */
+    uint32_t nlocrel;	/* number of local relocation entries */
 
 };	
 
@@ -707,63 +954,66 @@ struct dysymtab_command {
 
 /* a table of contents entry */
 struct dylib_table_of_contents {
-    unsigned long symbol_index;	/* the defined external symbol
+    uint32_t symbol_index;	/* the defined external symbol
 				   (index into the symbol table) */
-    unsigned long module_index;	/* index into the module table this symbol
+    uint32_t module_index;	/* index into the module table this symbol
 				   is defined in */
 };	
 
 /* a module table entry */
 struct dylib_module {
-    unsigned long module_name;	/* the module name (index into string table) */
+    uint32_t module_name;	/* the module name (index into string table) */
 
-    unsigned long iextdefsym;	/* index into externally defined symbols */
-    unsigned long nextdefsym;	/* number of externally defined symbols */
-    unsigned long irefsym;		/* index into reference symbol table */
-    unsigned long nrefsym;	/* number of reference symbol table entries */
-    unsigned long ilocalsym;	/* index into symbols for local symbols */
-    unsigned long nlocalsym;	/* number of local symbols */
+    uint32_t iextdefsym;	/* index into externally defined symbols */
+    uint32_t nextdefsym;	/* number of externally defined symbols */
+    uint32_t irefsym;		/* index into reference symbol table */
+    uint32_t nrefsym;		/* number of reference symbol table entries */
+    uint32_t ilocalsym;		/* index into symbols for local symbols */
+    uint32_t nlocalsym;		/* number of local symbols */
 
-    unsigned long iextrel;	/* index into external relocation entries */
-    unsigned long nextrel;	/* number of external relocation entries */
+    uint32_t iextrel;		/* index into external relocation entries */
+    uint32_t nextrel;		/* number of external relocation entries */
 
-    unsigned long iinit;	/* index into the init section */
-    unsigned long ninit;	/* number of init section entries */
+    uint32_t iinit_iterm;	/* low 16 bits are the index into the init
+				   section, high 16 bits are the index into
+			           the term section */
+    uint32_t ninit_nterm;	/* low 16 bits are the number of init section
+				   entries, high 16 bits are the number of
+				   term section entries */
 
-    unsigned long		/* for this module address of the start of */
+    uint32_t			/* for this module address of the start of */
 	objc_module_info_addr;  /*  the (__OBJC,__module_info) section */
-    unsigned long		/* for this module size of */
+    uint32_t			/* for this module size of */
 	objc_module_info_size;	/*  the (__OBJC,__module_info) section */
 };	
 
 /* a 64-bit module table entry */
 struct dylib_module_64 {
-	uint32_t module_name;	/* the module name (index into string table) */
+    uint32_t module_name;	/* the module name (index into string table) */
 
-	uint32_t iextdefsym;	/* index into externally defined symbols */
-	uint32_t nextdefsym;	/* number of externally defined symbols */
-	uint32_t irefsym;	/* index into reference symbol table */
-	uint32_t nrefsym;	/* number of reference symbol table entries */
-	uint32_t ilocalsym;	/* index into symbols for local symbols */
-	uint32_t nlocalsym;	/* number of local symbols */
+    uint32_t iextdefsym;	/* index into externally defined symbols */
+    uint32_t nextdefsym;	/* number of externally defined symbols */
+    uint32_t irefsym;		/* index into reference symbol table */
+    uint32_t nrefsym;		/* number of reference symbol table entries */
+    uint32_t ilocalsym;		/* index into symbols for local symbols */
+    uint32_t nlocalsym;		/* number of local symbols */
 
-	uint32_t iextrel;	/* index into external relocation entries */
-	uint32_t nextrel;	/* number of external relocation entries */
+    uint32_t iextrel;		/* index into external relocation entries */
+    uint32_t nextrel;		/* number of external relocation entries */
 
-	uint32_t iinit_iterm;	/* low 16 bits are the index into the init
+    uint32_t iinit_iterm;	/* low 16 bits are the index into the init
 				   section, high 16 bits are the index into
 				   the term section */
-	uint32_t ninit_nterm;	/* low 16 bits are the number of init section
-				   entries, high 16 bits are the number of
-				   term section entries */
-
-	uint32_t		/* for this module size of the */
-		objc_module_info_size;	/* (__OBJC,__module_info) section */
-	uint64_t		/* for this module address of the start of */
-		objc_module_info_addr;	/* the (__OBJC,__module_info) section */
+    uint32_t ninit_nterm;      /* low 16 bits are the number of init section
+				  entries, high 16 bits are the number of
+				  term section entries */
+
+    uint32_t			/* for this module size of */
+        objc_module_info_size;	/*  the (__OBJC,__module_info) section */
+    uint64_t			/* for this module address of the start of */
+        objc_module_info_addr;	/*  the (__OBJC,__module_info) section */
 };
 
-
 /* 
  * The entries in the reference symbol table are used when loading the module
  * (both by the static and dynamic link editors) and if the module is unloaded
@@ -773,48 +1023,128 @@ struct dylib_module_64 {
  * <mach-o/nlist.h> as they are also used for symbol table entries.
  */
 struct dylib_reference {
-    unsigned long isym:24,	/* index into the symbol table */
+    uint32_t isym:24,		/* index into the symbol table */
     		  flags:8;	/* flags to indicate the type of reference */
 };
 
+/*
+ * The twolevel_hints_command contains the offset and number of hints in the
+ * two-level namespace lookup hints table.
+ */
+struct twolevel_hints_command {
+    uint32_t cmd;	/* LC_TWOLEVEL_HINTS */
+    uint32_t cmdsize;	/* sizeof(struct twolevel_hints_command) */
+    uint32_t offset;	/* offset to the hint table */
+    uint32_t nhints;	/* number of hints in the hint table */
+};
+
+/*
+ * The entries in the two-level namespace lookup hints table are twolevel_hint
+ * structs.  These provide hints to the dynamic link editor where to start
+ * looking for an undefined symbol in a two-level namespace image.  The
+ * isub_image field is an index into the sub-images (sub-frameworks and
+ * sub-umbrellas list) that made up the two-level image that the undefined
+ * symbol was found in when it was built by the static link editor.  If
+ * isub-image is 0 the the symbol is expected to be defined in library and not
+ * in the sub-images.  If isub-image is non-zero it is an index into the array
+ * of sub-images for the umbrella with the first index in the sub-images being
+ * 1. The array of sub-images is the ordered list of sub-images of the umbrella
+ * that would be searched for a symbol that has the umbrella recorded as its
+ * primary library.  The table of contents index is an index into the
+ * library's table of contents.  This is used as the starting point of the
+ * binary search or a directed linear search.
+ */
+struct twolevel_hint {
+    uint32_t 
+	isub_image:8,	/* index into the sub images */
+	itoc:24;	/* index into the table of contents */
+};
+
+/*
+ * The prebind_cksum_command contains the value of the original check sum for
+ * prebound files or zero.  When a prebound file is first created or modified
+ * for other than updating its prebinding information the value of the check sum
+ * is set to zero.  When the file has it prebinding re-done and if the value of
+ * the check sum is zero the original check sum is calculated and stored in
+ * cksum field of this load command in the output file.  If when the prebinding
+ * is re-done and the cksum field is non-zero it is left unchanged from the
+ * input file.
+ */
+struct prebind_cksum_command {
+    uint32_t cmd;	/* LC_PREBIND_CKSUM */
+    uint32_t cmdsize;	/* sizeof(struct prebind_cksum_command) */
+    uint32_t cksum;	/* the check sum or zero */
+};
+
+/*
+ * The uuid load command contains a single 128-bit unique random number that
+ * identifies an object produced by the static link editor.
+ */
+struct uuid_command {
+    uint32_t	cmd;		/* LC_UUID */
+    uint32_t	cmdsize;	/* sizeof(struct uuid_command) */
+    uint8_t	uuid[16];	/* the 128-bit uuid */
+};
+
+/*
+ * The rpath_command contains a path which at runtime should be added to
+ * the current run path used to find @rpath prefixed dylibs.
+ */
+struct rpath_command {
+    uint32_t	 cmd;		/* LC_RPATH */
+    uint32_t	 cmdsize;	/* includes string */
+    union lc_str path;		/* path to add to run path */
+};
+
+/*
+ * The linkedit_data_command contains the offsets and sizes of a blob
+ * of data in the __LINKEDIT segment.  
+ */
+struct linkedit_data_command {
+    uint32_t	cmd;		/* LC_CODE_SIGNATURE or LC_SEGMENT_SPLIT_INFO */
+    uint32_t	cmdsize;	/* sizeof(struct linkedit_data_command) */
+    uint32_t	dataoff;	/* file offset of data in __LINKEDIT segment */
+    uint32_t	datasize;	/* file size of data in __LINKEDIT segment  */
+};
+
 /*
  * The symseg_command contains the offset and size of the GNU style
  * symbol table information as described in the header file <symseg.h>.
  * The symbol roots of the symbol segments must also be aligned properly
  * in the file.  So the requirement of keeping the offsets aligned to a
- * multiple of a sizeof(long) translates to the length field of the symbol
+ * multiple of a 4 bytes translates to the length field of the symbol
  * roots also being a multiple of a long.  Also the padding must again be
  * zeroed. (THIS IS OBSOLETE and no longer supported).
  */
 struct symseg_command {
-	unsigned long	cmd;		/* LC_SYMSEG */
-	unsigned long	cmdsize;	/* sizeof(struct symseg_command) */
-	unsigned long	offset;		/* symbol segment offset */
-	unsigned long	size;		/* symbol segment size in bytes */
+	uint32_t	cmd;		/* LC_SYMSEG */
+	uint32_t	cmdsize;	/* sizeof(struct symseg_command) */
+	uint32_t	offset;		/* symbol segment offset */
+	uint32_t	size;		/* symbol segment size in bytes */
 };
 
 /*
  * The ident_command contains a free format string table following the
  * ident_command structure.  The strings are null terminated and the size of
- * the command is padded out with zero bytes to a multiple of sizeof(long).
+ * the command is padded out with zero bytes to a multiple of 4 bytes/
  * (THIS IS OBSOLETE and no longer supported).
  */
 struct ident_command {
-	unsigned long cmd;		/* LC_IDENT */
-	unsigned long cmdsize;		/* strings that follow this command */
+	uint32_t cmd;		/* LC_IDENT */
+	uint32_t cmdsize;	/* strings that follow this command */
 };
 
 /*
  * The fvmfile_command contains a reference to a file to be loaded at the
- * specified virtual address.  (Presently, this command is reserved for NeXT
+ * specified virtual address.  (Presently, this command is reserved for
  * internal use.  The kernel ignores this command when loading a program into
  * memory).
  */
 struct fvmfile_command {
-	unsigned long cmd;		/* LC_FVMFILE */
-	unsigned long cmdsize;		/* includes pathname string */
+	uint32_t cmd;			/* LC_FVMFILE */
+	uint32_t cmdsize;		/* includes pathname string */
 	union lc_str	name;		/* files pathname */
-	unsigned long	header_addr;	/* files virtual address */
+	uint32_t	header_addr;	/* files virtual address */
 };
 
 #endif /* _MACHO_LOADER_H_ */
diff --git a/EXTERNAL_HEADERS/mach-o/nlist.h b/EXTERNAL_HEADERS/mach-o/nlist.h
index 3fe7c367c..bcd71f61a 100644
--- a/EXTERNAL_HEADERS/mach-o/nlist.h
+++ b/EXTERNAL_HEADERS/mach-o/nlist.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHO_NLIST_H_
 #define _MACHO_NLIST_H_
diff --git a/EXTERNAL_HEADERS/mach-o/reloc.h b/EXTERNAL_HEADERS/mach-o/reloc.h
index 8b8e29273..172a5b523 100644
--- a/EXTERNAL_HEADERS/mach-o/reloc.h
+++ b/EXTERNAL_HEADERS/mach-o/reloc.h
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.1 (the "License").  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON- INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: exec.h,v 1.6 1994/10/27 04:16:05 cgd Exp $	*/
 
diff --git a/EXTERNAL_HEADERS/machine/Makefile b/EXTERNAL_HEADERS/machine/Makefile
index f26245e7c..e6aee2489 100644
--- a/EXTERNAL_HEADERS/machine/Makefile
+++ b/EXTERNAL_HEADERS/machine/Makefile
@@ -13,6 +13,8 @@ INSTINC_SUBDIRS_PPC =
 
 INSTINC_SUBDIRS_I386 =
 
+INSTINC_SUBDIRS_ARM =
+
 EXPORT_FILES = \
 	limits.h
 
diff --git a/EXTERNAL_HEADERS/machine/limits.h b/EXTERNAL_HEADERS/machine/limits.h
index 2c5eac77a..65bd37494 100644
--- a/EXTERNAL_HEADERS/machine/limits.h
+++ b/EXTERNAL_HEADERS/machine/limits.h
@@ -1,29 +1,37 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #if !defined (_LIMITS_H___) && !defined (_MACH_MACHLIMITS_H_)
 #if defined (__ppc__)
 #include <ppc/limits.h>
 #elif defined (__i386__)
 #include <i386/limits.h>
+#elif defined (__arm__)
+#include <arm/limits.h>
 #else
 #error architecture not supported
 #endif
diff --git a/EXTERNAL_HEADERS/ppc/_limits.h b/EXTERNAL_HEADERS/ppc/_limits.h
index d512ec411..53f939493 100644
--- a/EXTERNAL_HEADERS/ppc/_limits.h
+++ b/EXTERNAL_HEADERS/ppc/_limits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_PPC__LIMITS_H_
 #define	_PPC__LIMITS_H_
diff --git a/EXTERNAL_HEADERS/ppc/limits.h b/EXTERNAL_HEADERS/ppc/limits.h
index 69d6991dc..f28b43326 100644
--- a/EXTERNAL_HEADERS/ppc/limits.h
+++ b/EXTERNAL_HEADERS/ppc/limits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1988, 1993
diff --git a/EXTERNAL_HEADERS/stdarg.h b/EXTERNAL_HEADERS/stdarg.h
index dc927b294..f178505e8 100644
--- a/EXTERNAL_HEADERS/stdarg.h
+++ b/EXTERNAL_HEADERS/stdarg.h
@@ -1,28 +1,32 @@
+/* Copyright (C) 1989, 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+GCC is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING.  If not, write to
+the Free Software Foundation, 59 Temple Place - Suite 330,
+Boston, MA 02111-1307, USA.  */
+
+/* As a special exception, if you include this header file into source
+   files compiled by GCC, this header file does not by itself cause
+   the resulting executable to be covered by the GNU General Public
+   License.  This exception does not however invalidate any other
+   reasons why the executable file might be covered by the GNU General
+   Public License.  */
+
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
+ * ISO C Standard:  7.15  Variable arguments  <stdarg.h>
  */
-/* stdarg.h for GNU.
-   Note that the type used in va_arg is supposed to match the
-   actual type **after default promotions**.
-   Thus, va_arg (..., short) is not valid.  */
 
 #ifndef _STDARG_H
 #ifndef _ANSI_STDARG_H_
@@ -32,100 +36,25 @@
 #endif /* not __need___va_list */
 #undef __need___va_list
 
-#ifdef __clipper__
-#include <va-clipper.h>
-#else
-#ifdef __m88k__
-#include <va-m88k.h>
-#else
-#ifdef __i860__
-#include <va-i860.h>
-#else
-#ifdef __hppa__
-#include <va-pa.h>
-#else
-#ifdef __mips__
-#include <va-mips.h>
-#else
-#ifdef __sparc__
-#include <va-sparc.h>
-#else
-#ifdef __i960__
-#include <va-i960.h>
-#else
-#ifdef __alpha__
-#include <va-alpha.h>
-#else
-#if defined (__H8300__) || defined (__H8300H__)
-#include <va-h8300.h>
-#else
-#if defined (__PPC__) && defined (_CALL_SYSV)
-#include <va-ppc.h>
-#else
-
 /* Define __gnuc_va_list.  */
 
 #ifndef __GNUC_VA_LIST
 #define __GNUC_VA_LIST
-#if defined(__svr4__) || defined(_AIX) || defined(_M_UNIX) || defined(__NetBSD__) || (defined(__APPLE__) && defined(__ppc__))
-typedef char *__gnuc_va_list;
-#else
-typedef void *__gnuc_va_list;
-#endif
+typedef __builtin_va_list __gnuc_va_list;
 #endif
 
 /* Define the standard macros for the user,
    if this invocation was from the user program.  */
 #ifdef _STDARG_H
 
-/* Amount of space required in an argument list for an arg of type TYPE.
-   TYPE may alternatively be an expression whose type is used.  */
-
-#if defined(sysV68)
-#define __va_rounded_size(TYPE)  \
-  (((sizeof (TYPE) + sizeof (short) - 1) / sizeof (short)) * sizeof (short))
-#else
-#define __va_rounded_size(TYPE)  \
-  (((sizeof (TYPE) + sizeof (int) - 1) / sizeof (int)) * sizeof (int))
+#define va_start(v,l)	__builtin_va_start(v,l)
+#define va_end(v)	__builtin_va_end(v)
+#define va_arg(v,l)	__builtin_va_arg(v,l)
+#if !defined(__STRICT_ANSI__) || __STDC_VERSION__ + 0 >= 199900L
+#define va_copy(d,s)	__builtin_va_copy(d,s)
 #endif
+#define __va_copy(d,s)	__builtin_va_copy(d,s)
 
-#define va_start(AP, LASTARG) 						\
- (AP = ((__gnuc_va_list) __builtin_next_arg (LASTARG)))
-
-#undef va_end
-void va_end (__gnuc_va_list);		/* Defined in libgcc.a */
-#define va_end(AP)	((void)0)
-
-/* We cast to void * and then to TYPE * because this avoids
-   a warning about increasing the alignment requirement.  */
-
-#if defined (__arm__) || defined (__i386__) || defined (__i860__) || defined (__ns32000__) || defined (__vax__)
-/* This is for little-endian machines; small args are padded upward.  */
-#define va_arg(AP, TYPE)						\
- (AP = (__gnuc_va_list) ((char *) (AP) + __va_rounded_size (TYPE)),	\
-  *((TYPE *) (void *) ((char *) (AP) - __va_rounded_size (TYPE))))
-#else /* big-endian */
-/* This is for big-endian machines; small args are padded downward.  */
-#define va_arg(AP, TYPE)						\
- (AP = (__gnuc_va_list) ((char *) (AP) + __va_rounded_size (TYPE)),	\
-  *((TYPE *) (void *) ((char *) (AP)					\
-		       - ((sizeof (TYPE) < __va_rounded_size (char)	\
-			   ? sizeof (TYPE) : __va_rounded_size (TYPE))))))
-#endif /* big-endian */
-#endif /* _STDARG_H */
-
-#endif /* not powerpc with V.4 calling sequence */
-#endif /* not h8300 */
-#endif /* not alpha */
-#endif /* not i960 */
-#endif /* not sparc */
-#endif /* not mips */
-#endif /* not hppa */
-#endif /* not i860 */
-#endif /* not m88k */
-#endif /* not clipper */
-
-#ifdef _STDARG_H
 /* Define va_list, if desired, from __gnuc_va_list. */
 /* We deliberately do not define va_list when called from
    stdio.h, because ANSI C says that stdio.h is not supposed to define
@@ -141,7 +70,7 @@ void va_end (__gnuc_va_list);		/* Defined in libgcc.a */
 #undef _BSD_VA_LIST
 #endif
 
-#ifdef __svr4__
+#if defined(__svr4__) || (defined(_SCO_DS) && !defined(__VA_LIST))
 /* SVR4.2 uses _VA_LIST for an internal alias for va_list,
    so we must avoid testing it and setting it here.
    SVR4 uses _VA_LIST as a flag in stdarg.h, but we should
@@ -154,8 +83,11 @@ void va_end (__gnuc_va_list);		/* Defined in libgcc.a */
 #endif
 #endif /* __i860__ */
 typedef __gnuc_va_list va_list;
+#ifdef _SCO_DS
+#define __VA_LIST
+#endif
 #endif /* _VA_LIST_ */
-#else /* not __svr4__ */
+#else /* not __svr4__ || _SCO_DS */
 
 /* The macro _VA_LIST_ is the same thing used by this file in Ultrix.
    But on BSD NET2 we must not test or define or undef it.
@@ -168,7 +100,10 @@ typedef __gnuc_va_list va_list;
 #ifndef _VA_LIST
 /* The macro _VA_LIST_T_H is used in the Bull dpx2  */
 #ifndef _VA_LIST_T_H
+/* The macro __va_list__ is used by BeOS.  */
+#ifndef __va_list__
 typedef __gnuc_va_list va_list;
+#endif /* not __va_list__ */
 #endif /* not _VA_LIST_T_H */
 #endif /* not _VA_LIST */
 #endif /* not _VA_LIST_DEFINED */
@@ -184,6 +119,9 @@ typedef __gnuc_va_list va_list;
 #ifndef _VA_LIST_T_H
 #define _VA_LIST_T_H
 #endif
+#ifndef __va_list__
+#define __va_list__
+#endif
 
 #endif /* not _VA_LIST_, except on certain systems */
 
diff --git a/EXTERNAL_HEADERS/stdint.h b/EXTERNAL_HEADERS/stdint.h
index 98fd43852..90164c0af 100644
--- a/EXTERNAL_HEADERS/stdint.h
+++ b/EXTERNAL_HEADERS/stdint.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000,2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
  * We build on <machine/types.h> rather than <sys/types.h> in order to
  * minimize the global namespace pollution (i.e., we'd like to define
@@ -84,10 +84,10 @@ typedef unsigned long long      uintmax_t;
 
 
 /* 7.18.2.1 Limits of exact-width integer types */
-#define INT8_MIN         -128
-#define INT16_MIN        -32768
-#define INT32_MIN        -2147483648
-#define INT64_MIN        -9223372036854775808LL
+#define INT8_MIN         (-127-1)
+#define INT16_MIN        (-32767-1)
+#define INT32_MIN        (-2147483647-1)
+#define INT64_MIN        (-9223372036854775807LL-1LL)
 
 #define INT8_MAX         +127
 #define INT16_MAX        +32767
@@ -132,11 +132,15 @@ typedef unsigned long long      uintmax_t;
 #define UINT_FAST64_MAX   UINT64_MAX
 
 /* 7.18.2.4 Limits of integer types capable of holding object pointers */
-
+#if defined(__LP64__)
+#define INTPTR_MIN        INT64_MIN
+#define INTPTR_MAX        INT64_MAX
+#define UINTPTR_MAX       UINT64_MAX
+#else
 #define INTPTR_MIN        INT32_MIN
 #define INTPTR_MAX        INT32_MAX
-                             
 #define UINTPTR_MAX       UINT32_MAX
+#endif
 
 /* 7.18.2.5 Limits of greatest-width integer types */
 #define INTMAX_MIN        INT64_MIN
@@ -145,9 +149,13 @@ typedef unsigned long long      uintmax_t;
 #define UINTMAX_MAX       UINT64_MAX
 
 /* 7.18.3 "Other" */
+#if defined(__LP64__)
+#define PTRDIFF_MIN       INT64_MIN
+#define PTRDIFF_MAX       INT64_MAX
+#else
 #define PTRDIFF_MIN       INT32_MIN
 #define PTRDIFF_MAX       INT32_MAX
-
+#endif
 /* We have no sig_atomic_t yet, so no SIG_ATOMIC_{MIN,MAX}.
    Should end up being {-127,127} or {0,255} ... or bigger.
    My bet would be on one of {U}INT32_{MIN,MAX}. */
diff --git a/Makefile b/Makefile
index 2855bb34c..c24af4cb4 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,8 @@ ALL_SUBDIRS = \
 	bsd  \
 	pexpert \
 	libkern \
-	libsa
+	libsa \
+	security
 
 CONFIG_SUBDIRS = config
 
@@ -39,12 +40,16 @@ INSTINC_SUBDIRS_PPC = $(INSTINC_SUBDIRS) EXTERNAL_HEADERS
 
 INSTINC_SUBDIRS_I386 = $(INSTINC_SUBDIRS) EXTERNAL_HEADERS
 
+INSTINC_SUBDIRS_ARM = $(INSTINC_SUBDIRS) EXTERNAL_HEADERS
+
 EXPINC_SUBDIRS = $(ALL_SUBDIRS)
 
 EXPINC_SUBDIRS_PPC =  $(EXPINC_SUBDIRS)
 
 EXPINC_SUBDIRS_I386 = $(EXPINC_SUBDIRS)
 
+EXPINC_SUBDIRS_ARM = $(EXPINC_SUBDIRS)
+
 COMP_SUBDIRS = $(ALL_SUBDIRS)
 
 
@@ -54,7 +59,8 @@ INST_SUBDIRS =	\
 	iokit	\
 	osfmk	\
 	bsd	\
-	config
+	config	\
+	security
 
 INSTALL_FILE_LIST= \
 	mach_kernel
diff --git a/README b/README
index 0d4d3285a..9ab5b012d 100644
--- a/README
+++ b/README
@@ -1,17 +1,50 @@
-How to build XNU:
+Table of contents:
+A. How to build XNU
+B. How to install a new header file from XNU
+
+=============================================
+A. How to build XNU:
 
 1) Type: "make"
 
-  This builds all the components for all architectures defined in 
-  ARCH_CONFIGS and for all kernel configurations defined in KERNEL_CONFIGS.
-  By default, ARCH_CONFIGS contains one architecture, the build machine 
-  architecture, and KERNEL_CONFIGS is set to build for RELEASE.
+  This builds all the components for kernel, architecture, and machine
+  configurations defined in TARGET_CONFIGS.  Additionally, we also support
+  architectures defined in ARCH_CONFIGS and kernel configurations defined in 
+  KERNEL_CONFIGS.  Note that TARGET_CONFIGS overrides any configurations defined
+  in ARCH_CONFIGS and KERNEL_CONFIGS.
+
+  By default, architecture defaults to the build machine 
+  architecture, and the kernel configuration is set to build for DEVELOPMENT.
+  The machine configuration defaults to MX31ADS for arm and nothing for i386 and ppc.
+  
   This will also create a bootable image, mach_kernel,  and a kernel binary 
   with symbols, mach_kernel.sys.
 
-  Example:
-    $(OBJROOT)/RELEASE_PPC/osfmk/RELEASE/osfmk.o: pre-linked object for osfmk component
-    $(OBJROOT)/RELEASE_PPC/mach_kernel: bootable image
+  Here are the valid arm machine configs:
+	LN2410SBC MX31ADS INTEGRATORCP S5I3000SMDK S5L8900XFPGA S5L8900XRB
+	OLOCREEK
+	
+  Examples:
+	/* make a debug kernel for MX31 arm board */
+	make TARGET_CONFIGS="debug arm MX31ADS"
+	
+    $(OBJROOT)/DEBUG_ARM_MX31ADS/osfmk/DEBUG/osfmk.o: pre-linked object for osfmk component
+    $(OBJROOT)/DEBUG_ARM_MX31ADS/mach_kernel: bootable image
+
+	/* make debug and development kernels for MX31 arm board */
+	make TARGET_CONFIGS="debug arm MX31ADS  development arm MX31ADS"
+	
+    $(OBJROOT)/DEBUG_ARM_MX31ADS/osfmk/DEBUG/osfmk.o: pre-linked object for osfmk component
+    $(OBJROOT)/DEBUG_ARM_MX31ADS/mach_kernel: bootable image
+    $(OBJROOT)/DEVELOPMENT_ARM/osfmk/DEVELOPMENT/osfmk.o: pre-linked object for osfmk component
+    $(OBJROOT)/DEVELOPMENT_ARM/mach_kernel: bootable image
+
+	/* this is all you need to do to build MX31ADS arm with DEVELOPMENT kernel configuration  */
+	make TARGET_CONFIGS="default arm default"
+	
+	or the following is equivalent
+	
+	make ARCH_CONFIGS=ARM
 
 2) Building a Component
 
@@ -30,11 +63,10 @@ How to build XNU:
 
     $ make all
 
-  This builds a component for all architectures defined in ARCH_CONFIGS 
-  and for all kernel configurations defined in KERNEL_CONFIGS. 
-  By default, ARCH_CONFIGS contains one architecture, the build machine 
-  architecture, and KERNEL_CONFIGS is set to build for RELEASE .
-
+  This builds a component for all architectures, kernel configurations, and
+  machine configurations defined in TARGET_CONFIGS (or alternately ARCH_CONFIGS
+  and KERNEL_CONFIGS).
+  
   Example:
     $(OBJROOT)/RELEASE_PPC/osfmk/RELEASE/osfmk.o: pre-linked object for osfmk component
 
@@ -50,14 +82,18 @@ How to build XNU:
 
 3) Building DEBUG
 
-  Define KERNEL_CONFIGS to DEBUG in your environment or when running a 
+  Define kernel configuration to DEBUG in your environment or when running a 
   make command.  Then, apply procedures 4, 5
 
+  $ make TARGET_CONFIGS="DEBUG PPC DEFAULT" all
+
+  or
+
   $ make KERNEL_CONFIGS=DEBUG all
 
   or
 
-  $ export KERNEL_CONFIGS=DEBUG
+  $ export TARGET_CONFIGS="DEBUG ARM MX31ADS"
   $ make all
 
   Example:
@@ -66,9 +102,13 @@ How to build XNU:
 
 4) Building fat
 
-  Define ARCH_CONFIGS in your environment or when running a make command.
+  Define architectures in your environment or when running a make command.
   Apply procedures 3, 4, 5
 
+  $ make TARGET_CONFIGS="RELEASE PPC default RELEASE I386 default" exporthdrs all
+
+  or
+
   $ make ARCH_CONFIGS="PPC I386" exporthdrs all
 
   or
@@ -76,19 +116,156 @@ How to build XNU:
   $ export ARCH_CONFIGS="PPC I386"
   $ make exporthdrs all
 
-5) Build check before integration
+5) Verbose make 
+   To display complete tool invocations rather than an abbreviated version,
+   $ make VERBOSE=YES
+
+6) Debug information formats
+   By default, a DWARF debug information repository is created during the install phase; this is a "bundle" named mach_kernel.dSYM
+   To select the older STABS debug information format (where debug information is embedded in the mach_kernel.sys image), set the BUILD_STABS environment variable.
+   $ export BUILD_STABS=1
+   $ make
+
+7) Build check before integration
 
   From the top directory, run:
 
     $ ~rc/bin/buildit . -arch ppc -arch i386 -noinstallsrc -nosum
-
-6) Creating tags and cscope
+	
+	or for multiple arm builds
+	
+    $ ~rc/bin/buildit . -noinstallsrc -nosum -- TARGET_CONFIGS="release arm MX31ADS release arm LN2410SBC"
+	
+	or for default arm build (kernel config DEVELOPMENT and machine config MX31ADS)
+	
+    $ ~rc/bin/buildit . -arch arm -noinstallsrc -nosum -- TARGET_CONFIGS="release arm MX31ADS release arm LN2410SBC"
+	
+
+8) Creating tags and cscope
 
   Set up your build environment as per instructions in 2a
 
   From the top directory, run:
 
-    $ make tags		# this will build ctags and etags
+    $ make tags		# this will build ctags and etags on a case-sensitive
+			# volume, only ctags on case-insensitive
+
+    $ make TAGS		# this will build etags
 
     $ make cscope	# this will build cscope database
 
+=============================================
+B. How to install a new header file from XNU
+
+[Note: This does not covers installing header file in IOKit framework]
+
+1) XNU installs header files at the following locations -
+	a. $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers
+	b. $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
+	c. $(DSTROOT)/System/Library/Frameworks/System.framework/Headers
+	d. $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
+	e. $(DSTROOT)/usr/include/
+
+	Kernel.framework is used by kernel extensions.  System.framework 
+	and /usr/include are used by user level applications.  The header 
+	files in framework's "PrivateHeaders" are only available for Apple 
+	Internal development.
+
+2) The directory containing the header file should have a Makefile that 
+   creates the list of files that should be installed at different locations.
+   If you are adding first header file in a directory, you will need to 
+   create Makefile similar to xnu/bsd/sys/Makefile.
+
+   Add your header file to the correct file list depending on where you want 
+   to install it.   The default locations where the header files are installed 
+   from each file list are -
+
+   a. DATAFILES : To make header file available in user level -
+	  $(DSTROOT)/System/Library/Frameworks/System.framework/Headers
+	  $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
+	  $(DSTROOT)/usr/include/
+
+   b. PRIVATE_DATAFILES : To make header file available to Apple internal in 
+      user level -
+	  $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
+
+   c. KERNELFILES : To make header file available in kernel level - 
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
+
+   d. PRIVATE_KERNELFILES : To make header file available to Apple internal 
+      for kernel extensions - 
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
+
+3) The Makefile combines the file lists mentioned above into different 
+   install lists which are used by build system to install the header files.
+
+   If the install list that you are interested does not exists, create it
+   by adding the appropriate file lists.  The default install lists, its 
+   member file lists and their default location are described below - 
+
+   a. INSTALL_MI_LIST : Installs header file to location that is available to 
+      everyone in user level. 
+      Locations -
+   	  $(DSTROOT)/System/Library/Frameworks/System.framework/Headers
+	  $(DSTROOT)/usr/include/
+      Definition -
+	  INSTALL_MI_LIST = ${DATAFILES}
+
+   b. INSTALL_MI_LCL_LIST : Installs header file to location that is available
+      for Apple internal in user level.
+      Locations -
+	  $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
+      Definition -
+	  INSTALL_MI_LCL_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
+
+   c. INSTALL_KF_MI_LIST : Installs header file to location that is available
+      to everyone for kernel extensions.
+      Locations -
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers
+      Definition -
+	  INSTALL_KF_MI_LIST = ${KERNELFILES}
+
+   d. INSTALL_KF_MI_LCL_LIST : Installs header file to location that is 
+      available for Apple internal for kernel extensions.
+      Locations -
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
+      Definition -
+	  INSTALL_KF_MI_LCL_LIST = ${KERNELFILES} ${PRIVATE_KERNELFILES}
+
+4) If you want to install the header file in a sub-directory of the paths 
+   described in (1), specify the directory name using two variable 
+   INSTALL_MI_DIR and EXPORT_MI_DIR as follows - 
+
+   INSTALL_MI_DIR = dirname
+   EXPORT_MI_DIR = dirname
+
+5) A single header file can exist at different locations using the steps 
+   mentioned above.  However it might not be desirable to make all the code
+   in the header file available at all the locations.  For example, you 
+   want to export a function only to kernel level but not user level.
+
+   You can use C language's pre-processor directive (#ifdef, #endif, #ifndef)
+   to control the text generated before a header file is installed.  The kernel 
+   only includes the code if the conditional macro is TRUE and strips out 
+   code for FALSE conditions from the header file.  
+
+   Some pre-defined macros and their descriptions are -
+   a. PRIVATE : If true, code is available to all of the xnu kernel and is 
+      not available in kernel extensions and user level header files.  The 
+      header files installed in all the paths described above in (1) will not 
+      have code enclosed within this macro.
+
+   b. KERNEL_PRIVATE : Same as PRIVATE
+
+   c. BSD_KERNEL_PRIVATE : If true, code is available to the xnu/bsd part of 
+      the kernel and is not available to rest of the kernel, kernel extensions 
+      and user level header files.  The header files installed in all the 
+      paths described above in (1) will not have code enclosed within this 
+      macro.
+
+   d. KERNEL :  If true, code is available only in kernel and kernel 
+      extensions and is not available in user level header files.  Only the 
+      header files installed in following paths will have the code - 
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers
+	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
diff --git a/bsd/Makefile b/bsd/Makefile
index 7d5fb5325..87836eda0 100644
--- a/bsd/Makefile
+++ b/bsd/Makefile
@@ -33,6 +33,9 @@ INSTINC_SUBDIRS_PPC = \
 INSTINC_SUBDIRS_I386 = \
 	i386 
 
+INSTINC_SUBDIRS_ARM = \
+	arm 
+
 EXPINC_SUBDIRS = \
 	bsm \
 	crypto \
@@ -60,6 +63,9 @@ EXPINC_SUBDIRS_PPC = \
 EXPINC_SUBDIRS_I386 = \
 	i386 
 
+EXPINC_SUBDIRS_ARM = \
+	arm 
+
 SETUP_SUBDIRS = 	\
 	conf
 
@@ -73,5 +79,3 @@ INSTMAN_SUBDIRS = \
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/bsd/bsm/audit.h b/bsd/bsm/audit.h
index 7ee808102..96550b3f0 100644
--- a/bsd/bsm/audit.h
+++ b/bsd/bsm/audit.h
@@ -1,14 +1,19 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2004 Apple Computer, Inc.  All Rights Reserved.
+ * Copyright (c) 1999-2007 Apple Inc.  All Rights Reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _BSM_AUDIT_H
diff --git a/bsd/bsm/audit_kernel.h b/bsd/bsm/audit_kernel.h
index 5719f3107..b68db2a6c 100644
--- a/bsd/bsm/audit_kernel.h
+++ b/bsd/bsm/audit_kernel.h
@@ -1,14 +1,19 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2004 Apple Computer, Inc.  All Rights Reserved.
+ * Copyright (c) 1999-2007 Apple Inc.  All Rights Reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,12 +23,23 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #ifndef _BSM_AUDIT_KERNEL_H
 #define	_BSM_AUDIT_KERNEL_H
 
+#if CONFIG_MACF
+#include <sys/queue.h>
+#include <security/mac_framework.h>
+#endif
+
 #ifdef KERNEL
 
 #include <bsm/audit.h>
@@ -95,6 +111,7 @@ extern int	audit_suspended;
 #define ARG_PROCESS		0x0000080000000000ULL
 #define ARG_MACHPORT1		0x0000100000000000ULL
 #define ARG_MACHPORT2		0x0000200000000000ULL
+#define ARG_MAC_STRING		0x0000400000000000ULL
 #define ARG_NONE		0x0000000000000000ULL
 #define ARG_ALL			0xFFFFFFFFFFFFFFFFULL
 
@@ -148,6 +165,21 @@ struct posix_ipc_perm {
 	mode_t			pipc_mode;
 };
 
+#if CONFIG_MACF
+
+#define MAC_AUDIT_LABEL_LEN 1024
+#define MAC_AUDIT_DATA_TYPE 0
+#define MAC_AUDIT_TEXT_TYPE 1
+
+struct mac_audit_record {
+	int type;		// one of the types defined above
+	int length;		// byte length of the data field
+	u_char *data;	// the payload
+	LIST_ENTRY(mac_audit_record) records;
+};
+
+#endif
+
 struct audit_record {
 	/* Audit record header. */
 	u_int32_t		ar_magic;
@@ -201,6 +233,12 @@ struct audit_record {
 	char				*ar_arg_upath2;
 	char				*ar_arg_kpath1;
 	char				*ar_arg_kpath2;
+#if CONFIG_MACF
+	char				*ar_vnode1_mac_labels;
+	char				*ar_vnode2_mac_labels;
+	char				*ar_cred_mac_labels;
+	char				*ar_arg_mac_string;
+#endif
 	char				*ar_arg_text;
 	struct au_mask			ar_arg_amask;
 	struct vnode_au_info		ar_arg_vnode1;
@@ -209,11 +247,21 @@ struct audit_record {
 	int				ar_arg_svipc_cmd;
 	struct ipc_perm			ar_arg_svipc_perm;
 	int				ar_arg_svipc_id;
-	void *				ar_arg_svipc_addr;
+	user_addr_t			ar_arg_svipc_addr;
 	struct posix_ipc_perm		ar_arg_pipc_perm;
 	mach_port_name_t		ar_arg_mach_port1;
 	mach_port_name_t		ar_arg_mach_port2;
 	union auditon_udata		ar_arg_auditon;
+
+#if CONFIG_MACF
+	/* MAC security related fields added by MAC policies
+	 * ar_forced_by_mac is 1 if mac_audit_check_preselect() forced this
+	 * call to be audited, 0 otherwise.
+	 */
+	LIST_HEAD(mac_audit_record_list_t, mac_audit_record)   *ar_mac_records;
+	int                             ar_forced_by_mac;
+#endif
+
 };
 
 /*
@@ -234,6 +282,8 @@ struct proc;
 struct vnode;
 struct componentname;
 
+int			 kau_will_audit(void);
+
 void			 audit_abort(struct kaudit_record *ar);
 void			 audit_commit(struct kaudit_record *ar, int error, 
 					int retval);
@@ -245,8 +295,17 @@ struct kaudit_record	*audit_new(int event, struct proc *p,
 
 void			 audit_syscall_enter(unsigned short code,
 				struct proc *proc, struct uthread *uthread);
+#if CONFIG_MACF
+/*
+ * The parameter list of audit_syscall_exit() was modified to also take the
+ * Darwin syscall number, which is required by mac_audit_check_postselect().
+ */
+void			 audit_syscall_exit(unsigned short code, int error,
+				struct proc *proc, struct uthread *uthread);
+#else
 void			 audit_syscall_exit(int error, struct proc *proc,
 				struct uthread *uthread);
+#endif
 void			 audit_mach_syscall_enter(unsigned short audit_event);
 void			 audit_mach_syscall_exit(int retval,
 				struct uthread *uthread);
@@ -277,7 +336,7 @@ token_t			*kau_to_header64(const struct timespec *ctime, int rec_size,
  * are wrapped by a macro, and the macro should be the only place in 
  * the source tree where these functions are referenced.
  */
-#ifdef AUDIT
+#if AUDIT
 void			 audit_arg_addr(user_addr_t addr);
 void			 audit_arg_len(user_size_t len);
 void			 audit_arg_fd(int fd);
@@ -299,11 +358,11 @@ void			 audit_arg_process(struct proc *p);
 void			 audit_arg_signum(u_int signum);
 void			 audit_arg_socket(int sodomain, int sotype, 
 						int soprotocol);
-void			 audit_arg_sockaddr(struct proc *p, 
+void			 audit_arg_sockaddr(struct vnode *cwd_vp,
 						struct sockaddr *so);
 void			 audit_arg_auid(uid_t auid);
 void			 audit_arg_auditinfo(const struct auditinfo *au_info);
-void			 audit_arg_upath(struct proc *p, char *upath, 
+void			 audit_arg_upath(struct vnode *cwd_vp, char *upath, 
 					 u_int64_t flags);
 void			 audit_arg_vnpath(struct vnode *vp, u_int64_t flags);
 void			 audit_arg_vnpath_withref(struct vnode *vp, u_int64_t flags);
@@ -312,7 +371,7 @@ void			 audit_arg_cmd(int cmd);
 void			 audit_arg_svipc_cmd(int cmd);
 void			 audit_arg_svipc_perm(const struct ipc_perm *perm);
 void			 audit_arg_svipc_id(int id);
-void			 audit_arg_svipc_addr(void *addr);
+void			 audit_arg_svipc_addr(user_addr_t addr);
 void			 audit_arg_posix_ipc_perm(uid_t uid, gid_t gid, 
 						 mode_t mode);
 void			 audit_arg_auditon(const union auditon_udata *udata);
@@ -327,6 +386,16 @@ void			 audit_proc_fork(struct proc *parent,
 					 struct proc *child);
 void			 audit_proc_free(struct proc *p);
 
+#if CONFIG_MACF
+/* 
+ * audit_mac_data() is the MAC Framework's entry point to the audit subsystem.
+ * It currently creates only text and data audit tokens.
+ */
+int			 audit_mac_data(int type, int len, u_char *data);
+void			 audit_arg_mac_string(const char *string);
+
+#endif
+
 /*
  * Define a macro to wrap the audit_arg_* calls by checking the global
  * audit_enabled flag before performing the actual call.
@@ -347,9 +416,9 @@ void			 audit_proc_free(struct proc *p);
  * auditing is enabled, or we have a audit record on the thread. It is 
  * possible that an audit record was begun before auditing was turned off.
  */
-#define AUDIT_SYSCALL_EXIT(error, proc, uthread)	do {		\
-	if (audit_enabled || (uthread->uu_ar != NULL)) {			\
-		audit_syscall_exit(error, proc, uthread);		\
+#define AUDIT_SYSCALL_EXIT(code, proc, uthread, error)	do {		\
+	if (audit_enabled || (uthread->uu_ar != NULL)) {		\
+		audit_syscall_exit(code, error, proc, uthread);		\
 	}								\
 	} while (0)
 
@@ -379,10 +448,14 @@ void			 audit_proc_free(struct proc *p);
 	} while (0)
 
 #else /* !AUDIT */
+
+#define	AUDIT_ARG(op, args...)	do {					\
+	} while (0)
+
 #define AUDIT_SYSCALL_ENTER(args...)	do {				\
 	} while (0)
 
-#define AUDIT_SYSCALL_EXIT(error, proc, uthread)	do {		\
+#define AUDIT_SYSCALL_EXIT(code, proc, uthread, error)	do {	\
 	} while (0)
 
 #define AUDIT_MACH_SYSCALL_ENTER(args...)       do {			\
diff --git a/bsd/bsm/audit_kevents.h b/bsd/bsm/audit_kevents.h
index 73bbf2ce2..6df33b22a 100644
--- a/bsd/bsm/audit_kevents.h
+++ b/bsd/bsm/audit_kevents.h
@@ -1,14 +1,19 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2004 Apple Computer, Inc.  All Rights Reserved.
+ * Copyright (c) 1999-2007 Apple Inc.  All Rights Reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,7 +23,13 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #ifndef _BSM_AUDIT_KEVENTS_H_
@@ -94,7 +105,7 @@
 #define	AUE_PIPE        185             /*42*/
 #define AUE_GETEGID     AUE_NULL        /*43*/
 #define AUE_PROFILE     305		/*44*/
-#define AUE_KTRACE      306		/*45*/
+#define AUE_KTRACE      AUE_NULL	/*45*/
 #define AUE_REBOOT      308
 #define AUE_SIGACTION   AUE_NULL        /*46*/    /*XXX*/
 #define AUE_GETGID	AUE_NULL	/*47*/
@@ -308,10 +319,6 @@
 #define AUE_SEMINIT     AUE_NULL        /*275*/   /*ENOSYS*/
 #define AUE_SEMDESTROY  AUE_NULL        /*276*/   /*ENOSYS*/
 
-#define AUE_LOADSHFILE  347		/*296*/
-#define AUE_RESETSHFILE 348		/*297*/
-#define AUE_NEWSYSTEMSHREG 349		/*298*/
-
 #define AUE_GETSID      AUE_NULL        /*310*/
 
 #define AUE_MLOCKALL    AUE_NULL        /*324*/   /*ENOSYS*/
@@ -329,6 +336,41 @@
 #define AUE_PIDFORTASK	359
 #define AUE_SYSCTL_NONADMIN	360
 
+// BSM events for security system calls
+#define AUE_MAC_GET_PROC		400
+#define AUE_MAC_SET_PROC		401
+#define AUE_MAC_GET_PID			402
+#define AUE_MAC_SET_FILE		403
+#define AUE_MAC_GET_FILE		404
+#define AUE_MAC_SET_LINK		405
+#define AUE_MAC_GET_LINK		406
+#define AUE_MAC_SET_FD			407
+#define AUE_MAC_GET_FD			408
+#define AUE_MAC_EXECVE			409
+#define AUE_MAC_SYSCALL			410
+#define	AUE_MAC_GET_LCID		411
+#define	AUE_MAC_GET_LCTX		412
+#define	AUE_MAC_SET_LCTX		413
+#define	AUE_SETLCID			414
+#define	AUE_GETLCID			415
+#define	AUE_MAC_MOUNT			416
+#define	AUE_MAC_GET_MOUNT		417
+#define	AUE_MAC_GETFSSTAT		418
+ 
+// BSM events for extended attributes
+#define AUE_EXTATTR_SET_FILE		451
+#define AUE_EXTATTR_GET_FILE		452
+#define AUE_EXTATTR_DELETE_FILE		453
+#define AUE_EXTATTR_LIST_FILE		454
+#define AUE_EXTATTR_SET_LINK		455
+#define AUE_EXTATTR_GET_LINK		456 
+#define AUE_EXTATTR_DELETE_LINK		457
+#define AUE_EXTATTR_LIST_LINK		458
+#define AUE_EXTATTR_SET_FD		459
+#define AUE_EXTATTR_GET_FD		460
+#define AUE_EXTATTR_DELETE_FD		461
+#define AUE_EXTATTR_LIST_FD		462
+
 // BSM events - Have to identify which ones are relevant to MacOSX
 #define AUE_ACLSET                      251
 #define AUE_AUDIT                       211
diff --git a/bsd/bsm/audit_klib.h b/bsd/bsm/audit_klib.h
index 00730f6a7..5f3e4717b 100644
--- a/bsd/bsm/audit_klib.h
+++ b/bsd/bsm/audit_klib.h
@@ -1,14 +1,19 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2004 Apple Computer, Inc.  All Rights Reserved.
+ * Copyright (c) 1999-2007 Apple Inc.  All Rights Reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _BSM_AUDIT_KLIB_H_
@@ -48,7 +53,7 @@ void au_evclassmap_init(void);
 void au_evclassmap_insert(au_event_t event, au_class_t class);
 au_class_t au_event_class(au_event_t event);
 
-int canon_path(struct proc *p, char *path, char *cpath);
+int canon_path(struct vnode *cwd_vp, char *path, char *cpath);
 
 
 
@@ -57,7 +62,6 @@ int canon_path(struct proc *p, char *path, char *cpath);
  * Define a system call to audit event mapping table.
  */
 extern au_event_t sys_au_event[];
-extern int nsys_au_event;	/* number of entries in this table */
 
 #endif /*KERNEL*/
 
diff --git a/bsd/bsm/audit_record.h b/bsd/bsm/audit_record.h
index da2c96388..fb58ccc05 100644
--- a/bsd/bsm/audit_record.h
+++ b/bsd/bsm/audit_record.h
@@ -1,14 +1,19 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2004 Apple Computer, Inc.  All Rights Reserved.
+ * Copyright (c) 1999-2007 Apple Inc.  All Rights Reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _BSM_AUDIT_RECORD_H_
@@ -241,15 +246,15 @@ token_t			*au_to_header64(int rec_size, au_event_t e_type,
 token_t			*au_to_me(void);
                                
 token_t			*au_to_arg(char n, char *text, u_int32_t v);
-token_t			*au_to_arg32(char n, char *text, u_int32_t v);
-token_t			*au_to_arg64(char n, char *text, u_int64_t v);
+token_t			*au_to_arg32(char n, const char *text, u_int32_t v);
+token_t			*au_to_arg64(char n, const char *text, u_int64_t v);
 token_t			*au_to_attr(struct vnode_attr *attr);
 token_t			*au_to_attr32(struct vnode_attr *attr);
 token_t			*au_to_attr64(struct vnode_attr *attr);
 token_t			*au_to_data(char unit_print, char unit_type,
-				char unit_count, char *p);
+				char unit_count, unsigned char *p);
 token_t			*au_to_exit(int retval, int err);
-token_t			*au_to_groups(int *groups);
+token_t			*au_to_groups(gid_t *groups);
 token_t			*au_to_newgroups(u_int16_t n, gid_t *groups);
 token_t			*au_to_in_addr(struct in_addr *internet_addr);
 token_t			*au_to_in_addr_ex(struct in6_addr *internet_addr);
@@ -280,7 +285,7 @@ token_t			*au_to_process64_ex(au_id_t auid, uid_t euid,
 token_t			*au_to_return(char status, u_int32_t ret);
 token_t			*au_to_return32(char status, u_int32_t ret);
 token_t			*au_to_return64(char status, u_int64_t ret);
-token_t			*au_to_seq(long audit_count);
+token_t			*au_to_seq(u_int32_t audit_count);
 token_t			*au_to_socket(struct socket *so);
 token_t			*au_to_socket_ex_32(u_int16_t lp, u_int16_t rp, 
 				struct sockaddr *la, struct sockaddr *ta);
@@ -310,7 +315,7 @@ token_t			*au_to_subject64_ex(au_id_t auid, uid_t euid,
 				au_asid_t sid, au_tid_addr_t *tid);
 token_t			*au_to_exec_args(const char **);
 token_t			*au_to_exec_env(const char **);
-token_t			*au_to_text(char *text);
+token_t			*au_to_text(const char *text);
 token_t			*au_to_kevent(struct kevent *kev);
 token_t			*au_to_trailer(int rec_size);
 
diff --git a/bsd/conf/MASTER b/bsd/conf/MASTER
index 9ccacbf12..8e308ece2 100644
--- a/bsd/conf/MASTER
+++ b/bsd/conf/MASTER
@@ -46,11 +46,7 @@
 #
 #  EXPERIMENTAL CONFIGURATION OPTIONS (select any combination, carefully)
 #
-#	nbc     = no buffer cache support
-#	simple  = non-rollover clock support
-#	timing  = precision timing support
 #	host	= host resource control support
-#	fixpri	= fixed priority threads
 #
 #  MULTI-PROCESSOR CONFIGURATION (select at most one)
 #
@@ -65,8 +61,7 @@
 #	medium = medium scale system configuration
 #	small  = small scale system configuration
 #	xsmall = extra small scale system configuration
-#	bsmall = special extra small scale system configuration for
-#		 (e.g. for boot floppies)
+#	bsmall = special extra small scale system configuration 
 #
 #######################################################################
 #  
@@ -85,15 +80,17 @@ ident		NeXT
 # obsolete timezone spec
 options		TIMEZONE=0, PST=0
 
-options		QUOTA		# 				# <quota>
-options		INET
+options		QUOTA			# 				# <quota>
+options		INET			#				# <inet>
 options		ABSOLUTETIME_SCALAR_TYPE
 options		NEW_VM_CODE	#				# <newvm>
 options		OLD_VM_CODE	#				# <!newvm>
 options		HW_AST		# Hardware ast support		# <hw_ast>
 options 	HW_FOOTPRINT	# Cache footprint support	# <hw_foot>
+options		CONFIG_LCTX	# Login Context
+
 options 	MACH		# Standard Mach features	# <mach>
-options		MACH_ASSERT	# Compile in assertions		# <test>
+options		MACH_ASSERT	# Compile in assertions		# <mach_assert>
 options		MACH_COMPAT	# Vendor syscall compatibility  # <mach>
 options		MACH_COUNTERS	# counters			# <count>
 options		MACH_DEBUG	# IPC debugging interface	# <mdebug>
@@ -106,7 +103,6 @@ options		MACH_IPC_TEST	# Testing code/printfs		# <ipc_test>
 options		MACH_LDEBUG	# Sanity-check simple locking	# <test>
 options 	MACH_NP		# Mach IPC support		# <np>
 options		MACH_NBC	# No buffer cache		# <nbc>
-options		REV_ENDIAN_FS	# Reverse Endian FS		# <revfs>
 options		MACH_NET	# Fast network access		# <mach_net>
 options		MACH_XP		# external pager support	# <xp>
 options		NORMA_IPC	# NORMA IPC support		# <norma_ipc>
@@ -126,63 +122,82 @@ options		ISO		# ISO stack			# <iso>
 options		LLC		# 802.2 support			# <llc>
 options		LOOP		# loopback support		# <loop>
 options		MROUTING	# multicast routing		# <mrouting>
-options		NS		# Netware			# <ns>
-options		PPP		# PPP				# <ppp>
 options		ROUTING		# routing			# <routing>
-options		TPIP		#				# <tpip>
-options		TUN		#				# <tun>
+options		NETMIBS		#				# <netmibs>
 options		VLAN		#				# <vlan>
 options		BOND		#				# <bond>
-options		NETMIBS		#				# <netmibs>
-options         IPDIVERT        # Divert sockets (for NAT)      # <ipdivert>
-options         IPFIREWALL      # IP Firewalling (used by NAT)  # <ipfirewall>
-#options        IPFIREWALL_VERBOSE      #                       # <ipfirewall>
-options         IPFIREWALL_FORWARD      #Transparent proxy      # <ipfirewall>
-options         IPFIREWALL_DEFAULT_TO_ACCEPT    # allow everything by default   # <ipfirewall>
-#options         IPFIREWALL_KEXT # Kernel extension              # <ipfirewall>
+options		IPDIVERT        # Divert sockets (for NAT)      # <ipdivert>
+options		IPFIREWALL      # IP Firewalling (used by NAT)  # <ipfirewall>
+options		IPFIREWALL_FORWARD      #Transparent proxy      # <ipfirewall>
+options		IPFIREWALL_DEFAULT_TO_ACCEPT    # allow everything by default   # <ipfirewall>
 options		DUMMYNET	# dummynet support		# <dummynet>
+options		TRAFFIC_MGT	# traffic management support		# <traffic_mgt>
 options		IPFW2		# IP firewall (new version)	# <ipfw2>
 options		MULTICAST	# Internet Protocol Class-D	$
 options		TCPDEBUG	# TCP debug			# <tcpdebug>
 options		RANDOM_IP_ID	# random (not sequential) ip ids	# <randomipid>
 options		TCP_DROP_SYNFIN	# Drop TCP packets with SYN+FIN set	# <tcpdrop_synfin>
 options		ICMP_BANDLIM	# ICMP bandwidth limiting sysctl
+options		IFNET_INPUT_SANITY_CHK	# allow dlil/ifnet input sanity check # <ifnet_input_chk>
 options		AUDIT		# Security event auditing	# <audit>
+options		SYSV_SEM	# SVID semaphores			# <sysv_sem>
+options		SYSV_MSG	# SVID messages				# <sysv_msg>
+options		SYSV_SHM	# SVID shared mem			# <sysv_shm>
+options		PANIC_INFO	# want kernel panic info	# <panic_info>
+options		DEVELOPMENT	# dev kernel				# <development>
 
+# secure_kernel - secure kernel from user programs
+options     SECURE_KERNEL       # <secure_kernel> 
 
 #
 #	4.4 general kernel 
 #
+options		SOCKETS		# socket support			# <inet, inet6, netat>
 options 	COMPAT_43_TTY	# 4.3 BSD tty compat	# <compat_43_tty>
-options		COMPAT_43_SOCKET	# 4.3 BSD socket compat	# <compat_43_socket>
 options 	DIAGNOSTIC	# diagnostics				# <diagnostic>
-options		KTRACE		# ktrace support			# <ktrace>
+options		CONFIG_DTRACE	# dtrace support			# <config_dtrace>
 options		GPROF		# build profiling			# <profile>
+options		SENDFILE	# sendfile					# <sendfile>
+options		NETWORKING	# networking layer			# <inet, inet6, netat>
+options		CONFIG_FSE	# file system events		# <config_fse>
+options		CONFIG_IMAGEBOOT	# local image boot	# <config_imageboot>
+options		CONFIG_SOWUPCALL	# SB_UPCALL on sowwakeup	# <config_sowupcall>
+options		CONFIG_FORCE_OUT_IFP	# Force IP output to use an interface # <config_force_out_ifp>
+options		CONFIG_MBUF_NOEXPAND	# limit mbuf expansion	# <config_mbuf_noexpand>
+options		CONFIG_MBUF_JUMBO	# jumbo cluster pool	# <config_mbuf_jumbo>
 
 #
 #	4.4 filesystems 
 #
-options		FFS		# Fast Filesystem Support	# <ffs>
-options		HFS		# HFS/HFS+ support		# <hfs>
+options		FFS			# Fast Filesystem Support	# <ffs>
+options		HFS			# HFS/HFS+ support		# <hfs>
 options		FIFO		# fifo support			# <fifo>
 options		UNION		# union_fs support		# <union>
 options		FDESC		# fdesc_fs support		# <fdesc>
 options		CD9660		# ISO 9660 CD-ROM support	# <cd9660>
-options		VOLFS		# volfs support			# <volfs>
 options		DEVFS		# devfs support			# <devfs>
-options		SYNTHFS		# synthfs support		# <synthfs>
+options		JOURNALING	# journaling support	# <journaling>
+
+#
+#	file system features
+#
+options		QUOTA		# file system quotas		# <quota>
+options		REV_ENDIAN_FS	# Reverse Endian FS		# <revfs>
+options		NAMEDSTREAMS	# named stream vnop support	# <namedstreams>
+options		CONFIG_VOLFS	# volfs path support (legacy)	# <config_volfs>
 
 #
 # NFS support
 #
-options		NFSCLIENT	# Be an NFS client		# <nfsserver>
-options		NFSSERVER	# Be an NFS server		# <nfsclient>
+options		NFSCLIENT	# Be an NFS client		# <nfsclient>
+options		NFSSERVER	# Be an NFS server		# <nfsserver>
 
 #
 # AppleTalk Support
 #
-options		NETAT		# AppleTalk support		# <netat>
+options		NETAT			# AppleTalk support				# <netat>
 #options		AURP_SUPPORT    # AppleTalk Update Routing      # <netat>
+
 #
 # Machine Independent Apple Features
 #
@@ -192,7 +207,7 @@ options		DRIVERKIT 	# driverkit support		# <driverkit>
 options		KERNOBJC	# Objective-C support		# <kernobjc>
 options		OBJCTEST	# Objc internal test		# <objctest>
 options		KERNEL_STACK	# MI kernel stack support	# <kernstack>
-profile				# build a profiling kernel	# <profile>
+profile						# build a profiling kernel	# <profile>
 
 #
 #  Point-to-Point Protocol support
@@ -215,6 +230,10 @@ pseudo-device   faith   1		# <faith>
 pseudo-device   stf 	1 		# <stf>
 
 options			crypto			# <ipsec,crypto>
+options			ALLCRYPTO		# <xlarge,large,medium>
+options			randomipid		# <inet,randomipid>
+
+options		ZLIB		# inflate/deflate support	# <zlib>
 
 makeoptions	LIBDRIVER = "libDriver_kern.o"			# <libdriver>
 makeoptions	LIBOBJC   = "libkobjc.o"			# <kernobjc>
@@ -223,7 +242,7 @@ maxusers	64		# <xlarge>
 maxusers	50		# <large>
 maxusers	32		# <medium>
 maxusers	16		# <small>
-maxusers	8		# <xsmall,msmall>
+maxusers	8		# <xsmall>
 maxusers	2		# <bsmall>
 
 #
@@ -235,22 +254,170 @@ pseudo-device	cpus		16	# <multi16>
 pseudo-device	cpus		2	# <multi2>
 pseudo-device	cpus		1	# <!multi64,multi32,multi16,multi2>
 
+#
+#  configurable kernel event related resources 
+#
+options   CONFIG_KN_HASHSIZE=64		# <medium,large,xlarge>
+options   CONFIG_KN_HASHSIZE=48		# <small,xsmall>
+options   CONFIG_KN_HASHSIZE=20		# <bsmall>
+
+#
+#  configurable vfs related resources 
+#  CONFIG_VNODES - used to pre allocate vnode related resources 
+#  CONFIG_VNODE_FREE_MIN - mininmum number of free vnodes 
+#  CONFIG_NC_HASH - name cache hash table allocation
+#  CONFIG_VFS_NAMES - name strings
+#
+#  263168 magic number for medium CONFIG_VNODES is based on memory 
+#  Number vnodes  is (memsize/64k) + 1024 
+#  This is the calculation that is used by launchd in tiger
+#  we are clipping the max based on 16G 
+#  ie ((16*1024*1024*1024)/(64 *1024)) + 1024 = 263168;
+
+options   CONFIG_VNODES=263168		# <large,xlarge>
+options   CONFIG_VNODES=263168		# <medium>
+options   CONFIG_VNODES=10240		# <small>
+options   CONFIG_VNODES=1024		# <xsmall>
+options   CONFIG_VNODES=512			# <bsmall>
+
+options   CONFIG_VNODE_FREE_MIN=500		# <large,xlarge>
+options   CONFIG_VNODE_FREE_MIN=300		# <medium>
+options   CONFIG_VNODE_FREE_MIN=200		# <small>
+options   CONFIG_VNODE_FREE_MIN=100		# <xsmall>
+options   CONFIG_VNODE_FREE_MIN=75		# <bsmall>
+
+options   CONFIG_NC_HASH=5120		# <large,xlarge>
+options   CONFIG_NC_HASH=4096		# <medium>
+options   CONFIG_NC_HASH=2048		# <small,xsmall>
+options   CONFIG_NC_HASH=1024		# <bsmall>
+
+options   CONFIG_VFS_NAMES=5120		# <large,xlarge>
+options   CONFIG_VFS_NAMES=4096		# <medium>
+options   CONFIG_VFS_NAMES=3072		# <small,xsmall>
+options   CONFIG_VFS_NAMES=2048		# <bsmall>
+
+#
+#  configurable kauth credential related resources 
+#
+options   KAUTH_CRED_PRIMES_COUNT=7		# <medium,large,xlarge>
+options   KAUTH_CRED_PRIMES_COUNT=3		# <bsmall,xsmall,small>
+
+options   KAUTH_CRED_PRIMES="{97, 241, 397, 743, 1499, 3989, 7499}"		# <medium,large,xlarge>
+options   KAUTH_CRED_PRIMES="{5, 17, 97}"								# <bsmall,xsmall,small>
+
+#
+#  configurable options for minumum number of buffers for kernel memory 
+#
+options   CONFIG_MIN_NBUF=256		# <medium,large,xlarge>
+options   CONFIG_MIN_NBUF=128		# <small>
+options   CONFIG_MIN_NBUF=80		# <xsmall>
+options   CONFIG_MIN_NBUF=64		# <bsmall>
+
+options   CONFIG_MIN_NIOBUF=128		# <medium,large,xlarge>
+options   CONFIG_MIN_NIOBUF=64		# <xsmall,small>
+options   CONFIG_MIN_NIOBUF=32		# <bsmall>
+
+#
+# set maximum space used for packet buffers
+#
+options        CONFIG_NMBCLUSTERS="((1024 * 1024) / MCLBYTES)"		# <large,xlarge>
+options        CONFIG_NMBCLUSTERS="((1024 * 512) / MCLBYTES)"		# <medium>
+options        CONFIG_NMBCLUSTERS="((1024 * 256) / MCLBYTES)"		# <bsmall,xsmall,small>
+
+#
+# Configure size of TCP hash table
+#
+options CONFIG_TCBHASHSIZE=4096		# <medium,large,xlarge>
+options CONFIG_TCBHASHSIZE=128		# <xsmall,small,bsmall>
+
+#
+#  configurable async IO options 
+#  CONFIG_AIO_MAX - system wide limit of async IO requests.
+#  CONFIG_AIO_PROCESS_MAX - process limit of async IO requests.
+#  CONFIG_AIO_THREAD_COUNT - number of async IO worker threads created.
+#
+options   CONFIG_AIO_MAX=360		# <xlarge>
+options   CONFIG_AIO_MAX=180		# <large>
+options   CONFIG_AIO_MAX=90			# <medium>
+options   CONFIG_AIO_MAX=45			# <small>
+options   CONFIG_AIO_MAX=20			# <xsmall>
+options   CONFIG_AIO_MAX=10			# <bsmall>
+
+options   CONFIG_AIO_PROCESS_MAX=64		# <xlarge>
+options   CONFIG_AIO_PROCESS_MAX=32		# <large>
+options   CONFIG_AIO_PROCESS_MAX=16		# <medium>
+options   CONFIG_AIO_PROCESS_MAX=12		# <small>
+options   CONFIG_AIO_PROCESS_MAX=8		# <xsmall>
+options   CONFIG_AIO_PROCESS_MAX=4		# <bsmall>
+
+options   CONFIG_AIO_THREAD_COUNT=16	# <xlarge>
+options   CONFIG_AIO_THREAD_COUNT=8		# <large>
+options   CONFIG_AIO_THREAD_COUNT=4		# <medium>
+options   CONFIG_AIO_THREAD_COUNT=3		# <small>
+options   CONFIG_AIO_THREAD_COUNT=2		# <xsmall,bsmall>
+
+#
+#  configurable kernel related resources (CONFIG_THREAD_MAX needs to stay in 
+#  sync with osfmk/conf/MASTER until we fix the config system...) todo XXX
+#
+options   CONFIG_THREAD_MAX=2560		# <medium,large,xlarge>
+options   CONFIG_THREAD_MAX=1536		# <small,xsmall>
+options   CONFIG_THREAD_MAX=1024		# <bsmall>
+
+options   CONFIG_MAXVIFS=32				# <medium,large,xlarge>
+options   CONFIG_MAXVIFS=16				# <small,xsmall>
+options   CONFIG_MAXVIFS=2				# <bsmall>
+
+options   CONFIG_MFCTBLSIZ=256			# <medium,large,xlarge>
+options   CONFIG_MFCTBLSIZ=128			# <small,xsmall>
+options   CONFIG_MFCTBLSIZ=16			# <bsmall>
+
+#
+#  configurable kernel - use these options to strip strings from panic
+#  and printf calls.
+#  no_panic_str - saves around 50K of kernel footprint.
+#  no_printf_str - saves around 45K of kernel footprint.
+#
+options   CONFIG_NO_PANIC_STRINGS		# <no_panic_str>
+options   CONFIG_NO_PRINTF_STRINGS		# <no_printf_str>
+options   CONFIG_NO_KPRINTF_STRINGS		# <no_kprintf_str>
+
+#
+# configurable kernel - general switch to say we are building for an
+# embedded device
+#
+options   CONFIG_EMBEDDED			# <config_embedded>
+
 #
 #  Ethernet (ARP)
 #
-pseudo-device	ether
+pseudo-device	ether			# <networking,inet,inet6,netat>
 #
 #  Network loopback device
 #
-pseudo-device	loop
+pseudo-device	loop			# <networking,inet,inet6,netat>
 #
 #  UCB pseudo terminal service
 #
-pseudo-device	pty	128 init pty_init
+pseudo-device  pty     512 init pty_init       # <xlarge>
+pseudo-device  pty     256 init pty_init       # <large>
+pseudo-device  pty     128 init pty_init       # <medium>
+pseudo-device  pty      48 init pty_init       # <small>
+pseudo-device  pty      16 init pty_init       # <xsmall>
+pseudo-device  pty       8 init pty_init       # <bsmall>
+#
+# Cloning pseudo terminal service
+#
+pseudo-device	ptmx	1 init ptmx_init
 
 #
 # vnode device
-pseudo-device	vndevice	4	init	vndevice_init
+#
+pseudo-device  vndevice	   16       init    vndevice_init   # <xlarge>
+pseudo-device  vndevice		8       init    vndevice_init   # <large>
+pseudo-device  vndevice		4       init    vndevice_init   # <medium>
+pseudo-device  vndevice		3       init    vndevice_init   # <small>
+pseudo-device  vndevice		2       init    vndevice_init   # <bsmall>
 
 #
 # memory device
@@ -260,11 +427,11 @@ pseudo-device	mdevdevice	1	init	mdevinit
 #
 # packet filter device
 #
-pseudo-device	bpfilter	4	init	bpf_init
+pseudo-device	bpfilter	4	init	bpf_init		# <networking,inet,inet6,netat>
 
 #
 # fsevents device
-pseudo-device	fsevents	1	init	fsevents_init
+pseudo-device	fsevents	1	init	fsevents_init	# <config_fse>
 
 #
 #  shim to "linux" mach disk drivers  (mach drivers must also be turned on)
@@ -273,4 +440,11 @@ pseudo-device	fsevents	1	init	fsevents_init
 #pseudo-device diskshim
 
 pseudo-device	random		1	init	random_init
+pseudo-device	dtrace		1	init	dtrace_init	# <config_dtrace>
+pseudo-device	helper		1	init	helper_init	# <config_dtrace>
+pseudo-device	lockstat	1	init	lockstat_init	# <config_dtrace>
+pseudo-device	sdt		1	init	sdt_init	# <config_dtrace>
+pseudo-device	systrace	1	init	systrace_init	# <config_dtrace>
+pseudo-device	fbt		1	init	fbt_init	# <config_dtrace>
+pseudo-device	profile_prvd	1	init	profile_init	# <config_dtrace>
 
diff --git a/bsd/conf/MASTER.i386 b/bsd/conf/MASTER.i386
index d7d200932..24125e3ce 100644
--- a/bsd/conf/MASTER.i386
+++ b/bsd/conf/MASTER.i386
@@ -44,10 +44,20 @@
 #  
 #  Standard Apple Research Configurations:
 #  -------- ----- -------- ---------------
-#
-#  RELEASE = [intel pc mach medium event vol pst gdb kernobjc libdriver fixpri simple_clock mdebug kernserv driverkit uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock volfs devfs revfs hfs mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs netat]
-#  PROFILE = [RELEASE profile]
-#  DEBUG = [intel pc mach medium event vol pst gdb kernobjc libdriver_g fixpri debug simple_clock mdebug kernserv driverkit xpr_debug uxpr kernstack ipc_compat ipc_debug nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs hfs volfs devfs mach_assert mrouting ipdivert ipfirewall ipfw2 dummynet ipv6firewall inet6 ipsec gif tcpdrop_synfin ktrace stf compat_43_tty compat_43_socket vlan bond netmibs netat]
+#  BASE =        [ intel mach medium config_dtrace vol pst gdb kernobjc fixpri simple_clock mdebug kernserv driverkit uxpr kernstack ipc_compat ipc_debug compat_43_tty sysv_sem sysv_msg sysv_shm audit panic_info config_imageboot ]
+#  FILESYS =	 [ devfs revfs hfs journaling fdesc config_fse quota namedstreams fifo ffs union cd9660 config_volfs ]
+#  NETWORKING =  [ inet inet6 compat_oldsock mrouting tcpdrop_synfin bpfilter ipdivert netat ipfirewall ipv6firewall ipfw2 dummynet traffic_mgt sendfile netmibs bond vlan gif stf zlib randomipid ifnet_input_chk config_mbuf_jumbo ]
+#  NFS =         [ nfsclient nfsserver ]
+#  VPN =         [ ipsec ]
+#  RELEASE =     [ BASE NETWORKING NFS VPN FILESYS libdriver ]
+#  PROFILE =     [ RELEASE profile ]
+#  DEBUG =       [ BASE NETWORKING NFS VPN FILESYS libdriver_g debug xpr_debug mach_assert ]
+#
+#  EMBEDDED_BASE =	[ intel mach bsmall vol pst gdb kernobjc fixpri simple_clock mdebug kernserv driverkit uxpr kernstack ipc_compat ipc_debug compat_43_tty sysv_sem sysv_msg sysv_shm audit panic_info config_imageboot ]
+#  EMBEDDED_FILESYS =	[ devfs hfs journaling fdesc fifo ]
+#  EMBEDDED_NET =	[ inet compat_oldsock mrouting tcpdrop_synfin bpfilter ipdivert config_mbuf_noexpand dummynet ipfirewall ipfw2 zlib ifnet_input_chk ]
+#  EMBEDDED =		[ EMBEDDED_BASE EMBEDDED_NET VPN EMBEDDED_FILESYS libdriver no_printf_str no_kprintf_str no_kdebug ]
+#  DEVELOPMENT =	[ EMBEDDED_BASE EMBEDDED_NET NFS VPN EMBEDDED_FILESYS libdriver netmibs development mach_assert config_dtrace ]
 #
 ######################################################################
 #
@@ -64,12 +74,23 @@ options		DEBUG		# general debugging code	# <debug>
 options		SHOW_SPACE	# print size of structures	# <debug>
 options		EVENTMETER	# event meter support		# <debug>
 options		FP_EMUL		# floating point emulation	# <fp>
-#options	PC_SUPPORT	# virtual PC support		# <pc>
 options		UXPR		# user-level XPR package	# <uxpr>
 config		mach_kernel	swap generic			# <mach>
-
 options		EVENT						# <event>
 
+#
+# Note: MAC options must be set in both bsd/conf and security/conf MASTER files
+#
+options		CONFIG_MACF					# Mandatory Access Control Framework
+options		CONFIG_MACF_SOCKET_SUBSET	# MAC socket subest (no labels)
+#options	CONFIG_MACF_SOCKET				# MAC socket labels
+#options	CONFIG_MACF_NET					# mbuf
+#options	CONFIG_MACF_DEBUG
+#options	CONFIG_MACF_MACH
+
+# app-profiling i.e. pre-heating - off? 
+options   CONFIG_APP_PROFILE=0  
+
 #
 #  Ipl measurement system
 #
diff --git a/bsd/conf/MASTER.ppc b/bsd/conf/MASTER.ppc
index ee3e6839b..4e1513cad 100644
--- a/bsd/conf/MASTER.ppc
+++ b/bsd/conf/MASTER.ppc
@@ -45,11 +45,15 @@
 #  Standard Apple Research Configurations:
 #  -------- ----- -------- ---------------
 #
-#  RELEASE = [ppc mach medium vol pst gdb simple_clock kernstack nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs noprofiling hfs volfs devfs netat mrouting ipdivert ipfirewall ipfw2 dummynet ktrace inet6 ipv6firewall ipsec tcpdrop_synfin gif stf compat_43_tty compat_43_socket vlan bond netmibs]
-#  RELEASE_TRACE = [RELEASE kdebug]
-#  PROFILE = [ppc mach medium vol pst gdb simple_clock kernstack nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs profile hfs volfs devfs netat mrouting ipdivert ipfirewall ipfw2 dummynet ktrace inet6 ipv6firewall ipsec tcpdrop_synfin gif stf compat_43_tty compat_43_socket vlan bond]
-#  DEBUG = [ppc mach medium vol pst gdb debug simple_clock kernstack nfsclient nfsserver quota fifo fdesc union ffs cd9660 compat_oldsock revfs profiling hfs volfs devfs netat mrouting mach_assert ipdivert ipfirewall ipfw2 dummynet ktrace inet6 ipv6firewall ipsec tcpdrop_synfin gif stf compat_43_tty compat_43_socket vlan bond netmibs]
-#  DEBUG_TRACE = [DEBUG kdebug]
+#  BASE =        [ ppc mach medium config_dtrace vol pst gdb noprofiling simple_clock kernstack compat_43_tty sysv_sem sysv_msg sysv_shm audit panic_info config_imageboot ]
+#  FILESYS =	 [ devfs revfs hfs journaling fdesc config_fse quota namedstreams fifo ffs union cd9660 config_volfs ]
+#  NETWORKING =  [ inet inet6 compat_oldsock mrouting tcpdrop_synfin bpfilter ipdivert netat ipfirewall ipv6firewall ipfw2 dummynet traffic_mgt sendfile netmibs bond vlan gif stf zlib randomipid ifnet_input_chk ]
+#  NFS =         [ nfsclient nfsserver ]
+#  VPN =         [ ipsec ]
+#  RELEASE =     [ BASE NETWORKING NFS VPN FILESYS libdriver ]
+#  DEVELOPMENT = [ RELEASE ]
+#  PROFILE =     [ RELEASE profile ]
+#  DEBUG =       [ BASE NETWORKING NFS VPN FILESYS libdriver_g debug xpr_debug mach_assert ]
 #
 ######################################################################
 #
@@ -64,6 +68,16 @@ options		FP_EMUL		# floating point emulation	# <fp>
 options		UXPR		# user-level XPR package	# <uxpr>
 config		mach_kernel	swap generic			# <mach>
 
+#
+# Note: MAC options must be set in both bsd/conf and security/conf MASTER files
+#
+options		CONFIG_MACF			# Mandatory Access Control Framework
+options		CONFIG_MACF_SOCKET_SUBSET	# MAC socket subest (no labels)
+#options	CONFIG_MACF_SOCKET		# MAC socket labels
+#options	CONFIG_MACF_NET			# mbuf
+#options	CONFIG_MACF_DEBUG
+#options	CONFIG_MACF_MACH
+
 options		EVENT						# <event>
 
 #
diff --git a/bsd/conf/Makefile b/bsd/conf/Makefile
index b4b5a7d2f..29ae6092f 100644
--- a/bsd/conf/Makefile
+++ b/bsd/conf/Makefile
@@ -3,10 +3,85 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+export ubc_subr.o_CFLAGS_ADD=-Wno-discard-qual
 export vnode_pager.o_CFLAGS_ADD=-Werror
 export vm_unix.o_CFLAGS_ADD=-Werror
 export dp_backing_file.o_CFLAGS_ADD=-Werror
 export if_mib.o_CFLAGS_ADD=-Wno-unused-parameter
+export adsp_Write.o_CFLAGS_ADD=-Wno-sign-compare
+export adsp_Packet.o_CFLAGS_ADD=-Wno-sign-compare
+export adsp_Control.o_CFLAGS_ADD=-Wno-sign-compare
+export adsp_RxAttn.o_CFLAGS_ADD=-Wno-sign-compare
+export adsp_attention.o_CFLAGS_ADD=-Wno-sign-compare
+export asp_proto.o_CFLAGS_ADD=-Wno-sign-compare
+export drv_dep.o_CFLAGS_ADD=-Wno-sign-compare
+export ddp_rtmp.o_CFLAGS_ADD=-Wno-sign-compare
+export ddp_lap.o_CFLAGS_ADD=-Wno-sign-compare
+export radix.o_CFLAGS_ADD=-Wno-sign-compare
+export route.o_CFLAGS_ADD=-Wno-sign-compare
+export rtsock.o_CFLAGS_ADD=-Wno-sign-compare
+export dhcp_options.o_CFLAGS_ADD=-Wno-sign-compare
+export igmp.o_CFLAGS_ADD=-Wno-sign-compare
+export in_cksum.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_divert.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_dummynet.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_flow.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_fw2.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_fw2_compat.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_icmp.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_input.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_mroute.o_CFLAGS_ADD=-Wno-sign-compare
+export ip_output.o_CFLAGS_ADD=-Wno-sign-compare
+export raw_ip.o_CFLAGS_ADD=-Wno-sign-compare
+export tcp_input.o_CFLAGS_ADD=-Wno-sign-compare
+export tcp_output.o_CFLAGS_ADD=-Wno-sign-compare
+export tcp_subr.o_CFLAGS_ADD=-Wno-sign-compare
+export tcp_usrreq.o_CFLAGS_ADD=-Wno-sign-compare
+export tcp_timer.o_CFLAGS_ADD=-Wno-sign-compare
+export udp_usrreq.o_CFLAGS_ADD=-Wno-sign-compare
+export ah_input.o_CFLAGS_ADD=-Wno-sign-compare
+export ah_core.o_CFLAGS_ADD=-Wno-sign-compare
+export ah_output.o_CFLAGS_ADD=-Wno-sign-compare
+export esp_core.o_CFLAGS_ADD=-Wno-sign-compare
+export esp_input.o_CFLAGS_ADD=-Wno-sign-compare
+export esp_output.o_CFLAGS_ADD=-Wno-sign-compare
+export esp_rijndael.o_CFLAGS_ADD=-Wno-sign-compare
+export ipsec.o_CFLAGS_ADD=-Wno-sign-compare
+export dest6.o_CFLAGS_ADD=-Wno-sign-compare
+export frag6.o_CFLAGS_ADD=-Wno-sign-compare
+export icmp6.o_CFLAGS_ADD=-Wno-sign-compare
+export in6.o_CFLAGS_ADD=-Wno-sign-compare
+export in6_src.o_CFLAGS_ADD=-Wno-sign-compare
+export in6_cksum.o_CFLAGS_ADD=-Wno-sign-compare
+export ip6_fw.o_CFLAGS_ADD=-Wno-sign-compare
+export ip6_forward.o_CFLAGS_ADD=-Wno-sign-compare
+export in6_ifattach.o_CFLAGS_ADD=-Wno-sign-compare
+export ip6_input.o_CFLAGS_ADD=-Wno-sign-compare
+export ip6_mroute.o_CFLAGS_ADD=-Wno-sign-compare
+export ip6_output.o_CFLAGS_ADD=-Wno-sign-compare
+export ipcomp_input.o_CFLAGS_ADD=-Wno-sign-compare
+export ipcomp_output.o_CFLAGS_ADD=-Wno-sign-compare
+export in6_proto.o_CFLAGS_ADD=-Wno-sign-compare
+export mld6.o_CFLAGS_ADD=-Wno-sign-compare
+export nd6.o_CFLAGS_ADD=-Wno-sign-compare
+export nd6_nbr.o_CFLAGS_ADD=-Wno-sign-compare
+export nd6_rtr.o_CFLAGS_ADD=-Wno-sign-compare
+export raw_ip6.o_CFLAGS_ADD=-Wno-sign-compare
+export route6.o_CFLAGS_ADD=-Wno-sign-compare
+export scope6.o_CFLAGS_ADD=-Wno-sign-compare
+export udp6_usrreq.o_CFLAGS_ADD=-Wno-sign-compare
+export key.o_CFLAGS_ADD=-Wno-sign-compare
+export keysock.o_CFLAGS_ADD=-Wno-sign-compare
+export atp_write.o_CFLAGS_ADD=-Wno-sign-compare
+export keydb.o_CFLAGS_ADD=-Wno-sign-compare
+export des_setkey.o_CFLAGS_ADD=-Wno-sign-compare
+export sys_socket.o_CFLAGS_ADD=-Wno-sign-compare
+export sys_glue.o_CFLAGS_ADD=-Wno-sign-compare
+export uipc_domain.o_CFLAGS_ADD=-Wno-sign-compare
+export uipc_mbuf.o_CFLAGS_ADD=-Wno-sign-compare
+export uipc_mbuf2.o_CFLAGS_ADD=-Wno-sign-compare
+export uipc_socket.o_CFLAGS_ADD=-Wno-sign-compare
+export uipc_socket2.o_CFLAGS_ADD=-Wno-sign-compare
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
@@ -22,10 +97,14 @@ ifndef BSD_KERNEL_CONFIG
 export BSD_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(BSD_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -33,33 +112,30 @@ $(COMPOBJROOT)/$(BSD_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(BSD_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(BSD_KERNEL_CONFIG) $(BSD_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(BSD_KERNEL_CONFIG) $(BSD_KERNEL_CONFIG); \
 	);
 
 .ORDER: $(COMPOBJROOT)/$(BSD_KERNEL_CONFIG)/Makefile 
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(BSD_KERNEL_CONFIG)/Makefile 
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(BSD_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(BSD_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(BSD_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(BSD_KERNEL_CONFIG)	\
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(BSD_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all: do_all 
 
diff --git a/bsd/conf/Makefile.i386 b/bsd/conf/Makefile.i386
index cbe71f0a8..07c022208 100644
--- a/bsd/conf/Makefile.i386
+++ b/bsd/conf/Makefile.i386
@@ -8,47 +8,17 @@ CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
 
 # Objects that don't compile cleanly:
 OBJS_NO_WERROR =		\
+	vfs_xattr.o		\
+	vfs_fsevents.o	\
+	fifo_vnops.o	\
+	subr_log.o		\
 	ioconf.o		\
 	aescrypt.o		\
 	aeskey.o		\
 	des_setkey.o		\
 	sha2.o			\
-	shadow.o		\
 	vn.o			\
-	MacOSStubs.o		\
-	hfs_attrlist.o		\
-	hfs_btreeio.o		\
-	hfs_catalog.o		\
-	hfs_chash.o		\
-	hfs_cnode.o		\
-	hfs_encodinghint.o	\
-	hfs_encodings.o		\
-	hfs_endian.o		\
-	hfs_hotfiles.o		\
-	hfs_link.o		\
-	hfs_lookup.o		\
-	hfs_notification.o	\
-	hfs_quota.o		\
-	hfs_readwrite.o		\
-	hfs_search.o		\
-	hfs_vfsops.o		\
-	hfs_vfsutils.o		\
-	hfs_vnops.o		\
-	hfs_xattr.o		\
-	BTree.o			\
-	BTreeAllocate.o		\
-	BTreeMiscOps.o		\
-	BTreeNodeOps.o		\
-	BTreeNodeReserve.o	\
-	BTreeScanner.o		\
 	BTreeTreeOps.o		\
-	CatalogUtilities.o	\
-	FileIDsServices.o	\
-	BTreeWrapper.o		\
-	FileExtentMapping.o	\
-	VolumeAllocation.o	\
-	UnicodeWrappers.o	\
-	cd9660_bmap.o		\
 	cd9660_lookup.o		\
 	cd9660_node.o		\
 	cd9660_rrip.o		\
@@ -56,127 +26,24 @@ OBJS_NO_WERROR =		\
 	cd9660_vfsops.o		\
 	cd9660_vnops.o		\
 	bsd_init.o		\
-	bsd_stubs.o		\
-	kdebug.o		\
-	kern_acct.o		\
-	kern_aio.o		\
-	kern_audit.o		\
-	kern_authorization.o	\
-	kern_bsm_audit.o	\
-	kern_bsm_klib.o		\
-	kern_bsm_token.o	\
-	kern_clock.o		\
-	kern_control.o		\
-	kern_core.o		\
-	kern_credential.o	\
-	kern_descrip.o		\
-	kern_event.o		\
-	kern_exec.o		\
-	kern_exit.o		\
-	kern_fork.o		\
-	kern_ktrace.o		\
-	kern_lock.o		\
-	kern_malloc.o		\
-	kern_mib.o		\
-	kern_mman.o		\
 	kern_newsysctl.o	\
-	kern_panicinfo.o	\
-	kern_pcsamples.o	\
-	kern_physio.o		\
 	kern_prot.o		\
-	kern_resource.o		\
-	kern_shutdown.o		\
-	kern_sig.o		\
-	kern_symfile.o		\
-	kern_synch.o		\
-	kern_sysctl.o		\
-	kern_time.o		\
-	kern_xxx.o		\
-	kpi_mbuf.o		\
 	kpi_socket.o		\
 	kpi_socketfilter.o	\
-	mach_fat.o		\
-	mach_header.o		\
-	mach_loader.o		\
-	mach_process.o		\
-	netboot.o		\
-	posix_sem.o		\
-	posix_shm.o		\
-	qsort.o			\
-	spl.o			\
-	subr_log.o		\
-	subr_prf.o		\
-	subr_prof.o		\
-	subr_xxx.o		\
-	sys_domain.o		\
-	sys_generic.o		\
-	sys_socket.o		\
-	sysctl_init.o		\
-	sysv_ipc.o		\
-	sys_pipe.o		\
-	sysv_sem.o		\
-	sysv_shm.o		\
-	tty.o			\
-	tty_compat.o		\
-	tty_conf.o		\
-	tty_pty.o		\
-	tty_subr.o		\
-	tty_tty.o		\
-	ubc_subr.o		\
 	uipc_domain.o		\
-	uipc_mbuf.o		\
-	uipc_mbuf2.o		\
 	uipc_proto.o		\
-	uipc_socket.o		\
-	uipc_socket2.o		\
-	uipc_syscalls.o		\
 	uipc_usrreq.o		\
-	random.o		\
-	dead_vnops.o		\
-	devfs_tree.o		\
-	devfs_vfsops.o		\
-	devfs_vnops.o		\
-	fdesc_vfsops.o		\
-	fdesc_vnops.o		\
-	fifo_vnops.o		\
-	spec_vnops.o		\
-	synthfs_util.o		\
-	synthfs_vfsops.o	\
-	synthfs_vnops.o		\
-	union_subr.o		\
-	union_vfsops.o		\
-	union_vnops.o		\
-	volfs_vfsops.o		\
-	volfs_vnops.o		\
-	bpf.o			\
-	dlil.o			\
-	ether_at_pr_module.o	\
-	ether_if_module.o	\
-	ether_inet6_pr_module.o	\
-	ether_inet_pr_module.o	\
-	if.o			\
-	if_bond.o		\
+	ether_if_module.o			\
 	if_ethersubr.o		\
 	if_gif.o		\
-	if_loop.o		\
 	if_media.o		\
 	if_stf.o		\
-	if_vlan.o		\
 	kext_net.o		\
-	kpi_interface.o		\
-	kpi_protocol.o		\
-	ndrv.o			\
 	netisr.o		\
-	net_osdep.o		\
-	radix.o			\
-	raw_usrreq.o		\
-	route.o			\
 	rtsock.o		\
-	zlib.o			\
 	dhcp_options.o		\
 	if_ether.o		\
 	igmp.o			\
-	in.o			\
 	in_bootp.o		\
 	in_cksum.o		\
 	ip_fw2.o		\
@@ -184,75 +51,28 @@ OBJS_NO_WERROR =		\
 	kpi_ipfilter.o		\
 	in_gif.o		\
 	in_pcb.o		\
-	in_proto.o		\
-	in_rmx.o		\
 	ip_divert.o		\
 	ip_dummynet.o		\
-	ip_encap.o		\
-	ip_flow.o		\
 	ip_icmp.o		\
 	ip_input.o		\
 	ip_mroute.o		\
 	ip_output.o		\
-	raw_ip.o		\
 	tcp_input.o		\
-	tcp_output.o		\
 	tcp_subr.o		\
 	tcp_timer.o		\
-	tcp_sack.o		\
-	tcp_usrreq.o		\
-	udp_usrreq.o		\
 	ah_core.o		\
-	ah_input.o		\
-	ah_output.o		\
-	dest6.o			\
 	esp_core.o		\
 	esp_input.o		\
-	esp_output.o		\
 	esp_rijndael.o		\
-	frag6.o			\
-	icmp6.o			\
-	in6.o			\
-	in6_cksum.o		\
 	in6_gif.o		\
-	in6_ifattach.o		\
-	in6_pcb.o		\
-	in6_prefix.o		\
 	in6_proto.o		\
-	in6_rmx.o		\
 	in6_src.o		\
-	ip6_forward.o		\
-	ip6_fw.o		\
-	ip6_input.o		\
-	ip6_mroute.o		\
 	ip6_output.o		\
-	ipcomp_core.o		\
-	ipcomp_input.o		\
-	ipcomp_output.o		\
 	ipsec.o			\
-	mld6.o			\
-	nd6.o			\
-	nd6_nbr.o		\
-	nd6_rtr.o		\
 	raw_ip6.o		\
-	route6.o		\
-	scope6.o		\
-	udp6_output.o		\
-	udp6_usrreq.o		\
 	key.o			\
 	keydb.o			\
-	keysock.o		\
 	krpc_subr.o		\
-	nfs_bio.o		\
-	nfs_boot.o		\
-	nfs_node.o		\
-	nfs_nqlease.o		\
-	nfs_socket.o		\
-	nfs_srvcache.o		\
-	nfs_subs.o		\
-	nfs_syscalls.o		\
-	nfs_vfsops.o		\
-	nfs_vnops.o		\
 	ffs_alloc.o		\
 	ffs_balloc.o		\
 	ffs_inode.o		\
@@ -271,95 +91,35 @@ OBJS_NO_WERROR =		\
 	ufs_vfsops.o		\
 	ufs_vnops.o		\
 	ux_exception.o		\
-	vfs_bio.o		\
-	vfs_cache.o		\
-	vfs_cluster.o		\
-	vfs_conf.o		\
-	vfs_fsevents.o		\
-	vfs_init.o		\
 	vfs_journal.o		\
-	vfs_lookup.o		\
-	vfs_quota.o		\
-	vfs_subr.o		\
-	vfs_support.o		\
 	vfs_syscalls.o		\
-	vfs_utfconv.o		\
-	vfs_vnops.o		\
-	vfs_xattr.o		\
-	kpi_vfs.o		\
-	vnode_if.o		\
 	sysctl.o		\
 	unix_startup.o 		\
-	memdev.o		\
 	randomdev.o		\
 	sha1mod.o		\
-	inet_ntop.o		\
-	vfs_attrlist.o	\
-	volfs_vnops.o	\
-	bpf_filter.o	\
-	devtimer.o		\
 	in_arp.o		\
-	ip_ecn.o		\
-	key_debug.o		\
-	nfs_serv.o		\
-	nfs_lock.o		\
-	kern_lockf.o	\
-	kern_subr.o		\
-	sysv_msg.o		\
 	vnode_pager.o	\
 	dp_backing_file.o \
 	vm_unix.o		\
-	param.o			\
 	mem.o			\
 	km.o			\
 	init_sysent.o		\
-	at.o			\
-	adsp.o			\
-	adsp_CLDeny.o		\
-	adsp_CLListen.o		\
-	adsp_Close.o		\
-	adsp_Control.o		\
-	adsp_Init.o		\
-	adsp_InitGlobals.o	\
-	adsp_NewCID.o		\
-	adsp_Open.o		\
-	adsp_Options.o		\
-	adsp_Packet.o		\
-	adsp_Read.o		\
-	adsp_RxAttn.o		\
-	adsp_RxData.o		\
-	adsp_Status.o		\
-	adsp_Timer.o		\
-	adsp_TimerElem.o	\
-	adsp_Write.o		\
-	adsp_attention.o	\
-	adsp_misc.o		\
-	adsp_reset.o		\
-	adsp_stream.o		\
-	at_proto.o		\
-	at_pcb.o		\
-	atp_alloc.o		\
-	atp_misc.o		\
-	atp_open.o		\
-	atp_read.o		\
-	atp_write.o		\
-	ddp_aarp.o		\
-	ddp_aep.o		\
-	ddp_brt.o		\
-	ddp_lap.o		\
-	ddp_nbp.o		\
-	ddp_proto.o		\
-	ddp_r_rtmp.o		\
-	ddp_r_zip.o		\
-	ddp_rtmp.o		\
-	ddp_rtmptable.o		\
-	ddp_sip.o		\
-	ddp.o			\
-	sys_dep.o		\
-	sys_glue.o		\
 	drv_dep.o		\
-	ddp_usrreq.o		\
-	asp_proto.o	
+	dtrace.o		\
+	lockstat.o		\
+	profile_prvd.o		\
+	systrace.o		\
+	blist.o			\
+	dtrace_glue.o		\
+	fbt.o			\
+	fbt_x86.o		\
+	sdt.o       \
+	sdt_subr.o       \
+	sdt_x86.o       \
+	dtrace_isa.o		\
+	dis_tables.o		\
+	aes_modes.o
+
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
 
diff --git a/bsd/conf/Makefile.ppc b/bsd/conf/Makefile.ppc
index 7786ccbd6..ac870fd86 100644
--- a/bsd/conf/Makefile.ppc
+++ b/bsd/conf/Makefile.ppc
@@ -2,6 +2,130 @@
 #BEGIN  Machine dependent Makefile fragment for ppc
 ######################################################################
  
+# Enable -Werror for ppc builds
+CFLAGS+=$(WERROR)
+CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
+
+# Objects that don't compile cleanly:
+OBJS_NO_WERROR =		\
+	vfs_xattr.o		\
+	vfs_fsevents.o	\
+	fifo_vnops.o	\
+	subr_log.o		\
+	ioconf.o		\
+	aescrypt.o		\
+	aeskey.o		\
+	des_setkey.o		\
+	sha2.o			\
+	shadow.o		\
+	vn.o			\
+	cd9660_lookup.o		\
+	cd9660_node.o		\
+	cd9660_rrip.o		\
+	cd9660_util.o		\
+	cd9660_vfsops.o		\
+	cd9660_vnops.o		\
+	bsd_init.o		\
+	bsd_stubs.o		\
+	kern_newsysctl.o	\
+	kern_prot.o		\
+	kpi_socket.o		\
+	kpi_socketfilter.o	\
+	uipc_domain.o		\
+	uipc_proto.o		\
+	uipc_usrreq.o		\
+	ether_if_module.o	\
+	if_ethersubr.o		\
+	if_gif.o		\
+	if_media.o		\
+	if_stf.o		\
+	kext_net.o		\
+	netisr.o		\
+	rtsock.o		\
+	dhcp_options.o		\
+	if_ether.o		\
+	igmp.o			\
+	in_bootp.o		\
+	in_cksum.o		\
+	ip_fw2.o		\
+	ip_fw2_compat.o		\
+	kpi_ipfilter.o		\
+	in_gif.o		\
+	in_pcb.o		\
+	ip_divert.o		\
+	ip_dummynet.o		\
+	ip_icmp.o		\
+	ip_input.o		\
+	ip_mroute.o		\
+	ip_output.o		\
+	tcp_input.o		\
+	tcp_subr.o		\
+	tcp_timer.o		\
+	ah_core.o		\
+	esp_core.o		\
+	esp_input.o		\
+	esp_rijndael.o		\
+	in6_gif.o		\
+	in6_proto.o		\
+	in6_src.o		\
+	ip6_output.o		\
+	ipsec.o			\
+	raw_ip6.o		\
+	key.o			\
+	keydb.o			\
+	krpc_subr.o		\
+	ffs_alloc.o		\
+	ffs_balloc.o		\
+	ffs_inode.o		\
+	ffs_subr.o		\
+	ffs_vfsops.o		\
+	ffs_vnops.o		\
+	ufs_attrlist.o		\
+	ufs_bmap.o		\
+	ufs_byte_order.o	\
+	ufs_ihash.o		\
+	ufs_inode.o		\
+	ufs_lockf.o		\
+	ufs_lookup.o		\
+	ufs_quota.o		\
+	ufs_readwrite.o		\
+	ufs_vfsops.o		\
+	ufs_vnops.o		\
+	ux_exception.o		\
+	vfs_journal.o		\
+	vfs_syscalls.o		\
+	sysctl.o		\
+	unix_startup.o 		\
+	randomdev.o		\
+	sha1mod.o		\
+	devtimer.o		\
+	in_arp.o		\
+	vnode_pager.o	\
+	dp_backing_file.o \
+	vm_unix.o		\
+	mem.o			\
+	km.o			\
+	at.o			\
+	drv_dep.o		\
+	dtrace.o		\
+	lockstat.o		\
+	profile_prvd.o		\
+	systrace.o		\
+	blist.o			\
+	dtrace_glue.o		\
+	fbt.o			\
+	fbt_ppc.o		\
+	sdt.o		\
+	sdt_subr.o		\
+	sdt_ppc.o		\
+	dtrace_isa.o		\
+	dtrace_subr_ppc.o
+
+
+OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
+
+$(OBJS_WERROR):		WERROR=-Werror
+
 ######################################################################
 #END    Machine dependent Makefile fragment for ppc
 ######################################################################
diff --git a/bsd/conf/Makefile.template b/bsd/conf/Makefile.template
index 7ba478229..1990dd5b6 100644
--- a/bsd/conf/Makefile.template
+++ b/bsd/conf/Makefile.template
@@ -46,7 +46,7 @@ include $(MakeInc_def)
 CFLAGS+= -imacros meta_features.h -DARCH_PRIVATE -DKERNEL -DDRIVER_PRIVATE \
 	-D_KERNEL_BUILD -DKERNEL_BUILD -DMACH_KERNEL -DBSD_BUILD \
 	-DBSD_KERNEL_PRIVATE -DNCPUS=1 -Wno-four-char-constants -fpascal-strings \
-	-D__APPLE__ -DLP64KERN=1 -DLP64_DEBUG=0  -I.
+	-D__APPLE__ -DLP64KERN=1 -DLP64_DEBUG=0  -I. $(CFLAGS_INLINE_CONFIG)
 
 # XXX: ld flags for bsd.o
 export LDFLAGS_COMPONENT += -keep_private_externs
@@ -103,12 +103,11 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS)
-	@echo "[ creating $(COMPONENT).o ]"
-	@echo [ updating $(COMPONENT).o ${BSD_KERNEL_CONFIG} ]
-	$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
+	@echo LD $(COMPONENT)
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`;
+	$(_v)${MD} -u Makedep -f -d `ls *.d`;
 	
 do_all: $(COMPONENT).o
 
@@ -118,4 +117,3 @@ do_build_all: do_depend
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/bsd/conf/compat_hdrs b/bsd/conf/compat_hdrs
index 9229d66db..017a7848d 100644
--- a/bsd/conf/compat_hdrs
+++ b/bsd/conf/compat_hdrs
@@ -9,6 +9,8 @@ kernserv	next	loadable_fs.h
 mach		kern	exc.defs
 mach		kern	exc.h
 mach		kern	mach.defs
+mach		kern	mach_exc.defs
+mach		kern	mach_exc.h
 mach		kern	mach_host.defs
 mach		kern	mach_host.h
 mach		kern	mach_interface.h
diff --git a/bsd/conf/files b/bsd/conf/files
index 268cb3175..502307c92 100644
--- a/bsd/conf/files
+++ b/bsd/conf/files
@@ -12,6 +12,10 @@
 OPTIONS/hw_ast				optional hw_ast
 OPTIONS/hw_footprint			optional hw_footprint
 OPTIONS/kernserv			optional kernserv
+OPTIONS/config_macf			optional config_macf
+OPTIONS/config_macf_socket_subset	optional config_macf_socket_subset
+OPTIONS/config_macf_socket		optional config_macf_socket
+OPTIONS/config_macf_net			optional config_macf_net
 OPTIONS/mach_assert			optional mach_assert
 OPTIONS/mach_compat			optional mach_compat
 OPTIONS/mach_counters			optional mach_counters
@@ -56,17 +60,28 @@ OPTIONS/new_vm_code			optional new_vm_code
 OPTIONS/old_vm_code			optional old_vm_code
 OPTIONS/compat_43			optional compat_43
 OPTIONS/compat_43_tty			optional compat_43_tty
-OPTIONS/compat_43_socket		optional compat_43_socket
 OPTIONS/diagnostic			optional diagnostic
-OPTIONS/ktrace				optional ktrace
+OPTIONS/config_dtrace			optional config_dtrace
 OPTIONS/profiling			optional profiling
 OPTIONS/vndevice			optional vndevice
 OPTIONS/audit				optional audit
-OPTIONS/fsevents			optional fsevents
+OPTIONS/config_fse			optional config_fse
+OPTIONS/sockets				optional sockets
+OPTIONS/kpidirect			optional kpidirect
+OPTIONS/development			optional development
+OPTIONS/sysv_sem			optional sysv_sem
+OPTIONS/sysv_msg			optional sysv_msg
+OPTIONS/sysv_shm			optional sysv_shm
+OPTIONS/panic_info			optional panic_info
+OPTIONS/no_bsd_inlines		optional no_bsd_inlines
 
 #
 # Network options
 #
+OPTIONS/networking			optional networking
+OPTIONS/inet				optional inet
+OPTIONS/inet6				optional inet6
+OPTIONS/ether				optional ether
 OPTIONS/multicast			optional multicast
 OPTIONS/mrouting			optional mrouting
 OPTIONS/routing				optional routing
@@ -79,13 +94,13 @@ OPTIONS/gateway				optional gateway
 OPTIONS/ipx				optional ipx
 OPTIONS/tun				optional tun
 OPTIONS/vlan				optional vlan
+OPTIONS/bond				optional bond
 OPTIONS/bpfilter			optional bpfilter
-OPTIONS/sl				optional sl
-OPTIONS/ppp				optional ppp
+OPTIONS/sl					optional sl
+OPTIONS/ppp					optional ppp
 OPTIONS/sppp				optional sppp
-OPTIONS/ppp_deflate			optional ppp_deflate
+OPTIONS/ppp_deflate			optional ppp_deflate zlib
 OPTIONS/disc				optional disc
-OPTIONS/ether				optional ether
 OPTIONS/fddi				optional fddi
 
 OPTIONS/ipdivert			optional ipdivert
@@ -96,23 +111,33 @@ OPTIONS/ipv6firewall		optional ipv6firewall
 OPTIONS/tcpdebug			optional tcpdebug
 OPTIONS/bridge				optional bridge
 OPTIONS/faith				optional faith
-OPTIONS/gif				optional gif
+OPTIONS/gif					optional gif
 OPTIONS/netat				optional netat
+OPTIONS/sendfile			optional sendfile
+OPTIONS/randomipid			optional randomipid
+
+OPTIONS/zlib				optional zlib
 
 #
 # Filesystem options
 #
-OPTIONS/ffs				optional ffs
-OPTIONS/hfs				optional hfs
-OPTIONS/mfs				optional mfs
+OPTIONS/ffs					optional ffs
+OPTIONS/hfs					optional hfs
+OPTIONS/mfs					optional mfs
 OPTIONS/fdesc				optional fdesc
 OPTIONS/fifo				optional fifo
 OPTIONS/nullfs				optional nullfs
 OPTIONS/union				optional union
 OPTIONS/cd9660				optional cd9660
-OPTIONS/volfs				optional volfs
 OPTIONS/devfs				optional devfs
-OPTIONS/synthfs				optional synthfs
+OPTIONS/crypto				optional crypto
+OPTIONS/allcrypto			optional allcrypto
+OPTIONS/journaling			optional journaling
+OPTIONS/crypto				optional crypto
+OPTIONS/allcrypto			optional allcrypto
+OPTIONS/journaling			optional journaling
+
+OPTIONS/config_imageboot              optional config_imageboot
 
 bsd/dev/random/randomdev.c							standard
 bsd/dev/random/YarrowCoreLib/port/smf.c				standard
@@ -132,6 +157,7 @@ bsd/libkern/crc32.c			standard
 bsd/libkern/random.c			standard
 bsd/libkern/scanc.c			standard
 bsd/libkern/skpc.c			standard
+bsd/libkern/strsep.c			standard
 bsd/libkern/inet_ntop.c			standard
 bsd/libkern/bcd.c			standard
 
@@ -140,9 +166,10 @@ bsd/vfs/vfs_bio.c			standard
 bsd/vfs/vfs_cache.c			standard
 bsd/vfs/vfs_cluster.c			standard
 bsd/vfs/vfs_conf.c			standard
+bsd/vfs/vfs_fslog.c			standard
 bsd/vfs/vfs_init.c			standard
 bsd/vfs/vfs_lookup.c			standard
-bsd/vfs/vfs_quota.c			standard
+bsd/vfs/vfs_quota.c			optional quota
 bsd/vfs/vfs_subr.c			standard
 bsd/vfs/vfs_syscalls.c			standard
 bsd/vfs/vfs_support.c			standard
@@ -151,14 +178,13 @@ bsd/vfs/vfs_vnops.c			standard
 bsd/vfs/vfs_xattr.c			standard
 bsd/vfs/vnode_if.c			standard
 bsd/vfs/kpi_vfs.c			standard
-bsd/vfs/vfs_journal.c			standard
-#bsd/vfs/vfs_fsevents.c			optional fsevents
+bsd/vfs/vfs_journal.c			optional journaling
 bsd/vfs/vfs_fsevents.c			standard
 
 bsd/miscfs/deadfs/dead_vnops.c		standard
 bsd/miscfs/fdesc/fdesc_vfsops.c		optional fdesc
 bsd/miscfs/fdesc/fdesc_vnops.c		optional fdesc
-bsd/miscfs/fifofs/fifo_vnops.c		optional fifo
+bsd/miscfs/fifofs/fifo_vnops.c		optional fifo sockets
 bsd/miscfs/nullfs/null_subr.c		optional nullfs
 bsd/miscfs/nullfs/null_vfsops.c		optional nullfs
 bsd/miscfs/nullfs/null_vnops.c		optional nullfs
@@ -167,17 +193,10 @@ bsd/miscfs/union/union_subr.c		optional union
 bsd/miscfs/union/union_vfsops.c		optional union
 bsd/miscfs/union/union_vnops.c		optional union
 
-bsd/miscfs/volfs/volfs_vfsops.c		optional volfs
-bsd/miscfs/volfs/volfs_vnops.c		optional volfs
-
 bsd/miscfs/devfs/devfs_tree.c		optional devfs
 bsd/miscfs/devfs/devfs_vnops.c		optional devfs
 bsd/miscfs/devfs/devfs_vfsops.c		optional devfs
 
-bsd/miscfs/synthfs/synthfs_vfsops.c	optional synthfs
-bsd/miscfs/synthfs/synthfs_vnops.c	optional synthfs
-bsd/miscfs/synthfs/synthfs_util.c	optional synthfs
-
 bsd/isofs/cd9660/cd9660_bmap.c		optional cd9660
 bsd/isofs/cd9660/cd9660_lookup.c	optional cd9660
 bsd/isofs/cd9660/cd9660_node.c		optional cd9660
@@ -186,88 +205,77 @@ bsd/isofs/cd9660/cd9660_util.c		optional cd9660
 bsd/isofs/cd9660/cd9660_vfsops.c	optional cd9660
 bsd/isofs/cd9660/cd9660_vnops.c		optional cd9660
 
-#bsd/net/slcompress.c		         optional i4bipr
 bsd/net/bpf.c				optional bpfilter
 bsd/net/bpf_filter.c			optional bpfilter
 bsd/net/bridge.c			optional bridge
 bsd/net/bsd_comp.c			optional ppp_bsdcomp
-bsd/net/if.c				standard
+bsd/net/if.c				optional networking
 bsd/net/if_atmsubr.c			optional atm
 bsd/net/if_disc.c			optional disc
-bsd/net/init.c				standard
-bsd/net/dlil.c				standard
+bsd/net/init.c				optional sockets
+bsd/net/dlil.c				optional networking
 bsd/net/ether_if_module.c		optional ether
-bsd/net/ether_at_pr_module.c		optional ether
-bsd/net/ether_inet_pr_module.c		optional ether
+bsd/net/ether_at_pr_module.c		optional ether netat
+bsd/net/ether_inet_pr_module.c		optional ether inet
 bsd/net/ether_inet6_pr_module.c		optional ether inet6
-#bsd/net/if_ethersubr.c			optional ether
 bsd/net/if_loop.c			optional loop
-#bsd/net/if_media.c			standard
 bsd/net/if_mib.c			optional netmibs
 bsd/net/if_sl.c				optional sl
 bsd/net/if_tun.c			optional tun
 bsd/net/if_vlan.c			optional vlan
-bsd/net/multicast_list.c		standard
+bsd/net/multicast_list.c		optional networking
 bsd/net/if_bond.c			optional bond
 bsd/net/devtimer.c			optional bond
-#bsd/net/kext_net.c		standard
-bsd/net/ndrv.c				standard
+bsd/net/ndrv.c				optional networking
 bsd/net/ppp_deflate.c			optional ppp_deflate
-bsd/net/radix.c				standard
-bsd/net/raw_cb.c			standard
-bsd/net/raw_usrreq.c			standard
-bsd/net/route.c				standard
-bsd/net/rtsock.c			standard
+bsd/net/radix.c				optional networking
+bsd/net/raw_cb.c			optional networking
+bsd/net/raw_usrreq.c			optional networking
+bsd/net/route.c				optional networking
+bsd/net/rtsock.c			optional networking
 bsd/net/slcompress.c			optional ppp
 bsd/net/slcompress.c			optional sl
-bsd/net/zlib.c				optional ppp_deflate
-#bsd/net/netisr.c			standard
-bsd/net/zlib.c				optional ipsec 
 bsd/net/if_dummy.c         		optional dummy
 bsd/net/if_gif.c          		optional gif
 bsd/net/if_stf.c          		optional stf
-#bsd/net/radish.c          		standard
-bsd/net/if_faith.c          		optional faith
-bsd/net/net_osdep.c			optional ipsec 
-bsd/net/net_osdep.c			optional inet6
-bsd/net/kpi_interface.c		standard
-bsd/net/kpi_protocol.c		standard
-bsd/net/kpi_interfacefilter.c	standard
+bsd/net/net_osdep.c			optional sockets
+bsd/net/kpi_interface.c		optional networking
+bsd/net/kpi_protocol.c		optional networking
+bsd/net/kpi_interfacefilter.c	optional networking
 
 
 bsd/netinet/if_atm.c			optional atm
-bsd/netinet/igmp.c			standard
-bsd/netinet/in.c			standard
-bsd/netinet/in_bootp.c			standard
-bsd/netinet/dhcp_options.c		standard
-bsd/netinet/in_arp.c			standard
-bsd/netinet/in_pcb.c			standard
-bsd/netinet/in_proto.c			standard
-bsd/netinet/in_rmx.c			standard
+bsd/netinet/igmp.c			optional inet
+bsd/netinet/in.c			optional inet
+bsd/netinet/in_dhcp.c			optional inet
+bsd/netinet/dhcp_options.c		optional inet
+bsd/netinet/in_arp.c			optional inet
+bsd/netinet/in_pcb.c			optional inet
+bsd/netinet/in_proto.c			optional inet
+bsd/netinet/in_rmx.c			optional inet
 bsd/netinet/ip_divert.c			optional ipdivert
 bsd/netinet/ip_dummynet.c  		optional dummynet
-bsd/netinet/ip_flow.c			standard
+bsd/netinet/ip_flow.c			optional inet
 bsd/netinet/ip_fw2.c			optional ipfw2
 bsd/netinet/ip_fw2_compat.c		optional ipfw2
-bsd/netinet/ip_icmp.c			standard
-bsd/netinet/ip_id.c			optional randomipid
-bsd/netinet/ip_input.c			standard
-bsd/netinet/ip_mroute.c			standard
-bsd/netinet/ip_output.c			standard
-bsd/netinet/raw_ip.c			standard
+bsd/netinet/ip_icmp.c			optional inet
+bsd/netinet/ip_id.c				optional randomipid inet
+bsd/netinet/ip_input.c			optional inet
+bsd/netinet/ip_mroute.c			optional mrouting
+bsd/netinet/ip_output.c			optional inet
+bsd/netinet/raw_ip.c			optional inet
 bsd/netinet/tcp_debug.c			optional tcpdebug
-bsd/netinet/tcp_input.c			standard
-bsd/netinet/tcp_output.c		standard
-bsd/netinet/tcp_sack.c			standard
-bsd/netinet/tcp_subr.c			standard
-bsd/netinet/tcp_timer.c			standard
-bsd/netinet/tcp_usrreq.c		standard
-bsd/netinet/udp_usrreq.c		standard
+bsd/netinet/tcp_input.c			optional inet
+bsd/netinet/tcp_output.c		optional inet
+bsd/netinet/tcp_sack.c			optional inet
+bsd/netinet/tcp_subr.c			optional inet
+bsd/netinet/tcp_timer.c			optional inet
+bsd/netinet/tcp_usrreq.c		optional inet
+bsd/netinet/udp_usrreq.c		optional inet
 bsd/netinet/in_gif.c      		optional gif inet
-bsd/netinet/ip_ecn.c          		optional inet inet6
-bsd/netinet/ip_ecn.c          		optional inet ipsec
+bsd/netinet/ip_ecn.c          		optional inet
 bsd/netinet/ip_encap.c         		optional inet 
-bsd/netinet/kpi_ipfilter.c		standard
+bsd/netinet/kpi_ipfilter.c		optional inet
 bsd/netinet6/ah_core.c      		optional ipsec
 bsd/netinet6/ah_input.c     		optional ipsec
 bsd/netinet6/ah_output.c   		optional ipsec
@@ -286,7 +294,7 @@ bsd/netinet6/ip6_fw.c      		    optional inet6
 bsd/netinet6/ip6_forward.c  		optional inet6
 bsd/netinet6/in6_ifattach.c 		optional inet6
 bsd/netinet6/ip6_input.c    		optional inet6
-bsd/netinet6/ip6_mroute.c   		optional inet6 
+bsd/netinet6/ip6_mroute.c   		optional mrouting inet6
 bsd/netinet6/ip6_output.c   		optional inet6
 bsd/netinet6/in6_src.c      		optional inet6
 bsd/netinet6/ipcomp_core.c  		optional ipsec
@@ -311,20 +319,13 @@ bsd/netkey/key_debug.c    		optional ipsec
 bsd/netkey/keysock.c      		optional ipsec
 bsd/netkey/keydb.c      		optional ipsec
 
-bsd/kern/md5c.c    		        optional inet
-bsd/kern/md5c.c    		        optional inet6
-bsd/kern/md5c.c    		        optional crypto
-bsd/crypto/sha1.c         		optional crypto
-bsd/crypto/sha2/sha2.c			optional crypto
+bsd/crypto/sha2/sha2.c			optional crypto allcrypto
 bsd/crypto/des/des_ecb.c  		optional crypto
 bsd/crypto/des/des_enc.c  		optional crypto
 bsd/crypto/des/des_setkey.c       	optional crypto
-bsd/crypto/blowfish/bf_enc.c      	optional crypto
-bsd/crypto/blowfish/bf_skey.c     	optional crypto
-bsd/crypto/cast128/cast128.c      	optional crypto
-bsd/crypto/aes/aescrypt.c		optional crypto
-bsd/crypto/aes/aeskey.c			optional crypto
-bsd/crypto/aes/aestab.c			optional crypto
+bsd/crypto/blowfish/bf_enc.c      	optional crypto allcrypto
+bsd/crypto/blowfish/bf_skey.c     	optional crypto allcrypto
+bsd/crypto/cast128/cast128.c      	optional crypto allcrypto
 bsd/crypto/rc4/rc4.c              	optional crypto
 
 #bsd/netpm/pm_aTT.c        optional pm
@@ -401,36 +402,43 @@ bsd/netat/aurp_zi.c			optional netat
 bsd/nfs/krpc_subr.c			optional nfsclient
 bsd/nfs/nfs_bio.c			optional nfsclient
 bsd/nfs/nfs_boot.c			optional nfsclient
+bsd/nfs/nfs_gss.c			optional nfsclient
+bsd/nfs/nfs_gss.c			optional nfsserver
+bsd/nfs/nfs_lock.c			optional nfsclient
 bsd/nfs/nfs_node.c			optional nfsclient
 bsd/nfs/nfs_serv.c			optional nfsserver
-bsd/nfs/nfs_socket.c			optional nfsclient nfsserver
+bsd/nfs/nfs_socket.c			optional nfsclient
+bsd/nfs/nfs_socket.c			optional nfsserver
 bsd/nfs/nfs_srvcache.c			optional nfsserver
-bsd/nfs/nfs_subs.c			optional nfsclient nfsserver
-bsd/nfs/nfs_syscalls.c			optional nfsclient nfsserver
+bsd/nfs/nfs_subs.c			optional nfsclient
+bsd/nfs/nfs_subs.c			optional nfsserver
+bsd/nfs/nfs_syscalls.c			optional nfsclient
+bsd/nfs/nfs_syscalls.c			optional nfsserver
 bsd/nfs/nfs_vfsops.c			optional nfsclient
 bsd/nfs/nfs_vnops.c			optional nfsclient
-bsd/nfs/nfs_lock.c			optional nfsclient
+bsd/nfs/nfs4_subs.c			optional nfsclient
+bsd/nfs/nfs4_vnops.c			optional nfsclient
 
 bsd/kern/netboot.c			optional nfsclient
 
-bsd/ufs/ffs/ffs_alloc.c			standard
-bsd/ufs/ffs/ffs_balloc.c		standard
-bsd/ufs/ffs/ffs_inode.c			standard
-bsd/ufs/ffs/ffs_subr.c			standard
-bsd/ufs/ffs/ffs_tables.c		standard
-bsd/ufs/ffs/ffs_vfsops.c		standard
-bsd/ufs/ffs/ffs_vnops.c			standard
+bsd/ufs/ffs/ffs_alloc.c			optional ffs
+bsd/ufs/ffs/ffs_balloc.c		optional ffs
+bsd/ufs/ffs/ffs_inode.c			optional ffs
+bsd/ufs/ffs/ffs_subr.c			optional ffs
+bsd/ufs/ffs/ffs_tables.c		optional ffs
+bsd/ufs/ffs/ffs_vfsops.c		optional ffs
+bsd/ufs/ffs/ffs_vnops.c			optional ffs
 bsd/ufs/mfs/mfs_vfsops.c		optional mfs
 bsd/ufs/mfs/mfs_vnops.c			optional mfs
-bsd/ufs/ufs/ufs_attrlist.c		standard
-bsd/ufs/ufs/ufs_bmap.c			standard
-bsd/ufs/ufs/ufs_byte_order.c		optional rev_endian_fs
-bsd/ufs/ufs/ufs_ihash.c			standard
-bsd/ufs/ufs/ufs_inode.c			standard
-bsd/ufs/ufs/ufs_lookup.c		standard
+bsd/ufs/ufs/ufs_attrlist.c		optional ffs
+bsd/ufs/ufs/ufs_bmap.c			optional ffs
+bsd/ufs/ufs/ufs_byte_order.c	optional rev_endian_fs
+bsd/ufs/ufs/ufs_ihash.c			optional ffs
+bsd/ufs/ufs/ufs_inode.c			optional ffs
+bsd/ufs/ufs/ufs_lookup.c		optional ffs
 bsd/ufs/ufs/ufs_quota.c			optional quota
-bsd/ufs/ufs/ufs_vfsops.c		standard
-bsd/ufs/ufs/ufs_vnops.c			standard
+bsd/ufs/ufs/ufs_vfsops.c		optional ffs
+bsd/ufs/ufs/ufs_vnops.c			optional ffs
 
 bsd/hfs/hfs_attrlist.c				optional hfs
 bsd/hfs/hfs_btreeio.c				optional hfs
@@ -472,27 +480,25 @@ bsd/kern/init_sysent.c			standard
 bsd/kern/kdebug.c			standard
 bsd/kern/kern_acct.c			standard
 bsd/kern/kern_aio.c			standard
-bsd/kern/kern_audit.c			standard
-bsd/kern/kern_authorization.c		standard
-bsd/kern/kern_bsm_token.c		standard
-bsd/kern/kern_bsm_audit.c		standard
-bsd/kern/kern_bsm_klib.c		standard
+bsd/kern/kern_audit.c			optional audit
+bsd/kern/kern_authorization.c	standard
+bsd/kern/kern_bsm_token.c		optional audit
+bsd/kern/kern_bsm_audit.c		optional audit
+bsd/kern/kern_bsm_klib.c		optional audit
 bsd/kern/kern_clock.c			standard
 bsd/kern/kern_core.c			standard
 bsd/kern/kern_credential.c		standard
 bsd/kern/kern_symfile.c			standard
 bsd/kern/kern_descrip.c			standard
 bsd/kern/kern_event.c			standard
-bsd/kern/kern_control.c			standard
+bsd/kern/kern_control.c			optional networking
 bsd/kern/kern_exec.c			standard
 bsd/kern/kern_exit.c			standard
 bsd/kern/kern_lockf.c			standard
 bsd/kern/kern_fork.c			standard
-bsd/kern/kern_ktrace.c			standard 
-bsd/kern/kern_lock.c			optional cpus
 bsd/kern/kern_malloc.c			standard
 bsd/kern/kern_mman.c			standard
-bsd/kern/kern_panicinfo.c		standard
+bsd/kern/kern_panicinfo.c		optional panic_info
 bsd/kern/kern_physio.c			standard
 bsd/kern/kern_proc.c			standard
 bsd/kern/kern_prot.c			standard
@@ -503,38 +509,41 @@ bsd/kern/kern_subr.c			standard
 bsd/kern/kern_synch.c			standard
 bsd/kern/kern_sysctl.c			standard
 bsd/kern/kern_newsysctl.c		standard
+bsd/kern/kern_memorystatus.c		optional config_embedded
 bsd/kern/kern_mib.c			standard
-bsd/kern/kpi_mbuf.c			standard
+bsd/kern/kpi_mbuf.c			optional sockets
 bsd/kern/sysctl_init.c			standard
 bsd/kern/kern_time.c			standard
 bsd/kern/kern_xxx.c			standard
 bsd/kern/mach_process.c			standard
-bsd/kern/kern_pcsamples.c		standard
+bsd/kern/mcache.c			optional sockets
 bsd/kern/spl.c				standard
 bsd/kern/subr_log.c			standard
 bsd/kern/subr_prf.c			standard
 bsd/kern/subr_prof.c			standard
+bsd/kern/subr_sbuf.c			standard
 bsd/kern/subr_xxx.c			standard
 bsd/kern/sys_generic.c			standard
 bsd/kern/sys_pipe.c			standard
-bsd/kern/sys_socket.c			standard
-bsd/kern/sys_domain.c	                standard
+bsd/kern/sys_socket.c			optional sockets
+bsd/kern/sys_domain.c			optional sockets
 bsd/kern/syscalls.c			standard
 bsd/kern/tty.c				standard
 bsd/kern/tty_compat.c			optional compat_43_tty
 bsd/kern/tty_conf.c			standard
 bsd/kern/tty_pty.c			optional pty
+bsd/kern/tty_ptmx.c			optional ptmx
 bsd/kern/tty_subr.c			standard
 bsd/kern/tty_tty.c			standard
 bsd/kern/ubc_subr.c			standard
-bsd/kern/uipc_domain.c			standard
-bsd/kern/uipc_mbuf.c			standard
-bsd/kern/uipc_mbuf2.c		 	standard	
-bsd/kern/uipc_proto.c			standard
-bsd/kern/uipc_socket.c			standard
-bsd/kern/uipc_socket2.c			standard
-bsd/kern/uipc_syscalls.c		standard
-bsd/kern/uipc_usrreq.c			standard
+bsd/kern/uipc_domain.c			optional sockets
+bsd/kern/uipc_mbuf.c			optional sockets
+bsd/kern/uipc_mbuf2.c		 	optional sockets
+bsd/kern/uipc_proto.c			optional sockets
+bsd/kern/uipc_socket.c			optional sockets
+bsd/kern/uipc_socket2.c			optional sockets
+bsd/kern/uipc_syscalls.c		optional sockets
+bsd/kern/uipc_usrreq.c			optional sockets
 bsd/kern/sysv_ipc.c			standard
 bsd/kern/sysv_shm.c			standard
 bsd/kern/sysv_sem.c			standard
@@ -545,14 +554,16 @@ bsd/kern/mach_loader.c			standard
 bsd/kern/posix_sem.c			standard
 bsd/kern/posix_shm.c			standard
 # XXXdbg - I need this in the journaling and block cache code
-bsd/kern/qsort.c			standard
-bsd/kern/kpi_socket.c		standard
-bsd/kern/kpi_socketfilter.c	standard
+bsd/kern/qsort.c				standard
+bsd/kern/kpi_socket.c			optional sockets
+bsd/kern/kpi_socketfilter.c		optional sockets
+bsd/kern/pthread_support.c	standard
+bsd/kern/pthread_synch.c	standard
 bsd/kern/proc_info.c			standard
-bsd/kern/socket_info.c			standard
+bsd/kern/socket_info.c			optional sockets
 
 bsd/vm/vnode_pager.c			standard
-bsd/vm/vm_unix.c			standard
+bsd/vm/vm_unix.c				standard
 bsd/vm/dp_backing_file.c		standard
 
 bsd/uxkern/ux_exception.c		standard
@@ -562,3 +573,19 @@ bsd/conf/param.c			standard
 
 bsd/dev/chud/chud_bsd_callback.c	standard
 bsd/dev/chud/chud_process.c		standard
+
+bsd/dev/dtrace/dtrace.c			optional config_dtrace
+bsd/dev/dtrace/lockstat.c		optional config_dtrace
+bsd/dev/dtrace/dtrace_ptss.c		optional config_dtrace
+bsd/dev/dtrace/dtrace_subr.c		optional config_dtrace
+bsd/dev/dtrace/dtrace_glue.c		standard
+bsd/dev/dtrace/dtrace_alloc.c		optional config_dtrace
+bsd/dev/dtrace/blist.c			optional config_dtrace
+bsd/dev/dtrace/fbt.c			optional config_dtrace
+bsd/dev/dtrace/sdt.c			optional config_dtrace
+bsd/dev/dtrace/sdt_subr.c		optional config_dtrace
+bsd/dev/dtrace/systrace.c		optional config_dtrace
+bsd/dev/dtrace/profile_prvd.c		optional config_dtrace
+bsd/dev/dtrace/fasttrap.c		optional config_dtrace
+
+bsd/kern/imageboot.c                  optional config_imageboot
diff --git a/bsd/conf/files.i386 b/bsd/conf/files.i386
index 3eb14dd0d..024d69708 100644
--- a/bsd/conf/files.i386
+++ b/bsd/conf/files.i386
@@ -15,8 +15,18 @@ bsd/dev/i386/lock_stubs.c	standard
 bsd/dev/i386/sysctl.c           standard
 bsd/dev/i386/unix_signal.c	standard
 bsd/dev/i386/munge.s		standard
+bsd/crypto/aes/i386/aes_x86_v2.s		optional crypto
+bsd/crypto/aes/i386/aes_modes.c			optional crypto
 
 
+bsd/dev/i386/dtrace_isa.c	optional config_dtrace
+bsd/dev/i386/dtrace_subr_x86.c	optional config_dtrace
+bsd/dev/i386/fbt_x86.c		optional config_dtrace
+bsd/dev/i386/sdt_x86.c		optional config_dtrace
+bsd/dev/i386/fasttrap_isa.c	optional config_dtrace
+bsd/dev/i386/instr_size.c	optional config_dtrace
+bsd/dev/i386/dis_tables.c	optional config_dtrace
+
 bsd/kern/bsd_stubs.c		standard
-bsd/netinet/in_cksum.c		standard
+bsd/netinet/in_cksum.c		optional inet
 
diff --git a/bsd/conf/files.ppc b/bsd/conf/files.ppc
index c24d147e5..57e8870a7 100644
--- a/bsd/conf/files.ppc
+++ b/bsd/conf/files.ppc
@@ -2,7 +2,7 @@ OPTIONS/show_space		optional show_space
 OPTIONS/gdb			optional gdb
 OPTIONS/iplmeas			optional iplmeas
 
-bsd/netinet/in_cksum.c		standard
+bsd/netinet/in_cksum.c		optional inet
 
 bsd/dev/ppc/conf.c		standard
 bsd/dev/ppc/cons.c		standard
@@ -18,6 +18,17 @@ bsd/dev/ppc/km.c		standard
 bsd/dev/ppc/xsumas.s		standard	
 bsd/dev/ppc/sysctl.c		standard	
 bsd/dev/ppc/munge.s		standard
+bsd/crypto/aes/ppc/aescrypt.c		optional crypto
+bsd/crypto/aes/ppc/aeskey.c			optional crypto
+bsd/crypto/aes/ppc/aestab.c			optional crypto
+
+
+bsd/dev/ppc/dtrace_isa.c	optional config_dtrace
+bsd/dev/ppc/dtrace_subr_ppc.c	optional config_dtrace
+bsd/dev/ppc/fbt_ppc.c		optional config_dtrace
+bsd/dev/ppc/sdt_ppc.c		optional config_dtrace
+bsd/dev/ppc/fasttrap_isa.c	optional config_dtrace
+
+bsd/kern/bsd_stubs.c			standard
 
-bsd/kern/bsd_stubs.c		standard
 
diff --git a/bsd/conf/param.c b/bsd/conf/param.c
index 2a4e0e347..5e7191195 100644
--- a/bsd/conf/param.c
+++ b/bsd/conf/param.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1989, 1993
@@ -82,12 +88,11 @@ struct	timezone tz = { TIMEZONE, PST };
 #define HNPROC 2500	/* based on thread_max */
 int	maxproc = NPROC;
 int	maxprocperuid = NPROC/2;
-__private_extern__ int hard_maxproc = HNPROC;	/* hardcoded limit */
+/*__private_extern__*/ int hard_maxproc = HNPROC;	/* hardcoded limit */
 int nprocs = 0; /* XXX */
 
-#define	NTEXT (80 + NPROC / 8)			/* actually the object cache */
-#define	NVNODE (NPROC + NTEXT + 300)
-int	desiredvnodes = NVNODE + 700;
+//#define	NTEXT (80 + NPROC / 8)			/* actually the object cache */
+int    desiredvnodes = CONFIG_VNODES;
 
 #define MAXFILES (OPEN_MAX + 2048)
 int	maxfiles = MAXFILES;
@@ -102,12 +107,9 @@ int	maxsockets = MAXSOCKETS;
 /*
  *  async IO (aio) configurable limits
  */
-#define AIO_MAX				90	/* system wide limit of async IO requests */
-#define AIO_PROCESS_MAX		AIO_LISTIO_MAX 	/* process limit of async IO requests */
-#define AIO_THREAD_COUNT	4	/* number of async IO worker threads created */
-int aio_max_requests = AIO_MAX;
-int aio_max_requests_per_process = AIO_PROCESS_MAX;
-int aio_worker_threads = AIO_THREAD_COUNT;
+int aio_max_requests = CONFIG_AIO_MAX;
+int aio_max_requests_per_process = CONFIG_AIO_PROCESS_MAX;
+int aio_worker_threads = CONFIG_AIO_THREAD_COUNT;
 
 /*
  * These have to be allocated somewhere; allocating
@@ -116,8 +118,8 @@ int aio_worker_threads = AIO_THREAD_COUNT;
  */
 struct 	callout *callout;
 struct	cblock *cfree;
-struct	cblock *cfreelist = 0;
+struct	cblock *cfreelist = NULL;
 int	cfreecount = 0;
-struct	buf *buf;
+struct	buf *buf_headers;
 struct	domain *domains;
 
diff --git a/bsd/conf/tools/doconf/Makefile b/bsd/conf/tools/doconf/Makefile
index 2bf0b7a10..7794a4ceb 100644
--- a/bsd/conf/tools/doconf/Makefile
+++ b/bsd/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,23 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/bsd/conf/tools/doconf/doconf.csh b/bsd/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/bsd/conf/tools/doconf/doconf.csh
+++ b/bsd/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/bsd/crypto/Makefile b/bsd/crypto/Makefile
index e878376c2..90a8d7eec 100644
--- a/bsd/crypto/Makefile
+++ b/bsd/crypto/Makefile
@@ -14,20 +14,24 @@ INSTINC_SUBDIRS = \
     rc4 \
 	aes \
 	sha2
-	
+
 
 INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 PRIVATE_DATAFILES = \
-	sha1.h md5.h
+	sha1.h
 
 INSTALL_MI_DIR = crypto
 
diff --git a/bsd/crypto/aes/Makefile b/bsd/crypto/aes/Makefile
index 9a6c0e847..f34372331 100644
--- a/bsd/crypto/aes/Makefile
+++ b/bsd/crypto/aes/Makefile
@@ -13,14 +13,18 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 PRIVATE_DATAFILES = \
-	aes.h	aesopt.h	aestab.h
+	aes.h
 
 INSTALL_MI_DIR = crypto
 
diff --git a/bsd/crypto/aes/aes.h b/bsd/crypto/aes/aes.h
index 8f8cefe0f..f75d02272 100644
--- a/bsd/crypto/aes/aes.h
+++ b/bsd/crypto/aes/aes.h
@@ -27,17 +27,15 @@
  in respect of its properties, including, but not limited to, correctness
  and/or fitness for purpose.
  ---------------------------------------------------------------------------
- Issue 28/01/2004
+ Issue 31/01/2006
 
  This file contains the definitions required to use AES in C. See aesopt.h
  for optimisation details.
 */
 
-#if !defined( _AES_H )
+#ifndef _AES_H
 #define _AES_H
 
-/*  This include is used to find 8 & 32 bit unsigned integer types  */
-#include <machine/limits.h>
 
 #if defined(__cplusplus)
 extern "C"
@@ -48,34 +46,30 @@ extern "C"
 #define AES_192     /* define if AES with 192 bit keys is needed    */
 #define AES_256     /* define if AES with 256 bit keys is needed    */
 #define AES_VAR     /* define if a variable key size is needed      */
+#define AES_MODES   /* define if support is needed for modes        */
 
 /* The following must also be set in assembler files if being used  */
 
 #define AES_ENCRYPT /* if support for encryption is needed          */
 #define AES_DECRYPT /* if support for decryption is needed          */
-//#define AES_ERR_CHK /* for parameter checks & error return codes    */
-
-#if UCHAR_MAX == 0xff                   /* an unsigned 8 bit type   */
-  typedef unsigned char      aes_08t;
-#else
-#  error Please define aes_08t as an 8-bit unsigned integer type in aes.h
-#endif
-
-#if UINT_MAX == 4294967295              /* an unsigned 32 bit type  */
-  typedef   unsigned int     aes_32t;
-#elif ULONG_MAX == 4294967295ul
-  typedef   unsigned long    aes_32t;
-#else
-#  error Please define aes_32t as a 32-bit unsigned integer type in aes.h
-#endif
+#define AES_ERR_CHK /* for parameter checks & error return codes    */
+#define AES_REV_DKS /* define to reverse decryption key schedule    */
 
 #define AES_BLOCK_SIZE  16  /* the AES block size in bytes          */
 #define N_COLS           4  /* the number of columns in the state   */
 
+typedef	unsigned long   uint_32t;
+typedef unsigned char   uint_8t;
+typedef unsigned short  uint_16t;
+typedef unsigned char   aes_08t;
+typedef	unsigned long   aes_32t;
+
+#define void_ret  void
+#define int_ret   int
+
 /* The key schedule length is 11, 13 or 15 16-byte blocks for 128,  */
 /* 192 or 256-bit keys respectively. That is 176, 208 or 240 bytes  */
-/* or 44, 52 or 60 32-bit words. For simplicity this code allocates */
-/* the maximum 60 word array for the key schedule for all key sizes */
+/* or 44, 52 or 60 32-bit words.                                    */
 
 #if defined( AES_VAR ) || defined( AES_256 )
 #define KS_LENGTH       60
@@ -85,6 +79,38 @@ extern "C"
 #define KS_LENGTH       44
 #endif
 
+
+
+/* the character array 'inf' in the following structures is used    */
+/* to hold AES context information. This AES code uses cx->inf.b[0] */
+/* to hold the number of rounds multiplied by 16. The other three   */
+/* elements can be used by code that implements additional modes    */
+
+#if defined (__i386__)
+
+#if defined( AES_ERR_CHK )
+#define aes_rval     int_ret
+#else
+#define aes_rval     void_ret
+#endif
+
+typedef union
+{   uint_32t l;
+    uint_8t b[4];
+} aes_inf;
+
+typedef struct
+{   uint_32t ks[KS_LENGTH];
+    aes_inf inf;
+} aes_encrypt_ctx;
+
+typedef struct
+{   uint_32t ks[KS_LENGTH];
+    aes_inf inf;
+} aes_decrypt_ctx;
+
+#else
+
 #if defined( AES_ERR_CHK )
 #define aes_ret     int
 #define aes_good    0
@@ -93,11 +119,7 @@ extern "C"
 #define aes_ret     void
 #endif
 
-#if !defined( AES_DLL )                 /* implement normal/DLL functions   */
 #define aes_rval    aes_ret
-#else
-#define aes_rval    aes_ret __declspec(dllexport) _stdcall
-#endif
 
 typedef struct
 {   aes_32t ks[KS_LENGTH];
@@ -109,6 +131,8 @@ typedef struct
     aes_32t rn;
 } aes_decrypt_ctx;
 
+#endif
+
 typedef struct
 {   
 	aes_decrypt_ctx decrypt;
@@ -116,58 +140,70 @@ typedef struct
 } aes_ctx;
 
 
-/* This routine must be called before first use if non-static       */
-/* tables are being used                                            */
+/* implemented in case of wrong call for fixed tables */
 
 void gen_tabs(void);
 
-/* The key length (klen) is input in bytes when it is in the range  */
-/* 16 <= klen <= 32 or in bits when in the range 128 <= klen <= 256 */
+
+/* Key lengths in the range 16 <= key_len <= 32 are given in bytes, */
+/* those in the range 128 <= key_len <= 256 are given in bits       */
 
 #if defined( AES_ENCRYPT )
 
 #if defined(AES_128) || defined(AES_VAR)
-aes_rval aes_encrypt_key128(const unsigned char *in_key, aes_encrypt_ctx cx[1]);
+aes_rval aes_encrypt_key128(const unsigned char *key, aes_encrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_192) || defined(AES_VAR)
-aes_rval aes_encrypt_key192(const unsigned char *in_key, aes_encrypt_ctx cx[1]);
+aes_rval aes_encrypt_key192(const unsigned char *key, aes_encrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_256) || defined(AES_VAR)
-aes_rval aes_encrypt_key256(const unsigned char *in_key, aes_encrypt_ctx cx[1]);
+aes_rval aes_encrypt_key256(const unsigned char *key, aes_encrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_VAR)
-aes_rval aes_encrypt_key(const unsigned char *in_key, int key_len, aes_encrypt_ctx cx[1]);
+aes_rval aes_encrypt_key(const unsigned char *key, int key_len, aes_encrypt_ctx cx[1]);
+#endif
+
+#if defined (__i386__)
+aes_rval aes_encrypt(const unsigned char *in, unsigned char *out, const aes_encrypt_ctx cx[1]);
 #endif
 
 aes_rval aes_encrypt_cbc(const unsigned char *in_blk, const unsigned char *in_iv, unsigned int num_blk,
 					 unsigned char *out_blk, const aes_encrypt_ctx cx[1]);
+
 #endif
 
 #if defined( AES_DECRYPT )
 
 #if defined(AES_128) || defined(AES_VAR)
-aes_rval aes_decrypt_key128(const unsigned char *in_key, aes_decrypt_ctx cx[1]);
+aes_rval aes_decrypt_key128(const unsigned char *key, aes_decrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_192) || defined(AES_VAR)
-aes_rval aes_decrypt_key192(const unsigned char *in_key, aes_decrypt_ctx cx[1]);
+aes_rval aes_decrypt_key192(const unsigned char *key, aes_decrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_256) || defined(AES_VAR)
-aes_rval aes_decrypt_key256(const unsigned char *in_key, aes_decrypt_ctx cx[1]);
+aes_rval aes_decrypt_key256(const unsigned char *key, aes_decrypt_ctx cx[1]);
 #endif
 
 #if defined(AES_VAR)
-aes_rval aes_decrypt_key(const unsigned char *in_key, int key_len, aes_decrypt_ctx cx[1]);
+aes_rval aes_decrypt_key(const unsigned char *key, int key_len, aes_decrypt_ctx cx[1]);
+#endif
+
+#if defined (__i386__)
+aes_rval aes_decrypt(const unsigned char *in, unsigned char *out, const aes_decrypt_ctx cx[1]);
 #endif
 
 aes_rval aes_decrypt_cbc(const unsigned char *in_blk, const unsigned char *in_iv, unsigned int num_blk,
 					 unsigned char *out_blk, const aes_decrypt_ctx cx[1]);
+
+
 #endif
 
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/iokit/include/bsddev/ppc/Makefile b/bsd/crypto/aes/i386/Makefile
similarity index 55%
rename from iokit/include/bsddev/ppc/Makefile
rename to bsd/crypto/aes/i386/Makefile
index b952d4510..f116db347 100644
--- a/iokit/include/bsddev/ppc/Makefile
+++ b/bsd/crypto/aes/i386/Makefile
@@ -3,6 +3,7 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
@@ -12,26 +13,22 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
-EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
-
-EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
+EXPINC_SUBDIRS = \
 
-EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_PPC = \
 
-DATAFILES = \
-            evsio.h		\
-	    evio.h		\
-	    event.h		\
-	    EventShmemLock.h
+EXPINC_SUBDIRS_I386 = \
 
-INSTALL_MD_LIST	= ${DATAFILES}
+PRIVATE_DATAFILES = \
+	aesopt.h edefs.h
 
-INSTALL_MD_DIR = dev/ppc
+INSTALL_MI_DIR = crypto
 
-EXPORT_MD_LIST	= \
+EXPORT_MI_DIR = ${INSTALL_MI_DIR}
 
-EXPORT_MD_DIR = \
+INSTALL_KF_MI_LIST =
 
+INSTALL_KF_MI_LCL_LIST = ${PRIVATE_DATAFILES}
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/bsd/crypto/aes/i386/aes_modes.c b/bsd/crypto/aes/i386/aes_modes.c
new file mode 100644
index 000000000..fd8b1401b
--- /dev/null
+++ b/bsd/crypto/aes/i386/aes_modes.c
@@ -0,0 +1,471 @@
+/*
+ ---------------------------------------------------------------------------
+ Copyright (c) 2003, Dr Brian Gladman, Worcester, UK.   All rights reserved.
+
+ LICENSE TERMS
+
+ The free distribution and use of this software in both source and binary
+ form is allowed (with or without changes) provided that:
+
+   1. distributions of this source code include the above copyright
+      notice, this list of conditions and the following disclaimer;
+
+   2. distributions in binary form include the above copyright
+      notice, this list of conditions and the following disclaimer
+      in the documentation and/or other associated materials;
+
+   3. the copyright holder's name is not used to endorse products
+      built using this software without specific written permission.
+
+ ALTERNATIVELY, provided that this notice is retained in full, this product
+ may be distributed under the terms of the GNU General Public License (GPL),
+ in which case the provisions of the GPL apply INSTEAD OF those given above.
+
+ DISCLAIMER
+
+ This software is provided 'as is' with no explicit or implied warranties
+ in respect of its properties, including, but not limited to, correctness
+ and/or fitness for purpose.
+ ---------------------------------------------------------------------------
+ Issue 31/01/2006
+
+ These subroutines implement multiple block AES modes for ECB, CBC, CFB,
+ OFB and CTR encryption,  The code provides support for the VIA Advanced 
+ Cryptography Engine (ACE).
+
+ NOTE: In the following subroutines, the AES contexts (ctx) must be
+ 16 byte aligned if VIA ACE is being used
+*/
+
+//#include <memory.h>
+#include <kern/assert.h>
+
+#include "aesopt.h"
+
+#if defined( AES_MODES )
+#if defined(__cplusplus)
+extern "C"
+{
+#endif
+
+#if defined( _MSC_VER ) && ( _MSC_VER > 800 )
+#pragma intrinsic(memcpy)
+#define in_line __inline
+#else
+#define in_line
+#endif
+
+#define BFR_BLOCKS      8
+
+/* These values are used to detect long word alignment in order to */
+/* speed up some buffer operations. This facility may not work on  */
+/* some machines so this define can be commented out if necessary  */
+
+#define FAST_BUFFER_OPERATIONS
+#pragma warning( disable : 4311 4312 )
+
+#define lp08(x)         ((uint_8t*)(x))
+#define lp32(x)         ((uint_32t*)(x))
+#define addr_mod_04(x)	((unsigned long)(x) & 3)
+#define addr_mod_16(x)  ((unsigned long)(x) & 15)
+
+#if defined( USE_VIA_ACE_IF_PRESENT )
+
+#include "via_ace.h"
+
+#pragma pack(16)
+
+aligned_array(unsigned long,    enc_gen_table, 12, 16) =    NEH_ENC_GEN_DATA;
+aligned_array(unsigned long,   enc_load_table, 12, 16) =   NEH_ENC_LOAD_DATA;
+aligned_array(unsigned long, enc_hybrid_table, 12, 16) = NEH_ENC_HYBRID_DATA;
+aligned_array(unsigned long,    dec_gen_table, 12, 16) =    NEH_DEC_GEN_DATA;
+aligned_array(unsigned long,   dec_load_table, 12, 16) =   NEH_DEC_LOAD_DATA;
+aligned_array(unsigned long, dec_hybrid_table, 12, 16) = NEH_DEC_HYBRID_DATA;
+
+/* NOTE: These control word macros must only be used after  */
+/* a key has been set up because they depend on key size    */
+
+#if NEH_KEY_TYPE == NEH_LOAD
+#define kd_adr(c)   ((uint_8t*)(c)->ks)
+#elif NEH_KEY_TYPE == NEH_GENERATE
+#define kd_adr(c)   ((uint_8t*)(c)->ks + (c)->inf.b[0])
+#else
+#define kd_adr(c)   ((uint_8t*)(c)->ks + ((c)->inf.b[0] == 160 ? 160 : 0))
+#endif
+
+#else
+
+#define aligned_array(type, name, no, stride) type name[no]
+#define aligned_auto(type, name, no, stride)  type name[no]
+
+#endif
+
+#if defined( _MSC_VER ) && _MSC_VER > 1200
+
+#define via_cwd(cwd, ty, dir, len) unsigned long* cwd = (dir##_##ty##_table + ((len - 128) >> 4)) 
+
+#else
+
+#define via_cwd(cwd, ty, dir, len)				\
+    aligned_auto(unsigned long, cwd, 4, 16);	\
+    cwd[1] = cwd[2] = cwd[3] = 0;				\
+    cwd[0] = neh_##dir##_##ty##_key(len)
+
+#endif
+
+/* implemented in case of wrong call for fixed tables */
+void gen_tabs(void)
+{
+}
+
+aes_rval aes_mode_reset(aes_encrypt_ctx ctx[1])
+{
+    ctx->inf.b[2] = 0; 
+    return 0;
+}
+
+aes_rval aes_ecb_encrypt(const unsigned char *ibuf, unsigned char *obuf,
+                    int len, const aes_encrypt_ctx ctx[1])
+{   int nb = len >> 4;
+
+    if(len & (AES_BLOCK_SIZE - 1))
+        return 1;
+
+#if defined( USE_VIA_ACE_IF_PRESENT )
+
+    if(ctx->inf.b[1] == 0xff)
+    {   uint_8t *ksp = (uint_8t*)(ctx->ks);
+        via_cwd(cwd, hybrid, enc, 2* ctx->inf.b[0] - 192);	
+
+        if(addr_mod_16(ctx))
+            return 1;
+
+        if(!addr_mod_16(ibuf) && !addr_mod_16(obuf))
+        {
+            via_ecb_op5(ksp,cwd,ibuf,obuf,nb);
+        }
+        else
+        {   aligned_auto(uint_8t, buf, BFR_BLOCKS * AES_BLOCK_SIZE, 16);
+            uint_8t *ip, *op;
+
+            while(nb)
+            {
+                int m = (nb > BFR_BLOCKS ? BFR_BLOCKS : nb);
+
+                ip = (addr_mod_16(ibuf) ? buf : (uint_8t*)ibuf);
+                op = (addr_mod_16(obuf) ? buf : obuf);
+
+                if(ip != ibuf)
+                    memcpy(buf, ibuf, m * AES_BLOCK_SIZE);
+
+                via_ecb_op5(ksp,cwd,ip,op,m);
+
+                if(op != obuf)
+                    memcpy(obuf, buf, m * AES_BLOCK_SIZE);
+
+                ibuf += m * AES_BLOCK_SIZE;
+                obuf += m * AES_BLOCK_SIZE;
+                nb -= m;
+            }
+        }
+
+        return 0;
+    }
+
+#endif
+
+#if !defined( ASSUME_VIA_ACE_PRESENT )
+    while(nb--)
+    {
+        aes_encrypt(ibuf, obuf, ctx);
+        ibuf += AES_BLOCK_SIZE;
+        obuf += AES_BLOCK_SIZE;
+    }
+#endif
+    return 0;
+}
+
+aes_rval aes_ecb_decrypt(const unsigned char *ibuf, unsigned char *obuf,
+                    int len, const aes_decrypt_ctx ctx[1])
+{   int nb = len >> 4;
+
+    if(len & (AES_BLOCK_SIZE - 1))
+        return 1;
+
+#if defined( USE_VIA_ACE_IF_PRESENT )
+
+    if(ctx->inf.b[1] == 0xff)
+    {   uint_8t *ksp = kd_adr(ctx);
+        via_cwd(cwd, hybrid, dec, 2* ctx->inf.b[0] - 192);	
+
+        if(addr_mod_16(ctx))
+            return 1;
+
+        if(!addr_mod_16(ibuf) && !addr_mod_16(obuf))
+        {
+            via_ecb_op5(ksp,cwd,ibuf,obuf,nb);
+        }
+        else
+        {   aligned_auto(uint_8t, buf, BFR_BLOCKS * AES_BLOCK_SIZE, 16);
+            uint_8t *ip, *op;
+
+            while(nb)
+            {
+                int m = (nb > BFR_BLOCKS ? BFR_BLOCKS : nb);
+
+                ip = (addr_mod_16(ibuf) ? buf : (uint_8t*)ibuf);
+                op = (addr_mod_16(obuf) ? buf : obuf);
+
+                if(ip != ibuf)
+                    memcpy(buf, ibuf, m * AES_BLOCK_SIZE);
+
+                via_ecb_op5(ksp,cwd,ip,op,m);
+
+                if(op != obuf)
+                    memcpy(obuf, buf, m * AES_BLOCK_SIZE);
+
+                ibuf += m * AES_BLOCK_SIZE;
+                obuf += m * AES_BLOCK_SIZE;
+                nb -= m;
+            }
+        }
+
+        return 0;
+    }
+
+#endif
+
+#if !defined( ASSUME_VIA_ACE_PRESENT )
+    while(nb--)
+    {
+        aes_decrypt(ibuf, obuf, ctx);
+        ibuf += AES_BLOCK_SIZE;
+        obuf += AES_BLOCK_SIZE;
+    }
+#endif
+    return 0;
+}
+
+aes_rval aes_cbc_encrypt(const unsigned char *ibuf, unsigned char *obuf,
+                    int len, unsigned char *iv, const aes_encrypt_ctx ctx[1])
+{   int nb = len >> 4;
+
+    if(len & (AES_BLOCK_SIZE - 1))
+        return 1;
+
+#if defined( USE_VIA_ACE_IF_PRESENT )
+
+    if(ctx->inf.b[1] == 0xff)
+    {   uint_8t *ksp = (uint_8t*)(ctx->ks), *ivp = iv;
+        aligned_auto(uint_8t, liv, AES_BLOCK_SIZE, 16);
+        via_cwd(cwd, hybrid, enc, 2* ctx->inf.b[0] - 192);	
+
+        if(addr_mod_16(ctx))
+            return 1;
+
+        if(addr_mod_16(iv))   /* ensure an aligned iv */
+        {
+            ivp = liv;
+            memcpy(liv, iv, AES_BLOCK_SIZE);
+        }
+
+        if(!addr_mod_16(ibuf) && !addr_mod_16(obuf) && !addr_mod_16(iv))
+        {
+            via_cbc_op7(ksp,cwd,ibuf,obuf,nb,ivp,ivp);
+        }
+        else
+        {   aligned_auto(uint_8t, buf, BFR_BLOCKS * AES_BLOCK_SIZE, 16);
+            uint_8t *ip, *op;
+
+            while(nb)
+            {
+                int m = (nb > BFR_BLOCKS ? BFR_BLOCKS : nb);
+
+                ip = (addr_mod_16(ibuf) ? buf : (uint_8t*)ibuf);
+                op = (addr_mod_16(obuf) ? buf : obuf);
+
+                if(ip != ibuf)
+                    memcpy(buf, ibuf, m * AES_BLOCK_SIZE);
+
+                via_cbc_op7(ksp,cwd,ip,op,m,ivp,ivp);
+
+                if(op != obuf)
+                    memcpy(obuf, buf, m * AES_BLOCK_SIZE);
+
+                ibuf += m * AES_BLOCK_SIZE;
+                obuf += m * AES_BLOCK_SIZE;
+                nb -= m;
+            }
+        }
+
+        if(iv != ivp)
+            memcpy(iv, ivp, AES_BLOCK_SIZE);
+
+        return 0;
+    }
+
+#endif
+
+#if !defined( ASSUME_VIA_ACE_PRESENT )
+# ifdef FAST_BUFFER_OPERATIONS
+    if(!addr_mod_04(ibuf) && !addr_mod_04(iv))
+        while(nb--)
+        {
+            lp32(iv)[0] ^= lp32(ibuf)[0];
+            lp32(iv)[1] ^= lp32(ibuf)[1];
+            lp32(iv)[2] ^= lp32(ibuf)[2];
+            lp32(iv)[3] ^= lp32(ibuf)[3];
+            aes_encrypt(iv, iv, ctx);
+            memcpy(obuf, iv, AES_BLOCK_SIZE);
+            ibuf += AES_BLOCK_SIZE;
+            obuf += AES_BLOCK_SIZE;
+        }
+    else
+# endif
+        while(nb--)
+        {
+            iv[ 0] ^= ibuf[ 0]; iv[ 1] ^= ibuf[ 1];
+            iv[ 2] ^= ibuf[ 2]; iv[ 3] ^= ibuf[ 3];
+            iv[ 4] ^= ibuf[ 4]; iv[ 5] ^= ibuf[ 5];
+            iv[ 6] ^= ibuf[ 6]; iv[ 7] ^= ibuf[ 7];
+            iv[ 8] ^= ibuf[ 8]; iv[ 9] ^= ibuf[ 9];
+            iv[10] ^= ibuf[10]; iv[11] ^= ibuf[11];
+            iv[12] ^= ibuf[12]; iv[13] ^= ibuf[13];
+            iv[14] ^= ibuf[14]; iv[15] ^= ibuf[15];
+            aes_encrypt(iv, iv, ctx);
+            memcpy(obuf, iv, AES_BLOCK_SIZE);
+            ibuf += AES_BLOCK_SIZE;
+            obuf += AES_BLOCK_SIZE;
+        }
+#endif
+    return 0;
+}
+
+aes_rval aes_encrypt_cbc(const unsigned char *in_blk, const unsigned char *in_iv, unsigned int num_blk,
+					 unsigned char *out_blk, const aes_encrypt_ctx cx[1])
+{
+		unsigned char tmp_iv[16];
+		int i;
+		
+		for (i = 0; i < 16; i++)
+			tmp_iv[i] = *(in_iv + i);
+		
+		return aes_cbc_encrypt(in_blk, out_blk, num_blk<<4, tmp_iv, cx);
+
+}
+
+aes_rval aes_cbc_decrypt(const unsigned char *ibuf, unsigned char *obuf,
+                    int len, unsigned char *iv, const aes_decrypt_ctx ctx[1])
+{   unsigned char tmp[AES_BLOCK_SIZE];
+    int nb = len >> 4;
+
+    if(len & (AES_BLOCK_SIZE - 1))
+        return 1;
+
+#if defined( USE_VIA_ACE_IF_PRESENT )
+
+    if(ctx->inf.b[1] == 0xff)
+    {   uint_8t *ksp = kd_adr(ctx), *ivp = iv;
+        aligned_auto(uint_8t, liv, AES_BLOCK_SIZE, 16);
+        via_cwd(cwd, hybrid, dec, 2* ctx->inf.b[0] - 192);	
+
+        if(addr_mod_16(ctx))
+            return 1;
+
+        if(addr_mod_16(iv))   /* ensure an aligned iv */
+        {
+            ivp = liv;
+            memcpy(liv, iv, AES_BLOCK_SIZE);
+        }
+
+        if(!addr_mod_16(ibuf) && !addr_mod_16(obuf) && !addr_mod_16(iv))
+        {
+            via_cbc_op6(ksp,cwd,ibuf,obuf,nb,ivp);
+        }
+        else
+        {   aligned_auto(uint_8t, buf, BFR_BLOCKS * AES_BLOCK_SIZE, 16);
+            uint_8t *ip, *op;
+
+            while(nb)
+            {
+                int m = (nb > BFR_BLOCKS ? BFR_BLOCKS : nb);
+
+                ip = (addr_mod_16(ibuf) ? buf : (uint_8t*)ibuf);
+                op = (addr_mod_16(obuf) ? buf : obuf);
+
+                if(ip != ibuf)
+                    memcpy(buf, ibuf, m * AES_BLOCK_SIZE);
+
+                via_cbc_op6(ksp,cwd,ip,op,m,ivp);
+
+                if(op != obuf)
+                    memcpy(obuf, buf, m * AES_BLOCK_SIZE);
+
+                ibuf += m * AES_BLOCK_SIZE;
+                obuf += m * AES_BLOCK_SIZE;
+                nb -= m;
+            }
+        }
+
+        if(iv != ivp)
+            memcpy(iv, ivp, AES_BLOCK_SIZE);
+
+        return 0;
+    }
+#endif
+
+#if !defined( ASSUME_VIA_ACE_PRESENT )
+# ifdef FAST_BUFFER_OPERATIONS
+    if(!addr_mod_04(obuf) && !addr_mod_04(iv))
+        while(nb--)
+        {
+            memcpy(tmp, ibuf, AES_BLOCK_SIZE);
+            aes_decrypt(ibuf, obuf, ctx);
+            lp32(obuf)[0] ^= lp32(iv)[0];
+            lp32(obuf)[1] ^= lp32(iv)[1];
+            lp32(obuf)[2] ^= lp32(iv)[2];
+            lp32(obuf)[3] ^= lp32(iv)[3];
+            memcpy(iv, tmp, AES_BLOCK_SIZE);
+            ibuf += AES_BLOCK_SIZE;
+            obuf += AES_BLOCK_SIZE;
+        }
+    else
+# endif
+        while(nb--)
+        {
+            memcpy(tmp, ibuf, AES_BLOCK_SIZE);
+            aes_decrypt(ibuf, obuf, ctx);
+            obuf[ 0] ^= iv[ 0]; obuf[ 1] ^= iv[ 1];
+            obuf[ 2] ^= iv[ 2]; obuf[ 3] ^= iv[ 3];
+            obuf[ 4] ^= iv[ 4]; obuf[ 5] ^= iv[ 5];
+            obuf[ 6] ^= iv[ 6]; obuf[ 7] ^= iv[ 7];
+            obuf[ 8] ^= iv[ 8]; obuf[ 9] ^= iv[ 9];
+            obuf[10] ^= iv[10]; obuf[11] ^= iv[11];
+            obuf[12] ^= iv[12]; obuf[13] ^= iv[13];
+            obuf[14] ^= iv[14]; obuf[15] ^= iv[15];
+            memcpy(iv, tmp, AES_BLOCK_SIZE);
+            ibuf += AES_BLOCK_SIZE;
+            obuf += AES_BLOCK_SIZE;
+        }
+#endif
+    return 0;
+}
+
+aes_rval aes_decrypt_cbc(const unsigned char *in_blk, const unsigned char *in_iv, unsigned int num_blk,
+					 unsigned char *out_blk, const aes_decrypt_ctx cx[1])
+{
+		unsigned char tmp_iv[16];
+		int i;
+		
+		for (i = 0; i < 16; i++)
+			tmp_iv[i] = *(in_iv + i);
+		
+		return aes_cbc_decrypt(in_blk, out_blk, num_blk<<4, tmp_iv, cx);
+
+}
+
+
+#if defined(__cplusplus)
+}
+#endif
+#endif
diff --git a/bsd/crypto/aes/i386/aes_x86_v2.s b/bsd/crypto/aes/i386/aes_x86_v2.s
new file mode 100644
index 000000000..7ed98adb8
--- /dev/null
+++ b/bsd/crypto/aes/i386/aes_x86_v2.s
@@ -0,0 +1,1298 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * ---------------------------------------------------------------------------
+ * Copyright (c) 2002, Dr Brian Gladman, Worcester, UK.   All rights reserved.
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software in both source and binary
+ * form is allowed (with or without changes) provided that:
+ *
+ *   1. distributions of this source code include the above copyright
+ *      notice, this list of conditions and the following disclaimer;
+ *
+ *   2. distributions in binary form include the above copyright
+ *      notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other associated materials;
+ *
+ *   3. the copyright holder's name is not used to endorse products
+ *      built using this software without specific written permission.
+ *
+ * ALTERNATIVELY, provided that this notice is retained in full, this product
+ * may be distributed under the terms of the GNU General Public License (GPL),
+ * in which case the provisions of the GPL apply INSTEAD OF those given above.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ * Issue 31/01/2006
+ *
+ * This code requires either ASM_X86_V2 or ASM_X86_V2C to be set in aesopt.h 
+ * and the same define to be set here as well. If AES_V2C is set this file 
+ * requires the C files aeskey.c and aestab.c for support.
+ *
+ * This is a full assembler implementation covering encryption, decryption and
+ * key scheduling. It uses 2k bytes of tables but its encryption and decryption
+ * performance is very close to that obtained using large tables.  Key schedule
+ * expansion is slower for both encryption and decryption but this is likely to
+ * be offset by the much smaller load that this version places on the processor
+ * cache. I acknowledge the contribution made by Daniel Bernstein to aspects of
+ * the design of the AES round function used here.
+ *
+ * This code provides the standard AES block size (128 bits, 16 bytes) and the
+ * three standard AES key sizes (128, 192 and 256 bits). It has the same call
+ * interface as my C implementation. The ebx, esi, edi and ebp registers are
+ * preserved across calls but eax, ecx and edx and the artihmetic status flags
+ * are not.
+ */
+
+#include <mach/i386/asm.h>
+
+#define AES_128          /* define if AES with 128 bit keys is needed */
+#define AES_192          /* define if AES with 192 bit keys is needed */
+#define AES_256          /* define if AES with 256 bit keys is needed */
+#define AES_VAR          /* define if a variable key size is needed */
+#define ENCRYPTION       /* define if encryption is needed */
+#define DECRYPTION       /* define if decryption is needed */
+#define AES_REV_DKS      /* define if key decryption schedule is reversed */
+
+#ifndef ASM_X86_V2C
+#define ENCRYPTION_KEY_SCHEDULE /* define if enc. key expansion is needed */
+#define DECRYPTION_KEY_SCHEDULE /* define if dec. key expansion is needed */
+#endif
+
+/*
+ * The encryption key schedule has the following in memory layout where N is the
+ * number of rounds (10, 12 or 14):
+ *
+ * lo: | input key (round 0)  |  ; each round is four 32-bit words
+ *     | encryption round 1   |
+ *     | encryption round 2   |
+ *     ....
+ *     | encryption round N-1 |
+ * hi: | encryption round N   |
+ *
+ * The decryption key schedule is normally set up so that it has the same
+ * layout as above by actually reversing the order of the encryption key
+ * schedule in memory (this happens when AES_REV_DKS is set):
+ *
+ * lo: | decryption round 0   | =              | encryption round N   |
+ *     | decryption round 1   | = INV_MIX_COL[ | encryption round N-1 | ]
+ *     | decryption round 2   | = INV_MIX_COL[ | encryption round N-2 | ]
+ *     ....                       ....
+ *     | decryption round N-1 | = INV_MIX_COL[ | encryption round 1   | ]
+ * hi: | decryption round N   | =              | input key (round 0)  |
+ *
+ * with rounds except the first and last modified using inv_mix_column()
+ * But if AES_REV_DKS is NOT set the order of keys is left as it is for
+ * encryption so that it has to be accessed in reverse when used for
+ * decryption (although the inverse mix column modifications are done)
+ *
+ * lo: | decryption round 0   | =              | input key (round 0)  |
+ *     | decryption round 1   | = INV_MIX_COL[ | encryption round 1   | ]
+ *     | decryption round 2   | = INV_MIX_COL[ | encryption round 2   | ]
+ *     ....                       ....
+ *     | decryption round N-1 | = INV_MIX_COL[ | encryption round N-1 | ]
+ * hi: | decryption round N   | =              | encryption round N   |
+ *
+ * This layout is faster when the assembler key scheduling provided here
+ * is used.
+ */
+
+/* End of user defines */
+
+#ifdef AES_VAR
+#ifndef AES_128
+#define AES_128 
+#endif
+#ifndef AES_192
+#define AES_192 
+#endif
+#ifndef AES_256
+#define AES_256 
+#endif
+#endif
+
+#ifdef AES_VAR
+#define KS_LENGTH 60
+#else
+#ifdef AES_256
+#define KS_LENGTH 60
+#else
+#ifdef AES_192
+#define KS_LENGTH 52
+#else 
+#define KS_LENGTH 44
+#endif
+#endif
+#endif
+
+/*
+ * These macros implement stack based local variables
+ */
+#define	save(r1)			\
+    movl    %r1, (%esp);
+
+#define	restore(r1)			\
+    movl    (%esp), %r1;
+
+#define	do_call(f, n)			\
+    call    EXT(f);			\
+    addl    $(n), %esp;
+
+/*
+ * finite field multiplies by {02}, {04} and {08}
+ */
+#define f2(x) ((x<<1)^(((x>>7)&1)*0x11b))
+#define f4(x) ((x<<2)^(((x>>6)&1)*0x11b)^(((x>>6)&2)*0x11b))
+#define f8(x) ((x<<3)^(((x>>5)&1)*0x11b)^(((x>>5)&2)*0x11b)^(((x>>5)&4)*0x11b))
+
+/*
+ * finite field multiplies required in table generation
+ */
+#define	f3(x) (f2(x) ^ x)
+#define	f9(x) (f8(x) ^ x)
+#define	fb(x) (f8(x) ^ f2(x) ^ x)
+#define	fd(x) (f8(x) ^ f4(x) ^ x)
+#define	fe(x) (f8(x) ^ f4(x) ^ f2(x))
+
+#define	etab_0(x) enc_tab+4(,x,8)
+#define	etab_1(x) enc_tab+3(,x,8)
+#define	etab_2(x) enc_tab+2(,x,8)
+#define	etab_3(x) enc_tab+1(,x,8)
+
+#define	etab_b(x) etab_3(x)
+
+#define	btab_0(x) enc_tab+6(,x,8)
+#define	btab_1(x) enc_tab+5(,x,8)
+#define	btab_2(x) enc_tab+4(,x,8)
+#define	btab_3(x) enc_tab+3(,x,8)
+
+/*
+ * ROUND FUNCTION.  Build column[2] on ESI and column[3] on EDI that have the
+ * round keys pre-loaded. Build column[0] in EBP and column[1] in EBX.
+ *
+ * Input:
+ *
+ *   EAX     column[0]
+ *   EBX     column[1]
+ *   ECX     column[2]
+ *   EDX     column[3]
+ *   ESI     column key[round][2]
+ *   EDI     column key[round][3]
+ *   EBP     scratch
+ *
+ * Output:
+ *
+ *   EBP     column[0]   unkeyed
+ *   EBX     column[1]   unkeyed
+ *   ESI     column[2]   keyed
+ *   EDI     column[3]   keyed
+ *   EAX     scratch
+ *   ECX     scratch
+ *   EDX     scratch
+ */
+#define	rnd_fun(m1, m2)			\
+    roll    $16, %ebx;			\
+					\
+    ## m1 ## _zo(esi, cl, 0, ebp);	\
+    m1(esi, dh, 1, ebp);		\
+    m1(esi, bh, 3, ebp);		\
+    ## m1 ## _zo(edi, dl, 0, ebp);	\
+    m1(edi, ah, 1, ebp);		\
+    m1(edi, bl, 2, ebp);		\
+    ## m2 ## _zo(ebp, al, 0, ebp);	\
+					\
+    shrl    $16, %ebx;			\
+    andl    $0xffff0000, %eax;		\
+    orl     %ebx, %eax;			\
+    shrl    $16, %edx;			\
+					\
+    m1(ebp, ah, 1, ebx);		\
+    m1(ebp, dh, 3, ebx);		\
+    m2(ebx, dl, 2, ebx);		\
+    m1(ebx, ch, 1, edx);		\
+    ## m1 ## _zo(ebx, al, 0, edx);	\
+					\
+    shrl    $16, %eax;			\
+    shrl    $16, %ecx;			\
+					\
+    m1(ebp, cl, 2, edx);		\
+    m1(edi, ch, 3, edx);		\
+    m1(esi, al, 2, edx);		\
+    m1(ebx, ah, 3, edx)
+
+/*
+ * Basic MOV and XOR Operations for normal rounds
+ */
+#define	nr_xor_zo	nr_xor
+#define	nr_xor(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    xorl    etab_ ## r3(%r4), %r1;
+
+#define	nr_mov_zo	nr_mov
+#define	nr_mov(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    movl    etab_ ## r3(%r4), %r1;
+
+/*
+ * Basic MOV and XOR Operations for last round
+ */
+
+#if 1
+
+#define	lr_xor_zo(r1, r2, r3, r4)	\
+    movzbl  %r2, %r4;			\
+    movzbl  etab_b(%r4), %r4;		\
+    xor     %r4, %r1;
+
+#define	lr_xor(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    movzbl  etab_b(%r4), %r4;		\
+    shll    $(8*r3), %r4;		\
+    xor     %r4, %r1;
+
+#define	lr_mov_zo(r1, r2, r3, r4)	\
+    movzbl  %r2, %r4;			\
+    movzbl  etab_b(%r4), %r1;
+
+#define	lr_mov(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    movzbl  etab_b(%r4), %r1;		\
+    shll    $(8*r3), %r1;
+
+#else        /* less effective but worth leaving as an option */
+
+#define	lr_xor_zo	lr_xor
+#define	lr_xor(r1, r2, r3, r4)			\
+    movzbl  %r2, %r4;				\
+    mov     btab_ ## r3(%r4), %r4;		\
+    andl    $(0x000000ff << 8 * r3), %r4;	\
+    xor     %r4, %r1;
+
+#define	lr_mov_zo	lr_mov
+#define	lr_mov(r1, r2, r3, r4)			\
+    movzbl  %r2, %r4;				\
+    mov     btab_ ## r3(%r4), %r1;		\
+    andl    $(0x000000ff << 8 * r3), %r1;
+
+#endif
+
+/*
+ * Apply S-Box to the 4 bytes in a 32-bit word and rotate left 3 byte positions
+ *
+ *   r1 : output is xored into this register
+ *   r2 : input: a => eax, b => ebx, c => ecx, d => edx
+ *   r3 : scratch register
+ */
+
+#define	l3s_col(r1, r2, r3)			\
+    lr_xor_zo(r1, ## r2 ## h, 0, r3);		\
+    lr_xor(r1, ## r2 ## l, 3, r3);		\
+    shrl    $16, %e ## r2 ## x;			\
+    lr_xor(r1, ## r2 ## h, 2, r3);		\
+    lr_xor(r1, ## r2 ## l, 1, r3);
+
+/*
+ * offsets to parameters
+ */
+#define	in_blk		4	/* input byte array address parameter */
+#define	out_blk		8	/* output byte array address parameter */
+#define	ctx		12	/* AES context structure */
+#define	stk_spc		20	/* stack space */
+
+#ifdef  ENCRYPTION
+
+#define ENCRYPTION_TABLE 
+
+#define	enc_round			\
+    addl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    rnd_fun(nr_xor, nr_mov);		\
+					\
+    movl    %ebp, %eax;			\
+    movl    %esi, %ecx;			\
+    movl    %edi, %edx;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+#define enc_last_round			\
+    addl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    rnd_fun(lr_xor, lr_mov);		\
+					\
+    movl    %ebp, %eax;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+    .section __TEXT, __text
+
+/*
+ * AES Encryption Subroutine
+ */
+Entry(aes_encrypt)
+
+    subl    $stk_spc, %esp
+    movl    %ebp, 16(%esp)
+    movl    %ebx, 12(%esp)
+    movl    %esi, 8(%esp)
+    movl    %edi, 4(%esp)
+
+    movl    in_blk+stk_spc(%esp), %esi	/* input pointer */
+    movl    (%esi), %eax
+    movl    4(%esi), %ebx
+    movl    8(%esi), %ecx
+    movl    12(%esi), %edx
+
+    movl    ctx+stk_spc(%esp), %ebp	/* key pointer */
+    movzbl  4*KS_LENGTH(%ebp), %edi
+    xorl    (%ebp), %eax
+    xorl    4(%ebp), %ebx
+    xorl    8(%ebp), %ecx
+    xorl    12(%ebp), %edx
+
+    /*
+     * determine the number of rounds
+     */
+    cmpl    $10*16, %edi
+    je     aes_encrypt.3
+    cmpl    $12*16, %edi
+    je     aes_encrypt.2
+    cmpl    $14*16, %edi
+    je      aes_encrypt.1
+    movl    $-1, %eax
+    jmp     aes_encrypt.5
+
+aes_encrypt.1:
+    enc_round
+    enc_round
+aes_encrypt.2:
+    enc_round
+    enc_round
+aes_encrypt.3:
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_round
+    enc_last_round
+
+    movl    out_blk+stk_spc(%esp), %edx
+    movl    %eax, (%edx)
+    movl    %ebx, 4(%edx)
+    movl    %esi, 8(%edx)
+    movl    %edi, 12(%edx)
+    xorl    %eax, %eax
+
+aes_encrypt.5:
+    movl    16(%esp), %ebp
+    movl    12(%esp), %ebx
+    movl    8(%esp), %esi
+    movl    4(%esp), %edi
+    addl    $stk_spc, %esp
+    ret
+
+#endif
+
+/*
+ * For r2 == 16, or r2 == 24 && r1 == 7, or r2 ==32 && r1 == 6
+ */
+#define	f_key(r1, r2, rc_val)		\
+    l3s_col(esi, a, ebx);		\
+    xorl    $rc_val, %esi;		\
+					\
+    movl    %esi, r1*r2(%ebp);		\
+    xorl    %esi, %edi;			\
+    movl    %edi, r1*r2+4(%ebp);	\
+    xorl    %edi, %ecx;			\
+    movl    %ecx, r1*r2+8(%ebp);	\
+    xorl    %ecx, %edx;			\
+    movl    %edx, r1*r2+12(%ebp);	\
+    movl    %edx, %eax;
+
+/*
+ * For r2 == 24 && r1 == 0 to 6
+ */
+#define	f_key_24(r1, r2, rc_val)	\
+    f_key(r1, r2, rc_val);		\
+					\
+    xorl    r1*r2+16-r2(%ebp), %eax;	\
+    movl    %eax, r1*r2+16(%ebp);	\
+    xorl    r1*r2+20-r2(%ebp), %eax;	\
+    movl    %eax, r1*r2+20(%ebp);
+
+/*
+ * For r2 ==32 && r1 == 0 to 5
+ */
+#define	f_key_32(r1, r2, rc_val)	\
+    f_key(r1, r2, rc_val);		\
+					\
+    roll    $8, %eax;			\
+    pushl   %edx;			\
+    movl    r1*r2+16-r2(%ebp), %edx;	\
+    l3s_col(edx, a, ebx);		\
+    movl    %edx, %eax;			\
+    popl    %edx;			\
+    movl    %eax, r1*r2+16(%ebp);	\
+    xorl    r1*r2+20-r2(%ebp), %eax;	\
+    movl    %eax, r1*r2+20(%ebp);	\
+    xorl    r1*r2+24-r2(%ebp), %eax;	\
+    movl    %eax, r1*r2+24(%ebp);	\
+    xorl    r1*r2+28-r2(%ebp), %eax;	\
+    movl    %eax, r1*r2+28(%ebp);
+
+#ifdef ENCRYPTION_KEY_SCHEDULE
+
+#ifdef  AES_128
+
+#ifndef ENCRYPTION_TABLE
+#define ENCRYPTION_TABLE 
+#endif
+
+Entry(aes_encrypt_key128)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+
+    movl    24(%esp), %ebp
+    movl    $10*16, 4*KS_LENGTH(%ebp)
+    movl    20(%esp), %ebx
+
+    movl    (%ebx), %esi
+    movl    %esi, (%ebp)
+    movl    4(%ebx), %edi
+    movl    %edi, 4(%ebp)
+    movl    8(%ebx), %ecx
+    movl    %ecx, 8(%ebp)
+    movl    12(%ebx), %edx
+    movl    %edx, 12(%ebp)
+    addl    $16, %ebp
+    movl    %edx, %eax
+
+    f_key(0, 16, 1)
+    f_key(1, 16, 2)
+    f_key(2, 16, 4)
+    f_key(3, 16, 8)
+    f_key(4, 16, 16)
+    f_key(5, 16, 32)
+    f_key(6, 16, 64)
+    f_key(7, 16, 128)
+    f_key(8, 16, 27)
+    f_key(9, 16, 54)
+
+    popl    %edi
+    popl    %esi
+    popl    %ebx
+    popl    %ebp
+    xorl    %eax, %eax
+    ret
+
+#endif
+
+#ifdef  AES_192
+
+#ifndef ENCRYPTION_TABLE
+#define ENCRYPTION_TABLE 
+#endif
+
+Entry(aes_encrypt_key192)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+
+    movl    24(%esp), %ebp
+    movl    $12*16, 4*KS_LENGTH(%ebp)
+    movl    20(%esp), %ebx
+
+    movl    (%ebx), %esi
+    movl    %esi, (%ebp)
+    movl    4(%ebx), %edi
+    movl    %edi, 4(%ebp)
+    movl    8(%ebx), %ecx
+    movl    %ecx, 8(%ebp)
+    movl    12(%ebx), %edx
+    movl    %edx, 12(%ebp)
+    movl    16(%ebx), %eax
+    movl    %eax, 16(%ebp)
+    movl    20(%ebx), %eax
+    movl    %eax, 20(%ebp)
+    addl    $24, %ebp
+
+    f_key_24(0, 24, 1)
+    f_key_24(1, 24, 2)
+    f_key_24(2, 24, 4)
+    f_key_24(3, 24, 8)
+    f_key_24(4, 24, 16)
+    f_key_24(5, 24, 32)
+    f_key_24(6, 24, 64)
+    f_key(7, 24, 128)
+
+    popl    %edi
+    popl    %esi
+    popl    %ebx
+    popl    %ebp
+    xorl    %eax, %eax
+    ret
+
+#endif
+
+#ifdef  AES_256
+
+#ifndef ENCRYPTION_TABLE
+#define ENCRYPTION_TABLE 
+#endif
+
+Entry(aes_encrypt_key256)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+
+    movl    24(%esp), %ebp
+    movl    $14*16, 4*KS_LENGTH(%ebp)
+    movl    20(%esp), %ebx
+
+    movl    (%ebx), %esi
+    movl    %esi, (%ebp)
+    movl    4(%ebx), %edi
+    movl    %edi, 4(%ebp)
+    movl    8(%ebx), %ecx
+    movl    %ecx, 8(%ebp)
+    movl    12(%ebx), %edx
+    movl    %edx, 12(%ebp)
+    movl    16(%ebx), %eax
+    movl    %eax, 16(%ebp)
+    movl    20(%ebx), %eax
+    movl    %eax, 20(%ebp)
+    movl    24(%ebx), %eax
+    movl    %eax, 24(%ebp)
+    movl    28(%ebx), %eax
+    movl    %eax, 28(%ebp)
+    addl    $32, %ebp
+
+    f_key_32(0, 32, 1)
+    f_key_32(1, 32, 2)
+    f_key_32(2, 32, 4)
+    f_key_32(3, 32, 8)
+    f_key_32(4, 32, 16)
+    f_key_32(5, 32, 32)
+    f_key(6, 32, 64)
+
+    popl    %edi
+    popl    %esi
+    popl    %ebx
+    popl    %ebp
+    xorl    %eax, %eax
+    ret
+
+#endif
+
+#ifdef  AES_VAR
+
+#ifndef ENCRYPTION_TABLE
+#define ENCRYPTION_TABLE 
+#endif
+
+Entry(aes_encrypt_key)
+
+    movl    4(%esp), %ecx
+    movl    8(%esp), %eax
+    movl    12(%esp), %edx
+    pushl   %edx
+    pushl   %ecx
+
+    cmpl    $16, %eax
+    je      aes_encrypt_key.1
+    cmpl    $128, %eax
+    je      aes_encrypt_key.1
+
+    cmpl    $24, %eax
+    je      aes_encrypt_key.2
+    cmpl    $192, %eax
+    je      aes_encrypt_key.2
+
+    cmpl    $32, %eax
+    je      aes_encrypt_key.3
+    cmpl    $256, %eax
+    je      aes_encrypt_key.3
+    movl    $-1, %eax
+    addl    $8, %esp
+    ret
+
+aes_encrypt_key.1:
+    do_call(aes_encrypt_key128, 8)
+    ret
+aes_encrypt_key.2:
+    do_call(aes_encrypt_key192, 8)
+    ret
+aes_encrypt_key.3:
+    do_call(aes_encrypt_key256, 8)
+    ret
+
+#endif
+
+#endif
+
+#ifdef ENCRYPTION_TABLE
+
+# S-box data - 256 entries
+
+    .section __DATA, __data
+    .align ALIGN
+
+#define u8(x) 0, x, x, f3(x), f2(x), x, x, f3(x)
+
+enc_tab: 
+   .byte u8(0x63),u8(0x7c),u8(0x77),u8(0x7b),u8(0xf2),u8(0x6b),u8(0x6f),u8(0xc5)
+   .byte u8(0x30),u8(0x01),u8(0x67),u8(0x2b),u8(0xfe),u8(0xd7),u8(0xab),u8(0x76)
+   .byte u8(0xca),u8(0x82),u8(0xc9),u8(0x7d),u8(0xfa),u8(0x59),u8(0x47),u8(0xf0)
+   .byte u8(0xad),u8(0xd4),u8(0xa2),u8(0xaf),u8(0x9c),u8(0xa4),u8(0x72),u8(0xc0)
+   .byte u8(0xb7),u8(0xfd),u8(0x93),u8(0x26),u8(0x36),u8(0x3f),u8(0xf7),u8(0xcc)
+   .byte u8(0x34),u8(0xa5),u8(0xe5),u8(0xf1),u8(0x71),u8(0xd8),u8(0x31),u8(0x15)
+   .byte u8(0x04),u8(0xc7),u8(0x23),u8(0xc3),u8(0x18),u8(0x96),u8(0x05),u8(0x9a)
+   .byte u8(0x07),u8(0x12),u8(0x80),u8(0xe2),u8(0xeb),u8(0x27),u8(0xb2),u8(0x75)
+   .byte u8(0x09),u8(0x83),u8(0x2c),u8(0x1a),u8(0x1b),u8(0x6e),u8(0x5a),u8(0xa0)
+   .byte u8(0x52),u8(0x3b),u8(0xd6),u8(0xb3),u8(0x29),u8(0xe3),u8(0x2f),u8(0x84)
+   .byte u8(0x53),u8(0xd1),u8(0x00),u8(0xed),u8(0x20),u8(0xfc),u8(0xb1),u8(0x5b)
+   .byte u8(0x6a),u8(0xcb),u8(0xbe),u8(0x39),u8(0x4a),u8(0x4c),u8(0x58),u8(0xcf)
+   .byte u8(0xd0),u8(0xef),u8(0xaa),u8(0xfb),u8(0x43),u8(0x4d),u8(0x33),u8(0x85)
+   .byte u8(0x45),u8(0xf9),u8(0x02),u8(0x7f),u8(0x50),u8(0x3c),u8(0x9f),u8(0xa8)
+   .byte u8(0x51),u8(0xa3),u8(0x40),u8(0x8f),u8(0x92),u8(0x9d),u8(0x38),u8(0xf5)
+   .byte u8(0xbc),u8(0xb6),u8(0xda),u8(0x21),u8(0x10),u8(0xff),u8(0xf3),u8(0xd2)
+   .byte u8(0xcd),u8(0x0c),u8(0x13),u8(0xec),u8(0x5f),u8(0x97),u8(0x44),u8(0x17)
+   .byte u8(0xc4),u8(0xa7),u8(0x7e),u8(0x3d),u8(0x64),u8(0x5d),u8(0x19),u8(0x73)
+   .byte u8(0x60),u8(0x81),u8(0x4f),u8(0xdc),u8(0x22),u8(0x2a),u8(0x90),u8(0x88)
+   .byte u8(0x46),u8(0xee),u8(0xb8),u8(0x14),u8(0xde),u8(0x5e),u8(0x0b),u8(0xdb)
+   .byte u8(0xe0),u8(0x32),u8(0x3a),u8(0x0a),u8(0x49),u8(0x06),u8(0x24),u8(0x5c)
+   .byte u8(0xc2),u8(0xd3),u8(0xac),u8(0x62),u8(0x91),u8(0x95),u8(0xe4),u8(0x79)
+   .byte u8(0xe7),u8(0xc8),u8(0x37),u8(0x6d),u8(0x8d),u8(0xd5),u8(0x4e),u8(0xa9)
+   .byte u8(0x6c),u8(0x56),u8(0xf4),u8(0xea),u8(0x65),u8(0x7a),u8(0xae),u8(0x08)
+   .byte u8(0xba),u8(0x78),u8(0x25),u8(0x2e),u8(0x1c),u8(0xa6),u8(0xb4),u8(0xc6)
+   .byte u8(0xe8),u8(0xdd),u8(0x74),u8(0x1f),u8(0x4b),u8(0xbd),u8(0x8b),u8(0x8a)
+   .byte u8(0x70),u8(0x3e),u8(0xb5),u8(0x66),u8(0x48),u8(0x03),u8(0xf6),u8(0x0e)
+   .byte u8(0x61),u8(0x35),u8(0x57),u8(0xb9),u8(0x86),u8(0xc1),u8(0x1d),u8(0x9e)
+   .byte u8(0xe1),u8(0xf8),u8(0x98),u8(0x11),u8(0x69),u8(0xd9),u8(0x8e),u8(0x94)
+   .byte u8(0x9b),u8(0x1e),u8(0x87),u8(0xe9),u8(0xce),u8(0x55),u8(0x28),u8(0xdf)
+   .byte u8(0x8c),u8(0xa1),u8(0x89),u8(0x0d),u8(0xbf),u8(0xe6),u8(0x42),u8(0x68)
+   .byte u8(0x41),u8(0x99),u8(0x2d),u8(0x0f),u8(0xb0),u8(0x54),u8(0xbb),u8(0x16)
+
+#endif
+
+#ifdef  DECRYPTION
+
+#define DECRYPTION_TABLE 
+
+#define dtab_0(x) dec_tab(,x,8)
+#define dtab_1(x) dec_tab+3(,x,8)
+#define dtab_2(x) dec_tab+2(,x,8)
+#define dtab_3(x) dec_tab+1(,x,8)
+#define dtab_x(x) dec_tab+7(,x,8)
+
+#define	irn_fun(m1, m2)			\
+    roll    $16, %eax;			\
+					\
+    ## m1 ## _zo(esi, cl, 0, ebp);	\
+    m1(esi, bh, 1, ebp);		\
+    m1(esi, al, 2, ebp);		\
+    ## m1 ## _zo(edi, dl, 0, ebp);	\
+    m1(edi, ch, 1, ebp);		\
+    m1(edi, ah, 3, ebp);		\
+    ## m2 ## _zo(ebp, bl, 0, ebp);	\
+					\
+    shrl    $16, %eax;			\
+    andl    $0xffff0000, %ebx;		\
+    orl     %eax, %ebx;			\
+    shrl    $16, %ecx;			\
+					\
+    m1(ebp, bh, 1, eax);		\
+    m1(ebp, ch, 3, eax);		\
+    m2(eax, cl, 2, ecx);		\
+    ## m1 ## _zo(eax, bl, 0, ecx);	\
+    m1(eax, dh, 1, ecx);		\
+					\
+    shrl    $16, %ebx;			\
+    shrl    $16, %edx;			\
+					\
+    m1(esi, dh, 3, ecx);		\
+    m1(ebp, dl, 2, ecx);		\
+    m1(eax, bh, 3, ecx);		\
+    m1(edi, bl, 2, ecx);
+
+/*
+ * Basic MOV and XOR Operations for normal rounds
+ */
+#define	ni_xor_zo	ni_xor
+#define	ni_xor(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    xorl    dtab_ ## r3 ## (%r4), %r1;
+
+#define	ni_mov_zo	ni_mov
+#define	ni_mov(r1, r2, r3, r4)		\
+    movzbl  %r2, %r4;			\
+    movl    dtab_ ## r3 ## (%r4), %r1;
+
+/*
+ * Basic MOV and XOR Operations for last round
+ */
+
+#define	li_xor_zo(r1, r2, r3, r4)	\
+    movzbl %r2, %r4;			\
+    movzbl dtab_x(%r4), %r4;		\
+    xor    %r4, %r1;
+
+#define	li_xor(r1, r2, r3, r4)		\
+    movzbl %r2, %r4;			\
+    movzbl dtab_x(%r4), %r4;		\
+    shll   $(8*r3), %r4;		\
+    xor    %r4, %r1;
+
+#define	li_mov_zo(r1, r2, r3, r4)	\
+    movzbl %r2, %r4;			\
+    movzbl dtab_x(%r4), %r1;
+
+#define	li_mov(r1, r2, r3, r4)		\
+    movzbl %r2, %r4;			\
+    movzbl dtab_x(%r4), %r1;		\
+    shl    $(8*r3), %r1;
+
+#ifdef AES_REV_DKS
+
+#define	dec_round			\
+    addl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    irn_fun(ni_xor, ni_mov);		\
+					\
+    movl    %ebp, %ebx;			\
+    movl    %esi, %ecx;			\
+    movl    %edi, %edx;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+#define	dec_last_round			\
+    addl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    irn_fun(li_xor, li_mov);		\
+					\
+    movl    %ebp, %ebx;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+#else
+
+#define	dec_round			\
+    subl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    irn_fun(ni_xor, ni_mov);		\
+					\
+    movl    %ebp, %ebx;			\
+    movl    %esi, %ecx;			\
+    movl    %edi, %edx;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+#define	dec_last_round			\
+    subl    $16, %ebp;			\
+    save(ebp);				\
+    movl    8(%ebp), %esi;		\
+    movl    12(%ebp), %edi;		\
+					\
+    irn_fun(li_xor, li_mov);		\
+					\
+    movl    %ebp, %ebx;			\
+    restore(ebp);			\
+    xorl    (%ebp), %eax;		\
+    xorl    4(%ebp), %ebx;
+
+#endif /* AES_REV_DKS */
+
+    .section __TEXT, __text
+
+/*
+ * AES Decryption Subroutine
+ */
+Entry(aes_decrypt)
+
+    subl    $stk_spc, %esp
+    movl    %ebp, 16(%esp)
+    movl    %ebx, 12(%esp)
+    movl    %esi, 8(%esp)
+    movl    %edi, 4(%esp)
+
+    /*
+     * input four columns and xor in first round key
+     */
+    movl    in_blk+stk_spc(%esp), %esi	/* input pointer */
+    movl    (%esi), %eax
+    movl    4(%esi), %ebx
+    movl    8(%esi), %ecx
+    movl    12(%esi), %edx
+    leal    16(%esi), %esi
+
+    movl    ctx+stk_spc(%esp), %ebp	/* key pointer */
+    movzbl  4*KS_LENGTH(%ebp), %edi
+#ifndef  AES_REV_DKS		/* if decryption key schedule is not reversed */
+    leal    (%ebp,%edi), %ebp	/* we have to access it from the top down */
+#endif
+    xorl    (%ebp), %eax	/* key schedule */
+    xorl    4(%ebp), %ebx
+    xorl    8(%ebp), %ecx
+    xorl    12(%ebp), %edx
+
+    /*
+     * determine the number of rounds
+     */
+    cmpl    $10*16, %edi
+    je     aes_decrypt.3
+    cmpl    $12*16, %edi
+    je     aes_decrypt.2
+    cmpl    $14*16, %edi
+    je      aes_decrypt.1
+    movl    $-1, %eax
+    jmp     aes_decrypt.5
+
+aes_decrypt.1:
+    dec_round
+    dec_round
+aes_decrypt.2:
+    dec_round
+    dec_round
+aes_decrypt.3:
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_round
+    dec_last_round
+
+    /*
+     * move final values to the output array.
+     */
+    movl    out_blk+stk_spc(%esp), %ebp
+    movl    %eax, (%ebp)
+    movl    %ebx, 4(%ebp)
+    movl    %esi, 8(%ebp)
+    movl    %edi, 12(%ebp)
+    xorl    %eax, %eax
+
+aes_decrypt.5:
+    movl    16(%esp), %ebp
+    movl    12(%esp), %ebx
+    movl    8(%esp), %esi
+    movl    4(%esp), %edi
+    addl    $stk_spc, %esp
+    ret
+
+#endif
+
+#define	inv_mix_col			\
+    movzbl  %dl, %ebx;			\
+    movzbl  etab_b(%ebx), %ebx;		\
+    movl    dtab_0(%ebx), %eax;		\
+    movzbl  %dh, %ebx;			\
+    shrl    $16, %edx;			\
+    movzbl  etab_b(%ebx), %ebx;		\
+    xorl    dtab_1(%ebx), %eax;		\
+    movzbl  %dl, %ebx;			\
+    movzbl  etab_b(%ebx), %ebx;		\
+    xorl    dtab_2(%ebx), %eax;		\
+    movzbl  %dh, %ebx;			\
+    movzbl  etab_b(%ebx), %ebx;		\
+    xorl    dtab_3(%ebx), %eax;
+
+#ifdef DECRYPTION_KEY_SCHEDULE
+
+#ifdef AES_128
+
+#ifndef DECRYPTION_TABLE
+#define DECRYPTION_TABLE 
+#endif
+
+Entry(aes_decrypt_key128)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+    movl    24(%esp), %eax	/* context */
+    movl    20(%esp), %edx	/* key */
+    pushl   %eax
+    pushl   %edx
+    do_call(aes_encrypt_key128, 8)
+    movl    $10*16, %eax
+    movl    24(%esp), %esi	/* pointer to first round key */
+    leal    (%esi,%eax), %edi	/* pointer to last round key */
+    addl    $32, %esi
+				/* the inverse mix column transformation */
+    movl    -16(%esi), %edx	/* needs to be applied to all round keys */
+    inv_mix_col
+    movl    %eax, -16(%esi)	/* transforming the four sub-keys in the */
+    movl    -12(%esi), %edx	/* second round key */
+    inv_mix_col
+    movl    %eax, -12(%esi)	/* transformations for subsequent rounds */
+    movl    -8(%esi), %edx	/* can then be made more efficient by */
+    inv_mix_col
+    movl    %eax, -8(%esi)	/* in the encryption round key ek[r]: */
+    movl    -4(%esi), %edx
+    inv_mix_col
+    movl    %eax, -4(%esi)	/* where n is 1..3. Hence the corresponding */
+
+aes_decrypt_key128.0:
+    movl    (%esi), %edx	/* subkeys in the decryption round key dk[r] */
+    inv_mix_col
+    movl    %eax, (%esi)	/* GF(256): */
+    xorl    -12(%esi), %eax
+    movl    %eax, 4(%esi)	/* dk[r][n] = dk[r][n-1] ^ dk[r-1][n] */
+    xorl    -8(%esi), %eax
+    movl    %eax, 8(%esi)	/* So we only need one inverse mix column */
+    xorl    -4(%esi), %eax	/* operation (n = 0) for each four word cycle */
+    movl    %eax, 12(%esi)	/* in the expanded key. */
+    addl    $16, %esi
+    cmpl    %esi, %edi
+    jg      aes_decrypt_key128.0
+    jmp     dec_end
+
+#endif
+
+#ifdef AES_192
+
+#ifndef DECRYPTION_TABLE
+#define DECRYPTION_TABLE 
+#endif
+
+Entry(aes_decrypt_key192)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+    movl    24(%esp), %eax	/* context */
+    movl    20(%esp), %edx	/* key */
+    pushl   %eax
+    pushl   %edx
+    do_call(aes_encrypt_key192, 8)
+    movl    $12*16, %eax
+    movl    24(%esp), %esi	/* first round key */
+    leal    (%esi,%eax), %edi	/* last round key */
+    addl    $48, %esi		/* the first 6 words are the key, of */
+				/* which the top 2 words are part of */
+    movl    -32(%esi), %edx	/* the second round key and hence */
+    inv_mix_col
+    movl    %eax, -32(%esi)	/* need to do a further six values prior */
+    movl    -28(%esi), %edx	/* to using a more efficient technique */
+    inv_mix_col
+    movl    %eax, -28(%esi)
+				/* dk[r][n] = dk[r][n-1] ^ dk[r-1][n] */
+    movl    -24(%esi), %edx
+    inv_mix_col
+    movl    %eax, -24(%esi)	/* cycle is now 6 words long */
+    movl    -20(%esi), %edx
+    inv_mix_col
+    movl    %eax, -20(%esi)
+    movl    -16(%esi), %edx
+    inv_mix_col
+    movl    %eax, -16(%esi)
+    movl    -12(%esi), %edx
+    inv_mix_col
+    movl    %eax, -12(%esi)
+    movl    -8(%esi), %edx
+    inv_mix_col
+    movl    %eax, -8(%esi)
+    movl    -4(%esi), %edx
+    inv_mix_col
+    movl    %eax, -4(%esi)
+
+aes_decrypt_key192.0:
+    movl    (%esi), %edx	/* expanded key is 13 * 4 = 44 32-bit words */
+    inv_mix_col
+    movl    %eax, (%esi)	/* using inv_mix_col.  We have already done 8 */
+    xorl    -20(%esi), %eax	/* of these so 36 are left - hence we need */
+    movl    %eax, 4(%esi)	/* exactly 6 loops of six here */
+    xorl    -16(%esi), %eax
+    movl    %eax, 8(%esi)
+    xorl    -12(%esi), %eax
+    movl    %eax, 12(%esi)
+    xorl    -8(%esi), %eax
+    movl    %eax, 16(%esi)
+    xorl    -4(%esi), %eax
+    movl    %eax, 20(%esi)
+    addl    $24, %esi
+    cmpl    %esi, %edi
+    jg      aes_decrypt_key192.0
+    jmp     dec_end
+
+#endif
+
+#ifdef AES_256
+
+#ifndef DECRYPTION_TABLE
+#define DECRYPTION_TABLE 
+#endif
+
+Entry(aes_decrypt_key256)
+
+    pushl   %ebp
+    pushl   %ebx
+    pushl   %esi
+    pushl   %edi
+    movl    24(%esp), %eax
+    movl    20(%esp), %edx
+    pushl   %eax
+    pushl   %edx
+    do_call(aes_encrypt_key256, 8)
+    movl    $14*16, %eax
+    movl    24(%esp), %esi
+    leal    (%esi,%eax), %edi
+    addl    $64, %esi
+
+    movl    -48(%esi), %edx	/* the primary key is 8 words, of which */
+    inv_mix_col
+    movl    %eax, -48(%esi)
+    movl    -44(%esi), %edx
+    inv_mix_col
+    movl    %eax, -44(%esi)
+    movl    -40(%esi), %edx
+    inv_mix_col
+    movl    %eax, -40(%esi)
+    movl    -36(%esi), %edx
+    inv_mix_col
+    movl    %eax, -36(%esi)
+
+    movl    -32(%esi), %edx	/* the encryption key expansion cycle is */
+    inv_mix_col
+    movl    %eax, -32(%esi)	/* start by doing one complete block */
+    movl    -28(%esi), %edx
+    inv_mix_col
+    movl    %eax, -28(%esi)
+    movl    -24(%esi), %edx
+    inv_mix_col
+    movl    %eax, -24(%esi)
+    movl    -20(%esi), %edx
+    inv_mix_col
+    movl    %eax, -20(%esi)
+    movl    -16(%esi), %edx
+    inv_mix_col
+    movl    %eax, -16(%esi)
+    movl    -12(%esi), %edx
+    inv_mix_col
+    movl    %eax, -12(%esi)
+    movl    -8(%esi), %edx
+    inv_mix_col
+    movl    %eax, -8(%esi)
+    movl    -4(%esi), %edx
+    inv_mix_col
+    movl    %eax, -4(%esi)
+
+aes_decrypt_key256.0:
+    movl    (%esi), %edx	/* we can now speed up the remaining */
+    inv_mix_col
+    movl    %eax, (%esi)	/* outlined earlier.  But note that */
+    xorl    -28(%esi), %eax	/* there is one extra inverse mix */
+    movl    %eax, 4(%esi)	/* column operation as the 256 bit */
+    xorl    -24(%esi), %eax	/* key has an extra non-linear step */
+    movl    %eax, 8(%esi)	/* for the midway element. */
+    xorl    -20(%esi), %eax
+    movl    %eax, 12(%esi)	/* the expanded key is 15 * 4 = 60 */
+    movl    16(%esi), %edx	/* 32-bit words of which 52 need to */
+    inv_mix_col
+    movl    %eax, 16(%esi)	/* 12 so 40 are left - which means */
+    xorl    -12(%esi), %eax	/* that we need exactly 5 loops of 8 */
+    movl    %eax, 20(%esi)
+    xorl    -8(%esi), %eax
+    movl    %eax, 24(%esi)
+    xorl    -4(%esi), %eax
+    movl    %eax, 28(%esi)
+    addl    $32, %esi
+    cmpl    %esi, %edi
+    jg      aes_decrypt_key256.0
+
+#endif
+
+dec_end: 
+
+#ifdef AES_REV_DKS
+
+    movl    24(%esp), %esi	/* this reverses the order of the */
+dec_end.1:
+    movl    (%esi), %eax	/* round keys if required */
+    movl    4(%esi), %ebx
+    movl    (%edi), %ebp
+    movl    4(%edi), %edx
+    movl    %ebp, (%esi)
+    movl    %edx, 4(%esi)
+    movl    %eax, (%edi)
+    movl    %ebx, 4(%edi)
+
+    movl    8(%esi), %eax
+    movl    12(%esi), %ebx
+    movl    8(%edi), %ebp
+    movl    12(%edi), %edx
+    movl    %ebp, 8(%esi)
+    movl    %edx, 12(%esi)
+    movl    %eax, 8(%edi)
+    movl    %ebx, 12(%edi)
+
+    addl    $16, %esi
+    subl    $16, %edi
+    cmpl    %esi, %edi
+    jg      dec_end.1
+
+#endif
+
+    popl    %edi
+    popl    %esi
+    popl    %ebx
+    popl    %ebp
+    xorl    %eax, %eax
+    ret
+
+#ifdef AES_VAR
+
+Entry(aes_decrypt_key)
+
+    movl    4(%esp), %ecx
+    movl    8(%esp), %eax
+    movl    12(%esp), %edx
+    pushl   %edx
+    pushl   %ecx
+
+    cmpl    $16, %eax
+    je      aes_decrypt_key.1
+    cmpl    $128, %eax
+    je      aes_decrypt_key.1
+
+    cmpl    $24, %eax
+    je      aes_decrypt_key.2
+    cmpl    $192, %eax
+    je      aes_decrypt_key.2
+
+    cmpl    $32, %eax
+    je      aes_decrypt_key.3
+    cmpl    $256, %eax
+    je      aes_decrypt_key.3
+    movl    $-1, %eax
+    addl    $8, %esp
+    ret
+
+aes_decrypt_key.1:
+    do_call(aes_decrypt_key128, 8)
+    ret
+aes_decrypt_key.2:
+    do_call(aes_decrypt_key192, 8)
+    ret
+aes_decrypt_key.3:
+    do_call(aes_decrypt_key256, 8)
+    ret
+
+#endif
+
+#endif
+
+#ifdef DECRYPTION_TABLE
+
+/*
+ * Inverse S-box data - 256 entries
+ */
+
+    .section __DATA, __data
+    .align ALIGN
+
+#define v8(x) fe(x), f9(x), fd(x), fb(x), fe(x), f9(x), fd(x), x
+
+dec_tab: 
+   .byte v8(0x52),v8(0x09),v8(0x6a),v8(0xd5),v8(0x30),v8(0x36),v8(0xa5),v8(0x38)
+   .byte v8(0xbf),v8(0x40),v8(0xa3),v8(0x9e),v8(0x81),v8(0xf3),v8(0xd7),v8(0xfb)
+   .byte v8(0x7c),v8(0xe3),v8(0x39),v8(0x82),v8(0x9b),v8(0x2f),v8(0xff),v8(0x87)
+   .byte v8(0x34),v8(0x8e),v8(0x43),v8(0x44),v8(0xc4),v8(0xde),v8(0xe9),v8(0xcb)
+   .byte v8(0x54),v8(0x7b),v8(0x94),v8(0x32),v8(0xa6),v8(0xc2),v8(0x23),v8(0x3d)
+   .byte v8(0xee),v8(0x4c),v8(0x95),v8(0x0b),v8(0x42),v8(0xfa),v8(0xc3),v8(0x4e)
+   .byte v8(0x08),v8(0x2e),v8(0xa1),v8(0x66),v8(0x28),v8(0xd9),v8(0x24),v8(0xb2)
+   .byte v8(0x76),v8(0x5b),v8(0xa2),v8(0x49),v8(0x6d),v8(0x8b),v8(0xd1),v8(0x25)
+   .byte v8(0x72),v8(0xf8),v8(0xf6),v8(0x64),v8(0x86),v8(0x68),v8(0x98),v8(0x16)
+   .byte v8(0xd4),v8(0xa4),v8(0x5c),v8(0xcc),v8(0x5d),v8(0x65),v8(0xb6),v8(0x92)
+   .byte v8(0x6c),v8(0x70),v8(0x48),v8(0x50),v8(0xfd),v8(0xed),v8(0xb9),v8(0xda)
+   .byte v8(0x5e),v8(0x15),v8(0x46),v8(0x57),v8(0xa7),v8(0x8d),v8(0x9d),v8(0x84)
+   .byte v8(0x90),v8(0xd8),v8(0xab),v8(0x00),v8(0x8c),v8(0xbc),v8(0xd3),v8(0x0a)
+   .byte v8(0xf7),v8(0xe4),v8(0x58),v8(0x05),v8(0xb8),v8(0xb3),v8(0x45),v8(0x06)
+   .byte v8(0xd0),v8(0x2c),v8(0x1e),v8(0x8f),v8(0xca),v8(0x3f),v8(0x0f),v8(0x02)
+   .byte v8(0xc1),v8(0xaf),v8(0xbd),v8(0x03),v8(0x01),v8(0x13),v8(0x8a),v8(0x6b)
+   .byte v8(0x3a),v8(0x91),v8(0x11),v8(0x41),v8(0x4f),v8(0x67),v8(0xdc),v8(0xea)
+   .byte v8(0x97),v8(0xf2),v8(0xcf),v8(0xce),v8(0xf0),v8(0xb4),v8(0xe6),v8(0x73)
+   .byte v8(0x96),v8(0xac),v8(0x74),v8(0x22),v8(0xe7),v8(0xad),v8(0x35),v8(0x85)
+   .byte v8(0xe2),v8(0xf9),v8(0x37),v8(0xe8),v8(0x1c),v8(0x75),v8(0xdf),v8(0x6e)
+   .byte v8(0x47),v8(0xf1),v8(0x1a),v8(0x71),v8(0x1d),v8(0x29),v8(0xc5),v8(0x89)
+   .byte v8(0x6f),v8(0xb7),v8(0x62),v8(0x0e),v8(0xaa),v8(0x18),v8(0xbe),v8(0x1b)
+   .byte v8(0xfc),v8(0x56),v8(0x3e),v8(0x4b),v8(0xc6),v8(0xd2),v8(0x79),v8(0x20)
+   .byte v8(0x9a),v8(0xdb),v8(0xc0),v8(0xfe),v8(0x78),v8(0xcd),v8(0x5a),v8(0xf4)
+   .byte v8(0x1f),v8(0xdd),v8(0xa8),v8(0x33),v8(0x88),v8(0x07),v8(0xc7),v8(0x31)
+   .byte v8(0xb1),v8(0x12),v8(0x10),v8(0x59),v8(0x27),v8(0x80),v8(0xec),v8(0x5f)
+   .byte v8(0x60),v8(0x51),v8(0x7f),v8(0xa9),v8(0x19),v8(0xb5),v8(0x4a),v8(0x0d)
+   .byte v8(0x2d),v8(0xe5),v8(0x7a),v8(0x9f),v8(0x93),v8(0xc9),v8(0x9c),v8(0xef)
+   .byte v8(0xa0),v8(0xe0),v8(0x3b),v8(0x4d),v8(0xae),v8(0x2a),v8(0xf5),v8(0xb0)
+   .byte v8(0xc8),v8(0xeb),v8(0xbb),v8(0x3c),v8(0x83),v8(0x53),v8(0x99),v8(0x61)
+   .byte v8(0x17),v8(0x2b),v8(0x04),v8(0x7e),v8(0xba),v8(0x77),v8(0xd6),v8(0x26)
+   .byte v8(0xe1),v8(0x69),v8(0x14),v8(0x63),v8(0x55),v8(0x21),v8(0x0c),v8(0x7d)
+
+#endif
diff --git a/bsd/crypto/aes/i386/aesopt.h b/bsd/crypto/aes/i386/aesopt.h
new file mode 100644
index 000000000..025eb5fcf
--- /dev/null
+++ b/bsd/crypto/aes/i386/aesopt.h
@@ -0,0 +1,719 @@
+/*
+ ---------------------------------------------------------------------------
+ Copyright (c) 2003, Dr Brian Gladman, Worcester, UK.   All rights reserved.
+
+ LICENSE TERMS
+
+ The free distribution and use of this software in both source and binary
+ form is allowed (with or without changes) provided that:
+
+   1. distributions of this source code include the above copyright
+      notice, this list of conditions and the following disclaimer;
+
+   2. distributions in binary form include the above copyright
+      notice, this list of conditions and the following disclaimer
+      in the documentation and/or other associated materials;
+
+   3. the copyright holder's name is not used to endorse products
+      built using this software without specific written permission.
+
+ ALTERNATIVELY, provided that this notice is retained in full, this product
+ may be distributed under the terms of the GNU General Public License (GPL),
+ in which case the provisions of the GPL apply INSTEAD OF those given above.
+
+ DISCLAIMER
+
+ This software is provided 'as is' with no explicit or implied warranties
+ in respect of its properties, including, but not limited to, correctness
+ and/or fitness for purpose.
+ ---------------------------------------------------------------------------
+ Issue 31/01/2006
+
+ This file contains the compilation options for AES (Rijndael) and code
+ that is common across encryption, key scheduling and table generation.
+
+ OPERATION
+
+ These source code files implement the AES algorithm Rijndael designed by
+ Joan Daemen and Vincent Rijmen. This version is designed for the standard
+ block size of 16 bytes and for key sizes of 128, 192 and 256 bits (16, 24
+ and 32 bytes).
+
+ This version is designed for flexibility and speed using operations on
+ 32-bit words rather than operations on bytes.  It can be compiled with
+ either big or little endian internal byte order but is faster when the
+ native byte order for the processor is used.
+
+ THE CIPHER INTERFACE
+
+ The cipher interface is implemented as an array of bytes in which lower
+ AES bit sequence indexes map to higher numeric significance within bytes.
+
+  uint_8t                 (an unsigned  8-bit type)
+  uint_32t                (an unsigned 32-bit type)
+  struct aes_encrypt_ctx  (structure for the cipher encryption context)
+  struct aes_decrypt_ctx  (structure for the cipher decryption context)
+  aes_rval                the function return type
+
+  C subroutine calls:
+
+  aes_rval aes_encrypt_key128(const unsigned char *key, aes_encrypt_ctx cx[1]);
+  aes_rval aes_encrypt_key192(const unsigned char *key, aes_encrypt_ctx cx[1]);
+  aes_rval aes_encrypt_key256(const unsigned char *key, aes_encrypt_ctx cx[1]);
+  aes_rval aes_encrypt(const unsigned char *in, unsigned char *out,
+                                                  const aes_encrypt_ctx cx[1]);
+
+  aes_rval aes_decrypt_key128(const unsigned char *key, aes_decrypt_ctx cx[1]);
+  aes_rval aes_decrypt_key192(const unsigned char *key, aes_decrypt_ctx cx[1]);
+  aes_rval aes_decrypt_key256(const unsigned char *key, aes_decrypt_ctx cx[1]);
+  aes_rval aes_decrypt(const unsigned char *in, unsigned char *out,
+                                                  const aes_decrypt_ctx cx[1]);
+
+ IMPORTANT NOTE: If you are using this C interface with dynamic tables make sure that
+ you call gen_tabs() before AES is used so that the tables are initialised.
+
+ C++ aes class subroutines:
+
+     Class AESencrypt  for encryption
+
+      Construtors:
+          AESencrypt(void)
+          AESencrypt(const unsigned char *key) - 128 bit key
+      Members:
+          aes_rval key128(const unsigned char *key)
+          aes_rval key192(const unsigned char *key)
+          aes_rval key256(const unsigned char *key)
+          aes_rval encrypt(const unsigned char *in, unsigned char *out) const
+
+      Class AESdecrypt  for encryption
+      Construtors:
+          AESdecrypt(void)
+          AESdecrypt(const unsigned char *key) - 128 bit key
+      Members:
+          aes_rval key128(const unsigned char *key)
+          aes_rval key192(const unsigned char *key)
+          aes_rval key256(const unsigned char *key)
+          aes_rval decrypt(const unsigned char *in, unsigned char *out) const
+*/
+
+#if !defined( _AESOPT_H )
+#define _AESOPT_H
+
+#if defined( __cplusplus )
+#include "aescpp.h"
+#else
+#include "crypto/aes.h"
+#endif
+
+/*  PLATFORM SPECIFIC INCLUDES */
+
+#include "edefs.h"
+
+/*  CONFIGURATION - THE USE OF DEFINES
+
+    Later in this section there are a number of defines that control the
+    operation of the code.  In each section, the purpose of each define is
+    explained so that the relevant form can be included or excluded by
+    setting either 1's or 0's respectively on the branches of the related
+    #if clauses.  The following local defines should not be changed.
+*/
+
+#define ENCRYPTION_IN_C     1
+#define DECRYPTION_IN_C     2
+#define ENC_KEYING_IN_C     4
+#define DEC_KEYING_IN_C     8
+
+#define NO_TABLES           0
+#define ONE_TABLE           1
+#define FOUR_TABLES         4
+#define NONE                0
+#define PARTIAL             1
+#define FULL                2
+
+/*  --- START OF USER CONFIGURED OPTIONS --- */
+
+/*  1. BYTE ORDER WITHIN 32 BIT WORDS
+
+    The fundamental data processing units in Rijndael are 8-bit bytes. The
+    input, output and key input are all enumerated arrays of bytes in which
+    bytes are numbered starting at zero and increasing to one less than the
+    number of bytes in the array in question. This enumeration is only used
+    for naming bytes and does not imply any adjacency or order relationship
+    from one byte to another. When these inputs and outputs are considered
+    as bit sequences, bits 8*n to 8*n+7 of the bit sequence are mapped to
+    byte[n] with bit 8n+i in the sequence mapped to bit 7-i within the byte.
+    In this implementation bits are numbered from 0 to 7 starting at the
+    numerically least significant end of each byte (bit n represents 2^n).
+
+    However, Rijndael can be implemented more efficiently using 32-bit
+    words by packing bytes into words so that bytes 4*n to 4*n+3 are placed
+    into word[n]. While in principle these bytes can be assembled into words
+    in any positions, this implementation only supports the two formats in
+    which bytes in adjacent positions within words also have adjacent byte
+    numbers. This order is called big-endian if the lowest numbered bytes
+    in words have the highest numeric significance and little-endian if the
+    opposite applies.
+
+    This code can work in either order irrespective of the order used by the
+    machine on which it runs. Normally the internal byte order will be set
+    to the order of the processor on which the code is to be run but this
+    define can be used to reverse this in special situations
+
+    WARNING: Assembler code versions rely on PLATFORM_BYTE_ORDER being set.
+    This define will hence be redefined later (in section 4) if necessary
+*/
+
+#if 1 
+#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER
+#elif 0
+#define ALGORITHM_BYTE_ORDER IS_LITTLE_ENDIAN
+#elif 0
+#define ALGORITHM_BYTE_ORDER IS_BIG_ENDIAN
+#else
+#error The algorithm byte order is not defined
+#endif
+
+/*  2. VIA ACE SUPPORT
+
+    Define this option if support for the VIA ACE is required. This uses 
+    inline assembler instructions and is only implemented for the Microsoft, 
+    Intel and GCC compilers.  If VIA ACE is known to be present, then defining
+    ASSUME_VIA_ACE_PRESENT will remove the ordinary encryption/decryption 
+    code.  If USE_VIA_ACE_IF_PRESENT is defined then VIA ACE will be used if
+    it is detected (both present and enabled) but the normal AES code will 
+    also be present. 
+    
+    When VIA ACE is to be used, all AES encryption contexts MUST be 16 byte 
+    aligned; other input/output buffers do not need to be 16 byte aligned 
+    but there are very large performance gains if this can be arranged.  
+    VIA ACE also requires the decryption key schedule to be in reverse 
+    order (which the following defines ensure).
+*/
+
+#if 0 && !defined( _WIN64 ) && !defined( USE_VIA_ACE_IF_PRESENT )
+#define USE_VIA_ACE_IF_PRESENT
+#endif
+
+#if 0 && !defined( _WIN64 ) && !defined( ASSUME_VIA_ACE_PRESENT )
+#define ASSUME_VIA_ACE_PRESENT
+#endif
+
+/*  3. ASSEMBLER SUPPORT
+
+    This define (which can be on the command line) enables the use of the
+    assembler code routines for encryption, decryption and key scheduling
+    as follows:
+
+    ASM_X86_V1C uses the assembler (aes_x86_v1.asm) with large tables for 
+                encryption and decryption and but with key scheduling in C
+    ASM_X86_V2  uses assembler (aes_x86_v2.asm) with compressed tables for
+                encryption, decryption and key scheduling
+    ASM_X86_V2C	uses assembler (aes_x86_v2.asm) with compressed tables for
+                encryption and decryption and but with key scheduling in C
+    ASM_AMD64_C	uses assembler (aes_amd64.asm) with compressed tables for
+                encryption and decryption and but with key scheduling in C
+
+    Change one 'if 0' below to 'if 1' to select the version or define 
+    as a compilation option.
+*/
+
+#if defined ( ASM_X86_V1C ) || defined( ASM_X86_V2 ) || defined( ASM_X86_V2C )
+#  if defined( _M_IX86 )
+#    if 0 && !defined( ASM_X86_V1C )
+#      define ASM_X86_V1C
+#    elif 0 && !defined( ASM_X86_V2  )
+#      define ASM_X86_V2
+#    elif 0 && !defined( ASM_X86_V2C )
+#      define ASM_X86_V2C
+#    endif
+#  else
+#    error Assembler code is only available for x86 and AMD64 systems
+#  endif
+#elif defined( ASM_AMD64_C )
+#  if defined( _M_X64 )
+#    if 0 && !defined( ASM_AMD64_C )
+#      define ASM_AMD64_C
+#    endif
+#  else
+#    error Assembler code is only available for x86 and AMD64 systems
+#  endif
+#endif
+
+/*  4. FAST INPUT/OUTPUT OPERATIONS.
+
+    On some machines it is possible to improve speed by transferring the
+    bytes in the input and output arrays to and from the internal 32-bit
+    variables by addressing these arrays as if they are arrays of 32-bit
+    words.  On some machines this will always be possible but there may
+    be a large performance penalty if the byte arrays are not aligned on
+    the normal word boundaries. On other machines this technique will
+    lead to memory access errors when such 32-bit word accesses are not
+    properly aligned. The option SAFE_IO avoids such problems but will
+    often be slower on those machines that support misaligned access
+    (especially so if care is taken to align the input  and output byte
+    arrays on 32-bit word boundaries). If SAFE_IO is not defined it is
+    assumed that access to byte arrays as if they are arrays of 32-bit
+    words will not cause problems when such accesses are misaligned.
+*/
+#if 1 && !defined( _MSC_VER )
+#define SAFE_IO
+#endif
+
+/*  5. LOOP UNROLLING
+
+    The code for encryption and decrytpion cycles through a number of rounds
+    that can be implemented either in a loop or by expanding the code into a
+    long sequence of instructions, the latter producing a larger program but
+    one that will often be much faster. The latter is called loop unrolling.
+    There are also potential speed advantages in expanding two iterations in
+    a loop with half the number of iterations, which is called partial loop
+    unrolling.  The following options allow partial or full loop unrolling
+    to be set independently for encryption and decryption
+*/
+#if 1
+#define ENC_UNROLL  FULL
+#elif 0
+#define ENC_UNROLL  PARTIAL
+#else
+#define ENC_UNROLL  NONE
+#endif
+
+#if 1
+#define DEC_UNROLL  FULL
+#elif 0
+#define DEC_UNROLL  PARTIAL
+#else
+#define DEC_UNROLL  NONE
+#endif
+
+/*  6. FAST FINITE FIELD OPERATIONS
+
+    If this section is included, tables are used to provide faster finite
+    field arithmetic (this has no effect if FIXED_TABLES is defined).
+*/
+#if 1
+#define FF_TABLES
+#endif
+
+/*  7. INTERNAL STATE VARIABLE FORMAT
+
+    The internal state of Rijndael is stored in a number of local 32-bit
+    word varaibles which can be defined either as an array or as individual
+    names variables. Include this section if you want to store these local
+    varaibles in arrays. Otherwise individual local variables will be used.
+*/
+#if 1
+#define ARRAYS
+#endif
+
+/*  8. FIXED OR DYNAMIC TABLES
+
+    When this section is included the tables used by the code are compiled
+    statically into the binary file.  Otherwise the subroutine gen_tabs()
+    must be called to compute them before the code is first used.
+*/
+#if 0 && !(defined( _MSC_VER ) && ( _MSC_VER <= 800 )) 
+#define FIXED_TABLES
+#endif
+
+/*  9. TABLE ALIGNMENT
+
+    On some sytsems speed will be improved by aligning the AES large lookup
+    tables on particular boundaries. This define should be set to a power of
+    two giving the desired alignment. It can be left undefined if alignment
+    is not needed.  This option is specific to the Microsft VC++ compiler -
+    it seems to sometimes cause trouble for the VC++ version 6 compiler.
+*/
+
+#if 1 && defined( _MSC_VER ) && ( _MSC_VER >= 1300 )
+#define TABLE_ALIGN 32
+#endif
+
+/*  10. TABLE OPTIONS
+    
+    This cipher proceeds by repeating in a number of cycles known as 'rounds'
+    which are implemented by a round function which can optionally be speeded
+    up using tables.  The basic tables are each 256 32-bit words, with either
+    one or four tables being required for each round function depending on
+    how much speed is required. The encryption and decryption round functions
+    are different and the last encryption and decrytpion round functions are
+    different again making four different round functions in all.
+
+    This means that:
+      1. Normal encryption and decryption rounds can each use either 0, 1
+         or 4 tables and table spaces of 0, 1024 or 4096 bytes each.
+      2. The last encryption and decryption rounds can also use either 0, 1
+         or 4 tables and table spaces of 0, 1024 or 4096 bytes each.
+
+    Include or exclude the appropriate definitions below to set the number
+    of tables used by this implementation.
+*/
+
+#if 1   /* set tables for the normal encryption round */
+#define ENC_ROUND   FOUR_TABLES
+#elif 0
+#define ENC_ROUND   ONE_TABLE
+#else
+#define ENC_ROUND   NO_TABLES
+#endif
+
+#if 1   /* set tables for the last encryption round */
+#define LAST_ENC_ROUND  FOUR_TABLES
+#elif 0
+#define LAST_ENC_ROUND  ONE_TABLE
+#else
+#define LAST_ENC_ROUND  NO_TABLES
+#endif
+
+#if 1   /* set tables for the normal decryption round */
+#define DEC_ROUND   FOUR_TABLES
+#elif 0
+#define DEC_ROUND   ONE_TABLE
+#else
+#define DEC_ROUND   NO_TABLES
+#endif
+
+#if 1   /* set tables for the last decryption round */
+#define LAST_DEC_ROUND  FOUR_TABLES
+#elif 0
+#define LAST_DEC_ROUND  ONE_TABLE
+#else
+#define LAST_DEC_ROUND  NO_TABLES
+#endif
+
+/*  The decryption key schedule can be speeded up with tables in the same
+    way that the round functions can.  Include or exclude the following
+    defines to set this requirement.
+*/
+#if 1
+#define KEY_SCHED   FOUR_TABLES
+#elif 0
+#define KEY_SCHED   ONE_TABLE
+#else
+#define KEY_SCHED   NO_TABLES
+#endif
+
+/*  ---- END OF USER CONFIGURED OPTIONS ---- */
+
+/* VIA ACE support is only available for VC++ and GCC */
+
+#if !defined( _MSC_VER ) && !defined( __GNUC__ )
+#  if defined( ASSUME_VIA_ACE_PRESENT )
+#    undef ASSUME_VIA_ACE_PRESENT
+#  endif
+#  if defined( USE_VIA_ACE_IF_PRESENT )
+#    undef USE_VIA_ACE_IF_PRESENT
+#  endif
+#endif
+
+#if defined( ASSUME_VIA_ACE_PRESENT ) && !defined( USE_VIA_ACE_IF_PRESENT )
+#define USE_VIA_ACE_IF_PRESENT
+#endif
+
+#if defined( USE_VIA_ACE_IF_PRESENT ) && !defined ( AES_REV_DKS )
+#define AES_REV_DKS
+#endif
+
+/* Assembler support requires the use of platform byte order */
+
+#if ( defined( ASM_X86_V1C ) || defined( ASM_X86_V2C ) || defined( ASM_AMD64_C ) ) && (ALGORITHM_BYTE_ORDER != PLATFORM_BYTE_ORDER)
+#undef  ALGORITHM_BYTE_ORDER
+#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER
+#endif
+
+/* In this implementation the columns of the state array are each held in
+   32-bit words. The state array can be held in various ways: in an array
+   of words, in a number of individual word variables or in a number of
+   processor registers. The following define maps a variable name x and
+   a column number c to the way the state array variable is to be held.
+   The first define below maps the state into an array x[c] whereas the
+   second form maps the state into a number of individual variables x0,
+   x1, etc.  Another form could map individual state colums to machine
+   register names.
+*/
+
+#if defined( ARRAYS )
+#define s(x,c) x[c]
+#else
+#define s(x,c) x##c
+#endif
+
+/*  This implementation provides subroutines for encryption, decryption
+    and for setting the three key lengths (separately) for encryption
+    and decryption. Since not all functions are needed, masks are set 
+    up here to determine which will be implemented in C
+*/
+
+#if !defined( AES_ENCRYPT )
+#  define EFUNCS_IN_C   0
+#elif defined( ASSUME_VIA_ACE_PRESENT ) || defined( ASM_X86_V1C )
+    || defined( ASM_X86_V2C ) || defined( ASM_AMD64_C )
+#  define EFUNCS_IN_C   ENC_KEYING_IN_C
+#elif !defined( ASM_X86_V2 )
+#  define EFUNCS_IN_C   ( ENCRYPTION_IN_C | ENC_KEYING_IN_C )
+#else
+#  define EFUNCS_IN_C   0
+#endif
+
+#if !defined( AES_DECRYPT )
+#  define DFUNCS_IN_C   0
+#elif defined( ASSUME_VIA_ACE_PRESENT ) || defined( ASM_X86_V1C )
+    || defined( ASM_X86_V2C ) || defined( ASM_AMD64_C )	
+#  define DFUNCS_IN_C   DEC_KEYING_IN_C
+#elif !defined( ASM_X86_V2 )
+#  define DFUNCS_IN_C   ( DECRYPTION_IN_C | DEC_KEYING_IN_C )
+#else
+#  define DFUNCS_IN_C   0
+#endif
+
+#define FUNCS_IN_C  ( EFUNCS_IN_C | DFUNCS_IN_C )
+
+/* END OF CONFIGURATION OPTIONS */
+
+#define RC_LENGTH   (5 * (AES_BLOCK_SIZE / 4 - 2))
+
+/* Disable or report errors on some combinations of options */
+
+#if ENC_ROUND == NO_TABLES && LAST_ENC_ROUND != NO_TABLES
+#undef  LAST_ENC_ROUND
+#define LAST_ENC_ROUND  NO_TABLES
+#elif ENC_ROUND == ONE_TABLE && LAST_ENC_ROUND == FOUR_TABLES
+#undef  LAST_ENC_ROUND
+#define LAST_ENC_ROUND  ONE_TABLE
+#endif
+
+#if ENC_ROUND == NO_TABLES && ENC_UNROLL != NONE
+#undef  ENC_UNROLL
+#define ENC_UNROLL  NONE
+#endif
+
+#if DEC_ROUND == NO_TABLES && LAST_DEC_ROUND != NO_TABLES
+#undef  LAST_DEC_ROUND
+#define LAST_DEC_ROUND  NO_TABLES
+#elif DEC_ROUND == ONE_TABLE && LAST_DEC_ROUND == FOUR_TABLES
+#undef  LAST_DEC_ROUND
+#define LAST_DEC_ROUND  ONE_TABLE
+#endif
+
+#if DEC_ROUND == NO_TABLES && DEC_UNROLL != NONE
+#undef  DEC_UNROLL
+#define DEC_UNROLL  NONE
+#endif
+
+#if defined( bswap32 )
+#define aes_sw32    bswap32
+#elif defined( bswap_32 )
+#define aes_sw32    bswap_32
+#else
+#define brot(x,n)   (((uint_32t)(x) <<  n) | ((uint_32t)(x) >> (32 - n)))
+#define aes_sw32(x) ((brot((x),8) & 0x00ff00ff) | (brot((x),24) & 0xff00ff00))
+#endif
+
+/*  upr(x,n):  rotates bytes within words by n positions, moving bytes to
+               higher index positions with wrap around into low positions
+    ups(x,n):  moves bytes by n positions to higher index positions in
+               words but without wrap around
+    bval(x,n): extracts a byte from a word
+
+    WARNING:   The definitions given here are intended only for use with
+               unsigned variables and with shift counts that are compile
+               time constants
+*/
+
+#if ( ALGORITHM_BYTE_ORDER == IS_LITTLE_ENDIAN )
+#define upr(x,n)        (((uint_32t)(x) << (8 * (n))) | ((uint_32t)(x) >> (32 - 8 * (n))))
+#define ups(x,n)        ((uint_32t) (x) << (8 * (n)))
+#define bval(x,n)       ((uint_8t)((x) >> (8 * (n))))
+#define bytes2word(b0, b1, b2, b3)  \
+        (((uint_32t)(b3) << 24) | ((uint_32t)(b2) << 16) | ((uint_32t)(b1) << 8) | (b0))
+#endif
+
+#if ( ALGORITHM_BYTE_ORDER == IS_BIG_ENDIAN )
+#define upr(x,n)        (((uint_32t)(x) >> (8 * (n))) | ((uint_32t)(x) << (32 - 8 * (n))))
+#define ups(x,n)        ((uint_32t) (x) >> (8 * (n)))
+#define bval(x,n)       ((uint_8t)((x) >> (24 - 8 * (n))))
+#define bytes2word(b0, b1, b2, b3)  \
+        (((uint_32t)(b0) << 24) | ((uint_32t)(b1) << 16) | ((uint_32t)(b2) << 8) | (b3))
+#endif
+
+#if defined( SAFE_IO )
+
+#define word_in(x,c)    bytes2word(((const uint_8t*)(x)+4*c)[0], ((const uint_8t*)(x)+4*c)[1], \
+                                   ((const uint_8t*)(x)+4*c)[2], ((const uint_8t*)(x)+4*c)[3])
+#define word_out(x,c,v) { ((uint_8t*)(x)+4*c)[0] = bval(v,0); ((uint_8t*)(x)+4*c)[1] = bval(v,1); \
+                          ((uint_8t*)(x)+4*c)[2] = bval(v,2); ((uint_8t*)(x)+4*c)[3] = bval(v,3); }
+
+#elif ( ALGORITHM_BYTE_ORDER == PLATFORM_BYTE_ORDER )
+
+#define word_in(x,c)    (*((uint_32t*)(x)+(c)))
+#define word_out(x,c,v) (*((uint_32t*)(x)+(c)) = (v))
+
+#else
+
+#define word_in(x,c)    aes_sw32(*((uint_32t*)(x)+(c)))
+#define word_out(x,c,v) (*((uint_32t*)(x)+(c)) = aes_sw32(v))
+
+#endif
+
+/* the finite field modular polynomial and elements */
+
+#define WPOLY   0x011b
+#define BPOLY     0x1b
+
+/* multiply four bytes in GF(2^8) by 'x' {02} in parallel */
+
+#define m1  0x80808080
+#define m2  0x7f7f7f7f
+#define gf_mulx(x)  ((((x) & m2) << 1) ^ ((((x) & m1) >> 7) * BPOLY))
+
+/* The following defines provide alternative definitions of gf_mulx that might
+   give improved performance if a fast 32-bit multiply is not available. Note
+   that a temporary variable u needs to be defined where gf_mulx is used.
+
+#define gf_mulx(x) (u = (x) & m1, u |= (u >> 1), ((x) & m2) << 1) ^ ((u >> 3) | (u >> 6))
+#define m4  (0x01010101 * BPOLY)
+#define gf_mulx(x) (u = (x) & m1, ((x) & m2) << 1) ^ ((u - (u >> 7)) & m4)
+*/
+
+/* Work out which tables are needed for the different options   */
+
+#if defined( ASM_X86_V1C )
+#if defined( ENC_ROUND )
+#undef  ENC_ROUND
+#endif
+#define ENC_ROUND   FOUR_TABLES
+#if defined( LAST_ENC_ROUND )
+#undef  LAST_ENC_ROUND
+#endif
+#define LAST_ENC_ROUND  FOUR_TABLES
+#if defined( DEC_ROUND )
+#undef  DEC_ROUND
+#endif
+#define DEC_ROUND   FOUR_TABLES
+#if defined( LAST_DEC_ROUND )
+#undef  LAST_DEC_ROUND
+#endif
+#define LAST_DEC_ROUND  FOUR_TABLES
+#if defined( KEY_SCHED )
+#undef  KEY_SCHED
+#define KEY_SCHED   FOUR_TABLES
+#endif
+#endif
+
+#if ( FUNCS_IN_C & ENCRYPTION_IN_C ) || defined( ASM_X86_V1C )
+#if ENC_ROUND == ONE_TABLE
+#define FT1_SET
+#elif ENC_ROUND == FOUR_TABLES
+#define FT4_SET
+#else
+#define SBX_SET
+#endif
+#if LAST_ENC_ROUND == ONE_TABLE
+#define FL1_SET
+#elif LAST_ENC_ROUND == FOUR_TABLES
+#define FL4_SET
+#elif !defined( SBX_SET )
+#define SBX_SET
+#endif
+#endif
+
+#if ( FUNCS_IN_C & DECRYPTION_IN_C ) || defined( ASM_X86_V1C )
+#if DEC_ROUND == ONE_TABLE
+#define IT1_SET
+#elif DEC_ROUND == FOUR_TABLES
+#define IT4_SET
+#else
+#define ISB_SET
+#endif
+#if LAST_DEC_ROUND == ONE_TABLE
+#define IL1_SET
+#elif LAST_DEC_ROUND == FOUR_TABLES
+#define IL4_SET
+#elif !defined(ISB_SET)
+#define ISB_SET
+#endif
+#endif
+
+#if (FUNCS_IN_C & ENC_KEYING_IN_C) || (FUNCS_IN_C & DEC_KEYING_IN_C)
+#if KEY_SCHED == ONE_TABLE
+#define LS1_SET
+#elif KEY_SCHED == FOUR_TABLES
+#define LS4_SET
+#elif !defined( SBX_SET )
+#define SBX_SET
+#endif
+#endif
+
+#if (FUNCS_IN_C & DEC_KEYING_IN_C)
+#if KEY_SCHED == ONE_TABLE
+#define IM1_SET
+#elif KEY_SCHED == FOUR_TABLES
+#define IM4_SET
+#elif !defined( SBX_SET )
+#define SBX_SET
+#endif
+#endif
+
+/* generic definitions of Rijndael macros that use tables    */
+
+#define no_table(x,box,vf,rf,c) bytes2word( \
+    box[bval(vf(x,0,c),rf(0,c))], \
+    box[bval(vf(x,1,c),rf(1,c))], \
+    box[bval(vf(x,2,c),rf(2,c))], \
+    box[bval(vf(x,3,c),rf(3,c))])
+
+#define one_table(x,op,tab,vf,rf,c) \
+ (     tab[bval(vf(x,0,c),rf(0,c))] \
+  ^ op(tab[bval(vf(x,1,c),rf(1,c))],1) \
+  ^ op(tab[bval(vf(x,2,c),rf(2,c))],2) \
+  ^ op(tab[bval(vf(x,3,c),rf(3,c))],3))
+
+#define four_tables(x,tab,vf,rf,c) \
+ (  tab[0][bval(vf(x,0,c),rf(0,c))] \
+  ^ tab[1][bval(vf(x,1,c),rf(1,c))] \
+  ^ tab[2][bval(vf(x,2,c),rf(2,c))] \
+  ^ tab[3][bval(vf(x,3,c),rf(3,c))])
+
+#define vf1(x,r,c)  (x)
+#define rf1(r,c)    (r)
+#define rf2(r,c)    ((8+r-c)&3)
+
+/* perform forward and inverse column mix operation on four bytes in long word x in */
+/* parallel. NOTE: x must be a simple variable, NOT an expression in these macros.  */
+
+#if defined( FM4_SET )    /* not currently used */
+#define fwd_mcol(x)       four_tables(x,t_use(f,m),vf1,rf1,0)
+#elif defined( FM1_SET )  /* not currently used */
+#define fwd_mcol(x)       one_table(x,upr,t_use(f,m),vf1,rf1,0)
+#else
+#define dec_fmvars        uint_32t g2
+#define fwd_mcol(x)       (g2 = gf_mulx(x), g2 ^ upr((x) ^ g2, 3) ^ upr((x), 2) ^ upr((x), 1))
+#endif
+
+#if defined( IM4_SET )
+#define inv_mcol(x)       four_tables(x,t_use(i,m),vf1,rf1,0)
+#elif defined( IM1_SET )
+#define inv_mcol(x)       one_table(x,upr,t_use(i,m),vf1,rf1,0)
+#else
+#define dec_imvars        uint_32t g2, g4, g9
+#define inv_mcol(x)       (g2 = gf_mulx(x), g4 = gf_mulx(g2), g9 = (x) ^ gf_mulx(g4), g4 ^= g9, \
+                          (x) ^ g2 ^ g4 ^ upr(g2 ^ g9, 3) ^ upr(g4, 2) ^ upr(g9, 1))
+#endif
+
+#if defined( FL4_SET )
+#define ls_box(x,c)       four_tables(x,t_use(f,l),vf1,rf2,c)
+#elif   defined( LS4_SET )
+#define ls_box(x,c)       four_tables(x,t_use(l,s),vf1,rf2,c)
+#elif defined( FL1_SET )
+#define ls_box(x,c)       one_table(x,upr,t_use(f,l),vf1,rf2,c)
+#elif defined( LS1_SET )
+#define ls_box(x,c)       one_table(x,upr,t_use(l,s),vf1,rf2,c)
+#else
+#define ls_box(x,c)     no_table(x,t_use(s,box),vf1,rf2,c)
+#endif
+
+#if defined( ASM_X86_V1C ) && defined( AES_DECRYPT ) && !defined( ISB_SET )
+#define ISB_SET
+#endif
+
+#endif
diff --git a/bsd/crypto/aes/i386/edefs.h b/bsd/crypto/aes/i386/edefs.h
new file mode 100644
index 000000000..d25bef89c
--- /dev/null
+++ b/bsd/crypto/aes/i386/edefs.h
@@ -0,0 +1,130 @@
+/*
+ ---------------------------------------------------------------------------
+ Copyright (c) 2003, Dr Brian Gladman, Worcester, UK.   All rights reserved.
+
+ LICENSE TERMS
+
+ The free distribution and use of this software in both source and binary
+ form is allowed (with or without changes) provided that:
+
+   1. distributions of this source code include the above copyright
+      notice, this list of conditions and the following disclaimer;
+
+   2. distributions in binary form include the above copyright
+      notice, this list of conditions and the following disclaimer
+      in the documentation and/or other associated materials;
+
+   3. the copyright holder's name is not used to endorse products
+      built using this software without specific written permission.
+
+ ALTERNATIVELY, provided that this notice is retained in full, this product
+ may be distributed under the terms of the GNU General Public License (GPL),
+ in which case the provisions of the GPL apply INSTEAD OF those given above.
+
+ DISCLAIMER
+
+ This software is provided 'as is' with no explicit or implied warranties
+ in respect of its properties, including, but not limited to, correctness
+ and/or fitness for purpose.
+ ---------------------------------------------------------------------------
+ Issue 31/01/2006
+*/
+
+#ifndef EDEFS_H
+#define EDEFS_H
+#if defined(__cplusplus)
+extern "C"
+{
+#endif
+
+#define IS_LITTLE_ENDIAN   1234 /* byte 0 is least significant (i386) */
+#define IS_BIG_ENDIAN      4321 /* byte 0 is most significant (mc68k) */
+
+#if defined(__GNUC__) || defined(__GNU_LIBRARY__)
+#  if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__)
+#    include <sys/endian.h>
+#  elif defined( BSD ) && ( BSD >= 199103 ) || defined( __DJGPP__ ) || defined( __CYGWIN32__ ) 
+#      include <machine/endian.h>
+#  elif defined(__APPLE__)
+#    if defined(__BIG_ENDIAN__) && !defined( BIG_ENDIAN )
+#      define BIG_ENDIAN
+#    elif defined(__LITTLE_ENDIAN__) && !defined( LITTLE_ENDIAN )
+#      define LITTLE_ENDIAN
+#    endif
+#  elif !defined( __MINGW32__ )
+#    include <endian.h>
+#    if !defined(__BEOS__)
+#      include <byteswap.h>
+#    endif
+#  endif
+#endif
+
+#if !defined(PLATFORM_BYTE_ORDER)
+#  if defined(LITTLE_ENDIAN) || defined(BIG_ENDIAN)
+#    if    defined(LITTLE_ENDIAN) && !defined(BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif !defined(LITTLE_ENDIAN) &&  defined(BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#    elif defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#    endif
+#  elif defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)
+#    if    defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif !defined(_LITTLE_ENDIAN) &&  defined(_BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#    elif defined(_BYTE_ORDER) && (_BYTE_ORDER == _LITTLE_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif defined(_BYTE_ORDER) && (_BYTE_ORDER == _BIG_ENDIAN)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#   endif
+#  elif defined(__LITTLE_ENDIAN__) || defined(__BIG_ENDIAN__)
+#    if    defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif !defined(__LITTLE_ENDIAN__) &&  defined(__BIG_ENDIAN__)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#    elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __LITTLE_ENDIAN__)
+#      define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#    elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __BIG_ENDIAN__)
+#      define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#    endif
+#  endif
+#endif
+
+/*  if the platform is still unknown, try to find its byte order    */
+/*  from commonly used machine defines                              */
+
+#if !defined(PLATFORM_BYTE_ORDER)
+
+#if   defined( __alpha__ ) || defined( __alpha ) || defined( i386 )       || \
+      defined( __i386__ )  || defined( _M_I86 )  || defined( _M_IX86 )    || \
+      defined( __OS2__ )   || defined( sun386 )  || defined( __TURBOC__ ) || \
+      defined( vax )       || defined( vms )     || defined( VMS )        || \
+      defined( __VMS )     || defined( _M_X64 )
+#  define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+
+#elif defined( AMIGA )    || defined( applec )  || defined( __AS400__ )  || \
+      defined( _CRAY )    || defined( __hppa )  || defined( __hp9000 )   || \
+      defined( ibm370 )   || defined( mc68000 ) || defined( m68k )       || \
+      defined( __MRC__ )  || defined( __MVS__ ) || defined( __MWERKS__ ) || \
+      defined( sparc )    || defined( __sparc)  || defined( SYMANTEC_C ) || \
+      defined( __TANDEM ) || defined( THINK_C ) || defined( __VMCMS__ )  || \
+	  defined( __VOS__ )
+#  define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+
+#elif 0     /* **** EDIT HERE IF NECESSARY **** */
+#  define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN
+#elif 0     /* **** EDIT HERE IF NECESSARY **** */
+#  define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN
+#else
+#  error Please edit edefs.h (lines 117 or 119) to set the platform byte order
+#endif
+
+#endif
+
+#if defined(__cplusplus)
+}
+#endif
+#endif
diff --git a/iokit/include/bsddev/i386/Makefile b/bsd/crypto/aes/ppc/Makefile
similarity index 55%
rename from iokit/include/bsddev/i386/Makefile
rename to bsd/crypto/aes/ppc/Makefile
index fd80bdd0b..99755ad2e 100644
--- a/iokit/include/bsddev/i386/Makefile
+++ b/bsd/crypto/aes/ppc/Makefile
@@ -3,6 +3,7 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
@@ -12,26 +13,22 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
-EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
-
-EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
+EXPINC_SUBDIRS = \
 
-EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_PPC = \
 
-DATAFILES = \
-            evsio.h		\
-	    evio.h		\
-	    event.h		\
-	    EventShmemLock.h
+EXPINC_SUBDIRS_I386 = \
 
-INSTALL_MD_LIST	= ${DATAFILES}
+PRIVATE_DATAFILES = \
+	aestab.h aesopt.h
 
-INSTALL_MD_DIR = dev/i386
+INSTALL_MI_DIR = crypto
 
-EXPORT_MD_LIST	= \
+EXPORT_MI_DIR = ${INSTALL_MI_DIR}
 
-EXPORT_MD_DIR = \
+INSTALL_KF_MI_LIST =
 
+INSTALL_KF_MI_LCL_LIST = ${PRIVATE_DATAFILES}
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/bsd/crypto/aes/aescrypt.c b/bsd/crypto/aes/ppc/aescrypt.c
similarity index 100%
rename from bsd/crypto/aes/aescrypt.c
rename to bsd/crypto/aes/ppc/aescrypt.c
diff --git a/bsd/crypto/aes/aeskey.c b/bsd/crypto/aes/ppc/aeskey.c
similarity index 100%
rename from bsd/crypto/aes/aeskey.c
rename to bsd/crypto/aes/ppc/aeskey.c
diff --git a/bsd/crypto/aes/aesopt.h b/bsd/crypto/aes/ppc/aesopt.h
similarity index 99%
rename from bsd/crypto/aes/aesopt.h
rename to bsd/crypto/aes/ppc/aesopt.h
index 976ae902c..2b78eb920 100644
--- a/bsd/crypto/aes/aesopt.h
+++ b/bsd/crypto/aes/ppc/aesopt.h
@@ -134,7 +134,7 @@
 #if !defined( _AESOPT_H )
 #define _AESOPT_H
 
-#include "aes.h"
+#include <crypto/aes/aes.h>
 
 /*  CONFIGURATION - USE OF DEFINES
 
diff --git a/bsd/crypto/aes/aestab.c b/bsd/crypto/aes/ppc/aestab.c
similarity index 100%
rename from bsd/crypto/aes/aestab.c
rename to bsd/crypto/aes/ppc/aestab.c
diff --git a/bsd/crypto/aes/aestab.h b/bsd/crypto/aes/ppc/aestab.h
similarity index 100%
rename from bsd/crypto/aes/aestab.h
rename to bsd/crypto/aes/ppc/aestab.h
diff --git a/bsd/crypto/blowfish/bf_enc.c b/bsd/crypto/blowfish/bf_enc.c
index 4a31e2ee9..f17980d9b 100644
--- a/bsd/crypto/blowfish/bf_enc.c
+++ b/bsd/crypto/blowfish/bf_enc.c
@@ -75,11 +75,9 @@ to modify the code.
 
 /* XXX "data" is host endian */
 void
-BF_encrypt(data, key)
-	BF_LONG *data;
-	BF_KEY *key;
+BF_encrypt(BF_LONG *data, BF_KEY *key)
 {
-	register BF_LONG l, r, *p, *s;
+	BF_LONG l, r, *p, *s;
 
 	p = key->P;
 	s= &key->S[0];
@@ -117,11 +115,9 @@ BF_encrypt(data, key)
 
 /* XXX "data" is host endian */
 void
-BF_decrypt(data, key)
-	BF_LONG *data;
-	BF_KEY *key;
+BF_decrypt(BF_LONG *data, BF_KEY *key)
 {
-	register BF_LONG l, r, *p, *s;
+	BF_LONG l, r, *p, *s;
 
 	p = key->P;
 	s= &key->S[0];
diff --git a/bsd/crypto/des/Makefile b/bsd/crypto/des/Makefile
index 08483b21a..c4c411272 100644
--- a/bsd/crypto/des/Makefile
+++ b/bsd/crypto/des/Makefile
@@ -13,12 +13,16 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 PRIVATE_DATAFILES = \
 	des.h
 
diff --git a/bsd/crypto/des/des_ecb.c b/bsd/crypto/des/des_ecb.c
index cc9c5697b..4a3ea9959 100644
--- a/bsd/crypto/des/des_ecb.c
+++ b/bsd/crypto/des/des_ecb.c
@@ -89,7 +89,8 @@ char *des_options(void)
                         size="int";
                 else
                         size="long";
-                sprintf(buf,"des(%s,%s,%s,%s)",ptr,risc,unroll,size);
+                snprintf(buf, sizeof(buf), "des(%s,%s,%s,%s)",
+				ptr, risc, unroll, size);
                 init=0;
                 }
         return(buf);
diff --git a/bsd/crypto/md5.c b/bsd/crypto/md5.c
deleted file mode 100644
index 734232dac..000000000
--- a/bsd/crypto/md5.c
+++ /dev/null
@@ -1,308 +0,0 @@
-/*	$FreeBSD: src/sys/crypto/md5.c,v 1.1.2.2 2001/07/03 11:01:27 ume Exp $	*/
-/*	$KAME: md5.c,v 1.5 2000/11/08 06:13:08 itojun Exp $	*/
-
-/*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/types.h>
-#include <sys/cdefs.h>
-#include <sys/time.h>
-#include <sys/systm.h>
-#include <crypto/md5.h>
-
-#define SHIFT(X, s) (((X) << (s)) | ((X) >> (32 - (s))))
-
-#define F(X, Y, Z) (((X) & (Y)) | ((~X) & (Z)))
-#define G(X, Y, Z) (((X) & (Z)) | ((Y) & (~Z)))
-#define H(X, Y, Z) ((X) ^ (Y) ^ (Z))
-#define I(X, Y, Z) ((Y) ^ ((X) | (~Z)))
-
-#define ROUND1(a, b, c, d, k, s, i) { \
-	(a) = (a) + F((b), (c), (d)) + X[(k)] + T[(i)]; \
-	(a) = SHIFT((a), (s)); \
-	(a) = (b) + (a); \
-}
-
-#define ROUND2(a, b, c, d, k, s, i) { \
-	(a) = (a) + G((b), (c), (d)) + X[(k)] + T[(i)]; \
-	(a) = SHIFT((a), (s)); \
-	(a) = (b) + (a); \
-}
-
-#define ROUND3(a, b, c, d, k, s, i) { \
-	(a) = (a) + H((b), (c), (d)) + X[(k)] + T[(i)]; \
-	(a) = SHIFT((a), (s)); \
-	(a) = (b) + (a); \
-}
-
-#define ROUND4(a, b, c, d, k, s, i) { \
-	(a) = (a) + I((b), (c), (d)) + X[(k)] + T[(i)]; \
-	(a) = SHIFT((a), (s)); \
-	(a) = (b) + (a); \
-}
-
-#define Sa	 7
-#define Sb	12
-#define Sc	17
-#define Sd	22
-
-#define Se	 5
-#define Sf	 9
-#define Sg	14
-#define Sh	20
-
-#define Si	 4
-#define Sj	11
-#define Sk	16
-#define Sl	23
-
-#define Sm	 6
-#define Sn	10
-#define So	15
-#define Sp	21
-
-#define MD5_A0	0x67452301
-#define MD5_B0	0xefcdab89
-#define MD5_C0	0x98badcfe
-#define MD5_D0	0x10325476
-
-/* Integer part of 4294967296 times abs(sin(i)), where i is in radians. */
-static const u_int32_t T[65] = {
-	0,
-	0xd76aa478, 	0xe8c7b756,	0x242070db,	0xc1bdceee,
-	0xf57c0faf,	0x4787c62a, 	0xa8304613,	0xfd469501,
-	0x698098d8,	0x8b44f7af,	0xffff5bb1,	0x895cd7be,
-	0x6b901122, 	0xfd987193, 	0xa679438e,	0x49b40821,
-
-	0xf61e2562,	0xc040b340, 	0x265e5a51, 	0xe9b6c7aa,
-	0xd62f105d,	0x2441453,	0xd8a1e681,	0xe7d3fbc8,
-	0x21e1cde6,	0xc33707d6, 	0xf4d50d87, 	0x455a14ed,
-	0xa9e3e905,	0xfcefa3f8, 	0x676f02d9, 	0x8d2a4c8a,
-
-	0xfffa3942,	0x8771f681, 	0x6d9d6122, 	0xfde5380c,
-	0xa4beea44, 	0x4bdecfa9, 	0xf6bb4b60, 	0xbebfbc70,
-	0x289b7ec6, 	0xeaa127fa, 	0xd4ef3085,	0x4881d05,
-	0xd9d4d039, 	0xe6db99e5, 	0x1fa27cf8, 	0xc4ac5665,
-
-	0xf4292244, 	0x432aff97, 	0xab9423a7, 	0xfc93a039,
-	0x655b59c3, 	0x8f0ccc92, 	0xffeff47d, 	0x85845dd1,
-	0x6fa87e4f, 	0xfe2ce6e0, 	0xa3014314, 	0x4e0811a1,
-	0xf7537e82, 	0xbd3af235, 	0x2ad7d2bb, 	0xeb86d391,
-};
-
-static const u_int8_t md5_paddat[MD5_BUFLEN] = {
-	0x80,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,
-	0,	0,	0,	0,	0,	0,	0,	0,	
-};
-
-static void md5_calc(u_int8_t *, md5_ctxt *);
-
-void md5_init(ctxt)
-	md5_ctxt *ctxt;
-{
-	ctxt->md5_n = 0;
-	ctxt->md5_i = 0;
-	ctxt->md5_sta = MD5_A0;
-	ctxt->md5_stb = MD5_B0;
-	ctxt->md5_stc = MD5_C0;
-	ctxt->md5_std = MD5_D0;
-	bzero(ctxt->md5_buf, sizeof(ctxt->md5_buf));
-}
-
-void md5_loop(ctxt, input, len)
-	md5_ctxt *ctxt;
-	u_int8_t *input;
-	u_int len; /* number of bytes */
-{
-	u_int gap, i;
-
-	ctxt->md5_n += len * 8; /* byte to bit */
-	gap = MD5_BUFLEN - ctxt->md5_i;
-
-	if (len >= gap) {
-		bcopy((void *)input, (void *)(ctxt->md5_buf + ctxt->md5_i),
-			gap);
-		md5_calc(ctxt->md5_buf, ctxt);
-
-		for (i = gap; i + MD5_BUFLEN <= len; i += MD5_BUFLEN) {
-			md5_calc((u_int8_t *)(input + i), ctxt);
-		}
-		
-		ctxt->md5_i = len - i;
-		bcopy((void *)(input + i), (void *)ctxt->md5_buf, ctxt->md5_i);
-	} else {
-		bcopy((void *)input, (void *)(ctxt->md5_buf + ctxt->md5_i),
-			len);
-		ctxt->md5_i += len;
-	}
-}
-
-void md5_pad(ctxt)
-	md5_ctxt *ctxt;
-{
-	u_int gap;
-
-	/* Don't count up padding. Keep md5_n. */	
-	gap = MD5_BUFLEN - ctxt->md5_i;
-	if (gap > 8) {
-		bcopy((void *)md5_paddat,
-		      (void *)(ctxt->md5_buf + ctxt->md5_i),
-		      gap - sizeof(ctxt->md5_n));
-	} else {
-		/* including gap == 8 */
-		bcopy((void *)md5_paddat, (void *)(ctxt->md5_buf + ctxt->md5_i),
-			gap);
-		md5_calc(ctxt->md5_buf, ctxt);
-		bcopy((void *)(md5_paddat + gap),
-		      (void *)ctxt->md5_buf,
-		      MD5_BUFLEN - sizeof(ctxt->md5_n));
-	}
-
-	/* 8 byte word */	
-#if BYTE_ORDER == LITTLE_ENDIAN
-	bcopy(&ctxt->md5_n8[0], &ctxt->md5_buf[56], 8);
-#endif
-#if BYTE_ORDER == BIG_ENDIAN
-	ctxt->md5_buf[56] = ctxt->md5_n8[7];
-	ctxt->md5_buf[57] = ctxt->md5_n8[6];
-	ctxt->md5_buf[58] = ctxt->md5_n8[5];
-	ctxt->md5_buf[59] = ctxt->md5_n8[4];
-	ctxt->md5_buf[60] = ctxt->md5_n8[3];
-	ctxt->md5_buf[61] = ctxt->md5_n8[2];
-	ctxt->md5_buf[62] = ctxt->md5_n8[1];
-	ctxt->md5_buf[63] = ctxt->md5_n8[0];
-#endif
-
-	md5_calc(ctxt->md5_buf, ctxt);
-}
-
-void md5_result(digest, ctxt)
-	u_int8_t *digest;
-	md5_ctxt *ctxt;
-{
-	/* 4 byte words */
-#if BYTE_ORDER == LITTLE_ENDIAN
-	bcopy(&ctxt->md5_st8[0], digest, 16);
-#endif
-#if BYTE_ORDER == BIG_ENDIAN
-	digest[ 0] = ctxt->md5_st8[ 3]; digest[ 1] = ctxt->md5_st8[ 2];
-	digest[ 2] = ctxt->md5_st8[ 1]; digest[ 3] = ctxt->md5_st8[ 0];
-	digest[ 4] = ctxt->md5_st8[ 7]; digest[ 5] = ctxt->md5_st8[ 6];
-	digest[ 6] = ctxt->md5_st8[ 5]; digest[ 7] = ctxt->md5_st8[ 4];
-	digest[ 8] = ctxt->md5_st8[11]; digest[ 9] = ctxt->md5_st8[10];
-	digest[10] = ctxt->md5_st8[ 9]; digest[11] = ctxt->md5_st8[ 8];
-	digest[12] = ctxt->md5_st8[15]; digest[13] = ctxt->md5_st8[14];
-	digest[14] = ctxt->md5_st8[13]; digest[15] = ctxt->md5_st8[12];
-#endif
-}
-
-#if BYTE_ORDER == BIG_ENDIAN
-u_int32_t X[16];
-#endif
-
-static void md5_calc(b64, ctxt)
-	u_int8_t *b64;
-	md5_ctxt *ctxt;
-{
-	u_int32_t A = ctxt->md5_sta;
-	u_int32_t B = ctxt->md5_stb;
-	u_int32_t C = ctxt->md5_stc;
-	u_int32_t D = ctxt->md5_std;
-#if BYTE_ORDER == LITTLE_ENDIAN
-	u_int32_t *X = (u_int32_t *)b64;
-#endif	
-#if BYTE_ORDER == BIG_ENDIAN
-	/* 4 byte words */
-	/* what a brute force but fast! */
-	u_int8_t *y = (u_int8_t *)X;
-	y[ 0] = b64[ 3]; y[ 1] = b64[ 2]; y[ 2] = b64[ 1]; y[ 3] = b64[ 0];
-	y[ 4] = b64[ 7]; y[ 5] = b64[ 6]; y[ 6] = b64[ 5]; y[ 7] = b64[ 4];
-	y[ 8] = b64[11]; y[ 9] = b64[10]; y[10] = b64[ 9]; y[11] = b64[ 8];
-	y[12] = b64[15]; y[13] = b64[14]; y[14] = b64[13]; y[15] = b64[12];
-	y[16] = b64[19]; y[17] = b64[18]; y[18] = b64[17]; y[19] = b64[16];
-	y[20] = b64[23]; y[21] = b64[22]; y[22] = b64[21]; y[23] = b64[20];
-	y[24] = b64[27]; y[25] = b64[26]; y[26] = b64[25]; y[27] = b64[24];
-	y[28] = b64[31]; y[29] = b64[30]; y[30] = b64[29]; y[31] = b64[28];
-	y[32] = b64[35]; y[33] = b64[34]; y[34] = b64[33]; y[35] = b64[32];
-	y[36] = b64[39]; y[37] = b64[38]; y[38] = b64[37]; y[39] = b64[36];
-	y[40] = b64[43]; y[41] = b64[42]; y[42] = b64[41]; y[43] = b64[40];
-	y[44] = b64[47]; y[45] = b64[46]; y[46] = b64[45]; y[47] = b64[44];
-	y[48] = b64[51]; y[49] = b64[50]; y[50] = b64[49]; y[51] = b64[48];
-	y[52] = b64[55]; y[53] = b64[54]; y[54] = b64[53]; y[55] = b64[52];
-	y[56] = b64[59]; y[57] = b64[58]; y[58] = b64[57]; y[59] = b64[56];
-	y[60] = b64[63]; y[61] = b64[62]; y[62] = b64[61]; y[63] = b64[60];
-#endif
-
-	ROUND1(A, B, C, D,  0, Sa,  1); ROUND1(D, A, B, C,  1, Sb,  2);
-	ROUND1(C, D, A, B,  2, Sc,  3); ROUND1(B, C, D, A,  3, Sd,  4);
-	ROUND1(A, B, C, D,  4, Sa,  5); ROUND1(D, A, B, C,  5, Sb,  6);
-	ROUND1(C, D, A, B,  6, Sc,  7); ROUND1(B, C, D, A,  7, Sd,  8);
-	ROUND1(A, B, C, D,  8, Sa,  9); ROUND1(D, A, B, C,  9, Sb, 10);
-	ROUND1(C, D, A, B, 10, Sc, 11); ROUND1(B, C, D, A, 11, Sd, 12);
-	ROUND1(A, B, C, D, 12, Sa, 13); ROUND1(D, A, B, C, 13, Sb, 14);
-	ROUND1(C, D, A, B, 14, Sc, 15); ROUND1(B, C, D, A, 15, Sd, 16);
-	
-	ROUND2(A, B, C, D,  1, Se, 17); ROUND2(D, A, B, C,  6, Sf, 18);
-	ROUND2(C, D, A, B, 11, Sg, 19); ROUND2(B, C, D, A,  0, Sh, 20);
-	ROUND2(A, B, C, D,  5, Se, 21); ROUND2(D, A, B, C, 10, Sf, 22);
-	ROUND2(C, D, A, B, 15, Sg, 23); ROUND2(B, C, D, A,  4, Sh, 24);
-	ROUND2(A, B, C, D,  9, Se, 25); ROUND2(D, A, B, C, 14, Sf, 26);
-	ROUND2(C, D, A, B,  3, Sg, 27); ROUND2(B, C, D, A,  8, Sh, 28);
-	ROUND2(A, B, C, D, 13, Se, 29); ROUND2(D, A, B, C,  2, Sf, 30);
-	ROUND2(C, D, A, B,  7, Sg, 31); ROUND2(B, C, D, A, 12, Sh, 32);
-
-	ROUND3(A, B, C, D,  5, Si, 33); ROUND3(D, A, B, C,  8, Sj, 34);
-	ROUND3(C, D, A, B, 11, Sk, 35); ROUND3(B, C, D, A, 14, Sl, 36);
-	ROUND3(A, B, C, D,  1, Si, 37); ROUND3(D, A, B, C,  4, Sj, 38);
-	ROUND3(C, D, A, B,  7, Sk, 39); ROUND3(B, C, D, A, 10, Sl, 40);
-	ROUND3(A, B, C, D, 13, Si, 41); ROUND3(D, A, B, C,  0, Sj, 42);
-	ROUND3(C, D, A, B,  3, Sk, 43); ROUND3(B, C, D, A,  6, Sl, 44);
-	ROUND3(A, B, C, D,  9, Si, 45); ROUND3(D, A, B, C, 12, Sj, 46);
-	ROUND3(C, D, A, B, 15, Sk, 47); ROUND3(B, C, D, A,  2, Sl, 48);
-	
-	ROUND4(A, B, C, D,  0, Sm, 49); ROUND4(D, A, B, C,  7, Sn, 50);	
-	ROUND4(C, D, A, B, 14, So, 51); ROUND4(B, C, D, A,  5, Sp, 52);	
-	ROUND4(A, B, C, D, 12, Sm, 53); ROUND4(D, A, B, C,  3, Sn, 54);	
-	ROUND4(C, D, A, B, 10, So, 55); ROUND4(B, C, D, A,  1, Sp, 56);	
-	ROUND4(A, B, C, D,  8, Sm, 57); ROUND4(D, A, B, C, 15, Sn, 58);	
-	ROUND4(C, D, A, B,  6, So, 59); ROUND4(B, C, D, A, 13, Sp, 60);	
-	ROUND4(A, B, C, D,  4, Sm, 61); ROUND4(D, A, B, C, 11, Sn, 62);	
-	ROUND4(C, D, A, B,  2, So, 63); ROUND4(B, C, D, A,  9, Sp, 64);
-
-	ctxt->md5_sta += A;
-	ctxt->md5_stb += B;
-	ctxt->md5_stc += C;
-	ctxt->md5_std += D;
-}
diff --git a/bsd/crypto/md5.h b/bsd/crypto/md5.h
deleted file mode 100644
index 8a99300b8..000000000
--- a/bsd/crypto/md5.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*	$FreeBSD: src/sys/crypto/md5.h,v 1.1.2.1 2000/07/15 07:14:18 kris Exp $	*/
-/*	$KAME: md5.h,v 1.4 2000/03/27 04:36:22 sumikawa Exp $	*/
-
-/*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef _NETINET6_MD5_H_
-#define _NETINET6_MD5_H_
-
-#define MD5_BUFLEN	64
-
-typedef struct {
-	union {
-		u_int32_t	md5_state32[4];
-		u_int8_t	md5_state8[16];
-	} md5_st;
-
-#define md5_sta		md5_st.md5_state32[0]
-#define md5_stb		md5_st.md5_state32[1]
-#define md5_stc		md5_st.md5_state32[2]
-#define md5_std		md5_st.md5_state32[3]
-#define md5_st8		md5_st.md5_state8
-
-	union {
-		u_int64_t	md5_count64;
-		u_int8_t	md5_count8[8];
-	} md5_count;
-#define md5_n	md5_count.md5_count64
-#define md5_n8	md5_count.md5_count8
-
-	u_int	md5_i;
-	u_int8_t	md5_buf[MD5_BUFLEN];
-} md5_ctxt;
-
-extern void md5_init(md5_ctxt *);
-extern void md5_loop(md5_ctxt *, u_int8_t *, u_int);
-extern void md5_pad(md5_ctxt *);
-extern void md5_result(u_int8_t *, md5_ctxt *);
-
-/* compatibility */
-#define MD5_CTX		md5_ctxt
-#define MD5Init(x)	md5_init((x))
-#define MD5Update(x, y, z)	md5_loop((x), (y), (z))
-#define MD5Final(x, y) \
-do {				\
-	md5_pad((y));		\
-	md5_result((x), (y));	\
-} while (0)
-
-#endif /* ! _NETINET6_MD5_H_*/
diff --git a/bsd/crypto/rc4/Makefile b/bsd/crypto/rc4/Makefile
index 23432a57e..7efd5ff1e 100644
--- a/bsd/crypto/rc4/Makefile
+++ b/bsd/crypto/rc4/Makefile
@@ -13,12 +13,16 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 PRIVATE_DATAFILES = \
 	rc4.h
 
diff --git a/bsd/crypto/sha1.c b/bsd/crypto/sha1.c
deleted file mode 100644
index cf6bbe72a..000000000
--- a/bsd/crypto/sha1.c
+++ /dev/null
@@ -1,274 +0,0 @@
-/*	$FreeBSD: src/sys/crypto/sha1.c,v 1.2.2.4 2001/07/03 11:01:27 ume Exp $	*/
-/*	$KAME: sha1.c,v 1.5 2000/11/08 06:13:08 itojun Exp $	*/
-
-/*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-/*
- * FIPS pub 180-1: Secure Hash Algorithm (SHA-1)
- * based on: http://csrc.nist.gov/fips/fip180-1.txt
- * implemented by Jun-ichiro itojun Itoh <itojun@itojun.org>
- */
-
-#include <sys/types.h>
-#include <sys/cdefs.h>
-#include <sys/time.h>
-#include <sys/systm.h>
-
-#include <crypto/sha1.h>
-
-/* sanity check */
-#if BYTE_ORDER != BIG_ENDIAN
-# if BYTE_ORDER != LITTLE_ENDIAN
-#  define unsupported 1
-# endif
-#endif
-
-#ifndef unsupported
-
-/* constant table */
-static u_int32_t _K[] = { 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 };
-#define	K(t)	_K[(t) / 20]
-
-#define	F0(b, c, d)	(((b) & (c)) | ((~(b)) & (d)))
-#define	F1(b, c, d)	(((b) ^ (c)) ^ (d))
-#define	F2(b, c, d)	(((b) & (c)) | ((b) & (d)) | ((c) & (d)))
-#define	F3(b, c, d)	(((b) ^ (c)) ^ (d))
-
-#define	S(n, x)		(((x) << (n)) | ((x) >> (32 - n)))
-
-#define	H(n)	(ctxt->h.b32[(n)])
-#define	COUNT	(ctxt->count)
-#define	BCOUNT	(ctxt->c.b64[0] / 8)
-#define	W(n)	(ctxt->m.b32[(n)])
-
-#define	PUTBYTE(x)	{ \
-	ctxt->m.b8[(COUNT % 64)] = (x);		\
-	COUNT++;				\
-	COUNT %= 64;				\
-	ctxt->c.b64[0] += 8;			\
-	if (COUNT % 64 == 0)			\
-		sha1_step(ctxt);		\
-     }
-
-#define	PUTPAD(x)	{ \
-	ctxt->m.b8[(COUNT % 64)] = (x);		\
-	COUNT++;				\
-	COUNT %= 64;				\
-	if (COUNT % 64 == 0)			\
-		sha1_step(ctxt);		\
-     }
-
-static void sha1_step(struct sha1_ctxt *);
-
-static void
-sha1_step(ctxt)
-	struct sha1_ctxt *ctxt;
-{
-	u_int32_t	a, b, c, d, e;
-	size_t t, s;
-	u_int32_t	tmp;
-
-#if BYTE_ORDER == LITTLE_ENDIAN
-	struct sha1_ctxt tctxt;
-	bcopy(&ctxt->m.b8[0], &tctxt.m.b8[0], 64);
-	ctxt->m.b8[0] = tctxt.m.b8[3]; ctxt->m.b8[1] = tctxt.m.b8[2];
-	ctxt->m.b8[2] = tctxt.m.b8[1]; ctxt->m.b8[3] = tctxt.m.b8[0];
-	ctxt->m.b8[4] = tctxt.m.b8[7]; ctxt->m.b8[5] = tctxt.m.b8[6];
-	ctxt->m.b8[6] = tctxt.m.b8[5]; ctxt->m.b8[7] = tctxt.m.b8[4];
-	ctxt->m.b8[8] = tctxt.m.b8[11]; ctxt->m.b8[9] = tctxt.m.b8[10];
-	ctxt->m.b8[10] = tctxt.m.b8[9]; ctxt->m.b8[11] = tctxt.m.b8[8];
-	ctxt->m.b8[12] = tctxt.m.b8[15]; ctxt->m.b8[13] = tctxt.m.b8[14];
-	ctxt->m.b8[14] = tctxt.m.b8[13]; ctxt->m.b8[15] = tctxt.m.b8[12];
-	ctxt->m.b8[16] = tctxt.m.b8[19]; ctxt->m.b8[17] = tctxt.m.b8[18];
-	ctxt->m.b8[18] = tctxt.m.b8[17]; ctxt->m.b8[19] = tctxt.m.b8[16];
-	ctxt->m.b8[20] = tctxt.m.b8[23]; ctxt->m.b8[21] = tctxt.m.b8[22];
-	ctxt->m.b8[22] = tctxt.m.b8[21]; ctxt->m.b8[23] = tctxt.m.b8[20];
-	ctxt->m.b8[24] = tctxt.m.b8[27]; ctxt->m.b8[25] = tctxt.m.b8[26];
-	ctxt->m.b8[26] = tctxt.m.b8[25]; ctxt->m.b8[27] = tctxt.m.b8[24];
-	ctxt->m.b8[28] = tctxt.m.b8[31]; ctxt->m.b8[29] = tctxt.m.b8[30];
-	ctxt->m.b8[30] = tctxt.m.b8[29]; ctxt->m.b8[31] = tctxt.m.b8[28];
-	ctxt->m.b8[32] = tctxt.m.b8[35]; ctxt->m.b8[33] = tctxt.m.b8[34];
-	ctxt->m.b8[34] = tctxt.m.b8[33]; ctxt->m.b8[35] = tctxt.m.b8[32];
-	ctxt->m.b8[36] = tctxt.m.b8[39]; ctxt->m.b8[37] = tctxt.m.b8[38];
-	ctxt->m.b8[38] = tctxt.m.b8[37]; ctxt->m.b8[39] = tctxt.m.b8[36];
-	ctxt->m.b8[40] = tctxt.m.b8[43]; ctxt->m.b8[41] = tctxt.m.b8[42];
-	ctxt->m.b8[42] = tctxt.m.b8[41]; ctxt->m.b8[43] = tctxt.m.b8[40];
-	ctxt->m.b8[44] = tctxt.m.b8[47]; ctxt->m.b8[45] = tctxt.m.b8[46];
-	ctxt->m.b8[46] = tctxt.m.b8[45]; ctxt->m.b8[47] = tctxt.m.b8[44];
-	ctxt->m.b8[48] = tctxt.m.b8[51]; ctxt->m.b8[49] = tctxt.m.b8[50];
-	ctxt->m.b8[50] = tctxt.m.b8[49]; ctxt->m.b8[51] = tctxt.m.b8[48];
-	ctxt->m.b8[52] = tctxt.m.b8[55]; ctxt->m.b8[53] = tctxt.m.b8[54];
-	ctxt->m.b8[54] = tctxt.m.b8[53]; ctxt->m.b8[55] = tctxt.m.b8[52];
-	ctxt->m.b8[56] = tctxt.m.b8[59]; ctxt->m.b8[57] = tctxt.m.b8[58];
-	ctxt->m.b8[58] = tctxt.m.b8[57]; ctxt->m.b8[59] = tctxt.m.b8[56];
-	ctxt->m.b8[60] = tctxt.m.b8[63]; ctxt->m.b8[61] = tctxt.m.b8[62];
-	ctxt->m.b8[62] = tctxt.m.b8[61]; ctxt->m.b8[63] = tctxt.m.b8[60];
-#endif
-
-	a = H(0); b = H(1); c = H(2); d = H(3); e = H(4);
-
-	for (t = 0; t < 20; t++) {
-		s = t & 0x0f;
-		if (t >= 16) {
-			W(s) = S(1, W((s+13) & 0x0f) ^ W((s+8) & 0x0f) ^ W((s+2) & 0x0f) ^ W(s));
-		}
-		tmp = S(5, a) + F0(b, c, d) + e + W(s) + K(t);
-		e = d; d = c; c = S(30, b); b = a; a = tmp;
-	}
-	for (t = 20; t < 40; t++) {
-		s = t & 0x0f;
-		W(s) = S(1, W((s+13) & 0x0f) ^ W((s+8) & 0x0f) ^ W((s+2) & 0x0f) ^ W(s));
-		tmp = S(5, a) + F1(b, c, d) + e + W(s) + K(t);
-		e = d; d = c; c = S(30, b); b = a; a = tmp;
-	}
-	for (t = 40; t < 60; t++) {
-		s = t & 0x0f;
-		W(s) = S(1, W((s+13) & 0x0f) ^ W((s+8) & 0x0f) ^ W((s+2) & 0x0f) ^ W(s));
-		tmp = S(5, a) + F2(b, c, d) + e + W(s) + K(t);
-		e = d; d = c; c = S(30, b); b = a; a = tmp;
-	}
-	for (t = 60; t < 80; t++) {
-		s = t & 0x0f;
-		W(s) = S(1, W((s+13) & 0x0f) ^ W((s+8) & 0x0f) ^ W((s+2) & 0x0f) ^ W(s));
-		tmp = S(5, a) + F3(b, c, d) + e + W(s) + K(t);
-		e = d; d = c; c = S(30, b); b = a; a = tmp;
-	}
-
-	H(0) = H(0) + a;
-	H(1) = H(1) + b;
-	H(2) = H(2) + c;
-	H(3) = H(3) + d;
-	H(4) = H(4) + e;
-
-	bzero(&ctxt->m.b8[0], 64);
-}
-
-/*------------------------------------------------------------*/
-
-void
-sha1_init(ctxt)
-	struct sha1_ctxt *ctxt;
-{
-	bzero(ctxt, sizeof(struct sha1_ctxt));
-	H(0) = 0x67452301;
-	H(1) = 0xefcdab89;
-	H(2) = 0x98badcfe;
-	H(3) = 0x10325476;
-	H(4) = 0xc3d2e1f0;
-}
-
-void
-sha1_pad(ctxt)
-	struct sha1_ctxt *ctxt;
-{
-	size_t padlen;		/*pad length in bytes*/
-	size_t padstart;
-
-	PUTPAD(0x80);
-
-	padstart = COUNT % 64;
-	padlen = 64 - padstart;
-	if (padlen < 8) {
-		bzero(&ctxt->m.b8[padstart], padlen);
-		COUNT += padlen;
-		COUNT %= 64;
-		sha1_step(ctxt);
-		padstart = COUNT % 64;	/* should be 0 */
-		padlen = 64 - padstart;	/* should be 64 */
-	}
-	bzero(&ctxt->m.b8[padstart], padlen - 8);
-	COUNT += (padlen - 8);
-	COUNT %= 64;
-#if BYTE_ORDER == BIG_ENDIAN
-	PUTPAD(ctxt->c.b8[0]); PUTPAD(ctxt->c.b8[1]);
-	PUTPAD(ctxt->c.b8[2]); PUTPAD(ctxt->c.b8[3]);
-	PUTPAD(ctxt->c.b8[4]); PUTPAD(ctxt->c.b8[5]);
-	PUTPAD(ctxt->c.b8[6]); PUTPAD(ctxt->c.b8[7]);
-#else
-	PUTPAD(ctxt->c.b8[7]); PUTPAD(ctxt->c.b8[6]);
-	PUTPAD(ctxt->c.b8[5]); PUTPAD(ctxt->c.b8[4]);
-	PUTPAD(ctxt->c.b8[3]); PUTPAD(ctxt->c.b8[2]);
-	PUTPAD(ctxt->c.b8[1]); PUTPAD(ctxt->c.b8[0]);
-#endif
-}
-
-void
-sha1_loop(ctxt, input, len)
-	struct sha1_ctxt *ctxt;
-	const u_int8_t *input;
-	size_t len;
-{
-	size_t gaplen;
-	size_t gapstart;
-	size_t off;
-	size_t copysiz;
-
-	off = 0;
-
-	while (off < len) {
-		gapstart = COUNT % 64;
-		gaplen = 64 - gapstart;
-
-		copysiz = (gaplen < len - off) ? gaplen : len - off;
-		bcopy(&input[off], &ctxt->m.b8[gapstart], copysiz);
-		COUNT += copysiz;
-		COUNT %= 64;
-		ctxt->c.b64[0] += copysiz * 8;
-		if (COUNT % 64 == 0)
-			sha1_step(ctxt);
-		off += copysiz;
-	}
-}
-
-void
-sha1_result(ctxt, digest0)
-	struct sha1_ctxt *ctxt;
-	caddr_t digest0;
-{
-	u_int8_t *digest;
-
-	digest = (u_int8_t *)digest0;
-	sha1_pad(ctxt);
-#if BYTE_ORDER == BIG_ENDIAN
-	bcopy(&ctxt->h.b8[0], digest, 20);
-#else
-	digest[0] = ctxt->h.b8[3]; digest[1] = ctxt->h.b8[2];
-	digest[2] = ctxt->h.b8[1]; digest[3] = ctxt->h.b8[0];
-	digest[4] = ctxt->h.b8[7]; digest[5] = ctxt->h.b8[6];
-	digest[6] = ctxt->h.b8[5]; digest[7] = ctxt->h.b8[4];
-	digest[8] = ctxt->h.b8[11]; digest[9] = ctxt->h.b8[10];
-	digest[10] = ctxt->h.b8[9]; digest[11] = ctxt->h.b8[8];
-	digest[12] = ctxt->h.b8[15]; digest[13] = ctxt->h.b8[14];
-	digest[14] = ctxt->h.b8[13]; digest[15] = ctxt->h.b8[12];
-	digest[16] = ctxt->h.b8[19]; digest[17] = ctxt->h.b8[18];
-	digest[18] = ctxt->h.b8[17]; digest[19] = ctxt->h.b8[16];
-#endif
-}
-
-#endif /*unsupported*/
diff --git a/bsd/crypto/sha1.h b/bsd/crypto/sha1.h
index f5dbac6eb..bfc874833 100644
--- a/bsd/crypto/sha1.h
+++ b/bsd/crypto/sha1.h
@@ -1,72 +1,33 @@
-/*	$FreeBSD: src/sys/crypto/sha1.h,v 1.3.2.3 2000/10/12 18:59:31 archie Exp $	*/
-/*	$KAME: sha1.h,v 1.5 2000/03/27 04:36:23 sumikawa Exp $	*/
-
 /*
- * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 /*
- * FIPS pub 180-1: Secure Hash Algorithm (SHA-1)
- * based on: http://csrc.nist.gov/fips/fip180-1.txt
- * implemented by Jun-ichiro itojun Itoh <itojun@itojun.org>
+ * This header file is kept for legacy reasons and may be removed in
+ * future; the supported interface resides in <libkern/crypto/sha1.h>.
  */
-
-#ifndef _NETINET6_SHA1_H_
-#define _NETINET6_SHA1_H_
-
-struct sha1_ctxt {
-	union {
-		u_int8_t	b8[20];
-		u_int32_t	b32[5];
-	} h;
-	union {
-		u_int8_t	b8[8];
-		u_int64_t	b64[1];
-	} c;
-	union {
-		u_int8_t	b8[64];
-		u_int32_t	b32[16];
-	} m;
-	u_int8_t	count;
-};
-
-#ifdef KERNEL
-extern void sha1_init(struct sha1_ctxt *);
-extern void sha1_pad(struct sha1_ctxt *);
-extern void sha1_loop(struct sha1_ctxt *, const u_int8_t *, size_t);
-extern void sha1_result(struct sha1_ctxt *, caddr_t);
-
-/* compatibilty with other SHA1 source codes */
-typedef struct sha1_ctxt SHA1_CTX;
-#define SHA1Init(x)		sha1_init((x))
-#define SHA1Update(x, y, z)	sha1_loop((x), (y), (z))
-#define SHA1Final(x, y)		sha1_result((y), (x))
-#endif /* KERNEL */
-
-#define	SHA1_RESULTLEN	(160/8)
-
-#endif /*_NETINET6_SHA1_H_*/
+#include <libkern/crypto/sha1.h>
diff --git a/bsd/crypto/sha2/Makefile b/bsd/crypto/sha2/Makefile
index 72820c951..a89ecdaff 100644
--- a/bsd/crypto/sha2/Makefile
+++ b/bsd/crypto/sha2/Makefile
@@ -13,12 +13,16 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 PRIVATE_DATAFILES = \
 	sha2.h
 
diff --git a/bsd/crypto/sha2/sha2.c b/bsd/crypto/sha2/sha2.c
index 53b5b201c..c306068dc 100644
--- a/bsd/crypto/sha2/sha2.c
+++ b/bsd/crypto/sha2/sha2.c
@@ -64,7 +64,7 @@
  */
 
 #ifndef assert(x)
-#define assert(x)
+#define assert(x) do {} while(0)
 #endif
 
 /*** SHA-256/384/512 Machine Architecture Definitions *****************/
@@ -208,7 +208,7 @@ void SHA512_Transform(SHA512_CTX*, const sha2_word64*);
 
 /*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************/
 /* Hash constant words K for SHA-256: */
-const static sha2_word32 K256[64] = {
+static const sha2_word32 K256[64] = {
 	0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL,
 	0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL,
 	0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL,
@@ -228,7 +228,7 @@ const static sha2_word32 K256[64] = {
 };
 
 /* Initial hash value H for SHA-256: */
-const static sha2_word32 sha256_initial_hash_value[8] = {
+static const sha2_word32 sha256_initial_hash_value[8] = {
 	0x6a09e667UL,
 	0xbb67ae85UL,
 	0x3c6ef372UL,
@@ -240,7 +240,7 @@ const static sha2_word32 sha256_initial_hash_value[8] = {
 };
 
 /* Hash constant words K for SHA-384 and SHA-512: */
-const static sha2_word64 K512[80] = {
+static const sha2_word64 K512[80] = {
 	0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL,
 	0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL,
 	0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
@@ -284,7 +284,7 @@ const static sha2_word64 K512[80] = {
 };
 
 /* Initial hash value H for SHA-384 */
-const static sha2_word64 sha384_initial_hash_value[8] = {
+static const sha2_word64 sha384_initial_hash_value[8] = {
 	0xcbbb9d5dc1059ed8ULL,
 	0x629a292a367cd507ULL,
 	0x9159015a3070dd17ULL,
@@ -296,7 +296,7 @@ const static sha2_word64 sha384_initial_hash_value[8] = {
 };
 
 /* Initial hash value H for SHA-512 */
-const static sha2_word64 sha512_initial_hash_value[8] = {
+static const sha2_word64 sha512_initial_hash_value[8] = {
 	0x6a09e667f3bcc908ULL,
 	0xbb67ae8584caa73bULL,
 	0x3c6ef372fe94f82bULL,
diff --git a/bsd/dev/busvar.h b/bsd/dev/busvar.h
index e6508ebed..2674eb6d7 100644
--- a/bsd/dev/busvar.h
+++ b/bsd/dev/busvar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1987 Next, Inc.
@@ -35,7 +41,7 @@
 /* pseudo device initialization routine support */
 struct pseudo_init {
 	int	ps_count;
-	int	(*ps_func)();
+	int	(*ps_func)(int count);
 };
 extern struct pseudo_init pseudo_inits[];
 
diff --git a/bsd/dev/chud/chud_bsd_callback.c b/bsd/dev/chud/chud_bsd_callback.c
index bc2f00422..bfb0aeb61 100644
--- a/bsd/dev/chud/chud_bsd_callback.c
+++ b/bsd/dev/chud/chud_bsd_callback.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -26,7 +32,7 @@
 
 #include <sys/syscall.h>
 #include <sys/types.h> /* u_int */
-#include <sys/proc.h> /* struct proc */
+#include <sys/proc.h> /* proc_t */
 #include <sys/systm.h> /* struct sysent */
 #include <sys/sysproto.h>
 
@@ -84,13 +90,9 @@ static chudxnu_syscall_callback_func_t syscall_callback_fn = NULL;
 kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func);
 kern_return_t chudxnu_syscall_callback_cancel(void);
 
-int chud(p, uap, retval)
-     struct proc *p;
-     struct chud_args *uap;
-     register_t *retval;
+int
+chud(__unused proc_t p, struct chud_args *uap, register_t *retval)
 {
-#pragma unused (p)
-
     chudxnu_syscall_callback_func_t fn = syscall_callback_fn;
     
 	if(!fn) {
@@ -102,15 +104,15 @@ int chud(p, uap, retval)
 	return 0;
 }
 
-__private_extern__
-kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
+__private_extern__ kern_return_t 
+chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
 {
 	syscall_callback_fn = func;
     return KERN_SUCCESS;
 }
 
-__private_extern__
-kern_return_t chudxnu_syscall_callback_cancel(void)
+__private_extern__ kern_return_t
+chudxnu_syscall_callback_cancel(void)
 {
 	syscall_callback_fn = NULL;
     return KERN_SUCCESS;
diff --git a/bsd/dev/chud/chud_process.c b/bsd/dev/chud/chud_process.c
index c409c6275..cc82b9890 100644
--- a/bsd/dev/chud/chud_process.c
+++ b/bsd/dev/chud/chud_process.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/systm.h>
@@ -33,25 +39,28 @@ int	chudxnu_current_pid(void);
 __private_extern__ int
 chudxnu_pid_for_task(task_t task)
 {
-    struct proc *p;
+     proc_t p;
+    int pid = -1;
 
     if(task!=TASK_NULL) {
-        p = (struct proc *)(get_bsdtask_info(task));
+        p = (proc_t)(get_bsdtask_info(task));
         if(p) {
-            return (p->p_pid);
+            return (proc_pid(p));
         }
     }
-    return -1;
+    return pid;
 }
 
 __private_extern__ task_t
 chudxnu_task_for_pid(int pid)
 {
-    struct proc *p = pfind(pid);
+	task_t t = TASK_NULL;
+     proc_t p = proc_find(pid);
     if(p) {
-        return p->task;
+        t =  p->task;
+	proc_rele(p);
     }
-    return TASK_NULL;
+    return (t);
 }
 
 __private_extern__ int
@@ -63,12 +72,13 @@ chudxnu_current_pid(void)
 
     if(t != TASK_NULL) {
         pid = chudxnu_pid_for_task(t);
-    } else {
+    }
+    if(-1 == pid) {
         // no task, so try looking in the uthread and/or proc
-        pid = current_proc()->p_pid;
+        pid = proc_pid(current_proc());
 
-        if(ut && ut->uu_proc) {
-            pid = ut->uu_proc->p_pid;
+        if(-1 == pid && ut && ut->uu_proc) {
+            pid = proc_pid(ut->uu_proc);
         }
     }
     
diff --git a/bsd/dev/dtrace/blist.c b/bsd/dev/dtrace/blist.c
new file mode 100644
index 000000000..cb6177675
--- /dev/null
+++ b/bsd/dev/dtrace/blist.c
@@ -0,0 +1,926 @@
+/*
+ * BLIST.C -	Bitmap allocator/deallocator, using a radix tree with hinting
+ *
+ *	(c)Copyright 1998, Matthew Dillon.  Terms for use and redistribution
+ *	are covered by the BSD Copyright as found in /usr/src/COPYRIGHT.
+ *
+ *	This module implements a general bitmap allocator/deallocator.  The
+ *	allocator eats around 2 bits per 'block'.  The module does not 
+ *	try to interpret the meaning of a 'block' other then to return 
+ *	SWAPBLK_NONE on an allocation failure.
+ *
+ *	A radix tree is used to maintain the bitmap.  Two radix constants are
+ *	involved:  One for the bitmaps contained in the leaf nodes (typically
+ *	32), and one for the meta nodes (typically 16).  Both meta and leaf
+ *	nodes have a hint field.  This field gives us a hint as to the largest
+ *	free contiguous range of blocks under the node.  It may contain a
+ *	value that is too high, but will never contain a value that is too 
+ *	low.  When the radix tree is searched, allocation failures in subtrees
+ *	update the hint. 
+ *
+ *	The radix tree also implements two collapsed states for meta nodes:
+ *	the ALL-ALLOCATED state and the ALL-FREE state.  If a meta node is
+ *	in either of these two states, all information contained underneath
+ *	the node is considered stale.  These states are used to optimize
+ *	allocation and freeing operations.
+ *
+ * 	The hinting greatly increases code efficiency for allocations while
+ *	the general radix structure optimizes both allocations and frees.  The
+ *	radix tree should be able to operate well no matter how much 
+ *	fragmentation there is and no matter how large a bitmap is used.
+ *
+ *	Unlike the rlist code, the blist code wires all necessary memory at
+ *	creation time.  Neither allocations nor frees require interaction with
+ *	the memory subsystem.  In contrast, the rlist code may allocate memory 
+ *	on an rlist_free() call.  The non-blocking features of the blist code
+ *	are used to great advantage in the swap code (vm/nswap_pager.c).  The
+ *	rlist code uses a little less overall memory then the blist code (but
+ *	due to swap interleaving not all that much less), but the blist code 
+ *	scales much, much better.
+ *
+ *	LAYOUT: The radix tree is layed out recursively using a
+ *	linear array.  Each meta node is immediately followed (layed out
+ *	sequentially in memory) by BLIST_META_RADIX lower level nodes.  This
+ *	is a recursive structure but one that can be easily scanned through
+ *	a very simple 'skip' calculation.  In order to support large radixes, 
+ *	portions of the tree may reside outside our memory allocation.  We 
+ *	handle this with an early-termination optimization (when bighint is 
+ *	set to -1) on the scan.  The memory allocation is only large enough 
+ *	to cover the number of blocks requested at creation time even if it
+ *	must be encompassed in larger root-node radix.
+ *
+ *	NOTE: the allocator cannot currently allocate more then 
+ *	BLIST_BMAP_RADIX blocks per call.  It will panic with 'allocation too 
+ *	large' if you try.  This is an area that could use improvement.  The 
+ *	radix is large enough that this restriction does not effect the swap 
+ *	system, though.  Currently only the allocation code is effected by
+ *	this algorithmic unfeature.  The freeing code can handle arbitrary
+ *	ranges.
+ *
+ *	This code can be compiled stand-alone for debugging.
+ *
+ * $FreeBSD: src/sys/kern/subr_blist.c,v 1.5.2.1 2000/03/17 10:47:29 ps Exp $
+ */
+
+#if !defined(__APPLE__)
+#ifdef _KERNEL
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/kernel.h>
+#include <sys/blist.h>
+#include <sys/malloc.h>
+#include <vm/vm.h>
+#include <vm/vm_object.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
+#else
+
+#ifndef BLIST_NO_DEBUG
+#define BLIST_DEBUG
+#endif
+
+#define SWAPBLK_NONE ((daddr_t)-1)
+
+#include <sys/types.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdarg.h>
+
+#define malloc(a,b,c)	malloc(a)
+#define free(a,b)	free(a)
+
+typedef unsigned int u_daddr_t;
+
+#include <sys/blist.h>
+
+void panic(const char *ctl, ...);
+
+#endif
+#else /* is MacOS X */
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+typedef unsigned int u_daddr_t;
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/lock.h>
+#include <sys/kernel.h>
+/* #include <sys/blist.h> */
+#include "blist.h"
+#include <sys/malloc.h>
+
+#define SWAPBLK_NONE ((daddr_t)-1)
+#define malloc _MALLOC
+#define free _FREE
+#define M_SWAP M_TEMP
+
+#endif /* __APPLE__ */
+
+/*
+ * static support functions
+ */
+
+static daddr_t blst_leaf_alloc(blmeta_t *scan, daddr_t blk, int count);
+static daddr_t blst_meta_alloc(blmeta_t *scan, daddr_t blk, 
+				daddr_t count, daddr_t radix, int skip);
+static void blst_leaf_free(blmeta_t *scan, daddr_t relblk, int count);
+static void blst_meta_free(blmeta_t *scan, daddr_t freeBlk, daddr_t count, 
+					daddr_t radix, int skip, daddr_t blk);
+static void blst_copy(blmeta_t *scan, daddr_t blk, daddr_t radix, 
+				daddr_t skip, blist_t dest, daddr_t count);
+static daddr_t	blst_radix_init(blmeta_t *scan, daddr_t radix, 
+						int skip, daddr_t count);
+#ifndef _KERNEL
+static void	blst_radix_print(blmeta_t *scan, daddr_t blk, 
+					daddr_t radix, int skip, int tab);
+#endif
+
+#ifdef _KERNEL
+static MALLOC_DEFINE(M_SWAP, "SWAP", "Swap space");
+#endif
+
+/*
+ * blist_create() - create a blist capable of handling up to the specified
+ *		    number of blocks
+ *
+ *	blocks must be greater then 0
+ *
+ *	The smallest blist consists of a single leaf node capable of 
+ *	managing BLIST_BMAP_RADIX blocks.
+ */
+
+blist_t 
+blist_create(daddr_t blocks)
+{
+	blist_t bl;
+	int radix;
+	int skip = 0;
+
+	/*
+	 * Calculate radix and skip field used for scanning.
+	 */
+	radix = BLIST_BMAP_RADIX;
+
+	while (radix < blocks) {
+		radix <<= BLIST_META_RADIX_SHIFT;
+		skip = (skip + 1) << BLIST_META_RADIX_SHIFT;
+	}
+
+	bl = malloc(sizeof(struct blist), M_SWAP, M_WAITOK);
+
+	bzero(bl, sizeof(*bl));
+
+	bl->bl_blocks = blocks;
+	bl->bl_radix = radix;
+	bl->bl_skip = skip;
+	bl->bl_rootblks = 1 +
+	    blst_radix_init(NULL, bl->bl_radix, bl->bl_skip, blocks);
+	bl->bl_root = malloc(sizeof(blmeta_t) * bl->bl_rootblks, M_SWAP, M_WAITOK);
+
+#if defined(BLIST_DEBUG)
+	printf(
+		"BLIST representing %d blocks (%d MB of swap)"
+		", requiring %dK of ram\n",
+		bl->bl_blocks,
+		bl->bl_blocks * 4 / 1024,
+		(bl->bl_rootblks * sizeof(blmeta_t) + 1023) / 1024
+	);
+	printf("BLIST raw radix tree contains %d records\n", bl->bl_rootblks);
+#endif
+	blst_radix_init(bl->bl_root, bl->bl_radix, bl->bl_skip, blocks);
+
+	return(bl);
+}
+
+void 
+blist_destroy(blist_t bl)
+{
+	free(bl->bl_root, M_SWAP);
+	free(bl, M_SWAP);
+}
+
+/*
+ * blist_alloc() - reserve space in the block bitmap.  Return the base
+ *		     of a contiguous region or SWAPBLK_NONE if space could
+ *		     not be allocated.
+ */
+
+daddr_t 
+blist_alloc(blist_t bl, daddr_t count)
+{
+	daddr_t blk = SWAPBLK_NONE;
+
+	if (bl) {
+		if (bl->bl_radix == BLIST_BMAP_RADIX)
+			blk = blst_leaf_alloc(bl->bl_root, 0, count);
+		else
+			blk = blst_meta_alloc(bl->bl_root, 0, count,
+					      bl->bl_radix, bl->bl_skip);
+		if (blk != SWAPBLK_NONE)
+			bl->bl_free -= count;
+	}
+	return(blk);
+}
+
+/*
+ * blist_free() -	free up space in the block bitmap.  Return the base
+ *		     	of a contiguous region.  Panic if an inconsistancy is
+ *			found.
+ */
+
+void 
+blist_free(blist_t bl, daddr_t blkno, daddr_t count)
+{
+	if (bl) {
+		if (bl->bl_radix == BLIST_BMAP_RADIX)
+			blst_leaf_free(bl->bl_root, blkno, count);
+		else
+			blst_meta_free(bl->bl_root, blkno, count,
+				       bl->bl_radix, bl->bl_skip, 0);
+		bl->bl_free += count;
+	}
+}
+
+/*
+ * blist_resize() -	resize an existing radix tree to handle the
+ *			specified number of blocks.  This will reallocate
+ *			the tree and transfer the previous bitmap to the new
+ *			one.  When extending the tree you can specify whether
+ *			the new blocks are to left allocated or freed.
+ */
+
+void
+blist_resize(blist_t *pbl, daddr_t count, int freenew)
+{
+    blist_t newbl = blist_create(count);
+    blist_t save = *pbl;
+
+    *pbl = newbl;
+    if (count > save->bl_blocks)
+	    count = save->bl_blocks;
+    blst_copy(save->bl_root, 0, save->bl_radix, save->bl_skip, newbl, count);
+
+    /*
+     * If resizing upwards, should we free the new space or not?
+     */
+    if (freenew && count < newbl->bl_blocks)
+	    blist_free(newbl, count, newbl->bl_blocks - count);
+    blist_destroy(save);
+}
+
+#ifdef BLIST_DEBUG
+
+/*
+ * blist_print()    - dump radix tree
+ */
+
+void
+blist_print(blist_t bl)
+{
+	printf("BLIST {\n");
+	blst_radix_print(bl->bl_root, 0, bl->bl_radix, bl->bl_skip, 4);
+	printf("}\n");
+}
+
+#endif
+
+/************************************************************************
+ *			  ALLOCATION SUPPORT FUNCTIONS			*
+ ************************************************************************
+ *
+ *	These support functions do all the actual work.  They may seem 
+ *	rather longish, but that's because I've commented them up.  The
+ *	actual code is straight forward.
+ *
+ */
+
+/*
+ * blist_leaf_alloc() -	allocate at a leaf in the radix tree (a bitmap).
+ *
+ *	This is the core of the allocator and is optimized for the 1 block
+ *	and the BLIST_BMAP_RADIX block allocation cases.  Other cases are
+ *	somewhat slower.  The 1 block allocation case is log2 and extremely
+ *	quick.
+ */
+
+static daddr_t
+blst_leaf_alloc(blmeta_t *scan, daddr_t blk, int count)
+{
+	u_daddr_t orig = scan->u.bmu_bitmap;
+
+	if (orig == 0) {
+		/*
+		 * Optimize bitmap all-allocated case.  Also, count = 1
+		 * case assumes at least 1 bit is free in the bitmap, so
+		 * we have to take care of this case here.
+		 */
+		scan->bm_bighint = 0;
+		return(SWAPBLK_NONE);
+	}
+	if (count == 1) {
+		/*
+		 * Optimized code to allocate one bit out of the bitmap
+		 */
+		u_daddr_t mask;
+		int j = BLIST_BMAP_RADIX/2;
+		int r = 0;
+
+		mask = (u_daddr_t)-1 >> (BLIST_BMAP_RADIX/2);
+
+		while (j) {
+			if ((orig & mask) == 0) {
+			    r += j;
+			    orig >>= j;
+			}
+			j >>= 1;
+			mask >>= j;
+		}
+		scan->u.bmu_bitmap &= ~(1 << r);
+		return(blk + r);
+	}
+	if (count <= BLIST_BMAP_RADIX) {
+		/*
+		 * non-optimized code to allocate N bits out of the bitmap.
+		 * The more bits, the faster the code runs.  It will run
+		 * the slowest allocating 2 bits, but since there aren't any
+		 * memory ops in the core loop (or shouldn't be, anyway),
+		 * you probably won't notice the difference.
+		 */
+		int j;
+		int n = BLIST_BMAP_RADIX - count;
+		u_daddr_t mask;
+
+		mask = (u_daddr_t)-1 >> n;
+
+		for (j = 0; j <= n; ++j) {
+			if ((orig & mask) == mask) {
+				scan->u.bmu_bitmap &= ~mask;
+				return(blk + j);
+			}
+			mask = (mask << 1);
+		}
+	}
+	/*
+	 * We couldn't allocate count in this subtree, update bighint.
+	 */
+	scan->bm_bighint = count - 1;
+	return(SWAPBLK_NONE);
+}
+
+/*
+ * blist_meta_alloc() -	allocate at a meta in the radix tree.
+ *
+ *	Attempt to allocate at a meta node.  If we can't, we update
+ *	bighint and return a failure.  Updating bighint optimize future
+ *	calls that hit this node.  We have to check for our collapse cases
+ *	and we have a few optimizations strewn in as well.
+ */
+
+static daddr_t
+blst_meta_alloc(blmeta_t *scan, daddr_t blk, daddr_t count, daddr_t radix,
+		int skip)
+{
+	int i;
+	int next_skip = (skip >> BLIST_META_RADIX_SHIFT);
+
+	if (scan->u.bmu_avail == 0)  {
+		/*
+		 * ALL-ALLOCATED special case
+		 */
+		scan->bm_bighint = count;
+		return(SWAPBLK_NONE);
+	}
+
+	if (scan->u.bmu_avail == radix) {
+		radix >>= BLIST_META_RADIX_SHIFT;
+
+		/*
+		 * ALL-FREE special case, initialize uninitialize
+		 * sublevel.
+		 */
+		for (i = 1; i <= skip; i += next_skip) {
+			if (scan[i].bm_bighint == (daddr_t)-1)
+				break;
+			if (next_skip == 1) {
+				scan[i].u.bmu_bitmap = (u_daddr_t)-1;
+				scan[i].bm_bighint = BLIST_BMAP_RADIX;
+			} else {
+				scan[i].bm_bighint = radix;
+				scan[i].u.bmu_avail = radix;
+			}
+		}
+	} else {
+		radix >>= BLIST_META_RADIX_SHIFT;
+	}
+
+	for (i = 1; i <= skip; i += next_skip) {
+		if (count <= scan[i].bm_bighint) {
+			/*
+			 * count fits in object
+			 */
+			daddr_t r;
+			if (next_skip == 1)
+				r = blst_leaf_alloc(&scan[i], blk, count);
+			else
+				r = blst_meta_alloc(&scan[i], blk, count,
+						    radix, next_skip - 1);
+			if (r != SWAPBLK_NONE) {
+				scan->u.bmu_avail -= count;
+				if (scan->bm_bighint > scan->u.bmu_avail)
+					scan->bm_bighint = scan->u.bmu_avail;
+				return r;
+			}
+		} else if (scan[i].bm_bighint == (daddr_t)-1) {
+			/*
+			 * Terminator
+			 */
+			break;
+		} else if (count > radix) {
+			/*
+			 * count does not fit in object even if it were
+			 * complete free.
+			 */
+			panic("blist_meta_alloc: allocation too large");
+		}
+		blk += radix;
+	}
+
+	/*
+	 * We couldn't allocate count in this subtree, update bighint.
+	 */
+	if (scan->bm_bighint >= count)
+		scan->bm_bighint = count - 1;
+	return(SWAPBLK_NONE);
+}
+
+/*
+ * BLST_LEAF_FREE() -	free allocated block from leaf bitmap
+ *
+ */
+
+static void
+blst_leaf_free(blmeta_t *scan, daddr_t blk, int count)
+{
+	/*
+	 * free some data in this bitmap
+	 *
+	 * e.g.
+	 *	0000111111111110000
+	 *          \_________/\__/
+	 *		v        n
+	 */
+	int n = blk & (BLIST_BMAP_RADIX - 1);
+	u_daddr_t mask;
+
+	mask = ((u_daddr_t)-1 << n) &
+	    ((u_daddr_t)-1 >> (BLIST_BMAP_RADIX - count - n));
+
+	if (scan->u.bmu_bitmap & mask)
+		panic("blst_radix_free: freeing free block");
+	scan->u.bmu_bitmap |= mask;
+
+	/*
+	 * We could probably do a better job here.  We are required to make
+	 * bighint at least as large as the biggest contiguous block of 
+	 * data.  If we just shoehorn it, a little extra overhead will
+	 * be incured on the next allocation (but only that one typically).
+	 */
+	scan->bm_bighint = BLIST_BMAP_RADIX;
+}
+
+/*
+ * BLST_META_FREE() - free allocated blocks from radix tree meta info
+ *
+ *	This support routine frees a range of blocks from the bitmap.
+ *	The range must be entirely enclosed by this radix node.  If a
+ *	meta node, we break the range down recursively to free blocks
+ *	in subnodes (which means that this code can free an arbitrary
+ *	range whereas the allocation code cannot allocate an arbitrary
+ *	range).
+ */
+
+static void 
+blst_meta_free(blmeta_t *scan, daddr_t freeBlk, daddr_t count, daddr_t radix,
+	       int skip, daddr_t blk)
+{
+	int i;
+	int next_skip = (skip >> BLIST_META_RADIX_SHIFT);
+
+#if 0
+	printf("FREE (%x,%d) FROM (%x,%d)\n",
+	    freeBlk, count,
+	    blk, radix
+	);
+#endif
+
+	if (scan->u.bmu_avail == 0) {
+		/*
+		 * ALL-ALLOCATED special case, with possible
+		 * shortcut to ALL-FREE special case.
+		 */
+		scan->u.bmu_avail = count;
+		scan->bm_bighint = count;
+
+		if (count != radix)  {
+			for (i = 1; i <= skip; i += next_skip) {
+				if (scan[i].bm_bighint == (daddr_t)-1)
+					break;
+				scan[i].bm_bighint = 0;
+				if (next_skip == 1)
+					scan[i].u.bmu_bitmap = 0;
+				else
+					scan[i].u.bmu_avail = 0;
+			}
+			/* fall through */
+		}
+	} else {
+		scan->u.bmu_avail += count;
+		/* scan->bm_bighint = radix; */
+	}
+
+	/*
+	 * ALL-FREE special case.
+	 */
+
+	if (scan->u.bmu_avail == radix)
+		return;
+	if (scan->u.bmu_avail > radix)
+		panic("blst_meta_free: freeing already free blocks (%d) %d/%d", count, scan->u.bmu_avail, radix);
+
+	/*
+	 * Break the free down into its components
+	 */
+
+	radix >>= BLIST_META_RADIX_SHIFT;
+
+	i = (freeBlk - blk) / radix;
+	blk += i * radix;
+	i = i * next_skip + 1;
+
+	while (i <= skip && blk < freeBlk + count) {
+		daddr_t v;
+
+		v = blk + radix - freeBlk;
+		if (v > count)
+			v = count;
+
+		if (scan->bm_bighint == (daddr_t)-1)
+			panic("blst_meta_free: freeing unexpected range");
+
+		if (next_skip == 1)
+			blst_leaf_free(&scan[i], freeBlk, v);
+		else
+			blst_meta_free(&scan[i], freeBlk, v, radix,
+				       next_skip - 1, blk);
+		if (scan->bm_bighint < scan[i].bm_bighint)
+		    scan->bm_bighint = scan[i].bm_bighint;
+		count -= v;
+		freeBlk += v;
+		blk += radix;
+		i += next_skip;
+	}
+}
+
+/*
+ * BLIST_RADIX_COPY() - copy one radix tree to another
+ *
+ *	Locates free space in the source tree and frees it in the destination
+ *	tree.  The space may not already be free in the destination.
+ */
+
+static void blst_copy(blmeta_t *scan, daddr_t blk, daddr_t radix,
+		      daddr_t skip, blist_t dest, daddr_t count)
+{
+	int next_skip;
+	int i;
+
+	/*
+	 * Leaf node
+	 */
+
+	if (radix == BLIST_BMAP_RADIX) {
+		u_daddr_t v = scan->u.bmu_bitmap;
+
+		if (v == (u_daddr_t)-1) {
+			blist_free(dest, blk, count);
+		} else if (v != 0) {
+			int i;
+
+			for (i = 0; i < BLIST_BMAP_RADIX && i < count; ++i)
+				if (v & (1 << i))
+					blist_free(dest, blk + i, 1);
+		}
+		return;
+	}
+
+	/*
+	 * Meta node
+	 */
+
+	/*
+	 * Source all allocated, leave dest allocated
+	 */
+	if (scan->u.bmu_avail == 0)
+		return;
+	if (scan->u.bmu_avail == radix) {
+		/*
+		 * Source all free, free entire dest
+		 */
+		if (count < radix)
+			blist_free(dest, blk, count);
+		else
+			blist_free(dest, blk, radix);
+		return;
+	}
+
+	radix >>= BLIST_META_RADIX_SHIFT;
+	next_skip = (skip >> BLIST_META_RADIX_SHIFT);
+
+	for (i = 1; count && i <= skip; i += next_skip) {
+		if (scan[i].bm_bighint == (daddr_t)-1)
+			break;
+
+		if (count >= radix) {
+			blst_copy(
+			    &scan[i],
+			    blk,
+			    radix,
+			    next_skip - 1,
+			    dest,
+			    radix
+			);
+			count -= radix;
+		} else {
+			if (count) {
+				blst_copy(
+				    &scan[i],
+				    blk,
+				    radix,
+				    next_skip - 1,
+				    dest,
+				    count
+				);
+			}
+			count = 0;
+		}
+		blk += radix;
+	}
+}
+
+/*
+ * BLST_RADIX_INIT() - initialize radix tree
+ *
+ *	Initialize our meta structures and bitmaps and calculate the exact
+ *	amount of space required to manage 'count' blocks - this space may
+ *	be considerably less then the calculated radix due to the large
+ *	RADIX values we use.
+ */
+
+static daddr_t	
+blst_radix_init(blmeta_t *scan, daddr_t radix, int skip, daddr_t count)
+{
+	int i;
+	int next_skip;
+	daddr_t memindex = 0;
+
+	/*
+	 * Leaf node
+	 */
+
+	if (radix == BLIST_BMAP_RADIX) {
+		if (scan) {
+			scan->bm_bighint = 0;
+			scan->u.bmu_bitmap = 0;
+		}
+		return(memindex);
+	}
+
+	/*
+	 * Meta node.  If allocating the entire object we can special
+	 * case it.  However, we need to figure out how much memory
+	 * is required to manage 'count' blocks, so we continue on anyway.
+	 */
+
+	if (scan) {
+		scan->bm_bighint = 0;
+		scan->u.bmu_avail = 0;
+	}
+
+	radix >>= BLIST_META_RADIX_SHIFT;
+	next_skip = (skip >> BLIST_META_RADIX_SHIFT);
+
+	for (i = 1; i <= skip; i += next_skip) {
+		if (count >= radix) {
+			/*
+			 * Allocate the entire object
+			 */
+			memindex = i + blst_radix_init(
+			    ((scan) ? &scan[i] : NULL),
+			    radix,
+			    next_skip - 1,
+			    radix
+			);
+			count -= radix;
+		} else if (count > 0) {
+			/*
+			 * Allocate a partial object
+			 */
+			memindex = i + blst_radix_init(
+			    ((scan) ? &scan[i] : NULL),
+			    radix,
+			    next_skip - 1,
+			    count
+			);
+			count = 0;
+		} else {
+			/*
+			 * Add terminator and break out
+			 */
+			if (scan)
+				scan[i].bm_bighint = (daddr_t)-1;
+			break;
+		}
+	}
+	if (memindex < i)
+		memindex = i;
+	return(memindex);
+}
+
+#ifdef BLIST_DEBUG
+
+static void	
+blst_radix_print(blmeta_t *scan, daddr_t blk, daddr_t radix, int skip, int tab)
+{
+	int i;
+	int next_skip;
+	int lastState = 0;
+
+	if (radix == BLIST_BMAP_RADIX) {
+		printf(
+		    "%*.*s(%04x,%d): bitmap %08x big=%d\n", 
+		    tab, tab, "",
+		    blk, radix,
+		    scan->u.bmu_bitmap,
+		    scan->bm_bighint
+		);
+		return;
+	}
+
+	if (scan->u.bmu_avail == 0) {
+		printf(
+		    "%*.*s(%04x,%d) ALL ALLOCATED\n",
+		    tab, tab, "",
+		    blk,
+		    radix
+		);
+		return;
+	}
+	if (scan->u.bmu_avail == radix) {
+		printf(
+		    "%*.*s(%04x,%d) ALL FREE\n",
+		    tab, tab, "",
+		    blk,
+		    radix
+		);
+		return;
+	}
+
+	printf(
+	    "%*.*s(%04x,%d): subtree (%d/%d) big=%d {\n",
+	    tab, tab, "",
+	    blk, radix,
+	    scan->u.bmu_avail,
+	    radix,
+	    scan->bm_bighint
+	);
+
+	radix >>= BLIST_META_RADIX_SHIFT;
+	next_skip = (skip >> BLIST_META_RADIX_SHIFT);
+	tab += 4;
+
+	for (i = 1; i <= skip; i += next_skip) {
+		if (scan[i].bm_bighint == (daddr_t)-1) {
+			printf(
+			    "%*.*s(%04x,%d): Terminator\n",
+			    tab, tab, "",
+			    blk, radix
+			);
+			lastState = 0;
+			break;
+		}
+		blst_radix_print(
+		    &scan[i],
+		    blk,
+		    radix,
+		    next_skip - 1,
+		    tab
+		);
+		blk += radix;
+	}
+	tab -= 4;
+
+	printf(
+	    "%*.*s}\n",
+	    tab, tab, ""
+	);
+}
+
+#endif
+
+#ifdef BLIST_DEBUG
+
+int
+main(int ac, char **av)
+{
+	int size = 1024;
+	int i;
+	blist_t bl;
+
+	for (i = 1; i < ac; ++i) {
+		const char *ptr = av[i];
+		if (*ptr != '-') {
+			size = strtol(ptr, NULL, 0);
+			continue;
+		}
+		ptr += 2;
+		fprintf(stderr, "Bad option: %s\n", ptr - 2);
+		exit(1);
+	}
+	bl = blist_create(size);
+	blist_free(bl, 0, size);
+
+	for (;;) {
+		char buf[1024];
+		daddr_t da = 0;
+		daddr_t count = 0;
+
+
+		printf("%d/%d/%d> ", bl->bl_free, size, bl->bl_radix);
+		fflush(stdout);
+		if (fgets(buf, sizeof(buf), stdin) == NULL)
+			break;
+		switch(buf[0]) {
+		case 'r':
+			if (sscanf(buf + 1, "%d", &count) == 1) {
+				blist_resize(&bl, count, 1);
+			} else {
+				printf("?\n");
+			}
+		case 'p':
+			blist_print(bl);
+			break;
+		case 'a':
+			if (sscanf(buf + 1, "%d", &count) == 1) {
+				daddr_t blk = blist_alloc(bl, count);
+				printf("    R=%04x\n", blk);
+			} else {
+				printf("?\n");
+			}
+			break;
+		case 'f':
+			if (sscanf(buf + 1, "%x %d", &da, &count) == 2) {
+				blist_free(bl, da, count);
+			} else {
+				printf("?\n");
+			}
+			break;
+		case '?':
+		case 'h':
+			puts(
+			    "p          -print\n"
+			    "a %d       -allocate\n"
+			    "f %x %d    -free\n"
+			    "r %d       -resize\n"
+			    "h/?        -help"
+			);
+			break;
+		default:
+			printf("?\n");
+			break;
+		}
+	}
+	return(0);
+}
+
+void
+panic(const char *ctl, ...)
+{
+	va_list va;
+
+	va_start(va, ctl);
+	vfprintf(stderr, ctl, va);
+	fprintf(stderr, "\n");
+	va_end(va);
+	exit(1);
+}
+
+#endif
diff --git a/bsd/dev/dtrace/blist.h b/bsd/dev/dtrace/blist.h
new file mode 100644
index 000000000..6988eb4f4
--- /dev/null
+++ b/bsd/dev/dtrace/blist.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 1998 Matthew Dillon.  Terms of use and redistribution in all
+ * forms are covered by the BSD copyright in the file "/usr/src/COPYRIGHT".
+ *
+ * Implements bitmap resource lists.
+ *
+ *	Usage:
+ *		blist = blist_create(blocks)
+ *		(void)  blist_destroy(blist)
+ *		blkno = blist_alloc(blist, count)
+ *		(void)  blist_free(blist, blkno, count)
+ *		(void)  blist_resize(&blist, count, freeextra)
+ *		
+ *
+ *	Notes:
+ *		on creation, the entire list is marked reserved.  You should
+ *		first blist_free() the sections you want to make available
+ *		for allocation before doing general blist_alloc()/free()
+ *		ops.
+ *
+ *		SWAPBLK_NONE is returned on failure.  This module is typically
+ *		capable of managing up to (2^31) blocks per blist, though
+ *		the memory utilization would be insane if you actually did
+ *		that.  Managing something like 512MB worth of 4K blocks 
+ *		eats around 32 KBytes of memory. 
+ *
+ * $FreeBSD: src/sys/sys/blist.h,v 1.2 1999/08/28 00:51:33 peter Exp $
+ */
+
+#ifndef _SYS_BLIST_H_
+#define _SYS_BLIST_H_
+
+#define LOG2(v)		(((u_daddr_t)(v) >= 0x80000000U) ? 31 : \
+			((u_daddr_t)(v) >= 0x40000000U) ? 30 : \
+			((u_daddr_t)(v) >= 0x20000000U) ? 29 : \
+			((u_daddr_t)(v) >= 0x10000000U) ? 28 : \
+			((u_daddr_t)(v) >= 0x08000000U) ? 27 : \
+			((u_daddr_t)(v) >= 0x04000000U) ? 26 : \
+			((u_daddr_t)(v) >= 0x02000000U) ? 25 : \
+			((u_daddr_t)(v) >= 0x01000000U) ? 24 : \
+			((u_daddr_t)(v) >= 0x00800000U) ? 23 : \
+			((u_daddr_t)(v) >= 0x00400000U) ? 22 : \
+			((u_daddr_t)(v) >= 0x00200000U) ? 21 : \
+			((u_daddr_t)(v) >= 0x00100000U) ? 20 : \
+			((u_daddr_t)(v) >= 0x00080000U) ? 19 : \
+			((u_daddr_t)(v) >= 0x00040000U) ? 18 : \
+			((u_daddr_t)(v) >= 0x00020000U) ? 17 : \
+			((u_daddr_t)(v) >= 0x00010000U) ? 16 : \
+			((u_daddr_t)(v) >= 0x00008000U) ? 15 : \
+			((u_daddr_t)(v) >= 0x00004000U) ? 14 : \
+			((u_daddr_t)(v) >= 0x00002000U) ? 13 : \
+			((u_daddr_t)(v) >= 0x00001000U) ? 12 : \
+			((u_daddr_t)(v) >= 0x00000800U) ? 11 : \
+			((u_daddr_t)(v) >= 0x00000400U) ? 10 : \
+			((u_daddr_t)(v) >= 0x00000200U) ? 9 : \
+			((u_daddr_t)(v) >= 0x00000100U) ? 8 : \
+			((u_daddr_t)(v) >= 0x00000080U) ? 7 : \
+			((u_daddr_t)(v) >= 0x00000040U) ? 6 : \
+			((u_daddr_t)(v) >= 0x00000020U) ? 5 : \
+			((u_daddr_t)(v) >= 0x00000010U) ? 4 : \
+			((u_daddr_t)(v) >= 0x00000008U) ? 3 : \
+			((u_daddr_t)(v) >= 0x00000004U) ? 2 : \
+			((u_daddr_t)(v) >= 0x00000002U) ? 1 : 0)
+
+/*
+ * blmeta and bl_bitmap_t MUST be a power of 2 in size.
+ */
+
+typedef struct blmeta {
+	union {
+	    daddr_t	bmu_avail;	/* space available under us	*/
+	    u_daddr_t	bmu_bitmap;	/* bitmap if we are a leaf	*/
+	} u;
+	daddr_t		bm_bighint;	/* biggest contiguous block hint*/
+} blmeta_t;
+
+typedef struct blist {
+	daddr_t		bl_blocks;	/* area of coverage		*/
+	daddr_t		bl_radix;	/* coverage radix		*/
+	daddr_t		bl_skip;	/* starting skip		*/
+	daddr_t		bl_free;	/* number of free blocks	*/
+	blmeta_t	*bl_root;	/* root of radix tree		*/
+	daddr_t		bl_rootblks;	/* daddr_t blks allocated for tree */
+} *blist_t;
+
+#define BLIST_META_RADIX	16
+#define BLIST_META_RADIX_SHIFT	LOG2(BLIST_META_RADIX)
+#define BLIST_BMAP_RADIX	(sizeof(u_daddr_t)*8)
+#define BLIST_BMAP_RADIX_SHIFT	LOG2(BLIST_BMAP_RADIX)
+
+#define BLIST_MAX_ALLOC		BLIST_BMAP_RADIX
+
+extern blist_t blist_create(daddr_t blocks);
+extern void blist_destroy(blist_t blist);
+extern daddr_t blist_alloc(blist_t blist, daddr_t count);
+extern void blist_free(blist_t blist, daddr_t blkno, daddr_t count);
+extern void blist_print(blist_t blist);
+extern void blist_resize(blist_t *pblist, daddr_t count, int freenew);
+
+#endif	/* _SYS_BLIST_H_ */
diff --git a/bsd/dev/dtrace/dtrace.c b/bsd/dev/dtrace/dtrace.c
new file mode 100644
index 000000000..f34ed4fd0
--- /dev/null
+++ b/bsd/dev/dtrace/dtrace.c
@@ -0,0 +1,16547 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)dtrace.c	1.49	06/08/11 SMI" */
+
+/*
+ * DTrace - Dynamic Tracing for Solaris
+ *
+ * This is the implementation of the Solaris Dynamic Tracing framework
+ * (DTrace).  The user-visible interface to DTrace is described at length in
+ * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
+ * library, the in-kernel DTrace framework, and the DTrace providers are
+ * described in the block comments in the <sys/dtrace.h> header file.  The
+ * internal architecture of DTrace is described in the block comments in the
+ * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
+ * implementation very much assume mastery of all of these sources; if one has
+ * an unanswered question about the implementation, one should consult them
+ * first.
+ *
+ * The functions here are ordered roughly as follows:
+ *
+ *   - Probe context functions
+ *   - Probe hashing functions
+ *   - Non-probe context utility functions
+ *   - Matching functions
+ *   - Provider-to-Framework API functions
+ *   - Probe management functions
+ *   - DIF object functions
+ *   - Format functions
+ *   - Predicate functions
+ *   - ECB functions
+ *   - Buffer functions
+ *   - Enabling functions
+ *   - DOF functions
+ *   - Anonymous enabling functions
+ *   - Consumer state functions
+ *   - Helper functions
+ *   - Hook functions
+ *   - Driver cookbook functions
+ *
+ * Each group of functions begins with a block comment labelled the "DTrace
+ * [Group] Functions", allowing one to find each block by searching forward
+ * on capital-f functions.
+ */
+
+#define _DTRACE_WANT_PROC_GLUE_	1
+
+#include <sys/errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/dtrace_impl.h>
+#include <sys/param.h>
+#include <sys/ioctl.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+#include <sys/malloc.h>
+#include <sys/kernel_types.h>
+#include <sys/proc_internal.h>
+#include <sys/uio_internal.h>
+#include <sys/kauth.h>
+#include <vm/pmap.h>
+#include <sys/user.h>
+#include <mach/exception_types.h>
+#include <sys/signalvar.h>
+#include <kern/zalloc.h>
+
+#define t_predcache t_dtrace_predcache /* Cosmetic. Helps readability of thread.h */
+
+extern void dtrace_suspend(void);
+extern void dtrace_resume(void);
+extern void dtrace_init(void);
+extern void helper_init(void);
+
+#if defined(__APPLE__)
+
+#include "../../../osfmk/chud/chud_dtrace.h"
+
+extern kern_return_t chudxnu_dtrace_callback
+	(uint64_t selector, uint64_t *args, uint32_t count);
+#endif
+
+/*
+ * DTrace Tunable Variables
+ *
+ * The following variables may be tuned by adding a line to /etc/system that
+ * includes both the name of the DTrace module ("dtrace") and the name of the
+ * variable.  For example:
+ *
+ *   set dtrace:dtrace_destructive_disallow = 1
+ *
+ * In general, the only variables that one should be tuning this way are those
+ * that affect system-wide DTrace behavior, and for which the default behavior
+ * is undesirable.  Most of these variables are tunable on a per-consumer
+ * basis using DTrace options, and need not be tuned on a system-wide basis.
+ * When tuning these variables, avoid pathological values; while some attempt
+ * is made to verify the integrity of these variables, they are not considered
+ * part of the supported interface to DTrace, and they are therefore not
+ * checked comprehensively.  Further, these variables should not be tuned
+ * dynamically via "mdb -kw" or other means; they should only be tuned via
+ * /etc/system.
+ */
+int		dtrace_destructive_disallow = 0;
+#if defined(__APPLE__)
+#define proc_t struct proc
+#endif /* __APPLE__ */
+dtrace_optval_t	dtrace_nonroot_maxsize = (16 * 1024 * 1024);
+size_t		dtrace_difo_maxsize = (256 * 1024);
+dtrace_optval_t	dtrace_dof_maxsize = (256 * 1024);
+size_t		dtrace_global_maxsize = (16 * 1024);
+size_t		dtrace_actions_max = (16 * 1024);
+size_t		dtrace_retain_max = 1024;
+dtrace_optval_t	dtrace_helper_actions_max = 32;
+dtrace_optval_t	dtrace_helper_providers_max = 32;
+dtrace_optval_t	dtrace_dstate_defsize = (1 * 1024 * 1024);
+size_t		dtrace_strsize_default = 256;
+dtrace_optval_t	dtrace_cleanrate_default = 9900990;		/* 101 hz */
+dtrace_optval_t	dtrace_cleanrate_min = 200000;			/* 5000 hz */
+dtrace_optval_t	dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;	/* 1/minute */
+dtrace_optval_t	dtrace_aggrate_default = NANOSEC;		/* 1 hz */
+dtrace_optval_t	dtrace_statusrate_default = NANOSEC;		/* 1 hz */
+dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;	 /* 6/minute */
+dtrace_optval_t	dtrace_switchrate_default = NANOSEC;		/* 1 hz */
+dtrace_optval_t	dtrace_nspec_default = 1;
+dtrace_optval_t	dtrace_specsize_default = 32 * 1024;
+dtrace_optval_t dtrace_stackframes_default = 20;
+dtrace_optval_t dtrace_ustackframes_default = 20;
+dtrace_optval_t dtrace_jstackframes_default = 50;
+dtrace_optval_t dtrace_jstackstrsize_default = 512;
+int		dtrace_msgdsize_max = 128;
+hrtime_t	dtrace_chill_max = 500 * (NANOSEC / MILLISEC);	/* 500 ms */
+hrtime_t	dtrace_chill_interval = NANOSEC;		/* 1000 ms */
+int		dtrace_devdepth_max = 32;
+int		dtrace_err_verbose;
+hrtime_t	dtrace_deadman_interval = NANOSEC;
+hrtime_t	dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
+hrtime_t	dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
+
+/*
+ * DTrace External Variables
+ *
+ * As dtrace(7D) is a kernel module, any DTrace variables are obviously
+ * available to DTrace consumers via the backtick (`) syntax.  One of these,
+ * dtrace_zero, is made deliberately so:  it is provided as a source of
+ * well-known, zero-filled memory.  While this variable is not documented,
+ * it is used by some translators as an implementation detail.
+ */
+const char	dtrace_zero[256] = { 0 };	/* zero-filled memory */
+
+/*
+ * DTrace Internal Variables
+ */
+static dev_info_t	*dtrace_devi;		/* device info */
+static vmem_t		*dtrace_arena;		/* probe ID arena */
+static vmem_t		*dtrace_minor;		/* minor number arena */
+static taskq_t		*dtrace_taskq;		/* task queue */
+static dtrace_probe_t	**dtrace_probes;	/* array of all probes */
+static int		dtrace_nprobes;		/* number of probes */
+static dtrace_provider_t *dtrace_provider;	/* provider list */
+static dtrace_meta_t	*dtrace_meta_pid;	/* user-land meta provider */
+static int		dtrace_opens;		/* number of opens */
+static int		dtrace_helpers;		/* number of helpers */
+static void		*dtrace_softstate;	/* softstate pointer */
+static dtrace_hash_t	*dtrace_bymod;		/* probes hashed by module */
+static dtrace_hash_t	*dtrace_byfunc;		/* probes hashed by function */
+static dtrace_hash_t	*dtrace_byname;		/* probes hashed by name */
+static dtrace_toxrange_t *dtrace_toxrange;	/* toxic range array */
+static int		dtrace_toxranges;	/* number of toxic ranges */
+static int		dtrace_toxranges_max;	/* size of toxic range array */
+static dtrace_anon_t	dtrace_anon;		/* anonymous enabling */
+static kmem_cache_t	*dtrace_state_cache;	/* cache for dynamic state */
+static uint64_t		dtrace_vtime_references; /* number of vtimestamp refs */
+static kthread_t	*dtrace_panicked;	/* panicking thread */
+static dtrace_ecb_t	*dtrace_ecb_create_cache; /* cached created ECB */
+static dtrace_genid_t	dtrace_probegen;	/* current probe generation */
+static dtrace_helpers_t *dtrace_deferred_pid;	/* deferred helper list */
+static dtrace_enabling_t *dtrace_retained;	/* list of retained enablings */
+static dtrace_dynvar_t	dtrace_dynhash_sink;	/* end of dynamic hash chains */
+#if defined(__APPLE__)
+static int		dtrace_dof_mode;	/* dof mode */
+#endif
+
+#if defined(__APPLE__)
+
+/*
+ * To save memory, some common memory allocations are given a
+ * unique zone. In example, dtrace_probe_t is 72 bytes in size,
+ * which means it would fall into the kalloc.128 bucket. With
+ * 20k elements allocated, the space saved is substantial.
+ */
+
+struct zone *dtrace_probe_t_zone;
+
+#endif
+
+/*
+ * DTrace Locking
+ * DTrace is protected by three (relatively coarse-grained) locks:
+ *
+ * (1) dtrace_lock is required to manipulate essentially any DTrace state,
+ *     including enabling state, probes, ECBs, consumer state, helper state,
+ *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
+ *     probe context is lock-free -- synchronization is handled via the
+ *     dtrace_sync() cross call mechanism.
+ *
+ * (2) dtrace_provider_lock is required when manipulating provider state, or
+ *     when provider state must be held constant.
+ *
+ * (3) dtrace_meta_lock is required when manipulating meta provider state, or
+ *     when meta provider state must be held constant.
+ *
+ * The lock ordering between these three locks is dtrace_meta_lock before
+ * dtrace_provider_lock before dtrace_lock.  (In particular, there are
+ * several places where dtrace_provider_lock is held by the framework as it
+ * calls into the providers -- which then call back into the framework,
+ * grabbing dtrace_lock.)
+ *
+ * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
+ * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
+ * role as a coarse-grained lock; it is acquired before both of these locks.
+ * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
+ * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
+ * mod_lock is similar with respect to dtrace_provider_lock in that it must be
+ * acquired _between_ dtrace_provider_lock and dtrace_lock.
+ */
+
+/*
+ * APPLE NOTE:
+ *
+ * All kmutex_t vars have been changed to lck_mtx_t.
+ * Note that lck_mtx_t's require explicit initialization.
+ *
+ * mutex_enter() becomes lck_mtx_lock()
+ * mutex_exit() becomes lck_mtx_unlock()
+ *
+ * Lock asserts are changed like this:
+ *
+ * ASSERT(MUTEX_HELD(&cpu_lock));
+ *	becomes:
+ * lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+ *
+ * Due to the number of these changes, they are not called out explicitly.
+ */
+static lck_mtx_t	dtrace_lock;		/* probe state lock */
+static lck_mtx_t	dtrace_provider_lock;	/* provider state lock */
+static lck_mtx_t	dtrace_meta_lock;	/* meta-provider state lock */
+#if defined(__APPLE__)
+static lck_rw_t		dtrace_dof_mode_lock;	/* dof mode lock */
+#endif
+
+/*
+ * DTrace Provider Variables
+ *
+ * These are the variables relating to DTrace as a provider (that is, the
+ * provider of the BEGIN, END, and ERROR probes).
+ */
+static dtrace_pattr_t	dtrace_provider_attr = {
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+};
+
+static void
+dtrace_nullop(void)
+{}
+
+static dtrace_pops_t	dtrace_provider_ops = {
+	(void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
+	(void (*)(void *, struct modctl *))dtrace_nullop,
+	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
+	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
+	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
+	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
+	NULL,
+	NULL,
+	NULL,
+	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop
+};
+
+static dtrace_id_t	dtrace_probeid_begin;	/* special BEGIN probe */
+static dtrace_id_t	dtrace_probeid_end;	/* special END probe */
+dtrace_id_t		dtrace_probeid_error;	/* special ERROR probe */
+
+/*
+ * DTrace Helper Tracing Variables
+ */
+uint32_t dtrace_helptrace_next = 0;
+uint32_t dtrace_helptrace_nlocals;
+char	*dtrace_helptrace_buffer;
+int	dtrace_helptrace_bufsize = 512 * 1024;
+
+#ifdef DEBUG
+int	dtrace_helptrace_enabled = 1;
+#else
+int	dtrace_helptrace_enabled = 0;
+#endif
+
+/*
+ * DTrace Error Hashing
+ *
+ * On DEBUG kernels, DTrace will track the errors that has seen in a hash
+ * table.  This is very useful for checking coverage of tests that are
+ * expected to induce DIF or DOF processing errors, and may be useful for
+ * debugging problems in the DIF code generator or in DOF generation .  The
+ * error hash may be examined with the ::dtrace_errhash MDB dcmd.
+ */
+#ifdef DEBUG
+static dtrace_errhash_t	dtrace_errhash[DTRACE_ERRHASHSZ];
+static const char *dtrace_errlast;
+static kthread_t *dtrace_errthread;
+static lck_mtx_t dtrace_errlock;
+#endif
+
+/*
+ * DTrace Macros and Constants
+ *
+ * These are various macros that are useful in various spots in the
+ * implementation, along with a few random constants that have no meaning
+ * outside of the implementation.  There is no real structure to this cpp
+ * mishmash -- but is there ever?
+ */
+#define	DTRACE_HASHSTR(hash, probe)	\
+	dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
+
+#define	DTRACE_HASHNEXT(hash, probe)	\
+	(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
+
+#define	DTRACE_HASHPREV(hash, probe)	\
+	(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
+
+#define	DTRACE_HASHEQ(hash, lhs, rhs)	\
+	(strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
+	    *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
+
+#define	DTRACE_AGGHASHSIZE_SLEW		17
+
+/*
+ * The key for a thread-local variable consists of the lower 61 bits of the
+ * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
+ * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
+ * equal to a variable identifier.  This is necessary (but not sufficient) to
+ * assure that global associative arrays never collide with thread-local
+ * variables.  To guarantee that they cannot collide, we must also define the
+ * order for keying dynamic variables.  That order is:
+ *
+ *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
+ *
+ * Because the variable-key and the tls-key are in orthogonal spaces, there is
+ * no way for a global variable key signature to match a thread-local key
+ * signature.
+ */
+#if !defined(__APPLE__)
+#define	DTRACE_TLS_THRKEY(where) { \
+	uint_t intr = 0; \
+	uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
+	for (; actv; actv >>= 1) \
+		intr++; \
+	ASSERT(intr < (1 << 3)); \
+	(where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
+	    (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
+}
+#else
+#define	DTRACE_TLS_THRKEY(where) { \
+	uint_t intr = ml_at_interrupt_context(); /* XXX just one measely bit */ \
+	uint_t thr = (uint_t)current_thread(); \
+	uint_t pid = (uint_t)proc_selfpid(); \
+	ASSERT(intr < (1 << 3)); \
+	(where) = ((((uint64_t)thr << 32 | pid) + DIF_VARIABLE_MAX) & \
+	    (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
+}
+#endif /* __APPLE__ */
+
+#define	DTRACE_STORE(type, tomax, offset, what) \
+	*((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
+
+#if !defined(__APPLE__)
+#if !(defined(__i386__) || defined (__x86_64__))
+#define	DTRACE_ALIGNCHECK(addr, size, flags)				\
+	if (addr & (size - 1)) {					\
+		*flags |= CPU_DTRACE_BADALIGN;				\
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
+		return (0);						\
+	}
+#else
+#define	DTRACE_ALIGNCHECK(addr, size, flags)
+#endif
+
+#define	DTRACE_LOADFUNC(bits)						\
+/*CSTYLED*/								\
+uint##bits##_t								\
+dtrace_load##bits(uintptr_t addr)					\
+{									\
+	size_t size = bits / NBBY;					\
+	/*CSTYLED*/							\
+	uint##bits##_t rval;						\
+	int i;								\
+	volatile uint16_t *flags = (volatile uint16_t *)		\
+	    &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;			\
+									\
+	DTRACE_ALIGNCHECK(addr, size, flags);				\
+									\
+	for (i = 0; i < dtrace_toxranges; i++) {			\
+		if (addr >= dtrace_toxrange[i].dtt_limit)		\
+			continue;					\
+									\
+		if (addr + size <= dtrace_toxrange[i].dtt_base)		\
+			continue;					\
+									\
+		/*							\
+		 * This address falls within a toxic region; return 0.	\
+		 */							\
+		*flags |= CPU_DTRACE_BADADDR;				\
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
+		return (0);						\
+	}								\
+									\
+	*flags |= CPU_DTRACE_NOFAULT;					\
+	/*CSTYLED*/							\
+	rval = *((volatile uint##bits##_t *)addr);			\
+	*flags &= ~CPU_DTRACE_NOFAULT;					\
+									\
+	return (rval);							\
+}
+#else
+#define	DTRACE_ALIGNCHECK(addr, size, flags)				\
+	if (addr & (MIN(size,4) - 1)) {					\
+		*flags |= CPU_DTRACE_BADALIGN;				\
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
+		return (0);						\
+	}
+
+#define RECOVER_LABEL(bits) __asm__ volatile("_dtraceLoadRecover" #bits ":" );
+
+#define	DTRACE_LOADFUNC(bits)						\
+/*CSTYLED*/								\
+extern vm_offset_t dtraceLoadRecover##bits;				\
+uint##bits##_t dtrace_load##bits(uintptr_t addr);			\
+									\
+uint##bits##_t								\
+dtrace_load##bits(uintptr_t addr)					\
+{									\
+	size_t size = bits / NBBY;					\
+	/*CSTYLED*/							\
+	uint##bits##_t rval = 0;					\
+	int i;								\
+	ppnum_t pp;							\
+	volatile uint16_t *flags = (volatile uint16_t *)		\
+	    &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;			\
+									\
+	DTRACE_ALIGNCHECK(addr, size, flags);				\
+									\
+	for (i = 0; i < dtrace_toxranges; i++) {			\
+		if (addr >= dtrace_toxrange[i].dtt_limit)		\
+			continue;					\
+									\
+		if (addr + size <= dtrace_toxrange[i].dtt_base)		\
+			continue;					\
+									\
+		/*							\
+		 * This address falls within a toxic region; return 0.	\
+		 */							\
+		*flags |= CPU_DTRACE_BADADDR;				\
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
+		return (0);						\
+	}								\
+									\
+	pp = pmap_find_phys(kernel_pmap, addr);				\
+									\
+	if (0 == pp || /* pmap_find_phys failed ? */			\
+	    !dtxnu_is_RAM_page(pp) /* Backed by RAM? */ ) {		\
+		*flags |= CPU_DTRACE_BADADDR;				\
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
+		return (0);						\
+	}								\
+									\
+	{								\
+	volatile vm_offset_t recover = (vm_offset_t)&dtraceLoadRecover##bits;		\
+	*flags |= CPU_DTRACE_NOFAULT;					\
+	recover = dtrace_set_thread_recover(current_thread(), recover);	\
+	/*CSTYLED*/							\
+	rval = *((volatile uint##bits##_t *)addr);			\
+	RECOVER_LABEL(bits);						\
+	(void)dtrace_set_thread_recover(current_thread(), recover);	\
+	*flags &= ~CPU_DTRACE_NOFAULT;					\
+	}								\
+									\
+	return (rval);							\
+}
+#endif /* __APPLE__ */
+
+
+#ifdef __LP64__
+#define	dtrace_loadptr	dtrace_load64
+#else
+#define	dtrace_loadptr	dtrace_load32
+#endif
+
+#define	DTRACE_DYNHASH_FREE	0
+#define	DTRACE_DYNHASH_SINK	1
+#define	DTRACE_DYNHASH_VALID	2
+
+#define	DTRACE_MATCH_NEXT	0
+#define	DTRACE_MATCH_DONE	1
+#define	DTRACE_ANCHORED(probe)	((probe)->dtpr_func[0] != '\0')
+#define	DTRACE_STATE_ALIGN	64
+
+#define	DTRACE_FLAGS2FLT(flags)						\
+	(((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :		\
+	((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :		\
+	((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :		\
+	((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :		\
+	((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :		\
+	((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :		\
+	((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :		\
+	((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :	\
+	DTRACEFLT_UNKNOWN)
+
+#define	DTRACEACT_ISSTRING(act)						\
+	((act)->dta_kind == DTRACEACT_DIFEXPR &&			\
+	(act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING)
+
+static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
+static void dtrace_enabling_provide(dtrace_provider_t *);
+static int dtrace_enabling_match(dtrace_enabling_t *, int *);
+static void dtrace_enabling_matchall(void);
+static dtrace_state_t *dtrace_anon_grab(void);
+static uint64_t dtrace_helper(int, dtrace_mstate_t *,
+    dtrace_state_t *, uint64_t, uint64_t);
+static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
+static void dtrace_buffer_drop(dtrace_buffer_t *);
+static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
+    dtrace_state_t *, dtrace_mstate_t *);
+static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
+    dtrace_optval_t);
+static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
+static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *);
+
+/*
+ * DTrace Probe Context Functions
+ *
+ * These functions are called from probe context.  Because probe context is
+ * any context in which C may be called, arbitrarily locks may be held,
+ * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
+ * As a result, functions called from probe context may only call other DTrace
+ * support functions -- they may not interact at all with the system at large.
+ * (Note that the ASSERT macro is made probe-context safe by redefining it in
+ * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
+ * loads are to be performed from probe context, they _must_ be in terms of
+ * the safe dtrace_load*() variants.
+ *
+ * Some functions in this block are not actually called from probe context;
+ * for these functions, there will be a comment above the function reading
+ * "Note:  not called from probe context."
+ */
+void
+dtrace_panic(const char *format, ...)
+{
+	va_list alist;
+
+	va_start(alist, format);
+	dtrace_vpanic(format, alist);
+	va_end(alist);
+}
+
+int
+dtrace_assfail(const char *a, const char *f, int l)
+{
+	dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
+
+	/*
+	 * We just need something here that even the most clever compiler
+	 * cannot optimize away.
+	 */
+	return (a[(uintptr_t)f]);
+}
+
+/*
+ * Atomically increment a specified error counter from probe context.
+ */
+static void
+dtrace_error(uint32_t *counter)
+{
+	/*
+	 * Most counters stored to in probe context are per-CPU counters.
+	 * However, there are some error conditions that are sufficiently
+	 * arcane that they don't merit per-CPU storage.  If these counters
+	 * are incremented concurrently on different CPUs, scalability will be
+	 * adversely affected -- but we don't expect them to be white-hot in a
+	 * correctly constructed enabling...
+	 */
+	uint32_t oval, nval;
+
+	do {
+		oval = *counter;
+
+		if ((nval = oval + 1) == 0) {
+			/*
+			 * If the counter would wrap, set it to 1 -- assuring
+			 * that the counter is never zero when we have seen
+			 * errors.  (The counter must be 32-bits because we
+			 * aren't guaranteed a 64-bit compare&swap operation.)
+			 * To save this code both the infamy of being fingered
+			 * by a priggish news story and the indignity of being
+			 * the target of a neo-puritan witch trial, we're
+			 * carefully avoiding any colorful description of the
+			 * likelihood of this condition -- but suffice it to
+			 * say that it is only slightly more likely than the
+			 * overflow of predicate cache IDs, as discussed in
+			 * dtrace_predicate_create().
+			 */
+			nval = 1;
+		}
+	} while (dtrace_cas32(counter, oval, nval) != oval);
+}
+
+/*
+ * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
+ * uint8_t, a uint16_t, a uint32_t and a uint64_t.
+ */
+DTRACE_LOADFUNC(8)
+DTRACE_LOADFUNC(16)
+DTRACE_LOADFUNC(32)
+DTRACE_LOADFUNC(64)
+
+static int
+dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
+{
+	if (dest < mstate->dtms_scratch_base)
+		return (0);
+
+	if (dest + size < dest)
+		return (0);
+
+	if (dest + size > mstate->dtms_scratch_ptr)
+		return (0);
+
+	return (1);
+}
+
+static int
+dtrace_canstore_statvar(uint64_t addr, size_t sz,
+    dtrace_statvar_t **svars, int nsvars)
+{
+	int i;
+
+	for (i = 0; i < nsvars; i++) {
+		dtrace_statvar_t *svar = svars[i];
+
+		if (svar == NULL || svar->dtsv_size == 0)
+			continue;
+
+		if (addr - svar->dtsv_data < svar->dtsv_size &&
+		    addr + sz <= svar->dtsv_data + svar->dtsv_size)
+			return (1);
+	}
+
+	return (0);
+}
+
+/*
+ * Check to see if the address is within a memory region to which a store may
+ * be issued.  This includes the DTrace scratch areas, and any DTrace variable
+ * region.  The caller of dtrace_canstore() is responsible for performing any
+ * alignment checks that are needed before stores are actually executed.
+ */
+static int
+dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
+    dtrace_vstate_t *vstate)
+{
+	uintptr_t a;
+	size_t s;
+
+	/*
+	 * First, check to see if the address is in scratch space...
+	 */
+	a = mstate->dtms_scratch_base;
+	s = mstate->dtms_scratch_size;
+
+	if (addr - a < s && addr + sz <= a + s)
+		return (1);
+
+	/*
+	 * Now check to see if it's a dynamic variable.  This check will pick
+	 * up both thread-local variables and any global dynamically-allocated
+	 * variables.
+	 */
+	a = (uintptr_t)vstate->dtvs_dynvars.dtds_base;
+	s = vstate->dtvs_dynvars.dtds_size;
+	if (addr - a < s && addr + sz <= a + s)
+		return (1);
+
+	/*
+	 * Finally, check the static local and global variables.  These checks
+	 * take the longest, so we perform them last.
+	 */
+	if (dtrace_canstore_statvar(addr, sz,
+	    vstate->dtvs_locals, vstate->dtvs_nlocals))
+		return (1);
+
+	if (dtrace_canstore_statvar(addr, sz,
+	    vstate->dtvs_globals, vstate->dtvs_nglobals))
+		return (1);
+
+	return (0);
+}
+
+/*
+ * Compare two strings using safe loads.
+ */
+static int
+dtrace_strncmp(char *s1, char *s2, size_t limit)
+{
+	uint8_t c1, c2;
+	volatile uint16_t *flags;
+
+	if (s1 == s2 || limit == 0)
+		return (0);
+
+	flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+
+	do {
+		if (s1 == NULL)
+			c1 = '\0';
+		else
+			c1 = dtrace_load8((uintptr_t)s1++);
+
+		if (s2 == NULL)
+			c2 = '\0';
+		else
+			c2 = dtrace_load8((uintptr_t)s2++);
+
+		if (c1 != c2)
+			return (c1 - c2);
+	} while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
+
+	return (0);
+}
+
+/*
+ * Compute strlen(s) for a string using safe memory accesses.  The additional
+ * len parameter is used to specify a maximum length to ensure completion.
+ */
+static size_t
+dtrace_strlen(const char *s, size_t lim)
+{
+	uint_t len;
+
+	for (len = 0; len != lim; len++)
+		if (dtrace_load8((uintptr_t)s++) == '\0')
+			break;
+
+	return (len);
+}
+
+/*
+ * Check if an address falls within a toxic region.
+ */
+static int
+dtrace_istoxic(uintptr_t kaddr, size_t size)
+{
+	uintptr_t taddr, tsize;
+	int i;
+
+	for (i = 0; i < dtrace_toxranges; i++) {
+		taddr = dtrace_toxrange[i].dtt_base;
+		tsize = dtrace_toxrange[i].dtt_limit - taddr;
+
+		if (kaddr - taddr < tsize) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
+			return (1);
+		}
+
+		if (taddr - kaddr < size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
+			return (1);
+		}
+	}
+
+	return (0);
+}
+
+/*
+ * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
+ * memory specified by the DIF program.  The dst is assumed to be safe memory
+ * that we can store to directly because it is managed by DTrace.  As with
+ * standard bcopy, overlapping copies are handled properly.
+ */
+static void
+dtrace_bcopy(const void *src, void *dst, size_t len)
+{
+	if (len != 0) {
+		uint8_t *s1 = dst;
+		const uint8_t *s2 = src;
+
+		if (s1 <= s2) {
+			do {
+				*s1++ = dtrace_load8((uintptr_t)s2++);
+			} while (--len != 0);
+		} else {
+			s2 += len;
+			s1 += len;
+
+			do {
+				*--s1 = dtrace_load8((uintptr_t)--s2);
+			} while (--len != 0);
+		}
+	}
+}
+
+/*
+ * Copy src to dst using safe memory accesses, up to either the specified
+ * length, or the point that a nul byte is encountered.  The src is assumed to
+ * be unsafe memory specified by the DIF program.  The dst is assumed to be
+ * safe memory that we can store to directly because it is managed by DTrace.
+ * Unlike dtrace_bcopy(), overlapping regions are not handled.
+ */
+static void
+dtrace_strcpy(const void *src, void *dst, size_t len)
+{
+	if (len != 0) {
+		uint8_t *s1 = dst, c;
+		const uint8_t *s2 = src;
+
+		do {
+			*s1++ = c = dtrace_load8((uintptr_t)s2++);
+		} while (--len != 0 && c != '\0');
+	}
+}
+
+/*
+ * Copy src to dst, deriving the size and type from the specified (BYREF)
+ * variable type.  The src is assumed to be unsafe memory specified by the DIF
+ * program.  The dst is assumed to be DTrace variable memory that is of the
+ * specified type; we assume that we can store to directly.
+ */
+static void
+dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
+{
+	ASSERT(type->dtdt_flags & DIF_TF_BYREF);
+
+	if (type->dtdt_kind == DIF_TYPE_STRING)
+		dtrace_strcpy(src, dst, type->dtdt_size);
+	else
+		dtrace_bcopy(src, dst, type->dtdt_size);
+}
+
+/*
+ * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
+ * unsafe memory specified by the DIF program.  The s2 data is assumed to be
+ * safe memory that we can access directly because it is managed by DTrace.
+ */
+static int
+dtrace_bcmp(const void *s1, const void *s2, size_t len)
+{
+	volatile uint16_t *flags;
+
+	flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+
+	if (s1 == s2)
+		return (0);
+
+	if (s1 == NULL || s2 == NULL)
+		return (1);
+
+	if (s1 != s2 && len != 0) {
+		const uint8_t *ps1 = s1;
+		const uint8_t *ps2 = s2;
+
+		do {
+			if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
+				return (1);
+		} while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
+	}
+	return (0);
+}
+
+/*
+ * Zero the specified region using a simple byte-by-byte loop.  Note that this
+ * is for safe DTrace-managed memory only.
+ */
+static void
+dtrace_bzero(void *dst, size_t len)
+{
+	uchar_t *cp;
+
+	for (cp = dst; len != 0; len--)
+		*cp++ = 0;
+}
+
+/*
+ * This privilege check should be used by actions and subroutines to
+ * verify that the user credentials of the process that enabled the
+ * invoking ECB match the target credentials
+ */
+static int
+dtrace_priv_proc_common_user(dtrace_state_t *state)
+{
+	cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
+
+	/*
+	 * We should always have a non-NULL state cred here, since if cred
+	 * is null (anonymous tracing), we fast-path bypass this routine.
+	 */
+	ASSERT(s_cr != NULL);
+
+#if !defined(__APPLE__)
+	if ((cr = CRED()) != NULL &&
+#else
+	if ((cr = dtrace_CRED()) != NULL &&
+#endif /* __APPLE__ */
+	    s_cr->cr_uid == cr->cr_uid &&
+	    s_cr->cr_uid == cr->cr_ruid &&
+	    s_cr->cr_uid == cr->cr_suid &&
+	    s_cr->cr_gid == cr->cr_gid &&
+	    s_cr->cr_gid == cr->cr_rgid &&
+	    s_cr->cr_gid == cr->cr_sgid)
+		return (1);
+
+	return (0);
+}
+
+/*
+ * This privilege check should be used by actions and subroutines to
+ * verify that the zone of the process that enabled the invoking ECB
+ * matches the target credentials
+ */
+static int
+dtrace_priv_proc_common_zone(dtrace_state_t *state)
+{
+	cred_t *cr, *s_cr = state->dts_cred.dcr_cred;
+
+	/*
+	 * We should always have a non-NULL state cred here, since if cred
+	 * is null (anonymous tracing), we fast-path bypass this routine.
+	 */
+	ASSERT(s_cr != NULL);
+
+#if !defined(__APPLE__)
+	if ((cr = CRED()) != NULL &&
+	    s_cr->cr_zone == cr->cr_zone)
+		return (1);
+
+	return (0);
+#else
+	return 1; /* Darwin doesn't do zones. */
+#endif /* __APPLE__ */
+}
+
+/*
+ * This privilege check should be used by actions and subroutines to
+ * verify that the process has not setuid or changed credentials.
+ */
+#if !defined(__APPLE__)
+static int
+dtrace_priv_proc_common_nocd()
+{
+	proc_t *proc;
+
+	if ((proc = ttoproc(curthread)) != NULL &&
+	    !(proc->p_flag & SNOCD))
+		return (1);
+
+	return (0);
+}
+#else
+static int
+dtrace_priv_proc_common_nocd(void)
+{
+	return 1; /* Darwin omits "No Core Dump" flag. */
+}
+#endif /* __APPLE__ */
+
+static int
+dtrace_priv_proc_destructive(dtrace_state_t *state)
+{
+	int action = state->dts_cred.dcr_action;
+
+	if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) &&
+	    dtrace_priv_proc_common_zone(state) == 0)
+		goto bad;
+
+	if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) &&
+	    dtrace_priv_proc_common_user(state) == 0)
+		goto bad;
+
+	if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) &&
+	    dtrace_priv_proc_common_nocd() == 0)
+		goto bad;
+
+	return (1);
+
+bad:
+	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
+
+	return (0);
+}
+
+static int
+dtrace_priv_proc_control(dtrace_state_t *state)
+{
+	if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
+		return (1);
+
+	if (dtrace_priv_proc_common_zone(state) &&
+	    dtrace_priv_proc_common_user(state) &&
+	    dtrace_priv_proc_common_nocd())
+		return (1);
+
+	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
+
+	return (0);
+}
+
+static int
+dtrace_priv_proc(dtrace_state_t *state)
+{
+	if (state->dts_cred.dcr_action & DTRACE_CRA_PROC)
+		return (1);
+
+	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
+
+	return (0);
+}
+
+static int
+dtrace_priv_kernel(dtrace_state_t *state)
+{
+	if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
+		return (1);
+
+	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
+
+	return (0);
+}
+
+static int
+dtrace_priv_kernel_destructive(dtrace_state_t *state)
+{
+	if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
+		return (1);
+
+	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
+
+	return (0);
+}
+
+/*
+ * Note:  not called from probe context.  This function is called
+ * asynchronously (and at a regular interval) from outside of probe context to
+ * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
+ * cleaning is explained in detail in <sys/dtrace_impl.h>.
+ */
+#if defined(__APPLE__)
+static 
+#endif /* __APPLE__ */
+void
+dtrace_dynvar_clean(dtrace_dstate_t *dstate)
+{
+	dtrace_dynvar_t *dirty;
+	dtrace_dstate_percpu_t *dcpu;
+	int i, work = 0;
+
+	for (i = 0; i < NCPU; i++) {
+		dcpu = &dstate->dtds_percpu[i];
+
+		ASSERT(dcpu->dtdsc_rinsing == NULL);
+
+		/*
+		 * If the dirty list is NULL, there is no dirty work to do.
+		 */
+		if (dcpu->dtdsc_dirty == NULL)
+			continue;
+
+		/*
+		 * If the clean list is non-NULL, then we're not going to do
+		 * any work for this CPU -- it means that there has not been
+		 * a dtrace_dynvar() allocation on this CPU (or from this CPU)
+		 * since the last time we cleaned house.
+		 */
+		if (dcpu->dtdsc_clean != NULL)
+			continue;
+
+		work = 1;
+
+		/*
+		 * Atomically move the dirty list aside.
+		 */
+		do {
+			dirty = dcpu->dtdsc_dirty;
+
+			/*
+			 * Before we zap the dirty list, set the rinsing list.
+			 * (This allows for a potential assertion in
+			 * dtrace_dynvar():  if a free dynamic variable appears
+			 * on a hash chain, either the dirty list or the
+			 * rinsing list for some CPU must be non-NULL.)
+			 */
+			dcpu->dtdsc_rinsing = dirty;
+			dtrace_membar_producer();
+		} while (dtrace_casptr(&dcpu->dtdsc_dirty,
+		    dirty, NULL) != dirty);
+	}
+
+	if (!work) {
+		/*
+		 * We have no work to do; we can simply return.
+		 */
+		return;
+	}
+
+	dtrace_sync();
+
+	for (i = 0; i < NCPU; i++) {
+		dcpu = &dstate->dtds_percpu[i];
+
+		if (dcpu->dtdsc_rinsing == NULL)
+			continue;
+
+		/*
+		 * We are now guaranteed that no hash chain contains a pointer
+		 * into this dirty list; we can make it clean.
+		 */
+		ASSERT(dcpu->dtdsc_clean == NULL);
+		dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
+		dcpu->dtdsc_rinsing = NULL;
+	}
+
+	/*
+	 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
+	 * sure that all CPUs have seen all of the dtdsc_clean pointers.
+	 * This prevents a race whereby a CPU incorrectly decides that
+	 * the state should be something other than DTRACE_DSTATE_CLEAN
+	 * after dtrace_dynvar_clean() has completed.
+	 */
+	dtrace_sync();
+
+	dstate->dtds_state = DTRACE_DSTATE_CLEAN;
+}
+
+/*
+ * Depending on the value of the op parameter, this function looks-up,
+ * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
+ * allocation is requested, this function will return a pointer to a
+ * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
+ * variable can be allocated.  If NULL is returned, the appropriate counter
+ * will be incremented.
+ */
+#if defined(__APPLE__)
+static 
+#endif /* __APPLE__ */
+dtrace_dynvar_t *
+dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
+    dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op)
+{
+	uint64_t hashval = DTRACE_DYNHASH_VALID;
+	dtrace_dynhash_t *hash = dstate->dtds_hash;
+	dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
+	processorid_t me = CPU->cpu_id, cpu = me;
+	dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
+	size_t bucket, ksize;
+	size_t chunksize = dstate->dtds_chunksize;
+	uintptr_t kdata, lock, nstate;
+	uint_t i;
+
+	ASSERT(nkeys != 0);
+
+	/*
+	 * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
+	 * algorithm.  For the by-value portions, we perform the algorithm in
+	 * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
+	 * bit, and seems to have only a minute effect on distribution.  For
+	 * the by-reference data, we perform "One-at-a-time" iterating (safely)
+	 * over each referenced byte.  It's painful to do this, but it's much
+	 * better than pathological hash distribution.  The efficacy of the
+	 * hashing algorithm (and a comparison with other algorithms) may be
+	 * found by running the ::dtrace_dynstat MDB dcmd.
+	 */
+	for (i = 0; i < nkeys; i++) {
+		if (key[i].dttk_size == 0) {
+			uint64_t val = key[i].dttk_value;
+
+			hashval += (val >> 48) & 0xffff;
+			hashval += (hashval << 10);
+			hashval ^= (hashval >> 6);
+
+			hashval += (val >> 32) & 0xffff;
+			hashval += (hashval << 10);
+			hashval ^= (hashval >> 6);
+
+			hashval += (val >> 16) & 0xffff;
+			hashval += (hashval << 10);
+			hashval ^= (hashval >> 6);
+
+			hashval += val & 0xffff;
+			hashval += (hashval << 10);
+			hashval ^= (hashval >> 6);
+		} else {
+			/*
+			 * This is incredibly painful, but it beats the hell
+			 * out of the alternative.
+			 */
+			uint64_t j, size = key[i].dttk_size;
+			uintptr_t base = (uintptr_t)key[i].dttk_value;
+
+			for (j = 0; j < size; j++) {
+				hashval += dtrace_load8(base + j);
+				hashval += (hashval << 10);
+				hashval ^= (hashval >> 6);
+			}
+		}
+	}
+
+	hashval += (hashval << 3);
+	hashval ^= (hashval >> 11);
+	hashval += (hashval << 15);
+
+	/*
+	 * There is a remote chance (ideally, 1 in 2^31) that our hashval
+	 * comes out to be one of our two sentinel hash values.  If this
+	 * actually happens, we set the hashval to be a value known to be a
+	 * non-sentinel value.
+	 */
+	if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK)
+		hashval = DTRACE_DYNHASH_VALID;
+
+	/*
+	 * Yes, it's painful to do a divide here.  If the cycle count becomes
+	 * important here, tricks can be pulled to reduce it.  (However, it's
+	 * critical that hash collisions be kept to an absolute minimum;
+	 * they're much more painful than a divide.)  It's better to have a
+	 * solution that generates few collisions and still keeps things
+	 * relatively simple.
+	 */
+	bucket = hashval % dstate->dtds_hashsize;
+
+	if (op == DTRACE_DYNVAR_DEALLOC) {
+		volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
+
+		for (;;) {
+			while ((lock = *lockp) & 1)
+				continue;
+
+			if (dtrace_casptr((void *)lockp,
+			    (void *)lock, (void *)(lock + 1)) == (void *)lock)
+				break;
+		}
+
+		dtrace_membar_producer();
+	}
+
+top:
+	prev = NULL;
+	lock = hash[bucket].dtdh_lock;
+
+	dtrace_membar_consumer();
+
+	start = hash[bucket].dtdh_chain;
+	ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK ||
+	    start->dtdv_hashval != DTRACE_DYNHASH_FREE ||
+	    op != DTRACE_DYNVAR_DEALLOC));
+
+	for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
+		dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
+		dtrace_key_t *dkey = &dtuple->dtt_key[0];
+
+		if (dvar->dtdv_hashval != hashval) {
+			if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) {
+				/*
+				 * We've reached the sink, and therefore the
+				 * end of the hash chain; we can kick out of
+				 * the loop knowing that we have seen a valid
+				 * snapshot of state.
+				 */
+				ASSERT(dvar->dtdv_next == NULL);
+				ASSERT(dvar == &dtrace_dynhash_sink);
+				break;
+			}
+
+			if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) {
+				/*
+				 * We've gone off the rails:  somewhere along
+				 * the line, one of the members of this hash
+				 * chain was deleted.  Note that we could also
+				 * detect this by simply letting this loop run
+				 * to completion, as we would eventually hit
+				 * the end of the dirty list.  However, we
+				 * want to avoid running the length of the
+				 * dirty list unnecessarily (it might be quite
+				 * long), so we catch this as early as
+				 * possible by detecting the hash marker.  In
+				 * this case, we simply set dvar to NULL and
+				 * break; the conditional after the loop will
+				 * send us back to top.
+				 */
+				dvar = NULL;
+				break;
+			}
+
+			goto next;
+		}
+
+		if (dtuple->dtt_nkeys != nkeys)
+			goto next;
+
+		for (i = 0; i < nkeys; i++, dkey++) {
+			if (dkey->dttk_size != key[i].dttk_size)
+				goto next; /* size or type mismatch */
+
+			if (dkey->dttk_size != 0) {
+				if (dtrace_bcmp(
+				    (void *)(uintptr_t)key[i].dttk_value,
+				    (void *)(uintptr_t)dkey->dttk_value,
+				    dkey->dttk_size))
+					goto next;
+			} else {
+				if (dkey->dttk_value != key[i].dttk_value)
+					goto next;
+			}
+		}
+
+		if (op != DTRACE_DYNVAR_DEALLOC)
+			return (dvar);
+
+		ASSERT(dvar->dtdv_next == NULL ||
+		    dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE);
+
+		if (prev != NULL) {
+			ASSERT(hash[bucket].dtdh_chain != dvar);
+			ASSERT(start != dvar);
+			ASSERT(prev->dtdv_next == dvar);
+			prev->dtdv_next = dvar->dtdv_next;
+		} else {
+			if (dtrace_casptr(&hash[bucket].dtdh_chain,
+			    start, dvar->dtdv_next) != start) {
+				/*
+				 * We have failed to atomically swing the
+				 * hash table head pointer, presumably because
+				 * of a conflicting allocation on another CPU.
+				 * We need to reread the hash chain and try
+				 * again.
+				 */
+				goto top;
+			}
+		}
+
+		dtrace_membar_producer();
+
+		/*
+		 * Now set the hash value to indicate that it's free.
+		 */
+		ASSERT(hash[bucket].dtdh_chain != dvar);
+		dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
+
+		dtrace_membar_producer();
+
+		/*
+		 * Set the next pointer to point at the dirty list, and
+		 * atomically swing the dirty pointer to the newly freed dvar.
+		 */
+		do {
+			next = dcpu->dtdsc_dirty;
+			dvar->dtdv_next = next;
+		} while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
+
+		/*
+		 * Finally, unlock this hash bucket.
+		 */
+		ASSERT(hash[bucket].dtdh_lock == lock);
+		ASSERT(lock & 1);
+		hash[bucket].dtdh_lock++;
+
+		return (NULL);
+next:
+		prev = dvar;
+		continue;
+	}
+
+	if (dvar == NULL) {
+		/*
+		 * If dvar is NULL, it is because we went off the rails:
+		 * one of the elements that we traversed in the hash chain
+		 * was deleted while we were traversing it.  In this case,
+		 * we assert that we aren't doing a dealloc (deallocs lock
+		 * the hash bucket to prevent themselves from racing with
+		 * one another), and retry the hash chain traversal.
+		 */
+		ASSERT(op != DTRACE_DYNVAR_DEALLOC);
+		goto top;
+	}
+
+	if (op != DTRACE_DYNVAR_ALLOC) {
+		/*
+		 * If we are not to allocate a new variable, we want to
+		 * return NULL now.  Before we return, check that the value
+		 * of the lock word hasn't changed.  If it has, we may have
+		 * seen an inconsistent snapshot.
+		 */
+		if (op == DTRACE_DYNVAR_NOALLOC) {
+			if (hash[bucket].dtdh_lock != lock)
+				goto top;
+		} else {
+			ASSERT(op == DTRACE_DYNVAR_DEALLOC);
+			ASSERT(hash[bucket].dtdh_lock == lock);
+			ASSERT(lock & 1);
+			hash[bucket].dtdh_lock++;
+		}
+
+		return (NULL);
+	}
+
+	/*
+	 * We need to allocate a new dynamic variable.  The size we need is the
+	 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
+	 * size of any auxiliary key data (rounded up to 8-byte alignment) plus
+	 * the size of any referred-to data (dsize).  We then round the final
+	 * size up to the chunksize for allocation.
+	 */
+	for (ksize = 0, i = 0; i < nkeys; i++)
+		ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
+
+	/*
+	 * This should be pretty much impossible, but could happen if, say,
+	 * strange DIF specified the tuple.  Ideally, this should be an
+	 * assertion and not an error condition -- but that requires that the
+	 * chunksize calculation in dtrace_difo_chunksize() be absolutely
+	 * bullet-proof.  (That is, it must not be able to be fooled by
+	 * malicious DIF.)  Given the lack of backwards branches in DIF,
+	 * solving this would presumably not amount to solving the Halting
+	 * Problem -- but it still seems awfully hard.
+	 */
+	if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
+	    ksize + dsize > chunksize) {
+		dcpu->dtdsc_drops++;
+		return (NULL);
+	}
+
+	nstate = DTRACE_DSTATE_EMPTY;
+
+	do {
+retry:
+		free = dcpu->dtdsc_free;
+
+		if (free == NULL) {
+			dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
+			void *rval;
+
+			if (clean == NULL) {
+				/*
+				 * We're out of dynamic variable space on
+				 * this CPU.  Unless we have tried all CPUs,
+				 * we'll try to allocate from a different
+				 * CPU.
+				 */
+				switch (dstate->dtds_state) {
+				case DTRACE_DSTATE_CLEAN: {
+					void *sp = &dstate->dtds_state;
+
+					if (++cpu >= NCPU)
+						cpu = 0;
+
+					if (dcpu->dtdsc_dirty != NULL &&
+					    nstate == DTRACE_DSTATE_EMPTY)
+						nstate = DTRACE_DSTATE_DIRTY;
+
+					if (dcpu->dtdsc_rinsing != NULL)
+						nstate = DTRACE_DSTATE_RINSING;
+
+					dcpu = &dstate->dtds_percpu[cpu];
+
+					if (cpu != me)
+						goto retry;
+
+					(void) dtrace_cas32(sp,
+					    DTRACE_DSTATE_CLEAN, nstate);
+
+					/*
+					 * To increment the correct bean
+					 * counter, take another lap.
+					 */
+					goto retry;
+				}
+
+				case DTRACE_DSTATE_DIRTY:
+					dcpu->dtdsc_dirty_drops++;
+					break;
+
+				case DTRACE_DSTATE_RINSING:
+					dcpu->dtdsc_rinsing_drops++;
+					break;
+
+				case DTRACE_DSTATE_EMPTY:
+					dcpu->dtdsc_drops++;
+					break;
+				}
+
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
+				return (NULL);
+			}
+
+			/*
+			 * The clean list appears to be non-empty.  We want to
+			 * move the clean list to the free list; we start by
+			 * moving the clean pointer aside.
+			 */
+			if (dtrace_casptr(&dcpu->dtdsc_clean,
+			    clean, NULL) != clean) {
+				/*
+				 * We are in one of two situations:
+				 *
+				 *  (a)	The clean list was switched to the
+				 *	free list by another CPU.
+				 *
+				 *  (b)	The clean list was added to by the
+				 *	cleansing cyclic.
+				 *
+				 * In either of these situations, we can
+				 * just reattempt the free list allocation.
+				 */
+				goto retry;
+			}
+
+			ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE);
+
+			/*
+			 * Now we'll move the clean list to the free list.
+			 * It's impossible for this to fail:  the only way
+			 * the free list can be updated is through this
+			 * code path, and only one CPU can own the clean list.
+			 * Thus, it would only be possible for this to fail if
+			 * this code were racing with dtrace_dynvar_clean().
+			 * (That is, if dtrace_dynvar_clean() updated the clean
+			 * list, and we ended up racing to update the free
+			 * list.)  This race is prevented by the dtrace_sync()
+			 * in dtrace_dynvar_clean() -- which flushes the
+			 * owners of the clean lists out before resetting
+			 * the clean lists.
+			 */
+			rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
+			ASSERT(rval == NULL);
+			goto retry;
+		}
+
+		dvar = free;
+		new_free = dvar->dtdv_next;
+	} while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
+
+	/*
+	 * We have now allocated a new chunk.  We copy the tuple keys into the
+	 * tuple array and copy any referenced key data into the data space
+	 * following the tuple array.  As we do this, we relocate dttk_value
+	 * in the final tuple to point to the key data address in the chunk.
+	 */
+	kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
+	dvar->dtdv_data = (void *)(kdata + ksize);
+	dvar->dtdv_tuple.dtt_nkeys = nkeys;
+
+	for (i = 0; i < nkeys; i++) {
+		dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
+		size_t kesize = key[i].dttk_size;
+
+		if (kesize != 0) {
+			dtrace_bcopy(
+			    (const void *)(uintptr_t)key[i].dttk_value,
+			    (void *)kdata, kesize);
+			dkey->dttk_value = kdata;
+			kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
+		} else {
+			dkey->dttk_value = key[i].dttk_value;
+		}
+
+		dkey->dttk_size = kesize;
+	}
+
+	ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE);
+	dvar->dtdv_hashval = hashval;
+	dvar->dtdv_next = start;
+
+	if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
+		return (dvar);
+
+	/*
+	 * The cas has failed.  Either another CPU is adding an element to
+	 * this hash chain, or another CPU is deleting an element from this
+	 * hash chain.  The simplest way to deal with both of these cases
+	 * (though not necessarily the most efficient) is to free our
+	 * allocated block and tail-call ourselves.  Note that the free is
+	 * to the dirty list and _not_ to the free list.  This is to prevent
+	 * races with allocators, above.
+	 */
+	dvar->dtdv_hashval = DTRACE_DYNHASH_FREE;
+
+	dtrace_membar_producer();
+
+	do {
+		free = dcpu->dtdsc_dirty;
+		dvar->dtdv_next = free;
+	} while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
+
+	return (dtrace_dynvar(dstate, nkeys, key, dsize, op));
+}
+
+/*ARGSUSED*/
+static void
+dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
+{
+	if (nval < *oval)
+		*oval = nval;
+}
+
+/*ARGSUSED*/
+static void
+dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
+{
+	if (nval > *oval)
+		*oval = nval;
+}
+
+static void
+dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
+{
+	int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
+	int64_t val = (int64_t)nval;
+
+	if (val < 0) {
+		for (i = 0; i < zero; i++) {
+			if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
+				quanta[i] += incr;
+				return;
+			}
+		}
+	} else {
+		for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
+			if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
+				quanta[i - 1] += incr;
+				return;
+			}
+		}
+
+		quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
+		return;
+	}
+
+	ASSERT(0);
+}
+
+static void
+dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
+{
+	uint64_t arg = *lquanta++;
+	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
+	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
+	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
+	int32_t val = (int32_t)nval, level;
+
+	ASSERT(step != 0);
+	ASSERT(levels != 0);
+
+	if (val < base) {
+		/*
+		 * This is an underflow.
+		 */
+		lquanta[0] += incr;
+		return;
+	}
+
+	level = (val - base) / step;
+
+	if (level < levels) {
+		lquanta[level + 1] += incr;
+		return;
+	}
+
+	/*
+	 * This is an overflow.
+	 */
+	lquanta[levels + 1] += incr;
+}
+
+/*ARGSUSED*/
+static void
+dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
+{
+	data[0]++;
+	data[1] += nval;
+}
+
+/*ARGSUSED*/
+static void
+dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
+{
+	*oval = *oval + 1;
+}
+
+/*ARGSUSED*/
+static void
+dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
+{
+	*oval += nval;
+}
+
+/*
+ * Aggregate given the tuple in the principal data buffer, and the aggregating
+ * action denoted by the specified dtrace_aggregation_t.  The aggregation
+ * buffer is specified as the buf parameter.  This routine does not return
+ * failure; if there is no space in the aggregation buffer, the data will be
+ * dropped, and a corresponding counter incremented.
+ */
+static void
+dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
+    intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
+{
+	dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
+	uint32_t i, ndx, size, fsize;
+	uint32_t align = sizeof (uint64_t) - 1;
+	dtrace_aggbuffer_t *agb;
+	dtrace_aggkey_t *key;
+	uint32_t hashval = 0, limit, isstr;
+	caddr_t tomax, data, kdata;
+	dtrace_actkind_t action;
+	dtrace_action_t *act;
+	uintptr_t offs;
+
+	if (buf == NULL)
+		return;
+
+	if (!agg->dtag_hasarg) {
+		/*
+		 * Currently, only quantize() and lquantize() take additional
+		 * arguments, and they have the same semantics:  an increment
+		 * value that defaults to 1 when not present.  If additional
+		 * aggregating actions take arguments, the setting of the
+		 * default argument value will presumably have to become more
+		 * sophisticated...
+		 */
+		arg = 1;
+	}
+
+	action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
+	size = rec->dtrd_offset - agg->dtag_base;
+	fsize = size + rec->dtrd_size;
+
+	ASSERT(dbuf->dtb_tomax != NULL);
+	data = dbuf->dtb_tomax + offset + agg->dtag_base;
+
+	if ((tomax = buf->dtb_tomax) == NULL) {
+		dtrace_buffer_drop(buf);
+		return;
+	}
+
+	/*
+	 * The metastructure is always at the bottom of the buffer.
+	 */
+	agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
+	    sizeof (dtrace_aggbuffer_t));
+
+	if (buf->dtb_offset == 0) {
+		/*
+		 * We just kludge up approximately 1/8th of the size to be
+		 * buckets.  If this guess ends up being routinely
+		 * off-the-mark, we may need to dynamically readjust this
+		 * based on past performance.
+		 */
+		uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
+
+		if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
+		    (uintptr_t)tomax || hashsize == 0) {
+			/*
+			 * We've been given a ludicrously small buffer;
+			 * increment our drop count and leave.
+			 */
+			dtrace_buffer_drop(buf);
+			return;
+		}
+
+		/*
+		 * And now, a pathetic attempt to try to get a an odd (or
+		 * perchance, a prime) hash size for better hash distribution.
+		 */
+		if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
+			hashsize -= DTRACE_AGGHASHSIZE_SLEW;
+
+		agb->dtagb_hashsize = hashsize;
+		agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
+		    agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
+		agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
+
+		for (i = 0; i < agb->dtagb_hashsize; i++)
+			agb->dtagb_hash[i] = NULL;
+	}
+
+	ASSERT(agg->dtag_first != NULL);
+	ASSERT(agg->dtag_first->dta_intuple);
+
+	/*
+	 * Calculate the hash value based on the key.  Note that we _don't_
+	 * include the aggid in the hashing (but we will store it as part of
+	 * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
+	 * algorithm: a simple, quick algorithm that has no known funnels, and
+	 * gets good distribution in practice.  The efficacy of the hashing
+	 * algorithm (and a comparison with other algorithms) may be found by
+	 * running the ::dtrace_aggstat MDB dcmd.
+	 */
+	for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
+		i = act->dta_rec.dtrd_offset - agg->dtag_base;
+		limit = i + act->dta_rec.dtrd_size;
+		ASSERT(limit <= size);
+		isstr = DTRACEACT_ISSTRING(act);
+
+		for (; i < limit; i++) {
+			hashval += data[i];
+			hashval += (hashval << 10);
+			hashval ^= (hashval >> 6);
+
+			if (isstr && data[i] == '\0')
+				break;
+		}
+	}
+
+	hashval += (hashval << 3);
+	hashval ^= (hashval >> 11);
+	hashval += (hashval << 15);
+
+	/*
+	 * Yes, the divide here is expensive -- but it's generally the least
+	 * of the performance issues given the amount of data that we iterate
+	 * over to compute hash values, compare data, etc.
+	 */
+	ndx = hashval % agb->dtagb_hashsize;
+
+	for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
+		ASSERT((caddr_t)key >= tomax);
+		ASSERT((caddr_t)key < tomax + buf->dtb_size);
+
+		if (hashval != key->dtak_hashval || key->dtak_size != size)
+			continue;
+
+		kdata = key->dtak_data;
+		ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
+
+		for (act = agg->dtag_first; act->dta_intuple;
+		    act = act->dta_next) {
+			i = act->dta_rec.dtrd_offset - agg->dtag_base;
+			limit = i + act->dta_rec.dtrd_size;
+			ASSERT(limit <= size);
+			isstr = DTRACEACT_ISSTRING(act);
+
+			for (; i < limit; i++) {
+				if (kdata[i] != data[i])
+					goto next;
+
+				if (isstr && data[i] == '\0')
+					break;
+			}
+		}
+
+		if (action != key->dtak_action) {
+			/*
+			 * We are aggregating on the same value in the same
+			 * aggregation with two different aggregating actions.
+			 * (This should have been picked up in the compiler,
+			 * so we may be dealing with errant or devious DIF.)
+			 * This is an error condition; we indicate as much,
+			 * and return.
+			 */
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+			return;
+		}
+
+		/*
+		 * This is a hit:  we need to apply the aggregator to
+		 * the value at this key.
+		 */
+		agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
+		return;
+next:
+		continue;
+	}
+
+	/*
+	 * We didn't find it.  We need to allocate some zero-filled space,
+	 * link it into the hash table appropriately, and apply the aggregator
+	 * to the (zero-filled) value.
+	 */
+	offs = buf->dtb_offset;
+	while (offs & (align - 1))
+		offs += sizeof (uint32_t);
+
+	/*
+	 * If we don't have enough room to both allocate a new key _and_
+	 * its associated data, increment the drop count and return.
+	 */
+	if ((uintptr_t)tomax + offs + fsize >
+	    agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
+		dtrace_buffer_drop(buf);
+		return;
+	}
+
+	/*CONSTCOND*/
+	ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
+	key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
+	agb->dtagb_free -= sizeof (dtrace_aggkey_t);
+
+	key->dtak_data = kdata = tomax + offs;
+	buf->dtb_offset = offs + fsize;
+
+	/*
+	 * Now copy the data across.
+	 */
+	*((dtrace_aggid_t *)kdata) = agg->dtag_id;
+
+	for (i = sizeof (dtrace_aggid_t); i < size; i++)
+		kdata[i] = data[i];
+
+	/*
+	 * Because strings are not zeroed out by default, we need to iterate
+	 * looking for actions that store strings, and we need to explicitly
+	 * pad these strings out with zeroes.
+	 */
+	for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) {
+		int nul;
+
+		if (!DTRACEACT_ISSTRING(act))
+			continue;
+
+		i = act->dta_rec.dtrd_offset - agg->dtag_base;
+		limit = i + act->dta_rec.dtrd_size;
+		ASSERT(limit <= size);
+
+		for (nul = 0; i < limit; i++) {
+			if (nul) {
+				kdata[i] = '\0';
+				continue;
+			}
+
+			if (data[i] != '\0')
+				continue;
+
+			nul = 1;
+		}
+	}
+
+	for (i = size; i < fsize; i++)
+		kdata[i] = 0;
+
+	key->dtak_hashval = hashval;
+	key->dtak_size = size;
+	key->dtak_action = action;
+	key->dtak_next = agb->dtagb_hash[ndx];
+	agb->dtagb_hash[ndx] = key;
+
+	/*
+	 * Finally, apply the aggregator.
+	 */
+	*((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
+	agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
+}
+
+/*
+ * Given consumer state, this routine finds a speculation in the INACTIVE
+ * state and transitions it into the ACTIVE state.  If there is no speculation
+ * in the INACTIVE state, 0 is returned.  In this case, no error counter is
+ * incremented -- it is up to the caller to take appropriate action.
+ */
+static int
+dtrace_speculation(dtrace_state_t *state)
+{
+	int i = 0;
+	dtrace_speculation_state_t current;
+	uint32_t *stat = &state->dts_speculations_unavail, count;
+
+	while (i < state->dts_nspeculations) {
+		dtrace_speculation_t *spec = &state->dts_speculations[i];
+
+		current = spec->dtsp_state;
+
+		if (current != DTRACESPEC_INACTIVE) {
+			if (current == DTRACESPEC_COMMITTINGMANY ||
+			    current == DTRACESPEC_COMMITTING ||
+			    current == DTRACESPEC_DISCARDING)
+				stat = &state->dts_speculations_busy;
+			i++;
+			continue;
+		}
+
+		if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
+		    current, DTRACESPEC_ACTIVE) == current)
+			return (i + 1);
+	}
+
+	/*
+	 * We couldn't find a speculation.  If we found as much as a single
+	 * busy speculation buffer, we'll attribute this failure as "busy"
+	 * instead of "unavail".
+	 */
+	do {
+		count = *stat;
+	} while (dtrace_cas32(stat, count, count + 1) != count);
+
+	return (0);
+}
+
+/*
+ * This routine commits an active speculation.  If the specified speculation
+ * is not in a valid state to perform a commit(), this routine will silently do
+ * nothing.  The state of the specified speculation is transitioned according
+ * to the state transition diagram outlined in <sys/dtrace_impl.h>
+ */
+static void
+dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
+    dtrace_specid_t which)
+{
+	dtrace_speculation_t *spec;
+	dtrace_buffer_t *src, *dest;
+	uintptr_t daddr, saddr, dlimit;
+	dtrace_speculation_state_t current, new;
+	intptr_t offs;
+
+	if (which == 0)
+		return;
+
+	if (which > state->dts_nspeculations) {
+		cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+		return;
+	}
+
+	spec = &state->dts_speculations[which - 1];
+	src = &spec->dtsp_buffer[cpu];
+	dest = &state->dts_buffer[cpu];
+
+	do {
+		current = spec->dtsp_state;
+
+		if (current == DTRACESPEC_COMMITTINGMANY)
+			break;
+
+		switch (current) {
+		case DTRACESPEC_INACTIVE:
+		case DTRACESPEC_DISCARDING:
+			return;
+
+		case DTRACESPEC_COMMITTING:
+			/*
+			 * This is only possible if we are (a) commit()'ing
+			 * without having done a prior speculate() on this CPU
+			 * and (b) racing with another commit() on a different
+			 * CPU.  There's nothing to do -- we just assert that
+			 * our offset is 0.
+			 */
+			ASSERT(src->dtb_offset == 0);
+			return;
+
+		case DTRACESPEC_ACTIVE:
+			new = DTRACESPEC_COMMITTING;
+			break;
+
+		case DTRACESPEC_ACTIVEONE:
+			/*
+			 * This speculation is active on one CPU.  If our
+			 * buffer offset is non-zero, we know that the one CPU
+			 * must be us.  Otherwise, we are committing on a
+			 * different CPU from the speculate(), and we must
+			 * rely on being asynchronously cleaned.
+			 */
+			if (src->dtb_offset != 0) {
+				new = DTRACESPEC_COMMITTING;
+				break;
+			}
+			/*FALLTHROUGH*/
+
+		case DTRACESPEC_ACTIVEMANY:
+			new = DTRACESPEC_COMMITTINGMANY;
+			break;
+
+		default:
+			ASSERT(0);
+		}
+	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
+	    current, new) != current);
+
+	/*
+	 * We have set the state to indicate that we are committing this
+	 * speculation.  Now reserve the necessary space in the destination
+	 * buffer.
+	 */
+	if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
+	    sizeof (uint64_t), state, NULL)) < 0) {
+		dtrace_buffer_drop(dest);
+		goto out;
+	}
+
+	/*
+	 * We have the space; copy the buffer across.  (Note that this is a
+	 * highly subobtimal bcopy(); in the unlikely event that this becomes
+	 * a serious performance issue, a high-performance DTrace-specific
+	 * bcopy() should obviously be invented.)
+	 */
+	daddr = (uintptr_t)dest->dtb_tomax + offs;
+	dlimit = daddr + src->dtb_offset;
+	saddr = (uintptr_t)src->dtb_tomax;
+
+	/*
+	 * First, the aligned portion.
+	 */
+	while (dlimit - daddr >= sizeof (uint64_t)) {
+		*((uint64_t *)daddr) = *((uint64_t *)saddr);
+
+		daddr += sizeof (uint64_t);
+		saddr += sizeof (uint64_t);
+	}
+
+	/*
+	 * Now any left-over bit...
+	 */
+	while (dlimit - daddr)
+		*((uint8_t *)daddr++) = *((uint8_t *)saddr++);
+
+	/*
+	 * Finally, commit the reserved space in the destination buffer.
+	 */
+	dest->dtb_offset = offs + src->dtb_offset;
+
+out:
+	/*
+	 * If we're lucky enough to be the only active CPU on this speculation
+	 * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
+	 */
+	if (current == DTRACESPEC_ACTIVE ||
+	    (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
+		uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
+		    DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
+
+		ASSERT(rval == DTRACESPEC_COMMITTING);
+	}
+
+	src->dtb_offset = 0;
+	src->dtb_xamot_drops += src->dtb_drops;
+	src->dtb_drops = 0;
+}
+
+/*
+ * This routine discards an active speculation.  If the specified speculation
+ * is not in a valid state to perform a discard(), this routine will silently
+ * do nothing.  The state of the specified speculation is transitioned
+ * according to the state transition diagram outlined in <sys/dtrace_impl.h>
+ */
+static void
+dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
+    dtrace_specid_t which)
+{
+	dtrace_speculation_t *spec;
+	dtrace_speculation_state_t current, new;
+	dtrace_buffer_t *buf;
+
+	if (which == 0)
+		return;
+
+	if (which > state->dts_nspeculations) {
+		cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+		return;
+	}
+
+	spec = &state->dts_speculations[which - 1];
+	buf = &spec->dtsp_buffer[cpu];
+
+	do {
+		current = spec->dtsp_state;
+
+		switch (current) {
+		case DTRACESPEC_INACTIVE:
+		case DTRACESPEC_COMMITTINGMANY:
+		case DTRACESPEC_COMMITTING:
+		case DTRACESPEC_DISCARDING:
+			return;
+
+		case DTRACESPEC_ACTIVE:
+		case DTRACESPEC_ACTIVEMANY:
+			new = DTRACESPEC_DISCARDING;
+			break;
+
+		case DTRACESPEC_ACTIVEONE:
+			if (buf->dtb_offset != 0) {
+				new = DTRACESPEC_INACTIVE;
+			} else {
+				new = DTRACESPEC_DISCARDING;
+			}
+			break;
+
+		default:
+			ASSERT(0);
+		}
+	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
+	    current, new) != current);
+
+	buf->dtb_offset = 0;
+	buf->dtb_drops = 0;
+}
+
+/*
+ * Note:  not called from probe context.  This function is called
+ * asynchronously from cross call context to clean any speculations that are
+ * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
+ * transitioned back to the INACTIVE state until all CPUs have cleaned the
+ * speculation.
+ */
+static void
+dtrace_speculation_clean_here(dtrace_state_t *state)
+{
+	dtrace_icookie_t cookie;
+	processorid_t cpu = CPU->cpu_id;
+	dtrace_buffer_t *dest = &state->dts_buffer[cpu];
+	dtrace_specid_t i;
+
+	cookie = dtrace_interrupt_disable();
+
+	if (dest->dtb_tomax == NULL) {
+		dtrace_interrupt_enable(cookie);
+		return;
+	}
+
+	for (i = 0; i < state->dts_nspeculations; i++) {
+		dtrace_speculation_t *spec = &state->dts_speculations[i];
+		dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
+
+		if (src->dtb_tomax == NULL)
+			continue;
+
+		if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
+			src->dtb_offset = 0;
+			continue;
+		}
+
+		if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
+			continue;
+
+		if (src->dtb_offset == 0)
+			continue;
+
+		dtrace_speculation_commit(state, cpu, i + 1);
+	}
+
+	dtrace_interrupt_enable(cookie);
+}
+
+/*
+ * Note:  not called from probe context.  This function is called
+ * asynchronously (and at a regular interval) to clean any speculations that
+ * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
+ * is work to be done, it cross calls all CPUs to perform that work;
+ * COMMITMANY and DISCARDING speculations may not be transitioned back to the
+ * INACTIVE state until they have been cleaned by all CPUs.
+ */
+static void
+dtrace_speculation_clean(dtrace_state_t *state)
+{
+	int work = 0, rv;
+	dtrace_specid_t i;
+
+	for (i = 0; i < state->dts_nspeculations; i++) {
+		dtrace_speculation_t *spec = &state->dts_speculations[i];
+
+		ASSERT(!spec->dtsp_cleaning);
+
+		if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
+		    spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
+			continue;
+
+		work++;
+		spec->dtsp_cleaning = 1;
+	}
+
+	if (!work)
+		return;
+
+	dtrace_xcall(DTRACE_CPUALL,
+	    (dtrace_xcall_t)dtrace_speculation_clean_here, state);
+
+	/*
+	 * We now know that all CPUs have committed or discarded their
+	 * speculation buffers, as appropriate.  We can now set the state
+	 * to inactive.
+	 */
+	for (i = 0; i < state->dts_nspeculations; i++) {
+		dtrace_speculation_t *spec = &state->dts_speculations[i];
+		dtrace_speculation_state_t current, new;
+
+		if (!spec->dtsp_cleaning)
+			continue;
+
+		current = spec->dtsp_state;
+		ASSERT(current == DTRACESPEC_DISCARDING ||
+		    current == DTRACESPEC_COMMITTINGMANY);
+
+		new = DTRACESPEC_INACTIVE;
+
+		rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
+		ASSERT(rv == current);
+		spec->dtsp_cleaning = 0;
+	}
+}
+
+/*
+ * Called as part of a speculate() to get the speculative buffer associated
+ * with a given speculation.  Returns NULL if the specified speculation is not
+ * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
+ * the active CPU is not the specified CPU -- the speculation will be
+ * atomically transitioned into the ACTIVEMANY state.
+ */
+static dtrace_buffer_t *
+dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
+    dtrace_specid_t which)
+{
+	dtrace_speculation_t *spec;
+	dtrace_speculation_state_t current, new;
+	dtrace_buffer_t *buf;
+
+	if (which == 0)
+		return (NULL);
+
+	if (which > state->dts_nspeculations) {
+		cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
+		return (NULL);
+	}
+
+	spec = &state->dts_speculations[which - 1];
+	buf = &spec->dtsp_buffer[cpuid];
+
+	do {
+		current = spec->dtsp_state;
+
+		switch (current) {
+		case DTRACESPEC_INACTIVE:
+		case DTRACESPEC_COMMITTINGMANY:
+		case DTRACESPEC_DISCARDING:
+			return (NULL);
+
+		case DTRACESPEC_COMMITTING:
+			ASSERT(buf->dtb_offset == 0);
+			return (NULL);
+
+		case DTRACESPEC_ACTIVEONE:
+			/*
+			 * This speculation is currently active on one CPU.
+			 * Check the offset in the buffer; if it's non-zero,
+			 * that CPU must be us (and we leave the state alone).
+			 * If it's zero, assume that we're starting on a new
+			 * CPU -- and change the state to indicate that the
+			 * speculation is active on more than one CPU.
+			 */
+			if (buf->dtb_offset != 0)
+				return (buf);
+
+			new = DTRACESPEC_ACTIVEMANY;
+			break;
+
+		case DTRACESPEC_ACTIVEMANY:
+			return (buf);
+
+		case DTRACESPEC_ACTIVE:
+			new = DTRACESPEC_ACTIVEONE;
+			break;
+
+		default:
+			ASSERT(0);
+		}
+	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
+	    current, new) != current);
+
+	ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
+	return (buf);
+}
+
+/*
+ * This function implements the DIF emulator's variable lookups.  The emulator
+ * passes a reserved variable identifier and optional built-in array index.
+ */
+static uint64_t
+dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
+    uint64_t ndx)
+{
+	/*
+	 * If we're accessing one of the uncached arguments, we'll turn this
+	 * into a reference in the args array.
+	 */
+	if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
+		ndx = v - DIF_VAR_ARG0;
+		v = DIF_VAR_ARGS;
+	}
+
+	switch (v) {
+	case DIF_VAR_ARGS:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
+		if (ndx >= sizeof (mstate->dtms_arg) /
+		    sizeof (mstate->dtms_arg[0])) {
+#if !defined(__APPLE__)
+			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
+#else
+			/* Account for introduction of __dtrace_probe() on xnu. */
+			int aframes = mstate->dtms_probe->dtpr_aframes + 3;
+#endif /* __APPLE__ */
+			dtrace_provider_t *pv;
+			uint64_t val;
+
+			pv = mstate->dtms_probe->dtpr_provider;
+			if (pv->dtpv_pops.dtps_getargval != NULL)
+				val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
+				    mstate->dtms_probe->dtpr_id,
+				    mstate->dtms_probe->dtpr_arg, ndx, aframes);
+#if defined(__APPLE__)
+			/* Special case access of arg5 as passed to dtrace_probeid_error (which see.) */
+			else if (mstate->dtms_probe->dtpr_id == dtrace_probeid_error && ndx == 5) {
+				return ((dtrace_state_t *)(mstate->dtms_arg[0]))->dts_arg_error_illval;
+			}
+#endif /* __APPLE__ */
+			else
+				val = dtrace_getarg(ndx, aframes);
+
+			/*
+			 * This is regrettably required to keep the compiler
+			 * from tail-optimizing the call to dtrace_getarg().
+			 * The condition always evaluates to true, but the
+			 * compiler has no way of figuring that out a priori.
+			 * (None of this would be necessary if the compiler
+			 * could be relied upon to _always_ tail-optimize
+			 * the call to dtrace_getarg() -- but it can't.)
+			 */
+			if (mstate->dtms_probe != NULL)
+				return (val);
+
+			ASSERT(0);
+		}
+
+		return (mstate->dtms_arg[ndx]);
+
+#if !defined(__APPLE__)
+	case DIF_VAR_UREGS: {
+		klwp_t *lwp;
+
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		if ((lwp = curthread->t_lwp) == NULL) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
+			return (0);
+		}
+
+		return (dtrace_getreg(lwp->lwp_regs, ndx));
+	}
+#else
+	case DIF_VAR_UREGS: {
+		thread_t thread;
+
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		if ((thread = current_thread()) == NULL) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = 0;
+			return (0);
+		}
+
+		return (dtrace_getreg(find_user_regs(thread), ndx));
+	}
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_CURTHREAD:
+		if (!dtrace_priv_kernel(state))
+			return (0);
+		return ((uint64_t)(uintptr_t)curthread);
+#else
+	case DIF_VAR_CURTHREAD:
+		if (!dtrace_priv_kernel(state))
+			return (0);
+
+		return ((uint64_t)(uintptr_t)current_thread());
+#endif /* __APPLE__ */
+
+	case DIF_VAR_TIMESTAMP:
+		if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
+			mstate->dtms_timestamp = dtrace_gethrtime();
+			mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
+		}
+		return (mstate->dtms_timestamp);
+
+#if !defined(__APPLE__)
+	case DIF_VAR_VTIMESTAMP:
+		ASSERT(dtrace_vtime_references != 0);
+		return (curthread->t_dtrace_vtime);
+#else
+	case DIF_VAR_VTIMESTAMP:
+		ASSERT(dtrace_vtime_references != 0);
+		return (dtrace_get_thread_vtime(current_thread()));
+#endif /* __APPLE__ */
+
+	case DIF_VAR_WALLTIMESTAMP:
+		if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
+			mstate->dtms_walltimestamp = dtrace_gethrestime();
+			mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
+		}
+		return (mstate->dtms_walltimestamp);
+
+	case DIF_VAR_IPL:
+		if (!dtrace_priv_kernel(state))
+			return (0);
+		if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
+			mstate->dtms_ipl = dtrace_getipl();
+			mstate->dtms_present |= DTRACE_MSTATE_IPL;
+		}
+		return (mstate->dtms_ipl);
+
+	case DIF_VAR_EPID:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
+		return (mstate->dtms_epid);
+
+	case DIF_VAR_ID:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
+		return (mstate->dtms_probe->dtpr_id);
+
+	case DIF_VAR_STACKDEPTH:
+		if (!dtrace_priv_kernel(state))
+			return (0);
+		if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
+#if !defined(__APPLE__)
+			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
+#else
+			/* Account for introduction of __dtrace_probe() on xnu. */
+			int aframes = mstate->dtms_probe->dtpr_aframes + 3;
+#endif /* __APPLE__ */
+
+			mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
+			mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
+		}
+		return (mstate->dtms_stackdepth);
+
+	case DIF_VAR_USTACKDEPTH:
+		if (!dtrace_priv_proc(state))
+			return (0);
+		if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
+			/*
+			 * See comment in DIF_VAR_PID.
+			 */
+			if (DTRACE_ANCHORED(mstate->dtms_probe) &&
+			    CPU_ON_INTR(CPU)) {
+				mstate->dtms_ustackdepth = 0;
+			} else {
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+				mstate->dtms_ustackdepth =
+				    dtrace_getustackdepth();
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+			}
+			mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
+		}
+		return (mstate->dtms_ustackdepth);
+
+	case DIF_VAR_CALLER:
+		if (!dtrace_priv_kernel(state))
+			return (0);
+		if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
+#if !defined(__APPLE__)
+			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
+#else
+			/* Account for introduction of __dtrace_probe() on xnu. */
+			int aframes = mstate->dtms_probe->dtpr_aframes + 3;
+#endif /* __APPLE__ */
+
+			if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
+				/*
+				 * If this is an unanchored probe, we are
+				 * required to go through the slow path:
+				 * dtrace_caller() only guarantees correct
+				 * results for anchored probes.
+				 */
+				pc_t caller[2];
+
+				dtrace_getpcstack(caller, 2, aframes,
+				    (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
+				mstate->dtms_caller = caller[1];
+			} else if ((mstate->dtms_caller =
+			    dtrace_caller(aframes)) == -1) {
+				/*
+				 * We have failed to do this the quick way;
+				 * we must resort to the slower approach of
+				 * calling dtrace_getpcstack().
+				 */
+				pc_t caller;
+
+				dtrace_getpcstack(&caller, 1, aframes, NULL);
+				mstate->dtms_caller = caller;
+			}
+
+			mstate->dtms_present |= DTRACE_MSTATE_CALLER;
+		}
+		return (mstate->dtms_caller);
+
+	case DIF_VAR_UCALLER:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
+			uint64_t ustack[3];
+
+			/*
+			 * dtrace_getupcstack() fills in the first uint64_t
+			 * with the current PID.  The second uint64_t will
+			 * be the program counter at user-level.  The third
+			 * uint64_t will contain the caller, which is what
+			 * we're after.
+			 */
+			ustack[2] = NULL;
+			dtrace_getupcstack(ustack, 3);
+			mstate->dtms_ucaller = ustack[2];
+			mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
+		}
+
+		return (mstate->dtms_ucaller);
+
+	case DIF_VAR_PROBEPROV:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
+		return ((uint64_t)(uintptr_t)
+		    mstate->dtms_probe->dtpr_provider->dtpv_name);
+
+	case DIF_VAR_PROBEMOD:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
+		return ((uint64_t)(uintptr_t)
+		    mstate->dtms_probe->dtpr_mod);
+
+	case DIF_VAR_PROBEFUNC:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
+		return ((uint64_t)(uintptr_t)
+		    mstate->dtms_probe->dtpr_func);
+
+	case DIF_VAR_PROBENAME:
+		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
+		return ((uint64_t)(uintptr_t)
+		    mstate->dtms_probe->dtpr_name);
+
+#if !defined(__APPLE__)
+	case DIF_VAR_PID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * Note that we are assuming that an unanchored probe is
+		 * always due to a high-level interrupt.  (And we're assuming
+		 * that there is only a single high level interrupt.)
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (pid0.pid_id);
+
+		/*
+		 * It is always safe to dereference one's own t_procp pointer:
+		 * it always points to a valid, allocated proc structure.
+		 * Further, it is always safe to dereference the p_pidp member
+		 * of one's own proc structure.  (These are truisms becuase
+		 * threads and processes don't clean up their own state --
+		 * they leave that task to whomever reaps them.)
+		 */
+		return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
+
+#else
+	case DIF_VAR_PID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * Note that we are assuming that an unanchored probe is
+		 * always due to a high-level interrupt.  (And we're assuming
+		 * that there is only a single high level interrupt.)
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			/* Anchored probe that fires while on an interrupt accrues to process 0 */
+			return 0; 
+
+		return ((uint64_t)proc_selfpid());
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_PPID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (pid0.pid_id);
+
+		return ((uint64_t)curthread->t_procp->p_ppid);
+#else
+	case DIF_VAR_PPID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		return ((uint64_t)(uintptr_t)(current_proc()->p_ppid));
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_TID:
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		return ((uint64_t)curthread->t_tid);
+#else
+	case DIF_VAR_TID:
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		return ((uint64_t)(uintptr_t)current_thread()); /* Is user's (pthread_t)t->kernel_thread */
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_EXECNAME:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
+
+		/*
+		 * It is always safe to dereference one's own t_procp pointer:
+		 * it always points to a valid, allocated proc structure.
+		 * (This is true because threads don't clean up their own
+		 * state -- they leave that task to whomever reaps them.)
+		 */
+		return ((uint64_t)(uintptr_t)
+		    curthread->t_procp->p_user.u_comm);
+#else
+	case DIF_VAR_EXECNAME:
+	{
+		char *xname = (char *)mstate->dtms_scratch_ptr;
+		size_t scratch_size = MAXCOMLEN+1;
+		
+		/* The scratch allocation's lifetime is that of the clause. */
+		if (mstate->dtms_scratch_ptr + scratch_size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size)
+			return 0;
+			
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		mstate->dtms_scratch_ptr += scratch_size;
+		proc_selfname( xname, MAXCOMLEN );
+
+		return ((uint64_t)(uintptr_t)xname);
+	}
+#endif /* __APPLE__ */
+#if !defined(__APPLE__)
+	case DIF_VAR_ZONENAME:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
+
+		/*
+		 * It is always safe to dereference one's own t_procp pointer:
+		 * it always points to a valid, allocated proc structure.
+		 * (This is true because threads don't clean up their own
+		 * state -- they leave that task to whomever reaps them.)
+		 */
+		return ((uint64_t)(uintptr_t)
+		    curthread->t_procp->p_zone->zone_name);
+
+#else
+	case DIF_VAR_ZONENAME:
+		if (!dtrace_priv_proc(state))
+			return (0);
+		
+		return ((uint64_t)(uintptr_t)NULL); /* Darwin doesn't do "zones" */
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_UID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return ((uint64_t)p0.p_cred->cr_uid);
+
+		return ((uint64_t)curthread->t_cred->cr_uid);
+#else
+	case DIF_VAR_UID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		if (dtrace_CRED() != NULL)
+			return ((uint64_t)kauth_getuid());
+		else
+			return -1LL;
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_GID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return ((uint64_t)p0.p_cred->cr_gid);
+
+		return ((uint64_t)curthread->t_cred->cr_gid);
+#else
+	case DIF_VAR_GID:
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		if (dtrace_CRED() != NULL)
+			return ((uint64_t)kauth_getgid());
+		else
+			return -1LL;
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_VAR_ERRNO: {
+		klwp_t *lwp;
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		if ((lwp = curthread->t_lwp) == NULL)
+			return (0);
+
+		return ((uint64_t)lwp->lwp_errno);
+	}
+#else
+	case DIF_VAR_ERRNO: {
+		uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+		if (!dtrace_priv_proc(state))
+			return (0);
+
+		/*
+		 * See comment in DIF_VAR_PID.
+		 */
+		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
+			return (0);
+
+		return (uthread ? uthread->t_dtrace_errno : -1);
+	}
+#endif /* __APPLE__ */
+
+	default:
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+		return (0);
+	}
+}
+
+/*
+ * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
+ * Notice that we don't bother validating the proper number of arguments or
+ * their types in the tuple stack.  This isn't needed because all argument
+ * interpretation is safe because of our load safety -- the worst that can
+ * happen is that a bogus program can obtain bogus results.
+ */
+static void
+dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
+    dtrace_key_t *tupregs, int nargs,
+    dtrace_mstate_t *mstate, dtrace_state_t *state)
+{
+	volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+#if !defined(__APPLE__)
+	volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
+#else
+	volatile uint64_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	union {
+		mutex_impl_t mi;
+		uint64_t mx;
+	} m;
+
+	union {
+		krwlock_t ri;
+		uintptr_t rw;
+	} r;
+#else
+/* XXX awaits lock/mutex work */
+#endif /* __APPLE__ */
+
+	switch (subr) {
+	case DIF_SUBR_RAND:
+		regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
+		break;
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_MUTEX_OWNED:
+		m.mx = dtrace_load64(tupregs[0].dttk_value);
+		if (MUTEX_TYPE_ADAPTIVE(&m.mi))
+			regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
+		else
+			regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
+		break;
+
+	case DIF_SUBR_MUTEX_OWNER:
+		m.mx = dtrace_load64(tupregs[0].dttk_value);
+		if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
+		    MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
+			regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
+		else
+			regs[rd] = 0;
+		break;
+
+	case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
+		m.mx = dtrace_load64(tupregs[0].dttk_value);
+		regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
+		break;
+
+	case DIF_SUBR_MUTEX_TYPE_SPIN:
+		m.mx = dtrace_load64(tupregs[0].dttk_value);
+		regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
+		break;
+
+	case DIF_SUBR_RW_READ_HELD: {
+		uintptr_t tmp;
+
+		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
+		regs[rd] = _RW_READ_HELD(&r.ri, tmp);
+		break;
+	}
+
+	case DIF_SUBR_RW_WRITE_HELD:
+		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
+		regs[rd] = _RW_WRITE_HELD(&r.ri);
+		break;
+
+	case DIF_SUBR_RW_ISWRITER:
+		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
+		regs[rd] = _RW_ISWRITER(&r.ri);
+		break;
+#else
+/* XXX awaits lock/mutex work */
+#endif /* __APPLE__ */
+
+	case DIF_SUBR_BCOPY: {
+		/*
+		 * We need to be sure that the destination is in the scratch
+		 * region -- no other region is allowed.
+		 */
+		uintptr_t src = tupregs[0].dttk_value;
+		uintptr_t dest = tupregs[1].dttk_value;
+		size_t size = tupregs[2].dttk_value;
+
+		if (!dtrace_inscratch(dest, size, mstate)) {
+			*flags |= CPU_DTRACE_BADADDR;
+			*illval = regs[rd];
+			break;
+		}
+
+		dtrace_bcopy((void *)src, (void *)dest, size);
+		break;
+	}
+
+	case DIF_SUBR_ALLOCA:
+	case DIF_SUBR_COPYIN: {
+		uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
+		uint64_t size =
+		    tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
+		size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
+
+		/*
+		 * This action doesn't require any credential checks since
+		 * probes will not activate in user contexts to which the
+		 * enabling user does not have permissions.
+		 */
+		if (mstate->dtms_scratch_ptr + scratch_size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		if (subr == DIF_SUBR_COPYIN) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			dtrace_copyin(tupregs[0].dttk_value, dest, size);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		}
+
+		mstate->dtms_scratch_ptr += scratch_size;
+		regs[rd] = dest;
+		break;
+	}
+
+	case DIF_SUBR_COPYINTO: {
+		uint64_t size = tupregs[1].dttk_value;
+		uintptr_t dest = tupregs[2].dttk_value;
+
+		/*
+		 * This action doesn't require any credential checks since
+		 * probes will not activate in user contexts to which the
+		 * enabling user does not have permissions.
+		 */
+		if (!dtrace_inscratch(dest, size, mstate)) {
+			*flags |= CPU_DTRACE_BADADDR;
+			*illval = regs[rd];
+			break;
+		}
+
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		dtrace_copyin(tupregs[0].dttk_value, dest, size);
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		break;
+	}
+
+	case DIF_SUBR_COPYINSTR: {
+		uintptr_t dest = mstate->dtms_scratch_ptr;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+
+		if (nargs > 1 && tupregs[1].dttk_value < size)
+			size = tupregs[1].dttk_value + 1;
+
+		/*
+		 * This action doesn't require any credential checks since
+		 * probes will not activate in user contexts to which the
+		 * enabling user does not have permissions.
+		 */
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		dtrace_copyinstr(tupregs[0].dttk_value, dest, size);
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+		((char *)dest)[size - 1] = '\0';
+		mstate->dtms_scratch_ptr += size;
+		regs[rd] = dest;
+		break;
+	}
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_MSGSIZE:
+	case DIF_SUBR_MSGDSIZE: {
+		uintptr_t baddr = tupregs[0].dttk_value, daddr;
+		uintptr_t wptr, rptr;
+		size_t count = 0;
+		int cont = 0;
+
+		while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
+			wptr = dtrace_loadptr(baddr +
+			    offsetof(mblk_t, b_wptr));
+
+			rptr = dtrace_loadptr(baddr +
+			    offsetof(mblk_t, b_rptr));
+
+			if (wptr < rptr) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = tupregs[0].dttk_value;
+				break;
+			}
+
+			daddr = dtrace_loadptr(baddr +
+			    offsetof(mblk_t, b_datap));
+
+			baddr = dtrace_loadptr(baddr +
+			    offsetof(mblk_t, b_cont));
+
+			/*
+			 * We want to prevent against denial-of-service here,
+			 * so we're only going to search the list for
+			 * dtrace_msgdsize_max mblks.
+			 */
+			if (cont++ > dtrace_msgdsize_max) {
+				*flags |= CPU_DTRACE_ILLOP;
+				break;
+			}
+
+			if (subr == DIF_SUBR_MSGDSIZE) {
+				if (dtrace_load8(daddr +
+				    offsetof(dblk_t, db_type)) != M_DATA)
+					continue;
+			}
+
+			count += wptr - rptr;
+		}
+
+		if (!(*flags & CPU_DTRACE_FAULT))
+			regs[rd] = count;
+
+		break;
+	}
+#else
+	case DIF_SUBR_MSGSIZE:
+	case DIF_SUBR_MSGDSIZE: {
+		/* Darwin does not implement SysV streams messages */
+		regs[rd] = 0;
+		break;
+	}
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_PROGENYOF: {
+		pid_t pid = tupregs[0].dttk_value;
+		proc_t *p;
+		int rval = 0;
+
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+
+		for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
+			if (p->p_pidp->pid_id == pid) {
+				rval = 1;
+				break;
+			}
+		}
+
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+		regs[rd] = rval;
+		break;
+	}
+#else
+	case DIF_SUBR_PROGENYOF: {
+		pid_t pid = tupregs[0].dttk_value;
+		struct proc *p = current_proc();
+		int rval = 0, lim = nprocs;
+
+		while(p && (lim-- > 0)) {
+			pid_t ppid;
+
+			ppid = (pid_t)dtrace_load32((uintptr_t)&(p->p_pid));
+			if (*flags & CPU_DTRACE_FAULT)
+				break;
+
+			if (ppid == pid) {
+				rval = 1;
+				break;
+			}
+
+			if (ppid == 0)
+				break; /* Can't climb process tree any further. */
+
+			p = (struct proc *)dtrace_loadptr((uintptr_t)&(p->p_pptr));
+			if (*flags & CPU_DTRACE_FAULT)
+				break;
+		}
+
+		regs[rd] = rval;
+		break;
+	}
+#endif /* __APPLE__ */
+
+	case DIF_SUBR_SPECULATION:
+		regs[rd] = dtrace_speculation(state);
+		break;
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_COPYOUT: {
+		uintptr_t kaddr = tupregs[0].dttk_value;
+		uintptr_t uaddr = tupregs[1].dttk_value;
+		uint64_t size = tupregs[2].dttk_value;
+
+		if (!dtrace_destructive_disallow &&
+		    dtrace_priv_proc_control(state) &&
+		    !dtrace_istoxic(kaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			dtrace_copyout(kaddr, uaddr, size);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		}
+		break;
+	}
+
+	case DIF_SUBR_COPYOUTSTR: {
+		uintptr_t kaddr = tupregs[0].dttk_value;
+		uintptr_t uaddr = tupregs[1].dttk_value;
+		uint64_t size = tupregs[2].dttk_value;
+
+		if (!dtrace_destructive_disallow &&
+		    dtrace_priv_proc_control(state) &&
+		    !dtrace_istoxic(kaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			dtrace_copyoutstr(kaddr, uaddr, size);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		}
+		break;
+	}
+#else
+	case DIF_SUBR_COPYOUT: {
+		uintptr_t kaddr = tupregs[0].dttk_value;
+		user_addr_t uaddr = tupregs[1].dttk_value;
+		uint64_t size = tupregs[2].dttk_value;
+
+		if (!dtrace_destructive_disallow &&
+		    dtrace_priv_proc_control(state) &&
+		    !dtrace_istoxic(kaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			dtrace_copyout(kaddr, uaddr, size);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		}
+		break;
+	}
+
+	case DIF_SUBR_COPYOUTSTR: {
+		uintptr_t kaddr = tupregs[0].dttk_value;
+		user_addr_t uaddr = tupregs[1].dttk_value;
+		uint64_t size = tupregs[2].dttk_value;
+
+		if (!dtrace_destructive_disallow &&
+		    dtrace_priv_proc_control(state) &&
+		    !dtrace_istoxic(kaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			dtrace_copyoutstr(kaddr, uaddr, size);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+		}
+		break;
+	}
+#endif /* __APPLE__ */
+
+	case DIF_SUBR_STRLEN:
+		regs[rd] = dtrace_strlen((char *)(uintptr_t)
+		    tupregs[0].dttk_value,
+		    state->dts_options[DTRACEOPT_STRSIZE]);
+		break;
+
+	case DIF_SUBR_STRCHR:
+	case DIF_SUBR_STRRCHR: {
+		/*
+		 * We're going to iterate over the string looking for the
+		 * specified character.  We will iterate until we have reached
+		 * the string length or we have found the character.  If this
+		 * is DIF_SUBR_STRRCHR, we will look for the last occurrence
+		 * of the specified character instead of the first.
+		 */
+		uintptr_t addr = tupregs[0].dttk_value;
+		uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
+		char c, target = (char)tupregs[1].dttk_value;
+
+		for (regs[rd] = NULL; addr < limit; addr++) {
+			if ((c = dtrace_load8(addr)) == target) {
+				regs[rd] = addr;
+
+				if (subr == DIF_SUBR_STRCHR)
+					break;
+			}
+
+			if (c == '\0')
+				break;
+		}
+
+		break;
+	}
+
+	case DIF_SUBR_STRSTR:
+	case DIF_SUBR_INDEX:
+	case DIF_SUBR_RINDEX: {
+		/*
+		 * We're going to iterate over the string looking for the
+		 * specified string.  We will iterate until we have reached
+		 * the string length or we have found the string.  (Yes, this
+		 * is done in the most naive way possible -- but considering
+		 * that the string we're searching for is likely to be
+		 * relatively short, the complexity of Rabin-Karp or similar
+		 * hardly seems merited.)
+		 */
+		char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
+		char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		size_t len = dtrace_strlen(addr, size);
+		size_t sublen = dtrace_strlen(substr, size);
+		char *limit = addr + len, *orig = addr;
+		int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
+		int inc = 1;
+
+		regs[rd] = notfound;
+
+		/*
+		 * strstr() and index()/rindex() have similar semantics if
+		 * both strings are the empty string: strstr() returns a
+		 * pointer to the (empty) string, and index() and rindex()
+		 * both return index 0 (regardless of any position argument).
+		 */
+		if (sublen == 0 && len == 0) {
+			if (subr == DIF_SUBR_STRSTR)
+				regs[rd] = (uintptr_t)addr;
+			else
+				regs[rd] = 0;
+			break;
+		}
+
+		if (subr != DIF_SUBR_STRSTR) {
+			if (subr == DIF_SUBR_RINDEX) {
+				limit = orig - 1;
+				addr += len;
+				inc = -1;
+			}
+
+			/*
+			 * Both index() and rindex() take an optional position
+			 * argument that denotes the starting position.
+			 */
+			if (nargs == 3) {
+				int64_t pos = (int64_t)tupregs[2].dttk_value;
+
+				/*
+				 * If the position argument to index() is
+				 * negative, Perl implicitly clamps it at
+				 * zero.  This semantic is a little surprising
+				 * given the special meaning of negative
+				 * positions to similar Perl functions like
+				 * substr(), but it appears to reflect a
+				 * notion that index() can start from a
+				 * negative index and increment its way up to
+				 * the string.  Given this notion, Perl's
+				 * rindex() is at least self-consistent in
+				 * that it implicitly clamps positions greater
+				 * than the string length to be the string
+				 * length.  Where Perl completely loses
+				 * coherence, however, is when the specified
+				 * substring is the empty string ("").  In
+				 * this case, even if the position is
+				 * negative, rindex() returns 0 -- and even if
+				 * the position is greater than the length,
+				 * index() returns the string length.  These
+				 * semantics violate the notion that index()
+				 * should never return a value less than the
+				 * specified position and that rindex() should
+				 * never return a value greater than the
+				 * specified position.  (One assumes that
+				 * these semantics are artifacts of Perl's
+				 * implementation and not the results of
+				 * deliberate design -- it beggars belief that
+				 * even Larry Wall could desire such oddness.)
+				 * While in the abstract one would wish for
+				 * consistent position semantics across
+				 * substr(), index() and rindex() -- or at the
+				 * very least self-consistent position
+				 * semantics for index() and rindex() -- we
+				 * instead opt to keep with the extant Perl
+				 * semantics, in all their broken glory.  (Do
+				 * we have more desire to maintain Perl's
+				 * semantics than Perl does?  Probably.)
+				 */
+				if (subr == DIF_SUBR_RINDEX) {
+					if (pos < 0) {
+						if (sublen == 0)
+							regs[rd] = 0;
+						break;
+					}
+
+					if (pos > len)
+						pos = len;
+				} else {
+					if (pos < 0)
+						pos = 0;
+
+					if (pos >= len) {
+						if (sublen == 0)
+							regs[rd] = len;
+						break;
+					}
+				}
+
+				addr = orig + pos;
+			}
+		}
+
+		for (regs[rd] = notfound; addr != limit; addr += inc) {
+			if (dtrace_strncmp(addr, substr, sublen) == 0) {
+				if (subr != DIF_SUBR_STRSTR) {
+					/*
+					 * As D index() and rindex() are
+					 * modeled on Perl (and not on awk),
+					 * we return a zero-based (and not a
+					 * one-based) index.  (For you Perl
+					 * weenies: no, we're not going to add
+					 * $[ -- and shouldn't you be at a con
+					 * or something?)
+					 */
+					regs[rd] = (uintptr_t)(addr - orig);
+					break;
+				}
+
+				ASSERT(subr == DIF_SUBR_STRSTR);
+				regs[rd] = (uintptr_t)addr;
+				break;
+			}
+		}
+
+		break;
+	}
+
+	case DIF_SUBR_STRTOK: {
+		uintptr_t addr = tupregs[0].dttk_value;
+		uintptr_t tokaddr = tupregs[1].dttk_value;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		uintptr_t limit, toklimit = tokaddr + size;
+		uint8_t c, tokmap[32];	 /* 256 / 8 */
+		char *dest = (char *)mstate->dtms_scratch_ptr;
+		int i;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		if (addr == NULL) {
+			/*
+			 * If the address specified is NULL, we use our saved
+			 * strtok pointer from the mstate.  Note that this
+			 * means that the saved strtok pointer is _only_
+			 * valid within multiple enablings of the same probe --
+			 * it behaves like an implicit clause-local variable.
+			 */
+			addr = mstate->dtms_strtok;
+		}
+
+		/*
+		 * First, zero the token map, and then process the token
+		 * string -- setting a bit in the map for every character
+		 * found in the token string.
+		 */
+		for (i = 0; i < sizeof (tokmap); i++)
+			tokmap[i] = 0;
+
+		for (; tokaddr < toklimit; tokaddr++) {
+			if ((c = dtrace_load8(tokaddr)) == '\0')
+				break;
+
+			ASSERT((c >> 3) < sizeof (tokmap));
+			tokmap[c >> 3] |= (1 << (c & 0x7));
+		}
+
+		for (limit = addr + size; addr < limit; addr++) {
+			/*
+			 * We're looking for a character that is _not_ contained
+			 * in the token string.
+			 */
+			if ((c = dtrace_load8(addr)) == '\0')
+				break;
+
+			if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
+				break;
+		}
+
+		if (c == '\0') {
+			/*
+			 * We reached the end of the string without finding
+			 * any character that was not in the token string.
+			 * We return NULL in this case, and we set the saved
+			 * address to NULL as well.
+			 */
+			regs[rd] = NULL;
+			mstate->dtms_strtok = NULL;
+			break;
+		}
+
+		/*
+		 * From here on, we're copying into the destination string.
+		 */
+		for (i = 0; addr < limit && i < size - 1; addr++) {
+			if ((c = dtrace_load8(addr)) == '\0')
+				break;
+
+			if (tokmap[c >> 3] & (1 << (c & 0x7)))
+				break;
+
+			ASSERT(i < size);
+			dest[i++] = c;
+		}
+
+		ASSERT(i < size);
+		dest[i] = '\0';
+		regs[rd] = (uintptr_t)dest;
+		mstate->dtms_scratch_ptr += size;
+		mstate->dtms_strtok = addr;
+		break;
+	}
+
+	case DIF_SUBR_SUBSTR: {
+		uintptr_t s = tupregs[0].dttk_value;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		char *d = (char *)mstate->dtms_scratch_ptr;
+		int64_t index = (int64_t)tupregs[1].dttk_value;
+		int64_t remaining = (int64_t)tupregs[2].dttk_value;
+		size_t len = dtrace_strlen((char *)s, size);
+		int64_t i = 0;
+
+		if (nargs <= 2)
+			remaining = (int64_t)size;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		if (index < 0) {
+			index += len;
+
+			if (index < 0 && index + remaining > 0) {
+				remaining += index;
+				index = 0;
+			}
+		}
+
+		if (index >= len || index < 0)
+			index = len;
+
+		for (d[0] = '\0'; remaining > 0; remaining--) {
+			if ((d[i++] = dtrace_load8(s++ + index)) == '\0')
+				break;
+
+			if (i == size) {
+				d[i - 1] = '\0';
+				break;
+			}
+		}
+
+		mstate->dtms_scratch_ptr += size;
+		regs[rd] = (uintptr_t)d;
+		break;
+	}
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_GETMAJOR:
+#ifdef __LP64__
+		regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
+#else
+		regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
+#endif
+		break;
+
+#else  /* __APPLE__ */
+	case DIF_SUBR_GETMAJOR:
+		regs[rd] = (uintptr_t)major( (dev_t)tupregs[0].dttk_value );
+		break;
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_GETMINOR:
+#ifdef __LP64__
+		regs[rd] = tupregs[0].dttk_value & MAXMIN64;
+#else
+		regs[rd] = tupregs[0].dttk_value & MAXMIN;
+#endif
+		break;
+
+#else  /* __APPLE__ */
+	case DIF_SUBR_GETMINOR:
+		regs[rd] = (uintptr_t)minor( (dev_t)tupregs[0].dttk_value );
+		break;
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+	case DIF_SUBR_DDI_PATHNAME: {
+		/*
+		 * This one is a galactic mess.  We are going to roughly
+		 * emulate ddi_pathname(), but it's made more complicated
+		 * by the fact that we (a) want to include the minor name and
+		 * (b) must proceed iteratively instead of recursively.
+		 */
+		uintptr_t dest = mstate->dtms_scratch_ptr;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		char *start = (char *)dest, *end = start + size - 1;
+		uintptr_t daddr = tupregs[0].dttk_value;
+		int64_t minor = (int64_t)tupregs[1].dttk_value;
+		char *s;
+		int i, len, depth = 0;
+
+		if (size == 0 || mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		*end = '\0';
+
+		/*
+		 * We want to have a name for the minor.  In order to do this,
+		 * we need to walk the minor list from the devinfo.  We want
+		 * to be sure that we don't infinitely walk a circular list,
+		 * so we check for circularity by sending a scout pointer
+		 * ahead two elements for every element that we iterate over;
+		 * if the list is circular, these will ultimately point to the
+		 * same element.  You may recognize this little trick as the
+		 * answer to a stupid interview question -- one that always
+		 * seems to be asked by those who had to have it laboriously
+		 * explained to them, and who can't even concisely describe
+		 * the conditions under which one would be forced to resort to
+		 * this technique.  Needless to say, those conditions are
+		 * found here -- and probably only here.  Is this is the only
+		 * use of this infamous trick in shipping, production code?
+		 * If it isn't, it probably should be...
+		 */
+		if (minor != -1) {
+			uintptr_t maddr = dtrace_loadptr(daddr +
+			    offsetof(struct dev_info, devi_minor));
+
+			uintptr_t next = offsetof(struct ddi_minor_data, next);
+			uintptr_t name = offsetof(struct ddi_minor_data,
+			    d_minor) + offsetof(struct ddi_minor, name);
+			uintptr_t dev = offsetof(struct ddi_minor_data,
+			    d_minor) + offsetof(struct ddi_minor, dev);
+			uintptr_t scout;
+
+			if (maddr != NULL)
+				scout = dtrace_loadptr(maddr + next);
+
+			while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
+				uint64_t m;
+#ifdef __LP64__
+				m = dtrace_load64(maddr + dev) & MAXMIN64;
+#else
+				m = dtrace_load32(maddr + dev) & MAXMIN;
+#endif
+				if (m != minor) {
+					maddr = dtrace_loadptr(maddr + next);
+
+					if (scout == NULL)
+						continue;
+
+					scout = dtrace_loadptr(scout + next);
+
+					if (scout == NULL)
+						continue;
+
+					scout = dtrace_loadptr(scout + next);
+
+					if (scout == NULL)
+						continue;
+
+					if (scout == maddr) {
+						*flags |= CPU_DTRACE_ILLOP;
+						break;
+					}
+
+					continue;
+				}
+
+				/*
+				 * We have the minor data.  Now we need to
+				 * copy the minor's name into the end of the
+				 * pathname.
+				 */
+				s = (char *)dtrace_loadptr(maddr + name);
+				len = dtrace_strlen(s, size);
+
+				if (*flags & CPU_DTRACE_FAULT)
+					break;
+
+				if (len != 0) {
+					if ((end -= (len + 1)) < start)
+						break;
+
+					*end = ':';
+				}
+
+				for (i = 1; i <= len; i++)
+					end[i] = dtrace_load8((uintptr_t)s++);
+				break;
+			}
+		}
+
+		while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
+			ddi_node_state_t devi_state;
+
+			devi_state = dtrace_load32(daddr +
+			    offsetof(struct dev_info, devi_node_state));
+
+			if (*flags & CPU_DTRACE_FAULT)
+				break;
+
+			if (devi_state >= DS_INITIALIZED) {
+				s = (char *)dtrace_loadptr(daddr +
+				    offsetof(struct dev_info, devi_addr));
+				len = dtrace_strlen(s, size);
+
+				if (*flags & CPU_DTRACE_FAULT)
+					break;
+
+				if (len != 0) {
+					if ((end -= (len + 1)) < start)
+						break;
+
+					*end = '@';
+				}
+
+				for (i = 1; i <= len; i++)
+					end[i] = dtrace_load8((uintptr_t)s++);
+			}
+
+			/*
+			 * Now for the node name...
+			 */
+			s = (char *)dtrace_loadptr(daddr +
+			    offsetof(struct dev_info, devi_node_name));
+
+			daddr = dtrace_loadptr(daddr +
+			    offsetof(struct dev_info, devi_parent));
+
+			/*
+			 * If our parent is NULL (that is, if we're the root
+			 * node), we're going to use the special path
+			 * "devices".
+			 */
+			if (daddr == NULL)
+				s = "devices";
+
+			len = dtrace_strlen(s, size);
+			if (*flags & CPU_DTRACE_FAULT)
+				break;
+
+			if ((end -= (len + 1)) < start)
+				break;
+
+			for (i = 1; i <= len; i++)
+				end[i] = dtrace_load8((uintptr_t)s++);
+			*end = '/';
+
+			if (depth++ > dtrace_devdepth_max) {
+				*flags |= CPU_DTRACE_ILLOP;
+				break;
+			}
+		}
+
+		if (end < start)
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+
+		if (daddr == NULL) {
+			regs[rd] = (uintptr_t)end;
+			mstate->dtms_scratch_ptr += size;
+		}
+
+		break;
+	}
+#else
+	case DIF_SUBR_DDI_PATHNAME: {
+		/* XXX awaits galactic disentanglement ;-} */
+		regs[rd] = NULL;
+		break;
+	}
+#endif /* __APPLE__ */
+
+	case DIF_SUBR_STRJOIN: {
+		char *d = (char *)mstate->dtms_scratch_ptr;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		uintptr_t s1 = tupregs[0].dttk_value;
+		uintptr_t s2 = tupregs[1].dttk_value;
+		int i = 0;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		for (;;) {
+			if (i >= size) {
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+				regs[rd] = NULL;
+				break;
+			}
+
+			if ((d[i++] = dtrace_load8(s1++)) == '\0') {
+				i--;
+				break;
+			}
+		}
+
+		for (;;) {
+			if (i >= size) {
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+				regs[rd] = NULL;
+				break;
+			}
+
+			if ((d[i++] = dtrace_load8(s2++)) == '\0')
+				break;
+		}
+
+		if (i < size) {
+			mstate->dtms_scratch_ptr += i;
+			regs[rd] = (uintptr_t)d;
+		}
+
+		break;
+	}
+
+	case DIF_SUBR_LLTOSTR: {
+		int64_t i = (int64_t)tupregs[0].dttk_value;
+		int64_t val = i < 0 ? i * -1 : i;
+		uint64_t size = 22;	/* enough room for 2^64 in decimal */
+		char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		for (*end-- = '\0'; val; val /= 10)
+			*end-- = '0' + (val % 10);
+
+		if (i == 0)
+			*end-- = '0';
+
+		if (i < 0)
+			*end-- = '-';
+
+		regs[rd] = (uintptr_t)end + 1;
+		mstate->dtms_scratch_ptr += size;
+		break;
+	}
+
+	case DIF_SUBR_DIRNAME:
+	case DIF_SUBR_BASENAME: {
+		char *dest = (char *)mstate->dtms_scratch_ptr;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		uintptr_t src = tupregs[0].dttk_value;
+		int i, j, len = dtrace_strlen((char *)src, size);
+		int lastbase = -1, firstbase = -1, lastdir = -1;
+		int start, end;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		/*
+		 * The basename and dirname for a zero-length string is
+		 * defined to be "."
+		 */
+		if (len == 0) {
+			len = 1;
+			src = (uintptr_t)".";
+		}
+
+		/*
+		 * Start from the back of the string, moving back toward the
+		 * front until we see a character that isn't a slash.  That
+		 * character is the last character in the basename.
+		 */
+		for (i = len - 1; i >= 0; i--) {
+			if (dtrace_load8(src + i) != '/')
+				break;
+		}
+
+		if (i >= 0)
+			lastbase = i;
+
+		/*
+		 * Starting from the last character in the basename, move
+		 * towards the front until we find a slash.  The character
+		 * that we processed immediately before that is the first
+		 * character in the basename.
+		 */
+		for (; i >= 0; i--) {
+			if (dtrace_load8(src + i) == '/')
+				break;
+		}
+
+		if (i >= 0)
+			firstbase = i + 1;
+
+		/*
+		 * Now keep going until we find a non-slash character.  That
+		 * character is the last character in the dirname.
+		 */
+		for (; i >= 0; i--) {
+			if (dtrace_load8(src + i) != '/')
+				break;
+		}
+
+		if (i >= 0)
+			lastdir = i;
+
+		ASSERT(!(lastbase == -1 && firstbase != -1));
+		ASSERT(!(firstbase == -1 && lastdir != -1));
+
+		if (lastbase == -1) {
+			/*
+			 * We didn't find a non-slash character.  We know that
+			 * the length is non-zero, so the whole string must be
+			 * slashes.  In either the dirname or the basename
+			 * case, we return '/'.
+			 */
+			ASSERT(firstbase == -1);
+			firstbase = lastbase = lastdir = 0;
+		}
+
+		if (firstbase == -1) {
+			/*
+			 * The entire string consists only of a basename
+			 * component.  If we're looking for dirname, we need
+			 * to change our string to be just "."; if we're
+			 * looking for a basename, we'll just set the first
+			 * character of the basename to be 0.
+			 */
+			if (subr == DIF_SUBR_DIRNAME) {
+				ASSERT(lastdir == -1);
+				src = (uintptr_t)".";
+				lastdir = 0;
+			} else {
+				firstbase = 0;
+			}
+		}
+
+		if (subr == DIF_SUBR_DIRNAME) {
+			if (lastdir == -1) {
+				/*
+				 * We know that we have a slash in the name --
+				 * or lastdir would be set to 0, above.  And
+				 * because lastdir is -1, we know that this
+				 * slash must be the first character.  (That
+				 * is, the full string must be of the form
+				 * "/basename".)  In this case, the last
+				 * character of the directory name is 0.
+				 */
+				lastdir = 0;
+			}
+
+			start = 0;
+			end = lastdir;
+		} else {
+			ASSERT(subr == DIF_SUBR_BASENAME);
+			ASSERT(firstbase != -1 && lastbase != -1);
+			start = firstbase;
+			end = lastbase;
+		}
+
+		for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
+			dest[j] = dtrace_load8(src + i);
+
+		dest[j] = '\0';
+		regs[rd] = (uintptr_t)dest;
+		mstate->dtms_scratch_ptr += size;
+		break;
+	}
+
+	case DIF_SUBR_CLEANPATH: {
+		char *dest = (char *)mstate->dtms_scratch_ptr, c;
+		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
+		uintptr_t src = tupregs[0].dttk_value;
+		int i = 0, j = 0;
+
+		if (mstate->dtms_scratch_ptr + size >
+		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+			regs[rd] = NULL;
+			break;
+		}
+
+		/*
+		 * Move forward, loading each character.
+		 */
+		do {
+			c = dtrace_load8(src + i++);
+next:
+			if (j + 5 >= size)	/* 5 = strlen("/..c\0") */
+				break;
+
+			if (c != '/') {
+				dest[j++] = c;
+				continue;
+			}
+
+			c = dtrace_load8(src + i++);
+
+			if (c == '/') {
+				/*
+				 * We have two slashes -- we can just advance
+				 * to the next character.
+				 */
+				goto next;
+			}
+
+			if (c != '.') {
+				/*
+				 * This is not "." and it's not ".." -- we can
+				 * just store the "/" and this character and
+				 * drive on.
+				 */
+				dest[j++] = '/';
+				dest[j++] = c;
+				continue;
+			}
+
+			c = dtrace_load8(src + i++);
+
+			if (c == '/') {
+				/*
+				 * This is a "/./" component.  We're not going
+				 * to store anything in the destination buffer;
+				 * we're just going to go to the next component.
+				 */
+				goto next;
+			}
+
+			if (c != '.') {
+				/*
+				 * This is not ".." -- we can just store the
+				 * "/." and this character and continue
+				 * processing.
+				 */
+				dest[j++] = '/';
+				dest[j++] = '.';
+				dest[j++] = c;
+				continue;
+			}
+
+			c = dtrace_load8(src + i++);
+
+			if (c != '/' && c != '\0') {
+				/*
+				 * This is not ".." -- it's "..[mumble]".
+				 * We'll store the "/.." and this character
+				 * and continue processing.
+				 */
+				dest[j++] = '/';
+				dest[j++] = '.';
+				dest[j++] = '.';
+				dest[j++] = c;
+				continue;
+			}
+
+			/*
+			 * This is "/../" or "/..\0".  We need to back up
+			 * our destination pointer until we find a "/".
+			 */
+			i--;
+			while (j != 0 && dest[--j] != '/')
+				continue;
+
+			if (c == '\0')
+				dest[++j] = '/';
+		} while (c != '\0');
+
+		dest[j] = '\0';
+		regs[rd] = (uintptr_t)dest;
+		mstate->dtms_scratch_ptr += size;
+		break;
+	}
+#ifdef __APPLE__
+
+	/* CHUD callback ('chud(uint64_t, [uint64_t], [uint64_t] ...)') */
+	case DIF_SUBR_CHUD: {
+		uint64_t selector = tupregs[0].dttk_value;
+		uint64_t args[DIF_DTR_NREGS-1] = {0ULL};
+		uint32_t ii;
+
+		/* copy in any variadic argument list */
+		for(ii = 0; ii < DIF_DTR_NREGS-1; ii++) {
+			args[ii] = tupregs[ii+1].dttk_value;
+		}
+
+		kern_return_t ret = 
+			chudxnu_dtrace_callback(selector, args, DIF_DTR_NREGS-1);
+		if(KERN_SUCCESS != ret) {
+			/* error */
+		}
+		break;
+	}
+
+#endif /* __APPLE__ */
+
+	}
+}
+
+/*
+ * Emulate the execution of DTrace IR instructions specified by the given
+ * DIF object.  This function is deliberately void of assertions as all of
+ * the necessary checks are handled by a call to dtrace_difo_validate().
+ */
+static uint64_t
+dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
+    dtrace_vstate_t *vstate, dtrace_state_t *state)
+{
+	const dif_instr_t *text = difo->dtdo_buf;
+	const uint_t textlen = difo->dtdo_len;
+	const char *strtab = difo->dtdo_strtab;
+	const uint64_t *inttab = difo->dtdo_inttab;
+
+	uint64_t rval = 0;
+	dtrace_statvar_t *svar;
+	dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
+	dtrace_difv_t *v;
+	volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+#if !defined(__APPLE__)
+	volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
+#else
+	volatile uint64_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
+#endif /* __APPLE__ */
+
+	dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
+	uint64_t regs[DIF_DIR_NREGS];
+	uint64_t *tmp;
+
+	uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
+	int64_t cc_r;
+	uint_t pc = 0, id, opc;
+	uint8_t ttop = 0;
+	dif_instr_t instr;
+	uint_t r1, r2, rd;
+
+	regs[DIF_REG_R0] = 0; 		/* %r0 is fixed at zero */
+
+	while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
+		opc = pc;
+
+		instr = text[pc++];
+		r1 = DIF_INSTR_R1(instr);
+		r2 = DIF_INSTR_R2(instr);
+		rd = DIF_INSTR_RD(instr);
+
+		switch (DIF_INSTR_OP(instr)) {
+		case DIF_OP_OR:
+			regs[rd] = regs[r1] | regs[r2];
+			break;
+		case DIF_OP_XOR:
+			regs[rd] = regs[r1] ^ regs[r2];
+			break;
+		case DIF_OP_AND:
+			regs[rd] = regs[r1] & regs[r2];
+			break;
+		case DIF_OP_SLL:
+			regs[rd] = regs[r1] << regs[r2];
+			break;
+		case DIF_OP_SRL:
+			regs[rd] = regs[r1] >> regs[r2];
+			break;
+		case DIF_OP_SUB:
+			regs[rd] = regs[r1] - regs[r2];
+			break;
+		case DIF_OP_ADD:
+			regs[rd] = regs[r1] + regs[r2];
+			break;
+		case DIF_OP_MUL:
+			regs[rd] = regs[r1] * regs[r2];
+			break;
+		case DIF_OP_SDIV:
+			if (regs[r2] == 0) {
+				regs[rd] = 0;
+				*flags |= CPU_DTRACE_DIVZERO;
+			} else {
+				regs[rd] = (int64_t)regs[r1] /
+				    (int64_t)regs[r2];
+			}
+			break;
+
+		case DIF_OP_UDIV:
+			if (regs[r2] == 0) {
+				regs[rd] = 0;
+				*flags |= CPU_DTRACE_DIVZERO;
+			} else {
+				regs[rd] = regs[r1] / regs[r2];
+			}
+			break;
+
+		case DIF_OP_SREM:
+			if (regs[r2] == 0) {
+				regs[rd] = 0;
+				*flags |= CPU_DTRACE_DIVZERO;
+			} else {
+				regs[rd] = (int64_t)regs[r1] %
+				    (int64_t)regs[r2];
+			}
+			break;
+
+		case DIF_OP_UREM:
+			if (regs[r2] == 0) {
+				regs[rd] = 0;
+				*flags |= CPU_DTRACE_DIVZERO;
+			} else {
+				regs[rd] = regs[r1] % regs[r2];
+			}
+			break;
+
+		case DIF_OP_NOT:
+			regs[rd] = ~regs[r1];
+			break;
+		case DIF_OP_MOV:
+			regs[rd] = regs[r1];
+			break;
+		case DIF_OP_CMP:
+			cc_r = regs[r1] - regs[r2];
+			cc_n = cc_r < 0;
+			cc_z = cc_r == 0;
+			cc_v = 0;
+			cc_c = regs[r1] < regs[r2];
+			break;
+		case DIF_OP_TST:
+			cc_n = cc_v = cc_c = 0;
+			cc_z = regs[r1] == 0;
+			break;
+		case DIF_OP_BA:
+			pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BE:
+			if (cc_z)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BNE:
+			if (cc_z == 0)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BG:
+			if ((cc_z | (cc_n ^ cc_v)) == 0)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BGU:
+			if ((cc_c | cc_z) == 0)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BGE:
+			if ((cc_n ^ cc_v) == 0)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BGEU:
+			if (cc_c == 0)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BL:
+			if (cc_n ^ cc_v)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BLU:
+			if (cc_c)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BLE:
+			if (cc_z | (cc_n ^ cc_v))
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_BLEU:
+			if (cc_c | cc_z)
+				pc = DIF_INSTR_LABEL(instr);
+			break;
+		case DIF_OP_RLDSB:
+			if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDSB:
+			regs[rd] = (int8_t)dtrace_load8(regs[r1]);
+			break;
+		case DIF_OP_RLDSH:
+			if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDSH:
+			regs[rd] = (int16_t)dtrace_load16(regs[r1]);
+			break;
+		case DIF_OP_RLDSW:
+			if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDSW:
+			regs[rd] = (int32_t)dtrace_load32(regs[r1]);
+			break;
+		case DIF_OP_RLDUB:
+			if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDUB:
+			regs[rd] = dtrace_load8(regs[r1]);
+			break;
+		case DIF_OP_RLDUH:
+			if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDUH:
+			regs[rd] = dtrace_load16(regs[r1]);
+			break;
+		case DIF_OP_RLDUW:
+			if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDUW:
+			regs[rd] = dtrace_load32(regs[r1]);
+			break;
+		case DIF_OP_RLDX:
+			if (!dtrace_canstore(regs[r1], 8, mstate, vstate)) {
+				*flags |= CPU_DTRACE_KPRIV;
+				*illval = regs[r1];
+				break;
+			}
+			/*FALLTHROUGH*/
+		case DIF_OP_LDX:
+			regs[rd] = dtrace_load64(regs[r1]);
+			break;
+		case DIF_OP_ULDSB:
+			regs[rd] = (int8_t)
+			    dtrace_fuword8(regs[r1]);
+			break;
+		case DIF_OP_ULDSH:
+			regs[rd] = (int16_t)
+			    dtrace_fuword16(regs[r1]);
+			break;
+		case DIF_OP_ULDSW:
+			regs[rd] = (int32_t)
+			    dtrace_fuword32(regs[r1]);
+			break;
+		case DIF_OP_ULDUB:
+			regs[rd] =
+			    dtrace_fuword8(regs[r1]);
+			break;
+		case DIF_OP_ULDUH:
+			regs[rd] =
+			    dtrace_fuword16(regs[r1]);
+			break;
+		case DIF_OP_ULDUW:
+			regs[rd] =
+			    dtrace_fuword32(regs[r1]);
+			break;
+		case DIF_OP_ULDX:
+			regs[rd] =
+			    dtrace_fuword64(regs[r1]);
+			break;
+		case DIF_OP_RET:
+			rval = regs[rd];
+			break;
+		case DIF_OP_NOP:
+			break;
+		case DIF_OP_SETX:
+			regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
+			break;
+		case DIF_OP_SETS:
+			regs[rd] = (uint64_t)(uintptr_t)
+			    (strtab + DIF_INSTR_STRING(instr));
+			break;
+		case DIF_OP_SCMP:
+			cc_r = dtrace_strncmp((char *)(uintptr_t)regs[r1],
+			    (char *)(uintptr_t)regs[r2],
+			    state->dts_options[DTRACEOPT_STRSIZE]);
+
+			cc_n = cc_r < 0;
+			cc_z = cc_r == 0;
+			cc_v = cc_c = 0;
+			break;
+		case DIF_OP_LDGA:
+			regs[rd] = dtrace_dif_variable(mstate, state,
+			    r1, regs[r2]);
+			break;
+		case DIF_OP_LDGS:
+			id = DIF_INSTR_VAR(instr);
+
+			if (id >= DIF_VAR_OTHER_UBASE) {
+				uintptr_t a;
+
+				id -= DIF_VAR_OTHER_UBASE;
+				svar = vstate->dtvs_globals[id];
+				ASSERT(svar != NULL);
+				v = &svar->dtsv_var;
+
+				if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
+					regs[rd] = svar->dtsv_data;
+					break;
+				}
+
+				a = (uintptr_t)svar->dtsv_data;
+
+				if (*(uint8_t *)a == UINT8_MAX) {
+					/*
+					 * If the 0th byte is set to UINT8_MAX
+					 * then this is to be treated as a
+					 * reference to a NULL variable.
+					 */
+					regs[rd] = NULL;
+				} else {
+					regs[rd] = a + sizeof (uint64_t);
+				}
+
+				break;
+			}
+
+			regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
+			break;
+
+		case DIF_OP_STGS:
+			id = DIF_INSTR_VAR(instr);
+
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+
+			svar = vstate->dtvs_globals[id];
+			ASSERT(svar != NULL);
+			v = &svar->dtsv_var;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				uintptr_t a = (uintptr_t)svar->dtsv_data;
+
+				ASSERT(a != NULL);
+				ASSERT(svar->dtsv_size != 0);
+
+				if (regs[rd] == NULL) {
+					*(uint8_t *)a = UINT8_MAX;
+					break;
+				} else {
+					*(uint8_t *)a = 0;
+					a += sizeof (uint64_t);
+				}
+
+				dtrace_vcopy((void *)(uintptr_t)regs[rd],
+				    (void *)a, &v->dtdv_type);
+				break;
+			}
+
+			svar->dtsv_data = regs[rd];
+			break;
+
+		case DIF_OP_LDTA:
+			/*
+			 * There are no DTrace built-in thread-local arrays at
+			 * present.  This opcode is saved for future work.
+			 */
+			*flags |= CPU_DTRACE_ILLOP;
+			regs[rd] = 0;
+			break;
+
+		case DIF_OP_LDLS:
+			id = DIF_INSTR_VAR(instr);
+
+			if (id < DIF_VAR_OTHER_UBASE) {
+				/*
+				 * For now, this has no meaning.
+				 */
+				regs[rd] = 0;
+				break;
+			}
+
+			id -= DIF_VAR_OTHER_UBASE;
+
+			ASSERT(id < vstate->dtvs_nlocals);
+			ASSERT(vstate->dtvs_locals != NULL);
+
+			svar = vstate->dtvs_locals[id];
+			ASSERT(svar != NULL);
+			v = &svar->dtsv_var;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				uintptr_t a = (uintptr_t)svar->dtsv_data;
+				size_t sz = v->dtdv_type.dtdt_size;
+
+				sz += sizeof (uint64_t);
+				ASSERT(svar->dtsv_size == NCPU * sz);
+				a += CPU->cpu_id * sz;
+
+				if (*(uint8_t *)a == UINT8_MAX) {
+					/*
+					 * If the 0th byte is set to UINT8_MAX
+					 * then this is to be treated as a
+					 * reference to a NULL variable.
+					 */
+					regs[rd] = NULL;
+				} else {
+					regs[rd] = a + sizeof (uint64_t);
+				}
+
+				break;
+			}
+
+			ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
+			tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
+			regs[rd] = tmp[CPU->cpu_id];
+			break;
+
+		case DIF_OP_STLS:
+			id = DIF_INSTR_VAR(instr);
+
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+			ASSERT(id < vstate->dtvs_nlocals);
+
+			ASSERT(vstate->dtvs_locals != NULL);
+			svar = vstate->dtvs_locals[id];
+			ASSERT(svar != NULL);
+			v = &svar->dtsv_var;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				uintptr_t a = (uintptr_t)svar->dtsv_data;
+				size_t sz = v->dtdv_type.dtdt_size;
+
+				sz += sizeof (uint64_t);
+				ASSERT(svar->dtsv_size == NCPU * sz);
+				a += CPU->cpu_id * sz;
+
+				if (regs[rd] == NULL) {
+					*(uint8_t *)a = UINT8_MAX;
+					break;
+				} else {
+					*(uint8_t *)a = 0;
+					a += sizeof (uint64_t);
+				}
+
+				dtrace_vcopy((void *)(uintptr_t)regs[rd],
+				    (void *)a, &v->dtdv_type);
+				break;
+			}
+
+			ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
+			tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
+			tmp[CPU->cpu_id] = regs[rd];
+			break;
+
+		case DIF_OP_LDTS: {
+			dtrace_dynvar_t *dvar;
+			dtrace_key_t *key;
+
+			id = DIF_INSTR_VAR(instr);
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+			v = &vstate->dtvs_tlocals[id];
+
+			key = &tupregs[DIF_DTR_NREGS];
+			key[0].dttk_value = (uint64_t)id;
+			key[0].dttk_size = 0;
+			DTRACE_TLS_THRKEY(key[1].dttk_value);
+			key[1].dttk_size = 0;
+
+			dvar = dtrace_dynvar(dstate, 2, key,
+			    sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC);
+
+			if (dvar == NULL) {
+				regs[rd] = 0;
+				break;
+			}
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
+			} else {
+				regs[rd] = *((uint64_t *)dvar->dtdv_data);
+			}
+
+			break;
+		}
+
+		case DIF_OP_STTS: {
+			dtrace_dynvar_t *dvar;
+			dtrace_key_t *key;
+
+			id = DIF_INSTR_VAR(instr);
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+
+			key = &tupregs[DIF_DTR_NREGS];
+			key[0].dttk_value = (uint64_t)id;
+			key[0].dttk_size = 0;
+			DTRACE_TLS_THRKEY(key[1].dttk_value);
+			key[1].dttk_size = 0;
+			v = &vstate->dtvs_tlocals[id];
+
+			dvar = dtrace_dynvar(dstate, 2, key,
+			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
+			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
+			    regs[rd] ? DTRACE_DYNVAR_ALLOC :
+			    DTRACE_DYNVAR_DEALLOC);
+
+			/*
+			 * Given that we're storing to thread-local data,
+			 * we need to flush our predicate cache.
+			 */
+#if !defined(__APPLE__)
+			curthread->t_predcache = NULL;
+#else
+			dtrace_set_thread_predcache(current_thread(), 0);
+#endif /* __APPLE__ */
+
+
+			if (dvar == NULL)
+				break;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				dtrace_vcopy((void *)(uintptr_t)regs[rd],
+				    dvar->dtdv_data, &v->dtdv_type);
+			} else {
+				*((uint64_t *)dvar->dtdv_data) = regs[rd];
+			}
+
+			break;
+		}
+
+		case DIF_OP_SRA:
+			regs[rd] = (int64_t)regs[r1] >> regs[r2];
+			break;
+
+		case DIF_OP_CALL:
+			dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
+			    regs, tupregs, ttop, mstate, state);
+			break;
+
+		case DIF_OP_PUSHTR:
+			if (ttop == DIF_DTR_NREGS) {
+				*flags |= CPU_DTRACE_TUPOFLOW;
+				break;
+			}
+
+			if (r1 == DIF_TYPE_STRING) {
+				/*
+				 * If this is a string type and the size is 0,
+				 * we'll use the system-wide default string
+				 * size.  Note that we are _not_ looking at
+				 * the value of the DTRACEOPT_STRSIZE option;
+				 * had this been set, we would expect to have
+				 * a non-zero size value in the "pushtr".
+				 */
+				tupregs[ttop].dttk_size =
+				    dtrace_strlen((char *)(uintptr_t)regs[rd],
+				    regs[r2] ? regs[r2] :
+				    dtrace_strsize_default) + 1;
+			} else {
+				tupregs[ttop].dttk_size = regs[r2];
+			}
+
+			tupregs[ttop++].dttk_value = regs[rd];
+			break;
+
+		case DIF_OP_PUSHTV:
+			if (ttop == DIF_DTR_NREGS) {
+				*flags |= CPU_DTRACE_TUPOFLOW;
+				break;
+			}
+
+			tupregs[ttop].dttk_value = regs[rd];
+			tupregs[ttop++].dttk_size = 0;
+			break;
+
+		case DIF_OP_POPTS:
+			if (ttop != 0)
+				ttop--;
+			break;
+
+		case DIF_OP_FLUSHTS:
+			ttop = 0;
+			break;
+
+		case DIF_OP_LDGAA:
+		case DIF_OP_LDTAA: {
+			dtrace_dynvar_t *dvar;
+			dtrace_key_t *key = tupregs;
+			uint_t nkeys = ttop;
+
+			id = DIF_INSTR_VAR(instr);
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+
+			key[nkeys].dttk_value = (uint64_t)id;
+			key[nkeys++].dttk_size = 0;
+
+			if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
+				DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
+				key[nkeys++].dttk_size = 0;
+				v = &vstate->dtvs_tlocals[id];
+			} else {
+				v = &vstate->dtvs_globals[id]->dtsv_var;
+			}
+
+			dvar = dtrace_dynvar(dstate, nkeys, key,
+			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
+			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
+			    DTRACE_DYNVAR_NOALLOC);
+
+			if (dvar == NULL) {
+				regs[rd] = 0;
+				break;
+			}
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
+			} else {
+				regs[rd] = *((uint64_t *)dvar->dtdv_data);
+			}
+
+			break;
+		}
+
+		case DIF_OP_STGAA:
+		case DIF_OP_STTAA: {
+			dtrace_dynvar_t *dvar;
+			dtrace_key_t *key = tupregs;
+			uint_t nkeys = ttop;
+
+			id = DIF_INSTR_VAR(instr);
+			ASSERT(id >= DIF_VAR_OTHER_UBASE);
+			id -= DIF_VAR_OTHER_UBASE;
+
+			key[nkeys].dttk_value = (uint64_t)id;
+			key[nkeys++].dttk_size = 0;
+
+			if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
+				DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
+				key[nkeys++].dttk_size = 0;
+				v = &vstate->dtvs_tlocals[id];
+			} else {
+				v = &vstate->dtvs_globals[id]->dtsv_var;
+			}
+
+			dvar = dtrace_dynvar(dstate, nkeys, key,
+			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
+			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
+			    regs[rd] ? DTRACE_DYNVAR_ALLOC :
+			    DTRACE_DYNVAR_DEALLOC);
+
+			if (dvar == NULL)
+				break;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
+				dtrace_vcopy((void *)(uintptr_t)regs[rd],
+				    dvar->dtdv_data, &v->dtdv_type);
+			} else {
+				*((uint64_t *)dvar->dtdv_data) = regs[rd];
+			}
+
+			break;
+		}
+
+		case DIF_OP_ALLOCS: {
+			uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
+			size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
+
+			if (mstate->dtms_scratch_ptr + size >
+			    mstate->dtms_scratch_base +
+			    mstate->dtms_scratch_size) {
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+				regs[rd] = NULL;
+			} else {
+				dtrace_bzero((void *)
+				    mstate->dtms_scratch_ptr, size);
+				mstate->dtms_scratch_ptr += size;
+				regs[rd] = ptr;
+			}
+			break;
+		}
+
+		case DIF_OP_COPYS:
+			if (!dtrace_canstore(regs[rd], regs[r2],
+			    mstate, vstate)) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = regs[rd];
+				break;
+			}
+
+			dtrace_bcopy((void *)(uintptr_t)regs[r1],
+			    (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
+			break;
+
+		case DIF_OP_STB:
+			if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = regs[rd];
+				break;
+			}
+			*((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
+			break;
+
+		case DIF_OP_STH:
+			if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = regs[rd];
+				break;
+			}
+			if (regs[rd] & 1) {
+				*flags |= CPU_DTRACE_BADALIGN;
+				*illval = regs[rd];
+				break;
+			}
+			*((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
+			break;
+
+		case DIF_OP_STW:
+			if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = regs[rd];
+				break;
+			}
+			if (regs[rd] & 3) {
+				*flags |= CPU_DTRACE_BADALIGN;
+				*illval = regs[rd];
+				break;
+			}
+			*((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
+			break;
+
+		case DIF_OP_STX:
+			if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
+				*flags |= CPU_DTRACE_BADADDR;
+				*illval = regs[rd];
+				break;
+			}
+#if !defined(__APPLE__)
+			if (regs[rd] & 7) {
+#else
+			if (regs[rd] & 3) { /* Darwin kmem_zalloc() called from dtrace_difo_init() is 4-byte aligned. */
+#endif /* __APPLE__ */
+				*flags |= CPU_DTRACE_BADALIGN;
+				*illval = regs[rd];
+				break;
+			}
+			*((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
+			break;
+		}
+	}
+
+	if (!(*flags & CPU_DTRACE_FAULT))
+		return (rval);
+
+	mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
+	mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
+
+	return (0);
+}
+
+static void
+dtrace_action_breakpoint(dtrace_ecb_t *ecb)
+{
+	dtrace_probe_t *probe = ecb->dte_probe;
+	dtrace_provider_t *prov = probe->dtpr_provider;
+	char c[DTRACE_FULLNAMELEN + 80], *str;
+	char *msg = "dtrace: breakpoint action at probe ";
+	char *ecbmsg = " (ecb ";
+	uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
+	uintptr_t val = (uintptr_t)ecb;
+	int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
+
+	if (dtrace_destructive_disallow)
+		return;
+
+	/*
+	 * It's impossible to be taking action on the NULL probe.
+	 */
+	ASSERT(probe != NULL);
+
+	/*
+	 * This is a poor man's (destitute man's?) sprintf():  we want to
+	 * print the provider name, module name, function name and name of
+	 * the probe, along with the hex address of the ECB with the breakpoint
+	 * action -- all of which we must place in the character buffer by
+	 * hand.
+	 */
+	while (*msg != '\0')
+		c[i++] = *msg++;
+
+	for (str = prov->dtpv_name; *str != '\0'; str++)
+		c[i++] = *str;
+	c[i++] = ':';
+
+	for (str = probe->dtpr_mod; *str != '\0'; str++)
+		c[i++] = *str;
+	c[i++] = ':';
+
+	for (str = probe->dtpr_func; *str != '\0'; str++)
+		c[i++] = *str;
+	c[i++] = ':';
+
+	for (str = probe->dtpr_name; *str != '\0'; str++)
+		c[i++] = *str;
+
+	while (*ecbmsg != '\0')
+		c[i++] = *ecbmsg++;
+
+	while (shift >= 0) {
+		mask = (uintptr_t)0xf << shift;
+
+		if (val >= ((uintptr_t)1 << shift))
+			c[i++] = "0123456789abcdef"[(val & mask) >> shift];
+		shift -= 4;
+	}
+
+	c[i++] = ')';
+	c[i] = '\0';
+
+	debug_enter(c);
+}
+
+static void
+dtrace_action_panic(dtrace_ecb_t *ecb)
+{
+	dtrace_probe_t *probe = ecb->dte_probe;
+
+	/*
+	 * It's impossible to be taking action on the NULL probe.
+	 */
+	ASSERT(probe != NULL);
+
+	if (dtrace_destructive_disallow)
+		return;
+
+	if (dtrace_panicked != NULL)
+		return;
+
+#if !defined(__APPLE__)
+	if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
+		return;
+#else
+	if (dtrace_casptr(&dtrace_panicked, NULL, current_thread()) != NULL)
+		return;
+#endif /* __APPLE__ */
+
+	/*
+	 * We won the right to panic.  (We want to be sure that only one
+	 * thread calls panic() from dtrace_probe(), and that panic() is
+	 * called exactly once.)
+	 */
+	dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
+	    probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
+	    probe->dtpr_func, probe->dtpr_name, (void *)ecb);
+
+#if defined(__APPLE__)
+	/* Mac OS X debug feature -- can return from panic() */
+	dtrace_panicked = NULL;
+#endif /* __APPLE__ */
+}
+
+static void
+dtrace_action_raise(uint64_t sig)
+{
+	if (dtrace_destructive_disallow)
+		return;
+
+	if (sig >= NSIG) {
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+		return;
+	}
+
+#if !defined(__APPLE__)
+	/*
+	 * raise() has a queue depth of 1 -- we ignore all subsequent
+	 * invocations of the raise() action.
+	 */
+	if (curthread->t_dtrace_sig == 0)
+		curthread->t_dtrace_sig = (uint8_t)sig;
+
+	curthread->t_sig_check = 1;
+	aston(curthread);
+#else
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+	if (uthread && uthread->t_dtrace_sig == 0) {
+		uthread->t_dtrace_sig = sig;
+		psignal(current_proc(), (int)sig);
+	}
+#endif /* __APPLE__ */
+}
+
+static void
+dtrace_action_stop(void)
+{
+	if (dtrace_destructive_disallow)
+		return;
+
+#if !defined(__APPLE__)
+	if (!curthread->t_dtrace_stop) {
+		curthread->t_dtrace_stop = 1;
+		curthread->t_sig_check = 1;
+		aston(curthread);
+	}
+#else
+	psignal(current_proc(), SIGSTOP);
+#endif /* __APPLE__ */
+}
+
+static void
+dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
+{
+	hrtime_t now;
+	volatile uint16_t *flags;
+	cpu_t *cpu = CPU;
+
+	if (dtrace_destructive_disallow)
+		return;
+
+	flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
+
+	now = dtrace_gethrtime();
+
+	if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
+		/*
+		 * We need to advance the mark to the current time.
+		 */
+		cpu->cpu_dtrace_chillmark = now;
+		cpu->cpu_dtrace_chilled = 0;
+	}
+
+	/*
+	 * Now check to see if the requested chill time would take us over
+	 * the maximum amount of time allowed in the chill interval.  (Or
+	 * worse, if the calculation itself induces overflow.)
+	 */
+	if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
+	    cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
+		*flags |= CPU_DTRACE_ILLOP;
+		return;
+	}
+
+	while (dtrace_gethrtime() - now < val)
+		continue;
+
+	/*
+	 * Normally, we assure that the value of the variable "timestamp" does
+	 * not change within an ECB.  The presence of chill() represents an
+	 * exception to this rule, however.
+	 */
+	mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
+	cpu->cpu_dtrace_chilled += val;
+}
+
+static void
+dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
+    uint64_t *buf, uint64_t arg)
+{
+	int nframes = DTRACE_USTACK_NFRAMES(arg);
+	int strsize = DTRACE_USTACK_STRSIZE(arg);
+	uint64_t *pcs = &buf[1], *fps;
+	char *str = (char *)&pcs[nframes];
+	int size, offs = 0, i, j;
+	uintptr_t old = mstate->dtms_scratch_ptr, saved;
+	uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+	char *sym;
+
+	/*
+	 * Should be taking a faster path if string space has not been
+	 * allocated.
+	 */
+	ASSERT(strsize != 0);
+
+	/*
+	 * We will first allocate some temporary space for the frame pointers.
+	 */
+	fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
+	size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
+	    (nframes * sizeof (uint64_t));
+
+	if (mstate->dtms_scratch_ptr + size >
+	    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
+		/*
+		 * Not enough room for our frame pointers -- need to indicate
+		 * that we ran out of scratch space.
+		 */
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
+		return;
+	}
+
+	mstate->dtms_scratch_ptr += size;
+	saved = mstate->dtms_scratch_ptr;
+
+	/*
+	 * Now get a stack with both program counters and frame pointers.
+	 */
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	dtrace_getufpstack(buf, fps, nframes + 1);
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	/*
+	 * If that faulted, we're cooked.
+	 */
+	if (*flags & CPU_DTRACE_FAULT)
+		goto out;
+
+	/*
+	 * Now we want to walk up the stack, calling the USTACK helper.  For
+	 * each iteration, we restore the scratch pointer.
+	 */
+	for (i = 0; i < nframes; i++) {
+		mstate->dtms_scratch_ptr = saved;
+
+		if (offs >= strsize)
+			break;
+
+		sym = (char *)(uintptr_t)dtrace_helper(
+		    DTRACE_HELPER_ACTION_USTACK,
+		    mstate, state, pcs[i], fps[i]);
+
+		/*
+		 * If we faulted while running the helper, we're going to
+		 * clear the fault and null out the corresponding string.
+		 */
+		if (*flags & CPU_DTRACE_FAULT) {
+			*flags &= ~CPU_DTRACE_FAULT;
+			str[offs++] = '\0';
+			continue;
+		}
+
+		if (sym == NULL) {
+			str[offs++] = '\0';
+			continue;
+		}
+
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+
+		/*
+		 * Now copy in the string that the helper returned to us.
+		 */
+		for (j = 0; offs + j < strsize; j++) {
+			if ((str[offs + j] = sym[j]) == '\0')
+				break;
+		}
+
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+		offs += j + 1;
+	}
+
+	if (offs >= strsize) {
+		/*
+		 * If we didn't have room for all of the strings, we don't
+		 * abort processing -- this needn't be a fatal error -- but we
+		 * still want to increment a counter (dts_stkstroverflows) to
+		 * allow this condition to be warned about.  (If this is from
+		 * a jstack() action, it is easily tuned via jstackstrsize.)
+		 */
+		dtrace_error(&state->dts_stkstroverflows);
+	}
+
+	while (offs < strsize)
+		str[offs++] = '\0';
+
+out:
+	mstate->dtms_scratch_ptr = old;
+}
+
+/*
+ * If you're looking for the epicenter of DTrace, you just found it.  This
+ * is the function called by the provider to fire a probe -- from which all
+ * subsequent probe-context DTrace activity emanates.
+ */
+#if !defined(__APPLE__)
+void
+dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
+    uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
+#else
+static void
+__dtrace_probe(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
+    uint64_t arg2, uint64_t arg3, uint64_t arg4)
+#endif /* __APPLE__ */
+{
+	processorid_t cpuid;
+	dtrace_icookie_t cookie;
+	dtrace_probe_t *probe;
+	dtrace_mstate_t mstate;
+	dtrace_ecb_t *ecb;
+	dtrace_action_t *act;
+	intptr_t offs;
+	size_t size;
+	int vtime, onintr;
+	volatile uint16_t *flags;
+	hrtime_t now;
+
+#if !defined(__APPLE__)
+	/*
+	 * Kick out immediately if this CPU is still being born (in which case
+	 * curthread will be set to -1)
+	 */
+	if ((uintptr_t)curthread & 1)
+		return;
+#else
+#endif /* __APPLE__ */
+
+	cookie = dtrace_interrupt_disable();
+	probe = dtrace_probes[id - 1];
+	cpuid = CPU->cpu_id;
+	onintr = CPU_ON_INTR(CPU);
+
+#if !defined(__APPLE__)
+	if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
+	    probe->dtpr_predcache == curthread->t_predcache) {
+#else
+	if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
+	    probe->dtpr_predcache == dtrace_get_thread_predcache(current_thread())) {
+#endif /* __APPLE__ */
+		/*
+		 * We have hit in the predicate cache; we know that
+		 * this predicate would evaluate to be false.
+		 */
+		dtrace_interrupt_enable(cookie);
+		return;
+	}
+
+	if (panic_quiesce) {
+		/*
+		 * We don't trace anything if we're panicking.
+		 */
+		dtrace_interrupt_enable(cookie);
+		return;
+	}
+
+#if !defined(__APPLE__)
+	now = dtrace_gethrtime();
+	vtime = dtrace_vtime_references != 0;
+
+	if (vtime && curthread->t_dtrace_start)
+		curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
+#else
+	vtime = dtrace_vtime_references != 0;
+
+	if (vtime)
+	{
+		int64_t dtrace_accum_time, recent_vtime;
+		thread_t thread = current_thread();
+
+		dtrace_accum_time = dtrace_get_thread_tracing(thread); /* Time spent inside DTrace so far (nanoseconds) */
+
+		if (dtrace_accum_time >= 0) {
+			recent_vtime = dtrace_abs_to_nano(dtrace_calc_thread_recent_vtime(thread)); /* up to the moment thread vtime */
+
+			recent_vtime = recent_vtime - dtrace_accum_time; /* Time without DTrace contribution */
+		
+			dtrace_set_thread_vtime(thread, recent_vtime);
+		}
+	}
+
+	now = dtrace_gethrtime(); /* must not precede dtrace_calc_thread_recent_vtime() call! */
+#endif /* __APPLE__ */
+
+	mstate.dtms_probe = probe;
+	mstate.dtms_arg[0] = arg0;
+	mstate.dtms_arg[1] = arg1;
+	mstate.dtms_arg[2] = arg2;
+	mstate.dtms_arg[3] = arg3;
+	mstate.dtms_arg[4] = arg4;
+
+	flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
+
+	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
+		dtrace_predicate_t *pred = ecb->dte_predicate;
+		dtrace_state_t *state = ecb->dte_state;
+		dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
+		dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
+		dtrace_vstate_t *vstate = &state->dts_vstate;
+		dtrace_provider_t *prov = probe->dtpr_provider;
+		int committed = 0;
+		caddr_t tomax;
+
+		/*
+		 * A little subtlety with the following (seemingly innocuous)
+		 * declaration of the automatic 'val':  by looking at the
+		 * code, you might think that it could be declared in the
+		 * action processing loop, below.  (That is, it's only used in
+		 * the action processing loop.)  However, it must be declared
+		 * out of that scope because in the case of DIF expression
+		 * arguments to aggregating actions, one iteration of the
+		 * action loop will use the last iteration's value.
+		 */
+#ifdef lint
+		uint64_t val = 0;
+#else
+		uint64_t val;
+#endif
+
+		mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
+		*flags &= ~CPU_DTRACE_ERROR;
+
+		if (prov == dtrace_provider) {
+			/*
+			 * If dtrace itself is the provider of this probe,
+			 * we're only going to continue processing the ECB if
+			 * arg0 (the dtrace_state_t) is equal to the ECB's
+			 * creating state.  (This prevents disjoint consumers
+			 * from seeing one another's metaprobes.)
+			 */
+			if (arg0 != (uint64_t)(uintptr_t)state)
+				continue;
+		}
+
+		if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
+			/*
+			 * We're not currently active.  If our provider isn't
+			 * the dtrace pseudo provider, we're not interested.
+			 */
+			if (prov != dtrace_provider)
+				continue;
+
+			/*
+			 * Now we must further check if we are in the BEGIN
+			 * probe.  If we are, we will only continue processing
+			 * if we're still in WARMUP -- if one BEGIN enabling
+			 * has invoked the exit() action, we don't want to
+			 * evaluate subsequent BEGIN enablings.
+			 */
+			if (probe->dtpr_id == dtrace_probeid_begin &&
+			    state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
+				ASSERT(state->dts_activity ==
+				    DTRACE_ACTIVITY_DRAINING);
+				continue;
+			}
+		}
+
+#if defined(__APPLE__)
+        /*
+         * If the thread on which this probe has fired belongs to a process marked P_LNOATTACH
+         * then this enabling is not permitted to observe it. Move along, nothing to see here.
+         */
+        if (ISSET(current_proc()->p_lflag, P_LNOATTACH)) {
+            continue;
+        }
+#endif /* __APPLE__ */
+
+		if (ecb->dte_cond) {
+			/*
+			 * If the dte_cond bits indicate that this
+			 * consumer is only allowed to see user-mode firings
+			 * of this probe, call the provider's dtps_usermode()
+			 * entry point to check that the probe was fired
+			 * while in a user context. Skip this ECB if that's
+			 * not the case.
+			 */
+			if ((ecb->dte_cond & DTRACE_COND_USERMODE) &&
+			    prov->dtpv_pops.dtps_usermode(prov->dtpv_arg,
+			    probe->dtpr_id, probe->dtpr_arg) == 0)
+				continue;
+
+			/*
+			 * This is more subtle than it looks. We have to be
+			 * absolutely certain that CRED() isn't going to
+			 * change out from under us so it's only legit to
+			 * examine that structure if we're in constrained
+			 * situations. Currently, the only times we'll this
+			 * check is if a non-super-user has enabled the
+			 * profile or syscall providers -- providers that
+			 * allow visibility of all processes. For the
+			 * profile case, the check above will ensure that
+			 * we're examining a user context.
+			 */
+			if (ecb->dte_cond & DTRACE_COND_OWNER) {
+				cred_t *cr;
+				cred_t *s_cr =
+				    ecb->dte_state->dts_cred.dcr_cred;
+				proc_t *proc;
+
+				ASSERT(s_cr != NULL);
+
+#if !defined(__APPLE__)
+				if ((cr = CRED()) == NULL ||
+#else
+				if ((cr = dtrace_CRED()) == NULL ||
+#endif /* __APPLE__ */
+				    s_cr->cr_uid != cr->cr_uid ||
+				    s_cr->cr_uid != cr->cr_ruid ||
+				    s_cr->cr_uid != cr->cr_suid ||
+				    s_cr->cr_gid != cr->cr_gid ||
+				    s_cr->cr_gid != cr->cr_rgid ||
+				    s_cr->cr_gid != cr->cr_sgid ||
+#if !defined(__APPLE__)
+				    (proc = ttoproc(curthread)) == NULL ||
+				    (proc->p_flag & SNOCD))
+#else
+					1) /* Darwin omits "No Core Dump" flag. */
+#endif /* __APPLE__ */
+					continue;
+			}
+
+			if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
+				cred_t *cr;
+				cred_t *s_cr =
+				    ecb->dte_state->dts_cred.dcr_cred;
+
+				ASSERT(s_cr != NULL);
+
+#if !defined(__APPLE__) /* Darwin doesn't do zones. */
+				if ((cr = CRED()) == NULL ||
+				    s_cr->cr_zone->zone_id !=
+				    cr->cr_zone->zone_id)
+					continue;
+#endif /* __APPLE__ */
+			}
+		}
+
+		if (now - state->dts_alive > dtrace_deadman_timeout) {
+			/*
+			 * We seem to be dead.  Unless we (a) have kernel
+			 * destructive permissions (b) have expicitly enabled
+			 * destructive actions and (c) destructive actions have
+			 * not been disabled, we're going to transition into
+			 * the KILLED state, from which no further processing
+			 * on this state will be performed.
+			 */
+			if (!dtrace_priv_kernel_destructive(state) ||
+			    !state->dts_cred.dcr_destructive ||
+			    dtrace_destructive_disallow) {
+				void *activity = &state->dts_activity;
+				dtrace_activity_t current;
+
+				do {
+					current = state->dts_activity;
+				} while (dtrace_cas32(activity, current,
+				    DTRACE_ACTIVITY_KILLED) != current);
+
+				continue;
+			}
+		}
+
+		if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
+		    ecb->dte_alignment, state, &mstate)) < 0)
+			continue;
+
+		tomax = buf->dtb_tomax;
+		ASSERT(tomax != NULL);
+
+		if (ecb->dte_size != 0)
+			DTRACE_STORE(uint32_t, tomax, offs, ecb->dte_epid);
+
+		mstate.dtms_epid = ecb->dte_epid;
+		mstate.dtms_present |= DTRACE_MSTATE_EPID;
+
+		if (pred != NULL) {
+			dtrace_difo_t *dp = pred->dtp_difo;
+			int rval;
+
+			rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
+
+			if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
+				dtrace_cacheid_t cid = probe->dtpr_predcache;
+
+				if (cid != DTRACE_CACHEIDNONE && !onintr) {
+					/*
+					 * Update the predicate cache...
+					 */
+					ASSERT(cid == pred->dtp_cacheid);
+#if !defined(__APPLE__)
+					curthread->t_predcache = cid;
+#else
+					dtrace_set_thread_predcache(current_thread(), cid);
+#endif /* __APPLE__ */
+				}
+
+				continue;
+			}
+		}
+
+		for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
+		    act != NULL; act = act->dta_next) {
+			size_t valoffs;
+			dtrace_difo_t *dp;
+			dtrace_recdesc_t *rec = &act->dta_rec;
+
+			size = rec->dtrd_size;
+			valoffs = offs + rec->dtrd_offset;
+
+			if (DTRACEACT_ISAGG(act->dta_kind)) {
+				uint64_t v = 0xbad;
+				dtrace_aggregation_t *agg;
+
+				agg = (dtrace_aggregation_t *)act;
+
+				if ((dp = act->dta_difo) != NULL)
+					v = dtrace_dif_emulate(dp,
+					    &mstate, vstate, state);
+
+				if (*flags & CPU_DTRACE_ERROR)
+					continue;
+
+				/*
+				 * Note that we always pass the expression
+				 * value from the previous iteration of the
+				 * action loop.  This value will only be used
+				 * if there is an expression argument to the
+				 * aggregating action, denoted by the
+				 * dtag_hasarg field.
+				 */
+				dtrace_aggregate(agg, buf,
+				    offs, aggbuf, v, val);
+				continue;
+			}
+
+			switch (act->dta_kind) {
+			case DTRACEACT_STOP:
+				if (dtrace_priv_proc_destructive(state))
+					dtrace_action_stop();
+				continue;
+
+			case DTRACEACT_BREAKPOINT:
+				if (dtrace_priv_kernel_destructive(state))
+					dtrace_action_breakpoint(ecb);
+				continue;
+
+			case DTRACEACT_PANIC:
+				if (dtrace_priv_kernel_destructive(state))
+					dtrace_action_panic(ecb);
+				continue;
+
+			case DTRACEACT_STACK:
+				if (!dtrace_priv_kernel(state))
+					continue;
+
+				dtrace_getpcstack((pc_t *)(tomax + valoffs),
+				    size / sizeof (pc_t), probe->dtpr_aframes,
+				    DTRACE_ANCHORED(probe) ? NULL :
+				    (uint32_t *)arg0);
+
+				continue;
+
+			case DTRACEACT_JSTACK:
+			case DTRACEACT_USTACK:
+				if (!dtrace_priv_proc(state))
+					continue;
+
+				/*
+				 * See comment in DIF_VAR_PID.
+				 */
+				if (DTRACE_ANCHORED(mstate.dtms_probe) &&
+				    CPU_ON_INTR(CPU)) {
+					int depth = DTRACE_USTACK_NFRAMES(
+					    rec->dtrd_arg) + 1;
+
+					dtrace_bzero((void *)(tomax + valoffs),
+					    DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
+					    + depth * sizeof (uint64_t));
+
+					continue;
+				}
+
+				if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
+				    curproc->p_dtrace_helpers != NULL) {
+					/*
+					 * This is the slow path -- we have
+					 * allocated string space, and we're
+					 * getting the stack of a process that
+					 * has helpers.  Call into a separate
+					 * routine to perform this processing.
+					 */
+					dtrace_action_ustack(&mstate, state,
+					    (uint64_t *)(tomax + valoffs),
+					    rec->dtrd_arg);
+					continue;
+				}
+
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+				dtrace_getupcstack((uint64_t *)
+				    (tomax + valoffs),
+				    DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+				continue;
+
+			default:
+				break;
+			}
+
+			dp = act->dta_difo;
+			ASSERT(dp != NULL);
+
+			val = dtrace_dif_emulate(dp, &mstate, vstate, state);
+
+			if (*flags & CPU_DTRACE_ERROR)
+				continue;
+
+			switch (act->dta_kind) {
+			case DTRACEACT_SPECULATE:
+				ASSERT(buf == &state->dts_buffer[cpuid]);
+				buf = dtrace_speculation_buffer(state,
+				    cpuid, val);
+
+				if (buf == NULL) {
+					*flags |= CPU_DTRACE_DROP;
+					continue;
+				}
+
+				offs = dtrace_buffer_reserve(buf,
+				    ecb->dte_needed, ecb->dte_alignment,
+				    state, NULL);
+
+				if (offs < 0) {
+					*flags |= CPU_DTRACE_DROP;
+					continue;
+				}
+
+				tomax = buf->dtb_tomax;
+				ASSERT(tomax != NULL);
+
+				if (ecb->dte_size != 0)
+					DTRACE_STORE(uint32_t, tomax, offs,
+					    ecb->dte_epid);
+				continue;
+
+			case DTRACEACT_CHILL:
+				if (dtrace_priv_kernel_destructive(state))
+					dtrace_action_chill(&mstate, val);
+				continue;
+
+			case DTRACEACT_RAISE:
+				if (dtrace_priv_proc_destructive(state))
+					dtrace_action_raise(val);
+				continue;
+
+			case DTRACEACT_COMMIT:
+				ASSERT(!committed);
+
+				/*
+				 * We need to commit our buffer state.
+				 */
+				if (ecb->dte_size)
+					buf->dtb_offset = offs + ecb->dte_size;
+				buf = &state->dts_buffer[cpuid];
+				dtrace_speculation_commit(state, cpuid, val);
+				committed = 1;
+				continue;
+
+			case DTRACEACT_DISCARD:
+				dtrace_speculation_discard(state, cpuid, val);
+				continue;
+
+			case DTRACEACT_DIFEXPR:
+			case DTRACEACT_LIBACT:
+			case DTRACEACT_PRINTF:
+			case DTRACEACT_PRINTA:
+			case DTRACEACT_SYSTEM:
+			case DTRACEACT_FREOPEN:
+				break;
+
+			case DTRACEACT_SYM:
+			case DTRACEACT_MOD:
+				if (!dtrace_priv_kernel(state))
+					continue;
+				break;
+
+#if !defined(__APPLE__)
+			case DTRACEACT_USYM:
+			case DTRACEACT_UMOD:
+			case DTRACEACT_UADDR: {
+				struct pid *pid = curthread->t_procp->p_pidp;
+
+				if (!dtrace_priv_proc(state))
+					continue;
+
+				DTRACE_STORE(uint64_t, tomax,
+				    valoffs, (uint64_t)pid->pid_id);
+				DTRACE_STORE(uint64_t, tomax,
+				    valoffs + sizeof (uint64_t), val);
+
+				continue;
+			}
+#else
+			case DTRACEACT_USYM:
+			case DTRACEACT_UMOD:
+			case DTRACEACT_UADDR: {
+				if (!dtrace_priv_proc(state))
+					continue;
+
+				DTRACE_STORE(uint64_t, tomax,
+				    valoffs, (uint64_t)proc_selfpid());
+				DTRACE_STORE(uint64_t, tomax,
+				    valoffs + sizeof (uint64_t), val);
+
+				continue;
+			}
+#endif /* __APPLE__ */
+
+			case DTRACEACT_EXIT: {
+				/*
+				 * For the exit action, we are going to attempt
+				 * to atomically set our activity to be
+				 * draining.  If this fails (either because
+				 * another CPU has beat us to the exit action,
+				 * or because our current activity is something
+				 * other than ACTIVE or WARMUP), we will
+				 * continue.  This assures that the exit action
+				 * can be successfully recorded at most once
+				 * when we're in the ACTIVE state.  If we're
+				 * encountering the exit() action while in
+				 * COOLDOWN, however, we want to honor the new
+				 * status code.  (We know that we're the only
+				 * thread in COOLDOWN, so there is no race.)
+				 */
+				void *activity = &state->dts_activity;
+				dtrace_activity_t current = state->dts_activity;
+
+				if (current == DTRACE_ACTIVITY_COOLDOWN)
+					break;
+
+				if (current != DTRACE_ACTIVITY_WARMUP)
+					current = DTRACE_ACTIVITY_ACTIVE;
+
+				if (dtrace_cas32(activity, current,
+				    DTRACE_ACTIVITY_DRAINING) != current) {
+					*flags |= CPU_DTRACE_DROP;
+					continue;
+				}
+
+				break;
+			}
+
+			default:
+				ASSERT(0);
+			}
+
+			if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
+				uintptr_t end = valoffs + size;
+
+				/*
+				 * If this is a string, we're going to only
+				 * load until we find the zero byte -- after
+				 * which we'll store zero bytes.
+				 */
+				if (dp->dtdo_rtype.dtdt_kind ==
+				    DIF_TYPE_STRING) {
+					char c = '\0' + 1;
+					int intuple = act->dta_intuple;
+					size_t s;
+
+					for (s = 0; s < size; s++) {
+						if (c != '\0')
+							c = dtrace_load8(val++);
+
+						DTRACE_STORE(uint8_t, tomax,
+						    valoffs++, c);
+
+						if (c == '\0' && intuple)
+							break;
+					}
+
+					continue;
+				}
+
+				while (valoffs < end) {
+					DTRACE_STORE(uint8_t, tomax, valoffs++,
+					    dtrace_load8(val++));
+				}
+
+				continue;
+			}
+
+			switch (size) {
+			case 0:
+				break;
+
+			case sizeof (uint8_t):
+				DTRACE_STORE(uint8_t, tomax, valoffs, val);
+				break;
+			case sizeof (uint16_t):
+				DTRACE_STORE(uint16_t, tomax, valoffs, val);
+				break;
+			case sizeof (uint32_t):
+				DTRACE_STORE(uint32_t, tomax, valoffs, val);
+				break;
+			case sizeof (uint64_t):
+				DTRACE_STORE(uint64_t, tomax, valoffs, val);
+				break;
+			default:
+				/*
+				 * Any other size should have been returned by
+				 * reference, not by value.
+				 */
+				ASSERT(0);
+				break;
+			}
+		}
+
+		if (*flags & CPU_DTRACE_DROP)
+			continue;
+
+		if (*flags & CPU_DTRACE_FAULT) {
+			int ndx;
+			dtrace_action_t *err;
+
+			buf->dtb_errors++;
+
+			if (probe->dtpr_id == dtrace_probeid_error) {
+				/*
+				 * There's nothing we can do -- we had an
+				 * error on the error probe.  We bump an
+				 * error counter to at least indicate that
+				 * this condition happened.
+				 */
+				dtrace_error(&state->dts_dblerrors);
+				continue;
+			}
+
+			if (vtime) {
+				/*
+				 * Before recursing on dtrace_probe(), we
+				 * need to explicitly clear out our start
+				 * time to prevent it from being accumulated
+				 * into t_dtrace_vtime.
+				 */
+#if !defined(__APPLE__)
+				curthread->t_dtrace_start = 0;
+#else
+				/* Set the sign bit on t_dtrace_tracing to suspend accumulation to it. */
+				dtrace_set_thread_tracing(current_thread(), 
+							(1ULL<<63) | dtrace_get_thread_tracing(current_thread()));
+#endif /* __APPLE__ */
+			}
+
+			/*
+			 * Iterate over the actions to figure out which action
+			 * we were processing when we experienced the error.
+			 * Note that act points _past_ the faulting action; if
+			 * act is ecb->dte_action, the fault was in the
+			 * predicate, if it's ecb->dte_action->dta_next it's
+			 * in action #1, and so on.
+			 */
+			for (err = ecb->dte_action, ndx = 0;
+			    err != act; err = err->dta_next, ndx++)
+				continue;
+
+			dtrace_probe_error(state, ecb->dte_epid, ndx,
+			    (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
+			    mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
+			    cpu_core[cpuid].cpuc_dtrace_illval);
+
+			continue;
+		}
+
+		if (!committed)
+			buf->dtb_offset = offs + ecb->dte_size;
+	}
+
+#if !defined(__APPLE__)
+	if (vtime)
+		curthread->t_dtrace_start = dtrace_gethrtime();
+#else
+	if (vtime) {
+		thread_t thread = current_thread();
+		int64_t t = dtrace_get_thread_tracing(thread);
+		
+		if (t >= 0) { 
+			/* Usual case, accumulate time spent here into t_dtrace_tracing */
+			dtrace_set_thread_tracing(thread, t + (dtrace_gethrtime() - now));
+		} else { 
+			/* Return from error recursion. No accumulation, just clear the sign bit on t_dtrace_tracing. */
+			dtrace_set_thread_tracing(thread, (~(1ULL<<63)) & t); 
+		}
+	}
+#endif /* __APPLE__ */
+
+	dtrace_interrupt_enable(cookie);
+}
+
+#if defined(__APPLE__)
+/* Don't allow a thread to re-enter dtrace_probe() */
+void
+dtrace_probe(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
+    uint64_t arg2, uint64_t arg3, uint64_t arg4)
+{
+	thread_t thread = current_thread();
+
+	if (id == dtrace_probeid_error) {
+		__dtrace_probe(id, arg0, arg1, arg2, arg3, arg4);
+		dtrace_getfp(); /* Defeat tail-call optimization of __dtrace_probe() */
+	} else if (!dtrace_get_thread_reentering(thread)) {
+		dtrace_set_thread_reentering(thread, TRUE);
+		__dtrace_probe(id, arg0, arg1, arg2, arg3, arg4);
+		dtrace_set_thread_reentering(thread, FALSE);
+	}
+}
+#endif /* __APPLE__ */
+
+/*
+ * DTrace Probe Hashing Functions
+ *
+ * The functions in this section (and indeed, the functions in remaining
+ * sections) are not _called_ from probe context.  (Any exceptions to this are
+ * marked with a "Note:".)  Rather, they are called from elsewhere in the
+ * DTrace framework to look-up probes in, add probes to and remove probes from
+ * the DTrace probe hashes.  (Each probe is hashed by each element of the
+ * probe tuple -- allowing for fast lookups, regardless of what was
+ * specified.)
+ */
+static uint_t
+dtrace_hash_str(char *p)
+{
+	unsigned int g;
+	uint_t hval = 0;
+
+	while (*p) {
+		hval = (hval << 4) + *p++;
+		if ((g = (hval & 0xf0000000)) != 0)
+			hval ^= g >> 24;
+		hval &= ~g;
+	}
+	return (hval);
+}
+
+static dtrace_hash_t *
+dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
+{
+	dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
+
+	hash->dth_stroffs = stroffs;
+	hash->dth_nextoffs = nextoffs;
+	hash->dth_prevoffs = prevoffs;
+
+	hash->dth_size = 1;
+	hash->dth_mask = hash->dth_size - 1;
+
+	hash->dth_tab = kmem_zalloc(hash->dth_size *
+	    sizeof (dtrace_hashbucket_t *), KM_SLEEP);
+
+	return (hash);
+}
+
+#if !defined(__APPLE__) /* Quiet compiler warning */
+static void
+dtrace_hash_destroy(dtrace_hash_t *hash)
+{
+#ifdef DEBUG
+	int i;
+
+	for (i = 0; i < hash->dth_size; i++)
+		ASSERT(hash->dth_tab[i] == NULL);
+#endif
+
+	kmem_free(hash->dth_tab,
+	    hash->dth_size * sizeof (dtrace_hashbucket_t *));
+	kmem_free(hash, sizeof (dtrace_hash_t));
+}
+#endif /* __APPLE__ */
+
+static void
+dtrace_hash_resize(dtrace_hash_t *hash)
+{
+	int size = hash->dth_size, i, ndx;
+	int new_size = hash->dth_size << 1;
+	int new_mask = new_size - 1;
+	dtrace_hashbucket_t **new_tab, *bucket, *next;
+
+	ASSERT((new_size & new_mask) == 0);
+
+	new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
+
+	for (i = 0; i < size; i++) {
+		for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
+			dtrace_probe_t *probe = bucket->dthb_chain;
+
+			ASSERT(probe != NULL);
+			ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
+
+			next = bucket->dthb_next;
+			bucket->dthb_next = new_tab[ndx];
+			new_tab[ndx] = bucket;
+		}
+	}
+
+	kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
+	hash->dth_tab = new_tab;
+	hash->dth_size = new_size;
+	hash->dth_mask = new_mask;
+}
+
+static void
+dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
+{
+	int hashval = DTRACE_HASHSTR(hash, new);
+	int ndx = hashval & hash->dth_mask;
+	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
+	dtrace_probe_t **nextp, **prevp;
+
+	for (; bucket != NULL; bucket = bucket->dthb_next) {
+		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
+			goto add;
+	}
+
+	if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
+		dtrace_hash_resize(hash);
+		dtrace_hash_add(hash, new);
+		return;
+	}
+
+	bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
+	bucket->dthb_next = hash->dth_tab[ndx];
+	hash->dth_tab[ndx] = bucket;
+	hash->dth_nbuckets++;
+
+add:
+	nextp = DTRACE_HASHNEXT(hash, new);
+	ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
+	*nextp = bucket->dthb_chain;
+
+	if (bucket->dthb_chain != NULL) {
+		prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
+		ASSERT(*prevp == NULL);
+		*prevp = new;
+	}
+
+	bucket->dthb_chain = new;
+	bucket->dthb_len++;
+}
+
+static dtrace_probe_t *
+dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
+{
+	int hashval = DTRACE_HASHSTR(hash, template);
+	int ndx = hashval & hash->dth_mask;
+	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
+
+	for (; bucket != NULL; bucket = bucket->dthb_next) {
+		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
+			return (bucket->dthb_chain);
+	}
+
+	return (NULL);
+}
+
+static int
+dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
+{
+	int hashval = DTRACE_HASHSTR(hash, template);
+	int ndx = hashval & hash->dth_mask;
+	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
+
+	for (; bucket != NULL; bucket = bucket->dthb_next) {
+		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
+			return (bucket->dthb_len);
+	}
+
+	return (NULL);
+}
+
+static void
+dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
+{
+	int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
+	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
+
+	dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
+	dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
+
+	/*
+	 * Find the bucket that we're removing this probe from.
+	 */
+	for (; bucket != NULL; bucket = bucket->dthb_next) {
+		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
+			break;
+	}
+
+	ASSERT(bucket != NULL);
+
+	if (*prevp == NULL) {
+		if (*nextp == NULL) {
+			/*
+			 * The removed probe was the only probe on this
+			 * bucket; we need to remove the bucket.
+			 */
+			dtrace_hashbucket_t *b = hash->dth_tab[ndx];
+
+			ASSERT(bucket->dthb_chain == probe);
+			ASSERT(b != NULL);
+
+			if (b == bucket) {
+				hash->dth_tab[ndx] = bucket->dthb_next;
+			} else {
+				while (b->dthb_next != bucket)
+					b = b->dthb_next;
+				b->dthb_next = bucket->dthb_next;
+			}
+
+			ASSERT(hash->dth_nbuckets > 0);
+			hash->dth_nbuckets--;
+			kmem_free(bucket, sizeof (dtrace_hashbucket_t));
+			return;
+		}
+
+		bucket->dthb_chain = *nextp;
+	} else {
+		*(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
+	}
+
+	if (*nextp != NULL)
+		*(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
+}
+
+/*
+ * DTrace Utility Functions
+ *
+ * These are random utility functions that are _not_ called from probe context.
+ */
+static int
+dtrace_badattr(const dtrace_attribute_t *a)
+{
+	return (a->dtat_name > DTRACE_STABILITY_MAX ||
+	    a->dtat_data > DTRACE_STABILITY_MAX ||
+	    a->dtat_class > DTRACE_CLASS_MAX);
+}
+
+/*
+ * Return a duplicate copy of a string.  If the specified string is NULL,
+ * this function returns a zero-length string.
+ */
+static char *
+dtrace_strdup(const char *str)
+{
+	char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
+
+	if (str != NULL)
+		(void) strcpy(new, str);
+
+	return (new);
+}
+
+#define	DTRACE_ISALPHA(c)	\
+	(((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
+
+static int
+dtrace_badname(const char *s)
+{
+	char c;
+
+	if (s == NULL || (c = *s++) == '\0')
+		return (0);
+
+	if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
+		return (1);
+
+	while ((c = *s++) != '\0') {
+		if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
+		    c != '-' && c != '_' && c != '.' && c != '`')
+			return (1);
+	}
+
+	return (0);
+}
+
+static void
+dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp)
+{
+	uint32_t priv;
+
+	if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
+		/*
+		 * For DTRACE_PRIV_ALL, the uid and zoneid don't matter.
+		 */
+		priv = DTRACE_PRIV_ALL;
+	} else {
+		*uidp = crgetuid(cr);
+		*zoneidp = crgetzoneid(cr);
+
+		priv = 0;
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
+			priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
+		else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
+			priv |= DTRACE_PRIV_USER;
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
+			priv |= DTRACE_PRIV_PROC;
+		if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
+			priv |= DTRACE_PRIV_OWNER;
+		if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
+			priv |= DTRACE_PRIV_ZONEOWNER;
+	}
+
+	*privp = priv;
+}
+
+#ifdef DTRACE_ERRDEBUG
+static void
+dtrace_errdebug(const char *str)
+{
+	int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
+	int occupied = 0;
+
+	lck_mtx_lock(&dtrace_errlock);
+	dtrace_errlast = str;
+#if !defined(__APPLE__)
+	dtrace_errthread = curthread;
+#else
+	dtrace_errthread = current_thread();
+#endif /* __APPLE__ */
+
+	while (occupied++ < DTRACE_ERRHASHSZ) {
+		if (dtrace_errhash[hval].dter_msg == str) {
+			dtrace_errhash[hval].dter_count++;
+			goto out;
+		}
+
+		if (dtrace_errhash[hval].dter_msg != NULL) {
+			hval = (hval + 1) % DTRACE_ERRHASHSZ;
+			continue;
+		}
+
+		dtrace_errhash[hval].dter_msg = str;
+		dtrace_errhash[hval].dter_count = 1;
+		goto out;
+	}
+
+	panic("dtrace: undersized error hash");
+out:
+	lck_mtx_unlock(&dtrace_errlock);
+}
+#endif
+
+/*
+ * DTrace Matching Functions
+ *
+ * These functions are used to match groups of probes, given some elements of
+ * a probe tuple, or some globbed expressions for elements of a probe tuple.
+ */
+static int
+dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid,
+    zoneid_t zoneid)
+{
+	if (priv != DTRACE_PRIV_ALL) {
+		uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
+		uint32_t match = priv & ppriv;
+
+		/*
+		 * No PRIV_DTRACE_* privileges...
+		 */
+		if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
+		    DTRACE_PRIV_KERNEL)) == 0)
+			return (0);
+
+		/*
+		 * No matching bits, but there were bits to match...
+		 */
+		if (match == 0 && ppriv != 0)
+			return (0);
+
+		/*
+		 * Need to have permissions to the process, but don't...
+		 */
+		if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
+		    uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) {
+			return (0);
+		}
+
+		/*
+		 * Need to be in the same zone unless we possess the
+		 * privilege to examine all zones.
+		 */
+		if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 &&
+		    zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) {
+			return (0);
+		}
+	}
+
+	return (1);
+}
+
+/*
+ * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
+ * consists of input pattern strings and an ops-vector to evaluate them.
+ * This function returns >0 for match, 0 for no match, and <0 for error.
+ */
+static int
+dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
+    uint32_t priv, uid_t uid, zoneid_t zoneid)
+{
+	dtrace_provider_t *pvp = prp->dtpr_provider;
+	int rv;
+
+	if (pvp->dtpv_defunct)
+		return (0);
+
+	if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
+		return (rv);
+
+	if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
+		return (rv);
+
+	if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
+		return (rv);
+
+	if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
+		return (rv);
+
+	if (dtrace_match_priv(prp, priv, uid, zoneid) == 0)
+		return (0);
+
+	return (rv);
+}
+
+/*
+ * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
+ * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
+ * libc's version, the kernel version only applies to 8-bit ASCII strings.
+ * In addition, all of the recursion cases except for '*' matching have been
+ * unwound.  For '*', we still implement recursive evaluation, but a depth
+ * counter is maintained and matching is aborted if we recurse too deep.
+ * The function returns 0 if no match, >0 if match, and <0 if recursion error.
+ */
+static int
+dtrace_match_glob(const char *s, const char *p, int depth)
+{
+	const char *olds;
+	char s1, c;
+	int gs;
+
+	if (depth > DTRACE_PROBEKEY_MAXDEPTH)
+		return (-1);
+
+	if (s == NULL)
+		s = ""; /* treat NULL as empty string */
+
+top:
+	olds = s;
+	s1 = *s++;
+
+	if (p == NULL)
+		return (0);
+
+	if ((c = *p++) == '\0')
+		return (s1 == '\0');
+
+	switch (c) {
+	case '[': {
+		int ok = 0, notflag = 0;
+		char lc = '\0';
+
+		if (s1 == '\0')
+			return (0);
+
+		if (*p == '!') {
+			notflag = 1;
+			p++;
+		}
+
+		if ((c = *p++) == '\0')
+			return (0);
+
+		do {
+			if (c == '-' && lc != '\0' && *p != ']') {
+				if ((c = *p++) == '\0')
+					return (0);
+				if (c == '\\' && (c = *p++) == '\0')
+					return (0);
+
+				if (notflag) {
+					if (s1 < lc || s1 > c)
+						ok++;
+					else
+						return (0);
+				} else if (lc <= s1 && s1 <= c)
+					ok++;
+
+			} else if (c == '\\' && (c = *p++) == '\0')
+				return (0);
+
+			lc = c; /* save left-hand 'c' for next iteration */
+
+			if (notflag) {
+				if (s1 != c)
+					ok++;
+				else
+					return (0);
+			} else if (s1 == c)
+				ok++;
+
+			if ((c = *p++) == '\0')
+				return (0);
+
+		} while (c != ']');
+
+		if (ok)
+			goto top;
+
+		return (0);
+	}
+
+	case '\\':
+		if ((c = *p++) == '\0')
+			return (0);
+		/*FALLTHRU*/
+
+	default:
+		if (c != s1)
+			return (0);
+		/*FALLTHRU*/
+
+	case '?':
+		if (s1 != '\0')
+			goto top;
+		return (0);
+
+	case '*':
+		while (*p == '*')
+			p++; /* consecutive *'s are identical to a single one */
+
+		if (*p == '\0')
+			return (1);
+
+		for (s = olds; *s != '\0'; s++) {
+			if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
+				return (gs);
+		}
+
+		return (0);
+	}
+}
+
+/*ARGSUSED*/
+static int
+dtrace_match_string(const char *s, const char *p, int depth)
+{
+	return (s != NULL && strcmp(s, p) == 0);
+}
+
+/*ARGSUSED*/
+static int
+dtrace_match_nul(const char *s, const char *p, int depth)
+{
+	return (1); /* always match the empty pattern */
+}
+
+/*ARGSUSED*/
+static int
+dtrace_match_nonzero(const char *s, const char *p, int depth)
+{
+	return (s != NULL && s[0] != '\0');
+}
+
+static int
+dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
+    zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg)
+{
+	dtrace_probe_t template, *probe;
+	dtrace_hash_t *hash = NULL;
+	int len, best = INT_MAX, nmatched = 0;
+	dtrace_id_t i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	/*
+	 * If the probe ID is specified in the key, just lookup by ID and
+	 * invoke the match callback once if a matching probe is found.
+	 */
+	if (pkp->dtpk_id != DTRACE_IDNONE) {
+		if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
+		    dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) {
+			(void) (*matched)(probe, arg);
+			nmatched++;
+		}
+		return (nmatched);
+	}
+
+	template.dtpr_mod = (char *)pkp->dtpk_mod;
+	template.dtpr_func = (char *)pkp->dtpk_func;
+	template.dtpr_name = (char *)pkp->dtpk_name;
+
+	/*
+	 * We want to find the most distinct of the module name, function
+	 * name, and name.  So for each one that is not a glob pattern or
+	 * empty string, we perform a lookup in the corresponding hash and
+	 * use the hash table with the fewest collisions to do our search.
+	 */
+	if (pkp->dtpk_mmatch == &dtrace_match_string &&
+	    (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
+		best = len;
+		hash = dtrace_bymod;
+	}
+
+	if (pkp->dtpk_fmatch == &dtrace_match_string &&
+	    (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
+		best = len;
+		hash = dtrace_byfunc;
+	}
+
+	if (pkp->dtpk_nmatch == &dtrace_match_string &&
+	    (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
+		best = len;
+		hash = dtrace_byname;
+	}
+
+	/*
+	 * If we did not select a hash table, iterate over every probe and
+	 * invoke our callback for each one that matches our input probe key.
+	 */
+	if (hash == NULL) {
+		for (i = 0; i < dtrace_nprobes; i++) {
+			if ((probe = dtrace_probes[i]) == NULL ||
+			    dtrace_match_probe(probe, pkp, priv, uid,
+			    zoneid) <= 0)
+				continue;
+
+			nmatched++;
+
+			if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
+				break;
+		}
+
+		return (nmatched);
+	}
+
+	/*
+	 * If we selected a hash table, iterate over each probe of the same key
+	 * name and invoke the callback for every probe that matches the other
+	 * attributes of our input probe key.
+	 */
+	for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
+	    probe = *(DTRACE_HASHNEXT(hash, probe))) {
+
+		if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0)
+			continue;
+
+		nmatched++;
+
+		if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
+			break;
+	}
+
+	return (nmatched);
+}
+
+/*
+ * Return the function pointer dtrace_probecmp() should use to compare the
+ * specified pattern with a string.  For NULL or empty patterns, we select
+ * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
+ * For non-empty non-glob strings, we use dtrace_match_string().
+ */
+static dtrace_probekey_f *
+dtrace_probekey_func(const char *p)
+{
+	char c;
+
+	if (p == NULL || *p == '\0')
+		return (&dtrace_match_nul);
+
+	while ((c = *p++) != '\0') {
+		if (c == '[' || c == '?' || c == '*' || c == '\\')
+			return (&dtrace_match_glob);
+	}
+
+	return (&dtrace_match_string);
+}
+
+/*
+ * Build a probe comparison key for use with dtrace_match_probe() from the
+ * given probe description.  By convention, a null key only matches anchored
+ * probes: if each field is the empty string, reset dtpk_fmatch to
+ * dtrace_match_nonzero().
+ */
+static void
+dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
+{
+	pkp->dtpk_prov = pdp->dtpd_provider;
+	pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
+
+	pkp->dtpk_mod = pdp->dtpd_mod;
+	pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
+
+	pkp->dtpk_func = pdp->dtpd_func;
+	pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
+
+	pkp->dtpk_name = pdp->dtpd_name;
+	pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
+
+	pkp->dtpk_id = pdp->dtpd_id;
+
+	if (pkp->dtpk_id == DTRACE_IDNONE &&
+	    pkp->dtpk_pmatch == &dtrace_match_nul &&
+	    pkp->dtpk_mmatch == &dtrace_match_nul &&
+	    pkp->dtpk_fmatch == &dtrace_match_nul &&
+	    pkp->dtpk_nmatch == &dtrace_match_nul)
+		pkp->dtpk_fmatch = &dtrace_match_nonzero;
+}
+
+/*
+ * DTrace Provider-to-Framework API Functions
+ *
+ * These functions implement much of the Provider-to-Framework API, as
+ * described in <sys/dtrace.h>.  The parts of the API not in this section are
+ * the functions in the API for probe management (found below), and
+ * dtrace_probe() itself (found above).
+ */
+
+/*
+ * Register the calling provider with the DTrace framework.  This should
+ * generally be called by DTrace providers in their attach(9E) entry point.
+ */
+int
+dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
+    cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
+{
+	dtrace_provider_t *provider;
+
+	if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
+		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
+		    "arguments", name ? name : "<NULL>");
+		return (EINVAL);
+	}
+
+	if (name[0] == '\0' || dtrace_badname(name)) {
+		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
+		    "provider name", name);
+		return (EINVAL);
+	}
+
+	if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
+	    pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
+	    pops->dtps_destroy == NULL ||
+	    ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
+		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
+		    "provider ops", name);
+		return (EINVAL);
+	}
+
+	if (dtrace_badattr(&pap->dtpa_provider) ||
+	    dtrace_badattr(&pap->dtpa_mod) ||
+	    dtrace_badattr(&pap->dtpa_func) ||
+	    dtrace_badattr(&pap->dtpa_name) ||
+	    dtrace_badattr(&pap->dtpa_args)) {
+		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
+		    "provider attributes", name);
+		return (EINVAL);
+	}
+
+	if (priv & ~DTRACE_PRIV_ALL) {
+		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
+		    "privilege attributes", name);
+		return (EINVAL);
+	}
+
+	if ((priv & DTRACE_PRIV_KERNEL) &&
+	    (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
+	    pops->dtps_usermode == NULL) {
+		cmn_err(CE_WARN, "failed to register provider '%s': need "
+		    "dtps_usermode() op for given privilege attributes", name);
+		return (EINVAL);
+	}
+
+	provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
+	provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
+	(void) strcpy(provider->dtpv_name, name);
+
+	provider->dtpv_attr = *pap;
+	provider->dtpv_priv.dtpp_flags = priv;
+	if (cr != NULL) {
+		provider->dtpv_priv.dtpp_uid = crgetuid(cr);
+		provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr);
+	}
+	provider->dtpv_pops = *pops;
+
+	if (pops->dtps_provide == NULL) {
+		ASSERT(pops->dtps_provide_module != NULL);
+		provider->dtpv_pops.dtps_provide =
+		    (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
+	}
+
+	if (pops->dtps_provide_module == NULL) {
+		ASSERT(pops->dtps_provide != NULL);
+		provider->dtpv_pops.dtps_provide_module =
+		    (void (*)(void *, struct modctl *))dtrace_nullop;
+	}
+
+	if (pops->dtps_suspend == NULL) {
+		ASSERT(pops->dtps_resume == NULL);
+		provider->dtpv_pops.dtps_suspend =
+		    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
+		provider->dtpv_pops.dtps_resume =
+		    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
+	}
+
+	provider->dtpv_arg = arg;
+	*idp = (dtrace_provider_id_t)provider;
+
+	if (pops == &dtrace_provider_ops) {
+		lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED);
+		lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+		ASSERT(dtrace_anon.dta_enabling == NULL);
+
+		/*
+		 * We make sure that the DTrace provider is at the head of
+		 * the provider chain.
+		 */
+		provider->dtpv_next = dtrace_provider;
+		dtrace_provider = provider;
+		return (0);
+	}
+
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	/*
+	 * If there is at least one provider registered, we'll add this
+	 * provider after the first provider.
+	 */
+	if (dtrace_provider != NULL) {
+		provider->dtpv_next = dtrace_provider->dtpv_next;
+		dtrace_provider->dtpv_next = provider;
+	} else {
+		dtrace_provider = provider;
+	}
+
+	if (dtrace_retained != NULL) {
+		dtrace_enabling_provide(provider);
+
+		/*
+		 * Now we need to call dtrace_enabling_matchall() -- which
+		 * will acquire cpu_lock and dtrace_lock.  We therefore need
+		 * to drop all of our locks before calling into it...
+		 */
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+		dtrace_enabling_matchall();
+
+		return (0);
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	return (0);
+}
+
+/*
+ * Unregister the specified provider from the DTrace framework.  This should
+ * generally be called by DTrace providers in their detach(9E) entry point.
+ */
+int
+dtrace_unregister(dtrace_provider_id_t id)
+{
+	dtrace_provider_t *old = (dtrace_provider_t *)id;
+	dtrace_provider_t *prev = NULL;
+	int i, self = 0;
+	dtrace_probe_t *probe, *first = NULL;
+
+	if (old->dtpv_pops.dtps_enable ==
+	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) {
+		/*
+		 * If DTrace itself is the provider, we're called with locks
+		 * already held.
+		 */
+		ASSERT(old == dtrace_provider);
+		ASSERT(dtrace_devi != NULL);
+		lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED);
+		lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+		self = 1;
+
+		if (dtrace_provider->dtpv_next != NULL) {
+			/*
+			 * There's another provider here; return failure.
+			 */
+			return (EBUSY);
+		}
+	} else {
+		lck_mtx_lock(&dtrace_provider_lock);
+		lck_mtx_lock(&mod_lock);
+		lck_mtx_lock(&dtrace_lock);
+	}
+
+	/*
+	 * If anyone has /dev/dtrace open, or if there are anonymous enabled
+	 * probes, we refuse to let providers slither away, unless this
+	 * provider has already been explicitly invalidated.
+	 */
+	if (!old->dtpv_defunct &&
+	    (dtrace_opens || (dtrace_anon.dta_state != NULL &&
+	    dtrace_anon.dta_state->dts_necbs > 0))) {
+		if (!self) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&mod_lock);
+			lck_mtx_unlock(&dtrace_provider_lock);
+		}
+		return (EBUSY);
+	}
+
+	/*
+	 * Attempt to destroy the probes associated with this provider.
+	 */
+	for (i = 0; i < dtrace_nprobes; i++) {
+		if ((probe = dtrace_probes[i]) == NULL)
+			continue;
+
+		if (probe->dtpr_provider != old)
+			continue;
+
+		if (probe->dtpr_ecb == NULL)
+			continue;
+
+		/*
+		 * We have at least one ECB; we can't remove this provider.
+		 */
+		if (!self) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&mod_lock);
+			lck_mtx_unlock(&dtrace_provider_lock);
+		}
+		return (EBUSY);
+	}
+
+	/*
+	 * All of the probes for this provider are disabled; we can safely
+	 * remove all of them from their hash chains and from the probe array.
+	 */
+	for (i = 0; i < dtrace_nprobes; i++) {
+		if ((probe = dtrace_probes[i]) == NULL)
+			continue;
+
+		if (probe->dtpr_provider != old)
+			continue;
+
+		dtrace_probes[i] = NULL;
+
+		dtrace_hash_remove(dtrace_bymod, probe);
+		dtrace_hash_remove(dtrace_byfunc, probe);
+		dtrace_hash_remove(dtrace_byname, probe);
+
+		if (first == NULL) {
+			first = probe;
+			probe->dtpr_nextmod = NULL;
+		} else {
+			probe->dtpr_nextmod = first;
+			first = probe;
+		}
+	}
+
+	/*
+	 * The provider's probes have been removed from the hash chains and
+	 * from the probe array.  Now issue a dtrace_sync() to be sure that
+	 * everyone has cleared out from any probe array processing.
+	 */
+	dtrace_sync();
+
+	for (probe = first; probe != NULL; probe = first) {
+		first = probe->dtpr_nextmod;
+
+		old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
+		    probe->dtpr_arg);
+		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
+		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
+		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
+		vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
+#if !defined(__APPLE__)
+		kmem_free(probe, sizeof (dtrace_probe_t));
+#else
+		zfree(dtrace_probe_t_zone, probe);
+#endif
+	}
+
+	if ((prev = dtrace_provider) == old) {
+		ASSERT(self || dtrace_devi == NULL);
+		ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
+		dtrace_provider = old->dtpv_next;
+	} else {
+		while (prev != NULL && prev->dtpv_next != old)
+			prev = prev->dtpv_next;
+
+		if (prev == NULL) {
+			panic("attempt to unregister non-existent "
+			    "dtrace provider %p\n", (void *)id);
+		}
+
+		prev->dtpv_next = old->dtpv_next;
+	}
+
+	if (!self) {
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&mod_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+	}
+
+	kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
+	kmem_free(old, sizeof (dtrace_provider_t));
+
+	return (0);
+}
+
+/*
+ * Invalidate the specified provider.  All subsequent probe lookups for the
+ * specified provider will fail, but its probes will not be removed.
+ */
+void
+dtrace_invalidate(dtrace_provider_id_t id)
+{
+	dtrace_provider_t *pvp = (dtrace_provider_t *)id;
+
+	ASSERT(pvp->dtpv_pops.dtps_enable !=
+	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
+
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	pvp->dtpv_defunct = 1;
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+}
+
+/*
+ * Indicate whether or not DTrace has attached.
+ */
+int
+dtrace_attached(void)
+{
+	/*
+	 * dtrace_provider will be non-NULL iff the DTrace driver has
+	 * attached.  (It's non-NULL because DTrace is always itself a
+	 * provider.)
+	 */
+	return (dtrace_provider != NULL);
+}
+
+/*
+ * Remove all the unenabled probes for the given provider.  This function is
+ * not unlike dtrace_unregister(), except that it doesn't remove the provider
+ * -- just as many of its associated probes as it can.
+ */
+int
+dtrace_condense(dtrace_provider_id_t id)
+{
+	dtrace_provider_t *prov = (dtrace_provider_t *)id;
+	int i;
+	dtrace_probe_t *probe;
+
+	/*
+	 * Make sure this isn't the dtrace provider itself.
+	 */
+	ASSERT(prov->dtpv_pops.dtps_enable !=
+	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
+
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	/*
+	 * Attempt to destroy the probes associated with this provider.
+	 */
+	for (i = 0; i < dtrace_nprobes; i++) {
+		if ((probe = dtrace_probes[i]) == NULL)
+			continue;
+
+		if (probe->dtpr_provider != prov)
+			continue;
+
+		if (probe->dtpr_ecb != NULL)
+			continue;
+
+		dtrace_probes[i] = NULL;
+
+		dtrace_hash_remove(dtrace_bymod, probe);
+		dtrace_hash_remove(dtrace_byfunc, probe);
+		dtrace_hash_remove(dtrace_byname, probe);
+
+		prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
+		    probe->dtpr_arg);
+		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
+		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
+		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
+#if !defined(__APPLE__)
+		kmem_free(probe, sizeof (dtrace_probe_t));
+#else
+		zfree(dtrace_probe_t_zone, probe);
+#endif
+		vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	return (0);
+}
+
+/*
+ * DTrace Probe Management Functions
+ *
+ * The functions in this section perform the DTrace probe management,
+ * including functions to create probes, look-up probes, and call into the
+ * providers to request that probes be provided.  Some of these functions are
+ * in the Provider-to-Framework API; these functions can be identified by the
+ * fact that they are not declared "static".
+ */
+
+/*
+ * Create a probe with the specified module name, function name, and name.
+ */
+dtrace_id_t
+dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
+    const char *func, const char *name, int aframes, void *arg)
+{
+	dtrace_probe_t *probe, **probes;
+	dtrace_provider_t *provider = (dtrace_provider_t *)prov;
+	dtrace_id_t id;
+
+	if (provider == dtrace_provider) {
+		lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	} else {
+		lck_mtx_lock(&dtrace_lock);
+	}
+
+	id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
+	    VM_BESTFIT | VM_SLEEP);
+#if !defined(__APPLE__)
+	probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
+#else
+	probe = zalloc(dtrace_probe_t_zone);
+	bzero(probe, sizeof (dtrace_probe_t));
+#endif
+
+	probe->dtpr_id = id;
+	probe->dtpr_gen = dtrace_probegen++;
+	probe->dtpr_mod = dtrace_strdup(mod);
+	probe->dtpr_func = dtrace_strdup(func);
+	probe->dtpr_name = dtrace_strdup(name);
+	probe->dtpr_arg = arg;
+	probe->dtpr_aframes = aframes;
+	probe->dtpr_provider = provider;
+
+	dtrace_hash_add(dtrace_bymod, probe);
+	dtrace_hash_add(dtrace_byfunc, probe);
+	dtrace_hash_add(dtrace_byname, probe);
+
+	if (id - 1 >= dtrace_nprobes) {
+		size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
+		size_t nsize = osize << 1;
+
+		if (nsize == 0) {
+			ASSERT(osize == 0);
+			ASSERT(dtrace_probes == NULL);
+			nsize = sizeof (dtrace_probe_t *);
+		}
+
+		probes = kmem_zalloc(nsize, KM_SLEEP);
+
+		if (dtrace_probes == NULL) {
+			ASSERT(osize == 0);
+			dtrace_probes = probes;
+			dtrace_nprobes = 1;
+		} else {
+			dtrace_probe_t **oprobes = dtrace_probes;
+
+			bcopy(oprobes, probes, osize);
+			dtrace_membar_producer();
+			dtrace_probes = probes;
+
+			dtrace_sync();
+
+			/*
+			 * All CPUs are now seeing the new probes array; we can
+			 * safely free the old array.
+			 */
+			kmem_free(oprobes, osize);
+			dtrace_nprobes <<= 1;
+		}
+
+		ASSERT(id - 1 < dtrace_nprobes);
+	}
+
+	ASSERT(dtrace_probes[id - 1] == NULL);
+	dtrace_probes[id - 1] = probe;
+
+	if (provider != dtrace_provider)
+		lck_mtx_unlock(&dtrace_lock);
+
+	return (id);
+}
+
+static dtrace_probe_t *
+dtrace_probe_lookup_id(dtrace_id_t id)
+{
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (id == 0 || id > dtrace_nprobes)
+		return (NULL);
+
+	return (dtrace_probes[id - 1]);
+}
+
+static int
+dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
+{
+	*((dtrace_id_t *)arg) = probe->dtpr_id;
+
+	return (DTRACE_MATCH_DONE);
+}
+
+/*
+ * Look up a probe based on provider and one or more of module name, function
+ * name and probe name.
+ */
+dtrace_id_t
+dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
+    const char *func, const char *name)
+{
+	dtrace_probekey_t pkey;
+	dtrace_id_t id;
+	int match;
+
+	pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
+	pkey.dtpk_pmatch = &dtrace_match_string;
+	pkey.dtpk_mod = mod;
+	pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
+	pkey.dtpk_func = func;
+	pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
+	pkey.dtpk_name = name;
+	pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
+	pkey.dtpk_id = DTRACE_IDNONE;
+
+	lck_mtx_lock(&dtrace_lock);
+	match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0,
+	    dtrace_probe_lookup_match, &id);
+	lck_mtx_unlock(&dtrace_lock);
+
+	ASSERT(match == 1 || match == 0);
+	return (match ? id : 0);
+}
+
+/*
+ * Returns the probe argument associated with the specified probe.
+ */
+void *
+dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
+{
+	dtrace_probe_t *probe;
+	void *rval = NULL;
+
+	lck_mtx_lock(&dtrace_lock);
+
+	if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
+	    probe->dtpr_provider == (dtrace_provider_t *)id)
+		rval = probe->dtpr_arg;
+
+	lck_mtx_unlock(&dtrace_lock);
+
+	return (rval);
+}
+
+/*
+ * Copy a probe into a probe description.
+ */
+static void
+dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
+{
+	bzero(pdp, sizeof (dtrace_probedesc_t));
+	pdp->dtpd_id = prp->dtpr_id;
+
+	(void) strlcpy(pdp->dtpd_provider,
+	    prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN);
+
+	(void) strlcpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN);
+	(void) strlcpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN);
+	(void) strlcpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN);
+}
+
+/*
+ * Called to indicate that a probe -- or probes -- should be provided by a
+ * specfied provider.  If the specified description is NULL, the provider will
+ * be told to provide all of its probes.  (This is done whenever a new
+ * consumer comes along, or whenever a retained enabling is to be matched.) If
+ * the specified description is non-NULL, the provider is given the
+ * opportunity to dynamically provide the specified probe, allowing providers
+ * to support the creation of probes on-the-fly.  (So-called _autocreated_
+ * probes.)  If the provider is NULL, the operations will be applied to all
+ * providers; if the provider is non-NULL the operations will only be applied
+ * to the specified provider.  The dtrace_provider_lock must be held, and the
+ * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
+ * will need to grab the dtrace_lock when it reenters the framework through
+ * dtrace_probe_lookup(), dtrace_probe_create(), etc.
+ */
+static void
+dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
+{
+	struct modctl *ctl;
+	int all = 0;
+
+	lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (prv == NULL) {
+		all = 1;
+		prv = dtrace_provider;
+	}
+
+	do {
+		kmod_info_t *ktl;
+		/*
+		 * First, call the blanket provide operation.
+		 */
+		prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
+
+#if !defined(__APPLE__)
+		/*
+		 * Now call the per-module provide operation.  We will grab
+		 * mod_lock to prevent the list from being modified.  Note
+		 * that this also prevents the mod_busy bits from changing.
+		 * (mod_busy can only be changed with mod_lock held.)
+		 */
+		lck_mtx_lock(&mod_lock);
+
+		ctl = &modules;
+		do {
+			if (ctl->mod_busy || ctl->mod_mp == NULL)
+				continue;
+
+			prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
+
+		} while ((ctl = ctl->mod_next) != &modules);
+
+		lck_mtx_unlock(&mod_lock);
+#else
+#if 0 /* XXX Workaround for PR_4643546 XXX */
+		simple_lock(&kmod_lock);
+		
+		ktl = kmod;
+		while (ktl) {
+			prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ktl);
+			ktl = ktl->next;
+		}
+		
+		simple_unlock(&kmod_lock);
+#else
+		/*
+		 * Don't bother to iterate over the kmod list. At present only fbt
+		 * offers a provide_module in its dtpv_pops, and then it ignores the
+		 * module anyway.
+		 */
+		prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, NULL);
+#endif
+#endif /* __APPLE__ */
+	} while (all && (prv = prv->dtpv_next) != NULL);
+}
+
+/*
+ * Iterate over each probe, and call the Framework-to-Provider API function
+ * denoted by offs.
+ */
+static void
+dtrace_probe_foreach(uintptr_t offs)
+{
+	dtrace_provider_t *prov;
+	void (*func)(void *, dtrace_id_t, void *);
+	dtrace_probe_t *probe;
+	dtrace_icookie_t cookie;
+	int i;
+
+	/*
+	 * We disable interrupts to walk through the probe array.  This is
+	 * safe -- the dtrace_sync() in dtrace_unregister() assures that we
+	 * won't see stale data.
+	 */
+	cookie = dtrace_interrupt_disable();
+
+	for (i = 0; i < dtrace_nprobes; i++) {
+		if ((probe = dtrace_probes[i]) == NULL)
+			continue;
+
+		if (probe->dtpr_ecb == NULL) {
+			/*
+			 * This probe isn't enabled -- don't call the function.
+			 */
+			continue;
+		}
+
+		prov = probe->dtpr_provider;
+		func = *((void(**)(void *, dtrace_id_t, void *))
+		    ((uintptr_t)&prov->dtpv_pops + offs));
+
+		func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
+	}
+
+	dtrace_interrupt_enable(cookie);
+}
+
+static int
+dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
+{
+	dtrace_probekey_t pkey;
+	uint32_t priv;
+	uid_t uid;
+	zoneid_t zoneid;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	dtrace_ecb_create_cache = NULL;
+
+	if (desc == NULL) {
+		/*
+		 * If we're passed a NULL description, we're being asked to
+		 * create an ECB with a NULL probe.
+		 */
+		(void) dtrace_ecb_create_enable(NULL, enab);
+		return (0);
+	}
+
+	dtrace_probekey(desc, &pkey);
+	dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred,
+	    &priv, &uid, &zoneid);
+
+	return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable,
+	    enab));
+}
+
+/*
+ * DTrace Helper Provider Functions
+ */
+static void
+dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
+{
+	attr->dtat_name = DOF_ATTR_NAME(dofattr);
+	attr->dtat_data = DOF_ATTR_DATA(dofattr);
+	attr->dtat_class = DOF_ATTR_CLASS(dofattr);
+}
+
+static void
+dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
+    const dof_provider_t *dofprov, char *strtab)
+{
+	hprov->dthpv_provname = strtab + dofprov->dofpv_name;
+	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
+	    dofprov->dofpv_provattr);
+	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
+	    dofprov->dofpv_modattr);
+	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
+	    dofprov->dofpv_funcattr);
+	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
+	    dofprov->dofpv_nameattr);
+	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
+	    dofprov->dofpv_argsattr);
+}
+
+static void
+dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
+{
+	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
+	dof_hdr_t *dof = (dof_hdr_t *)daddr;
+	dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
+	dof_provider_t *provider;
+	dof_probe_t *probe;
+	uint32_t *off, *enoff;
+	uint8_t *arg;
+	char *strtab;
+	uint_t i, nprobes;
+	dtrace_helper_provdesc_t dhpv;
+	dtrace_helper_probedesc_t dhpb;
+	dtrace_meta_t *meta = dtrace_meta_pid;
+	dtrace_mops_t *mops = &meta->dtm_mops;
+	void *parg;
+
+	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+	    provider->dofpv_strtab * dof->dofh_secsize);
+	prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+	    provider->dofpv_probes * dof->dofh_secsize);
+	arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+	    provider->dofpv_prargs * dof->dofh_secsize);
+	off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+	    provider->dofpv_proffs * dof->dofh_secsize);
+
+	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
+	off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
+	arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
+	enoff = NULL;
+
+	/*
+	 * See dtrace_helper_provider_validate().
+	 */
+	if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
+	    provider->dofpv_prenoffs != DOF_SECT_NONE) {
+		enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+		    provider->dofpv_prenoffs * dof->dofh_secsize);
+		enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset);
+	}
+
+	nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
+
+	/*
+	 * Create the provider.
+	 */
+	dtrace_dofprov2hprov(&dhpv, provider, strtab);
+
+	if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
+		return;
+
+	meta->dtm_count++;
+
+	/*
+	 * Create the probes.
+	 */
+	for (i = 0; i < nprobes; i++) {
+		probe = (dof_probe_t *)(uintptr_t)(daddr +
+		    prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
+
+		dhpb.dthpb_mod = dhp->dofhp_mod;
+		dhpb.dthpb_func = strtab + probe->dofpr_func;
+		dhpb.dthpb_name = strtab + probe->dofpr_name;
+#if defined(__APPLE__)
+		dhpb.dthpb_base = dhp->dofhp_addr;
+#else
+		dhpb.dthpb_base = probe->dofpr_addr;
+#endif
+		dhpb.dthpb_offs = off + probe->dofpr_offidx;
+		dhpb.dthpb_noffs = probe->dofpr_noffs;
+		if (enoff != NULL) {
+			dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx;
+			dhpb.dthpb_nenoffs = probe->dofpr_nenoffs;
+		} else {
+			dhpb.dthpb_enoffs = NULL;
+			dhpb.dthpb_nenoffs = 0;
+		}
+		dhpb.dthpb_args = arg + probe->dofpr_argidx;
+		dhpb.dthpb_nargc = probe->dofpr_nargc;
+		dhpb.dthpb_xargc = probe->dofpr_xargc;
+		dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
+		dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
+
+		mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
+	}
+}
+
+static void
+dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
+{
+	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
+	dof_hdr_t *dof = (dof_hdr_t *)daddr;
+	int i;
+
+	lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
+		    dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (sec->dofs_type != DOF_SECT_PROVIDER)
+			continue;
+
+		dtrace_helper_provide_one(dhp, sec, pid);
+	}
+
+	/*
+	 * We may have just created probes, so we must now rematch against
+	 * any retained enablings.  Note that this call will acquire both
+	 * cpu_lock and dtrace_lock; the fact that we are holding
+	 * dtrace_meta_lock now is what defines the ordering with respect to
+	 * these three locks.
+	 */
+	dtrace_enabling_matchall();
+}
+
+static void
+dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
+{
+	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
+	dof_hdr_t *dof = (dof_hdr_t *)daddr;
+	dof_sec_t *str_sec;
+	dof_provider_t *provider;
+	char *strtab;
+	dtrace_helper_provdesc_t dhpv;
+	dtrace_meta_t *meta = dtrace_meta_pid;
+	dtrace_mops_t *mops = &meta->dtm_mops;
+
+	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
+	    provider->dofpv_strtab * dof->dofh_secsize);
+
+	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
+
+	/*
+	 * Create the provider.
+	 */
+	dtrace_dofprov2hprov(&dhpv, provider, strtab);
+
+	mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
+
+	meta->dtm_count--;
+}
+
+static void
+dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid)
+{
+	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
+	dof_hdr_t *dof = (dof_hdr_t *)daddr;
+	int i;
+
+	lck_mtx_assert(&dtrace_meta_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
+		    dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (sec->dofs_type != DOF_SECT_PROVIDER)
+			continue;
+
+		dtrace_helper_provider_remove_one(dhp, sec, pid);
+	}
+}
+
+/*
+ * DTrace Meta Provider-to-Framework API Functions
+ *
+ * These functions implement the Meta Provider-to-Framework API, as described
+ * in <sys/dtrace.h>.
+ */
+int
+dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
+    dtrace_meta_provider_id_t *idp)
+{
+	dtrace_meta_t *meta;
+	dtrace_helpers_t *help, *next;
+	int i;
+
+	*idp = DTRACE_METAPROVNONE;
+
+	/*
+	 * We strictly don't need the name, but we hold onto it for
+	 * debuggability. All hail error queues!
+	 */
+	if (name == NULL) {
+		cmn_err(CE_WARN, "failed to register meta-provider: "
+		    "invalid name");
+		return (EINVAL);
+	}
+
+	if (mops == NULL ||
+	    mops->dtms_create_probe == NULL ||
+	    mops->dtms_provide_pid == NULL ||
+	    mops->dtms_remove_pid == NULL) {
+		cmn_err(CE_WARN, "failed to register meta-register %s: "
+		    "invalid ops", name);
+		return (EINVAL);
+	}
+
+	meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
+	meta->dtm_mops = *mops;
+	meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
+	(void) strcpy(meta->dtm_name, name);
+	meta->dtm_arg = arg;
+
+	lck_mtx_lock(&dtrace_meta_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (dtrace_meta_pid != NULL) {
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&dtrace_meta_lock);
+		cmn_err(CE_WARN, "failed to register meta-register %s: "
+		    "user-land meta-provider exists", name);
+		kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
+		kmem_free(meta, sizeof (dtrace_meta_t));
+		return (EINVAL);
+	}
+
+	dtrace_meta_pid = meta;
+	*idp = (dtrace_meta_provider_id_t)meta;
+
+	/*
+	 * If there are providers and probes ready to go, pass them
+	 * off to the new meta provider now.
+	 */
+
+	help = dtrace_deferred_pid;
+	dtrace_deferred_pid = NULL;
+
+	lck_mtx_unlock(&dtrace_lock);
+
+	while (help != NULL) {
+		for (i = 0; i < help->dthps_nprovs; i++) {
+			dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
+			    help->dthps_pid);
+		}
+
+		next = help->dthps_next;
+		help->dthps_next = NULL;
+		help->dthps_prev = NULL;
+		help->dthps_deferred = 0;
+		help = next;
+	}
+
+	lck_mtx_unlock(&dtrace_meta_lock);
+
+	return (0);
+}
+
+int
+dtrace_meta_unregister(dtrace_meta_provider_id_t id)
+{
+	dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
+
+	lck_mtx_lock(&dtrace_meta_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (old == dtrace_meta_pid) {
+		pp = &dtrace_meta_pid;
+	} else {
+		panic("attempt to unregister non-existent "
+		    "dtrace meta-provider %p\n", (void *)old);
+	}
+
+	if (old->dtm_count != 0) {
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&dtrace_meta_lock);
+		return (EBUSY);
+	}
+
+	*pp = NULL;
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_meta_lock);
+
+	kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
+	kmem_free(old, sizeof (dtrace_meta_t));
+
+	return (0);
+}
+
+
+/*
+ * DTrace DIF Object Functions
+ */
+static int
+dtrace_difo_err(uint_t pc, const char *format, ...)
+{
+	if (dtrace_err_verbose) {
+		va_list alist;
+
+		(void) uprintf("dtrace DIF object error: [%u]: ", pc);
+		va_start(alist, format);
+		(void) vuprintf(format, alist);
+		va_end(alist);
+	}
+
+#ifdef DTRACE_ERRDEBUG
+	dtrace_errdebug(format);
+#endif
+	return (1);
+}
+
+/*
+ * Validate a DTrace DIF object by checking the IR instructions.  The following
+ * rules are currently enforced by dtrace_difo_validate():
+ *
+ * 1. Each instruction must have a valid opcode
+ * 2. Each register, string, variable, or subroutine reference must be valid
+ * 3. No instruction can modify register %r0 (must be zero)
+ * 4. All instruction reserved bits must be set to zero
+ * 5. The last instruction must be a "ret" instruction
+ * 6. All branch targets must reference a valid instruction _after_ the branch
+ */
+static int
+dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
+    cred_t *cr)
+{
+	int err = 0, i;
+	int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
+	int kcheck;
+	uint_t pc;
+
+	kcheck = cr == NULL ||
+	    PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE) == 0;
+
+	dp->dtdo_destructive = 0;
+
+	for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
+		dif_instr_t instr = dp->dtdo_buf[pc];
+
+		uint_t r1 = DIF_INSTR_R1(instr);
+		uint_t r2 = DIF_INSTR_R2(instr);
+		uint_t rd = DIF_INSTR_RD(instr);
+		uint_t rs = DIF_INSTR_RS(instr);
+		uint_t label = DIF_INSTR_LABEL(instr);
+		uint_t v = DIF_INSTR_VAR(instr);
+		uint_t subr = DIF_INSTR_SUBR(instr);
+		uint_t type = DIF_INSTR_TYPE(instr);
+		uint_t op = DIF_INSTR_OP(instr);
+
+		switch (op) {
+		case DIF_OP_OR:
+		case DIF_OP_XOR:
+		case DIF_OP_AND:
+		case DIF_OP_SLL:
+		case DIF_OP_SRL:
+		case DIF_OP_SRA:
+		case DIF_OP_SUB:
+		case DIF_OP_ADD:
+		case DIF_OP_MUL:
+		case DIF_OP_SDIV:
+		case DIF_OP_UDIV:
+		case DIF_OP_SREM:
+		case DIF_OP_UREM:
+		case DIF_OP_COPYS:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r2);
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_NOT:
+		case DIF_OP_MOV:
+		case DIF_OP_ALLOCS:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_LDSB:
+		case DIF_OP_LDSH:
+		case DIF_OP_LDSW:
+		case DIF_OP_LDUB:
+		case DIF_OP_LDUH:
+		case DIF_OP_LDUW:
+		case DIF_OP_LDX:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			if (kcheck)
+				dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
+				    DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
+			break;
+		case DIF_OP_RLDSB:
+		case DIF_OP_RLDSH:
+		case DIF_OP_RLDSW:
+		case DIF_OP_RLDUB:
+		case DIF_OP_RLDUH:
+		case DIF_OP_RLDUW:
+		case DIF_OP_RLDX:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_ULDSB:
+		case DIF_OP_ULDSH:
+		case DIF_OP_ULDSW:
+		case DIF_OP_ULDUB:
+		case DIF_OP_ULDUH:
+		case DIF_OP_ULDUW:
+		case DIF_OP_ULDX:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_STB:
+		case DIF_OP_STH:
+		case DIF_OP_STW:
+		case DIF_OP_STX:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to 0 address\n");
+			break;
+		case DIF_OP_CMP:
+		case DIF_OP_SCMP:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r2);
+			if (rd != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			break;
+		case DIF_OP_TST:
+			if (r1 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r1);
+			if (r2 != 0 || rd != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			break;
+		case DIF_OP_BA:
+		case DIF_OP_BE:
+		case DIF_OP_BNE:
+		case DIF_OP_BG:
+		case DIF_OP_BGU:
+		case DIF_OP_BGE:
+		case DIF_OP_BGEU:
+		case DIF_OP_BL:
+		case DIF_OP_BLU:
+		case DIF_OP_BLE:
+		case DIF_OP_BLEU:
+			if (label >= dp->dtdo_len) {
+				err += efunc(pc, "invalid branch target %u\n",
+				    label);
+			}
+			if (label <= pc) {
+				err += efunc(pc, "backward branch to %u\n",
+				    label);
+			}
+			break;
+		case DIF_OP_RET:
+			if (r1 != 0 || r2 != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			break;
+		case DIF_OP_NOP:
+		case DIF_OP_POPTS:
+		case DIF_OP_FLUSHTS:
+			if (r1 != 0 || r2 != 0 || rd != 0)
+				err += efunc(pc, "non-zero reserved bits\n");
+			break;
+		case DIF_OP_SETX:
+			if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
+				err += efunc(pc, "invalid integer ref %u\n",
+				    DIF_INSTR_INTEGER(instr));
+			}
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_SETS:
+			if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
+				err += efunc(pc, "invalid string ref %u\n",
+				    DIF_INSTR_STRING(instr));
+			}
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_LDGA:
+		case DIF_OP_LDTA:
+			if (r1 > DIF_VAR_ARRAY_MAX)
+				err += efunc(pc, "invalid array %u\n", r1);
+			if (r2 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r2);
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_LDGS:
+		case DIF_OP_LDTS:
+		case DIF_OP_LDLS:
+		case DIF_OP_LDGAA:
+		case DIF_OP_LDTAA:
+			if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
+				err += efunc(pc, "invalid variable %u\n", v);
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+			break;
+		case DIF_OP_STGS:
+		case DIF_OP_STTS:
+		case DIF_OP_STLS:
+		case DIF_OP_STGAA:
+		case DIF_OP_STTAA:
+			if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
+				err += efunc(pc, "invalid variable %u\n", v);
+			if (rs >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			break;
+		case DIF_OP_CALL:
+			if (subr > DIF_SUBR_MAX)
+				err += efunc(pc, "invalid subr %u\n", subr);
+			if (rd >= nregs)
+				err += efunc(pc, "invalid register %u\n", rd);
+			if (rd == 0)
+				err += efunc(pc, "cannot write to %r0\n");
+
+			if (subr == DIF_SUBR_COPYOUT ||
+			    subr == DIF_SUBR_COPYOUTSTR) {
+				dp->dtdo_destructive = 1;
+			}
+			break;
+		case DIF_OP_PUSHTR:
+			if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
+				err += efunc(pc, "invalid ref type %u\n", type);
+			if (r2 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r2);
+			if (rs >= nregs)
+				err += efunc(pc, "invalid register %u\n", rs);
+			break;
+		case DIF_OP_PUSHTV:
+			if (type != DIF_TYPE_CTF)
+				err += efunc(pc, "invalid val type %u\n", type);
+			if (r2 >= nregs)
+				err += efunc(pc, "invalid register %u\n", r2);
+			if (rs >= nregs)
+				err += efunc(pc, "invalid register %u\n", rs);
+			break;
+		default:
+			err += efunc(pc, "invalid opcode %u\n",
+			    DIF_INSTR_OP(instr));
+		}
+	}
+
+	if (dp->dtdo_len != 0 &&
+	    DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
+		err += efunc(dp->dtdo_len - 1,
+		    "expected 'ret' as last DIF instruction\n");
+	}
+
+	if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
+		/*
+		 * If we're not returning by reference, the size must be either
+		 * 0 or the size of one of the base types.
+		 */
+		switch (dp->dtdo_rtype.dtdt_size) {
+		case 0:
+		case sizeof (uint8_t):
+		case sizeof (uint16_t):
+		case sizeof (uint32_t):
+		case sizeof (uint64_t):
+			break;
+
+		default:
+			err += efunc(dp->dtdo_len - 1, "bad return size");
+		}
+	}
+
+	for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
+		dtrace_diftype_t *vt, *et;
+		uint_t id, ndx;
+
+		if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
+		    v->dtdv_scope != DIFV_SCOPE_THREAD &&
+		    v->dtdv_scope != DIFV_SCOPE_LOCAL) {
+			err += efunc(i, "unrecognized variable scope %d\n",
+			    v->dtdv_scope);
+			break;
+		}
+
+		if (v->dtdv_kind != DIFV_KIND_ARRAY &&
+		    v->dtdv_kind != DIFV_KIND_SCALAR) {
+			err += efunc(i, "unrecognized variable type %d\n",
+			    v->dtdv_kind);
+			break;
+		}
+
+		if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
+			err += efunc(i, "%d exceeds variable id limit\n", id);
+			break;
+		}
+
+		if (id < DIF_VAR_OTHER_UBASE)
+			continue;
+
+		/*
+		 * For user-defined variables, we need to check that this
+		 * definition is identical to any previous definition that we
+		 * encountered.
+		 */
+		ndx = id - DIF_VAR_OTHER_UBASE;
+
+		switch (v->dtdv_scope) {
+		case DIFV_SCOPE_GLOBAL:
+			if (ndx < vstate->dtvs_nglobals) {
+				dtrace_statvar_t *svar;
+
+				if ((svar = vstate->dtvs_globals[ndx]) != NULL)
+					existing = &svar->dtsv_var;
+			}
+
+			break;
+
+		case DIFV_SCOPE_THREAD:
+			if (ndx < vstate->dtvs_ntlocals)
+				existing = &vstate->dtvs_tlocals[ndx];
+			break;
+
+		case DIFV_SCOPE_LOCAL:
+			if (ndx < vstate->dtvs_nlocals) {
+				dtrace_statvar_t *svar;
+
+				if ((svar = vstate->dtvs_locals[ndx]) != NULL)
+					existing = &svar->dtsv_var;
+			}
+
+			break;
+		}
+
+		vt = &v->dtdv_type;
+
+		if (vt->dtdt_flags & DIF_TF_BYREF) {
+			if (vt->dtdt_size == 0) {
+				err += efunc(i, "zero-sized variable\n");
+				break;
+			}
+
+			if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
+			    vt->dtdt_size > dtrace_global_maxsize) {
+				err += efunc(i, "oversized by-ref global\n");
+				break;
+			}
+		}
+
+		if (existing == NULL || existing->dtdv_id == 0)
+			continue;
+
+		ASSERT(existing->dtdv_id == v->dtdv_id);
+		ASSERT(existing->dtdv_scope == v->dtdv_scope);
+
+		if (existing->dtdv_kind != v->dtdv_kind)
+			err += efunc(i, "%d changed variable kind\n", id);
+
+		et = &existing->dtdv_type;
+
+		if (vt->dtdt_flags != et->dtdt_flags) {
+			err += efunc(i, "%d changed variable type flags\n", id);
+			break;
+		}
+
+		if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
+			err += efunc(i, "%d changed variable type size\n", id);
+			break;
+		}
+	}
+
+	return (err);
+}
+
+/*
+ * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
+ * are much more constrained than normal DIFOs.  Specifically, they may
+ * not:
+ *
+ * 1. Make calls to subroutines other than copyin(), copyinstr() or
+ *    miscellaneous string routines
+ * 2. Access DTrace variables other than the args[] array, and the
+ *    curthread, pid, ppid, tid, execname, zonename, uid and gid variables.
+ * 3. Have thread-local variables.
+ * 4. Have dynamic variables.
+ */
+static int
+dtrace_difo_validate_helper(dtrace_difo_t *dp)
+{
+	int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
+	int err = 0;
+	uint_t pc;
+
+	for (pc = 0; pc < dp->dtdo_len; pc++) {
+		dif_instr_t instr = dp->dtdo_buf[pc];
+
+		uint_t v = DIF_INSTR_VAR(instr);
+		uint_t subr = DIF_INSTR_SUBR(instr);
+		uint_t op = DIF_INSTR_OP(instr);
+
+		switch (op) {
+		case DIF_OP_OR:
+		case DIF_OP_XOR:
+		case DIF_OP_AND:
+		case DIF_OP_SLL:
+		case DIF_OP_SRL:
+		case DIF_OP_SRA:
+		case DIF_OP_SUB:
+		case DIF_OP_ADD:
+		case DIF_OP_MUL:
+		case DIF_OP_SDIV:
+		case DIF_OP_UDIV:
+		case DIF_OP_SREM:
+		case DIF_OP_UREM:
+		case DIF_OP_COPYS:
+		case DIF_OP_NOT:
+		case DIF_OP_MOV:
+		case DIF_OP_RLDSB:
+		case DIF_OP_RLDSH:
+		case DIF_OP_RLDSW:
+		case DIF_OP_RLDUB:
+		case DIF_OP_RLDUH:
+		case DIF_OP_RLDUW:
+		case DIF_OP_RLDX:
+		case DIF_OP_ULDSB:
+		case DIF_OP_ULDSH:
+		case DIF_OP_ULDSW:
+		case DIF_OP_ULDUB:
+		case DIF_OP_ULDUH:
+		case DIF_OP_ULDUW:
+		case DIF_OP_ULDX:
+		case DIF_OP_STB:
+		case DIF_OP_STH:
+		case DIF_OP_STW:
+		case DIF_OP_STX:
+		case DIF_OP_ALLOCS:
+		case DIF_OP_CMP:
+		case DIF_OP_SCMP:
+		case DIF_OP_TST:
+		case DIF_OP_BA:
+		case DIF_OP_BE:
+		case DIF_OP_BNE:
+		case DIF_OP_BG:
+		case DIF_OP_BGU:
+		case DIF_OP_BGE:
+		case DIF_OP_BGEU:
+		case DIF_OP_BL:
+		case DIF_OP_BLU:
+		case DIF_OP_BLE:
+		case DIF_OP_BLEU:
+		case DIF_OP_RET:
+		case DIF_OP_NOP:
+		case DIF_OP_POPTS:
+		case DIF_OP_FLUSHTS:
+		case DIF_OP_SETX:
+		case DIF_OP_SETS:
+		case DIF_OP_LDGA:
+		case DIF_OP_LDLS:
+		case DIF_OP_STGS:
+		case DIF_OP_STLS:
+		case DIF_OP_PUSHTR:
+		case DIF_OP_PUSHTV:
+			break;
+
+		case DIF_OP_LDGS:
+			if (v >= DIF_VAR_OTHER_UBASE)
+				break;
+
+			if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
+				break;
+
+			if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
+			    v == DIF_VAR_PPID || v == DIF_VAR_TID ||
+			    v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME ||
+			    v == DIF_VAR_UID || v == DIF_VAR_GID)
+				break;
+
+			err += efunc(pc, "illegal variable %u\n", v);
+			break;
+
+		case DIF_OP_LDTA:
+		case DIF_OP_LDTS:
+		case DIF_OP_LDGAA:
+		case DIF_OP_LDTAA:
+			err += efunc(pc, "illegal dynamic variable load\n");
+			break;
+
+		case DIF_OP_STTS:
+		case DIF_OP_STGAA:
+		case DIF_OP_STTAA:
+			err += efunc(pc, "illegal dynamic variable store\n");
+			break;
+
+		case DIF_OP_CALL:
+			if (subr == DIF_SUBR_ALLOCA ||
+			    subr == DIF_SUBR_BCOPY ||
+			    subr == DIF_SUBR_COPYIN ||
+			    subr == DIF_SUBR_COPYINTO ||
+			    subr == DIF_SUBR_COPYINSTR ||
+			    subr == DIF_SUBR_INDEX ||
+			    subr == DIF_SUBR_LLTOSTR ||
+			    subr == DIF_SUBR_RINDEX ||
+			    subr == DIF_SUBR_STRCHR ||
+			    subr == DIF_SUBR_STRJOIN ||
+			    subr == DIF_SUBR_STRRCHR ||
+			    subr == DIF_SUBR_STRSTR ||
+				subr == DIF_SUBR_CHUD)
+				break;
+
+			err += efunc(pc, "invalid subr %u\n", subr);
+			break;
+
+		default:
+			err += efunc(pc, "invalid opcode %u\n",
+			    DIF_INSTR_OP(instr));
+		}
+	}
+
+	return (err);
+}
+
+/*
+ * Returns 1 if the expression in the DIF object can be cached on a per-thread
+ * basis; 0 if not.
+ */
+static int
+dtrace_difo_cacheable(dtrace_difo_t *dp)
+{
+	int i;
+
+	if (dp == NULL)
+		return (0);
+
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+
+		if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
+			continue;
+
+		switch (v->dtdv_id) {
+		case DIF_VAR_CURTHREAD:
+		case DIF_VAR_PID:
+		case DIF_VAR_TID:
+		case DIF_VAR_EXECNAME:
+		case DIF_VAR_ZONENAME:
+			break;
+
+		default:
+			return (0);
+		}
+	}
+
+	/*
+	 * This DIF object may be cacheable.  Now we need to look for any
+	 * array loading instructions, any memory loading instructions, or
+	 * any stores to thread-local variables.
+	 */
+	for (i = 0; i < dp->dtdo_len; i++) {
+		uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
+
+		if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
+		    (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
+		    (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
+		    op == DIF_OP_LDGA || op == DIF_OP_STTS)
+			return (0);
+	}
+
+	return (1);
+}
+
+static void
+dtrace_difo_hold(dtrace_difo_t *dp)
+{
+	int i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	dp->dtdo_refcnt++;
+	ASSERT(dp->dtdo_refcnt != 0);
+
+	/*
+	 * We need to check this DIF object for references to the variable
+	 * DIF_VAR_VTIMESTAMP.
+	 */
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+
+		if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
+			continue;
+
+		if (dtrace_vtime_references++ == 0)
+			dtrace_vtime_enable();
+	}
+}
+
+/*
+ * This routine calculates the dynamic variable chunksize for a given DIF
+ * object.  The calculation is not fool-proof, and can probably be tricked by
+ * malicious DIF -- but it works for all compiler-generated DIF.  Because this
+ * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
+ * if a dynamic variable size exceeds the chunksize.
+ */
+static void
+dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
+{
+	uint64_t sval;
+	dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
+	const dif_instr_t *text = dp->dtdo_buf;
+	uint_t pc, srd = 0;
+	uint_t ttop = 0;
+	size_t size, ksize;
+	uint_t id, i;
+
+	for (pc = 0; pc < dp->dtdo_len; pc++) {
+		dif_instr_t instr = text[pc];
+		uint_t op = DIF_INSTR_OP(instr);
+		uint_t rd = DIF_INSTR_RD(instr);
+		uint_t r1 = DIF_INSTR_R1(instr);
+		uint_t nkeys = 0;
+		uchar_t scope;
+
+		dtrace_key_t *key = tupregs;
+
+		switch (op) {
+		case DIF_OP_SETX:
+			sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
+			srd = rd;
+			continue;
+
+		case DIF_OP_STTS:
+			key = &tupregs[DIF_DTR_NREGS];
+			key[0].dttk_size = 0;
+			key[1].dttk_size = 0;
+			nkeys = 2;
+			scope = DIFV_SCOPE_THREAD;
+			break;
+
+		case DIF_OP_STGAA:
+		case DIF_OP_STTAA:
+			nkeys = ttop;
+
+			if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
+				key[nkeys++].dttk_size = 0;
+
+			key[nkeys++].dttk_size = 0;
+
+			if (op == DIF_OP_STTAA) {
+				scope = DIFV_SCOPE_THREAD;
+			} else {
+				scope = DIFV_SCOPE_GLOBAL;
+			}
+
+			break;
+
+		case DIF_OP_PUSHTR:
+			if (ttop == DIF_DTR_NREGS)
+				return;
+
+			if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
+				/*
+				 * If the register for the size of the "pushtr"
+				 * is %r0 (or the value is 0) and the type is
+				 * a string, we'll use the system-wide default
+				 * string size.
+				 */
+				tupregs[ttop++].dttk_size =
+				    dtrace_strsize_default;
+			} else {
+				if (srd == 0)
+					return;
+
+				tupregs[ttop++].dttk_size = sval;
+			}
+
+			break;
+
+		case DIF_OP_PUSHTV:
+			if (ttop == DIF_DTR_NREGS)
+				return;
+
+			tupregs[ttop++].dttk_size = 0;
+			break;
+
+		case DIF_OP_FLUSHTS:
+			ttop = 0;
+			break;
+
+		case DIF_OP_POPTS:
+			if (ttop != 0)
+				ttop--;
+			break;
+		}
+
+		sval = 0;
+		srd = 0;
+
+		if (nkeys == 0)
+			continue;
+
+		/*
+		 * We have a dynamic variable allocation; calculate its size.
+		 */
+		for (ksize = 0, i = 0; i < nkeys; i++)
+			ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
+
+		size = sizeof (dtrace_dynvar_t);
+		size += sizeof (dtrace_key_t) * (nkeys - 1);
+		size += ksize;
+
+		/*
+		 * Now we need to determine the size of the stored data.
+		 */
+		id = DIF_INSTR_VAR(instr);
+
+		for (i = 0; i < dp->dtdo_varlen; i++) {
+			dtrace_difv_t *v = &dp->dtdo_vartab[i];
+
+			if (v->dtdv_id == id && v->dtdv_scope == scope) {
+				size += v->dtdv_type.dtdt_size;
+				break;
+			}
+		}
+
+		if (i == dp->dtdo_varlen)
+			return;
+
+		/*
+		 * We have the size.  If this is larger than the chunk size
+		 * for our dynamic variable state, reset the chunk size.
+		 */
+		size = P2ROUNDUP(size, sizeof (uint64_t));
+
+		if (size > vstate->dtvs_dynvars.dtds_chunksize)
+			vstate->dtvs_dynvars.dtds_chunksize = size;
+	}
+}
+
+static void
+dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
+{
+	int i, oldsvars, osz, nsz, otlocals, ntlocals;
+	uint_t id;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
+
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+		dtrace_statvar_t *svar, ***svarp;
+		size_t dsize = 0;
+		uint8_t scope = v->dtdv_scope;
+		int *np;
+
+		if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
+			continue;
+
+		id -= DIF_VAR_OTHER_UBASE;
+
+		switch (scope) {
+		case DIFV_SCOPE_THREAD:
+			while (id >= (otlocals = vstate->dtvs_ntlocals)) {
+				dtrace_difv_t *tlocals;
+
+				if ((ntlocals = (otlocals << 1)) == 0)
+					ntlocals = 1;
+
+				osz = otlocals * sizeof (dtrace_difv_t);
+				nsz = ntlocals * sizeof (dtrace_difv_t);
+
+				tlocals = kmem_zalloc(nsz, KM_SLEEP);
+
+				if (osz != 0) {
+					bcopy(vstate->dtvs_tlocals,
+					    tlocals, osz);
+					kmem_free(vstate->dtvs_tlocals, osz);
+				}
+
+				vstate->dtvs_tlocals = tlocals;
+				vstate->dtvs_ntlocals = ntlocals;
+			}
+
+			vstate->dtvs_tlocals[id] = *v;
+			continue;
+
+		case DIFV_SCOPE_LOCAL:
+			np = &vstate->dtvs_nlocals;
+			svarp = &vstate->dtvs_locals;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
+				dsize = NCPU * (v->dtdv_type.dtdt_size +
+				    sizeof (uint64_t));
+			else
+				dsize = NCPU * sizeof (uint64_t);
+
+			break;
+
+		case DIFV_SCOPE_GLOBAL:
+			np = &vstate->dtvs_nglobals;
+			svarp = &vstate->dtvs_globals;
+
+			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
+				dsize = v->dtdv_type.dtdt_size +
+				    sizeof (uint64_t);
+
+			break;
+
+		default:
+			ASSERT(0);
+		}
+
+		while (id >= (oldsvars = *np)) {
+			dtrace_statvar_t **statics;
+			int newsvars, oldsize, newsize;
+
+			if ((newsvars = (oldsvars << 1)) == 0)
+				newsvars = 1;
+
+			oldsize = oldsvars * sizeof (dtrace_statvar_t *);
+			newsize = newsvars * sizeof (dtrace_statvar_t *);
+
+			statics = kmem_zalloc(newsize, KM_SLEEP);
+
+			if (oldsize != 0) {
+				bcopy(*svarp, statics, oldsize);
+				kmem_free(*svarp, oldsize);
+			}
+
+			*svarp = statics;
+			*np = newsvars;
+		}
+
+		if ((svar = (*svarp)[id]) == NULL) {
+			svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
+			svar->dtsv_var = *v;
+
+			if ((svar->dtsv_size = dsize) != 0) {
+				svar->dtsv_data = (uint64_t)(uintptr_t)
+				    kmem_zalloc(dsize, KM_SLEEP);
+			}
+
+			(*svarp)[id] = svar;
+		}
+
+		svar->dtsv_refcnt++;
+	}
+
+	dtrace_difo_chunksize(dp, vstate);
+	dtrace_difo_hold(dp);
+}
+
+static dtrace_difo_t *
+dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
+{
+	dtrace_difo_t *new;
+	size_t sz;
+
+	ASSERT(dp->dtdo_buf != NULL);
+	ASSERT(dp->dtdo_refcnt != 0);
+
+	new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
+
+	ASSERT(dp->dtdo_buf != NULL);
+	sz = dp->dtdo_len * sizeof (dif_instr_t);
+	new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
+	bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
+	new->dtdo_len = dp->dtdo_len;
+
+	if (dp->dtdo_strtab != NULL) {
+		ASSERT(dp->dtdo_strlen != 0);
+		new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
+		bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
+		new->dtdo_strlen = dp->dtdo_strlen;
+	}
+
+	if (dp->dtdo_inttab != NULL) {
+		ASSERT(dp->dtdo_intlen != 0);
+		sz = dp->dtdo_intlen * sizeof (uint64_t);
+		new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
+		bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
+		new->dtdo_intlen = dp->dtdo_intlen;
+	}
+
+	if (dp->dtdo_vartab != NULL) {
+		ASSERT(dp->dtdo_varlen != 0);
+		sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
+		new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
+		bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
+		new->dtdo_varlen = dp->dtdo_varlen;
+	}
+
+	dtrace_difo_init(new, vstate);
+	return (new);
+}
+
+static void
+dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
+{
+	int i;
+
+	ASSERT(dp->dtdo_refcnt == 0);
+
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+		dtrace_statvar_t *svar, **svarp;
+		uint_t id;
+		uint8_t scope = v->dtdv_scope;
+		int *np;
+
+		switch (scope) {
+		case DIFV_SCOPE_THREAD:
+			continue;
+
+		case DIFV_SCOPE_LOCAL:
+			np = &vstate->dtvs_nlocals;
+			svarp = vstate->dtvs_locals;
+			break;
+
+		case DIFV_SCOPE_GLOBAL:
+			np = &vstate->dtvs_nglobals;
+			svarp = vstate->dtvs_globals;
+			break;
+
+		default:
+			ASSERT(0);
+		}
+
+		if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
+			continue;
+
+		id -= DIF_VAR_OTHER_UBASE;
+		ASSERT(id < *np);
+
+		svar = svarp[id];
+		ASSERT(svar != NULL);
+		ASSERT(svar->dtsv_refcnt > 0);
+
+		if (--svar->dtsv_refcnt > 0)
+			continue;
+
+		if (svar->dtsv_size != 0) {
+			ASSERT(svar->dtsv_data != NULL);
+			kmem_free((void *)(uintptr_t)svar->dtsv_data,
+			    svar->dtsv_size);
+		}
+
+		kmem_free(svar, sizeof (dtrace_statvar_t));
+		svarp[id] = NULL;
+	}
+
+	kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
+	kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
+	kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
+	kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
+
+	kmem_free(dp, sizeof (dtrace_difo_t));
+}
+
+static void
+dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
+{
+	int i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dp->dtdo_refcnt != 0);
+
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+
+		if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
+			continue;
+
+		ASSERT(dtrace_vtime_references > 0);
+		if (--dtrace_vtime_references == 0)
+			dtrace_vtime_disable();
+	}
+
+	if (--dp->dtdo_refcnt == 0)
+		dtrace_difo_destroy(dp, vstate);
+}
+
+/*
+ * DTrace Format Functions
+ */
+static uint16_t
+dtrace_format_add(dtrace_state_t *state, char *str)
+{
+	char *fmt, **new;
+	uint16_t ndx, len = strlen(str) + 1;
+
+	fmt = kmem_zalloc(len, KM_SLEEP);
+	bcopy(str, fmt, len);
+
+	for (ndx = 0; ndx < state->dts_nformats; ndx++) {
+		if (state->dts_formats[ndx] == NULL) {
+			state->dts_formats[ndx] = fmt;
+			return (ndx + 1);
+		}
+	}
+
+	if (state->dts_nformats == USHRT_MAX) {
+		/*
+		 * This is only likely if a denial-of-service attack is being
+		 * attempted.  As such, it's okay to fail silently here.
+		 */
+		kmem_free(fmt, len);
+		return (0);
+	}
+
+	/*
+	 * For simplicity, we always resize the formats array to be exactly the
+	 * number of formats.
+	 */
+	ndx = state->dts_nformats++;
+	new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
+
+	if (state->dts_formats != NULL) {
+		ASSERT(ndx != 0);
+		bcopy(state->dts_formats, new, ndx * sizeof (char *));
+		kmem_free(state->dts_formats, ndx * sizeof (char *));
+	}
+
+	state->dts_formats = new;
+	state->dts_formats[ndx] = fmt;
+
+	return (ndx + 1);
+}
+
+static void
+dtrace_format_remove(dtrace_state_t *state, uint16_t format)
+{
+	char *fmt;
+
+	ASSERT(state->dts_formats != NULL);
+	ASSERT(format <= state->dts_nformats);
+	ASSERT(state->dts_formats[format - 1] != NULL);
+
+	fmt = state->dts_formats[format - 1];
+	kmem_free(fmt, strlen(fmt) + 1);
+	state->dts_formats[format - 1] = NULL;
+}
+
+static void
+dtrace_format_destroy(dtrace_state_t *state)
+{
+	int i;
+
+	if (state->dts_nformats == 0) {
+		ASSERT(state->dts_formats == NULL);
+		return;
+	}
+
+	ASSERT(state->dts_formats != NULL);
+
+	for (i = 0; i < state->dts_nformats; i++) {
+		char *fmt = state->dts_formats[i];
+
+		if (fmt == NULL)
+			continue;
+
+		kmem_free(fmt, strlen(fmt) + 1);
+	}
+
+	kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
+	state->dts_nformats = 0;
+	state->dts_formats = NULL;
+}
+
+/*
+ * DTrace Predicate Functions
+ */
+static dtrace_predicate_t *
+dtrace_predicate_create(dtrace_difo_t *dp)
+{
+	dtrace_predicate_t *pred;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dp->dtdo_refcnt != 0);
+
+	pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
+	pred->dtp_difo = dp;
+	pred->dtp_refcnt = 1;
+
+	if (!dtrace_difo_cacheable(dp))
+		return (pred);
+
+	if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
+		/*
+		 * This is only theoretically possible -- we have had 2^32
+		 * cacheable predicates on this machine.  We cannot allow any
+		 * more predicates to become cacheable:  as unlikely as it is,
+		 * there may be a thread caching a (now stale) predicate cache
+		 * ID. (N.B.: the temptation is being successfully resisted to
+		 * have this cmn_err() "Holy shit -- we executed this code!")
+		 */
+		return (pred);
+	}
+
+	pred->dtp_cacheid = dtrace_predcache_id++;
+
+	return (pred);
+}
+
+static void
+dtrace_predicate_hold(dtrace_predicate_t *pred)
+{
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
+	ASSERT(pred->dtp_refcnt > 0);
+
+	pred->dtp_refcnt++;
+}
+
+static void
+dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
+{
+	dtrace_difo_t *dp = pred->dtp_difo;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
+	ASSERT(pred->dtp_refcnt > 0);
+
+	if (--pred->dtp_refcnt == 0) {
+		dtrace_difo_release(pred->dtp_difo, vstate);
+		kmem_free(pred, sizeof (dtrace_predicate_t));
+	}
+}
+
+/*
+ * DTrace Action Description Functions
+ */
+static dtrace_actdesc_t *
+dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
+    uint64_t uarg, uint64_t arg)
+{
+	dtrace_actdesc_t *act;
+
+/*	ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
+  arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));*/
+
+	act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
+	act->dtad_kind = kind;
+	act->dtad_ntuple = ntuple;
+	act->dtad_uarg = uarg;
+	act->dtad_arg = arg;
+	act->dtad_refcnt = 1;
+
+	return (act);
+}
+
+static void
+dtrace_actdesc_hold(dtrace_actdesc_t *act)
+{
+	ASSERT(act->dtad_refcnt >= 1);
+	act->dtad_refcnt++;
+}
+
+static void
+dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
+{
+	dtrace_actkind_t kind = act->dtad_kind;
+	dtrace_difo_t *dp;
+
+	ASSERT(act->dtad_refcnt >= 1);
+
+	if (--act->dtad_refcnt != 0)
+		return;
+
+	if ((dp = act->dtad_difo) != NULL)
+		dtrace_difo_release(dp, vstate);
+
+	if (DTRACEACT_ISPRINTFLIKE(kind)) {
+		char *str = (char *)(uintptr_t)act->dtad_arg;
+
+/*		ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
+  (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));*/
+
+		if (str != NULL)
+			kmem_free(str, strlen(str) + 1);
+	}
+
+	kmem_free(act, sizeof (dtrace_actdesc_t));
+}
+
+/*
+ * DTrace ECB Functions
+ */
+static dtrace_ecb_t *
+dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
+{
+	dtrace_ecb_t *ecb;
+	dtrace_epid_t epid;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
+	ecb->dte_predicate = NULL;
+	ecb->dte_probe = probe;
+
+	/*
+	 * The default size is the size of the default action: recording
+	 * the epid.
+	 */
+	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
+	ecb->dte_alignment = sizeof (dtrace_epid_t);
+
+	epid = state->dts_epid++;
+
+	if (epid - 1 >= state->dts_necbs) {
+		dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
+		int necbs = state->dts_necbs << 1;
+
+		ASSERT(epid == state->dts_necbs + 1);
+
+		if (necbs == 0) {
+			ASSERT(oecbs == NULL);
+			necbs = 1;
+		}
+
+		ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
+
+		if (oecbs != NULL)
+			bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
+
+		dtrace_membar_producer();
+		state->dts_ecbs = ecbs;
+
+		if (oecbs != NULL) {
+			/*
+			 * If this state is active, we must dtrace_sync()
+			 * before we can free the old dts_ecbs array:  we're
+			 * coming in hot, and there may be active ring
+			 * buffer processing (which indexes into the dts_ecbs
+			 * array) on another CPU.
+			 */
+			if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
+				dtrace_sync();
+
+			kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
+		}
+
+		dtrace_membar_producer();
+		state->dts_necbs = necbs;
+	}
+
+	ecb->dte_state = state;
+
+	ASSERT(state->dts_ecbs[epid - 1] == NULL);
+	dtrace_membar_producer();
+	state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
+
+	return (ecb);
+}
+
+static void
+dtrace_ecb_enable(dtrace_ecb_t *ecb)
+{
+	dtrace_probe_t *probe = ecb->dte_probe;
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(ecb->dte_next == NULL);
+
+	if (probe == NULL) {
+		/*
+		 * This is the NULL probe -- there's nothing to do.
+		 */
+		return;
+	}
+
+	if (probe->dtpr_ecb == NULL) {
+		dtrace_provider_t *prov = probe->dtpr_provider;
+
+		/*
+		 * We're the first ECB on this probe.
+		 */
+		probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
+
+		if (ecb->dte_predicate != NULL)
+			probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
+
+		prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
+		    probe->dtpr_id, probe->dtpr_arg);
+	} else {
+		/*
+		 * This probe is already active.  Swing the last pointer to
+		 * point to the new ECB, and issue a dtrace_sync() to assure
+		 * that all CPUs have seen the change.
+		 */
+		ASSERT(probe->dtpr_ecb_last != NULL);
+		probe->dtpr_ecb_last->dte_next = ecb;
+		probe->dtpr_ecb_last = ecb;
+		probe->dtpr_predcache = 0;
+
+		dtrace_sync();
+	}
+}
+
+static void
+dtrace_ecb_resize(dtrace_ecb_t *ecb)
+{
+	uint32_t maxalign = sizeof (dtrace_epid_t);
+	uint32_t align = sizeof (uint8_t), offs, diff;
+	dtrace_action_t *act;
+	int wastuple = 0;
+	uint32_t aggbase = UINT32_MAX;
+	dtrace_state_t *state = ecb->dte_state;
+
+	/*
+	 * If we record anything, we always record the epid.  (And we always
+	 * record it first.)
+	 */
+	offs = sizeof (dtrace_epid_t);
+	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
+
+	for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
+		dtrace_recdesc_t *rec = &act->dta_rec;
+
+		if ((align = rec->dtrd_alignment) > maxalign)
+			maxalign = align;
+
+		if (!wastuple && act->dta_intuple) {
+			/*
+			 * This is the first record in a tuple.  Align the
+			 * offset to be at offset 4 in an 8-byte aligned
+			 * block.
+			 */
+			diff = offs + sizeof (dtrace_aggid_t);
+
+			if (diff = (diff & (sizeof (uint64_t) - 1)))
+				offs += sizeof (uint64_t) - diff;
+
+			aggbase = offs - sizeof (dtrace_aggid_t);
+			ASSERT(!(aggbase & (sizeof (uint64_t) - 1)));
+		}
+
+		/*LINTED*/
+		if (rec->dtrd_size != 0 && (diff = (offs & (align - 1)))) {
+			/*
+			 * The current offset is not properly aligned; align it.
+			 */
+			offs += align - diff;
+		}
+
+		rec->dtrd_offset = offs;
+
+		if (offs + rec->dtrd_size > ecb->dte_needed) {
+			ecb->dte_needed = offs + rec->dtrd_size;
+
+			if (ecb->dte_needed > state->dts_needed)
+				state->dts_needed = ecb->dte_needed;
+		}
+
+		if (DTRACEACT_ISAGG(act->dta_kind)) {
+			dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
+			dtrace_action_t *first = agg->dtag_first, *prev;
+
+			ASSERT(rec->dtrd_size != 0 && first != NULL);
+			ASSERT(wastuple);
+			ASSERT(aggbase != UINT32_MAX);
+
+			agg->dtag_base = aggbase;
+
+			while ((prev = first->dta_prev) != NULL &&
+			    DTRACEACT_ISAGG(prev->dta_kind)) {
+				agg = (dtrace_aggregation_t *)prev;
+				first = agg->dtag_first;
+			}
+
+			if (prev != NULL) {
+				offs = prev->dta_rec.dtrd_offset +
+				    prev->dta_rec.dtrd_size;
+			} else {
+				offs = sizeof (dtrace_epid_t);
+			}
+			wastuple = 0;
+		} else {
+			if (!act->dta_intuple)
+				ecb->dte_size = offs + rec->dtrd_size;
+
+			offs += rec->dtrd_size;
+		}
+
+		wastuple = act->dta_intuple;
+	}
+
+	if ((act = ecb->dte_action) != NULL &&
+	    !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
+	    ecb->dte_size == sizeof (dtrace_epid_t)) {
+		/*
+		 * If the size is still sizeof (dtrace_epid_t), then all
+		 * actions store no data; set the size to 0.
+		 */
+		ecb->dte_alignment = maxalign;
+		ecb->dte_size = 0;
+
+		/*
+		 * If the needed space is still sizeof (dtrace_epid_t), then
+		 * all actions need no additional space; set the needed
+		 * size to 0.
+		 */
+		if (ecb->dte_needed == sizeof (dtrace_epid_t))
+			ecb->dte_needed = 0;
+
+		return;
+	}
+
+	/*
+	 * Set our alignment, and make sure that the dte_size and dte_needed
+	 * are aligned to the size of an EPID.
+	 */
+	ecb->dte_alignment = maxalign;
+	ecb->dte_size = (ecb->dte_size + (sizeof (dtrace_epid_t) - 1)) &
+	    ~(sizeof (dtrace_epid_t) - 1);
+	ecb->dte_needed = (ecb->dte_needed + (sizeof (dtrace_epid_t) - 1)) &
+	    ~(sizeof (dtrace_epid_t) - 1);
+	ASSERT(ecb->dte_size <= ecb->dte_needed);
+}
+
+static dtrace_action_t *
+dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
+{
+	dtrace_aggregation_t *agg;
+	size_t size = sizeof (uint64_t);
+	int ntuple = desc->dtad_ntuple;
+	dtrace_action_t *act;
+	dtrace_recdesc_t *frec;
+	dtrace_aggid_t aggid;
+	dtrace_state_t *state = ecb->dte_state;
+
+	agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
+	agg->dtag_ecb = ecb;
+
+	ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
+
+	switch (desc->dtad_kind) {
+	case DTRACEAGG_MIN:
+		agg->dtag_initial = UINT64_MAX;
+		agg->dtag_aggregate = dtrace_aggregate_min;
+		break;
+
+	case DTRACEAGG_MAX:
+		agg->dtag_aggregate = dtrace_aggregate_max;
+		break;
+
+	case DTRACEAGG_COUNT:
+		agg->dtag_aggregate = dtrace_aggregate_count;
+		break;
+
+	case DTRACEAGG_QUANTIZE:
+		agg->dtag_aggregate = dtrace_aggregate_quantize;
+		size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
+		    sizeof (uint64_t);
+		break;
+
+	case DTRACEAGG_LQUANTIZE: {
+		uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
+		uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
+
+		agg->dtag_initial = desc->dtad_arg;
+		agg->dtag_aggregate = dtrace_aggregate_lquantize;
+
+		if (step == 0 || levels == 0)
+			goto err;
+
+		size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
+		break;
+	}
+
+	case DTRACEAGG_AVG:
+		agg->dtag_aggregate = dtrace_aggregate_avg;
+		size = sizeof (uint64_t) * 2;
+		break;
+
+	case DTRACEAGG_SUM:
+		agg->dtag_aggregate = dtrace_aggregate_sum;
+		break;
+
+	default:
+		goto err;
+	}
+
+	agg->dtag_action.dta_rec.dtrd_size = size;
+
+	if (ntuple == 0)
+		goto err;
+
+	/*
+	 * We must make sure that we have enough actions for the n-tuple.
+	 */
+	for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
+		if (DTRACEACT_ISAGG(act->dta_kind))
+			break;
+
+		if (--ntuple == 0) {
+			/*
+			 * This is the action with which our n-tuple begins.
+			 */
+			agg->dtag_first = act;
+			goto success;
+		}
+	}
+
+	/*
+	 * This n-tuple is short by ntuple elements.  Return failure.
+	 */
+	ASSERT(ntuple != 0);
+err:
+	kmem_free(agg, sizeof (dtrace_aggregation_t));
+	return (NULL);
+
+success:
+	/*
+	 * If the last action in the tuple has a size of zero, it's actually
+	 * an expression argument for the aggregating action.
+	 */
+	ASSERT(ecb->dte_action_last != NULL);
+	act = ecb->dte_action_last;
+
+	if (act->dta_kind == DTRACEACT_DIFEXPR) {
+		ASSERT(act->dta_difo != NULL);
+
+		if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
+			agg->dtag_hasarg = 1;
+	}
+
+	/*
+	 * We need to allocate an id for this aggregation.
+	 */
+	aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
+	    VM_BESTFIT | VM_SLEEP);
+
+	if (aggid - 1 >= state->dts_naggregations) {
+		dtrace_aggregation_t **oaggs = state->dts_aggregations;
+		dtrace_aggregation_t **aggs;
+		int naggs = state->dts_naggregations << 1;
+		int onaggs = state->dts_naggregations;
+
+		ASSERT(aggid == state->dts_naggregations + 1);
+
+		if (naggs == 0) {
+			ASSERT(oaggs == NULL);
+			naggs = 1;
+		}
+
+		aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
+
+		if (oaggs != NULL) {
+			bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
+			kmem_free(oaggs, onaggs * sizeof (*aggs));
+		}
+
+		state->dts_aggregations = aggs;
+		state->dts_naggregations = naggs;
+	}
+
+	ASSERT(state->dts_aggregations[aggid - 1] == NULL);
+	state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
+
+	frec = &agg->dtag_first->dta_rec;
+	if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
+		frec->dtrd_alignment = sizeof (dtrace_aggid_t);
+
+	for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
+		ASSERT(!act->dta_intuple);
+		act->dta_intuple = 1;
+	}
+
+	return (&agg->dtag_action);
+}
+
+static void
+dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
+{
+	dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
+	dtrace_state_t *state = ecb->dte_state;
+	dtrace_aggid_t aggid = agg->dtag_id;
+
+	ASSERT(DTRACEACT_ISAGG(act->dta_kind));
+	vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
+
+	ASSERT(state->dts_aggregations[aggid - 1] == agg);
+	state->dts_aggregations[aggid - 1] = NULL;
+
+	kmem_free(agg, sizeof (dtrace_aggregation_t));
+}
+
+static int
+dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
+{
+	dtrace_action_t *action, *last;
+	dtrace_difo_t *dp = desc->dtad_difo;
+	uint32_t size = 0, align = sizeof (uint8_t), mask;
+	uint16_t format = 0;
+	dtrace_recdesc_t *rec;
+	dtrace_state_t *state = ecb->dte_state;
+	dtrace_optval_t *opt = state->dts_options, nframes, strsize;
+	uint64_t arg = desc->dtad_arg;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
+
+	if (DTRACEACT_ISAGG(desc->dtad_kind)) {
+		/*
+		 * If this is an aggregating action, there must be neither
+		 * a speculate nor a commit on the action chain.
+		 */
+		dtrace_action_t *act;
+
+		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
+			if (act->dta_kind == DTRACEACT_COMMIT)
+				return (EINVAL);
+
+			if (act->dta_kind == DTRACEACT_SPECULATE)
+				return (EINVAL);
+		}
+
+		action = dtrace_ecb_aggregation_create(ecb, desc);
+
+		if (action == NULL)
+			return (EINVAL);
+	} else {
+		if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
+		    (desc->dtad_kind == DTRACEACT_DIFEXPR &&
+		    dp != NULL && dp->dtdo_destructive)) {
+			state->dts_destructive = 1;
+		}
+
+		switch (desc->dtad_kind) {
+		case DTRACEACT_PRINTF:
+		case DTRACEACT_PRINTA:
+		case DTRACEACT_SYSTEM:
+		case DTRACEACT_FREOPEN:
+			/*
+			 * We know that our arg is a string -- turn it into a
+			 * format.
+			 */
+			if (arg == NULL) {
+				ASSERT(desc->dtad_kind == DTRACEACT_PRINTA);
+				format = 0;
+			} else {
+				ASSERT(arg != NULL);
+				/* ASSERT(arg > KERNELBASE); */
+				format = dtrace_format_add(state,
+				    (char *)(uintptr_t)arg);
+			}
+
+			/*FALLTHROUGH*/
+		case DTRACEACT_LIBACT:
+		case DTRACEACT_DIFEXPR:
+			if (dp == NULL)
+				return (EINVAL);
+
+			if ((size = dp->dtdo_rtype.dtdt_size) != 0)
+				break;
+
+			if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
+				if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
+					return (EINVAL);
+
+				size = opt[DTRACEOPT_STRSIZE];
+			}
+
+			break;
+
+		case DTRACEACT_STACK:
+			if ((nframes = arg) == 0) {
+				nframes = opt[DTRACEOPT_STACKFRAMES];
+				ASSERT(nframes > 0);
+				arg = nframes;
+			}
+
+			size = nframes * sizeof (pc_t);
+			break;
+
+		case DTRACEACT_JSTACK:
+			if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
+				strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
+
+			if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
+				nframes = opt[DTRACEOPT_JSTACKFRAMES];
+
+			arg = DTRACE_USTACK_ARG(nframes, strsize);
+
+			/*FALLTHROUGH*/
+		case DTRACEACT_USTACK:
+			if (desc->dtad_kind != DTRACEACT_JSTACK &&
+			    (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
+				strsize = DTRACE_USTACK_STRSIZE(arg);
+				nframes = opt[DTRACEOPT_USTACKFRAMES];
+				ASSERT(nframes > 0);
+				arg = DTRACE_USTACK_ARG(nframes, strsize);
+			}
+
+			/*
+			 * Save a slot for the pid.
+			 */
+			size = (nframes + 1) * sizeof (uint64_t);
+			size += DTRACE_USTACK_STRSIZE(arg);
+			size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
+
+			break;
+
+		case DTRACEACT_SYM:
+		case DTRACEACT_MOD:
+			if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
+			    sizeof (uint64_t)) ||
+			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
+				return (EINVAL);
+			break;
+
+		case DTRACEACT_USYM:
+		case DTRACEACT_UMOD:
+		case DTRACEACT_UADDR:
+			if (dp == NULL ||
+			    (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
+			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
+				return (EINVAL);
+
+			/*
+			 * We have a slot for the pid, plus a slot for the
+			 * argument.  To keep things simple (aligned with
+			 * bitness-neutral sizing), we store each as a 64-bit
+			 * quantity.
+			 */
+			size = 2 * sizeof (uint64_t);
+			break;
+
+		case DTRACEACT_STOP:
+		case DTRACEACT_BREAKPOINT:
+		case DTRACEACT_PANIC:
+			break;
+
+		case DTRACEACT_CHILL:
+		case DTRACEACT_DISCARD:
+		case DTRACEACT_RAISE:
+			if (dp == NULL)
+				return (EINVAL);
+			break;
+
+		case DTRACEACT_EXIT:
+			if (dp == NULL ||
+			    (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
+			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
+				return (EINVAL);
+			break;
+
+		case DTRACEACT_SPECULATE:
+			if (ecb->dte_size > sizeof (dtrace_epid_t))
+				return (EINVAL);
+
+			if (dp == NULL)
+				return (EINVAL);
+
+			state->dts_speculates = 1;
+			break;
+
+		case DTRACEACT_COMMIT: {
+			dtrace_action_t *act = ecb->dte_action;
+
+			for (; act != NULL; act = act->dta_next) {
+				if (act->dta_kind == DTRACEACT_COMMIT)
+					return (EINVAL);
+			}
+
+			if (dp == NULL)
+				return (EINVAL);
+			break;
+		}
+
+		default:
+			return (EINVAL);
+		}
+
+		if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
+			/*
+			 * If this is a data-storing action or a speculate,
+			 * we must be sure that there isn't a commit on the
+			 * action chain.
+			 */
+			dtrace_action_t *act = ecb->dte_action;
+
+			for (; act != NULL; act = act->dta_next) {
+				if (act->dta_kind == DTRACEACT_COMMIT)
+					return (EINVAL);
+			}
+		}
+
+		action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
+		action->dta_rec.dtrd_size = size;
+	}
+
+	action->dta_refcnt = 1;
+	rec = &action->dta_rec;
+	size = rec->dtrd_size;
+
+	for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
+		if (!(size & mask)) {
+			align = mask + 1;
+			break;
+		}
+	}
+
+	action->dta_kind = desc->dtad_kind;
+
+	if ((action->dta_difo = dp) != NULL)
+		dtrace_difo_hold(dp);
+
+	rec->dtrd_action = action->dta_kind;
+	rec->dtrd_arg = arg;
+	rec->dtrd_uarg = desc->dtad_uarg;
+	rec->dtrd_alignment = (uint16_t)align;
+	rec->dtrd_format = format;
+
+	if ((last = ecb->dte_action_last) != NULL) {
+		ASSERT(ecb->dte_action != NULL);
+		action->dta_prev = last;
+		last->dta_next = action;
+	} else {
+		ASSERT(ecb->dte_action == NULL);
+		ecb->dte_action = action;
+	}
+
+	ecb->dte_action_last = action;
+
+	return (0);
+}
+
+static void
+dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
+{
+	dtrace_action_t *act = ecb->dte_action, *next;
+	dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
+	dtrace_difo_t *dp;
+	uint16_t format;
+
+	if (act != NULL && act->dta_refcnt > 1) {
+		ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
+		act->dta_refcnt--;
+	} else {
+		for (; act != NULL; act = next) {
+			next = act->dta_next;
+			ASSERT(next != NULL || act == ecb->dte_action_last);
+			ASSERT(act->dta_refcnt == 1);
+
+			if ((format = act->dta_rec.dtrd_format) != 0)
+				dtrace_format_remove(ecb->dte_state, format);
+
+			if ((dp = act->dta_difo) != NULL)
+				dtrace_difo_release(dp, vstate);
+
+			if (DTRACEACT_ISAGG(act->dta_kind)) {
+				dtrace_ecb_aggregation_destroy(ecb, act);
+			} else {
+				kmem_free(act, sizeof (dtrace_action_t));
+			}
+		}
+	}
+
+	ecb->dte_action = NULL;
+	ecb->dte_action_last = NULL;
+	ecb->dte_size = sizeof (dtrace_epid_t);
+}
+
+static void
+dtrace_ecb_disable(dtrace_ecb_t *ecb)
+{
+	/*
+	 * We disable the ECB by removing it from its probe.
+	 */
+	dtrace_ecb_t *pecb, *prev = NULL;
+	dtrace_probe_t *probe = ecb->dte_probe;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (probe == NULL) {
+		/*
+		 * This is the NULL probe; there is nothing to disable.
+		 */
+		return;
+	}
+
+	for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
+		if (pecb == ecb)
+			break;
+		prev = pecb;
+	}
+
+	ASSERT(pecb != NULL);
+
+	if (prev == NULL) {
+		probe->dtpr_ecb = ecb->dte_next;
+	} else {
+		prev->dte_next = ecb->dte_next;
+	}
+
+	if (ecb == probe->dtpr_ecb_last) {
+		ASSERT(ecb->dte_next == NULL);
+		probe->dtpr_ecb_last = prev;
+	}
+
+	/*
+	 * The ECB has been disconnected from the probe; now sync to assure
+	 * that all CPUs have seen the change before returning.
+	 */
+	dtrace_sync();
+
+	if (probe->dtpr_ecb == NULL) {
+		/*
+		 * That was the last ECB on the probe; clear the predicate
+		 * cache ID for the probe, disable it and sync one more time
+		 * to assure that we'll never hit it again.
+		 */
+		dtrace_provider_t *prov = probe->dtpr_provider;
+
+		ASSERT(ecb->dte_next == NULL);
+		ASSERT(probe->dtpr_ecb_last == NULL);
+		probe->dtpr_predcache = DTRACE_CACHEIDNONE;
+		prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
+		    probe->dtpr_id, probe->dtpr_arg);
+		dtrace_sync();
+	} else {
+		/*
+		 * There is at least one ECB remaining on the probe.  If there
+		 * is _exactly_ one, set the probe's predicate cache ID to be
+		 * the predicate cache ID of the remaining ECB.
+		 */
+		ASSERT(probe->dtpr_ecb_last != NULL);
+		ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
+
+		if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
+			dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
+
+			ASSERT(probe->dtpr_ecb->dte_next == NULL);
+
+			if (p != NULL)
+				probe->dtpr_predcache = p->dtp_cacheid;
+		}
+
+		ecb->dte_next = NULL;
+	}
+}
+
+static void
+dtrace_ecb_destroy(dtrace_ecb_t *ecb)
+{
+	dtrace_state_t *state = ecb->dte_state;
+	dtrace_vstate_t *vstate = &state->dts_vstate;
+	dtrace_predicate_t *pred;
+	dtrace_epid_t epid = ecb->dte_epid;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(ecb->dte_next == NULL);
+	ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
+
+	if ((pred = ecb->dte_predicate) != NULL)
+		dtrace_predicate_release(pred, vstate);
+
+	dtrace_ecb_action_remove(ecb);
+
+	ASSERT(state->dts_ecbs[epid - 1] == ecb);
+	state->dts_ecbs[epid - 1] = NULL;
+
+	kmem_free(ecb, sizeof (dtrace_ecb_t));
+}
+
+static dtrace_ecb_t *
+dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
+    dtrace_enabling_t *enab)
+{
+	dtrace_ecb_t *ecb;
+	dtrace_predicate_t *pred;
+	dtrace_actdesc_t *act;
+	dtrace_provider_t *prov;
+	dtrace_ecbdesc_t *desc = enab->dten_current;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(state != NULL);
+
+	ecb = dtrace_ecb_add(state, probe);
+	ecb->dte_uarg = desc->dted_uarg;
+
+	if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
+		dtrace_predicate_hold(pred);
+		ecb->dte_predicate = pred;
+	}
+
+	if (probe != NULL) {
+		/*
+		 * If the provider shows more leg than the consumer is old
+		 * enough to see, we need to enable the appropriate implicit
+		 * predicate bits to prevent the ecb from activating at
+		 * revealing times.
+		 *
+		 * Providers specifying DTRACE_PRIV_USER at register time
+		 * are stating that they need the /proc-style privilege
+		 * model to be enforced, and this is what DTRACE_COND_OWNER
+		 * and DTRACE_COND_ZONEOWNER will then do at probe time.
+		 */
+		prov = probe->dtpr_provider;
+		if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
+		    (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
+			ecb->dte_cond |= DTRACE_COND_OWNER;
+
+		if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) &&
+		    (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
+			ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
+
+		/*
+		 * If the provider shows us kernel innards and the user
+		 * is lacking sufficient privilege, enable the
+		 * DTRACE_COND_USERMODE implicit predicate.
+		 */
+		if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
+		    (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
+			ecb->dte_cond |= DTRACE_COND_USERMODE;
+	}
+
+	if (dtrace_ecb_create_cache != NULL) {
+		/*
+		 * If we have a cached ecb, we'll use its action list instead
+		 * of creating our own (saving both time and space).
+		 */
+		dtrace_ecb_t *cached = dtrace_ecb_create_cache;
+		dtrace_action_t *act = cached->dte_action;
+
+		if (act != NULL) {
+			ASSERT(act->dta_refcnt > 0);
+			act->dta_refcnt++;
+			ecb->dte_action = act;
+			ecb->dte_action_last = cached->dte_action_last;
+			ecb->dte_needed = cached->dte_needed;
+			ecb->dte_size = cached->dte_size;
+			ecb->dte_alignment = cached->dte_alignment;
+		}
+
+		return (ecb);
+	}
+
+	for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
+		if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
+			dtrace_ecb_destroy(ecb);
+			return (NULL);
+		}
+	}
+
+	dtrace_ecb_resize(ecb);
+
+	return (dtrace_ecb_create_cache = ecb);
+}
+
+static int
+dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
+{
+	dtrace_ecb_t *ecb;
+	dtrace_enabling_t *enab = arg;
+	dtrace_state_t *state = enab->dten_vstate->dtvs_state;
+
+	ASSERT(state != NULL);
+
+	if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
+		/*
+		 * This probe was created in a generation for which this
+		 * enabling has previously created ECBs; we don't want to
+		 * enable it again, so just kick out.
+		 */
+		return (DTRACE_MATCH_NEXT);
+	}
+
+	if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
+		return (DTRACE_MATCH_DONE);
+
+	dtrace_ecb_enable(ecb);
+	return (DTRACE_MATCH_NEXT);
+}
+
+static dtrace_ecb_t *
+dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
+{
+	dtrace_ecb_t *ecb;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (id == 0 || id > state->dts_necbs)
+		return (NULL);
+
+	ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
+	ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
+
+	return (state->dts_ecbs[id - 1]);
+}
+
+static dtrace_aggregation_t *
+dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
+{
+	dtrace_aggregation_t *agg;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (id == 0 || id > state->dts_naggregations)
+		return (NULL);
+
+	ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
+	ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
+	    agg->dtag_id == id);
+
+	return (state->dts_aggregations[id - 1]);
+}
+
+/*
+ * DTrace Buffer Functions
+ *
+ * The following functions manipulate DTrace buffers.  Most of these functions
+ * are called in the context of establishing or processing consumer state;
+ * exceptions are explicitly noted.
+ */
+
+/*
+ * Note:  called from cross call context.  This function switches the two
+ * buffers on a given CPU.  The atomicity of this operation is assured by
+ * disabling interrupts while the actual switch takes place; the disabling of
+ * interrupts serializes the execution with any execution of dtrace_probe() on
+ * the same CPU.
+ */
+static void
+dtrace_buffer_switch(dtrace_buffer_t *buf)
+{
+	caddr_t tomax = buf->dtb_tomax;
+	caddr_t xamot = buf->dtb_xamot;
+	dtrace_icookie_t cookie;
+
+	ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
+	ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
+
+	cookie = dtrace_interrupt_disable();
+	buf->dtb_tomax = xamot;
+	buf->dtb_xamot = tomax;
+	buf->dtb_xamot_drops = buf->dtb_drops;
+	buf->dtb_xamot_offset = buf->dtb_offset;
+	buf->dtb_xamot_errors = buf->dtb_errors;
+	buf->dtb_xamot_flags = buf->dtb_flags;
+	buf->dtb_offset = 0;
+	buf->dtb_drops = 0;
+	buf->dtb_errors = 0;
+	buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
+	dtrace_interrupt_enable(cookie);
+}
+
+/*
+ * Note:  called from cross call context.  This function activates a buffer
+ * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
+ * is guaranteed by the disabling of interrupts.
+ */
+static void
+dtrace_buffer_activate(dtrace_state_t *state)
+{
+	dtrace_buffer_t *buf;
+	dtrace_icookie_t cookie = dtrace_interrupt_disable();
+
+	buf = &state->dts_buffer[CPU->cpu_id];
+
+	if (buf->dtb_tomax != NULL) {
+		/*
+		 * We might like to assert that the buffer is marked inactive,
+		 * but this isn't necessarily true:  the buffer for the CPU
+		 * that processes the BEGIN probe has its buffer activated
+		 * manually.  In this case, we take the (harmless) action
+		 * re-clearing the bit INACTIVE bit.
+		 */
+		buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
+	}
+
+	dtrace_interrupt_enable(cookie);
+}
+
+static int
+dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
+    processorid_t cpu)
+{
+	cpu_t *cp;
+	dtrace_buffer_t *buf;
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (size > dtrace_nonroot_maxsize &&
+	    !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE))
+		return (EFBIG);
+
+#if defined(__APPLE__)
+	if (size > (sane_size / 8) / NCPU) /* As in kdbg_set_nkdbufs(), roughly. */
+		return (ENOMEM);
+#endif /* __APPLE__ */
+
+	cp = cpu_list;
+
+	do {
+		if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
+			continue;
+
+		buf = &bufs[cp->cpu_id];
+
+		/*
+		 * If there is already a buffer allocated for this CPU, it
+		 * is only possible that this is a DR event.  In this case,
+		 * the buffer size must match our specified size.
+		 */
+		if (buf->dtb_tomax != NULL) {
+			ASSERT(buf->dtb_size == size);
+			continue;
+		}
+
+		ASSERT(buf->dtb_xamot == NULL);
+
+		if ((buf->dtb_tomax = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
+			goto err;
+
+		buf->dtb_size = size;
+		buf->dtb_flags = flags;
+		buf->dtb_offset = 0;
+		buf->dtb_drops = 0;
+
+		if (flags & DTRACEBUF_NOSWITCH)
+			continue;
+
+		if ((buf->dtb_xamot = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
+			goto err;
+	} while ((cp = cp->cpu_next) != cpu_list);
+
+	return (0);
+
+err:
+	cp = cpu_list;
+
+	do {
+		if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
+			continue;
+
+		buf = &bufs[cp->cpu_id];
+
+		if (buf->dtb_xamot != NULL) {
+			ASSERT(buf->dtb_tomax != NULL);
+			ASSERT(buf->dtb_size == size);
+			kmem_free(buf->dtb_xamot, size);
+		}
+
+		if (buf->dtb_tomax != NULL) {
+			ASSERT(buf->dtb_size == size);
+			kmem_free(buf->dtb_tomax, size);
+		}
+
+		buf->dtb_tomax = NULL;
+		buf->dtb_xamot = NULL;
+		buf->dtb_size = 0;
+	} while ((cp = cp->cpu_next) != cpu_list);
+
+	return (ENOMEM);
+}
+
+/*
+ * Note:  called from probe context.  This function just increments the drop
+ * count on a buffer.  It has been made a function to allow for the
+ * possibility of understanding the source of mysterious drop counts.  (A
+ * problem for which one may be particularly disappointed that DTrace cannot
+ * be used to understand DTrace.)
+ */
+static void
+dtrace_buffer_drop(dtrace_buffer_t *buf)
+{
+	buf->dtb_drops++;
+}
+
+/*
+ * Note:  called from probe context.  This function is called to reserve space
+ * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
+ * mstate.  Returns the new offset in the buffer, or a negative value if an
+ * error has occurred.
+ */
+static intptr_t
+dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
+    dtrace_state_t *state, dtrace_mstate_t *mstate)
+{
+	intptr_t offs = buf->dtb_offset, soffs;
+	intptr_t woffs;
+	caddr_t tomax;
+	size_t total;
+
+	if (buf->dtb_flags & DTRACEBUF_INACTIVE)
+		return (-1);
+
+	if ((tomax = buf->dtb_tomax) == NULL) {
+		dtrace_buffer_drop(buf);
+		return (-1);
+	}
+
+	if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
+		while (offs & (align - 1)) {
+			/*
+			 * Assert that our alignment is off by a number which
+			 * is itself sizeof (uint32_t) aligned.
+			 */
+			ASSERT(!((align - (offs & (align - 1))) &
+			    (sizeof (uint32_t) - 1)));
+			DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
+			offs += sizeof (uint32_t);
+		}
+
+		if ((soffs = offs + needed) > buf->dtb_size) {
+			dtrace_buffer_drop(buf);
+			return (-1);
+		}
+
+		if (mstate == NULL)
+			return (offs);
+
+		mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
+		mstate->dtms_scratch_size = buf->dtb_size - soffs;
+		mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
+
+		return (offs);
+	}
+
+	if (buf->dtb_flags & DTRACEBUF_FILL) {
+		if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
+		    (buf->dtb_flags & DTRACEBUF_FULL))
+			return (-1);
+		goto out;
+	}
+
+	total = needed + (offs & (align - 1));
+
+	/*
+	 * For a ring buffer, life is quite a bit more complicated.  Before
+	 * we can store any padding, we need to adjust our wrapping offset.
+	 * (If we've never before wrapped or we're not about to, no adjustment
+	 * is required.)
+	 */
+	if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
+	    offs + total > buf->dtb_size) {
+		woffs = buf->dtb_xamot_offset;
+
+		if (offs + total > buf->dtb_size) {
+			/*
+			 * We can't fit in the end of the buffer.  First, a
+			 * sanity check that we can fit in the buffer at all.
+			 */
+			if (total > buf->dtb_size) {
+				dtrace_buffer_drop(buf);
+				return (-1);
+			}
+
+			/*
+			 * We're going to be storing at the top of the buffer,
+			 * so now we need to deal with the wrapped offset.  We
+			 * only reset our wrapped offset to 0 if it is
+			 * currently greater than the current offset.  If it
+			 * is less than the current offset, it is because a
+			 * previous allocation induced a wrap -- but the
+			 * allocation didn't subsequently take the space due
+			 * to an error or false predicate evaluation.  In this
+			 * case, we'll just leave the wrapped offset alone: if
+			 * the wrapped offset hasn't been advanced far enough
+			 * for this allocation, it will be adjusted in the
+			 * lower loop.
+			 */
+			if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
+				if (woffs >= offs)
+					woffs = 0;
+			} else {
+				woffs = 0;
+			}
+
+			/*
+			 * Now we know that we're going to be storing to the
+			 * top of the buffer and that there is room for us
+			 * there.  We need to clear the buffer from the current
+			 * offset to the end (there may be old gunk there).
+			 */
+			while (offs < buf->dtb_size)
+				tomax[offs++] = 0;
+
+			/*
+			 * We need to set our offset to zero.  And because we
+			 * are wrapping, we need to set the bit indicating as
+			 * much.  We can also adjust our needed space back
+			 * down to the space required by the ECB -- we know
+			 * that the top of the buffer is aligned.
+			 */
+			offs = 0;
+			total = needed;
+			buf->dtb_flags |= DTRACEBUF_WRAPPED;
+		} else {
+			/*
+			 * There is room for us in the buffer, so we simply
+			 * need to check the wrapped offset.
+			 */
+			if (woffs < offs) {
+				/*
+				 * The wrapped offset is less than the offset.
+				 * This can happen if we allocated buffer space
+				 * that induced a wrap, but then we didn't
+				 * subsequently take the space due to an error
+				 * or false predicate evaluation.  This is
+				 * okay; we know that _this_ allocation isn't
+				 * going to induce a wrap.  We still can't
+				 * reset the wrapped offset to be zero,
+				 * however: the space may have been trashed in
+				 * the previous failed probe attempt.  But at
+				 * least the wrapped offset doesn't need to
+				 * be adjusted at all...
+				 */
+				goto out;
+			}
+		}
+
+		while (offs + total > woffs) {
+			dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
+			size_t size;
+
+			if (epid == DTRACE_EPIDNONE) {
+				size = sizeof (uint32_t);
+			} else {
+				ASSERT(epid <= state->dts_necbs);
+				ASSERT(state->dts_ecbs[epid - 1] != NULL);
+
+				size = state->dts_ecbs[epid - 1]->dte_size;
+			}
+
+			ASSERT(woffs + size <= buf->dtb_size);
+			ASSERT(size != 0);
+
+			if (woffs + size == buf->dtb_size) {
+				/*
+				 * We've reached the end of the buffer; we want
+				 * to set the wrapped offset to 0 and break
+				 * out.  However, if the offs is 0, then we're
+				 * in a strange edge-condition:  the amount of
+				 * space that we want to reserve plus the size
+				 * of the record that we're overwriting is
+				 * greater than the size of the buffer.  This
+				 * is problematic because if we reserve the
+				 * space but subsequently don't consume it (due
+				 * to a failed predicate or error) the wrapped
+				 * offset will be 0 -- yet the EPID at offset 0
+				 * will not be committed.  This situation is
+				 * relatively easy to deal with:  if we're in
+				 * this case, the buffer is indistinguishable
+				 * from one that hasn't wrapped; we need only
+				 * finish the job by clearing the wrapped bit,
+				 * explicitly setting the offset to be 0, and
+				 * zero'ing out the old data in the buffer.
+				 */
+				if (offs == 0) {
+					buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
+					buf->dtb_offset = 0;
+					woffs = total;
+
+					while (woffs < buf->dtb_size)
+						tomax[woffs++] = 0;
+				}
+
+				woffs = 0;
+				break;
+			}
+
+			woffs += size;
+		}
+
+		/*
+		 * We have a wrapped offset.  It may be that the wrapped offset
+		 * has become zero -- that's okay.
+		 */
+		buf->dtb_xamot_offset = woffs;
+	}
+
+out:
+	/*
+	 * Now we can plow the buffer with any necessary padding.
+	 */
+	while (offs & (align - 1)) {
+		/*
+		 * Assert that our alignment is off by a number which
+		 * is itself sizeof (uint32_t) aligned.
+		 */
+		ASSERT(!((align - (offs & (align - 1))) &
+		    (sizeof (uint32_t) - 1)));
+		DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
+		offs += sizeof (uint32_t);
+	}
+
+	if (buf->dtb_flags & DTRACEBUF_FILL) {
+		if (offs + needed > buf->dtb_size - state->dts_reserve) {
+			buf->dtb_flags |= DTRACEBUF_FULL;
+			return (-1);
+		}
+	}
+
+	if (mstate == NULL)
+		return (offs);
+
+	/*
+	 * For ring buffers and fill buffers, the scratch space is always
+	 * the inactive buffer.
+	 */
+	mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
+	mstate->dtms_scratch_size = buf->dtb_size;
+	mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
+
+	return (offs);
+}
+
+static void
+dtrace_buffer_polish(dtrace_buffer_t *buf)
+{
+	ASSERT(buf->dtb_flags & DTRACEBUF_RING);
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
+		return;
+
+	/*
+	 * We need to polish the ring buffer.  There are three cases:
+	 *
+	 * - The first (and presumably most common) is that there is no gap
+	 *   between the buffer offset and the wrapped offset.  In this case,
+	 *   there is nothing in the buffer that isn't valid data; we can
+	 *   mark the buffer as polished and return.
+	 *
+	 * - The second (less common than the first but still more common
+	 *   than the third) is that there is a gap between the buffer offset
+	 *   and the wrapped offset, and the wrapped offset is larger than the
+	 *   buffer offset.  This can happen because of an alignment issue, or
+	 *   can happen because of a call to dtrace_buffer_reserve() that
+	 *   didn't subsequently consume the buffer space.  In this case,
+	 *   we need to zero the data from the buffer offset to the wrapped
+	 *   offset.
+	 *
+	 * - The third (and least common) is that there is a gap between the
+	 *   buffer offset and the wrapped offset, but the wrapped offset is
+	 *   _less_ than the buffer offset.  This can only happen because a
+	 *   call to dtrace_buffer_reserve() induced a wrap, but the space
+	 *   was not subsequently consumed.  In this case, we need to zero the
+	 *   space from the offset to the end of the buffer _and_ from the
+	 *   top of the buffer to the wrapped offset.
+	 */
+	if (buf->dtb_offset < buf->dtb_xamot_offset) {
+		bzero(buf->dtb_tomax + buf->dtb_offset,
+		    buf->dtb_xamot_offset - buf->dtb_offset);
+	}
+
+	if (buf->dtb_offset > buf->dtb_xamot_offset) {
+		bzero(buf->dtb_tomax + buf->dtb_offset,
+		    buf->dtb_size - buf->dtb_offset);
+		bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
+	}
+}
+
+static void
+dtrace_buffer_free(dtrace_buffer_t *bufs)
+{
+	int i;
+
+	for (i = 0; i < NCPU; i++) {
+		dtrace_buffer_t *buf = &bufs[i];
+
+		if (buf->dtb_tomax == NULL) {
+			ASSERT(buf->dtb_xamot == NULL);
+			ASSERT(buf->dtb_size == 0);
+			continue;
+		}
+
+		if (buf->dtb_xamot != NULL) {
+			ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
+			kmem_free(buf->dtb_xamot, buf->dtb_size);
+		}
+
+		kmem_free(buf->dtb_tomax, buf->dtb_size);
+		buf->dtb_size = 0;
+		buf->dtb_tomax = NULL;
+		buf->dtb_xamot = NULL;
+	}
+}
+
+/*
+ * DTrace Enabling Functions
+ */
+static dtrace_enabling_t *
+dtrace_enabling_create(dtrace_vstate_t *vstate)
+{
+	dtrace_enabling_t *enab;
+
+	enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
+	enab->dten_vstate = vstate;
+
+	return (enab);
+}
+
+static void
+dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
+{
+	dtrace_ecbdesc_t **ndesc;
+	size_t osize, nsize;
+
+	/*
+	 * We can't add to enablings after we've enabled them, or after we've
+	 * retained them.
+	 */
+	ASSERT(enab->dten_probegen == 0);
+	ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
+
+#if defined(__APPLE__)
+	if (ecb == NULL) return; /* XXX protection against gcc 4.0 botch on x86 */
+#endif /* __APPLE__ */
+
+	if (enab->dten_ndesc < enab->dten_maxdesc) {
+		enab->dten_desc[enab->dten_ndesc++] = ecb;
+		return;
+	}
+
+	osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
+
+	if (enab->dten_maxdesc == 0) {
+		enab->dten_maxdesc = 1;
+	} else {
+		enab->dten_maxdesc <<= 1;
+	}
+
+	ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
+
+	nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
+	ndesc = kmem_zalloc(nsize, KM_SLEEP);
+	bcopy(enab->dten_desc, ndesc, osize);
+	kmem_free(enab->dten_desc, osize);
+
+	enab->dten_desc = ndesc;
+	enab->dten_desc[enab->dten_ndesc++] = ecb;
+}
+
+static void
+dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
+    dtrace_probedesc_t *pd)
+{
+	dtrace_ecbdesc_t *new;
+	dtrace_predicate_t *pred;
+	dtrace_actdesc_t *act;
+
+	/*
+	 * We're going to create a new ECB description that matches the
+	 * specified ECB in every way, but has the specified probe description.
+	 */
+	new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
+
+	if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
+		dtrace_predicate_hold(pred);
+
+	for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
+		dtrace_actdesc_hold(act);
+
+	new->dted_action = ecb->dted_action;
+	new->dted_pred = ecb->dted_pred;
+	new->dted_probe = *pd;
+	new->dted_uarg = ecb->dted_uarg;
+
+	dtrace_enabling_add(enab, new);
+}
+
+static void
+dtrace_enabling_dump(dtrace_enabling_t *enab)
+{
+	int i;
+
+	for (i = 0; i < enab->dten_ndesc; i++) {
+		dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
+
+		cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
+		    desc->dtpd_provider, desc->dtpd_mod,
+		    desc->dtpd_func, desc->dtpd_name);
+	}
+}
+
+static void
+dtrace_enabling_destroy(dtrace_enabling_t *enab)
+{
+	int i;
+	dtrace_ecbdesc_t *ep;
+	dtrace_vstate_t *vstate = enab->dten_vstate;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (i = 0; i < enab->dten_ndesc; i++) {
+		dtrace_actdesc_t *act, *next;
+		dtrace_predicate_t *pred;
+
+		ep = enab->dten_desc[i];
+
+		if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
+			dtrace_predicate_release(pred, vstate);
+
+		for (act = ep->dted_action; act != NULL; act = next) {
+			next = act->dtad_next;
+			dtrace_actdesc_release(act, vstate);
+		}
+
+		kmem_free(ep, sizeof (dtrace_ecbdesc_t));
+	}
+
+	kmem_free(enab->dten_desc,
+	    enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
+
+	/*
+	 * If this was a retained enabling, decrement the dts_nretained count
+	 * and take it off of the dtrace_retained list.
+	 */
+	if (enab->dten_prev != NULL || enab->dten_next != NULL ||
+	    dtrace_retained == enab) {
+		ASSERT(enab->dten_vstate->dtvs_state != NULL);
+		ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
+		enab->dten_vstate->dtvs_state->dts_nretained--;
+	}
+
+	if (enab->dten_prev == NULL) {
+		if (dtrace_retained == enab) {
+			dtrace_retained = enab->dten_next;
+
+			if (dtrace_retained != NULL)
+				dtrace_retained->dten_prev = NULL;
+		}
+	} else {
+		ASSERT(enab != dtrace_retained);
+		ASSERT(dtrace_retained != NULL);
+		enab->dten_prev->dten_next = enab->dten_next;
+	}
+
+	if (enab->dten_next != NULL) {
+		ASSERT(dtrace_retained != NULL);
+		enab->dten_next->dten_prev = enab->dten_prev;
+	}
+
+	kmem_free(enab, sizeof (dtrace_enabling_t));
+}
+
+static int
+dtrace_enabling_retain(dtrace_enabling_t *enab)
+{
+	dtrace_state_t *state;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
+	ASSERT(enab->dten_vstate != NULL);
+
+	state = enab->dten_vstate->dtvs_state;
+	ASSERT(state != NULL);
+
+	/*
+	 * We only allow each state to retain dtrace_retain_max enablings.
+	 */
+	if (state->dts_nretained >= dtrace_retain_max)
+		return (ENOSPC);
+
+	state->dts_nretained++;
+
+	if (dtrace_retained == NULL) {
+		dtrace_retained = enab;
+		return (0);
+	}
+
+	enab->dten_next = dtrace_retained;
+	dtrace_retained->dten_prev = enab;
+	dtrace_retained = enab;
+
+	return (0);
+}
+
+static int
+dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
+    dtrace_probedesc_t *create)
+{
+	dtrace_enabling_t *new, *enab;
+	int found = 0, err = ENOENT;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
+	ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
+	ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
+	ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
+
+	new = dtrace_enabling_create(&state->dts_vstate);
+
+	/*
+	 * Iterate over all retained enablings, looking for enablings that
+	 * match the specified state.
+	 */
+	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
+		int i;
+
+		/*
+		 * dtvs_state can only be NULL for helper enablings -- and
+		 * helper enablings can't be retained.
+		 */
+		ASSERT(enab->dten_vstate->dtvs_state != NULL);
+
+		if (enab->dten_vstate->dtvs_state != state)
+			continue;
+
+		/*
+		 * Now iterate over each probe description; we're looking for
+		 * an exact match to the specified probe description.
+		 */
+		for (i = 0; i < enab->dten_ndesc; i++) {
+			dtrace_ecbdesc_t *ep = enab->dten_desc[i];
+			dtrace_probedesc_t *pd = &ep->dted_probe;
+
+			if (strcmp(pd->dtpd_provider, match->dtpd_provider))
+				continue;
+
+			if (strcmp(pd->dtpd_mod, match->dtpd_mod))
+				continue;
+
+			if (strcmp(pd->dtpd_func, match->dtpd_func))
+				continue;
+
+			if (strcmp(pd->dtpd_name, match->dtpd_name))
+				continue;
+
+			/*
+			 * We have a winning probe!  Add it to our growing
+			 * enabling.
+			 */
+			found = 1;
+			dtrace_enabling_addlike(new, ep, create);
+		}
+	}
+
+	if (!found || (err = dtrace_enabling_retain(new)) != 0) {
+		dtrace_enabling_destroy(new);
+		return (err);
+	}
+
+	return (0);
+}
+
+static void
+dtrace_enabling_retract(dtrace_state_t *state)
+{
+	dtrace_enabling_t *enab, *next;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	/*
+	 * Iterate over all retained enablings, destroy the enablings retained
+	 * for the specified state.
+	 */
+	for (enab = dtrace_retained; enab != NULL; enab = next) {
+		next = enab->dten_next;
+
+		/*
+		 * dtvs_state can only be NULL for helper enablings -- and
+		 * helper enablings can't be retained.
+		 */
+		ASSERT(enab->dten_vstate->dtvs_state != NULL);
+
+		if (enab->dten_vstate->dtvs_state == state) {
+			ASSERT(state->dts_nretained > 0);
+			dtrace_enabling_destroy(enab);
+		}
+	}
+
+	ASSERT(state->dts_nretained == 0);
+}
+
+static int
+dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
+{
+	int i = 0;
+	int matched = 0;
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (i = 0; i < enab->dten_ndesc; i++) {
+		dtrace_ecbdesc_t *ep = enab->dten_desc[i];
+
+		enab->dten_current = ep;
+		enab->dten_error = 0;
+
+		matched += dtrace_probe_enable(&ep->dted_probe, enab);
+
+		if (enab->dten_error != 0) {
+			/*
+			 * If we get an error half-way through enabling the
+			 * probes, we kick out -- perhaps with some number of
+			 * them enabled.  Leaving enabled probes enabled may
+			 * be slightly confusing for user-level, but we expect
+			 * that no one will attempt to actually drive on in
+			 * the face of such errors.  If this is an anonymous
+			 * enabling (indicated with a NULL nmatched pointer),
+			 * we cmn_err() a message.  We aren't expecting to
+			 * get such an error -- such as it can exist at all,
+			 * it would be a result of corrupted DOF in the driver
+			 * properties.
+			 */
+			if (nmatched == NULL) {
+				cmn_err(CE_WARN, "dtrace_enabling_match() "
+				    "error on %p: %d", (void *)ep,
+				    enab->dten_error);
+			}
+
+			return (enab->dten_error);
+		}
+	}
+
+	enab->dten_probegen = dtrace_probegen;
+	if (nmatched != NULL)
+		*nmatched = matched;
+
+	return (0);
+}
+
+static void
+dtrace_enabling_matchall(void)
+{
+	dtrace_enabling_t *enab;
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	/*
+	 * Because we can be called after dtrace_detach() has been called, we
+	 * cannot assert that there are retained enablings.  We can safely
+	 * load from dtrace_retained, however:  the taskq_destroy() at the
+	 * end of dtrace_detach() will block pending our completion.
+	 */
+	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next)
+		(void) dtrace_enabling_match(enab, NULL);
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&cpu_lock);
+}
+
+static int
+dtrace_enabling_matchstate(dtrace_state_t *state, int *nmatched)
+{
+	dtrace_enabling_t *enab;
+	int matched, total = 0, err;
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
+		ASSERT(enab->dten_vstate->dtvs_state != NULL);
+
+		if (enab->dten_vstate->dtvs_state != state)
+			continue;
+
+		if ((err = dtrace_enabling_match(enab, &matched)) != 0)
+			return (err);
+
+		total += matched;
+	}
+
+	if (nmatched != NULL)
+		*nmatched = total;
+
+	return (0);
+}
+
+/*
+ * If an enabling is to be enabled without having matched probes (that is, if
+ * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
+ * enabling must be _primed_ by creating an ECB for every ECB description.
+ * This must be done to assure that we know the number of speculations, the
+ * number of aggregations, the minimum buffer size needed, etc. before we
+ * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
+ * enabling any probes, we create ECBs for every ECB decription, but with a
+ * NULL probe -- which is exactly what this function does.
+ */
+static void
+dtrace_enabling_prime(dtrace_state_t *state)
+{
+	dtrace_enabling_t *enab;
+	int i;
+
+	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
+		ASSERT(enab->dten_vstate->dtvs_state != NULL);
+
+		if (enab->dten_vstate->dtvs_state != state)
+			continue;
+
+		/*
+		 * We don't want to prime an enabling more than once, lest
+		 * we allow a malicious user to induce resource exhaustion.
+		 * (The ECBs that result from priming an enabling aren't
+		 * leaked -- but they also aren't deallocated until the
+		 * consumer state is destroyed.)
+		 */
+		if (enab->dten_primed)
+			continue;
+
+		for (i = 0; i < enab->dten_ndesc; i++) {
+			enab->dten_current = enab->dten_desc[i];
+			(void) dtrace_probe_enable(NULL, enab);
+		}
+
+		enab->dten_primed = 1;
+	}
+}
+
+/*
+ * Called to indicate that probes should be provided due to retained
+ * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
+ * must take an initial lap through the enabling calling the dtps_provide()
+ * entry point explicitly to allow for autocreated probes.
+ */
+static void
+dtrace_enabling_provide(dtrace_provider_t *prv)
+{
+	int i, all = 0;
+	dtrace_probedesc_t desc;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&dtrace_provider_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (prv == NULL) {
+		all = 1;
+		prv = dtrace_provider;
+	}
+
+	do {
+		dtrace_enabling_t *enab = dtrace_retained;
+		void *parg = prv->dtpv_arg;
+
+		for (; enab != NULL; enab = enab->dten_next) {
+			for (i = 0; i < enab->dten_ndesc; i++) {
+				desc = enab->dten_desc[i]->dted_probe;
+				lck_mtx_unlock(&dtrace_lock);
+				prv->dtpv_pops.dtps_provide(parg, &desc);
+				lck_mtx_lock(&dtrace_lock);
+			}
+		}
+	} while (all && (prv = prv->dtpv_next) != NULL);
+
+	lck_mtx_unlock(&dtrace_lock);
+	dtrace_probe_provide(NULL, all ? NULL : prv);
+	lck_mtx_lock(&dtrace_lock);
+}
+
+/*
+ * DTrace DOF Functions
+ */
+/*ARGSUSED*/
+static void
+dtrace_dof_error(dof_hdr_t *dof, const char *str)
+{
+	if (dtrace_err_verbose)
+		cmn_err(CE_WARN, "failed to process DOF: %s", str);
+
+#ifdef DTRACE_ERRDEBUG
+	dtrace_errdebug(str);
+#endif
+}
+
+/*
+ * Create DOF out of a currently enabled state.  Right now, we only create
+ * DOF containing the run-time options -- but this could be expanded to create
+ * complete DOF representing the enabled state.
+ */
+static dof_hdr_t *
+dtrace_dof_create(dtrace_state_t *state)
+{
+	dof_hdr_t *dof;
+	dof_sec_t *sec;
+	dof_optdesc_t *opt;
+	int i, len = sizeof (dof_hdr_t) +
+	    roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
+	    sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	dof = dt_kmem_zalloc_aligned(len, 8, KM_SLEEP);
+	dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
+	dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
+	dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
+	dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
+
+	dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
+	dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
+	dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION;
+	dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
+	dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
+	dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
+
+	dof->dofh_flags = 0;
+	dof->dofh_hdrsize = sizeof (dof_hdr_t);
+	dof->dofh_secsize = sizeof (dof_sec_t);
+	dof->dofh_secnum = 1;	/* only DOF_SECT_OPTDESC */
+	dof->dofh_secoff = sizeof (dof_hdr_t);
+	dof->dofh_loadsz = len;
+	dof->dofh_filesz = len;
+	dof->dofh_pad = 0;
+
+	/*
+	 * Fill in the option section header...
+	 */
+	sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
+	sec->dofs_type = DOF_SECT_OPTDESC;
+	sec->dofs_align = sizeof (uint64_t);
+	sec->dofs_flags = DOF_SECF_LOAD;
+	sec->dofs_entsize = sizeof (dof_optdesc_t);
+
+	opt = (dof_optdesc_t *)((uintptr_t)sec +
+	    roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
+
+	sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
+	sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
+
+	for (i = 0; i < DTRACEOPT_MAX; i++) {
+		opt[i].dofo_option = i;
+		opt[i].dofo_strtab = DOF_SECIDX_NONE;
+		opt[i].dofo_value = state->dts_options[i];
+	}
+
+	return (dof);
+}
+
+static dof_hdr_t *
+#if defined(__APPLE__)
+dtrace_dof_copyin(user_addr_t uarg, int *errp)
+#else
+dtrace_dof_copyin(uintptr_t uarg, int *errp)
+#endif
+{
+	dof_hdr_t hdr, *dof;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED);
+
+	/*
+	 * First, we're going to copyin() the sizeof (dof_hdr_t).
+	 */
+#if defined(__APPLE__)
+	if (copyin(uarg, &hdr, sizeof (hdr)) != 0) {
+#else
+	if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
+#endif
+		dtrace_dof_error(NULL, "failed to copyin DOF header");
+		*errp = EFAULT;
+		return (NULL);
+	}
+
+	/*
+	 * Now we'll allocate the entire DOF and copy it in -- provided
+	 * that the length isn't outrageous.
+	 */
+	if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
+		dtrace_dof_error(&hdr, "load size exceeds maximum");
+		*errp = E2BIG;
+		return (NULL);
+	}
+
+	if (hdr.dofh_loadsz < sizeof (hdr)) {
+		dtrace_dof_error(&hdr, "invalid load size");
+		*errp = EINVAL;
+		return (NULL);
+	}
+
+	dof = dt_kmem_alloc_aligned(hdr.dofh_loadsz, 8, KM_SLEEP);
+
+#if defined(__APPLE__)
+	if (copyin(uarg, dof, hdr.dofh_loadsz) != 0) {
+#else
+	if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0) {
+#endif
+		dt_kmem_free_aligned(dof, hdr.dofh_loadsz);
+		*errp = EFAULT;
+		return (NULL);
+	}
+
+	return (dof);
+}
+
+#if defined(__APPLE__)
+
+static dof_hdr_t *
+dtrace_dof_copyin_from_proc(proc_t* p, user_addr_t uarg, int *errp)
+{
+	dof_hdr_t hdr, *dof;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED);
+
+	/*
+	 * First, we're going to copyin() the sizeof (dof_hdr_t).
+	 */
+	if (uread(p, &hdr, sizeof(hdr), uarg) != KERN_SUCCESS) {
+		dtrace_dof_error(NULL, "failed to copyin DOF header");
+		*errp = EFAULT;
+		return (NULL);
+	}
+
+	/*
+	 * Now we'll allocate the entire DOF and copy it in -- provided
+	 * that the length isn't outrageous.
+	 */
+	if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
+		dtrace_dof_error(&hdr, "load size exceeds maximum");
+		*errp = E2BIG;
+		return (NULL);
+	}
+
+	if (hdr.dofh_loadsz < sizeof (hdr)) {
+		dtrace_dof_error(&hdr, "invalid load size");
+		*errp = EINVAL;
+		return (NULL);
+	}
+
+	dof = dt_kmem_alloc_aligned(hdr.dofh_loadsz, 8, KM_SLEEP);
+
+	if (uread(p, dof, hdr.dofh_loadsz, uarg) != KERN_SUCCESS) {
+		dt_kmem_free_aligned(dof, hdr.dofh_loadsz);
+		*errp = EFAULT;
+		return (NULL);
+	}
+
+	return (dof);
+}
+
+#endif /* __APPLE__ */
+
+static dof_hdr_t *
+dtrace_dof_property(const char *name)
+{
+	uchar_t *buf;
+	uint64_t loadsz;
+	unsigned int len, i;
+	dof_hdr_t *dof;
+
+	/*
+	 * Unfortunately, array of values in .conf files are always (and
+	 * only) interpreted to be integer arrays.  We must read our DOF
+	 * as an integer array, and then squeeze it into a byte array.
+	 */
+	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
+	    (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
+		return (NULL);
+
+	for (i = 0; i < len; i++)
+		buf[i] = (uchar_t)(((int *)buf)[i]);
+
+	if (len < sizeof (dof_hdr_t)) {
+		ddi_prop_free(buf);
+		dtrace_dof_error(NULL, "truncated header");
+		return (NULL);
+	}
+
+	if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
+		ddi_prop_free(buf);
+		dtrace_dof_error(NULL, "truncated DOF");
+		return (NULL);
+	}
+
+	if (loadsz >= dtrace_dof_maxsize) {
+		ddi_prop_free(buf);
+		dtrace_dof_error(NULL, "oversized DOF");
+		return (NULL);
+	}
+
+	dof = dt_kmem_alloc_aligned(loadsz, 8, KM_SLEEP);
+	bcopy(buf, dof, loadsz);
+	ddi_prop_free(buf);
+
+	return (dof);
+}
+
+static void
+dtrace_dof_destroy(dof_hdr_t *dof)
+{
+	dt_kmem_free_aligned(dof, dof->dofh_loadsz);
+}
+
+/*
+ * Return the dof_sec_t pointer corresponding to a given section index.  If the
+ * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
+ * a type other than DOF_SECT_NONE is specified, the header is checked against
+ * this type and NULL is returned if the types do not match.
+ */
+static dof_sec_t *
+dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
+{
+	dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
+	    ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
+
+	if (i >= dof->dofh_secnum) {
+		dtrace_dof_error(dof, "referenced section index is invalid");
+		return (NULL);
+	}
+
+	if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
+		dtrace_dof_error(dof, "referenced section is not loadable");
+		return (NULL);
+	}
+
+	if (type != DOF_SECT_NONE && type != sec->dofs_type) {
+		dtrace_dof_error(dof, "referenced section is the wrong type");
+		return (NULL);
+	}
+
+	return (sec);
+}
+
+static dtrace_probedesc_t *
+dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
+{
+	dof_probedesc_t *probe;
+	dof_sec_t *strtab;
+	uintptr_t daddr = (uintptr_t)dof;
+	uintptr_t str;
+	size_t size;
+
+	if (sec->dofs_type != DOF_SECT_PROBEDESC) {
+		dtrace_dof_error(dof, "invalid probe section");
+		return (NULL);
+	}
+
+	if (sec->dofs_align != sizeof (dof_secidx_t)) {
+		dtrace_dof_error(dof, "bad alignment in probe description");
+		return (NULL);
+	}
+
+	if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
+		dtrace_dof_error(dof, "truncated probe description");
+		return (NULL);
+	}
+
+	probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
+
+	if (strtab == NULL)
+		return (NULL);
+
+	str = daddr + strtab->dofs_offset;
+	size = strtab->dofs_size;
+
+	if (probe->dofp_provider >= strtab->dofs_size) {
+		dtrace_dof_error(dof, "corrupt probe provider");
+		return (NULL);
+	}
+
+	(void) strncpy(desc->dtpd_provider,
+	    (char *)(str + probe->dofp_provider),
+	    MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
+
+	if (probe->dofp_mod >= strtab->dofs_size) {
+		dtrace_dof_error(dof, "corrupt probe module");
+		return (NULL);
+	}
+
+	(void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
+	    MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
+
+	if (probe->dofp_func >= strtab->dofs_size) {
+		dtrace_dof_error(dof, "corrupt probe function");
+		return (NULL);
+	}
+
+	(void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
+	    MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
+
+	if (probe->dofp_name >= strtab->dofs_size) {
+		dtrace_dof_error(dof, "corrupt probe name");
+		return (NULL);
+	}
+
+	(void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
+	    MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
+
+	return (desc);
+}
+
+static dtrace_difo_t *
+dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
+    cred_t *cr)
+{
+	dtrace_difo_t *dp;
+	size_t ttl = 0;
+	dof_difohdr_t *dofd;
+	uintptr_t daddr = (uintptr_t)dof;
+	size_t max = dtrace_difo_maxsize;
+	int i, l, n;
+
+	static const struct {
+		int section;
+		int bufoffs;
+		int lenoffs;
+		int entsize;
+		int align;
+		const char *msg;
+	} difo[] = {
+		{ DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
+		offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
+		sizeof (dif_instr_t), "multiple DIF sections" },
+
+		{ DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
+		offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
+		sizeof (uint64_t), "multiple integer tables" },
+
+		{ DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
+		offsetof(dtrace_difo_t, dtdo_strlen), 0,
+		sizeof (char), "multiple string tables" },
+
+		{ DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
+		offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
+		sizeof (uint_t), "multiple variable tables" },
+
+#if !defined(__APPLE__)
+		{ DOF_SECT_NONE, 0, 0, 0, NULL }
+#else
+		{ DOF_SECT_NONE, 0, 0, 0, 0, NULL }
+#endif /* __APPLE__ */
+	};
+
+	if (sec->dofs_type != DOF_SECT_DIFOHDR) {
+		dtrace_dof_error(dof, "invalid DIFO header section");
+		return (NULL);
+	}
+
+	if (sec->dofs_align != sizeof (dof_secidx_t)) {
+		dtrace_dof_error(dof, "bad alignment in DIFO header");
+		return (NULL);
+	}
+
+	if (sec->dofs_size < sizeof (dof_difohdr_t) ||
+	    sec->dofs_size % sizeof (dof_secidx_t)) {
+		dtrace_dof_error(dof, "bad size in DIFO header");
+		return (NULL);
+	}
+
+	dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
+
+	dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
+	dp->dtdo_rtype = dofd->dofd_rtype;
+
+	for (l = 0; l < n; l++) {
+		dof_sec_t *subsec;
+		void **bufp;
+		uint32_t *lenp;
+
+		if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
+		    dofd->dofd_links[l])) == NULL)
+			goto err; /* invalid section link */
+
+		if (ttl + subsec->dofs_size > max) {
+			dtrace_dof_error(dof, "exceeds maximum size");
+			goto err;
+		}
+
+		ttl += subsec->dofs_size;
+
+		for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
+			if (subsec->dofs_type != difo[i].section)
+				continue;
+
+			if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
+				dtrace_dof_error(dof, "section not loaded");
+				goto err;
+			}
+
+			if (subsec->dofs_align != difo[i].align) {
+				dtrace_dof_error(dof, "bad alignment");
+				goto err;
+			}
+
+			bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
+			lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
+
+			if (*bufp != NULL) {
+				dtrace_dof_error(dof, difo[i].msg);
+				goto err;
+			}
+
+			if (difo[i].entsize != subsec->dofs_entsize) {
+				dtrace_dof_error(dof, "entry size mismatch");
+				goto err;
+			}
+
+			if (subsec->dofs_entsize != 0 &&
+			    (subsec->dofs_size % subsec->dofs_entsize) != 0) {
+				dtrace_dof_error(dof, "corrupt entry size");
+				goto err;
+			}
+
+			*lenp = subsec->dofs_size;
+			*bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
+			bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
+			    *bufp, subsec->dofs_size);
+
+			if (subsec->dofs_entsize != 0)
+				*lenp /= subsec->dofs_entsize;
+
+			break;
+		}
+
+		/*
+		 * If we encounter a loadable DIFO sub-section that is not
+		 * known to us, assume this is a broken program and fail.
+		 */
+		if (difo[i].section == DOF_SECT_NONE &&
+		    (subsec->dofs_flags & DOF_SECF_LOAD)) {
+			dtrace_dof_error(dof, "unrecognized DIFO subsection");
+			goto err;
+		}
+	}
+
+	if (dp->dtdo_buf == NULL) {
+		/*
+		 * We can't have a DIF object without DIF text.
+		 */
+		dtrace_dof_error(dof, "missing DIF text");
+		goto err;
+	}
+
+	/*
+	 * Before we validate the DIF object, run through the variable table
+	 * looking for the strings -- if any of their size are under, we'll set
+	 * their size to be the system-wide default string size.  Note that
+	 * this should _not_ happen if the "strsize" option has been set --
+	 * in this case, the compiler should have set the size to reflect the
+	 * setting of the option.
+	 */
+	for (i = 0; i < dp->dtdo_varlen; i++) {
+		dtrace_difv_t *v = &dp->dtdo_vartab[i];
+		dtrace_diftype_t *t = &v->dtdv_type;
+
+		if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
+			continue;
+
+		if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
+			t->dtdt_size = dtrace_strsize_default;
+	}
+
+	if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
+		goto err;
+
+	dtrace_difo_init(dp, vstate);
+	return (dp);
+
+err:
+	kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
+	kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
+	kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
+	kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
+
+	kmem_free(dp, sizeof (dtrace_difo_t));
+	return (NULL);
+}
+
+static dtrace_predicate_t *
+dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
+    cred_t *cr)
+{
+	dtrace_difo_t *dp;
+
+	if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
+		return (NULL);
+
+	return (dtrace_predicate_create(dp));
+}
+
+static dtrace_actdesc_t *
+dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
+    cred_t *cr)
+{
+	dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
+	dof_actdesc_t *desc;
+	dof_sec_t *difosec;
+	size_t offs;
+	uintptr_t daddr = (uintptr_t)dof;
+	uint64_t arg;
+	dtrace_actkind_t kind;
+
+	if (sec->dofs_type != DOF_SECT_ACTDESC) {
+		dtrace_dof_error(dof, "invalid action section");
+		return (NULL);
+	}
+
+	if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
+		dtrace_dof_error(dof, "truncated action description");
+		return (NULL);
+	}
+
+	if (sec->dofs_align != sizeof (uint64_t)) {
+		dtrace_dof_error(dof, "bad alignment in action description");
+		return (NULL);
+	}
+
+	if (sec->dofs_size < sec->dofs_entsize) {
+		dtrace_dof_error(dof, "section entry size exceeds total size");
+		return (NULL);
+	}
+
+	if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
+		dtrace_dof_error(dof, "bad entry size in action description");
+		return (NULL);
+	}
+
+	if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
+		dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
+		return (NULL);
+	}
+
+	for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
+		desc = (dof_actdesc_t *)(daddr +
+		    (uintptr_t)sec->dofs_offset + offs);
+		kind = (dtrace_actkind_t)desc->dofa_kind;
+
+		if (DTRACEACT_ISPRINTFLIKE(kind) &&
+		    (kind != DTRACEACT_PRINTA ||
+		    desc->dofa_strtab != DOF_SECIDX_NONE)) {
+			dof_sec_t *strtab;
+			char *str, *fmt;
+			uint64_t i;
+
+			/*
+			 * printf()-like actions must have a format string.
+			 */
+			if ((strtab = dtrace_dof_sect(dof,
+			    DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
+				goto err;
+
+			str = (char *)((uintptr_t)dof +
+			    (uintptr_t)strtab->dofs_offset);
+
+			for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
+				if (str[i] == '\0')
+					break;
+			}
+
+			if (i >= strtab->dofs_size) {
+				dtrace_dof_error(dof, "bogus format string");
+				goto err;
+			}
+
+			if (i == desc->dofa_arg) {
+				dtrace_dof_error(dof, "empty format string");
+				goto err;
+			}
+
+			i -= desc->dofa_arg;
+			fmt = kmem_alloc(i + 1, KM_SLEEP);
+			bcopy(&str[desc->dofa_arg], fmt, i + 1);
+			arg = (uint64_t)(uintptr_t)fmt;
+		} else {
+			if (kind == DTRACEACT_PRINTA) {
+				ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
+				arg = 0;
+			} else {
+				arg = desc->dofa_arg;
+			}
+		}
+
+		act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
+		    desc->dofa_uarg, arg);
+
+		if (last != NULL) {
+			last->dtad_next = act;
+		} else {
+			first = act;
+		}
+
+		last = act;
+
+		if (desc->dofa_difo == DOF_SECIDX_NONE)
+			continue;
+
+		if ((difosec = dtrace_dof_sect(dof,
+		    DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
+			goto err;
+
+		act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
+
+		if (act->dtad_difo == NULL)
+			goto err;
+	}
+
+	ASSERT(first != NULL);
+	return (first);
+
+err:
+	for (act = first; act != NULL; act = next) {
+		next = act->dtad_next;
+		dtrace_actdesc_release(act, vstate);
+	}
+
+	return (NULL);
+}
+
+static dtrace_ecbdesc_t *
+dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
+    cred_t *cr)
+{
+	dtrace_ecbdesc_t *ep;
+	dof_ecbdesc_t *ecb;
+	dtrace_probedesc_t *desc;
+	dtrace_predicate_t *pred = NULL;
+
+	if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
+		dtrace_dof_error(dof, "truncated ECB description");
+		return (NULL);
+	}
+
+	if (sec->dofs_align != sizeof (uint64_t)) {
+		dtrace_dof_error(dof, "bad alignment in ECB description");
+		return (NULL);
+	}
+
+	ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
+	sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
+
+	if (sec == NULL)
+		return (NULL);
+
+	ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
+	ep->dted_uarg = ecb->dofe_uarg;
+	desc = &ep->dted_probe;
+
+	if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
+		goto err;
+
+	if (ecb->dofe_pred != DOF_SECIDX_NONE) {
+		if ((sec = dtrace_dof_sect(dof,
+		    DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
+			goto err;
+
+		if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
+			goto err;
+
+		ep->dted_pred.dtpdd_predicate = pred;
+	}
+
+	if (ecb->dofe_actions != DOF_SECIDX_NONE) {
+		if ((sec = dtrace_dof_sect(dof,
+		    DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
+			goto err;
+
+		ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
+
+		if (ep->dted_action == NULL)
+			goto err;
+	}
+
+	return (ep);
+
+err:
+	if (pred != NULL)
+		dtrace_predicate_release(pred, vstate);
+	kmem_free(ep, sizeof (dtrace_ecbdesc_t));
+	return (NULL);
+}
+
+#if !defined(__APPLE__) /* APPLE dyld has already done this for us */
+/*
+ * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
+ * specified DOF.  At present, this amounts to simply adding 'ubase' to the
+ * site of any user SETX relocations to account for load object base address.
+ * In the future, if we need other relocations, this function can be extended.
+ */
+static int
+dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
+{
+	uintptr_t daddr = (uintptr_t)dof;
+	dof_relohdr_t *dofr =
+	    (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	dof_sec_t *ss, *rs, *ts;
+	dof_relodesc_t *r;
+	uint_t i, n;
+
+	if (sec->dofs_size < sizeof (dof_relohdr_t) ||
+	    sec->dofs_align != sizeof (dof_secidx_t)) {
+		dtrace_dof_error(dof, "invalid relocation header");
+		return (-1);
+	}
+
+	ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
+	rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
+	ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
+
+	if (ss == NULL || rs == NULL || ts == NULL)
+		return (-1); /* dtrace_dof_error() has been called already */
+
+	if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
+	    rs->dofs_align != sizeof (uint64_t)) {
+		dtrace_dof_error(dof, "invalid relocation section");
+		return (-1);
+	}
+
+	r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
+	n = rs->dofs_size / rs->dofs_entsize;
+
+	for (i = 0; i < n; i++) {
+		uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
+
+		switch (r->dofr_type) {
+		case DOF_RELO_NONE:
+			break;
+		case DOF_RELO_SETX:
+			if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
+			    sizeof (uint64_t) > ts->dofs_size) {
+				dtrace_dof_error(dof, "bad relocation offset");
+				return (-1);
+			}
+
+			if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
+				dtrace_dof_error(dof, "misaligned setx relo");
+				return (-1);
+			}
+
+			*(uint64_t *)taddr += ubase;
+			break;
+		default:
+			dtrace_dof_error(dof, "invalid relocation type");
+			return (-1);
+		}
+
+		r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
+	}
+
+	return (0);
+}
+#endif /* __APPLE__ */
+
+/*
+ * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
+ * header:  it should be at the front of a memory region that is at least
+ * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
+ * size.  It need not be validated in any other way.
+ */
+static int
+dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
+    dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
+{
+	uint64_t len = dof->dofh_loadsz, seclen;
+	uintptr_t daddr = (uintptr_t)dof;
+	dtrace_ecbdesc_t *ep;
+	dtrace_enabling_t *enab;
+	uint_t i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
+
+	/*
+	 * Check the DOF header identification bytes.  In addition to checking
+	 * valid settings, we also verify that unused bits/bytes are zeroed so
+	 * we can use them later without fear of regressing existing binaries.
+	 */
+	if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
+	    DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
+		dtrace_dof_error(dof, "DOF magic string mismatch");
+		return (-1);
+	}
+
+	if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
+	    dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
+		dtrace_dof_error(dof, "DOF has invalid data model");
+		return (-1);
+	}
+
+	if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
+		dtrace_dof_error(dof, "DOF encoding mismatch");
+		return (-1);
+	}
+
+#if !defined(__APPLE__)
+	if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
+	    dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) {
+		dtrace_dof_error(dof, "DOF version mismatch");
+		return (-1);
+	}
+#else
+	/*
+	 * We only support DOF_VERSION_3 for now.
+	 */
+	if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_3) {
+		dtrace_dof_error(dof, "DOF version mismatch");
+		return (-1);
+	}
+#endif
+
+	if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
+		dtrace_dof_error(dof, "DOF uses unsupported instruction set");
+		return (-1);
+	}
+
+	if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
+		dtrace_dof_error(dof, "DOF uses too many integer registers");
+		return (-1);
+	}
+
+	if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
+		dtrace_dof_error(dof, "DOF uses too many tuple registers");
+		return (-1);
+	}
+
+	for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
+		if (dof->dofh_ident[i] != 0) {
+			dtrace_dof_error(dof, "DOF has invalid ident byte set");
+			return (-1);
+		}
+	}
+
+	if (dof->dofh_flags & ~DOF_FL_VALID) {
+		dtrace_dof_error(dof, "DOF has invalid flag bits set");
+		return (-1);
+	}
+
+	if (dof->dofh_secsize == 0) {
+		dtrace_dof_error(dof, "zero section header size");
+		return (-1);
+	}
+
+	/*
+	 * Check that the section headers don't exceed the amount of DOF
+	 * data.  Note that we cast the section size and number of sections
+	 * to uint64_t's to prevent possible overflow in the multiplication.
+	 */
+	seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
+
+	if (dof->dofh_secoff > len || seclen > len ||
+	    dof->dofh_secoff + seclen > len) {
+		dtrace_dof_error(dof, "truncated section headers");
+		return (-1);
+	}
+
+	if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
+		dtrace_dof_error(dof, "misaligned section headers");
+		return (-1);
+	}
+
+	if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
+		dtrace_dof_error(dof, "misaligned section size");
+		return (-1);
+	}
+
+	/*
+	 * Take an initial pass through the section headers to be sure that
+	 * the headers don't have stray offsets.  If the 'noprobes' flag is
+	 * set, do not permit sections relating to providers, probes, or args.
+	 */
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)(daddr +
+		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (noprobes) {
+			switch (sec->dofs_type) {
+			case DOF_SECT_PROVIDER:
+			case DOF_SECT_PROBES:
+			case DOF_SECT_PRARGS:
+			case DOF_SECT_PROFFS:
+				dtrace_dof_error(dof, "illegal sections "
+				    "for enabling");
+				return (-1);
+			}
+		}
+
+		if (!(sec->dofs_flags & DOF_SECF_LOAD))
+			continue; /* just ignore non-loadable sections */
+
+		if (sec->dofs_align & (sec->dofs_align - 1)) {
+			dtrace_dof_error(dof, "bad section alignment");
+			return (-1);
+		}
+
+		if (sec->dofs_offset & (sec->dofs_align - 1)) {
+			dtrace_dof_error(dof, "misaligned section");
+			return (-1);
+		}
+
+		if (sec->dofs_offset > len || sec->dofs_size > len ||
+		    sec->dofs_offset + sec->dofs_size > len) {
+			dtrace_dof_error(dof, "corrupt section header");
+			return (-1);
+		}
+
+		if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
+		    sec->dofs_offset + sec->dofs_size - 1) != '\0') {
+			dtrace_dof_error(dof, "non-terminating string table");
+			return (-1);
+		}
+	}
+
+#if !defined(__APPLE__)
+	/*
+	 * APPLE NOTE: We have no relocation to perform. All dof values are
+	 * relative offsets.
+	 */
+
+	/*
+	 * Take a second pass through the sections and locate and perform any
+	 * relocations that are present.  We do this after the first pass to
+	 * be sure that all sections have had their headers validated.
+	 */
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)(daddr +
+		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (!(sec->dofs_flags & DOF_SECF_LOAD))
+			continue; /* skip sections that are not loadable */
+
+		switch (sec->dofs_type) {
+		case DOF_SECT_URELHDR:
+			if (dtrace_dof_relocate(dof, sec, ubase) != 0)
+				return (-1);
+			break;
+		}
+	}
+#endif /* __APPLE__ */
+
+	if ((enab = *enabp) == NULL)
+		enab = *enabp = dtrace_enabling_create(vstate);
+
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)(daddr +
+		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (sec->dofs_type != DOF_SECT_ECBDESC)
+			continue;
+
+#if !defined(__APPLE__)
+		if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
+			dtrace_enabling_destroy(enab);
+			*enabp = NULL;
+			return (-1);
+		}
+#else
+		/* XXX Defend against gcc 4.0 botch on x86 (not all paths out of inlined dtrace_dof_ecbdesc
+		   are checked for the NULL return value.) */
+		ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr);
+		if (ep == NULL) {
+			dtrace_enabling_destroy(enab);
+			*enabp = NULL;
+			return (-1);
+		}
+#endif /* __APPLE__ */
+		
+		dtrace_enabling_add(enab, ep);
+	}
+
+	return (0);
+}
+
+/*
+ * Process DOF for any options.  This routine assumes that the DOF has been
+ * at least processed by dtrace_dof_slurp().
+ */
+static int
+dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
+{
+	int i, rval;
+	uint32_t entsize;
+	size_t offs;
+	dof_optdesc_t *desc;
+
+	for (i = 0; i < dof->dofh_secnum; i++) {
+		dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
+		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
+
+		if (sec->dofs_type != DOF_SECT_OPTDESC)
+			continue;
+
+		if (sec->dofs_align != sizeof (uint64_t)) {
+			dtrace_dof_error(dof, "bad alignment in "
+			    "option description");
+			return (EINVAL);
+		}
+
+		if ((entsize = sec->dofs_entsize) == 0) {
+			dtrace_dof_error(dof, "zeroed option entry size");
+			return (EINVAL);
+		}
+
+		if (entsize < sizeof (dof_optdesc_t)) {
+			dtrace_dof_error(dof, "bad option entry size");
+			return (EINVAL);
+		}
+
+		for (offs = 0; offs < sec->dofs_size; offs += entsize) {
+			desc = (dof_optdesc_t *)((uintptr_t)dof +
+			    (uintptr_t)sec->dofs_offset + offs);
+
+			if (desc->dofo_strtab != DOF_SECIDX_NONE) {
+				dtrace_dof_error(dof, "non-zero option string");
+				return (EINVAL);
+			}
+
+			if (desc->dofo_value == DTRACEOPT_UNSET) {
+				dtrace_dof_error(dof, "unset option");
+				return (EINVAL);
+			}
+
+			if ((rval = dtrace_state_option(state,
+			    desc->dofo_option, desc->dofo_value)) != 0) {
+				dtrace_dof_error(dof, "rejected option");
+				return (rval);
+			}
+		}
+	}
+
+	return (0);
+}
+
+/*
+ * DTrace Consumer State Functions
+ */
+#if defined(__APPLE__)
+static 
+#endif /* __APPLE__ */
+int
+dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
+{
+	size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
+	void *base;
+	uintptr_t limit;
+	dtrace_dynvar_t *dvar, *next, *start;
+	int i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
+
+	bzero(dstate, sizeof (dtrace_dstate_t));
+
+	if ((dstate->dtds_chunksize = chunksize) == 0)
+		dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
+
+	if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
+		size = min;
+
+	if ((base = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
+		return (ENOMEM);
+
+	dstate->dtds_size = size;
+	dstate->dtds_base = base;
+	dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
+	bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
+
+	hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
+
+	if (hashsize != 1 && (hashsize & 1))
+		hashsize--;
+
+	dstate->dtds_hashsize = hashsize;
+	dstate->dtds_hash = dstate->dtds_base;
+
+	/*
+	 * Set all of our hash buckets to point to the single sink, and (if
+	 * it hasn't already been set), set the sink's hash value to be the
+	 * sink sentinel value.  The sink is needed for dynamic variable
+	 * lookups to know that they have iterated over an entire, valid hash
+	 * chain.
+	 */
+	for (i = 0; i < hashsize; i++)
+		dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink;
+
+	if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK)
+		dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK;
+
+	/*
+	 * Determine number of active CPUs.  Divide free list evenly among
+	 * active CPUs.
+	 */
+	start = (dtrace_dynvar_t *)
+	    ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
+	limit = (uintptr_t)base + size;
+
+	maxper = (limit - (uintptr_t)start) / NCPU;
+	maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
+
+	for (i = 0; i < NCPU; i++) {
+		dstate->dtds_percpu[i].dtdsc_free = dvar = start;
+
+		/*
+		 * If we don't even have enough chunks to make it once through
+		 * NCPUs, we're just going to allocate everything to the first
+		 * CPU.  And if we're on the last CPU, we're going to allocate
+		 * whatever is left over.  In either case, we set the limit to
+		 * be the limit of the dynamic variable space.
+		 */
+		if (maxper == 0 || i == NCPU - 1) {
+			limit = (uintptr_t)base + size;
+			start = NULL;
+		} else {
+			limit = (uintptr_t)start + maxper;
+			start = (dtrace_dynvar_t *)limit;
+		}
+
+		ASSERT(limit <= (uintptr_t)base + size);
+
+		for (;;) {
+			next = (dtrace_dynvar_t *)((uintptr_t)dvar +
+			    dstate->dtds_chunksize);
+
+			if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
+				break;
+
+			dvar->dtdv_next = next;
+			dvar = next;
+		}
+
+		if (maxper == 0)
+			break;
+	}
+
+	return (0);
+}
+
+#if defined(__APPLE__)
+static 
+#endif /* __APPLE__ */
+void
+dtrace_dstate_fini(dtrace_dstate_t *dstate)
+{
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (dstate->dtds_base == NULL)
+		return;
+
+	kmem_free(dstate->dtds_base, dstate->dtds_size);
+	kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
+}
+
+static void
+dtrace_vstate_fini(dtrace_vstate_t *vstate)
+{
+	/*
+	 * Logical XOR, where are you?
+	 */
+	ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
+
+	if (vstate->dtvs_nglobals > 0) {
+		kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
+		    sizeof (dtrace_statvar_t *));
+	}
+
+	if (vstate->dtvs_ntlocals > 0) {
+		kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
+		    sizeof (dtrace_difv_t));
+	}
+
+	ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
+
+	if (vstate->dtvs_nlocals > 0) {
+		kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
+		    sizeof (dtrace_statvar_t *));
+	}
+}
+
+static void
+dtrace_state_clean(dtrace_state_t *state)
+{
+	if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
+		return;
+
+	dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
+	dtrace_speculation_clean(state);
+}
+
+static void
+dtrace_state_deadman(dtrace_state_t *state)
+{
+	hrtime_t now;
+
+	dtrace_sync();
+
+	now = dtrace_gethrtime();
+
+	if (state != dtrace_anon.dta_state &&
+	    now - state->dts_laststatus >= dtrace_deadman_user)
+		return;
+
+	/*
+	 * We must be sure that dts_alive never appears to be less than the
+	 * value upon entry to dtrace_state_deadman(), and because we lack a
+	 * dtrace_cas64(), we cannot store to it atomically.  We thus instead
+	 * store INT64_MAX to it, followed by a memory barrier, followed by
+	 * the new value.  This assures that dts_alive never appears to be
+	 * less than its true value, regardless of the order in which the
+	 * stores to the underlying storage are issued.
+	 */
+	state->dts_alive = INT64_MAX;
+	dtrace_membar_producer();
+	state->dts_alive = now;
+}
+
+#if defined(__APPLE__)
+static 
+#endif /* __APPLE__ */
+dtrace_state_t *
+dtrace_state_create(dev_t *devp, cred_t *cr)
+{
+	minor_t minor;
+	major_t major;
+	char c[30];
+	dtrace_state_t *state;
+	dtrace_optval_t *opt;
+	int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+#if !defined(__APPLE__)
+	minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
+	    VM_BESTFIT | VM_SLEEP);
+#else
+	/*
+	 * Darwin's DEVFS layer acquired the minor number for this "device" when it called
+	 * dtrace_devfs_clone_func(). At that time, dtrace_devfs_clone_func() proposed a minor number
+	 * (next unused according to vmem_alloc()) and then immediately put the number back in play
+	 * (by calling vmem_free()). Now that minor number is being used for an open, so committing it
+	 * to use. The following vmem_alloc() must deliver that same minor number.
+	 */
+
+	minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
+	    VM_BESTFIT | VM_SLEEP);
+
+	if (NULL != devp) {
+    	ASSERT(getminor(*devp) == minor);
+		if (getminor(*devp) != minor) {
+			printf("dtrace_open: couldn't re-acquire vended minor number %d. Instead got %d\n", 
+					getminor(*devp), minor);
+			vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
+			return NULL;
+		}
+	} else {
+        /* NULL==devp iff "Anonymous state" (see dtrace_anon_property),
+		 * so just vend the minor device number here de novo since no "open" has occurred. */
+	}
+
+#endif /* __APPLE__ */
+
+	if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
+		vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
+		return (NULL);
+	}
+
+	state = ddi_get_soft_state(dtrace_softstate, minor);
+	state->dts_epid = DTRACE_EPIDNONE + 1;
+
+	(void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
+	state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
+	    NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
+
+	if (devp != NULL) {
+		major = getemajor(*devp);
+	} else {
+		major = ddi_driver_major(dtrace_devi);
+	}
+
+	state->dts_dev = makedevice(major, minor);
+
+	if (devp != NULL)
+		*devp = state->dts_dev;
+
+	/*
+	 * We allocate NCPU buffers.  On the one hand, this can be quite
+	 * a bit of memory per instance (nearly 36K on a Starcat).  On the
+	 * other hand, it saves an additional memory reference in the probe
+	 * path.
+	 */
+	state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
+	state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
+	state->dts_cleaner = CYCLIC_NONE;
+	state->dts_deadman = CYCLIC_NONE;
+	state->dts_vstate.dtvs_state = state;
+
+	for (i = 0; i < DTRACEOPT_MAX; i++)
+		state->dts_options[i] = DTRACEOPT_UNSET;
+
+	/*
+	 * Set the default options.
+	 */
+	opt = state->dts_options;
+	opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
+	opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
+	opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
+	opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
+	opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
+	opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
+	opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
+	opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
+	opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
+	opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
+	opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
+	opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
+	opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
+	opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
+
+	state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
+
+	/*
+	 * Depending on the user credentials, we set flag bits which alter probe
+	 * visibility or the amount of destructiveness allowed.  In the case of
+	 * actual anonymous tracing, or the possession of all privileges, all of
+	 * the normal checks are bypassed.
+	 */
+	if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
+		state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
+		state->dts_cred.dcr_action = DTRACE_CRA_ALL;
+	} else {
+		/*
+		 * Set up the credentials for this instantiation.  We take a
+		 * hold on the credential to prevent it from disappearing on
+		 * us; this in turn prevents the zone_t referenced by this
+		 * credential from disappearing.  This means that we can
+		 * examine the credential and the zone from probe context.
+		 */
+		crhold(cr);
+		state->dts_cred.dcr_cred = cr;
+
+		/*
+		 * CRA_PROC means "we have *some* privilege for dtrace" and
+		 * unlocks the use of variables like pid, zonename, etc.
+		 */
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
+		    PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
+			state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
+		}
+
+		/*
+		 * dtrace_user allows use of syscall and profile providers.
+		 * If the user also has proc_owner and/or proc_zone, we
+		 * extend the scope to include additional visibility and
+		 * destructive power.
+		 */
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) {
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
+				state->dts_cred.dcr_visible |=
+				    DTRACE_CRV_ALLPROC;
+
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
+			}
+
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) {
+				state->dts_cred.dcr_visible |=
+				    DTRACE_CRV_ALLZONE;
+
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
+			}
+
+			/*
+			 * If we have all privs in whatever zone this is,
+			 * we can do destructive things to processes which
+			 * have altered credentials.
+			 */
+#if !defined(__APPLE__)
+			if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
+			    cr->cr_zone->zone_privset)) {
+				state->dts_cred.dcr_action |=
+					DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
+			}
+#else
+			/* Darwin doesn't do zones. */
+			state->dts_cred.dcr_action |=
+				DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
+#endif /* __APPLE__ */
+		}
+
+		/*
+		 * Holding the dtrace_kernel privilege also implies that
+		 * the user has the dtrace_user privilege from a visibility
+		 * perspective.  But without further privileges, some
+		 * destructive actions are not available.
+		 */
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
+			/*
+			 * Make all probes in all zones visible.  However,
+			 * this doesn't mean that all actions become available
+			 * to all zones.
+			 */
+			state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
+			    DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE;
+
+			state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
+			    DTRACE_CRA_PROC;
+			/*
+			 * Holding proc_owner means that destructive actions
+			 * for *this* zone are allowed.
+			 */
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
+
+			/*
+			 * Holding proc_zone means that destructive actions
+			 * for this user/group ID in all zones is allowed.
+			 */
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
+
+			/*
+			 * If we have all privs in whatever zone this is,
+			 * we can do destructive things to processes which
+			 * have altered credentials.
+			 */
+#if !defined(__APPLE__)
+			if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE),
+			    cr->cr_zone->zone_privset)) {
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
+			}
+#else
+			/* Darwin doesn't do zones. */
+			state->dts_cred.dcr_action |=
+				DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG;
+#endif /* __APPLE__ */
+		}
+
+		/*
+		 * Holding the dtrace_proc privilege gives control over fasttrap
+		 * and pid providers.  We need to grant wider destructive
+		 * privileges in the event that the user has proc_owner and/or
+		 * proc_zone.
+		 */
+		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER;
+
+			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE))
+				state->dts_cred.dcr_action |=
+				    DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE;
+		}
+	}
+
+	return (state);
+}
+
+static int
+dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
+{
+	dtrace_optval_t *opt = state->dts_options, size;
+	processorid_t cpu;
+	int flags = 0, rval;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(which < DTRACEOPT_MAX);
+	ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
+	    (state == dtrace_anon.dta_state &&
+	    state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
+
+	if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
+		return (0);
+
+	if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
+		cpu = opt[DTRACEOPT_CPU];
+
+	if (which == DTRACEOPT_SPECSIZE)
+		flags |= DTRACEBUF_NOSWITCH;
+
+	if (which == DTRACEOPT_BUFSIZE) {
+		if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
+			flags |= DTRACEBUF_RING;
+
+		if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
+			flags |= DTRACEBUF_FILL;
+
+		if (state != dtrace_anon.dta_state ||
+		    state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
+			flags |= DTRACEBUF_INACTIVE;
+	}
+
+	for (size = opt[which]; size >= sizeof (uint64_t); size >>= 1) {
+		/*
+		 * The size must be 8-byte aligned.  If the size is not 8-byte
+		 * aligned, drop it down by the difference.
+		 */
+		if (size & (sizeof (uint64_t) - 1))
+			size -= size & (sizeof (uint64_t) - 1);
+
+		if (size < state->dts_reserve) {
+			/*
+			 * Buffers always must be large enough to accommodate
+			 * their prereserved space.  We return E2BIG instead
+			 * of ENOMEM in this case to allow for user-level
+			 * software to differentiate the cases.
+			 */
+			return (E2BIG);
+		}
+
+		rval = dtrace_buffer_alloc(buf, size, flags, cpu);
+
+		if (rval != ENOMEM) {
+			opt[which] = size;
+			return (rval);
+		}
+
+		if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
+			return (rval);
+	}
+
+	return (ENOMEM);
+}
+
+static int
+dtrace_state_buffers(dtrace_state_t *state)
+{
+	dtrace_speculation_t *spec = state->dts_speculations;
+	int rval, i;
+
+	if ((rval = dtrace_state_buffer(state, state->dts_buffer,
+	    DTRACEOPT_BUFSIZE)) != 0)
+		return (rval);
+
+	if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
+	    DTRACEOPT_AGGSIZE)) != 0)
+		return (rval);
+
+	for (i = 0; i < state->dts_nspeculations; i++) {
+		if ((rval = dtrace_state_buffer(state,
+		    spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
+			return (rval);
+	}
+
+	return (0);
+}
+
+static void
+dtrace_state_prereserve(dtrace_state_t *state)
+{
+	dtrace_ecb_t *ecb;
+	dtrace_probe_t *probe;
+
+	state->dts_reserve = 0;
+
+	if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
+		return;
+
+	/*
+	 * If our buffer policy is a "fill" buffer policy, we need to set the
+	 * prereserved space to be the space required by the END probes.
+	 */
+	probe = dtrace_probes[dtrace_probeid_end - 1];
+	ASSERT(probe != NULL);
+
+	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
+		if (ecb->dte_state != state)
+			continue;
+
+		state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
+	}
+}
+
+static int
+dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
+{
+	dtrace_optval_t *opt = state->dts_options, sz, nspec;
+	dtrace_speculation_t *spec;
+	dtrace_buffer_t *buf;
+	cyc_handler_t hdlr;
+	cyc_time_t when;
+	int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
+	dtrace_icookie_t cookie;
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
+		rval = EBUSY;
+		goto out;
+	}
+
+	/*
+	 * Before we can perform any checks, we must prime all of the
+	 * retained enablings that correspond to this state.
+	 */
+	dtrace_enabling_prime(state);
+
+	if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
+		rval = EACCES;
+		goto out;
+	}
+
+	dtrace_state_prereserve(state);
+
+	/*
+	 * Now we want to do is try to allocate our speculations.
+	 * We do not automatically resize the number of speculations; if
+	 * this fails, we will fail the operation.
+	 */
+	nspec = opt[DTRACEOPT_NSPEC];
+	ASSERT(nspec != DTRACEOPT_UNSET);
+
+	if (nspec > INT_MAX) {
+		rval = ENOMEM;
+		goto out;
+	}
+
+	spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t), KM_NOSLEEP);
+
+	if (spec == NULL) {
+		rval = ENOMEM;
+		goto out;
+	}
+
+	state->dts_speculations = spec;
+	state->dts_nspeculations = (int)nspec;
+
+	for (i = 0; i < nspec; i++) {
+		if ((buf = kmem_zalloc(bufsize, KM_NOSLEEP)) == NULL) {
+			rval = ENOMEM;
+			goto err;
+		}
+
+		spec[i].dtsp_buffer = buf;
+	}
+
+	if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
+		if (dtrace_anon.dta_state == NULL) {
+			rval = ENOENT;
+			goto out;
+		}
+
+		if (state->dts_necbs != 0) {
+			rval = EALREADY;
+			goto out;
+		}
+
+		state->dts_anon = dtrace_anon_grab();
+		ASSERT(state->dts_anon != NULL);
+		state = state->dts_anon;
+
+		/*
+		 * We want "grabanon" to be set in the grabbed state, so we'll
+		 * copy that option value from the grabbing state into the
+		 * grabbed state.
+		 */
+		state->dts_options[DTRACEOPT_GRABANON] =
+		    opt[DTRACEOPT_GRABANON];
+
+		*cpu = dtrace_anon.dta_beganon;
+
+		/*
+		 * If the anonymous state is active (as it almost certainly
+		 * is if the anonymous enabling ultimately matched anything),
+		 * we don't allow any further option processing -- but we
+		 * don't return failure.
+		 */
+		if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
+			goto out;
+	}
+
+	if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
+	    opt[DTRACEOPT_AGGSIZE] != 0) {
+		if (state->dts_aggregations == NULL) {
+			/*
+			 * We're not going to create an aggregation buffer
+			 * because we don't have any ECBs that contain
+			 * aggregations -- set this option to 0.
+			 */
+			opt[DTRACEOPT_AGGSIZE] = 0;
+		} else {
+			/*
+			 * If we have an aggregation buffer, we must also have
+			 * a buffer to use as scratch.
+			 */
+			if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
+			    opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
+				opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
+			}
+		}
+	}
+
+	if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
+	    opt[DTRACEOPT_SPECSIZE] != 0) {
+		if (!state->dts_speculates) {
+			/*
+			 * We're not going to create speculation buffers
+			 * because we don't have any ECBs that actually
+			 * speculate -- set the speculation size to 0.
+			 */
+			opt[DTRACEOPT_SPECSIZE] = 0;
+		}
+	}
+
+	/*
+	 * The bare minimum size for any buffer that we're actually going to
+	 * do anything to is sizeof (uint64_t).
+	 */
+	sz = sizeof (uint64_t);
+
+	if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
+	    (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
+	    (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
+		/*
+		 * A buffer size has been explicitly set to 0 (or to a size
+		 * that will be adjusted to 0) and we need the space -- we
+		 * need to return failure.  We return ENOSPC to differentiate
+		 * it from failing to allocate a buffer due to failure to meet
+		 * the reserve (for which we return E2BIG).
+		 */
+		rval = ENOSPC;
+		goto out;
+	}
+
+	if ((rval = dtrace_state_buffers(state)) != 0)
+		goto err;
+
+	if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
+		sz = dtrace_dstate_defsize;
+
+	do {
+		rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
+
+		if (rval == 0)
+			break;
+
+		if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
+			goto err;
+	} while (sz >>= 1);
+
+	opt[DTRACEOPT_DYNVARSIZE] = sz;
+
+	if (rval != 0)
+		goto err;
+
+	if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
+		opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
+
+	if (opt[DTRACEOPT_CLEANRATE] == 0)
+		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
+
+	if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
+		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
+
+	if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
+		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
+
+	hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
+	hdlr.cyh_arg = state;
+	hdlr.cyh_level = CY_LOW_LEVEL;
+
+	when.cyt_when = 0;
+	when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
+
+	state->dts_cleaner = cyclic_add(&hdlr, &when);
+
+	hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
+	hdlr.cyh_arg = state;
+	hdlr.cyh_level = CY_LOW_LEVEL;
+
+	when.cyt_when = 0;
+	when.cyt_interval = dtrace_deadman_interval;
+
+	state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
+	state->dts_deadman = cyclic_add(&hdlr, &when);
+
+	state->dts_activity = DTRACE_ACTIVITY_WARMUP;
+
+	/*
+	 * Now it's time to actually fire the BEGIN probe.  We need to disable
+	 * interrupts here both to record the CPU on which we fired the BEGIN
+	 * probe (the data from this CPU will be processed first at user
+	 * level) and to manually activate the buffer for this CPU.
+	 */
+	cookie = dtrace_interrupt_disable();
+	*cpu = CPU->cpu_id;
+	ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
+	state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
+
+	dtrace_probe(dtrace_probeid_begin,
+	    (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
+	dtrace_interrupt_enable(cookie);
+	/*
+	 * We may have had an exit action from a BEGIN probe; only change our
+	 * state to ACTIVE if we're still in WARMUP.
+	 */
+	ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
+	    state->dts_activity == DTRACE_ACTIVITY_DRAINING);
+
+	if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
+		state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
+
+	/*
+	 * Regardless of whether or not now we're in ACTIVE or DRAINING, we
+	 * want each CPU to transition its principal buffer out of the
+	 * INACTIVE state.  Doing this assures that no CPU will suddenly begin
+	 * processing an ECB halfway down a probe's ECB chain; all CPUs will
+	 * atomically transition from processing none of a state's ECBs to
+	 * processing all of them.
+	 */
+	dtrace_xcall(DTRACE_CPUALL,
+	    (dtrace_xcall_t)dtrace_buffer_activate, state);
+	goto out;
+
+err:
+	dtrace_buffer_free(state->dts_buffer);
+	dtrace_buffer_free(state->dts_aggbuffer);
+
+	if ((nspec = state->dts_nspeculations) == 0) {
+		ASSERT(state->dts_speculations == NULL);
+		goto out;
+	}
+
+	spec = state->dts_speculations;
+	ASSERT(spec != NULL);
+
+	for (i = 0; i < state->dts_nspeculations; i++) {
+		if ((buf = spec[i].dtsp_buffer) == NULL)
+			break;
+
+		dtrace_buffer_free(buf);
+		kmem_free(buf, bufsize);
+	}
+
+	kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
+	state->dts_nspeculations = 0;
+	state->dts_speculations = NULL;
+
+out:
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&cpu_lock);
+
+	return (rval);
+}
+
+static int
+dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
+{
+	dtrace_icookie_t cookie;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
+	    state->dts_activity != DTRACE_ACTIVITY_DRAINING)
+		return (EINVAL);
+
+	/*
+	 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
+	 * to be sure that every CPU has seen it.  See below for the details
+	 * on why this is done.
+	 */
+	state->dts_activity = DTRACE_ACTIVITY_DRAINING;
+	dtrace_sync();
+
+	/*
+	 * By this point, it is impossible for any CPU to be still processing
+	 * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
+	 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
+	 * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
+	 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
+	 * iff we're in the END probe.
+	 */
+	state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
+	dtrace_sync();
+	ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
+
+	/*
+	 * Finally, we can release the reserve and call the END probe.  We
+	 * disable interrupts across calling the END probe to allow us to
+	 * return the CPU on which we actually called the END probe.  This
+	 * allows user-land to be sure that this CPU's principal buffer is
+	 * processed last.
+	 */
+	state->dts_reserve = 0;
+
+	cookie = dtrace_interrupt_disable();
+	*cpu = CPU->cpu_id;
+	dtrace_probe(dtrace_probeid_end,
+	    (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
+	dtrace_interrupt_enable(cookie);
+
+	state->dts_activity = DTRACE_ACTIVITY_STOPPED;
+	dtrace_sync();
+
+	return (0);
+}
+
+static int
+dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
+    dtrace_optval_t val)
+{
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
+		return (EBUSY);
+
+	if (option >= DTRACEOPT_MAX)
+		return (EINVAL);
+
+	if (option != DTRACEOPT_CPU && val < 0)
+		return (EINVAL);
+
+	switch (option) {
+	case DTRACEOPT_DESTRUCTIVE:
+		if (dtrace_destructive_disallow)
+			return (EACCES);
+
+		state->dts_cred.dcr_destructive = 1;
+		break;
+
+	case DTRACEOPT_BUFSIZE:
+	case DTRACEOPT_DYNVARSIZE:
+	case DTRACEOPT_AGGSIZE:
+	case DTRACEOPT_SPECSIZE:
+	case DTRACEOPT_STRSIZE:
+		if (val < 0)
+			return (EINVAL);
+
+		if (val >= LONG_MAX) {
+			/*
+			 * If this is an otherwise negative value, set it to
+			 * the highest multiple of 128m less than LONG_MAX.
+			 * Technically, we're adjusting the size without
+			 * regard to the buffer resizing policy, but in fact,
+			 * this has no effect -- if we set the buffer size to
+			 * ~LONG_MAX and the buffer policy is ultimately set to
+			 * be "manual", the buffer allocation is guaranteed to
+			 * fail, if only because the allocation requires two
+			 * buffers.  (We set the the size to the highest
+			 * multiple of 128m because it ensures that the size
+			 * will remain a multiple of a megabyte when
+			 * repeatedly halved -- all the way down to 15m.)
+			 */
+			val = LONG_MAX - (1 << 27) + 1;
+		}
+	}
+
+	state->dts_options[option] = val;
+
+	return (0);
+}
+
+static void
+dtrace_state_destroy(dtrace_state_t *state)
+{
+	dtrace_ecb_t *ecb;
+	dtrace_vstate_t *vstate = &state->dts_vstate;
+	minor_t minor = getminor(state->dts_dev);
+	int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
+	dtrace_speculation_t *spec = state->dts_speculations;
+	int nspec = state->dts_nspeculations;
+	uint32_t match;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+	/*
+	 * First, retract any retained enablings for this state.
+	 */
+	dtrace_enabling_retract(state);
+	ASSERT(state->dts_nretained == 0);
+
+	if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE ||
+	    state->dts_activity == DTRACE_ACTIVITY_DRAINING) {
+		/*
+		 * We have managed to come into dtrace_state_destroy() on a
+		 * hot enabling -- almost certainly because of a disorderly
+		 * shutdown of a consumer.  (That is, a consumer that is
+		 * exiting without having called dtrace_stop().) In this case,
+		 * we're going to set our activity to be KILLED, and then
+		 * issue a sync to be sure that everyone is out of probe
+		 * context before we start blowing away ECBs.
+		 */
+		state->dts_activity = DTRACE_ACTIVITY_KILLED;
+		dtrace_sync();
+	}
+
+	/*
+	 * Release the credential hold we took in dtrace_state_create().
+	 */
+	if (state->dts_cred.dcr_cred != NULL)
+		crfree(state->dts_cred.dcr_cred);
+
+	/*
+	 * Now we can safely disable and destroy any enabled probes.  Because
+	 * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress
+	 * (especially if they're all enabled), we take two passes through the
+	 * ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes, and
+	 * in the second we disable whatever is left over.
+	 */
+	for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
+		for (i = 0; i < state->dts_necbs; i++) {
+			if ((ecb = state->dts_ecbs[i]) == NULL)
+				continue;
+
+			if (match && ecb->dte_probe != NULL) {
+				dtrace_probe_t *probe = ecb->dte_probe;
+				dtrace_provider_t *prov = probe->dtpr_provider;
+
+				if (!(prov->dtpv_priv.dtpp_flags & match))
+					continue;
+			}
+
+			dtrace_ecb_disable(ecb);
+			dtrace_ecb_destroy(ecb);
+		}
+
+		if (!match)
+			break;
+	}
+
+	/*
+	 * Before we free the buffers, perform one more sync to assure that
+	 * every CPU is out of probe context.
+	 */
+	dtrace_sync();
+
+	dtrace_buffer_free(state->dts_buffer);
+	dtrace_buffer_free(state->dts_aggbuffer);
+
+	for (i = 0; i < nspec; i++)
+		dtrace_buffer_free(spec[i].dtsp_buffer);
+
+	if (state->dts_cleaner != CYCLIC_NONE)
+		cyclic_remove(state->dts_cleaner);
+
+	if (state->dts_deadman != CYCLIC_NONE)
+		cyclic_remove(state->dts_deadman);
+
+	dtrace_dstate_fini(&vstate->dtvs_dynvars);
+	dtrace_vstate_fini(vstate);
+	kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
+
+	if (state->dts_aggregations != NULL) {
+#ifdef DEBUG
+		for (i = 0; i < state->dts_naggregations; i++)
+			ASSERT(state->dts_aggregations[i] == NULL);
+#endif
+		ASSERT(state->dts_naggregations > 0);
+		kmem_free(state->dts_aggregations,
+		    state->dts_naggregations * sizeof (dtrace_aggregation_t *));
+	}
+
+	kmem_free(state->dts_buffer, bufsize);
+	kmem_free(state->dts_aggbuffer, bufsize);
+
+	for (i = 0; i < nspec; i++)
+		kmem_free(spec[i].dtsp_buffer, bufsize);
+
+	kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
+
+	dtrace_format_destroy(state);
+
+	vmem_destroy(state->dts_aggid_arena);
+	ddi_soft_state_free(dtrace_softstate, minor);
+	vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
+}
+
+/*
+ * DTrace Anonymous Enabling Functions
+ */
+static dtrace_state_t *
+dtrace_anon_grab(void)
+{
+	dtrace_state_t *state;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if ((state = dtrace_anon.dta_state) == NULL) {
+		ASSERT(dtrace_anon.dta_enabling == NULL);
+		return (NULL);
+	}
+
+	ASSERT(dtrace_anon.dta_enabling != NULL);
+	ASSERT(dtrace_retained != NULL);
+
+	dtrace_enabling_destroy(dtrace_anon.dta_enabling);
+	dtrace_anon.dta_enabling = NULL;
+	dtrace_anon.dta_state = NULL;
+
+	return (state);
+}
+
+static void
+dtrace_anon_property(void)
+{
+	int i, rv;
+	dtrace_state_t *state;
+	dof_hdr_t *dof;
+	char c[32];		/* enough for "dof-data-" + digits */
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+	for (i = 0; ; i++) {
+		(void) snprintf(c, sizeof (c), "dof-data-%d", i);
+
+		dtrace_err_verbose = 1;
+
+		if ((dof = dtrace_dof_property(c)) == NULL) {
+			dtrace_err_verbose = 0;
+			break;
+		}
+
+		/*
+		 * We want to create anonymous state, so we need to transition
+		 * the kernel debugger to indicate that DTrace is active.  If
+		 * this fails (e.g. because the debugger has modified text in
+		 * some way), we won't continue with the processing.
+		 */
+		if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
+			cmn_err(CE_NOTE, "kernel debugger active; anonymous "
+			    "enabling ignored.");
+			dtrace_dof_destroy(dof);
+			break;
+		}
+
+		/*
+		 * If we haven't allocated an anonymous state, we'll do so now.
+		 */
+		if ((state = dtrace_anon.dta_state) == NULL) {
+			state = dtrace_state_create(NULL, NULL);
+			dtrace_anon.dta_state = state;
+
+			if (state == NULL) {
+				/*
+				 * This basically shouldn't happen:  the only
+				 * failure mode from dtrace_state_create() is a
+				 * failure of ddi_soft_state_zalloc() that
+				 * itself should never happen.  Still, the
+				 * interface allows for a failure mode, and
+				 * we want to fail as gracefully as possible:
+				 * we'll emit an error message and cease
+				 * processing anonymous state in this case.
+				 */
+				cmn_err(CE_WARN, "failed to create "
+				    "anonymous state");
+				dtrace_dof_destroy(dof);
+				break;
+			}
+		}
+
+		rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
+		    &dtrace_anon.dta_enabling, 0, B_TRUE);
+
+		if (rv == 0)
+			rv = dtrace_dof_options(dof, state);
+
+		dtrace_err_verbose = 0;
+		dtrace_dof_destroy(dof);
+
+		if (rv != 0) {
+			/*
+			 * This is malformed DOF; chuck any anonymous state
+			 * that we created.
+			 */
+			ASSERT(dtrace_anon.dta_enabling == NULL);
+			dtrace_state_destroy(state);
+			dtrace_anon.dta_state = NULL;
+			break;
+		}
+
+		ASSERT(dtrace_anon.dta_enabling != NULL);
+	}
+
+	if (dtrace_anon.dta_enabling != NULL) {
+		int rval;
+
+		/*
+		 * dtrace_enabling_retain() can only fail because we are
+		 * trying to retain more enablings than are allowed -- but
+		 * we only have one anonymous enabling, and we are guaranteed
+		 * to be allowed at least one retained enabling; we assert
+		 * that dtrace_enabling_retain() returns success.
+		 */
+		rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
+		ASSERT(rval == 0);
+
+		dtrace_enabling_dump(dtrace_anon.dta_enabling);
+	}
+}
+
+/*
+ * DTrace Helper Functions
+ */
+static void
+dtrace_helper_trace(dtrace_helper_action_t *helper,
+    dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
+{
+	uint32_t size, next, nnext, i;
+	dtrace_helptrace_t *ent;
+	uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+
+	if (!dtrace_helptrace_enabled)
+		return;
+
+	ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
+
+	/*
+	 * What would a tracing framework be without its own tracing
+	 * framework?  (Well, a hell of a lot simpler, for starters...)
+	 */
+	size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
+	    sizeof (uint64_t) - sizeof (uint64_t);
+
+	/*
+	 * Iterate until we can allocate a slot in the trace buffer.
+	 */
+	do {
+		next = dtrace_helptrace_next;
+
+		if (next + size < dtrace_helptrace_bufsize) {
+			nnext = next + size;
+		} else {
+			nnext = size;
+		}
+	} while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
+
+	/*
+	 * We have our slot; fill it in.
+	 */
+	if (nnext == size)
+		next = 0;
+
+	ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
+	ent->dtht_helper = helper;
+	ent->dtht_where = where;
+	ent->dtht_nlocals = vstate->dtvs_nlocals;
+
+	ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
+	    mstate->dtms_fltoffs : -1;
+	ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
+	ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
+
+	for (i = 0; i < vstate->dtvs_nlocals; i++) {
+		dtrace_statvar_t *svar;
+
+		if ((svar = vstate->dtvs_locals[i]) == NULL)
+			continue;
+
+		ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
+		ent->dtht_locals[i] =
+		    ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
+	}
+}
+
+static uint64_t
+dtrace_helper(int which, dtrace_mstate_t *mstate,
+    dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
+{
+	uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+	uint64_t sarg0 = mstate->dtms_arg[0];
+	uint64_t sarg1 = mstate->dtms_arg[1];
+	uint64_t rval;
+	dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
+	dtrace_helper_action_t *helper;
+	dtrace_vstate_t *vstate;
+	dtrace_difo_t *pred;
+	int i, trace = dtrace_helptrace_enabled;
+
+	ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
+
+	if (helpers == NULL)
+		return (0);
+
+	if ((helper = helpers->dthps_actions[which]) == NULL)
+		return (0);
+
+	vstate = &helpers->dthps_vstate;
+	mstate->dtms_arg[0] = arg0;
+	mstate->dtms_arg[1] = arg1;
+
+	/*
+	 * Now iterate over each helper.  If its predicate evaluates to 'true',
+	 * we'll call the corresponding actions.  Note that the below calls
+	 * to dtrace_dif_emulate() may set faults in machine state.  This is
+	 * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
+	 * the stored DIF offset with its own (which is the desired behavior).
+	 * Also, note the calls to dtrace_dif_emulate() may allocate scratch
+	 * from machine state; this is okay, too.
+	 */
+	for (; helper != NULL; helper = helper->dtha_next) {
+		if ((pred = helper->dtha_predicate) != NULL) {
+			if (trace)
+				dtrace_helper_trace(helper, mstate, vstate, 0);
+
+			if (!dtrace_dif_emulate(pred, mstate, vstate, state))
+				goto next;
+
+			if (*flags & CPU_DTRACE_FAULT)
+				goto err;
+		}
+
+		for (i = 0; i < helper->dtha_nactions; i++) {
+			if (trace)
+				dtrace_helper_trace(helper,
+				    mstate, vstate, i + 1);
+
+			rval = dtrace_dif_emulate(helper->dtha_actions[i],
+			    mstate, vstate, state);
+
+			if (*flags & CPU_DTRACE_FAULT)
+				goto err;
+		}
+
+next:
+		if (trace)
+			dtrace_helper_trace(helper, mstate, vstate,
+			    DTRACE_HELPTRACE_NEXT);
+	}
+
+	if (trace)
+		dtrace_helper_trace(helper, mstate, vstate,
+		    DTRACE_HELPTRACE_DONE);
+
+	/*
+	 * Restore the arg0 that we saved upon entry.
+	 */
+	mstate->dtms_arg[0] = sarg0;
+	mstate->dtms_arg[1] = sarg1;
+
+	return (rval);
+
+err:
+	if (trace)
+		dtrace_helper_trace(helper, mstate, vstate,
+		    DTRACE_HELPTRACE_ERR);
+
+	/*
+	 * Restore the arg0 that we saved upon entry.
+	 */
+	mstate->dtms_arg[0] = sarg0;
+	mstate->dtms_arg[1] = sarg1;
+
+	return (NULL);
+}
+
+static void
+dtrace_helper_action_destroy(dtrace_helper_action_t *helper,
+    dtrace_vstate_t *vstate)
+{
+	int i;
+
+	if (helper->dtha_predicate != NULL)
+		dtrace_difo_release(helper->dtha_predicate, vstate);
+
+	for (i = 0; i < helper->dtha_nactions; i++) {
+		ASSERT(helper->dtha_actions[i] != NULL);
+		dtrace_difo_release(helper->dtha_actions[i], vstate);
+	}
+
+	kmem_free(helper->dtha_actions,
+	    helper->dtha_nactions * sizeof (dtrace_difo_t *));
+	kmem_free(helper, sizeof (dtrace_helper_action_t));
+}
+
+#if !defined(__APPLE__)
+static int
+dtrace_helper_destroygen(int gen)
+{
+	proc_t *p = curproc;
+#else
+static int
+dtrace_helper_destroygen(proc_t* p, int gen)
+{
+#endif
+	dtrace_helpers_t *help = p->p_dtrace_helpers;
+	dtrace_vstate_t *vstate;
+	int i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+	if (help == NULL || gen > help->dthps_generation)
+		return (EINVAL);
+
+	vstate = &help->dthps_vstate;
+
+	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
+		dtrace_helper_action_t *last = NULL, *h, *next;
+
+		for (h = help->dthps_actions[i]; h != NULL; h = next) {
+			next = h->dtha_next;
+
+			if (h->dtha_generation == gen) {
+				if (last != NULL) {
+					last->dtha_next = next;
+				} else {
+					help->dthps_actions[i] = next;
+				}
+
+				dtrace_helper_action_destroy(h, vstate);
+			} else {
+				last = h;
+			}
+		}
+	}
+
+	/*
+	 * Interate until we've cleared out all helper providers with the
+	 * given generation number.
+	 */
+	for (;;) {
+		dtrace_helper_provider_t *prov;
+
+		/*
+		 * Look for a helper provider with the right generation. We
+		 * have to start back at the beginning of the list each time
+		 * because we drop dtrace_lock. It's unlikely that we'll make
+		 * more than two passes.
+		 */
+		for (i = 0; i < help->dthps_nprovs; i++) {
+			prov = help->dthps_provs[i];
+
+			if (prov->dthp_generation == gen)
+				break;
+		}
+
+		/*
+		 * If there were no matches, we're done.
+		 */
+		if (i == help->dthps_nprovs)
+			break;
+
+		/*
+		 * Move the last helper provider into this slot.
+		 */
+		help->dthps_nprovs--;
+		help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs];
+		help->dthps_provs[help->dthps_nprovs] = NULL;
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		/*
+		 * If we have a meta provider, remove this helper provider.
+		 */
+		lck_mtx_lock(&dtrace_meta_lock);
+		if (dtrace_meta_pid != NULL) {
+			ASSERT(dtrace_deferred_pid == NULL);
+			dtrace_helper_provider_remove(&prov->dthp_prov,
+			    p->p_pid);
+		}
+		lck_mtx_unlock(&dtrace_meta_lock);
+
+		dtrace_helper_provider_destroy(prov);
+
+		lck_mtx_lock(&dtrace_lock);
+	}
+
+	return (0);
+}
+
+static int
+dtrace_helper_validate(dtrace_helper_action_t *helper)
+{
+	int err = 0, i;
+	dtrace_difo_t *dp;
+
+	if ((dp = helper->dtha_predicate) != NULL)
+		err += dtrace_difo_validate_helper(dp);
+
+	for (i = 0; i < helper->dtha_nactions; i++)
+		err += dtrace_difo_validate_helper(helper->dtha_actions[i]);
+
+	return (err == 0);
+}
+
+#if !defined(__APPLE__)
+static int
+dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
+#else
+static int
+dtrace_helper_action_add(proc_t* p, int which, dtrace_ecbdesc_t *ep)
+#endif
+{
+	dtrace_helpers_t *help;
+	dtrace_helper_action_t *helper, *last;
+	dtrace_actdesc_t *act;
+	dtrace_vstate_t *vstate;
+	dtrace_predicate_t *pred;
+	int count = 0, nactions = 0, i;
+
+	if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
+		return (EINVAL);
+
+#if !defined(__APPLE__)
+	help = curproc->p_dtrace_helpers;
+#else
+	help = p->p_dtrace_helpers;
+#endif
+	last = help->dthps_actions[which];
+	vstate = &help->dthps_vstate;
+
+	for (count = 0; last != NULL; last = last->dtha_next) {
+		count++;
+		if (last->dtha_next == NULL)
+			break;
+	}
+
+	/*
+	 * If we already have dtrace_helper_actions_max helper actions for this
+	 * helper action type, we'll refuse to add a new one.
+	 */
+	if (count >= dtrace_helper_actions_max)
+		return (ENOSPC);
+
+	helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
+	helper->dtha_generation = help->dthps_generation;
+
+	if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
+		ASSERT(pred->dtp_difo != NULL);
+		dtrace_difo_hold(pred->dtp_difo);
+		helper->dtha_predicate = pred->dtp_difo;
+	}
+
+	for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
+		if (act->dtad_kind != DTRACEACT_DIFEXPR)
+			goto err;
+
+		if (act->dtad_difo == NULL)
+			goto err;
+
+		nactions++;
+	}
+
+	helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
+	    (helper->dtha_nactions = nactions), KM_SLEEP);
+
+	for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
+		dtrace_difo_hold(act->dtad_difo);
+		helper->dtha_actions[i++] = act->dtad_difo;
+	}
+
+	if (!dtrace_helper_validate(helper))
+		goto err;
+
+	if (last == NULL) {
+		help->dthps_actions[which] = helper;
+	} else {
+		last->dtha_next = helper;
+	}
+
+	if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
+		dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
+		dtrace_helptrace_next = 0;
+	}
+
+	return (0);
+err:
+	dtrace_helper_action_destroy(helper, vstate);
+	return (EINVAL);
+}
+
+static void
+dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
+    dof_helper_t *dofhp)
+{
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED);
+
+	lck_mtx_lock(&dtrace_meta_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (!dtrace_attached() || dtrace_meta_pid == NULL) {
+		/*
+		 * If the dtrace module is loaded but not attached, or if
+		 * there aren't isn't a meta provider registered to deal with
+		 * these provider descriptions, we need to postpone creating
+		 * the actual providers until later.
+		 */
+
+		if (help->dthps_next == NULL && help->dthps_prev == NULL &&
+		    dtrace_deferred_pid != help) {
+			help->dthps_deferred = 1;
+			help->dthps_pid = p->p_pid;
+			help->dthps_next = dtrace_deferred_pid;
+			help->dthps_prev = NULL;
+			if (dtrace_deferred_pid != NULL)
+				dtrace_deferred_pid->dthps_prev = help;
+			dtrace_deferred_pid = help;
+		}
+
+		lck_mtx_unlock(&dtrace_lock);
+
+	} else if (dofhp != NULL) {
+		/*
+		 * If the dtrace module is loaded and we have a particular
+		 * helper provider description, pass that off to the
+		 * meta provider.
+		 */
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		dtrace_helper_provide(dofhp, p->p_pid);
+
+	} else {
+		/*
+		 * Otherwise, just pass all the helper provider descriptions
+		 * off to the meta provider.
+		 */
+
+		int i;
+		lck_mtx_unlock(&dtrace_lock);
+
+		for (i = 0; i < help->dthps_nprovs; i++) {
+			dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
+			    p->p_pid);
+		}
+	}
+
+	lck_mtx_unlock(&dtrace_meta_lock);
+}
+
+#if !defined(__APPLE__)
+static int
+dtrace_helper_provider_add(dof_helper_t *dofhp, int gen)
+#else
+static int
+dtrace_helper_provider_add(proc_t* p, dof_helper_t *dofhp, int gen)
+#endif
+{
+	dtrace_helpers_t *help;
+	dtrace_helper_provider_t *hprov, **tmp_provs;
+	uint_t tmp_maxprovs, i;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+#if !defined(__APPLE__)
+	help = curproc->p_dtrace_helpers;
+#else
+	help = p->p_dtrace_helpers;
+#endif
+	ASSERT(help != NULL);
+
+	/*
+	 * If we already have dtrace_helper_providers_max helper providers,
+	 * we're refuse to add a new one.
+	 */
+	if (help->dthps_nprovs >= dtrace_helper_providers_max)
+		return (ENOSPC);
+
+	/*
+	 * Check to make sure this isn't a duplicate.
+	 */
+	for (i = 0; i < help->dthps_nprovs; i++) {
+		if (dofhp->dofhp_addr ==
+		    help->dthps_provs[i]->dthp_prov.dofhp_addr)
+			return (EALREADY);
+	}
+
+	hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
+	hprov->dthp_prov = *dofhp;
+	hprov->dthp_ref = 1;
+	hprov->dthp_generation = gen;
+
+	/*
+	 * Allocate a bigger table for helper providers if it's already full.
+	 */
+	if (help->dthps_maxprovs == help->dthps_nprovs) {
+		tmp_maxprovs = help->dthps_maxprovs;
+		tmp_provs = help->dthps_provs;
+
+		if (help->dthps_maxprovs == 0)
+			help->dthps_maxprovs = 2;
+		else
+			help->dthps_maxprovs *= 2;
+		if (help->dthps_maxprovs > dtrace_helper_providers_max)
+			help->dthps_maxprovs = dtrace_helper_providers_max;
+
+		ASSERT(tmp_maxprovs < help->dthps_maxprovs);
+
+		help->dthps_provs = kmem_zalloc(help->dthps_maxprovs *
+		    sizeof (dtrace_helper_provider_t *), KM_SLEEP);
+
+		if (tmp_provs != NULL) {
+			bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs *
+			    sizeof (dtrace_helper_provider_t *));
+			kmem_free(tmp_provs, tmp_maxprovs *
+			    sizeof (dtrace_helper_provider_t *));
+		}
+	}
+
+	help->dthps_provs[help->dthps_nprovs] = hprov;
+	help->dthps_nprovs++;
+
+	return (0);
+}
+
+static void
+dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov)
+{
+	lck_mtx_lock(&dtrace_lock);
+
+	if (--hprov->dthp_ref == 0) {
+		dof_hdr_t *dof;
+		lck_mtx_unlock(&dtrace_lock);
+		dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
+		dtrace_dof_destroy(dof);
+		kmem_free(hprov, sizeof (dtrace_helper_provider_t));
+	} else {
+		lck_mtx_unlock(&dtrace_lock);
+	}
+}
+
+static int
+dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
+{
+	uintptr_t daddr = (uintptr_t)dof;
+	dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec;
+	dof_provider_t *provider;
+	dof_probe_t *probe;
+	uint8_t *arg;
+	char *strtab, *typestr;
+	dof_stridx_t typeidx;
+	size_t typesz;
+	uint_t nprobes, j, k;
+
+	ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
+
+	if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
+		dtrace_dof_error(dof, "misaligned section offset");
+		return (-1);
+	}
+
+	/*
+	 * The section needs to be large enough to contain the DOF provider
+	 * structure appropriate for the given version.
+	 */
+	if (sec->dofs_size <
+	    ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ?
+	    offsetof(dof_provider_t, dofpv_prenoffs) :
+	    sizeof (dof_provider_t))) {
+		dtrace_dof_error(dof, "provider section too small");
+		return (-1);
+	}
+
+	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
+	str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
+	prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
+	arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
+	off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
+
+	if (str_sec == NULL || prb_sec == NULL ||
+	    arg_sec == NULL || off_sec == NULL)
+		return (-1);
+
+	enoff_sec = NULL;
+
+	if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 &&
+	    provider->dofpv_prenoffs != DOF_SECT_NONE &&
+	    (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS,
+	    provider->dofpv_prenoffs)) == NULL)
+		return (-1);
+
+	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
+
+	if (provider->dofpv_name >= str_sec->dofs_size ||
+	    strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
+		dtrace_dof_error(dof, "invalid provider name");
+		return (-1);
+	}
+
+	if (prb_sec->dofs_entsize == 0 ||
+	    prb_sec->dofs_entsize > prb_sec->dofs_size) {
+		dtrace_dof_error(dof, "invalid entry size");
+		return (-1);
+	}
+
+	if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
+		dtrace_dof_error(dof, "misaligned entry size");
+		return (-1);
+	}
+
+	if (off_sec->dofs_entsize != sizeof (uint32_t)) {
+		dtrace_dof_error(dof, "invalid entry size");
+		return (-1);
+	}
+
+	if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
+		dtrace_dof_error(dof, "misaligned section offset");
+		return (-1);
+	}
+
+	if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
+		dtrace_dof_error(dof, "invalid entry size");
+		return (-1);
+	}
+
+	arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
+
+	nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
+
+	/*
+	 * Take a pass through the probes to check for errors.
+	 */
+	for (j = 0; j < nprobes; j++) {
+		probe = (dof_probe_t *)(uintptr_t)(daddr +
+		    prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
+
+		if (probe->dofpr_func >= str_sec->dofs_size) {
+			dtrace_dof_error(dof, "invalid function name");
+			return (-1);
+		}
+
+		if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
+			dtrace_dof_error(dof, "function name too long");
+			return (-1);
+		}
+
+		if (probe->dofpr_name >= str_sec->dofs_size ||
+		    strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
+			dtrace_dof_error(dof, "invalid probe name");
+			return (-1);
+		}
+
+		/*
+		 * The offset count must not wrap the index, and the offsets
+		 * must also not overflow the section's data.
+		 */
+		if (probe->dofpr_offidx + probe->dofpr_noffs <
+		    probe->dofpr_offidx ||
+		    (probe->dofpr_offidx + probe->dofpr_noffs) *
+		    off_sec->dofs_entsize > off_sec->dofs_size) {
+			dtrace_dof_error(dof, "invalid probe offset");
+			return (-1);
+		}
+
+		if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
+			/*
+			 * If there's no is-enabled offset section, make sure
+			 * there aren't any is-enabled offsets. Otherwise
+			 * perform the same checks as for probe offsets
+			 * (immediately above).
+			 */
+			if (enoff_sec == NULL) {
+				if (probe->dofpr_enoffidx != 0 ||
+				    probe->dofpr_nenoffs != 0) {
+					dtrace_dof_error(dof, "is-enabled "
+					    "offsets with null section");
+					return (-1);
+				}
+			} else if (probe->dofpr_enoffidx +
+			    probe->dofpr_nenoffs < probe->dofpr_enoffidx ||
+			    (probe->dofpr_enoffidx + probe->dofpr_nenoffs) *
+			    enoff_sec->dofs_entsize > enoff_sec->dofs_size) {
+				dtrace_dof_error(dof, "invalid is-enabled "
+				    "offset");
+				return (-1);
+			}
+
+			if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) {
+				dtrace_dof_error(dof, "zero probe and "
+				    "is-enabled offsets");
+				return (-1);
+			}
+		} else if (probe->dofpr_noffs == 0) {
+			dtrace_dof_error(dof, "zero probe offsets");
+			return (-1);
+		}
+
+		if (probe->dofpr_argidx + probe->dofpr_xargc <
+		    probe->dofpr_argidx ||
+		    (probe->dofpr_argidx + probe->dofpr_xargc) *
+		    arg_sec->dofs_entsize > arg_sec->dofs_size) {
+			dtrace_dof_error(dof, "invalid args");
+			return (-1);
+		}
+
+		typeidx = probe->dofpr_nargv;
+		typestr = strtab + probe->dofpr_nargv;
+		for (k = 0; k < probe->dofpr_nargc; k++) {
+			if (typeidx >= str_sec->dofs_size) {
+				dtrace_dof_error(dof, "bad "
+				    "native argument type");
+				return (-1);
+			}
+
+			typesz = strlen(typestr) + 1;
+			if (typesz > DTRACE_ARGTYPELEN) {
+				dtrace_dof_error(dof, "native "
+				    "argument type too long");
+				return (-1);
+			}
+			typeidx += typesz;
+			typestr += typesz;
+		}
+
+		typeidx = probe->dofpr_xargv;
+		typestr = strtab + probe->dofpr_xargv;
+		for (k = 0; k < probe->dofpr_xargc; k++) {
+			if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
+				dtrace_dof_error(dof, "bad "
+				    "native argument index");
+				return (-1);
+			}
+
+			if (typeidx >= str_sec->dofs_size) {
+				dtrace_dof_error(dof, "bad "
+				    "translated argument type");
+				return (-1);
+			}
+
+			typesz = strlen(typestr) + 1;
+			if (typesz > DTRACE_ARGTYPELEN) {
+				dtrace_dof_error(dof, "translated argument "
+				    "type too long");
+				return (-1);
+			}
+
+			typeidx += typesz;
+			typestr += typesz;
+		}
+	}
+
+	return (0);
+}
+
+#if !defined(__APPLE__)
+static int
+dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
+#else
+static int
+dtrace_helper_slurp(proc_t* p, dof_hdr_t *dof, dof_helper_t *dhp)
+#endif
+{
+	dtrace_helpers_t *help;
+	dtrace_vstate_t *vstate;
+	dtrace_enabling_t *enab = NULL;
+	int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1;
+	uintptr_t daddr = (uintptr_t)dof;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+
+#if !defined(__APPLE__)
+	if ((help = curproc->p_dtrace_helpers) == NULL)
+		help = dtrace_helpers_create(curproc);
+#else
+	if ((help = p->p_dtrace_helpers) == NULL)
+		help = dtrace_helpers_create(p);
+#endif
+
+	vstate = &help->dthps_vstate;
+
+	if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
+	    dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
+		dtrace_dof_destroy(dof);
+		return (rv);
+	}
+
+	/*
+	 * Look for helper providers and validate their descriptions.
+	 */
+	if (dhp != NULL) {
+		for (i = 0; i < dof->dofh_secnum; i++) {
+			dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
+			    dof->dofh_secoff + i * dof->dofh_secsize);
+
+			if (sec->dofs_type != DOF_SECT_PROVIDER)
+				continue;
+
+			if (dtrace_helper_provider_validate(dof, sec) != 0) {
+				dtrace_enabling_destroy(enab);
+				dtrace_dof_destroy(dof);
+				return (-1);
+			}
+
+			nprovs++;
+		}
+	}
+
+	/*
+	 * Now we need to walk through the ECB descriptions in the enabling.
+	 */
+	for (i = 0; i < enab->dten_ndesc; i++) {
+		dtrace_ecbdesc_t *ep = enab->dten_desc[i];
+		dtrace_probedesc_t *desc = &ep->dted_probe;
+
+		if (strcmp(desc->dtpd_provider, "dtrace") != 0)
+			continue;
+
+		if (strcmp(desc->dtpd_mod, "helper") != 0)
+			continue;
+
+		if (strcmp(desc->dtpd_func, "ustack") != 0)
+			continue;
+
+#if !defined(__APPLE__)
+		if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK, ep)) != 0)
+#else
+		if ((rv = dtrace_helper_action_add(p, DTRACE_HELPER_ACTION_USTACK, ep)) != 0)
+#endif
+		{
+			/*
+			 * Adding this helper action failed -- we are now going
+			 * to rip out the entire generation and return failure.
+			 */
+#if !defined(__APPLE__)
+			(void) dtrace_helper_destroygen(help->dthps_generation);
+#else
+			(void) dtrace_helper_destroygen(p, help->dthps_generation);
+#endif
+			dtrace_enabling_destroy(enab);
+			dtrace_dof_destroy(dof);
+			return (-1);
+		}
+
+		nhelpers++;
+	}
+
+	if (nhelpers < enab->dten_ndesc)
+		dtrace_dof_error(dof, "unmatched helpers");
+
+	gen = help->dthps_generation++;
+	dtrace_enabling_destroy(enab);
+
+	if (dhp != NULL && nprovs > 0) {
+		dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
+#if !defined(__APPLE__)
+		if (dtrace_helper_provider_add(dhp, gen) == 0) {
+#else
+		if (dtrace_helper_provider_add(p, dhp, gen) == 0) {
+#endif
+			lck_mtx_unlock(&dtrace_lock);
+#if !defined(__APPLE__)
+			dtrace_helper_provider_register(curproc, help, dhp);
+#else
+			dtrace_helper_provider_register(p, help, dhp);
+#endif
+			lck_mtx_lock(&dtrace_lock);
+
+			destroy = 0;
+		}
+	}
+
+	if (destroy)
+		dtrace_dof_destroy(dof);
+
+	return (gen);
+}
+
+#if defined(__APPLE__)
+
+/*
+ * DTrace lazy dof
+ *
+ * DTrace user static probes (USDT probes) and helper actions are loaded
+ * in a process by proccessing dof sections. The dof sections are passed
+ * into the kernel by dyld, in a dof_ioctl_data_t block. It is rather
+ * expensive to process dof for a process that will never use it. There
+ * is a memory cost (allocating the providers/probes), and a cpu cost
+ * (creating the providers/probes).
+ *
+ * To reduce this cost, we use "lazy dof". The normal proceedure for
+ * dof processing is to copyin the dof(s) pointed to by the dof_ioctl_data_t
+ * block, and invoke dof_slurp_helper() on them. When "lazy dof" is
+ * used, each process retains the dof_ioctl_data_t block, instead of
+ * copying in the data it points to.
+ *
+ * The dof_ioctl_data_t blocks are managed as if they were the actual
+ * processed dof; on fork the block is copied to the child, on exec and
+ * exit the block is freed.
+ *
+ * If the process loads library(s) containing additional dof, the
+ * new dof_ioctl_data_t is merged with the existing block.
+ *
+ * There are a few catches that make this slightly more difficult.
+ * When dyld registers dof_ioctl_data_t blocks, it expects a unique
+ * identifier value for each dof in the block. In non-lazy dof terms,
+ * this is the generation that dof was loaded in. If we hand back
+ * a UID for a lazy dof, that same UID must be able to unload the
+ * dof once it has become non-lazy. To meet this requirement, the
+ * code that loads lazy dof requires that the UID's for dof(s) in
+ * the lazy dof be sorted, and in ascending order. It is okay to skip
+ * UID's, I.E., 1 -> 5 -> 6 is legal.
+ *
+ * Once a process has become non-lazy, it will stay non-lazy. All
+ * future dof operations for that process will be non-lazy, even
+ * if the dof mode transitions back to lazy.
+ *
+ * Always do lazy dof checks before non-lazy (I.E. In fork, exit, exec.).
+ * That way if the lazy check fails due to transitioning to non-lazy, the
+ * right thing is done with the newly faulted in dof.
+ */
+
+/*
+ * This method is a bit squicky. It must handle:
+ *
+ * dof should not be lazy.
+ * dof should have been handled lazily, but there was an error
+ * dof was handled lazily, and needs to be freed.
+ * dof was handled lazily, and must not be freed.
+ *
+ * 
+ * Returns EACCESS if dof should be handled non-lazily.
+ * 
+ * KERN_SUCCESS and all other return codes indicate lazy handling of dof.
+ * 
+ * If the dofs data is claimed by this method, dofs_claimed will be set.
+ * Callers should not free claimed dofs.
+ */
+int
+dtrace_lazy_dofs_add(proc_t *p, dof_ioctl_data_t* incoming_dofs, int *dofs_claimed)
+{
+	ASSERT(p);
+	ASSERT(incoming_dofs && incoming_dofs->dofiod_count > 0);
+
+	int rval = 0;
+	*dofs_claimed = 0;
+
+	lck_rw_lock_shared(&dtrace_dof_mode_lock);
+
+	/*
+	 * If we have lazy dof, dof mode better be LAZY_ON.
+	 */
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON);
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || p->p_dtrace_helpers == NULL);
+	ASSERT(dtrace_dof_mode != DTRACE_DOF_MODE_NEVER);
+
+	/*
+	 * Any existing helpers force non-lazy behavior.
+	 */
+	if (dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON && (p->p_dtrace_helpers == NULL)) {
+		lck_mtx_lock(&p->p_dtrace_sprlock);
+
+		dof_ioctl_data_t* existing_dofs = p->p_dtrace_lazy_dofs;
+		unsigned int existing_dofs_count = (existing_dofs) ? existing_dofs->dofiod_count : 0;
+		unsigned int i, merged_dofs_count = incoming_dofs->dofiod_count + existing_dofs_count;
+
+		/*
+		 * Range check...
+		 */
+		if (merged_dofs_count == 0 || merged_dofs_count > 1024) {
+			dtrace_dof_error(NULL, "lazy_dofs_add merged_dofs_count out of range");
+			rval = EINVAL;
+			goto unlock;
+		}
+		
+		/*
+		 * Each dof being added must be assigned a unique generation.
+		 */
+		uint64_t generation = (existing_dofs) ? existing_dofs->dofiod_helpers[existing_dofs_count - 1].dofhp_dof + 1 : 1;
+		for (i=0; i<incoming_dofs->dofiod_count; i++) {
+			/*
+			 * We rely on these being the same so we can overwrite dofhp_dof and not lose info.
+			 */
+			ASSERT(incoming_dofs->dofiod_helpers[i].dofhp_dof == incoming_dofs->dofiod_helpers[i].dofhp_addr);
+			incoming_dofs->dofiod_helpers[i].dofhp_dof = generation++;
+		}
+
+		
+		if (existing_dofs) {
+			/*
+			 * Merge the existing and incoming dofs
+			 */
+			size_t merged_dofs_size = DOF_IOCTL_DATA_T_SIZE(merged_dofs_count);
+			dof_ioctl_data_t* merged_dofs = kmem_alloc(merged_dofs_size, KM_SLEEP);
+
+			bcopy(&existing_dofs->dofiod_helpers[0],
+			      &merged_dofs->dofiod_helpers[0],
+			      sizeof(dof_helper_t) * existing_dofs_count);
+			bcopy(&incoming_dofs->dofiod_helpers[0],
+			      &merged_dofs->dofiod_helpers[existing_dofs_count],
+			      sizeof(dof_helper_t) * incoming_dofs->dofiod_count);
+
+			merged_dofs->dofiod_count = merged_dofs_count;
+
+			kmem_free(existing_dofs, DOF_IOCTL_DATA_T_SIZE(existing_dofs_count));
+
+			p->p_dtrace_lazy_dofs = merged_dofs;
+		} else {
+			/*
+			 * Claim the incoming dofs
+			 */
+			*dofs_claimed = 1;
+			p->p_dtrace_lazy_dofs = incoming_dofs;
+		}
+
+#if DEBUG
+		dof_ioctl_data_t* all_dofs = p->p_dtrace_lazy_dofs;
+		for (i=0; i<all_dofs->dofiod_count-1; i++) {
+			ASSERT(all_dofs->dofiod_helpers[i].dofhp_dof < all_dofs->dofiod_helpers[i+1].dofhp_dof);
+		}
+#endif DEBUG
+
+unlock:
+		lck_mtx_unlock(&p->p_dtrace_sprlock);
+	} else {
+		rval = EACCES;
+	}
+
+ 	lck_rw_unlock_shared(&dtrace_dof_mode_lock);
+
+	return rval;
+}
+
+/*
+ * Returns:
+ *
+ * EINVAL: lazy dof is enabled, but the requested generation was not found.
+ * EACCES: This removal needs to be handled non-lazily. 
+ */
+int
+dtrace_lazy_dofs_remove(proc_t *p, int generation)
+{
+	int rval = EINVAL;
+
+	lck_rw_lock_shared(&dtrace_dof_mode_lock);
+
+	/*
+	 * If we have lazy dof, dof mode better be LAZY_ON.
+	 */
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON);
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || p->p_dtrace_helpers == NULL);
+	ASSERT(dtrace_dof_mode != DTRACE_DOF_MODE_NEVER);
+
+	/*
+	 * Any existing helpers force non-lazy behavior.
+	 */
+	if (dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON && (p->p_dtrace_helpers == NULL)) {
+		lck_mtx_lock(&p->p_dtrace_sprlock);
+
+		dof_ioctl_data_t* existing_dofs = p->p_dtrace_lazy_dofs;
+		
+		if (existing_dofs) {		
+			int index, existing_dofs_count = existing_dofs->dofiod_count;
+			for (index=0; index<existing_dofs_count; index++) {
+				if ((int)existing_dofs->dofiod_helpers[index].dofhp_dof == generation) {
+					dof_ioctl_data_t* removed_dofs = NULL;
+				
+					/*
+					 * If there is only 1 dof, we'll delete it and swap in NULL.
+					 */
+					if (existing_dofs_count > 1) {
+						int removed_dofs_count = existing_dofs_count - 1;
+						size_t removed_dofs_size = DOF_IOCTL_DATA_T_SIZE(removed_dofs_count);
+					
+						removed_dofs = kmem_alloc(removed_dofs_size, KM_SLEEP);
+						removed_dofs->dofiod_count = removed_dofs_count;
+					
+						/*
+						 * copy the remaining data.
+						 */
+						if (index > 0) {
+							bcopy(&existing_dofs->dofiod_helpers[0],
+							      &removed_dofs->dofiod_helpers[0],
+							      index * sizeof(dof_helper_t));
+						}
+					
+						if (index < existing_dofs_count-1) {
+							bcopy(&existing_dofs->dofiod_helpers[index+1],
+							      &removed_dofs->dofiod_helpers[index],
+							      (existing_dofs_count - index - 1) * sizeof(dof_helper_t));
+						}
+					}
+				
+					kmem_free(existing_dofs, DOF_IOCTL_DATA_T_SIZE(existing_dofs_count));
+				
+					p->p_dtrace_lazy_dofs = removed_dofs;
+
+					rval = KERN_SUCCESS;
+
+					break;
+				}
+			}
+
+#if DEBUG
+			dof_ioctl_data_t* all_dofs = p->p_dtrace_lazy_dofs;
+			if (all_dofs) {
+				unsigned int i;
+				for (i=0; i<all_dofs->dofiod_count-1; i++) {
+					ASSERT(all_dofs->dofiod_helpers[i].dofhp_dof < all_dofs->dofiod_helpers[i+1].dofhp_dof);
+				}
+			}
+#endif
+
+		}
+
+		lck_mtx_unlock(&p->p_dtrace_sprlock);
+	} else {		
+		rval = EACCES;
+	}
+	
+	lck_rw_unlock_shared(&dtrace_dof_mode_lock);
+	
+	return rval;
+}
+
+void
+dtrace_lazy_dofs_destroy(proc_t *p)
+{
+	lck_rw_lock_shared(&dtrace_dof_mode_lock);
+	lck_mtx_lock(&p->p_dtrace_sprlock);
+	
+	/*
+	 * If we have lazy dof, dof mode better be LAZY_ON, or we must be exiting.
+	 * We cannot assert against DTRACE_DOF_MODE_NEVER here, because we are called from
+	 * kern_exit.c and kern_exec.c.
+	 */
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON || p->p_lflag & P_LEXIT);
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || p->p_dtrace_helpers == NULL);
+
+	dof_ioctl_data_t* lazy_dofs = p->p_dtrace_lazy_dofs;
+	p->p_dtrace_lazy_dofs = NULL;
+
+	lck_mtx_unlock(&p->p_dtrace_sprlock);
+	lck_rw_unlock_shared(&dtrace_dof_mode_lock);
+
+	if (lazy_dofs) {
+		kmem_free(lazy_dofs, DOF_IOCTL_DATA_T_SIZE(lazy_dofs->dofiod_count));
+	}
+}
+
+void
+dtrace_lazy_dofs_duplicate(proc_t *parent, proc_t *child)
+{
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_NOTOWNED);
+	lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED);
+	lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED);
+
+	lck_rw_lock_shared(&dtrace_dof_mode_lock);
+	lck_mtx_lock(&parent->p_dtrace_sprlock);
+
+	/*
+	 * If we have lazy dof, dof mode better be LAZY_ON, or we must be exiting.
+	 * We cannot assert against DTRACE_DOF_MODE_NEVER here, because we are called from
+	 * kern_fork.c
+	 */
+	ASSERT(parent->p_dtrace_lazy_dofs == NULL || dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON);
+	ASSERT(parent->p_dtrace_lazy_dofs == NULL || parent->p_dtrace_helpers == NULL);
+	/*
+	 * In theory we should hold the child sprlock, but this is safe...
+	 */
+	ASSERT(child->p_dtrace_lazy_dofs == NULL && child->p_dtrace_helpers == NULL);
+
+	dof_ioctl_data_t* parent_dofs = parent->p_dtrace_lazy_dofs;
+	dof_ioctl_data_t* child_dofs = NULL;
+	if (parent_dofs) {
+		size_t parent_dofs_size = DOF_IOCTL_DATA_T_SIZE(parent_dofs->dofiod_count);
+		child_dofs = kmem_alloc(parent_dofs_size, KM_SLEEP);
+		bcopy(parent_dofs, child_dofs, parent_dofs_size);
+	}
+
+	lck_mtx_unlock(&parent->p_dtrace_sprlock);
+ 
+	if (child_dofs) {
+		lck_mtx_lock(&child->p_dtrace_sprlock);
+		child->p_dtrace_lazy_dofs = child_dofs;
+		lck_mtx_unlock(&child->p_dtrace_sprlock);
+	}
+
+	lck_rw_unlock_shared(&dtrace_dof_mode_lock);
+}
+
+static int
+dtrace_lazy_dofs_proc_iterate_filter(proc_t *p, void* ignored)
+{
+#pragma unused(ignored)
+	/*
+	 * Okay to NULL test without taking the sprlock.
+	 */
+	return p->p_dtrace_lazy_dofs != NULL;
+}
+
+static int
+dtrace_lazy_dofs_proc_iterate_doit(proc_t *p, void* ignored)
+{
+#pragma unused(ignored)
+	/*
+	 * It is possible this process may exit during our attempt to
+	 * fault in the dof. We could fix this by holding locks longer,
+	 * but the errors are benign.
+	 */
+	lck_mtx_lock(&p->p_dtrace_sprlock);
+
+	/*
+	 * In this case only, it is okay to have lazy dof when dof mode is DTRACE_DOF_MODE_LAZY_OFF
+	 */
+	ASSERT(p->p_dtrace_lazy_dofs == NULL || p->p_dtrace_helpers == NULL);
+	ASSERT(dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_OFF);
+
+
+	dof_ioctl_data_t* lazy_dofs = p->p_dtrace_lazy_dofs;
+	p->p_dtrace_lazy_dofs = NULL;
+
+	lck_mtx_unlock(&p->p_dtrace_sprlock);
+
+	/*
+	 * Process each dof_helper_t
+	 */
+	if (lazy_dofs != NULL) {
+		unsigned int i;
+		int rval;
+
+		for (i=0; i<lazy_dofs->dofiod_count; i++) {
+			/*
+			 * When loading lazy dof, we depend on the generations being sorted in ascending order.
+			 */
+			ASSERT(i >= (lazy_dofs->dofiod_count - 1) || lazy_dofs->dofiod_helpers[i].dofhp_dof < lazy_dofs->dofiod_helpers[i+1].dofhp_dof);
+
+			dof_helper_t *dhp = &lazy_dofs->dofiod_helpers[i];
+
+			/*
+			 * We stored the generation in dofhp_dof. Save it, and restore the original value.
+			 */
+			int generation = dhp->dofhp_dof;
+			dhp->dofhp_dof = dhp->dofhp_addr;
+
+			dof_hdr_t *dof = dtrace_dof_copyin_from_proc(p, dhp->dofhp_dof, &rval);
+		
+			if (dof != NULL) {
+				dtrace_helpers_t *help;
+								
+				lck_mtx_lock(&dtrace_lock);
+				
+				/*
+				 * This must be done with the dtrace_lock held
+				 */
+				if ((help = p->p_dtrace_helpers) == NULL)
+					help = dtrace_helpers_create(p);
+				
+				/*
+				 * If the generation value has been bumped, someone snuck in
+				 * when we released the dtrace lock. We have to dump this generation,
+				 * there is no safe way to load it.
+				 */
+				if (help->dthps_generation <= generation) {
+					help->dthps_generation = generation;
+					
+					/*
+					 * dtrace_helper_slurp() takes responsibility for the dof --
+					 * it may free it now or it may save it and free it later.
+					 */
+					if ((rval = dtrace_helper_slurp(p, dof, dhp)) != generation) {
+						dtrace_dof_error(NULL, "returned value did not match expected generation");
+					}
+				}
+				
+				lck_mtx_unlock(&dtrace_lock);
+			}
+		}
+
+		kmem_free(lazy_dofs, DOF_IOCTL_DATA_T_SIZE(lazy_dofs->dofiod_count));
+	}
+
+	return PROC_RETURNED;
+}
+
+#endif /* __APPLE__ */
+
+static dtrace_helpers_t *
+dtrace_helpers_create(proc_t *p)
+{
+	dtrace_helpers_t *help;
+
+	lck_mtx_assert(&dtrace_lock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(p->p_dtrace_helpers == NULL);
+
+	help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
+	help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
+	    DTRACE_NHELPER_ACTIONS, KM_SLEEP);
+
+	p->p_dtrace_helpers = help;
+	dtrace_helpers++;
+
+	return (help);
+}
+
+#if !defined(__APPLE__)
+static void
+dtrace_helpers_destroy(void)
+{
+	proc_t *p = curproc;
+#else
+static void
+dtrace_helpers_destroy(proc_t* p)
+{
+#endif
+	dtrace_helpers_t *help;
+	dtrace_vstate_t *vstate;
+	int i;
+
+	lck_mtx_lock(&dtrace_lock);
+
+	ASSERT(p->p_dtrace_helpers != NULL);
+	ASSERT(dtrace_helpers > 0);
+
+	help = p->p_dtrace_helpers;
+	vstate = &help->dthps_vstate;
+
+	/*
+	 * We're now going to lose the help from this process.
+	 */
+	p->p_dtrace_helpers = NULL;
+	dtrace_sync();
+
+	/*
+	 * Destory the helper actions.
+	 */
+	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
+		dtrace_helper_action_t *h, *next;
+
+		for (h = help->dthps_actions[i]; h != NULL; h = next) {
+			next = h->dtha_next;
+			dtrace_helper_action_destroy(h, vstate);
+			h = next;
+		}
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+
+	/*
+	 * Destroy the helper providers.
+	 */
+	if (help->dthps_maxprovs > 0) {
+		lck_mtx_lock(&dtrace_meta_lock);
+		if (dtrace_meta_pid != NULL) {
+			ASSERT(dtrace_deferred_pid == NULL);
+
+			for (i = 0; i < help->dthps_nprovs; i++) {
+				dtrace_helper_provider_remove(
+				    &help->dthps_provs[i]->dthp_prov, p->p_pid);
+			}
+		} else {
+			lck_mtx_lock(&dtrace_lock);
+			ASSERT(help->dthps_deferred == 0 ||
+			    help->dthps_next != NULL ||
+			    help->dthps_prev != NULL ||
+			    help == dtrace_deferred_pid);
+
+			/*
+			 * Remove the helper from the deferred list.
+			 */
+			if (help->dthps_next != NULL)
+				help->dthps_next->dthps_prev = help->dthps_prev;
+			if (help->dthps_prev != NULL)
+				help->dthps_prev->dthps_next = help->dthps_next;
+			if (dtrace_deferred_pid == help) {
+				dtrace_deferred_pid = help->dthps_next;
+				ASSERT(help->dthps_prev == NULL);
+			}
+
+			lck_mtx_unlock(&dtrace_lock);
+		}
+
+		lck_mtx_unlock(&dtrace_meta_lock);
+
+		for (i = 0; i < help->dthps_nprovs; i++) {
+			dtrace_helper_provider_destroy(help->dthps_provs[i]);
+		}
+
+		kmem_free(help->dthps_provs, help->dthps_maxprovs *
+		    sizeof (dtrace_helper_provider_t *));
+	}
+
+	lck_mtx_lock(&dtrace_lock);
+
+	dtrace_vstate_fini(&help->dthps_vstate);
+	kmem_free(help->dthps_actions,
+	    sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
+	kmem_free(help, sizeof (dtrace_helpers_t));
+
+	--dtrace_helpers;
+	lck_mtx_unlock(&dtrace_lock);
+}
+
+static void
+dtrace_helpers_duplicate(proc_t *from, proc_t *to)
+{
+	dtrace_helpers_t *help, *newhelp;
+	dtrace_helper_action_t *helper, *new, *last;
+	dtrace_difo_t *dp;
+	dtrace_vstate_t *vstate;
+	int i, j, sz, hasprovs = 0;
+
+	lck_mtx_lock(&dtrace_lock);
+	ASSERT(from->p_dtrace_helpers != NULL);
+	ASSERT(dtrace_helpers > 0);
+
+	help = from->p_dtrace_helpers;
+	newhelp = dtrace_helpers_create(to);
+	ASSERT(to->p_dtrace_helpers != NULL);
+
+	newhelp->dthps_generation = help->dthps_generation;
+	vstate = &newhelp->dthps_vstate;
+
+	/*
+	 * Duplicate the helper actions.
+	 */
+	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
+		if ((helper = help->dthps_actions[i]) == NULL)
+			continue;
+
+		for (last = NULL; helper != NULL; helper = helper->dtha_next) {
+			new = kmem_zalloc(sizeof (dtrace_helper_action_t),
+			    KM_SLEEP);
+			new->dtha_generation = helper->dtha_generation;
+
+			if ((dp = helper->dtha_predicate) != NULL) {
+				dp = dtrace_difo_duplicate(dp, vstate);
+				new->dtha_predicate = dp;
+			}
+
+			new->dtha_nactions = helper->dtha_nactions;
+			sz = sizeof (dtrace_difo_t *) * new->dtha_nactions;
+			new->dtha_actions = kmem_alloc(sz, KM_SLEEP);
+
+			for (j = 0; j < new->dtha_nactions; j++) {
+				dtrace_difo_t *dp = helper->dtha_actions[j];
+
+				ASSERT(dp != NULL);
+				dp = dtrace_difo_duplicate(dp, vstate);
+				new->dtha_actions[j] = dp;
+			}
+
+			if (last != NULL) {
+				last->dtha_next = new;
+			} else {
+				newhelp->dthps_actions[i] = new;
+			}
+
+			last = new;
+		}
+	}
+
+	/*
+	 * Duplicate the helper providers and register them with the
+	 * DTrace framework.
+	 */
+	if (help->dthps_nprovs > 0) {
+		newhelp->dthps_nprovs = help->dthps_nprovs;
+		newhelp->dthps_maxprovs = help->dthps_nprovs;
+		newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
+		    sizeof (dtrace_helper_provider_t *), KM_SLEEP);
+		for (i = 0; i < newhelp->dthps_nprovs; i++) {
+			newhelp->dthps_provs[i] = help->dthps_provs[i];
+			newhelp->dthps_provs[i]->dthp_ref++;
+		}
+
+		hasprovs = 1;
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+
+	if (hasprovs)
+		dtrace_helper_provider_register(to, newhelp, NULL);
+}
+
+/*
+ * DTrace Hook Functions
+ */
+static void
+dtrace_module_loaded(struct modctl *ctl)
+{
+	dtrace_provider_t *prv;
+
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&mod_lock);
+
+	// ASSERT(ctl->mod_busy);
+
+	/*
+	 * We're going to call each providers per-module provide operation
+	 * specifying only this module.
+	 */
+	for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
+		prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
+
+	lck_mtx_unlock(&mod_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	/*
+	 * If we have any retained enablings, we need to match against them.
+	 * Enabling probes requires that cpu_lock be held, and we cannot hold
+	 * cpu_lock here -- it is legal for cpu_lock to be held when loading a
+	 * module.  (In particular, this happens when loading scheduling
+	 * classes.)  So if we have any retained enablings, we need to dispatch
+	 * our task queue to do the match for us.
+	 */
+	lck_mtx_lock(&dtrace_lock);
+
+	if (dtrace_retained == NULL) {
+		lck_mtx_unlock(&dtrace_lock);
+		return;
+	}
+
+	(void) taskq_dispatch(dtrace_taskq,
+	    (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
+
+	lck_mtx_unlock(&dtrace_lock);
+
+	/*
+	 * And now, for a little heuristic sleaze:  in general, we want to
+	 * match modules as soon as they load.  However, we cannot guarantee
+	 * this, because it would lead us to the lock ordering violation
+	 * outlined above.  The common case, of course, is that cpu_lock is
+	 * _not_ held -- so we delay here for a clock tick, hoping that that's
+	 * long enough for the task queue to do its work.  If it's not, it's
+	 * not a serious problem -- it just means that the module that we
+	 * just loaded may not be immediately instrumentable.
+	 */
+	delay(1);
+}
+
+static void
+dtrace_module_unloaded(struct modctl *ctl)
+{
+	dtrace_probe_t template, *probe, *first, *next;
+	dtrace_provider_t *prov;
+
+	template.dtpr_mod = ctl->mod_modname;
+
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&mod_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (dtrace_bymod == NULL) {
+		/*
+		 * The DTrace module is loaded (obviously) but not attached;
+		 * we don't have any work to do.
+		 */
+		lck_mtx_unlock(&dtrace_provider_lock);
+		lck_mtx_unlock(&mod_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		return;
+	}
+
+	for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
+	    probe != NULL; probe = probe->dtpr_nextmod) {
+		if (probe->dtpr_ecb != NULL) {
+			lck_mtx_unlock(&dtrace_provider_lock);
+			lck_mtx_unlock(&mod_lock);
+			lck_mtx_unlock(&dtrace_lock);
+
+			/*
+			 * This shouldn't _actually_ be possible -- we're
+			 * unloading a module that has an enabled probe in it.
+			 * (It's normally up to the provider to make sure that
+			 * this can't happen.)  However, because dtps_enable()
+			 * doesn't have a failure mode, there can be an
+			 * enable/unload race.  Upshot:  we don't want to
+			 * assert, but we're not going to disable the
+			 * probe, either.
+			 */
+			if (dtrace_err_verbose) {
+				cmn_err(CE_WARN, "unloaded module '%s' had "
+				    "enabled probes", ctl->mod_modname);
+			}
+
+			return;
+		}
+	}
+
+	probe = first;
+
+	for (first = NULL; probe != NULL; probe = next) {
+		ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
+
+		dtrace_probes[probe->dtpr_id - 1] = NULL;
+
+		next = probe->dtpr_nextmod;
+		dtrace_hash_remove(dtrace_bymod, probe);
+		dtrace_hash_remove(dtrace_byfunc, probe);
+		dtrace_hash_remove(dtrace_byname, probe);
+
+		if (first == NULL) {
+			first = probe;
+			probe->dtpr_nextmod = NULL;
+		} else {
+			probe->dtpr_nextmod = first;
+			first = probe;
+		}
+	}
+
+	/*
+	 * We've removed all of the module's probes from the hash chains and
+	 * from the probe array.  Now issue a dtrace_sync() to be sure that
+	 * everyone has cleared out from any probe array processing.
+	 */
+	dtrace_sync();
+
+	for (probe = first; probe != NULL; probe = first) {
+		first = probe->dtpr_nextmod;
+		prov = probe->dtpr_provider;
+		prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
+		    probe->dtpr_arg);
+		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
+		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
+		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
+		vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
+#if !defined(__APPLE__)
+		kmem_free(probe, sizeof (dtrace_probe_t));
+#else
+		zfree(dtrace_probe_t_zone, probe);
+#endif
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&mod_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+}
+
+void
+dtrace_suspend(void)
+{
+	dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
+}
+
+void
+dtrace_resume(void)
+{
+	dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
+}
+
+static int
+dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
+{
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_lock(&dtrace_lock);
+
+	switch (what) {
+	case CPU_CONFIG: {
+		dtrace_state_t *state;
+		dtrace_optval_t *opt, rs, c;
+
+		/*
+		 * For now, we only allocate a new buffer for anonymous state.
+		 */
+		if ((state = dtrace_anon.dta_state) == NULL)
+			break;
+
+		if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
+			break;
+
+		opt = state->dts_options;
+		c = opt[DTRACEOPT_CPU];
+
+		if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
+			break;
+
+		/*
+		 * Regardless of what the actual policy is, we're going to
+		 * temporarily set our resize policy to be manual.  We're
+		 * also going to temporarily set our CPU option to denote
+		 * the newly configured CPU.
+		 */
+		rs = opt[DTRACEOPT_BUFRESIZE];
+		opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
+		opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
+
+		(void) dtrace_state_buffers(state);
+
+		opt[DTRACEOPT_BUFRESIZE] = rs;
+		opt[DTRACEOPT_CPU] = c;
+
+		break;
+	}
+
+	case CPU_UNCONFIG:
+		/*
+		 * We don't free the buffer in the CPU_UNCONFIG case.  (The
+		 * buffer will be freed when the consumer exits.)
+		 */
+		break;
+
+	default:
+		break;
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	return (0);
+}
+
+static void
+dtrace_cpu_setup_initial(processorid_t cpu)
+{
+	(void) dtrace_cpu_setup(CPU_CONFIG, cpu);
+}
+
+static void
+dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
+{
+	if (dtrace_toxranges >= dtrace_toxranges_max) {
+		int osize, nsize;
+		dtrace_toxrange_t *range;
+
+		osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
+
+		if (osize == 0) {
+			ASSERT(dtrace_toxrange == NULL);
+			ASSERT(dtrace_toxranges_max == 0);
+			dtrace_toxranges_max = 1;
+		} else {
+			dtrace_toxranges_max <<= 1;
+		}
+
+		nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
+		range = kmem_zalloc(nsize, KM_SLEEP);
+
+		if (dtrace_toxrange != NULL) {
+			ASSERT(osize != 0);
+			bcopy(dtrace_toxrange, range, osize);
+			kmem_free(dtrace_toxrange, osize);
+		}
+
+		dtrace_toxrange = range;
+	}
+
+	ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
+	ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
+
+	dtrace_toxrange[dtrace_toxranges].dtt_base = base;
+	dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
+	dtrace_toxranges++;
+}
+
+/*
+ * DTrace Driver Cookbook Functions
+ */
+/*ARGSUSED*/
+static int
+dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	dtrace_provider_id_t id;
+	dtrace_state_t *state = NULL;
+	dtrace_enabling_t *enab;
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (ddi_soft_state_init(&dtrace_softstate,
+	    sizeof (dtrace_state_t), 0) != 0) {
+		cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
+		lck_mtx_unlock(&cpu_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		return (DDI_FAILURE);
+	}
+
+#if !defined(__APPLE__)
+	if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
+	    DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
+	    ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
+	    DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
+		cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
+		ddi_remove_minor_node(devi, NULL);
+		ddi_soft_state_fini(&dtrace_softstate);
+		lck_mtx_unlock(&cpu_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		return (DDI_FAILURE);
+	}
+#endif /* __APPLE__ */
+
+	ddi_report_dev(devi);
+	dtrace_devi = devi;
+
+	dtrace_modload = dtrace_module_loaded;
+	dtrace_modunload = dtrace_module_unloaded;
+	dtrace_cpu_init = dtrace_cpu_setup_initial;
+	dtrace_helpers_cleanup = dtrace_helpers_destroy;
+	dtrace_helpers_fork = dtrace_helpers_duplicate;
+	dtrace_cpustart_init = dtrace_suspend;
+	dtrace_cpustart_fini = dtrace_resume;
+	dtrace_debugger_init = dtrace_suspend;
+	dtrace_debugger_fini = dtrace_resume;
+	dtrace_kreloc_init = dtrace_suspend;
+	dtrace_kreloc_fini = dtrace_resume;
+
+	register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+	dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
+	    NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
+	dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
+	    UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
+	    VM_SLEEP | VMC_IDENTIFIER);
+	dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
+	    1, INT_MAX, 0);
+
+	dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
+	    sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
+	    NULL, NULL, NULL, NULL, NULL, 0);
+
+	lck_mtx_assert(&cpu_lock, LCK_MTX_ASSERT_OWNED);
+
+	dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
+	    offsetof(dtrace_probe_t, dtpr_nextmod),
+	    offsetof(dtrace_probe_t, dtpr_prevmod));
+
+	dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
+	    offsetof(dtrace_probe_t, dtpr_nextfunc),
+	    offsetof(dtrace_probe_t, dtpr_prevfunc));
+
+	dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
+	    offsetof(dtrace_probe_t, dtpr_nextname),
+	    offsetof(dtrace_probe_t, dtpr_prevname));
+
+	if (dtrace_retain_max < 1) {
+		cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
+		    "setting to 1", dtrace_retain_max);
+		dtrace_retain_max = 1;
+	}
+
+	/*
+	 * Now discover our toxic ranges.
+	 */
+	dtrace_toxic_ranges(dtrace_toxrange_add);
+
+	/*
+	 * Before we register ourselves as a provider to our own framework,
+	 * we would like to assert that dtrace_provider is NULL -- but that's
+	 * not true if we were loaded as a dependency of a DTrace provider.
+	 * Once we've registered, we can assert that dtrace_provider is our
+	 * pseudo provider.
+	 */
+	(void) dtrace_register("dtrace", &dtrace_provider_attr,
+	    DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
+
+	ASSERT(dtrace_provider != NULL);
+	ASSERT((dtrace_provider_id_t)dtrace_provider == id);
+
+#if !defined(__APPLE__)
+	dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
+	dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "END", 0, NULL);
+	dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
+#elif defined(__ppc__) || defined(__ppc64__)
+	dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "BEGIN", 2, NULL);
+	dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "END", 1, NULL);
+	dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "ERROR", 4, NULL);
+#elif (defined(__i386__) || defined (__x86_64__))
+	dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "BEGIN", 1, NULL);
+	dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "END", 0, NULL);
+	dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "ERROR", 3, NULL);
+#elif defined(__arm__)
+	dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "BEGIN", 2, NULL);
+	dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "END", 1, NULL);
+	dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
+	    dtrace_provider, NULL, NULL, "ERROR", 4, NULL);
+#else
+#error Unknown Architecture
+#endif /* __APPLE__ */
+
+	dtrace_anon_property();
+	lck_mtx_unlock(&cpu_lock);
+
+	/*
+	 * If DTrace helper tracing is enabled, we need to allocate the
+	 * trace buffer and initialize the values.
+	 */
+	if (dtrace_helptrace_enabled) {
+		ASSERT(dtrace_helptrace_buffer == NULL);
+		dtrace_helptrace_buffer =
+		    kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
+		dtrace_helptrace_next = 0;
+	}
+
+	/*
+	 * If there are already providers, we must ask them to provide their
+	 * probes, and then match any anonymous enabling against them.  Note
+	 * that there should be no other retained enablings at this time:
+	 * the only retained enablings at this time should be the anonymous
+	 * enabling.
+	 */
+	if (dtrace_anon.dta_enabling != NULL) {
+		ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
+
+		dtrace_enabling_provide(NULL);
+		state = dtrace_anon.dta_state;
+
+		/*
+		 * We couldn't hold cpu_lock across the above call to
+		 * dtrace_enabling_provide(), but we must hold it to actually
+		 * enable the probes.  We have to drop all of our locks, pick
+		 * up cpu_lock, and regain our locks before matching the
+		 * retained anonymous enabling.
+		 */
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+
+		lck_mtx_lock(&cpu_lock);
+		lck_mtx_lock(&dtrace_provider_lock);
+		lck_mtx_lock(&dtrace_lock);
+
+		if ((enab = dtrace_anon.dta_enabling) != NULL)
+			(void) dtrace_enabling_match(enab, NULL);
+
+		lck_mtx_unlock(&cpu_lock);
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	if (state != NULL) {
+		/*
+		 * If we created any anonymous state, set it going now.
+		 */
+		(void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
+	}
+
+	return (DDI_SUCCESS);
+}
+
+extern void fasttrap_init(void);
+
+/*ARGSUSED*/
+static int
+dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
+{
+#pragma unused(flag, otyp)
+	dtrace_state_t *state;
+	uint32_t priv;
+	uid_t uid;
+	zoneid_t zoneid;
+
+#if !defined(__APPLE__)
+	if (getminor(*devp) == DTRACEMNRN_HELPER)
+		return (0);
+
+	/*
+	 * If this wasn't an open with the "helper" minor, then it must be
+	 * the "dtrace" minor.
+	 */
+	ASSERT(getminor(*devp) == DTRACEMNRN_DTRACE);
+#else
+	/* Darwin puts Helper on its own major device. */
+#endif /* __APPLE__ */
+
+	/*
+	 * If no DTRACE_PRIV_* bits are set in the credential, then the
+	 * caller lacks sufficient permission to do anything with DTrace.
+	 */
+	dtrace_cred2priv(cred_p, &priv, &uid, &zoneid);
+	if (priv == DTRACE_PRIV_NONE)
+		return (EACCES);
+
+#if defined(__APPLE__)
+	/*
+	 * We delay the initialization of fasttrap as late as possible.
+	 * It certainly can't be later than now!
+	 */
+	fasttrap_init();
+#endif /* __APPLE__ */
+
+	/*
+	 * Ask all providers to provide all their probes.
+	 */
+	lck_mtx_lock(&dtrace_provider_lock);
+	dtrace_probe_provide(NULL, NULL);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_lock);
+	dtrace_opens++;
+	dtrace_membar_producer();
+
+	/*
+	 * If the kernel debugger is active (that is, if the kernel debugger
+	 * modified text in some way), we won't allow the open.
+	 */
+	if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
+		dtrace_opens--;
+		lck_mtx_unlock(&cpu_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		return (EBUSY);
+	}
+
+	state = dtrace_state_create(devp, cred_p);
+	lck_mtx_unlock(&cpu_lock);
+
+	if (state == NULL) {
+		if (--dtrace_opens == 0)
+			(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
+		lck_mtx_unlock(&dtrace_lock);
+		return (EAGAIN);
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+
+#if defined(__APPLE__)
+	lck_rw_lock_exclusive(&dtrace_dof_mode_lock);
+
+	/*
+	 * If we are currently lazy, transition states.
+	 *
+	 * Unlike dtrace_close, we do not need to check the
+	 * value of dtrace_opens, as any positive value (and
+	 * we count as 1) means we transition states.
+	 */
+	if (dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_ON) {
+		dtrace_dof_mode = DTRACE_DOF_MODE_LAZY_OFF;
+		
+		/*
+		 * Iterate all existing processes and load lazy dofs.
+		 */
+		proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS,
+			     dtrace_lazy_dofs_proc_iterate_doit,
+			     NULL,
+			     dtrace_lazy_dofs_proc_iterate_filter,
+			     NULL);
+	}
+
+	lck_rw_unlock_exclusive(&dtrace_dof_mode_lock);
+#endif
+
+	return (0);
+}
+
+/*ARGSUSED*/
+static int
+dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
+{
+	minor_t minor = getminor(dev);
+	dtrace_state_t *state;
+
+#if !defined(__APPLE__)
+	if (minor == DTRACEMNRN_HELPER)
+		return (0);
+#else
+	/* Darwin puts Helper on its own major device. */
+#endif /* __APPLE__ */
+
+	state = ddi_get_soft_state(dtrace_softstate, minor);
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	if (state->dts_anon) {
+		/*
+		 * There is anonymous state. Destroy that first.
+		 */
+		ASSERT(dtrace_anon.dta_state == NULL);
+		dtrace_state_destroy(state->dts_anon);
+	}
+
+	dtrace_state_destroy(state);
+	ASSERT(dtrace_opens > 0);
+	if (--dtrace_opens == 0)
+		(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&cpu_lock);
+
+#if defined(__APPLE__)
+
+	/*
+	 * Lock ordering requires the dof mode lock be taken before
+	 * the dtrace_lock.
+	 */
+	lck_rw_lock_exclusive(&dtrace_dof_mode_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	/*
+	 * If we are currently lazy-off, and this is the last close, transition to
+	 * lazy state.
+	 */
+	if (dtrace_dof_mode == DTRACE_DOF_MODE_LAZY_OFF && dtrace_opens == 0) {
+		dtrace_dof_mode = DTRACE_DOF_MODE_LAZY_ON;
+	}
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_rw_unlock_exclusive(&dtrace_dof_mode_lock);
+#endif
+
+	return (0);
+}
+
+#if defined(__APPLE__)
+/*
+ * Introduce cast to quiet warnings.
+ * XXX: This hides a lot of brokenness.
+ */
+#define copyin(src, dst, len) copyin( (user_addr_t)(src), (dst), (len) )
+#define copyout(src, dst, len) copyout( (src), (user_addr_t)(dst), (len) )
+#endif /* __APPLE__ */
+
+#if defined(__APPLE__)
+/*ARGSUSED*/
+static int
+dtrace_ioctl_helper(int cmd, caddr_t arg, int *rv)
+{
+#pragma unused(rv)
+	/*
+	 * Safe to check this outside the dof mode lock
+	 */
+	if (dtrace_dof_mode == DTRACE_DOF_MODE_NEVER)
+		return KERN_SUCCESS;
+
+	switch (cmd) {
+		case DTRACEHIOC_ADDDOF: {
+			dof_helper_t *dhp = NULL;
+			size_t dof_ioctl_data_size;
+			dof_ioctl_data_t* multi_dof;
+			unsigned int i;
+			int rval = 0;
+			user_addr_t user_address = *(user_addr_t*)arg;
+			uint64_t dof_count;
+			int multi_dof_claimed = 0;
+			proc_t* p = current_proc();
+
+			/*
+			 * Read the number of DOF sections being passed in.
+			 */
+			if (copyin(user_address + offsetof(dof_ioctl_data_t, dofiod_count),
+				   &dof_count,
+				   sizeof(dof_count))) {
+				dtrace_dof_error(NULL, "failed to copyin dofiod_count");
+				return (EFAULT);
+			}
+				   			
+			/*
+			 * Range check the count.
+			 */
+			if (dof_count == 0 || dof_count > 1024) {
+				dtrace_dof_error(NULL, "dofiod_count is not valid");
+				return (EINVAL);
+			}
+			
+			/*
+			 * Allocate a correctly sized structure and copyin the data.
+			 */
+			dof_ioctl_data_size = DOF_IOCTL_DATA_T_SIZE(dof_count);
+			if ((multi_dof = kmem_alloc(dof_ioctl_data_size, KM_SLEEP)) == NULL) 
+				return (ENOMEM);
+			
+			/* NOTE! We can no longer exit this method via return */
+			if (copyin(user_address, multi_dof, dof_ioctl_data_size) != 0) {
+				dtrace_dof_error(NULL, "failed copyin of dof_ioctl_data_t");
+				rval = EFAULT;
+				goto cleanup;
+			}
+			
+			/*
+			 * Check that the count didn't change between the first copyin and the second.
+			 */
+			if (multi_dof->dofiod_count != dof_count) {
+				rval = EINVAL;
+				goto cleanup;
+			}
+
+			/*
+			 * Try to process lazily first.
+			 */
+			rval = dtrace_lazy_dofs_add(p, multi_dof, &multi_dof_claimed);
+
+			/*
+			 * If rval is EACCES, we must be non-lazy.
+			 */
+			if (rval == EACCES) {
+				rval = 0;
+				/*
+				 * Process each dof_helper_t
+				 */
+				i = 0;
+				do {
+					dhp = &multi_dof->dofiod_helpers[i];
+					
+					dof_hdr_t *dof = dtrace_dof_copyin(dhp->dofhp_dof, &rval);
+					
+					if (dof != NULL) {					
+						lck_mtx_lock(&dtrace_lock);
+						
+						/*
+						 * dtrace_helper_slurp() takes responsibility for the dof --
+						 * it may free it now or it may save it and free it later.
+						 */
+						if ((dhp->dofhp_dof = (uint64_t)dtrace_helper_slurp(p, dof, dhp)) == -1ULL) {
+							rval = EINVAL;
+						}
+						
+						lck_mtx_unlock(&dtrace_lock);
+					}
+				} while (++i < multi_dof->dofiod_count && rval == 0);
+			}
+
+			/*
+			 * We need to copyout the multi_dof struct, because it contains
+			 * the generation (unique id) values needed to call DTRACEHIOC_REMOVE
+			 *
+			 * This could certainly be better optimized.
+			 */
+			if (copyout(multi_dof, user_address, dof_ioctl_data_size) != 0) {
+				dtrace_dof_error(NULL, "failed copyout of dof_ioctl_data_t");
+				/* Don't overwrite pre-existing error code */
+				if (rval == 0) rval = EFAULT;
+			}
+			
+		cleanup:
+			/*
+			 * If we had to allocate struct memory, free it.
+			 */
+			if (multi_dof != NULL && !multi_dof_claimed) {
+				kmem_free(multi_dof, dof_ioctl_data_size);
+			}
+			
+			return rval;
+		}
+
+		case DTRACEHIOC_REMOVE: {
+			int generation = *(int*)arg;
+			proc_t* p = current_proc();
+
+			/*
+			 * Try lazy first.
+			 */
+			int rval = dtrace_lazy_dofs_remove(p, generation);
+			
+			/*
+			 * EACCES means non-lazy
+			 */
+			if (rval == EACCES) {
+				lck_mtx_lock(&dtrace_lock);
+				rval = dtrace_helper_destroygen(p, generation);
+				lck_mtx_unlock(&dtrace_lock);
+			}
+
+			return (rval);
+		}
+
+		default:
+			break;
+	}
+
+	return ENOTTY;
+}
+#endif /* __APPLE__ */
+
+/*ARGSUSED*/
+static int
+dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
+{
+	minor_t minor = getminor(dev);
+	dtrace_state_t *state;
+	int rval;
+
+#if !defined(__APPLE__)
+	if (minor == DTRACEMNRN_HELPER)
+		return (dtrace_ioctl_helper(cmd, arg, rv));
+#else
+	/* Darwin puts Helper on its own major device. */
+#endif /* __APPLE__ */
+
+	state = ddi_get_soft_state(dtrace_softstate, minor);
+
+	if (state->dts_anon) {
+		ASSERT(dtrace_anon.dta_state == NULL);
+		state = state->dts_anon;
+	}
+
+	switch (cmd) {
+	case DTRACEIOC_PROVIDER: {
+		dtrace_providerdesc_t pvd;
+		dtrace_provider_t *pvp;
+
+		if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
+			return (EFAULT);
+
+		pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
+		lck_mtx_lock(&dtrace_provider_lock);
+
+		for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
+			if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
+				break;
+		}
+
+		lck_mtx_unlock(&dtrace_provider_lock);
+
+		if (pvp == NULL)
+			return (ESRCH);
+
+		bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
+		bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
+		if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_EPROBE: {
+		dtrace_eprobedesc_t epdesc;
+		dtrace_ecb_t *ecb;
+		dtrace_action_t *act;
+		void *buf;
+		size_t size;
+		uintptr_t dest;
+		int nrecs;
+
+		if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
+			return (EFAULT);
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (EINVAL);
+		}
+
+		if (ecb->dte_probe == NULL) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (EINVAL);
+		}
+
+		epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
+		epdesc.dtepd_uarg = ecb->dte_uarg;
+		epdesc.dtepd_size = ecb->dte_size;
+
+		nrecs = epdesc.dtepd_nrecs;
+		epdesc.dtepd_nrecs = 0;
+		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
+			if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
+				continue;
+
+			epdesc.dtepd_nrecs++;
+		}
+
+		/*
+		 * Now that we have the size, we need to allocate a temporary
+		 * buffer in which to store the complete description.  We need
+		 * the temporary buffer to be able to drop dtrace_lock()
+		 * across the copyout(), below.
+		 */
+		size = sizeof (dtrace_eprobedesc_t) +
+		    (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
+
+		buf = kmem_alloc(size, KM_SLEEP);
+		dest = (uintptr_t)buf;
+
+		bcopy(&epdesc, (void *)dest, sizeof (epdesc));
+		dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
+
+		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
+			if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
+				continue;
+
+			if (nrecs-- == 0)
+				break;
+
+			bcopy(&act->dta_rec, (void *)dest,
+			    sizeof (dtrace_recdesc_t));
+			dest += sizeof (dtrace_recdesc_t);
+		}
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
+			kmem_free(buf, size);
+			return (EFAULT);
+		}
+
+		kmem_free(buf, size);
+		return (0);
+	}
+
+	case DTRACEIOC_AGGDESC: {
+		dtrace_aggdesc_t aggdesc;
+		dtrace_action_t *act;
+		dtrace_aggregation_t *agg;
+		int nrecs;
+		uint32_t offs;
+		dtrace_recdesc_t *lrec;
+		void *buf;
+		size_t size;
+		uintptr_t dest;
+
+		if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
+			return (EFAULT);
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (EINVAL);
+		}
+
+		aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
+
+		nrecs = aggdesc.dtagd_nrecs;
+		aggdesc.dtagd_nrecs = 0;
+
+		offs = agg->dtag_base;
+		lrec = &agg->dtag_action.dta_rec;
+		aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
+
+		for (act = agg->dtag_first; ; act = act->dta_next) {
+			ASSERT(act->dta_intuple ||
+			    DTRACEACT_ISAGG(act->dta_kind));
+
+			/*
+			 * If this action has a record size of zero, it
+			 * denotes an argument to the aggregating action.
+			 * Because the presence of this record doesn't (or
+			 * shouldn't) affect the way the data is interpreted,
+			 * we don't copy it out to save user-level the
+			 * confusion of dealing with a zero-length record.
+			 */
+			if (act->dta_rec.dtrd_size == 0) {
+				ASSERT(agg->dtag_hasarg);
+				continue;
+			}
+
+			aggdesc.dtagd_nrecs++;
+
+			if (act == &agg->dtag_action)
+				break;
+		}
+
+		/*
+		 * Now that we have the size, we need to allocate a temporary
+		 * buffer in which to store the complete description.  We need
+		 * the temporary buffer to be able to drop dtrace_lock()
+		 * across the copyout(), below.
+		 */
+		size = sizeof (dtrace_aggdesc_t) +
+		    (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
+
+		buf = kmem_alloc(size, KM_SLEEP);
+		dest = (uintptr_t)buf;
+
+		bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
+		dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
+
+		for (act = agg->dtag_first; ; act = act->dta_next) {
+			dtrace_recdesc_t rec = act->dta_rec;
+
+			/*
+			 * See the comment in the above loop for why we pass
+			 * over zero-length records.
+			 */
+			if (rec.dtrd_size == 0) {
+				ASSERT(agg->dtag_hasarg);
+				continue;
+			}
+
+			if (nrecs-- == 0)
+				break;
+
+			rec.dtrd_offset -= offs;
+			bcopy(&rec, (void *)dest, sizeof (rec));
+			dest += sizeof (dtrace_recdesc_t);
+
+			if (act == &agg->dtag_action)
+				break;
+		}
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
+			kmem_free(buf, size);
+			return (EFAULT);
+		}
+
+		kmem_free(buf, size);
+		return (0);
+	}
+
+	case DTRACEIOC_ENABLE: {
+		dof_hdr_t *dof;
+		dtrace_enabling_t *enab = NULL;
+		dtrace_vstate_t *vstate;
+		int err = 0;
+
+		*rv = 0;
+
+		/*
+		 * If a NULL argument has been passed, we take this as our
+		 * cue to reevaluate our enablings.
+		 */
+		if (arg == NULL) {
+			lck_mtx_lock(&cpu_lock);
+			lck_mtx_lock(&dtrace_lock);
+			err = dtrace_enabling_matchstate(state, rv);
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&cpu_lock);
+
+			return (err);
+		}
+
+		if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
+			return (rval);
+
+		lck_mtx_lock(&cpu_lock);
+		lck_mtx_lock(&dtrace_lock);
+		vstate = &state->dts_vstate;
+
+		if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&cpu_lock);
+			dtrace_dof_destroy(dof);
+			return (EBUSY);
+		}
+
+		if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&cpu_lock);
+			dtrace_dof_destroy(dof);
+			return (EINVAL);
+		}
+
+		if ((rval = dtrace_dof_options(dof, state)) != 0) {
+			dtrace_enabling_destroy(enab);
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&cpu_lock);
+			dtrace_dof_destroy(dof);
+			return (rval);
+		}
+
+		if ((err = dtrace_enabling_match(enab, rv)) == 0) {
+			err = dtrace_enabling_retain(enab);
+		} else {
+			dtrace_enabling_destroy(enab);
+		}
+
+		lck_mtx_unlock(&cpu_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		dtrace_dof_destroy(dof);
+
+		return (err);
+	}
+
+	case DTRACEIOC_REPLICATE: {
+		dtrace_repldesc_t desc;
+		dtrace_probedesc_t *match = &desc.dtrpd_match;
+		dtrace_probedesc_t *create = &desc.dtrpd_create;
+		int err;
+
+		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
+		match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
+		match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
+		match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
+
+		create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
+		create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
+		create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
+		create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
+
+		lck_mtx_lock(&dtrace_lock);
+		err = dtrace_enabling_replicate(state, match, create);
+		lck_mtx_unlock(&dtrace_lock);
+
+		return (err);
+	}
+
+	case DTRACEIOC_PROBEMATCH:
+	case DTRACEIOC_PROBES: {
+		dtrace_probe_t *probe = NULL;
+		dtrace_probedesc_t desc;
+		dtrace_probekey_t pkey;
+		dtrace_id_t i;
+		int m = 0;
+		uint32_t priv;
+		uid_t uid;
+		zoneid_t zoneid;
+
+		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
+		desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
+		desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
+		desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
+
+		/*
+		 * Before we attempt to match this probe, we want to give
+		 * all providers the opportunity to provide it.
+		 */
+		if (desc.dtpd_id == DTRACE_IDNONE) {
+			lck_mtx_lock(&dtrace_provider_lock);
+			dtrace_probe_provide(&desc, NULL);
+			lck_mtx_unlock(&dtrace_provider_lock);
+			desc.dtpd_id++;
+		}
+
+		if (cmd == DTRACEIOC_PROBEMATCH)  {
+			dtrace_probekey(&desc, &pkey);
+			pkey.dtpk_id = DTRACE_IDNONE;
+		}
+
+		dtrace_cred2priv(cr, &priv, &uid, &zoneid);
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if (cmd == DTRACEIOC_PROBEMATCH) {
+			for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
+				if ((probe = dtrace_probes[i - 1]) != NULL &&
+				    (m = dtrace_match_probe(probe, &pkey,
+				    priv, uid, zoneid)) != 0)
+					break;
+			}
+
+			if (m < 0) {
+				lck_mtx_unlock(&dtrace_lock);
+				return (EINVAL);
+			}
+
+		} else {
+			for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
+				if ((probe = dtrace_probes[i - 1]) != NULL &&
+				    dtrace_match_priv(probe, priv, uid, zoneid))
+					break;
+			}
+		}
+
+		if (probe == NULL) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (ESRCH);
+		}
+
+		dtrace_probe_description(probe, &desc);
+		lck_mtx_unlock(&dtrace_lock);
+
+		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_PROBEARG: {
+		dtrace_argdesc_t desc;
+		dtrace_probe_t *probe;
+		dtrace_provider_t *prov;
+
+		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		if (desc.dtargd_id == DTRACE_IDNONE)
+			return (EINVAL);
+
+		if (desc.dtargd_ndx == DTRACE_ARGNONE)
+			return (EINVAL);
+
+		lck_mtx_lock(&dtrace_provider_lock);
+		lck_mtx_lock(&mod_lock);
+		lck_mtx_lock(&dtrace_lock);
+
+		if (desc.dtargd_id > dtrace_nprobes) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&mod_lock);
+			lck_mtx_unlock(&dtrace_provider_lock);
+			return (EINVAL);
+		}
+
+		if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
+			lck_mtx_unlock(&dtrace_lock);
+			lck_mtx_unlock(&mod_lock);
+			lck_mtx_unlock(&dtrace_provider_lock);
+			return (EINVAL);
+		}
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		prov = probe->dtpr_provider;
+
+		if (prov->dtpv_pops.dtps_getargdesc == NULL) {
+			/*
+			 * There isn't any typed information for this probe.
+			 * Set the argument number to DTRACE_ARGNONE.
+			 */
+			desc.dtargd_ndx = DTRACE_ARGNONE;
+		} else {
+			desc.dtargd_native[0] = '\0';
+			desc.dtargd_xlate[0] = '\0';
+			desc.dtargd_mapping = desc.dtargd_ndx;
+
+			prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
+			    probe->dtpr_id, probe->dtpr_arg, &desc);
+		}
+
+		lck_mtx_unlock(&mod_lock);
+		lck_mtx_unlock(&dtrace_provider_lock);
+
+		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_GO: {
+		processorid_t cpuid;
+		rval = dtrace_state_go(state, &cpuid);
+
+		if (rval != 0)
+			return (rval);
+
+		if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_STOP: {
+		processorid_t cpuid;
+
+		lck_mtx_lock(&dtrace_lock);
+		rval = dtrace_state_stop(state, &cpuid);
+		lck_mtx_unlock(&dtrace_lock);
+
+		if (rval != 0)
+			return (rval);
+
+		if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_DOFGET: {
+		dof_hdr_t hdr, *dof;
+		uint64_t len;
+
+		if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
+			return (EFAULT);
+
+		lck_mtx_lock(&dtrace_lock);
+		dof = dtrace_dof_create(state);
+		lck_mtx_unlock(&dtrace_lock);
+
+		len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
+		rval = copyout(dof, (void *)arg, len);
+		dtrace_dof_destroy(dof);
+
+		return (rval == 0 ? 0 : EFAULT);
+	}
+
+	case DTRACEIOC_AGGSNAP:
+	case DTRACEIOC_BUFSNAP: {
+		dtrace_bufdesc_t desc;
+		caddr_t cached;
+		dtrace_buffer_t *buf;
+
+		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
+			return (EINVAL);
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if (cmd == DTRACEIOC_BUFSNAP) {
+			buf = &state->dts_buffer[desc.dtbd_cpu];
+		} else {
+			buf = &state->dts_aggbuffer[desc.dtbd_cpu];
+		}
+
+		if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
+			size_t sz = buf->dtb_offset;
+
+			if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
+				lck_mtx_unlock(&dtrace_lock);
+				return (EBUSY);
+			}
+
+			/*
+			 * If this buffer has already been consumed, we're
+			 * going to indicate that there's nothing left here
+			 * to consume.
+			 */
+			if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
+				lck_mtx_unlock(&dtrace_lock);
+
+				desc.dtbd_size = 0;
+				desc.dtbd_drops = 0;
+				desc.dtbd_errors = 0;
+				desc.dtbd_oldest = 0;
+				sz = sizeof (desc);
+
+				if (copyout(&desc, (void *)arg, sz) != 0)
+					return (EFAULT);
+
+				return (0);
+			}
+
+			/*
+			 * If this is a ring buffer that has wrapped, we want
+			 * to copy the whole thing out.
+			 */
+			if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
+				dtrace_buffer_polish(buf);
+				sz = buf->dtb_size;
+			}
+
+			if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
+				lck_mtx_unlock(&dtrace_lock);
+				return (EFAULT);
+			}
+
+			desc.dtbd_size = sz;
+			desc.dtbd_drops = buf->dtb_drops;
+			desc.dtbd_errors = buf->dtb_errors;
+			desc.dtbd_oldest = buf->dtb_xamot_offset;
+
+			lck_mtx_unlock(&dtrace_lock);
+
+			if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
+				return (EFAULT);
+
+			buf->dtb_flags |= DTRACEBUF_CONSUMED;
+
+			return (0);
+		}
+
+		if (buf->dtb_tomax == NULL) {
+			ASSERT(buf->dtb_xamot == NULL);
+			lck_mtx_unlock(&dtrace_lock);
+			return (ENOENT);
+		}
+
+		cached = buf->dtb_tomax;
+		ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
+
+		dtrace_xcall(desc.dtbd_cpu,
+		    (dtrace_xcall_t)dtrace_buffer_switch, buf);
+
+		state->dts_errors += buf->dtb_xamot_errors;
+
+		/*
+		 * If the buffers did not actually switch, then the cross call
+		 * did not take place -- presumably because the given CPU is
+		 * not in the ready set.  If this is the case, we'll return
+		 * ENOENT.
+		 */
+		if (buf->dtb_tomax == cached) {
+			ASSERT(buf->dtb_xamot != cached);
+			lck_mtx_unlock(&dtrace_lock);
+			return (ENOENT);
+		}
+
+		ASSERT(cached == buf->dtb_xamot);
+
+		/*
+		 * We have our snapshot; now copy it out.
+		 */
+		if (copyout(buf->dtb_xamot, desc.dtbd_data,
+		    buf->dtb_xamot_offset) != 0) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (EFAULT);
+		}
+
+		desc.dtbd_size = buf->dtb_xamot_offset;
+		desc.dtbd_drops = buf->dtb_xamot_drops;
+		desc.dtbd_errors = buf->dtb_xamot_errors;
+		desc.dtbd_oldest = 0;
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		/*
+		 * Finally, copy out the buffer description.
+		 */
+		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_CONF: {
+		dtrace_conf_t conf;
+
+		bzero(&conf, sizeof (conf));
+		conf.dtc_difversion = DIF_VERSION;
+		conf.dtc_difintregs = DIF_DIR_NREGS;
+		conf.dtc_diftupregs = DIF_DTR_NREGS;
+		conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
+
+		if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_STATUS: {
+		dtrace_status_t stat;
+		dtrace_dstate_t *dstate;
+		int i, j;
+		uint64_t nerrs;
+
+		/*
+		 * See the comment in dtrace_state_deadman() for the reason
+		 * for setting dts_laststatus to INT64_MAX before setting
+		 * it to the correct value.
+		 */
+		state->dts_laststatus = INT64_MAX;
+		dtrace_membar_producer();
+		state->dts_laststatus = dtrace_gethrtime();
+
+		bzero(&stat, sizeof (stat));
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (ENOENT);
+		}
+
+		if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
+			stat.dtst_exiting = 1;
+
+		nerrs = state->dts_errors;
+		dstate = &state->dts_vstate.dtvs_dynvars;
+
+		for (i = 0; i < NCPU; i++) {
+			dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
+
+			stat.dtst_dyndrops += dcpu->dtdsc_drops;
+			stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
+			stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
+
+			if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
+				stat.dtst_filled++;
+
+			nerrs += state->dts_buffer[i].dtb_errors;
+
+			for (j = 0; j < state->dts_nspeculations; j++) {
+				dtrace_speculation_t *spec;
+				dtrace_buffer_t *buf;
+
+				spec = &state->dts_speculations[j];
+				buf = &spec->dtsp_buffer[i];
+				stat.dtst_specdrops += buf->dtb_xamot_drops;
+			}
+		}
+
+		stat.dtst_specdrops_busy = state->dts_speculations_busy;
+		stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
+		stat.dtst_stkstroverflows = state->dts_stkstroverflows;
+		stat.dtst_dblerrors = state->dts_dblerrors;
+		stat.dtst_killed =
+		    (state->dts_activity == DTRACE_ACTIVITY_KILLED);
+		stat.dtst_errors = nerrs;
+
+		lck_mtx_unlock(&dtrace_lock);
+
+		if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	case DTRACEIOC_FORMAT: {
+		dtrace_fmtdesc_t fmt;
+		char *str;
+		int len;
+
+		if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
+			return (EFAULT);
+
+		lck_mtx_lock(&dtrace_lock);
+
+		if (fmt.dtfd_format == 0 ||
+		    fmt.dtfd_format > state->dts_nformats) {
+			lck_mtx_unlock(&dtrace_lock);
+			return (EINVAL);
+		}
+
+		/*
+		 * Format strings are allocated contiguously and they are
+		 * never freed; if a format index is less than the number
+		 * of formats, we can assert that the format map is non-NULL
+		 * and that the format for the specified index is non-NULL.
+		 */
+		ASSERT(state->dts_formats != NULL);
+		str = state->dts_formats[fmt.dtfd_format - 1];
+		ASSERT(str != NULL);
+
+		len = strlen(str) + 1;
+
+		if (len > fmt.dtfd_length) {
+			fmt.dtfd_length = len;
+
+			if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
+				lck_mtx_unlock(&dtrace_lock);
+				return (EINVAL);
+			}
+		} else {
+			if (copyout(str, fmt.dtfd_string, len) != 0) {
+				lck_mtx_unlock(&dtrace_lock);
+				return (EINVAL);
+			}
+		}
+
+		lck_mtx_unlock(&dtrace_lock);
+		return (0);
+	}
+
+	default:
+		break;
+	}
+
+	return (ENOTTY);
+}
+
+#if defined(__APPLE__)
+#undef copyin
+#undef copyout
+#endif /* __APPLE__ */
+
+#if !defined(__APPLE__)
+/*ARGSUSED*/
+static int
+dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
+{
+	dtrace_state_t *state;
+
+	switch (cmd) {
+	case DDI_DETACH:
+		break;
+
+	case DDI_SUSPEND:
+		return (DDI_SUCCESS);
+
+	default:
+		return (DDI_FAILURE);
+	}
+
+	lck_mtx_lock(&cpu_lock);
+	lck_mtx_lock(&dtrace_provider_lock);
+	lck_mtx_lock(&dtrace_lock);
+
+	ASSERT(dtrace_opens == 0);
+
+	if (dtrace_helpers > 0) {
+		lck_mtx_unlock(&dtrace_provider_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&cpu_lock);
+		return (DDI_FAILURE);
+	}
+
+	if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
+		lck_mtx_unlock(&dtrace_provider_lock);
+		lck_mtx_unlock(&dtrace_lock);
+		lck_mtx_unlock(&cpu_lock);
+		return (DDI_FAILURE);
+	}
+
+	dtrace_provider = NULL;
+
+	if ((state = dtrace_anon_grab()) != NULL) {
+		/*
+		 * If there were ECBs on this state, the provider should
+		 * have not been allowed to detach; assert that there is
+		 * none.
+		 */
+		ASSERT(state->dts_necbs == 0);
+		dtrace_state_destroy(state);
+
+		/*
+		 * If we're being detached with anonymous state, we need to
+		 * indicate to the kernel debugger that DTrace is now inactive.
+		 */
+		(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
+	}
+
+	bzero(&dtrace_anon, sizeof (dtrace_anon_t));
+	unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
+	dtrace_cpu_init = NULL;
+	dtrace_helpers_cleanup = NULL;
+	dtrace_helpers_fork = NULL;
+	dtrace_cpustart_init = NULL;
+	dtrace_cpustart_fini = NULL;
+	dtrace_debugger_init = NULL;
+	dtrace_debugger_fini = NULL;
+	dtrace_kreloc_init = NULL;
+	dtrace_kreloc_fini = NULL;
+	dtrace_modload = NULL;
+	dtrace_modunload = NULL;
+
+	lck_mtx_unlock(&cpu_lock);
+
+	if (dtrace_helptrace_enabled) {
+		kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
+		dtrace_helptrace_buffer = NULL;
+	}
+
+	kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
+	dtrace_probes = NULL;
+	dtrace_nprobes = 0;
+
+	dtrace_hash_destroy(dtrace_bymod);
+	dtrace_hash_destroy(dtrace_byfunc);
+	dtrace_hash_destroy(dtrace_byname);
+	dtrace_bymod = NULL;
+	dtrace_byfunc = NULL;
+	dtrace_byname = NULL;
+
+	kmem_cache_destroy(dtrace_state_cache);
+	vmem_destroy(dtrace_minor);
+	vmem_destroy(dtrace_arena);
+
+	if (dtrace_toxrange != NULL) {
+		kmem_free(dtrace_toxrange,
+		    dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
+		dtrace_toxrange = NULL;
+		dtrace_toxranges = 0;
+		dtrace_toxranges_max = 0;
+	}
+
+	ddi_remove_minor_node(dtrace_devi, NULL);
+	dtrace_devi = NULL;
+
+	ddi_soft_state_fini(&dtrace_softstate);
+
+	ASSERT(dtrace_vtime_references == 0);
+	ASSERT(dtrace_opens == 0);
+	ASSERT(dtrace_retained == NULL);
+
+	lck_mtx_unlock(&dtrace_lock);
+	lck_mtx_unlock(&dtrace_provider_lock);
+
+	/*
+	 * We don't destroy the task queue until after we have dropped our
+	 * locks (taskq_destroy() may block on running tasks).  To prevent
+	 * attempting to do work after we have effectively detached but before
+	 * the task queue has been destroyed, all tasks dispatched via the
+	 * task queue must check that DTrace is still attached before
+	 * performing any operation.
+	 */
+	taskq_destroy(dtrace_taskq);
+	dtrace_taskq = NULL;
+
+	return (DDI_SUCCESS);
+}
+
+/*ARGSUSED*/
+static int
+dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
+{
+	int error;
+
+	switch (infocmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)dtrace_devi;
+		error = DDI_SUCCESS;
+		break;
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		error = DDI_SUCCESS;
+		break;
+	default:
+		error = DDI_FAILURE;
+	}
+	return (error);
+}
+
+static struct cb_ops dtrace_cb_ops = {
+	dtrace_open,		/* open */
+	dtrace_close,		/* close */
+	nulldev,		/* strategy */
+	nulldev,		/* print */
+	nodev,			/* dump */
+	nodev,			/* read */
+	nodev,			/* write */
+	dtrace_ioctl,		/* ioctl */
+	nodev,			/* devmap */
+	nodev,			/* mmap */
+	nodev,			/* segmap */
+	nochpoll,		/* poll */
+	ddi_prop_op,		/* cb_prop_op */
+	0,			/* streamtab  */
+	D_NEW | D_MP		/* Driver compatibility flag */
+};
+
+static struct dev_ops dtrace_ops = {
+	DEVO_REV,		/* devo_rev */
+	0,			/* refcnt */
+	dtrace_info,		/* get_dev_info */
+	nulldev,		/* identify */
+	nulldev,		/* probe */
+	dtrace_attach,		/* attach */
+	dtrace_detach,		/* detach */
+	nodev,			/* reset */
+	&dtrace_cb_ops,		/* driver operations */
+	NULL,			/* bus operations */
+	nodev			/* dev power */
+};
+
+static struct modldrv modldrv = {
+	&mod_driverops,		/* module type (this is a pseudo driver) */
+	"Dynamic Tracing",	/* name of module */
+	&dtrace_ops,		/* driver ops */
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	(void *)&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	return (mod_install(&modlinkage));
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+int
+_fini(void)
+{
+	return (mod_remove(&modlinkage));
+}
+#else
+
+d_open_t _dtrace_open, helper_open;
+d_close_t _dtrace_close, helper_close;
+d_ioctl_t _dtrace_ioctl, helper_ioctl;
+
+int 
+_dtrace_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(p)
+	dev_t locdev = dev;
+
+	return  dtrace_open( &locdev, flags, devtype, CRED());
+}
+
+int
+helper_open(dev_t dev, int flags, int devtype, struct proc *p)
+{	
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+int
+_dtrace_close(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(p)
+	return dtrace_close( dev, flags, devtype, CRED());
+}
+
+int
+helper_close(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+int
+_dtrace_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, struct proc *p)
+{
+#pragma unused(p)
+	int err, rv = 0;
+
+	err = dtrace_ioctl(dev, (int)cmd, *(intptr_t *)data, fflag, CRED(), &rv);
+	
+	/* XXX Darwin's BSD ioctls only return -1 or zero. Overload errno to mimic Solaris. 20 bits suffice. */
+	if (err != 0) {
+		ASSERT( (err & 0xfffff000) == 0 );
+		return (err & 0xfff); /* ioctl returns -1 and errno set to an error code < 4096 */
+	} else if (rv != 0) {
+		ASSERT( (rv & 0xfff00000) == 0 );
+		return (((rv & 0xfffff) << 12)); /* ioctl returns -1 and errno set to a return value >= 4096 */
+	} else 
+		return 0;
+}
+
+int
+helper_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, struct proc *p)
+{
+#pragma unused(dev,fflag,p)
+	int err, rv = 0;
+	
+	err = dtrace_ioctl_helper((int)cmd, data, &rv);
+	/* XXX Darwin's BSD ioctls only return -1 or zero. Overload errno to mimic Solaris. 20 bits suffice. */
+	if (err != 0) {
+		ASSERT( (err & 0xfffff000) == 0 );
+		return (err & 0xfff); /* ioctl returns -1 and errno set to an error code < 4096 */
+	} else if (rv != 0) {
+		ASSERT( (rv & 0xfff00000) == 0 );
+		return (((rv & 0xfffff) << 20)); /* ioctl returns -1 and errno set to a return value >= 4096 */
+	} else 
+		return 0;
+}
+
+#define HELPER_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw helper_cdevsw =
+{
+	helper_open,		/* open */
+	helper_close,		/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	helper_ioctl,		/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int helper_majdevno = 0;
+
+static int gDTraceInited = 0;
+
+void
+helper_init( void )
+{
+	/*
+	 * Once the "helper" is initialized, it can take ioctl calls that use locks
+	 * and zones initialized in dtrace_init. Make certain dtrace_init was called
+	 * before us.
+	 */
+
+	if (!gDTraceInited) {
+		panic("helper_init before dtrace_init\n");
+	}
+
+	if (0 >= helper_majdevno)
+	{
+		helper_majdevno = cdevsw_add(HELPER_MAJOR, &helper_cdevsw);
+		
+		if (helper_majdevno < 0) {
+			printf("helper_init: failed to allocate a major number!\n");
+			return;
+		}
+		
+		if (NULL == devfs_make_node( makedev(helper_majdevno, 0), DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, 
+					DTRACEMNR_HELPER, 0 )) {
+			printf("dtrace_init: failed to devfs_make_node for helper!\n");
+			return;
+		}
+	} else
+		panic("helper_init: called twice!\n");
+}
+
+#undef HELPER_MAJOR
+
+/*
+ * Called with DEVFS_LOCK held, so vmem_alloc's underlying blist structures are protected.
+ */
+static int
+dtrace_clone_func(dev_t dev, int action)
+{
+#pragma unused(dev)
+
+	if (action == DEVFS_CLONE_ALLOC) {
+		if (NULL == dtrace_minor) /* Arena not created yet!?! */
+			return 0;
+		else {
+			/*
+			 * Propose a minor number, namely the next number that vmem_alloc() will return.
+			 * Immediately put it back in play by calling vmem_free().
+			 */
+			int ret = (int)(uintptr_t)vmem_alloc(dtrace_minor, 1, VM_BESTFIT | VM_SLEEP);
+
+			vmem_free(dtrace_minor, (void *)(uintptr_t)ret, 1);
+
+			return ret;
+		}
+	}
+	else if (action == DEVFS_CLONE_FREE) {
+		return 0;
+	}
+	else return -1;
+}
+
+#define DTRACE_MAJOR  -24 /* let the kernel pick the device number */
+
+static struct cdevsw dtrace_cdevsw =
+{
+	_dtrace_open,		/* open */
+	_dtrace_close,		/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	_dtrace_ioctl,		/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+lck_attr_t* dtrace_lck_attr;
+lck_grp_attr_t* dtrace_lck_grp_attr;
+lck_grp_t* dtrace_lck_grp;
+
+static int gMajDevNo;
+
+void
+dtrace_init( void )
+{
+	if (0 == gDTraceInited) {
+		int i, ncpu = NCPU;
+
+		gMajDevNo = cdevsw_add(DTRACE_MAJOR, &dtrace_cdevsw);
+
+		if (gMajDevNo < 0) {
+			printf("dtrace_init: failed to allocate a major number!\n");
+			gDTraceInited = 0;
+			return;
+		}
+
+		if (NULL == devfs_make_node_clone( makedev(gMajDevNo, 0), DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, 
+					dtrace_clone_func, DTRACEMNR_DTRACE, 0 )) {
+			printf("dtrace_init: failed to devfs_make_node_clone for dtrace!\n");
+			gDTraceInited = 0;
+			return;
+		}
+
+#if defined(DTRACE_MEMORY_ZONES)
+
+		/*
+		 * Initialize the dtrace kalloc-emulation zones.
+		 */
+		dtrace_alloc_init();
+
+#endif /* DTRACE_MEMORY_ZONES */
+
+		/*
+		 * Allocate the dtrace_probe_t zone
+		 */
+		dtrace_probe_t_zone = zinit(sizeof(dtrace_probe_t),
+					    1024 * sizeof(dtrace_probe_t),
+					    sizeof(dtrace_probe_t),
+					    "dtrace.dtrace_probe_t");
+
+		/*
+		 * Create the dtrace lock group and attrs.
+		 */
+		dtrace_lck_attr = lck_attr_alloc_init();
+		dtrace_lck_grp_attr= lck_grp_attr_alloc_init();		
+		dtrace_lck_grp = lck_grp_alloc_init("dtrace",  dtrace_lck_grp_attr);
+
+		/*
+		 * We have to initialize all locks explicitly
+		 */
+		lck_mtx_init(&dtrace_lock, dtrace_lck_grp, dtrace_lck_attr);
+		lck_mtx_init(&dtrace_provider_lock, dtrace_lck_grp, dtrace_lck_attr);
+		lck_mtx_init(&dtrace_meta_lock, dtrace_lck_grp, dtrace_lck_attr);
+#ifdef DEBUG
+		lck_mtx_init(&dtrace_errlock, dtrace_lck_grp, dtrace_lck_attr);
+#endif
+		lck_rw_init(&dtrace_dof_mode_lock, dtrace_lck_grp, dtrace_lck_attr);
+
+		/*
+		 * The cpu_core structure consists of per-CPU state available in any context.
+		 * On some architectures, this may mean that the page(s) containing the
+		 * NCPU-sized array of cpu_core structures must be locked in the TLB -- it
+		 * is up to the platform to assure that this is performed properly.  Note that
+		 * the structure is sized to avoid false sharing.
+		 */
+		lck_mtx_init(&cpu_lock, dtrace_lck_grp, dtrace_lck_attr);
+		lck_mtx_init(&mod_lock, dtrace_lck_grp, dtrace_lck_attr);
+
+		cpu_core = (cpu_core_t *)kmem_zalloc( ncpu * sizeof(cpu_core_t), KM_SLEEP );
+		for (i = 0; i < ncpu; ++i) {
+			lck_mtx_init(&cpu_core[i].cpuc_pid_lock, dtrace_lck_grp, dtrace_lck_attr);
+		}
+
+		cpu_list = (cpu_t *)kmem_zalloc( ncpu * sizeof(cpu_t), KM_SLEEP );
+		for (i = 0; i < ncpu; ++i) {
+			cpu_list[i].cpu_id = (processorid_t)i;
+			cpu_list[i].cpu_next = &(cpu_list[(i+1) % ncpu]);
+			lck_rw_init(&cpu_list[i].cpu_ft_lock, dtrace_lck_grp, dtrace_lck_attr);
+		}
+
+		lck_mtx_lock(&cpu_lock);
+		for (i = 0; i < ncpu; ++i) 
+			dtrace_cpu_setup_initial( (processorid_t)i ); /* In lieu of register_cpu_setup_func() callback */
+		lck_mtx_unlock(&cpu_lock);
+
+		(void)dtrace_abs_to_nano(0LL); /* Force once only call to clock_timebase_info (which can take a lock) */
+
+		/*
+		 * See dtrace_impl.h for a description of dof modes.
+		 * The default is lazy dof.
+		 *
+		 * XXX Warn if state is LAZY_OFF? It won't break anything, but
+		 * makes no sense...
+		 */
+		if (!PE_parse_boot_arg("dtrace_dof_mode", &dtrace_dof_mode)) {
+			dtrace_dof_mode = DTRACE_DOF_MODE_LAZY_ON;
+		}
+
+		/*
+		 * Sanity check of dof mode value.
+		 */
+		switch (dtrace_dof_mode) {
+			case DTRACE_DOF_MODE_NEVER:
+			case DTRACE_DOF_MODE_LAZY_ON:
+				/* valid modes, but nothing else we need to do */
+				break;
+
+			case DTRACE_DOF_MODE_LAZY_OFF:
+			case DTRACE_DOF_MODE_NON_LAZY:
+				/* Cannot wait for a dtrace_open to init fasttrap */
+				fasttrap_init();
+				break;
+
+			default:
+				/* Invalid, clamp to non lazy */
+				dtrace_dof_mode = DTRACE_DOF_MODE_NON_LAZY;
+				fasttrap_init();
+				break;
+		}
+
+		gDTraceInited = 1;
+
+	} else
+		panic("dtrace_init: called twice!\n");
+}
+
+void
+dtrace_postinit(void)
+{
+		dtrace_attach( (dev_info_t *)makedev(gMajDevNo, 0), 0 );
+}
+#undef DTRACE_MAJOR
+
+/*
+ * Routines used to register interest in cpu's being added to or removed
+ * from the system.
+ */
+void
+register_cpu_setup_func(cpu_setup_func_t *ignore1, void *ignore2)
+{
+#pragma unused(ignore1,ignore2)
+}
+
+void
+unregister_cpu_setup_func(cpu_setup_func_t *ignore1, void *ignore2)
+{
+#pragma unused(ignore1,ignore2)
+}
+#endif /* __APPLE__ */
diff --git a/bsd/dev/dtrace/dtrace_alloc.c b/bsd/dev/dtrace/dtrace_alloc.c
new file mode 100644
index 000000000..df0107bd3
--- /dev/null
+++ b/bsd/dev/dtrace/dtrace_alloc.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2005-2007 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * DTrace kalloc emulation.
+ *
+ * This is a subset of kalloc functionality, to allow dtrace
+ * specific allocation to be accounted for separately from the
+ * general kalloc pool.
+ *
+ * Note that allocations greater than dalloc_max still go into
+ * the kalloc.large bucket, as it seems impossible to emulate
+ * that functionality in the bsd kern.
+ */
+
+#include <stdarg.h>
+#include <string.h>
+#include <sys/malloc.h>
+#include <sys/dtrace.h>
+#include <kern/zalloc.h>
+
+#if defined(DTRACE_MEMORY_ZONES)
+
+#define DTRACE_ALLOC_MINSIZE 16
+
+vm_size_t dtrace_alloc_max;
+vm_size_t dtrace_alloc_max_prerounded;
+int first_d_zone = -1;
+struct zone *d_zone[16];
+static const char *d_zone_name[16] = {
+	"dtrace.1",		"dtrace.2",
+	"dtrace.4",		"dtrace.8",
+	"dtrace.16",		"dtrace.32",
+	"dtrace.64",		"dtrace.128",
+	"dtrace.256",		"dtrace.512",
+	"dtrace.1024",		"dtrace.2048",
+	"dtrace.4096",		"dtrace.8192",
+	"dtrace.16384",		"dtrace.32768"
+};
+
+unsigned long d_zone_max[16] = {
+      1024,		/*      1 Byte  */
+      1024,		/*      2 Byte  */
+      1024,		/*      4 Byte  */
+      1024,		/*      8 Byte  */
+      1024,		/*     16 Byte  */
+      4096,		/*     32 Byte  */
+      4096,		/*     64 Byte  */
+      4096,		/*    128 Byte  */
+      4096,		/*    256 Byte  */
+      1024,		/*    512 Byte  */
+      1024,		/*   1024 Byte  */
+      1024,		/*   2048 Byte  */
+      1024,		/*   4096 Byte  */
+      4096,		/*   8192 Byte  */
+      64,		/*  16384 Byte  */
+      64,		/*  32768 Byte  */
+};
+
+void dtrace_alloc_init(void)
+{
+	vm_size_t size;
+	int i;
+
+	if (PAGE_SIZE < 16*1024)
+		dtrace_alloc_max = 16*1024;
+	else
+		dtrace_alloc_max = PAGE_SIZE;
+	dtrace_alloc_max_prerounded = dtrace_alloc_max / 2 + 1;
+
+	/*
+	 *	Allocate a zone for each size we are going to handle.
+	 *	We specify non-paged memory.
+	 */
+	for (i = 0, size = 1; size < dtrace_alloc_max; i++, size <<= 1) {
+		if (size < DTRACE_ALLOC_MINSIZE) {
+			d_zone[i] = NULL;
+			continue;
+		}
+		if (size == DTRACE_ALLOC_MINSIZE) {
+			first_d_zone = i;
+		}
+		d_zone[i] = zinit(size, d_zone_max[i] * size, size, d_zone_name[i]);
+	}
+}
+
+void *dtrace_alloc(vm_size_t size)
+{
+	int zindex;
+	vm_size_t allocsize;
+
+	/*
+	 * If size is too large for a zone, then use kmem_alloc.
+	 * (We use kmem_alloc instead of kmem_alloc_wired so that
+	 * krealloc can use kmem_realloc.)
+	 */
+
+	if (size >= dtrace_alloc_max_prerounded) {
+		return _MALLOC(size, M_TEMP, M_WAITOK);
+	}
+
+	/* compute the size of the block that we will actually allocate */
+	allocsize = DTRACE_ALLOC_MINSIZE;
+	zindex = first_d_zone;
+	while (allocsize < size) {
+		allocsize <<= 1;
+		zindex++;
+	}
+
+	return(zalloc_canblock(d_zone[zindex], TRUE));
+}
+
+void dtrace_free(void *data, vm_size_t size)
+{
+	int zindex;
+	vm_size_t freesize;
+
+	if (size >= dtrace_alloc_max_prerounded) {
+		_FREE(data, M_TEMP);
+		return;
+	}
+
+	/* compute the size of the block that we actually allocated from */
+	freesize = DTRACE_ALLOC_MINSIZE;
+	zindex = first_d_zone;
+	while (freesize < size) {
+		freesize <<= 1;
+		zindex++;
+	}
+
+	/* free to the appropriate zone */
+	zfree(d_zone[zindex], data);
+}
+
+#endif /* DTRACE_MEMORY_ZONES */
diff --git a/bsd/dev/dtrace/dtrace_glue.c b/bsd/dev/dtrace/dtrace_glue.c
new file mode 100644
index 000000000..035150aa7
--- /dev/null
+++ b/bsd/dev/dtrace/dtrace_glue.c
@@ -0,0 +1,1600 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+
+/*
+ * APPLE NOTE: This file is compiled even if dtrace is unconfig'd. A symbol
+ * from this file (_dtrace_register_anon_DOF) always needs to be exported for
+ * an external kext to link against.
+ */
+
+#if CONFIG_DTRACE
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+
+#include <stdarg.h>
+#include <string.h>
+#include <sys/malloc.h>
+#include <sys/time.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/user.h>
+#include <sys/systm.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
+#include <kern/task.h>
+#include <kern/sched_prim.h>
+#include <kern/queue.h>
+#include <miscfs/devfs/devfs.h>
+#include <kern/kalloc.h>
+
+#include <mach/vm_param.h>
+#include <mach/mach_vm.h>
+#include <mach/task.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h> /* All the bits we care about are guarded by MACH_KERNEL_PRIVATE :-( */
+
+/*
+ * pid/proc
+ */
+#define proc_t struct proc
+
+/* Not called from probe context */
+proc_t * 
+sprlock(pid_t pid)
+{
+	proc_t* p;
+
+	if ((p = proc_find(pid)) == PROC_NULL) {
+		return PROC_NULL;
+	}
+
+	task_suspend(p->task);
+
+	proc_lock(p);
+
+	lck_mtx_lock(&p->p_dtrace_sprlock);
+
+	return p;
+}
+
+/* Not called from probe context */
+void
+sprunlock(proc_t *p)
+{
+	if (p != PROC_NULL) {
+		lck_mtx_unlock(&p->p_dtrace_sprlock);
+
+		proc_unlock(p);
+
+		task_resume(p->task);
+
+		proc_rele(p);
+	}
+}
+
+/*
+ * uread/uwrite
+ */
+
+// These are not exported from vm_map.h.
+extern kern_return_t vm_map_read_user(vm_map_t map, vm_map_address_t src_addr, void *dst_p, vm_size_t size);
+extern kern_return_t vm_map_write_user(vm_map_t map, void *src_p, vm_map_address_t dst_addr, vm_size_t size);
+
+/* Not called from probe context */
+int
+uread(proc_t *p, void *buf, user_size_t len, user_addr_t a)
+{
+	kern_return_t ret;
+
+	ASSERT(p != PROC_NULL);
+	ASSERT(p->task != NULL);
+
+	task_t task = p->task;
+
+	/*
+	 * Grab a reference to the task vm_map_t to make sure
+	 * the map isn't pulled out from under us.
+	 *
+	 * Because the proc_lock is not held at all times on all code
+	 * paths leading here, it is possible for the proc to have
+	 * exited. If the map is null, fail.
+	 */
+	vm_map_t map = get_task_map_reference(task);
+	if (map) {
+		ret = vm_map_read_user( map, (vm_map_address_t)a, buf, (vm_size_t)len);
+		vm_map_deallocate(map);
+	} else
+		ret = KERN_TERMINATED;
+	
+	return (int)ret;
+}
+
+
+/* Not called from probe context */
+int
+uwrite(proc_t *p, void *buf, user_size_t len, user_addr_t a)
+{
+	kern_return_t ret;
+
+	ASSERT(p != NULL);
+	ASSERT(p->task != NULL);
+
+	task_t task = p->task;
+
+	/*
+	 * Grab a reference to the task vm_map_t to make sure
+	 * the map isn't pulled out from under us.
+	 *
+	 * Because the proc_lock is not held at all times on all code
+	 * paths leading here, it is possible for the proc to have
+	 * exited. If the map is null, fail.
+	 */
+	vm_map_t map = get_task_map_reference(task);
+	if (map) {
+		/* Find the memory permissions. */
+		uint32_t nestingDepth=999999;
+		vm_region_submap_short_info_data_64_t info;
+		mach_msg_type_number_t count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
+		mach_vm_address_t address = (mach_vm_address_t)a;
+		mach_vm_size_t sizeOfRegion = (mach_vm_size_t)len;
+	
+		ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count);
+		if (ret != KERN_SUCCESS)
+			goto done;
+
+		vm_prot_t reprotect;
+
+		if (!(info.protection & VM_PROT_WRITE)) {
+			/* Save the original protection values for restoration later */
+			reprotect = info.protection;
+
+			if (info.max_protection & VM_PROT_WRITE) {
+				/* The memory is not currently writable, but can be made writable. */
+				ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect | VM_PROT_WRITE);
+			} else {
+				/*
+				 * The memory is not currently writable, and cannot be made writable. We need to COW this memory.
+				 *
+				 * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails.
+				 */
+				ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE);
+			}
+
+			if (ret != KERN_SUCCESS)
+				goto done;
+
+		} else {
+			/* The memory was already writable. */
+			reprotect = VM_PROT_NONE;
+		}
+
+		ret = vm_map_write_user( map,
+					 buf,
+					 (vm_map_address_t)a,
+					 (vm_size_t)len);
+
+		if (ret != KERN_SUCCESS)
+			goto done;
+
+		if (reprotect != VM_PROT_NONE) {
+			ASSERT(reprotect & VM_PROT_EXECUTE);
+			ret = mach_vm_protect (map, (mach_vm_offset_t)a, (mach_vm_size_t)len, 0, reprotect);
+		}
+
+done:
+		vm_map_deallocate(map);
+	} else 
+		ret = KERN_TERMINATED;
+
+	return (int)ret;
+}
+
+/*
+ * cpuvar
+ */
+lck_mtx_t cpu_lock;
+lck_mtx_t mod_lock;
+
+cpu_t *cpu_list;
+cpu_core_t *cpu_core; /* XXX TLB lockdown? */
+
+/*
+ * cred_t
+ */
+
+/*
+ * dtrace_CRED() can be called from probe context. We cannot simply call kauth_cred_get() since
+ * that function may try to resolve a lazy credential binding, which entails taking the proc_lock.
+ */ 
+cred_t *
+dtrace_CRED(void)
+{
+	struct uthread *uthread = get_bsdthread_info(current_thread());
+
+	if (uthread == NULL)
+		return NULL;
+	else
+		return uthread->uu_ucred; /* May return NOCRED which is defined to be 0 */
+}
+
+#define	HAS_ALLPRIVS(cr)	priv_isfullset(&CR_OEPRIV(cr))
+#define	HAS_PRIVILEGE(cr, pr)	((pr) == PRIV_ALL ? \
+					HAS_ALLPRIVS(cr) : \
+					PRIV_ISASSERT(&CR_OEPRIV(cr), pr))
+
+int PRIV_POLICY_CHOICE(void* cred, int priv, int all)
+{
+#pragma unused(priv, all)
+	return kauth_cred_issuser(cred); /* XXX TODO: How is this different from PRIV_POLICY_ONLY? */
+}
+
+int 
+PRIV_POLICY_ONLY(void *cr, int priv, int boolean)
+{
+#pragma unused(priv, boolean)
+	return kauth_cred_issuser(cr); /* XXX TODO: HAS_PRIVILEGE(cr, priv); */
+}
+
+gid_t
+crgetgid(const cred_t *cr) { return cr->cr_groups[0]; }
+
+uid_t
+crgetuid(const cred_t *cr) { return cr->cr_uid; }
+
+/*
+ * "cyclic"
+ */
+
+/* osfmk/kern/timer_call.h */
+typedef void            *call_entry_param_t;
+typedef void            (*call_entry_func_t)(
+								call_entry_param_t      param0,
+								call_entry_param_t      param1);
+
+typedef struct call_entry {
+	queue_chain_t       q_link;
+	call_entry_func_t   func;
+	call_entry_param_t  param0;
+	call_entry_param_t  param1;
+	uint64_t            deadline;
+	enum {
+		IDLE,
+		PENDING,
+		DELAYED }         state;
+} call_entry_data_t;
+
+
+typedef struct call_entry   *timer_call_t;
+typedef void                *timer_call_param_t;
+typedef void                (*timer_call_func_t)(
+	timer_call_param_t      param0,
+	timer_call_param_t      param1);
+
+extern void
+timer_call_setup(
+	timer_call_t            call,
+	timer_call_func_t       func,
+	timer_call_param_t      param0);
+
+extern boolean_t
+timer_call_enter1(
+	timer_call_t            call,
+	timer_call_param_t      param1,
+	uint64_t                deadline);
+
+extern boolean_t
+timer_call_cancel(
+	timer_call_t            call);
+
+typedef struct wrap_timer_call {
+	cyc_handler_t hdlr;
+	cyc_time_t when;
+	uint64_t deadline;
+	struct call_entry call;
+} wrap_timer_call_t;
+
+#define WAKEUP_REAPER 0x7FFFFFFFFFFFFFFFLL
+#define NEARLY_FOREVER 0x7FFFFFFFFFFFFFFELL
+
+static void
+_timer_call_apply_cyclic( void *ignore, void *vTChdl )
+{
+#pragma unused(ignore)
+	wrap_timer_call_t *wrapTC = (wrap_timer_call_t *)vTChdl;
+
+	(*(wrapTC->hdlr.cyh_func))( wrapTC->hdlr.cyh_arg );
+
+	clock_deadline_for_periodic_event( wrapTC->when.cyt_interval, mach_absolute_time(), &(wrapTC->deadline) );
+	timer_call_enter1( &(wrapTC->call), (void *)wrapTC, wrapTC->deadline );
+
+	/* Did timer_call_remove_cyclic request a wakeup call when this timer call was re-armed? */
+	if (wrapTC->when.cyt_interval == WAKEUP_REAPER)
+		thread_wakeup((event_t)wrapTC);
+}
+
+static cyclic_id_t
+timer_call_add_cyclic(wrap_timer_call_t *wrapTC, cyc_handler_t *handler, cyc_time_t *when)
+{
+	uint64_t now;
+
+	timer_call_setup( &(wrapTC->call),  _timer_call_apply_cyclic, NULL );
+	wrapTC->hdlr = *handler;
+	wrapTC->when = *when;
+
+	nanoseconds_to_absolutetime( wrapTC->when.cyt_interval, (uint64_t *)&wrapTC->when.cyt_interval );
+
+	now = mach_absolute_time();
+	wrapTC->deadline = now;
+
+	clock_deadline_for_periodic_event( wrapTC->when.cyt_interval, now, &(wrapTC->deadline) );
+	timer_call_enter1( &(wrapTC->call), (void *)wrapTC, wrapTC->deadline );
+
+	return (cyclic_id_t)wrapTC;
+}
+
+static void
+timer_call_remove_cyclic(cyclic_id_t cyclic)
+{
+	wrap_timer_call_t *wrapTC = (wrap_timer_call_t *)cyclic;
+
+	while (!timer_call_cancel(&(wrapTC->call))) {
+		int ret = assert_wait(wrapTC, THREAD_UNINT);
+		ASSERT(ret == THREAD_WAITING);
+
+		wrapTC->when.cyt_interval = WAKEUP_REAPER;
+
+		ret = thread_block(THREAD_CONTINUE_NULL);
+		ASSERT(ret == THREAD_AWAKENED);
+	}
+}
+
+static void *
+timer_call_get_cyclic_arg(cyclic_id_t cyclic)
+{       
+	wrap_timer_call_t *wrapTC = (wrap_timer_call_t *)cyclic;
+ 	
+	return (wrapTC ? wrapTC->hdlr.cyh_arg : NULL);
+}   
+
+cyclic_id_t
+cyclic_timer_add(cyc_handler_t *handler, cyc_time_t *when)
+{
+	wrap_timer_call_t *wrapTC = _MALLOC(sizeof(wrap_timer_call_t), M_TEMP, M_ZERO | M_WAITOK);
+	if (NULL == wrapTC)
+		return CYCLIC_NONE;
+	else
+		return timer_call_add_cyclic( wrapTC, handler, when );
+}
+
+void 
+cyclic_timer_remove(cyclic_id_t cyclic)
+{
+	ASSERT( cyclic != CYCLIC_NONE );
+
+	timer_call_remove_cyclic( cyclic );
+	_FREE((void *)cyclic, M_TEMP);
+}
+
+static void
+_cyclic_add_omni(cyclic_id_list_t cyc_list)
+{
+	cyc_time_t cT;
+	cyc_handler_t cH;
+	wrap_timer_call_t *wrapTC;
+	cyc_omni_handler_t *omni = (cyc_omni_handler_t *)cyc_list;
+	char *t;
+
+	(omni->cyo_online)(omni->cyo_arg, CPU, &cH, &cT); 
+
+	t = (char *)cyc_list;
+	t += sizeof(cyc_omni_handler_t);
+	cyc_list = (cyclic_id_list_t)t;
+
+	t += sizeof(cyclic_id_t)*NCPU;
+	t += (sizeof(wrap_timer_call_t))*cpu_number();
+	wrapTC = (wrap_timer_call_t *)t;
+
+	cyc_list[cpu_number()] = timer_call_add_cyclic(wrapTC, &cH, &cT);
+}
+
+cyclic_id_list_t
+cyclic_add_omni(cyc_omni_handler_t *omni)
+{
+	cyclic_id_list_t cyc_list = 
+		_MALLOC( (sizeof(wrap_timer_call_t))*NCPU + 
+				 sizeof(cyclic_id_t)*NCPU + 
+				 sizeof(cyc_omni_handler_t), M_TEMP, M_ZERO | M_WAITOK);
+	if (NULL == cyc_list)
+		return (cyclic_id_list_t)CYCLIC_NONE;
+
+	*(cyc_omni_handler_t *)cyc_list = *omni;
+	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)_cyclic_add_omni, (void *)cyc_list);
+
+	return cyc_list;
+}
+
+static void
+_cyclic_remove_omni(cyclic_id_list_t cyc_list)
+{
+	cyc_omni_handler_t *omni = (cyc_omni_handler_t *)cyc_list;
+	void *oarg;
+	cyclic_id_t cid;
+	char *t;
+
+	t = (char *)cyc_list;
+	t += sizeof(cyc_omni_handler_t);
+	cyc_list = (cyclic_id_list_t)t;
+
+	cid = cyc_list[cpu_number()];
+	oarg = timer_call_get_cyclic_arg(cid);
+
+	timer_call_remove_cyclic( cid );
+	(omni->cyo_offline)(omni->cyo_arg, CPU, oarg);
+}
+
+void
+cyclic_remove_omni(cyclic_id_list_t cyc_list)
+{
+	ASSERT( cyc_list != (cyclic_id_list_t)CYCLIC_NONE );
+
+	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)_cyclic_remove_omni, (void *)cyc_list);
+	_FREE(cyc_list, M_TEMP);
+}
+
+typedef struct wrap_thread_call {
+	thread_call_t TChdl;
+	cyc_handler_t hdlr;
+	cyc_time_t when;
+	uint64_t deadline;
+} wrap_thread_call_t;
+
+/*
+ * _cyclic_apply will run on some thread under kernel_task. That's OK for the 
+ * cleaner and the deadman, but too distant in time and place for the profile provider.
+ */
+static void
+_cyclic_apply( void *ignore, void *vTChdl )
+{
+#pragma unused(ignore)
+	wrap_thread_call_t *wrapTC = (wrap_thread_call_t *)vTChdl;
+
+	(*(wrapTC->hdlr.cyh_func))( wrapTC->hdlr.cyh_arg );
+
+	clock_deadline_for_periodic_event( wrapTC->when.cyt_interval, mach_absolute_time(), &(wrapTC->deadline) );
+	(void)thread_call_enter1_delayed( wrapTC->TChdl, (void *)wrapTC, wrapTC->deadline );
+
+	/* Did cyclic_remove request a wakeup call when this thread call was re-armed? */
+	if (wrapTC->when.cyt_interval == WAKEUP_REAPER)
+		thread_wakeup((event_t)wrapTC);
+}
+
+cyclic_id_t
+cyclic_add(cyc_handler_t *handler, cyc_time_t *when)
+{
+	uint64_t now;
+
+	wrap_thread_call_t *wrapTC = _MALLOC(sizeof(wrap_thread_call_t), M_TEMP, M_ZERO | M_WAITOK);
+	if (NULL == wrapTC)
+		return CYCLIC_NONE;
+
+	wrapTC->TChdl = thread_call_allocate( _cyclic_apply, NULL );
+	wrapTC->hdlr = *handler;
+	wrapTC->when = *when;
+
+	ASSERT(when->cyt_when == 0);
+	ASSERT(when->cyt_interval < WAKEUP_REAPER);
+
+	nanoseconds_to_absolutetime(wrapTC->when.cyt_interval, (uint64_t *)&wrapTC->when.cyt_interval);
+
+	now = mach_absolute_time();
+	wrapTC->deadline = now;
+
+	clock_deadline_for_periodic_event( wrapTC->when.cyt_interval, now, &(wrapTC->deadline) );
+	(void)thread_call_enter1_delayed( wrapTC->TChdl, (void *)wrapTC, wrapTC->deadline );
+
+	return (cyclic_id_t)wrapTC;
+}
+
+static void
+noop_cyh_func(void * ignore)
+{
+#pragma unused(ignore)
+}
+
+void
+cyclic_remove(cyclic_id_t cyclic)
+{
+	wrap_thread_call_t *wrapTC = (wrap_thread_call_t *)cyclic;
+
+	ASSERT(cyclic != CYCLIC_NONE);
+
+	while (!thread_call_cancel(wrapTC->TChdl)) {
+		int ret = assert_wait(wrapTC, THREAD_UNINT);
+		ASSERT(ret == THREAD_WAITING);
+
+		wrapTC->when.cyt_interval = WAKEUP_REAPER;
+
+		ret = thread_block(THREAD_CONTINUE_NULL);
+		ASSERT(ret == THREAD_AWAKENED);
+	}
+
+	if (thread_call_free(wrapTC->TChdl))
+		_FREE(wrapTC, M_TEMP);
+	else {
+		/* Gut this cyclic and move on ... */
+		wrapTC->hdlr.cyh_func = noop_cyh_func;
+		wrapTC->when.cyt_interval = NEARLY_FOREVER;
+	}
+}
+
+/*
+ * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision)
+ */ 
+
+thread_call_t
+dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos)
+{
+#pragma unused(arg)
+	thread_call_t call = thread_call_allocate(func, NULL);
+
+	nanoseconds_to_absolutetime(nanos, &nanos);
+
+	/*
+	 * This method does not use clock_deadline_for_periodic_event() because it is a one-shot,
+	 * and clock drift on later invocations is not a worry.
+	 */
+	uint64_t deadline = mach_absolute_time() + nanos;
+
+	thread_call_enter_delayed(call, deadline);
+
+	return call;
+}
+
+/*
+ * ddi
+ */
+void
+ddi_report_dev(dev_info_t *devi)
+{
+#pragma unused(devi)
+}
+
+#define NSOFT_STATES 32 /* XXX No more than 32 clients at a time, please. */
+static void *soft[NSOFT_STATES];
+
+int
+ddi_soft_state_init(void **state_p, size_t size, size_t n_items)
+{
+#pragma unused(n_items)
+	int i;
+	
+	for (i = 0; i < NSOFT_STATES; ++i) soft[i] = _MALLOC(size, M_TEMP, M_ZERO | M_WAITOK);
+	*(size_t *)state_p = size;
+	return 0;
+}
+
+int
+ddi_soft_state_zalloc(void *state, int item)
+{
+#pragma unused(state)
+	if (item < NSOFT_STATES)
+		return DDI_SUCCESS;
+	else
+		return DDI_FAILURE;
+}
+
+void *
+ddi_get_soft_state(void *state, int item)
+{
+#pragma unused(state)
+	ASSERT(item < NSOFT_STATES);
+	return soft[item];
+}
+
+int
+ddi_soft_state_free(void *state, int item)
+{
+	ASSERT(item < NSOFT_STATES);
+	bzero( soft[item], (size_t)state );
+	return DDI_SUCCESS;
+}
+
+void
+ddi_soft_state_fini(void **state_p)
+{
+#pragma unused(state_p)
+	int i;
+	
+	for (i = 0; i < NSOFT_STATES; ++i) _FREE( soft[i], M_TEMP );
+}
+
+static unsigned int gRegisteredProps = 0;
+static struct {
+	char name[32];		/* enough for "dof-data-" + digits */
+	int *data;
+	uint_t nelements;
+} gPropTable[16];
+
+kern_return_t _dtrace_register_anon_DOF(char *, uchar_t *, uint_t);
+
+kern_return_t
+_dtrace_register_anon_DOF(char *name, uchar_t *data, uint_t nelements)
+{
+	if (gRegisteredProps < sizeof(gPropTable)/sizeof(gPropTable[0])) {
+		int *p = (int *)_MALLOC(nelements*sizeof(int), M_TEMP, M_WAITOK);
+		
+		if (NULL == p)
+			return KERN_FAILURE;
+			
+		strlcpy(gPropTable[gRegisteredProps].name, name, sizeof(gPropTable[0].name));
+		gPropTable[gRegisteredProps].nelements = nelements;
+		gPropTable[gRegisteredProps].data = p;
+			
+		while (nelements-- > 0) {
+			*p++ = (int)(*data++);
+		}
+		
+		gRegisteredProps++;
+		return KERN_SUCCESS;
+	}
+	else
+		return KERN_FAILURE;
+}
+
+int
+ddi_prop_lookup_int_array(dev_t match_dev, dev_info_t *dip, uint_t flags,
+    char *name, int **data, uint_t *nelements)
+{
+#pragma unused(match_dev,dip,flags)
+	unsigned int i;
+	for (i = 0; i < gRegisteredProps; ++i)
+	{
+		if (0 == strncmp(name, gPropTable[i].name,
+					sizeof(gPropTable[i].name))) {
+			*data = gPropTable[i].data;
+			*nelements = gPropTable[i].nelements;
+			return DDI_SUCCESS;
+		}
+	}
+	return DDI_FAILURE;
+}
+	
+int
+ddi_prop_free(void *buf)
+{
+	_FREE(buf, M_TEMP);
+	return DDI_SUCCESS;
+}
+
+int
+ddi_driver_major(dev_info_t	*devi) { return (int)major(devi); }
+
+int
+ddi_create_minor_node(dev_info_t *dip, const char *name, int spec_type,
+    minor_t minor_num, const char *node_type, int flag)
+{
+#pragma unused(spec_type,node_type,flag)
+	dev_t dev = makedev( (uint32_t)dip, minor_num );
+
+	if (NULL == devfs_make_node( dev, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, name, 0 ))
+		return DDI_FAILURE;
+	else
+		return DDI_SUCCESS;
+} 
+
+void
+ddi_remove_minor_node(dev_info_t *dip, char *name)
+{
+#pragma unused(dip,name)
+/* XXX called from dtrace_detach, so NOTREACHED for now. */
+}
+
+major_t
+getemajor( dev_t d )
+{
+	return (major_t) major(d);
+}
+
+minor_t
+getminor ( dev_t d )
+{
+	return (minor_t) minor(d);
+}
+
+dev_t 
+makedevice(major_t major, minor_t minor)
+{
+	return makedev( major, minor );
+}
+
+int ddi_getprop(dev_t dev, dev_info_t *dip, int flags, const char *name, int defvalue)
+{
+#pragma unused(dev, dip, flags, name)
+
+	return defvalue;
+}
+
+/*
+ * Kernel Debug Interface
+ */
+int
+kdi_dtrace_set(kdi_dtrace_set_t ignore)
+{
+#pragma unused(ignore)
+	return 0; /* Success */
+}
+
+extern void Debugger(const char*);
+
+void
+debug_enter(char *c) { Debugger(c); }
+
+/*
+ * kmem
+ */
+
+void *
+dt_kmem_alloc(size_t size, int kmflag)
+{
+#pragma unused(kmflag)
+
+/*
+ * We ignore the M_NOWAIT bit in kmflag (all of kmflag, in fact).
+ * Requests larger than 8K with M_NOWAIT fail in kalloc_canblock.
+ */
+#if defined(DTRACE_MEMORY_ZONES)
+	return dtrace_alloc(size);
+#else
+	return kalloc(size);
+#endif
+}
+
+void *
+dt_kmem_zalloc(size_t size, int kmflag)
+{
+#pragma unused(kmflag)
+
+/*
+ * We ignore the M_NOWAIT bit in kmflag (all of kmflag, in fact).
+ * Requests larger than 8K with M_NOWAIT fail in kalloc_canblock.
+ */
+#if defined(DTRACE_MEMORY_ZONES)
+	void* buf = dtrace_alloc(size);
+#else
+	void* buf = kalloc(size);
+#endif
+
+	if(!buf)
+		return NULL;
+
+	bzero(buf, size);
+
+	return buf;
+}
+
+void
+dt_kmem_free(void *buf, size_t size)
+{
+#pragma unused(size)
+	/*
+	 * DTrace relies on this, its doing a lot of NULL frees.
+	 * A null free causes the debug builds to panic.
+	 */
+	if (buf == NULL) return;
+
+	ASSERT(size > 0);
+
+#if defined(DTRACE_MEMORY_ZONES)
+	dtrace_free(buf, size);
+#else
+	kfree(buf, size);
+#endif
+}
+
+
+
+/*
+ * aligned kmem allocator
+ * align should be a power of two
+ */
+
+void* dt_kmem_alloc_aligned(size_t size, size_t align, int kmflag)
+{
+	void* buf;
+	intptr_t p;
+	void** buf_backup;
+
+	buf = dt_kmem_alloc(align + sizeof(void*) + size, kmflag);
+
+	if(!buf)
+		return NULL;
+
+	p = (intptr_t)buf;
+	p += sizeof(void*);				/* now we have enough room to store the backup */
+	p = P2ROUNDUP(p, align);			/* and now we're aligned */
+
+	buf_backup = (void**)(p - sizeof(void*));
+	*buf_backup = buf;				/* back up the address we need to free */
+
+	return (void*)p;
+}
+
+void* dt_kmem_zalloc_aligned(size_t size, size_t align, int kmflag)
+{
+	void* buf;
+
+	buf = dt_kmem_alloc_aligned(size, align, kmflag);
+
+	if(!buf)
+		return NULL;
+
+	bzero(buf, size);
+
+	return buf;
+}
+
+void dt_kmem_free_aligned(void* buf, size_t size)
+{
+#pragma unused(size)
+	intptr_t p;
+	void** buf_backup;
+
+	p = (intptr_t)buf;
+	p -= sizeof(void*);
+	buf_backup = (void**)(p);
+
+	dt_kmem_free(*buf_backup, size + ((char*)buf - (char*)*buf_backup));
+}
+
+/*
+ * dtrace wants to manage just a single block: dtrace_state_percpu_t * NCPU, and
+ * doesn't specify constructor, destructor, or reclaim methods.
+ * At present, it always zeroes the block it obtains from kmem_cache_alloc().
+ * We'll manage this constricted use of kmem_cache with ordinary _MALLOC and _FREE.
+ */
+kmem_cache_t *
+kmem_cache_create(
+    char *name,		/* descriptive name for this cache */
+    size_t bufsize,		/* size of the objects it manages */
+    size_t align,		/* required object alignment */
+    int (*constructor)(void *, void *, int), /* object constructor */
+    void (*destructor)(void *, void *),	/* object destructor */
+    void (*reclaim)(void *), /* memory reclaim callback */
+    void *private,		/* pass-thru arg for constr/destr/reclaim */
+    vmem_t *vmp,		/* vmem source for slab allocation */
+    int cflags)		/* cache creation flags */
+{
+#pragma unused(name,align,constructor,destructor,reclaim,private,vmp,cflags)
+	return (kmem_cache_t *)bufsize; /* A cookie that tracks the single object size. */
+}
+	
+void *
+kmem_cache_alloc(kmem_cache_t *cp, int kmflag)
+{
+#pragma unused(kmflag)
+	size_t bufsize = (size_t)cp;
+	return (void *)_MALLOC(bufsize, M_TEMP, M_WAITOK);
+}
+
+void
+kmem_cache_free(kmem_cache_t *cp, void *buf)
+{
+#pragma unused(cp)
+	_FREE(buf, M_TEMP);
+}
+
+void
+kmem_cache_destroy(kmem_cache_t *cp)
+{
+#pragma unused(cp)
+}
+
+/*
+ * taskq
+ */
+extern void thread_call_setup(thread_call_t, thread_call_func_t, thread_call_param_t); /* XXX MACH_KERNEL_PRIVATE */
+
+static void
+_taskq_apply( task_func_t func, thread_call_param_t arg )
+{
+	func( (void *)arg );
+}
+
+taskq_t *
+taskq_create(const char *name, int nthreads, pri_t pri, int minalloc,
+    int maxalloc, uint_t flags)
+{
+#pragma unused(name,nthreads,pri,minalloc,maxalloc,flags)
+
+	return (taskq_t *)thread_call_allocate( (thread_call_func_t)_taskq_apply, NULL );
+}
+
+taskqid_t
+taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
+{
+#pragma unused(flags)
+	thread_call_setup( (thread_call_t) tq, (thread_call_func_t)_taskq_apply, (thread_call_param_t)func );
+	thread_call_enter1( (thread_call_t) tq, (thread_call_param_t)arg );
+	return (taskqid_t) tq /* for lack of anything better */;
+}
+
+void	
+taskq_destroy(taskq_t *tq)
+{
+	thread_call_cancel( (thread_call_t) tq );
+	thread_call_free( (thread_call_t) tq );
+}
+
+pri_t maxclsyspri;
+
+/*
+ * vmem (Solaris "slab" allocator) used by DTrace solely to hand out resource ids
+ */
+typedef unsigned int u_daddr_t;
+#include "blist.h"
+
+/* By passing around blist *handles*, the underlying blist can be resized as needed. */
+struct blist_hdl {
+	blist_t blist; 
+};
+
+vmem_t * 
+vmem_create(const char *name, void *base, size_t size, size_t quantum, void *ignore5,
+					void *ignore6, vmem_t *source, size_t qcache_max, int vmflag)
+{
+#pragma unused(name,quantum,ignore5,ignore6,source,qcache_max,vmflag)
+	blist_t bl;
+	struct blist_hdl *p = _MALLOC(sizeof(struct blist_hdl), M_TEMP, M_WAITOK);
+	
+	ASSERT(quantum == 1);
+	ASSERT(NULL == ignore5);
+	ASSERT(NULL == ignore6);
+	ASSERT(NULL == source);
+	ASSERT(0 == qcache_max);
+	ASSERT(vmflag & VMC_IDENTIFIER);
+	
+	size = MIN(128, size); /* Clamp to 128 initially, since the underlying data structure is pre-allocated */
+	
+	p->blist = bl = blist_create( size );
+	blist_free(bl, 0, size);
+	if (base) blist_alloc( bl, (daddr_t)base ); /* Chomp off initial ID(s) */
+	
+	return (vmem_t *)p;
+}
+ 
+void *
+vmem_alloc(vmem_t *vmp, size_t size, int vmflag)
+{
+#pragma unused(vmflag)
+	struct blist_hdl *q = (struct blist_hdl *)vmp;
+	blist_t bl = q->blist;
+	daddr_t p;
+	
+	p = blist_alloc(bl, (daddr_t)size);
+	
+	if ((daddr_t)-1 == p) {
+		blist_resize(&bl, (bl->bl_blocks) << 1, 1);
+		q->blist = bl;
+		p = blist_alloc(bl, (daddr_t)size);
+		if ((daddr_t)-1 == p) 
+			panic("vmem_alloc: failure after blist_resize!");
+	}
+	
+	return (void *)p;
+}
+
+void
+vmem_free(vmem_t *vmp, void *vaddr, size_t size)
+{
+	struct blist_hdl *p = (struct blist_hdl *)vmp;
+	
+	blist_free( p->blist, (daddr_t)vaddr, (daddr_t)size );
+}
+
+void
+vmem_destroy(vmem_t *vmp)
+{
+	struct blist_hdl *p = (struct blist_hdl *)vmp;
+	
+	blist_destroy( p->blist );
+	_FREE( p, sizeof(struct blist_hdl) );
+}
+
+/*
+ * Timing
+ */
+
+/*
+ * dtrace_gethrestime() provides the "walltimestamp", a value that is anchored at 
+ * January 1, 1970. Because it can be called from probe context, it must take no locks.
+ */
+
+hrtime_t
+dtrace_gethrestime(void)
+{
+	uint32_t		secs, nanosecs;
+	uint64_t		secs64, ns64;
+    
+	clock_get_calendar_nanotime_nowait(&secs, &nanosecs);
+	secs64 = (uint64_t)secs;
+	ns64 = (uint64_t)nanosecs;
+
+	ns64 = ns64 + (secs64 * 1000000000LL);
+	return ns64;
+}
+
+/*
+ * dtrace_gethrtime() provides high-resolution timestamps with machine-dependent origin.
+ * Hence its primary use is to specify intervals.
+ */
+
+hrtime_t
+dtrace_abs_to_nano(uint64_t elapsed)
+{
+	static mach_timebase_info_data_t    sTimebaseInfo = { 0, 0 };
+
+	/*
+	 * If this is the first time we've run, get the timebase.
+	 * We can use denom == 0 to indicate that sTimebaseInfo is
+	 * uninitialised because it makes no sense to have a zero
+	 * denominator in a fraction.
+	 */
+
+	if ( sTimebaseInfo.denom == 0 ) {
+		(void) clock_timebase_info(&sTimebaseInfo);
+	}
+
+	/*
+	 * Convert to nanoseconds.
+	 * return (elapsed * (uint64_t)sTimebaseInfo.numer)/(uint64_t)sTimebaseInfo.denom;
+	 *
+	 * Provided the final result is representable in 64 bits the following maneuver will
+	 * deliver that result without intermediate overflow.
+	 */
+	if (sTimebaseInfo.denom == sTimebaseInfo.numer)
+		return elapsed;
+	else if (sTimebaseInfo.denom == 1)
+		return elapsed * (uint64_t)sTimebaseInfo.numer;
+	else {
+		/* Decompose elapsed = eta32 * 2^32 + eps32: */
+		uint64_t eta32 = elapsed >> 32;
+		uint64_t eps32 = elapsed & 0x00000000ffffffffLL;
+
+		uint32_t numer = sTimebaseInfo.numer, denom = sTimebaseInfo.denom;
+
+		/* Form product of elapsed64 (decomposed) and numer: */
+		uint64_t mu64 = numer * eta32;
+		uint64_t lambda64 = numer * eps32;
+
+		/* Divide the constituents by denom: */
+		uint64_t q32 = mu64/denom;
+		uint64_t r32 = mu64 - (q32 * denom); /* mu64 % denom */
+
+		return (q32 << 32) + ((r32 << 32) + lambda64)/denom;
+	}
+}
+
+hrtime_t
+dtrace_gethrtime(void)
+{
+    static uint64_t        start = 0;
+    
+	if (start == 0)
+		start = mach_absolute_time();
+		
+    return dtrace_abs_to_nano(mach_absolute_time() - start);
+}
+
+/*
+ * Atomicity and synchronization
+ */
+uint32_t
+dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new)
+{
+	if (OSCompareAndSwap( cmp, new, (unsigned long *)target ))
+		return cmp;
+	else
+		return ~cmp; /* Must return something *other* than cmp */
+}
+
+void *
+dtrace_casptr(void *target, void *cmp, void *new)
+{
+#if defined(__LP64__)
+#error dtrace_casptr implementation missing for LP64
+#else
+	if (OSCompareAndSwap( (uint32_t)cmp, (uint32_t)new, (unsigned long *)target ))
+		return cmp;
+	else
+		return (void *)(~(uintptr_t)cmp); /* Must return something *other* than cmp */
+#endif
+}
+
+/*
+ * Interrupt manipulation
+ */
+dtrace_icookie_t
+dtrace_interrupt_disable(void)
+{
+	return (dtrace_icookie_t)ml_set_interrupts_enabled(FALSE);
+}
+
+void
+dtrace_interrupt_enable(dtrace_icookie_t reenable)
+{
+	(void)ml_set_interrupts_enabled((boolean_t)reenable);
+}
+
+/*
+ * MP coordination
+ */
+static void
+dtrace_sync_func(void) {}
+
+/*
+ * dtrace_sync() is not called from probe context.
+ */
+void
+dtrace_sync(void)
+{
+	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
+}
+
+/*
+ * The dtrace_copyin/out/instr and dtrace_fuword* routines can be called from probe context.
+ */
+
+extern kern_return_t dtrace_copyio_preflight(addr64_t);
+extern kern_return_t dtrace_copyio_postflight(addr64_t);
+ 
+static int
+dtrace_copycheck(user_addr_t uaddr, uintptr_t kaddr, size_t size)
+{
+#pragma unused(kaddr)
+
+	vm_offset_t recover = dtrace_set_thread_recover( current_thread(), 0 ); /* Snare any extant recovery point. */
+	dtrace_set_thread_recover( current_thread(), recover ); /* Put it back. We *must not* re-enter and overwrite. */
+
+	ASSERT(kaddr + size >= kaddr);
+
+	if (ml_at_interrupt_context() ||	/* Avoid possible copyio page fault on int stack, which panics! */ 
+		0 != recover ||					/* Avoid reentrancy into copyio facility. */
+		uaddr + size < uaddr ||			/* Avoid address wrap. */
+		KERN_FAILURE == dtrace_copyio_preflight(uaddr)) /* Machine specific setup/constraints. */
+	{
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+		return (0);
+	}
+	return (1);
+}
+
+void
+dtrace_copyin(user_addr_t src, uintptr_t dst, size_t len)
+{
+	if (dtrace_copycheck( src, dst, len )) {
+		if (copyin((const user_addr_t)src, (char *)dst, (vm_size_t)len)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = src;
+		}
+		dtrace_copyio_postflight(src);
+	}
+}
+
+void
+dtrace_copyinstr(user_addr_t src, uintptr_t dst, size_t len)
+{
+	size_t actual;
+	
+	if (dtrace_copycheck( src, dst, len )) {
+		if (copyinstr((const user_addr_t)src, (char *)dst, (vm_size_t)len, &actual)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = src;
+		}
+		dtrace_copyio_postflight(src);
+	}
+}
+
+void
+dtrace_copyout(uintptr_t src, user_addr_t dst, size_t len)
+{
+	if (dtrace_copycheck( dst, src, len )) {
+		if (copyout((const void *)src, dst, (vm_size_t)len)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = dst;
+		}
+		dtrace_copyio_postflight(dst);
+	}
+}
+
+void
+dtrace_copyoutstr(uintptr_t src, user_addr_t dst, size_t len)
+{
+	size_t actual;
+
+	if (dtrace_copycheck( dst, src, len )) {
+		if (copyoutstr((const void *)src, dst, (size_t)len, &actual)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = dst;
+		}
+		dtrace_copyio_postflight(dst);
+	}
+}
+
+uint8_t
+dtrace_fuword8(user_addr_t uaddr)
+{
+	uint8_t ret = 0;
+
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	if (dtrace_copycheck( uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const user_addr_t)uaddr, (char *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+		}
+		dtrace_copyio_postflight(uaddr);
+	}
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return(ret);
+}
+
+uint16_t
+dtrace_fuword16(user_addr_t uaddr)
+{
+	uint16_t ret = 0;
+
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	if (dtrace_copycheck( uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const user_addr_t)uaddr, (char *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+		}
+		dtrace_copyio_postflight(uaddr);
+	}
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return(ret);
+}
+
+uint32_t
+dtrace_fuword32(user_addr_t uaddr)
+{
+	uint32_t ret = 0;
+
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	if (dtrace_copycheck( uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const user_addr_t)uaddr, (char *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+		}
+		dtrace_copyio_postflight(uaddr);
+	}
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return(ret);
+}
+
+uint64_t
+dtrace_fuword64(user_addr_t uaddr)
+{
+	uint64_t ret = 0;
+
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	if (dtrace_copycheck( uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const user_addr_t)uaddr, (char *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+		}
+		dtrace_copyio_postflight(uaddr);
+	}
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return(ret);
+}
+
+/*
+ * Emulation of Solaris fuword / suword
+ * Called from the fasttrap provider, so the use of copyin/out requires fewer safegaurds.
+ */
+
+int
+fuword8(user_addr_t uaddr, uint8_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint8_t)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+fuword16(user_addr_t uaddr, uint16_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint16_t)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+fuword32(user_addr_t uaddr, uint32_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint32_t)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+fuword64(user_addr_t uaddr, uint64_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint64_t)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+void
+fuword8_noerr(user_addr_t uaddr, uint8_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint8_t))) {
+		*value = 0;
+	}
+}
+
+void
+fuword16_noerr(user_addr_t uaddr, uint16_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint16_t))) {
+		*value = 0;
+	}
+}
+
+void
+fuword32_noerr(user_addr_t uaddr, uint32_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint32_t))) {
+		*value = 0;
+	}
+}
+
+void
+fuword64_noerr(user_addr_t uaddr, uint64_t *value)
+{
+	if (copyin((const user_addr_t)uaddr, (char *)value, sizeof(uint64_t))) {
+		*value = 0;
+	}
+}
+
+int
+suword64(user_addr_t addr, uint64_t value)
+{
+	if (copyout((const void *)&value, addr, sizeof(value)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+suword32(user_addr_t addr, uint32_t value)
+{
+	if (copyout((const void *)&value, addr, sizeof(value)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+suword16(user_addr_t addr, uint16_t value)
+{
+	if (copyout((const void *)&value, addr, sizeof(value)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+suword8(user_addr_t addr, uint8_t value)
+{
+	if (copyout((const void *)&value, addr, sizeof(value)) != 0) {
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
+ * Miscellaneous
+ */
+extern boolean_t dtrace_tally_fault(user_addr_t);
+
+boolean_t
+dtrace_tally_fault(user_addr_t uaddr)
+{
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+	cpu_core[CPU->cpu_id].cpuc_dtrace_illval = uaddr;
+	return( DTRACE_CPUFLAG_ISSET(CPU_DTRACE_NOFAULT) ? TRUE : FALSE );
+}
+
+void
+dtrace_vpanic(const char *format, va_list alist)
+{
+	vuprintf( format, alist );
+	panic("dtrace_vpanic");
+}
+
+#define TOTTY   0x02
+extern int prf(const char *, va_list, int, struct tty *); /* bsd/kern/subr_prf.h */
+
+int
+vuprintf(const char *format, va_list ap)
+{
+	return prf(format, ap, TOTTY, NULL);
+}
+
+/* Not called from probe context */
+void cmn_err( int level, const char *format, ... )
+{
+#pragma unused(level)
+	va_list alist;
+
+	va_start(alist, format);
+	vuprintf(format, alist);
+	va_end(alist);
+	uprintf("\n");
+}
+
+/*
+ * History:
+ *  2002-01-24 	gvdl	Initial implementation of strstr
+ */
+
+__private_extern__ char *
+strstr(const char *in, const char *str)
+{
+    char c;
+    size_t len;
+
+    c = *str++;
+    if (!c)
+        return (char *) in;	// Trivial empty string case
+
+    len = strlen(str);
+    do {
+        char sc;
+
+        do {
+            sc = *in++;
+            if (!sc)
+                return (char *) 0;
+        } while (sc != c);
+    } while (strncmp(in, str, len) != 0);
+
+    return (char *) (in - 1);
+}
+
+/*
+ * Runtime and ABI
+ */
+uintptr_t
+dtrace_caller(int ignore)
+{
+#pragma unused(ignore)
+	return -1; /* Just as in Solaris dtrace_asm.s */
+}
+
+int
+dtrace_getstackdepth(int aframes)
+{
+	struct frame *fp = (struct frame *)dtrace_getfp();
+	struct frame *nextfp, *minfp, *stacktop;
+	int depth = 0;
+	int on_intr;
+
+	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
+		stacktop = (struct frame *)dtrace_get_cpu_int_stack_top();
+	else
+		stacktop = (struct frame *)(dtrace_get_kernel_stack(current_thread()) + KERNEL_STACK_SIZE);
+
+	minfp = fp;
+
+	aframes++;
+
+	for (;;) {
+		depth++;
+
+		nextfp = *(struct frame **)fp;
+
+		if (nextfp <= minfp || nextfp >= stacktop) {
+			if (on_intr) {
+				/*
+				 * Hop from interrupt stack to thread stack.
+				 */
+                                vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
+
+                                minfp = (struct frame *)kstack_base;
+                                stacktop = (struct frame *)(kstack_base + KERNEL_STACK_SIZE);
+
+				on_intr = 0;
+				continue;
+			}
+			break;
+		}
+
+		fp = nextfp;
+		minfp = fp;
+	}
+
+	if (depth <= aframes)
+		return (0);
+
+	return (depth - aframes);
+}
+
+/*
+ * Unconsidered
+ */
+void
+dtrace_vtime_enable(void) {}
+
+void
+dtrace_vtime_disable(void) {}
+
+#else /* else ! CONFIG_DTRACE */
+
+#include <sys/types.h>
+#include <mach/vm_types.h>
+#include <mach/kmod.h>
+
+/*
+ * This exists to prevent build errors when dtrace is unconfigured.
+ */
+
+kern_return_t _dtrace_register_anon_DOF(char *, unsigned char *, uint32_t);
+
+kern_return_t _dtrace_register_anon_DOF(char *arg1, unsigned char *arg2, uint32_t arg3) {
+#pragma unused(arg1, arg2, arg3)
+
+        return KERN_FAILURE;
+}
+
+#endif /* CONFIG_DTRACE */
diff --git a/bsd/dev/dtrace/dtrace_ptss.c b/bsd/dev/dtrace/dtrace_ptss.c
new file mode 100644
index 000000000..8e2ec272e
--- /dev/null
+++ b/bsd/dev/dtrace/dtrace_ptss.c
@@ -0,0 +1,309 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/types.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/systm.h>
+#include <sys/user.h>
+#include <sys/dtrace_ptss.h>
+
+#include <mach/vm_param.h>
+#include <mach/mach_vm.h>
+
+#include <kern/task.h>
+
+#include <vm/vm_map.h>
+
+/*
+ * This function requires the sprlock to be held
+ *
+ * In general, it will not block. If it needs to allocate a new
+ * page of memory, the underlying kernel _MALLOC may block.
+ */
+struct dtrace_ptss_page_entry*
+dtrace_ptss_claim_entry_locked(struct proc* p) {
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+
+	struct dtrace_ptss_page_entry* entry = NULL;
+
+	while (TRUE) {
+		struct dtrace_ptss_page_entry* temp = p->p_dtrace_ptss_free_list;
+
+		if (temp == NULL) {
+			// Nothing on the free list. Allocate a new page, its okay if multiple threads race here.
+			struct dtrace_ptss_page* page = dtrace_ptss_allocate_page(p);
+
+			// Make sure we actually got a page
+			if (page == NULL)
+				return NULL;
+
+			// Add the page to the page list
+			page->next = p->p_dtrace_ptss_pages;
+			p->p_dtrace_ptss_pages = page;
+
+			// CAS the entries onto the free list.
+			do {
+				page->entries[DTRACE_PTSS_ENTRIES_PER_PAGE-1].next = p->p_dtrace_ptss_free_list;
+			} while (!OSCompareAndSwap((UInt32)page->entries[DTRACE_PTSS_ENTRIES_PER_PAGE-1].next,
+						   (UInt32)&page->entries[0],
+						   (volatile UInt32 *)&p->p_dtrace_ptss_free_list));
+				 
+			// Now that we've added to the free list, try again.
+			continue;
+		}
+
+		// Claim temp
+		if (!OSCompareAndSwap((UInt32)temp, (UInt32)temp->next, (volatile UInt32 *)&p->p_dtrace_ptss_free_list))
+			continue;
+
+		// At this point, we own temp.
+		entry = temp;
+
+		break;
+	}
+
+	return entry;
+}
+
+/*
+ * This function does not require any locks to be held on entry.
+ */
+struct dtrace_ptss_page_entry*
+dtrace_ptss_claim_entry(struct proc* p) {
+	// Verify no locks held on entry
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED);
+	lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
+
+	struct dtrace_ptss_page_entry* entry = NULL;
+
+	while (TRUE) {
+		struct dtrace_ptss_page_entry* temp = p->p_dtrace_ptss_free_list;
+
+		if (temp == NULL) {
+			lck_mtx_lock(&p->p_dtrace_sprlock);
+			temp = dtrace_ptss_claim_entry_locked(p);
+			lck_mtx_unlock(&p->p_dtrace_sprlock);
+			return temp;
+		}
+
+		// Claim temp
+		if (!OSCompareAndSwap((UInt32)temp, (UInt32)temp->next, (volatile UInt32 *)&p->p_dtrace_ptss_free_list))
+			continue;
+
+		// At this point, we own temp.
+		entry = temp;
+
+		break;
+	}
+
+	return entry;
+}
+
+/*
+ * This function does not require any locks to be held on entry.
+ */
+void
+dtrace_ptss_release_entry(struct proc* p, struct dtrace_ptss_page_entry* e) {
+	if (p && e) {
+		do {
+			e->next = p->p_dtrace_ptss_free_list;
+		} while (!OSCompareAndSwap((UInt32)e->next, (UInt32)e, (volatile UInt32 *)&p->p_dtrace_ptss_free_list));
+	}
+}
+
+/*
+ * This function allocates a new page in the target process's address space.
+ * 
+ * It returns a dtrace_ptss_page that has its entries chained, with the last
+ * entries next field set to NULL. It does not add the page or the entries to
+ * the process's page/entry lists.
+ *
+ * This function does not require that any locks be held when it is invoked.
+ */
+struct dtrace_ptss_page*
+dtrace_ptss_allocate_page(struct proc* p)
+{
+	// Allocate the kernel side data
+	struct dtrace_ptss_page* ptss_page = _MALLOC(sizeof(struct dtrace_ptss_page), M_TEMP, M_ZERO | M_WAITOK);
+	if (ptss_page == NULL)
+		return NULL;
+
+	// Now allocate a page in user space and set its protections to allow execute.
+	task_t task = p->task;
+	vm_map_t map = get_task_map_reference(task);
+
+	mach_vm_address_t addr = 0LL;
+	mach_vm_size_t size = PAGE_SIZE; // We need some way to assert that this matches vm_map_round_page() !!!
+
+	kern_return_t kr = mach_vm_allocate(map, &addr, size, VM_FLAGS_ANYWHERE);
+	if (kr != KERN_SUCCESS) {
+		goto err;
+	}
+
+	kr = mach_vm_protect(map, addr, size, 0, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
+	if (kr != KERN_SUCCESS) {
+		mach_vm_deallocate(map, addr, size);
+		goto err;
+	}	
+
+	// Chain the page entries.
+	int i;
+	for (i=0; i<DTRACE_PTSS_ENTRIES_PER_PAGE; i++) {
+		ptss_page->entries[i].addr = addr + (i * DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD);
+		ptss_page->entries[i].next = &ptss_page->entries[i+1];
+	}
+
+	// The last entry should point to NULL
+	ptss_page->entries[DTRACE_PTSS_ENTRIES_PER_PAGE-1].next = NULL;
+
+	vm_map_deallocate(map);
+
+	return ptss_page;
+
+err:
+	_FREE(ptss_page, M_TEMP);
+
+	vm_map_deallocate(map);
+
+	return NULL;
+}
+
+/*
+ * This function frees an existing page in the target process's address space.
+ * 
+ * It does not alter any of the process's page/entry lists.
+ *
+ * TODO: Inline in dtrace_ptrace_exec_exit?
+ */
+void
+dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page)
+{
+	// Grab the task and get a reference to its vm_map
+	task_t task = p->task;
+	vm_map_t map = get_task_map_reference(task);
+
+	mach_vm_address_t addr = ptss_page->entries[0].addr;
+	mach_vm_size_t size = PAGE_SIZE; // We need some way to assert that this matches vm_map_round_page() !!!
+
+	// Silent failures, no point in checking return code.
+	mach_vm_deallocate(map, addr, size);
+
+	vm_map_deallocate(map);
+}
+
+/*
+ * This function assumes that the target process has been
+ * suspended, and the proc_lock & sprlock is held 
+ */
+void
+dtrace_ptss_enable(struct proc* p) {
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED);
+
+	struct uthread* uth;
+	/*
+	 * XXX There has been a concern raised about holding the proc_lock
+	 * while calling dtrace_ptss_claim_entry(), due to the fact
+	 * that dtrace_ptss_claim_entry() can potentially malloc.
+	 */
+	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
+		uth->t_dtrace_scratch = dtrace_ptss_claim_entry_locked(p);
+	}
+}
+
+/*
+ * This function is not thread safe.
+ *
+ * It assumes the sprlock is held, and the proc_lock is not.
+ */
+void
+dtrace_ptss_exec_exit(struct proc* p) {
+	/*
+	 * Should hold sprlock to touch the pages list. Must not
+	 * hold the proc lock to avoid deadlock.
+	 */
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
+
+	p->p_dtrace_ptss_free_list = NULL;
+
+	struct dtrace_ptss_page* temp = p->p_dtrace_ptss_pages;
+	p->p_dtrace_ptss_pages = NULL;
+
+	while (temp != NULL) {
+		struct dtrace_ptss_page* next = temp->next;
+		
+		// Do we need to specifically mach_vm_deallocate the user pages?
+		// This can be called when the process is exiting, I believe the proc's
+		// vm_map_t may already be toast.
+		
+		// Must be certain to free the kernel memory!
+		_FREE(temp, M_TEMP);
+		temp = next;
+	}
+}
+
+/*
+ * This function is not thread safe. It is not used for vfork.
+ *
+ * The child proc ptss fields are initialized to NULL at fork time.
+ * Pages allocated in the parent are copied as part of the vm_map copy, though.
+ * We need to deallocate those pages.
+ *
+ * Parent and child sprlock should be held, and proc_lock must NOT be held.
+ */
+void
+dtrace_ptss_fork(struct proc* parent, struct proc* child) {
+	// The child should not have any pages/entries allocated at this point.
+	// ASSERT(child->p_dtrace_ptss_pages == NULL);
+	// ASSERT(child->p_dtrace_ptss_free_list == NULL);
+
+	/*
+	 * The parent's sprlock should be held, to protect its pages list
+	 * from changing while the child references it. The child's sprlock
+	 * must also be held, because we are modifying its pages list.
+	 * Finally, to prevent a deadlock with the fasttrap cleanup code,
+	 * neither the parent or child proc_lock should be held.
+	 */
+	lck_mtx_assert(&parent->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&parent->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
+	lck_mtx_assert(&child->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&child->p_mlock, LCK_MTX_ASSERT_NOTOWNED);
+
+	// Get page list from *PARENT*
+	struct dtrace_ptss_page* temp = parent->p_dtrace_ptss_pages;
+
+	while (temp != NULL) {		
+		// Freeing the page in the *CHILD*
+		dtrace_ptss_free_page(child, temp);
+
+		// Do not free the kernel memory, it belong to the parent.
+		temp = temp->next;
+	}
+}
diff --git a/bsd/dev/dtrace/dtrace_subr.c b/bsd/dev/dtrace/dtrace_subr.c
new file mode 100644
index 000000000..2a2a4b3de
--- /dev/null
+++ b/bsd/dev/dtrace/dtrace_subr.c
@@ -0,0 +1,156 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)dtrace_subr.c	1.7	06/04/24 SMI"
+ */
+
+#include <stdarg.h>
+#include <string.h>
+#include <sys/malloc.h>
+#include <sys/time.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <kern/debug.h>
+
+#if defined(__APPLE__)
+#define proc_t struct proc
+#endif
+
+/* Copied from an arch specific dtrace_subr.c. */
+int (*dtrace_fasttrap_probe_ptr)(struct regs *);
+
+/*
+ * Following DTrace hooks are taken from Solaris' dtrace_subr.c
+ * They're assigned in dtrace.c but Darwin never calls them.
+ */
+void (*dtrace_cpu_init)(processorid_t);
+void (*dtrace_modload)(struct modctl *);
+void (*dtrace_modunload)(struct modctl *);
+#if defined(__APPLE__)
+void (*dtrace_helpers_cleanup)(proc_t *);
+#endif
+void (*dtrace_helpers_fork)(proc_t *, proc_t *);
+void (*dtrace_cpustart_init)(void);
+void (*dtrace_cpustart_fini)(void);
+
+void (*dtrace_kreloc_init)(void);
+void (*dtrace_kreloc_fini)(void);
+
+void (*dtrace_debugger_init)(void);
+void (*dtrace_debugger_fini)(void);
+
+dtrace_vtime_state_t dtrace_vtime_active = 0;
+dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1;
+
+void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
+void (*dtrace_fasttrap_exec_ptr)(proc_t *);
+void (*dtrace_fasttrap_exit_ptr)(proc_t *);
+
+/*
+ * This function is called by cfork() in the event that it appears that
+ * there may be dtrace tracepoints active in the parent process's address
+ * space. This first confirms the existence of dtrace tracepoints in the
+ * parent process and calls into the fasttrap module to remove the
+ * corresponding tracepoints from the child. By knowing that there are
+ * existing tracepoints, and ensuring they can't be removed, we can rely
+ * on the fasttrap module remaining loaded.
+ */
+void
+dtrace_fasttrap_fork(proc_t *p, proc_t *cp)
+{
+#if !defined(__APPLE__)
+	ASSERT(p->p_proc_flag & P_PR_LOCK);
+	ASSERT(p->p_dtrace_count > 0);
+#endif /* __APPLE__ */
+
+	if (dtrace_fasttrap_fork_ptr) {
+		(*dtrace_fasttrap_fork_ptr)(p, cp);
+	}
+}
+
+typedef struct dtrace_invop_hdlr {
+	int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t);
+	struct dtrace_invop_hdlr *dtih_next;
+} dtrace_invop_hdlr_t;
+
+dtrace_invop_hdlr_t *dtrace_invop_hdlr;
+
+int
+dtrace_invop(uintptr_t, uintptr_t *, uintptr_t);
+
+int
+dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
+{
+	dtrace_invop_hdlr_t *hdlr;
+	int rval;
+
+	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) {
+		if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0)
+			return (rval);
+	}
+
+	return (0);
+}
+
+void
+dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
+{
+	dtrace_invop_hdlr_t *hdlr;
+
+	hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
+	hdlr->dtih_func = func;
+	hdlr->dtih_next = dtrace_invop_hdlr;
+	dtrace_invop_hdlr = hdlr;
+}
+
+void
+dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
+{
+	dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL;
+
+	for (;;) {
+		if (hdlr == NULL)
+			panic("attempt to remove non-existent invop handler");
+
+		if (hdlr->dtih_func == func)
+			break;
+
+		prev = hdlr;
+		hdlr = hdlr->dtih_next;
+	}
+
+	if (prev == NULL) {
+		ASSERT(dtrace_invop_hdlr == hdlr);
+		dtrace_invop_hdlr = hdlr->dtih_next;
+	} else {
+		ASSERT(dtrace_invop_hdlr != hdlr);
+		prev->dtih_next = hdlr->dtih_next;
+	}
+
+	kmem_free(hdlr, sizeof (dtrace_invop_hdlr_t));
+}
+
diff --git a/bsd/dev/dtrace/fasttrap.c b/bsd/dev/dtrace/fasttrap.c
new file mode 100644
index 000000000..b0828e6dc
--- /dev/null
+++ b/bsd/dev/dtrace/fasttrap.c
@@ -0,0 +1,2579 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)fasttrap.c	1.21	06/06/12 SMI"
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/kauth.h>
+
+#include <sys/fasttrap.h>
+#include <sys/fasttrap_impl.h>
+#include <sys/fasttrap_isa.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/proc.h>
+
+#include <miscfs/devfs/devfs.h>
+#include <sys/proc_internal.h>
+#include <sys/dtrace_glue.h>
+#include <sys/dtrace_ptss.h>
+
+#include <kern/zalloc.h>
+
+#define proc_t struct proc
+
+/*
+ * User-Land Trap-Based Tracing
+ * ----------------------------
+ *
+ * The fasttrap provider allows DTrace consumers to instrument any user-level
+ * instruction to gather data; this includes probes with semantic
+ * signifigance like entry and return as well as simple offsets into the
+ * function. While the specific techniques used are very ISA specific, the
+ * methodology is generalizable to any architecture.
+ *
+ *
+ * The General Methodology
+ * -----------------------
+ *
+ * With the primary goal of tracing every user-land instruction and the
+ * limitation that we can't trust user space so don't want to rely on much
+ * information there, we begin by replacing the instructions we want to trace
+ * with trap instructions. Each instruction we overwrite is saved into a hash
+ * table keyed by process ID and pc address. When we enter the kernel due to
+ * this trap instruction, we need the effects of the replaced instruction to
+ * appear to have occurred before we proceed with the user thread's
+ * execution.
+ *
+ * Each user level thread is represented by a ulwp_t structure which is
+ * always easily accessible through a register. The most basic way to produce
+ * the effects of the instruction we replaced is to copy that instruction out
+ * to a bit of scratch space reserved in the user thread's ulwp_t structure
+ * (a sort of kernel-private thread local storage), set the PC to that
+ * scratch space and single step. When we reenter the kernel after single
+ * stepping the instruction we must then adjust the PC to point to what would
+ * normally be the next instruction. Of course, special care must be taken
+ * for branches and jumps, but these represent such a small fraction of any
+ * instruction set that writing the code to emulate these in the kernel is
+ * not too difficult.
+ *
+ * Return probes may require several tracepoints to trace every return site,
+ * and, conversely, each tracepoint may activate several probes (the entry
+ * and offset 0 probes, for example). To solve this muliplexing problem,
+ * tracepoints contain lists of probes to activate and probes contain lists
+ * of tracepoints to enable. If a probe is activated, it adds its ID to
+ * existing tracepoints or creates new ones as necessary.
+ *
+ * Most probes are activated _before_ the instruction is executed, but return
+ * probes are activated _after_ the effects of the last instruction of the
+ * function are visible. Return probes must be fired _after_ we have
+ * single-stepped the instruction whereas all other probes are fired
+ * beforehand.
+ *
+ *
+ * Lock Ordering
+ * -------------
+ *
+ * The lock ordering below -- both internally and with respect to the DTrace
+ * framework -- is a little tricky and bears some explanation. Each provider
+ * has a lock (ftp_mtx) that protects its members including reference counts
+ * for enabled probes (ftp_rcount), consumers actively creating probes
+ * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
+ * from being freed. A provider is looked up by taking the bucket lock for the
+ * provider hash table, and is returned with its lock held. The provider lock
+ * may be taken in functions invoked by the DTrace framework, but may not be
+ * held while calling functions in the DTrace framework.
+ *
+ * To ensure consistency over multiple calls to the DTrace framework, the
+ * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
+ * not be taken when holding the provider lock as that would create a cyclic
+ * lock ordering. In situations where one would naturally take the provider
+ * lock and then the creation lock, we instead up a reference count to prevent
+ * the provider from disappearing, drop the provider lock, and acquire the
+ * creation lock.
+ *
+ * Briefly:
+ * 	bucket lock before provider lock
+ *	DTrace before provider lock
+ *	creation lock before DTrace
+ *	never hold the provider lock and creation lock simultaneously
+ */
+
+static dev_info_t *fasttrap_devi;
+static dtrace_meta_provider_id_t fasttrap_meta_id;
+
+static thread_call_t fasttrap_timeout;
+static lck_mtx_t fasttrap_cleanup_mtx;
+static uint_t fasttrap_cleanup_work;
+
+/*
+ * Generation count on modifications to the global tracepoint lookup table.
+ */
+static volatile uint64_t fasttrap_mod_gen;
+
+#if !defined(__APPLE__)
+/*
+ * When the fasttrap provider is loaded, fasttrap_max is set to either
+ * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
+ * fasttrap.conf file. Each time a probe is created, fasttrap_total is
+ * incremented by the number of tracepoints that may be associated with that
+ * probe; fasttrap_total is capped at fasttrap_max.
+ */
+#define	FASTTRAP_MAX_DEFAULT		2500000
+#endif
+
+static uint32_t fasttrap_max;
+static uint32_t fasttrap_total;
+
+
+#define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
+#define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
+#define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
+
+fasttrap_hash_t			fasttrap_tpoints;
+static fasttrap_hash_t		fasttrap_provs;
+static fasttrap_hash_t		fasttrap_procs;
+
+static uint64_t			fasttrap_pid_count;	/* pid ref count */
+static lck_mtx_t       		fasttrap_count_mtx;	/* lock on ref count */
+
+#define	FASTTRAP_ENABLE_FAIL	1
+#define	FASTTRAP_ENABLE_PARTIAL	2
+
+static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
+static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
+
+#if defined(__APPLE__)
+static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, fasttrap_provider_type_t, const char *,
+    const dtrace_pattr_t *);
+#endif
+static void fasttrap_provider_retire(pid_t, const char *, int);
+static void fasttrap_provider_free(fasttrap_provider_t *);
+
+static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
+static void fasttrap_proc_release(fasttrap_proc_t *);
+
+#define	FASTTRAP_PROVS_INDEX(pid, name) \
+	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
+
+#define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
+
+#if defined(__APPLE__)
+
+/*
+ * To save memory, some common memory allocations are given a
+ * unique zone. In example, dtrace_probe_t is 72 bytes in size,
+ * which means it would fall into the kalloc.128 bucket. With
+ * 20k elements allocated, the space saved is substantial.
+ */
+
+struct zone *fasttrap_tracepoint_t_zone;
+
+/*
+ * fasttrap_probe_t's are variable in size. Some quick profiling has shown
+ * that the sweet spot for reducing memory footprint is covering the first
+ * three sizes. Everything larger goes into the common pool.
+ */
+#define FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS 4
+
+struct zone *fasttrap_probe_t_zones[FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS];
+
+static const char *fasttrap_probe_t_zone_names[FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS] = {
+	"",
+	"dtrace.fasttrap_probe_t[1]",
+	"dtrace.fasttrap_probe_t[2]",
+	"dtrace.fasttrap_probe_t[3]"
+};
+
+/*
+ * We have to manage locks explicitly
+ */
+lck_grp_t*			fasttrap_lck_grp;
+lck_grp_attr_t*			fasttrap_lck_grp_attr;
+lck_attr_t*			fasttrap_lck_attr;
+#endif
+
+static int
+fasttrap_highbit(ulong_t i)
+{
+	int h = 1;
+
+	if (i == 0)
+		return (0);
+#ifdef _LP64
+	if (i & 0xffffffff00000000ul) {
+		h += 32; i >>= 32;
+	}
+#endif
+	if (i & 0xffff0000) {
+		h += 16; i >>= 16;
+	}
+	if (i & 0xff00) {
+		h += 8; i >>= 8;
+	}
+	if (i & 0xf0) {
+		h += 4; i >>= 4;
+	}
+	if (i & 0xc) {
+		h += 2; i >>= 2;
+	}
+	if (i & 0x2) {
+		h += 1;
+	}
+	return (h);
+}
+
+static uint_t
+fasttrap_hash_str(const char *p)
+{
+	unsigned int g;
+	uint_t hval = 0;
+
+	while (*p) {
+		hval = (hval << 4) + *p++;
+		if ((g = (hval & 0xf0000000)) != 0)
+			hval ^= g >> 24;
+		hval &= ~g;
+	}
+	return (hval);
+}
+
+/*
+ * FIXME - needs implementation
+ */
+void
+fasttrap_sigtrap(proc_t *p, uthread_t t, user_addr_t pc)
+{
+#pragma unused(p, t, pc)
+
+#if 0
+	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
+
+	sqp->sq_info.si_signo = SIGTRAP;
+	sqp->sq_info.si_code = TRAP_DTRACE;
+	sqp->sq_info.si_addr = (caddr_t)pc;
+
+	mutex_enter(&p->p_lock);
+	sigaddqa(p, t, sqp);
+	mutex_exit(&p->p_lock);
+
+	if (t != NULL)
+		aston(t);
+#endif
+
+	printf("fasttrap_sigtrap called with no implementation.\n");
+}
+
+/*
+ * This function ensures that no threads are actively using the memory
+ * associated with probes that were formerly live.
+ */
+static void
+fasttrap_mod_barrier(uint64_t gen)
+{
+	unsigned int i;
+
+	if (gen < fasttrap_mod_gen)
+		return;
+
+	fasttrap_mod_gen++;
+
+	for (i = 0; i < NCPU; i++) {
+		lck_mtx_lock(&cpu_core[i].cpuc_pid_lock);
+		lck_mtx_unlock(&cpu_core[i].cpuc_pid_lock);
+	}
+}
+
+/*
+ * This is the timeout's callback for cleaning up the providers and their
+ * probes.
+ */
+/*ARGSUSED*/
+static void
+fasttrap_pid_cleanup_cb(void *ignored, void* ignored2)
+{
+#pragma unused(ignored, ignored2)
+	fasttrap_provider_t **fpp, *fp;
+	fasttrap_bucket_t *bucket;
+	dtrace_provider_id_t provid;
+	unsigned int i, later = 0;
+
+	static volatile int in = 0;
+	ASSERT(in == 0);
+	in = 1;
+
+	lck_mtx_lock(&fasttrap_cleanup_mtx);
+	while (fasttrap_cleanup_work) {
+		fasttrap_cleanup_work = 0;
+		lck_mtx_unlock(&fasttrap_cleanup_mtx);
+
+		later = 0;
+
+		/*
+		 * Iterate over all the providers trying to remove the marked
+		 * ones. If a provider is marked but not retired, we just
+		 * have to take a crack at removing it -- it's no big deal if
+		 * we can't.
+		 */
+		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
+			bucket = &fasttrap_provs.fth_table[i];
+			lck_mtx_lock(&bucket->ftb_mtx);
+			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
+
+			while ((fp = *fpp) != NULL) {
+				if (!fp->ftp_marked) {
+					fpp = &fp->ftp_next;
+					continue;
+				}
+
+				lck_mtx_lock(&fp->ftp_mtx);
+
+				/*
+				 * If this provider has consumers actively
+				 * creating probes (ftp_ccount) or is a USDT
+				 * provider (ftp_mcount), we can't unregister
+				 * or even condense.
+				 */
+				if (fp->ftp_ccount != 0 ||
+				    fp->ftp_mcount != 0) {
+					fp->ftp_marked = 0;
+					lck_mtx_unlock(&fp->ftp_mtx);
+					continue;
+				}
+
+				if (!fp->ftp_retired || fp->ftp_rcount != 0)
+					fp->ftp_marked = 0;
+
+				lck_mtx_unlock(&fp->ftp_mtx);
+
+				/*
+				 * If we successfully unregister this
+				 * provider we can remove it from the hash
+				 * chain and free the memory. If our attempt
+				 * to unregister fails and this is a retired
+				 * provider, increment our flag to try again
+				 * pretty soon. If we've consumed more than
+				 * half of our total permitted number of
+				 * probes call dtrace_condense() to try to
+				 * clean out the unenabled probes.
+				 */
+				provid = fp->ftp_provid;
+				if (dtrace_unregister(provid) != 0) {
+					if (fasttrap_total > fasttrap_max / 2)
+						(void) dtrace_condense(provid);
+					later += fp->ftp_marked;
+					fpp = &fp->ftp_next;
+				} else {
+					*fpp = fp->ftp_next;
+					fasttrap_provider_free(fp);
+				}
+			}
+			lck_mtx_unlock(&bucket->ftb_mtx);
+		}
+
+		lck_mtx_lock(&fasttrap_cleanup_mtx);
+	}
+
+	ASSERT(fasttrap_timeout != 0);
+
+	/*
+	 * APPLE NOTE: You must hold the fasttrap_cleanup_mtx to do this!
+	 */
+	if (fasttrap_timeout != (thread_call_t)1)
+		thread_call_free(fasttrap_timeout);
+
+	/*
+	 * If we were unable to remove a retired provider, try again after
+	 * a second. This situation can occur in certain circumstances where
+	 * providers cannot be unregistered even though they have no probes
+	 * enabled because of an execution of dtrace -l or something similar.
+	 * If the timeout has been disabled (set to 1 because we're trying
+	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
+	 * get a chance to do that work if and when the timeout is reenabled
+	 * (if detach fails).
+	 */
+	if (later > 0 && fasttrap_timeout != (thread_call_t)1)
+		/* The time value passed to dtrace_timeout is in nanos */
+		fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / SEC);
+	else if (later > 0)
+		fasttrap_cleanup_work = 1;
+	else
+		fasttrap_timeout = 0;
+
+	lck_mtx_unlock(&fasttrap_cleanup_mtx);
+	in = 0;
+}
+
+/*
+ * Activates the asynchronous cleanup mechanism.
+ */
+static void
+fasttrap_pid_cleanup(void)
+{
+	lck_mtx_lock(&fasttrap_cleanup_mtx);
+	fasttrap_cleanup_work = 1;
+	if (fasttrap_timeout == 0)
+		fasttrap_timeout = dtrace_timeout(&fasttrap_pid_cleanup_cb, NULL, NANOSEC / MILLISEC);
+	lck_mtx_unlock(&fasttrap_cleanup_mtx);
+}
+
+/*
+ * This is called from cfork() via dtrace_fasttrap_fork(). The child
+ * process's address space is a (roughly) a copy of the parent process's so
+ * we have to remove all the instrumentation we had previously enabled in the
+ * parent.
+ */
+static void
+fasttrap_fork(proc_t *p, proc_t *cp)
+{
+	pid_t ppid = p->p_pid;
+	unsigned int i;
+
+	ASSERT(current_proc() == p);
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_OWNED);
+	ASSERT(p->p_dtrace_count > 0);
+	ASSERT(cp->p_dtrace_count == 0);
+
+	/*
+	 * This would be simpler and faster if we maintained per-process
+	 * hash tables of enabled tracepoints. It could, however, potentially
+	 * slow down execution of a tracepoint since we'd need to go
+	 * through two levels of indirection. In the future, we should
+	 * consider either maintaining per-process ancillary lists of
+	 * enabled tracepoints or hanging a pointer to a per-process hash
+	 * table of enabled tracepoints off the proc structure.
+	 */
+
+	/*
+	 * We don't have to worry about the child process disappearing
+	 * because we're in fork().
+	 */
+	if (cp != sprlock(cp->p_pid)) {
+		printf("fasttrap_fork: sprlock(%d) returned a differt proc\n", cp->p_pid);
+		return;
+	}
+	proc_unlock(cp);
+
+	/*
+	 * Iterate over every tracepoint looking for ones that belong to the
+	 * parent process, and remove each from the child process.
+	 */
+	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
+		fasttrap_tracepoint_t *tp;
+		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
+
+		lck_mtx_lock(&bucket->ftb_mtx);
+		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+			if (tp->ftt_pid == ppid &&
+			    !tp->ftt_proc->ftpc_defunct) {
+				fasttrap_tracepoint_remove(cp, tp);
+			}
+		}
+		lck_mtx_unlock(&bucket->ftb_mtx);
+	}
+
+	/*
+	 * Free any ptss pages/entries in the child.
+	 */
+	dtrace_ptss_fork(p, cp);
+
+	proc_lock(cp);
+	sprunlock(cp);
+}
+
+/*
+ * This is called from proc_exit() or from exec_common() if p_dtrace_probes
+ * is set on the proc structure to indicate that there is a pid provider
+ * associated with this process.
+ */
+static void
+fasttrap_exec_exit(proc_t *p)
+{
+	ASSERT(p == current_proc());
+	lck_mtx_assert(&p->p_mlock, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(&p->p_dtrace_sprlock, LCK_MTX_ASSERT_NOTOWNED);
+
+
+	/* APPLE NOTE: Okay, the locking here is really odd and needs some
+	 * explaining. This method is always called with the proc_lock held.
+	 * We must drop the proc_lock before calling fasttrap_provider_retire
+	 * to avoid a deadlock when it takes the bucket lock.
+	 * 
+	 * Next, the dtrace_ptss_exec_exit function requires the sprlock
+	 * be held, but not the proc_lock. 
+	 *
+	 * Finally, we must re-acquire the proc_lock
+	 */
+	proc_unlock(p);
+
+	/*
+	 * We clean up the pid provider for this process here; user-land
+	 * static probes are handled by the meta-provider remove entry point.
+	 */
+	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
+#if defined(__APPLE__)
+	/*
+	 * We also need to remove any aliased providers.
+	 * XXX optimization: track which provider types are instantiated
+	 * and only retire as needed.
+	 */
+	fasttrap_provider_retire(p->p_pid, FASTTRAP_OBJC_NAME, 0);
+	fasttrap_provider_retire(p->p_pid, FASTTRAP_ONESHOT_NAME, 0);
+#endif /* __APPLE__ */
+
+	/*
+	 * This should be called after it is no longer possible for a user
+	 * thread to execute (potentially dtrace instrumented) instructions.
+	 */
+	lck_mtx_lock(&p->p_dtrace_sprlock);
+	dtrace_ptss_exec_exit(p);
+	lck_mtx_unlock(&p->p_dtrace_sprlock);
+
+	proc_lock(p);
+}
+
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+#pragma unused(arg, desc)
+	/*
+	 * There are no "default" pid probes.
+	 */
+}
+
+static int
+fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
+{
+	fasttrap_tracepoint_t *tp, *new_tp = NULL;
+	fasttrap_bucket_t *bucket;
+	fasttrap_id_t *id;
+	pid_t pid;
+	user_addr_t pc;
+
+	ASSERT(index < probe->ftp_ntps);
+
+	pid = probe->ftp_pid;
+	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
+	id = &probe->ftp_tps[index].fit_id;
+
+	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
+
+	//ASSERT(!(p->p_flag & SVFORK));
+
+	/*
+	 * Before we make any modifications, make sure we've imposed a barrier
+	 * on the generation in which this probe was last modified.
+	 */
+	fasttrap_mod_barrier(probe->ftp_gen);
+
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	/*
+	 * If the tracepoint has already been enabled, just add our id to the
+	 * list of interested probes. This may be our second time through
+	 * this path in which case we'll have constructed the tracepoint we'd
+	 * like to install. If we can't find a match, and have an allocated
+	 * tracepoint ready to go, enable that one now.
+	 *
+	 * A tracepoint whose process is defunct is also considered defunct.
+	 */
+again:
+	lck_mtx_lock(&bucket->ftb_mtx);
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
+		    tp->ftt_proc->ftpc_defunct)
+			continue;
+
+		/*
+		 * Now that we've found a matching tracepoint, it would be
+		 * a decent idea to confirm that the tracepoint is still
+		 * enabled and the trap instruction hasn't been overwritten.
+		 * Since this is a little hairy, we'll punt for now.
+		 */
+
+		/*
+		 * This can't be the first interested probe. We don't have
+		 * to worry about another thread being in the midst of
+		 * deleting this tracepoint (which would be the only valid
+		 * reason for a tracepoint to have no interested probes)
+		 * since we're holding P_PR_LOCK for this process.
+		 */
+		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
+
+		switch (id->fti_ptype) {
+		case DTFTP_ENTRY:
+		case DTFTP_OFFSETS:
+		case DTFTP_IS_ENABLED:
+			id->fti_next = tp->ftt_ids;
+			dtrace_membar_producer();
+			tp->ftt_ids = id;
+			dtrace_membar_producer();
+			break;
+
+		case DTFTP_RETURN:
+		case DTFTP_POST_OFFSETS:
+			id->fti_next = tp->ftt_retids;
+			dtrace_membar_producer();
+			tp->ftt_retids = id;
+			dtrace_membar_producer();
+			break;
+
+		default:
+			ASSERT(0);
+		}
+
+		lck_mtx_unlock(&bucket->ftb_mtx);
+
+		if (new_tp != NULL) {
+			new_tp->ftt_ids = NULL;
+			new_tp->ftt_retids = NULL;
+		}
+
+		return (0);
+	}
+
+	/*
+	 * If we have a good tracepoint ready to go, install it now while
+	 * we have the lock held and no one can screw with us.
+	 */
+	if (new_tp != NULL) {
+		int rc = 0;
+
+		new_tp->ftt_next = bucket->ftb_data;
+		dtrace_membar_producer();
+		bucket->ftb_data = new_tp;
+		dtrace_membar_producer();
+		lck_mtx_unlock(&bucket->ftb_mtx);
+
+		/*
+		 * Activate the tracepoint in the ISA-specific manner.
+		 * If this fails, we need to report the failure, but
+		 * indicate that this tracepoint must still be disabled
+		 * by calling fasttrap_tracepoint_disable().
+		 */
+		if (fasttrap_tracepoint_install(p, new_tp) != 0)
+			rc = FASTTRAP_ENABLE_PARTIAL;
+
+		/*
+		 * Increment the count of the number of tracepoints active in
+		 * the victim process.
+		 */
+		//ASSERT(p->p_proc_flag & P_PR_LOCK);
+		p->p_dtrace_count++;
+
+		return (rc);
+	}
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	/*
+	 * Initialize the tracepoint that's been preallocated with the probe.
+	 */
+	new_tp = probe->ftp_tps[index].fit_tp;
+
+	ASSERT(new_tp->ftt_pid == pid);
+	ASSERT(new_tp->ftt_pc == pc);
+	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
+	ASSERT(new_tp->ftt_ids == NULL);
+	ASSERT(new_tp->ftt_retids == NULL);
+
+	switch (id->fti_ptype) {
+	case DTFTP_ENTRY:
+	case DTFTP_OFFSETS:
+	case DTFTP_IS_ENABLED:
+		id->fti_next = NULL;
+		new_tp->ftt_ids = id;
+		break;
+
+	case DTFTP_RETURN:
+	case DTFTP_POST_OFFSETS:
+		id->fti_next = NULL;
+		new_tp->ftt_retids = id;
+		break;
+
+	default:
+		ASSERT(0);
+	}
+
+	/*
+	 * If the ISA-dependent initialization goes to plan, go back to the
+	 * beginning and try to install this freshly made tracepoint.
+	 */
+	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
+		goto again;
+
+	new_tp->ftt_ids = NULL;
+	new_tp->ftt_retids = NULL;
+
+	return (FASTTRAP_ENABLE_FAIL);
+}
+
+static void
+fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
+{
+	fasttrap_bucket_t *bucket;
+	fasttrap_provider_t *provider = probe->ftp_prov;
+	fasttrap_tracepoint_t **pp, *tp;
+	fasttrap_id_t *id, **idp;
+	pid_t pid;
+	user_addr_t pc;
+
+	ASSERT(index < probe->ftp_ntps);
+
+	pid = probe->ftp_pid;
+	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
+	id = &probe->ftp_tps[index].fit_id;
+
+	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
+
+	/*
+	 * Find the tracepoint and make sure that our id is one of the
+	 * ones registered with it.
+	 */
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+	lck_mtx_lock(&bucket->ftb_mtx);
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
+		    tp->ftt_proc == provider->ftp_proc)
+			break;
+	}
+
+	/*
+	 * If we somehow lost this tracepoint, we're in a world of hurt.
+	 */
+	ASSERT(tp != NULL);
+
+	switch (id->fti_ptype) {
+		case DTFTP_ENTRY:
+		case DTFTP_OFFSETS:
+		case DTFTP_IS_ENABLED:
+			ASSERT(tp->ftt_ids != NULL);
+			idp = &tp->ftt_ids;
+			break;
+			
+		case DTFTP_RETURN:
+		case DTFTP_POST_OFFSETS:
+			ASSERT(tp->ftt_retids != NULL);
+			idp = &tp->ftt_retids;
+			break;
+			
+		default:
+			/* Fix compiler warning... */
+			idp = NULL;
+			ASSERT(0);
+	}
+
+	while ((*idp)->fti_probe != probe) {
+		idp = &(*idp)->fti_next;
+		ASSERT(*idp != NULL);
+	}
+
+	id = *idp;
+	*idp = id->fti_next;
+	dtrace_membar_producer();
+
+	ASSERT(id->fti_probe == probe);
+
+	/*
+	 * If there are other registered enablings of this tracepoint, we're
+	 * all done, but if this was the last probe assocated with this
+	 * this tracepoint, we need to remove and free it.
+	 */
+	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
+
+		/*
+		 * If the current probe's tracepoint is in use, swap it
+		 * for an unused tracepoint.
+		 */
+		if (tp == probe->ftp_tps[index].fit_tp) {
+			fasttrap_probe_t *tmp_probe;
+			fasttrap_tracepoint_t **tmp_tp;
+			uint_t tmp_index;
+
+			if (tp->ftt_ids != NULL) {
+				tmp_probe = tp->ftt_ids->fti_probe;
+				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
+				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
+			} else {
+				tmp_probe = tp->ftt_retids->fti_probe;
+				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
+				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
+			}
+
+			ASSERT(*tmp_tp != NULL);
+			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
+			ASSERT((*tmp_tp)->ftt_ids == NULL);
+			ASSERT((*tmp_tp)->ftt_retids == NULL);
+
+			probe->ftp_tps[index].fit_tp = *tmp_tp;
+			*tmp_tp = tp;
+
+		}
+
+		lck_mtx_unlock(&bucket->ftb_mtx);
+
+		/*
+		 * Tag the modified probe with the generation in which it was
+		 * changed.
+		 */
+		probe->ftp_gen = fasttrap_mod_gen;
+		return;
+	}
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	/*
+	 * We can't safely remove the tracepoint from the set of active
+	 * tracepoints until we've actually removed the fasttrap instruction
+	 * from the process's text. We can, however, operate on this
+	 * tracepoint secure in the knowledge that no other thread is going to
+	 * be looking at it since we hold P_PR_LOCK on the process if it's
+	 * live or we hold the provider lock on the process if it's dead and
+	 * gone.
+	 */
+
+	/*
+	 * We only need to remove the actual instruction if we're looking
+	 * at an existing process
+	 */
+	if (p != NULL) {
+		/*
+		 * If we fail to restore the instruction we need to kill
+		 * this process since it's in a completely unrecoverable
+		 * state.
+		 */
+		if (fasttrap_tracepoint_remove(p, tp) != 0)
+			fasttrap_sigtrap(p, NULL, pc);
+
+		/*
+		 * Decrement the count of the number of tracepoints active
+		 * in the victim process.
+		 */
+		//ASSERT(p->p_proc_flag & P_PR_LOCK);
+		p->p_dtrace_count--;
+	}
+
+	/*
+	 * Remove the probe from the hash table of active tracepoints.
+	 */
+	lck_mtx_lock(&bucket->ftb_mtx);
+	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
+	ASSERT(*pp != NULL);
+	while (*pp != tp) {
+		pp = &(*pp)->ftt_next;
+		ASSERT(*pp != NULL);
+	}
+
+	*pp = tp->ftt_next;
+	dtrace_membar_producer();
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	/*
+	 * Tag the modified probe with the generation in which it was changed.
+	 */
+	probe->ftp_gen = fasttrap_mod_gen;
+}
+
+static void
+fasttrap_enable_callbacks(void)
+{
+	/*
+	 * We don't have to play the rw lock game here because we're
+	 * providing something rather than taking something away --
+	 * we can be sure that no threads have tried to follow this
+	 * function pointer yet.
+	 */
+	lck_mtx_lock(&fasttrap_count_mtx);
+	if (fasttrap_pid_count == 0) {
+		ASSERT(dtrace_pid_probe_ptr == NULL);
+		ASSERT(dtrace_return_probe_ptr == NULL);
+		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
+		dtrace_return_probe_ptr = &fasttrap_return_probe;
+	}
+	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
+	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
+	fasttrap_pid_count++;
+	lck_mtx_unlock(&fasttrap_count_mtx);
+}
+
+static void
+fasttrap_disable_callbacks(void)
+{
+	//ASSERT(MUTEX_HELD(&cpu_lock));
+
+	lck_mtx_lock(&fasttrap_count_mtx);
+	ASSERT(fasttrap_pid_count > 0);
+	fasttrap_pid_count--;
+	if (fasttrap_pid_count == 0) {
+		cpu_t *cur, *cpu = CPU;
+
+		/*
+		 * APPLE NOTE: This loop seems broken, it touches every CPU
+		 * but the one we're actually running on. Need to ask Sun folks
+		 * if that is safe. Scenario is this: We're running on CPU A,
+		 * and lock all but A. Then we get preempted, and start running
+		 * on CPU B. A probe fires on A, and is allowed to enter. BOOM!
+		 */
+		for (cur = cpu->cpu_next; cur != cpu; cur = cur->cpu_next) {
+			lck_rw_lock_exclusive(&cur->cpu_ft_lock);
+			// rw_enter(&cur->cpu_ft_lock, RW_WRITER);
+		}
+
+		dtrace_pid_probe_ptr = NULL;
+		dtrace_return_probe_ptr = NULL;
+
+		for (cur = cpu->cpu_next; cur != cpu; cur = cur->cpu_next) {
+			lck_rw_unlock_exclusive(&cur->cpu_ft_lock);
+			// rw_exit(&cur->cpu_ft_lock);
+		}
+	}
+	lck_mtx_unlock(&fasttrap_count_mtx);
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg, id)
+	fasttrap_probe_t *probe = parg;
+	proc_t *p;
+	int i, rc;
+
+	ASSERT(probe != NULL);
+	ASSERT(!probe->ftp_enabled);
+	ASSERT(id == probe->ftp_id);
+	// ASSERT(MUTEX_HELD(&cpu_lock));
+
+	/*
+	 * Increment the count of enabled probes on this probe's provider;
+	 * the provider can't go away while the probe still exists. We
+	 * must increment this even if we aren't able to properly enable
+	 * this probe.
+	 */
+	lck_mtx_lock(&probe->ftp_prov->ftp_mtx);
+	probe->ftp_prov->ftp_rcount++;
+	lck_mtx_unlock(&probe->ftp_prov->ftp_mtx);
+
+	/*
+	 * If this probe's provider is retired (meaning it was valid in a
+	 * previously exec'ed incarnation of this address space), bail out. The
+	 * provider can't go away while we're in this code path.
+	 */
+	if (probe->ftp_prov->ftp_retired)
+		return;
+
+	/*
+	 * If we can't find the process, it may be that we're in the context of
+	 * a fork in which the traced process is being born and we're copying
+	 * USDT probes. Otherwise, the process is gone so bail.
+	 */
+	if ((p = sprlock(probe->ftp_pid)) == PROC_NULL) {
+#if defined(__APPLE__)
+		/*
+		 * APPLE NOTE: We should never end up here. The Solaris sprlock()
+		 * does not return process's with SIDL set, but we always return
+		 * the child process.
+		 */
+		return;
+#else
+
+		if ((curproc->p_flag & SFORKING) == 0)
+			return;
+
+		lck_mtx_lock(&pidlock);
+		p = prfind(probe->ftp_pid);
+
+		/*
+		 * Confirm that curproc is indeed forking the process in which
+		 * we're trying to enable probes.
+		 */
+		ASSERT(p != NULL);
+		//ASSERT(p->p_parent == curproc);
+		ASSERT(p->p_stat == SIDL);
+
+		lck_mtx_lock(&p->p_lock);
+		lck_mtx_unlock(&pidlock);
+
+		sprlock_proc(p);
+#endif
+	}
+
+	/*
+	 * APPLE NOTE: We do not have an equivalent thread structure to Solaris.
+	 * Solaris uses its ulwp_t struct for scratch space to support the pid provider.
+	 * To mimic this, we allocate on demand scratch space. If this is the first
+	 * time a probe has been enabled in this process, we need to allocate scratch
+	 * space for each already existing thread. Now is a good time to do this, as
+	 * the target process is suspended and the proc_lock is held.
+	 */
+	if (p->p_dtrace_ptss_pages == NULL) {
+		dtrace_ptss_enable(p);
+	}
+
+	// ASSERT(!(p->p_flag & SVFORK));
+	proc_unlock(p);
+
+	/*
+	 * We have to enable the trap entry point before any user threads have
+	 * the chance to execute the trap instruction we're about to place
+	 * in their process's text.
+	 */
+	fasttrap_enable_callbacks();
+
+	/*
+	 * Enable all the tracepoints and add this probe's id to each
+	 * tracepoint's list of active probes.
+	 */
+	for (i = 0; i < (int)probe->ftp_ntps; i++) {
+		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
+			/*
+			 * If enabling the tracepoint failed completely,
+			 * we don't have to disable it; if the failure
+			 * was only partial we must disable it.
+			 */
+			if (rc == FASTTRAP_ENABLE_FAIL)
+				i--;
+			else
+				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
+
+			/*
+			 * Back up and pull out all the tracepoints we've
+			 * created so far for this probe.
+			 */
+			while (i >= 0) {
+				fasttrap_tracepoint_disable(p, probe, i);
+				i--;
+			}
+
+			proc_lock(p);
+			sprunlock(p);
+
+			/*
+			 * Since we're not actually enabling this probe,
+			 * drop our reference on the trap table entry.
+			 */
+			fasttrap_disable_callbacks();
+			return;
+		}
+	}
+
+	proc_lock(p);
+	sprunlock(p);
+
+	probe->ftp_enabled = 1;
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg, id)
+	fasttrap_probe_t *probe = parg;
+	fasttrap_provider_t *provider = probe->ftp_prov;
+	proc_t *p;
+	int i, whack = 0;
+
+	ASSERT(id == probe->ftp_id);
+
+	/*
+	 * We won't be able to acquire a /proc-esque lock on the process
+	 * iff the process is dead and gone. In this case, we rely on the
+	 * provider lock as a point of mutual exclusion to prevent other
+	 * DTrace consumers from disabling this probe.
+	 */
+	if ((p = sprlock(probe->ftp_pid)) != PROC_NULL) {
+		// ASSERT(!(p->p_flag & SVFORK));
+		proc_unlock(p);
+	}
+
+	lck_mtx_lock(&provider->ftp_mtx);
+
+	/*
+	 * Disable all the associated tracepoints (for fully enabled probes).
+	 */
+	if (probe->ftp_enabled) {
+		for (i = 0; i < (int)probe->ftp_ntps; i++) {
+			fasttrap_tracepoint_disable(p, probe, i);
+		}
+	}
+
+	ASSERT(provider->ftp_rcount > 0);
+	provider->ftp_rcount--;
+
+	if (p != NULL) {
+		/*
+		 * Even though we may not be able to remove it entirely, we
+		 * mark this retired provider to get a chance to remove some
+		 * of the associated probes.
+		 */
+		if (provider->ftp_retired && !provider->ftp_marked)
+			whack = provider->ftp_marked = 1;
+		lck_mtx_unlock(&provider->ftp_mtx);
+
+		proc_lock(p);
+		sprunlock(p);
+	} else {
+		/*
+		 * If the process is dead, we're just waiting for the
+		 * last probe to be disabled to be able to free it.
+		 */
+		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
+			whack = provider->ftp_marked = 1;
+		lck_mtx_unlock(&provider->ftp_mtx);
+	}
+
+	if (whack)
+		fasttrap_pid_cleanup();
+
+	if (!probe->ftp_enabled)
+		return;
+
+	probe->ftp_enabled = 0;
+
+	// ASSERT(MUTEX_HELD(&cpu_lock));
+	fasttrap_disable_callbacks();
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
+    dtrace_argdesc_t *desc)
+{
+#pragma unused(arg, id)
+	fasttrap_probe_t *probe = parg;
+	char *str;
+	int i;
+
+	desc->dtargd_native[0] = '\0';
+	desc->dtargd_xlate[0] = '\0';
+
+	if (probe->ftp_prov->ftp_retired != 0 ||
+	    desc->dtargd_ndx >= probe->ftp_nargs) {
+		desc->dtargd_ndx = DTRACE_ARGNONE;
+		return;
+	}
+
+	/*
+	 * We only need to set this member if the argument is remapped.
+	 */
+	if (probe->ftp_argmap != NULL)
+		desc->dtargd_mapping = probe->ftp_argmap[desc->dtargd_ndx];
+
+	str = probe->ftp_ntypes;
+	for (i = 0; i < desc->dtargd_mapping; i++) {
+		str += strlen(str) + 1;
+	}
+
+	(void) strlcpy(desc->dtargd_native, str, sizeof(desc->dtargd_native));
+
+	if (probe->ftp_xtypes == NULL)
+		return;
+
+	str = probe->ftp_xtypes;
+	for (i = 0; i < desc->dtargd_ndx; i++) {
+		str += strlen(str) + 1;
+	}
+
+	(void) strlcpy(desc->dtargd_xlate, str, sizeof(desc->dtargd_xlate));
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg, id)
+	fasttrap_probe_t *probe = parg;
+	unsigned int i;
+
+	ASSERT(probe != NULL);
+	ASSERT(!probe->ftp_enabled);
+	ASSERT(fasttrap_total >= probe->ftp_ntps);
+
+	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
+#if !defined(__APPLE__)
+	size_t size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
+#endif
+
+	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
+		fasttrap_mod_barrier(probe->ftp_gen);
+
+	for (i = 0; i < probe->ftp_ntps; i++) {
+#if !defined(__APPLE__)
+		kmem_free(probe->ftp_tps[i].fit_tp, sizeof (fasttrap_tracepoint_t));
+#else
+		zfree(fasttrap_tracepoint_t_zone, probe->ftp_tps[i].fit_tp);
+#endif
+	}
+
+#if !defined(__APPLE__)
+	kmem_free(probe, size);
+#else
+	if (probe->ftp_ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
+		zfree(fasttrap_probe_t_zones[probe->ftp_ntps], probe);
+	} else {
+		size_t size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
+		kmem_free(probe, size);
+	}
+#endif
+}
+
+
+static const dtrace_pattr_t pid_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+};
+
+static dtrace_pops_t pid_pops = {
+	fasttrap_pid_provide,
+	NULL,
+	fasttrap_pid_enable,
+	fasttrap_pid_disable,
+	NULL,
+	NULL,
+	fasttrap_pid_getargdesc,
+	fasttrap_pid_getarg,
+	NULL,
+	fasttrap_pid_destroy
+};
+
+static dtrace_pops_t usdt_pops = {
+	fasttrap_pid_provide,
+	NULL,
+	fasttrap_pid_enable,
+	fasttrap_pid_disable,
+	NULL,
+	NULL,
+	fasttrap_pid_getargdesc,
+	fasttrap_usdt_getarg,
+	NULL,
+	fasttrap_pid_destroy
+};
+
+static fasttrap_proc_t *
+fasttrap_proc_lookup(pid_t pid)
+{
+	fasttrap_bucket_t *bucket;
+	fasttrap_proc_t *fprc, *new_fprc;
+
+	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
+		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
+			lck_mtx_lock(&fprc->ftpc_mtx);
+			lck_mtx_unlock(&bucket->ftb_mtx);
+			fprc->ftpc_count++;
+			lck_mtx_unlock(&fprc->ftpc_mtx);
+
+			return (fprc);
+		}
+	}
+
+	/*
+	 * Drop the bucket lock so we don't try to perform a sleeping
+	 * allocation under it.
+	 */
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
+	ASSERT(new_fprc != NULL);
+	new_fprc->ftpc_pid = pid;
+	new_fprc->ftpc_count = 1;
+
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	/*
+	 * Take another lap through the list to make sure a proc hasn't
+	 * been created for this pid while we weren't under the bucket lock.
+	 */
+	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
+		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
+			lck_mtx_lock(&fprc->ftpc_mtx);
+			lck_mtx_unlock(&bucket->ftb_mtx);
+			fprc->ftpc_count++;
+			lck_mtx_unlock(&fprc->ftpc_mtx);
+
+			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
+
+			return (fprc);
+		}
+	}
+
+#if defined(__APPLE__)
+	/*
+	 * We have to initialize all locks explicitly
+	 */
+	lck_mtx_init(&new_fprc->ftpc_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+#endif
+
+	new_fprc->ftpc_next = bucket->ftb_data;
+	bucket->ftb_data = new_fprc;
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	return (new_fprc);
+}
+
+static void
+fasttrap_proc_release(fasttrap_proc_t *proc)
+{
+	fasttrap_bucket_t *bucket;
+	fasttrap_proc_t *fprc, **fprcp;
+	pid_t pid = proc->ftpc_pid;
+
+	lck_mtx_lock(&proc->ftpc_mtx);
+
+	ASSERT(proc->ftpc_count != 0);
+
+	if (--proc->ftpc_count != 0) {
+		lck_mtx_unlock(&proc->ftpc_mtx);
+		return;
+	}
+
+	lck_mtx_unlock(&proc->ftpc_mtx);
+
+	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
+	while ((fprc = *fprcp) != NULL) {
+		if (fprc == proc)
+			break;
+
+		fprcp = &fprc->ftpc_next;
+	}
+
+	/*
+	 * Something strange has happened if we can't find the proc.
+	 */
+	ASSERT(fprc != NULL);
+
+	*fprcp = fprc->ftpc_next;
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+#if defined(__APPLE__)
+	/*
+	 * Apple explicit lock management. Not 100% certain we need this, the
+	 * memory is freed even without the destroy. Maybe accounting cleanup?
+	 */
+	lck_mtx_destroy(&fprc->ftpc_mtx, fasttrap_lck_grp);
+#endif
+
+	kmem_free(fprc, sizeof (fasttrap_proc_t));
+}
+
+/*
+ * Lookup a fasttrap-managed provider based on its name and associated pid.
+ * If the pattr argument is non-NULL, this function instantiates the provider
+ * if it doesn't exist otherwise it returns NULL. The provider is returned
+ * with its lock held.
+ */
+#if defined(__APPLE__)
+static fasttrap_provider_t *
+fasttrap_provider_lookup(pid_t pid, fasttrap_provider_type_t provider_type, const char *name,
+    const dtrace_pattr_t *pattr)
+#endif /* __APPLE__ */
+{
+	fasttrap_provider_t *fp, *new_fp = NULL;
+	fasttrap_bucket_t *bucket;
+	char provname[DTRACE_PROVNAMELEN];
+	proc_t *p;
+	cred_t *cred;
+
+	ASSERT(strlen(name) < sizeof (fp->ftp_name));
+	ASSERT(pattr != NULL);
+
+	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	/*
+	 * Take a lap through the list and return the match if we find it.
+	 */
+	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
+		if (fp->ftp_pid == pid &&
+#if defined(__APPLE__)
+		    fp->ftp_provider_type == provider_type &&
+#endif /* __APPLE__ */
+		    strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
+		    !fp->ftp_retired) {
+			lck_mtx_lock(&fp->ftp_mtx);
+			lck_mtx_unlock(&bucket->ftb_mtx);
+			return (fp);
+		}
+	}
+
+	/*
+	 * Drop the bucket lock so we don't try to perform a sleeping
+	 * allocation under it.
+	 */
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	/*
+	 * Make sure the process exists, isn't a child created as the result
+	 * of a vfork(2), and isn't a zombie (but may be in fork).
+	 */
+	if ((p = proc_find(pid)) == NULL) {
+		return NULL;
+	}
+	proc_lock(p);
+	if (p->p_lflag & (P_LINVFORK | P_LEXIT)) {
+		proc_unlock(p);
+		proc_rele(p);
+		return (NULL);
+	}
+
+	/*
+	 * Increment p_dtrace_probes so that the process knows to inform us
+	 * when it exits or execs. fasttrap_provider_free() decrements this
+	 * when we're done with this provider.
+	 */
+	p->p_dtrace_probes++;
+
+	/*
+	 * Grab the credentials for this process so we have
+	 * something to pass to dtrace_register().
+	 */
+#if !defined(__APPLE__)
+	mutex_enter(&p->p_crlock);
+	crhold(p->p_cred);
+	cred = p->p_cred;
+	mutex_exit(&p->p_crlock);
+	mutex_exit(&p->p_lock);
+#else
+	// lck_mtx_lock(&p->p_crlock);
+	// Seems like OS X has no equivalent to crhold, even though it has a cr_ref field in ucred
+	crhold(p->p_ucred);
+	cred = p->p_ucred;
+	// lck_mtx_unlock(&p->p_crlock);
+	proc_unlock(p);
+	proc_rele(p);
+#endif /* __APPLE__ */
+
+	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
+	ASSERT(new_fp != NULL);
+	new_fp->ftp_pid = pid;
+	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
+#if defined(__APPLE__)
+	new_fp->ftp_provider_type = provider_type;
+
+	/*
+	 * Apple locks require explicit init.
+	 */
+	lck_mtx_init(&new_fp->ftp_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+	lck_mtx_init(&new_fp->ftp_cmtx, fasttrap_lck_grp, fasttrap_lck_attr);
+#endif /* __APPLE__ */
+
+	ASSERT(new_fp->ftp_proc != NULL);
+
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	/*
+	 * Take another lap through the list to make sure a provider hasn't
+	 * been created for this pid while we weren't under the bucket lock.
+	 */
+	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
+		if (fp->ftp_pid == pid && strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
+		    !fp->ftp_retired) {
+			lck_mtx_lock(&fp->ftp_mtx);
+			lck_mtx_unlock(&bucket->ftb_mtx);
+			fasttrap_provider_free(new_fp);
+			crfree(cred);
+			return (fp);
+		}
+	}
+
+	(void) strlcpy(new_fp->ftp_name, name, sizeof(new_fp->ftp_name));
+
+	/*
+	 * Fail and return NULL if either the provider name is too long
+	 * or we fail to register this new provider with the DTrace
+	 * framework. Note that this is the only place we ever construct
+	 * the full provider name -- we keep it in pieces in the provider
+	 * structure.
+	 */
+	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
+	    (int)sizeof (provname) ||
+	    dtrace_register(provname, pattr,
+	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
+	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
+	    &new_fp->ftp_provid) != 0) {
+		lck_mtx_unlock(&bucket->ftb_mtx);
+		fasttrap_provider_free(new_fp);
+		crfree(cred);
+		return (NULL);
+	}
+
+	new_fp->ftp_next = bucket->ftb_data;
+	bucket->ftb_data = new_fp;
+
+	lck_mtx_lock(&new_fp->ftp_mtx);
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	crfree(cred);
+	return (new_fp);
+}
+
+static void
+fasttrap_provider_free(fasttrap_provider_t *provider)
+{
+	pid_t pid = provider->ftp_pid;
+	proc_t *p;
+
+	/*
+	 * There need to be no associated enabled probes, no consumers
+	 * creating probes, and no meta providers referencing this provider.
+	 */
+	ASSERT(provider->ftp_rcount == 0);
+	ASSERT(provider->ftp_ccount == 0);
+	ASSERT(provider->ftp_mcount == 0);
+
+	fasttrap_proc_release(provider->ftp_proc);
+
+#if defined(__APPLE__)
+	/*
+	 * Apple explicit lock management. Not 100% certain we need this, the
+	 * memory is freed even without the destroy. Maybe accounting cleanup?
+	 */
+	lck_mtx_destroy(&provider->ftp_mtx, fasttrap_lck_grp);
+	lck_mtx_destroy(&provider->ftp_cmtx, fasttrap_lck_grp);
+#endif
+
+	kmem_free(provider, sizeof (fasttrap_provider_t));
+
+	/*
+	 * Decrement p_dtrace_probes on the process whose provider we're
+	 * freeing. We don't have to worry about clobbering somone else's
+	 * modifications to it because we have locked the bucket that
+	 * corresponds to this process's hash chain in the provider hash
+	 * table. Don't sweat it if we can't find the process.
+	 */
+	if ((p = proc_find(pid)) == NULL) {
+		return;
+	}
+
+	proc_lock(p);
+	p->p_dtrace_probes--;
+	proc_unlock(p);
+	
+	proc_rele(p);
+}
+
+static void
+fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
+{
+	fasttrap_provider_t *fp;
+	fasttrap_bucket_t *bucket;
+	dtrace_provider_id_t provid;
+
+	ASSERT(strlen(name) < sizeof (fp->ftp_name));
+
+	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
+	lck_mtx_lock(&bucket->ftb_mtx);
+
+	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
+		if (fp->ftp_pid == pid && strncmp(fp->ftp_name, name, sizeof(fp->ftp_name)) == 0 &&
+		    !fp->ftp_retired)
+			break;
+	}
+
+	if (fp == NULL) {
+		lck_mtx_unlock(&bucket->ftb_mtx);
+		return;
+	}
+
+	lck_mtx_lock(&fp->ftp_mtx);
+	ASSERT(!mprov || fp->ftp_mcount > 0);
+	if (mprov && --fp->ftp_mcount != 0)  {
+		lck_mtx_unlock(&fp->ftp_mtx);
+		lck_mtx_unlock(&bucket->ftb_mtx);
+		return;
+	}
+
+	/*
+	 * Mark the provider to be removed in our post-processing step,
+	 * mark it retired, and mark its proc as defunct (though it may
+	 * already be marked defunct by another provider that shares the
+	 * same proc). Marking it indicates that we should try to remove it;
+	 * setting the retired flag indicates that we're done with this
+	 * provider; setting the proc to be defunct indicates that all
+	 * tracepoints associated with the traced process should be ignored.
+	 *
+	 * We obviously need to take the bucket lock before the provider lock
+	 * to perform the lookup, but we need to drop the provider lock
+	 * before calling into the DTrace framework since we acquire the
+	 * provider lock in callbacks invoked from the DTrace framework. The
+	 * bucket lock therefore protects the integrity of the provider hash
+	 * table.
+	 */
+	fp->ftp_proc->ftpc_defunct = 1;
+	fp->ftp_retired = 1;
+	fp->ftp_marked = 1;
+	provid = fp->ftp_provid;
+	lck_mtx_unlock(&fp->ftp_mtx);
+
+	/*
+	 * We don't have to worry about invalidating the same provider twice
+	 * since fasttrap_provider_lookup() will ignore provider that have
+	 * been marked as retired.
+	 */
+	dtrace_invalidate(provid);
+
+	lck_mtx_unlock(&bucket->ftb_mtx);
+
+	fasttrap_pid_cleanup();
+}
+
+static int
+fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
+{
+	fasttrap_provider_t *provider;
+	fasttrap_probe_t *pp;
+	fasttrap_tracepoint_t *tp;
+	const char *name;
+	unsigned int i, aframes, whack;
+
+#if defined(__APPLE__)
+	switch (pdata->ftps_probe_type) {
+#endif
+	case DTFTP_ENTRY:
+		name = "entry";
+		aframes = FASTTRAP_ENTRY_AFRAMES;
+		break;
+	case DTFTP_RETURN:
+		name = "return";
+		aframes = FASTTRAP_RETURN_AFRAMES;
+		break;
+	case DTFTP_OFFSETS:
+		aframes = 0;
+		name = NULL;
+		break;
+	default:
+		return (EINVAL);
+	}
+
+#if defined(__APPLE__)
+	const char* provider_name;
+	switch (pdata->ftps_provider_type) {
+		case DTFTP_PROVIDER_PID:
+			provider_name = FASTTRAP_PID_NAME;
+			break;
+		case DTFTP_PROVIDER_OBJC:
+			provider_name = FASTTRAP_OBJC_NAME;
+			break;
+		case DTFTP_PROVIDER_ONESHOT:
+			provider_name = FASTTRAP_ONESHOT_NAME;
+			break;
+		default:
+			return (EINVAL);
+	}
+
+	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, pdata->ftps_provider_type,
+						 provider_name, &pid_attr)) == NULL)
+		return (ESRCH);
+#endif /* __APPLE__ */
+
+	/*
+	 * Increment this reference count to indicate that a consumer is
+	 * actively adding a new probe associated with this provider. This
+	 * prevents the provider from being deleted -- we'll need to check
+	 * for pending deletions when we drop this reference count.
+	 */
+	provider->ftp_ccount++;
+	lck_mtx_unlock(&provider->ftp_mtx);
+
+	/*
+	 * Grab the creation lock to ensure consistency between calls to
+	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
+	 * other threads creating probes. We must drop the provider lock
+	 * before taking this lock to avoid a three-way deadlock with the
+	 * DTrace framework.
+	 */
+	lck_mtx_lock(&provider->ftp_cmtx);
+
+	if (name == NULL) {
+		for (i = 0; i < pdata->ftps_noffs; i++) {
+			char name_str[17];
+
+			(void) snprintf(name_str, sizeof(name_str), "%llx",
+			    (unsigned long long)pdata->ftps_offs[i]);
+
+			if (dtrace_probe_lookup(provider->ftp_provid,
+			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
+				continue;
+
+			atomic_add_32(&fasttrap_total, 1);
+
+			if (fasttrap_total > fasttrap_max) {
+				atomic_add_32(&fasttrap_total, -1);
+				goto no_mem;
+			}
+
+#if !defined(__APPLE__)
+			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
+			ASSERT(pp != NULL);
+#else
+			pp = zalloc(fasttrap_probe_t_zones[1]);
+			bzero(pp, sizeof (fasttrap_probe_t));
+#endif
+
+			pp->ftp_prov = provider;
+			pp->ftp_faddr = pdata->ftps_pc;
+			pp->ftp_fsize = pdata->ftps_size;
+			pp->ftp_pid = pdata->ftps_pid;
+			pp->ftp_ntps = 1;
+
+#if !defined(__APPLE__)
+			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
+#else
+			tp = zalloc(fasttrap_tracepoint_t_zone);			
+			bzero(tp, sizeof (fasttrap_tracepoint_t));
+#endif
+
+			tp->ftt_proc = provider->ftp_proc;
+			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
+			tp->ftt_pid = pdata->ftps_pid;
+
+			pp->ftp_tps[0].fit_tp = tp;
+			pp->ftp_tps[0].fit_id.fti_probe = pp;
+#if defined(__APPLE__)
+			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_probe_type;
+#endif
+			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
+			    pdata->ftps_mod, pdata->ftps_func, name_str,
+			    FASTTRAP_OFFSET_AFRAMES, pp);
+		}
+
+	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
+	    pdata->ftps_func, name) == 0) {
+		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
+
+		if (fasttrap_total > fasttrap_max) {
+			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
+			goto no_mem;
+		}
+
+		ASSERT(pdata->ftps_noffs > 0);
+#if !defined(__APPLE__)
+		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
+					  ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
+		ASSERT(pp != NULL);
+#else
+		if (pdata->ftps_noffs < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
+			pp = zalloc(fasttrap_probe_t_zones[pdata->ftps_noffs]);
+			bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[pdata->ftps_noffs]));
+		} else {
+			pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
+		}
+#endif
+
+		pp->ftp_prov = provider;
+		pp->ftp_faddr = pdata->ftps_pc;
+		pp->ftp_fsize = pdata->ftps_size;
+		pp->ftp_pid = pdata->ftps_pid;
+		pp->ftp_ntps = pdata->ftps_noffs;
+
+		for (i = 0; i < pdata->ftps_noffs; i++) {
+#if !defined(__APPLE__)
+			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
+#else
+			tp = zalloc(fasttrap_tracepoint_t_zone);
+			bzero(tp, sizeof (fasttrap_tracepoint_t));
+#endif
+
+			tp->ftt_proc = provider->ftp_proc;
+			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
+			tp->ftt_pid = pdata->ftps_pid;
+
+			pp->ftp_tps[i].fit_tp = tp;
+			pp->ftp_tps[i].fit_id.fti_probe = pp;
+#if defined(__APPLE__)
+			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_probe_type;
+#endif
+		}
+
+		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
+		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
+	}
+
+	lck_mtx_unlock(&provider->ftp_cmtx);
+
+	/*
+	 * We know that the provider is still valid since we incremented the
+	 * creation reference count. If someone tried to clean up this provider
+	 * while we were using it (e.g. because the process called exec(2) or
+	 * exit(2)), take note of that and try to clean it up now.
+	 */
+	lck_mtx_lock(&provider->ftp_mtx);
+	provider->ftp_ccount--;
+	whack = provider->ftp_retired;
+	lck_mtx_unlock(&provider->ftp_mtx);
+
+	if (whack)
+		fasttrap_pid_cleanup();
+
+	return (0);
+
+no_mem:
+	/*
+	 * If we've exhausted the allowable resources, we'll try to remove
+	 * this provider to free some up. This is to cover the case where
+	 * the user has accidentally created many more probes than was
+	 * intended (e.g. pid123:::).
+	 */
+	lck_mtx_unlock(&provider->ftp_cmtx);
+	lck_mtx_lock(&provider->ftp_mtx);
+	provider->ftp_ccount--;
+	provider->ftp_marked = 1;
+	lck_mtx_unlock(&provider->ftp_mtx);
+
+	fasttrap_pid_cleanup();
+
+	return (ENOMEM);
+}
+
+/*ARGSUSED*/
+static void *
+fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
+{
+#pragma unused(arg)
+	fasttrap_provider_t *provider;
+
+	/*
+	 * A 32-bit unsigned integer (like a pid for example) can be
+	 * expressed in 10 or fewer decimal digits. Make sure that we'll
+	 * have enough space for the provider name.
+	 */
+	if (strlen(dhpv->dthpv_provname) + 10 >=
+	    sizeof (provider->ftp_name)) {
+		cmn_err(CE_WARN, "failed to instantiate provider %s: "
+		    "name too long to accomodate pid", dhpv->dthpv_provname);
+		return (NULL);
+	}
+
+	/*
+	 * Don't let folks spoof the true pid provider.
+	 */
+	if (strncmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME, sizeof(FASTTRAP_PID_NAME)) == 0) {
+		cmn_err(CE_WARN, "failed to instantiate provider %s: "
+		    "%s is an invalid name", dhpv->dthpv_provname,
+		    FASTTRAP_PID_NAME);
+		return (NULL);
+	}
+#if defined(__APPLE__)
+	/*
+	 * We also need to check the other pid provider types
+	 */
+	if (strncmp(dhpv->dthpv_provname, FASTTRAP_OBJC_NAME, sizeof(FASTTRAP_OBJC_NAME)) == 0) {
+		cmn_err(CE_WARN, "failed to instantiate provider %s: "
+		    "%s is an invalid name", dhpv->dthpv_provname,
+		    FASTTRAP_OBJC_NAME);
+		return (NULL);
+	}
+	if (strncmp(dhpv->dthpv_provname, FASTTRAP_ONESHOT_NAME, sizeof(FASTTRAP_ONESHOT_NAME)) == 0) {
+		cmn_err(CE_WARN, "failed to instantiate provider %s: "
+		    "%s is an invalid name", dhpv->dthpv_provname,
+		    FASTTRAP_ONESHOT_NAME);
+		return (NULL);
+	}
+#endif /* __APPLE__ */
+
+	/*
+	 * The highest stability class that fasttrap supports is ISA; cap
+	 * the stability of the new provider accordingly.
+	 */
+	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class >= DTRACE_CLASS_COMMON)
+		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
+	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class >= DTRACE_CLASS_COMMON)
+		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
+	if (dhpv->dthpv_pattr.dtpa_func.dtat_class >= DTRACE_CLASS_COMMON)
+		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
+	if (dhpv->dthpv_pattr.dtpa_name.dtat_class >= DTRACE_CLASS_COMMON)
+		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
+	if (dhpv->dthpv_pattr.dtpa_args.dtat_class >= DTRACE_CLASS_COMMON)
+		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
+
+#if defined(__APPLE__)
+	if ((provider = fasttrap_provider_lookup(pid, DTFTP_PROVIDER_USDT, dhpv->dthpv_provname,
+	    &dhpv->dthpv_pattr)) == NULL) {
+		cmn_err(CE_WARN, "failed to instantiate provider %s for "
+		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
+		return (NULL);
+	}
+
+	/*
+	 * APPLE NOTE!
+	 *
+	 * USDT probes (fasttrap meta probes) are very expensive to create.
+	 * Profiling has shown that the largest single cost is verifying that
+	 * dtrace hasn't already created a given meta_probe. The reason for
+	 * this is dtrace_match() often has to strcmp ~100 hashed entries for
+	 * each static probe being created. We want to get rid of that check.
+	 * The simplest way of eliminating it is to deny the ability to add
+	 * probes to an existing provider. If the provider already exists, BZZT!
+	 * This still leaves the possibility of intentionally malformed DOF
+	 * having duplicate probes. However, duplicate probes are not fatal,
+	 * and there is no way to get that by accident, so we will not check
+	 * for that case.
+	 */
+
+	if (provider->ftp_mcount != 0) {
+		/* This is the duplicate provider case. */
+		lck_mtx_unlock(&provider->ftp_mtx);
+		return NULL;
+	}
+#endif /* __APPLE__ */
+
+	/*
+	 * Up the meta provider count so this provider isn't removed until
+	 * the meta provider has been told to remove it.
+	 */
+	provider->ftp_mcount++;
+
+	lck_mtx_unlock(&provider->ftp_mtx);
+
+	return (provider);
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_meta_create_probe(void *arg, void *parg,
+    dtrace_helper_probedesc_t *dhpb)
+{
+#pragma unused(arg)
+	fasttrap_provider_t *provider = parg;
+	fasttrap_probe_t *pp;
+	fasttrap_tracepoint_t *tp;
+	unsigned int i, j;
+	uint32_t ntps;
+
+	/*
+	 * Since the meta provider count is non-zero we don't have to worry
+	 * about this provider disappearing.
+	 */
+	ASSERT(provider->ftp_mcount > 0);
+
+	/*
+	 * Grab the creation lock to ensure consistency between calls to
+	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
+	 * other threads creating probes.
+	 */
+	lck_mtx_lock(&provider->ftp_cmtx);
+
+#if !defined(__APPLE__)
+	/*
+	 * APPLE NOTE: This is hideously expensive. See note in 
+	 * fasttrap_meta_provide() for why we can get away without
+	 * checking here.
+	 */
+	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
+	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
+		lck_mtx_unlock(&provider->ftp_cmtx);
+		return;
+	}
+#endif
+
+	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
+	ASSERT(ntps > 0);
+
+	atomic_add_32(&fasttrap_total, ntps);
+
+	if (fasttrap_total > fasttrap_max) {
+		atomic_add_32(&fasttrap_total, -ntps);
+		lck_mtx_unlock(&provider->ftp_cmtx);
+		return;
+	}
+
+#if !defined(__APPLE__)
+	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
+	ASSERT(pp != NULL);
+#else
+	if (ntps < FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS) {
+		pp = zalloc(fasttrap_probe_t_zones[ntps]);
+		bzero(pp, offsetof(fasttrap_probe_t, ftp_tps[ntps]));
+	} else {
+		pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
+	}
+#endif
+
+	pp->ftp_prov = provider;
+	pp->ftp_pid = provider->ftp_pid;
+	pp->ftp_ntps = ntps;
+	pp->ftp_nargs = dhpb->dthpb_xargc;
+	pp->ftp_xtypes = dhpb->dthpb_xtypes;
+	pp->ftp_ntypes = dhpb->dthpb_ntypes;
+
+	/*
+	 * First create a tracepoint for each actual point of interest.
+	 */
+	for (i = 0; i < dhpb->dthpb_noffs; i++) {
+#if !defined(__APPLE__)
+		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
+#else
+		tp = zalloc(fasttrap_tracepoint_t_zone);
+		bzero(tp, sizeof (fasttrap_tracepoint_t));
+#endif
+
+		tp->ftt_proc = provider->ftp_proc;
+#if defined(__APPLE__)
+		/*
+		 * APPLE NOTE: We have linker support when creating DOF to handle all relocations for us.
+		 * Unfortunately, a side effect of this is that the relocations do not point at exactly
+		 * the location we want. We need to fix up the addresses here. The fixups vary by arch and type.
+		 */
+#if defined(__i386__)
+		/*
+		 * Both 32 & 64 bit want to go back one byte, to point at the first NOP
+		 */
+		tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i] - 1;
+#elif defined(__ppc__)
+		/* All PPC probes are zero offset. */
+		tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_offs[i];
+#else
+#error "Architecture not supported"
+#endif
+
+#else
+		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
+#endif
+		tp->ftt_pid = provider->ftp_pid;
+
+		pp->ftp_tps[i].fit_tp = tp;
+		pp->ftp_tps[i].fit_id.fti_probe = pp;
+#ifdef __sparc
+		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
+#else
+		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
+#endif
+	}
+
+	/*
+	 * Then create a tracepoint for each is-enabled point.
+	 */
+	for (j = 0; i < ntps; i++, j++) {
+#if !defined(__APPLE__)
+		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
+#else
+		tp = zalloc(fasttrap_tracepoint_t_zone);
+		bzero(tp, sizeof (fasttrap_tracepoint_t));
+#endif
+
+		tp->ftt_proc = provider->ftp_proc;
+#if defined(__APPLE__)
+		/*
+		 * APPLE NOTE: We have linker support when creating DOF to handle all relocations for us.
+		 * Unfortunately, a side effect of this is that the relocations do not point at exactly
+		 * the location we want. We need to fix up the addresses here. The fixups vary by arch and type.
+		 */
+#if defined(__i386__)
+		/*
+		 * Both 32 & 64 bit want to go forward two bytes, to point at a single byte nop.
+		 */
+		tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j] + 2;
+#elif defined(__ppc__)
+		/* All PPC is-enabled probes are zero offset. */
+		tp->ftt_pc = dhpb->dthpb_base + (int64_t)dhpb->dthpb_enoffs[j];
+#else
+#error "Architecture not supported"
+#endif
+
+#else
+		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
+#endif
+		tp->ftt_pid = provider->ftp_pid;
+
+		pp->ftp_tps[i].fit_tp = tp;
+		pp->ftp_tps[i].fit_id.fti_probe = pp;
+		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
+	}
+
+	/*
+	 * If the arguments are shuffled around we set the argument remapping
+	 * table. Later, when the probe fires, we only remap the arguments
+	 * if the table is non-NULL.
+	 */
+	for (i = 0; i < dhpb->dthpb_xargc; i++) {
+		if (dhpb->dthpb_args[i] != i) {
+			pp->ftp_argmap = dhpb->dthpb_args;
+			break;
+		}
+	}
+
+	/*
+	 * The probe is fully constructed -- register it with DTrace.
+	 */
+	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
+	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
+
+	lck_mtx_unlock(&provider->ftp_cmtx);
+}
+
+/*ARGSUSED*/
+static void
+fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
+{
+#pragma unused(arg)
+	/*
+	 * Clean up the USDT provider. There may be active consumers of the
+	 * provider busy adding probes, no damage will actually befall the
+	 * provider until that count has dropped to zero. This just puts
+	 * the provider on death row.
+	 */
+	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
+}
+
+static dtrace_mops_t fasttrap_mops = {
+	fasttrap_meta_create_probe,
+	fasttrap_meta_provide,
+	fasttrap_meta_remove
+};
+
+/*ARGSUSED*/
+static int
+fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
+{
+#pragma unused(dev, md, rv)
+	if (!dtrace_attached())
+		return (EAGAIN);
+
+	if (cmd == FASTTRAPIOC_MAKEPROBE) {
+		// FIXME! What size is arg? If it is not 64 bit, how do we pass in a 64 bit value?
+		fasttrap_probe_spec_t *uprobe = (void *)arg;
+		fasttrap_probe_spec_t *probe;
+		uint64_t noffs;
+		size_t size, i;
+		int ret;
+		char *c;
+
+		/*
+		 * FIXME! How does this work? The kern is running in 32 bit mode. It has a 32 bit pointer,
+		 * uprobe. We do address manipulations on it, and still have a 64 bit value? This seems
+		 * broken. What is the right way to do this?
+		 */
+		if (copyin((user_addr_t)(unsigned long)&uprobe->ftps_noffs, &noffs,
+		    sizeof (uprobe->ftps_noffs)))
+			return (EFAULT);
+
+		/*
+		 * Probes must have at least one tracepoint.
+		 */
+		if (noffs == 0)
+			return (EINVAL);
+
+		/*
+		 * We want to check the number of noffs before doing
+		 * sizing math, to prevent potential buffer overflows.
+		 */
+		if (noffs > ((1024 * 1024) - sizeof(fasttrap_probe_spec_t)) / sizeof(probe->ftps_offs[0]))
+			return (ENOMEM);
+
+		size = sizeof (fasttrap_probe_spec_t) +
+		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
+
+		probe = kmem_alloc(size, KM_SLEEP);
+
+		if (copyin((user_addr_t)(unsigned long)uprobe, probe, size) != 0) {
+			kmem_free(probe, size);
+			return (EFAULT);
+		}
+
+		/*
+		 * Verify that the function and module strings contain no
+		 * funny characters.
+		 */
+		for (i = 0, c = &probe->ftps_func[0]; i < sizeof(probe->ftps_func) && *c != '\0'; i++, c++) {
+			if (*c < 0x20 || 0x7f <= *c) {
+				ret = EINVAL;
+				goto err;
+			}
+		}
+		if (*c != '\0') {
+			ret = EINVAL;
+			goto err;
+		}
+
+		for (i = 0, c = &probe->ftps_mod[0]; i < sizeof(probe->ftps_mod) && *c != '\0'; i++, c++) {
+			if (*c < 0x20 || 0x7f <= *c) {
+				ret = EINVAL;
+				goto err;
+			}
+		}
+		if (*c != '\0') {
+			ret = EINVAL;
+			goto err;
+		}
+
+		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
+			proc_t *p;
+			pid_t pid = probe->ftps_pid;
+
+			/*
+			 * Report an error if the process doesn't exist
+			 * or is actively being birthed.
+			 */
+			if ((p = proc_find(pid)) == PROC_NULL || p->p_stat == SIDL) {
+				if (p != PROC_NULL)
+					proc_rele(p);
+				return (ESRCH);
+			}
+			// proc_lock(p);
+			// FIXME! How is this done on OS X?
+			// if ((ret = priv_proc_cred_perm(cr, p, NULL,
+			//     VREAD | VWRITE)) != 0) {
+			// 	mutex_exit(&p->p_lock);
+			// 	return (ret);
+			// }
+			// proc_unlock(p);
+			proc_rele(p);
+		}
+
+		ret = fasttrap_add_probe(probe);
+
+err:
+		kmem_free(probe, size);
+
+		return (ret);
+
+	} else if (cmd == FASTTRAPIOC_GETINSTR) {
+		fasttrap_instr_query_t instr;
+		fasttrap_tracepoint_t *tp;
+		uint_t index;
+		// int ret;
+
+		if (copyin((user_addr_t)(unsigned long)arg, &instr, sizeof (instr)) != 0)
+			return (EFAULT);
+
+		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
+			proc_t *p;
+			pid_t pid = instr.ftiq_pid;
+
+			/*
+			 * Report an error if the process doesn't exist
+			 * or is actively being birthed.
+			 */
+			if ((p = proc_find(pid)) == NULL || p->p_stat == SIDL) {
+				if (p != PROC_NULL)
+					proc_rele(p);
+				return (ESRCH);
+			}
+			//proc_lock(p);
+			// FIXME! How is this done on OS X?
+			// if ((ret = priv_proc_cred_perm(cr, p, NULL,
+			//     VREAD)) != 0) {
+			// 	mutex_exit(&p->p_lock);
+			// 	return (ret);
+			// }
+			// proc_unlock(p);
+			proc_rele(p);
+		}
+
+		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
+
+		lck_mtx_lock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
+		tp = fasttrap_tpoints.fth_table[index].ftb_data;
+		while (tp != NULL) {
+			if (instr.ftiq_pid == tp->ftt_pid &&
+			    instr.ftiq_pc == tp->ftt_pc &&
+			    !tp->ftt_proc->ftpc_defunct)
+				break;
+
+			tp = tp->ftt_next;
+		}
+
+		if (tp == NULL) {
+			lck_mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
+			return (ENOENT);
+		}
+
+		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
+		    sizeof (instr.ftiq_instr));
+		lck_mtx_unlock(&fasttrap_tpoints.fth_table[index].ftb_mtx);
+
+		if (copyout(&instr, (user_addr_t)(unsigned long)arg, sizeof (instr)) != 0)
+			return (EFAULT);
+
+		return (0);
+	}
+
+	return (EINVAL);
+}
+
+static int
+fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	ulong_t nent;
+
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	fasttrap_devi = devi;
+
+	/*
+	 * Install our hooks into fork(2), exec(2), and exit(2).
+	 */
+	dtrace_fasttrap_fork_ptr = &fasttrap_fork;
+	dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
+	dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
+
+#if !defined(__APPLE__)
+	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
+	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
+#else
+	/*
+	 * We're sizing based on system memory. 100k probes per 256M of system memory.
+	 * Yes, this is a WAG.
+	 */
+	fasttrap_max = (sane_size >> 28) * 100000;
+#endif
+	fasttrap_total = 0;
+
+	/*
+	 * Conjure up the tracepoints hashtable...
+	 */
+	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
+	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
+
+	if (nent <= 0 || nent > 0x1000000)
+		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
+
+	if ((nent & (nent - 1)) == 0)
+		fasttrap_tpoints.fth_nent = nent;
+	else
+		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
+	ASSERT(fasttrap_tpoints.fth_nent > 0);
+	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
+	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
+	    sizeof (fasttrap_bucket_t), KM_SLEEP);
+	ASSERT(fasttrap_tpoints.fth_table != NULL);
+#if defined(__APPLE__)
+	/*
+	 * We have to explicitly initialize all locks...
+	 */
+	unsigned int i;
+	for (i=0; i<fasttrap_tpoints.fth_nent; i++) {
+		lck_mtx_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+	}
+#endif
+
+	/*
+	 * ... and the providers hash table...
+	 */
+	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
+	if ((nent & (nent - 1)) == 0)
+		fasttrap_provs.fth_nent = nent;
+	else
+		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
+	ASSERT(fasttrap_provs.fth_nent > 0);
+	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
+	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
+	    sizeof (fasttrap_bucket_t), KM_SLEEP);
+	ASSERT(fasttrap_provs.fth_table != NULL);
+#if defined(__APPLE__)
+	/*
+	 * We have to explicitly initialize all locks...
+	 */
+	for (i=0; i<fasttrap_provs.fth_nent; i++) {
+		lck_mtx_init(&fasttrap_provs.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+	}
+#endif
+
+	/*
+	 * ... and the procs hash table.
+	 */
+	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
+	if ((nent & (nent - 1)) == 0)
+		fasttrap_procs.fth_nent = nent;
+	else
+		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
+	ASSERT(fasttrap_procs.fth_nent > 0);
+	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
+	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
+	    sizeof (fasttrap_bucket_t), KM_SLEEP);
+	ASSERT(fasttrap_procs.fth_table != NULL);
+#if defined(__APPLE__)
+	/*
+	 * We have to explicitly initialize all locks...
+	 */
+	for (i=0; i<fasttrap_procs.fth_nent; i++) {
+		lck_mtx_init(&fasttrap_procs.fth_table[i].ftb_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+	}
+#endif
+
+	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
+	    &fasttrap_meta_id);
+
+	return (DDI_SUCCESS);
+}
+
+static int 
+_fasttrap_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev, flags, devtype, p)
+	return  0;
+}
+
+static int
+_fasttrap_ioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, struct proc *p)
+{
+#pragma unused(p)
+	int err, rv = 0;
+
+	/*
+	 * FIXME! 64 bit problem with the data var.
+	 */
+	err = fasttrap_ioctl(dev, (int)cmd, *(intptr_t *)data, fflag, CRED(), &rv);
+
+	/* XXX Darwin's BSD ioctls only return -1 or zero. Overload errno to mimic Solaris. 20 bits suffice. */
+	if (err != 0) {
+		ASSERT( (err & 0xfffff000) == 0 );
+		return (err & 0xfff); /* ioctl returns -1 and errno set to an error code < 4096 */
+	} else if (rv != 0) {
+		ASSERT( (rv & 0xfff00000) == 0 );
+		return (((rv & 0xfffff) << 12)); /* ioctl returns -1 and errno set to a return value >= 4096 */
+	} else 
+		return 0;
+}
+
+static int gFasttrapInited = 0;
+
+#define FASTTRAP_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+
+static struct cdevsw fasttrap_cdevsw =
+{
+	_fasttrap_open,         /* open */
+	eno_opcl,               /* close */
+	eno_rdwrt,              /* read */
+	eno_rdwrt,              /* write */
+	_fasttrap_ioctl,        /* ioctl */
+	(stop_fcn_t *)nulldev,  /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,                   /* tty's */
+	eno_select,             /* select */
+	eno_mmap,               /* mmap */
+	eno_strat,              /* strategy */
+	eno_getc,               /* getc */
+	eno_putc,               /* putc */
+	0                       /* type */
+};
+
+void fasttrap_init(void);
+
+void
+fasttrap_init( void )
+{
+	/*
+	 * This method is now invoked from multiple places. Any open of /dev/dtrace,
+	 * also dtrace_init if the dtrace_dof_mode is DTRACE_DOF_MODE_NON_LAZY.
+	 *
+	 * The reason is to delay allocating the (rather large) resources as late as possible.
+	 */
+	if (0 == gFasttrapInited) {
+		int majdevno = cdevsw_add(FASTTRAP_MAJOR, &fasttrap_cdevsw);
+
+		if (majdevno < 0) {
+			// FIX ME! What kind of error reporting to do here?
+			printf("fasttrap_init: failed to allocate a major number!\n");
+			return;
+		}
+
+		dev_t device = makedev( (uint32_t)majdevno, 0 );
+		if (NULL == devfs_make_node( device, DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, "fasttrap", 0 )) {
+			return;
+		}
+
+		/*
+		 * Allocate the fasttrap_tracepoint_t zone
+		 */
+		fasttrap_tracepoint_t_zone = zinit(sizeof(fasttrap_tracepoint_t),
+						   1024 * sizeof(fasttrap_tracepoint_t),
+						   sizeof(fasttrap_tracepoint_t),
+						   "dtrace.fasttrap_tracepoint_t");
+
+		/*
+		 * fasttrap_probe_t's are variable in size. We use an array of zones to
+		 * cover the most common sizes.
+		 */
+		int i;
+		for (i=1; i<FASTTRAP_PROBE_T_ZONE_MAX_TRACEPOINTS; i++) {
+			size_t zone_element_size = offsetof(fasttrap_probe_t, ftp_tps[i]);
+			fasttrap_probe_t_zones[i] = zinit(zone_element_size,
+							  1024 * zone_element_size,
+							  zone_element_size,
+							  fasttrap_probe_t_zone_names[i]);
+		}
+
+		
+		/*
+		 * Create the fasttrap lock group. Must be done before fasttrap_attach()!
+		 */
+		fasttrap_lck_attr = lck_attr_alloc_init();
+		fasttrap_lck_grp_attr= lck_grp_attr_alloc_init();		
+		fasttrap_lck_grp = lck_grp_alloc_init("fasttrap",  fasttrap_lck_grp_attr);
+
+		/*
+		 * Initialize global locks
+		 */
+		lck_mtx_init(&fasttrap_cleanup_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+		lck_mtx_init(&fasttrap_count_mtx, fasttrap_lck_grp, fasttrap_lck_attr);
+
+		if (DDI_FAILURE == fasttrap_attach((dev_info_t *)device, 0 )) {
+			// FIX ME! Do we remove the devfs node here?
+			// What kind of error reporting?
+			printf("fasttrap_init: Call to fasttrap_attach failed.\n");
+			return;
+		}
+
+		gFasttrapInited = 1;		
+	}
+}
+
+#undef FASTTRAP_MAJOR
diff --git a/bsd/dev/dtrace/fbt.c b/bsd/dev/dtrace/fbt.c
new file mode 100644
index 000000000..f1b85f42f
--- /dev/null
+++ b/bsd/dev/dtrace/fbt.c
@@ -0,0 +1,483 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)fbt.c	1.15	05/09/19 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#include <mach-o/loader.h> 
+#include <kern/mach_header.h>
+
+extern struct mach_header _mh_execute_header; /* the kernel's mach header */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+#include <pexpert/pexpert.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/fbt.h>
+
+#include <sys/dtrace_glue.h>
+
+/* #include <machine/trap.h> */
+struct savearea_t; /* Used anonymously */
+typedef kern_return_t (*perfCallback)(int, struct savearea_t *, int, int);
+
+#if defined (__ppc__) || defined (__ppc64__)
+extern perfCallback tempDTraceTrapHook, tempDTraceIntHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
+extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int);
+#else
+extern perfCallback tempDTraceTrapHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
+#endif
+
+#define	FBT_ADDR2NDX(addr)	((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
+#define	FBT_PROBETAB_SIZE	0x8000		/* 32k entries -- 128K total */
+
+static dev_info_t		*fbt_devi;
+static int				fbt_probetab_size;
+dtrace_provider_id_t	fbt_id;
+fbt_probe_t				**fbt_probetab;
+int						fbt_probetab_mask;
+static int				fbt_verbose = 0;
+
+void fbt_init( void );
+
+/*ARGSUSED*/
+static void
+fbt_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	fbt_probe_t *fbt = parg, *next, *hash, *last;
+	int ndx;
+
+	do {
+		/*
+		 * Now we need to remove this probe from the fbt_probetab.
+		 */
+		ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
+		last = NULL;
+		hash = fbt_probetab[ndx];
+
+		while (hash != fbt) {
+			ASSERT(hash != NULL);
+			last = hash;
+			hash = hash->fbtp_hashnext;
+		}
+
+		if (last != NULL) {
+			last->fbtp_hashnext = fbt->fbtp_hashnext;
+		} else {
+			fbt_probetab[ndx] = fbt->fbtp_hashnext;
+		}
+
+		next = fbt->fbtp_next;
+		kmem_free(fbt, sizeof (fbt_probe_t));
+
+		fbt = next;
+	} while (fbt != NULL);
+}
+
+/*ARGSUSED*/
+static void
+fbt_enable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	fbt_probe_t *fbt = parg;
+	struct modctl *ctl = fbt->fbtp_ctl;
+
+#if defined (__ppc__) || defined (__ppc64__)
+	dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
+	if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
+		if (fbt_verbose) {
+			cmn_err(CE_NOTE, "fbt_enable is failing for probe %s "
+			    "in module %s: tempDTraceIntHook already occupied.",
+			    fbt->fbtp_name, ctl->mod_modname);
+		}
+		return;
+	}
+#endif
+	
+	dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
+	if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
+		if (fbt_verbose) {
+			cmn_err(CE_NOTE, "fbt_enable is failing for probe %s "
+			    "in module %s: tempDTraceTrapHook already occupied.",
+			    fbt->fbtp_name, ctl->mod_modname);
+		}
+		return;
+	}
+
+	for (; fbt != NULL; fbt = fbt->fbtp_next)
+		(void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint, 
+								sizeof(fbt->fbtp_patchval));
+		
+	dtrace_membar_consumer();
+}
+
+/*ARGSUSED*/
+static void
+fbt_disable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	fbt_probe_t *fbt = parg;
+
+	for (; fbt != NULL; fbt = fbt->fbtp_next)
+		(void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_savedval, (vm_offset_t)fbt->fbtp_patchpoint, 
+								sizeof(fbt->fbtp_savedval));
+		
+	dtrace_membar_consumer();
+}
+
+/*ARGSUSED*/
+static void
+fbt_suspend(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	fbt_probe_t *fbt = parg;
+
+	for (; fbt != NULL; fbt = fbt->fbtp_next)
+		(void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_savedval, (vm_offset_t)fbt->fbtp_patchpoint, 
+								sizeof(fbt->fbtp_savedval));
+		
+	dtrace_membar_consumer();
+}
+
+/*ARGSUSED*/
+static void
+fbt_resume(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	fbt_probe_t *fbt = parg;
+	struct modctl *ctl = fbt->fbtp_ctl;
+
+#if defined (__ppc__) || defined (__ppc64__)
+	dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
+	if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
+		if (fbt_verbose) {
+			cmn_err(CE_NOTE, "fbt_enable is failing for probe %s "
+			    "in module %s: tempDTraceIntHook already occupied.",
+			    fbt->fbtp_name, ctl->mod_modname);
+		}
+		return;
+	}
+#endif
+	
+	dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
+	if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
+		if (fbt_verbose) {
+			cmn_err(CE_NOTE, "fbt_resume is failing for probe %s "
+			    "in module %s: tempDTraceTrapHook already occupied.",
+			    fbt->fbtp_name, ctl->mod_modname);
+		}
+		return;
+	}
+	
+	for (; fbt != NULL; fbt = fbt->fbtp_next)
+		(void)ml_nofault_copy( (vm_offset_t)&fbt->fbtp_patchval, (vm_offset_t)fbt->fbtp_patchpoint, 
+								sizeof(fbt->fbtp_patchval));
+		
+	dtrace_membar_consumer();
+}
+
+#if !defined(__APPLE__)
+/*ARGSUSED*/
+static void
+fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
+{
+	fbt_probe_t *fbt = parg;
+	struct modctl *ctl = fbt->fbtp_ctl;
+	struct module *mp = ctl->mod_mp;
+	ctf_file_t *fp = NULL, *pfp;
+	ctf_funcinfo_t f;
+	int error;
+	ctf_id_t argv[32], type;
+	int argc = sizeof (argv) / sizeof (ctf_id_t);
+	const char *parent;
+
+	if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
+		goto err;
+
+	if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) {
+		(void) strlcpy(desc->dtargd_native, "int",
+			       sizeof(desc->dtargd_native));
+		return;
+	}
+
+	if ((fp = ctf_modopen(mp, &error)) == NULL) {
+		/*
+		 * We have no CTF information for this module -- and therefore
+		 * no args[] information.
+		 */
+		goto err;
+	}
+
+	/*
+	 * If we have a parent container, we must manually import it.
+	 */
+	if ((parent = ctf_parent_name(fp)) != NULL) {
+		struct modctl *mod;
+
+		/*
+		 * We must iterate over all modules to find the module that
+		 * is our parent.
+		 */
+		for (mod = &modules; mod != NULL; mod = mod->mod_next) {
+			if (strcmp(mod->mod_filename, parent) == 0)
+				break;
+		}
+
+		if (mod == NULL)
+			goto err;
+
+		if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
+			goto err;
+
+		if (ctf_import(fp, pfp) != 0) {
+			ctf_close(pfp);
+			goto err;
+		}
+
+		ctf_close(pfp);
+	}
+
+	if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
+		goto err;
+
+	if (fbt->fbtp_roffset != 0) {
+		if (desc->dtargd_ndx > 1)
+			goto err;
+
+		ASSERT(desc->dtargd_ndx == 1);
+		type = f.ctc_return;
+	} else {
+		if (desc->dtargd_ndx + 1 > f.ctc_argc)
+			goto err;
+
+		if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
+			goto err;
+
+		type = argv[desc->dtargd_ndx];
+	}
+
+	if (ctf_type_name(fp, type, desc->dtargd_native,
+	    DTRACE_ARGTYPELEN) != NULL) {
+		ctf_close(fp);
+		return;
+	}
+err:
+	if (fp != NULL)
+		ctf_close(fp);
+
+	desc->dtargd_ndx = DTRACE_ARGNONE;
+}
+#endif /* __APPLE__ */
+
+static dtrace_pattr_t fbt_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pops_t fbt_pops = {
+	NULL,
+	fbt_provide_module,
+	fbt_enable,
+	fbt_disable,
+	fbt_suspend,
+	fbt_resume,
+#if !defined(__APPLE__)
+	fbt_getargdesc,
+#else
+	NULL, /* XXX where to look for xnu? */
+#endif /* __APPLE__ */
+	NULL,
+	NULL,
+	fbt_destroy
+};
+
+static void
+fbt_cleanup(dev_info_t *devi)
+{
+	dtrace_invop_remove(fbt_invop);
+	ddi_remove_minor_node(devi, NULL);
+	kmem_free(fbt_probetab, fbt_probetab_size * sizeof (fbt_probe_t *));
+	fbt_probetab = NULL;
+	fbt_probetab_mask = 0;
+}
+
+static int
+fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (fbt_probetab_size == 0)
+		fbt_probetab_size = FBT_PROBETAB_SIZE;
+
+	fbt_probetab_mask = fbt_probetab_size - 1;
+	fbt_probetab =
+	    kmem_zalloc(fbt_probetab_size * sizeof (fbt_probe_t *), KM_SLEEP);
+
+	dtrace_invop_add(fbt_invop);
+
+	if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
+	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
+	    dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
+	    &fbt_pops, NULL, &fbt_id) != 0) {
+		fbt_cleanup(devi);
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	fbt_devi = devi;
+
+	return (DDI_SUCCESS);
+}
+
+static d_open_t _fbt_open;
+
+static int
+_fbt_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+#define FBT_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw fbt_cdevsw =
+{
+	_fbt_open,		/* open */
+	eno_opcl,			/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	eno_ioctl,			/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int gDisableFBT = 0;
+struct modctl g_fbt_kernctl;
+#undef kmem_alloc /* from its binding to dt_kmem_alloc glue */
+#undef kmem_free /* from its binding to dt_kmem_free glue */
+#include <vm/vm_kern.h>
+
+void
+fbt_init( void )
+{
+
+	PE_parse_boot_arg("DisableFBT", &gDisableFBT);
+
+	if (0 == gDisableFBT)
+	{
+		int majdevno = cdevsw_add(FBT_MAJOR, &fbt_cdevsw);
+		int size = 0, header_size, round_size;
+	   	kern_return_t ret;
+		void *p, *q;
+		
+		if (majdevno < 0) {
+			printf("fbt_init: failed to allocate a major number!\n");
+			return;
+		}
+
+		/*
+		 * Capture the kernel's mach_header in its entirety and the contents of
+		 * its LINKEDIT segment (and only that segment). This is sufficient to
+		 * build all the fbt probes lazily the first time a client looks to
+		 * the fbt provider. Remeber thes on the global struct modctl g_fbt_kernctl.
+		 */
+		header_size = sizeof(struct mach_header) + _mh_execute_header.sizeofcmds;
+		p = getsegdatafromheader(&_mh_execute_header, SEG_LINKEDIT, &size);
+
+        round_size = round_page_32(header_size + size);
+		ret = kmem_alloc_pageable(kernel_map, (vm_offset_t *)&q, round_size);
+
+		if (p && (ret == KERN_SUCCESS)) {
+			struct segment_command *sgp;
+
+			bcopy( (void *)&_mh_execute_header, q, header_size);
+			bcopy( p, (char *)q + header_size, size);
+
+			sgp = getsegbynamefromheader(q, SEG_LINKEDIT);
+
+			if (sgp) {
+				sgp->vmaddr = (unsigned long)((char *)q + header_size);
+				g_fbt_kernctl.address = (vm_address_t)q;
+				g_fbt_kernctl.size = header_size + size;
+			} else {
+				kmem_free(kernel_map, (vm_offset_t)q, round_size);
+				g_fbt_kernctl.address = (vm_address_t)NULL;
+				g_fbt_kernctl.size = 0;
+			}
+		} else {
+			if (ret == KERN_SUCCESS)
+				kmem_free(kernel_map, (vm_offset_t)q, round_size);
+			g_fbt_kernctl.address = (vm_address_t)NULL;
+			g_fbt_kernctl.size = 0;
+		}
+
+		strncpy((char *)&(g_fbt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
+
+		fbt_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+
+		gDisableFBT = 1; /* Ensure this initialization occurs just one time. */
+	}
+	else
+		printf("fbt_init: DisableFBT non-zero, no FBT probes will be provided.\n");
+}
+#undef FBT_MAJOR
diff --git a/bsd/dev/dtrace/lockstat.c b/bsd/dev/dtrace/lockstat.c
new file mode 100644
index 000000000..f466c873e
--- /dev/null
+++ b/bsd/dev/dtrace/lockstat.c
@@ -0,0 +1,402 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)lockstat.c	1.11	06/03/24 SMI" */
+
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include <sys/dtrace_glue.h>
+
+#include <sys/lockstat.h>
+
+#include <kern/processor.h>
+
+#define membar_producer dtrace_membar_producer
+
+/*
+ * Hot patch values, x86
+ */
+#ifdef	__i386__
+#define	NOP	0x90
+#define	RET	0xc3
+#define LOCKSTAT_AFRAMES 1
+#elif	__ppc__
+#define	NOP	0x60000000
+#define RET	0x4e800020	/* blr */
+#define LOCKSTAT_AFRAMES 2
+#else
+#error "not ported to this architecture"
+#endif
+
+
+typedef struct lockstat_probe {
+	const char	*lsp_func;
+	const char	*lsp_name;
+	int		lsp_probe;
+	dtrace_id_t	lsp_id;
+} lockstat_probe_t;
+
+lockstat_probe_t lockstat_probes[] =
+{
+#ifndef	__PPC__
+	/* Not implemented yet on PPC... */
+	{ LS_LCK_MTX_LOCK,	LSA_ACQUIRE,	LS_LCK_MTX_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_LOCK,	LSA_SPIN,	LS_LCK_MTX_LOCK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_MTX_TRY_LOCK,	LSA_ACQUIRE,	LS_LCK_MTX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_TRY_SPIN_LOCK, LSA_ACQUIRE, LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_UNLOCK,	LSA_RELEASE,	LS_LCK_MTX_UNLOCK_RELEASE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_EXT_LOCK,	LSA_ACQUIRE,	LS_LCK_MTX_EXT_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_EXT_LOCK,	LSA_SPIN,	LS_LCK_MTX_EXT_LOCK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_MTX_EXT_TRY_LOCK, LSA_ACQUIRE,	LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_MTX_UNLOCK,	LSA_RELEASE,	LS_LCK_MTX_EXT_UNLOCK_RELEASE, DTRACE_IDNONE },
+	{ LS_MUTEX_LOCK,	LSA_ACQUIRE,	LS_MUTEX_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_MUTEX_UNLOCK,	LSA_RELEASE,	LS_MUTEX_UNLOCK_RELEASE, DTRACE_IDNONE },
+	{ LS_MUTEX_TRY_LOCK,	LSA_ACQUIRE,	LS_MUTEX_TRY_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_MUTEX_TRY_SPIN,	LSA_ACQUIRE,	LS_MUTEX_TRY_SPIN_ACQUIRE, DTRACE_IDNONE },
+	{ LS_MUTEX_LOCK_SPIN,	LSA_ACQUIRE,	LS_MUTEX_LOCK_SPIN_ACQUIRE, DTRACE_IDNONE },
+#endif
+	{ LS_LCK_MTX_LOCK,	LSA_BLOCK,	LS_LCK_MTX_LOCK_BLOCK, DTRACE_IDNONE },
+	{ LS_LCK_MTX_EXT_LOCK,	LSA_BLOCK,	LS_LCK_MTX_EXT_LOCK_BLOCK, DTRACE_IDNONE },
+
+	{ LS_MUTEX_CONVERT_SPIN,	LSA_ACQUIRE,	LS_MUTEX_CONVERT_SPIN_ACQUIRE, DTRACE_IDNONE },
+
+	{ LS_LCK_RW_LOCK_SHARED,	LSR_ACQUIRE,	LS_LCK_RW_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_SHARED,	LSR_BLOCK,	LS_LCK_RW_LOCK_SHARED_BLOCK, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_SHARED,	LSR_SPIN,	LS_LCK_RW_LOCK_SHARED_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_EXCL,		LSR_ACQUIRE,	LS_LCK_RW_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_EXCL,		LSR_BLOCK,	LS_LCK_RW_LOCK_EXCL_BLOCK, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_EXCL,		LSR_SPIN,	LS_LCK_RW_LOCK_EXCL_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_RW_DONE,		LSR_RELEASE,	LS_LCK_RW_DONE_RELEASE, DTRACE_IDNONE },
+	{ LS_LCK_RW_TRY_LOCK_SHARED,	LSR_ACQUIRE,	LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_RW_TRY_LOCK_EXCL,	LSR_ACQUIRE,	LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_SHARED_TO_EXCL, LSR_UPGRADE,	LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_SHARED_TO_EXCL,	LSR_BLOCK,	LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_SHARED_TO_EXCL,	LSR_SPIN,	LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_RW_LOCK_EXCL_TO_SHARED,	LSR_DOWNGRADE,	LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, DTRACE_IDNONE },
+
+
+#ifdef	LATER
+	/* Interlock and spinlock measurements would be nice, but later */
+	{ LS_LCK_SPIN_LOCK,	LSS_ACQUIRE,	LS_LCK_SPIN_LOCK_ACQUIRE, DTRACE_IDNONE },
+	{ LS_LCK_SPIN_LOCK,	LSS_SPIN,	LS_LCK_SPIN_LOCK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_SPIN_UNLOCK,	LSS_RELEASE,	LS_LCK_SPIN_UNLOCK_RELEASE, DTRACE_IDNONE },
+
+	{ LS_LCK_RW_LOCK_EXCL_TO_SHARED,	LSA_ILK_SPIN,	LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_MTX_LOCK,	LSA_ILK_SPIN,	LS_LCK_MTX_LOCK_ILK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_MTX_EXT_LOCK,	LSA_ILK_SPIN,	LS_LCK_MTX_EXT_LOCK_ILK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_RW_TRY_LOCK_EXCL,	LSA_ILK_SPIN,	LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN, DTRACE_IDNONE },
+	{ LS_LCK_RW_TRY_LOCK_SHARED,	LSA_SPIN,	LS_LCK_RW_TRY_LOCK_SHARED_SPIN, DTRACE_IDNONE },
+#endif
+
+	{ NULL }
+};
+
+dtrace_id_t lockstat_probemap[LS_NPROBES];
+
+extern void lck_mtx_lock_lockstat_patch_point();
+extern void lck_mtx_try_lock_lockstat_patch_point();
+extern void lck_mtx_try_lock_spin_lockstat_patch_point();
+extern void lck_mtx_unlock_lockstat_patch_point();
+extern void lck_mtx_unlock2_lockstat_patch_point();
+extern void mutex_lock_lockstat_patch_point();
+extern void mutex_unlock_lockstat_patch_point();
+extern void mutex_unlock2_lockstat_patch_point();
+extern void mutex_try_lockstat_patch_point();
+extern void mutex_try_spin_lockstat_patch_point();
+extern void mutex_lock_spin_lockstat_patch_point();
+extern void mutex_convert_spin_lockstat_patch_point();
+extern void lck_rw_done_lockstat_patch_point();
+extern void lck_rw_lock_shared_lockstat_patch_point();
+extern void lck_mtx_lock_ext_lockstat_patch_point();
+extern void lck_mtx_ext_unlock_lockstat_patch_point();
+
+vm_offset_t *assembly_probes[] = {
+#if	defined(__i386__)
+	/*
+	 * On x86 these points are better done via hot patches, which ensure
+	 * there is zero overhead when not in use.  On x86 these patch points
+	 * are swapped between the return instruction and a no-op, with the
+	 * Dtrace call following the return.
+	 */ 
+	(vm_offset_t *) lck_mtx_lock_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_try_lock_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_try_lock_spin_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_unlock_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_unlock2_lockstat_patch_point,
+	(vm_offset_t *) lck_rw_lock_shared_lockstat_patch_point,
+	(vm_offset_t *) lck_rw_done_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_lock_ext_lockstat_patch_point,
+	(vm_offset_t *) lck_mtx_ext_unlock_lockstat_patch_point,
+	(vm_offset_t *) mutex_lock_lockstat_patch_point,
+	(vm_offset_t *) mutex_try_spin_lockstat_patch_point,
+	(vm_offset_t *) mutex_try_lockstat_patch_point,
+	(vm_offset_t *) mutex_unlock_lockstat_patch_point,
+	(vm_offset_t *) mutex_unlock2_lockstat_patch_point,
+	(vm_offset_t *) mutex_lock_spin_lockstat_patch_point,
+	(vm_offset_t *) mutex_convert_spin_lockstat_patch_point,
+#endif
+	(vm_offset_t *) lck_mtx_unlock_lockstat_patch_point,
+	NULL
+};
+/*
+ * Hot patch switches back and forth the probe points between NOP and RET.
+ * The argument indicates whether the probe point is on or off.
+ */
+void lockstat_hot_patch(boolean_t active)
+{
+	int i;
+
+
+	for (i = 0; assembly_probes[i]; i++) {
+#ifdef __i386__
+		uint8_t instr;
+		instr = (active ? NOP : RET );
+		(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i]), 
+								sizeof(instr));
+#endif
+#ifdef __ppc__
+		uint32_t instr;
+		instr = (active ? NOP : RET );
+		(void) ml_nofault_copy( (vm_offset_t)&instr, *(assembly_probes[i]), sizeof(instr));
+#endif
+	}
+}
+
+
+
+void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
+				    uint64_t, uint64_t, uint64_t);
+
+/*
+ * An initial value for lockstat_probe. See lockstat_attach(). Think safety.
+ */
+static void
+lockstat_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
+				    uint64_t arg2, uint64_t arg3, uint64_t arg4)
+{
+#pragma unused(id,arg0,arg1,arg2,arg3,arg4)
+}
+
+
+static dev_info_t	*lockstat_devi;	/* saved in xxattach() for xxinfo() */
+static dtrace_provider_id_t lockstat_id;
+
+/*ARGSUSED*/
+static void
+lockstat_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	lockstat_probe_t *probe = parg;
+
+	ASSERT(!lockstat_probemap[probe->lsp_probe]);
+
+	lockstat_probemap[probe->lsp_probe] = id;
+	membar_producer();
+
+	lockstat_probe = dtrace_probe;
+	membar_producer();
+
+	lockstat_hot_patch(TRUE);
+	membar_producer();
+
+}
+
+/*ARGSUSED*/
+static void
+lockstat_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	lockstat_probe_t *probe = parg;
+	int i;
+
+	ASSERT(lockstat_probemap[probe->lsp_probe]);
+
+	lockstat_probemap[probe->lsp_probe] = 0;
+	lockstat_hot_patch(FALSE);
+	membar_producer();
+
+	/*
+	 * See if we have any probes left enabled.
+	 */
+	for (i = 0; i < LS_NPROBES; i++) {
+		if (lockstat_probemap[i]) {
+			/*
+			 * This probe is still enabled.  We don't need to deal
+			 * with waiting for all threads to be out of the
+			 * lockstat critical sections; just return.
+			 */
+			return;
+		}
+	}
+
+}
+
+/*ARGSUSED*/
+static void
+lockstat_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+	int i = 0;
+
+	for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
+		lockstat_probe_t *probe = &lockstat_probes[i];
+
+		if (dtrace_probe_lookup(lockstat_id, "mach_kernel",
+		    probe->lsp_func, probe->lsp_name) != 0)
+			continue;
+
+		ASSERT(!probe->lsp_id);
+		probe->lsp_id = dtrace_probe_create(lockstat_id,
+		    "mach_kernel", probe->lsp_func, probe->lsp_name,
+		    LOCKSTAT_AFRAMES, probe);
+	}
+}
+
+
+/*ARGSUSED*/
+static void
+lockstat_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+	lockstat_probe_t *probe = parg;
+
+	ASSERT(!lockstat_probemap[probe->lsp_probe]);
+	probe->lsp_id = 0;
+}
+
+static dtrace_pattr_t lockstat_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+};
+
+static dtrace_pops_t lockstat_pops = {
+	lockstat_provide,
+	NULL,
+	lockstat_enable,
+	lockstat_disable,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	lockstat_destroy
+};
+
+static int
+lockstat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (ddi_create_minor_node(devi, "lockstat", S_IFCHR, 0,
+	    DDI_PSEUDO, 0) == DDI_FAILURE ||
+	    dtrace_register("lockstat", &lockstat_attr, DTRACE_PRIV_KERNEL,
+	    NULL, &lockstat_pops, NULL, &lockstat_id) != 0) {
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	lockstat_devi = devi;
+	lockstat_probe = lockstat_stub;
+	return (DDI_SUCCESS);
+}
+
+d_open_t _lockstat_open;
+
+int _lockstat_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+#define LOCKSTAT_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw lockstat_cdevsw =
+{
+	_lockstat_open,		/* open */
+	eno_opcl,			/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	eno_ioctl,			/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int gLockstatInited = 0;
+
+void lockstat_init( void );
+
+void lockstat_init( void )
+{
+	if (0 == gLockstatInited)
+	{
+		int majdevno = cdevsw_add(LOCKSTAT_MAJOR, &lockstat_cdevsw);
+		
+		if (majdevno < 0) {
+			printf("lockstat_init: failed to allocate a major number!\n");
+			gLockstatInited = 0;
+			return;
+		}
+
+		lockstat_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+		gLockstatInited = 1;
+	} else
+		panic("lockstat_init: called twice!\n");
+}
+#undef LOCKSTAT_MAJOR
diff --git a/bsd/dev/dtrace/profile_prvd.c b/bsd/dev/dtrace/profile_prvd.c
new file mode 100644
index 000000000..249060cd3
--- /dev/null
+++ b/bsd/dev/dtrace/profile_prvd.c
@@ -0,0 +1,840 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)profile.c	1.6	06/03/24 SMI" */
+
+#if !defined(__APPLE__)
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/modctl.h>
+#include <sys/conf.h>
+#include <sys/systm.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/cpuvar.h>
+#include <sys/kmem.h>
+#include <sys/strsubr.h>
+#include <sys/dtrace.h>
+#include <sys/cyclic.h>
+#include <sys/atomic.h>
+#else
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/cpu_data.h>
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include <sys/dtrace_glue.h>
+
+#if defined(__ppc__) || defined(__ppc64__)
+extern struct savearea *find_kern_regs(thread_t);
+#elif defined(__i386__) || defined(__x86_64__)
+extern x86_saved_state32_t *find_kern_regs(thread_t);
+#elif defined (__arm__)
+extern struct arm_saved_state *find_kern_regs(thread_t);
+#else
+#error Unknown architecture
+#endif
+
+#undef ASSERT
+#define ASSERT(x) do {} while(0)
+
+extern void profile_init(void);
+#endif /* __APPLE__ */
+
+static dev_info_t *profile_devi;
+static dtrace_provider_id_t profile_id;
+
+/*
+ * Regardless of platform, there are five artificial frames in the case of the
+ * profile provider:
+ *
+ *	profile_fire
+ *	cyclic_expire
+ *	cyclic_fire
+ *	[ cbe ]
+ *	[ locore ]
+ *
+ * On amd64, there are two frames associated with locore:  one in locore, and
+ * another in common interrupt dispatch code.  (i386 has not been modified to
+ * use this common layer.)  Further, on i386, the interrupted instruction
+ * appears as its own stack frame.  All of this means that we need to add one
+ * frame for amd64, and then take one away for both amd64 and i386.
+ *
+ * On SPARC, the picture is further complicated because the compiler
+ * optimizes away tail-calls -- so the following frames are optimized away:
+ *
+ * 	profile_fire
+ *	cyclic_expire
+ *
+ * This gives three frames.  However, on DEBUG kernels, the cyclic_expire
+ * frame cannot be tail-call eliminated, yielding four frames in this case.
+ *
+ * All of the above constraints lead to the mess below.  Yes, the profile
+ * provider should ideally figure this out on-the-fly by hiting one of its own
+ * probes and then walking its own stack trace.  This is complicated, however,
+ * and the static definition doesn't seem to be overly brittle.  Still, we
+ * allow for a manual override in case we get it completely wrong.
+ */
+#if !defined(__APPLE__)
+
+#ifdef __x86_64__
+#define	PROF_ARTIFICIAL_FRAMES	7
+#else
+#ifdef __i386__
+#define	PROF_ARTIFICIAL_FRAMES	6
+#else
+#ifdef __sparc
+#ifdef DEBUG
+#define	PROF_ARTIFICIAL_FRAMES	4
+#else
+#define	PROF_ARTIFICIAL_FRAMES	3
+#endif
+#endif
+#endif
+#endif
+
+#else /* is Mac OS X */
+
+#if defined(__ppc__) || defined(__ppc64__)
+#define PROF_ARTIFICIAL_FRAMES 8
+#elif defined(__i386__) || defined(__x86_64__)
+#define PROF_ARTIFICIAL_FRAMES  9
+#elif defined(__arm__)
+#define PROF_ARTIFICIAL_FRAMES 3 /* XXX BOGUS ARMTODO */
+#else
+#error Unknown architecture
+#endif
+
+#endif /* __APPLE__ */
+
+#define	PROF_NAMELEN		15
+
+#define	PROF_PROFILE		0
+#define	PROF_TICK		1
+#define	PROF_PREFIX_PROFILE	"profile-"
+#define	PROF_PREFIX_TICK	"tick-"
+
+typedef struct profile_probe {
+	char		prof_name[PROF_NAMELEN];
+	dtrace_id_t	prof_id;
+	int		prof_kind;
+	hrtime_t	prof_interval;
+	cyclic_id_t	prof_cyclic;
+} profile_probe_t;
+
+typedef struct profile_probe_percpu {
+	hrtime_t	profc_expected;
+	hrtime_t	profc_interval;
+	profile_probe_t	*profc_probe;
+} profile_probe_percpu_t;
+
+hrtime_t	profile_interval_min = NANOSEC / 5000;		/* 5000 hz */
+int		profile_aframes = 0;				/* override */
+
+static int profile_rates[] = {
+    97, 199, 499, 997, 1999,
+    4001, 4999, 0, 0, 0,
+    0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0
+};
+
+static int profile_ticks[] = {
+    1, 10, 100, 500, 1000,
+    5000, 0, 0, 0, 0,
+    0, 0, 0, 0, 0
+};
+
+/*
+ * profile_max defines the upper bound on the number of profile probes that
+ * can exist (this is to prevent malicious or clumsy users from exhausing
+ * system resources by creating a slew of profile probes). At mod load time,
+ * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's
+ * present in the profile.conf file.
+ */
+#define	PROFILE_MAX_DEFAULT	1000	/* default max. number of probes */
+static uint32_t profile_max;		/* maximum number of profile probes */
+static uint32_t profile_total;	/* current number of profile probes */
+
+static void
+profile_fire(void *arg)
+{
+	profile_probe_percpu_t *pcpu = arg;
+	profile_probe_t *prof = pcpu->profc_probe;
+	hrtime_t late;
+
+	late = dtrace_gethrtime() - pcpu->profc_expected;
+	pcpu->profc_expected += pcpu->profc_interval;
+
+#if !defined(__APPLE__)
+	dtrace_probe(prof->prof_id, CPU->cpu_profile_pc,
+	    CPU->cpu_profile_upc, late, 0, 0);
+#else
+#if defined(__ppc__) || defined(__ppc64__)
+	struct savearea *sv = find_kern_regs(current_thread());
+
+	if (sv) {
+		if (USERMODE(sv->save_srr1)) {
+			dtrace_probe(prof->prof_id, 0x0, sv->save_srr0, late, 0, 0);
+		} else {
+			dtrace_probe(prof->prof_id, sv->save_srr0, 0x0, late, 0, 0);
+		}
+	} else {
+		dtrace_probe(prof->prof_id, 0xcafebabe,
+	    	0x0, late, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */
+	}
+#elif defined(__i386__) || defined(__x86_64__)
+	x86_saved_state32_t *kern_regs = find_kern_regs(current_thread());
+
+	if (NULL != kern_regs) {
+		/* Kernel was interrupted. */
+		dtrace_probe(prof->prof_id, kern_regs->eip,  0x0, 0, 0, 0);
+	} else {
+		/* Possibly a user interrupt */
+		x86_saved_state_t   *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
+
+		if (NULL == tagged_regs) {
+			/* Too bad, so sad, no useful interrupt state. */
+			dtrace_probe(prof->prof_id, 0xcafebabe,
+	    		0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */
+		} else if (is_saved_state64(tagged_regs)) {
+			x86_saved_state64_t *regs = saved_state64(tagged_regs);
+
+			dtrace_probe(prof->prof_id, 0x0, regs->isf.rip, 0, 0, 0);
+		} else {
+			x86_saved_state32_t *regs = saved_state32(tagged_regs);
+
+			dtrace_probe(prof->prof_id, 0x0, regs->eip, 0, 0, 0);
+		}	
+	}
+#else
+#error Unknown architecture
+#endif
+#endif /* __APPLE__ */
+}
+
+static void
+profile_tick(void *arg)
+{
+	profile_probe_t *prof = arg;
+
+#if !defined(__APPLE__)
+	dtrace_probe(prof->prof_id, CPU->cpu_profile_pc,
+	    CPU->cpu_profile_upc, 0, 0, 0);
+#else
+#if defined(__ppc__) || defined(__ppc64__)
+	struct savearea *sv = find_kern_regs(current_thread());
+
+	if (sv) {
+		if (USERMODE(sv->save_srr1)) {
+			dtrace_probe(prof->prof_id, 0x0, sv->save_srr0, 0, 0, 0);
+		} else {
+			dtrace_probe(prof->prof_id, sv->save_srr0, 0x0, 0, 0, 0);
+		}
+	} else {
+		dtrace_probe(prof->prof_id, 0xcafebabe,
+	    	0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */
+	}
+#elif defined(__i386__) || defined(__x86_64__)
+	x86_saved_state32_t *kern_regs = find_kern_regs(current_thread());
+
+	if (NULL != kern_regs) {
+		/* Kernel was interrupted. */
+		dtrace_probe(prof->prof_id, kern_regs->eip,  0x0, 0, 0, 0);
+	} else {
+		/* Possibly a user interrupt */
+		x86_saved_state_t   *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
+
+		if (NULL == tagged_regs) {
+			/* Too bad, so sad, no useful interrupt state. */
+			dtrace_probe(prof->prof_id, 0xcafebabe,
+	    		0x0, 0, 0, 0); /* XXX_BOGUS also see profile_usermode() below. */
+		} else if (is_saved_state64(tagged_regs)) {
+			x86_saved_state64_t *regs = saved_state64(tagged_regs);
+
+			dtrace_probe(prof->prof_id, 0x0, regs->isf.rip, 0, 0, 0);
+		} else {
+			x86_saved_state32_t *regs = saved_state32(tagged_regs);
+
+			dtrace_probe(prof->prof_id, 0x0, regs->eip, 0, 0, 0);
+		}	
+	}
+#else
+#error Unknown architecture
+#endif
+#endif /* __APPLE__ */
+}
+
+static void
+profile_create(hrtime_t interval, const char *name, int kind)
+{
+	profile_probe_t *prof;
+
+	if (interval < profile_interval_min)
+		return;
+
+	if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
+		return;
+
+	atomic_add_32(&profile_total, 1);
+	if (profile_total > profile_max) {
+		atomic_add_32(&profile_total, -1);
+		return;
+	}
+
+#if !defined(__APPLE__)
+	prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
+#else
+	if (PROF_TICK == kind)
+		prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
+	else
+		prof = kmem_zalloc(sizeof (profile_probe_t) + NCPU*sizeof(profile_probe_percpu_t), KM_SLEEP);
+#endif /* __APPLE__ */
+	(void) strlcpy(prof->prof_name, name, sizeof(prof->prof_name));
+	prof->prof_interval = interval;
+	prof->prof_cyclic = CYCLIC_NONE;
+	prof->prof_kind = kind;
+	prof->prof_id = dtrace_probe_create(profile_id,
+	    NULL, NULL, name,
+	    profile_aframes ? profile_aframes : PROF_ARTIFICIAL_FRAMES, prof);
+}
+
+/*ARGSUSED*/
+static void
+profile_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+	int i, j, rate, kind;
+	hrtime_t val = 0, mult = 1, len;
+	const char *name, *suffix = NULL;
+
+	const struct {
+		char *prefix;
+		int kind;
+	} types[] = {
+		{ PROF_PREFIX_PROFILE, PROF_PROFILE },
+		{ PROF_PREFIX_TICK, PROF_TICK },
+		{ NULL, NULL }
+	};
+
+	const struct {
+		char *name;
+		hrtime_t mult;
+	} suffixes[] = {
+		{ "ns", 	NANOSEC / NANOSEC },
+		{ "nsec",	NANOSEC / NANOSEC },
+		{ "us",		NANOSEC / MICROSEC },
+		{ "usec",	NANOSEC / MICROSEC },
+		{ "ms",		NANOSEC / MILLISEC },
+		{ "msec",	NANOSEC / MILLISEC },
+		{ "s",		NANOSEC / SEC },
+		{ "sec",	NANOSEC / SEC },
+		{ "m",		NANOSEC * (hrtime_t)60 },
+		{ "min",	NANOSEC * (hrtime_t)60 },
+		{ "h",		NANOSEC * (hrtime_t)(60 * 60) },
+		{ "hour",	NANOSEC * (hrtime_t)(60 * 60) },
+		{ "d",		NANOSEC * (hrtime_t)(24 * 60 * 60) },
+		{ "day",	NANOSEC * (hrtime_t)(24 * 60 * 60) },
+		{ "hz",		0 },
+#if !defined(__APPLE__)
+		{ NULL }
+#else
+		{ NULL, 0 }
+#endif /* __APPLE__ */
+	};
+
+	if (desc == NULL) {
+		char n[PROF_NAMELEN];
+
+		/*
+		 * If no description was provided, provide all of our probes.
+		 */
+		for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) {
+			if ((rate = profile_rates[i]) == 0)
+				continue;
+
+			(void) snprintf(n, PROF_NAMELEN, "%s%d",
+			    PROF_PREFIX_PROFILE, rate);
+			profile_create(NANOSEC / rate, n, PROF_PROFILE);
+		}
+
+		for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) {
+			if ((rate = profile_ticks[i]) == 0)
+				continue;
+
+			(void) snprintf(n, PROF_NAMELEN, "%s%d",
+			    PROF_PREFIX_TICK, rate);
+			profile_create(NANOSEC / rate, n, PROF_TICK);
+		}
+
+		return;
+	}
+
+	name = desc->dtpd_name;
+
+	for (i = 0; types[i].prefix != NULL; i++) {
+		len = strlen(types[i].prefix);
+
+		if (strncmp(name, types[i].prefix, len) != 0)
+			continue;
+		break;
+	}
+
+	if (types[i].prefix == NULL)
+		return;
+
+	kind = types[i].kind;
+	j = strlen(name) - len;
+
+	/*
+	 * We need to start before any time suffix.
+	 */
+	for (j = strlen(name); j >= len; j--) {
+		if (name[j] >= '0' && name[j] <= '9')
+			break;
+		suffix = &name[j];
+	}
+
+	ASSERT(suffix != NULL);
+
+	/*
+	 * Now determine the numerical value present in the probe name.
+	 */
+	for (; j >= len; j--) {
+		if (name[j] < '0' || name[j] > '9')
+			return;
+
+		val += (name[j] - '0') * mult;
+		mult *= (hrtime_t)10;
+	}
+
+	if (val == 0)
+		return;
+
+	/*
+	 * Look-up the suffix to determine the multiplier.
+	 */
+	for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
+		if (strcasecmp(suffixes[i].name, suffix) == 0) {
+			mult = suffixes[i].mult;
+			break;
+		}
+	}
+
+	if (suffixes[i].name == NULL && *suffix != '\0')
+		return;
+
+	if (mult == 0) {
+		/*
+		 * The default is frequency-per-second.
+		 */
+		val = NANOSEC / val;
+	} else {
+		val *= mult;
+	}
+
+	profile_create(val, name, kind);
+}
+
+/*ARGSUSED*/
+static void
+profile_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+	profile_probe_t *prof = parg;
+
+	ASSERT(prof->prof_cyclic == CYCLIC_NONE);
+#if !defined(__APPLE__)
+	kmem_free(prof, sizeof (profile_probe_t));
+#else
+	if (prof->prof_kind == PROF_TICK)
+		kmem_free(prof, sizeof (profile_probe_t));
+	else
+		kmem_free(prof, sizeof (profile_probe_t) + NCPU*sizeof(profile_probe_percpu_t));
+#endif /* __APPLE__ */
+
+	ASSERT(profile_total >= 1);
+	atomic_add_32(&profile_total, -1);
+}
+
+/*ARGSUSED*/
+static void
+profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
+{
+	profile_probe_t *prof = arg;
+	profile_probe_percpu_t *pcpu;
+
+#if !defined(__APPLE__)
+	pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP);
+#else
+	pcpu = ((profile_probe_percpu_t *)(&(prof[1]))) + cpu_number();
+#endif /* __APPLE__ */
+	pcpu->profc_probe = prof;
+
+	hdlr->cyh_func = profile_fire;
+	hdlr->cyh_arg = pcpu;
+	hdlr->cyh_level = CY_HIGH_LEVEL;
+
+	when->cyt_interval = prof->prof_interval;
+#if !defined(__APPLE__)
+	when->cyt_when = dtrace_gethrtime() + when->cyt_interval;
+#else
+	when->cyt_when = 0;
+#endif /* __APPLE__ */
+
+	pcpu->profc_expected = when->cyt_when;
+	pcpu->profc_interval = when->cyt_interval;
+}
+
+/*ARGSUSED*/
+static void
+profile_offline(void *arg, cpu_t *cpu, void *oarg)
+{
+	profile_probe_percpu_t *pcpu = oarg;
+
+	ASSERT(pcpu->profc_probe == arg);
+#if !defined(__APPLE__)
+	kmem_free(pcpu, sizeof (profile_probe_percpu_t));
+#endif /* __APPLE__ */
+}
+
+/*ARGSUSED*/
+static void
+profile_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	profile_probe_t *prof = parg;
+	cyc_omni_handler_t omni;
+	cyc_handler_t hdlr;
+	cyc_time_t when;
+
+	ASSERT(prof->prof_interval != 0);
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+	if (prof->prof_kind == PROF_TICK) {
+		hdlr.cyh_func = profile_tick;
+		hdlr.cyh_arg = prof;
+		hdlr.cyh_level = CY_HIGH_LEVEL;
+
+		when.cyt_interval = prof->prof_interval;
+#if !defined(__APPLE__)
+		when.cyt_when = dtrace_gethrtime() + when.cyt_interval;
+#else
+		when.cyt_when = 0;
+#endif /* __APPLE__ */
+	} else {
+		ASSERT(prof->prof_kind == PROF_PROFILE);
+		omni.cyo_online = profile_online;
+		omni.cyo_offline = profile_offline;
+		omni.cyo_arg = prof;
+	}
+
+#if !defined(__APPLE__)
+	if (prof->prof_kind == PROF_TICK) {
+		prof->prof_cyclic = cyclic_add(&hdlr, &when);
+	} else {
+		prof->prof_cyclic = cyclic_add_omni(&omni);
+	}
+#else
+	if (prof->prof_kind == PROF_TICK) {
+		prof->prof_cyclic = cyclic_timer_add(&hdlr, &when);
+	} else {
+		prof->prof_cyclic = (cyclic_id_t)cyclic_add_omni(&omni); /* cast puns cyclic_id_list_t with cyclic_id_t */
+	}
+#endif /* __APPLE__ */
+}
+
+/*ARGSUSED*/
+static void
+profile_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	profile_probe_t *prof = parg;
+
+	ASSERT(prof->prof_cyclic != CYCLIC_NONE);
+	ASSERT(MUTEX_HELD(&cpu_lock));
+
+#if !defined(__APPLE__)
+	cyclic_remove(prof->prof_cyclic);
+#else
+	if (prof->prof_kind == PROF_TICK) {
+		cyclic_timer_remove(prof->prof_cyclic);
+	} else {
+		cyclic_remove_omni((cyclic_id_list_t)prof->prof_cyclic); /* cast puns cyclic_id_list_t with cyclic_id_t */
+	}
+#endif /* __APPLE__ */
+	prof->prof_cyclic = CYCLIC_NONE;
+}
+
+#if !defined(__APPLE__)
+/*ARGSUSED*/
+static int
+profile_usermode(void *arg, dtrace_id_t id, void *parg)
+{
+	return (CPU->cpu_profile_pc == 0);
+}
+#else
+static int
+profile_usermode(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id,parg)
+	return 1; /* XXX_BOGUS */
+}
+#endif /* __APPLE__ */
+
+static dtrace_pattr_t profile_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+};
+
+static dtrace_pops_t profile_pops = {
+	profile_provide,
+	NULL,
+	profile_enable,
+	profile_disable,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	profile_usermode,
+	profile_destroy
+};
+
+static int
+profile_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (ddi_create_minor_node(devi, "profile", S_IFCHR, 0,
+	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
+	    dtrace_register("profile", &profile_attr,
+	    DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER, NULL,
+	    &profile_pops, NULL, &profile_id) != 0) {
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+#if !defined(__APPLE__)
+	profile_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
+	    "profile-max-probes", PROFILE_MAX_DEFAULT);
+#else
+	profile_max = PROFILE_MAX_DEFAULT;
+#endif /* __APPLE__ */
+
+	ddi_report_dev(devi);
+	profile_devi = devi;
+	return (DDI_SUCCESS);
+}
+
+#if !defined(__APPLE__)
+static int
+profile_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_DETACH:
+		break;
+	case DDI_SUSPEND:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (dtrace_unregister(profile_id) != 0)
+		return (DDI_FAILURE);
+
+	ddi_remove_minor_node(devi, NULL);
+	return (DDI_SUCCESS);
+}
+
+/*ARGSUSED*/
+static int
+profile_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
+{
+	int error;
+
+	switch (infocmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)profile_devi;
+		error = DDI_SUCCESS;
+		break;
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		error = DDI_SUCCESS;
+		break;
+	default:
+		error = DDI_FAILURE;
+	}
+	return (error);
+}
+
+/*ARGSUSED*/
+static int
+profile_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
+{
+	return (0);
+}
+
+static struct cb_ops profile_cb_ops = {
+	profile_open,		/* open */
+	nodev,			/* close */
+	nulldev,		/* strategy */
+	nulldev,		/* print */
+	nodev,			/* dump */
+	nodev,			/* read */
+	nodev,			/* write */
+	nodev,			/* ioctl */
+	nodev,			/* devmap */
+	nodev,			/* mmap */
+	nodev,			/* segmap */
+	nochpoll,		/* poll */
+	ddi_prop_op,		/* cb_prop_op */
+	0,			/* streamtab  */
+	D_NEW | D_MP		/* Driver compatibility flag */
+};
+
+static struct dev_ops profile_ops = {
+	DEVO_REV,		/* devo_rev, */
+	0,			/* refcnt  */
+	profile_info,		/* get_dev_info */
+	nulldev,		/* identify */
+	nulldev,		/* probe */
+	profile_attach,		/* attach */
+	profile_detach,		/* detach */
+	nodev,			/* reset */
+	&profile_cb_ops,	/* driver operations */
+	NULL,			/* bus operations */
+	nodev			/* dev power */
+};
+
+/*
+ * Module linkage information for the kernel.
+ */
+static struct modldrv modldrv = {
+	&mod_driverops,		/* module type (this is a pseudo driver) */
+	"Profile Interrupt Tracing",	/* name of module */
+	&profile_ops,		/* driver ops */
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	(void *)&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	return (mod_install(&modlinkage));
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+int
+_fini(void)
+{
+	return (mod_remove(&modlinkage));
+}
+#else
+d_open_t _profile_open;
+
+int _profile_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+#define PROFILE_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw profile_cdevsw =
+{
+	_profile_open,		/* open */
+	eno_opcl,			/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	eno_ioctl,			/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int gProfileInited = 0;
+
+void profile_init( void )
+{
+	if (0 == gProfileInited)
+	{
+		int majdevno = cdevsw_add(PROFILE_MAJOR, &profile_cdevsw);
+		
+		if (majdevno < 0) {
+			printf("profile_init: failed to allocate a major number!\n");
+			gProfileInited = 0;
+			return;
+		}
+
+		profile_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+
+		gProfileInited = 1;
+	} else
+		panic("profile_init: called twice!\n");
+}
+#undef PROFILE_MAJOR
+#endif /* __APPLE__ */
diff --git a/bsd/dev/dtrace/sdt.c b/bsd/dev/dtrace/sdt.c
new file mode 100644
index 000000000..640bfae34
--- /dev/null
+++ b/bsd/dev/dtrace/sdt.c
@@ -0,0 +1,673 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)sdt.c	1.6	06/03/24 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include <sys/dtrace_glue.h>
+
+#include <sys/sdt_impl.h>
+
+struct savearea_t; /* Used anonymously */
+typedef kern_return_t (*perfCallback)(int, struct savearea_t *, int, int);
+
+#if defined (__ppc__) || defined (__ppc64__)
+extern perfCallback tempDTraceTrapHook, tempDTraceIntHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
+extern kern_return_t fbt_perfIntCallback(int, struct savearea_t *, int, int);
+
+#define	SDT_PATCHVAL	0x7c810808
+#define SDT_AFRAMES     6
+#elif defined(__i386__) || defined(__x86_64__)
+extern perfCallback tempDTraceTrapHook;
+extern kern_return_t fbt_perfCallback(int, struct savearea_t *, int, int);
+
+#define	SDT_PATCHVAL	0xf0
+#define	SDT_AFRAMES		6
+#else
+#error Unknown architecture
+#endif
+
+#define	SDT_PROBETAB_SIZE	0x1000		/* 4k entries -- 16K total */
+
+static dev_info_t		*sdt_devi;
+static int			sdt_verbose = 0;
+sdt_probe_t		**sdt_probetab;
+int			sdt_probetab_size;
+int			sdt_probetab_mask;
+
+/*ARGSUSED*/
+static void
+__sdt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(arg)
+	struct module *mp = (struct module *)ctl->address;
+	char *modname = ctl->mod_modname;
+	sdt_probedesc_t *sdpd;
+	sdt_probe_t *sdp, *old;
+	sdt_provider_t *prov;
+	int len;
+
+	/*
+	 * One for all, and all for one:  if we haven't yet registered all of
+	 * our providers, we'll refuse to provide anything.
+	 */
+	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
+		if (prov->sdtp_id == DTRACE_PROVNONE)
+			return;
+	}
+
+	if (mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
+		return;
+
+	for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
+		char *name = sdpd->sdpd_name, *func, *nname;
+		int i, j;
+		dtrace_id_t id;
+
+		for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
+			char *prefpart, *prefix = prov->sdtp_prefix;
+
+			if ((prefpart = strstr(name, prefix))) {
+				name = prefpart + strlen(prefix);
+				break;
+			}
+		}
+
+		nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
+
+		for (i = 0, j = 0; name[j] != '\0'; i++) {
+			if (name[j] == '_' && name[j + 1] == '_') {
+				nname[i] = '-';
+				j += 2;
+			} else {
+				nname[i] = name[j++];
+			}
+		}
+
+		nname[i] = '\0';
+
+		sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
+		sdp->sdp_loadcnt = ctl->mod_loadcnt;
+		sdp->sdp_ctl = ctl;
+		sdp->sdp_name = nname;
+		sdp->sdp_namelen = len;
+		sdp->sdp_provider = prov;
+
+		func = sdpd->sdpd_func;
+
+		if (func == NULL)
+			func = "<unknown>";
+
+		/*
+		 * We have our provider.  Now create the probe.
+		 */
+		if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
+		    func, nname)) != DTRACE_IDNONE) {
+			old = dtrace_probe_arg(prov->sdtp_id, id);
+			ASSERT(old != NULL);
+
+			sdp->sdp_next = old->sdp_next;
+			sdp->sdp_id = id;
+			old->sdp_next = sdp;
+		} else {
+			sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
+			    modname, func, nname, SDT_AFRAMES, sdp);
+
+			mp->sdt_nprobes++;
+		}
+
+		sdp->sdp_hashnext =
+		    sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
+		sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
+
+		sdp->sdp_patchval = SDT_PATCHVAL;
+		sdp->sdp_patchpoint = (sdt_instr_t *)sdpd->sdpd_offset;
+		sdp->sdp_savedval = *sdp->sdp_patchpoint;
+	}
+}
+
+/*ARGSUSED*/
+static void
+sdt_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	sdt_probe_t *sdp = parg, *old, *last, *hash;
+	int ndx;
+#if !defined(__APPLE__)
+	struct modctl *ctl = sdp->sdp_ctl;
+
+	if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
+		if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
+		    ctl->mod_loaded)) {
+			((struct module *)(ctl->mod_mp))->sdt_nprobes--;
+		}
+	}
+#endif /* __APPLE__ */
+
+	while (sdp != NULL) {
+		old = sdp;
+
+		/*
+		 * Now we need to remove this probe from the sdt_probetab.
+		 */
+		ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
+		last = NULL;
+		hash = sdt_probetab[ndx];
+
+		while (hash != sdp) {
+			ASSERT(hash != NULL);
+			last = hash;
+			hash = hash->sdp_hashnext;
+		}
+
+		if (last != NULL) {
+			last->sdp_hashnext = sdp->sdp_hashnext;
+		} else {
+			sdt_probetab[ndx] = sdp->sdp_hashnext;
+		}
+
+		kmem_free(sdp->sdp_name, sdp->sdp_namelen);
+		sdp = sdp->sdp_next;
+		kmem_free(old, sizeof (sdt_probe_t));
+	}
+}
+
+/*ARGSUSED*/
+static void
+sdt_enable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	sdt_probe_t *sdp = parg;
+	struct modctl *ctl = sdp->sdp_ctl;
+
+#if !defined(__APPLE__)
+	ctl->mod_nenabled++;
+
+	/*
+	 * If this module has disappeared since we discovered its probes,
+	 * refuse to enable it.
+	 */
+	if (!ctl->mod_loaded) {
+		if (sdt_verbose) {
+			cmn_err(CE_NOTE, "sdt is failing for probe %s "
+			    "(module %s unloaded)",
+			    sdp->sdp_name, ctl->mod_modname);
+		}
+		goto err;
+	}
+
+	/*
+	 * Now check that our modctl has the expected load count.  If it
+	 * doesn't, this module must have been unloaded and reloaded -- and
+	 * we're not going to touch it.
+	 */
+	if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
+		if (sdt_verbose) {
+			cmn_err(CE_NOTE, "sdt is failing for probe %s "
+			    "(module %s reloaded)",
+			    sdp->sdp_name, ctl->mod_modname);
+		}
+		goto err;
+	}
+#endif /* __APPLE__ */
+
+#if defined (__ppc__) || defined (__ppc64__)
+	dtrace_casptr(&tempDTraceIntHook, NULL, fbt_perfIntCallback);
+	if (tempDTraceIntHook != (perfCallback)fbt_perfIntCallback) {
+		if (sdt_verbose) {
+			cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
+			    "in module %s: tempDTraceIntHook already occupied.",
+			    sdp->sdp_name, ctl->mod_modname);
+		}
+		return;
+	}
+#endif
+	
+	dtrace_casptr(&tempDTraceTrapHook, NULL, fbt_perfCallback);
+	if (tempDTraceTrapHook != (perfCallback)fbt_perfCallback) {
+		if (sdt_verbose) {
+			cmn_err(CE_NOTE, "sdt_enable is failing for probe %s "
+			    "in module %s: tempDTraceTrapHook already occupied.",
+			    sdp->sdp_name, ctl->mod_modname);
+		}
+		return;
+	}
+
+	while (sdp != NULL) {
+		(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_patchval, (vm_offset_t)sdp->sdp_patchpoint, 
+								sizeof(sdp->sdp_patchval));
+		sdp = sdp->sdp_next;
+	}
+err:
+	;
+}
+
+/*ARGSUSED*/
+static void
+sdt_disable(void *arg, dtrace_id_t id, void *parg)
+{
+#pragma unused(arg,id)
+	sdt_probe_t *sdp = parg;
+#if !defined(__APPLE__)
+	struct modctl *ctl = sdp->sdp_ctl;
+
+	ctl->mod_nenabled--;
+
+	if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
+		goto err;
+#endif /* __APPLE__ */
+
+	while (sdp != NULL) {
+		(void)ml_nofault_copy( (vm_offset_t)&sdp->sdp_savedval, (vm_offset_t)sdp->sdp_patchpoint, 
+								sizeof(sdp->sdp_savedval));
+		sdp = sdp->sdp_next;
+	}
+
+err:
+	;
+}
+
+static dtrace_pops_t sdt_pops = {
+	NULL,
+	sdt_provide_module,
+	sdt_enable,
+	sdt_disable,
+	NULL,
+	NULL,
+	sdt_getargdesc,
+	NULL,
+	NULL,
+	sdt_destroy
+};
+
+/*ARGSUSED*/
+static int
+sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+#pragma unused(cmd)
+	sdt_provider_t *prov;
+
+	if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
+	    0, DDI_PSEUDO, 0) == DDI_FAILURE) {
+		cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	sdt_devi = devi;
+
+	if (sdt_probetab_size == 0)
+		sdt_probetab_size = SDT_PROBETAB_SIZE;
+
+	sdt_probetab_mask = sdt_probetab_size - 1;
+	sdt_probetab =
+	    kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
+	dtrace_invop_add(sdt_invop);
+
+	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
+		if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
+		    DTRACE_PRIV_KERNEL, NULL,
+		    &sdt_pops, prov, &prov->sdtp_id) != 0) {
+			cmn_err(CE_WARN, "failed to register sdt provider %s",
+			    prov->sdtp_name);
+		}
+	}
+
+	return (DDI_SUCCESS);
+}
+
+#if !defined(__APPLE__)
+/*ARGSUSED*/
+static int
+sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
+{
+	sdt_provider_t *prov;
+
+	switch (cmd) {
+	case DDI_DETACH:
+		break;
+
+	case DDI_SUSPEND:
+		return (DDI_SUCCESS);
+
+	default:
+		return (DDI_FAILURE);
+	}
+
+	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
+		if (prov->sdtp_id != DTRACE_PROVNONE) {
+			if (dtrace_unregister(prov->sdtp_id) != 0)
+				return (DDI_FAILURE);
+
+			prov->sdtp_id = DTRACE_PROVNONE;
+		}
+	}
+
+	dtrace_invop_remove(sdt_invop);
+	kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
+
+	return (DDI_SUCCESS);
+}
+
+/*ARGSUSED*/
+static int
+sdt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
+{
+	int error;
+
+	switch (infocmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)sdt_devi;
+		error = DDI_SUCCESS;
+		break;
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		error = DDI_SUCCESS;
+		break;
+	default:
+		error = DDI_FAILURE;
+	}
+	return (error);
+}
+
+/*ARGSUSED*/
+static int
+sdt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
+{
+	return (0);
+}
+
+static struct cb_ops sdt_cb_ops = {
+	sdt_open,		/* open */
+	nodev,			/* close */
+	nulldev,		/* strategy */
+	nulldev,		/* print */
+	nodev,			/* dump */
+	nodev,			/* read */
+	nodev,			/* write */
+	nodev,			/* ioctl */
+	nodev,			/* devmap */
+	nodev,			/* mmap */
+	nodev,			/* segmap */
+	nochpoll,		/* poll */
+	ddi_prop_op,		/* cb_prop_op */
+	0,			/* streamtab  */
+	D_NEW | D_MP		/* Driver compatibility flag */
+};
+
+static struct dev_ops sdt_ops = {
+	DEVO_REV,		/* devo_rev, */
+	0,			/* refcnt  */
+	sdt_info,		/* get_dev_info */
+	nulldev,		/* identify */
+	nulldev,		/* probe */
+	sdt_attach,		/* attach */
+	sdt_detach,		/* detach */
+	nodev,			/* reset */
+	&sdt_cb_ops,		/* driver operations */
+	NULL,			/* bus operations */
+	nodev			/* dev power */
+};
+
+/*
+ * Module linkage information for the kernel.
+ */
+static struct modldrv modldrv = {
+	&mod_driverops,		/* module type (this is a pseudo driver) */
+	"Statically Defined Tracing",	/* name of module */
+	&sdt_ops,		/* driver ops */
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	(void *)&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	return (mod_install(&modlinkage));
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+int
+_fini(void)
+{
+	return (mod_remove(&modlinkage));
+}
+#else
+d_open_t _sdt_open;
+
+int _sdt_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+#define SDT_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw sdt_cdevsw =
+{
+	_sdt_open,		/* open */
+	eno_opcl,			/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	eno_ioctl,			/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int gSDTInited = 0;
+static struct modctl g_sdt_kernctl;
+static struct module g_sdt_mach_module;
+
+#include <mach-o/loader.h>
+#include <mach-o/nlist.h>
+
+extern struct mach_header _mh_execute_header; /* the kernel's mach header */
+
+void sdt_init( void )
+{
+	if (0 == gSDTInited)
+	{
+		int majdevno = cdevsw_add(SDT_MAJOR, &sdt_cdevsw);
+		
+		if (majdevno < 0) {
+			printf("sdt_init: failed to allocate a major number!\n");
+			gSDTInited = 0;
+			return;
+		}
+
+		if (MH_MAGIC != _mh_execute_header.magic) {
+        	g_sdt_kernctl.address = (vm_address_t)NULL;
+        	g_sdt_kernctl.size = 0;
+		} else {
+		struct mach_header          *mh;
+    		struct load_command         *cmd;
+    		struct segment_command      *orig_ts = NULL, *orig_le = NULL;
+    		struct symtab_command       *orig_st = NULL;
+    		struct nlist                *sym = NULL;
+    		char                        *strings;
+    		unsigned int i;
+
+		g_sdt_mach_module.sdt_nprobes = 0;
+		g_sdt_mach_module.sdt_probes = NULL;
+
+        	g_sdt_kernctl.address = (vm_address_t)&g_sdt_mach_module;
+        	g_sdt_kernctl.size = 0;
+		strncpy((char *)&(g_sdt_kernctl.mod_modname), "mach_kernel", KMOD_MAX_NAME);
+
+		mh = &_mh_execute_header;
+    		cmd = (struct load_command *) &mh[1];
+    		for (i = 0; i < mh->ncmds; i++) {
+        		if (cmd->cmd == LC_SEGMENT) {
+            		struct segment_command *orig_sg = (struct segment_command *) cmd;
+
+            		if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
+                		orig_ts = orig_sg;
+            		else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
+                		orig_le = orig_sg;
+            		else if (strcmp("", orig_sg->segname) == 0)
+                		orig_ts = orig_sg; /* kexts have a single unnamed segment */
+        		}
+        		else if (cmd->cmd == LC_SYMTAB)
+            		orig_st = (struct symtab_command *) cmd;
+	
+        		cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
+    		}
+	
+    		if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
+        		return;
+
+    		sym = (struct nlist *)orig_le->vmaddr;
+    		strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist);
+
+    		for (i = 0; i < orig_st->nsyms; i++) {
+        		uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
+        		char *name = strings + sym[i].n_un.n_strx;
+				char *prev_name;
+				unsigned long best;
+				unsigned int j;
+
+        		/* Check that the symbol is a global and that it has a name. */
+        		if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
+            		continue;
+
+        		if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
+            		continue;
+
+        		/* Lop off omnipresent leading underscore. */
+        		if (*name == '_')
+            		name += 1;
+
+				if (strstr(name, "_dtrace_probe$")) {
+					sdt_probedesc_t *sdpd = kmem_alloc(sizeof(sdt_probedesc_t), KM_SLEEP);
+					int len = strlen(name) + 1;
+
+					sdpd->sdpd_name = kmem_alloc(len, KM_SLEEP);
+					strncpy(sdpd->sdpd_name, name, len); /* NUL termination is ensured. */
+
+					prev_name = "<unknown>";
+					best = 0;
+					for (j = 0; j < orig_st->nsyms; j++) {
+						uint8_t n_type = sym[j].n_type & (N_TYPE | N_EXT);
+						char *name = strings + sym[j].n_un.n_strx;
+
+						if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
+							continue;
+
+						if (0 == sym[j].n_un.n_strx) /* iff a null, "", name. */
+							continue;
+
+						if (*name == '_')
+							name += 1;
+						if (strstr(name, "_dtrace_probe$"))
+							continue;
+
+						if (*(unsigned long *)sym[i].n_value <= (unsigned long)sym[j].n_value)
+							continue;
+
+						if ((unsigned long)sym[j].n_value > best) {
+							best = (unsigned long)sym[j].n_value;
+							prev_name = name;
+						}
+					}
+
+					sdpd->sdpd_func = kmem_alloc((len = strlen(prev_name) + 1), KM_SLEEP);
+					strncpy(sdpd->sdpd_func, prev_name, len); /* NUL termination is ensured. */
+
+					sdpd->sdpd_offset = *(unsigned long *)sym[i].n_value;
+
+					sdpd->sdpd_next = g_sdt_mach_module.sdt_probes;
+					g_sdt_mach_module.sdt_probes = sdpd;
+				} else {
+					prev_name = name;
+				}
+			}
+		}
+
+		sdt_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+
+		gSDTInited = 1;
+	} else
+		panic("sdt_init: called twice!\n");
+}
+#undef SDT_MAJOR
+
+/*ARGSUSED*/
+void
+sdt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(ctl)
+    __sdt_provide_module(arg, &g_sdt_kernctl);
+
+	sdt_probedesc_t *sdpd = g_sdt_mach_module.sdt_probes;
+	while (sdpd) {
+		sdt_probedesc_t *this_sdpd = sdpd;
+		kmem_free((void *)sdpd->sdpd_name, strlen(sdpd->sdpd_name) + 1);
+		kmem_free((void *)sdpd->sdpd_func, strlen(sdpd->sdpd_func) + 1);
+		sdpd = sdpd->sdpd_next;
+		kmem_free((void *)this_sdpd, sizeof(sdt_probedesc_t));
+	}
+	g_sdt_mach_module.sdt_probes = NULL;
+}
+
+#endif /* __APPLE__ */
diff --git a/bsd/dev/dtrace/sdt_subr.c b/bsd/dev/dtrace/sdt_subr.c
new file mode 100644
index 000000000..f57c5d614
--- /dev/null
+++ b/bsd/dev/dtrace/sdt_subr.c
@@ -0,0 +1,229 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)sdt_subr.c	1.7	06/04/03 SMI" */
+
+#include <sys/sdt_impl.h>
+
+static dtrace_pattr_t vtrace_attr = {
+{ DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_UNSTABLE, DTRACE_STABILITY_UNSTABLE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pattr_t info_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pattr_t fpu_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_CPU },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pattr_t fsinfo_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pattr_t stab_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pattr_t sdt_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+sdt_provider_t sdt_providers[] = {
+	{ "vtrace", "__vtrace____", &vtrace_attr, 0 },
+	{ "sysinfo", "__cpu_sysinfo____", &info_attr, 0 },
+	{ "vminfo", "__vminfo____", &info_attr, 0 },
+	{ "fpuinfo", "__fpuinfo____", &fpu_attr, 0 },
+	{ "sched", "__sched____", &stab_attr, 0 },
+	{ "proc", "__proc____", &stab_attr, 0 },
+	{ "io", "__io____", &stab_attr, 0 },
+	{ "mib", "__mib____", &stab_attr, 0 },
+	{ "fsinfo", "__fsinfo____", &fsinfo_attr, 0 },
+	{ "sdt", "__sdt____", &sdt_attr, 0 },
+	{ NULL }
+};
+
+#warning !!! Need xnu cognate for disp_t.
+#warning !!! Need translators for bufinfo_t, cpuinfo_t, devinfo_t, fileinfo_t.
+sdt_argdesc_t sdt_args[] = {
+	{ "sched", "wakeup", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "wakeup", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "dequeue", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "dequeue", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "dequeue", 2, 1, "disp_t *", "cpuinfo_t *" },
+	{ "sched", "enqueue", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "enqueue", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "enqueue", 2, 1, "disp_t *", "cpuinfo_t *" },
+	{ "sched", "enqueue", 3, 2, "int", NULL },
+	{ "sched", "off-cpu", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "off-cpu", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "tick", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "tick", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "change-pri", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "change-pri", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "change-pri", 2, 1, "pri_t", NULL },
+	{ "sched", "schedctl-nopreempt", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "schedctl-nopreempt", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "schedctl-nopreempt", 2, 1, "int", NULL },
+	{ "sched", "schedctl-preempt", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "schedctl-preempt", 1, 0, "struct proc *", "psinfo_t *" },
+	{ "sched", "schedctl-yield", 0, 0, "int", NULL },
+	{ "sched", "surrender", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "sched", "surrender", 1, 0, "struct proc *", "psinfo_t *" },
+
+	{ "proc", "create", 0, 0, "struct proc *", "psinfo_t *" },
+	{ "proc", "exec", 0, 0, "string", NULL },
+	{ "proc", "exec-failure", 0, 0, "int", NULL },
+	/* proc:::exec-success has no arguments */
+	{ "proc", "exit", 0, 0, "int", NULL },
+	{ "proc", "fault", 0, 0, "int", NULL },
+	{ "proc", "fault", 1, 1, "siginfo_t *", NULL },
+	{ "proc", "lwp-create", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "proc", "lwp-create", 1, 1, "struct proc *", "psinfo_t *" },
+	/* proc:::lwp-start has no arguments */
+	/* proc:::lwp-exit has no arguments */
+	{ "proc", "signal-clear", 0, 0, "int", NULL },
+	{ "proc", "signal-clear", 1, 1, "siginfo_t *", NULL },
+	{ "proc", "signal-discard", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "proc", "signal-discard", 1, 1, "struct proc *", "psinfo_t *" },
+	{ "proc", "signal-discard", 2, 2, "int", NULL },
+	{ "proc", "signal-handle", 0, 0, "int", NULL },
+	{ "proc", "signal-handle", 1, 1, "siginfo_t *", NULL },
+	{ "proc", "signal-handle", 2, 2, "void (*)(void)", NULL },
+	{ "proc", "signal-send", 0, 0, "struct thread *", "lwpsinfo_t *" },
+	{ "proc", "signal-send", 1, 1, "struct proc *", "psinfo_t *" },
+	{ "proc", "signal-send", 2, 2, "int", NULL },
+	/* proc:::start has no arguments */
+
+	{ "io", "start", 0, 0, "struct buf *", "bufinfo_t *" },
+	{ "io", "start", 1, 0, "struct buf *", "devinfo_t *" },
+	{ "io", "start", 2, 0, "struct buf *", "fileinfo_t *" },
+	{ "io", "done", 0, 0, "struct buf *", "bufinfo_t *" },
+	{ "io", "done", 1, 0, "struct buf *", "devinfo_t *" },
+	{ "io", "done", 2, 0, "struct buf *", "fileinfo_t *" },
+	{ "io", "wait-start", 0, 0, "struct buf *", "bufinfo_t *" },
+	{ "io", "wait-start", 1, 0, "struct buf *", "devinfo_t *" },
+	{ "io", "wait-start", 2, 0, "struct buf *", "fileinfo_t *" },
+	{ "io", "wait-done", 0, 0, "struct buf *", "bufinfo_t *" },
+	{ "io", "wait-done", 1, 0, "struct buf *", "devinfo_t *" },
+	{ "io", "wait-done", 2, 0, "struct buf *", "fileinfo_t *" },
+
+	{ "vminfo", "anonfree", 0, 0, "int", NULL },
+	{ "vminfo", "anonpgin", 0, 0, "int", NULL },
+	{ "vminfo", "anonpgout", 0, 0, "int", NULL },
+	{ "vminfo", "as_fault", 0, 0, "int", NULL },
+	{ "vminfo", "cow_fault", 0, 0, "int", NULL },
+	{ "vminfo", "dfree", 0, 0, "int", NULL },
+	{ "vminfo", "execfree", 0, 0, "int", NULL },
+	{ "vminfo", "execpgin", 0, 0, "int", NULL },
+	{ "vminfo", "execpgout", 0, 0, "int", NULL },
+	{ "vminfo", "fsfree", 0, 0, "int", NULL },
+	{ "vminfo", "fspgin", 0, 0, "int", NULL },
+	{ "vminfo", "fspgout", 0, 0, "int", NULL },
+	{ "vminfo", "kerenl_asflt", 0, 0, "int", NULL },
+	{ "vminfo", "maj_fault", 0, 0, "int", NULL },
+	{ "vminfo", "pgfrec", 0, 0, "int", NULL },
+	{ "vminfo", "pgin", 0, 0, "int", NULL },
+	{ "vminfo", "pgout", 0, 0, "int", NULL },
+	{ "vminfo", "pgpgin", 0, 0, "int", NULL },
+	{ "vminfo", "pgpgout", 0, 0, "int", NULL },
+	{ "vminfo", "pgrec", 0, 0, "int", NULL },
+	{ "vminfo", "pgrrun", 0, 0, "int", NULL },
+	{ "vminfo", "pgswapin", 0, 0, "int", NULL },
+	{ "vminfo", "pgswapout", 0, 0, "int", NULL },
+	{ "vminfo", "prot_fault", 0, 0, "int", NULL },
+	{ "vminfo", "rev", 0, 0, "int", NULL },
+	{ "vminfo", "scan", 0, 0, "int", NULL },
+	{ "vminfo", "softlock", 0, 0, "int", NULL },
+	{ "vminfo", "swapin", 0, 0, "int", NULL },
+	{ "vminfo", "swapout", 0, 0, "int", NULL },
+	{ "vminfo", "zfod", 0, 0, "int", NULL },
+
+	{ "mib", NULL, 0, 0, "int", NULL },
+	{ "fsinfo", NULL, 0, 0, "struct vnode *", "fileinfo_t *" },
+	{ "fsinfo", NULL, 1, 1, "int", "int" },
+	{ NULL }
+};
+
+/*ARGSUSED*/
+void
+sdt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
+{
+#pragma unused(arg, id)
+	sdt_probe_t *sdp = parg;
+	int i;
+
+	desc->dtargd_native[0] = '\0';
+	desc->dtargd_xlate[0] = '\0';
+
+	for (i = 0; sdt_args[i].sda_provider != NULL; i++) {
+		sdt_argdesc_t *a = &sdt_args[i];
+
+		if (strcmp(sdp->sdp_provider->sdtp_name, a->sda_provider) != 0)
+			continue;
+
+		if (a->sda_name != NULL &&
+		    strcmp(sdp->sdp_name, a->sda_name) != 0)
+			continue;
+
+		if (desc->dtargd_ndx != a->sda_ndx)
+			continue;
+
+		if (a->sda_native != NULL)
+			(void) strcpy(desc->dtargd_native, a->sda_native);
+
+		if (a->sda_xlate != NULL)
+			(void) strcpy(desc->dtargd_xlate, a->sda_xlate);
+
+		desc->dtargd_mapping = a->sda_mapping;
+		return;
+	}
+
+	desc->dtargd_ndx = DTRACE_ARGNONE;
+}
diff --git a/bsd/dev/dtrace/systrace.c b/bsd/dev/dtrace/systrace.c
new file mode 100644
index 000000000..d625d7e80
--- /dev/null
+++ b/bsd/dev/dtrace/systrace.c
@@ -0,0 +1,1131 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)systrace.c	1.5	06/03/24 SMI" */
+
+#if !defined(__APPLE__)
+#include <sys/dtrace.h>
+#include <sys/systrace.h>
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/ddi.h>
+#include <sys/sunddi.h>
+#include <sys/atomic.h>
+#define	SYSTRACE_ARTIFICIAL_FRAMES	1
+#else
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+/* XXX All of these should really be derived from syscall_sw.h */
+#if defined(__i386__) || defined (__x86_64__)
+#define SYSCALL_CLASS_SHIFT 24
+#define SYSCALL_CLASS_MASK  (0xFF << SYSCALL_CLASS_SHIFT)
+#define SYSCALL_NUMBER_MASK (~SYSCALL_CLASS_MASK)
+#define I386_SYSCALL_NUMBER_MASK (0xFFFF)
+
+typedef x86_saved_state_t savearea_t;
+#elif defined(__arm__)
+typedef struct arm_saved_state savearea_t;
+#endif
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/errno.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include "systrace.h"
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/user.h>
+
+#if defined (__ppc__) || defined (__ppc64__)
+#define	SYSTRACE_ARTIFICIAL_FRAMES	3
+#define MACHTRACE_ARTIFICIAL_FRAMES 4
+#elif defined(__i386__) || defined (__x86_64__)
+#define	SYSTRACE_ARTIFICIAL_FRAMES	2
+#define MACHTRACE_ARTIFICIAL_FRAMES 3
+#elif defined(__arm__)
+#define	SYSTRACE_ARTIFICIAL_FRAMES	2 /* XXX ARMTODO */
+#define MACHTRACE_ARTIFICIAL_FRAMES 3 /* XXX ARMTODO */
+#else
+#error Unknown Architecture
+#endif
+
+#include <sys/sysent.h>
+#define sy_callc sy_call /* Map Solaris slot name to Darwin's */
+#define NSYSCALL nsysent /* and is less than 500 or so */
+
+extern const char *syscallnames[];
+
+#include <sys/dtrace_glue.h>
+#define casptr dtrace_casptr
+#define membar_enter dtrace_membar_producer
+
+#define LOADABLE_SYSCALL(a) 0 /* Not pertinent to Darwin. */
+#define LOADED_SYSCALL(a) 1 /* Not pertinent to Darwin. */
+
+systrace_sysent_t *systrace_sysent = NULL;
+void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t,
+    uint64_t, uint64_t, uint64_t);
+
+void
+systrace_stub(dtrace_id_t id, uint64_t arg0, uint64_t arg1,
+    uint64_t arg2, uint64_t arg3, uint64_t arg4)
+{
+#pragma unused(id,arg0,arg1,arg2,arg3,arg4)
+}
+
+int32_t
+dtrace_systrace_syscall(struct proc *pp, void *uap, int *rv)
+{
+	boolean_t           flavor;
+	unsigned short      code;
+
+	systrace_sysent_t *sy;
+	dtrace_id_t id;
+	int32_t rval;
+#if 0 /* XXX */
+	proc_t *p;
+#endif
+	syscall_arg_t *ip = (syscall_arg_t *)uap;
+
+#if defined (__ppc__) || defined (__ppc64__)
+	{
+		savearea_t *regs = (savearea_t *)find_user_regs(current_thread());
+
+		flavor = (((unsigned int)regs->save_r0) == 0)? 1: 0;
+
+		if (flavor)
+			code = regs->save_r3;
+		else
+			code = regs->save_r0;
+	}
+#elif defined(__i386__) || defined (__x86_64__)
+#pragma unused(flavor)
+	{
+		x86_saved_state_t   *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
+
+		if (is_saved_state64(tagged_regs)) {
+			x86_saved_state64_t *regs = saved_state64(tagged_regs);
+			code = regs->rax & SYSCALL_NUMBER_MASK;
+			/*
+			 * Check for indirect system call... system call number
+			 * passed as 'arg0'
+			 */
+			if (code == 0) {
+				code = regs->rdi;
+			}
+		} else {
+			code = saved_state32(tagged_regs)->eax & I386_SYSCALL_NUMBER_MASK;
+			/*
+			 * TODO: handle indirect system calls
+			 */
+		}
+	}
+#elif defined(__arm__)
+	do {} while(0); /* XXX what is the right ABI */
+#else
+#error Unknown Architecture
+#endif
+
+	// Bounds "check" the value of code a la unix_syscall
+	sy = (code >= NUM_SYSENT) ? &systrace_sysent[63] : &systrace_sysent[code];
+
+	if ((id = sy->stsy_entry) != DTRACE_IDNONE)
+		(*systrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4));
+
+#if 0 /* XXX */
+	/*
+	 * We want to explicitly allow DTrace consumers to stop a process
+	 * before it actually executes the meat of the syscall.
+	 */
+	p = ttoproc(curthread);
+	mutex_enter(&p->p_lock);
+	if (curthread->t_dtrace_stop && !curthread->t_lwp->lwp_nostop) {
+		curthread->t_dtrace_stop = 0;
+		stop(PR_REQUESTED, 0);
+	}
+	mutex_exit(&p->p_lock);
+#endif
+
+	rval = (*sy->stsy_underlying)(pp, uap, rv);
+
+	if ((id = sy->stsy_return) != DTRACE_IDNONE) {
+		uint64_t munged_rv;
+    	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+		if (uthread)
+			uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */
+
+		/*
+	 	 * "Decode" rv for use in the call to dtrace_probe()
+	 	 */
+		if (rval == ERESTART) {
+			munged_rv = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */
+		} else if (rval != EJUSTRETURN) {
+			if (rval) {
+				munged_rv = -1LL; /* Mimic what libc will do. */
+			} else {
+				switch (sy->stsy_return_type) {
+				case _SYSCALL_RET_INT_T:
+					munged_rv = rv[0];
+					break;
+				case _SYSCALL_RET_UINT_T:
+					munged_rv = ((u_int)rv[0]);
+					break;
+				case _SYSCALL_RET_OFF_T:
+					munged_rv = *(u_int64_t *)rv;
+					break;
+				case _SYSCALL_RET_ADDR_T:
+				case _SYSCALL_RET_SIZE_T:
+				case _SYSCALL_RET_SSIZE_T:
+					munged_rv = *(user_addr_t *)rv;
+					break;
+				case _SYSCALL_RET_NONE:
+					munged_rv = 0LL;
+					break;
+				default:
+					munged_rv = 0LL;
+					break;
+				}
+			}
+		} else
+			munged_rv = 0LL;
+
+		(*systrace_probe)(id, munged_rv, munged_rv, (uint64_t)rval, 0, 0);
+	}
+
+	return (rval);
+}
+
+void
+dtrace_systrace_syscall_return(unsigned short code, int rval, int *rv)
+{
+	systrace_sysent_t *sy;
+	dtrace_id_t id;
+
+	// Bounds "check" the value of code a la unix_syscall_return
+	sy = (code >= NUM_SYSENT) ? &systrace_sysent[63] : &systrace_sysent[code];
+
+	if ((id = sy->stsy_return) != DTRACE_IDNONE) {
+		uint64_t munged_rv;
+    	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+		if (uthread)
+			uthread->t_dtrace_errno = rval; /* Establish t_dtrace_errno now in case this enabling refers to it. */
+
+		/*
+	 	 * "Decode" rv for use in the call to dtrace_probe()
+	 	 */
+		if (rval == ERESTART) {
+			munged_rv = -1LL; /* System call will be reissued in user mode. Make DTrace report a -1 return. */
+		} else if (rval != EJUSTRETURN) {
+			if (rval) {
+				munged_rv = -1LL; /* Mimic what libc will do. */
+			} else {
+				switch (sy->stsy_return_type) {
+				case _SYSCALL_RET_INT_T:
+					munged_rv = rv[0];
+					break;
+				case _SYSCALL_RET_UINT_T:
+					munged_rv = ((u_int)rv[0]);
+					break;
+				case _SYSCALL_RET_OFF_T:
+					munged_rv = *(u_int64_t *)rv;
+					break;
+				case _SYSCALL_RET_ADDR_T:
+				case _SYSCALL_RET_SIZE_T:
+				case _SYSCALL_RET_SSIZE_T:
+					munged_rv = *(user_addr_t *)rv;
+					break;
+				case _SYSCALL_RET_NONE:
+					munged_rv = 0LL;
+					break;
+				default:
+					munged_rv = 0LL;
+					break;
+				}
+			}
+		} else
+			munged_rv = 0LL;
+
+		(*systrace_probe)(id, munged_rv, munged_rv, (uint64_t)rval, 0, 0);
+	}
+}
+#endif /* __APPLE__ */
+
+#define	SYSTRACE_SHIFT			16
+#define	SYSTRACE_ISENTRY(x)		((int)(x) >> SYSTRACE_SHIFT)
+#define	SYSTRACE_SYSNUM(x)		((int)(x) & ((1 << SYSTRACE_SHIFT) - 1))
+#define	SYSTRACE_ENTRY(id)		((1 << SYSTRACE_SHIFT) | (id))
+#define	SYSTRACE_RETURN(id)		(id)
+
+#if ((1 << SYSTRACE_SHIFT) <= NSYSCALL)
+#error 1 << SYSTRACE_SHIFT must exceed number of system calls
+#endif
+
+static dev_info_t *systrace_devi;
+static dtrace_provider_id_t systrace_id;
+
+#if defined(__APPLE__)
+#define systrace_init _systrace_init /* Avoid name clash with Darwin automagic conf symbol */
+#endif
+static void
+systrace_init(struct sysent *actual, systrace_sysent_t **interposed)
+{
+	systrace_sysent_t *sysent = *interposed;
+	int i;
+
+	if (sysent == NULL) {
+		*interposed = sysent = kmem_zalloc(sizeof (systrace_sysent_t) *
+		    NSYSCALL, KM_SLEEP);
+	}
+
+	for (i = 0; i < NSYSCALL; i++) {
+		struct sysent *a = &actual[i];
+		systrace_sysent_t *s = &sysent[i];
+
+		if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a))
+			continue;
+
+		if (a->sy_callc == dtrace_systrace_syscall)
+			continue;
+
+#ifdef _SYSCALL32_IMPL
+		if (a->sy_callc == dtrace_systrace_syscall32)
+			continue;
+#endif
+
+		s->stsy_underlying = a->sy_callc;
+#if defined(__APPLE__)
+		s->stsy_return_type = a->sy_return_type;
+#endif
+	}
+}
+
+/*ARGSUSED*/
+static void
+systrace_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+	int i;
+
+	if (desc != NULL)
+		return;
+
+	systrace_init(sysent, &systrace_sysent);
+#ifdef _SYSCALL32_IMPL
+	systrace_init(sysent32, &systrace_sysent32);
+#endif
+
+	for (i = 0; i < NSYSCALL; i++) {
+		if (systrace_sysent[i].stsy_underlying == NULL)
+			continue;
+
+		if (dtrace_probe_lookup(systrace_id, NULL,
+		    syscallnames[i], "entry") != 0)
+			continue;
+
+		(void) dtrace_probe_create(systrace_id, NULL, syscallnames[i],
+		    "entry", SYSTRACE_ARTIFICIAL_FRAMES,
+		    (void *)((uintptr_t)SYSTRACE_ENTRY(i)));
+		(void) dtrace_probe_create(systrace_id, NULL, syscallnames[i],
+		    "return", SYSTRACE_ARTIFICIAL_FRAMES,
+		    (void *)((uintptr_t)SYSTRACE_RETURN(i)));
+
+		systrace_sysent[i].stsy_entry = DTRACE_IDNONE;
+		systrace_sysent[i].stsy_return = DTRACE_IDNONE;
+#ifdef _SYSCALL32_IMPL
+		systrace_sysent32[i].stsy_entry = DTRACE_IDNONE;
+		systrace_sysent32[i].stsy_return = DTRACE_IDNONE;
+#endif
+	}
+}
+#if defined(__APPLE__)
+#undef systrace_init
+#endif
+
+/*ARGSUSED*/
+static void
+systrace_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+
+	/*
+	 * There's nothing to do here but assert that we have actually been
+	 * disabled.
+	 */
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		ASSERT(systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE);
+#ifdef _SYSCALL32_IMPL
+		ASSERT(systrace_sysent32[sysnum].stsy_entry == DTRACE_IDNONE);
+#endif
+	} else {
+		ASSERT(systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
+#ifdef _SYSCALL32_IMPL
+		ASSERT(systrace_sysent32[sysnum].stsy_return == DTRACE_IDNONE);
+#endif
+	}
+}
+
+/*ARGSUSED*/
+static void
+systrace_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+	int enabled = (systrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE ||
+	    systrace_sysent[sysnum].stsy_return != DTRACE_IDNONE);
+
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		systrace_sysent[sysnum].stsy_entry = id;
+#ifdef _SYSCALL32_IMPL
+		systrace_sysent32[sysnum].stsy_entry = id;
+#endif
+	} else {
+		systrace_sysent[sysnum].stsy_return = id;
+#ifdef _SYSCALL32_IMPL
+		systrace_sysent32[sysnum].stsy_return = id;
+#endif
+	}
+
+	if (enabled) {
+		ASSERT(sysent[sysnum].sy_callc == dtrace_systrace_syscall);
+		return;
+	}
+
+	(void) casptr(&sysent[sysnum].sy_callc,
+	    (void *)systrace_sysent[sysnum].stsy_underlying,
+	    (void *)dtrace_systrace_syscall);
+#ifdef _SYSCALL32_IMPL
+	(void) casptr(&sysent32[sysnum].sy_callc,
+	    (void *)systrace_sysent32[sysnum].stsy_underlying,
+	    (void *)dtrace_systrace_syscall32);
+#endif
+}
+
+/*ARGSUSED*/
+static void
+systrace_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+	int disable = (systrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE ||
+	    systrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
+
+	if (disable) {
+		(void) casptr(&sysent[sysnum].sy_callc,
+		    (void *)dtrace_systrace_syscall,
+		    (void *)systrace_sysent[sysnum].stsy_underlying);
+
+#ifdef _SYSCALL32_IMPL
+		(void) casptr(&sysent32[sysnum].sy_callc,
+		    (void *)dtrace_systrace_syscall32,
+		    (void *)systrace_sysent32[sysnum].stsy_underlying);
+#endif
+	}
+
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		systrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE;
+#ifdef _SYSCALL32_IMPL
+		systrace_sysent32[sysnum].stsy_entry = DTRACE_IDNONE;
+#endif
+	} else {
+		systrace_sysent[sysnum].stsy_return = DTRACE_IDNONE;
+#ifdef _SYSCALL32_IMPL
+		systrace_sysent32[sysnum].stsy_return = DTRACE_IDNONE;
+#endif
+	}
+}
+
+static dtrace_pattr_t systrace_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pops_t systrace_pops = {
+	systrace_provide,
+	NULL,
+	systrace_enable,
+	systrace_disable,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	systrace_destroy
+};
+
+static int
+systrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_ATTACH:
+		break;
+	case DDI_RESUME:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	systrace_probe = dtrace_probe;
+	membar_enter();
+
+	if (ddi_create_minor_node(devi, "systrace", S_IFCHR, 0,
+	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
+	    dtrace_register("syscall", &systrace_attr, DTRACE_PRIV_USER, NULL,
+	    &systrace_pops, NULL, &systrace_id) != 0) {
+		systrace_probe = systrace_stub;
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	systrace_devi = devi;
+
+	return (DDI_SUCCESS);
+}
+
+#if !defined(__APPLE__)
+static int
+systrace_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
+{
+	switch (cmd) {
+	case DDI_DETACH:
+		break;
+	case DDI_SUSPEND:
+		return (DDI_SUCCESS);
+	default:
+		return (DDI_FAILURE);
+	}
+
+	if (dtrace_unregister(systrace_id) != 0)
+		return (DDI_FAILURE);
+
+	ddi_remove_minor_node(devi, NULL);
+	systrace_probe = systrace_stub;
+	return (DDI_SUCCESS);
+}
+
+/*ARGSUSED*/
+static int
+systrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
+{
+	int error;
+
+	switch (infocmd) {
+	case DDI_INFO_DEVT2DEVINFO:
+		*result = (void *)systrace_devi;
+		error = DDI_SUCCESS;
+		break;
+	case DDI_INFO_DEVT2INSTANCE:
+		*result = (void *)0;
+		error = DDI_SUCCESS;
+		break;
+	default:
+		error = DDI_FAILURE;
+	}
+	return (error);
+}
+
+/*ARGSUSED*/
+static int
+systrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
+{
+	return (0);
+}
+
+static struct cb_ops systrace_cb_ops = {
+	systrace_open,		/* open */
+	nodev,			/* close */
+	nulldev,		/* strategy */
+	nulldev,		/* print */
+	nodev,			/* dump */
+	nodev,			/* read */
+	nodev,			/* write */
+	nodev,			/* ioctl */
+	nodev,			/* devmap */
+	nodev,			/* mmap */
+	nodev,			/* segmap */
+	nochpoll,		/* poll */
+	ddi_prop_op,		/* cb_prop_op */
+	0,			/* streamtab  */
+	D_NEW | D_MP		/* Driver compatibility flag */
+};
+
+static struct dev_ops systrace_ops = {
+	DEVO_REV,		/* devo_rev, */
+	0,			/* refcnt  */
+	systrace_info,		/* get_dev_info */
+	nulldev,		/* identify */
+	nulldev,		/* probe */
+	systrace_attach,	/* attach */
+	systrace_detach,	/* detach */
+	nodev,			/* reset */
+	&systrace_cb_ops,	/* driver operations */
+	NULL,			/* bus operations */
+	nodev			/* dev power */
+};
+
+/*
+ * Module linkage information for the kernel.
+ */
+static struct modldrv modldrv = {
+	&mod_driverops,		/* module type (this is a pseudo driver) */
+	"System Call Tracing",	/* name of module */
+	&systrace_ops,		/* driver ops */
+};
+
+static struct modlinkage modlinkage = {
+	MODREV_1,
+	(void *)&modldrv,
+	NULL
+};
+
+int
+_init(void)
+{
+	return (mod_install(&modlinkage));
+}
+
+int
+_info(struct modinfo *modinfop)
+{
+	return (mod_info(&modlinkage, modinfop));
+}
+
+int
+_fini(void)
+{
+	return (mod_remove(&modlinkage));
+}
+#else
+typedef kern_return_t (*mach_call_t)(void *);
+
+/* XXX From #include <kern/syscall_sw.h> which may be changed for 64 bit! */
+typedef void    mach_munge_t(const void *, void *);
+
+typedef struct {
+        int                     mach_trap_arg_count;
+        int                     (*mach_trap_function)(void);
+#if defined(__i386__)
+        boolean_t               mach_trap_stack;
+#else
+        mach_munge_t            *mach_trap_arg_munge32; /* system call arguments for 32-bit */
+        mach_munge_t            *mach_trap_arg_munge64; /* system call arguments for 64-bit */
+#endif
+#if     !MACH_ASSERT
+        int                     mach_trap_unused;
+#else
+        const char*             mach_trap_name;
+#endif /* !MACH_ASSERT */
+} mach_trap_t;
+
+#define MACH_TRAP_TABLE_COUNT   128
+
+extern mach_trap_t              mach_trap_table[];
+extern int                      mach_trap_count;
+
+#define MACH_TRAP(name, foo, bar, baz) #name
+
+/* XXX From osfmk/kern/syscall_sw.c */
+static const char *	mach_name_table[MACH_TRAP_TABLE_COUNT] = {
+/* 0 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 1 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 2 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 3 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 4 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 5 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 6 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 7 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 8 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 9 */		MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 10 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 11 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 12 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 13 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 14 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 15 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 16 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 17 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 18 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 19 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 20 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 21 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 22 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 23 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 24 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 25 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 26 */	MACH_TRAP(mach_reply_port, 0, NULL, NULL),
+/* 27 */	MACH_TRAP(thread_self_trap, 0, NULL, NULL),
+/* 28 */	MACH_TRAP(task_self_trap, 0, NULL, NULL),
+/* 29 */	MACH_TRAP(host_self_trap, 0, NULL, NULL),
+/* 30 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 31 */	MACH_TRAP(mach_msg_trap, 7, munge_wwwwwww, munge_ddddddd),
+/* 32 */	MACH_TRAP(mach_msg_overwrite_trap, 8, munge_wwwwwwww, munge_dddddddd),
+/* 33 */	MACH_TRAP(semaphore_signal_trap, 1, munge_w, munge_d),
+/* 34 */	MACH_TRAP(semaphore_signal_all_trap, 1, munge_w, munge_d),
+/* 35 */	MACH_TRAP(semaphore_signal_thread_trap, 2, munge_ww, munge_dd),
+/* 36 */	MACH_TRAP(semaphore_wait_trap, 1, munge_w, munge_d),
+/* 37 */	MACH_TRAP(semaphore_wait_signal_trap, 2, munge_ww, munge_dd),
+/* 38 */	MACH_TRAP(semaphore_timedwait_trap, 3, munge_www, munge_ddd),
+/* 39 */	MACH_TRAP(semaphore_timedwait_signal_trap, 4, munge_wwww, munge_dddd),
+/* 40 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 41 */	MACH_TRAP(init_process, 0, NULL, NULL),
+/* 42 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 43 */	MACH_TRAP(map_fd, 5, munge_wwwww, munge_ddddd),
+/* 44 */	MACH_TRAP(task_name_for_pid, 3, munge_www, munge_ddd),
+/* 45 */ 	MACH_TRAP(task_for_pid, 3, munge_www, munge_ddd),
+/* 46 */	MACH_TRAP(pid_for_task, 2, munge_ww,munge_dd),
+/* 47 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 48 */	MACH_TRAP(macx_swapon, 4, munge_wwww, munge_dddd),
+/* 49 */	MACH_TRAP(macx_swapoff, 2, munge_ww, munge_dd),
+/* 50 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 51 */	MACH_TRAP(macx_triggers, 4, munge_wwww, munge_dddd),
+/* 52 */	MACH_TRAP(macx_backing_store_suspend, 1, munge_w, munge_d),
+/* 53 */	MACH_TRAP(macx_backing_store_recovery, 1, munge_w, munge_d),
+/* 54 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 55 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 56 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 57 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 58 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 59 */ 	MACH_TRAP(swtch_pri, 0, NULL, NULL),
+/* 60 */	MACH_TRAP(swtch, 0, NULL, NULL),
+/* 61 */	MACH_TRAP(thread_switch, 3, munge_www, munge_ddd),
+/* 62 */	MACH_TRAP(clock_sleep_trap, 5, munge_wwwww, munge_ddddd),
+/* 63 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* traps 64 - 95 reserved (debo) */
+/* 64 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 65 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 66 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 67 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 68 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 69 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 70 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 71 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 72 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 73 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 74 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 75 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 76 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 77 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 78 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 79 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 80 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 81 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 82 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 83 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 84 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 85 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 86 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 87 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 88 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 89 */	MACH_TRAP(mach_timebase_info_trap, 1, munge_w, munge_d),
+/* 90 */	MACH_TRAP(mach_wait_until_trap, 2, munge_l, munge_d),
+/* 91 */	MACH_TRAP(mk_timer_create_trap, 0, NULL, NULL),
+/* 92 */	MACH_TRAP(mk_timer_destroy_trap, 1, munge_w, munge_d),
+/* 93 */	MACH_TRAP(mk_timer_arm_trap, 3, munge_wl, munge_dd),
+/* 94 */	MACH_TRAP(mk_timer_cancel_trap, 2, munge_ww, munge_dd),		
+/* 95 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),		
+/* traps 64 - 95 reserved (debo) */
+/* 96 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 97 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 98 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 99 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* traps 100-107 reserved for iokit (esb) */ 
+/* 100 */	MACH_TRAP(iokit_user_client_trap, 8, munge_wwwwwwww, munge_dddddddd),
+/* 101 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 102 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 103 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 104 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 105 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 106 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 107 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* traps 108-127 unused */			
+/* 108 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 109 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 110 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 111 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 112 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 113 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 114 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 115 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 116 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 117 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 118 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 119 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 120 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 121 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 122 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 123 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 124 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 125 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 126 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),
+/* 127 */	MACH_TRAP(kern_invalid, 0, NULL, NULL),			
+};
+
+/* XXX From osfmk/i386/bsd_i386.c */
+struct mach_call_args {
+        syscall_arg_t arg1;
+        syscall_arg_t arg2;
+        syscall_arg_t arg3;
+        syscall_arg_t arg4;
+        syscall_arg_t arg5;
+        syscall_arg_t arg6;
+        syscall_arg_t arg7;
+        syscall_arg_t arg8;
+        syscall_arg_t arg9;
+};
+
+#undef NSYSCALL
+#define NSYSCALL mach_trap_count
+
+#if ((1 << SYSTRACE_SHIFT) <= NSYSCALL)
+#error 1 << SYSTRACE_SHIFT must exceed number of Mach traps
+#endif
+
+typedef systrace_sysent_t machtrace_sysent_t;
+
+static machtrace_sysent_t *machtrace_sysent = NULL;
+
+void (*machtrace_probe)(dtrace_id_t, uint64_t, uint64_t,
+    uint64_t, uint64_t, uint64_t);
+
+static dev_info_t *machtrace_devi;
+static dtrace_provider_id_t machtrace_id;
+
+static kern_return_t
+dtrace_machtrace_syscall(struct mach_call_args *args)
+{
+	boolean_t           flavor;
+	unsigned short      code;
+
+	machtrace_sysent_t *sy;
+	dtrace_id_t id;
+	kern_return_t rval;
+#if 0 /* XXX */
+	proc_t *p;
+#endif
+	syscall_arg_t *ip = (syscall_arg_t *)args;
+	mach_call_t mach_call;
+
+#if defined (__ppc__) || defined (__ppc64__)
+	{
+		savearea_t *regs = (savearea_t *)find_user_regs(current_thread());
+
+		flavor = (((unsigned int)regs->save_r0) == 0)? 1: 0;
+
+		if (flavor)
+			code = -regs->save_r3;
+		else
+			code = -regs->save_r0;
+	}
+#elif defined(__i386__) || defined (__x86_64__)
+#pragma unused(flavor)
+	{
+		x86_saved_state_t   *tagged_regs = (x86_saved_state_t *)find_user_regs(current_thread());
+
+		if (is_saved_state64(tagged_regs)) {
+			code = -saved_state64(tagged_regs)->rax & SYSCALL_NUMBER_MASK;
+		} else {
+			code = -saved_state32(tagged_regs)->eax;
+		}
+	}
+#elif defined(__arm__)
+	do {} while(0); /* XXX ARMTODO */
+#else
+#error Unknown Architecture
+#endif
+
+	sy = &machtrace_sysent[code];
+
+	if ((id = sy->stsy_entry) != DTRACE_IDNONE)
+		(*machtrace_probe)(id, *ip, *(ip+1), *(ip+2), *(ip+3), *(ip+4));
+
+#if 0 /* XXX */
+	/*
+	 * We want to explicitly allow DTrace consumers to stop a process
+	 * before it actually executes the meat of the syscall.
+	 */
+	p = ttoproc(curthread);
+	mutex_enter(&p->p_lock);
+	if (curthread->t_dtrace_stop && !curthread->t_lwp->lwp_nostop) {
+		curthread->t_dtrace_stop = 0;
+		stop(PR_REQUESTED, 0);
+	}
+	mutex_exit(&p->p_lock);
+#endif
+
+	mach_call = (mach_call_t)(*sy->stsy_underlying);
+	rval = mach_call(args);
+
+	if ((id = sy->stsy_return) != DTRACE_IDNONE)
+		(*machtrace_probe)(id, (uint64_t)rval, 0, 0, 0, 0);
+
+	return (rval);
+}
+
+static void
+machtrace_init(mach_trap_t *actual, machtrace_sysent_t **interposed)
+{
+	machtrace_sysent_t *msysent = *interposed;
+	int i;
+
+	if (msysent == NULL) {
+		*interposed = msysent = kmem_zalloc(sizeof (machtrace_sysent_t) *
+				NSYSCALL, KM_SLEEP);
+	}
+
+	for (i = 0; i < NSYSCALL; i++) {
+		mach_trap_t *a = &actual[i];
+		machtrace_sysent_t *s = &msysent[i];
+
+		if (LOADABLE_SYSCALL(a) && !LOADED_SYSCALL(a))
+			continue;
+
+		if ((mach_call_t)(a->mach_trap_function) == (mach_call_t)(dtrace_machtrace_syscall))
+			continue;
+
+		s->stsy_underlying = a->mach_trap_function;
+	}
+}
+
+/*ARGSUSED*/
+static void
+machtrace_provide(void *arg, const dtrace_probedesc_t *desc)
+{
+	int i;
+
+	if (desc != NULL)
+		return;
+
+	machtrace_init(mach_trap_table, &machtrace_sysent);
+
+	for (i = 0; i < NSYSCALL; i++) {
+		
+		if (machtrace_sysent[i].stsy_underlying == NULL)
+			continue;
+
+		if (dtrace_probe_lookup(machtrace_id, NULL,
+					mach_name_table[i], "entry") != 0)
+			continue;
+
+		(void) dtrace_probe_create(machtrace_id, NULL, mach_name_table[i],
+					   "entry", MACHTRACE_ARTIFICIAL_FRAMES,
+					   (void *)((uintptr_t)SYSTRACE_ENTRY(i)));
+		(void) dtrace_probe_create(machtrace_id, NULL, mach_name_table[i],
+					   "return", MACHTRACE_ARTIFICIAL_FRAMES,
+					   (void *)((uintptr_t)SYSTRACE_RETURN(i)));
+
+		machtrace_sysent[i].stsy_entry = DTRACE_IDNONE;
+		machtrace_sysent[i].stsy_return = DTRACE_IDNONE;
+	}
+}
+
+/*ARGSUSED*/
+static void
+machtrace_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+
+	/*
+	 * There's nothing to do here but assert that we have actually been
+	 * disabled.
+	 */
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		ASSERT(machtrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE);
+	} else {
+		ASSERT(machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
+	}
+}
+
+/*ARGSUSED*/
+static void
+machtrace_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+	int enabled = (machtrace_sysent[sysnum].stsy_entry != DTRACE_IDNONE ||
+			machtrace_sysent[sysnum].stsy_return != DTRACE_IDNONE);
+
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		machtrace_sysent[sysnum].stsy_entry = id;
+	} else {
+		machtrace_sysent[sysnum].stsy_return = id;
+	}
+
+	if (enabled) {
+		ASSERT(sysent[sysnum].sy_callc == dtrace_machtrace_syscall);
+		return;
+	}
+
+	(void) casptr(&mach_trap_table[sysnum].mach_trap_function,
+		      (void *)machtrace_sysent[sysnum].stsy_underlying,
+		      (void *)dtrace_machtrace_syscall);
+}
+
+/*ARGSUSED*/
+static void
+machtrace_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	int sysnum = SYSTRACE_SYSNUM((uintptr_t)parg);
+	int disable = (machtrace_sysent[sysnum].stsy_entry == DTRACE_IDNONE ||
+			machtrace_sysent[sysnum].stsy_return == DTRACE_IDNONE);
+
+	if (disable) {
+		(void) casptr(&mach_trap_table[sysnum].mach_trap_function,
+			      (void *)dtrace_machtrace_syscall,
+			      (void *)machtrace_sysent[sysnum].stsy_underlying);
+
+	}
+
+	if (SYSTRACE_ISENTRY((uintptr_t)parg)) {
+		machtrace_sysent[sysnum].stsy_entry = DTRACE_IDNONE;
+	} else {
+		machtrace_sysent[sysnum].stsy_return = DTRACE_IDNONE;
+	}
+}
+
+static dtrace_pattr_t machtrace_attr = {
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
+};
+
+static dtrace_pops_t machtrace_pops = {
+	machtrace_provide,
+	NULL,
+	machtrace_enable,
+	machtrace_disable,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+	machtrace_destroy
+};
+
+static int
+machtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
+{
+	switch (cmd) {
+		case DDI_ATTACH:
+			break;
+		case DDI_RESUME:
+			return (DDI_SUCCESS);
+		default:
+			return (DDI_FAILURE);
+	}
+
+	machtrace_probe = dtrace_probe;
+	membar_enter();
+
+	if (ddi_create_minor_node(devi, "machtrace", S_IFCHR, 0,
+				DDI_PSEUDO, NULL) == DDI_FAILURE ||
+			dtrace_register("mach_trap", &machtrace_attr, DTRACE_PRIV_USER, NULL,
+				&machtrace_pops, NULL, &machtrace_id) != 0) {
+		machtrace_probe = systrace_stub;
+		ddi_remove_minor_node(devi, NULL);
+		return (DDI_FAILURE);
+	}
+
+	ddi_report_dev(devi);
+	machtrace_devi = devi;
+
+	return (DDI_SUCCESS);
+}
+
+d_open_t _systrace_open;
+
+int _systrace_open(dev_t dev, int flags, int devtype, struct proc *p)
+{
+#pragma unused(dev,flags,devtype,p)
+	return 0;
+}
+
+#define SYSTRACE_MAJOR  -24 /* let the kernel pick the device number */
+
+/*
+ * A struct describing which functions will get invoked for certain
+ * actions.
+ */
+static struct cdevsw systrace_cdevsw =
+{
+	_systrace_open,		/* open */
+	eno_opcl,		/* close */
+	eno_rdwrt,			/* read */
+	eno_rdwrt,			/* write */
+	eno_ioctl,		/* ioctl */
+	(stop_fcn_t *)nulldev, /* stop */
+	(reset_fcn_t *)nulldev, /* reset */
+	NULL,				/* tty's */
+	eno_select,			/* select */
+	eno_mmap,			/* mmap */
+	eno_strat,			/* strategy */
+	eno_getc,			/* getc */
+	eno_putc,			/* putc */
+	0					/* type */
+};
+
+static int gSysTraceInited = 0;
+
+void systrace_init( void );
+
+void systrace_init( void )
+{
+	if (0 == gSysTraceInited) {
+		int majdevno = cdevsw_add(SYSTRACE_MAJOR, &systrace_cdevsw);
+
+		if (majdevno < 0) {
+			printf("systrace_init: failed to allocate a major number!\n");
+			gSysTraceInited = 0;
+			return;
+		}
+
+		systrace_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+		machtrace_attach( (dev_info_t	*)majdevno, DDI_ATTACH );
+
+		gSysTraceInited = 1;
+	} else
+		panic("systrace_init: called twice!\n");
+}
+#undef SYSTRACE_MAJOR
+#endif /* __APPLE__ */
diff --git a/bsd/dev/dtrace/systrace.h b/bsd/dev/dtrace/systrace.h
new file mode 100644
index 000000000..454b71515
--- /dev/null
+++ b/bsd/dev/dtrace/systrace.h
@@ -0,0 +1,94 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_SYSTRACE_H
+#define	_SYS_SYSTRACE_H
+
+/* #pragma ident	"@(#)systrace.h	1.2	05/06/08 SMI" */
+
+#if defined(__APPLE__)
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#include <sys/dtrace.h>
+
+#endif /* __APPLE__ */
+#include <sys/dtrace.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#ifdef _KERNEL
+
+typedef struct systrace_sysent {
+	dtrace_id_t	stsy_entry;
+	dtrace_id_t	stsy_return;
+#if !defined(__APPLE__)
+	int64_t		(*stsy_underlying)();
+#else
+	int32_t		(*stsy_underlying)();
+	int32_t		stsy_return_type;
+#endif /* __APPLE__ */
+} systrace_sysent_t;
+
+extern systrace_sysent_t *systrace_sysent;
+extern systrace_sysent_t *systrace_sysent32;
+
+#if !defined(__APPLE__)
+extern void (*systrace_probe)(dtrace_id_t, uintptr_t, uintptr_t,
+    uintptr_t, uintptr_t, uintptr_t);
+extern void systrace_stub(dtrace_id_t, uintptr_t, uintptr_t,
+    uintptr_t, uintptr_t, uintptr_t);
+
+extern int64_t dtrace_systrace_syscall(uintptr_t arg0, uintptr_t arg1,
+    uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5);
+#else
+extern void (*systrace_probe)(dtrace_id_t, uint64_t, uint64_t,
+    uint64_t, uint64_t, uint64_t);
+extern void systrace_stub(dtrace_id_t, uint64_t, uint64_t,
+    uint64_t, uint64_t, uint64_t);
+
+extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
+
+extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
+#endif /* __APPLE__ */
+
+#ifdef _SYSCALL32_IMPL
+extern int64_t dtrace_systrace_syscall32(uintptr_t arg0, uintptr_t arg1,
+    uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5);
+#endif
+
+#endif
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_SYSTRACE_H */
diff --git a/bsd/dev/i386/conf.c b/bsd/dev/i386/conf.c
index 0e5ea99b4..0b39c35cd 100644
--- a/bsd/dev/i386/conf.c
+++ b/bsd/dev/i386/conf.c
@@ -1,26 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved
  * Copyright (c) 1993 NeXT Computer, Inc.
  *
  * UNIX Device switch tables.
@@ -49,14 +54,14 @@ struct bdevsw	bdevsw[] =
 {
 	/*
 	 *	For block devices, every other block of 8 slots is 
-	 *	reserved to NeXT.  The other slots are available for
+	 *	reserved for Apple.  The other slots are available for
 	 *	the user.  This way we can both add new entries without
-	 *	running into each other.  Be sure to fill in NeXT's
+	 *	running into each other.  Be sure to fill in Apple's
 	 *	8 reserved slots when you jump over us -- we'll do the
 	 *	same for you.
 	 */
 
-	/* 0 - 7 are reserved to NeXT */
+	/* 0 - 7 are reserved for Apple */
 
 	NO_BDEVICE,							/* 0*/
 	NO_BDEVICE,							/* 1*/
@@ -77,7 +82,7 @@ struct bdevsw	bdevsw[] =
 	NO_BDEVICE,							/*14*/
 	NO_BDEVICE,							/*15*/
 
-	/* 16 - 23 are reserved to NeXT */
+	/* 16 - 23 are reserved for Apple */
 	NO_BDEVICE,							/*16*/
 	NO_BDEVICE,							/*17*/
 	NO_BDEVICE,							/*18*/
@@ -187,14 +192,14 @@ struct cdevsw	cdevsw[] =
 {
 	/*
 	 *	For character devices, every other block of 16 slots is
-	 *	reserved to NeXT.  The other slots are available for
+	 *	reserved for Apple.  The other slots are available for
 	 *	the user.  This way we can both add new entries without
-	 *	running into each other.  Be sure to fill in NeXT's
+	 *	running into each other.  Be sure to fill in Apple's
 	 *	16 reserved slots when you jump over us -- we'll do the
 	 *	same for you.
 	 */
 
-	/* 0 - 15 are reserved to NeXT */
+	/* 0 - 15 are reserved for Apple */
 
     {
 	cnopen,		cnclose,	cnread,		cnwrite,	/* 0*/
diff --git a/bsd/dev/i386/cons.c b/bsd/dev/i386/cons.c
index 43f98d1ef..b5e3e7289 100644
--- a/bsd/dev/i386/cons.c
+++ b/bsd/dev/i386/cons.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1987, 1988 NeXT, Inc.
@@ -267,7 +273,7 @@ alert(
 {
 	char smsg[200];
 	
-	sprintf(smsg, msg,  p1, p2, p3, p4, p5, p6, p7, p8);
+	snprintf(smsg, sizeof(smsg), msg,  p1, p2, p3, p4, p5, p6, p7, p8);
 #if FIXME  /* [ */
 	/* DoAlert(title, smsg); */
 #else
diff --git a/bsd/dev/i386/cons.h b/bsd/dev/i386/cons.h
index e14004aa2..3ea2d28d6 100644
--- a/bsd/dev/i386/cons.h
+++ b/bsd/dev/i386/cons.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1987 NeXT, Inc.
diff --git a/bsd/dev/i386/dis_tables.c b/bsd/dev/i386/dis_tables.c
new file mode 100644
index 000000000..08519b456
--- /dev/null
+++ b/bsd/dev/i386/dis_tables.c
@@ -0,0 +1,3305 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1988 AT&T	*/
+/*	  All Rights Reserved  	*/
+
+
+/* #pragma ident	"@(#)dis_tables.c	1.13	06/06/15 SMI" */
+
+#if !defined(__APPLE__)
+#include	"dis_tables.h"
+#else
+#include <sys/dtrace.h>
+#include <sys/dtrace_glue.h>
+
+#include <sys/dis_tables.h>
+#endif /* __APPLE__ */
+
+/* BEGIN CSTYLED */
+
+/*
+ * Disassembly begins in dis_distable, which is equivalent to the One-byte
+ * Opcode Map in the Intel IA32 ISA Reference (page A-6 in my copy).  The
+ * decoding loops then traverse out through the other tables as necessary to
+ * decode a given instruction.
+ *
+ * The behavior of this file can be controlled by one of the following flags:
+ *
+ * 	DIS_TEXT	Include text for disassembly
+ * 	DIS_MEM		Include memory-size calculations
+ *
+ * Either or both of these can be defined.
+ *
+ * This file is not, and will never be, cstyled.  If anything, the tables should
+ * be taken out another tab stop or two so nothing overlaps.
+ */
+
+/*
+ * These functions must be provided for the consumer to do disassembly.
+ */
+#ifdef DIS_TEXT
+extern char *strncpy(char *, const char *, size_t);
+extern size_t strlen(const char *);
+extern int strcmp(const char *, const char *);
+extern int strncmp(const char *, const char *, size_t);
+extern size_t strlcat(char *, const char *, size_t);
+#endif
+
+
+#define		TERM 	0	/* used to indicate that the 'indirect' */
+				/* field terminates - no pointer.	*/
+
+/* Used to decode instructions. */
+typedef struct	instable {
+	struct instable	*it_indirect;	/* for decode op codes */
+	uchar_t		it_adrmode;
+#ifdef DIS_TEXT
+	char		it_name[NCPS];
+	uint_t		it_suffix:1;		/* mnem + "w", "l", or "d" */
+#endif
+#ifdef DIS_MEM
+	uint_t		it_size:16;
+#endif
+	uint_t		it_invalid64:1;		/* opcode invalid in amd64 */
+	uint_t		it_always64:1;		/* 64 bit when in 64 bit mode */
+	uint_t		it_invalid32:1;		/* invalid in IA32 */
+	uint_t		it_stackop:1;		/* push/pop stack operation */
+} instable_t;
+
+/*
+ * Instruction formats.
+ */
+enum {
+	UNKNOWN,
+	MRw,
+	IMlw,
+	IMw,
+	IR,
+	OA,
+	AO,
+	MS,
+	SM,
+	Mv,
+	Mw,
+	M,		/* register or memory */
+	Mb,		/* register or memory, always byte sized */
+	MO,		/* memory only (no registers) */
+	PREF,
+	SWAPGS,
+	R,
+	RA,
+	SEG,
+	MR,
+	RM,
+	IA,
+	MA,
+	SD,
+	AD,
+	SA,
+	D,
+	INM,
+	SO,
+	BD,
+	I,
+	P,
+	V,
+	DSHIFT,		/* for double shift that has an 8-bit immediate */
+	U,
+	OVERRIDE,
+	NORM,		/* instructions w/o ModR/M byte, no memory access */
+	IMPLMEM,	/* instructions w/o ModR/M byte, implicit mem access */
+	O,		/* for call	*/
+	JTAB,		/* jump table 	*/
+	IMUL,		/* for 186 iimul instr  */
+	CBW,		/* so data16 can be evaluated for cbw and variants */
+	MvI,		/* for 186 logicals */
+	ENTER,		/* for 186 enter instr  */
+	RMw,		/* for 286 arpl instr */
+	Ib,		/* for push immediate byte */
+	F,		/* for 287 instructions */
+	FF,		/* for 287 instructions */
+	FFC,		/* for 287 instructions */
+	DM,		/* 16-bit data */
+	AM,		/* 16-bit addr */
+	LSEG,		/* for 3-bit seg reg encoding */
+	MIb,		/* for 386 logicals */
+	SREG,		/* for 386 special registers */
+	PREFIX,		/* a REP instruction prefix */
+	LOCK,		/* a LOCK instruction prefix */
+	INT3,		/* The int 3 instruction, which has a fake operand */
+	INTx,		/* The normal int instruction, with explicit int num */
+	DSHIFTcl,	/* for double shift that implicitly uses %cl */
+	CWD,		/* so data16 can be evaluated for cwd and variants */
+	RET,		/* single immediate 16-bit operand */
+	MOVZ,		/* for movs and movz, with different size operands */
+	XADDB,		/* for xaddb */
+	MOVSXZ,		/* AMD64 mov sign extend 32 to 64 bit instruction */
+
+/*
+ * MMX/SIMD addressing modes.
+ */
+
+	MMO,		/* Prefixable MMX/SIMD-Int	mm/mem	-> mm */
+	MMOIMPL,	/* Prefixable MMX/SIMD-Int	mm	-> mm (mem) */
+	MMO3P,		/* Prefixable MMX/SIMD-Int	mm	-> r32,imm8 */
+	MMOM3,		/* Prefixable MMX/SIMD-Int	mm	-> r32 	*/
+	MMOS,		/* Prefixable MMX/SIMD-Int	mm	-> mm/mem */
+	MMOMS,		/* Prefixable MMX/SIMD-Int	mm	-> mem */
+	MMOPM,		/* MMX/SIMD-Int			mm/mem	-> mm,imm8 */
+	MMOPRM,		/* Prefixable MMX/SIMD-Int	r32/mem	-> mm,imm8 */
+	MMOSH,		/* Prefixable MMX		mm,imm8	*/
+	MM,		/* MMX/SIMD-Int			mm/mem	-> mm	*/
+	MMS,		/* MMX/SIMD-Int			mm	-> mm/mem */
+	MMSH,		/* MMX				mm,imm8 */
+	XMMO,		/* Prefixable SIMD		xmm/mem	-> xmm */
+	XMMOS,		/* Prefixable SIMD		xmm	-> xmm/mem */
+	XMMOPM,		/* Prefixable SIMD		xmm/mem	w/to xmm,imm8 */
+	XMMOMX,		/* Prefixable SIMD		mm/mem	-> xmm */
+	XMMOX3,		/* Prefixable SIMD		xmm	-> r32 */
+	XMMOXMM,	/* Prefixable SIMD		xmm/mem	-> mm	*/
+	XMMOM,		/* Prefixable SIMD		xmm	-> mem */
+	XMMOMS,		/* Prefixable SIMD		mem	-> xmm */
+	XMM,		/* SIMD 			xmm/mem	-> xmm */
+	XMMXIMPL,	/* SIMD				xmm	-> xmm (mem) */
+	XMM3P,		/* SIMD				xmm	-> r32,imm8 */
+	XMMP,		/* SIMD 			xmm/mem w/to xmm,imm8 */
+	XMMPRM,		/* SIMD 			r32/mem -> xmm,imm8 */
+	XMMS,		/* SIMD				xmm	-> xmm/mem */
+	XMMM,		/* SIMD 			mem	-> xmm */
+	XMMMS,		/* SIMD				xmm	-> mem */
+	XMM3MX,		/* SIMD 			r32/mem -> xmm */
+	XMM3MXS,	/* SIMD 			xmm	-> r32/mem */
+	XMMSH,		/* SIMD 			xmm,imm8 */
+	XMMXM3,		/* SIMD 			xmm/mem -> r32 */
+	XMMX3,		/* SIMD 			xmm	-> r32 */
+	XMMXMM,		/* SIMD 			xmm/mem	-> mm */
+	XMMMX,		/* SIMD 			mm	-> xmm */
+	XMMXM,		/* SIMD 			xmm	-> mm */
+	XMMFENCE,	/* SIMD lfence or mfence */
+	XMMSFNC		/* SIMD sfence (none or mem) */
+};
+
+#define	FILL	0x90	/* Fill byte used for alignment (nop)	*/
+
+/*
+** Register numbers for the i386
+*/
+#define	EAX_REGNO 0
+#define	ECX_REGNO 1
+#define	EDX_REGNO 2
+#define	EBX_REGNO 3
+#define	ESP_REGNO 4
+#define	EBP_REGNO 5
+#define	ESI_REGNO 6
+#define	EDI_REGNO 7
+
+/*
+ * modes for immediate values
+ */
+#define	MODE_NONE	0
+#define	MODE_IPREL	1	/* signed IP relative value */
+#define	MODE_SIGNED	2	/* sign extended immediate */
+#define	MODE_IMPLIED	3	/* constant value implied from opcode */
+#define	MODE_OFFSET	4	/* offset part of an address */
+#define	MODE_RIPREL	5	/* like IPREL, but from %rip (amd64) */
+
+/*
+ * The letters used in these macros are:
+ *   IND - indirect to another to another table
+ *   "T" - means to Terminate indirections (this is the final opcode)
+ *   "S" - means "operand length suffix required"
+ *   "NS" - means "no suffix" which is the operand length suffix of the opcode
+ *   "Z" - means instruction size arg required
+ *   "u" - means the opcode is invalid in IA32 but valid in amd64
+ *   "x" - means the opcode is invalid in amd64, but not IA32
+ *   "y" - means the operand size is always 64 bits in 64 bit mode
+ *   "p" - means push/pop stack operation
+ */
+
+#if defined(DIS_TEXT) && defined(DIS_MEM)
+#define	IND(table)		{(instable_t *)table, 0, "", 0, 0, 0, 0, 0, 0}
+#define	INDx(table)		{(instable_t *)table, 0, "", 0, 0, 1, 0, 0, 0}
+#define	TNS(name, amode)	{TERM, amode, name, 0, 0, 0, 0, 0, 0}
+#define	TNSu(name, amode)	{TERM, amode, name, 0, 0, 0, 0, 1, 0}
+#define	TNSx(name, amode)	{TERM, amode, name, 0, 0, 1, 0, 0, 0}
+#define	TNSy(name, amode)	{TERM, amode, name, 0, 0, 0, 1, 0, 0}
+#define	TNSyp(name, amode)	{TERM, amode, name, 0, 0, 0, 1, 0, 1}
+#define	TNSZ(name, amode, sz)	{TERM, amode, name, 0, sz, 0, 0, 0, 0}
+#define	TNSZy(name, amode, sz)	{TERM, amode, name, 0, sz, 0, 1, 0, 0}
+#define	TS(name, amode)		{TERM, amode, name, 1, 0, 0, 0, 0, 0}
+#define	TSx(name, amode)	{TERM, amode, name, 1, 0, 1, 0, 0, 0}
+#define	TSy(name, amode)	{TERM, amode, name, 1, 0, 0, 1, 0, 0}
+#define	TSp(name, amode)	{TERM, amode, name, 1, 0, 0, 0, 0, 1}
+#define	TSZ(name, amode, sz)	{TERM, amode, name, 1, sz, 0, 0, 0, 0}
+#define	TSZx(name, amode, sz)	{TERM, amode, name, 1, sz, 1, 0, 0, 0}
+#define	TSZy(name, amode, sz)	{TERM, amode, name, 1, sz, 0, 1, 0, 0}
+#define	INVALID			{TERM, UNKNOWN, "", 0, 0, 0, 0, 0}
+#elif defined(DIS_TEXT)
+#define	IND(table)		{(instable_t *)table, 0, "", 0, 0, 0, 0, 0}
+#define	INDx(table)		{(instable_t *)table, 0, "", 0, 1, 0, 0, 0}
+#define	TNS(name, amode)	{TERM, amode, name, 0, 0, 0, 0, 0}
+#define	TNSu(name, amode)	{TERM, amode, name, 0, 0, 0, 1, 0}
+#define	TNSx(name, amode)	{TERM, amode, name, 0, 1, 0, 0, 0}
+#define	TNSy(name, amode)	{TERM, amode, name, 0, 0, 1, 0, 0}
+#define	TNSyp(name, amode)	{TERM, amode, name, 0, 0, 1, 0, 1}
+#define	TNSZ(name, amode, sz)	{TERM, amode, name, 0, 0, 0, 0, 0}
+#define	TNSZy(name, amode, sz)	{TERM, amode, name, 0, 0, 1, 0, 0}
+#define	TS(name, amode)		{TERM, amode, name, 1, 0, 0, 0, 0}
+#define	TSx(name, amode)	{TERM, amode, name, 1, 1, 0, 0, 0}
+#define	TSy(name, amode)	{TERM, amode, name, 1, 0, 1, 0, 0}
+#define	TSp(name, amode)	{TERM, amode, name, 1, 0, 0, 0, 1}
+#define	TSZ(name, amode, sz)	{TERM, amode, name, 1, 0, 0, 0, 0}
+#define	TSZx(name, amode, sz)	{TERM, amode, name, 1, 1, 0, 0, 0}
+#define	TSZy(name, amode, sz)	{TERM, amode, name, 1, 0, 1, 0, 0}
+#define	INVALID			{TERM, UNKNOWN, "", 0, 0, 0, 0, 0}
+#elif defined(DIS_MEM)
+#define	IND(table)		{(instable_t *)table, 0, 0, 0, 0, 0, 0}
+#define	INDx(table)		{(instable_t *)table, 0, 0, 1, 0, 0, 0}
+#define	TNS(name, amode)	{TERM, amode,  0, 0, 0, 0, 0}
+#define	TNSu(name, amode)	{TERM, amode,  0, 0, 0, 1, 0}
+#define	TNSy(name, amode)	{TERM, amode,  0, 0, 1, 0, 0}
+#define	TNSyp(name, amode)	{TERM, amode,  0, 0, 1, 0, 1}
+#define	TNSx(name, amode)	{TERM, amode,  0, 1, 0, 0, 0}
+#define	TNSZ(name, amode, sz)	{TERM, amode, sz, 0, 0, 0, 0}
+#define	TNSZy(name, amode, sz)	{TERM, amode, sz, 0, 1, 0, 0}
+#define	TS(name, amode)		{TERM, amode,  0, 0, 0, 0, 0}
+#define	TSx(name, amode)	{TERM, amode,  0, 1, 0, 0, 0}
+#define	TSy(name, amode)	{TERM, amode,  0, 0, 1, 0, 0}
+#define	TSp(name, amode)	{TERM, amode,  0, 0, 0, 0, 1}
+#define	TSZ(name, amode, sz)	{TERM, amode, sz, 0, 0, 0, 0}
+#define	TSZx(name, amode, sz)	{TERM, amode, sz, 1, 0, 0, 0}
+#define	TSZy(name, amode, sz)	{TERM, amode, sz, 0, 1, 0, 0}
+#define	INVALID			{TERM, UNKNOWN, 0, 0, 0, 0, 0}
+#else
+#define	IND(table)		{(instable_t *)table, 0, 0, 0, 0, 0}
+#define	INDx(table)		{(instable_t *)table, 0, 1, 0, 0, 0}
+#define	TNS(name, amode)	{TERM, amode,  0, 0, 0, 0}
+#define	TNSu(name, amode)	{TERM, amode,  0, 0, 1, 0}
+#define	TNSy(name, amode)	{TERM, amode,  0, 1, 0, 0}
+#define	TNSyp(name, amode)	{TERM, amode,  0, 1, 0, 1}
+#define	TNSx(name, amode)	{TERM, amode,  1, 0, 0, 0}
+#define	TNSZ(name, amode, sz)	{TERM, amode,  0, 0, 0, 0}
+#define	TNSZy(name, amode, sz)	{TERM, amode,  0, 1, 0, 0}
+#define	TS(name, amode)		{TERM, amode,  0, 0, 0, 0}
+#define	TSx(name, amode)	{TERM, amode,  1, 0, 0, 0}
+#define	TSy(name, amode)	{TERM, amode,  0, 1, 0, 0}
+#define	TSp(name, amode)	{TERM, amode,  0, 0, 0, 1}
+#define	TSZ(name, amode, sz)	{TERM, amode,  0, 0, 0, 0}
+#define	TSZx(name, amode, sz)	{TERM, amode,  1, 0, 0, 0}
+#define	TSZy(name, amode, sz)	{TERM, amode,  0, 1, 0, 0}
+#define	INVALID			{TERM, UNKNOWN, 0, 0, 0, 0}
+#endif
+
+#ifdef DIS_TEXT
+/*
+ * this decodes the r_m field for mode's 0, 1, 2 in 16 bit mode
+ */
+const char *const dis_addr16[3][8] = {
+"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di)", "",
+									"(%bx)",
+"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di", "(%bp)",
+									"(%bx)",
+"(%bx,%si)", "(%bx,%di)", "(%bp,%si)", "(%bp,%di)", "(%si)", "(%di)", "(%bp)",
+									"(%bx)",
+};
+
+
+/*
+ * This decodes 32 bit addressing mode r_m field for modes 0, 1, 2
+ */
+const char *const dis_addr32_mode0[16] = {
+  "(%eax)", "(%ecx)", "(%edx)",  "(%ebx)",  "", "",        "(%esi)",  "(%edi)",
+  "(%r8d)", "(%r9d)", "(%r10d)", "(%r11d)", "", "",        "(%r14d)", "(%r15d)"
+};
+
+const char *const dis_addr32_mode12[16] = {
+  "(%eax)", "(%ecx)", "(%edx)",  "(%ebx)",  "", "(%ebp)",  "(%esi)",  "(%edi)",
+  "(%r8d)", "(%r9d)", "(%r10d)", "(%r11d)", "", "(%r13d)", "(%r14d)", "(%r15d)"
+};
+
+/*
+ * This decodes 64 bit addressing mode r_m field for modes 0, 1, 2
+ */
+const char *const dis_addr64_mode0[16] = {
+ "(%rax)", "(%rcx)", "(%rdx)", "(%rbx)", "",       "(%rip)", "(%rsi)", "(%rdi)",
+ "(%r8)",  "(%r9)",  "(%r10)", "(%r11)", "(%r12)", "(%rip)", "(%r14)", "(%r15)"
+};
+const char *const dis_addr64_mode12[16] = {
+ "(%rax)", "(%rcx)", "(%rdx)", "(%rbx)", "",       "(%rbp)", "(%rsi)", "(%rdi)",
+ "(%r8)",  "(%r9)",  "(%r10)", "(%r11)", "(%r12)", "(%r13)", "(%r14)", "(%r15)"
+};
+
+/*
+ * decode for scale from SIB byte
+ */
+const char *const dis_scale_factor[4] = { ")", ",2)", ",4)", ",8)" };
+
+/*
+ * register decoding for normal references to registers (ie. not addressing)
+ */
+const char *const dis_REG8[16] = {
+	"%al",  "%cl",  "%dl",   "%bl",   "%ah",   "%ch",   "%dh",   "%bh",
+	"%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b"
+};
+
+const char *const dis_REG8_REX[16] = {
+	"%al",  "%cl",  "%dl",   "%bl",   "%spl",  "%bpl",  "%sil",  "%dil",
+	"%r8b", "%r9b", "%r10b", "%r11b", "%r12b", "%r13b", "%r14b", "%r15b"
+};
+
+const char *const dis_REG16[16] = {
+	"%ax",  "%cx",  "%dx",   "%bx",   "%sp",   "%bp",   "%si",   "%di",
+	"%r8w", "%r9w", "%r10w", "%r11w", "%r12w", "%r13w", "%r14w", "%r15w"
+};
+
+const char *const dis_REG32[16] = {
+	"%eax", "%ecx", "%edx",  "%ebx",  "%esp",  "%ebp",  "%esi",  "%edi",
+	"%r8d", "%r9d", "%r10d", "%r11d", "%r12d", "%r13d", "%r14d", "%r15d"
+};
+
+const char *const dis_REG64[16] = {
+	"%rax", "%rcx", "%rdx",  "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
+	"%r8",  "%r9",  "%r10",  "%r11", "%r12", "%r13", "%r14", "%r15"
+};
+
+const char *const dis_DEBUGREG[16] = {
+	"%db0", "%db1", "%db2",  "%db3",  "%db4",  "%db5",  "%db6",  "%db7",
+	"%db8", "%db9", "%db10", "%db11", "%db12", "%db13", "%db14", "%db15"
+};
+
+const char *const dis_CONTROLREG[16] = {
+    "%cr0", "%cr1", "%cr2", "%cr3", "%cr4", "%cr5?", "%cr6?", "%cr7?",
+    "%cr8", "%cr9?", "%cr10?", "%cr11?", "%cr12?", "%cr13?", "%cr14?", "%cr15?"
+};
+
+const char *const dis_TESTREG[16] = {
+	"%tr0?", "%tr1?", "%tr2?", "%tr3", "%tr4", "%tr5", "%tr6", "%tr7",
+	"%tr0?", "%tr1?", "%tr2?", "%tr3", "%tr4", "%tr5", "%tr6", "%tr7"
+};
+
+const char *const dis_MMREG[16] = {
+	"%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7",
+	"%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7"
+};
+
+const char *const dis_XMMREG[16] = {
+    "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7",
+    "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15"
+};
+
+const char *const dis_SEGREG[16] = {
+	"%es", "%cs", "%ss", "%ds", "%fs", "%gs", "<reserved>", "<reserved>",
+	"%es", "%cs", "%ss", "%ds", "%fs", "%gs", "<reserved>", "<reserved>"
+};
+
+/*
+ * SIMD predicate suffixes
+ */
+const char *const dis_PREDSUFFIX[8] = {
+	"eq", "lt", "le", "unord", "neq", "nlt", "nle", "ord"
+};
+
+
+
+#endif	/* DIS_TEXT */
+
+
+
+
+/*
+ *	"decode table" for 64 bit mode MOVSXD instruction (opcode 0x63)
+ */
+const instable_t dis_opMOVSLD = TNS("movslq",MOVSXZ);
+
+/*
+ *	"decode table" for pause and clflush instructions
+ */
+const instable_t dis_opPause = TNS("pause", NORM);
+
+/*
+ *	Decode table for 0x0F00 opcodes
+ */
+const instable_t dis_op0F00[8] = {
+
+/*  [0]  */	TNS("sldt",M),		TNS("str",M),		TNSy("lldt",M), 	TNSy("ltr",M),
+/*  [4]  */	TNSZ("verr",M,2),	TNSZ("verw",M,2),	INVALID,		INVALID,
+};
+
+
+/*
+ *	Decode table for 0x0F01 opcodes
+ */
+const instable_t dis_op0F01[8] = {
+
+/*  [0]  */	TNSZ("sgdt",MO,6),	TNSZ("sidt",MO,6), 	TNSZ("lgdt",MO,6),	TNSZ("lidt",MO,6),
+/*  [4]  */	TNSZ("smsw",M,2),	INVALID, 		TNSZ("lmsw",M,2),	TNS("invlpg",SWAPGS),
+};
+
+/*
+ *	Decode table for 0x0F18 opcodes -- SIMD prefetch
+ */
+const instable_t dis_op0F18[8] = {
+
+/*  [0]  */	TNS("prefetchnta",PREF),TNS("prefetcht0",PREF),	TNS("prefetcht1",PREF),	TNS("prefetcht2",PREF),
+/*  [4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+};
+
+/*
+ * 	Decode table for 0x0FAE opcodes -- SIMD state save/restore
+ */
+const instable_t dis_op0FAE[8] = {
+/*  [0]  */	TNSZ("fxsave",M,512),	TNSZ("fxrstor",M,512),	TNS("ldmxcsr",M),	TNS("stmxcsr",M),
+/*  [4]  */	INVALID,		TNS("lfence",XMMFENCE), TNS("mfence",XMMFENCE),	TNS("sfence",XMMSFNC),
+};
+
+/*
+ *	Decode table for 0x0FBA opcodes
+ */
+
+const instable_t dis_op0FBA[8] = {
+
+/*  [0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [4]  */	TS("bt",MIb),		TS("bts",MIb),		TS("btr",MIb),		TS("btc",MIb),
+};
+
+/*
+ * 	Decode table for 0x0FC7 opcode
+ */
+
+const instable_t dis_op0FC7[8] = {
+
+/*  [0]  */	INVALID,		TNS("cmpxchg8b",M),	INVALID,		INVALID,
+/*  [4]  */	INVALID,		INVALID,	INVALID,		 INVALID,
+};
+
+
+/*
+ *	Decode table for 0x0FC8 opcode -- 486 bswap instruction
+ *
+ *bit pattern: 0000 1111 1100 1reg
+ */
+const instable_t dis_op0FC8[4] = {
+/*  [0]  */	TNS("bswap",R),		INVALID,		INVALID,		INVALID,
+};
+
+/*
+ *	Decode table for 0x0F71, 0x0F72, and 0x0F73 opcodes -- MMX instructions
+ */
+const instable_t dis_op0F7123[4][8] = {
+{
+/*  [70].0 */	INVALID,		INVALID,		INVALID,		INVALID,
+/*      .4 */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [71].0 */	INVALID,		INVALID,		TNS("psrlw",MMOSH),	INVALID,
+/*      .4 */	TNS("psraw",MMOSH),	INVALID,		TNS("psllw",MMOSH),	INVALID,
+}, {
+/*  [72].0 */	INVALID,		INVALID,		TNS("psrld",MMOSH),	INVALID,
+/*      .4 */	TNS("psrad",MMOSH),	INVALID,		TNS("pslld",MMOSH),	INVALID,
+}, {
+/*  [73].0 */	INVALID,		INVALID,		TNS("psrlq",MMOSH),	TNS("INVALID",MMOSH),
+/*      .4 */	INVALID,		INVALID, 		TNS("psllq",MMOSH),	TNS("INVALID",MMOSH),
+} };
+
+/*
+ *	Decode table for SIMD extensions to above 0x0F71-0x0F73 opcodes.
+ */
+const instable_t dis_opSIMD7123[32] = {
+/* [70].0 */	INVALID,		INVALID,		INVALID,		INVALID,
+/*     .4 */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/* [71].0 */	INVALID,		INVALID,		TNS("psrlw",XMMSH),	INVALID,
+/*     .4 */	TNS("psraw",XMMSH),	INVALID,		TNS("psllw",XMMSH),	INVALID,
+
+/* [72].0 */	INVALID,		INVALID,		TNS("psrld",XMMSH),	INVALID,
+/*     .4 */	TNS("psrad",XMMSH),	INVALID,		TNS("pslld",XMMSH),	INVALID,
+
+/* [73].0 */	INVALID,		INVALID,		TNS("psrlq",XMMSH),	TNS("psrldq",XMMSH),
+/*     .4 */	INVALID,		INVALID,		TNS("psllq",XMMSH),	TNS("pslldq",XMMSH),
+};
+
+/*
+ *	SIMD instructions have been wedged into the existing IA32 instruction
+ *	set through the use of prefixes.  That is, while 0xf0 0x58 may be
+ *	addps, 0xf3 0xf0 0x58 (literally, repz addps) is a completely different
+ *	instruction - addss.  At present, three prefixes have been coopted in
+ *	this manner - address size (0x66), repnz (0xf2) and repz (0xf3).  The
+ *	following tables are used to provide the prefixed instruction names.
+ *	The arrays are sparse, but they're fast.
+ */
+
+/*
+ *	Decode table for SIMD instructions with the address size (0x66) prefix.
+ */
+const instable_t dis_opSIMDdata16[256] = {
+/*  [00]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [04]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [08]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [10]  */	TNSZ("movupd",XMM,16),	TNSZ("movupd",XMMS,16),	TNSZ("movlpd",XMMM,8),	TNSZ("movlpd",XMMMS,8),
+/*  [14]  */	TNSZ("unpcklpd",XMM,16),TNSZ("unpckhpd",XMM,16),TNSZ("movhpd",XMMM,8),	TNSZ("movhpd",XMMMS,8),
+/*  [18]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [1C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [20]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [24]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [28]  */	TNSZ("movapd",XMM,16),	TNSZ("movapd",XMMS,16),	TNSZ("cvtpi2pd",XMMOMX,8),TNSZ("movntpd",XMMOMS,16),
+/*  [2C]  */	TNSZ("cvttpd2pi",XMMXMM,16),TNSZ("cvtpd2pi",XMMXMM,16),TNSZ("ucomisd",XMM,8),TNSZ("comisd",XMM,8),
+
+/*  [30]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [34]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [38]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [3C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [40]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [44]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [48]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [4C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [50]  */	TNS("movmskpd",XMMOX3),	TNSZ("sqrtpd",XMM,16),	INVALID,		INVALID,
+/*  [54]  */	TNSZ("andpd",XMM,16),	TNSZ("andnpd",XMM,16),	TNSZ("orpd",XMM,16),	TNSZ("xorpd",XMM,16),
+/*  [58]  */	TNSZ("addpd",XMM,16),	TNSZ("mulpd",XMM,16),	TNSZ("cvtpd2ps",XMM,16),TNSZ("cvtps2dq",XMM,16),
+/*  [5C]  */	TNSZ("subpd",XMM,16),	TNSZ("minpd",XMM,16),	TNSZ("divpd",XMM,16),	TNSZ("maxpd",XMM,16),
+
+/*  [60]  */	TNSZ("punpcklbw",XMM,16),TNSZ("punpcklwd",XMM,16),TNSZ("punpckldq",XMM,16),TNSZ("packsswb",XMM,16),
+/*  [64]  */	TNSZ("pcmpgtb",XMM,16),	TNSZ("pcmpgtw",XMM,16),	TNSZ("pcmpgtd",XMM,16),	TNSZ("packuswb",XMM,16),
+/*  [68]  */	TNSZ("punpckhbw",XMM,16),TNSZ("punpckhwd",XMM,16),TNSZ("punpckhdq",XMM,16),TNSZ("packssdw",XMM,16),
+/*  [6C]  */	TNSZ("punpcklqdq",XMM,16),TNSZ("punpckhqdq",XMM,16),TNSZ("movd",XMM3MX,4),TNSZ("movdqa",XMM,16),
+
+/*  [70]  */	TNSZ("pshufd",XMMP,16),	INVALID,		INVALID,		INVALID,
+/*  [74]  */	TNSZ("pcmpeqb",XMM,16),	TNSZ("pcmpeqw",XMM,16),	TNSZ("pcmpeqd",XMM,16),	INVALID,
+/*  [78]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [7C]  */	INVALID,		INVALID,		TNSZ("movd",XMM3MXS,4),	TNSZ("movdqa",XMMS,16),
+
+/*  [80]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [84]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [88]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [8C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [90]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [94]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [98]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [9C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [A0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [AC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [B0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [BC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [C0]  */	INVALID,		INVALID,		TNSZ("cmppd",XMMP,16),	INVALID,
+/*  [C4]  */	TNSZ("pinsrw",XMMPRM,2),TNS("pextrw",XMM3P),	TNSZ("shufpd",XMMP,16),	INVALID,
+/*  [C8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [CC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [D0]  */	INVALID,		TNSZ("psrlw",XMM,16),	TNSZ("psrld",XMM,16),	TNSZ("psrlq",XMM,16),
+/*  [D4]  */	TNSZ("paddq",XMM,16),	TNSZ("pmullw",XMM,16),	TNSZ("movq",XMMS,8),	TNS("pmovmskb",XMMX3),
+/*  [D8]  */	TNSZ("psubusb",XMM,16),	TNSZ("psubusw",XMM,16),	TNSZ("pminub",XMM,16),	TNSZ("pand",XMM,16),
+/*  [DC]  */	TNSZ("paddusb",XMM,16),	TNSZ("paddusw",XMM,16),	TNSZ("pmaxub",XMM,16),	TNSZ("pandn",XMM,16),
+
+/*  [E0]  */	TNSZ("pavgb",XMM,16),	TNSZ("psraw",XMM,16),	TNSZ("psrad",XMM,16),	TNSZ("pavgw",XMM,16),
+/*  [E4]  */	TNSZ("pmulhuw",XMM,16),	TNSZ("pmulhw",XMM,16),	TNSZ("cvttpd2dq",XMM,16),TNSZ("movntdq",XMMS,16),
+/*  [E8]  */	TNSZ("psubsb",XMM,16),	TNSZ("psubsw",XMM,16),	TNSZ("pminsw",XMM,16),	TNSZ("por",XMM,16),
+/*  [EC]  */	TNSZ("paddsb",XMM,16),	TNSZ("paddsw",XMM,16),	TNSZ("pmaxsw",XMM,16),	TNSZ("pxor",XMM,16),
+
+/*  [F0]  */	INVALID,		TNSZ("psllw",XMM,16),	TNSZ("pslld",XMM,16),	TNSZ("psllq",XMM,16),
+/*  [F4]  */	TNSZ("pmuludq",XMM,16),	TNSZ("pmaddwd",XMM,16),	TNSZ("psadbw",XMM,16),	TNSZ("maskmovdqu", XMMXIMPL,16),
+/*  [F8]  */	TNSZ("psubb",XMM,16),	TNSZ("psubw",XMM,16),	TNSZ("psubd",XMM,16),	TNSZ("psubq",XMM,16),
+/*  [FC]  */	TNSZ("paddb",XMM,16),	TNSZ("paddw",XMM,16),	TNSZ("paddd",XMM,16),	INVALID,
+};
+
+/*
+ *	Decode table for SIMD instructions with the repnz (0xf2) prefix.
+ */
+const instable_t dis_opSIMDrepnz[256] = {
+/*  [00]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [04]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [08]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [10]  */	TNSZ("movsd",XMM,8),	TNSZ("movsd",XMMS,8),	INVALID,		INVALID,
+/*  [14]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [18]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [1C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [20]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [24]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [28]  */	INVALID,		INVALID,		TNSZ("cvtsi2sd",XMM3MX,4),INVALID,
+/*  [2C]  */	TNSZ("cvttsd2si",XMMXM3,8),TNSZ("cvtsd2si",XMMXM3,8),INVALID,		INVALID,
+
+/*  [30]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [34]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [38]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [3C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [40]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [44]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [48]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [4C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [50]  */	INVALID,		TNSZ("sqrtsd",XMM,8),	INVALID,		INVALID,
+/*  [54]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [58]  */	TNSZ("addsd",XMM,8),	TNSZ("mulsd",XMM,8),	TNSZ("cvtsd2ss",XMM,8),	INVALID,
+/*  [5C]  */	TNSZ("subsd",XMM,8),	TNSZ("minsd",XMM,8),	TNSZ("divsd",XMM,8),	TNSZ("maxsd",XMM,8),
+
+/*  [60]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [64]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [68]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [6C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [70]  */	TNSZ("pshuflw",XMMP,16),INVALID,		INVALID,		INVALID,
+/*  [74]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [78]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [7C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [80]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [84]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [88]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [90]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [94]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [98]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [9C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [A0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [AC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [B0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [BC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [C0]  */	INVALID,		INVALID,		TNSZ("cmpsd",XMMP,8),	INVALID,
+/*  [C4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [C8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [CC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [D0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [D4]  */	INVALID,		INVALID,		TNS("movdq2q",XMMXM),	INVALID,
+/*  [D8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [DC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [E0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [E4]  */	INVALID,		INVALID,		TNSZ("cvtpd2dq",XMM,16),INVALID,
+/*  [E8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [EC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [F0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [F4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [F8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [FC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+};
+
+/*
+ *	Decode table for SIMD instructions with the repz (0xf3) prefix.
+ */
+const instable_t dis_opSIMDrepz[256] = {
+/*  [00]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [04]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [08]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [10]  */	TNSZ("movss",XMM,4),	TNSZ("movss",XMMS,4),	INVALID,		INVALID,
+/*  [14]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [18]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [1C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [20]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [24]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [28]  */	INVALID,		INVALID,		TNSZ("cvtsi2ss",XMM3MX,4),INVALID,
+/*  [2C]  */	TNSZ("cvttss2si",XMMXM3,4),TNSZ("cvtss2si",XMMXM3,4),INVALID,		INVALID,
+
+/*  [30]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [34]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [38]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [3C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [40]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [44]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [48]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [4C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [50]  */	INVALID,		TNSZ("sqrtss",XMM,4),	TNSZ("rsqrtss",XMM,4),	TNSZ("rcpss",XMM,4),
+/*  [54]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [58]  */	TNSZ("addss",XMM,4),	TNSZ("mulss",XMM,4),	TNSZ("cvtss2sd",XMM,4),	TNSZ("cvttps2dq",XMM,16),
+/*  [5C]  */	TNSZ("subss",XMM,4),	TNSZ("minss",XMM,4),	TNSZ("divss",XMM,4),	TNSZ("maxss",XMM,4),
+
+/*  [60]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [64]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [68]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [6C]  */	INVALID,		INVALID,		INVALID,		TNSZ("movdqu",XMM,16),
+
+/*  [70]  */	TNSZ("pshufhw",XMMP,16),INVALID,		INVALID,		INVALID,
+/*  [74]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [78]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [7C]  */	INVALID,		INVALID,		TNSZ("movq",XMM,8),	TNSZ("movdqu",XMMS,16),
+
+/*  [80]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [84]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [88]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [90]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [94]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [98]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [9C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [A0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [A8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [AC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [B0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [B8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [BC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [C0]  */	INVALID,		INVALID,		TNSZ("cmpss",XMMP,4),	INVALID,
+/*  [C4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [C8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [CC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [D0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [D4]  */	INVALID,		INVALID,		TNS("movq2dq",XMMMX),	INVALID,
+/*  [D8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [DC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [E0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [E4]  */	INVALID,		INVALID,		TNSZ("cvtdq2pd",XMM,8),	INVALID,
+/*  [E8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [EC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+
+/*  [F0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [F4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [F8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [FC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+};
+
+/*
+ *	Decode table for 0x0F opcodes
+ */
+
+const instable_t dis_op0F[16][16] = {
+{
+/*  [00]  */	IND(dis_op0F00),	IND(dis_op0F01),	TNS("lar",MR),		TNS("lsl",MR),
+/*  [04]  */	INVALID,		TNS("syscall",NORM),	TNS("clts",NORM),	TNS("sysret",NORM),
+/*  [08]  */	TNS("invd",NORM),	TNS("wbinvd",NORM),	INVALID,		TNS("ud2",NORM),
+/*  [0C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [10]  */	TNSZ("movups",XMMO,16),	TNSZ("movups",XMMOS,16),TNSZ("movlps",XMMO,8),	TNSZ("movlps",XMMOS,8),
+/*  [14]  */	TNSZ("unpcklps",XMMO,16),TNSZ("unpckhps",XMMO,16),TNSZ("movhps",XMMOM,8),TNSZ("movhps",XMMOMS,8),
+/*  [18]  */	IND(dis_op0F18),	INVALID,		INVALID,		INVALID,
+/*  [1C]  */	INVALID,		INVALID,		INVALID,		TS("nop",Mw),
+}, {
+/*  [20]  */	TSy("mov",SREG),	TSy("mov",SREG),	TSy("mov",SREG),	TSy("mov",SREG),
+/*  [24]  */	TSx("mov",SREG),	INVALID,		TSx("mov",SREG),	INVALID,
+/*  [28]  */	TNSZ("movaps",XMMO,16),	TNSZ("movaps",XMMOS,16),TNSZ("cvtpi2ps",XMMOMX,8),TNSZ("movntps",XMMOS,16),
+/*  [2C]  */	TNSZ("cvttps2pi",XMMOXMM,8),TNSZ("cvtps2pi",XMMOXMM,8),TNSZ("ucomiss",XMMO,4),TNSZ("comiss",XMMO,4),
+}, {
+/*  [30]  */	TNS("wrmsr",NORM),	TNS("rdtsc",NORM),	TNS("rdmsr",NORM),	TNS("rdpmc",NORM),
+/*  [34]  */	TNSx("sysenter",NORM),	TNSx("sysexit",NORM),	INVALID,		INVALID,
+/*  [38]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [3C]  */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [40]  */	TS("cmovx.o",MR),	TS("cmovx.no",MR),	TS("cmovx.b",MR),	TS("cmovx.ae",MR),
+/*  [44]  */	TS("cmovx.e",MR),	TS("cmovx.ne",MR),	TS("cmovx.be",MR),	TS("cmovx.a",MR),
+/*  [48]  */	TS("cmovx.s",MR),	TS("cmovx.ns",MR),	TS("cmovx.pe",MR),	TS("cmovx.po",MR),
+/*  [4C]  */	TS("cmovx.l",MR),	TS("cmovx.ge",MR),	TS("cmovx.le",MR),	TS("cmovx.g",MR),
+}, {
+/*  [50]  */	TNS("movmskps",XMMOX3),	TNSZ("sqrtps",XMMO,16),	TNSZ("rsqrtps",XMMO,16),TNSZ("rcpps",XMMO,16),
+/*  [54]  */	TNSZ("andps",XMMO,16),	TNSZ("andnps",XMMO,16),	TNSZ("orps",XMMO,16),	TNSZ("xorps",XMMO,16),
+/*  [58]  */	TNSZ("addps",XMMO,16),	TNSZ("mulps",XMMO,16),	TNSZ("cvtps2pd",XMMO,8),TNSZ("cvtdq2ps",XMMO,16),
+/*  [5C]  */	TNSZ("subps",XMMO,16),	TNSZ("minps",XMMO,16),	TNSZ("divps",XMMO,16),	TNSZ("maxps",XMMO,16),
+}, {
+/*  [60]  */	TNSZ("punpcklbw",MMO,4),TNSZ("punpcklwd",MMO,4),TNSZ("punpckldq",MMO,4),TNSZ("packsswb",MMO,8),
+/*  [64]  */	TNSZ("pcmpgtb",MMO,8),	TNSZ("pcmpgtw",MMO,8),	TNSZ("pcmpgtd",MMO,8),	TNSZ("packuswb",MMO,8),
+/*  [68]  */	TNSZ("punpckhbw",MMO,8),TNSZ("punpckhwd",MMO,8),TNSZ("punpckhdq",MMO,8),TNSZ("packssdw",MMO,8),
+/*  [6C]  */	TNSZ("INVALID",MMO,0),	TNSZ("INVALID",MMO,0),	TNSZ("movd",MMO,4),	TNSZ("movq",MMO,8),
+}, {
+/*  [70]  */	TNSZ("pshufw",MMOPM,8),	TNS("psrXXX",MR),	TNS("psrXXX",MR),	TNS("psrXXX",MR),
+/*  [74]  */	TNSZ("pcmpeqb",MMO,8),	TNSZ("pcmpeqw",MMO,8),	TNSZ("pcmpeqd",MMO,8),	TNS("emms",NORM),
+/*  [78]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [7C]  */	INVALID,		INVALID,		TNSZ("movd",MMOS,4),	TNSZ("movq",MMOS,8),
+}, {
+/*  [80]  */	TNS("jo",D),		TNS("jno",D),		TNS("jb",D),		TNS("jae",D),
+/*  [84]  */	TNS("je",D),		TNS("jne",D),		TNS("jbe",D),		TNS("ja",D),
+/*  [88]  */	TNS("js",D),		TNS("jns",D),		TNS("jp",D),		TNS("jnp",D),
+/*  [8C]  */	TNS("jl",D),		TNS("jge",D),		TNS("jle",D),		TNS("jg",D),
+}, {
+/*  [90]  */	TNS("seto",Mb),		TNS("setno",Mb),	TNS("setb",Mb),		TNS("setae",Mb),
+/*  [94]  */	TNS("sete",Mb),		TNS("setne",Mb),	TNS("setbe",Mb),	TNS("seta",Mb),
+/*  [98]  */	TNS("sets",Mb),		TNS("setns",Mb),	TNS("setp",Mb),		TNS("setnp",Mb),
+/*  [9C]  */	TNS("setl",Mb),		TNS("setge",Mb),	TNS("setle",Mb),	TNS("setg",Mb),
+}, {
+/*  [A0]  */	TSp("push",LSEG),	TSp("pop",LSEG),	TNS("cpuid",NORM),	TS("bt",RMw),
+/*  [A4]  */	TS("shld",DSHIFT),	TS("shld",DSHIFTcl),	INVALID,		INVALID,
+/*  [A8]  */	TSp("push",LSEG),	TSp("pop",LSEG),	TNS("rsm",NORM),	TS("bts",RMw),
+/*  [AC]  */	TS("shrd",DSHIFT),	TS("shrd",DSHIFTcl),	IND(dis_op0FAE),	TS("imul",MRw),
+}, {
+/*  [B0]  */	TNS("cmpxchgb",RMw),	TS("cmpxchg",RMw),	TS("lss",MR),		TS("btr",RMw),
+/*  [B4]  */	TS("lfs",MR),		TS("lgs",MR),		TS("movzb",MOVZ),	TNS("movzwl",MOVZ),
+/*  [B8]  */	INVALID,		INVALID,		IND(dis_op0FBA),	TS("btc",RMw),
+/*  [BC]  */	TS("bsf",MRw),		TS("bsr",MRw),		TS("movsb",MOVZ),	TNS("movswl",MOVZ),
+}, {
+/*  [C0]  */	TNS("xaddb",XADDB),	TS("xadd",RMw),		TNSZ("cmpps",XMMOPM,16),TNS("movnti",RM),
+/*  [C4]  */	TNSZ("pinsrw",MMOPRM,2),TNS("pextrw",MMO3P), 	TNSZ("shufps",XMMOPM,16),IND(dis_op0FC7),
+/*  [C8]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [CC]  */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [D0]  */	INVALID,		TNSZ("psrlw",MMO,8),	TNSZ("psrld",MMO,8),	TNSZ("psrlq",MMO,8),
+/*  [D4]  */	TNSZ("paddq",MMO,8),	TNSZ("pmullw",MMO,8),	TNSZ("INVALID",MMO,0),	TNS("pmovmskb",MMOM3),
+/*  [D8]  */	TNSZ("psubusb",MMO,8),	TNSZ("psubusw",MMO,8),	TNSZ("pminub",MMO,8),	TNSZ("pand",MMO,8),
+/*  [DC]  */	TNSZ("paddusb",MMO,8),	TNSZ("paddusw",MMO,8),	TNSZ("pmaxub",MMO,8),	TNSZ("pandn",MMO,8),
+}, {
+/*  [E0]  */	TNSZ("pavgb",MMO,8),	TNSZ("psraw",MMO,8),	TNSZ("psrad",MMO,8),	TNSZ("pavgw",MMO,8),
+/*  [E4]  */	TNSZ("pmulhuw",MMO,8),	TNSZ("pmulhw",MMO,8),	TNS("INVALID",XMMO),	TNSZ("movntq",MMOMS,8),
+/*  [E8]  */	TNSZ("psubsb",MMO,8),	TNSZ("psubsw",MMO,8),	TNSZ("pminsw",MMO,8),	TNSZ("por",MMO,8),
+/*  [EC]  */	TNSZ("paddsb",MMO,8),	TNSZ("paddsw",MMO,8),	TNSZ("pmaxsw",MMO,8),	TNSZ("pxor",MMO,8),
+}, {
+/*  [F0]  */	INVALID,		TNSZ("psllw",MMO,8),	TNSZ("pslld",MMO,8),	TNSZ("psllq",MMO,8),
+/*  [F4]  */	TNSZ("pmuludq",MMO,8),	TNSZ("pmaddwd",MMO,8),	TNSZ("psadbw",MMO,8),	TNSZ("maskmovq",MMOIMPL,8),
+/*  [F8]  */	TNSZ("psubb",MMO,8),	TNSZ("psubw",MMO,8),	TNSZ("psubd",MMO,8),	TNSZ("psubq",MMO,8),
+/*  [FC]  */	TNSZ("paddb",MMO,8),	TNSZ("paddw",MMO,8),	TNSZ("paddd",MMO,8),	INVALID,
+} };
+
+
+/*
+ *	Decode table for 0x80 opcodes
+ */
+
+const instable_t dis_op80[8] = {
+
+/*  [0]  */	TNS("addb",IMlw),	TNS("orb",IMw),		TNS("adcb",IMlw),	TNS("sbbb",IMlw),
+/*  [4]  */	TNS("andb",IMw),	TNS("subb",IMlw),	TNS("xorb",IMw),	TNS("cmpb",IMlw),
+};
+
+
+/*
+ *	Decode table for 0x81 opcodes.
+ */
+
+const instable_t dis_op81[8] = {
+
+/*  [0]  */	TS("add",IMlw),		TS("or",IMw),		TS("adc",IMlw),		TS("sbb",IMlw),
+/*  [4]  */	TS("and",IMw),		TS("sub",IMlw),		TS("xor",IMw),		TS("cmp",IMlw),
+};
+
+
+/*
+ *	Decode table for 0x82 opcodes.
+ */
+
+const instable_t dis_op82[8] = {
+
+/*  [0]  */	TNSx("addb",IMlw),	TNSx("orb",IMlw),	TNSx("adcb",IMlw),	TNSx("sbbb",IMlw),
+/*  [4]  */	TNSx("andb",IMlw),	TNSx("subb",IMlw),	TNSx("xorb",IMlw),	TNSx("cmpb",IMlw),
+};
+/*
+ *	Decode table for 0x83 opcodes.
+ */
+
+const instable_t dis_op83[8] = {
+
+/*  [0]  */	TS("add",IMlw),		TS("or",IMlw),		TS("adc",IMlw),		TS("sbb",IMlw),
+/*  [4]  */	TS("and",IMlw),		TS("sub",IMlw),		TS("xor",IMlw),		TS("cmp",IMlw),
+};
+
+/*
+ *	Decode table for 0xC0 opcodes.
+ */
+
+const instable_t dis_opC0[8] = {
+
+/*  [0]  */	TNS("rolb",MvI),	TNS("rorb",MvI),	TNS("rclb",MvI),	TNS("rcrb",MvI),
+/*  [4]  */	TNS("shlb",MvI),	TNS("shrb",MvI),	INVALID,		TNS("sarb",MvI),
+};
+
+/*
+ *	Decode table for 0xD0 opcodes.
+ */
+
+const instable_t dis_opD0[8] = {
+
+/*  [0]  */	TNS("rolb",Mv),		TNS("rorb",Mv),		TNS("rclb",Mv),		TNS("rcrb",Mv),
+/*  [4]  */	TNS("shlb",Mv),		TNS("shrb",Mv),		TNS("salb",Mv),		TNS("sarb",Mv),
+};
+
+/*
+ *	Decode table for 0xC1 opcodes.
+ *	186 instruction set
+ */
+
+const instable_t dis_opC1[8] = {
+
+/*  [0]  */	TS("rol",MvI),		TS("ror",MvI),		TS("rcl",MvI),		TS("rcr",MvI),
+/*  [4]  */	TS("shl",MvI),		TS("shr",MvI),		TS("sal",MvI),		TS("sar",MvI),
+};
+
+/*
+ *	Decode table for 0xD1 opcodes.
+ */
+
+const instable_t dis_opD1[8] = {
+
+/*  [0]  */	TS("rol",Mv),		TS("ror",Mv),		TS("rcl",Mv),		TS("rcr",Mv),
+/*  [4]  */	TS("shl",Mv),		TS("shr",Mv),		TS("sal",Mv),		TS("sar",Mv),
+};
+
+
+/*
+ *	Decode table for 0xD2 opcodes.
+ */
+
+const instable_t dis_opD2[8] = {
+
+/*  [0]  */	TNS("rolb",Mv),		TNS("rorb",Mv),		TNS("rclb",Mv),		TNS("rcrb",Mv),
+/*  [4]  */	TNS("shlb",Mv),		TNS("shrb",Mv),		TNS("salb",Mv),		TNS("sarb",Mv),
+};
+/*
+ *	Decode table for 0xD3 opcodes.
+ */
+
+const instable_t dis_opD3[8] = {
+
+/*  [0]  */	TS("rol",Mv),		TS("ror",Mv),		TS("rcl",Mv),		TS("rcr",Mv),
+/*  [4]  */	TS("shl",Mv),		TS("shr",Mv),		TS("salb",Mv),		TS("sar",Mv),
+};
+
+
+/*
+ *	Decode table for 0xF6 opcodes.
+ */
+
+const instable_t dis_opF6[8] = {
+
+/*  [0]  */	TNS("testb",IMw),	TNS("testb",IMw),	TNS("notb",Mw),		TNS("negb",Mw),
+/*  [4]  */	TNS("mulb",MA),		TNS("imulb",MA),	TNS("divb",MA),		TNS("idivb",MA),
+};
+
+
+/*
+ *	Decode table for 0xF7 opcodes.
+ */
+
+const instable_t dis_opF7[8] = {
+
+/*  [0]  */	TS("test",IMw),		TS("test",IMw),		TS("not",Mw),		TS("neg",Mw),
+/*  [4]  */	TS("mul",MA),		TS("imul",MA),		TS("div",MA),		TS("idiv",MA),
+};
+
+
+/*
+ *	Decode table for 0xFE opcodes.
+ */
+
+const instable_t dis_opFE[8] = {
+
+/*  [0]  */	TNS("incb",Mw),		TNS("decb",Mw),		INVALID,		INVALID,
+/*  [4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+};
+/*
+ *	Decode table for 0xFF opcodes.
+ */
+
+const instable_t dis_opFF[8] = {
+
+/*  [0]  */	TS("inc",Mw),		TS("dec",Mw),		TNSyp("call",INM),	TNS("lcall",INM),
+/*  [4]  */	TNSy("jmp",INM),	TNS("ljmp",INM),	TSp("push",M),		INVALID,
+};
+
+/* for 287 instructions, which are a mess to decode */
+
+const instable_t dis_opFP1n2[8][8] = {
+{
+/* bit pattern:	1101 1xxx MODxx xR/M */
+/*  [0,0] */	TNS("fadds",M),		TNS("fmuls",M),		TNS("fcoms",M),		TNS("fcomps",M),
+/*  [0,4] */	TNS("fsubs",M),		TNS("fsubrs",M),	TNS("fdivs",M),		TNS("fdivrs",M),
+}, {
+/*  [1,0]  */	TNS("flds",M),		INVALID,		TNS("fsts",M),		TNS("fstps",M),
+/*  [1,4]  */	TNSZ("fldenv",M,28),	TNSZ("fldcw",M,2),	TNSZ("fnstenv",M,28),	TNSZ("fnstcw",M,2),
+}, {
+/*  [2,0]  */	TNS("fiaddl",M),	TNS("fimull",M),	TNS("ficoml",M),	TNS("ficompl",M),
+/*  [2,4]  */	TNS("fisubl",M),	TNS("fisubrl",M),	TNS("fidivl",M),	TNS("fidivrl",M),
+}, {
+/*  [3,0]  */	TNS("fildl",M),		INVALID,		TNS("fistl",M),		TNS("fistpl",M),
+/*  [3,4]  */	INVALID,		TNSZ("fldt",M,10),	INVALID,		TNSZ("fstpt",M,10),
+}, {
+/*  [4,0]  */	TNSZ("faddl",M,8),	TNSZ("fmull",M,8),	TNSZ("fcoml",M,8),	TNSZ("fcompl",M,8),
+/*  [4,1]  */	TNSZ("fsubl",M,8),	TNSZ("fsubrl",M,8),	TNSZ("fdivl",M,8),	TNSZ("fdivrl",M,8),
+}, {
+/*  [5,0]  */	TNSZ("fldl",M,8),	INVALID,		TNSZ("fstl",M,8),	TNSZ("fstpl",M,8),
+/*  [5,4]  */	TNSZ("frstor",M,108),	INVALID,		TNSZ("fnsave",M,108),	TNSZ("fnstsw",M,2),
+}, {
+/*  [6,0]  */	TNSZ("fiadd",M,2),	TNSZ("fimul",M,2),	TNSZ("ficom",M,2),	TNSZ("ficomp",M,2),
+/*  [6,4]  */	TNSZ("fisub",M,2),	TNSZ("fisubr",M,2),	TNSZ("fidiv",M,2),	TNSZ("fidivr",M,2),
+}, {
+/*  [7,0]  */	TNSZ("fild",M,2),	INVALID,		TNSZ("fist",M,2),	TNSZ("fistp",M,2),
+/*  [7,4]  */	TNSZ("fbld",M,10),	TNSZ("fildll",M,8),	TNSZ("fbstp",M,10),	TNSZ("fistpll",M,8),
+} };
+
+const instable_t dis_opFP3[8][8] = {
+{
+/* bit  pattern:	1101 1xxx 11xx xREG */
+/*  [0,0]  */	TNS("fadd",FF),		TNS("fmul",FF),		TNS("fcom",F),		TNS("fcomp",F),
+/*  [0,4]  */	TNS("fsub",FF),		TNS("fsubr",FF),	TNS("fdiv",FF),		TNS("fdivr",FF),
+}, {
+/*  [1,0]  */	TNS("fld",F),		TNS("fxch",F),		TNS("fnop",NORM),	TNS("fstp",F),
+/*  [1,4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [2,0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [2,4]  */	INVALID,		TNS("fucompp",NORM),	INVALID,		INVALID,
+}, {
+/*  [3,0]  */	INVALID,		INVALID,		INVALID,		INVALID,
+/*  [3,4]  */	INVALID,		INVALID,		INVALID,		INVALID,
+}, {
+/*  [4,0]  */	TNS("fadd",FF),		TNS("fmul",FF),		TNS("fcom",F),		TNS("fcomp",F),
+/*  [4,4]  */	TNS("fsub",FF),		TNS("fsubr",FF),	TNS("fdiv",FF),		TNS("fdivr",FF),
+}, {
+/*  [5,0]  */	TNS("ffree",F),		TNS("fxch",F),		TNS("fst",F),		TNS("fstp",F),
+/*  [5,4]  */	TNS("fucom",F),		TNS("fucomp",F),	INVALID,		INVALID,
+}, {
+/*  [6,0]  */	TNS("faddp",FF),	TNS("fmulp",FF),	TNS("fcomp",F),		TNS("fcompp",NORM),
+/*  [6,4]  */	TNS("fsubp",FF),	TNS("fsubrp",FF),	TNS("fdivp",FF),	TNS("fdivrp",FF),
+}, {
+/*  [7,0]  */	TNS("ffreep",F),		TNS("fxch",F),		TNS("fstp",F),		TNS("fstp",F),
+/*  [7,4]  */	TNS("fnstsw",M),	TNS("fucomip",FFC),	TNS("fcomip",FFC),	INVALID,
+} };
+
+const instable_t dis_opFP4[4][8] = {
+{
+/* bit pattern:	1101 1001 111x xxxx */
+/*  [0,0]  */	TNS("fchs",NORM),	TNS("fabs",NORM),	INVALID,		INVALID,
+/*  [0,4]  */	TNS("ftst",NORM),	TNS("fxam",NORM),	TNS("ftstp",NORM),	INVALID,
+}, {
+/*  [1,0]  */	TNS("fld1",NORM),	TNS("fldl2t",NORM),	TNS("fldl2e",NORM),	TNS("fldpi",NORM),
+/*  [1,4]  */	TNS("fldlg2",NORM),	TNS("fldln2",NORM),	TNS("fldz",NORM),	INVALID,
+}, {
+/*  [2,0]  */	TNS("f2xm1",NORM),	TNS("fyl2x",NORM),	TNS("fptan",NORM),	TNS("fpatan",NORM),
+/*  [2,4]  */	TNS("fxtract",NORM),	TNS("fprem1",NORM),	TNS("fdecstp",NORM),	TNS("fincstp",NORM),
+}, {
+/*  [3,0]  */	TNS("fprem",NORM),	TNS("fyl2xp1",NORM),	TNS("fsqrt",NORM),	TNS("fsincos",NORM),
+/*  [3,4]  */	TNS("frndint",NORM),	TNS("fscale",NORM),	TNS("fsin",NORM),	TNS("fcos",NORM),
+} };
+
+const instable_t dis_opFP5[8] = {
+/* bit pattern:	1101 1011 111x xxxx */
+/*  [0]  */	TNS("feni",NORM),	TNS("fdisi",NORM),	TNS("fnclex",NORM),	TNS("fninit",NORM),
+/*  [4]  */	TNS("fsetpm",NORM),	TNS("frstpm",NORM),	INVALID,		INVALID,
+};
+
+const instable_t dis_opFP6[8] = {
+/* bit pattern:	1101 1011 11yy yxxx */
+/*  [00]  */	TNS("fcmov.nb",FF),	TNS("fcmov.ne",FF),	TNS("fcmov.nbe",FF),	TNS("fcmov.nu",FF),
+/*  [04]  */	INVALID,		TNS("fucomi",F),	TNS("fcomi",F),		INVALID,
+};
+
+const instable_t dis_opFP7[8] = {
+/* bit pattern:	1101 1010 11yy yxxx */
+/*  [00]  */	TNS("fcmov.b",FF),	TNS("fcmov.e",FF),	TNS("fcmov.be",FF),	TNS("fcmov.u",FF),
+/*  [04]  */	INVALID,		INVALID,		INVALID,		INVALID,
+};
+
+/*
+ *	Main decode table for the op codes.  The first two nibbles
+ *	will be used as an index into the table.  If there is a
+ *	a need to further decode an instruction, the array to be
+ *	referenced is indicated with the other two entries being
+ *	empty.
+ */
+
+const instable_t dis_distable[16][16] = {
+{
+/* [0,0] */	TNS("addb",RMw),	TS("add",RMw),		TNS("addb",MRw),	TS("add",MRw),
+/* [0,4] */	TNS("addb",IA),		TS("add",IA),		TSx("push",SEG),	TSx("pop",SEG),
+/* [0,8] */	TNS("orb",RMw),		TS("or",RMw),		TNS("orb",MRw),		TS("or",MRw),
+/* [0,C] */	TNS("orb",IA),		TS("or",IA),		TSx("push",SEG),	IND(dis_op0F),
+}, {
+/* [1,0] */	TNS("adcb",RMw),	TS("adc",RMw),		TNS("adcb",MRw),	TS("adc",MRw),
+/* [1,4] */	TNS("adcb",IA),		TS("adc",IA),		TSx("push",SEG),	TSx("pop",SEG),
+/* [1,8] */	TNS("sbbb",RMw),	TS("sbb",RMw),		TNS("sbbb",MRw),	TS("sbb",MRw),
+/* [1,C] */	TNS("sbbb",IA),		TS("sbb",IA),		TSx("push",SEG),	TSx("pop",SEG),
+}, {
+/* [2,0] */	TNS("andb",RMw),	TS("and",RMw),		TNS("andb",MRw),	TS("and",MRw),
+/* [2,4] */	TNS("andb",IA),		TS("and",IA),		TNSx("%es:",OVERRIDE),	TNSx("daa",NORM),
+/* [2,8] */	TNS("subb",RMw),	TS("sub",RMw),		TNS("subb",MRw),	TS("sub",MRw),
+/* [2,C] */	TNS("subb",IA),		TS("sub",IA),		TNSx("%cs:",OVERRIDE),	TNSx("das",NORM),
+}, {
+/* [3,0] */	TNS("xorb",RMw),	TS("xor",RMw),		TNS("xorb",MRw),	TS("xor",MRw),
+/* [3,4] */	TNS("xorb",IA),		TS("xor",IA),		TNSx("%ss:",OVERRIDE),	TNSx("aaa",NORM),
+/* [3,8] */	TNS("cmpb",RMw),	TS("cmp",RMw),		TNS("cmpb",MRw),	TS("cmp",MRw),
+/* [3,C] */	TNS("cmpb",IA),		TS("cmp",IA),		TNSx("%ds:",OVERRIDE),	TNSx("aas",NORM),
+}, {
+/* [4,0] */	TSx("inc",R),		TSx("inc",R),		TSx("inc",R),		TSx("inc",R),
+/* [4,4] */	TSx("inc",R),		TSx("inc",R),		TSx("inc",R),		TSx("inc",R),
+/* [4,8] */	TSx("dec",R),		TSx("dec",R),		TSx("dec",R),		TSx("dec",R),
+/* [4,C] */	TSx("dec",R),		TSx("dec",R),		TSx("dec",R),		TSx("dec",R),
+}, {
+/* [5,0] */	TSp("push",R),		TSp("push",R),		TSp("push",R),		TSp("push",R),
+/* [5,4] */	TSp("push",R),		TSp("push",R),		TSp("push",R),		TSp("push",R),
+/* [5,8] */	TSp("pop",R),		TSp("pop",R),		TSp("pop",R),		TSp("pop",R),
+/* [5,C] */	TSp("pop",R),		TSp("pop",R),		TSp("pop",R),		TSp("pop",R),
+}, {
+/* [6,0] */	TSZx("pusha",IMPLMEM,28),TSZx("popa",IMPLMEM,28), TSx("bound",MR),	TNS("arpl",RMw),
+/* [6,4] */	TNS("%fs:",OVERRIDE),	TNS("%gs:",OVERRIDE),	TNS("data16",DM),	TNS("addr16",AM),
+/* [6,8] */	TSp("push",I),		TS("imul",IMUL),	TSp("push",Ib),	TS("imul",IMUL),
+/* [6,C] */	TNSZ("insb",IMPLMEM,1),	TSZ("ins",IMPLMEM,4),	TNSZ("outsb",IMPLMEM,1),TSZ("outs",IMPLMEM,4),
+}, {
+/* [7,0] */	TNSy("jo",BD),		TNSy("jno",BD),		TNSy("jb",BD),		TNSy("jae",BD),
+/* [7,4] */	TNSy("je",BD),		TNSy("jne",BD),		TNSy("jbe",BD),		TNSy("ja",BD),
+/* [7,8] */	TNSy("js",BD),		TNSy("jns",BD),		TNSy("jp",BD),		TNSy("jnp",BD),
+/* [7,C] */	TNSy("jl",BD),		TNSy("jge",BD),		TNSy("jle",BD),		TNSy("jg",BD),
+}, {
+/* [8,0] */	IND(dis_op80),		IND(dis_op81),		INDx(dis_op82),		IND(dis_op83),
+/* [8,4] */	TNS("testb",RMw),	TS("test",RMw),		TNS("xchgb",RMw),	TS("xchg",RMw),
+/* [8,8] */	TNS("movb",RMw),	TS("mov",RMw),		TNS("movb",MRw),	TS("mov",MRw),
+/* [8,C] */	TNS("movw",SM),		TS("lea",MR),		TNS("movw",MS),		TSp("pop",M),
+}, {
+/* [9,0] */	TNS("nop",NORM),	TS("xchg",RA),		TS("xchg",RA),		TS("xchg",RA),
+/* [9,4] */	TS("xchg",RA),		TS("xchg",RA),		TS("xchg",RA),		TS("xchg",RA),
+/* [9,8] */	TNS("cXtX",CBW),	TNS("cXtX",CWD),	TNSx("lcall",SO),	TNS("fwait",NORM),
+/* [9,C] */	TSZy("pushf",IMPLMEM,4),TSZy("popf",IMPLMEM,4),	TNSx("sahf",NORM),	TNSx("lahf",NORM),
+}, {
+/* [A,0] */	TNS("movb",OA),		TS("mov",OA),		TNS("movb",AO),		TS("mov",AO),
+/* [A,4] */	TNSZ("movsb",SD,1),	TS("movs",SD),		TNSZ("cmpsb",SD,1),	TS("cmps",SD),
+/* [A,8] */	TNS("testb",IA),	TS("test",IA),		TNS("stosb",AD),	TS("stos",AD),
+/* [A,C] */	TNS("lodsb",SA),	TS("lods",SA),		TNS("scasb",AD),	TS("scas",AD),
+}, {
+/* [B,0] */	TNS("movb",IR),		TNS("movb",IR),		TNS("movb",IR),		TNS("movb",IR),
+/* [B,4] */	TNS("movb",IR),		TNS("movb",IR),		TNS("movb",IR),		TNS("movb",IR),
+/* [B,8] */	TS("mov",IR),		TS("mov",IR),		TS("mov",IR),		TS("mov",IR),
+/* [B,C] */	TS("mov",IR),		TS("mov",IR),		TS("mov",IR),		TS("mov",IR),
+}, {
+/* [C,0] */	IND(dis_opC0),		IND(dis_opC1), 		TNSyp("ret",RET),	TNSyp("ret",NORM),
+/* [C,4] */	TNSx("les",MR),		TNSx("lds",MR),		TNS("movb",IMw),	TS("mov",IMw),
+/* [C,8] */	TNSyp("enter",ENTER),	TNSyp("leave",NORM),	TNS("lret",RET),	TNS("lret",NORM),
+/* [C,C] */	TNS("int",INT3),	TNS("int",INTx),	TNSx("into",NORM),	TNS("iret",NORM),
+}, {
+/* [D,0] */	IND(dis_opD0),		IND(dis_opD1),		IND(dis_opD2),		IND(dis_opD3),
+/* [D,4] */	TNSx("aam",U),		TNSx("aad",U),		TNSx("falc",NORM),	TNSZ("xlat",IMPLMEM,1),
+
+/* 287 instructions.  Note that although the indirect field		*/
+/* indicates opFP1n2 for further decoding, this is not necessarily	*/
+/* the case since the opFP arrays are not partitioned according to key1	*/
+/* and key2.  opFP1n2 is given only to indicate that we haven't		*/
+/* finished decoding the instruction.					*/
+/* [D,8] */	IND(dis_opFP1n2),	IND(dis_opFP1n2),	IND(dis_opFP1n2),	IND(dis_opFP1n2),
+/* [D,C] */	IND(dis_opFP1n2),	IND(dis_opFP1n2),	IND(dis_opFP1n2),	IND(dis_opFP1n2),
+}, {
+/* [E,0] */	TNSy("loopnz",BD),	TNSy("loopz",BD),	TNSy("loop",BD),	TNSy("jcxz",BD),
+/* [E,4] */	TNS("inb",P),		TS("in",P),		TNS("outb",P),		TS("out",P),
+/* [E,8] */	TNSyp("call",D),	TNSy("jmp",D),		TNSx("ljmp",SO),		TNSy("jmp",BD),
+/* [E,C] */	TNS("inb",V),		TS("in",V),		TNS("outb",V),		TS("out",V),
+}, {
+/* [F,0] */	TNS("lock",LOCK),	TNS("icebp", NORM),	TNS("repnz",PREFIX),	TNS("repz",PREFIX),
+/* [F,4] */	TNS("hlt",NORM),	TNS("cmc",NORM),	IND(dis_opF6),		IND(dis_opF7),
+/* [F,8] */	TNS("clc",NORM),	TNS("stc",NORM),	TNS("cli",NORM),	TNS("sti",NORM),
+/* [F,C] */	TNS("cld",NORM),	TNS("std",NORM),	IND(dis_opFE),		IND(dis_opFF),
+} };
+
+/* END CSTYLED */
+
+/*
+ * common functions to decode and disassemble an x86 or amd64 instruction
+ */
+
+/*
+ * These are the individual fields of a REX prefix. Note that a REX
+ * prefix with none of these set is still needed to:
+ *	- use the MOVSXD (sign extend 32 to 64 bits) instruction
+ *	- access the %sil, %dil, %bpl, %spl registers
+ */
+#define	REX_W 0x08	/* 64 bit operand size when set */
+#define	REX_R 0x04	/* high order bit extension of ModRM reg field */
+#define	REX_X 0x02	/* high order bit extension of SIB index field */
+#define	REX_B 0x01	/* extends ModRM r_m, SIB base, or opcode reg */
+
+static uint_t opnd_size;	/* SIZE16, SIZE32 or SIZE64 */
+static uint_t addr_size;	/* SIZE16, SIZE32 or SIZE64 */
+
+/*
+ * Even in 64 bit mode, usually only 4 byte immediate operands are supported.
+ */
+static int isize[] = {1, 2, 4, 4};
+static int isize64[] = {1, 2, 4, 8};
+
+/*
+ * Just a bunch of useful macros.
+ */
+#define	WBIT(x)	(x & 0x1)		/* to get w bit	*/
+#define	REGNO(x) (x & 0x7)		/* to get 3 bit register */
+#define	VBIT(x)	((x)>>1 & 0x1)		/* to get 'v' bit */
+#define	OPSIZE(osize, wbit) ((wbit) ? isize[osize] : 1)
+#define	OPSIZE64(osize, wbit) ((wbit) ? isize64[osize] : 1)
+
+#define	REG_ONLY 3	/* mode to indicate a register operand (not memory) */
+
+#define	BYTE_OPND	0	/* w-bit value indicating byte register */
+#define	LONG_OPND	1	/* w-bit value indicating opnd_size register */
+#define	MM_OPND		2	/* "value" used to indicate a mmx reg */
+#define	XMM_OPND	3	/* "value" used to indicate a xmm reg */
+#define	SEG_OPND	4	/* "value" used to indicate a segment reg */
+#define	CONTROL_OPND	5	/* "value" used to indicate a control reg */
+#define	DEBUG_OPND	6	/* "value" used to indicate a debug reg */
+#define	TEST_OPND	7	/* "value" used to indicate a test reg */
+#define	WORD_OPND	8	/* w-bit value indicating word size reg */
+
+/*
+ * Get the next byte and separate the op code into the high and low nibbles.
+ */
+static int
+dtrace_get_opcode(dis86_t *x, uint_t *high, uint_t *low)
+{
+	int byte;
+
+	/*
+	 * x86 instructions have a maximum length of 15 bytes.  Bail out if
+	 * we try to read more.
+	 */
+	if (x->d86_len >= 15)
+		return (x->d86_error = 1);
+
+	if (x->d86_error)
+		return (1);
+	byte = x->d86_get_byte(x->d86_data);
+	if (byte < 0)
+		return (x->d86_error = 1);
+	x->d86_bytes[x->d86_len++] = byte;
+	*low = byte & 0xf;		/* ----xxxx low 4 bits */
+	*high = byte >> 4 & 0xf;	/* xxxx---- bits 7 to 4 */
+	return (0);
+}
+
+/*
+ * Get and decode an SIB (scaled index base) byte
+ */
+static void
+dtrace_get_SIB(dis86_t *x, uint_t *ss, uint_t *index, uint_t *base)
+{
+	int byte;
+
+	if (x->d86_error)
+		return;
+
+	byte = x->d86_get_byte(x->d86_data);
+	if (byte < 0) {
+		x->d86_error = 1;
+		return;
+	}
+	x->d86_bytes[x->d86_len++] = byte;
+
+	*base = byte & 0x7;
+	*index = (byte >> 3) & 0x7;
+	*ss = (byte >> 6) & 0x3;
+}
+
+/*
+ * Get the byte following the op code and separate it into the
+ * mode, register, and r/m fields.
+ */
+static void
+dtrace_get_modrm(dis86_t *x, uint_t *mode, uint_t *reg, uint_t *r_m)
+{
+	if (x->d86_got_modrm == 0) {
+		if (x->d86_rmindex == -1)
+			x->d86_rmindex = x->d86_len;
+		dtrace_get_SIB(x, mode, reg, r_m);
+		x->d86_got_modrm = 1;
+	}
+}
+
+/*
+ * Adjust register selection based on any REX prefix bits present.
+ */
+/*ARGSUSED*/
+static void
+dtrace_rex_adjust(uint_t rex_prefix, uint_t mode, uint_t *reg, uint_t *r_m)
+{
+	if (reg != NULL && r_m == NULL) {
+		if (rex_prefix & REX_B)
+			*reg += 8;
+	} else {
+		if (reg != NULL && (REX_R & rex_prefix) != 0)
+			*reg += 8;
+		if (r_m != NULL && (REX_B & rex_prefix) != 0)
+			*r_m += 8;
+	}
+}
+
+/*
+ * Get an immediate operand of the given size, with sign extension.
+ */
+static void
+dtrace_imm_opnd(dis86_t *x, int wbit, int size, int opindex)
+{
+	int i;
+	int byte;
+	int valsize;
+
+	if (x->d86_numopnds < opindex + 1)
+		x->d86_numopnds = opindex + 1;
+
+	switch (wbit) {
+	case BYTE_OPND:
+		valsize = 1;
+		break;
+	case LONG_OPND:
+		if (x->d86_opnd_size == SIZE16)
+			valsize = 2;
+		else if (x->d86_opnd_size == SIZE32)
+			valsize = 4;
+		else
+			valsize = 8;
+		break;
+	case MM_OPND:
+	case XMM_OPND:
+	case SEG_OPND:
+	case CONTROL_OPND:
+	case DEBUG_OPND:
+	case TEST_OPND:
+		valsize = size;
+		break;
+	case WORD_OPND:
+		valsize = 2;
+		break;
+	}
+	if (valsize < size)
+		valsize = size;
+
+	if (x->d86_error)
+		return;
+	x->d86_opnd[opindex].d86_value = 0;
+	for (i = 0; i < size; ++i) {
+		byte = x->d86_get_byte(x->d86_data);
+		if (byte < 0) {
+			x->d86_error = 1;
+			return;
+		}
+		x->d86_bytes[x->d86_len++] = byte;
+		x->d86_opnd[opindex].d86_value |= (uint64_t)byte << (i * 8);
+	}
+	/* Do sign extension */
+	if (x->d86_bytes[x->d86_len - 1] & 0x80) {
+		for (; i < sizeof (uint64_t); i++)
+			x->d86_opnd[opindex].d86_value |=
+			    (uint64_t)0xff << (i * 8);
+	}
+#ifdef DIS_TEXT
+	x->d86_opnd[opindex].d86_mode = MODE_SIGNED;
+	x->d86_opnd[opindex].d86_value_size = valsize;
+	x->d86_imm_bytes += size;
+#endif
+}
+
+/*
+ * Get an ip relative operand of the given size, with sign extension.
+ */
+static void
+dtrace_disp_opnd(dis86_t *x, int wbit, int size, int opindex)
+{
+	dtrace_imm_opnd(x, wbit, size, opindex);
+#ifdef DIS_TEXT
+	x->d86_opnd[opindex].d86_mode = MODE_IPREL;
+#endif
+}
+
+/*
+ * Check to see if there is a segment override prefix pending.
+ * If so, print it in the current 'operand' location and set
+ * the override flag back to false.
+ */
+/*ARGSUSED*/
+static void
+dtrace_check_override(dis86_t *x, int opindex)
+{
+#ifdef DIS_TEXT
+	if (x->d86_seg_prefix) {
+		(void) strlcat(x->d86_opnd[opindex].d86_prefix,
+		    x->d86_seg_prefix, PFIXLEN);
+	}
+#endif
+	x->d86_seg_prefix = NULL;
+}
+
+
+/*
+ * Process a single instruction Register or Memory operand.
+ *
+ * mode = addressing mode from ModRM byte
+ * r_m = r_m (or reg if mode == 3) field from ModRM byte
+ * wbit = indicates which register (8bit, 16bit, ... MMX, etc.) set to use.
+ * o = index of operand that we are processing (0, 1 or 2)
+ *
+ * the value of reg or r_m must have already been adjusted for any REX prefix.
+ */
+/*ARGSUSED*/
+static void
+dtrace_get_operand(dis86_t *x, uint_t mode, uint_t r_m, int wbit, int opindex)
+{
+	int have_SIB = 0;	/* flag presence of scale-index-byte */
+	uint_t ss;		/* scale-factor from opcode */
+	uint_t index;		/* index register number */
+	uint_t base;		/* base register number */
+	int dispsize;   	/* size of displacement in bytes */
+#ifdef DIS_TEXT
+	char *opnd = x->d86_opnd[opindex].d86_opnd;
+#endif
+
+	if (x->d86_numopnds < opindex + 1)
+		x->d86_numopnds = opindex + 1;
+
+	if (x->d86_error)
+		return;
+
+	/*
+	 * first handle a simple register
+	 */
+	if (mode == REG_ONLY) {
+#ifdef DIS_TEXT
+		switch (wbit) {
+		case MM_OPND:
+			(void) strlcat(opnd, dis_MMREG[r_m], OPLEN);
+			break;
+		case XMM_OPND:
+			(void) strlcat(opnd, dis_XMMREG[r_m], OPLEN);
+			break;
+		case SEG_OPND:
+			(void) strlcat(opnd, dis_SEGREG[r_m], OPLEN);
+			break;
+		case CONTROL_OPND:
+			(void) strlcat(opnd, dis_CONTROLREG[r_m], OPLEN);
+			break;
+		case DEBUG_OPND:
+			(void) strlcat(opnd, dis_DEBUGREG[r_m], OPLEN);
+			break;
+		case TEST_OPND:
+			(void) strlcat(opnd, dis_TESTREG[r_m], OPLEN);
+			break;
+		case BYTE_OPND:
+			if (x->d86_rex_prefix == 0)
+				(void) strlcat(opnd, dis_REG8[r_m], OPLEN);
+			else
+				(void) strlcat(opnd, dis_REG8_REX[r_m], OPLEN);
+			break;
+		case WORD_OPND:
+			(void) strlcat(opnd, dis_REG16[r_m], OPLEN);
+			break;
+		case LONG_OPND:
+			if (x->d86_opnd_size == SIZE16)
+				(void) strlcat(opnd, dis_REG16[r_m], OPLEN);
+			else if (x->d86_opnd_size == SIZE32)
+				(void) strlcat(opnd, dis_REG32[r_m], OPLEN);
+			else
+				(void) strlcat(opnd, dis_REG64[r_m], OPLEN);
+			break;
+		}
+#endif /* DIS_TEXT */
+		return;
+	}
+
+	/*
+	 * if symbolic representation, skip override prefix, if any
+	 */
+	dtrace_check_override(x, opindex);
+
+	/*
+	 * Handle 16 bit memory references first, since they decode
+	 * the mode values more simply.
+	 * mode 1 is r_m + 8 bit displacement
+	 * mode 2 is r_m + 16 bit displacement
+	 * mode 0 is just r_m, unless r_m is 6 which is 16 bit disp
+	 */
+	if (x->d86_addr_size == SIZE16) {
+		if ((mode == 0 && r_m == 6) || mode == 2)
+			dtrace_imm_opnd(x, WORD_OPND, 2, opindex);
+		else if (mode == 1)
+			dtrace_imm_opnd(x, BYTE_OPND, 1, opindex);
+#ifdef DIS_TEXT
+		if (mode == 0 && r_m == 6)
+			x->d86_opnd[opindex].d86_mode = MODE_SIGNED;
+		else if (mode == 0)
+			x->d86_opnd[opindex].d86_mode = MODE_NONE;
+		else
+			x->d86_opnd[opindex].d86_mode = MODE_OFFSET;
+		(void) strlcat(opnd, dis_addr16[mode][r_m], OPLEN);
+#endif
+		return;
+	}
+
+	/*
+	 * 32 and 64 bit addressing modes are more complex since they
+	 * can involve an SIB (scaled index and base) byte to decode.
+	 */
+	if (r_m == ESP_REGNO || r_m == ESP_REGNO + 8) {
+		have_SIB = 1;
+		dtrace_get_SIB(x, &ss, &index, &base);
+		if (x->d86_error)
+			return;
+		if (base != 5 || mode != 0)
+			if (x->d86_rex_prefix & REX_B)
+				base += 8;
+		if (x->d86_rex_prefix & REX_X)
+			index += 8;
+	} else {
+		base = r_m;
+	}
+
+	/*
+	 * Compute the displacement size and get its bytes
+	 */
+	dispsize = 0;
+
+	if (mode == 1)
+		dispsize = 1;
+	else if (mode == 2)
+		dispsize = 4;
+	else if ((r_m & 7) == EBP_REGNO ||
+	    (have_SIB && (base & 7) == EBP_REGNO))
+		dispsize = 4;
+
+	if (dispsize > 0) {
+		dtrace_imm_opnd(x, dispsize == 4 ? LONG_OPND : BYTE_OPND,
+		    dispsize, opindex);
+		if (x->d86_error)
+			return;
+	}
+
+#ifdef DIS_TEXT
+	if (dispsize > 0)
+		x->d86_opnd[opindex].d86_mode = MODE_OFFSET;
+
+	if (have_SIB == 0) {
+		if (x->d86_mode == SIZE32) {
+			if (mode == 0)
+				(void) strlcat(opnd, dis_addr32_mode0[r_m],
+				    OPLEN);
+			else
+				(void) strlcat(opnd, dis_addr32_mode12[r_m],
+				    OPLEN);
+		} else {
+			if (mode == 0) {
+				(void) strlcat(opnd, dis_addr64_mode0[r_m],
+				    OPLEN);
+				if (r_m == 5) {
+					x->d86_opnd[opindex].d86_mode =
+					    MODE_RIPREL;
+				}
+			} else {
+				(void) strlcat(opnd, dis_addr64_mode12[r_m],
+				    OPLEN);
+			}
+		}
+	} else {
+		uint_t need_paren = 0;
+		char **regs;
+		if (x->d86_mode == SIZE32) /* NOTE this is not addr_size! */
+			regs = (char **)dis_REG32;
+		else
+			regs = (char **)dis_REG64;
+
+		/*
+		 * print the base (if any)
+		 */
+		if (base == EBP_REGNO && mode == 0) {
+			if (index != ESP_REGNO) {
+				(void) strlcat(opnd, "(", OPLEN);
+				need_paren = 1;
+			}
+		} else {
+			(void) strlcat(opnd, "(", OPLEN);
+			(void) strlcat(opnd, regs[base], OPLEN);
+			need_paren = 1;
+		}
+
+		/*
+		 * print the index (if any)
+		 */
+		if (index != ESP_REGNO) {
+			(void) strlcat(opnd, ",", OPLEN);
+			(void) strlcat(opnd, regs[index], OPLEN);
+			(void) strlcat(opnd, dis_scale_factor[ss], OPLEN);
+		} else
+			if (need_paren)
+				(void) strlcat(opnd, ")", OPLEN);
+	}
+#endif
+}
+
+/*
+ * Operand sequence for standard instruction involving one register
+ * and one register/memory operand.
+ * wbit indicates a byte(0) or opnd_size(1) operation
+ * vbit indicates direction (0 for "opcode r,r_m") or (1 for "opcode r_m, r")
+ */
+#define	STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, vbit)  {	\
+		dtrace_get_modrm(x, &mode, &reg, &r_m);			\
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);	\
+		dtrace_get_operand(x, mode, r_m, wbit, vbit);		\
+		dtrace_get_operand(x, REG_ONLY, reg, wbit, 1 - vbit);	\
+}
+
+/*
+ * Similar to above, but allows for the two operands to be of different
+ * classes (ie. wbit).
+ *	wbit is for the r_m operand
+ *	w2 is for the reg operand
+ */
+#define	MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, w2, vbit)	{	\
+		dtrace_get_modrm(x, &mode, &reg, &r_m);			\
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);	\
+		dtrace_get_operand(x, mode, r_m, wbit, vbit);		\
+		dtrace_get_operand(x, REG_ONLY, reg, w2, 1 - vbit);	\
+}
+
+/*
+ * Similar, but for 2 operands plus an immediate.
+ */
+#define	THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, immsize) { \
+		dtrace_get_modrm(x, &mode, &reg, &r_m);			\
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);	\
+		dtrace_get_operand(x, mode, r_m, wbit, 1);		\
+		dtrace_get_operand(x, REG_ONLY, reg, w2, 2);		\
+		dtrace_imm_opnd(x, wbit, immsize, 0);			\
+}
+
+/*
+ * Dissassemble a single x86 or amd64 instruction.
+ *
+ * Mode determines the default operating mode (SIZE16, SIZE32 or SIZE64)
+ * for interpreting instructions.
+ *
+ * returns non-zero for bad opcode
+ */
+int
+dtrace_disx86(dis86_t *x, uint_t cpu_mode)
+{
+	instable_t *dp;		/* decode table being used */
+#ifdef DIS_TEXT
+	uint_t i;
+#endif
+#ifdef DIS_MEM
+	uint_t nomem = 0;
+#define	NOMEM	(nomem = 1)
+#else
+#define	NOMEM	/* nothing */
+#endif
+	uint_t wbit;		/* opcode wbit, 0 is 8 bit, !0 for opnd_size */
+	uint_t w2;		/* wbit value for second operand */
+	uint_t vbit;
+	uint_t mode = 0;	/* mode value from ModRM byte */
+	uint_t reg;		/* reg value from ModRM byte */
+	uint_t r_m;		/* r_m value from ModRM byte */
+
+	uint_t opcode1;		/* high nibble of 1st byte */
+	uint_t opcode2;		/* low nibble of 1st byte */
+	uint_t opcode3;		/* extra opcode bits usually from ModRM byte */
+	uint_t opcode4;		/* high nibble of 2nd byte */
+	uint_t opcode5;		/* low nibble of 2nd byte */
+	uint_t opcode6;		/* high nibble of 3rd byte */
+	uint_t opcode7;		/* low nibble of 3rd byte */
+	uint_t opcode_bytes = 1;
+
+	/*
+	 * legacy prefixes come in 5 flavors, you should have only one of each
+	 */
+	uint_t	opnd_size_prefix = 0;
+	uint_t	addr_size_prefix = 0;
+	uint_t	segment_prefix = 0;
+	uint_t	lock_prefix = 0;
+	uint_t	rep_prefix = 0;
+	uint_t	rex_prefix = 0;	/* amd64 register extension prefix */
+	size_t	off;
+
+	x->d86_len = 0;
+	x->d86_rmindex = -1;
+	x->d86_error = 0;
+#ifdef DIS_TEXT
+	x->d86_numopnds = 0;
+	x->d86_seg_prefix = NULL;
+	x->d86_mnem[0] = 0;
+	for (i = 0; i < 3; ++i) {
+		x->d86_opnd[i].d86_opnd[0] = 0;
+		x->d86_opnd[i].d86_prefix[0] = 0;
+		x->d86_opnd[i].d86_value_size = 0;
+		x->d86_opnd[i].d86_value = 0;
+		x->d86_opnd[i].d86_mode = MODE_NONE;
+	}
+#endif
+	x->d86_error = 0;
+	x->d86_memsize = 0;
+
+	if (cpu_mode == SIZE16) {
+		opnd_size = SIZE16;
+		addr_size = SIZE16;
+	} else if (cpu_mode == SIZE32) {
+		opnd_size = SIZE32;
+		addr_size = SIZE32;
+	} else {
+		opnd_size = SIZE32;
+		addr_size = SIZE64;
+	}
+
+	/*
+	 * Get one opcode byte and check for zero padding that follows
+	 * jump tables.
+	 */
+	if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0)
+		goto error;
+
+	if (opcode1 == 0 && opcode2 == 0 &&
+	    x->d86_check_func != NULL && x->d86_check_func(x->d86_data)) {
+#ifdef DIS_TEXT
+		(void) strncpy(x->d86_mnem, ".byte\t0", OPLEN);
+#endif
+		goto done;
+	}
+
+	/*
+	 * Gather up legacy x86 prefix bytes.
+	 */
+	for (;;) {
+		uint_t *which_prefix = NULL;
+
+		dp = (instable_t *)&dis_distable[opcode1][opcode2];
+
+		switch (dp->it_adrmode) {
+		case PREFIX:
+			which_prefix = &rep_prefix;
+			break;
+		case LOCK:
+			which_prefix = &lock_prefix;
+			break;
+		case OVERRIDE:
+			which_prefix = &segment_prefix;
+#ifdef DIS_TEXT
+			x->d86_seg_prefix = (char *)dp->it_name;
+#endif
+			if (dp->it_invalid64 && cpu_mode == SIZE64)
+				goto error;
+			break;
+		case AM:
+			which_prefix = &addr_size_prefix;
+			break;
+		case DM:
+			which_prefix = &opnd_size_prefix;
+			break;
+		}
+		if (which_prefix == NULL)
+			break;
+		*which_prefix = (opcode1 << 4) | opcode2;
+		if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0)
+			goto error;
+	}
+
+	/*
+	 * Handle amd64 mode PREFIX values.
+	 * Some of the segment prefixes are no-ops. (only FS/GS actually work)
+	 * We might have a REX prefix (opcodes 0x40-0x4f)
+	 */
+	if (cpu_mode == SIZE64) {
+		if (segment_prefix != 0x64 && segment_prefix != 0x65)
+			segment_prefix = 0;
+
+		if (opcode1 == 0x4) {
+			rex_prefix = (opcode1 << 4) | opcode2;
+			if (dtrace_get_opcode(x, &opcode1, &opcode2) != 0)
+				goto error;
+			dp = (instable_t *)&dis_distable[opcode1][opcode2];
+		}
+	}
+
+	/*
+	 * Deal with selection of operand and address size now.
+	 * Note that the REX.W bit being set causes opnd_size_prefix to be
+	 * ignored.
+	 */
+	if (cpu_mode == SIZE64) {
+		if (rex_prefix & REX_W)
+			opnd_size = SIZE64;
+		else if (opnd_size_prefix)
+			opnd_size = SIZE16;
+
+		if (addr_size_prefix)
+			addr_size = SIZE32;
+	} else if (cpu_mode == SIZE32) {
+		if (opnd_size_prefix)
+			opnd_size = SIZE16;
+		if (addr_size_prefix)
+			addr_size = SIZE16;
+	} else {
+		if (opnd_size_prefix)
+			opnd_size = SIZE32;
+		if (addr_size_prefix)
+			addr_size = SIZE32;
+	}
+
+	/*
+	 * The pause instruction - a repz'd nop.  This doesn't fit
+	 * with any of the other prefix goop added for SSE, so we'll
+	 * special-case it here.
+	 */
+	if (rep_prefix == 0xf3 && opcode1 == 0x9 && opcode2 == 0x0) {
+		rep_prefix = 0;
+		dp = (instable_t *)&dis_opPause;
+	}
+
+	/*
+	 * Some 386 instructions have 2 bytes of opcode before the mod_r/m
+	 * byte so we may need to perform a table indirection.
+	 */
+	if (dp->it_indirect == (instable_t *)dis_op0F) {
+		if (dtrace_get_opcode(x, &opcode4, &opcode5) != 0)
+			goto error;
+		opcode_bytes = 2;
+		if (opcode4 == 0x7 && opcode5 >= 0x1 && opcode5 <= 0x3) {
+			uint_t	subcode;
+
+			if (dtrace_get_opcode(x, &opcode6, &opcode7) != 0)
+				goto error;
+			opcode_bytes = 3;
+			subcode = ((opcode6 & 0x3) << 1) |
+			    ((opcode7 & 0x8) >> 3);
+			dp = (instable_t *)&dis_op0F7123[opcode5][subcode];
+		} else if ((opcode4 == 0xc) && (opcode5 >= 0x8)) {
+			dp = (instable_t *)&dis_op0FC8[0];
+		} else {
+			dp = (instable_t *)&dis_op0F[opcode4][opcode5];
+		}
+	}
+
+	/*
+	 * If still not at a TERM decode entry, then a ModRM byte
+	 * exists and its fields further decode the instruction.
+	 */
+	x->d86_got_modrm = 0;
+	if (dp->it_indirect != TERM) {
+		dtrace_get_modrm(x, &mode, &opcode3, &r_m);
+		if (x->d86_error)
+			goto error;
+		reg = opcode3;
+
+		/*
+		 * decode 287 instructions (D8-DF) from opcodeN
+		 */
+		if (opcode1 == 0xD && opcode2 >= 0x8) {
+			if (opcode2 == 0xB && mode == 0x3 && opcode3 == 4)
+				dp = (instable_t *)&dis_opFP5[r_m];
+			else if (opcode2 == 0xA && mode == 0x3 && opcode3 < 4)
+				dp = (instable_t *)&dis_opFP7[opcode3];
+			else if (opcode2 == 0xB && mode == 0x3)
+				dp = (instable_t *)&dis_opFP6[opcode3];
+			else if (opcode2 == 0x9 && mode == 0x3 && opcode3 >= 4)
+				dp = (instable_t *)&dis_opFP4[opcode3 - 4][r_m];
+			else if (mode == 0x3)
+				dp = (instable_t *)
+				    &dis_opFP3[opcode2 - 8][opcode3];
+			else
+				dp = (instable_t *)
+				    &dis_opFP1n2[opcode2 - 8][opcode3];
+		} else {
+			dp = (instable_t *)dp->it_indirect + opcode3;
+		}
+	}
+
+	/*
+	 * In amd64 bit mode, ARPL opcode is changed to MOVSXD
+	 * (sign extend 32bit to 64 bit)
+	 */
+	if (cpu_mode == SIZE64 && opcode1 == 0x6 && opcode2 == 0x3)
+		dp = (instable_t *)&dis_opMOVSLD;
+
+	/*
+	 * at this point we should have a correct (or invalid) opcode
+	 */
+	if (cpu_mode == SIZE64 && dp->it_invalid64 ||
+	    cpu_mode != SIZE64 && dp->it_invalid32)
+		goto error;
+	if (dp->it_indirect != TERM)
+		goto error;
+
+	/*
+	 * deal with MMX/SSE opcodes which are changed by prefixes
+	 */
+	switch (dp->it_adrmode) {
+	case MMO:
+	case MMOIMPL:
+	case MMO3P:
+	case MMOM3:
+	case MMOMS:
+	case MMOPM:
+	case MMOPRM:
+	case MMOS:
+	case XMMO:
+	case XMMOM:
+	case XMMOMS:
+	case XMMOPM:
+	case XMMOS:
+	case XMMOMX:
+	case XMMOX3:
+	case XMMOXMM:
+		/*
+		 * This is horrible.  Some SIMD instructions take the
+		 * form 0x0F 0x?? ..., which is easily decoded using the
+		 * existing tables.  Other SIMD instructions use various
+		 * prefix bytes to overload existing instructions.  For
+		 * Example, addps is F0, 58, whereas addss is F3 (repz),
+		 * F0, 58.  Presumably someone got a raise for this.
+		 *
+		 * If we see one of the instructions which can be
+		 * modified in this way (if we've got one of the SIMDO*
+		 * address modes), we'll check to see if the last prefix
+		 * was a repz.  If it was, we strip the prefix from the
+		 * mnemonic, and we indirect using the dis_opSIMDrepz
+		 * table.
+		 */
+
+		/*
+		 * Calculate our offset in dis_op0F
+		 */
+		if ((uintptr_t)dp - (uintptr_t)dis_op0F > sizeof (dis_op0F))
+			goto error;
+
+		off = ((uintptr_t)dp - (uintptr_t)dis_op0F) /
+		    sizeof (instable_t);
+
+		/*
+		 * Rewrite if this instruction used one of the magic prefixes.
+		 */
+		if (rep_prefix) {
+			if (rep_prefix == 0xf2)
+				dp = (instable_t *)&dis_opSIMDrepnz[off];
+			else
+				dp = (instable_t *)&dis_opSIMDrepz[off];
+			rep_prefix = 0;
+		} else if (opnd_size_prefix) {
+			dp = (instable_t *)&dis_opSIMDdata16[off];
+			opnd_size_prefix = 0;
+			if (opnd_size == SIZE16)
+				opnd_size = SIZE32;
+		}
+		break;
+
+	case MMOSH:
+		/*
+		 * As with the "normal" SIMD instructions, the MMX
+		 * shuffle instructions are overloaded.  These
+		 * instructions, however, are special in that they use
+		 * an extra byte, and thus an extra table.  As of this
+		 * writing, they only use the opnd_size prefix.
+		 */
+
+		/*
+		 * Calculate our offset in dis_op0F7123
+		 */
+		if ((uintptr_t)dp - (uintptr_t)dis_op0F7123 >
+		    sizeof (dis_op0F7123))
+			goto error;
+
+		if (opnd_size_prefix) {
+			off = ((uintptr_t)dp - (uintptr_t)dis_op0F7123) /
+			    sizeof (instable_t);
+			dp = (instable_t *)&dis_opSIMD7123[off];
+			opnd_size_prefix = 0;
+			if (opnd_size == SIZE16)
+				opnd_size = SIZE32;
+		}
+		break;
+	}
+
+	/*
+	 * In 64 bit mode, some opcodes automatically use opnd_size == SIZE64.
+	 */
+	if (cpu_mode == SIZE64)
+		if (dp->it_always64 || (opnd_size == SIZE32 && dp->it_stackop))
+			opnd_size = SIZE64;
+
+#ifdef DIS_TEXT
+	/*
+	 * At this point most instructions can format the opcode mnemonic
+	 * including the prefixes.
+	 */
+	if (lock_prefix)
+		(void) strlcat(x->d86_mnem, "lock ", OPLEN);
+
+	if (rep_prefix == 0xf2)
+		(void) strlcat(x->d86_mnem, "repnz ", OPLEN);
+	else if (rep_prefix == 0xf3)
+		(void) strlcat(x->d86_mnem, "repz ", OPLEN);
+
+	if (cpu_mode == SIZE64 && addr_size_prefix)
+		(void) strlcat(x->d86_mnem, "addr32 ", OPLEN);
+
+	if (dp->it_adrmode != CBW &&
+	    dp->it_adrmode != CWD &&
+	    dp->it_adrmode != XMMSFNC) {
+		if (strcmp(dp->it_name, "INVALID") == 0)
+			goto error;
+		(void) strlcat(x->d86_mnem, dp->it_name, OPLEN);
+		if (dp->it_suffix) {
+			char *types[] = {"", "w", "l", "q"};
+			if (opcode_bytes == 2 && opcode4 == 4) {
+				/* It's a cmovx.yy. Replace the suffix x */
+				for (i = 5; i < OPLEN; i++) {
+					if (x->d86_mnem[i] == '.')
+						break;
+				}
+				x->d86_mnem[i - 1] = *types[opnd_size];
+			} else {
+				(void) strlcat(x->d86_mnem, types[opnd_size],
+				    OPLEN);
+			}
+		}
+	}
+#endif
+
+	/*
+	 * Process operands based on the addressing modes.
+	 */
+	x->d86_mode = cpu_mode;
+	x->d86_rex_prefix = rex_prefix;
+	x->d86_opnd_size = opnd_size;
+	x->d86_addr_size = addr_size;
+	vbit = 0;		/* initialize for mem/reg -> reg */
+	switch (dp->it_adrmode) {
+		/*
+		 * amd64 instruction to sign extend 32 bit reg/mem operands
+		 * into 64 bit register values
+		 */
+	case MOVSXZ:
+#ifdef DIS_TEXT
+		if (rex_prefix == 0)
+			(void) strncpy(x->d86_mnem, "movzld", OPLEN);
+#endif
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		x->d86_opnd_size = SIZE64;
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1);
+		x->d86_opnd_size = opnd_size = SIZE32;
+		wbit = LONG_OPND;
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		break;
+
+		/*
+		 * movsbl movsbw movsbq (0x0FBE) or movswl movswq (0x0FBF)
+		 * movzbl movzbw movzbq (0x0FB6) or movzwl movzwq (0x0FB7)
+		 * wbit lives in 2nd byte, note that operands
+		 * are different sized
+		 */
+	case MOVZ:
+		if (rex_prefix & REX_W) {
+			/* target register size = 64 bit */
+			x->d86_mnem[5] = 'q';
+		}
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1);
+		x->d86_opnd_size = opnd_size = SIZE16;
+		wbit = WBIT(opcode5);
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		break;
+
+	/*
+	 * imul instruction, with either 8-bit or longer immediate
+	 * opcode 0x6B for byte, sign-extended displacement, 0x69 for word(s)
+	 */
+	case IMUL:
+		wbit = LONG_OPND;
+		THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND,
+		    OPSIZE(opnd_size, opcode2 == 0x9));
+		break;
+
+	/* memory or register operand to register, with 'w' bit	*/
+	case MRw:
+		wbit = WBIT(opcode2);
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0);
+		break;
+
+	/* register to memory or register operand, with 'w' bit	*/
+	/* arpl happens to fit here also because it is odd */
+	case RMw:
+		if (opcode_bytes == 2)
+			wbit = WBIT(opcode5);
+		else
+			wbit = WBIT(opcode2);
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1);
+		break;
+
+	/* xaddb instruction */
+	case XADDB:
+		wbit = 0;
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1);
+		break;
+
+	/* MMX register to memory or register operand		*/
+	case MMS:
+	case MMOS:
+#ifdef DIS_TEXT
+		wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND;
+#else
+		wbit = LONG_OPND;
+#endif
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 1);
+		break;
+
+	/* MMX register to memory */
+	case MMOMS:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode == REG_ONLY)
+			goto error;
+		wbit = MM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 1);
+		break;
+
+	/* Double shift. Has immediate operand specifying the shift. */
+	case DSHIFT:
+		wbit = LONG_OPND;
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 2);
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1);
+		dtrace_imm_opnd(x, wbit, 1, 0);
+		break;
+
+	/*
+	 * Double shift. With no immediate operand, specifies using %cl.
+	 */
+	case DSHIFTcl:
+		wbit = LONG_OPND;
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1);
+		break;
+
+	/* immediate to memory or register operand */
+	case IMlw:
+		wbit = WBIT(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 1);
+		/*
+		 * Have long immediate for opcode 0x81, but not 0x80 nor 0x83
+		 */
+		dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, opcode2 == 1), 0);
+		break;
+
+	/* immediate to memory or register operand with the	*/
+	/* 'w' bit present					*/
+	case IMw:
+		wbit = WBIT(opcode2);
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 1);
+		dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, wbit), 0);
+		break;
+
+	/* immediate to register with register in low 3 bits	*/
+	/* of op code						*/
+	case IR:
+		/* w-bit here (with regs) is bit 3 */
+		wbit = opcode2 >>3 & 0x1;
+		reg = REGNO(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, NULL);
+		mode = REG_ONLY;
+		r_m = reg;
+		dtrace_get_operand(x, mode, r_m, wbit, 1);
+		dtrace_imm_opnd(x, wbit, OPSIZE64(opnd_size, wbit), 0);
+		break;
+
+	/* MMX immediate shift of register */
+	case MMSH:
+	case MMOSH:
+		wbit = MM_OPND;
+		goto mm_shift;	/* in next case */
+
+	/* SIMD immediate shift of register */
+	case XMMSH:
+		wbit = XMM_OPND;
+mm_shift:
+		reg = REGNO(opcode7);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, NULL);
+		dtrace_get_operand(x, REG_ONLY, reg, wbit, 1);
+		dtrace_imm_opnd(x, wbit, 1, 0);
+		NOMEM;
+		break;
+
+	/* accumulator to memory operand */
+	case AO:
+		vbit = 1;
+		/*FALLTHROUGH*/
+
+	/* memory operand to accumulator */
+	case OA:
+		wbit = WBIT(opcode2);
+		dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1 - vbit);
+		dtrace_imm_opnd(x, wbit, OPSIZE64(addr_size, LONG_OPND), vbit);
+#ifdef DIS_TEXT
+		x->d86_opnd[vbit].d86_mode = MODE_OFFSET;
+#endif
+		break;
+
+
+	/* segment register to memory or register operand */
+	case SM:
+		vbit = 1;
+		/*FALLTHROUGH*/
+
+	/* memory or register operand to segment register */
+	case MS:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, LONG_OPND, vbit);
+		dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 1 - vbit);
+		break;
+
+	/*
+	 * rotate or shift instructions, which may shift by 1 or
+	 * consult the cl register, depending on the 'v' bit
+	 */
+	case Mv:
+		vbit = VBIT(opcode2);
+		wbit = WBIT(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 1);
+#ifdef DIS_TEXT
+		if (vbit) {
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "%cl", OPLEN);
+		} else {
+			x->d86_opnd[0].d86_mode = MODE_SIGNED;
+			x->d86_opnd[0].d86_value_size = 1;
+			x->d86_opnd[0].d86_value = 1;
+		}
+#endif
+		break;
+	/*
+	 * immediate rotate or shift instructions
+	 */
+	case MvI:
+		wbit = WBIT(opcode2);
+normal_imm_mem:
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 1);
+		dtrace_imm_opnd(x, wbit, 1, 0);
+		break;
+
+	/* bit test instructions */
+	case MIb:
+		wbit = LONG_OPND;
+		goto normal_imm_mem;
+
+	/* single memory or register operand with 'w' bit present */
+	case Mw:
+		wbit = WBIT(opcode2);
+just_mem:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		break;
+
+	case SWAPGS:
+		if (cpu_mode == SIZE64 && mode == 3 && r_m == 0) {
+#ifdef DIS_TEXT
+			(void) strncpy(x->d86_mnem, "swapgs", OPLEN);
+#endif
+			NOMEM;
+			break;
+		}
+		/*FALLTHROUGH*/
+
+	/* prefetch instruction - memory operand, but no memory acess */
+	case PREF:
+		NOMEM;
+		/*FALLTHROUGH*/
+
+	/* single memory or register operand */
+	case M:
+		wbit = LONG_OPND;
+		goto just_mem;
+
+	/* single memory or register byte operand */
+	case Mb:
+		wbit = BYTE_OPND;
+		goto just_mem;
+
+	case MO:
+		/* Similar to M, but only memory (no direct registers) */
+		wbit = LONG_OPND;
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode == 3)
+			goto error;
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		break;
+
+	/* move special register to register or reverse if vbit */
+	case SREG:
+		switch (opcode5) {
+
+		case 2:
+			vbit = 1;
+			/*FALLTHROUGH*/
+		case 0:
+			wbit = CONTROL_OPND;
+			break;
+
+		case 3:
+			vbit = 1;
+			/*FALLTHROUGH*/
+		case 1:
+			wbit = DEBUG_OPND;
+			break;
+
+		case 6:
+			vbit = 1;
+			/*FALLTHROUGH*/
+		case 4:
+			wbit = TEST_OPND;
+			break;
+
+		}
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		dtrace_get_operand(x, REG_ONLY, reg, wbit, vbit);
+		dtrace_get_operand(x, REG_ONLY, r_m, LONG_OPND, 1 - vbit);
+		NOMEM;
+		break;
+
+	/*
+	 * single register operand with register in the low 3
+	 * bits of op code
+	 */
+	case R:
+		if (opcode_bytes == 2)
+			reg = REGNO(opcode5);
+		else
+			reg = REGNO(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, NULL);
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0);
+		NOMEM;
+		break;
+
+	/*
+	 * register to accumulator with register in the low 3
+	 * bits of op code, xchg instructions
+	 */
+	case RA:
+		NOMEM;
+		reg = REGNO(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, &reg, NULL);
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 0);
+		dtrace_get_operand(x, REG_ONLY, EAX_REGNO, LONG_OPND, 1);
+		break;
+
+	/*
+	 * single segment register operand, with register in
+	 * bits 3-4 of op code byte
+	 */
+	case SEG:
+		NOMEM;
+		reg = (x->d86_bytes[x->d86_len - 1] >> 3) & 0x3;
+		dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 0);
+		break;
+
+	/*
+	 * single segment register operand, with register in
+	 * bits 3-5 of op code
+	 */
+	case LSEG:
+		NOMEM;
+		/* long seg reg from opcode */
+		reg = (x->d86_bytes[x->d86_len - 1] >> 3) & 0x7;
+		dtrace_get_operand(x, REG_ONLY, reg, SEG_OPND, 0);
+		break;
+
+	/* memory or register operand to register */
+	case MR:
+		wbit = LONG_OPND;
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0);
+		break;
+
+	case RM:
+		wbit = LONG_OPND;
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 1);
+		break;
+
+	/* MMX/SIMD-Int memory or mm reg to mm reg		*/
+	case MM:
+	case MMO:
+#ifdef DIS_TEXT
+		wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND;
+#else
+		wbit = LONG_OPND;
+#endif
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 0);
+		break;
+
+	case MMOIMPL:
+#ifdef DIS_TEXT
+		wbit = strcmp(dp->it_name, "movd") ? MM_OPND : LONG_OPND;
+#else
+		wbit = LONG_OPND;
+#endif
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode != REG_ONLY)
+			goto error;
+
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		dtrace_get_operand(x, REG_ONLY, reg, MM_OPND, 1);
+		mode = 0;	/* change for memory access size... */
+		break;
+
+	/* MMX/SIMD-Int and SIMD-FP predicated mm reg to r32 */
+	case MMO3P:
+		wbit = MM_OPND;
+		goto xmm3p;
+	case XMM3P:
+		wbit = XMM_OPND;
+xmm3p:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode != REG_ONLY)
+			goto error;
+
+		THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 1);
+		NOMEM;
+		break;
+
+	/* MMX/SIMD-Int predicated r32/mem to mm reg */
+	case MMOPRM:
+		wbit = LONG_OPND;
+		w2 = MM_OPND;
+		goto xmmprm;
+	case XMMPRM:
+		wbit = LONG_OPND;
+		w2 = XMM_OPND;
+xmmprm:
+		THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, w2, 1);
+		break;
+
+	/* MMX/SIMD-Int predicated mm/mem to mm reg */
+	case MMOPM:
+		wbit = w2 = MM_OPND;
+		goto xmmprm;
+
+	/* MMX/SIMD-Int mm reg to r32 */
+	case MMOM3:
+		NOMEM;
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode != REG_ONLY)
+			goto error;
+		wbit = MM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 0);
+		break;
+
+	/* SIMD memory or xmm reg operand to xmm reg		*/
+	case XMM:
+	case XMMO:
+	case XMMXIMPL:
+		wbit = XMM_OPND;
+		STANDARD_MODRM(x, mode, reg, r_m, rex_prefix, wbit, 0);
+
+		if (dp->it_adrmode == XMMXIMPL && mode != REG_ONLY)
+			goto error;
+
+#ifdef DIS_TEXT
+		/*
+		 * movlps and movhlps share opcodes.  They differ in the
+		 * addressing modes allowed for their operands.
+		 * movhps and movlhps behave similarly.
+		 */
+		if (mode == REG_ONLY) {
+			if (strcmp(dp->it_name, "movlps") == 0)
+				(void) strncpy(x->d86_mnem, "movhlps", OPLEN);
+			else if (strcmp(dp->it_name, "movhps") == 0)
+				(void) strncpy(x->d86_mnem, "movlhps", OPLEN);
+		}
+#endif
+		if (dp->it_adrmode == XMMXIMPL)
+			mode = 0;	/* change for memory access size... */
+		break;
+
+	/* SIMD xmm reg to memory or xmm reg */
+	case XMMS:
+	case XMMOS:
+	case XMMMS:
+	case XMMOMS:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+#ifdef DIS_TEXT
+		if ((strcmp(dp->it_name, "movlps") == 0 ||
+		    strcmp(dp->it_name, "movhps") == 0 ||
+		    strcmp(dp->it_name, "movntps") == 0) &&
+		    mode == REG_ONLY)
+			goto error;
+#endif
+		wbit = XMM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1);
+		break;
+
+	/* SIMD memory to xmm reg */
+	case XMMM:
+	case XMMOM:
+		wbit = XMM_OPND;
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+#ifdef DIS_TEXT
+		if (mode == REG_ONLY) {
+			if (strcmp(dp->it_name, "movhps") == 0)
+				(void) strncpy(x->d86_mnem, "movlhps", OPLEN);
+			else
+				goto error;
+		}
+#endif
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0);
+		break;
+
+	/* SIMD memory or r32 to xmm reg			*/
+	case XMM3MX:
+		wbit = LONG_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0);
+		break;
+
+	case XMM3MXS:
+		wbit = LONG_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1);
+		break;
+
+	/* SIMD memory or mm reg to xmm reg			*/
+	case XMMOMX:
+	/* SIMD mm to xmm */
+	case XMMMX:
+		wbit = MM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 0);
+		break;
+
+	/* SIMD memory or xmm reg to mm reg			*/
+	case XMMXMM:
+	case XMMOXMM:
+	case XMMXM:
+		wbit = XMM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, MM_OPND, 0);
+		break;
+
+
+	/* SIMD memory or xmm reg to r32			*/
+	case XMMXM3:
+		wbit = XMM_OPND;
+		MIXED_MM(x, mode, reg, r_m, rex_prefix, wbit, LONG_OPND, 0);
+		break;
+
+	/* SIMD xmm to r32					*/
+	case XMMX3:
+	case XMMOX3:
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+		if (mode != REG_ONLY)
+			goto error;
+		dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+		dtrace_get_operand(x, mode, r_m, XMM_OPND, 0);
+		dtrace_get_operand(x, REG_ONLY, reg, LONG_OPND, 1);
+		NOMEM;
+		break;
+
+	/* SIMD predicated memory or xmm reg with/to xmm reg */
+	case XMMP:
+	case XMMOPM:
+		wbit = XMM_OPND;
+		THREEOPERAND(x, mode, reg, r_m, rex_prefix, wbit, XMM_OPND, 1);
+
+#ifdef DIS_TEXT
+		/*
+		 * cmpps and cmpss vary their instruction name based
+		 * on the value of imm8.  Other XMMP instructions,
+		 * such as shufps, require explicit specification of
+		 * the predicate.
+		 */
+		if (dp->it_name[0] == 'c' &&
+		    dp->it_name[1] == 'm' &&
+		    dp->it_name[2] == 'p' &&
+		    strlen(dp->it_name) == 5) {
+			uchar_t pred = x->d86_opnd[0].d86_value & 0xff;
+
+			if (pred >= (sizeof (dis_PREDSUFFIX) / sizeof (char *)))
+				goto error;
+
+			(void) strncpy(x->d86_mnem, "cmp", OPLEN);
+			(void) strlcat(x->d86_mnem, dis_PREDSUFFIX[pred],
+			    OPLEN);
+			(void) strlcat(x->d86_mnem,
+			    dp->it_name + strlen(dp->it_name) - 2,
+			    OPLEN);
+			x->d86_opnd[0] = x->d86_opnd[1];
+			x->d86_opnd[1] = x->d86_opnd[2];
+			x->d86_numopnds = 2;
+		}
+#endif
+		break;
+
+	/* immediate operand to accumulator */
+	case IA:
+		wbit = WBIT(opcode2);
+		dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1);
+		dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, wbit), 0);
+		NOMEM;
+		break;
+
+	/* memory or register operand to accumulator */
+	case MA:
+		wbit = WBIT(opcode2);
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, wbit, 0);
+		break;
+
+	/* si register to di register used to reference memory		*/
+	case SD:
+#ifdef DIS_TEXT
+		dtrace_check_override(x, 0);
+		x->d86_numopnds = 2;
+		if (addr_size == SIZE64) {
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%rsi)",
+			    OPLEN);
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%rdi)",
+			    OPLEN);
+		} else if (addr_size == SIZE32) {
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%esi)",
+			    OPLEN);
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%edi)",
+			    OPLEN);
+		} else {
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%si)",
+			    OPLEN);
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%di)",
+			    OPLEN);
+		}
+#endif
+		wbit = LONG_OPND;
+		break;
+
+	/* accumulator to di register				*/
+	case AD:
+		wbit = WBIT(opcode2);
+#ifdef DIS_TEXT
+		dtrace_check_override(x, 1);
+		x->d86_numopnds = 2;
+		dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 0);
+		if (addr_size == SIZE64)
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%rdi)",
+			    OPLEN);
+		else if (addr_size == SIZE32)
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%edi)",
+			    OPLEN);
+		else
+			(void) strlcat(x->d86_opnd[1].d86_opnd, "(%di)",
+			    OPLEN);
+#endif
+		break;
+
+	/* si register to accumulator				*/
+	case SA:
+		wbit = WBIT(opcode2);
+#ifdef DIS_TEXT
+		dtrace_check_override(x, 0);
+		x->d86_numopnds = 2;
+		if (addr_size == SIZE64)
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%rsi)",
+			    OPLEN);
+		else if (addr_size == SIZE32)
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%esi)",
+			    OPLEN);
+		else
+			(void) strlcat(x->d86_opnd[0].d86_opnd, "(%si)",
+			    OPLEN);
+		dtrace_get_operand(x, REG_ONLY, EAX_REGNO, wbit, 1);
+#endif
+		break;
+
+	/*
+	 * single operand, a 16/32 bit displacement
+	 */
+	case D:
+		wbit = LONG_OPND;
+		dtrace_disp_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 0);
+		NOMEM;
+		break;
+
+	/* jmp/call indirect to memory or register operand		*/
+	case INM:
+#ifdef DIS_TEXT
+		(void) strlcat(x->d86_opnd[0].d86_prefix, "*", OPLEN);
+#endif
+		dtrace_rex_adjust(rex_prefix, mode, NULL, &r_m);
+		dtrace_get_operand(x, mode, r_m, LONG_OPND, 0);
+		wbit = LONG_OPND;
+		break;
+
+	/*
+	 * for long jumps and long calls -- a new code segment
+	 * register and an offset in IP -- stored in object
+	 * code in reverse order. Note - not valid in amd64
+	 */
+	case SO:
+		dtrace_check_override(x, 1);
+		wbit = LONG_OPND;
+		dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 1);
+#ifdef DIS_TEXT
+		x->d86_opnd[1].d86_mode = MODE_SIGNED;
+#endif
+		/* will now get segment operand */
+		dtrace_imm_opnd(x, wbit, 2, 0);
+		break;
+
+	/*
+	 * jmp/call. single operand, 8 bit displacement.
+	 * added to current EIP in 'compofff'
+	 */
+	case BD:
+		dtrace_disp_opnd(x, BYTE_OPND, 1, 0);
+		NOMEM;
+		break;
+
+	/* single 32/16 bit immediate operand			*/
+	case I:
+		wbit = LONG_OPND;
+		dtrace_imm_opnd(x, wbit, OPSIZE(opnd_size, LONG_OPND), 0);
+		break;
+
+	/* single 8 bit immediate operand			*/
+	case Ib:
+		wbit = LONG_OPND;
+		dtrace_imm_opnd(x, wbit, 1, 0);
+		break;
+
+	case ENTER:
+		wbit = LONG_OPND;
+		dtrace_imm_opnd(x, wbit, 2, 0);
+		dtrace_imm_opnd(x, wbit, 1, 1);
+		switch (opnd_size) {
+		case SIZE64:
+			x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 8;
+			break;
+		case SIZE32:
+			x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 4;
+			break;
+		case SIZE16:
+			x->d86_memsize = (x->d86_opnd[1].d86_value + 1) * 2;
+			break;
+		}
+
+		break;
+
+	/* 16-bit immediate operand */
+	case RET:
+		wbit = LONG_OPND;
+		dtrace_imm_opnd(x, wbit, 2, 0);
+		break;
+
+	/* single 8 bit port operand				*/
+	case P:
+		dtrace_check_override(x, 0);
+		dtrace_imm_opnd(x, BYTE_OPND, 1, 0);
+		NOMEM;
+		break;
+
+	/* single operand, dx register (variable port instruction) */
+	case V:
+		x->d86_numopnds = 1;
+		dtrace_check_override(x, 0);
+#ifdef DIS_TEXT
+		(void) strlcat(x->d86_opnd[0].d86_opnd, "(%dx)", OPLEN);
+#endif
+		NOMEM;
+		break;
+
+	/*
+	 * The int instruction, which has two forms:
+	 * int 3 (breakpoint) or
+	 * int n, where n is indicated in the subsequent
+	 * byte (format Ib).  The int 3 instruction (opcode 0xCC),
+	 * where, although the 3 looks  like an operand,
+	 * it is implied by the opcode. It must be converted
+	 * to the correct base and output.
+	 */
+	case INT3:
+#ifdef DIS_TEXT
+		x->d86_numopnds = 1;
+		x->d86_opnd[0].d86_mode = MODE_SIGNED;
+		x->d86_opnd[0].d86_value_size = 1;
+		x->d86_opnd[0].d86_value = 3;
+#endif
+		NOMEM;
+		break;
+
+	/* single 8 bit immediate operand			*/
+	case INTx:
+		dtrace_imm_opnd(x, BYTE_OPND, 1, 0);
+		NOMEM;
+		break;
+
+	/* an unused byte must be discarded */
+	case U:
+		if (x->d86_get_byte(x->d86_data) < 0)
+			goto error;
+		x->d86_len++;
+		NOMEM;
+		break;
+
+	case CBW:
+#ifdef DIS_TEXT
+		if (opnd_size == SIZE16)
+			(void) strlcat(x->d86_mnem, "cbtw", OPLEN);
+		else if (opnd_size == SIZE32)
+			(void) strlcat(x->d86_mnem, "cwtl", OPLEN);
+		else
+			(void) strlcat(x->d86_mnem, "cltq", OPLEN);
+#endif
+		wbit = LONG_OPND;
+		NOMEM;
+		break;
+
+	case CWD:
+#ifdef DIS_TEXT
+		if (opnd_size == SIZE16)
+			(void) strlcat(x->d86_mnem, "cwtd", OPLEN);
+		else if (opnd_size == SIZE32)
+			(void) strlcat(x->d86_mnem, "cltd", OPLEN);
+		else
+			(void) strlcat(x->d86_mnem, "cqtd", OPLEN);
+#endif
+		wbit = LONG_OPND;
+		NOMEM;
+		break;
+
+	case XMMSFNC:
+		/*
+		 * sfence is sfence if mode is REG_ONLY.  If mode isn't
+		 * REG_ONLY, mnemonic should be 'clflush'.
+		 */
+		dtrace_get_modrm(x, &mode, &reg, &r_m);
+
+		/* sfence doesn't take operands */
+#ifdef DIS_TEXT
+		if (mode == REG_ONLY) {
+			(void) strlcat(x->d86_mnem, "sfence", OPLEN);
+		} else {
+			(void) strlcat(x->d86_mnem, "clflush", OPLEN);
+			dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+			dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0);
+			NOMEM;
+		}
+#else
+		if (mode != REG_ONLY) {
+			dtrace_rex_adjust(rex_prefix, mode, &reg, &r_m);
+			dtrace_get_operand(x, mode, r_m, BYTE_OPND, 0);
+			NOMEM;
+		}
+#endif
+		break;
+
+	/*
+	 * no disassembly, the mnemonic was all there was so go on
+	 */
+	case NORM:
+		if (dp->it_invalid32 && cpu_mode != SIZE64)
+			goto error;
+		NOMEM;
+		/*FALLTHROUGH*/
+	case IMPLMEM:
+		break;
+
+	case XMMFENCE:
+		/*
+		 * Only the following exact byte sequences are allowed:
+		 *
+		 * 	0f ae e8	lfence
+		 * 	0f ae f0	mfence
+		 */
+		if ((uint8_t)x->d86_bytes[x->d86_len - 1] != 0xe8 &&
+		    (uint8_t)x->d86_bytes[x->d86_len - 1] != 0xf0)
+			goto error;
+
+		break;
+
+
+	/* float reg */
+	case F:
+#ifdef DIS_TEXT
+		x->d86_numopnds = 1;
+		(void) strlcat(x->d86_opnd[0].d86_opnd, "%st(X)", OPLEN);
+		x->d86_opnd[0].d86_opnd[4] = r_m + '0';
+#endif
+		NOMEM;
+		break;
+
+	/* float reg to float reg, with ret bit present */
+	case FF:
+		vbit = opcode2 >> 2 & 0x1;	/* vbit = 1: st -> st(i) */
+		/*FALLTHROUGH*/
+	case FFC:				/* case for vbit always = 0 */
+#ifdef DIS_TEXT
+		x->d86_numopnds = 2;
+		(void) strlcat(x->d86_opnd[1 - vbit].d86_opnd, "%st", OPLEN);
+		(void) strlcat(x->d86_opnd[vbit].d86_opnd, "%st(X)", OPLEN);
+		x->d86_opnd[vbit].d86_opnd[4] = r_m + '0';
+#endif
+		NOMEM;
+		break;
+
+	/* an invalid op code */
+	case AM:
+	case DM:
+	case OVERRIDE:
+	case PREFIX:
+	case UNKNOWN:
+		NOMEM;
+	default:
+		goto error;
+	} /* end switch */
+	if (x->d86_error)
+		goto error;
+
+done:
+#ifdef DIS_MEM
+	/*
+	 * compute the size of any memory accessed by the instruction
+	 */
+	if (x->d86_memsize != 0) {
+		return (0);
+	} else if (dp->it_stackop) {
+		switch (opnd_size) {
+		case SIZE16:
+			x->d86_memsize = 2;
+			break;
+		case SIZE32:
+			x->d86_memsize = 4;
+			break;
+		case SIZE64:
+			x->d86_memsize = 8;
+			break;
+		}
+	} else if (nomem || mode == REG_ONLY) {
+		x->d86_memsize = 0;
+
+	} else if (dp->it_size != 0) {
+		/*
+		 * In 64 bit mode descriptor table entries
+		 * go up to 10 bytes and popf/pushf are always 8 bytes
+		 */
+		if (x->d86_mode == SIZE64 && dp->it_size == 6)
+			x->d86_memsize = 10;
+		else if (x->d86_mode == SIZE64 && opcode1 == 0x9 &&
+		    (opcode2 == 0xc || opcode2 == 0xd))
+			x->d86_memsize = 8;
+		else
+			x->d86_memsize = dp->it_size;
+
+	} else if (wbit == 0) {
+		x->d86_memsize = 1;
+
+	} else if (wbit == LONG_OPND) {
+		if (opnd_size == SIZE64)
+			x->d86_memsize = 8;
+		else if (opnd_size == SIZE32)
+			x->d86_memsize = 4;
+		else
+			x->d86_memsize = 2;
+
+	} else if (wbit == SEG_OPND) {
+		x->d86_memsize = 4;
+
+	} else {
+		x->d86_memsize = 8;
+	}
+#endif
+	return (0);
+
+error:
+#ifdef DIS_TEXT
+	(void) strlcat(x->d86_mnem, "undef", OPLEN);
+#endif
+	return (1);
+}
+
+#ifdef DIS_TEXT
+
+/*
+ * Some instructions should have immediate operands printed
+ * as unsigned integers. We compare against this table.
+ */
+static char *unsigned_ops[] = {
+	"or", "and", "xor", "test", "in", "out", "lcall", "ljmp",
+	"rcr", "rcl", "ror", "rol", "shl", "shr", "sal", "psr", "psl",
+	0
+};
+
+
+static int
+isunsigned_op(char *opcode)
+{
+	char *where;
+	int i;
+	int is_unsigned = 0;
+
+	/*
+	 * Work back to start of last mnemonic, since we may have
+	 * prefixes on some opcodes.
+	 */
+	where = opcode + strlen(opcode) - 1;
+	while (where > opcode && *where != ' ')
+		--where;
+	if (*where == ' ')
+		++where;
+
+	for (i = 0; unsigned_ops[i]; ++i) {
+		if (strncmp(where, unsigned_ops[i],
+		    strlen(unsigned_ops[i])))
+			continue;
+		is_unsigned = 1;
+		break;
+	}
+	return (is_unsigned);
+}
+
+/*
+ * Print a numeric immediate into end of buf, maximum length buflen.
+ * The immediate may be an address or a displacement.  Mask is set
+ * for address size.  If the immediate is a "small negative", or
+ * if it's a negative displacement of any magnitude, print as -<absval>.
+ * Respect the "octal" flag.  "Small negative" is defined as "in the
+ * interval [NEG_LIMIT, 0)".
+ *
+ * Also, "isunsigned_op()" instructions never print negatives.
+ *
+ * Return whether we decided to print a negative value or not.
+ */
+
+#define	NEG_LIMIT	-255
+enum {IMM, DISP};
+enum {POS, TRY_NEG};
+
+static int
+print_imm(dis86_t *dis, uint64_t usv, uint64_t mask, char *buf,
+    size_t buflen, int disp, int try_neg)
+{
+	int curlen;
+	int64_t sv = (int64_t)usv;
+	int octal = dis->d86_flags & DIS_F_OCTAL;
+
+	curlen = strlen(buf);
+
+	if (try_neg == TRY_NEG && sv < 0 &&
+	    (disp || sv >= NEG_LIMIT) &&
+	    !isunsigned_op(dis->d86_mnem)) {
+		dis->d86_sprintf_func(buf + curlen, buflen - curlen,
+		    octal ? "-0%llo" : "-0x%llx", (-sv) & mask);
+		return (1);
+	} else {
+		if (disp == DISP)
+			dis->d86_sprintf_func(buf + curlen, buflen - curlen,
+			    octal ? "+0%llo" : "+0x%llx", usv & mask);
+		else
+			dis->d86_sprintf_func(buf + curlen, buflen - curlen,
+			    octal ? "0%llo" : "0x%llx", usv & mask);
+		return (0);
+
+	}
+}
+
+
+static int
+log2(int size)
+{
+	switch (size) {
+	case 1: return (0);
+	case 2: return (1);
+	case 4: return (2);
+	case 8: return (3);
+	}
+	return (0);
+}
+
+/* ARGSUSED */
+void
+dtrace_disx86_str(dis86_t *dis, uint_t mode, uint64_t pc, char *buf,
+    size_t buflen)
+{
+	uint64_t reltgt = 0;
+	uint64_t tgt = 0;
+	int curlen;
+	int (*lookup)(void *, uint64_t, char *, size_t);
+	int i;
+	int64_t sv;
+	uint64_t usv, mask, save_mask, save_usv;
+	static uint64_t masks[] =
+	    {0xffU, 0xffffU, 0xffffffffU, 0xffffffffffffffffULL};
+	save_usv = 0;
+
+	dis->d86_sprintf_func(buf, buflen, "%-6s ", dis->d86_mnem);
+
+	/*
+	 * For PC-relative jumps, the pc is really the next pc after executing
+	 * this instruction, so increment it appropriately.
+	 */
+	pc += dis->d86_len;
+
+	for (i = 0; i < dis->d86_numopnds; i++) {
+		d86opnd_t *op = &dis->d86_opnd[i];
+
+		if (i != 0)
+			(void) strlcat(buf, ",", buflen);
+
+		(void) strlcat(buf, op->d86_prefix, buflen);
+
+		/*
+		 * sv is for the signed, possibly-truncated immediate or
+		 * displacement; usv retains the original size and
+		 * unsignedness for symbol lookup.
+		 */
+
+		sv = usv = op->d86_value;
+
+		/*
+		 * About masks: for immediates that represent
+		 * addresses, the appropriate display size is
+		 * the effective address size of the instruction.
+		 * This includes MODE_OFFSET, MODE_IPREL, and
+		 * MODE_RIPREL.  Immediates that are simply
+		 * immediate values should display in the operand's
+		 * size, however, since they don't represent addresses.
+		 */
+
+		/* d86_addr_size is SIZEnn, which is log2(real size) */
+		mask = masks[dis->d86_addr_size];
+
+		/* d86_value_size and d86_imm_bytes are in bytes */
+		if (op->d86_mode == MODE_SIGNED ||
+		    op->d86_mode == MODE_IMPLIED)
+			mask = masks[log2(op->d86_value_size)];
+
+		switch (op->d86_mode) {
+
+		case MODE_NONE:
+
+			(void) strlcat(buf, op->d86_opnd, buflen);
+			break;
+
+		case MODE_SIGNED:
+		case MODE_IMPLIED:
+		case MODE_OFFSET:
+
+			tgt = usv;
+
+			if (dis->d86_seg_prefix)
+				(void) strlcat(buf, dis->d86_seg_prefix,
+				    buflen);
+
+			if (op->d86_mode == MODE_SIGNED ||
+			    op->d86_mode == MODE_IMPLIED) {
+				(void) strlcat(buf, "$", buflen);
+			}
+
+			if (print_imm(dis, usv, mask, buf, buflen,
+			    IMM, TRY_NEG) &&
+			    (op->d86_mode == MODE_SIGNED ||
+			    op->d86_mode == MODE_IMPLIED)) {
+
+				/*
+				 * We printed a negative value for an
+				 * immediate that wasn't a
+				 * displacement.  Note that fact so we can
+				 * print the positive value as an
+				 * annotation.
+				 */
+
+				save_usv = usv;
+				save_mask = mask;
+			}
+			(void) strlcat(buf, op->d86_opnd, buflen);
+
+			break;
+
+		case MODE_IPREL:
+		case MODE_RIPREL:
+
+			reltgt = pc + sv;
+
+			switch (mode) {
+			case SIZE16:
+				reltgt = (uint16_t)reltgt;
+				break;
+			case SIZE32:
+				reltgt = (uint32_t)reltgt;
+				break;
+			}
+
+			(void) print_imm(dis, usv, mask, buf, buflen,
+			    DISP, TRY_NEG);
+
+			if (op->d86_mode == MODE_RIPREL)
+				(void) strlcat(buf, "(%rip)", buflen);
+			break;
+		}
+	}
+
+	/*
+	 * The symbol lookups may result in false positives,
+	 * particularly on object files, where small numbers may match
+	 * the 0-relative non-relocated addresses of symbols.
+	 */
+
+	lookup = dis->d86_sym_lookup;
+	if (tgt != 0) {
+		/* Print symbol, if found, for tgt */
+		if (lookup(dis->d86_data, tgt, NULL, 0) == 0) {
+			(void) strlcat(buf, "\t<", buflen);
+			curlen = strlen(buf);
+			lookup(dis->d86_data, tgt, buf + curlen,
+			    buflen - curlen);
+			(void) strlcat(buf, ">", buflen);
+		}
+
+		/*
+		 * If we printed a negative immediate above, print the
+		 * positive in case our heuristic was unhelpful
+		 */
+		if (save_usv) {
+			(void) strlcat(buf, "\t<", buflen);
+			(void) print_imm(dis, save_usv, save_mask, buf, buflen,
+			    IMM, POS);
+			(void) strlcat(buf, ">", buflen);
+		}
+	}
+
+	if (reltgt != 0) {
+		/* Print symbol or effective address for reltgt */
+
+		(void) strlcat(buf, "\t<", buflen);
+		curlen = strlen(buf);
+		lookup(dis->d86_data, reltgt, buf + curlen,
+		    buflen - curlen);
+		(void) strlcat(buf, ">", buflen);
+	}
+}
+
+#endif /* DIS_TEXT */
diff --git a/bsd/dev/i386/dtrace_isa.c b/bsd/dev/i386/dtrace_isa.c
new file mode 100644
index 000000000..304532c6b
--- /dev/null
+++ b/bsd/dev/i386/dtrace_isa.c
@@ -0,0 +1,649 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+
+typedef x86_saved_state_t savearea_t;
+
+#include <stdarg.h>
+#include <string.h>
+#include <sys/malloc.h>
+#include <sys/time.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
+#include <kern/task.h>
+#include <kern/sched_prim.h>
+#include <miscfs/devfs/devfs.h>
+#include <mach/vm_param.h>
+
+extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
+
+void
+dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
+    int fault, int fltoffs, uint64_t illval)
+{
+    /*
+     * For the case of the error probe firing lets
+     * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
+     */
+    state->dts_arg_error_illval = illval;
+    dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fault, fltoffs );
+}
+
+/*
+ * Atomicity and synchronization
+ */
+void
+dtrace_membar_producer(void)
+{
+	__asm__ volatile("sfence");
+}
+
+void
+dtrace_membar_consumer(void)
+{
+	__asm__ volatile("lfence");
+}
+
+/*
+ * Interrupt manipulation
+ * XXX dtrace_getipl() can be called from probe context.
+ */
+int
+dtrace_getipl(void)
+{
+	/*
+	 * XXX Drat, get_interrupt_level is MACH_KERNEL_PRIVATE
+	 * in osfmk/kern/cpu_data.h
+	 */
+	/* return get_interrupt_level(); */
+	return (ml_at_interrupt_context() ? 1: 0);
+}
+
+/*
+ * MP coordination
+ */
+
+extern void mp_broadcast(
+       void (*action_func)(void *),
+       void *arg);
+
+typedef struct xcArg {
+	processorid_t cpu;
+	dtrace_xcall_t f;
+	void *arg;
+} xcArg_t;
+
+static void
+xcRemote( void *foo )
+{
+	xcArg_t *pArg = (xcArg_t *)foo;
+	
+	if ( pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL ) {
+		(pArg->f)(pArg->arg);
+	}
+}
+
+/*
+ * dtrace_xcall() is not called from probe context.
+ */
+void
+dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
+{
+	xcArg_t xcArg;
+	
+	xcArg.cpu = cpu;
+	xcArg.f = f;
+	xcArg.arg = arg;
+
+	mp_broadcast( xcRemote, (void *)&xcArg);
+}
+
+/*
+ * Runtime and ABI
+ */
+extern greg_t
+dtrace_getfp(void)
+{
+	return (greg_t)__builtin_frame_address(0);
+}
+
+uint64_t
+dtrace_getreg(struct regs *savearea, uint_t reg)
+{
+	boolean_t is64Bit = proc_is64bit(current_proc());
+	x86_saved_state_t *regs = (x86_saved_state_t *)savearea;
+	
+	if (is64Bit) {
+		/* beyond register SS */
+		if (reg > x86_SAVED_STATE64_COUNT - 1) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+			return (0);
+		}
+		return ((uint64_t *)(&(regs->ss_64.gs)))[reg];
+	} else {
+		/* beyond register SS */
+		if (reg > x86_SAVED_STATE32_COUNT - 1) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+			return (0);
+		}
+		return (uint64_t)((unsigned int *)(&(regs->ss_32.gs)))[reg];
+	}
+	
+}
+
+#define RETURN_OFFSET 4
+#define RETURN_OFFSET64 8
+
+static int
+dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, user_addr_t pc,
+    user_addr_t sp)
+{
+#if 0
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+
+	uintptr_t oldcontext = lwp->lwp_oldcontext; /* XXX signal stack crawl */
+	size_t s1, s2;
+#endif
+	int ret = 0;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	ASSERT(pcstack == NULL || pcstack_limit > 0);
+	
+#if 0 /* XXX signal stack crawl */
+	if (p->p_model == DATAMODEL_NATIVE) {
+		s1 = sizeof (struct frame) + 2 * sizeof (long);
+		s2 = s1 + sizeof (siginfo_t);
+	} else {
+		s1 = sizeof (struct frame32) + 3 * sizeof (int);
+		s2 = s1 + sizeof (siginfo32_t);
+	}
+#endif
+
+	while (pc != 0) {
+		ret++;
+		if (pcstack != NULL) {
+			*pcstack++ = (uint64_t)pc;
+			pcstack_limit--;
+			if (pcstack_limit <= 0)
+				break;
+		}
+
+		if (sp == 0)
+			break;
+
+#if 0 /* XXX signal stack crawl */
+		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
+			if (p->p_model == DATAMODEL_NATIVE) {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fulword(&gregs[REG_FP]);
+				pc = dtrace_fulword(&gregs[REG_PC]);
+
+				oldcontext = dtrace_fulword(&ucp->uc_link);
+			} else {
+				ucontext32_t *ucp = (ucontext32_t *)oldcontext;
+				greg32_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fuword32(&gregs[EBP]);
+				pc = dtrace_fuword32(&gregs[EIP]);
+
+				oldcontext = dtrace_fuword32(&ucp->uc_link);
+			}
+		} 
+		else
+#endif
+		{
+			if (is64Bit) {
+				pc = dtrace_fuword64((sp + RETURN_OFFSET64));
+				sp = dtrace_fuword64(sp);
+			} else {
+				pc = dtrace_fuword32((sp + RETURN_OFFSET));
+				sp = dtrace_fuword32(sp);
+			}
+		}
+
+#if 0 /* XXX */
+		/*
+		 * This is totally bogus:  if we faulted, we're going to clear
+		 * the fault and break.  This is to deal with the apparently
+		 * broken Java stacks on x86.
+		 */
+		if (*flags & CPU_DTRACE_FAULT) {
+			*flags &= ~CPU_DTRACE_FAULT;
+			break;
+		}
+#endif
+	}
+
+	return (ret);
+}
+
+void
+dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
+{
+	thread_t thread = current_thread();
+	x86_saved_state_t *regs;
+	user_addr_t pc, sp, fp;
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+	int n;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (*flags & CPU_DTRACE_FAULT)
+		return;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	/*
+	 * If there's no user context we still need to zero the stack.
+	 */
+	if (thread == NULL)
+		goto zero;
+
+	regs = (x86_saved_state_t *)find_user_regs(thread);
+	if (regs == NULL)
+		goto zero;
+		
+	*pcstack++ = (uint64_t)proc_selfpid();
+	pcstack_limit--;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	if (is64Bit) {
+		pc = regs->ss_64.isf.rip;
+		sp = regs->ss_64.isf.rsp;
+		fp = regs->ss_64.rbp;
+	} else {
+		pc = regs->ss_32.eip;
+		sp = regs->ss_32.uesp;
+		fp = regs->ss_32.ebp;
+	}
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		*pcstack++ = (uint64_t)pc;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			return;
+
+		if (is64Bit)
+			pc = dtrace_fuword64(sp);
+		else
+			pc = dtrace_fuword32(sp);
+	}
+
+	/*
+	 * Note that unlike ppc, the x86 code does not use
+	 * CPU_DTRACE_USTACK_FP. This is because x86 always
+	 * traces from the fp, even in syscall/profile/fbt
+	 * providers.
+	 */
+	n = dtrace_getustack_common(pcstack, pcstack_limit, pc, fp);
+	ASSERT(n >= 0);
+	ASSERT(n <= pcstack_limit);
+
+	pcstack += n;
+	pcstack_limit -= n;
+
+zero:
+	while (pcstack_limit-- > 0)
+		*pcstack++ = 0;
+}
+
+int
+dtrace_getustackdepth(void)
+{
+	thread_t thread = current_thread();
+	x86_saved_state_t *regs;
+	user_addr_t pc, sp, fp;
+	int n = 0;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (thread == NULL)
+		return 0;
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
+		return (-1);
+
+	regs = (x86_saved_state_t *)find_user_regs(thread);
+	if (regs == NULL)
+		return 0;
+
+	if (is64Bit) {
+		pc = regs->ss_64.isf.rip;
+		sp = regs->ss_64.isf.rsp;
+		fp = regs->ss_64.rbp;
+	} else {
+		pc = regs->ss_32.eip;
+		sp = regs->ss_32.uesp;
+		fp = regs->ss_32.ebp;
+	}
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		n++;
+
+		if (is64Bit)
+			pc = dtrace_fuword64(sp);
+		else
+			pc = dtrace_fuword32(sp);
+	}
+
+	/*
+	 * Note that unlike ppc, the x86 code does not use
+	 * CPU_DTRACE_USTACK_FP. This is because x86 always
+	 * traces from the fp, even in syscall/profile/fbt
+	 * providers.
+	 */
+
+	n += dtrace_getustack_common(NULL, 0, pc, fp);
+
+	return (n);
+}
+
+void
+dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
+{
+	thread_t thread = current_thread();
+	savearea_t *regs;
+	user_addr_t pc, sp;
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+#if 0
+	uintptr_t oldcontext;
+	size_t s1, s2;
+#endif
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (*flags & CPU_DTRACE_FAULT)
+		return;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	/*
+	 * If there's no user context we still need to zero the stack.
+	 */
+	if (thread == NULL)
+		goto zero;
+
+	regs = (savearea_t *)find_user_regs(thread);
+	if (regs == NULL)
+		goto zero;
+		
+	*pcstack++ = (uint64_t)proc_selfpid();
+	pcstack_limit--;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	pc = regs->ss_32.eip;
+	sp = regs->ss_32.ebp;
+	
+#if 0 /* XXX signal stack crawl */
+	oldcontext = lwp->lwp_oldcontext;
+
+	if (p->p_model == DATAMODEL_NATIVE) {
+		s1 = sizeof (struct frame) + 2 * sizeof (long);
+		s2 = s1 + sizeof (siginfo_t);
+	} else {
+		s1 = sizeof (struct frame32) + 3 * sizeof (int);
+		s2 = s1 + sizeof (siginfo32_t);
+	}
+#endif
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		*pcstack++ = (uint64_t)pc;
+		*fpstack++ = 0;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			return;
+
+		if (is64Bit)
+			pc = dtrace_fuword64(sp);
+		else
+			pc = dtrace_fuword32(sp);
+	}
+
+	while (pc != 0) {
+		*pcstack++ = (uint64_t)pc;
+		*fpstack++ = sp;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			break;
+
+		if (sp == 0)
+			break;
+
+#if 0 /* XXX signal stack crawl */
+		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
+			if (p->p_model == DATAMODEL_NATIVE) {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fulword(&gregs[REG_FP]);
+				pc = dtrace_fulword(&gregs[REG_PC]);
+
+				oldcontext = dtrace_fulword(&ucp->uc_link);
+			} else {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fuword32(&gregs[EBP]);
+				pc = dtrace_fuword32(&gregs[EIP]);
+
+				oldcontext = dtrace_fuword32(&ucp->uc_link);
+			}
+		} 
+		else
+#endif
+		{
+			if (is64Bit) {
+				pc = dtrace_fuword64((sp + RETURN_OFFSET64));
+				sp = dtrace_fuword64(sp);
+			} else {
+				pc = dtrace_fuword32((sp + RETURN_OFFSET));
+				sp = dtrace_fuword32(sp);
+			}
+		}
+
+#if 0 /* XXX */
+		/*
+		 * This is totally bogus:  if we faulted, we're going to clear
+		 * the fault and break.  This is to deal with the apparently
+		 * broken Java stacks on x86.
+		 */
+		if (*flags & CPU_DTRACE_FAULT) {
+			*flags &= ~CPU_DTRACE_FAULT;
+			break;
+		}
+#endif
+	}
+
+zero:
+	while (pcstack_limit-- > 0)
+		*pcstack++ = 0;
+}
+
+void
+dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes,
+		  uint32_t *intrpc)
+{
+	struct frame *fp = (struct frame *)dtrace_getfp();
+	struct frame *nextfp, *minfp, *stacktop;
+	int depth = 0;
+	int last = 0;
+	uintptr_t pc;
+	uintptr_t caller = CPU->cpu_dtrace_caller;
+	int on_intr;
+
+	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
+		stacktop = (struct frame *)dtrace_get_cpu_int_stack_top();
+	else
+		stacktop = (struct frame *)(dtrace_get_kernel_stack(current_thread()) + KERNEL_STACK_SIZE);
+
+	minfp = fp;
+
+	aframes++;
+
+	if (intrpc != NULL && depth < pcstack_limit)
+		pcstack[depth++] = (pc_t)intrpc;
+
+	while (depth < pcstack_limit) {
+		nextfp = *(struct frame **)fp;
+		pc = *(uintptr_t *)(((uint32_t)fp) + RETURN_OFFSET);
+
+		if (nextfp <= minfp || nextfp >= stacktop) {
+			if (on_intr) {
+				/*
+				 * Hop from interrupt stack to thread stack.
+				 */
+				vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
+
+				minfp = (struct frame *)kstack_base;
+				stacktop = (struct frame *)(kstack_base + KERNEL_STACK_SIZE);
+
+				on_intr = 0;
+				continue;
+			}
+			/*
+			 * This is the last frame we can process; indicate
+			 * that we should return after processing this frame.
+			 */
+			last = 1;
+		}
+
+		if (aframes > 0) {
+			if (--aframes == 0 && caller != 0) {
+				/*
+				 * We've just run out of artificial frames,
+				 * and we have a valid caller -- fill it in
+				 * now.
+				 */
+				ASSERT(depth < pcstack_limit);
+				pcstack[depth++] = (pc_t)caller;
+				caller = 0;
+			}
+		} else {
+			if (depth < pcstack_limit)
+				pcstack[depth++] = (pc_t)pc;
+		}
+
+		if (last) {
+			while (depth < pcstack_limit)
+				pcstack[depth++] = 0;
+			return;
+		}
+
+		fp = nextfp;
+		minfp = fp;
+	}
+}
+
+struct frame {
+	struct frame *backchain;
+	uintptr_t retaddr;
+};
+
+uint64_t
+dtrace_getarg(int arg, int aframes)
+{
+	uint64_t val;
+	struct frame *fp = (struct frame *)dtrace_getfp();
+	uintptr_t *stack;
+	uintptr_t pc;
+	int i;
+
+	for (i = 1; i <= aframes; i++) {
+		fp = fp->backchain;
+		pc = fp->retaddr;
+
+		if (pc  == (uintptr_t)dtrace_invop_callsite) {
+			/*
+			 * If we pass through the invalid op handler, we will
+			 * use the pointer that it passed to the stack as the
+			 * second argument to dtrace_invop() as the pointer to
+			 * the frame we're hunting for.
+			 */
+
+			stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
+			fp = (struct frame *)stack[1]; /* Grab *second* argument */
+			stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+			val = (uint64_t)(stack[arg]);
+			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+			return val;
+		}
+	}
+
+	/*
+	 * Arrive here when provider has called dtrace_probe directly.
+	 */
+	stack = (uintptr_t *)&fp[1]; /* Find marshalled arguments */
+	stack++; /* Advance past probeID */
+
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	val = *(((uint64_t *)stack) + arg); /* dtrace_probe arguments arg0 .. arg4 are 64bits wide */
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return (val);
+}
+
+/*
+ * Load/Store Safety
+ */
+void
+dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
+{
+	/*
+	 * "base" is the smallest toxic address in the range, "limit" is the first
+	 * VALID address greater than "base".
+	 */
+	func(0x0, VM_MIN_KERNEL_ADDRESS);
+	func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
+}
+
+extern boolean_t pmap_valid_page(ppnum_t pn);
+
+boolean_t
+dtxnu_is_RAM_page(ppnum_t pn)
+{
+	return pmap_valid_page(pn);
+}
+
diff --git a/bsd/dev/i386/dtrace_subr_x86.c b/bsd/dev/i386/dtrace_subr_x86.c
new file mode 100644
index 000000000..ae29f8416
--- /dev/null
+++ b/bsd/dev/i386/dtrace_subr_x86.c
@@ -0,0 +1,308 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)dtrace_subr.c	1.13	06/06/12 SMI"
+ */
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_glue.h>
+#include <sys/dtrace_impl.h>
+#include <sys/fasttrap.h>
+#include <sys/vm.h>
+#include <sys/user.h>
+#include <sys/kauth.h>
+#include <kern/debug.h>
+
+int (*dtrace_pid_probe_ptr)(x86_saved_state_t *);
+int (*dtrace_return_probe_ptr)(x86_saved_state_t *);
+
+/*
+ * HACK! There doesn't seem to be an easy way to include trap.h from
+ * here. FIXME!
+ */
+#define	T_INT3			3		/* int 3 instruction */
+#define T_DTRACE_RET		0x7f		/* DTrace pid return */
+
+kern_return_t
+dtrace_user_probe(x86_saved_state_t *);
+
+kern_return_t
+dtrace_user_probe(x86_saved_state_t *regs)
+{
+	x86_saved_state64_t *regs64;
+	x86_saved_state32_t *regs32;
+        int trapno;
+
+	/*
+	 * FIXME!
+	 *
+	 * The only call path into this method is always a user trap.
+	 * We don't need to test for user trap, but should assert it.
+	 */
+	boolean_t user_mode = TRUE;
+
+        if (is_saved_state64(regs) == TRUE) {
+                regs64 = saved_state64(regs);
+		regs32 = NULL;
+                trapno = regs64->isf.trapno;
+                user_mode = TRUE; // By default, because xnu is 32 bit only
+        } else {
+		regs64 = NULL;
+                regs32 = saved_state32(regs);
+                if (regs32->cs & 0x03) user_mode = TRUE;
+                trapno = regs32->trapno;
+        }
+
+	lck_rw_t *rwp;
+	struct proc *p = current_proc();
+
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+	if (user_mode /*|| (rp->r_ps & PS_VM)*/) {
+		/*
+		 * DTrace accesses t_cred in probe context.  t_cred
+		 * must always be either NULL, or point to a valid,
+		 * allocated cred structure.
+		 */
+		kauth_cred_uthread_update(uthread, p);
+	}
+
+	if (trapno == T_DTRACE_RET) {
+		uint8_t step = uthread->t_dtrace_step;
+		uint8_t ret = uthread->t_dtrace_ret;
+		user_addr_t npc = uthread->t_dtrace_npc;
+
+		if (uthread->t_dtrace_ast) {
+			printf("dtrace_user_probe() should be calling aston()\n");
+			// aston(uthread);
+			// uthread->t_sig_check = 1;
+		}
+
+		/*
+		 * Clear all user tracing flags.
+		 */
+		uthread->t_dtrace_ft = 0;
+
+		/*
+		 * If we weren't expecting to take a return probe trap, kill
+		 * the process as though it had just executed an unassigned
+		 * trap instruction.
+		 */
+		if (step == 0) {
+			/*
+			 * APPLE NOTE: We're returning KERN_FAILURE, which causes 
+			 * the generic signal handling code to take over, which will effectively
+			 * deliver a EXC_BAD_INSTRUCTION to the user process.
+			 */
+	 		return KERN_FAILURE;
+		}
+
+		/*
+		 * If we hit this trap unrelated to a return probe, we're
+		 * just here to reset the AST flag since we deferred a signal
+		 * until after we logically single-stepped the instruction we
+		 * copied out.
+		 */
+		if (ret == 0) {
+			if (regs64) {
+				regs64->isf.rip = npc;
+			} else {
+				regs32->eip = npc;
+			}
+			return KERN_SUCCESS;
+		}
+
+		/*
+		 * We need to wait until after we've called the
+		 * dtrace_return_probe_ptr function pointer to set %pc.
+		 */
+		rwp = &CPU->cpu_ft_lock;
+		lck_rw_lock_shared(rwp);
+
+		if (dtrace_return_probe_ptr != NULL)
+			(void) (*dtrace_return_probe_ptr)(regs);
+		lck_rw_unlock_shared(rwp);
+
+		if (regs64) {
+			regs64->isf.rip = npc;
+		} else {
+			regs32->eip = npc;
+		}
+
+		return KERN_SUCCESS;
+	} else if (trapno == T_INT3) {
+		uint8_t instr;
+		rwp = &CPU->cpu_ft_lock;
+
+		/*
+		 * The DTrace fasttrap provider uses the breakpoint trap
+		 * (int 3). We let DTrace take the first crack at handling
+		 * this trap; if it's not a probe that DTrace knowns about,
+		 * we call into the trap() routine to handle it like a
+		 * breakpoint placed by a conventional debugger.
+		 */
+
+		/*
+		 * APPLE NOTE: I believe the purpose of the reader/writers lock
+		 * is thus: There are times which dtrace needs to prevent calling
+		 * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain
+		 * mutex here. However, that serialized all probe calls, and
+		 * destroyed MP behavior. So now they use a RW lock, with probes
+		 * as readers, and the top level synchronization as a writer.
+		 */
+		lck_rw_lock_shared(rwp);
+		if (dtrace_pid_probe_ptr != NULL && 
+		    (*dtrace_pid_probe_ptr)(regs) == 0) {
+			lck_rw_unlock_shared(rwp);
+			return KERN_SUCCESS;
+		}
+		lck_rw_unlock_shared(rwp);
+
+
+		/*
+		 * If the instruction that caused the breakpoint trap doesn't
+		 * look like an int 3 anymore, it may be that this tracepoint
+		 * was removed just after the user thread executed it. In
+		 * that case, return to user land to retry the instuction.
+		 */
+		user_addr_t pc = (regs64) ? regs64->isf.rip : (user_addr_t)regs32->eip;
+		if (fuword8(pc - 1, &instr) == 0 && instr != FASTTRAP_INSTR) {
+			if (regs64) {
+				regs64->isf.rip--;
+			} else {
+				regs32->eip--;
+			}
+			return KERN_SUCCESS;
+		}
+
+	}
+
+	return KERN_FAILURE;
+}
+
+void
+dtrace_safe_synchronous_signal(void)
+{
+#if 0
+	kthread_t *t = curthread;
+	struct regs *rp = lwptoregs(ttolwp(t));
+	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
+
+	ASSERT(t->t_dtrace_on);
+
+	/*
+	 * If we're not in the range of scratch addresses, we're not actually
+	 * tracing user instructions so turn off the flags. If the instruction
+	 * we copied out caused a synchonous trap, reset the pc back to its
+	 * original value and turn off the flags.
+	 */
+	if (rp->r_pc < t->t_dtrace_scrpc ||
+			rp->r_pc > t->t_dtrace_astpc + isz) {
+		t->t_dtrace_ft = 0;
+	} else if (rp->r_pc == t->t_dtrace_scrpc ||
+			rp->r_pc == t->t_dtrace_astpc) {
+		rp->r_pc = t->t_dtrace_pc;
+		t->t_dtrace_ft = 0;
+	}
+#endif /* 0 */
+}
+
+int
+dtrace_safe_defer_signal(void)
+{
+#if 0
+	kthread_t *t = curthread;
+	struct regs *rp = lwptoregs(ttolwp(t));
+	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
+
+	ASSERT(t->t_dtrace_on);
+
+	/*
+	 * If we're not in the range of scratch addresses, we're not actually
+	 * tracing user instructions so turn off the flags.
+	 */
+	if (rp->r_pc < t->t_dtrace_scrpc ||
+			rp->r_pc > t->t_dtrace_astpc + isz) {
+		t->t_dtrace_ft = 0;
+		return (0);
+	}
+
+	/*
+	 * If we've executed the original instruction, but haven't performed
+	 * the jmp back to t->t_dtrace_npc or the clean up of any registers
+	 * used to emulate %rip-relative instructions in 64-bit mode, do that
+	 * here and take the signal right away. We detect this condition by
+	 * seeing if the program counter is the range [scrpc + isz, astpc).
+	 */
+	if (t->t_dtrace_astpc - rp->r_pc <
+			t->t_dtrace_astpc - t->t_dtrace_scrpc - isz) {
+#ifdef __sol64
+		/*
+		 * If there is a scratch register and we're on the
+		 * instruction immediately after the modified instruction,
+		 * restore the value of that scratch register.
+		 */
+		if (t->t_dtrace_reg != 0 &&
+				rp->r_pc == t->t_dtrace_scrpc + isz) {
+			switch (t->t_dtrace_reg) {
+				case REG_RAX:
+					rp->r_rax = t->t_dtrace_regv;
+					break;
+				case REG_RCX:
+					rp->r_rcx = t->t_dtrace_regv;
+					break;
+				case REG_R8:
+					rp->r_r8 = t->t_dtrace_regv;
+					break;
+				case REG_R9:
+					rp->r_r9 = t->t_dtrace_regv;
+					break;
+			}
+		}
+#endif
+		rp->r_pc = t->t_dtrace_npc;
+		t->t_dtrace_ft = 0;
+		return (0);
+	}
+
+	/*
+	 * Otherwise, make sure we'll return to the kernel after executing
+	 * the copied out instruction and defer the signal.
+	 */
+	if (!t->t_dtrace_step) {
+		ASSERT(rp->r_pc < t->t_dtrace_astpc);
+		rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc;
+		t->t_dtrace_step = 1;
+	}
+
+	t->t_dtrace_ast = 1;
+
+	return (1);
+
+#endif /* 0 */
+
+	return 0;
+}
diff --git a/bsd/dev/i386/fasttrap_isa.c b/bsd/dev/i386/fasttrap_isa.c
new file mode 100644
index 000000000..478429efc
--- /dev/null
+++ b/bsd/dev/i386/fasttrap_isa.c
@@ -0,0 +1,2237 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)fasttrap_isa.c	1.23	06/09/19 SMI"
+ */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#include <sys/fasttrap_isa.h>
+#include <sys/fasttrap_impl.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include "fasttrap_regset.h"
+
+#include <sys/dtrace_ptss.h>
+#include <kern/debug.h>
+
+#define proc_t struct proc
+
+/*
+ * Lossless User-Land Tracing on x86
+ * ---------------------------------
+ *
+ * The execution of most instructions is not dependent on the address; for
+ * these instructions it is sufficient to copy them into the user process's
+ * address space and execute them. To effectively single-step an instruction
+ * in user-land, we copy out the following sequence of instructions to scratch
+ * space in the user thread's ulwp_t structure.
+ *
+ * We then set the program counter (%eip or %rip) to point to this scratch
+ * space. Once execution resumes, the original instruction is executed and
+ * then control flow is redirected to what was originally the subsequent
+ * instruction. If the kernel attemps to deliver a signal while single-
+ * stepping, the signal is deferred and the program counter is moved into the
+ * second sequence of instructions. The second sequence ends in a trap into
+ * the kernel where the deferred signal is then properly handled and delivered.
+ *
+ * For instructions whose execute is position dependent, we perform simple
+ * emulation. These instructions are limited to control transfer
+ * instructions in 32-bit mode, but in 64-bit mode there's the added wrinkle
+ * of %rip-relative addressing that means that almost any instruction can be
+ * position dependent. For all the details on how we emulate generic
+ * instructions included %rip-relative instructions, see the code in
+ * fasttrap_pid_probe() below where we handle instructions of type
+ * FASTTRAP_T_COMMON (under the header: Generic Instruction Tracing).
+ */
+
+#define	FASTTRAP_MODRM_MOD(modrm)	(((modrm) >> 6) & 0x3)
+#define	FASTTRAP_MODRM_REG(modrm)	(((modrm) >> 3) & 0x7)
+#define	FASTTRAP_MODRM_RM(modrm)	((modrm) & 0x7)
+#define	FASTTRAP_MODRM(mod, reg, rm)	(((mod) << 6) | ((reg) << 3) | (rm))
+
+#define	FASTTRAP_SIB_SCALE(sib)		(((sib) >> 6) & 0x3)
+#define	FASTTRAP_SIB_INDEX(sib)		(((sib) >> 3) & 0x7)
+#define	FASTTRAP_SIB_BASE(sib)		((sib) & 0x7)
+
+#define	FASTTRAP_REX_W(rex)		(((rex) >> 3) & 1)
+#define	FASTTRAP_REX_R(rex)		(((rex) >> 2) & 1)
+#define	FASTTRAP_REX_X(rex)		(((rex) >> 1) & 1)
+#define	FASTTRAP_REX_B(rex)		((rex) & 1)
+#define	FASTTRAP_REX(w, r, x, b)	\
+	(0x40 | ((w) << 3) | ((r) << 2) | ((x) << 1) | (b))
+
+/*
+ * Single-byte op-codes.
+ */
+#define	FASTTRAP_PUSHL_EBP	0x55
+
+#define	FASTTRAP_JO		0x70
+#define	FASTTRAP_JNO		0x71
+#define	FASTTRAP_JB		0x72
+#define	FASTTRAP_JAE		0x73
+#define	FASTTRAP_JE		0x74
+#define	FASTTRAP_JNE		0x75
+#define	FASTTRAP_JBE		0x76
+#define	FASTTRAP_JA		0x77
+#define	FASTTRAP_JS		0x78
+#define	FASTTRAP_JNS		0x79
+#define	FASTTRAP_JP		0x7a
+#define	FASTTRAP_JNP		0x7b
+#define	FASTTRAP_JL		0x7c
+#define	FASTTRAP_JGE		0x7d
+#define	FASTTRAP_JLE		0x7e
+#define	FASTTRAP_JG		0x7f
+
+#define	FASTTRAP_NOP		0x90
+
+#define	FASTTRAP_MOV_EAX	0xb8
+#define	FASTTRAP_MOV_ECX	0xb9
+
+#define	FASTTRAP_RET16		0xc2
+#define	FASTTRAP_RET		0xc3
+
+#define	FASTTRAP_LOOPNZ		0xe0
+#define	FASTTRAP_LOOPZ		0xe1
+#define	FASTTRAP_LOOP		0xe2
+#define	FASTTRAP_JCXZ		0xe3
+
+#define	FASTTRAP_CALL		0xe8
+#define	FASTTRAP_JMP32		0xe9
+#define	FASTTRAP_JMP8		0xeb
+
+#define	FASTTRAP_INT3		0xcc
+#define	FASTTRAP_INT		0xcd
+#define	T_DTRACE_RET		0x7f
+
+#define	FASTTRAP_2_BYTE_OP	0x0f
+#define	FASTTRAP_GROUP5_OP	0xff
+
+/*
+ * Two-byte op-codes (second byte only).
+ */
+#define	FASTTRAP_0F_JO		0x80
+#define	FASTTRAP_0F_JNO		0x81
+#define	FASTTRAP_0F_JB		0x82
+#define	FASTTRAP_0F_JAE		0x83
+#define	FASTTRAP_0F_JE		0x84
+#define	FASTTRAP_0F_JNE		0x85
+#define	FASTTRAP_0F_JBE		0x86
+#define	FASTTRAP_0F_JA		0x87
+#define	FASTTRAP_0F_JS		0x88
+#define	FASTTRAP_0F_JNS		0x89
+#define	FASTTRAP_0F_JP		0x8a
+#define	FASTTRAP_0F_JNP		0x8b
+#define	FASTTRAP_0F_JL		0x8c
+#define	FASTTRAP_0F_JGE		0x8d
+#define	FASTTRAP_0F_JLE		0x8e
+#define	FASTTRAP_0F_JG		0x8f
+
+#define	FASTTRAP_EFLAGS_OF	0x800
+#define	FASTTRAP_EFLAGS_DF	0x400
+#define	FASTTRAP_EFLAGS_SF	0x080
+#define	FASTTRAP_EFLAGS_ZF	0x040
+#define	FASTTRAP_EFLAGS_AF	0x010
+#define	FASTTRAP_EFLAGS_PF	0x004
+#define	FASTTRAP_EFLAGS_CF	0x001
+
+/*
+ * Instruction prefixes.
+ */
+#define	FASTTRAP_PREFIX_OPERAND	0x66
+#define	FASTTRAP_PREFIX_ADDRESS	0x67
+#define	FASTTRAP_PREFIX_CS	0x2E
+#define	FASTTRAP_PREFIX_DS	0x3E
+#define	FASTTRAP_PREFIX_ES	0x26
+#define	FASTTRAP_PREFIX_FS	0x64
+#define	FASTTRAP_PREFIX_GS	0x65
+#define	FASTTRAP_PREFIX_SS	0x36
+#define	FASTTRAP_PREFIX_LOCK	0xF0
+#define	FASTTRAP_PREFIX_REP	0xF3
+#define	FASTTRAP_PREFIX_REPNE	0xF2
+
+#define	FASTTRAP_NOREG	0xff
+
+/*
+ * Map between instruction register encodings and the kernel constants which
+ * correspond to indicies into struct regs.
+ */
+
+/*
+ * APPLE NOTE: We are cheating here. The regmap is used to decode which register
+ * a given instruction is trying to reference. OS X does not have extended registers
+ * for 32 bit apps, but the *order* is the same. So for 32 bit state, we will return:
+ *
+ * REG_RAX -> EAX
+ * REG_RCX -> ECX
+ * ...
+ * REG_RDI -> EDI
+ *
+ * The fasttrap_getreg function knows how to make the correct transformation.
+ */
+#if __sol64 || defined(__APPLE__)
+static const uint8_t regmap[16] = {
+	REG_RAX, REG_RCX, REG_RDX, REG_RBX, REG_RSP, REG_RBP, REG_RSI, REG_RDI,
+	REG_R8, REG_R9, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15,
+};
+#else
+static const uint8_t regmap[8] = {
+	EAX, ECX, EDX, EBX, UESP, EBP, ESI, EDI
+};
+#endif
+
+static user_addr_t fasttrap_getreg(x86_saved_state_t *, uint_t);
+
+static uint64_t
+fasttrap_anarg(x86_saved_state_t *regs, int function_entry, int argno)
+{
+	uint64_t value;
+	int shift = function_entry ? 1 : 0;
+
+	x86_saved_state64_t *regs64;
+	x86_saved_state32_t *regs32;
+	unsigned int p_model;
+
+        if (is_saved_state64(regs)) {
+                regs64 = saved_state64(regs);
+		regs32 = NULL;
+		p_model = DATAMODEL_LP64;
+        } else {
+		regs64 = NULL;
+                regs32 = saved_state32(regs);
+		p_model = DATAMODEL_ILP32;
+        }
+
+	if (p_model == DATAMODEL_LP64) {
+		user_addr_t stack;
+		
+		/*
+		 * In 64-bit mode, the first six arguments are stored in
+		 * registers.
+		 */
+		if (argno < 6)
+			return ((&regs64->rdi)[argno]);
+
+		stack = regs64->isf.rsp + sizeof(uint64_t) * (argno - 6 + shift);
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		value = dtrace_fuword64(stack);
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
+	} else {
+		uint32_t *stack = (uint32_t *)regs32->uesp;
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		value = dtrace_fuword32((user_addr_t)(unsigned long)&stack[argno + shift]);
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
+	}
+
+	return (value);
+}
+
+/*ARGSUSED*/
+int
+fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, user_addr_t pc,
+    fasttrap_probe_type_t type)
+{
+#pragma unused(type)
+	uint8_t instr[FASTTRAP_MAX_INSTR_SIZE + 10];
+	size_t len = FASTTRAP_MAX_INSTR_SIZE;
+	size_t first = MIN(len, PAGE_SIZE - (pc & PAGE_MASK));
+	uint_t start = 0;
+	size_t size;
+	int rmindex;
+	uint8_t seg, rex = 0;
+	unsigned int p_model = (p->p_flag & P_LP64) ? DATAMODEL_LP64 : DATAMODEL_ILP32;
+
+	/*
+	 * Read the instruction at the given address out of the process's
+	 * address space. We don't have to worry about a debugger
+	 * changing this instruction before we overwrite it with our trap
+	 * instruction since P_PR_LOCK is set. Since instructions can span
+	 * pages, we potentially read the instruction in two parts. If the
+	 * second part fails, we just zero out that part of the instruction.
+	 */
+	/*
+	 * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey...
+	 */
+	if (uread(p, &instr[0], first, pc) != 0)
+		return (-1);
+	if (len > first &&
+	    uread(p, &instr[first], len - first, pc + first) != 0) {
+		bzero(&instr[first], len - first);
+		len = first;
+	}
+
+	/*
+	 * If the disassembly fails, then we have a malformed instruction.
+	 */
+	if ((size = dtrace_instr_size_isa(instr, p_model, &rmindex)) <= 0)
+		return (-1);
+
+	/*
+	 * Make sure the disassembler isn't completely broken.
+	 */
+	ASSERT(-1 <= rmindex && rmindex < (int)size);
+
+	/*
+	 * If the computed size is greater than the number of bytes read,
+	 * then it was a malformed instruction possibly because it fell on a
+	 * page boundary and the subsequent page was missing or because of
+	 * some malicious user.
+	 */
+	if (size > len)
+		return (-1);
+
+	tp->ftt_size = (uint8_t)size;
+	tp->ftt_segment = FASTTRAP_SEG_NONE;
+
+	/*
+	 * Find the start of the instruction's opcode by processing any
+	 * legacy prefixes.
+	 */
+	for (;;) {
+		seg = 0;
+		switch (instr[start]) {
+		case FASTTRAP_PREFIX_SS:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_GS:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_FS:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_ES:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_DS:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_CS:
+			seg++;
+			/*FALLTHRU*/
+		case FASTTRAP_PREFIX_OPERAND:
+		case FASTTRAP_PREFIX_ADDRESS:
+		case FASTTRAP_PREFIX_LOCK:
+		case FASTTRAP_PREFIX_REP:
+		case FASTTRAP_PREFIX_REPNE:
+			if (seg != 0) {
+				/*
+				 * It's illegal for an instruction to specify
+				 * two segment prefixes -- give up on this
+				 * illegal instruction.
+				 */
+				if (tp->ftt_segment != FASTTRAP_SEG_NONE)
+					return (-1);
+
+				tp->ftt_segment = seg;
+			}
+			start++;
+			continue;
+		}
+		break;
+	}
+
+#if __sol64 || defined(__APPLE__)
+	/*
+	 * Identify the REX prefix on 64-bit processes.
+	 */
+	if (p_model == DATAMODEL_LP64 && (instr[start] & 0xf0) == 0x40)
+		rex = instr[start++];
+#endif
+
+	/*
+	 * Now that we're pretty sure that the instruction is okay, copy the
+	 * valid part to the tracepoint.
+	 */
+	bcopy(instr, tp->ftt_instr, FASTTRAP_MAX_INSTR_SIZE);
+
+	tp->ftt_type = FASTTRAP_T_COMMON;
+	if (instr[start] == FASTTRAP_2_BYTE_OP) {
+		switch (instr[start + 1]) {
+		case FASTTRAP_0F_JO:
+		case FASTTRAP_0F_JNO:
+		case FASTTRAP_0F_JB:
+		case FASTTRAP_0F_JAE:
+		case FASTTRAP_0F_JE:
+		case FASTTRAP_0F_JNE:
+		case FASTTRAP_0F_JBE:
+		case FASTTRAP_0F_JA:
+		case FASTTRAP_0F_JS:
+		case FASTTRAP_0F_JNS:
+		case FASTTRAP_0F_JP:
+		case FASTTRAP_0F_JNP:
+		case FASTTRAP_0F_JL:
+		case FASTTRAP_0F_JGE:
+		case FASTTRAP_0F_JLE:
+		case FASTTRAP_0F_JG:
+			tp->ftt_type = FASTTRAP_T_JCC;
+			tp->ftt_code = (instr[start + 1] & 0x0f) | FASTTRAP_JO;
+			tp->ftt_dest = pc + tp->ftt_size +
+			    *(int32_t *)&instr[start + 2];
+			break;
+		}
+	} else if (instr[start] == FASTTRAP_GROUP5_OP) {
+		uint_t mod = FASTTRAP_MODRM_MOD(instr[start + 1]);
+		uint_t reg = FASTTRAP_MODRM_REG(instr[start + 1]);
+		uint_t rm = FASTTRAP_MODRM_RM(instr[start + 1]);
+
+		if (reg == 2 || reg == 4) {
+			uint_t i, sz;
+
+			if (reg == 2)
+				tp->ftt_type = FASTTRAP_T_CALL;
+			else
+				tp->ftt_type = FASTTRAP_T_JMP;
+
+			if (mod == 3)
+				tp->ftt_code = 2;
+			else
+				tp->ftt_code = 1;
+
+			ASSERT(p_model == DATAMODEL_LP64 || rex == 0);
+
+			/*
+			 * See AMD x86-64 Architecture Programmer's Manual
+			 * Volume 3, Section 1.2.7, Table 1-12, and
+			 * Appendix A.3.1, Table A-15.
+			 */
+			if (mod != 3 && rm == 4) {
+				uint8_t sib = instr[start + 2];
+				uint_t index = FASTTRAP_SIB_INDEX(sib);
+				uint_t base = FASTTRAP_SIB_BASE(sib);
+
+				tp->ftt_scale = FASTTRAP_SIB_SCALE(sib);
+
+				tp->ftt_index = (index == 4) ?
+				    FASTTRAP_NOREG :
+				    regmap[index | (FASTTRAP_REX_X(rex) << 3)];
+				tp->ftt_base = (mod == 0 && base == 5) ?
+				    FASTTRAP_NOREG :
+				    regmap[base | (FASTTRAP_REX_B(rex) << 3)];
+
+				i = 3;
+				sz = mod == 1 ? 1 : 4;
+			} else {
+				/*
+				 * In 64-bit mode, mod == 0 and r/m == 5
+				 * denotes %rip-relative addressing; in 32-bit
+				 * mode, the base register isn't used. In both
+				 * modes, there is a 32-bit operand.
+				 */
+				if (mod == 0 && rm == 5) {
+#if __sol64 || defined(__APPLE__)
+					if (p_model == DATAMODEL_LP64)
+						tp->ftt_base = REG_RIP;
+					else
+#endif
+						tp->ftt_base = FASTTRAP_NOREG;
+					sz = 4;
+				} else  {
+					uint8_t base = rm |
+					    (FASTTRAP_REX_B(rex) << 3);
+
+					tp->ftt_base = regmap[base];
+					sz = mod == 1 ? 1 : mod == 2 ? 4 : 0;
+				}
+				tp->ftt_index = FASTTRAP_NOREG;
+				i = 2;
+			}
+
+			if (sz == 1)
+				tp->ftt_dest = *(int8_t *)&instr[start + i];
+			else if (sz == 4)
+				tp->ftt_dest = *(int32_t *)&instr[start + i];
+			else
+				tp->ftt_dest = 0;
+		}
+	} else {
+		switch (instr[start]) {
+		case FASTTRAP_RET:
+			tp->ftt_type = FASTTRAP_T_RET;
+			break;
+
+		case FASTTRAP_RET16:
+			tp->ftt_type = FASTTRAP_T_RET16;
+			tp->ftt_dest = *(uint16_t *)&instr[start + 1];
+			break;
+
+		case FASTTRAP_JO:
+		case FASTTRAP_JNO:
+		case FASTTRAP_JB:
+		case FASTTRAP_JAE:
+		case FASTTRAP_JE:
+		case FASTTRAP_JNE:
+		case FASTTRAP_JBE:
+		case FASTTRAP_JA:
+		case FASTTRAP_JS:
+		case FASTTRAP_JNS:
+		case FASTTRAP_JP:
+		case FASTTRAP_JNP:
+		case FASTTRAP_JL:
+		case FASTTRAP_JGE:
+		case FASTTRAP_JLE:
+		case FASTTRAP_JG:
+			tp->ftt_type = FASTTRAP_T_JCC;
+			tp->ftt_code = instr[start];
+			tp->ftt_dest = pc + tp->ftt_size +
+			    (int8_t)instr[start + 1];
+			break;
+
+		case FASTTRAP_LOOPNZ:
+		case FASTTRAP_LOOPZ:
+		case FASTTRAP_LOOP:
+			tp->ftt_type = FASTTRAP_T_LOOP;
+			tp->ftt_code = instr[start];
+			tp->ftt_dest = pc + tp->ftt_size +
+			    (int8_t)instr[start + 1];
+			break;
+
+		case FASTTRAP_JCXZ:
+			tp->ftt_type = FASTTRAP_T_JCXZ;
+			tp->ftt_dest = pc + tp->ftt_size +
+			    (int8_t)instr[start + 1];
+			break;
+
+		case FASTTRAP_CALL:
+			tp->ftt_type = FASTTRAP_T_CALL;
+			tp->ftt_dest = pc + tp->ftt_size +
+			    *(int32_t *)&instr[start + 1];
+			tp->ftt_code = 0;
+			break;
+
+		case FASTTRAP_JMP32:
+			tp->ftt_type = FASTTRAP_T_JMP;
+			tp->ftt_dest = pc + tp->ftt_size +
+			    *(int32_t *)&instr[start + 1];
+			break;
+		case FASTTRAP_JMP8:
+			tp->ftt_type = FASTTRAP_T_JMP;
+			tp->ftt_dest = pc + tp->ftt_size +
+			    (int8_t)instr[start + 1];
+			break;
+
+		case FASTTRAP_PUSHL_EBP:
+			if (start == 0)
+				tp->ftt_type = FASTTRAP_T_PUSHL_EBP;
+			break;
+
+		case FASTTRAP_NOP:
+#if __sol64 || defined(__APPLE__)
+			ASSERT(p_model == DATAMODEL_LP64 || rex == 0);
+
+			/*
+			 * On sol64 we have to be careful not to confuse a nop
+			 * (actually xchgl %eax, %eax) with an instruction using
+			 * the same opcode, but that does something different
+			 * (e.g. xchgl %r8d, %eax or xcghq %r8, %rax).
+			 */
+			if (FASTTRAP_REX_B(rex) == 0)
+#endif
+				tp->ftt_type = FASTTRAP_T_NOP;
+			break;
+
+		case FASTTRAP_INT3:
+			/*
+			 * The pid provider shares the int3 trap with debugger
+			 * breakpoints so we can't instrument them.
+			 */
+			ASSERT(instr[start] == FASTTRAP_INSTR);
+			return (-1);
+
+		case FASTTRAP_INT:
+			/*
+			 * Interrupts seem like they could be traced with
+			 * no negative implications, but it's possible that
+			 * a thread could be redirected by the trap handling
+			 * code which would eventually return to the
+			 * instruction after the interrupt. If the interrupt
+			 * were in our scratch space, the subsequent
+			 * instruction might be overwritten before we return.
+			 * Accordingly we refuse to instrument any interrupt.
+			 */
+			return (-1);
+		}
+	}
+
+#if __sol64 || defined(__APPLE__)
+	if (p_model == DATAMODEL_LP64 && tp->ftt_type == FASTTRAP_T_COMMON) {
+		/*
+		 * If the process is 64-bit and the instruction type is still
+		 * FASTTRAP_T_COMMON -- meaning we're going to copy it out an
+		 * execute it -- we need to watch for %rip-relative
+		 * addressing mode. See the portion of fasttrap_pid_probe()
+		 * below where we handle tracepoints with type
+		 * FASTTRAP_T_COMMON for how we emulate instructions that
+		 * employ %rip-relative addressing.
+		 */
+		if (rmindex != -1) {
+			uint_t mod = FASTTRAP_MODRM_MOD(instr[rmindex]);
+			uint_t reg = FASTTRAP_MODRM_REG(instr[rmindex]);
+			uint_t rm = FASTTRAP_MODRM_RM(instr[rmindex]);
+
+			ASSERT(rmindex > (int)start);
+
+			if (mod == 0 && rm == 5) {
+				/*
+				 * We need to be sure to avoid other
+				 * registers used by this instruction. While
+				 * the reg field may determine the op code
+				 * rather than denoting a register, assuming
+				 * that it denotes a register is always safe.
+				 * We leave the REX field intact and use
+				 * whatever value's there for simplicity.
+				 */
+				if (reg != 0) {
+					tp->ftt_ripmode = FASTTRAP_RIP_1 |
+					    (FASTTRAP_RIP_X *
+					    FASTTRAP_REX_B(rex));
+					rm = 0;
+				} else {
+					tp->ftt_ripmode = FASTTRAP_RIP_2 |
+					    (FASTTRAP_RIP_X *
+					    FASTTRAP_REX_B(rex));
+					rm = 1;
+				}
+
+				tp->ftt_modrm = tp->ftt_instr[rmindex];
+				tp->ftt_instr[rmindex] =
+				    FASTTRAP_MODRM(2, reg, rm);
+			}
+		}
+	}
+#endif
+
+	return (0);
+}
+
+int
+fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	fasttrap_instr_t instr = FASTTRAP_INSTR;
+
+	if (uwrite(p, &instr, 1, tp->ftt_pc) != 0)
+		return (-1);
+
+	return (0);
+}
+
+int
+fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	uint8_t instr;
+
+	/*
+	 * Distinguish between read or write failures and a changed
+	 * instruction.
+	 */
+	if (uread(p, &instr, 1, tp->ftt_pc) != 0)
+		return (0);
+	if (instr != FASTTRAP_INSTR)
+		return (0);
+	if (uwrite(p, &tp->ftt_instr[0], 1, tp->ftt_pc) != 0)
+		return (-1);
+
+	return (0);
+}
+
+static void
+fasttrap_return_common(x86_saved_state_t *regs, user_addr_t pc, pid_t pid,
+    user_addr_t new_pc)
+{
+	x86_saved_state64_t *regs64;
+	x86_saved_state32_t *regs32;
+	unsigned int p_model;
+
+        if (is_saved_state64(regs)) {
+                regs64 = saved_state64(regs);
+		regs32 = NULL;
+		p_model = DATAMODEL_LP64;
+        } else {
+		regs64 = NULL;
+                regs32 = saved_state32(regs);
+		p_model = DATAMODEL_ILP32;
+        }
+
+	fasttrap_tracepoint_t *tp;
+	fasttrap_bucket_t *bucket;
+	fasttrap_id_t *id;
+	lck_mtx_t *pid_mtx;
+
+	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
+	lck_mtx_lock(pid_mtx);
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    !tp->ftt_proc->ftpc_defunct)
+			break;
+	}
+
+	/*
+	 * Don't sweat it if we can't find the tracepoint again; unlike
+	 * when we're in fasttrap_pid_probe(), finding the tracepoint here
+	 * is not essential to the correct execution of the process.
+	 */
+	if (tp == NULL) {
+		lck_mtx_unlock(pid_mtx);
+		return;
+	}
+
+	for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
+		/*
+		 * If there's a branch that could act as a return site, we
+		 * need to trace it, and check here if the program counter is
+		 * external to the function.
+		 */
+		if (tp->ftt_type != FASTTRAP_T_RET &&
+		    tp->ftt_type != FASTTRAP_T_RET16 &&
+		    new_pc - id->fti_probe->ftp_faddr <
+		    id->fti_probe->ftp_fsize)
+			continue;
+
+		if (p_model == DATAMODEL_LP64) {
+			dtrace_probe(id->fti_probe->ftp_id,
+				     pc - id->fti_probe->ftp_faddr,
+				     regs64->rax, regs64->rdx, 0, 0);
+		} else {
+			dtrace_probe(id->fti_probe->ftp_id,
+				     pc - id->fti_probe->ftp_faddr,
+				     regs32->eax, regs32->edx, 0, 0);
+		}
+	}
+
+	lck_mtx_unlock(pid_mtx);
+}
+
+static void
+fasttrap_sigsegv(proc_t *p, uthread_t t, user_addr_t addr)
+{	
+	proc_lock(p);
+
+	/* Set fault address and mark signal */
+	t->uu_code = addr;
+	t->uu_siglist |= sigmask(SIGSEGV);
+
+	/* 
+         * XXX These two line may be redundant; if not, then we need
+	 * XXX to potentially set the data address in the machine
+	 * XXX specific thread state structure to indicate the address.
+	 */
+	t->uu_exception = KERN_INVALID_ADDRESS;		/* SIGSEGV */
+	t->uu_subcode = 0;	/* XXX pad */
+
+	proc_unlock(p);
+
+	/* raise signal */
+	signal_setast(t->uu_context.vc_thread);
+}
+
+static void
+fasttrap_usdt_args64(fasttrap_probe_t *probe, x86_saved_state64_t *regs64, int argc,
+    uint64_t *argv)
+{
+	int i, x, cap = MIN(argc, probe->ftp_nargs);
+	user_addr_t stack = (user_addr_t)regs64->isf.rsp;
+
+	for (i = 0; i < cap; i++) {
+		x = probe->ftp_argmap[i];
+
+		if (x < 6) {
+			/* FIXME! This may be broken, needs testing */
+			argv[i] = (&regs64->rdi)[x];
+		} else {
+			fasttrap_fuword64_noerr(stack + (x * sizeof(uint64_t)), &argv[i]);
+		}
+	}
+
+	for (; i < argc; i++) {
+		argv[i] = 0;
+	}
+}
+
+static void
+fasttrap_usdt_args32(fasttrap_probe_t *probe, x86_saved_state32_t *regs32, int argc,
+    uint32_t *argv)
+{
+	int i, x, cap = MIN(argc, probe->ftp_nargs);
+	uint32_t *stack = (uint32_t *)regs32->uesp;
+
+	for (i = 0; i < cap; i++) {
+		x = probe->ftp_argmap[i];
+
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[x], &argv[i]);
+	}
+
+	for (; i < argc; i++) {
+		argv[i] = 0;
+	}
+}
+
+/*
+ * FIXME!
+ */
+static int
+fasttrap_do_seg(fasttrap_tracepoint_t *tp, x86_saved_state_t *rp, user_addr_t *addr) // 64 bit
+{
+#pragma unused(tp, rp, addr)
+	printf("fasttrap_do_seg() called while unimplemented.\n");
+#if 0
+	proc_t *p = curproc;
+	user_desc_t *desc;
+	uint16_t sel, ndx, type;
+	uintptr_t limit;
+
+	switch (tp->ftt_segment) {
+	case FASTTRAP_SEG_CS:
+		sel = rp->r_cs;
+		break;
+	case FASTTRAP_SEG_DS:
+		sel = rp->r_ds;
+		break;
+	case FASTTRAP_SEG_ES:
+		sel = rp->r_es;
+		break;
+	case FASTTRAP_SEG_FS:
+		sel = rp->r_fs;
+		break;
+	case FASTTRAP_SEG_GS:
+		sel = rp->r_gs;
+		break;
+	case FASTTRAP_SEG_SS:
+		sel = rp->r_ss;
+		break;
+	}
+
+	/*
+	 * Make sure the given segment register specifies a user priority
+	 * selector rather than a kernel selector.
+	 */
+	if (!SELISUPL(sel))
+		return (-1);
+
+	ndx = SELTOIDX(sel);
+
+	/*
+	 * Check the bounds and grab the descriptor out of the specified
+	 * descriptor table.
+	 */
+	if (SELISLDT(sel)) {
+		if (ndx > p->p_ldtlimit)
+			return (-1);
+
+		desc = p->p_ldt + ndx;
+
+	} else {
+		if (ndx >= NGDT)
+			return (-1);
+
+		desc = cpu_get_gdt() + ndx;
+	}
+
+	/*
+	 * The descriptor must have user privilege level and it must be
+	 * present in memory.
+	 */
+	if (desc->usd_dpl != SEL_UPL || desc->usd_p != 1)
+		return (-1);
+
+	type = desc->usd_type;
+
+	/*
+	 * If the S bit in the type field is not set, this descriptor can
+	 * only be used in system context.
+	 */
+	if ((type & 0x10) != 0x10)
+		return (-1);
+
+	limit = USEGD_GETLIMIT(desc) * (desc->usd_gran ? PAGESIZE : 1);
+
+	if (tp->ftt_segment == FASTTRAP_SEG_CS) {
+		/*
+		 * The code/data bit and readable bit must both be set.
+		 */
+		if ((type & 0xa) != 0xa)
+			return (-1);
+
+		if (*addr > limit)
+			return (-1);
+	} else {
+		/*
+		 * The code/data bit must be clear.
+		 */
+		if ((type & 0x8) != 0)
+			return (-1);
+
+		/*
+		 * If the expand-down bit is clear, we just check the limit as
+		 * it would naturally be applied. Otherwise, we need to check
+		 * that the address is the range [limit + 1 .. 0xffff] or
+		 * [limit + 1 ... 0xffffffff] depending on if the default
+		 * operand size bit is set.
+		 */
+		if ((type & 0x4) == 0) {
+			if (*addr > limit)
+				return (-1);
+		} else if (desc->usd_def32) {
+			if (*addr < limit + 1 || 0xffff < *addr)
+				return (-1);
+		} else {
+			if (*addr < limit + 1 || 0xffffffff < *addr)
+				return (-1);
+		}
+	}
+
+	*addr += USEGD_GETBASE(desc);
+#endif /* 0 */
+	return (0);
+}
+
+/*
+ * Due to variances between Solaris and xnu, I have split this into a 32 bit and 64 bit
+ * code path. It still takes an x86_saved_state_t* argument, because it must sometimes
+ * call other methods that require a x86_saved_state_t.
+ *
+ * NOTE!!!!
+ *
+ * Any changes made to this method must be echo'd in fasttrap_pid_probe64!
+ *
+ */
+static int
+fasttrap_pid_probe32(x86_saved_state_t *regs)
+{
+	ASSERT(is_saved_state32(regs));
+
+	x86_saved_state32_t *regs32  = saved_state32(regs);
+	user_addr_t pc = regs32->eip - 1;
+	proc_t *p = current_proc();
+	user_addr_t new_pc = 0;
+	fasttrap_bucket_t *bucket;
+	lck_mtx_t *pid_mtx;
+	fasttrap_tracepoint_t *tp, tp_local;
+	pid_t pid;
+	dtrace_icookie_t cookie;
+	uint_t is_enabled = 0;
+
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+	/*
+	 * It's possible that a user (in a veritable orgy of bad planning)
+	 * could redirect this thread's flow of control before it reached the
+	 * return probe fasttrap. In this case we need to kill the process
+	 * since it's in a unrecoverable state.
+	 */
+	if (uthread->t_dtrace_step) {
+		ASSERT(uthread->t_dtrace_on);
+		fasttrap_sigtrap(p, uthread, pc);
+		return (0);
+	}
+
+	/*
+	 * Clear all user tracing flags.
+	 */
+	uthread->t_dtrace_ft = 0;
+	uthread->t_dtrace_pc = 0;
+	uthread->t_dtrace_npc = 0;
+	uthread->t_dtrace_scrpc = 0;
+	uthread->t_dtrace_astpc = 0;
+
+	/*
+	 * Treat a child created by a call to vfork(2) as if it were its
+	 * parent. We know that there's only one thread of control in such a
+	 * process: this one.
+	 */
+	/*
+	 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
+	 * FIXME: How do we assert this?
+	 */
+	while (p->p_lflag & P_LINVFORK)
+		p = p->p_pptr;
+
+	pid = p->p_pid;
+	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
+	lck_mtx_lock(pid_mtx);
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	/*
+	 * Lookup the tracepoint that the process just hit.
+	 */
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    !tp->ftt_proc->ftpc_defunct)
+			break;
+	}
+
+	/*
+	 * If we couldn't find a matching tracepoint, either a tracepoint has
+	 * been inserted without using the pid<pid> ioctl interface (see
+	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
+	 */
+	if (tp == NULL) {
+		lck_mtx_unlock(pid_mtx);
+		return (-1);
+	}
+
+	/*
+	 * Set the program counter to the address of the traced instruction
+	 * so that it looks right in ustack() output.
+	 */
+	regs32->eip = pc;
+
+	if (tp->ftt_ids != NULL) {
+		fasttrap_id_t *id;
+		
+		uint32_t s0, s1, s2, s3, s4, s5;
+		uint32_t *stack = (uint32_t *)regs32->uesp;
+		
+		/*
+		 * In 32-bit mode, all arguments are passed on the
+		 * stack. If this is a function entry probe, we need
+		 * to skip the first entry on the stack as it
+		 * represents the return address rather than a
+		 * parameter to the function.
+		 */
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[0], &s0);
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[1], &s1);
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[2], &s2);
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[3], &s3);
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[4], &s4);
+		fasttrap_fuword32_noerr((user_addr_t)(unsigned long)&stack[5], &s5);
+		
+		for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
+			fasttrap_probe_t *probe = id->fti_probe;
+			
+			if (id->fti_ptype == DTFTP_ENTRY) {
+				/*
+				 * We note that this was an entry
+				 * probe to help ustack() find the
+				 * first caller.
+				 */
+				cookie = dtrace_interrupt_disable();
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
+				dtrace_probe(probe->ftp_id, s1, s2,
+					     s3, s4, s5);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
+				dtrace_interrupt_enable(cookie);
+			} else if (id->fti_ptype == DTFTP_IS_ENABLED) {
+				/*
+				 * Note that in this case, we don't
+				 * call dtrace_probe() since it's only
+				 * an artificial probe meant to change
+				 * the flow of control so that it
+				 * encounters the true probe.
+				 */
+				is_enabled = 1;
+			} else if (probe->ftp_argmap == NULL) {
+				dtrace_probe(probe->ftp_id, s0, s1,
+					     s2, s3, s4);
+			} else {
+				uint32_t t[5];
+				
+				fasttrap_usdt_args32(probe, regs32,
+						     sizeof (t) / sizeof (t[0]), t);
+				
+				dtrace_probe(probe->ftp_id, t[0], t[1],
+					     t[2], t[3], t[4]);
+			}
+
+			/* APPLE NOTE: Oneshot probes get one and only one chance... */
+			if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) {
+				fasttrap_tracepoint_remove(p, tp);
+			}
+		}
+	}
+
+	/*
+	 * We're about to do a bunch of work so we cache a local copy of
+	 * the tracepoint to emulate the instruction, and then find the
+	 * tracepoint again later if we need to light up any return probes.
+	 */
+	tp_local = *tp;
+	lck_mtx_unlock(pid_mtx);
+	tp = &tp_local;
+
+	/*
+	 * Set the program counter to appear as though the traced instruction
+	 * had completely executed. This ensures that fasttrap_getreg() will
+	 * report the expected value for REG_RIP.
+	 */
+	regs32->eip = pc + tp->ftt_size;
+
+	/*
+	 * If there's an is-enabled probe connected to this tracepoint it
+	 * means that there was a 'xorl %eax, %eax' or 'xorq %rax, %rax'
+	 * instruction that was placed there by DTrace when the binary was
+	 * linked. As this probe is, in fact, enabled, we need to stuff 1
+	 * into %eax or %rax. Accordingly, we can bypass all the instruction
+	 * emulation logic since we know the inevitable result. It's possible
+	 * that a user could construct a scenario where the 'is-enabled'
+	 * probe was on some other instruction, but that would be a rather
+	 * exotic way to shoot oneself in the foot.
+	 */
+	if (is_enabled) {
+		regs32->eax = 1;
+		new_pc = regs32->eip;
+		goto done;
+	}
+
+	/*
+	 * We emulate certain types of instructions to ensure correctness
+	 * (in the case of position dependent instructions) or optimize
+	 * common cases. The rest we have the thread execute back in user-
+	 * land.
+	 */
+	switch (tp->ftt_type) {
+		case FASTTRAP_T_RET:
+		case FASTTRAP_T_RET16:
+		{
+			user_addr_t dst;
+			user_addr_t addr;
+			int ret;
+
+			/*
+			 * We have to emulate _every_ facet of the behavior of a ret
+			 * instruction including what happens if the load from %esp
+			 * fails; in that case, we send a SIGSEGV.
+			 */
+			uint32_t dst32;
+			ret = fasttrap_fuword32((user_addr_t)regs32->uesp, &dst32);
+			dst = dst32;
+			addr = regs32->uesp + sizeof (uint32_t);
+
+			if (ret == -1) {
+				fasttrap_sigsegv(p, uthread, (user_addr_t)regs32->uesp);
+				new_pc = pc;
+				break;
+			}
+			
+			if (tp->ftt_type == FASTTRAP_T_RET16)
+				addr += tp->ftt_dest;
+			
+			regs32->uesp = addr;
+			new_pc = dst;
+			break;
+		}
+
+		case FASTTRAP_T_JCC:
+		{
+			uint_t taken;
+			
+			switch (tp->ftt_code) {
+				case FASTTRAP_JO:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_OF) != 0;
+					break;
+				case FASTTRAP_JNO:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_OF) == 0;
+					break;
+				case FASTTRAP_JB:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_CF) != 0;
+					break;
+				case FASTTRAP_JAE:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_CF) == 0;
+					break;
+				case FASTTRAP_JE:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0;
+					break;
+				case FASTTRAP_JNE:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0;
+					break;
+				case FASTTRAP_JBE:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_CF) != 0 ||
+						(regs32->efl & FASTTRAP_EFLAGS_ZF) != 0;
+					break;
+				case FASTTRAP_JA:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_CF) == 0 &&
+						(regs32->efl & FASTTRAP_EFLAGS_ZF) == 0;
+					break;
+				case FASTTRAP_JS:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_SF) != 0;
+					break;
+				case FASTTRAP_JNS:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_SF) == 0;
+					break;
+				case FASTTRAP_JP:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_PF) != 0;
+					break;
+				case FASTTRAP_JNP:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_PF) == 0;
+					break;
+				case FASTTRAP_JL:
+					taken = ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) !=
+						((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JGE:
+					taken = ((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) ==
+						((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JLE:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0 ||
+						((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) !=
+						((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JG:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0 &&
+						((regs32->efl & FASTTRAP_EFLAGS_SF) == 0) ==
+						((regs32->efl & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				default:
+					taken = FALSE;
+			}
+			
+			if (taken)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+
+		case FASTTRAP_T_LOOP:
+		{
+			uint_t taken;
+			greg_t cx = regs32->ecx--;
+
+			switch (tp->ftt_code) {
+				case FASTTRAP_LOOPNZ:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) == 0 &&
+						cx != 0;
+					break;
+				case FASTTRAP_LOOPZ:
+					taken = (regs32->efl & FASTTRAP_EFLAGS_ZF) != 0 &&
+						cx != 0;
+					break;
+				case FASTTRAP_LOOP:
+					taken = (cx != 0);
+					break;
+				default:
+					taken = FALSE;
+			}
+			
+			if (taken)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+
+		case FASTTRAP_T_JCXZ:
+		{
+			greg_t cx = regs32->ecx;
+			
+			if (cx == 0)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+
+		case FASTTRAP_T_PUSHL_EBP:
+		{
+			user_addr_t addr = regs32->uesp - sizeof (uint32_t);
+			int ret = fasttrap_suword32(addr, (uint32_t)regs32->ebp);
+			
+			if (ret == -1) {
+				fasttrap_sigsegv(p, uthread, addr);
+				new_pc = pc;
+				break;
+			}
+			
+			regs32->uesp = addr;
+			new_pc = pc + tp->ftt_size;
+			break;
+		}
+		
+		case FASTTRAP_T_NOP:
+			new_pc = pc + tp->ftt_size;
+			break;
+
+		case FASTTRAP_T_JMP:
+		case FASTTRAP_T_CALL:
+			if (tp->ftt_code == 0) {
+				new_pc = tp->ftt_dest;
+			} else {
+				user_addr_t /* value ,*/ addr = tp->ftt_dest;
+
+				if (tp->ftt_base != FASTTRAP_NOREG)
+					addr += fasttrap_getreg(regs, tp->ftt_base);
+				if (tp->ftt_index != FASTTRAP_NOREG)
+					addr += fasttrap_getreg(regs, tp->ftt_index) <<
+						tp->ftt_scale;
+				
+				if (tp->ftt_code == 1) {
+					/*
+					 * If there's a segment prefix for this
+					 * instruction, we'll need to check permissions
+					 * and bounds on the given selector, and adjust
+					 * the address accordingly.
+					 */
+					if (tp->ftt_segment != FASTTRAP_SEG_NONE &&
+					    fasttrap_do_seg(tp, regs, &addr) != 0) {
+						fasttrap_sigsegv(p, uthread, addr);
+						new_pc = pc;
+						break;
+					}
+					
+					uint32_t value32;
+					addr = (user_addr_t)(uint32_t)addr;
+					if (fasttrap_fuword32(addr, &value32) == -1) {
+						fasttrap_sigsegv(p, uthread, addr);
+						new_pc = pc;
+						break;
+					}
+					new_pc = value32;
+				} else {
+					new_pc = addr;
+				}
+			}
+
+			/*
+			 * If this is a call instruction, we need to push the return
+			 * address onto the stack. If this fails, we send the process
+			 * a SIGSEGV and reset the pc to emulate what would happen if
+			 * this instruction weren't traced.
+			 */
+			if (tp->ftt_type == FASTTRAP_T_CALL) {
+				user_addr_t addr = regs32->uesp - sizeof (uint32_t);
+				int ret = fasttrap_suword32(addr, (uint32_t)(pc + tp->ftt_size));
+				
+				if (ret == -1) {
+					fasttrap_sigsegv(p, uthread, addr);
+					new_pc = pc;
+					break;
+				}
+				
+				regs32->uesp = addr;
+			}
+			break;
+
+		case FASTTRAP_T_COMMON:
+		{
+			user_addr_t addr;
+			uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 5 + 2];
+			uint_t i = 0;
+
+			/*
+			 * Generic Instruction Tracing
+			 * ---------------------------
+			 *
+			 * This is the layout of the scratch space in the user-land
+			 * thread structure for our generated instructions.
+			 *
+			 *	32-bit mode			bytes
+			 *	------------------------	-----
+			 * a:	<original instruction>		<= 15
+			 *	jmp	<pc + tp->ftt_size>	    5
+			 * b:	<original instrction>		<= 15
+			 *	int	T_DTRACE_RET		    2
+			 *					-----
+			 *					<= 37
+			 *
+			 *	64-bit mode			bytes
+			 *	------------------------	-----
+			 * a:	<original instruction>		<= 15
+			 *	jmp	0(%rip)			    6
+			 *	<pc + tp->ftt_size>		    8
+			 * b:	<original instruction>		<= 15
+			 * 	int	T_DTRACE_RET		    2
+			 * 					-----
+			 * 					<= 46
+			 *
+			 * The %pc is set to a, and curthread->t_dtrace_astpc is set
+			 * to b. If we encounter a signal on the way out of the
+			 * kernel, trap() will set %pc to curthread->t_dtrace_astpc
+			 * so that we execute the original instruction and re-enter
+			 * the kernel rather than redirecting to the next instruction.
+			 *
+			 * If there are return probes (so we know that we're going to
+			 * need to reenter the kernel after executing the original
+			 * instruction), the scratch space will just contain the
+			 * original instruction followed by an interrupt -- the same
+			 * data as at b.
+			 */
+
+			addr = uthread->t_dtrace_scratch->addr;
+
+			if (addr == 0LL) {
+				fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc
+				new_pc = pc;
+				break;
+			}
+
+			ASSERT(tp->ftt_size < FASTTRAP_MAX_INSTR_SIZE);
+
+			uthread->t_dtrace_scrpc = addr;
+			bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
+			i += tp->ftt_size;
+
+			/*
+			 * Set up the jmp to the next instruction; note that
+			 * the size of the traced instruction cancels out.
+			 */
+			scratch[i++] = FASTTRAP_JMP32;
+			*(uint32_t *)&scratch[i] = pc - addr - 5;
+			i += sizeof (uint32_t);
+
+			uthread->t_dtrace_astpc = addr + i;
+			bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
+			i += tp->ftt_size;
+			scratch[i++] = FASTTRAP_INT;
+			scratch[i++] = T_DTRACE_RET;
+			
+			if (fasttrap_copyout(scratch, addr, i)) {
+				fasttrap_sigtrap(p, uthread, pc);
+				new_pc = pc;
+				break;
+			}
+			
+			if (tp->ftt_retids != NULL) {
+				uthread->t_dtrace_step = 1;
+				uthread->t_dtrace_ret = 1;
+				new_pc = uthread->t_dtrace_astpc;
+			} else {
+				new_pc = uthread->t_dtrace_scrpc;
+			}
+			
+			uthread->t_dtrace_pc = pc;
+			uthread->t_dtrace_npc = pc + tp->ftt_size;
+			uthread->t_dtrace_on = 1;
+			break;
+		}
+		
+		default:
+			panic("fasttrap: mishandled an instruction");
+	}
+	
+done:
+	/*
+	 * APPLE NOTE:
+	 *
+	 * We're setting this earlier than Solaris does, to get a "correct"
+	 * ustack() output. In the Sun code,  a() -> b() -> c() -> d() is
+	 * reported at: d, b, a. The new way gives c, b, a, which is closer
+	 * to correct, as the return instruction has already exectued.
+	 */
+	regs32->eip = new_pc;
+
+	/*
+	 * If there were no return probes when we first found the tracepoint,
+	 * we should feel no obligation to honor any return probes that were
+	 * subsequently enabled -- they'll just have to wait until the next
+	 * time around.
+	 */
+	if (tp->ftt_retids != NULL) {
+		/*
+		 * We need to wait until the results of the instruction are
+		 * apparent before invoking any return probes. If this
+		 * instruction was emulated we can just call
+		 * fasttrap_return_common(); if it needs to be executed, we
+		 * need to wait until the user thread returns to the kernel.
+		 */
+		if (tp->ftt_type != FASTTRAP_T_COMMON) {
+			fasttrap_return_common(regs, pc, pid, new_pc);
+		} else {
+			ASSERT(uthread->t_dtrace_ret != 0);
+			ASSERT(uthread->t_dtrace_pc == pc);
+			ASSERT(uthread->t_dtrace_scrpc != 0);
+			ASSERT(new_pc == uthread->t_dtrace_astpc);
+		}
+	}
+
+	return (0);
+}
+
+/*
+ * Due to variances between Solaris and xnu, I have split this into a 32 bit and 64 bit
+ * code path. It still takes an x86_saved_state_t* argument, because it must sometimes
+ * call other methods that require a x86_saved_state_t.
+ *
+ * NOTE!!!!
+ *
+ * Any changes made to this method must be echo'd in fasttrap_pid_probe32!
+ *
+ */
+static int
+fasttrap_pid_probe64(x86_saved_state_t *regs)
+{
+	ASSERT(is_saved_state64(regs));
+
+	x86_saved_state64_t *regs64 = saved_state64(regs);
+	user_addr_t pc = regs64->isf.rip - 1;
+	proc_t *p = current_proc();
+	user_addr_t new_pc = 0;
+	fasttrap_bucket_t *bucket;
+	lck_mtx_t *pid_mtx;
+	fasttrap_tracepoint_t *tp, tp_local;
+	pid_t pid;
+	dtrace_icookie_t cookie;
+	uint_t is_enabled = 0;
+
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+	/*
+	 * It's possible that a user (in a veritable orgy of bad planning)
+	 * could redirect this thread's flow of control before it reached the
+	 * return probe fasttrap. In this case we need to kill the process
+	 * since it's in a unrecoverable state.
+	 */
+	if (uthread->t_dtrace_step) {
+		ASSERT(uthread->t_dtrace_on);
+		fasttrap_sigtrap(p, uthread, pc);
+		return (0);
+	}
+
+	/*
+	 * Clear all user tracing flags.
+	 */
+	uthread->t_dtrace_ft = 0;
+	uthread->t_dtrace_pc = 0;
+	uthread->t_dtrace_npc = 0;
+	uthread->t_dtrace_scrpc = 0;
+	uthread->t_dtrace_astpc = 0;
+	uthread->t_dtrace_regv = 0;
+
+	/*
+	 * Treat a child created by a call to vfork(2) as if it were its
+	 * parent. We know that there's only one thread of control in such a
+	 * process: this one.
+	 */
+	/*
+	 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
+	 * FIXME: How do we assert this?
+	 */
+	while (p->p_lflag & P_LINVFORK)
+		p = p->p_pptr;
+
+	pid = p->p_pid;
+	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
+	lck_mtx_lock(pid_mtx);
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	/*
+	 * Lookup the tracepoint that the process just hit.
+	 */
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    !tp->ftt_proc->ftpc_defunct)
+			break;
+	}
+
+	/*
+	 * If we couldn't find a matching tracepoint, either a tracepoint has
+	 * been inserted without using the pid<pid> ioctl interface (see
+	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
+	 */
+	if (tp == NULL) {
+		lck_mtx_unlock(pid_mtx);
+		return (-1);
+	}
+
+	/*
+	 * Set the program counter to the address of the traced instruction
+	 * so that it looks right in ustack() output.
+	 */
+	regs64->isf.rip = pc;
+
+	if (tp->ftt_ids != NULL) {
+		fasttrap_id_t *id;
+
+		for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
+			fasttrap_probe_t *probe = id->fti_probe;
+			
+			if (id->fti_ptype == DTFTP_ENTRY) {
+				/*
+				 * We note that this was an entry
+				 * probe to help ustack() find the
+				 * first caller.
+				 */
+				cookie = dtrace_interrupt_disable();
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
+				dtrace_probe(probe->ftp_id, regs64->rdi,
+					     regs64->rsi, regs64->rdx, regs64->rcx,
+					     regs64->r8);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
+				dtrace_interrupt_enable(cookie);
+			} else if (id->fti_ptype == DTFTP_IS_ENABLED) {
+				/*
+				 * Note that in this case, we don't
+				 * call dtrace_probe() since it's only
+				 * an artificial probe meant to change
+				 * the flow of control so that it
+				 * encounters the true probe.
+				 */
+				is_enabled = 1;
+			} else if (probe->ftp_argmap == NULL) {
+				dtrace_probe(probe->ftp_id, regs64->rdi,
+					     regs64->rsi, regs64->rdx, regs64->rcx,
+					     regs64->r8);
+			} else {
+				uint64_t t[5];
+				
+				fasttrap_usdt_args64(probe, regs64,
+						     sizeof (t) / sizeof (t[0]), t);
+				
+				dtrace_probe(probe->ftp_id, t[0], t[1],
+					     t[2], t[3], t[4]);
+			}
+
+			/* APPLE NOTE: Oneshot probes get one and only one chance... */
+			if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) {
+				fasttrap_tracepoint_remove(p, tp);
+			}
+		}
+	}
+
+	/*
+	 * We're about to do a bunch of work so we cache a local copy of
+	 * the tracepoint to emulate the instruction, and then find the
+	 * tracepoint again later if we need to light up any return probes.
+	 */
+	tp_local = *tp;
+	lck_mtx_unlock(pid_mtx);
+	tp = &tp_local;
+
+	/*
+	 * Set the program counter to appear as though the traced instruction
+	 * had completely executed. This ensures that fasttrap_getreg() will
+	 * report the expected value for REG_RIP.
+	 */
+	regs64->isf.rip = pc + tp->ftt_size;
+
+	/*
+	 * If there's an is-enabled probe connected to this tracepoint it
+	 * means that there was a 'xorl %eax, %eax' or 'xorq %rax, %rax'
+	 * instruction that was placed there by DTrace when the binary was
+	 * linked. As this probe is, in fact, enabled, we need to stuff 1
+	 * into %eax or %rax. Accordingly, we can bypass all the instruction
+	 * emulation logic since we know the inevitable result. It's possible
+	 * that a user could construct a scenario where the 'is-enabled'
+	 * probe was on some other instruction, but that would be a rather
+	 * exotic way to shoot oneself in the foot.
+	 */
+	if (is_enabled) {
+		regs64->rax = 1;
+		new_pc = regs64->isf.rip;
+		goto done;
+	}
+
+	/*
+	 * We emulate certain types of instructions to ensure correctness
+	 * (in the case of position dependent instructions) or optimize
+	 * common cases. The rest we have the thread execute back in user-
+	 * land.
+	 */
+	switch (tp->ftt_type) {
+		case FASTTRAP_T_RET:
+		case FASTTRAP_T_RET16:
+		{
+			user_addr_t dst;
+			user_addr_t addr;
+			int ret;
+			
+			/*
+			 * We have to emulate _every_ facet of the behavior of a ret
+			 * instruction including what happens if the load from %esp
+			 * fails; in that case, we send a SIGSEGV.
+			 */
+			ret = fasttrap_fuword64((user_addr_t)regs64->isf.rsp, &dst);
+			addr = regs64->isf.rsp + sizeof (uint64_t);
+			
+			if (ret == -1) {
+				fasttrap_sigsegv(p, uthread, (user_addr_t)regs64->isf.rsp);
+				new_pc = pc;
+				break;
+			}
+			
+			if (tp->ftt_type == FASTTRAP_T_RET16)
+				addr += tp->ftt_dest;
+			
+			regs64->isf.rsp = addr;
+			new_pc = dst;
+			break;
+		}
+		
+		case FASTTRAP_T_JCC:
+		{
+			uint_t taken;
+			
+			switch (tp->ftt_code) {
+				case FASTTRAP_JO:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_OF) != 0;
+					break;
+				case FASTTRAP_JNO:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0;
+					break;
+				case FASTTRAP_JB:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) != 0;
+					break;
+				case FASTTRAP_JAE:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) == 0;
+					break;
+				case FASTTRAP_JE:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0;
+					break;
+				case FASTTRAP_JNE:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0;
+					break;
+				case FASTTRAP_JBE:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) != 0 ||
+						(regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0;
+					break;
+				case FASTTRAP_JA:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_CF) == 0 &&
+						(regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0;
+					break;
+				case FASTTRAP_JS:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_SF) != 0;
+					break;
+				case FASTTRAP_JNS:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0;
+					break;
+				case FASTTRAP_JP:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_PF) != 0;
+					break;
+				case FASTTRAP_JNP:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_PF) == 0;
+					break;
+				case FASTTRAP_JL:
+					taken = ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) !=
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JGE:
+					taken = ((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) ==
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JLE:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0 ||
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) !=
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				case FASTTRAP_JG:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_SF) == 0) ==
+						((regs64->isf.rflags & FASTTRAP_EFLAGS_OF) == 0);
+					break;
+				default:
+					taken = FALSE;
+			}
+			
+			if (taken)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+
+		case FASTTRAP_T_LOOP:
+		{
+			uint_t taken;
+			uint64_t cx = regs64->rcx--;
+			
+			switch (tp->ftt_code) {
+				case FASTTRAP_LOOPNZ:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) == 0 &&
+						cx != 0;
+					break;
+				case FASTTRAP_LOOPZ:
+					taken = (regs64->isf.rflags & FASTTRAP_EFLAGS_ZF) != 0 &&
+						cx != 0;
+					break;
+				case FASTTRAP_LOOP:
+					taken = (cx != 0);
+					break;
+				default:
+					taken = FALSE;
+			}
+			
+			if (taken)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+		
+		case FASTTRAP_T_JCXZ:
+		{
+			uint64_t cx = regs64->rcx;
+
+			if (cx == 0)
+				new_pc = tp->ftt_dest;
+			else
+				new_pc = pc + tp->ftt_size;
+			break;
+		}
+
+		case FASTTRAP_T_PUSHL_EBP:
+		{
+			user_addr_t addr = regs64->isf.rsp - sizeof (uint64_t);
+			int ret = fasttrap_suword64(addr, (uint64_t)regs64->rbp);
+			
+			if (ret == -1) {
+				fasttrap_sigsegv(p, uthread, addr);
+				new_pc = pc;
+				break;
+			}
+			
+			regs64->isf.rsp = addr;
+			new_pc = pc + tp->ftt_size;
+			break;
+		}
+		
+		case FASTTRAP_T_NOP:
+			new_pc = pc + tp->ftt_size;
+			break;
+
+		case FASTTRAP_T_JMP:
+		case FASTTRAP_T_CALL:
+			if (tp->ftt_code == 0) {
+				new_pc = tp->ftt_dest;
+			} else {
+				user_addr_t value, addr = tp->ftt_dest;
+				
+				if (tp->ftt_base != FASTTRAP_NOREG)
+					addr += fasttrap_getreg(regs, tp->ftt_base);
+				if (tp->ftt_index != FASTTRAP_NOREG)
+					addr += fasttrap_getreg(regs, tp->ftt_index) <<
+						tp->ftt_scale;
+				
+				if (tp->ftt_code == 1) {
+					/*
+					 * If there's a segment prefix for this
+					 * instruction, we'll need to check permissions
+					 * and bounds on the given selector, and adjust
+					 * the address accordingly.
+					 */
+					if (tp->ftt_segment != FASTTRAP_SEG_NONE &&
+					    fasttrap_do_seg(tp, regs, &addr) != 0) {
+						fasttrap_sigsegv(p, uthread, addr);
+						new_pc = pc;
+						break;
+					}
+					
+					if (fasttrap_fuword64(addr, &value) == -1) {
+						fasttrap_sigsegv(p, uthread, addr);
+						new_pc = pc;
+						break;
+					}
+					new_pc = value;
+				} else {
+					new_pc = addr;
+				}
+			}
+
+			/*
+			 * If this is a call instruction, we need to push the return
+			 * address onto the stack. If this fails, we send the process
+			 * a SIGSEGV and reset the pc to emulate what would happen if
+			 * this instruction weren't traced.
+			 */
+			if (tp->ftt_type == FASTTRAP_T_CALL) {
+				user_addr_t addr = regs64->isf.rsp - sizeof (uint64_t);
+				int ret = fasttrap_suword64(addr, pc + tp->ftt_size);
+				
+				if (ret == -1) {
+					fasttrap_sigsegv(p, uthread, addr);
+					new_pc = pc;
+					break;
+				}
+				
+				regs64->isf.rsp = addr;
+			}
+			break;
+
+		case FASTTRAP_T_COMMON:
+		{
+			user_addr_t addr;
+			uint8_t scratch[2 * FASTTRAP_MAX_INSTR_SIZE + 5 + 2];
+			uint_t i = 0;
+			
+			/*
+			 * Generic Instruction Tracing
+			 * ---------------------------
+			 *
+			 * This is the layout of the scratch space in the user-land
+			 * thread structure for our generated instructions.
+			 *
+			 *	32-bit mode			bytes
+			 *	------------------------	-----
+			 * a:	<original instruction>		<= 15
+			 *	jmp	<pc + tp->ftt_size>	    5
+			 * b:	<original instrction>		<= 15
+			 *	int	T_DTRACE_RET		    2
+			 *					-----
+			 *					<= 37
+			 *
+			 *	64-bit mode			bytes
+			 *	------------------------	-----
+			 * a:	<original instruction>		<= 15
+			 *	jmp	0(%rip)			    6
+			 *	<pc + tp->ftt_size>		    8
+			 * b:	<original instruction>		<= 15
+			 * 	int	T_DTRACE_RET		    2
+			 * 					-----
+			 * 					<= 46
+			 *
+			 * The %pc is set to a, and curthread->t_dtrace_astpc is set
+			 * to b. If we encounter a signal on the way out of the
+			 * kernel, trap() will set %pc to curthread->t_dtrace_astpc
+			 * so that we execute the original instruction and re-enter
+			 * the kernel rather than redirecting to the next instruction.
+			 *
+			 * If there are return probes (so we know that we're going to
+			 * need to reenter the kernel after executing the original
+			 * instruction), the scratch space will just contain the
+			 * original instruction followed by an interrupt -- the same
+			 * data as at b.
+			 *
+			 * %rip-relative Addressing
+			 * ------------------------
+			 *
+			 * There's a further complication in 64-bit mode due to %rip-
+			 * relative addressing. While this is clearly a beneficial
+			 * architectural decision for position independent code, it's
+			 * hard not to see it as a personal attack against the pid
+			 * provider since before there was a relatively small set of
+			 * instructions to emulate; with %rip-relative addressing,
+			 * almost every instruction can potentially depend on the
+			 * address at which it's executed. Rather than emulating
+			 * the broad spectrum of instructions that can now be
+			 * position dependent, we emulate jumps and others as in
+			 * 32-bit mode, and take a different tack for instructions
+			 * using %rip-relative addressing.
+			 *
+			 * For every instruction that uses the ModRM byte, the
+			 * in-kernel disassembler reports its location. We use the
+			 * ModRM byte to identify that an instruction uses
+			 * %rip-relative addressing and to see what other registers
+			 * the instruction uses. To emulate those instructions,
+			 * we modify the instruction to be %rax-relative rather than
+			 * %rip-relative (or %rcx-relative if the instruction uses
+			 * %rax; or %r8- or %r9-relative if the REX.B is present so
+			 * we don't have to rewrite the REX prefix). We then load
+			 * the value that %rip would have been into the scratch
+			 * register and generate an instruction to reset the scratch
+			 * register back to its original value. The instruction
+			 * sequence looks like this:
+			 *
+			 *	64-mode %rip-relative		bytes
+			 *	------------------------	-----
+			 * a:	<modified instruction>		<= 15
+			 *	movq	$<value>, %<scratch>	    6
+			 *	jmp	0(%rip)			    6
+			 *	<pc + tp->ftt_size>		    8
+			 * b:	<modified instruction>  	<= 15
+			 * 	int	T_DTRACE_RET		    2
+			 * 					-----
+			 *					   52
+			 *
+			 * We set curthread->t_dtrace_regv so that upon receiving
+			 * a signal we can reset the value of the scratch register.
+			 */
+
+			addr = uthread->t_dtrace_scratch->addr;
+
+			if (addr == 0LL) {
+				fasttrap_sigtrap(p, uthread, pc); // Should be killing target proc
+				new_pc = pc;
+				break;
+			}
+
+			ASSERT(tp->ftt_size < FASTTRAP_MAX_INSTR_SIZE);
+
+			uthread->t_dtrace_scrpc = addr;
+			bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
+			i += tp->ftt_size;
+
+			if (tp->ftt_ripmode != 0) {
+				uint64_t* reg;
+				
+				ASSERT(tp->ftt_ripmode &
+				       (FASTTRAP_RIP_1 | FASTTRAP_RIP_2));
+				
+				/*
+				 * If this was a %rip-relative instruction, we change
+				 * it to be either a %rax- or %rcx-relative
+				 * instruction (depending on whether those registers
+				 * are used as another operand; or %r8- or %r9-
+				 * relative depending on the value of REX.B). We then
+				 * set that register and generate a movq instruction
+				 * to reset the value.
+				 */
+				if (tp->ftt_ripmode & FASTTRAP_RIP_X)
+					scratch[i++] = FASTTRAP_REX(1, 0, 0, 1);
+				else
+					scratch[i++] = FASTTRAP_REX(1, 0, 0, 0);
+				
+				if (tp->ftt_ripmode & FASTTRAP_RIP_1)
+					scratch[i++] = FASTTRAP_MOV_EAX;
+				else
+					scratch[i++] = FASTTRAP_MOV_ECX;
+				
+				switch (tp->ftt_ripmode) {
+					case FASTTRAP_RIP_1:
+						reg = &regs64->rax;
+						uthread->t_dtrace_reg = REG_RAX;
+						break;
+					case FASTTRAP_RIP_2:
+						reg = &regs64->rcx;
+						uthread->t_dtrace_reg = REG_RCX;
+						break;
+					case FASTTRAP_RIP_1 | FASTTRAP_RIP_X:
+						reg = &regs64->r8;
+						uthread->t_dtrace_reg = REG_R8;
+						break;
+					case FASTTRAP_RIP_2 | FASTTRAP_RIP_X:
+						reg = &regs64->r9;
+						uthread->t_dtrace_reg = REG_R9;
+						break;
+					default:
+						reg = NULL;
+						panic("unhandled ripmode in fasttrap_pid_probe64");
+				}
+				
+				*(uint64_t *)&scratch[i] = *reg;
+				uthread->t_dtrace_regv = *reg;
+				*reg = pc + tp->ftt_size;
+				i += sizeof (uint64_t);
+			}
+
+			/*
+			 * Generate the branch instruction to what would have
+			 * normally been the subsequent instruction. In 32-bit mode,
+			 * this is just a relative branch; in 64-bit mode this is a
+			 * %rip-relative branch that loads the 64-bit pc value
+			 * immediately after the jmp instruction.
+			 */
+			scratch[i++] = FASTTRAP_GROUP5_OP;
+			scratch[i++] = FASTTRAP_MODRM(0, 4, 5);
+			*(uint32_t *)&scratch[i] = 0;
+			i += sizeof (uint32_t);
+			*(uint64_t *)&scratch[i] = pc + tp->ftt_size;
+			i += sizeof (uint64_t);
+
+			uthread->t_dtrace_astpc = addr + i;
+			bcopy(tp->ftt_instr, &scratch[i], tp->ftt_size);
+			i += tp->ftt_size;
+			scratch[i++] = FASTTRAP_INT;
+			scratch[i++] = T_DTRACE_RET;
+
+			if (fasttrap_copyout(scratch, addr, i)) {
+				fasttrap_sigtrap(p, uthread, pc);
+				new_pc = pc;
+				break;
+			}
+
+			if (tp->ftt_retids != NULL) {
+				uthread->t_dtrace_step = 1;
+				uthread->t_dtrace_ret = 1;
+				new_pc = uthread->t_dtrace_astpc;
+			} else {
+				new_pc = uthread->t_dtrace_scrpc;
+			}
+			
+			uthread->t_dtrace_pc = pc;
+			uthread->t_dtrace_npc = pc + tp->ftt_size;
+			uthread->t_dtrace_on = 1;
+			break;
+		}
+		
+		default:
+			panic("fasttrap: mishandled an instruction");
+	}
+	
+done:
+	/*
+	 * APPLE NOTE:
+	 *
+	 * We're setting this earlier than Solaris does, to get a "correct"
+	 * ustack() output. In the Sun code,  a() -> b() -> c() -> d() is
+	 * reported at: d, b, a. The new way gives c, b, a, which is closer
+	 * to correct, as the return instruction has already exectued.
+	 */
+	regs64->isf.rip = new_pc;
+
+
+	/*
+	 * If there were no return probes when we first found the tracepoint,
+	 * we should feel no obligation to honor any return probes that were
+	 * subsequently enabled -- they'll just have to wait until the next
+	 * time around.
+	 */
+	if (tp->ftt_retids != NULL) {
+		/*
+		 * We need to wait until the results of the instruction are
+		 * apparent before invoking any return probes. If this
+		 * instruction was emulated we can just call
+		 * fasttrap_return_common(); if it needs to be executed, we
+		 * need to wait until the user thread returns to the kernel.
+		 */
+		if (tp->ftt_type != FASTTRAP_T_COMMON) {
+			fasttrap_return_common(regs, pc, pid, new_pc);
+		} else {
+			ASSERT(uthread->t_dtrace_ret != 0);
+			ASSERT(uthread->t_dtrace_pc == pc);
+			ASSERT(uthread->t_dtrace_scrpc != 0);
+			ASSERT(new_pc == uthread->t_dtrace_astpc);
+		}
+	}
+
+	return (0);
+}
+
+int
+fasttrap_pid_probe(x86_saved_state_t *regs)
+{
+        if (is_saved_state64(regs))
+		return fasttrap_pid_probe64(regs);
+
+	return fasttrap_pid_probe32(regs);
+}
+
+int
+fasttrap_return_probe(x86_saved_state_t *regs)
+{
+	x86_saved_state64_t *regs64;
+	x86_saved_state32_t *regs32;
+	unsigned int p_model;
+
+        if (is_saved_state64(regs)) {
+                regs64 = saved_state64(regs);
+		regs32 = NULL;
+		p_model = DATAMODEL_LP64;
+        } else {
+		regs64 = NULL;
+                regs32 = saved_state32(regs);
+		p_model = DATAMODEL_ILP32;
+        }
+
+	proc_t *p = current_proc();
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+	user_addr_t pc = uthread->t_dtrace_pc;
+	user_addr_t npc = uthread->t_dtrace_npc;
+
+	uthread->t_dtrace_pc = 0;
+	uthread->t_dtrace_npc = 0;
+	uthread->t_dtrace_scrpc = 0;
+	uthread->t_dtrace_astpc = 0;
+
+	/*
+	 * Treat a child created by a call to vfork(2) as if it were its
+	 * parent. We know that there's only one thread of control in such a
+	 * process: this one.
+	 */
+	/*
+	 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
+	 * How do we assert this?
+	 */
+	while (p->p_lflag & P_LINVFORK) {
+		p = p->p_pptr;
+	}
+
+	/*
+	 * We set rp->r_pc to the address of the traced instruction so
+	 * that it appears to dtrace_probe() that we're on the original
+	 * instruction, and so that the user can't easily detect our
+	 * complex web of lies. dtrace_return_probe() (our caller)
+	 * will correctly set %pc after we return.
+	 */
+	if (p_model == DATAMODEL_LP64)
+		regs64->isf.rip = pc;
+	else
+		regs32->eip = pc;
+
+	fasttrap_return_common(regs, pc, p->p_pid, npc);
+
+	return (0);
+}
+
+uint64_t
+fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+#pragma unused(arg, id, parg, aframes)
+	return (fasttrap_anarg((x86_saved_state_t *)find_user_regs(current_thread()), 1, argno));
+}
+
+uint64_t
+fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+#pragma unused(arg, id, parg, aframes)
+	return (fasttrap_anarg((x86_saved_state_t *)find_user_regs(current_thread()), 0, argno));
+}
+
+/*
+ * APPLE NOTE: See comments by regmap array definition. We are cheating
+ * when returning 32 bit registers.
+ */
+static user_addr_t
+fasttrap_getreg(x86_saved_state_t *regs, uint_t reg)
+{
+	if (is_saved_state64(regs)) {
+		x86_saved_state64_t *regs64 = saved_state64(regs);
+
+		switch (reg) {
+			case REG_RAX:		return regs64->rax;
+			case REG_RCX:		return regs64->rcx;
+			case REG_RDX:		return regs64->rdx;
+			case REG_RBX:		return regs64->rbx;
+			case REG_RSP:		return regs64->isf.rsp;
+			case REG_RBP:		return regs64->rbp;
+			case REG_RSI:		return regs64->rsi;
+			case REG_RDI:		return regs64->rdi;
+			case REG_R8:		return regs64->r8;
+			case REG_R9:		return regs64->r9;
+			case REG_R10:		return regs64->r10;
+			case REG_R11:		return regs64->r11;
+			case REG_R12:		return regs64->r12;
+			case REG_R13:		return regs64->r13;
+			case REG_R14:		return regs64->r14;
+			case REG_R15:		return regs64->r15;
+		}
+
+		panic("dtrace: unhandled x86_64 getreg() constant");
+	} else {
+		x86_saved_state32_t *regs32 = saved_state32(regs);
+
+		switch (reg) {
+			case REG_RAX:		return regs32->eax;
+			case REG_RCX:		return regs32->ecx;
+			case REG_RDX:		return regs32->edx;
+			case REG_RBX:		return regs32->ebx;
+			case REG_RSP:		return regs32->uesp;
+			case REG_RBP:		return regs32->ebp;
+			case REG_RSI:		return regs32->esi;
+			case REG_RDI:		return regs32->edi;
+		}
+
+		panic("dtrace: unhandled i386 getreg() constant");
+	}
+
+	return 0;
+}
diff --git a/bsd/dev/i386/fasttrap_regset.h b/bsd/dev/i386/fasttrap_regset.h
new file mode 100644
index 000000000..805fa83c4
--- /dev/null
+++ b/bsd/dev/i386/fasttrap_regset.h
@@ -0,0 +1,122 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
+
+/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T		*/
+/*	All Rights Reserved	*/
+
+#ifndef	_FASTTRAP_REGSET_H
+#define	_FASTTRAP_REGSET_H
+
+/*
+ * APPLE NOTE: This file was orginally uts/intel/sys/regset.h
+ */
+
+/*
+ * #pragma ident	"@(#)regset.h	1.11	05/06/08 SMI"
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * The names and offsets defined here should be specified by the
+ * AMD64 ABI suppl.
+ *
+ * We make fsbase and gsbase part of the lwp context (since they're
+ * the only way to access the full 64-bit address range via the segment
+ * registers) and thus belong here too.  However we treat them as
+ * read-only; if %fs or %gs are updated, the results of the descriptor
+ * table lookup that those updates implicitly cause will be reflected
+ * in the corresponding fsbase and/or gsbase values the next time the
+ * context can be inspected.  However it is NOT possible to override
+ * the fsbase/gsbase settings via this interface.
+ *
+ * Direct modification of the base registers (thus overriding the
+ * descriptor table base address) can be achieved with _lwp_setprivate.
+ */
+
+#define	REG_GSBASE	27
+#define	REG_FSBASE	26
+#define	REG_DS		25
+#define	REG_ES		24
+
+#define	REG_GS		23
+#define	REG_FS		22
+#define	REG_SS		21
+#define	REG_RSP		20
+#define	REG_RFL		19
+#define	REG_CS		18
+#define	REG_RIP		17
+#define	REG_ERR		16
+#define	REG_TRAPNO	15
+#define	REG_RAX		14
+#define	REG_RCX		13
+#define	REG_RDX		12
+#define	REG_RBX		11
+#define	REG_RBP		10
+#define	REG_RSI		9
+#define	REG_RDI		8
+#define	REG_R8		7
+#define	REG_R9		6
+#define	REG_R10		5
+#define	REG_R11		4
+#define	REG_R12		3
+#define	REG_R13		2
+#define	REG_R14		1
+#define	REG_R15		0
+
+/*
+ * The names and offsets defined here are specified by i386 ABI suppl.
+ */
+
+#define	SS		18	/* only stored on a privilege transition */
+#define	UESP		17	/* only stored on a privilege transition */
+#define	EFL		16
+#define	CS		15
+#define	EIP		14
+#define	ERR		13
+#define	TRAPNO		12
+#define	EAX		11
+#define	ECX		10
+#define	EDX		9
+#define	EBX		8
+#define	ESP		7
+#define	EBP		6
+#define	ESI		5
+#define	EDI		4
+#define	DS		3
+#define	ES		2
+#define	FS		1
+#define	GS		0
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _FASTTRAP_REGSET_H */
diff --git a/bsd/dev/i386/fbt_x86.c b/bsd/dev/i386/fbt_x86.c
new file mode 100644
index 000000000..219224f80
--- /dev/null
+++ b/bsd/dev/i386/fbt_x86.c
@@ -0,0 +1,824 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)fbt.c	1.15	05/09/19 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+#include <mach/vm_param.h>
+#include <mach-o/loader.h> 
+#include <mach-o/nlist.h>
+
+extern struct mach_header _mh_execute_header; /* the kernel's mach header */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/fbt.h>
+
+#include <sys/dtrace_glue.h>
+
+#define DTRACE_INVOP_NOP_SKIP 1
+#define DTRACE_INVOP_MOVL_ESP_EBP 10
+#define DTRACE_INVOP_MOVL_ESP_EBP_SKIP 2
+#define DTRACE_INVOP_LEAVE_SKIP 1
+
+#define	FBT_PUSHL_EBP			0x55
+#define	FBT_MOVL_ESP_EBP0_V0	0x8b
+#define	FBT_MOVL_ESP_EBP1_V0	0xec
+#define	FBT_MOVL_ESP_EBP0_V1	0x89
+#define	FBT_MOVL_ESP_EBP1_V1	0xe5
+#define	FBT_REX_RSP_RBP			0x48
+
+#define	FBT_POPL_EBP			0x5d
+#define	FBT_RET					0xc3
+#define	FBT_RET_IMM16			0xc2
+#define	FBT_LEAVE				0xc9
+#define	FBT_JMP_SHORT_REL		0xeb /* Jump short, relative, displacement relative to next instr. */
+#define	FBT_JMP_NEAR_REL		0xe9 /* Jump near, relative, displacement relative to next instr. */
+#define	FBT_JMP_FAR_ABS			0xea /* Jump far, absolute, address given in operand */
+#define FBT_RET_LEN				1
+#define FBT_RET_IMM16_LEN		3
+#define	FBT_JMP_SHORT_REL_LEN	2
+#define	FBT_JMP_NEAR_REL_LEN	5
+#define	FBT_JMP_FAR_ABS_LEN		5
+
+#define	FBT_PATCHVAL			0xf0
+#define FBT_AFRAMES_ENTRY		7
+#define FBT_AFRAMES_RETURN		6
+
+#define	FBT_ENTRY	"entry"
+#define	FBT_RETURN	"return"
+#define	FBT_ADDR2NDX(addr)	((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
+
+extern dtrace_provider_id_t	fbt_id;
+extern fbt_probe_t		**fbt_probetab;
+extern int			fbt_probetab_mask;
+
+/*
+ * Critical routines that must not be probed. PR_5221096, PR_5379018.
+ */
+
+static const char * critical_blacklist[] =
+{
+	"bcopy_phys",
+	"console_cpu_alloc", 
+	"console_cpu_free", 
+	"cpu_IA32e_disable", 
+	"cpu_IA32e_enable", 
+	"cpu_control", 
+	"cpu_data_alloc", 
+	"cpu_desc_init", 
+	"cpu_desc_init64", 
+	"cpu_desc_load64", 
+	"cpu_exit_wait", 
+	"cpu_info", 
+	"cpu_info_count", 
+	"cpu_init", 
+	"cpu_interrupt", 
+	"cpu_machine_init", 
+	"cpu_mode_init", 
+	"cpu_processor_alloc", 
+	"cpu_processor_free", 
+	"cpu_signal_handler", 
+	"cpu_sleep", 
+	"cpu_start", 
+	"cpu_subtype", 
+	"cpu_thread_alloc", 
+	"cpu_thread_halt", 
+	"cpu_thread_init", 
+	"cpu_threadtype", 
+	"cpu_to_processor", 
+	"cpu_topology_start", 
+	"cpu_type", 
+	"cpu_window_init", 
+	"cpuid_cpu_display",
+	"handle_pending_TLB_flushes",
+	"hw_compare_and_store",
+	"machine_idle_cstate",
+	"mca_cpu_alloc",
+	"mca_cpu_init",
+	"ml_nofault_copy",
+	"pmap_cpu_alloc", 
+	"pmap_cpu_free", 
+	"pmap_cpu_high_map_vaddr", 
+	"pmap_cpu_high_shared_remap", 
+	"pmap_cpu_init",
+	"rdHPET",
+	"register_cpu_setup_func",
+	"unregister_cpu_setup_func"
+};
+#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0]))
+
+/*
+ * The transitive closure of entry points that can be reached from probe context.
+ * (Apart from routines whose names begin with dtrace_ or dtxnu_.)
+ */
+static const char * probe_ctx_closure[] =
+{
+	"Debugger",
+	"OSCompareAndSwap",
+	"absolutetime_to_microtime",
+	"ast_pending",
+	"clock_get_calendar_nanotime_nowait",
+	"copyin",
+	"copyin_user",
+	"copyinstr",
+	"copyout",
+	"copyoutstr",
+	"cpu_number",
+	"current_proc",
+	"current_processor",
+	"current_task",
+	"current_thread",
+	"debug_enter",
+	"find_user_regs",
+	"flush_tlb64",
+	"get_bsdtask_info",
+	"get_bsdthread_info",
+	"hw_atomic_and",
+	"kauth_cred_get",
+	"kauth_getgid",
+	"kauth_getuid",
+	"kernel_preempt_check",
+	"mach_absolute_time",
+	"max_valid_stack_address",
+	"ml_at_interrupt_context",
+	"ml_phys_write_byte_64",
+	"ml_phys_write_half_64",
+	"ml_phys_write_word_64",
+	"ml_set_interrupts_enabled",
+	"panic",
+	"pmap64_pde",
+	"pmap64_pdpt",
+	"pmap_find_phys",
+	"pmap_get_mapwindow",
+	"pmap_pde",
+	"pmap_pte",
+	"pmap_put_mapwindow",
+	"pmap_valid_page",
+	"prf",
+	"proc_is64bit",
+	"proc_selfname",
+	"proc_selfpid",
+	"psignal_lock",
+	"rtc_nanotime_load",
+	"rtc_nanotime_read",
+	"strlcpy",
+	"sync_iss_to_iks_unconditionally",
+	"timer_grab"
+};
+#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0]))
+
+
+static int _cmp(const void *a, const void *b)
+{
+	return strcmp((const char *)a, *(const char **)b);
+}
+
+static const void * bsearch(
+	register const void *key,
+	const void *base0,
+	size_t nmemb,
+	register size_t size,
+	register int (*compar)(const void *, const void *)) {
+
+	register const char *base = base0;
+	register size_t lim;
+	register int cmp;
+	register const void *p;
+
+	for (lim = nmemb; lim != 0; lim >>= 1) {
+		p = base + (lim >> 1) * size;
+		cmp = (*compar)(key, p);
+		if (cmp == 0)
+			return p;
+		if (cmp > 0) {	/* key > p: move right */
+			base = (const char *)p + size;
+			lim--;
+		}		/* else move left */
+	}
+	return (NULL);
+}
+
+int
+fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
+{
+	uintptr_t stack0 = 0, stack1 = 0, stack2 = 0, stack3 = 0, stack4 = 0;
+	fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
+
+	for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
+		if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
+
+			if (fbt->fbtp_roffset == 0) {
+				uintptr_t *stacktop;
+        			if (CPU_ON_INTR(CPU))
+                			stacktop = (uintptr_t *)dtrace_get_cpu_int_stack_top();
+        			else
+                			stacktop = (uintptr_t *)(dtrace_get_kernel_stack(current_thread()) + KERNEL_STACK_SIZE);
+
+				stack += 1; /* skip over the target's pushl'd %ebp */
+
+				if (stack <= stacktop)
+					CPU->cpu_dtrace_caller = *stack++;
+				if (stack <= stacktop)
+					stack0 = *stack++;
+				if (stack <= stacktop)
+					stack1 = *stack++;
+				if (stack <= stacktop)
+					stack2 = *stack++;
+				if (stack <= stacktop)
+					stack3 = *stack++;
+				if (stack <= stacktop)
+					stack4 = *stack++;
+
+				dtrace_probe(fbt->fbtp_id, stack0, stack1, stack2, stack3, stack4);
+				CPU->cpu_dtrace_caller = 0;
+			} else {
+				dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
+				CPU->cpu_dtrace_caller = 0;
+			}
+
+			return (fbt->fbtp_rval);
+		}
+	}
+
+	return (0);
+}
+
+#define IS_USER_TRAP(regs) (regs && (((regs)->cs & 3) != 0))
+#define T_INVALID_OPCODE 6
+#define FBT_EXCEPTION_CODE T_INVALID_OPCODE
+
+kern_return_t
+fbt_perfCallback(
+                int         		trapno,
+                x86_saved_state_t 	*tagged_regs,
+                __unused int        unused1,
+                __unused int        unused2)
+{
+	kern_return_t retval = KERN_FAILURE;
+	x86_saved_state32_t *saved_state = saved_state32(tagged_regs);
+	struct x86_saved_state32_from_kernel *regs = (struct x86_saved_state32_from_kernel *)saved_state;
+
+	if (FBT_EXCEPTION_CODE == trapno && !IS_USER_TRAP(saved_state)) {
+		boolean_t oldlevel, cpu_64bit;
+		uint32_t esp_probe, *ebp, edi, fp, *pDst, delta = 0;
+		int emul;
+
+		cpu_64bit = ml_is64bit();
+		oldlevel = ml_set_interrupts_enabled(FALSE);
+
+		/* Calculate where the stack pointer was when the probe instruction "fired." */
+		if (cpu_64bit) {
+			esp_probe = saved_state->uesp; /* Easy, x86_64 establishes this value in idt64.s */
+		} else {
+			esp_probe = (uint32_t)&(regs[1]); /* Nasty, infer the location above the save area */
+		}
+
+		emul = dtrace_invop( saved_state->eip, (uintptr_t *)esp_probe, saved_state->eax );
+		__asm__ volatile(".globl _dtrace_invop_callsite");
+		__asm__ volatile("_dtrace_invop_callsite:");
+
+		switch (emul) {
+		case DTRACE_INVOP_NOP:
+			saved_state->eip += DTRACE_INVOP_NOP_SKIP;	/* Skip over the patched NOP */
+			retval = KERN_SUCCESS;
+			break;
+
+		case DTRACE_INVOP_MOVL_ESP_EBP:
+			saved_state->ebp = esp_probe;						/* Emulate patched movl %esp,%ebp */
+			saved_state->eip += DTRACE_INVOP_MOVL_ESP_EBP_SKIP;	/* Skip over the bytes of the patched movl %esp,%ebp */
+			retval = KERN_SUCCESS;
+			break;
+
+		case DTRACE_INVOP_POPL_EBP:
+		case DTRACE_INVOP_LEAVE:
+/*
+ * Emulate first micro-op of patched leave: movl %ebp,%esp
+ * fp points just below the return address slot for target's ret 
+ * and at the slot holding the frame pointer saved by the target's prologue.
+ */
+			fp = saved_state->ebp;
+/* Emulate second micro-op of patched leave: patched popl %ebp
+ * savearea ebp is set for the frame of the caller to target
+ * The *live* %esp will be adjusted below for pop increment(s)
+ */
+			saved_state->ebp = *(uint32_t *)fp;
+/* Skip over the patched leave */
+			saved_state->eip += DTRACE_INVOP_LEAVE_SKIP;
+/*
+ * Lift the stack to account for the emulated leave
+ * Account for words local in this frame
+ * (in "case DTRACE_INVOP_POPL_EBP:" this is zero.)
+ */
+			delta = ((uint32_t *)fp) - ((uint32_t *)esp_probe);
+/* Account for popping off the ebp (just accomplished by the emulation
+ * above...)
+ */
+			delta += 1;
+			
+			if (cpu_64bit)
+				saved_state->uesp += (delta << 2);
+
+/* XXX Fragile in the extreme. Obtain the value of %edi that our caller pushed
+ * (on behalf of its caller -- trap_from_kernel()). Ultimately,
+ * trap_from_kernel's stack pointer is restored from this slot.
+ * This is sensitive to the manner in which the compiler preserves %edi,
+ * and trap_from_kernel()'s internals.
+ */
+			ebp = (uint32_t *)__builtin_frame_address(0);
+			ebp = (uint32_t *)*ebp;
+			edi = *(ebp - 1);
+/* Shift contents of stack */
+			for (pDst = (uint32_t *)fp;
+			     pDst > (((uint32_t *)edi));
+				 pDst--)
+				*pDst = pDst[-delta];
+/* Now adjust the value of %edi in our caller (kernel_trap)'s frame */
+			*(ebp - 1) = edi + (delta << 2);
+
+			retval = KERN_SUCCESS;
+			break;
+			
+		default:
+			retval = KERN_FAILURE;
+			break;
+		}
+		ml_set_interrupts_enabled(oldlevel);
+	}
+
+	return retval;
+}
+
+/*ARGSUSED*/
+static void
+__fbt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(arg)
+	struct mach_header			*mh;
+	struct load_command         *cmd;
+	struct segment_command      *orig_ts = NULL, *orig_le = NULL;
+	struct symtab_command       *orig_st = NULL;
+	struct nlist                *sym = NULL;
+	char						*strings;
+	uintptr_t					instrLow, instrHigh;
+	char						*modname;
+	unsigned int i, j;
+
+	int gIgnoreFBTBlacklist = 0;
+    PE_parse_boot_arg("IgnoreFBTBlacklist", &gIgnoreFBTBlacklist);
+
+	mh = (struct mach_header *)(ctl->address);
+	modname = ctl->mod_modname;
+
+	if (0 == ctl->address || 0 == ctl->size) /* Has the linker been jettisoned? */
+		return;
+
+	/*
+	 * Employees of dtrace and their families are ineligible.  Void
+	 * where prohibited.
+	 */
+
+	if (strcmp(modname, "com.apple.driver.dtrace") == 0)
+		return;
+
+	if (strstr(modname, "CHUD") != NULL)
+		return;
+
+	if (mh->magic != MH_MAGIC)
+		return;
+
+	cmd = (struct load_command *) &mh[1];
+	for (i = 0; i < mh->ncmds; i++) {
+		if (cmd->cmd == LC_SEGMENT) {
+			struct segment_command *orig_sg = (struct segment_command *) cmd;
+
+			if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
+				orig_ts = orig_sg;
+			else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
+				orig_le = orig_sg;
+			else if (strcmp("", orig_sg->segname) == 0)
+				orig_ts = orig_sg; /* kexts have a single unnamed segment */
+		}
+		else if (cmd->cmd == LC_SYMTAB)
+			orig_st = (struct symtab_command *) cmd;
+
+		cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
+	}
+
+	if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
+		return;
+
+	sym = (struct nlist *)orig_le->vmaddr;
+	strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist);
+
+	/* Find extent of the TEXT section */
+	instrLow = (uintptr_t)orig_ts->vmaddr;
+	instrHigh = (uintptr_t)(orig_ts->vmaddr + orig_ts->vmsize);
+
+	for (i = 0; i < orig_st->nsyms; i++) {
+		fbt_probe_t *fbt, *retfbt;
+		machine_inst_t *instr, *limit, theInstr, i1, i2;
+		uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
+		char *name = strings + sym[i].n_un.n_strx;
+		int size;
+
+		/* Check that the symbol is a global and that it has a name. */
+		if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
+			continue;
+
+		if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
+			continue;
+		
+		/* Lop off omnipresent leading underscore. */			
+		if (*name == '_')
+			name += 1;
+
+		if (strstr(name, "dtrace_") == name &&
+		    strstr(name, "dtrace_safe_") != name) {
+			/*
+			 * Anything beginning with "dtrace_" may be called
+			 * from probe context unless it explitly indicates
+			 * that it won't be called from probe context by
+			 * using the prefix "dtrace_safe_".
+			 */
+			continue;
+		}
+
+		if (strstr(name, "dsmos_") == name) 
+			continue; /* Don't Steal Mac OS X! */
+
+        if (strstr(name, "dtxnu_") == name ||
+			strstr(name, "_dtrace") == name)
+			continue; /* Shims in dtrace.c */
+
+		if (strstr(name, "chud") == name)
+			continue; /* Professional courtesy. */
+		
+		if (strstr(name, "hibernate_") == name)
+			continue; /* Let sleeping dogs lie. */
+		
+		if (0 == strcmp(name, "ZN9IOService14newTemperatureElPS_") || /* IOService::newTemperature */
+			0 == strcmp(name, "ZN9IOService26temperatureCriticalForZoneEPS_")) /* IOService::temperatureCriticalForZone */
+			continue; /* Per the fire code */
+
+		/*
+		 * Place no probes (illegal instructions) in the exception handling path!
+		 */
+		if (0 == strcmp(name, "t_invop") ||
+			0 == strcmp(name, "enter_lohandler") ||
+			0 == strcmp(name, "lo_alltraps") ||
+			0 == strcmp(name, "kernel_trap") ||
+			0 == strcmp(name, "i386_astintr"))
+			continue;
+
+		if (0 == strcmp(name, "current_thread") ||
+			0 == strcmp(name, "ast_pending") ||
+			0 == strcmp(name, "fbt_perfCallback") ||
+			0 == strcmp(name, "machine_thread_get_kern_state") ||
+			0 == strcmp(name, "ml_set_interrupts_enabled") ||
+			0 == strcmp(name, "dtrace_invop") ||
+			0 == strcmp(name, "fbt_invop") ||
+			0 == strcmp(name, "sdt_invop") ||
+			0 == strcmp(name, "max_valid_stack_address"))
+			continue;
+
+		/*
+		 * Voodoo.
+		 */
+		if (strstr(name, "machine_stack_") == name ||
+			strstr(name, "mapping_") == name ||
+			strstr(name, "hpet_") == name ||
+
+			0 == strcmp(name, "rdHPET") ||
+			0 == strcmp(name, "HPETInterrupt") ||
+			0 == strcmp(name, "tmrCvt") ||
+
+			strstr(name, "tsc_") == name ||
+
+			strstr(name, "pmCPU") == name ||
+			0 == strcmp(name, "Cstate_table_set") ||
+			0 == strcmp(name, "pmHPETInterrupt") ||
+			0 == strcmp(name, "pmKextRegister") ||
+			0 == strcmp(name, "pmSafeMode") ||
+			0 == strcmp(name, "pmUnregister") ||
+			strstr(name, "pms") == name ||
+			0 == strcmp(name, "power_management_init") ||
+			strstr(name, "usimple_") == name ||
+
+			strstr(name, "rtc_") == name ||
+			strstr(name, "_rtc_") == name ||
+			strstr(name, "rtclock_") == name ||
+			strstr(name, "clock_") == name ||
+			strstr(name, "absolutetime_to_") == name ||
+			0 == strcmp(name, "setPop") ||
+			0 == strcmp(name, "nanoseconds_to_absolutetime") ||
+			0 == strcmp(name, "nanotime_to_absolutetime") ||
+
+			strstr(name, "etimer_") == name ||
+
+			strstr(name, "commpage_") == name ||
+			strstr(name, "pmap_") == name ||
+			strstr(name, "ml_") == name ||
+			strstr(name, "PE_") == name ||
+			strstr(name, "lapic_") == name ||
+			strstr(name, "acpi_") == name)
+			continue;
+
+        /*
+         * Avoid machine_ routines. PR_5346750.
+         */
+        if (strstr(name, "machine_") == name)
+            continue;
+
+		if (0 == strcmp(name, "handle_pending_TLB_flushes"))
+			continue;
+
+        /*
+         * Place no probes on critical routines. PR_5221096
+         */
+        if (!gIgnoreFBTBlacklist &&
+            bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL)
+                continue;
+
+        /*
+		 * Place no probes that could be hit in probe context.
+		 */
+		if (!gIgnoreFBTBlacklist && 
+			bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL)
+			continue;
+
+		/*
+		 * Place no probes that could be hit on the way to the debugger.
+		 */
+		if (strstr(name, "kdp_") == name ||
+			strstr(name, "kdb_") == name ||
+			strstr(name, "kdbg_") == name ||
+			strstr(name, "kdebug_") == name ||
+			0 == strcmp(name, "kernel_debug") ||
+			0 == strcmp(name, "Debugger") ||
+			0 == strcmp(name, "Call_DebuggerC") ||
+			0 == strcmp(name, "lock_debugger") ||
+			0 == strcmp(name, "unlock_debugger") ||
+			0 == strcmp(name, "SysChoked")) 
+			continue;
+
+		/*
+		 * Place no probes that could be hit on the way to a panic.
+		 */
+		if (NULL != strstr(name, "panic_") ||
+			0 == strcmp(name, "panic") ||
+			0 == strcmp(name, "handleMck") ||
+			0 == strcmp(name, "unresolved_kernel_trap"))
+			continue;
+		
+		if (dtrace_probe_lookup(fbt_id, modname, name, NULL) != 0)
+			continue;
+
+		for (j = 0, instr = (machine_inst_t *)sym[i].n_value, theInstr = 0;
+			 (j < 4) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)(instr + 2)); 
+			 j++) {
+			theInstr = instr[0];
+			if (theInstr == FBT_PUSHL_EBP || theInstr == FBT_RET || theInstr == FBT_RET_IMM16)
+				break;
+
+			if ((size = dtrace_instr_size(instr)) <= 0)
+				break;
+ 
+			instr += size;
+		}
+
+		if (theInstr != FBT_PUSHL_EBP)
+			continue;
+
+		i1 = instr[1];
+		i2 = instr[2];
+
+		limit = (machine_inst_t *)instrHigh;
+
+		if ((i1 == FBT_MOVL_ESP_EBP0_V0 && i2 == FBT_MOVL_ESP_EBP1_V0) ||
+			(i1 == FBT_MOVL_ESP_EBP0_V1 && i2 == FBT_MOVL_ESP_EBP1_V1)) {
+				instr += 1; /* Advance to the movl %esp,%ebp */
+				theInstr = i1;
+		} else {
+			/*
+			 * Sometimes, the compiler will schedule an intervening instruction
+			 * in the function prologue. Example:
+			 *
+			 * _mach_vm_read:
+			 * 000006d8        pushl   %ebp
+			 * 000006d9        movl    $0x00000004,%edx
+			 * 000006de        movl    %esp,%ebp
+			 * 
+			 * Try the next instruction, to see if it is a movl %esp,%ebp
+			 */
+
+			instr += 1; /* Advance past the pushl %ebp */
+			if ((size = dtrace_instr_size(instr)) <= 0)
+				continue;
+ 
+			instr += size;
+
+			if ((instr + 1) >= limit)
+				continue;
+
+			i1 = instr[0];
+			i2 = instr[1];
+
+			if (!(i1 == FBT_MOVL_ESP_EBP0_V0 && i2 == FBT_MOVL_ESP_EBP1_V0) &&
+				!(i1 == FBT_MOVL_ESP_EBP0_V1 && i2 == FBT_MOVL_ESP_EBP1_V1))
+				continue;
+
+			/* instr already points at the movl %esp,%ebp */
+			theInstr = i1;
+		}
+
+		fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
+		strlcpy( (char *)&(fbt->fbtp_name), name, MAX_FBTP_NAME_CHARS );
+		fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, FBT_AFRAMES_ENTRY, fbt);
+		fbt->fbtp_patchpoint = instr;
+		fbt->fbtp_ctl = ctl;
+		fbt->fbtp_loadcnt = ctl->mod_loadcnt;
+		fbt->fbtp_rval = DTRACE_INVOP_MOVL_ESP_EBP;
+		fbt->fbtp_savedval = theInstr;
+		fbt->fbtp_patchval = FBT_PATCHVAL;
+
+		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
+		fbt->fbtp_symndx = i;
+		fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
+
+		retfbt = NULL;
+again:
+		if (instr >= limit)
+			continue;
+
+		/*
+		 * If this disassembly fails, then we've likely walked off into
+		 * a jump table or some other unsuitable area.  Bail out of the
+		 * disassembly now.
+		 */
+		if ((size = dtrace_instr_size(instr)) <= 0)
+			continue;
+
+		/*
+		 * We (desperately) want to avoid erroneously instrumenting a
+		 * jump table, especially given that our markers are pretty
+		 * short:  two bytes on x86, and just one byte on amd64.  To
+		 * determine if we're looking at a true instruction sequence
+		 * or an inline jump table that happens to contain the same
+		 * byte sequences, we resort to some heuristic sleeze:  we
+		 * treat this instruction as being contained within a pointer,
+		 * and see if that pointer points to within the body of the
+		 * function.  If it does, we refuse to instrument it.
+		 */
+		for (j = 0; j < sizeof (uintptr_t); j++) {
+			uintptr_t check = (uintptr_t)instr - j;
+			uint8_t *ptr;
+
+			if (check < sym[i].n_value)
+				break;
+
+			if (check + sizeof (uintptr_t) > (uintptr_t)limit)
+				continue;
+
+			ptr = *(uint8_t **)check;
+
+			if (ptr >= (uint8_t *)sym[i].n_value && ptr < limit) {
+				instr += size;
+				goto again;
+			}
+		}
+
+		/*
+		 * OK, it's an instruction.
+		 */
+		theInstr = instr[0];
+
+		/* Walked onto the start of the next routine? If so, bail out of this function. */
+		if (theInstr == FBT_PUSHL_EBP)
+			continue;
+
+		if (!(size == 1 && (theInstr == FBT_POPL_EBP || theInstr == FBT_LEAVE))) {
+			instr += size;
+			goto again;
+		}
+
+		/*
+		 * Found the popl %ebp; or leave.
+		 */
+		machine_inst_t *patch_instr = instr;
+
+		/*
+		 * Scan forward for a "ret", or "jmp".
+		 */
+		instr += size;
+		if (instr >= limit)
+			continue;
+
+		size = dtrace_instr_size(instr);
+		if (size <= 0) /* Failed instruction decode? */
+			continue;
+
+		theInstr = instr[0];
+
+		if (!(size == FBT_RET_LEN && (theInstr == FBT_RET)) &&
+			!(size == FBT_RET_IMM16_LEN && (theInstr == FBT_RET_IMM16)) &&
+			!(size == FBT_JMP_SHORT_REL_LEN && (theInstr == FBT_JMP_SHORT_REL)) &&
+			!(size == FBT_JMP_NEAR_REL_LEN && (theInstr == FBT_JMP_NEAR_REL)) &&
+			!(size == FBT_JMP_FAR_ABS_LEN && (theInstr == FBT_JMP_FAR_ABS)))
+			continue;
+
+		/*
+		 * popl %ebp; ret; or leave; ret; or leave; jmp tailCalledFun; -- We have a winner!
+		 */
+		fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
+		strlcpy( (char *)&(fbt->fbtp_name), name, MAX_FBTP_NAME_CHARS );
+
+		if (retfbt == NULL) {
+			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
+			    name, FBT_RETURN, FBT_AFRAMES_RETURN, fbt);
+		} else {
+			retfbt->fbtp_next = fbt;
+			fbt->fbtp_id = retfbt->fbtp_id;
+		}
+
+		retfbt = fbt;
+		fbt->fbtp_patchpoint = patch_instr;
+		fbt->fbtp_ctl = ctl;
+		fbt->fbtp_loadcnt = ctl->mod_loadcnt;
+
+		if (*patch_instr == FBT_POPL_EBP) {
+			fbt->fbtp_rval = DTRACE_INVOP_POPL_EBP;
+		} else {
+			ASSERT(*patch_instr == FBT_LEAVE);
+			fbt->fbtp_rval = DTRACE_INVOP_LEAVE;
+		}
+		fbt->fbtp_roffset =
+		    (uintptr_t)(patch_instr - (uint8_t *)sym[i].n_value);
+
+		fbt->fbtp_savedval = *patch_instr;
+		fbt->fbtp_patchval = FBT_PATCHVAL;
+		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(patch_instr)];
+		fbt->fbtp_symndx = i;
+		fbt_probetab[FBT_ADDR2NDX(patch_instr)] = fbt;
+
+		instr += size;
+		goto again;
+	}
+}
+
+extern struct modctl g_fbt_kernctl;
+#undef kmem_alloc /* from its binding to dt_kmem_alloc glue */
+#undef kmem_free /* from its binding to dt_kmem_free glue */
+#include <vm/vm_kern.h>
+
+/*ARGSUSED*/
+void
+fbt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(ctl)
+	__fbt_provide_module(arg, &g_fbt_kernctl);
+
+	kmem_free(kernel_map, (vm_offset_t)g_fbt_kernctl.address, round_page_32(g_fbt_kernctl.size));
+	g_fbt_kernctl.address = 0;
+	g_fbt_kernctl.size = 0;
+}
diff --git a/bsd/dev/i386/instr_size.c b/bsd/dev/i386/instr_size.c
new file mode 100644
index 000000000..2982b8d66
--- /dev/null
+++ b/bsd/dev/i386/instr_size.c
@@ -0,0 +1,138 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1988 AT&T	*/
+/*	  All Rights Reserved	*/
+
+
+/*
+ * #pragma ident	"@(#)instr_size.c	1.14	05/07/08 SMI"
+ */
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_glue.h>
+
+#include <sys/dis_tables.h>
+
+/*
+ * This subsystem (with the minor exception of the instr_size() function) is
+ * is called from DTrace probe context.  This imposes several requirements on
+ * the implementation:
+ *
+ * 1. External subsystems and functions may not be referenced.  The one current
+ *    exception is for cmn_err, but only to signal the detection of table
+ *    errors.  Assuming the tables are correct, no combination of input is to
+ *    trigger a cmn_err call.
+ *
+ * 2. These functions can't be allowed to be traced.  To prevent this,
+ *    all functions in the probe path (everything except instr_size()) must
+ *    have names that begin with "dtrace_".
+ */
+
+typedef enum dis_isize {
+	DIS_ISIZE_INSTR,
+	DIS_ISIZE_OPERAND
+} dis_isize_t;
+
+
+/*
+ * get a byte from instruction stream
+ */
+static int
+dtrace_dis_get_byte(void *p)
+{
+	int ret;
+	uchar_t **instr = p;
+
+	ret = **instr;
+	*instr += 1;
+
+	return (ret);
+}
+
+/*
+ * Returns either the size of a given instruction, in bytes, or the size of that
+ * instruction's memory access (if any), depending on the value of `which'.
+ * If a programming error in the tables is detected, the system will panic to
+ * ease diagnosis.  Invalid instructions will not be flagged.  They will appear
+ * to have an instruction size between 1 and the actual size, and will be
+ * reported as having no memory impact.
+ */
+/* ARGSUSED2 */
+static int
+dtrace_dis_isize(uchar_t *instr, dis_isize_t which, model_t model, int *rmindex)
+{
+	int sz;
+	dis86_t	x;
+	uint_t mode = SIZE32;
+
+	mode = (model == DATAMODEL_LP64) ? SIZE64 : SIZE32;
+
+	x.d86_data = (void **)&instr;
+	x.d86_get_byte = dtrace_dis_get_byte;
+	x.d86_check_func = NULL;
+
+	if (dtrace_disx86(&x, mode) != 0)
+		return (-1);
+
+	if (which == DIS_ISIZE_INSTR)
+		sz = x.d86_len;		/* length of the instruction */
+	else
+		sz = x.d86_memsize;	/* length of memory operand */
+
+	if (rmindex != NULL)
+		*rmindex = x.d86_rmindex;
+	return (sz);
+}
+
+int
+dtrace_instr_size_isa(uchar_t *instr, model_t model, int *rmindex)
+{
+	return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, model, rmindex));
+}
+
+int
+dtrace_instr_size(uchar_t *instr)
+{
+	return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, DATAMODEL_NATIVE,
+	    NULL));
+}
+
+#if !defined(__APPLE__)
+/*ARGSUSED*/
+int
+instr_size(struct regs *rp, caddr_t *addrp, enum seg_rw rw)
+{
+	uchar_t instr[16];	/* maximum size instruction */
+	caddr_t pc = (caddr_t)rp->r_pc;
+
+	(void) copyin_nowatch(pc, (caddr_t)instr, sizeof (instr));
+
+	return (dtrace_dis_isize(instr,
+	    rw == S_EXEC ? DIS_ISIZE_INSTR : DIS_ISIZE_OPERAND,
+	    curproc->p_model, NULL));
+}
+#endif /* __APPLE__ */
diff --git a/bsd/dev/i386/kern_machdep.c b/bsd/dev/i386/kern_machdep.c
index 6ff80a540..bb9851e2e 100644
--- a/bsd/dev/i386/kern_machdep.c
+++ b/bsd/dev/i386/kern_machdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (C) 1990,  NeXT, Inc.
@@ -43,14 +49,14 @@
  *		binary, either directly or via an imputed interpreter.
  **********************************************************************/
 int
-grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype)
+grade_binary(cpu_type_t exectype, __unused cpu_subtype_t execsubtype)
 {
 	switch(exectype) {
 	case CPU_TYPE_X86:		/* native */
 	case CPU_TYPE_POWERPC:		/* via translator */
 		return 1;
 	case CPU_TYPE_X86_64:		/* native 64-bit */
-		return (ml_is64bit() && execsubtype == CPU_SUBTYPE_X86_64_ALL) ? 2 : 0;
+		return ml_is64bit() ? 2 : 0;
 	default:			/* all other binary types */
 		return 0;
 	}
diff --git a/bsd/dev/i386/km.c b/bsd/dev/i386/km.c
index a75e4ea54..d5ecffc29 100644
--- a/bsd/dev/i386/km.c
+++ b/bsd/dev/i386/km.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 	Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. 
  *
diff --git a/bsd/dev/i386/lock_stubs.c b/bsd/dev/i386/lock_stubs.c
index 2b207335a..1362cd986 100644
--- a/bsd/dev/i386/lock_stubs.c
+++ b/bsd/dev/i386/lock_stubs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #if 0
 #define	_KERNEL
diff --git a/bsd/dev/i386/mem.c b/bsd/dev/i386/mem.c
index 7bcebdae6..9f4eeddde 100644
--- a/bsd/dev/i386/mem.c
+++ b/bsd/dev/i386/mem.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1988 University of Utah.
diff --git a/bsd/dev/i386/memmove.c b/bsd/dev/i386/memmove.c
index 5ef7f1291..58ca278a7 100644
--- a/bsd/dev/i386/memmove.c
+++ b/bsd/dev/i386/memmove.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991,1993 NeXT Computer, Inc.  All rights reserved.
  * 
diff --git a/bsd/dev/i386/munge.s b/bsd/dev/i386/munge.s
index 5a4dc52b2..18a3dc4bb 100644
--- a/bsd/dev/i386/munge.s
+++ b/bsd/dev/i386/munge.s
@@ -1,29 +1,35 @@
 /*
  * Coyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
  * Syscall argument mungers.
  *
- * The data to be munged has been explicitly copied in to the arguement area,
+ * The data to be munged has been explicitly copied in to the argument area,
  * and will be munged in place in the uu_arg[] array.  Because of this, the
  * functions all take the same arguments as their PPC equivalents, but the
  * first argument is ignored.  These mungers are for 32-bit app's syscalls,
diff --git a/bsd/dev/i386/pci_device.h b/bsd/dev/i386/pci_device.h
index 92f6a0905..6bb2dcd78 100644
--- a/bsd/dev/i386/pci_device.h
+++ b/bsd/dev/i386/pci_device.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/bsd/dev/i386/sdt_x86.c b/bsd/dev/i386/sdt_x86.c
new file mode 100644
index 000000000..eb1c2ecff
--- /dev/null
+++ b/bsd/dev/i386/sdt_x86.c
@@ -0,0 +1,84 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)sdt.c	1.6	06/03/24 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/cpu_data.h>
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+#include <mach/vm_param.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include <sys/dtrace_glue.h>
+
+#include <sys/sdt_impl.h>
+
+extern sdt_probe_t      **sdt_probetab;
+
+/*ARGSUSED*/
+int
+sdt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
+{
+#pragma unused(eax)
+	uintptr_t stack0 = 0, stack1 = 0, stack2 = 0, stack3 = 0, stack4 = 0;
+	sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)];
+
+	for (; sdt != NULL; sdt = sdt->sdp_hashnext) {
+		if ((uintptr_t)sdt->sdp_patchpoint == addr) {
+                        uintptr_t *stacktop;
+                        if (CPU_ON_INTR(CPU))
+                                stacktop = (uintptr_t *)dtrace_get_cpu_int_stack_top();
+                        else
+                                stacktop = (uintptr_t *)(dtrace_get_kernel_stack(current_thread()) + KERNEL_STACK_SIZE);
+
+            if (stack <= stacktop)
+                stack0 = *stack++;
+            if (stack <= stacktop)
+                stack1 = *stack++;
+            if (stack <= stacktop)
+                stack2 = *stack++;
+            if (stack <= stacktop)
+                stack3 = *stack++;
+            if (stack <= stacktop)
+                stack4 = *stack++;
+
+			dtrace_probe(sdt->sdp_id, stack0, stack1, stack2, stack3, stack4);
+
+			return (DTRACE_INVOP_NOP);
+		}
+	}
+
+	return (0);
+}
+
diff --git a/bsd/dev/i386/sel.h b/bsd/dev/i386/sel.h
index 3d8824286..4f4e4889b 100644
--- a/bsd/dev/i386/sel.h
+++ b/bsd/dev/i386/sel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/bsd/dev/i386/sel_inline.h b/bsd/dev/i386/sel_inline.h
index e972af7b5..39ab9b69f 100644
--- a/bsd/dev/i386/sel_inline.h
+++ b/bsd/dev/i386/sel_inline.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/bsd/dev/i386/stubs.c b/bsd/dev/i386/stubs.c
index d98a823d6..ddc759c29 100644
--- a/bsd/dev/i386/stubs.c
+++ b/bsd/dev/i386/stubs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved
@@ -40,7 +46,6 @@
 #include <machine/machine_routines.h>
 
 /* XXX should be elsewhere (cpeak) */
-extern struct proc *i386_current_proc(void);
 extern void	*get_bsduthreadarg(thread_t);
 extern int	*get_bsduthreadrval(thread_t);
 extern void	*find_user_regs(thread_t);
@@ -112,46 +117,11 @@ copywithin(void *src, void *dst, size_t count)
 	return 0;
 }
 
-/*
- * This is just current_proc() from bsd/kern/bsd_stubs.c, but instead of
- * returning kernproc in the non-vfork() case, it can return NULL.  This is
- * needed because the system call entry point is in osfmk/i386/bsd_i386.c
- * instead of bsd/dev/i386, and therefore cannot see some BSD thread
- * internals.  We need to distinguish kernproc defaulting in the vfork and
- * non-vfork cases vs. actually being the real process context.
- */     
-struct proc *
-i386_current_proc(void)
-{       
-	struct uthread * ut;
-	struct proc *p; 
-	thread_t thr_act = current_thread();
-
-	ut = (struct uthread *)get_bsdthread_info(thr_act);
-	if (ut &&  (ut->uu_flag & UT_VFORK)) { 
-		if (ut->uu_proc) {
-			p = ut->uu_proc; 
-			if ((p->p_flag & P_INVFORK) == 0)
-				panic("returning child proc not under vfork");
-			if (p->p_vforkact != (void *)thr_act)
-				panic("returning child proc which is not cur_act");
-			return(p);
-		} else {
-			return (kernproc);
-		}
-	}
-
-	/* Not in vfork - may return NULL */
-	p = (struct proc *)get_bsdtask_info(current_task());
-
-	return (p);
-}
-
 void *
 get_bsduthreadarg(thread_t th)
 {
         void	*arg_ptr;
-	struct uthread *ut;
+struct uthread *ut;
   
 	ut = get_bsdthread_info(th);
 
@@ -166,7 +136,7 @@ get_bsduthreadarg(thread_t th)
 int *
 get_bsduthreadrval(thread_t th)
 {
-        struct uthread *ut;
+struct uthread *ut;
 
 	ut = get_bsdthread_info(th);
 	return(&ut->uu_rval[0]);
diff --git a/bsd/dev/i386/sysctl.c b/bsd/dev/i386/sysctl.c
index 1cb9f08d7..c7b04d296 100644
--- a/bsd/dev/i386/sysctl.c
+++ b/bsd/dev/i386/sysctl.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <string.h>
@@ -79,7 +85,22 @@ hw_cpu_extfeatures SYSCTL_HANDLER_ARGS
     return SYSCTL_OUT(req, buf, strlen(buf) + 1);
 }
 
-SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RW, 0,
+static int
+hw_cpu_logical_per_package SYSCTL_HANDLER_ARGS
+{
+	__unused struct sysctl_oid *unused_oidp = oidp;
+	__unused void *unused_arg1 = arg1;
+	__unused int unused_arg2 = arg2;
+	i386_cpu_info_t *cpu_info = cpuid_info();
+
+	if (!(cpuid_features() & CPUID_FEATURE_HTT))
+		return ENOENT;
+
+	return SYSCTL_OUT(req, &cpu_info->cpuid_logical_per_package,
+			  sizeof(cpu_info->cpuid_logical_per_package));
+}
+
+SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"CPU info");
 
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, vendor, CTLTYPE_STRING | CTLFLAG_RD, 
@@ -90,10 +111,6 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, brand_string, CTLTYPE_STRING | CTLFLAG_RD,
 	    (void *)offsetof(i386_cpu_info_t, cpuid_brand_string), 0,
 	    hw_cpu_sysctl, "A", "CPU brand string");
 
-SYSCTL_PROC(_machdep_cpu, OID_AUTO, model_string, CTLTYPE_STRING | CTLFLAG_RD, 
-	    (void *)offsetof(i386_cpu_info_t, cpuid_model_string), -1,
-	    hw_cpu_sysctl, "A", "CPU model string");
-
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, family, CTLTYPE_INT | CTLFLAG_RD, 
 	    (void *)offsetof(i386_cpu_info_t, cpuid_family), sizeof(uint8_t),
 	    hw_cpu_sysctl, "I", "CPU family");
@@ -140,9 +157,8 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, extfeatures, CTLTYPE_STRING | CTLFLAG_RD,
 
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, logical_per_package,
 	    CTLTYPE_INT | CTLFLAG_RD, 
-	    (void *)offsetof(i386_cpu_info_t, cpuid_logical_per_package),
-	    sizeof(uint32_t),
-	    hw_cpu_sysctl, "I", "CPU logical cpus per package");
+	    0, 0,
+	    hw_cpu_logical_per_package, "I", "CPU logical cpus per package");
 
 SYSCTL_PROC(_machdep_cpu, OID_AUTO, cores_per_package,
 	    CTLTYPE_INT | CTLFLAG_RD, 
@@ -151,25 +167,154 @@ SYSCTL_PROC(_machdep_cpu, OID_AUTO, cores_per_package,
 	    hw_cpu_sysctl, "I", "CPU cores per package");
 
 
-struct sysctl_oid *machdep_sysctl_list[] =
-{
-    &sysctl__machdep_cpu,
-    &sysctl__machdep_cpu_vendor,
-    &sysctl__machdep_cpu_brand_string,
-    &sysctl__machdep_cpu_model_string,
-    &sysctl__machdep_cpu_family,
-    &sysctl__machdep_cpu_model,
-    &sysctl__machdep_cpu_extmodel,
-    &sysctl__machdep_cpu_extfamily,
-    &sysctl__machdep_cpu_feature_bits,
-    &sysctl__machdep_cpu_extfeature_bits,
-    &sysctl__machdep_cpu_stepping,
-    &sysctl__machdep_cpu_signature,
-    &sysctl__machdep_cpu_brand,
-    &sysctl__machdep_cpu_features,
-    &sysctl__machdep_cpu_extfeatures,
-    &sysctl__machdep_cpu_logical_per_package,
-    &sysctl__machdep_cpu_cores_per_package,
-    (struct sysctl_oid *) 0
-};
+SYSCTL_NODE(_machdep_cpu, OID_AUTO, mwait, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"mwait");
+
+SYSCTL_PROC(_machdep_cpu_mwait, OID_AUTO, linesize_min,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_mwait_linesize_min),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Monitor/mwait minimum line size");
+
+SYSCTL_PROC(_machdep_cpu_mwait, OID_AUTO, linesize_max,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_mwait_linesize_max),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Monitor/mwait maximum line size");
+
+SYSCTL_PROC(_machdep_cpu_mwait, OID_AUTO, extensions,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_mwait_extensions),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Monitor/mwait extensions");
+
+SYSCTL_PROC(_machdep_cpu_mwait, OID_AUTO, sub_Cstates,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_mwait_sub_Cstates),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Monitor/mwait sub C-states");
+
+
+SYSCTL_NODE(_machdep_cpu, OID_AUTO, thermal, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"thermal");
+
+SYSCTL_PROC(_machdep_cpu_thermal, OID_AUTO, sensor,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_thermal_sensor),
+	    sizeof(boolean_t),
+	    hw_cpu_sysctl, "I", "Thermal sensor present");
+
+SYSCTL_PROC(_machdep_cpu_thermal, OID_AUTO, dynamic_acceleration,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_thermal_dynamic_acceleration),
+	    sizeof(boolean_t),
+	    hw_cpu_sysctl, "I", "Dynamic Acceleration Technology");
+
+SYSCTL_PROC(_machdep_cpu_thermal, OID_AUTO, thresholds,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_thermal_thresholds),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Number of interrupt thresholds");
+
+SYSCTL_PROC(_machdep_cpu_thermal, OID_AUTO, ACNT_MCNT,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_thermal_ACNT_MCNT),
+	    sizeof(boolean_t),
+	    hw_cpu_sysctl, "I", "ACNT_MCNT capability");
+
+
+SYSCTL_NODE(_machdep_cpu, OID_AUTO, arch_perf, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"arch_perf");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, version,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_version),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Architectural Performance Version Number");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, number,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_number),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Number of counters per logical cpu");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, width,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_width),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Bit width of counters");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, events_number,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_events_number),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Number of monitoring events");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, events,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_events),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Bit vector of events");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, fixed_number,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_fixed_number),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Number of fixed-function counters");
+
+SYSCTL_PROC(_machdep_cpu_arch_perf, OID_AUTO, fixed_width,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_arch_perf_fixed_width),
+	    sizeof(uint8_t),
+	    hw_cpu_sysctl, "I", "Bit-width of fixed-function counters");
+
+
+SYSCTL_NODE(_machdep_cpu, OID_AUTO, cache, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"cache");
+
+SYSCTL_PROC(_machdep_cpu_cache, OID_AUTO, linesize,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_cache_linesize),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Cacheline size");
+
+SYSCTL_PROC(_machdep_cpu_cache, OID_AUTO, L2_associativity,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_cache_L2_associativity),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "L2 cache associativity");
+
+SYSCTL_PROC(_machdep_cpu_cache, OID_AUTO, size,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_cache_size),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Cache size (in Kbytes)");
+
+
+SYSCTL_NODE(_machdep_cpu, OID_AUTO, address_bits, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"address_bits");
+
+SYSCTL_PROC(_machdep_cpu_address_bits, OID_AUTO, physical,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_address_bits_physical),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Number of physical address bits");
+
+SYSCTL_PROC(_machdep_cpu_address_bits, OID_AUTO, virtual,
+	    CTLTYPE_INT | CTLFLAG_RD, 
+	    (void *)offsetof(i386_cpu_info_t, cpuid_address_bits_virtual),
+	    sizeof(uint32_t),
+	    hw_cpu_sysctl, "I", "Number of virtual address bits");
+
+uint64_t pmap_pv_hashlist_walks;
+uint64_t pmap_pv_hashlist_cnts;
+uint32_t pmap_pv_hashlist_max;
+
+/*extern struct sysctl_oid_list sysctl__machdep_pmap_children;*/
+
+SYSCTL_NODE(_machdep, OID_AUTO, pmap, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+	"PMAP info");
 
+SYSCTL_QUAD    (_machdep_pmap, OID_AUTO, hashwalks, CTLFLAG_RD | CTLFLAG_KERN, &pmap_pv_hashlist_walks, "");
+SYSCTL_QUAD    (_machdep_pmap, OID_AUTO, hashcnts, CTLFLAG_RD | CTLFLAG_KERN, &pmap_pv_hashlist_cnts, "");
+SYSCTL_INT     (_machdep_pmap, OID_AUTO, hashmax, CTLFLAG_RD | CTLFLAG_KERN, &pmap_pv_hashlist_max, 0, "");
diff --git a/bsd/dev/i386/systemcalls.c b/bsd/dev/i386/systemcalls.c
index 6c68bfd35..fa6db0a3d 100644
--- a/bsd/dev/i386/systemcalls.c
+++ b/bsd/dev/i386/systemcalls.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <kern/task.h>
 #include <kern/thread.h>
@@ -35,7 +41,6 @@
 #include <sys/systm.h>
 #include <sys/user.h>
 #include <sys/errno.h>
-#include <sys/ktrace.h>
 #include <sys/kdebug.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
@@ -48,15 +53,18 @@
 #include <i386/machine_routines.h>
 #include <mach/i386/syscall_sw.h>
 
+#if CONFIG_DTRACE
+extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
+extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
+#endif
+
 extern void unix_syscall(x86_saved_state_t *);
 extern void unix_syscall64(x86_saved_state_t *);
-extern void unix_syscall_return(int);
 extern void *find_user_regs(thread_t);
-extern void IOSleep(int);
-extern void exit_funnel_section(void);
-
-extern void Debugger(const char      * message);
+extern void throttle_lowpri_io(int *lowpri_window, mount_t v_mount);
 
+extern void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid);
+extern boolean_t x86_sysenter_arg_store_isvalid(thread_t thread);
 /*
  * Function:	unix_syscall
  *
@@ -67,25 +75,24 @@ extern void Debugger(const char      * message);
 void
 unix_syscall(x86_saved_state_t *state)
 {
-	thread_t	thread;
-	void		*vt; 
-	unsigned int	code;
-	struct sysent	*callp;
-	int		nargs;
-	int		error;
-	int		funnel_type;
-	vm_offset_t	params;
-	struct proc	*p;
-	struct uthread	*uthread;
-	unsigned int cancel_enable;
+	thread_t		thread;
+	void			*vt;
+	unsigned int		code;
+	struct sysent		*callp;
+
+	int			error;
+	vm_offset_t		params;
+	struct proc		*p;
+	struct uthread		*uthread;
 	x86_saved_state32_t	*regs;
+	boolean_t		args_in_uthread;
 
 	assert(is_saved_state32(state));
 	regs = saved_state32(state);
-
+#if DEBUG
 	if (regs->eax == 0x800)
 		thread_exception_return();
-
+#endif
 	thread = current_thread();
 	uthread = get_bsdthread_info(thread);
 
@@ -104,35 +111,44 @@ unix_syscall(x86_saved_state_t *state)
 		/* NOTREACHED */
 	}
 
-	code = regs->eax;
+	code = regs->eax & I386_SYSCALL_NUMBER_MASK;
+	args_in_uthread = ((regs->eax & I386_SYSCALL_ARG_BYTES_MASK) != 0) && x86_sysenter_arg_store_isvalid(thread);
 	params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-	callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+
+	regs->efl &= ~(EFL_CF);
+
+	callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 
 	if (callp == sysent) {
 		code = fuword(params);
-		params += sizeof (int);
-		callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+		params += sizeof(int);
+		callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 	}
+
 	vt = (void *)uthread->uu_arg;
 
-	nargs = callp->sy_narg * sizeof (syscall_arg_t);
-	if (nargs != 0) {
+	if (callp->sy_arg_bytes != 0) {
 		sy_munge_t	*mungerp;
 
-		assert(nargs <= 8);
-
-		error = copyin((user_addr_t) params, (char *) vt, nargs);
-		if (error) {
-			regs->eax = error;
-			regs->efl |= EFL_CF;
-			thread_exception_return();
-			/* NOTREACHED */
+		assert((unsigned) callp->sy_arg_bytes <= sizeof (uthread->uu_arg));
+		if (!args_in_uthread)
+		{
+			uint32_t nargs;
+			nargs = callp->sy_arg_bytes;
+			error = copyin((user_addr_t) params, (char *) vt, nargs);
+			if (error) {
+				regs->eax = error;
+				regs->efl |= EFL_CF;
+				thread_exception_return();
+				/* NOTREACHED */
+			}
 		}
+
 		if (code != 180) {
-		        int *ip = (int *)vt;
+	        	int *ip = (int *)vt;
 
 			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-					      *ip, *(ip+1), *(ip+2), *(ip+3), 0);
+				      *ip, *(ip+1), *(ip+2), *(ip+3), 0);
 		}
 		mungerp = callp->sy_arg_munge32;
 
@@ -146,52 +162,37 @@ unix_syscall(x86_saved_state_t *state)
 		if (mungerp != NULL)
 			(*mungerp)(NULL, vt);
 	} else
-	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-				      0, 0, 0, 0, 0);
+		KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
+			0, 0, 0, 0, 0);
+
 	/*
 	 * Delayed binding of thread credential to process credential, if we
 	 * are not running with an explicitly set thread credential.
 	 */
-	if (uthread->uu_ucred != p->p_ucred &&
-	    (uthread->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = uthread->uu_ucred;
-		uthread->uu_ucred = kauth_cred_proc_ref(p);
-		if (IS_VALID_CRED(old))
-			kauth_cred_unref(&old);
-	}
+	kauth_cred_uthread_update(uthread, p);
 
 	uthread->uu_rval[0] = 0;
 	uthread->uu_rval[1] = regs->edx;
+	uthread->uu_flag |= UT_NOTCANCELPT;
 
-	cancel_enable = callp->sy_cancel;
-	
-	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-	        uthread->uu_flag |= UT_NOTCANCELPT;
-	} else {
-		if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-			if (cancel_enable == _SYSCALL_CANCEL_PRE) {
-					/* system call cancelled; return to handle cancellation */
-					regs->eax = (long long)EINTR;
-					regs->efl |= EFL_CF;
-					thread_exception_return();
-					/* NOTREACHED */
- 			} else {
-			        thread_abort_safely(thread);
-			}
-		}
-	}
 
-	funnel_type = (callp->sy_funnel & FUNNEL_MASK);
-	if (funnel_type == KERNEL_FUNNEL)
-		thread_funnel_set(kernel_flock, TRUE);
-
-	if (KTRPOINT(p, KTR_SYSCALL))
-	        ktrsyscall(p, code, callp->sy_narg, vt);
+#ifdef JOE_DEBUG
+        uthread->uu_iocount = 0;
+        uthread->uu_vpindex = 0;
+#endif
 
 	AUDIT_SYSCALL_ENTER(code, p, uthread);
 	error = (*(callp->sy_call))((void *) p, (void *) vt, &(uthread->uu_rval[0]));
-	AUDIT_SYSCALL_EXIT(error, p, uthread);
-	
+        AUDIT_SYSCALL_EXIT(code, p, uthread, error);
+
+#ifdef JOE_DEBUG
+        if (uthread->uu_iocount)
+                joe_debug("system call returned with uu_iocount != 0");
+#endif
+#if CONFIG_DTRACE
+	uthread->t_dtrace_errno = error;
+#endif /* CONFIG_DTRACE */
+
 	if (error == ERESTART) {
 		/*
 		 * Move the user's pc back to repeat the syscall:
@@ -199,8 +200,9 @@ unix_syscall(x86_saved_state_t *state)
 		 * The SYSENTER_TF_CS covers single-stepping over a sysenter
 		 * - see debug trap handler in idt.s/idt64.s
 		 */
-		if (regs->cs == SYSENTER_CS || regs->cs == SYSENTER_TF_CS)
+		if (regs->cs == SYSENTER_CS || regs->cs == SYSENTER_TF_CS) {
 			regs->eip -= 5;
+		}
 		else
 			regs->eip -= 2;
 	}
@@ -211,24 +213,17 @@ unix_syscall(x86_saved_state_t *state)
 		} else { /* (not error) */
 		    regs->eax = uthread->uu_rval[0];
 		    regs->edx = uthread->uu_rval[1];
-		    regs->efl &= ~EFL_CF;
 		} 
 	}
 
-	if (KTRPOINT(p, KTR_SYSRET))
-	        ktrsysret(p, code, error, uthread->uu_rval[0]);
-
-	if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
-
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
+#if DEBUG
 	/*
-	 * if we're holding the funnel
-	 * than drop it regardless of whether
-	 * we took it on system call entry
+	 * if we're holding the funnel panic
 	 */
-	exit_funnel_section();
-
-	if (uthread->uu_lowpri_delay) {
+	syscall_exit_funnelcheck();
+#endif /* DEBUG */
+	if (uthread->uu_lowpri_window && uthread->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -236,8 +231,7 @@ unix_syscall(x86_saved_state_t *state)
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(uthread->uu_lowpri_delay);
-	        uthread->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
 	}
 	if (code != 180)
 	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -257,10 +251,8 @@ unix_syscall64(x86_saved_state_t *state)
 	void		*uargp;
 	int		args_in_regs;
 	int		error;
-	int		funnel_type;
 	struct proc	*p;
 	struct uthread	*uthread;
-	unsigned int cancel_enable;
 	x86_saved_state64_t *regs;
 
 	assert(is_saved_state64(state));
@@ -289,7 +281,7 @@ unix_syscall64(x86_saved_state_t *state)
 	args_in_regs = 6;
 
 	code = regs->rax & SYSCALL_NUMBER_MASK;
-	callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+	callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 	uargp = (void *)(&regs->rdi);
 
 	if (callp == sysent) {
@@ -297,29 +289,29 @@ unix_syscall64(x86_saved_state_t *state)
 		 * indirect system call... system call number
 		 * passed as 'arg0'
 		 */
-		code = regs->rdi;
-		callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+	        code = regs->rdi;
+		callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 		uargp = (void *)(&regs->rsi);
 		args_in_regs = 5;
 	}
 
 	if (callp->sy_narg != 0) {
 		if (code != 180) {
-		        uint64_t *ip = (uint64_t *)uargp;
+			uint64_t *ip = (uint64_t *)uargp;
 
 			KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
-					      (int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
+					(int)(*ip), (int)(*(ip+1)), (int)(*(ip+2)), (int)(*(ip+3)), 0);
 		}
-	        assert(callp->sy_narg <= 8);
+		assert(callp->sy_narg <= 8);
 
 		if (callp->sy_narg > args_in_regs) {
-		        int copyin_count;
+			int copyin_count;
 
-		        copyin_count = (callp->sy_narg - args_in_regs) * sizeof(uint64_t);
+			copyin_count = (callp->sy_narg - args_in_regs) * sizeof(uint64_t);
 
-		        error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count);
+			error = copyin((user_addr_t)(regs->isf.rsp + sizeof(user_addr_t)), (char *)&regs->v_arg6, copyin_count);
 			if (error) {
-			        regs->rax = error;
+				regs->rax = error;
 				regs->isf.rflags |= EFL_CF;
 				thread_exception_return();
 				/* NOTREACHED */
@@ -328,11 +320,10 @@ unix_syscall64(x86_saved_state_t *state)
 		/*
 		 * XXX Turn 64 bit unsafe calls into nosys()
 		 */
-		if (callp->sy_funnel & UNSAFE_64BIT) {
-		        callp = &sysent[63];
+		if (callp->sy_flags & UNSAFE_64BIT) {
+			callp = &sysent[63];
 			goto unsafe;
 		}
-
 	} else
 	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_START,
 				      0, 0, 0, 0, 0);
@@ -342,45 +333,22 @@ unsafe:
 	 * Delayed binding of thread credential to process credential, if we
 	 * are not running with an explicitly set thread credential.
 	 */
-	if (uthread->uu_ucred != p->p_ucred &&
-	    (uthread->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = uthread->uu_ucred;
-		uthread->uu_ucred = kauth_cred_proc_ref(p);
-		if (IS_VALID_CRED(old))
-			kauth_cred_unref(&old);
-	}
+	kauth_cred_uthread_update(uthread, p);
 
 	uthread->uu_rval[0] = 0;
 	uthread->uu_rval[1] = 0;
 
-	cancel_enable = callp->sy_cancel;
 	
-	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-	        uthread->uu_flag |= UT_NOTCANCELPT;
-	} else {
-		if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-			if (cancel_enable == _SYSCALL_CANCEL_PRE) {
-					/* system call cancelled; return to handle cancellation */
-					regs->rax = EINTR;
-					regs->isf.rflags |= EFL_CF;
-					thread_exception_return();
-					/* NOTREACHED */
- 			} else {
-			        thread_abort_safely(thread);
-			}
-		}
-	}
-
-	funnel_type = (callp->sy_funnel & FUNNEL_MASK);
-	if (funnel_type == KERNEL_FUNNEL)
-		thread_funnel_set(kernel_flock, TRUE);
+	uthread->uu_flag |= UT_NOTCANCELPT;
 
-	if (KTRPOINT(p, KTR_SYSCALL))
-	        ktrsyscall(p, code, callp->sy_narg, uargp);
 
 	AUDIT_SYSCALL_ENTER(code, p, uthread);
 	error = (*(callp->sy_call))((void *) p, uargp, &(uthread->uu_rval[0]));
-	AUDIT_SYSCALL_EXIT(error, p, uthread);
+        AUDIT_SYSCALL_EXIT(code, p, uthread, error);
+
+#if CONFIG_DTRACE
+	uthread->t_dtrace_errno = error;
+#endif /* CONFIG_DTRACE */
 	
 	if (error == ERESTART) {
 		/*
@@ -392,7 +360,7 @@ unsafe:
 	}
 	else if (error != EJUSTRETURN) {
 		if (error) {
-		        regs->rax = error;
+			regs->rax = error;
 			regs->isf.rflags |= EFL_CF;	/* carry bit */
 		} else { /* (not error) */
 
@@ -422,20 +390,15 @@ unsafe:
 		} 
 	}
 
-	if (KTRPOINT(p, KTR_SYSRET))
-	        ktrsysret(p, code, error, uthread->uu_rval[0]);
 
-	if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
 
 	/*
-	 * if we're holding the funnel
-	 * than drop it regardless of whether
-	 * we took it on system call entry
+	 * if we're holding the funnel panic
 	 */
-	exit_funnel_section();
+	syscall_exit_funnelcheck();
 
-	if (uthread->uu_lowpri_delay) {
+	if (uthread->uu_lowpri_window && uthread->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -443,8 +406,7 @@ unsafe:
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(uthread->uu_lowpri_delay);
-	        uthread->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
 	}
 	if (code != 180)
 	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -464,7 +426,6 @@ unix_syscall_return(int error)
 	unsigned int code;
 	vm_offset_t params;
 	struct sysent *callp;
-	unsigned int cancel_enable;
 
 	thread = current_thread();
 	uthread = get_bsdthread_info(thread);
@@ -472,103 +433,107 @@ unix_syscall_return(int error)
 	p = current_proc();
 
 	if (proc_is64bit(p)) {
-	        x86_saved_state64_t *regs;
+		x86_saved_state64_t *regs;
 
 		regs = saved_state64(find_user_regs(thread));
 
-	        /* reconstruct code for tracing before blasting rax */
+		/* reconstruct code for tracing before blasting rax */
 		code = regs->rax & SYSCALL_NUMBER_MASK;
-		callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+		callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 
 		if (callp == sysent)
-		        /*
+			/*
 			 * indirect system call... system call number
 			 * passed as 'arg0'
 			 */
-		        code = regs->rdi;
+			code = regs->rdi;
+
+#if CONFIG_DTRACE
+		if (callp->sy_call == dtrace_systrace_syscall)
+			dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
+#endif /* CONFIG_DTRACE */
 
 		if (error == ERESTART) {
-		        /*
+			/*
 			 * all system calls come through via the syscall instruction
 			 * in 64 bit mode... its 2 bytes in length
 			 * move the user's pc back to repeat the syscall:
 			 */
-		        regs->isf.rip -= 2;
+			regs->isf.rip -= 2;
 		}
 		else if (error != EJUSTRETURN) {
-		        if (error) {
-			        regs->rax = error;
+			if (error) {
+				regs->rax = error;
 				regs->isf.rflags |= EFL_CF;	/* carry bit */
 			} else { /* (not error) */
 
-			        switch (callp->sy_return_type) {
+				switch (callp->sy_return_type) {
 				case _SYSCALL_RET_INT_T:
-				        regs->rax = uthread->uu_rval[0];
+					regs->rax = uthread->uu_rval[0];
 					regs->rdx = uthread->uu_rval[1];
 					break;
 				case _SYSCALL_RET_UINT_T:
-				        regs->rax = ((u_int)uthread->uu_rval[0]);
+					regs->rax = ((u_int)uthread->uu_rval[0]);
 					regs->rdx = ((u_int)uthread->uu_rval[1]);
 					break;
 				case _SYSCALL_RET_OFF_T:
 				case _SYSCALL_RET_ADDR_T:
 				case _SYSCALL_RET_SIZE_T:
 				case _SYSCALL_RET_SSIZE_T:
-				        regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
+					regs->rax = *((uint64_t *)(&uthread->uu_rval[0]));
 					regs->rdx = 0;
 					break;
 				case _SYSCALL_RET_NONE:
-				        break;
+					break;
 				default:
-				        panic("unix_syscall: unknown return type");
+					panic("unix_syscall: unknown return type");
 					break;
 				}
 				regs->isf.rflags &= ~EFL_CF;
 			} 
 		}
 	} else {
-	        x86_saved_state32_t	*regs;
+		x86_saved_state32_t	*regs;
 
 		regs = saved_state32(find_user_regs(thread));
 
+		regs->efl &= ~(EFL_CF);
 		/* reconstruct code for tracing before blasting eax */
-		code = regs->eax;
-		callp = (code >= (unsigned int)nsysent) ? &sysent[63] : &sysent[code];
+		code = regs->eax & I386_SYSCALL_NUMBER_MASK;
+		callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
+
+#if CONFIG_DTRACE
+		if (callp->sy_call == dtrace_systrace_syscall)
+			dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
+#endif /* CONFIG_DTRACE */
 
 		if (callp == sysent) {
-		        params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
-		        code = fuword(params);
+			params = (vm_offset_t) ((caddr_t)regs->uesp + sizeof (int));
+			code = fuword(params);
 		}
 		if (error == ERESTART) {
-		        regs->eip -= ((regs->cs & 0xffff) == SYSENTER_CS) ? 5 : 2;
+			regs->eip -= ((regs->cs & 0xffff) == SYSENTER_CS) ? 5 : 2;
 		}
 		else if (error != EJUSTRETURN) {
-		        if (error) {
-			        regs->eax = error;
+			if (error) {
+				regs->eax = error;
 				regs->efl |= EFL_CF;	/* carry bit */
 			} else { /* (not error) */
-			        regs->eax = uthread->uu_rval[0];
+				regs->eax = uthread->uu_rval[0];
 				regs->edx = uthread->uu_rval[1];
-				regs->efl &= ~EFL_CF;
 			} 
 		}
 	}
-	if (KTRPOINT(p, KTR_SYSRET))
-	        ktrsysret(p, code, error, uthread->uu_rval[0]);
 
-	cancel_enable = callp->sy_cancel;
 
-	if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
 
 	/*
-	 * if we're holding the funnel
-	 * than drop it regardless of whether
-	 * we took it on system call entry
+	 * if we're holding the funnel panic
 	 */
-	exit_funnel_section();
+	syscall_exit_funnelcheck();
 
-	if (uthread->uu_lowpri_delay) {
+	if (uthread->uu_lowpri_window && uthread->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -576,8 +541,7 @@ unix_syscall_return(int error)
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(uthread->uu_lowpri_delay);
-	        uthread->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
 	}
 	if (code != 180)
 	        KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_EXCP_SC, code) | DBG_FUNC_END,
@@ -607,3 +571,34 @@ munge_wwwlww(
 	arg64[1] = arg32[1];	/* wWwlww */
 	arg64[0] = arg32[0];	/* Wwwlww */
 }	
+
+
+void
+munge_wwlwww(
+	__unused const void	*in32,
+	void			*out64)
+{
+	uint32_t	*arg32;
+	uint64_t	*arg64;
+
+	/* we convert in place in out64 */
+	arg32 = (uint32_t *) out64;
+	arg64 = (uint64_t *) out64;
+
+	arg64[5] = arg32[6];	/* wwlwwW */
+	arg64[4] = arg32[5];	/* wwlwWw */
+	arg64[3] = arg32[4];	/* wwlWww  */
+	arg32[5] = arg32[3];	/* wwLwww (hi) */
+	arg32[4] = arg32[2];	/* wwLwww (lo) */
+	arg64[1] = arg32[1];	/* wWlwww */
+	arg64[0] = arg32[0];	/* Wwlwww */
+}	
+
+#ifdef JOE_DEBUG
+joe_debug(char *p) {
+
+        printf("%s\n", p);
+}
+#endif
+
+
diff --git a/bsd/dev/i386/table_inline.h b/bsd/dev/i386/table_inline.h
index a4bb2937a..b99eddd0b 100644
--- a/bsd/dev/i386/table_inline.h
+++ b/bsd/dev/i386/table_inline.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/bsd/dev/i386/unix_signal.c b/bsd/dev/i386/unix_signal.c
index 8c6305d52..7a9291cae 100644
--- a/bsd/dev/i386/unix_signal.c
+++ b/bsd/dev/i386/unix_signal.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1992 NeXT, Inc.
@@ -50,8 +56,11 @@
 
 #include <sys/kdebug.h>
 
+#include <sys/sdt.h>
+
 /* Forward: */
-extern boolean_t machine_exception(int, int, int, int *, int *);
+extern boolean_t machine_exception(int, mach_exception_code_t, 
+		mach_exception_subcode_t, int *, mach_exception_subcode_t *);
 extern kern_return_t thread_getstatus(register thread_t act, int flavor,
 			thread_state_t tstate, mach_msg_type_number_t *count);
 extern kern_return_t thread_setstatus(thread_t thread, int flavor,
@@ -61,6 +70,8 @@ extern kern_return_t thread_setstatus(thread_t thread, int flavor,
 /* These defns should match the Libc implmn */
 #define UC_TRAD			1
 #define UC_FLAVOR		30
+#define	UC_SET_ALT_STACK	0x40000000
+#define	UC_RESET_ALT_STACK	0x80000000
 
 #define	C_32_STK_ALIGN		16
 #define	C_64_STK_ALIGN		16
@@ -108,34 +119,29 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 	int oonstack, flavor; 
 	void * state;
 	mach_msg_type_number_t state_count;
-	int uthsigaltstack = 0;
-	int altstack = 0;
 
-	thread_t thread = current_thread();
+	thread_t thread;
 	struct uthread * ut;
 	int stack_size = 0;
 	int infostyle = UC_TRAD;
     
+	thread = current_thread();
+	ut = get_bsdthread_info(thread);
+
 	if (p->p_sigacts->ps_siginfo & sigmask(sig))
 		infostyle = UC_FLAVOR;
 
-	ut = get_bsdthread_info(thread);
-
-	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
+	oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
 
-	if (uthsigaltstack != 0 )  {
-		oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
-		altstack = ut->uu_flag & UT_ALTSTACK;
-	} else {
-		oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
-		altstack = ps->ps_flags & SAS_ALTSTACK;
-	}
 	/*
 	 * init siginfo
 	 */
+	proc_unlock(p);
+
 	bzero((caddr_t)&sinfo64, sizeof(user_siginfo_t));
 	sinfo64.si_signo = sig;
 		
+
 	if (proc_is64bit(p)) {
 	        x86_thread_state64_t	*tstate64;
 	        struct user_ucontext64 	uctx64;
@@ -160,20 +166,16 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 
 		tstate64 = &mctx.mctx64.ss;
 
-		if (altstack && !oonstack && (ps->ps_sigonstack & sigmask(sig))) {
-		        if (uthsigaltstack != 0) {
-			        ua_sp = ut->uu_sigstk.ss_sp;
-				stack_size = ut->uu_sigstk.ss_size;
-				ua_sp += stack_size;
-				ut->uu_sigstk.ss_flags |= SA_ONSTACK;
-			} else {
-			        ua_sp = ps->ps_sigstk.ss_sp;
-				stack_size = ps->ps_sigstk.ss_size;
-				ua_sp += stack_size;
-				ps->ps_sigstk.ss_flags |= SA_ONSTACK;
-			}
-		} else
+		/* figure out where our new stack lives */
+		if ((ut->uu_flag & UT_ALTSTACK) && !oonstack &&
+		    (ps->ps_sigonstack & sigmask(sig))) {
+			ua_sp = ut->uu_sigstk.ss_sp;
+			stack_size = ut->uu_sigstk.ss_size;
+			ua_sp += stack_size;
+			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		} else {
 		        ua_sp = tstate64->rsp;
+		}
 		ua_cr2 = mctx.mctx64.es.faultvaddr;
 
 		/* The x86_64 ABI defines a 128-byte red zone. */
@@ -271,20 +273,16 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 
 		tstate32 = &mctx.mctx32.ss;
 
-		if (altstack && !oonstack && (ps->ps_sigonstack & sigmask(sig))) {
-		        if (uthsigaltstack != 0) {
-			        ua_sp = ut->uu_sigstk.ss_sp;
-				stack_size = ut->uu_sigstk.ss_size;
-				ua_sp += stack_size;
-				ut->uu_sigstk.ss_flags |= SA_ONSTACK;
-			} else {
-			        ua_sp = ps->ps_sigstk.ss_sp;
-				stack_size = ps->ps_sigstk.ss_size;
-				ua_sp += stack_size;
-				ps->ps_sigstk.ss_flags |= SA_ONSTACK;
-			}
-		} else
+		/* figure out where our new stack lives */
+		if ((ut->uu_flag & UT_ALTSTACK) && !oonstack &&
+		    (ps->ps_sigonstack & sigmask(sig))) {
+			ua_sp = ut->uu_sigstk.ss_sp;
+			stack_size = ut->uu_sigstk.ss_size;
+			ua_sp += stack_size;
+			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		} else {
 		        ua_sp = tstate32->esp;
+		}
 		ua_cr2 = mctx.mctx32.es.faultvaddr;
 
 		ua_sp -= sizeof (struct ucontext);
@@ -341,7 +339,7 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 
 		uctx32.uc_mcsize = sizeof(struct mcontext32);
 
-		uctx32.uc_mcontext = CAST_DOWN(struct mcontext *, ua_mctxp);
+		uctx32.uc_mcontext = CAST_DOWN(_STRUCT_MCONTEXT32 *, ua_mctxp);
 		
 		if (copyout((caddr_t)&uctx32, ua_uctxp, sizeof (uctx32))) 
 		        goto bad;
@@ -354,36 +352,13 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 	}
 
 	switch (sig) {
-		case SIGCHLD:
-			sinfo64.si_pid = p->si_pid;
-			p->si_pid =0;
-			sinfo64.si_status = p->si_status;
-			p->si_status = 0;
-			sinfo64.si_uid = p->si_uid;
-			p->si_uid =0;
-			sinfo64.si_code = p->si_code;
-			p->si_code = 0;
-			if (sinfo64.si_code == CLD_EXITED) {
-				if (WIFEXITED(sinfo64.si_status)) 
-					sinfo64.si_code = CLD_EXITED;
-				else if (WIFSIGNALED(sinfo64.si_status)) {
-					if (WCOREDUMP(sinfo64.si_status))
-						sinfo64.si_code = CLD_DUMPED;
-					else	
-						sinfo64.si_code = CLD_KILLED;
-				}
-			}
-			break;
 		case SIGILL:
 			switch (ut->uu_code) {
 				case EXC_I386_INVOP:
 					sinfo64.si_code = ILL_ILLOPC;
 					break;
-				case EXC_I386_GPFLT:
-					sinfo64.si_code = ILL_PRVOPC;
-					break;
 				default:
-					printf("unknown SIGILL code %d\n", ut->uu_code);
+					printf("unknown SIGILL code %ld\n", (long) ut->uu_code);
 					sinfo64.si_code = ILL_NOOP;
 			}
 			break;
@@ -405,8 +380,8 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 			} else if (ut->uu_subcode & (1 << FP_IE)) {
 				sinfo64.si_code = FPE_FLTINV;
 			} else {
-				printf("unknown SIGFPE code %d, subcode %x\n",
-				       ut->uu_code, ut->uu_subcode);
+				printf("unknown SIGFPE code %ld, subcode %lx\n",
+				       (long) ut->uu_code, (long) ut->uu_subcode);
 				sinfo64.si_code = FPE_NOOP;
 			}
 			break;
@@ -421,6 +396,12 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 		        sinfo64.si_addr = ua_cr2;
 
 			switch (ut->uu_code) {
+				case EXC_I386_GPFLT:
+					/* CR2 is meaningless after GP fault */
+					/* XXX namespace clash! */
+					sinfo64.si_addr = 0ULL;
+					sinfo64.si_code = 0;
+					break;
 				case KERN_PROTECTION_FAILURE:
 					sinfo64.si_code = SEGV_ACCERR;
 					break;
@@ -428,14 +409,62 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 					sinfo64.si_code = SEGV_MAPERR;
 					break;
 				default:
-					printf("unknown SIGSEGV code %d\n", ut->uu_code);
+					printf("unknown SIGSEGV code %ld\n", (long) ut->uu_code);
 					sinfo64.si_code = FPE_NOOP;
 			}
 				break;
 		default:
+		{
+			int status_and_exitcode;
+
+			/*
+			 * All other signals need to fill out a minimum set of
+			 * information for the siginfo structure passed into
+			 * the signal handler, if SA_SIGINFO was specified.
+			 *
+			 * p->si_status actually contains both the status and
+			 * the exit code; we save it off in its own variable
+			 * for later breakdown.
+			 */
+			proc_lock(p);
+			sinfo64.si_pid = p->si_pid;
+			p->si_pid =0;
+			status_and_exitcode = p->si_status;
+			p->si_status = 0;
+			sinfo64.si_uid = p->si_uid;
+			p->si_uid =0;
+			sinfo64.si_code = p->si_code;
+			p->si_code = 0;
+			proc_unlock(p);
+			if (sinfo64.si_code == CLD_EXITED) {
+				if (WIFEXITED(status_and_exitcode)) 
+					sinfo64.si_code = CLD_EXITED;
+				else if (WIFSIGNALED(status_and_exitcode)) {
+					if (WCOREDUMP(status_and_exitcode)) {
+						sinfo64.si_code = CLD_DUMPED;
+						status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode);
+					} else {
+						sinfo64.si_code = CLD_KILLED;
+						status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode);
+					}
+				}
+			}
+			/*
+			 * The recorded status contains the exit code and the
+			 * signal information, but the information to be passed
+			 * in the siginfo to the handler is supposed to only
+			 * contain the status, so we have to shift it out.
+			 */
+			sinfo64.si_status = WEXITSTATUS(status_and_exitcode);
 			break;
+		}
 	}
 	if (proc_is64bit(p)) {
+
+		/* XXX truncates catcher address to uintptr_t */
+		DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &sinfo64,
+			void (*)(void), CAST_DOWN(sig_t, ua_catcher));
+
 	        if (copyout((caddr_t)&sinfo64, ua_sip, sizeof (sinfo64))) 
 		        goto bad;
 
@@ -454,7 +483,10 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 		sinfo32.si_uid    = sinfo64.si_uid;
 		sinfo32.si_status = sinfo64.si_status;
 		sinfo32.si_addr   = CAST_DOWN(void *, sinfo64.si_addr);
-		sinfo32.pad[0]    = sinfo64.pad[0];
+		sinfo32.__pad[0]    = sinfo64.pad[0];
+
+		DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &sinfo32,
+			void (*)(void), CAST_DOWN(sig_t, ua_catcher));
 
 	        if (copyout((caddr_t)&sinfo32, ua_sip, sizeof (sinfo32))) 
 		        goto bad;
@@ -480,16 +512,21 @@ sendsig(struct proc *p, user_addr_t ua_catcher, int sig, int mask, __unused u_lo
 	        goto bad;
 	ml_fp_setvalid(FALSE);
 
+	proc_lock(p);
+
 	return;
 
 bad:
+	proc_lock(p);
 	SIGACTION(p, SIGILL) = SIG_DFL;
 	sig = sigmask(SIGILL);
 	p->p_sigignore &= ~sig;
 	p->p_sigcatch &= ~sig;
 	ut->uu_sigmask &= ~sig;
 	/* sendsig is called with signal lock held */
-	psignal_lock(p, SIGILL, 0);
+	proc_unlock(p);
+	psignal_locked(p, SIGILL);
+	proc_lock(p);
 	return;
 }
 
@@ -505,10 +542,7 @@ bad:
  */
 
 int
-sigreturn(
-	struct proc *p,
-	struct sigreturn_args *uap,
-	__unused int *retval)
+sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 {
         union {
 	    struct mcontext32	mctx32;
@@ -517,7 +551,6 @@ sigreturn(
 	thread_t thread = current_thread();
 	struct uthread * ut;
 	int	error;
-	int	uthsigaltstack = 0;
 	int	onstack = 0;
 
 	mach_msg_type_number_t ts_count;
@@ -528,7 +561,18 @@ sigreturn(
 	void		    *  fs;
 
 	ut = (struct uthread *)get_bsdthread_info(thread);
-	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
+
+	/*
+	 * If we are being asked to change the altstack flag on the thread, we
+	 * just set/reset it and return (the uap->uctx is not used).
+	 */
+	if ((unsigned int)uap->infostyle == UC_SET_ALT_STACK) {
+		ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		return (0);
+	} else if ((unsigned int)uap->infostyle == UC_RESET_ALT_STACK) {
+		ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
+		return (0);
+	}
 
 	if (proc_is64bit(p)) {
 	        struct user_ucontext64	uctx64;
@@ -570,22 +614,18 @@ sigreturn(
 		fs_count  = x86_FLOAT_STATE32_COUNT;
 		fs = (void *)&mctx.mctx32.fs;
 	}
-	if (onstack) {
-		if (uthsigaltstack != 0)
-			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
-		else
-			p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
-	} else {
-		if (uthsigaltstack != 0)
-			ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
-		else
-			p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
-	}
+
+	if (onstack)
+		ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+	else
+		ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
+
 	if (ut->uu_siglist & ~ut->uu_sigmask)
 		signal_setast(thread);
 
 	/*
-	 * thread_set_state() does all the needed checks for the passed in content
+	 * thread_set_state() does all the needed checks for the passed in
+	 * content
 	 */
 	if (thread_setstatus(thread, ts_flavor, ts, ts_count) != KERN_SUCCESS)
 	        return(EINVAL);
@@ -606,78 +646,49 @@ sigreturn(
 
 boolean_t
 machine_exception(
-    int			exception,
-    int			code,
-    __unused int	subcode,
-    int			*unix_signal,
-    int			*unix_code
-)
-{
-
-    switch(exception) {
-
-    case EXC_BAD_INSTRUCTION:
-	*unix_signal = SIGILL;
-	*unix_code = code;
-	break;
-
-    case EXC_ARITHMETIC:
-	*unix_signal = SIGFPE;
-	*unix_code = code;
-	break;
-
-    default:
-	return(FALSE);
-    }
-   
-    return(TRUE);
-}
-
-#include <sys/systm.h>
-#include <sys/sysent.h>
-
-int __pthread_cset(struct sysent *);
-void __pthread_creset(struct sysent *);
-
-int
-__pthread_cset(struct sysent *callp)
+	int				exception,
+	mach_exception_code_t		code,
+	__unused mach_exception_subcode_t subcode,
+	int 				*unix_signal,
+	mach_exception_code_t		*unix_code)
 {
-	unsigned int cancel_enable;
-	thread_t thread;
-	struct uthread * uthread; 
 
-	thread = current_thread();
-	uthread = get_bsdthread_info(thread);
+	switch(exception) {
 
-	cancel_enable = callp->sy_cancel;
-	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-		uthread->uu_flag |= UT_NOTCANCELPT;
-	} else {
-		if((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-			if (cancel_enable == _SYSCALL_CANCEL_PRE)
-				return(EINTR);
-			else
-				thread_abort_safely(thread);
+	case EXC_BAD_ACCESS:
+		/* Map GP fault to SIGSEGV, otherwise defer to caller */
+		if (code == EXC_I386_GPFLT) {
+			*unix_signal = SIGSEGV;
+			*unix_code = code;
+			break;
+		}
+		return(FALSE);
+
+	case EXC_BAD_INSTRUCTION:
+		*unix_signal = SIGILL;
+		*unix_code = code;
+		break;
+
+	case EXC_ARITHMETIC:
+		*unix_signal = SIGFPE;
+		*unix_code = code;
+		break;
+
+	case EXC_SOFTWARE:
+		if (code == EXC_I386_BOUND) {
+			/*
+			 * Map #BR, the Bound Range Exceeded exception, to
+			 * SIGTRAP.
+			 */
+			*unix_signal = SIGTRAP;
+			*unix_code = code;
+			break;
 		}
-	}   
-	return(0);
-}
-
-
-void
-__pthread_creset(struct sysent *callp)
-{
-
-	unsigned int cancel_enable;
-	thread_t thread;
-	struct uthread * uthread; 
-
-	thread = current_thread();
-	uthread = get_bsdthread_info(thread);
-	
-	cancel_enable = callp->sy_cancel;
-	if (!cancel_enable) 
-		uthread->uu_flag &= ~UT_NOTCANCELPT;
 
+	default:
+		return(FALSE);
+	}
+   
+	return(TRUE);
 }
 
diff --git a/bsd/dev/kmreg_com.h b/bsd/dev/kmreg_com.h
index 1d4a7224f..032d74acc 100644
--- a/bsd/dev/kmreg_com.h
+++ b/bsd/dev/kmreg_com.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 	Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. 
  *
diff --git a/bsd/dev/ldd.h b/bsd/dev/ldd.h
index 30f5fa39d..ff8c020c4 100644
--- a/bsd/dev/ldd.h
+++ b/bsd/dev/ldd.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)ldd.h	2.0	03/20/90	(c) 1990 NeXT	
  *
diff --git a/bsd/dev/memdev.c b/bsd/dev/memdev.c
index a01fcb258..307ad77d9 100644
--- a/bsd/dev/memdev.c
+++ b/bsd/dev/memdev.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 University of Utah.
@@ -88,6 +94,7 @@
 #include <vm/pmap.h>
 #include <vm/vm_pager.h>
 #include <mach/memory_object_types.h>
+#include <kern/debug.h>
 
 #include <miscfs/devfs/devfs.h>
 
@@ -134,7 +141,7 @@ static struct cdevsw mdevcdevsw = {
 	/* ioctl */	mdevcioctl,
 	/* stop */	eno_stop,
 	/* reset */	eno_reset,
-	/* ttys */	0,
+	/* ttys */	NULL,
 	/* select */	eno_select,
 	/* mmap */	eno_mmap,
 	/* strategy */	eno_strat,
@@ -544,10 +551,10 @@ dev_t mdevadd(int devid, ppnum_t base, unsigned int size, int phys) {
 	}
 	else {
 		if(devid >= 16) {								/* Giving us something bogus? */
-			panic("mdevadd: attempt to explicitly add a bogus memory device: &08X\n", devid);
+			panic("mdevadd: attempt to explicitly add a bogus memory device: %08X\n", devid);
 		}
 		if(mdev[devid].mdFlags &mdInited) {				/* Already there? */
-			panic("mdevadd: attempt to explicitly add a previously defined memory device: &08X\n", devid);
+			panic("mdevadd: attempt to explicitly add a previously defined memory device: %08X\n", devid);
 		}
 	}
 	
diff --git a/bsd/dev/ppc/conf.c b/bsd/dev/ppc/conf.c
index 463700245..0e4a33aff 100644
--- a/bsd/dev/ppc/conf.c
+++ b/bsd/dev/ppc/conf.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved
@@ -38,10 +44,9 @@
 #include <sys/ioctl.h>
 #include <sys/tty.h>
 #include <sys/conf.h>
+#include <dev/ppc/cons.h>
 
 
-extern int	nulldev();
-
 struct bdevsw	bdevsw[] =
 {
 	/*
@@ -88,15 +93,26 @@ struct bdevsw	bdevsw[] =
 int	nblkdev = sizeof (bdevsw) / sizeof (bdevsw[0]);
 
 extern struct tty *km_tty[];
-extern int	consopen(), consclose(), consread(), conswrite(), consioctl(),
-		consselect(), cons_getc(), cons_putc();
-extern int	kmopen(),kmclose(),kmread(),kmwrite(),kmioctl(),
-		kmgetc(), kmputc(dev_t dev, char c);
 
-extern int	cttyopen(), cttyread(), cttywrite(), cttyioctl(), cttyselect();
+dev_t chrtoblk(dev_t dev);
+int chrtoblk_set(int cdev, int bdev);
+int isdisk(dev_t dev, int type);
+int iskmemdev(dev_t dev);
+
 
-extern int 	mmread(),mmwrite(),mmioctl();
-#define	mmselect	seltrue
+/* XXX No support for linker sets, so must declare here */
+int cttyopen(dev_t dev, int flag, int mode, struct proc *p);
+int cttyread(dev_t dev, struct uio *uio, int flag);
+int cttywrite(dev_t dev, struct uio *uio, int flag);
+int cttyioctl(dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p);
+int cttyselect(dev_t dev, int flag, void* wql, struct proc *p);
+
+/* XXX bsd/dev/ppc/mem.c */
+int mmread(dev_t dev, struct uio *uio, int flag);
+int mmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p);
+int mmwrite(dev_t dev, struct uio *uio, int flag);
+
+#define	mmselect	(select_fcn_t *)seltrue
 
 #if 1
 #ifdef NPTY
@@ -108,9 +124,18 @@ extern int 	mmread(),mmwrite(),mmioctl();
 #endif /* 1 */
 #if NPTY > 0
 extern struct tty *pt_tty[];
-extern int	ptsopen(),ptsclose(),ptsread(),ptswrite(),ptsstop(),ptsputc();
-extern int	ptcopen(),ptcclose(),ptcread(),ptcwrite(),ptcselect(),
-		ptyioctl();
+extern d_open_t		ptsopen;
+extern d_close_t	ptsclose;
+extern d_read_t		ptsread;
+extern d_write_t	ptswrite;
+extern d_stop_t		ptsstop;
+extern d_putc_t		ptsputc;
+extern d_open_t		ptcopen;
+extern d_close_t	ptcclose;
+extern d_read_t		ptcread;
+extern d_write_t	ptcwrite;
+extern d_select_t	ptcselect;
+extern d_ioctl_t	ptyioctl;
 #else
 #define ptsopen		eno_opcl
 #define ptsclose	eno_opcl
@@ -127,8 +152,11 @@ extern int	ptcopen(),ptcclose(),ptcread(),ptcwrite(),ptcselect(),
 #define ptyioctl	eno_ioctl
 #endif
 
-extern int	logopen(),logclose(),logread(),logioctl(),logselect();
-extern int	seltrue();
+extern d_open_t         logopen;
+extern d_close_t        logclose;
+extern d_read_t         logread;
+extern d_ioctl_t        logioctl;
+extern d_select_t       logselect;
 
 struct cdevsw	cdevsw[] =
 {
@@ -145,33 +173,46 @@ struct cdevsw	cdevsw[] =
 
     {
 	consopen,	consclose,	consread,	conswrite,	/* 0*/
-	consioctl,	nulldev,	nulldev,	0,	consselect,
-	eno_mmap,	eno_strat,	(getc_fcn_t *)cons_getc,	(putc_fcn_t *)cons_putc, D_TTY
+	consioctl,	((stop_fcn_t *)&nulldev),
+					((reset_fcn_t *)&nulldev),
+							0,	consselect,
+	eno_mmap,	eno_strat,	cons_getc,	cons_putc, D_TTY
    },
     NO_CDEVICE,								/* 1*/
     {
-	cttyopen,	nulldev,	cttyread,	cttywrite,	/* 2*/
-	cttyioctl,	nulldev,	nulldev,	0,		cttyselect,
+	cttyopen,	((open_close_fcn_t *)&nulldev),
+					cttyread,	cttywrite,	/* 2*/
+	cttyioctl,	((stop_fcn_t *)&nulldev),
+					((reset_fcn_t *)&nulldev),
+							0,	cttyselect,
 	eno_mmap,	eno_strat,	eno_getc,	eno_putc,	D_TTY
     },
     {
-	nulldev,	nulldev,	mmread,		mmwrite,	/* 3*/
-	mmioctl,	nulldev,	nulldev,	0,		(select_fcn_t *)mmselect,
+	((open_close_fcn_t *)&nulldev),
+			((open_close_fcn_t *)&nulldev),
+					mmread,		mmwrite,	/* 3*/
+	mmioctl,	((stop_fcn_t *)&nulldev),
+					((reset_fcn_t *)&nulldev),
+							0,	mmselect,
 	eno_mmap,		eno_strat,	eno_getc,	eno_putc,	D_DISK
     },
     {
 	ptsopen,	ptsclose,	ptsread,	ptswrite,	/* 4*/
-	ptyioctl,	ptsstop,	nulldev,	pt_tty,		ttselect,
+	ptyioctl,	ptsstop,	((reset_fcn_t *)&nulldev),
+							pt_tty,		ttselect,
 	eno_mmap,	eno_strat,	eno_getc,	eno_putc,	D_TTY
     },
     {
 	ptcopen,	ptcclose,	ptcread,	ptcwrite,	/* 5*/
-	ptyioctl,	nulldev,	nulldev,	0,		ptcselect,
+	ptyioctl,	((stop_fcn_t *)&nulldev),
+					((reset_fcn_t *)&nulldev),
+							0,		ptcselect,
 	eno_mmap,	eno_strat,	eno_getc,	eno_putc,	D_TTY
     },
     {
 	logopen,	logclose,	logread,	eno_rdwrt,	/* 6*/
-	logioctl,	eno_stop,	nulldev,	0,		logselect,
+	logioctl,	eno_stop,	((reset_fcn_t *)&nulldev),
+							0,		logselect,
 	eno_mmap,	eno_strat,	eno_getc,	eno_putc,	0
     },
     NO_CDEVICE,								/* 7*/
@@ -181,7 +222,9 @@ struct cdevsw	cdevsw[] =
     NO_CDEVICE,								/*11*/
     {
 	kmopen,		kmclose,	kmread,		kmwrite,	/*12*/
-	kmioctl,	nulldev,	nulldev,	km_tty,		ttselect,
+	kmioctl,	((stop_fcn_t *)&nulldev),
+					((reset_fcn_t *)&nulldev),
+							km_tty,		ttselect,
 	eno_mmap,	eno_strat,	kmgetc,		kmputc,		0
     },
     NO_CDEVICE,								/*13*/
@@ -229,9 +272,7 @@ int	nchrdev = sizeof (cdevsw) / sizeof (cdevsw[0]);
  * return true if a disk
  */
 int
-isdisk(dev, type)
-	dev_t dev;
-	int type;
+isdisk(dev_t dev, int type)
 {
 	dev_t	maj = major(dev);
 
@@ -282,8 +323,7 @@ static int chrtoblktab[] = {
  * convert chr dev to blk dev
  */
 dev_t
-chrtoblk(dev)
-	dev_t dev;
+chrtoblk(dev_t dev)
 {
 	int blkmaj;
 
@@ -309,8 +349,8 @@ chrtoblk_set(int cdev, int bdev)
 /*
  * Returns true if dev is /dev/mem or /dev/kmem.
  */
-int iskmemdev(dev)
-	dev_t dev;
+int
+iskmemdev(dev_t dev)
 {
 
 	return (major(dev) == 3 && minor(dev) < 2);
diff --git a/bsd/dev/ppc/cons.c b/bsd/dev/ppc/cons.c
index b9d966909..2e1fec1a6 100644
--- a/bsd/dev/ppc/cons.c
+++ b/bsd/dev/ppc/cons.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1987, 1988 NeXT, Inc.
@@ -47,10 +53,7 @@ struct tty	*constty;		/* current console device */
 
 /*ARGSUSED*/
 int
-consopen(dev, flag, devtype, pp)
-	dev_t dev;
-	int flag, devtype;
-	struct proc *pp;
+consopen(__unused dev_t dev, int flag, int devtype, struct proc *pp)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -70,10 +73,7 @@ consopen(dev, flag, devtype, pp)
 
 /*ARGSUSED*/
 int
-consclose(dev, flag, mode, pp)
-	dev_t dev;
-	int flag, mode;
-	struct proc *pp;
+consclose(__unused dev_t dev, int flag, int mode, struct proc *pp)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -94,10 +94,7 @@ consclose(dev, flag, mode, pp)
 
 /*ARGSUSED*/
 int
-consread(dev, uio, ioflag)
-	dev_t dev;
-	struct uio *uio;
-	int ioflag;
+consread(__unused dev_t dev, struct uio *uio, int ioflag)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -116,10 +113,7 @@ consread(dev, uio, ioflag)
 
 /*ARGSUSED*/
 int
-conswrite(dev, uio, ioflag)
-	dev_t dev;
-	struct uio *uio;
-	int ioflag;
+conswrite(__unused dev_t dev, struct uio *uio, int ioflag)
 {
     dev_t device;
 	boolean_t funnel_state;
@@ -138,12 +132,7 @@ conswrite(dev, uio, ioflag)
 
 /*ARGSUSED*/
 int
-consioctl(dev, cmd, addr, flag, p)
-	dev_t dev;
-	int cmd;
-	caddr_t addr;
-	int flag;
-	struct proc *p;
+consioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -159,7 +148,7 @@ consioctl(dev, cmd, addr, flag, p)
 	 * Superuser can always use this to wrest control of console
 	 * output from the "virtual" console.
 	 */
-	if (cmd == TIOCCONS && constty) {
+	if ((unsigned int)cmd == TIOCCONS && constty) {
 		error = proc_suser(p);
 		if (error) {
 			goto out;
@@ -178,11 +167,7 @@ out:
 /*ARGSUSED*/
 /* called with funnel held */
 int
-consselect(dev, flag, wql, p)
-	dev_t dev;
-	int flag;
-	void *wql;
-	struct proc *p;
+consselect(__unused dev_t dev, int flag, void *wql, struct proc *p)
 {
 	dev_t device;
 
@@ -194,7 +179,7 @@ consselect(dev, flag, wql, p)
 }
 
 int
-cons_getc()
+cons_getc(__unused dev_t dev)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -211,10 +196,8 @@ cons_getc()
 	return(error);
 }
 
-/*ARGSUSED*/
 int
-cons_putc(c)
-	char c;
+cons_putc(__unused dev_t dev, char c)
 {
 	dev_t device;
 	boolean_t funnel_state;
@@ -239,9 +222,9 @@ cons_putc(c)
  */
 int 
 alert(
-	int width, 
-	int height, 
-	const char *title, 
+	__unused int width, 
+	__unused int height, 
+	__unused const char *title, 
 	const char *msg, 
 	int p1, 
 	int p2, 
@@ -254,7 +237,7 @@ alert(
 {
 	char smsg[200];
 	
-	sprintf(smsg, msg,  p1, p2, p3, p4, p5, p6, p7, p8);
+	snprintf(smsg, sizeof(smsg), msg,  p1, p2, p3, p4, p5, p6, p7, p8);
 #if FIXME  /* [ */
 	/* DoAlert(title, smsg); */
 #else
@@ -265,7 +248,7 @@ alert(
 }
 
 int 
-alert_done()
+alert_done(void)
 {
 	/* DoRestore(); */
 	return 0;
diff --git a/bsd/dev/ppc/cons.h b/bsd/dev/ppc/cons.h
index 6da1b0ae7..acad7e39e 100644
--- a/bsd/dev/ppc/cons.h
+++ b/bsd/dev/ppc/cons.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1987 NeXT, Inc.
@@ -53,5 +59,30 @@ extern	struct tty *cn_tty;
 
 extern struct tty	cons;
 extern struct tty	*constty;		/* current console device */
+
+int consopen(dev_t, int, int, struct proc *);
+int consclose(dev_t, int, int, struct proc *);
+int consread(dev_t, struct uio *, int);
+int conswrite(dev_t, struct uio *, int);
+int consioctl(dev_t, u_long, caddr_t, int, struct proc *);
+int consselect(dev_t, int, void *, struct proc *);
+int cons_getc(dev_t);
+int cons_putc(dev_t, char);
+int alert(int, int, const char *, const char *, int, int, int, int, int, int, int, int);
+int alert_done(void);
+
+/*
+ * These really want their own header file, but this is the only one in
+ * common, and the km device is the keyboard monitor, so it's technically a
+ * part of the console.
+ */
+int kmopen(dev_t, int, int, struct proc *);
+int kmclose(dev_t, int, int, struct proc *);
+int kmread(dev_t, struct uio *, int);
+int kmwrite(dev_t, struct uio *, int);
+int kmioctl(dev_t, u_long, caddr_t, int, struct proc *);
+int kmgetc(dev_t);
+int kmputc(dev_t, char);
+
 #endif
 
diff --git a/bsd/dev/ppc/dtrace_isa.c b/bsd/dev/ppc/dtrace_isa.c
new file mode 100644
index 000000000..18449879a
--- /dev/null
+++ b/bsd/dev/ppc/dtrace_isa.c
@@ -0,0 +1,601 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+#include <stdarg.h>
+#include <string.h>
+#include <sys/malloc.h>
+#include <sys/time.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
+#include <kern/task.h>
+#include <kern/sched_prim.h>
+#include <miscfs/devfs/devfs.h>
+#include <mach/vm_param.h>
+#include <machine/cpu_capabilities.h>
+
+extern dtrace_id_t      dtrace_probeid_error;   /* special ERROR probe */
+
+void
+dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
+    int fault, int fltoffs, uint64_t illval)
+{
+	/*
+	 * dtrace_getarg() is a lost cause on PPC. For the case of the error probe firing lets
+	 * stash away "illval" here, and special-case retrieving it in DIF_VARIABLE_ARG.
+	 */
+	state->dts_arg_error_illval = illval;
+	dtrace_probe( dtrace_probeid_error, (uint64_t)(uintptr_t)state, epid, which, fault, fltoffs );
+}
+
+/*
+ * Atomicity and synchronization
+ */
+void
+dtrace_membar_producer(void)
+{
+	__asm__ volatile("sync");
+}
+
+void
+dtrace_membar_consumer(void)
+{
+	__asm__ volatile("isync");
+}
+
+/*
+ * Interrupt manipulation
+ * XXX dtrace_getipl() can be called from probe context.
+ */
+int
+dtrace_getipl(void)
+{
+	return (ml_at_interrupt_context() ? 1: 0);
+}
+
+/*
+ * MP coordination
+ */
+typedef void (*broadcastFunc) (uint32_t);
+
+int32_t cpu_broadcast(uint32_t *, broadcastFunc, uint32_t); /* osfmk/ppc/machine_cpu.h */
+
+typedef struct xcArg {
+	processorid_t cpu;
+	dtrace_xcall_t f;
+	void *arg;
+	uint32_t waitVar;
+} xcArg_t;
+
+static void
+xcRemote( uint32_t foo )
+{
+	xcArg_t *pArg = (xcArg_t *)foo;
+	
+	if ( pArg->cpu == CPU->cpu_id || pArg->cpu == DTRACE_CPUALL ) {
+		(pArg->f)(pArg->arg);
+	}
+	
+    if(!hw_atomic_sub(&(pArg->waitVar), 1)) {      /* Drop the wait count */
+        thread_wakeup((event_t)&(pArg->waitVar));  /* If we were the last, wake up the signaller */
+    }
+}
+
+/*
+ * dtrace_xcall() is not called from probe context.
+ */
+void
+dtrace_xcall(processorid_t cpu, dtrace_xcall_t f, void *arg)
+{
+	xcArg_t xcArg;
+	
+	/* Talking to ourselves, are we? */
+	if ( cpu == CPU->cpu_id ) {
+		(*f)(arg);
+		return;
+	}
+	
+	if ( cpu == DTRACE_CPUALL ) {
+		(*f)(arg);
+	}
+	
+	xcArg.cpu = cpu;
+	xcArg.f = f;
+	xcArg.arg = arg;
+    xcArg.waitVar = 0;
+
+	(void)cpu_broadcast(&(xcArg.waitVar), xcRemote, (uint32_t)&xcArg);
+}
+
+/*
+ * Runtime and ABI
+ */
+extern greg_t
+dtrace_getfp(void)
+{
+	return (greg_t)__builtin_frame_address(0);
+}
+
+uint64_t
+dtrace_getreg(struct regs *savearea, uint_t reg)
+{
+	ppc_saved_state_t *regs = (ppc_saved_state_t *)savearea;
+    uint64_t mask = (_cpu_capabilities & k64Bit) ? 0xffffffffffffffffULL : 0x00000000ffffffffULL;
+	
+	/* See osfmk/ppc/savearea.h */
+	if (reg > 68) { /* beyond mmcr2 */
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+		return (0);
+	}
+	
+	switch (reg) {
+		/* First 38 registers are saved to 64 bits r0-r31, srr0, srr1, xer, lr, ctr, dar. */
+		default:
+			return (((uint64_t *)(&(regs->save_r0)))[reg]) & mask;
+
+		/* Handle the 32-bit registers */
+		case 38: case 39: case 40: case 41: /* cr, dsisr, exception, vrsave */
+		case 42: case 43: case 44: case 45: /* vscr[4] */
+		case 46: case 47: case 48: case 49:     /* fpscrpad, fpscr, save_1d8[2] */
+		case 50: case 51: case 52: case 53: /* save_1E0[8] */
+		case 54: case 55: case 56: case 57: 
+		case 58: case 59: case 60: case 61: /* save_pmc[8] */
+		case 62: case 63: case 64: case 65: 
+			return (uint64_t)(((unsigned int *)(&(regs->save_cr)))[reg - 38]);
+			
+		case 66:
+			return regs->save_mmcr0 & mask;
+		case 67:
+			return regs->save_mmcr1 & mask;
+		case 68:
+			return regs->save_mmcr2 & mask;
+	}
+}
+
+#define RETURN_OFFSET 8
+#define RETURN_OFFSET64 16
+#define REGPC save_srr0
+#define REGSP save_r1
+
+/*
+ * XXX dtrace_getustack_common() can be called from probe context.
+ */
+static int
+dtrace_getustack_common(uint64_t *pcstack, int pcstack_limit, user_addr_t pc,
+    user_addr_t sp)
+{
+#if 0
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+
+	uintptr_t oldcontext = lwp->lwp_oldcontext; /* XXX signal stack crawl*/
+	size_t s1, s2;
+#endif
+	int ret = 0;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	ASSERT(pcstack == NULL || pcstack_limit > 0);
+	
+#if 0 /* XXX signal stack crawl*/
+	if (p->p_model == DATAMODEL_NATIVE) {
+		s1 = sizeof (struct frame) + 2 * sizeof (long);
+		s2 = s1 + sizeof (siginfo_t);
+	} else {
+		s1 = sizeof (struct frame32) + 3 * sizeof (int);
+		s2 = s1 + sizeof (siginfo32_t);
+	}
+#endif
+
+	while (pc != 0) {
+		ret++;
+		if (pcstack != NULL) {
+			*pcstack++ = (uint64_t)pc;
+			pcstack_limit--;
+			if (pcstack_limit <= 0)
+				break;
+		}
+
+		if (sp == 0)
+			break;
+
+#if 0 /* XXX signal stack crawl*/
+		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
+			if (p->p_model == DATAMODEL_NATIVE) {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fulword(&gregs[REG_FP]);
+				pc = dtrace_fulword(&gregs[REG_PC]);
+
+				oldcontext = dtrace_fulword(&ucp->uc_link);
+			} else {
+				ucontext32_t *ucp = (ucontext32_t *)oldcontext;
+				greg32_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fuword32(&gregs[EBP]);
+				pc = dtrace_fuword32(&gregs[EIP]);
+
+				oldcontext = dtrace_fuword32(&ucp->uc_link);
+			}
+		} 
+		else
+#endif
+		{
+			if (is64Bit) {
+				pc = dtrace_fuword64((sp + RETURN_OFFSET64));
+				sp = dtrace_fuword64(sp);
+			} else {
+				pc = dtrace_fuword32((sp + RETURN_OFFSET));
+				sp = dtrace_fuword32(sp);
+			}
+		}
+	}
+
+	return (ret);
+}
+
+void
+dtrace_getupcstack(uint64_t *pcstack, int pcstack_limit)
+{
+	thread_t thread = current_thread();
+	ppc_saved_state_t *regs;
+	user_addr_t pc, sp;
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+	int n;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (*flags & CPU_DTRACE_FAULT)
+		return;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	/*
+	 * If there's no user context we still need to zero the stack.
+	 */
+	if (thread == NULL)
+		goto zero;
+
+	regs = (ppc_saved_state_t *)find_user_regs(thread);
+	if (regs == NULL)
+		goto zero;
+		
+	*pcstack++ = (uint64_t)proc_selfpid();
+	pcstack_limit--;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	pc = regs->REGPC;
+	sp = regs->REGSP;
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		*pcstack++ = (uint64_t)pc;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			return;
+
+		pc = regs->save_lr;
+	}
+	
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_USTACK_FP)) {
+		/*
+		 * If the ustack fp flag is set, the stack frame from sp to
+		 * fp contains no valid call information. Start with the fp.
+		 */
+		if (is64Bit)
+			sp = dtrace_fuword64(sp);
+		else
+			sp = (user_addr_t)dtrace_fuword32(sp);
+	}
+
+	n = dtrace_getustack_common(pcstack, pcstack_limit, pc, sp);
+	ASSERT(n >= 0);
+	ASSERT(n <= pcstack_limit);
+
+	pcstack += n;
+	pcstack_limit -= n;
+
+zero:
+	while (pcstack_limit-- > 0)
+		*pcstack++ = 0;
+}
+
+int
+dtrace_getustackdepth(void)
+{
+	thread_t thread = current_thread();
+	ppc_saved_state_t *regs;
+	user_addr_t pc, sp;
+	int n = 0;
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (thread == NULL)
+		return 0;
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT))
+		return (-1);
+
+	regs = (ppc_saved_state_t *)find_user_regs(thread);
+	if (regs == NULL)
+		return 0;
+
+	pc = regs->REGPC;
+	sp = regs->REGSP;
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		n++;
+		pc = regs->save_lr;
+	}
+	
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_USTACK_FP)) {
+		/*
+		 * If the ustack fp flag is set, the stack frame from sp to
+		 * fp contains no valid call information. Start with the fp.
+		 */
+		if (is64Bit)
+			sp = dtrace_fuword64(sp);
+		else
+			sp = (user_addr_t)dtrace_fuword32(sp);
+	}
+
+	n += dtrace_getustack_common(NULL, 0, pc, sp);
+
+	return (n);
+}
+
+void
+dtrace_getufpstack(uint64_t *pcstack, uint64_t *fpstack, int pcstack_limit)
+{
+	thread_t thread = current_thread();
+	ppc_saved_state_t *regs;
+	user_addr_t pc, sp;
+	volatile uint16_t *flags =
+	    (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
+#if 0
+	uintptr_t oldcontext;
+	size_t s1, s2;
+#endif
+	boolean_t is64Bit = proc_is64bit(current_proc());
+
+	if (*flags & CPU_DTRACE_FAULT)
+		return;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	/*
+	 * If there's no user context we still need to zero the stack.
+	 */
+	if (thread == NULL)
+		goto zero;
+
+	regs = (ppc_saved_state_t *)find_user_regs(thread);
+	if (regs == NULL)
+		goto zero;
+		
+	*pcstack++ = (uint64_t)proc_selfpid();
+	pcstack_limit--;
+
+	if (pcstack_limit <= 0)
+		return;
+
+	pc = regs->REGPC;
+	sp = regs->REGSP;
+	
+#if 0 /* XXX signal stack crawl*/
+	oldcontext = lwp->lwp_oldcontext;
+
+	if (p->p_model == DATAMODEL_NATIVE) {
+		s1 = sizeof (struct frame) + 2 * sizeof (long);
+		s2 = s1 + sizeof (siginfo_t);
+	} else {
+		s1 = sizeof (struct frame32) + 3 * sizeof (int);
+		s2 = s1 + sizeof (siginfo32_t);
+	}
+#endif
+
+	if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_ENTRY)) {
+		*pcstack++ = (uint64_t)pc;
+		*fpstack++ = 0;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			return;
+
+		/*
+		 * XXX This is wrong, but we do not yet support stack helpers.
+		 */
+		if (is64Bit)
+			pc = dtrace_fuword64(sp);
+		else
+			pc = dtrace_fuword32(sp);
+	}
+
+	while (pc != 0) {
+		*pcstack++ = (uint64_t)pc;
+		*fpstack++ = sp;
+		pcstack_limit--;
+		if (pcstack_limit <= 0)
+			break;
+
+		if (sp == 0)
+			break;
+
+#if 0 /* XXX signal stack crawl*/
+		if (oldcontext == sp + s1 || oldcontext == sp + s2) {
+			if (p->p_model == DATAMODEL_NATIVE) {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fulword(&gregs[REG_FP]);
+				pc = dtrace_fulword(&gregs[REG_PC]);
+
+				oldcontext = dtrace_fulword(&ucp->uc_link);
+			} else {
+				ucontext_t *ucp = (ucontext_t *)oldcontext;
+				greg_t *gregs = ucp->uc_mcontext.gregs;
+
+				sp = dtrace_fuword32(&gregs[EBP]);
+				pc = dtrace_fuword32(&gregs[EIP]);
+
+				oldcontext = dtrace_fuword32(&ucp->uc_link);
+			}
+		} 
+		else
+#endif
+		{
+			if (is64Bit) {
+				pc = dtrace_fuword64((sp + RETURN_OFFSET64));
+				sp = dtrace_fuword64(sp);
+			} else {
+				pc = dtrace_fuword32((sp + RETURN_OFFSET));
+				sp = dtrace_fuword32(sp);
+			}
+		}
+	}
+
+zero:
+	while (pcstack_limit-- > 0)
+		*pcstack++ = 0;
+}
+
+void
+dtrace_getpcstack(pc_t *pcstack, int pcstack_limit, int aframes,
+    uint32_t *intrpc)
+{
+	struct frame *fp = (struct frame *)dtrace_getfp();
+	struct frame *nextfp, *minfp, *stacktop;
+	int depth = 0;
+	int last = 0;
+	uintptr_t pc;
+	uintptr_t caller = CPU->cpu_dtrace_caller;
+	int on_intr;
+
+	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
+		stacktop = (struct frame *)dtrace_get_cpu_int_stack_top();
+	else
+		stacktop = (struct frame *)(dtrace_get_kernel_stack(current_thread()) + KERNEL_STACK_SIZE);
+
+	minfp = fp;
+
+	aframes++;
+
+	if (intrpc != NULL && depth < pcstack_limit)
+		pcstack[depth++] = (pc_t)intrpc;
+
+	while (depth < pcstack_limit) {
+		nextfp = *(struct frame **)fp;
+		pc = *(uintptr_t *)(((uint32_t)fp) + RETURN_OFFSET);
+
+		if (nextfp <= minfp || nextfp >= stacktop) {
+			if (on_intr) {
+				/*
+				 * Hop from interrupt stack to thread stack.
+				 */
+				vm_offset_t kstack_base = dtrace_get_kernel_stack(current_thread());
+
+				minfp = (struct frame *)kstack_base;
+				stacktop = (struct frame *)(kstack_base + KERNEL_STACK_SIZE);
+
+				on_intr = 0;
+				continue;
+			}
+			/*
+			 * This is the last frame we can process; indicate
+			 * that we should return after processing this frame.
+			 */
+			last = 1;
+		}
+
+		if (aframes > 0) {
+			if (--aframes == 0 && caller != 0) {
+				/*
+				 * We've just run out of artificial frames,
+				 * and we have a valid caller -- fill it in
+				 * now.
+				 */
+				ASSERT(depth < pcstack_limit);
+				pcstack[depth++] = (pc_t)caller;
+				caller = 0;
+			}
+		} else {
+			if (depth < pcstack_limit)
+				pcstack[depth++] = (pc_t)pc;
+		}
+
+		if (last) {
+			while (depth < pcstack_limit)
+				pcstack[depth++] = 0;
+			return;
+		}
+
+		fp = nextfp;
+		minfp = fp;
+	}
+}
+
+uint64_t
+dtrace_getarg(int arg, int aframes)
+{
+#pragma unused(arg,aframes)
+	return 0xfeedfacedeafbeadLL; /* XXX Only called for arg >= 5 */
+}
+
+/*
+ * Load/Store Safety
+ */
+
+void
+dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
+{
+	/*
+	 * "base" is the smallest toxic address in the range, "limit" is the first
+	 * VALID address greater than "base".
+	 */
+	func(0x0, VM_MIN_KERNEL_ADDRESS);
+	func(VM_MAX_KERNEL_ADDRESS + 1, ~(uintptr_t)0);
+}
+
+extern void *mapping_phys_lookup(ppnum_t, unsigned int *);
+
+boolean_t
+dtxnu_is_RAM_page(ppnum_t pn)
+{
+	unsigned int ignore;
+	return (NULL == mapping_phys_lookup(pn, &ignore)) ? FALSE : TRUE;
+}
+
diff --git a/bsd/dev/ppc/dtrace_subr_ppc.c b/bsd/dev/ppc/dtrace_subr_ppc.c
new file mode 100644
index 000000000..5040a9183
--- /dev/null
+++ b/bsd/dev/ppc/dtrace_subr_ppc.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)dtrace_subr.c	1.12	05/06/08 SMI"
+ */
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <sys/dtrace.h>
+#include <sys/dtrace_glue.h>
+#include <sys/dtrace_impl.h>
+#include <sys/fasttrap.h>
+#include <sys/vm.h>
+#include <sys/user.h>
+#include <sys/kauth.h>
+#include <kern/debug.h>
+
+int (*dtrace_pid_probe_ptr)(ppc_saved_state_t *);
+int (*dtrace_return_probe_ptr)(ppc_saved_state_t *);
+kern_return_t dtrace_user_probe(ppc_saved_state_t *sv);
+
+kern_return_t
+dtrace_user_probe(ppc_saved_state_t *sv)
+{
+
+	lck_rw_t *rwp;
+	struct proc *p = current_proc();
+
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+	/*
+	 * DTrace accesses t_cred in probe context.	 t_cred
+	 * must always be either NULL, or point to a valid,
+	 * allocated cred structure.
+	 */
+	kauth_cred_uthread_update(uthread, p);
+
+	if (sv->save_exception == T_DTRACE_RET) {
+
+/*
+ *		T_DTRACE_RET is generated by the kernel when an emulation sequence 
+ *		ends.  Unlike the x86 implementation, this can not be caused by
+ *		a user state trap instruction.  It is a system error if it occurs
+ *		when not stepping and is, therefore, a panickable offence.
+ */
+
+		if(uthread->t_dtrace_step == 0) {	/* Are we supposed to be tracing? */
+			panic("dtrace_user_probe: T_DTRACE_RET when not stepping\n");
+		}
+
+		if (uthread->t_dtrace_ast) {
+			printf("dtrace_user_probe() should be calling aston()\n");
+			// aston(uthread);
+			// uthread->t_sig_check = 1;
+		}
+
+		/*
+		 * Clear all user tracing flags.
+		 */
+		uthread->t_dtrace_ft = 0;
+
+		/*
+		 * We need to wait until after we've called the
+		 * dtrace_return_probe_ptr function pointer to step the pc.
+		 */
+		rwp = &CPU->cpu_ft_lock;
+		lck_rw_lock_shared(rwp);
+
+		if (dtrace_return_probe_ptr != NULL) (void)(*dtrace_return_probe_ptr)(sv);
+		lck_rw_unlock_shared(rwp);
+
+		sv->save_srr0 = sv->save_srr0 + 4;	/* Step to next instruction */
+		if(!(sv->save_srr1 & 0x8000000000000000ULL)) sv->save_srr0 &= 0x00000000FFFFFFFF;	/* Trim if in 32-bit mode */
+
+		return KERN_SUCCESS;
+		
+	} else {
+
+/*
+ *	We have taken our normal trap to get here.  Make sure we expect it
+ */
+		uint32_t instr;
+		rwp = &CPU->cpu_ft_lock;
+
+		/*
+		 * The DTrace fasttrap provider uses a trap, "twi 31,r31,0xDDDD".
+		 * We will only be here if dtrace (or someone pretending to be us)
+		 * sets the trap.
+		 * We let DTrace take the first crack at handling
+		 * this trap; if it's not a probe that DTrace knowns about,
+		 * we call into the trap() routine to handle it like a
+		 * breakpoint placed by a conventional debugger.
+		 */
+
+		/*
+		 * APPLE NOTE: I believe the purpose of the reader/writers lock
+		 * is thus: There are times which dtrace needs to prevent calling
+		 * dtrace_pid_probe_ptr(). Sun's original impl grabbed a plain
+		 * mutex here. However, that serialized all probe calls, and
+		 * destroyed MP behavior. So now they use a RW lock, with probes
+		 * as readers, and the top level synchronization as a writer.
+		 */
+		lck_rw_lock_shared(rwp);
+		if (dtrace_pid_probe_ptr != NULL && 
+			(*dtrace_pid_probe_ptr)(sv) == 0) {
+			lck_rw_unlock_shared(rwp);
+			return KERN_SUCCESS;
+		}
+		lck_rw_unlock_shared(rwp);
+
+		/*
+		 * If the instruction that caused the breakpoint trap doesn't
+		 * look like our trap anymore, it may be that this tracepoint
+		 * was removed just after the user thread executed it. In
+		 * that case, return to user land to retry the instuction.
+		 *
+		 * Note that the PC is correct because we do not advance it until after emulation.
+		 */
+		if (fuword32(sv->save_srr0, &instr) == 0 && instr != FASTTRAP_INSTR) {
+			return KERN_SUCCESS;
+		}
+
+	}
+
+/*
+ *	If we get here, we go back to throw an exception
+ */
+
+	return KERN_FAILURE;
+}
+
+void
+dtrace_safe_synchronous_signal(void)
+{
+// This is commented out of the x86 code and is never called.
+}
+
+int
+dtrace_safe_defer_signal(void)
+{
+// This is commented out of the x86 code and is never called.
+	return 0;
+}
diff --git a/bsd/dev/ppc/fasttrap_isa.c b/bsd/dev/ppc/fasttrap_isa.c
new file mode 100644
index 000000000..33b7ff4d4
--- /dev/null
+++ b/bsd/dev/ppc/fasttrap_isa.c
@@ -0,0 +1,722 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*
+ * #pragma ident	"@(#)fasttrap_isa.c	1.23	06/09/19 SMI"
+ */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <sys/fasttrap_isa.h>
+#include <sys/fasttrap_impl.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/dtrace_ptss.h>
+#include <kern/debug.h>
+#include <ppc/decodePPC.h>
+#include <kern/task.h>
+#include <mach/vm_param.h>
+#include <mach/mach_vm.h>
+#include <mach/task.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h> /* All the bits we care about are guarded by MACH_KERNEL_PRIVATE :-( */
+
+#define proc_t struct proc
+
+static int32_t branchtaken(int32_t bo, int32_t bi, ppc_saved_state_t *sv);
+static int32_t dtrace_decode_ppc(uint32_t inst);
+int patchInst(task_t task, addr64_t vaddr, uint32_t inst);
+kern_return_t dtrace_user_probe(ppc_saved_state_t *sv);
+
+/*
+ * Lossless User-Land Tracing on PPC
+ * ---------------------------------
+ *
+ * PPC uses a different technique to emulate user-land instruction replaces by a probe
+ * trap than x86.
+ *
+ * Like x86, it will emulate all forms of branch instructions.  We will not attempt
+ * to emulate any instruction that we know will cause an interruption or exception
+ * (system call, trap, privileged instruction, instruction that uses a privileged
+ * register).
+ *
+ * NOTE: I am thinking that we should punish tight loopers, e.g., branch-to-dot.
+ * Depending upon clock resolution and how fast we can process these guys, it is
+ * possible that its quantum will never decrease.  Maybe we could just manually
+ * end the guy's quantum and let the next guy go...
+ *
+ * When fasttrap_tracepoint_init is called, we fetch the instruction and decode it.
+ * If we don't recognize it or find it is a "banned" instruction, we return -1,
+ * telling our caller to forget it.  Otherwise we save the instruction image and
+ * enough of the decode to quickly handle it at probe time.  We cram it into
+ * the fasttrap_machtp_t structure.
+ *
+ * When the probe hits, we verify that the PC is still a probe point and if not,
+ * we bail.  Otherwise we have a bit more to do.
+ *
+ * If DTFTP_ENTRY is set, we have an entry probe and need to call dtrace_probe.
+ *
+ * If DTFTP_IS_ENABLED is set, all we need to do is to return a 1.
+ *
+ * If ftp_argmap is NULL, we call dtrace_probe
+ *
+ * Otherwise, we figure out what the arguments are and pass them to dtrace_probe
+ *
+ * Next, we need to set up to emulate the probed instruction and here is where we are
+ * the most different than the x86 code.
+ *
+ * Like x86, we first check to see if the instruction is any form of branch.  If so, 
+ * we emulate it completely within the kernel and are done.
+ *
+ * If it is anything else, we build a code stream within the kernel to execute the
+ * instruction.  Note that this is very different from x86 which build the code in
+ * userland.
+ *
+ * The generated stream needs to be executed within the kernel's code space but with
+ * the user address space and registers.  Because PPC allows different translation modes
+ * for instruction fetch and data fetch, this is not too difficult.
+ *
+ * There are two kinds streams needed: execute and continue, and execute and return,
+ * which are used for entry/offset and exit probes respectivily. 
+ *
+ * The probe code will copy the instruction image into the current user savearea (which
+ * also contains the complete user state register context).  A flag that requests either
+ * execute/continue or execute/return is also set in the savearea.
+ *
+ * We now exit the dtrace code and the marked context makes its way back to the point
+ * where it will be dispatched on the processor.
+ *
+ * The exception return code will start to restore the user context, including registers
+ * and address space.  However, before dispatching the user, it will notice that the
+ * emulate flags are set.  At this point the code will build a code stream 
+ * in an area in the per_proc that consists of
+ * the original instruction followed by a trap instruction.  It will set the new MSR (in
+ * SRR1) to have address translation enable for data, translation disabled for instruction
+ * fetches, interruptions disabled, and supervisor state.
+ *
+ * The new PC and MSR are loaded via a RFID and the generated stream is executed. If a
+ * synchronous fault occurs, it is either handled (PTE miss, FPU or vector unavailable),
+ * emulated (alignment or denorm), or passed on to the user.
+ *
+ * Assuming the emulated instruction completes, the trap will execute.  When that happens, 
+ * low-level trap handler will check its flags.  If the trap corresponds to an
+ * execute/continue stream, the trap handler will adjust the PC and complete the
+ * transition into user space. 
+ *
+ * If the trap corresponds to an execute/return stream, the handler will generate 
+ * a T_DTRACE_RET exception and let the trap handler pass it along to dtrace_user_probe.
+ *
+ */
+
+
+static uint64_t
+fasttrap_anarg(ppc_saved_state_t *sv, int function_entry, int argno)
+{
+#pragma unused(function_entry)
+	uint32_t farg;
+ 	uint64_t value;
+ 	
+ 	/* The first 8 arguments (argno 0-7) are in registers */
+ 	if (argno < 8) {
+ 		value = (&sv->save_r3)[argno];
+ 	} else {
+ 		if (sv->save_srr1 & 0x8000000000000000ULL) {
+ 			/* 64-bit */
+ 			/* Grab argument >= 8 from stack */
+ 			fasttrap_fuword64_noerr(sv->save_r1 + 48 + ((argno)* sizeof(uint64_t)), &value);
+ 		} else {
+ 			/* 32-bit */
+			/* Grab argument >= 8 from stack */
+ 			fasttrap_fuword32_noerr(sv->save_r1 + 24 + ((argno) * sizeof(uint32_t)), &farg);
+			value = (uint64_t)farg;
+ 		}
+ 	}
+ 	
+ 	return (value);
+}
+
+/*ARGSUSED*/
+int
+fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, user_addr_t pc,
+    fasttrap_probe_type_t type)
+{
+#pragma unused(type)
+
+	uint32_t instr, testr1, testr2, testr3;
+	user_addr_t targpc;
+	int32_t target, optype;
+
+	/*
+	 * Read the instruction at the given address out of the process's
+	 * address space. We don't have to worry about a debugger
+	 * changing this instruction before we overwrite it with our trap
+	 * instruction since P_PR_LOCK is set. Since instructions can span
+	 * pages, we potentially read the instruction in two parts. If the
+	 * second part fails, we just zero out that part of the instruction.
+	 */
+	/*
+	 * APPLE NOTE: Of course, we do not have a P_PR_LOCK, so this is racey...
+	 */
+
+	if (uread(p, &instr, 4, pc) != 0) return (-1);	/* Grab instruction, return suddenly if read fails... */
+		
+	optype = dtrace_decode_ppc(instr);		/* See if we have an instruction we can probe */
+	
+	tp->ftt_instr = instr;					/* Save the instruction image */
+	testr1 = tp->ftt_bo = (uint8_t)((instr >> (31 - 10)) & 0x1F);	/* Extract branch options */
+	testr2 = tp->ftt_bi = (uint8_t)((instr >> (31 - 15)) & 0x1F);	/* Extract condition register bit */
+	testr3 = (instr >> (31 - 20)) & 0x1F;	/* Get that last register */
+	tp->ftt_flgs = (uint8_t)(instr & 3);	/* Set the absolute address and link flags */
+
+	switch(optype) {						/* Do instruction specific decode */
+		
+		case diCMN:							/* Common instruction */
+			tp->ftt_type = ftmtCommon;		/* Mark as common instruction */
+			break;
+			
+		case diINV: 						/* Invalid */
+		case diTRP:							/* Trap */
+		case diSC:							/* System Call */
+		case diRFI: 						/* Return from interrupt */
+		case diPRV:							/* Priviliged instruction */
+			return (-1);					/* We will not emulate these... */
+			break;
+		
+		case diB:							/* Branch */
+			tp->ftt_type = ftmtB;			/* Mark as branch instruction */
+			target = instr & 0x03FFFFFC;	/* Extract address or offset */
+			if(target & 0x02000000) target |= 0xFC000000;	/* Sign extend */
+			tp->ftt_trgt = target;			/* Trim back down and save */
+			
+			targpc = (user_addr_t)((int64_t)target);	/* Generate a target address, hopefully we sign extend... */
+			if(!(tp->ftt_flgs & ftmtAbs)) {	/* Are we dealing with an offset here? */
+				targpc = targpc + pc;		/* Apply offset to get target address */
+			}
+			
+			if(targpc == pc) return -1;		/* Branching to self is a sin and is forbidden... */
+			break;
+			
+		case diBC:							/* Branch conditional */
+			tp->ftt_type = ftmtBC;			/* Mark as branch conditional */
+			target = instr & 0x0000FFFC;	/* Extract address or offset */
+			if(target & 0x00008000) target |= 0xFFFF0000;	/* Sign extend */
+			tp->ftt_trgt = target;			/* Trim back down and save */
+			
+			targpc = (user_addr_t)((int64_t)target);	/* Generate a target address, hopefully we sign extend... */
+			if(!(tp->ftt_flgs & ftmtAbs)) {		/* Are we dealing with an offset here? */
+				targpc = targpc + pc;		/* Apply offset to get target address */
+			}
+			
+			if(targpc == pc) return -1;		/* Branching to self is a sin and is forbidden... */
+			break;
+			
+		case diBLR:							/* Branch conditional to link register */
+			tp->ftt_type = ftmtBLR;			/* Mark as branch conditional to link register */
+			break;
+			
+		case diBCTR:						/* Branch conditional to count register */
+			tp->ftt_type = ftmtBCTR;		/* Mark as branch conditional to count register */
+			break;
+			
+		case diOR:							/* OR */
+			if((instr >> 26) == 24) {		/* Is this the ORI nop? */
+				if((testr1 == testr2) && ((instr & 0x0000FFFF) == 0)) tp->ftt_type = ftmtNOP;	/* Remember if this is a NOP instruction */
+				else tp->ftt_type = ftmtCommon;	/* Otherwise it is a common ORI instruction */
+			}
+			else if((testr1 == testr2) && (testr1 == testr3)) tp->ftt_type = ftmtNOP;	/* If all three registers are the same, this is a NOP */
+			else tp->ftt_type = ftmtCommon;	/* Otherwise it is a common OR instruction */
+
+			break;
+			
+		default:
+			panic("fasttrap_tracepoint_init: invalid branch decode, inst = %08X, optype = %d\n", instr, optype);
+			break;
+			
+	}
+
+	return (0);
+}
+
+int
+fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	return patchInst(p->task, tp->ftt_pc, FASTTRAP_INSTR);	/* Patch the instruction and flush it */
+}
+
+extern void dbgTrace(uint32_t, uint32_t, uint32_t, uint32_t, uint32_t);
+
+int
+fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	uint32_t instr;
+
+	/*
+	 * Distinguish between read or write failures and a changed
+	 * instruction.
+	 */
+	if (uread(p, &instr, 4, tp->ftt_pc) != 0) return (0);	/* Get the instruction, but exit if not mapped */
+
+//	dbgTrace(0x99999999, (uint32_t)tp->ftt_pc, tp->ftt_instr, instr, 0);	/* (TRACE/DEBUG) */
+
+	if (instr != FASTTRAP_INSTR) return (0);	/* Did someone change it? If so, just leave */
+
+	return patchInst(p->task, tp->ftt_pc, tp->ftt_instr);	/* Patch the old instruction back in and flush it */
+}
+
+static void
+fasttrap_return_common(ppc_saved_state_t *sv, user_addr_t pc, pid_t pid, user_addr_t new_pc)
+{
+
+	fasttrap_tracepoint_t *tp;
+	fasttrap_bucket_t *bucket;
+	fasttrap_id_t *id;
+	lck_mtx_t *pid_mtx;
+
+	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
+	lck_mtx_lock(pid_mtx);
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    !tp->ftt_proc->ftpc_defunct)
+			break;
+	}
+
+	/*
+	 * Don't sweat it if we can't find the tracepoint again. Unlike
+	 * when we're in fasttrap_pid_probe(), finding the tracepoint here
+	 * is not essential to the correct execution of the process.
+	 */
+	if (tp == NULL) {
+		lck_mtx_unlock(pid_mtx);
+		return;
+	}
+
+	for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
+		/*
+		 * If there's a branch that could act as a return site, we
+		 * need to trace it, and check here if the program counter is
+		 * external to the function.
+		 */
+		if((new_pc - id->fti_probe->ftp_faddr) < id->fti_probe->ftp_fsize)	/* Is target within the function? */
+			continue;							/* Yeah, skip this one... */
+
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_USTACK_FP);
+		dtrace_probe(id->fti_probe->ftp_id,
+			pc - id->fti_probe->ftp_faddr,
+			sv->save_r3, sv->save_r4, 0, 0);
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_USTACK_FP);
+	}
+
+	lck_mtx_unlock(pid_mtx);
+}
+
+static void
+fasttrap_usdt_args(fasttrap_probe_t *probe, ppc_saved_state_t *sv, int argc,
+    uint64_t *argv)
+{
+	int i, x, cap = MIN(argc, probe->ftp_nargs);
+	uint32_t farg;
+
+	for (i = 0; i < cap; i++) {
+		x = probe->ftp_argmap[i];
+
+		if (x <= 8) {							/* Is this argument in a register? */
+			argv[i] = (&sv->save_r0)[x];
+		} else {
+			if(sv->save_srr1 & 0x8000000000000000ULL) {	/* Are we running in 64-bit? */
+				fasttrap_fuword64_noerr(sv->save_r1 + 48 + (x * sizeof(uint64_t)), &argv[i]);	/* Grab argument > 8 from stack */
+			}
+			else {
+				fasttrap_fuword32_noerr(sv->save_r1 + 24 + (x * sizeof(uint32_t)), &farg);	/* Grab argument > 8 from stack */
+				argv[i] = (uint64_t)farg;		/* Convert to 64-bit */
+			}
+		}
+	}
+
+	for (; i < argc; i++) {
+		argv[i] = 0;
+	}
+}
+
+int
+fasttrap_pid_probe(ppc_saved_state_t *sv)
+{
+	proc_t *p = current_proc();
+	fasttrap_bucket_t *bucket;
+	lck_mtx_t *pid_mtx;
+	fasttrap_tracepoint_t *tp, tp_local;
+	pid_t pid;
+	dtrace_icookie_t cookie;
+	uint_t is_enabled = 0;
+	user_addr_t new_pc = 0;
+	user_addr_t pc;
+	user_addr_t addrmask;
+
+	pc = sv->save_srr0;							/* Remember the PC for later */
+	if(sv->save_srr1 & 0x8000000000000000ULL) addrmask = 0xFFFFFFFFFFFFFFFFULL;	/* Set 64-bit addressing if enabled */
+	else addrmask = 0x00000000FFFFFFFFULL;		/* Otherwise set 32-bit */
+
+	uthread_t uthread = (uthread_t)get_bsdthread_info(current_thread());
+
+	/*
+	 * Clear all user tracing flags.
+	 */
+	uthread->t_dtrace_ft = 0;
+
+	/*
+	 * Treat a child created by a call to vfork(2) as if it were its
+	 * parent. We know that there's only one thread of control in such a
+	 * process: this one.
+	 */
+	/*
+	 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
+	 * FIXME: How do we assert this?
+	 */
+	while (p->p_lflag & P_LINVFORK) p = p->p_pptr;	/* Search the end */
+
+	pid = p->p_pid;
+	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
+	lck_mtx_lock(pid_mtx);
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, sv->save_srr0)];	/* Get the bucket that corresponds to out PC */
+
+	/*
+	 * Lookup the tracepoint that the process just hit.
+	 */
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && (sv->save_srr0 == tp->ftt_pc) &&
+		    !tp->ftt_proc->ftpc_defunct)
+			break;
+	}
+
+	/*
+	 * If we couldn't find a matching tracepoint, either a tracepoint has
+	 * been inserted without using the pid<pid> ioctl interface (see
+	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
+	 */
+	if (tp == NULL) {
+		lck_mtx_unlock(pid_mtx);
+		return (-1);
+	}
+
+	if (tp->ftt_ids != NULL) {
+		fasttrap_id_t *id;
+		
+		for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
+			fasttrap_probe_t *probe = id->fti_probe;
+			
+			if (id->fti_ptype == DTFTP_ENTRY) {
+				/*
+				 * We note that this was an entry
+				 * probe to help ustack() find the
+				 * first caller.
+				 */
+				cookie = dtrace_interrupt_disable();
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_USTACK_FP | CPU_DTRACE_ENTRY);
+				dtrace_probe(probe->ftp_id, sv->save_r3, sv->save_r4,	/* Call the main probe routine with the first 5 args */
+					sv->save_r5, sv->save_r6, sv->save_r7);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_USTACK_FP | CPU_DTRACE_ENTRY);
+				dtrace_interrupt_enable(cookie);
+				
+			} else if (id->fti_ptype == DTFTP_IS_ENABLED) {
+				/*
+				 * Note that in this case, we don't
+				 * call dtrace_probe() since it's only
+				 * an artificial probe meant to change
+				 * the flow of control so that it
+				 * encounters the true probe.
+				 */
+				is_enabled = 1;
+				
+			} else if (probe->ftp_argmap == NULL) {
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_USTACK_FP);
+				dtrace_probe(probe->ftp_id, sv->save_r3, sv->save_r4,	/* Call the main probe routine with the first 5 args */
+					     sv->save_r5, sv->save_r6, sv->save_r7);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_USTACK_FP);
+					     
+			} else {
+				uint64_t t[5];
+				
+				fasttrap_usdt_args(probe, sv, 5, t);	/* Grab 5 arguments */
+				
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_USTACK_FP);
+				dtrace_probe(probe->ftp_id, t[0], t[1],
+					     t[2], t[3], t[4]);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_USTACK_FP);
+			}
+
+			/* APPLE NOTE: Oneshot probes get one and only one chance... */
+			if (probe->ftp_prov->ftp_provider_type == DTFTP_PROVIDER_ONESHOT) {
+				fasttrap_tracepoint_remove(p, tp);
+			}
+		}
+	}
+
+	/*
+	 * We're about to do a bunch of work so we cache a local copy of
+	 * the tracepoint to emulate the instruction, and then find the
+	 * tracepoint again later if we need to light up any return probes.
+	 */
+	tp_local = *tp;
+	lck_mtx_unlock(pid_mtx);
+	tp = &tp_local;
+
+	/*
+	 * If there's an is-enabled probe connected to this tracepoint it
+	 * means that there was a 'xor r3,r3,r3'
+	 * instruction that was placed there by DTrace when the binary was
+	 * linked. As this probe is, in fact, enabled, we need to stuff 1
+	 * into R3. Accordingly, we can bypass all the instruction
+	 * emulation logic since we know the inevitable result. It's possible
+	 * that a user could construct a scenario where the 'is-enabled'
+	 * probe was on some other instruction, but that would be a rather
+	 * exotic way to shoot oneself in the foot.
+	 */
+	if (is_enabled) {
+		sv->save_r3 = 1;				/* Set condition to true */
+		new_pc = (sv->save_srr0 + 4) & addrmask;		/* Just fall through to the next instruction */
+		goto done;
+	}
+
+	/*
+	 * We emulate certain types of instructions to ensure correctness
+	 * (in the case of position dependent instructions) or optimize
+	 * common cases. The rest we execute in the kernel, but with
+	 * most of the user's context active.
+	 */
+	switch (tp->ftt_type) {
+	
+		case ftmtNOP:					/* NOP  */
+			new_pc = (sv->save_srr0 + 4) & addrmask;	/* Just fall through to the next instruction */
+			break;
+
+		case ftmtB:						/* Plain unconditional branch */
+			new_pc = (user_addr_t)((int64_t)tp->ftt_trgt);	/* Assume target is absolute address for the moment */
+			if(!(tp->ftt_flgs & ftmtAbs)) new_pc = (new_pc + sv->save_srr0) & addrmask;	/* We don't have absolute address, use as offset from instruction address */
+
+			if(tp->ftt_flgs & ftmtLink) sv->save_lr = (sv->save_srr0 + 4) & addrmask;	/* Set the LR to the next instruction if needed */
+			break;
+		
+		case ftmtBC:					/* Conditional PC relative or absolute branch */
+			new_pc = (user_addr_t)((int64_t)tp->ftt_trgt);	/* Assume target is absolute address for the moment */
+			if(!(tp->ftt_flgs & ftmtAbs)) new_pc = new_pc + sv->save_srr0;	/* We don't have absolute address, use as offset from instruction address */
+
+			if(tp->ftt_flgs & ftmtLink) sv->save_lr = (sv->save_srr0 + 4) & addrmask;	/* Set the LR to the next instruction if needed */
+			if(!branchtaken(tp->ftt_bo, tp->ftt_bi, sv)) new_pc = (sv->save_srr0 + 4) & addrmask;	/* If branch was not taken, set PC to next address */
+			break;
+		
+		case ftmtBLR:					/* Conditional branch to LR */
+			new_pc = sv->save_lr;		/* Branch target comes from the LR */
+
+			if(tp->ftt_flgs & ftmtLink) sv->save_lr = (sv->save_srr0 + 4) & addrmask;	/* Set the LR to the next instruction if needed */			
+			if(!branchtaken(tp->ftt_bo, tp->ftt_bi, sv)) new_pc = (sv->save_srr0 + 4) & addrmask;	/* If branch was not taken, set PC to next address */
+			break;
+		
+		case ftmtBCTR:					/* Conditional branch to CTR */
+			new_pc = sv->save_ctr;		/* Branch target comes from the CTR */
+
+			if(tp->ftt_flgs & ftmtLink) sv->save_lr = (sv->save_srr0 + 4) & addrmask;	/* Set the LR to the next instruction if needed */			
+			if(!branchtaken(tp->ftt_bo, tp->ftt_bi, sv)) new_pc = (sv->save_srr0 + 4) & addrmask;	/* If branch was not taken, set PC to next address */
+			break;
+		
+		case ftmtCommon:				/* Common, non-in-kernel emulated instruction */
+			sv->save_instr[0] = 1;		/* We only have one instruction to inject */
+			sv->save_instr[1] = tp->ftt_instr;	/* Set the instruction */
+			sv->save_hdr.save_flags = sv->save_hdr.save_flags | SAVinject;	/* Tell low-level exception return to inject the instruction */
+			uthread->t_dtrace_step = 1;	/* Let it be known that a trace return is imminent */
+			return 0;					/* Go and don't dome back until you are done... */
+			
+		default:
+			panic("fasttrap_pid_probe: invalid ftt_type = %08X\n", tp->ftt_type);	/* Huh, wha happened? */
+			break;
+	}
+		
+
+done:
+	
+	/*
+	 * If there were no return probes when we first found the tracepoint,
+	 * we should feel no obligation to honor any return probes that were
+	 * subsequently enabled -- they'll just have to wait until the next
+	 * time around.
+	 */
+	sv->save_srr0 = new_pc;				/* Set the new PC */
+	if (tp->ftt_retids != NULL) fasttrap_return_common(sv, pc, pid, new_pc);
+
+	return (0);
+}
+
+
+int
+fasttrap_return_probe(ppc_saved_state_t *sv)
+{
+
+	user_addr_t pc, npc;
+	
+	proc_t *p = current_proc();
+
+
+	/*
+	 * Treat a child created by a call to vfork(2) as if it were its
+	 * parent. We know that there's only one thread of control in such a
+	 * process: this one.
+	 */
+	/*
+	 * APPLE NOTE: Terry says: "You need to hold the process locks (currently: kernel funnel) for this traversal"
+	 * How do we assert this?
+	 */
+	while (p->p_lflag & P_LINVFORK) {
+		p = p->p_pptr;
+	}
+
+	pc = sv->save_srr0;		/* Get the PC of the probed instruction */
+	npc = pc + 4;			/* Get next PC */	
+	if(!(sv->save_srr1 & 0x8000000000000000ULL)) npc &= 0x00000000FFFFFFFF;	/* Wrap new PC if running 32-bit */
+	fasttrap_return_common(sv, pc, p->p_pid, npc);
+
+	return (0);
+}
+
+uint64_t
+fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+#pragma unused(arg, id, parg, aframes)
+	return (fasttrap_anarg((ppc_saved_state_t *)find_user_regs(current_thread()), 1, argno));
+}
+
+uint64_t
+fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+#pragma unused(arg, id, parg, aframes)
+	return (fasttrap_anarg((ppc_saved_state_t *)find_user_regs(current_thread()), 0, argno));
+}
+
+
+static int32_t branchtaken(int32_t bo, int32_t bi, ppc_saved_state_t *sv) {
+	int32_t bcond, czero, crmatch;
+	uint64_t ctr;
+	
+	if((bo & 0x14) == 0x14) return 1;	/* If this is a branch always, exit with true... */
+	
+	czero = 0;							/* Assume that we have not just decremented the CTR to 0 */
+	
+	if(!(bo & 4)) {						/* Skip the next bit if we do NOT muck with the CTR */
+		ctr = sv->save_ctr = sv->save_ctr - 1;	/* Decrement the CTR */
+		if(!(sv->save_srr1 & 0x8000000000000000ULL)) ctr &= 0x00000000FFFFFFFF;	/* Only look at the bottom 32 bits if 32-bit mode */
+		czero = (ctr == 0);				/* Remember if we just hit zero */
+	}
+	
+	bcond = (bo >> 3);					/* If 1, branch if CR flag is 1.  If 0, branch if 0 */
+	crmatch = bo >> 4;					/* If bo[0] is set, do not check CR flag */
+	crmatch = crmatch | (((sv->save_cr >> (31 - bi)) ^ bcond) ^ 1);	/* Low bit is now set if CR flag matches or CR is not checked. Other bits are trash. */
+
+//	dbgTrace(0x77777777, bo, bi, sv->save_cr, ((czero | crmatch) & 1));	/* (TRACE/DEBUG) */
+
+	return ((czero | crmatch) & 1);		/* Return 1 if branch taken, 0 if not... */	
+}
+
+static int32_t dtrace_decode_ppc(uint32_t inst) {
+
+	int32_t curdcd, lastmask, newmask, spr, bit, bito, word;
+	uint16_t xop = 0;
+	dcdtab *dcd;
+	
+	curdcd = inst >> 26;				/* Isolate major op code to start decode */
+	lastmask = 99;						/* Always force a new xop at the start */
+	
+	while(1) {							/* Loop until we find instruction or fail */
+		dcd = &insts[curdcd];			/* Point to the current decode table entry */
+		if(dcd->dcdFlgs & dcdJump) {	/* Should we jump to a new spot in the decode table? */
+			curdcd = dcd->dcdMatch;		/* Jump */
+			continue;
+		}
+		
+		newmask = dcd->dcdFlgs & dcdMask;	/* Isolate the mask index */
+		if(lastmask != newmask) {		/* Are we changing masks? */
+			if(!newmask) break;			/* If the mask is 0, we match everything and succeed... (note: lastmask can never be 0) */
+			xop = inst & masktab[newmask];	/* Clear all extra bits to make match */
+			lastmask = newmask;			/* Remember */
+		}
+		
+		if(xop == dcd->dcdMatch) break;	/* We found our guy! */
+		
+		if(!(dcd->dcdFlgs & dcdStep)) {	/* No stepping, we failed */
+			dcd = &dcdfail;				/* Point to a failure entry */
+			break;						/* Leave... */
+		}
+		
+		curdcd = curdcd + 1;			/* Step to the next decode entry */
+	}
+
+	if(dcd->dcdType != diSPR) return (int32_t)(dcd->dcdType);	/* Return what we found */
+	
+	spr = (inst >> (31 - 20)) & 0x3FF;	/* Get the source */
+	spr = ((spr << 5) & 0x3E0) | ((spr >> 5) & 0x1F);	/* Flip to right order */
+	
+	word = spr >> 5;					/* Get word index into table */
+	bito = spr & 0x1F;					/* Get bit offset into entry */
+	bit = 0x80000000 >> bito;			/* Position bit for a test */
+	
+	if(!(sprtbl[word] & bit)) return (diINV);	/* Bogus SPR so whole instruction is invalid... */
+	
+	if(spr & 0x10) return (diPRV);		/* This is a priviliged SPR so instruction is priviliged... */
+	return (diCMN);						/* Just a common SPR so instruction is the same... */
+}
diff --git a/bsd/dev/ppc/fbt_ppc.c b/bsd/dev/ppc/fbt_ppc.c
new file mode 100644
index 000000000..e2ffc4216
--- /dev/null
+++ b/bsd/dev/ppc/fbt_ppc.c
@@ -0,0 +1,680 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)fbt.c	1.15	05/09/19 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/cpu_data.h>
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+
+#include <mach-o/loader.h> 
+#include <mach-o/nlist.h>
+
+extern struct mach_header _mh_execute_header; /* the kernel's mach header */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <miscfs/devfs/devfs.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <sys/fbt.h>
+
+#include <sys/dtrace_glue.h>
+#include <machine/cpu_capabilities.h>
+
+#define DTRACE_INVOP_NOP_SKIP 4
+
+#define DTRACE_INVOP_MFLR_R0 11
+#define DTRACE_INVOP_MFLR_R0_SKIP 4
+
+#define FBT_MFLR_R0		0x7c0802a6
+
+#define FBT_MTLR_R0		0x7c0803a6
+#define FBT_BLR			0x4e800020
+#define FBT_BCTR		0x4e800420
+
+#define FBT_LI_MASK 0x03fffffc
+#define FBT_JUMP	0x48000000
+#define IS_JUMP(instr) (((instr) & ~FBT_LI_MASK) == FBT_JUMP) /* Relative, No LR update -- AA == 0b, LK == 0b */
+#define FBT_LI_EXTD64(instr) \
+	(((instr) & 0x02000000) ? \
+	 	(((uint64_t)((instr) & FBT_LI_MASK)) | 0xfffffffffc000000ULL) : \
+	 	 ((uint64_t)((instr) & FBT_LI_MASK)))
+
+#define FBT_PATCHVAL	0x7c810808
+#define FBT_AFRAMES_ENTRY		6
+#define FBT_AFRAMES_RETURN		6
+
+#define	FBT_ENTRY	"entry"
+#define	FBT_RETURN	"return"
+#define	FBT_ADDR2NDX(addr)	((((uintptr_t)(addr)) >> 4) & fbt_probetab_mask)
+
+extern dtrace_provider_id_t	fbt_id;
+extern fbt_probe_t		**fbt_probetab;
+extern int			fbt_probetab_mask;
+
+/*
+ * Critical routines that must not be probed. PR_5221096, PR_5379018.
+ */
+
+static const char * critical_blacklist[] =
+{
+	"bcopy_phys",
+	"bcopy_physvir_32",
+	"cpu_control",
+	"cpu_exit_wait",
+	"cpu_info",
+	"cpu_info_count",
+	"cpu_init",
+	"cpu_machine_init",
+	"cpu_per_proc_alloc",
+	"cpu_per_proc_free",
+	"cpu_signal_handler",
+	"cpu_sleep",
+	"cpu_start",
+	"cpu_subtype",
+	"cpu_threadtype",
+	"cpu_to_processor",
+	"cpu_type",
+	"mapSkipListVerifyC",
+	"ml_nofault_copy",
+	"register_cpu_setup_func",
+	"unregister_cpu_setup_func"
+};
+#define CRITICAL_BLACKLIST_COUNT (sizeof(critical_blacklist)/sizeof(critical_blacklist[0]))
+
+/*
+ * The transitive closure of entry points that can be reached from probe context.
+ * (Apart from routines whose names begin with dtrace_ or dtxnu_.)
+ */
+static const char * probe_ctx_closure[] =
+{
+	"Debugger",
+	"MapUserMemoryWindow",
+	"OSCompareAndSwap",
+	"absolutetime_to_microtime",
+	"bcopy",
+	"clock_get_calendar_nanotime_nowait",
+	"copyin",
+	"copyinstr",
+	"copyout",
+	"copyoutstr",
+	"cpu_number",
+	"current_proc",
+	"current_processor",
+	"current_task",
+	"current_thread",
+	"debug_enter",
+	"find_user_regs",
+	"getPerProc",
+	"get_bsdtask_info",
+	"get_bsdthread_info",
+	"get_threadtask",
+	"hw_atomic_and",
+	"hw_compare_and_store",
+	"hw_find_map",
+	"kauth_cred_get",
+	"kauth_getgid",
+	"kauth_getuid",
+	"mach_absolute_time",
+	"mapping_drop_busy",
+	"mapping_find",
+	"mapping_phys_lookup",
+	"max_valid_stack_address",
+	"ml_at_interrupt_context",
+	"ml_phys_write_byte_64",
+	"ml_phys_write_half_64",
+	"ml_phys_write_word_64",
+	"ml_set_interrupts_enabled",
+	"panic",
+	"pmap_find_phys",
+	"prf",
+	"proc_is64bit",
+	"proc_selfname",
+	"proc_selfpid",
+	"psignal_lock",
+	"splhigh",
+	"splx",
+	"strlcpy",
+	"timer_grab"
+};
+#define PROBE_CTX_CLOSURE_COUNT (sizeof(probe_ctx_closure)/sizeof(probe_ctx_closure[0]))
+
+static int _cmp(const void *a, const void *b)
+{
+	return strcmp((const char *)a, *(const char **)b);
+}
+
+static const void * bsearch(
+	register const void *key,
+	const void *base0,
+	size_t nmemb,
+	register size_t size,
+	register int (*compar)(const void *, const void *)) {
+
+	register const char *base = base0;
+	register size_t lim;
+	register int cmp;
+	register const void *p;
+
+	for (lim = nmemb; lim != 0; lim >>= 1) {
+		p = base + (lim >> 1) * size;
+		cmp = (*compar)(key, p);
+		if (cmp == 0)
+			return p;
+		if (cmp > 0) {	/* key > p: move right */
+			base = (const char *)p + size;
+			lim--;
+		}		/* else move left */
+	}
+	return (NULL);
+}
+
+int
+fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
+{
+	fbt_probe_t *fbt = fbt_probetab[FBT_ADDR2NDX(addr)];
+	uint64_t mask = (_cpu_capabilities & k64Bit) ? 0xffffffffffffffffULL : 0x00000000ffffffffULL;
+
+	for (; fbt != NULL; fbt = fbt->fbtp_hashnext) {
+		if ((uintptr_t)fbt->fbtp_patchpoint == addr) {
+			
+			if (fbt->fbtp_roffset == 0) {
+				ppc_saved_state_t *regs = (ppc_saved_state_t *)stack;
+
+				CPU->cpu_dtrace_caller = addr;
+				
+				dtrace_probe(fbt->fbtp_id, regs->save_r3 & mask, regs->save_r4 & mask,
+					regs->save_r5 & mask, regs->save_r6 & mask, regs->save_r7 & mask);
+					
+				CPU->cpu_dtrace_caller = NULL;
+			} else {
+			
+				dtrace_probe(fbt->fbtp_id, fbt->fbtp_roffset, rval, 0, 0, 0);
+
+				if (fbt->fbtp_rval == DTRACE_INVOP_TAILJUMP) {
+					ppc_saved_state_t *regs = (ppc_saved_state_t *)stack;
+
+					regs->save_srr0 = (uint64_t)fbt->fbtp_patchpoint + FBT_LI_EXTD64(fbt->fbtp_savedval);
+					regs->save_srr0 &= mask;
+				}
+				
+				CPU->cpu_dtrace_caller = NULL;
+			}
+
+			return (fbt->fbtp_rval);
+		}
+	}
+
+	return (0);
+}
+
+#include <ppc/proc_reg.h> /* For USER_MODE */
+#define IS_USER_TRAP(regs) USER_MODE((regs)->save_srr1)
+#define T_VECTOR_SIZE   4               /* function pointer size */
+#define T_PROGRAM       (0x07 * T_VECTOR_SIZE)
+#define FBT_EXCEPTION_CODE T_PROGRAM
+
+kern_return_t
+fbt_perfCallback(
+                int         trapno,
+                ppc_saved_state_t *regs,
+                int         unused1,
+                int         unused2)
+{
+#pragma unused (unused1)
+#pragma unused (unused2)
+	kern_return_t retval = KERN_FAILURE;
+	
+	if (!IS_USER_TRAP(regs) && FBT_EXCEPTION_CODE == trapno) {
+		boolean_t oldlevel;
+		
+		oldlevel = ml_set_interrupts_enabled(FALSE);
+						
+		switch (dtrace_invop( regs->save_srr0, (uintptr_t *)regs, regs->save_r3 )) {
+		case DTRACE_INVOP_NOP:
+			regs->save_srr0 += DTRACE_INVOP_NOP_SKIP;	/* Skip over the bytes of the patched NOP */
+			retval = KERN_SUCCESS;
+			break;
+
+		case DTRACE_INVOP_MFLR_R0:
+			regs->save_r0 = regs->save_lr;					/* Emulate patched mflr r0 */
+			regs->save_srr0 += DTRACE_INVOP_MFLR_R0_SKIP;	/* Skip over the bytes of the patched mflr r0 */
+			retval = KERN_SUCCESS;
+			break;
+
+		case DTRACE_INVOP_RET:
+			regs->save_srr0 = regs->save_lr;				/* Emulate patched blr by resuming execution at the LR */
+			retval = KERN_SUCCESS;
+			break;
+			
+		case DTRACE_INVOP_BCTR:
+			regs->save_srr0 = regs->save_ctr;				/* Emulate patched bctr by resuming execution at the CTR */
+			retval = KERN_SUCCESS;
+			break;
+			
+		case DTRACE_INVOP_TAILJUMP:
+			retval = KERN_SUCCESS;
+			break;
+			
+		default:
+			retval = KERN_FAILURE;
+			break;
+		}
+		ml_set_interrupts_enabled(oldlevel);
+	}
+	
+	return retval;
+}
+
+kern_return_t
+fbt_perfIntCallback(
+                int         trapno,
+                ppc_saved_state_t *regs,
+                int         unused1,
+                int         unused2)
+{
+	kern_return_t retval = KERN_FAILURE;
+	
+	if (KERN_SUCCESS == (retval = fbt_perfCallback(trapno, regs, unused1, unused2)))
+		enable_preemption();
+	
+	return retval;
+}
+
+/*ARGSUSED*/
+static void
+__fbt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(arg)
+	struct mach_header			*mh;
+	struct load_command         *cmd;
+    struct segment_command      *orig_ts = NULL, *orig_le = NULL;
+    struct symtab_command       *orig_st = NULL;
+	struct nlist                *sym = NULL;
+	char						*strings;
+	uintptr_t					instrLow, instrHigh;
+	char						*modname;
+	unsigned int i;
+
+    int gIgnoreFBTBlacklist = 0;
+	PE_parse_boot_arg("IgnoreFBTBlacklist", &gIgnoreFBTBlacklist);
+
+	mh = (struct mach_header *)(ctl->address);
+	modname = ctl->mod_modname;
+	
+	if (0 == ctl->address || 0 == ctl->size) /* Has the linker been jettisoned? */
+		return;
+		
+	/*
+	 * Employees of dtrace and their families are ineligible.  Void
+	 * where prohibited.
+	 */
+
+	if (strcmp(modname, "com.apple.driver.dtrace") == 0)
+		return;
+
+	if (strstr(modname, "CHUD") != NULL)
+		return;
+		
+	if (mh->magic != MH_MAGIC)
+		return;
+		
+	cmd = (struct load_command *) &mh[1];
+	for (i = 0; i < mh->ncmds; i++) {
+        if (cmd->cmd == LC_SEGMENT) {
+            struct segment_command *orig_sg = (struct segment_command *) cmd;
+ 
+            if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
+                orig_ts = orig_sg;
+            else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
+                orig_le = orig_sg;
+            else if (strcmp("", orig_sg->segname) == 0)
+                orig_ts = orig_sg; /* kexts have a single unnamed segment */
+        }
+        else if (cmd->cmd == LC_SYMTAB)
+            orig_st = (struct symtab_command *) cmd;
+
+        cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
+    }
+
+	if ((orig_ts == NULL) || (orig_st == NULL) || (orig_le == NULL))
+		return;
+
+    sym = (struct nlist *)orig_le->vmaddr;
+    strings = ((char *)sym) + orig_st->nsyms * sizeof(struct nlist);
+	
+	/* Find extent of the TEXT section */
+	instrLow = (uintptr_t)orig_ts->vmaddr;
+	instrHigh = (uintptr_t)(orig_ts->vmaddr + orig_ts->vmsize);
+
+	for (i = 0; i < orig_st->nsyms; i++) {
+		fbt_probe_t *fbt, *retfbt;
+		machine_inst_t *instr, *limit, theInstr;
+        uint8_t n_type = sym[i].n_type & (N_TYPE | N_EXT);
+		char *name = strings + sym[i].n_un.n_strx;
+		int j;
+
+		/* Check that the symbol is a global and that it has a name. */
+        if (((N_SECT | N_EXT) != n_type && (N_ABS | N_EXT) != n_type))
+            continue;
+			
+		if (0 == sym[i].n_un.n_strx) /* iff a null, "", name. */
+			continue;
+ 
+		/* Lop off omnipresent leading underscore. */			
+		if (*name == '_')
+			name += 1;
+		
+		if (strstr(name, "dtrace_") == name &&
+		    strstr(name, "dtrace_safe_") != name) {
+			/*
+			 * Anything beginning with "dtrace_" may be called
+			 * from probe context unless it explitly indicates
+			 * that it won't be called from probe context by
+			 * using the prefix "dtrace_safe_".
+			 */
+			continue;
+		}
+		
+        if (strstr(name, "dsmos_") == name) 
+            continue; /* Don't Steal Mac OS X! */
+
+        if (strstr(name, "dtxnu_") == name ||
+			strstr(name, "_dtrace") == name) 
+			continue; /* Shims in dtrace.c */
+		
+		if (strstr(name, "chud") == name)
+			continue; /* Professional courtesy. */
+
+        if (strstr(name, "hibernate_") == name)
+            continue; /* Let sleeping dogs lie. */
+        
+        if (0 == strcmp(name, "ZN9IOService14newTemperatureElPS_") || /* IOService::newTemperature */
+            0 == strcmp(name, "ZN9IOService26temperatureCriticalForZoneEPS_")) /* IOService::temperatureCriticalForZone */
+            continue; /* Per the fire code */
+
+		/*
+		 * Place no probes (illegal instructions) in the exception handling path!
+		 */
+		if (0 == strcmp(name, "L_handler700") ||
+			0 == strcmp(name, "save_get_phys_64") ||
+			0 == strcmp(name, "save_get_phys_32") ||
+			0 == strcmp(name, "EmulExit") ||
+			0 == strcmp(name, "Emulate") ||
+			0 == strcmp(name, "Emulate64") ||
+			0 == strcmp(name, "switchSegs") ||
+			0 == strcmp(name, "save_ret_phys"))
+			continue;
+
+		if (0 == strcmp(name, "thandler") ||
+			0 == strcmp(name, "versave") ||
+			0 == strcmp(name, "timer_event") ||
+			0 == strcmp(name, "hw_atomic_or") ||
+			0 == strcmp(name, "trap"))
+			continue;
+
+		if (0 == strcmp(name, "fbt_perfCallback") ||
+			0 == strcmp(name, "fbt_perfIntCallback") ||
+			0 == strcmp(name, "ml_set_interrupts_enabled") ||
+			0 == strcmp(name, "dtrace_invop") ||
+			0 == strcmp(name, "fbt_invop") ||
+			0 == strcmp(name, "sdt_invop") ||
+			0 == strcmp(name, "max_valid_stack_address"))
+			continue;
+
+		/*
+		 * Probes encountered while we're on the interrupt stack are routed along
+		 * the interrupt handling path. No probes allowed there either!
+		 */
+		if (0 == strcmp(name, "ihandler") ||
+			0 == strcmp(name, "interrupt") ||
+			0 == strcmp(name, "disable_preemption"))
+			continue;
+
+		/*
+		 * Avoid weird stack voodoo in and under machine_stack_handoff et al
+		 */
+        if (strstr(name, "machine_stack") == name ||
+            0 == strcmp(name, "getPerProc") ||     /* Called in machine_stack_handoff with weird stack state */
+            0 == strcmp(name, "fpu_save") ||     /* Called in machine_stack_handoff with weird stack state */
+            0 == strcmp(name, "vec_save") ||     /* Called in machine_stack_handoff with weird stack state */
+            0 == strcmp(name, "pmap_switch"))     /* Called in machine_stack_handoff with weird stack state */
+				continue;
+
+		/*
+		 * Avoid machine_ routines. PR_5346750.
+		 */
+		if (strstr(name, "machine_") == name)
+			continue;
+
+		/*
+		 * Avoid low level pmap and virtual machine monitor PowerPC routines. See PR_5379018.
+		 */
+
+		if (strstr(name, "hw_") == name ||
+			strstr(name, "mapping_") == name ||
+			strstr(name, "commpage_") == name ||
+			strstr(name, "pmap_") == name ||
+			strstr(name, "vmm_") == name)
+				continue;
+		/*
+		 * Place no probes on critical routines. PR_5221096
+		 */
+		if (!gIgnoreFBTBlacklist && 
+			bsearch( name, critical_blacklist, CRITICAL_BLACKLIST_COUNT, sizeof(name), _cmp ) != NULL)
+				continue;
+
+		/*
+		 * Place no probes that could be hit in probe context.
+		 */
+		if (!gIgnoreFBTBlacklist && 
+			bsearch( name, probe_ctx_closure, PROBE_CTX_CLOSURE_COUNT, sizeof(name), _cmp ) != NULL)
+				continue;
+
+		/*
+		 * Place no probes that could be hit on the way to the debugger.
+		 */
+		if (strstr(name, "kdp_") == name ||
+			strstr(name, "kdb_") == name ||
+			strstr(name, "kdbg_") == name ||
+			strstr(name, "kdebug_") == name ||
+			0 == strcmp(name, "kernel_debug") ||
+			0 == strcmp(name, "Debugger") ||
+			0 == strcmp(name, "Call_DebuggerC") ||
+			0 == strcmp(name, "lock_debugger") ||
+			0 == strcmp(name, "unlock_debugger") ||
+			0 == strcmp(name, "SysChoked")) 
+			continue;
+		
+		/*
+		 * Place no probes that could be hit on the way to a panic.
+		 */
+		if (NULL != strstr(name, "panic_") ||
+			0 == strcmp(name, "panic") ||
+			0 == strcmp(name, "handleMck") ||
+			0 == strcmp(name, "unresolved_kernel_trap"))
+			continue;
+		
+		if (dtrace_probe_lookup(fbt_id, modname, name, NULL) != 0)
+			continue;
+			
+		/*
+		 * Scan forward for mflr r0.
+		 */
+		for (j = 0, instr = (machine_inst_t *)sym[i].n_value, theInstr = 0;
+			 (j < 4) && ((uintptr_t)instr >= instrLow) && (instrHigh > (uintptr_t)instr);
+			 j++, instr++) 
+		{
+			theInstr = *instr;
+			if (theInstr == FBT_MFLR_R0) /* Place the entry probe here. */
+				break;
+			if (theInstr == FBT_MTLR_R0) /* We've gone too far, bail. */
+				break;
+			if (theInstr == FBT_BLR) /* We've gone too far, bail. */
+				break;
+		}
+			
+		if (theInstr != FBT_MFLR_R0)
+			continue;
+			
+		limit = (machine_inst_t *)instrHigh;
+
+		fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
+		strlcpy( (char *)&(fbt->fbtp_name), name, MAX_FBTP_NAME_CHARS );
+		fbt->fbtp_id = dtrace_probe_create(fbt_id, modname, name, FBT_ENTRY, FBT_AFRAMES_ENTRY, fbt);
+		fbt->fbtp_patchpoint = instr;
+		fbt->fbtp_ctl = ctl;
+		fbt->fbtp_loadcnt = ctl->mod_loadcnt;
+		fbt->fbtp_rval = DTRACE_INVOP_MFLR_R0;
+		fbt->fbtp_savedval = theInstr;
+		fbt->fbtp_patchval = FBT_PATCHVAL;
+
+		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
+		fbt->fbtp_symndx = i;
+		fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
+
+		instr++; /* Move on down the line */
+		retfbt = NULL;
+again:
+		if (instr >= limit)
+			continue;
+
+		/*
+		 * We (desperately) want to avoid erroneously instrumenting a
+		 * jump table. To determine if we're looking at a true instruction
+		 * or an inline jump table that happens to contain the same
+		 * byte sequences, we resort to some heuristic sleeze:  we
+		 * treat this instruction as being contained within a pointer,
+		 * and see if that pointer points to within the body of the
+		 * function.  If it does, we refuse to instrument it.
+		 */
+		{
+			machine_inst_t *ptr = *(machine_inst_t **)instr;
+
+			if (ptr >= (machine_inst_t *)sym[i].n_value && ptr < limit) {
+				instr++;
+				goto again;
+			}
+		}
+
+		/*
+		 * OK, it's an instruction.
+		 */
+		theInstr = *instr;
+
+		/* Walked onto the start of the next routine? If so, bail out from this function. */
+		if (theInstr == FBT_MFLR_R0)
+			continue;
+
+		if (theInstr != FBT_MTLR_R0) {
+			instr++;
+			goto again;
+		}
+
+		/*
+		 * Found mtlr r0;
+		 * Scan forward for a blr, bctr, or a jump (relative, no LR change).
+		 */
+		instr++;
+		for (j = 0; (j < 12) && (instr < limit); j++, instr++) {
+			theInstr = *instr;
+			if (theInstr == FBT_BLR || theInstr == FBT_BCTR || IS_JUMP(theInstr) || 
+				theInstr == FBT_MFLR_R0 || theInstr == FBT_MTLR_R0)
+				break;
+		}
+
+		if (!(theInstr == FBT_BLR || theInstr == FBT_BCTR || IS_JUMP(theInstr)))
+			goto again;
+
+		/*
+		 * We have a winner: "mtlr r0; ... ; {blr, bctr, j}" !
+		 */
+		fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
+		strlcpy( (char *)&(fbt->fbtp_name), name, MAX_FBTP_NAME_CHARS );
+
+		if (retfbt == NULL) {
+			fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
+			    name, FBT_RETURN, FBT_AFRAMES_RETURN, fbt);
+		} else {
+			retfbt->fbtp_next = fbt;
+			fbt->fbtp_id = retfbt->fbtp_id;
+		}
+
+		retfbt = fbt;
+		fbt->fbtp_patchpoint = instr;
+		fbt->fbtp_ctl = ctl;
+		fbt->fbtp_loadcnt = ctl->mod_loadcnt;
+
+		if (theInstr == FBT_BLR)
+			fbt->fbtp_rval = DTRACE_INVOP_RET;
+		else if (theInstr == FBT_BCTR)
+			fbt->fbtp_rval = DTRACE_INVOP_BCTR;
+		else
+			fbt->fbtp_rval = DTRACE_INVOP_TAILJUMP;
+
+		fbt->fbtp_roffset =
+		    (uintptr_t)((uint8_t *)instr - (uint8_t *)sym[i].n_value);
+
+		fbt->fbtp_savedval = *instr;
+		fbt->fbtp_patchval = FBT_PATCHVAL;
+		fbt->fbtp_hashnext = fbt_probetab[FBT_ADDR2NDX(instr)];
+		fbt->fbtp_symndx = i;
+		fbt_probetab[FBT_ADDR2NDX(instr)] = fbt;
+		instr++;
+		goto again;
+	}
+}
+
+extern struct modctl g_fbt_kernctl;
+#undef kmem_alloc /* from its binding to dt_kmem_alloc glue */
+#undef kmem_free /* from its binding to dt_kmem_free glue */
+#include <vm/vm_kern.h>
+
+/*ARGSUSED*/
+void
+fbt_provide_module(void *arg, struct modctl *ctl)
+{
+#pragma unused(ctl)
+	__fbt_provide_module(arg, &g_fbt_kernctl);
+
+    kmem_free(kernel_map, (vm_offset_t)g_fbt_kernctl.address, round_page_32(g_fbt_kernctl.size));
+	g_fbt_kernctl.address = 0;
+	g_fbt_kernctl.size = 0;
+}
diff --git a/bsd/dev/ppc/ffs.c b/bsd/dev/ppc/ffs.c
index fa02a4795..c3f06a74f 100644
--- a/bsd/dev/ppc/ffs.c
+++ b/bsd/dev/ppc/ffs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991 NeXT Computer, Inc.  All rights reserved.
  *
diff --git a/bsd/dev/ppc/ffs.s b/bsd/dev/ppc/ffs.s
index 75ec0dea4..290053a82 100644
--- a/bsd/dev/ppc/ffs.s
+++ b/bsd/dev/ppc/ffs.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1992, 1997-1998 Apple Computer, Inc.  All rights reserved.
  *
diff --git a/bsd/dev/ppc/kern_machdep.c b/bsd/dev/ppc/kern_machdep.c
index bdf477f7f..1f45bd131 100644
--- a/bsd/dev/ppc/kern_machdep.c
+++ b/bsd/dev/ppc/kern_machdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (C) 1990, 1993  NeXT, Inc.
@@ -37,6 +43,9 @@
 #include	<kern/cpu_number.h>
 #include	<machine/exec.h>
 
+boolean_t kernacc(off_t, size_t );
+
+
 /*
  * Routine: grade_binary()
  *
@@ -220,6 +229,8 @@ grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype)
 	/* NOTREACHED */
 }
 
+extern vm_map_offset_t kvtophys64(vm_map_offset_t);
+
 boolean_t
 kernacc(
     off_t 	start,
@@ -233,7 +244,7 @@ kernacc(
 	end = start + len;
 	
 	while (base < end) {
-		if(kvtophys((vm_offset_t)base) == NULL)
+		if(kvtophys64((vm_map_offset_t)base) == (vm_map_offset_t)0)
 			return(FALSE);
 		base += page_size;
 	}   
@@ -244,16 +255,9 @@ kernacc(
 void
 md_prepare_for_shutdown(int paniced, int howto, char * command);
 
-extern void IOSystemShutdownNotification(void);
-
 void
 md_prepare_for_shutdown(__unused int paniced, __unused int howto,
 			__unused char * command)
 {
-
-    /*
-     * Temporary hack to notify the power management root domain
-     * that the system will shut down.
-     */
-    IOSystemShutdownNotification();
+	return;
 }
diff --git a/bsd/dev/ppc/km.c b/bsd/dev/ppc/km.c
index db5d95169..27abea786 100644
--- a/bsd/dev/ppc/km.c
+++ b/bsd/dev/ppc/km.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 	Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. 
  *
@@ -54,15 +60,6 @@ int disableConsoleOutput;
 
 static int initialized = 0;
 
-// Function prototypes
-extern d_open_t         kmopen;
-extern d_close_t        kmclose;
-extern d_read_t         kmread;
-extern d_write_t        kmwrite;
-extern d_ioctl_t        kmioctl;
-extern d_getc_t         kmgetc;
-extern d_putc_t         kmputc;
-
 extern void kminit(void);
 
 // used by or implemented in the osfmk project
@@ -306,7 +303,7 @@ kmoutput(struct tty *tp)
 		if (cc == 0)
 			break;
 		cc = min(cc, sizeof buf);
-		(void) q_to_b(&tp->t_outq, buf, cc);
+		(void) q_to_b(&tp->t_outq, (unsigned char *)buf, cc);
 		for (cp = buf; cp < &buf[cc]; cp++)
 		    kmputc(tp->t_dev, *cp & 0x7f);
 	}
diff --git a/bsd/dev/ppc/machdep.c b/bsd/dev/ppc/machdep.c
index 6e44b5eb9..8f912037b 100644
--- a/bsd/dev/ppc/machdep.c
+++ b/bsd/dev/ppc/machdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 Apple Computer, Inc.  All rights reserved.
@@ -39,51 +45,14 @@ int reboot_how;
 extern struct tty	cons;
 extern struct tty	*constty;		/* current console device */
 
-extern int getchar();
+extern int getchar(void);
+extern int cngetc(void);
+extern void cnputc(char);
 
 #define putchar cnputc
 
-void
-gets(buf)
-	char *buf;
-{
-	register char *lp;
-	register c;
-
-	lp = buf;
-	for (;;) {
-		c = getchar() & 0177;
-		switch(c) {
-		case '\n':
-		case '\r':
-			*lp++ = '\0';
-			return;
-		case '\b':
-			if (lp > buf) {
-				lp--;
-				putchar(' ');
-				putchar('\b');
-			}
-			continue;
-		case '#':
-		case '\177':
-			lp--;
-			if (lp < buf)
-				lp = buf;
-			continue;
-		case '@':
-		case 'u'&037:
-			lp = buf;
-			putchar('\n');		/* XXX calls 'cnputc' on mips */
-			continue;
-		default:
-			*lp++ = c;
-		}
-	}
-}
-
 int
-getchar()
+getchar(void)
 {
 	int c;
 
diff --git a/bsd/dev/ppc/mem.c b/bsd/dev/ppc/mem.c
index 54fe48e68..301ade89d 100644
--- a/bsd/dev/ppc/mem.c
+++ b/bsd/dev/ppc/mem.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1988 University of Utah.
@@ -85,28 +91,23 @@
 
 static caddr_t devzerobuf;
 
-extern pmap_t kernel_pmap;
 extern boolean_t kernacc(off_t, size_t );
 extern int setup_kmem;
 
-int mmread(dev_t dev, struct uio *uio);
+int mmread(dev_t dev, struct uio *uio, int flag);
 int mmrw(dev_t dev, struct uio *uio, enum uio_rw rw);
 int mmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p);
-int mmwrite(dev_t dev, struct uio *uio);
+int mmwrite(dev_t dev, struct uio *uio, int flag);
 
 int
-mmread(dev, uio)
-	dev_t dev;
-	struct uio *uio;
+mmread(dev_t dev, struct uio *uio, __unused int flag)
 {
 
 	return (mmrw(dev, uio, UIO_READ));
 }
 
 int
-mmwrite(dev, uio)
-	dev_t dev;
-	struct uio *uio;
+mmwrite(dev_t dev, struct uio *uio, __unused int flag)
 {
 
 	return (mmrw(dev, uio, UIO_WRITE));
@@ -149,7 +150,6 @@ mmrw(dev, uio, rw)
 	int error = 0;
 	vm_offset_t	where;
 
-
 	while (uio_resid(uio) > 0 && error == 0) {
 		if (uio_iov_len(uio) == 0) {
 			uio_next_iov(uio);
diff --git a/bsd/dev/ppc/memmove.c b/bsd/dev/ppc/memmove.c
index c9a091bb1..e102c248a 100644
--- a/bsd/dev/ppc/memmove.c
+++ b/bsd/dev/ppc/memmove.c
@@ -1,31 +1,37 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991,1993 NeXT Computer, Inc.  All rights reserved.
  * 
  */
+#include <sys/systm.h>
 
 
-
-void ovbcopy(const char *src, char *dst, unsigned int ulen)
+void ovbcopy(const void *src, void *dst, size_t ulen)
 {
 	bcopy(src, dst, ulen);
 }
diff --git a/bsd/dev/ppc/munge.s b/bsd/dev/ppc/munge.s
index fdf7c7060..2af7c6e1c 100644
--- a/bsd/dev/ppc/munge.s
+++ b/bsd/dev/ppc/munge.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -267,6 +273,35 @@ _munge_wwwlww:
         
         blr
 
+
+        .align  5
+        .globl  _munge_wwlwww
+_munge_wwlwww:
+        li      r0,0
+        lwz     r5,0*8+4(r3)	// Wwlwww
+        lwz     r6,1*8+4(r3)	// wWlwww
+        lwz     r7,2*8+4(r3)	// wwLwww (hi)
+        lwz     r8,3*8+4(r3)	// wwLwww (lo)
+        lwz     r9,4*8+4(r3)	// wwlWww
+        lwz     r10,5*8+4(r3)	// wwlwWw
+        lwz     r11,6*8+4(r3)	// wwlwwW
+        
+        stw     r0,0*8+0(r4)	// 0wlwww
+        stw     r5,0*8+4(r4)	// Wwlwww
+        stw     r0,1*8+0(r4)	// w0lwww
+        stw     r6,1*8+4(r4)	// wWlwww
+        stw     r7,2*8+0(r4)	// wwLwww (hi)
+        stw     r8,2*8+4(r4)	// wwLwww (lo)
+        stw     r0,3*8+0(r4)	// wwl0ww 
+        stw     r9,3*8+4(r4)	// wwlwww
+        stw     r0, 4*8+0(r4)	// wwlw0w
+        stw     r10,4*8+4(r4)	// wwlwWw
+        stw     r0, 5*8+0(r4)	// wwlww0
+        stw     r11,5*8+4(r4)	// wwlwwW
+        
+        blr
+
+
         .align  5
         .globl  _munge_wwwwl	// 4 'w's and an l
 _munge_wwwwl:
diff --git a/bsd/dev/ppc/nvram.c b/bsd/dev/ppc/nvram.c
index bf466872a..0114a93b9 100644
--- a/bsd/dev/ppc/nvram.c
+++ b/bsd/dev/ppc/nvram.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * BSD driver for Non-volatile RAM.
diff --git a/bsd/dev/ppc/ppc_init.c b/bsd/dev/ppc/ppc_init.c
index 293ac2a6a..489610109 100644
--- a/bsd/dev/ppc/ppc_init.c
+++ b/bsd/dev/ppc/ppc_init.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
diff --git a/bsd/dev/ppc/sdt_ppc.c b/bsd/dev/ppc/sdt_ppc.c
new file mode 100644
index 000000000..bcd10c967
--- /dev/null
+++ b/bsd/dev/ppc/sdt_ppc.c
@@ -0,0 +1,71 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/* #pragma ident	"@(#)sdt.c	1.6	06/03/24 SMI" */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#define MACH__POSIX_C_SOURCE_PRIVATE 1 /* pulls in suitable savearea from mach/ppc/thread_status.h */
+#include <kern/cpu_data.h>
+#include <kern/thread.h>
+#include <mach/thread_status.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+
+#include <sys/dtrace_glue.h>
+
+#include <sys/sdt_impl.h>
+#include <machine/cpu_capabilities.h>
+
+extern sdt_probe_t      **sdt_probetab;
+
+/*ARGSUSED*/
+int
+sdt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
+{
+    uint64_t mask = (_cpu_capabilities & k64Bit) ? 0xffffffffffffffffULL : 0x00000000ffffffffULL;
+
+#pragma unused(eax)
+	sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)];
+
+	for (; sdt != NULL; sdt = sdt->sdp_hashnext) {
+		if ((uintptr_t)sdt->sdp_patchpoint == addr) {
+			ppc_saved_state_t *regs = (ppc_saved_state_t *)stack;
+			
+            dtrace_probe(sdt->sdp_id, regs->save_r3 & mask, regs->save_r4 & mask,
+                regs->save_r5 & mask, regs->save_r6 & mask, regs->save_r7 & mask);
+				
+			return (DTRACE_INVOP_NOP);
+		}
+	}
+
+	return (0);
+}
+
diff --git a/bsd/dev/ppc/stubs.c b/bsd/dev/ppc/stubs.c
index 3f0df507d..55e2f0170 100644
--- a/bsd/dev/ppc/stubs.c
+++ b/bsd/dev/ppc/stubs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 by Apple Computer, Inc., all rights reserved
@@ -36,6 +42,8 @@
 #include <kern/task.h>
 #include <vm/vm_map.h>
 
+extern void	*get_bsduthreadarg(thread_t);
+extern int	*get_bsduthreadrval(thread_t);
 
 /* 
  * copy a null terminated string from one point to another in 
@@ -52,7 +60,8 @@ int
 copystr(const void *vfrom, void *vto, size_t maxlen, size_t *lencopied)
 {
     register unsigned l;
-	caddr_t from, to;
+	const char *from;
+	char *to;
 
 	from = vfrom;
 	to = vto;
diff --git a/bsd/dev/ppc/sysctl.c b/bsd/dev/ppc/sysctl.c
index 8693ac273..7bc509e16 100644
--- a/bsd/dev/ppc/sysctl.c
+++ b/bsd/dev/ppc/sysctl.c
@@ -1,31 +1,28 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#include <sys/param.h>
-#include <sys/kernel.h>
-#include <sys/sysctl.h>
-
-struct sysctl_oid *machdep_sysctl_list[] =
-{
-    (struct sysctl_oid *) 0
-};
-
diff --git a/bsd/dev/ppc/systemcalls.c b/bsd/dev/ppc/systemcalls.c
index 4d0a3d397..11f9eb9c4 100644
--- a/bsd/dev/ppc/systemcalls.c
+++ b/bsd/dev/ppc/systemcalls.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #include <kern/task.h>
@@ -27,6 +39,7 @@
 #include <kern/locks.h>
 #include <kern/sched_prim.h>
 #include <mach/machine/thread_status.h>
+#include <mach/thread_act.h>
 #include <ppc/savearea.h>
 
 #include <sys/kernel.h>
@@ -36,7 +49,6 @@
 #include <sys/systm.h>
 #include <sys/user.h>
 #include <sys/errno.h>
-#include <sys/ktrace.h>
 #include <sys/kdebug.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
@@ -44,17 +56,20 @@
 
 #include <bsm/audit_kernel.h>
 
+#if CONFIG_DTRACE
+extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
+extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
+#endif
+
 extern void
 unix_syscall(struct savearea *regs);
-void
-unix_syscall_return(int error);
 
 extern struct savearea * 
 find_user_regs(
 	thread_t act);
 
-extern void enter_funnel_section(funnel_t *funnel_lock);
-extern void exit_funnel_section(void);
+extern lck_spin_t * tz_slock;
+extern void throttle_lowpri_io(int *lowpri_window, mount_t v_mount);
 
 /*
  * Function:	unix_syscall
@@ -71,10 +86,8 @@ unix_syscall(struct savearea	*regs)
 	struct proc			*proc;
 	struct sysent		*callp;
 	int					error;
-	unsigned short		code;
+	unsigned int		code;
 	boolean_t			flavor;
-	int funnel_type;
-	unsigned int cancel_enable;
 
 	flavor = (((unsigned int)regs->save_r0) == 0)? 1: 0;
 
@@ -113,15 +126,9 @@ unix_syscall(struct savearea	*regs)
 	 * Delayed binding of thread credential to process credential, if we
 	 * are not running with an explicitly set thread credential.
 	 */
-	if (uthread->uu_ucred != proc->p_ucred &&
-	    (uthread->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = uthread->uu_ucred;
-		uthread->uu_ucred = kauth_cred_proc_ref(proc);
-		if (IS_VALID_CRED(old))
-			kauth_cred_unref(&old);
-	}
+	kauth_cred_uthread_update(uthread, proc);
 
-	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+	callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
 
 	if (callp->sy_narg != 0) {
 		void 		*regsp;
@@ -129,7 +136,7 @@ unix_syscall(struct savearea	*regs)
 		
 		if (IS_64BIT_PROCESS(proc)) {
 			/* XXX Turn 64 bit unsafe calls into nosys() */
-			if (callp->sy_funnel & UNSAFE_64BIT) {
+			if (callp->sy_flags & UNSAFE_64BIT) {
 				callp = &sysent[63];
 				goto unsafe;
 			}
@@ -153,27 +160,9 @@ unix_syscall(struct savearea	*regs)
 	}
 
 unsafe:
-	cancel_enable = callp->sy_cancel;
 	
-	if (cancel_enable == _SYSCALL_CANCEL_NONE) {
-			uthread->uu_flag |= UT_NOTCANCELPT;
-	} else {
-		if((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
-			if (cancel_enable == _SYSCALL_CANCEL_PRE) {
-					/* system call cancelled; return to handle cancellation */
-					regs->save_r3 = (long long)EINTR;
-					thread_exception_return();
-					/* NOTREACHED */
- 			} else {
-                        thread_abort_safely(thread_act);
-			}
-		}
-	}
+	uthread->uu_flag |= UT_NOTCANCELPT;
 
-	funnel_type = (int)(callp->sy_funnel & FUNNEL_MASK);
-	if (funnel_type == KERNEL_FUNNEL) 
-		 enter_funnel_section(kernel_flock);
-	
 	uthread->uu_rval[0] = 0;
 
 	/*
@@ -191,21 +180,26 @@ unsafe:
 	 */
 	regs->save_srr0 += 4;
 
-	if (KTRPOINT(proc, KTR_SYSCALL))
-		ktrsyscall(proc, code, callp->sy_narg, uthread->uu_arg);
-
 #ifdef JOE_DEBUG
 	uthread->uu_iocount = 0;
 	uthread->uu_vpindex = 0;
 #endif
 	AUDIT_SYSCALL_ENTER(code, proc, uthread);
 	error = (*(callp->sy_call))(proc, (void *)uthread->uu_arg, &(uthread->uu_rval[0]));
-	AUDIT_SYSCALL_EXIT(error, proc, uthread);
+	AUDIT_SYSCALL_EXIT(code, proc, uthread, error);
+#if CONFIG_MACF
+	mac_thread_userret(code, error, thread_act);
+#endif
+
 
 #ifdef JOE_DEBUG
 	if (uthread->uu_iocount)
 	        joe_debug("system call returned with uu_iocount != 0");
 #endif
+#if CONFIG_DTRACE
+	uthread->t_dtrace_errno = error;
+#endif /* CONFIG_DTRACE */
+
 	regs = find_user_regs(thread_act);
 
 	if (error == ERESTART) {
@@ -262,29 +256,12 @@ unsafe:
 	/* else  (error == EJUSTRETURN) { nothing } */
 
 
-	if (KTRPOINT(proc, KTR_SYSRET)) {
-		switch(callp->sy_return_type) {
-		case _SYSCALL_RET_ADDR_T:
-		case _SYSCALL_RET_SIZE_T:
-		case _SYSCALL_RET_SSIZE_T:
-			/*
-			 * Trace the value of the least significant bits,
-			 * until we can revise the ktrace API safely.
-			 */
-			ktrsysret(proc, code, error, uthread->uu_rval[1]);
-			break;
-		default:
-			ktrsysret(proc, code, error, uthread->uu_rval[0]);
-			break;
-		}
-	}
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
 
-	if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+	/* panic if funnel is held */
+	syscall_exit_funnelcheck();
 
-	exit_funnel_section();
-
-	if (uthread->uu_lowpri_delay) {
+	if (uthread->uu_lowpri_window && uthread->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -292,8 +269,7 @@ unsafe:
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(uthread->uu_lowpri_delay);
-	        uthread->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
 	}
 	if (kdebug_enable && (code != 180)) {
 
@@ -316,9 +292,8 @@ unix_syscall_return(int error)
 	struct uthread				*uthread;
 	struct proc					*proc;
 	struct savearea				*regs;
-	unsigned short				code;
+	unsigned int				code;
 	struct sysent				*callp;
-	unsigned int cancel_enable;
 
 	thread_act = current_thread();
 	proc = current_proc();
@@ -331,7 +306,12 @@ unix_syscall_return(int error)
 	else
 		code = regs->save_r3;
 
-	callp = (code >= nsysent) ? &sysent[63] : &sysent[code];
+	callp = (code >= NUM_SYSENT) ? &sysent[63] : &sysent[code];
+
+#if CONFIG_DTRACE
+        if (callp->sy_call == dtrace_systrace_syscall)
+                dtrace_systrace_syscall_return( code, error, uthread->uu_rval );
+#endif /* CONFIG_DTRACE */
 
 	/*
 	 * Get index into sysent table
@@ -387,31 +367,13 @@ unix_syscall_return(int error)
 	}
 	/* else  (error == EJUSTRETURN) { nothing } */
 
-	if (KTRPOINT(proc, KTR_SYSRET)) {
-		switch(callp->sy_return_type) {
-		case _SYSCALL_RET_ADDR_T:
-		case _SYSCALL_RET_SIZE_T:
-		case _SYSCALL_RET_SSIZE_T:
-			/*
-			 * Trace the value of the least significant bits,
-			 * until we can revise the ktrace API safely.
-			 */
-			ktrsysret(proc, code, error, uthread->uu_rval[1]);
-			break;
-		default:
-			ktrsysret(proc, code, error, uthread->uu_rval[0]);
-			break;
-		}
-	}
-
-	cancel_enable = callp->sy_cancel;
 
-	if (cancel_enable == _SYSCALL_CANCEL_NONE)
-                uthread->uu_flag &= ~UT_NOTCANCELPT;
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
 
-	exit_funnel_section();
+	/* panic if funnel is held */
+	syscall_exit_funnelcheck();
 
-	if (uthread->uu_lowpri_delay) {
+	if (uthread->uu_lowpri_window && uthread->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -419,8 +381,7 @@ unix_syscall_return(int error)
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(uthread->uu_lowpri_delay);
-	        uthread->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&uthread->uu_lowpri_window,uthread->v_mount);
 	}
 	if (kdebug_enable && (code != 180)) {
 	        if (callp->sy_return_type == _SYSCALL_RET_SSIZE_T)
diff --git a/bsd/dev/ppc/unix_signal.c b/bsd/dev/ppc/unix_signal.c
index 83bab1f9d..435123101 100644
--- a/bsd/dev/ppc/unix_signal.c
+++ b/bsd/dev/ppc/unix_signal.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
@@ -42,12 +48,13 @@
 #include <mach/ppc/thread_status.h>
 #include <ppc/proc_reg.h>
 
+#include <sys/sdt.h>
+
 // #include <machine/thread.h> XXX include path messed up for some reason...
 
 /* XXX functions not in a Mach headers */
 extern kern_return_t thread_getstatus(register thread_t act, int flavor,
 			thread_state_t tstate, mach_msg_type_number_t *count);
-extern int is_64signalregset(void);
 extern unsigned int get_msr_exportmask(void);
 extern kern_return_t thread_setstatus(thread_t thread, int flavor,
 			thread_state_t tstate, mach_msg_type_number_t count);
@@ -95,6 +102,8 @@ extern int thread_enable_fpe(thread_t act, int onoff);
 #define UC_FLAVOR64_VEC		45
 #define UC_DUAL			50
 #define UC_DUAL_VEC		55
+#define	UC_SET_ALT_STACK	0x40000000
+#define UC_RESET_ALT_STACK	0x80000000
 
  /* The following are valid mcontext sizes */
 #define UC_FLAVOR_SIZE ((PPC_THREAD_STATE_COUNT + PPC_EXCEPTION_STATE_COUNT + PPC_FLOAT_STATE_COUNT) * sizeof(int))
@@ -163,7 +172,7 @@ siginfo_64to32(user_siginfo_t *in, siginfo_t *out)
 	/* following cast works for sival_int because of padding */
 	out->si_value.sival_ptr	= CAST_DOWN(void *,in->si_value.sival_ptr);
 	out->si_band	= in->si_band;			/* range reduction */
-	out->pad[0]	= in->pad[0];			/* mcontext.ss.r1 */
+	out->__pad[0]	= in->pad[0];			/* mcontext.ss.r1 */
 }
 
 
@@ -196,38 +205,81 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 	int stack_size = 0;
 	void * tstate;
 	int flavor;
-	int ctx32 = 1;
-	int uthsigaltstack = 0;
-	int altstack = 0;
-	
-	
+        int ctx32 = 1;
+
 	th_act = current_thread();
 	ut = get_bsdthread_info(th_act);
 
-	
+	/*
+	 * XXX We conditionalize type passed here based on SA_SIGINFO, but
+	 * XXX we always send up all the information, regardless; perhaps
+	 * XXX this should not be conditionalized?  Defer making this change
+	 * XXX now, due to possible tools impact.
+	 */
 	if (p->p_sigacts->ps_siginfo & sigmask(sig)) {
-		infostyle = UC_FLAVOR;
-        }
-	if(is_64signalregset() && (infostyle == UC_FLAVOR)) {
-		dualcontext = 1;
-		infostyle = UC_DUAL;	
-	}
-	if (p->p_sigacts->ps_64regset & sigmask(sig)) {
-		dualcontext = 0;
-                ctx32 = 0;
-		infostyle = UC_FLAVOR64;
-	}
-	/* treat 64 bit processes as having used 64 bit registers */
-        if ((IS_64BIT_PROCESS(p) || is_64signalregset()) &&
-	    (infostyle == UC_TRAD)) {
-                ctx32=0;
- 		infostyle = UC_TRAD64;
-	}
-	if (IS_64BIT_PROCESS(p)) {
-		ctx32=0;
-		dualcontext = 0;
+		/*
+		 * If SA_SIGINFO is set, then we must provide the user
+		 * process both a siginfo_t and a context argument.  We call
+		 * this "FLAVORED", as opposed to "TRADITIONAL", which doesn't
+		 * expect a context.  "DUAL" is a type of "FLAVORED".
+		 */
+		if (is_64signalregset()) {
+			/*
+			 * If this is a 64 bit CPU, we must include a 64 bit
+			 * context in the data we pass to user space; we may
+			 * or may not also include a 32 bit context at the
+			 * same time, for non-leaf functions.
+			 *
+			 * The user may also explicitly choose to not receive
+			 * a 32 bit context, at their option; we only allow
+			 * this to happen on 64 bit processors, for obvious
+			 * reasons.
+			 */
+			if (IS_64BIT_PROCESS(p) ||
+			    (p->p_sigacts->ps_64regset & sigmask(sig))) {
+				 /*
+				  * For a 64 bit process, there is no 32 bit
+				  * context.
+				  */
+				ctx32 = 0;
+				infostyle = UC_FLAVOR64;
+			} else {
+				/*
+				 * For a 32 bit process on a 64 bit CPU, we
+				 * may have 64 bit leaf functions, so we need
+				 * both contexts.
+				 */
+				dualcontext = 1;
+				infostyle = UC_DUAL;
+			}
+		} else {
+			/*
+			 * If this is a 32 bit CPU, then we only have a 32 bit
+			 * context to contend with.
+			 */
+			infostyle = UC_FLAVOR;
+		}
+        } else {
+		/*
+		 * If SA_SIGINFO is not set, then we have a traditional style
+		 * call which does not need additional context passed.  The
+		 * default is 32 bit traditional.
+		 *
+		 * XXX The second check is redundant on PPC32; keep it anyway.
+		 */
+		if (is_64signalregset() || IS_64BIT_PROCESS(p)) {
+			/*
+			 * However, if this is a 64 bit CPU, we need to change
+			 * this to 64 bit traditional, and drop the 32 bit
+			 * context.
+			 */
+			ctx32 = 0;
+			infostyle = UC_TRAD64;
+		}
 	}
-	
+
+	proc_unlock(p);
+
 	/* I need this for SIGINFO anyway */
 	flavor = PPC_THREAD_STATE;
 	tstate = (void *)&mctx.ss;
@@ -303,31 +355,15 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 	}  
 
 	trampact = ps->ps_trampact[sig];
-	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
-	
-	if (uthsigaltstack != 0 )  {
-		oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
-		altstack = ut->uu_flag & UT_ALTSTACK;
-	} else {
-		oonstack = ps->ps_sigstk.ss_flags & SA_ONSTACK;
-		altstack = ps->ps_flags & SAS_ALTSTACK;
-	}
-	
+	oonstack = ut->uu_sigstk.ss_flags & SA_ONSTACK;
 
 	/* figure out where our new stack lives */
-	if (altstack && !oonstack &&
+	if ((ut->uu_flag & UT_ALTSTACK) && !oonstack &&
 		(ps->ps_sigonstack & sigmask(sig))) {
-		if (uthsigaltstack != 0) {
-			sp = ut->uu_sigstk.ss_sp;
-			sp += ut->uu_sigstk.ss_size;
-			stack_size = ut->uu_sigstk.ss_size;
-			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
-		} else {
-			sp = ps->ps_sigstk.ss_sp;
-			sp += ps->ps_sigstk.ss_size;
-			stack_size = ps->ps_sigstk.ss_size;
-			ps->ps_sigstk.ss_flags |= SA_ONSTACK;
-		}
+		sp = ut->uu_sigstk.ss_sp;
+		sp += ut->uu_sigstk.ss_size;
+		stack_size = ut->uu_sigstk.ss_size;
+		ut->uu_sigstk.ss_flags |= SA_ONSTACK;
 	}
 	else {
 		if (ctx32 == 0)
@@ -417,26 +453,6 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 	}
 
 	switch (sig) {
-		case SIGCHLD:
-			sinfo.si_pid = p->si_pid;
-			p->si_pid =0;
-			sinfo.si_status = p->si_status;
-			p->si_status = 0;
-			sinfo.si_uid = p->si_uid;
-			p->si_uid =0;
-			sinfo.si_code = p->si_code;
-			p->si_code = 0;
-			if (sinfo.si_code == CLD_EXITED) {
-				if (WIFEXITED(sinfo.si_status)) 
-					sinfo.si_code = CLD_EXITED;
-				else if (WIFSIGNALED(sinfo.si_status)) {
-					if (WCOREDUMP(sinfo.si_status))
-						sinfo.si_code = CLD_DUMPED;
-					else	
-						sinfo.si_code = CLD_KILLED;
-				}
-			}
-			break;
 		case SIGILL:
 			/*
 			 * If it's 64 bit and not a dual context, mctx will
@@ -540,12 +556,60 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 			}
 			break;
 		default:
+		{
+			int status_and_exitcode;
+
+			/*
+			 * All other signals need to fill out a minimum set of
+			 * information for the siginfo structure passed into
+			 * the signal handler, if SA_SIGINFO was specified.
+			 *
+			 * p->si_status actually contains both the status and
+			 * the exit code; we save it off in its own variable
+			 * for later breakdown.
+			 */
+			proc_lock(p);
+			sinfo.si_pid = p->si_pid;
+			p->si_pid = 0;
+			status_and_exitcode = p->si_status;
+			p->si_status = 0;
+			sinfo.si_uid = p->si_uid;
+			p->si_uid = 0;
+			sinfo.si_code = p->si_code;
+			p->si_code = 0;
+			proc_unlock(p);
+			if (sinfo.si_code == CLD_EXITED) {
+				if (WIFEXITED(status_and_exitcode)) 
+					sinfo.si_code = CLD_EXITED;
+				else if (WIFSIGNALED(status_and_exitcode)) {
+					if (WCOREDUMP(status_and_exitcode)) {
+						sinfo.si_code = CLD_DUMPED;
+						status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode);
+					} else {
+						sinfo.si_code = CLD_KILLED;
+						status_and_exitcode = W_EXITCODE(status_and_exitcode,status_and_exitcode);
+					}
+				}
+			}
+			/*
+			 * The recorded status contains the exit code and the
+			 * signal information, but the information to be passed
+			 * in the siginfo to the handler is supposed to only
+			 * contain the status, so we have to shift it out.
+			 */
+			sinfo.si_status = WEXITSTATUS(status_and_exitcode);
 			break;
+		}
 	}
 
 
 	/* copy info out to user space */
 	if (IS_64BIT_PROCESS(p)) {
+
+		/* XXX truncates catcher address to uintptr_t */
+		DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &sinfo,
+			void (*)(void), CAST_DOWN(sig_t, catcher));
+
 		if (copyout(&uctx, p_uctx, sizeof(struct user_ucontext64)))
 			goto bad;
 		if (copyout(&sinfo, p_sinfo, sizeof(user_siginfo_t)))
@@ -555,10 +619,14 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 		siginfo_t sinfo32;
 
 		ucontext_64to32(&uctx, &uctx32);
+		siginfo_64to32(&sinfo,&sinfo32);
+
+		DTRACE_PROC3(signal__handle, int, sig, siginfo_t *, &sinfo32,
+			void (*)(void), CAST_DOWN(sig_t, catcher));
+
 		if (copyout(&uctx32, p_uctx, sizeof(struct ucontext64)))
 			goto bad;
 
-		siginfo_64to32(&sinfo,&sinfo32);
 		if (copyout(&sinfo32, p_sinfo, sizeof(siginfo_t)))
 			goto bad;
 	}
@@ -608,16 +676,21 @@ sendsig(struct proc *p, user_addr_t catcher, int sig, int mask, __unused u_long
 			panic("sendsig: thread_setstatus failed, ret = %08X\n", kretn);
 		}	
 	}
+
+	proc_lock(p);
 	return;
 
 bad:
+	proc_lock(p);
 	SIGACTION(p, SIGILL) = SIG_DFL;
 	sig = sigmask(SIGILL);
 	p->p_sigignore &= ~sig;
 	p->p_sigcatch &= ~sig;
 	ut->uu_sigmask &= ~sig;
 	/* sendsig is called with signal lock held */
-	psignal_lock(p, SIGILL, 0);
+	proc_unlock(p);
+	psignal_locked(p, SIGILL);
+	proc_lock(p);
 	return;
 }
 
@@ -651,12 +724,24 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 	struct uthread * ut;
 	int vec_used = 0;
 	void *tsptr, *fptr, *vptr;
-	int infostyle = uap->infostyle;
-	int uthsigaltstack = 0;
-	
+        int infostyle = uap->infostyle;
+
 	th_act = current_thread();
 
 	ut = (struct uthread *)get_bsdthread_info(th_act);
+
+	/*
+	 * If we are being asked to change the altstack flag on the thread, we
+	 * just rest it and return (the uap->uctx is not used).
+	 */
+	if (infostyle == UC_SET_ALT_STACK) {
+		ut->uu_sigstk.ss_flags |= SA_ONSTACK;
+		return (0);
+	} else if ((unsigned int)infostyle == UC_RESET_ALT_STACK) {
+		ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
+		return (0);
+	}
+
 	if (IS_64BIT_PROCESS(p)) {
 		error = copyin(uap->uctx, &uctx, sizeof(struct user_ucontext64));
 		if (error)
@@ -695,21 +780,11 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 	error = copyin(uctx.uc_mcontext64, mactx, uctx.uc_mcsize);
 	if (error)
 		return(error);
-
-	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
-
-
-	if (uctx.uc_onstack & 01) {
-		if (uthsigaltstack != 0)
+	
+	if ((uctx.uc_onstack & 01))
 			ut->uu_sigstk.ss_flags |= SA_ONSTACK;
-		else
-			p->p_sigacts->ps_sigstk.ss_flags |= SA_ONSTACK;
-	} else {
-		if (uthsigaltstack != 0)
+	else
 			ut->uu_sigstk.ss_flags &= ~SA_ONSTACK;
-		else
-			p->p_sigacts->ps_sigstk.ss_flags &= ~SA_ONSTACK;
-	}
 
 	ut->uu_sigmask = uctx.uc_sigmask & ~sigcantmask;
 	if (ut->uu_siglist & ~ut->uu_sigmask)
@@ -782,12 +857,11 @@ sigreturn(struct proc *p, struct sigreturn_args *uap, __unused int *retval)
 
 boolean_t
 machine_exception(
-    int		exception,
-    int		code,
-    __unused int subcode,
-    int		*unix_signal,
-    int		*unix_code
-)
+		int				exception,
+		mach_exception_code_t		code,
+		__unused mach_exception_subcode_t subcode,
+		int				*unix_signal,
+		mach_exception_code_t		*unix_code)
 {
     switch(exception) {
 
diff --git a/bsd/dev/ppc/xsumas.s b/bsd/dev/ppc/xsumas.s
index dae54fb13..6ac06e947 100644
--- a/bsd/dev/ppc/xsumas.s
+++ b/bsd/dev/ppc/xsumas.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #define kShort  11
diff --git a/bsd/dev/random/YarrowCoreLib/include/WindowsTypesForMac.h b/bsd/dev/random/YarrowCoreLib/include/WindowsTypesForMac.h
index bdab3ea8a..dee0d2e56 100644
--- a/bsd/dev/random/YarrowCoreLib/include/WindowsTypesForMac.h
+++ b/bsd/dev/random/YarrowCoreLib/include/WindowsTypesForMac.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/include/yarrow.h b/bsd/dev/random/YarrowCoreLib/include/yarrow.h
index af6b23bf1..e1ce58dd0 100644
--- a/bsd/dev/random/YarrowCoreLib/include/yarrow.h
+++ b/bsd/dev/random/YarrowCoreLib/include/yarrow.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/include/yarrowUtils.h b/bsd/dev/random/YarrowCoreLib/include/yarrowUtils.h
index c25314991..cdeb4bbf4 100644
--- a/bsd/dev/random/YarrowCoreLib/include/yarrowUtils.h
+++ b/bsd/dev/random/YarrowCoreLib/include/yarrowUtils.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/port/smf.c b/bsd/dev/random/YarrowCoreLib/port/smf.c
index 297fe0f58..6bdd9ca07 100644
--- a/bsd/dev/random/YarrowCoreLib/port/smf.c
+++ b/bsd/dev/random/YarrowCoreLib/port/smf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -47,7 +53,7 @@ SMFAPI MMPTR mmMalloc(DWORD request)
     if (mem == 0) // oops, it didn't appear to work
     {
         printf ("Couldn't allocate kernel memory!\n");
-        return 0;
+        return NULL;
     }
     
     return (MMPTR) mem;
diff --git a/bsd/dev/random/YarrowCoreLib/src/assertverify.h b/bsd/dev/random/YarrowCoreLib/src/assertverify.h
index feff7245c..3e8613462 100644
--- a/bsd/dev/random/YarrowCoreLib/src/assertverify.h
+++ b/bsd/dev/random/YarrowCoreLib/src/assertverify.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef ASSERT_VERIFY_H
diff --git a/bsd/dev/random/YarrowCoreLib/src/comp.c b/bsd/dev/random/YarrowCoreLib/src/comp.c
index 8d2faeea0..c22784095 100644
--- a/bsd/dev/random/YarrowCoreLib/src/comp.c
+++ b/bsd/dev/random/YarrowCoreLib/src/comp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/src/comp.h b/bsd/dev/random/YarrowCoreLib/src/comp.h
index e264e2c00..6c7989a91 100644
--- a/bsd/dev/random/YarrowCoreLib/src/comp.h
+++ b/bsd/dev/random/YarrowCoreLib/src/comp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/src/entropysources.h b/bsd/dev/random/YarrowCoreLib/src/entropysources.h
index ad9cfb2ec..52f387571 100644
--- a/bsd/dev/random/YarrowCoreLib/src/entropysources.h
+++ b/bsd/dev/random/YarrowCoreLib/src/entropysources.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* entropysources.h */
diff --git a/bsd/dev/random/YarrowCoreLib/src/macOnly.h b/bsd/dev/random/YarrowCoreLib/src/macOnly.h
index ea1d86b42..c42c7ae89 100644
--- a/bsd/dev/random/YarrowCoreLib/src/macOnly.h
+++ b/bsd/dev/random/YarrowCoreLib/src/macOnly.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/src/prng.c b/bsd/dev/random/YarrowCoreLib/src/prng.c
index e2ba0a2ee..bbb56785f 100644
--- a/bsd/dev/random/YarrowCoreLib/src/prng.c
+++ b/bsd/dev/random/YarrowCoreLib/src/prng.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -98,12 +104,12 @@ static DWORD mutexCreatorId = 0;
 static void 
 prng_do_SHA1(GEN_CTX *ctx) 
 {
-	SHA1_CTX sha;
+	YSHA1_CTX sha;
 
-	SHA1Init(&sha);
-	SHA1Update(&sha,ctx->IV,20);
-	SHA1Update(&sha,ctx->out,20);
-	SHA1Final(ctx->out,&sha);
+	YSHA1Init(&sha);
+	YSHA1Update(&sha,ctx->IV,20);
+	YSHA1Update(&sha,ctx->out,20);
+	YSHA1Final(ctx->out,&sha);
 	ctx->index = 0;
 }
 
@@ -117,12 +123,12 @@ prng_do_SHA1(GEN_CTX *ctx)
 static void 
 prng_make_new_state(GEN_CTX *ctx,BYTE *newState) 
 {
-	SHA1_CTX sha;
+	YSHA1_CTX sha;
 
 	memcpy(ctx->IV,newState,20);
-	SHA1Init(&sha);
-	SHA1Update(&sha,ctx->IV,20);
-	SHA1Final(ctx->out,&sha);
+	YSHA1Init(&sha);
+	YSHA1Update(&sha,ctx->IV,20);
+	YSHA1Final(ctx->out,&sha);
 	ctx->numout = 0;
 	ctx->index = 0;
 }
@@ -139,7 +145,7 @@ static void
 prng_slow_init(PRNG *p)
 /* This fails silently and must be fixed. */
 {
-	SHA1_CTX* ctx = NULL;
+	YSHA1_CTX* ctx = NULL;
 	MMPTR mmctx = MM_NULL;
 	BYTE* bigbuf = NULL;
 	MMPTR mmbigbuf = MM_NULL;
@@ -155,19 +161,19 @@ prng_slow_init(PRNG *p)
 	if(mmbuf == MM_NULL) {goto cleanup_slow_init;}
 	buf = (BYTE*)mmGetPtr(mmbuf);
 
-	mmctx = mmMalloc(sizeof(SHA1_CTX));
+	mmctx = mmMalloc(sizeof(YSHA1_CTX));
 	if(mmctx == MM_NULL) {goto cleanup_slow_init;}
-	ctx = (SHA1_CTX*)mmGetPtr(mmctx);
+	ctx = (YSHA1_CTX*)mmGetPtr(mmctx);
 
 
 	/* Initialize the secret state. */
 	/* Init entropy pool */
-	SHA1Init(&p->pool);
+	YSHA1Init(&p->pool);
 	/* Init output generator */
 	polllength = prng_slow_poll(bigbuf,SPLEN);
-	SHA1Init(ctx);
-	SHA1Update(ctx,bigbuf,polllength);
-	SHA1Final(buf,ctx);
+	YSHA1Init(ctx);
+	YSHA1Update(ctx,bigbuf,polllength);
+	YSHA1Final(buf,ctx);
 	prng_make_new_state(&p->outstate, buf);
 
 cleanup_slow_init:
@@ -259,7 +265,7 @@ prngInitialize(PrngRef *prng)
 	/* Initialize the secret state. */
 	/* FIXME - might want to make this an option here and have the caller
 	 * do it after we return....? */
-	SHA1Init(&p->pool);
+	YSHA1Init(&p->pool);
 #if		SLOW_POLL_ENABLE
 	prng_slow_init(p);	/* Does a slow poll and then calls prng_make_state(...) */
 #else	
@@ -372,15 +378,15 @@ prngForceReseed(PRNG *p, LONGLONG ticks)
 	{
 		/* Do a couple of iterations between time checks */
 		prngOutput(p, buf,64);
-		SHA1Update(&p->pool,buf,64);
+		YSHA1Update(&p->pool,buf,64);
 		prngOutput(p, buf,64);
-		SHA1Update(&p->pool,buf,64);
+		YSHA1Update(&p->pool,buf,64);
 		prngOutput(p, buf,64);
-		SHA1Update(&p->pool,buf,64);
+		YSHA1Update(&p->pool,buf,64);
 		prngOutput(p, buf,64);
-		SHA1Update(&p->pool,buf,64);
+		YSHA1Update(&p->pool,buf,64);
 		prngOutput(p, buf,64);
-		SHA1Update(&p->pool,buf,64);
+		YSHA1Update(&p->pool,buf,64);
 
 #if		defined(macintosh) || defined(__APPLE__)
 	#if		defined(TARGET_API_MAC_OSX) || defined(KERNEL_BUILD)
@@ -399,12 +405,12 @@ prngForceReseed(PRNG *p, LONGLONG ticks)
 #else
 	} while ( (now-start) < ticks) ;
 #endif
-	SHA1Final(dig,&p->pool);
-	SHA1Update(&p->pool,dig,20); 
-	SHA1Final(dig,&p->pool);
+	YSHA1Final(dig,&p->pool);
+	YSHA1Update(&p->pool,dig,20); 
+	YSHA1Final(dig,&p->pool);
 
 	/* Reset secret state */
-	SHA1Init(&p->pool);
+	YSHA1Init(&p->pool);
 	prng_make_new_state(&p->outstate,dig);
 
 	/* Clear counter variables */
@@ -431,9 +437,9 @@ prngProcessSeedBuffer(PRNG *p, BYTE *buf,LONGLONG ticks)
 	PCHECK(buf);
 
 	/* Put the data into the entropy, add some data from the unknown state, reseed */
-	SHA1Update(&p->pool,buf,20);			/* Put it into the entropy pool */
+	YSHA1Update(&p->pool,buf,20);			/* Put it into the entropy pool */
 	prng_do_SHA1(&p->outstate);				/* Output 20 more bytes and     */
-	SHA1Update(&p->pool,p->outstate.out,20);/* add it to the pool as well.  */
+	YSHA1Update(&p->pool,p->outstate.out,20);/* add it to the pool as well.  */
 	prngForceReseed(p, ticks); 				/* Do a reseed */
 	return prngOutput(p, buf,20); /* Return the first 20 bytes of output in buf */
 }
@@ -444,7 +450,7 @@ prngProcessSeedBuffer(PRNG *p, BYTE *buf,LONGLONG ticks)
 prng_error_status
 prngStretch(BYTE *inbuf,UINT inbuflen,BYTE *outbuf,UINT outbuflen) {
 	long int left,prev;
-	SHA1_CTX ctx;
+	YSHA1_CTX ctx;
 	BYTE dig[20];
 
 	PCHECK(inbuf);
@@ -457,13 +463,13 @@ prngStretch(BYTE *inbuf,UINT inbuflen,BYTE *outbuf,UINT outbuflen) {
 	}
 	else  /* Extend using SHA1 hash of inbuf */
 	{
-		SHA1Init(&ctx);
-		SHA1Update(&ctx,inbuf,inbuflen);
-		SHA1Final(dig,&ctx);
+		YSHA1Init(&ctx);
+		YSHA1Update(&ctx,inbuf,inbuflen);
+		YSHA1Final(dig,&ctx);
 		for(prev=0,left=outbuflen;left>0;prev+=20,left-=20) 
 		{
-			SHA1Update(&ctx,dig,20);
-			SHA1Final(dig,&ctx);
+			YSHA1Update(&ctx,dig,20);
+			YSHA1Final(dig,&ctx);
 			memcpy(outbuf+prev,dig,(left>20)?20:left);
 		}
 		trashMemory(dig,20*sizeof(BYTE));
@@ -489,7 +495,7 @@ prngInput(PRNG *p, BYTE *inbuf,UINT inbuflen,UINT poolnum, __unused UINT estbits
 	if(poolnum >= TOTAL_SOURCES) {return PRNG_ERR_OUT_OF_BOUNDS;}
 
 	/* Add to entropy pool */
-	SHA1Update(&p->pool,inbuf,inbuflen);
+	YSHA1Update(&p->pool,inbuf,inbuflen);
 	
 	#ifndef	YARROW_KERNEL
 	/* skip this step for the kernel */
diff --git a/bsd/dev/random/YarrowCoreLib/src/prng.h b/bsd/dev/random/YarrowCoreLib/src/prng.h
index 8524068a6..df8a53886 100644
--- a/bsd/dev/random/YarrowCoreLib/src/prng.h
+++ b/bsd/dev/random/YarrowCoreLib/src/prng.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/src/prngpriv.h b/bsd/dev/random/YarrowCoreLib/src/prngpriv.h
index e2919d0d2..5030deb85 100644
--- a/bsd/dev/random/YarrowCoreLib/src/prngpriv.h
+++ b/bsd/dev/random/YarrowCoreLib/src/prngpriv.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -78,7 +84,7 @@ struct PRNG {
 	GEN_CTX outstate;
 
 	/* Entropy Pools (somewhat unlike a gene pool) */
-	SHA1_CTX pool;
+	YSHA1_CTX pool;
 	UINT poolSize[TOTAL_SOURCES];			/* Note that size is in bytes and est in bits */
 	UINT poolEstBits[TOTAL_SOURCES];
 	COMP_CTX comp_state[COMP_SOURCES];
diff --git a/bsd/dev/random/YarrowCoreLib/src/sha1mod.c b/bsd/dev/random/YarrowCoreLib/src/sha1mod.c
index f58585865..f240dd9af 100644
--- a/bsd/dev/random/YarrowCoreLib/src/sha1mod.c
+++ b/bsd/dev/random/YarrowCoreLib/src/sha1mod.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -61,7 +67,8 @@ By Steve Reid <steve@edmweb.com>
 
 /* Hash a single 512-bit block. This is the core of the algorithm. */
 
-void SHA1Transform(unsigned long state[5], const unsigned char buffer[64])
+__private_extern__ void
+YSHA1Transform(unsigned long state[5], const unsigned char buffer[64])
 {
 unsigned long a, b, c, d, e;
 typedef union {
@@ -114,9 +121,10 @@ static unsigned char workspace[64];
 }
 
 
-/* SHA1Init - Initialize new context */
+/* YSHA1Init - Initialize new context */
 
-void SHA1Init(SHA1_CTX* context)
+__private_extern__ void
+YSHA1Init(YSHA1_CTX* context)
 {
     /* SHA1 initialization constants */
     context->state[0] = 0x67452301;
@@ -130,7 +138,8 @@ void SHA1Init(SHA1_CTX* context)
 
 /* Run your data through this. */
 
-void SHA1Update(SHA1_CTX* context, const unsigned char* data, unsigned int len)
+__private_extern__ void
+YSHA1Update(YSHA1_CTX* context, const unsigned char* data, unsigned int len)
 {
 unsigned int i, j;
 
@@ -139,9 +148,9 @@ unsigned int i, j;
     context->count[1] += (len >> 29);
     if ((j + len) > 63) {
         memcpy(&context->buffer[j], data, (i = 64-j));
-        SHA1Transform(context->state, context->buffer);
+        YSHA1Transform(context->state, context->buffer);
         for ( ; i + 63 < len; i += 64) {
-            SHA1Transform(context->state, &data[i]);
+            YSHA1Transform(context->state, &data[i]);
         }
         j = 0;
     }
@@ -152,7 +161,8 @@ unsigned int i, j;
 
 /* Add padding and return the message digest. */
 
-void SHA1Final(unsigned char digest[20], SHA1_CTX* context)
+__private_extern__ void
+YSHA1Final(unsigned char digest[20], YSHA1_CTX* context)
 {
 unsigned long i, j;
 unsigned char finalcount[8];
@@ -161,11 +171,11 @@ unsigned char finalcount[8];
         finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)]
          >> ((3-(i & 3)) * 8) ) & 255);  /* Endian independent */
     }
-    SHA1Update(context, "\200", 1);
+    YSHA1Update(context, (unsigned char *)"\200", 1);
     while ((context->count[0] & 504) != 448) {
-        SHA1Update(context, "\0", 1);
+        YSHA1Update(context, (unsigned char *)"\0", 1);
     }
-    SHA1Update(context, finalcount, 8);  /* Should cause a SHA1Transform() */
+    YSHA1Update(context, finalcount, 8);  /* Should cause a YSHA1Transform() */
     for (i = 0; i < 20; i++) {
         digest[i] = (unsigned char)
          ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
@@ -176,8 +186,8 @@ unsigned char finalcount[8];
     memset(context->state, 0, 20);
     memset(context->count, 0, 8);
     memset(finalcount, 0, 8);
-#ifdef SHA1HANDSOFF  /* make SHA1Transform overwrite it's own static vars */
-    SHA1Transform(context->state, context->buffer);
+#ifdef SHA1HANDSOFF  /* make YSHA1Transform overwrite it's own static vars */
+    YSHA1Transform(context->state, context->buffer);
 #endif
 }
 
@@ -191,7 +201,7 @@ unsigned char finalcount[8];
 int main(int argc, char** argv)
 {
 int i, j;
-SHA1_CTX context;
+YSHA1_CTX context;
 unsigned char digest[20], buffer[16384];
 FILE* file;
 
@@ -209,12 +219,12 @@ FILE* file;
             exit(-1);
         }
     } 
-    SHA1Init(&context);
+    YSHA1Init(&context);
     while (!feof(file)) {  /* note: what if ferror(file) */
         i = fread(buffer, 1, 16384, file);
-        SHA1Update(&context, buffer, i);
+        YSHA1Update(&context, buffer, i);
     }
-    SHA1Final(digest, &context);
+    YSHA1Final(digest, &context);
     fclose(file);
     for (i = 0; i < 5; i++) {
         for (j = 0; j < 4; j++) {
diff --git a/bsd/dev/random/YarrowCoreLib/src/sha1mod.h b/bsd/dev/random/YarrowCoreLib/src/sha1mod.h
index c066767bb..60b5c64cf 100644
--- a/bsd/dev/random/YarrowCoreLib/src/sha1mod.h
+++ b/bsd/dev/random/YarrowCoreLib/src/sha1mod.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -50,12 +56,15 @@ typedef struct {
     unsigned long state[5];
     unsigned long count[2];
     unsigned char buffer[64];
-} SHA1_CTX;
+} YSHA1_CTX;
 
 //Function forward declerations
-void SHA1Transform(unsigned long state[5], const unsigned char buffer[64]);
-void SHA1Init(SHA1_CTX* context);
-void SHA1Update(SHA1_CTX* context, const unsigned char* data, unsigned int len);
-void SHA1Final(unsigned char digest[20], SHA1_CTX* context);
+__private_extern__ void YSHA1Transform(unsigned long state[5],
+    const unsigned char buffer[64]);
+__private_extern__ void YSHA1Init(YSHA1_CTX* context);
+__private_extern__ void YSHA1Update(YSHA1_CTX* context,
+    const unsigned char* data, unsigned int len);
+__private_extern__ void YSHA1Final(unsigned char digest[20],
+    YSHA1_CTX* context);
 
 #endif	/* __SHA1_H__ */
diff --git a/bsd/dev/random/YarrowCoreLib/src/smf.h b/bsd/dev/random/YarrowCoreLib/src/smf.h
index ad4fcf321..a332a64f8 100644
--- a/bsd/dev/random/YarrowCoreLib/src/smf.h
+++ b/bsd/dev/random/YarrowCoreLib/src/smf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -76,7 +82,7 @@ extern "C" {
 #endif /* macintosh */
 
 
-#define MM_NULL	0
+#define MM_NULL	((void *)0)
 
 /* Function forward declarations */
 SMFAPI void mmInit( void );
diff --git a/bsd/dev/random/YarrowCoreLib/src/userdefines.h b/bsd/dev/random/YarrowCoreLib/src/userdefines.h
index 9ca48cc25..3021f5885 100644
--- a/bsd/dev/random/YarrowCoreLib/src/userdefines.h
+++ b/bsd/dev/random/YarrowCoreLib/src/userdefines.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/YarrowCoreLib/src/yarrowUtils.c b/bsd/dev/random/YarrowCoreLib/src/yarrowUtils.c
index bd8b19794..c57f50dce 100644
--- a/bsd/dev/random/YarrowCoreLib/src/yarrowUtils.c
+++ b/bsd/dev/random/YarrowCoreLib/src/yarrowUtils.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/dev/random/randomdev.c b/bsd/dev/random/randomdev.c
index 04e1883a4..4a7741e2a 100644
--- a/bsd/dev/random/randomdev.c
+++ b/bsd/dev/random/randomdev.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c)1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>
@@ -30,13 +36,17 @@
 #include <string.h>
 #include <miscfs/devfs/devfs.h>
 #include <kern/lock.h>
+#include <kern/clock.h>
 #include <sys/time.h>
 #include <sys/malloc.h>
 #include <sys/uio_internal.h>
 
 #include <dev/random/randomdev.h>
 #include <dev/random/YarrowCoreLib/include/yarrow.h>
-#include <crypto/sha1.h>
+#include <libkern/crypto/sha1.h>
+
+#include <mach/mach_time.h>
+#include <machine/machine_routines.h>
 
 #define RANDOM_MAJOR  -1 /* let the kernel pick the device number */
 
@@ -68,7 +78,12 @@ static struct cdevsw random_cdevsw =
 static int gRandomInstalled = 0;
 static PrngRef gPrngRef;
 static int gRandomError = 1;
-static mutex_t *gYarrowMutex = 0;
+static lck_grp_t *gYarrowGrp;
+static lck_attr_t *gYarrowAttr;
+static lck_grp_attr_t *gYarrowGrpAttr;
+static lck_mtx_t *gYarrowMutex = 0;
+
+void CheckReseed(void);
 
 #define RESEED_TICKS 50 /* how long a reseed operation can take */
 
@@ -84,7 +99,7 @@ typedef BlockWord Block[kBSize];
 
 void add_blocks(Block a, Block b, BlockWord carry);
 void fips_initialize(void);
-void random_block(Block b, int addOptional);
+void random_block(Block b);
 u_int32_t CalculateCRC(u_int8_t* buffer, size_t length);
 
 /*
@@ -179,21 +194,18 @@ u_int32_t CalculateCRC(u_int8_t* buffer, size_t length)
  * get a random block of data per fips 186-2
  */
 void
-random_block(Block b, int addOptional)
+random_block(Block b)
 {
 	int repeatCount = 0;
 	do
 	{
-		if (addOptional)
-		{
-			// do one iteration
-			Block xSeed;
-			prngOutput (gPrngRef, (BYTE*) &xSeed, sizeof (xSeed));
-			
-			// add the seed to the previous value of g_xkey
-			add_blocks (g_xkey, xSeed, 0);
-		}
+		// do one iteration
+		Block xSeed;
+		prngOutput (gPrngRef, (BYTE*) &xSeed, sizeof (xSeed));
 		
+		// add the seed to the previous value of g_xkey
+		add_blocks (g_xkey, xSeed, 0);
+
 		// compute "G"
 		SHA1Update (&g_sha1_ctx, (const u_int8_t *) &g_xkey, sizeof (g_xkey));
 		
@@ -249,8 +261,6 @@ void
 PreliminarySetup(void)
 {
     prng_error_status perr;
-    struct timeval tt;
-    char buffer [16];
 
     /* create a Yarrow object */
     perr = prngInitialize(&gPrngRef);
@@ -262,6 +272,10 @@ PreliminarySetup(void)
 	/* clear the error flag, reads and write should then work */
     gRandomError = 0;
 
+   {
+    struct timeval tt;
+    char buffer [16];
+
     /* get a little non-deterministic data as an initial seed. */
     microtime(&tt);
 
@@ -280,24 +294,26 @@ PreliminarySetup(void)
     }
     
     /* turn the data around */
-    perr = prngOutput(gPrngRef, (BYTE*)buffer, sizeof (buffer));
+    perr = prngOutput(gPrngRef, (BYTE*) buffer, sizeof (buffer));
     
     /* and scramble it some more */
     perr = prngForceReseed(gPrngRef, RESEED_TICKS);
+    }
     
     /* make a mutex to control access */
-    gYarrowMutex = mutex_alloc(0);
+    gYarrowGrpAttr = lck_grp_attr_alloc_init();
+    gYarrowGrp     = lck_grp_alloc_init("random", gYarrowGrpAttr);
+    gYarrowAttr    = lck_attr_alloc_init();
+    gYarrowMutex   = lck_mtx_alloc_init(gYarrowGrp, gYarrowAttr);
 	
 	fips_initialize ();
 }
 
-const Block kKnownAnswer = {0x92b404e5, 0x56588ced, 0x6c1acd4e, 0xbf053f68, 0x9f73a93};
-
 void
 fips_initialize(void)
 {
-	/* So that we can do the self test, set the seed to zero */
-	memset(&g_xkey, 0, sizeof(g_xkey));
+	/* Read the initial value of g_xkey from yarrow */
+	prngOutput (gPrngRef, (BYTE*) &g_xkey, sizeof (g_xkey));
 	
 	/* initialize our SHA1 generator */
 	SHA1Init (&g_sha1_ctx);
@@ -305,20 +321,7 @@ fips_initialize(void)
 	/* other initializations */
 	memset (zeros, 0, sizeof (zeros));
 	g_bytes_used = 0;
-	random_block(g_random_data, FALSE);
-	
-	// check here to see if we got the initial data we were expecting
-	int i;
-	for (i = 0; i < kBSize; ++i)
-	{
-		if (kKnownAnswer[i] != g_random_data[i])
-		{
-			panic("FIPS random self test failed");
-		}
-	}
-
-	// now do the random block again to make sure that userland doesn't get predictable data
-	random_block(g_random_data, TRUE);
+	random_block(g_random_data);
 }
 
 /*
@@ -336,8 +339,10 @@ random_init(void)
 	/* install us in the file system */
 	gRandomInstalled = 1;
 
+#ifndef ARM_BOARD_CONFIG_S5L8900XFPGA_1136JFS
 	/* setup yarrow and the mutex */
 	PreliminarySetup();
+#endif
 
 	ret = cdevsw_add(RANDOM_MAJOR, &random_cdevsw);
 	if (ret < 0) {
@@ -428,7 +433,7 @@ random_write (__unused dev_t dev, struct uio *uio, __unused int ioflag)
     }
     
     /* get control of the Yarrow instance, Yarrow is NOT thread safe */
-    mutex_lock(gYarrowMutex);
+    lck_mtx_lock(gYarrowMutex);
     
     /* Security server is sending us entropy */
 
@@ -441,7 +446,7 @@ random_write (__unused dev_t dev, struct uio *uio, __unused int ioflag)
             goto /*ugh*/ error_exit;
         
         /* put it in Yarrow */
-        if (prngInput(gPrngRef, (BYTE*)rdBuffer,
+        if (prngInput(gPrngRef, (BYTE*) rdBuffer,
 			bytesToInput, SYSTEM_SOURCE,
         	bytesToInput * 8) != 0) {
             retCode = EIO;
@@ -458,14 +463,15 @@ random_write (__unused dev_t dev, struct uio *uio, __unused int ioflag)
     /* retCode should be 0 at this point */
     
 error_exit: /* do this to make sure the mutex unlocks. */
-    mutex_unlock(gYarrowMutex);
+    lck_mtx_unlock(gYarrowMutex);
     return (retCode);
 }
 
 /*
  * return data to the caller.  Results unpredictable.
  */ 
-int random_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
+int
+random_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
 {
     int retCode = 0;
 	
@@ -473,8 +479,9 @@ int random_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
         return (ENOTSUP);
 
    /* lock down the mutex */
-    mutex_lock(gYarrowMutex);
+    lck_mtx_lock(gYarrowMutex);
 
+    CheckReseed();
 	int bytes_remaining = uio_resid(uio);
     while (bytes_remaining > 0 && retCode == 0) {
         /* get the user's data */
@@ -483,14 +490,14 @@ int random_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
 		int bytes_available = kBSizeInBytes - g_bytes_used;
         if (bytes_available == 0)
 		{
-			random_block(g_random_data, TRUE);
+			random_block(g_random_data);
 			g_bytes_used = 0;
 			bytes_available = kBSizeInBytes;
 		}
 		
 		bytes_to_read = min (bytes_remaining, bytes_available);
 		
-        retCode = uiomove(((u_int8_t*)g_random_data)+ g_bytes_used, bytes_to_read, uio);
+        retCode = uiomove(((caddr_t)g_random_data)+ g_bytes_used, bytes_to_read, uio);
         g_bytes_used += bytes_to_read;
 
         if (retCode != 0)
@@ -502,7 +509,7 @@ int random_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
     retCode = 0;
     
 error_exit:
-    mutex_unlock(gYarrowMutex);
+    lck_mtx_unlock(gYarrowMutex);
     return retCode;
 }
 
@@ -511,11 +518,14 @@ void
 read_random(void* buffer, u_int numbytes)
 {
     if (gYarrowMutex == 0) { /* are we initialized? */
+#ifndef ARM_BOARD_CONFIG_S5L8900XFPGA_1136JFS
         PreliminarySetup ();
+#endif
     }
     
-    mutex_lock(gYarrowMutex);
-	
+    lck_mtx_lock(gYarrowMutex);
+    CheckReseed();
+
 	int bytes_read = 0;
 
 	int bytes_remaining = numbytes;
@@ -523,18 +533,18 @@ read_random(void* buffer, u_int numbytes)
         int bytes_to_read = min(bytes_remaining, kBSizeInBytes - g_bytes_used);
         if (bytes_to_read == 0)
 		{
-			random_block(g_random_data, TRUE);
+			random_block(g_random_data);
 			g_bytes_used = 0;
 			bytes_to_read = min(bytes_remaining, kBSizeInBytes);
 		}
 		
-		memmove (buffer, ((u_int8_t*)g_random_data)+ bytes_read, bytes_to_read);
+		memmove ((u_int8_t*) buffer + bytes_read, ((u_int8_t*)g_random_data)+ g_bytes_used, bytes_to_read);
 		g_bytes_used += bytes_to_read;
 		bytes_read += bytes_to_read;
 		bytes_remaining -= bytes_to_read;
     }
 
-    mutex_unlock(gYarrowMutex);
+    lck_mtx_unlock(gYarrowMutex);
 }
 
 /*
@@ -548,3 +558,7 @@ RandomULong(void)
 	return (buf);
 }
 
+void
+CheckReseed(void)
+{
+}
diff --git a/bsd/dev/random/randomdev.h b/bsd/dev/random/randomdev.h
index e5c65aea1..54e11cfa2 100644
--- a/bsd/dev/random/randomdev.h
+++ b/bsd/dev/random/randomdev.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __DEV_RANDOMDEV_H__
diff --git a/bsd/dev/unix_startup.c b/bsd/dev/unix_startup.c
index 018ea1583..d3df77196 100644
--- a/bsd/dev/unix_startup.c
+++ b/bsd/dev/unix_startup.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992,7 NeXT Computer, Inc.
@@ -43,10 +49,12 @@
 
 extern vm_map_t mb_map;
 
+#if INET || INET6
 extern u_long   tcp_sendspace;
 extern u_long   tcp_recvspace;
+#endif
 
-void            bsd_bufferinit(void);
+void            bsd_bufferinit(void) __attribute__((section("__TEXT, initcode")));
 extern void     md_prepare_for_shutdown(int, int, char *);
 
 int		bsd_mbuf_cluster_reserve(void);
@@ -57,17 +65,17 @@ int		bsd_mbuf_cluster_reserve(void);
 
 #ifdef	NBUF
 int             max_nbuf_headers = NBUF;
-int             niobuf = NBUF / 2;
+int             niobuf_headers = NBUF / 2;
 int 		nbuf_hashelements = NBUF;
-int 		nbuf = NBUF;
+int 		nbuf_headers = NBUF;
 #else
 int             max_nbuf_headers = 0;
-int             niobuf = 0;
+int             niobuf_headers = 0;
 int 		nbuf_hashelements = 0;
-int		nbuf = 0;
+int		nbuf_headers = 0;
 #endif
 
-SYSCTL_INT (_kern, OID_AUTO, nbuf, CTLFLAG_RD, &nbuf, 0, "");
+SYSCTL_INT (_kern, OID_AUTO, nbuf, CTLFLAG_RD, &nbuf_headers, 0, "");
 SYSCTL_INT (_kern, OID_AUTO, maxnbuf, CTLFLAG_RW, &max_nbuf_headers, 0, "");
 
 __private_extern__ int customnbuf = 0;
@@ -78,7 +86,7 @@ vm_map_t        buffer_map;
 vm_map_t        bufferhdr_map;
 
 
-extern void     bsd_startupearly(void);
+extern void     bsd_startupearly(void) __attribute__((section("__TEXT, initcode")));
 
 void
 bsd_startupearly(void)
@@ -92,8 +100,8 @@ bsd_startupearly(void)
 		max_nbuf_headers = atop(sane_size / 50);	/* Get 2% of ram, but no more than we can map */
 	if ((customnbuf == 0) && (max_nbuf_headers > 16384))
 		max_nbuf_headers = 16384;
-	if (max_nbuf_headers < 256)
-		max_nbuf_headers = 256;
+	if (max_nbuf_headers < CONFIG_MIN_NBUF)
+		max_nbuf_headers = CONFIG_MIN_NBUF;
 
 	/* clip the number of hash elements  to 200000 */
 	if ( (customnbuf == 0 ) && nbuf_hashelements == 0) {
@@ -103,14 +111,14 @@ bsd_startupearly(void)
 	} else
 		nbuf_hashelements = max_nbuf_headers;
 
-	if (niobuf == 0)
-		niobuf = max_nbuf_headers;
-	if (niobuf > 4096)
-		niobuf = 4096;
-	if (niobuf < 128)
-		niobuf = 128;
+	if (niobuf_headers == 0)
+		niobuf_headers = max_nbuf_headers;
+	if (niobuf_headers > 4096)
+		niobuf_headers = 4096;
+	if (niobuf_headers < CONFIG_MIN_NIOBUF)
+		niobuf_headers = CONFIG_MIN_NIOBUF;
 
-	size = (max_nbuf_headers + niobuf) * sizeof(struct buf);
+	size = (max_nbuf_headers + niobuf_headers) * sizeof(struct buf);
 	size = round_page(size);
 
 	ret = kmem_suballoc(kernel_map,
@@ -132,24 +140,28 @@ bsd_startupearly(void)
 	if (ret != KERN_SUCCESS)
 		panic("Failed to allocate bufferhdr_map");
 
-	buf = (struct buf *) firstaddr;
-	bzero(buf, size);
+	buf_headers = (struct buf *) firstaddr;
+	bzero(buf_headers, size);
 
+#if SOCKETS
 	{
 		int             scale;
 
 		nmbclusters = bsd_mbuf_cluster_reserve() / MCLBYTES;
 
+#if INET || INET6
 		if ((scale = nmbclusters / NMBCLUSTERS) > 1) {
 			tcp_sendspace *= scale;
 			tcp_recvspace *= scale;
 
-			if (tcp_sendspace > (32 * 1024))
-				tcp_sendspace = 32 * 1024;
-			if (tcp_recvspace > (32 * 1024))
-				tcp_recvspace = 32 * 1024;
+			if (tcp_sendspace > (64 * 1024))
+				tcp_sendspace = 64 * 1024;
+			if (tcp_recvspace > (64 * 1024))
+				tcp_recvspace = 64 * 1024;
 		}
+#endif /* INET || INET6 */
 	}
+#endif /* SOCKETS */
 
 	/*
 	 * Size vnodes based on memory 
@@ -157,10 +169,12 @@ bsd_startupearly(void)
 	 * This is the calculation that is used by launchd in tiger
 	 * we are clipping the max based on 16G 
 	 * ie ((16*1024*1024*1024)/(64 *1024)) + 1024 = 263168;
+	 * CONFIG_VNODES is set to 263168 for "medium" configurations (the default)
+	 * but can be smaller or larger. 
 	 */
 	desiredvnodes  = (sane_size/65536) + 1024;
-	if (desiredvnodes > 263168)
-		desiredvnodes = 263168;
+	if (desiredvnodes > CONFIG_VNODES)
+		desiredvnodes = CONFIG_VNODES;
 }
 
 void
@@ -172,6 +186,7 @@ bsd_bufferinit(void)
 
 	bsd_startupearly();
 
+#if SOCKETS
 	ret = kmem_suballoc(kernel_map,
 			    (vm_offset_t *) & mbutl,
 			    (vm_size_t) (nmbclusters * MCLBYTES),
@@ -181,6 +196,7 @@ bsd_bufferinit(void)
 
 	if (ret != KERN_SUCCESS)
 		panic("Failed to allocate mb_map\n");
+#endif /* SOCKETS */
 
 	/*
 	 * Set up buffers, so they can be used to read disk labels.
@@ -188,6 +204,7 @@ bsd_bufferinit(void)
 	bufinit();
 }
 
+
 /*
  * this has been broken out into a separate routine that
  * can be called from the x86 early vm initialization to
@@ -198,12 +215,15 @@ bsd_bufferinit(void)
 int
 bsd_mbuf_cluster_reserve(void)
 {
-	if (sane_size > (64 * 1024 * 1024) || ncl) {
+        if (sane_size > (64 * 1024 * 1024) || ncl) {
 
 	        if ((nmbclusters = ncl) == 0) {
 		        if ((nmbclusters = ((sane_size / 16)/MCLBYTES)) > 32768)
 			        nmbclusters = 32768;
 		}
+		/* Make sure it's not odd in case ncl is manually set */
+		if (nmbclusters & 0x1)
+			--nmbclusters;
 	}
 
 	return (nmbclusters * MCLBYTES);
diff --git a/bsd/dev/vn/Makefile b/bsd/dev/vn/Makefile
index 58a7c8500..313fb2a7b 100644
--- a/bsd/dev/vn/Makefile
+++ b/bsd/dev/vn/Makefile
@@ -3,6 +3,7 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+CFLAGS+=$(WERROR)
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
diff --git a/bsd/dev/vn/shadow.c b/bsd/dev/vn/shadow.c
index cb5fbfd6d..52ed5dfac 100644
--- a/bsd/dev/vn/shadow.c
+++ b/bsd/dev/vn/shadow.c
@@ -1,24 +1,29 @@
-
 /*
- * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2001-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -560,9 +565,9 @@ shadow_map_t *
 shadow_map_create(off_t file_size, off_t shadow_size, 
 		  u_long band_size, u_long block_size)
 {
-    void *		block_bitmap = 0;
+    void *		block_bitmap = NULL;
     u_long		bitmap_size;
-    band_number_t *	bands = 0;
+    band_number_t *	bands = NULL;
     shadow_map_t *	map;
     u_long		n_bands = 0;
 
@@ -629,8 +634,8 @@ shadow_map_free(shadow_map_t * map)
 	my_free(map->block_bitmap);
     if (map->bands)
 	my_free(map->bands);
-    map->block_bitmap = 0;
-    map->bands = 0;
+    map->block_bitmap = NULL;
+    map->bands = NULL;
     my_free(map);
     return;
 }
diff --git a/bsd/dev/vn/shadow.h b/bsd/dev/vn/shadow.h
index b610fd828..48b2c6b7a 100644
--- a/bsd/dev/vn/shadow.h
+++ b/bsd/dev/vn/shadow.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __VN_SHADOW_H__
diff --git a/bsd/dev/vn/vn.c b/bsd/dev/vn/vn.c
index c0642c1d0..4ac024c27 100644
--- a/bsd/dev/vn/vn.c
+++ b/bsd/dev/vn/vn.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -114,6 +120,8 @@
 
 
 #include "shadow.h"
+static void 
+vndevice_do_init(void) __attribute__((section("__TEXT, initcode")));
 
 static ioctl_fcn_t		vnioctl_chr;
 static ioctl_fcn_t		vnioctl_blk;
@@ -151,7 +159,7 @@ static struct cdevsw vn_cdevsw = {
 	/* ioctl */	vnioctl_chr,
 	/* stop */	eno_stop,
 	/* reset */	eno_reset,
-	/* ttys */	0,
+	/* ttys */	NULL,
 	/* select */	eno_select,
 	/* mmap */	eno_mmap,
 	/* strategy */	eno_strat,
@@ -171,7 +179,7 @@ struct vn_softc {
 	struct vnode	*sc_shadow_vp;	/* shadow vnode if not NULL	*/
 	uint32_t	sc_shadow_vid;
 	shadow_map_t *	sc_shadow_map;	/* shadow map if not NULL	*/
-	kauth_cred_t sc_cred;	/* credentials 			*/
+	kauth_cred_t	sc_cred;	/* credentials 			*/
 	u_int32_t	sc_options;	/* options 			*/
 	void *		sc_bdev;
 	void *		sc_cdev;
@@ -188,9 +196,8 @@ static u_int32_t	vn_options;
 #define IFOPT(vn,opt) if (((vn)->sc_options|vn_options) & (opt))
 #define TESTOPT(vn,opt) (((vn)->sc_options|vn_options) & (opt))
 
-static int	setcred(struct vnode * vp, struct proc * p, 
-			kauth_cred_t cred);
-static void	vnclear (struct vn_softc *vn, struct proc * p);
+static int	setcred(struct vnode * vp, kauth_cred_t cred);
+static void	vnclear (struct vn_softc *vn, vfs_context_t  ctx);
 static void vn_ioctl_to_64(struct vn_ioctl *from, struct user_vn_ioctl *to);
 void vndevice_init(void);
 int vndevice_root_image(char * path, char devname[], dev_t * dev_p);
@@ -200,13 +207,13 @@ vniocattach_file(struct vn_softc *vn,
 		 struct user_vn_ioctl *vniop,
 		 dev_t dev,
 		 int in_kernel,
-		 struct proc *p);
+		 proc_t p);
 static int
 vniocattach_shadow(struct vn_softc * vn,
 		   struct user_vn_ioctl *vniop,
 		   dev_t dev,
 		   int in_kernel,
-		   struct proc *p);
+		   proc_t p);
 static __inline__ int
 vnunit(dev_t dev)
 {
@@ -215,13 +222,13 @@ vnunit(dev_t dev)
 
 static	int
 vnclose(__unused dev_t dev, __unused int flags, 
-		__unused int devtype, __unused struct proc *p)
+		__unused int devtype, __unused proc_t p)
 {
 	return (0);
 }
 
 static	int
-vnopen(dev_t dev, int flags, __unused int devtype, __unused struct proc *p)
+vnopen(dev_t dev, int flags, __unused int devtype, __unused proc_t p)
 {
 	struct vn_softc *vn;
 	int unit;
@@ -238,7 +245,7 @@ vnopen(dev_t dev, int flags, __unused int devtype, __unused struct proc *p)
 }
 
 static int
-file_io(struct vnode * vp, struct vfs_context * context_p, 
+file_io(struct vnode * vp, vfs_context_t ctx, 
 	enum uio_rw op, char * base, off_t offset, user_ssize_t count,
 	user_ssize_t * resid)
 {
@@ -250,9 +257,9 @@ file_io(struct vnode * vp, struct vfs_context * context_p,
 				    &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, CAST_USER_ADDR_T(base), count);
 	if (op == UIO_READ)
-		error = VNOP_READ(vp, auio, IO_SYNC, context_p);
+		error = VNOP_READ(vp, auio, IO_SYNC, ctx);
 	else
-		error = VNOP_WRITE(vp, auio, IO_SYNC, context_p);
+		error = VNOP_WRITE(vp, auio, IO_SYNC, ctx);
 
 	if (resid != NULL) {
 		*resid = uio_resid(auio);
@@ -280,7 +287,7 @@ block_remainder(off_t o, int blocksize)
 
 static int
 vnread_shadow(struct vn_softc * vn, struct uio *uio, int ioflag, 
-	      struct vfs_context * context_p)
+	      vfs_context_t ctx)
 {
 	u_long		blocksize = vn->sc_secsize;
 	int 		error = 0;
@@ -320,7 +327,7 @@ vnread_shadow(struct vn_softc * vn, struct uio *uio, int ioflag,
 			this_resid = resid;
 		}
 		uio_setresid(uio, this_resid);
-		error = VNOP_READ(vp, uio, ioflag, context_p);
+		error = VNOP_READ(vp, uio, ioflag, ctx);
 		if (error) {
 			break;
 		}
@@ -340,7 +347,7 @@ vnread_shadow(struct vn_softc * vn, struct uio *uio, int ioflag,
 }
 
 static int
-vncopy_block_to_shadow(struct vn_softc * vn, struct vfs_context * context_p,
+vncopy_block_to_shadow(struct vn_softc * vn, vfs_context_t ctx,
 		       u_long file_block, u_long shadow_block)
 {
 	int	error;
@@ -351,14 +358,14 @@ vncopy_block_to_shadow(struct vn_softc * vn, struct vfs_context * context_p,
 	    return (ENOMEM);
 	}
 	/* read one block from file at file_block offset */
-	error = file_io(vn->sc_vp, context_p, UIO_READ,
+	error = file_io(vn->sc_vp, ctx, UIO_READ,
 			tmpbuf, (off_t)file_block * vn->sc_secsize, 
 			vn->sc_secsize, NULL);
 	if (error) {
 		goto done;
 	}
 	/* write one block to shadow file at shadow_block offset */
-	error = file_io(vn->sc_shadow_vp, context_p, UIO_WRITE,
+	error = file_io(vn->sc_shadow_vp, ctx, UIO_WRITE,
 			tmpbuf, (off_t)shadow_block * vn->sc_secsize, 
 			vn->sc_secsize, NULL);
  done:
@@ -373,7 +380,7 @@ enum {
 
 static int
 vnwrite_shadow(struct vn_softc * vn, struct uio *uio, int ioflag, 
-	       struct vfs_context * context_p)
+	       vfs_context_t ctx)
 {
 	u_long		blocksize = vn->sc_secsize;
 	int 		error = 0;
@@ -421,8 +428,7 @@ vnwrite_shadow(struct vn_softc * vn, struct uio *uio, int ioflag,
 			off_t	size;
 			size = (off_t)shadow_map_shadow_size(vn->sc_shadow_map) 
 				* vn->sc_secsize;
-			vnode_setsize(vn->sc_shadow_vp, size, IO_SYNC, 
-				      context_p);
+			vnode_setsize(vn->sc_shadow_vp, size, IO_SYNC, ctx);
 #endif 0
 		}
 		/* write the blocks (or parts thereof) */
@@ -440,11 +446,10 @@ vnwrite_shadow(struct vn_softc * vn, struct uio *uio, int ioflag,
 					+ resid_block_count - 1;
 				d = shadow_block_number 
 					+ shadow_block_count - 1;
-				error = vncopy_block_to_shadow(vn, context_p,
-							       s, d);
+				error = vncopy_block_to_shadow(vn, ctx, s, d);
 				if (error) {
 					printf("vnwrite_shadow: failed to copy"
-					       " block %d to shadow block %d\n",
+					       " block %lu to shadow block %lu\n",
 					       s, d);
 					break;
 				}
@@ -453,18 +458,18 @@ vnwrite_shadow(struct vn_softc * vn, struct uio *uio, int ioflag,
 		uio_setresid(uio, this_resid);
 		if ((flags & FLAGS_FIRST_BLOCK_PARTIAL) != 0) {
 			/* copy the first block to the shadow */
-			error = vncopy_block_to_shadow(vn, context_p,
+			error = vncopy_block_to_shadow(vn, ctx,
 						       offset_block_number,
 						       shadow_block_number);
 			if (error) {
 				printf("vnwrite_shadow: failed to"
-				       " copy block %d to shadow block %d\n", 
+				       " copy block %lu to shadow block %lu\n", 
 				       offset_block_number, 
 				       shadow_block_number);
 				break;
 			}
 		}
-		error = VNOP_WRITE(vn->sc_shadow_vp, uio, ioflag, context_p);
+		error = VNOP_WRITE(vn->sc_shadow_vp, uio, ioflag, ctx);
 		if (error) {
 			break;
 		}
@@ -489,7 +494,7 @@ vnread(dev_t dev, struct uio *uio, int ioflag)
 	int 			error = 0;
 	boolean_t   		funnel_state;
 	off_t			offset;
-	struct proc *		p;
+	proc_t			p;
 	user_ssize_t		resid;
 	struct vn_softc *	vn;
 	int 			unit;
@@ -505,11 +510,15 @@ vnread(dev_t dev, struct uio *uio, int ioflag)
 		error = ENXIO;
 		goto done;
 	}
+
+	context.vc_thread = current_thread();
+	context.vc_ucred = vn->sc_cred;
+
 	error = vnode_getwithvid(vn->sc_vp, vn->sc_vid);
 	if (error != 0) {
 		/* the vnode is no longer available, abort */
 		error = ENXIO;
-		vnclear(vn, p);
+		vnclear(vn, &context);
 		goto done;
 	}
 
@@ -534,8 +543,6 @@ vnread(dev_t dev, struct uio *uio, int ioflag)
 		uio_setresid(uio, resid);
 	}
 
-	context.vc_proc = p;
-	context.vc_ucred = vn->sc_cred;
 	if (vn->sc_shadow_vp != NULL) {
 		error = vnode_getwithvid(vn->sc_shadow_vp,
 					 vn->sc_shadow_vid);
@@ -543,7 +550,7 @@ vnread(dev_t dev, struct uio *uio, int ioflag)
 			/* the vnode is no longer available, abort */
 			error = ENXIO;
 			vnode_put(vn->sc_vp);
-			vnclear(vn, p);
+			vnclear(vn, &context);
 			goto done;
 		}
 		error = vnread_shadow(vn, uio, ioflag, &context);
@@ -564,7 +571,7 @@ vnwrite(dev_t dev, struct uio *uio, int ioflag)
 	int 			error;
 	boolean_t   		funnel_state;
 	off_t			offset;
-	struct proc *		p;
+	proc_t			p;
 	user_ssize_t		resid;
 	struct vn_softc *	vn;
 	int 			unit;
@@ -584,11 +591,15 @@ vnwrite(dev_t dev, struct uio *uio, int ioflag)
 		error = EROFS;
 		goto done;
 	}
+
+	context.vc_thread = current_thread();
+	context.vc_ucred = vn->sc_cred;
+
 	error = vnode_getwithvid(vn->sc_vp, vn->sc_vid);
 	if (error != 0) {
 		/* the vnode is no longer available, abort */
 		error = ENXIO;
-		vnclear(vn, p);
+		vnclear(vn, &context);
 		goto done;
 	}
 	resid = uio_resid(uio);
@@ -612,9 +623,6 @@ vnwrite(dev_t dev, struct uio *uio, int ioflag)
 		uio_setresid(uio, resid);
 	}
 
-	context.vc_proc = p;
-	context.vc_ucred = vn->sc_cred;
-
 	if (vn->sc_shadow_vp != NULL) {
 		error = vnode_getwithvid(vn->sc_shadow_vp,
 					 vn->sc_shadow_vid);
@@ -622,7 +630,7 @@ vnwrite(dev_t dev, struct uio *uio, int ioflag)
 			/* the vnode is no longer available, abort */
 			error = ENXIO;
 			vnode_put(vn->sc_vp);
-			vnclear(vn, p);
+			vnclear(vn, &context);
 			goto done;
 		}
 		error = vnwrite_shadow(vn, uio, ioflag, &context);
@@ -637,18 +645,16 @@ vnwrite(dev_t dev, struct uio *uio, int ioflag)
 }
 
 static int
-shadow_read(struct vn_softc * vn, struct buf * bp, char * base, struct proc * p)
+shadow_read(struct vn_softc * vn, struct buf * bp, char * base,
+	vfs_context_t ctx)
 {
 	u_long		blocksize = vn->sc_secsize;
-	struct vfs_context  context; 
 	int 		error = 0;
 	u_long		offset;
 	boolean_t	read_shadow;
 	u_long		resid;
 	u_long		start = 0;
 
-	context.vc_proc = p;
-	context.vc_ucred = vn->sc_cred;
 	offset = buf_blkno(bp);
 	resid =  buf_resid(bp) / blocksize;
 	while (resid > 0) {
@@ -666,7 +672,7 @@ shadow_read(struct vn_softc * vn, struct buf * bp, char * base, struct proc * p)
 		else {
 			vp = vn->sc_vp;
 		}
-		error = file_io(vp, &context, UIO_READ, base + start,
+		error = file_io(vp, ctx, UIO_READ, base + start,
 				(off_t)this_offset * blocksize,
 				(user_ssize_t)this_resid * blocksize, 
 				&temp_resid);
@@ -688,18 +694,15 @@ shadow_read(struct vn_softc * vn, struct buf * bp, char * base, struct proc * p)
 
 static int
 shadow_write(struct vn_softc * vn, struct buf * bp, char * base, 
-	     struct proc * p)
+	     vfs_context_t ctx)
 {
 	u_long		blocksize = vn->sc_secsize;
-	struct vfs_context  context; 
 	int 		error = 0;
 	u_long		offset;
 	boolean_t	shadow_grew;
 	u_long		resid;
 	u_long		start = 0;
 
-	context.vc_proc = p;
-	context.vc_ucred = vn->sc_cred;
 	offset = buf_blkno(bp);
 	resid =  buf_resid(bp) / blocksize;
 	while (resid > 0) {
@@ -716,11 +719,10 @@ shadow_write(struct vn_softc * vn, struct buf * bp, char * base,
 			/* truncate the file to its new length before write */
 			size = (off_t)shadow_map_shadow_size(vn->sc_shadow_map) 
 				* blocksize;
-			vnode_setsize(vn->sc_shadow_vp, size, IO_SYNC,
-				      &context);
+			vnode_setsize(vn->sc_shadow_vp, size, IO_SYNC, ctx);
 #endif
 		}
-		error = file_io(vn->sc_shadow_vp, &context, UIO_WRITE, 
+		error = file_io(vn->sc_shadow_vp, ctx, UIO_WRITE, 
 				base + start,
 				(off_t)this_offset * blocksize,
 				(user_ssize_t)this_resid * blocksize, 
@@ -742,25 +744,20 @@ shadow_write(struct vn_softc * vn, struct buf * bp, char * base,
 }
 
 static int
-vn_readwrite_io(struct vn_softc * vn, struct buf * bp, struct proc * p)
+vn_readwrite_io(struct vn_softc * vn, struct buf * bp, vfs_context_t ctx)
 {
 	int			error = 0;
 	char *			iov_base;
 	caddr_t 		vaddr;
-	
 
 	if (buf_map(bp, &vaddr)) 
 	        panic("vn device: buf_map failed");
 	iov_base = (char *)vaddr;
 
 	if (vn->sc_shadow_vp == NULL) {
-		struct vfs_context  	context; 
 	        user_ssize_t		temp_resid;
 
-		context.vc_proc = p;
-		context.vc_ucred = vn->sc_cred;
-
-		error = file_io(vn->sc_vp, &context,
+		error = file_io(vn->sc_vp, ctx,
 				buf_flags(bp) & B_READ ? UIO_READ : UIO_WRITE,
 				iov_base,
 				(off_t)buf_blkno(bp) * vn->sc_secsize,
@@ -769,9 +766,9 @@ vn_readwrite_io(struct vn_softc * vn, struct buf * bp, struct proc * p)
 	}
 	else {
 		if (buf_flags(bp) & B_READ)
-			error = shadow_read(vn, bp, iov_base, p);
+			error = shadow_read(vn, bp, iov_base, ctx);
 		else
-			error = shadow_write(vn, bp, iov_base, p);
+			error = shadow_write(vn, bp, iov_base, ctx);
 	}
 	buf_unmap(bp);
 
@@ -786,9 +783,9 @@ vnstrategy(struct buf *bp)
 	long sz;	/* in sc_secsize chunks */
 	daddr64_t blk_num;
 	boolean_t   		funnel_state;
-	struct proc * 		p = current_proc();
 	struct vnode *		shadow_vp = NULL;
 	struct vnode *		vp = NULL;
+	struct vfs_context  	context; 
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	vn = vn_table + vnunit(buf_device(bp));
@@ -797,6 +794,9 @@ vnstrategy(struct buf *bp)
 		goto done;
 	}
 
+	context.vc_thread = current_thread();
+	context.vc_ucred = vn->sc_cred;
+
 	buf_setresid(bp, buf_count(bp));
 	/*
 	 * Check for required alignment.  Transfers must be a valid
@@ -831,11 +831,12 @@ vnstrategy(struct buf *bp)
 		error = ENXIO;
 		goto done;
 	}
+
 	error = vnode_getwithvid(vp, vn->sc_vid);
 	if (error != 0) {
 		/* the vnode is no longer available, abort */
 		error = ENXIO;
-		vnclear(vn, p);
+		vnclear(vn, &context);
 		goto done;
 	}
 	shadow_vp = vn->sc_shadow_vp;
@@ -846,11 +847,12 @@ vnstrategy(struct buf *bp)
 			/* the vnode is no longer available, abort */
 			error = ENXIO;
 			vnode_put(vn->sc_vp);
-			vnclear(vn, p);
+			vnclear(vn, &context);
 			goto done;
 		}
 	}
-	error = vn_readwrite_io(vn, bp, p);
+
+	error = vn_readwrite_io(vn, bp, &context);
 	vnode_put(vp);
 	if (shadow_vp != NULL) {
 		vnode_put(shadow_vp);
@@ -868,7 +870,7 @@ vnstrategy(struct buf *bp)
 /* ARGSUSED */
 static	int
 vnioctl(dev_t dev, u_long cmd, caddr_t data,
-	__unused int flag, struct proc *p,
+	__unused int flag, proc_t p,
 	int is_char)
 {
 	struct vn_softc *vn;
@@ -880,6 +882,7 @@ vnioctl(dev_t dev, u_long cmd, caddr_t data,
 	struct vfsioattr ioattr;
 	struct user_vn_ioctl user_vnio;
 	boolean_t   		funnel_state;
+	struct vfs_context  	context; 
 
 	unit = vnunit(dev);
 	if (vnunit(dev) >= NVNDEVICE) {
@@ -893,6 +896,9 @@ vnioctl(dev_t dev, u_long cmd, caddr_t data,
 		goto done;
 	}
 
+	context.vc_thread = current_thread();
+	context.vc_ucred = vn->sc_cred;
+
 	viop = (struct user_vn_ioctl *)data;
 	f = (u_int32_t *)data;
 	o = (u_int64_t *)data;
@@ -1048,7 +1054,7 @@ vnioctl(dev_t dev, u_long cmd, caddr_t data,
 		 * How are these problems handled for removable and failing
 		 * hardware devices? (Hint: They are not)
 		 */
-		vnclear(vn, p);
+		vnclear(vn, &context);
 		break;
 
 	case VNIOCGSET:
@@ -1081,13 +1087,13 @@ vnioctl(dev_t dev, u_long cmd, caddr_t data,
 }
 
 static	int
-vnioctl_chr(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
+vnioctl_chr(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p)
 {
 	return (vnioctl(dev, cmd, data, flag, p, TRUE));
 }
 
 static	int
-vnioctl_blk(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
+vnioctl_blk(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p)
 {
 	return (vnioctl(dev, cmd, data, flag, p, FALSE));
 }
@@ -1104,47 +1110,42 @@ vniocattach_file(struct vn_softc *vn,
 		 struct user_vn_ioctl *vniop,
 		 dev_t dev,
 		 int in_kernel,
-		 struct proc *p)
+		 proc_t p)
 {
 	dev_t	cdev;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	kauth_cred_t cred;
 	struct nameidata nd;
 	off_t file_size;
 	int error, flags;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);
-	
 	flags = FREAD|FWRITE;
 	if (in_kernel) {
-		NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, vniop->vn_file, &context);
+		NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, vniop->vn_file, ctx);
 	}
 	else {
 		NDINIT(&nd, LOOKUP, FOLLOW, 
 			   (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32), 
-			   vniop->vn_file, &context);
+			   vniop->vn_file, ctx);
 	}
 	/* vn_open gives both long- and short-term references */
 	error = vn_open(&nd, flags, 0);
 	if (error) {
 		if (error != EACCES && error != EPERM && error != EROFS) {
-			kauth_cred_unref(&context.vc_ucred);
 			return (error);
 		}
 		flags &= ~FWRITE;
 		if (in_kernel) {
 			NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, 
-			       vniop->vn_file, &context);
+			       vniop->vn_file, ctx);
 		}
 		else {
 			NDINIT(&nd, LOOKUP, FOLLOW, 
 				   (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32), 
-			       vniop->vn_file, &context);
+			       vniop->vn_file, ctx);
 		}
 		error = vn_open(&nd, flags, 0);
 		if (error) {
-			kauth_cred_unref(&context.vc_ucred);
 			return (error);
 		}
 	}
@@ -1152,22 +1153,20 @@ vniocattach_file(struct vn_softc *vn,
 		error = EINVAL;
 	}
 	else {
-		error = vnode_size(nd.ni_vp, &file_size, &context);
+		error = vnode_size(nd.ni_vp, &file_size, ctx);
 	}
 	if (error != 0) {
-		(void) vn_close(nd.ni_vp, flags, context.vc_ucred, p);
+		(void) vn_close(nd.ni_vp, flags, ctx);
 		vnode_put(nd.ni_vp);
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	cred = kauth_cred_proc_ref(p);
 	nd.ni_vp->v_flag |= VNOCACHE_DATA;
-	error = setcred(nd.ni_vp, p, cred);
+	error = setcred(nd.ni_vp, cred);
 	if (error) {
-		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
+		(void)vn_close(nd.ni_vp, flags, ctx);
 		vnode_put(nd.ni_vp);
 		kauth_cred_unref(&cred);
-		kauth_cred_unref(&context.vc_ucred);
 		return(error);
 	}
 	vn->sc_secsize = DEV_BSIZE;
@@ -1187,53 +1186,46 @@ vniocattach_file(struct vn_softc *vn,
 		vn->sc_flags |= VNF_READONLY;
 	/* lose the short-term reference */
 	vnode_put(nd.ni_vp);
-	kauth_cred_unref(&context.vc_ucred);
 	return(0);
 }
 
 static int
 vniocattach_shadow(struct vn_softc *vn, struct user_vn_ioctl *vniop, 
-				   __unused int dev, int in_kernel, struct proc *p)
+				   __unused int dev, int in_kernel, proc_t p)
 {
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct nameidata nd;
 	int error, flags;
 	shadow_map_t *	map;
 	off_t file_size;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);
-	
 	flags = FREAD|FWRITE;
 	if (in_kernel) {
-		NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, vniop->vn_file, &context);
+		NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE32, vniop->vn_file, ctx);
 	}
 	else {
 		NDINIT(&nd, LOOKUP, FOLLOW, 
 			   (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32), 
-			   vniop->vn_file, &context);
+			   vniop->vn_file, ctx);
 	}
 	/* vn_open gives both long- and short-term references */
 	error = vn_open(&nd, flags, 0);
 	if (error) {
 		/* shadow MUST be writable! */
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	if (nd.ni_vp->v_type != VREG 
-	    || (error = vnode_size(nd.ni_vp, &file_size, &context))) {
-		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
+	    || (error = vnode_size(nd.ni_vp, &file_size, ctx))) {
+		(void)vn_close(nd.ni_vp, flags, ctx);
 		vnode_put(nd.ni_vp);
-		kauth_cred_unref(&context.vc_ucred);
 		return (error ? error : EINVAL);
 	}
 	map = shadow_map_create(vn->sc_fsize, file_size,
 				0, vn->sc_secsize);
 	if (map == NULL) {
-		(void)vn_close(nd.ni_vp, flags, context.vc_ucred, p);
+		(void)vn_close(nd.ni_vp, flags, ctx);
 		vnode_put(nd.ni_vp);
 		vn->sc_shadow_vp = NULL;
-		kauth_cred_unref(&context.vc_ucred);
 		return (ENOMEM);
 	}
 	vn->sc_shadow_vp = nd.ni_vp;
@@ -1244,7 +1236,6 @@ vniocattach_shadow(struct vn_softc *vn, struct user_vn_ioctl *vniop,
 
 	/* lose the short-term reference */
 	vnode_put(nd.ni_vp);
-	kauth_cred_unref(&context.vc_ucred);
 	return(0);
 }
 
@@ -1261,7 +1252,7 @@ vndevice_root_image(char * path, char devname[], dev_t * dev_p)
 	vn = vn_table + ROOT_IMAGE_UNIT;
 	*dev_p = makedev(vndevice_bdev_major, 
 			 ROOT_IMAGE_UNIT);
-	sprintf(devname, "vn%d", ROOT_IMAGE_UNIT);
+	snprintf(devname, 16, "vn%d", ROOT_IMAGE_UNIT);
 	error = vniocattach_file(vn, &vnio, *dev_p, 1, current_proc());
 	return (error);
 }
@@ -1273,7 +1264,7 @@ vndevice_root_image(char * path, char devname[], dev_t * dev_p)
  * if some other uid can write directly to the mapped file (NFS).
  */
 static int
-setcred(struct vnode * vp, struct proc * p, kauth_cred_t cred)
+setcred(struct vnode * vp, kauth_cred_t cred)
 {
 	char *tmpbuf;
 	int error = 0;
@@ -1282,7 +1273,7 @@ setcred(struct vnode * vp, struct proc * p, kauth_cred_t cred)
 	/*
 	 * Horrible kludge to establish credentials for NFS  XXX.
 	 */
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = cred;
 	tmpbuf = _MALLOC(DEV_BSIZE, M_TEMP, M_WAITOK);
 	error = file_io(vp, &context, UIO_READ, tmpbuf, 0, DEV_BSIZE, NULL);
@@ -1291,17 +1282,16 @@ setcred(struct vnode * vp, struct proc * p, kauth_cred_t cred)
 }
 
 void
-vnclear(struct vn_softc *vn, struct proc * p)
+vnclear(struct vn_softc *vn, vfs_context_t ctx)
 {
 	if (vn->sc_vp != NULL) {
 		/* release long-term reference */
-		(void)vn_close(vn->sc_vp, vn->sc_open_flags, vn->sc_cred, p);
+		(void)vn_close(vn->sc_vp, vn->sc_open_flags, ctx);
 		vn->sc_vp = NULL;
 	}
 	if (vn->sc_shadow_vp != NULL) {
 		/* release long-term reference */
-		(void)vn_close(vn->sc_shadow_vp, FREAD | FWRITE, 
-			       vn->sc_cred, p);
+		(void)vn_close(vn->sc_shadow_vp, FREAD | FWRITE, ctx);
 		vn->sc_shadow_vp = NULL;
 	}
 	if (vn->sc_shadow_map != NULL) {
@@ -1350,10 +1340,17 @@ static int vndevice_inited = 0;
 void 
 vndevice_init(void)
 {
-	int 	i;
-
 	if (vndevice_inited)
 		return;
+		
+	vndevice_do_init();
+}
+		
+static void 
+vndevice_do_init( void )
+{
+	int 	i;
+
 	vndevice_bdev_major = bdevsw_add(BDEV_MAJOR, &vn_bdevsw);
 
 	if (vndevice_bdev_major < 0) {
diff --git a/bsd/hfs/MacOSStubs.c b/bsd/hfs/MacOSStubs.c
index 1b0fee293..43459f1ae 100644
--- a/bsd/hfs/MacOSStubs.c
+++ b/bsd/hfs/MacOSStubs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -25,11 +31,12 @@
 #include <sys/malloc.h>
 #include <sys/types.h>
 
+#include <pexpert/pexpert.h>
+
 #include "hfs.h"
 #include "hfs_dbg.h"
 #include "hfscommon/headers/FileMgrInternal.h"
 
-
 /* 
  * gTimeZone should only be used for HFS volumes!
  * It is initialized when an HFS volume is mounted.
@@ -41,7 +48,7 @@ struct timezone gTimeZone = {8*60,1};
  *
  * called by the Catalog Manager when creating/updating HFS Plus records
  */
-UInt32 GetTimeUTC(void)
+u_int32_t GetTimeUTC(void)
 {
 	struct timeval tv;
 
@@ -55,9 +62,9 @@ UInt32 GetTimeUTC(void)
  * LocalToUTC - convert from Mac OS local time to Mac OS GMT time.
  * This should only be called for HFS volumes (not for HFS Plus).
  */
-UInt32 LocalToUTC(UInt32 localTime)
+u_int32_t LocalToUTC(u_int32_t localTime)
 {
-	UInt32 gtime = localTime;
+	u_int32_t gtime = localTime;
 	
 	if (gtime != 0) {
 		gtime += (gTimeZone.tz_minuteswest * 60);
@@ -76,9 +83,9 @@ UInt32 LocalToUTC(UInt32 localTime)
  * UTCToLocal - convert from Mac OS GMT time to Mac OS local time.
  * This should only be called for HFS volumes (not for HFS Plus).
  */
-UInt32 UTCToLocal(UInt32 utcTime)
+u_int32_t UTCToLocal(u_int32_t utcTime)
 {
-	UInt32 ltime = utcTime;
+	u_int32_t ltime = utcTime;
 	
 	if (ltime != 0) {
 		ltime -= (gTimeZone.tz_minuteswest * 60);
@@ -150,7 +157,14 @@ void DisposePtr (Ptr p)
 }
 
 
-void DebugStr (ConstStr255Param  debuggerMsg)
+void
+DebugStr(
+#if CONFIG_NO_KPRINTF_STRINGS
+	__unused ConstStr255Param debuggerMsg
+#else
+	ConstStr255Param debuggerMsg
+#endif
+	)
 {
     kprintf ("*** Mac OS Debugging Message: %s\n", &debuggerMsg[1]);
 	DEBUG_BREAK;
diff --git a/bsd/hfs/hfs.h b/bsd/hfs/hfs.h
index 6ca91334a..5568cf9c5 100644
--- a/bsd/hfs/hfs.h
+++ b/bsd/hfs/hfs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __HFS__
@@ -25,6 +31,10 @@
 
 #define HFS_SPARSE_DEV 1
 
+#ifdef DEBUG
+#define HFS_CHECK_LOCK_ORDER 1
+#endif
+
 #include <sys/appleapiopts.h>
 
 #ifdef KERNEL
@@ -57,9 +67,9 @@
 
 #define HFS_MAX_DEFERED_ALLOC	(1024*1024)
 
-// 400 mb is a "big" file (i.e. one that when deleted
+// 400 megs is a "big" file (i.e. one that when deleted
 // would touch enough data that we should break it into
-// multiple separate transactions
+// multiple separate transactions)
 #define HFS_BIGFILE_SIZE (400LL * 1024LL * 1024LL)
 
 
@@ -74,15 +84,6 @@ enum { kMDBOffset = kMasterDirectoryBlock * 512 };	/* MDB offset on disk in byte
 /* number of locked buffer caches to hold for b-tree meta data */
 #define kMaxLockedMetaBuffers		32		
 
-/*
- *	File type and creator for symbolic links
- */
-enum {
-	kSymLinkFileType  = 0x736C6E6B,	/* 'slnk' */
-	kSymLinkCreator   = 0x72686170	/* 'rhap' */
-};
-
-
 extern struct timezone gTimeZone;
 
 
@@ -118,13 +119,6 @@ extern struct timezone gTimeZone;
 
 /* Internal Data structures*/
 
-
-#define kHFS_DamagedVolume  0x1	/* This volume has errors, unmount dirty */
-
-/* XXX */
-#define MARK_VOLUMEDAMAGED(fcb) 
-
-
 /* This structure describes the HFS specific mount structure data. */
 typedef struct hfsmount {
 	u_int32_t     hfs_flags;              /* see below */
@@ -141,6 +135,13 @@ typedef struct hfsmount {
 	struct vnode *		hfs_catalog_vp;
 	struct vnode *		hfs_allocation_vp;
 	struct vnode *		hfs_attribute_vp;
+	struct vnode *		hfs_startup_vp;
+	struct vnode *		hfs_attrdata_vp;   /* pseudo file */
+	struct cnode *		hfs_extents_cp;
+	struct cnode *		hfs_catalog_cp;
+	struct cnode *		hfs_allocation_cp;
+	struct cnode *		hfs_attribute_cp;
+	struct cnode *		hfs_startup_cp;
 	dev_t			hfs_raw_dev;			/* device mounted */
 	u_int32_t		hfs_logBlockSize;		/* Size of buffer cache buffer for I/O */
 	
@@ -149,7 +150,7 @@ typedef struct hfsmount {
 	gid_t         hfs_gid;            /* gid to set as owner of the files */
 	mode_t        hfs_dir_mask;       /* mask to and with directory protection bits */
 	mode_t        hfs_file_mask;      /* mask to and with file protection bits */
-	u_long        hfs_encoding;       /* Defualt encoding for non hfs+ volumes */	
+	u_long        hfs_encoding;       /* Default encoding for non hfs+ volumes */	
 
 	/* Persistent fields (on disk, dynamic) */
 	time_t        hfs_mtime;          /* file system last modification time */
@@ -157,7 +158,7 @@ typedef struct hfsmount {
 	u_int32_t     hfs_dircount;       /* number of directories in file system */
 	u_int32_t     freeBlocks;	  /* free allocation blocks */
 	u_int32_t     nextAllocation;	  /* start of next allocation search */
-	u_int32_t     vcbNxtCNID;         /* next unused catalog node ID */
+	u_int32_t     vcbNxtCNID;         /* next unused catalog node ID - protected by catalog lock */
 	u_int32_t     vcbWrCnt;           /* file system write count */
 	u_int64_t     encodingsBitmap;    /* in-use encodings */
 	u_int16_t     vcbNmFls;           /* HFS Only - root dir file count */
@@ -165,13 +166,21 @@ typedef struct hfsmount {
 
 	/* Persistent fields (on disk, static) */
 	u_int16_t 			vcbSigWord;
-	int16_t				vcbFlags;
+	int16_t				vcbFlags; /* Runtime flag to indicate if volume is dirty/clean */
 	u_int32_t 			vcbAtrb;
 	u_int32_t 			vcbJinfoBlock;
 	time_t        hfs_itime;   /* file system creation time */
 	time_t        hfs_btime;   /* file system last backup time */
 	u_int32_t 			blockSize;	/* size of allocation blocks */
 	u_int32_t 			totalBlocks;	/* total allocation blocks */
+	u_int32_t			allocLimit;	/* Do not allocate this block or beyond */
+	/*
+	 * NOTE: When resizing a volume to make it smaller, allocLimit is set to the allocation
+	 * block number which will contain the new alternate volume header.  At all other times,
+	 * allocLimit is set to totalBlocks.  The allocation code uses allocLimit instead of
+	 * totalBlocks to limit which blocks may be allocated, so that during a resize, we don't
+	 * put new content into the blocks we're trying to truncate away.
+	 */
 	int32_t 			vcbClpSiz;
 	u_int32_t     vcbFndrInfo[8];
 	int16_t 			vcbVBMSt;		/* HFS only */
@@ -191,8 +200,15 @@ typedef struct hfsmount {
 	u_int32_t		loanedBlocks;		/* blocks on loan for delayed allocations */
 	
 	u_int32_t 			localCreateDate;	/* creation times for HFS+ volumes are in local time */
-	struct cat_desc		hfs_privdir_desc;
-	struct cat_attr		hfs_privdir_attr;
+
+	/*
+	 * HFS+ Private system directories (two). Any access
+	 * (besides looking at the cd_cnid) requires holding
+	 * the Catalog File lock.
+	 */
+	struct cat_desc     hfs_private_desc[2];
+	struct cat_attr     hfs_private_attr[2];
+
 	u_int32_t		hfs_metadata_createdate;
 	hfs_to_unicode_func_t	hfs_get_unicode;
 	unicode_to_hfs_func_t	hfs_get_hfsname;
@@ -241,6 +257,7 @@ typedef struct hfsmount {
 #ifdef HFS_SPARSE_DEV
 	/* Sparse device variables: */
 	struct vnode * hfs_backingfs_rootvp;
+	u_int32_t      hfs_last_backingstatfs;
 	int            hfs_sparsebandblks;
 #endif
 	size_t         hfs_max_inline_attrsize;
@@ -256,7 +273,7 @@ typedef struct hfsmount {
 
 typedef hfsmount_t  ExtendedVCB;
 
-/* Aliases for legacy field names */
+/* Aliases for legacy (Mac OS 9) field names */
 #define vcbCrDate          hfs_itime
 #define vcbLsMod           hfs_mtime
 #define vcbVolBkUp         hfs_btime
@@ -266,22 +283,72 @@ typedef hfsmount_t  ExtendedVCB;
 #define vcbFilCnt          hfs_filecount
 #define vcbDirCnt          hfs_dircount
 
+/* Inline functions to set/reset vcbFlags.  Upper 8 bits indicate if the volume 
+ * header/VCB is clean/dirty --- if set, volume header is dirty, and 
+ * if clear, volume header is clean.  This value is checked to determine
+ * if the in-memory copy of volume header should be flushed to the disk
+ * or not. 
+ */
+/* Set runtime flag to indicate that volume is dirty */
+static __inline__ void MarkVCBDirty(ExtendedVCB *vcb)
+{ 
+	vcb->vcbFlags |= 0xFF00;
+}
+
+/* Clear runtime flag to indicate that volume is dirty */
+static __inline__ void MarkVCBClean(ExtendedVCB *vcb)
+{
+	vcb->vcbFlags &= 0x00FF;
+}
+
+/* Check runtime flag to determine if the volume is dirty or not */
+static __inline__ Boolean IsVCBDirty(ExtendedVCB *vcb)
+{
+	return (vcb->vcbFlags & 0xFF00 ? true  : false);
+}
 
-/* HFS mount point flags */
-#define HFS_READ_ONLY             0x001
-#define HFS_UNKNOWN_PERMS         0x002
-#define HFS_WRITEABLE_MEDIA       0x004
-#define HFS_CLEANED_ORPHANS       0x008
-#define HFS_X                     0x010
-#define HFS_CASE_SENSITIVE        0x020
-#define HFS_STANDARD              0x040
-#define HFS_METADATA_ZONE         0x080
-#define HFS_FRAGMENTED_FREESPACE  0x100
-#define HFS_NEED_JNL_RESET        0x200
-#define HFS_HAS_SPARSE_DEVICE     0x400
-#define HFS_RESIZE_IN_PROGRESS    0x800
+/*
+ * There are two private directories in HFS+.
+ *
+ * One contains inodes for files that are hardlinked or open/unlinked.
+ * The other contains inodes for directories that are hardlinked.
+ */
+enum privdirtype {FILE_HARDLINKS, DIR_HARDLINKS};
 
 
+/* HFS mount point flags */
+#define HFS_READ_ONLY             0x00001
+#define HFS_UNKNOWN_PERMS         0x00002
+#define HFS_WRITEABLE_MEDIA       0x00004
+#define HFS_CLEANED_ORPHANS       0x00008
+#define HFS_X                     0x00010
+#define HFS_CASE_SENSITIVE        0x00020
+#define HFS_STANDARD              0x00040
+#define HFS_METADATA_ZONE         0x00080
+#define HFS_FRAGMENTED_FREESPACE  0x00100
+#define HFS_NEED_JNL_RESET        0x00200
+#define HFS_HAS_SPARSE_DEVICE     0x00400
+#define HFS_RESIZE_IN_PROGRESS    0x00800
+#define HFS_QUOTAS                0x01000
+#define HFS_CREATING_BTREE        0x02000
+/* When set, do not update nextAllocation in the mount structure */
+#define HFS_SKIP_UPDATE_NEXT_ALLOCATION 0x04000	
+#define HFS_XATTR_EXTENTS         0x08000	
+#define	HFS_FOLDERCOUNT           0x10000
+/* When set, the file system exists on a virtual device, like disk image */
+#define HFS_VIRTUAL_DEVICE        0x20000
+
+
+/* Macro to update next allocation block in the HFS mount structure.  If 
+ * the HFS_SKIP_UPDATE_NEXT_ALLOCATION is set, do not update 
+ * nextAllocation block.
+ */
+#define HFS_UPDATE_NEXT_ALLOCATION(hfsmp, new_nextAllocation)			\
+	{								\
+		if ((hfsmp->hfs_flags & HFS_SKIP_UPDATE_NEXT_ALLOCATION) == 0)\
+			hfsmp->nextAllocation = new_nextAllocation;	\
+	}								\
+
 #define HFS_MOUNT_LOCK(hfsmp, metadata)                      \
 	{                                                    \
 		if ((metadata) && 1)                         \
@@ -295,19 +362,42 @@ typedef hfsmount_t  ExtendedVCB;
 	}                                                    \
 
 #define hfs_global_exclusive_lock_acquire(hfsmp) lck_rw_lock_exclusive(&(hfsmp)->hfs_global_lock)
-#define hfs_global_exclusive_lock_release(hfsmp) lck_rw_done(&(hfsmp)->hfs_global_lock)
-
-
-#define MAXHFSVNODELEN		31
+#define hfs_global_exclusive_lock_release(hfsmp) lck_rw_unlock_exclusive(&(hfsmp)->hfs_global_lock)
 
+/* Macro for incrementing and decrementing the folder count in a cnode 
+ * attribute only if the HFS_FOLDERCOUNT bit is set in the mount flags 
+ * and kHFSHasFolderCount bit is set in the cnode flags.  Currently these 
+ * bits are only set for case sensitive HFS+ volumes.
+ */
+#define INC_FOLDERCOUNT(hfsmp, cattr) 				\
+	if ((hfsmp->hfs_flags & HFS_FOLDERCOUNT) &&		\
+	    (cattr.ca_recflags & kHFSHasFolderCountMask)) { 	\
+		cattr.ca_dircount++;				\
+	}							\
+
+#define DEC_FOLDERCOUNT(hfsmp, cattr) 				\
+	if ((hfsmp->hfs_flags & HFS_FOLDERCOUNT) &&		\
+	    (cattr.ca_recflags & kHFSHasFolderCountMask) && 	\
+	    (cattr.ca_dircount > 0)) { 				\
+		cattr.ca_dircount--;				\
+	}							\
 
 typedef struct filefork FCB;
 
+/*
+ * Macros for creating item names for our special/private directories.
+ */
+#define MAKE_INODE_NAME(name, size, linkno) \
+	    (void) snprintf((name), size, "%s%d", HFS_INODE_PREFIX, (linkno))
+#define HFS_INODE_PREFIX_LEN	5
 
-#define MAKE_INODE_NAME(name,linkno) \
-	    (void) sprintf((name), "%s%d", HFS_INODE_PREFIX, (linkno))
+#define MAKE_DIRINODE_NAME(name, size, linkno) \
+	    (void) snprintf((name), size, "%s%d", HFS_DIRINODE_PREFIX, (linkno))
+#define HFS_DIRINODE_PREFIX_LEN   4
 
-#define HFS_INODE_PREFIX_LEN	5
+#define MAKE_DELETED_NAME(NAME, size, FID) \
+	    (void) snprintf((NAME), size, "%s%d", HFS_DELETE_PREFIX, (FID))
+#define HFS_DELETE_PREFIX_LEN	4
 
 
 #define HFS_AVERAGE_NAME_SIZE	22
@@ -376,6 +466,7 @@ enum { kHFSPlusMaxFileNameBytes = kHFSPlusMaxFileNameChars * 3 };
 #define HFS_GET_LAST_MTIME  (FCNTL_FS_SPECIFIC_BASE + 0x00003)
 #define HFS_GET_BOOT_INFO   (FCNTL_FS_SPECIFIC_BASE + 0x00004)
 #define HFS_SET_BOOT_INFO   (FCNTL_FS_SPECIFIC_BASE + 0x00005)
+#define HFS_EXT_BULKACCESS  (FCNTL_FS_SPECIFIC_BASE + 0x00006)
 
 
 /*
@@ -386,119 +477,166 @@ enum { kHFSPlusMaxFileNameBytes = kHFSPlusMaxFileNameChars * 3 };
 #define MAC_GMT_FACTOR		2082844800UL
 
 
-time_t to_bsd_time(u_int32_t hfs_time);
-u_int32_t to_hfs_time(time_t bsd_time);
+/*****************************************************************************
+	FUNCTION PROTOTYPES 
+******************************************************************************/
 
-int hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush);
-#define HFS_ALTFLUSH	1
 
-extern int hfsUnmount(struct hfsmount *hfsmp, struct proc *p);
+/*****************************************************************************
+	hfs_vnop_xxx functions from different files 
+******************************************************************************/
+int hfs_vnop_readdirattr(struct vnop_readdirattr_args *);  /* in hfs_attrlist.c */
 
-extern int hfs_getnewvnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname *cnp,
-                           struct cat_desc *descp, int wantrsrc, struct cat_attr *attrp,
-                           struct cat_fork *forkp, struct vnode **vpp);
+int hfs_vnop_inactive(struct vnop_inactive_args *);        /* in hfs_cnode.c */
+int hfs_vnop_reclaim(struct vnop_reclaim_args *);          /* in hfs_cnode.c */
 
-extern u_int32_t hfs_freeblks(struct hfsmount * hfsmp, int wantreserve);
+int hfs_vnop_link(struct vnop_link_args *);                /* in hfs_link.c */
 
-extern void hfs_remove_orphans(struct hfsmount *);
+int hfs_vnop_lookup(struct vnop_lookup_args *);            /* in hfs_lookup.c */
 
+int hfs_vnop_search(struct vnop_searchfs_args *);          /* in hfs_search.c */
 
-short MacToVFSError(OSErr err);
+int hfs_vnop_read(struct vnop_read_args *);           /* in hfs_readwrite.c */
+int hfs_vnop_write(struct vnop_write_args *);         /* in hfs_readwrite.c */
+int hfs_vnop_ioctl(struct vnop_ioctl_args *);         /* in hfs_readwrite.c */
+int hfs_vnop_select(struct vnop_select_args *);       /* in hfs_readwrite.c */
+int hfs_vnop_strategy(struct vnop_strategy_args *);   /* in hfs_readwrite.c */
+int hfs_vnop_allocate(struct vnop_allocate_args *);   /* in hfs_readwrite.c */
+int hfs_vnop_pagein(struct vnop_pagein_args *);       /* in hfs_readwrite.c */
+int hfs_vnop_pageout(struct vnop_pageout_args *);     /* in hfs_readwrite.c */
+int hfs_vnop_bwrite(struct vnop_bwrite_args *);       /* in hfs_readwrite.c */
+int hfs_vnop_blktooff(struct vnop_blktooff_args *);   /* in hfs_readwrite.c */
+int hfs_vnop_offtoblk(struct vnop_offtoblk_args *);   /* in hfs_readwrite.c */
+int hfs_vnop_blockmap(struct vnop_blockmap_args *);   /* in hfs_readwrite.c */
 
-extern int hfs_owner_rights(struct hfsmount *hfsmp, uid_t cnode_uid, struct ucred *cred,
-		struct proc *p, int invokesuperuserstatus);
+int hfs_vnop_getxattr(struct vnop_getxattr_args *);        /* in hfs_xattr.c */
+int hfs_vnop_setxattr(struct vnop_setxattr_args *);        /* in hfs_xattr.c */
+int hfs_vnop_removexattr(struct vnop_removexattr_args *);  /* in hfs_xattr.c */
+int hfs_vnop_listxattr(struct vnop_listxattr_args *);      /* in hfs_xattr.c */
+#if NAMEDSTREAMS
+extern int hfs_vnop_getnamedstream(struct vnop_getnamedstream_args*);
+extern int hfs_vnop_makenamedstream(struct vnop_makenamedstream_args*);
+extern int hfs_vnop_removenamedstream(struct vnop_removenamedstream_args*);
+#endif
 
-u_long FindMetaDataDirectory(ExtendedVCB *vcb);
 
-#define  kMaxSecsForFsync	5
-#define  HFS_SYNCTRANS		1
+/*****************************************************************************
+	Functions from MacOSStubs.c
+******************************************************************************/
+time_t to_bsd_time(u_int32_t hfs_time);
 
-extern int hfs_btsync(struct vnode *vp, int sync_transaction);
-// used as a callback by the journaling code
-extern void hfs_sync_metadata(void *arg);
+u_int32_t to_hfs_time(time_t bsd_time);
 
-short make_dir_entry(FCB **fileptr, char *name, u_int32_t fileID);
 
+/*****************************************************************************
+	Functions from hfs_encodinghint.c
+******************************************************************************/
+u_int32_t hfs_pickencoding(const u_int16_t *src, int len);
 
-unsigned long BestBlockSizeFit(unsigned long allocationBlockSize,
-                               unsigned long blockSizeLimit,
-                               unsigned long baseMultiple);
+u_int32_t hfs_getencodingbias(void);
 
-OSErr	hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
-		struct proc *p);
-OSErr	hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
-		off_t embeddedOffset, u_int64_t disksize, struct proc *p, void *args, kauth_cred_t cred);
+void hfs_setencodingbias(u_int32_t bias);
 
-extern int     hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
-							   void *_args, off_t embeddedOffset, daddr64_t mdb_offset,
-							   HFSMasterDirectoryBlock *mdbp, struct ucred *cred);
-extern u_long  GetFileInfo(ExtendedVCB *vcb, u_int32_t dirid, const char *name,
-						   struct cat_attr *fattr, struct cat_fork *forkinfo);
+
+/*****************************************************************************
+	Functions from hfs_encodings.c
+******************************************************************************/
+void hfs_converterinit(void);
 
 int hfs_getconverter(u_int32_t encoding, hfs_to_unicode_func_t *get_unicode,
 		     unicode_to_hfs_func_t *get_hfsname);
 
 int hfs_relconverter(u_int32_t encoding);
 
-int hfs_to_utf8(ExtendedVCB *vcb, Str31 hfs_str, ByteCount maxDstLen,
+int hfs_to_utf8(ExtendedVCB *vcb, const Str31 hfs_str, ByteCount maxDstLen,
 		ByteCount *actualDstLen, unsigned char* dstStr);
 
 int utf8_to_hfs(ExtendedVCB *vcb, ByteCount srcLen, const unsigned char* srcStr,
 		Str31 dstStr);
 
-int mac_roman_to_utf8(Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen,
+int mac_roman_to_utf8(const Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen,
 		unsigned char* dstStr);
 
 int utf8_to_mac_roman(ByteCount srcLen, const unsigned char* srcStr, Str31 dstStr);
 
-u_int32_t hfs_pickencoding(const u_int16_t *src, int len);
+int mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str, u_int32_t maxCharLen, u_int32_t *usedCharLen);
 
-enum volop {VOL_UPDATE, VOL_MKDIR, VOL_RMDIR, VOL_MKFILE, VOL_RMFILE};
+int unicode_to_hfs(ExtendedVCB *vcb, ByteCount srcLen, u_int16_t* srcStr, Str31 dstStr, int retry);
 
-extern int hfs_volupdate(struct hfsmount *hfsmp, enum volop op, int inroot);
 
-extern void hfs_setencodingbits(struct hfsmount *hfsmp, u_int32_t encoding);
+/*****************************************************************************
+	Functions from hfs_notifications.c
+******************************************************************************/
+void hfs_generate_volume_notifications(struct hfsmount *hfsmp);
 
 
-extern void replace_desc(struct cnode *cp, struct cat_desc *cdp);
+/*****************************************************************************
+	Functions from hfs_readwrite.c
+******************************************************************************/
+extern int  hfs_relocate(struct  vnode *, u_int32_t, kauth_cred_t, struct  proc *);
 
-extern int hfs_namecmp(const char *, size_t, const char *, size_t);
+extern int hfs_truncate(struct vnode *, off_t, int, int, vfs_context_t);
 
-extern int  hfs_virtualmetafile(struct cnode *);
+extern int hfs_bmap(struct vnode *, daddr_t, struct vnode **, daddr64_t *, unsigned int *);
 
-void hfs_generate_volume_notifications(struct hfsmount *hfsmp);
+extern int hfs_fsync(struct vnode *, int, int, struct proc *);
 
-__private_extern__ u_int32_t hfs_getencodingbias(void);
-__private_extern__ void hfs_setencodingbias(u_int32_t bias);
+extern int hfs_access(struct vnode *, mode_t, kauth_cred_t, struct proc *);
 
-extern int hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp,
-			struct vnode **rvpp, struct proc *p);
+extern int hfs_removeallattr(struct hfsmount *hfsmp, u_int32_t fileid);
 
-extern int hfs_update(struct vnode *, int);
+extern int hfs_set_volxattr(struct hfsmount *hfsmp, unsigned int xattrtype, int state);
 
-extern int hfs_truncate(struct vnode *, off_t, int, int, vfs_context_t);
+extern void hfs_check_volxattr(struct hfsmount *hfsmp, unsigned int xattrtype);
 
-extern int hfs_fsync(struct vnode *, int, int, struct proc *);
+extern int  hfs_isallocated(struct hfsmount *, u_long, u_long);
 
-extern int hfs_access(struct vnode *, mode_t, struct ucred *, struct proc *);
 
-extern int hfs_vget(struct hfsmount *, cnid_t, struct vnode **, int);
+/*****************************************************************************
+	Functions from hfs_vfsops.c
+******************************************************************************/
+int hfs_mountroot(mount_t mp, vnode_t rvp, vfs_context_t context);
 
-extern int hfs_bmap(struct vnode *, daddr_t, struct vnode **, daddr64_t *, int *);
+/* used as a callback by the journaling code */
+extern void hfs_sync_metadata(void *arg);
 
-extern int hfs_removeallattr(struct hfsmount *hfsmp, u_int32_t fileid);
+extern int hfs_vget(struct hfsmount *, cnid_t, struct vnode **, int);
 
-__private_extern__ int hfs_start_transaction(struct hfsmount *hfsmp);
-__private_extern__ int hfs_end_transaction(struct hfsmount *hfsmp);
+extern void hfs_setencodingbits(struct hfsmount *hfsmp, u_int32_t encoding);
 
-extern int  hfs_setextendedsecurity(struct hfsmount *hfsmp, int state);
-extern void hfs_checkextendedsecurity(struct hfsmount *hfsmp);
+enum volop {VOL_UPDATE, VOL_MKDIR, VOL_RMDIR, VOL_MKFILE, VOL_RMFILE};
+extern int hfs_volupdate(struct hfsmount *hfsmp, enum volop op, int inroot);
+
+int hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush);
+#define HFS_ALTFLUSH	1
 
 extern int  hfs_extendfs(struct hfsmount *, u_int64_t, vfs_context_t);
 extern int  hfs_truncatefs(struct hfsmount *, u_int64_t, vfs_context_t);
 extern int  hfs_resize_progress(struct hfsmount *, u_int32_t *);
 
-extern int  hfs_isallocated(struct hfsmount *, u_long, u_long);
+/* If a runtime corruption is detected, mark the volume inconsistent 
+ * bit in the volume attributes.
+ */
+void hfs_mark_volume_inconsistent(struct hfsmount *hfsmp);
+
+/*****************************************************************************
+	Functions from hfs_vfsutils.c
+******************************************************************************/
+unsigned long BestBlockSizeFit(unsigned long allocationBlockSize,
+                               unsigned long blockSizeLimit,
+                               unsigned long baseMultiple);
+
+OSErr	hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
+		struct proc *p);
+OSErr	hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
+		off_t embeddedOffset, u_int64_t disksize, struct proc *p, void *args, kauth_cred_t cred);
+
+extern int hfsUnmount(struct hfsmount *hfsmp, struct proc *p);
+
+extern int overflow_extents(struct filefork *fp);
+
+extern int hfs_owner_rights(struct hfsmount *hfsmp, uid_t cnode_uid, kauth_cred_t cred,
+		struct proc *p, int invokesuperuserstatus);
 
 
 /* HFS System file locking */
@@ -506,11 +644,90 @@ extern int  hfs_isallocated(struct hfsmount *, u_long, u_long);
 #define SFL_EXTENTS     0x0002
 #define SFL_BITMAP      0x0004
 #define SFL_ATTRIBUTE   0x0008
-#define SFL_VALIDMASK   (SFL_CATALOG | SFL_EXTENTS | SFL_BITMAP | SFL_ATTRIBUTE)
+#define SFL_STARTUP	0x0010
+#define SFL_VALIDMASK   (SFL_CATALOG | SFL_EXTENTS | SFL_BITMAP | SFL_ATTRIBUTE | SFL_STARTUP)
 
 extern int  hfs_systemfile_lock(struct hfsmount *, int, enum hfslocktype);
 extern void hfs_systemfile_unlock(struct hfsmount *, int);
 
+extern u_long  GetFileInfo(ExtendedVCB *vcb, u_int32_t dirid, const char *name,
+						   struct cat_attr *fattr, struct cat_fork *forkinfo);
+
+extern void hfs_remove_orphans(struct hfsmount *);
+
+u_int32_t GetLogicalBlockSize(struct vnode *vp);
+
+extern u_int32_t hfs_freeblks(struct hfsmount * hfsmp, int wantreserve);
+
+short MacToVFSError(OSErr err);
+
+/* HFS directory hint functions. */
+extern directoryhint_t * hfs_getdirhint(struct cnode *, int, int);
+extern void  hfs_reldirhint(struct cnode *, directoryhint_t *);
+extern void  hfs_reldirhints(struct cnode *, int);
+extern void  hfs_insertdirhint(struct cnode *, directoryhint_t *);
+
+extern int hfs_namecmp(const u_int8_t *str1, size_t len1, const u_int8_t *str2, size_t len2);
+
+extern int     hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
+			   void *_args, off_t embeddedOffset, daddr64_t mdb_offset,
+			   HFSMasterDirectoryBlock *mdbp, kauth_cred_t cred);
+
+extern int  hfs_virtualmetafile(struct cnode *);
+
+extern int hfs_start_transaction(struct hfsmount *hfsmp);
+extern int hfs_end_transaction(struct hfsmount *hfsmp);
+
+
+/*****************************************************************************
+	Functions from hfs_vnops.c
+******************************************************************************/
+int hfs_write_access(struct vnode *vp, kauth_cred_t cred, struct proc *p, Boolean considerFlags);
+
+int hfs_chmod(struct vnode *vp, int mode, kauth_cred_t cred, struct proc *p);
+
+int hfs_chown(struct vnode *vp, uid_t uid, gid_t gid, kauth_cred_t cred, struct proc *p);
+
+#define  kMaxSecsForFsync	5
+#define  HFS_SYNCTRANS		1
+extern int hfs_btsync(struct vnode *vp, int sync_transaction);
+
+extern void replace_desc(struct cnode *cp, struct cat_desc *cdp);
+
+extern int hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp,
+			struct vnode **rvpp, int can_drop_lock);
+
+extern int hfs_update(struct vnode *, int);
+
+
+/*****************************************************************************
+	Functions from hfs_xattr.c
+******************************************************************************/
+int  hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKey);
+int  hfs_buildattrkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key);
+void hfs_xattr_init(struct hfsmount * hfsmp);
+int file_attribute_exist(struct hfsmount *hfsmp, uint32_t fileID);
+
+
+
+/*****************************************************************************
+	Functions from hfs_link.c
+******************************************************************************/
+
+extern int  hfs_unlink(struct hfsmount *hfsmp, struct vnode *dvp, struct vnode *vp,
+                       struct componentname *cnp, int skip_reserve);
+extern int  hfs_lookuplink(struct hfsmount *hfsmp, cnid_t linkfileid,
+                           cnid_t *prevlinkid,  cnid_t *nextlinkid);
+extern void  hfs_privatedir_init(struct hfsmount *, enum privdirtype);
+
+extern void  hfs_savelinkorigin(cnode_t *cp, cnid_t parentcnid);
+extern void  hfs_relorigins(struct cnode *cp);
+extern void  hfs_relorigin(struct cnode *cp, cnid_t parentcnid);
+extern int   hfs_haslinkorigin(cnode_t *cp);
+extern cnid_t  hfs_currentparent(cnode_t *cp);
+extern cnid_t  hfs_currentcnid(cnode_t *cp);
+
+
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
 #endif /* __HFS__ */
diff --git a/bsd/hfs/hfs_attrlist.c b/bsd/hfs/hfs_attrlist.c
index 8e15541b6..a3e9c55b7 100644
--- a/bsd/hfs/hfs_attrlist.c
+++ b/bsd/hfs/hfs_attrlist.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -43,36 +49,12 @@
 #include "hfs_mount.h"
 #include "hfs_dbg.h"
 #include "hfs_attrlist.h"
-
-
-
-/* Routines that are shared by hfs_setattr: */
-extern int hfs_write_access(struct vnode *vp, kauth_cred_t cred,
-			struct proc *p, Boolean considerFlags);
-
-extern int hfs_chflags(struct vnode *vp, uint32_t flags, kauth_cred_t cred,
-			struct proc *p);
-
-extern int hfs_chmod(struct vnode *vp, int mode, kauth_cred_t cred,
-			struct proc *p);
-
-extern int hfs_chown(struct vnode *vp, uid_t uid, gid_t gid, kauth_cred_t cred,
-			struct proc *p);
-
-__private_extern__ int hfs_vnop_readdirattr(struct vnop_readdirattr_args *ap);
-
-__private_extern__ int hfs_vnop_setattrlist(struct vnop_setattrlist_args  *ap);
-
-__private_extern__ int hfs_vnop_getattrlist(struct vnop_getattrlist_args *ap);
+#include "hfs_btreeio.h"
 
 /* Packing routines: */
 
-
-static void packvolcommonattr(struct attrblock *abp, struct hfsmount *hfsmp,
-			struct vnode *vp, struct proc *p);
-
-static void packvolattr(struct attrblock *abp, struct hfsmount *hfsmp,
-			struct vnode *vp);
+static void packnameattr(struct attrblock *abp, struct vnode *vp,
+			const u_int8_t *name, int namelen);
 
 static void packcommonattr(struct attrblock *abp, struct hfsmount *hfsmp,
 			struct vnode *vp, struct cat_desc * cdp,
@@ -86,505 +68,7 @@ static void packdirattr(struct attrblock *abp, struct hfsmount *hfsmp,
 			struct vnode *vp, struct cat_desc * descp,
 			struct cat_attr * cattrp);
 
-
-#if 0
-static int unpackattrblk(struct attrblock *abp, struct vnode *vp);
-
-static void unpackcommonattr(struct attrblock *abp, struct vnode *vp);
-
-static int unpackvolattr(struct attrblock *abp, struct hfsmount *hfsmp,
-			struct vnode *root_vp);
-
-
-/*
- * Get a list of attributes.
- */
-__private_extern__
-int
-hfs_vnop_getattrlist(ap)
-	struct vnop_getattrlist_args /* {
-		struct vnode *a_vp;
-		struct attrlist *a_alist
-		struct uio *a_uio;
-		int a_options;
-		vfs_context_t a_context;
-	} */ *ap;
-{
-	struct vnode *vp = ap->a_vp;
-	struct cnode *cp;
-	struct hfsmount *hfsmp;
-	struct attrlist *alist = ap->a_alist;
-	proc_t p = vfs_context_proc(ap->a_context);
-	int fixedblocksize;
-	int attrblocksize;
-	int attrbufsize;
-	void *attrbufptr = NULL;
-	void *attrptr;
-	void *varptr;
-	struct attrblock attrblk;
-	struct cat_fork *datafp = NULL;
-	struct cat_fork *rsrcfp = NULL;
-	struct cat_fork rsrcfork;
-	int lockflags;
-	int error = 0;
-
-	if ((alist->bitmapcount != ATTR_BIT_MAP_COUNT) ||
-	    ((alist->commonattr & ~ATTR_CMN_VALIDMASK) != 0) ||
-	    ((alist->volattr & ~ATTR_VOL_VALIDMASK) != 0) ||
-	    ((alist->dirattr & ~ATTR_DIR_VALIDMASK) != 0) ||
-	    ((alist->fileattr & ~ATTR_FILE_VALIDMASK) != 0)) {
-		return (EINVAL);
-	}
-
-	/*
-	 * Requesting volume information requires setting the
-	 * ATTR_VOL_INFO bit. Also, volume info requests are
-	 * mutually exclusive with all other info requests.
-	 */
-	if ((alist->volattr != 0) &&
-	    (((alist->volattr & ATTR_VOL_INFO) == 0) ||
-	     (alist->dirattr != 0) || (alist->fileattr != 0))) {
-		return (EINVAL);
-	}
-
-	/* Reject requests for unsupported options for now: */
-	if ((alist->commonattr & (ATTR_CMN_NAMEDATTRCOUNT | ATTR_CMN_NAMEDATTRLIST)) ||
-	    (alist->fileattr & (ATTR_FILE_FILETYPE | ATTR_FILE_FORKCOUNT | ATTR_FILE_FORKLIST))) {
-		return (EINVAL);
-	}
-
-	if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK)))
-		return (error);
-	cp = VTOC(vp);
-	hfsmp = VTOHFS(vp);
-
-	/* Requesting volume information requires root vnode */ 
-	if ((alist->volattr) && cp->c_fileid != kHFSRootFolderID) {
-		error = EINVAL;
-		goto exit;
-	}
-	/* Asking for data fork attributes from the rsrc fork is not supported */
-	if (VNODE_IS_RSRC(vp) && (alist->fileattr & ATTR_DATAFORK_MASK)) {
-		error = EINVAL;
-		goto exit;
-	}
-	/* This file no longer exists! */
-	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-		error = ENOENT;
-		goto exit;
-	}
-	/* This file doesn't have a name! */
-	if ((cp->c_desc.cd_namelen == 0) && (alist->commonattr & ATTR_CMN_NAME)) {
-		error = ENOENT;
-		goto exit;
-	}
-
-	/* Update cnode times if needed */
-	hfs_touchtimes(hfsmp, cp);
-
-	/*
-	 * If a File ID (ATTR_CMN_OBJPERMANENTID) is requested on
-	 * an HFS volume we must be sure to create the thread
-	 * record before returning it. (yikes)
-	 */
-	if (vnode_isreg(vp) &&
-	    (alist->commonattr & ATTR_CMN_OBJPERMANENTID) &&
-	    (VTOVCB(vp)->vcbSigWord != kHFSPlusSigWord)) {
-
-	    	cat_cookie_t cookie;
-
-		if (hfsmp->hfs_flags & HFS_READ_ONLY) {
-			error = EROFS;
-			goto exit;
-		}
-		if ((error = hfs_write_access(vp, vfs_context_ucred(ap->a_context),
-		                              p, false)) != 0) {
-        		goto exit;
-        	}
-		/*
-		 * Reserve some space in the Catalog file.
-		 */
-		bzero(&cookie, sizeof(cookie));
-		error = cat_preflight(hfsmp, CAT_CREATE, &cookie, p);
-		if (error) {
-        		goto exit;
-        	}
-
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
-
-		error = cat_insertfilethread(hfsmp, &cp->c_desc);
-
-		hfs_systemfile_unlock(hfsmp, lockflags);
-
-		cat_postflight(hfsmp, &cookie, p);
-
-		if (error)
-			goto exit;
-	}
-	bzero(&rsrcfork, sizeof(rsrcfork));
-	/* Establish known fork data */
-	if (cp->c_datafork != NULL) {
-		datafp = &cp->c_datafork->ff_data;
-		if ((cp->c_rsrcfork == NULL) &&
-		    (cp->c_blocks == datafp->cf_blocks))
-			rsrcfp = &rsrcfork;	/* rsrc fork is empty */
-	}
-	if (cp->c_rsrcfork != NULL)
-		rsrcfp = &cp->c_rsrcfork->ff_data;
-	
-	/*
-	 * When resource fork data is requested and its not available
-	 * in the cnode and the fork is not empty then it needs to be
-	 * fetched from the catalog.
-	 */
-	if ((alist->fileattr & ATTR_RSRCFORK_MASK) && (rsrcfp == NULL)) {
-
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-
-		/* Get resource fork data */
-		error = cat_lookup(hfsmp, &cp->c_desc, 1,
-				(struct cat_desc *)0, (struct cat_attr *)0, &rsrcfork, NULL);
-
-		hfs_systemfile_unlock(hfsmp, lockflags);
-
-		if (error)
-			goto exit;
-
-		rsrcfp = &rsrcfork;
-	}
-
-	fixedblocksize = hfs_attrblksize(alist);
-	attrblocksize = fixedblocksize + (sizeof(uint32_t));  /* uint32_t for length word */
-	if (alist->commonattr & ATTR_CMN_NAME)
-		attrblocksize += kHFSPlusMaxFileNameBytes + 1;
-	if (alist->volattr & ATTR_VOL_MOUNTPOINT)
-		attrblocksize += PATH_MAX;
-	if (alist->volattr & ATTR_VOL_NAME)
-		attrblocksize += kHFSPlusMaxFileNameBytes + 1;
-#if 0
-	if (alist->commonattr & ATTR_CMN_NAMEDATTRLIST)
-		attrblocksize += 0;
-	if (alist->fileattr & ATTR_FILE_FORKLIST)
-		attrblocksize += 0;
-#endif
-	attrbufsize = MIN(uio_resid(ap->a_uio), attrblocksize);
-	MALLOC(attrbufptr, void *, attrblocksize, M_TEMP, M_WAITOK);
-	attrptr = attrbufptr;
-	*((uint32_t *)attrptr) = 0;  /* Set buffer length in case of errors */
-	++((uint32_t *)attrptr);     /* Reserve space for length field */
-	varptr = ((char *)attrptr) + fixedblocksize;
-
-	attrblk.ab_attrlist = alist;
-	attrblk.ab_attrbufpp = &attrptr;
-	attrblk.ab_varbufpp = &varptr;
-	attrblk.ab_flags = 0;
-	attrblk.ab_blocksize = attrblocksize;
-
-	hfs_packattrblk(&attrblk, hfsmp, vp, &cp->c_desc, &cp->c_attr,
-			datafp, rsrcfp, p);
-
-	/* Don't copy out more data than was generated */
-	attrbufsize = MIN((u_int)attrbufsize, (u_int)varptr - (u_int)attrbufptr);
-	 /* Set actual buffer length for return to caller */
-	*((uint32_t *)attrbufptr) = attrbufsize;
-	error = uiomove((caddr_t)attrbufptr, attrbufsize, ap->a_uio);
-exit:
-	if (attrbufptr)
-		FREE(attrbufptr, M_TEMP);
-	hfs_unlock(cp);
-	return (error);
-}
-
-
-/*
- * Set a list of attributes.
- */
-__private_extern__
-int
-hfs_vnop_setattrlist(ap)
-	struct vnop_setattrlist_args /* {
-		struct vnode *a_vp;
-		struct attrlist *a_alist
-		struct uio *a_uio;
-		int a_options;
-		vfs_context_t a_context;
-	} */ *ap;
-{
-	struct vnode *vp = ap->a_vp;
-	struct cnode *cp;
-	struct hfsmount * hfsmp;
-	struct attrlist *alist = ap->a_alist;
-	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
-	struct proc *p = vfs_context_proc(ap->a_context);
-	int attrblocksize;
-	void *attrbufptr = NULL;
-	void *attrptr;
-	void *varptr = NULL;
-	struct attrblock attrblk;
-	uid_t saved_uid;
-	gid_t saved_gid;
-	mode_t saved_mode;
-	uint32_t saved_flags;
-	int error = 0;
-
-	hfsmp = VTOHFS(vp);
-
-	if (hfsmp->hfs_flags & HFS_READ_ONLY)
-		return (EROFS);
-	if ((alist->bitmapcount != ATTR_BIT_MAP_COUNT)     ||
-	    ((alist->commonattr & ~ATTR_CMN_SETMASK) != 0) ||
-	    ((alist->volattr & ~ATTR_VOL_SETMASK) != 0)    ||
-	    ((alist->dirattr & ~ATTR_DIR_SETMASK) != 0)    ||
-	    ((alist->fileattr & ~ATTR_FILE_SETMASK) != 0)) {
-		return (EINVAL);
-	}
-	if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK)))
-		return (error);
-	cp = VTOC(vp);
-
-	/* 
-	 * When setting volume attributes make sure
-	 * that ATTR_VOL_INFO is set and that all
-	 * the attributes are valid.
-	 */
-	if ((alist->volattr != 0) &&
-	    (((alist->volattr & ATTR_VOL_INFO) == 0) ||
-	    (alist->commonattr & ~ATTR_CMN_VOLSETMASK) ||
-	    (cp->c_fileid != kHFSRootFolderID))) {
-	    	if ((alist->volattr & ATTR_VOL_INFO) == 0)
-			printf("hfs_setattrlist: you forgot to set ATTR_VOL_INFO bit!\n");
-		else
-			printf("hfs_setattrlist: you cannot set bits 0x%08X!\n",
-				alist->commonattr & ~ATTR_CMN_VOLSETMASK);
-		error = EINVAL;
-		goto ErrorExit;
-	}
-	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-		error = ENOENT;
-		goto ErrorExit;
-	}
-	// XXXdbg - don't allow modifying the journal or journal_info_block
-	if (hfsmp->jnl && cp->c_datafork) {
-		struct HFSPlusExtentDescriptor *extd;
-		
-		extd = &cp->c_datafork->ff_extents[0];
-		if (extd->startBlock == HFSTOVCB(hfsmp)->vcbJinfoBlock || extd->startBlock == hfsmp->jnl_start) {
-			error = EPERM;
-			goto ErrorExit;
-		}
-	}
-
-	/*
-	 * Ownership of a file is required in one of two classes of calls:
-	 *
-	 * (a) When setting any ownership-requiring attribute other
-	 *     than ATTR_CMN_FLAGS, or
-	 * (b) When setting ATTR_CMN_FLAGS on a volume that's not
-	 *     plain HFS (for which no real per-object ownership
-	 *     information is stored)
-	 */
-	if ((alist->commonattr & (ATTR_OWNERSHIP_SETMASK & ~ATTR_CMN_FLAGS)) ||
-	    ((alist->commonattr & ATTR_CMN_FLAGS) &&
-	     (VTOVCB(vp)->vcbSigWord != kHFSSigWord))) {
-		/*
-		 * NOTE: The following isn't ENTIRELY complete: even if
-		 * you're the superuser you cannot change the flags as
-		 * long as SF_IMMUTABLE or SF_APPEND is set and the
-		 * securelevel > 0.  This is verified in hfs_chflags
-		 * which gets invoked to do the actual flags field
-		 * change so this check is sufficient for now.
-		 */
-		if ((error = hfs_owner_rights(hfsmp, cp->c_uid, cred, p, true)) != 0)
-        		goto ErrorExit;
-	}
-	/*
-	 * For any other attributes, check to see if the user has
-	 * write access to the cnode in question [unlike vn_access,
-	 * ignore IMMUTABLE here]:
-	 */ 
-	if (((alist->commonattr & ~ATTR_OWNERSHIP_SETMASK) != 0) ||
-	    (alist->volattr != 0) || (alist->dirattr != 0) ||
-	    (alist->fileattr != 0)) {
-		if ((error = hfs_write_access(vp, cred, p, false)) != 0)
-        		goto ErrorExit;
-	}
-
-	/*
-	 * Allocate the buffer now to minimize the time we might
-	 * be blocked holding the catalog lock.
-	 */
-	// LP64todo - fix this
-	attrblocksize = uio_resid(ap->a_uio);
-	if (attrblocksize < hfs_attrblksize(alist)) {
-		error = EINVAL;
-		goto ErrorExit;
-	}
-
-	MALLOC(attrbufptr, void *, attrblocksize, M_TEMP, M_WAITOK);
-
-	error = uiomove((caddr_t)attrbufptr, attrblocksize, ap->a_uio);
-	if (error)
-		goto ErrorExit;
-
-	/* Save original state so changes can be detected. */
-	saved_uid = cp->c_uid;
-	saved_gid = cp->c_gid;
-	saved_mode = cp->c_mode;
-	saved_flags = cp->c_flags;
-
-	attrptr = attrbufptr;
-	attrblk.ab_attrlist = alist;
-	attrblk.ab_attrbufpp = &attrptr;
-	attrblk.ab_varbufpp = &varptr;
-	attrblk.ab_flags = 0;
-	attrblk.ab_blocksize = attrblocksize;
-	error = unpackattrblk(&attrblk, vp);
-	if (error)
-		goto ErrorExit;
-
-	/* If unpacking changed the owner/group then call hfs_chown() */
-	if ((saved_uid != cp->c_uid) || (saved_gid != cp->c_gid)) {
-		uid_t uid;
-		gid_t gid;
-		
-		uid = cp->c_uid;
- 		cp->c_uid = saved_uid;
-		gid = cp->c_gid;
-		cp->c_gid = saved_gid;
-		if ((error = hfs_chown(vp, uid, gid, cred, p)))
-			goto ErrorExit;
-	}
-	/* If unpacking changed the mode then call hfs_chmod() */
-	if (saved_mode != cp->c_mode) {
-		mode_t mode;
-
-		mode = cp->c_mode;
-		cp->c_mode = saved_mode;
-		if ((error = hfs_chmod(vp, mode, cred, p)))
-			goto ErrorExit;
-	}
-	/* If unpacking changed the flags then call hfs_chflags() */
-	if (saved_flags !=cp->c_flags) {
-		uint32_t flags;
-
-		flags = cp->c_flags;
-		cp->c_flags = saved_flags;
-		if ((error = hfs_chflags(vp, flags, cred, p)))
-			goto ErrorExit;
-	}
-	/*
-	 * If any cnode attributes changed then do an update.
-	 */
-	if (alist->volattr == 0) {
-		cp->c_flag |= C_MODIFIED;
-		if ((error = hfs_update(vp, TRUE))) {
-			goto ErrorExit;
-		}
-	}
-	/* Volume Rename */
-	if (alist->volattr & ATTR_VOL_NAME) {
-		ExtendedVCB *vcb = VTOVCB(vp);
-	
-		if (vcb->vcbVN[0] == 0) {
-			/*
-			 * Ignore attempts to rename a volume to a zero-length name:
-			 * restore the original name from the cnode.
-			 */
-			copystr(cp->c_desc.cd_nameptr, vcb->vcbVN, sizeof(vcb->vcbVN), NULL);
-		} else {
-			struct cat_desc to_desc;
-			struct cat_desc todir_desc;
-			struct cat_desc new_desc;
-			cat_cookie_t cookie;
-			int catreserve = 0;
-			int catlocked = 0;
-			int started_tr = 0;
-			int lockflags;
-
-			bzero(&to_desc, sizeof(to_desc));
-			bzero(&todir_desc, sizeof(todir_desc));
-			bzero(&new_desc, sizeof(new_desc));
-			bzero(&cookie, sizeof(cookie));
-
-			todir_desc.cd_parentcnid = kHFSRootParentID;
-			todir_desc.cd_cnid = kHFSRootFolderID;
-			todir_desc.cd_flags = CD_ISDIR;
-
-			to_desc.cd_nameptr = vcb->vcbVN;
-			to_desc.cd_namelen = strlen(vcb->vcbVN);
-			to_desc.cd_parentcnid = kHFSRootParentID;
-			to_desc.cd_cnid = cp->c_cnid;
-			to_desc.cd_flags = CD_ISDIR;
-
-			if ((error = hfs_start_transaction(hfsmp) != 0)) {
-			    goto rename_out;
-			}
-			started_tr = 1;
-
-			/*
-			 * Reserve some space in the Catalog file.
-			 */
-			error = cat_preflight(hfsmp, CAT_RENAME, &cookie, p);
-			if (error) {
-				goto rename_out;
-			}
-			catreserve = 1;
-
-			lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
-			catlocked = 1;
-
-			error = cat_rename(hfsmp, &cp->c_desc, &todir_desc, &to_desc, &new_desc);
-rename_out:			
-			if (catlocked) {
-				hfs_systemfile_unlock(hfsmp, lockflags);
-			}
-			if (catreserve) {
-				cat_postflight(hfsmp, &cookie, p);
-			}
-			(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
-			if (started_tr) {
-			    hfs_end_transaction(hfsmp);
-			}
-			
-			if (error) {
-				/* Restore the old name in the VCB */
-				copystr(cp->c_desc.cd_nameptr, vcb->vcbVN, sizeof(vcb->vcbVN), NULL);
-				vcb->vcbFlags |= 0xFF00;
-				goto ErrorExit;
-			}
-			/* Release old allocated name buffer */
-			if (cp->c_desc.cd_flags & CD_HASBUF) {
-				char *name = cp->c_desc.cd_nameptr;
-		
-				cp->c_desc.cd_nameptr = 0;
-				cp->c_desc.cd_namelen = 0;
-				cp->c_desc.cd_flags &= ~CD_HASBUF;
-				vfs_removename(name);
-			}			
-			/* Update cnode's catalog descriptor */
-			replace_desc(cp, &new_desc);
-			vcb->volumeNameEncodingHint = new_desc.cd_encoding;
-			cp->c_touch_chgtime = TRUE;
-		}
-	}
-
-	/*
-	 * When the volume name changes or the volume's finder info
-	 * changes then force them to disk immediately.
-	 */
-	if ((alist->volattr & ATTR_VOL_INFO) &&
-	    ((alist->volattr & ATTR_VOL_NAME) ||
-	     (alist->commonattr & ATTR_CMN_FNDRINFO))) {
-		(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
-	}
-ErrorExit:
-	if (attrbufptr)
-		FREE(attrbufptr, M_TEMP);
-
-	hfs_unlock(cp);
-	return (error);
-}
-#endif
+static u_int32_t hfs_real_user_access(vnode_t vp, vfs_context_t ctx);
 
 /*
  * readdirattr operation will return attributes for the items in the
@@ -595,30 +79,7 @@ ErrorExit:
  * different (ufs) file system. The attributes that apply for it may not 
  * apply for the file system you are doing the readdirattr on. To make life 
  * simpler, this call will only return entries in its directory, hfs like.
- * TO DO LATER: 
- * 1. more than one for uiovcnt support.
- * 2. put knohint (hints) in state for next call in
- * 3. credentials checking when rest of hfs does it.
- * 4. Do return permissions concatenation ???
  */
-
-/* 			
-#
-#% readdirattr	vp	L L L
-#
-vnop_readdirattr {
-	IN struct vnode *vp;
-	IN struct attrlist *alist;
-	INOUT struct uio *uio;
-	IN u_long maxcount:
-	IN u_long options;
-	OUT u_long *newstate;
-	OUT int *eofflag;
-	OUT u_long *actualCount;
-	OUT u_long **cookies;
-	IN kauth_cred_t cred;
-};
-*/
 __private_extern__
 int
 hfs_vnop_readdirattr(ap)
@@ -641,70 +102,64 @@ hfs_vnop_readdirattr(ap)
 	uio_t uio = ap->a_uio;
 	int maxcount = ap->a_maxcount;
 	struct proc *p = vfs_context_proc(ap->a_context);
-	uint32_t fixedblocksize;
-	uint32_t maxattrblocksize;
-	uint32_t currattrbufsize;
+	u_int32_t fixedblocksize;
+	u_int32_t maxattrblocksize;
+	u_int32_t currattrbufsize;
 	void *attrbufptr = NULL;
 	void *attrptr;
 	void *varptr;
 	struct attrblock attrblk;
 	int error = 0;
-	int depleted = 0;
-	int index;
-	int i, dir_entries;
+	int index = 0;
+	int i, dir_entries = 0;
 	struct cat_desc *lastdescp = NULL;
 	struct cat_entrylist *ce_list = NULL;
 	directoryhint_t *dirhint = NULL;
 	unsigned int tag;
-	int shared_cnode_lock = 0;
+	int maxentries;
+	int lockflags;
 
 	*(ap->a_actualcount) = 0;
 	*(ap->a_eofflag) = 0;
 
 	/* Check for invalid options and buffer space. */
-	if (((ap->a_options & ~(FSOPT_NOINMEMUPDATE | FSOPT_NOFOLLOW)) != 0)
-	||  (uio_resid(uio) <= 0) || (uio_iovcnt(uio) > 1) || (maxcount <= 0))
+	if (((ap->a_options & ~(FSOPT_NOINMEMUPDATE | FSOPT_NOFOLLOW)) != 0) ||
+	    (uio_resid(uio) <= 0) || (uio_iovcnt(uio) > 1) || (maxcount <= 0)) {
 		return (EINVAL);
-
-	/* This call doesn't take volume attributes. */
+	}
+	/*
+	 * Reject requests for unsupported attributes.
+	 */
 	if ((alist->bitmapcount != ATTR_BIT_MAP_COUNT) ||
-	    ((alist->commonattr & ~ATTR_CMN_VALIDMASK) != 0) ||
+	    (alist->commonattr & ~HFS_ATTR_CMN_VALID) ||
 	    (alist->volattr  != 0) ||
-	    ((alist->dirattr & ~ATTR_DIR_VALIDMASK) != 0) ||
-	    ((alist->fileattr & ~ATTR_FILE_VALIDMASK) != 0)) 
+	    (alist->dirattr & ~HFS_ATTR_DIR_VALID) ||
+	    (alist->fileattr & ~HFS_ATTR_FILE_VALID) ||
+	    (alist->forkattr != 0)) {
 		return (EINVAL);
-
-	if ((error = hfs_lock(VTOC(dvp), HFS_EXCLUSIVE_LOCK)))
+	}
+	/*
+	 * Take an exclusive directory lock since we manipulate the directory hints
+	 */
+	if ((error = hfs_lock(VTOC(dvp), HFS_EXCLUSIVE_LOCK))) {
 		return (error);
+	}
 	dcp = VTOC(dvp);
 	hfsmp = VTOHFS(dvp);
 
-	/* Reject requests for unsupported options. */
-	if ((alist->commonattr & (ATTR_CMN_NAMEDATTRCOUNT | ATTR_CMN_NAMEDATTRLIST |
-	     ATTR_CMN_OBJPERMANENTID)) ||
-	    (alist->fileattr & (ATTR_FILE_FILETYPE | ATTR_FILE_FORKCOUNT |
-	     ATTR_FILE_FORKLIST | ATTR_FILE_DATAEXTENTS | ATTR_FILE_RSRCEXTENTS))) {
-		printf("readdirattr: unsupported attributes! (%s)\n", dcp->c_desc.cd_nameptr);
-		error = EINVAL;
-		goto exit;
-	}
-
 	dir_entries = dcp->c_entries;
-	if (dcp->c_attr.ca_fileid == kHFSRootFolderID && (hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))) {
-		dir_entries -= 3;
-	}
 
-	/* Convert uio_offset into a directory index. */
+	/* Extract directory index and tag (sequence number) from uio_offset */
 	index = uio_offset(uio) & HFS_INDEX_MASK;
 	tag = uio_offset(uio) & ~HFS_INDEX_MASK;
 	if ((index + 1) > dir_entries) {
 		*(ap->a_eofflag) = 1;
 		error = 0;
-		goto exit;
+		goto exit2;
 	}
 
 	/* Get a buffer to hold packed attributes. */
-	fixedblocksize = (sizeof(uint32_t) + hfs_attrblksize(alist)); /* 4 bytes for length */
+	fixedblocksize = (sizeof(u_int32_t) + hfs_attrblksize(alist)); /* 4 bytes for length */
 	maxattrblocksize = fixedblocksize;
 	if (alist->commonattr & ATTR_CMN_NAME) 
 		maxattrblocksize += kHFSPlusMaxFileNameBytes + 1;
@@ -712,13 +167,8 @@ hfs_vnop_readdirattr(ap)
 	attrptr = attrbufptr;
 	varptr = (char *)attrbufptr + fixedblocksize;  /* Point to variable-length storage */
 
-	/* Initialize a catalog entry list. */
-	MALLOC(ce_list, struct cat_entrylist *, sizeof(*ce_list), M_TEMP, M_WAITOK);
-	bzero(ce_list, sizeof(*ce_list));
-	ce_list->maxentries = MAXCATENTRIES;
-
-	/* Get a directory hint (cnode must be locked exclusive) */	
-	dirhint = hfs_getdirhint(dcp, ((index - 1) & HFS_INDEX_MASK) | tag);
+	/* Get a detached directory hint (cnode must be locked exclusive) */	
+	dirhint = hfs_getdirhint(dcp, ((index - 1) & HFS_INDEX_MASK) | tag, TRUE);
 
 	/* Hide tag from catalog layer. */
 	dirhint->dh_index &= HFS_INDEX_MASK;
@@ -727,200 +177,217 @@ hfs_vnop_readdirattr(ap)
 	}
 
 	/*
-	 * An ATTR_CMN_USERACCESS attribute request can result in a
-	 * call to kauth_cred_ismember_gid().  So when requesting
-	 * this attribute we downgrade our exclusive lock on dcp to
-	 * a shared lock in case kauth_cred_ismember_gid generates
-	 * an indirect call back into the file system.
+	 * Obtain a list of catalog entries and pack their attributes until
+	 * the output buffer is full or maxcount entries have been packed.
+	 */
+
+	/*
+	 * Constrain our list size.
 	 */
-	if (alist->commonattr & ATTR_CMN_USERACCESS) {
-		lck_rw_lock_exclusive_to_shared(&dcp->c_rwlock);
-		dcp->c_lockowner = HFS_SHARED_OWNER;
-		shared_cnode_lock = 1;
+	maxentries = uio_resid(uio) / (fixedblocksize + HFS_AVERAGE_NAME_SIZE);
+	maxentries = min(maxentries, dcp->c_entries - index);
+	maxentries = min(maxentries, maxcount);
+	maxentries = min(maxentries, MAXCATENTRIES);
+	if (maxentries < 1) {
+		error = EINVAL;
+		goto exit2;
 	}
+
+	/* Initialize a catalog entry list. */
+	MALLOC(ce_list, struct cat_entrylist *, CE_LIST_SIZE(maxentries), M_TEMP, M_WAITOK);
+	bzero(ce_list, CE_LIST_SIZE(maxentries));
+	ce_list->maxentries = maxentries;
+
 	/*
-	 * Obtain a list of catalog entries and pack their attributes until
-	 * the output buffer is full or maxcount entries have been packed.
+	 * Populate the ce_list from the catalog file.
 	 */
-	while (!depleted) {
-		int maxentries;
-		int lockflags;
-
-		/* Constrain our list size. */
-		maxentries = uio_resid(uio) / (fixedblocksize + HFS_AVERAGE_NAME_SIZE);
-		maxentries = min(maxentries, dcp->c_entries - index);
-		maxentries = min(maxentries, maxcount);
-		ce_list->maxentries = min(maxentries, ce_list->maxentries);
-		lastdescp = NULL;
-
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-		 
-		error = cat_getentriesattr(hfsmp, dirhint, ce_list);
-		/* Don't forget to release the descriptors later! */
-
-		hfs_systemfile_unlock(hfsmp, lockflags);
- 
-		if (error == ENOENT) {
-			*(ap->a_eofflag) = TRUE;
-			error = 0;
-			depleted = 1;
-		}
-		if (error)
-			break;
- 
-		/* Process the catalog entries. */
-		for (i = 0; i < (int)ce_list->realentries; ++i) {
-			struct cnode *cp = NULL;
-			struct vnode *vp = NULL;
-			struct cat_desc * cdescp;
-			struct cat_attr * cattrp;
-			struct cat_fork c_datafork;
-			struct cat_fork c_rsrcfork;
-
-			bzero(&c_datafork, sizeof(c_datafork));
-			bzero(&c_rsrcfork, sizeof(c_rsrcfork));
-			cdescp = &ce_list->entry[i].ce_desc;
-			cattrp = &ce_list->entry[i].ce_attr;
-			c_datafork.cf_size   = ce_list->entry[i].ce_datasize;
-			c_datafork.cf_blocks = ce_list->entry[i].ce_datablks;
-			c_rsrcfork.cf_size   = ce_list->entry[i].ce_rsrcsize;
-			c_rsrcfork.cf_blocks = ce_list->entry[i].ce_rsrcblks;
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+	 
+	error = cat_getentriesattr(hfsmp, dirhint, ce_list);
+	/* Don't forget to release the descriptors later! */
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+
+	if (error == ENOENT) {
+		*(ap->a_eofflag) = TRUE;
+		error = 0;
+	}
+	if (error) {
+		goto exit1;
+	}
+
+	/*
+	 * Drop the directory lock so we don't deadlock when we:
+	 *   - acquire a child cnode lock
+	 *   - make calls to vnode_authorize()
+	 *   - make calls to kauth_cred_ismember_gid()
+	 */
+	hfs_unlock(dcp);
+	dcp = NULL;
+
+	/* Process the catalog entries. */
+	for (i = 0; i < (int)ce_list->realentries; ++i) {
+		struct cnode *cp = NULL;
+		struct vnode *vp = NULL;
+		struct cat_desc * cdescp;
+		struct cat_attr * cattrp;
+		struct cat_fork c_datafork;
+		struct cat_fork c_rsrcfork;
+
+		bzero(&c_datafork, sizeof(c_datafork));
+		bzero(&c_rsrcfork, sizeof(c_rsrcfork));
+		cdescp = &ce_list->entry[i].ce_desc;
+		cattrp = &ce_list->entry[i].ce_attr;
+		c_datafork.cf_size   = ce_list->entry[i].ce_datasize;
+		c_datafork.cf_blocks = ce_list->entry[i].ce_datablks;
+		c_rsrcfork.cf_size   = ce_list->entry[i].ce_rsrcsize;
+		c_rsrcfork.cf_blocks = ce_list->entry[i].ce_rsrcblks;
+
+		if ((alist->commonattr & ATTR_CMN_USERACCESS) &&
+		    (cattrp->ca_recflags & kHFSHasSecurityMask)) {
 			/*
-			 * Get in memory cnode data (if any).
+			 * Obtain vnode for our vnode_authorize() calls.
 			 */
-			if (!(ap->a_options & FSOPT_NOINMEMUPDATE)) {
-				vp = hfs_chash_getvnode(dcp->c_dev, cattrp->ca_fileid, 0, 0);
-
-				if (vp != NULL) {
-					cp = VTOC(vp);
-					/* Only use cnode's decriptor for non-hardlinks */
-					if (!(cp->c_flag & C_HARDLINK))
-						cdescp = &cp->c_desc;
-					cattrp = &cp->c_attr;
-					if (cp->c_datafork) {
-						c_datafork.cf_size   = cp->c_datafork->ff_size;
-						c_datafork.cf_blocks = cp->c_datafork->ff_blocks;
-					}
-					if (cp->c_rsrcfork) {
-						c_rsrcfork.cf_size   = cp->c_rsrcfork->ff_size;
-						c_rsrcfork.cf_blocks = cp->c_rsrcfork->ff_blocks;
-					}
-				}
-			}
-			*((uint32_t *)attrptr)++ = 0; /* move it past length */
-			attrblk.ab_attrlist = alist;
-			attrblk.ab_attrbufpp = &attrptr;
-			attrblk.ab_varbufpp = &varptr;
-			attrblk.ab_flags = 0;
-			attrblk.ab_blocksize = maxattrblocksize;
-
-			/* Pack catalog entries into attribute buffer. */
-			hfs_packattrblk(&attrblk, hfsmp, vp, cdescp, cattrp,
-					&c_datafork, &c_rsrcfork, p);
-			currattrbufsize = ((char *)varptr - (char *)attrbufptr);
-		
-			/* All done with cnode. */
-			if (vp != NULL) {
-				hfs_unlock(VTOC(vp));
-				vnode_put(vp);
+			if (hfs_vget(hfsmp, cattrp->ca_fileid, &vp, 0) != 0) {
 				vp = NULL;
-				cp = NULL;
 			}
-
-			/* Make sure there's enough buffer space remaining. */
-			// LP64todo - fix this!
-			if (uio_resid(uio) < 0 || currattrbufsize > (uint32_t)uio_resid(uio)) {
-				depleted = 1;
-				break;
-			} else {
-				*((uint32_t *)attrbufptr) = currattrbufsize;
-				error = uiomove((caddr_t)attrbufptr, currattrbufsize, ap->a_uio);
-				if (error != E_NONE) {
-					depleted = 1;
-					break;
-				}
-				attrptr = attrbufptr;
-				varptr = (char *)attrbufptr + fixedblocksize;  /* Point to variable-length storage */
-				/* Save the last valid catalog entry */
-				lastdescp = &ce_list->entry[i].ce_desc;
-				index++;
-				*ap->a_actualcount += 1;
-
-				/* Termination checks */
-				if ((--maxcount <= 0) ||
-					// LP64todo - fix this!
-					uio_resid(uio) < 0 ||
-				    ((uint32_t)uio_resid(uio) < (fixedblocksize + HFS_AVERAGE_NAME_SIZE)) ||
-				    (index >= dir_entries)) {
-					depleted = 1;
-					break;
-				}
+		} else if (!(ap->a_options & FSOPT_NOINMEMUPDATE)) {
+			/* Get in-memory cnode data (if any). */
+			vp = hfs_chash_getvnode(hfsmp->hfs_raw_dev, cattrp->ca_fileid, 0, 0);
+		}
+		if (vp != NULL) {
+			cp = VTOC(vp);
+			/* Only use cnode's decriptor for non-hardlinks */
+			if (!(cp->c_flag & C_HARDLINK))
+				cdescp = &cp->c_desc;
+			cattrp = &cp->c_attr;
+			if (cp->c_datafork) {
+				c_datafork.cf_size   = cp->c_datafork->ff_size;
+				c_datafork.cf_blocks = cp->c_datafork->ff_blocks;
+			}
+			if (cp->c_rsrcfork) {
+				c_rsrcfork.cf_size   = cp->c_rsrcfork->ff_size;
+				c_rsrcfork.cf_blocks = cp->c_rsrcfork->ff_blocks;
 			}
-		} /* for each catalog entry */
+			/* All done with cnode. */
+			hfs_unlock(cp);
+			cp = NULL;
+		}
 
-		/* If there are more entries then save the last name. */
-		if (index < dir_entries
-		&&  !(*(ap->a_eofflag))
-		&&  lastdescp != NULL) {
+		*((u_int32_t *)attrptr) = 0; 
+		attrptr = ((u_int32_t *)attrptr) + 1;
+		attrblk.ab_attrlist = alist;
+		attrblk.ab_attrbufpp = &attrptr;
+		attrblk.ab_varbufpp = &varptr;
+		attrblk.ab_flags = 0;
+		attrblk.ab_blocksize = maxattrblocksize;
+		attrblk.ab_context = ap->a_context;
+
+		/* Pack catalog entries into attribute buffer. */
+		hfs_packattrblk(&attrblk, hfsmp, vp, cdescp, cattrp, &c_datafork, &c_rsrcfork, p);
+		currattrbufsize = ((char *)varptr - (char *)attrbufptr);
+	
+		/* All done with vnode. */
+		if (vp != NULL) {
+			vnode_put(vp);
+			vp = NULL;
+		}
 
-			/* Remember last entry */
-			if (dirhint->dh_desc.cd_nameptr != NULL) {
-				vfs_removename(dirhint->dh_desc.cd_nameptr);
+		/* Make sure there's enough buffer space remaining. */
+		// LP64todo - fix this!
+		if (uio_resid(uio) < 0 || currattrbufsize > (u_int32_t)uio_resid(uio)) {
+			break;
+		} else {
+			*((u_int32_t *)attrbufptr) = currattrbufsize;
+			error = uiomove((caddr_t)attrbufptr, currattrbufsize, uio);
+			if (error != E_NONE) {
+				break;
+			}
+			attrptr = attrbufptr;
+			/* Point to variable-length storage */
+			varptr = (char *)attrbufptr + fixedblocksize; 
+			/* Save the last valid catalog entry */
+			lastdescp = &ce_list->entry[i].ce_desc;
+			index++;
+			*ap->a_actualcount += 1;
+
+			/* Termination checks */
+			if ((--maxcount <= 0) ||
+			    // LP64todo - fix this!
+			    uio_resid(uio) < 0 ||
+			    ((u_int32_t)uio_resid(uio) < (fixedblocksize + HFS_AVERAGE_NAME_SIZE)) ||
+			    (index >= dir_entries)) {
+				break;
 			}
-			dirhint->dh_desc.cd_namelen = lastdescp->cd_namelen;
-			dirhint->dh_desc.cd_nameptr =
-				vfs_addname(lastdescp->cd_nameptr, lastdescp->cd_namelen, 0, 0);
-			dirhint->dh_index = index - 1;
-			dirhint->dh_desc.cd_cnid = lastdescp->cd_cnid;
-			dirhint->dh_desc.cd_hint = lastdescp->cd_hint;
-			dirhint->dh_desc.cd_encoding = lastdescp->cd_encoding;
 		}
-			
-		/* All done with the catalog descriptors. */
-		for (i = 0; i < (int)ce_list->realentries; ++i)
-			cat_releasedesc(&ce_list->entry[i].ce_desc);
-		ce_list->realentries = 0;
+	} /* for each catalog entry */
 
-	} /* while not depleted */
+	/* For small directories, check if we're all done. */
+	if (*ap->a_actualcount == (u_long)dir_entries) {
+		*(ap->a_eofflag) = TRUE;
+	}
 
-	*ap->a_newstate = dcp->c_mtime;
+	/* If we skipped catalog entries for reserved files that should
+	 * not be listed in namespace, update the index accordingly.
+	 */
+	if (ce_list->skipentries) {
+		index += ce_list->skipentries;
+		ce_list->skipentries = 0;
+	}
 
-	/* Make sure dcp is locked exclusive before changing c_dirhinttag. */
-	if (shared_cnode_lock) {
-		/*
-		 * If the upgrade fails we loose the lock and
-		 * have to take the exclusive lock on our own.
-		 */
-		if (lck_rw_lock_shared_to_exclusive(&dcp->c_rwlock) != 0)
-			lck_rw_lock_exclusive(&dcp->c_rwlock);
-		dcp->c_lockowner = current_thread();
-		shared_cnode_lock = 0;
+	/* If there are more entries then save the last name. */
+	if (index < dir_entries
+	&&  !(*(ap->a_eofflag))
+	&&  lastdescp != NULL) {
+
+		/* Remember last entry */
+		if ((dirhint->dh_desc.cd_flags & CD_HASBUF) &&
+		    (dirhint->dh_desc.cd_nameptr != NULL)) {
+			dirhint->dh_desc.cd_flags &= ~CD_HASBUF;
+			vfs_removename((const char *)dirhint->dh_desc.cd_nameptr);
+		}
+		dirhint->dh_desc.cd_namelen = lastdescp->cd_namelen;
+		dirhint->dh_desc.cd_nameptr = (const u_int8_t *)
+		vfs_addname((const char *)lastdescp->cd_nameptr, lastdescp->cd_namelen, 0, 0);
+		dirhint->dh_desc.cd_flags |= CD_HASBUF;
+		dirhint->dh_index = index - 1;
+		dirhint->dh_desc.cd_cnid = lastdescp->cd_cnid;
+		dirhint->dh_desc.cd_hint = lastdescp->cd_hint;
+		dirhint->dh_desc.cd_encoding = lastdescp->cd_encoding;
+	} else {
+		/* No more entries. */
+		*(ap->a_eofflag) = TRUE;
 	}
+		
+	/* All done with the catalog descriptors. */
+	for (i = 0; i < (int)ce_list->realentries; ++i)
+		cat_releasedesc(&ce_list->entry[i].ce_desc);
+	ce_list->realentries = 0;
+
+	(void) hfs_lock(VTOC(dvp), HFS_FORCE_LOCK);
+	dcp = VTOC(dvp);
 
-	/* Convert directory index back into a uio_offset. */
+exit1:
+	/* Pack directory index and tag into uio_offset. */
 	while (tag == 0) tag = (++dcp->c_dirhinttag) << HFS_INDEX_BITS;	
 	uio_setoffset(uio, index | tag);
 	dirhint->dh_index |= tag;
 
-exit:
+exit2:
+	*ap->a_newstate = dcp->c_dirchangecnt;
+
 	/* Drop directory hint on error or if there are no more entries */
-	if (dirhint && (error || index >= dir_entries)) {
-		if (shared_cnode_lock) {
-			/*
-			 * If the upgrade fails we loose the lock and
-			 * have to take the exclusive lock on our own.
-			 */
-			if (lck_rw_lock_shared_to_exclusive(&dcp->c_rwlock) != 0)
-				lck_rw_lock_exclusive(&dcp->c_rwlock);
-			dcp->c_lockowner = current_thread();
-		}
-		hfs_reldirhint(dcp, dirhint);
+	if (dirhint) {
+		if ((error != 0) || (index >= dir_entries) || *(ap->a_eofflag))
+			hfs_reldirhint(dcp, dirhint);
+		else
+			hfs_insertdirhint(dcp, dirhint);
 	}
 	if (attrbufptr)
 		FREE(attrbufptr, M_TEMP);
 	if (ce_list)
 		FREE(ce_list, M_TEMP);
-		
+
 	hfs_unlock(dcp);
 	return (error);
 }
@@ -940,489 +407,103 @@ hfs_packattrblk(struct attrblock *abp,
 		struct cat_attr *attrp,
 		struct cat_fork *datafork,
 		struct cat_fork *rsrcfork,
-		struct proc *p)
-{
-	struct attrlist *attrlistp = abp->ab_attrlist;
-
-	if (attrlistp->volattr) {
-		if (attrlistp->commonattr)
-			packvolcommonattr(abp, hfsmp, vp, p);
-
-		if (attrlistp->volattr & ~ATTR_VOL_INFO)
-			packvolattr(abp, hfsmp, vp);
-	} else {
-		if (attrlistp->commonattr)
-			packcommonattr(abp, hfsmp, vp, descp, attrp, p);
-	
-		if (attrlistp->dirattr && S_ISDIR(attrp->ca_mode))
-			packdirattr(abp, hfsmp, vp, descp,attrp);
-	
-		if (attrlistp->fileattr && !S_ISDIR(attrp->ca_mode))
-			packfileattr(abp, hfsmp, attrp, datafork, rsrcfork);
-	}
-}
-
-
-static char*
-mountpointname(struct mount *mp)
-{
-	size_t namelength = strlen(mp->mnt_vfsstat.f_mntonname);
-	int foundchars = 0;
-	char *c;
-	
-	if (namelength == 0)
-		return (NULL);
-	
-	/*
-	 * Look backwards through the name string, looking for
-	 * the first slash encountered (which must precede the
-	 * last part of the pathname).
-	 */
-	for (c = mp->mnt_vfsstat.f_mntonname + namelength - 1;
-	     namelength > 0; --c, --namelength) {
-		if (*c != '/') {
-			foundchars = 1;
-		} else if (foundchars) {
-			return (c + 1);
-		}
-	}
-	
-	return (mp->mnt_vfsstat.f_mntonname);
-}
-
-
-static void
-packnameattr(
-	struct attrblock *abp,
-	struct vnode *vp,
-	char *name,
-	int namelen)
-{
-	void *varbufptr;
-	struct attrreference * attr_refptr;
-	char *mpname;
-	size_t mpnamelen;
-	uint32_t attrlength;
-	char empty = 0;
-	
-	/* A cnode's name may be incorrect for the root of a mounted
-	 * filesystem (it can be mounted on a different directory name
-	 * than the name of the volume, such as "blah-1").  So for the
-	 * root directory, it's best to return the last element of the
-	 location where the volume's mounted:
-	 */
-	if ((vp != NULL) && vnode_isvroot(vp) &&
-	    (mpname = mountpointname(vnode_mount(vp)))) {
-		mpnamelen = strlen(mpname);
-		
-		/* Trim off any trailing slashes: */
-		while ((mpnamelen > 0) && (mpname[mpnamelen-1] == '/'))
-			--mpnamelen;
-
-		/* If there's anything left, use it instead of the volume's name */
-		if (mpnamelen > 0) {
-			name = mpname;
-			namelen = mpnamelen;
-		}
-	}
-	if (name == NULL) {
-		name = &empty;
-		namelen = 0;
-	}
-
-	varbufptr = *abp->ab_varbufpp;
-	attr_refptr = (struct attrreference *)(*abp->ab_attrbufpp);
-
-	attrlength = namelen + 1;
-	attr_refptr->attr_dataoffset = (char *)varbufptr - (char *)attr_refptr;
-	attr_refptr->attr_length = attrlength;
-	(void) strncpy((unsigned char *)varbufptr, name, attrlength);
-	/*
-	 * Advance beyond the space just allocated and
-	 * round up to the next 4-byte boundary:
-	 */
-	(char *)(varbufptr) += attrlength + ((4 - (attrlength & 3)) & 3);
-	++attr_refptr;
-
-	*abp->ab_attrbufpp = attr_refptr;
-	*abp->ab_varbufpp = varbufptr;
-}
-
-/*
- * Pack common volume attributes.
- */
-static void
-packvolcommonattr(struct attrblock *abp, struct hfsmount *hfsmp, struct vnode *vp, struct proc *p)
-{
-	attrgroup_t attr;
-	void *attrbufptr = *abp->ab_attrbufpp;
-	void *varbufptr = *abp->ab_varbufpp;
-	struct cnode *cp = VTOC(vp);
-	struct mount *mp = VTOVFS(vp);
-	ExtendedVCB *vcb = HFSTOVCB(hfsmp);
-	u_int32_t attrlength;
-	boolean_t is_64_bit = proc_is64bit(p);
-
-	attr = abp->ab_attrlist->commonattr;
-
-	if (ATTR_CMN_NAME & attr) {
-		packnameattr(abp, vp, cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen);
-		attrbufptr = *abp->ab_attrbufpp;
-		varbufptr = *abp->ab_varbufpp;
-	}
-	if (ATTR_CMN_DEVID & attr) {
-		*((dev_t *)attrbufptr)++ = hfsmp->hfs_raw_dev;
-	}
-	if (ATTR_CMN_FSID & attr) {
-		fsid_t fsid;
-		
-		fsid.val[0] = (long)hfsmp->hfs_raw_dev;
-		fsid.val[1] = (long)vfs_typenum(mp);
-		*((fsid_t *)attrbufptr) = fsid;
-		++((fsid_t *)attrbufptr);
-	}
-	if (ATTR_CMN_OBJTYPE & attr) {
-		*((fsobj_type_t *)attrbufptr)++ = 0;
-	}
-	if (ATTR_CMN_OBJTAG & attr) {
-		*((fsobj_tag_t *)attrbufptr)++ = VT_HFS;
-	}
-	if (ATTR_CMN_OBJID & attr)	{
-		((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
-	}
-        if (ATTR_CMN_OBJPERMANENTID & attr) {
-		((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
-        }
-	if (ATTR_CMN_PAROBJID & attr) {
-		((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
-	}
-        if (ATTR_CMN_SCRIPT & attr) {
-        	uint32_t encoding;
- 
-        	if (vcb->vcbSigWord == kHFSPlusSigWord)
-        		encoding = vcb->volumeNameEncodingHint;
-        	else
-        		encoding = hfsmp->hfs_encoding;
-        	*((text_encoding_t *)attrbufptr)++ = encoding;
-	}
-	if (ATTR_CMN_CRTIME & attr) {
-	    if (is_64_bit) {
-            ((struct user_timespec *)attrbufptr)->tv_sec = vcb->vcbCrDate;
-            ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            ((struct timespec *)attrbufptr)->tv_sec = vcb->vcbCrDate;
-            ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_MODTIME & attr) {
-	    if (is_64_bit) {
-            ((struct user_timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            ((struct timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_CHGTIME & attr) {
-	    if (is_64_bit) {
-            ((struct user_timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            ((struct timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_ACCTIME & attr) {
-	    if (is_64_bit) {
-            ((struct user_timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            ((struct timespec *)attrbufptr)->tv_sec = vcb->vcbLsMod;
-            ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_BKUPTIME & attr) {
-	    if (is_64_bit) {
-            ((struct user_timespec *)attrbufptr)->tv_sec = vcb->vcbVolBkUp;
-            ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            ((struct timespec *)attrbufptr)->tv_sec = vcb->vcbVolBkUp;
-            ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_FNDRINFO & attr) {
-		bcopy (&vcb->vcbFndrInfo, attrbufptr, sizeof(vcb->vcbFndrInfo));
-		(char *)attrbufptr += sizeof(vcb->vcbFndrInfo);
-	}
-	if (ATTR_CMN_OWNERID & attr) {
-		if (cp->c_uid == UNKNOWNUID)
-			*((uid_t *)attrbufptr)++ = kauth_cred_getuid(proc_ucred(p));
-		else
-			*((uid_t *)attrbufptr)++ = cp->c_uid;
-	}
-	if (ATTR_CMN_GRPID & attr) {
-		*((gid_t *)attrbufptr)++ = cp->c_gid;
-	}
+		struct proc *p)
+{
+	struct attrlist *attrlistp = abp->ab_attrlist;
 
-	if (ATTR_CMN_ACCESSMASK & attr) {
-		/*
-		 * [2856576]  Since we are dynamically changing the owner, also
-		 * effectively turn off the set-user-id and set-group-id bits,
-		 * just like chmod(2) would when changing ownership.  This prevents
-		 * a security hole where set-user-id programs run as whoever is
-		 * logged on (or root if nobody is logged in yet!)
-		 */
-		*((uint32_t *)attrbufptr)++ =
-			(cp->c_uid == UNKNOWNUID) ? cp->c_mode & ~(S_ISUID | S_ISGID) : cp->c_mode;
-	}
-	if (ATTR_CMN_NAMEDATTRCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = 0;	/* XXX PPD TBC */
-	}
-	if (ATTR_CMN_NAMEDATTRLIST & attr) {
-		attrlength = 0;
-		((struct attrreference *)attrbufptr)->attr_dataoffset = 0;
-		((struct attrreference *)attrbufptr)->attr_length = attrlength;
-		/*
-		 * Advance beyond the space just allocated and
-		 * round up to the next 4-byte boundary:
-		 */
-		(char *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-		++((struct attrreference *)attrbufptr);
-	}
-	if (ATTR_CMN_FLAGS & attr) {
-		*((uint32_t *)attrbufptr)++ = cp->c_flags;
-	}
-	if (ATTR_CMN_USERACCESS & attr) {
-		*((uint32_t *)attrbufptr)++ =
-			DerivePermissionSummary(cp->c_uid, cp->c_gid, cp->c_mode,
-				VTOVFS(vp), kauth_cred_get(), proc_self());
-	}
+	if (attrlistp->commonattr)
+		packcommonattr(abp, hfsmp, vp, descp, attrp, p);
 
-	*abp->ab_attrbufpp = attrbufptr;
-	*abp->ab_varbufpp = varbufptr;
+	if (attrlistp->dirattr && S_ISDIR(attrp->ca_mode))
+		packdirattr(abp, hfsmp, vp, descp,attrp);
+
+	if (attrlistp->fileattr && !S_ISDIR(attrp->ca_mode))
+		packfileattr(abp, hfsmp, attrp, datafork, rsrcfork);
 }
 
 
-static void
-packvolattr(struct attrblock *abp, struct hfsmount *hfsmp, struct vnode *vp)
+static char*
+mountpointname(struct mount *mp)
 {
-	attrgroup_t attr;
-	void *attrbufptr = *abp->ab_attrbufpp;
-	void *varbufptr = *abp->ab_varbufpp;
-	struct cnode *cp = VTOC(vp);
-	struct mount *mp = VTOVFS(vp);
-	ExtendedVCB *vcb = HFSTOVCB(hfsmp);
-	uint32_t attrlength;
+	size_t namelength = strlen(mp->mnt_vfsstat.f_mntonname);
+	int foundchars = 0;
+	char *c;
+	
+	if (namelength == 0)
+		return (NULL);
+	
+	/*
+	 * Look backwards through the name string, looking for
+	 * the first slash encountered (which must precede the
+	 * last part of the pathname).
+	 */
+	for (c = mp->mnt_vfsstat.f_mntonname + namelength - 1;
+	     namelength > 0; --c, --namelength) {
+		if (*c != '/') {
+			foundchars = 1;
+		} else if (foundchars) {
+			return (c + 1);
+		}
+	}
+	
+	return (mp->mnt_vfsstat.f_mntonname);
+}
 
-	attr = abp->ab_attrlist->volattr;
 
-	if (ATTR_VOL_FSTYPE & attr) {
-		*((uint32_t *)attrbufptr)++ = (uint32_t)vfs_typenum(mp);
-	}
-	if (ATTR_VOL_SIGNATURE & attr) {
-		*((uint32_t *)attrbufptr)++ = (uint32_t)vcb->vcbSigWord;
-	}
-	if (ATTR_VOL_SIZE & attr) {
-		*((off_t *)attrbufptr)++ =
-				(off_t)vcb->totalBlocks * (off_t)vcb->blockSize;
-	}
-	if (ATTR_VOL_SPACEFREE & attr) {
-		*((off_t *)attrbufptr)++ = (off_t)hfs_freeblks(hfsmp, 0) *
-		                           (off_t)vcb->blockSize;
-	}
-	if (ATTR_VOL_SPACEAVAIL & attr) {
-		*((off_t *)attrbufptr)++ = (off_t)hfs_freeblks(hfsmp, 1) *
-		                           (off_t)vcb->blockSize;
-	}
-	if (ATTR_VOL_MINALLOCATION & attr) {
-		*((off_t *)attrbufptr)++ = (off_t)vcb->blockSize;
-	}
-	if (ATTR_VOL_ALLOCATIONCLUMP & attr) {
-		*((off_t *)attrbufptr)++ = (off_t)(vcb->vcbClpSiz);
-	}
-	if (ATTR_VOL_IOBLOCKSIZE & attr) {
-        *((uint32_t *)attrbufptr)++ = hfsmp->hfs_logBlockSize;
-	}
-	if (ATTR_VOL_OBJCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = 
-		    (uint32_t)vcb->vcbFilCnt + (uint32_t)vcb->vcbDirCnt;
-	}
-	if (ATTR_VOL_FILECOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = (uint32_t)vcb->vcbFilCnt;
-	}
-	if (ATTR_VOL_DIRCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = (uint32_t)vcb->vcbDirCnt;
-	}
-	if (ATTR_VOL_MAXOBJCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = 0xFFFFFFFF;
-	}
-	if (ATTR_VOL_MOUNTPOINT & attr) {
-		((struct attrreference *)attrbufptr)->attr_dataoffset =
-				(char *)varbufptr - (char *)attrbufptr;
-		((struct attrreference *)attrbufptr)->attr_length =
-				strlen(mp->mnt_vfsstat.f_mntonname) + 1;
-		attrlength = ((struct attrreference *)attrbufptr)->attr_length;
-		/* round up to the next 4-byte boundary: */
-		attrlength = attrlength + ((4 - (attrlength & 3)) & 3);
-		(void) bcopy(mp->mnt_vfsstat.f_mntonname, varbufptr, attrlength);
-			
-		/* Advance beyond the space just allocated: */
-		(char *)varbufptr += attrlength;
-		++((struct attrreference *)attrbufptr);
-	}
-	if (ATTR_VOL_NAME & attr) {
-		((struct attrreference *)attrbufptr)->attr_dataoffset =
-				(char *)varbufptr - (char *)attrbufptr;
-		((struct attrreference *)attrbufptr)->attr_length =
-				cp->c_desc.cd_namelen + 1;
-		attrlength = ((struct attrreference *)attrbufptr)->attr_length;
-		/* round up to the next 4-byte boundary: */
-		attrlength = attrlength + ((4 - (attrlength & 3)) & 3);
-		/* XXX this could read off the end of cd_nameptr! */
-		bcopy(cp->c_desc.cd_nameptr, varbufptr, attrlength);
-
-		/* Advance beyond the space just allocated: */
-		(char *)varbufptr += attrlength;
-		++((struct attrreference *)attrbufptr);
-	}
-    if (ATTR_VOL_MOUNTFLAGS & attr) {
-        *((uint32_t *)attrbufptr)++ = (uint32_t)vfs_flags(mp);
-    }
-	if (ATTR_VOL_MOUNTEDDEVICE & attr) {
-		((struct attrreference *)attrbufptr)->attr_dataoffset =
-				(char *)varbufptr - (char *)attrbufptr;
-		((struct attrreference *)attrbufptr)->attr_length =
-				strlen(mp->mnt_vfsstat.f_mntfromname) + 1;
-		attrlength = ((struct attrreference *)attrbufptr)->attr_length;
-		/* round up to the next 4-byte boundary: */
-		attrlength = attrlength + ((4 - (attrlength & 3)) & 3);
-		(void) bcopy(mp->mnt_vfsstat.f_mntfromname, varbufptr, attrlength);
-			
-		/* Advance beyond the space just allocated: */
-		(char *)varbufptr += attrlength;
-		++((struct attrreference *)attrbufptr);
-        }
-	if (ATTR_VOL_ENCODINGSUSED & attr) {
-		*((unsigned long long *)attrbufptr)++ =
-				(unsigned long long)vcb->encodingsBitmap;
-	}
-	if (ATTR_VOL_CAPABILITIES & attr) {
-		vol_capabilities_attr_t *vcapattrptr;
+static void
+packnameattr(
+	struct attrblock *abp,
+	struct vnode *vp,
+	const u_int8_t *name,
+	int namelen)
+{
+	void *varbufptr;
+	struct attrreference * attr_refptr;
+	char *mpname;
+	size_t mpnamelen;
+	u_int32_t attrlength;
+	u_int8_t empty = 0;
 	
-		vcapattrptr = (vol_capabilities_attr_t *)attrbufptr;
-
-		if (vcb->vcbSigWord == kHFSPlusSigWord) {
-			u_int32_t journal_active_cap;
-			u_int32_t case_sensitive;
-			
-			if (hfsmp->jnl)
-				journal_active_cap = VOL_CAP_FMT_JOURNAL_ACTIVE;
-			else
-				journal_active_cap = 0;
-
-			if (hfsmp->hfs_flags & HFS_CASE_SENSITIVE)
-				case_sensitive = VOL_CAP_FMT_CASE_SENSITIVE;
-			else
-				case_sensitive = 0;
-			
-			vcapattrptr->capabilities[VOL_CAPABILITIES_FORMAT] =
-					VOL_CAP_FMT_PERSISTENTOBJECTIDS |
-					VOL_CAP_FMT_SYMBOLICLINKS |
-					VOL_CAP_FMT_HARDLINKS |
-					VOL_CAP_FMT_JOURNAL |
-					journal_active_cap |
-					case_sensitive |
-					VOL_CAP_FMT_CASE_PRESERVING |
-					VOL_CAP_FMT_FAST_STATFS | 
-					VOL_CAP_FMT_2TB_FILESIZE;
-		} else { /* Plain HFS */
-			vcapattrptr->capabilities[VOL_CAPABILITIES_FORMAT] =
-					VOL_CAP_FMT_PERSISTENTOBJECTIDS |
-					VOL_CAP_FMT_CASE_PRESERVING |
-					VOL_CAP_FMT_FAST_STATFS ;
+	/* A cnode's name may be incorrect for the root of a mounted
+	 * filesystem (it can be mounted on a different directory name
+	 * than the name of the volume, such as "blah-1").  So for the
+	 * root directory, it's best to return the last element of the
+	 location where the volume's mounted:
+	 */
+	if ((vp != NULL) && vnode_isvroot(vp) &&
+	    (mpname = mountpointname(vnode_mount(vp)))) {
+		mpnamelen = strlen(mpname);
+		
+		/* Trim off any trailing slashes: */
+		while ((mpnamelen > 0) && (mpname[mpnamelen-1] == '/'))
+			--mpnamelen;
+
+		/* If there's anything left, use it instead of the volume's name */
+		if (mpnamelen > 0) {
+			name = (u_int8_t *)mpname;
+			namelen = mpnamelen;
 		}
-        	vcapattrptr->capabilities[VOL_CAPABILITIES_INTERFACES] =
-        				VOL_CAP_INT_SEARCHFS |
-        				VOL_CAP_INT_ATTRLIST |
-        				VOL_CAP_INT_NFSEXPORT |
-        				VOL_CAP_INT_READDIRATTR |
-        				VOL_CAP_INT_EXCHANGEDATA |
-        				VOL_CAP_INT_ALLOCATE |
-        				VOL_CAP_INT_VOL_RENAME |
-        				VOL_CAP_INT_ADVLOCK |
-        				VOL_CAP_INT_FLOCK ;
-        	vcapattrptr->capabilities[VOL_CAPABILITIES_RESERVED1] = 0;
-        	vcapattrptr->capabilities[VOL_CAPABILITIES_RESERVED2] = 0;
-
-		vcapattrptr->valid[VOL_CAPABILITIES_FORMAT] =
-					VOL_CAP_FMT_PERSISTENTOBJECTIDS |
-					VOL_CAP_FMT_SYMBOLICLINKS |
-					VOL_CAP_FMT_HARDLINKS |
-					VOL_CAP_FMT_JOURNAL |
-					VOL_CAP_FMT_JOURNAL_ACTIVE |
-					VOL_CAP_FMT_NO_ROOT_TIMES |
-					VOL_CAP_FMT_SPARSE_FILES |
-					VOL_CAP_FMT_ZERO_RUNS |
-					VOL_CAP_FMT_CASE_SENSITIVE |
-					VOL_CAP_FMT_CASE_PRESERVING |
-					VOL_CAP_FMT_FAST_STATFS |
-					VOL_CAP_FMT_2TB_FILESIZE;
-        	vcapattrptr->valid[VOL_CAPABILITIES_INTERFACES] =
-        				VOL_CAP_INT_SEARCHFS |
-        				VOL_CAP_INT_ATTRLIST |
-        				VOL_CAP_INT_NFSEXPORT |
-        				VOL_CAP_INT_READDIRATTR |
-        				VOL_CAP_INT_EXCHANGEDATA |
-        				VOL_CAP_INT_COPYFILE |
-        				VOL_CAP_INT_ALLOCATE |
-        				VOL_CAP_INT_VOL_RENAME |
-        				VOL_CAP_INT_ADVLOCK |
-        				VOL_CAP_INT_FLOCK ;
-        	vcapattrptr->valid[VOL_CAPABILITIES_RESERVED1] = 0;
-        	vcapattrptr->valid[VOL_CAPABILITIES_RESERVED2] = 0;
-
-		++((vol_capabilities_attr_t *)attrbufptr);
 	}
-	if (ATTR_VOL_ATTRIBUTES & attr) {
-		vol_attributes_attr_t *volattrattrp;
-		
-		volattrattrp = (vol_attributes_attr_t *)attrbufptr;
-        	volattrattrp->validattr.commonattr = ATTR_CMN_VALIDMASK;
-        	volattrattrp->validattr.volattr = ATTR_VOL_VALIDMASK;
-        	volattrattrp->validattr.dirattr = ATTR_DIR_VALIDMASK;
-        	volattrattrp->validattr.fileattr = ATTR_FILE_VALIDMASK;
-        	volattrattrp->validattr.forkattr = ATTR_FORK_VALIDMASK;
-
-        	volattrattrp->nativeattr.commonattr = ATTR_CMN_VALIDMASK;
-        	volattrattrp->nativeattr.volattr = ATTR_VOL_VALIDMASK;
-        	volattrattrp->nativeattr.dirattr = ATTR_DIR_VALIDMASK;
-        	volattrattrp->nativeattr.fileattr = ATTR_FILE_VALIDMASK;
-        	volattrattrp->nativeattr.forkattr = ATTR_FORK_VALIDMASK;
-		++((vol_attributes_attr_t *)attrbufptr);
+	if (name == NULL) {
+		name = &empty;
+		namelen = 0;
 	}
-	
-	*abp->ab_attrbufpp = attrbufptr;
+
+	varbufptr = *abp->ab_varbufpp;
+	attr_refptr = (struct attrreference *)(*abp->ab_attrbufpp);
+
+	attrlength = namelen + 1;
+	attr_refptr->attr_dataoffset = (char *)varbufptr - (char *)attr_refptr;
+	attr_refptr->attr_length = attrlength;
+	(void) strncpy((char *)varbufptr, (const char *) name, attrlength);
+	/*
+	 * Advance beyond the space just allocated and
+	 * round up to the next 4-byte boundary:
+	 */
+	varbufptr = ((char *)varbufptr) + attrlength + ((4 - (attrlength & 3)) & 3);
+	++attr_refptr;
+
+	*abp->ab_attrbufpp = attr_refptr;
 	*abp->ab_varbufpp = varbufptr;
 }
 
@@ -1440,8 +521,14 @@ packcommonattr(
 	struct mount *mp = HFSTOVFS(hfsmp);
 	void *attrbufptr = *abp->ab_attrbufpp;
 	void *varbufptr = *abp->ab_varbufpp;
-	uint32_t attrlength = 0;
 	boolean_t is_64_bit = proc_is64bit(p);
+	uid_t cuid = 1;
+	int isroot = 0;
+
+	if (attr & (ATTR_CMN_OWNERID | ATTR_CMN_GRPID)) {
+		cuid = kauth_cred_getuid(proc_ucred(p));
+		isroot = cuid == 0;
+	}
 	
 	if (ATTR_CMN_NAME & attr) {
 		packnameattr(abp, vp, cdp->cd_nameptr, cdp->cd_namelen);
@@ -1449,7 +536,8 @@ packcommonattr(
 		varbufptr = *abp->ab_varbufpp;
 	}
 	if (ATTR_CMN_DEVID & attr) {
-		*((dev_t *)attrbufptr)++ = hfsmp->hfs_raw_dev;
+		*((dev_t *)attrbufptr) = hfsmp->hfs_raw_dev;
+		attrbufptr = ((dev_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_FSID & attr) {
 		fsid_t fsid;
@@ -1457,13 +545,15 @@ packcommonattr(
 		fsid.val[0] = (long)hfsmp->hfs_raw_dev;
 		fsid.val[1] = (long)vfs_typenum(mp);
 		*((fsid_t *)attrbufptr) = fsid;
-		++((fsid_t *)attrbufptr);
+		attrbufptr = ((fsid_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_OBJTYPE & attr) {
-		*((fsobj_type_t *)attrbufptr)++ = IFTOVT(cap->ca_mode);
+		*((fsobj_type_t *)attrbufptr) = IFTOVT(cap->ca_mode);
+		attrbufptr = ((fsobj_type_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_OBJTAG & attr) {
-		*((fsobj_tag_t *)attrbufptr)++ = VT_HFS;
+		*((fsobj_tag_t *)attrbufptr) = VT_HFS;
+		attrbufptr = ((fsobj_tag_t *)attrbufptr) + 1;
 	}
 	/*
 	 * Exporting file IDs from HFS Plus:
@@ -1477,94 +567,123 @@ packcommonattr(
 	 * and Carbon APIs, which are hardlink-ignorant, will always
 	 * receive the c_cnid (from getattrlist).
 	 */
-    if (ATTR_CMN_OBJID & attr) {
+	if (ATTR_CMN_OBJID & attr) {
 		((fsobj_id_t *)attrbufptr)->fid_objno = cdp->cd_cnid;
 		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
+		attrbufptr = ((fsobj_id_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_OBJPERMANENTID & attr) {
 		((fsobj_id_t *)attrbufptr)->fid_objno = cdp->cd_cnid;
 		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
+		attrbufptr = ((fsobj_id_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_PAROBJID & attr) {
 		((fsobj_id_t *)attrbufptr)->fid_objno = cdp->cd_parentcnid;
 		((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-		++((fsobj_id_t *)attrbufptr);
+		attrbufptr = ((fsobj_id_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_SCRIPT & attr) {
-		*((text_encoding_t *)attrbufptr)++ = cdp->cd_encoding;
+		*((text_encoding_t *)attrbufptr) = cdp->cd_encoding;
+		attrbufptr = ((text_encoding_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_CRTIME & attr) {
 	    if (is_64_bit) {
             ((struct user_timespec *)attrbufptr)->tv_sec = cap->ca_itime;
             ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
+			attrbufptr = ((struct user_timespec *)attrbufptr) + 1;
 	    }
 	    else {
             ((struct timespec *)attrbufptr)->tv_sec = cap->ca_itime;
             ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
+			attrbufptr = ((struct timespec *)attrbufptr) + 1;
 	    }
 	}
 	if (ATTR_CMN_MODTIME & attr) {
 	    if (is_64_bit) {
              ((struct user_timespec *)attrbufptr)->tv_sec = cap->ca_mtime;
              ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-             ++((struct user_timespec *)attrbufptr);
+			 attrbufptr = ((struct user_timespec *)attrbufptr) + 1;
 	    }
 	    else {
             ((struct timespec *)attrbufptr)->tv_sec = cap->ca_mtime;
             ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
+			attrbufptr = ((struct timespec *)attrbufptr) + 1;
 	    }
 	}
 	if (ATTR_CMN_CHGTIME & attr) {
 	    if (is_64_bit) {
             ((struct user_timespec *)attrbufptr)->tv_sec = cap->ca_ctime;
             ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
+			attrbufptr = ((struct user_timespec *)attrbufptr) + 1;
 	    }
 	    else {
             ((struct timespec *)attrbufptr)->tv_sec = cap->ca_ctime;
             ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
+			attrbufptr = ((struct timespec *)attrbufptr) + 1;
 	    }
 	}
 	if (ATTR_CMN_ACCTIME & attr) {
 	    if (is_64_bit) {
             ((struct user_timespec *)attrbufptr)->tv_sec = cap->ca_atime;
             ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
+			attrbufptr = ((struct user_timespec *)attrbufptr) + 1;
 	    }
 	    else {
             ((struct timespec *)attrbufptr)->tv_sec = cap->ca_atime;
             ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
+			attrbufptr = ((struct timespec *)attrbufptr) + 1;
 	    }
 	}
 	if (ATTR_CMN_BKUPTIME & attr) {
 	    if (is_64_bit) {
             ((struct user_timespec *)attrbufptr)->tv_sec = cap->ca_btime;
             ((struct user_timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct user_timespec *)attrbufptr);
+			attrbufptr = ((struct user_timespec *)attrbufptr) + 1;
 	    }
 	    else {
             ((struct timespec *)attrbufptr)->tv_sec = cap->ca_btime;
             ((struct timespec *)attrbufptr)->tv_nsec = 0;
-            ++((struct timespec *)attrbufptr);
+			attrbufptr = ((struct timespec *)attrbufptr) + 1;
 	    }
 	}
 	if (ATTR_CMN_FNDRINFO & attr) {
 		bcopy(&cap->ca_finderinfo, attrbufptr, sizeof(u_int8_t) * 32);
-		(char *)attrbufptr += sizeof(u_int8_t) * 32;
+		/* Don't expose a symlink's private type/creator. */
+		if (S_ISLNK(cap->ca_mode)) {
+			struct FndrFileInfo *fip;
+
+			fip = (struct FndrFileInfo *)attrbufptr;
+			fip->fdType = 0;
+			fip->fdCreator = 0;
+		}
+		attrbufptr = (char *)attrbufptr + sizeof(u_int8_t) * 32;
 	}
 	if (ATTR_CMN_OWNERID & attr) {
-		*((uid_t *)attrbufptr)++ =
-			(cap->ca_uid == UNKNOWNUID) ? kauth_cred_getuid(proc_ucred(p)) : cap->ca_uid;
+		uid_t nuid = cap->ca_uid;
+
+		if (!isroot) {
+			if (((unsigned int)vfs_flags(HFSTOVFS(hfsmp))) & MNT_UNKNOWNPERMISSIONS)
+				nuid = cuid;
+			else if (nuid == UNKNOWNUID)
+				nuid = cuid;
+		}
+
+		*((uid_t *)attrbufptr) = nuid;
+		attrbufptr = ((uid_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_GRPID & attr) {
-		*((gid_t *)attrbufptr)++ = cap->ca_gid;
+		gid_t ngid = cap->ca_gid;
+
+		if (!isroot) {
+			gid_t cgid = kauth_cred_getgid(proc_ucred(p));
+			if (((unsigned int)vfs_flags(HFSTOVFS(hfsmp))) & MNT_UNKNOWNPERMISSIONS)
+				ngid = cgid;
+			else if (ngid == UNKNOWNUID)
+				ngid = cgid;
+		}
+
+		*((gid_t *)attrbufptr) = ngid;
+		attrbufptr = ((gid_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_ACCESSMASK & attr) {
 		/*
@@ -1574,31 +693,44 @@ packcommonattr(
 		 * a security hole where set-user-id programs run as whoever is
 		 * logged on (or root if nobody is logged in yet!)
 		 */
-		*((uint32_t *)attrbufptr)++ =
-			(cap->ca_uid == UNKNOWNUID) ? cap->ca_mode & ~(S_ISUID | S_ISGID) : cap->ca_mode;
-	}
-	if (ATTR_CMN_NAMEDATTRCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = 0;
-	}
-	if (ATTR_CMN_NAMEDATTRLIST & attr) {
-		attrlength = 0;
-		((struct attrreference *)attrbufptr)->attr_dataoffset = 0;
-		((struct attrreference *)attrbufptr)->attr_length = attrlength;
-		/*
-		 * Advance beyond the space just allocated and
-		 * round up to the next 4-byte boundary:
-		 */
-		(char *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-		++((struct attrreference *)attrbufptr);
+		*((u_int32_t *)attrbufptr) = (cap->ca_uid == UNKNOWNUID) ?
+			cap->ca_mode & ~(S_ISUID | S_ISGID) : cap->ca_mode;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_FLAGS & attr) {
-		*((uint32_t *)attrbufptr)++ = cap->ca_flags;
+		*((u_int32_t *)attrbufptr) = cap->ca_flags;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_CMN_USERACCESS & attr) {
-		*((uint32_t *)attrbufptr)++ =
-			DerivePermissionSummary(cap->ca_uid, cap->ca_gid,
-				cap->ca_mode, mp, proc_ucred(current_proc()),
-				current_proc());
+		u_int32_t user_access;
+
+		/* Take the long path when we have an ACL */
+		if ((vp != NULLVP) && (cap->ca_recflags & kHFSHasSecurityMask)) {
+			user_access = hfs_real_user_access(vp, abp->ab_context);
+		} else {
+			user_access = DerivePermissionSummary(cap->ca_uid, cap->ca_gid,
+			                  cap->ca_mode, mp, proc_ucred(current_proc()), 0);
+		}
+		/* Also consider READ-ONLY file system. */
+		if (vfs_flags(mp) & MNT_RDONLY) {
+			user_access &= ~W_OK;
+		}
+		/* Locked objects are not writable either */
+		if ((cap->ca_flags & UF_IMMUTABLE) && (vfs_context_suser(abp->ab_context) != 0))
+			user_access &= ~W_OK;
+		if ((cap->ca_flags & SF_IMMUTABLE) && (vfs_context_suser(abp->ab_context) == 0))
+			user_access &= ~W_OK;
+
+		*((u_int32_t *)attrbufptr) = user_access;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
+	}
+	if (ATTR_CMN_FILEID & attr) {
+		*((u_int64_t *)attrbufptr) = cap->ca_fileid;
+		attrbufptr = ((u_int64_t *)attrbufptr) + 1;
+	}
+	if (ATTR_CMN_PARENTID & attr) {
+		*((u_int64_t *)attrbufptr) = cdp->cd_parentcnid;
+		attrbufptr = ((u_int64_t *)attrbufptr) + 1;
 	}
 	
 	*abp->ab_attrbufpp = attrbufptr;
@@ -1615,26 +747,40 @@ packdirattr(
 {
 	attrgroup_t attr = abp->ab_attrlist->dirattr;
 	void *attrbufptr = *abp->ab_attrbufpp;
-	
-	if (ATTR_DIR_LINKCOUNT & attr)
-		*((uint32_t *)attrbufptr)++ = cattrp->ca_nlink;
+	u_int32_t entries;
+
+	/*
+	 * The DIR_LINKCOUNT is the count of real directory hard links.
+	 * (i.e. its not the sum of the implied "." and ".." references
+	 *  typically used in stat's st_nlink field)
+	 */
+	if (ATTR_DIR_LINKCOUNT & attr) {
+		*((u_int32_t *)attrbufptr) = cattrp->ca_linkcount;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
+	}
 	if (ATTR_DIR_ENTRYCOUNT & attr) {
-		uint32_t entries = cattrp->ca_entries;
+		entries = cattrp->ca_entries;
 
 		if (descp->cd_parentcnid == kHFSRootParentID) {
-			if (hfsmp->hfs_privdir_desc.cd_cnid != 0)
+			if (hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid != 0)
+				--entries;	    /* hide private dir */
+			if (hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid != 0)
 				--entries;	    /* hide private dir */
-			if (hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))
+			if (hfsmp->jnl ||
+			    ((hfsmp->vcbAtrb & kHFSVolumeJournaledMask) &&
+			     (hfsmp->hfs_flags & HFS_READ_ONLY)))
 				entries -= 2;	/* hide the journal files */
 		}
 
-		*((uint32_t *)attrbufptr)++ = entries;
+		*((u_int32_t *)attrbufptr) = entries;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_DIR_MOUNTSTATUS & attr) {
 		if (vp != NULL && vnode_mountedhere(vp) != NULL)
-			*((uint32_t *)attrbufptr)++ = DIR_MNTSTATUS_MNTPOINT;
+			*((u_int32_t *)attrbufptr) = DIR_MNTSTATUS_MNTPOINT;
 		else
-			*((uint32_t *)attrbufptr)++ = 0;
+			*((u_int32_t *)attrbufptr) = 0;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	*abp->ab_attrbufpp = attrbufptr;
 }
@@ -1650,263 +796,60 @@ packfileattr(
 	attrgroup_t attr = abp->ab_attrlist->fileattr;
 	void *attrbufptr = *abp->ab_attrbufpp;
 	void *varbufptr = *abp->ab_varbufpp;
-	uint32_t attrlength;
-	uint32_t allocblksize;
+	u_int32_t allocblksize;
 
 	allocblksize = HFSTOVCB(hfsmp)->blockSize;
 
 	if (ATTR_FILE_LINKCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = cattrp->ca_nlink;
+		*((u_int32_t *)attrbufptr) = cattrp->ca_linkcount;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_TOTALSIZE & attr) {
-		*((off_t *)attrbufptr)++ = datafork->cf_size + rsrcfork->cf_size;
+		*((off_t *)attrbufptr) = datafork->cf_size + rsrcfork->cf_size;
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_ALLOCSIZE & attr) {
-		*((off_t *)attrbufptr)++ =
+		*((off_t *)attrbufptr) =
 			(off_t)cattrp->ca_blocks * (off_t)allocblksize;
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_IOBLOCKSIZE & attr) {
-		*((uint32_t *)attrbufptr)++ = hfsmp->hfs_logBlockSize;
+		*((u_int32_t *)attrbufptr) = hfsmp->hfs_logBlockSize;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_CLUMPSIZE & attr) {
-		*((uint32_t *)attrbufptr)++ = HFSTOVCB(hfsmp)->vcbClpSiz;
+		*((u_int32_t *)attrbufptr) = hfsmp->vcbClpSiz;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_DEVTYPE & attr) {
 		if (S_ISBLK(cattrp->ca_mode) || S_ISCHR(cattrp->ca_mode))
-			*((uint32_t *)attrbufptr)++ = (uint32_t)cattrp->ca_rdev;
+			*((u_int32_t *)attrbufptr) = (u_int32_t)cattrp->ca_rdev;
 		else
-			*((uint32_t *)attrbufptr)++ = 0;
-	}
-	if (ATTR_FILE_FILETYPE & attr) {
-		*((uint32_t *)attrbufptr)++ = 0;
-	}
-	if (ATTR_FILE_FORKCOUNT & attr) {
-		*((uint32_t *)attrbufptr)++ = 2;
-	}
-	if (ATTR_FILE_FORKLIST & attr) {
-		attrlength = 0;
-		((struct attrreference *)attrbufptr)->attr_dataoffset = 0;
-		((struct attrreference *)attrbufptr)->attr_length = attrlength;	
-		/*
-		 * Advance beyond the space just allocated and
-		 * round up to the next 4-byte boundary:
-		 */
-		(char *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-		++((struct attrreference *)attrbufptr);
+			*((u_int32_t *)attrbufptr) = 0;
+		attrbufptr = ((u_int32_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_DATALENGTH & attr) {
-		*((off_t *)attrbufptr)++ = datafork->cf_size;
+		*((off_t *)attrbufptr) = datafork->cf_size;
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_DATAALLOCSIZE & attr) {
-		*((off_t *)attrbufptr)++ =
+		*((off_t *)attrbufptr) =
 			(off_t)datafork->cf_blocks * (off_t)allocblksize;
-	}
-	if (ATTR_FILE_DATAEXTENTS & attr) {
-		bcopy(&datafork->cf_extents, attrbufptr, sizeof(extentrecord));
-		(char *)attrbufptr += sizeof(extentrecord);
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_RSRCLENGTH & attr) {
-		*((off_t *)attrbufptr)++ = rsrcfork->cf_size;
+		*((off_t *)attrbufptr) = rsrcfork->cf_size;
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	if (ATTR_FILE_RSRCALLOCSIZE & attr) {
-		*((off_t *)attrbufptr)++ =
+		*((off_t *)attrbufptr) =
 			(off_t)rsrcfork->cf_blocks * (off_t)allocblksize;
-	}
-	if (ATTR_FILE_RSRCEXTENTS & attr) {
-		bcopy(&rsrcfork->cf_extents, attrbufptr, sizeof(extentrecord));
-		(char *)attrbufptr += sizeof(extentrecord);
+		attrbufptr = ((off_t *)attrbufptr) + 1;
 	}
 	*abp->ab_attrbufpp = attrbufptr;
 	*abp->ab_varbufpp = varbufptr;
 }
 
-#if 0
-static int
-unpackattrblk(struct attrblock *abp, struct vnode *vp)
-{
-	struct attrlist *attrlistp = abp->ab_attrlist;
-	int error;
-
-	if (attrlistp->volattr) {
-		error = unpackvolattr(abp, VTOHFS(vp), vp);
-		if (error)
-			return (error);
-	} else if (attrlistp->commonattr) {
-		unpackcommonattr(abp, vp);
-	}
-	return (0);
-}
-
-
-static void
-unpackcommonattr(
-	struct attrblock *abp,
-	struct vnode *vp)
-{
-	attrgroup_t attr = abp->ab_attrlist->commonattr;
-	void *attrbufptr = *abp->ab_attrbufpp;
-	struct cnode *cp = VTOC(vp);
-	boolean_t is_64_bit = proc_is64bit(current_proc());
-
-	if (ATTR_CMN_SCRIPT & attr) {
-		cp->c_encoding = (u_int32_t)*((text_encoding_t *)attrbufptr)++;
-		hfs_setencodingbits(VTOHFS(vp), cp->c_encoding);
-	}
-	if (ATTR_CMN_CRTIME & attr) {
-	    if (is_64_bit) {
-            cp->c_itime = ((struct user_timespec *)attrbufptr)->tv_sec;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            cp->c_itime = ((struct timespec *)attrbufptr)->tv_sec;
-            ++((struct timespec *)attrbufptr);
-	    }
-	}
-	if (ATTR_CMN_MODTIME & attr) {
-		cp->c_mtime = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-		cp->c_touch_modtime = FALSE;
-	}
-	if (ATTR_CMN_CHGTIME & attr) {
-		cp->c_ctime = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-		cp->c_touch_chgtime = FALSE;
-	}
-	if (ATTR_CMN_ACCTIME & attr) {
-		cp->c_atime = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-		cp->c_touch_acctime = FALSE;
-	}
-	if (ATTR_CMN_BKUPTIME & attr) {
-		cp->c_btime = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-	}
-	if (ATTR_CMN_FNDRINFO & attr) {
-		bcopy(attrbufptr, &cp->c_attr.ca_finderinfo,
-			sizeof(cp->c_attr.ca_finderinfo));
-		(char *)attrbufptr += sizeof(cp->c_attr.ca_finderinfo);
-	}
-	if (ATTR_CMN_OWNERID & attr) {
-		if (VTOVCB(vp)->vcbSigWord == kHFSPlusSigWord) {
-			u_int32_t uid = (u_int32_t)*((uid_t *)attrbufptr)++;
-			if (uid != (uid_t)VNOVAL)
-				cp->c_uid = uid;
-		} else {
-			((uid_t *)attrbufptr)++;
-		}
-	}
-	if (ATTR_CMN_GRPID & attr) {
-		u_int32_t gid = (u_int32_t)*((gid_t *)attrbufptr)++;
-		if (VTOVCB(vp)->vcbSigWord == kHFSPlusSigWord) {
-		    if (gid != (gid_t)VNOVAL)
-			cp->c_gid = gid;
-		}
-	}
-	if (ATTR_CMN_ACCESSMASK & attr) {
-        	u_int16_t mode = (u_int16_t)*((uint32_t *)attrbufptr)++;
-        	if (VTOVCB(vp)->vcbSigWord == kHFSPlusSigWord) {
-			if (mode != (mode_t)VNOVAL) {
-                		cp->c_mode &= ~ALLPERMS;
-                		cp->c_mode |= (mode & ALLPERMS);
-			}
-        	}
-	}
-	if (ATTR_CMN_FLAGS & attr) {
-		uint32_t flags = *((uint32_t *)attrbufptr)++;
-		/*
-		 * Flags are settable only on HFS+ volumes.  A special
-		 * exception is made for the IMMUTABLE flags
-		 * (SF_IMMUTABLE and UF_IMMUTABLE), which can be set on
-		 * HFS volumes as well:
-		 */
-		if ((VTOVCB(vp)->vcbSigWord == kHFSPlusSigWord) ||
-		    ((VTOVCB(vp)->vcbSigWord == kHFSSigWord) &&
-		     ((flags & ~IMMUTABLE) == 0))) {
-			if (flags != (uint32_t)VNOVAL) {
-				cp->c_flags = flags;
-			}
-		}
-	}
-	*abp->ab_attrbufpp = attrbufptr;
-}
-
-
-static int
-unpackvolattr(
-	struct attrblock *abp,
-	struct hfsmount *hfsmp,
-	struct vnode *root_vp)
-{
-	void *attrbufptr = *abp->ab_attrbufpp;
-	attrgroup_t attr;
-	int error = 0;
-	boolean_t is_64_bit = proc_is64bit(current_proc());
-
-	HFS_MOUNT_LOCK(hfsmp, TRUE);
-
-	attr = abp->ab_attrlist->commonattr;
-	if (attr == 0)
-		goto volattr;
-
-	if (ATTR_CMN_SCRIPT & attr) {
-		hfsmp->volumeNameEncodingHint =
-				(u_int32_t)*(((text_encoding_t *)attrbufptr)++);
-	}
-	if (ATTR_CMN_CRTIME & attr) {
-	    if (is_64_bit) {
-            hfsmp->vcbCrDate = ((struct user_timespec *)attrbufptr)->tv_sec;
-            ++((struct user_timespec *)attrbufptr);
-	    }
-	    else {
-            hfsmp->vcbCrDate = ((struct timespec *)attrbufptr)->tv_sec;
-            ++((struct timespec *)attrbufptr);
-	    }
-		
-		/* The volume's create date comes from the root directory */
-		VTOC(root_vp)->c_itime = hfsmp->vcbCrDate;
-		VTOC(root_vp)->c_flag |= C_MODIFIED;
-		/*
-		 * XXX Should we also do a relative change to the
-		 * the volume header's create date in local time?
-		 */
-	}
-	if (ATTR_CMN_MODTIME & attr) {
-		hfsmp->vcbLsMod = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-	}
-	if (ATTR_CMN_BKUPTIME & attr) {
-		hfsmp->vcbVolBkUp = ((struct timespec *)attrbufptr)->tv_sec;
-		++((struct timespec *)attrbufptr);
-	}
-	if (ATTR_CMN_FNDRINFO & attr) {
-		bcopy(attrbufptr, &hfsmp->vcbFndrInfo, sizeof(hfsmp->vcbFndrInfo));
-		(char *)attrbufptr += sizeof(hfsmp->vcbFndrInfo);
-	}
-
-volattr:	
-	attr = abp->ab_attrlist->volattr & ~ATTR_VOL_INFO;
-	/*
-	 * XXX - no validation is done on the name!
-	 * It could be empty or garbage (bad UTF-8).
-	 */
-	if (ATTR_VOL_NAME & attr) {
-		attrreference_t * attr_refp = (attrreference_t *) attrbufptr;
-		
-		error = copystr(((char *)attrbufptr) + attr_refp->attr_dataoffset,
-		                hfsmp->vcbVN, MIN(attr_refp->attr_length, sizeof(hfsmp->vcbVN)),
-		                NULL);
-		if (error == 0)
-			(char *)attrbufptr += sizeof(struct attrreference);
-	}
-	*abp->ab_attrbufpp = attrbufptr;
-
-	hfsmp->vcbFlags |= 0xFF00;
-	HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-
-	return (error);
-}
-#endif
-
 /*
  * Calculate the total size of an attribute block.
  */
@@ -1924,55 +867,20 @@ hfs_attrblksize(struct attrlist *attrlist)
     else
         sizeof_timespec = sizeof(struct timespec);
 
-#if ((ATTR_CMN_NAME | ATTR_CMN_DEVID | ATTR_CMN_FSID | ATTR_CMN_OBJTYPE	|  \
-      ATTR_CMN_OBJTAG | ATTR_CMN_OBJID | ATTR_CMN_OBJPERMANENTID |         \
-      ATTR_CMN_PAROBJID | ATTR_CMN_SCRIPT | ATTR_CMN_CRTIME |              \
-      ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | ATTR_CMN_ACCTIME |             \
-      ATTR_CMN_BKUPTIME | ATTR_CMN_FNDRINFO | ATTR_CMN_OWNERID |           \
-      ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK | ATTR_CMN_NAMEDATTRCOUNT |     \
-      ATTR_CMN_NAMEDATTRLIST | ATTR_CMN_FLAGS | ATTR_CMN_USERACCESS)       \
-      != ATTR_CMN_VALIDMASK)
-#error	hfs_attrblksize: Missing bits in common mask computation!
-#endif
 	DBG_ASSERT((attrlist->commonattr & ~ATTR_CMN_VALIDMASK) == 0);
 
-#if ((ATTR_VOL_FSTYPE | ATTR_VOL_SIGNATURE | ATTR_VOL_SIZE |                \
-      ATTR_VOL_SPACEFREE | ATTR_VOL_SPACEAVAIL | ATTR_VOL_MINALLOCATION |   \
-      ATTR_VOL_ALLOCATIONCLUMP | ATTR_VOL_IOBLOCKSIZE |                     \
-      ATTR_VOL_OBJCOUNT | ATTR_VOL_FILECOUNT | ATTR_VOL_DIRCOUNT |          \
-      ATTR_VOL_MAXOBJCOUNT | ATTR_VOL_MOUNTPOINT | ATTR_VOL_NAME |          \
-      ATTR_VOL_MOUNTFLAGS | ATTR_VOL_INFO | ATTR_VOL_MOUNTEDDEVICE |        \
-      ATTR_VOL_ENCODINGSUSED | ATTR_VOL_CAPABILITIES | ATTR_VOL_ATTRIBUTES) \
-      != ATTR_VOL_VALIDMASK)
-#error	hfs_attrblksize: Missing bits in volume mask computation!
-#endif
 	DBG_ASSERT((attrlist->volattr & ~ATTR_VOL_VALIDMASK) == 0);
 
-#if ((ATTR_DIR_LINKCOUNT | ATTR_DIR_ENTRYCOUNT | ATTR_DIR_MOUNTSTATUS)  \
-      != ATTR_DIR_VALIDMASK)
-#error	hfs_attrblksize: Missing bits in directory mask computation!
-#endif
 	DBG_ASSERT((attrlist->dirattr & ~ATTR_DIR_VALIDMASK) == 0);
 
-#if ((ATTR_FILE_LINKCOUNT | ATTR_FILE_TOTALSIZE | ATTR_FILE_ALLOCSIZE |        \
-      ATTR_FILE_IOBLOCKSIZE | ATTR_FILE_CLUMPSIZE | ATTR_FILE_DEVTYPE |        \
-      ATTR_FILE_FILETYPE | ATTR_FILE_FORKCOUNT | ATTR_FILE_FORKLIST |          \
-      ATTR_FILE_DATALENGTH | ATTR_FILE_DATAALLOCSIZE | ATTR_FILE_DATAEXTENTS | \
-      ATTR_FILE_RSRCLENGTH | ATTR_FILE_RSRCALLOCSIZE | ATTR_FILE_RSRCEXTENTS)  \
-      != ATTR_FILE_VALIDMASK)
-#error	hfs_attrblksize: Missing bits in file mask computation!
-#endif
 	DBG_ASSERT((attrlist->fileattr & ~ATTR_FILE_VALIDMASK) == 0);
 
-#if ((ATTR_FORK_TOTALSIZE | ATTR_FORK_ALLOCSIZE) != ATTR_FORK_VALIDMASK)
-#error	hfs_attrblksize: Missing bits in fork mask computation!
-#endif
 	DBG_ASSERT((attrlist->forkattr & ~ATTR_FORK_VALIDMASK) == 0);
 
 	size = 0;
 	
 	if ((a = attrlist->commonattr) != 0) {
-        if (a & ATTR_CMN_NAME) size += sizeof(struct attrreference);
+		if (a & ATTR_CMN_NAME) size += sizeof(struct attrreference);
 		if (a & ATTR_CMN_DEVID) size += sizeof(dev_t);
 		if (a & ATTR_CMN_FSID) size += sizeof(fsid_t);
 		if (a & ATTR_CMN_OBJTYPE) size += sizeof(fsobj_type_t);
@@ -1981,81 +889,92 @@ hfs_attrblksize(struct attrlist *attrlist)
 		if (a & ATTR_CMN_OBJPERMANENTID) size += sizeof(fsobj_id_t);
 		if (a & ATTR_CMN_PAROBJID) size += sizeof(fsobj_id_t);
 		if (a & ATTR_CMN_SCRIPT) size += sizeof(text_encoding_t);
-        if (a & ATTR_CMN_CRTIME) size += sizeof_timespec;
-        if (a & ATTR_CMN_MODTIME) size += sizeof_timespec;
-        if (a & ATTR_CMN_CHGTIME) size += sizeof_timespec;
-        if (a & ATTR_CMN_ACCTIME) size += sizeof_timespec;
-        if (a & ATTR_CMN_BKUPTIME) size += sizeof_timespec;
+		if (a & ATTR_CMN_CRTIME) size += sizeof_timespec;
+		if (a & ATTR_CMN_MODTIME) size += sizeof_timespec;
+		if (a & ATTR_CMN_CHGTIME) size += sizeof_timespec;
+		if (a & ATTR_CMN_ACCTIME) size += sizeof_timespec;
+		if (a & ATTR_CMN_BKUPTIME) size += sizeof_timespec;
 		if (a & ATTR_CMN_FNDRINFO) size += 32 * sizeof(u_int8_t);
 		if (a & ATTR_CMN_OWNERID) size += sizeof(uid_t);
 		if (a & ATTR_CMN_GRPID) size += sizeof(gid_t);
-		if (a & ATTR_CMN_ACCESSMASK) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_NAMEDATTRCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_NAMEDATTRLIST) size += sizeof(struct attrreference);
-		if (a & ATTR_CMN_FLAGS) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_USERACCESS) size += sizeof(uint32_t);
-	};
-	if ((a = attrlist->volattr) != 0) {
-		if (a & ATTR_VOL_FSTYPE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_SIGNATURE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_SIZE) size += sizeof(off_t);
-		if (a & ATTR_VOL_SPACEFREE) size += sizeof(off_t);
-		if (a & ATTR_VOL_SPACEAVAIL) size += sizeof(off_t);
-		if (a & ATTR_VOL_MINALLOCATION) size += sizeof(off_t);
-		if (a & ATTR_VOL_ALLOCATIONCLUMP) size += sizeof(off_t);
-		if (a & ATTR_VOL_IOBLOCKSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_OBJCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_FILECOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_DIRCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_MAXOBJCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_MOUNTPOINT) size += sizeof(struct attrreference);
-		if (a & ATTR_VOL_NAME) size += sizeof(struct attrreference);
-		if (a & ATTR_VOL_MOUNTFLAGS) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_MOUNTEDDEVICE) size += sizeof(struct attrreference);
-		if (a & ATTR_VOL_ENCODINGSUSED) size += sizeof(unsigned long long);
-		if (a & ATTR_VOL_CAPABILITIES) size += sizeof(vol_capabilities_attr_t);
-		if (a & ATTR_VOL_ATTRIBUTES) size += sizeof(vol_attributes_attr_t);
-	};
+		if (a & ATTR_CMN_ACCESSMASK) size += sizeof(u_int32_t);
+		if (a & ATTR_CMN_FLAGS) size += sizeof(u_int32_t);
+		if (a & ATTR_CMN_USERACCESS) size += sizeof(u_int32_t);
+		if (a & ATTR_CMN_FILEID) size += sizeof(u_int64_t);
+		if (a & ATTR_CMN_PARENTID) size += sizeof(u_int64_t);
+	}
 	if ((a = attrlist->dirattr) != 0) {
-		if (a & ATTR_DIR_LINKCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_DIR_ENTRYCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_DIR_MOUNTSTATUS) size += sizeof(uint32_t);
-	};
+		if (a & ATTR_DIR_LINKCOUNT) size += sizeof(u_int32_t);
+		if (a & ATTR_DIR_ENTRYCOUNT) size += sizeof(u_int32_t);
+		if (a & ATTR_DIR_MOUNTSTATUS) size += sizeof(u_int32_t);
+	}
 	if ((a = attrlist->fileattr) != 0) {
-		if (a & ATTR_FILE_LINKCOUNT) size += sizeof(uint32_t);
+		if (a & ATTR_FILE_LINKCOUNT) size += sizeof(u_int32_t);
 		if (a & ATTR_FILE_TOTALSIZE) size += sizeof(off_t);
 		if (a & ATTR_FILE_ALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_IOBLOCKSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_CLUMPSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_DEVTYPE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FILETYPE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FORKCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FORKLIST) size += sizeof(struct attrreference);
+		if (a & ATTR_FILE_IOBLOCKSIZE) size += sizeof(u_int32_t);
+		if (a & ATTR_FILE_CLUMPSIZE) size += sizeof(u_int32_t);
+		if (a & ATTR_FILE_DEVTYPE) size += sizeof(u_int32_t);
 		if (a & ATTR_FILE_DATALENGTH) size += sizeof(off_t);
 		if (a & ATTR_FILE_DATAALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_DATAEXTENTS) size += sizeof(extentrecord);
 		if (a & ATTR_FILE_RSRCLENGTH) size += sizeof(off_t);
 		if (a & ATTR_FILE_RSRCALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_RSRCEXTENTS) size += sizeof(extentrecord);
-	};
-	if ((a = attrlist->forkattr) != 0) {
-		if (a & ATTR_FORK_TOTALSIZE) size += sizeof(off_t);
-		if (a & ATTR_FORK_ALLOCSIZE) size += sizeof(off_t);
-	};
+	}
 
-	return size;
+	return (size);
 }
 
+#define KAUTH_DIR_WRITE_RIGHTS		(KAUTH_VNODE_ACCESS | KAUTH_VNODE_ADD_FILE | \
+                                	 KAUTH_VNODE_ADD_SUBDIRECTORY | \
+                                	 KAUTH_VNODE_DELETE_CHILD)
+
+#define KAUTH_DIR_READ_RIGHTS		(KAUTH_VNODE_ACCESS | KAUTH_VNODE_LIST_DIRECTORY)
+
+#define KAUTH_DIR_EXECUTE_RIGHTS	(KAUTH_VNODE_ACCESS | KAUTH_VNODE_SEARCH)
+
+#define KAUTH_FILE_WRITE_RIGHTS		(KAUTH_VNODE_ACCESS | KAUTH_VNODE_WRITE_DATA)
+
+#define KAUTH_FILE_READRIGHTS		(KAUTH_VNODE_ACCESS | KAUTH_VNODE_READ_DATA)
+
+#define KAUTH_FILE_EXECUTE_RIGHTS	(KAUTH_VNODE_ACCESS | KAUTH_VNODE_EXECUTE)
+
+
+/*
+ * Compute the same [expensive] user_access value as getattrlist does
+ */
+static u_int32_t
+hfs_real_user_access(vnode_t vp, vfs_context_t ctx)
+{
+	u_int32_t user_access = 0;
+
+	if (vnode_isdir(vp)) {
+		if (vnode_authorize(vp, NULLVP, KAUTH_DIR_WRITE_RIGHTS, ctx) == 0)
+			user_access |= W_OK;
+		if (vnode_authorize(vp, NULLVP, KAUTH_DIR_READ_RIGHTS, ctx) == 0)
+			user_access |= R_OK;
+		if (vnode_authorize(vp, NULLVP, KAUTH_DIR_EXECUTE_RIGHTS, ctx) == 0)
+			user_access |= X_OK;
+	} else {
+		if (vnode_authorize(vp, NULLVP, KAUTH_FILE_WRITE_RIGHTS, ctx) == 0)
+			user_access |= W_OK;
+		if (vnode_authorize(vp, NULLVP, KAUTH_FILE_READRIGHTS, ctx) == 0)
+			user_access |= R_OK;
+		if (vnode_authorize(vp, NULLVP, KAUTH_FILE_EXECUTE_RIGHTS, ctx) == 0)
+			user_access |= X_OK;
+	}
+	return (user_access);
+}
+		
 
 __private_extern__
 unsigned long
 DerivePermissionSummary(uid_t obj_uid, gid_t obj_gid, mode_t obj_mode,
-		struct mount *mp, kauth_cred_t cred, struct proc *p)
+		struct mount *mp, kauth_cred_t cred, __unused struct proc *p)
 {
 	unsigned long permissions;
 
 	if (obj_uid == UNKNOWNUID)
-		obj_uid = kauth_cred_getuid(proc_ucred(p));
+		obj_uid = kauth_cred_getuid(cred);
 
 	/* User id 0 (root) always gets access. */
 	if (!suser(cred, NULL)) {
@@ -2064,7 +983,7 @@ DerivePermissionSummary(uid_t obj_uid, gid_t obj_gid, mode_t obj_mode,
 	};
 
 	/* Otherwise, check the owner. */
-	if (hfs_owner_rights(VFSTOHFS(mp), obj_uid, cred, p, false) == 0) {
+	if (hfs_owner_rights(VFSTOHFS(mp), obj_uid, cred, NULL, false) == 0) {
 		permissions = ((unsigned long)obj_mode & S_IRWXU) >> 6;
 		goto Exit;
 	}
diff --git a/bsd/hfs/hfs_attrlist.h b/bsd/hfs/hfs_attrlist.h
index e4e203eaf..990adf227 100644
--- a/bsd/hfs/hfs_attrlist.h
+++ b/bsd/hfs/hfs_attrlist.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _HFS_ATTRLIST_H_
 #define _HFS_ATTRLIST_H_
@@ -39,25 +45,45 @@ struct attrblock {
 	void **		  ab_varbufpp;
 	int		  ab_flags;
 	int		  ab_blocksize;
+	vfs_context_t	  ab_context;
 };
 
+/* 
+ * The following define the attributes that HFS supports:
+ */
 
-#define	ATTR_OWNERSHIP_SETMASK	(ATTR_CMN_OWNERID | ATTR_CMN_GRPID | \
-	ATTR_CMN_ACCESSMASK | ATTR_CMN_FLAGS | ATTR_CMN_CRTIME |     \
-	ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME | ATTR_CMN_ACCTIME)
+#define HFS_ATTR_CMN_VALID				\
+	(ATTR_CMN_NAME | ATTR_CMN_DEVID	|		\
+	 ATTR_CMN_FSID | ATTR_CMN_OBJTYPE |		\
+	 ATTR_CMN_OBJTAG | ATTR_CMN_OBJID |		\
+	 ATTR_CMN_OBJPERMANENTID | ATTR_CMN_PAROBJID |	\
+	 ATTR_CMN_SCRIPT | ATTR_CMN_CRTIME |		\
+	 ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME |		\
+	 ATTR_CMN_ACCTIME | ATTR_CMN_BKUPTIME |		\
+	 ATTR_CMN_FNDRINFO |ATTR_CMN_OWNERID |		\
+	 ATTR_CMN_GRPID | ATTR_CMN_ACCESSMASK |		\
+	 ATTR_CMN_FLAGS | ATTR_CMN_USERACCESS |		\
+	 ATTR_CMN_EXTENDED_SECURITY | ATTR_CMN_UUID |	\
+	 ATTR_CMN_GRPUUID | ATTR_CMN_FILEID |		\
+	 ATTR_CMN_PARENTID )
 
-#define ATTR_DATAFORK_MASK	(ATTR_FILE_TOTALSIZE | \
-    ATTR_FILE_DATALENGTH | ATTR_FILE_DATAALLOCSIZE | ATTR_FILE_DATAEXTENTS)
+#define HFS_ATTR_DIR_VALID				\
+	(ATTR_DIR_LINKCOUNT | ATTR_DIR_ENTRYCOUNT | ATTR_DIR_MOUNTSTATUS)
 
-#define ATTR_RSRCFORK_MASK	(ATTR_FILE_TOTALSIZE | \
-    ATTR_FILE_RSRCLENGTH | ATTR_FILE_RSRCALLOCSIZE | ATTR_FILE_RSRCEXTENTS)
+#define HFS_ATTR_FILE_VALID				  \
+	(ATTR_FILE_LINKCOUNT |ATTR_FILE_TOTALSIZE |	  \
+	 ATTR_FILE_ALLOCSIZE | ATTR_FILE_IOBLOCKSIZE |	  \
+	 ATTR_FILE_CLUMPSIZE | ATTR_FILE_DEVTYPE |	  \
+	 ATTR_FILE_FORKCOUNT | ATTR_FILE_FORKLIST |	  \
+	 ATTR_FILE_DATALENGTH | ATTR_FILE_DATAALLOCSIZE | \
+	 ATTR_FILE_RSRCLENGTH | ATTR_FILE_RSRCALLOCSIZE)
 
 
 extern int hfs_attrblksize(struct attrlist *attrlist);
 
 extern unsigned long DerivePermissionSummary(uid_t obj_uid, gid_t obj_gid,
 			mode_t obj_mode, struct mount *mp,
-			struct ucred *cred, struct proc *p);
+			kauth_cred_t cred, struct proc *p);
 
 extern void hfs_packattrblk(struct attrblock *abp, struct hfsmount *hfsmp,
 		struct vnode *vp, struct cat_desc *descp, struct cat_attr *attrp,
diff --git a/bsd/hfs/hfs_btreeio.c b/bsd/hfs/hfs_btreeio.c
index 1c6818aea..0c81c879f 100644
--- a/bsd/hfs/hfs_btreeio.c
+++ b/bsd/hfs/hfs_btreeio.c
@@ -1,28 +1,35 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/buf.h>
+#include <sys/buf_internal.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mount.h>
@@ -33,18 +40,22 @@
 #include "hfs_cnode.h"
 #include "hfs_dbg.h"
 #include "hfs_endian.h"
+#include "hfs_btreeio.h"
 
 #include "hfscommon/headers/FileMgrInternal.h"
 #include "hfscommon/headers/BTreesPrivate.h"
 
 #define FORCESYNCBTREEWRITES 0
 
+/* From bsd/vfs/vfs_bio.c */
+extern int bdwrite_internal(struct buf *, int);
 
 static int ClearBTNodes(struct vnode *vp, long blksize, off_t offset, off_t amount);
+static int btree_journal_modify_block_end(struct hfsmount *hfsmp, struct buf *bp);
 
 
 __private_extern__
-OSStatus SetBTreeBlockSize(FileReference vp, ByteCount blockSize, ItemCount minBlockCount)
+OSStatus SetBTreeBlockSize(FileReference vp, ByteCount blockSize, __unused ItemCount minBlockCount)
 {
 	BTreeControlBlockPtr	bTreePtr;
 	
@@ -61,7 +72,7 @@ OSStatus SetBTreeBlockSize(FileReference vp, ByteCount blockSize, ItemCount minB
 
 
 __private_extern__
-OSStatus GetBTreeBlock(FileReference vp, UInt32 blockNum, GetBlockOptions options, BlockDescriptor *block)
+OSStatus GetBTreeBlock(FileReference vp, u_int32_t blockNum, GetBlockOptions options, BlockDescriptor *block)
 {
     OSStatus	 retval = E_NONE;
     struct buf   *bp = NULL;
@@ -112,7 +123,7 @@ OSStatus GetBTreeBlock(FileReference vp, UInt32 blockNum, GetBlockOptions option
             	 * This is necessary on big endian since the test below won't trigger.
             	 */
                 retval = hfs_swap_BTNode (block, vp, kSwapBTNodeBigToHost);
-            } else if (*((UInt16 *)((char *)block->buffer + (block->blockSize - sizeof (UInt16)))) == 0x0e00) {
+            } else if (*((u_int16_t *)((char *)block->buffer + (block->blockSize - sizeof (u_int16_t)))) == 0x0e00) {
 				/*
 				 * The node was left in the cache in non-native order, so swap it.
 				 * This only happens on little endian, after the node is written
@@ -156,7 +167,7 @@ void ModifyBlockStart(FileReference vp, BlockDescPtr blockPtr)
 	
     bp = (struct buf *) blockPtr->blockHeader;
     if (bp == NULL) {
-		panic("ModifyBlockStart: null bp  for blockdescptr 0x%x?!?\n", blockPtr);
+		panic("ModifyBlockStart: null bp  for blockdescptr %p?!?\n", blockPtr);
 		return;
     }
 
@@ -164,9 +175,10 @@ void ModifyBlockStart(FileReference vp, BlockDescPtr blockPtr)
 	blockPtr->isModified = 1;
 }
 
-static int
-btree_journal_modify_block_end(struct hfsmount *hfsmp, struct buf *bp)
+static void
+btree_swap_node(struct buf *bp, __unused void *arg)
 {
+    //	struct hfsmount *hfsmp = (struct hfsmount *)arg;
 	int retval;
     struct vnode *vp = buf_vnode(bp);
     BlockDescriptor block;
@@ -179,12 +191,17 @@ btree_journal_modify_block_end(struct hfsmount *hfsmp, struct buf *bp)
     block.blockReadFromDisk = (buf_fromcache(bp) == 0);
     block.blockSize = buf_count(bp);
 
-    // XXXdbg have to swap the data before it goes in the journal
+    // swap the data now that this node is ready to go to disk
     retval = hfs_swap_BTNode (&block, vp, kSwapBTNodeHostToBig);
     if (retval)
-    	panic("btree_journal_modify_block_end: about to write corrupt node!\n");
+    	panic("btree_swap_node: about to write corrupt node!\n");
+}
+
 
-    return journal_modify_block_end(hfsmp->jnl, bp);
+static int
+btree_journal_modify_block_end(struct hfsmount *hfsmp, struct buf *bp)
+{
+    return journal_modify_block_end(hfsmp->jnl, bp, btree_swap_node, hfsmp);
 }
 
 
@@ -192,7 +209,6 @@ __private_extern__
 OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, ReleaseBlockOptions options)
 {
     struct hfsmount	*hfsmp = VTOHFS(vp);
-    extern int bdwrite_internal(struct buf *, int);
     OSStatus	retval = E_NONE;
     struct buf *bp = NULL;
 
@@ -215,7 +231,7 @@ OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, ReleaseBlock
         if (options & kForceWriteBlock) {
 			if (hfsmp->jnl) {
 				if (blockPtr->isModified == 0) {
-					panic("hfs: releaseblock: modified is 0 but forcewrite set! bp 0x%x\n", bp);
+					panic("hfs: releaseblock: modified is 0 but forcewrite set! bp %p\n", bp);
 				}
 
 				retval = btree_journal_modify_block_end(hfsmp, bp);
@@ -235,8 +251,6 @@ OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, ReleaseBlock
                  * isn't going to work.
                  *
                  */
-                extern int count_lock_queue(void);
-
                 /* Don't hog all the buffers... */
                 if (count_lock_queue() > kMaxLockedMetaBuffers) {
                      hfs_btsync(vp, HFS_SYNCTRANS);
@@ -253,7 +267,7 @@ OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, ReleaseBlock
              */
 			if (hfsmp->jnl) {
 				if (blockPtr->isModified == 0) {
-					panic("hfs: releaseblock: modified is 0 but markdirty set! bp 0x%x\n", bp);
+					panic("hfs: releaseblock: modified is 0 but markdirty set! bp %p\n", bp);
 				}
 				retval = btree_journal_modify_block_end(hfsmp, bp);
 				blockPtr->isModified = 0;
@@ -390,8 +404,8 @@ OSStatus ExtendBTreeFile(FileReference vp, FSSize minEOF, FSSize maxEOF)
 	if ((retval == 0) &&
 	    ((VCBTOHFS(vcb)->hfs_flags & HFS_METADATA_ZONE) == 0) &&
 	    (vcb->nextAllocation > startAllocation) &&
-	    ((vcb->nextAllocation + fileblocks) < vcb->totalBlocks)) {
-		vcb->nextAllocation += fileblocks;
+	    ((vcb->nextAllocation + fileblocks) < vcb->allocLimit)) {
+		HFS_UPDATE_NEXT_ALLOCATION(vcb, vcb->nextAllocation + fileblocks); 
 	}
 		
 	filePtr->fcbEOF = (u_int64_t)filePtr->ff_blocks * (u_int64_t)vcb->blockSize;
@@ -543,19 +557,15 @@ ClearBTNodes(struct vnode *vp, long blksize, off_t offset, off_t amount)
 
 extern char  hfs_attrname[];
 
-extern int  hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKey);
-
-int  hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecnt);
-
 /*
  * Create an HFS+ Attribute B-tree File.
  *
- * A journal transaction must be already started.
+ * No global resources should be held.
  */
 int
-hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecnt)
+hfs_create_attr_btree(struct hfsmount *hfsmp, u_int32_t nodesize, u_int32_t nodecnt)
 {
-	struct vnode* vp = NULL;
+	struct vnode* vp = NULLVP;
 	struct cat_desc cndesc;
 	struct cat_attr cnattr;
 	struct cat_fork cfork;
@@ -567,22 +577,45 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 	void * buffer;
 	u_int16_t *index;
 	u_int16_t  offset;
+	int intrans = 0;
 	int result;
+again:
+	/*
+	 * Serialize creation using HFS_CREATING_BTREE flag.
+	 */
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	if (hfsmp->hfs_flags & HFS_CREATING_BTREE) {
+			/* Someone else beat us, wait for them to finish. */
+			(void) msleep(hfsmp->hfs_attribute_cp, &hfsmp->hfs_mutex,
+			              PDROP | PINOD, "hfs_create_attr_btree", 0);
+			if (hfsmp->hfs_attribute_vp) {
+				return (0);
+			}
+			goto again;
+	}
+	hfsmp->hfs_flags |= HFS_CREATING_BTREE;
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
 
-	printf("Creating HFS+ Attribute B-tree File (%d nodes) on %s\n", nodecnt, hfsmp->vcbVN);
+	/* Check if were out of usable disk space. */
+	if ((hfs_freeblks(hfsmp, 1) == 0)) {
+		result = ENOSPC;
+		goto exit;
+	}
 
 	/*
 	 * Set up Attribute B-tree vnode
+	 * (this must be done before we start a transaction
+	 *  or take any system file locks)
 	 */
 	bzero(&cndesc, sizeof(cndesc));
 	cndesc.cd_parentcnid = kHFSRootParentID;
 	cndesc.cd_flags |= CD_ISMETA;
-	cndesc.cd_nameptr = hfs_attrname;
+	cndesc.cd_nameptr = (const u_int8_t *)hfs_attrname;
 	cndesc.cd_namelen = strlen(hfs_attrname);
 	cndesc.cd_cnid = kHFSAttributesFileID;
 
 	bzero(&cnattr, sizeof(cnattr));
-	cnattr.ca_nlink = 1;
+	cnattr.ca_linkcount = 1;
 	cnattr.ca_mode = S_IFREG;
 	cnattr.ca_fileid = cndesc.cd_cnid;
 
@@ -590,9 +623,9 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 	cfork.cf_clump = nodesize * nodecnt;
 
 	result = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork, &vp);
-	if (result)
-		return (result);
-
+	if (result) {
+		goto exit;
+	}
 	/*
 	 * Set up Attribute B-tree control block
 	 */
@@ -616,6 +649,13 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 	/*
 	 * Allocate some space
 	 */
+	if (hfs_start_transaction(hfsmp) != 0) {
+		result = EINVAL;
+		goto exit;
+	}
+	intrans = 1;
+
+	/* Note ExtendBTreeFile will acquire the necessary system file locks. */
 	result = ExtendBTreeFile(vp, nodesize, cfork.cf_clump);
 	if (result)
 		goto exit;
@@ -644,7 +684,7 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 		panic("hfs_create_attr_btree: bad buffer size (%d)\n", buf_size(bp));
 
 	bzero(buffer, nodesize);
-	index = (int16_t *)buffer;
+	index = (u_int16_t *)buffer;
 
 	/* FILL IN THE NODE DESCRIPTOR:  */
 	ndp = (BTNodeDescriptor *)buffer;
@@ -654,7 +694,7 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 	index[(nodesize / 2) - 1] = offset;
 
 	/* FILL IN THE HEADER RECORD:  */
-	bthp = (BTHeaderRec *)((UInt8 *)buffer + offset);
+	bthp = (BTHeaderRec *)((u_int8_t *)buffer + offset);
 	bthp->nodeSize     = nodesize;
 	bthp->totalNodes   = btcb->totalNodes;
 	bthp->freeNodes    = btcb->freeNodes;
@@ -684,21 +724,38 @@ hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecn
 	if (result)
 		goto exit;
 
-	/* Publish new btree file */
+	/* Update vp/cp for attribute btree */
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	hfsmp->hfs_attribute_cp = VTOC(vp);
 	hfsmp->hfs_attribute_vp = vp;
-	(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
 
+	(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
 exit:
-	hfs_unlock(VTOC(vp));
+	if (vp) {
+		hfs_unlock(VTOC(vp));
+	}
 	if (result) {
 		if (btcb) {
 			FREE (btcb, M_TEMP);
 		}
-		vnode_put(vp);
-	//	hfs_truncate();  /* XXX need to give back blocks */
+		if (vp) {
+			vnode_put(vp);
+		}
+		/* XXX need to give back blocks ? */
+	}
+	if (intrans) {
+		hfs_end_transaction(hfsmp);
 	}
-	return (result);
-}
 
+	/*
+	 * All done, clear HFS_CREATING_BTREE, and wake up any sleepers.
+	 */
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	hfsmp->hfs_flags &= ~HFS_CREATING_BTREE;
+	wakeup((caddr_t)hfsmp->hfs_attribute_cp);
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
 
+	return (result);
+}
 
diff --git a/bsd/hfs/hfs_btreeio.h b/bsd/hfs/hfs_btreeio.h
new file mode 100644
index 000000000..f307cca43
--- /dev/null
+++ b/bsd/hfs/hfs_btreeio.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _HFS_BTREEIO_H_
+#define _HFS_BTREEIO_H_
+
+#include <sys/appleapiopts.h>
+
+#ifdef KERNEL
+#ifdef __APPLE_API_PRIVATE
+
+#include "hfs.h"
+#include "hfscommon/headers/BTreesInternal.h"
+
+/* BTree accessor routines */
+extern OSStatus SetBTreeBlockSize(FileReference vp, ByteCount blockSize, 
+				ItemCount minBlockCount);
+
+extern OSStatus GetBTreeBlock(FileReference vp, u_int32_t blockNum, 
+				GetBlockOptions options, BlockDescriptor *block);
+
+extern OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, 
+				ReleaseBlockOptions options);
+
+extern OSStatus ExtendBTreeFile(FileReference vp, FSSize minEOF, FSSize maxEOF);
+
+extern void ModifyBlockStart(FileReference vp, BlockDescPtr blockPtr);
+
+int hfs_create_attr_btree(struct hfsmount *hfsmp, u_int32_t nodesize, u_int32_t nodecnt);
+
+#endif /* __APPLE_API_PRIVATE */
+#endif /* KERNEL */
+#endif /* ! _HFS_BTREEIO_H_ */
diff --git a/bsd/hfs/hfs_catalog.c b/bsd/hfs/hfs_catalog.c
index 75c7701ee..0a4953cbd 100644
--- a/bsd/hfs/hfs_catalog.c
+++ b/bsd/hfs/hfs_catalog.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/systm.h>
@@ -38,6 +44,7 @@
 #include "hfs_endian.h"
 
 #include "hfscommon/headers/BTreesInternal.h"
+#include "hfscommon/headers/BTreesPrivate.h"
 #include "hfscommon/headers/HFSUnicodeWrappers.h"
 
 
@@ -83,19 +90,12 @@ u_char modetodirtype[16] = {
 #define MODE_TO_DT(mode)  (modetodirtype[((mode) & S_IFMT) >> 12])
 
 
-static int cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantrsrc,
+static int cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, int allow_system_files, u_long hint, int wantrsrc,
                   struct cat_desc *descp, struct cat_attr *attrp, struct cat_fork *forkp, cnid_t *desc_cnid);
 
 static int cat_lookupmangled(struct hfsmount *hfsmp, struct cat_desc *descp, int wantrsrc,
                   struct cat_desc *outdescp, struct cat_attr *attrp, struct cat_fork *forkp);
 
-extern int mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str,
-                                UInt32 maxCharLen, UInt32 *unicodeChars);
-
-extern int unicode_to_hfs(ExtendedVCB *vcb, ByteCount srcLen,
-                          const u_int16_t* srcStr, Str31 dstStr, int retry);
-
-
 /* Internal catalog support routines */
 
 static int cat_findposition(const CatalogKey *ckp, const CatalogRecord *crp,
@@ -110,7 +110,7 @@ static int buildkey(struct hfsmount *hfsmp, struct cat_desc *descp,
 
 static void buildthreadkey(HFSCatalogNodeID parentID, int std_hfs, CatalogKey *key);
 
-static void buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding, CatalogRecord *crp, int *recordSize);
+static void buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding, CatalogRecord *crp, u_int32_t *recordSize);
 
 static int catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *state);
 
@@ -131,40 +131,40 @@ static int isadir(const CatalogRecord *crp);
 
 static int buildthread(void *keyp, void *recp, int std_hfs, int directory);
 
+static int cat_makealias(struct hfsmount *hfsmp, u_int32_t inode_num, struct HFSPlusCatalogFile *crp);
+
 
 __private_extern__
 int
-cat_preflight(struct hfsmount *hfsmp, catops_t ops, cat_cookie_t *cookie, struct proc *p)
+cat_preflight(struct hfsmount *hfsmp, catops_t ops, cat_cookie_t *cookie, __unused proc_t p)
 {
-	FCB *fcb;
-	int lockflags;
+	int lockflags = 0;
 	int result;
 
-	fcb = GetFileControlBlock(hfsmp->hfs_catalog_vp);
-
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
+	if (hfsmp->hfs_catalog_cp->c_lockowner != current_thread())
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 	 
-	result = BTReserveSpace(fcb, ops, (void*)cookie);
-	
-	hfs_systemfile_unlock(hfsmp, lockflags);
+	result = BTReserveSpace(hfsmp->hfs_catalog_cp->c_datafork, ops, (void*)cookie);
+
+	if (lockflags)
+		hfs_systemfile_unlock(hfsmp, lockflags);
 
 	return MacToVFSError(result);
 }
 
 __private_extern__
 void
-cat_postflight(struct hfsmount *hfsmp, cat_cookie_t *cookie, struct proc *p)
+cat_postflight(struct hfsmount *hfsmp, cat_cookie_t *cookie, __unused proc_t p)
 {
-	FCB *fcb;
-	int lockflags;
+	int lockflags = 0;
 
-	fcb = GetFileControlBlock(hfsmp->hfs_catalog_vp);
+	if (hfsmp->hfs_catalog_cp->c_lockowner != current_thread())
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
+	(void) BTReleaseReserve(hfsmp->hfs_catalog_cp->c_datafork, (void*)cookie);
 
-	(void) BTReleaseReserve(fcb, (void*)cookie);
-
-	hfs_systemfile_unlock(hfsmp, lockflags);
+	if (lockflags)
+		hfs_systemfile_unlock(hfsmp, lockflags);
 }
 
  
@@ -262,7 +262,7 @@ __private_extern__
 void
 cat_releasedesc(struct cat_desc *descp)
 {
-	char * name;
+	const u_int8_t * name;
 
 	if (descp == NULL)
 		return;
@@ -272,11 +272,11 @@ cat_releasedesc(struct cat_desc *descp)
 	    	name = descp->cd_nameptr;
 		descp->cd_nameptr = NULL;
 		descp->cd_namelen = 0;
-		descp->cd_flags &= ~CD_HASBUF;
-		vfs_removename(name);
+		vfs_removename((const char *)name);
 	}
 	descp->cd_nameptr = NULL;
 	descp->cd_namelen = 0;
+	descp->cd_flags &= ~CD_HASBUF;
 }
 
 /*
@@ -305,7 +305,7 @@ cat_lookup(struct hfsmount *hfsmp, struct cat_desc *descp, int wantrsrc,
 	if (result)
 		goto exit;
 
-	result = cat_lookupbykey(hfsmp, keyp, descp->cd_hint, wantrsrc, outdescp, attrp, forkp, desc_cnid);
+	result = cat_lookupbykey(hfsmp, keyp, 0, descp->cd_hint, wantrsrc, outdescp, attrp, forkp, desc_cnid);
 	
 	if (result == ENOENT) {
 		if (!std_hfs) {
@@ -341,7 +341,7 @@ cat_insertfilethread(struct hfsmount *hfsmp, struct cat_desc *descp)
 	struct BTreeIterator *iterator;
 	struct FSBufferDescriptor file_data;
 	struct HFSCatalogFile file_rec;
-	UInt16 datasize;
+	u_int16_t datasize;
 	FCB *fcb;
 	int result;
 
@@ -467,12 +467,12 @@ exit:
  */
 __private_extern__
 int
-cat_idlookup(struct hfsmount *hfsmp, cnid_t cnid, struct cat_desc *outdescp,
-                 struct cat_attr *attrp, struct cat_fork *forkp)
+cat_idlookup(struct hfsmount *hfsmp, cnid_t cnid, int allow_system_files,
+    struct cat_desc *outdescp, struct cat_attr *attrp, struct cat_fork *forkp)
 {
 	struct BTreeIterator * iterator;
 	FSBufferDescriptor btdata;
-	UInt16	datasize;
+	u_int16_t	datasize;
 	CatalogKey * keyp;
 	CatalogRecord * recp;
 	int result;
@@ -512,19 +512,21 @@ cat_idlookup(struct hfsmount *hfsmp, cnid_t cnid, struct cat_desc *outdescp,
 		goto exit;
 	}
 
-	result = cat_lookupbykey(hfsmp, keyp, 0, 0, outdescp, attrp, forkp, NULL);
-	if (!result && outdescp) {
+	result = cat_lookupbykey(hfsmp, keyp, allow_system_files, 0, 0, outdescp, attrp, forkp, NULL);
+	/* No corresponding file/folder record found for a thread record,
+	 * mark the volume inconsistent.
+	 */
+	if (result == 0 && outdescp) {
 		cnid_t dcnid = outdescp->cd_cnid;
 		/*
-		 * Just for sanity's sake, let's make sure that
+		 * Just for sanity's case, let's make sure that
 		 * the key in the thread matches the key in the record.
 		 */
 		if (cnid != dcnid) {
-			printf("Requested cnid (%d / 0x%08lx) != dcnid (%d / 0x%08lx)\n", cnid, cnid, dcnid, dcnid);			
+			printf("Requested cnid (%d / %08x) != dcnid (%d / %08x)\n", cnid, cnid, dcnid, dcnid);
 			result = ENOENT;
 		}
 	}
-
 exit:
 	FREE(recp, M_TEMP);
 	FREE(iterator, M_TEMP);
@@ -541,33 +543,32 @@ cat_lookupmangled(struct hfsmount *hfsmp, struct cat_desc *descp, int wantrsrc,
                   struct cat_desc *outdescp, struct cat_attr *attrp, struct cat_fork *forkp)
 {
 	cnid_t fileID;
-	int prefixlen;
+	u_int32_t prefixlen;
 	int result;
 	
 	if (wantrsrc)
 		return (ENOENT);
 
 	fileID = GetEmbeddedFileID(descp->cd_nameptr, descp->cd_namelen, &prefixlen);
-
 	if (fileID < (cnid_t)kHFSFirstUserCatalogNodeID)
 		return (ENOENT);
 
-	if(fileID == hfsmp->hfs_privdir_desc.cd_cnid ||
+	if (fileID == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+		fileID == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid ||
 		fileID == hfsmp->hfs_jnlfileid ||
 		fileID == hfsmp->hfs_jnlinfoblkid) {
 		return (ENOENT);
 	}
 
-	result = cat_idlookup(hfsmp, fileID, outdescp, attrp, forkp);
+	result = cat_idlookup(hfsmp, fileID, 0, outdescp, attrp, forkp);
 	if (result)
 		return (ENOENT);
-
 	/* It must be in the correct directory */
 	if (descp->cd_parentcnid != outdescp->cd_parentcnid)
 		goto falsematch;
 
-	if ((outdescp->cd_namelen < prefixlen) ||
-	    bcmp(outdescp->cd_nameptr, descp->cd_nameptr, prefixlen-6) != 0)
+	if (((u_int16_t)outdescp->cd_namelen < prefixlen) ||
+		bcmp(outdescp->cd_nameptr, descp->cd_nameptr, prefixlen-6) != 0)
 		goto falsematch;
 
 	return (0);
@@ -582,13 +583,13 @@ falsematch:
  * cat_lookupbykey - lookup a catalog node using a cnode key
  */
 static int
-cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantrsrc,
+cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, int allow_system_files, u_long hint, int wantrsrc,
                   struct cat_desc *descp, struct cat_attr *attrp, struct cat_fork *forkp, cnid_t *desc_cnid)
 {
 	struct BTreeIterator * iterator;
 	FSBufferDescriptor btdata;
 	CatalogRecord * recp;
-	UInt16  datasize;
+	u_int16_t  datasize;
 	int result;
 	int std_hfs;
 	u_long ilink = 0;
@@ -609,34 +610,45 @@ cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantr
 	if (result) 
 		goto exit;
 
-	/* Save the cnid now in case there's a hard link */
+	/* Save the cnid and encoding now in case there's a hard link */
 	cnid = getcnid(recp);
 	encoding = getencoding(recp);
 	hint = iterator->hint.nodeNum;
 
 	/* Hide the journal files (if any) */
 	if ((hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY))) &&
-		((cnid == hfsmp->hfs_jnlfileid) ||
-		 (cnid == hfsmp->hfs_jnlinfoblkid))) {
+		((cnid == hfsmp->hfs_jnlfileid) || (cnid == hfsmp->hfs_jnlinfoblkid)) &&
+		 !allow_system_files) {
 
 		result = ENOENT;
 		goto exit;
 	}
-
+	
 	/*
-	 * When a hardlink link is encountered, auto resolve it
+	 * When a hardlink link is encountered, auto resolve it.
+	 *
+	 * The catalog record will change, and possibly its type.
 	 */
 	if (!std_hfs
 	    && (attrp || forkp) 
 	    && (recp->recordType == kHFSPlusFileRecord)
-	    && (SWAP_BE32(recp->hfsPlusFile.userInfo.fdType) == kHardLinkFileType)
-	    && (SWAP_BE32(recp->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator)
-	    && ((to_bsd_time(recp->hfsPlusFile.createDate) == (time_t)HFSTOVCB(hfsmp)->vcbCrDate) ||
+	    && ((to_bsd_time(recp->hfsPlusFile.createDate) == (time_t)hfsmp->vcbCrDate) ||
 	        (to_bsd_time(recp->hfsPlusFile.createDate) == (time_t)hfsmp->hfs_metadata_createdate))) {
-
-		ilink = recp->hfsPlusFile.bsdInfo.special.iNodeNum;
-
-		(void) resolvelink(hfsmp, ilink, (struct HFSPlusCatalogFile *)recp);
+		int isdirlink = 0;
+		int isfilelink = 0;
+
+		if ((SWAP_BE32(recp->hfsPlusFile.userInfo.fdType) == kHardLinkFileType) &&
+		    (SWAP_BE32(recp->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator)) {
+			isfilelink = 1;
+		} else if ((recp->hfsPlusFile.flags & kHFSHasLinkChainMask) && 
+		           (SWAP_BE32(recp->hfsPlusFile.userInfo.fdType) == kHFSAliasType) && 
+			   (SWAP_BE32(recp->hfsPlusFile.userInfo.fdCreator) == kHFSAliasCreator)) {
+			isdirlink = 1;
+		}
+		if (isfilelink || isdirlink) {
+			ilink = recp->hfsPlusFile.hl_linkReference;
+			(void) cat_resolvelink(hfsmp, ilink, isdirlink, (struct HFSPlusCatalogFile *)recp);
+		}
 	}
 
 	if (attrp != NULL) {
@@ -648,7 +660,7 @@ cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantr
 		} else {
 			getbsdattr(hfsmp, (struct HFSPlusCatalogFile *)recp, attrp);
 			if (ilink)
-				attrp->ca_rdev = ilink;
+				attrp->ca_linkref = ilink;
 		}
 	}
 	if (forkp != NULL) {
@@ -707,13 +719,13 @@ cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantr
 			}
 			/* Adjust for any missing blocks. */
 			if ((validblks < forkp->cf_blocks) && (forkp->cf_extents[7].blockCount == 0)) {
-				u_int64_t psize;
+				off_t psize;
 
 				forkp->cf_blocks = validblks;
 				if (attrp != NULL) {
 					attrp->ca_blocks = validblks + recp->hfsPlusFile.resourceFork.totalBlocks;
 				}
-				psize = (u_int64_t)validblks * (u_int64_t)hfsmp->blockSize;
+				psize = (off_t)validblks * (off_t)hfsmp->blockSize;
 				if (psize < forkp->cf_size) {
 					forkp->cf_size = psize;
 				}
@@ -728,9 +740,9 @@ cat_lookupbykey(struct hfsmount *hfsmp, CatalogKey *keyp, u_long hint, int wantr
 			MALLOC(pluskey, HFSPlusCatalogKey *, sizeof(HFSPlusCatalogKey), M_TEMP, M_WAITOK);
 			promotekey(hfsmp, (HFSCatalogKey *)&iterator->key, pluskey, &encoding);
 	
-		} else
+		} else {
 			pluskey = (HFSPlusCatalogKey *)&iterator->key;
-
+		}
 		builddesc(pluskey, cnid, hint, encoding, isadir(recp), descp);
 		if (std_hfs) {
 			FREE(pluskey, M_TEMP);
@@ -750,13 +762,15 @@ exit:
 
 /*
  * cat_create - create a node in the catalog
+ *
+ * NOTE: both the catalog file and attribute file locks must
+ *       be held before calling this function.
  */
 __private_extern__
 int
 cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attrp,
 	struct cat_desc *out_descp)
 {
-	ExtendedVCB * vcb;
 	FCB * fcb;
 	struct btobj * bto;
 	FSBufferDescriptor btdata;
@@ -764,41 +778,31 @@ cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 	u_int32_t datalen;
 	int std_hfs;
 	int result = 0;
-	u_long encoding;
+	u_long encoding = kTextEncodingMacRoman;
 	int modeformat;
-	int mntlock = 0;
 
 	modeformat = attrp->ca_mode & S_IFMT;
 
-	vcb = HFSTOVCB(hfsmp);
-	fcb = GetFileControlBlock(vcb->catalogRefNum);
-	std_hfs = (vcb->vcbSigWord == kHFSSigWord);
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+	std_hfs = (hfsmp->hfs_flags & HFS_STANDARD);
 
 	/*
-	 * Atomically get the next CNID.  If we have wrapped the CNIDs
-	 * then keep the hfsmp lock held until we have found a CNID.
+	 * Get the next CNID. We can change it since we hold the catalog lock.
 	 */
-	HFS_MOUNT_LOCK(hfsmp, TRUE);
-	mntlock = 1;
 	nextCNID = hfsmp->vcbNxtCNID;
 	if (nextCNID == 0xFFFFFFFF) {
 		if (std_hfs) {
-			result = ENOSPC;
+			return (ENOSPC);
 		} else {
+			HFS_MOUNT_LOCK(hfsmp, TRUE)
 			hfsmp->vcbNxtCNID = kHFSFirstUserCatalogNodeID;
 			hfsmp->vcbAtrb |= kHFSCatalogNodeIDsReusedMask;
+			HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 		}
 	} else {
 		hfsmp->vcbNxtCNID++;
 	}
-	hfsmp->vcbFlags |= 0xFF00;
-	/* OK to drop lock if CNIDs are not wrapping */
-	if ((hfsmp->vcbAtrb & kHFSCatalogNodeIDsReusedMask) == 0) {
-		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-		mntlock = 0;
-		if (result)
-			return (result);  /* HFS only exit */
-	}
+	MarkVCBDirty(hfsmp);
 
 	/* Get space for iterator, key and data */	
 	MALLOC(bto, struct btobj *, sizeof(struct btobj), M_TEMP, M_WAITOK);
@@ -825,10 +829,21 @@ cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 		btdata.itemCount = 1;
 		
 		for (;;) {
+			// this call requires the attribute file lock to be held
+			result = file_attribute_exist(hfsmp, nextCNID);
+			if (result == EEXIST) {
+				// that cnid has orphaned attributes so just skip it.
+				if (++nextCNID < kHFSFirstUserCatalogNodeID) {
+					nextCNID = kHFSFirstUserCatalogNodeID;
+				}
+				continue;
+			}
+			if (result) goto exit;
+			
 			buildthreadkey(nextCNID, std_hfs, (CatalogKey *) &bto->iterator.key);
 
 			result = BTInsertRecord(fcb, &bto->iterator, &btdata, datalen);
-			if ((result == btExists) && !std_hfs && mntlock) {
+			if ((result == btExists) && !std_hfs && (hfsmp->vcbAtrb & kHFSCatalogNodeIDsReusedMask)) {
 				/*
 				 * Allow CNIDs on HFS Plus volumes to wrap around
 				 */
@@ -844,15 +859,13 @@ cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 	
 	/*
 	 * CNID is now established. If we have wrapped then
-	 * update the vcbNxtCNID and drop the vcb lock.
+	 * update the vcbNxtCNID.
 	 */
-	if (mntlock) {
+	if ((hfsmp->vcbAtrb & kHFSCatalogNodeIDsReusedMask)) {
 		hfsmp->vcbNxtCNID = nextCNID + 1;
 		if (hfsmp->vcbNxtCNID < kHFSFirstUserCatalogNodeID) {
 			hfsmp->vcbNxtCNID = kHFSFirstUserCatalogNodeID;
 		}
-		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-		mntlock = 0;
 	}
 
 	/*
@@ -873,16 +886,20 @@ cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 		/* Back out the thread record */
 		if (!std_hfs || S_ISDIR(attrp->ca_mode)) {
 			buildthreadkey(nextCNID, std_hfs, (CatalogKey *)&bto->iterator.key);
-			(void) BTDeleteRecord(fcb, &bto->iterator);
+			if (BTDeleteRecord(fcb, &bto->iterator)) {
+				/* Error on deleting extra thread record, mark 
+				 * volume inconsistent 
+				 */
+				printf ("hfs: cat_create() failed to delete thread record on volume %s\n", hfsmp->vcbVN);
+				hfs_mark_volume_inconsistent(hfsmp);
+			}
 		}
 		goto exit;
 	}
 
 	/*
-	 * Insert was Successfull, update name, parent and volume
+	 * Insert was successful, update name, parent and volume
 	 */
-
-
 	if (out_descp != NULL) {
 		HFSPlusCatalogKey * pluskey = NULL;
 
@@ -902,9 +919,6 @@ cat_create(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 	attrp->ca_fileid = nextCNID;
 
 exit:
-	if (mntlock)
-		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-
 	(void) BTFlushPath(fcb);
 	FREE(bto, M_TEMP);
 
@@ -940,11 +954,12 @@ cat_rename (
 	HFSPlusCatalogKey * to_key;
 	ExtendedVCB * vcb;
 	FCB * fcb;
-	UInt16	datasize;
+	u_int16_t	datasize;
 	int result = 0;
 	int sourcegone = 0;
 	int skipthread = 0;
 	int directory = from_cdp->cd_flags & CD_ISDIR;
+	int is_dirlink = 0;
 	int std_hfs;
 	u_long encoding = 0;
 
@@ -986,7 +1001,7 @@ cat_rename (
 		}
 		bzero(&iterator, sizeof(iterator));
 		/*
-		 * Traverese destination path all the way back to the root
+		 * Traverse destination path all the way back to the root
 		 * making sure that source directory is not encountered.
 		 *
 		 */
@@ -998,7 +1013,7 @@ cat_rename (
 			if (result) goto exit;
 			
 			pathcnid = getparentcnid(recp);
-			if (pathcnid == cnid) {
+			if (pathcnid == cnid || pathcnid == 0) {
 				result = EINVAL;
 				goto exit;
 			}
@@ -1013,14 +1028,14 @@ cat_rename (
 	if (result) {
 		if (std_hfs || (result != btNotFound)) 
 			goto exit;
-	
+
 		struct cat_desc temp_desc;
 
 		/* Probably the node has mangled name */
 		result = cat_lookupmangled(hfsmp, from_cdp, 0, &temp_desc, NULL, NULL); 
-		if (result) 
+		if (result)
 			goto exit;
-			
+
 		/* The file has mangled name.  Search the cnode data using full name */
 		bzero(from_iterator, sizeof(*from_iterator));
 		result = buildkey(hfsmp, &temp_desc, (HFSPlusCatalogKey *)&from_iterator->key, 0);
@@ -1034,20 +1049,35 @@ cat_rename (
 			cat_releasedesc(&temp_desc);
 			goto exit;
 		}
-		
+
 		cat_releasedesc(&temp_desc);
 	}
 
-	/* Update the text encoding (on disk and in descriptor) */
-	if (!std_hfs) {
-		encoding = hfs_pickencoding(to_key->nodeName.unicode,
-				to_key->nodeName.length);
+	/* Check if the source is directory hard link.  We do not change 
+	 * directory flag because it is later used to initialize result descp
+	 */
+	if ((!std_hfs) && 
+	    (directory) && 
+	    (recp->recordType == kHFSPlusFileRecord) &&
+	    (recp->hfsPlusFile.flags & kHFSHasLinkChainMask)) {
+	    	is_dirlink  = 1;
+	}
+
+	/*
+	 * Update the text encoding (on disk and in descriptor).
+	 *
+	 * Note that hardlink inodes don't require a text encoding hint.
+	 */
+	if (!std_hfs &&
+	    todir_cdp->cd_parentcnid != hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid &&
+	    todir_cdp->cd_parentcnid != hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
+		encoding = hfs_pickencoding(to_key->nodeName.unicode, to_key->nodeName.length);
 		hfs_setencodingbits(hfsmp, encoding);
 		recp->hfsPlusFile.textEncoding = encoding;
 		if (out_cdp)
 			out_cdp->cd_encoding = encoding;
 	}
-	
+
 	if (std_hfs && !directory &&
 	    !(recp->hfsFile.flags & kHFSThreadExistsMask))
 		skipthread = 1;
@@ -1096,8 +1126,12 @@ cat_rename (
 		       {
 		       	int err;
 			err = BTInsertRecord(fcb, from_iterator, &btdata, datasize);
-			if (err)
-				panic("cat_create: could not undo (BTInsert = %d)", err);
+			if (err) {
+				printf("cat_create: could not undo (BTInsert = %d)", err);
+				hfs_mark_volume_inconsistent(hfsmp);
+				result = err;
+				goto exit;
+			}
 		       }
 		    #else
 			(void) BTInsertRecord(fcb, from_iterator, &btdata, datasize);
@@ -1119,8 +1153,12 @@ cat_rename (
 		     {
 		     	int err;
 			err = BTDeleteRecord(fcb, to_iterator);
-			if (err)
-				panic("cat_create: could not undo (BTDelete = %d)", err);
+			if (err) {
+				printf("cat_create: could not undo (BTDelete = %d)", err);
+				hfs_mark_volume_inconsistent(hfsmp);
+				result = err;
+				goto exit;
+			}
 		     }			
 		  #else
 			(void) BTDeleteRecord(fcb, to_iterator);
@@ -1142,7 +1180,14 @@ cat_rename (
 	 * (optional for HFS files)
 	 */
 	if (!skipthread) {
-		datasize = buildthread(&to_iterator->key, recp, std_hfs, directory);
+		/* For directory hard links, always create a file thread 
+		 * record.  For everything else, use the directory flag.
+		 */
+		if (is_dirlink) {
+			datasize = buildthread(&to_iterator->key, recp, std_hfs, false);
+		} else {
+			datasize = buildthread(&to_iterator->key, recp, std_hfs, directory);
+		}
 		btdata.itemSize = datasize;
 		buildthreadkey(from_cdp->cd_cnid, std_hfs, (CatalogKey *)&from_iterator->key);
 		result = BTInsertRecord(fcb, from_iterator, &btdata, datasize);
@@ -1196,16 +1241,14 @@ __private_extern__
 int
 cat_delete(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attrp)
 {
-	ExtendedVCB * vcb;
 	FCB * fcb;
 	BTreeIterator *iterator;
 	cnid_t cnid;
 	int std_hfs;
 	int result;
 
-	vcb = HFSTOVCB(hfsmp);
-	fcb = GetFileControlBlock(vcb->catalogRefNum);
-	std_hfs = (vcb->vcbSigWord == kHFSSigWord);
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+	std_hfs = (hfsmp->hfs_flags & HFS_STANDARD);
 
 	/* Preflight check:
 	 *
@@ -1219,8 +1262,8 @@ cat_delete(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 
 	/* XXX Preflight Missing */
 	
-	/* Get space for iterator */	
-	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+	iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
 	iterator->hint.nodeNum = 0;
 
 	/*
@@ -1268,13 +1311,17 @@ cat_delete(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 		cat_releasedesc(&temp_desc);
 	}
 
-	/* Delete thread record, ignore errors */
+	/* Delete thread record.  On error, mark volume inconsistent */
 	buildthreadkey(cnid, std_hfs, (CatalogKey *)&iterator->key);
-	(void) BTDeleteRecord(fcb, iterator);
+	if (BTDeleteRecord(fcb, iterator)) {
+		if (!std_hfs) {
+			printf ("hfs: cat_delete() failed to delete thread record on volume %s\n", hfsmp->vcbVN);
+			hfs_mark_volume_inconsistent(hfsmp);
+		}
+	}
 
 exit:
 	(void) BTFlushPath(fcb);
-	FREE(iterator, M_TEMP);
 
 	return MacToVFSError(result);
 }
@@ -1289,16 +1336,14 @@ int
 cat_update(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attrp,
 	struct cat_fork *dataforkp, struct cat_fork *rsrcforkp)
 {
-	ExtendedVCB * vcb;
 	FCB * fcb;
 	BTreeIterator * iterator;
 	struct update_state state;
 	int std_hfs;
 	int result;
 
-	vcb = HFSTOVCB(hfsmp);
-	fcb = GetFileControlBlock(vcb->catalogRefNum);
-	std_hfs = (vcb->vcbSigWord == kHFSSigWord);
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+	std_hfs = (hfsmp->hfs_flags & HFS_STANDARD);
 
 	state.s_desc = descp;
 	state.s_attr = attrp;
@@ -1306,8 +1351,8 @@ cat_update(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 	state.s_rsrcfork = rsrcforkp;
 	state.s_hfsmp = hfsmp;
 
-	/* Get space for iterator */	
-	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+	iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
 
 	/*
 	 * For open-deleted files we need to do a lookup by cnid
@@ -1317,10 +1362,13 @@ cat_update(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 	 * itself (not the link record) so a lookup by fileid
 	 * (i.e. thread rec) is needed.
 	 */
-	if ((descp->cd_cnid != attrp->ca_fileid) || (descp->cd_namelen == 0))
+	if ((descp->cd_cnid != attrp->ca_fileid) ||
+	    (descp->cd_namelen == 0) ||
+	    (attrp->ca_recflags & kHFSHasLinkChainMask)) {
 		result = getkey(hfsmp, attrp->ca_fileid, (CatalogKey *)&iterator->key);
-	else
+	} else {
 		result = buildkey(hfsmp, descp, (HFSPlusCatalogKey *)&iterator->key, 0);
+	}
 	if (result)
 		goto exit;
 
@@ -1337,7 +1385,6 @@ cat_update(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attr
 
 exit:
 	(void) BTFlushPath(fcb);
-	FREE(iterator, M_TEMP);
 
 	return MacToVFSError(result);
 }
@@ -1413,6 +1460,12 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 					(u_int16_t)forkp->cf_extents[i].blockCount;
 			}
 		}
+
+		/* Synchronize the lock state */
+		if (attrp->ca_flags & (SF_IMMUTABLE | UF_IMMUTABLE))
+			file->flags |= kHFSFileLockedMask;
+		else
+			file->flags &= ~kHFSFileLockedMask;
 		break;
 	}
 	case kHFSPlusFolderRecord: {
@@ -1420,9 +1473,10 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 		
 		dir = (struct HFSPlusCatalogFolder *)crp;
 		/* Do a quick sanity check */
-		if ((ckp->hfsPlus.parentID != descp->cd_parentcnid) ||
-			(dir->folderID != descp->cd_cnid)) 
+		if (dir->folderID != attrp->ca_fileid) {
+			printf("catrec_update: id %d != %d\n", dir->folderID, attrp->ca_fileid);
 			return (btNotFound);
+		}
 		dir->flags            = attrp->ca_recflags;
 		dir->valence          = attrp->ca_entries;
 		dir->createDate       = to_hfs_time(attrp->ca_itime);
@@ -1431,8 +1485,11 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 		dir->accessDate       = to_hfs_time(attrp->ca_atime);
 		attrp->ca_atimeondisk = attrp->ca_atime;	
 		dir->attributeModDate = to_hfs_time(attrp->ca_ctime);
-		dir->textEncoding     = descp->cd_encoding;
-		dir->attrBlocks       = attrp->ca_attrblks;
+		/* Note: directory hardlink inodes don't require a text encoding hint. */
+		if (ckp->hfsPlus.parentID != hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
+			dir->textEncoding = descp->cd_encoding;
+		}
+		dir->folderCount      = attrp->ca_dircount;
 		bcopy(&attrp->ca_finderinfo[0], &dir->userInfo, 32);
 		/*
 		 * Update the BSD Info if it was already initialized on
@@ -1469,6 +1526,10 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 			dir->bsdInfo.ownerFlags = attrp->ca_flags & 0x000000FF;
 			dir->bsdInfo.adminFlags = attrp->ca_flags >> 16;
 			dir->bsdInfo.fileMode   = attrp->ca_mode;
+			/* A directory hardlink has a link count. */
+			if (attrp->ca_linkcount > 1 || dir->hl_linkCount > 1) {
+				dir->hl_linkCount = attrp->ca_linkcount;
+			}
 		}
 		break;
 	}
@@ -1486,8 +1547,15 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 		file->accessDate       = to_hfs_time(attrp->ca_atime);
 		attrp->ca_atimeondisk  = attrp->ca_atime;	
 		file->attributeModDate = to_hfs_time(attrp->ca_ctime);
-		file->textEncoding     = descp->cd_encoding;
-		file->attrBlocks       = attrp->ca_attrblks;
+		/*
+		 * Note: file hardlink inodes don't require a text encoding
+		 * hint, but they do have a first link value.
+		 */
+		if (ckp->hfsPlus.parentID == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) {
+			file->hl_firstLinkID = attrp->ca_firstlink;
+		} else {
+			file->textEncoding = descp->cd_encoding;
+		}
 		bcopy(&attrp->ca_finderinfo[0], &file->userInfo, 32);
 		/*
 		 * Update the BSD Info if it was already initialized on
@@ -1548,8 +1616,9 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 
 		if ((file->resourceFork.extents[0].startBlock != 0) &&
 		    (file->resourceFork.extents[0].startBlock ==
-		     file->dataFork.extents[0].startBlock))
+		     file->dataFork.extents[0].startBlock)) {
 			panic("catrec_update: rsrc fork == data fork");
+		}
 
 		/* Synchronize the lock state */
 		if (attrp->ca_flags & (SF_IMMUTABLE | UF_IMMUTABLE))
@@ -1558,11 +1627,11 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 			file->flags &= ~kHFSFileLockedMask;
 
 		/* Push out special field if necessary */
-		if (S_ISBLK(attrp->ca_mode) || S_ISCHR(attrp->ca_mode))
+		if (S_ISBLK(attrp->ca_mode) || S_ISCHR(attrp->ca_mode)) {
 			file->bsdInfo.special.rawDevice = attrp->ca_rdev;
-		else if (descp->cd_cnid != attrp->ca_fileid
-		     ||  attrp->ca_nlink == 2)
-			file->bsdInfo.special.linkCount = attrp->ca_nlink;
+		} else if (descp->cd_cnid != attrp->ca_fileid || attrp->ca_linkcount == 2) {
+			file->hl_linkCount = attrp->ca_linkcount;
+		}
 		break;
 	}
 	default:
@@ -1571,142 +1640,799 @@ catrec_update(const CatalogKey *ckp, CatalogRecord *crp, struct update_state *st
 	return (0);
 }
 
-/*
- * Callback to collect directory entries.
- * Called with readattr_state for each item in a directory.
+/* This function sets kHFSHasChildLinkBit in a directory hierarchy in the 
+ * catalog btree of given cnid by walking up the parent chain till it reaches 
+ * either the root folder, or the private metadata directory for storing 
+ * directory hard links.  This function updates the corresponding in-core 
+ * cnode, if any, and the directory record in the catalog btree.
+ * On success, returns zero.  On failure, returns non-zero value.
  */
-struct readattr_state {
-	struct hfsmount *hfsmp;
-	struct cat_entrylist *list;
-	cnid_t	dir_cnid;
-	int stdhfs;
-	int error;
-};
-
-static int
-cat_readattr(const CatalogKey *key, const CatalogRecord *rec,
-             struct readattr_state *state)
+__private_extern__
+int 
+cat_set_childlinkbit(struct hfsmount *hfsmp, cnid_t cnid)
 {
-	struct cat_entrylist *list = state->list;
-	struct hfsmount *hfsmp = state->hfsmp;
-	struct cat_entry *cep;
-	cnid_t parentcnid;
+	int retval = 0;
+	int lockflags = 0;
+	struct cat_desc desc;
+	struct cat_attr attr;
+
+	while ((cnid != kHFSRootFolderID) && (cnid != kHFSRootParentID) &&
+	       (cnid != hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid)) {
+		/* Update the bit in corresponding cnode, if any, in the hash.
+		 * If the cnode has the bit already set, stop the traversal.
+		 */
+		retval = hfs_chash_set_childlinkbit(hfsmp->hfs_raw_dev, cnid);
+		if (retval == 0) {
+			break;
+		}
 
-	if (list->realentries >= list->maxentries)
-		return (0);  /* stop */
-	
-	parentcnid = state->stdhfs ? key->hfs.parentID : key->hfsPlus.parentID;
+		/* Update the catalog record on disk if either cnode was not
+		 * found in the hash, or if a cnode was found and the cnode 
+		 * did not have the bit set previously.
+		 */
+		retval = hfs_start_transaction(hfsmp);
+		if (retval) {
+			break;
+		}
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 
-	switch(rec->recordType) {
-	case kHFSPlusFolderRecord:
-	case kHFSPlusFileRecord:
-	case kHFSFolderRecord:
-	case kHFSFileRecord:
-		if (parentcnid != state->dir_cnid) {
-			state->error = ENOENT;
-			return (0);	/* stop */
+		/* Look up our catalog folder record */
+		retval = cat_idlookup(hfsmp, cnid, 0, &desc, &attr, NULL);
+		if (retval) {
+			hfs_systemfile_unlock(hfsmp, lockflags);
+			hfs_end_transaction(hfsmp);
+			break;
 		}
-		break;
-	default:
-		state->error = ENOENT;
-		return (0);	/* stop */
-	}
 
-	/* Hide the private meta data directory and journal files */
-	if (parentcnid == kHFSRootFolderID) {
-		if ((rec->recordType == kHFSPlusFolderRecord) &&
-		    (rec->hfsPlusFolder.folderID == hfsmp->hfs_privdir_desc.cd_cnid)) {
-			return (1);	/* continue */
+		/* Update the bit in the catalog record */
+		attr.ca_recflags |= kHFSHasChildLinkMask;
+		retval = cat_update(hfsmp, &desc, &attr, NULL, NULL);
+		if (retval) {
+			hfs_systemfile_unlock(hfsmp, lockflags);
+			hfs_end_transaction(hfsmp);
+			break;
 		}
-		if ((hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY))) &&
-		    (rec->recordType == kHFSPlusFileRecord) &&
-		    ((rec->hfsPlusFile.fileID == hfsmp->hfs_jnlfileid) ||
-		     (rec->hfsPlusFile.fileID == hfsmp->hfs_jnlinfoblkid))) {
 
-			return (1);	/* continue */
+		hfs_systemfile_unlock(hfsmp, lockflags);
+		hfs_end_transaction(hfsmp);
+
+		cnid = desc.cd_parentcnid;
+	}
+
+	return retval;
+}
+
+/* This function traverses the parent directory hierarchy from the given 
+ * directory to one level below root directory and checks if any of its 
+ * ancestors is - 
+ * 	1. A directory hard link.  
+ * 	2. The 'pointed at' directory.
+ * If any of these conditions fail or an internal error is encountered 
+ * during look up of the catalog record, this function returns non-zero value.
+ */
+__private_extern__
+int
+cat_check_link_ancestry(struct hfsmount *hfsmp, cnid_t cnid, cnid_t pointed_at_cnid)
+{
+	HFSPlusCatalogKey *keyp;
+	BTreeIterator *ip;
+	FSBufferDescriptor btdata;
+	HFSPlusCatalogFolder folder;
+	FCB *fcb;
+	int invalid;
+	int result;
+
+	invalid = 0;
+	BDINIT(btdata, &folder);
+	MALLOC(ip, BTreeIterator *, sizeof(*ip), M_TEMP, M_WAITOK);
+	keyp = (HFSPlusCatalogKey *)&ip->key;
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+
+	while (cnid != kHFSRootParentID) {
+		/* Check if the 'pointed at' directory is an ancestor */
+		if (pointed_at_cnid == cnid) {
+			invalid = 1;
+			break;
+		}
+		if ((result = getkey(hfsmp, cnid, (CatalogKey *)keyp))) {
+			printf("cat_check_link_ancestry: getkey for %u failed\n", cnid);
+			invalid = 1;  /* On errors, assume an invalid parent */
+			break;
+		}
+		if ((result = BTSearchRecord(fcb, ip, &btdata, NULL, NULL))) {
+			printf("cat_check_link_ancestry: cannot find %u\n", cnid);
+			invalid = 1;  /* On errors, assume an invalid parent */
+			break;
+		}
+		/* Check if this ancestor is a directory hard link */
+		if (folder.flags & kHFSHasLinkChainMask) {
+			invalid = 1;
+			break;
 		}
+		cnid = keyp->parentID;
 	}
+	FREE(ip, M_TEMP);
+	return (invalid);
+}
 
-	cep = &list->entry[list->realentries++];
 
-	if (state->stdhfs) {
-		struct HFSPlusCatalogFile cnoderec;
-		HFSPlusCatalogKey * pluskey;
-		long encoding;
+/*
+ * updatelink_callback - update a link's chain
+ */
 
-		promoteattr(hfsmp, rec, &cnoderec);
-		getbsdattr(hfsmp, &cnoderec, &cep->ce_attr);
+struct linkupdate_state {
+	cnid_t filelinkid;
+	cnid_t prevlinkid;
+	cnid_t nextlinkid;
+};
 
-		MALLOC(pluskey, HFSPlusCatalogKey *, sizeof(HFSPlusCatalogKey), M_TEMP, M_WAITOK);
-		promotekey(hfsmp, (HFSCatalogKey *)key, pluskey, &encoding);
-		builddesc(pluskey, getcnid(rec), 0, encoding, isadir(rec), &cep->ce_desc);
-		FREE(pluskey, M_TEMP);
+static int
+updatelink_callback(__unused const CatalogKey *ckp, CatalogRecord *crp, struct linkupdate_state *state)
+{
+	HFSPlusCatalogFile *file;
 
-		if (rec->recordType == kHFSFileRecord) {
-			int blksize = HFSTOVCB(hfsmp)->blockSize;
+	if (crp->recordType != kHFSPlusFileRecord) {
+		printf("updatelink_callback: unexpected rec type %d\n", crp->recordType);
+		return (btNotFound);
+	}
 
-			cep->ce_datasize = rec->hfsFile.dataLogicalSize;
-			cep->ce_datablks = rec->hfsFile.dataPhysicalSize / blksize;
-			cep->ce_rsrcsize = rec->hfsFile.rsrcLogicalSize;
-			cep->ce_rsrcblks = rec->hfsFile.rsrcPhysicalSize / blksize;
+	file = (struct HFSPlusCatalogFile *)crp;
+	if (file->flags & kHFSHasLinkChainMask) {
+		if (state->prevlinkid != HFS_IGNORABLE_LINK) {
+			file->hl_prevLinkID = state->prevlinkid;
 		}
-	} else {
-		getbsdattr(hfsmp, (struct HFSPlusCatalogFile *)rec, &cep->ce_attr);
-		builddesc((HFSPlusCatalogKey *)key, getcnid(rec), 0, getencoding(rec),
-			isadir(rec), &cep->ce_desc);
-		
-		if (rec->recordType == kHFSPlusFileRecord) {
-			cep->ce_datasize = rec->hfsPlusFile.dataFork.logicalSize;
-			cep->ce_datablks = rec->hfsPlusFile.dataFork.totalBlocks;
-			cep->ce_rsrcsize = rec->hfsPlusFile.resourceFork.logicalSize;
-			cep->ce_rsrcblks = rec->hfsPlusFile.resourceFork.totalBlocks;
-			
-			/* Save link reference for later processing. */
-			if ((SWAP_BE32(rec->hfsPlusFile.userInfo.fdType) == kHardLinkFileType)
-			&&  (SWAP_BE32(rec->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator))
-				cep->ce_attr.ca_rdev = rec->hfsPlusFile.bsdInfo.special.iNodeNum;
+		if (state->nextlinkid != HFS_IGNORABLE_LINK) {
+			file->hl_nextLinkID = state->nextlinkid;
 		}
+	} else {
+		printf("updatelink_callback: file %d isn't a chain\n", file->fileID);
 	}
-
-	return (list->realentries < list->maxentries);
+	return (0);
 }
 
 /*
- * Pack a cat_entrylist buffer with attributes from the catalog
- *
- * Note: index is zero relative
+ * cat_updatelink - update a link's chain
  */
 __private_extern__
 int
-cat_getentriesattr(struct hfsmount *hfsmp, directoryhint_t *dirhint, struct cat_entrylist *ce_list)
+cat_updatelink(struct hfsmount *hfsmp, cnid_t linkfileid, cnid_t prevlinkid, cnid_t nextlinkid)
 {
-	FCB* fcb;
-	CatalogKey * key;
+	FCB * fcb;
 	BTreeIterator * iterator;
-	struct readattr_state state;
-	cnid_t parentcnid;
-	int i;
-	int std_hfs;
-	int index;
-	int have_key;
-	int result = 0;
+	struct linkupdate_state state;
+	int result;
 
-	ce_list->realentries = 0;
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+	state.filelinkid = linkfileid;
+	state.prevlinkid = prevlinkid;
+	state.nextlinkid = nextlinkid;
 
-	fcb = GetFileControlBlock(HFSTOVCB(hfsmp)->catalogRefNum);
-	std_hfs = (HFSTOVCB(hfsmp)->vcbSigWord == kHFSSigWord);
-	parentcnid = dirhint->dh_desc.cd_parentcnid;
+	/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+	iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
+	iterator->hint.nodeNum = 0;
 
-	state.hfsmp = hfsmp;
-	state.list = ce_list;
-	state.dir_cnid = parentcnid;
-	state.stdhfs = std_hfs;
-	state.error = 0;
+	result = getkey(hfsmp, linkfileid, (CatalogKey *)&iterator->key);
+	if (result == 0) {
+		result = BTUpdateRecord(fcb, iterator, (IterateCallBackProcPtr)updatelink_callback, &state);
+		(void) BTFlushPath(fcb);
+	} else {
+		printf("cat_updatelink: couldn't resolve cnid %d\n", linkfileid);
+	}
+	return MacToVFSError(result);
+}
 
-	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
-	bzero(iterator, sizeof(*iterator));
-	key = (CatalogKey *)&iterator->key;
-	have_key = 0;
+/*
+ * cat_lookuplink - lookup a link by it's name
+ */
+__private_extern__
+int
+cat_lookuplink(struct hfsmount *hfsmp, struct cat_desc *descp, cnid_t *linkfileid, cnid_t *prevlinkid,  cnid_t *nextlinkid)
+{
+	FCB * fcb;
+	BTreeIterator * iterator;
+	struct FSBufferDescriptor btdata;
+	struct HFSPlusCatalogFile file;
+	int result;
+
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+
+	/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+	iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
+	iterator->hint.nodeNum = 0;
+
+	if ((result = buildkey(hfsmp, descp, (HFSPlusCatalogKey *)&iterator->key, 0))) {
+		goto exit;
+	}
+	BDINIT(btdata, &file);
+
+	if ((result = BTSearchRecord(fcb, iterator, &btdata, NULL, NULL))) {
+		goto exit;
+	}
+	if (file.recordType != kHFSPlusFileRecord) {
+		result = ENOENT;
+		goto exit;
+	}
+	*linkfileid = file.fileID;
+
+	if (file.flags & kHFSHasLinkChainMask) {
+		*prevlinkid = file.hl_prevLinkID;
+		*nextlinkid = file.hl_nextLinkID;
+	} else {
+		*prevlinkid = 0;
+		*nextlinkid = 0;
+	}
+exit:
+	return MacToVFSError(result);
+}
+
+
+/*
+ * cat_lookuplink - lookup a link by its cnid
+ */
+__private_extern__
+int
+cat_lookuplinkbyid(struct hfsmount *hfsmp, cnid_t linkfileid, cnid_t *prevlinkid,  cnid_t *nextlinkid)
+{
+	FCB * fcb;
+	BTreeIterator * iterator;
+	struct FSBufferDescriptor btdata;
+	struct HFSPlusCatalogFile file;
+	int result;
+
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+
+	/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+	iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
+	iterator->hint.nodeNum = 0;
+
+	if ((result = getkey(hfsmp, linkfileid, (CatalogKey *)&iterator->key))) {
+		printf("cat_lookuplinkbyid: getkey for %d failed %d\n", linkfileid, result);
+		goto exit;
+	}
+	BDINIT(btdata, &file);
+
+	if ((result = BTSearchRecord(fcb, iterator, &btdata, NULL, NULL))) {
+		printf("cat_lookuplinkbyid: cannot find %d\n", linkfileid);
+		goto exit;
+	}
+	/* The prev/next chain is only valid when kHFSHasLinkChainMask is set. */
+	if (file.flags & kHFSHasLinkChainMask) {
+		cnid_t parent;
+
+		parent = ((HFSPlusCatalogKey *)&iterator->key)->parentID;
+
+		/* ADL inodes don't have a chain (its in an EA) */
+		if (parent == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
+			result = ENOLINK;  /* signal to caller to get head of list */
+		} else if (parent == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) {
+			*prevlinkid = 0;
+			*nextlinkid = file.hl_firstLinkID;
+		} else {
+			*prevlinkid = file.hl_prevLinkID;
+			*nextlinkid = file.hl_nextLinkID;
+		}
+	} else {
+		*prevlinkid = 0;
+		*nextlinkid = 0;
+	}
+exit:
+	return MacToVFSError(result);
+}
+
+
+/*
+ * cat_createlink - create a link in the catalog
+ *
+ * The following cat_attr fields are expected to be set:
+ *	 ca_linkref
+ *	 ca_itime
+ *	 ca_mode (S_IFREG)
+ *	 ca_recflags
+ *	 ca_flags
+ *	 ca_finderinfo (type and creator)
+ */
+__private_extern__
+int
+cat_createlink(struct hfsmount *hfsmp, struct cat_desc *descp, struct cat_attr *attrp,
+               cnid_t nextlinkid, cnid_t *linkfileid)
+{
+	FCB * fcb;
+	struct btobj * bto;
+	FSBufferDescriptor btdata;
+	HFSPlusForkData *rsrcforkp;
+	u_int32_t nextCNID;
+	u_int32_t datalen;
+	u_long encoding;
+	int thread_inserted = 0;
+	int alias_allocated = 0;
+	int result = 0;
+
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
+
+	/*
+	 * Get the next CNID. We can change it since we hold the catalog lock.
+	 */
+	nextCNID = hfsmp->vcbNxtCNID;
+	if (nextCNID == 0xFFFFFFFF) {
+		HFS_MOUNT_LOCK(hfsmp, TRUE)
+		hfsmp->vcbNxtCNID = kHFSFirstUserCatalogNodeID;
+		hfsmp->vcbAtrb |= kHFSCatalogNodeIDsReusedMask;
+		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
+	} else {
+		hfsmp->vcbNxtCNID++;
+	}
+	MarkVCBDirty(hfsmp);
+
+	/* Get space for iterator, key and data */	
+	MALLOC(bto, struct btobj *, sizeof(struct btobj), M_TEMP, M_WAITOK);
+	bto->iterator.hint.nodeNum = 0;
+	rsrcforkp = &bto->data.hfsPlusFile.resourceFork;
+
+	result = buildkey(hfsmp, descp, &bto->key, 0);
+	if (result) {
+		printf("cat_createlink: err %d from buildkey\n", result);
+		goto exit;
+	}
+
+	/* This is our only chance to set the encoding (other than a rename). */
+	encoding = hfs_pickencoding(bto->key.nodeName.unicode, bto->key.nodeName.length);
+
+	/* Insert the thread record first. */
+	datalen = buildthread((void*)&bto->key, &bto->data, 0, 0);
+	btdata.bufferAddress = &bto->data;
+	btdata.itemSize = datalen;
+	btdata.itemCount = 1;
+	
+	for (;;) {
+		buildthreadkey(nextCNID, 0, (CatalogKey *) &bto->iterator.key);
+
+		result = BTInsertRecord(fcb, &bto->iterator, &btdata, datalen);
+		if ((result == btExists) && (hfsmp->vcbAtrb & kHFSCatalogNodeIDsReusedMask)) {
+			/*
+			 * Allow CNIDs on HFS Plus volumes to wrap around
+			 */
+			if (++nextCNID < kHFSFirstUserCatalogNodeID) {
+				nextCNID = kHFSFirstUserCatalogNodeID;
+			}
+			continue;
+		}
+		if (result == 0) {
+			thread_inserted = 1;
+		}
+		break;
+	}
+	if (result)
+		goto exit;
+	
+	/*
+	 * CNID is now established. If we have wrapped then
+	 * update the vcbNxtCNID.
+	 */
+	if ((hfsmp->vcbAtrb & kHFSCatalogNodeIDsReusedMask)) {
+		hfsmp->vcbNxtCNID = nextCNID + 1;
+		if (hfsmp->vcbNxtCNID < kHFSFirstUserCatalogNodeID) {
+			hfsmp->vcbNxtCNID = kHFSFirstUserCatalogNodeID;
+		}
+	}
+
+	/*
+	 * Now insert the link record.
+	 */
+	buildrecord(attrp, nextCNID, 0, encoding, &bto->data, &datalen);
+	
+	bto->data.hfsPlusFile.hl_prevLinkID = 0;
+	bto->data.hfsPlusFile.hl_nextLinkID = nextlinkid;
+	bto->data.hfsPlusFile.hl_linkReference = attrp->ca_linkref;
+
+	/* For directory hard links, create alias in resource fork */
+	if (descp->cd_flags & CD_ISDIR) {
+		if ((result = cat_makealias(hfsmp, attrp->ca_linkref, &bto->data.hfsPlusFile))) {
+			goto exit;
+		}
+		alias_allocated = 1;
+	}
+	btdata.bufferAddress = &bto->data;
+	btdata.itemSize = datalen;
+	btdata.itemCount = 1;
+	
+	bcopy(&bto->key, &bto->iterator.key, sizeof(bto->key));
+
+	result = BTInsertRecord(fcb, &bto->iterator, &btdata, datalen);
+	if (result) {
+		if (result == btExists)
+			result = EEXIST;
+		goto exit;
+	}
+	if (linkfileid != NULL) {
+		*linkfileid = nextCNID;
+	}
+exit:	
+	if (result) {
+		if (thread_inserted) {
+			printf("cat_createlink: err %d from BTInsertRecord\n", MacToVFSError(result));
+
+			buildthreadkey(nextCNID, 0, (CatalogKey *)&bto->iterator.key);
+			if (BTDeleteRecord(fcb, &bto->iterator)) {
+	    			printf("hfs: cat_createlink() failed to delete thread record on volume %s\n", hfsmp->vcbVN);
+				hfs_mark_volume_inconsistent(hfsmp);
+			}
+		}
+		if (alias_allocated && rsrcforkp->extents[0].startBlock != 0) {
+			(void) BlockDeallocate(hfsmp, rsrcforkp->extents[0].startBlock,
+					       rsrcforkp->extents[0].blockCount);
+			rsrcforkp->extents[0].startBlock = 0;
+			rsrcforkp->extents[0].blockCount = 0;
+		}
+	}
+	(void) BTFlushPath(fcb);
+	FREE(bto, M_TEMP);
+
+	return MacToVFSError(result);
+}
+
+/* Directory hard links are visible as aliases on pre-Leopard systems and 
+ * as normal directories on Leopard or later.  All directory hard link aliases 
+ * have the same resource fork content except for the three uniquely 
+ * identifying values that are updated in the resource fork data when the alias 
+ * is created.  The following array is the constant resource fork data used 
+ * only for creating directory hard link aliases.
+ */
+static const char hfs_dirlink_alias_rsrc[] = {
+	0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x9e, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x32, 
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x2b,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
+	0x01, 0x00, 0x00, 0x00, 0x01, 0x9e, 0x00, 0x00, 0x00, 0x9e, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x32, 0x00, 0x00, 0x61, 0x6c, 0x69, 0x73,
+	0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+};
+
+/* Constants for directory hard link alias */
+enum {
+	/* Size of resource fork data array for directory hard link alias */
+	kHFSAliasSize                = 0x1d0,
+
+	/* Volume type for ejectable devices like disk image */
+	kHFSAliasVolTypeEjectable    = 0x5,
+
+	/* Offset for volume create date, in Mac OS local time */
+	kHFSAliasVolCreateDateOffset = 0x12a,
+
+	/* Offset for the type of volume */
+	kHFSAliasVolTypeOffset       = 0x130,
+
+	/* Offset for folder ID of the parent directory of the directory inode */
+	kHFSAliasParentIDOffset      = 0x132,
+
+	/* Offset for folder ID of the directory inode */
+	kHFSAliasTargetIDOffset	     = 0x176,
+};
+
+/* Create and write an alias that points at the directory represented by given
+ * inode number on the same volume.  Directory hard links are visible as 
+ * aliases in pre-Leopard systems and this function creates these aliases.  
+ *
+ * Note: This code is very specific to creating alias for the purpose 
+ * of directory hard links only, and should not be generalized.
+ */
+static int
+cat_makealias(struct hfsmount *hfsmp, u_int32_t inode_num, struct HFSPlusCatalogFile *crp)
+{
+	struct buf *bp;
+	daddr64_t blkno;
+	u_int32_t blkcount;
+	int blksize;
+	int sectorsize;
+	int result;
+	HFSPlusForkData *rsrcforkp;
+	char *alias;
+	uint32_t *valptr;
+
+	rsrcforkp = &(crp->resourceFork);
+
+	blksize = hfsmp->blockSize;
+	blkcount = howmany(kHFSAliasSize, blksize);
+	sectorsize = hfsmp->hfs_phys_block_size;
+	bzero(rsrcforkp, sizeof(HFSPlusForkData));
+
+	/* Allocate some disk space for the alias content. */
+	result = BlockAllocate(hfsmp, 0, blkcount, blkcount, 1, 1,
+	                       &rsrcforkp->extents[0].startBlock,
+	                       &rsrcforkp->extents[0].blockCount);
+	if (result) {
+		rsrcforkp->extents[0].startBlock = 0;
+		goto exit;
+	}
+
+	/* Acquire a buffer cache block for our block. */
+	blkno = ((u_int64_t)rsrcforkp->extents[0].startBlock * (u_int64_t)blksize) / sectorsize;
+	blkno += hfsmp->hfsPlusIOPosOffset / sectorsize;
+
+	bp = buf_getblk(hfsmp->hfs_devvp, blkno, roundup(kHFSAliasSize, hfsmp->hfs_phys_block_size), 0, 0, BLK_META);
+	if (hfsmp->jnl) {
+		journal_modify_block_start(hfsmp->jnl, bp);
+	}
+
+	/* Generate alias content */
+	alias = (char *)buf_dataptr(bp);
+	bzero(alias, buf_size(bp));
+	bcopy(hfs_dirlink_alias_rsrc, alias, kHFSAliasSize);
+
+	/* Set the volume create date, local time in Mac OS format */
+	valptr = (uint32_t *)(alias + kHFSAliasVolCreateDateOffset);
+	*valptr = OSSwapHostToBigInt32(hfsmp->localCreateDate);
+
+	/* If the file system is on a virtual device like disk image, 
+	 * update the volume type to be ejectable device.  
+	 */
+	if (hfsmp->hfs_flags & HFS_VIRTUAL_DEVICE) {
+		*(uint16_t *)(alias + kHFSAliasVolTypeOffset) = 
+		OSSwapHostToBigInt16(kHFSAliasVolTypeEjectable);
+	}
+
+	/* Set id of the parent of the target directory */
+	valptr = (uint32_t *)(alias + kHFSAliasParentIDOffset);
+	*valptr = OSSwapHostToBigInt32(hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid);
+
+	/* Set id of the target directory */
+	valptr = (uint32_t *)(alias + kHFSAliasTargetIDOffset);
+	*valptr = OSSwapHostToBigInt32(inode_num);
+
+	/* Write alias content to disk. */
+	if (hfsmp->jnl) {
+		journal_modify_block_end(hfsmp->jnl, bp, NULL, NULL);
+	} else if ((result = buf_bwrite(bp))) {
+		goto exit;
+	}
+
+	/* Finish initializing the fork data. */
+	rsrcforkp->logicalSize = kHFSAliasSize;
+	rsrcforkp->totalBlocks = rsrcforkp->extents[0].blockCount;
+
+exit:
+	if (result && rsrcforkp->extents[0].startBlock != 0) {
+		(void) BlockDeallocate(hfsmp, rsrcforkp->extents[0].startBlock, rsrcforkp->extents[0].blockCount);
+		rsrcforkp->extents[0].startBlock = 0;
+		rsrcforkp->extents[0].blockCount = 0;
+		rsrcforkp->logicalSize = 0;
+		rsrcforkp->totalBlocks = 0;
+	}
+	return (result);
+}
+
+/*
+ * cat_deletelink - delete a link from the catalog
+ */
+__private_extern__
+int
+cat_deletelink(struct hfsmount *hfsmp, struct cat_desc *descp)
+{
+	struct HFSPlusCatalogFile file;
+	struct cat_attr cattr;
+	uint32_t totalBlocks;
+	int i;
+	int result;
+
+	bzero(&file, sizeof (file));
+	bzero(&cattr, sizeof (cattr));
+	cattr.ca_fileid = descp->cd_cnid;
+
+	/* Directory links have alias content to remove. */
+	if (descp->cd_flags & CD_ISDIR) {
+		FCB * fcb;
+		BTreeIterator * iterator;
+		struct FSBufferDescriptor btdata;
+	
+		fcb = hfsmp->hfs_catalog_cp->c_datafork;
+	
+		/* Borrow the btcb iterator since we have an exclusive catalog lock. */	
+		iterator = &((BTreeControlBlockPtr)(fcb->ff_sysfileinfo))->iterator;
+		iterator->hint.nodeNum = 0;
+	
+		if ((result = buildkey(hfsmp, descp, (HFSPlusCatalogKey *)&iterator->key, 0))) {
+			goto exit;
+		}
+		BDINIT(btdata, &file);
+	
+		if ((result = BTSearchRecord(fcb, iterator, &btdata, NULL, NULL))) {
+			goto exit;
+		}
+	}
+
+	result = cat_delete(hfsmp, descp, &cattr);
+	
+	if ((result == 0) && 
+	    (descp->cd_flags & CD_ISDIR) && 
+	    (file.recordType == kHFSPlusFileRecord)) {
+
+		totalBlocks = file.resourceFork.totalBlocks;
+
+		for (i = 0; (i < 8) && (totalBlocks > 0); i++) {
+			if ((file.resourceFork.extents[i].blockCount == 0) &&
+			    (file.resourceFork.extents[i].startBlock == 0)) {
+				break;
+			}
+
+			(void) BlockDeallocate(hfsmp, 
+				file.resourceFork.extents[i].startBlock, 
+				file.resourceFork.extents[i].blockCount);
+
+			totalBlocks -= file.resourceFork.extents[i].blockCount;
+			file.resourceFork.extents[i].startBlock = 0;
+			file.resourceFork.extents[i].blockCount = 0;
+		}
+	}
+exit:	
+	return (result);
+}
+
+
+/*
+ * Callback to collect directory entries.
+ * Called with readattr_state for each item in a directory.
+ */
+struct readattr_state {
+	struct hfsmount *hfsmp;
+	struct cat_entrylist *list;
+	cnid_t	dir_cnid;
+	int stdhfs;
+	int error;
+};
+
+static int
+getentriesattr_callback(const CatalogKey *key, const CatalogRecord *rec,
+             struct readattr_state *state)
+{
+	struct cat_entrylist *list = state->list;
+	struct hfsmount *hfsmp = state->hfsmp;
+	struct cat_entry *cep;
+	cnid_t parentcnid;
+
+	if (list->realentries >= list->maxentries)
+		return (0);  /* stop */
+	
+	parentcnid = state->stdhfs ? key->hfs.parentID : key->hfsPlus.parentID;
+
+	switch(rec->recordType) {
+	case kHFSPlusFolderRecord:
+	case kHFSPlusFileRecord:
+	case kHFSFolderRecord:
+	case kHFSFileRecord:
+		if (parentcnid != state->dir_cnid) {
+			state->error = ENOENT;
+			return (0);	/* stop */
+		}
+		break;
+	default:
+		state->error = ENOENT;
+		return (0);	/* stop */
+	}
+
+	/* Hide the private system directories and journal files */
+	if (parentcnid == kHFSRootFolderID) {
+		if (rec->recordType == kHFSPlusFolderRecord) {
+			if (rec->hfsPlusFolder.folderID == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+			    rec->hfsPlusFolder.folderID == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
+			    	list->skipentries++;
+				return (1);	/* continue */
+			}
+		}
+		if ((hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY))) &&
+		    (rec->recordType == kHFSPlusFileRecord) &&
+		    ((rec->hfsPlusFile.fileID == hfsmp->hfs_jnlfileid) ||
+		     (rec->hfsPlusFile.fileID == hfsmp->hfs_jnlinfoblkid))) {
+			list->skipentries++;
+			return (1);	/* continue */
+		}
+	}
+
+	cep = &list->entry[list->realentries++];
+
+	if (state->stdhfs) {
+		struct HFSPlusCatalogFile cnoderec;
+		HFSPlusCatalogKey * pluskey;
+		u_long encoding;
+
+		promoteattr(hfsmp, rec, &cnoderec);
+		getbsdattr(hfsmp, &cnoderec, &cep->ce_attr);
+
+		MALLOC(pluskey, HFSPlusCatalogKey *, sizeof(HFSPlusCatalogKey), M_TEMP, M_WAITOK);
+		promotekey(hfsmp, (const HFSCatalogKey *)key, pluskey, &encoding);
+		builddesc(pluskey, getcnid(rec), 0, encoding, isadir(rec), &cep->ce_desc);
+		FREE(pluskey, M_TEMP);
+
+		if (rec->recordType == kHFSFileRecord) {
+			int blksize = HFSTOVCB(hfsmp)->blockSize;
+
+			cep->ce_datasize = rec->hfsFile.dataLogicalSize;
+			cep->ce_datablks = rec->hfsFile.dataPhysicalSize / blksize;
+			cep->ce_rsrcsize = rec->hfsFile.rsrcLogicalSize;
+			cep->ce_rsrcblks = rec->hfsFile.rsrcPhysicalSize / blksize;
+		}
+	} else {
+		getbsdattr(hfsmp, (const struct HFSPlusCatalogFile *)rec, &cep->ce_attr);
+		builddesc((const HFSPlusCatalogKey *)key, getcnid(rec), 0, getencoding(rec),
+			isadir(rec), &cep->ce_desc);
+		
+		if (rec->recordType == kHFSPlusFileRecord) {
+			cep->ce_datasize = rec->hfsPlusFile.dataFork.logicalSize;
+			cep->ce_datablks = rec->hfsPlusFile.dataFork.totalBlocks;
+			cep->ce_rsrcsize = rec->hfsPlusFile.resourceFork.logicalSize;
+			cep->ce_rsrcblks = rec->hfsPlusFile.resourceFork.totalBlocks;
+			
+			/* Save link reference for later processing. */
+			if ((SWAP_BE32(rec->hfsPlusFile.userInfo.fdType) == kHardLinkFileType) &&
+			    (SWAP_BE32(rec->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator)) {
+				cep->ce_attr.ca_linkref = rec->hfsPlusFile.bsdInfo.special.iNodeNum;
+			} else if ((rec->hfsPlusFile.flags & kHFSHasLinkChainMask) && 
+			           (SWAP_BE32(rec->hfsPlusFile.userInfo.fdType) == kHFSAliasType) && 
+				   (SWAP_BE32(rec->hfsPlusFile.userInfo.fdCreator) == kHFSAliasCreator)) {
+				cep->ce_attr.ca_linkref = rec->hfsPlusFile.bsdInfo.special.iNodeNum;
+			}
+		}
+	}
+
+	return (list->realentries < list->maxentries);
+}
+
+/*
+ * Pack a cat_entrylist buffer with attributes from the catalog
+ *
+ * Note: index is zero relative
+ */
+__private_extern__
+int
+cat_getentriesattr(struct hfsmount *hfsmp, directoryhint_t *dirhint, struct cat_entrylist *ce_list)
+{
+	FCB* fcb;
+	CatalogKey * key;
+	BTreeIterator * iterator;
+	struct readattr_state state;
+	cnid_t parentcnid;
+	int i;
+	int std_hfs;
+	int index;
+	int have_key;
+	int result = 0;
+
+	ce_list->realentries = 0;
+
+	fcb = GetFileControlBlock(HFSTOVCB(hfsmp)->catalogRefNum);
+	std_hfs = (HFSTOVCB(hfsmp)->vcbSigWord == kHFSSigWord);
+	parentcnid = dirhint->dh_desc.cd_parentcnid;
+
+	state.hfsmp = hfsmp;
+	state.list = ce_list;
+	state.dir_cnid = parentcnid;
+	state.stdhfs = std_hfs;
+	state.error = 0;
+
+	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	bzero(iterator, sizeof(*iterator));
+	key = (CatalogKey *)&iterator->key;
+	have_key = 0;
 	iterator->hint.nodeNum = dirhint->dh_desc.cd_hint;
 	index = dirhint->dh_index + 1;
 
@@ -1762,7 +2488,7 @@ cat_getentriesattr(struct hfsmount *hfsmp, directoryhint_t *dirhint, struct cat_
 
 	/* Fill list with entries starting at iterator->key. */
 	result = BTIterateRecords(fcb, kBTreeNextRecord, iterator,
-			(IterateCallBackProcPtr)cat_readattr, &state);
+			(IterateCallBackProcPtr)getentriesattr_callback, &state);
 
 	if (state.error)
 		result = state.error;
@@ -1781,22 +2507,29 @@ cat_getentriesattr(struct hfsmount *hfsmp, directoryhint_t *dirhint, struct cat_
 		struct FndrFileInfo *fip;
 		struct cat_entry *cep;
 		struct HFSPlusCatalogFile filerec;
+		int isdirlink = 0;
+		int isfilelink = 0;
 
 		cep = &ce_list->entry[i];
-		if (!S_ISREG(cep->ce_attr.ca_mode))
+		if (cep->ce_attr.ca_linkref == 0)
 			continue;
-	
+		
 		/* Note: Finder info is still in Big Endian */
 		fip = (struct FndrFileInfo *)&cep->ce_attr.ca_finderinfo;
 
-		/* Check for hard link signature. */
-		if ((cep->ce_attr.ca_rdev != 0)
-		&&  (SWAP_BE32(fip->fdType) == kHardLinkFileType)
-		&&  (SWAP_BE32(fip->fdCreator) == kHFSPlusCreator)
-		&&  ((cep->ce_attr.ca_itime == (time_t)HFSTOVCB(hfsmp)->vcbCrDate) ||
-		     (cep->ce_attr.ca_itime == (time_t)hfsmp->hfs_metadata_createdate))) {
-
-			if (resolvelink(hfsmp, cep->ce_attr.ca_rdev, &filerec) != 0)
+		if (S_ISREG(cep->ce_attr.ca_mode) &&
+		    (SWAP_BE32(fip->fdType) == kHardLinkFileType) &&
+		    (SWAP_BE32(fip->fdCreator) == kHFSPlusCreator)) {
+			isfilelink = 1;
+		}
+		if (S_ISREG(cep->ce_attr.ca_mode) &&
+		    (SWAP_BE32(fip->fdType) == kHFSAliasType) &&
+		    (SWAP_BE32(fip->fdCreator) == kHFSAliasCreator) &&
+		    (cep->ce_attr.ca_recflags & kHFSHasLinkChainMask)) {
+			isdirlink = 1;
+		}
+		if (isfilelink || isdirlink) {
+			if (cat_resolvelink(hfsmp, cep->ce_attr.ca_linkref, isdirlink, &filerec) != 0)
 				continue;
 			/* Repack entry from inode record. */
 			getbsdattr(hfsmp, &filerec, &cep->ce_attr);		
@@ -1826,7 +2559,7 @@ struct linkinfo {
 };
 typedef struct linkinfo linkinfo_t;
 
-/* State information for the cat_packdirentry callback function. */
+/* State information for the getdirentries_callback function. */
 struct packdirentry_state {
 	int            cbs_extended;
 	u_int32_t      cbs_parentID;
@@ -1838,8 +2571,11 @@ struct packdirentry_state {
 	int32_t        cbs_maxlinks;
 	linkinfo_t *   cbs_linkinfo;
 	struct cat_desc * cbs_desc;
-//	struct dirent  * cbs_stdentry;
-	// followign fields are only used for NFS readdir, which uses the next file id as the seek offset of each entry
+	u_int8_t        * cbs_namebuf;
+	/*
+	 * The following fields are only used for NFS readdir, which
+	 * uses the next file id as the seek offset of each entry.
+	 */
 	struct direntry * cbs_direntry;
 	struct direntry * cbs_prevdirentry;
 	u_int32_t      cbs_previlinkref;
@@ -1847,12 +2583,15 @@ struct packdirentry_state {
 	Boolean        cbs_eof;
 };
 
+/*
+ * getdirentries callback for HFS Plus directories.
+ */
 static int
-cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
+getdirentries_callback(const CatalogKey *ckp, const CatalogRecord *crp,
                  struct packdirentry_state *state)
 {
 	struct hfsmount *hfsmp;
-	CatalogName *cnp;
+	const CatalogName *cnp;
 	cnid_t curID;
 	OSErr result;
 	struct dirent catent;
@@ -1862,10 +2601,10 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 	u_int32_t curlinkref = 0;
 	cnid_t  cnid;
 	int hide = 0;
-	u_int8_t type;
+	u_int8_t type = DT_UNKNOWN;
 	u_int8_t is_mangled = 0;
-	char *nameptr;
-	user_addr_t uiobase;
+	u_int8_t *nameptr;
+	user_addr_t uiobase = (user_addr_t)NULL;
 	size_t namelen = 0;
 	size_t maxnamelen;
 	size_t uiosize = 0;
@@ -1873,17 +2612,15 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 	Boolean stop_after_pack = false;
 	
 	hfsmp = state->cbs_hfsmp;
-
-	if (hfsmp->hfs_flags & HFS_STANDARD)
-		curID = ckp->hfs.parentID;
-	else
-		curID = ckp->hfsPlus.parentID;
+	curID = ckp->hfsPlus.parentID;
 
 	/* We're done when parent directory changes */
 	if (state->cbs_parentID != curID) {
 		if (state->cbs_extended) {
-			if (state->cbs_hasprevdirentry) { /* the last record haven't been returned yet, so we want to stop after
-											   * packing the last item */
+			/* The last record has not been returned yet, so we 
+			 * want to stop after packing the last item 
+			 */
+			if (state->cbs_hasprevdirentry) { 
 				stop_after_pack = true;
 			} else {
 				state->cbs_result = ENOENT;
@@ -1897,87 +2634,105 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 
 	if (state->cbs_extended) {
 		entry = state->cbs_direntry;
-		nameptr = &entry->d_name[0];
+		nameptr = (u_int8_t *)&entry->d_name[0];
 		maxnamelen = NAME_MAX;
 	} else {
-		nameptr = &catent.d_name[0];
+		nameptr = (u_int8_t *)&catent.d_name[0];
 		maxnamelen = NAME_MAX;
 	}
 
 	if (state->cbs_extended && stop_after_pack) {
-		cnid = INT_MAX;			/* the last item returns a non-zero invalid cookie */
+		/* The last item returns a non-zero invalid cookie */
+		cnid = INT_MAX;		
 	} else {
-		if (!(hfsmp->hfs_flags & HFS_STANDARD)) {
-			switch(crp->recordType) {
-			case kHFSPlusFolderRecord:
-				type = DT_DIR;
-				cnid = crp->hfsPlusFolder.folderID;
-				/* Hide our private meta data directory */
-				if ((curID == kHFSRootFolderID) &&
-					(cnid == hfsmp->hfs_privdir_desc.cd_cnid)) {
+		switch(crp->recordType) {
+		case kHFSPlusFolderRecord:
+			type = DT_DIR;
+			cnid = crp->hfsPlusFolder.folderID;
+			/* Hide our private system directories. */
+			if (curID == kHFSRootFolderID) {
+				if (cnid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+				    cnid == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
 					hide = 1;
 				}
-
-				break;
-			case kHFSPlusFileRecord:
-				itime = to_bsd_time(crp->hfsPlusFile.createDate);
-				/*
-				 * When a hardlink link is encountered save its link ref.
-				 */
-				if ((SWAP_BE32(crp->hfsPlusFile.userInfo.fdType) == kHardLinkFileType) &&
-					(SWAP_BE32(crp->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator) &&
-					((itime == (time_t)hfsmp->hfs_itime) ||
-					 (itime == (time_t)hfsmp->hfs_metadata_createdate))) {
-					ilinkref = crp->hfsPlusFile.bsdInfo.special.iNodeNum;
+			}
+			break;
+		case kHFSPlusFileRecord:
+			itime = to_bsd_time(crp->hfsPlusFile.createDate);
+			type = MODE_TO_DT(crp->hfsPlusFile.bsdInfo.fileMode);
+			cnid = crp->hfsPlusFile.fileID;
+			/*
+			 * When a hardlink link is encountered save its link ref.
+			 */
+			if ((SWAP_BE32(crp->hfsPlusFile.userInfo.fdType) == kHardLinkFileType) &&
+				(SWAP_BE32(crp->hfsPlusFile.userInfo.fdCreator) == kHFSPlusCreator) &&
+				((itime == (time_t)hfsmp->hfs_itime) ||
+				 (itime == (time_t)hfsmp->hfs_metadata_createdate))) {
+				/* If link ref is inode's file id then use it directly. */
+				if (crp->hfsPlusFile.flags & kHFSHasLinkChainMask) {
+					cnid = crp->hfsPlusFile.hl_linkReference;
+				} else {
+					ilinkref = crp->hfsPlusFile.hl_linkReference;
 				}
-				type = MODE_TO_DT(crp->hfsPlusFile.bsdInfo.fileMode);
-				cnid = crp->hfsPlusFile.fileID;
-				/* Hide the journal files */
-				if ((curID == kHFSRootFolderID) &&
-					((hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))) &&
-					((cnid == hfsmp->hfs_jnlfileid) ||
-					 (cnid == hfsmp->hfs_jnlinfoblkid))) {
-					hide = 1;
+			} else if ((SWAP_BE32(crp->hfsPlusFile.userInfo.fdType) == kHFSAliasType) &&
+				(SWAP_BE32(crp->hfsPlusFile.userInfo.fdCreator) == kHFSAliasCreator) &&
+				(crp->hfsPlusFile.flags & kHFSHasLinkChainMask) &&
+				(crp->hfsPlusFile.hl_linkReference >= kHFSFirstUserCatalogNodeID) &&
+				((itime == (time_t)hfsmp->hfs_itime) ||
+				 (itime == (time_t)hfsmp->hfs_metadata_createdate))) {
+				/* A directory's link resolves to a directory. */
+				type = DT_DIR;
+				/* A directory's link ref is always inode's file id. */
+				cnid = crp->hfsPlusFile.hl_linkReference;
+			}
+			/* Hide the journal files */
+			if ((curID == kHFSRootFolderID) &&
+				((hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))) &&
+				((cnid == hfsmp->hfs_jnlfileid) ||
+				 (cnid == hfsmp->hfs_jnlinfoblkid))) {
+				hide = 1;
+			}
+			break;
+		default:
+			return (0);	/* stop */
+		};
+
+		cnp = (const CatalogName*) &ckp->hfsPlus.nodeName;
+
+		namelen = cnp->ustr.length;
+		/*
+		 * For MacRoman encoded names, assume that its ascii and
+		 * convert it directly in an attempt to avoid the more
+		 * expensive utf8_encodestr conversion.
+		 */
+		if ((namelen < maxnamelen) && (crp->hfsPlusFile.textEncoding == 0)) {
+			int i;
+			u_int16_t ch;
+			const u_int16_t *chp;
+
+			chp = &cnp->ustr.unicode[0];
+			for (i = 0; i < (int)namelen; ++i) {
+				ch = *chp++;
+				if (ch > 0x007f || ch == 0x0000) {
+					/* Perform expensive utf8_encodestr conversion */
+					goto encodestr;
 				}
-				break;
-			default:
-				return (0);	/* stop */
-			};
-
-			cnp = (CatalogName*) &ckp->hfsPlus.nodeName;
-			result = utf8_encodestr(cnp->ustr.unicode, cnp->ustr.length * sizeof(UniChar),
-									nameptr, &namelen, maxnamelen + 1, ':', 0);
-			if (result == ENAMETOOLONG) {
-				result = ConvertUnicodeToUTF8Mangled(cnp->ustr.length * sizeof(UniChar),
-													 cnp->ustr.unicode, maxnamelen + 1,
-													 (ByteCount*)&namelen, nameptr,
-													 cnid);		
-				is_mangled = 1;
+				nameptr[i] = (ch == '/') ? ':' : (u_int8_t)ch;
 			}
-		} else { /* hfs */
-			switch(crp->recordType) {
-			case kHFSFolderRecord:
-				type = DT_DIR;
-				cnid = crp->hfsFolder.folderID;
-				break;
-			case kHFSFileRecord:
-				type = DT_REG;
-				cnid = crp->hfsFile.fileID;
-				break;
-			default:
-				return (0);	/* stop */
-			};
+			nameptr[namelen] = '\0';
+			result = 0;
+		} else {
+encodestr:
+			result = utf8_encodestr(cnp->ustr.unicode, namelen * sizeof(UniChar),
+						nameptr, &namelen, maxnamelen + 1, ':', 0);
+		}
 
-			cnp = (CatalogName*) ckp->hfs.nodeName;
-			result = hfs_to_utf8(hfsmp, cnp->pstr, maxnamelen + 1,
-								 (ByteCount *)&namelen, nameptr);
-			/*
-			 * When an HFS name cannot be encoded with the current
-			 * volume encoding we use MacRoman as a fallback.
-			 */
-			if (result)
-				result = mac_roman_to_utf8(cnp->pstr, maxnamelen + 1,
-										   (ByteCount *)&namelen, nameptr);
+		/* Check result returned from encoding the filename to utf8 */
+		if (result == ENAMETOOLONG) {
+			result = ConvertUnicodeToUTF8Mangled(cnp->ustr.length * sizeof(UniChar),
+							     cnp->ustr.unicode, maxnamelen + 1,
+							     (ByteCount*)&namelen, nameptr, cnid);		
+			is_mangled = 1;
 		}
 	}
 
@@ -1985,8 +2740,9 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 		/*
 		 * The index is 1 relative and includes "." and ".."
 		 *
-		 * Also stuff the cnid in the upper 32 bits of the cookie.  The cookie is stored to the previous entry, which
-		 * will be packed and copied this time
+		 * Also stuff the cnid in the upper 32 bits of the cookie.
+		 * The cookie is stored to the previous entry, which will
+		 * be packed and copied this time
 		 */
 		state->cbs_prevdirentry->d_seekoff = (state->cbs_index + 3) | ((u_int64_t)cnid << 32);
 		uiosize = state->cbs_prevdirentry->d_reclen;
@@ -2003,8 +2759,9 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 	}
 
 	/* Save current base address for post processing of hard-links. */
-	uiobase = uio_curriovbase(state->cbs_uio);
-
+	if (ilinkref || state->cbs_previlinkref) {
+		uiobase = uio_curriovbase(state->cbs_uio);
+	}
 	/* If this entry won't fit then we're done */
 	if ((uiosize > uio_resid(state->cbs_uio)) ||
 	    (ilinkref != 0 && state->cbs_nlinks == state->cbs_maxlinks)) {
@@ -2024,34 +2781,33 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 				state->cbs_desc->cd_flags &= ~CD_ISDIR;
 			}
 			if (state->cbs_desc->cd_nameptr != NULL) {
-				vfs_removename(state->cbs_desc->cd_nameptr);
+				state->cbs_desc->cd_namelen = 0;
 			}
 #if 0
 			state->cbs_desc->cd_encoding = xxxx;
 #endif
 			if (!is_mangled) {
 				state->cbs_desc->cd_namelen = namelen;
-				state->cbs_desc->cd_nameptr = vfs_addname(nameptr, namelen, 0, 0);
+				bcopy(nameptr, state->cbs_namebuf, namelen + 1);
 			} else {
 				/* Store unmangled name for the directory hint else it will 
 				 * restart readdir at the last location again 
 				 */
-				char *new_nameptr;
+				u_int8_t *new_nameptr;
 				size_t bufsize;
 				size_t tmp_namelen = 0;
 			
-				cnp = (CatalogName *)&ckp->hfsPlus.nodeName;
+				cnp = (const CatalogName *)&ckp->hfsPlus.nodeName;
 				bufsize = 1 + utf8_encodelen(cnp->ustr.unicode,
-											 cnp->ustr.length * sizeof(UniChar),
-											 ':', 0);
-				MALLOC(new_nameptr, char *, bufsize, M_TEMP, M_WAITOK);
+				                             cnp->ustr.length * sizeof(UniChar),
+				                             ':', 0);
+				MALLOC(new_nameptr, u_int8_t *, bufsize, M_TEMP, M_WAITOK);
 				result = utf8_encodestr(cnp->ustr.unicode,
-										cnp->ustr.length * sizeof(UniChar),
-										new_nameptr, &tmp_namelen,
-										bufsize, ':', 0);
+				                        cnp->ustr.length * sizeof(UniChar),
+				                        new_nameptr, &tmp_namelen, bufsize, ':', 0);
 			
 				state->cbs_desc->cd_namelen = tmp_namelen;
-				state->cbs_desc->cd_nameptr = vfs_addname(new_nameptr, tmp_namelen, 0, 0);
+				bcopy(new_nameptr, state->cbs_namebuf, tmp_namelen + 1);
 			
 				FREE(new_nameptr, M_TEMP);
 			} 
@@ -2075,7 +2831,8 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 		}
 	}
 
-	if (state->cbs_extended) {	/* fill the direntry to be used the next time */
+	/* Fill the direntry to be used the next time */
+	if (state->cbs_extended) {	
 		if (stop_after_pack) {
 			state->cbs_eof = true;
 			return (0);	/* stop */
@@ -2083,10 +2840,12 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 		entry->d_type = type;
 		entry->d_namlen = namelen;
 		entry->d_reclen = EXT_DIRENT_LEN(namelen);
-		if (hide)
-			entry->d_fileno = 0;  /* file number = 0 means skip entry */
-		else
+		if (hide) {
+			/* File number = 0 means skip entry */
+			entry->d_fileno = 0;  
+		} else {
 			entry->d_fileno = cnid;
+		}
 		/* swap the current and previous entry */
 		struct direntry * tmp;
 		tmp = state->cbs_direntry;
@@ -2101,6 +2860,93 @@ cat_packdirentry(const CatalogKey *ckp, const CatalogRecord *crp,
 		uio_resid(state->cbs_uio) >= SMALL_DIRENTRY_SIZE);
 }
 
+/*
+ * getdirentries callback for standard HFS (non HFS+) directories.
+ */
+static int
+getdirentries_std_callback(const CatalogKey *ckp, const CatalogRecord *crp,
+                           struct packdirentry_state *state)
+{
+	struct hfsmount *hfsmp;
+	const CatalogName *cnp;
+	cnid_t curID;
+	OSErr result;
+	struct dirent catent;
+	cnid_t  cnid;
+	u_int8_t type = DT_UNKNOWN;
+	u_int8_t *nameptr;
+	size_t namelen = 0;
+	size_t maxnamelen;
+	size_t uiosize = 0;
+	caddr_t uioaddr;
+	
+	hfsmp = state->cbs_hfsmp;
+
+	curID = ckp->hfs.parentID;
+
+	/* We're done when parent directory changes */
+	if (state->cbs_parentID != curID) {
+		state->cbs_result = ENOENT;
+		return (0);	/* stop */
+	}
+
+	nameptr = (u_int8_t *)&catent.d_name[0];
+	maxnamelen = NAME_MAX;
+
+	switch(crp->recordType) {
+	case kHFSFolderRecord:
+		type = DT_DIR;
+		cnid = crp->hfsFolder.folderID;
+		break;
+	case kHFSFileRecord:
+		type = DT_REG;
+		cnid = crp->hfsFile.fileID;
+		break;
+	default:
+		return (0);	/* stop */
+	};
+
+	cnp = (const CatalogName*) ckp->hfs.nodeName;
+	result = hfs_to_utf8(hfsmp, cnp->pstr, maxnamelen + 1, (ByteCount *)&namelen, nameptr);
+	/*
+	 * When an HFS name cannot be encoded with the current
+	 * volume encoding we use MacRoman as a fallback.
+	 */
+	if (result) {
+		result = mac_roman_to_utf8(cnp->pstr, maxnamelen + 1, (ByteCount *)&namelen, nameptr);
+	}
+	catent.d_type = type;
+	catent.d_namlen = namelen;
+	catent.d_reclen = uiosize = STD_DIRENT_LEN(namelen);
+	catent.d_fileno = cnid;
+	uioaddr = (caddr_t) &catent;
+
+	/* If this entry won't fit then we're done */
+	if (uiosize > uio_resid(state->cbs_uio)) {
+		return (0);	/* stop */
+	}
+
+	state->cbs_result = uiomove(uioaddr, uiosize, state->cbs_uio);
+	if (state->cbs_result == 0) {
+		++state->cbs_index;
+
+		/* Remember previous entry */
+		state->cbs_desc->cd_cnid = cnid;
+		if (type == DT_DIR) {
+			state->cbs_desc->cd_flags |= CD_ISDIR;
+		} else {
+			state->cbs_desc->cd_flags &= ~CD_ISDIR;
+		}
+		if (state->cbs_desc->cd_nameptr != NULL) {
+			state->cbs_desc->cd_namelen = 0;
+		}
+		state->cbs_desc->cd_namelen = namelen;
+		bcopy(nameptr, state->cbs_namebuf, namelen + 1);
+	}
+
+	/* Continue iteration if there's room */
+	return (state->cbs_result == 0  && uio_resid(state->cbs_uio) >= SMALL_DIRENTRY_SIZE);
+}
 
 /*
  * Pack a uio buffer with directory entries from the catalog
@@ -2120,14 +2966,17 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	int result;
 	int index;
 	int have_key;
-	
-	fcb = GetFileControlBlock(hfsmp->hfs_catalog_vp);
+
+	if (extended && (hfsmp->hfs_flags & HFS_STANDARD)) {
+		return (ENOTSUP);
+	}
+	fcb = hfsmp->hfs_catalog_cp->c_datafork;
 
 	/*
 	 * Get a buffer for link info array, btree iterator and a direntry:
 	 */
 	maxlinks = MIN(entrycnt, uio_resid(uio) / SMALL_DIRENTRY_SIZE);
-	bufsize = (maxlinks * sizeof(linkinfo_t)) + sizeof(*iterator);
+	bufsize = MAXPATHLEN + (maxlinks * sizeof(linkinfo_t)) + sizeof(*iterator);
 	if (extended) {
 		bufsize += 2*sizeof(struct direntry);
 	}
@@ -2139,9 +2988,9 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	state.cbs_previlinkref = 0;
 	state.cbs_nlinks = 0;
 	state.cbs_maxlinks = maxlinks;
-	state.cbs_linkinfo = (linkinfo_t *) buffer;
+	state.cbs_linkinfo = (linkinfo_t *)((char *)buffer + MAXPATHLEN);
 
-	iterator = (BTreeIterator *) ((char *)buffer + (maxlinks * sizeof(linkinfo_t)));
+	iterator = (BTreeIterator *) ((char *)state.cbs_linkinfo + (maxlinks * sizeof(linkinfo_t)));
 	key = (CatalogKey *)&iterator->key;
 	have_key = 0;
 	index = dirhint->dh_index + 1;
@@ -2155,14 +3004,26 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	 */
 	if (dirhint->dh_desc.cd_namelen != 0) {
 		if (buildkey(hfsmp, &dirhint->dh_desc, (HFSPlusCatalogKey *)key, 0) == 0) {
+			iterator->hint.nodeNum = dirhint->dh_desc.cd_hint;
 			have_key = 1;
 		}
 	}
 
+	if (index == 0 && dirhint->dh_threadhint != 0) {
+		/*
+		 * Position the iterator at the directory's thread record.
+		 * (i.e. just before the first entry)
+		 */
+		buildthreadkey(dirhint->dh_desc.cd_parentcnid, (hfsmp->hfs_flags & HFS_STANDARD), key);
+		iterator->hint.nodeNum = dirhint->dh_threadhint;
+		iterator->hint.index = 0;
+		have_key = 1;
+	}
+
 	/*
 	 * If the last entry wasn't cached then position the btree iterator
 	 */
-	if ((index == 0) || !have_key) {
+	if (!have_key) {
 		/*
 		 * Position the iterator at the directory's thread record.
 		 * (i.e. just before the first entry)
@@ -2173,7 +3034,9 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 			result = MacToVFSError(result);
 			goto cleanup;
 		}
-	
+		if (index == 0) {
+			dirhint->dh_threadhint = iterator->hint.nodeNum;
+		}
 		/*
 		 * Iterate until we reach the entry just
 		 * before the one we want to start with.
@@ -2204,9 +3067,20 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	state.cbs_hfsmp = hfsmp;
 	state.cbs_uio = uio;
 	state.cbs_desc = &dirhint->dh_desc;
+	state.cbs_namebuf = (u_int8_t *)buffer;
 	state.cbs_result = 0;
 	state.cbs_parentID = dirhint->dh_desc.cd_parentcnid;
 
+	/* Use a temporary buffer to hold intermediate descriptor names. */
+	if (dirhint->dh_desc.cd_namelen > 0 && dirhint->dh_desc.cd_nameptr != NULL) {
+		bcopy(dirhint->dh_desc.cd_nameptr, buffer, dirhint->dh_desc.cd_namelen+1);
+		if (dirhint->dh_desc.cd_flags & CD_HASBUF) {
+			dirhint->dh_desc.cd_flags &= ~CD_HASBUF;
+			vfs_removename((const char *)dirhint->dh_desc.cd_nameptr);
+		}
+	}
+	dirhint->dh_desc.cd_nameptr = (u_int8_t *)buffer;
+
 	enum BTreeIterationOperations op;
 	if (extended && index != 0 && have_key)
 		op = kBTreeCurrentRecord;
@@ -2216,21 +3090,29 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	/*
 	 * Process as many entries as possible starting at iterator->key.
 	 */
-	result = BTIterateRecords(fcb, op, iterator,
-	                          (IterateCallBackProcPtr)cat_packdirentry, &state);
-
-	/* If readdir is called for NFS and BTIterateRecords reaches the end of the
-	 * Catalog BTree, call cat_packdirentry() with dummy values to copy previous 
-	 * direntry stored in state to the user buffer.
-	 */
-	if (state.cbs_extended && (result == fsBTRecordNotFoundErr)) {
-		CatalogKey ckp;
-		CatalogRecord crp;
+	if (hfsmp->hfs_flags & HFS_STANDARD)
+		result = BTIterateRecords(fcb, op, iterator,
+	                          (IterateCallBackProcPtr)getdirentries_std_callback, &state);
+	else {
+		result = BTIterateRecords(fcb, op, iterator,
+	                          (IterateCallBackProcPtr)getdirentries_callback, &state);
+	
+		/* For extended calls, every call to getdirentries_callback() 
+		 * transfers the previous directory entry found to the user 
+		 * buffer.  Therefore when BTIterateRecords reaches the end of 
+		 * Catalog BTree, call getdirentries_callback() again with 
+		 * dummy values to copy the last directory entry stored in 
+		 * packdirentry_state 
+		 */
+		if (state.cbs_extended && (result == fsBTRecordNotFoundErr)) {
+			CatalogKey ckp;
+			CatalogRecord crp;
 
-		bzero(&ckp, sizeof(ckp));
-		bzero(&crp, sizeof(crp));
+			bzero(&ckp, sizeof(ckp));
+			bzero(&crp, sizeof(crp));
 
-		result = cat_packdirentry(&ckp, &crp, &state);
+			result = getdirentries_callback(&ckp, &crp, &state);
+		}
 	}
 
 	/* Note that state.cbs_index is still valid on errors */
@@ -2246,6 +3128,15 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 	dirhint->dh_desc.cd_flags |= CD_DECOMPOSED;
 	dirhint->dh_index = index - 1;
 	
+	/* Fix up the name. */
+	if (dirhint->dh_desc.cd_namelen > 0) {
+		dirhint->dh_desc.cd_nameptr = (const u_int8_t *)vfs_addname((char *)buffer, dirhint->dh_desc.cd_namelen, 0, 0);
+		dirhint->dh_desc.cd_flags |= CD_HASBUF;
+	} else {
+		dirhint->dh_desc.cd_nameptr = NULL;
+		dirhint->dh_desc.cd_namelen = 0;
+	}
+	
 	/*
 	 * Post process any hard links to get the real file id.
 	 */
@@ -2262,11 +3153,19 @@ cat_getdirentries(struct hfsmount *hfsmp, int entrycnt, directoryhint_t *dirhint
 			if (address == (user_addr_t)0)
 				continue;
 			if (uio_isuserspace(uio)) {
-				(void) copyout(&fileid, address,
-				               extended ? sizeof(ino64_t) : sizeof(ino_t));
+				if (extended) {
+					ino64_t fileid_64 = (ino64_t)fileid;
+					(void) copyout(&fileid_64, address, sizeof(fileid_64));
+				} else {
+					(void) copyout(&fileid, address, sizeof(fileid));
+				}
 			} else /* system space */ {
-				ino64_t *inoptr = (ino64_t *)CAST_DOWN(caddr_t, address);
-				*inoptr = fileid;
+				if (extended) {
+					ino64_t fileid_64 = (ino64_t)fileid;
+					bcopy(&fileid_64, (void*) CAST_DOWN(caddr_t, address), sizeof(fileid_64));
+				} else {
+					bcopy(&fileid, (void*) CAST_DOWN(caddr_t, address), sizeof(fileid));
+				}
 			}
 		}
 	}
@@ -2455,7 +3354,7 @@ static int
 buildkey(struct hfsmount *hfsmp, struct cat_desc *descp,
 	HFSPlusCatalogKey *key, int retry)
 {
-	int utf8_flags = 0;
+	int utf8_flags = UTF_ESCAPE_ILLEGAL;
 	int result = 0;
 	size_t unicodeBytes = 0;
 
@@ -2513,24 +3412,32 @@ buildkey(struct hfsmount *hfsmp, struct cat_desc *descp,
  */
 __private_extern__
 int
-resolvelink(struct hfsmount *hfsmp, u_long linkref, struct HFSPlusCatalogFile *recp)
+cat_resolvelink(struct hfsmount *hfsmp, u_long linkref, int isdirlink, struct HFSPlusCatalogFile *recp)
 {
 	FSBufferDescriptor btdata;
 	struct BTreeIterator *iterator;
 	struct cat_desc idesc;
 	char inodename[32];
+	cnid_t parentcnid;
 	int result = 0;
 
 	BDINIT(btdata, recp);
-	MAKE_INODE_NAME(inodename, linkref);
+
+	if (isdirlink) {
+		MAKE_DIRINODE_NAME(inodename, sizeof(inodename), (unsigned int)linkref);
+		parentcnid = hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid;
+	} else {
+		MAKE_INODE_NAME(inodename, sizeof(inodename), (unsigned int)linkref);
+		parentcnid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+	}
 
 	/* Get space for iterator */	
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
 	bzero(iterator, sizeof(*iterator));
 
 	/* Build a descriptor for private dir. */	
-	idesc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
-	idesc.cd_nameptr = inodename;
+	idesc.cd_parentcnid = parentcnid;
+	idesc.cd_nameptr = (const u_int8_t *)inodename;
 	idesc.cd_namelen = strlen(inodename);
 	idesc.cd_flags = 0;
 	idesc.cd_hint = 0;
@@ -2542,8 +3449,8 @@ resolvelink(struct hfsmount *hfsmp, u_long linkref, struct HFSPlusCatalogFile *r
 
 	if (result == 0) {
 		/* Make sure there's a reference */
-		if (recp->bsdInfo.special.linkCount == 0)
-			recp->bsdInfo.special.linkCount = 2;
+		if (recp->hl_linkCount == 0)
+			recp->hl_linkCount = 2;
 	} else {
 		printf("HFS resolvelink: can't find %s\n", inodename);
 	}
@@ -2561,8 +3468,14 @@ resolvelinkid(struct hfsmount *hfsmp, u_long linkref, ino_t *ino)
 {
 	struct HFSPlusCatalogFile record;
 	int error;
-	
-	error = resolvelink(hfsmp, linkref, &record);
+
+	/*
+	 * Since we know resolvelinkid is only called from
+	 * cat_getdirentries, we can assume that only file
+	 * hardlinks need to be resolved (cat_getdirentries
+	 * can resolve directory hardlinks in place).
+	 */
+	error = cat_resolvelink(hfsmp, linkref, 0, &record);
 	if (error == 0) {
 		if (record.fileID == 0)
 			error = ENOENT;
@@ -2580,7 +3493,7 @@ getkey(struct hfsmount *hfsmp, cnid_t cnid, CatalogKey * key)
 {
 	struct BTreeIterator * iterator;
 	FSBufferDescriptor btdata;
-	UInt16	datasize;
+	u_int16_t	datasize;
 	CatalogKey * keyp;
 	CatalogRecord * recp;
 	int result;
@@ -2633,6 +3546,9 @@ exit:
  * getkeyplusattr - From id, fetch the key and the bsd attrs for a file/dir (could pass
  * null arguments to cat_idlookup instead, but we save around 10% by not building the 
  * cat_desc here). Both key and attrp must point to real structures.
+ *
+ * The key's parent id is the only part of the key expected to be used by the caller.
+ * The name portion of the key may not always be valid (ie in the case of a hard link).
  */
 __private_extern__
 int
@@ -2643,9 +3559,32 @@ cat_getkeyplusattr(struct hfsmount *hfsmp, cnid_t cnid, CatalogKey * key, struct
 	result = getkey(hfsmp, cnid, key);
        
 	if (result == 0) {
-		result = cat_lookupbykey(hfsmp, key, 0, 0, NULL, attrp, NULL, NULL);
+		result = cat_lookupbykey(hfsmp, key, 0, 0, 0, NULL, attrp, NULL, NULL);
 	}
+	/*
+	 * Check for a raw file hardlink inode.
+	 * Fix up the parent id in the key if necessary.
+	 * Only hard links created by Mac OS X 10.5 or later can be resolved here.
+	 */
+	if ((result == 0) &&
+	    (key->hfsPlus.parentID == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) &&
+	    (attrp->ca_recflags & kHFSHasLinkChainMask)) {
+		cnid_t nextlinkid = 0;
+		cnid_t prevlinkid = 0;
+		struct cat_desc linkdesc;
 
+		/*
+		 * Pick up the first link in the chain and get a descriptor for it.
+		 * This allows blind bulk access checks to work for hardlinks.
+		 */
+		if ((cat_lookuplinkbyid(hfsmp, cnid, &prevlinkid,  &nextlinkid) == 0) &&
+		    (nextlinkid != 0)) {
+			if (cat_findname(hfsmp, nextlinkid, &linkdesc) == 0) {
+				key->hfsPlus.parentID = linkdesc.cd_parentcnid;
+				cat_releasedesc(&linkdesc);
+			}
+		}	
+	}
 	return MacToVFSError(result);
 }
 
@@ -2655,7 +3594,7 @@ cat_getkeyplusattr(struct hfsmount *hfsmp, cnid_t cnid, CatalogKey * key, struct
  */
 static void
 buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding,
-            CatalogRecord *crp, int *recordSize)
+            CatalogRecord *crp, u_int32_t *recordSize)
 {
 	int type = attrp->ca_mode & S_IFMT;
 	u_int32_t createtime = to_hfs_time(attrp->ca_itime);
@@ -2682,11 +3621,10 @@ buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding
 		}
 	} else {
 		struct HFSPlusBSDInfo * bsdp = NULL;
-		struct FndrFileInfo * fip = NULL;
 
 		if (type == S_IFDIR) {
 			crp->recordType = kHFSPlusFolderRecord;
-			crp->hfsPlusFolder.flags = 0;
+			crp->hfsPlusFolder.flags = attrp->ca_recflags;
 			crp->hfsPlusFolder.valence = 0;
 			crp->hfsPlusFolder.folderID = cnid;	
 			crp->hfsPlusFolder.createDate = createtime;
@@ -2695,14 +3633,14 @@ buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding
 			crp->hfsPlusFolder.accessDate = createtime;
 			crp->hfsPlusFolder.backupDate = 0;
 			crp->hfsPlusFolder.textEncoding = encoding;
-			crp->hfsPlusFolder.attrBlocks = 0;
+			crp->hfsPlusFolder.folderCount = 0;
 			bcopy(attrp->ca_finderinfo, &crp->hfsPlusFolder.userInfo, 32);
 			bsdp = &crp->hfsPlusFolder.bsdInfo;
-			bsdp->special.rawDevice = 0;
+			bsdp->special.linkCount = 1;
 			*recordSize = sizeof(HFSPlusCatalogFolder);
 		} else {
 			crp->recordType = kHFSPlusFileRecord;
-			crp->hfsPlusFile.flags = kHFSThreadExistsMask;
+			crp->hfsPlusFile.flags = attrp->ca_recflags;
 			crp->hfsPlusFile.reserved1 = 0;
 			crp->hfsPlusFile.fileID = cnid;
 			crp->hfsPlusFile.createDate = createtime;
@@ -2711,29 +3649,14 @@ buildrecord(struct cat_attr *attrp, cnid_t cnid, int std_hfs, u_int32_t encoding
 			crp->hfsPlusFile.attributeModDate = createtime;
 			crp->hfsPlusFile.backupDate = 0;
 			crp->hfsPlusFile.textEncoding = encoding;
-			crp->hfsPlusFile.attrBlocks = 0;
+			crp->hfsPlusFile.reserved2 = 0;
+			bcopy(attrp->ca_finderinfo, &crp->hfsPlusFile.userInfo, 32);
 			bsdp = &crp->hfsPlusFile.bsdInfo;
-			bsdp->special.rawDevice = 0;
-			switch(type) {
-			case S_IFBLK:
-			case S_IFCHR:
-				/* BLK/CHR need to save the device info */
+			/* BLK/CHR need to save the device info */
+			if (type == S_IFBLK || type == S_IFCHR) {
 				bsdp->special.rawDevice = attrp->ca_rdev;
-				bzero(&crp->hfsPlusFile.userInfo, 32);
-				break;
-			case S_IFREG:
-				/* Hardlink links need to save the linkref */
-				fip = (FndrFileInfo *)&attrp->ca_finderinfo;
-				if ((SWAP_BE32(fip->fdType) == kHardLinkFileType) &&
-				    (SWAP_BE32(fip->fdCreator) == kHFSPlusCreator)) {
-					bsdp->special.iNodeNum = attrp->ca_rdev;
-				}
-				bcopy(attrp->ca_finderinfo, &crp->hfsPlusFile.userInfo, 32);
-				break;
-			case S_IFLNK:
-				/* Symlinks also have a type and creator */
-				bcopy(attrp->ca_finderinfo, &crp->hfsPlusFile.userInfo, 32);
-				break;
+			} else {
+				bsdp->special.linkCount = 1;
 			}
 			bzero(&crp->hfsPlusFile.dataFork, 2*sizeof(HFSPlusForkData));
 			*recordSize = sizeof(HFSPlusCatalogFile);
@@ -2755,15 +3678,15 @@ builddesc(const HFSPlusCatalogKey *key, cnid_t cnid, u_long hint, u_long encodin
 	int isdir, struct cat_desc *descp)
 {
 	int result = 0;
-	char * nameptr;
+	unsigned char * nameptr;
 	size_t bufsize;
 	size_t utf8len;
-	char tmpbuff[128];
+	unsigned char tmpbuff[128];
 
 	/* guess a size... */
 	bufsize = (3 * key->nodeName.length) + 1;
 	if (bufsize >= sizeof(tmpbuff) - 1) {
-	    MALLOC(nameptr, char *, bufsize, M_TEMP, M_WAITOK);
+	    MALLOC(nameptr, unsigned char *, bufsize, M_TEMP, M_WAITOK);
 	} else {
 	    nameptr = &tmpbuff[0];
 	}
@@ -2778,7 +3701,7 @@ builddesc(const HFSPlusCatalogKey *key, cnid_t cnid, u_long hint, u_long encodin
 		                             key->nodeName.length * sizeof(UniChar),
 		                             ':', 0);
 		FREE(nameptr, M_TEMP);
-		MALLOC(nameptr, char *, bufsize, M_TEMP, M_WAITOK);
+		MALLOC(nameptr, unsigned char *, bufsize, M_TEMP, M_WAITOK);
 
 		result = utf8_encodestr(key->nodeName.unicode,
 		                        key->nodeName.length * sizeof(UniChar),
@@ -2786,7 +3709,7 @@ builddesc(const HFSPlusCatalogKey *key, cnid_t cnid, u_long hint, u_long encodin
 		                        bufsize, ':', 0);
 	}
 	descp->cd_parentcnid = key->parentID;
-	descp->cd_nameptr = vfs_addname(nameptr, utf8len, 0, 0);
+	descp->cd_nameptr = (const u_int8_t *)vfs_addname((char *)nameptr, utf8len, 0, 0);
 	descp->cd_namelen = utf8len;
 	descp->cd_cnid = cnid;
 	descp->cd_hint = hint;
@@ -2812,7 +3735,6 @@ getbsdattr(struct hfsmount *hfsmp, const struct HFSPlusCatalogFile *crp, struct
 	const struct HFSPlusBSDInfo *bsd = &crp->bsdInfo;
 
 	attrp->ca_recflags = crp->flags;
-	attrp->ca_nlink = 1;
 	attrp->ca_atime = to_bsd_time(crp->accessDate);
 	attrp->ca_atimeondisk = attrp->ca_atime;	
 	attrp->ca_mtime = to_bsd_time(crp->contentModDate);
@@ -2824,12 +3746,15 @@ getbsdattr(struct hfsmount *hfsmp, const struct HFSPlusCatalogFile *crp, struct
 		attrp->ca_flags = 0;
 		attrp->ca_uid = hfsmp->hfs_uid;
 		attrp->ca_gid = hfsmp->hfs_gid;
-		if (isDirectory)
+		if (isDirectory) {
 			attrp->ca_mode = S_IFDIR | (hfsmp->hfs_dir_mask & ACCESSPERMS);
-		else
+		} else {
 			attrp->ca_mode = S_IFREG | (hfsmp->hfs_file_mask & ACCESSPERMS);
+		}
+		attrp->ca_linkcount = 1;
 		attrp->ca_rdev = 0;
 	} else {
+		attrp->ca_linkcount = 1;  /* may be overridden below */
 		attrp->ca_rdev = 0;
 		attrp->ca_uid = bsd->ownerID;
 		attrp->ca_gid = bsd->groupID;
@@ -2840,24 +3765,33 @@ getbsdattr(struct hfsmount *hfsmp, const struct HFSPlusCatalogFile *crp, struct
 		case S_IFBLK:
 			attrp->ca_rdev = bsd->special.rawDevice;
 			break;
+			
+		case S_IFDIR: /* fall through */
 		case S_IFREG:
 			/* Pick up the hard link count */
 			if (bsd->special.linkCount > 0)
-				attrp->ca_nlink = bsd->special.linkCount;
+				attrp->ca_linkcount = bsd->special.linkCount;
 			break;
 		}
 
-		if (((unsigned int)vfs_flags(HFSTOVFS(hfsmp))) & MNT_UNKNOWNPERMISSIONS) {
-			/*
-			 *  Override the permissions as determined by the mount auguments
-			 *  in ALMOST the same way unset permissions are treated but keep
-			 *  track of whether or not the file or folder is hfs locked
-			 *  by leaving the h_pflags field unchanged from what was unpacked
-			 *  out of the catalog.
-			 */
-			attrp->ca_uid = hfsmp->hfs_uid;
-			attrp->ca_gid = hfsmp->hfs_gid;
-		}
+		/*
+		 *  Override the permissions as determined by the mount auguments
+		 *  in ALMOST the same way unset permissions are treated but keep
+		 *  track of whether or not the file or folder is hfs locked
+		 *  by leaving the h_pflags field unchanged from what was unpacked
+		 *  out of the catalog.
+		 */
+		/*
+		 * This code was used to do UID translation with MNT_IGNORE_OWNERS
+		 * (aka MNT_UNKNOWNPERMISSIONS) at the HFS layer.  It's largely done
+		 * at the VFS layer, so there is no need to do it here now; this also
+		 * allows VFS to let root see the real UIDs.
+		 *
+		 * if (((unsigned int)vfs_flags(HFSTOVFS(hfsmp))) & MNT_UNKNOWNPERMISSIONS) {
+		 * 	attrp->ca_uid = hfsmp->hfs_uid;
+		 * 	attrp->ca_gid = hfsmp->hfs_gid;
+		 * }
+		 */
 	}
 
 	if (isDirectory) {
@@ -2865,9 +3799,13 @@ getbsdattr(struct hfsmount *hfsmp, const struct HFSPlusCatalogFile *crp, struct
 			attrp->ca_mode &= ~S_IFMT;
 			attrp->ca_mode |= S_IFDIR;
 		}
-		attrp->ca_nlink = 2 + ((HFSPlusCatalogFolder *)crp)->valence;
-		attrp->ca_entries = ((HFSPlusCatalogFolder *)crp)->valence;
-		attrp->ca_attrblks = ((HFSPlusCatalogFolder *)crp)->attrBlocks;
+		attrp->ca_entries = ((const HFSPlusCatalogFolder *)crp)->valence;
+		attrp->ca_dircount = ((hfsmp->hfs_flags & HFS_FOLDERCOUNT) && (attrp->ca_recflags & kHFSHasFolderCountMask)) ?
+					     ((const HFSPlusCatalogFolder *)crp)->folderCount : 0;
+			
+		/* Keep UF_HIDDEN bit in sync with Finder Info's invisible bit */
+		if (((const HFSPlusCatalogFolder *)crp)->userInfo.frFlags & OSSwapHostToBigConstInt16(kFinderInvisibleMask))
+			attrp->ca_flags |= UF_HIDDEN;
 	} else {
 		/* Keep IMMUTABLE bits in sync with HFS locked flag */
 		if (crp->flags & kHFSFileLockedMask) {
@@ -2879,12 +3817,18 @@ getbsdattr(struct hfsmount *hfsmp, const struct HFSPlusCatalogFile *crp, struct
 			/* The file's supposed to be unlocked: */
 			attrp->ca_flags &= ~(SF_IMMUTABLE | UF_IMMUTABLE);
 		}
+		/* Keep UF_HIDDEN bit in sync with Finder Info's invisible bit */
+		if (crp->userInfo.fdFlags & OSSwapHostToBigConstInt16(kFinderInvisibleMask))
+			attrp->ca_flags |= UF_HIDDEN;
 		/* get total blocks (both forks) */
 		attrp->ca_blocks = crp->dataFork.totalBlocks + crp->resourceFork.totalBlocks;
-		attrp->ca_attrblks = crp->attrBlocks;
+
 		/* On HFS+ the ThreadExists flag must always be set. */
 		if ((hfsmp->hfs_flags & HFS_STANDARD) == 0)
 			attrp->ca_recflags |= kHFSThreadExistsMask;
+
+		/* Pick up the hardlink first link, if any. */
+		attrp->ca_firstlink = (attrp->ca_recflags & kHFSHasLinkChainMask) ? crp->hl_firstLinkID : 0;
 	}
 	
 	attrp->ca_fileid = crp->fileID;
@@ -2901,7 +3845,7 @@ promotekey(struct hfsmount *hfsmp, const HFSCatalogKey *hfskey,
            HFSPlusCatalogKey *keyp, u_long *encoding)
 {
 	hfs_to_unicode_func_t hfs_get_unicode = hfsmp->hfs_get_unicode;
-	UInt32 uniCount;
+	u_int32_t uniCount;
 	int error;
 
 	*encoding = hfsmp->hfs_encoding;
@@ -2963,7 +3907,7 @@ promotefork(struct hfsmount *hfsmp, const struct HFSCatalogFile *filep,
 }
 
 /*
- * promoteattr - promote hfs catalog attributes to hfs plus
+ * promoteattr - promote standard hfs catalog attributes to hfs plus
  *
  */
 static void
@@ -2972,9 +3916,9 @@ promoteattr(struct hfsmount *hfsmp, const CatalogRecord *dataPtr, struct HFSPlus
 	u_long blocksize = HFSTOVCB(hfsmp)->blockSize;
 
 	if (dataPtr->recordType == kHFSFolderRecord) {
-		struct HFSCatalogFolder * folder;
+		const struct HFSCatalogFolder * folder;
 
-		folder = (struct HFSCatalogFolder *) dataPtr;
+		folder = (const struct HFSCatalogFolder *) dataPtr;
 		crp->recordType       = kHFSPlusFolderRecord;
 		crp->flags            = folder->flags;
 		crp->fileID           = folder->folderID;
@@ -2982,11 +3926,12 @@ promoteattr(struct hfsmount *hfsmp, const CatalogRecord *dataPtr, struct HFSPlus
 		crp->contentModDate   = LocalToUTC(folder->modifyDate);
 		crp->backupDate       = LocalToUTC(folder->backupDate);
 		crp->reserved1        = folder->valence;
+		crp->reserved2        = 0;
 		bcopy(&folder->userInfo, &crp->userInfo, 32);
 	} else /* file */ {
-		struct HFSCatalogFile * file;
+		const struct HFSCatalogFile * file;
 
-		file = (struct HFSCatalogFile *) dataPtr;
+		file = (const struct HFSCatalogFile *) dataPtr;
 		crp->recordType       = kHFSPlusFileRecord;
 		crp->flags            = file->flags;
 		crp->fileID           = file->fileID;
@@ -2994,6 +3939,7 @@ promoteattr(struct hfsmount *hfsmp, const CatalogRecord *dataPtr, struct HFSPlus
 		crp->contentModDate   = LocalToUTC(file->modifyDate);
 		crp->backupDate       = LocalToUTC(file->backupDate);
 		crp->reserved1        = 0;
+		crp->reserved2        = 0;
 		bcopy(&file->userInfo, &crp->userInfo, 16);
 		bcopy(&file->finderInfo, &crp->finderInfo, 16);
 		crp->dataFork.totalBlocks = file->dataPhysicalSize / blocksize;
@@ -3003,7 +3949,6 @@ promoteattr(struct hfsmount *hfsmp, const CatalogRecord *dataPtr, struct HFSPlus
 	crp->attributeModDate = crp->contentModDate;
 	crp->accessDate = crp->contentModDate;
 	bzero(&crp->bsdInfo, sizeof(HFSPlusBSDInfo));
-	crp->attrBlocks = 0;
 }
 
 /*
@@ -3108,7 +4053,7 @@ getcnid(const CatalogRecord *crp)
 		cnid = crp->hfsPlusFile.fileID;
 		break;
 	default:
-		printf("hfs: getcnid: unknown recordType (crp @ 0x%x)\n", crp);
+		panic("hfs: getcnid: unknown recordType (crp @ %p)\n", crp);
 		break;
 	}
 
@@ -3134,7 +4079,7 @@ getparentcnid(const CatalogRecord *recp)
 		cnid = recp->hfsPlusThread.parentID;
 		break;
 	default:
-		panic("hfs: getparentcnid: unknown recordType (crp @ 0x%x)\n", recp);
+		panic("hfs: getparentcnid: unknown recordType (crp @ %p)\n", recp);
 		break;
 	}
 
diff --git a/bsd/hfs/hfs_catalog.h b/bsd/hfs/hfs_catalog.h
index 2f91eae90..03c36567b 100644
--- a/bsd/hfs/hfs_catalog.h
+++ b/bsd/hfs/hfs_catalog.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __HFS_CATALOG__
 #define __HFS_CATALOG__
@@ -51,7 +57,7 @@ struct cat_desc {
 	u_int8_t  cd_flags;       /* see below (8 bits) */
 	u_int8_t  cd_encoding;    /* name encoding */
 	int16_t   cd_namelen;     /* length of cnode name */
-	char *    cd_nameptr;     /* pointer to cnode name */
+	const u_int8_t * cd_nameptr; /* pointer to cnode name */
 	cnid_t    cd_parentcnid;  /* parent directory CNID */
 	u_long    cd_hint;        /* catalog file hint */
 	cnid_t    cd_cnid;        /* cnode id (for getattrlist) */
@@ -70,10 +76,13 @@ struct cat_attr {
 	cnid_t		ca_fileid;	/* inode number (for stat) normally == cnid */
 	mode_t		ca_mode;	/* file access mode and type (16 bits) */
 	u_int16_t	ca_recflags;	/* catalog record flags (16 bit integer) */
-	u_int32_t	ca_nlink;	/* file link count */
+	u_int32_t	ca_linkcount;	/* real hard link count */
 	uid_t		ca_uid;		/* file owner */
 	gid_t		ca_gid;		/* file group */
-	dev_t		ca_rdev;	/* device a special file represents */
+	union {
+	    dev_t	cau_rdev;	/* special file device (VBLK or VCHAR only) */
+	    u_int32_t	cau_linkref;	/* hardlink reference number */
+	} ca_union1;
 	time_t		ca_atime;	/* last access time */
 	time_t		ca_atimeondisk;	/* access time value on disk */
 	time_t		ca_mtime;	/* last data modification time */
@@ -82,15 +91,23 @@ struct cat_attr {
 	time_t		ca_btime;	/* last backup time */
 	u_int32_t	ca_flags;	/* status flags (chflags) */
 	union {
-		u_int32_t  cau_blocks;	/* total file blocks used (rsrc + data) */
-		u_int32_t  cau_entries;	/* total directory entries (valence) */
-	} ca_union;
+	    u_int32_t	cau_blocks;	/* total file blocks used (rsrc + data) */
+	    u_int32_t	cau_entries;	/* total directory entries (valence) */
+	} ca_union2;
+	union {
+	    u_int32_t	cau_dircount;	/* count of sub dirs (for posix nlink) */
+	    u_int32_t	cau_firstlink;	/* first hardlink link (files only) */
+	} ca_union3;
 	u_int8_t 	ca_finderinfo[32]; /* Opaque Finder information */
-	u_int32_t	ca_attrblks;    /* cached count of attribute data blocks */
 };
+
 /* Aliases for common fields */
-#define	ca_blocks	ca_union.cau_blocks
-#define	ca_entries	ca_union.cau_entries
+#define	ca_rdev		ca_union1.cau_rdev	
+#define	ca_linkref	ca_union1.cau_linkref
+#define	ca_blocks	ca_union2.cau_blocks
+#define	ca_entries	ca_union2.cau_entries
+#define	ca_dircount	ca_union3.cau_dircount
+#define	ca_firstlink	ca_union3.cau_firstlink
 
 /*
  * Catalog Node Fork (runtime)
@@ -98,7 +115,7 @@ struct cat_attr {
  * NOTE: this is not the same as a struct HFSPlusForkData
  */
 struct cat_fork {
-	u_int64_t      cf_size;        /* fork's logical size in bytes */
+	off_t          cf_size;        /* fork's logical size in bytes */
 	union {
 	    u_int32_t  cfu_clump;      /* fork's clump size in bytes (sys files only) */
 	    u_int64_t  cfu_bytesread;  /* bytes read from this fork */
@@ -120,6 +137,7 @@ struct cat_fork {
 struct directoryhint {
 	TAILQ_ENTRY(directoryhint) dh_link; /* chain */
 	int     dh_index;                   /* index into directory (zero relative) */
+	u_int32_t  dh_threadhint;           /* node hint of a directory's thread record */
 	u_int32_t  dh_time;
 	struct  cat_desc  dh_desc;          /* entry's descriptor */
 };
@@ -153,18 +171,33 @@ struct cat_entry {
 	u_long		ce_rsrcblks;
 };
 
-#define MAXCATENTRIES 8
+/*
+ * Starting in 10.5, hfs_vnop_readdirattr() only makes one
+ * call to cat_getentriesattr(). So we increased MAXCATENTRIES
+ * while keeping the total size of the CE LIST buffer <= 8K
+ * (which works out to be 60 entries per call).  The 8K limit
+ * keeps the memory coming from a kalloc zone instead of
+ * valuable/fragment-able kernel map space.
+ */
+#define MAXCATENTRIES	\
+	(1 + (8192 - sizeof (struct cat_entrylist)) / sizeof (struct cat_entry))
+
 /*
  * Catalog Node Entry List
  *
  * A cat_entrylist is a list of Catalog Node Entries.
  */
 struct cat_entrylist {
-	u_long  maxentries;    /* length of list */
-	u_long  realentries;   /* valid entry count */
-	struct cat_entry  entry[MAXCATENTRIES];   /* array of entries */
+	u_long  maxentries;    /* number of entries requested */
+	u_long  realentries;   /* number of valid entries returned */
+	u_long  skipentries;   /* number of entries skipped (reserved HFS+ files) */
+	struct cat_entry  entry[1];   /* array of entries */
 };
 
+#define CE_LIST_SIZE(entries)	\
+	sizeof (*ce_list) + (((entries) - 1) * sizeof (struct cat_entry))
+
+
 /*
  * Catalog Operations Hint
  *
@@ -172,10 +205,10 @@ struct cat_entrylist {
  * upper 16 bits: count of B-tree delete operations
  *
  */
-#define CAT_DELETE	0x00020000
+#define CAT_DELETE	0x00010000
 #define CAT_CREATE	0x00000002
-#define CAT_RENAME	0x00020002
-#define CAT_EXCHANGE	0x00020002
+#define CAT_RENAME	0x00010002
+#define CAT_EXCHANGE	0x00010002
 
 typedef u_int32_t	catops_t;
 
@@ -238,6 +271,7 @@ extern int cat_lookup (	struct hfsmount *hfsmp,
 
 extern int cat_idlookup (struct hfsmount *hfsmp,
 			cnid_t cnid,
+			int allow_system_files,
 			struct cat_desc *outdescp,
 			struct cat_attr *attrp,
 			struct cat_fork *forkp);
@@ -312,17 +346,62 @@ extern int cat_convertkey(
 			CatalogRecord * recp,
 			struct cat_desc *descp);
 
-extern int resolvelink(
-			struct hfsmount *hfsmp,
-			u_long linkref,
-			struct HFSPlusCatalogFile *recp);
-
 extern int cat_getkeyplusattr(
 			struct hfsmount *hfsmp, 
 			cnid_t cnid, 
 			CatalogKey *key, 
 			struct cat_attr *attrp);
 
+/* Hard link functions. */
+
+extern int cat_check_link_ancestry(
+			struct hfsmount *hfsmp,
+			cnid_t parentid, 
+			cnid_t pointed_at_cnid);
+
+extern int cat_set_childlinkbit(
+			struct hfsmount *hfsmp, 
+			cnid_t cnid);
+
+#define HFS_IGNORABLE_LINK  0x00000001
+
+extern int cat_resolvelink( struct hfsmount *hfsmp,
+                            u_long linkref,
+                            int isdirlink,
+                            struct HFSPlusCatalogFile *recp);
+
+extern int cat_createlink( struct hfsmount *hfsmp,
+                           struct cat_desc *descp,
+                           struct cat_attr *attr,
+                           cnid_t nextlinkid,
+                           cnid_t *linkfileid);
+
+/* Finder Info's file type and creator for directory hard link alias */
+enum {
+	kHFSAliasType    = 0x66647270,  /* 'fdrp' */
+	kHFSAliasCreator = 0x4D414353   /* 'MACS' */
+};
+
+extern int cat_deletelink( struct hfsmount *hfsmp,
+                           struct cat_desc *descp);
+
+extern int cat_updatelink( struct hfsmount *hfsmp,
+                           cnid_t linkfileid,
+                           cnid_t prevlinkid,
+                           cnid_t nextlinkid);
+
+extern int cat_lookuplink( struct hfsmount *hfsmp,
+                           struct cat_desc *descp,
+                           cnid_t *linkfileid,
+                           cnid_t *prevlinkid,
+                           cnid_t *nextlinkid);
+
+extern int cat_lookuplinkbyid( struct hfsmount *hfsmp,
+                               cnid_t linkfileid,
+                               cnid_t *prevlinkid,
+                               cnid_t *nextlinkid);
+
+
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
 #endif /* __HFS_CATALOG__ */
diff --git a/bsd/hfs/hfs_chash.c b/bsd/hfs/hfs_chash.c
index c93b1f1a4..4c6401fcd 100644
--- a/bsd/hfs/hfs_chash.c
+++ b/bsd/hfs/hfs_chash.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -85,7 +91,6 @@ u_long	cnodehash;		/* size of hash table - 1 */
 
 lck_mtx_t  hfs_chash_mutex;
 
-
 /*
  * Initialize cnode hash table.
  */
@@ -93,8 +98,6 @@ __private_extern__
 void
 hfs_chashinit()
 {
-	cnodehashtbl = hashinit(desiredvnodes, M_HFSMNT, &cnodehash);
-
 	chash_lck_grp_attr= lck_grp_attr_alloc_init();
 	chash_lck_grp  = lck_grp_alloc_init("cnode_hash", chash_lck_grp_attr);
 	chash_lck_attr = lck_attr_alloc_init();
@@ -102,6 +105,26 @@ hfs_chashinit()
 	lck_mtx_init(&hfs_chash_mutex, chash_lck_grp, chash_lck_attr);
 }
 
+static void hfs_chash_lock(void) 
+{
+	lck_mtx_lock(&hfs_chash_mutex);
+}
+
+static void hfs_chash_unlock(void) 
+{
+	lck_mtx_unlock(&hfs_chash_mutex);
+}
+
+__private_extern__
+void
+hfs_chashinit_finish()
+{
+	hfs_chash_lock();
+	if (!cnodehashtbl)
+		cnodehashtbl = hashinit(desiredvnodes, M_HFSMNT, &cnodehash);
+	hfs_chash_unlock();
+}
+
 
 /*
  * Use the device, inum pair to find the incore cnode.
@@ -115,7 +138,7 @@ hfs_chash_getvnode(dev_t dev, ino_t inum, int wantrsrc, int skiplock)
 	struct cnode *cp;
 	struct vnode *vp;
 	int error;
-	uint32_t vid;
+	u_int32_t vid;
 
 	/* 
 	 * Go through the hash list
@@ -123,7 +146,7 @@ hfs_chash_getvnode(dev_t dev, ino_t inum, int wantrsrc, int skiplock)
 	 * allocated, wait for it to be finished and then try again.
 	 */
 loop:
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 	for (cp = CNODEHASH(dev, inum)->lh_first; cp; cp = cp->c_hash.le_next) {
 		if ((cp->c_fileid != inum) || (cp->c_dev != dev))
 			continue;
@@ -135,22 +158,13 @@ loop:
 			              "hfs_chash_getvnode", 0);
 			goto loop;
 		}
-		/*
-		 * Skip cnodes that are not in the name space anymore
-		 * note that this check is done outside of the proper
-		 * lock to catch nodes already in this state... this
-		 * state must be rechecked after we acquire the cnode lock
-		 */
-		if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-			continue;
-		}
 		/* Obtain the desired vnode. */
 		vp = wantrsrc ? cp->c_rsrc_vp : cp->c_vp;
 		if (vp == NULLVP)
 			goto exit;
 
 		vid = vnode_vid(vp);
-		lck_mtx_unlock(&hfs_chash_mutex);
+		hfs_chash_unlock();
 
 		if ((error = vnode_getwithvid(vp, vid))) {
 		        /*
@@ -166,10 +180,10 @@ loop:
 
 		/*
 		 * Skip cnodes that are not in the name space anymore
-		 * we need to check again with the cnode lock held
-		 * because we may have blocked acquiring the vnode ref
-		 * or the lock on the cnode which would allow the node
-		 * to be unlinked
+		 * we need to check with the cnode lock held because
+		 * we may have blocked acquiring the vnode ref or the
+		 * lock on the cnode which would allow the node to be
+		 * unlinked
 		 */
 		if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
 			if (!skiplock)
@@ -181,16 +195,13 @@ loop:
 		return (vp);
 	}
 exit:
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 	return (NULL);
 }
 
 
 /*
- * Use the device, fileid pair to find the incore cnode.
- * If no cnode if found one is created
- *
- * If it is in core, but locked, wait for it.
+ * Use the device, fileid pair to snoop an incore cnode.
  */
 __private_extern__
 int
@@ -205,7 +216,7 @@ hfs_chash_snoop(dev_t dev, ino_t inum, int (*callout)(const struct cat_desc *,
 	 * If a cnode is in the process of being cleaned out or being
 	 * allocated, wait for it to be finished and then try again.
 	 */
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 	for (cp = CNODEHASH(dev, inum)->lh_first; cp; cp = cp->c_hash.le_next) {
 		if ((cp->c_fileid != inum) || (cp->c_dev != dev))
 			continue;
@@ -215,7 +226,7 @@ hfs_chash_snoop(dev_t dev, ino_t inum, int (*callout)(const struct cat_desc *,
 		}
 		break;
 	}
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 	return (result);
 }
 
@@ -225,6 +236,9 @@ hfs_chash_snoop(dev_t dev, ino_t inum, int (*callout)(const struct cat_desc *,
  * If no cnode if found one is created
  *
  * If it is in core, but locked, wait for it.
+ *
+ * If the cnode is C_DELETED, then return NULL since that 
+ * inum is no longer valid for lookups (open-unlinked file).
  */
 __private_extern__
 struct cnode *
@@ -233,7 +247,7 @@ hfs_chash_getcnode(dev_t dev, ino_t inum, struct vnode **vpp, int wantrsrc, int
 	struct cnode	*cp;
 	struct cnode	*ncp = NULL;
 	vnode_t		vp;
-	uint32_t	vid;
+	u_int32_t	vid;
 
 	/* 
 	 * Go through the hash list
@@ -241,7 +255,7 @@ hfs_chash_getcnode(dev_t dev, ino_t inum, struct vnode **vpp, int wantrsrc, int
 	 * allocated, wait for it to be finished and then try again.
 	 */
 loop:
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
 loop_with_lock:
 	for (cp = CNODEHASH(dev, inum)->lh_first; cp; cp = cp->c_hash.le_next) {
@@ -257,15 +271,6 @@ loop_with_lock:
 			              "hfs_chash_getcnode", 0);
 			goto loop_with_lock;
 		}
-		/*
-		 * Skip cnodes that are not in the name space anymore
-		 * note that this check is done outside of the proper
-		 * lock to catch nodes already in this state... this
-		 * state must be rechecked after we acquire the cnode lock
-		 */
-		if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-			continue;
-		}
 		vp = wantrsrc ? cp->c_rsrc_vp : cp->c_vp;
 		if (vp == NULL) {
 			/*
@@ -273,11 +278,11 @@ loop_with_lock:
 			 */
 			SET(cp->c_hflag, H_ATTACH);
 
-			lck_mtx_unlock(&hfs_chash_mutex);
+			hfs_chash_unlock();
 		} else {
 			vid = vnode_vid(vp);
 
-			lck_mtx_unlock(&hfs_chash_mutex);
+			hfs_chash_unlock();
 
 			if (vnode_getwithvid(vp, vid))
 		        	goto loop;
@@ -291,32 +296,37 @@ loop_with_lock:
 		        FREE_ZONE(ncp, sizeof(struct cnode), M_HFSNODE);
 			ncp = NULL;
 		}
-		if (!skiplock && hfs_lock(cp, HFS_EXCLUSIVE_LOCK) != 0) {
-			if (vp != NULLVP)
-				vnode_put(vp);
-			lck_mtx_lock(&hfs_chash_mutex);
 
-			if (vp == NULLVP)
-			        CLR(cp->c_hflag, H_ATTACH);
-			goto loop_with_lock;
+		if (!skiplock) {
+			hfs_lock(cp, HFS_FORCE_LOCK);
 		}
+
 		/*
 		 * Skip cnodes that are not in the name space anymore
-		 * we need to check again with the cnode lock held
-		 * because we may have blocked acquiring the vnode ref
-		 * or the lock on the cnode which would allow the node
-		 * to be unlinked
+		 * we need to check with the cnode lock held because
+		 * we may have blocked acquiring the vnode ref or the
+		 * lock on the cnode which would allow the node to be
+		 * unlinked.
+		 *
+		 * Don't return a cnode in this case since the inum
+		 * is no longer valid for lookups.
 		 */
-		if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
+		if ((cp->c_flag & (C_NOEXISTS | C_DELETED)) && !wantrsrc) {
 			if (!skiplock)
 				hfs_unlock(cp);
-			if (vp != NULLVP)
+			if (vp != NULLVP) {
 				vnode_put(vp);
-			lck_mtx_lock(&hfs_chash_mutex);
-
-			if (vp == NULLVP)
-			        CLR(cp->c_hflag, H_ATTACH);
-			goto loop_with_lock;
+			} else {
+				hfs_chash_lock();
+		        	CLR(cp->c_hflag, H_ATTACH);
+				if (ISSET(cp->c_hflag, H_WAITING)) {
+					CLR(cp->c_hflag, H_WAITING);
+					wakeup((caddr_t)cp);
+				}
+				hfs_chash_unlock();
+			}
+			vp = NULL;
+			cp = NULL;
 		}
 		*vpp = vp;
 		return (cp);
@@ -325,11 +335,11 @@ loop_with_lock:
 	/* 
 	 * Allocate a new cnode
 	 */
-	if (skiplock)
+	if (skiplock && !wantrsrc)
 		panic("%s - should never get here when skiplock is set \n", __FUNCTION__);
 
 	if (ncp == NULL) {
-		lck_mtx_unlock(&hfs_chash_mutex);
+		hfs_chash_unlock();
 
 	        MALLOC_ZONE(ncp, struct cnode *, sizeof(struct cnode), M_HFSNODE, M_WAITOK);
 		/*
@@ -345,6 +355,7 @@ loop_with_lock:
 	ncp->c_fileid = inum;
 	ncp->c_dev = dev;
 	TAILQ_INIT(&ncp->c_hintlist); /* make the list empty */
+	TAILQ_INIT(&ncp->c_originlist);
 
 	lck_rw_init(&ncp->c_rwlock, hfs_rwlock_group, hfs_lock_attr);
 	if (!skiplock)
@@ -352,7 +363,7 @@ loop_with_lock:
 
 	/* Insert the new cnode with it's H_ALLOC flag set */
 	LIST_INSERT_HEAD(CNODEHASH(dev, inum), ncp, c_hash);
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 
 	*vpp = NULL;
 	return (ncp);
@@ -363,7 +374,7 @@ __private_extern__
 void
 hfs_chashwakeup(struct cnode *cp, int hflags)
 {
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
 	CLR(cp->c_hflag, hflags);
 
@@ -371,7 +382,7 @@ hfs_chashwakeup(struct cnode *cp, int hflags)
 	        CLR(cp->c_hflag, H_WAITING);
 		wakeup((caddr_t)cp);
 	}
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 }
 
 
@@ -382,14 +393,14 @@ __private_extern__
 void
 hfs_chash_rehash(struct cnode *cp1, struct cnode *cp2)
 {
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
 	LIST_REMOVE(cp1, c_hash);
 	LIST_REMOVE(cp2, c_hash);
 	LIST_INSERT_HEAD(CNODEHASH(cp1->c_dev, cp1->c_fileid), cp1, c_hash);
 	LIST_INSERT_HEAD(CNODEHASH(cp2->c_dev, cp2->c_fileid), cp2, c_hash);
 
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 }
 
 
@@ -400,18 +411,19 @@ __private_extern__
 int
 hfs_chashremove(struct cnode *cp)
 {
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
 	/* Check if a vnode is getting attached */
 	if (ISSET(cp->c_hflag, H_ATTACH)) {
-		lck_mtx_unlock(&hfs_chash_mutex);
+		hfs_chash_unlock();
 		return (EBUSY);
 	}
-	LIST_REMOVE(cp, c_hash);
-	cp->c_hash.le_next = NULL;
-	cp->c_hash.le_prev = NULL;
-	
-	lck_mtx_unlock(&hfs_chash_mutex);
+	if (cp->c_hash.le_next || cp->c_hash.le_prev) {
+	    LIST_REMOVE(cp, c_hash);
+	    cp->c_hash.le_next = NULL;
+	    cp->c_hash.le_prev = NULL;
+	}
+	hfs_chash_unlock();
 	return (0);
 }
 
@@ -422,7 +434,7 @@ __private_extern__
 void
 hfs_chash_abort(struct cnode *cp)
 {
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
 	LIST_REMOVE(cp, c_hash);
 	cp->c_hash.le_next = NULL;
@@ -433,20 +445,77 @@ hfs_chash_abort(struct cnode *cp)
 	        CLR(cp->c_hflag, H_WAITING);
 		wakeup((caddr_t)cp);
 	}
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
 }
 
 
 /*
- * mark a cnode as in transistion
+ * mark a cnode as in transition
  */
 __private_extern__
 void
 hfs_chash_mark_in_transit(struct cnode *cp)
 {
-	lck_mtx_lock(&hfs_chash_mutex);
+	hfs_chash_lock();
 
         SET(cp->c_hflag, H_TRANSIT);
 
-	lck_mtx_unlock(&hfs_chash_mutex);
+	hfs_chash_unlock();
+}
+
+/* Search a cnode in the hash.  This function does not return cnode which 
+ * are getting created, destroyed or in transition.  Note that this function
+ * does not acquire the cnode hash mutex, and expects the caller to acquire it.
+ * On success, returns pointer to the cnode found.  On failure, returns NULL.
+ */
+static 
+struct cnode *
+hfs_chash_search_cnid(dev_t dev, cnid_t cnid) 
+{
+	struct cnode *cp;
+
+	for (cp = CNODEHASH(dev, cnid)->lh_first; cp; cp = cp->c_hash.le_next) {
+		if ((cp->c_fileid == cnid) && (cp->c_dev == dev)) {
+			break;
+		}
+	}
+
+	/* If cnode is being created or reclaimed, return error. */
+	if (cp && ISSET(cp->c_hflag, H_ALLOC | H_TRANSIT | H_ATTACH)) {
+		cp = NULL;
+	}
+
+	return cp;
+}
+
+/* Search a cnode corresponding to given device and ID in the hash.  If the 
+ * found cnode has kHFSHasChildLinkBit cleared, set it.  If the cnode is not 
+ * found, no new cnode is created and error is returned.
+ * 
+ * Return values - 
+ *	-1 : The cnode was not found.
+ * 	 0 : The cnode was found, and the kHFSHasChildLinkBit was already set.
+ *	 1 : The cnode was found, the kHFSHasChildLinkBit was not set, and the 
+ *	     function had to set that bit.
+ */
+__private_extern__ 
+int
+hfs_chash_set_childlinkbit(dev_t dev, cnid_t cnid)
+{
+	int retval = -1;
+	struct cnode *cp;
+
+	hfs_chash_lock();
+	cp = hfs_chash_search_cnid(dev, cnid);
+	if (cp) {
+		if (cp->c_attr.ca_recflags & kHFSHasChildLinkMask) {
+			retval = 0;
+		} else {
+			cp->c_attr.ca_recflags |= kHFSHasChildLinkMask;
+			retval = 1;
+		}
+	}
+	hfs_chash_unlock();
+
+	return retval;
 }
diff --git a/bsd/hfs/hfs_cnode.c b/bsd/hfs/hfs_cnode.c
index e7d86d973..1f434da3d 100644
--- a/bsd/hfs/hfs_cnode.c
+++ b/bsd/hfs/hfs_cnode.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -51,14 +57,8 @@ static int  hfs_filedone(struct vnode *vp, vfs_context_t context);
 
 static void  hfs_reclaim_cnode(struct cnode *);
 
-static int  hfs_valid_cnode(struct hfsmount *, struct vnode *, struct componentname *, cnid_t);
-
 static int hfs_isordered(struct cnode *, struct cnode *);
 
-int hfs_vnop_inactive(struct vnop_inactive_args *);
-
-int hfs_vnop_reclaim(struct vnop_reclaim_args *);
-
 
 /*
  * Last reference to an cnode.  If necessary, write or delete it.
@@ -98,33 +98,38 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 		return (0);
 	}
 
-	if ((v_type == VREG) &&
-	    (ISSET(cp->c_flag, C_DELETED) || VTOF(vp)->ff_blocks)) {
+	if ((v_type == VREG || v_type == VLNK)) {
 		hfs_lock_truncate(cp, TRUE);
 		took_trunc_lock = 1;
 	}
 
+	(void) hfs_lock(cp, HFS_FORCE_LOCK);
+
 	/*
-	 * We do the ubc_setsize before we take the cnode
-	 * lock and before the hfs_truncate (since we'll
-	 * be inside a transaction).
+	 * We should lock cnode before checking the flags in the 
+	 * condition below and should unlock the cnode before calling 
+	 * ubc_setsize() as cluster code can call other HFS vnops which
+	 * will try to acquire the same cnode lock and cause deadlock.
 	 */
 	if ((v_type == VREG || v_type == VLNK) &&
 	    (cp->c_flag & C_DELETED) &&
 	    (VTOF(vp)->ff_blocks != 0)) {
+	    	hfs_unlock(cp); 
 		ubc_setsize(vp, 0);
+		(void) hfs_lock(cp, HFS_FORCE_LOCK);
 	}
 
-	(void) hfs_lock(cp, HFS_FORCE_LOCK);
-
 	if (v_type == VREG && !ISSET(cp->c_flag, C_DELETED) && VTOF(vp)->ff_blocks) {
 		hfs_filedone(vp, ap->a_context);
 	}
 	/* 
-	 * Remove any directory hints
+	 * Remove any directory hints or cached origins
 	 */
-	if (v_type == VDIR)
+	if (v_type == VDIR) {
 		hfs_reldirhints(cp, 0);
+		if (cp->c_flag & C_HARDLINK)
+			hfs_relorigins(cp);
+	}
 
 	if (cp->c_datafork)
 		++forkcount;
@@ -134,22 +139,6 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 	/* If needed, get rid of any fork's data for a deleted file */
 	if ((v_type == VREG || v_type == VLNK) && (cp->c_flag & C_DELETED)) {
 		if (VTOF(vp)->ff_blocks != 0) {
-		    // start the transaction out here so that
-		    // the truncate and the removal of the file
-		    // are all in one transaction.  otherwise
-		    // because this cnode is marked for deletion
-		    // the truncate won't cause the catalog entry
-		    // to get updated which means that we could
-		    // free blocks but still keep a reference to
-		    // them in the catalog entry and then double
-		    // free them later.
-		    //
-//		    if (hfs_start_transaction(hfsmp) != 0) {
-//			error = EINVAL;
-//			goto out;
-//		    }
-//		    started_tr = 1;
-		    
 			/*
 			 * Since we're already inside a transaction,
 			 * tell hfs_truncate to skip the ubc_setsize.
@@ -160,6 +149,37 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 			truncated = 1;
 		}
 		recycle = 1;
+
+		/*
+		 * Check if there's any resource fork blocks that need to
+		 * be reclaimed.  This covers the case where there is a
+		 * resource fork but its not in core.
+		 */
+		if ((cp->c_blocks > 0) && (forkcount == 1) && (vp != cp->c_rsrc_vp)) {
+			struct vnode *rvp = NULLVP;
+
+			error = hfs_vgetrsrc(hfsmp, vp, &rvp, FALSE);
+			if (error)
+				goto out;
+			/*
+			 * Defer the vnode_put and ubc_setsize on rvp until hfs_unlock().
+			 */
+			cp->c_flag |= C_NEED_RVNODE_PUT | C_NEED_RSRC_SETSIZE;
+			error = hfs_truncate(rvp, (off_t)0, IO_NDELAY, 1, ap->a_context);
+			if (error)
+				goto out;
+			vnode_recycle(rvp);  /* all done with this vnode */
+		}
+	}
+
+	// If needed, get rid of any xattrs that this file may have.
+	// Note that this must happen outside of any other transactions
+	// because it starts/ends its own transactions and grabs its
+	// own locks.  This is to prevent a file with a lot of attributes
+	// from creating a transaction that is too large (which panics).
+	//
+	if ((cp->c_attr.ca_recflags & kHFSHasAttributesMask) != 0 && (cp->c_flag & C_DELETED)) {
+		hfs_removeallattr(hfsmp, cp->c_fileid);
 	}
 
 	/*
@@ -171,9 +191,13 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 		 * Mark cnode in transit so that no one can get this 
 		 * cnode from cnode hash.
 		 */
-	        hfs_chash_mark_in_transit(cp);
+	        // hfs_chash_mark_in_transit(cp);
+	        // XXXdbg - remove the cnode from the hash table since it's deleted
+	        //          otherwise someone could go to sleep on the cnode and not
+	        //          be woken up until this vnode gets recycled which could be
+	        //          a very long time...
+		hfs_chashremove(cp);
 
-		cp->c_flag &= ~C_DELETED;
 		cp->c_flag |= C_NOEXISTS;   // XXXdbg
 		cp->c_rdev = 0;
 
@@ -219,14 +243,12 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 
   		/* Update HFS Private Data dir */
 		if (error == 0) {
-			hfsmp->hfs_privdir_attr.ca_entries--;
-			(void)cat_update(hfsmp, &hfsmp->hfs_privdir_desc,
-				&hfsmp->hfs_privdir_attr, NULL, NULL);
-		}
-
-		if (error == 0) {
-			/* Delete any attributes, ignore errors */
-			(void) hfs_removeallattr(hfsmp, cp->c_fileid);
+			hfsmp->hfs_private_attr[FILE_HARDLINKS].ca_entries--;
+			if (vnode_isdir(vp)) {
+				DEC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[FILE_HARDLINKS]);
+			}
+			(void)cat_update(hfsmp, &hfsmp->hfs_private_desc[FILE_HARDLINKS],
+				&hfsmp->hfs_private_attr[FILE_HARDLINKS], NULL, NULL);
 		}
 
 		hfs_systemfile_unlock(hfsmp, lockflags);
@@ -235,16 +257,17 @@ hfs_vnop_inactive(struct vnop_inactive_args *ap)
 			goto out;
 
 #if QUOTA
-		(void)hfs_chkiq(cp, -1, NOCRED, 0);
+		if (hfsmp->hfs_flags & HFS_QUOTAS)
+			(void)hfs_chkiq(cp, -1, NOCRED, 0);
 #endif /* QUOTA */
 
 		cp->c_mode = 0;
-		cp->c_flag |= C_NOEXISTS;
+		cp->c_flag &= ~C_DELETED;
 		cp->c_touch_chgtime = TRUE;
 		cp->c_touch_modtime = TRUE;
 
 		if (error == 0)
- 			hfs_volupdate(hfsmp, VOL_RMFILE, 0);
+			hfs_volupdate(hfsmp, (v_type == VDIR) ? VOL_RMDIR : VOL_RMFILE, 0);
 	}
 
 	if ((cp->c_flag & C_MODIFIED) ||
@@ -264,7 +287,7 @@ out:
 	hfs_unlock(cp);
 
 	if (took_trunc_lock)
-		hfs_unlock_truncate(cp);
+	    hfs_unlock_truncate(cp, TRUE);
 
 	/*
 	 * If we are done with the vnode, reclaim it
@@ -368,9 +391,11 @@ hfs_vnop_reclaim(struct vnop_reclaim_args *ap)
 	/*
 	 * Keep track of an inactive hot file.
 	 */
-	if (!vnode_isdir(vp) && !vnode_issystem(vp))
+	if (!vnode_isdir(vp) &&
+	    !vnode_issystem(vp) &&
+	    !(cp->c_flag & (C_DELETED | C_NOEXISTS)) ) {
   		(void) hfs_addhotfile(vp);
-
+	}
 	vnode_removefsref(vp);
 
 	/*
@@ -432,7 +457,9 @@ hfs_vnop_reclaim(struct vnop_reclaim_args *ap)
 
 extern int (**hfs_vnodeop_p) (void *);
 extern int (**hfs_specop_p)  (void *);
+#if FIFO
 extern int (**hfs_fifoop_p)  (void *);
+#endif
 
 /*
  * hfs_getnewvnode - get new default vnode
@@ -446,7 +473,7 @@ hfs_getnewvnode(
 	struct vnode *dvp,
 	struct componentname *cnp,
 	struct cat_desc *descp,
-	int wantrsrc,
+	int flags,
 	struct cat_attr *attrp,
 	struct cat_fork *forkp,
 	struct vnode **vpp)
@@ -457,11 +484,14 @@ hfs_getnewvnode(
 	struct vnode *tvp = NULLVP;
 	struct cnode *cp = NULL;
 	struct filefork *fp = NULL;
-	int i;
 	int retval;
 	int issystemfile;
+	int wantrsrc;
 	struct vnode_fsparam vfsp;
 	enum vtype vtype;
+#if QUOTA
+	int i;
+#endif /* QUOTA */
 
 	if (attrp->ca_fileid == 0) {
 		*vpp = NULL;
@@ -473,15 +503,35 @@ hfs_getnewvnode(
 		*vpp = NULL;
 		return (ENOTSUP);
 	}
-#endif
+#endif /* !FIFO */
 	vtype = IFTOVT(attrp->ca_mode);
 	issystemfile = (descp->cd_flags & CD_ISMETA) && (vtype == VREG);
+	wantrsrc = flags & GNV_WANTRSRC;
+
+#ifdef HFS_CHECK_LOCK_ORDER
+	/*
+	 * The only case were its permissible to hold the parent cnode
+	 * lock is during a create operation (hfs_makenode) or when
+	 * we don't need the cnode lock (GNV_SKIPLOCK).
+	 */
+	if ((dvp != NULL) &&
+	    (flags & (GNV_CREATE | GNV_SKIPLOCK)) == 0 &&
+	    VTOC(dvp)->c_lockowner == current_thread()) {
+		panic("hfs_getnewvnode: unexpected hold of parent cnode %p", VTOC(dvp));
+	}
+#endif /* HFS_CHECK_LOCK_ORDER */
 
 	/*
 	 * Get a cnode (new or existing)
-	 * skip getting the cnode lock if we are getting resource fork (wantrsrc == 2)
 	 */
-	cp = hfs_chash_getcnode(hfsmp->hfs_raw_dev, attrp->ca_fileid, vpp, wantrsrc, (wantrsrc == 2));
+	cp = hfs_chash_getcnode(hfsmp->hfs_raw_dev, attrp->ca_fileid, vpp, wantrsrc, (flags & GNV_SKIPLOCK));
+
+	/*
+	 * If the id is no longer valid for lookups we'll get back a NULL cp.
+	 */
+	if (cp == NULL) {
+		return (ENOENT);
+	}
 
 	/* Hardlinks may need an updated catalog descriptor */
 	if ((cp->c_flag & C_HARDLINK) && descp->cd_nameptr && descp->cd_namelen > 0) {
@@ -498,7 +548,8 @@ hfs_getnewvnode(
 		lck_rw_init(&cp->c_truncatelock, hfs_rwlock_group, hfs_lock_attr);
 
 		/* Make sure its still valid (ie exists on disk). */
-		if (!hfs_valid_cnode(hfsmp, dvp, (wantrsrc ? NULL : cnp), cp->c_fileid)) {
+		if (!(flags & GNV_CREATE) &&
+		    !hfs_valid_cnode(hfsmp, dvp, (wantrsrc ? NULL : cnp), cp->c_fileid)) {
 			hfs_chash_abort(cp);
 			hfs_reclaim_cnode(cp);
 			*vpp = NULL;
@@ -513,23 +564,48 @@ hfs_getnewvnode(
 		descp->cd_flags &= ~CD_HASBUF;
 
 		/* Tag hardlinks */
-		if (IFTOVT(cp->c_mode) == VREG &&
-		    (descp->cd_cnid != attrp->ca_fileid)) {
+		if ((vtype == VREG || vtype == VDIR) &&
+		    ((descp->cd_cnid != attrp->ca_fileid) ||
+		     (attrp->ca_recflags & kHFSHasLinkChainMask))) {
 			cp->c_flag |= C_HARDLINK;
 		}
+		/*
+		 * Fix-up dir link counts.
+		 *
+		 * Earlier versions of Leopard used ca_linkcount for posix
+		 * nlink support (effectively the sub-directory count + 2).
+		 * That is now accomplished using the ca_dircount field with
+		 * the corresponding kHFSHasFolderCountMask flag.
+		 *
+		 * For directories the ca_linkcount is the true link count,
+		 * tracking the number of actual hardlinks to a directory.
+		 *
+		 * We only do this if the mount has HFS_FOLDERCOUNT set;
+		 * at the moment, we only set that for HFSX volumes.
+		 */
+		if ((hfsmp->hfs_flags & HFS_FOLDERCOUNT) && 
+		    (vtype == VDIR) &&
+		    !(attrp->ca_recflags & kHFSHasFolderCountMask) &&
+		    (cp->c_attr.ca_linkcount > 1)) {
+			if (cp->c_attr.ca_entries == 0)
+				cp->c_attr.ca_dircount = 0;
+			else
+				cp->c_attr.ca_dircount = cp->c_attr.ca_linkcount - 2;
 
-		/* Take one dev reference for each non-directory cnode */
-		if (IFTOVT(cp->c_mode) != VDIR) {
-			cp->c_devvp = hfsmp->hfs_devvp;
-			vnode_ref(cp->c_devvp);
+			cp->c_attr.ca_linkcount = 1;
+			cp->c_attr.ca_recflags |= kHFSHasFolderCountMask;
+			if ( !(hfsmp->hfs_flags & HFS_READ_ONLY) )
+				cp->c_flag |= C_MODIFIED;
 		}
 #if QUOTA
-		for (i = 0; i < MAXQUOTAS; i++)
-			cp->c_dquot[i] = NODQUOT;
+		if (hfsmp->hfs_flags & HFS_QUOTAS) {
+			for (i = 0; i < MAXQUOTAS; i++)
+				cp->c_dquot[i] = NODQUOT;
+		}
 #endif /* QUOTA */
 	}
 
-	if (IFTOVT(cp->c_mode) == VDIR) {
+	if (vtype == VDIR) {
 	        if (cp->c_vp != NULL)
 		        panic("hfs_getnewvnode: orphaned vnode (data)");
 		cvpp = &cp->c_vp;
@@ -597,12 +673,20 @@ hfs_getnewvnode(
 	vfsp.vnfs_mp = mp;
 	vfsp.vnfs_vtype = vtype;
 	vfsp.vnfs_str = "hfs";
-	vfsp.vnfs_dvp = dvp;
+	if ((cp->c_flag & C_HARDLINK) && (vtype == VDIR)) {
+		vfsp.vnfs_dvp = NULL;  /* no parent for me! */
+		vfsp.vnfs_cnp = NULL;  /* no name for me! */
+	} else {
+		vfsp.vnfs_dvp = dvp;
+		vfsp.vnfs_cnp = cnp;
+	}
 	vfsp.vnfs_fsnode = cp;
-	vfsp.vnfs_cnp = cnp;
+#if FIFO
 	if (vtype == VFIFO )
 		vfsp.vnfs_vops = hfs_fifoop_p;
-	else if (vtype == VBLK || vtype == VCHR)
+	else
+#endif
+	if (vtype == VBLK || vtype == VCHR)
 		vfsp.vnfs_vops = hfs_specop_p;
 	else
 		vfsp.vnfs_vops = hfs_vnodeop_p;
@@ -617,10 +701,9 @@ hfs_getnewvnode(
 	else
 		vfsp.vnfs_filesize = 0;
 
-	if (dvp && cnp && (cnp->cn_flags & MAKEENTRY))
-		vfsp.vnfs_flags = 0;
-	else
-		vfsp.vnfs_flags = VNFS_NOCACHE;
+	vfsp.vnfs_flags = VNFS_ADDFSREF;
+	if (dvp == NULLVP || cnp == NULL || !(cnp->cn_flags & MAKEENTRY))
+		vfsp.vnfs_flags |= VNFS_NOCACHE;
 
 	/* Tag system files */
 	vfsp.vnfs_marksystem = issystemfile;
@@ -655,17 +738,18 @@ hfs_getnewvnode(
 		return (retval);
 	}
 	vp = *cvpp;
-	vnode_addfsref(vp);
 	vnode_settag(vp, VT_HFS);
-	if (cp->c_flag & C_HARDLINK)
-		vnode_set_hard_link(vp);
+	if (cp->c_flag & C_HARDLINK) {
+		vnode_setmultipath(vp);
+	}
 	hfs_chashwakeup(cp, H_ALLOC | H_ATTACH);
 
 	/*
 	 * Stop tracking an active hot file.
 	 */
-	if (!vnode_isdir(vp) && !vnode_issystem(vp))
+	if (!(flags & GNV_CREATE) && (vtype != VDIR) && !issystemfile) {
 		(void) hfs_removehotfile(vp);
+	}
 
 	*vpp = vp;
 	return (0);
@@ -686,20 +770,13 @@ hfs_reclaim_cnode(struct cnode *cp)
 	}
 #endif /* QUOTA */
 
-	if (cp->c_devvp) {
-		struct vnode *tmp_vp = cp->c_devvp;
-
-		cp->c_devvp = NULL;
-		vnode_rele(tmp_vp);
-	}
-
 	/* 
 	 * If the descriptor has a name then release it
 	 */
-	if (cp->c_desc.cd_flags & CD_HASBUF) {
-		char *nameptr;
+	if ((cp->c_desc.cd_flags & CD_HASBUF) && (cp->c_desc.cd_nameptr != 0)) {
+		const char *nameptr;
 
-		nameptr = cp->c_desc.cd_nameptr;
+		nameptr = (const char *) cp->c_desc.cd_nameptr;
 		cp->c_desc.cd_nameptr = 0;
 		cp->c_desc.cd_flags &= ~CD_HASBUF;
 		cp->c_desc.cd_namelen = 0;
@@ -713,7 +790,8 @@ hfs_reclaim_cnode(struct cnode *cp)
 }
 
 
-static int
+__private_extern__
+int
 hfs_valid_cnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname *cnp, cnid_t cnid)
 {
 	struct cat_attr attr;
@@ -731,9 +809,9 @@ hfs_valid_cnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname
 
 	if (dvp && cnp) {
 		bzero(&cndesc, sizeof(cndesc));
-		cndesc.cd_nameptr = cnp->cn_nameptr;
+		cndesc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 		cndesc.cd_namelen = cnp->cn_namelen;
-		cndesc.cd_parentcnid = VTOC(dvp)->c_cnid;
+		cndesc.cd_parentcnid = VTOC(dvp)->c_fileid;
 		cndesc.cd_hint = VTOC(dvp)->c_childhint;
 
 		if ((cat_lookup(hfsmp, &cndesc, 0, NULL, &attr, NULL, NULL) == 0) &&
@@ -741,7 +819,7 @@ hfs_valid_cnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname
 			stillvalid = 1;
 		}
 	} else {
-		if (cat_idlookup(hfsmp, cnid, NULL, NULL, NULL) == 0) {
+		if (cat_idlookup(hfsmp, cnid, 0, NULL, NULL, NULL) == 0) {
 			stillvalid = 1;
 		}
 	}
@@ -761,11 +839,29 @@ __private_extern__
 void
 hfs_touchtimes(struct hfsmount *hfsmp, struct cnode* cp)
 {
+	/* don't modify times if volume is read-only */
+	if (hfsmp->hfs_flags & HFS_READ_ONLY) {
+		cp->c_touch_acctime = FALSE;
+		cp->c_touch_chgtime = FALSE;
+		cp->c_touch_modtime = FALSE;
+	}
+	else if (hfsmp->hfs_flags & HFS_STANDARD) {
 	/* HFS Standard doesn't support access times */
-	if (hfsmp->hfs_flags & HFS_STANDARD) {
 		cp->c_touch_acctime = FALSE;
 	}
 
+	/*
+	 * Skip access time updates if:
+	 *	. MNT_NOATIME is set
+	 *	. a file system freeze is in progress
+	 *	. a file system resize is in progress
+	 */
+	if (cp->c_touch_acctime) {
+		if ((vfs_flags(hfsmp->hfs_mp) & MNT_NOATIME) ||
+		    (hfsmp->hfs_freezing_proc != NULL) ||
+		    (hfsmp->hfs_flags & HFS_RESIZE_IN_PROGRESS))
+			cp->c_touch_acctime = FALSE;
+	}
 	if (cp->c_touch_acctime || cp->c_touch_chgtime || cp->c_touch_modtime) {
 		struct timeval tv;
 		int touchvol = 0;
@@ -808,7 +904,7 @@ hfs_touchtimes(struct hfsmount *hfsmp, struct cnode* cp)
 
 		/* Touch the volume modtime if needed */
 		if (touchvol) {
-			HFSTOVCB(hfsmp)->vcbFlags |= 0xFF00;
+			MarkVCBDirty(hfsmp);
 			HFSTOVCB(hfsmp)->vcbLsMod = tv.tv_sec;
 		}
 	}
@@ -823,35 +919,68 @@ hfs_lock(struct cnode *cp, enum hfslocktype locktype)
 {
 	void * thread = current_thread();
 
-	/* System files need to keep track of owner */
-	if ((cp->c_fileid < kHFSFirstUserCatalogNodeID) &&
-	    (cp->c_fileid > kHFSRootFolderID) &&
-	    (locktype != HFS_SHARED_LOCK)) {
-
+	if (cp->c_lockowner == thread) {
 		/*
-		 * The extents and bitmap file locks support
-		 * recursion and are always taken exclusive.
+		 * Only the extents and bitmap file's support lock recursion.
 		 */
-		if (cp->c_fileid == kHFSExtentsFileID ||
-		    cp->c_fileid == kHFSAllocationFileID) {
-			if (cp->c_lockowner == thread) {
-				cp->c_syslockcount++;
-			} else {
-				lck_rw_lock_exclusive(&cp->c_rwlock);
-				cp->c_lockowner = thread;
-				cp->c_syslockcount = 1;
-			}
+		if ((cp->c_fileid == kHFSExtentsFileID) ||
+		    (cp->c_fileid == kHFSAllocationFileID)) {
+			cp->c_syslockcount++;
 		} else {
-			lck_rw_lock_exclusive(&cp->c_rwlock);
-			cp->c_lockowner = thread;
+			panic("hfs_lock: locking against myself!");
 		}
 	} else if (locktype == HFS_SHARED_LOCK) {
 		lck_rw_lock_shared(&cp->c_rwlock);
 		cp->c_lockowner = HFS_SHARED_OWNER;
-	} else {
+
+	} else /* HFS_EXCLUSIVE_LOCK */ {
 		lck_rw_lock_exclusive(&cp->c_rwlock);
 		cp->c_lockowner = thread;
+
+		/*
+		 * Only the extents and bitmap file's support lock recursion.
+		 */
+		if ((cp->c_fileid == kHFSExtentsFileID) ||
+		    (cp->c_fileid == kHFSAllocationFileID)) {
+			cp->c_syslockcount = 1;
+		}
+	}
+
+#ifdef HFS_CHECK_LOCK_ORDER
+	/*
+	 * Regular cnodes (non-system files) cannot be locked
+	 * while holding the journal lock or a system file lock.
+	 */
+	if (!(cp->c_desc.cd_flags & CD_ISMETA) &&
+            ((cp->c_fileid > kHFSFirstUserCatalogNodeID) || (cp->c_fileid == kHFSRootFolderID))) {
+		vnode_t vp = NULLVP;
+
+		/* Find corresponding vnode. */
+		if (cp->c_vp != NULLVP && VTOC(cp->c_vp) == cp) {
+			vp = cp->c_vp;
+		} else if (cp->c_rsrc_vp != NULLVP && VTOC(cp->c_rsrc_vp) == cp) {
+			vp = cp->c_rsrc_vp;
+		}
+		if (vp != NULLVP) {
+			struct hfsmount *hfsmp = VTOHFS(vp);
+
+			if (hfsmp->jnl && (journal_owner(hfsmp->jnl) == thread)) {
+				/* This will eventually be a panic here. */
+				printf("hfs_lock: bad lock order (cnode after journal)\n");
+			}
+			if (hfsmp->hfs_catalog_cp && hfsmp->hfs_catalog_cp->c_lockowner == thread) {
+				panic("hfs_lock: bad lock order (cnode after catalog)");
+			}
+			if (hfsmp->hfs_attribute_cp && hfsmp->hfs_attribute_cp->c_lockowner == thread) {
+				panic("hfs_lock: bad lock order (cnode after attribute)");
+			}
+			if (hfsmp->hfs_extents_cp && hfsmp->hfs_extents_cp->c_lockowner == thread) {
+				panic("hfs_lock: bad lock order (cnode after extents)");
+			}
+		}
 	}
+#endif /* HFS_CHECK_LOCK_ORDER */
+	
 	/*
 	 * Skip cnodes that no longer exist (were deleted).
 	 */
@@ -882,13 +1011,9 @@ hfs_lockpair(struct cnode *cp1, struct cnode *cp2, enum hfslocktype locktype)
 	}
 
 	/*
-	 * Lock in cnode parent-child order (if there is a relationship);
-	 * otherwise lock in cnode address order.
+	 * Lock in cnode address order.
 	 */
-	if ((IFTOVT(cp1->c_mode) == VDIR) && (cp1->c_fileid == cp2->c_parentcnid)) {
-		first = cp1;
-		last = cp2;
-	} else if (cp1 < cp2) {
+	if (cp1 < cp2) {
 		first = cp1;
 		last = cp2;
 	} else {
@@ -918,18 +1043,15 @@ hfs_isordered(struct cnode *cp1, struct cnode *cp2)
 		return (1);
 	if (cp2 == NULL || cp1 == (struct cnode *)0xffffffff)
 		return (0);
-	if (cp1->c_fileid == cp2->c_parentcnid)
-		return (1);  /* cp1 is the parent and should go first */
-	if (cp2->c_fileid == cp1->c_parentcnid)
-		return (0);  /* cp1 is the child and should go last */
-
-	return (cp1 < cp2);  /* fall-back is to use address order */
+	/*
+	 * Locking order is cnode address order.
+	 */
+	return (cp1 < cp2);
 }
 
 /*
  * Acquire 4 cnode locks.
- *   - locked in cnode parent-child order (if there is a relationship)
- *     otherwise lock in cnode address order (lesser address first).
+ *   - locked in cnode address order (lesser address first).
  *   - all or none of the locks are taken
  *   - only one lock taken per cnode (dup cnodes are skipped)
  *   - some of the cnode pointers may be null
@@ -996,34 +1118,35 @@ hfs_unlock(struct cnode *cp)
 {
         vnode_t rvp = NULLVP;
         vnode_t vp = NULLVP;
-	u_int32_t c_flag;
+        u_int32_t c_flag;
+	void *lockowner;
 
-	/* System files need to keep track of owner */
-	if ((cp->c_fileid < kHFSFirstUserCatalogNodeID) &&
-	    (cp->c_fileid > kHFSRootFolderID) &&
-	    (cp->c_datafork != NULL)) {
-		/*
-		 * The extents and bitmap file locks support
-		 * recursion and are always taken exclusive.
-		 */
-		if (cp->c_fileid == kHFSExtentsFileID ||
-		    cp->c_fileid == kHFSAllocationFileID) {
-			if (--cp->c_syslockcount > 0) {
-				return;
-			}
+	/*
+	 * Only the extents and bitmap file's support lock recursion.
+	 */
+	if ((cp->c_fileid == kHFSExtentsFileID) ||
+	    (cp->c_fileid == kHFSAllocationFileID)) {
+		if (--cp->c_syslockcount > 0) {
+			return;
 		}
 	}
 	c_flag = cp->c_flag;
 	cp->c_flag &= ~(C_NEED_DVNODE_PUT | C_NEED_RVNODE_PUT | C_NEED_DATA_SETSIZE | C_NEED_RSRC_SETSIZE);
+
 	if (c_flag & (C_NEED_DVNODE_PUT | C_NEED_DATA_SETSIZE)) {
-		vp = cp->c_vp;
+	        vp = cp->c_vp;
 	}
 	if (c_flag & (C_NEED_RVNODE_PUT | C_NEED_RSRC_SETSIZE)) {
-                rvp = cp->c_rsrc_vp;
+	        rvp = cp->c_rsrc_vp;
 	}
 
-	cp->c_lockowner = NULL;
-	lck_rw_done(&cp->c_rwlock);
+	lockowner = cp->c_lockowner;
+	if (lockowner == current_thread()) {
+	    cp->c_lockowner = NULL;
+	    lck_rw_unlock_exclusive(&cp->c_rwlock);
+	} else {
+	    lck_rw_unlock_shared(&cp->c_rwlock);
+	}
 
 	/* Perform any vnode post processing after cnode lock is dropped. */
 	if (vp) {
@@ -1036,7 +1159,7 @@ hfs_unlock(struct cnode *cp)
 		if (c_flag & C_NEED_RSRC_SETSIZE)
 			ubc_setsize(rvp, 0);
 		if (c_flag & C_NEED_RVNODE_PUT)
-			vnode_put(rvp);
+	        	vnode_put(rvp);
 	}
 }
 
@@ -1108,8 +1231,10 @@ __private_extern__
 void
 hfs_lock_truncate(struct cnode *cp, int exclusive)
 {
+#ifdef HFS_CHECK_LOCK_ORDER
 	if (cp->c_lockowner == current_thread())
-		panic("hfs_lock_truncate: cnode 0x%08x locked!", cp);
+		panic("hfs_lock_truncate: cnode %p locked!", cp);
+#endif /* HFS_CHECK_LOCK_ORDER */
 
 	if (exclusive)
 		lck_rw_lock_exclusive(&cp->c_truncatelock);
@@ -1119,9 +1244,13 @@ hfs_lock_truncate(struct cnode *cp, int exclusive)
 
 __private_extern__
 void
-hfs_unlock_truncate(struct cnode *cp)
+hfs_unlock_truncate(struct cnode *cp, int exclusive)
 {
-	lck_rw_done(&cp->c_truncatelock);
+    if (exclusive) {
+	lck_rw_unlock_exclusive(&cp->c_truncatelock);
+    } else {
+	lck_rw_unlock_shared(&cp->c_truncatelock);
+    }
 }
 
 
diff --git a/bsd/hfs/hfs_cnode.h b/bsd/hfs/hfs_cnode.h
index c819e792e..22c5c0784 100644
--- a/bsd/hfs/hfs_cnode.h
+++ b/bsd/hfs/hfs_cnode.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _HFS_CNODE_H_
 #define _HFS_CNODE_H_
@@ -71,7 +77,22 @@ typedef struct filefork filefork_t;
 #define	fcbBTCBPtr       ff_sysfileinfo
 
 typedef u_int8_t atomicflag_t;
- 
+
+
+/*
+ * Hardlink Origin (for hardlinked directories).
+ */
+struct linkorigin {
+	TAILQ_ENTRY(linkorigin)  lo_link;  /* chain */
+	void *  lo_thread;      /* thread that performed the lookup */
+	cnid_t  lo_cnid;        /* hardlink's cnid */
+	cnid_t  lo_parentcnid;  /* hardlink's parent cnid */
+};
+typedef struct linkorigin linkorigin_t;
+
+#define MAX_CACHED_ORIGINS  10
+
+
 /*
  * The cnode is used to represent each active (or recently active)
  * file or directory in the HFS filesystem.
@@ -87,19 +108,21 @@ struct cnode {
 	u_int32_t		c_hflag;	/* cnode's flags for maintaining hash - protected by global hash lock */
 	struct vnode		*c_vp;		/* vnode for data fork or dir */
 	struct vnode		*c_rsrc_vp;	/* vnode for resource fork */
-	struct vnode		*c_devvp;	/* vnode for block I/O */
 	dev_t			c_dev;		/* cnode's device */
         struct dquot		*c_dquot[MAXQUOTAS]; /* cnode's quota info */
 	struct klist		c_knotes;	/* knotes attached to this vnode */
-	u_long			c_childhint;	/* catalog hint for children */
+	u_int32_t		c_childhint;	 /* catalog hint for children (small dirs only) */
+	u_int32_t		c_dirthreadhint; /* catalog hint for directory's thread rec */
 	struct cat_desc		c_desc;		/* cnode's descriptor */
 	struct cat_attr		c_attr;		/* cnode's attributes */
-	TAILQ_HEAD(hfs_hinthead, directoryhint) c_hintlist;  /* directory hint list */
+	TAILQ_HEAD(hfs_originhead, linkorigin) c_originlist;  /* hardlink origin cache */
+	TAILQ_HEAD(hfs_hinthead, directoryhint) c_hintlist;  /* readdir directory hint list */
   	int16_t			c_dirhinttag;	/* directory hint tag */
 	union {
 	    int16_t     cu_dirhintcnt;          /* directory hint count */
 	    int16_t     cu_syslockcount;        /* system file use only */
 	} c_union;
+	u_int32_t		c_dirchangecnt; /* changes each insert/delete (in-core only) */
  	struct filefork		*c_datafork;	/* cnode's data fork */
 	struct filefork		*c_rsrcfork;	/* cnode's rsrc fork */
 	atomicflag_t	c_touch_acctime;
@@ -116,10 +139,10 @@ typedef struct cnode cnode_t;
 
 #define c_fileid	c_attr.ca_fileid
 #define c_mode		c_attr.ca_mode
-#define c_nlink		c_attr.ca_nlink
+#define c_linkcount	c_attr.ca_linkcount
 #define c_uid		c_attr.ca_uid
 #define c_gid		c_attr.ca_gid
-#define c_rdev		c_attr.ca_rdev
+#define c_rdev		c_attr.ca_union1.cau_rdev
 #define c_atime		c_attr.ca_atime
 #define c_mtime		c_attr.ca_mtime
 #define c_ctime		c_attr.ca_ctime
@@ -127,9 +150,8 @@ typedef struct cnode cnode_t;
 #define c_btime		c_attr.ca_btime
 #define c_flags		c_attr.ca_flags
 #define c_finderinfo	c_attr.ca_finderinfo
-#define c_blocks	c_attr.ca_blocks
-#define c_attrblks	c_attr.ca_attrblks
-#define c_entries	c_attr.ca_entries
+#define c_blocks	c_attr.ca_union2.cau_blocks
+#define c_entries	c_attr.ca_union2.cau_entries
 #define c_zftimeout	c_childhint
 
 #define c_dirhintcnt    c_union.cu_dirhintcnt
@@ -152,17 +174,26 @@ typedef struct cnode cnode_t;
 #define C_MODIFIED         0x00010  /* CNode has been modified */
 #define C_NOEXISTS         0x00020  /* CNode has been deleted, catalog entry is gone */
 #define C_DELETED          0x00040  /* CNode has been marked to be deleted */
-#define C_HARDLINK         0x00080  /* CNode is a hard link */
+#define C_HARDLINK         0x00080  /* CNode is a hard link (file or dir) */
 
 #define C_FORCEUPDATE      0x00100  /* force the catalog entry update */
 #define C_HASXATTRS        0x00200  /* cnode has extended attributes */
+#define C_NEG_ENTRIES      0x00400  /* directory has negative name entries */
+#define C_WARNED_RSRC      0x00800  /* cnode lookup warning has been issued */ 
 
 #define C_NEED_DATA_SETSIZE  0x01000  /* Do a ubc_setsize(0) on c_rsrc_vp after the unlock */
 #define C_NEED_RSRC_SETSIZE  0x02000  /* Do a ubc_setsize(0) on c_vp after the unlock */
-
+#define C_DIR_MODIFICATION   0x04000  /* Directory is being modified, wait for lookups */
 
 #define ZFTIMELIMIT	(5 * 60)
 
+/*
+ * The following is the "invisible" bit from the fdFlags field
+ * in the FndrFileInfo.
+ */
+enum { kFinderInvisibleMask = 1 << 14 };
+
+
 /*
  * Convert between cnode pointers and vnode pointers
  */
@@ -183,6 +214,10 @@ typedef struct cnode cnode_t;
 			 VTOC((vp))->c_rsrcfork :		\
 			 VTOC((vp))->c_datafork)
 
+#define VCTOF(vp, cp)	((vp) == (cp)->c_rsrc_vp ?	\
+			 (cp)->c_rsrcfork :		\
+			 (cp)->c_datafork)
+
 #define FTOV(fp)	((fp) == FTOC(fp)->c_rsrcfork ?		\
 			 FTOC(fp)->c_rsrc_vp :			\
 			 FTOC(fp)->c_vp)
@@ -206,12 +241,25 @@ struct hfsfid {
 };
 
 
+/* Get new default vnode */
+extern int hfs_getnewvnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname *cnp,
+                           struct cat_desc *descp, int flags, struct cat_attr *attrp,
+                           struct cat_fork *forkp, struct vnode **vpp);
+
+
+#define GNV_WANTRSRC   0x01  /* Request the resource fork vnode. */
+#define GNV_SKIPLOCK   0x02  /* Skip taking the cnode lock (when getting resource fork). */
+#define GNV_CREATE     0x04  /* The vnode is for a newly created item. */
+
+
+/* Touch cnode times based on c_touch_xxx flags */
 extern void hfs_touchtimes(struct hfsmount *, struct cnode *);
 
 /*
  * HFS cnode hash functions.
  */
 extern void  hfs_chashinit(void);
+extern void  hfs_chashinit_finish(void);
 extern void  hfs_chashinsert(struct cnode *cp);
 extern int   hfs_chashremove(struct cnode *cp);
 extern void  hfs_chash_abort(struct cnode *cp);
@@ -223,13 +271,9 @@ extern struct vnode * hfs_chash_getvnode(dev_t dev, ino_t inum, int wantrsrc, in
 extern struct cnode * hfs_chash_getcnode(dev_t dev, ino_t inum, struct vnode **vpp, int wantrsrc, int skiplock);
 extern int hfs_chash_snoop(dev_t, ino_t, int (*)(const struct cat_desc *,
                             const struct cat_attr *, void *), void *);
+extern int hfs_valid_cnode(struct hfsmount *hfsmp, struct vnode *dvp, struct componentname *cnp, cnid_t cnid);
 				
-/*
- * HFS directory hint functions.
- */
-extern directoryhint_t * hfs_getdirhint(struct cnode *, int);
-extern void  hfs_reldirhint(struct cnode *, directoryhint_t *);
-extern void  hfs_reldirhints(struct cnode *, int);
+extern int hfs_chash_set_childlinkbit(dev_t dev, cnid_t cnid);
 
 /*
  * HFS cnode lock functions.
@@ -242,8 +286,9 @@ extern void  hfs_reldirhints(struct cnode *, int);
  *  4. system files (as needed)
  *       A. Catalog B-tree file
  *       B. Attributes B-tree file
- *       C. Allocation Bitmap file (always exclusive, supports recursion)
- *       D. Overflow Extents B-tree file (always exclusive, supports recursion)
+ *       C. Startup file (if there is one)
+ *       D. Allocation Bitmap file (always exclusive, supports recursion)
+ *       E. Overflow Extents B-tree file (always exclusive, supports recursion)
  *  5. hfs mount point (always last)
  *
  */
@@ -260,7 +305,7 @@ extern void hfs_unlockpair(struct cnode *, struct cnode *);
 extern void hfs_unlockfour(struct cnode *, struct cnode *, struct cnode *, struct cnode *);
 
 extern void hfs_lock_truncate(struct cnode *, int);
-extern void hfs_unlock_truncate(struct cnode *);
+extern void hfs_unlock_truncate(struct cnode *, int);
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
diff --git a/bsd/hfs/hfs_dbg.h b/bsd/hfs/hfs_dbg.h
index fe1d71c89..ef0423083 100644
--- a/bsd/hfs/hfs_dbg.h
+++ b/bsd/hfs/hfs_dbg.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	hfs_dbg.h
  *
@@ -33,6 +39,7 @@
  */
 
 struct componentname;
+extern void Debugger(const char *message);
 
 /* Define the debugging stage...
 		4 -> Do all, aggresive, call_kdp
diff --git a/bsd/hfs/hfs_encodinghint.c b/bsd/hfs/hfs_encodinghint.c
index 02a1fce32..94b926e9e 100644
--- a/bsd/hfs/hfs_encodinghint.c
+++ b/bsd/hfs/hfs_encodinghint.c
@@ -1,24 +1,31 @@
 /*
  * Copyright (c) 2001-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+#if HFS
 
 #include <sys/param.h>
 #include <hfs/hfs_macos_defs.h>
@@ -924,3 +931,17 @@ hfs_setencodingbias(u_int32_t bias)
 	lck_mtx_unlock(&encodinglst_mutex);
 }
 
+#else /* not HFS - temp workaround until 4277828 is fixed */
+/* stubs for exported routines that aren't present when we build kernel without HFS */
+
+#include <sys/types.h>
+
+u_int32_t hfs_pickencoding(u_int16_t *src, int len);
+
+u_int32_t hfs_pickencoding(__unused u_int16_t *src, __unused int len)
+{
+	return(0);
+}
+
+#endif /* HFS */
+
diff --git a/bsd/hfs/hfs_encodings.c b/bsd/hfs/hfs_encodings.c
index e2b13ca1c..d0e89e8d8 100644
--- a/bsd/hfs/hfs_encodings.c
+++ b/bsd/hfs/hfs_encodings.c
@@ -1,24 +1,31 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+#if HFS
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -26,6 +33,8 @@
 #include <sys/malloc.h>
 #include <sys/queue.h>
 #include <sys/utfconv.h>
+#include <kern/host.h>
+#include <mach/host_priv.h>
 
 #include "hfs.h"
 
@@ -47,21 +56,14 @@ struct	hfs_encoding {
 	SLIST_ENTRY(hfs_encoding)  link;
 	int			refcount;
 	int			kmod_id;
-	UInt32			encoding;
+	u_int32_t	encoding;
 	hfs_to_unicode_func_t	get_unicode_func;
 	unicode_to_hfs_func_t	get_hfsname_func;
 };
 
-/* XXX We should use an "official" interface! */
-extern kern_return_t kmod_destroy(host_priv_t host_priv, kmod_t id);
-extern struct host realhost;
-
 #define MAX_HFS_UNICODE_CHARS	(15*5)
 
-int mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str, UInt32 maxCharLen, UInt32 *usedCharLen);
-
-static int unicode_to_mac_roman(UniChar *uni_str, UInt32 unicodeChars, Str31 hfs_str);
-
+static int unicode_to_mac_roman(UniChar *uni_str, u_int32_t unicodeChars, Str31 hfs_str);
 
 void
 hfs_converterinit(void)
@@ -92,7 +94,7 @@ hfs_converterinit(void)
  *
  */
 int
-hfs_addconverter(int id, UInt32 encoding, hfs_to_unicode_func_t get_unicode, unicode_to_hfs_func_t get_hfsname)
+hfs_addconverter(int id, u_int32_t encoding, hfs_to_unicode_func_t get_unicode, unicode_to_hfs_func_t get_hfsname)
 {
 	struct hfs_encoding *encp;
 	
@@ -125,7 +127,7 @@ hfs_addconverter(int id, UInt32 encoding, hfs_to_unicode_func_t get_unicode, uni
  * The call is initiated from within the kernel during the unmounting of an hfs voulume.
  */
 int
-hfs_remconverter(int id, UInt32 encoding)
+hfs_remconverter(int id, u_int32_t encoding)
 {
 	struct hfs_encoding *encp;
 
@@ -159,7 +161,7 @@ hfs_remconverter(int id, UInt32 encoding)
  * Normally called during the mounting of an hfs voulume.
  */
 int
-hfs_getconverter(UInt32 encoding, hfs_to_unicode_func_t *get_unicode, unicode_to_hfs_func_t *get_hfsname)
+hfs_getconverter(u_int32_t encoding, hfs_to_unicode_func_t *get_unicode, unicode_to_hfs_func_t *get_hfsname)
 {
 	struct hfs_encoding *encp;
 	int found = 0;
@@ -192,7 +194,7 @@ hfs_getconverter(UInt32 encoding, hfs_to_unicode_func_t *get_unicode, unicode_to
  * Normally called during the unmounting of an hfs voulume.
  */
 int
-hfs_relconverter(UInt32 encoding)
+hfs_relconverter(u_int32_t encoding)
 {
 	struct hfs_encoding *encp;
 
@@ -229,7 +231,7 @@ hfs_relconverter(UInt32 encoding)
  * '/' chars are converted to ':'
  */
 int
-hfs_to_utf8(ExtendedVCB *vcb, Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen, unsigned char* dstStr)
+hfs_to_utf8(ExtendedVCB *vcb, const Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen, unsigned char* dstStr)
 {
 	int error;
 	UniChar uniStr[MAX_HFS_UNICODE_CHARS];
@@ -259,7 +261,7 @@ hfs_to_utf8(ExtendedVCB *vcb, Str31 hfs_str, ByteCount maxDstLen, ByteCount *act
  * volume encoding then MacRoman is used as a fallback.
  */
 int
-mac_roman_to_utf8(Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen, unsigned char* dstStr)
+mac_roman_to_utf8(const Str31 hfs_str, ByteCount maxDstLen, ByteCount *actualDstLen, unsigned char* dstStr)
 {
 	int error;
 	UniChar uniStr[MAX_HFS_UNICODE_CHARS];
@@ -342,7 +344,7 @@ utf8_to_mac_roman(ByteCount srcLen, const unsigned char* srcStr, Str31 dstStr)
  */
 
 /* 0x00A0 - 0x00FF = Latin 1 Supplement (30 total) */
-static UInt8 gLatin1Table[] = {
+static u_int8_t gLatin1Table[] = {
   /*		  0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F  */
   /* 0x00A0 */	0xCA, 0xC1, 0xA2, 0xA3, 0xDB, 0xB4,  '?', 0xA4, 0xAC, 0xA9, 0xBB, 0xC7, 0xC2,  '?', 0xA8, 0xF8,
   /* 0x00B0 */	0xA1, 0XB1,  '?',  '?', 0xAB, 0xB5, 0xA6, 0xe1, 0xFC,  '?', 0xBC, 0xC8,  '?',  '?',  '?', 0xC0,
@@ -353,14 +355,14 @@ static UInt8 gLatin1Table[] = {
 };
 
 /* 0x02C0 - 0x02DF = Spacing Modifiers (8 total) */
-static UInt8 gSpaceModsTable[] = {
+static u_int8_t gSpaceModsTable[] = {
   /*		  0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F  */
   /* 0x02C0 */	 '?',  '?',  '?',  '?',  '?',  '?', 0xF6, 0xFF,  '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?',
   /* 0x02D0 */	 '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?', 0xF9, 0xFA, 0xFB, 0xFE, 0xF7, 0xFD,  '?',  '?'
 };
 
 /* 0x2010 - 0x20AF = General Punctuation (17 total) */
-static UInt8 gPunctTable[] = {
+static u_int8_t gPunctTable[] = {
   /*		  0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F  */
   /* 0x2010 */	 '?',  '?',  '?', 0xd0, 0xd1,  '?',  '?',  '?', 0xd4, 0xd5, 0xe2,  '?', 0xd2, 0xd3, 0xe3,  '?',
   /* 0x2020 */	0xa0, 0xe0, 0xa5,  '?',  '?',  '?', 0xc9,  '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?',
@@ -375,7 +377,7 @@ static UInt8 gPunctTable[] = {
 };
 
 /* 0x22xx = Mathematical Operators (11 total) */
-static UInt8 gMathTable[] = {
+static u_int8_t gMathTable[] = {
   /*		  0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F  */
   /* 0x2200 */	 '?',  '?', 0xb6,  '?',  '?',  '?', 0xc6,  '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?', 0xb8,
   /* 0x2210 */	 '?', 0xb7,  '?',  '?',  '?',  '?',  '?',  '?',  '?',  '?', 0xc3,  '?',  '?',  '?', 0xb0,  '?',
@@ -387,7 +389,7 @@ static UInt8 gMathTable[] = {
 };
 
 /* */
-static UInt8 gReverseCombTable[] = {
+static u_int8_t gReverseCombTable[] = {
   /*		  0     1     2     3     4     5     6     7     8     9     A     B     C     D     E     F  */
   /* 0x40 */	0xDA, 0x40, 0xDA, 0xDA, 0xDA, 0x56, 0xDA, 0xDA, 0xDA, 0x6C, 0xDA, 0xDA, 0xDA, 0xDA, 0x82, 0x98,
   /* 0x50 */	0xDA, 0xDA, 0xDA, 0xDA, 0xDA, 0xAE, 0xDA, 0xDA, 0xDA, 0xC4, 0xDA, 0xDA, 0xDA, 0xDA, 0xDA, 0xDA,
@@ -448,18 +450,18 @@ static UInt8 gReverseCombTable[] = {
  *
  * Assumes Unicode input is fully decomposed
  */
-static int unicode_to_mac_roman(UniChar *uni_str, UInt32 unicodeChars, Str31 hfs_str)
+static int unicode_to_mac_roman(UniChar *uni_str, u_int32_t unicodeChars, Str31 hfs_str)
 {
-	UInt8		*p;
+	u_int8_t	*p;
 	const UniChar	*u;
 	UniChar		c;
 	UniChar		mask;
-	UInt16		inputChars;
-	UInt16		pascalChars;
+	u_int16_t	inputChars;
+	u_int16_t	pascalChars;
 	OSErr		result = noErr;
-	UInt8		lsb;
-	UInt8		prevChar;
-	UInt8		mc;
+	u_int8_t	lsb;
+	u_int8_t	prevChar;
+	u_int8_t	mc;
 
 	mask = (UniChar) 0xFF80;
 	p = &hfs_str[1];
@@ -469,7 +471,7 @@ static int unicode_to_mac_roman(UniChar *uni_str, UInt32 unicodeChars, Str31 hfs
 	
 	while (inputChars) {
 		c = *(u++);
-		lsb = (UInt8) c;
+		lsb = (u_int8_t) c;
 
 		/*
 		 * If its not 7-bit ascii, then we need to map it
@@ -623,12 +625,12 @@ static UniChar gHiBitCombUnicode[128] = {
  */
 int
 mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str,
-				UInt32 maxCharLen, UInt32 *unicodeChars)
+				__unused u_int32_t maxCharLen, u_int32_t *unicodeChars)
 {
-	const UInt8  *p;
+	const u_int8_t  *p;
 	UniChar  *u;
-	UInt16  pascalChars;
-	UInt8  c;
+	u_int16_t  pascalChars;
+	u_int8_t  c;
 
 	p = hfs_str;
 	u = uni_str;
@@ -638,7 +640,7 @@ mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str,
 	while (pascalChars--) {
 		c = *(p++);
 
-		if ( (SInt8) c >= 0 ) {		/* check if seven bit ascii */
+		if ( (int8_t) c >= 0 ) {		/* check if seven bit ascii */
 			*(u++) = (UniChar) c;	/* just pad high byte with zero */
 		} else { /* its a hi bit character */
 			UniChar uc;
@@ -660,3 +662,38 @@ mac_roman_to_unicode(const Str31 hfs_str, UniChar *uni_str,
 	return noErr;
 }
 
+#else /* not HFS - temp workaround until 4277828 is fixed */
+/* stubs for exported routines that aren't present when we build kernel without HFS */
+
+#include <sys/types.h>
+#include <sys/errno.h>
+
+int hfs_addconverter(int id, u_int32_t encoding, void * get_unicode, void * get_hfsname);
+int hfs_getconverter(u_int32_t encoding, void *get_unicode, void *get_hfsname);
+int hfs_relconverter(u_int32_t encoding);
+int hfs_remconverter(int id, u_int32_t encoding);
+
+int hfs_addconverter( __unused int id, 
+					  __unused u_int32_t encoding, 
+					  __unused void * get_unicode, 
+					  __unused void * get_hfsname )
+{
+	return(0);
+}
+
+int hfs_getconverter(__unused u_int32_t encoding, __unused void *get_unicode, __unused void *get_hfsname)
+{
+	return(EINVAL);
+}
+
+int hfs_relconverter(__unused u_int32_t encoding)
+{
+	return(EINVAL);
+}
+
+int hfs_remconverter(__unused int id, __unused u_int32_t encoding)
+{
+	return(0);
+}
+
+#endif /* HFS */
diff --git a/bsd/hfs/hfs_encodings.h b/bsd/hfs/hfs_encodings.h
index 6b3e1e3ce..8d9dc4b08 100644
--- a/bsd/hfs/hfs_encodings.h
+++ b/bsd/hfs/hfs_encodings.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2002, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997-2000 Apple Computer, Inc. All Rights Reserved
@@ -49,17 +55,17 @@
  */
 
 typedef int (* hfs_to_unicode_func_t)(const Str31 hfs_str, UniChar *uni_str,
-		UInt32 maxCharLen, UInt32 *usedCharLen);
+		u_int32_t maxCharLen, u_int32_t *usedCharLen);
 
-typedef int (* unicode_to_hfs_func_t)(UniChar *uni_str, UInt32 unicodeChars,
+typedef int (* unicode_to_hfs_func_t)(UniChar *uni_str, u_int32_t unicodeChars,
 		Str31 hfs_str);
 
 
-int hfs_addconverter(int kmod_id, UInt32 encoding,
+int hfs_addconverter(int kmod_id, u_int32_t encoding,
 		hfs_to_unicode_func_t get_unicode,
 		unicode_to_hfs_func_t get_hfsname);
 
-int hfs_remconverter(int kmod_id, UInt32 encoding);
+int hfs_remconverter(int kmod_id, u_int32_t encoding);
 
 #endif /* __APPLE_API_UNSTABLE */
 
diff --git a/bsd/hfs/hfs_endian.c b/bsd/hfs/hfs_endian.c
index b9837f669..692b598a9 100644
--- a/bsd/hfs/hfs_endian.c
+++ b/bsd/hfs/hfs_endian.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -81,9 +87,9 @@ hfs_swap_BTNode (
 )
 {
     BTNodeDescriptor *srcDesc = src->buffer;
-    UInt16 *srcOffs = NULL;
+    u_int16_t *srcOffs = NULL;
 	BTreeControlBlockPtr btcb = (BTreeControlBlockPtr)VTOF(vp)->fcbBTCBPtr;
-    UInt32 i;
+    u_int32_t i;
     int error = 0;
 
 #ifdef ENDIAN_DEBUG
@@ -113,12 +119,12 @@ hfs_swap_BTNode (
     	 */
     	if (btcb->totalNodes != 0) {
 			if (srcDesc->fLink >= btcb->totalNodes) {
-				printf("hfs_swap_BTNode: invalid forward link (0x%08X)\n", srcDesc->fLink);
+				printf("hfs_swap_BTNode: invalid forward link (0x%08x >= 0x%08x)\n", srcDesc->fLink, btcb->totalNodes);
 				error = fsBTInvalidHeaderErr;
 				goto fail;
 			}
 			if (srcDesc->bLink >= btcb->totalNodes) {
-				printf("hfs_swap_BTNode: invalid backward link (0x%08X)\n", srcDesc->bLink);
+				printf("hfs_swap_BTNode: invalid backward link (0x%08x >= 0x%08x)\n", srcDesc->bLink, btcb->totalNodes);
 				error = fsBTInvalidHeaderErr;
 				goto fail;
 			}
@@ -149,7 +155,7 @@ hfs_swap_BTNode (
         /*
          * Swap the node offsets (including the free space one!).
          */
-        srcOffs = (UInt16 *)((char *)src->buffer + (src->blockSize - ((srcDesc->numRecords + 1) * sizeof (UInt16))));
+        srcOffs = (u_int16_t *)((char *)src->buffer + (src->blockSize - ((srcDesc->numRecords + 1) * sizeof (u_int16_t))));
 
         /*
          * Sanity check that the record offsets are within the node itself.
@@ -280,7 +286,7 @@ hfs_swap_BTNode (
         /*
          * Swap the node offsets (including the free space one!).
          */
-        srcOffs = (UInt16 *)((char *)src->buffer + (src->blockSize - ((srcDesc->numRecords + 1) * sizeof (UInt16))));
+        srcOffs = (u_int16_t *)((char *)src->buffer + (src->blockSize - ((srcDesc->numRecords + 1) * sizeof (u_int16_t))));
 
         /*
          * Sanity check that the record offsets are within the node itself.
@@ -332,7 +338,7 @@ fail:
 		 */
 		printf("node=%lld fileID=%u volume=%s device=%s\n", src->blockNum, VTOC(vp)->c_fileid,
 			VTOVCB(vp)->vcbVN, vfs_statfs(vnode_mount(vp))->f_mntfromname);
-		VTOVCB(vp)->vcbFlags |= kHFS_DamagedVolume;
+		hfs_mark_volume_inconsistent(VTOVCB(vp));
 	}
 	
     return (error);
@@ -346,10 +352,16 @@ hfs_swap_HFSPlusBTInternalNode (
 )
 {
     BTNodeDescriptor *srcDesc = src->buffer;
-    UInt16 *srcOffs = (UInt16 *)((char *)src->buffer + (src->blockSize - (srcDesc->numRecords * sizeof (UInt16))));
-	char *nextRecord;	/*  Points to start of record following current one */
-    UInt32 i;
-    UInt32 j;
+    u_int16_t *srcOffs = (u_int16_t *)((char *)src->buffer + (src->blockSize - (srcDesc->numRecords * sizeof (u_int16_t))));
+    char *nextRecord;	/*  Points to start of record following current one */
+    
+    /*
+     * i is an int32 because it needs to be negative to index the offset to free space.
+     * srcDesc->numRecords is a u_int16_t and is unlikely to become 32-bit so this should be ok.
+     */
+
+    int32_t i;
+    u_int32_t j;
 
     if (fileID == kHFSExtentsFileID) {
         HFSPlusExtentKey *srcKey;
@@ -357,7 +369,7 @@ hfs_swap_HFSPlusBTInternalNode (
 		size_t recordSize;	/* Size of the data part of the record, or node number for index nodes */
         
         if (srcDesc->kind == kBTIndexNode)
-        	recordSize = sizeof(UInt32);
+        	recordSize = sizeof(u_int32_t);
         else
         	recordSize = sizeof(HFSPlusExtentDescriptor);
 
@@ -401,7 +413,7 @@ hfs_swap_HFSPlusBTInternalNode (
             
             if (srcDesc->kind == kBTIndexNode) {
             	/* For index nodes, the record data is just a child node number. */
-                *((UInt32 *)srcRec) = SWAP_BE32 (*((UInt32 *)srcRec));
+                *((u_int32_t *)srcRec) = SWAP_BE32 (*((u_int32_t *)srcRec));
             } else {
 				/* Swap the extent data */
 				for (j = 0; j < kHFSPlusExtentDensity; j++) {
@@ -413,7 +425,7 @@ hfs_swap_HFSPlusBTInternalNode (
 
     } else if (fileID == kHFSCatalogFileID) {
         HFSPlusCatalogKey *srcKey;
-        SInt16 *srcPtr;
+        int16_t *srcPtr;
         u_int16_t keyLength;
 
         for (i = 0; i < srcDesc->numRecords; i++) {
@@ -453,8 +465,8 @@ hfs_swap_HFSPlusBTInternalNode (
              * Make sure that we can safely dereference the record's type field or
              * an index node's child node number.
              */
-            srcPtr = (SInt16 *)((char *)srcKey + keyLength + sizeof(srcKey->keyLength));
-            if ((char *)srcPtr + sizeof(UInt32) > nextRecord) {
+            srcPtr = (int16_t *)((char *)srcKey + keyLength + sizeof(srcKey->keyLength));
+            if ((char *)srcPtr + sizeof(u_int32_t) > nextRecord) {
 				printf("hfs_swap_HFSPlusBTInternalNode: catalog key #%d too big\n", srcDesc->numRecords-i-1);
 				return fsBTInvalidNodeErr;
             }
@@ -469,7 +481,7 @@ hfs_swap_HFSPlusBTInternalNode (
             /* Make sure name length is consistent with key length */
             if (keyLength < sizeof(srcKey->parentID) + sizeof(srcKey->nodeName.length) +
                 srcKey->nodeName.length*sizeof(srcKey->nodeName.unicode[0])) {
-				printf("hfs_swap_HFSPlusBTInternalNode: catalog record #%d keyLength=%d expected=%d\n",
+				printf("hfs_swap_HFSPlusBTInternalNode: catalog record #%d keyLength=%d expected=%lu\n",
 					srcDesc->numRecords-i, keyLength, sizeof(srcKey->parentID) + sizeof(srcKey->nodeName.length) +
                     srcKey->nodeName.length*sizeof(srcKey->nodeName.unicode[0]));
 				return fsBTInvalidNodeErr;
@@ -485,7 +497,7 @@ hfs_swap_HFSPlusBTInternalNode (
              * Skip over swapping the various types of catalog record.
              */
             if (srcDesc->kind == kBTIndexNode) {
-                *((UInt32 *)srcPtr) = SWAP_BE32 (*((UInt32 *)srcPtr));
+                *((u_int32_t *)srcPtr) = SWAP_BE32 (*((u_int32_t *)srcPtr));
                 continue;
             }
             
@@ -522,8 +534,8 @@ hfs_swap_HFSPlusBTInternalNode (
     
                 /* Don't swap srcRec->userInfo */
                 /* Don't swap srcRec->finderInfo */
-                /* Don't swap srcRec->reserved */
-    
+                srcRec->folderCount = SWAP_BE32 (srcRec->folderCount);
+   
             } else if (srcPtr[0] == kHFSPlusFileRecord) {
                 HFSPlusCatalogFile *srcRec = (HFSPlusCatalogFile *)srcPtr;
                 if ((char *)srcRec + sizeof(*srcRec) > nextRecord) {
@@ -551,8 +563,12 @@ hfs_swap_HFSPlusBTInternalNode (
                 srcRec->bsdInfo.special.iNodeNum	= SWAP_BE32 (srcRec->bsdInfo.special.iNodeNum);
     
                 srcRec->textEncoding		= SWAP_BE32 (srcRec->textEncoding);
-    
-                /* Don't swap srcRec->reserved1 */
+			
+                /* If kHFSHasLinkChainBit is set, reserved1 is hl_FirstLinkID.  
+		 * In all other context, it is expected to be zero.
+		 */
+                srcRec->reserved1 = SWAP_BE32 (srcRec->reserved1);
+
                 /* Don't swap srcRec->userInfo */
                 /* Don't swap srcRec->finderInfo */
                 /* Don't swap srcRec->reserved2 */
@@ -667,7 +683,7 @@ hfs_swap_HFSPlusBTInternalNode (
              * Skip over swapping the various types of attribute record.
              */
             if (srcDesc->kind == kBTIndexNode) {
-                *((UInt32 *)srcRec) = SWAP_BE32 (*((UInt32 *)srcRec));
+                *((u_int32_t *)srcRec) = SWAP_BE32 (*((u_int32_t *)srcRec));
                 continue;
             }
             
@@ -735,7 +751,7 @@ hfs_swap_HFSPlusBTInternalNode (
     } else if (fileID > kHFSFirstUserCatalogNodeID) {
     	/* The only B-tree with a non-system CNID that we use is the hotfile B-tree */
 		HotFileKey *srcKey;
-		UInt32 *srcRec;
+		u_int32_t *srcRec;
         
 		for (i = 0; i < srcDesc->numRecords; i++) {
         	/* Point to the start of the record we're currently checking. */
@@ -748,8 +764,8 @@ hfs_swap_HFSPlusBTInternalNode (
              */
 			nextRecord = (char *)src->buffer + srcOffs[i-1];
 
-			/* Make sure there is room for the key (HotFileKey) and data (UInt32) */
-			if ((char *)srcKey + sizeof(HotFileKey) + sizeof(UInt32) > nextRecord) {
+			/* Make sure there is room for the key (HotFileKey) and data (u_int32_t) */
+			if ((char *)srcKey + sizeof(HotFileKey) + sizeof(u_int32_t) > nextRecord) {
 				printf("hfs_swap_HFSPlusBTInternalNode: hotfile #%d offset too big (0x%04X)\n", srcDesc->numRecords-i-1, srcOffs[i]);
 				return fsBTInvalidNodeErr;
 			}
@@ -771,7 +787,7 @@ hfs_swap_HFSPlusBTInternalNode (
 			srcKey->temperature = SWAP_BE32 (srcKey->temperature);
 			srcKey->fileID = SWAP_BE32 (srcKey->fileID);
              
-			*((UInt32 *)srcRec) = SWAP_BE32 (*((UInt32 *)srcRec));
+			*((u_int32_t *)srcRec) = SWAP_BE32 (*((u_int32_t *)srcRec));
 		}
     } else {
         panic ("hfs_swap_HFSPlusBTInternalNode: fileID %u is not a system B-tree\n", fileID);
@@ -789,11 +805,15 @@ hfs_swap_HFSBTInternalNode (
 )
 {
     BTNodeDescriptor *srcDesc = src->buffer;
-    UInt16 *srcOffs = (UInt16 *)((char *)src->buffer + (src->blockSize - (srcDesc->numRecords * sizeof (UInt16))));
+    u_int16_t *srcOffs = (u_int16_t *)((char *)src->buffer + (src->blockSize - (srcDesc->numRecords * sizeof (u_int16_t))));
 	char *nextRecord;	/*  Points to start of record following current one */
 
-    UInt32 i;
-    UInt32 j;
+    /*
+     * i is an int32 because it needs to be negative to index the offset to free space.
+     * srcDesc->numRecords is a u_int16_t and is unlikely to become 32-bit so this should be ok.
+     */
+    int32_t i;
+    u_int32_t j;
 
     if (fileID == kHFSExtentsFileID) {
         HFSExtentKey *srcKey;
@@ -801,7 +821,7 @@ hfs_swap_HFSBTInternalNode (
 		size_t recordSize;	/* Size of the data part of the record, or node number for index nodes */
         
         if (srcDesc->kind == kBTIndexNode)
-        	recordSize = sizeof(UInt32);
+        	recordSize = sizeof(u_int32_t);
         else
         	recordSize = sizeof(HFSExtentDescriptor);
 
@@ -843,7 +863,7 @@ hfs_swap_HFSBTInternalNode (
     
             if (srcDesc->kind == kBTIndexNode) {
             	/* For index nodes, the record data is just a child node number. */
-                *((UInt32 *)srcRec) = SWAP_BE32 (*((UInt32 *)srcRec));
+                *((u_int32_t *)srcRec) = SWAP_BE32 (*((u_int32_t *)srcRec));
             } else {
 				/* Swap the extent data */
 				for (j = 0; j < kHFSExtentDensity; j++) {
@@ -855,7 +875,7 @@ hfs_swap_HFSBTInternalNode (
         
     } else if (fileID == kHFSCatalogFileID) {
         HFSCatalogKey *srcKey;
-        SInt16 *srcPtr;
+        int16_t *srcPtr;
         unsigned expectedKeyLength;
 
         for (i = 0; i < srcDesc->numRecords; i++) {
@@ -904,13 +924,13 @@ hfs_swap_HFSBTInternalNode (
             }
 
             /* Point to record data (round up to even byte boundary) */
-            srcPtr = (SInt16 *)((char *)srcKey + ((srcKey->keyLength + 2) & ~1));
+            srcPtr = (int16_t *)((char *)srcKey + ((srcKey->keyLength + 2) & ~1));
             
             /*
              * Make sure that we can safely dereference the record's type field or
              * and index node's child node number.
              */
-            if ((char *)srcPtr + sizeof(UInt32) > nextRecord) {
+            if ((char *)srcPtr + sizeof(u_int32_t) > nextRecord) {
 				printf("hfs_swap_HFSBTInternalNode: catalog key #%d too big\n", srcDesc->numRecords-i-1);
 				return fsBTInvalidNodeErr;
             }
@@ -920,7 +940,7 @@ hfs_swap_HFSBTInternalNode (
              * Skip over swapping the various types of catalog record.
              */
             if (srcDesc->kind == kBTIndexNode) {
-                *((UInt32 *)srcPtr) = SWAP_BE32 (*((UInt32 *)srcPtr));
+                *((u_int32_t *)srcPtr) = SWAP_BE32 (*((u_int32_t *)srcPtr));
                 continue;
             }
     
@@ -977,7 +997,7 @@ hfs_swap_HFSBTInternalNode (
     
                 srcRec->clumpSize			= SWAP_BE16 (srcRec->clumpSize);
                 
-                /* Swap the two sets of extents as an array of six (three each) UInt16 */
+                /* Swap the two sets of extents as an array of six (three each) u_int16_t */
                 for (j = 0; j < kHFSExtentDensity * 2; j++) {
                     srcRec->dataExtents[j].startBlock	= SWAP_BE16 (srcRec->dataExtents[j].startBlock);
                     srcRec->dataExtents[j].blockCount	= SWAP_BE16 (srcRec->dataExtents[j].blockCount);
diff --git a/bsd/hfs/hfs_endian.h b/bsd/hfs/hfs_endian.h
index 784299705..519c40104 100644
--- a/bsd/hfs/hfs_endian.h
+++ b/bsd/hfs/hfs_endian.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __HFS_ENDIAN_H__
 #define __HFS_ENDIAN_H__
diff --git a/bsd/hfs/hfs_format.h b/bsd/hfs/hfs_format.h
index a285f0da8..f0d60d8bb 100644
--- a/bsd/hfs/hfs_format.h
+++ b/bsd/hfs/hfs_format.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __HFS_FORMAT__
 #define __HFS_FORMAT__
@@ -40,7 +46,6 @@ extern "C" {
 #endif
 
 /* some on-disk hfs structures have 68K alignment (misaligned) */
-#pragma options align=mac68k
 
 /* Signatures used to differentiate between HFS and HFS Plus volumes */
 enum {
@@ -59,26 +64,47 @@ enum {
 
 #ifdef __APPLE_API_PRIVATE
 /*
- * Mac OS X has a special directory for linked and unlinked files (HFS Plus only).
- * This directory and its contents are never exported from the filesystem under
- * Mac OS X.
+ * Mac OS X has two special directories on HFS+ volumes for hardlinked files
+ * and hardlinked directories as well as for open-unlinked files.
  *
- * To make this folder name sort last,  it has embedded null prefix.
- * (0xC0, 0x80 in UTF-8)
+ * These directories and their contents are not exported from the filesystem
+ * under Mac OS X.
  */
-#define HFSPLUSMETADATAFOLDER  "\xC0\x80\xC0\x80\xC0\x80\xC0\x80HFS+ Private Data"
+#define HFSPLUSMETADATAFOLDER       "\xE2\x90\x80\xE2\x90\x80\xE2\x90\x80\xE2\x90\x80HFS+ Private Data"
+#define HFSPLUS_DIR_METADATA_FOLDER ".HFS+ Private Directory Data\xd"
 
 /*
- * Files in the HFS Private Data folder have one of the following prefixes
- * followed by a decimal number (no leading zeros).  For indirect nodes this
- * number is a 32 bit random number.  For unlinked (deleted) files that are
- * still open, the number is the file ID for that file.
+ * Files in the "HFS+ Private Data" folder have one of the following prefixes
+ * followed by a decimal number (no leading zeros) for the file ID.
+ *
+ * Note: Earlier version of Mac OS X used a 32 bit random number for the link
+ * ref number instead of the file id.
  *
  * e.g.  iNode7182000 and temp3296
  */
 #define HFS_INODE_PREFIX	"iNode"
 #define HFS_DELETE_PREFIX	"temp"
 
+/*
+ * Files in the ".HFS+ Private Directory Data" folder have the following 
+ * prefix followed by a decimal number (no leading zeros) for the file ID.
+ *
+ * e.g. dir_555
+ */
+#define HFS_DIRINODE_PREFIX	"dir_"
+
+/*
+ * Hardlink inodes save the head of the link chain in
+ * an extended attribute named FIRST_LINK_XATTR_NAME.
+ * The attribute data is the decimal value in ASCII
+ * of the cnid for the first link in the chain.
+ *
+ * This extended attribute is private (i.e. its not
+ * exported in the getxattr/listxattr POSIX APIs).
+ */
+#define FIRST_LINK_XATTR_NAME	"com.apple.system.hfs.firstlink"
+#define FIRST_LINK_XATTR_REC_SIZE (sizeof(HFSPlusAttrData) - 2 + 12)
+
 #endif /* __APPLE_API_PRIVATE */
 
 /*
@@ -90,13 +116,22 @@ enum {
 };
 
 
+/*
+ *	File type and creator for symbolic links
+ */
+enum {
+      kSymLinkFileType  = 0x736C6E6B, /* 'slnk' */
+      kSymLinkCreator   = 0x72686170  /* 'rhap' */
+};
+
+
 #ifndef _HFSUNISTR255_DEFINED_
 #define _HFSUNISTR255_DEFINED_
 /* Unicode strings are used for HFS Plus file and folder names */
 struct HFSUniStr255 {
 	u_int16_t	length;		/* number of unicode characters */
 	u_int16_t	unicode[255];	/* unicode characters */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSUniStr255 HFSUniStr255;
 typedef const HFSUniStr255 *ConstHFSUniStr255Param;
 #endif /* _HFSUNISTR255_DEFINED_ */
@@ -116,7 +151,7 @@ struct HFSExtentKey {
 	u_int8_t 	forkType;	/* 0 = data fork, FF = resource fork */
 	u_int32_t 	fileID;		/* file ID */
 	u_int16_t 	startBlock;	/* first file allocation block number in this extent */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSExtentKey HFSExtentKey;
 
 /* HFS Plus Extent key */
@@ -126,7 +161,7 @@ struct HFSPlusExtentKey {
 	u_int8_t 	pad;			/* make the other fields align on 32-bit boundary */
 	u_int32_t 	fileID;			/* file ID */
 	u_int32_t 	startBlock;		/* first file allocation block number in this extent */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusExtentKey HFSPlusExtentKey;
 
 /* Number of extent descriptors per extent record */
@@ -139,14 +174,14 @@ enum {
 struct HFSExtentDescriptor {
 	u_int16_t 	startBlock;		/* first allocation block */
 	u_int16_t 	blockCount;		/* number of allocation blocks */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSExtentDescriptor HFSExtentDescriptor;
 
 /* HFS Plus extent descriptor */
 struct HFSPlusExtentDescriptor {
 	u_int32_t 	startBlock;		/* first allocation block */
 	u_int32_t 	blockCount;		/* number of allocation blocks */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusExtentDescriptor HFSPlusExtentDescriptor;
 
 /* HFS extent record */
@@ -166,7 +201,7 @@ struct FndrFileInfo {
 	    int16_t	h;
 	} fdLocation;
 	int16_t 	opaque;
-};
+} __attribute__((aligned(2), packed));
 typedef struct FndrFileInfo FndrFileInfo;
 
 struct FndrDirInfo {
@@ -182,12 +217,12 @@ struct FndrDirInfo {
 	    u_int16_t	h;
 	} frLocation;
 	int16_t 	opaque;
-};
+} __attribute__((aligned(2), packed));
 typedef struct FndrDirInfo FndrDirInfo;
 
 struct FndrOpaqueInfo {
 	int8_t opaque[16];
-};
+} __attribute__((aligned(2), packed));
 typedef struct FndrOpaqueInfo FndrOpaqueInfo;
 
 
@@ -197,7 +232,7 @@ struct HFSPlusForkData {
 	u_int32_t 		clumpSize;	/* fork's clump size in bytes */
 	u_int32_t 		totalBlocks;	/* total blocks used by this fork */
 	HFSPlusExtentRecord 	extents;	/* initial set of extents */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusForkData HFSPlusForkData;
 
 
@@ -207,8 +242,8 @@ typedef struct HFSPlusForkData HFSPlusForkData;
  * should preserve, but not change, this information.
  */
 struct HFSPlusBSDInfo {
-	u_int32_t 	ownerID;	/* user or group ID of file/folder owner */
-	u_int32_t 	groupID;	/* additional user of group ID */
+	u_int32_t 	ownerID;	/* user-id of owner or hard link chain previous link */
+	u_int32_t 	groupID;	/* group-id of owner or hard link chain next link */
 	u_int8_t 	adminFlags;	/* super-user changeable flags */
 	u_int8_t 	ownerFlags;	/* owner changeable flags */
 	u_int16_t 	fileMode;	/* file type and permission bits */
@@ -217,9 +252,28 @@ struct HFSPlusBSDInfo {
 	    u_int32_t	linkCount;	/* links that refer to this indirect node */
 	    u_int32_t	rawDevice;	/* special file device (FBLK and FCHR only) */
 	} special;
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusBSDInfo HFSPlusBSDInfo;
 
+/*
+ * Hardlink "links" resolve to an inode
+ * and the actual uid/gid comes from that
+ * inode.
+ *
+ * We repurpose the links's uid/gid fields
+ * for the hardlink link chain. The chain
+ * consists of a doubly linked list of file
+ * ids.
+ */
+ 
+#define hl_firstLinkID     reserved1         /* Valid only if HasLinkChain flag is set (indirect nodes only) */
+
+#define hl_prevLinkID      bsdInfo.ownerID   /* Valid only if HasLinkChain flag is set */
+#define hl_nextLinkID      bsdInfo.groupID   /* Valid only if HasLinkChain flag is set */
+
+#define hl_linkReference   bsdInfo.special.iNodeNum
+#define hl_linkCount       bsdInfo.special.linkCount
+
 
 /* Catalog file data structures */
 
@@ -232,6 +286,8 @@ enum {
 	kHFSAllocationFileID		= 6,	/* File ID of the allocation file (HFS Plus only) */
 	kHFSStartupFileID		= 7,	/* File ID of the startup file (HFS Plus only) */
 	kHFSAttributesFileID		= 8,	/* File ID of the attribute file (HFS Plus only) */
+	kHFSAttributeDataFileID         = 13,	/* Used in Mac OS X runtime for extent based attributes */
+	                                        /* kHFSAttributeDataFileID is never stored on disk. */
 	kHFSRepairCatalogFileID		= 14,	/* Used when rebuilding Catalog B-tree */
 	kHFSBogusExtentFileID		= 15,	/* Used for exchanging extents in extents file */
 	kHFSFirstUserCatalogNodeID	= 16
@@ -243,7 +299,7 @@ struct HFSCatalogKey {
 	u_int8_t 	reserved;		/* reserved (set to zero) */
 	u_int32_t 	parentID;		/* parent folder ID */
 	u_int8_t 	nodeName[kHFSMaxFileNameChars + 1]; /* catalog node name */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSCatalogKey HFSCatalogKey;
 
 /* HFS Plus catalog key */
@@ -251,7 +307,7 @@ struct HFSPlusCatalogKey {
 	u_int16_t 		keyLength;	/* key length (in bytes) */
 	u_int32_t 		parentID;	/* parent folder ID */
 	HFSUniStr255 		nodeName;	/* catalog node name */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusCatalogKey HFSPlusCatalogKey;
 
 /* Catalog record types */
@@ -282,7 +338,16 @@ enum {
 	kHFSHasAttributesMask	= 0x0004,
 
 	kHFSHasSecurityBit	= 0x0003,	/* object has security data (ACLs) */
-	kHFSHasSecurityMask	= 0x0008
+	kHFSHasSecurityMask	= 0x0008,
+
+	kHFSHasFolderCountBit	= 0x0004,	/* only for HFSX, folder maintains a separate sub-folder count */
+	kHFSHasFolderCountMask	= 0x0010,	/* (sum of folder records and directory hard links) */
+
+	kHFSHasLinkChainBit	= 0x0005,	/* has hardlink chain (inode or link) */
+	kHFSHasLinkChainMask	= 0x0020,
+
+	kHFSHasChildLinkBit	= 0x0006,	/* folder has a child that's a dir link */
+	kHFSHasChildLinkMask	= 0x0040
 };
 
 
@@ -298,14 +363,14 @@ struct HFSCatalogFolder {
 	FndrDirInfo 		userInfo;		/* Finder information */
 	FndrOpaqueInfo		finderInfo;		/* additional Finder information */
 	u_int32_t 		reserved[4];		/* reserved - initialized as zero */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSCatalogFolder HFSCatalogFolder;
 
 /* HFS Plus catalog folder record - 88 bytes */
 struct HFSPlusCatalogFolder {
 	int16_t 		recordType;		/* == kHFSPlusFolderRecord */
 	u_int16_t 		flags;			/* file flags */
-	u_int32_t 		valence;		/* folder's valence (limited to 2^16 in Mac OS) */
+	u_int32_t 		valence;		/* folder's item count */
 	u_int32_t 		folderID;		/* folder ID */
 	u_int32_t 		createDate;		/* date and time of creation */
 	u_int32_t 		contentModDate;		/* date and time of last content modification */
@@ -316,8 +381,8 @@ struct HFSPlusCatalogFolder {
 	FndrDirInfo 		userInfo;		/* Finder information */
 	FndrOpaqueInfo	 	finderInfo;		/* additional Finder information */
 	u_int32_t 		textEncoding;		/* hint for name conversions */
-	u_int32_t 		attrBlocks;		/* cached count of attribute data blocks */
-};
+	u_int32_t 		folderCount;		/* number of enclosed folders, active when HasFolderCount is set */
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusCatalogFolder HFSPlusCatalogFolder;
 
 /* HFS catalog file record - 102 bytes */
@@ -341,7 +406,7 @@ struct HFSCatalogFile {
 	HFSExtentRecord		dataExtents;		/* first data fork extent record */
 	HFSExtentRecord		rsrcExtents;		/* first resource fork extent record */
 	u_int32_t		reserved;		/* reserved - initialized as zero */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSCatalogFile HFSCatalogFile;
 
 /* HFS Plus catalog file record - 248 bytes */
@@ -359,12 +424,12 @@ struct HFSPlusCatalogFile {
 	FndrFileInfo 		userInfo;		/* Finder information */
 	FndrOpaqueInfo	 	finderInfo;		/* additional Finder information */
 	u_int32_t 		textEncoding;		/* hint for name conversions */
-	u_int32_t 		attrBlocks;		/* cached count of attribute data blocks */
+	u_int32_t 		reserved2;		/* reserved - initialized as zero */
 
-	/* Note: these start on double long (64 bit) boundry */
+	/* Note: these start on double long (64 bit) boundary */
 	HFSPlusForkData 	dataFork;		/* size and block data for data fork */
 	HFSPlusForkData 	resourceFork;		/* size and block data for resource fork */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusCatalogFile HFSPlusCatalogFile;
 
 /* HFS catalog thread record - 46 bytes */
@@ -373,7 +438,7 @@ struct HFSCatalogThread {
 	int32_t 	reserved[2];		/* reserved - initialized as zero */
 	u_int32_t 	parentID;		/* parent ID for this catalog node */
 	u_int8_t 	nodeName[kHFSMaxFileNameChars + 1]; /* name of this catalog node */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSCatalogThread HFSCatalogThread;
 
 /* HFS Plus catalog thread record -- 264 bytes */
@@ -382,7 +447,7 @@ struct HFSPlusCatalogThread {
 	int16_t 	reserved;		/* reserved - initialized as zero */
 	u_int32_t 	parentID;		/* parent ID for this catalog node */
 	HFSUniStr255 	nodeName;		/* name of this catalog node (variable length) */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusCatalogThread HFSPlusCatalogThread;
 
 #ifdef __APPLE_API_UNSTABLE
@@ -391,23 +456,23 @@ typedef struct HFSPlusCatalogThread HFSPlusCatalogThread;
   	chosen so that they wouldn't conflict with the catalog record types.
 */
 enum {
-	kHFSPlusAttrInlineData	= 0x10,    /* if size <  kAttrOverflowSize */
-	kHFSPlusAttrForkData	= 0x20,    /* if size >= kAttrOverflowSize */
-	kHFSPlusAttrExtents	= 0x30     /* overflow extents for large attributes */
+	kHFSPlusAttrInlineData	= 0x10,   /* attributes whose data fits in a b-tree node */
+	kHFSPlusAttrForkData	= 0x20,   /* extent based attributes (data lives in extents) */
+	kHFSPlusAttrExtents	= 0x30    /* overflow extents for large attributes */
 };
 
 
 /*
   	HFSPlusAttrForkData
   	For larger attributes, whose value is stored in allocation blocks.
-  	If the attribute has more than 8 extents, there will be additonal
+  	If the attribute has more than 8 extents, there will be additional
   	records (of type HFSPlusAttrExtents) for this attribute.
 */
 struct HFSPlusAttrForkData {
 	u_int32_t 	recordType;		/* == kHFSPlusAttrForkData*/
 	u_int32_t 	reserved;
 	HFSPlusForkData theFork;		/* size and first extents of value*/
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusAttrForkData HFSPlusAttrForkData;
 
 /*
@@ -419,7 +484,7 @@ struct HFSPlusAttrExtents {
 	u_int32_t 		recordType;	/* == kHFSPlusAttrExtents*/
 	u_int32_t 		reserved;
 	HFSPlusExtentRecord	extents;	/* additional extents*/
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusAttrExtents HFSPlusAttrExtents;
 
 /*
@@ -433,7 +498,7 @@ struct HFSPlusAttrData {
 	u_int32_t    reserved[2];
 	u_int32_t    attrSize;     /* size of attribute data in bytes */
 	u_int8_t     attrData[2];  /* variable length */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusAttrData HFSPlusAttrData;
 
 
@@ -443,7 +508,7 @@ struct HFSPlusAttrInlineData {
 	u_int32_t 	reserved;
 	u_int32_t 	logicalSize;
 	u_int8_t 	userData[2];
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusAttrInlineData HFSPlusAttrInlineData;
 
 
@@ -463,10 +528,10 @@ struct HFSPlusAttrKey {
 	u_int16_t     keyLength;       /* key length (in bytes) */
 	u_int16_t     pad;	       /* set to zero */
 	u_int32_t     fileID;          /* file associated with attribute */
-	u_int32_t     startBlock;      /* first attribue allocation block number for extents */
+	u_int32_t     startBlock;      /* first allocation block number for extents */
 	u_int16_t     attrNameLen;     /* number of unicode characters */
 	u_int16_t     attrName[kHFSMaxAttrNameLen];   /* attribute name (Unicode) */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusAttrKey HFSPlusAttrKey;
 
 #define kHFSPlusAttrKeyMaximumLength   (sizeof(HFSPlusAttrKey) - sizeof(u_int16_t))
@@ -546,7 +611,7 @@ struct HFSMasterDirectoryBlock {
 	HFSExtentRecord		drXTExtRec;	/* extent record for extents overflow file */
 	u_int32_t 		drCTFlSize;	/* size of catalog file */
 	HFSExtentRecord 	drCTExtRec;	/* extent record for catalog file */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSMasterDirectoryBlock	HFSMasterDirectoryBlock;
 
 
@@ -594,7 +659,7 @@ struct HFSPlusVolumeHeader {
 	HFSPlusForkData  catalogFile;		/* catalog B-tree file */
 	HFSPlusForkData  attributesFile;	/* extended attributes B-tree file */
 	HFSPlusForkData	 startupFile;		/* boot file (secondary loader) */
-};
+} __attribute__((aligned(2), packed));
 typedef struct HFSPlusVolumeHeader HFSPlusVolumeHeader;
 
 
@@ -619,7 +684,7 @@ struct BTNodeDescriptor {
 	u_int8_t 	height;			/* zero for header, map; child is one more than parent*/
 	u_int16_t 	numRecords;		/* number of records in this node*/
 	u_int16_t 	reserved;		/* reserved - initialized as zero */
-};
+} __attribute__((aligned(2), packed));
 typedef struct BTNodeDescriptor BTNodeDescriptor;
 
 /* Constants for BTNodeDescriptor kind */
@@ -647,7 +712,7 @@ struct BTHeaderRec {
 	u_int8_t 	keyCompareType;		/* Key string Comparison Type */
 	u_int32_t 	attributes;		/* persistent attributes about the tree */
 	u_int32_t 	reserved3[16];		/* reserved */
-};
+} __attribute__((aligned(2), packed));
 typedef struct BTHeaderRec BTHeaderRec;
 
 /* Constants for BTHeaderRec attributes */
@@ -671,7 +736,7 @@ struct JournalInfoBlock {
 	u_int64_t       offset;               // byte offset to the journal on the device
 	u_int64_t       size;                 // size in bytes of the journal
 	u_int32_t 	reserved[32];
-};
+} __attribute__((aligned(2), packed));
 typedef struct JournalInfoBlock JournalInfoBlock;
 
 enum {
@@ -681,8 +746,6 @@ enum {
 };
 
 
-#pragma options align=reset
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/bsd/hfs/hfs_fsctl.h b/bsd/hfs/hfs_fsctl.h
index 440cabc55..fffe15d26 100644
--- a/bsd/hfs/hfs_fsctl.h
+++ b/bsd/hfs/hfs_fsctl.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _HFS_FSCTL_H_
@@ -25,6 +31,7 @@
 
 #include <sys/appleapiopts.h>
 
+#include <sys/param.h>
 #include <sys/ioccom.h>
 #include <sys/time.h>
 
@@ -39,6 +46,9 @@ struct hfs_backingstoreinfo {
 };
 
 
+typedef char pathname_t[MAXPATHLEN];
+
+
 /* HFS FS CONTROL COMMANDS */
 
 #define HFSIOC_RESIZE_PROGRESS  _IOR('h', 1, u_int32_t)
@@ -49,6 +59,10 @@ struct hfs_backingstoreinfo {
 
 #define HFSIOC_CHANGE_NEXT_ALLOCATION  _IOWR('h', 3, u_int32_t)
 #define HFS_CHANGE_NEXT_ALLOCATION  IOCBASECMD(HFSIOC_CHANGE_NEXT_ALLOCATION)
+/* Magic value for next allocation to use with fcntl to set next allocation
+ * to zero and never update it again on new block allocation.
+ */
+#define HFS_NO_UPDATE_NEXT_ALLOCATION 	0xffffFFFF
 
 #define HFSIOC_GETCREATETIME  _IOR('h', 4, time_t)
 #define HFS_GETCREATETIME  IOCBASECMD(HFSIOC_GETCREATETIME)
@@ -59,9 +73,31 @@ struct hfs_backingstoreinfo {
 #define HFSIOC_CLRBACKINGSTOREINFO  _IO('h', 8)
 #define HFS_CLRBACKINGSTOREINFO  IOCBASECMD(HFSIOC_CLRBACKINGSTOREINFO)
 
+#define HFSIOC_BULKACCESS _IOW('h', 9, struct access_t)
+#define HFS_BULKACCESS_FSCTL IOCBASECMD(HFSIOC_BULKACCESS)
+
 #define HFSIOC_SETACLSTATE  _IOW('h', 10, int32_t)
 #define HFS_SETACLSTATE  IOCBASECMD(HFSIOC_SETACLSTATE)
 
+#define HFSIOC_PREV_LINK  _IOWR('h', 11, u_int32_t)
+#define HFS_PREV_LINK  IOCBASECMD(HFSIOC_PREV_LINK)
+
+#define HFSIOC_NEXT_LINK  _IOWR('h', 12, u_int32_t)
+#define HFS_NEXT_LINK  IOCBASECMD(HFSIOC_NEXT_LINK)
+
+#define HFSIOC_GETPATH  _IOWR('h', 13, pathname_t)
+#define HFS_GETPATH  IOCBASECMD(HFSIOC_GETPATH)
+
+/* Enable/disable extent-based extended attributes */
+#define HFSIOC_SET_XATTREXTENTS_STATE  _IOW('h', 14, u_int32_t)
+#define HFS_SET_XATTREXTENTS_STATE  IOCBASECMD(HFSIOC_SET_XATTREXTENTS_STATE)
+
+#define HFSIOC_EXT_BULKACCESS _IOW('h', 15, struct ext_access_t)
+#define HFS_EXT_BULKACCESS_FSCTL IOCBASECMD(HFSIOC_EXT_BULKACCESS)
+
+#define HFSIOC_MARK_BOOT_CORRUPT _IO('h', 16)
+#define HFS_MARK_BOOT_CORRUPT IOCBASECMD(HFSIOC_MARK_BOOT_CORRUPT)
+
 #endif /* __APPLE_API_UNSTABLE */
 
 
diff --git a/bsd/hfs/hfs_hotfiles.c b/bsd/hfs/hfs_hotfiles.c
index e4d57acb7..4b23e1b56 100644
--- a/bsd/hfs/hfs_hotfiles.c
+++ b/bsd/hfs/hfs_hotfiles.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>
@@ -26,6 +32,7 @@
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/ubc.h>
+#include <sys/ubc_internal.h>
 #include <sys/vnode.h>
 #include <sys/vnode_internal.h>
 #include <sys/kauth.h>
@@ -43,6 +50,11 @@
 #define HFC_VERBOSE 0
 
 
+/*
+ * Minimum post Tiger base time.
+ * Thu Mar 31 17:00:00 2005
+ */
+#define HFC_MIN_BASE_TIME   0x424c8f00L
 
 /*
  * Hot File List (runtime).
@@ -117,7 +129,7 @@ static void  hf_printtree (hotfile_entry_t *);
 static int  hotfiles_collect (struct hfsmount *);
 static int  hotfiles_age (struct hfsmount *);
 static int  hotfiles_adopt (struct hfsmount *);
-static int  hotfiles_evict (struct hfsmount *, struct proc *);
+static int  hotfiles_evict (struct hfsmount *, vfs_context_t);
 static int  hotfiles_refine (struct hfsmount *);
 static int  hotextents(struct hfsmount *, HFSPlusExtentDescriptor *);
 static int  hfs_addhotfile_internal(struct vnode *);
@@ -126,7 +138,7 @@ static int  hfs_addhotfile_internal(struct vnode *);
 /*
  * Hot File Cluster B-tree (on disk) functions.
  */
-static int  hfc_btree_create (struct hfsmount *, int, int);
+static int  hfc_btree_create (struct hfsmount *, unsigned int, unsigned int);
 static int  hfc_btree_open (struct hfsmount *, struct vnode **);
 static int  hfc_btree_close (struct hfsmount *, struct vnode *);
 static int  hfc_comparekeys (HotFileKey *, HotFileKey *);
@@ -134,9 +146,6 @@ static int  hfc_comparekeys (HotFileKey *, HotFileKey *);
 
 char hfc_tag[] = "CLUSTERED HOT FILES B-TREE     ";
 
-extern int  UBCINFOEXISTS(struct vnode * vp);
-extern int  hfs_vnop_write(struct vnop_write_args *ap);
-
 
 /*
  *========================================================================
@@ -183,7 +192,7 @@ hfs_recording_start(struct hfsmount *hfsmp)
 		FREE(tmp, M_TEMP);
 	}
 
-	microuptime(&tv);
+	microtime(&tv);  /* Times are base on GMT time. */
 
 	/*
 	 * On first startup check for suspended recording.
@@ -200,6 +209,10 @@ hfs_recording_start(struct hfsmount *hfsmp)
 			hfsmp->hfc_maxfiles = SWAP_BE32 (hotfileinfo.maxfilecnt);
 			hfsmp->hfc_timeout = SWAP_BE32 (hotfileinfo.timeleft) + tv.tv_sec ;
 			hfsmp->hfc_timebase = SWAP_BE32 (hotfileinfo.timebase);
+			/* Fix up any bogus timebase values. */
+			if (hfsmp->hfc_timebase < HFC_MIN_BASE_TIME) {
+				hfsmp->hfc_timebase = hfsmp->hfc_timeout - HFC_DEFAULT_DURATION;
+			}
 #if HFC_VERBOSE
 			printf("Resume recording hot files on %s (%d secs left)\n",
 				hfsmp->vcbVN, SWAP_BE32 (hotfileinfo.timeleft));
@@ -292,6 +305,7 @@ hfs_recording_stop(struct hfsmount *hfsmp)
 
 	hotfiles_collect(hfsmp);
 
+
 	/*
 	 * Convert hot file data into a simple file id list....
 	 *
@@ -320,7 +334,7 @@ hfs_recording_stop(struct hfsmount *hfsmp)
 
 	/* Open the B-tree file for writing... */
 	if (hfsmp->hfc_filevp)
-		panic("hfs_recording_stop: hfc_filevp exists (vp = 0x%08x)", hfsmp->hfc_filevp);
+		panic("hfs_recording_stop: hfc_filevp exists (vp = %p)", hfsmp->hfc_filevp);
 
 	error = hfc_btree_open(hfsmp, &hfsmp->hfc_filevp);
 	if (error) {
@@ -343,10 +357,17 @@ hfs_recording_stop(struct hfsmount *hfsmp)
 	size = sizeof(hotfilelist_t);
 	size += sizeof(hotfileinfo_t) * (hotdata->activefiles - 1);
 	MALLOC(listp, hotfilelist_t *, size, M_TEMP, M_WAITOK);
+	if (listp == NULL) {
+		error = ENOMEM;
+		(void) hfc_btree_close(hfsmp, hfsmp->hfc_filevp);
+		hfsmp->hfc_filevp = NULL;
+		goto out;
+	}
+
 	bzero(listp, size);
 
 	hf_getsortedlist(hotdata, listp);	/* NOTE: destroys hot file tree! */
-	microuptime(&tv);
+	microtime(&tv);
 	listp->hfl_duration = tv.tv_sec - hfsmp->hfc_timebase;
 	hfsmp->hfc_recdata = listp;
 
@@ -443,10 +464,10 @@ hfs_recording_suspend(struct hfsmount *hfsmp)
 	}
 	if (hfs_lock(VTOC(hfsmp->hfc_filevp), HFS_EXCLUSIVE_LOCK) != 0) {
 		error = EPERM;
-		goto out;
+		goto end_transaction;
 	}
 
-	microuptime(&tv);
+	microtime(&tv);
 	hotfileinfo.magic       = SWAP_BE32 (HFC_MAGIC);
 	hotfileinfo.version     = SWAP_BE32 (HFC_VERSION);
 	hotfileinfo.duration    = SWAP_BE32 (HFC_DEFAULT_DURATION);
@@ -455,11 +476,14 @@ hfs_recording_suspend(struct hfsmount *hfsmp)
 	hotfileinfo.threshold   = SWAP_BE32 (hotdata->threshold);
 	hotfileinfo.maxfileblks = SWAP_BE32 (hotdata->maxblocks);
 	hotfileinfo.maxfilecnt  = SWAP_BE32 (HFC_DEFAULT_FILE_COUNT);
-	strcpy(hotfileinfo.tag, hfc_tag);
+	strlcpy((char *)hotfileinfo.tag, hfc_tag, sizeof hotfileinfo.tag);
 	(void) BTSetUserData(VTOF(hfsmp->hfc_filevp), &hotfileinfo, sizeof(hotfileinfo));
 
 	hfs_unlock(VTOC(hfsmp->hfc_filevp));
+
+end_transaction:
 	hfs_end_transaction(hfsmp);
+
 out:
 	if (hfsmp->hfc_filevp) {
 		(void) hfc_btree_close(hfsmp, hfsmp->hfc_filevp);
@@ -471,7 +495,7 @@ out:
 	}
 	hfsmp->hfc_stage = HFC_DISABLED;
 	wakeup((caddr_t)&hfsmp->hfc_stage);
-exit:
+
 	lck_mtx_unlock(&hfsmp->hfc_mutex);
 	return (error);
 }
@@ -509,6 +533,16 @@ hfs_recording_init(struct hfsmount *hfsmp)
 		return (EPERM);
 	}
 
+	/*
+	 * Tracking of hot files requires up-to-date access times.
+	 * So if access time updates are disabled, then we disable
+	 * hot files, too.
+	 */
+	if (vfs_flags(HFSTOVFS(hfsmp)) & MNT_NOATIME) {
+		hfsmp->hfc_stage = HFC_DISABLED;
+		return EPERM;
+	}
+	
 	/*
 	 * If the Hot File btree exists then metadata zone is ready.
 	 */
@@ -529,7 +563,7 @@ hfs_recording_init(struct hfsmount *hfsmp)
 	 * Open the Hot File B-tree file for writing.
 	 */
 	if (hfsmp->hfc_filevp)
-		panic("hfs_recording_init: hfc_filevp exists (vp = 0x%08x)", hfsmp->hfc_filevp);
+		panic("hfs_recording_init: hfc_filevp exists (vp = %p)", hfsmp->hfc_filevp);
 	error = hfc_btree_open(hfsmp, &hfsmp->hfc_filevp);
 	if (error) {
 #if HFC_VERBOSE
@@ -567,7 +601,7 @@ hfs_recording_init(struct hfsmount *hfsmp)
 	} 
 	if (hfs_lock(VTOC(hfsmp->hfc_filevp), HFS_EXCLUSIVE_LOCK) != 0) {
 		error = EPERM;
-		goto out1;
+		goto out0;
 	}
 	filefork = VTOF(hfsmp->hfc_filevp);
 
@@ -641,6 +675,7 @@ hfs_recording_init(struct hfsmount *hfsmp)
 	(void) BTFlushPath(filefork);
 	hfs_unlock(VTOC(hfsmp->hfc_filevp));
 
+out0:
 	hfs_end_transaction(hfsmp);
 #if HFC_VERBOSE
 	printf("%d files identified out of %d\n", inserted, filecount);
@@ -665,7 +700,7 @@ out2:
  */
 __private_extern__
 int
-hfs_hotfilesync(struct hfsmount *hfsmp, struct proc *p)
+hfs_hotfilesync(struct hfsmount *hfsmp, vfs_context_t ctx)
 {
 	if (hfsmp->hfc_stage) {
 		struct timeval tv;
@@ -678,13 +713,13 @@ hfs_hotfilesync(struct hfsmount *hfsmp, struct proc *p)
 			break;
 	
 		case HFC_RECORDING:
-			microuptime(&tv);
+			microtime(&tv);
 			if (tv.tv_sec > hfsmp->hfc_timeout)
 				(void) hfs_recording_stop(hfsmp);
 			break;
 	
 		case HFC_EVICTION:
-			(void) hotfiles_evict(hfsmp, p);
+			(void) hotfiles_evict(hfsmp, ctx);
 			break;
 	
 		case HFC_ADOPTION:
@@ -933,7 +968,7 @@ hotfiles_refine(struct hfsmount *hfsmp)
 	} 
 	if (hfs_lock(VTOC(hfsmp->hfc_filevp), HFS_EXCLUSIVE_LOCK) != 0) {
 		error = EPERM;
-		goto out;
+		goto out1;
 	}
 	filefork = VTOF(hfsmp->hfc_filevp);
 
@@ -999,6 +1034,7 @@ hotfiles_refine(struct hfsmount *hfsmp)
 	(void) BTFlushPath(filefork);
 	hfs_unlock(VTOC(hfsmp->hfc_filevp));
 
+out1:
 	hfs_end_transaction(hfsmp);
 out:
 	FREE(iterator, M_TEMP);	
@@ -1108,26 +1144,19 @@ hotfiles_adopt(struct hfsmount *hfsmp)
 			vnode_put(vp);
 			break;  /* adopt this entry the next time around */
 		}
-		/* Start a new transaction. */
-		if (hfs_start_transaction(hfsmp) != 0) {
-		    error = EINVAL;
-		    hfs_unlock(VTOC(vp));
-		    vnode_put(vp);
-		    break;
-		}
-		startedtrans = 1;
-
 		if (VTOC(vp)->c_desc.cd_nameptr)
-			data = *(u_int32_t *)(VTOC(vp)->c_desc.cd_nameptr);
+			data = *(const u_int32_t *)(VTOC(vp)->c_desc.cd_nameptr);
 		else
 			data = 0x3f3f3f3f;
 
 		error = hfs_relocate(vp, hfsmp->hfs_hotfile_start, kauth_cred_get(), current_proc());
 		hfs_unlock(VTOC(vp));
 		vnode_put(vp);
-		if (error)
-			break;
-		
+		if (error) {
+			/* Move on to next item. */
+			listp->hfl_next++;
+			continue;
+		}
 		/* Keep hot file free space current. */
 		hfsmp->hfs_hotfile_freeblks -= fileblocks;
 		listp->hfl_totalblocks -= fileblocks;
@@ -1138,6 +1167,13 @@ hotfiles_adopt(struct hfsmount *hfsmp)
 		key->fileID      = listp->hfl_hotfile[i].hf_fileid;
 		key->forkType    = 0;
 
+		/* Start a new transaction before calling BTree code. */
+		if (hfs_start_transaction(hfsmp) != 0) {
+		    error = EINVAL;
+		    break;
+		}
+		startedtrans = 1;
+
 		error = BTInsertRecord(filefork, iterator, &record, record.itemSize);
 		if (error) {
 			printf("hotfiles_adopt: BTInsertRecord failed %d (fileid %d)\n", error, key->fileID);
@@ -1215,7 +1251,7 @@ hotfiles_adopt(struct hfsmount *hfsmp)
  * Requires that the hfc_mutex be held.
  */
 static int
-hotfiles_evict(struct hfsmount *hfsmp, struct proc *p)
+hotfiles_evict(struct hfsmount *hfsmp, vfs_context_t ctx)
 {
 	BTreeIterator * iterator;
 	struct vnode *vp;
@@ -1286,18 +1322,6 @@ hotfiles_evict(struct hfsmount *hfsmp, struct proc *p)
 		 * Aquire the vnode for this file.
 		 */
 		error = hfs_vget(hfsmp, key->fileID, &vp, 0);
-
-		/* Start a new transaction. */
-		if (hfs_start_transaction(hfsmp) != 0) {
-		    if (error == 0) {
-			hfs_unlock(VTOC(vp));
-			vnode_put(vp);
-		    }
-		    error = EINVAL;
-		    break;
-		}
-		startedtrans = 1;
-
 		if (error) {
 			if (error == ENOENT) {
 				goto delete;  /* stale entry, go to next */
@@ -1335,7 +1359,7 @@ hotfiles_evict(struct hfsmount *hfsmp, struct proc *p)
 		/*
 		 * Relocate file out of hot area.
 		 */
-		error = hfs_relocate(vp, HFSTOVCB(hfsmp)->nextAllocation, proc_ucred(p), p);
+		error = hfs_relocate(vp, HFSTOVCB(hfsmp)->nextAllocation, vfs_context_ucred(ctx), vfs_context_proc(ctx));
 		if (error) {
 			printf("hotfiles_evict: err %d relocating file %d\n", error, key->fileID);
 			hfs_unlock(VTOC(vp));
@@ -1361,6 +1385,13 @@ hotfiles_evict(struct hfsmount *hfsmp, struct proc *p)
 		blksmoved += fileblocks;
 		filesmoved++;
 delete:
+		/* Start a new transaction before calling BTree code. */
+		if (hfs_start_transaction(hfsmp) != 0) {
+		    error = EINVAL;
+		    break;
+		}
+		startedtrans = 1;
+
 		error = BTDeleteRecord(filefork, iterator);
 		if (error) {
 			error = MacToVFSError(error);
@@ -1610,7 +1641,7 @@ hotextents(struct hfsmount *hfsmp, HFSPlusExtentDescriptor * extents)
 static int
 hfc_btree_open(struct hfsmount *hfsmp, struct vnode **vpp)
 {
-	struct proc *p;
+	proc_t p;
 	struct vnode *vp;
 	struct cat_desc  cdesc;
 	struct cat_attr  cattr;
@@ -1625,7 +1656,7 @@ hfc_btree_open(struct hfsmount *hfsmp, struct vnode **vpp)
 
 	bzero(&cdesc, sizeof(cdesc));
 	cdesc.cd_parentcnid = kRootDirID;
-	cdesc.cd_nameptr = filename;
+	cdesc.cd_nameptr = (const u_int8_t *)filename;
 	cdesc.cd_namelen = strlen(filename);
 
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
@@ -1674,10 +1705,7 @@ again:
 	vnode_put(vp);
 
 	if (!vnode_issystem(vp))
-		panic("hfc_btree_open: not a system file (vp = 0x%08x)", vp);
-
-	if (UBCINFOEXISTS(vp))
-		panic("hfc_btree_open: has UBCInfo (vp = 0x%08x)", vp);
+		panic("hfc_btree_open: not a system file (vp = %p)", vp);
 
 	return (error);
 }
@@ -1690,7 +1718,7 @@ again:
 static int
 hfc_btree_close(struct hfsmount *hfsmp, struct vnode *vp)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
 	int  error = 0;
 
 
@@ -1718,30 +1746,27 @@ hfc_btree_close(struct hfsmount *hfsmp, struct vnode *vp)
  *
  */
 static int
-hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
+hfc_btree_create(struct hfsmount *hfsmp, unsigned int nodesize, unsigned int entries)
 {
 	struct vnode *dvp = NULL;
 	struct vnode *vp = NULL;
 	struct cnode *cp = NULL;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct vnode_attr va;
 	struct componentname cname;
 	static char filename[] = HFC_FILENAME;
 	int  error;
 
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
-
 	if (hfsmp->hfc_filevp)
-		panic("hfc_btree_create: hfc_filevp exists (vp = 0x%08x)", hfsmp->hfc_filevp);
+		panic("hfc_btree_create: hfc_filevp exists (vp = %p)", hfsmp->hfc_filevp);
 
-	error = VFS_ROOT(HFSTOVFS(hfsmp), &dvp, &context);
+	error = VFS_ROOT(HFSTOVFS(hfsmp), &dvp, ctx);
 	if (error) {
 		return (error);
 	}
 	cname.cn_nameiop = CREATE;
 	cname.cn_flags = ISLASTCN;
-	cname.cn_context = &context;
+	cname.cn_context = ctx;
 	cname.cn_pnbuf = filename;
 	cname.cn_pnlen = sizeof(filename);
 	cname.cn_nameptr = filename;
@@ -1756,7 +1781,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 	VATTR_SET(&va, va_gid, 0);
 
 	/* call ourselves directly, ignore the higher-level VFS file creation code */
-	error = VNOP_CREATE(dvp, &vp, &cname, &va, &context);
+	error = VNOP_CREATE(dvp, &vp, &cname, &va, ctx);
 	if (error) {
 		printf("HFS: error %d creating HFBT on %s\n", error, HFSTOVCB(hfsmp)->vcbVN);
 		goto out;
@@ -1771,14 +1796,14 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 	cp = VTOC(vp);
 
 	/* Don't use non-regular files or files with links. */
-	if (!vnode_isreg(vp) || cp->c_nlink != 1) {
+	if (!vnode_isreg(vp) || cp->c_linkcount != 1) {
 		error = EFTYPE;
 		goto out;
 	}
 
 	printf("HFS: created HFBT on %s\n", HFSTOVCB(hfsmp)->vcbVN);
 
-	if (VTOF(vp)->ff_size < (u_int64_t)nodesize) {
+	if (VTOF(vp)->ff_size < nodesize) {
 		caddr_t  buffer;
 		u_int16_t *index;
 		u_int16_t  offset;
@@ -1800,7 +1825,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 			goto out;
 		}	
 		bzero(buffer, nodesize);
-		index = (int16_t *)buffer;
+		index = (u_int16_t *)buffer;
 	
 		entirespernode = (nodesize - sizeof(BTNodeDescriptor) - 2) /
 				 (sizeof(HotFileKey) + 6);
@@ -1816,7 +1841,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 		index[(nodesize / 2) - 1] = SWAP_BE16 (offset);
 	
 		/* FILL IN THE HEADER RECORD:  */
-		bthp = (BTHeaderRec *)((UInt8 *)buffer + offset);
+		bthp = (BTHeaderRec *)((u_int8_t *)buffer + offset);
 		bthp->nodeSize     = SWAP_BE16 (nodesize);
 		bthp->totalNodes   = SWAP_BE32 (filesize / nodesize);
 		bthp->freeNodes    = SWAP_BE32 (nodecnt - 1);
@@ -1828,7 +1853,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 		index[(nodesize / 2) - 2] = SWAP_BE16 (offset);
 	
 		/* FILL IN THE USER RECORD:  */
-		hotfileinfo = (HotFilesInfo *)((UInt8 *)buffer + offset);
+		hotfileinfo = (HotFilesInfo *)((u_int8_t *)buffer + offset);
 		hotfileinfo->magic       = SWAP_BE32 (HFC_MAGIC);
 		hotfileinfo->version     = SWAP_BE32 (HFC_VERSION);
 		hotfileinfo->duration    = SWAP_BE32 (HFC_DEFAULT_DURATION);
@@ -1837,7 +1862,8 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 		hotfileinfo->threshold   = SWAP_BE32 (HFC_MINIMUM_TEMPERATURE);
 		hotfileinfo->maxfileblks = SWAP_BE32 (HFC_MAXIMUM_FILESIZE / HFSTOVCB(hfsmp)->blockSize);
 		hotfileinfo->maxfilecnt  = SWAP_BE32 (HFC_DEFAULT_FILE_COUNT);
-		strcpy(hotfileinfo->tag, hfc_tag);
+		strlcpy((char *)hotfileinfo->tag, hfc_tag,
+			sizeof hotfileinfo->tag);
 		offset += kBTreeHeaderUserBytes;
 		index[(nodesize / 2) - 3] = SWAP_BE16 (offset);
 	
@@ -1848,7 +1874,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 		index[(nodesize / 2) - 4] = SWAP_BE16 (offset);
 
 		vnode_setnoflush(vp);
-		error = hfs_truncate(vp, (off_t)filesize, IO_NDELAY, 0, &context);
+		error = hfs_truncate(vp, (off_t)filesize, IO_NDELAY, 0, ctx);
 		if (error) {
 			printf("HFS: error %d growing HFBT on %s\n", error, HFSTOVCB(hfsmp)->vcbVN);
 			goto out;
@@ -1867,7 +1893,7 @@ hfc_btree_create(struct hfsmount *hfsmp, int nodesize, int entries)
 			args.a_vp = vp;
 			args.a_uio = auio;
 			args.a_ioflag = 0;
-			args.a_context = &context;
+			args.a_context = ctx;
 
 			hfs_unlock(cp);
 			cp = NULL;
diff --git a/bsd/hfs/hfs_hotfiles.h b/bsd/hfs/hfs_hotfiles.h
index a9db6d619..5c1ac29bb 100644
--- a/bsd/hfs/hfs_hotfiles.h
+++ b/bsd/hfs/hfs_hotfiles.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __HFS_HOTFILES__
 #define __HFS_HOTFILES__
@@ -36,7 +42,7 @@
  */
 #define HFC_DEFAULT_FILE_COUNT	 1000
 #define HFC_DEFAULT_DURATION   	 (3600 * 60)
-#define HFC_CUMULATIVE_CYCLES	 4
+#define HFC_CUMULATIVE_CYCLES	 3
 #define HFC_MAXIMUM_FILE_COUNT	 5000
 #define HFC_MAXIMUM_FILESIZE	 (10 * 1024 * 1024)
 #define HFC_MINIMUM_TEMPERATURE  24
@@ -84,9 +90,9 @@ typedef struct HotFileKey HotFileKey;
 struct HotFilesInfo {
 	u_int32_t	magic;
 	u_int32_t	version;
-	u_int32_t	duration;    /* duration of sample period */
-	u_int32_t	timebase;   /* recording period start time */
-	u_int32_t	timeleft;    /* recording period stop time */
+	u_int32_t	duration;    /* duration of sample period (secs) */
+	u_int32_t	timebase;    /* start of recording period (GMT time in secs) */
+	u_int32_t	timeleft;    /* time remaining in recording period (secs) */
 	u_int32_t	threshold;
 	u_int32_t	maxfileblks;
 	u_int32_t	maxfilecnt;
@@ -105,7 +111,7 @@ struct vnode;
 /*
  * Hot File interface functions.
  */
-int  hfs_hotfilesync (struct hfsmount *, struct proc *);
+int  hfs_hotfilesync (struct hfsmount *, vfs_context_t ctx);
 
 int  hfs_recording_init(struct hfsmount *);
 int  hfs_recording_suspend (struct hfsmount *);
@@ -113,9 +119,6 @@ int  hfs_recording_suspend (struct hfsmount *);
 int  hfs_addhotfile (struct vnode *);
 int  hfs_removehotfile (struct vnode *);
 
-int  hfs_relocate(struct  vnode *, u_int32_t, kauth_cred_t, struct  proc *);
-
-
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
 #endif /* __HFS_HOTFILES__ */
diff --git a/bsd/hfs/hfs_link.c b/bsd/hfs/hfs_link.c
index 8ab33cf97..65f5e9ee8 100644
--- a/bsd/hfs/hfs_link.c
+++ b/bsd/hfs/hfs_link.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -38,59 +44,86 @@
 
 static int cur_link_id = 0;
 
+/*
+ * Private directories where hardlink inodes reside.
+ */
+const char *hfs_private_names[] = {
+	HFSPLUSMETADATAFOLDER,      /* FILE HARDLINKS */
+	HFSPLUS_DIR_METADATA_FOLDER /* DIRECTORY HARDLINKS */
+};
+
+
+/*
+ * Hardlink inodes save the head of their link chain in a
+ * private extended attribute.  The following calls are
+ * used to access this attribute.
+ */
+static int  setfirstlink(struct hfsmount * hfsmp, cnid_t fileid, cnid_t firstlink);
+static int  getfirstlink(struct hfsmount * hfsmp, cnid_t fileid, cnid_t *firstlink);
 
 /*
- * Create a new indirect link
+ * Create a new catalog link record
+ *
+ * An indirect link is a reference to an inode (the real
+ * file or directory record).
+ *
+ * All the indirect links for a given inode are chained
+ * together in a doubly linked list.
  *
- * An indirect link is a reference to a data node.  The only useable
- * fields in the link are the link number, parentID, name and text
- * encoding.  All other catalog fields are ignored.
+ * Pre-Leopard file hard links do not have kHFSHasLinkChainBit 
+ * set and do not have first/prev/next link IDs i.e. the values 
+ * are zero.  If a new link is being added to an existing 
+ * pre-Leopard file hard link chain, do not set kHFSHasLinkChainBit.
  */
 static int
-createindirectlink(struct hfsmount *hfsmp, u_int32_t linknum,
-			u_int32_t linkparid, char *linkName, cnid_t *linkcnid)
+createindirectlink(struct hfsmount *hfsmp, u_int32_t linknum, struct cat_desc *descp,
+                   cnid_t nextcnid, cnid_t *linkcnid, int is_inode_linkchain_set)
 {
 	struct FndrFileInfo *fip;
-	struct cat_desc desc;
 	struct cat_attr attr;
-	int result;
 
-	/* Setup the descriptor */
-	bzero(&desc, sizeof(desc));
-	desc.cd_nameptr = linkName;
-	desc.cd_namelen = strlen(linkName);
-	desc.cd_parentcnid = linkparid;
+	if (linknum == 0) {
+		printf("createindirectlink: linknum is zero!\n");
+		return (EINVAL);
+	}
 
 	/* Setup the default attributes */
 	bzero(&attr, sizeof(attr));
 	
-	/* links are matched to data nodes by link ID and to volumes by create date */
-	attr.ca_rdev = linknum;  /* note: cat backend overloads ca_rdev to be the linknum when nlink = 0 */
-	attr.ca_itime = HFSTOVCB(hfsmp)->vcbCrDate;
-	attr.ca_mode = S_IFREG;
-
+	/* Links are matched to inodes by link ID and to volumes by create date */
+	attr.ca_linkref = linknum;
+	attr.ca_itime = hfsmp->hfs_itime;
+	attr.ca_mode = S_IFREG | S_IRUSR | S_IRGRP | S_IROTH;
+	attr.ca_recflags = kHFSHasLinkChainMask | kHFSThreadExistsMask;
+	attr.ca_flags = UF_IMMUTABLE;
 	fip = (struct FndrFileInfo *)&attr.ca_finderinfo;
-	fip->fdType    = SWAP_BE32 (kHardLinkFileType);	/* 'hlnk' */
-	fip->fdCreator = SWAP_BE32 (kHFSPlusCreator);	/* 'hfs+' */
-	fip->fdFlags   = SWAP_BE16 (kHasBeenInited);
 
+	if (descp->cd_flags & CD_ISDIR) {
+		fip->fdType    = SWAP_BE32 (kHFSAliasType);
+		fip->fdCreator = SWAP_BE32 (kHFSAliasCreator);
+		fip->fdFlags   = SWAP_BE16 (kIsAlias);
+	} else /* file */ {
+		fip->fdType    = SWAP_BE32 (kHardLinkFileType);
+		fip->fdCreator = SWAP_BE32 (kHFSPlusCreator);
+		fip->fdFlags   = SWAP_BE16 (kHasBeenInited);
+		/* If the file inode does not have kHFSHasLinkChainBit set 
+		 * and the next link chain ID is zero, assume that this 
+		 * is pre-Leopard file inode.  Therefore clear the bit.
+		 */
+		if ((is_inode_linkchain_set == 0) && (nextcnid == 0)) {
+			attr.ca_recflags &= ~kHFSHasLinkChainMask;
+		}
+	}
 	/* Create the indirect link directly in the catalog */
-	result = cat_create(hfsmp, &desc, &attr, NULL);
-
-	if (result == 0 && linkcnid != NULL)
-		*linkcnid = attr.ca_fileid;
-
-	return (result);
+	return cat_createlink(hfsmp, descp, &attr, nextcnid, linkcnid);
 }
 
 
 /*
- * 2 locks are needed (dvp and vp)
- * also need catalog lock
+ * Make a link to the cnode cp in the directory dp
+ * using the name in cnp.
  *
- * caller's responsibility:
- *		componentname cleanup
- *		unlocking dvp and vp
+ * The cnodes cp and dcp must be locked.
  */
 static int
 hfs_makelink(struct hfsmount *hfsmp, struct cnode *cp, struct cnode *dcp,
@@ -101,23 +134,27 @@ hfs_makelink(struct hfsmount *hfsmp, struct cnode *cp, struct cnode *dcp,
 	u_int32_t indnodeno = 0;
 	char inodename[32]; 
 	struct cat_desc to_desc;
+	struct cat_desc link_desc;
 	int newlink = 0;
 	int lockflags;
-	int retval;
+	int retval = 0;
 	cat_cookie_t cookie;
 	cnid_t orig_cnid;
+	cnid_t linkcnid;
+	cnid_t orig_firstlink;
+	enum privdirtype type;
+
+	type = S_ISDIR(cp->c_mode) ? DIR_HARDLINKS : FILE_HARDLINKS;
 
 	if (cur_link_id == 0) {
-	    cur_link_id = ((random() & 0x3fffffff) + 100);
-	    // printf("hfs: initializing cur link id to: 0x%.8x\n", cur_link_id);
+		cur_link_id = ((random() & 0x3fffffff) + 100);
 	}
 	
-	/* We don't allow link nodes in our Private Meta Data folder! */
-	if (dcp->c_fileid == hfsmp->hfs_privdir_desc.cd_cnid)
+	/* We don't allow link nodes in our private system directories. */
+	if (dcp->c_fileid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+	    dcp->c_fileid == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
 		return (EPERM);
-
-	if (hfs_freeblks(hfsmp, 0) == 0)
-		return (ENOSPC);
+	}
 
 	bzero(&cookie, sizeof(cat_cookie_t));
 	/* Reserve some space in the Catalog file. */
@@ -125,147 +162,208 @@ hfs_makelink(struct hfsmount *hfsmp, struct cnode *cp, struct cnode *dcp,
 		return (retval);
 	}
 
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
+	lockflags = SFL_CATALOG | SFL_ATTRIBUTE;
+	/* Directory hard links allocate space for a symlink. */
+	if (type == DIR_HARDLINKS) {
+		lockflags |= SFL_BITMAP;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, lockflags, HFS_EXCLUSIVE_LOCK);
 
-	// save off a copy of the current cnid so we can put 
-	// it back if we get errors down below
+	/* Save the current cnid value so we restore it if an error occurs. */
 	orig_cnid = cp->c_desc.cd_cnid;
 
 	/*
-	 * If this is a new hardlink then we need to create the data
-	 * node (inode) and replace the original file with a link node.
+	 * If this is a new hardlink then we need to create the inode
+	 * and replace the original file/dir object with a link node.
 	 */
-	if (cp->c_nlink == 2 && (cp->c_flag & C_HARDLINK) == 0) {
+	if ((cp->c_linkcount == 2) && !(cp->c_flag & C_HARDLINK)) {
 		newlink = 1;
 		bzero(&to_desc, sizeof(to_desc));
-		to_desc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
+		to_desc.cd_parentcnid = hfsmp->hfs_private_desc[type].cd_cnid;
 		to_desc.cd_cnid = cp->c_fileid;
+		to_desc.cd_flags = (type == DIR_HARDLINKS) ? CD_ISDIR : 0;
 
 		do {
-			/* get a unique indirect node number */
-			if (retval == 0) {
-			    indnodeno = cp->c_fileid;
+			if (type == DIR_HARDLINKS) {
+				/* Directory hardlinks always use the cnid. */
+				indnodeno = cp->c_fileid;
+				MAKE_DIRINODE_NAME(inodename, sizeof(inodename),
+							indnodeno);
 			} else {
-			    indnodeno = cur_link_id++;
+				/* Get a unique indirect node number */
+				if (retval == 0) {
+					indnodeno = cp->c_fileid;
+				} else {
+					indnodeno = cur_link_id++;
+				}
+				MAKE_INODE_NAME(inodename, sizeof(inodename),
+						indnodeno);
 			}
-
-			MAKE_INODE_NAME(inodename, indnodeno);
-
-			/* move source file to data node directory */
-			to_desc.cd_nameptr = inodename;
+			/* Move original file/dir to data node directory */
+			to_desc.cd_nameptr = (const u_int8_t *)inodename;
 			to_desc.cd_namelen = strlen(inodename);
 		
-			retval = cat_rename(hfsmp, &cp->c_desc, &hfsmp->hfs_privdir_desc,
+			retval = cat_rename(hfsmp, &cp->c_desc, &hfsmp->hfs_private_desc[type],
 					&to_desc, NULL);
 
 			if (retval != 0 && retval != EEXIST) {
 			    printf("hfs_makelink: cat_rename to %s failed (%d). fileid %d\n",
 				inodename, retval, cp->c_fileid);
 			}
-
-		} while (retval == EEXIST);
+		} while ((retval == EEXIST) && (type == FILE_HARDLINKS));
 		if (retval)
 			goto out;
 
-		/* Replace source file with link node */
-		retval = createindirectlink(hfsmp, indnodeno, cp->c_parentcnid,
-				cp->c_desc.cd_nameptr, &cp->c_desc.cd_cnid);
+		/*
+		 * Replace original file/dir with a link record.
+		 */
+		
+		bzero(&link_desc, sizeof(link_desc));
+		link_desc.cd_nameptr = cp->c_desc.cd_nameptr;
+		link_desc.cd_namelen = cp->c_desc.cd_namelen;
+		link_desc.cd_parentcnid = cp->c_parentcnid;
+		link_desc.cd_flags = S_ISDIR(cp->c_mode) ? CD_ISDIR : 0;
+
+		retval = createindirectlink(hfsmp, indnodeno, &link_desc, 0, &linkcnid, true);
 		if (retval) {
-		    /* put it source file back */
-		    int err;
+			int err;
 
-		    // Put this back to what it was before.
-		    cp->c_desc.cd_cnid = orig_cnid;
+			/* Restore the cnode's cnid. */
+			cp->c_desc.cd_cnid = orig_cnid;
 
-		    err = cat_rename(hfsmp, &to_desc, &dcp->c_desc, &cp->c_desc, NULL);
-		    if (err)
-			panic("hfs_makelink: error %d from cat_rename backout 1", err);
-		    goto out;
+			/* Put the original file back. */
+			err = cat_rename(hfsmp, &to_desc, &dcp->c_desc, &cp->c_desc, NULL);
+			if (err && err != EIO && err != ENXIO)
+				panic("hfs_makelink: error %d from cat_rename backout 1", err);
+			goto out;
+		}
+		cp->c_attr.ca_linkref = indnodeno;
+		cp->c_desc.cd_cnid = linkcnid;
+		/* Directory hard links store the first link in an attribute. */
+		if (type == DIR_HARDLINKS) {
+			if (setfirstlink(hfsmp, cp->c_fileid, linkcnid) == 0)
+				cp->c_attr.ca_recflags |= kHFSHasAttributesMask;
+		} else /* FILE_HARDLINKS */ {
+			cp->c_attr.ca_firstlink = linkcnid;
 		}
-		cp->c_rdev = indnodeno;
+		cp->c_attr.ca_recflags |= kHFSHasLinkChainMask;
 	} else {
-		indnodeno = cp->c_rdev;
+		indnodeno = cp->c_attr.ca_linkref;
 	}
 
 	/*
 	 * Create a catalog entry for the new link (parentID + name).
 	 */
-	retval = createindirectlink(hfsmp, indnodeno, dcp->c_fileid, cnp->cn_nameptr, NULL);
+	
+	bzero(&link_desc, sizeof(link_desc));
+	link_desc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
+	link_desc.cd_namelen = strlen(cnp->cn_nameptr);
+	link_desc.cd_parentcnid = dcp->c_fileid;
+	link_desc.cd_flags = S_ISDIR(cp->c_mode) ? CD_ISDIR : 0;
+
+	/* Directory hard links store the first link in an attribute. */
+	if (type == DIR_HARDLINKS) {
+		retval = getfirstlink(hfsmp, cp->c_fileid, &orig_firstlink);
+	} else /* FILE_HARDLINKS */ {
+		orig_firstlink = cp->c_attr.ca_firstlink;
+	}
+	if (retval == 0)
+		retval = createindirectlink(hfsmp, indnodeno, &link_desc, 
+				orig_firstlink, &linkcnid, 
+				(cp->c_attr.ca_recflags & kHFSHasLinkChainMask));
 	if (retval && newlink) {
-	    int err;
+		int err;
 
-	    /* Get rid of new link */
-	    (void) cat_delete(hfsmp, &cp->c_desc, &cp->c_attr);
-	    
-	    // Put this back to what it was before.
-	    cp->c_desc.cd_cnid = orig_cnid;
+		/* Get rid of new link */
+		(void) cat_delete(hfsmp, &cp->c_desc, &cp->c_attr);
+		
+		/* Restore the cnode's cnid. */
+		cp->c_desc.cd_cnid = orig_cnid;
+		
+		/* Put the original file back. */
+		err = cat_rename(hfsmp, &to_desc, &dcp->c_desc, &cp->c_desc, NULL);
+		if (err && err != EIO && err != ENXIO)
+			panic("hfs_makelink: error %d from cat_rename backout 2", err);
 
-	    /* Put the source file back */
-	    err = cat_rename(hfsmp, &to_desc, &dcp->c_desc, &cp->c_desc, NULL);
-	    if (err)
-		panic("hfs_makelink: error %d from cat_rename backout 2", err);
+		cp->c_attr.ca_linkref = 0;
+		goto out;
+	} else if (retval == 0) {
 
-	    goto out;
-	}
+	    /* Update the original first link to point back to the new first link. */
+	    if (cp->c_attr.ca_recflags & kHFSHasLinkChainMask) {
+		(void) cat_updatelink(hfsmp, orig_firstlink, linkcnid, HFS_IGNORABLE_LINK);
 
-	/*
-	 * Finally, if this is a new hardlink then:
-	 *  - update HFS Private Data dir
-	 *  - mark the cnode as a hard link
-	 */
-	if (newlink) {
+		/* Update the inode's first link value. */
+		if (type == DIR_HARDLINKS) {
+		    if (setfirstlink(hfsmp, cp->c_fileid, linkcnid) == 0)
+			cp->c_attr.ca_recflags |= kHFSHasAttributesMask;
+		} else {
+		    cp->c_attr.ca_firstlink = linkcnid;
+		}
+	    }
+	    /*
+	     * Finally, if this is a new hardlink then:
+	     *  - update the private system directory
+	     *  - mark the cnode as a hard link
+	     */
+	    if (newlink) {
 		vnode_t vp;
 		
-	    if (retval != 0) {
-		panic("hfs_makelink: retval %d but newlink = 1!\n", retval);
-	    }
-	    
-		hfsmp->hfs_privdir_attr.ca_entries++;
-		retval = cat_update(hfsmp, &hfsmp->hfs_privdir_desc,
-				    &hfsmp->hfs_privdir_attr, NULL, NULL);
 		if (retval != 0) {
-		    panic("hfs_makelink: cat_update of privdir failed! (%d)\n",
-			  retval);
+		    panic("hfs_makelink: retval %d but newlink = 1!\n", retval);
+		}
+
+		hfsmp->hfs_private_attr[type].ca_entries++;
+		/* From application perspective, directory hard link is a 
+		 * normal directory.  Therefore count the new directory 
+		 * hard link for folder count calculation.
+		 */
+		if (type == DIR_HARDLINKS) {
+			INC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[type]);
+		}
+		retval = cat_update(hfsmp, &hfsmp->hfs_private_desc[type],
+		    &hfsmp->hfs_private_attr[type], NULL, NULL);
+		if (retval != 0 && retval != EIO && retval != ENXIO) {
+		    panic("hfs_makelink: cat_update of privdir failed! (%d)\n", retval);
 		}
-		hfs_volupdate(hfsmp, VOL_MKFILE, 0);
 		cp->c_flag |= C_HARDLINK;
 		if ((vp = cp->c_vp) != NULLVP) {
-			if (vnode_get(vp) == 0) {
-				vnode_set_hard_link(vp);
-				vnode_put(vp);
-			}
+		    if (vnode_get(vp) == 0) {
+			vnode_setmultipath(vp);
+			vnode_put(vp);
+		    }
 		}
 		if ((vp = cp->c_rsrc_vp) != NULLVP) {
-			if (vnode_get(vp) == 0) {
-				vnode_set_hard_link(vp);
-				vnode_put(vp);
-			}
+		    if (vnode_get(vp) == 0) {
+			vnode_setmultipath(vp);
+			vnode_put(vp);
+		    }
 		}
 		cp->c_touch_chgtime = TRUE;
 		cp->c_flag |= C_FORCEUPDATE;
+	    }
+	    dcp->c_flag |= C_FORCEUPDATE;
 	}
-	dcp->c_flag |= C_FORCEUPDATE;
-
 out:
 	hfs_systemfile_unlock(hfsmp, lockflags);
 
 	cat_postflight(hfsmp, &cookie, p);
+	
+	if (retval == 0 && newlink) {
+		hfs_volupdate(hfsmp, VOL_MKFILE, 0);
+	}
 	return (retval);
 }
 
 
 /*
- * link vnode call
-#% link		vp	U U U
-#% link		tdvp	L U U
-#
- vnop_link {
-     IN WILLRELE struct vnode *vp;
-     IN struct vnode *targetPar_vp;
-     IN struct componentname *cnp;
-     IN vfs_context_t context;
-
-     */
+ * link vnode operation
+ *
+ *  IN vnode_t  a_vp;
+ *  IN vnode_t  a_tdvp;
+ *  IN struct componentname  *a_cnp;
+ *  IN vfs_context_t  a_context;
+ */
 __private_extern__
 int
 hfs_vnop_link(struct vnop_link_args *ap)
@@ -273,30 +371,74 @@ hfs_vnop_link(struct vnop_link_args *ap)
 	struct hfsmount *hfsmp;
 	struct vnode *vp = ap->a_vp;
 	struct vnode *tdvp = ap->a_tdvp;
+	struct vnode *fdvp = NULLVP;
 	struct componentname *cnp = ap->a_cnp;
 	struct cnode *cp;
 	struct cnode *tdcp;
+	struct cnode *fdcp = NULL;
+	struct cat_desc todesc;
+	int lockflags = 0;
+	int intrans = 0;
 	enum vtype v_type;
-	int error, ret, lockflags;
-	struct cat_desc cndesc;
+	int error, ret;
+
+	hfsmp = VTOHFS(vp);
+	v_type = vnode_vtype(vp);
 
-	if (VTOVCB(tdvp)->vcbSigWord != kHFSPlusSigWord) {
-		return err_link(ap);	/* hfs disks don't support hard links */
+	/* No hard links in HFS standard file systems. */
+	if (hfsmp->hfs_flags & HFS_STANDARD) {
+		return (ENOTSUP);
 	}
-	if (VTOHFS(vp)->hfs_privdir_desc.cd_cnid == 0) {
-		return err_link(ap);	/* no private metadata dir, no links possible */
+	/* Linking to a special file is not permitted. */
+	if (v_type == VBLK || v_type == VCHR) {
+		return (EPERM);  
 	}
-	if (vnode_mount(tdvp) != vnode_mount(vp)) {
-		return (EXDEV);
+	if (v_type == VDIR) {
+		/* Make sure our private directory exists. */
+		if (hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid == 0) {
+			return (EPERM);
+		}
+		/*
+		 * Directory hardlinks (ADLs) have only been qualified on
+		 * journaled HFS+.  If/when they are tested on non-journaled
+		 * file systems then this test can be removed.
+		 */
+		if (hfsmp->jnl == NULL) {
+			return (EPERM);
+		}
+		/* Directory hardlinks also need the parent of the original directory. */
+		if ((error = hfs_vget(hfsmp, hfs_currentparent(VTOC(vp)), &fdvp, 1))) {
+			return (error);
+		}
+	} else {
+		/* Make sure our private directory exists. */
+		if (hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid == 0) {
+			return (ENOTSUP);
+		}
 	}
-	if ((error = hfs_lockpair(VTOC(tdvp), VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
-		return (error);
+	if (hfs_freeblks(hfsmp, 0) == 0) {
+		if (fdvp) {
+			vnode_put(fdvp);
+		}
+		return (ENOSPC);
+	}
+	/* Lock the cnodes. */
+	if (fdvp) {
+		if ((error = hfs_lockfour(VTOC(tdvp), VTOC(vp), VTOC(fdvp), NULL, HFS_EXCLUSIVE_LOCK))) {
+			if (fdvp) {
+				vnode_put(fdvp);
+		    	}
+			return (error);
+		}
+		fdcp = VTOC(fdvp);
+	} else {
+		if ((error = hfs_lockpair(VTOC(tdvp), VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+			return (error);
+		}
 	}
 	tdcp = VTOC(tdvp);
 	cp = VTOC(vp);
-	hfsmp = VTOHFS(vp);
-
-	if (cp->c_nlink >= HFS_LINK_MAX) {
+	if (cp->c_linkcount >= HFS_LINK_MAX) {
 		error = EMLINK;
 		goto out;
 	}
@@ -305,60 +447,807 @@ hfs_vnop_link(struct vnop_link_args *ap)
 		goto out;
 	}
 	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-	    error = ENOENT;
-	    goto out;
-	}
-	
-	v_type = vnode_vtype(vp);
-	if (v_type == VBLK || v_type == VCHR) {
-		error = EINVAL;  /* cannot link to a special file */
+		error = ENOENT;
 		goto out;
 	}
 
+	tdcp->c_flag |= C_DIR_MODIFICATION;
+
 	if (hfs_start_transaction(hfsmp) != 0) {
-	    error = EINVAL;  /* cannot link to a special file */
-	    goto out;
+		error = EINVAL;
+		goto out;
 	}
+	intrans = 1;
 
-	cp->c_nlink++;
-	cp->c_touch_chgtime = TRUE;
+	todesc.cd_flags = (v_type == VDIR) ? CD_ISDIR : 0;
+	todesc.cd_encoding = 0;
+	todesc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
+	todesc.cd_namelen = cnp->cn_namelen;
+	todesc.cd_parentcnid = tdcp->c_fileid;
+	todesc.cd_hint = 0;
+	todesc.cd_cnid = 0;
 
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+
+	/* If destination exists then we lost a race with create. */
+	if (cat_lookup(hfsmp, &todesc, 0, NULL, NULL, NULL, NULL) == 0) {
+		error = EEXIST;
+		goto out;
+	}
+	if (cp->c_flag & C_HARDLINK) {
+		struct cat_attr cattr;
+
+		/* If inode is missing then we lost a race with unlink. */
+		if ((cat_idlookup(hfsmp, cp->c_fileid, 0, NULL, &cattr, NULL) != 0) ||
+		    (cattr.ca_fileid != cp->c_fileid)) {
+			error = ENOENT;
+			goto out;
+		}
+	} else {
+		cnid_t fileid;
+
+		/* If source is missing then we lost a race with unlink. */
+		if ((cat_lookup(hfsmp, &cp->c_desc, 0, NULL, NULL, NULL, &fileid) != 0) ||
+		    (fileid != cp->c_fileid)) {
+			error = ENOENT;
+			goto out;
+		}
+	}
+	/* 
+	 * All directory links must reside in an non-ARCHIVED hierarchy.
+	 */
+	if (v_type == VDIR) {
+		/*
+		 * - Source parent and destination parent cannot match
+		 * - A link is not permitted in the root directory
+		 * - Parent of 'pointed at' directory is not the root directory
+		 * - The 'pointed at' directory (source) is not an ancestor
+		 *   of the new directory hard link (destination).
+		 * - No ancestor of the new directory hard link (destination) 
+		 *   is a directory hard link.
+		 */
+		if ((cp->c_parentcnid == tdcp->c_fileid) ||
+		    (tdcp->c_fileid == kHFSRootFolderID) ||
+		    (cp->c_parentcnid == kHFSRootFolderID) ||
+		    cat_check_link_ancestry(hfsmp, tdcp->c_fileid, cp->c_fileid)) {
+			error = EPERM;  /* abide by the rules, you did not */
+			goto out;
+		}
+	}
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	lockflags = 0;
+
+	cp->c_linkcount++;
+	cp->c_touch_chgtime = TRUE;
 	error = hfs_makelink(hfsmp, cp, tdcp, cnp);
 	if (error) {
-		cp->c_nlink--;
+		cp->c_linkcount--;
 		hfs_volupdate(hfsmp, VOL_UPDATE, 0);
 	} else {
 		/* Invalidate negative cache entries in the destination directory */
-		if (hfsmp->hfs_flags & HFS_CASE_SENSITIVE)
+		if (tdcp->c_flag & C_NEG_ENTRIES) {
 			cache_purge_negatives(tdvp);
+			tdcp->c_flag &= ~C_NEG_ENTRIES;
+		}
 
 		/* Update the target directory and volume stats */
-		tdcp->c_nlink++;
 		tdcp->c_entries++;
+		if (v_type == VDIR) {
+			INC_FOLDERCOUNT(hfsmp, tdcp->c_attr);
+			tdcp->c_attr.ca_recflags |= kHFSHasChildLinkMask;
+
+			/* Set kHFSHasChildLinkBit in the destination hierarchy */
+			error = cat_set_childlinkbit(hfsmp, tdcp->c_parentcnid);
+			if (error) {
+				printf ("hfs_vnop_link: error updating destination parent chain for %u\n", tdcp->c_cnid);
+				error = 0;
+			}
+		}
+		tdcp->c_dirchangecnt++;
 		tdcp->c_touch_chgtime = TRUE;
 		tdcp->c_touch_modtime = TRUE;
 		tdcp->c_flag |= C_FORCEUPDATE;
 
 		error = hfs_update(tdvp, 0);
-		if (error) {
-		    panic("hfs_vnop_link: error updating tdvp 0x%x\n", tdvp);
+		if (error && error != EIO && error != ENXIO) {
+			panic("hfs_vnop_link: error updating tdvp %p\n", tdvp);
 		}
+		
+		if ((v_type == VDIR) && 
+		    (fdcp != NULL) && 
+		    ((fdcp->c_attr.ca_recflags & kHFSHasChildLinkMask) == 0)) {
 
+			fdcp->c_attr.ca_recflags |= kHFSHasChildLinkMask;
+			fdcp->c_touch_chgtime = TRUE;
+			fdcp->c_flag |= C_FORCEUPDATE;
+			error = hfs_update(fdvp, 0);
+			if (error && error != EIO && error != ENXIO) {
+				panic("hfs_vnop_link: error updating fdvp %p\n", fdvp);
+			}
+
+			/* Set kHFSHasChildLinkBit in the source hierarchy */
+			error = cat_set_childlinkbit(hfsmp, fdcp->c_parentcnid);
+			if (error) {
+				printf ("hfs_vnop_link: error updating source parent chain for %u\n", fdcp->c_cnid);
+				error = 0;
+			}
+		}
 		hfs_volupdate(hfsmp, VOL_MKFILE,
 			(tdcp->c_cnid == kHFSRootFolderID));
 	}
+	/* Make sure update occurs inside transaction */
+	cp->c_flag |= C_FORCEUPDATE;  
 
-	cp->c_flag |= C_FORCEUPDATE;    // otherwise hfs_update() might skip the update
-
-	if ((ret = hfs_update(vp, TRUE)) != 0) {
-	    panic("hfs_vnop_link: error %d updating vp @ 0x%x\n", ret, vp);
+	if ((error == 0) && (ret = hfs_update(vp, TRUE)) != 0 && ret != EIO && ret != ENXIO) {
+		panic("hfs_vnop_link: error %d updating vp @ %p\n", ret, vp);
 	}
 
-	hfs_end_transaction(hfsmp);
-	
 	HFS_KNOTE(vp, NOTE_LINK);
 	HFS_KNOTE(tdvp, NOTE_WRITE);
 out:
-	hfs_unlockpair(tdcp, cp);
+	if (lockflags) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
+	}
+	if (intrans) {
+		hfs_end_transaction(hfsmp);
+	}
+
+	tdcp->c_flag &= ~C_DIR_MODIFICATION;
+	wakeup((caddr_t)&tdcp->c_flag);
+
+	if (fdcp) {
+		hfs_unlockfour(tdcp, cp, fdcp, NULL);
+	} else {
+		hfs_unlockpair(tdcp, cp);
+	}
+	if (fdvp) {
+		vnode_put(fdvp);
+	}
 	return (error);
 }
+
+
+/*
+ * Remove a link to a hardlink file/dir.
+ *
+ * Note: dvp and vp cnodes are already locked.
+ */
+__private_extern__
+int
+hfs_unlink(struct hfsmount *hfsmp, struct vnode *dvp, struct vnode *vp, struct componentname *cnp, int skip_reserve)
+{
+	struct cnode *cp;
+	struct cnode *dcp;
+	struct cat_desc cndesc;
+	struct timeval tv;
+	char inodename[32];
+	cnid_t  prevlinkid;
+	cnid_t  nextlinkid;
+	int lockflags = 0;
+	int started_tr;
+	int rm_priv_file = 0;
+	int error;
+	
+	if (hfsmp->hfs_flags & HFS_STANDARD) {
+		return (EPERM);
+	}
+	cp = VTOC(vp);
+	dcp = VTOC(dvp);
+
+	dcp->c_flag |= C_DIR_MODIFICATION;
+	
+	/* Remove the entry from the namei cache: */
+	cache_purge(vp);
+
+	if ((error = hfs_start_transaction(hfsmp)) != 0) {
+		started_tr = 0;
+		goto out;
+	}
+	started_tr = 1;
+
+	/* 
+	 * Protect against a race with rename by using the component
+	 * name passed in and parent id from dvp (instead of using 
+	 * the cp->c_desc which may have changed).  
+	 *
+	 * Re-lookup the component name so we get the correct cnid
+	 * for the name (as opposed to the c_cnid in the cnode which
+	 * could have changed before the cnode was locked).
+	 */
+	cndesc.cd_flags = vnode_isdir(vp) ? CD_ISDIR : 0;
+	cndesc.cd_encoding = cp->c_desc.cd_encoding;
+	cndesc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
+	cndesc.cd_namelen = cnp->cn_namelen;
+	cndesc.cd_parentcnid = dcp->c_fileid;
+	cndesc.cd_hint = dcp->c_childhint;
+
+	lockflags = SFL_CATALOG | SFL_ATTRIBUTE;
+	if (cndesc.cd_flags & CD_ISDIR) {
+		/* We'll be removing the alias resource allocation blocks. */
+		lockflags |= SFL_BITMAP;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, lockflags, HFS_EXCLUSIVE_LOCK);
+
+	if ((error = cat_lookuplink(hfsmp, &cndesc, &cndesc.cd_cnid, &prevlinkid, &nextlinkid))) {
+		goto out;
+	}
+
+	/* Reserve some space in the catalog file. */
+	if (!skip_reserve && (error = cat_preflight(hfsmp, 2 * CAT_DELETE, NULL, 0))) {
+		goto out;
+	}
+
+	/* Purge any cached origin entries for a directory hard link. */
+	if (cndesc.cd_flags & CD_ISDIR) {
+		hfs_relorigin(cp, dcp->c_fileid);
+		if (dcp->c_fileid != dcp->c_cnid) {
+			hfs_relorigin(cp, dcp->c_cnid);
+		}
+	}
+
+	/* Delete the link record. */
+	if ((error = cat_deletelink(hfsmp, &cndesc))) {
+		goto out;
+	}
+
+	/* Update the parent directory. */
+	if (dcp->c_entries > 0) {
+		dcp->c_entries--;
+	}
+	if (cndesc.cd_flags & CD_ISDIR) {
+		DEC_FOLDERCOUNT(hfsmp, dcp->c_attr);
+	}
+	dcp->c_dirchangecnt++;
+	microtime(&tv);
+	dcp->c_ctime = tv.tv_sec;
+	dcp->c_mtime = tv.tv_sec;
+	(void ) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
+
+	/*
+	 * If this is the last link then we need to process the inode.
+	 * Otherwise we need to fix up the link chain.
+	 */
+	--cp->c_linkcount;
+	if (cp->c_linkcount < 1) {
+		char delname[32];
+		struct cat_desc to_desc;
+		struct cat_desc from_desc;
+
+		/*
+		 * If a file inode or directory inode is being deleted, rename 
+		 * it to an open deleted file.  This ensures that deletion 
+		 * of inode and its corresponding extended attributes does 
+		 * not overflow the journal.  This inode will be deleted 
+		 * either in hfs_vnop_inactive() or in hfs_remove_orphans(). 
+		 * Note: a rename failure here is not fatal.
+		 */	
+		bzero(&from_desc, sizeof(from_desc));
+		bzero(&to_desc, sizeof(to_desc));
+		if (vnode_isdir(vp)) {
+			if (cp->c_entries != 0) {
+				panic("hfs_unlink: dir not empty (id %d, %d entries)", cp->c_fileid, cp->c_entries);
+			}
+			MAKE_DIRINODE_NAME(inodename, sizeof(inodename),
+						cp->c_attr.ca_linkref);
+			from_desc.cd_parentcnid = hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid;
+			from_desc.cd_flags = CD_ISDIR;
+			to_desc.cd_flags = CD_ISDIR;
+		} else { 
+			MAKE_INODE_NAME(inodename, sizeof(inodename),
+					cp->c_attr.ca_linkref);
+			from_desc.cd_parentcnid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+			from_desc.cd_flags = 0;
+			to_desc.cd_flags = 0;
+		}
+		from_desc.cd_nameptr = (const u_int8_t *)inodename;
+		from_desc.cd_namelen = strlen(inodename);
+		from_desc.cd_cnid = cp->c_fileid;
+
+		MAKE_DELETED_NAME(delname, sizeof(delname), cp->c_fileid);
+		to_desc.cd_nameptr = (const u_int8_t *)delname;
+		to_desc.cd_namelen = strlen(delname);
+		to_desc.cd_parentcnid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+		to_desc.cd_cnid = cp->c_fileid;
+
+		error = cat_rename(hfsmp, &from_desc, &hfsmp->hfs_private_desc[FILE_HARDLINKS],
+				   &to_desc, (struct cat_desc *)NULL);
+		if (error == 0) {
+			cp->c_flag |= C_DELETED;
+			cp->c_attr.ca_recflags &= ~kHFSHasLinkChainMask;
+			cp->c_attr.ca_firstlink = 0;
+			if (vnode_isdir(vp)) {
+				hfsmp->hfs_private_attr[DIR_HARDLINKS].ca_entries--;
+				DEC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[DIR_HARDLINKS]);
+
+				hfsmp->hfs_private_attr[FILE_HARDLINKS].ca_entries++;
+				INC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[FILE_HARDLINKS]);
+
+				(void)cat_update(hfsmp, &hfsmp->hfs_private_desc[DIR_HARDLINKS],
+					&hfsmp->hfs_private_attr[DIR_HARDLINKS], NULL, NULL);
+				(void)cat_update(hfsmp, &hfsmp->hfs_private_desc[FILE_HARDLINKS],
+					&hfsmp->hfs_private_attr[FILE_HARDLINKS], NULL, NULL);
+			}
+		} else {
+			error = 0;  /* rename failure here is not fatal */
+		}
+	} else /* Still some links left */ {
+		cnid_t firstlink;
+
+		/*
+		 * Update the start of the link chain.
+		 * Note: Directory hard links store the first link in an attribute.
+		 */
+		if (vnode_isdir(vp) &&
+		    getfirstlink(hfsmp, cp->c_fileid, &firstlink) == 0 &&
+		    firstlink == cndesc.cd_cnid) {
+			if (setfirstlink(hfsmp, cp->c_fileid, nextlinkid) == 0)
+				cp->c_attr.ca_recflags |= kHFSHasAttributesMask;
+		} else if (vnode_isreg(vp) && cp->c_attr.ca_firstlink == cndesc.cd_cnid) {
+			cp->c_attr.ca_firstlink = nextlinkid;
+		}
+		/* Update previous link. */
+		if (prevlinkid) {
+			(void) cat_updatelink(hfsmp, prevlinkid, HFS_IGNORABLE_LINK, nextlinkid);
+		}
+		/* Update next link. */
+		if (nextlinkid) {
+			(void) cat_updatelink(hfsmp, nextlinkid, prevlinkid, HFS_IGNORABLE_LINK);
+		}
+	}
+
+	/* Push new link count to disk. */
+	cp->c_ctime = tv.tv_sec;	
+	(void) cat_update(hfsmp, &cp->c_desc, &cp->c_attr, NULL, NULL);
+
+	/* All done with the system files. */
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	lockflags = 0;
+
+	/* Update file system stats. */
+	hfs_volupdate(hfsmp, VOL_RMFILE, (dcp->c_cnid == kHFSRootFolderID));
+	/* The last link of a directory removed the inode. */
+	if (rm_priv_file) {
+		hfs_volupdate(hfsmp, VOL_RMFILE, 0);
+	}
+	/*
+	 * All done with this cnode's descriptor...
+	 *
+	 * Note: all future catalog calls for this cnode may be
+	 * by fileid only.  This is OK for HFS (which doesn't have
+	 * file thread records) since HFS doesn't support hard links.
+	 */
+	cat_releasedesc(&cp->c_desc);
+
+	HFS_KNOTE(dvp, NOTE_WRITE);
+	HFS_KNOTE(vp, NOTE_DELETE);
+out:
+	if (lockflags) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
+	}
+	if (started_tr) {
+		hfs_end_transaction(hfsmp);
+	}
+
+	dcp->c_flag &= ~C_DIR_MODIFICATION;
+	wakeup((caddr_t)&dcp->c_flag);
+
+	return (error);
+}
+
+
+/*
+ * Initialize the HFS+ private system directories.
+ *
+ * These directories are used to hold the inodes
+ * for file and directory hardlinks as well as
+ * open-unlinked files.
+ *
+ * If they don't yet exist they will get created.
+ *
+ * This call is assumed to be made during mount.
+ */
+__private_extern__
+void
+hfs_privatedir_init(struct hfsmount * hfsmp, enum privdirtype type)
+{
+	struct vnode * dvp = NULLVP;
+	struct cnode * dcp = NULL;
+	struct cat_desc *priv_descp;
+	struct cat_attr *priv_attrp;
+	struct FndrDirInfo * fndrinfo;
+	struct timeval tv;
+	int lockflags;
+	int trans = 0;
+	int error;
+	
+	if (hfsmp->hfs_flags & HFS_STANDARD) {
+		return;
+	}
+
+	priv_descp = &hfsmp->hfs_private_desc[type];
+	priv_attrp = &hfsmp->hfs_private_attr[type];
+
+	/* Check if directory already exists. */
+	if (priv_descp->cd_cnid != 0) {
+		return;
+	}
+
+	priv_descp->cd_parentcnid = kRootDirID;
+	priv_descp->cd_nameptr = (const u_int8_t *)hfs_private_names[type];
+	priv_descp->cd_namelen = strlen((const char *)priv_descp->cd_nameptr);
+	priv_descp->cd_flags = CD_ISDIR | CD_DECOMPOSED;
+
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+	error = cat_lookup(hfsmp, priv_descp, 0, NULL, priv_attrp, NULL, NULL);
+	hfs_systemfile_unlock(hfsmp, lockflags);
+
+	if (error == 0) {
+		if (type == FILE_HARDLINKS) {
+			hfsmp->hfs_metadata_createdate = priv_attrp->ca_itime;
+		}
+		priv_descp->cd_cnid = priv_attrp->ca_fileid;
+		goto exit;
+	}
+
+	/* Directory is missing, if this is read-only then we're done. */
+	if (hfsmp->hfs_flags & HFS_READ_ONLY) {
+		goto exit;
+	}
+
+	/* Grab the root directory so we can update it later. */
+	if (hfs_vget(hfsmp, kRootDirID, &dvp, 0) != 0) {
+		goto exit;
+	}
+	dcp = VTOC(dvp);
+
+	/* Setup the default attributes */
+	bzero(priv_attrp, sizeof(struct cat_attr));
+	priv_attrp->ca_flags = UF_IMMUTABLE | UF_HIDDEN;
+	priv_attrp->ca_mode = S_IFDIR;
+	if (type == DIR_HARDLINKS) {
+		priv_attrp->ca_mode |= S_ISVTX | S_IRUSR | S_IXUSR | S_IRGRP |
+		                       S_IXGRP | S_IROTH | S_IXOTH;
+	}
+	priv_attrp->ca_linkcount = 1;
+	priv_attrp->ca_itime = hfsmp->hfs_itime;
+	priv_attrp->ca_recflags = kHFSHasFolderCountMask;
+	
+	fndrinfo = (struct FndrDirInfo *)&priv_attrp->ca_finderinfo;
+	fndrinfo->frLocation.v = SWAP_BE16(16384);
+	fndrinfo->frLocation.h = SWAP_BE16(16384);
+	fndrinfo->frFlags = SWAP_BE16(kIsInvisible + kNameLocked);		
+
+	if (hfs_start_transaction(hfsmp) != 0) {
+		goto exit;
+	}
+	trans = 1;
+
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
+
+	/* Make sure there's space in the Catalog file. */
+	if (cat_preflight(hfsmp, CAT_CREATE, NULL, 0) != 0) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
+		goto exit;
+	}
+
+	/* Create the private directory on disk. */
+	error = cat_create(hfsmp, priv_descp, priv_attrp, NULL);
+	if (error == 0) {
+		priv_descp->cd_cnid = priv_attrp->ca_fileid;
+
+		/* Update the parent directory */
+		dcp->c_entries++;
+		INC_FOLDERCOUNT(hfsmp, dcp->c_attr);
+		dcp->c_dirchangecnt++;
+		microtime(&tv);
+		dcp->c_ctime = tv.tv_sec;
+		dcp->c_mtime = tv.tv_sec;
+		(void) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
+	}
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	
+	if (error) {
+		goto exit;
+	}
+	if (type == FILE_HARDLINKS) {
+		hfsmp->hfs_metadata_createdate = hfsmp->hfs_itime;
+	}
+	hfs_volupdate(hfsmp, VOL_MKDIR, 1);
+exit:
+	if (trans) {
+		hfs_end_transaction(hfsmp);
+	}
+	if (dvp) {
+		hfs_unlock(dcp);
+		vnode_put(dvp);
+	}
+	if ((error == 0) && (type == DIR_HARDLINKS)) {
+		hfs_xattr_init(hfsmp);
+	}
+}
+
+
+/*
+ * Lookup a hardlink link (from chain)
+ */
+__private_extern__
+int
+hfs_lookuplink(struct hfsmount *hfsmp, cnid_t linkfileid, cnid_t *prevlinkid,  cnid_t *nextlinkid)
+{
+	int lockflags;
+	int error;
+
+	*prevlinkid = 0;
+	*nextlinkid = 0;
+
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+
+	error = cat_lookuplinkbyid(hfsmp, linkfileid, prevlinkid, nextlinkid);
+	if (error == ENOLINK) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_SHARED_LOCK);
+
+		error = getfirstlink(hfsmp, linkfileid, nextlinkid);
+	}
+	hfs_systemfile_unlock(hfsmp, lockflags);
+
+	return (error);
+}
+
+/*
+ * Cache the orgin of a directory hard link
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+void
+hfs_savelinkorigin(cnode_t *cp, cnid_t parentcnid)
+{
+	linkorigin_t *origin = NULL;
+	void * thread = current_thread();
+	int count = 0;
+
+	/*
+	 *  Look for an existing origin first.  If not found, create/steal one.
+	 */
+	TAILQ_FOREACH(origin, &cp->c_originlist, lo_link) {
+		++count;
+		if (origin->lo_thread == thread) {
+			TAILQ_REMOVE(&cp->c_originlist, origin, lo_link);
+			break;
+		}
+	}
+	if (origin == NULL) {
+		/* Recycle the last (i.e., the oldest) if we have too many. */
+		if (count > MAX_CACHED_ORIGINS) {
+			origin = TAILQ_LAST(&cp->c_originlist, hfs_originhead);
+			TAILQ_REMOVE(&cp->c_originlist, origin, lo_link);
+		} else {
+			MALLOC(origin, linkorigin_t *, sizeof(linkorigin_t), M_TEMP, M_WAITOK);
+		}
+		origin->lo_thread = thread;
+	}
+	origin->lo_cnid = cp->c_cnid;
+	origin->lo_parentcnid = parentcnid;
+	TAILQ_INSERT_HEAD(&cp->c_originlist, origin, lo_link);
+}
+
+/*
+ * Release any cached origins for a directory hard link
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+void
+hfs_relorigins(struct cnode *cp)
+{
+	linkorigin_t *origin, *prev;
+
+	TAILQ_FOREACH_SAFE(origin, &cp->c_originlist, lo_link, prev) {
+		FREE(origin, M_TEMP);
+	}
+	TAILQ_INIT(&cp->c_originlist);
+}
+
+/*
+ * Release a specific origin for a directory hard link
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+void
+hfs_relorigin(struct cnode *cp, cnid_t parentcnid)
+{
+	linkorigin_t *origin = NULL;
+	void * thread = current_thread();
+
+	TAILQ_FOREACH(origin, &cp->c_originlist, lo_link) {
+		if ((origin->lo_thread == thread) ||
+		    (origin->lo_parentcnid == parentcnid)) {
+			TAILQ_REMOVE(&cp->c_originlist, origin, lo_link);
+			break;
+		}
+	}
+}
+
+/*
+ * Test if a directory hard link has a cached origin
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+int
+hfs_haslinkorigin(cnode_t *cp)
+{
+	if (cp->c_flag & C_HARDLINK) {
+		linkorigin_t *origin;
+		void * thread = current_thread();
+	
+		TAILQ_FOREACH(origin, &cp->c_originlist, lo_link) {
+			if (origin->lo_thread == thread) {
+				return (1);
+			}
+		}
+	}
+	return (0);
+}
+
+/*
+ * Obtain the current parent cnid of a directory hard link
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+cnid_t
+hfs_currentparent(cnode_t *cp)
+{
+	if (cp->c_flag & C_HARDLINK) {
+		linkorigin_t *origin;
+		void * thread = current_thread();
+	
+		TAILQ_FOREACH(origin, &cp->c_originlist, lo_link) {
+			if (origin->lo_thread == thread) {
+				return (origin->lo_parentcnid);
+			}
+		}
+	}
+	return (cp->c_parentcnid);
+}
+
+/*
+ * Obtain the current cnid of a directory hard link
+ *
+ * cnode must be lock on entry
+ */
+__private_extern__
+cnid_t
+hfs_currentcnid(cnode_t *cp)
+{
+	if (cp->c_flag & C_HARDLINK) {
+		linkorigin_t *origin;
+		void * thread = current_thread();
+	
+		TAILQ_FOREACH(origin, &cp->c_originlist, lo_link) {
+			if (origin->lo_thread == thread) {
+				return (origin->lo_cnid);
+			}
+		}
+	}
+	return (cp->c_cnid);
+}
+
+
+/*
+ * Set the first link attribute for a given file id.
+ *
+ * The attributes b-tree must already be locked.
+ * If journaling is enabled, a transaction must already be started.
+ */
+static int
+setfirstlink(struct hfsmount * hfsmp, cnid_t fileid, cnid_t firstlink)
+{
+	FCB * btfile;
+	BTreeIterator * iterator;
+	FSBufferDescriptor btdata;
+	u_int8_t attrdata[FIRST_LINK_XATTR_REC_SIZE];
+	HFSPlusAttrData *dataptr;
+	int result;
+	u_int16_t datasize;
+
+	if (hfsmp->hfs_attribute_cp == NULL) {
+		return (EPERM);
+	}
+	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	bzero(iterator, sizeof(*iterator));
+
+	result = hfs_buildattrkey(fileid, FIRST_LINK_XATTR_NAME, (HFSPlusAttrKey *)&iterator->key);
+	if (result) {
+		goto out;
+	}
+	dataptr = (HFSPlusAttrData *)&attrdata[0];
+	dataptr->recordType = kHFSPlusAttrInlineData;
+	dataptr->reserved[0] = 0;
+	dataptr->reserved[1] = 0;
+
+	/*
+	 * Since attrData is variable length, we calculate the size of
+	 * attrData by subtracting the size of all other members of
+	 * structure HFSPlusAttData from the size of attrdata.
+	 */
+	(void)snprintf((char *)&dataptr->attrData[0],
+			sizeof(dataptr) - (4 * sizeof(uint32_t)),
+		        "%lu", (unsigned long)firstlink);
+	dataptr->attrSize = 1 + strlen((char *)&dataptr->attrData[0]);
+
+	/* Calculate size of record rounded up to multiple of 2 bytes. */
+	datasize = sizeof(HFSPlusAttrData) - 2 + dataptr->attrSize + ((dataptr->attrSize & 1) ? 1 : 0);
+
+	btdata.bufferAddress = dataptr;
+	btdata.itemSize = datasize;
+	btdata.itemCount = 1;
+
+	btfile = hfsmp->hfs_attribute_cp->c_datafork;
+
+	/* Insert the attribute. */
+	result = BTInsertRecord(btfile, iterator, &btdata, datasize);
+	if (result == btExists) {
+		result = BTReplaceRecord(btfile, iterator, &btdata, datasize);
+	}
+	(void) BTFlushPath(btfile);
+out:
+	FREE(iterator, M_TEMP);
+
+	return MacToVFSError(result);
+}
+
+/*
+ * Get the first link attribute for a given file id.
+ *
+ * The attributes b-tree must already be locked.
+ */
+static int
+getfirstlink(struct hfsmount * hfsmp, cnid_t fileid, cnid_t *firstlink)
+{
+	FCB * btfile;
+	BTreeIterator * iterator;
+	FSBufferDescriptor btdata;
+	u_int8_t attrdata[FIRST_LINK_XATTR_REC_SIZE];
+	HFSPlusAttrData *dataptr;
+	int result;
+	u_int16_t datasize;
+
+	if (hfsmp->hfs_attribute_cp == NULL) {
+		return (EPERM);
+	}
+	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	bzero(iterator, sizeof(*iterator));
+
+	result = hfs_buildattrkey(fileid, FIRST_LINK_XATTR_NAME, (HFSPlusAttrKey *)&iterator->key);
+	if (result)
+		goto out;
+
+	dataptr = (HFSPlusAttrData *)&attrdata[0];
+	datasize = sizeof(attrdata);
+
+	btdata.bufferAddress = dataptr;
+	btdata.itemSize = sizeof(attrdata);
+	btdata.itemCount = 1;
+
+	btfile = hfsmp->hfs_attribute_cp->c_datafork;
+
+	result = BTSearchRecord(btfile, iterator, &btdata, NULL, NULL);
+	if (result)
+		goto out;
+
+	if (dataptr->attrSize < 3) {
+		result = ENOENT;
+		goto out;
+	}
+	*firstlink = strtoul((char*)&dataptr->attrData[0], NULL, 10);
+out:
+	FREE(iterator, M_TEMP);
+
+	return MacToVFSError(result);
+}
+
diff --git a/bsd/hfs/hfs_lookup.c b/bsd/hfs/hfs_lookup.c
index 1942c91d0..b59038b3b 100644
--- a/bsd/hfs/hfs_lookup.c
+++ b/bsd/hfs/hfs_lookup.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1989, 1993
@@ -71,23 +77,14 @@
 #include <sys/mount.h>
 #include <sys/vnode.h>
 #include <sys/malloc.h>
-#include <sys/paths.h>
 #include <sys/kdebug.h>
 #include <sys/kauth.h>
+#include <sys/namei.h>
 
 #include "hfs.h"
 #include "hfs_catalog.h"
 #include "hfs_cnode.h"
 
-#define LEGACY_FORK_NAMES	1
-
-static int forkcomponent(struct componentname *cnp, int *rsrcfork);
-
-#define _PATH_DATAFORKSPEC	"/..namedfork/data"
-
-#if LEGACY_FORK_NAMES
-#define LEGACY_RSRCFORKSPEC	"/rsrc"
-#endif
 
 /*	
  * FROM FREEBSD 3.1
@@ -153,15 +150,11 @@ static int forkcomponent(struct componentname *cnp, int *rsrcfork);
  *	When should we lock parent_hp in here ??
  */
 static int
-hfs_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, vfs_context_t context, int *cnode_locked)
+hfs_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, int *cnode_locked)
 {
 	struct cnode *dcp;	/* cnode for directory being searched */
 	struct vnode *tvp;	/* target vnode */
 	struct hfsmount *hfsmp;
-	kauth_cred_t cred;
-	struct proc *p;
-	int wantrsrc = 0;
-	int forknamelen = 0;
 	int flags;
 	int nameiop;
 	int retval = 0;
@@ -172,7 +165,8 @@ hfs_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, vfs
 	struct cat_fork fork;
 	int lockflags;
 
-	dcp = VTOC(dvp);
+  retry:
+	dcp = NULL;
 	hfsmp = VTOHFS(dvp);
 	*vpp = NULL;
 	*cnode_locked = 0;
@@ -182,9 +176,6 @@ hfs_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, vfs
 	flags = cnp->cn_flags;
 	bzero(&desc, sizeof(desc));
 
-	cred = vfs_context_ucred(context);
-	p    = vfs_context_proc(context);
-
 	/*
 	 * First check to see if it is a . or .., else look it up.
 	 */
@@ -196,50 +187,66 @@ hfs_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp, vfs
 		cnp->cn_flags &= ~MAKEENTRY;
 		goto found;	/* We always know who we are */
 	} else {
-		/* Check fork suffix to see if we want the resource fork */
-		forknamelen = forkcomponent(cnp, &wantrsrc);
-		
-		/* Resource fork names are not cached. */
-		if (wantrsrc)
-			cnp->cn_flags &= ~MAKEENTRY;
+		if (hfs_lock(VTOC(dvp), HFS_EXCLUSIVE_LOCK) != 0) {
+			retval = ENOENT;  /* The parent no longer exists ? */
+			goto exit;
+		}
+		dcp = VTOC(dvp);
 
-		if (hfs_lock(dcp, HFS_EXCLUSIVE_LOCK) != 0) {
-			goto notfound;
+		if (dcp->c_flag & C_DIR_MODIFICATION) {
+		    // XXXdbg - if we could msleep on a lck_rw_t then we would do that
+		    //          but since we can't we have to unlock, delay for a bit
+		    //          and then retry...
+		    // msleep((caddr_t)&dcp->c_flag, &dcp->c_rwlock, PINOD, "hfs_vnop_lookup", 0);
+		    hfs_unlock(dcp);
+		    tsleep((caddr_t)dvp, PRIBIO, "hfs_lookup", 1);
+
+		    goto retry;
 		}
 
 		/* No need to go to catalog if there are no children */
 		if (dcp->c_entries == 0) {
-			hfs_unlock(dcp);
 			goto notfound;
 		}
 
 		bzero(&cndesc, sizeof(cndesc));
-		cndesc.cd_nameptr = cnp->cn_nameptr;
+		cndesc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 		cndesc.cd_namelen = cnp->cn_namelen;
-		cndesc.cd_parentcnid = dcp->c_cnid;
+		cndesc.cd_parentcnid = dcp->c_fileid;
 		cndesc.cd_hint = dcp->c_childhint;
 
 		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
 
-		retval = cat_lookup(hfsmp, &cndesc, wantrsrc, &desc, &attr, &fork, NULL);
+		retval = cat_lookup(hfsmp, &cndesc, 0, &desc, &attr, &fork, NULL);
 		
 		hfs_systemfile_unlock(hfsmp, lockflags);
 
 		if (retval == 0) {
 			dcp->c_childhint = desc.cd_hint;
-			hfs_unlock(dcp);
+			/*
+			 * Note: We must drop the parent lock here before calling
+			 * hfs_getnewvnode (which takes the child lock).
+			 */
+		    	hfs_unlock(dcp);
+		    	dcp = NULL;
 			goto found;
 		}
-		hfs_unlock(dcp);
 notfound:
-		/* ENAMETOOLONG supersedes other errors */
-		if (((nameiop != CREATE) && (nameiop != RENAME)) && 
-			(retval != ENAMETOOLONG) && 
-		    (cnp->cn_namelen > kHFSPlusMaxFileNameChars)) {
+		/*
+		 * ENAMETOOLONG supersedes other errors
+		 *
+		 * For a CREATE or RENAME operation on the last component
+		 * the ENAMETOOLONG will be handled in the next VNOP.
+		 */
+		if ((retval != ENAMETOOLONG) && 
+		    (cnp->cn_namelen > kHFSPlusMaxFileNameChars) &&
+		    (((flags & ISLASTCN) == 0) || ((nameiop != CREATE) && (nameiop != RENAME)))) {
 			retval = ENAMETOOLONG;
 		} else if (retval == 0) {
 			retval = ENOENT;
 		}
+		if (retval != ENOENT)
+			goto exit;
 		/*
 		 * This is a non-existing entry
 		 *
@@ -252,39 +259,23 @@ notfound:
 		    (cnp->cn_flags & DOWHITEOUT) &&
 		    (cnp->cn_flags & ISWHITEOUT))) &&
 		    (flags & ISLASTCN) &&
-		    (retval == ENOENT)) {
+		    !(ISSET(dcp->c_flag, C_DELETED | C_NOEXISTS))) {
 			retval = EJUSTRETURN;
 			goto exit;
 		}
 		/*
-		 * Insert name into cache (as non-existent) if appropriate.
-		 *
-		 * Only done for case-sensitive HFS+ volumes.
+		 * Insert name into the name cache (as non-existent).
 		 */
-		if ((retval == ENOENT) &&
-		    (hfsmp->hfs_flags & HFS_CASE_SENSITIVE) &&
-		    (cnp->cn_flags & MAKEENTRY) && nameiop != CREATE) {
+		if ((hfsmp->hfs_flags & HFS_STANDARD) == 0 &&
+		    (cnp->cn_flags & MAKEENTRY) &&
+		    (nameiop != CREATE)) {
 			cache_enter(dvp, NULL, cnp);
+			dcp->c_flag |= C_NEG_ENTRIES;
 		}
 		goto exit;
 	}
 
 found:
-	/*
-	 * Process any fork specifiers
-	 */
-	if (forknamelen && S_ISREG(attr.ca_mode)) {
-		/* fork names are only for lookups */
-		if ((nameiop != LOOKUP) && (nameiop != CREATE)) {
-			retval = EPERM;  
-			goto exit;
-		}
-		cnp->cn_consume = forknamelen;
-		flags |= ISLASTCN;
-	} else {
-		wantrsrc = 0;
-		forknamelen = 0;
-	}
 	if (flags & ISLASTCN) {
 		switch(nameiop) {
 		case DELETE:
@@ -306,37 +297,90 @@ found:
 			goto exit;
 		*vpp = dvp;
 	} else if (flags & ISDOTDOT) {
-		if ((retval = hfs_vget(hfsmp, dcp->c_parentcnid, &tvp, 0)))
+		/*
+		 * Directory hard links can have multiple parents so
+		 * find the appropriate parent for the current thread.
+		 */
+		if ((retval = hfs_vget(hfsmp, hfs_currentparent(VTOC(dvp)), &tvp, 0))) {
 			goto exit;
+		}
 		*cnode_locked = 1;
 		*vpp = tvp;
 	} else {
 		int type = (attr.ca_mode & S_IFMT);
+#if NAMEDRSRCFORK
+		int rsrc_warn = 0;
 
+		/*
+		 * Check if caller wants the resource fork but utilized
+		 * the legacy "file/rsrc" access path.
+		 *
+		 * This is deprecated behavior and support for it will not
+		 * be allowed beyond case insensitive HFS+ and even that
+		 * support will be removed in the next major OS release.
+		 */
+		if ((type == S_IFREG) &&
+		    ((flags & ISLASTCN) == 0) &&
+		    (cnp->cn_nameptr[cnp->cn_namelen] == '/') &&
+		    (bcmp(&cnp->cn_nameptr[cnp->cn_namelen+1], "rsrc", 5) == 0) &&
+		    ((hfsmp->hfs_flags & (HFS_STANDARD | HFS_CASE_SENSITIVE)) == 0)) {
+		
+			cnp->cn_consume = 5;
+			cnp->cn_flags |= CN_WANTSRSRCFORK | ISLASTCN | NOCACHE;
+			cnp->cn_flags &= ~MAKEENTRY;
+			flags |= ISLASTCN;
+			rsrc_warn = 1;
+		}
+#endif
 		if (!(flags & ISLASTCN) && (type != S_IFDIR) && (type != S_IFLNK)) {
 			retval = ENOTDIR;
 			goto exit;
 		}
-
+		/* Don't cache directory hardlink names. */
+		if (attr.ca_recflags & kHFSHasLinkChainMask) {
+			cnp->cn_flags &= ~MAKEENTRY;
+		}
 		/* Names with composed chars are not cached. */
 		if (cnp->cn_namelen != desc.cd_namelen)
 			cnp->cn_flags &= ~MAKEENTRY;
 
-		/* Resource fork vnode names include the fork specifier. */
-		if (wantrsrc && (flags & ISLASTCN))
-			cnp->cn_namelen += forknamelen;
+		retval = hfs_getnewvnode(hfsmp, dvp, cnp, &desc, 0, &attr, &fork, &tvp);
 
-		retval = hfs_getnewvnode(hfsmp, dvp, cnp, &desc, wantrsrc, &attr, &fork, &tvp);
-
-		if (wantrsrc && (flags & ISLASTCN))
-			cnp->cn_namelen -= forknamelen;
-
-		if (retval)
+		if (retval) {
+			/*
+			 * If this was a create operation lookup and another
+			 * process removed the object before we had a chance
+			 * to create the vnode, then just treat it as the not
+			 * found case above and return EJUSTRETURN.
+			 */
+			if ((retval == ENOENT) &&
+			    (cnp->cn_nameiop == CREATE) &&
+			    (flags & ISLASTCN)) {
+				retval = EJUSTRETURN;
+			}
 			goto exit;
+		}
+
+		/* Save the origin info of a directory link for future ".." requests. */
+		if (S_ISDIR(attr.ca_mode) && (attr.ca_recflags & kHFSHasLinkChainMask)) {
+			hfs_savelinkorigin(VTOC(tvp), VTOC(dvp)->c_fileid);
+		}
 		*cnode_locked = 1;
 		*vpp = tvp;
+#if NAMEDRSRCFORK
+		if (rsrc_warn) {
+			if ((VTOC(tvp)->c_flag & C_WARNED_RSRC) == 0) {
+				VTOC(tvp)->c_flag |= C_WARNED_RSRC;
+				printf("%.200s: file access by '/rsrc' was deprecated in 10.4\n",
+				       cnp->cn_nameptr);
+			}
+		}
+#endif
 	}
 exit:
+	if (dcp) {
+		hfs_unlock(dcp);
+	}
 	cat_releasedesc(&desc);
 	return (retval);
 }
@@ -393,9 +437,9 @@ hfs_vnop_lookup(struct vnop_lookup_args *ap)
 	 */
 	error = cache_lookup(dvp, vpp, cnp);
 	if (error != -1) {
-		if (error == ENOENT)		/* found a negative cache entry */
-			goto exit;
-		goto lookup;			/* did not find it in the cache */
+		if ((error == ENOENT) && (cnp->cn_nameiop != CREATE))		
+			goto exit;	/* found a negative cache entry */
+		goto lookup;		/* did not find it in the cache */
 	}
 	/*
 	 * We have a name that matched
@@ -414,7 +458,7 @@ hfs_vnop_lookup(struct vnop_lookup_args *ap)
 
 	if ((flags & ISLASTCN) && (cp->c_flag & C_HARDLINK)) {
 		hfs_lock(cp, HFS_FORCE_LOCK);
-		if ((cp->c_parentcnid != VTOC(dvp)->c_cnid) ||
+		if ((cp->c_parentcnid != dcp->c_cnid) ||
 		    (bcmp(cnp->cn_nameptr, cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen) != 0)) {
 			struct cat_desc desc;
 			int lockflags;
@@ -422,11 +466,13 @@ hfs_vnop_lookup(struct vnop_lookup_args *ap)
 			/*
 			 * Get an updated descriptor
 			 */
-			bzero(&desc, sizeof(desc));
-			desc.cd_nameptr = cnp->cn_nameptr;
+			desc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 			desc.cd_namelen = cnp->cn_namelen;
-			desc.cd_parentcnid = VTOC(dvp)->c_cnid;
-			desc.cd_hint = VTOC(dvp)->c_childhint;
+			desc.cd_parentcnid = dcp->c_fileid;
+			desc.cd_hint = dcp->c_childhint;
+			desc.cd_encoding = 0;
+			desc.cd_cnid = 0;
+			desc.cd_flags = S_ISDIR(cp->c_mode) ? CD_ISDIR : 0;
 	
 			lockflags = hfs_systemfile_lock(VTOHFS(dvp), SFL_CATALOG, HFS_SHARED_LOCK);		
 			if (cat_lookup(VTOHFS(vp), &desc, 0, &desc, NULL, NULL, NULL) == 0)
@@ -435,46 +481,33 @@ hfs_vnop_lookup(struct vnop_lookup_args *ap)
 		}
 		hfs_unlock(cp);
 	}
-	if (dvp != vp && !(flags & ISDOTDOT)) {
-		if ((flags & ISLASTCN) == 0 && vnode_isreg(vp)) {
-			int wantrsrc = 0;
-
-			cnp->cn_consume = forkcomponent(cnp, &wantrsrc);
-			if (cnp->cn_consume) {
-				flags |= ISLASTCN;
-				/* Fork names are only for lookups */
-				if (cnp->cn_nameiop != LOOKUP &&
-				    cnp->cn_nameiop != CREATE) {
-				        vnode_put(vp);
-					error = EPERM;
-					goto exit;
-				}
-			}
-			/*
-			 * Use cnode's rsrcfork vnode if possible.
-			 */
-			if (wantrsrc) {
-			        int	vid;
-
-			        *vpp = NULL;
-
-			        if (cp->c_rsrc_vp == NULL) {
-				        vnode_put(vp);
-				        goto lookup;
-				}
-				vid = vnode_vid(cp->c_rsrc_vp);
-
-				error = vnode_getwithvid(cp->c_rsrc_vp, vid);
-				if (error) {
-					vnode_put(vp);
-				        goto lookup;
-				}
-				*vpp = cp->c_rsrc_vp;
-				vnode_put(vp);
-				vp = *vpp;
-			}
+#if NAMEDRSRCFORK
+	/*
+	 * Check if caller wants the resource fork but utilized
+	 * the legacy "file/rsrc" access path.
+	 *
+	 * This is deprecated behavior and support for it will not
+	 * be allowed beyond case insensitive HFS+ and even that
+	 * support will be removed in the next major OS release.
+	 */
+	if ((dvp != vp) &&
+	    ((flags & ISLASTCN) == 0) &&
+	    vnode_isreg(vp) &&
+	    (cnp->cn_nameptr[cnp->cn_namelen] == '/') &&
+	    (bcmp(&cnp->cn_nameptr[cnp->cn_namelen+1], "rsrc", 5) == 0) &&
+	    ((VTOHFS(vp)->hfs_flags & (HFS_STANDARD | HFS_CASE_SENSITIVE)) == 0)) {		
+		cnp->cn_consume = 5;
+		cnp->cn_flags |= CN_WANTSRSRCFORK | ISLASTCN | NOCACHE;
+		cnp->cn_flags &= ~MAKEENTRY;
+
+		hfs_lock(cp, HFS_FORCE_LOCK);
+		if ((cp->c_flag & C_WARNED_RSRC) == 0) {
+			cp->c_flag |= C_WARNED_RSRC;
+			printf("%.200s: file access by '/rsrc' was deprecated in 10.4\n", cnp->cn_nameptr);
 		}
+		hfs_unlock(cp);
 	}
+#endif
 	return (error);
 	
 lookup:
@@ -485,7 +518,7 @@ lookup:
 	 */
 	cnode_locked = 0;
 
-	error = hfs_lookup(dvp, vpp, cnp, ap->a_context, &cnode_locked);
+	error = hfs_lookup(dvp, vpp, cnp, &cnode_locked);
 	
 	if (cnode_locked)
 		hfs_unlock(VTOC(*vpp));
@@ -494,39 +527,3 @@ exit:
 }
 
 
-/*
- * forkcomponent - look for a fork suffix in the component name
- *
- */
-static int
-forkcomponent(struct componentname *cnp, int *rsrcfork)
-{
-	char *suffix = cnp->cn_nameptr + cnp->cn_namelen;
-	int consume = 0;
-
-	*rsrcfork = 0;
-	if (*suffix == '\0')
-		return (0);
-	/*
-	 * There are only 3 valid fork suffixes:
-	 *	"/..namedfork/rsrc"
-	 *	"/..namedfork/data"
-	 *	"/rsrc"  (legacy)
-	 */
-	if (bcmp(suffix, _PATH_RSRCFORKSPEC, sizeof(_PATH_RSRCFORKSPEC)) == 0) {
-		consume = sizeof(_PATH_RSRCFORKSPEC) - 1;
-		*rsrcfork = 1;
-	} else if (bcmp(suffix, _PATH_DATAFORKSPEC, sizeof(_PATH_DATAFORKSPEC)) == 0) {
-		consume = sizeof(_PATH_DATAFORKSPEC) - 1;
-	}
-
-#if LEGACY_FORK_NAMES
-	else if (bcmp(suffix, LEGACY_RSRCFORKSPEC, sizeof(LEGACY_RSRCFORKSPEC)) == 0) {
-		consume = sizeof(LEGACY_RSRCFORKSPEC) - 1;
-		*rsrcfork = 1;
-		printf("HFS: /rsrc paths are deprecated (%s)\n", cnp->cn_nameptr);
-	}
-#endif
-	return (consume);
-}
-
diff --git a/bsd/hfs/hfs_macos_defs.h b/bsd/hfs/hfs_macos_defs.h
index 56a0f2296..7bc43b99f 100644
--- a/bsd/hfs/hfs_macos_defs.h
+++ b/bsd/hfs/hfs_macos_defs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __HFS_MACOS_TYPES__
@@ -29,11 +35,10 @@
 #ifdef __APPLE_API_PRIVATE
 
 #include <sys/param.h>
-	#ifdef KERNEL
-	#include <libkern/libkern.h>
-	#include <sys/systm.h>
-	#endif
 
+#include <libkern/OSTypes.h>
+#include <libkern/libkern.h>
+#include <sys/systm.h>
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/proc.h>
@@ -105,31 +110,16 @@
 	#define nil NULL
 #endif
 
-
-typedef u_int8_t 		UInt8;
-typedef int8_t		 	SInt8;
-typedef u_int16_t 		UInt16;
-typedef int16_t 		SInt16;
-typedef u_int32_t 		UInt32;
-typedef int32_t 		SInt32;
-typedef u_int64_t  		UInt64;
-typedef int64_t 		SInt64;
-
 typedef char *			Ptr;
 typedef long 			Size;
 
-typedef SInt16 			OSErr;
-typedef SInt32 			OSStatus;
-typedef UInt32 			ItemCount;
-typedef void *			LogicalAddress;
-typedef UInt32 			ByteCount;
-typedef UInt8 *			BytePtr;
-typedef UInt32 			ByteOffset;
-typedef UInt32 			OptionBits;
-typedef unsigned long 		FourCharCode;
-typedef FourCharCode 		OSType;
-
-typedef UInt16 			UniChar;
+typedef int16_t 			OSErr;
+typedef u_int32_t 			ItemCount;
+typedef u_int32_t 			ByteCount;
+typedef u_int8_t *			BytePtr;
+typedef u_int32_t 			ByteOffset;
+
+typedef u_int16_t 			UniChar;
 typedef unsigned char 		Str255[256];
 typedef unsigned char 		Str31[32];
 typedef unsigned char *		StringPtr;
@@ -137,10 +127,10 @@ typedef const unsigned char *	ConstStr255Param;
 typedef const unsigned char *	ConstStr31Param;
 typedef const unsigned char *	ConstUTF8Param;
 
-typedef UInt8 			Byte;
+typedef u_int8_t 			Byte;
 
-typedef UInt32 			TextEncoding;
-typedef UniChar *		UniCharArrayPtr;
+typedef u_int32_t 			TextEncoding;
+typedef UniChar *			UniCharArrayPtr;
 typedef const UniChar *		ConstUniCharArrayPtr;
 
 
@@ -168,10 +158,6 @@ enum {
 
 #endif  /*  !TYPE_BOOL */
 
-typedef unsigned char 				Boolean;
-
-
-
 
 EXTERN_API( void ) DebugStr(ConstStr255Param debuggerMsg);
 
@@ -298,7 +284,7 @@ BlockMoveData(const void * srcPtr, void * destPtr, Size byteCount);
 #define BlockMoveData(src, dest, len)	bcopy((src), (dest), (len))
 
 EXTERN_API_C( void )
-ClearMemory(void * start, UInt32 length);
+ClearMemory(void * start, u_int32_t length);
 
 #define ClearMemory(start, length)	bzero((start), (size_t)(length));
 
diff --git a/bsd/hfs/hfs_mount.h b/bsd/hfs/hfs_mount.h
index d09334757..5782bd6f6 100644
--- a/bsd/hfs/hfs_mount.h
+++ b/bsd/hfs/hfs_mount.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997-2002 Apple Computer, Inc. All Rights Reserved
@@ -50,7 +56,7 @@ struct hfs_mount_args {
 	uid_t	hfs_uid;		/* uid that owns hfs files (standard HFS only) */
 	gid_t	hfs_gid;		/* gid that owns hfs files (standard HFS only) */
 	mode_t	hfs_mask;		/* mask to be applied for hfs perms  (standard HFS only) */
-	uint32_t hfs_encoding;		/* encoding for this volume (standard HFS only) */
+	u_int32_t hfs_encoding;	/* encoding for this volume (standard HFS only) */
 	struct	timezone hfs_timezone;	/* user time zone info (standard HFS only) */
 	int		flags;			/* mounting flags, see below */
 	int     journal_tbuffer_size;   /* size in bytes of the journal transaction buffer */
@@ -72,6 +78,7 @@ struct hfs_mount_args {
 #define HFS_DISABLE_JOURNALING  0x031272
 #define HFS_GET_JOURNAL_INFO    0x6a6e6c69
 #define HFS_SET_PKG_EXTENSIONS  0x121031
+#define HFS_REPLAY_JOURNAL	0x6a6e6c72
 
 #endif /* __APPLE_API_UNSTABLE */
 
diff --git a/bsd/hfs/hfs_notification.c b/bsd/hfs/hfs_notification.c
index 60e96c5b1..5ea5825a6 100644
--- a/bsd/hfs/hfs_notification.c
+++ b/bsd/hfs/hfs_notification.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (C) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (C) 2003, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/systm.h>
@@ -46,8 +52,8 @@
 
 
 
-void hfs_generate_volume_notifications(struct hfsmount *hfsmp) {
-	ExtendedVCB *vcb = HFSTOVCB(hfsmp);
+void hfs_generate_volume_notifications(struct hfsmount *hfsmp) 
+{
 	fsid_t fsid;
 		
 	fsid.val[0] = (long)hfsmp->hfs_raw_dev;
@@ -57,13 +63,13 @@ void hfs_generate_volume_notifications(struct hfsmount *hfsmp) {
 		/* Check to see whether the free space is back above the minimal level: */
 		if (hfs_freeblks(hfsmp, 1) > hfsmp->hfs_freespace_notify_desiredlevel) {
             hfsmp->hfs_notification_conditions &= ~VQ_LOWDISK;
-            vfs_event_signal(&fsid, hfsmp->hfs_notification_conditions, NULL);
+            vfs_event_signal(&fsid, hfsmp->hfs_notification_conditions, (intptr_t)NULL);
 		}
 	} else {
 		/* Check to see whether the free space fell below the requested limit: */
 		if (hfs_freeblks(hfsmp, 1) < hfsmp->hfs_freespace_notify_warninglimit) {
             hfsmp->hfs_notification_conditions |= VQ_LOWDISK;
-            vfs_event_signal(&fsid, hfsmp->hfs_notification_conditions, NULL);
+            vfs_event_signal(&fsid, hfsmp->hfs_notification_conditions, (intptr_t)NULL);
 		}
 	};
 }
diff --git a/bsd/hfs/hfs_quota.c b/bsd/hfs/hfs_quota.c
index 5a1a459c4..56bc0e4dc 100644
--- a/bsd/hfs/hfs_quota.c
+++ b/bsd/hfs/hfs_quota.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993, 1995
@@ -67,6 +73,7 @@
 #include <sys/proc.h>
 #include <sys/kauth.h>
 #include <sys/vnode.h>
+#include <sys/vnode_internal.h>
 #include <sys/quota.h>
 #include <sys/proc_internal.h>
 #include <kern/kalloc.h>
@@ -76,6 +83,7 @@
 #include <hfs/hfs_quota.h>
 #include <hfs/hfs_mount.h>
 
+
 /*
  * Quota name to error message mapping.
  */
@@ -351,7 +359,7 @@ hfs_chkiqchg(cp, change, cred, type)
 	int type;
 {
 	register struct dquot *dq = cp->c_dquot[type];
-	long ncurinodes;
+	unsigned long ncurinodes;
 	struct vnode *vp = cp->c_vp ? cp->c_vp : cp->c_rsrc_vp;
 
 	dqlock(dq);
@@ -475,6 +483,9 @@ hfs_quotaon(p, mp, type, fnamep)
 	int error = 0;
 	struct hfs_quotaon_cargs args;
 
+	/* Finish setting up quota structures. */
+	dqhashinit();
+
 	qfp = &hfsmp->hfs_qfiles[type];
 
 	if ( (qf_get(qfp, QTF_OPENING)) )
@@ -489,7 +500,10 @@ hfs_quotaon(p, mp, type, fnamep)
 		error = EACCES;
 		goto out;
 	}
-	vfs_setflags(mp, (uint64_t)((unsigned int)MNT_QUOTA));
+	vfs_setflags(mp, (u_int64_t)((unsigned int)MNT_QUOTA));
+	HFS_MOUNT_LOCK(hfsmp, TRUE)
+	hfsmp->hfs_flags |= HFS_QUOTAS;
+	HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 	vnode_setnoflush(vp);
 	/*
 	 * Save the credential of the process that turned on quotas.
@@ -573,6 +587,12 @@ hfs_quotaoff(__unused struct proc *p, struct mount *mp, register int type)
 	int error;
 	struct hfs_quotaoff_cargs args;
 
+	/*
+	 * If quotas haven't been initialized, there's no work to be done.
+	 */
+	if (!dqisinitialized())
+		return (0);
+
 	qfp = &hfsmp->hfs_qfiles[type];
 	
 	if ( (qf_get(qfp, QTF_CLOSING)) )
@@ -611,8 +631,12 @@ hfs_quotaoff(__unused struct proc *p, struct mount *mp, register int type)
 	for (type = 0; type < MAXQUOTAS; type++)
 		if (hfsmp->hfs_qfiles[type].qf_vp != NULLVP)
 			break;
-	if (type == MAXQUOTAS)
-		vfs_clearflags(mp, (uint64_t)((unsigned int)MNT_QUOTA));
+	if (type == MAXQUOTAS) {
+		vfs_clearflags(mp, (u_int64_t)((unsigned int)MNT_QUOTA));
+		HFS_MOUNT_LOCK(hfsmp, TRUE)
+		hfsmp->hfs_flags &= ~HFS_QUOTAS;
+		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
+	}
 
 	qf_put(qfp, QTF_CLOSING);
 
@@ -784,6 +808,9 @@ hfs_qsync(mp)
 	struct hfsmount *hfsmp = VFSTOHFS(mp);
 	int i;
 
+	if (!dqisinitialized())
+		return (0);
+
 	/*
 	 * Check if the mount point has any quotas.
 	 * If not, simply return.
diff --git a/bsd/hfs/hfs_quota.h b/bsd/hfs/hfs_quota.h
index bde8fc5cd..c36025242 100644
--- a/bsd/hfs/hfs_quota.h
+++ b/bsd/hfs/hfs_quota.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -72,12 +78,16 @@
 struct cnode;
 struct mount;
 struct proc;
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
 struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
 __BEGIN_DECLS
-int	hfs_chkdq(struct cnode *, int64_t, struct ucred *, int);
-int	hfs_chkdqchg(struct cnode *, int64_t, struct ucred *, int);
-int	hfs_chkiq(struct cnode *, long, struct ucred *, int);
-int	hfs_chkiqchg(struct cnode *, long, struct ucred *, int);
+int	hfs_chkdq(struct cnode *, int64_t, kauth_cred_t, int);
+int	hfs_chkdqchg(struct cnode *, int64_t, kauth_cred_t, int);
+int	hfs_chkiq(struct cnode *, long, kauth_cred_t, int);
+int	hfs_chkiqchg(struct cnode *, long, kauth_cred_t, int);
 int	hfs_getinoquota(struct cnode *);
 int	hfs_getquota(struct mount *, u_long, int, caddr_t);
 int	hfs_qsync(struct mount *mp);
diff --git a/bsd/hfs/hfs_readwrite.c b/bsd/hfs/hfs_readwrite.c
index c3a2de87b..daf533c2e 100644
--- a/bsd/hfs/hfs_readwrite.c
+++ b/bsd/hfs/hfs_readwrite.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)hfs_readwrite.c	1.0
  *
@@ -38,69 +44,51 @@
 #include <sys/proc.h>
 #include <sys/kauth.h>
 #include <sys/vnode.h>
+#include <sys/vnode_internal.h>
 #include <sys/uio.h>
 #include <sys/vfs_context.h>
+#include <sys/fsevents.h>
+#include <kern/kalloc.h>
 #include <sys/disk.h>
 #include <sys/sysctl.h>
 
 #include <miscfs/specfs/specdev.h>
 
 #include <sys/ubc.h>
+#include <sys/ubc_internal.h>
+
 #include <vm/vm_pageout.h>
 #include <vm/vm_kern.h>
 
 #include <sys/kdebug.h>
 
 #include	"hfs.h"
+#include	"hfs_attrlist.h"
 #include	"hfs_endian.h"
-#include  "hfs_fsctl.h"
+#include  	"hfs_fsctl.h"
 #include	"hfs_quota.h"
 #include	"hfscommon/headers/FileMgrInternal.h"
 #include	"hfscommon/headers/BTreesInternal.h"
 #include	"hfs_cnode.h"
 #include	"hfs_dbg.h"
 
-extern int overflow_extents(struct filefork *fp);
-
 #define can_cluster(size) ((((size & (4096-1))) == 0) && (size <= (MAXPHYSIO/2)))
 
 enum {
 	MAXHFSFILESIZE = 0x7FFFFFFF		/* this needs to go in the mount structure */
 };
 
-extern u_int32_t GetLogicalBlockSize(struct vnode *vp);
-
-extern int  hfs_setextendedsecurity(struct hfsmount *, int);
-
+/* from bsd/vfs/vfs_cluster.c */
+extern int is_file_clean(vnode_t vp, off_t filesize);
 
 static int  hfs_clonelink(struct vnode *, int, kauth_cred_t, struct proc *);
 static int  hfs_clonefile(struct vnode *, int, int, int);
 static int  hfs_clonesysfile(struct vnode *, int, int, int, kauth_cred_t, struct proc *);
 
-
 int flush_cache_on_write = 0;
 SYSCTL_INT (_kern, OID_AUTO, flush_cache_on_write, CTLFLAG_RW, &flush_cache_on_write, 0, "always flush the drive cache on writes to uncached files");
 
 
-/*****************************************************************************
-*
-*	I/O Operations on vnodes
-*
-*****************************************************************************/
-int  hfs_vnop_read(struct vnop_read_args *);
-int  hfs_vnop_write(struct vnop_write_args *);
-int  hfs_vnop_ioctl(struct vnop_ioctl_args *);
-int  hfs_vnop_select(struct vnop_select_args *);
-int  hfs_vnop_blktooff(struct vnop_blktooff_args *);
-int  hfs_vnop_offtoblk(struct vnop_offtoblk_args *);
-int  hfs_vnop_blockmap(struct vnop_blockmap_args *);
-int  hfs_vnop_strategy(struct vnop_strategy_args *);
-int  hfs_vnop_allocate(struct vnop_allocate_args *);
-int  hfs_vnop_pagein(struct vnop_pagein_args *);
-int  hfs_vnop_pageout(struct vnop_pageout_args *);
-int  hfs_vnop_bwrite(struct vnop_bwrite_args *);
-
-
 /*
  * Read data from a file.
  */
@@ -152,7 +140,7 @@ hfs_vnop_read(struct vnop_read_args *ap)
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 12)) | DBG_FUNC_START,
 		(int)uio_offset(uio), uio_resid(uio), (int)filesize, (int)filebytes, 0);
 
-	retval = cluster_read(vp, uio, filesize, 0);
+	retval = cluster_read(vp, uio, filesize, ap->a_ioflag);
 
 	cp->c_touch_acctime = TRUE;
 
@@ -162,7 +150,7 @@ hfs_vnop_read(struct vnop_read_args *ap)
 	/*
 	 * Keep track blocks read
 	 */
-	if (VTOHFS(vp)->hfc_stage == HFC_RECORDING && retval == 0) {
+	if (hfsmp->hfc_stage == HFC_RECORDING && retval == 0) {
 		int took_cnode_lock = 0;
 		off_t bytesread;
 
@@ -177,7 +165,7 @@ hfs_vnop_read(struct vnop_read_args *ap)
 		 * If this file hasn't been seen since the start of
 		 * the current sampling period then start over.
 		 */
-		if (cp->c_atime < VTOHFS(vp)->hfc_timebase) {
+		if (cp->c_atime < hfsmp->hfc_timebase) {
 			struct timeval tv;
 
 			fp->ff_bytesread = bytesread;
@@ -190,7 +178,7 @@ hfs_vnop_read(struct vnop_read_args *ap)
 			hfs_unlock(cp);
 	}
 exit:
-	hfs_unlock_truncate(cp);
+	hfs_unlock_truncate(cp, 0);
 	return (retval);
 }
 
@@ -208,7 +196,7 @@ hfs_vnop_write(struct vnop_write_args *ap)
 	kauth_cred_t cred = NULL;
 	off_t origFileSize;
 	off_t writelimit;
-	off_t bytesToAdd;
+	off_t bytesToAdd = 0;
 	off_t actualBytesAdded;
 	off_t filebytes;
 	off_t offset;
@@ -218,11 +206,17 @@ hfs_vnop_write(struct vnop_write_args *ap)
 	int retval = 0;
 	int lockflags;
 	int cnode_locked = 0;
+	int partialwrite = 0;
+	int exclusive_lock = 0;
 
 	// LP64todo - fix this! uio_resid may be 64-bit value
 	resid = uio_resid(uio);
 	offset = uio_offset(uio);
 
+	if (ioflag & IO_APPEND) {
+	    exclusive_lock = 1;
+	}
+	
 	if (offset < 0)
 		return (EINVAL);
 	if (resid == 0)
@@ -230,31 +224,11 @@ hfs_vnop_write(struct vnop_write_args *ap)
 	if (!vnode_isreg(vp))
 		return (EPERM);  /* Can only write regular files */
 
-	/* Protect against a size change. */
-	hfs_lock_truncate(VTOC(vp), TRUE);
-
-	if ( (retval = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
-		hfs_unlock_truncate(VTOC(vp));
-		return (retval);
-	}
-	cnode_locked = 1;
 	cp = VTOC(vp);
 	fp = VTOF(vp);
 	hfsmp = VTOHFS(vp);
-	filebytes = (off_t)fp->ff_blocks * (off_t)hfsmp->blockSize;
 
-	if (ioflag & IO_APPEND) {
-		uio_setoffset(uio, fp->ff_size);
-		offset = fp->ff_size;
-	}
-	if ((cp->c_flags & APPEND) && offset != fp->ff_size) {
-		retval = EPERM;
-		goto exit;
-	}
-
-	origFileSize = fp->ff_size;
 	eflags = kEFDeferMask;	/* defer file block allocations */
-
 #ifdef HFS_SPARSE_DEV
 	/* 
 	 * When the underlying device is sparse and space
@@ -268,19 +242,57 @@ hfs_vnop_write(struct vnop_write_args *ap)
 	}
 #endif /* HFS_SPARSE_DEV */
 
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 0)) | DBG_FUNC_START,
-		(int)offset, uio_resid(uio), (int)fp->ff_size, (int)filebytes, 0);
+again:
+	/* Protect against a size change. */
+	hfs_lock_truncate(cp, exclusive_lock);
 
-	/* Now test if we need to extend the file */
-	/* Doing so will adjust the filebytes for us */
+	if (ioflag & IO_APPEND) {
+		uio_setoffset(uio, fp->ff_size);
+		offset = fp->ff_size;
+	}
+	if ((cp->c_flags & APPEND) && offset != fp->ff_size) {
+		retval = EPERM;
+		goto exit;
+	}
 
+	origFileSize = fp->ff_size;
 	writelimit = offset + resid;
-	if (writelimit <= filebytes)
+	filebytes = (off_t)fp->ff_blocks * (off_t)hfsmp->blockSize;
+
+	/* If the truncate lock is shared, and if we either have virtual 
+	 * blocks or will need to extend the file, upgrade the truncate 
+	 * to exclusive lock.  If upgrade fails, we lose the lock and 
+	 * have to get exclusive lock again 
+	 */
+	if ((exclusive_lock == 0) && 
+	    ((fp->ff_unallocblocks != 0) || (writelimit > filebytes))) {
+	    	exclusive_lock = 1;
+		/* Lock upgrade failed and we lost our shared lock, try again */
+		if (lck_rw_lock_shared_to_exclusive(&cp->c_truncatelock) == FALSE) {
+			goto again;
+		} 
+	}
+
+	if ( (retval = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+		goto exit;
+	}
+	cnode_locked = 1;
+	
+	if (!exclusive_lock) {
+		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 0)) | DBG_FUNC_START,
+		             (int)offset, uio_resid(uio), (int)fp->ff_size,
+		             (int)filebytes, 0);
+	}
+
+	/* Check if we do not need to extend the file */
+	if (writelimit <= filebytes) {
 		goto sizeok;
+	}
 
 	cred = vfs_context_ucred(ap->a_context);
-#if QUOTA
 	bytesToAdd = writelimit - filebytes;
+
+#if QUOTA
 	retval = hfs_chkdq(cp, (int64_t)(roundup(bytesToAdd, hfsmp->blockSize)), 
 			   cred, 0);
 	if (retval)
@@ -324,6 +336,17 @@ hfs_vnop_write(struct vnop_write_args *ap)
 	(void) hfs_volupdate(hfsmp, VOL_UPDATE, 0);
 	(void) hfs_end_transaction(hfsmp);
 
+	/*
+	 * If we didn't grow the file enough try a partial write.
+	 * POSIX expects this behavior.
+	 */
+	if ((retval == ENOSPC) && (filebytes > offset)) {
+		retval = 0;
+		partialwrite = 1;
+		uio_setresid(uio, (uio_resid(uio) - bytesToAdd));
+		resid -= bytesToAdd;
+		writelimit = filebytes;
+	}
 sizeok:
 	if (retval == E_NONE) {
 		off_t filesize;
@@ -340,7 +363,7 @@ sizeok:
 		else
 			filesize = fp->ff_size;
 
-		lflag = (ioflag & IO_SYNC);
+		lflag = ioflag & ~(IO_TAILZEROFILL | IO_HEADZEROFILL | IO_NOZEROVALID | IO_NOZERODIRTY);
 
 		if (offset <= fp->ff_size) {
 			zero_off = offset & ~PAGE_MASK_64;
@@ -456,6 +479,9 @@ sizeok:
 		cnode_locked = 0;
 		retval = cluster_write(vp, uio, fp->ff_size, filesize, zero_off,
 				tail_off, lflag | IO_NOZERODIRTY);
+		if (retval) {
+			goto ioerr_exit;
+		}
 		offset = uio_offset(uio);
 		if (offset > fp->ff_size) {
 			fp->ff_size = offset;
@@ -470,8 +496,12 @@ sizeok:
 			cp->c_touch_modtime = TRUE;
 		}
 	}
+	if (partialwrite) {
+		uio_setresid(uio, (uio_resid(uio) + bytesToAdd));
+		resid += bytesToAdd;
+	}
 
-	// XXXdbg - testing for vivek and paul lambert
+	// XXXdbg - see radar 4871353 for more info
 	{
 	    if (flush_cache_on_write && ((ioflag & IO_NOCACHE) || vnode_isnocache(vp))) {
 		VNOP_IOCTL(hfsmp->hfs_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, NULL);
@@ -523,31 +553,22 @@ ioerr_exit:
 exit:
 	if (cnode_locked)
 		hfs_unlock(cp);
-	hfs_unlock_truncate(cp);
+	hfs_unlock_truncate(cp, exclusive_lock);
 	return (retval);
 }
 
 /* support for the "bulk-access" fcntl */
 
-#define CACHE_ELEMS 64
 #define CACHE_LEVELS 16
+#define NUM_CACHE_ENTRIES (64*16)
 #define PARENT_IDS_FLAG 0x100
 
-/* from hfs_attrlist.c */
-extern unsigned long DerivePermissionSummary(uid_t obj_uid, gid_t obj_gid,
-			mode_t obj_mode, struct mount *mp,
-			kauth_cred_t cred, struct proc *p);
-
-/* from vfs/vfs_fsevents.c */
-extern char *get_pathbuff(void);
-extern void release_pathbuff(char *buff);
-
 struct access_cache {
        int numcached;
        int cachehits; /* these two for statistics gathering */
        int lookups;
        unsigned int *acache;
-       Boolean *haveaccess;
+       unsigned char *haveaccess;
 };
 
 struct access_t {
@@ -564,74 +585,115 @@ struct user_access_t {
 	uid_t		uid;			/* IN: effective user id */
 	short		flags;			/* IN: access requested (i.e. R_OK) */
 	short		num_groups;		/* IN: number of groups user belongs to */
-	int			num_files;		/* IN: number of files to process */
+	int		num_files;		/* IN: number of files to process */
 	user_addr_t	file_ids;		/* IN: array of file ids */
 	user_addr_t	groups;			/* IN: array of groups */
 	user_addr_t	access;			/* OUT: access info for each file (0 for 'has access') */
 };
 
+
+// these are the "extended" versions of the above structures
+// note that it is crucial that they be different sized than
+// the regular version
+struct ext_access_t {
+	uint32_t   flags;           /* IN: access requested (i.e. R_OK) */
+	uint32_t   num_files;       /* IN: number of files to process */
+	uint32_t   map_size;        /* IN: size of the bit map */
+	uint32_t  *file_ids;        /* IN: Array of file ids */
+	char      *bitmap;          /* OUT: hash-bitmap of interesting directory ids */
+	short     *access;          /* OUT: access info for each file (0 for 'has access') */
+	uint32_t   num_parents;   /* future use */
+	cnid_t      *parents;   /* future use */
+};
+
+struct ext_user_access_t {
+	uint32_t      flags;        /* IN: access requested (i.e. R_OK) */
+	uint32_t      num_files;    /* IN: number of files to process */
+	uint32_t      map_size;     /* IN: size of the bit map */
+	user_addr_t   file_ids;     /* IN: array of file ids */
+	user_addr_t   bitmap;       /* IN: array of groups */
+	user_addr_t   access;       /* OUT: access info for each file (0 for 'has access') */
+	uint32_t      num_parents;/* future use */
+	user_addr_t   parents;/* future use */
+};
+
+
 /*
  * Perform a binary search for the given parent_id. Return value is 
- * found/not found boolean, and indexp will be the index of the item 
- * or the index at which to insert the item if it's not found.
+ * the index if there is a match.  If no_match_indexp is non-NULL it
+ * will be assigned with the index to insert the item (even if it was
+ * not found).
  */
-static int
-lookup_bucket(struct access_cache *cache, int *indexp, cnid_t parent_id)
+static int cache_binSearch(cnid_t *array, unsigned int hi, cnid_t parent_id, int *no_match_indexp)
 {
-	unsigned int lo, hi;
-	int index, matches = 0;
+    int index=-1;
+    unsigned int lo=0;
 	
-	if (cache->numcached == 0) {
-		*indexp = 0;
-		return 0; // table is empty, so insert at index=0 and report no match
+    do {
+	unsigned int mid = ((hi - lo)/2) + lo;
+	unsigned int this_id = array[mid];
+		
+	if (parent_id == this_id) {
+	    hi = mid;
+	    break;
 	}
-	
-	if (cache->numcached > CACHE_ELEMS) {
-		/*printf("EGAD! numcached is %d... cut our losses and trim to %d\n",
-		  cache->numcached, CACHE_ELEMS);*/
-		cache->numcached = CACHE_ELEMS;
+		
+	if (parent_id < this_id) {
+	    hi = mid;
+	    continue;
 	}
+		
+	if (parent_id > this_id) {
+	    lo = mid + 1;
+	    continue;
+	}
+    } while(lo < hi);
+
+    /* check if lo and hi converged on the match */
+    if (parent_id == array[hi]) {
+	index = hi;
+    }
 	
-	lo = 0;
-	hi = cache->numcached - 1;
-	index = -1;
+    if (no_match_indexp) {
+	*no_match_indexp = hi;
+    }
+
+    return index;
+}
+ 
+ 
+static int
+lookup_bucket(struct access_cache *cache, int *indexp, cnid_t parent_id)
+{
+    unsigned int hi;
+    int matches = 0;
+    int index, no_match_index;
 	
-	/* perform binary search for parent_id */
-	do {
-		unsigned int mid = (hi - lo)/2 + lo;
-		unsigned int this_id = cache->acache[mid];
-		
-		if (parent_id == this_id) {
-			index = mid;
-			break;
-		}
-		
-		if (parent_id < this_id) {
-			hi = mid;
-			continue;
-		}
-		
-		if (parent_id > this_id) {
-			lo = mid + 1;
-			continue;
-		}
-	} while(lo < hi);
+    if (cache->numcached == 0) {
+	*indexp = 0;
+	return 0; // table is empty, so insert at index=0 and report no match
+    }
 	
-	/* check if lo and hi converged on the match */
-	if (parent_id == cache->acache[hi]) {
-		index = hi;
-	}
+    if (cache->numcached > NUM_CACHE_ENTRIES) {
+	/*printf("EGAD! numcached is %d... cut our losses and trim to %d\n",
+	  cache->numcached, NUM_CACHE_ENTRIES);*/
+	cache->numcached = NUM_CACHE_ENTRIES;
+    }
 	
-	/* if no existing entry found, find index for new one */
-	if (index == -1) {
-		index = (parent_id < cache->acache[hi]) ? hi : hi + 1;
-		matches = 0;
-	} else {
-		matches = 1;
-	}
+    hi = cache->numcached - 1;
 	
-	*indexp = index;
-	return matches;
+    index = cache_binSearch(cache->acache, hi, parent_id, &no_match_index);
+
+    /* if no existing entry found, find index for new one */
+    if (index == -1) {
+	index = no_match_index;
+	matches = 0;
+    } else {
+	matches = 1;
+    }
+	
+    *indexp = index;
+    return matches;
 }
 
 /*
@@ -642,63 +704,71 @@ lookup_bucket(struct access_cache *cache, int *indexp, cnid_t parent_id)
 static void
 add_node(struct access_cache *cache, int index, cnid_t nodeID, int access)
 {
-       int lookup_index = -1;
-
-       /* need to do a lookup first if -1 passed for index */
-       if (index == -1) {
-               if (lookup_bucket(cache, &lookup_index, nodeID)) {
-                       if (cache->haveaccess[lookup_index] != access) {
-                               /* change access info for existing entry... should never happen */
-			       cache->haveaccess[lookup_index] = access;
-                       }
-
-		       /* mission accomplished */
-                       return;
-               } else {
-                       index = lookup_index;
-               }
-
-       }
-
-       /* if the cache is full, do a replace rather than an insert */
-       if (cache->numcached >= CACHE_ELEMS) {
-               //printf("cache is full (%d). replace at index %d\n", cache->numcached, index);
-               cache->numcached = CACHE_ELEMS-1;
-
-               if (index > cache->numcached) {
-                 //    printf("index %d pinned to %d\n", index, cache->numcached);
-                       index = cache->numcached;
-               }
-       } else if (index >= 0 && index < cache->numcached) {
-               /* only do bcopy if we're inserting */
-               bcopy( cache->acache+index, cache->acache+(index+1), (cache->numcached - index)*sizeof(int) );
-               bcopy( cache->haveaccess+index, cache->haveaccess+(index+1), (cache->numcached - index)*sizeof(Boolean) );
-       }
-
-       cache->acache[index] = nodeID;
-       cache->haveaccess[index] = access;
-       cache->numcached++;
+    int lookup_index = -1;
+
+    /* need to do a lookup first if -1 passed for index */
+    if (index == -1) {
+	if (lookup_bucket(cache, &lookup_index, nodeID)) {
+	    if (cache->haveaccess[lookup_index] != access && cache->haveaccess[lookup_index] == ESRCH) {
+		// only update an entry if the previous access was ESRCH (i.e. a scope checking error)
+		cache->haveaccess[lookup_index] = access;
+	    }
+
+	    /* mission accomplished */
+	    return;
+	} else {
+	    index = lookup_index;
+	}
+
+    }
+
+    /* if the cache is full, do a replace rather than an insert */
+    if (cache->numcached >= NUM_CACHE_ENTRIES) {
+	//printf("cache is full (%d). replace at index %d\n", cache->numcached, index);
+	cache->numcached = NUM_CACHE_ENTRIES-1;
+
+	if (index > cache->numcached) {
+	    //    printf("index %d pinned to %d\n", index, cache->numcached);
+	    index = cache->numcached;
+	}
+    }
+
+    if (index < cache->numcached && index < NUM_CACHE_ENTRIES && nodeID > cache->acache[index]) {
+	index++;
+    }
+
+    if (index >= 0 && index < cache->numcached) {
+	/* only do bcopy if we're inserting */
+	bcopy( cache->acache+index, cache->acache+(index+1), (cache->numcached - index)*sizeof(int) );
+	bcopy( cache->haveaccess+index, cache->haveaccess+(index+1), (cache->numcached - index)*sizeof(unsigned char) );
+    }
+
+    cache->acache[index] = nodeID;
+    cache->haveaccess[index] = access;
+    cache->numcached++;
 }
 
 
 struct cinfo {
-	uid_t   uid;
-	gid_t   gid;
-	mode_t  mode;
-	cnid_t  parentcnid;
+    uid_t   uid;
+    gid_t   gid;
+    mode_t  mode;
+    cnid_t  parentcnid;
+    u_int16_t recflags;
 };
 
 static int
 snoop_callback(const struct cat_desc *descp, const struct cat_attr *attrp, void * arg)
 {
-	struct cinfo *cip = (struct cinfo *)arg;
+    struct cinfo *cip = (struct cinfo *)arg;
 
-	cip->uid = attrp->ca_uid;
-	cip->gid = attrp->ca_gid;
-	cip->mode = attrp->ca_mode;
-	cip->parentcnid = descp->cd_parentcnid;
+    cip->uid = attrp->ca_uid;
+    cip->gid = attrp->ca_gid;
+    cip->mode = attrp->ca_mode;
+    cip->parentcnid = descp->cd_parentcnid;
+    cip->recflags = attrp->ca_recflags;
 	
-	return (0);
+    return (0);
 }
 
 /*
@@ -707,134 +777,506 @@ snoop_callback(const struct cat_desc *descp, const struct cat_attr *attrp, void
  */ 
 static int
 do_attr_lookup(struct hfsmount *hfsmp, struct access_cache *cache, dev_t dev, cnid_t cnid, 
-	       struct cnode *skip_cp, CatalogKey *keyp, struct cat_attr *cnattrp, struct proc *p)
+    struct cnode *skip_cp, CatalogKey *keyp, struct cat_attr *cnattrp)
 {
-	int error = 0;
-
-	/* if this id matches the one the fsctl was called with, skip the lookup */
-	if (cnid == skip_cp->c_cnid) {
-		cnattrp->ca_uid = skip_cp->c_uid;
-		cnattrp->ca_gid = skip_cp->c_gid;
-		cnattrp->ca_mode = skip_cp->c_mode;
-		keyp->hfsPlus.parentID = skip_cp->c_parentcnid;
+    int error = 0;
+
+    /* if this id matches the one the fsctl was called with, skip the lookup */
+    if (cnid == skip_cp->c_cnid) {
+	cnattrp->ca_uid = skip_cp->c_uid;
+	cnattrp->ca_gid = skip_cp->c_gid;
+	cnattrp->ca_mode = skip_cp->c_mode;
+	keyp->hfsPlus.parentID = skip_cp->c_parentcnid;
+    } else {
+	struct cinfo c_info;
+
+	/* otherwise, check the cnode hash incase the file/dir is incore */
+	if (hfs_chash_snoop(dev, cnid, snoop_callback, &c_info) == 0) {
+	    cnattrp->ca_uid = c_info.uid;
+	    cnattrp->ca_gid = c_info.gid;
+	    cnattrp->ca_mode = c_info.mode;
+	    cnattrp->ca_recflags = c_info.recflags;
+	    keyp->hfsPlus.parentID = c_info.parentcnid;
 	} else {
-		struct cinfo c_info;
-
-		/* otherwise, check the cnode hash incase the file/dir is incore */
-		if (hfs_chash_snoop(dev, cnid, snoop_callback, &c_info) == 0) {
-			cnattrp->ca_uid = c_info.uid;
-			cnattrp->ca_gid = c_info.gid;
-			cnattrp->ca_mode = c_info.mode;
-			keyp->hfsPlus.parentID = c_info.parentcnid;
-		} else {
-			int lockflags;
+	    int lockflags;
 			
-			lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+	    lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
 			
-			/* lookup this cnid in the catalog */
-			error = cat_getkeyplusattr(hfsmp, cnid, keyp, cnattrp);
+	    /* lookup this cnid in the catalog */
+	    error = cat_getkeyplusattr(hfsmp, cnid, keyp, cnattrp);
 			
-			hfs_systemfile_unlock(hfsmp, lockflags);
+	    hfs_systemfile_unlock(hfsmp, lockflags);
 			
-			cache->lookups++;
-		}
+	    cache->lookups++;
 	}
+    }
 	
-	return (error);
+    return (error);
 }
 
+
 /*
  * Compute whether we have access to the given directory (nodeID) and all its parents. Cache
  * up to CACHE_LEVELS as we progress towards the root.
  */
 static int 
 do_access_check(struct hfsmount *hfsmp, int *err, struct access_cache *cache, HFSCatalogNodeID nodeID, 
-		struct cnode *skip_cp, struct proc *theProcPtr, kauth_cred_t myp_ucred, dev_t dev )
+    struct cnode *skip_cp, struct proc *theProcPtr, kauth_cred_t myp_ucred, dev_t dev,
+    struct vfs_context *my_context,
+    char *bitmap,
+    uint32_t map_size,
+    cnid_t* parents,
+    uint32_t num_parents)
 {
-       int                     myErr = 0;
-       int                     myResult;
-       HFSCatalogNodeID        thisNodeID;
-       unsigned long           myPerms;
-       struct cat_attr         cnattr;
-       int                     cache_index = -1;
-       CatalogKey              catkey;
-
-       int i = 0, ids_to_cache = 0;
-       int parent_ids[CACHE_LEVELS];
-
-       /* root always has access */
-       if (!suser(myp_ucred, NULL)) {
-               return (1);
-       }
-
-       thisNodeID = nodeID;
-       while (thisNodeID >=  kRootDirID) {
-               myResult = 0;   /* default to "no access" */
-       
-               /* check the cache before resorting to hitting the catalog */
-
-               /* ASSUMPTION: access info of cached entries is "final"... i.e. no need
-                * to look any further after hitting cached dir */
-
-               if (lookup_bucket(cache, &cache_index, thisNodeID)) {
-                       cache->cachehits++;
-                       myResult = cache->haveaccess[cache_index];
-                       goto ExitThisRoutine;
-               }
-
-               /* remember which parents we want to cache */
-               if (ids_to_cache < CACHE_LEVELS) {
-                       parent_ids[ids_to_cache] = thisNodeID;
-                       ids_to_cache++;
-               }
+    int                     myErr = 0;
+    int                     myResult;
+    HFSCatalogNodeID        thisNodeID;
+    unsigned int            myPerms;
+    struct cat_attr         cnattr;
+    int                     cache_index = -1, scope_index = -1, scope_idx_start = -1;
+    CatalogKey              catkey;
+
+    int i = 0, ids_to_cache = 0;
+    int parent_ids[CACHE_LEVELS];
+
+    thisNodeID = nodeID;
+    while (thisNodeID >=  kRootDirID) {
+	myResult = 0;   /* default to "no access" */
 	       
-	       /* do the lookup (checks the cnode hash, then the catalog) */
-	       myErr = do_attr_lookup(hfsmp, cache, dev, thisNodeID, skip_cp, &catkey, &cnattr, theProcPtr);
-	       if (myErr) {
-		       goto ExitThisRoutine; /* no access */
-	       }
-
-               myPerms = DerivePermissionSummary(cnattr.ca_uid, cnattr.ca_gid,
-                                                 cnattr.ca_mode, hfsmp->hfs_mp,
-                                                 myp_ucred, theProcPtr);
-
-               if ( (myPerms & X_OK) == 0 ) {
-		       myResult = 0;
-                       goto ExitThisRoutine;   /* no access */
-	       } 
-
-               /* up the hierarchy we go */
-               thisNodeID = catkey.hfsPlus.parentID;
-       }
-
-       /* if here, we have access to this node */
-       myResult = 1;
-
- ExitThisRoutine:
-       if (myErr) {
-	       //printf("*** error %d from catalog looking up parent %d/%d!\n", myErr, dev, thisNodeID);
-               myResult = 0;
-       }
-       *err = myErr;
-
-       /* cache the parent directory(ies) */
-       for (i = 0; i < ids_to_cache; i++) {
-               /* small optimization: get rid of double-lookup for all these */
-	       // printf("adding %d to cache with result: %d\n", parent_ids[i], myResult);
-               add_node(cache, -1, parent_ids[i], myResult);
-       }
-
-       return (myResult);
+	/* check the cache before resorting to hitting the catalog */
+
+	/* ASSUMPTION: access info of cached entries is "final"... i.e. no need
+	 * to look any further after hitting cached dir */
+
+	if (lookup_bucket(cache, &cache_index, thisNodeID)) {
+	    cache->cachehits++;
+	    myErr = cache->haveaccess[cache_index];
+	    if (scope_index != -1) {
+		if (myErr == ESRCH) {
+		    myErr = 0;
+		}
+	    } else {
+		scope_index = 0;   // so we'll just use the cache result 
+		scope_idx_start = ids_to_cache;
+	    }
+	    myResult = (myErr == 0) ? 1 : 0;
+	    goto ExitThisRoutine;
+	}
+
+
+	if (parents) {
+	    int tmp;
+	    tmp = cache_binSearch(parents, num_parents-1, thisNodeID, NULL);
+	    if (scope_index == -1)
+		scope_index = tmp;
+	    if (tmp != -1 && scope_idx_start == -1 && ids_to_cache < CACHE_LEVELS) {
+		scope_idx_start = ids_to_cache;
+	    }
+	}	   
+
+	/* remember which parents we want to cache */
+	if (ids_to_cache < CACHE_LEVELS) {
+	    parent_ids[ids_to_cache] = thisNodeID;
+	    ids_to_cache++;
+	}
+	// Inefficient (using modulo) and we might want to use a hash function, not rely on the node id to be "nice"...
+	if (bitmap && map_size) {
+	    bitmap[(thisNodeID/8)%(map_size)]|=(1<<(thisNodeID&7));	       
+	}
+	       
+
+	/* do the lookup (checks the cnode hash, then the catalog) */
+	myErr = do_attr_lookup(hfsmp, cache, dev, thisNodeID, skip_cp, &catkey, &cnattr);
+	if (myErr) {
+	    goto ExitThisRoutine; /* no access */
+	}
+
+	/* Root always gets access. */
+	if (suser(myp_ucred, NULL) == 0) {
+		thisNodeID = catkey.hfsPlus.parentID;
+		myResult = 1;
+		continue;
+	}
+
+	// if the thing has acl's, do the full permission check
+	if ((cnattr.ca_recflags & kHFSHasSecurityMask) != 0) {
+	    struct vnode *vp;
+
+	    /* get the vnode for this cnid */
+	    myErr = hfs_vget(hfsmp, thisNodeID, &vp, 0);
+	    if ( myErr ) {
+		myResult = 0;
+		goto ExitThisRoutine;
+	    }
+
+	    thisNodeID = VTOC(vp)->c_parentcnid;
+
+	    hfs_unlock(VTOC(vp));
+
+	    if (vnode_vtype(vp) == VDIR) {
+		myErr = vnode_authorize(vp, NULL, (KAUTH_VNODE_SEARCH | KAUTH_VNODE_LIST_DIRECTORY), my_context);
+	    } else {
+		myErr = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, my_context);
+	    }
+
+	    vnode_put(vp);
+	    if (myErr) {
+		myResult = 0;
+		goto ExitThisRoutine;
+	    }
+	} else {
+	    unsigned int flags;
+		   
+	    myPerms = DerivePermissionSummary(cnattr.ca_uid, cnattr.ca_gid,
+		cnattr.ca_mode, hfsmp->hfs_mp,
+		myp_ucred, theProcPtr);
+
+	    if (cnattr.ca_mode & S_IFDIR) {
+		flags = R_OK | X_OK;
+	    } else {
+		flags = R_OK;
+	    }
+	    if ( (myPerms & flags) != flags) {
+		myResult = 0;
+		myErr = EACCES;
+		goto ExitThisRoutine;   /* no access */
+	    }
+
+	    /* up the hierarchy we go */
+	    thisNodeID = catkey.hfsPlus.parentID;
+	}
+    }
+
+    /* if here, we have access to this node */
+    myResult = 1;
+
+  ExitThisRoutine:
+    if (parents && myErr == 0 && scope_index == -1) {
+	myErr = ESRCH;
+    }
+				
+    if (myErr) {
+	myResult = 0;
+    }
+    *err = myErr;
+
+    /* cache the parent directory(ies) */
+    for (i = 0; i < ids_to_cache; i++) {
+	if (myErr == 0 && parents && (scope_idx_start == -1 || i > scope_idx_start)) {
+	    add_node(cache, -1, parent_ids[i], ESRCH);
+	} else {
+	    add_node(cache, -1, parent_ids[i], myErr);
+	}
+    }
+
+    return (myResult);
 }
-/* end "bulk-access" support */
 
+static int
+do_bulk_access_check(struct hfsmount *hfsmp, struct vnode *vp,
+    struct vnop_ioctl_args *ap, int arg_size, vfs_context_t context)
+{
+    boolean_t is64bit;
+
+    /*
+     * NOTE: on entry, the vnode is locked. Incase this vnode
+     * happens to be in our list of file_ids, we'll note it
+     * avoid calling hfs_chashget_nowait() on that id as that
+     * will cause a "locking against myself" panic.
+     */
+    Boolean check_leaf = true;
+		
+    struct ext_user_access_t *user_access_structp;
+    struct ext_user_access_t tmp_user_access;
+    struct access_cache cache;
+		
+    int error = 0;
+    unsigned int i;
+		
+    dev_t dev = VTOC(vp)->c_dev;
+		
+    short flags;
+    unsigned int num_files = 0;
+    int map_size = 0;
+    int num_parents = 0;
+    int *file_ids=NULL;
+    short *access=NULL;
+    char *bitmap=NULL;
+    cnid_t *parents=NULL;
+    int leaf_index;
+	
+    cnid_t cnid;
+    cnid_t prevParent_cnid = 0;
+    unsigned int myPerms;
+    short myaccess = 0;
+    struct cat_attr cnattr;
+    CatalogKey catkey;
+    struct cnode *skip_cp = VTOC(vp);
+    kauth_cred_t cred = vfs_context_ucred(context);
+    proc_t p = vfs_context_proc(context);
+
+    is64bit = proc_is64bit(p);
+
+    /* initialize the local cache and buffers */
+    cache.numcached = 0;
+    cache.cachehits = 0;
+    cache.lookups = 0;
+    cache.acache = NULL;
+    cache.haveaccess = NULL;
+		
+    /* struct copyin done during dispatch... need to copy file_id array separately */
+    if (ap->a_data == NULL) {
+	error = EINVAL;
+	goto err_exit_bulk_access;
+    }
+
+    if (is64bit) {
+	if (arg_size != sizeof(struct ext_user_access_t)) {
+	    error = EINVAL;
+	    goto err_exit_bulk_access;
+	}
+
+	user_access_structp = (struct ext_user_access_t *)ap->a_data;
+
+    } else if (arg_size == sizeof(struct access_t)) {
+	struct access_t *accessp = (struct access_t *)ap->a_data;
+
+	// convert an old style bulk-access struct to the new style
+	tmp_user_access.flags     = accessp->flags;
+	tmp_user_access.num_files = accessp->num_files;
+	tmp_user_access.map_size  = 0;
+	tmp_user_access.file_ids  = CAST_USER_ADDR_T(accessp->file_ids);
+	tmp_user_access.bitmap    = (user_addr_t)NULL;
+	tmp_user_access.access    = CAST_USER_ADDR_T(accessp->access);
+	tmp_user_access.num_parents = 0;
+	user_access_structp = &tmp_user_access;
+
+    } else if (arg_size == sizeof(struct ext_access_t)) {
+	struct ext_access_t *accessp = (struct ext_access_t *)ap->a_data;
+
+	// up-cast from a 32-bit version of the struct
+	tmp_user_access.flags     = accessp->flags;
+	tmp_user_access.num_files = accessp->num_files;
+	tmp_user_access.map_size  = accessp->map_size;
+	tmp_user_access.num_parents  = accessp->num_parents;
+
+	tmp_user_access.file_ids  = CAST_USER_ADDR_T(accessp->file_ids);
+	tmp_user_access.bitmap    = CAST_USER_ADDR_T(accessp->bitmap);
+	tmp_user_access.access    = CAST_USER_ADDR_T(accessp->access);
+	tmp_user_access.parents    = CAST_USER_ADDR_T(accessp->parents);
+
+	user_access_structp = &tmp_user_access;
+    } else {
+	error = EINVAL;
+	goto err_exit_bulk_access;
+    }
+		
+    map_size = user_access_structp->map_size;
+
+    num_files = user_access_structp->num_files;
+
+    num_parents= user_access_structp->num_parents;
+
+    if (num_files < 1) {
+	goto err_exit_bulk_access;
+    }
+    if (num_files > 1024) {
+	error = EINVAL;
+	goto err_exit_bulk_access;
+    }
+
+    if (num_parents > 1024) {
+	error = EINVAL;
+	goto err_exit_bulk_access;
+    }
+		
+    file_ids = (int *) kalloc(sizeof(int) * num_files);
+    access = (short *) kalloc(sizeof(short) * num_files);
+    if (map_size) {
+	bitmap = (char *) kalloc(sizeof(char) * map_size);
+    }
+
+    if (num_parents) {
+	parents = (cnid_t *) kalloc(sizeof(cnid_t) * num_parents);
+    }
+
+    cache.acache = (unsigned int *) kalloc(sizeof(int) * NUM_CACHE_ENTRIES);
+    cache.haveaccess = (unsigned char *) kalloc(sizeof(unsigned char) * NUM_CACHE_ENTRIES);
+		
+    if (file_ids == NULL || access == NULL || (map_size != 0 && bitmap == NULL) || cache.acache == NULL || cache.haveaccess == NULL) {
+	if (file_ids) {
+	    kfree(file_ids, sizeof(int) * num_files);
+	}
+	if (bitmap) {
+	    kfree(bitmap, sizeof(char) * map_size);
+	}
+	if (access) {
+	    kfree(access, sizeof(short) * num_files);
+	}
+	if (cache.acache) {
+	    kfree(cache.acache, sizeof(int) * NUM_CACHE_ENTRIES);
+	}
+	if (cache.haveaccess) {
+	    kfree(cache.haveaccess, sizeof(unsigned char) * NUM_CACHE_ENTRIES);
+	}
+	if (parents) {
+	    kfree(parents, sizeof(cnid_t) * num_parents);
+	}			
+	return ENOMEM;
+    }
+		
+    // make sure the bitmap is zero'ed out...
+    if (bitmap) {
+	bzero(bitmap, (sizeof(char) * map_size));
+    }
+
+    if ((error = copyin(user_access_structp->file_ids, (caddr_t)file_ids,
+		num_files * sizeof(int)))) {
+	goto err_exit_bulk_access;
+    }
+	
+    if (num_parents) {
+	if ((error = copyin(user_access_structp->parents, (caddr_t)parents,
+		    num_parents * sizeof(cnid_t)))) {
+	    goto err_exit_bulk_access;
+	}
+    }
+	
+    flags = user_access_structp->flags;
+    if ((flags & (F_OK | R_OK | W_OK | X_OK)) == 0) {
+	flags = R_OK;
+    }
+		
+    /* check if we've been passed leaf node ids or parent ids */
+    if (flags & PARENT_IDS_FLAG) {
+	check_leaf = false;
+    }
+		
+    /* Check access to each file_id passed in */
+    for (i = 0; i < num_files; i++) {
+	leaf_index=-1;
+	cnid = (cnid_t) file_ids[i];
+			
+	/* root always has access */
+	if ((!parents) && (!suser(cred, NULL))) {
+	    access[i] = 0;
+	    continue;
+	}
+			
+	if (check_leaf) {
+	    /* do the lookup (checks the cnode hash, then the catalog) */
+	    error = do_attr_lookup(hfsmp, &cache, dev, cnid, skip_cp, &catkey, &cnattr);
+	    if (error) {
+		access[i] = (short) error;
+		continue;
+	    }
+	    
+	    if (parents) {
+		// Check if the leaf matches one of the parent scopes
+		leaf_index = cache_binSearch(parents, num_parents-1, cnid, NULL);
+	    }
+
+	    // if the thing has acl's, do the full permission check
+	    if ((cnattr.ca_recflags & kHFSHasSecurityMask) != 0) {
+		struct vnode *cvp;
+		int myErr = 0;
+		/* get the vnode for this cnid */
+		myErr = hfs_vget(hfsmp, cnid, &cvp, 0);
+		if ( myErr ) {
+		    access[i] = myErr;
+		    continue;
+		}
+		
+		hfs_unlock(VTOC(cvp));
+		
+		if (vnode_vtype(cvp) == VDIR) {
+		    myErr = vnode_authorize(cvp, NULL, (KAUTH_VNODE_SEARCH | KAUTH_VNODE_LIST_DIRECTORY), context);
+		} else {
+		    myErr = vnode_authorize(cvp, NULL, KAUTH_VNODE_READ_DATA, context);
+		}
+		
+		vnode_put(cvp);
+		if (myErr) {
+		    access[i] = myErr;
+		    continue;
+		}
+	    } else {
+		/* before calling CheckAccess(), check the target file for read access */
+		myPerms = DerivePermissionSummary(cnattr.ca_uid, cnattr.ca_gid,
+		    cnattr.ca_mode, hfsmp->hfs_mp, cred, p);
+		
+		/* fail fast if no access */ 
+		if ((myPerms & flags) == 0) {
+		    access[i] = EACCES;
+		    continue;
+		}		  					
+	    }
+	} else {
+	    /* we were passed an array of parent ids */
+	    catkey.hfsPlus.parentID = cnid;
+	}
+			
+	/* if the last guy had the same parent and had access, we're done */
+	if (i > 0 && catkey.hfsPlus.parentID == prevParent_cnid && access[i-1] == 0) {
+	    cache.cachehits++;
+	    access[i] = 0;
+	    continue;
+	}
+			
+	myaccess = do_access_check(hfsmp, &error, &cache, catkey.hfsPlus.parentID, 
+	    skip_cp, p, cred, dev, context,bitmap, map_size, parents, num_parents);
+			
+	if (myaccess || (error == ESRCH && leaf_index != -1)) {
+	    access[i] = 0; // have access.. no errors to report
+	} else {
+	    access[i] = (error != 0 ? (short) error : EACCES);
+	}
+			
+	prevParent_cnid = catkey.hfsPlus.parentID;
+    }
+		
+    /* copyout the access array */
+    if ((error = copyout((caddr_t)access, user_access_structp->access, 
+		num_files * sizeof (short)))) {
+	goto err_exit_bulk_access;
+    }
+    if (map_size && bitmap) {
+	if ((error = copyout((caddr_t)bitmap, user_access_structp->bitmap, 
+		    map_size * sizeof (char)))) {
+	    goto err_exit_bulk_access;
+	}
+    }
+	
+		
+  err_exit_bulk_access:
+		
+    //printf("on exit (err %d), numfiles/numcached/cachehits/lookups is %d/%d/%d/%d\n", error, num_files, cache.numcached, cache.cachehits, cache.lookups);
+		
+    if (file_ids) 
+	kfree(file_ids, sizeof(int) * num_files);
+    if (parents) 
+	kfree(parents, sizeof(cnid_t) * num_parents);
+    if (bitmap) 
+	kfree(bitmap, sizeof(char) * map_size);
+    if (access)
+	kfree(access, sizeof(short) * num_files);
+    if (cache.acache)
+	kfree(cache.acache, sizeof(int) * NUM_CACHE_ENTRIES);
+    if (cache.haveaccess)
+	kfree(cache.haveaccess, sizeof(unsigned char) * NUM_CACHE_ENTRIES);
+		
+    return (error);
+}
+
+
+/* end "bulk-access" support */
 
 
 /*
  * Callback for use with freeze ioctl.
  */
 static int
-hfs_freezewrite_callback(struct vnode *vp, void *cargs)
+hfs_freezewrite_callback(struct vnode *vp, __unused void *cargs)
 {
 	vnode_waitforwrites(vp, 0, 0, 0, "hfs freeze");
 
@@ -865,6 +1307,69 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 	switch (ap->a_command) {
 
+	case HFS_GETPATH:
+	{
+		struct vnode *file_vp;
+		cnid_t  cnid;
+		int  outlen;
+		char *bufptr;
+		int error;
+
+		/* Caller must be owner of file system. */
+		vfsp = vfs_statfs(HFSTOVFS(hfsmp));
+		if (suser(cred, NULL) &&
+			kauth_cred_getuid(cred) != vfsp->f_owner) {
+			return (EACCES);
+		}
+		/* Target vnode must be file system's root. */
+		if (!vnode_isvroot(vp)) {
+			return (EINVAL);
+		}
+		bufptr = (char *)ap->a_data;
+		cnid = strtoul(bufptr, NULL, 10);
+
+		if ((error = hfs_vget(hfsmp, cnid, &file_vp, 1))) {
+			return (error);
+		}
+		error = build_path(file_vp, bufptr, sizeof(pathname_t), &outlen, 0, context);
+		vnode_put(file_vp);
+
+		return (error);
+	}
+
+	case HFS_PREV_LINK:
+	case HFS_NEXT_LINK:
+	{
+		cnid_t linkfileid;
+		cnid_t nextlinkid;
+		cnid_t prevlinkid;
+		int error;
+
+		/* Caller must be owner of file system. */
+		vfsp = vfs_statfs(HFSTOVFS(hfsmp));
+		if (suser(cred, NULL) &&
+			kauth_cred_getuid(cred) != vfsp->f_owner) {
+			return (EACCES);
+		}
+		/* Target vnode must be file system's root. */
+		if (!vnode_isvroot(vp)) {
+			return (EINVAL);
+		}
+		linkfileid = *(cnid_t *)ap->a_data;
+		if (linkfileid < kHFSFirstUserCatalogNodeID) {
+			return (EINVAL);
+		}
+		if ((error = hfs_lookuplink(hfsmp, linkfileid, &prevlinkid, &nextlinkid))) {
+			return (error);
+		}
+		if (ap->a_command == HFS_NEXT_LINK) {
+			*(cnid_t *)ap->a_data = nextlinkid;
+		} else {
+			*(cnid_t *)ap->a_data = prevlinkid;
+		}
+		return (0);
+	}
+
 	case HFS_RESIZE_PROGRESS: {
 
 		vfsp = vfs_statfs(HFSTOVFS(hfsmp));
@@ -877,6 +1382,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		}
 		return hfs_resize_progress(hfsmp, (u_int32_t *)ap->a_data);
 	}
+
 	case HFS_RESIZE_VOLUME: {
 		u_int64_t newsize;
 		u_int64_t cursize;
@@ -901,6 +1407,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		}
 	}
 	case HFS_CHANGE_NEXT_ALLOCATION: {
+		int error = 0;		/* Assume success */
 		u_int32_t location;
 
 		if (vnode_vfsisrdonly(vp)) {
@@ -914,17 +1421,30 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		if (!vnode_isvroot(vp)) {
 			return (EINVAL);
 		}
+		HFS_MOUNT_LOCK(hfsmp, TRUE);
 		location = *(u_int32_t *)ap->a_data;
-		if (location > hfsmp->totalBlocks - 1) {
-			return (EINVAL);
+		if ((location >= hfsmp->allocLimit) &&
+			(location != HFS_NO_UPDATE_NEXT_ALLOCATION)) {
+			error = EINVAL;
+			goto fail_change_next_allocation;
 		}
 		/* Return previous value. */
 		*(u_int32_t *)ap->a_data = hfsmp->nextAllocation;
-		HFS_MOUNT_LOCK(hfsmp, TRUE);
-		hfsmp->nextAllocation = location;
-		hfsmp->vcbFlags |= 0xFF00;
+		if (location == HFS_NO_UPDATE_NEXT_ALLOCATION) {
+			/* On magic value for location, set nextAllocation to next block
+			 * after metadata zone and set flag in mount structure to indicate 
+			 * that nextAllocation should not be updated again.
+			 */
+			HFS_UPDATE_NEXT_ALLOCATION(hfsmp, hfsmp->hfs_metazone_end + 1);
+			hfsmp->hfs_flags |= HFS_SKIP_UPDATE_NEXT_ALLOCATION; 
+		} else {
+			hfsmp->hfs_flags &= ~HFS_SKIP_UPDATE_NEXT_ALLOCATION; 
+			HFS_UPDATE_NEXT_ALLOCATION(hfsmp, location);
+		}
+		MarkVCBDirty(hfsmp);
+fail_change_next_allocation:
 		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-		return (0);
+		return (error);
 	}
 
 #ifdef HFS_SPARSE_DEV
@@ -978,6 +1498,8 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		hfsmp->hfs_sparsebandblks = bsdata->bandsize / HFSTOVCB(hfsmp)->blockSize;
 		hfsmp->hfs_sparsebandblks *= 4;
 
+		vfs_markdependency(hfsmp->hfs_mp);
+
 		(void)vnode_put(di_vp);
 		file_drop(bsdata->backingfd);
 		return (0);
@@ -1005,7 +1527,6 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 	case F_FREEZE_FS: {
 		struct mount *mp;
-		task_t task;
  
 		if (!is_suser())
 			return (EACCES);
@@ -1018,9 +1539,6 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 		lck_rw_lock_exclusive(&hfsmp->hfs_insync);
  
-		task = current_task();
-		task_working_set_disable(task);
-
 		// flush things before we get started to try and prevent
 		// dirty data from being paged out while we're frozen.
 		// note: can't do this after taking the lock as it will
@@ -1066,245 +1584,37 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		return (0);
 	}
 
-#define HFSIOC_BULKACCESS _IOW('h', 9, struct access_t)
-#define HFS_BULKACCESS_FSCTL IOCBASECMD(HFSIOC_BULKACCESS)
-
-	case HFS_BULKACCESS_FSCTL:
-	case HFS_BULKACCESS: {
-		/*
-		 * NOTE: on entry, the vnode is locked. Incase this vnode
-		 * happens to be in our list of file_ids, we'll note it
-		 * avoid calling hfs_chashget_nowait() on that id as that
-		 * will cause a "locking against myself" panic.
-		 */
-		Boolean check_leaf = true;
-		
-		struct user_access_t *user_access_structp;
-		struct user_access_t tmp_user_access_t;
-		struct access_cache cache;
-		
-		int error = 0, i;
-		
-		dev_t dev = VTOC(vp)->c_dev;
-		
-		short flags;
-		struct ucred myucred;
-		int num_files;
-		int *file_ids = NULL;
-		short *access = NULL;
-		
-		cnid_t cnid;
-		cnid_t prevParent_cnid = 0;
-		unsigned long myPerms;
-		short myaccess = 0;
-		struct cat_attr cnattr;
-		CatalogKey catkey;
-		struct cnode *skip_cp = VTOC(vp);
-		struct vfs_context	my_context;
-
-		/* set up front for common exit code */
-		my_context.vc_ucred = NOCRED;
-
-		/* first, return error if not run as root */
-		if (cred->cr_ruid != 0) {
-			return EPERM;
-		}
-		
-		/* initialize the local cache and buffers */
-		cache.numcached = 0;
-		cache.cachehits = 0;
-		cache.lookups = 0;
-		
-		file_ids = (int *) get_pathbuff();
-		access = (short *) get_pathbuff();
-		cache.acache = (int *) get_pathbuff();
-		cache.haveaccess = (Boolean *) get_pathbuff();
-		
-		if (file_ids == NULL || access == NULL || cache.acache == NULL || cache.haveaccess == NULL) {
-			release_pathbuff((char *) file_ids);
-			release_pathbuff((char *) access);
-			release_pathbuff((char *) cache.acache);
-			release_pathbuff((char *) cache.haveaccess);
-			
-			return ENOMEM;
-		}
-		
-		/* struct copyin done during dispatch... need to copy file_id array separately */
-		if (ap->a_data == NULL) {
-			error = EINVAL;
-			goto err_exit_bulk_access;
-		}
-
-		if (is64bit) {
-			user_access_structp = (struct user_access_t *)ap->a_data;
-		}
-		else {
-			struct access_t *       accessp = (struct access_t *)ap->a_data;
-			tmp_user_access_t.uid = accessp->uid;
-			tmp_user_access_t.flags = accessp->flags;
-			tmp_user_access_t.num_groups = accessp->num_groups;
-			tmp_user_access_t.num_files = accessp->num_files;
-			tmp_user_access_t.file_ids = CAST_USER_ADDR_T(accessp->file_ids);
-			tmp_user_access_t.groups = CAST_USER_ADDR_T(accessp->groups);
-			tmp_user_access_t.access = CAST_USER_ADDR_T(accessp->access);
-			user_access_structp = &tmp_user_access_t;
-		}
-		
-		num_files = user_access_structp->num_files;
-		if (num_files < 1) {
-			goto err_exit_bulk_access;
-		}
-		if (num_files > 256) {
-			error = EINVAL;
-			goto err_exit_bulk_access;
-		}
-		
-		if ((error = copyin(user_access_structp->file_ids, (caddr_t)file_ids,
-							num_files * sizeof(int)))) {
-			goto err_exit_bulk_access;
-		}
-		
-		/* fill in the ucred structure */
-		flags = user_access_structp->flags;
-		if ((flags & (F_OK | R_OK | W_OK | X_OK)) == 0) {
-			flags = R_OK;
-		}
-		
-		/* check if we've been passed leaf node ids or parent ids */
-		if (flags & PARENT_IDS_FLAG) {
-			check_leaf = false;
-		}
-		
-		/*
-		 * Create a templated credential; this credential may *NOT*
-		 * be used unless instantiated with a kauth_cred_create();
-		 * there must be a correcponding kauth_cred_unref() when it
-		 * is no longer in use (i.e. before it goes out of scope).
-		 */
-		memset(&myucred, 0, sizeof(myucred));
-		myucred.cr_ref = 1;
-		myucred.cr_uid = myucred.cr_ruid = myucred.cr_svuid = user_access_structp->uid;
-		myucred.cr_ngroups = user_access_structp->num_groups;
-		if (myucred.cr_ngroups < 1 || myucred.cr_ngroups > 16) {
-			myucred.cr_ngroups = 0;
-		} else if ((error = copyin(user_access_structp->groups, (caddr_t)myucred.cr_groups,
-					  myucred.cr_ngroups * sizeof(gid_t)))) {
-			goto err_exit_bulk_access;
-		}
-		myucred.cr_rgid = myucred.cr_svgid = myucred.cr_groups[0];
-		myucred.cr_gmuid = myucred.cr_uid;
-		
-		my_context.vc_proc = p;
-		my_context.vc_ucred = kauth_cred_create(&myucred);
+	case HFS_BULKACCESS_FSCTL: {
+	    int size;
+	    
+	    if (hfsmp->hfs_flags & HFS_STANDARD) {
+		return EINVAL;
+	    }
 
-		/* Check access to each file_id passed in */
-		for (i = 0; i < num_files; i++) {
-#if 0
-			cnid = (cnid_t) file_ids[i];
-			
-			/* root always has access */
-			if (!suser(my_context.vc_ucred, NULL)) {
-				access[i] = 0;
-				continue;
-			}
-			
-			if (check_leaf) {
-				
-				/* do the lookup (checks the cnode hash, then the catalog) */
-				error = do_attr_lookup(hfsmp, &cache, dev, cnid, skip_cp, &catkey, &cnattr, p);
-				if (error) {
-					access[i] = (short) error;
-					continue;
-				}
-							
-				/* before calling CheckAccess(), check the target file for read access */
-				myPerms = DerivePermissionSummary(cnattr.ca_uid, cnattr.ca_gid,
-								  cnattr.ca_mode, hfsmp->hfs_mp, my_context.vc_ucred, p  );
-				
-				
-				/* fail fast if no access */ 
-				if ((myPerms & flags) == 0) {
-					access[i] = EACCES;
-					continue;
-				}
-			} else {
-				/* we were passed an array of parent ids */
-				catkey.hfsPlus.parentID = cnid;
-			}
-			
-			/* if the last guy had the same parent and had access, we're done */
-			if (i > 0 && catkey.hfsPlus.parentID == prevParent_cnid && access[i-1] == 0) {
-				cache.cachehits++;
-				access[i] = 0;
-				continue;
-			}
-			
-			myaccess = do_access_check(hfsmp, &error, &cache, catkey.hfsPlus.parentID, 
-						   skip_cp, p, my_context.vc_ucred, dev);
-			
-			if ( myaccess ) {
-				access[i] = 0; // have access.. no errors to report
-			} else {
-				access[i] = (error != 0 ? (short) error : EACCES);
-			}
-			
-			prevParent_cnid = catkey.hfsPlus.parentID;
-#else
-			int myErr;
-			
-			cnid = (cnid_t)file_ids[i];
-			
-			while (cnid >= kRootDirID) {
-			    /* get the vnode for this cnid */
-			    myErr = hfs_vget(hfsmp, cnid, &vp, 0);
-			    if ( myErr ) {
-				access[i] = EACCES;
-				break;
-			    }
+	    if (is64bit) {
+		size = sizeof(struct user_access_t);
+	    } else {
+		size = sizeof(struct access_t);
+	    }
+	    
+	    return do_bulk_access_check(hfsmp, vp, ap, size, context);
+	} 
 
-			    cnid = VTOC(vp)->c_parentcnid;
+	case HFS_EXT_BULKACCESS_FSCTL: {
+	    int size;
+	    
+	    if (hfsmp->hfs_flags & HFS_STANDARD) {
+		return EINVAL;
+	    }
 
-			    hfs_unlock(VTOC(vp));
-			    if (vnode_vtype(vp) == VDIR) {
-			    	/*
-				 * XXX This code assumes that none of the
-				 * XXX callbacks from vnode_authorize() will
-				 * XXX take a persistent ref on the context
-				 * XXX credential, which is a bad assumption.
-				 */
-				myErr = vnode_authorize(vp, NULL, (KAUTH_VNODE_SEARCH | KAUTH_VNODE_LIST_DIRECTORY), &my_context);
-			    } else {
-				myErr = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, &my_context);
-			    }
-			    vnode_put(vp);
-			    access[i] = myErr;
-			    if (myErr) {
-				break;
-			    }
-			}
-#endif			
-		}
-		
-		/* copyout the access array */
-		if ((error = copyout((caddr_t)access, user_access_structp->access, 
-				     num_files * sizeof (short)))) {
-			goto err_exit_bulk_access;
-		}
-		
-	err_exit_bulk_access:
-		
-		//printf("on exit (err %d), numfiles/numcached/cachehits/lookups is %d/%d/%d/%d\n", error, num_files, cache.numcached, cache.cachehits, cache.lookups);
-		
-		release_pathbuff((char *) cache.acache);
-		release_pathbuff((char *) cache.haveaccess);
-		release_pathbuff((char *) file_ids);
-		release_pathbuff((char *) access);
-		/* clean up local context, if needed */
-		if (IS_VALID_CRED(my_context.vc_ucred))
-			kauth_cred_unref(&my_context.vc_ucred);
-		
-		return (error);
-	} /* HFS_BULKACCESS */
+	    if (is64bit) {
+		size = sizeof(struct ext_user_access_t);
+	    } else {
+		size = sizeof(struct ext_access_t);
+	    }
+	    
+	    return do_bulk_access_check(hfsmp, vp, ap, size, context);
+	} 
 
 	case HFS_SETACLSTATE: {
 		int state;
@@ -1322,7 +1632,28 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 			return (EPERM);
 		}
 		if (state == 0 || state == 1)
-			return hfs_setextendedsecurity(hfsmp, state);
+			return hfs_set_volxattr(hfsmp, HFS_SETACLSTATE, state);
+		else
+			return (EINVAL);	
+	}
+
+	case HFS_SET_XATTREXTENTS_STATE: {
+		int state;
+
+		if (ap->a_data == NULL) {
+			return (EINVAL);
+		}
+
+		state = *(int *)ap->a_data;
+
+		/* Super-user can enable or disable extent-based extended 
+		 * attribute support on a volume 
+		 */
+		if (!is_suser()) {
+			return (EPERM);
+		}
+		if (state == 0 || state == 1)
+			return hfs_set_volxattr(hfsmp, HFS_SET_XATTREXTENTS_STATE, state);
 		else
 			return (EINVAL);	
 	}
@@ -1332,7 +1663,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 
 		error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK);
 		if (error == 0) {
-			error = hfs_fsync(vp, MNT_NOWAIT, TRUE, p);
+			error = hfs_fsync(vp, MNT_WAIT, TRUE, p);
 			hfs_unlock(VTOC(vp));
 		}
 
@@ -1380,7 +1711,7 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 			error = advisory_read(vp, fp->ff_size, ra->ra_offset, ra->ra_count);
 		}
 
-		hfs_unlock_truncate(VTOC(vp));
+		hfs_unlock_truncate(VTOC(vp), TRUE);
 		return (error);
 	}
 
@@ -1494,6 +1825,25 @@ hfs_vnop_ioctl( struct vnop_ioctl_args /* {
 		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 		break;
 
+	case HFS_MARK_BOOT_CORRUPT:
+		/* Mark the boot volume corrupt by setting 
+		 * kHFSVolumeInconsistentBit in the volume header.  This will 
+		 * force fsck_hfs on next mount.
+		 */
+		if (!is_suser()) {
+			return EACCES;
+		}
+		
+		/* Allowed only on the root vnode of the boot volume */
+		if (!(vfs_flags(HFSTOVFS(hfsmp)) & MNT_ROOTFS) || 
+		    !vnode_isvroot(vp)) {
+			return EINVAL;
+		}
+
+		printf ("hfs_vnop_ioctl: Marking the boot volume corrupt.\n");
+		hfs_mark_volume_inconsistent(hfsmp);
+		break;
+
 	default:
 		return (ENOTTY);
 	}
@@ -1530,13 +1880,12 @@ hfs_vnop_select(__unused struct vnop_select_args *ap)
  * The block run is returned in logical blocks, and is the REMAINING amount of blocks
  */
 int
-hfs_bmap(struct vnode *vp, daddr_t bn, struct vnode **vpp, daddr64_t *bnp, int *runp)
+hfs_bmap(struct vnode *vp, daddr_t bn, struct vnode **vpp, daddr64_t *bnp, unsigned int *runp)
 {
-	struct cnode *cp = VTOC(vp);
 	struct filefork *fp = VTOF(vp);
 	struct hfsmount *hfsmp = VTOHFS(vp);
 	int  retval = E_NONE;
-	daddr_t  logBlockSize;
+	u_int32_t  logBlockSize;
 	size_t  bytesContAvail = 0;
 	off_t  blockposition;
 	int lockExtBtree;
@@ -1547,17 +1896,17 @@ hfs_bmap(struct vnode *vp, daddr_t bn, struct vnode **vpp, daddr64_t *bnp, int *
 	 * to physical mapping is requested.
 	 */
 	if (vpp != NULL)
-		*vpp = cp->c_devvp;
+		*vpp = hfsmp->hfs_devvp;
 	if (bnp == NULL)
 		return (0);
 
 	logBlockSize = GetLogicalBlockSize(vp);
-	blockposition = (off_t)bn * (off_t)logBlockSize;
+	blockposition = (off_t)bn * logBlockSize;
 
 	lockExtBtree = overflow_extents(fp);
 
 	if (lockExtBtree)
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS, HFS_SHARED_LOCK);
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS, HFS_EXCLUSIVE_LOCK);
 
 	retval = MacToVFSError(
                             MapFileBlockC (HFSTOVCB(hfsmp),
@@ -1627,6 +1976,15 @@ hfs_vnop_offtoblk(struct vnop_offtoblk_args *ap)
 /*
  * Map file offset to physical block number.
  *
+ * If this function is called for write operation, and if the file
+ * had virtual blocks allocated (delayed allocation), real blocks
+ * are allocated by calling ExtendFileC().
+ * 
+ * If this function is called for read operation, and if the file
+ * had virtual blocks allocated (delayed allocation), no change 
+ * to the size of file is done, and if required, rangelist is 
+ * searched for mapping.
+ *
  * System file cnodes are expected to be locked (shared or exclusive).
  */
 int
@@ -1669,14 +2027,10 @@ hfs_vnop_blockmap(struct vnop_blockmap_args *ap)
 	if (ap->a_bpn == NULL)
 		return (0);
 
-	if ( !vnode_issystem(vp) && !vnode_islnk(vp)) {
+	if ( !vnode_issystem(vp) && !vnode_islnk(vp) && !vnode_isswap(vp)) {
 		if (VTOC(vp)->c_lockowner != current_thread()) {
 			hfs_lock(VTOC(vp), HFS_FORCE_LOCK);
 			tooklock = 1;
-		} else {
-			cp = VTOC(vp);
-			panic("blockmap: %s cnode lock already held!\n",
-				cp->c_desc.cd_nameptr ? cp->c_desc.cd_nameptr : "");
 		}
 	}
 	hfsmp = VTOHFS(vp);
@@ -1684,7 +2038,8 @@ hfs_vnop_blockmap(struct vnop_blockmap_args *ap)
 	fp = VTOF(vp);
 
 retry:
-	if (fp->ff_unallocblocks) {
+	/* Check virtual blocks only when performing write operation */
+	if ((ap->a_flags & VNODE_WRITE) && (fp->ff_unallocblocks != 0)) {
 		if (hfs_start_transaction(hfsmp) != 0) {
 			retval = EINVAL;
 			goto exit;
@@ -1703,8 +2058,8 @@ retry:
 	/*
 	 * Check for any delayed allocations.
 	 */
-	if (fp->ff_unallocblocks) {
-		SInt64 actbytes;
+	if ((ap->a_flags & VNODE_WRITE) && (fp->ff_unallocblocks != 0)) {
+		int64_t actbytes;
 		u_int32_t loanedBlocks;
 
 		// 
@@ -1740,9 +2095,7 @@ retry:
 			HFS_MOUNT_LOCK(hfsmp, TRUE);
 			hfsmp->loanedBlocks += loanedBlocks;
 			HFS_MOUNT_UNLOCK(hfsmp, TRUE);
-		}
 
-		if (retval) {
 			hfs_systemfile_unlock(hfsmp, lockflags);
 			cp->c_flag |= C_MODIFIED;
 			if (started_tr) {
@@ -1750,6 +2103,7 @@ retry:
 				(void) hfs_volupdate(hfsmp, VOL_UPDATE, 0);
 
 				hfs_end_transaction(hfsmp);
+				started_tr = 0;
 			}
 			goto exit;
 		}
@@ -1769,10 +2123,63 @@ retry:
 		started_tr = 0;
 	}	
 	if (retval) {
+		/* On write, always return error because virtual blocks, if any, 
+		 * should have been allocated in ExtendFileC().  We do not 
+		 * allocate virtual blocks on read, therefore return error 
+		 * only if no virtual blocks are allocated.  Otherwise we search
+		 * rangelist for zero-fills
+		 */
+		if ((MacToVFSError(retval) != ERANGE) ||
+		    (ap->a_flags & VNODE_WRITE) ||
+		    ((ap->a_flags & VNODE_READ) && (fp->ff_unallocblocks == 0))) {
+			goto exit;
+		} 
+		
+		/* Validate if the start offset is within logical file size */
+		if (ap->a_foffset > fp->ff_size) {
+		    	goto exit;
+		}
+
+		/* Searching file extents has failed for read operation, therefore 
+		 * search rangelist for any uncommitted holes in the file. 
+		 */
+		overlaptype = rl_scan(&fp->ff_invalidranges, ap->a_foffset,
+	        	              ap->a_foffset + (off_t)(ap->a_size - 1),
+	                	      &invalid_range);
+		switch(overlaptype) {
+		case RL_OVERLAPISCONTAINED:
+			/* start_offset <= rl_start, end_offset >= rl_end */
+			if (ap->a_foffset != invalid_range->rl_start) {
+				break;
+			}
+		case RL_MATCHINGOVERLAP:
+			/* start_offset = rl_start, end_offset = rl_end */
+		case RL_OVERLAPCONTAINSRANGE:
+			/* start_offset >= rl_start, end_offset <= rl_end */
+		case RL_OVERLAPSTARTSBEFORE:
+			/* start_offset > rl_start, end_offset >= rl_start */
+			if ((off_t)fp->ff_size > (invalid_range->rl_end + 1)) {
+				bytesContAvail = (invalid_range->rl_end + 1) - ap->a_foffset;
+			} else {
+				bytesContAvail = fp->ff_size - ap->a_foffset;
+			}
+			if (bytesContAvail > ap->a_size) {
+				bytesContAvail = ap->a_size;
+			}
+			*ap->a_bpn = (daddr64_t)-1;
+			retval = 0;
+			break;
+		case RL_OVERLAPENDSAFTER:
+			/* start_offset < rl_start, end_offset < rl_end */
+		case RL_NOOVERLAP:
+			break;
+		}
 		goto exit;
 	}
 
-	/* Adjust the mapping information for invalid file ranges: */
+	/* MapFileC() found a valid extent in the filefork.  Search the 
+	 * mapping information further for invalid file ranges 
+	 */
 	overlaptype = rl_scan(&fp->ff_invalidranges, ap->a_foffset,
 	                      ap->a_foffset + (off_t)bytesContAvail - 1,
 	                      &invalid_range);
@@ -1781,7 +2188,7 @@ retry:
 		case RL_MATCHINGOVERLAP:
 		case RL_OVERLAPCONTAINSRANGE:
 		case RL_OVERLAPSTARTSBEFORE:
-			/* There's no valid block for this byte offset: */
+			/* There's no valid block for this byte offset */
 			*ap->a_bpn = (daddr64_t)-1;
 			/* There's no point limiting the amount to be returned
 			 * if the invalid range that was hit extends all the way 
@@ -1814,13 +2221,17 @@ retry:
 		} /* end switch */
 		if (bytesContAvail > ap->a_size)
 			bytesContAvail = ap->a_size;
+	} 
+		
+exit:
+	if (retval == 0) {
+		if (ap->a_run)
+			*ap->a_run = bytesContAvail;
+
+		if (ap->a_poff)
+			*(int *)ap->a_poff = 0;
 	}
-	if (ap->a_run)
-		*ap->a_run = bytesContAvail;
 
-	if (ap->a_poff)
-		*(int *)ap->a_poff = 0;
-exit:
 	if (tooklock)
 		hfs_unlock(cp);
 
@@ -1839,14 +2250,13 @@ hfs_vnop_strategy(struct vnop_strategy_args *ap)
 {
 	buf_t	bp = ap->a_bp;
 	vnode_t	vp = buf_vnode(bp);
-	struct cnode *cp = VTOC(vp);
 
-	return (buf_strategy(cp->c_devvp, ap));
+	return (buf_strategy(VTOHFS(vp)->hfs_devvp, ap));
 }
 
 
 static int
-do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_context_t context)
+do_hfs_truncate(struct vnode *vp, off_t length, int flags, vfs_context_t context)
 {
 	register struct cnode *cp = VTOC(vp);
     	struct filefork *fp = VTOF(vp);
@@ -1856,7 +2266,6 @@ do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_
 	off_t bytesToAdd;
 	off_t actualBytesAdded;
 	off_t filebytes;
-	u_int64_t old_filesize;
 	u_long fileblocks;
 	int blksize;
 	struct hfsmount *hfsmp;
@@ -1865,7 +2274,6 @@ do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_
 	blksize = VTOVCB(vp)->blockSize;
 	fileblocks = fp->ff_blocks;
 	filebytes = (off_t)fileblocks * (off_t)blksize;
-	old_filesize = fp->ff_size;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 7)) | DBG_FUNC_START,
 		 (int)length, (int)fp->ff_size, (int)filebytes, 0, 0);
@@ -1978,7 +2386,7 @@ do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_
 		}
  
 		if (!(flags & IO_NOZEROFILL)) {
-			if (UBCINFOEXISTS(vp) && retval == E_NONE) {
+			if (UBCINFOEXISTS(vp)  && (vnode_issystem(vp) == 0) && retval == E_NONE) {
 				struct rl_entry *invalid_range;
 				off_t zero_limit;
 			
@@ -2028,36 +2436,9 @@ do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_
 		cp->c_touch_modtime = TRUE;
 		fp->ff_size = length;
 
-		/* Nested transactions will do their own ubc_setsize. */
-		if (!skipsetsize) {
-			/*
-			 * ubc_setsize can cause a pagein here 
-			 * so we need to drop cnode lock. 
-			 */
-			hfs_unlock(cp);
-			ubc_setsize(vp, length);
-			hfs_lock(cp, HFS_FORCE_LOCK);
-		}
-
 	} else { /* Shorten the size of the file */
 
 		if ((off_t)fp->ff_size > length) {
-			/*
-			 * Any buffers that are past the truncation point need to be
-			 * invalidated (to maintain buffer cache consistency).
-			 */
-
-		         /* Nested transactions will do their own ubc_setsize. */
-		         if (!skipsetsize) {
-		         	/*
-		         	 * ubc_setsize can cause a pageout here 
-		         	 * so we need to drop cnode lock. 
-		         	 */
-				hfs_unlock(cp);
-				ubc_setsize(vp, length);
-				hfs_lock(cp, HFS_FORCE_LOCK);
-			}
-	    
 			/* Any space previously marked as invalid is now irrelevant: */
 			rl_remove(length, fp->ff_size - 1, &fp->ff_invalidranges);
 		}
@@ -2138,11 +2519,12 @@ do_hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize, vfs_
 #endif /* QUOTA */
 		}
 		/* Only set update flag if the logical length changes */
-		if (old_filesize != length)
+		if ((off_t)fp->ff_size != length)
 			cp->c_touch_modtime = TRUE;
 		fp->ff_size = length;
 	}
-	cp->c_touch_chgtime = TRUE;
+	cp->c_touch_chgtime = TRUE;	/* status changed */
+	cp->c_touch_modtime = TRUE;	/* file data was modified */
 	retval = hfs_update(vp, MNT_WAIT);
 	if (retval) {
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 7)) | DBG_FUNC_NONE,
@@ -2174,13 +2556,30 @@ hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize,
 	int blksize, error = 0;
 	struct cnode *cp = VTOC(vp);
 
-	if (vnode_isdir(vp))
-		return (EISDIR);	/* cannot truncate an HFS directory! */
+	/* Cannot truncate an HFS directory! */
+	if (vnode_isdir(vp)) {
+		return (EISDIR);
+	}
+	/* A swap file cannot change size. */
+	if (vnode_isswap(vp) && (length != 0)) {
+		return (EPERM);
+	}
 
 	blksize = VTOVCB(vp)->blockSize;
 	fileblocks = fp->ff_blocks;
 	filebytes = (off_t)fileblocks * (off_t)blksize;
 
+	//
+	// Have to do this here so that we don't wind up with
+	// i/o pending for blocks that are about to be released
+	// if we truncate the file.
+	//
+	// If skipsetsize is set, then the caller is responsible
+	// for the ubc_setsize.
+	//
+	if (!skipsetsize)
+		ubc_setsize(vp, length);
+
 	// have to loop truncating or growing files that are
 	// really big because otherwise transactions can get
 	// enormous and consume too many kernel resources.
@@ -2193,7 +2592,7 @@ hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize,
 		    		filebytes = length;
 			}
 			cp->c_flag |= C_FORCEUPDATE;
-			error = do_hfs_truncate(vp, filebytes, flags, skipsetsize, context);
+			error = do_hfs_truncate(vp, filebytes, flags, context);
 			if (error)
 				break;
 		}
@@ -2205,13 +2604,13 @@ hfs_truncate(struct vnode *vp, off_t length, int flags, int skipsetsize,
 				filebytes = length;
 			}
 			cp->c_flag |= C_FORCEUPDATE;
-			error = do_hfs_truncate(vp, filebytes, flags, skipsetsize, context);
+			error = do_hfs_truncate(vp, filebytes, flags, context);
 			if (error)
 				break;
 		}
 	} else /* Same logical size */ {
 
-		error = do_hfs_truncate(vp, length, flags, skipsetsize, context);
+		error = do_hfs_truncate(vp, length, flags, context);
 	}
 	/* Files that are changing size are not hot file candidates. */
 	if (VTOHFS(vp)->hfc_stage == HFC_RECORDING) {
@@ -2247,8 +2646,8 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 	off_t filebytes;
 	u_long fileblocks;
 	int retval, retval2;
-	UInt32 blockHint;
-	UInt32 extendFlags;   /* For call to ExtendFileC */
+	u_int32_t blockHint;
+	u_int32_t extendFlags;   /* For call to ExtendFileC */
 	struct hfsmount *hfsmp;
 	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
 	int lockflags;
@@ -2259,10 +2658,15 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 		return (EISDIR);
 	if (length < (off_t)0)
 		return (EINVAL);
-
-	if ((retval = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK)))
-		return (retval);
+	
 	cp = VTOC(vp);
+
+	hfs_lock_truncate(cp, TRUE);
+
+	if ((retval = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
+		goto Err_Exit;
+	}
+	
 	fp = VTOF(vp);
 	hfsmp = VTOHFS(vp);
 	vcb = VTOVCB(vp);
@@ -2304,7 +2708,9 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 	 * value of filebytes is 0, length will be at least 1.
 	 */
 	if (length > filebytes) {
-		moreBytesRequested = length - filebytes;
+		off_t total_bytes_added = 0, orig_request_size;
+
+		orig_request_size = moreBytesRequested = length - filebytes;
 		
 #if QUOTA
 		retval = hfs_chkdq(cp,
@@ -2333,35 +2739,54 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 			}
 		}
 
-		if (hfs_start_transaction(hfsmp) != 0) {
-		    retval = EINVAL;
-		    goto Err_Exit;
-		}
 
-		/* Protect extents b-tree and allocation bitmap */
-		lockflags = SFL_BITMAP;
-		if (overflow_extents(fp))
+		while ((length > filebytes) && (retval == E_NONE)) {
+		    off_t bytesRequested;
+		    
+		    if (hfs_start_transaction(hfsmp) != 0) {
+			retval = EINVAL;
+			goto Err_Exit;
+		    }
+
+		    /* Protect extents b-tree and allocation bitmap */
+		    lockflags = SFL_BITMAP;
+		    if (overflow_extents(fp))
 			lockflags |= SFL_EXTENTS;
-		lockflags = hfs_systemfile_lock(hfsmp, lockflags, HFS_EXCLUSIVE_LOCK);
+		    lockflags = hfs_systemfile_lock(hfsmp, lockflags, HFS_EXCLUSIVE_LOCK);
+
+		    if (moreBytesRequested >= HFS_BIGFILE_SIZE) {
+			bytesRequested = HFS_BIGFILE_SIZE;
+		    } else {
+			bytesRequested = moreBytesRequested;
+		    }
 
-		retval = MacToVFSError(ExtendFileC(vcb,
+		    retval = MacToVFSError(ExtendFileC(vcb,
 						(FCB*)fp,
-						moreBytesRequested,
+						bytesRequested,
 						blockHint,
 						extendFlags,
 						&actualBytesAdded));
 
-		*(ap->a_bytesallocated) = actualBytesAdded;
-		filebytes = (off_t)fp->ff_blocks * (off_t)vcb->blockSize;
-
-		hfs_systemfile_unlock(hfsmp, lockflags);
+		    if (retval == E_NONE) {
+			*(ap->a_bytesallocated) += actualBytesAdded;
+			total_bytes_added += actualBytesAdded;
+			moreBytesRequested -= actualBytesAdded;
+			if (blockHint != 0) {
+			    blockHint += actualBytesAdded / vcb->blockSize;
+			}
+		    }
+		    filebytes = (off_t)fp->ff_blocks * (off_t)vcb->blockSize;
+		    
+		    hfs_systemfile_unlock(hfsmp, lockflags);
 
-		if (hfsmp->jnl) {
+		    if (hfsmp->jnl) {
 			(void) hfs_update(vp, TRUE);
 			(void) hfs_volupdate(hfsmp, VOL_UPDATE, 0);
+		    }
+
+		    hfs_end_transaction(hfsmp);
 		}
 
-		hfs_end_transaction(hfsmp);
 
 		/*
 		 * if we get an error and no changes were made then exit
@@ -2377,9 +2802,9 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 		 * until the file is closed, when we truncate the file to allocation
 		 * block size.
 		 */
-		if ((actualBytesAdded != 0) && (moreBytesRequested < actualBytesAdded))
+		if (total_bytes_added != 0 && orig_request_size < total_bytes_added)
 			*(ap->a_bytesallocated) =
-				roundup(moreBytesRequested, (off_t)vcb->blockSize);
+				roundup(orig_request_size, (off_t)vcb->blockSize);
 
 	} else { /* Shorten the size of the file */
 
@@ -2390,31 +2815,9 @@ hfs_vnop_allocate(struct vnop_allocate_args /* {
 			 */
 		}
 
-		if (hfs_start_transaction(hfsmp) != 0) {
-		    retval = EINVAL;
-		    goto Err_Exit;
-		}
-
-		/* Protect extents b-tree and allocation bitmap */
-		lockflags = SFL_BITMAP;
-		if (overflow_extents(fp))
-			lockflags |= SFL_EXTENTS;
-		lockflags = hfs_systemfile_lock(hfsmp, lockflags, HFS_EXCLUSIVE_LOCK);
-
-		retval = MacToVFSError(TruncateFileC(vcb, (FCB*)fp, length, false));
-
-		hfs_systemfile_unlock(hfsmp, lockflags);
-
+		retval = hfs_truncate(vp, length, 0, 0, ap->a_context);
 		filebytes = (off_t)fp->ff_blocks * (off_t)vcb->blockSize;
 
-		if (hfsmp->jnl) {
-			(void) hfs_update(vp, TRUE);
-			(void) hfs_volupdate(hfsmp, VOL_UPDATE, 0);
-		}
-
-		hfs_end_transaction(hfsmp);
-		
-
 		/*
 		 * if we get an error and no changes were made then exit
 		 * otherwise we must do the hfs_update to reflect the changes
@@ -2442,6 +2845,7 @@ Std_Exit:
 	if (retval == 0)
 		retval = retval2;
 Err_Exit:
+	hfs_unlock_truncate(cp, TRUE);
 	hfs_unlock(cp);
 	return (retval);
 }
@@ -2472,7 +2876,7 @@ hfs_vnop_pagein(struct vnop_pagein_args *ap)
 	/*
 	 * Keep track of blocks read.
 	 */
-	if (VTOHFS(vp)->hfc_stage == HFC_RECORDING && error == 0) {
+	if (!vnode_isswap(vp) && VTOHFS(vp)->hfc_stage == HFC_RECORDING && error == 0) {
 		struct cnode *cp;
 		struct filefork *fp;
 		int bytesread;
@@ -2487,7 +2891,7 @@ hfs_vnop_pagein(struct vnop_pagein_args *ap)
 			bytesread = ap->a_size;
 
 		/* When ff_bytesread exceeds 32-bits, update it behind the cnode lock. */
-		if ((fp->ff_bytesread + bytesread) > 0x00000000ffffffff) {
+		if ((fp->ff_bytesread + bytesread) > 0x00000000ffffffff && cp->c_lockowner != current_thread()) {
 			hfs_lock(cp, HFS_FORCE_LOCK);
 			took_cnode_lock = 1;
 		}
@@ -2532,36 +2936,45 @@ hfs_vnop_pageout(struct vnop_pageout_args *ap)
 	struct cnode *cp;
 	struct filefork *fp;
 	int retval;
-	off_t end_of_range;
 	off_t filesize;
 
 	cp = VTOC(vp);
-	if (cp->c_lockowner == current_thread()) {
-		panic("pageout: %s cnode lock already held!\n",
-		      cp->c_desc.cd_nameptr ? cp->c_desc.cd_nameptr : "");
-	}
-	if ( (retval = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
-		if (!(ap->a_flags & UPL_NOCOMMIT)) {
-		        ubc_upl_abort_range(ap->a_pl,
-					    ap->a_pl_offset,
-					    ap->a_size,
-					    UPL_ABORT_FREE_ON_EMPTY);
-		}
-		return (retval);
-	}
 	fp = VTOF(vp);
+	
+	if (vnode_isswap(vp)) {
+		filesize = fp->ff_size;
+	} else {
+		off_t end_of_range;
+		int tooklock = 0;
+
+		if (cp->c_lockowner != current_thread()) {
+		    if ( (retval = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
+			if (!(ap->a_flags & UPL_NOCOMMIT)) {
+				ubc_upl_abort_range(ap->a_pl,
+						    ap->a_pl_offset,
+						    ap->a_size,
+						    UPL_ABORT_FREE_ON_EMPTY);
+			}
+			return (retval);
+		    }
+		    tooklock = 1;
+		}
+	
+		filesize = fp->ff_size;
+		end_of_range = ap->a_f_offset + ap->a_size - 1;
+	
+		if (end_of_range >= filesize) {
+			end_of_range = (off_t)(filesize - 1);
+		}
+		if (ap->a_f_offset < filesize) {
+			rl_remove(ap->a_f_offset, end_of_range, &fp->ff_invalidranges);
+			cp->c_flag |= C_MODIFIED;  /* leof is dirty */
+		}
 
-	filesize = fp->ff_size;
-	end_of_range = ap->a_f_offset + ap->a_size - 1;
-
-	if (end_of_range >= filesize) {
-	        end_of_range = (off_t)(filesize - 1);
-	}
-	if (ap->a_f_offset < filesize) {
-	        rl_remove(ap->a_f_offset, end_of_range, &fp->ff_invalidranges);
-	        cp->c_flag |= C_MODIFIED;  /* leof is dirty */
+		if (tooklock) {
+		    hfs_unlock(cp);
+		}
 	}
-	hfs_unlock(cp);
 
 	retval = cluster_pageout(vp, ap->a_pl, ap->a_pl_offset, ap->a_f_offset,
 	                         ap->a_size, filesize, ap->a_flags);
@@ -2603,10 +3016,10 @@ hfs_vnop_bwrite(struct vnop_bwrite_args *ap)
 		 * Swap and validate the node if it is in native byte order.
 		 * This is always be true on big endian, so we always validate
 		 * before writing here.  On little endian, the node typically has
-		 * been swapped and validatated when it was written to the journal,
+		 * been swapped and validated when it was written to the journal,
 		 * so we won't do anything here.
 		 */
-		if (((UInt16 *)((char *)buf_dataptr(bp) + buf_count(bp) - 2))[0] == 0x000e) {
+		if (((u_int16_t *)((char *)buf_dataptr(bp) + buf_count(bp) - 2))[0] == 0x000e) {
 			/* Prepare the block pointer */
 			block.blockHeader = bp;
 			block.buffer = (char *)buf_dataptr(bp);
@@ -2626,7 +3039,7 @@ hfs_vnop_bwrite(struct vnop_bwrite_args *ap)
 	if ((buf_flags(bp) & B_LOCKED)) {
 	        // XXXdbg
 	        if (VTOHFS(vp)->jnl) {
-		        panic("hfs: CLEARING the lock bit on bp 0x%x\n", bp);
+		        panic("hfs: CLEARING the lock bit on bp %p\n", bp);
 		}
 		buf_clearflags(bp, B_LOCKED);
 	}
@@ -2650,7 +3063,7 @@ hfs_vnop_bwrite(struct vnop_bwrite_args *ap)
  * 0               N (file offset)
  *
  * -----------------     -----------------
- * |///////////////|     |               |     STEP 1 (aquire new blocks)
+ * |///////////////|     |               |     STEP 1 (acquire new blocks)
  * -----------------     -----------------
  * 0               N     N+1             2N
  *
@@ -2667,7 +3080,7 @@ hfs_vnop_bwrite(struct vnop_bwrite_args *ap)
  * During steps 2 and 3 page-outs to file offsets less
  * than or equal to N are suspended.
  *
- * During step 3 page-ins to the file get supended.
+ * During step 3 page-ins to the file get suspended.
  */
 __private_extern__
 int
@@ -2683,7 +3096,6 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 	u_int32_t  growsize;
 	u_int32_t  nextallocsave;
 	daddr64_t  sector_a,  sector_b;
-	int disabled_caching = 0;
 	int eflags;
 	off_t  newbytes;
 	int  retval;
@@ -2710,7 +3122,7 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 	if (blockHint == 0)
 		blockHint = hfsmp->nextAllocation;
 
-	if ((fp->ff_size > (u_int64_t)0x7fffffff) ||
+	if ((fp->ff_size > 0x7fffffff) ||
 	    ((fp->ff_size > blksize) && vnodetype == VLNK)) {
 		return (EFBIG);
 	}
@@ -2729,10 +3141,16 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 	if (!vnode_issystem(vp) && (vnodetype != VLNK)) {
 		hfs_unlock(cp);
 		hfs_lock_truncate(cp, TRUE);
-		if ((retval = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
-			hfs_unlock_truncate(cp);
+		/* Force lock since callers expects lock to be held. */
+		if ((retval = hfs_lock(cp, HFS_FORCE_LOCK))) {
+			hfs_unlock_truncate(cp, TRUE);
 			return (retval);
 		}
+		/* No need to continue if file was removed. */
+		if (cp->c_flag & C_NOEXISTS) {
+			hfs_unlock_truncate(cp, TRUE);
+			return (ENOENT);
+		}
 		took_trunc_lock = 1;
 	}
 	headblks = fp->ff_blocks;
@@ -2745,7 +3163,7 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 
 	if (hfs_start_transaction(hfsmp) != 0) {
 		if (took_trunc_lock)
-	    		hfs_unlock_truncate(cp);
+			hfs_unlock_truncate(cp, TRUE);
 	    return (EINVAL);
 	}
 	started_tr = 1;
@@ -2765,19 +3183,14 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 	}
 
 	/*
-	 * STEP 1 - aquire new allocation blocks.
+	 * STEP 1 - acquire new allocation blocks.
 	 */
-	if (!vnode_isnocache(vp)) {
-		vnode_setnocache(vp);
-		disabled_caching = 1;
-
-	}
 	nextallocsave = hfsmp->nextAllocation;
 	retval = ExtendFileC(hfsmp, (FCB*)fp, growsize, blockHint, eflags, &newbytes);
 	if (eflags & kEFMetadataMask) {
 		HFS_MOUNT_LOCK(hfsmp, TRUE);
-		hfsmp->nextAllocation = nextallocsave;
-		hfsmp->vcbFlags |= 0xFF00;
+		HFS_UPDATE_NEXT_ALLOCATION(hfsmp, nextallocsave);
+		MarkVCBDirty(hfsmp);
 		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 	}
 
@@ -2802,7 +3215,17 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 		} else if ((eflags & kEFMetadataMask) &&
 		           ((((u_int64_t)sector_b * hfsmp->hfs_phys_block_size) / blksize) >
 		              hfsmp->hfs_metazone_end)) {
-			printf("hfs_relocate: didn't move into metadata zone\n");
+			const char * filestr;
+			char emptystr = '\0';
+
+			if (cp->c_desc.cd_nameptr != NULL) {
+				filestr = (const char *)&cp->c_desc.cd_nameptr[0];
+			} else if (vnode_name(vp) != NULL) {
+				filestr = vnode_name(vp);
+			} else {
+				filestr = &emptystr;
+			}
+			printf("hfs_relocate: %s didn't move into MDZ (%d blks)\n", filestr, fp->ff_blocks);
 			retval = ENOSPC;
 			goto restore;
 		}
@@ -2859,7 +3282,7 @@ hfs_relocate(struct  vnode *vp, u_int32_t  blockHint, kauth_cred_t cred,
 		goto restore;
 out:
 	if (took_trunc_lock)
-		hfs_unlock_truncate(cp);
+		hfs_unlock_truncate(cp, TRUE);
 
 	if (lockflags) {
 		hfs_systemfile_unlock(hfsmp, lockflags);
@@ -2870,7 +3293,6 @@ out:
 	if (retval == 0) {
 		(void) hfs_update(vp, MNT_WAIT);
 	}
-
 	if (hfsmp->jnl) {
 		if (cp->c_cnid < kHFSFirstUserCatalogNodeID)
 			(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
@@ -2878,17 +3300,17 @@ out:
 			(void) hfs_flushvolumeheader(hfsmp, MNT_NOWAIT, 0);
 	}
 exit:
-	if (disabled_caching) {
-		vnode_clearnocache(vp);
-	}
 	if (started_tr)
 		hfs_end_transaction(hfsmp);
 
 	return (retval);
 
 restore:
-	if (fp->ff_blocks == headblks)
+	if (fp->ff_blocks == headblks) {
+		if (took_trunc_lock)
+			hfs_unlock_truncate(cp, TRUE);
 		goto exit;
+	}
 	/*
 	 * Give back any newly allocated space.
 	 */
@@ -2905,7 +3327,7 @@ restore:
 	lockflags = 0;
 
 	if (took_trunc_lock)
-		hfs_unlock_truncate(cp);
+		hfs_unlock_truncate(cp, TRUE);
 	goto exit;
 }
 
@@ -2915,7 +3337,7 @@ restore:
  *
  */
 static int
-hfs_clonelink(struct vnode *vp, int blksize, kauth_cred_t cred, struct proc *p)
+hfs_clonelink(struct vnode *vp, int blksize, kauth_cred_t cred, __unused struct proc *p)
 {
 	struct buf *head_bp = NULL;
 	struct buf *tail_bp = NULL;
@@ -2979,7 +3401,7 @@ hfs_clonefile(struct vnode *vp, int blkstart, int blkcnt, int blksize)
 		uio_reset(auio, offset, UIO_SYSSPACE32, UIO_READ);
 		uio_addiov(auio, (uintptr_t)bufp, iosize);
 
-		error = cluster_read(vp, auio, copysize, 0);
+		error = cluster_read(vp, auio, copysize, IO_NOCACHE);
 		if (error) {
 			printf("hfs_clonefile: cluster_read failed - %d\n", error);
 			break;
diff --git a/bsd/hfs/hfs_search.c b/bsd/hfs/hfs_search.c
index 1432a7c69..5c48b3282 100644
--- a/bsd/hfs/hfs_search.c
+++ b/bsd/hfs/hfs_search.c
@@ -1,26 +1,38 @@
 /*
- * Copyright (c) 1997-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  *	@(#)hfs_search.c
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -37,6 +49,10 @@
 #include <sys/kauth.h>
 #include <sys/vnode_internal.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 #include "hfs.h"
 #include "hfs_dbg.h"
 #include "hfs_catalog.h"
@@ -75,7 +91,7 @@ struct searchinfospec
 	struct timespec		changeDate;	
 	struct timespec		accessDate;		
 	struct timespec		lastBackupDate;	
-	uint8_t			finderInfo[32];
+	u_int8_t		finderInfo[32];
 	uid_t			uid;	
 	gid_t			gid;
 	mode_t			mask;
@@ -84,7 +100,7 @@ struct searchinfospec
 };
 typedef struct searchinfospec searchinfospec_t;
 
-static void ResolveHardlink(ExtendedVCB *vcb, HFSPlusCatalogFile *recp);
+static void ResolveHardlink(struct hfsmount *hfsmp, HFSPlusCatalogFile *recp);
 
 
 static int UnpackSearchAttributeBlock(struct hfsmount *hfsmp, struct attrlist *alist,
@@ -96,8 +112,7 @@ static int CheckCriteria(	ExtendedVCB *vcb,
 							CatalogRecord *rec,
 							CatalogKey *key, 
 							searchinfospec_t *searchInfo1,
-							searchinfospec_t *searchInfo2,
-							Boolean lookForDup );
+							searchinfospec_t *searchInfo2 );
 
 static int CheckAccess(ExtendedVCB *vcb, u_long searchBits, CatalogKey *key, struct vfs_context *ctx);
 
@@ -120,12 +135,6 @@ static Boolean CompareWideRange( u_int64_t val, u_int64_t low, u_int64_t high )
 }
 //#define CompareRange(val, low, high)	((val >= low) && (val <= high))
 			
-#if 1 // Installer workaround (2940423)
-static Boolean IsTargetName( searchinfospec_t * searchInfoPtr, Boolean isHFSPlus );
-#endif // Installer workaround
-
-__private_extern__ int hfs_vnop_search(struct vnop_searchfs_args *ap);
-
 
 /************************************************************************/
 /* Entry for searchfs()                                                 */
@@ -174,11 +183,10 @@ hfs_vnop_search(ap)
 	void *variableBuffer;
 	u_long fixedBlockSize;
 	u_long eachReturnBufferSize;
-	struct proc *p = proc_self();
+	struct proc *p = current_proc();
 	int err = E_NONE;
 	int isHFSPlus;
 	int timerExpired = false;
-	int doQuickExit = false;
 	CatalogKey * myCurrentKeyPtr;
 	CatalogRecord * myCurrentDataPtr;
 	CatPosition * myCatPositionPtr;
@@ -195,6 +203,17 @@ hfs_vnop_search(ap)
 	if (ap->a_options & ~SRCHFS_VALIDOPTIONSMASK)
 		return (EINVAL);
 
+	/*
+	 * Reject requests for unsupported attributes.
+	 */
+	if ((ap->a_returnattrs->commonattr & ~HFS_ATTR_CMN_VALID) ||
+	    (ap->a_returnattrs->volattr != 0) ||
+	    (ap->a_returnattrs->dirattr & ~HFS_ATTR_DIR_VALID) ||
+	    (ap->a_returnattrs->fileattr & ~HFS_ATTR_FILE_VALID) ||
+	    (ap->a_returnattrs->forkattr != 0)) {
+		return (EINVAL);
+	}
+
 	/* SRCHFS_SKIPLINKS requires root access.
 	 * This option cannot be used with either
 	 * the ATTR_CMN_NAME or ATTR_CMN_PAROBJID
@@ -210,6 +229,16 @@ hfs_vnop_search(ap)
 			return (err);
 	}
 
+	// If both 32-bit and 64-bit parent ids or file ids are given	   
+	// then return an error.
+
+	attrgroup_t test_attrs=ap->a_searchattrs->commonattr;
+
+	if (((test_attrs & ATTR_CMN_OBJID) && (test_attrs & ATTR_CMN_FILEID)) ||
+			((test_attrs & ATTR_CMN_PARENTID) && (test_attrs & ATTR_CMN_PAROBJID)))
+		return (EINVAL);
+
+
 	if (uio_resid(ap->a_uio) <= 0)
 		return (EINVAL);
 
@@ -231,7 +260,13 @@ hfs_vnop_search(ap)
 				&searchInfo2, ap->a_searchparams2);
 	if (err) return err;
 
-    fixedBlockSize = sizeof(uint32_t) + hfs_attrblksize(ap->a_returnattrs);	/* uint32_t for length word */
+	//shadow search bits if 64-bit file/parent ids are used	
+	if (ap->a_searchattrs->commonattr & ATTR_CMN_FILEID) 
+		ap->a_searchattrs->commonattr |= ATTR_CMN_OBJID;
+	if (ap->a_searchattrs->commonattr & ATTR_CMN_PARENTID) 
+		ap->a_searchattrs->commonattr |= ATTR_CMN_PAROBJID;
+
+	fixedBlockSize = sizeof(u_int32_t) + hfs_attrblksize(ap->a_returnattrs);	/* u_int32_t for length word */
 
 	eachReturnBufferSize = fixedBlockSize;
 
@@ -273,53 +308,11 @@ hfs_vnop_search(ap)
 		}
 
 		ap->a_options &= ~SRCHFS_START;
-		bzero( (caddr_t)myCatPositionPtr, sizeof( *myCatPositionPtr ) );
+		bzero((caddr_t)myCatPositionPtr, sizeof(*myCatPositionPtr));
 		err = BTScanInitialize(catalogFCB, 0, 0, 0, kCatSearchBufferSize, &myBTScanState);
-		
-#if 1 // Installer workaround (2940423)
-		// hack to get around installer problems when the installer expects search results 
-		// to be in key order.  At this point the problem appears to be limited to 
-		// searches for "Library".  The idea here is to go get the "Library" at root
-		// and return it first to the caller then continue the search as normal with
-		// the exception of taking care not to return a duplicate hit (see CheckCriteria) 
-		if ( err == E_NONE &&
-			 (ap->a_searchattrs->commonattr & ATTR_CMN_NAME) != 0 &&
-			 IsTargetName( &searchInfo1, isHFSPlus )  )
-		{
-			CatalogRecord		rec;
-			BTreeIterator       iterator;
-			FSBufferDescriptor  btrec;
-			CatalogKey *		keyp;
-			UInt16              reclen;
-			OSErr				result;
-		
-			bzero( (caddr_t)&iterator, sizeof( iterator ) );
-			keyp = (CatalogKey *) &iterator.key;
-			(void) BuildCatalogKeyUTF8(vcb, kRootDirID, "Library", kUndefinedStrLen, keyp, NULL);
-
-			btrec.bufferAddress = &rec;
-			btrec.itemCount = 1;
-			btrec.itemSize = sizeof( rec );
-
-			result = BTSearchRecord( catalogFCB, &iterator, &btrec, &reclen, &iterator );
-			if ( result == E_NONE ) {
-				// need to unlock since CheckAccess assumes no lock held
-				hfs_systemfile_unlock(hfsmp, lockflags);
-				if (CheckCriteria(vcb, ap->a_options, ap->a_searchattrs, &rec,
-								  keyp, &searchInfo1, &searchInfo2, false) &&
-					CheckAccess(vcb, ap->a_options, keyp, ap->a_context)) {
-		
-					result = InsertMatch(hfsmp, ap->a_uio, &rec, 
-									  keyp, ap->a_returnattrs,
-									  attributesBuffer, variableBuffer,
-									  ap->a_nummatches);
-					if (result == E_NONE && *(ap->a_nummatches) >= ap->a_maxmatches)
-						doQuickExit = true;
-				}
-				lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-			}
+		if (err) {
+			goto ExitThisRoutine;
 		}
-#endif // Installer workaround
 	} else {
 		/* Resuming a search. */
 		err = BTScanInitialize(catalogFCB, myCatPositionPtr->nextNode, 
@@ -338,10 +331,6 @@ hfs_vnop_search(ap)
 
 	if (err)
 		goto ExitThisRoutine;
-#if 1 // Installer workaround (2940423)
-	if ( doQuickExit )
-		goto QuickExit;
-#endif // Installer workaround
 
 	/*
 	 * Check all the catalog btree records...
@@ -359,12 +348,10 @@ hfs_vnop_search(ap)
 
 		/* Resolve any hardlinks */
 		if (isHFSPlus && (ap->a_options & SRCHFS_SKIPLINKS) == 0) {
-			lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-			ResolveHardlink(vcb, (HFSPlusCatalogFile *) myCurrentDataPtr);
-			hfs_systemfile_unlock(hfsmp, lockflags);
+			ResolveHardlink(vcb, (HFSPlusCatalogFile *)myCurrentDataPtr);
 		}
 		if (CheckCriteria( vcb, ap->a_options, ap->a_searchattrs, myCurrentDataPtr,
-				myCurrentKeyPtr, &searchInfo1, &searchInfo2, true )
+				myCurrentKeyPtr, &searchInfo1, &searchInfo2 )
 		&&  CheckAccess(vcb, ap->a_options, myCurrentKeyPtr, ap->a_context)) {
 			err = InsertMatch(hfsmp, ap->a_uio, myCurrentDataPtr, 
 					myCurrentKeyPtr, ap->a_returnattrs,
@@ -397,7 +384,6 @@ hfs_vnop_search(ap)
 		}
 	}
 
-QuickExit:
 	/* Update catalog position */
 	myCatPositionPtr->writeCount = myBTScanState.btcb->writeCount;
 
@@ -430,29 +416,53 @@ ExitThisRoutine:
 
 
 static void
-ResolveHardlink(ExtendedVCB *vcb, HFSPlusCatalogFile *recp)
+ResolveHardlink(struct hfsmount *hfsmp, HFSPlusCatalogFile *recp)
 {
-	if ((recp->recordType == kHFSPlusFileRecord)
-	&&  (SWAP_BE32(recp->userInfo.fdType) == kHardLinkFileType)
-	&&  (SWAP_BE32(recp->userInfo.fdCreator) == kHFSPlusCreator)
-	&&  ((to_bsd_time(recp->createDate) == vcb->vcbCrDate) ||
-	     (to_bsd_time(recp->createDate) == VCBTOHFS(vcb)->hfs_metadata_createdate))) {
+	u_int32_t type, creator;
+	int isdirlink = 0;
+	int isfilelink = 0;
+	time_t filecreatedate;
+ 
+	if (recp->recordType != kHFSPlusFileRecord) {
+		return;
+	}
+	type = SWAP_BE32(recp->userInfo.fdType);
+	creator = SWAP_BE32(recp->userInfo.fdCreator);
+	filecreatedate = to_bsd_time(recp->createDate);
+
+	if ((type == kHardLinkFileType && creator == kHFSPlusCreator) &&
+	    (filecreatedate == (time_t)hfsmp->vcbCrDate ||
+	     filecreatedate == (time_t)hfsmp->hfs_metadata_createdate)) {
+		isfilelink = 1;
+	} else if ((type == kHFSAliasType && creator == kHFSAliasCreator) &&
+	           (recp->flags & kHFSHasLinkChainMask) &&
+	           (filecreatedate == (time_t)hfsmp->vcbCrDate ||
+	            filecreatedate == (time_t)hfsmp->hfs_metadata_createdate)) {
+		isdirlink = 1;
+	}
+
+	if (isfilelink || isdirlink) {
 		cnid_t saved_cnid;
+		int lockflags;
 
 		/* Export link's cnid (a unique value) instead of inode's cnid */
 		saved_cnid = recp->fileID;
-		(void) resolvelink(VCBTOHFS(vcb), recp->bsdInfo.special.iNodeNum, recp);
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+
+		(void) cat_resolvelink(hfsmp, recp->hl_linkReference, isdirlink, recp);
+
 		recp->fileID = saved_cnid;
+		hfs_systemfile_unlock(hfsmp, lockflags);
 	}
 }
 
 
 static Boolean
-CompareMasked(const UInt32 *thisValue, const UInt32 *compareData,
-		const UInt32 *compareMask, UInt32 count)
+CompareMasked(const u_int32_t *thisValue, const u_int32_t *compareData,
+		const u_int32_t *compareMask, u_int32_t count)
 {
 	Boolean	matched;
-	UInt32	i;
+	u_int32_t	i;
 	
 	matched = true;		/* Assume it will all match */
 	
@@ -514,14 +524,14 @@ ComparePartialPascalName ( register ConstStr31Param str, register ConstStr31Para
 // that's limited to /System.
 //
 static int
-is_inappropriate_name(char *name, int len)
+is_inappropriate_name(const char *name, int len)
 {
     const char *bad_names[] = { "System" };
     int   bad_len[]   = {        6 };
     int  i;
     
     for(i=0; i < (int) (sizeof(bad_names) / sizeof(bad_names[0])); i++) {
-	if (len == bad_len[i] && strcmp(name, bad_names[i]) == 0) {
+	if (len == bad_len[i] && strncmp(name, bad_names[i], strlen(bad_names[i]) + 1) == 0) {
 	    return 1;
 	}
     }
@@ -576,7 +586,7 @@ CheckAccess(ExtendedVCB *theVCBPtr, u_long searchBits, CatalogKey *theKeyPtr, st
 		if ( searchBits & SRCHFS_SKIPPACKAGES ) {
 		    if (   (SWAP_BE16(finfop->frFlags) & kHasBundle)
 			|| (cp->c_desc.cd_nameptr != NULL
-			    && is_package_name(cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen)) ) {
+			    && is_package_name((const char *)cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen)) ) {
 				myResult = 0;
 				goto ExitThisRoutine;
 		    }
@@ -584,7 +594,7 @@ CheckAccess(ExtendedVCB *theVCBPtr, u_long searchBits, CatalogKey *theKeyPtr, st
 
 		if ( searchBits & SRCHFS_SKIPINAPPROPRIATE ) {
 		    if ( cp->c_parentcnid == kRootDirID && cp->c_desc.cd_nameptr != NULL &&
-			     is_inappropriate_name(cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen) ) {
+			     is_inappropriate_name((const char *)cp->c_desc.cd_nameptr, cp->c_desc.cd_namelen) ) {
 				myResult = 0;
 				goto ExitThisRoutine;
 		    }
@@ -598,6 +608,20 @@ CheckAccess(ExtendedVCB *theVCBPtr, u_long searchBits, CatalogKey *theKeyPtr, st
 
 		myNodeID = cp->c_parentcnid;	/* move up the hierarchy */
 		hfs_unlock(VTOC(vp));
+
+#if CONFIG_MACF
+		if (vp->v_type == VDIR) {
+			myErr = mac_vnode_check_readdir(ctx, vp);
+		} else {
+			myErr = mac_vnode_check_stat(ctx, NOCRED, vp);
+		}
+		if (myErr) {
+			vnode_put(vp);
+			vp = NULL;
+			goto ExitThisRoutine;
+		}
+#endif /* MAC */
+
 		if (vp->v_type == VDIR) {
 		    myErr = vnode_authorize(vp, NULL, (KAUTH_VNODE_SEARCH | KAUTH_VNODE_LIST_DIRECTORY), ctx);
 		} else {
@@ -627,8 +651,7 @@ CheckCriteria(	ExtendedVCB *vcb,
 				CatalogRecord *rec, 
 				CatalogKey *key,
 				searchinfospec_t  *searchInfo1, 
-				searchinfospec_t *searchInfo2,
-				Boolean lookForDup )
+				searchinfospec_t *searchInfo2 )
 {
 	Boolean matched, atleastone;
 	Boolean isHFSPlus;
@@ -664,12 +687,18 @@ CheckCriteria(	ExtendedVCB *vcb,
 
 			if ((SWAP_BE32(filep->userInfo.fdType) == kHardLinkFileType) &&
 			    (SWAP_BE32(filep->userInfo.fdCreator) == kHFSPlusCreator)) {
-				return (false);	/* skip over link records */
-			} else if ((parid == VCBTOHFS(vcb)->hfs_privdir_desc.cd_cnid) &&
+				return (false);	/* skip over file link records */
+			} else if ((parid == vcb->hfs_private_desc[FILE_HARDLINKS].cd_cnid) &&
 			           (filep->bsdInfo.special.linkCount == 0)) {
 				return (false);	/* skip over unlinked files */
+			} else if ((SWAP_BE32(filep->userInfo.fdType) == kHFSAliasType) &&
+			           (SWAP_BE32(filep->userInfo.fdCreator) == kHFSAliasCreator) &&
+			           (filep->flags & kHFSHasLinkChainMask)) {
+				return (false);	/* skip over dir link records */
 			}
-		} else if (key->hfsPlus.parentID == VCBTOHFS(vcb)->hfs_privdir_desc.cd_cnid) {
+		} else if (key->hfsPlus.parentID == vcb->hfs_private_desc[FILE_HARDLINKS].cd_cnid) {
+			return (false);	/* skip over private files */
+		} else if (key->hfsPlus.parentID == vcb->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
 			return (false);	/* skip over private files */
 		}
 
@@ -711,20 +740,6 @@ CheckCriteria(	ExtendedVCB *vcb,
 				matched = (FastRelString(key->hfs.nodeName, (u_char*)searchInfo1->name) == 0);
 		}
 
-#if 1 // Installer workaround (2940423)
-		if ( lookForDup ) {
-			HFSCatalogNodeID parentID;
-			if (isHFSPlus)
-				parentID = key->hfsPlus.parentID;
-			else
-				parentID = key->hfs.parentID;
-	
-			if ( matched && parentID == kRootDirID && 
-				 IsTargetName( searchInfo1, isHFSPlus )  )
-				matched = false;
-		}
-#endif // Installer workaround
-
 		if ( matched == false || (searchBits & ~SRCHFS_MATCHPARTIALNAMES) == 0 )
 			goto TestDone;	/* no match, or nothing more to compare */
 
@@ -881,16 +896,16 @@ CheckCriteria(	ExtendedVCB *vcb,
 
 		/* Finder Info & Extended Finder Info where extFinderInfo is last 32 bytes */
 		if ( searchAttributes & ATTR_CMN_FNDRINFO ) {
-			UInt32 *thisValue;
-			thisValue = (UInt32 *) &c_attr.ca_finderinfo;
+			u_int32_t *thisValue;
+			thisValue = (u_int32_t *) &c_attr.ca_finderinfo;
 
 			/* 
 			 * Note: ioFlFndrInfo and ioDrUsrWds have the same offset in search info, so
 			 * no need to test the object type here.
 			 */
 			matched = CompareMasked(thisValue,
-					(UInt32 *)&searchInfo1->finderInfo,
-					(UInt32 *) &searchInfo2->finderInfo, 8);
+					(u_int32_t *)&searchInfo1->finderInfo,
+					(u_int32_t *) &searchInfo2->finderInfo, 8);
 			if (matched == false) goto TestDone;
 			atleastone = true;
 		}
@@ -958,9 +973,9 @@ CheckCriteria(	ExtendedVCB *vcb,
 
 		/* mode */
 		if ( searchAttributes & ATTR_CMN_ACCESSMASK ) {
-			matched = CompareRange((uint32_t)c_attr.ca_mode, 
-					(uint32_t)searchInfo1->mask,
-					(uint32_t)searchInfo2->mask);
+			matched = CompareRange((u_int32_t)c_attr.ca_mode, 
+					(u_int32_t)searchInfo1->mask,
+					(u_int32_t)searchInfo2->mask);
 			if (matched == false) goto TestDone;
 			atleastone = true;
 		}
@@ -994,7 +1009,6 @@ InsertMatch(struct hfsmount *hfsmp, uio_t a_uio, CatalogRecord *rec,
 	void *rovingAttributesBuffer;
 	void *rovingVariableBuffer;
 	u_long packedBufferSize;
-	u_long privateDir = hfsmp->hfs_privdir_desc.cd_cnid;
 	struct attrblock attrblk;
 	struct cat_desc c_desc;
 	struct cat_attr c_attr;
@@ -1009,8 +1023,9 @@ InsertMatch(struct hfsmount *hfsmp, uio_t a_uio, CatalogRecord *rec,
 	/* Convert catalog record into cat_attr format. */
 	cat_convertattr(hfsmp, rec, &c_attr, &datafork, &rsrcfork);
 
-	/* hide our private meta data directory */
-	if ((privateDir != 0) && (c_attr.ca_fileid == privateDir)) {
+	/* Hide our private meta data directories */
+	if (c_attr.ca_fileid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+	    c_attr.ca_fileid == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
 		err = 0;
 		goto exit;
 	}
@@ -1038,6 +1053,7 @@ InsertMatch(struct hfsmount *hfsmp, uio_t a_uio, CatalogRecord *rec,
 	attrblk.ab_varbufpp = &rovingVariableBuffer;
 	attrblk.ab_flags = 0;
 	attrblk.ab_blocksize = 0;
+	attrblk.ab_context = vfs_context_current();
 
 	hfs_packattrblk(&attrblk, hfsmp, NULL, &c_desc, &c_attr, &datafork, &rsrcfork, current_proc());
 
@@ -1069,11 +1085,11 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
     
     is_64_bit = proc_is64bit(current_proc());
 
-    bufferSize = *((uint32_t *)attributeBuffer);
+    bufferSize = *((u_int32_t *)attributeBuffer);
 	if (bufferSize == 0)
 		return (EINVAL);	/* XXX -DJB is a buffer size of zero ever valid for searchfs? */
 
-    ++((uint32_t *)attributeBuffer);	/* advance past the size */
+	attributeBuffer = (u_int32_t *)attributeBuffer + 1; /* advance past the size */
 	
 	/* 
 	 * UnPack common attributes
@@ -1094,49 +1110,50 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
 				/* Convert name to pascal string to match HFS B-Tree names */
 
 				if (len > 0) {
-					if (utf8_to_hfs(HFSTOVCB(hfsmp), len-1, s, (u_char*)searchInfo->name) != 0)
+					if (utf8_to_hfs(HFSTOVCB(hfsmp), len-1, (u_char *)s, (u_char*)searchInfo->name) != 0)
 						return (EINVAL);
 
 					searchInfo->nameLength = searchInfo->name[0];
 				} else {
 					searchInfo->name[0] = searchInfo->nameLength = 0;
 				}
-                ++((attrreference_t *)attributeBuffer);
+				attributeBuffer = (attrreference_t *)attributeBuffer + 1;
 			} else {
 				size_t ucslen;
 				/* Convert name to Unicode to match HFS Plus B-Tree names */
 
 				if (len > 0) {
-					if (utf8_decodestr(s, len-1, (UniChar*)searchInfo->name, &ucslen,
-					    sizeof(searchInfo->name), ':', UTF_DECOMPOSED))
+					if (utf8_decodestr((u_int8_t *)s, len-1, (UniChar*)searchInfo->name, &ucslen,
+					    sizeof(searchInfo->name), ':', UTF_DECOMPOSED | UTF_ESCAPE_ILLEGAL))
 						return (EINVAL);
 
 					searchInfo->nameLength = ucslen / sizeof(UniChar);
 				} else {
 					searchInfo->nameLength = 0;
 				}
-                ++((attrreference_t *)attributeBuffer);
+				attributeBuffer = (attrreference_t *)attributeBuffer + 1;
 			}
 		}
 		if ( a & ATTR_CMN_OBJID ) {
 			searchInfo->nodeID = ((fsobj_id_t *) attributeBuffer)->fid_objno;	/* ignore fid_generation */
-			++((fsobj_id_t *)attributeBuffer);
+			attributeBuffer = (fsobj_id_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_CMN_PAROBJID ) {
-			searchInfo->parentDirID = ((fsobj_id_t *) attributeBuffer)->fid_objno;	/* ignore fid_generation */
-			++((fsobj_id_t *)attributeBuffer);
+			searchInfo->parentDirID = ((fsobj_id_t *) attributeBuffer)->fid_objno;  /* ignore fid_generation */
+			attributeBuffer = (fsobj_id_t *)attributeBuffer + 1;
 		}
+
 		if ( a & ATTR_CMN_CRTIME ) {
             if (is_64_bit) {
                 struct user_timespec tmp;
                 tmp = *((struct user_timespec *)attributeBuffer);
                 searchInfo->creationDate.tv_sec = (time_t)tmp.tv_sec;
                 searchInfo->creationDate.tv_nsec = tmp.tv_nsec;
-                ++((struct user_timespec *)attributeBuffer);
+				attributeBuffer = (struct user_timespec *)attributeBuffer + 1;
             }
             else {
                 searchInfo->creationDate = *((struct timespec *)attributeBuffer);
-                ++((struct timespec *)attributeBuffer);
+				attributeBuffer = (struct timespec *)attributeBuffer + 1;
             }
 		}
 		if ( a & ATTR_CMN_MODTIME ) {
@@ -1145,11 +1162,11 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
                 tmp = *((struct user_timespec *)attributeBuffer);
                 searchInfo->modificationDate.tv_sec = (time_t)tmp.tv_sec;
                 searchInfo->modificationDate.tv_nsec = tmp.tv_nsec;
-                ++((struct user_timespec *)attributeBuffer);
+				attributeBuffer = (struct user_timespec *)attributeBuffer + 1;
             }
             else {
                 searchInfo->modificationDate = *((struct timespec *)attributeBuffer);
-                ++((struct timespec *)attributeBuffer);
+				attributeBuffer = (struct timespec *)attributeBuffer + 1;
             }
 		}
 		if ( a & ATTR_CMN_CHGTIME ) {
@@ -1158,11 +1175,11 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
                 tmp = *((struct user_timespec *)attributeBuffer);
                 searchInfo->changeDate.tv_sec = (time_t)tmp.tv_sec;
                 searchInfo->changeDate.tv_nsec = tmp.tv_nsec;
-                ++((struct user_timespec *)attributeBuffer);
+				attributeBuffer = (struct user_timespec *)attributeBuffer + 1;
             }
             else {
                 searchInfo->changeDate = *((struct timespec *)attributeBuffer);
-                ++((struct timespec *)attributeBuffer);
+				attributeBuffer = (struct timespec *)attributeBuffer + 1;
             }
 		}
 		if ( a & ATTR_CMN_ACCTIME ) {
@@ -1171,11 +1188,11 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
                 tmp = *((struct user_timespec *)attributeBuffer);
                 searchInfo->accessDate.tv_sec = (time_t)tmp.tv_sec;
                 searchInfo->accessDate.tv_nsec = tmp.tv_nsec;
-                ++((struct user_timespec *)attributeBuffer);
+				attributeBuffer = (struct user_timespec *)attributeBuffer + 1;
             }
             else {
                 searchInfo->accessDate = *((struct timespec *)attributeBuffer);
-                ++((struct timespec *)attributeBuffer);
+				attributeBuffer = (struct timespec *)attributeBuffer + 1;
             }
 		}
 		if ( a & ATTR_CMN_BKUPTIME ) {
@@ -1184,28 +1201,36 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
                 tmp = *((struct user_timespec *)attributeBuffer);
                 searchInfo->lastBackupDate.tv_sec = (time_t)tmp.tv_sec;
                 searchInfo->lastBackupDate.tv_nsec = tmp.tv_nsec;
-                ++((struct user_timespec *)attributeBuffer);
+				attributeBuffer = (struct user_timespec *)attributeBuffer + 1;
             }
             else {
                 searchInfo->lastBackupDate = *((struct timespec *)attributeBuffer);
-                ++((struct timespec *)attributeBuffer);
+				attributeBuffer = (struct timespec *)attributeBuffer + 1;
             }
 		}
 		if ( a & ATTR_CMN_FNDRINFO ) {
 			bcopy( attributeBuffer, searchInfo->finderInfo, sizeof(searchInfo->finderInfo) );
-			(uint8_t *)attributeBuffer += 32;
+			attributeBuffer = (u_int8_t *)attributeBuffer + 32;
 		}
 		if ( a & ATTR_CMN_OWNERID ) {
 			searchInfo->uid = *((uid_t *)attributeBuffer);
-			++((uid_t *)attributeBuffer);
+			attributeBuffer = (uid_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_CMN_GRPID ) {
 			searchInfo->gid = *((gid_t *)attributeBuffer);
-			++((gid_t *)attributeBuffer);
+			attributeBuffer = (gid_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_CMN_ACCESSMASK ) {
 			searchInfo->mask = *((mode_t *)attributeBuffer);
-			++((mode_t *)attributeBuffer);
+			attributeBuffer = (mode_t *)attributeBuffer + 1;
+		}
+		if ( a & ATTR_CMN_FILEID ) {
+			searchInfo->nodeID = (u_int32_t)*((u_int64_t *) attributeBuffer);	
+			attributeBuffer = (u_int64_t *)attributeBuffer + 1;
+		}
+		if ( a & ATTR_CMN_PARENTID ) {
+			searchInfo->parentDirID = (u_int32_t)*((u_int64_t *) attributeBuffer);	
+			attributeBuffer = (u_int64_t *)attributeBuffer + 1;
 		}
 	}
 
@@ -1213,7 +1238,7 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
 	if ( a != 0 ) {
 		if ( a & ATTR_DIR_ENTRYCOUNT ) {
 			searchInfo->d.numFiles = *((u_int32_t *)attributeBuffer);
-			++((u_int32_t *)attributeBuffer);
+			attributeBuffer = (u_int32_t *)attributeBuffer + 1;
 		}
 	}
 
@@ -1221,59 +1246,22 @@ UnpackSearchAttributeBlock( struct hfsmount *hfsmp, struct attrlist	*alist, sear
 	if ( a != 0 ) {
 		if ( a & ATTR_FILE_DATALENGTH ) {
 			searchInfo->f.dataLogicalLength = *((off_t *)attributeBuffer);
-			++((off_t *)attributeBuffer);
+			attributeBuffer = (off_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_FILE_DATAALLOCSIZE ) {
 			searchInfo->f.dataPhysicalLength = *((off_t *)attributeBuffer);
-			++((off_t *)attributeBuffer);
+			attributeBuffer = (off_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_FILE_RSRCLENGTH ) {
 			searchInfo->f.resourceLogicalLength = *((off_t *)attributeBuffer);
-			++((off_t *)attributeBuffer);
+			attributeBuffer = (off_t *)attributeBuffer + 1;
 		}
 		if ( a & ATTR_FILE_RSRCALLOCSIZE ) {
 			searchInfo->f.resourcePhysicalLength = *((off_t *)attributeBuffer);
-			++((off_t *)attributeBuffer);
+			attributeBuffer = (off_t *)attributeBuffer + 1;
 		}
 	}
 
 	return (0);
 }
 
-
-#if 1 // Installer workaround (2940423)
-/* this routine was added as part of the work around where some installers would fail */
-/* because they incorrectly assumed search results were in some kind of order.  */
-/* This routine is used to indentify the problematic target.  At this point we */
-/* only know of one.  This routine could be modified for more (I hope not). */
-static Boolean IsTargetName( searchinfospec_t * searchInfoPtr, Boolean isHFSPlus )
-{
-	if ( searchInfoPtr->name == NULL )
-		return( false );
-		
-	if (isHFSPlus) {
-		HFSUniStr255 myName = {
-			7, 	/* number of unicode characters */
-			{
-				'L','i','b','r','a','r','y'
-			}
-		};		
-		if ( FastUnicodeCompare( myName.unicode, myName.length,
-								 (UniChar*)searchInfoPtr->name,
-								 searchInfoPtr->nameLength ) == 0 )  {
-			return( true );
-		}
-						  
-	} else {
-		u_char		myName[32] = {
-			0x07,'L','i','b','r','a','r','y'
-		};
-		if ( FastRelString(myName, (u_char*)searchInfoPtr->name) == 0 )  {
-			return( true );
-		}
-	}
-	return( false );
-	
-} /* IsTargetName */
-#endif // Installer workaround
-
diff --git a/bsd/hfs/hfs_vfsops.c b/bsd/hfs/hfs_vfsops.c
index a1053d3c4..d5a05045b 100644
--- a/bsd/hfs/hfs_vfsops.c
+++ b/bsd/hfs/hfs_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1991, 1993, 1994
@@ -79,6 +85,7 @@
 #include <sys/paths.h>
 #include <sys/utfconv.h>
 #include <sys/kdebug.h>
+#include <sys/fslog.h>
 
 #include <kern/locks.h>
 
@@ -98,7 +105,6 @@
 #include "hfscommon/headers/FileMgrInternal.h"
 #include "hfscommon/headers/BTreesInternal.h"
 
-
 #if	HFS_DIAGNOSTIC
 int hfs_dbg_all = 0;
 int hfs_dbg_err = 0;
@@ -110,16 +116,8 @@ lck_attr_t *  hfs_lock_attr;
 lck_grp_t *  hfs_mutex_group;
 lck_grp_t *  hfs_rwlock_group;
 
-
 extern struct vnodeopv_desc hfs_vnodeop_opv_desc;
 
-extern void hfs_converterinit(void);
-
-extern void inittodr(time_t base);
-
-extern int  hfs_write_access(struct vnode *, kauth_cred_t, struct proc *, Boolean);
-
-
 static int hfs_changefs(struct mount *mp, struct hfs_mount_args *args);
 static int hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp, vfs_context_t context);
 static int hfs_flushfiles(struct mount *, int, struct proc *);
@@ -127,8 +125,8 @@ static int hfs_flushMDB(struct hfsmount *hfsmp, int waitfor, int altflush);
 static int hfs_getmountpoint(struct vnode *vp, struct hfsmount **hfsmpp);
 static int hfs_init(struct vfsconf *vfsp);
 static int hfs_mount(struct mount *mp, vnode_t  devvp, user_addr_t data, vfs_context_t context);
-static int hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args, vfs_context_t context);
-static int hfs_reload(struct mount *mp, kauth_cred_t cred, struct proc *p);
+static int hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args, int journal_replay_only, vfs_context_t context);
+static int hfs_reload(struct mount *mp);
 static int hfs_vfs_root(struct mount *mp, struct vnode **vpp, vfs_context_t context);
 static int hfs_quotactl(struct mount *, int, uid_t, caddr_t, vfs_context_t context);
 static int hfs_start(struct mount *mp, int flags, vfs_context_t context);
@@ -140,9 +138,10 @@ static int hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context);
 static int hfs_vfs_vget(struct mount *mp, ino64_t ino, struct vnode **vpp, vfs_context_t context);
 static int hfs_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, vfs_context_t context);
 
-static int hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks);
+static int hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks, vfs_context_t context);
 static int hfs_overlapped_overflow_extents(struct hfsmount *hfsmp, u_int32_t startblk,
                                            u_int32_t catblks, u_int32_t fileID, int rsrcfork);
+static int hfs_journal_replay(const char *devnode, vfs_context_t context);
 
 
 /*
@@ -157,8 +156,10 @@ hfs_mountroot(mount_t mp, vnode_t rvp, vfs_context_t context)
 	ExtendedVCB *vcb;
 	struct vfsstatfs *vfsp;
 	int error;
+
+	hfs_chashinit_finish();
 	
-	if ((error = hfs_mountfs(rvp, mp, NULL, context)))
+	if ((error = hfs_mountfs(rvp, mp, NULL, 0, context)))
 		return (error);
 
 	/* Init hfsmp */
@@ -194,19 +195,19 @@ hfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data, vfs_context_t conte
 	struct hfsmount *hfsmp = NULL;
 	struct hfs_mount_args args;
 	int retval = E_NONE;
-	uint32_t cmdflags;
+	u_int32_t cmdflags;
 
 	if ((retval = copyin(data, (caddr_t)&args, sizeof(args)))) {
 		return (retval);
 	}
-	cmdflags = (uint32_t)vfs_flags(mp) & MNT_CMDFLAGS;
+	cmdflags = (u_int32_t)vfs_flags(mp) & MNT_CMDFLAGS;
 	if (cmdflags & MNT_UPDATE) {
 		hfsmp = VFSTOHFS(mp);
 
 		/* Reload incore data after an fsck. */
 		if (cmdflags & MNT_RELOAD) {
 			if (vfs_isrdonly(mp))
-				return hfs_reload(mp, vfs_context_ucred(context), p);
+				return hfs_reload(mp);
 			else
 				return (EINVAL);
 		}
@@ -260,6 +261,18 @@ hfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data, vfs_context_t conte
 
 		/* Change to a writable file system. */
 		if (vfs_iswriteupgrade(mp)) {
+
+			/*
+			 * On inconsistent disks, do not allow read-write mount
+			 * unless it is the boot volume being mounted.
+			 */
+			if (!(vfs_flags(mp) & MNT_ROOTFS) &&
+					(hfsmp->vcbAtrb & kHFSVolumeInconsistentMask)) {
+				retval = EINVAL;
+				goto out;
+			}
+
+
 			retval = hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
 			if (retval != E_NONE)
 				goto out;
@@ -304,8 +317,10 @@ hfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data, vfs_context_t conte
 			hfsmp->hfs_flags &= ~HFS_READ_ONLY;
 
 			if (!(hfsmp->hfs_flags & (HFS_READ_ONLY & HFS_STANDARD))) {
-				/* setup private/hidden directory for unlinked files */
-				FindMetaDataDirectory(HFSTOVCB(hfsmp));
+				/* Setup private/hidden directories for hardlinks. */
+				hfs_privatedir_init(hfsmp, FILE_HARDLINKS);
+				hfs_privatedir_init(hfsmp, DIR_HARDLINKS);
+
 				hfs_remove_orphans(hfsmp);
 
 				/*
@@ -314,6 +329,10 @@ hfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data, vfs_context_t conte
 				if (hfsmp->hfs_flags & HFS_METADATA_ZONE) {
 					(void) hfs_recording_init(hfsmp);
 				}
+				/* Force ACLs on HFS+ file systems. */
+				if (vfs_extendedsecurity(HFSTOVFS(hfsmp)) == 0) {
+					vfs_setextendedsecurity(HFSTOVFS(hfsmp));
+				}
 			}
 		}
 
@@ -323,9 +342,11 @@ hfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data, vfs_context_t conte
 	} else /* not an update request */ {
 
 		/* Set the mount flag to indicate that we support volfs  */
-		vfs_setflags(mp, (uint64_t)((unsigned int)MNT_DOVOLFS));
+		vfs_setflags(mp, (u_int64_t)((unsigned int)MNT_DOVOLFS));
 
-		retval = hfs_mountfs(devvp, mp, &args, context);
+		hfs_chashinit_finish();
+
+		retval = hfs_mountfs(devvp, mp, &args, 0, context);
 	}
 out:
 	if (retval == 0) {
@@ -383,7 +404,7 @@ hfs_changefs_callback(struct vnode *vp, void *cargs)
 		replace_desc(cp, &cndesc);
 
 		if (cndesc.cd_cnid == kHFSRootFolderID) {
-		        strncpy(vcb->vcbVN, cp->c_desc.cd_nameptr, NAME_MAX);
+		        strlcpy((char *)vcb->vcbVN, (const char *)cp->c_desc.cd_nameptr, NAME_MAX+1);
 			cp->c_desc.cd_encoding = args->hfsmp->hfs_encoding;
 		}
 	} else {
@@ -404,7 +425,7 @@ hfs_changefs(struct mount *mp, struct hfs_mount_args *args)
 	unicode_to_hfs_func_t	get_hfsname_func;
 	u_long old_encoding = 0;
 	struct hfs_changefs_cargs cargs;
-	uint32_t mount_flags;
+	u_int32_t mount_flags;
 
 	hfsmp = VFSTOHFS(mp);
 	vcb = HFSTOVCB(hfsmp);
@@ -417,7 +438,7 @@ hfs_changefs(struct mount *mp, struct hfs_mount_args *args)
 
 	/* The root filesystem must operate with actual permissions: */
 	if (permswitch && (mount_flags & MNT_ROOTFS) && (mount_flags & MNT_UNKNOWNPERMISSIONS)) {
-		vfs_clearflags(mp, (uint64_t)((unsigned int)MNT_UNKNOWNPERMISSIONS));	/* Just say "No". */
+		vfs_clearflags(mp, (u_int64_t)((unsigned int)MNT_UNKNOWNPERMISSIONS));	/* Just say "No". */
 		return EINVAL;
 	}
 	if (mount_flags & MNT_UNKNOWNPERMISSIONS)
@@ -427,6 +448,14 @@ hfs_changefs(struct mount *mp, struct hfs_mount_args *args)
 
 	namefix = permfix = 0;
 
+	/*
+	 * Tracking of hot files requires up-to-date access times.  So if
+	 * access time updates are disabled, we must also disable hot files.
+	 */
+	if (mount_flags & MNT_NOATIME) {
+		(void) hfs_recording_suspend(hfsmp);
+	}
+	
 	/* Change the timezone (Note: this affects all hfs volumes and hfs+ volume create dates) */
 	if (args->hfs_timezone.tz_minuteswest != VNOVAL) {
 		gTimeZone = args->hfs_timezone;
@@ -523,8 +552,6 @@ exit:
 
 struct hfs_reload_cargs {
 	struct hfsmount *hfsmp;
-        kauth_cred_t cred;
-        struct proc	*p;
         int		error;
 };
 
@@ -557,7 +584,7 @@ hfs_reload_callback(struct vnode *vp, void *cargs)
 		datafork = cp->c_datafork ? &cp->c_datafork->ff_data : NULL;
 
 		/* lookup by fileID since name could have changed */
-		if ((args->error = cat_idlookup(args->hfsmp, cp->c_fileid, &desc, &cp->c_attr, datafork)))
+		if ((args->error = cat_idlookup(args->hfsmp, cp->c_fileid, 0, &desc, &cp->c_attr, datafork)))
 		        return (VNODE_RETURNED_DONE);
 
 		/* update cnode's catalog descriptor */
@@ -581,7 +608,7 @@ hfs_reload_callback(struct vnode *vp, void *cargs)
  *	re-read cnode data for all active vnodes.
  */
 static int
-hfs_reload(struct mount *mountp, kauth_cred_t cred, struct proc *p)
+hfs_reload(struct mount *mountp)
 {
 	register struct vnode *devvp;
 	struct buf *bp;
@@ -608,8 +635,6 @@ hfs_reload(struct mount *mountp, kauth_cred_t cred, struct proc *p)
 		panic("hfs_reload: dirty1");
 
 	args.hfsmp = hfsmp;
-	args.cred = cred;
-	args.p = p;
 	args.error = 0;
 	/*
 	 * hfs_reload_callback will be called for each vnode
@@ -658,7 +683,7 @@ hfs_reload(struct mount *mountp, kauth_cred_t cred, struct proc *p)
 	vcb->vcbWrCnt		= SWAP_BE32 (vhp->writeCount);
 	vcb->vcbFilCnt		= SWAP_BE32 (vhp->fileCount);
 	vcb->vcbDirCnt		= SWAP_BE32 (vhp->folderCount);
-	vcb->nextAllocation 	= SWAP_BE32 (vhp->nextAllocation);
+	HFS_UPDATE_NEXT_ALLOCATION(vcb, SWAP_BE32 (vhp->nextAllocation));
 	vcb->totalBlocks	= SWAP_BE32 (vhp->totalBlocks);
 	vcb->freeBlocks		= SWAP_BE32 (vhp->freeBlocks);
 	vcb->encodingsBitmap	= SWAP_BE64 (vhp->encodingsBitmap);
@@ -736,14 +761,15 @@ hfs_reload(struct mount *mountp, kauth_cred_t cred, struct proc *p)
 	}
 
 	/* Reload the volume name */
-	if ((error = cat_idlookup(hfsmp, kHFSRootFolderID, &cndesc, NULL, NULL)))
+	if ((error = cat_idlookup(hfsmp, kHFSRootFolderID, 0, &cndesc, NULL, NULL)))
 		return (error);
 	vcb->volumeNameEncodingHint = cndesc.cd_encoding;
 	bcopy(cndesc.cd_nameptr, vcb->vcbVN, min(255, cndesc.cd_namelen));
 	cat_releasedesc(&cndesc);
 
-	/* Re-establish private/hidden directory for unlinked files */
-	FindMetaDataDirectory(vcb);
+	/* Re-establish private/hidden directories. */
+	hfs_privatedir_init(hfsmp, FILE_HARDLINKS);
+	hfs_privatedir_init(hfsmp, DIR_HARDLINKS);
 
 	/* In case any volume information changed to trigger a notification */
 	hfs_generate_volume_notifications(hfsmp);
@@ -757,7 +783,7 @@ hfs_reload(struct mount *mountp, kauth_cred_t cred, struct proc *p)
  */
 static int
 hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
-            vfs_context_t context)
+            int journal_replay_only, vfs_context_t context)
 {
 	struct proc *p = vfs_context_proc(context);
 	int retval = E_NONE;
@@ -766,7 +792,9 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	dev_t dev;
 	HFSMasterDirectoryBlock *mdbp;
 	int ronly;
+#if QUOTA
 	int i;
+#endif
 	int mntwrapper;
 	kauth_cred_t cred;
 	u_int64_t disksize;
@@ -775,6 +803,7 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	u_int32_t minblksize;
 	u_int32_t iswritable;
 	daddr64_t mdb_offset;
+	int isvirtual = 0;
 
 	ronly = vfs_isrdonly(mp);
 	dev = vnode_specrdev(devvp);
@@ -805,6 +834,9 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	}
 	/* Get the number of 512 byte physical blocks. */
 	if (VNOP_IOCTL(devvp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, context)) {
+		/* resetting block size may fail if getting block count did */
+		(void)VNOP_IOCTL(devvp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, context);
+
 		retval = ENXIO;
 		goto error_exit;
 	}
@@ -817,11 +849,11 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	 * worth of blocks but to insure compatibility with
 	 * pre-Tiger systems we have to do it.
 	 */
-	if (blkcnt > (u_int64_t)0x000000007fffffff) {
+	if (blkcnt > 0x000000007fffffff) {
 		minblksize = blksize = 4096;
 	}
 	
-	/* Now switch to our prefered physical block size. */
+	/* Now switch to our preferred physical block size. */
 	if (blksize > 512) {
 		if (VNOP_IOCTL(devvp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, context)) {
 			retval = ENXIO;
@@ -836,7 +868,7 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	/*
 	 * At this point:
 	 *   minblksize is the minimum physical block size
-	 *   blksize has our prefered physical block size
+	 *   blksize has our preferred physical block size
 	 *   blkcnt has the total number of physical blocks
 	 */
 
@@ -865,6 +897,7 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	hfsmp->hfs_mp = mp;			/* Make VFSTOHFS work */
 	hfsmp->hfs_raw_dev = vnode_specrdev(devvp);
 	hfsmp->hfs_devvp = devvp;
+	vnode_ref(devvp);  /* Hold a ref on the device, dropped when hfsmp is freed. */
 	hfsmp->hfs_phys_block_size = blksize;
 	hfsmp->hfs_phys_block_count = blkcnt;
 	hfsmp->hfs_flags |= HFS_WRITEABLE_MEDIA;
@@ -872,8 +905,11 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 		hfsmp->hfs_flags |= HFS_READ_ONLY;
 	if (((unsigned int)vfs_flags(mp)) & MNT_UNKNOWNPERMISSIONS)
 		hfsmp->hfs_flags |= HFS_UNKNOWN_PERMS;
+
+#if QUOTA
 	for (i = 0; i < MAXQUOTAS; i++)
 		dqfileinit(&hfsmp->hfs_qfiles[i]);
+#endif
 
 	if (args) {
 		hfsmp->hfs_uid = (args->hfs_uid == (uid_t)VNOVAL) ? UNKNOWNUID : args->hfs_uid;
@@ -914,15 +950,20 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	}
 
 	// record the current time at which we're mounting this volume
-	{
-	    struct timeval tv;
-	    microtime(&tv);
-	    hfsmp->hfs_mount_time = tv.tv_sec;
-	}
+	struct timeval tv;
+	microtime(&tv);
+	hfsmp->hfs_mount_time = tv.tv_sec;
 
 	/* Mount a standard HFS disk */
 	if ((SWAP_BE16(mdbp->drSigWord) == kHFSSigWord) &&
 	    (mntwrapper || (SWAP_BE16(mdbp->drEmbedSigWord) != kHFSPlusSigWord))) {
+
+	    	/* If only journal replay is requested, exit immediately */
+		if (journal_replay_only) {
+			retval = 0;
+			goto error_exit;
+		}
+
 	        if ((vfs_flags(mp) & MNT_ROOTFS)) {	
 			retval = EINVAL;  /* Cannot root from HFS standard disks */
 			goto error_exit;
@@ -1015,11 +1056,24 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 			vhp = (HFSPlusVolumeHeader*) mdbp;
 		}
 
+		/*
+		 * On inconsistent disks, do not allow read-write mount
+		 * unless it is the boot volume being mounted.
+		 */
+		if (!(vfs_flags(mp) & MNT_ROOTFS) &&
+				(SWAP_BE32(vhp->attributes) & kHFSVolumeInconsistentMask) &&
+				!(hfsmp->hfs_flags & HFS_READ_ONLY)) {
+			retval = EINVAL;
+			goto error_exit;
+		}
+
+
 		// XXXdbg
 		//
 		hfsmp->jnl = NULL;
 		hfsmp->jvp = NULL;
-		if (args != NULL && (args->flags & HFSFSMNT_EXTENDED_ARGS) && args->journal_disable) {
+		if (args != NULL && (args->flags & HFSFSMNT_EXTENDED_ARGS) && 
+		    args->journal_disable) {
 		    jnl_disable = 1;
 		}
 				
@@ -1040,7 +1094,7 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 			// point as journaled.
 			//
 			if (hfs_early_journal_init(hfsmp, vhp, args, embeddedOffset, mdb_offset, mdbp, cred) == 0) {
-				vfs_setflags(mp, (uint64_t)((unsigned int)MNT_JOURNALED));
+				vfs_setflags(mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
 			} else {
 				// if the journal failed to open, then set the lastMountedVersion
 				// to be "FSK!" which fsck_hfs will see and force the fsck instead
@@ -1086,6 +1140,15 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 		}
 		// XXXdbg
 	
+		/* Either the journal is replayed successfully, or there 
+		 * was nothing to replay, or no journal exists.  In any case,
+		 * return success.
+		 */
+		if (journal_replay_only) {
+			retval = 0;
+			goto error_exit;
+		}
+
 		(void) hfs_getconverter(0, &hfsmp->hfs_get_unicode, &hfsmp->hfs_get_hfsname);
 
 		retval = hfs_MountHFSPlusVolume(hfsmp, vhp, embeddedOffset, disksize, p, args, cred);
@@ -1115,7 +1178,7 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 			    journal_close(hfsmp->jnl);
 			    hfsmp->jnl = NULL;
 			    if (hfs_early_journal_init(hfsmp, vhp, args, embeddedOffset, mdb_offset, mdbp, cred) == 0) {
-					vfs_setflags(mp, (uint64_t)((unsigned int)MNT_JOURNALED));
+					vfs_setflags(mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
 				} else {
 					// if the journal failed to open, then set the lastMountedVersion
 					// to be "FSK!" which fsck_hfs will see and force the fsck instead
@@ -1180,6 +1243,16 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 	vfs_setmaxsymlen(mp, 0);
 	mp->mnt_vtable->vfc_threadsafe = TRUE;
 	mp->mnt_vtable->vfc_vfsflags |= VFC_VFSNATIVEXATTR;
+#if NAMEDSTREAMS
+	mp->mnt_kern_flag |= MNTK_NAMED_STREAMS;
+#endif
+	if (!(hfsmp->hfs_flags & HFS_STANDARD)) {
+		/* Tell VFS that we support directory hard links. */
+		mp->mnt_vtable->vfc_vfsflags |= VFC_VFSDIRLINKS;
+	} else {
+		/* HFS standard doesn't support extended readdir! */
+		mp->mnt_vtable->vfc_vfsflags &= ~VFC_VFSREADDIR_EXTENDED;
+	}
 
 	if (args) {
 		/*
@@ -1213,13 +1286,20 @@ hfs_mountfs(struct vnode *devvp, struct mount *mp, struct hfs_mount_args *args,
 				(HFSTOVCB(hfsmp)->totalBlocks / 100) * HFS_ROOTLOWDISKSHUTOFFFRACTION);
 	};
 	
+	/* Check if the file system exists on virtual device, like disk image */
+	if (VNOP_IOCTL(devvp, DKIOCISVIRTUAL, (caddr_t)&isvirtual, 0, context) == 0) {
+		if (isvirtual) {
+			hfsmp->hfs_flags |= HFS_VIRTUAL_DEVICE;
+		}
+	}
+
 	/*
 	 * Start looking for free space to drop below this level and generate a
 	 * warning immediately if needed:
 	 */
 	hfsmp->hfs_notification_conditions = 0;
 	hfs_generate_volume_notifications(hfsmp);
-	
+
 	if (ronly == 0) {
 		(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
 	}
@@ -1237,6 +1317,9 @@ error_exit:
 		hfsmp->jvp = NULL;
 	}
 	if (hfsmp) {
+		if (hfsmp->hfs_devvp) {
+			vnode_rele(hfsmp->hfs_devvp);
+		}
 		FREE(hfsmp, M_HFSMNT);
 		vfs_setfsprivate(mp, NULL);
 	}
@@ -1286,8 +1369,20 @@ hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context)
 	 * Flush out the b-trees, volume bitmap and Volume Header
 	 */
 	if ((hfsmp->hfs_flags & HFS_READ_ONLY) == 0) {
-		hfs_start_transaction(hfsmp);
-		started_tr = 1;
+		retval = hfs_start_transaction(hfsmp);
+		if (retval == 0) {
+		    started_tr = 1;
+		} else if (!force) {
+		    goto err_exit;
+		}
+
+		if (hfsmp->hfs_startup_vp) {
+			(void) hfs_lock(VTOC(hfsmp->hfs_startup_vp), HFS_EXCLUSIVE_LOCK);
+			retval = hfs_fsync(hfsmp->hfs_startup_vp, MNT_WAIT, 0, p);
+			hfs_unlock(VTOC(hfsmp->hfs_startup_vp));
+			if (retval && !force)
+				goto err_exit;
+		}
 
 		if (hfsmp->hfs_attribute_vp) {
 			(void) hfs_lock(VTOC(hfsmp->hfs_attribute_vp), HFS_EXCLUSIVE_LOCK);
@@ -1322,16 +1417,16 @@ hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context)
 			if (retval && !force)
 				goto err_exit;
 		}
-#if 0		
-		/* See if this volume is damaged, is so do not unmount cleanly */
-		if (HFSTOVCB(hfsmp)->vcbFlags & kHFS_DamagedVolume) {
+
+		/* If runtime corruption was detected, indicate that the volume
+		 * was not unmounted cleanly.
+		 */
+		if (hfsmp->vcbAtrb & kHFSVolumeInconsistentMask) {
 			HFSTOVCB(hfsmp)->vcbAtrb &= ~kHFSVolumeUnmountedMask;
 		} else {
 			HFSTOVCB(hfsmp)->vcbAtrb |= kHFSVolumeUnmountedMask;
 		}
-#else
-		HFSTOVCB(hfsmp)->vcbAtrb |= kHFSVolumeUnmountedMask;
-#endif
+
 		retval = hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
 		if (retval) {
 			HFSTOVCB(hfsmp)->vcbAtrb &= ~kHFSVolumeUnmountedMask;
@@ -1339,8 +1434,10 @@ hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context)
 				goto err_exit;	/* could not flush everything */
 		}
 
-		hfs_end_transaction(hfsmp);
-		started_tr = 0;
+		if (started_tr) {
+		    hfs_end_transaction(hfsmp);
+		    started_tr = 0;
+		}
 	}
 
 	if (hfsmp->jnl) {
@@ -1389,6 +1486,7 @@ hfs_unmount(struct mount *mp, int mntflags, vfs_context_t context)
 	}
 #endif /* HFS_SPARSE_DEV */
 	lck_mtx_destroy(&hfsmp->hfc_mutex, hfs_mutex_group);
+	vnode_rele(hfsmp->hfs_devvp);
 	FREE(hfsmp, M_HFSMNT);
 
 	return (0);
@@ -1414,16 +1512,20 @@ hfs_vfs_root(struct mount *mp, struct vnode **vpp, __unused vfs_context_t contex
 /*
  * Do operations associated with quotas
  */
+#if !QUOTA
+static int
+hfs_quotactl(__unused struct mount *mp, __unused int cmds, __unused uid_t uid, __unused caddr_t datap, __unused vfs_context_t context)
+{
+	return (ENOTSUP);
+}
+#else
 static int
 hfs_quotactl(struct mount *mp, int cmds, uid_t uid, caddr_t datap, vfs_context_t context)
 {
 	struct proc *p = vfs_context_proc(context);
 	int cmd, type, error;
 
-#if !QUOTA
-	return (ENOTSUP);
-#else
-	if (uid == -1)
+	if (uid == ~0U)
 		uid = vfs_context_ucred(context)->cr_ruid;
 	cmd = cmds >> SUBCMDSHIFT;
 
@@ -1483,8 +1585,8 @@ hfs_quotactl(struct mount *mp, int cmds, uid_t uid, caddr_t datap, vfs_context_t
 	vfs_unbusy(mp);
 
 	return (error);
-#endif /* QUOTA */
 }
+#endif /* QUOTA */
 
 /* Subtype is composite of bits */
 #define HFS_SUBTYPE_JOURNALED      0x01
@@ -1501,17 +1603,17 @@ hfs_statfs(struct mount *mp, register struct vfsstatfs *sbp, __unused vfs_contex
 	ExtendedVCB *vcb = VFSTOVCB(mp);
 	struct hfsmount *hfsmp = VFSTOHFS(mp);
 	u_long freeCNIDs;
-	uint16_t subtype = 0;
+	u_int16_t subtype = 0;
 
 	freeCNIDs = (u_long)0xFFFFFFFF - (u_long)vcb->vcbNxtCNID;
 
-	sbp->f_bsize = (uint32_t)vcb->blockSize;
+	sbp->f_bsize = (u_int32_t)vcb->blockSize;
 	sbp->f_iosize = (size_t)(MAX_UPL_TRANSFER * PAGE_SIZE);
-	sbp->f_blocks = (uint64_t)((unsigned long)vcb->totalBlocks);
-	sbp->f_bfree = (uint64_t)((unsigned long )hfs_freeblks(hfsmp, 0));
-	sbp->f_bavail = (uint64_t)((unsigned long )hfs_freeblks(hfsmp, 1));
-	sbp->f_files = (uint64_t)((unsigned long )(vcb->totalBlocks - 2));  /* max files is constrained by total blocks */
-	sbp->f_ffree = (uint64_t)((unsigned long )(MIN(freeCNIDs, sbp->f_bavail)));
+	sbp->f_blocks = (u_int64_t)((unsigned long)vcb->totalBlocks);
+	sbp->f_bfree = (u_int64_t)((unsigned long )hfs_freeblks(hfsmp, 0));
+	sbp->f_bavail = (u_int64_t)((unsigned long )hfs_freeblks(hfsmp, 1));
+	sbp->f_files = (u_int64_t)((unsigned long )(vcb->totalBlocks - 2));  /* max files is constrained by total blocks */
+	sbp->f_ffree = (u_int64_t)((unsigned long )(MIN(freeCNIDs, sbp->f_bavail)));
 
 	/*
 	 * Subtypes (flavors) for HFS
@@ -1563,9 +1665,9 @@ hfs_sync_metadata(void *arg)
 	priIDSector = (daddr64_t)((vcb->hfsPlusIOPosOffset / sectorsize) +
 				  HFS_PRI_SECTOR(sectorsize));
 	retval = (int)buf_meta_bread(hfsmp->hfs_devvp, priIDSector, sectorsize, NOCRED, &bp);
-	if ((retval != 0) && (retval != ENXIO)) {
+	if ((retval != 0 ) && (retval != ENXIO)) {
 		printf("hfs_sync_metadata: can't read volume header at %d! (retval 0x%x)\n",
-			priIDSector, retval);
+		       (int)priIDSector, retval);
 	}
 
 	if (retval == 0 && ((buf_flags(bp) & (B_DELWRI | B_LOCKED)) == B_DELWRI)) {
@@ -1648,7 +1750,7 @@ hfs_sync(struct mount *mp, int waitfor, vfs_context_t context)
 	 * During MNT_UPDATE hfs_changefs might be manipulating
 	 * vnodes so back off
 	 */
-	if (((uint32_t)vfs_flags(mp)) & MNT_UPDATE)	/* XXX MNT_UPDATE may not be visible here */
+	if (((u_int32_t)vfs_flags(mp)) & MNT_UPDATE)	/* XXX MNT_UPDATE may not be visible here */
 		return (0);
 
 	hfsmp = VFSTOHFS(mp);
@@ -1659,7 +1761,7 @@ hfs_sync(struct mount *mp, int waitfor, vfs_context_t context)
 	if (!lck_rw_try_lock_shared(&hfsmp->hfs_insync))
 		return 0;
 
-	args.cred = vfs_context_proc(context);
+	args.cred = kauth_cred_get();
 	args.waitfor = waitfor;
 	args.p = p;
 	args.error = 0;
@@ -1724,11 +1826,11 @@ hfs_sync(struct mount *mp, int waitfor, vfs_context_t context)
 	hfs_qsync(mp);
 #endif /* QUOTA */
 
-	hfs_hotfilesync(hfsmp, p);
+	hfs_hotfilesync(hfsmp, vfs_context_kernel());
+
 	/*
 	 * Write back modified superblock.
 	 */
-
 	if (IsVCBDirty(vcb)) {
 		error = hfs_flushvolumeheader(hfsmp, waitfor, 0);
 		if (error)
@@ -1755,7 +1857,7 @@ hfs_sync(struct mount *mp, int waitfor, vfs_context_t context)
  *   those rights via. exflagsp and credanonp
  */
 static int
-hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp, vfs_context_t context)
+hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp, __unused vfs_context_t context)
 {
 	struct hfsfid *hfsfhp;
 	struct vnode *nvp;
@@ -1764,7 +1866,7 @@ hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp,
 	*vpp = NULL;
 	hfsfhp = (struct hfsfid *)fhp;
 
-	if (fhlen < sizeof(struct hfsfid))
+	if (fhlen < (int)sizeof(struct hfsfid))
 		return (EINVAL);
 
 	result = hfs_vget(VFSTOHFS(mp), ntohl(hfsfhp->hfsfid_cnid), &nvp, 0);
@@ -1785,7 +1887,7 @@ hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp,
 	 * error prone. Future, would be change the "wrap bit" to a unique
 	 * wrap number and use that for generation number. For now do this.
 	 */  
-	if ((ntohl(hfsfhp->hfsfid_gen) < VTOC(nvp)->c_itime)) {
+	if (((time_t)(ntohl(hfsfhp->hfsfid_gen)) < VTOC(nvp)->c_itime)) {
 		hfs_unlock(VTOC(nvp));
 		vnode_put(nvp);
 		return (ESTALE);
@@ -1802,7 +1904,7 @@ hfs_fhtovp(struct mount *mp, int fhlen, unsigned char *fhp, struct vnode **vpp,
  */
 /* ARGSUSED */
 static int
-hfs_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, vfs_context_t context)
+hfs_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, __unused vfs_context_t context)
 {
 	struct cnode *cp;
 	struct hfsfid *hfsfhp;
@@ -1836,9 +1938,6 @@ hfs_init(__unused struct vfsconf *vfsp)
 	done = 1;
 	hfs_chashinit();
 	hfs_converterinit();
-#if QUOTA
-	dqinit();
-#endif /* QUOTA */
 
 	BTReserveSetup();
 	
@@ -1853,9 +1952,7 @@ hfs_init(__unused struct vfsconf *vfsp)
 }
 
 static int
-hfs_getmountpoint(vp, hfsmpp)
-	struct vnode *vp;
-	struct hfsmount **hfsmpp;
+hfs_getmountpoint(struct vnode *vp, struct hfsmount **hfsmpp)
 {
 	struct hfsmount * hfsmp;
 	char fstypename[MFSNAMELEN];
@@ -1867,7 +1964,7 @@ hfs_getmountpoint(vp, hfsmpp)
 		return (EINVAL);
 
 	vnode_vfsname(vp, fstypename);
-	if (strcmp(fstypename, "hfs") != 0)
+	if (strncmp(fstypename, "hfs", sizeof(fstypename)) != 0)
 		return (EINVAL);
 
 	hfsmp = VTOHFS(vp);
@@ -1897,7 +1994,7 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	/* all sysctl names at this level are terminal */
 
 	if (name[0] == HFS_ENCODINGBIAS) {
-		u_int32_t bias;
+		int bias;
 
 		bias = hfs_getencodingbias();
 		error = sysctl_int(oldp, oldlenp, newp, newlen, &bias);
@@ -1907,13 +2004,13 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 
 	} else if (name[0] == HFS_EXTEND_FS) {
         u_int64_t  newsize;
-		vnode_t vp = p->p_fd->fd_cdir;
+		vnode_t vp = vfs_context_cwd(context);
 
-		if (newp == USER_ADDR_NULL || vp == NULL)
+		if (newp == USER_ADDR_NULL || vp == NULLVP)
 			return (EINVAL);
 		if ((error = hfs_getmountpoint(vp, &hfsmp)))
 			return (error);
-		error = sysctl_quad(oldp, oldlenp, newp, newlen, &newsize);
+		error = sysctl_quad(oldp, oldlenp, newp, newlen, (quad_t *)&newsize);
 		if (error)
 			return (error);
 	
@@ -1927,17 +2024,20 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		u_int16_t *unicode_name;
 		char *filename;
 
+		if ((newlen <= 0) || (newlen > MAXPATHLEN)) 
+			return (EINVAL);
+
 		bufsize = MAX(newlen * 3, MAXPATHLEN);
 		MALLOC(filename, char *, newlen, M_TEMP, M_WAITOK);
 		MALLOC(unicode_name, u_int16_t *, bufsize, M_TEMP, M_WAITOK);
 
 		error = copyin(newp, (caddr_t)filename, newlen);
 		if (error == 0) {
-			error = utf8_decodestr(filename, newlen - 1, unicode_name,
+			error = utf8_decodestr((u_int8_t *)filename, newlen - 1, unicode_name,
 			                       &bytes, bufsize, 0, UTF_DECOMPOSED);
 			if (error == 0) {
 				hint = hfs_pickencoding(unicode_name, bytes / 2);
-				error = sysctl_int(oldp, oldlenp, USER_ADDR_NULL, 0, &hint);
+				error = sysctl_int(oldp, oldlenp, USER_ADDR_NULL, 0, (int32_t *)&hint);
 			}
 		}
 		FREE(unicode_name, M_TEMP);
@@ -1946,7 +2046,8 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 
 	} else if (name[0] == HFS_ENABLE_JOURNALING) {
 		// make the file system journaled...
-		struct vnode *vp = p->p_fd->fd_cdir, *jvp;
+		vnode_t vp = vfs_context_cwd(context);
+		vnode_t jvp;
 		ExtendedVCB *vcb;
 		struct cat_attr jnl_attr, jinfo_attr;
 		struct cat_fork jnl_fork, jinfo_fork;
@@ -1957,7 +2058,7 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		if (!is_suser()) {
 			return (EPERM);
 		}
-		if (vp == NULL)
+		if (vp == NULLVP)
 		        return EINVAL;
 
 		hfsmp = VTOHFS(vp);
@@ -1970,7 +2071,7 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		}
 
 		if (hfsmp->jnl) {
-		    printf("hfs: volume @ mp 0x%x is already journaled!\n", vnode_mount(vp));
+		    printf("hfs: volume @ mp %p is already journaled!\n", vnode_mount(vp));
 		    return EAGAIN;
 		}
 
@@ -2041,7 +2142,7 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		hfsmp->hfs_jnlinfoblkid = jinfo_attr.ca_fileid;
 		hfsmp->hfs_jnlfileid    = jnl_attr.ca_fileid;
 
-		vfs_setflags(hfsmp->hfs_mp, (uint64_t)((unsigned int)MNT_JOURNALED));
+		vfs_setflags(hfsmp->hfs_mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
 
 		hfs_global_exclusive_lock_release(hfsmp);
 		hfs_flushvolumeheader(hfsmp, MNT_WAIT, 1);
@@ -2049,18 +2150,27 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		return 0;
 	} else if (name[0] == HFS_DISABLE_JOURNALING) {
 		// clear the journaling bit 
-		struct vnode *vp = p->p_fd->fd_cdir;
+		vnode_t vp = vfs_context_cwd(context);
 		
 		/* Only root can disable journaling */
 		if (!is_suser()) {
 			return (EPERM);
 		}
-		if (vp == NULL)
+		if (vp == NULLVP)
 		        return EINVAL;
 
 		hfsmp = VTOHFS(vp);
 
-		printf("hfs: disabling journaling for mount @ 0x%x\n", vnode_mount(vp));
+		/* 
+		 * Disabling journaling is disallowed on volumes with directory hard links
+		 * because we have not tested the relevant code path.
+		 */  
+		if (hfsmp->hfs_private_attr[DIR_HARDLINKS].ca_entries != 0){
+			printf("hfs: cannot disable journaling on volumes with directory hardlinks\n");
+			return EPERM;
+		}
+
+		printf("hfs: disabling journaling for mount @ %p\n", vnode_mount(vp));
 
 		hfs_global_exclusive_lock_acquire(hfsmp);
 
@@ -2072,7 +2182,7 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 			VNOP_CLOSE(hfsmp->jvp, hfsmp->hfs_flags & HFS_READ_ONLY ? FREAD : FREAD|FWRITE, context);
 		}
 		hfsmp->jvp = NULL;
-		vfs_clearflags(hfsmp->hfs_mp, (uint64_t)((unsigned int)MNT_JOURNALED));
+		vfs_clearflags(hfsmp->hfs_mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
 		hfsmp->jnl_start        = 0;
 		hfsmp->hfs_jnlinfoblkid = 0;
 		hfsmp->hfs_jnlfileid    = 0;
@@ -2084,10 +2194,10 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 
 		return 0;
 	} else if (name[0] == HFS_GET_JOURNAL_INFO) {
-		struct vnode *vp = p->p_fd->fd_cdir;
+		vnode_t vp = vfs_context_cwd(context);
 		off_t jnl_start, jnl_size;
 
-		if (vp == NULL)
+		if (vp == NULLVP)
 		        return EINVAL;
 
 		hfsmp = VTOHFS(vp);
@@ -2140,7 +2250,27 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		bzero(&vq, sizeof(vq));
 		vq.vq_flags = hfsmp->hfs_notification_conditions;
 		return SYSCTL_OUT(req, &vq, sizeof(vq));;
-	};
+	} else if (name[0] == HFS_REPLAY_JOURNAL) {
+		char *devnode = NULL;
+		size_t devnode_len;
+
+		devnode_len = *oldlenp;
+		MALLOC(devnode, char *, devnode_len + 1, M_TEMP, M_WAITOK);
+		if (devnode == NULL) {
+			return ENOMEM;
+		}
+
+		error = copyin(oldp, (caddr_t)devnode, devnode_len);
+		if (error) {
+			FREE(devnode, M_TEMP);
+			return error;
+		}
+		devnode[devnode_len] = 0;
+
+		error = hfs_journal_replay(devnode, context);
+		FREE(devnode, M_TEMP);
+		return error;
+	}
 
 	return (ENOTSUP);
 }
@@ -2149,7 +2279,33 @@ hfs_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp,
 static int
 hfs_vfs_vget(struct mount *mp, ino64_t ino, struct vnode **vpp, __unused vfs_context_t context)
 {
-	return hfs_vget(VFSTOHFS(mp), (cnid_t)ino, vpp, 1);
+	int error;
+
+	error = hfs_vget(VFSTOHFS(mp), (cnid_t)ino, vpp, 1);
+	if (error)
+		return (error);
+
+	/*
+	 * ADLs may need to have their origin state updated
+	 * since build_path needs a valid parent.
+	 */
+	if (vnode_isdir(*vpp) &&
+	    (VTOC(*vpp)->c_flag & C_HARDLINK) &&
+	    (hfs_lock(VTOC(*vpp), HFS_EXCLUSIVE_LOCK) == 0)) {
+		cnode_t *cp = VTOC(*vpp);
+		struct cat_desc cdesc;
+		
+		if (!hfs_haslinkorigin(cp) &&
+		    (cat_findname(VFSTOHFS(mp), (cnid_t)ino, &cdesc) == 0)) {
+			if (cdesc.cd_parentcnid !=
+			    VFSTOHFS(mp)->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
+				hfs_savelinkorigin(cp, cdesc.cd_parentcnid);
+			}
+			cat_releasedesc(&cdesc);
+		}
+		hfs_unlock(cp);
+	}
+	return (0);
 }
 
 
@@ -2164,23 +2320,23 @@ __private_extern__
 int
 hfs_vget(struct hfsmount *hfsmp, cnid_t cnid, struct vnode **vpp, int skiplock)
 {
-	struct vnode *vp = NULL;
+	struct vnode *vp = NULLVP;
 	struct cat_desc cndesc;
 	struct cat_attr cnattr;
 	struct cat_fork cnfork;
-	struct componentname cn;
 	u_int32_t linkref = 0;
 	int error;
 	
 	/* Check for cnids that should't be exported. */
-	if ((cnid < kHFSFirstUserCatalogNodeID)
-	&&  (cnid != kHFSRootFolderID && cnid != kHFSRootParentID))
+	if ((cnid < kHFSFirstUserCatalogNodeID) &&
+	    (cnid != kHFSRootFolderID && cnid != kHFSRootParentID)) {
 		return (ENOENT);
-
-	/* Don't export HFS Private Data dir. */
-	if (cnid == hfsmp->hfs_privdir_desc.cd_cnid)
+	}
+	/* Don't export our private directories. */
+	if (cnid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid ||
+	    cnid == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) {
 		return (ENOENT);
-
+	}
 	/*
 	 * Check the hash first
 	 */
@@ -2200,21 +2356,24 @@ hfs_vget(struct hfsmount *hfsmp, cnid_t cnid, struct vnode **vpp, int skiplock)
 	if (cnid == kHFSRootParentID) {
 		static char hfs_rootname[] = "/";
 
-		cndesc.cd_nameptr = &hfs_rootname[0];
+		cndesc.cd_nameptr = (const u_int8_t *)&hfs_rootname[0];
 		cndesc.cd_namelen = 1;
 		cndesc.cd_parentcnid = kHFSRootParentID;
 		cndesc.cd_cnid = kHFSRootFolderID;
 		cndesc.cd_flags = CD_ISDIR;
 
 		cnattr.ca_fileid = kHFSRootFolderID;
-		cnattr.ca_nlink = 2;
+		cnattr.ca_linkcount = 1;
 		cnattr.ca_entries = 1;
+		cnattr.ca_dircount = 1;
 		cnattr.ca_mode = (S_IFDIR | S_IRWXU | S_IRWXG | S_IRWXO);
 	} else {
 		int lockflags;
+		cnid_t pid;
+		const char *nameptr;
 
 		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-		error = cat_idlookup(hfsmp, cnid, &cndesc, &cnattr, &cnfork);
+		error = cat_idlookup(hfsmp, cnid, 0, &cndesc, &cnattr, &cnfork);
 		hfs_systemfile_unlock(hfsmp, lockflags);
 
 		if (error) {
@@ -2223,42 +2382,85 @@ hfs_vget(struct hfsmount *hfsmp, cnid_t cnid, struct vnode **vpp, int skiplock)
 		}
 
 		/*
-		 * If we just looked up a raw hardlink inode,
-		 * then finish initializing it.
+		 * Check for a raw hardlink inode and save its linkref.
 		 */
-		if ((cndesc.cd_parentcnid == hfsmp->hfs_privdir_desc.cd_cnid) &&
-		    (bcmp(cndesc.cd_nameptr, HFS_INODE_PREFIX, HFS_INODE_PREFIX_LEN) == 0)) {
-			linkref = strtoul((const char*)&cndesc.cd_nameptr[HFS_INODE_PREFIX_LEN], NULL, 10);
-			cnattr.ca_rdev = linkref;
+		pid = cndesc.cd_parentcnid;
+		nameptr = (const char *)cndesc.cd_nameptr;
+
+		if ((pid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) &&
+		    (bcmp(nameptr, HFS_INODE_PREFIX, HFS_INODE_PREFIX_LEN) == 0)) {
+			linkref = strtoul(&nameptr[HFS_INODE_PREFIX_LEN], NULL, 10);
+
+		} else if ((pid == hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid) &&
+		           (bcmp(nameptr, HFS_DIRINODE_PREFIX, HFS_DIRINODE_PREFIX_LEN) == 0)) {
+			linkref = strtoul(&nameptr[HFS_DIRINODE_PREFIX_LEN], NULL, 10);
+
+		} else if ((pid == hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) &&
+		           (bcmp(nameptr, HFS_DELETE_PREFIX, HFS_DELETE_PREFIX_LEN) == 0)) {
+			*vpp = NULL;
+			return (ENOENT);  /* open unlinked file */
 		}
 	}
 
 	/*
-	 * Supply hfs_getnewvnode with a component name. 
+	 * Finish initializing cnode descriptor for hardlinks.
+	 *
+	 * We need a valid name and parent for reverse lookups.
 	 */
-	MALLOC_ZONE(cn.cn_pnbuf, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
-	cn.cn_nameiop = LOOKUP;
-	cn.cn_flags = ISLASTCN | HASBUF;
-	cn.cn_context = NULL;
-	cn.cn_pnlen = MAXPATHLEN;
-	cn.cn_nameptr = cn.cn_pnbuf;
-	cn.cn_namelen = cndesc.cd_namelen;
-	cn.cn_hash = 0;
-	cn.cn_consume = 0;
-	bcopy(cndesc.cd_nameptr, cn.cn_nameptr, cndesc.cd_namelen + 1);
+	if (linkref) {
+		cnid_t nextlinkid;
+		cnid_t prevlinkid;
+		struct cat_desc linkdesc;
+
+		cnattr.ca_linkref = linkref;
 
-	/* XXX should we supply the parent as well... ? */
-	error = hfs_getnewvnode(hfsmp, NULLVP, &cn, &cndesc, 0, &cnattr, &cnfork, &vp);
-	if (error == 0 && linkref != 0) {
-		VTOC(vp)->c_flag |= C_HARDLINK;
+		/*
+		 * Pick up the first link in the chain and get a descriptor for it.
+		 * This allows blind volfs paths to work for hardlinks.
+		 */
+		if ((hfs_lookuplink(hfsmp, linkref, &prevlinkid,  &nextlinkid) == 0) &&
+		    (nextlinkid != 0)) {
+			if (cat_findname(hfsmp, nextlinkid, &linkdesc) == 0) {
+				cat_releasedesc(&cndesc);
+				bcopy(&linkdesc, &cndesc, sizeof(linkdesc));
+			}
+		}	
 	}
 
-	FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+	if (linkref) {
+		error = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cnfork, &vp);
+		if (error == 0) {
+			VTOC(vp)->c_flag |= C_HARDLINK;
+			vnode_setmultipath(vp);
+		}
+	} else {
+		struct componentname cn;
+
+		/* Supply hfs_getnewvnode with a component name. */
+		MALLOC_ZONE(cn.cn_pnbuf, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
+		cn.cn_nameiop = LOOKUP;
+		cn.cn_flags = ISLASTCN | HASBUF;
+		cn.cn_context = NULL;
+		cn.cn_pnlen = MAXPATHLEN;
+		cn.cn_nameptr = cn.cn_pnbuf;
+		cn.cn_namelen = cndesc.cd_namelen;
+		cn.cn_hash = 0;
+		cn.cn_consume = 0;
+		bcopy(cndesc.cd_nameptr, cn.cn_nameptr, cndesc.cd_namelen + 1);
+	
+		error = hfs_getnewvnode(hfsmp, NULLVP, &cn, &cndesc, 0, &cnattr, &cnfork, &vp);
 
+		if (error == 0 && (VTOC(vp)->c_flag & C_HARDLINK) && vnode_isdir(vp)) {
+			hfs_savelinkorigin(VTOC(vp), cndesc.cd_parentcnid);
+		}
+		FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+	}
 	cat_releasedesc(&cndesc);
+
 	*vpp = vp;
-	if (vp && skiplock)
+	if (vp && skiplock) {
 		hfs_unlock(VTOC(vp));
+	}
 	return (error);
 }
 
@@ -2267,13 +2469,19 @@ hfs_vget(struct hfsmount *hfsmp, cnid_t cnid, struct vnode **vpp, int skiplock)
  * Flush out all the files in a filesystem.
  */
 static int
+#if QUOTA
 hfs_flushfiles(struct mount *mp, int flags, struct proc *p)
+#else
+hfs_flushfiles(struct mount *mp, int flags, __unused struct proc *p)
+#endif /* QUOTA */
 {
 	struct hfsmount *hfsmp;
 	struct vnode *skipvp = NULLVP;
+	int error;
+#if QUOTA
 	int quotafilecnt;
 	int i;
-	int error;
+#endif
 
 	hfsmp = VFSTOHFS(mp);
 
@@ -2343,7 +2551,7 @@ hfs_setencodingbits(struct hfsmount *hfsmp, u_int32_t encoding)
 #define  kIndexMacUkrainian	48  /* MacUkrainian encoding is 152 */
 #define  kIndexMacFarsi		49  /* MacFarsi encoding is 140 */
 
-	UInt32	index;
+	u_int32_t	index;
 
 	switch (encoding) {
 	case kTextEncodingMacUkrainian:
@@ -2357,10 +2565,10 @@ hfs_setencodingbits(struct hfsmount *hfsmp, u_int32_t encoding)
 		break;
 	}
 
-	if (index < 64) {
+	if (index < 64 && (hfsmp->encodingsBitmap & (u_int64_t)(1ULL << index)) == 0) {
 		HFS_MOUNT_LOCK(hfsmp, TRUE)
 		hfsmp->encodingsBitmap |= (u_int64_t)(1ULL << index);
-		hfsmp->vcbFlags |= 0xFF00;
+		MarkVCBDirty(hfsmp);
 		HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 	}
 }
@@ -2380,7 +2588,7 @@ hfs_volupdate(struct hfsmount *hfsmp, enum volop op, int inroot)
 
 	lck_mtx_lock(&hfsmp->hfs_mutex);
 
-	hfsmp->vcbFlags |= 0xFF00;
+	MarkVCBDirty(hfsmp);
 	hfsmp->hfs_mtime = tv.tv_sec;
 
 	switch (op) {
@@ -2454,7 +2662,7 @@ hfs_flushMDB(struct hfsmount *hfsmp, int waitfor, int altflush)
 	mdb->drNxtCNID	= SWAP_BE32 (vcb->vcbNxtCNID);
 	mdb->drFreeBks	= SWAP_BE16 (vcb->freeBlocks);
 
-	namelen = strlen(vcb->vcbVN);
+	namelen = strlen((char *)vcb->vcbVN);
 	retval = utf8_to_hfs(vcb, namelen, vcb->vcbVN, mdb->drVN);
 	/* Retry with MacRoman in case that's how it was exported. */
 	if (retval)
@@ -2534,18 +2742,17 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 	int i;
 	int sectorsize;
 	daddr64_t priIDSector;
-	int critical = 0;
+	int critical;
 	u_int16_t  signature;
 	u_int16_t  hfsversion;
 
 	if (hfsmp->hfs_flags & HFS_READ_ONLY) {
 		return(0);
 	}
-	if (vcb->vcbSigWord == kHFSSigWord)
+	if (hfsmp->hfs_flags & HFS_STANDARD) {
 		return hfs_flushMDB(hfsmp, waitfor, altflush);
-
-	if (altflush)
-		critical = 1;
+	}
+	critical = altflush;
 	sectorsize = hfsmp->hfs_phys_block_size;
 	priIDSector = (daddr64_t)((vcb->hfsPlusIOPosOffset / sectorsize) +
 				  HFS_PRI_SECTOR(sectorsize));
@@ -2617,7 +2824,7 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 				mdb->drCrDate = SWAP_BE32 (vcb->localCreateDate);	/* pick up the new create date */
 
 				if (hfsmp->jnl) {
-					journal_modify_block_end(hfsmp->jnl, bp2);
+					journal_modify_block_end(hfsmp->jnl, bp2, NULL, NULL);
 				} else {
 					(void) VNOP_BWRITE(bp2);		/* write out the changes */
 				}
@@ -2629,9 +2836,7 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 		  }	
 	}
 
-	if (1 /* hfsmp->jnl == 0 */) {
-		lck_mtx_lock(&hfsmp->hfs_mutex);
-	}
+	lck_mtx_lock(&hfsmp->hfs_mutex);
 
 	/* Note: only update the lower 16 bits worth of attributes */
 	volumeHeader->attributes       = SWAP_BE32 (vcb->vcbAtrb);
@@ -2660,6 +2865,13 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 		critical = 1;
 	}
 
+	/*
+	 * System files are only dirty when altflush is set.
+	 */
+	if (altflush == 0) {
+		goto done;
+	}
+
 	/* Sync Extents over-flow file meta data */
 	fp = VTOF(vcb->extentsRefNum);
 	if (FTOC(fp)->c_flag & C_MODIFIED) {
@@ -2720,12 +2932,27 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 		volumeHeader->attributesFile.clumpSize   = SWAP_BE32 (fp->ff_clumpsize);
 	}
 
-	vcb->vcbFlags &= 0x00FF;
-
-	if (1 /* hfsmp->jnl == 0 */) {
-		lck_mtx_unlock(&hfsmp->hfs_mutex);
+	/* Sync Startup file meta data */
+	if (hfsmp->hfs_startup_vp) {
+		fp = VTOF(hfsmp->hfs_startup_vp);
+		if (FTOC(fp)->c_flag & C_MODIFIED) {
+			for (i = 0; i < kHFSPlusExtentDensity; i++) {
+				volumeHeader->startupFile.extents[i].startBlock =
+					SWAP_BE32 (fp->ff_extents[i].startBlock);
+				volumeHeader->startupFile.extents[i].blockCount =
+					SWAP_BE32 (fp->ff_extents[i].blockCount);
+			}
+			volumeHeader->startupFile.logicalSize = SWAP_BE64 (fp->ff_size);
+			volumeHeader->startupFile.totalBlocks = SWAP_BE32 (fp->ff_blocks);
+			volumeHeader->startupFile.clumpSize   = SWAP_BE32 (fp->ff_clumpsize);
+			FTOC(fp)->c_flag &= ~C_MODIFIED;
+		}
 	}
 
+done:
+	MarkVCBClean(hfsmp);
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
+
 	/* If requested, flush out the alternate volume header */
 	if (altflush && hfsmp->hfs_alt_id_sector) {
 		struct buf *alt_bp = NULL;
@@ -2738,7 +2965,7 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 			bcopy(volumeHeader, (char *)buf_dataptr(alt_bp) + HFS_ALT_OFFSET(sectorsize), kMDBSize);
 
 			if (hfsmp->jnl) {
-				journal_modify_block_end(hfsmp->jnl, alt_bp);
+				journal_modify_block_end(hfsmp->jnl, alt_bp, NULL, NULL);
 			} else {
 				(void) VNOP_BWRITE(alt_bp);
 			}
@@ -2747,7 +2974,7 @@ hfs_flushvolumeheader(struct hfsmount *hfsmp, int waitfor, int altflush)
 	}
 
 	if (hfsmp->jnl) {
-		journal_modify_block_end(hfsmp->jnl, bp);
+		journal_modify_block_end(hfsmp->jnl, bp, NULL, NULL);
 	} else {
 		if (waitfor != MNT_WAIT)
 			buf_bawrite(bp);
@@ -2791,7 +3018,9 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 	daddr_t	   bitmapblks;
 	int  lockflags;
 	int  error;
-
+	int64_t oldBitmapSize;
+	Boolean  usedExtendFileC = false;
+	
 	devvp = hfsmp->hfs_devvp;
 	vcb = HFSTOVCB(hfsmp);
 
@@ -2863,7 +3092,11 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 		return (EINVAL);
 	}
 
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+	/*
+	 * Note: we take the attributes lock in case we have an attribute data vnode
+	 * which needs to change size.
+	 */
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE | SFL_EXTENTS | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
 	vp = vcb->allocationsRefNum;
 	fp = VTOF(vp);
 	bcopy(&fp->ff_data, &forkdata, sizeof(forkdata));
@@ -2871,7 +3104,8 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 	/*
 	 * Calculate additional space required (if any) by allocation bitmap.
 	 */
-	bitmapblks = roundup(newblkcnt / 8, vcb->vcbVBMIOSize) / vcb->blockSize;
+	oldBitmapSize = fp->ff_size;
+	bitmapblks = roundup((newblkcnt+7) / 8, vcb->vcbVBMIOSize) / vcb->blockSize;
 	if (bitmapblks > (daddr_t)fp->ff_blocks)
 		bitmapblks -= fp->ff_blocks;
 	else
@@ -2880,26 +3114,59 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 	if (bitmapblks > 0) {
 		daddr64_t blkno;
 		daddr_t blkcnt;
+		off_t bytesAdded;
 
 		/*
-		 * Add a new extent to the allocation bitmap file.
+		 * Get the bitmap's current size (in allocation blocks) so we know
+		 * where to start zero filling once the new space is added.  We've
+		 * got to do this before the bitmap is grown.
 		 */
-		error = AddFileExtent(vcb, fp, vcb->totalBlocks, bitmapblks);
-		if (error) {
-			printf("hfs_extendfs: error %d adding extents\n", error);
-			goto out;
-		}
-		blkcnt = bitmapblks;
 		blkno  = (daddr64_t)fp->ff_blocks;
-		fp->ff_blocks += bitmapblks;
+
+		/*
+		 * Try to grow the allocation file in the normal way, using allocation
+		 * blocks already existing in the file system.  This way, we might be
+		 * able to grow the bitmap contiguously, or at least in the metadata
+		 * zone.
+		 */
+		error = ExtendFileC(vcb, fp, bitmapblks * vcb->blockSize, 0,
+				kEFAllMask | kEFNoClumpMask | kEFReserveMask | kEFMetadataMask,
+				&bytesAdded);
+
+		if (error == 0) {
+			usedExtendFileC = true;
+		} else {
+			/*
+			 * If the above allocation failed, fall back to allocating the new
+			 * extent of the bitmap from the space we're going to add.  Since those
+			 * blocks don't yet belong to the file system, we have to update the
+			 * extent list directly, and manually adjust the file size.
+			 */
+			bytesAdded = 0;
+			error = AddFileExtent(vcb, fp, vcb->totalBlocks, bitmapblks);
+			if (error) {
+				printf("hfs_extendfs: error %d adding extents\n", error);
+				goto out;
+			}
+			fp->ff_blocks += bitmapblks;
+			VTOC(vp)->c_blocks = fp->ff_blocks;
+			VTOC(vp)->c_flag |= C_MODIFIED;
+		}
+		
+		/*
+		 * Update the allocation file's size to include the newly allocated
+		 * blocks.  Note that ExtendFileC doesn't do this, which is why this
+		 * statement is outside the above "if" statement.
+		 */
 		fp->ff_size += (u_int64_t)bitmapblks * (u_int64_t)vcb->blockSize;
-		VTOC(vp)->c_blocks = fp->ff_blocks;
+		
 		/*
 		 * Zero out the new bitmap blocks.
 		 */
 		{
 	
 			bp = NULL;
+			blkcnt = bitmapblks;
 			while (blkcnt > 0) {
 				error = (int)buf_meta_bread(vp, blkno, vcb->blockSize, NOCRED, &bp);
 				if (error) {
@@ -2923,11 +3190,20 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 		}
 		/*
 		 * Mark the new bitmap space as allocated.
+		 *
+		 * Note that ExtendFileC will have marked any blocks it allocated, so
+		 * this is only needed if we used AddFileExtent.  Also note that this
+		 * has to come *after* the zero filling of new blocks in the case where
+		 * we used AddFileExtent (since the part of the bitmap we're touching
+		 * is in those newly allocated blocks).
 		 */
-		error = BlockMarkAllocated(vcb, vcb->totalBlocks, bitmapblks);
-		if (error) {
-			printf("hfs_extendfs: error %d setting bitmap\n", error);
-			goto out;
+		if (!usedExtendFileC) {
+			error = BlockMarkAllocated(vcb, vcb->totalBlocks, bitmapblks);
+			if (error) {
+				printf("hfs_extendfs: error %d setting bitmap\n", error);
+				goto out;
+			}
+			vcb->freeBlocks -= bitmapblks;
 		}
 	}
 	/*
@@ -2955,7 +3231,7 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 	prev_alt_sector = hfsmp->hfs_alt_id_sector;
 
 	vcb->totalBlocks += addblks;
-	vcb->freeBlocks += addblks - bitmapblks;
+	vcb->freeBlocks += addblks;
 	hfsmp->hfs_phys_block_count = newsize / sectorsize;
 	hfsmp->hfs_alt_id_sector = (hfsmp->hfsPlusIOPosOffset / sectorsize) +
 	                          HFS_ALT_SECTOR(sectorsize, hfsmp->hfs_phys_block_count);
@@ -2966,9 +3242,20 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 		/*
 		 * Restore to old state.
 		 */
-		fp->ff_size -= (u_int64_t)bitmapblks * (u_int64_t)vcb->blockSize;
+		if (usedExtendFileC) {
+			(void) TruncateFileC(vcb, fp, oldBitmapSize, false);
+		} else {
+			fp->ff_blocks -= bitmapblks;
+			fp->ff_size -= (u_int64_t)bitmapblks * (u_int64_t)vcb->blockSize;
+			/*
+			 * No need to mark the excess blocks free since those bitmap blocks
+			 * are no longer part of the bitmap.  But we do need to undo the
+			 * effect of the "vcb->freeBlocks -= bitmapblks" above.
+			 */
+			vcb->freeBlocks += bitmapblks;
+		}
 		vcb->totalBlocks -= addblks;
-		vcb->freeBlocks -= addblks - bitmapblks;
+		vcb->freeBlocks -= addblks;
 		hfsmp->hfs_phys_block_count = prev_phys_block_count;
 		hfsmp->hfs_alt_id_sector = prev_alt_sector;
 		MarkVCBDirty(vcb);
@@ -2989,11 +3276,36 @@ hfs_extendfs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 	
 			bzero((char *)buf_dataptr(bp) + HFS_ALT_OFFSET(sectorsize), kMDBSize);
 	
-			journal_modify_block_end(hfsmp->jnl, bp);
+			journal_modify_block_end(hfsmp->jnl, bp, NULL, NULL);
 		} else if (bp) {
 			buf_brelse(bp);
 		}
 	}
+	
+	/*
+	 * TODO: Adjust the size of the metadata zone based on new volume size?
+	 */
+	 
+	/*
+	 * Adjust the size of hfsmp->hfs_attrdata_vp
+	 */
+	if (hfsmp->hfs_attrdata_vp) {
+		struct cnode *attr_cp;
+		struct filefork *attr_fp;
+		
+		if (vnode_get(hfsmp->hfs_attrdata_vp) == 0) {
+			attr_cp = VTOC(hfsmp->hfs_attrdata_vp);
+			attr_fp = VTOF(hfsmp->hfs_attrdata_vp);
+			
+			attr_cp->c_blocks = newblkcnt;
+			attr_fp->ff_blocks = newblkcnt;
+			attr_fp->ff_extents[0].blockCount = newblkcnt;
+			attr_fp->ff_size = (off_t) newblkcnt * hfsmp->blockSize;
+			ubc_setsize(hfsmp->hfs_attrdata_vp, attr_fp->ff_size);
+			vnode_put(hfsmp->hfs_attrdata_vp);
+		}
+	}
+
 out:
 	if (error && fp) {
 		/* Restore allocation fork. */
@@ -3014,17 +3326,16 @@ out:
  */
 __private_extern__
 int
-hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t context)
+hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, vfs_context_t context)
 {
 	struct  buf *bp = NULL;
 	u_int64_t oldsize;
 	u_int32_t newblkcnt;
-	u_int32_t reclaimblks;
+	u_int32_t reclaimblks = 0;
 	int lockflags = 0;
 	int transaction_begun = 0;
 	int error;
 
-	
 	lck_mtx_lock(&hfsmp->hfs_mutex);
 	if (hfsmp->hfs_flags & HFS_RESIZE_IN_PROGRESS) {
 		lck_mtx_unlock(&hfsmp->hfs_mutex);
@@ -3057,12 +3368,14 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 	}
 	/* Make sure there's enough space to work with. */
 	if (reclaimblks >= hfs_freeblks(hfsmp, 1)) {
+		printf("hfs_truncatefs: insufficient space (need %u blocks; have %u blocks)\n", reclaimblks, hfs_freeblks(hfsmp, 1));
 		error = ENOSPC;
 		goto out;
 	}
+	
 	/* Start with a clean journal. */
 	journal_flush(hfsmp->jnl);
-
+	
 	if (hfs_start_transaction(hfsmp) != 0) {
 		error = EINVAL;
 		goto out;
@@ -3070,9 +3383,27 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 	transaction_begun = 1;
 
 	/*
-	 * Look for files that have blocks beyond newblkcnt.
+	 * Prevent new allocations from using the part we're trying to truncate.
+	 *
+	 * NOTE: allocLimit is set to the allocation block number where the new
+	 * alternate volume header will be.  That way there will be no files to
+	 * interfere with allocating the new alternate volume header, and no files
+	 * in the allocation blocks beyond (i.e. the blocks we're trying to
+	 * truncate away.
+	 */
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	if (hfsmp->blockSize == 512) 
+		hfsmp->allocLimit = newblkcnt - 2;
+	else
+		hfsmp->allocLimit = newblkcnt - 1;
+	hfsmp->freeBlocks -= reclaimblks;
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
+	
+	/*
+	 * Look for files that have blocks at or beyond the location of the
+	 * new alternate volume header.
 	 */
-	if (hfs_isallocated(hfsmp, newblkcnt, reclaimblks - 1)) {
+	if (hfs_isallocated(hfsmp, hfsmp->allocLimit, reclaimblks)) {
 		/*
 		 * hfs_reclaimspace will use separate transactions when
 		 * relocating files (so we don't overwhelm the journal).
@@ -3081,7 +3412,7 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 		transaction_begun = 0;
 
 		/* Attempt to reclaim some space. */ 
-		if (hfs_reclaimspace(hfsmp, newblkcnt, reclaimblks) != 0) {
+		if (hfs_reclaimspace(hfsmp, hfsmp->allocLimit, reclaimblks, context) != 0) {
 			printf("hfs_truncatefs: couldn't reclaim space on %s\n", hfsmp->vcbVN);
 			error = ENOSPC;
 			goto out;
@@ -3093,39 +3424,41 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 		transaction_begun = 1;
 		
 		/* Check if we're clear now. */
-		if (hfs_isallocated(hfsmp, newblkcnt, reclaimblks - 1)) {
+		if (hfs_isallocated(hfsmp, hfsmp->allocLimit, reclaimblks)) {
 			printf("hfs_truncatefs: didn't reclaim enough space on %s\n", hfsmp->vcbVN);
 			error = EAGAIN;  /* tell client to try again */
 			goto out;
 		}
 	}
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+	
+	/*
+	 * Note: we take the attributes lock in case we have an attribute data vnode
+	 * which needs to change size.
+	 */
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE | SFL_EXTENTS | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
 
 	/*
 	 * Mark the old alternate volume header as free. 
 	 * We don't bother shrinking allocation bitmap file.
 	 */
-	 if (hfsmp->blockSize == 512) 
+	if (hfsmp->blockSize == 512) 
 		(void) BlockMarkFree(hfsmp, hfsmp->totalBlocks - 2, 2);
 	else 
 		(void) BlockMarkFree(hfsmp, hfsmp->totalBlocks - 1, 1);
 
 	/*
-	 * Allocate last block for alternate volume header.
+	 * Allocate last 1KB for alternate volume header.
 	 */
-	if (hfsmp->blockSize == 512)
-		error = BlockMarkAllocated(hfsmp, newblkcnt - 2, 2);
-	else
-		error = BlockMarkAllocated(hfsmp, newblkcnt - 1, 1);
-
+	error = BlockMarkAllocated(hfsmp, hfsmp->allocLimit, (hfsmp->blockSize == 512) ? 2 : 1);
 	if (error) {
+		printf("hfs_truncatefs: Error %d allocating new alternate volume header\n", error);
 		goto out;
 	}
 
 	/*
 	 * Invalidate the existing alternate volume header.
 	 *
-	 * Don't do this as a transaction (don't call journal_modify_block)
+	 * Don't include this in a transaction (don't call journal_modify_block)
 	 * since this block will be outside of the truncated file system!
 	 */
 	if (hfsmp->hfs_alt_id_sector) {
@@ -3147,7 +3480,6 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 	/*
 	 * Adjust file system variables and flush them to disk.
 	 */
-	hfsmp->freeBlocks -= hfsmp->totalBlocks - newblkcnt;
 	hfsmp->totalBlocks = newblkcnt;
 	hfsmp->hfs_phys_block_count = newsize / hfsmp->hfs_phys_block_size;
 	hfsmp->hfs_alt_id_sector = HFS_ALT_SECTOR(hfsmp->hfs_phys_block_size, hfsmp->hfs_phys_block_count);
@@ -3155,7 +3487,42 @@ hfs_truncatefs(struct hfsmount *hfsmp, u_int64_t newsize, __unused vfs_context_t
 	error = hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
 	if (error)
 		panic("hfs_truncatefs: unexpected error flushing volume header (%d)\n", error);
+	
+	/*
+	 * TODO: Adjust the size of the metadata zone based on new volume size?
+	 */
+	 
+	/*
+	 * Adjust the size of hfsmp->hfs_attrdata_vp
+	 */
+	if (hfsmp->hfs_attrdata_vp) {
+		struct cnode *cp;
+		struct filefork *fp;
+		
+		if (vnode_get(hfsmp->hfs_attrdata_vp) == 0) {
+			cp = VTOC(hfsmp->hfs_attrdata_vp);
+			fp = VTOF(hfsmp->hfs_attrdata_vp);
+			
+			cp->c_blocks = newblkcnt;
+			fp->ff_blocks = newblkcnt;
+			fp->ff_extents[0].blockCount = newblkcnt;
+			fp->ff_size = (off_t) newblkcnt * hfsmp->blockSize;
+			ubc_setsize(hfsmp->hfs_attrdata_vp, fp->ff_size);
+			vnode_put(hfsmp->hfs_attrdata_vp);
+		}
+	}
+	
 out:
+	if (error)
+		hfsmp->freeBlocks += reclaimblks;
+	
+	lck_mtx_lock(&hfsmp->hfs_mutex);
+	hfsmp->allocLimit = hfsmp->totalBlocks;
+	if (hfsmp->nextAllocation >= hfsmp->allocLimit)
+		hfsmp->nextAllocation = hfsmp->hfs_metazone_end + 1;
+	hfsmp->hfs_flags &= ~HFS_RESIZE_IN_PROGRESS;
+	lck_mtx_unlock(&hfsmp->hfs_mutex);
+	
 	if (lockflags) {
 		hfs_systemfile_unlock(hfsmp, lockflags);
 	}
@@ -3164,84 +3531,733 @@ out:
 		journal_flush(hfsmp->jnl);
 	}
 
-	lck_mtx_lock(&hfsmp->hfs_mutex);
-	hfsmp->hfs_flags &= ~HFS_RESIZE_IN_PROGRESS;
-	lck_mtx_unlock(&hfsmp->hfs_mutex);
-
 	return (error);
 }
 
 
 /*
- * Reclaim space at the end of a file system.
+ * Invalidate the physical block numbers associated with buffer cache blocks
+ * in the given extent of the given vnode.
  */
+struct hfs_inval_blk_no {
+	daddr64_t sectorStart;
+	daddr64_t sectorCount;
+};
 static int
-hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks)
+hfs_invalidate_block_numbers_callback(buf_t bp, void *args_in)
 {
-	struct vnode *vp = NULL;
-	FCB *fcb;
-	struct BTreeIterator * iterator = NULL;
-	struct FSBufferDescriptor btdata;
-	struct HFSPlusCatalogFile filerec;
-	struct filefork *fp;
-	u_int32_t  saved_next_allocation;
-	cnid_t * cnidbufp;
-	size_t cnidbufsize;
-	int filecnt = 0;
-	int maxfilecnt;
-	u_long block;
-	u_long datablks;
-	u_long rsrcblks;
-	u_long blkstomove = 0;
-	int lockflags;
-	int i;
-	int error;
-	int lastprogress = 0;
+	daddr64_t blkno;
+	struct hfs_inval_blk_no *args;
+	
+	blkno = buf_blkno(bp);
+	args = args_in;
+	
+	if (blkno >= args->sectorStart && blkno < args->sectorStart+args->sectorCount)
+		buf_setblkno(bp, buf_lblkno(bp));
 
+	return BUF_RETURNED;
+}
+static void
+hfs_invalidate_sectors(struct vnode *vp, daddr64_t sectorStart, daddr64_t sectorCount)
+{
+	struct hfs_inval_blk_no args;
+	args.sectorStart = sectorStart;
+	args.sectorCount = sectorCount;
+	
+	buf_iterate(vp, hfs_invalidate_block_numbers_callback, BUF_SCAN_DIRTY|BUF_SCAN_CLEAN, &args);
+}
 
-	/* Check if Attributes file overlaps reclaim area. */
-	if (hfsmp->hfs_attribute_vp) {
-		fp = VTOF(hfsmp->hfs_attribute_vp);
-		datablks = 0;
-		for (i = 0; i < kHFSPlusExtentDensity; ++i) {
-			if (fp->ff_extents[i].blockCount == 0) {
-				break;
-			}
-			datablks += fp->ff_extents[i].blockCount;
-			block = fp->ff_extents[i].startBlock + fp->ff_extents[i].blockCount;
-			if (block >= startblk) {
-				printf("hfs_reclaimspace: Attributes file can't move\n");
-				return (EPERM);
-			}
+
+/*
+ * Copy the contents of an extent to a new location.  Also invalidates the
+ * physical block number of any buffer cache block in the copied extent
+ * (so that if the block is written, it will go through VNOP_BLOCKMAP to
+ * determine the new physical block number).
+ */
+static int
+hfs_copy_extent(
+	struct hfsmount *hfsmp,
+	struct vnode *vp,		/* The file whose extent is being copied. */
+	u_int32_t oldStart,		/* The start of the source extent. */
+	u_int32_t newStart,		/* The start of the destination extent. */
+	u_int32_t blockCount,	/* The number of allocation blocks to copy. */
+	vfs_context_t context)
+{
+	int err = 0;
+	size_t bufferSize;
+	void *buffer = NULL;
+	struct vfsioattr ioattr;
+	buf_t bp = NULL;
+	off_t resid;
+	size_t ioSize;
+	u_int32_t ioSizeSectors;	/* Device sectors in this I/O */
+	daddr64_t srcSector, destSector;
+	u_int32_t sectorsPerBlock = hfsmp->blockSize / hfsmp->hfs_phys_block_size;
+
+	/*
+	 * Sanity check that we have locked the vnode of the file we're copying.
+	 *
+	 * But since hfs_systemfile_lock() doesn't actually take the lock on
+	 * the allocation file if a journal is active, ignore the check if the
+	 * file being copied is the allocation file.
+	 */
+	struct cnode *cp = VTOC(vp);
+	if (cp != hfsmp->hfs_allocation_cp && cp->c_lockowner != current_thread())
+		panic("hfs_copy_extent: vp=%p (cp=%p) not owned?\n", vp, cp);
+
+	/*
+	 * Wait for any in-progress writes to this vnode to complete, so that we'll
+	 * be copying consistent bits.  (Otherwise, it's possible that an async
+	 * write will complete to the old extent after we read from it.  That
+	 * could lead to corruption.)
+	 */
+	err = vnode_waitforwrites(vp, 0, 0, 0, "hfs_copy_extent");
+	if (err) {
+		printf("hfs_copy_extent: Error %d from vnode_waitforwrites\n", err);
+		return err;
+	}
+	
+	/*
+	 * Determine the I/O size to use
+	 *
+	 * NOTE: Many external drives will result in an ioSize of 128KB.
+	 * TODO: Should we use a larger buffer, doing several consecutive
+	 * reads, then several consecutive writes?
+	 */
+	vfs_ioattr(hfsmp->hfs_mp, &ioattr);
+	bufferSize = MIN(ioattr.io_maxreadcnt, ioattr.io_maxwritecnt);
+	if (kmem_alloc(kernel_map, (vm_offset_t*) &buffer, bufferSize))
+		return ENOMEM;
+
+	/* Get a buffer for doing the I/O */
+	bp = buf_alloc(hfsmp->hfs_devvp);
+	buf_setdataptr(bp, (uintptr_t)buffer);
+	
+	resid = (off_t) blockCount * (off_t) hfsmp->blockSize;
+	srcSector = (daddr64_t) oldStart * hfsmp->blockSize / hfsmp->hfs_phys_block_size;
+	destSector = (daddr64_t) newStart * hfsmp->blockSize / hfsmp->hfs_phys_block_size;
+	while (resid > 0) {
+		ioSize = MIN(bufferSize, resid);
+		ioSizeSectors = ioSize / hfsmp->hfs_phys_block_size;
+		
+		/* Prepare the buffer for reading */
+		buf_reset(bp, B_READ);
+		buf_setsize(bp, ioSize);
+		buf_setcount(bp, ioSize);
+		buf_setblkno(bp, srcSector);
+		buf_setlblkno(bp, srcSector);
+		
+		/* Do the read */
+		err = VNOP_STRATEGY(bp);
+		if (!err)
+			err = buf_biowait(bp);
+		if (err) {
+			printf("hfs_copy_extent: Error %d from VNOP_STRATEGY (read)\n", err);
+			break;
 		}
-		if ((i == kHFSPlusExtentDensity) && (fp->ff_blocks > datablks)) {
-			if (hfs_overlapped_overflow_extents(hfsmp, startblk, datablks, kHFSAttributesFileID, 0)) {
-				printf("hfs_reclaimspace: Attributes file can't move\n");
-				return (EPERM);
-			}
+		
+		/* Prepare the buffer for writing */
+		buf_reset(bp, B_WRITE);
+		buf_setsize(bp, ioSize);
+		buf_setcount(bp, ioSize);
+		buf_setblkno(bp, destSector);
+		buf_setlblkno(bp, destSector);
+		if (journal_uses_fua(hfsmp->jnl))
+			buf_markfua(bp);
+			
+		/* Do the write */
+		vnode_startwrite(hfsmp->hfs_devvp);
+		err = VNOP_STRATEGY(bp);
+		if (!err)
+			err = buf_biowait(bp);
+		if (err) {
+			printf("hfs_copy_extent: Error %d from VNOP_STRATEGY (write)\n", err);
+			break;
 		}
+		
+		resid -= ioSize;
+		srcSector += ioSizeSectors;
+		destSector += ioSizeSectors;
+	}
+	if (bp)
+		buf_free(bp);
+	if (buffer)
+		kmem_free(kernel_map, (vm_offset_t)buffer, bufferSize);
+
+	/* Make sure all writes have been flushed to disk. */
+	if (!journal_uses_fua(hfsmp->jnl)) {
+		err = VNOP_IOCTL(hfsmp->hfs_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, context);
+		if (err) {
+			printf("hfs_copy_extent: DKIOCSYNCHRONIZECACHE failed (%d)\n", err);
+			err = 0;	/* Don't fail the copy. */
+		}
+	}
+
+	if (!err)
+		hfs_invalidate_sectors(vp, (daddr64_t)oldStart*sectorsPerBlock, (daddr64_t)blockCount*sectorsPerBlock);
+
+	return err;
+}
+
+
+/*
+ * Reclaim space at the end of a volume, used by a given system file.
+ *
+ * This routine attempts to move any extent which contains allocation blocks
+ * at or after "startblk."  A separate transaction is used to do the move.
+ * The contents of any moved extents are read and written via the volume's
+ * device vnode -- NOT via "vp."  During the move, moved blocks which are part
+ * of a transaction have their physical block numbers invalidated so they will
+ * eventually be written to their new locations.
+ *
+ * This routine can be used to move overflow extents for the allocation file.
+ *
+ * Inputs:
+ *    hfsmp       The volume being resized.
+ *    startblk    Blocks >= this allocation block need to be moved.
+ *    locks       Which locks need to be taken for the given system file.
+ *    vp          The vnode for the system file.
+ *
+ * Outputs:
+ *    moved       Set to true if any extents were moved.
+ */
+static int
+hfs_relocate_callback(__unused HFSPlusExtentKey *key, HFSPlusExtentRecord *record, HFSPlusExtentRecord *state)
+{
+	bcopy(state, record, sizeof(HFSPlusExtentRecord));
+	return 0;
+}
+static int
+hfs_reclaim_sys_file(struct hfsmount *hfsmp, struct vnode *vp, u_long startblk, int locks, Boolean *moved, vfs_context_t context)
+{
+	int error;
+	int lockflags;
+	int i;
+	u_long datablks;
+	u_long block;
+	u_int32_t oldStartBlock;
+	u_int32_t newStartBlock;
+	u_int32_t blockCount;
+	struct filefork *fp;
+
+	/* If there is no vnode for this file, then there's nothing to do. */	
+	if (vp == NULL)
+		return 0;
+
+	/* printf("hfs_reclaim_sys_file: %.*s\n", VTOC(vp)->c_desc.cd_namelen, VTOC(vp)->c_desc.cd_nameptr); */
+	
+	/* We always need the allocation bitmap and extents B-tree */
+	locks |= SFL_BITMAP | SFL_EXTENTS;
+	
+	error = hfs_start_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_sys_file: hfs_start_transaction returned %d\n", error);
+		return error;
 	}
-	/* Check if Catalog file overlaps reclaim area. */
-	fp = VTOF(hfsmp->hfs_catalog_vp);
+	lockflags = hfs_systemfile_lock(hfsmp, locks, HFS_EXCLUSIVE_LOCK);
+	fp = VTOF(vp);
 	datablks = 0;
+
+	/* Relocate non-overflow extents */
 	for (i = 0; i < kHFSPlusExtentDensity; ++i) {
-		if (fp->ff_extents[i].blockCount == 0) {
+		if (fp->ff_extents[i].blockCount == 0)
 			break;
+		oldStartBlock = fp->ff_extents[i].startBlock;
+		blockCount = fp->ff_extents[i].blockCount;
+		datablks += blockCount;
+		block = oldStartBlock + blockCount;
+		if (block > startblk) {
+			error = BlockAllocate(hfsmp, 1, blockCount, blockCount, true, true, &newStartBlock, &blockCount);
+			if (error) {
+				printf("hfs_reclaim_sys_file: BlockAllocate returned %d\n", error);
+				goto fail;
+			}
+			if (blockCount != fp->ff_extents[i].blockCount) {
+				printf("hfs_reclaim_sys_file: new blockCount=%u, original blockCount=%u", blockCount, fp->ff_extents[i].blockCount);
+				goto free_fail;
+			}
+			error = hfs_copy_extent(hfsmp, vp, oldStartBlock, newStartBlock, blockCount, context);
+			if (error) {
+				printf("hfs_reclaim_sys_file: hfs_copy_extent returned %d\n", error);
+				goto free_fail;
+			}
+			fp->ff_extents[i].startBlock = newStartBlock;
+			VTOC(vp)->c_flag |= C_MODIFIED;
+			*moved = true;
+			error = BlockDeallocate(hfsmp, oldStartBlock, blockCount);
+			if (error) {
+				/* TODO: Mark volume inconsistent? */
+				printf("hfs_reclaim_sys_file: BlockDeallocate returned %d\n", error);
+				goto fail;
+			}
+			error = hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
+			if (error) {
+				/* TODO: Mark volume inconsistent? */
+				printf("hfs_reclaim_sys_file: hfs_flushvolumeheader returned %d\n", error);
+				goto fail;
+			}
 		}
-		datablks += fp->ff_extents[i].blockCount;
-		block = fp->ff_extents[i].startBlock + fp->ff_extents[i].blockCount;
-		if (block >= startblk) {
-			printf("hfs_reclaimspace: Catalog file can't move\n");
-			return (EPERM);
+	}
+
+	/* Relocate overflow extents (if any) */
+	if (i == kHFSPlusExtentDensity && fp->ff_blocks > datablks) {
+		struct BTreeIterator *iterator = NULL;
+		struct FSBufferDescriptor btdata;
+		HFSPlusExtentRecord record;
+		HFSPlusExtentKey *key;
+		FCB *fcb;
+		u_int32_t fileID;
+		u_int8_t forktype;
+
+		forktype = VNODE_IS_RSRC(vp) ? 0xFF : 0;
+		fileID = VTOC(vp)->c_cnid;
+		if (kmem_alloc(kernel_map, (vm_offset_t*) &iterator, sizeof(*iterator))) {
+			printf("hfs_reclaim_sys_file: kmem_alloc failed!\n");
+			error = ENOMEM;
+			goto fail;
+		}
+
+		bzero(iterator, sizeof(*iterator));
+		key = (HFSPlusExtentKey *) &iterator->key;
+		key->keyLength = kHFSPlusExtentKeyMaximumLength;
+		key->forkType = forktype;
+		key->fileID = fileID;
+		key->startBlock = datablks;
+	
+		btdata.bufferAddress = &record;
+		btdata.itemSize = sizeof(record);
+		btdata.itemCount = 1;
+	
+		fcb = VTOF(hfsmp->hfs_extents_vp);
+
+		error = BTSearchRecord(fcb, iterator, &btdata, NULL, iterator);
+		while (error == 0) {
+			/* Stop when we encounter a different file or fork. */
+			if ((key->fileID != fileID) ||
+				(key->forkType != forktype)) {
+				break;
+			}
+			/* 
+			 * Check if the file overlaps target space.
+			 */
+			for (i = 0; i < kHFSPlusExtentDensity; ++i) {
+				if (record[i].blockCount == 0) {
+					goto overflow_done;
+				}
+				oldStartBlock = record[i].startBlock;
+				blockCount = record[i].blockCount;
+				block = oldStartBlock + blockCount;
+				if (block > startblk) {
+					error = BlockAllocate(hfsmp, 1, blockCount, blockCount, true, true, &newStartBlock, &blockCount);
+					if (error) {
+						printf("hfs_reclaim_sys_file: BlockAllocate returned %d\n", error);
+						goto overflow_done;
+					}
+					if (blockCount != record[i].blockCount) {
+						printf("hfs_reclaim_sys_file: new blockCount=%u, original blockCount=%u", blockCount, fp->ff_extents[i].blockCount);
+						kmem_free(kernel_map, (vm_offset_t)iterator, sizeof(*iterator));
+						goto free_fail;
+					}
+					error = hfs_copy_extent(hfsmp, vp, oldStartBlock, newStartBlock, blockCount, context);
+					if (error) {
+						printf("hfs_reclaim_sys_file: hfs_copy_extent returned %d\n", error);
+						kmem_free(kernel_map, (vm_offset_t)iterator, sizeof(*iterator));
+						goto free_fail;
+					}
+					record[i].startBlock = newStartBlock;
+					VTOC(vp)->c_flag |= C_MODIFIED;
+					*moved = true;
+					/*
+					 * NOTE: To support relocating overflow extents of the
+					 * allocation file, we must update the BTree record BEFORE
+					 * deallocating the old extent so that BlockDeallocate will
+					 * use the extent's new location to calculate physical block
+					 * numbers.  (This is for the case where the old extent's
+					 * bitmap bits actually reside in the extent being moved.)
+					 */
+					error = BTUpdateRecord(fcb, iterator, (IterateCallBackProcPtr) hfs_relocate_callback, &record);
+					if (error) {
+						/* TODO: Mark volume inconsistent? */
+						printf("hfs_reclaim_sys_file: BTUpdateRecord returned %d\n", error);
+						goto overflow_done;
+					}
+					error = BlockDeallocate(hfsmp, oldStartBlock, blockCount);
+					if (error) {
+						/* TODO: Mark volume inconsistent? */
+						printf("hfs_reclaim_sys_file: BlockDeallocate returned %d\n", error);
+						goto overflow_done;
+					}
+				}
+			}
+			/* Look for more records. */
+			error = BTIterateRecord(fcb, kBTreeNextRecord, iterator, &btdata, NULL);
+			if (error == btNotFound) {
+				error = 0;
+				break;
+			}
+		}
+overflow_done:
+		kmem_free(kernel_map, (vm_offset_t)iterator, sizeof(*iterator));
+		if (error) {
+			goto fail;
 		}
 	}
-	if ((i == kHFSPlusExtentDensity) && (fp->ff_blocks > datablks)) {
-		if (hfs_overlapped_overflow_extents(hfsmp, startblk, datablks, kHFSCatalogFileID, 0)) {
-			printf("hfs_reclaimspace: Catalog file can't move\n");
-			return (EPERM);
+	
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	error = hfs_end_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_sys_file: hfs_end_transaction returned %d\n", error);
+	}
+
+	return error;
+
+free_fail:
+	(void) BlockDeallocate(hfsmp, newStartBlock, blockCount);
+fail:
+	(void) hfs_systemfile_unlock(hfsmp, lockflags);
+	(void) hfs_end_transaction(hfsmp);
+	return error;
+}
+
+
+/*
+ * This journal_relocate callback updates the journal info block to point
+ * at the new journal location.  This write must NOT be done using the
+ * transaction.  We must write the block immediately.  We must also force
+ * it to get to the media so that the new journal location will be seen by
+ * the replay code before we can safely let journaled blocks be written
+ * to their normal locations.
+ *
+ * The tests for journal_uses_fua below are mildly hacky.  Since the journal
+ * and the file system are both on the same device, I'm leveraging what
+ * the journal has decided about FUA.
+ */
+struct hfs_journal_relocate_args {
+	struct hfsmount *hfsmp;
+	vfs_context_t context;
+	u_int32_t newStartBlock;
+};
+
+static errno_t
+hfs_journal_relocate_callback(void *_args)
+{
+	int error;
+	struct hfs_journal_relocate_args *args = _args;
+	struct hfsmount *hfsmp = args->hfsmp;
+	buf_t bp;
+	JournalInfoBlock *jibp;
+
+	error = buf_meta_bread(hfsmp->hfs_devvp,
+		hfsmp->vcbJinfoBlock * (hfsmp->blockSize/hfsmp->hfs_phys_block_size),
+		hfsmp->blockSize, vfs_context_ucred(args->context), &bp);
+	if (error) {
+		printf("hfs_reclaim_journal_file: failed to read JIB (%d)\n", error);
+		return error;
+	}
+	jibp = (JournalInfoBlock*) buf_dataptr(bp);
+	jibp->offset = SWAP_BE64((u_int64_t)args->newStartBlock * hfsmp->blockSize);
+	jibp->size = SWAP_BE64(hfsmp->jnl_size);
+	if (journal_uses_fua(hfsmp->jnl))
+		buf_markfua(bp);
+	error = buf_bwrite(bp);
+	if (error) {
+		printf("hfs_reclaim_journal_file: failed to write JIB (%d)\n", error);
+		return error;
+	}
+	if (!journal_uses_fua(hfsmp->jnl)) {
+		error = VNOP_IOCTL(hfsmp->hfs_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, args->context);
+		if (error) {
+			printf("hfs_reclaim_journal_file: DKIOCSYNCHRONIZECACHE failed (%d)\n", error);
+			error = 0;		/* Don't fail the operation. */
 		}
 	}
 
+	return error;
+}
+
+
+static int
+hfs_reclaim_journal_file(struct hfsmount *hfsmp, vfs_context_t context)
+{
+	int error;
+	int lockflags;
+	u_int32_t newStartBlock;
+	u_int32_t oldBlockCount;
+	u_int32_t newBlockCount;
+	struct cat_desc journal_desc;
+	struct cat_attr journal_attr;
+	struct cat_fork journal_fork;
+	struct hfs_journal_relocate_args callback_args;
+
+	error = hfs_start_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_journal_file: hfs_start_transaction returned %d\n", error);
+		return error;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+	
+	oldBlockCount = hfsmp->jnl_size / hfsmp->blockSize;
+	
+	/* TODO: Allow the journal to change size based on the new volume size. */
+	error = BlockAllocate(hfsmp, 1, oldBlockCount, oldBlockCount, true, true, &newStartBlock, &newBlockCount);
+	if (error) {
+		printf("hfs_reclaim_journal_file: BlockAllocate returned %d\n", error);
+		goto fail;
+	}
+	if (newBlockCount != oldBlockCount) {
+		printf("hfs_reclaim_journal_file: newBlockCount != oldBlockCount (%u, %u)\n", newBlockCount, oldBlockCount);
+		goto free_fail;
+	}
+	
+	error = BlockDeallocate(hfsmp, hfsmp->jnl_start, oldBlockCount);
+	if (error) {
+		printf("hfs_reclaim_journal_file: BlockDeallocate returned %d\n", error);
+		goto free_fail;
+	}
+
+	/* Update the catalog record for .journal */
+	error = cat_idlookup(hfsmp, hfsmp->hfs_jnlfileid, 1, &journal_desc, &journal_attr, &journal_fork);
+	if (error) {
+		printf("hfs_reclaim_journal_file: cat_idlookup returned %d\n", error);
+		goto free_fail;
+	}
+	journal_fork.cf_size = newBlockCount * hfsmp->blockSize;
+	journal_fork.cf_extents[0].startBlock = newStartBlock;
+	journal_fork.cf_extents[0].blockCount = newBlockCount;
+	journal_fork.cf_blocks = newBlockCount;
+	error = cat_update(hfsmp, &journal_desc, &journal_attr, &journal_fork, NULL);
+	if (error) {
+		printf("hfs_reclaim_journal_file: cat_update returned %d\n", error);
+		goto free_fail;
+	}
+	callback_args.hfsmp = hfsmp;
+	callback_args.context = context;
+	callback_args.newStartBlock = newStartBlock;
+	
+	error = journal_relocate(hfsmp->jnl, (off_t)newStartBlock*hfsmp->blockSize,
+		(off_t)newBlockCount*hfsmp->blockSize, 0,
+		hfs_journal_relocate_callback, &callback_args);
+	if (error) {
+		/* NOTE: journal_relocate will mark the journal invalid. */
+		printf("hfs_reclaim_journal_file: journal_relocate returned %d\n", error);
+		goto fail;
+	}
+	hfsmp->jnl_start = newStartBlock;
+	hfsmp->jnl_size = (off_t)newBlockCount * hfsmp->blockSize;
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	error = hfs_end_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_journal_file: hfs_end_transaction returned %d\n", error);
+	}
+	
+	return error;
+
+free_fail:
+	(void) BlockDeallocate(hfsmp, newStartBlock, newBlockCount);
+fail:
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	(void) hfs_end_transaction(hfsmp);
+	return error;
+}
+
+
+/*
+ * Move the journal info block to a new location.  We have to make sure the
+ * new copy of the journal info block gets to the media first, then change
+ * the field in the volume header and the catalog record.
+ */
+static int
+hfs_reclaim_journal_info_block(struct hfsmount *hfsmp, vfs_context_t context)
+{
+	int error;
+	int lockflags;
+	u_int32_t newBlock;
+	u_int32_t blockCount;
+	struct cat_desc jib_desc;
+	struct cat_attr jib_attr;
+	struct cat_fork jib_fork;
+	buf_t old_bp, new_bp;
+	
+	error = hfs_start_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: hfs_start_transaction returned %d\n", error);
+		return error;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+	
+	error = BlockAllocate(hfsmp, 1, 1, 1, true, true, &newBlock, &blockCount);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: BlockAllocate returned %d\n", error);
+		goto fail;
+	}
+	if (blockCount != 1) {
+		printf("hfs_reclaim_journal_info_block: blockCount != 1 (%u)\n", blockCount);
+		goto free_fail;
+	}
+	error = BlockDeallocate(hfsmp, hfsmp->vcbJinfoBlock, 1);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: BlockDeallocate returned %d\n", error);
+		goto free_fail;
+	}
+	
+	/* Copy the old journal info block content to the new location */
+	error = buf_meta_bread(hfsmp->hfs_devvp,
+		hfsmp->vcbJinfoBlock * (hfsmp->blockSize/hfsmp->hfs_phys_block_size),
+		hfsmp->blockSize, vfs_context_ucred(context), &old_bp);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: failed to read JIB (%d)\n", error);
+		goto free_fail;
+	}
+	new_bp = buf_getblk(hfsmp->hfs_devvp,
+		newBlock * (hfsmp->blockSize/hfsmp->hfs_phys_block_size),
+		hfsmp->blockSize, 0, 0, BLK_META);
+	bcopy((char*)buf_dataptr(old_bp), (char*)buf_dataptr(new_bp), hfsmp->blockSize);
+	buf_brelse(old_bp);
+	if (journal_uses_fua(hfsmp->jnl))
+		buf_markfua(new_bp);
+	error = buf_bwrite(new_bp);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: failed to write new JIB (%d)\n", error);
+		goto free_fail;
+	}
+	if (!journal_uses_fua(hfsmp->jnl)) {
+		error = VNOP_IOCTL(hfsmp->hfs_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, context);
+		if (error) {
+			printf("hfs_reclaim_journal_info_block: DKIOCSYNCHRONIZECACHE failed (%d)\n", error);
+			/* Don't fail the operation. */
+		}
+	}
+	
+	/* Update the catalog record for .journal_info_block */
+	error = cat_idlookup(hfsmp, hfsmp->hfs_jnlinfoblkid, 1, &jib_desc, &jib_attr, &jib_fork);
+	if (error) {
+		printf("hfs_reclaim_journal_file: cat_idlookup returned %d\n", error);
+		goto fail;
+	}
+	jib_fork.cf_size = hfsmp->blockSize;
+	jib_fork.cf_extents[0].startBlock = newBlock;
+	jib_fork.cf_extents[0].blockCount = 1;
+	jib_fork.cf_blocks = 1;
+	error = cat_update(hfsmp, &jib_desc, &jib_attr, &jib_fork, NULL);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: cat_update returned %d\n", error);
+		goto fail;
+	}
+	
+	/* Update the pointer to the journal info block in the volume header. */
+	hfsmp->vcbJinfoBlock = newBlock;
+	error = hfs_flushvolumeheader(hfsmp, MNT_WAIT, HFS_ALTFLUSH);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: hfs_flushvolumeheader returned %d\n", error);
+		goto fail;
+	}
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	error = hfs_end_transaction(hfsmp);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: hfs_end_transaction returned %d\n", error);
+	}
+	error = journal_flush(hfsmp->jnl);
+	if (error) {
+		printf("hfs_reclaim_journal_info_block: journal_flush returned %d\n", error);
+	}
+	return error;
+
+free_fail:
+	(void) BlockDeallocate(hfsmp, newBlock, blockCount);
+fail:
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	(void) hfs_end_transaction(hfsmp);
+	return error;
+}
+
+
+/*
+ * Reclaim space at the end of a file system.
+ */
+static int
+hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks, vfs_context_t context)
+{
+	struct vnode *vp = NULL;
+	FCB *fcb;
+	struct BTreeIterator * iterator = NULL;
+	struct FSBufferDescriptor btdata;
+	struct HFSPlusCatalogFile filerec;
+	u_int32_t  saved_next_allocation;
+	cnid_t * cnidbufp;
+	size_t cnidbufsize;
+	int filecnt = 0;
+	int maxfilecnt;
+	u_long block;
+	u_long datablks;
+	u_long rsrcblks;
+	u_long blkstomove = 0;
+	int lockflags;
+	int i;
+	int error;
+	int lastprogress = 0;
+	Boolean system_file_moved = false;
+
+	/* Relocate extents of the Allocation file if they're in the way. */
+	error = hfs_reclaim_sys_file(hfsmp, hfsmp->hfs_allocation_vp, startblk, SFL_BITMAP, &system_file_moved, context);
+	if (error) {
+		printf("hfs_reclaimspace: reclaim allocation file returned %d\n", error);
+		return error;
+	}
+	/* Relocate extents of the Extents B-tree if they're in the way. */
+	error = hfs_reclaim_sys_file(hfsmp, hfsmp->hfs_extents_vp, startblk, SFL_EXTENTS, &system_file_moved, context);
+	if (error) {
+		printf("hfs_reclaimspace: reclaim extents b-tree returned %d\n", error);
+		return error;
+	}
+	/* Relocate extents of the Catalog B-tree if they're in the way. */
+	error = hfs_reclaim_sys_file(hfsmp, hfsmp->hfs_catalog_vp, startblk, SFL_CATALOG, &system_file_moved, context);
+	if (error) {
+		printf("hfs_reclaimspace: reclaim catalog b-tree returned %d\n", error);
+		return error;
+	}
+	/* Relocate extents of the Attributes B-tree if they're in the way. */
+	error = hfs_reclaim_sys_file(hfsmp, hfsmp->hfs_attribute_vp, startblk, SFL_ATTRIBUTE, &system_file_moved, context);
+	if (error) {
+		printf("hfs_reclaimspace: reclaim attribute b-tree returned %d\n", error);
+		return error;
+	}
+	/* Relocate extents of the Startup File if there is one and they're in the way. */
+	error = hfs_reclaim_sys_file(hfsmp, hfsmp->hfs_startup_vp, startblk, SFL_STARTUP, &system_file_moved, context);
+	if (error) {
+		printf("hfs_reclaimspace: reclaim startup file returned %d\n", error);
+		return error;
+	}
+	
+	/*
+	 * We need to make sure the alternate volume header gets flushed if we moved
+	 * any extents in the volume header.  But we need to do that before
+	 * shrinking the size of the volume, or else the journal code will panic
+	 * with an invalid (too large) block number.
+	 *
+	 * Note that system_file_moved will be set if ANY extent was moved, even
+	 * if it was just an overflow extent.  In this case, the journal_flush isn't
+	 * strictly required, but shouldn't hurt.
+	 */
+	if (system_file_moved)
+		journal_flush(hfsmp->jnl);
+
+	if (hfsmp->jnl_start + (hfsmp->jnl_size / hfsmp->blockSize) > startblk) {
+		error = hfs_reclaim_journal_file(hfsmp, context);
+		if (error) {
+			printf("hfs_reclaimspace: hfs_reclaim_journal_file failed (%d)\n", error);
+			return error;
+		}
+	}
+	
+	if (hfsmp->vcbJinfoBlock >= startblk) {
+		error = hfs_reclaim_journal_info_block(hfsmp, context);
+		if (error) {
+			printf("hfs_reclaimspace: hfs_reclaim_journal_info_block failed (%d)\n", error);
+			return error;
+		}
+	}
+	
 	/* For now move a maximum of 250,000 files. */
 	maxfilecnt = MIN(hfsmp->hfs_filecount, 250000);
 	maxfilecnt = MIN((u_long)maxfilecnt, reclaimblks);
@@ -3255,7 +4271,7 @@ hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks)
 	}	
 
 	saved_next_allocation = hfsmp->nextAllocation;
-	hfsmp->nextAllocation = hfsmp->hfs_metazone_start;
+	HFS_UPDATE_NEXT_ALLOCATION(hfsmp, hfsmp->hfs_metazone_start);
 
 	fcb = VTOF(hfsmp->hfs_catalog_vp);
 	bzero(iterator, sizeof(*iterator));
@@ -3337,12 +4353,13 @@ hfs_reclaimspace(struct hfsmount *hfsmp, u_long startblk, u_long reclaimblks)
 	}
 
 end_iteration:
-	if (filecnt == 0) {
+	if (filecnt == 0 && !system_file_moved) {
+		printf("hfs_reclaimspace: no files moved\n");
 		error = ENOSPC;
 	}
 	/* All done with catalog. */
 	hfs_systemfile_unlock(hfsmp, lockflags);
-	if (error)
+	if (error || filecnt == 0)
 		goto out;
 
 	/*
@@ -3364,7 +4381,8 @@ end_iteration:
 	 * <------------------------ Original Total Blocks ----------------------->
 	 *
 	 */
-	if ((reclaimblks + blkstomove) >= hfs_freeblks(hfsmp, 1)) {
+	if (blkstomove >= hfs_freeblks(hfsmp, 1)) {
+		printf("hfs_truncatefs: insufficient space (need %lu blocks; have %u blocks)\n", blkstomove, hfs_freeblks(hfsmp, 1));
 		error = ENOSPC;
 		goto out;
 	}
@@ -3387,7 +4405,7 @@ end_iteration:
 
 		/* Relocate any resource fork blocks. */
 		if ((VTOC((vp))->c_blocks - VTOF((vp))->ff_blocks) > 0) {
-			error = hfs_vgetrsrc(hfsmp, vp, &rvp, current_proc());
+			error = hfs_vgetrsrc(hfsmp, vp, &rvp, TRUE);
 			if (error)
 				break;
 			error = hfs_relocate(rvp, hfsmp->hfs_metazone_end + 1, kauth_cred_get(), current_proc());
@@ -3430,7 +4448,7 @@ out:
 	 * (but only if we didn't move any files)
 	 */
 	if (error && hfsmp->hfs_resize_filesmoved == 0) {
-		hfsmp->nextAllocation = saved_next_allocation;
+		HFS_UPDATE_NEXT_ALLOCATION(hfsmp, saved_next_allocation);
 	}
 	return (error);
 }
@@ -3525,26 +4543,29 @@ hfs_resize_progress(struct hfsmount *hfsmp, u_int32_t *progress)
 static int
 hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t context)
 {
+#define HFS_ATTR_CMN_VALIDMASK (ATTR_CMN_VALIDMASK & ~(ATTR_CMN_NAMEDATTRCOUNT | ATTR_CMN_NAMEDATTRLIST))
+#define HFS_ATTR_FILE_VALIDMASK (ATTR_FILE_VALIDMASK & ~(ATTR_FILE_FILETYPE | ATTR_FILE_FORKCOUNT | ATTR_FILE_FORKLIST))
+
 	ExtendedVCB *vcb = VFSTOVCB(mp);
 	struct hfsmount *hfsmp = VFSTOHFS(mp);
 	u_long freeCNIDs;
 
 	freeCNIDs = (u_long)0xFFFFFFFF - (u_long)hfsmp->vcbNxtCNID;
 
-	VFSATTR_RETURN(fsap, f_objcount, (uint64_t)hfsmp->vcbFilCnt + (uint64_t)hfsmp->vcbDirCnt);
-	VFSATTR_RETURN(fsap, f_filecount, (uint64_t)hfsmp->vcbFilCnt);
-	VFSATTR_RETURN(fsap, f_dircount, (uint64_t)hfsmp->vcbDirCnt);
-	VFSATTR_RETURN(fsap, f_maxobjcount, (uint64_t)0xFFFFFFFF);
+	VFSATTR_RETURN(fsap, f_objcount, (u_int64_t)hfsmp->vcbFilCnt + (u_int64_t)hfsmp->vcbDirCnt);
+	VFSATTR_RETURN(fsap, f_filecount, (u_int64_t)hfsmp->vcbFilCnt);
+	VFSATTR_RETURN(fsap, f_dircount, (u_int64_t)hfsmp->vcbDirCnt);
+	VFSATTR_RETURN(fsap, f_maxobjcount, (u_int64_t)0xFFFFFFFF);
 	VFSATTR_RETURN(fsap, f_iosize, (size_t)(MAX_UPL_TRANSFER * PAGE_SIZE));
-	VFSATTR_RETURN(fsap, f_blocks, (uint64_t)hfsmp->totalBlocks);
-	VFSATTR_RETURN(fsap, f_bfree, (uint64_t)hfs_freeblks(hfsmp, 0));
-	VFSATTR_RETURN(fsap, f_bavail, (uint64_t)hfs_freeblks(hfsmp, 1));
-	VFSATTR_RETURN(fsap, f_bsize, (uint32_t)vcb->blockSize);
+	VFSATTR_RETURN(fsap, f_blocks, (u_int64_t)hfsmp->totalBlocks);
+	VFSATTR_RETURN(fsap, f_bfree, (u_int64_t)hfs_freeblks(hfsmp, 0));
+	VFSATTR_RETURN(fsap, f_bavail, (u_int64_t)hfs_freeblks(hfsmp, 1));
+	VFSATTR_RETURN(fsap, f_bsize, (u_int32_t)vcb->blockSize);
 	/* XXX needs clarification */
 	VFSATTR_RETURN(fsap, f_bused, hfsmp->totalBlocks - hfs_freeblks(hfsmp, 1));
 	/* Maximum files is constrained by total blocks. */
-	VFSATTR_RETURN(fsap, f_files, (uint64_t)(hfsmp->totalBlocks - 2));
-	VFSATTR_RETURN(fsap, f_ffree, MIN((uint64_t)freeCNIDs, (uint64_t)hfs_freeblks(hfsmp, 1)));
+	VFSATTR_RETURN(fsap, f_files, (u_int64_t)(hfsmp->totalBlocks - 2));
+	VFSATTR_RETURN(fsap, f_ffree, MIN((u_int64_t)freeCNIDs, (u_int64_t)hfs_freeblks(hfsmp, 1)));
 
 	fsap->f_fsid.val[0] = hfsmp->hfs_raw_dev;
 	fsap->f_fsid.val[1] = vfs_typenum(mp);
@@ -3562,18 +4583,23 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 			cap->capabilities[VOL_CAPABILITIES_FORMAT] =
 				VOL_CAP_FMT_PERSISTENTOBJECTIDS |
 				VOL_CAP_FMT_CASE_PRESERVING |
-				VOL_CAP_FMT_FAST_STATFS;
+				VOL_CAP_FMT_FAST_STATFS |
+				VOL_CAP_FMT_HIDDEN_FILES |
+				VOL_CAP_FMT_PATH_FROM_ID;
 		} else {
 			cap->capabilities[VOL_CAPABILITIES_FORMAT] =
 				VOL_CAP_FMT_PERSISTENTOBJECTIDS |
 				VOL_CAP_FMT_SYMBOLICLINKS |
 				VOL_CAP_FMT_HARDLINKS |
 				VOL_CAP_FMT_JOURNAL |
+				VOL_CAP_FMT_ZERO_RUNS |
 				(hfsmp->jnl ? VOL_CAP_FMT_JOURNAL_ACTIVE : 0) |
 				(hfsmp->hfs_flags & HFS_CASE_SENSITIVE ? VOL_CAP_FMT_CASE_SENSITIVE : 0) |
 				VOL_CAP_FMT_CASE_PRESERVING |
 				VOL_CAP_FMT_FAST_STATFS | 
-				VOL_CAP_FMT_2TB_FILESIZE;
+				VOL_CAP_FMT_2TB_FILESIZE |
+				VOL_CAP_FMT_HIDDEN_FILES |
+				VOL_CAP_FMT_PATH_FROM_ID;
 		}
 		cap->capabilities[VOL_CAPABILITIES_INTERFACES] =
 			VOL_CAP_INT_SEARCHFS |
@@ -3584,7 +4610,13 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 			VOL_CAP_INT_ALLOCATE |
 			VOL_CAP_INT_VOL_RENAME |
 			VOL_CAP_INT_ADVLOCK |
-			VOL_CAP_INT_FLOCK;
+			VOL_CAP_INT_FLOCK |
+#if NAMEDSTREAMS
+			VOL_CAP_INT_EXTENDED_ATTR |
+			VOL_CAP_INT_NAMEDSTREAMS;
+#else
+			VOL_CAP_INT_EXTENDED_ATTR;
+#endif
 		cap->capabilities[VOL_CAPABILITIES_RESERVED1] = 0;
 		cap->capabilities[VOL_CAPABILITIES_RESERVED2] = 0;
 
@@ -3600,7 +4632,10 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 			VOL_CAP_FMT_CASE_SENSITIVE |
 			VOL_CAP_FMT_CASE_PRESERVING |
 			VOL_CAP_FMT_FAST_STATFS |
-			VOL_CAP_FMT_2TB_FILESIZE;
+			VOL_CAP_FMT_2TB_FILESIZE |
+			VOL_CAP_FMT_OPENDENYMODES |
+			VOL_CAP_FMT_HIDDEN_FILES |
+			VOL_CAP_FMT_PATH_FROM_ID;
 		cap->valid[VOL_CAPABILITIES_INTERFACES] =
 			VOL_CAP_INT_SEARCHFS |
 			VOL_CAP_INT_ATTRLIST |
@@ -3611,7 +4646,14 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 			VOL_CAP_INT_ALLOCATE |
 			VOL_CAP_INT_VOL_RENAME |
 			VOL_CAP_INT_ADVLOCK |
-			VOL_CAP_INT_FLOCK;
+			VOL_CAP_INT_FLOCK |
+			VOL_CAP_INT_MANLOCK |
+#if NAMEDSTREAMS
+			VOL_CAP_INT_EXTENDED_ATTR |
+			VOL_CAP_INT_NAMEDSTREAMS;
+#else
+			VOL_CAP_INT_EXTENDED_ATTR;
+#endif
 		cap->valid[VOL_CAPABILITIES_RESERVED1] = 0;
 		cap->valid[VOL_CAPABILITIES_RESERVED2] = 0;
 		VFSATTR_SET_SUPPORTED(fsap, f_capabilities);
@@ -3619,16 +4661,16 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 	if (VFSATTR_IS_ACTIVE(fsap, f_attributes)) {
 		vol_attributes_attr_t *attrp = &fsap->f_attributes;
 
-        	attrp->validattr.commonattr = ATTR_CMN_VALIDMASK;
+        	attrp->validattr.commonattr = HFS_ATTR_CMN_VALIDMASK;
         	attrp->validattr.volattr = ATTR_VOL_VALIDMASK & ~ATTR_VOL_INFO;
         	attrp->validattr.dirattr = ATTR_DIR_VALIDMASK;
-        	attrp->validattr.fileattr = ATTR_FILE_VALIDMASK;
+        	attrp->validattr.fileattr = HFS_ATTR_FILE_VALIDMASK;
         	attrp->validattr.forkattr = 0;
 
-        	attrp->nativeattr.commonattr = ATTR_CMN_VALIDMASK;
+        	attrp->nativeattr.commonattr = HFS_ATTR_CMN_VALIDMASK;
         	attrp->nativeattr.volattr = ATTR_VOL_VALIDMASK & ~ATTR_VOL_INFO;
         	attrp->nativeattr.dirattr = ATTR_DIR_VALIDMASK;
-        	attrp->nativeattr.fileattr = ATTR_FILE_VALIDMASK;
+        	attrp->nativeattr.fileattr = HFS_ATTR_FILE_VALIDMASK;
         	attrp->nativeattr.forkattr = 0;
 		VFSATTR_SET_SUPPORTED(fsap, f_attributes);
 	}	
@@ -3643,7 +4685,7 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 	fsap->f_backup_time.tv_nsec = 0;
 	VFSATTR_SET_SUPPORTED(fsap, f_backup_time);
 	if (VFSATTR_IS_ACTIVE(fsap, f_fssubtype)) {
-		uint16_t subtype = 0;
+		u_int16_t subtype = 0;
 
 		/*
 		 * Subtypes (flavors) for HFS
@@ -3668,8 +4710,7 @@ hfs_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t
 	}
 
 	if (VFSATTR_IS_ACTIVE(fsap, f_vol_name)) {
-		strncpy(fsap->f_vol_name, hfsmp->vcbVN, MAXPATHLEN);
-		fsap->f_vol_name[MAXPATHLEN - 1] = 0;
+		strlcpy(fsap->f_vol_name, (char *) hfsmp->vcbVN, MAXPATHLEN);
 		VFSATTR_SET_SUPPORTED(fsap, f_vol_name);
 	}
 	return (0);
@@ -3706,7 +4747,7 @@ hfs_rename_volume(struct vnode *vp, const char *name, proc_t p)
 	todir_desc.cd_cnid = kHFSRootFolderID;
 	todir_desc.cd_flags = CD_ISDIR;
 
-	to_desc.cd_nameptr = name;
+	to_desc.cd_nameptr = (const u_int8_t *)name;
 	to_desc.cd_namelen = strlen(name);
 	to_desc.cd_parentcnid = kHFSRootParentID;
 	to_desc.cd_cnid = cp->c_cnid;
@@ -3723,15 +4764,14 @@ hfs_rename_volume(struct vnode *vp, const char *name, proc_t p)
 				 * If successful, update the name in the VCB, ensure it's terminated.
 				 */
 				if (!error) {
-					strncpy(vcb->vcbVN, name, sizeof(vcb->vcbVN));
-					vcb->vcbVN[sizeof(vcb->vcbVN) - 1] = 0;
+					strlcpy((char *)vcb->vcbVN, name, sizeof(vcb->vcbVN));
 				}
 
 				hfs_systemfile_unlock(hfsmp, lockflags);
 				cat_postflight(hfsmp, &cookie, p);
 			
 				if (error)
-					vcb->vcbFlags |= 0xFF00;
+					MarkVCBDirty(vcb);
 				(void) hfs_flushvolumeheader(hfsmp, MNT_WAIT, 0);
 			}
 			hfs_end_transaction(hfsmp);
@@ -3739,12 +4779,12 @@ hfs_rename_volume(struct vnode *vp, const char *name, proc_t p)
 		if (!error) {
 			/* Release old allocated name buffer */
 			if (cp->c_desc.cd_flags & CD_HASBUF) {
-				char *name = cp->c_desc.cd_nameptr;
+				const char *tmp_name = (const char *)cp->c_desc.cd_nameptr;
 		
 				cp->c_desc.cd_nameptr = 0;
 				cp->c_desc.cd_namelen = 0;
 				cp->c_desc.cd_flags &= ~CD_HASBUF;
-				vfs_removename(name);
+				vfs_removename(tmp_name);
 			}			
 			/* Update cnode's catalog descriptor */
 			replace_desc(cp, &new_desc);
@@ -3792,6 +4832,73 @@ out:
 	return error;
 }
 
+/* If a runtime corruption is detected, set the volume inconsistent 
+ * bit in the volume attributes.  The volume inconsistent bit is a persistent
+ * bit which represents that the volume is corrupt and needs repair.  
+ * The volume inconsistent bit can be set from the kernel when it detects
+ * runtime corruption or from file system repair utilities like fsck_hfs when
+ * a repair operation fails.  The bit should be cleared only from file system 
+ * verify/repair utility like fsck_hfs when a verify/repair succeeds.
+ */
+void hfs_mark_volume_inconsistent(struct hfsmount *hfsmp)
+{
+	HFS_MOUNT_LOCK(hfsmp, TRUE);	
+	if ((hfsmp->vcbAtrb & kHFSVolumeInconsistentMask) == 0) {
+		hfsmp->vcbAtrb |= kHFSVolumeInconsistentMask;
+		MarkVCBDirty(hfsmp);
+	}
+	/* Log information to ASL log */
+	fslog_fs_corrupt(hfsmp->hfs_mp);
+	printf("HFS: Runtime corruption detected on %s, fsck will be forced on next mount.\n", hfsmp->vcbVN);
+	HFS_MOUNT_UNLOCK(hfsmp, TRUE);
+}
+
+/* Replay the journal on the device node provided.  Returns zero if 
+ * journal replay succeeded or no journal was supposed to be replayed.
+ */
+static int hfs_journal_replay(const char *devnode, vfs_context_t context)
+{
+	int retval = 0;
+	struct vnode *devvp = NULL;
+	struct mount *mp = NULL;
+	struct hfs_mount_args *args = NULL;
+
+	/* Lookup vnode for given raw device path */
+	retval = vnode_open(devnode, FREAD|FWRITE, 0, 0, &devvp, NULL);
+	if (retval) {
+		goto out;
+	}
+
+	/* Replay allowed only on raw devices */
+	if (!vnode_ischr(devvp)) {
+		retval = EINVAL;
+		goto out;
+	}
+
+	/* Create dummy mount structures */
+	MALLOC(mp, struct mount *, sizeof(struct mount), M_TEMP, M_WAITOK);
+	bzero(mp, sizeof(struct mount));
+	mount_lock_init(mp);
+
+	MALLOC(args, struct hfs_mount_args *, sizeof(struct hfs_mount_args), M_TEMP, M_WAITOK);
+	bzero(args, sizeof(struct hfs_mount_args));
+
+	retval = hfs_mountfs(devvp, mp, args, 1, context);
+	buf_flushdirtyblks(devvp, MNT_WAIT, 0, "hfs_journal_replay");
+
+out:
+	if (mp) {
+		mount_lock_destroy(mp);
+		FREE(mp, M_TEMP);
+	}
+	if (args) {
+		FREE(args, M_TEMP);
+	}
+	if (devvp) {
+		vnode_close(devvp, FREAD|FWRITE, NULL);
+	}
+	return retval;
+}
 
 /*
  * hfs vfs operations.
@@ -3809,5 +4916,6 @@ struct vfsops hfs_vfsops = {
 	hfs_vptofh,
 	hfs_init,
 	hfs_sysctl,
-	hfs_vfs_setattr
+	hfs_vfs_setattr,
+	{NULL}
 };
diff --git a/bsd/hfs/hfs_vfsutils.c b/bsd/hfs/hfs_vfsutils.c
index 09ec831df..0433aec32 100644
--- a/bsd/hfs/hfs_vfsutils.c
+++ b/bsd/hfs/hfs_vfsutils.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)hfs_vfsutils.c	4.0
 *
@@ -33,10 +39,15 @@
 #include <sys/stat.h>
 #include <sys/mount.h>
 #include <sys/buf.h>
+#include <sys/buf_internal.h>
 #include <sys/ubc.h>
 #include <sys/unistd.h>
 #include <sys/utfconv.h>
 #include <sys/kauth.h>
+#include <sys/fcntl.h>
+#include <sys/vnode_internal.h>
+
+#include <libkern/OSAtomic.h>
 
 #include "hfs.h"
 #include "hfs_catalog.h"
@@ -44,15 +55,12 @@
 #include "hfs_mount.h"
 #include "hfs_endian.h"
 #include "hfs_cnode.h"
+#include "hfs_fsctl.h"
 
 #include "hfscommon/headers/FileMgrInternal.h"
 #include "hfscommon/headers/BTreesInternal.h"
 #include "hfscommon/headers/HFSUnicodeWrappers.h"
 
-
-extern int count_lock_queue(void);
-
-
 static void ReleaseMetaFileVNode(struct vnode *vp);
 static int  hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_args);
 
@@ -60,12 +68,6 @@ static void hfs_metadatazone_init(struct hfsmount *);
 static u_int32_t hfs_hotfile_freeblocks(struct hfsmount *);
 
 
-
-u_int32_t GetLogicalBlockSize(struct vnode *vp);
-
-extern int hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKey);
-
-
 //*******************************************************************************
 // Note: Finder information in the HFS/HFS+ metadata are considered opaque and
 //       hence are not in the right byte order on little endian machines. It is
@@ -77,17 +79,16 @@ extern int hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKe
 //
 //
 //*******************************************************************************
-char hfs_catname[] = "Catalog B-tree";
-char hfs_extname[] = "Extents B-tree";
-char hfs_vbmname[] = "Volume Bitmap";
-char hfs_attrname[] = "Attribute B-tree";
+unsigned char hfs_catname[] = "Catalog B-tree";
+unsigned char hfs_extname[] = "Extents B-tree";
+unsigned char hfs_vbmname[] = "Volume Bitmap";
+unsigned char hfs_attrname[] = "Attribute B-tree";
+unsigned char hfs_startupname[] = "Startup File";
 
-char hfs_privdirname[] =
-	"\xE2\x90\x80\xE2\x90\x80\xE2\x90\x80\xE2\x90\x80HFS+ Private Data";
 
 __private_extern__
 OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
-		struct proc *p)
+		__unused struct proc *p)
 {
 	ExtendedVCB *vcb = HFSTOVCB(hfsmp);
 	int error;
@@ -121,6 +122,7 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 	vcb->vcbVBMSt		= SWAP_BE16 (mdb->drVBMSt);
 	vcb->nextAllocation	= SWAP_BE16 (mdb->drAllocPtr);
 	vcb->totalBlocks	= SWAP_BE16 (mdb->drNmAlBlks);
+	vcb->allocLimit		= vcb->totalBlocks;
 	vcb->blockSize		= SWAP_BE32 (mdb->drAlBlkSiz);
 	vcb->vcbClpSiz		= SWAP_BE32 (mdb->drClpSiz);
 	vcb->vcbAlBlSt		= SWAP_BE16 (mdb->drAlBlSt);
@@ -154,7 +156,7 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 	cndesc.cd_parentcnid = kHFSRootParentID;
 	cndesc.cd_flags |= CD_ISMETA;
 	bzero(&cnattr, sizeof(cnattr));
-	cnattr.ca_nlink = 1;
+	cnattr.ca_linkcount = 1;
 	cnattr.ca_mode = S_IFREG;
 	bzero(&fork, sizeof(fork));
 
@@ -162,7 +164,7 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 	 * Set up Extents B-tree vnode
 	 */
 	cndesc.cd_nameptr = hfs_extname;
-	cndesc.cd_namelen = strlen(hfs_extname);
+	cndesc.cd_namelen = strlen((char *)hfs_extname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSExtentsFileID;
 	fork.cf_size = SWAP_BE32(mdb->drXTFlSize);
 	fork.cf_blocks = fork.cf_size / vcb->blockSize;
@@ -185,12 +187,13 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto MtVolErr;
 	}
+	hfsmp->hfs_extents_cp = VTOC(hfsmp->hfs_extents_vp);
 
 	/*
 	 * Set up Catalog B-tree vnode...
 	 */ 
 	cndesc.cd_nameptr = hfs_catname;
-	cndesc.cd_namelen = strlen(hfs_catname);
+	cndesc.cd_namelen = strlen((char *)hfs_catname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSCatalogFileID;
 	fork.cf_size = SWAP_BE32(mdb->drCTFlSize);
 	fork.cf_blocks = fork.cf_size / vcb->blockSize;
@@ -217,12 +220,13 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto MtVolErr;
 	}
+	hfsmp->hfs_catalog_cp = VTOC(hfsmp->hfs_catalog_vp);
 
 	/*
 	 * Set up dummy Allocation file vnode (used only for locking bitmap)
 	 */  
 	cndesc.cd_nameptr = hfs_vbmname;
-	cndesc.cd_namelen = strlen(hfs_vbmname);
+	cndesc.cd_namelen = strlen((char *)hfs_vbmname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSAllocationFileID;
 	bzero(&fork, sizeof(fork));
 	cnattr.ca_blocks = 0;
@@ -234,6 +238,7 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto MtVolErr;
 	}
+	hfsmp->hfs_allocation_cp = VTOC(hfsmp->hfs_allocation_vp);
 
       	/* mark the volume dirty (clear clean unmount bit) */
 	vcb->vcbAtrb &=	~kHFSVolumeUnmountedMask;
@@ -245,10 +250,10 @@ OSErr hfs_MountHFSVolume(struct hfsmount *hfsmp, HFSMasterDirectoryBlock *mdb,
 	hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
 	hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 
-	if (error == noErr)
-	  {
-	    error = cat_idlookup(hfsmp, kHFSRootFolderID, NULL, NULL, NULL);
-	  }
+    if (error == noErr)
+      {
+		error = cat_idlookup(hfsmp, kHFSRootFolderID, 0, NULL, NULL, NULL);
+      }
 
     if ( error == noErr )
       {
@@ -276,31 +281,31 @@ CmdDone:
 
 __private_extern__
 OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
-	off_t embeddedOffset, u_int64_t disksize, struct proc *p, void *args, kauth_cred_t cred)
+	off_t embeddedOffset, u_int64_t disksize, __unused struct proc *p, void *args, kauth_cred_t cred)
 {
 	register ExtendedVCB *vcb;
 	struct cat_desc cndesc;
 	struct cat_attr cnattr;
 	struct cat_fork cfork;
-	UInt32 blockSize;
+	u_int32_t blockSize;
 	daddr64_t spare_sectors;
 	struct BTreeInfoRec btinfo;
 	u_int16_t  signature;
-	u_int16_t  version;
+	u_int16_t  hfs_version;
 	int  i;
 	OSErr retval;
 
 	signature = SWAP_BE16(vhp->signature);
-	version = SWAP_BE16(vhp->version);
+	hfs_version = SWAP_BE16(vhp->version);
 
 	if (signature == kHFSPlusSigWord) {
-		if (version != kHFSPlusVersion) {
-			printf("hfs_mount: invalid HFS+ version: %d\n", version);
+		if (hfs_version != kHFSPlusVersion) {
+			printf("hfs_mount: invalid HFS+ version: %d\n", hfs_version);
 			return (EINVAL);
 		}
 	} else if (signature == kHFSXSigWord) {
-		if (version != kHFSXVersion) {
-			printf("hfs_mount: invalid HFSX version: %d\n", version);
+		if (hfs_version != kHFSXVersion) {
+			printf("hfs_mount: invalid HFSX version: %d\n", hfs_version);
 			return (EINVAL);
 		}
 		/* The in-memory signature is always 'H+'. */
@@ -356,6 +361,7 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	/* Now fill in the Extended VCB info */
 	vcb->nextAllocation	= SWAP_BE32(vhp->nextAllocation);
 	vcb->totalBlocks	= SWAP_BE32(vhp->totalBlocks);
+	vcb->allocLimit		= vcb->totalBlocks;
 	vcb->freeBlocks		= SWAP_BE32(vhp->freeBlocks);
 	vcb->blockSize		= blockSize;
 	vcb->encodingsBitmap	= SWAP_BE64(vhp->encodingsBitmap);
@@ -393,14 +399,14 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	cndesc.cd_parentcnid = kHFSRootParentID;
 	cndesc.cd_flags |= CD_ISMETA;
 	bzero(&cnattr, sizeof(cnattr));
-	cnattr.ca_nlink = 1;
+	cnattr.ca_linkcount = 1;
 	cnattr.ca_mode = S_IFREG;
 
 	/*
 	 * Set up Extents B-tree vnode
 	 */
 	cndesc.cd_nameptr = hfs_extname;
-	cndesc.cd_namelen = strlen(hfs_extname);
+	cndesc.cd_namelen = strlen((char *)hfs_extname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSExtentsFileID;
 
 	cfork.cf_size    = SWAP_BE64 (vhp->extentsFile.logicalSize);
@@ -416,20 +422,20 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	}
 	retval = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork,
 	                         &hfsmp->hfs_extents_vp);
+	if (retval)
+		goto ErrorExit;
+	hfsmp->hfs_extents_cp = VTOC(hfsmp->hfs_extents_vp);
+	hfs_unlock(hfsmp->hfs_extents_cp);
 
-	if (retval) goto ErrorExit;
 	retval = MacToVFSError(BTOpenPath(VTOF(hfsmp->hfs_extents_vp),
 	                                  (KeyCompareProcPtr) CompareExtentKeysPlus));
-	if (retval) {
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
+	if (retval)
 		goto ErrorExit;
-	}
-
 	/*
 	 * Set up Catalog B-tree vnode
 	 */ 
 	cndesc.cd_nameptr = hfs_catname;
-	cndesc.cd_namelen = strlen(hfs_catname);
+	cndesc.cd_namelen = strlen((char *)hfs_catname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSCatalogFileID;
 
 	cfork.cf_size    = SWAP_BE64 (vhp->catalogFile.logicalSize);
@@ -446,14 +452,14 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	retval = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork,
 	                         &hfsmp->hfs_catalog_vp);
 	if (retval) {
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto ErrorExit;
 	}
+	hfsmp->hfs_catalog_cp = VTOC(hfsmp->hfs_catalog_vp);
+	hfs_unlock(hfsmp->hfs_catalog_cp);
+
 	retval = MacToVFSError(BTOpenPath(VTOF(hfsmp->hfs_catalog_vp),
 	                                  (KeyCompareProcPtr) CompareExtendedCatalogKeys));
 	if (retval) {
-		hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto ErrorExit;
 	}
 	if ((hfsmp->hfs_flags & HFS_X) &&
@@ -470,7 +476,7 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	 * Set up Allocation file vnode
 	 */  
 	cndesc.cd_nameptr = hfs_vbmname;
-	cndesc.cd_namelen = strlen(hfs_vbmname);
+	cndesc.cd_namelen = strlen((char *)hfs_vbmname);
 	cndesc.cd_cnid = cnattr.ca_fileid = kHFSAllocationFileID;
 
 	cfork.cf_size    = SWAP_BE64 (vhp->allocationFile.logicalSize);
@@ -487,17 +493,17 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	retval = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork,
 	                         &hfsmp->hfs_allocation_vp);
 	if (retval) {
-		hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto ErrorExit;
 	}
+	hfsmp->hfs_allocation_cp = VTOC(hfsmp->hfs_allocation_vp);
+	hfs_unlock(hfsmp->hfs_allocation_cp);
 
 	/*
 	 * Set up Attribute B-tree vnode
 	 */
 	if (vhp->attributesFile.totalBlocks != 0) {
 		cndesc.cd_nameptr = hfs_attrname;
-		cndesc.cd_namelen = strlen(hfs_attrname);
+		cndesc.cd_namelen = strlen((char *)hfs_attrname);
 		cndesc.cd_cnid = cnattr.ca_fileid = kHFSAttributesFileID;
 	
 		cfork.cf_size    = SWAP_BE64 (vhp->attributesFile.logicalSize);
@@ -514,31 +520,48 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		retval = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork,
 					 &hfsmp->hfs_attribute_vp);
 		if (retval) {
-			hfs_unlock(VTOC(hfsmp->hfs_allocation_vp));
-			hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-			hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 			goto ErrorExit;
 		}
+		hfsmp->hfs_attribute_cp = VTOC(hfsmp->hfs_attribute_vp);
+		hfs_unlock(hfsmp->hfs_attribute_cp);
 		retval = MacToVFSError(BTOpenPath(VTOF(hfsmp->hfs_attribute_vp),
 						  (KeyCompareProcPtr) hfs_attrkeycompare));
 		if (retval) {
-			hfs_unlock(VTOC(hfsmp->hfs_attribute_vp));
-			hfs_unlock(VTOC(hfsmp->hfs_allocation_vp));
-			hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-			hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 			goto ErrorExit;
 		}
 	}
 
-
+	/*
+	 * Set up Startup file vnode
+	 */
+	if (vhp->startupFile.totalBlocks != 0) {
+		cndesc.cd_nameptr = hfs_startupname;
+		cndesc.cd_namelen = strlen((char *)hfs_startupname);
+		cndesc.cd_cnid = cnattr.ca_fileid = kHFSStartupFileID;
+	
+		cfork.cf_size    = SWAP_BE64 (vhp->startupFile.logicalSize);
+		cfork.cf_clump   = SWAP_BE32 (vhp->startupFile.clumpSize);
+		cfork.cf_blocks  = SWAP_BE32 (vhp->startupFile.totalBlocks);
+		cfork.cf_vblocks = 0;
+		cnattr.ca_blocks = cfork.cf_blocks;
+		for (i = 0; i < kHFSPlusExtentDensity; i++) {
+			cfork.cf_extents[i].startBlock =
+					SWAP_BE32 (vhp->startupFile.extents[i].startBlock);
+			cfork.cf_extents[i].blockCount =
+					SWAP_BE32 (vhp->startupFile.extents[i].blockCount);
+		}
+		retval = hfs_getnewvnode(hfsmp, NULL, NULL, &cndesc, 0, &cnattr, &cfork,
+					 &hfsmp->hfs_startup_vp);
+		if (retval) {
+			goto ErrorExit;
+		}
+		hfsmp->hfs_startup_cp = VTOC(hfsmp->hfs_startup_vp);
+		hfs_unlock(hfsmp->hfs_startup_cp);
+	}
+	
 	/* Pick up volume name and create date */
-	retval = cat_idlookup(hfsmp, kHFSRootFolderID, &cndesc, &cnattr, NULL);
+	retval = cat_idlookup(hfsmp, kHFSRootFolderID, 0, &cndesc, &cnattr, NULL);
 	if (retval) {
-		if (hfsmp->hfs_attribute_vp)
-			hfs_unlock(VTOC(hfsmp->hfs_attribute_vp));
-		hfs_unlock(VTOC(hfsmp->hfs_allocation_vp));
-		hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
 		goto ErrorExit;
 	}
 	vcb->vcbCrDate = cnattr.ca_itime;
@@ -552,14 +575,10 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		hfs_flushvolumeheader(hfsmp, TRUE, 0);
 	}
 
-	/*
-	 * all done with metadata files so we can unlock now...
-	 */
-	if (hfsmp->hfs_attribute_vp)
-		hfs_unlock(VTOC(hfsmp->hfs_attribute_vp));
-	hfs_unlock(VTOC(hfsmp->hfs_allocation_vp));
-	hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
-	hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
+	/* kHFSHasFolderCount is only supported/updated on HFSX volumes */
+	if ((hfsmp->hfs_flags & HFS_X) != 0) {
+		hfsmp->hfs_flags |= HFS_FOLDERCOUNT;
+	}
 
 	//
 	// Check if we need to do late journal initialization.  This only
@@ -569,7 +588,7 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	//
 	if (   (vcb->vcbAtrb & kHFSVolumeJournaledMask)
 		&& (SWAP_BE32(vhp->lastMountedVersion) != kHFSJMountVersion)
-	        && (hfsmp->jnl == NULL)) {
+		&& (hfsmp->jnl == NULL)) {
 
 		retval = hfs_late_journal_init(hfsmp, vhp, args);
 		if (retval != 0) {
@@ -609,7 +628,7 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 			retval = EINVAL;
 			goto ErrorExit;
 		} else if (hfsmp->jnl) {
-			vfs_setflags(hfsmp->hfs_mp, (uint64_t)((unsigned int)MNT_JOURNALED));
+			vfs_setflags(hfsmp->hfs_mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
 		}
 	} else if (hfsmp->jnl || ((vcb->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY))) {
 		struct cat_attr jinfo_attr, jnl_attr;
@@ -630,6 +649,10 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		if (hfsmp->hfs_flags & HFS_READ_ONLY) {
 		    vcb->vcbAtrb |= kHFSVolumeJournaledMask;
 		}
+
+		if (hfsmp->jnl == NULL) {
+		    vfs_clearflags(hfsmp->hfs_mp, (u_int64_t)((unsigned int)MNT_JOURNALED));
+		}
 	}
 
 	/*
@@ -644,12 +667,14 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		/* Keep the roving allocator out of the metadata zone. */
 		if (vcb->nextAllocation >= hfsmp->hfs_metazone_start &&
 		    vcb->nextAllocation <= hfsmp->hfs_metazone_end) {	    
-			vcb->nextAllocation = hfsmp->hfs_metazone_end + 1;
+			HFS_UPDATE_NEXT_ALLOCATION(hfsmp, hfsmp->hfs_metazone_end + 1);
 		}
 	}
 
-	/* setup private/hidden directory for unlinked files */
-	FindMetaDataDirectory(vcb);
+	/* Setup private/hidden directories for hardlinks. */
+	hfs_privatedir_init(hfsmp, FILE_HARDLINKS);
+	hfs_privatedir_init(hfsmp, DIR_HARDLINKS);
+
 	if ((hfsmp->hfs_flags & HFS_READ_ONLY) == 0) 
 		hfs_remove_orphans(hfsmp);
 
@@ -666,7 +691,11 @@ OSErr hfs_MountHFSPlusVolume(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		(void) hfs_recording_init(hfsmp);
 	}
 
-	hfs_checkextendedsecurity(hfsmp);
+	/* Force ACLs on HFS+ file systems. */
+	vfs_setextendedsecurity(HFSTOVFS(hfsmp));
+
+	/* Check if volume supports writing of extent-based extended attributes */
+	hfs_check_volxattr(hfsmp, HFS_SET_XATTREXTENTS_STATE);
 
 	return (0);
 
@@ -717,8 +746,22 @@ static void ReleaseMetaFileVNode(struct vnode *vp)
 
 __private_extern__
 int
-hfsUnmount( register struct hfsmount *hfsmp, struct proc *p)
+hfsUnmount( register struct hfsmount *hfsmp, __unused struct proc *p)
 {
+	/* Get rid of our attribute data vnode (if any). */
+	if (hfsmp->hfs_attrdata_vp) {
+		vnode_t advp = hfsmp->hfs_attrdata_vp;
+	
+		if (vnode_get(advp) == 0) {
+			vnode_rele_ext(advp, O_EVTONLY, 0);
+			vnode_put(advp);
+		}
+		hfsmp->hfs_attrdata_vp = NULLVP;
+	}
+
+	if (hfsmp->hfs_startup_vp)
+		ReleaseMetaFileVNode(hfsmp->hfs_startup_vp);
+
 	if (hfsmp->hfs_allocation_vp)
 		ReleaseMetaFileVNode(hfsmp->hfs_allocation_vp);
 
@@ -728,6 +771,18 @@ hfsUnmount( register struct hfsmount *hfsmp, struct proc *p)
 	ReleaseMetaFileVNode(hfsmp->hfs_catalog_vp);
 	ReleaseMetaFileVNode(hfsmp->hfs_extents_vp);
 
+	/*
+	 * Setting these pointers to NULL so that any references
+	 * past this point will fail, and tell us the point of failure.
+	 * Also, facilitates a check in hfs_update for a null catalog
+	 * vp
+	 */
+	hfsmp->hfs_allocation_vp = NULL;
+	hfsmp->hfs_attribute_vp = NULL;
+	hfsmp->hfs_catalog_vp = NULL;
+	hfsmp->hfs_extents_vp = NULL;
+	hfsmp->hfs_startup_vp = NULL;
+
 	return (0);
 }
 
@@ -741,14 +796,14 @@ overflow_extents(struct filefork *fp)
 {
 	u_long blocks;
 
-       //
-       // If the vnode pointer is NULL then we're being called
-       // from hfs_remove_orphans() with a faked-up filefork
-       // and therefore it has to be an HFS+ volume.  Otherwise
-       // we check through the volume header to see what type
-       // of volume we're on.
-       //
-       if (FTOV(fp) == NULL || VTOVCB(FTOV(fp))->vcbSigWord == kHFSPlusSigWord) {
+	//
+	// If the vnode pointer is NULL then we're being called
+	// from hfs_remove_orphans() with a faked-up filefork
+	// and therefore it has to be an HFS+ volume.  Otherwise
+	// we check through the volume header to see what type
+	// of volume we're on.
+        //
+	if (FTOV(fp) == NULL || VTOVCB(FTOV(fp))->vcbSigWord == kHFSPlusSigWord) {
 		if (fp->ff_extents[7].blockCount == 0)
 			return (0);
 
@@ -780,13 +835,24 @@ __private_extern__
 int
 hfs_systemfile_lock(struct hfsmount *hfsmp, int flags, enum hfslocktype locktype)
 {
-	if (flags & ~SFL_VALIDMASK)
-		panic("hfs_systemfile_lock: invalid lock request (0x%x)", (unsigned long) flags);
 	/*
-	 * Locking order is Catalog file, Attributes file, Bitmap file, Extents file
+	 * Locking order is Catalog file, Attributes file, Startup file, Bitmap file, Extents file
 	 */
 	if (flags & SFL_CATALOG) {
-		(void) hfs_lock(VTOC(hfsmp->hfs_catalog_vp), locktype);
+
+#ifdef HFS_CHECK_LOCK_ORDER
+		if (hfsmp->hfs_attribute_cp && hfsmp->hfs_attribute_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Attributes before Catalog)");
+		}
+		if (hfsmp->hfs_startup_cp && hfsmp->hfs_startup_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Startup before Catalog)");
+		}
+		if (hfsmp-> hfs_extents_cp && hfsmp->hfs_extents_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Extents before Catalog)");
+		}
+#endif /* HFS_CHECK_LOCK_ORDER */
+
+		(void) hfs_lock(hfsmp->hfs_catalog_cp, locktype);
 		/*
 		 * When the catalog file has overflow extents then
 		 * also acquire the extents b-tree lock if its not
@@ -798,8 +864,18 @@ hfs_systemfile_lock(struct hfsmount *hfsmp, int flags, enum hfslocktype locktype
 		}
 	}
 	if (flags & SFL_ATTRIBUTE) {
-		if (hfsmp->hfs_attribute_vp) {
-			(void) hfs_lock(VTOC(hfsmp->hfs_attribute_vp), locktype);
+
+#ifdef HFS_CHECK_LOCK_ORDER
+		if (hfsmp->hfs_startup_cp && hfsmp->hfs_startup_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Startup before Attributes)");
+		}
+		if (hfsmp->hfs_extents_cp && hfsmp->hfs_extents_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Extents before Attributes)");
+		}
+#endif /* HFS_CHECK_LOCK_ORDER */
+
+		if (hfsmp->hfs_attribute_cp) {
+			(void) hfs_lock(hfsmp->hfs_attribute_cp, locktype);
 			/*
 			 * When the attribute file has overflow extents then
 			 * also acquire the extents b-tree lock if its not
@@ -813,7 +889,29 @@ hfs_systemfile_lock(struct hfsmount *hfsmp, int flags, enum hfslocktype locktype
 			flags &= ~SFL_ATTRIBUTE;
 		}
 	}
-	if (flags & SFL_BITMAP) {
+	if (flags & SFL_STARTUP) {
+#ifdef HFS_CHECK_LOCK_ORDER
+		if (hfsmp-> hfs_extents_cp && hfsmp->hfs_extents_cp->c_lockowner == current_thread()) {
+			panic("hfs_systemfile_lock: bad lock order (Extents before Startup)");
+		}
+#endif /* HFS_CHECK_LOCK_ORDER */
+
+		(void) hfs_lock(hfsmp->hfs_startup_cp, locktype);
+		/*
+		 * When the startup file has overflow extents then
+		 * also acquire the extents b-tree lock if its not
+		 * already requested.
+		 */
+		if ((flags & SFL_EXTENTS) == 0 &&
+		    overflow_extents(VTOF(hfsmp->hfs_startup_vp))) {
+			flags |= SFL_EXTENTS;
+		}
+	}
+	/* 
+	 * To prevent locks being taken in the wrong order, the extent lock
+	 * gets a bitmap lock as well.
+	 */
+	if (flags & (SFL_BITMAP | SFL_EXTENTS)) {
 		/*
 		 * Since the only bitmap operations are clearing and
 		 * setting bits we always need exclusive access. And
@@ -821,10 +919,15 @@ hfs_systemfile_lock(struct hfsmount *hfsmp, int flags, enum hfslocktype locktype
 		 * lock since we can only change the bitmap from
 		 * within a transaction.
 		 */
-		if (hfsmp->jnl) {
+		if (hfsmp->jnl || (hfsmp->hfs_allocation_cp == NULL)) {
 			flags &= ~SFL_BITMAP;
 		} else {
-			(void) hfs_lock(VTOC(hfsmp->hfs_allocation_vp), HFS_EXCLUSIVE_LOCK);
+			(void) hfs_lock(hfsmp->hfs_allocation_cp, HFS_EXCLUSIVE_LOCK);
+			/* The bitmap lock is also grabbed when only extent lock 
+			 * was requested. Set the bitmap lock bit in the lock
+			 * flags which callers will use during unlock.
+			 */
+			flags |= SFL_BITMAP;
 		}
 	}
 	if (flags & SFL_EXTENTS) {
@@ -832,7 +935,7 @@ hfs_systemfile_lock(struct hfsmount *hfsmp, int flags, enum hfslocktype locktype
 		 * Since the extents btree lock is recursive we always
 		 * need exclusive access.
 		 */
-		(void) hfs_lock(VTOC(hfsmp->hfs_extents_vp), HFS_EXCLUSIVE_LOCK);
+		(void) hfs_lock(hfsmp->hfs_extents_cp, HFS_EXCLUSIVE_LOCK);
 	}
 	return (flags);
 }
@@ -848,13 +951,14 @@ hfs_systemfile_unlock(struct hfsmount *hfsmp, int flags)
 	u_int32_t lastfsync;
 	int numOfLockedBuffs;
 
-	microuptime(&tv);
-	lastfsync = tv.tv_sec;
-	
-	if (flags & ~SFL_VALIDMASK)
-		panic("hfs_systemfile_unlock: invalid lock request (0x%x)", (unsigned long) flags);
-
-	if (flags & SFL_ATTRIBUTE && hfsmp->hfs_attribute_vp) {
+	if (hfsmp->jnl == NULL) {
+		microuptime(&tv);
+		lastfsync = tv.tv_sec;
+	}
+	if (flags & SFL_STARTUP && hfsmp->hfs_startup_cp) {
+		hfs_unlock(hfsmp->hfs_startup_cp);
+	}
+	if (flags & SFL_ATTRIBUTE && hfsmp->hfs_attribute_cp) {
 		if (hfsmp->jnl == NULL) {
 			BTGetLastSync((FCB*)VTOF(hfsmp->hfs_attribute_vp), &lastfsync);
 			numOfLockedBuffs = count_lock_queue();
@@ -864,7 +968,7 @@ hfs_systemfile_unlock(struct hfsmount *hfsmp, int flags)
 				hfs_btsync(hfsmp->hfs_attribute_vp, HFS_SYNCTRANS);
 			}
 		}
-		hfs_unlock(VTOC(hfsmp->hfs_attribute_vp));
+		hfs_unlock(hfsmp->hfs_attribute_cp);
 	}
 	if (flags & SFL_CATALOG) {
 		if (hfsmp->jnl == NULL) {
@@ -876,10 +980,10 @@ hfs_systemfile_unlock(struct hfsmount *hfsmp, int flags)
 				hfs_btsync(hfsmp->hfs_catalog_vp, HFS_SYNCTRANS);
 			}
 		}
-		hfs_unlock(VTOC(hfsmp->hfs_catalog_vp));
+		hfs_unlock(hfsmp->hfs_catalog_cp);
 	}
 	if (flags & SFL_BITMAP) {
-		hfs_unlock(VTOC(hfsmp->hfs_allocation_vp));
+		hfs_unlock(hfsmp->hfs_allocation_cp);
 	}
 	if (flags & SFL_EXTENTS) {
 		if (hfsmp->jnl == NULL) {
@@ -891,7 +995,7 @@ hfs_systemfile_unlock(struct hfsmount *hfsmp, int flags)
 				hfs_btsync(hfsmp->hfs_extents_vp, HFS_SYNCTRANS);
 			}
 		}
-		hfs_unlock(VTOC(hfsmp->hfs_extents_vp));
+		hfs_unlock(hfsmp->hfs_extents_cp);
 	}
 }
 
@@ -928,6 +1032,8 @@ void RequireFileLock(FileReference vp, int shareable)
 			if (VTOHFS(vp)->jnl == NULL)
 				panic("allocation file not locked! v: 0x%08X\n #\n", (u_int)vp);
 			break;
+		case kHFSStartupFileID:
+			panic("startup file not locked! v: 0x%08X\n #\n", (u_int)vp);
 		case kHFSAttributesFileID:
 			panic("attributes btree not locked! v: 0x%08X\n #\n", (u_int)vp);
 			break;
@@ -951,7 +1057,7 @@ void RequireFileLock(FileReference vp, int shareable)
  */
 int
 hfs_owner_rights(struct hfsmount *hfsmp, uid_t cnode_uid, kauth_cred_t cred,
-		struct proc *p, int invokesuperuserstatus)
+		__unused struct proc *p, int invokesuperuserstatus)
 {
 	if ((kauth_cred_getuid(cred) == cnode_uid) ||                                    /* [1a] */
 	    (cnode_uid == UNKNOWNUID) ||  									  /* [1b] */
@@ -1009,148 +1115,12 @@ unsigned long BestBlockSizeFit(unsigned long allocationBlockSize,
 }
 
 
-/*
- * To make the HFS Plus filesystem follow UFS unlink semantics, a remove
- * of an active vnode is translated to a move/rename so the file appears
- * deleted. The destination folder for these move/renames is setup here
- * and a reference to it is place in hfsmp->hfs_privdir_desc.
- */
-__private_extern__
-u_long
-FindMetaDataDirectory(ExtendedVCB *vcb)
-{
-	struct hfsmount * hfsmp;
-	struct vnode * dvp = NULL;
-	struct cnode * dcp = NULL;
-	struct FndrDirInfo * fndrinfo;
-	struct cat_desc out_desc = {0};
-	struct proc *p = current_proc();
-	struct timeval tv;
-	cat_cookie_t cookie;
-	int lockflags;
-	int error;
-	
-	if (vcb->vcbSigWord != kHFSPlusSigWord)
-		return (0);
-
-	hfsmp = VCBTOHFS(vcb);
-
-	if (hfsmp->hfs_privdir_desc.cd_parentcnid == 0) {
-		hfsmp->hfs_privdir_desc.cd_parentcnid = kRootDirID;
-		hfsmp->hfs_privdir_desc.cd_nameptr = hfs_privdirname;
-		hfsmp->hfs_privdir_desc.cd_namelen = strlen(hfs_privdirname);
-		hfsmp->hfs_privdir_desc.cd_flags = CD_ISDIR;
-	}
-
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-
-	error = cat_lookup(hfsmp, &hfsmp->hfs_privdir_desc, 0, NULL,
-			&hfsmp->hfs_privdir_attr, NULL, NULL);
-
-	hfs_systemfile_unlock(hfsmp, lockflags);
-
-	if (error == 0) {
-		hfsmp->hfs_metadata_createdate = hfsmp->hfs_privdir_attr.ca_itime;
-		hfsmp->hfs_privdir_desc.cd_cnid = hfsmp->hfs_privdir_attr.ca_fileid;
-		/*
-		 * Clear the system immutable flag if set...
-		 */
-		if ((hfsmp->hfs_privdir_attr.ca_flags & SF_IMMUTABLE) &&
-		    (hfsmp->hfs_flags & HFS_READ_ONLY) == 0) {
-			hfsmp->hfs_privdir_attr.ca_flags &= ~SF_IMMUTABLE;
-
-			if ((error = hfs_start_transaction(hfsmp)) != 0) {
-			    return (hfsmp->hfs_privdir_attr.ca_fileid);
-			}
-
-			lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-			(void) cat_update(hfsmp, &hfsmp->hfs_privdir_desc,
-			                 &hfsmp->hfs_privdir_attr, NULL, NULL);
-			hfs_systemfile_unlock(hfsmp, lockflags);
-
-			hfs_end_transaction(hfsmp);
-		}
-		return (hfsmp->hfs_privdir_attr.ca_fileid);
-
-	} else if (hfsmp->hfs_flags & HFS_READ_ONLY) {
-
-		return (0);
-	}
-    
-	/* Setup the default attributes */
-	bzero(&hfsmp->hfs_privdir_attr, sizeof(struct cat_attr));
-	hfsmp->hfs_privdir_attr.ca_mode = S_IFDIR;
-	hfsmp->hfs_privdir_attr.ca_nlink = 2;
-	hfsmp->hfs_privdir_attr.ca_itime = vcb->vcbCrDate;
-	microtime(&tv);
-	hfsmp->hfs_privdir_attr.ca_mtime = tv.tv_sec;
-
-	/* hidden and off the desktop view */
-	fndrinfo = (struct FndrDirInfo *)&hfsmp->hfs_privdir_attr.ca_finderinfo;
-	fndrinfo->frLocation.v = SWAP_BE16 (22460);
-	fndrinfo->frLocation.h = SWAP_BE16 (22460);
-	fndrinfo->frFlags |= SWAP_BE16 (kIsInvisible + kNameLocked);		
-
-	if ((error = hfs_start_transaction(hfsmp)) != 0) {
-	    return (0);
-	}
-	/* Reserve some space in the Catalog file. */
-	if (cat_preflight(hfsmp, CAT_CREATE, &cookie, p) != 0) {
-	    hfs_end_transaction(hfsmp);
-
-	    return (0);
-	}
-
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
-
-		error = cat_create(hfsmp, &hfsmp->hfs_privdir_desc,
-				&hfsmp->hfs_privdir_attr, &out_desc);
-
-	hfs_systemfile_unlock(hfsmp, lockflags);
-
-	cat_postflight(hfsmp, &cookie, p);
-	
-	if (error) {
-		hfs_volupdate(hfsmp, VOL_UPDATE, 0);
-
-		hfs_end_transaction(hfsmp);
-
-		return (0);
-	}
-
-	hfsmp->hfs_privdir_desc.cd_hint = out_desc.cd_hint;
-	hfsmp->hfs_privdir_desc.cd_cnid = out_desc.cd_cnid;
-	hfsmp->hfs_privdir_attr.ca_fileid = out_desc.cd_cnid;
-	hfsmp->hfs_metadata_createdate = vcb->vcbCrDate;
-
-	if (hfs_vget(hfsmp, kRootDirID, &dvp, 0) == 0) {
-		dcp = VTOC(dvp);
-		dcp->c_childhint = out_desc.cd_hint;
-		dcp->c_nlink++;
-		dcp->c_entries++;
-		dcp->c_touch_chgtime = TRUE;
-		dcp->c_touch_modtime = TRUE;
-		(void) hfs_update(dvp, 0);
-		hfs_unlock(dcp);
-		vnode_put(dvp);
-	}
-	hfs_volupdate(hfsmp, VOL_MKDIR, 1);
-	hfs_end_transaction(hfsmp);
-
-	cat_releasedesc(&out_desc);
-
-	return (out_desc.cd_cnid);
-}
-
 __private_extern__
 u_long
-GetFileInfo(ExtendedVCB *vcb, u_int32_t dirid, const char *name,
+GetFileInfo(ExtendedVCB *vcb, __unused u_int32_t dirid, const char *name,
 			struct cat_attr *fattr, struct cat_fork *forkinfo)
 {
 	struct hfsmount * hfsmp;
-	struct vnode * dvp = NULL;
-	struct cnode * dcp = NULL;
-	struct FndrDirInfo * fndrinfo;
 	struct cat_desc jdesc;
 	int lockflags;
 	int error;
@@ -1162,7 +1132,7 @@ GetFileInfo(ExtendedVCB *vcb, u_int32_t dirid, const char *name,
 
 	memset(&jdesc, 0, sizeof(struct cat_desc));
 	jdesc.cd_parentcnid = kRootDirID;
-	jdesc.cd_nameptr = name;
+	jdesc.cd_nameptr = (const u_int8_t *)name;
 	jdesc.cd_namelen = strlen(name);
 
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
@@ -1180,9 +1150,9 @@ GetFileInfo(ExtendedVCB *vcb, u_int32_t dirid, const char *name,
 
 
 /*
- * On HFS Plus Volume, there can be orphaned files.  These
- * are files that were unlinked while busy. If the volume
- * was not cleanly unmounted then some of these files may
+ * On HFS Plus Volumes, there can be orphaned files or directories
+ * These are files or directories that were unlinked while busy. 
+ * If the volume was not cleanly unmounted then some of these may
  * have persisted and need to be removed.
  */
 __private_extern__
@@ -1224,7 +1194,7 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 	
 	/* Build a key to "temp" */
 	keyp = (HFSPlusCatalogKey*)&iterator->key;
-	keyp->parentID = hfsmp->hfs_privdir_desc.cd_cnid;
+	keyp->parentID = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
 	keyp->nodeName.length = 4;  /* "temp" */
 	keyp->keyLength = kHFSPlusCatalogKeyMinimumLength + keyp->nodeName.length * 2;
 	keyp->nodeName.unicode[0] = 't';
@@ -1233,32 +1203,31 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 	keyp->nodeName.unicode[3] = 'p';
 
 	/*
-	 * Position the iterator just before the first real temp file.
+	 * Position the iterator just before the first real temp file/dir.
 	 */
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 	(void) BTSearchRecord(fcb, iterator, NULL, NULL, iterator);
 	hfs_systemfile_unlock(hfsmp, lockflags);
 
-	/* Visit all the temp files in the HFS+ private directory. */
+	/* Visit all the temp files/dirs in the HFS+ private directory. */
 	for (;;) {
 		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 		result = BTIterateRecord(fcb, kBTreeNextRecord, iterator, &btdata, NULL);
 		hfs_systemfile_unlock(hfsmp, lockflags);
 		if (result)
 			break;
-		if (keyp->parentID != hfsmp->hfs_privdir_desc.cd_cnid)
+		if (keyp->parentID != hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid)
 			break;
-		if (filerec.recordType != kHFSPlusFileRecord)
-			continue;
 		
 		(void) utf8_encodestr(keyp->nodeName.unicode, keyp->nodeName.length * 2,
-		                      filename, &namelen, sizeof(filename), 0, 0);
+		                      (u_int8_t *)filename, &namelen, sizeof(filename), 0, 0);
 		
-		(void) sprintf(tempname, "%s%d", HFS_DELETE_PREFIX, filerec.fileID);
+		(void) snprintf(tempname, sizeof(tempname), "%s%d",
+				HFS_DELETE_PREFIX, filerec.fileID);
 		
 		/*
-		 * Delete all files named "tempxxx", where
-		 * xxx is the file's cnid in decimal.
+		 * Delete all files (and directories) named "tempxxx", 
+		 * where xxx is the file's cnid in decimal.
 		 *
 		 */
 		if (bcmp(tempname, filename, namelen) == 0) {
@@ -1270,6 +1239,9 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 			bzero(&rfork, sizeof(rfork));
 			bzero(&cnode, sizeof(cnode));
 			
+			/* Delete any attributes, ignore errors */
+			(void) hfs_removeallattr(hfsmp, filerec.fileID);
+			
 			if (hfs_start_transaction(hfsmp) != 0) {
 			    printf("hfs_remove_orphans: failed to start transaction\n");
 			    goto exit;
@@ -1291,8 +1263,8 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 			/* Build a fake cnode */
 			cat_convertattr(hfsmp, (CatalogRecord *)&filerec, &cnode.c_attr,
 			                &dfork.ff_data, &rfork.ff_data);
-			cnode.c_desc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
-			cnode.c_desc.cd_nameptr = filename;
+			cnode.c_desc.cd_parentcnid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+			cnode.c_desc.cd_nameptr = (const u_int8_t *)filename;
 			cnode.c_desc.cd_namelen = namelen;
 			cnode.c_desc.cd_cnid = cnode.c_attr.ca_fileid;
 			cnode.c_blocks = dfork.ff_blocks + rfork.ff_blocks;
@@ -1312,7 +1284,7 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 				cnode.c_rsrcfork = NULL;
 				fsize = (u_int64_t)dfork.ff_blocks * (u_int64_t)HFSTOVCB(hfsmp)->blockSize;
 				while (fsize > 0) {
-					if (fsize > HFS_BIGFILE_SIZE && overflow_extents(&dfork)) {
+				    if (fsize > HFS_BIGFILE_SIZE && overflow_extents(&dfork)) {
 						fsize -= HFS_BIGFILE_SIZE;
 					} else {
 						fsize = 0;
@@ -1329,11 +1301,19 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 					// that no one transaction gets too big.
 					//
 					if (fsize > 0 && started_tr) {
+						/* Drop system file locks before starting 
+						 * another transaction to preserve lock order.
+						 */
+						hfs_systemfile_unlock(hfsmp, lockflags);
+						catlock = 0;
 						hfs_end_transaction(hfsmp);
+
 						if (hfs_start_transaction(hfsmp) != 0) {
 							started_tr = 0;
 							break;
 						}
+						lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE | SFL_EXTENTS | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+						catlock = 1;
 					}
 				}
 			}
@@ -1348,27 +1328,41 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 				}
 			}
 
-			/* Remove the file record from the Catalog */	
+			/* Remove the file or folder record from the Catalog */	
 			if (cat_delete(hfsmp, &cnode.c_desc, &cnode.c_attr) != 0) {
-				printf("hfs_remove_oprhans: error deleting cat rec for id %d!\n", cnode.c_desc.cd_cnid);
+				printf("hfs_remove_orphans: error deleting cat rec for id %d!\n", cnode.c_desc.cd_cnid);
+				hfs_systemfile_unlock(hfsmp, lockflags);
+				catlock = 0;
 				hfs_volupdate(hfsmp, VOL_UPDATE, 0);
 				break;
 			}
 			++orphanedlinks;
 
-			/* Delete any attributes, ignore errors */
-			(void) hfs_removeallattr(hfsmp, cnode.c_fileid);
-			
 			/* Update parent and volume counts */	
-			hfsmp->hfs_privdir_attr.ca_entries--;
-			(void)cat_update(hfsmp, &hfsmp->hfs_privdir_desc,
-			                 &hfsmp->hfs_privdir_attr, NULL, NULL);
- 			hfs_volupdate(hfsmp, VOL_RMFILE, 0);
+			hfsmp->hfs_private_attr[FILE_HARDLINKS].ca_entries--;
+			if (cnode.c_attr.ca_mode & S_IFDIR) {
+				DEC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[FILE_HARDLINKS]);
+			}
+
+			(void)cat_update(hfsmp, &hfsmp->hfs_private_desc[FILE_HARDLINKS],
+			                 &hfsmp->hfs_private_attr[FILE_HARDLINKS], NULL, NULL);
 
 			/* Drop locks and end the transaction */
 			hfs_systemfile_unlock(hfsmp, lockflags);
 			cat_postflight(hfsmp, &cookie, p);
 			catlock = catreserve = 0;
+
+			/* 
+			   Now that Catalog is unlocked, update the volume info, making
+			   sure to differentiate between files and directories
+			*/
+			if (cnode.c_attr.ca_mode & S_IFDIR) {
+				hfs_volupdate(hfsmp, VOL_RMDIR, 0);
+			}
+			else{
+ 				hfs_volupdate(hfsmp, VOL_RMFILE, 0);
+			}
+
 			if (started_tr) {
 				hfs_end_transaction(hfsmp);
 				started_tr = 0;
@@ -1377,7 +1371,7 @@ hfs_remove_orphans(struct hfsmount * hfsmp)
 		} /* end if */
 	} /* end for */
 	if (orphanedlinks > 0)
-		printf("HFS: Removed %d orphaned unlinked files\n", orphanedlinks);
+		printf("HFS: Removed %d orphaned unlinked files or directories \n", orphanedlinks);
 exit:
 	if (catlock) {
 		hfs_systemfile_unlock(hfsmp, lockflags);
@@ -1439,22 +1433,29 @@ __private_extern__
 u_int32_t
 hfs_freeblks(struct hfsmount * hfsmp, int wantreserve)
 {
-	ExtendedVCB *vcb = HFSTOVCB(hfsmp);
 	u_int32_t freeblks;
+	u_int32_t rsrvblks;
+	u_int32_t loanblks;
 
-	HFS_MOUNT_LOCK(hfsmp, TRUE);
-	freeblks = vcb->freeBlocks;
+	/*
+	 * We don't bother taking the mount lock
+	 * to look at these values since the values
+	 * themselves are each updated automically
+	 * on aligned addresses.
+	 */
+	freeblks = hfsmp->freeBlocks;
+	rsrvblks = hfsmp->reserveBlocks;
+	loanblks = hfsmp->loanedBlocks;
 	if (wantreserve) {
-		if (freeblks > vcb->reserveBlocks)
-			freeblks -= vcb->reserveBlocks;
+		if (freeblks > rsrvblks)
+			freeblks -= rsrvblks;
 		else
 			freeblks = 0;
 	}
-	if (freeblks > vcb->loanedBlocks)
-		freeblks -= vcb->loanedBlocks;
+	if (freeblks > loanblks)
+		freeblks -= loanblks;
 	else
 		freeblks = 0;
-	HFS_MOUNT_UNLOCK(hfsmp, TRUE);
 
 #ifdef HFS_SPARSE_DEV
 	/* 
@@ -1463,26 +1464,33 @@ hfs_freeblks(struct hfsmount * hfsmp, int wantreserve)
 	 */
 	if ((hfsmp->hfs_flags & HFS_HAS_SPARSE_DEVICE) && hfsmp->hfs_backingfs_rootvp) {
 		struct vfsstatfs *vfsp;  /* 272 bytes */
-		u_int32_t vfreeblks;
+		u_int64_t vfreeblks;
 		u_int32_t loanedblks;
 		struct mount * backingfs_mp;
+		struct timeval now;
 
 		backingfs_mp = vnode_mount(hfsmp->hfs_backingfs_rootvp);
 
-		if (vfsp = vfs_statfs(backingfs_mp)) {
+		microtime(&now);
+		if ((now.tv_sec - hfsmp->hfs_last_backingstatfs) >= 1) {
+		    vfs_update_vfsstat(backingfs_mp, vfs_context_kernel(), VFS_KERNEL_EVENT);
+		    hfsmp->hfs_last_backingstatfs = now.tv_sec;
+		}
+
+		if ((vfsp = vfs_statfs(backingfs_mp))) {
 			HFS_MOUNT_LOCK(hfsmp, TRUE);
-			vfreeblks = (u_int32_t)vfsp->f_bavail;
+			vfreeblks = vfsp->f_bavail;
 			/* Normalize block count if needed. */
-			if (vfsp->f_bsize != vcb->blockSize) {
-				vfreeblks = ((u_int64_t)vfreeblks * (u_int64_t)(vfsp->f_bsize)) / vcb->blockSize;
+			if (vfsp->f_bsize != hfsmp->blockSize) {
+				vfreeblks = ((u_int64_t)vfreeblks * (u_int64_t)(vfsp->f_bsize)) / hfsmp->blockSize;
 			}
-			if (vfreeblks > hfsmp->hfs_sparsebandblks)
+			if (vfreeblks > (unsigned int)hfsmp->hfs_sparsebandblks)
 				vfreeblks -= hfsmp->hfs_sparsebandblks;
 			else
 				vfreeblks = 0;
 			
 			/* Take into account any delayed allocations. */
-			loanedblks = 2 * vcb->loanedBlocks;
+			loanedblks = 2 * hfsmp->loanedBlocks;
 			if (vfreeblks > loanedblks)
 				vfreeblks -= loanedblks;
 			else
@@ -1556,15 +1564,17 @@ short MacToVFSError(OSErr err)
  * Find the current thread's directory hint for a given index.
  *
  * Requires an exclusive lock on directory cnode.
+ *
+ * Use detach if the cnode lock must be dropped while the hint is still active.
  */
 __private_extern__
 directoryhint_t *
-hfs_getdirhint(struct cnode *dcp, int index)
+hfs_getdirhint(struct cnode *dcp, int index, int detach)
 {
 	struct timeval tv;
 	directoryhint_t *hint;
 	boolean_t need_remove, need_init;
-	char * name;
+	const u_int8_t * name;
 
 	microuptime(&tv);
 
@@ -1590,9 +1600,12 @@ hfs_getdirhint(struct cnode *dcp, int index)
 			need_remove = false;
 		} else {				/* recycle the last (i.e., the oldest) hint */
 			hint = TAILQ_LAST(&dcp->c_hintlist, hfs_hinthead);
-			if ((name = hint->dh_desc.cd_nameptr)) {
+			if ((hint->dh_desc.cd_flags & CD_HASBUF) &&
+			    (name = hint->dh_desc.cd_nameptr)) {
 				hint->dh_desc.cd_nameptr = NULL;
-				vfs_removename(name);
+				hint->dh_desc.cd_namelen = 0;
+				hint->dh_desc.cd_flags &= ~CD_HASBUF;				
+				vfs_removename((const char *)name);
 			}
 			need_remove = true;
 		}
@@ -1601,7 +1614,10 @@ hfs_getdirhint(struct cnode *dcp, int index)
 	if (need_remove)
 		TAILQ_REMOVE(&dcp->c_hintlist, hint, dh_link);
 
-	TAILQ_INSERT_HEAD(&dcp->c_hintlist, hint, dh_link);
+	if (detach)
+		--dcp->c_dirhintcnt;
+	else
+		TAILQ_INSERT_HEAD(&dcp->c_hintlist, hint, dh_link);
 
 	if (need_init) {
 		hint->dh_index = index;
@@ -1609,7 +1625,7 @@ hfs_getdirhint(struct cnode *dcp, int index)
 		hint->dh_desc.cd_encoding = 0;
 		hint->dh_desc.cd_namelen = 0;
 		hint->dh_desc.cd_nameptr = NULL;
-		hint->dh_desc.cd_parentcnid = dcp->c_cnid;
+		hint->dh_desc.cd_parentcnid = dcp->c_fileid;
 		hint->dh_desc.cd_hint = dcp->c_childhint;
 		hint->dh_desc.cd_cnid = 0;
 	}
@@ -1626,16 +1642,25 @@ __private_extern__
 void
 hfs_reldirhint(struct cnode *dcp, directoryhint_t * relhint)
 {
-	char * name;
+	const u_int8_t * name;
+	directoryhint_t *hint;
 
-	TAILQ_REMOVE(&dcp->c_hintlist, relhint, dh_link);
+	/* Check if item is on list (could be detached) */
+	TAILQ_FOREACH(hint, &dcp->c_hintlist, dh_link) {
+		if (hint == relhint) {
+			TAILQ_REMOVE(&dcp->c_hintlist, relhint, dh_link);
+			--dcp->c_dirhintcnt;
+			break;
+		}
+	}
 	name = relhint->dh_desc.cd_nameptr;
-	if (name != NULL) {
+	if ((relhint->dh_desc.cd_flags & CD_HASBUF) && (name != NULL)) {
 		relhint->dh_desc.cd_nameptr = NULL;
-		vfs_removename(name);
+		relhint->dh_desc.cd_namelen = 0;
+		relhint->dh_desc.cd_flags &= ~CD_HASBUF;
+		vfs_removename((const char *)name);
 	}
 	FREE_ZONE(relhint, sizeof(directoryhint_t), M_HFSDIRHINT);
-	--dcp->c_dirhintcnt;
 }
 
 /*
@@ -1649,7 +1674,7 @@ hfs_reldirhints(struct cnode *dcp, int stale_hints_only)
 {
 	struct timeval tv;
 	directoryhint_t *hint, *prev;
-	char * name;
+	const u_int8_t * name;
 
 	if (stale_hints_only)
 		microuptime(&tv);
@@ -1659,9 +1684,11 @@ hfs_reldirhints(struct cnode *dcp, int stale_hints_only)
 		if (stale_hints_only && (tv.tv_sec - hint->dh_time) < HFS_DIRHINT_TTL)
 			break;  /* stop here if this entry is too new */
 		name = hint->dh_desc.cd_nameptr;
-		if (name != NULL) {
+		if ((hint->dh_desc.cd_flags & CD_HASBUF) && (name != NULL)) {
 			hint->dh_desc.cd_nameptr = NULL;
-			vfs_removename(name);
+			hint->dh_desc.cd_namelen = 0;
+			hint->dh_desc.cd_flags &= ~CD_HASBUF;
+			vfs_removename((const char *)name);
 		}
 		prev = TAILQ_PREV(hint, hfs_hinthead, dh_link); /* must save this pointer before calling FREE_ZONE on this node */
 		TAILQ_REMOVE(&dcp->c_hintlist, hint, dh_link);
@@ -1670,6 +1697,25 @@ hfs_reldirhints(struct cnode *dcp, int stale_hints_only)
 	}
 }
 
+/*
+ * Insert a detached directory hint back into the list of dirhints.
+ *
+ * Requires an exclusive lock on directory cnode.
+ */
+__private_extern__
+void
+hfs_insertdirhint(struct cnode *dcp, directoryhint_t * hint)
+{
+	directoryhint_t *test;
+
+	TAILQ_FOREACH(test, &dcp->c_hintlist, dh_link) {
+		if (test == hint)
+			panic("hfs_insertdirhint: hint %p already on list!", hint);
+	}
+
+	TAILQ_INSERT_HEAD(&dcp->c_hintlist, hint, dh_link);
+	++dcp->c_dirhintcnt;
+}
 
 /*
  * Perform a case-insensitive compare of two UTF-8 filenames.
@@ -1678,7 +1724,7 @@ hfs_reldirhints(struct cnode *dcp, int stale_hints_only)
  */
 __private_extern__
 int
-hfs_namecmp(const char *str1, size_t len1, const char *str2, size_t len2)
+hfs_namecmp(const u_int8_t *str1, size_t len1, const u_int8_t *str2, size_t len2)
 {
 	u_int16_t *ustr1, *ustr2;
 	size_t ulen1, ulen2;
@@ -1716,7 +1762,10 @@ hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	int               retval, blksize = hfsmp->hfs_phys_block_size;
 	struct vnode     *devvp;
 	struct hfs_mount_args *args = _args;
-
+	u_int32_t	  jib_flags;
+	u_int64_t	  jib_offset;
+	u_int64_t	  jib_size;
+	
 	devvp = hfsmp->hfs_devvp;
 
 	if (args != NULL && (args->flags & HFSFSMNT_EXTENDED_ARGS)) {
@@ -1734,11 +1783,11 @@ hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 		return retval;
 
 	jibp = (JournalInfoBlock *)buf_dataptr(jinfo_bp);
-	jibp->flags  = SWAP_BE32(jibp->flags);
-	jibp->offset = SWAP_BE64(jibp->offset);
-	jibp->size   = SWAP_BE64(jibp->size);
+	jib_flags  = SWAP_BE32(jibp->flags);
+	jib_offset = SWAP_BE64(jibp->offset);
+	jib_size   = SWAP_BE64(jibp->size);
 
-	if (jibp->flags & kJIJournalInFSMask) {
+	if (jib_flags & kJIJournalInFSMask) {
 		hfsmp->jvp = hfsmp->hfs_devvp;
 	} else {
 		printf("hfs: journal not stored in fs! don't know what to do.\n");
@@ -1747,16 +1796,16 @@ hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	}
 
 	// save this off for the hack-y check in hfs_remove()
-	hfsmp->jnl_start = jibp->offset / SWAP_BE32(vhp->blockSize);
-	hfsmp->jnl_size  = jibp->size;
+	hfsmp->jnl_start = jib_offset / SWAP_BE32(vhp->blockSize);
+	hfsmp->jnl_size  = jib_size;
 
 	if ((hfsmp->hfs_flags & HFS_READ_ONLY) && (vfs_flags(hfsmp->hfs_mp) & MNT_ROOTFS) == 0) {
 	    // if the file system is read-only, check if the journal is empty.
 	    // if it is, then we can allow the mount.  otherwise we have to
 	    // return failure.
 	    retval = journal_is_clean(hfsmp->jvp,
-		                      jibp->offset + embeddedOffset,
-				      jibp->size,
+		                      jib_offset + embeddedOffset,
+				      jib_size,
 				      devvp,
 		                      hfsmp->hfs_phys_block_size);
 
@@ -1772,12 +1821,12 @@ hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 	    return retval;
 	}
 
-	if (jibp->flags & kJIJournalNeedInitMask) {
+	if (jib_flags & kJIJournalNeedInitMask) {
 		printf("hfs: Initializing the journal (joffset 0x%llx sz 0x%llx)...\n",
-			   jibp->offset + embeddedOffset, jibp->size);
+			   jib_offset + embeddedOffset, jib_size);
 		hfsmp->jnl = journal_create(hfsmp->jvp,
-									jibp->offset + embeddedOffset,
-									jibp->size,
+									jib_offset + embeddedOffset,
+									jib_size,
 									devvp,
 									blksize,
 									arg_flags,
@@ -1786,21 +1835,19 @@ hfs_early_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp,
 
 		// no need to start a transaction here... if this were to fail
 		// we'd just re-init it on the next mount.
-		jibp->flags &= ~kJIJournalNeedInitMask;
-		jibp->flags  = SWAP_BE32(jibp->flags);
-		jibp->offset = SWAP_BE64(jibp->offset);
-		jibp->size   = SWAP_BE64(jibp->size);
+		jib_flags &= ~kJIJournalNeedInitMask;
+		jibp->flags  = SWAP_BE32(jib_flags);
 		buf_bwrite(jinfo_bp);
 		jinfo_bp = NULL;
 		jibp     = NULL;
 	} else { 
 		//printf("hfs: Opening the journal (joffset 0x%llx sz 0x%llx vhp_blksize %d)...\n",
-		//	   jibp->offset + embeddedOffset,
-		//	   jibp->size, SWAP_BE32(vhp->blockSize));
+		//	   jib_offset + embeddedOffset,
+		//	   jib_size, SWAP_BE32(vhp->blockSize));
 				
 		hfsmp->jnl = journal_open(hfsmp->jvp,
-								  jibp->offset + embeddedOffset,
-								  jibp->size,
+								  jib_offset + embeddedOffset,
+								  jib_size,
 								  devvp,
 								  blksize,
 								  arg_flags,
@@ -1862,15 +1909,18 @@ static int
 hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_args)
 {
 	JournalInfoBlock *jibp;
-	struct buf       *jinfo_bp, *bp;
+	struct buf       *jinfo_bp;
 	int               sectors_per_fsblock, arg_flags=0, arg_tbufsz=0;
-	int               retval, need_flush = 0, write_jibp = 0;
+	int               retval, write_jibp = 0, recreate_journal = 0;
 	struct vnode     *devvp;
 	struct cat_attr   jib_attr, jattr;
 	struct cat_fork   jib_fork, jfork;
 	ExtendedVCB      *vcb;
 	u_long            fid;
 	struct hfs_mount_args *args = _args;
+	u_int32_t	  jib_flags;
+	u_int64_t	  jib_offset;
+	u_int64_t	  jib_size;
 	
 	devvp = hfsmp->hfs_devvp;
 	vcb = HFSTOVCB(hfsmp);
@@ -1900,6 +1950,7 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 
 		vcb->vcbJinfoBlock    = jib_fork.cf_extents[0].startBlock;
 		vhp->journalInfoBlock = SWAP_BE32(jib_fork.cf_extents[0].startBlock);
+		recreate_journal = 1;
 	}
 
 
@@ -1915,9 +1966,9 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 	}
 
 	jibp = (JournalInfoBlock *)buf_dataptr(jinfo_bp);
-	jibp->flags  = SWAP_BE32(jibp->flags);
-	jibp->offset = SWAP_BE64(jibp->offset);
-	jibp->size   = SWAP_BE64(jibp->size);
+	jib_flags  = SWAP_BE32(jibp->flags);
+	jib_offset = SWAP_BE64(jibp->offset);
+	jib_size   = SWAP_BE64(jibp->size);
 
 	fid = GetFileInfo(vcb, kRootDirID, ".journal", &jattr, &jfork);
 	if (fid == 0 || jfork.cf_extents[0].startBlock == 0 || jfork.cf_size == 0) {
@@ -1930,24 +1981,26 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 	hfsmp->hfs_jnlfileid = fid;
 
 	// make sure the journal file begins where we think it should.
-	if ((jibp->offset / (u_int64_t)vcb->blockSize) != jfork.cf_extents[0].startBlock) {
+	if ((jib_offset / (u_int64_t)vcb->blockSize) != jfork.cf_extents[0].startBlock) {
 		printf("hfs: The journal file moved (was: %lld; is: %d).  Fixing up\n",
-			   (jibp->offset / (u_int64_t)vcb->blockSize), jfork.cf_extents[0].startBlock);
+			   (jib_offset / (u_int64_t)vcb->blockSize), jfork.cf_extents[0].startBlock);
 
-		jibp->offset = (u_int64_t)jfork.cf_extents[0].startBlock * (u_int64_t)vcb->blockSize;
+		jib_offset = (u_int64_t)jfork.cf_extents[0].startBlock * (u_int64_t)vcb->blockSize;
 		write_jibp   = 1;
+		recreate_journal = 1;
 	}
 
 	// check the size of the journal file.
-	if (jibp->size != (u_int64_t)jfork.cf_extents[0].blockCount*vcb->blockSize) {
+	if (jib_size != (u_int64_t)jfork.cf_extents[0].blockCount*vcb->blockSize) {
 		printf("hfs: The journal file changed size! (was %lld; is %lld).  Fixing up.\n",
-			   jibp->size, (u_int64_t)jfork.cf_extents[0].blockCount*vcb->blockSize);
+			   jib_size, (u_int64_t)jfork.cf_extents[0].blockCount*vcb->blockSize);
 		
-		jibp->size = (u_int64_t)jfork.cf_extents[0].blockCount * vcb->blockSize;
+		jib_size = (u_int64_t)jfork.cf_extents[0].blockCount * vcb->blockSize;
 		write_jibp = 1;
+		recreate_journal = 1;
 	}
 	
-	if (jibp->flags & kJIJournalInFSMask) {
+	if (jib_flags & kJIJournalInFSMask) {
 		hfsmp->jvp = hfsmp->hfs_devvp;
 	} else {
 		printf("hfs: journal not stored in fs! don't know what to do.\n");
@@ -1956,16 +2009,16 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 	}
 
 	// save this off for the hack-y check in hfs_remove()
-	hfsmp->jnl_start = jibp->offset / SWAP_BE32(vhp->blockSize);
-	hfsmp->jnl_size  = jibp->size;
+	hfsmp->jnl_start = jib_offset / SWAP_BE32(vhp->blockSize);
+	hfsmp->jnl_size  = jib_size;
 
 	if ((hfsmp->hfs_flags & HFS_READ_ONLY) && (vfs_flags(hfsmp->hfs_mp) & MNT_ROOTFS) == 0) {
 	    // if the file system is read-only, check if the journal is empty.
 	    // if it is, then we can allow the mount.  otherwise we have to
 	    // return failure.
 	    retval = journal_is_clean(hfsmp->jvp,
-		                      jibp->offset + (off_t)vcb->hfsPlusIOPosOffset,
-				      jibp->size,
+		                      jib_offset + (off_t)vcb->hfsPlusIOPosOffset,
+				      jib_size,
 				      devvp,
 		                      hfsmp->hfs_phys_block_size);
 
@@ -1981,12 +2034,12 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 	    return retval;
 	}
 
-	if (jibp->flags & kJIJournalNeedInitMask) {
+	if ((jib_flags & kJIJournalNeedInitMask) || recreate_journal) {
 		printf("hfs: Initializing the journal (joffset 0x%llx sz 0x%llx)...\n",
-			   jibp->offset + (off_t)vcb->hfsPlusIOPosOffset, jibp->size);
+			   jib_offset + (off_t)vcb->hfsPlusIOPosOffset, jib_size);
 		hfsmp->jnl = journal_create(hfsmp->jvp,
-									jibp->offset + (off_t)vcb->hfsPlusIOPosOffset,
-									jibp->size,
+									jib_offset + (off_t)vcb->hfsPlusIOPosOffset,
+									jib_size,
 									devvp,
 									hfsmp->hfs_phys_block_size,
 									arg_flags,
@@ -1995,7 +2048,7 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 
 		// no need to start a transaction here... if this were to fail
 		// we'd just re-init it on the next mount.
-		jibp->flags &= ~kJIJournalNeedInitMask;
+		jib_flags &= ~kJIJournalNeedInitMask;
 		write_jibp   = 1;
 
 	} else { 
@@ -2010,12 +2063,12 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 		arg_flags |= JOURNAL_RESET;
 		
 		//printf("hfs: Opening the journal (joffset 0x%llx sz 0x%llx vhp_blksize %d)...\n",
-		//	   jibp->offset + (off_t)vcb->hfsPlusIOPosOffset,
-		//	   jibp->size, SWAP_BE32(vhp->blockSize));
+		//	   jib_offset + (off_t)vcb->hfsPlusIOPosOffset,
+		//	   jib_size, SWAP_BE32(vhp->blockSize));
 				
 		hfsmp->jnl = journal_open(hfsmp->jvp,
-								  jibp->offset + (off_t)vcb->hfsPlusIOPosOffset,
-								  jibp->size,
+								  jib_offset + (off_t)vcb->hfsPlusIOPosOffset,
+								  jib_size,
 								  devvp,
 								  hfsmp->hfs_phys_block_size,
 								  arg_flags,
@@ -2025,9 +2078,9 @@ hfs_late_journal_init(struct hfsmount *hfsmp, HFSPlusVolumeHeader *vhp, void *_a
 			
 
 	if (write_jibp) {
-		jibp->flags  = SWAP_BE32(jibp->flags);
-		jibp->offset = SWAP_BE64(jibp->offset);
-		jibp->size   = SWAP_BE64(jibp->size);
+		jibp->flags  = SWAP_BE32(jib_flags);
+		jibp->offset = SWAP_BE64(jib_offset);
+		jibp->size   = SWAP_BE64(jib_size);
 
 		buf_bwrite(jinfo_bp);
 	} else {
@@ -2248,21 +2301,21 @@ __private_extern__
 int
 hfs_virtualmetafile(struct cnode *cp)
 {
-	char * filename;
+	const char * filename;
 
 
 	if (cp->c_parentcnid != kHFSRootFolderID)
 		return (0);
 
-	filename = cp->c_desc.cd_nameptr;
+	filename = (const char *)cp->c_desc.cd_nameptr;
 	if (filename == NULL)
 		return (0);
 
-	if ((strcmp(filename, ".journal") == 0) ||
-	    (strcmp(filename, ".journal_info_block") == 0) ||
-	    (strcmp(filename, ".quota.user") == 0) ||
-	    (strcmp(filename, ".quota.group") == 0) ||
-	    (strcmp(filename, ".hotfiles.btree") == 0))
+	if ((strncmp(filename, ".journal", sizeof(".journal")) == 0) ||
+	    (strncmp(filename, ".journal_info_block", sizeof(".journal_info_block")) == 0) ||
+	    (strncmp(filename, ".quota.user", sizeof(".quota.user")) == 0) ||
+	    (strncmp(filename, ".quota.group", sizeof(".quota.group")) == 0) ||
+	    (strncmp(filename, ".hotfiles.btree", sizeof(".hotfiles.btree")) == 0))
 		return (1);
 
 	return (0);
@@ -2273,23 +2326,43 @@ __private_extern__
 int
 hfs_start_transaction(struct hfsmount *hfsmp)
 {
-    int ret;
+	int ret, unlock_on_err=0;
+	void * thread = current_thread();
+
+#ifdef HFS_CHECK_LOCK_ORDER
+	/*
+	 * You cannot start a transaction while holding a system
+	 * file lock. (unless the transaction is nested.)
+	 */
+	if (hfsmp->jnl && journal_owner(hfsmp->jnl) != thread) {
+		if (hfsmp->hfs_catalog_cp && hfsmp->hfs_catalog_cp->c_lockowner == thread) {
+			panic("hfs_start_transaction: bad lock order (cat before jnl)\n");
+		}
+		if (hfsmp->hfs_attribute_cp && hfsmp->hfs_attribute_cp->c_lockowner == thread) {
+			panic("hfs_start_transaction: bad lock order (attr before jnl)\n");
+		}
+		if (hfsmp->hfs_extents_cp && hfsmp->hfs_extents_cp->c_lockowner == thread) {
+			panic("hfs_start_transaction: bad lock order (ext before jnl)\n");
+		}
+	}
+#endif /* HFS_CHECK_LOCK_ORDER */
 
-    if (hfsmp->jnl == NULL || journal_owner(hfsmp->jnl) != current_thread()) {
+    if (hfsmp->jnl == NULL || journal_owner(hfsmp->jnl) != thread) {
 	lck_rw_lock_shared(&hfsmp->hfs_global_lock);
+	unlock_on_err = 1;
     }
 
     if (hfsmp->jnl) {
 	ret = journal_start_transaction(hfsmp->jnl);
 	if (ret == 0) {
-	    OSAddAtomic(1, &hfsmp->hfs_global_lock_nesting);
+	    OSAddAtomic(1, (SInt32 *)&hfsmp->hfs_global_lock_nesting);
 	}
     } else {
 	ret = 0;
     }
 
-    if (ret != 0) {
-	lck_rw_done(&hfsmp->hfs_global_lock);
+    if (ret != 0 && unlock_on_err) {
+	lck_rw_unlock_shared(&hfsmp->hfs_global_lock);
     }
 
     return ret;
@@ -2303,7 +2376,7 @@ hfs_end_transaction(struct hfsmount *hfsmp)
 
     if (    hfsmp->jnl == NULL
 	|| (   journal_owner(hfsmp->jnl) == current_thread()
-	    && (OSAddAtomic(-1, &hfsmp->hfs_global_lock_nesting) == 1)) ) {
+	    && (OSAddAtomic(-1, (SInt32 *)&hfsmp->hfs_global_lock_nesting) == 1)) ) {
 
 	    need_unlock = 1;
     } 
@@ -2315,7 +2388,7 @@ hfs_end_transaction(struct hfsmount *hfsmp)
     }
 
     if (need_unlock) {
-	lck_rw_done(&hfsmp->hfs_global_lock);
+	lck_rw_unlock_shared(&hfsmp->hfs_global_lock);
     }
 
     return ret;
diff --git a/bsd/hfs/hfs_vnops.c b/bsd/hfs/hfs_vnops.c
index 726dee5e0..cac1f5b75 100644
--- a/bsd/hfs/hfs_vnops.c
+++ b/bsd/hfs/hfs_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/systm.h>
@@ -27,9 +33,11 @@
 #include <sys/stat.h>
 #include <sys/buf.h>
 #include <sys/mount.h>
+#include <sys/vnode_if.h>
 #include <sys/vnode_internal.h>
 #include <sys/malloc.h>
 #include <sys/ubc.h>
+#include <sys/ubc_internal.h>
 #include <sys/paths.h>
 #include <sys/quota.h>
 #include <sys/time.h>
@@ -56,8 +64,6 @@
 #include "hfscommon/headers/BTreesInternal.h"
 #include "hfscommon/headers/FileMgrInternal.h"
 
-#define MAKE_DELETED_NAME(NAME,FID) \
-	    (void) sprintf((NAME), "%s%d", HFS_DELETE_PREFIX, (FID))
 
 #define KNDETACH_VNLOCKED 0x00000001
 
@@ -70,19 +76,28 @@
 int always_do_fullfsync = 0;
 SYSCTL_INT (_kern, OID_AUTO, always_do_fullfsync, CTLFLAG_RW, &always_do_fullfsync, 0, "always F_FULLFSYNC when fsync is called");
 
-extern unsigned long strtoul(const char *, char **, int);
-
 static int hfs_makenode(struct vnode *dvp, struct vnode **vpp,
                         struct componentname *cnp, struct vnode_attr *vap,
                         vfs_context_t ctx);
 
-static int hfs_metasync(struct hfsmount *hfsmp, daddr64_t node, struct proc *p);
+static int hfs_metasync(struct hfsmount *hfsmp, daddr64_t node, __unused struct proc *p);
+static int hfs_metasync_all(struct hfsmount *hfsmp);
 
 static int hfs_removedir(struct vnode *, struct vnode *, struct componentname *,
                          int);
 
 static int hfs_removefile(struct vnode *, struct vnode *, struct componentname *,
-                          int, int);
+                          int, int, int);
+
+#if FIFO
+static int hfsfifo_read(struct vnop_read_args *);
+static int hfsfifo_write(struct vnop_write_args *);
+static int hfsfifo_close(struct vnop_close_args *);
+static int hfsfifo_kqfilt_add(struct vnop_kqfilt_add_args *);
+static int hfsfifo_kqfilt_remove(struct vnop_kqfilt_remove_args *);
+
+extern int (**fifo_vnodeop_p)(void *);
+#endif /* FIFO */
 
 static int hfs_vnop_close(struct vnop_close_args*);
 static int hfs_vnop_create(struct vnop_create_args*);
@@ -98,17 +113,19 @@ static int hfs_vnop_rename(struct vnop_rename_args*);
 static int hfs_vnop_rmdir(struct vnop_rmdir_args*);
 static int hfs_vnop_symlink(struct vnop_symlink_args*);
 static int hfs_vnop_setattr(struct vnop_setattr_args*);
+static int hfs_vnop_readlink(struct vnop_readlink_args *);
+static int hfs_vnop_pathconf(struct vnop_pathconf_args *);
+static int hfs_vnop_kqfiltremove(struct vnop_kqfilt_remove_args *);
+static int hfs_vnop_whiteout(struct vnop_whiteout_args *);
+static int hfsspec_read(struct vnop_read_args *);
+static int hfsspec_write(struct vnop_write_args *);
+static int hfsspec_close(struct vnop_close_args *);
 
 /* Options for hfs_removedir and hfs_removefile */
 #define HFSRM_SKIP_RESERVE  0x01
 
 
-int hfs_write_access(struct vnode *vp, kauth_cred_t cred, struct proc *p, Boolean considerFlags);
 
-int hfs_chmod(struct vnode *vp, int mode, kauth_cred_t cred,
-			struct proc *p);
-int hfs_chown(struct vnode *vp, uid_t uid, gid_t gid,
-			kauth_cred_t cred, struct proc *p);
 
 /*****************************************************************************
 *
@@ -122,7 +139,41 @@ int hfs_chown(struct vnode *vp, uid_t uid, gid_t gid,
 static int
 hfs_vnop_create(struct vnop_create_args *ap)
 {
-	return hfs_makenode(ap->a_dvp, ap->a_vpp, ap->a_cnp, ap->a_vap, ap->a_context);
+	int error;
+
+again:
+	error = hfs_makenode(ap->a_dvp, ap->a_vpp, ap->a_cnp, ap->a_vap, ap->a_context);
+
+	/*
+	 * We speculatively skipped the original lookup of the leaf
+	 * for CREATE.  Since it exists, go get it as long as they
+	 * didn't want an exclusive create.
+	 */
+	if ((error == EEXIST) && !(ap->a_vap->va_vaflags & VA_EXCLUSIVE)) {
+		struct vnop_lookup_args args;
+
+		args.a_desc = &vnop_lookup_desc;
+		args.a_dvp = ap->a_dvp;
+		args.a_vpp = ap->a_vpp;
+		args.a_cnp = ap->a_cnp;
+		args.a_context = ap->a_context;
+		args.a_cnp->cn_nameiop = LOOKUP;
+		error = hfs_vnop_lookup(&args);
+		/*
+		 * We can also race with remove for this file.
+		 */
+		if (error == ENOENT) {
+			goto again;
+		}
+
+		/* Make sure it was file. */
+		if ((error == 0) && !vnode_isreg(*args.a_vpp)) {
+			vnode_put(*args.a_vpp);
+			error = EEXIST;
+		}
+		args.a_cnp->cn_nameiop = CREATE;
+	}
+	return (error);
 }
 
 /*
@@ -188,7 +239,11 @@ hfs_vnop_open(struct vnop_open_args *ap)
 	 */
 	if ((VTOHFS(vp)->hfs_flags & HFS_READ_ONLY) ||
 	    (VTOHFS(vp)->jnl == NULL) ||
+#if NAMEDSTREAMS
+	    !vnode_isreg(vp) || vnode_isinuse(vp, 0) || vnode_isnamedstream(vp)) {
+#else
 	    !vnode_isreg(vp) || vnode_isinuse(vp, 0)) {
+#endif
 		return (0);
 	}
 
@@ -198,11 +253,18 @@ hfs_vnop_open(struct vnop_open_args *ap)
 	if (fp->ff_blocks &&
 	    fp->ff_extents[7].blockCount != 0 &&
 	    fp->ff_size <= (20 * 1024 * 1024)) {
+		struct timeval now;
+		struct cnode *cp = VTOC(vp);
 		/* 
 		 * Wait until system bootup is done (3 min).
+		 * And don't relocate a file that's been modified
+		 * within the past minute -- this can lead to
+		 * system thrashing.
 		 */
 		microuptime(&tv);
-		if (tv.tv_sec > (60 * 3)) {
+		microtime(&now);
+		if (tv.tv_sec > (60 * 3) &&
+		   ((now.tv_sec - cp->c_mtime) > 60)) {
 			(void) hfs_relocate(vp, VTOVCB(vp)->nextAllocation + 4096,
 			                    vfs_context_ucred(ap->a_context),
 			                    vfs_context_proc(ap->a_context));
@@ -264,59 +326,139 @@ hfs_vnop_close(ap)
 static int
 hfs_vnop_getattr(struct vnop_getattr_args *ap)
 {
+#define VNODE_ATTR_TIMES  \
+	(VNODE_ATTR_va_access_time|VNODE_ATTR_va_change_time|VNODE_ATTR_va_modify_time)
+#define VNODE_ATTR_AUTH  \
+	(VNODE_ATTR_va_mode | VNODE_ATTR_va_uid | VNODE_ATTR_va_gid | \
+         VNODE_ATTR_va_flags | VNODE_ATTR_va_acl)
+
 	struct vnode *vp = ap->a_vp;
 	struct vnode_attr *vap = ap->a_vap;
 	struct vnode *rvp = NULL;
 	struct hfsmount *hfsmp;
 	struct cnode *cp;
+	uint64_t data_size;
 	enum vtype v_type;
 	int error = 0;
 
-	if ((error = hfs_lock(VTOC(vp), HFS_SHARED_LOCK))) {
-		return (error);
-	}
 	cp = VTOC(vp);
+
+	/*
+	 * Shortcut for vnode_authorize path.  Each of the attributes
+	 * in this set is updated atomically so we don't need to take
+	 * the cnode lock to access them.
+	 */
+	if ((vap->va_active & ~VNODE_ATTR_AUTH) == 0) {
+		/* Make sure file still exists. */
+		if (cp->c_flag & C_NOEXISTS)
+			return (ENOENT);
+
+		vap->va_uid = cp->c_uid;
+		vap->va_gid = cp->c_gid;
+		vap->va_mode = cp->c_mode;
+		vap->va_flags = cp->c_flags;
+		vap->va_supported |= VNODE_ATTR_AUTH & ~VNODE_ATTR_va_acl;
+
+		if ((cp->c_attr.ca_recflags & kHFSHasSecurityMask) == 0) {
+			vap->va_acl = (kauth_acl_t) KAUTH_FILESEC_NONE;
+			VATTR_SET_SUPPORTED(vap, va_acl);
+		}
+		return (0);
+	}
 	hfsmp = VTOHFS(vp);
-	hfs_touchtimes(hfsmp, cp);
 	v_type = vnode_vtype(vp);
 
-	VATTR_RETURN(vap, va_rdev, (v_type == VBLK || v_type == VCHR) ? cp->c_rdev : 0);
+	/*
+	 * If time attributes are requested and we have cnode times
+	 * that require updating, then acquire an exclusive lock on
+	 * the cnode before updating the times.  Otherwise we can
+	 * just acquire a shared lock.
+	 */
+	if ((vap->va_active & VNODE_ATTR_TIMES) &&
+	    (cp->c_touch_acctime || cp->c_touch_chgtime || cp->c_touch_modtime)) {
+		if ((error = hfs_lock(cp, HFS_EXCLUSIVE_LOCK)))
+			return (error);
+		hfs_touchtimes(hfsmp, cp);
+	} else {
+		if ((error = hfs_lock(cp, HFS_SHARED_LOCK)))
+			return (error);
+	}
+
 	if (v_type == VDIR) {
+		data_size = (cp->c_entries + 2) * AVERAGE_HFSDIRENTRY_SIZE;
+
 		if (VATTR_IS_ACTIVE(vap, va_nlink)) {
-			int entries;
+			int nlink;
 	
-			entries = cp->c_nlink;
-			if (vnode_isvroot(vp)) {
-				if (hfsmp->hfs_privdir_desc.cd_cnid != 0)
-					--entries;     /* hide private dir */
-				if (hfsmp->jnl || ((HFSTOVCB(hfsmp)->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))
-					entries -= 2;  /* hide the journal files */
+			/*
+			 * For directories, the va_nlink is esentially a count
+			 * of the ".." references to a directory plus the "."
+			 * reference and the directory itself. So for HFS+ this
+			 * becomes the sub-directory count plus two.
+			 *
+			 * In the absence of a sub-directory count we use the
+			 * directory's item count.  This will be too high in
+			 * most cases since it also includes files.
+			 */
+			if ((hfsmp->hfs_flags & HFS_FOLDERCOUNT) && 
+			    (cp->c_attr.ca_recflags & kHFSHasFolderCountMask))
+				nlink = cp->c_attr.ca_dircount;  /* implied ".." entries */
+			else
+				nlink = cp->c_entries;
+
+			/* Account for ourself and our "." entry */
+			nlink += 2;  
+			 /* Hide our private directories. */
+			if (cp->c_cnid == kHFSRootFolderID) {
+				if (hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid != 0) {
+					--nlink;    
+				}
+				if (hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid != 0) {
+					--nlink;
+				}
 			}
-			VATTR_RETURN(vap, va_nlink, (uint64_t)entries);
-		}
-		
+			VATTR_RETURN(vap, va_nlink, (u_int64_t)nlink);
+		}		
 		if (VATTR_IS_ACTIVE(vap, va_nchildren)) {
 			int entries;
 	
 			entries = cp->c_entries;
-			if (vnode_isvroot(vp)) {
-				if (hfsmp->hfs_privdir_desc.cd_cnid != 0)
-					--entries;     /* hide private dir */
-				if (hfsmp->jnl)
-					entries -= 2;  /* hide the journal files */
+			/* Hide our private files and directories. */
+			if (cp->c_cnid == kHFSRootFolderID) {
+				if (hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid != 0)
+					--entries;
+				if (hfsmp->hfs_private_desc[DIR_HARDLINKS].cd_cnid != 0)
+					--entries;
+				if (hfsmp->jnl || ((hfsmp->vcbAtrb & kHFSVolumeJournaledMask) && (hfsmp->hfs_flags & HFS_READ_ONLY)))
+					entries -= 2;   /* hide the journal files */
 			}
 			VATTR_RETURN(vap, va_nchildren, entries);
 		}
-	} else {
-		VATTR_RETURN(vap, va_nlink, (uint64_t)cp->c_nlink);
+		/*
+		 * The va_dirlinkcount is the count of real directory hard links.
+		 * (i.e. its not the sum of the implied "." and ".." references)
+		 */
+		if (VATTR_IS_ACTIVE(vap, va_dirlinkcount)) {
+			VATTR_RETURN(vap, va_dirlinkcount, (uint32_t)cp->c_linkcount);
+		}
+	} else /* !VDIR */ {
+		data_size = VCTOF(vp, cp)->ff_size;
+
+		VATTR_RETURN(vap, va_nlink, (u_int64_t)cp->c_linkcount);
+		if (VATTR_IS_ACTIVE(vap, va_data_alloc)) {
+			u_int64_t blocks;
+	
+			blocks = VCTOF(vp, cp)->ff_blocks;
+			VATTR_RETURN(vap, va_data_alloc, blocks * (u_int64_t)hfsmp->blockSize);
+		}
 	}
 
 	/* conditional because 64-bit arithmetic can be expensive */
 	if (VATTR_IS_ACTIVE(vap, va_total_size)) {
 		if (v_type == VDIR) {
-			VATTR_RETURN(vap, va_total_size, cp->c_nlink * AVERAGE_HFSDIRENTRY_SIZE);
+			VATTR_RETURN(vap, va_total_size, (cp->c_entries + 2) * AVERAGE_HFSDIRENTRY_SIZE);
 		} else {
-			uint64_t total_size = 0;
+			u_int64_t total_size = 0;
 			struct cnode *rcp;
 			
 			if (cp->c_datafork) {
@@ -324,8 +466,7 @@ hfs_vnop_getattr(struct vnop_getattr_args *ap)
 			}
 
 			if (cp->c_blocks - VTOF(vp)->ff_blocks) {
-				/* hfs_vgetrsrc does not use struct proc - therefore passing NULL */ 
-				error = hfs_vgetrsrc(hfsmp, vp, &rvp, NULL);
+				error = hfs_vgetrsrc(hfsmp, vp, &rvp, TRUE);
 				if (error) {
 					goto out;
 				}
@@ -337,44 +478,15 @@ hfs_vnop_getattr(struct vnop_getattr_args *ap)
 			}
 
 			VATTR_RETURN(vap, va_total_size, total_size);
-			/* Include size of attibute data (extents), if any */
-			if (cp->c_attrblks) {
-				vap->va_total_size += (uint64_t)cp->c_attrblks * (uint64_t)hfsmp->blockSize;
-			}
 		}
 	}
 	if (VATTR_IS_ACTIVE(vap, va_total_alloc)) {
 		if (v_type == VDIR) {
 			VATTR_RETURN(vap, va_total_alloc, 0);
 		} else {
-			VATTR_RETURN(vap, va_total_alloc, (uint64_t)cp->c_blocks * (uint64_t)hfsmp->blockSize);
-			/* Include size of attibute data (extents), if any */
-			if (cp->c_attrblks) {
-				vap->va_total_alloc += (uint64_t)cp->c_attrblks * (uint64_t)hfsmp->blockSize;
-			}
+			VATTR_RETURN(vap, va_total_alloc, (u_int64_t)cp->c_blocks * (u_int64_t)hfsmp->blockSize);
 		}
 	}
-	/* XXX broken... if ask for "data size" of rsrc fork vp you get rsrc fork size! */
-	if (v_type == VDIR) {
-		VATTR_RETURN(vap, va_data_size, cp->c_nlink * AVERAGE_HFSDIRENTRY_SIZE);
-	} else {
-		VATTR_RETURN(vap, va_data_size, VTOF(vp)->ff_size);
-	}	
-	if (VATTR_IS_ACTIVE(vap, va_data_alloc) && (v_type != VDIR)) {
-			/* XXX do we need to account for ff_unallocblocks ? */
-		VATTR_RETURN(vap, va_data_alloc, (uint64_t)VTOF(vp)->ff_blocks * (uint64_t)hfsmp->blockSize);
-	}
-	/* XXX is this really a good 'optimal I/O size'? */
-	VATTR_RETURN(vap, va_iosize, hfsmp->hfs_logBlockSize);
-	VATTR_RETURN(vap, va_uid, cp->c_uid);
-	VATTR_RETURN(vap, va_gid, cp->c_gid);
-	VATTR_RETURN(vap, va_mode, cp->c_mode);
-#if 0
-	/* XXX is S_IFXATTR still needed ??? */
-	if (VNODE_IS_RSRC(vp))
-		vap->va_mode |= S_IFXATTR;
-#endif
-	VATTR_RETURN(vap, va_flags, cp->c_flags);
 
 	/*
 	 * If the VFS wants extended security data, and we know that we
@@ -385,16 +497,12 @@ hfs_vnop_getattr(struct vnop_getattr_args *ap)
 	 */
 	if (VATTR_IS_ACTIVE(vap, va_acl)) {
 		if ((cp->c_attr.ca_recflags & kHFSHasSecurityMask) == 0) {
-			vap->va_acl = KAUTH_FILESEC_NONE;
+			vap->va_acl = (kauth_acl_t) KAUTH_FILESEC_NONE;
 			VATTR_SET_SUPPORTED(vap, va_acl);
 		}
 	}
-	vap->va_create_time.tv_sec = cp->c_itime;
-	vap->va_create_time.tv_nsec = 0;
-	VATTR_SET_SUPPORTED(vap, va_create_time);
-
 	if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-		/* Access times are lazyily updated, get current time if needed */
+		/* Access times are lazily updated, get current time if needed */
 		if (cp->c_touch_acctime) {
 			struct timeval tv;
 	
@@ -406,15 +514,21 @@ hfs_vnop_getattr(struct vnop_getattr_args *ap)
 		vap->va_access_time.tv_nsec = 0;
 		VATTR_SET_SUPPORTED(vap, va_access_time);
 	}
+	vap->va_create_time.tv_sec = cp->c_itime;
+	vap->va_create_time.tv_nsec = 0;
 	vap->va_modify_time.tv_sec = cp->c_mtime;
 	vap->va_modify_time.tv_nsec = 0;
-	VATTR_SET_SUPPORTED(vap, va_modify_time);
 	vap->va_change_time.tv_sec = cp->c_ctime;
 	vap->va_change_time.tv_nsec = 0;
-	VATTR_SET_SUPPORTED(vap, va_change_time);
 	vap->va_backup_time.tv_sec = cp->c_btime;
-	vap->va_backup_time.tv_nsec = 0;
-	VATTR_SET_SUPPORTED(vap, va_backup_time);
+	vap->va_backup_time.tv_nsec = 0;	
+
+	/* XXX is this really a good 'optimal I/O size'? */
+	vap->va_iosize = hfsmp->hfs_logBlockSize;
+	vap->va_uid = cp->c_uid;
+	vap->va_gid = cp->c_gid;
+	vap->va_mode = cp->c_mode;
+	vap->va_flags = cp->c_flags;
 
 	/*
 	 * Exporting file IDs from HFS Plus:
@@ -427,29 +541,96 @@ hfs_vnop_getattr(struct vnop_getattr_args *ap)
 	 * The stat call (getattr) uses va_fileid and the Carbon APIs,
 	 * which are hardlink-ignorant, will ask for va_linkid.
 	 */
-	VATTR_RETURN(vap, va_fileid, (uint64_t)cp->c_fileid);
-	VATTR_RETURN(vap, va_linkid, (uint64_t)cp->c_cnid);
-	VATTR_RETURN(vap, va_parentid, (uint64_t)cp->c_parentcnid);
-	VATTR_RETURN(vap, va_fsid, cp->c_dev);
-	VATTR_RETURN(vap, va_filerev, 0);
-
-	VATTR_RETURN(vap, va_encoding, cp->c_encoding);
+	vap->va_fileid = (u_int64_t)cp->c_fileid;
+	/* Hardlinked directories have multiple cnids and parents (one per link). */
+	if ((v_type == VDIR) && (cp->c_flag & C_HARDLINK)) {
+		vap->va_linkid = (u_int64_t)hfs_currentcnid(cp);
+		vap->va_parentid = (u_int64_t)hfs_currentparent(cp);
+	} else {
+		vap->va_linkid = (u_int64_t)cp->c_cnid;
+		vap->va_parentid = (u_int64_t)cp->c_parentcnid;
+	}
+	vap->va_fsid = cp->c_dev;
+	vap->va_filerev = 0;
+	vap->va_encoding = cp->c_encoding;
+	vap->va_rdev = (v_type == VBLK || v_type == VCHR) ? cp->c_rdev : 0;
+	vap->va_data_size = data_size;
+
+	/* Mark them all at once instead of individual VATTR_SET_SUPPORTED calls. */
+	vap->va_supported |= VNODE_ATTR_va_create_time | VNODE_ATTR_va_modify_time |
+	                     VNODE_ATTR_va_change_time| VNODE_ATTR_va_backup_time |
+	                     VNODE_ATTR_va_iosize | VNODE_ATTR_va_uid |
+	                     VNODE_ATTR_va_gid | VNODE_ATTR_va_mode |
+	                     VNODE_ATTR_va_flags |VNODE_ATTR_va_fileid |
+	                     VNODE_ATTR_va_linkid | VNODE_ATTR_va_parentid |
+	                     VNODE_ATTR_va_fsid | VNODE_ATTR_va_filerev |
+	                     VNODE_ATTR_va_encoding | VNODE_ATTR_va_rdev |
+	                     VNODE_ATTR_va_data_size;
 
 	/* if this is the root, let VFS to find out the mount name, which may be different from the real name */
-	if (VATTR_IS_ACTIVE(vap, va_name) && !vnode_isvroot(vp)) {
+	if (VATTR_IS_ACTIVE(vap, va_name) && (cp->c_cnid != kHFSRootFolderID)) {
 		/* Return the name for ATTR_CMN_NAME */
 		if (cp->c_desc.cd_namelen == 0) {
+		    if ((cp->c_flag & C_HARDLINK) && ((cp->c_flag & C_DELETED) == 0 || (cp->c_linkcount > 1))) {
+			cnid_t nextlinkid;
+			cnid_t prevlinkid;
+			struct vnode *file_vp;
+			
+			if ((error = hfs_lookuplink(hfsmp, cp->c_fileid, &prevlinkid, &nextlinkid))) {
+			    goto out;
+			}
+
+			//
+			// don't bother trying to get a linkid that's the same
+			// as the current cnid
+			//
+			if (nextlinkid == VTOC(vp)->c_cnid) {
+			    if (prevlinkid == VTOC(vp)->c_cnid) {
+				hfs_unlock(cp);
+				goto out2;
+			    } else {
+				nextlinkid = prevlinkid;
+			    }
+			}
+			    
+			hfs_unlock(cp);
+
+			if (nextlinkid == 0 || (error = hfs_vget(hfsmp, nextlinkid, &file_vp, 1))) {
+			    if (prevlinkid == 0 || (error = hfs_vget(hfsmp, prevlinkid, &file_vp, 1))) {
+				goto out2;
+			    }
+			}
+				
+			cp = VTOC(file_vp);
+			if (hfs_lock(cp, HFS_SHARED_LOCK) == 0) {
+			    if (cp->c_desc.cd_namelen) {
+				strlcpy(vap->va_name, (const char *)cp->c_desc.cd_nameptr, MAXPATHLEN);
+			    }
+			    hfs_unlock(cp);
+			    vnode_put(file_vp);
+			    goto out2;
+			}
+			
+			if (vnode_name(file_vp)) {
+			    strlcpy(vap->va_name, vnode_name(file_vp), MAXPATHLEN);
+			} else {
+			    error = ENOENT;
+			}
+			vnode_put(file_vp);
+			goto out2;
+		    } else {
 			error = ENOENT;
 			goto out;
+		    }
+		} else {
+		    strlcpy(vap->va_name, (const char *)cp->c_desc.cd_nameptr, MAXPATHLEN);
+		    VATTR_SET_SUPPORTED(vap, va_name);
 		}
-		
-		strncpy(vap->va_name, cp->c_desc.cd_nameptr, MAXPATHLEN);
-		vap->va_name[MAXPATHLEN-1] = '\0';
-		VATTR_SET_SUPPORTED(vap, va_name);
 	}
 
 out:
 	hfs_unlock(cp);
+out2:
 	if (rvp) {
 		vnode_put(rvp);
 	}
@@ -491,15 +672,19 @@ hfs_vnop_setattr(ap)
 
 		/* Take truncate lock before taking cnode lock. */
 		hfs_lock_truncate(VTOC(vp), TRUE);
+		
+		/* Perform the ubc_setsize before taking the cnode lock. */
+		ubc_setsize(vp, vap->va_data_size);
+
 		if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
-			hfs_unlock_truncate(VTOC(vp));
+			hfs_unlock_truncate(VTOC(vp), TRUE);
 			return (error);
 		}
 		cp = VTOC(vp);
 
-		error = hfs_truncate(vp, vap->va_data_size, vap->va_vaflags & 0xffff, 0, ap->a_context);
+		error = hfs_truncate(vp, vap->va_data_size, vap->va_vaflags & 0xffff, 1, ap->a_context);
 
-		hfs_unlock_truncate(cp);
+		hfs_unlock_truncate(cp, TRUE);
 		if (error)
 			goto out;
 	}
@@ -509,6 +694,21 @@ hfs_vnop_setattr(ap)
 		cp = VTOC(vp);
 	}
 
+	/*
+	 * If it is just an access time update request by itself
+	 * we know the request is from kernel level code, and we
+	 * can delay it without being as worried about consistency.
+	 * This change speeds up mmaps, in the rare case that they
+	 * get caught behind a sync.
+	 */
+
+	if (vap->va_active == VNODE_ATTR_va_access_time) {
+		cp->c_touch_acctime=TRUE;
+		goto out;
+	}
+
+
+
 	/*
 	 * Owner/group change request.
 	 * We are guaranteed that the new owner/group is valid and legal.
@@ -538,31 +738,23 @@ hfs_vnop_setattr(ap)
 	 */
 	VATTR_SET_SUPPORTED(vap, va_flags);
 	if (VATTR_IS_ACTIVE(vap, va_flags)) {
+		u_int16_t *fdFlags;
+
 		cp->c_flags = vap->va_flags;
 		cp->c_touch_chgtime = TRUE;
-	}
-
-	/*
-	 * If the file's extended security data is being changed, we
-	 * need to note the change.  Note that because we don't store
-	 * the data, we do not set the SUPPORTED bit; this will cause
-	 * the VFS to use a fallback strategy.
-	 */
-	if (VATTR_IS_ACTIVE(vap, va_acl)) {
-		/* Remember if any ACL data was set or cleared. */
-		if (vap->va_acl == NULL) {
-			/* being cleared */
-			if (cp->c_attr.ca_recflags & kHFSHasSecurityMask) {
-				cp->c_attr.ca_recflags &= ~kHFSHasSecurityMask;
-				cp->c_touch_chgtime = TRUE;
-			}
-		} else {
-			/* being set */
-			if ((cp->c_attr.ca_recflags & kHFSHasSecurityMask) == 0) {
-				cp->c_attr.ca_recflags |= kHFSHasSecurityMask;
-				cp->c_touch_chgtime = TRUE;
-			}
-		}
+		
+		/*
+		 * Mirror the UF_HIDDEN flag to the invisible bit of the Finder Info.
+		 *
+		 * The fdFlags for files and frFlags for folders are both 8 bytes
+		 * into the userInfo (the first 16 bytes of the Finder Info).  They
+		 * are both 16-bit fields.
+		 */
+		fdFlags = (u_int16_t *) &cp->c_finderinfo[8];
+		if (vap->va_flags & UF_HIDDEN)
+			*fdFlags |= OSSwapHostToBigConstInt16(kFinderInvisibleMask);
+		else
+			*fdFlags &= ~OSSwapHostToBigConstInt16(kFinderInvisibleMask);
 	}
 
 	/*
@@ -577,8 +769,6 @@ hfs_vnop_setattr(ap)
 	    VATTR_IS_ACTIVE(vap, va_access_time) ||
 	    VATTR_IS_ACTIVE(vap, va_modify_time) ||
 	    VATTR_IS_ACTIVE(vap, va_backup_time)) {
-		if (vnode_islnk(vp))
-			goto done;
 		if (VATTR_IS_ACTIVE(vap, va_create_time))
 			cp->c_itime = vap->va_create_time.tv_sec;
 		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
@@ -616,9 +806,8 @@ hfs_vnop_setattr(ap)
 		hfs_setencodingbits(hfsmp, cp->c_encoding);
 	}
 
-done:
 	if ((error = hfs_update(vp, TRUE)) != 0)
-	    goto out;
+		goto out;
 	HFS_KNOTE(vp, NOTE_ATTRIB);
 out:
 	if (cp)
@@ -633,10 +822,9 @@ out:
  */
 __private_extern__
 int
-hfs_chmod(struct vnode *vp, int mode, kauth_cred_t cred, struct proc *p)
+hfs_chmod(struct vnode *vp, int mode, __unused kauth_cred_t cred, __unused struct proc *p)
 {
 	register struct cnode *cp = VTOC(vp);
-	int error;
 
 	if (VTOVCB(vp)->vcbSigWord != kHFSPlusSigWord)
 		return (0);
@@ -715,15 +903,19 @@ hfs_write_access(struct vnode *vp, kauth_cred_t cred, struct proc *p, Boolean co
  */
 __private_extern__
 int
+#if !QUOTA
+hfs_chown(struct vnode *vp, uid_t uid, gid_t gid, __unused kauth_cred_t cred,
+	__unused struct proc *p)
+#else 
 hfs_chown(struct vnode *vp, uid_t uid, gid_t gid, kauth_cred_t cred,
-	struct proc *p)
+	__unused struct proc *p)
+#endif
 {
 	register struct cnode *cp = VTOC(vp);
 	uid_t ouid;
 	gid_t ogid;
-	int error = 0;
-	int is_member;
 #if QUOTA
+	int error = 0;
 	register int i;
 	int64_t change;
 #endif /* QUOTA */
@@ -825,7 +1017,12 @@ good:
 		panic("hfs_chown: lost quota");
 #endif /* QUOTA */
 
-	if (ouid != uid || ogid != gid)
+
+	/*
+	  According to the SUSv3 Standard, chown() shall mark
+	  for update the st_ctime field of the file.
+	  (No exceptions mentioned)
+	*/
 		cp->c_touch_chgtime = TRUE;
 	return (0);
 }
@@ -854,6 +1051,13 @@ hfs_vnop_exchange(ap)
 	struct hfsmount *hfsmp;
 	struct cat_desc tempdesc;
 	struct cat_attr tempattr;
+	const unsigned char *from_nameptr;
+	const unsigned char *to_nameptr;
+	char from_iname[32];
+	char to_iname[32];
+	u_int32_t tempflag;
+	cnid_t  from_parid;
+	cnid_t  to_parid;
 	int lockflags;
 	int error = 0, started_tr = 0, got_cookie = 0;
 	cat_cookie_t cookie;
@@ -874,7 +1078,6 @@ hfs_vnop_exchange(ap)
 
 	/* Only normal files can be exchanged. */
 	if (!vnode_isreg(from_vp) || !vnode_isreg(to_vp) ||
-	    (from_cp->c_flag & C_HARDLINK) || (to_cp->c_flag & C_HARDLINK) ||
 	    VNODE_IS_RSRC(from_vp) || VNODE_IS_RSRC(to_vp)) {
 		error = EINVAL;
 		goto exit;
@@ -909,26 +1112,42 @@ hfs_vnop_exchange(ap)
 	/*
 	 * Reserve some space in the Catalog file.
 	 */
-	bzero(&cookie, sizeof(cookie));
 	if ((error = cat_preflight(hfsmp, CAT_EXCHANGE, &cookie, vfs_context_proc(ap->a_context)))) {
 		goto exit;
 	}
 	got_cookie = 1;
 
 	/* The backend code always tries to delete the virtual
-	 * extent id for exchanging files so we neeed to lock
+	 * extent id for exchanging files so we need to lock
 	 * the extents b-tree.
 	 */
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_EXTENTS | SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
 
+	/* Account for the location of the catalog objects. */
+	if (from_cp->c_flag & C_HARDLINK) {
+		MAKE_INODE_NAME(from_iname, sizeof(from_iname),
+				from_cp->c_attr.ca_linkref);
+		from_nameptr = (unsigned char *)from_iname;
+		from_parid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+		from_cp->c_hint = 0;
+	} else {
+		from_nameptr = from_cp->c_desc.cd_nameptr;
+		from_parid = from_cp->c_parentcnid;
+	}
+	if (to_cp->c_flag & C_HARDLINK) {
+		MAKE_INODE_NAME(to_iname, sizeof(to_iname),
+				to_cp->c_attr.ca_linkref);
+		to_nameptr = (unsigned char *)to_iname;
+		to_parid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
+		to_cp->c_hint = 0;
+	} else {
+		to_nameptr = to_cp->c_desc.cd_nameptr;
+		to_parid = to_cp->c_parentcnid;
+	}
+
 	/* Do the exchange */
-	error = ExchangeFileIDs(hfsmp,
-	                        from_cp->c_desc.cd_nameptr,
-	                        to_cp->c_desc.cd_nameptr,
-	                        from_cp->c_parentcnid,
-	                        to_cp->c_parentcnid,
-	                        from_cp->c_hint,
-	                        to_cp->c_hint);
+	error = ExchangeFileIDs(hfsmp, from_nameptr, to_nameptr, from_parid,
+	                        to_parid, from_cp->c_hint, to_cp->c_hint);
 	hfs_systemfile_unlock(hfsmp, lockflags);
 
 	/*
@@ -950,6 +1169,7 @@ hfs_vnop_exchange(ap)
 	/* Save a copy of from attributes before swapping. */
 	bcopy(&from_cp->c_desc, &tempdesc, sizeof(struct cat_desc));
 	bcopy(&from_cp->c_attr, &tempattr, sizeof(struct cat_attr));
+	tempflag = from_cp->c_flag & (C_HARDLINK | C_HASXATTRS);
 
 	/*
 	 * Swap the descriptors and all non-fork related attributes.
@@ -967,6 +1187,8 @@ hfs_vnop_exchange(ap)
 	from_cp->c_uid = to_cp->c_uid;
 	from_cp->c_flags = to_cp->c_flags;
 	from_cp->c_mode = to_cp->c_mode;
+	from_cp->c_linkcount = to_cp->c_linkcount;
+	from_cp->c_flag = to_cp->c_flag & (C_HARDLINK | C_HASXATTRS);
 	from_cp->c_attr.ca_recflags = to_cp->c_attr.ca_recflags;
 	bcopy(to_cp->c_finderinfo, from_cp->c_finderinfo, 32);
 
@@ -981,6 +1203,8 @@ hfs_vnop_exchange(ap)
 	to_cp->c_uid = tempattr.ca_uid;
 	to_cp->c_flags = tempattr.ca_flags;
 	to_cp->c_mode = tempattr.ca_mode;
+	to_cp->c_linkcount = tempattr.ca_linkcount;
+	to_cp->c_flag = tempflag;
 	to_cp->c_attr.ca_recflags = tempattr.ca_recflags;
 	bcopy(tempattr.ca_finderinfo, to_cp->c_finderinfo, 32);
 
@@ -1035,7 +1259,9 @@ hfs_fsync(struct vnode *vp, int waitfor, int fullsync, struct proc *p)
 	int took_trunc_lock = 0;
 
 	wait = (waitfor == MNT_WAIT);
-
+	if (always_do_fullfsync)
+		fullsync = 1;
+	
 	/* HFS directories don't have any data blocks. */
 	if (vnode_isdir(vp))
 		goto metasync;
@@ -1057,7 +1283,7 @@ hfs_fsync(struct vnode *vp, int waitfor, int fullsync, struct proc *p)
 		took_trunc_lock = 1;
 
 		/* Don't hold cnode lock when calling into cluster layer. */
-		(void) cluster_push(vp, 0);
+		(void) cluster_push(vp, wait ? IO_SYNC : 0);
 
 		hfs_lock(cp, HFS_FORCE_LOCK);
 	}
@@ -1070,10 +1296,10 @@ hfs_fsync(struct vnode *vp, int waitfor, int fullsync, struct proc *p)
 	 */
 	if ((wait || (cp->c_flag & C_ZFWANTSYNC)) &&
 	    ((cp->c_flags & UF_NODUMP) == 0) &&
-	    UBCINFOEXISTS(vp) && (fp = VTOF(vp)) &&
+	    UBCINFOEXISTS(vp) && (vnode_issystem(vp) ==0) && (fp = VTOF(vp)) &&
 	    cp->c_zftimeout != 0) {
 		microuptime(&tv);
-		if (tv.tv_sec < cp->c_zftimeout) {
+		if (!fullsync && tv.tv_sec < (long)cp->c_zftimeout) {
 			/* Remember that a force sync was requested. */
 			cp->c_flag |= C_ZFWANTSYNC;
 			goto datasync;
@@ -1105,7 +1331,7 @@ hfs_fsync(struct vnode *vp, int waitfor, int fullsync, struct proc *p)
 			cp->c_flag |= C_MODIFIED;
 		}
 		hfs_unlock(cp);
-		(void) cluster_push(vp, 0);
+		(void) cluster_push(vp, wait ? IO_SYNC : 0);
 		hfs_lock(cp, HFS_FORCE_LOCK);
 
 		cp->c_flag &= ~C_ZFWANTSYNC;
@@ -1113,7 +1339,7 @@ hfs_fsync(struct vnode *vp, int waitfor, int fullsync, struct proc *p)
 	}
 datasync:
 	if (took_trunc_lock)
-		hfs_unlock_truncate(cp);
+		hfs_unlock_truncate(cp, TRUE);
 	
 	/*
 	 * if we have a journal and if journal_active() returns != 0 then the
@@ -1147,20 +1373,26 @@ metasync:
 	} else if ( !(vp->v_flag & VSWAP) ) /* User file */ {
 		retval = hfs_update(vp, wait);
 
-		/* When MNT_WAIT is requested push out any delayed meta data */
-   		if ((retval == 0) && wait && cp->c_hint &&
+		/*
+		 * When MNT_WAIT is requested push out the catalog record for
+		 * this file.  If they asked for a full fsync, we can skip this
+		 * because the journal_flush or hfs_metasync_all will push out
+		 * all of the metadata changes.
+		 */
+   		if ((retval == 0) && wait && !fullsync && cp->c_hint &&
    		    !ISSET(cp->c_flag, C_DELETED | C_NOEXISTS)) {
    			hfs_metasync(VTOHFS(vp), (daddr64_t)cp->c_hint, p);
-   		}
+		}
 
-		// make sure that we've really been called from the user
-		// fsync() and if so push out any pending transactions 
-		// that this file might is a part of (and get them on
-		// stable storage).
-		if (fullsync || always_do_fullfsync) {
+		/*
+		 * If this was a full fsync, make sure all metadata
+		 * changes get to stable storage.
+		 */
+		if (fullsync) {
 		    if (hfsmp->jnl) {
 			journal_flush(hfsmp->jnl);
 		    } else {
+			retval = hfs_metasync_all(hfsmp);
 		    	/* XXX need to pass context! */
 			VNOP_IOCTL(hfsmp->hfs_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, NULL);
 		    }
@@ -1173,7 +1405,7 @@ metasync:
 
 /* Sync an hfs catalog b-tree node */
 static int
-hfs_metasync(struct hfsmount *hfsmp, daddr64_t node, struct proc *p)
+hfs_metasync(struct hfsmount *hfsmp, daddr64_t node, __unused struct proc *p)
 {
 	vnode_t	vp;
 	buf_t	bp;
@@ -1212,9 +1444,46 @@ hfs_metasync(struct hfsmount *hfsmp, daddr64_t node, struct proc *p)
 }
 
 
+/*
+ * Sync all hfs B-trees.  Use this instead of journal_flush for a volume
+ * without a journal.  Note that the volume bitmap does not get written;
+ * we rely on fsck_hfs to fix that up (which it can do without any loss
+ * of data).
+ */
+static int
+hfs_metasync_all(struct hfsmount *hfsmp)
+{
+	int lockflags;
+
+	/* Lock all of the B-trees so we get a mutually consistent state */
+	lockflags = hfs_systemfile_lock(hfsmp,
+		SFL_CATALOG|SFL_EXTENTS|SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
+
+	/* Sync each of the B-trees */
+	if (hfsmp->hfs_catalog_vp)
+		hfs_btsync(hfsmp->hfs_catalog_vp, 0);
+	if (hfsmp->hfs_extents_vp)
+		hfs_btsync(hfsmp->hfs_extents_vp, 0);
+	if (hfsmp->hfs_attribute_vp)
+		hfs_btsync(hfsmp->hfs_attribute_vp, 0);
+	
+	/* Wait for all of the writes to complete */
+	if (hfsmp->hfs_catalog_vp)
+		vnode_waitforwrites(hfsmp->hfs_catalog_vp, 0, 0, 0, "hfs_metasync_all");
+	if (hfsmp->hfs_extents_vp)
+		vnode_waitforwrites(hfsmp->hfs_extents_vp, 0, 0, 0, "hfs_metasync_all");
+	if (hfsmp->hfs_attribute_vp)
+		vnode_waitforwrites(hfsmp->hfs_attribute_vp, 0, 0, 0, "hfs_metasync_all");
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	
+	return 0;
+}
+
+
 /*ARGSUSED 1*/
 static int
-hfs_btsync_callback(struct buf *bp, void *dummy)
+hfs_btsync_callback(struct buf *bp, __unused void *dummy)
 {
 	buf_clearflags(bp, B_LOCKED);
 	(void) buf_bawrite(bp);
@@ -1262,20 +1531,21 @@ hfs_vnop_rmdir(ap)
 {
 	struct vnode *dvp = ap->a_dvp;
 	struct vnode *vp = ap->a_vp;
+	struct cnode *dcp = VTOC(dvp);
+	struct cnode *cp = VTOC(vp);
 	int error;
 
-	if (!vnode_isdir(vp)) {
+	if (!S_ISDIR(cp->c_mode)) {
 		return (ENOTDIR);
 	}
 	if (dvp == vp) {
 		return (EINVAL);
 	}
-	if ((error = hfs_lockpair(VTOC(dvp), VTOC(vp), HFS_EXCLUSIVE_LOCK)))
+	if ((error = hfs_lockpair(dcp, cp, HFS_EXCLUSIVE_LOCK))) {
 		return (error);
-
+	}
 	error = hfs_removedir(dvp, vp, ap->a_cnp, 0);
-
-	hfs_unlockpair(VTOC(dvp), VTOC(vp));
+	hfs_unlockpair(dcp, cp);
 
 	return (error);
 }
@@ -1289,42 +1559,50 @@ static int
 hfs_removedir(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
               int skip_reserve)
 {
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 	struct cnode *cp;
 	struct cnode *dcp;
 	struct hfsmount * hfsmp;
 	struct cat_desc desc;
-	cat_cookie_t cookie;
 	int lockflags;
-	int error = 0, started_tr = 0, got_cookie = 0;
+	int error = 0, started_tr = 0;
 
 	cp = VTOC(vp);
 	dcp = VTOC(dvp);
 	hfsmp = VTOHFS(vp);
 
-	if (dcp == cp)
+	if (dcp == cp) {
 		return (EINVAL);	/* cannot remove "." */
+	}
+	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
+		return (0);
+	}
+	if (cp->c_entries != 0) {
+		return (ENOTEMPTY);
+	}
+
+	/* Check if we're removing the last link to an empty directory. */
+	if (cp->c_flag & C_HARDLINK) {
+		/* We could also return EBUSY here */
+		return hfs_unlink(hfsmp, dvp, vp, cnp, skip_reserve);
+	}
+
+	if ((hfsmp->hfs_attribute_vp != NULL) &&
+	    (cp->c_attr.ca_recflags & kHFSHasAttributesMask) != 0) {
+
+	    return hfs_removefile(dvp, vp, cnp, 0, 0, 1);
+	}
+
+	dcp->c_flag |= C_DIR_MODIFICATION;
 
 #if QUOTA
-	(void)hfs_getinoquota(cp);
+	if (hfsmp->hfs_flags & HFS_QUOTAS)
+		(void)hfs_getinoquota(cp);
 #endif
 	if ((error = hfs_start_transaction(hfsmp)) != 0) {
 	    goto out;
 	}
 	started_tr = 1;
 
-	if (!skip_reserve) {
-		/*
-		 * Reserve some space in the Catalog file.
-		 */
-		bzero(&cookie, sizeof(cookie));
-		if ((error = cat_preflight(hfsmp, CAT_DELETE, &cookie, p))) {
-			goto out;
-		}
-		got_cookie = 1;
-	}
-
 	/*
 	 * Verify the directory is empty (and valid).
 	 * (Rmdir ".." won't be valid since
@@ -1332,18 +1610,11 @@ hfs_removedir(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	 *  the current directory and thus be
 	 *  non-empty.)
 	 */
-	if (cp->c_entries != 0) {
-		error = ENOTEMPTY;
-		goto out;
-	}
 	if ((dcp->c_flags & APPEND) || (cp->c_flags & (IMMUTABLE | APPEND))) {
 		error = EPERM;
 		goto out;
 	}
 
-	if (cp->c_entries > 0)
-		panic("hfs_rmdir: attempting to delete a non-empty directory!");
-
 	/* Remove the entry from the namei cache: */
 	cache_purge(vp);
 
@@ -1352,51 +1623,77 @@ hfs_removedir(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	 * name passed in and parent id from dvp (instead of using 
 	 * the cp->c_desc which may have changed).
 	 */
-	bzero(&desc, sizeof(desc));
-	desc.cd_nameptr = cnp->cn_nameptr;
+	desc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 	desc.cd_namelen = cnp->cn_namelen;
-	desc.cd_parentcnid = dcp->c_cnid;
+	desc.cd_parentcnid = dcp->c_fileid;
 	desc.cd_cnid = cp->c_cnid;
+	desc.cd_flags = CD_ISDIR;
+	desc.cd_encoding = cp->c_encoding;
+	desc.cd_hint = 0;
+
+	if (!hfs_valid_cnode(hfsmp, dvp, cnp, cp->c_fileid)) {
+	    error = 0;
+	    goto out;
+	}
 
 	/* Remove entry from catalog */
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+
+	if (!skip_reserve) {
+		/*
+		 * Reserve some space in the Catalog file.
+		 */
+		if ((error = cat_preflight(hfsmp, CAT_DELETE, NULL, 0))) {
+			hfs_systemfile_unlock(hfsmp, lockflags);
+			goto out;
+		}
+	}
+
 	error = cat_delete(hfsmp, &desc, &cp->c_attr);
 	if (error == 0) {
-		/* Delete any attributes, ignore errors */
-		(void) hfs_removeallattr(hfsmp, cp->c_fileid);
+		/* The parent lost a child */
+		if (dcp->c_entries > 0)
+			dcp->c_entries--;
+		DEC_FOLDERCOUNT(hfsmp, dcp->c_attr);
+		dcp->c_dirchangecnt++;
+		dcp->c_touch_chgtime = TRUE;
+		dcp->c_touch_modtime = TRUE;
+		hfs_touchtimes(hfsmp, cp);
+		(void) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
+		cp->c_flag &= ~(C_MODIFIED | C_FORCEUPDATE);
 	}
+
 	hfs_systemfile_unlock(hfsmp, lockflags);
 
 	if (error)
 		goto out;
 
 #if QUOTA
-	(void)hfs_chkiq(cp, -1, NOCRED, 0);
+	if (hfsmp->hfs_flags & HFS_QUOTAS)
+		(void)hfs_chkiq(cp, -1, NOCRED, 0);
 #endif /* QUOTA */
 
-	/* The parent lost a child */
-	if (dcp->c_entries > 0)
-		dcp->c_entries--;
-	if (dcp->c_nlink > 0)
-		dcp->c_nlink--;
-	dcp->c_touch_chgtime = TRUE;
-	dcp->c_touch_modtime = TRUE;
-
-	dcp->c_flag |= C_FORCEUPDATE;  // XXXdbg - don't screw around, force this guy out
-	
-	(void) hfs_update(dvp, 0);
-	HFS_KNOTE(dvp, NOTE_WRITE | NOTE_LINK);
+	HFS_KNOTE(dvp, NOTE_WRITE | NOTE_LINK | NOTE_ATTRIB);
 
 	hfs_volupdate(hfsmp, VOL_RMDIR, (dcp->c_cnid == kHFSRootFolderID));
 
-	cp->c_mode = 0;  /* Makes the vnode go away...see inactive */
-	cp->c_flag |= C_NOEXISTS;
+	/*
+	 * directory open or in use (e.g. opendir() or current working
+	 * directory for some process); wait for inactive to actually
+	 * remove catalog entry
+	 */
+	if (vnode_isinuse(vp, 0)) {
+		cp->c_flag |= C_DELETED;
+	} else {
+		cp->c_mode = 0;  /* Makes the vnode go away...see inactive */
+		cp->c_flag |= C_NOEXISTS;
+	}
 out:
+	dcp->c_flag &= ~C_DIR_MODIFICATION;
+	wakeup((caddr_t)&dcp->c_flag);
+
 	HFS_KNOTE(vp, NOTE_DELETE);
 
-	if (got_cookie) {
-		cat_postflight(hfsmp, &cookie, p);
-	}
 	if (started_tr) { 
 	    hfs_end_transaction(hfsmp);
 	}
@@ -1420,22 +1717,46 @@ hfs_vnop_remove(ap)
 {
 	struct vnode *dvp = ap->a_dvp;
 	struct vnode *vp = ap->a_vp;
-	int error;
+	struct cnode *dcp = VTOC(dvp);
+	struct cnode *cp = VTOC(vp);
+	struct vnode *rvp = cp->c_rsrc_vp;
+	int error=0, recycle_rsrc=0, rvid=0;
 
 	if (dvp == vp) {
 		return (EINVAL);
 	}
 
-	hfs_lock_truncate(VTOC(vp), TRUE);
+	hfs_lock_truncate(cp, TRUE);
 
-	if ((error = hfs_lockpair(VTOC(dvp), VTOC(vp), HFS_EXCLUSIVE_LOCK)))
+	if ((error = hfs_lockpair(dcp, cp, HFS_EXCLUSIVE_LOCK)))
 		goto out;
 
-	error = hfs_removefile(dvp, vp, ap->a_cnp, ap->a_flags, 0);
+	error = hfs_removefile(dvp, vp, ap->a_cnp, ap->a_flags, 0, 0);
+
+	//
+	// If the remove succeeded and it's an open-unlinked file that has
+	// a resource fork vnode that's not in use, we will want to recycle
+	// the rvp *after* we're done unlocking everything.  Otherwise the
+	// resource vnode will keep a v_parent reference on this vnode which
+	// prevents it from going through inactive/reclaim which means that
+	// the disk space associated with this file won't get free'd until
+	// something forces the resource vnode to get recycled (and that can
+	// take a very long time).
+	//
+	if (error == 0 && (cp->c_flag & C_DELETED) && rvp && !vnode_isinuse(rvp, 0)) {
+	    rvid = vnode_vid(rvp);
+	    recycle_rsrc = 1;
+	}
 
-	hfs_unlockpair(VTOC(dvp), VTOC(vp));
+	hfs_unlockpair(dcp, cp);
 out:
-	hfs_unlock_truncate(VTOC(vp));
+	hfs_unlock_truncate(cp, TRUE);
+
+	if (recycle_rsrc && vnode_getwithvid(rvp, rvid) == 0) {
+		vnode_recycle(rvp);
+		vnode_put(rvp);
+	} 
+	
 	return (error);
 }
 
@@ -1444,7 +1765,7 @@ static int
 hfs_removefile_callback(struct buf *bp, void *hfsmp) {
 
         if ( !(buf_flags(bp) & B_META))
-	        panic("hfs: symlink bp @ 0x%x is not marked meta-data!\n", bp);
+	        panic("hfs: symlink bp @ %p is not marked meta-data!\n", bp);
 	/*
 	 * it's part of the current transaction, kill it.
 	 */
@@ -1462,7 +1783,7 @@ hfs_removefile_callback(struct buf *bp, void *hfsmp) {
  */
 static int
 hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
-               int flags, int skip_reserve)
+               int flags, int skip_reserve, int allow_dirs)
 {
 	struct vnode *rvp = NULL;
 	struct cnode *cp;
@@ -1474,73 +1795,100 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	int dataforkbusy = 0;
 	int rsrcforkbusy = 0;
 	int truncated = 0;
-	cat_cookie_t cookie;
 	int lockflags;
 	int error = 0;
-	int started_tr = 0, got_cookie = 0;
-	int isbigfile = 0;
-	cnid_t real_cnid = 0;
-
-	/* Directories should call hfs_rmdir! */
-	if (vnode_isdir(vp)) {
-		return (EISDIR);
-	}
+	int started_tr = 0;
+	int isbigfile = 0, hasxattrs=0, isdir=0;
 
 	cp = VTOC(vp);
 	dcp = VTOC(dvp);
 	hfsmp = VTOHFS(vp);
 
+	/* Check if we lost a race post lookup. */
 	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
-	    return 0;
+		return (0);
 	}
-	
-	// if it's not a hardlink, check that the parent
-	// cnid is the same as the directory cnid
-	if (   (cp->c_flag & C_HARDLINK) == 0
-	    && (cp->c_parentcnid != hfsmp->hfs_privdir_desc.cd_cnid)
-	    && (cp->c_parentcnid != dcp->c_cnid)) {
-		error = EINVAL;
-		goto out;
+
+	if (!hfs_valid_cnode(hfsmp, dvp, cnp, cp->c_fileid)) {
+	    return 0;
 	}
 
 	/* Make sure a remove is permitted */
 	if (VNODE_IS_RSRC(vp)) {
-		error = EPERM;
-		goto out;
+		return (EPERM);
 	}
+	/* Don't allow deleting the journal or journal_info_block. */
+	if (hfsmp->jnl &&
+	    (cp->c_fileid == hfsmp->hfs_jnlfileid || cp->c_fileid == hfsmp->hfs_jnlinfoblkid)) {
+		return (EPERM);
+	}
+	/*
+	 * Hard links require special handling.
+	 */
+	if (cp->c_flag & C_HARDLINK) {
+		if ((flags & VNODE_REMOVE_NODELETEBUSY) && vnode_isinuse(vp, 0)) {
+			return (EBUSY);
+		} else {
+			/* A directory hard link with a link count of one is 
+			 * treated as a regular directory.  Therefore it should 
+			 * only be removed using rmdir().
+			 */
+			if ((vnode_isdir(vp) == 1) && (cp->c_linkcount == 1) && 
+			    (allow_dirs == 0)) {
+			    	return (EPERM);
+			}
+			return hfs_unlink(hfsmp, dvp, vp, cnp, skip_reserve);
+		}
+	}
+	/* Directories should call hfs_rmdir! (unless they have a lot of attributes) */
+	if (vnode_isdir(vp)) {
+		if (allow_dirs == 0)
+			return (EPERM);  /* POSIX */
+		isdir = 1;
+	}
+	/* Sanity check the parent ids. */
+	if ((cp->c_parentcnid != hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid) &&
+	    (cp->c_parentcnid != dcp->c_fileid)) {
+		return (EINVAL);
+	}
+
+	dcp->c_flag |= C_DIR_MODIFICATION;
+
+	// this guy is going away so mark him as such
+	cp->c_flag |= C_DELETED;
+
+
+	/* Remove our entry from the namei cache. */
+	cache_purge(vp);
 
 	/*
-	 * Aquire a vnode for a non-empty resource fork.
+	 * Acquire a vnode for a non-empty resource fork.
 	 * (needed for hfs_truncate)
 	 */
-	if (cp->c_blocks - VTOF(vp)->ff_blocks) {
-		error = hfs_vgetrsrc(hfsmp, vp, &rvp, 0);
+	if (isdir == 0 && (cp->c_blocks - VTOF(vp)->ff_blocks)) {
+		error = hfs_vgetrsrc(hfsmp, vp, &rvp, FALSE);
 		if (error)
 			goto out;
+		/* Defer the vnode_put on rvp until the hfs_unlock(). */
+		cp->c_flag |= C_NEED_RVNODE_PUT;
 	}
-
-	// XXXdbg - don't allow deleting the journal or journal_info_block
-	if (hfsmp->jnl && cp->c_datafork) {
-		struct HFSPlusExtentDescriptor *extd;
-
-		extd = &cp->c_datafork->ff_extents[0];
-		if (extd->startBlock == HFSTOVCB(hfsmp)->vcbJinfoBlock || extd->startBlock == hfsmp->jnl_start) {
-			error = EPERM;
-			goto out;
-		}
+	/* Check if this file is being used. */
+	if (isdir == 0) {
+		dataforkbusy = vnode_isinuse(vp, 0);
+		rsrcforkbusy = rvp ? vnode_isinuse(rvp, 0) : 0;
+	}
+	
+	/* Check if we have to break the deletion into multiple pieces. */
+	if (isdir == 0) {
+		isbigfile = ((cp->c_datafork->ff_size >= HFS_BIGFILE_SIZE) && overflow_extents(VTOF(vp)));
 	}
 
-	/*
-	 * Check if this file is being used.
-	 */
-	if (vnode_isinuse(vp, 0))
-		dataforkbusy = 1;
-	if (rvp && vnode_isinuse(rvp, 0))
-		rsrcforkbusy = 1;
-
-	// need this to check if we have to break the deletion 
-	// into multiple pieces
-	isbigfile = (VTOC(vp)->c_datafork->ff_size >= HFS_BIGFILE_SIZE);
+	/* Check if the file has xattrs.  If it does we'll have to delete them in
+	   individual transactions in case there are too many */
+	if ((hfsmp->hfs_attribute_vp != NULL) &&
+	    (cp->c_attr.ca_recflags & kHFSHasAttributesMask) != 0) {
+	    hasxattrs = 1;
+	}
 
 	/*
 	 * Carbon semantics prohibit deleting busy files.
@@ -1548,22 +1896,19 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	 */
 	if (dataforkbusy || rsrcforkbusy) {
 		if ((flags & VNODE_REMOVE_NODELETEBUSY) ||
-		    (hfsmp->hfs_privdir_desc.cd_cnid == 0)) {
+		    (hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid == 0)) {
 			error = EBUSY;
 			goto out;
 		}
 	}
 
 #if QUOTA
-	(void)hfs_getinoquota(cp);
+	if (hfsmp->hfs_flags & HFS_QUOTAS)
+		(void)hfs_getinoquota(cp);
 #endif /* QUOTA */
 
-	/*
-	 * We do the ubc_setsize before the hfs_truncate
-	 * since we'll be inside a transaction.
-	 */
-	if ((cp->c_flag & C_HARDLINK) == 0 &&
-	    (!dataforkbusy || !rsrcforkbusy)) {
+	/* Check if we need a ubc_setsize. */
+	if (isdir == 0 && (!dataforkbusy || !rsrcforkbusy)) {
 		/*
 		 * A ubc_setsize can cause a pagein so defer it
 		 * until after the cnode lock is dropped.  The
@@ -1576,28 +1921,6 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 		if (!rsrcforkbusy && rvp) {
 			cp->c_flag |= C_NEED_RSRC_SETSIZE;
 		}
-	} else {
-	    struct cat_desc cndesc;
-
-	    // for hard links, re-lookup the name that was passed
-	    // in so we get the correct cnid for the name (as
-	    // opposed to the c_cnid in the cnode which could have
-	    // been changed before this node got locked).
-	    bzero(&cndesc, sizeof(cndesc));
-	    cndesc.cd_nameptr = cnp->cn_nameptr;
-	    cndesc.cd_namelen = cnp->cn_namelen;
-	    cndesc.cd_parentcnid = VTOC(dvp)->c_cnid;
-	    cndesc.cd_hint = VTOC(dvp)->c_childhint;
-	    
-	    lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
-
-	    if (cat_lookup(hfsmp, &cndesc, 0, NULL, NULL, NULL, &real_cnid) != 0) {
-		hfs_systemfile_unlock(hfsmp, lockflags);
-		error = ENOENT;
-		goto out;
-	    }
-	    
-	    hfs_systemfile_unlock(hfsmp, lockflags);
 	}
 
 	if ((error = hfs_start_transaction(hfsmp)) != 0) {
@@ -1605,33 +1928,18 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	}
 	started_tr = 1;
 
-	if (!skip_reserve) {
-		/*
-		 * Reserve some space in the Catalog file.
-		 */
-		if ((error = cat_preflight(hfsmp, CAT_DELETE, &cookie, 0))) {
-			goto out;
-		}
-		got_cookie = 1;
-	}
-
-	/* Remove our entry from the namei cache. */
-	cache_purge(vp);
-
 	// XXXdbg - if we're journaled, kill any dirty symlink buffers 
 	if (hfsmp->jnl && vnode_islnk(vp))
 	        buf_iterate(vp, hfs_removefile_callback, BUF_SKIP_NONLOCKED, (void *)hfsmp);
 
 	/*
 	 * Truncate any non-busy forks.  Busy forks will
-	 * get trucated when their vnode goes inactive.
+	 * get truncated when their vnode goes inactive.
 	 *
 	 * Since we're already inside a transaction,
 	 * tell hfs_truncate to skip the ubc_setsize.
-	 *
-	 * (Note: hard links are truncated in VOP_INACTIVE)
 	 */
-	if ((cp->c_flag & C_HARDLINK) == 0) {
+	if (isdir == 0) {
 		int mode = cp->c_mode;
 
 		if (!dataforkbusy && !isbigfile && cp->c_datafork->ff_blocks != 0) {
@@ -1643,7 +1951,7 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 			truncated = 1;
 		}
 		if (!rsrcforkbusy && rvp) {
-			cp->c_mode = 0;            /* Suppress hfs_update */
+			cp->c_mode = 0;  /* Suppress hfs_update */
 			error = hfs_truncate(rvp, (off_t)0, IO_NDELAY, 1, ctx);
 			cp->c_mode = mode;
 			if (error)
@@ -1659,93 +1967,19 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	 */
 	desc.cd_flags = 0;
 	desc.cd_encoding = cp->c_desc.cd_encoding;
-	desc.cd_nameptr = cnp->cn_nameptr;
+	desc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 	desc.cd_namelen = cnp->cn_namelen;
-	desc.cd_parentcnid = dcp->c_cnid;
+	desc.cd_parentcnid = dcp->c_fileid;
 	desc.cd_hint = cp->c_desc.cd_hint;
-	if (real_cnid) {
-	    // if it was a hardlink we had to re-lookup the cnid
-	    desc.cd_cnid = real_cnid;
-	} else {
-	    desc.cd_cnid = cp->c_cnid;
-	}
+	desc.cd_cnid = cp->c_cnid;
 	microtime(&tv);
 
 	/*
-	 * There are 3 remove cases to consider:
-	 *   1. File is a hardlink    ==> remove the link
-	 *   2. File is busy (in use) ==> move/rename the file
-	 *   3. File is not in use    ==> remove the file
+	 * There are two cases to consider:
+	 *  1. File is busy/big   ==> move/rename the file
+	 *  2. File is not in use ==> remove the file
 	 */
-
-	if (cp->c_flag & C_HARDLINK) {
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
-
-		/* Delete the link record */
-		error = cat_delete(hfsmp, &desc, &cp->c_attr);
-		if (error == 0) {
-			/* Update the parent directory */
-			if (dcp->c_entries > 0)
-				dcp->c_entries--;
-			if (dcp->c_nlink > 0)
-				dcp->c_nlink--;
-			dcp->c_ctime = tv.tv_sec;
-			dcp->c_mtime = tv.tv_sec;
-			(void ) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
-
-			if (--cp->c_nlink < 1) {
-				char inodename[32];
-				char delname[32];
-				struct cat_desc to_desc;
-				struct cat_desc from_desc;
-
-				/*
-			 	 * This is now esentially an open deleted file.
-				 * Rename it to reflect this state which makes
-				 * orphan file cleanup easier (see hfs_remove_orphans).
-			 	 * Note: a rename failure here is not fatal.
-			 	 */	
-				MAKE_INODE_NAME(inodename, cp->c_rdev);
-				bzero(&from_desc, sizeof(from_desc));
-				from_desc.cd_nameptr = inodename;
-				from_desc.cd_namelen = strlen(inodename);
-				from_desc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
-				from_desc.cd_flags = 0;
-				from_desc.cd_cnid = cp->c_fileid;
-
-				MAKE_DELETED_NAME(delname, cp->c_fileid);		
-				bzero(&to_desc, sizeof(to_desc));
-				to_desc.cd_nameptr = delname;
-				to_desc.cd_namelen = strlen(delname);
-				to_desc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
-				to_desc.cd_flags = 0;
-				to_desc.cd_cnid = cp->c_fileid;
-	
-				error = cat_rename(hfsmp, &from_desc, &hfsmp->hfs_privdir_desc,
-						   &to_desc, (struct cat_desc *)NULL);
-				if (error != 0) {
-				    panic("hfs_removefile: error %d from cat_rename(%s %s) cp 0x%x\n",
-					  inodename, delname, cp);
-				}
-				if (error == 0) {
-					/* Update the file's state */
-					cp->c_flag |= C_DELETED;
-					cp->c_ctime = tv.tv_sec;	
-					(void) cat_update(hfsmp, &to_desc, &cp->c_attr, NULL, NULL);
-				}
-			} else {
-				/* Update the file's state */
-				cp->c_ctime = tv.tv_sec;	
-				(void) cat_update(hfsmp, &cp->c_desc, &cp->c_attr, NULL, NULL);
-			}
-		}
-		hfs_systemfile_unlock(hfsmp, lockflags);
-		if (error != 0)
-			goto out;
-
-		hfs_volupdate(hfsmp, VOL_RMFILE, (dcp->c_cnid == kHFSRootFolderID));
-
-	} else if (dataforkbusy || rsrcforkbusy || isbigfile) {
+	if (dataforkbusy || rsrcforkbusy || isbigfile || hasxattrs) {
 		char delname[32];
 		struct cat_desc to_desc;
 		struct cat_desc todir_desc;
@@ -1756,29 +1990,40 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 		bzero(&todir_desc, sizeof(todir_desc));
 		todir_desc.cd_parentcnid = 2;
 
-		MAKE_DELETED_NAME(delname, cp->c_fileid);		
+		MAKE_DELETED_NAME(delname, sizeof(delname), cp->c_fileid);
 		bzero(&to_desc, sizeof(to_desc));
-		to_desc.cd_nameptr = delname;
+		to_desc.cd_nameptr = (const u_int8_t *)delname;
 		to_desc.cd_namelen = strlen(delname);
-		to_desc.cd_parentcnid = hfsmp->hfs_privdir_desc.cd_cnid;
+		to_desc.cd_parentcnid = hfsmp->hfs_private_desc[FILE_HARDLINKS].cd_cnid;
 		to_desc.cd_flags = 0;
 		to_desc.cd_cnid = cp->c_cnid;
 
 		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
+		if (!skip_reserve) {
+			if ((error = cat_preflight(hfsmp, CAT_RENAME, NULL, 0))) {
+				hfs_systemfile_unlock(hfsmp, lockflags);
+				goto out;
+			}
+		}
 
 		error = cat_rename(hfsmp, &desc, &todir_desc,
 				&to_desc, (struct cat_desc *)NULL);
 
 		if (error == 0) {
-			hfsmp->hfs_privdir_attr.ca_entries++;
-			(void) cat_update(hfsmp, &hfsmp->hfs_privdir_desc,
-			                  &hfsmp->hfs_privdir_attr, NULL, NULL);
+			hfsmp->hfs_private_attr[FILE_HARDLINKS].ca_entries++;
+			if (isdir == 1) {
+				INC_FOLDERCOUNT(hfsmp, hfsmp->hfs_private_attr[FILE_HARDLINKS]);
+			}
+			(void) cat_update(hfsmp, &hfsmp->hfs_private_desc[FILE_HARDLINKS],
+			                  &hfsmp->hfs_private_attr[FILE_HARDLINKS], NULL, NULL);
 
 			/* Update the parent directory */
 			if (dcp->c_entries > 0)
 				dcp->c_entries--;
-			if (dcp->c_nlink > 0)
-				dcp->c_nlink--;
+			if (isdir == 1) {
+				DEC_FOLDERCOUNT(hfsmp, dcp->c_attr);
+			}
+			dcp->c_dirchangecnt++;
 			dcp->c_ctime = tv.tv_sec;
 			dcp->c_mtime = tv.tv_sec;
 			(void) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
@@ -1786,7 +2031,7 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 			/* Update the file's state */
 			cp->c_flag |= C_DELETED;
 			cp->c_ctime = tv.tv_sec;
-			--cp->c_nlink;
+			--cp->c_linkcount;
 			(void) cat_update(hfsmp, &to_desc, &cp->c_attr, NULL, NULL);
 		}
 		hfs_systemfile_unlock(hfsmp, lockflags);
@@ -1802,7 +2047,13 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 			goto out;
 		}
 
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+		if (!skip_reserve) {
+			if ((error = cat_preflight(hfsmp, CAT_DELETE, NULL, 0))) {
+				hfs_systemfile_unlock(hfsmp, lockflags);
+				goto out;
+			}
+		}
 
 		error = cat_delete(hfsmp, &desc, &cp->c_attr);
 
@@ -1817,14 +2068,10 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 			}
 		}
 		if (error == 0) {
-			/* Delete any attributes, ignore errors */
-			(void) hfs_removeallattr(hfsmp, cp->c_fileid);
-
 			/* Update the parent directory */
 			if (dcp->c_entries > 0)
 				dcp->c_entries--;
-			if (dcp->c_nlink > 0)
-				dcp->c_nlink--;
+			dcp->c_dirchangecnt++;
 			dcp->c_ctime = tv.tv_sec;
 			dcp->c_mtime = tv.tv_sec;
 			(void) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
@@ -1834,14 +2081,16 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 			goto out;
 
 #if QUOTA
-		(void)hfs_chkiq(cp, -1, NOCRED, 0);
+		if (hfsmp->hfs_flags & HFS_QUOTAS)
+			(void)hfs_chkiq(cp, -1, NOCRED, 0);
 #endif /* QUOTA */
 
 		cp->c_mode = 0;
 		truncated  = 0;    // because the catalog entry is gone
 		cp->c_flag |= C_NOEXISTS;
+		cp->c_flag &= ~C_DELETED;
 		cp->c_touch_chgtime = TRUE;   /* XXX needed ? */
-		--cp->c_nlink;
+		--cp->c_linkcount;
 
 		hfs_volupdate(hfsmp, VOL_RMFILE, (dcp->c_cnid == kHFSRootFolderID));
 	}
@@ -1849,18 +2098,18 @@ hfs_removefile(struct vnode *dvp, struct vnode *vp, struct componentname *cnp,
 	/*
 	 * All done with this cnode's descriptor...
 	 *
-	 * Note: all future catalog calls for this cnode must be
-	 * by fileid only.  This is OK for HFS (which doesn't have
-	 * file thread records) since HFS doesn't support hard
-	 * links or the removal of busy files.
+	 * Note: all future catalog calls for this cnode must be by
+	 * fileid only.  This is OK for HFS (which doesn't have file
+	 * thread records) since HFS doesn't support the removal of
+	 * busy files.
 	 */
 	cat_releasedesc(&cp->c_desc);
 
 	HFS_KNOTE(dvp, NOTE_WRITE);
 
 out:
-	if (got_cookie) {
-		cat_postflight(hfsmp, &cookie, 0);
+	if (error) {
+	    cp->c_flag &= ~C_DELETED;
 	}
 
 	/* Commit the truncation to the catalog record */
@@ -1875,12 +2124,13 @@ out:
 	    hfs_end_transaction(hfsmp);
 	}
 
+	dcp->c_flag &= ~C_DIR_MODIFICATION;
+	wakeup((caddr_t)&dcp->c_flag);
+
 	HFS_KNOTE(vp, NOTE_DELETE);
 	if (rvp) {
 		HFS_KNOTE(rvp, NOTE_DELETE);
-		/* Defer the vnode_put on rvp until the hfs_unlock(). */
-		cp->c_flag |= C_NEED_RVNODE_PUT;
-	};
+	}
 
 	return (error);
 }
@@ -1889,18 +2139,19 @@ out:
 __private_extern__ void
 replace_desc(struct cnode *cp, struct cat_desc *cdp)
 {
+	// fixes 4348457 and 4463138
 	if (&cp->c_desc == cdp) {
-		return;
+	    return;
 	}
 
 	/* First release allocated name buffer */
 	if (cp->c_desc.cd_flags & CD_HASBUF && cp->c_desc.cd_nameptr != 0) {
-		char *name = cp->c_desc.cd_nameptr;
+		const u_int8_t *name = cp->c_desc.cd_nameptr;
 
 		cp->c_desc.cd_nameptr = 0;
 		cp->c_desc.cd_namelen = 0;
 		cp->c_desc.cd_flags &= ~CD_HASBUF;
-		vfs_removename(name);
+		vfs_removename((const char *)name);
 	}
 	bcopy(cdp, &cp->c_desc, sizeof(cp->c_desc));
 
@@ -1956,16 +2207,17 @@ hfs_vnop_rename(ap)
 	int error;
 
 	/* When tvp exist, take the truncate lock for the hfs_removefile(). */
-	if (tvp && vnode_isreg(tvp)) {
+	if (tvp && (vnode_isreg(tvp) || vnode_islnk(tvp))) {
 		hfs_lock_truncate(VTOC(tvp), TRUE);
 		took_trunc_lock = 1;
 	}
 
+  retry:
 	error = hfs_lockfour(VTOC(fdvp), VTOC(fvp), VTOC(tdvp), tvp ? VTOC(tvp) : NULL,
 	                     HFS_EXCLUSIVE_LOCK);
 	if (error) {
 		if (took_trunc_lock)
-			hfs_unlock_truncate(VTOC(tvp));	
+			hfs_unlock_truncate(VTOC(tvp), TRUE);	
 		return (error);
 	}
 
@@ -1976,11 +2228,58 @@ hfs_vnop_rename(ap)
 	hfsmp = VTOHFS(tdvp);
 
 	/* Check for a race against unlink. */
-	if (fcp->c_flag & C_NOEXISTS) {
+	if ((fcp->c_flag & (C_NOEXISTS | C_DELETED)) || !hfs_valid_cnode(hfsmp, fdvp, fcnp, fcp->c_fileid)) {
 		error = ENOENT;
 		goto out;
 	}
 
+	if (tcp && ((tcp->c_flag & (C_NOEXISTS | C_DELETED)) || !hfs_valid_cnode(hfsmp, tdvp, tcnp, tcp->c_fileid))) {
+	    //
+	    // hmm, the destination vnode isn't valid any more.
+	    // in this case we can just drop him and pretend he
+	    // never existed in the first place.
+	    //
+	    if (took_trunc_lock) {
+		hfs_unlock_truncate(VTOC(tvp), TRUE);
+		took_trunc_lock = 0;
+	    }
+
+	    hfs_unlockfour(fdcp, fcp, tdcp, tcp);
+
+	    tcp = NULL;
+	    tvp = NULL;
+	    
+	    // retry the locking with tvp null'ed out
+	    goto retry;
+	}
+
+	fdcp->c_flag |= C_DIR_MODIFICATION;
+	if (fdvp != tdvp) {
+	    tdcp->c_flag |= C_DIR_MODIFICATION;
+	}
+
+	/*
+	 * Disallow renaming of a directory hard link if the source and 
+	 * destination parent directories are different, or a directory whose 
+	 * descendant is a directory hard link and the one of the ancestors
+	 * of the destination directory is a directory hard link.
+	 */
+	if (vnode_isdir(fvp) && (fdvp != tdvp)) {
+		if (fcp->c_flag & C_HARDLINK) {
+			error = EPERM;
+			goto out;
+		}
+		if (fcp->c_attr.ca_recflags & kHFSHasChildLinkMask) {
+		    lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+		    if (cat_check_link_ancestry(hfsmp, tdcp->c_fileid, 0)) {
+				error = EPERM;
+				hfs_systemfile_unlock(hfsmp, lockflags);
+				goto out;
+			}
+			hfs_systemfile_unlock(hfsmp, lockflags);
+		}
+	}
+
 	/*
 	 * The following edge case is caught here:
 	 * (to cannot be a descendent of from)
@@ -1996,7 +2295,7 @@ hfs_vnop_rename(ap)
 	 *     /
 	 *    o tvp
 	 */
-	if (tdcp->c_parentcnid == fcp->c_cnid) {
+	if (tdcp->c_parentcnid == fcp->c_fileid) {
 		error = EINVAL;
 		goto out;
 	}
@@ -2075,23 +2374,25 @@ hfs_vnop_rename(ap)
 	    vnode_isreg(fvp) &&
 	    (fdvp != tdvp) &&
 	    (fdcp->c_desc.cd_nameptr != NULL) &&
-	    (strcmp(fdcp->c_desc.cd_nameptr, CARBON_TEMP_DIR_NAME) == 0)) {
+	    (strncmp((const char *)fdcp->c_desc.cd_nameptr,
+		     CARBON_TEMP_DIR_NAME,
+		     sizeof(CARBON_TEMP_DIR_NAME)) == 0)) {
 		fcp->c_flags &= ~UF_NODUMP;
 		fcp->c_touch_chgtime = TRUE;
 		(void) hfs_update(fvp, 0);
 	}
 
 	bzero(&from_desc, sizeof(from_desc));
-	from_desc.cd_nameptr = fcnp->cn_nameptr;
+	from_desc.cd_nameptr = (const u_int8_t *)fcnp->cn_nameptr;
 	from_desc.cd_namelen = fcnp->cn_namelen;
-	from_desc.cd_parentcnid = fdcp->c_cnid;
+	from_desc.cd_parentcnid = fdcp->c_fileid;
 	from_desc.cd_flags = fcp->c_desc.cd_flags & ~(CD_HASBUF | CD_DECOMPOSED);
 	from_desc.cd_cnid = fcp->c_cnid;
 
 	bzero(&to_desc, sizeof(to_desc));
-	to_desc.cd_nameptr = tcnp->cn_nameptr;
+	to_desc.cd_nameptr = (const u_int8_t *)tcnp->cn_nameptr;
 	to_desc.cd_namelen = tcnp->cn_namelen;
-	to_desc.cd_parentcnid = tdcp->c_cnid;
+	to_desc.cd_parentcnid = tdcp->c_fileid;
 	to_desc.cd_flags = fcp->c_desc.cd_flags & ~(CD_HASBUF | CD_DECOMPOSED);
 	to_desc.cd_cnid = fcp->c_cnid;
 
@@ -2100,6 +2401,25 @@ hfs_vnop_rename(ap)
 	}
 	started_tr = 1;
 
+	/* hfs_vnop_link() and hfs_vnop_rename() set kHFSHasChildLinkMask 
+	 * inside a journal transaction and without holding a cnode lock.  
+	 * As setting of this bit depends on being in journal transaction for 
+	 * concurrency, check this bit again after we start journal transaction for rename
+	 * to ensure that this directory does not have any descendant that
+	 * is a directory hard link. 
+	 */
+	if (vnode_isdir(fvp) && (fdvp != tdvp)) {
+		if (fcp->c_attr.ca_recflags & kHFSHasChildLinkMask) {
+		    lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+		    if (cat_check_link_ancestry(hfsmp, tdcp->c_fileid, 0)) {
+				error = EPERM;
+				hfs_systemfile_unlock(hfsmp, lockflags);
+				goto out;
+			}
+			hfs_systemfile_unlock(hfsmp, lockflags);
+		}
+	}
+
 	// if it's a hardlink then re-lookup the name so
 	// that we get the correct cnid in from_desc (see
 	// the comment in hfs_removefile for more details)
@@ -2108,11 +2428,12 @@ hfs_vnop_rename(ap)
 	    struct cat_desc tmpdesc;
 	    cnid_t real_cnid;
 
-	    bzero(&tmpdesc, sizeof(tmpdesc));
-	    tmpdesc.cd_nameptr = fcnp->cn_nameptr;
+	    tmpdesc.cd_nameptr = (const u_int8_t *)fcnp->cn_nameptr;
 	    tmpdesc.cd_namelen = fcnp->cn_namelen;
-	    tmpdesc.cd_parentcnid = fdcp->c_cnid;
+	    tmpdesc.cd_parentcnid = fdcp->c_fileid;
 	    tmpdesc.cd_hint = fdcp->c_childhint;
+	    tmpdesc.cd_flags = fcp->c_desc.cd_flags & CD_ISDIR;
+	    tmpdesc.cd_encoding = 0;
 	    
 	    lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
 
@@ -2129,7 +2450,6 @@ hfs_vnop_rename(ap)
 	/*
 	 * Reserve some space in the Catalog file.
 	 */
-	bzero(&cookie, sizeof(cookie));
 	if ((error = cat_preflight(hfsmp, CAT_RENAME + CAT_DELETE, &cookie, p))) {
 		goto out;
 	}
@@ -2140,36 +2460,36 @@ hfs_vnop_rename(ap)
 	 */
 	if (tvp) {
 		/*
-		 * When fvp matches tvp they must be case variants
-		 * or hard links.
+		 * When fvp matches tvp they could be case variants
+		 * or matching hard links.
 		 */
 		if (fvp == tvp) {
-			/*
-			 * If this a hard link with different parents
-			 * and its not a case variant then tvp should
-			 * be removed.
-			 */
-			if (!((fcp->c_flag & C_HARDLINK) &&
-			    ((fdvp != tdvp) ||
-			     (hfs_namecmp(fcnp->cn_nameptr, fcnp->cn_namelen,
-					  tcnp->cn_nameptr, tcnp->cn_namelen) != 0)))) {
-				goto skip;
+			if (!(fcp->c_flag & C_HARDLINK)) {
+				goto skip_rm;  /* simple case variant */
+
+			} else if ((fdvp != tdvp) ||
+			           (hfsmp->hfs_flags & HFS_CASE_SENSITIVE)) {
+				goto out;  /* matching hardlinks, nothing to do */
+
+			} else if (hfs_namecmp((const u_int8_t *)fcnp->cn_nameptr, fcnp->cn_namelen,
+			                       (const u_int8_t *)tcnp->cn_nameptr, tcnp->cn_namelen) == 0) {
+				goto skip_rm;  /* case-variant hardlink in the same dir */
+			} else {
+				goto out;  /* matching hardlink, nothing to do */
 			}
-		} else {
-			cache_purge(tvp);
 		}
 
 		if (vnode_isdir(tvp))
 			error = hfs_removedir(tdvp, tvp, tcnp, HFSRM_SKIP_RESERVE);
 		else {
-			error = hfs_removefile(tdvp, tvp, tcnp, 0, HFSRM_SKIP_RESERVE);
+			error = hfs_removefile(tdvp, tvp, tcnp, 0, HFSRM_SKIP_RESERVE, 0);
 		}
 
 		if (error)
 			goto out;
 		tvp_deleted = 1;
 	}
-skip:
+skip_rm:
 	/*
 	 * All done with tvp and fvp
 	 */
@@ -2183,12 +2503,14 @@ skip:
 	}
 
 	/* Invalidate negative cache entries in the destination directory */
-	if (hfsmp->hfs_flags & HFS_CASE_SENSITIVE)
+	if (tdcp->c_flag & C_NEG_ENTRIES) {
 		cache_purge_negatives(tdvp);
+		tdcp->c_flag &= ~C_NEG_ENTRIES;
+	}
 
 	/* Update cnode's catalog descriptor */
 	replace_desc(fcp, &out_desc);
-	fcp->c_parentcnid = tdcp->c_cnid;
+	fcp->c_parentcnid = tdcp->c_fileid;
 	fcp->c_hint = 0;
 
 	hfs_volupdate(hfsmp, vnode_isdir(fvp) ? VOL_RMDIR : VOL_RMFILE,
@@ -2198,12 +2520,30 @@ skip:
 
 	/* Update both parent directories. */
 	if (fdvp != tdvp) {
-		tdcp->c_nlink++;
+		if (vnode_isdir(fvp)) {
+			/* If the source directory has directory hard link 
+			 * descendants, set the kHFSHasChildLinkBit in the 
+			 * destination parent hierarchy 
+			 */
+			if ((fcp->c_attr.ca_recflags & kHFSHasChildLinkMask) && 
+			    !(tdcp->c_attr.ca_recflags & kHFSHasChildLinkMask)) {
+
+				tdcp->c_attr.ca_recflags |= kHFSHasChildLinkMask;
+
+				error = cat_set_childlinkbit(hfsmp, tdcp->c_parentcnid);
+				if (error) {
+					printf ("hfs_vnop_rename: error updating parent chain for %u\n", tdcp->c_cnid);
+					error = 0;
+				}
+			}
+			INC_FOLDERCOUNT(hfsmp, tdcp->c_attr);
+			DEC_FOLDERCOUNT(hfsmp, fdcp->c_attr);
+		}
 		tdcp->c_entries++;
-		if (fdcp->c_nlink > 0)
-			fdcp->c_nlink--;
+		tdcp->c_dirchangecnt++;
 		if (fdcp->c_entries > 0)
 			fdcp->c_entries--;
+		fdcp->c_dirchangecnt++;
 		fdcp->c_touch_chgtime = TRUE;
 		fdcp->c_touch_modtime = TRUE;
 
@@ -2233,8 +2573,15 @@ out:
 		if (tdvp != fdvp) HFS_KNOTE(tdvp, NOTE_WRITE);
 	};
 
+	fdcp->c_flag &= ~C_DIR_MODIFICATION;
+	wakeup((caddr_t)&fdcp->c_flag);
+	if (fdvp != tdvp) {
+	    tdcp->c_flag &= ~C_DIR_MODIFICATION;
+	    wakeup((caddr_t)&tdcp->c_flag);
+	}
+
 	if (took_trunc_lock)
-		hfs_unlock_truncate(VTOC(tvp));	
+		hfs_unlock_truncate(VTOC(tvp), TRUE);	
 
 	hfs_unlockfour(fdcp, fcp, tdcp, tcp);
 
@@ -2267,12 +2614,14 @@ hfs_vnop_symlink(struct vnop_symlink_args *ap)
 	struct vnode **vpp = ap->a_vpp;
 	struct vnode *dvp = ap->a_dvp;
 	struct vnode *vp = NULL;
+	struct cnode *cp = NULL;
 	struct hfsmount *hfsmp;
 	struct filefork *fp;
 	struct buf *bp = NULL;
 	char *datap;
 	int started_tr = 0;
-	int len, error;
+	u_int32_t len;
+	int error;
 
 	/* HFS standard disks don't support symbolic links */
 	if (VTOVCB(dvp)->vcbSigWord != kHFSPlusSigWord)
@@ -2282,20 +2631,32 @@ hfs_vnop_symlink(struct vnop_symlink_args *ap)
 	if (ap->a_target[0] == 0)
 		return (EINVAL);
 
+	hfsmp = VTOHFS(dvp);
+	len = strlen(ap->a_target);
+
+	/* Check for free space */
+	if (((u_int64_t)hfs_freeblks(hfsmp, 0) * (u_int64_t)hfsmp->blockSize) < len) {
+		return (ENOSPC);
+	}
+
 	/* Create the vnode */
 	ap->a_vap->va_mode |= S_IFLNK;
 	if ((error = hfs_makenode(dvp, vpp, ap->a_cnp, ap->a_vap, ap->a_context))) {
 		goto out;
 	}
 	vp = *vpp;
-	if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK)))
-		return (error);
+	if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+		goto out;
+	}
+	cp = VTOC(vp);
 	fp = VTOF(vp);
-	hfsmp = VTOHFS(dvp);
-	len = strlen(ap->a_target);
+
+	if (cp->c_flag & (C_NOEXISTS | C_DELETED)) {
+	    goto out;
+	}
 
 #if QUOTA
-	(void)hfs_getinoquota(VTOC(vp));
+	(void)hfs_getinoquota(cp);
 #endif /* QUOTA */
 
 	if ((error = hfs_start_transaction(hfsmp)) != 0) {
@@ -2312,8 +2673,30 @@ hfs_vnop_symlink(struct vnop_symlink_args *ap)
 	 * Don't need truncate lock since a symlink is treated as a system file.
 	 */
 	error = hfs_truncate(vp, len, IO_NOZEROFILL, 1, ap->a_context);
-	if (error)
-		goto out;	/* XXX need to remove link */
+
+	/* On errors, remove the symlink file */
+	if (error) {
+		/*
+		 * End the transaction so we don't re-take the cnode lock
+		 * below while inside a transaction (lock order violation).
+		 */
+		hfs_end_transaction(hfsmp);
+
+		/* hfs_removefile() requires holding the truncate lock */
+		hfs_unlock(cp);
+		hfs_lock_truncate(cp, TRUE);
+		hfs_lock(cp, HFS_FORCE_LOCK);
+
+		if (hfs_start_transaction(hfsmp) != 0) {
+			started_tr = 0;
+			hfs_unlock_truncate(cp, TRUE);
+			goto out;
+		}
+		
+		(void) hfs_removefile(dvp, vp, ap->a_cnp, 0, 0, 0);
+		hfs_unlock_truncate(cp, TRUE);
+		goto out;	
+	}
 
 	/* Write the link to disk */
 	bp = buf_getblk(vp, (daddr64_t)0, roundup((int)fp->ff_size, VTOHFS(vp)->hfs_phys_block_size),
@@ -2326,7 +2709,7 @@ hfs_vnop_symlink(struct vnop_symlink_args *ap)
 	bcopy(ap->a_target, datap, len);
 
 	if (hfsmp->jnl) {
-		journal_modify_block_end(hfsmp->jnl, bp);
+		journal_modify_block_end(hfsmp->jnl, bp, NULL, NULL);
 	} else {
 		buf_bawrite(bp);
 	}
@@ -2341,8 +2724,14 @@ hfs_vnop_symlink(struct vnop_symlink_args *ap)
 out:
 	if (started_tr)
 	    hfs_end_transaction(hfsmp);
-	if (vp) {
-		hfs_unlock(VTOC(vp));
+	if ((cp != NULL) && (vp != NULL)) {
+		hfs_unlock(cp);
+	}
+	if (error) {
+		if (vp) {
+			vnode_put(vp);
+		}
+		*vpp = NULL;
 	}
 	return (error);
 }
@@ -2397,6 +2786,9 @@ typedef union {
  *  Each tag/index pair is tied to a unique directory hint.  The hint
  *  contains information (filename) needed to build the catalog b-tree
  *  key for finding the next set of entries.
+ *
+ * If the directory is marked as deleted-but-in-use (cp->c_flag & C_DELETED),
+ * do NOT synthesize entries for "." and "..".
  */
 static int
 hfs_vnop_readdir(ap)
@@ -2457,9 +2849,10 @@ hfs_vnop_readdir(ap)
 		}
 	}
 	/*
-	 * Synthesize entries for "." and ".."
+	 * Synthesize entries for "." and "..", unless the directory has
+	 * been deleted, but not closed yet (lazy delete in progress).
 	 */
-	if (offset == 0) {
+	if (offset == 0 && !(cp->c_flag & C_DELETED)) {
 		hfs_dotentry_t  dotentry[2];
 		size_t  uiosize;
 
@@ -2528,7 +2921,7 @@ hfs_vnop_readdir(ap)
 	// here and we can't service our page fault because VM is
 	// blocked trying to start a transaction as a result of
 	// trying to free up pages for our page fault.  It's messy
-	// but it does happen on dual-procesors that are paging
+	// but it does happen on dual-processors that are paging
 	// heavily (see radar 3082639 for more info).  By locking
 	// the buffer up-front we prevent ourselves from faulting
 	// while holding the shared catalog file lock.
@@ -2556,7 +2949,7 @@ hfs_vnop_readdir(ap)
 	/* When called from NFS, try and resolve a cnid hint. */
 	if (nfs_cookies && cnid_hint != 0) {
 		if (cat_findname(hfsmp, cnid_hint, &localhint.dh_desc) == 0) {
-			if ( localhint.dh_desc.cd_parentcnid == cp->c_cnid) {
+			if ( localhint.dh_desc.cd_parentcnid == cp->c_fileid) {
 				localhint.dh_index = index - 1;
 				localhint.dh_time = 0;
 				bzero(&localhint.dh_link, sizeof(localhint.dh_link));
@@ -2569,7 +2962,7 @@ hfs_vnop_readdir(ap)
 
 	/* Get a directory hint (cnode must be locked exclusive) */
 	if (dirhint == NULL) {
-		dirhint = hfs_getdirhint(cp, ((index - 1) & HFS_INDEX_MASK) | tag);
+		dirhint = hfs_getdirhint(cp, ((index - 1) & HFS_INDEX_MASK) | tag, 0);
 
 		/* Hide tag from catalog layer. */
 		dirhint->dh_index &= HFS_INDEX_MASK;
@@ -2578,9 +2971,17 @@ hfs_vnop_readdir(ap)
 		}
 	}
 	
+	if (index == 0) {
+		dirhint->dh_threadhint = cp->c_dirthreadhint;
+	}
+
 	/* Pack the buffer with dirent entries. */
 	error = cat_getdirentries(hfsmp, cp->c_entries, dirhint, uio, extended, &items, &eofflag);
 
+	if (index == 0 && error == 0) {
+		cp->c_dirthreadhint = dirhint->dh_threadhint;
+	}
+
 	hfs_systemfile_unlock(hfsmp, lockflags);
 
 	if (error != 0) {
@@ -2657,7 +3058,7 @@ hfs_vnop_readlink(ap)
    
 	/* Zero length sym links are not allowed */
 	if (fp->ff_size == 0 || fp->ff_size > MAXPATHLEN) {
-		VTOVCB(vp)->vcbFlags |= kHFS_DamagedVolume;
+		printf("hfs: zero length symlink on fileid %d\n", cp->c_fileid);
 		error = EINVAL;
 		goto exit;
 	}
@@ -2745,10 +3146,10 @@ hfs_vnop_pathconf(ap)
 		*ap->a_retval = PIPE_BUF;
 		break;
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		break;
 	case _PC_NO_TRUNC:
-		*ap->a_retval = 0;
+		*ap->a_retval = 200112;		/* _POSIX_NO_TRUNC */
 		break;
 	case _PC_NAME_CHARS_MAX:
 		*ap->a_retval = kHFSPlusMaxFileNameChars;
@@ -2762,6 +3163,9 @@ hfs_vnop_pathconf(ap)
 	case _PC_CASE_PRESERVING:
 		*ap->a_retval = 1;
 		break;
+	case _PC_FILESIZEBITS:
+		*ap->a_retval = 64;	/* number of bits to store max file size */
+		break;
 	default:
 		return (EINVAL);
 	}
@@ -2794,7 +3198,8 @@ hfs_update(struct vnode *vp, __unused int waitfor)
 	p = current_proc();
 	hfsmp = VTOHFS(vp);
 
-	if (vnode_issystem(vp) && (cp->c_cnid < kHFSFirstUserCatalogNodeID)) {
+	if (((vnode_issystem(vp) && (cp->c_cnid < kHFSFirstUserCatalogNodeID))) || 
+	   	hfsmp->hfs_catalog_vp == NULL){
 		return (0);
 	}
 	if ((hfsmp->hfs_flags & HFS_READ_ONLY) || (cp->c_mode == 0)) {
@@ -2828,7 +3233,7 @@ hfs_update(struct vnode *vp, __unused int waitfor)
 	 * we have to do the update.
 	 */
 	if (ISSET(cp->c_flag, C_FORCEUPDATE) == 0 &&
-	    (ISSET(cp->c_flag, C_DELETED) ||
+	    (ISSET(cp->c_flag, C_DELETED) || 
 	    (dataforkp && cp->c_datafork->ff_unallocblocks) ||
 	    (rsrcforkp && cp->c_rsrcfork->ff_unallocblocks))) {
 	//	cp->c_flag &= ~(C_ACCESS | C_CHANGE | C_UPDATE);
@@ -2869,10 +3274,8 @@ hfs_update(struct vnode *vp, __unused int waitfor)
 
 	/*
 	 * Lock the Catalog b-tree file.
-	 * A shared lock is sufficient since an update doesn't change
-	 * the tree and the lock on vp protects the cnode.
 	 */
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_SHARED_LOCK);
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 
 	/* XXX - waitfor is not enforced */
 	error = cat_update(hfsmp, &cp->c_desc, &cp->c_attr, dataforkp, rsrcforkp);
@@ -2891,6 +3294,7 @@ hfs_update(struct vnode *vp, __unused int waitfor)
 
 /*
  * Allocate a new node
+ * Note - Function does not create and return a vnode for whiteout creation.
  */
 static int
 hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
@@ -2903,29 +3307,30 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	struct cat_desc in_desc, out_desc;
 	struct cat_attr attr;
 	struct timeval tv;
-	cat_cookie_t cookie;
 	int lockflags;
-	int error, started_tr = 0, got_cookie = 0;
+	int error, started_tr = 0;
 	enum vtype vnodetype;
 	int mode;
 
-	if ((error = hfs_lock(VTOC(dvp), HFS_EXCLUSIVE_LOCK)))
-		return (error);
 	dcp = VTOC(dvp);
+	if ((error = hfs_lock(dcp, HFS_EXCLUSIVE_LOCK)))
+		return (error);
+
+	dcp->c_flag |= C_DIR_MODIFICATION;
+	
 	hfsmp = VTOHFS(dvp);
 	*vpp = NULL;
 	tvp = NULL;
 	out_desc.cd_flags = 0;
 	out_desc.cd_nameptr = NULL;
 
-	mode = MAKEIMODE(vap->va_type, vap->va_mode);
-
-	if ((mode & S_IFMT) == 0)
-		mode |= S_IFREG;
-	vnodetype = IFTOVT(mode);
+	vnodetype = vap->va_type;
+	if (vnodetype == VNON)
+		vnodetype = VREG;
+	mode = MAKEIMODE(vnodetype, vap->va_mode);
 
 	/* Check if were out of usable disk space. */
-	if ((hfs_freeblks(hfsmp, 1) <= 0) && (suser(vfs_context_ucred(ctx), NULL) != 0)) {
+	if ((hfs_freeblks(hfsmp, 1) == 0) && (vfs_context_suser(ctx) != 0)) {
 		error = ENOSPC;
 		goto exit;
 	}
@@ -2935,16 +3340,38 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	/* Setup the default attributes */
 	bzero(&attr, sizeof(attr));
 	attr.ca_mode = mode;
-	attr.ca_nlink = vnodetype == VDIR ? 2 : 1;
-	attr.ca_mtime = tv.tv_sec;
-	if ((VTOVCB(dvp)->vcbSigWord == kHFSSigWord) && gTimeZone.tz_dsttime) {
-		attr.ca_mtime += 3600;	/* Same as what hfs_update does */
+	attr.ca_linkcount = 1;
+	if (VATTR_IS_ACTIVE(vap, va_rdev)) {
+		attr.ca_rdev = vap->va_rdev;
+	}
+	if (VATTR_IS_ACTIVE(vap, va_create_time)) {
+		VATTR_SET_SUPPORTED(vap, va_create_time);
+		attr.ca_itime = vap->va_create_time.tv_sec;
+	} else {
+		attr.ca_itime = tv.tv_sec;
+	}
+	if ((hfsmp->hfs_flags & HFS_STANDARD) && gTimeZone.tz_dsttime) {
+		attr.ca_itime += 3600;	/* Same as what hfs_update does */
 	}
-	attr.ca_atime = attr.ca_ctime = attr.ca_itime = attr.ca_mtime;
+	attr.ca_atime = attr.ca_ctime = attr.ca_mtime = attr.ca_itime;
 	attr.ca_atimeondisk = attr.ca_atime;
-	/* On HFS+ the ThreadExists flag must always be set for files. */
-	if (vnodetype != VDIR && (hfsmp->hfs_flags & HFS_STANDARD) == 0)
-		attr.ca_recflags = kHFSThreadExistsMask;
+	if (VATTR_IS_ACTIVE(vap, va_flags)) {
+		VATTR_SET_SUPPORTED(vap, va_flags);
+		attr.ca_flags = vap->va_flags;
+	}
+	
+	/* 
+	 * HFS+ only: all files get ThreadExists
+	 * HFSX only: dirs get HasFolderCount
+	 */
+	if (!(hfsmp->hfs_flags & HFS_STANDARD)) {
+		if (vnodetype == VDIR) {
+			if (hfsmp->hfs_flags & HFS_FOLDERCOUNT)
+				attr.ca_recflags = kHFSHasFolderCountMask;
+		} else {
+			attr.ca_recflags = kHFSThreadExistsMask;
+		}
+	}
 
 	attr.ca_uid = vap->va_uid;
 	attr.ca_gid = vap->va_gid;
@@ -2964,9 +3391,9 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 		attr.ca_flags |= UF_OPAQUE;
 
 	/* Setup the descriptor */
-	in_desc.cd_nameptr = cnp->cn_nameptr;
+	in_desc.cd_nameptr = (const u_int8_t *)cnp->cn_nameptr;
 	in_desc.cd_namelen = cnp->cn_namelen;
-	in_desc.cd_parentcnid = dcp->c_cnid;
+	in_desc.cd_parentcnid = dcp->c_fileid;
 	in_desc.cd_flags = S_ISDIR(mode) ? CD_ISDIR : 0;
 	in_desc.cd_hint = dcp->c_childhint;
 	in_desc.cd_encoding = 0;
@@ -2976,25 +3403,25 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	}
 	started_tr = 1;
 
-	/*
-	 * Reserve some space in the Catalog file.
-	 *
-	 * (we also add CAT_DELETE since our getnewvnode
-	 *  request can cause an hfs_inactive call to
-	 *  delete an unlinked file)
-	 */
-	if ((error = cat_preflight(hfsmp, CAT_CREATE | CAT_DELETE, &cookie, 0))) {
+	// have to also lock the attribute file because cat_create() needs
+	// to check that any fileID it wants to use does not have orphaned
+	// attributes in it.
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG | SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
+
+	/* Reserve some space in the Catalog file. */
+	if ((error = cat_preflight(hfsmp, CAT_CREATE, NULL, 0))) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
 		goto exit;
 	}
-	got_cookie = 1;
-
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_CATALOG, HFS_EXCLUSIVE_LOCK);
 	error = cat_create(hfsmp, &in_desc, &attr, &out_desc);
 	if (error == 0) {
 		/* Update the parent directory */
 		dcp->c_childhint = out_desc.cd_hint;	/* Cache directory's location */
-		dcp->c_nlink++;
 		dcp->c_entries++;
+		if (vnodetype == VDIR) {
+			INC_FOLDERCOUNT(hfsmp, dcp->c_attr);
+		}
+		dcp->c_dirchangecnt++;
 		dcp->c_ctime = tv.tv_sec;
 		dcp->c_mtime = tv.tv_sec;
 		(void) cat_update(hfsmp, &dcp->c_desc, &dcp->c_attr, NULL, NULL);
@@ -3005,8 +3432,10 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 		goto exit;
 	
 	/* Invalidate negative cache entries in the directory */
-	if (hfsmp->hfs_flags & HFS_CASE_SENSITIVE)
+	if (dcp->c_flag & C_NEG_ENTRIES) {
 		cache_purge_negatives(dvp);
+		dcp->c_flag &= ~C_NEG_ENTRIES;
+	}
 
 	if (vnodetype == VDIR) {
 		HFS_KNOTE(dvp, NOTE_WRITE | NOTE_LINK);
@@ -3034,18 +3463,20 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	    started_tr = 0;
 	}
 
+	/* Do not create vnode for whiteouts */
+	if (S_ISWHT(mode)) {
+		goto exit;
+	}
+
 	/*
 	 * Create a vnode for the object just created.
 	 *
 	 * The cnode is locked on successful return.
 	 */
-	error = hfs_getnewvnode(hfsmp, dvp, cnp, &out_desc, 0, &attr, NULL, &tvp);
+	error = hfs_getnewvnode(hfsmp, dvp, cnp, &out_desc, GNV_CREATE, &attr, NULL, &tvp);
 	if (error)
 		goto exit;
 
-	// XXXdbg
-	//cache_enter(dvp, tvp, cnp);
-
 	cp = VTOC(tvp);
 #if QUOTA
 	/* 
@@ -3053,7 +3484,7 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	 * fall through to the rmdir we actually have 
 	 * accounted for the inode
 	*/
-	if (vfs_flags(HFSTOVFS(hfsmp)) & MNT_QUOTA) {
+	if (hfsmp->hfs_flags & HFS_QUOTAS) {
 		if ((error = hfs_getinoquota(cp)) ||
 		    (error = hfs_chkiq(cp, 1, vfs_context_ucred(ctx), FORCE))) {
 	
@@ -3063,8 +3494,8 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 				hfs_unlock(cp);
 				hfs_lock_truncate(cp, TRUE);
 				hfs_lock(cp, HFS_FORCE_LOCK);
-				(void) hfs_removefile(dvp, tvp, cnp, 0, 0);
-				hfs_unlock_truncate(cp);
+				(void) hfs_removefile(dvp, tvp, cnp, 0, 0, 0);
+				hfs_unlock_truncate(cp, TRUE);
 			}
 			/*
 			 * we successfully allocated a new vnode, but
@@ -3079,20 +3510,10 @@ hfs_makenode(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 	}
 #endif /* QUOTA */
 
-	/* Remember if any ACL data was set. */
-	if (VATTR_IS_ACTIVE(vap, va_acl) &&
-	    (vap->va_acl != NULL)) {
-		cp->c_attr.ca_recflags |= kHFSHasSecurityMask;
-		cp->c_touch_chgtime = TRUE;
-		(void) hfs_update(tvp, TRUE);
-	}
 	*vpp = tvp;
 exit:
 	cat_releasedesc(&out_desc);
 
-	if (got_cookie) {
-		cat_postflight(hfsmp, &cookie, 0);
-	}
 	/*
 	 * Check if a file is located in the "Cleanup At Startup"
 	 * directory.  If it is then tag it as NODUMP so that we
@@ -3100,9 +3521,14 @@ exit:
 	 */
 	if ((error == 0) && dvp && (vnodetype == VREG) &&
 	    (dcp->c_desc.cd_nameptr != NULL) &&
-	    (strcmp(dcp->c_desc.cd_nameptr, CARBON_TEMP_DIR_NAME) == 0)) {
+	    (strncmp((const char *)dcp->c_desc.cd_nameptr,
+		     CARBON_TEMP_DIR_NAME,
+		     sizeof(CARBON_TEMP_DIR_NAME)) == 0)) {
 	   	struct vnode *ddvp;
 
+		dcp->c_flag &= ~C_DIR_MODIFICATION;
+		wakeup((caddr_t)&dcp->c_flag);
+
 		hfs_unlock(dcp);
 		dvp = NULL;
 
@@ -3114,7 +3540,7 @@ exit:
 			if (VTOC(ddvp)->c_desc.cd_nameptr) {
 				uid_t uid;
 
-				uid = strtoul(VTOC(ddvp)->c_desc.cd_nameptr, 0, 0);
+				uid = strtoul((const char *)VTOC(ddvp)->c_desc.cd_nameptr, 0, 0);
 				if ((uid == cp->c_uid) ||
 				    (uid == vfs_context_ucred(ctx)->cr_uid)) {
 					cp->c_flags |= UF_NODUMP;
@@ -3126,6 +3552,9 @@ exit:
 		}
 	}
 	if (dvp) {
+		dcp->c_flag &= ~C_DIR_MODIFICATION;
+		wakeup((caddr_t)&dcp->c_flag);
+		
 		hfs_unlock(dcp);
 	}
 	if (error == 0 && cp != NULL) {
@@ -3141,24 +3570,48 @@ exit:
 
 
 /*
- * WARNING - assumes caller has cnode lock.
+ * Return a referenced vnode for the resource fork
+ *
+ * cnode for vnode vp must already be locked.
+ *
+ * can_drop_lock is true if its safe to temporally drop/re-acquire the cnode lock
  */
 __private_extern__
 int
-hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp, struct vnode **rvpp, __unused struct proc *p)
+hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp, struct vnode **rvpp, int can_drop_lock)
 {
 	struct vnode *rvp;
 	struct cnode *cp = VTOC(vp);
 	int error;
 	int vid;
 
+	/* Attempt to use exising vnode */
 	if ((rvp = cp->c_rsrc_vp)) {
 	        vid = vnode_vid(rvp);
 
-		/* Use exising vnode */
+		/*
+		 * It is not safe to hold the cnode lock when calling vnode_getwithvid()
+		 * for the alternate fork -- vnode_getwithvid() could deadlock waiting
+		 * for a VL_WANTTERM while another thread has an iocount on the alternate
+		 * fork vnode and is attempting to acquire the common cnode lock.
+		 *
+		 * But it's also not safe to drop the cnode lock when we're holding
+		 * multiple cnode locks, like during a hfs_removefile() operation
+		 * since we could lock out of order when re-acquiring the cnode lock.
+		 *
+		 * So we can only drop the lock here if its safe to drop it -- which is
+		 * most of the time with the exception being hfs_removefile().
+		 */
+		if (can_drop_lock)
+			hfs_unlock(cp);
+
 		error = vnode_getwithvid(rvp, vid);
+
+		if (can_drop_lock)
+			(void) hfs_lock(cp, HFS_FORCE_LOCK);
+
 		if (error) {
-			char * name = VTOC(vp)->c_desc.cd_nameptr;
+			const char * name = (const char *)VTOC(vp)->c_desc.cd_nameptr;
 
 			if (name)
 				printf("hfs_vgetrsrc: couldn't get"
@@ -3177,11 +3630,13 @@ hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp, struct vnode **rvpp, __un
 		 * and that its safe to have the cnode lock dropped and reacquired.
 		 */
 		if (cp->c_lockowner != current_thread()) {
+			if (!can_drop_lock)
+				return (EINVAL);
 			/*
 			 * If the upgrade fails we loose the lock and
 			 * have to take the exclusive lock on our own.
 			 */
-			if (lck_rw_lock_shared_to_exclusive(&cp->c_rwlock) != 0)
+			if (lck_rw_lock_shared_to_exclusive(&cp->c_rwlock) == FALSE)
 				lck_rw_lock_exclusive(&cp->c_rwlock);
 			cp->c_lockowner = current_thread();
 		}
@@ -3209,10 +3664,13 @@ hfs_vgetrsrc(struct hfsmount *hfsmp, struct vnode *vp, struct vnode **rvpp, __un
 			cn.cn_nameptr = cn.cn_pnbuf;
 			cn.cn_hash = 0;
 			cn.cn_consume = 0;
-			cn.cn_namelen = sprintf(cn.cn_nameptr, "%s%s", cp->c_desc.cd_nameptr, _PATH_RSRCFORKSPEC);
+			cn.cn_namelen = snprintf(cn.cn_nameptr, MAXPATHLEN,
+						 "%s%s", cp->c_desc.cd_nameptr,
+						 _PATH_RSRCFORKSPEC);
 		}
 		error = hfs_getnewvnode(hfsmp, vnode_parent(vp), cn.cn_pnbuf ? &cn : NULL,
-		                        &cp->c_desc, 2, &cp->c_attr, &rsrcfork, &rvp);
+		                        &cp->c_desc, GNV_WANTRSRC | GNV_SKIPLOCK, &cp->c_attr,
+		                        &rsrcfork, &rvp);
 		if (cn.cn_pnbuf)
 			FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
 		if (error)
@@ -3266,8 +3724,19 @@ filt_hfsread(struct knote *kn, long hint)
 	}
 
 	/* poll(2) semantics dictate always saying there is data */
-	kn->kn_data = (!(kn->kn_flags & EV_POLL)) ?
-		VTOF(vp)->ff_size - kn->kn_fp->f_fglob->fg_offset : 1;
+	if (!(kn->kn_flags & EV_POLL)) {
+		off_t amount;
+ 
+		amount = VTOF(vp)->ff_size - kn->kn_fp->f_fglob->fg_offset;
+		if (amount > (off_t)INTPTR_MAX)
+			kn->kn_data = INTPTR_MAX;
+		else if (amount < (off_t)INTPTR_MIN)
+			kn->kn_data = INTPTR_MIN;
+		else
+			kn->kn_data = (intptr_t)amount;
+	} else {
+		kn->kn_data = 1;
+	}
 
 	if  (dropvp)
 		vnode_put(vp);
@@ -3279,13 +3748,13 @@ filt_hfsread(struct knote *kn, long hint)
 static int
 filt_hfswrite(struct knote *kn, long hint)
 {
-	int dropvp = 0;
-	
+	struct vnode *vp = (struct vnode *)kn->kn_hook;
+
 	if (hint == 0)  {
-		if ((vnode_getwithvid(kn->kn_hook, kn->kn_hookid) != 0)) {
+		if ((vnode_getwithvid(vp, kn->kn_hookid) != 0)) {
 			hint = NOTE_REVOKE;
 		} else 
-			vnode_put(kn->kn_hook);
+			vnode_put(vp);
 	}
 	if (hint == NOTE_REVOKE) {
 		/*
@@ -3303,12 +3772,13 @@ filt_hfswrite(struct knote *kn, long hint)
 static int
 filt_hfsvnode(struct knote *kn, long hint)
 {
+	struct vnode *vp = (struct vnode *)kn->kn_hook;
 
 	if (hint == 0)  {
-		if ((vnode_getwithvid(kn->kn_hook, kn->kn_hookid) != 0)) {
+		if ((vnode_getwithvid(vp, kn->kn_hookid) != 0)) {
 			hint = NOTE_REVOKE;
 		} else
-			vnode_put(kn->kn_hook);
+			vnode_put(vp);
 	}
 	if (kn->kn_sfflags & hint)
 		kn->kn_fflags |= hint;
@@ -3385,7 +3855,7 @@ hfs_vnop_kqfiltremove(ap)
 		struct vnode *a_vp;
 		uintptr_t ident;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */__unused *ap;
 {
 	int result;
 
@@ -3472,8 +3942,6 @@ hfsfifo_read(ap)
 		vfs_context_t a_context;
 	} */ *ap;
 {
-	extern int (**fifo_vnodeop_p)(void *);
-
 	/*
 	 * Set access flag.
 	 */
@@ -3493,8 +3961,6 @@ hfsfifo_write(ap)
 		vfs_context_t a_context;
 	} */ *ap;
 {
-	extern int (**fifo_vnodeop_p)(void *);
-
 	/*
 	 * Set update and change flags.
 	 */
@@ -3516,7 +3982,6 @@ hfsfifo_close(ap)
 		vfs_context_t a_context;
 	} */ *ap;
 {
-	extern int (**fifo_vnodeop_p)(void *);
 	struct vnode *vp = ap->a_vp;
 	struct cnode *cp;
 
@@ -3539,7 +4004,6 @@ int
 hfsfifo_kqfilt_add(ap)
 	struct vnop_kqfilt_add_args *ap;
 {
-	extern int (**fifo_vnodeop_p)(void *);
 	int error;
 
 	error = VOCALL(fifo_vnodeop_p, VOFFSET(vnop_kqfilt_add), ap);
@@ -3557,7 +4021,6 @@ int
 hfsfifo_kqfilt_remove(ap)
 	struct vnop_kqfilt_remove_args *ap;
 {
-	extern int (**fifo_vnodeop_p)(void *);
 	int error;
 
 	error = VOCALL(fifo_vnodeop_p, VOFFSET(vnop_kqfilt_remove), ap);
@@ -3596,34 +4059,73 @@ hfs_vnop_fsync(ap)
 	return (error);
 }
 
-/*****************************************************************************
-*
-*	VOP Tables
-*
-*****************************************************************************/
-int hfs_vnop_readdirattr(struct vnop_readdirattr_args *);  /* in hfs_attrlist.c */
-int hfs_vnop_inactive(struct vnop_inactive_args *);        /* in hfs_cnode.c */
-int hfs_vnop_reclaim(struct vnop_reclaim_args *);          /* in hfs_cnode.c */
-int hfs_vnop_link(struct vnop_link_args *);                /* in hfs_link.c */
-int hfs_vnop_lookup(struct vnop_lookup_args *);            /* in hfs_lookup.c */
-int hfs_vnop_search(struct vnop_searchfs_args *);          /* in hfs_search.c */
-
-int hfs_vnop_read(struct vnop_read_args *);           /* in hfs_readwrite.c */
-int hfs_vnop_write(struct vnop_write_args *);         /* in hfs_readwrite.c */
-int hfs_vnop_ioctl(struct vnop_ioctl_args *);         /* in hfs_readwrite.c */
-int hfs_vnop_select(struct vnop_select_args *);       /* in hfs_readwrite.c */
-int hfs_vnop_strategy(struct vnop_strategy_args *);   /* in hfs_readwrite.c */
-int hfs_vnop_allocate(struct vnop_allocate_args *);   /* in hfs_readwrite.c */
-int hfs_vnop_pagein(struct vnop_pagein_args *);       /* in hfs_readwrite.c */
-int hfs_vnop_pageout(struct vnop_pageout_args *);     /* in hfs_readwrite.c */
-int hfs_vnop_bwrite(struct vnop_bwrite_args *);       /* in hfs_readwrite.c */
-int hfs_vnop_blktooff(struct vnop_blktooff_args *);   /* in hfs_readwrite.c */
-int hfs_vnop_offtoblk(struct vnop_offtoblk_args *);   /* in hfs_readwrite.c */
-int hfs_vnop_blockmap(struct vnop_blockmap_args *);   /* in hfs_readwrite.c */
-int hfs_vnop_getxattr(struct vnop_getxattr_args *);        /* in hfs_xattr.c */
-int hfs_vnop_setxattr(struct vnop_setxattr_args *);        /* in hfs_xattr.c */
-int hfs_vnop_removexattr(struct vnop_removexattr_args *);  /* in hfs_xattr.c */
-int hfs_vnop_listxattr(struct vnop_listxattr_args *);      /* in hfs_xattr.c */
+
+static int
+hfs_vnop_whiteout(ap) 
+	struct vnop_whiteout_args /* {
+		struct vnode *a_dvp;
+		struct componentname *a_cnp;
+		int a_flags;
+		vfs_context_t a_context;
+	} */ *ap;
+{
+	int error = 0;
+	struct vnode *vp = NULL;
+	struct vnode_attr va;
+	struct vnop_lookup_args lookup_args;
+	struct vnop_remove_args remove_args;
+	struct hfsmount *hfsmp;
+
+	hfsmp = VTOHFS(ap->a_dvp);
+	if (hfsmp->hfs_flags & HFS_STANDARD) {
+		error = ENOTSUP;
+		goto exit;
+	}
+
+	switch (ap->a_flags) {
+		case LOOKUP:
+			error = 0;
+			break;
+
+		case CREATE: 
+			VATTR_INIT(&va);
+			VATTR_SET(&va, va_type, VREG);
+			VATTR_SET(&va, va_mode, S_IFWHT);
+			VATTR_SET(&va, va_uid, 0);
+			VATTR_SET(&va, va_gid, 0);
+			
+			error = hfs_makenode(ap->a_dvp, &vp, ap->a_cnp, &va, ap->a_context);
+			/* No need to release the vnode as no vnode is created for whiteouts */
+			break;
+
+		case DELETE:
+			lookup_args.a_dvp = ap->a_dvp;
+			lookup_args.a_vpp = &vp;
+			lookup_args.a_cnp = ap->a_cnp;
+			lookup_args.a_context = ap->a_context;
+
+			error = hfs_vnop_lookup(&lookup_args);
+			if (error) {
+				break;
+			}
+			
+			remove_args.a_dvp = ap->a_dvp;
+			remove_args.a_vp = vp;
+			remove_args.a_cnp = ap->a_cnp;
+			remove_args.a_flags = 0;
+			remove_args.a_context = ap->a_context;
+
+			error = hfs_vnop_remove(&remove_args);
+			vnode_put(vp);
+			break;
+
+		default:
+			panic("hfs_vnop_whiteout: unknown operation (flag = %x)\n", ap->a_flags);
+	};
+	
+exit:
+	return (error);
+}
 
 int (**hfs_vnodeop_p)(void *);
 
@@ -3675,6 +4177,12 @@ struct vnodeopv_entry_desc hfs_vnodeop_entries[] = {
     { &vnop_setxattr_desc, (VOPFUNC)hfs_vnop_setxattr},
     { &vnop_removexattr_desc, (VOPFUNC)hfs_vnop_removexattr},
     { &vnop_listxattr_desc, (VOPFUNC)hfs_vnop_listxattr},
+    { &vnop_whiteout_desc, (VOPFUNC)hfs_vnop_whiteout},
+#if NAMEDSTREAMS
+    { &vnop_getnamedstream_desc, (VOPFUNC)hfs_vnop_getnamedstream },
+    { &vnop_makenamedstream_desc, (VOPFUNC)hfs_vnop_makenamedstream },
+    { &vnop_removenamedstream_desc, (VOPFUNC)hfs_vnop_removenamedstream },
+#endif
     { NULL, (VOPFUNC)NULL }
 };
 
diff --git a/bsd/hfs/hfs_xattr.c b/bsd/hfs/hfs_xattr.c
index 1dd671913..37dca768b 100644
--- a/bsd/hfs/hfs_xattr.c
+++ b/bsd/hfs/hfs_xattr.c
@@ -1,32 +1,41 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
+#include <sys/ubc.h>
 #include <sys/utfconv.h>
 #include <sys/vnode.h>
 #include <sys/xattr.h>
+#include <sys/fcntl.h>
+#include <sys/vnode_internal.h>
 #include <sys/kauth.h>
 
 #include "hfs.h"
@@ -34,9 +43,12 @@
 #include "hfs_mount.h"
 #include "hfs_format.h"
 #include "hfs_endian.h"
+#include "hfs_btreeio.h"
+#include "hfs_fsctl.h"
 
 #include "hfscommon/headers/BTreesInternal.h"
 
+#define HFS_XATTR_VERBOSE  0
 
 #define  ATTRIBUTE_FILE_NODE_SIZE   8192
 
@@ -49,38 +61,146 @@ struct listattr_callback_state {
 	size_t      size;
 };
 
-#define HFS_MAXATTRIBUTESIZE    (1024*1024)
+#define HFS_MAXATTRIBUTESIZE    (128 * 1024)
+#define HFS_MAXATTRBLKS         (32 * 1024)
+
 
 /* HFS Internal Names */
 #define	XATTR_EXTENDEDSECURITY_NAME   "system.extendedsecurity"
+#define XATTR_XATTREXTENTS_NAME	      "system.xattrextents"
 
-
-#define RESOURCE_FORK_EXISTS(VP)   \
-	((VTOC((VP))->c_blocks - VTOF((VP))->ff_blocks) > 0)
+/* Faster version if we already know this is the data fork. */
+#define RSRC_FORK_EXISTS(CP)   \
+	(((CP)->c_attr.ca_blocks - (CP)->c_datafork->ff_data.cf_blocks) > 0)
 
 static u_int32_t emptyfinfo[8] = {0};
 
-
-extern int  hfs_create_attr_btree(struct hfsmount *hfsmp, uint32_t nodesize, uint32_t nodecnt);
-
-
-int  hfs_vnop_getxattr(struct vnop_getxattr_args *ap);
-int  hfs_vnop_setxattr(struct vnop_setxattr_args *ap);
-int  hfs_vnop_removexattr(struct vnop_removexattr_args *ap);
-int  hfs_vnop_listxattr(struct vnop_listxattr_args *ap);
-int  hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKey);
-
-static int file_attribute_exist(struct hfsmount *hfsmp, uint32_t fileID);
+const char hfs_attrdatafilename[] = "Attribute Data";
 
 static int  listattr_callback(const HFSPlusAttrKey *key, const HFSPlusAttrData *data,
                        struct listattr_callback_state *state);
 
-static int  buildkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key);
+static int  remove_attribute_records(struct hfsmount *hfsmp, BTreeIterator * iterator);
 
 static int  getnodecount(struct hfsmount *hfsmp, size_t nodesize);
 
 static size_t  getmaxinlineattrsize(struct vnode * attrvp);
 
+static int  read_attr_data(struct hfsmount *hfsmp, uio_t uio, size_t datasize, HFSPlusExtentDescriptor *extents);
+
+static int  write_attr_data(struct hfsmount *hfsmp, uio_t uio, size_t datasize, HFSPlusExtentDescriptor *extents);
+
+static int  alloc_attr_blks(struct hfsmount *hfsmp, size_t attrsize, size_t extentbufsize, HFSPlusExtentDescriptor *extents, int *blocks);
+
+static void  free_attr_blks(struct hfsmount *hfsmp, int blkcnt, HFSPlusExtentDescriptor *extents);
+
+static int  has_overflow_extents(HFSPlusForkData *forkdata);
+
+static int  count_extent_blocks(int maxblks, HFSPlusExtentRecord extents);
+
+#if NAMEDSTREAMS
+/*
+ * Obtain the vnode for a stream.
+ */
+int
+hfs_vnop_getnamedstream(struct vnop_getnamedstream_args* ap)
+{
+	vnode_t vp = ap->a_vp;
+	vnode_t *svpp = ap->a_svpp;
+	struct cnode *cp;
+	int error = 0;
+
+	*svpp = NULL;
+
+	/*
+	 * We only support the "com.apple.ResourceFork" stream.
+	 */
+	if (bcmp(ap->a_name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		return (ENOATTR);
+	}
+	cp = VTOC(vp);
+	if ( !S_ISREG(cp->c_mode) ) {
+		return (EPERM);
+	}
+	if ((error = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+		return (error);
+	}
+	if (!RSRC_FORK_EXISTS(cp) && (ap->a_operation != NS_OPEN)) {
+		hfs_unlock(cp);
+		return (ENOATTR);
+	}
+	error = hfs_vgetrsrc(VTOHFS(vp), vp, svpp, TRUE);
+	hfs_unlock(cp);
+
+	return (error);
+}
+
+/*
+ * Create a stream.
+ */
+int
+hfs_vnop_makenamedstream(struct vnop_makenamedstream_args* ap)
+{
+	vnode_t vp = ap->a_vp;
+	vnode_t *svpp = ap->a_svpp;
+	struct cnode *cp;
+	int error = 0;
+
+	*svpp = NULL;
+
+	/*
+	 * We only support the "com.apple.ResourceFork" stream.
+	 */
+	if (bcmp(ap->a_name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		return (ENOATTR);
+	}
+	cp = VTOC(vp);
+	if ( !S_ISREG(cp->c_mode) ) {
+		return (EPERM);
+	}
+	if ((error = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
+		return (error);
+	}
+	error = hfs_vgetrsrc(VTOHFS(vp), vp, svpp, TRUE);
+	hfs_unlock(cp);
+
+	return (error);
+}
+
+/*
+ * Remove a stream.
+ */
+int
+hfs_vnop_removenamedstream(struct vnop_removenamedstream_args* ap)
+{
+	vnode_t svp = ap->a_svp;
+	struct cnode *scp;
+	int error = 0;
+
+	/*
+	 * We only support the "com.apple.ResourceFork" stream.
+	 */
+	if (bcmp(ap->a_name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		return (ENOATTR);
+	}
+	scp = VTOC(svp);
+
+	/* Take truncate lock before taking cnode lock. */
+	hfs_lock_truncate(scp, TRUE);
+	if ((error = hfs_lock(scp, HFS_EXCLUSIVE_LOCK))) {
+		goto out;
+	}
+	if (VTOF(svp)->ff_size != 0) {
+		error = hfs_truncate(svp, 0, IO_NDELAY, 0, ap->a_context);
+	}
+	hfs_unlock(scp);
+out:
+	hfs_unlock_truncate(scp, TRUE);
+	return (error);
+}
+#endif
+
+
 /*
  * Retrieve the data of an extended attribute.
  */
@@ -100,29 +220,42 @@ hfs_vnop_getxattr(struct vnop_getxattr_args *ap)
 */
 {
 	struct vnode *vp = ap->a_vp;
+	struct cnode *cp;
 	struct hfsmount *hfsmp;
 	uio_t uio = ap->a_uio;
 	struct BTreeIterator * iterator = NULL;
 	struct filefork *btfile;
 	FSBufferDescriptor btdata;
-	HFSPlusAttrData * datap = NULL;
+	HFSPlusAttrRecord * recp = NULL;
 	size_t bufsize;
-	UInt16 datasize;
+	u_int16_t datasize;
 	int lockflags;
 	int result;
 
-	if (ap->a_name == NULL || ap->a_name[0] == '\0') {
-		return (EINVAL);  /* invalid name */
-	}
-	hfsmp = VTOHFS(vp);
-
-	if (!VNODE_IS_RSRC(vp)) {
+	cp = VTOC(vp);
+	if (vp == cp->c_vp) {
 		/* Get the Finder Info. */
 		if (bcmp(ap->a_name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) {
+			u_int8_t finderinfo[32];
 			bufsize = 32;
 
+			if ((result = hfs_lock(cp, HFS_SHARED_LOCK))) {
+				return (result);
+			}
+			/* Make a copy since we may not export all of it. */
+			bcopy(cp->c_finderinfo, finderinfo, sizeof(finderinfo));
+			hfs_unlock(cp);
+
+			/* Don't expose a symlink's private type/creator. */
+			if (vnode_islnk(vp)) {
+				struct FndrFileInfo *fip;
+
+				fip = (struct FndrFileInfo *)&finderinfo;
+				fip->fdType = 0;
+				fip->fdCreator = 0;
+			}
 			/* If Finder Info is empty then it doesn't exist. */
-			if (bcmp(VTOC(vp)->c_finderinfo, emptyfinfo, sizeof(emptyfinfo)) == 0) {
+			if (bcmp(finderinfo, emptyfinfo, sizeof(emptyfinfo)) == 0) {
 				return (ENOATTR);
 			}
 			if (uio == NULL) {
@@ -132,7 +265,7 @@ hfs_vnop_getxattr(struct vnop_getxattr_args *ap)
 			if (uio_resid(uio) < bufsize)
 				return (ERANGE);
 
-			result = uiomove((caddr_t) &VTOC(vp)->c_finderinfo , bufsize, uio);
+			result = uiomove((caddr_t)&finderinfo , bufsize, uio);
 
 			return (result);
 		}
@@ -140,17 +273,19 @@ hfs_vnop_getxattr(struct vnop_getxattr_args *ap)
 		if (bcmp(ap->a_name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) == 0) {
 			struct vnode *rvp = NULL;
 
-			if ( !vnode_isreg(vp) ) {
+			if ( !S_ISREG(cp->c_mode) ) {
 				return (EPERM);
 			}
-			if ( !RESOURCE_FORK_EXISTS(vp)) {
-				return (ENOATTR);
-			}
-			if ((result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+			if ((result = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
 				return (result);
 			}
-			result = hfs_vgetrsrc(hfsmp, vp, &rvp, vfs_context_proc(ap->a_context));
-			hfs_unlock(VTOC(vp));
+			if ( !RSRC_FORK_EXISTS(cp)) {
+				hfs_unlock(cp);
+				return (ENOATTR);
+			}
+			hfsmp = VTOHFS(vp);
+			result = hfs_vgetrsrc(hfsmp, vp, &rvp, TRUE);
+			hfs_unlock(cp);
 			if (result) {
 				return (result);
 			}
@@ -163,31 +298,46 @@ hfs_vnop_getxattr(struct vnop_getxattr_args *ap)
 			return (result);
 		}
 	}
+	hfsmp = VTOHFS(vp);
 	/*
 	 * Standard HFS only supports native FinderInfo and Resource Forks.
 	 */
 	if (hfsmp->hfs_flags & HFS_STANDARD) {
 		return (EPERM);
 	}
+
+	if ((result = hfs_lock(cp, HFS_SHARED_LOCK))) {
+		return (result);
+	}
 	/* Bail if we don't have any extended attributes. */
 	if ((hfsmp->hfs_attribute_vp == NULL) ||
-	    (VTOC(vp)->c_attr.ca_recflags & kHFSHasAttributesMask) == 0) {
-		return (ENOATTR);
+	    (cp->c_attr.ca_recflags & kHFSHasAttributesMask) == 0) {
+	    	result = ENOATTR;
+		goto exit;
 	}
 	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		result = ENOMEM;
+		goto exit;
+	}
 	bzero(iterator, sizeof(*iterator));
 
 	bufsize = sizeof(HFSPlusAttrData) - 2;
 	if (uio)
 		bufsize += uio_resid(uio);
-	MALLOC(datap, HFSPlusAttrData *, bufsize, M_TEMP, M_WAITOK);
-	btdata.bufferAddress = datap;
+	bufsize = MAX(bufsize, sizeof(HFSPlusAttrRecord));
+	MALLOC(recp, HFSPlusAttrRecord *, bufsize, M_TEMP, M_WAITOK);
+	if (recp == NULL) {
+		result = ENOMEM;
+		goto exit;
+	}
+	btdata.bufferAddress = recp;
 	btdata.itemSize = bufsize;
 	btdata.itemCount = 1;
 
-	result = buildkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
+	result = hfs_buildattrkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
 	if (result)
 		goto exit;	
 
@@ -202,19 +352,123 @@ hfs_vnop_getxattr(struct vnop_getxattr_args *ap)
 		goto exit;
 	}
 
-	*ap->a_size = datap->attrSize;
+	switch (recp->recordType) {
+	case kHFSPlusAttrInlineData:
+		/*
+		 * Sanity check record size. It's not required to have any
+		 * user data, so the minimum size is 2 bytes less that the
+		 * size of HFSPlusAttrData (since HFSPlusAttrData struct
+		 * has 2 bytes set aside for attribute data).
+		 */
+		if (datasize < (sizeof(HFSPlusAttrData) - 2)) {
+			printf("hfs_getxattr: %d,%s invalid record size %d (expecting %lu)\n", 
+				VTOC(vp)->c_fileid, ap->a_name, datasize, sizeof(HFSPlusAttrData));
+			result = ENOATTR;
+			break;
+		}
+		*ap->a_size = recp->attrData.attrSize;
+		if (uio && recp->attrData.attrSize != 0) {
+			if (*ap->a_size > uio_resid(uio))
+				result = ERANGE;
+			else
+				result = uiomove((caddr_t) &recp->attrData.attrData , recp->attrData.attrSize, uio);
+		}
+		break;
 
-	/* Copy out the attribute data. */
-	if (uio) {
-		if (datap->attrSize > uio_resid(uio))
+	case kHFSPlusAttrForkData:
+		if (datasize < sizeof(HFSPlusAttrForkData)) {
+			printf("hfs_getxattr: %d,%s invalid record size %d (expecting %lu)\n", 
+				VTOC(vp)->c_fileid, ap->a_name, datasize, sizeof(HFSPlusAttrForkData));
+			result = ENOATTR;
+			break;
+		}
+		*ap->a_size = recp->forkData.theFork.logicalSize;
+		if (uio == NULL) {
+			break;
+		}
+		if (*ap->a_size > uio_resid(uio)) {
 			result = ERANGE;
-		else
-			result = uiomove((caddr_t) &datap->attrData , datap->attrSize, uio);
+			break;
+		}
+		/* Process overflow extents if necessary. */
+		if (has_overflow_extents(&recp->forkData.theFork)) {
+			HFSPlusExtentDescriptor *extentbuf;
+			HFSPlusExtentDescriptor *extentptr;
+			size_t extentbufsize;
+			u_int32_t totalblocks;
+			u_int32_t blkcnt;
+			u_int32_t attrlen;
+
+			totalblocks = recp->forkData.theFork.totalBlocks;
+			/* Ignore bogus block counts. */
+			if (totalblocks > HFS_MAXATTRBLKS) {
+				result = ERANGE;
+				break;
+			}
+			attrlen = recp->forkData.theFork.logicalSize;
+
+			/* Get a buffer to hold the worst case amount of extents. */
+			extentbufsize = totalblocks * sizeof(HFSPlusExtentDescriptor);
+			extentbufsize = roundup(extentbufsize, sizeof(HFSPlusExtentRecord));
+			MALLOC(extentbuf, HFSPlusExtentDescriptor *, extentbufsize, M_TEMP, M_WAITOK);
+			if (extentbuf == NULL) {
+				result = ENOMEM;
+				break;
+			}
+			bzero(extentbuf, extentbufsize);
+			extentptr = extentbuf;
+
+			/* Grab the first 8 extents. */
+			bcopy(&recp->forkData.theFork.extents[0], extentptr, sizeof(HFSPlusExtentRecord));
+			extentptr += kHFSPlusExtentDensity;
+			blkcnt = count_extent_blocks(totalblocks, recp->forkData.theFork.extents);
+
+			/* Now lookup the overflow extents. */
+			lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_SHARED_LOCK);
+			while (blkcnt < totalblocks) {
+				((HFSPlusAttrKey *)&iterator->key)->startBlock = blkcnt;
+				result = BTSearchRecord(btfile, iterator, &btdata, &datasize, NULL);
+				if (result ||
+				    (recp->recordType != kHFSPlusAttrExtents) ||
+				    (datasize < sizeof(HFSPlusAttrExtents))) {
+					printf("hfs_getxattr: %s missing extents, only %d blks of %d found\n",
+						ap->a_name, blkcnt, totalblocks);
+					result = ENOATTR;
+					break;   /* break from while */
+				}
+				/* Grab the next 8 extents. */
+				bcopy(&recp->overflowExtents.extents[0], extentptr, sizeof(HFSPlusExtentRecord));
+				extentptr += kHFSPlusExtentDensity;
+				blkcnt += count_extent_blocks(totalblocks, recp->overflowExtents.extents);
+			}
+			hfs_systemfile_unlock(hfsmp, lockflags);
+
+			if (blkcnt < totalblocks) {
+				result = ENOATTR;
+			} else {
+				result = read_attr_data(hfsmp, uio, attrlen, extentbuf);
+			}
+			FREE(extentbuf, M_TEMP);
+
+		} else /* No overflow extents. */ {
+			result = read_attr_data(hfsmp, uio, recp->forkData.theFork.logicalSize, recp->forkData.theFork.extents);
+		}
+		break;
+
+	default:
+		result = ENOATTR;
+		break;		
 	}
 exit:
-	FREE(datap, M_TEMP);
-	FREE(iterator, M_TEMP);
+	hfs_unlock(cp);
 
+	if (iterator) {
+		FREE(iterator, M_TEMP);
+	}
+	if (recp) {
+		FREE(recp, M_TEMP);
+	}
+	
 	return MacToVFSError(result);
 }
 
@@ -236,15 +490,21 @@ hfs_vnop_setxattr(struct vnop_setxattr_args *ap)
 */
 {
 	struct vnode *vp = ap->a_vp;
+	struct cnode *cp = NULL;
 	struct hfsmount *hfsmp;
 	uio_t uio = ap->a_uio;
 	struct BTreeIterator * iterator = NULL;
-	struct filefork *btfile;
+	struct filefork *btfile = NULL;
 	size_t attrsize;
 	FSBufferDescriptor btdata;
-	HFSPlusAttrData * datap = NULL;
-	UInt16 datasize;
-	int lockflags;
+	HFSPlusAttrRecord *recp = NULL;
+	HFSPlusExtentDescriptor *extentptr = NULL;
+	HFSPlusAttrRecord attrdata;  /* 90 bytes */
+	void * user_data_ptr = NULL;
+	int started_transaction = 0;
+	int lockflags = 0;
+	int exists;
+	int allocatedblks = 0;
 	int result;
 
 	if (ap->a_name == NULL || ap->a_name[0] == '\0') {
@@ -256,28 +516,83 @@ hfs_vnop_setxattr(struct vnop_setxattr_args *ap)
 	}
 	/* Set the Finder Info. */
 	if (bcmp(ap->a_name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) {
-		attrsize = 32;
+		u_int8_t finderinfo[32];
+		struct FndrFileInfo *fip;
+		void * finderinfo_start;
+		u_int16_t fdFlags;
+
+		attrsize = sizeof(VTOC(vp)->c_finderinfo);
+
+		if (uio_resid(uio) != attrsize) {
+			return (ERANGE);
+		}
+		/* Grab the new Finder Info data. */
+		if ((result = uiomove((caddr_t)&finderinfo , attrsize, uio))) {
+			return (result);
+		}
+		fip = (struct FndrFileInfo *)&finderinfo;
+
+		if ((result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+			return (result);
+		}
+		cp = VTOC(vp);
+
+		/* Symlink's don't have an external type/creator. */
+		if (vnode_islnk(vp)) {
+			/* Skip over type/creator fields. */
+			finderinfo_start = &cp->c_finderinfo[8];
+			attrsize -= 8;
+		} else {
+			finderinfo_start = &cp->c_finderinfo[0];
+			/*
+			 * Don't allow the external setting of
+			 * file type to kHardLinkFileType.
+			 */
+			if (fip->fdType == SWAP_BE32(kHardLinkFileType)) {
+				hfs_unlock(cp);
+				return (EPERM);
+			} 
+		}
 
-		if (bcmp(VTOC(vp)->c_finderinfo, emptyfinfo, sizeof(emptyfinfo))) {
+		if (bcmp(finderinfo_start, emptyfinfo, attrsize)) {
 			/* attr exists and "create" was specified. */
 			if (ap->a_options & XATTR_CREATE) {
+				hfs_unlock(cp);
 				return (EEXIST);
 			}
-		} else {
+		} else /* empty */ {
 			/* attr doesn't exists and "replace" was specified. */
 			if (ap->a_options & XATTR_REPLACE) {
+				hfs_unlock(cp);
 				return (ENOATTR);
 			}
 		}
-		if (uio_resid(uio) != attrsize)
-			return (ERANGE);
+		/* Set the cnode's Finder Info. */
+		if (attrsize == sizeof(cp->c_finderinfo))
+			bcopy(&finderinfo[0], finderinfo_start, attrsize);
+		else
+			bcopy(&finderinfo[8], finderinfo_start, attrsize);
 
-		result = uiomove((caddr_t) &VTOC(vp)->c_finderinfo , attrsize, uio);
-		if (result == 0) {
-			VTOC(vp)->c_touch_chgtime = TRUE;
-			VTOC(vp)->c_flag |= C_MODIFIED;
-			result = hfs_update(vp, FALSE);
-		}
+		/* Updating finderInfo updates change time and modified time */
+		cp->c_touch_chgtime = TRUE;
+		cp->c_flag |= C_MODIFIED;
+
+		/*
+		 * Mirror the invisible bit to the UF_HIDDEN flag.
+		 *
+		 * The fdFlags for files and frFlags for folders are both 8 bytes
+		 * into the userInfo (the first 16 bytes of the Finder Info).  They
+		 * are both 16-bit fields.
+		 */
+		fdFlags = *((u_int16_t *) &cp->c_finderinfo[8]);
+		if (fdFlags & OSSwapHostToBigConstInt16(kFinderInvisibleMask))
+			cp->c_flags |= UF_HIDDEN;
+		else
+			cp->c_flags &= ~UF_HIDDEN;
+
+		result = hfs_update(vp, FALSE);
+
+		hfs_unlock(cp);
 		return (result);
 	}
 	/* Write the Resource Fork. */
@@ -287,25 +602,30 @@ hfs_vnop_setxattr(struct vnop_setxattr_args *ap)
 		if (!vnode_isreg(vp)) {
 			return (EPERM);
 		}
-		if (RESOURCE_FORK_EXISTS(vp)) {
+		if ((result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+			return (result);
+		}
+		cp = VTOC(vp);
+
+		if (RSRC_FORK_EXISTS(cp)) {
 			/* attr exists and "create" was specified. */
 			if (ap->a_options & XATTR_CREATE) {
+				hfs_unlock(cp);
 				return (EEXIST);
 			}
 		} else {
 			/* attr doesn't exists and "replace" was specified. */
 			if (ap->a_options & XATTR_REPLACE) {
+				hfs_unlock(cp);
 				return (ENOATTR);
 			}
 		}
-		if ((result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
-			return (result);
-		}
-		result = hfs_vgetrsrc(hfsmp, vp, &rvp, vfs_context_proc(ap->a_context));
-		hfs_unlock(VTOC(vp));
+		result = hfs_vgetrsrc(hfsmp, vp, &rvp, TRUE);
+		hfs_unlock(cp);
 		if (result) {
 			return (result);
 		}
+		/* VNOP_WRITE will update timestamps accordingly */
 		result = VNOP_WRITE(rvp, uio, 0, ap->a_context);
 		vnode_put(rvp);
 		return (result);
@@ -316,123 +636,279 @@ hfs_vnop_setxattr(struct vnop_setxattr_args *ap)
 	if (hfsmp->hfs_flags & HFS_STANDARD) {
 		return (EPERM);
 	}
-	if (hfsmp->hfs_max_inline_attrsize == 0) {
-		hfsmp->hfs_max_inline_attrsize = getmaxinlineattrsize(hfsmp->hfs_attribute_vp);
-	}
 	attrsize = uio_resid(uio);
-	if (attrsize > hfsmp->hfs_max_inline_attrsize) {
-		/*
-		 * XXX Need to support extent-based attributes XXX
-		 */
+
+	/* Enforce an upper limit. */
+	if (attrsize > HFS_MAXATTRIBUTESIZE) {
 		return (E2BIG);
 	}
-	/* Calculate size of record rounded up to multiple of 2 bytes. */
-	datasize = sizeof(HFSPlusAttrData) - 2 + attrsize + ((attrsize & 1) ? 1 : 0);
-
-	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
-	bzero(iterator, sizeof(*iterator));
 
-	MALLOC(datap, HFSPlusAttrData *, datasize, M_TEMP, M_WAITOK);
-	btdata.bufferAddress = datap;
-	btdata.itemSize = datasize;
-	btdata.itemCount = 1;
-	datap->recordType = kHFSPlusAttrInlineData;
-	datap->reserved[0] = 0;
-	datap->reserved[1] = 0;
-	datap->attrSize = attrsize;
+	/*
+	 * Attempt to copy the users attr data before taking any locks.
+	 */
+	if (attrsize > 0 &&
+	    hfsmp->hfs_max_inline_attrsize != 0 &&
+	    attrsize < hfsmp->hfs_max_inline_attrsize) {
+		MALLOC(user_data_ptr, void *, attrsize, M_TEMP, M_WAITOK);
+		if (user_data_ptr == NULL) {
+			return (ENOMEM);
+		}
 
-	/* Copy in the attribute data. */
-	result = uiomove((caddr_t) &datap->attrData , attrsize, uio);
-	if (result) {
-		goto exit2;
+		result = uiomove((caddr_t)user_data_ptr, attrsize, uio);
+		if (result) {
+			goto exit;
+		}
 	}
-	/* Build a b-tree key. */
-	result = buildkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
+
+	result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK);
 	if (result) {
-		goto exit2;
+		goto exit;
 	}
+	cp = VTOC(vp);
+
 	/* Start a transaction for our changes. */
 	if (hfs_start_transaction(hfsmp) != 0) {
 	    result = EINVAL;
-	    goto exit2;
+	    goto exit;
 	}
+	started_transaction = 1;
 
-	/* once we started the transaction, nobody can compete with us, so make sure this file is still there */
-	struct cnode *cp;
-	cp = VTOC(vp);
-	if (cp->c_flag & C_NOEXISTS) {				 /* this file has already been removed */
+	/*
+	 * Once we started the transaction, nobody can compete
+	 * with us, so make sure this file is still there.
+	 */
+	if (cp->c_flag & C_NOEXISTS) {
 		result = ENOENT;
-		goto exit1;
+		goto exit;
 	}
 
 	/*
 	 * If there isn't an attributes b-tree then create one.
 	 */
 	if (hfsmp->hfs_attribute_vp == NULL) {
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS, HFS_EXCLUSIVE_LOCK);
 		result = hfs_create_attr_btree(hfsmp, ATTRIBUTE_FILE_NODE_SIZE,
 		                               getnodecount(hfsmp, ATTRIBUTE_FILE_NODE_SIZE));
-		hfs_systemfile_unlock(hfsmp, lockflags);
 		if (result) {
-			goto exit1;
+			goto exit;
 		}
 	}
-	btfile = VTOF(hfsmp->hfs_attribute_vp);
+	if (hfsmp->hfs_max_inline_attrsize == 0) {
+		hfsmp->hfs_max_inline_attrsize = getmaxinlineattrsize(hfsmp->hfs_attribute_vp);
+	}
 
+	/* Take exclusive access to the attributes b-tree. */
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
 
-	if (ap->a_options & XATTR_REPLACE) {
-		result = BTReplaceRecord(btfile, iterator, &btdata, datasize);
-		if (result)
-			goto exit0;
-		else
-			goto exit;
+	/* Build the b-tree key. */
+	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		result = ENOMEM;
+		goto exit;
 	}
-
-	/* Insert the attribute. */
-	result = BTInsertRecord(btfile, iterator, &btdata, datasize);
+	bzero(iterator, sizeof(*iterator));
+	result = hfs_buildattrkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
 	if (result) {
-		if (result != btExists) {
-			goto exit0;
+		goto exit;
+	}
+
+	/* Preflight for replace/create semantics. */
+	btfile = VTOF(hfsmp->hfs_attribute_vp);
+	btdata.bufferAddress = &attrdata;
+	btdata.itemSize = sizeof(attrdata);
+	btdata.itemCount = 1;
+	exists = BTSearchRecord(btfile, iterator, &btdata, NULL, NULL) == 0;
+
+	/* Replace requires that the attribute already exists. */
+	if ((ap->a_options & XATTR_REPLACE) && !exists) {
+		result = ENOATTR;
+		goto exit;	
+	}
+	/* Create requires that the attribute doesn't exist. */
+	if ((ap->a_options & XATTR_CREATE) && exists) {
+		result = EEXIST;
+		goto exit;	
+	}
+	/* If it won't fit inline then use extent-based attributes. */
+	if (attrsize > hfsmp->hfs_max_inline_attrsize) {
+		size_t extentbufsize;
+		int blkcnt;
+		int extentblks;
+		u_int32_t *keystartblk;
+		int i;
+
+		/* Check if volume supports extent-based attributes */
+		if ((hfsmp->hfs_flags & HFS_XATTR_EXTENTS) == 0) {
+			result = E2BIG;
+			goto exit;
+		} 
+
+		/* Get some blocks. */
+		blkcnt = howmany(attrsize, hfsmp->blockSize);
+		extentbufsize = blkcnt * sizeof(HFSPlusExtentDescriptor);
+		extentbufsize = roundup(extentbufsize, sizeof(HFSPlusExtentRecord));
+		MALLOC(extentptr, HFSPlusExtentDescriptor *, extentbufsize, M_TEMP, M_WAITOK);
+		if (extentptr == NULL) {
+			result = ENOMEM;
+			goto exit;
+		}
+		bzero(extentptr, extentbufsize);
+		result = alloc_attr_blks(hfsmp, attrsize, extentbufsize, extentptr, &allocatedblks);
+		if (result) {
+			allocatedblks = 0;
+			goto exit;  /* no more space */
+		}
+		/* Copy data into the blocks. */
+		result = write_attr_data(hfsmp, uio, attrsize, extentptr);
+		if (result) {
+			printf("hfs_setxattr: write_attr_data err (%d) %s:%s\n",
+				result, vnode_name(vp) ? vnode_name(vp) : "", ap->a_name);
+			goto exit;
+		}
+
+		/* Now remove any previous attribute. */
+		if (exists) {
+			result = remove_attribute_records(hfsmp, iterator);
+			if (result) {
+				printf("hfs_setxattr: remove_attribute_records err (%d) %s:%s\n",
+					result, vnode_name(vp) ? vnode_name(vp) : "", ap->a_name);
+				goto exit; 
+			}
+		}
+
+		/* Create attribute fork data record. */
+		MALLOC(recp, HFSPlusAttrRecord *, sizeof(HFSPlusAttrRecord), M_TEMP, M_WAITOK);
+		if (recp == NULL) {
+			result = ENOMEM;
+			goto exit;
+		}
+		btdata.bufferAddress = recp;
+		btdata.itemCount = 1;
+		btdata.itemSize = sizeof(HFSPlusAttrForkData);
+
+		recp->recordType = kHFSPlusAttrForkData;
+		recp->forkData.reserved = 0;
+		recp->forkData.theFork.logicalSize = attrsize;
+		recp->forkData.theFork.clumpSize = 0;
+		recp->forkData.theFork.totalBlocks = blkcnt;
+		bcopy(extentptr, recp->forkData.theFork.extents, sizeof(HFSPlusExtentRecord));
+
+		(void) hfs_buildattrkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
+
+		result = BTInsertRecord(btfile, iterator, &btdata, btdata.itemSize);
+		if (result) {
+#if HFS_XATTR_VERBOSE
+			printf("hfs_setxattr: BTInsertRecord err (%d) %s:%s\n",
+			       MacToVFSError(result), vnode_name(vp) ? vnode_name(vp) : "", ap->a_name);
+#endif
+			goto exit; 
+		}
+		extentblks = count_extent_blocks(blkcnt, recp->forkData.theFork.extents);
+		blkcnt -= extentblks;
+		keystartblk = &((HFSPlusAttrKey *)&iterator->key)->startBlock;
+		i = 0;
+
+		/* Create overflow extents as needed. */
+		while (blkcnt > 0) {
+			/* Initialize the key and record. */
+			*keystartblk += (u_int32_t)extentblks;
+			btdata.itemSize = sizeof(HFSPlusAttrExtents);
+			recp->recordType = kHFSPlusAttrExtents;
+			recp->overflowExtents.reserved = 0;
+
+			/* Copy the next set of extents. */
+			i += kHFSPlusExtentDensity;
+			bcopy(&extentptr[i], recp->overflowExtents.extents, sizeof(HFSPlusExtentRecord));
+
+			result = BTInsertRecord(btfile, iterator, &btdata, btdata.itemSize);
+			if (result) {
+				printf("hfs_setxattr: BTInsertRecord err (%d) %s:%s\n",
+					MacToVFSError(result), vnode_name(vp) ? vnode_name(vp) : "", ap->a_name);
+				goto exit;
+			}
+			extentblks = count_extent_blocks(blkcnt, recp->overflowExtents.extents);
+			blkcnt -= extentblks;
+		}
+	} else /* Inline data */ {
+		if (exists) {
+			result = remove_attribute_records(hfsmp, iterator);
+			if (result) {
+				goto exit;
+			}
 		}
 
-		// if it exists and XATTR_CREATE was specified,
-		// the spec says to return EEXIST
-		if (ap->a_options & XATTR_CREATE) {
-			result = EEXIST;
-			goto exit0;
+		/* Calculate size of record rounded up to multiple of 2 bytes. */
+		btdata.itemSize = sizeof(HFSPlusAttrData) - 2 + attrsize + ((attrsize & 1) ? 1 : 0);
+		MALLOC(recp, HFSPlusAttrRecord *, btdata.itemSize, M_TEMP, M_WAITOK);
+		if (recp == NULL) {
+			result = ENOMEM;
+			goto exit;
 		}
-		/* XXX need to account for old size in c_attrblks */
-		result = BTReplaceRecord(btfile, iterator, &btdata, datasize);
+		recp->recordType = kHFSPlusAttrInlineData;
+		recp->attrData.reserved[0] = 0;
+		recp->attrData.reserved[1] = 0;
+		recp->attrData.attrSize = attrsize;
+	
+		/* Copy in the attribute data (if any). */
+		if (attrsize > 0) {
+			if (user_data_ptr)
+				bcopy(user_data_ptr, &recp->attrData.attrData, attrsize);
+			else
+				result = uiomove((caddr_t)&recp->attrData.attrData, attrsize, uio);
+			if (result) {
+				goto exit;
+			}
+		}
+
+		(void) hfs_buildattrkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
+
+		btdata.bufferAddress = recp;
+		btdata.itemCount = 1;
+		result = BTInsertRecord(btfile, iterator, &btdata, btdata.itemSize);
 	}
 exit:
-	(void) BTFlushPath(btfile);
-exit0:
-	hfs_systemfile_unlock(hfsmp, lockflags);
+	if (btfile && started_transaction) {
+		(void) BTFlushPath(btfile);
+	}
+	if (lockflags) {
+		hfs_systemfile_unlock(hfsmp, lockflags);
+	}
 	if (result == 0) {
-		struct cnode * cp;
-
 		cp = VTOC(vp);
+		/* Setting an attribute only updates change time and not 
+		 * modified time of the file.
+		 */
 		cp->c_touch_chgtime = TRUE;
-		if ((cp->c_attr.ca_recflags & kHFSHasAttributesMask) == 0) {
-			cp->c_attr.ca_recflags |= kHFSHasAttributesMask;
-			(void) hfs_update(vp, 0);
+		cp->c_attr.ca_recflags |= kHFSHasAttributesMask;
+		if ((bcmp(ap->a_name, KAUTH_FILESEC_XATTR, sizeof(KAUTH_FILESEC_XATTR)) == 0)) {
+			cp->c_attr.ca_recflags |= kHFSHasSecurityMask;
+		}
+		(void) hfs_update(vp, 0);
+	}
+	if (started_transaction) {
+		if (result && allocatedblks) {
+			free_attr_blks(hfsmp, allocatedblks, extentptr);
 		}
+		hfs_end_transaction(hfsmp);
+	}
+	if (cp) {
+		hfs_unlock(cp);
+	}
+	if (result == 0) {
 		HFS_KNOTE(vp, NOTE_ATTRIB);
 	}
-exit1:
-	/* Finish the transaction of our changes. */
-	hfs_end_transaction(hfsmp);
-exit2:
-	FREE(datap, M_TEMP);
-	FREE(iterator, M_TEMP);
-
-	if (result == btNotFound)
-		result = ENOATTR;
-	else
-		result = MacToVFSError(result);
-
-	return (result);
+	if (user_data_ptr) {
+		FREE(user_data_ptr, M_TEMP);
+	}
+	if (recp) {
+		FREE(recp, M_TEMP);
+	}
+	if (extentptr) {
+		FREE(extentptr, M_TEMP);
+	}
+	if (iterator) {
+		FREE(iterator, M_TEMP);
+	}
+	return (result == btNotFound ? ENOATTR : MacToVFSError(result));
 }
 
 /*
@@ -452,12 +928,9 @@ hfs_vnop_removexattr(struct vnop_removexattr_args *ap)
 */
 {
 	struct vnode *vp = ap->a_vp;
+	struct cnode *cp = VTOC(vp);
 	struct hfsmount *hfsmp;
 	struct BTreeIterator * iterator = NULL;
-	struct filefork *btfile;
-	struct proc *p = vfs_context_proc(ap->a_context);
-	FSBufferDescriptor btdata;
-	HFSPlusAttrData attrdata;
 	int lockflags;
 	int result;
 
@@ -476,26 +949,45 @@ hfs_vnop_removexattr(struct vnop_removexattr_args *ap)
 		if ( !vnode_isreg(vp) ) {
 			return (EPERM);
 		}
-		if ( !RESOURCE_FORK_EXISTS(vp) ) {
-			return (ENOATTR);
-		}
-		if ((result = hfs_lock(VTOC(vp), HFS_EXCLUSIVE_LOCK))) {
+		if ((result = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
 			return (result);
 		}
-		result = hfs_vgetrsrc(hfsmp, vp, &rvp, p);
-		hfs_unlock(VTOC(vp));
+		if ( !RSRC_FORK_EXISTS(cp)) {
+			hfs_unlock(cp);
+			return (ENOATTR);
+		}
+		result = hfs_vgetrsrc(hfsmp, vp, &rvp, TRUE);
+		hfs_unlock(cp);
 		if (result) {
 			return (result);
 		}
+
 		hfs_lock_truncate(VTOC(rvp), TRUE);
 		if ((result = hfs_lock(VTOC(rvp), HFS_EXCLUSIVE_LOCK))) {
-			hfs_unlock_truncate(VTOC(vp));
+			hfs_unlock_truncate(cp, TRUE);
 			vnode_put(rvp);
 			return (result);
 		}
+
+		/* Start a transaction for encapsulating changes in 
+		 * hfs_truncate() and hfs_update()
+		 */
+		if ((result = hfs_start_transaction(hfsmp))) {
+			hfs_unlock_truncate(cp, TRUE);
+			hfs_unlock(cp);
+			vnode_put(rvp);
+			return (result);
+		}
+
 		result = hfs_truncate(rvp, (off_t)0, IO_NDELAY, 0, ap->a_context);
+		if (result == 0) {
+			cp->c_touch_chgtime = TRUE;
+			cp->c_flag |= C_MODIFIED;
+			result = hfs_update(vp, FALSE);
+		}
 
-		hfs_unlock_truncate(VTOC(rvp));
+		hfs_end_transaction(hfsmp);
+		hfs_unlock_truncate(VTOC(rvp), TRUE);
 		hfs_unlock(VTOC(rvp));
 
 		vnode_put(rvp);
@@ -503,10 +995,36 @@ hfs_vnop_removexattr(struct vnop_removexattr_args *ap)
 	}
 	/* Clear out the Finder Info. */
 	if (bcmp(ap->a_name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) {
-		if (bcmp(VTOC(vp)->c_finderinfo, emptyfinfo, sizeof(emptyfinfo)) == 0) {
+		void * finderinfo_start;
+		int finderinfo_size;
+
+		if ((result = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
+			return (result);
+		}
+
+		/* Symlink's don't have an external type/creator. */
+		if (vnode_islnk(vp)) {
+			/* Skip over type/creator fields. */
+			finderinfo_start = &cp->c_finderinfo[8];
+			finderinfo_size = sizeof(cp->c_finderinfo) - 8;
+		} else {
+			finderinfo_start = &cp->c_finderinfo[0];
+			finderinfo_size = sizeof(cp->c_finderinfo);
+		}
+		if (bcmp(finderinfo_start, emptyfinfo, finderinfo_size) == 0) {
+			hfs_unlock(cp);
 			return (ENOATTR);
 		}
-		bzero(VTOC(vp)->c_finderinfo, sizeof(emptyfinfo));
+
+		bzero(finderinfo_start, finderinfo_size);
+
+		/* Updating finderInfo updates change time and modified time */
+		cp->c_touch_chgtime = TRUE;
+		cp->c_flag |= C_MODIFIED;
+		hfs_update(vp, FALSE);
+
+		hfs_unlock(cp);
+
 		return (0);
 	}
 	/*
@@ -518,64 +1036,62 @@ hfs_vnop_removexattr(struct vnop_removexattr_args *ap)
 	if (hfsmp->hfs_attribute_vp == NULL) {
 		return (ENOATTR);
 	}
-	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		return (ENOMEM);
+	}
 	bzero(iterator, sizeof(*iterator));
 
-	if (hfs_start_transaction(hfsmp) != 0) {
-	    result = EINVAL;
-	    goto exit2;
+	if ((result = hfs_lock(cp, HFS_EXCLUSIVE_LOCK))) {
+		return (result);
 	}
 
-	result = buildkey(VTOC(vp)->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
-	if (result)
-		goto exit2;	
-
-	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
+	result = hfs_buildattrkey(cp->c_fileid, ap->a_name, (HFSPlusAttrKey *)&iterator->key);
+	if (result) {
+		goto exit;	
+	}
 
-	btdata.bufferAddress = &attrdata;
-	btdata.itemSize = sizeof(attrdata);
-	btdata.itemCount = 1;
-	result = BTSearchRecord(btfile, iterator, &btdata, NULL, NULL);
-	if (result)
-		goto exit1;	
+	if (hfs_start_transaction(hfsmp) != 0) {
+	    result = EINVAL;
+	    goto exit;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+	
+	result = remove_attribute_records(hfsmp, iterator);
 
-	result = BTDeleteRecord(btfile, iterator);
-	(void) BTFlushPath(btfile);
-exit1:
 	hfs_systemfile_unlock(hfsmp, lockflags);
+
 	if (result == 0) {
-		VTOC(vp)->c_touch_chgtime = TRUE;
-		HFS_KNOTE(vp, NOTE_ATTRIB);
+		cp->c_touch_chgtime = TRUE;
 
-		/* If no more attributes exist, clear the attribute bit */
 		lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_SHARED_LOCK);
-		result = file_attribute_exist(hfsmp, VTOC(vp)->c_fileid);
+
+		/* If no more attributes exist, clear attribute bit */
+		result = file_attribute_exist(hfsmp, cp->c_fileid);
 		if (result == 0) {
-			VTOC(vp)->c_attr.ca_recflags &= ~kHFSHasAttributesMask;
-			VTOC(vp)->c_flag |= C_MODIFIED;
-		} else if (result == EEXIST) {
+			cp->c_attr.ca_recflags &= ~kHFSHasAttributesMask;
+		}
+		if (result == EEXIST) {
 			result = 0;
 		}
+
 		hfs_systemfile_unlock(hfsmp, lockflags);
 
 		/* If ACL was removed, clear security bit */
 		if ((bcmp(ap->a_name, KAUTH_FILESEC_XATTR, sizeof(KAUTH_FILESEC_XATTR)) == 0)) {
-			VTOC(vp)->c_attr.ca_recflags &= ~kHFSHasSecurityMask;
-			VTOC(vp)->c_flag |= C_MODIFIED;
+			cp->c_attr.ca_recflags &= ~kHFSHasSecurityMask;
 		}
-
 		(void) hfs_update(vp, 0);
 	}
-exit2:
-	if (result == btNotFound) {
-		result = ENOATTR;
-	}
-	hfs_end_transaction(hfsmp);
 
+	hfs_end_transaction(hfsmp);
+exit:
+	hfs_unlock(cp);
+	if (result == 0) {
+		HFS_KNOTE(vp, NOTE_ATTRIB);
+	}
 	FREE(iterator, M_TEMP);
-
 	return MacToVFSError(result);
 }
 
@@ -591,7 +1107,7 @@ exit2:
  *	0	- Attribute was not found
  *	(other)	- Other error (such as EIO) 
  */
-static int
+int
 file_attribute_exist(struct hfsmount *hfsmp, uint32_t fileID)
 {
 	HFSPlusAttrKey *key;
@@ -613,7 +1129,7 @@ file_attribute_exist(struct hfsmount *hfsmp, uint32_t fileID)
 	bzero(iterator, sizeof(*iterator));
 	key = (HFSPlusAttrKey *)&iterator->key;
 
-	result = buildkey(fileID, NULL, key);
+	result = hfs_buildattrkey(fileID, NULL, key);
 	if (result) {
 		goto out;
 	}
@@ -628,21 +1144,106 @@ file_attribute_exist(struct hfsmount *hfsmp, uint32_t fileID)
 	/* If no next record was found or fileID for next record did not match,
 	 * no more attributes exist for this fileID
 	 */
-	if ((result && (result == btNotFound)) || (key->fileID != fileID)) {
-		result = 0;	
+	if ((result && (result == btNotFound)) || (key->fileID != fileID)) {
+		result = 0;	
+	} else {
+		result = EEXIST;
+	}
+
+out:
+	if (iterator) {
+		FREE(iterator, M_TEMP);
+	}
+	return result;
+}
+
+
+/*
+ * Remove all the records for a given attribute.
+ *
+ * - Used by hfs_vnop_removexattr, hfs_vnop_setxattr and hfs_removeallattr.
+ * - A transaction must have been started.
+ * - The Attribute b-tree file must be locked exclusive.
+ * - The Allocation Bitmap file must be locked exclusive.
+ * - The iterator key must be initialized.
+ */
+static int
+remove_attribute_records(struct hfsmount *hfsmp, BTreeIterator * iterator)
+{
+	struct filefork *btfile;
+	FSBufferDescriptor btdata;
+	HFSPlusAttrRecord attrdata;  /* 90 bytes */
+	u_int16_t datasize;
+	int result;
+
+	btfile = VTOF(hfsmp->hfs_attribute_vp);
+
+	btdata.bufferAddress = &attrdata;
+	btdata.itemSize = sizeof(attrdata);
+	btdata.itemCount = 1;
+	result = BTSearchRecord(btfile, iterator, &btdata, &datasize, NULL);
+	if (result) {
+		goto exit; /* no records. */
+	}
+	/*
+	 * Free the blocks from extent based attributes.
+	 *
+	 * Note that the block references (btree records) are removed
+	 * before releasing the blocks in the allocation bitmap.
+	 */
+	if (attrdata.recordType == kHFSPlusAttrForkData) {
+		int totalblks;
+		int extentblks;
+		u_int32_t *keystartblk;
+
+#if HFS_XATTR_VERBOSE
+		if (datasize < sizeof(HFSPlusAttrForkData)) {
+			printf("remove_attribute_records: bad record size %d (expecting %d)\n", datasize, sizeof(HFSPlusAttrForkData));
+		}
+#endif
+		totalblks = attrdata.forkData.theFork.totalBlocks;
+
+		/* Process the first 8 extents. */
+		extentblks = count_extent_blocks(totalblks, attrdata.forkData.theFork.extents);
+		if (extentblks > totalblks)
+			panic("remove_attribute_records: corruption...");
+		if (BTDeleteRecord(btfile, iterator) == 0) {
+			free_attr_blks(hfsmp, extentblks, attrdata.forkData.theFork.extents);
+		}
+		totalblks -= extentblks;
+		keystartblk = &((HFSPlusAttrKey *)&iterator->key)->startBlock;
+
+		/* Process any overflow extents. */
+		while (totalblks) {
+			*keystartblk += (u_int32_t)extentblks;
+
+			result = BTSearchRecord(btfile, iterator, &btdata, &datasize, NULL);
+			if (result ||
+			    (attrdata.recordType != kHFSPlusAttrExtents) ||
+			    (datasize < sizeof(HFSPlusAttrExtents))) {
+				printf("remove_attribute_records: BTSearchRecord %d (%d), totalblks %d\n",
+					MacToVFSError(result), attrdata.recordType != kHFSPlusAttrExtents, totalblks);
+				result = ENOATTR;
+				break;   /* break from while */
+			}
+			/* Process the next 8 extents. */
+			extentblks = count_extent_blocks(totalblks, attrdata.overflowExtents.extents);
+			if (extentblks > totalblks)
+				panic("remove_attribute_records: corruption...");
+			if (BTDeleteRecord(btfile, iterator) == 0) {
+				free_attr_blks(hfsmp, extentblks, attrdata.overflowExtents.extents);
+			}
+			totalblks -= extentblks;
+		}
 	} else {
-		result = EEXIST;
-	}
-
-out:
-	if (iterator) {
-		FREE(iterator, M_TEMP);
+		result = BTDeleteRecord(btfile, iterator);
 	}
-	return result;
+	(void) BTFlushPath(btfile);
+exit:
+	return (result == btNotFound ? ENOATTR :  MacToVFSError(result));
 }
 
 
-
 /*
  * Retrieve the list of extended attribute names.
  */
@@ -660,11 +1261,16 @@ hfs_vnop_listxattr(struct vnop_listxattr_args *ap)
 */
 {
 	struct vnode *vp = ap->a_vp;
+	struct cnode *cp = VTOC(vp);
 	struct hfsmount *hfsmp;
 	uio_t uio = ap->a_uio;
 	struct BTreeIterator * iterator = NULL;
 	struct filefork *btfile;
 	struct listattr_callback_state state;
+	void * finderinfo_start;
+	int finderinfo_size;
+	user_addr_t user_start = 0;
+	user_size_t user_len = 0;
 	int lockflags;
 	int result;
 
@@ -674,30 +1280,45 @@ hfs_vnop_listxattr(struct vnop_listxattr_args *ap)
 	hfsmp = VTOHFS(vp);
 	*ap->a_size = 0;
 
-	/* If Finder Info is non-empty then export it. */
-	if (bcmp(VTOC(vp)->c_finderinfo, emptyfinfo, sizeof(emptyfinfo)) != 0) {
+	if ((result = hfs_lock(cp, HFS_SHARED_LOCK))) {
+		return (result);
+	}
+
+	/* Don't expose a symlink's private type/creator. */
+	if (vnode_islnk(vp)) {
+		/* Skip over type/creator fields. */
+		finderinfo_start = &cp->c_finderinfo[8];
+		finderinfo_size = sizeof(cp->c_finderinfo) - 8;
+	} else {
+		finderinfo_start = &cp->c_finderinfo[0];
+		finderinfo_size = sizeof(cp->c_finderinfo);
+	}
+	/* If Finder Info is non-empty then export it's name. */
+	if (bcmp(finderinfo_start, emptyfinfo, finderinfo_size) != 0) {
 		if (uio == NULL) {
 			*ap->a_size += sizeof(XATTR_FINDERINFO_NAME);
 		} else if (uio_resid(uio) < sizeof(XATTR_FINDERINFO_NAME)) {
-			return (ERANGE);
+			result = ERANGE;
+			goto exit;
 		} else {
-			result = uiomove((caddr_t)XATTR_FINDERINFO_NAME,
+			result = uiomove(XATTR_FINDERINFO_NAME,
 			                  sizeof(XATTR_FINDERINFO_NAME), uio);
 			if (result)
-				return (result);
+				goto exit;
 		}
 	}
-	/* If Resource Fork is non-empty then export it. */
-	if (vnode_isreg(vp) && RESOURCE_FORK_EXISTS(vp)) {
+	/* If Resource Fork is non-empty then export it's name. */
+	if (S_ISREG(cp->c_mode) && RSRC_FORK_EXISTS(cp)) {
 		if (uio == NULL) {
 			*ap->a_size += sizeof(XATTR_RESOURCEFORK_NAME);
 		} else if (uio_resid(uio) < sizeof(XATTR_RESOURCEFORK_NAME)) {
-			return (ERANGE);
+			result = ERANGE;
+			goto exit;
 		} else {
-			result = uiomove((caddr_t)XATTR_RESOURCEFORK_NAME,
+			result = uiomove(XATTR_RESOURCEFORK_NAME,
 			                 sizeof(XATTR_RESOURCEFORK_NAME), uio);
 			if (result)
-				return (result);
+				goto exit;
 		}
 	}
 	/*
@@ -705,21 +1326,40 @@ hfs_vnop_listxattr(struct vnop_listxattr_args *ap)
 	 * Return at this point.
 	 */
 	if (hfsmp->hfs_flags & HFS_STANDARD) {
-		return (0);
+		result = 0;
+		goto exit;
 	}
 	/* Bail if we don't have any extended attributes. */
 	if ((hfsmp->hfs_attribute_vp == NULL) ||
-	    (VTOC(vp)->c_attr.ca_recflags & kHFSHasAttributesMask) == 0) {
-		return (0);
+	    (cp->c_attr.ca_recflags & kHFSHasAttributesMask) == 0) {
+		result = 0;
+		goto exit;
 	}
 	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		result = ENOMEM;
+		goto exit;
+	}
 	bzero(iterator, sizeof(*iterator));
-	result = buildkey(VTOC(vp)->c_fileid, NULL, (HFSPlusAttrKey *)&iterator->key);
+	result = hfs_buildattrkey(cp->c_fileid, NULL, (HFSPlusAttrKey *)&iterator->key);
 	if (result)
 		goto exit;	
 
+	/*
+	 * Lock the user's buffer here so that we won't fault on
+	 * it in uiomove while holding the attributes b-tree lock.
+	 */
+	if (uio && uio_isuserspace(uio)) {
+		user_start = uio_curriovbase(uio);
+		user_len = uio_curriovlen(uio);
+
+		if ((result = vslock(user_start, user_len)) != 0) {
+			user_start = 0;
+			goto exit;
+		}
+	}
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_SHARED_LOCK);
 
 	result = BTSearchRecord(btfile, iterator, NULL, NULL, NULL);
@@ -728,7 +1368,7 @@ hfs_vnop_listxattr(struct vnop_listxattr_args *ap)
 		goto exit;
 	}
 
-	state.fileID = VTOC(vp)->c_fileid;
+	state.fileID = cp->c_fileid;
 	state.result = 0;
 	state.uio = uio;
 	state.size = 0;
@@ -742,12 +1382,18 @@ hfs_vnop_listxattr(struct vnop_listxattr_args *ap)
 	if (uio == NULL) {
 		*ap->a_size += state.size;
 	}
-exit:
-	FREE(iterator, M_TEMP);
-	
+
 	if (state.result || result == btNotFound)
 		result = state.result;
 
+exit:
+	if (user_start) {
+		vsunlock(user_start, user_len, TRUE);
+	}
+	FREE(iterator, M_TEMP);
+
+	hfs_unlock(cp);
+	
 	return MacToVFSError(result);
 }
 
@@ -773,8 +1419,9 @@ listattr_callback(const HFSPlusAttrKey *key, __unused const HFSPlusAttrData *dat
 		return (1);	/* continue */
 	}
 
+	/* Convert the attribute name into UTF-8. */
 	result = utf8_encodestr(key->attrName, key->attrNameLen * sizeof(UniChar),
-				attrname, &bytecount, sizeof(attrname), 0, 0);
+				(u_int8_t *)attrname, &bytecount, sizeof(attrname), '/', 0);
 	if (result) {
 		state->result = result;
 		return (0);	/* stop */
@@ -800,77 +1447,95 @@ listattr_callback(const HFSPlusAttrKey *key, __unused const HFSPlusAttrData *dat
 	return (1); /* continue */
 }
 
-
 /*
  * Remove all the attributes from a cnode.
  *
- * A jornal transaction must be already started.
- * Attributes b-Tree must have exclusive lock held.
+ * This function creates/ends its own transaction so that each
+ * attribute is deleted in its own transaction (to avoid having
+ * a transaction grow too large).
+ *
+ * This function takes the necessary locks on the attribute
+ * b-tree file and the allocation (bitmap) file.
  */
 __private_extern__
 int
 hfs_removeallattr(struct hfsmount *hfsmp, u_int32_t fileid)
 {
-	BTreeIterator *next_iterator, *del_iterator;
-	HFSPlusAttrKey *next_key;
+	BTreeIterator *iterator;
+	HFSPlusAttrKey *key;
 	struct filefork *btfile;
-	int result, iter_result;
+	int result, lockflags;
 
 	if (hfsmp->hfs_attribute_vp == NULL) {
 		return (0);
 	}
 	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
-	MALLOC(next_iterator, BTreeIterator *, sizeof(BTreeIterator) * 2, M_TEMP, M_WAITOK);
-	bzero(next_iterator, sizeof(BTreeIterator) * 2);
-	del_iterator = &next_iterator[1];
-	next_key = (HFSPlusAttrKey *)&next_iterator->key;
-
-	/*
-	 * Go to first possible attribute key/record pair
-	 */
-	(void) buildkey(fileid, NULL, next_key);
-	result = BTIterateRecord(btfile, kBTreeNextRecord, next_iterator, NULL, NULL);
-	if (result || next_key->fileID != fileid) {
-		goto exit;
-	}
-	/* Remember iterator of attribute to delete */
-	bcopy(next_iterator, del_iterator, sizeof(BTreeIterator));
+	MALLOC(iterator, BTreeIterator *, sizeof(BTreeIterator), M_TEMP, M_WAITOK);
+	bzero(iterator, sizeof(BTreeIterator));
+	key = (HFSPlusAttrKey *)&iterator->key;
 
 	/* Loop until there are no more attributes for this file id */
 	for(;;) {
-		iter_result = BTIterateRecord(btfile, kBTreeNextRecord, next_iterator, NULL, NULL);
-
-		/* XXX need to free and extents for record types 0x20 and 0x30 */
-		result = BTDeleteRecord(btfile, del_iterator);
-		if (result) {
+		if (hfs_start_transaction(hfsmp) != 0) {
+			result = EINVAL;
 			goto exit;
 		}
-		if (iter_result) {
-			result = iter_result;
-			break;
+
+		/* Lock the attribute b-tree and the allocation (bitmap) files */
+		lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE | SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+
+		/*
+		 * Go to first possible attribute key/record pair
+		 */
+		(void) hfs_buildattrkey(fileid, NULL, key);
+		result = BTIterateRecord(btfile, kBTreeNextRecord, iterator, NULL, NULL);
+		if (result || key->fileID != fileid) {
+			hfs_systemfile_unlock(hfsmp, lockflags);
+			hfs_end_transaction(hfsmp);
+			goto exit;
 		}
-		if (iter_result || next_key->fileID != fileid) {
-			break;  /* end of attributes for this file id */
+		result = remove_attribute_records(hfsmp, iterator);
+
+#if HFS_XATTR_VERBOSE
+		if (result) {
+			printf("hfs_removeallattr: unexpected err %d\n", result);
 		}
-		bcopy(next_iterator, del_iterator, sizeof(BTreeIterator));
+#endif
+		hfs_systemfile_unlock(hfsmp, lockflags);
+		hfs_end_transaction(hfsmp);
 	}
 exit:
-	(void) BTFlushPath(btfile);
+	FREE(iterator, M_TEMP);
+	return (result == btNotFound ? 0: MacToVFSError(result));
+}
 
-	if (result == btNotFound) {
-		result = 0;
+__private_extern__
+void
+hfs_xattr_init(struct hfsmount * hfsmp)
+{
+	/*
+	 * If there isn't an attributes b-tree then create one.
+	 */
+	if (!(hfsmp->hfs_flags & HFS_STANDARD) &&
+	    (hfsmp->hfs_attribute_vp == NULL) &&
+	    !(hfsmp->hfs_flags & HFS_READ_ONLY)) {
+		(void) hfs_create_attr_btree(hfsmp, ATTRIBUTE_FILE_NODE_SIZE,
+		                             getnodecount(hfsmp, ATTRIBUTE_FILE_NODE_SIZE));
 	}
-	FREE(next_iterator, M_TEMP);
-	return (result);
+	if (hfsmp->hfs_attribute_vp)
+		hfsmp->hfs_max_inline_attrsize = getmaxinlineattrsize(hfsmp->hfs_attribute_vp);
 }
 
 /*
- * Enable/Disable extended security (ACLs).
+ * Enable/Disable volume attributes stored as EA for root file system.
+ * Supported attributes are - 
+ *	1. ACLs
+ *	2. Extent-based Extended Attributes 
  */
 __private_extern__
 int
-hfs_setextendedsecurity(struct hfsmount *hfsmp, int state)
+hfs_set_volxattr(struct hfsmount *hfsmp, unsigned int xattrtype, int state)
 {
 	struct BTreeIterator * iterator = NULL;
 	struct filefork *btfile;
@@ -881,32 +1546,44 @@ hfs_setextendedsecurity(struct hfsmount *hfsmp, int state)
 		return (ENOTSUP);
 	}
 
+	/*
+	 * If there isn't an attributes b-tree then create one.
+	 */
+	if (hfsmp->hfs_attribute_vp == NULL) {
+		result = hfs_create_attr_btree(hfsmp, ATTRIBUTE_FILE_NODE_SIZE,
+		                               getnodecount(hfsmp, ATTRIBUTE_FILE_NODE_SIZE));
+		if (result) {
+			return (result);
+		}
+	}
+
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		return (ENOMEM);
+	} 
 	bzero(iterator, sizeof(*iterator));
 
 	/*
 	 * Build a b-tree key.
 	 * We use the root's parent id (1) to hold this volume attribute.
 	 */
-	(void) buildkey(kHFSRootParentID, XATTR_EXTENDEDSECURITY_NAME,
-	                  (HFSPlusAttrKey *)&iterator->key);
+	if (xattrtype == HFS_SETACLSTATE) {
+		/* ACL */
+		(void) hfs_buildattrkey(kHFSRootParentID, XATTR_EXTENDEDSECURITY_NAME,
+		                      (HFSPlusAttrKey *)&iterator->key);
+	} else if (xattrtype == HFS_SET_XATTREXTENTS_STATE) {
+		/* Extent-based extended attributes */
+		(void) hfs_buildattrkey(kHFSRootParentID, XATTR_XATTREXTENTS_NAME,
+		                      (HFSPlusAttrKey *)&iterator->key);
+	} else {
+		result = EINVAL;
+		goto exit;
+	}
 
 	/* Start a transaction for our changes. */
 	if (hfs_start_transaction(hfsmp) != 0) {
 		result = EINVAL;
-		goto exit2;
-	}
-	/*
-	 * If there isn't an attributes b-tree then create one.
-	 */
-	if (hfsmp->hfs_attribute_vp == NULL) {
-		lockflags = hfs_systemfile_lock(hfsmp, SFL_EXTENTS, HFS_EXCLUSIVE_LOCK);
-		result = hfs_create_attr_btree(hfsmp, ATTRIBUTE_FILE_NODE_SIZE,
-		                               getnodecount(hfsmp, ATTRIBUTE_FILE_NODE_SIZE));
-		hfs_systemfile_unlock(hfsmp, lockflags);
-		if (result) {
-			goto exit1;
-		}
+		goto exit;
 	}
 	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
@@ -920,7 +1597,7 @@ hfs_setextendedsecurity(struct hfsmount *hfsmp, int state)
 	} else {
 		FSBufferDescriptor btdata;
 		HFSPlusAttrData attrdata;
-		UInt16 datasize;
+		u_int16_t datasize;
 
 		datasize = sizeof(attrdata);
 		btdata.bufferAddress = &attrdata;
@@ -941,30 +1618,45 @@ hfs_setextendedsecurity(struct hfsmount *hfsmp, int state)
 	(void) BTFlushPath(btfile);
 
 	hfs_systemfile_unlock(hfsmp, lockflags);
-exit1:
+
 	/* Finish the transaction of our changes. */
 	hfs_end_transaction(hfsmp);
-exit2:
-	FREE(iterator, M_TEMP);
-
+exit:
+	if (iterator) {
+		FREE(iterator, M_TEMP);
+	}
 	if (result == 0) {
-		if (state == 0)
-			vfs_clearextendedsecurity(HFSTOVFS(hfsmp));
-		else
-			vfs_setextendedsecurity(HFSTOVFS(hfsmp));
-		printf("hfs: %s extended security on %s\n",
-		       state == 0 ? "disabling" : "enabling", hfsmp->vcbVN);
+		if (xattrtype == HFS_SETACLSTATE) {
+			if (state == 0) {
+				vfs_clearextendedsecurity(HFSTOVFS(hfsmp));
+			} else {
+				vfs_setextendedsecurity(HFSTOVFS(hfsmp));
+			}
+		} else { 
+			/* HFS_SET_XATTREXTENTS_STATE */
+			HFS_MOUNT_LOCK(hfsmp, TRUE);
+			if (state == 0) {
+				hfsmp->hfs_flags &= ~HFS_XATTR_EXTENTS; 
+			} else {
+				hfsmp->hfs_flags |= HFS_XATTR_EXTENTS; 
+			}
+			HFS_MOUNT_UNLOCK(hfsmp, TRUE); 
+		}
 	}
 
 	return MacToVFSError(result);
 }
 
-/*
- * Check for extended security (ACLs).
+
+ /*
+ * Check for volume attributes stored as EA for root file system.
+ * Supported attributes are - 
+ *	1. ACLs
+ *	2. Extent-based Extended Attributes 
  */
 __private_extern__
 void
-hfs_checkextendedsecurity(struct hfsmount *hfsmp)
+hfs_check_volxattr(struct hfsmount *hfsmp, unsigned int xattrtype)
 {
 	struct BTreeIterator * iterator;
 	struct filefork *btfile;
@@ -977,15 +1669,24 @@ hfs_checkextendedsecurity(struct hfsmount *hfsmp)
 	}
 
 	MALLOC(iterator, BTreeIterator *, sizeof(*iterator), M_TEMP, M_WAITOK);
+	if (iterator == NULL) {
+		return;
+	}
 	bzero(iterator, sizeof(*iterator));
 
 	/*
 	 * Build a b-tree key.
 	 * We use the root's parent id (1) to hold this volume attribute.
 	 */
-	(void) buildkey(kHFSRootParentID, XATTR_EXTENDEDSECURITY_NAME,
-	                  (HFSPlusAttrKey *)&iterator->key);
-
+	if (xattrtype == HFS_SETACLSTATE) {
+		/* ACLs */
+		(void) hfs_buildattrkey(kHFSRootParentID, XATTR_EXTENDEDSECURITY_NAME,
+	        	              (HFSPlusAttrKey *)&iterator->key);
+	} else {
+		/* Extent-based extended attributes */
+		(void) hfs_buildattrkey(kHFSRootParentID, XATTR_XATTREXTENTS_NAME,
+	        	              (HFSPlusAttrKey *)&iterator->key);
+	}
 	btfile = VTOF(hfsmp->hfs_attribute_vp);
 
 	lockflags = hfs_systemfile_lock(hfsmp, SFL_ATTRIBUTE, HFS_EXCLUSIVE_LOCK);
@@ -997,8 +1698,13 @@ hfs_checkextendedsecurity(struct hfsmount *hfsmp)
 	FREE(iterator, M_TEMP);
 
 	if (result == 0) {
-		vfs_setextendedsecurity(HFSTOVFS(hfsmp));
-		printf("hfs mount: enabling extended security on %s\n", hfsmp->vcbVN);
+		if (xattrtype == HFS_SETACLSTATE) {
+			vfs_setextendedsecurity(HFSTOVFS(hfsmp));
+		} else {
+			HFS_MOUNT_LOCK(hfsmp, TRUE);
+			hfsmp->hfs_flags |= HFS_XATTR_EXTENTS; 
+			HFS_MOUNT_UNLOCK(hfsmp, TRUE); 
+		}
 	}
 }
 
@@ -1071,10 +1777,11 @@ hfs_attrkeycompare(HFSPlusAttrKey *searchKey, HFSPlusAttrKey *trialKey)
 
 
 /*
- * buildkey - build an Attribute b-tree key
+ * hfs_buildattrkey - build an Attribute b-tree key
  */
-static int
-buildkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key)
+__private_extern__
+int
+hfs_buildattrkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key)
 {
 	int result = 0;
 	size_t unicodeBytes = 0;
@@ -1083,7 +1790,7 @@ buildkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key)
 		/*
 		 * Convert filename from UTF-8 into Unicode
 		 */	
-		result = utf8_decodestr(attrname, strlen(attrname), key->attrName,
+		result = utf8_decodestr((const u_int8_t *)attrname, strlen(attrname), key->attrName,
 					&unicodeBytes, sizeof(key->attrName), 0, 0);
 		if (result) {
 			if (result != ENAMETOOLONG)
@@ -1109,22 +1816,24 @@ buildkey(u_int32_t fileID, const char *attrname, HFSPlusAttrKey *key)
 static int
 getnodecount(struct hfsmount *hfsmp, size_t nodesize)
 {
-	int avedatasize;
-	int recpernode;
-	int count;
+	u_int64_t freebytes;
+	u_int64_t calcbytes;
 
-	avedatasize = sizeof(u_int16_t);  /* index slot */
-	avedatasize += kHFSPlusAttrKeyMinimumLength + HFS_AVERAGE_NAME_SIZE * sizeof(u_int16_t);
-	avedatasize += sizeof(HFSPlusAttrData) + 32;
-
-	recpernode = (nodesize - sizeof(BTNodeDescriptor)) / avedatasize;
+	/*
+	 * 10.4: Scale base on current catalog file size (20 %) up to 20 MB.
+	 * 10.5: Attempt to be as big as the catalog clump size.
+	 *
+	 * Use no more than 10 % of the remaining free space.
+	 */
+	freebytes = (u_int64_t)hfs_freeblks(hfsmp, 0) * (u_int64_t)hfsmp->blockSize;
 
-	count = (hfsmp->hfs_filecount + hfsmp->hfs_dircount) / 8;
-	count /= recpernode;
+	calcbytes = MIN(hfsmp->hfs_catalog_cp->c_datafork->ff_size / 5, 20 * 1024 * 1024);
 
-	/* XXX should also consider volume size XXX */
+	calcbytes = MAX(calcbytes, hfsmp->hfs_catalog_cp->c_datafork->ff_clumpsize);
+	
+	calcbytes = MIN(calcbytes, freebytes / 10);
 
-	return (MAX(count, (int)(1024 * 1024) / (int)nodesize));
+	return (MAX(2, (int)(calcbytes / nodesize)));
 }
 
 
@@ -1148,7 +1857,7 @@ getmaxinlineattrsize(struct vnode * attrvp)
 	}
 	maxsize = nodesize;
 	maxsize -= sizeof(BTNodeDescriptor);     /* minus node descriptor */
-	maxsize -= 3 * sizeof(UInt16);           /* minus 3 index slots */
+	maxsize -= 3 * sizeof(u_int16_t);        /* minus 3 index slots */
 	maxsize /= 2;                            /* 2 key/rec pairs minumum */
 	maxsize -= sizeof(HFSPlusAttrKey);       /* minus maximum key size */
 	maxsize -= sizeof(HFSPlusAttrData) - 2;  /* minus data header */
@@ -1157,4 +1866,334 @@ getmaxinlineattrsize(struct vnode * attrvp)
 	return (maxsize);
 }
 
+/*
+ * Get a referenced vnode for attribute data I/O.
+ */
+static int
+get_attr_data_vnode(struct hfsmount *hfsmp, vnode_t *vpp)
+{
+	vnode_t vp;
+	int result = 0;
+
+	vp = hfsmp->hfs_attrdata_vp;
+	if (vp == NULLVP) {
+		struct cat_desc cat_desc;
+		struct cat_attr cat_attr;
+		struct cat_fork cat_fork;
+
+		/* We don't tag it as a system file since we intend to use cluster I/O. */
+		bzero(&cat_desc, sizeof(cat_desc));
+		cat_desc.cd_parentcnid = kHFSRootParentID;
+		cat_desc.cd_nameptr = (const u_int8_t *)hfs_attrdatafilename;
+		cat_desc.cd_namelen = strlen(hfs_attrdatafilename);
+		cat_desc.cd_cnid = kHFSAttributeDataFileID;
+
+		bzero(&cat_attr, sizeof(cat_attr));
+		cat_attr.ca_linkcount = 1;
+		cat_attr.ca_mode = S_IFREG;
+		cat_attr.ca_fileid = cat_desc.cd_cnid;
+		cat_attr.ca_blocks = hfsmp->totalBlocks;
+
+		/*
+		 * The attribute data file is a virtual file that spans the
+		 * entire file system space.
+		 *
+		 * Each extent-based attribute occupies a unique portion of
+		 * in this virtual file.  The cluster I/O is done using actual
+		 * allocation block offsets so no additional mapping is needed
+		 * for the VNOP_BLOCKMAP call.
+		 *
+		 * This approach allows the attribute data to be cached without
+		 * incurring the high cost of using a separate vnode per attribute.
+		 *
+		 * Since we need to acquire the attribute b-tree file lock anyways,
+		 * the virtual file doesn't introduce any additional serialization.
+		 */
+		bzero(&cat_fork, sizeof(cat_fork));
+		cat_fork.cf_size = (u_int64_t)hfsmp->totalBlocks * (u_int64_t)hfsmp->blockSize;
+		cat_fork.cf_blocks = hfsmp->totalBlocks;
+		cat_fork.cf_extents[0].startBlock = 0;
+		cat_fork.cf_extents[0].blockCount = cat_fork.cf_blocks;
+	
+		result = hfs_getnewvnode(hfsmp, NULL, NULL, &cat_desc, 0, &cat_attr, &cat_fork, &vp);
+		if (result == 0) {
+			HFS_MOUNT_LOCK(hfsmp, 1);
+			/* Check if someone raced us for creating this vnode. */
+			if (hfsmp->hfs_attrdata_vp != NULLVP) {
+				HFS_MOUNT_UNLOCK(hfsmp, 1);
+				vnode_put(vp);
+				vnode_recycle(vp);
+				vp = hfsmp->hfs_attrdata_vp;
+			} else {
+				hfsmp->hfs_attrdata_vp = vp;
+				HFS_MOUNT_UNLOCK(hfsmp, 1);
+				/* Keep a reference on this vnode until unmount */
+				vnode_ref_ext(vp, O_EVTONLY);
+				hfs_unlock(VTOC(vp));
+			}
+		}
+	} else {
+		if ((result = vnode_get(vp)))
+			vp = NULLVP;
+	}
+	*vpp = vp;
+	return (result);
+}
+
+/*
+ * Read an extent based attribute.
+ */
+static int
+read_attr_data(struct hfsmount *hfsmp, uio_t uio, size_t datasize, HFSPlusExtentDescriptor *extents)
+{
+	vnode_t evp = NULLVP;
+	int bufsize;
+	int iosize;
+	int attrsize;
+	int blksize;
+	int i;
+	int result = 0;
+
+	if ((result = get_attr_data_vnode(hfsmp, &evp))) {
+		return (result);
+	}
+	hfs_lock_truncate(VTOC(evp), 0);
+
+	bufsize = (int)uio_resid(uio);
+	attrsize = (int)datasize;
+	blksize = (int)hfsmp->blockSize;
+
+	/*
+	 * Read the attribute data one extent at a time.
+	 * For the typical case there is only one extent.
+	 */
+	for (i = 0; (attrsize > 0) && (bufsize > 0) && (extents[i].startBlock != 0); ++i) {
+		iosize = (int)extents[i].blockCount * blksize;
+		iosize = MIN(iosize, attrsize);
+		iosize = MIN(iosize, bufsize);
+		uio_setresid(uio, iosize);
+		uio_setoffset(uio, (u_int64_t)extents[i].startBlock * (u_int64_t)blksize);
+
+		result = cluster_read(evp, uio, VTOF(evp)->ff_size, IO_SYNC | IO_UNIT);
+
+#if HFS_XATTR_VERBOSE
+		printf("read_attr_data: cr iosize %d [%d, %d] (%d)\n",
+			iosize, extents[i].startBlock, extents[i].blockCount, result);
+#endif
+		if (result)
+			break;
+		attrsize -= iosize;
+		bufsize -= iosize;
+	}
+	uio_setresid(uio, bufsize);
+	uio_setoffset(uio, datasize);
+
+	hfs_unlock_truncate(VTOC(evp), 0);
+	vnode_put(evp);
+	return (result);
+}
+
+/*
+ * Write an extent based attribute.
+ */
+static int
+write_attr_data(struct hfsmount *hfsmp, uio_t uio, size_t datasize, HFSPlusExtentDescriptor *extents)
+{
+	vnode_t evp = NULLVP;
+	off_t filesize;
+	int bufsize;
+	int attrsize;
+	int iosize;
+	int blksize;
+	int i;
+	int result = 0;
+
+	/* Get exclusive use of attribute data vnode. */
+	if ((result = get_attr_data_vnode(hfsmp, &evp))) {
+		return (result);
+	}
+	hfs_lock_truncate(VTOC(evp), 0);
+
+	bufsize = uio_resid(uio);
+	attrsize = (int) datasize;
+	blksize = (int) hfsmp->blockSize;
+	filesize = VTOF(evp)->ff_size;
+
+	/*
+	 * Write the attribute data one extent at a time.
+	 */
+	for (i = 0; (attrsize > 0) && (bufsize > 0) && (extents[i].startBlock != 0); ++i) {
+		iosize = (int)extents[i].blockCount * blksize;
+		iosize = MIN(iosize, attrsize);
+		iosize = MIN(iosize, bufsize);
+		uio_setresid(uio, iosize);
+		uio_setoffset(uio, (u_int64_t)extents[i].startBlock * (u_int64_t)blksize);
+
+		result = cluster_write(evp, uio, filesize, filesize, filesize,
+		                       (off_t) 0, IO_SYNC | IO_UNIT);
+#if HFS_XATTR_VERBOSE
+		printf("write_attr_data: cw iosize %d [%d, %d] (%d)\n",
+			iosize, extents[i].startBlock, extents[i].blockCount, result);
+#endif
+		if (result)
+			break;
+		attrsize -= iosize;
+		bufsize -= iosize;
+	}
+	uio_setresid(uio, bufsize);
+	uio_setoffset(uio, datasize);
+
+	hfs_unlock_truncate(VTOC(evp), 0);
+	vnode_put(evp);
+	return (result);
+}
+
+/*
+ * Allocate blocks for an extent based attribute.
+ */
+static int
+alloc_attr_blks(struct hfsmount *hfsmp, size_t attrsize, size_t extentbufsize, HFSPlusExtentDescriptor *extents, int *blocks)
+{
+	int blkcnt;
+	int startblk;
+	int lockflags;
+	int i;
+	int maxextents;
+	int result = 0;
+
+	startblk = hfsmp->hfs_metazone_end;
+	blkcnt = howmany(attrsize, hfsmp->blockSize);
+	if (blkcnt > (int)hfs_freeblks(hfsmp, 0)) {
+		return (ENOSPC);
+	}
+	*blocks = blkcnt;
+	maxextents = extentbufsize / sizeof(HFSPlusExtentDescriptor);
+
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+
+	for (i = 0; (blkcnt > 0) && (i < maxextents); i++) {
+		result = BlockAllocate(hfsmp, startblk, blkcnt, blkcnt, 0, 0,
+				       &extents[i].startBlock, &extents[i].blockCount);
+#if HFS_XATTR_VERBOSE
+		printf("alloc_attr_blks: BA blkcnt %d [%d, %d] (%d)\n",
+			blkcnt, extents[i].startBlock, extents[i].blockCount, result);
+#endif
+		if (result) {
+			extents[i].startBlock = 0;
+			extents[i].blockCount = 0;
+			break;
+		}
+		blkcnt -= extents[i].blockCount;
+		startblk = extents[i].startBlock + extents[i].blockCount;
+	}
+	/*
+	 * If it didn't fit in the extents buffer then bail.
+	 */
+	if (blkcnt) {
+		result = ENOSPC;
+
+#if HFS_XATTR_VERBOSE
+		printf("alloc_attr_blks: unexpected failure, %d blocks unallocated\n", blkcnt);
+#endif
+		for (; i <= 0; i--) {
+			if ((blkcnt = extents[i].blockCount) != 0) {
+				(void) BlockDeallocate(hfsmp, extents[i].startBlock, blkcnt);
+				extents[i].startBlock = 0;
+				extents[i].blockCount = 0;
+		    }
+		}
+	}
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	return MacToVFSError(result);
+}
+
+/*
+ * Release blocks from an extent based attribute.
+ */
+static void
+free_attr_blks(struct hfsmount *hfsmp, int blkcnt, HFSPlusExtentDescriptor *extents)
+{
+	vnode_t evp = NULLVP;
+	int remblks = blkcnt;
+	int lockflags;
+	int i;
+
+	if (get_attr_data_vnode(hfsmp, &evp) != 0) {
+		evp = NULLVP;
+	}
+	lockflags = hfs_systemfile_lock(hfsmp, SFL_BITMAP, HFS_EXCLUSIVE_LOCK);
+
+	for (i = 0; (remblks > 0) && (extents[i].blockCount != 0); i++) {
+		if (extents[i].blockCount > (u_int32_t)blkcnt) {
+#if HFS_XATTR_VERBOSE
+			printf("free_attr_blks: skipping bad extent [%d, %d]\n",
+				extents[i].startBlock, extents[i].blockCount);
+#endif
+			extents[i].blockCount = 0;
+			continue;
+		}
+		if (extents[i].startBlock == 0) {
+			break;
+		}
+		(void)BlockDeallocate(hfsmp, extents[i].startBlock, extents[i].blockCount);
+		extents[i].startBlock = 0;
+		extents[i].blockCount = 0;
+		remblks -= extents[i].blockCount;
+
+#if HFS_XATTR_VERBOSE
+		printf("free_attr_blks: BlockDeallocate [%d, %d]\n",
+		       extents[i].startBlock, extents[i].blockCount);
+#endif
+		/* Discard any resident pages for this block range. */
+		if (evp) {
+			off_t  start, end;
+
+			start = (u_int64_t)extents[i].startBlock * (u_int64_t)hfsmp->blockSize;
+			end = start + (u_int64_t)extents[i].blockCount * (u_int64_t)hfsmp->blockSize;
+			(void) ubc_msync(hfsmp->hfs_attrdata_vp, start, end, &start, UBC_INVALIDATE);
+		}
+	}
+
+	hfs_systemfile_unlock(hfsmp, lockflags);
+	if (evp) {
+		vnode_put(evp);
+	}
+}
+
+static int
+has_overflow_extents(HFSPlusForkData *forkdata)
+{
+	u_int32_t blocks;
+
+	if (forkdata->extents[7].blockCount == 0)
+		return (0);
+
+	blocks = forkdata->extents[0].blockCount +
+		 forkdata->extents[1].blockCount +
+		 forkdata->extents[2].blockCount +
+		 forkdata->extents[3].blockCount +
+		 forkdata->extents[4].blockCount +
+		 forkdata->extents[5].blockCount +
+		 forkdata->extents[6].blockCount +
+		 forkdata->extents[7].blockCount;	
+
+	return (forkdata->totalBlocks > blocks);
+}
 
+static int
+count_extent_blocks(int maxblks, HFSPlusExtentRecord extents)
+{
+	int blocks;
+	int i;
+
+	for (i = 0, blocks = 0; i < kHFSPlusExtentDensity; ++i) {
+		/* Ignore obvious bogus extents. */
+		if (extents[i].blockCount > (u_int32_t)maxblks)
+			continue;
+		if (extents[i].startBlock == 0 || extents[i].blockCount == 0)
+			break;
+		blocks += extents[i].blockCount;
+	}
+	return (blocks);
+}
diff --git a/bsd/hfs/hfscommon/BTree/BTree.c b/bsd/hfs/hfscommon/BTree/BTree.c
index 9920c8742..2ee3159e3 100644
--- a/bsd/hfs/hfscommon/BTree/BTree.c
+++ b/bsd/hfs/hfscommon/BTree/BTree.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTree.c
@@ -145,6 +151,7 @@
 */
 
 #include "../headers/BTreesPrivate.h"
+#include "../../hfs_btreeio.h"
 
 /* 
  * The amount that the BTree header leaf count can be wrong before we assume
@@ -152,12 +159,6 @@
  */
 #define	kNumLeafRecSlack 10		
 
-/* BTree accessor routines */
-extern OSStatus GetBTreeBlock(FileReference vp, UInt32 blockNum, GetBlockOptions options, BlockDescriptor *block);
-extern OSStatus SetBTreeBlockSize(FileReference vp, ByteCount blockSize, ItemCount minBlockCount);
-extern OSStatus ExtendBTreeFile(FileReference vp, FSSize minEOF, FSSize maxEOF);
-extern OSStatus ReleaseBTreeBlock(FileReference vp, BlockDescPtr blockPtr, ReleaseBlockOptions options);
-
 //////////////////////////////////// Globals ////////////////////////////////////
 
 
@@ -456,29 +457,39 @@ Result:		noErr			- success, record contains copy of record found
 OSStatus	BTSearchRecord		(FCB						*filePtr,
 								 BTreeIterator				*searchIterator,
 								 FSBufferDescriptor			*record,
-								 UInt16						*recordLen,
+								 u_int16_t					*recordLen,
 								 BTreeIterator				*resultIterator )
 {
 	OSStatus				err;
 	BTreeControlBlockPtr	btreePtr;
 	TreePathTable			treePathTable;
-	UInt32					nodeNum;
+	u_int32_t				nodeNum;
 	BlockDescriptor			node;
-	UInt16					index;
+	u_int16_t				index;
 	BTreeKeyPtr				keyPtr;
 	RecordPtr				recordPtr;
-	UInt16					len;
+	u_int16_t				len;
 	Boolean					foundRecord;
 	Boolean					validHint;
 
-	if (filePtr == nil)									return	paramErr;
-	if (searchIterator == nil)							return	paramErr;
+	if (filePtr == nil) 
+	{
+		return	paramErr;
+	}
+
+	if (searchIterator == nil) 
+	{
+		return	paramErr;
+	}
 
 	node.buffer = nil;
 	node.blockHeader = nil;
 
 	btreePtr = (BTreeControlBlockPtr) filePtr->fcbBTCBPtr;
-	if (btreePtr == nil)								return	fsBTInvalidFileErr;
+	if (btreePtr == nil) 
+	{
+		return	fsBTInvalidFileErr;
+	}
 
 	REQUIRE_FILE_LOCK(btreePtr->fileRefNum, true);
 
@@ -527,9 +538,13 @@ OSStatus	BTSearchRecord		(FCB						*filePtr,
 		err = SearchTree ( btreePtr, &searchIterator->key, treePathTable, &nodeNum, &node, &index);
 		switch (err)
 		{
-			case noErr:			foundRecord = true;				break;
-			case fsBTRecordNotFoundErr:									break;
-			default:				goto ErrorExit;
+			case noErr:			
+				foundRecord = true;				
+				break;
+			case fsBTRecordNotFoundErr:
+				break;
+			default:
+				goto ErrorExit;
 		}
 	}
 
@@ -560,9 +575,11 @@ OSStatus	BTSearchRecord		(FCB						*filePtr,
 
 	if (resultIterator != nil)
 	{
-		resultIterator->hint.writeCount	= btreePtr->writeCount;
-		resultIterator->hint.nodeNum = nodeNum;
-		resultIterator->hint.index = index;
+		if (foundRecord) {
+			resultIterator->hint.writeCount	= btreePtr->writeCount;
+			resultIterator->hint.nodeNum = nodeNum;
+			resultIterator->hint.index = index;
+		}
 #if DEBUG_BUILD
 		resultIterator->hint.reserved1 = 0;
 		resultIterator->hint.reserved2 = 0;
@@ -633,19 +650,19 @@ OSStatus	BTIterateRecord		(FCB						*filePtr,
 								 BTreeIterationOperation	 operation,
 								 BTreeIterator				*iterator,
 								 FSBufferDescriptor			*record,
-								 UInt16						*recordLen )
+								 u_int16_t					*recordLen )
 {
 	OSStatus					err;
 	BTreeControlBlockPtr		btreePtr;
 	BTreeKeyPtr					keyPtr;
 	RecordPtr					recordPtr;
-	UInt16						len;
+	u_int16_t					len;
 
 	Boolean						foundRecord;
-	UInt32						nodeNum;
+	u_int32_t					nodeNum;
 
 	BlockDescriptor				left,		node,		right;
-	UInt16						index;
+	u_int16_t					index;
 
 
 	////////////////////////// Priliminary Checks ///////////////////////////////
@@ -704,7 +721,8 @@ OSStatus	BTIterateRecord		(FCB						*filePtr,
 			M_ExitOnError (err);
 
 			err = fsBTInvalidNodeErr;
-			MARK_VOLUMEDAMAGED(filePtr);
+			printf ("hfs: BTIterateRecord() found invalid btree node on volume %s\n", FCBTOVCB(filePtr)->vcbVN);
+			hfs_mark_volume_inconsistent(FCBTOVCB(filePtr));
 			goto ErrorExit;
 		}
 
@@ -717,6 +735,7 @@ OSStatus	BTIterateRecord		(FCB						*filePtr,
 
 	//////////////////////// Find Iterator Position /////////////////////////////
 
+	// Not called for (operation == kBTreeFirstRecord || operation == kBTreeLastRecord)
 	err = FindIteratorPosition (btreePtr, iterator,
 								&left, &node, &right, &nodeNum, &index, &foundRecord);
 	M_ExitOnError (err);
@@ -737,8 +756,19 @@ OSStatus	BTIterateRecord		(FCB						*filePtr,
 				nodeNum = ((NodeDescPtr) node.buffer)->bLink;
 				if ( nodeNum > 0)
 				{
+					// BTree nodes are always grabbed in left to right order.  
+					// Therefore release the current node before looking up the 
+					// left node.
+					err = ReleaseNode(btreePtr, &node);
+					M_ExitOnError(err);
+
+					// Look up the left node 
 					err = GetNode (btreePtr, nodeNum, &left);
 					M_ExitOnError (err);
+
+					// Look up the current node again
+					err = GetRightSiblingNode (btreePtr, left.buffer, &node);
+					M_ExitOnError (err);
 				} else {
 					err = fsBTStartOfIterationErr;
 					goto ErrorExit;
@@ -859,7 +889,8 @@ CopyData:
 		if (iterator->hitCount > iterator->maxLeafRecs + kNumLeafRecSlack)
 		{
 			err = fsBTInvalidNodeErr;
-			MARK_VOLUMEDAMAGED(filePtr);
+			printf ("hfs: BTIterateRecord() found invalid btree node on volume %s\n", FCBTOVCB(filePtr)->vcbVN);
+			hfs_mark_volume_inconsistent(FCBTOVCB(filePtr));
 			goto ErrorExit;
 		}
 #endif
@@ -943,11 +974,11 @@ BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation, BTreeIterator
 	BTreeControlBlockPtr	btreePtr;
 	BTreeKeyPtr		keyPtr;
 	RecordPtr		recordPtr;
-	UInt16			len;
+	u_int16_t		len;
 	Boolean			foundRecord;
-	UInt32			nodeNum;
+	u_int32_t		nodeNum;
 	BlockDescriptor		left, node, right;
-	UInt16			index;
+	u_int16_t		index;
 
 
 	////////////////////////// Priliminary Checks ///////////////////////////////
@@ -998,7 +1029,8 @@ BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation, BTreeIterator
 			M_ExitOnError(err);
 
 			err = fsBTInvalidNodeErr;
-			MARK_VOLUMEDAMAGED(filePtr);
+			printf ("hfs: BTIterateRecords() found invalid btree node on volume %s\n", FCBTOVCB(filePtr)->vcbVN);
+			hfs_mark_volume_inconsistent(FCBTOVCB(filePtr));
 			goto ErrorExit;
 		}
 
@@ -1012,6 +1044,7 @@ BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation, BTreeIterator
 
 	//////////////////////// Find Iterator Position /////////////////////////////
 
+	// Not called for (operation == kBTreeFirstRecord || operation == kBTreeLastRecord)
 	err = FindIteratorPosition(btreePtr, iterator, &left, &node, &right,
 				   &nodeNum, &index, &foundRecord);
 	if (err == fsBTRecordNotFoundErr)
@@ -1034,8 +1067,19 @@ BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation, BTreeIterator
 				nodeNum = ((NodeDescPtr) node.buffer)->bLink;
 				if ( nodeNum > 0)
 				{
-					err = GetNode(btreePtr, nodeNum, &left);
+					// BTree nodes are always grabbed in left to right order.  
+					// Therefore release the current node before looking up the 
+					// left node.
+					err = ReleaseNode(btreePtr, &node);
 					M_ExitOnError(err);
+
+					// Look up the left node 
+					err = GetNode (btreePtr, nodeNum, &left);
+					M_ExitOnError (err);
+
+					// Look up the current node again
+					err = GetRightSiblingNode (btreePtr, left.buffer, &node);
+					M_ExitOnError (err);
 				} else {
 					err = fsBTStartOfIterationErr;
 					goto ErrorExit;
@@ -1217,15 +1261,15 @@ ErrorExit:
 OSStatus	BTInsertRecord		(FCB						*filePtr,
 								 BTreeIterator				*iterator,
 								 FSBufferDescriptor			*record,
-								 UInt16						 recordLen )
+								 u_int16_t					 recordLen )
 {
 	OSStatus				err;
 	BTreeControlBlockPtr	btreePtr;
 	TreePathTable			treePathTable;
-	SInt32					nodesNeeded;
+	u_int32_t			nodesNeeded;
 	BlockDescriptor			nodeRec;
-	UInt32					insertNodeNum;
-	UInt16					index;
+	u_int32_t				insertNodeNum;
+	u_int16_t				index;
 	Boolean					recordFit;
 
 	////////////////////////// Priliminary Checks ///////////////////////////////
@@ -1256,7 +1300,7 @@ OSStatus	BTInsertRecord		(FCB						*filePtr,
 
 		case fsBTEmptyErr:	// if tree empty add 1st leaf node
 
-								if (BTAvailableNodes(btreePtr) == 0)
+								if (btreePtr->freeNodes == 0)
 								{
 									err = ExtendBTree (btreePtr, btreePtr->totalNodes + 1);
 									M_ExitOnError (err);
@@ -1322,10 +1366,9 @@ OSStatus	BTInsertRecord		(FCB						*filePtr,
 
 	/////////////////////// Extend File If Necessary ////////////////////////////
 
-	nodesNeeded = (SInt32)btreePtr->treeDepth + 1 - BTAvailableNodes(btreePtr);
-	if (nodesNeeded > 0)
+	if ((btreePtr->treeDepth + 1UL) > btreePtr->freeNodes)
 	{
-		nodesNeeded += (SInt32)btreePtr->totalNodes;
+		nodesNeeded = btreePtr->treeDepth + 1 + btreePtr->totalNodes - btreePtr->freeNodes;
 		if (nodesNeeded > CalcMapBits (btreePtr))	// we'll need to add a map node too!
 			++nodesNeeded;
 
@@ -1381,15 +1424,15 @@ ErrorExit:
 OSStatus	BTReplaceRecord		(FCB						*filePtr,
 								 BTreeIterator				*iterator,
 								 FSBufferDescriptor			*record,
-								 UInt16						 recordLen )
+								 u_int16_t					 recordLen )
 {
 	OSStatus				err;
 	BTreeControlBlockPtr	btreePtr;
 	TreePathTable			treePathTable;
-	SInt32					nodesNeeded;
+	u_int32_t			nodesNeeded;
 	BlockDescriptor			nodeRec;
-	UInt32					insertNodeNum;
-	UInt16					index;
+	u_int32_t				insertNodeNum;
+	u_int16_t				index;
 	Boolean					recordFit;
 	Boolean					validHint;
 
@@ -1474,10 +1517,9 @@ OSStatus	BTReplaceRecord		(FCB						*filePtr,
 
 	//////////////////////////// Make Some Room /////////////////////////////////
 
-	nodesNeeded = (SInt32)btreePtr->treeDepth + 1 - BTAvailableNodes(btreePtr);
-	if (nodesNeeded > 0)
+	if ((btreePtr->treeDepth + 1UL) > btreePtr->freeNodes)
 	{
-		nodesNeeded += (SInt32)btreePtr->totalNodes;
+		nodesNeeded = btreePtr->treeDepth + 1 + btreePtr->totalNodes - btreePtr->freeNodes;
 		if (nodesNeeded > CalcMapBits (btreePtr))	// we'll need to add a map node too!
 			++nodesNeeded;
 
@@ -1536,9 +1578,9 @@ BTUpdateRecord(FCB *filePtr, BTreeIterator *iterator,
 	BlockDescriptor			nodeRec;
 	RecordPtr				recordPtr;
 	BTreeKeyPtr				keyPtr;
-	UInt32					insertNodeNum;
-	UInt16					recordLen;
-	UInt16					index;
+	u_int32_t				insertNodeNum;
+	u_int16_t				recordLen;
+	u_int16_t				index;
 	Boolean					validHint;
 
 
@@ -1647,9 +1689,9 @@ OSStatus	BTDeleteRecord		(FCB						*filePtr,
 	BTreeControlBlockPtr	btreePtr;
 	TreePathTable			treePathTable;
 	BlockDescriptor			nodeRec;
-	SInt32 nodesNeeded;
-	UInt32					nodeNum;
-	UInt16					index;
+	u_int32_t 			nodesNeeded;
+	u_int32_t				nodeNum;
+	u_int16_t				index;
 
 
 	////////////////////////// Priliminary Checks ///////////////////////////////
@@ -1672,7 +1714,7 @@ OSStatus	BTDeleteRecord		(FCB						*filePtr,
 
 	/////////////////////////////// Find Key ////////////////////////////////////
 
-	// check hint for simple delete case (index > 0, numRecords > 2)
+	// check hint for simple delete case (index > 0, numRecords > 2)
 
 	err = SearchTree (btreePtr, &iterator->key, treePathTable, &nodeNum, &nodeRec, &index);
 	M_ExitOnError (err);					// record must exit for Delete
@@ -1680,10 +1722,9 @@ OSStatus	BTDeleteRecord		(FCB						*filePtr,
 
 	/////////////////////// Extend File If Necessary ////////////////////////////
 
-	nodesNeeded = (SInt32)btreePtr->treeDepth + 1 - BTAvailableNodes(btreePtr);
-	if ((btreePtr->attributes & kBTVariableIndexKeysMask) && (nodesNeeded > 0))
+	if ((btreePtr->treeDepth + 1UL) > btreePtr->totalNodes)
 	{
-		nodesNeeded += (SInt32)btreePtr->totalNodes;
+		nodesNeeded = btreePtr->treeDepth + 1 + btreePtr->totalNodes;
 		if (nodesNeeded > CalcMapBits (btreePtr))
 			++nodesNeeded;
 
@@ -1715,10 +1756,10 @@ ErrorExit:
 
 
 OSStatus	BTGetInformation	(FCB					*filePtr,
-								 UInt16					 version,
+								 u_int16_t				 file_version,
 								 BTreeInfoRec			*info )
 {
-#pragma unused (version)
+#pragma unused (file_version)
 
 	BTreeControlBlockPtr	btreePtr;
 
@@ -1897,7 +1938,7 @@ Result:		noErr			- success
 
 
 OSStatus	BTGetLastSync		(FCB					*filePtr,
-								 UInt32					*lastsync)
+								 u_int32_t				*lastsync)
 {
 	BTreeControlBlockPtr	btreePtr;
 
@@ -1936,7 +1977,7 @@ Result:		noErr			- success
 
 
 OSStatus	BTSetLastSync		(FCB					*filePtr,
-								 UInt32					lastsync)
+								 u_int32_t				lastsync)
 {
 	BTreeControlBlockPtr	btreePtr;
 
@@ -1949,7 +1990,7 @@ OSStatus	BTSetLastSync		(FCB					*filePtr,
 	REQUIRE_FILE_LOCK(btreePtr->fileRefNum, true);
 
 	M_ReturnErrorIf (btreePtr == nil,	fsBTInvalidFileErr);
-	M_ReturnErrorIf (lastsync == nil,	paramErr);
+	M_ReturnErrorIf (lastsync == 0,	paramErr);
 
 	btreePtr->lastfsync = lastsync;
 
diff --git a/bsd/hfs/hfscommon/BTree/BTreeAllocate.c b/bsd/hfs/hfscommon/BTree/BTreeAllocate.c
index d3fa1d3eb..ff917113e 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeAllocate.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeAllocate.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2003, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreeAllocate.c
@@ -73,15 +79,16 @@
 
 */
 
+#include "../../hfs_btreeio.h"
 #include "../../hfs_endian.h"
 #include "../headers/BTreesPrivate.h"
 
 ///////////////////// Routines Internal To BTreeAllocate.c //////////////////////
 
-OSStatus	GetMapNode (BTreeControlBlockPtr	  btreePtr,
+static OSStatus	GetMapNode (BTreeControlBlockPtr	  btreePtr,
 						BlockDescriptor			 *nodePtr,
-						UInt16					**mapPtr,
-						UInt16					 *mapSize );
+						u_int16_t					**mapPtr,
+						u_int16_t					 *mapSize );
 
 /////////////////////////////////////////////////////////////////////////////////
 
@@ -110,16 +117,16 @@ Result:		noErr			- success
 			!= noErr		- failure
 -------------------------------------------------------------------------------*/
 
-OSStatus	AllocateNode (BTreeControlBlockPtr		btreePtr, UInt32	*nodeNum)
+OSStatus	AllocateNode (BTreeControlBlockPtr		btreePtr, u_int32_t	*nodeNum)
 {
 	OSStatus		 err;
 	BlockDescriptor	 node;
-	UInt16			*mapPtr, *pos;
-	UInt16			 mapSize, size;
-	UInt16			 freeWord;
-	UInt16			 mask;
-	UInt16			 bitOffset;
-	UInt32			 nodeNumber;
+	u_int16_t		*mapPtr, *pos;
+	u_int16_t		 mapSize, size;
+	u_int16_t		 freeWord;
+	u_int16_t		 mask;
+	u_int16_t		 bitOffset;
+	u_int32_t		 nodeNumber;
 	
 	
 	nodeNumber		= 0;				// first node number of header map record
@@ -228,14 +235,14 @@ Result:		noErr			- success
 			!= noErr		- GetNode or ReleaseNode encountered some difficulty
 -------------------------------------------------------------------------------*/
 
-OSStatus	FreeNode (BTreeControlBlockPtr		btreePtr, UInt32	nodeNum)
+OSStatus	FreeNode (BTreeControlBlockPtr		btreePtr, u_int32_t	nodeNum)
 {
 	OSStatus		 err;
 	BlockDescriptor	 node;
-	UInt32			 nodeIndex;
-	UInt16			 mapSize;
-	UInt16			*mapPos;
-	UInt16			 bitOffset;
+	u_int32_t		 nodeIndex;
+	u_int16_t		 mapSize;
+	u_int16_t		*mapPos;
+	u_int16_t		 bitOffset;
 	
 
 	//////////////////////////// Find Map Record ////////////////////////////////
@@ -302,25 +309,25 @@ Result:		noErr		- success
 -------------------------------------------------------------------------------*/
 
 OSStatus	ExtendBTree	(BTreeControlBlockPtr	btreePtr,
-						 UInt32					newTotalNodes )
+						 u_int32_t				newTotalNodes )
 {
 	OSStatus				 err;
 	FCB						*filePtr;
 	FSSize					 minEOF, maxEOF;	
-	UInt16					 nodeSize;
-	UInt32					 oldTotalNodes;
-	UInt32					 newMapNodes;
-	UInt32					 mapBits, totalMapBits;
-	UInt32					 recStartBit;
-	UInt32					 nodeNum, nextNodeNum;
-	UInt32					 firstNewMapNodeNum, lastNewMapNodeNum;
+	u_int16_t				 nodeSize;
+	u_int32_t				 oldTotalNodes;
+	u_int32_t				 newMapNodes;
+	u_int32_t				 mapBits, totalMapBits;
+	u_int32_t				 recStartBit;
+	u_int32_t				 nodeNum, nextNodeNum;
+	u_int32_t				 firstNewMapNodeNum, lastNewMapNodeNum;
 	BlockDescriptor			 mapNode, newNode;
-	UInt16					*mapPos;
-	UInt16					*mapStart;
-	UInt16					 mapSize;
-	UInt16					 mapNodeRecSize;
-	UInt32					 bitInWord, bitInRecord;
-	UInt16					 mapIndex;
+	u_int16_t				*mapPos;
+	u_int16_t				*mapStart;
+	u_int16_t				 mapSize;
+	u_int16_t				 mapNodeRecSize;
+	u_int32_t				 bitInWord, bitInRecord;
+	u_int16_t				 mapIndex;
 
 
 	oldTotalNodes	 	= btreePtr->totalNodes;
@@ -356,10 +363,10 @@ OSStatus	ExtendBTree	(BTreeControlBlockPtr	btreePtr,
 		
 	/////////////////////// Extend LEOF If Necessary ////////////////////////////
 
-	minEOF = (UInt64)newTotalNodes * (UInt64)nodeSize;
-	if ( filePtr->fcbEOF < minEOF )
+	minEOF = (u_int64_t)newTotalNodes * (u_int64_t)nodeSize;
+	if ( (u_int64_t)filePtr->fcbEOF < minEOF )
 	{
-		maxEOF = (UInt64)0x7fffffffLL * (UInt64)nodeSize;
+		maxEOF = (u_int64_t)0x7fffffffLL * (u_int64_t)nodeSize;
 
 		err = btreePtr->setEndOfForkProc (btreePtr->fileRefNum, minEOF, maxEOF);
 		M_ExitOnError (err);
@@ -411,7 +418,7 @@ OSStatus	ExtendBTree	(BTreeControlBlockPtr	btreePtr,
 		((NodeDescPtr)newNode.buffer)->kind = kBTMapNode;
 		
 		// set free space offset
-		*(UInt16 *)((Ptr)newNode.buffer + nodeSize - 4) = nodeSize - 6;
+		*(u_int16_t *)((Ptr)newNode.buffer + nodeSize - 4) = nodeSize - 6;
 
 		if (nodeNum++ == lastNewMapNodeNum)
 			break;
@@ -452,7 +459,7 @@ OSStatus	ExtendBTree	(BTreeControlBlockPtr	btreePtr,
 
 			mapIndex = 0;
 			
-			mapStart	 = (UInt16 *) GetRecordAddress (btreePtr, mapNode.buffer, mapIndex);
+			mapStart	 = (u_int16_t *) GetRecordAddress (btreePtr, mapNode.buffer, mapIndex);
 			mapSize		 = GetRecordSize (btreePtr, mapNode.buffer, mapIndex);
 			
 			if (DEBUG_BUILD && mapSize != M_MapRecordSize (btreePtr->nodeSize) )
@@ -529,14 +536,15 @@ Result:		noErr			- success
 			!= noErr		- failure
 -------------------------------------------------------------------------------*/
 
+static
 OSStatus	GetMapNode (BTreeControlBlockPtr	  btreePtr,
 						BlockDescriptor			 *nodePtr,
-						UInt16					**mapPtr,
-						UInt16					 *mapSize )
+						u_int16_t				**mapPtr, 			
+						u_int16_t				 *mapSize )
 {
 	OSStatus	err;
-	UInt16		mapIndex;
-	UInt32		nextNodeNum;
+	u_int16_t	mapIndex;
+	u_int32_t	nextNodeNum;
 	
 	if (nodePtr->buffer != nil)		// if iterator is valid...
 	{
@@ -575,7 +583,7 @@ OSStatus	GetMapNode (BTreeControlBlockPtr	  btreePtr,
 	}
 	
 		
-	*mapPtr		= (UInt16 *) GetRecordAddress (btreePtr, nodePtr->buffer, mapIndex);
+	*mapPtr		= (u_int16_t *) GetRecordAddress (btreePtr, nodePtr->buffer, mapIndex);
 	*mapSize	= GetRecordSize (btreePtr, nodePtr->buffer, mapIndex);
 	
 	return noErr;
@@ -595,9 +603,9 @@ ErrorExit:
 
 ////////////////////////////////// CalcMapBits //////////////////////////////////
 
-UInt32		CalcMapBits	(BTreeControlBlockPtr	 btreePtr)
+u_int32_t		CalcMapBits	(BTreeControlBlockPtr	 btreePtr)
 {
-	UInt32		mapBits;
+	u_int32_t		mapBits;
 	
 	mapBits		= M_HeaderMapRecordSize (btreePtr->nodeSize) << 3;
 	
diff --git a/bsd/hfs/hfscommon/BTree/BTreeMiscOps.c b/bsd/hfs/hfscommon/BTree/BTreeMiscOps.c
index 31d4af9ba..0e47310ee 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeMiscOps.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeMiscOps.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2003, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreeMiscOps.c
@@ -106,6 +112,7 @@
 */
 
 #include "../headers/BTreesPrivate.h"
+#include "../../hfs_btreeio.h"
 
 
 ////////////////////////////// Routine Definitions //////////////////////////////
@@ -121,11 +128,11 @@ Input:		keySize		- length of key (including length field)
 
 Output:		none
 			
-Result:		UInt16		- size of combined key/record that will be inserted in btree
+Result:		u_int16_t	- size of combined key/record that will be inserted in btree
 -------------------------------------------------------------------------------*/
 
-UInt16		CalcKeyRecordSize		(UInt16					 keySize,
-									 UInt16					 recSize )
+u_int16_t		CalcKeyRecordSize		(u_int16_t				 keySize,
+										 u_int16_t				 recSize )
 {
 	if ( M_IsOdd (keySize) )	keySize += 1;	// pad byte
 	
@@ -153,8 +160,8 @@ Result:		noErr		- success
 OSStatus	VerifyHeader	(FCB				*filePtr,
 							 BTHeaderRec			 *header )
 {
-	UInt64		forkSize;
-	UInt32		totalNodes;
+	u_int64_t		forkSize;
+	u_int32_t		totalNodes;
 	
 
 	switch (header->nodeSize)							// node size == 512*2^n
@@ -171,9 +178,9 @@ OSStatus	VerifyHeader	(FCB				*filePtr,
 	
 	totalNodes = header->totalNodes;
 
-	forkSize = (UInt64)totalNodes * (UInt64)header->nodeSize;
+	forkSize = (u_int64_t)totalNodes * (u_int64_t)header->nodeSize;
 	
-	if ( forkSize > filePtr->fcbEOF )
+	if ( forkSize > (u_int64_t)filePtr->fcbEOF )
 		return fsBTInvalidHeaderErr;
 	
 	if ( header->freeNodes >= totalNodes )
@@ -235,7 +242,7 @@ OSStatus UpdateHeader(BTreeControlBlockPtr btreePtr, Boolean forceWrite)
 	OSStatus				err;
 	BlockDescriptor			node;
 	BTHeaderRec	*header;	
-	UInt32 options;
+	u_int32_t options;
 
 	if ((btreePtr->flags & kBTHeaderDirty) == 0)			// btree info already flushed
 	return	noErr;
@@ -309,20 +316,20 @@ OSStatus	FindIteratorPosition	(BTreeControlBlockPtr	 btreePtr,
 									 BlockDescriptor		*left,
 									 BlockDescriptor		*middle,
 									 BlockDescriptor		*right,
-									 UInt32					*returnNodeNum,
-									 UInt16					*returnIndex,
+									 u_int32_t				*returnNodeNum,
+									 u_int16_t				*returnIndex,
 									 Boolean				*foundRecord )
 {
 	OSStatus		err;
 	Boolean			foundIt;
-	UInt32			nodeNum;
-	UInt16			leftIndex,	index,	rightIndex;
+	u_int32_t		nodeNum;
+	u_int16_t		leftIndex,	index,	rightIndex;
 	Boolean			validHint;
 
 	// assume btreePtr valid
 	// assume left, middle, right point to BlockDescriptors
-	// assume nodeNum points to UInt32
-	// assume index points to UInt16
+	// assume nodeNum points to u_int32_t
+	// assume index points to u_int16_t
 	// assume foundRecord points to Boolean
 	
 	left->buffer		= nil;
@@ -359,16 +366,16 @@ OSStatus	FindIteratorPosition	(BTreeControlBlockPtr	 btreePtr,
 		 ((NodeDescPtr) middle->buffer)->numRecords <= 0 )
 	{	
 		goto SearchTheTree;
-	}
-		
-	++btreePtr->numValidHints;
+	}	
 	
 	foundIt = SearchNode (btreePtr, middle->buffer, &iterator->key, &index);
 	if (foundIt == true)
 	{
+		++btreePtr->numValidHints;
 		goto SuccessfulExit;
 	}
-	
+	iterator->hint.nodeNum = 0;
+
 	if (index == 0)
 	{
 		if (((NodeDescPtr) middle->buffer)->bLink == 0)		// before 1st btree record
@@ -378,9 +385,20 @@ OSStatus	FindIteratorPosition	(BTreeControlBlockPtr	 btreePtr,
 		
 		nodeNum = ((NodeDescPtr) middle->buffer)->bLink;
 		
-		err = GetLeftSiblingNode (btreePtr, middle->buffer, left);
+		// BTree nodes are always grabbed in left to right order.  
+		// Therefore release the current node before looking up the 
+		// left node.
+		err = ReleaseNode(btreePtr, middle);
+		M_ExitOnError(err);
+
+		// Look up the left node 
+		err = GetNode (btreePtr, nodeNum, left);
 		M_ExitOnError (err);
-		
+
+		// Look up the current node again
+		err = GetRightSiblingNode (btreePtr, left->buffer, middle);
+		M_ExitOnError (err);
+
 		if ( ((NodeDescPtr) left->buffer)->kind != kBTLeafNode ||
 			 ((NodeDescPtr) left->buffer)->numRecords <= 0 )
 		{	
@@ -506,7 +524,7 @@ ErrorExit:
 OSStatus	CheckInsertParams		(FCB						*filePtr,
 									 BTreeIterator				*iterator,
 									 FSBufferDescriptor			*record,
-									 UInt16						 recordLen )
+									 u_int16_t					 recordLen )
 {
 	BTreeControlBlockPtr	btreePtr;
 	
@@ -560,13 +578,13 @@ OSStatus	TrySimpleReplace		(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 nodePtr,
 									 BTreeIterator			*iterator,
 									 FSBufferDescriptor		*record,
-									 UInt16					 recordLen,
+									 u_int16_t				 recordLen,
 									 Boolean				*recordInserted )
 {
-	UInt32				oldSpace;
-	UInt32				spaceNeeded;
-	UInt16				index;
-	UInt16				keySize;
+	u_int32_t			oldSpace;
+	u_int32_t			spaceNeeded;
+	u_int16_t			index;
+	u_int16_t			keySize;
 	Boolean				foundIt;
 	Boolean				didItFit;
 	
@@ -589,7 +607,7 @@ OSStatus	TrySimpleReplace		(BTreeControlBlockPtr	 btreePtr,
 	
 	if ( spaceNeeded == oldSpace )
 	{
-		UInt8 *		dst;
+		u_int8_t *		dst;
 
 		dst = GetRecordAddress (btreePtr, nodePtr, index);
 
diff --git a/bsd/hfs/hfscommon/BTree/BTreeNodeOps.c b/bsd/hfs/hfscommon/BTree/BTreeNodeOps.c
index 590dfecc5..ab2962683 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeNodeOps.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeNodeOps.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000, 2002, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreeNodeOps.c
@@ -136,34 +142,34 @@
 
 ////////////////////// Routines Internal To BTreeNodeOps.c //////////////////////
 
-UInt16		GetRecordOffset		(BTreeControlBlockPtr	 btree,
+u_int16_t	GetRecordOffset		(BTreeControlBlockPtr	 btree,
 								 NodeDescPtr			 node,
-								 UInt16					 index );
+								 u_int16_t				 index );
 
-UInt16	   *GetOffsetAddress	(BTreeControlBlockPtr	btreePtr,
+u_int16_t	*GetOffsetAddress	(BTreeControlBlockPtr	btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					index );
+								 u_int16_t				index );
 								 
 void		InsertOffset		(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					 index,
-								 UInt16					 delta );
+								 u_int16_t				 index,
+								 u_int16_t				 delta );
 
 void		DeleteOffset		(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					 index );
+								 u_int16_t				 index );
 
 
 /////////////////////////////////////////////////////////////////////////////////
 
-#define GetRecordOffset(btreePtr,node,index)		(*(short *) ((UInt8 *)(node) + (btreePtr)->nodeSize - ((index) << 1) - kOffsetSize))
+#define GetRecordOffset(btreePtr,node,index)		(*(short *) ((u_int8_t *)(node) + (btreePtr)->nodeSize - ((index) << 1) - kOffsetSize))
 
 #if HFS_DIAGNOSTIC
 		#include <sys/systm.h>
 	    #define PRINTIT kprintf
+static void PrintNode(const NodeDescPtr node, u_int16_t nodeSize, u_int32_t nodeNumber);
 #endif /* HFS_DIAGNOSTIC */
 
-static void PrintNode(const NodeDescPtr node, UInt16 nodeSize, UInt32 nodeNumber);
 
 
 
@@ -184,7 +190,7 @@ Result:
 -------------------------------------------------------------------------------*/
 
 OSStatus	GetNode		(BTreeControlBlockPtr	 btreePtr,
-						 UInt32					 nodeNum,
+						 u_int32_t				 nodeNum,
 						 NodeRec				*nodePtr )
 {
 	OSStatus			err;
@@ -243,7 +249,7 @@ Result:		noErr		- success
 -------------------------------------------------------------------------------*/
 
 OSStatus	GetNewNode	(BTreeControlBlockPtr	 btreePtr,
-						 UInt32					 nodeNum,
+						 u_int32_t				 nodeNum,
 						 NodeRec				*returnNodePtr )
 {
 	OSStatus			 err;
@@ -278,7 +284,7 @@ OSStatus	GetNewNode	(BTreeControlBlockPtr	 btreePtr,
 	ClearNode (btreePtr, node);						// clear the node
 
 	pos = (char *)node + btreePtr->nodeSize - 2;	// find address of last offset
-	*(UInt16 *)pos = sizeof (BTNodeDescriptor);		// set offset to beginning of free space
+	*(u_int16_t *)pos = sizeof (BTNodeDescriptor);	// set offset to beginning of free space
 
 
 	return noErr;
@@ -385,9 +391,11 @@ Result:		noErr		- success
 
 OSStatus	UpdateNode	(BTreeControlBlockPtr	 btreePtr,
 						 NodePtr				 nodePtr,
-						 UInt32					 transactionID,
-						 UInt32					 flags )
+						 u_int32_t				 transactionID,
+						 u_int32_t				 flags )
 {
+#pragma unused(transactionID)
+
 	OSStatus			 err;
 	ReleaseBlockProcPtr	 releaseNodeProc;
 	
@@ -417,19 +425,19 @@ ErrorExit:
 
 
 #if HFS_DIAGNOSTIC
-static void PrintNode(const NodeDescPtr node, UInt16 nodeSize, UInt32 nodeNumber)
+static void PrintNode(const NodeDescPtr node, u_int16_t nodeSize, u_int32_t nodeNumber)
 {
 	struct row {
-		UInt16	word[8];
+		u_int16_t	word[8];
 	};
 	struct row	*offset;
-	UInt16	rows;
-	UInt32	*lp;
+	u_int16_t	rows;
+	u_int32_t	*lp;
 
 	PRINTIT("Dump of B-tree node #%ld ($%08lX)\n", nodeNumber, nodeNumber);
 
 	rows = nodeSize/16;
-	lp = (UInt32*) node;
+	lp = (u_int32_t*) node;
 	offset = 0;
 	
 	while (rows-- > 0)
@@ -474,14 +482,14 @@ Result:		noErr		- success
 
 Boolean		InsertRecord	(BTreeControlBlockPtr	btreePtr,
 							 NodeDescPtr 			node,
-							 UInt16	 				index,
+							 u_int16_t	 			index,
 							 RecordPtr				recPtr,
-							 UInt16					recSize )
+							 u_int16_t				recSize )
 {
-	UInt16		freeSpace;
-	UInt16		indexOffset;
-	UInt16		freeOffset;
-	UInt16		bytesToMove;
+	u_int16_t	freeSpace;
+	u_int16_t	indexOffset;
+	u_int16_t	freeOffset;
+	u_int16_t	bytesToMove;
 	void	   *src;
 	void	   *dst;
 	
@@ -544,28 +552,28 @@ Result:		noErr		- success
 
 Boolean		InsertKeyRecord		(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr 			 node,
-								 UInt16	 				 index,
+								 u_int16_t	 			 index,
 								 KeyPtr					 keyPtr,
-								 UInt16					 keyLength,
+								 u_int16_t				 keyLength,
 								 RecordPtr				 recPtr,
-								 UInt16					 recSize )
+								 u_int16_t				 recSize )
 {
-	UInt16		freeSpace;
-	UInt16		indexOffset;
-	UInt16		freeOffset;
-	UInt16		bytesToMove;
-	UInt8 *		src;
-	UInt8 *		dst;
-	UInt16		keySize;
-	UInt16		rawKeyLength;
-	UInt16		sizeOfLength;
+	u_int16_t		freeSpace;
+	u_int16_t		indexOffset;
+	u_int16_t		freeOffset;
+	u_int16_t		bytesToMove;
+	u_int8_t *		src;
+	u_int8_t *		dst;
+	u_int16_t		keySize;
+	u_int16_t		rawKeyLength;
+	u_int16_t		sizeOfLength;
 	
 	//// calculate actual key size
 
 	if ( btreePtr->attributes & kBTBigKeysMask )
-		keySize = keyLength + sizeof(UInt16);
+		keySize = keyLength + sizeof(u_int16_t);
 	else
-		keySize = keyLength + sizeof(UInt8);
+		keySize = keyLength + sizeof(u_int8_t);
 	
 	if ( M_IsOdd (keySize) )
 		++keySize;			// add pad byte
@@ -586,8 +594,8 @@ Boolean		InsertKeyRecord		(BTreeControlBlockPtr	 btreePtr,
 	indexOffset = GetRecordOffset (btreePtr, node, index);
 	freeOffset	= GetRecordOffset (btreePtr, node, node->numRecords);
 
-	src = ((UInt8 *) node) + indexOffset;
-	dst = ((UInt8 *) src) + keySize + recSize;
+	src = ((u_int8_t *) node) + indexOffset;
+	dst = ((u_int8_t *) src) + keySize + recSize;
 	bytesToMove = freeOffset - indexOffset;
 	if (bytesToMove)
 		MoveRecordsRight (src, dst, bytesToMove);
@@ -600,11 +608,12 @@ Boolean		InsertKeyRecord		(BTreeControlBlockPtr	 btreePtr,
 
 	//// copy record key
 
-	dst = ((UInt8 *) node) + indexOffset;
+	dst = ((u_int8_t *) node) + indexOffset;
 
 	if ( btreePtr->attributes & kBTBigKeysMask )
 	{
-		*((UInt16*) dst)++ = keyLength;		// use keyLength rather than key.length
+		*((u_int16_t *)dst) = keyLength;			// use keyLength rather than key.length
+  		dst = (u_int8_t *) (((u_int16_t *)dst) + 1);
 		rawKeyLength = keyPtr->length16;
 		sizeOfLength = 2;
 	}
@@ -615,7 +624,7 @@ Boolean		InsertKeyRecord		(BTreeControlBlockPtr	 btreePtr,
 		sizeOfLength = 1;
 	}
 	
-	MoveRecordsLeft ( ((UInt8 *) keyPtr) + sizeOfLength, dst, rawKeyLength);	// copy key
+	MoveRecordsLeft ( ((u_int8_t *) keyPtr) + sizeOfLength, dst, rawKeyLength);	// copy key
 
 	// any pad bytes?
 	bytesToMove = keySize - rawKeyLength;
@@ -625,7 +634,7 @@ Boolean		InsertKeyRecord		(BTreeControlBlockPtr	 btreePtr,
 
 	//// copy record data
 
-	dst = ((UInt8 *) node) + indexOffset + keySize;
+	dst = ((u_int8_t *) node) + indexOffset + keySize;
 	MoveRecordsLeft (recPtr, dst, recSize);
 
 	return true;
@@ -648,12 +657,12 @@ Result:		none
 
 void		DeleteRecord	(BTreeControlBlockPtr	btreePtr,
 							 NodeDescPtr 			node,
-							 UInt16	 				index )
+							 u_int16_t	 			index )
 {
-	SInt16		indexOffset;
-	SInt16		nextOffset;
-	SInt16		freeOffset;
-	SInt16		bytesToMove;
+	int16_t		indexOffset;
+	int16_t		nextOffset;
+	int16_t		freeOffset;
+	int16_t		bytesToMove;
 	void	   *src;
 	void	   *dst;
 	
@@ -702,24 +711,24 @@ Boolean
 SearchNode( BTreeControlBlockPtr btreePtr,
 	    NodeDescPtr node,
 	    KeyPtr searchKey,
-	    UInt16 *returnIndex )
+	    u_int16_t *returnIndex )
 {
-	SInt32		lowerBound;
-	SInt32		upperBound;
-	SInt32		index;
-	SInt32		result;
+	int32_t		lowerBound;
+	int32_t		upperBound;
+	int32_t		index;
+	int32_t		result;
 	KeyPtr		trialKey;
-	UInt16		*offset;
+	u_int16_t	*offset;
 	KeyCompareProcPtr compareProc = btreePtr->keyCompareProc;
 
 	lowerBound = 0;
 	upperBound = node->numRecords - 1;
-	offset = (UInt16 *) ((UInt8 *)(node) + (btreePtr)->nodeSize - kOffsetSize);
+	offset = (u_int16_t *) ((u_int8_t *)(node) + (btreePtr)->nodeSize - kOffsetSize);
 	
 	while (lowerBound <= upperBound) {
 		index = (lowerBound + upperBound) >> 1;
 
-		trialKey = (KeyPtr) ((UInt8 *)node + *(offset - index));
+		trialKey = (KeyPtr) ((u_int8_t *)node + *(offset - index));
 		
 		result = compareProc(searchKey, trialKey);
 
@@ -759,14 +768,14 @@ Result:		none
 
 OSStatus	GetRecordByIndex	(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					 index,
+								 u_int16_t				 index,
 								 KeyPtr					*keyPtr,
-								 UInt8 *				*dataPtr,
-								 UInt16					*dataSize )
+								 u_int8_t *				*dataPtr,
+								 u_int16_t				*dataSize )
 {
-	UInt16		offset;
-	UInt16		nextOffset;
-	UInt16		keySize;
+	u_int16_t		offset;
+	u_int16_t		nextOffset;
+	u_int16_t		keySize;
 	
 	//
 	//	Make sure index is valid (in range 0..numRecords-1)
@@ -784,7 +793,7 @@ OSStatus	GetRecordByIndex	(BTreeControlBlockPtr	 btreePtr,
 		++keySize;	// add pad byte
 
 	offset += keySize;			// add the key length to find data offset
-	*dataPtr = (UInt8 *) node + offset;
+	*dataPtr = (u_int8_t *) node + offset;
 	
 	//// find dataSize
 	nextOffset	= GetRecordOffset (btreePtr, node, index + 1);
@@ -808,9 +817,9 @@ Input:		btreePtr		- pointer to BTree control block
 Result:		- number of bytes used for data and offsets in the node.
 -------------------------------------------------------------------------------*/
 
-UInt16		GetNodeDataSize	(BTreeControlBlockPtr	btreePtr, NodeDescPtr	 node )
+u_int16_t	GetNodeDataSize	(BTreeControlBlockPtr	btreePtr, NodeDescPtr	 node )
 {
-	UInt16 freeOffset;
+	u_int16_t freeOffset;
 	
 	freeOffset = GetRecordOffset (btreePtr, node, node->numRecords);
 	
@@ -831,9 +840,9 @@ Input:		btreePtr		- pointer to BTree control block
 Result:		- number of bytes of free space in the node.
 -------------------------------------------------------------------------------*/
 
-UInt16		GetNodeFreeSize	(BTreeControlBlockPtr	btreePtr, NodeDescPtr	 node )
+u_int16_t		GetNodeFreeSize	(BTreeControlBlockPtr	btreePtr, NodeDescPtr	 node )
 {
-	UInt16	freeOffset;
+	u_int16_t	freeOffset;
 	
 	freeOffset = GetRecordOffset (btreePtr, node, node->numRecords);	// inline?
 	
@@ -856,14 +865,14 @@ Result:		- offset (in bytes) from beginning of node of record specified by index
 -------------------------------------------------------------------------------*/
 // make this a macro (for inlining)
 #if 0
-UInt16		GetRecordOffset	(BTreeControlBlockPtr	btreePtr,
+u_int16_t	GetRecordOffset	(BTreeControlBlockPtr	btreePtr,
 							 NodeDescPtr			node,
-							 UInt16					index )
+							 u_int16_t				index )
 {
 	void	*pos;
 	
 		
-	pos = (UInt8 *)node + btreePtr->nodeSize - (index << 1) - kOffsetSize;
+	pos = (u_int8_t *)node + btreePtr->nodeSize - (index << 1) - kOffsetSize;
 	
 	return *(short *)pos;
 }
@@ -885,13 +894,13 @@ Result:		- pointer to record "index".
 -------------------------------------------------------------------------------*/
 // make this a macro (for inlining)
 #if 0
-UInt8 *		GetRecordAddress	(BTreeControlBlockPtr	btreePtr,
+u_int8_t *	GetRecordAddress	(BTreeControlBlockPtr	btreePtr,
 								 NodeDescPtr			node,
-								 UInt16					index )
+								 u_int16_t				index )
 {
-	UInt8 *		pos;
+	u_int8_t *	pos;
 	
-	pos = (UInt8 *)node + GetRecordOffset (btreePtr, node, index);
+	pos = (u_int8_t *)node + GetRecordOffset (btreePtr, node, index);
 	
 	return pos;
 }
@@ -914,13 +923,13 @@ Input:		btreePtr		- pointer to BTree control block
 Result:		- size of record "index".
 -------------------------------------------------------------------------------*/
 
-UInt16		GetRecordSize		(BTreeControlBlockPtr	btreePtr,
+u_int16_t	GetRecordSize		(BTreeControlBlockPtr	btreePtr,
 								 NodeDescPtr			node,
-								 UInt16					index )
+								 u_int16_t				index )
 {
-	UInt16	*pos;
+	u_int16_t	*pos;
 		
-	pos = (UInt16 *) ((Ptr)node + btreePtr->nodeSize - (index << 1) - kOffsetSize);
+	pos = (u_int16_t *) ((Ptr)node + btreePtr->nodeSize - (index << 1) - kOffsetSize);
 	
 	return  *(pos-1) - *pos;
 }
@@ -939,15 +948,15 @@ Input:		btreePtr		- pointer to BTree control block
 Result:		- pointer to offset for record "index".
 -------------------------------------------------------------------------------*/
 
-UInt16	   *GetOffsetAddress	(BTreeControlBlockPtr	btreePtr,
+u_int16_t	 *GetOffsetAddress	(BTreeControlBlockPtr	btreePtr,
 								 NodeDescPtr			node,
-								 UInt16					index )
+								 u_int16_t				index )
 {
 	void	*pos;
 	
 	pos = (Ptr)node + btreePtr->nodeSize - (index << 1) -2;
 	
-	return (UInt16 *)pos;
+	return (u_int16_t *)pos;
 }
 
 
@@ -955,7 +964,7 @@ UInt16	   *GetOffsetAddress	(BTreeControlBlockPtr	btreePtr,
 /*-------------------------------------------------------------------------------
 Routine:	GetChildNodeNum	-	Return child node number from index record "index".
 
-Function:	Returns the first UInt32 stored after the key for record "index".
+Function:	Returns the first u_int32_t stored after the key for record "index".
 
 Assumes:	The node is an Index Node.
 			The key.length stored at record "index" is ODD. // change for variable length index keys
@@ -967,16 +976,16 @@ Input:		btreePtr		- pointer to BTree control block
 Result:		- child node number from record "index".
 -------------------------------------------------------------------------------*/
 
-UInt32		GetChildNodeNum			(BTreeControlBlockPtr	 btreePtr,
+u_int32_t	GetChildNodeNum			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 nodePtr,
-									 UInt16					 index )
+									 u_int16_t				 index )
 {
-	UInt8 *		pos;
+	u_int8_t *		pos;
 	
 	pos = GetRecordAddress (btreePtr, nodePtr, index);
 	pos += CalcKeySize(btreePtr, (BTreeKey *) pos);		// key.length + size of length field
 	
-	return	*(UInt32 *)pos;
+	return	*(u_int32_t *)pos;
 }
 
 
@@ -998,11 +1007,11 @@ Result:		none
 
 void		InsertOffset		(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					 index,
-								 UInt16					 delta )
+								 u_int16_t				 index,
+								 u_int16_t				 delta )
 {
-	UInt16		*src, *dst;
-	UInt16		 numOffsets;
+	u_int16_t	*src, *dst;
+	u_int16_t	 numOffsets;
 	
 	src = GetOffsetAddress (btreePtr, node, node->numRecords);	// point to free offset
 	dst = src - 1; 												// point to new offset
@@ -1032,11 +1041,11 @@ Result:		none
 
 void		DeleteOffset		(BTreeControlBlockPtr	 btreePtr,
 								 NodeDescPtr			 node,
-								 UInt16					 index )
+								 u_int16_t				 index )
 {
-	UInt16		*src, *dst;
-	UInt16		 numOffsets;
-	UInt16		 delta;
+	u_int16_t		*src, *dst;
+	u_int16_t		 numOffsets;
+	u_int16_t		 delta;
 	
 	dst			= GetOffsetAddress (btreePtr, node, index);
 	src			= dst - 1;
diff --git a/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c b/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
index 00874f229..547a0cf50 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeNodeReserve.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include "../headers/BTreesPrivate.h"
 #include "sys/malloc.h"
@@ -30,7 +36,6 @@
  * BTReserveSpace
  * BTReleaseReserve
  * BTUpdateReserve
- * BTAvailableNodes
  *
  * Each kernel thread can have it's own reserve of b-tree
  * nodes. This reserve info is kept in a hash table.
@@ -74,7 +79,6 @@ lck_mtx_t  nr_mutex;
 /* Internal Node Reserve Hash Routines (private) */
 static void nr_insert (struct vnode *, struct nreserve *nrp, int);
 static void nr_delete (struct vnode *, struct nreserve *nrp, int *);
-static int  nr_lookup (struct vnode *);
 static void nr_update (struct vnode *, int);
 
 
@@ -99,26 +103,13 @@ BTReserveSetup()
 }
 
 
-/*
- * BTAvailNodes - obtain the actual available nodes (for current thread)
- *
- */
-__private_extern__
-SInt32
-BTAvailableNodes(BTreeControlBlock *btree)
-{
-	SInt32 availNodes;
-
-	availNodes = (SInt32)btree->freeNodes - (SInt32)btree->reservedNodes;
-
-	return (availNodes + nr_lookup(btree->fileRefNum));
-}
-
-
 /*
  * BTReserveSpace - obtain a node reserve (for current thread)
  *
  * Used by the Catalog Layer (hfs_catalog.c) to reserve space.
+ *
+ * When data is NULL, we only insure that there's enough space
+ * but it is not reserved (assumes you keep the b-tree lock).
  */
 __private_extern__
 int
@@ -128,9 +119,11 @@ BTReserveSpace(FCB *file, int operations, void* data)
 	int rsrvNodes, availNodes, totalNodes;
 	int height;
 	int inserts, deletes;
+	u_int32_t clumpsize;
 	int err = 0;
 
 	btree = (BTreeControlBlockPtr)file->fcbBTCBPtr;
+	clumpsize = file->ff_clumpsize;
 
 	REQUIRE_FILE_LOCK(btree->fileRefNum, true);
 
@@ -140,30 +133,73 @@ BTReserveSpace(FCB *file, int operations, void* data)
 	 * tree.
 	 */
 	height = btree->treeDepth;
+	if (height < 2)
+		height = 2;  /* prevent underflow in rsrvNodes calculation */
 	inserts = operations & 0xffff;
 	deletes = operations >> 16;
 	
-	rsrvNodes = 1; /* allow for at least one root split */
-	if (deletes)
-		rsrvNodes += (deletes * (height - 1)) - 1;
-	if (inserts)
-		rsrvNodes += (inserts * height) + 1;
+	/*
+	 * Allow for at least one root split.
+	 *
+	 * Each delete operation can propogate a big key up the
+	 * index. This can cause a split at each level up.
+	 *
+	 * Each insert operation can cause a local split and a
+	 * split at each level up.
+	 */
+	rsrvNodes = 1 + (deletes * (height - 2)) + (inserts * (height - 1));
 
 	availNodes = btree->freeNodes - btree->reservedNodes;
 	
 	if (rsrvNodes > availNodes) {
+		u_int32_t reqblks, freeblks, rsrvblks;
+		struct hfsmount *hfsmp;
+
+		/* Try and reserve the last 5% of the disk space for file blocks. */
+		hfsmp = VTOVCB(btree->fileRefNum);
+		rsrvblks = ((u_int64_t)hfsmp->allocLimit * 5) / 100;
+		rsrvblks = MIN(rsrvblks, HFS_MAXRESERVE / hfsmp->blockSize);
+		freeblks = hfs_freeblks(hfsmp, 0);
+		if (freeblks <= rsrvblks) {
+			/* When running low, disallow adding new items. */
+			if ((inserts > 0) && (deletes == 0)) {
+				return (ENOSPC);
+			}
+			freeblks = 0;
+		} else {
+			freeblks -= rsrvblks;
+		}
+		reqblks = clumpsize / hfsmp->blockSize;
+
+		if (reqblks > freeblks) {
+			reqblks = ((rsrvNodes - availNodes) * btree->nodeSize) / hfsmp->blockSize;
+			/* When running low, disallow adding new items. */
+			if ((reqblks > freeblks) && (inserts > 0) && (deletes == 0)) {
+				return (ENOSPC);
+			}
+			file->ff_clumpsize = freeblks * hfsmp->blockSize;
+		}
 		totalNodes = rsrvNodes + btree->totalNodes - availNodes;
 		
 		/* See if we also need a map node */
-		if (totalNodes > (int)CalcMapBits(btree))
+		if (totalNodes > (int)CalcMapBits(btree)) {
 			++totalNodes;
-		if ((err = ExtendBTree(btree, totalNodes)))
-			return (err);
-	}	
+		}
+		if ((err = ExtendBTree(btree, totalNodes))) {
+			goto out;
+		}
+	}
+	/* Save this reserve if this is a persistent request. */
+	if (data) {
+		btree->reservedNodes += rsrvNodes;
+		nr_insert(btree->fileRefNum, (struct nreserve *)data, rsrvNodes);
+	}
+out:
+	/* Put clump size back if it was changed. */
+	if (file->ff_clumpsize != clumpsize)
+		file->ff_clumpsize = clumpsize;
 
-	btree->reservedNodes += rsrvNodes;
-	nr_insert(btree->fileRefNum, (struct nreserve *)data, rsrvNodes);
-	return (0);
+	return (err);
 }
 
 
@@ -228,6 +264,7 @@ nr_insert(struct vnode * btvp, struct nreserve *nrp, int nodecnt)
 	     tmp_nrp = tmp_nrp->nr_hash.le_next) {
 		if ((tmp_nrp->nr_tag == tag) && (tmp_nrp->nr_btvp == btvp)) {
 			nrp->nr_tag = 0;
+			tmp_nrp->nr_nodecnt += nodecnt;
 			lck_mtx_unlock(&nr_mutex);
 			return;
 		}
@@ -253,7 +290,7 @@ nr_delete(struct vnode * btvp, struct nreserve *nrp, int *nodecnt)
 	lck_mtx_lock(&nr_mutex);
 	if (nrp->nr_tag) {
 		if ((nrp->nr_tag != tag) || (nrp->nr_btvp != btvp))
-			panic("nr_delete: invalid NR (%08x)", nrp);
+			panic("nr_delete: invalid NR (%p)", nrp);
 		LIST_REMOVE(nrp, nr_hash);
 		*nodecnt = nrp->nr_nodecnt;
 		bzero(nrp, sizeof(struct nreserve));
@@ -264,28 +301,6 @@ nr_delete(struct vnode * btvp, struct nreserve *nrp, int *nodecnt)
 	lck_mtx_unlock(&nr_mutex);
 }
 
-/*
- * Lookup a node reserve.
- */
-static int
-nr_lookup(struct vnode * btvp)
-{
-	struct nodereserve *nrhead;
-	struct nreserve *nrp;
-	void* tag = NR_GET_TAG();
-
-	lck_mtx_lock(&nr_mutex);
-
-	nrhead = NR_HASH(btvp, tag);
-	for (nrp = nrhead->lh_first; nrp; nrp = nrp->nr_hash.le_next) {
-		if ((nrp->nr_tag == tag) && (nrp->nr_btvp == btvp)) {
-			lck_mtx_unlock(&nr_mutex);
-			return (nrp->nr_nodecnt - nrp->nr_newnodes);
-		}
-	}
-	lck_mtx_unlock(&nr_mutex);
-	return (0);
-}
 
 /*
  * Update a node reserve for any allocations that occurred.
diff --git a/bsd/hfs/hfscommon/BTree/BTreeScanner.c b/bsd/hfs/hfscommon/BTree/BTreeScanner.c
index 1139f6415..35aeafc18 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeScanner.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeScanner.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1996-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  *	@(#)BTreeScanner.c
  */
@@ -81,7 +87,7 @@ int BTScanNextRecord(	BTScanState *	scanState,
 		//	See if we have a record in the current node
 		err = GetRecordByIndex( scanState->btcb, scanState->currentNodePtr, 
 								scanState->recordNum, (KeyPtr *) key, 
-								(UInt8 **) data, &dataSizeShort  );
+								(u_int8_t **) data, &dataSizeShort  );
 
 		if ( err == noErr )
 		{
@@ -179,7 +185,8 @@ static int FindNextLeafNode(	BTScanState *scanState, Boolean avoidIO )
 				continue; 
 			}
 
-			(u_int8_t *) scanState->currentNodePtr += scanState->btcb->nodeSize;
+			scanState->currentNodePtr = (BTNodeDescriptor *)(((u_int8_t *)scanState->currentNodePtr) 
+										+ scanState->btcb->nodeSize);
 		}
 		
 		/* Fake a BlockDescriptor */
@@ -229,7 +236,7 @@ static int ReadMultipleNodes( BTScanState *theScanStatePtr )
 	daddr64_t				myPhyBlockNum;
 	u_int32_t				myBufferSize;
 	struct vnode *			myDevPtr;
-	int						myBlockRun;
+	unsigned int			myBlockRun;
 	u_int32_t				myBlocksInBufferCount;
 
 	// release old buffer if we have one
diff --git a/bsd/hfs/hfscommon/BTree/BTreeTreeOps.c b/bsd/hfs/hfscommon/BTree/BTreeTreeOps.c
index 777e6f0fc..2aad0a7b1 100644
--- a/bsd/hfs/hfscommon/BTree/BTreeTreeOps.c
+++ b/bsd/hfs/hfscommon/BTree/BTreeTreeOps.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreeTreeOps.c
@@ -89,6 +95,7 @@
 */
 
 #include "../headers/BTreesPrivate.h"
+#include "../../hfs_btreeio.h"
 
 //
 /////////////////////// Routines Internal To BTree Module ///////////////////////
@@ -108,14 +115,14 @@ static OSStatus   CollapseTree		(BTreeControlBlockPtr		 btreePtr,
 static OSStatus   RotateLeft		(BTreeControlBlockPtr		 btreePtr,
 									 NodeDescPtr				 leftNode,
 									 NodeDescPtr				 rightNode,
-									 UInt16						 rightInsertIndex,
+									 u_int16_t					 rightInsertIndex,
 									 KeyPtr						 keyPtr,
-									 UInt8 *					 recPtr,
-									 UInt16						 recSize,
-									 UInt16						*insertIndex,
-									 UInt32						*insertNodeNum,
+									 u_int8_t *					 recPtr,
+									 u_int16_t					 recSize,
+									 u_int16_t					*insertIndex,
+									 u_int32_t					*insertNodeNum,
 									 Boolean					*recordFit,
-									 UInt16						*recsRotated );
+									 u_int16_t					*recsRotated );
 
 static Boolean	   RotateRecordLeft	(BTreeControlBlockPtr		 btreePtr,
 									 NodeDescPtr				 leftNode,
@@ -124,14 +131,14 @@ static Boolean	   RotateRecordLeft	(BTreeControlBlockPtr		 btreePtr,
 static OSStatus	   SplitLeft		(BTreeControlBlockPtr		 btreePtr,
 									 BlockDescriptor			*leftNode,
 									 BlockDescriptor			*rightNode,
-									 UInt32						 rightNodeNum,
-									 UInt16						 index,
+									 u_int32_t					 rightNodeNum,
+									 u_int16_t					 index,
 									 KeyPtr						 keyPtr,
-									 UInt8 *					 recPtr,
-									 UInt16						 recSize,
-									 UInt16						*insertIndex,
-									 UInt32						*insertNodeNum,
-									 UInt16						*recsRotated );
+									 u_int8_t *					 recPtr,
+									 u_int16_t					 recSize,
+									 u_int16_t					*insertIndex,
+									 u_int32_t					*insertNodeNum,
+									 u_int16_t					*recsRotated );
 								 
 
 
@@ -140,23 +147,23 @@ static	OSStatus	InsertLevel		(BTreeControlBlockPtr		 btreePtr,
 									 InsertKey					*primaryKey,
 									 InsertKey					*secondaryKey,
 									 BlockDescriptor			*targetNode,
-									 UInt16						 index,
-									 UInt16						 level,
-									 UInt32						*insertNode );
+									 u_int16_t					 index,
+									 u_int16_t					 level,
+									 u_int32_t					*insertNode );
 						 
 static OSErr		InsertNode 		(BTreeControlBlockPtr		 btreePtr,
 									 InsertKey					*key,
 									 BlockDescriptor			*rightNode,
-									 UInt32						 node,
-									 UInt16	 					 index,
-									 UInt32						*newNode,	
-									 UInt16						*newIndex,
+									 u_int32_t					 node,
+									 u_int16_t	 				 index,
+									 u_int32_t					*newNode,	
+									 u_int16_t					*newIndex,
 									 BlockDescriptor			*leftNode,
 									 Boolean					*updateParent,
 									 Boolean					*insertParent,
 									 Boolean					*rootSplit );
 									 
-static UInt16		GetKeyLength	(const BTreeControlBlock *btreePtr,
+static u_int16_t		GetKeyLength	(const BTreeControlBlock *btreePtr,
 									 const BTreeKey *key,
 									 Boolean forLeafNode );
 
@@ -189,20 +196,20 @@ Result:		noErr			- key found, index is record index
 OSStatus	SearchTree	(BTreeControlBlockPtr	 btreePtr,
 						 BTreeKeyPtr			 searchKey,
 						 TreePathTable			 treePathTable,
-						 UInt32					*nodeNum,
+						 u_int32_t				*nodeNum,
 						 BlockDescriptor		*nodePtr,
-						 UInt16					*returnIndex )
+						 u_int16_t				*returnIndex )
 {
 	OSStatus	err;
-	SInt16		level;					//	Expected depth of current node
-	UInt32		curNodeNum;				//	Current node we're searching
+	int16_t		level;					//	Expected depth of current node
+	u_int32_t	curNodeNum;				//	Current node we're searching
 	NodeRec		nodeRec;
-	UInt16		index;
+	u_int16_t	index;
 	Boolean		keyFound;
-	SInt8		nodeKind;				//	Kind of current node (index/leaf)
+	int8_t		nodeKind;				//	Kind of current node (index/leaf)
 	KeyPtr		keyPtr;
-	UInt8 *		dataPtr;
-	UInt16		dataSize;
+	u_int8_t *	dataPtr;
+	u_int16_t	dataSize;
 	
 	
 	curNodeNum		= btreePtr->rootNode;
@@ -302,7 +309,7 @@ OSStatus	SearchTree	(BTreeControlBlockPtr	 btreePtr,
         
         //	Get the child pointer out of this index node.  We're now done with the current
         //	node and can continue the search with the child node.
-        curNodeNum = *(UInt32 *)dataPtr;
+        curNodeNum = *(u_int32_t *)dataPtr;
         err = ReleaseNode (btreePtr, &nodeRec);
         if (err != noErr)
         {
@@ -344,13 +351,13 @@ ErrorExit:
 OSStatus	InsertTree ( BTreeControlBlockPtr		 btreePtr,
 						 TreePathTable				 treePathTable,
 						 KeyPtr						 keyPtr,
-						 UInt8 *					 recPtr,
-						 UInt16						 recSize,
+						 u_int8_t *					 recPtr,
+						 u_int16_t					 recSize,
 						 BlockDescriptor			*targetNode,
-						 UInt16						 index,
-						 UInt16						 level,
+						 u_int16_t					 index,
+						 u_int16_t					 level,
 						 Boolean					 replacingKey,
-						 UInt32						*insertNode )
+						 u_int32_t					*insertNode )
 {
 	InsertKey			primaryKey;
 	OSStatus			err;
@@ -377,15 +384,15 @@ OSStatus	InsertLevel (BTreeControlBlockPtr		 btreePtr,
 						 InsertKey					*primaryKey,
 						 InsertKey					*secondaryKey,
 						 BlockDescriptor			*targetNode,
-						 UInt16						 index,
-						 UInt16						 level,
-						 UInt32						*insertNode )
+						 u_int16_t					 index,
+						 u_int16_t					 level,
+						 u_int32_t					*insertNode )
 {
 	OSStatus			 err;
 	BlockDescriptor		 leftNode;
-	UInt32				 targetNodeNum;
-	UInt32				 newNodeNum;
-	UInt16				 newIndex;
+	u_int32_t			 targetNodeNum;
+	u_int32_t			 newNodeNum;
+	u_int16_t			 newIndex;
 	Boolean				 insertParent;
 	Boolean				 updateParent;
 	Boolean				 newRoot;
@@ -445,10 +452,10 @@ OSStatus	InsertLevel (BTreeControlBlockPtr		 btreePtr,
 	if ( insertParent || updateParent )
 	{
 		BlockDescriptor		parentNode;
-		UInt32				parentNodeNum;
+		u_int32_t			parentNodeNum;
 		KeyPtr				keyPtr;
-		UInt8 *				recPtr;
-		UInt16				recSize;
+		u_int8_t *			recPtr;
+		u_int16_t			recSize;
 		
 		parentNode.buffer = nil;
 		parentNode.blockHeader = nil;
@@ -480,7 +487,7 @@ OSStatus	InsertLevel (BTreeControlBlockPtr		 btreePtr,
 
 			// debug: check if ptr == targetNodeNum
 			GetRecordByIndex (btreePtr, parentNode.buffer, index, &keyPtr, &recPtr, &recSize);
-			PanicIf( (*(UInt32 *) recPtr) != targetNodeNum, "\p InsertLevel: parent ptr doesn't match target node!");
+			PanicIf( (*(u_int32_t *) recPtr) != targetNodeNum, "\p InsertLevel: parent ptr doesn't match target node!");
 			
 			// need to delete and re-insert this parent key/ptr
 			// we delete it here and it gets re-inserted in the
@@ -489,7 +496,7 @@ OSStatus	InsertLevel (BTreeControlBlockPtr		 btreePtr,
 	
 			primaryKey->keyPtr		 = (KeyPtr) GetRecordAddress( btreePtr, targetNode->buffer, 0 );
 			primaryKey->keyLength	 = GetKeyLength(btreePtr, primaryKey->keyPtr, false);
-			primaryKey->recPtr		 = (UInt8 *) &targetNodeNum;
+			primaryKey->recPtr		 = (u_int8_t *) &targetNodeNum;
 			primaryKey->recSize		 = sizeof(targetNodeNum);
 			primaryKey->replacingKey = kReplaceRecord;
 			primaryKey->skipRotate   = insertParent;		// don't rotate left if we have two inserts occuring
@@ -513,8 +520,8 @@ OSStatus	InsertLevel (BTreeControlBlockPtr		 btreePtr,
 			
 			insertKeyPtr->keyPtr		= (KeyPtr) GetRecordAddress (btreePtr, leftNode.buffer, 0);
 			insertKeyPtr->keyLength		= GetKeyLength(btreePtr, insertKeyPtr->keyPtr, false);
-			insertKeyPtr->recPtr		= (UInt8 *) &((NodeDescPtr)targetNode->buffer)->bLink;
-			insertKeyPtr->recSize		= sizeof(UInt32);
+			insertKeyPtr->recPtr		= (u_int8_t *) &((NodeDescPtr)targetNode->buffer)->bLink;
+			insertKeyPtr->recSize		= sizeof(u_int32_t);
 			insertKeyPtr->replacingKey	= kInsertRecord;
 			insertKeyPtr->skipRotate	= false;		// a rotate is OK during second insert
 		}	
@@ -551,11 +558,11 @@ static OSErr	InsertNode	(BTreeControlBlockPtr	 btreePtr,
 							 InsertKey				*key,
 
 							 BlockDescriptor		*rightNode,
-							 UInt32					 node,
-							 UInt16	 				 index,
+							 u_int32_t				 node,
+							 u_int16_t	 			 index,
 
-							 UInt32					*newNode,	
-							 UInt16					*newIndex,
+							 u_int32_t				*newNode,	
+							 u_int16_t				*newIndex,
 
 							 BlockDescriptor		*leftNode,
 							 Boolean				*updateParent,
@@ -563,8 +570,8 @@ static OSErr	InsertNode	(BTreeControlBlockPtr	 btreePtr,
 							 Boolean				*rootSplit )
 {
 	BlockDescriptor		*targetNode;
-	UInt32				 leftNodeNum;
-	UInt16				 recsRotated;
+	u_int32_t			 leftNodeNum;
+	u_int16_t			 recsRotated;
 	OSErr				 err;
 	Boolean				 recordFit;
 
@@ -679,13 +686,13 @@ Result:		noErr		- success
 OSStatus	DeleteTree			(BTreeControlBlockPtr		 btreePtr,
 								 TreePathTable				 treePathTable,
 								 BlockDescriptor			*targetNode,
-								 UInt16						 index,
-								 UInt16						 level )
+								 u_int16_t					 index,
+								 u_int16_t					 level )
 {
 	OSStatus			err;
 	BlockDescriptor		parentNode;
 	BTNodeDescriptor	*targetNodePtr;
-	UInt32				targetNodeNum;
+	u_int32_t			targetNodeNum;
 	Boolean				deleteRequired;
 	Boolean				updateRequired;
 
@@ -711,7 +718,7 @@ OSStatus	DeleteTree			(BTreeControlBlockPtr		 btreePtr,
 	if ( targetNodePtr->numRecords == 0 )	// did we delete the last record?
 	{
 		BlockDescriptor		siblingNode;
-		UInt32				siblingNodeNum;
+		u_int32_t			siblingNodeNum;
 
 		deleteRequired = true;
 		
@@ -802,22 +809,22 @@ OSStatus	DeleteTree			(BTreeControlBlockPtr		 btreePtr,
 		if ( updateRequired )
 		{
 			 KeyPtr		keyPtr;
-			 UInt8 *	recPtr;
-			 UInt16		recSize;
-			 UInt32		insertNode;
+			 u_int8_t *	recPtr;
+			 u_int16_t	recSize;
+			 u_int32_t	insertNode;
 			 
 			 // XXXdbg
 			 ModifyBlockStart(btreePtr->fileRefNum, &parentNode);
 
 			// debug: check if ptr == targetNodeNum
 			GetRecordByIndex (btreePtr, parentNode.buffer, index, &keyPtr, &recPtr, &recSize);
-			PanicIf( (*(UInt32 *) recPtr) != targetNodeNum, "\p DeleteTree: parent ptr doesn't match targetNodeNum!!");
+			PanicIf( (*(u_int32_t *) recPtr) != targetNodeNum, "\p DeleteTree: parent ptr doesn't match targetNodeNum!!");
 			
 			// need to delete and re-insert this parent key/ptr
 			DeleteRecord (btreePtr, parentNode.buffer, index);
 	
 			keyPtr = (KeyPtr) GetRecordAddress( btreePtr, targetNode->buffer, 0 );
-			recPtr = (UInt8 *) &targetNodeNum;
+			recPtr = (u_int8_t *) &targetNodeNum;
 			recSize = sizeof(targetNodeNum);
 			
 			err = InsertTree (btreePtr, treePathTable, keyPtr, recPtr, recSize,
@@ -854,8 +861,8 @@ static OSStatus	CollapseTree	(BTreeControlBlockPtr		btreePtr,
 							 	 BlockDescriptor			*blockPtr )
 {
 	OSStatus		err;
-	UInt32			originalRoot;
-	UInt32			nodeNum;
+	u_int32_t		originalRoot;
+	u_int32_t		nodeNum;
 	
 	originalRoot	= btreePtr->rootNode;
 	
@@ -937,23 +944,23 @@ Result:		noErr		- success
 static OSStatus	RotateLeft		(BTreeControlBlockPtr		 btreePtr,
 								 NodeDescPtr				 leftNode,
 								 NodeDescPtr				 rightNode,
-								 UInt16						 rightInsertIndex,
+								 u_int16_t					 rightInsertIndex,
 								 KeyPtr						 keyPtr,
-								 UInt8 *					 recPtr,
-								 UInt16						 recSize,
-								 UInt16						*insertIndex,
-								 UInt32						*insertNodeNum,
+								 u_int8_t *					 recPtr,
+								 u_int16_t					 recSize,
+								 u_int16_t					*insertIndex,
+								 u_int32_t					*insertNodeNum,
 								 Boolean					*recordFit,
-								 UInt16						*recsRotated )
+								 u_int16_t					*recsRotated )
 {
 	OSStatus			err;
-	SInt32				insertSize;
-	SInt32				nodeSize;
-	SInt32				leftSize, rightSize;
-	SInt32				moveSize = 0;
-	UInt16				keyLength;
-	UInt16				lengthFieldSize;
-	UInt16				index, moveIndex;
+	int32_t				insertSize;
+	int32_t				nodeSize;
+	int32_t				leftSize, rightSize;
+	int32_t				moveSize = 0;
+	u_int16_t			keyLength;
+	u_int16_t			lengthFieldSize;
+	u_int16_t			index, moveIndex;
 	Boolean				didItFit;
 
 	///////////////////// Determine If Record Will Fit //////////////////////////
@@ -962,11 +969,11 @@ static OSStatus	RotateLeft		(BTreeControlBlockPtr		 btreePtr,
 
 	// the key's length field is 8-bits in HFS and 16-bits in HFS+
 	if ( btreePtr->attributes & kBTBigKeysMask )
-		lengthFieldSize = sizeof(UInt16);
+		lengthFieldSize = sizeof(u_int16_t);
 	else
-		lengthFieldSize = sizeof(UInt8);
+		lengthFieldSize = sizeof(u_int8_t);
 
-	insertSize = keyLength + lengthFieldSize + recSize + sizeof(UInt16);
+	insertSize = keyLength + lengthFieldSize + recSize + sizeof(u_int16_t);
 
 	if ( M_IsOdd (insertSize) )
 		++insertSize;	// add pad byte;
@@ -1029,7 +1036,7 @@ static OSStatus	RotateLeft		(BTreeControlBlockPtr		 btreePtr,
 	{
 		if ( index == rightInsertIndex )	// insert new record in left node
 		{
-			UInt16	leftInsertIndex;
+			u_int16_t	leftInsertIndex;
 			
 			leftInsertIndex = leftNode->numRecords;
 
@@ -1099,18 +1106,18 @@ ErrorExit:
 static OSStatus	SplitLeft		(BTreeControlBlockPtr		 btreePtr,
 								 BlockDescriptor			*leftNode,
 								 BlockDescriptor			*rightNode,
-								 UInt32						 rightNodeNum,
-								 UInt16						 index,
+								 u_int32_t					 rightNodeNum,
+								 u_int16_t					 index,
 								 KeyPtr						 keyPtr,
-								 UInt8 *					 recPtr,
-								 UInt16						 recSize,
-								 UInt16						*insertIndex,
-								 UInt32						*insertNodeNum,
-								 UInt16						*recsRotated )
+								 u_int8_t *					 recPtr,
+								 u_int16_t					 recSize,
+								 u_int16_t					*insertIndex,
+								 u_int32_t					*insertNodeNum,
+								 u_int16_t					*recsRotated )
 {
 	OSStatus			err;
 	NodeDescPtr			left, right;
-	UInt32				newNodeNum;
+	u_int32_t			newNodeNum;
 	Boolean				recordFit;
 	
 	
@@ -1217,8 +1224,8 @@ static Boolean RotateRecordLeft (BTreeControlBlockPtr		btreePtr,
 								 NodeDescPtr				leftNode,
 							 	 NodeDescPtr				rightNode )
 {
-	UInt16		size;
-	UInt8 *		recPtr;
+	u_int16_t	size;
+	u_int8_t *	recPtr;
 	Boolean		recordFit;
 	
 	size	= GetRecordSize (btreePtr, rightNode, 0);
@@ -1243,10 +1250,10 @@ static OSStatus	AddNewRootNode	(BTreeControlBlockPtr	 btreePtr,
 {
 	OSStatus			err;
 	BlockDescriptor		rootNode;
-	UInt32				rootNum;
+	u_int32_t			rootNum;
 	KeyPtr				keyPtr;
 	Boolean				didItFit;
-	UInt16				keyLength;	
+	u_int16_t			keyLength;	
 	
 	rootNode.buffer = nil;
 	rootNode.blockHeader = nil;
@@ -1276,7 +1283,7 @@ static OSStatus	AddNewRootNode	(BTreeControlBlockPtr	 btreePtr,
 	keyLength = GetKeyLength(btreePtr, keyPtr, false);
 
 	didItFit = InsertKeyRecord ( btreePtr, rootNode.buffer, 0, keyPtr, keyLength,
-								 (UInt8 *) &rightNode->bLink, 4 );
+								 (u_int8_t *) &rightNode->bLink, 4 );
 
 	PanicIf ( !didItFit, "\pAddNewRootNode:InsertKeyRecord failed for left index record");
 
@@ -1287,7 +1294,7 @@ static OSStatus	AddNewRootNode	(BTreeControlBlockPtr	 btreePtr,
 	keyLength = GetKeyLength(btreePtr, keyPtr, false);
 
 	didItFit = InsertKeyRecord ( btreePtr, rootNode.buffer, 1, keyPtr, keyLength,
-								 (UInt8 *) &leftNode->fLink, 4 );
+								 (u_int8_t *) &leftNode->fLink, 4 );
 
 	PanicIf ( !didItFit, "\pAddNewRootNode:InsertKeyRecord failed for right index record");
 
@@ -1313,9 +1320,9 @@ ErrorExit:
 }
 
 
-static UInt16	GetKeyLength ( const BTreeControlBlock *btreePtr, const BTreeKey *key, Boolean forLeafNode )
+static u_int16_t	GetKeyLength ( const BTreeControlBlock *btreePtr, const BTreeKey *key, Boolean forLeafNode )
 {
-	UInt16 length;
+	u_int16_t length;
 
 	if ( forLeafNode || btreePtr->attributes & kBTVariableIndexKeysMask )
 		length = KeyLength (btreePtr, key);		// just use actual key length
diff --git a/bsd/hfs/hfscommon/Catalog/CatalogUtilities.c b/bsd/hfs/hfscommon/Catalog/CatalogUtilities.c
index cad8b871e..1e24463f1 100644
--- a/bsd/hfs/hfscommon/Catalog/CatalogUtilities.c
+++ b/bsd/hfs/hfscommon/Catalog/CatalogUtilities.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2002, 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <sys/param.h>
 #include <sys/utfconv.h>
@@ -40,7 +46,7 @@
 
 OSErr
 LocateCatalogNode(const ExtendedVCB *volume, HFSCatalogNodeID folderID, const CatalogName *name,
-					UInt32 hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, UInt32 *newHint)
+					u_int32_t hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, u_int32_t *newHint)
 {
 	OSErr				result;
 	CatalogName 		*nodeName = NULL;	/* To ward off uninitialized use warnings from compiler */
@@ -84,17 +90,19 @@ LocateCatalogNode(const ExtendedVCB *volume, HFSCatalogNodeID folderID, const Ca
 //
 
 OSErr
-LocateCatalogNodeByKey(const ExtendedVCB *volume, UInt32 hint, CatalogKey *keyPtr,
-						CatalogRecord *dataPtr, UInt32 *newHint)
+LocateCatalogNodeByKey(const ExtendedVCB *volume, u_int32_t hint, CatalogKey *keyPtr,
+						CatalogRecord *dataPtr, u_int32_t *newHint)
 {
 	OSErr				result;
 	CatalogName 		*nodeName = NULL;
 	HFSCatalogNodeID	threadParentID;
-	UInt16 tempSize;
+	u_int16_t tempSize;
 	FSBufferDescriptor	 btRecord;
-	BTreeIterator		 searchIterator = {0};
+	BTreeIterator		 searchIterator; 
 	FCB			*fcb;
 
+	bzero(&searchIterator, sizeof(searchIterator));
+
 	fcb = GetFileControlBlock(volume->catalogRefNum);
 
 	btRecord.bufferAddress = dataPtr;
@@ -155,11 +163,11 @@ LocateCatalogNodeByKey(const ExtendedVCB *volume, UInt32 hint, CatalogKey *keyPt
 
 OSErr
 LocateCatalogRecord(const ExtendedVCB *volume, HFSCatalogNodeID folderID, const CatalogName *name,
-					UInt32 hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, UInt32 *newHint)
+					u_int32_t hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, u_int32_t *newHint)
 {
 	OSErr			result;
 	CatalogKey		tempKey;	// 518 bytes
-	UInt16			tempSize;
+	u_int16_t		tempSize;
 
 	BuildCatalogKey(folderID, name, (volume->vcbSigWord == kHFSPlusSigWord), &tempKey);
 
@@ -211,15 +219,15 @@ BuildCatalogKey(HFSCatalogNodeID parentID, const CatalogName *cName, Boolean isH
 }
 
 OSErr
-BuildCatalogKeyUTF8(ExtendedVCB *volume, HFSCatalogNodeID parentID, const char *name, UInt32 nameLength,
-		    CatalogKey *key, UInt32 *textEncoding)
+BuildCatalogKeyUTF8(ExtendedVCB *volume, HFSCatalogNodeID parentID, const unsigned char *name, u_int32_t nameLength,
+		    CatalogKey *key, u_int32_t *textEncoding)
 {
 	OSErr err = 0;
 
     if ( name == NULL)
         nameLength = 0;
     else if (nameLength == kUndefinedStrLen)
-        nameLength = strlen(name);
+        nameLength = strlen((const char *)name);
 
 	if ( volume->vcbSigWord == kHFSPlusSigWord ) {
 		size_t unicodeBytes = 0;
@@ -292,7 +300,7 @@ FlushCatalog(ExtendedVCB *volume)
 		if ( 0 /*fcb->fcbFlags & fcbModifiedMask*/ )
 		{
 			HFS_MOUNT_LOCK(volume, TRUE);
-			volume->vcbFlags |= 0xFF00;		// Mark the VCB dirty
+			MarkVCBDirty(volume);	// Mark the VCB dirty
 			volume->vcbLsMod = GetTimeUTC();	// update last modified date
 			HFS_MOUNT_UNLOCK(volume, TRUE);
 
@@ -329,7 +337,7 @@ UpdateCatalogName(ConstStr31Param srcName, Str31 destName)
 void
 CopyCatalogName(const CatalogName *srcName, CatalogName *dstName, Boolean isHFSPLus)
 {
-	UInt32	length;
+	u_int32_t	length;
 	
 	if ( srcName == NULL )
 	{
@@ -341,7 +349,7 @@ CopyCatalogName(const CatalogName *srcName, CatalogName *dstName, Boolean isHFSP
 	if (isHFSPLus)
 		length = sizeof(UniChar) * (srcName->ustr.length + 1);
 	else
-		length = sizeof(UInt8) + srcName->pstr[0];
+		length = sizeof(u_int8_t) + srcName->pstr[0];
 
 	if ( length > 1 )
 		BlockMoveData(srcName, dstName, length);
diff --git a/bsd/hfs/hfscommon/Catalog/FileIDsServices.c b/bsd/hfs/hfscommon/Catalog/FileIDsServices.c
index 812f3e58c..ab2cfdb29 100644
--- a/bsd/hfs/hfscommon/Catalog/FileIDsServices.c
+++ b/bsd/hfs/hfscommon/Catalog/FileIDsServices.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include "../../hfs_macos_defs.h"
@@ -35,24 +41,24 @@ struct ExtentsRecBuffer {
 typedef struct ExtentsRecBuffer ExtentsRecBuffer;
 
 
-static UInt32 CheckExtents( void *extents, UInt32 blocks, Boolean isHFSPlus );
-static OSErr  DeleteExtents( ExtendedVCB *vcb, UInt32 fileNumber, Boolean isHFSPlus );
-static OSErr  MoveExtents( ExtendedVCB *vcb, UInt32 srcFileID, UInt32 destFileID, Boolean isHFSPlus );
+static u_int32_t CheckExtents( void *extents, u_int32_t blocks, Boolean isHFSPlus );
+static OSErr  DeleteExtents( ExtendedVCB *vcb, u_int32_t fileNumber, Boolean isHFSPlus );
+static OSErr  MoveExtents( ExtendedVCB *vcb, u_int32_t srcFileID, u_int32_t destFileID, Boolean isHFSPlus );
 static void  CopyCatalogNodeInfo( CatalogRecord *src, CatalogRecord *dest );
 static void  CopyBigCatalogNodeInfo( CatalogRecord *src, CatalogRecord *dest );
-static void  CopyExtentInfo( ExtentKey *key, ExtentRecord *data, ExtentsRecBuffer *buffer, UInt16 bufferCount );
+static void  CopyExtentInfo( ExtentKey *key, ExtentRecord *data, ExtentsRecBuffer *buffer, u_int16_t bufferCount );
 
 
 
-OSErr ExchangeFileIDs( ExtendedVCB *vcb, ConstUTF8Param srcName, ConstUTF8Param destName, HFSCatalogNodeID srcID, HFSCatalogNodeID destID, UInt32 srcHint, UInt32 destHint )
+OSErr ExchangeFileIDs( ExtendedVCB *vcb, ConstUTF8Param srcName, ConstUTF8Param destName, HFSCatalogNodeID srcID, HFSCatalogNodeID destID, u_int32_t srcHint, u_int32_t destHint )
 {
 	CatalogKey	srcKey;		// 518 bytes
 	CatalogKey	destKey;	// 518 bytes
 	CatalogRecord	srcData;	// 520 bytes
 	CatalogRecord	destData;	// 520 bytes
 	CatalogRecord	swapData;	// 520 bytes
-	SInt16		numSrcExtentBlocks;
-	SInt16		numDestExtentBlocks;
+	int16_t		numSrcExtentBlocks;
+	int16_t		numDestExtentBlocks;
 	OSErr		err;
 	Boolean		isHFSPlus = ( vcb->vcbSigWord == kHFSPlusSigWord );
 
@@ -367,7 +373,7 @@ static void  CopyBigCatalogNodeInfo( CatalogRecord *src, CatalogRecord *dest )
 }
 
 
-static OSErr  MoveExtents( ExtendedVCB *vcb, UInt32 srcFileID, UInt32 destFileID, Boolean isHFSPlus )
+static OSErr  MoveExtents( ExtendedVCB *vcb, u_int32_t srcFileID, u_int32_t destFileID, Boolean isHFSPlus )
 {
 	FCB *				fcb;
 	ExtentsRecBuffer	extentsBuffer[kNumExtentsToCache];
@@ -375,9 +381,9 @@ static OSErr  MoveExtents( ExtendedVCB *vcb, UInt32 srcFileID, UInt32 destFileID
 	ExtentRecord		extentData;
 	BTreeIterator		btIterator;
 	FSBufferDescriptor	btRecord;
-	UInt16				btKeySize;
-	UInt16				btRecordSize;
-	SInt16				i, j;
+	u_int16_t			btKeySize;
+	u_int16_t			btRecordSize;
+	int16_t				i, j;
 	OSErr				err;
 	
 
@@ -525,7 +531,7 @@ static OSErr  MoveExtents( ExtendedVCB *vcb, UInt32 srcFileID, UInt32 destFileID
 }
 
 
-static void  CopyExtentInfo( ExtentKey *key, ExtentRecord *data, ExtentsRecBuffer *buffer, UInt16 bufferCount )
+static void  CopyExtentInfo( ExtentKey *key, ExtentRecord *data, ExtentsRecBuffer *buffer, u_int16_t bufferCount )
 {
 	BlockMoveData( key, &(buffer[bufferCount].extentKey), sizeof( ExtentKey ) );
 	BlockMoveData( data, &(buffer[bufferCount].extentData), sizeof( ExtentRecord ) );
@@ -533,14 +539,14 @@ static void  CopyExtentInfo( ExtentKey *key, ExtentRecord *data, ExtentsRecBuffe
 
 
 //--	Delete all extents in extent file that have the ID given.
-static OSErr  DeleteExtents( ExtendedVCB *vcb, UInt32 fileID, Boolean isHFSPlus )
+static OSErr  DeleteExtents( ExtendedVCB *vcb, u_int32_t fileID, Boolean isHFSPlus )
 {
 	FCB *				fcb;
 	ExtentKey *			extentKeyPtr;
 	ExtentRecord		extentData;
 	BTreeIterator		btIterator;
 	FSBufferDescriptor	btRecord;
-	UInt16				btRecordSize;
+	u_int16_t			btRecordSize;
 	OSErr				err;
 
 	fcb = GetFileControlBlock(vcb->extentsRefNum);
@@ -611,10 +617,10 @@ static OSErr  DeleteExtents( ExtendedVCB *vcb, UInt32 fileID, Boolean isHFSPlus
 
 
 //	Check if there are extents represented in the extents overflow file.
-static UInt32  CheckExtents( void *extents, UInt32 totalBlocks, Boolean isHFSPlus )
+static u_int32_t  CheckExtents( void *extents, u_int32_t totalBlocks, Boolean isHFSPlus )
 {
-	UInt32		extentAllocationBlocks;
-	UInt16		i;
+	u_int32_t		extentAllocationBlocks;
+	u_int16_t		i;
 
 
 	if ( totalBlocks == 0 )
diff --git a/bsd/hfs/hfscommon/Misc/BTreeWrapper.c b/bsd/hfs/hfscommon/Misc/BTreeWrapper.c
index 767e54bec..eb7fc628f 100644
--- a/bsd/hfs/hfscommon/Misc/BTreeWrapper.c
+++ b/bsd/hfs/hfscommon/Misc/BTreeWrapper.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000,2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000, 2002, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include "../headers/BTreesPrivate.h"
@@ -25,12 +31,12 @@
 
 // local routines
 static OSErr	CheckBTreeKey(const BTreeKey *key, const BTreeControlBlock *btcb);
-static Boolean	ValidHFSRecord(const void *record, const BTreeControlBlock *btcb, UInt16 recordSize);
+static Boolean	ValidHFSRecord(const void *record, const BTreeControlBlock *btcb, u_int16_t recordSize);
 
 
 
 
-OSErr SearchBTreeRecord(FileReference refNum, const void* key, UInt32 hint, void* foundKey, void* data, UInt16 *dataSize, UInt32 *newHint)
+OSErr SearchBTreeRecord(__unused FileReference refNum, __unused const void* key, __unused u_int32_t hint, __unused void* foundKey, __unused void* data, __unused u_int16_t *dataSize, __unused u_int32_t *newHint)
 {
 	panic("SearchBTreeRecord is dead code!");
 	return (-1);
@@ -88,7 +94,7 @@ ErrorExit:
 }
 
 
-OSErr ReplaceBTreeRecord(FileReference refNum, const void* key, UInt32 hint, void *newData, UInt16 dataSize, UInt32 *newHint)
+OSErr ReplaceBTreeRecord(FileReference refNum, const void* key, u_int32_t hint, void *newData, u_int16_t dataSize, u_int32_t *newHint)
 {
 	FSBufferDescriptor	btRecord;
 	BTreeIterator		iterator;
@@ -106,10 +112,10 @@ OSErr ReplaceBTreeRecord(FileReference refNum, const void* key, UInt32 hint, voi
 
 	iterator.hint.nodeNum = hint;
 
-	result = CheckBTreeKey((BTreeKey *) key, btcb);
+	result = CheckBTreeKey((const BTreeKey *) key, btcb);
 	ExitOnError(result);
 
-	BlockMoveData(key, &iterator.key, CalcKeySize(btcb, (BTreeKey *) key));		// should we range check against maxkeylen?
+	BlockMoveData(key, &iterator.key, CalcKeySize(btcb, (const BTreeKey *) key));		// should we range check against maxkeylen?
 
 	if ( DEBUG_BUILD && !ValidHFSRecord(newData, btcb, dataSize) )
 		DebugStr("\pReplaceBTreeRecord: bad record?");
@@ -129,7 +135,7 @@ ErrorExit:
 
 static OSErr CheckBTreeKey(const BTreeKey *key, const BTreeControlBlock *btcb)
 {
-	UInt16	keyLen;
+	u_int16_t	keyLen;
 	
 	if ( btcb->attributes & kBTBigKeysMask )
 		keyLen = key->length16;
@@ -147,9 +153,9 @@ static OSErr CheckBTreeKey(const BTreeKey *key, const BTreeControlBlock *btcb)
 }
 
 
-static Boolean ValidHFSRecord(const void *record, const BTreeControlBlock *btcb, UInt16 recordSize)
+static Boolean ValidHFSRecord(const void *record, const BTreeControlBlock *btcb, u_int16_t recordSize)
 {
-	UInt32			cNodeID;
+	u_int32_t			cNodeID;
 	
 	if ( btcb->maxKeyLength == kHFSExtentKeyMaximumLength )
 	{
@@ -161,7 +167,7 @@ static Boolean ValidHFSRecord(const void *record, const BTreeControlBlock *btcb,
 	}
 	else // Catalog record
 	{
-		CatalogRecord *catalogRecord = (CatalogRecord*) record;
+		const CatalogRecord *catalogRecord = (const CatalogRecord*) record;
 
 		switch(catalogRecord->recordType)
 		{
@@ -199,7 +205,7 @@ static Boolean ValidHFSRecord(const void *record, const BTreeControlBlock *btcb,
 	
 			case kHFSFileRecord:
 			{
-//				UInt16					i;
+//				u_int16_t					i;
 				HFSExtentDescriptor	*dataExtent;
 				HFSExtentDescriptor	*rsrcExtent;
 				
@@ -241,7 +247,7 @@ static Boolean ValidHFSRecord(const void *record, const BTreeControlBlock *btcb,
 	
 			case kHFSPlusFileRecord:
 			{
-//				UInt16					i;
+//				u_int16_t					i;
 				HFSPlusExtentDescriptor	*dataExtent;
 				HFSPlusExtentDescriptor	*rsrcExtent;
 				
diff --git a/bsd/hfs/hfscommon/Misc/FileExtentMapping.c b/bsd/hfs/hfscommon/Misc/FileExtentMapping.c
index 76c6a407a..bdfe2f2de 100644
--- a/bsd/hfs/hfscommon/Misc/FileExtentMapping.c
+++ b/bsd/hfs/hfscommon/Misc/FileExtentMapping.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -91,7 +97,7 @@ Internal Routines:
 					and was in the extents file, then delete the record instead.
 */
 
-static const SInt64 kTwoGigabytes = 0x80000000LL;
+static const int64_t kTwoGigabytes = 0x80000000LL;
 
 enum
 {
@@ -108,25 +114,25 @@ static OSErr HFSPlusToHFSExtents(
 
 static OSErr FindExtentRecord(
 	const ExtendedVCB		*vcb,
-	UInt8					forkType,
-	UInt32					fileID,
-	UInt32					startBlock,
+	u_int8_t				forkType,
+	u_int32_t				fileID,
+	u_int32_t				startBlock,
 	Boolean					allowPrevious,
 	HFSPlusExtentKey		*foundKey,
 	HFSPlusExtentRecord		foundData,
-	UInt32					*foundHint);
+	u_int32_t				*foundHint);
 
 static OSErr DeleteExtentRecord(
 	const ExtendedVCB		*vcb,
-	UInt8					forkType,
-	UInt32					fileID,
-	UInt32					startBlock);
+	u_int8_t				forkType,
+	u_int32_t				fileID,
+	u_int32_t				startBlock);
 
 static OSErr CreateExtentRecord(
 	ExtendedVCB		*vcb,
 	HFSPlusExtentKey		*key,
 	HFSPlusExtentRecord		extents,
-	UInt32					*hint);
+	u_int32_t				*hint);
 
 
 static OSErr GetFCBExtentRecord(
@@ -136,40 +142,40 @@ static OSErr GetFCBExtentRecord(
 static OSErr SearchExtentFile(
 	ExtendedVCB		*vcb,
 	const FCB	 			*fcb,
-	SInt64 					filePosition,
+	int64_t 				filePosition,
 	HFSPlusExtentKey		*foundExtentKey,
 	HFSPlusExtentRecord		foundExtentData,
-	UInt32					*foundExtentDataIndex,
-	UInt32					*extentBTreeHint,
-	UInt32					*endingFABNPlusOne );
+	u_int32_t				*foundExtentDataIndex,
+	u_int32_t				*extentBTreeHint,
+	u_int32_t				*endingFABNPlusOne );
 
 static OSErr SearchExtentRecord(
 	ExtendedVCB		*vcb,
-	UInt32					searchFABN,
+	u_int32_t				searchFABN,
 	const HFSPlusExtentRecord	extentData,
-	UInt32					extentDataStartFABN,
-	UInt32					*foundExtentDataOffset,
-	UInt32					*endingFABNPlusOne,
+	u_int32_t				extentDataStartFABN,
+	u_int32_t				*foundExtentDataOffset,
+	u_int32_t				*endingFABNPlusOne,
 	Boolean					*noMoreExtents);
 
 static OSErr ReleaseExtents(
 	ExtendedVCB				*vcb,
 	const HFSPlusExtentRecord	extentRecord,
-	UInt32					*numReleasedAllocationBlocks,
+	u_int32_t				*numReleasedAllocationBlocks,
 	Boolean 				*releasedLastExtent);
 
 static OSErr DeallocateFork(
 	ExtendedVCB 		*vcb,
 	HFSCatalogNodeID	fileID,
-	UInt8			forkType,
+	u_int8_t			forkType,
 	HFSPlusExtentRecord	catalogExtents,
 	Boolean *		recordDeleted);
 
 static OSErr TruncateExtents(
 	ExtendedVCB			*vcb,
-	UInt8				forkType,
-	UInt32				fileID,
-	UInt32				startBlock,
+	u_int8_t			forkType,
+	u_int32_t			fileID,
+	u_int32_t			startBlock,
 	Boolean *			recordDeleted);
 
 static OSErr UpdateExtentRecord (
@@ -177,12 +183,12 @@ static OSErr UpdateExtentRecord (
 	FCB						*fcb,
 	const HFSPlusExtentKey	*extentFileKey,
 	const HFSPlusExtentRecord	extentData,
-	UInt32					extentBTreeHint);
+	u_int32_t					extentBTreeHint);
 
 static Boolean ExtentsAreIntegral(
 	const HFSPlusExtentRecord extentRecord,
-	UInt32		mask,
-	UInt32		*blocksChecked,
+	u_int32_t	mask,
+	u_int32_t	*blocksChecked,
 	Boolean		*checkedLastExtent);
 
 //_________________________________________________________________________________
@@ -209,19 +215,19 @@ static Boolean ExtentsAreIntegral(
 //_________________________________________________________________________________
 static OSErr FindExtentRecord(
 	const ExtendedVCB	*vcb,
-	UInt8				forkType,
-	UInt32				fileID,
-	UInt32				startBlock,
+	u_int8_t			forkType,
+	u_int32_t			fileID,
+	u_int32_t			startBlock,
 	Boolean				allowPrevious,
 	HFSPlusExtentKey	*foundKey,
 	HFSPlusExtentRecord	foundData,
-	UInt32				*foundHint)
+	u_int32_t			*foundHint)
 {
 	FCB *				fcb;
 	BTreeIterator		*btIterator;
 	FSBufferDescriptor	btRecord;
 	OSErr				err;
-	UInt16				btRecordSize;
+	u_int16_t			btRecordSize;
 	
 	err = noErr;
 	if (foundHint)
@@ -263,7 +269,7 @@ static OSErr FindExtentRecord(
 		}
 
 		if (err == noErr) {
-			UInt16	i;
+			u_int16_t	i;
 			
 			// Copy the found key back for the caller
 			if (foundKey) {
@@ -341,11 +347,11 @@ static OSErr CreateExtentRecord(
 	ExtendedVCB	*vcb,
 	HFSPlusExtentKey	*key,
 	HFSPlusExtentRecord	extents,
-	UInt32				*hint)
+	u_int32_t			*hint)
 {
 	BTreeIterator * btIterator;
 	FSBufferDescriptor	btRecord;
-	UInt16  btRecordSize;
+	u_int16_t  btRecordSize;
 	int  lockflags;
 	OSErr  err;
 	
@@ -407,9 +413,9 @@ static OSErr CreateExtentRecord(
 
 static OSErr DeleteExtentRecord(
 	const ExtendedVCB	*vcb,
-	UInt8				forkType,
-	UInt32				fileID,
-	UInt32				startBlock)
+	u_int8_t			forkType,
+	u_int32_t			fileID,
+	u_int32_t			startBlock)
 {
 	BTreeIterator * btIterator;
 	OSErr				err;
@@ -466,17 +472,17 @@ OSErr MapFileBlockC (
 	size_t			*availableBytes)	// number of contiguous bytes (up to numberOfBytes)
 {
 	OSErr				err;
-	UInt32				allocBlockSize;			//	Size of the volume's allocation block
-	UInt32			sectorSize;
+	u_int32_t			allocBlockSize;			//	Size of the volume's allocation block
+	u_int32_t			sectorSize;
 	HFSPlusExtentKey	foundKey;
 	HFSPlusExtentRecord	foundData;
-	UInt32				foundIndex;
-	UInt32				hint;
-	UInt32				firstFABN;				// file allocation block of first block in found extent
-	UInt32				nextFABN;				// file allocation block of block after end of found extent
+	u_int32_t			foundIndex;
+	u_int32_t			hint;
+	u_int32_t			firstFABN;				// file allocation block of first block in found extent
+	u_int32_t			nextFABN;				// file allocation block of block after end of found extent
 	off_t				dataEnd;				// (offset) end of range that is contiguous
-	UInt32				sectorsPerBlock;		// Number of sectors per allocation block
-	UInt32				startBlock;				// volume allocation block corresponding to firstFABN
+	u_int32_t			sectorsPerBlock;		// Number of sectors per allocation block
+	u_int32_t			startBlock;				// volume allocation block corresponding to firstFABN
 	daddr64_t			temp;
 	off_t				tmpOff;
 
@@ -548,11 +554,11 @@ OSErr MapFileBlockC (
 static OSErr ReleaseExtents(
 	ExtendedVCB 			*vcb,
 	const HFSPlusExtentRecord	extentRecord,
-	UInt32					*numReleasedAllocationBlocks,
+	u_int32_t				*numReleasedAllocationBlocks,
 	Boolean 				*releasedLastExtent)
 {
-	UInt32	extentIndex;
-	UInt32	numberOfExtents;
+	u_int32_t	extentIndex;
+	u_int32_t	numberOfExtents;
 	OSErr	err = noErr;
 	
 	*numReleasedAllocationBlocks = 0;
@@ -565,7 +571,7 @@ static OSErr ReleaseExtents(
 
 	for( extentIndex = 0; extentIndex < numberOfExtents; extentIndex++)
 	{
-		UInt32	numAllocationBlocks;
+		u_int32_t	numAllocationBlocks;
 		
 		// Loop over the extent record and release the blocks associated with each extent.
 		
@@ -604,15 +610,15 @@ static OSErr ReleaseExtents(
 
 static OSErr TruncateExtents(
 	ExtendedVCB		*vcb,
-	UInt8			forkType,
-	UInt32			fileID,
-	UInt32			startBlock,
+	u_int8_t		forkType,
+	u_int32_t		fileID,
+	u_int32_t		startBlock,
 	Boolean *		recordDeleted)
 {
 	OSErr				err;
-	UInt32				numberExtentsReleased;
+	u_int32_t			numberExtentsReleased;
 	Boolean				releasedLastExtent;
-	UInt32				hint;
+	u_int32_t			hint;
 	HFSPlusExtentKey	key;
 	HFSPlusExtentRecord	extents;
 	int  lockflags;
@@ -659,12 +665,12 @@ static OSErr TruncateExtents(
 static OSErr DeallocateFork(
 	ExtendedVCB 		*vcb,
 	HFSCatalogNodeID	fileID,
-	UInt8			forkType,
+	u_int8_t			forkType,
 	HFSPlusExtentRecord	catalogExtents,
 	Boolean *		recordDeleted) /* true if a record was deleted */
 {
 	OSErr				err;
-	UInt32				numReleasedAllocationBlocks;
+	u_int32_t			numReleasedAllocationBlocks;
 	Boolean				releasedLastExtent;
 	
 	//	Release the catalog extents
@@ -718,9 +724,9 @@ OSErr FlushExtentFile( ExtendedVCB *vcb )
 //
 
 __private_extern__
-SInt32 CompareExtentKeys( const HFSExtentKey *searchKey, const HFSExtentKey *trialKey )
+int32_t CompareExtentKeys( const HFSExtentKey *searchKey, const HFSExtentKey *trialKey )
 {
-	SInt32	result;		//	± 1
+	int32_t	result;		//	± 1
 	
 	#if DEBUG_BUILD
 		if (searchKey->keyLength != kHFSExtentKeyMaximumLength)
@@ -782,9 +788,9 @@ SInt32 CompareExtentKeys( const HFSExtentKey *searchKey, const HFSExtentKey *tri
 //
 
 __private_extern__
-SInt32 CompareExtentKeysPlus( const HFSPlusExtentKey *searchKey, const HFSPlusExtentKey *trialKey )
+int32_t CompareExtentKeysPlus( const HFSPlusExtentKey *searchKey, const HFSPlusExtentKey *trialKey )
 {
-	SInt32	result;		//	± 1
+	int32_t	result;		//	± 1
 	
 	#if DEBUG_BUILD
 		if (searchKey->keyLength != kHFSPlusExtentKeyMaximumLength)
@@ -844,18 +850,18 @@ SInt32 CompareExtentKeysPlus( const HFSPlusExtentKey *searchKey, const HFSPlusEx
  */
 __private_extern__
 int
-AddFileExtent(ExtendedVCB *vcb, FCB *fcb, UInt32 startBlock, UInt32 blockCount)
+AddFileExtent(ExtendedVCB *vcb, FCB *fcb, u_int32_t startBlock, u_int32_t blockCount)
 {
 	HFSPlusExtentKey foundKey;
 	HFSPlusExtentRecord foundData;
-	UInt32 foundIndex;
-	UInt32 hint;
-	UInt32 nextBlock;
-	SInt64 peof;
+	u_int32_t foundIndex;
+	u_int32_t hint;
+	u_int32_t nextBlock;
+	int64_t peof;
 	int i;
 	int error;
 
-	peof = (SInt64)(fcb->ff_blocks + blockCount) * (SInt64)vcb->blockSize;
+	peof = (int64_t)(fcb->ff_blocks + blockCount) * (int64_t)vcb->blockSize;
 
 	error = SearchExtentFile(vcb, fcb, peof-1, &foundKey, foundData, &foundIndex, &hint, &nextBlock);
 	if (error != fxRangeErr)
@@ -916,32 +922,33 @@ __private_extern__
 OSErr ExtendFileC (
 	ExtendedVCB		*vcb,				// volume that file resides on
 	FCB				*fcb,				// FCB of file to truncate
-	SInt64			bytesToAdd,			// number of bytes to allocate
-	UInt32			blockHint,			// desired starting allocation block
-	UInt32			flags,				// EFContig and/or EFAll
-	SInt64			*actualBytesAdded)	// number of bytes actually allocated
+	int64_t			bytesToAdd,			// number of bytes to allocate
+	u_int32_t		blockHint,			// desired starting allocation block
+	u_int32_t		flags,				// EFContig and/or EFAll
+	int64_t			*actualBytesAdded)	// number of bytes actually allocated
 {
 	OSErr				err;
-	UInt32				volumeBlockSize;
-	SInt64				blocksToAdd;
-	SInt64				bytesThisExtent;
+	u_int32_t			volumeBlockSize;
+	int64_t				blocksToAdd;
+	int64_t				bytesThisExtent;
 	HFSPlusExtentKey	foundKey;
 	HFSPlusExtentRecord	foundData;
-	UInt32				foundIndex;
-	UInt32				hint;
-	UInt32				nextBlock;
-	UInt32				startBlock;
+	u_int32_t			foundIndex;
+	u_int32_t			hint;
+	u_int32_t			nextBlock;
+	u_int32_t			startBlock;
 	Boolean				allOrNothing;
 	Boolean				forceContig;
 	Boolean				wantContig;
 	Boolean				useMetaZone;
 	Boolean				needsFlush;
-	UInt32				actualStartBlock;
-	UInt32				actualNumBlocks;
-	UInt32				numExtentsPerRecord;
-	SInt64				maximumBytes;
-	SInt64				peof;
-	UInt32				prevblocks;
+	u_int32_t			actualStartBlock;
+	u_int32_t			actualNumBlocks;
+	u_int32_t			numExtentsPerRecord;
+	int64_t				maximumBytes;
+	int64_t 			availbytes;
+	int64_t				peof;
+	u_int32_t			prevblocks;
 	
 
 	needsFlush = false;
@@ -962,7 +969,7 @@ OSErr ExtendFileC (
 	if (vcb->vcbSigWord == kHFSSigWord) {
 		if (bytesToAdd >=  kTwoGigabytes)
 			goto Overflow;
-		if ((((SInt64)fcb->ff_blocks * (SInt64)volumeBlockSize) + bytesToAdd) >= kTwoGigabytes)
+		if ((((int64_t)fcb->ff_blocks * (int64_t)volumeBlockSize) + bytesToAdd) >= kTwoGigabytes)
 			goto Overflow;
 	}
 	//
@@ -970,14 +977,14 @@ OSErr ExtendFileC (
 	//	Round up the number of desired bytes to add.
 	//
 	blocksToAdd = howmany(bytesToAdd, volumeBlockSize);
-	bytesToAdd = (SInt64)((SInt64)blocksToAdd * (SInt64)volumeBlockSize);
+	bytesToAdd = (int64_t)((int64_t)blocksToAdd * (int64_t)volumeBlockSize);
 
 	/*
 	 * For deferred allocations just reserve the blocks.
 	 */
 	if ((flags & kEFDeferMask)
 	&&  (vcb->vcbSigWord == kHFSPlusSigWord)
-	&&  (bytesToAdd < (SInt64)HFS_MAX_DEFERED_ALLOC)
+	&&  (bytesToAdd < (int64_t)HFS_MAX_DEFERED_ALLOC)
 	&&  (blocksToAdd < hfs_freeblks(VCBTOHFS(vcb), 1))) {
 		HFS_MOUNT_LOCK(vcb, TRUE);
 		vcb->loanedBlocks += blocksToAdd;
@@ -999,7 +1006,7 @@ OSErr ExtendFileC (
 
 		loanedBlocks = fcb->ff_unallocblocks;
 		blocksToAdd += loanedBlocks;
-		bytesToAdd = (SInt64)blocksToAdd * (SInt64)volumeBlockSize;
+		bytesToAdd = (int64_t)blocksToAdd * (int64_t)volumeBlockSize;
 		FTOC(fcb)->c_blocks -= loanedBlocks;
 		fcb->ff_blocks -= loanedBlocks;
 		fcb->ff_unallocblocks  = 0;
@@ -1015,9 +1022,9 @@ OSErr ExtendFileC (
 	//	rounded up to a multiple of the clump size.
 	//
 	if ((vcb->vcbClpSiz > (int32_t)volumeBlockSize)
-	&&  (bytesToAdd < (SInt64)HFS_MAX_DEFERED_ALLOC)
+	&&  (bytesToAdd < (int64_t)HFS_MAX_DEFERED_ALLOC)
 	&&  (flags & kEFNoClumpMask) == 0) {
-		maximumBytes = (SInt64)howmany(bytesToAdd, vcb->vcbClpSiz);
+		maximumBytes = (int64_t)howmany(bytesToAdd, vcb->vcbClpSiz);
 		maximumBytes *= vcb->vcbClpSiz;
 	} else {
 		maximumBytes = bytesToAdd;
@@ -1027,7 +1034,7 @@ OSErr ExtendFileC (
 	//	Compute new physical EOF, rounded up to a multiple of a block.
 	//
 	if ( (vcb->vcbSigWord == kHFSSigWord) &&		//	Too big?
-		 ((((SInt64)fcb->ff_blocks * (SInt64)volumeBlockSize) + bytesToAdd) >= kTwoGigabytes) ) {
+		 ((((int64_t)fcb->ff_blocks * (int64_t)volumeBlockSize) + bytesToAdd) >= kTwoGigabytes) ) {
 		if (allOrNothing)					// Yes, must they have it all?
 			goto Overflow;						// Yes, can't have it
 		else {
@@ -1049,7 +1056,7 @@ OSErr ExtendFileC (
 	//
 	//	See if there are already enough blocks allocated to the file.
 	//
-	peof = ((SInt64)fcb->ff_blocks * (SInt64)volumeBlockSize) + bytesToAdd;  // potential new PEOF
+	peof = ((int64_t)fcb->ff_blocks * (int64_t)volumeBlockSize) + bytesToAdd;  // potential new PEOF
 	err = SearchExtentFile(vcb, fcb, peof-1, &foundKey, foundData, &foundIndex, &hint, &nextBlock);
 	if (err == noErr) {
 		//	Enough blocks are already allocated.  Just update the FCB to reflect the new length.
@@ -1064,8 +1071,8 @@ OSErr ExtendFileC (
 	//
 	//	Adjust the PEOF to the end of the last extent.
 	//
-	peof = (SInt64)((SInt64)nextBlock * (SInt64)volumeBlockSize);			// currently allocated PEOF
-	bytesThisExtent = (SInt64)(nextBlock - fcb->ff_blocks) * (SInt64)volumeBlockSize;
+	peof = (int64_t)((int64_t)nextBlock * (int64_t)volumeBlockSize);			// currently allocated PEOF
+	bytesThisExtent = (int64_t)(nextBlock - fcb->ff_blocks) * (int64_t)volumeBlockSize;
 	if (bytesThisExtent != 0) {
 		fcb->ff_blocks = nextBlock;
 		FTOC(fcb)->c_blocks += (bytesThisExtent / volumeBlockSize);
@@ -1090,22 +1097,19 @@ OSErr ExtendFileC (
 		else
 			startBlock = foundData[foundIndex].startBlock + foundData[foundIndex].blockCount;
 
-		/* Force reserve checking if requested. */
-		if (flags & kEFReserveMask) {
-			SInt64 availbytes;
-
-			actualNumBlocks = 0;
-			actualStartBlock = 0;
+		actualNumBlocks = 0;
+		actualStartBlock = 0;
 			
-			availbytes = (SInt64)hfs_freeblks(VCBTOHFS(vcb), 1) *
-			             (SInt64)volumeBlockSize;
-			if (availbytes <= 0) {
+		/* Find number of free blocks based on reserved block flag option */
+		availbytes = (int64_t)hfs_freeblks(VCBTOHFS(vcb), flags & kEFReserveMask) *
+		             (int64_t)volumeBlockSize;
+		if (availbytes <= 0) {
+			err = dskFulErr;
+		} else {
+			if (wantContig && (availbytes < bytesToAdd))
 				err = dskFulErr;
-			} else {
-				if (wantContig && (availbytes < bytesToAdd))
-					err = dskFulErr;
-				else {
-					err = BlockAllocate(
+			else {
+				err = BlockAllocate(
 						  vcb,
 						  startBlock,
 						  howmany(MIN(bytesToAdd, availbytes), volumeBlockSize),
@@ -1114,12 +1118,7 @@ OSErr ExtendFileC (
 						  useMetaZone,
 						  &actualStartBlock,
 						  &actualNumBlocks);
-				}
 			}
-		} else {
-			err = BlockAllocate(vcb, startBlock, howmany(bytesToAdd, volumeBlockSize),
-			                    howmany(maximumBytes, volumeBlockSize), wantContig, useMetaZone,
-			                    &actualStartBlock, &actualNumBlocks);
 		}
 		if (err == dskFulErr) {
 			if (forceContig)
@@ -1154,7 +1153,7 @@ OSErr ExtendFileC (
 				if (err != noErr) break;
 			}
 			else {
-				UInt16	i;
+				u_int16_t	i;
 
 				//	Need to add a new extent.  See if there is room in the current record.
 				if (foundData[foundIndex].blockCount != 0)	//	Is current extent free to use?
@@ -1213,7 +1212,7 @@ OSErr ExtendFileC (
 			// NOTE: BlockAllocate could have allocated more than we asked for.
 			// Don't set the PEOF beyond what our client asked for.
 			nextBlock += actualNumBlocks;
-			bytesThisExtent = (SInt64)((SInt64)actualNumBlocks * (SInt64)volumeBlockSize);
+			bytesThisExtent = (int64_t)((int64_t)actualNumBlocks * (int64_t)volumeBlockSize);
 			if (bytesThisExtent > bytesToAdd) {
 				bytesToAdd = 0;
 			}
@@ -1242,13 +1241,13 @@ Exit:
 		if (vcb->nextAllocation >= VCBTOHFS(vcb)->hfs_metazone_start &&
 		    vcb->nextAllocation <= VCBTOHFS(vcb)->hfs_metazone_end) {
 			HFS_MOUNT_LOCK(vcb, TRUE);
-			vcb->nextAllocation = VCBTOHFS(vcb)->hfs_metazone_end + 1;
-			vcb->vcbFlags |= 0xFF00;
+			HFS_UPDATE_NEXT_ALLOCATION(vcb, VCBTOHFS(vcb)->hfs_metazone_end + 1);	
+			MarkVCBDirty(vcb);
 			HFS_MOUNT_UNLOCK(vcb, TRUE);
 		}
 	}
 	if (prevblocks < fcb->ff_blocks) {
-		*actualBytesAdded = (SInt64)(fcb->ff_blocks - prevblocks) * (SInt64)volumeBlockSize;
+		*actualBytesAdded = (int64_t)(fcb->ff_blocks - prevblocks) * (int64_t)volumeBlockSize;
 	} else {
 		*actualBytesAdded = 0;
 	}
@@ -1280,22 +1279,22 @@ __private_extern__
 OSErr TruncateFileC (
 	ExtendedVCB		*vcb,				// volume that file resides on
 	FCB				*fcb,				// FCB of file to truncate
-	SInt64			peof,				// new physical size for file
+	int64_t			peof,				// new physical size for file
 	Boolean			truncateToExtent)	// if true, truncate to end of extent containing newPEOF
 {
 	OSErr				err;
-	UInt32				nextBlock;		//	next file allocation block to consider
-	UInt32				startBlock;		//	Physical (volume) allocation block number of start of a range
-	UInt32				physNumBlocks;	//	Number of allocation blocks in file (according to PEOF)
-	UInt32				numBlocks;
+	u_int32_t			nextBlock;		//	next file allocation block to consider
+	u_int32_t			startBlock;		//	Physical (volume) allocation block number of start of a range
+	u_int32_t			physNumBlocks;	//	Number of allocation blocks in file (according to PEOF)
+	u_int32_t			numBlocks;
 	HFSPlusExtentKey	key;			//	key for current extent record; key->keyLength == 0 if FCB's extent record
-	UInt32				hint;			//	BTree hint corresponding to key
+	u_int32_t			hint;			//	BTree hint corresponding to key
 	HFSPlusExtentRecord	extentRecord;
-	UInt32				extentIndex;
-	UInt32				extentNextBlock;
-	UInt32				numExtentsPerRecord;
-	SInt64                          temp64;
-	UInt8				forkType;
+	u_int32_t			extentIndex;
+	u_int32_t			extentNextBlock;
+	u_int32_t			numExtentsPerRecord;
+	int64_t             temp64;
+	u_int8_t			forkType;
 	Boolean				extentChanged;	// true if we actually changed an extent
 	Boolean				recordDeleted;	// true if an extent record got deleted
 
@@ -1312,7 +1311,7 @@ OSErr TruncateFileC (
 		forkType = kDataForkType;
 
 	temp64 = fcb->ff_blocks;
-	physNumBlocks = (UInt32)temp64;
+	physNumBlocks = (u_int32_t)temp64;
 
 	//
 	//	Round newPEOF up to a multiple of the allocation block size.  If new size is
@@ -1320,7 +1319,7 @@ OSErr TruncateFileC (
 	//	shouldn't that be an error?).
 	//
 	nextBlock = howmany(peof, vcb->blockSize);	// number of allocation blocks to remain in file
-	peof = (SInt64)((SInt64)nextBlock * (SInt64)vcb->blockSize);					// number of bytes in those blocks
+	peof = (int64_t)((int64_t)nextBlock * (int64_t)vcb->blockSize);					// number of bytes in those blocks
 	if ((vcb->vcbSigWord == kHFSSigWord) && (peof >= kTwoGigabytes)) {
 		#if DEBUG_BUILD
 			DebugStr("\pHFS: Trying to truncate a file to 2GB or more");
@@ -1456,15 +1455,15 @@ __private_extern__
 OSErr HeadTruncateFile (
 	ExtendedVCB  *vcb,
 	FCB  *fcb,
-	UInt32  headblks)
+	u_int32_t  headblks)
 {
 	HFSPlusExtentRecord  extents;
 	HFSPlusExtentRecord  tailExtents;
 	HFSCatalogNodeID  fileID;
-	UInt8  forkType;
-	UInt32  blkcnt;
-	UInt32  startblk;
-	UInt32  blksfreed;
+	u_int8_t  forkType;
+	u_int32_t  blkcnt;
+	u_int32_t  startblk;
+	u_int32_t  blksfreed;
 	int  i, j;
 	int  error = 0;
 	int  lockflags;
@@ -1501,7 +1500,7 @@ OSErr HeadTruncateFile (
 				else {
 					error = 0;
 					printf("HeadTruncateFile: problems deallocating %s (%d)\n",
-					       FTOC(fcb)->c_desc.cd_nameptr ? FTOC(fcb)->c_desc.cd_nameptr : "", error);
+					       FTOC(fcb)->c_desc.cd_nameptr ? (const char *)FTOC(fcb)->c_desc.cd_nameptr : "", error);
 				}
 			}
 
@@ -1525,7 +1524,7 @@ OSErr HeadTruncateFile (
 	 * Process overflow extents
 	 */
 	for (;;) {
-		UInt32  extblks;
+		u_int32_t  extblks;
 
 		error = FindExtentRecord(vcb, forkType, fileID, startblk, false, NULL, extents, NULL);
 		if (error) {
@@ -1536,7 +1535,7 @@ OSErr HeadTruncateFile (
 			 */
 			if (error != btNotFound)
 				printf("HeadTruncateFile: problems finding extents %s (%d)\n",
-				       FTOC(fcb)->c_desc.cd_nameptr ? FTOC(fcb)->c_desc.cd_nameptr : "", error);
+				       FTOC(fcb)->c_desc.cd_nameptr ? (const char *)FTOC(fcb)->c_desc.cd_nameptr : "", error);
 			error = 0;
 			break;
 		}
@@ -1550,7 +1549,7 @@ OSErr HeadTruncateFile (
 				error = BlockDeallocate(vcb, extents[i].startBlock, blkcnt);
 				if (error) {
 					printf("HeadTruncateFile: problems deallocating %s (%d)\n",
-					       FTOC(fcb)->c_desc.cd_nameptr ? FTOC(fcb)->c_desc.cd_nameptr : "", error);
+					       FTOC(fcb)->c_desc.cd_nameptr ? (const char *)FTOC(fcb)->c_desc.cd_nameptr : "", error);
 					error = 0;
 				}
 				blksfreed += blkcnt;
@@ -1565,7 +1564,7 @@ OSErr HeadTruncateFile (
 		error = DeleteExtentRecord(vcb, forkType, fileID, startblk);
 		if (error) {
 			printf("HeadTruncateFile: problems deallocating %s (%d)\n",
-				FTOC(fcb)->c_desc.cd_nameptr ? FTOC(fcb)->c_desc.cd_nameptr : "", error);
+				FTOC(fcb)->c_desc.cd_nameptr ? (const char *)FTOC(fcb)->c_desc.cd_nameptr : "", error);
 			error = 0;
 		}
 		
@@ -1580,7 +1579,7 @@ CopyExtents:
 	if (blksfreed) {
 		bcopy(tailExtents, fcb->fcbExtents, sizeof(tailExtents));
 		blkcnt = fcb->ff_blocks - headblks;
-		FTOC(fcb)->c_blocks -= blkcnt;
+		FTOC(fcb)->c_blocks -= headblks;
 		fcb->ff_blocks = blkcnt;
 
 		FTOC(fcb)->c_flag |= C_FORCEUPDATE;
@@ -1619,17 +1618,17 @@ ErrorExit:
 
 static OSErr SearchExtentRecord(
 	ExtendedVCB		*vcb,
-	UInt32					searchFABN,
+	u_int32_t				searchFABN,
 	const HFSPlusExtentRecord	extentData,
-	UInt32					extentDataStartFABN,
-	UInt32					*foundExtentIndex,
-	UInt32					*endingFABNPlusOne,
+	u_int32_t				extentDataStartFABN,
+	u_int32_t				*foundExtentIndex,
+	u_int32_t				*endingFABNPlusOne,
 	Boolean					*noMoreExtents)
 {
 	OSErr	err = noErr;
-	UInt32	extentIndex;
-	UInt32	numberOfExtents;
-	UInt32	numAllocationBlocks;
+	u_int32_t	extentIndex;
+	u_int32_t	numberOfExtents;
+	u_int32_t	numAllocationBlocks;
 	Boolean	foundExtent;
 	
 	*endingFABNPlusOne 	= extentDataStartFABN;
@@ -1721,21 +1720,21 @@ static OSErr SearchExtentRecord(
 static OSErr SearchExtentFile(
 	ExtendedVCB 	*vcb,
 	const FCB	 		*fcb,
-	SInt64 				filePosition,
+	int64_t 			filePosition,
 	HFSPlusExtentKey	*foundExtentKey,
 	HFSPlusExtentRecord	foundExtentData,
-	UInt32				*foundExtentIndex,
-	UInt32				*extentBTreeHint,
-	UInt32				*endingFABNPlusOne )
+	u_int32_t			*foundExtentIndex,
+	u_int32_t			*extentBTreeHint,
+	u_int32_t			*endingFABNPlusOne )
 {
 	OSErr				err;
-	UInt32				filePositionBlock;
-	SInt64                          temp64;
+	u_int32_t			filePositionBlock;
+	int64_t                         temp64;
 	Boolean				noMoreExtents;
 	int  lockflags;
 	
-	temp64 = filePosition / (SInt64)vcb->blockSize;
-	filePositionBlock = (UInt32)temp64;
+	temp64 = filePosition / (int64_t)vcb->blockSize;
+	filePositionBlock = (u_int32_t)temp64;
 
     bcopy ( fcb->fcbExtents, foundExtentData, sizeof(HFSPlusExtentRecord));
 	
@@ -1826,7 +1825,7 @@ static OSErr UpdateExtentRecord (
 	FCB  *fcb,
 	const HFSPlusExtentKey  *extentFileKey,
 	const HFSPlusExtentRecord  extentData,
-	UInt32  extentBTreeHint)
+	u_int32_t  extentBTreeHint)
 {
     OSErr err = noErr;
 	
@@ -1837,7 +1836,7 @@ static OSErr UpdateExtentRecord (
 	else {
 		BTreeIterator * btIterator;
 		FSBufferDescriptor btRecord;
-		UInt16 btRecordSize;
+		u_int16_t btRecordSize;
 		FCB * btFCB;
 		int lockflags;
 
@@ -1964,12 +1963,12 @@ static OSErr GetFCBExtentRecord(
 
 static Boolean ExtentsAreIntegral(
 	const HFSPlusExtentRecord extentRecord,
-	UInt32		mask,
-	UInt32		*blocksChecked,
+	u_int32_t	mask,
+	u_int32_t	*blocksChecked,
 	Boolean		*checkedLastExtent)
 {
-	UInt32		blocks;
-	UInt32		extentIndex;
+	u_int32_t	blocks;
+	u_int32_t	extentIndex;
 
 	*blocksChecked = 0;
 	*checkedLastExtent = false;
@@ -2006,12 +2005,12 @@ __private_extern__
 Boolean NodesAreContiguous(
 	ExtendedVCB	*vcb,
 	FCB			*fcb,
-	UInt32		nodeSize)
+	u_int32_t	nodeSize)
 {
-	UInt32				mask;
-	UInt32				startBlock;
-	UInt32				blocksChecked;
-	UInt32				hint;
+	u_int32_t			mask;
+	u_int32_t			startBlock;
+	u_int32_t			blocksChecked;
+	u_int32_t			hint;
 	HFSPlusExtentKey	key;
 	HFSPlusExtentRecord	extents;
 	OSErr				result;
@@ -2030,7 +2029,7 @@ Boolean NodesAreContiguous(
 		return FALSE;
 
 	if ( lastExtentReached || 
-		 (SInt64)((SInt64)blocksChecked * (SInt64)vcb->blockSize) >= (SInt64)fcb->ff_size)
+		 (int64_t)((int64_t)blocksChecked * (int64_t)vcb->blockSize) >= (int64_t)fcb->ff_size)
 		return TRUE;
 
 	startBlock = blocksChecked;
diff --git a/bsd/hfs/hfscommon/Misc/VolumeAllocation.c b/bsd/hfs/hfscommon/Misc/VolumeAllocation.c
index 55bec894b..0a999100e 100644
--- a/bsd/hfs/hfscommon/Misc/VolumeAllocation.c
+++ b/bsd/hfs/hfscommon/Misc/VolumeAllocation.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		VolumeAllocation.c
@@ -104,48 +110,48 @@ enum {
 
 static OSErr ReadBitmapBlock(
 	ExtendedVCB		*vcb,
-	UInt32			bit,
-	UInt32			**buffer,
-	UInt32			*blockRef);
+	u_int32_t		bit,
+	u_int32_t		**buffer,
+	u_int32_t		*blockRef);
 
 static OSErr ReleaseBitmapBlock(
 	ExtendedVCB		*vcb,
-	UInt32			blockRef,
+	u_int32_t		blockRef,
 	Boolean			dirty);
 
 static OSErr BlockAllocateAny(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	UInt32			endingBlock,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	u_int32_t		endingBlock,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks);
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks);
 
 static OSErr BlockAllocateContig(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	UInt32			minBlocks,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	u_int32_t		minBlocks,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks);
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks);
 
 static OSErr BlockFindContiguous(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	UInt32			endingBlock,
-	UInt32			minBlocks,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	u_int32_t		endingBlock,
+	u_int32_t		minBlocks,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks);
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks);
 
 static OSErr BlockAllocateKnown(
 	ExtendedVCB		*vcb,
-	UInt32			maxBlocks,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks);
+	u_int32_t		maxBlocks,
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks);
 
 
 /*
@@ -192,18 +198,18 @@ static OSErr BlockAllocateKnown(
 __private_extern__
 OSErr BlockAllocate (
 	ExtendedVCB		*vcb,				/* which volume to allocate space on */
-	UInt32			startingBlock,		/* preferred starting block, or 0 for no preference */
-	UInt32			minBlocks,		/* desired number of blocks to allocate */
-	UInt32			maxBlocks,		/* maximum number of blocks to allocate */
+	u_int32_t		startingBlock,		/* preferred starting block, or 0 for no preference */
+	u_int32_t		minBlocks,		/* desired number of blocks to allocate */
+	u_int32_t		maxBlocks,		/* maximum number of blocks to allocate */
 	Boolean			forceContiguous,	/* non-zero to force contiguous allocation and to force */
 							/* minBlocks bytes to actually be allocated */
 							
 	Boolean	useMetaZone,
-	UInt32			*actualStartBlock,	/* actual first block of allocation */
-	UInt32			*actualNumBlocks)	/* number of blocks actually allocated; if forceContiguous */
+	u_int32_t		*actualStartBlock,	/* actual first block of allocation */
+	u_int32_t		*actualNumBlocks)	/* number of blocks actually allocated; if forceContiguous */
 							/* was zero, then this may represent fewer than minBlocks */
 {
-	UInt32  freeBlocks;
+	u_int32_t  freeBlocks;
 	OSErr			err;
 	Boolean			updateAllocPtr = false;		//	true if nextAllocation needs to be updated
 
@@ -245,7 +251,7 @@ OSErr BlockAllocate (
 		HFS_MOUNT_UNLOCK(vcb, TRUE);
 		updateAllocPtr = true;
 	}
-	if (startingBlock >= vcb->totalBlocks) {
+	if (startingBlock >= vcb->allocLimit) {
 		startingBlock = 0; /* overflow so start at beginning */
 	}
 
@@ -258,14 +264,14 @@ OSErr BlockAllocate (
 		                          useMetaZone, actualStartBlock, actualNumBlocks);
 		/*
 		 * If we allocated from a new position then
-		 * also update the roving allocatior.
+		 * also update the roving allocator.
 		 */
 		if ((err == noErr) &&
 		    (*actualStartBlock > startingBlock) &&
 		    ((*actualStartBlock < VCBTOHFS(vcb)->hfs_metazone_start) ||
 	    	     (*actualStartBlock > VCBTOHFS(vcb)->hfs_metazone_end))) {
 			HFS_MOUNT_LOCK(vcb, TRUE);
-			vcb->nextAllocation = *actualStartBlock;
+			HFS_UPDATE_NEXT_ALLOCATION(vcb, *actualStartBlock);
 			HFS_MOUNT_UNLOCK(vcb, TRUE);
 		}
 	} else {
@@ -278,7 +284,7 @@ OSErr BlockAllocate (
 		 */
 		err = BlockAllocateKnown(vcb, maxBlocks, actualStartBlock, actualNumBlocks);
 		if (err == dskFulErr)
-			err = BlockAllocateAny(vcb, startingBlock, vcb->totalBlocks,
+			err = BlockAllocateAny(vcb, startingBlock, vcb->allocLimit,
 			                       maxBlocks, useMetaZone, actualStartBlock,
 			                       actualNumBlocks);
 		if (err == dskFulErr)
@@ -307,7 +313,7 @@ Exit:
 		if (updateAllocPtr &&
 		    ((*actualStartBlock < VCBTOHFS(vcb)->hfs_metazone_start) ||
 	    	     (*actualStartBlock > VCBTOHFS(vcb)->hfs_metazone_end))) {
-			vcb->nextAllocation = *actualStartBlock;
+			HFS_UPDATE_NEXT_ALLOCATION(vcb, *actualStartBlock);
 		}
 		//
 		//	Update the number of free blocks on the volume
@@ -346,8 +352,8 @@ Exit:
 __private_extern__
 OSErr BlockDeallocate (
 	ExtendedVCB		*vcb,			//	Which volume to deallocate space on
-	UInt32			firstBlock,		//	First block in range to deallocate
-	UInt32			numBlocks)		//	Number of contiguous blocks to deallocate
+	u_int32_t		firstBlock,		//	First block in range to deallocate
+	u_int32_t		numBlocks)		//	Number of contiguous blocks to deallocate
 {
 	OSErr			err;
 	
@@ -372,7 +378,7 @@ OSErr BlockDeallocate (
 	HFS_MOUNT_LOCK(vcb, TRUE);
 	vcb->freeBlocks += numBlocks;
 	if (vcb->nextAllocation == (firstBlock + numBlocks))
-		vcb->nextAllocation -= numBlocks;
+		HFS_UPDATE_NEXT_ALLOCATION(vcb, (vcb->nextAllocation - numBlocks));
 	MarkVCBDirty(vcb);
   	HFS_MOUNT_UNLOCK(vcb, TRUE); 
 
@@ -383,24 +389,24 @@ Exit:
 }
 
 
-UInt8 freebitcount[16] = {
+u_int8_t freebitcount[16] = {
 	4, 3, 3, 2, 3, 2, 2, 1,  /* 0 1 2 3 4 5 6 7 */
 	3, 2, 2, 1, 2, 1, 1, 0,  /* 8 9 A B C D E F */
 };
 
 __private_extern__
-UInt32
+u_int32_t
 MetaZoneFreeBlocks(ExtendedVCB *vcb)
 {
-	UInt32 freeblocks;
-	UInt32 *currCache;
-	UInt32 blockRef;
-	UInt32 bit;
-	UInt32 lastbit;
+	u_int32_t freeblocks;
+	u_int32_t *currCache;
+	u_int32_t blockRef;
+	u_int32_t bit;
+	u_int32_t lastbit;
 	int bytesleft;
 	int bytesperblock;
-	UInt8 byte;
-	UInt8 *buffer;
+	u_int8_t byte;
+	u_int8_t *buffer;
 
 	blockRef = 0;
 	bytesleft = freeblocks = 0;
@@ -427,7 +433,7 @@ MetaZoneFreeBlocks(ExtendedVCB *vcb)
 			if (ReadBitmapBlock(vcb, bit, &currCache, &blockRef) != 0) {
 				return (0);
 			}
-			buffer = (UInt8 *)currCache;
+			buffer = (u_int8_t *)currCache;
 			bytesleft = bytesperblock;
 		}
 		byte = *buffer++;
@@ -447,9 +453,9 @@ MetaZoneFreeBlocks(ExtendedVCB *vcb)
  * Obtain the next allocation block (bit) that's
  * outside the metadata allocation zone.
  */
-static UInt32 NextBitmapBlock(
+static u_int32_t NextBitmapBlock(
 	ExtendedVCB		*vcb,
-	UInt32			bit)
+	u_int32_t		bit)
 {
 	struct  hfsmount *hfsmp = VCBTOHFS(vcb);
 
@@ -485,22 +491,22 @@ static UInt32 NextBitmapBlock(
 */
 static OSErr ReadBitmapBlock(
 	ExtendedVCB		*vcb,
-	UInt32			bit,
-	UInt32			**buffer,
-	UInt32			*blockRef)
+	u_int32_t		bit,
+	u_int32_t		**buffer,
+	u_int32_t		*blockRef)
 {
 	OSErr			err;
 	struct buf *bp = NULL;
 	struct vnode *vp = NULL;
 	daddr64_t block;
-	UInt32 blockSize;
+	u_int32_t blockSize;
 
 	/*
 	 * volume bitmap blocks are protected by the allocation file lock
 	 */
 	REQUIRE_FILE_LOCK(vcb->hfs_allocation_vp, false);	
 
-	blockSize = (UInt32)vcb->vcbVBMIOSize;
+	blockSize = (u_int32_t)vcb->vcbVBMIOSize;
 	block = (daddr64_t)(bit / (blockSize * kBitsPerByte));
 
 	if (vcb->vcbSigWord == kHFSPlusSigWord) {
@@ -516,11 +522,11 @@ static OSErr ReadBitmapBlock(
 	if (bp) {
 		if (err) {
 			buf_brelse(bp);
-			*blockRef = NULL;
+			*blockRef = 0;
 			*buffer = NULL;
 		} else {
-			*blockRef = (UInt32)bp;
-			*buffer = (UInt32 *)buf_dataptr(bp);
+			*blockRef = (u_int32_t)bp;
+			*buffer = (u_int32_t *)buf_dataptr(bp);
 		}
 	}
 
@@ -543,7 +549,7 @@ static OSErr ReadBitmapBlock(
 */
 static OSErr ReleaseBitmapBlock(
 	ExtendedVCB		*vcb,
-	UInt32			blockRef,
+	u_int32_t		blockRef,
 	Boolean			dirty)
 {
 	struct buf *bp = (struct buf *)blockRef;
@@ -560,7 +566,7 @@ static OSErr ReleaseBitmapBlock(
 			struct hfsmount *hfsmp = VCBTOHFS(vcb);
 			
 			if (hfsmp->jnl) {
-				journal_modify_block_end(hfsmp->jnl, bp);
+				journal_modify_block_end(hfsmp->jnl, bp, NULL, NULL);
 			} else {
 				buf_bdwrite(bp);
 			}
@@ -597,12 +603,12 @@ _______________________________________________________________________
 */
 static OSErr BlockAllocateContig(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	UInt32			minBlocks,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	u_int32_t		minBlocks,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks)
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks)
 {
 	OSErr	err;
 
@@ -620,7 +626,7 @@ static OSErr BlockAllocateContig(
 	 * with the free extent cache, this can lead to duplicate entries
 	 * in the cache, causing the same blocks to be allocated twice.
 	 */
-	err = BlockFindContiguous(vcb, startingBlock, vcb->totalBlocks, minBlocks,
+	err = BlockFindContiguous(vcb, startingBlock, vcb->allocLimit, minBlocks,
 	                          maxBlocks, useMetaZone, actualStartBlock, actualNumBlocks);
 	if (err == dskFulErr && startingBlock != 0) {
 		/*
@@ -630,22 +636,11 @@ static OSErr BlockAllocateContig(
 		err = BlockFindContiguous(vcb, 1, startingBlock, minBlocks, maxBlocks,
 		                          useMetaZone, actualStartBlock, actualNumBlocks);
 	}
-	if (err != noErr) goto Exit;
-
-	// sanity check
-	if ((*actualStartBlock + *actualNumBlocks) > vcb->totalBlocks)
-		panic("BlockAllocateContig: allocation overflow on \"%s\"", vcb->vcbVN);
-
 	//
 	//	Now mark those blocks allocated.
 	//
-	err = BlockMarkAllocated(vcb, *actualStartBlock, *actualNumBlocks);
-	
-Exit:
-	if (err != noErr) {
-		*actualStartBlock = 0;
-		*actualNumBlocks = 0;
-	}
+	if (err == noErr)
+		err = BlockMarkAllocated(vcb, *actualStartBlock, *actualNumBlocks);
 	
 	return err;
 }
@@ -674,37 +669,49 @@ _______________________________________________________________________
 */
 static OSErr BlockAllocateAny(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	register UInt32	endingBlock,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	register u_int32_t	endingBlock,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks)
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks)
 {
 	OSErr			err;
-	register UInt32	block;			//	current block number
-	register UInt32	currentWord;	//	Pointer to current word within bitmap block
-	register UInt32	bitMask;		//	Word with given bits already set (ready to OR in)
-	register UInt32	wordsLeft;		//	Number of words left in this bitmap block
-	UInt32  *buffer = NULL;
-	UInt32  *currCache = NULL;
-	UInt32  blockRef;
-	UInt32  bitsPerBlock;
-	UInt32  wordsPerBlock;
+	register u_int32_t	block;			//	current block number
+	register u_int32_t	currentWord;	//	Pointer to current word within bitmap block
+	register u_int32_t	bitMask;		//	Word with given bits already set (ready to OR in)
+	register u_int32_t	wordsLeft;		//	Number of words left in this bitmap block
+	u_int32_t  *buffer = NULL;
+	u_int32_t  *currCache = NULL;
+	u_int32_t  blockRef;
+	u_int32_t  bitsPerBlock;
+	u_int32_t  wordsPerBlock;
 	Boolean dirty = false;
 	struct hfsmount *hfsmp = VCBTOHFS(vcb);
 
+	/*
+	 * When we're skipping the metadata zone and the start/end
+	 * range overlaps with the metadata zone then adjust the 
+	 * start to be outside of the metadata zone.  If the range
+	 * is entirely inside the metadata zone then we can deny the
+	 * request (dskFulErr).
+	 */
+	if (!useMetaZone && (vcb->hfs_flags & HFS_METADATA_ZONE)) {
+		if (startingBlock <= vcb->hfs_metazone_end) {
+			if (endingBlock > (vcb->hfs_metazone_end + 2))
+				startingBlock = vcb->hfs_metazone_end + 1;
+			else {
+				err = dskFulErr;
+				goto Exit;
+			}
+		}
+	}
+
 	//	Since this routine doesn't wrap around
 	if (maxBlocks > (endingBlock - startingBlock)) {
 		maxBlocks = endingBlock - startingBlock;
 	}
 
-	/*
-	 * Skip over metadata blocks.
-	 */
-	if (!useMetaZone)
-		startingBlock = NextBitmapBlock(vcb, startingBlock);
-
 	//
 	//	Pre-read the first bitmap block
 	//
@@ -716,7 +723,7 @@ static OSErr BlockAllocateAny(
 	//	Set up the current position within the block
 	//
 	{
-		UInt32 wordIndexInBlock;
+		u_int32_t wordIndexInBlock;
 		
 		bitsPerBlock  = vcb->vcbVBMIOSize * kBitsPerByte;
 		wordsPerBlock = vcb->vcbVBMIOSize / kBytesPerWord;
@@ -755,11 +762,12 @@ static OSErr BlockAllocateAny(
 				 */
 				if (!useMetaZone) {
 					block = NextBitmapBlock(vcb, block);
-					if (block >= endingBlock) {
-						err = dskFulErr;
-						goto Exit;
-					}
 				}
+				if (block >= endingBlock) {
+					err = dskFulErr;
+					goto Exit;
+				}
+
 				err = ReadBitmapBlock(vcb, block, &currCache, &blockRef);
 				if (err != noErr) goto Exit;
 				buffer = currCache;
@@ -825,7 +833,7 @@ static OSErr BlockAllocateAny(
 				 * Skip over metadata blocks.
 				 */
 				if (!useMetaZone) {
-					UInt32 nextBlock;
+					u_int32_t nextBlock;
 
 					nextBlock = NextBitmapBlock(vcb, block);
 					if (nextBlock != block) {
@@ -855,7 +863,7 @@ Exit:
 		*actualNumBlocks = block - *actualStartBlock;
 
 	// sanity check
-	if ((*actualStartBlock + *actualNumBlocks) > vcb->totalBlocks)
+	if ((*actualStartBlock + *actualNumBlocks) > vcb->allocLimit)
 		panic("BlockAllocateAny: allocation overflow on \"%s\"", vcb->vcbVN);
 	}
 	else {
@@ -892,14 +900,14 @@ _______________________________________________________________________
 */
 static OSErr BlockAllocateKnown(
 	ExtendedVCB		*vcb,
-	UInt32			maxBlocks,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks)
+	u_int32_t		maxBlocks,
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks)
 {
-	OSErr			err;
-	UInt32			i;
-	UInt32			foundBlocks;
-	UInt32			newStartBlock, newBlockCount;
+	OSErr			err;	
+	u_int32_t		i;
+	u_int32_t		foundBlocks;
+	u_int32_t		newStartBlock, newBlockCount;
 	
 	if (vcb->vcbFreeExtCnt == 0)
 		return dskFulErr;
@@ -946,22 +954,23 @@ static OSErr BlockAllocateKnown(
 	}
 	
 	// sanity check
-	if ((*actualStartBlock + *actualNumBlocks) > vcb->totalBlocks)
+	if ((*actualStartBlock + *actualNumBlocks) > vcb->allocLimit) 
 	{
-		printf("BlockAllocateKnown: allocation overflow on \"%s\"", vcb->vcbVN);
+		printf ("hfs: BlockAllocateKnown() found allocation overflow on \"%s\"", vcb->vcbVN);
+		hfs_mark_volume_inconsistent(vcb);
 		*actualStartBlock = 0;
 		*actualNumBlocks = 0;
 		err = EIO;
-	}
-
-	else
+	} 
+	else 
 	{
 		//
 		//	Now mark the found extent in the bitmap
 		//
-	err = BlockMarkAllocated(vcb, *actualStartBlock, *actualNumBlocks);
+		err = BlockMarkAllocated(vcb, *actualStartBlock, *actualNumBlocks);
 	}
-	return err;	
+
+	return err;
 }
 
 
@@ -984,19 +993,19 @@ _______________________________________________________________________
 __private_extern__
 OSErr BlockMarkAllocated(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	register UInt32	numBlocks)
+	u_int32_t		startingBlock,
+	register u_int32_t	numBlocks)
 {
 	OSErr			err;
-	register UInt32	*currentWord;	//	Pointer to current word within bitmap block
-	register UInt32	wordsLeft;		//	Number of words left in this bitmap block
-	register UInt32	bitMask;		//	Word with given bits already set (ready to OR in)
-	UInt32			firstBit;		//	Bit index within word of first bit to allocate
-	UInt32			numBits;		//	Number of bits in word to allocate
-	UInt32			*buffer = NULL;
-	UInt32  blockRef;
-	UInt32  bitsPerBlock;
-	UInt32  wordsPerBlock;
+	register u_int32_t	*currentWord;	//	Pointer to current word within bitmap block
+	register u_int32_t	wordsLeft;		//	Number of words left in this bitmap block
+	register u_int32_t	bitMask;		//	Word with given bits already set (ready to OR in)
+	u_int32_t		firstBit;		//	Bit index within word of first bit to allocate
+	u_int32_t		numBits;		//	Number of bits in word to allocate
+	u_int32_t		*buffer = NULL;
+	u_int32_t  blockRef;
+	u_int32_t  bitsPerBlock;
+	u_int32_t  wordsPerBlock;
 	// XXXdbg
 	struct hfsmount *hfsmp = VCBTOHFS(vcb);
 
@@ -1011,7 +1020,7 @@ OSErr BlockMarkAllocated(
 	//	Initialize currentWord, and wordsLeft.
 	//
 	{
-		UInt32 wordIndexInBlock;
+		u_int32_t wordIndexInBlock;
 		
 		bitsPerBlock  = vcb->vcbVBMIOSize * kBitsPerByte;
 		wordsPerBlock = vcb->vcbVBMIOSize / kBytesPerWord;
@@ -1153,27 +1162,31 @@ _______________________________________________________________________
 __private_extern__
 OSErr BlockMarkFree(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	register UInt32	numBlocks)
+	u_int32_t		startingBlock,
+	register u_int32_t	numBlocks)
 {
 	OSErr			err;
-	register UInt32	*currentWord;	//	Pointer to current word within bitmap block
-	register UInt32	wordsLeft;		//	Number of words left in this bitmap block
-	register UInt32	bitMask;		//	Word with given bits already set (ready to OR in)
-	UInt32			firstBit;		//	Bit index within word of first bit to allocate
-	UInt32			numBits;		//	Number of bits in word to allocate
-	UInt32			*buffer = NULL;
-	UInt32  blockRef;
-	UInt32  bitsPerBlock;
-	UInt32  wordsPerBlock;
+	register u_int32_t	*currentWord;	//	Pointer to current word within bitmap block
+	register u_int32_t	wordsLeft;		//	Number of words left in this bitmap block
+	register u_int32_t	bitMask;		//	Word with given bits already set (ready to OR in)
+	u_int32_t			firstBit;		//	Bit index within word of first bit to allocate
+	u_int32_t			numBits;		//	Number of bits in word to allocate
+	u_int32_t			*buffer = NULL;
+	u_int32_t  blockRef;
+	u_int32_t  bitsPerBlock;
+	u_int32_t  wordsPerBlock;
     // XXXdbg
 	struct hfsmount *hfsmp = VCBTOHFS(vcb);
 
+	/*
+	 * NOTE: We use vcb->totalBlocks instead of vcb->allocLimit because we
+	 * need to be able to free blocks being relocated during hfs_truncatefs.
+	 */
 	if (startingBlock + numBlocks > vcb->totalBlocks) {
-	    printf("hfs: block mark free: trying to free non-existent blocks (%d %d %d)\n",
-		  startingBlock, numBlocks, vcb->totalBlocks);
-	    err = EIO;
-	    goto Exit;
+		printf ("hfs: BlockMarkFree() trying to free non-existent blocks starting at %u (numBlock=%u) on volume %s\n", startingBlock, numBlocks, vcb->vcbVN);
+		hfs_mark_volume_inconsistent(vcb);
+		err = EIO;
+		goto Exit;
 	}
 
 
@@ -1192,7 +1205,7 @@ OSErr BlockMarkFree(
 	//	Initialize currentWord, and wordsLeft.
 	//
 	{
-		UInt32 wordIndexInBlock;
+		u_int32_t wordIndexInBlock;
 		
 		bitsPerBlock  = vcb->vcbVBMIOSize * kBitsPerByte;
 		wordsPerBlock = vcb->vcbVBMIOSize / kBytesPerWord;
@@ -1306,9 +1319,8 @@ Corruption:
 #if DEBUG_BUILD
 	panic("BlockMarkFree: blocks not allocated!");
 #else
-	printf("hfs: WARNING - blocks on volume %s not allocated!\n", vcb->vcbVN);
-	vcb->vcbAtrb |= kHFSVolumeInconsistentMask;
-	MarkVCBDirty(vcb);
+	printf ("hfs: BlockMarkFree() trying to free unallocated blocks on volume %s\n", vcb->vcbVN);
+	hfs_mark_volume_inconsistent(vcb);
 	err = EIO;
 	goto Exit;
 #endif
@@ -1344,26 +1356,26 @@ _______________________________________________________________________
 
 static OSErr BlockFindContiguous(
 	ExtendedVCB		*vcb,
-	UInt32			startingBlock,
-	UInt32			endingBlock,
-	UInt32			minBlocks,
-	UInt32			maxBlocks,
+	u_int32_t		startingBlock,
+	u_int32_t		endingBlock,
+	u_int32_t		minBlocks,
+	u_int32_t		maxBlocks,
 	Boolean			useMetaZone,
-	UInt32			*actualStartBlock,
-	UInt32			*actualNumBlocks)
+	u_int32_t		*actualStartBlock,
+	u_int32_t		*actualNumBlocks)
 {
 	OSErr			err;
-	register UInt32	currentBlock;		//	Block we're currently looking at.
-	UInt32			firstBlock;			//	First free block in current extent.
-	UInt32			stopBlock;			//	If we get to this block, stop searching for first free block.
-	UInt32			foundBlocks;		//	Number of contiguous free blocks in current extent.
-	UInt32			*buffer = NULL;
-	register UInt32	*currentWord;
-	register UInt32	bitMask;
-	register UInt32	wordsLeft;
-	register UInt32	tempWord;
-	UInt32  blockRef;
-	UInt32  wordsPerBlock;
+	register u_int32_t	currentBlock;		//	Block we're currently looking at.
+	u_int32_t			firstBlock;			//	First free block in current extent.
+	u_int32_t			stopBlock;			//	If we get to this block, stop searching for first free block.
+	u_int32_t			foundBlocks;		//	Number of contiguous free blocks in current extent.
+	u_int32_t			*buffer = NULL;
+	register u_int32_t	*currentWord;
+	register u_int32_t	bitMask;
+	register u_int32_t	wordsLeft;
+	register u_int32_t	tempWord;
+	u_int32_t  blockRef;
+	u_int32_t  wordsPerBlock;
 
 	/*
 	 * When we're skipping the metadata zone and the start/end
@@ -1546,7 +1558,7 @@ FoundUnused:
 				 * Skip over metadata blocks.
 				 */
 				if (!useMetaZone) {
-					UInt32 nextBlock;
+					u_int32_t nextBlock;
 
 					nextBlock = NextBitmapBlock(vcb, currentBlock);
 					if (nextBlock != currentBlock) {
@@ -1635,6 +1647,14 @@ ErrorExit:
 		err = noErr;
 		*actualStartBlock = firstBlock;
 		*actualNumBlocks = foundBlocks;
+		/*
+		 * Sanity check for overflow
+		 */
+		if ((firstBlock + foundBlocks) > vcb->allocLimit) {
+			panic("blk allocation overflow on \"%s\" sb:0x%08x eb:0x%08x cb:0x%08x fb:0x%08x stop:0x%08x min:0x%08x found:0x%08x",
+				vcb->vcbVN, startingBlock, endingBlock, currentBlock,
+				firstBlock, stopBlock, minBlocks, foundBlocks);
+		}
 	}
 	
 	if (buffer)
@@ -1652,15 +1672,15 @@ __private_extern__
 int 
 hfs_isallocated(struct hfsmount *hfsmp, u_long startingBlock, u_long numBlocks)
 {
-	UInt32  *currentWord;   // Pointer to current word within bitmap block
-	UInt32  wordsLeft;      // Number of words left in this bitmap block
-	UInt32  bitMask;        // Word with given bits already set (ready to test)
-	UInt32  firstBit;       // Bit index within word of first bit to allocate
-	UInt32  numBits;        // Number of bits in word to allocate
-	UInt32  *buffer = NULL;
-	UInt32  blockRef;
-	UInt32  bitsPerBlock;
-	UInt32  wordsPerBlock;
+	u_int32_t  *currentWord;   // Pointer to current word within bitmap block
+	u_int32_t  wordsLeft;      // Number of words left in this bitmap block
+	u_int32_t  bitMask;        // Word with given bits already set (ready to test)
+	u_int32_t  firstBit;       // Bit index within word of first bit to allocate
+	u_int32_t  numBits;        // Number of bits in word to allocate
+	u_int32_t  *buffer = NULL;
+	u_int32_t  blockRef;
+	u_int32_t  bitsPerBlock;
+	u_int32_t  wordsPerBlock;
 	int  inuse = 0;
 	int  error;
 
@@ -1675,7 +1695,7 @@ hfs_isallocated(struct hfsmount *hfsmp, u_long startingBlock, u_long numBlocks)
 	 * Initialize currentWord, and wordsLeft.
 	 */
 	{
-		UInt32 wordIndexInBlock;
+		u_int32_t wordIndexInBlock;
 		
 		bitsPerBlock  = hfsmp->vcbVBMIOSize * kBitsPerByte;
 		wordsPerBlock = hfsmp->vcbVBMIOSize / kBytesPerWord;
diff --git a/bsd/hfs/hfscommon/Unicode/UCStringCompareData.h b/bsd/hfs/hfscommon/Unicode/UCStringCompareData.h
index b975465d3..c00d15117 100644
--- a/bsd/hfs/hfscommon/Unicode/UCStringCompareData.h
+++ b/bsd/hfs/hfscommon/Unicode/UCStringCompareData.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2002, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		UCStringCompareData.h
@@ -69,7 +75,7 @@
  *
  * Note: 0x0000 now maps to 0 so that it will be ignored
  */
-UInt16 gLatinCaseFold[] = {
+u_int16_t gLatinCaseFold[] = {
 	/* 0 */	0xFFFF, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,
 	/* 1 */	0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F,
 	/* 2 */	0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
@@ -94,7 +100,7 @@ UInt16 gLatinCaseFold[] = {
 	ignored characters in that block. Ignored characters are mapped to zero.
  */
 
-UInt16 gLowerCaseTable[] = {
+u_int16_t gLowerCaseTable[] = {
 
 	/* High-byte indices ( == 0 iff no case mapping and no ignorables ) */
 
diff --git a/bsd/hfs/hfscommon/Unicode/UnicodeWrappers.c b/bsd/hfs/hfscommon/Unicode/UnicodeWrappers.c
index 91b3e7a98..4436f0146 100644
--- a/bsd/hfs/hfscommon/Unicode/UnicodeWrappers.c
+++ b/bsd/hfs/hfscommon/Unicode/UnicodeWrappers.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		UnicodeWrappers.c
@@ -25,6 +31,8 @@
 	Contains:	Wrapper routines for Unicode conversion and comparison.
 
 */
+
+#if HFS
 #include <sys/param.h>
 #include <sys/utfconv.h>
 
@@ -45,14 +53,14 @@ enum {
 				 ((c) >= 0x30 && (c) <= 0x39))
 
 
-#define IsHexDigit(c)		(((c) >= (UInt8) '0' && (c) <= (UInt8) '9') || \
-				 ((c) >= (UInt8) 'A' && (c) <= (UInt8) 'F'))
+#define IsHexDigit(c)		(((c) >= (u_int8_t) '0' && (c) <= (u_int8_t) '9') || \
+				 ((c) >= (u_int8_t) 'A' && (c) <= (u_int8_t) 'F'))
 
 
 static void	GetFilenameExtension( ItemCount length, ConstUniCharArrayPtr unicodeStr, char* extStr );
 
 
-static UInt32	HexStringToInteger( UInt32 length, const UInt8 *hexStr );
+static u_int32_t	HexStringToInteger( u_int32_t length, const u_int8_t *hexStr );
 
 
 /*
@@ -61,10 +69,10 @@ static UInt32	HexStringToInteger( UInt32 length, const UInt8 *hexStr );
 static void
 GetFilenameExtension(ItemCount length, ConstUniCharArrayPtr unicodeStr, char * extStr)
 {
-	UInt32	i;
+	u_int32_t	i;
 	UniChar	c;
-	UInt16	extChars;	/* number of extension chars (excluding dot) */
-	UInt16	maxExtChars;
+	u_int16_t	extChars;	/* number of extension chars (excluding dot) */
+	u_int16_t	maxExtChars;
 	Boolean	foundExtension;
 
 	extStr[0] = '\0';	/* assume there's no extension */
@@ -99,11 +107,11 @@ GetFilenameExtension(ItemCount length, ConstUniCharArrayPtr unicodeStr, char * e
 	
 	/* if we found one then copy it */
 	if ( foundExtension ) {
-		UInt8 *extStrPtr = extStr;
+		u_int8_t *extStrPtr = (u_int8_t *)extStr;
 		const UniChar *unicodeStrPtr = &unicodeStr[i];
 		
 		for ( i = 0; i <= extChars; ++i )
-			*(extStrPtr++) = (UInt8) *(unicodeStrPtr++);
+			*(extStrPtr++) = (u_int8_t) *(unicodeStrPtr++);
 		extStr[extChars + 1] = '\0';	/* terminate extension + dot */
 	}
 }
@@ -113,13 +121,13 @@ GetFilenameExtension(ItemCount length, ConstUniCharArrayPtr unicodeStr, char * e
 /*
  * Count filename extension characters (if any)
  */
-static UInt32
-CountFilenameExtensionChars( const unsigned char * filename, UInt32 length )
+static u_int32_t
+CountFilenameExtensionChars( const unsigned char * filename, u_int32_t length )
 {
-	UInt32	i;
+	u_int32_t	i;
 	UniChar	c;
-	UInt32	extChars;	/* number of extension chars (excluding dot) */
-	UInt16	maxExtChars;
+	u_int32_t	extChars;	/* number of extension chars (excluding dot) */
+	u_int16_t	maxExtChars;
 	Boolean	foundExtension;
 
 	if ( length < 3 )
@@ -138,7 +146,7 @@ CountFilenameExtensionChars( const unsigned char * filename, UInt32 length )
 		c = filename[i--];
 
 		/* look for leading dot */
-		if ( c == (UInt8) '.' )	{
+		if ( c == (u_int8_t) '.' )	{
 			if ( extChars > 0 )	/* cannot end with a dot */
 				return (extChars);
 
@@ -159,11 +167,11 @@ CountFilenameExtensionChars( const unsigned char * filename, UInt32 length )
  * extract the file id from a mangled name
  */
 HFSCatalogNodeID
-GetEmbeddedFileID(const unsigned char * filename, UInt32 length, UInt32 *prefixLength)
+GetEmbeddedFileID(const unsigned char * filename, u_int32_t length, u_int32_t *prefixLength)
 {
 	short	extChars;
 	short	i;
-	UInt8	c;
+	u_int8_t	c;
 
 	*prefixLength = 0;
 
@@ -205,13 +213,13 @@ GetEmbeddedFileID(const unsigned char * filename, UInt32 length, UInt32 *prefixL
 
 
 
-static UInt32
-HexStringToInteger(UInt32 length, const UInt8 *hexStr)
+static u_int32_t
+HexStringToInteger(u_int32_t length, const u_int8_t *hexStr)
 {
-	UInt32		value;
-	UInt32		i;
-	UInt8		c;
-	const UInt8	*p;
+	u_int32_t		value;
+	u_int32_t		i;
+	u_int8_t		c;
+	const u_int8_t	*p;
 
 	value = 0;
 	p = hexStr;
@@ -221,7 +229,7 @@ HexStringToInteger(UInt32 length, const UInt8 *hexStr)
 
 		if (c >= '0' && c <= '9') {
 			value = value << 4;
-			value += (UInt32) c - (UInt32) '0';
+			value += (u_int32_t) c - (u_int32_t) '0';
 		} else if (c >= 'A' && c <= 'F') {
 			value = value << 4;
 			value += 10 + ((unsigned int) c - (unsigned int) 'A');
@@ -242,12 +250,12 @@ HexStringToInteger(UInt32 length, const UInt8 *hexStr)
  *		return	 0 if equal
  *
  */
-SInt32	FastRelString( ConstStr255Param str1, ConstStr255Param str2 )
+int32_t	FastRelString( ConstStr255Param str1, ConstStr255Param str2 )
 {
-	UInt16*			compareTable;
-	SInt32	 		bestGuess;
-	UInt8 	 		length, length2;
-	UInt8 	 		delta;
+	u_int16_t*		compareTable;
+	int32_t	 		bestGuess;
+	u_int8_t 	 	length, length2;
+	u_int8_t 	 	delta;
 
 	delta = 0;
 	length = *(str1++);
@@ -266,18 +274,18 @@ SInt32	FastRelString( ConstStr255Param str1, ConstStr255Param str2 )
 		length = length2;
 	}
 
-	compareTable = (UInt16*) gCompareTable;
+	compareTable = (u_int16_t*) gCompareTable;
 
 	while (length--)
 	{
-		UInt8	aChar, bChar;
+		u_int8_t	aChar, bChar;
 
 		aChar = *(str1++);
 		bChar = *(str2++);
 		
 		if (aChar != bChar)		//	If they don't match exacly, do case conversion
 		{	
-			UInt16	aSortWord, bSortWord;
+			u_int16_t	aSortWord, bSortWord;
 
 			aSortWord = compareTable[aChar];
 			bSortWord = compareTable[bChar];
@@ -356,14 +364,14 @@ SInt32	FastRelString( ConstStr255Param str1, ConstStr255Param str2 )
 //			return 1;
 //
 
-SInt32 FastUnicodeCompare ( register ConstUniCharArrayPtr str1, register ItemCount length1,
+int32_t FastUnicodeCompare ( register ConstUniCharArrayPtr str1, register ItemCount length1,
 							register ConstUniCharArrayPtr str2, register ItemCount length2)
 {
-	register UInt16		c1,c2;
-	register UInt16		temp;
-	register UInt16*	lowerCaseTable;
+	register u_int16_t		c1,c2;
+	register u_int16_t		temp;
+	register u_int16_t*	lowerCaseTable;
 
-	lowerCaseTable = (UInt16*) gLowerCaseTable;
+	lowerCaseTable = (u_int16_t*) gLowerCaseTable;
 
 	while (1) {
 		/* Set default values for c1, c2 in case there are no more valid chars */
@@ -422,7 +430,7 @@ ConvertUnicodeToUTF8Mangled(ByteCount srcLen, ConstUniCharArrayPtr srcStr, ByteC
 	char fileIDStr[15];
 	char extStr[15];
 
-	sprintf(fileIDStr, "#%X", cnid);
+	snprintf(fileIDStr, sizeof(fileIDStr), "#%X", cnid);
 	GetFilenameExtension(srcLen/sizeof(UniChar), srcStr, extStr);
 
 	/* remove extension chars from source */
@@ -431,10 +439,29 @@ ConvertUnicodeToUTF8Mangled(ByteCount srcLen, ConstUniCharArrayPtr srcStr, ByteC
 
 	(void) utf8_encodestr(srcStr, srcLen, dstStr, &utf8len, subMaxLen, ':', 0);
 
-	strcat(dstStr, fileIDStr);
-	strcat(dstStr, extStr);
+	strlcat((char *)dstStr, fileIDStr, maxDstLen);
+	strlcat((char *)dstStr, extStr, maxDstLen);
 	*actualDstLen = utf8len + (strlen(extStr) + strlen(fileIDStr));
 
 	return noErr;
 }
 
+#else /* not HFS - temp workaround until 4277828 is fixed */
+/* stubs for exported routines that aren't present when we build kernel without HFS */
+
+#include <sys/types.h>
+
+int32_t FastUnicodeCompare( void * str1, u_int32_t length1, void * str2, u_int32_t length2 );
+
+
+int32_t FastUnicodeCompare( __unused void * str1, 
+							__unused u_int32_t length1, 
+							__unused void * str2, 
+							__unused u_int32_t length2 )
+{
+	return( 0 );
+}
+
+
+#endif /* HFS */
+
diff --git a/bsd/hfs/hfscommon/headers/BTreeScanner.h b/bsd/hfs/hfscommon/headers/BTreeScanner.h
index 368dd18c1..a806102e6 100644
--- a/bsd/hfs/hfscommon/headers/BTreeScanner.h
+++ b/bsd/hfs/hfscommon/headers/BTreeScanner.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1996-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  *	@(#)BTreeScanner.h
  */
diff --git a/bsd/hfs/hfscommon/headers/BTreesInternal.h b/bsd/hfs/hfscommon/headers/BTreesInternal.h
index 5747e31aa..a5f151953 100644
--- a/bsd/hfs/hfscommon/headers/BTreesInternal.h
+++ b/bsd/hfs/hfscommon/headers/BTreesInternal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreesInternal.h
@@ -155,15 +161,15 @@ enum {
 };
 typedef OptionBits	ReleaseBlockOptions;
 
-typedef	UInt64	FSSize;
-typedef	UInt32	ForkBlockNumber;
+typedef	u_int64_t	FSSize;
+typedef	u_int32_t	ForkBlockNumber;
 
 /*============================================================================
 	Fork Level Buffered I/O Access Method
 ============================================================================*/
 
 typedef	OSStatus	(* GetBlockProcPtr)		(FileReference				 fileRefNum,
-											 UInt32						 blockNum,
+											 u_int32_t					 blockNum,
 											 GetBlockOptions			 options,
 											 BlockDescriptor			*block );
 							 
@@ -202,7 +208,7 @@ enum BTreeIterationOperations{
 	kBTreeLastRecord,
 	kBTreeCurrentRecord
 };
-typedef UInt16 BTreeIterationOperation;
+typedef u_int16_t BTreeIterationOperation;
 
 
 /*
@@ -228,30 +234,30 @@ typedef BTreeKey *BTreeKeyPtr;
 	BTreeInfoRec Structure - for BTGetInformation
 */
 struct BTreeInfoRec{
-	UInt16				version;
-	UInt16				nodeSize;
-	UInt16				maxKeyLength;
-	UInt16				treeDepth;
-	UInt32				lastfsync;		/* Last time that this was fsynced  */
+	u_int16_t			version;
+	u_int16_t			nodeSize;
+	u_int16_t			maxKeyLength;
+	u_int16_t			treeDepth;
+	u_int32_t			lastfsync;		/* Last time that this was fsynced  */
 	ItemCount			numRecords;
 	ItemCount			numNodes;
 	ItemCount			numFreeNodes;
-	UInt8				keyCompareType;
-	UInt8				reserved[3];
+	u_int8_t			keyCompareType;
+	u_int8_t			reserved[3];
 };
 typedef struct BTreeInfoRec BTreeInfoRec;
 typedef BTreeInfoRec *BTreeInfoPtr;
 
 /*
-	BTreeHint can never be exported to the outside. Use UInt32 BTreeHint[4],
-	UInt8 BTreeHint[16], etc.
+	BTreeHint can never be exported to the outside. Use u_int32_t BTreeHint[4],
+	u_int8_t BTreeHint[16], etc.
  */
 struct BTreeHint{
 	ItemCount				writeCount;
-	UInt32					nodeNum;			// node the key was last seen in
-	UInt16					index;				// index then key was last seen at
-	UInt16					reserved1;
-	UInt32					reserved2;
+	u_int32_t				nodeNum;			// node the key was last seen in
+	u_int16_t				index;				// index then key was last seen at
+	u_int16_t				reserved1;
+	u_int32_t				reserved2;
 };
 typedef struct BTreeHint BTreeHint;
 typedef BTreeHint *BTreeHintPtr;
@@ -261,10 +267,10 @@ typedef BTreeHint *BTreeHintPtr;
 */
 struct BTreeIterator{
 	BTreeHint				hint;
-	UInt16					version;
-	UInt16					reserved;
-	UInt32					hitCount;			// Total number of leaf records hit
-	UInt32					maxLeafRecs;		// Max leaf records over iteration
+	u_int16_t				version;
+	u_int16_t				reserved;
+	u_int32_t				hitCount;			// Total number of leaf records hit
+	u_int32_t				maxLeafRecs;		// Max leaf records over iteration
 	BTreeKey				key;
 };
 typedef struct BTreeIterator BTreeIterator;
@@ -278,10 +284,10 @@ typedef BTreeIterator *BTreeIteratorPtr;
 /*
 	Key Comparison Function ProcPtr Type - for BTOpenPath
 */
-//typedef SInt32 				(* KeyCompareProcPtr)(BTreeKeyPtr a, BTreeKeyPtr b);
+//typedef int32_t 				(* KeyCompareProcPtr)(BTreeKeyPtr a, BTreeKeyPtr b);
 
 
-typedef SInt32 (* IterateCallBackProcPtr)(BTreeKeyPtr key, void * record, void * state);
+typedef int32_t (* IterateCallBackProcPtr)(BTreeKeyPtr key, void * record, void * state);
 
 
 extern OSStatus	BTOpenPath(FCB *filePtr, KeyCompareProcPtr keyCompareProc);
@@ -292,14 +298,14 @@ extern OSStatus	BTClosePath			(FCB		 				*filePtr );
 extern OSStatus	BTSearchRecord		(FCB		 				*filePtr,
 									 BTreeIterator				*searchIterator,
 									 FSBufferDescriptor			*btRecord,
-									 UInt16						*recordLen,
+									 u_int16_t					*recordLen,
 									 BTreeIterator				*resultIterator );
 
 extern OSStatus	BTIterateRecord		(FCB		 				*filePtr,
 									 BTreeIterationOperation	 operation,
 									 BTreeIterator				*iterator,
 									 FSBufferDescriptor			*btRecord,
-									 UInt16						*recordLen );
+									 u_int16_t					*recordLen );
 
 
 extern OSStatus BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation, BTreeIterator *iterator,
@@ -308,12 +314,12 @@ extern OSStatus BTIterateRecords(FCB *filePtr, BTreeIterationOperation operation
 extern OSStatus	BTInsertRecord		(FCB		 				*filePtr,
 									 BTreeIterator				*iterator,
 									 FSBufferDescriptor			*btrecord,
-									 UInt16						 recordLen );
+									 u_int16_t					 recordLen );
 
 extern OSStatus	BTReplaceRecord		(FCB		 				*filePtr,
 									 BTreeIterator				*iterator,
 									 FSBufferDescriptor			*btRecord,
-									 UInt16						 recordLen );
+									 u_int16_t					 recordLen );
 
 extern OSStatus	BTUpdateRecord		(FCB		 				*filePtr,
 									 BTreeIterator				*iterator,
@@ -324,9 +330,11 @@ extern OSStatus	BTDeleteRecord		(FCB		 				*filePtr,
 									 BTreeIterator				*iterator );
 
 extern OSStatus	BTGetInformation	(FCB		 				*filePtr,
-									 UInt16						 vers,
+									 u_int16_t					 vers,
 									 BTreeInfoRec				*info );
 
+extern OSStatus BTIsDirty(FCB *filePtr);
+
 extern OSStatus	BTFlushPath			(FCB		 				*filePtr );
 
 extern OSStatus BTReloadData		(FCB *filePtr);
@@ -334,10 +342,10 @@ extern OSStatus BTReloadData		(FCB *filePtr);
 extern OSStatus	BTInvalidateHint	(BTreeIterator				*iterator );
 
 extern OSStatus	BTGetLastSync		(FCB		 				*filePtr,
-									 UInt32						*lastfsync );
+									 u_int32_t					*lastfsync );
 
 extern OSStatus	BTSetLastSync		(FCB		 				*filePtr,
-									 UInt32						lastfsync );
+									 u_int32_t					lastfsync );
 
 extern OSStatus	BTHasContiguousNodes(FCB		 				*filePtr);
 
diff --git a/bsd/hfs/hfscommon/headers/BTreesPrivate.h b/bsd/hfs/hfscommon/headers/BTreesPrivate.h
index 852942dd0..c5d5ef4ad 100644
--- a/bsd/hfs/hfscommon/headers/BTreesPrivate.h
+++ b/bsd/hfs/hfscommon/headers/BTreesPrivate.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		BTreesPrivate.h
@@ -185,50 +191,51 @@ typedef enum {
 
 typedef struct BTreeControlBlock {					// fields specific to BTree CBs
 
-	UInt8		keyCompareType;   /* Key string Comparison Type */
-	UInt8						 btreeType;
-	UInt16						 treeDepth;
+	u_int8_t		keyCompareType;   /* Key string Comparison Type */
+	u_int8_t					 btreeType;
+	u_int16_t					 treeDepth;
 	FileReference				 fileRefNum;		// refNum of btree file
 	KeyCompareProcPtr			 keyCompareProc;
-	UInt32						 rootNode;
-	UInt32						 leafRecords;
-	UInt32						 firstLeafNode;
-	UInt32						 lastLeafNode;
-	UInt16						 nodeSize;
-	UInt16						 maxKeyLength;
-	UInt32						 totalNodes;
-	UInt32						 freeNodes;
+	u_int32_t					 rootNode;
+	u_int32_t					 leafRecords;
+	u_int32_t					 firstLeafNode;
+	u_int32_t					 lastLeafNode;
+	u_int16_t					 nodeSize;
+	u_int16_t					 maxKeyLength;
+	u_int32_t					 totalNodes;
+	u_int32_t					 freeNodes;
 
-	UInt16						 reserved3;			// 4-byte alignment
+	u_int16_t					 reserved3;			// 4-byte alignment
 
 	// new fields
-	SInt16						 version;
-	UInt32						 flags;				// dynamic flags
-	UInt32						 attributes;		// persistent flags
-	UInt32						 writeCount;
-	UInt32						 lastfsync;		/* Last time that this was fsynced  */
+	int16_t						 version;
+	u_int32_t					 flags;				// dynamic flags
+	u_int32_t					 attributes;		// persistent flags
+	u_int32_t					 writeCount;
+	u_int32_t					 lastfsync;		/* Last time that this was fsynced  */
 
 	GetBlockProcPtr			 	 getBlockProc;
 	ReleaseBlockProcPtr			 releaseBlockProc;
 	SetEndOfForkProcPtr			 setEndOfForkProc;
 
 	// statistical information
-	UInt32						 numGetNodes;
-	UInt32						 numGetNewNodes;
-	UInt32						 numReleaseNodes;
-	UInt32						 numUpdateNodes;
-	UInt32						 numMapNodesRead;	// map nodes beyond header node
-	UInt32						 numHintChecks;
-	UInt32						 numPossibleHints;	// Looks like a formated hint
-	UInt32						 numValidHints;		// Hint used to find correct record.
-	UInt32					reservedNodes;
+	u_int32_t					 numGetNodes;
+	u_int32_t					 numGetNewNodes;
+	u_int32_t					 numReleaseNodes;
+	u_int32_t					 numUpdateNodes;
+	u_int32_t					 numMapNodesRead;	// map nodes beyond header node
+	u_int32_t					 numHintChecks;
+	u_int32_t					 numPossibleHints;	// Looks like a formated hint
+	u_int32_t					 numValidHints;		// Hint used to find correct record.
+	u_int32_t					reservedNodes;
+	BTreeIterator   iterator; // useable when holding exclusive b-tree lock
 } BTreeControlBlock, *BTreeControlBlockPtr;
 
 
-UInt32 CalcKeySize(const BTreeControlBlock *btcb, const BTreeKey *key);
+u_int32_t CalcKeySize(const BTreeControlBlock *btcb, const BTreeKey *key);
 #define CalcKeySize(btcb, key)			( ((btcb)->attributes & kBTBigKeysMask) ? ((key)->length16 + 2) : ((key)->length8 + 1) )
 
-UInt32 KeyLength(const BTreeControlBlock *btcb, const BTreeKey *key);
+u_int32_t KeyLength(const BTreeControlBlock *btcb, const BTreeKey *key);
 #define KeyLength(btcb, key)			( ((btcb)->attributes & kBTBigKeysMask) ? (key)->length16 : (key)->length8 )
 
 
@@ -238,7 +245,7 @@ typedef enum {
 }	BTreeFlags;
 
 
-typedef	SInt8				*NodeBuffer;
+typedef	int8_t				*NodeBuffer;
 typedef BlockDescriptor		 NodeRec, *NodePtr;		// remove this someday...
 
 
@@ -247,9 +254,9 @@ typedef BlockDescriptor		 NodeRec, *NodePtr;		//
 //// Tree Path Table - constructed by SearchTree, used by InsertTree and DeleteTree
 
 typedef struct {
-	UInt32				node;				// node number
-	UInt16				index;
-	UInt16				reserved;			// align size to a power of 2
+	u_int32_t				node;				// node number
+	u_int16_t				index;
+	u_int16_t				reserved;			// align size to a power of 2
 } TreePathRecord, *TreePathRecordPtr;
 
 typedef TreePathRecord		TreePathTable [kMaxTreeDepth];
@@ -259,9 +266,9 @@ typedef TreePathRecord		TreePathTable [kMaxTreeDepth];
 
 struct InsertKey {
 	BTreeKeyPtr		keyPtr;
-	UInt8 *			recPtr;
-	UInt16			keyLength;
-	UInt16			recSize;
+	u_int8_t *		recPtr;
+	u_int16_t		keyLength;
+	u_int16_t		recSize;
 	Boolean			replacingKey;
 	Boolean			skipRotate;
 };
@@ -272,7 +279,7 @@ typedef struct InsertKey InsertKey;
 //// For Notational Convenience
 
 typedef	BTNodeDescriptor*	 NodeDescPtr;
-typedef UInt8				*RecordPtr;
+typedef u_int8_t			*RecordPtr;
 typedef BTreeKeyPtr			 KeyPtr;
 
 
@@ -283,46 +290,45 @@ typedef BTreeKeyPtr			 KeyPtr;
 
 #if DEBUG_BUILD
 	#define Panic( message )					DebugStr( (ConstStr255Param) message )
-	#define PanicIf( condition, message )		if ( condition != 0 )	DebugStr( message )
+	#define PanicIf( condition, message )		do { if ( condition != 0 )	DebugStr( message ); } while(0)
 #else
-	#define Panic( message )
-	#define PanicIf( condition, message )
+	#define Panic( message )				do { } while(0)
+	#define PanicIf( condition, message )	do { } while(0)
 #endif
 
 //	Exit function on error
-#define M_ExitOnError( result )	if ( ( result ) != noErr )	goto ErrorExit; else ;
+#define M_ExitOnError( result )	do { if ( ( result ) != noErr )	goto ErrorExit; } while(0)
 
 //	Test for passed condition and return if true
-#define	M_ReturnErrorIf( condition, error )	if ( condition )	return( error )
+#define	M_ReturnErrorIf( condition, error )	do { if ( condition )	return( error ); } while(0)
 
 //////////////////////////////// Key Operations /////////////////////////////////
 
-SInt32		CompareKeys				(BTreeControlBlockPtr	 btreePtr,
+int32_t		CompareKeys				(BTreeControlBlockPtr	 btreePtr,
 									 KeyPtr					 searchKey,
 									 KeyPtr					 trialKey );
 
 //////////////////////////////// Map Operations /////////////////////////////////
 
 OSStatus	AllocateNode			(BTreeControlBlockPtr	 btreePtr,
-									 UInt32					*nodeNum);
+									 u_int32_t				*nodeNum);
 
 OSStatus	FreeNode				(BTreeControlBlockPtr	 btreePtr,
-									 UInt32					 nodeNum);
+									 u_int32_t				 nodeNum);
 
 OSStatus	ExtendBTree				(BTreeControlBlockPtr	 btreePtr,
-									 UInt32					 nodes );
+									 u_int32_t				 nodes );
 
-UInt32		CalcMapBits				(BTreeControlBlockPtr	 btreePtr);
+u_int32_t	CalcMapBits				(BTreeControlBlockPtr	 btreePtr);
 
-SInt32		BTAvailableNodes			(BTreeControlBlock *btree);
 
 void 		BTUpdateReserve				(BTreeControlBlockPtr btreePtr,
                                                          int nodes);
 
 //////////////////////////////// Misc Operations ////////////////////////////////
 
-UInt16		CalcKeyRecordSize		(UInt16					 keySize,
-									 UInt16					 recSize );
+u_int16_t	CalcKeyRecordSize		(u_int16_t				 keySize,
+									 u_int16_t				 recSize );
 
 OSStatus	VerifyHeader			(FCB					*filePtr,
 									 BTHeaderRec				*header );
@@ -335,32 +341,34 @@ OSStatus	FindIteratorPosition	(BTreeControlBlockPtr	 btreePtr,
 									 BlockDescriptor		*left,
 									 BlockDescriptor		*middle,
 									 BlockDescriptor		*right,
-									 UInt32					*nodeNum,
-									 UInt16					*index,
+									 u_int32_t				*nodeNum,
+									 u_int16_t				*index,
 									 Boolean				*foundRecord );
 
 OSStatus	CheckInsertParams		(FCB					*filePtr,
 									 BTreeIterator			*iterator,
 									 FSBufferDescriptor		*record,
-									 UInt16					 recordLen );
+									 u_int16_t				 recordLen );
 
 OSStatus	TrySimpleReplace		(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 nodePtr,
 									 BTreeIterator			*iterator,
 									 FSBufferDescriptor		*record,
-									 UInt16					 recordLen,
+									 u_int16_t				 recordLen,
 									 Boolean				*recordInserted );
 
 OSStatus	IsItAHint				(BTreeControlBlockPtr 	 btreePtr, 
 									 BTreeIterator 			*iterator, 
 									 Boolean 				*answer );
 
+extern OSStatus TreeIsDirty(BTreeControlBlockPtr btreePtr);
+
 //////////////////////////////// Node Operations ////////////////////////////////
 
 //// Node Operations
 
 OSStatus	GetNode					(BTreeControlBlockPtr	 btreePtr,
-									 UInt32					 nodeNum,
+									 u_int32_t				 nodeNum,
 									 NodeRec				*returnNodePtr );
 
 OSStatus	GetLeftSiblingNode		(BTreeControlBlockPtr	 btreePtr,
@@ -377,7 +385,7 @@ OSStatus	GetRightSiblingNode		(BTreeControlBlockPtr	 btreePtr,
 
 
 OSStatus	GetNewNode				(BTreeControlBlockPtr	 btreePtr,
-									 UInt32					 nodeNum,
+									 u_int32_t				 nodeNum,
 									 NodeRec				*returnNodePtr );
 
 OSStatus	ReleaseNode				(BTreeControlBlockPtr	 btreePtr,
@@ -386,29 +394,20 @@ OSStatus	ReleaseNode				(BTreeControlBlockPtr	 btreePtr,
 OSStatus	TrashNode				(BTreeControlBlockPtr	 btreePtr,
 									 NodePtr				 nodePtr );
 
-// XXXdbg
-void ModifyBlockStart(FileReference vp, BlockDescPtr blockPtr);
-// XXXdbg
-
 OSStatus	UpdateNode				(BTreeControlBlockPtr	 btreePtr,
 									 NodePtr				 nodePtr,
-									 UInt32					 transactionID,
-									 UInt32					 flags );
-
-OSStatus	GetMapNode				(BTreeControlBlockPtr	 btreePtr,
-									 BlockDescriptor		 *nodePtr,
-									 UInt16					 **mapPtr,
-									 UInt16					 *mapSize );
+									 u_int32_t				 transactionID,
+									 u_int32_t				 flags );
 
 //// Node Buffer Operations
 
 void		ClearNode				(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 node );
 
-UInt16		GetNodeDataSize			(BTreeControlBlockPtr	 btreePtr,
+u_int16_t	GetNodeDataSize			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 node );
 
-UInt16		GetNodeFreeSize			(BTreeControlBlockPtr	 btreePtr,
+u_int16_t	GetNodeFreeSize			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 node );
 
 
@@ -416,59 +415,59 @@ UInt16		GetNodeFreeSize			(BTreeControlBlockPtr	 btreePtr,
 
 Boolean		InsertRecord			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr	 		 node,
-									 UInt16	 				 index,
+									 u_int16_t	 			 index,
 									 RecordPtr				 recPtr,
-									 UInt16					 recSize );
+									 u_int16_t				 recSize );
 
 Boolean		InsertKeyRecord			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr 			 node,
-									 UInt16	 				 index,
+									 u_int16_t	 			 index,
 									 KeyPtr					 keyPtr,
-									 UInt16					 keyLength,
+									 u_int16_t				 keyLength,
 									 RecordPtr				 recPtr,
-									 UInt16					 recSize );
+									 u_int16_t				 recSize );
 
 void		DeleteRecord			(BTreeControlBlockPtr	btree,
 									 NodeDescPtr	 		node,
-									 UInt16	 				index );
+									 u_int16_t	 			index );
 
 
 Boolean		SearchNode				(BTreeControlBlockPtr	 btree,
 									 NodeDescPtr			 node,
 									 KeyPtr					 searchKey,
-									 UInt16					*index );
+									 u_int16_t				*index );
 
 OSStatus	GetRecordByIndex		(BTreeControlBlockPtr	 btree,
 									 NodeDescPtr			 node,
-									 UInt16					 index,
+									 u_int16_t				 index,
 									 KeyPtr					*keyPtr,
-									 UInt8 *				*dataPtr,
-									 UInt16					*dataSize );
+									 u_int8_t *				*dataPtr,
+									 u_int16_t				*dataSize );
 
-UInt8 *		GetRecordAddress		(BTreeControlBlockPtr	 btree,
+u_int8_t *	GetRecordAddress		(BTreeControlBlockPtr	 btree,
 									 NodeDescPtr			 node,
-									 UInt16					 index );
+									 u_int16_t				 index );
 
-#define GetRecordAddress(btreePtr,node,index)		((UInt8 *)(node) + (*(short *) ((UInt8 *)(node) + (btreePtr)->nodeSize - ((index) << 1) - kOffsetSize)))
+#define GetRecordAddress(btreePtr,node,index)		((u_int8_t *)(node) + (*(short *) ((u_int8_t *)(node) + (btreePtr)->nodeSize - ((index) << 1) - kOffsetSize)))
 
 
-UInt16		GetRecordSize			(BTreeControlBlockPtr	 btree,
+u_int16_t	GetRecordSize			(BTreeControlBlockPtr	 btree,
 									 NodeDescPtr			 node,
-									 UInt16					 index );
+									 u_int16_t				 index );
 
-UInt32		GetChildNodeNum			(BTreeControlBlockPtr	 btreePtr,
+u_int32_t	GetChildNodeNum			(BTreeControlBlockPtr	 btreePtr,
 									 NodeDescPtr			 nodePtr,
-									 UInt16					 index );
+									 u_int16_t				 index );
 
-void		MoveRecordsLeft			(UInt8 *				 src,
-									 UInt8 *				 dst,
-									 UInt16					 bytesToMove );
+void		MoveRecordsLeft			(u_int8_t *				 src,
+									 u_int8_t *				 dst,
+									 u_int16_t				 bytesToMove );
 
 #define		MoveRecordsLeft(src,dst,bytes)			bcopy((src),(dst),(bytes))
 
-void		MoveRecordsRight		(UInt8 *				 src,
-									 UInt8 *				 dst,
-									 UInt16					 bytesToMove );
+void		MoveRecordsRight		(u_int8_t *				 src,
+									 u_int8_t *				 dst,
+									 u_int16_t				 bytesToMove );
 
 #define		MoveRecordsRight(src,dst,bytes)			bcopy((src),(dst),(bytes))
 
@@ -478,26 +477,26 @@ void		MoveRecordsRight		(UInt8 *				 src,
 OSStatus	SearchTree				(BTreeControlBlockPtr	 btreePtr,
 									 BTreeKeyPtr			 keyPtr,
 									 TreePathTable			 treePathTable,
-									 UInt32					*nodeNum,
+									 u_int32_t				*nodeNum,
 									 BlockDescriptor		*nodePtr,
-									 UInt16					*index );
+									 u_int16_t				*index );
 
 OSStatus	InsertTree				(BTreeControlBlockPtr	 btreePtr,
 									 TreePathTable			 treePathTable,
 									 KeyPtr					 keyPtr,
-									 UInt8 *				 recPtr,
-									 UInt16					 recSize,
+									 u_int8_t *				 recPtr,
+									 u_int16_t				 recSize,
 									 BlockDescriptor		*targetNode,
-									 UInt16					 index,
-									 UInt16					 level,
+									 u_int16_t				 index,
+									 u_int16_t				 level,
 									 Boolean				 replacingKey,
-									 UInt32					*insertNode );
+									 u_int32_t				*insertNode );
 
 OSStatus	DeleteTree				(BTreeControlBlockPtr	 btreePtr,
 									 TreePathTable			 treePathTable,
 									 BlockDescriptor		*targetNode,
-									 UInt16					 index,
-									 UInt16					 level );
+									 u_int16_t				 index,
+									 u_int16_t				 level );
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
diff --git a/bsd/hfs/hfscommon/headers/CatalogPrivate.h b/bsd/hfs/hfscommon/headers/CatalogPrivate.h
index fcf12ac7c..319cd0ad9 100644
--- a/bsd/hfs/hfscommon/headers/CatalogPrivate.h
+++ b/bsd/hfs/hfscommon/headers/CatalogPrivate.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		CatalogPrivate.h
@@ -85,17 +91,17 @@
 
 
 extern	OSErr	LocateCatalogNode(	const ExtendedVCB *volume, HFSCatalogNodeID folderID, const CatalogName *name,
-									UInt32 hint, CatalogKey *key, CatalogRecord *data, UInt32 *newHint);
+									u_int32_t hint, CatalogKey *key, CatalogRecord *data, u_int32_t *newHint);
 
-extern OSErr	LocateCatalogNodeByKey ( const ExtendedVCB *volume, UInt32 hint, CatalogKey *keyPtr,
-										 CatalogRecord *dataPtr, UInt32 *newHint );
+extern OSErr	LocateCatalogNodeByKey ( const ExtendedVCB *volume, u_int32_t hint, CatalogKey *keyPtr,
+										 CatalogRecord *dataPtr, u_int32_t *newHint );
 
 extern OSErr	LocateCatalogRecord( const ExtendedVCB *volume, HFSCatalogNodeID folderID, const CatalogName *name,
-									 UInt32 hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, UInt32 *newHint);
+									 u_int32_t hint, CatalogKey *keyPtr, CatalogRecord *dataPtr, u_int32_t *newHint);
 
 extern OSErr	LocateCatalogNodeWithRetry ( const ExtendedVCB *volume, HFSCatalogNodeID folderID, ConstStr31Param pascalName,
-											 CatalogName *unicodeName, UInt32 hint, CatalogKey *keyPtr, CatalogRecord *dataPtr,
-											 UInt32 *newHint );
+											 CatalogName *unicodeName, u_int32_t hint, CatalogKey *keyPtr, CatalogRecord *dataPtr,
+											 u_int32_t *newHint );
 extern OSErr	FlushCatalog( ExtendedVCB *volume);
 
 
@@ -105,8 +111,8 @@ extern void		ConvertInputNameToUnicode(ConstStr31Param name, TextEncoding encodi
 extern	void	BuildCatalogKey( HFSCatalogNodeID parentID, const CatalogName *name, Boolean isHFSPlus,
 								 CatalogKey *key);
 
-extern	OSErr	BuildCatalogKeyUTF8(ExtendedVCB *volume, HFSCatalogNodeID parentID, const char *name,
-				    UInt32 length, CatalogKey *key, UInt32 *textEncoding);
+extern	OSErr	BuildCatalogKeyUTF8(ExtendedVCB *volume, HFSCatalogNodeID parentID, const unsigned char *name,
+				    u_int32_t length, CatalogKey *key, u_int32_t *textEncoding);
 
 extern void		CopyCatalogName( const CatalogName *srcName, CatalogName *dstName, Boolean isHFSPLus);
 
diff --git a/bsd/hfs/hfscommon/headers/FileMgrInternal.h b/bsd/hfs/hfscommon/headers/FileMgrInternal.h
index 15ccb6e63..7bb16bc5a 100644
--- a/bsd/hfs/hfscommon/headers/FileMgrInternal.h
+++ b/bsd/hfs/hfscommon/headers/FileMgrInternal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		FilesInternal.h
@@ -67,7 +73,7 @@ extern "C" {
 #endif
 
 /* CatalogNodeID is used to track catalog objects */
-typedef UInt32		HFSCatalogNodeID;
+typedef u_int32_t		HFSCatalogNodeID;
 
 /* internal error codes*/
 
@@ -174,34 +180,17 @@ typedef union CatalogName CatalogName;
 #define GetFileControlBlock(fref)		VTOF((fref))
 #define GetFileRefNumFromFCB(fcb)		FTOV((fcb))
 
-
-/*	The following macro marks a VCB as dirty by setting the upper 8 bits of the flags*/
-EXTERN_API_C( void )
-MarkVCBDirty					(ExtendedVCB *vcb);
-
-EXTERN_API_C( void )
-MarkVCBClean					(ExtendedVCB *vcb);
-
-EXTERN_API_C( Boolean )
-IsVCBDirty						(ExtendedVCB *vcb);
-
-
-#define	MarkVCBDirty(vcb)		{ ((vcb)->vcbFlags |= 0xFF00); }
-#define	MarkVCBClean(vcb)		{ ((vcb)->vcbFlags &= 0x00FF); }
-#define	IsVCBDirty(vcb)			((Boolean) ((vcb->vcbFlags & 0xFF00) != 0))
-
-
 /*	Test for error and return if error occurred*/
 EXTERN_API_C( void )
 ReturnIfError					(OSErr 					result);
 
-#define	ReturnIfError(result)					if ( (result) != noErr ) return (result); else ;
+#define	ReturnIfError(result)				do {	if ( (result) != noErr ) return (result); } while(0)
 
 /*	Exit function on error*/
 EXTERN_API_C( void )
 ExitOnError						(OSErr 					result);
 
-#define	ExitOnError( result )					if ( ( result ) != noErr )	goto ErrorExit; else ;
+#define	ExitOnError( result )				do {	if ( ( result ) != noErr )	goto ErrorExit; } while(0)
 
 
 
@@ -213,83 +202,83 @@ ExchangeFileIDs					(ExtendedVCB *			volume,
 								 ConstUTF8Param			destName,
 								 HFSCatalogNodeID		srcID,
 								 HFSCatalogNodeID		destID,
-								 UInt32					srcHint,
-								 UInt32					destHint );
+								 u_int32_t				srcHint,
+								 u_int32_t				destHint );
 
 
 /* BTree Manager Routines*/
 
-typedef CALLBACK_API_C( SInt32 , KeyCompareProcPtr )(void *a, void *b);
+typedef CALLBACK_API_C( int32_t , KeyCompareProcPtr )(void *a, void *b);
 
 
 EXTERN_API_C( OSErr )
 SearchBTreeRecord				(FileReference 				refNum,
 								 const void *			key,
-								 UInt32 				hint,
+								 u_int32_t 				hint,
 								 void *					foundKey,
 								 void *					data,
-								 UInt16 *				dataSize,
-								 UInt32 *				newHint);
+								 u_int16_t *			dataSize,
+								 u_int32_t *			newHint);
 
 EXTERN_API_C( OSErr )
 ReplaceBTreeRecord				(FileReference 				refNum,
 								 const void *			key,
-								 UInt32 				hint,
+								 u_int32_t 				hint,
 								 void *					newData,
-								 UInt16 				dataSize,
-								 UInt32 *				newHint);
+								 u_int16_t 				dataSize,
+								 u_int32_t *			newHint);
 
 
 /*	Prototypes for exported routines in VolumeAllocation.c*/
 EXTERN_API_C( OSErr )
 BlockAllocate					(ExtendedVCB *			vcb,
-								 UInt32 				startingBlock,
-								 UInt32 				minBlocks,
-								 UInt32 				maxBlocks,
+								 u_int32_t 				startingBlock,
+								 u_int32_t 				minBlocks,
+								 u_int32_t 				maxBlocks,
 								 Boolean 				forceContiguous,
 								 Boolean				useMetaZone,
-								 UInt32 *				startBlock,
-								 UInt32 *				actualBlocks);
+								 u_int32_t *			startBlock,
+								 u_int32_t *			actualBlocks);
 
 EXTERN_API_C( OSErr )
 BlockDeallocate					(ExtendedVCB *			vcb,
-								 UInt32 				firstBlock,
-								 UInt32 				numBlocks);
+								 u_int32_t 				firstBlock,
+								 u_int32_t 				numBlocks);
 
 EXTERN_API_C( OSErr )
-BlockMarkAllocated(ExtendedVCB *vcb, UInt32 startingBlock, UInt32 numBlocks);
+BlockMarkAllocated(ExtendedVCB *vcb, u_int32_t startingBlock, u_int32_t numBlocks);
 
 EXTERN_API_C( OSErr )
-BlockMarkFree( ExtendedVCB *vcb, UInt32 startingBlock, UInt32 numBlocks);
+BlockMarkFree( ExtendedVCB *vcb, u_int32_t startingBlock, u_int32_t numBlocks);
 
-EXTERN_API_C( UInt32 )
+EXTERN_API_C( u_int32_t )
 MetaZoneFreeBlocks(ExtendedVCB *vcb);
 
 /*	File Extent Mapping routines*/
 EXTERN_API_C( OSErr )
 FlushExtentFile					(ExtendedVCB *			vcb);
 
-EXTERN_API_C( SInt32 )
+EXTERN_API_C( int32_t )
 CompareExtentKeys				(const HFSExtentKey *	searchKey,
 								 const HFSExtentKey *	trialKey);
 
-EXTERN_API_C( SInt32 )
+EXTERN_API_C( int32_t )
 CompareExtentKeysPlus			(const HFSPlusExtentKey *searchKey,
 								 const HFSPlusExtentKey *trialKey);
 
 EXTERN_API_C( OSErr )
 TruncateFileC					(ExtendedVCB *			vcb,
 								 FCB *					fcb,
-								 SInt64 				peof,
+								 int64_t 				peof,
 								 Boolean 				truncateToExtent);
 
 EXTERN_API_C( OSErr )
 ExtendFileC						(ExtendedVCB *			vcb,
 								 FCB *					fcb,
-								 SInt64 				bytesToAdd,
-								 UInt32 				blockHint,
-								 UInt32 				flags,
-								 SInt64 *				actualBytesAdded);
+								 int64_t 				bytesToAdd,
+								 u_int32_t 				blockHint,
+								 u_int32_t 				flags,
+								 int64_t *				actualBytesAdded);
 
 EXTERN_API_C( OSErr )
 MapFileBlockC					(ExtendedVCB *			vcb,
@@ -299,29 +288,29 @@ MapFileBlockC					(ExtendedVCB *			vcb,
 								 daddr64_t *				startBlock,
 								 size_t *				availableBytes);
 
-OSErr HeadTruncateFile(ExtendedVCB  *vcb, FCB  *fcb, UInt32  headblks);
+OSErr HeadTruncateFile(ExtendedVCB  *vcb, FCB  *fcb, u_int32_t  headblks);
 
 EXTERN_API_C( int )
-AddFileExtent (ExtendedVCB *vcb, FCB *fcb, UInt32 startBlock, UInt32 blockCount);
+AddFileExtent (ExtendedVCB *vcb, FCB *fcb, u_int32_t startBlock, u_int32_t blockCount);
 
 #if TARGET_API_MACOS_X
 EXTERN_API_C( Boolean )
 NodesAreContiguous				(ExtendedVCB *			vcb,
 								 FCB *					fcb,
-								 UInt32					nodeSize);
+								 u_int32_t				nodeSize);
 #endif
 
 
 
 /*	Get the current time in UTC (GMT)*/
-EXTERN_API_C( UInt32 )
+EXTERN_API_C( u_int32_t )
 GetTimeUTC						(void);
 
-EXTERN_API_C( UInt32 )
-LocalToUTC						(UInt32 				localTime);
+EXTERN_API_C( u_int32_t )
+LocalToUTC						(u_int32_t 				localTime);
 
-EXTERN_API_C( UInt32 )
-UTCToLocal						(UInt32 				utcTime);
+EXTERN_API_C( u_int32_t )
+UTCToLocal						(u_int32_t 				utcTime);
 
 
 #if PRAGMA_STRUCT_ALIGN
diff --git a/bsd/hfs/hfscommon/headers/HFSUnicodeWrappers.h b/bsd/hfs/hfscommon/headers/HFSUnicodeWrappers.h
index 56a7e4bdd..8fbb287f9 100644
--- a/bsd/hfs/hfscommon/headers/HFSUnicodeWrappers.h
+++ b/bsd/hfs/hfscommon/headers/HFSUnicodeWrappers.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2003, 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		HFSUnicodeWrappers.h
@@ -97,14 +103,14 @@ extern OSErr ConvertUnicodeToUTF8Mangled ( ByteCount srcLen,
 	negative number may be returned.
 */
 
-extern SInt32 FastUnicodeCompare(register ConstUniCharArrayPtr str1, register ItemCount length1,
+extern int32_t FastUnicodeCompare(register ConstUniCharArrayPtr str1, register ItemCount length1,
 								 register ConstUniCharArrayPtr str2, register ItemCount length2);
 
 
-extern SInt32 FastRelString( ConstStr255Param str1, ConstStr255Param str2 );
+extern int32_t FastRelString( ConstStr255Param str1, ConstStr255Param str2 );
 
 
-extern HFSCatalogNodeID GetEmbeddedFileID( ConstStr31Param filename, UInt32 length, UInt32 *prefixLength );
+extern HFSCatalogNodeID GetEmbeddedFileID( ConstStr31Param filename, u_int32_t length, u_int32_t *prefixLength );
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL */
diff --git a/bsd/hfs/rangelist.c b/bsd/hfs/rangelist.c
index cc062004f..e21a962dd 100644
--- a/bsd/hfs/rangelist.c
+++ b/bsd/hfs/rangelist.c
@@ -1,25 +1,33 @@
 /*
  * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#if HFS
+
 #include <sys/param.h>
 #include <mach/boolean.h>
 #include <sys/time.h>
@@ -433,3 +441,35 @@ rl_collapse_neighbors(struct rl_head *rangelist, struct rl_entry *range)
     rl_collapse_forwards(rangelist, range);
     rl_collapse_backwards(rangelist, range);
 }
+
+#else /* not HFS - temp workaround until 4277828 is fixed */
+/* stubs for exported routines that aren't present when we build kernel without HFS */
+
+#include <sys/types.h>
+
+void rl_add(off_t start, off_t end, void *rangelist);
+void rl_init(void *rangelist);
+void rl_remove(off_t start, off_t end, void *rangelist);
+int rl_scan(void *rangelist, off_t start, off_t end, void **overlap);
+
+void rl_add(__unused off_t start, __unused off_t end, __unused void *rangelist)
+{
+	return;
+}
+
+void rl_init(__unused void *rangelist)
+{
+	return;
+}
+
+void rl_remove(__unused off_t start, __unused off_t end, __unused void *rangelist)
+{
+	return;
+}
+
+int rl_scan(__unused void *rangelist, __unused off_t start, __unused off_t end, __unused void **overlap)
+{
+	return(0);
+}
+
+#endif /* HFS */
diff --git a/bsd/hfs/rangelist.h b/bsd/hfs/rangelist.h
index 86b4c0c06..a859d222a 100644
--- a/bsd/hfs/rangelist.h
+++ b/bsd/hfs/rangelist.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _HFS_RANGELIST_H_
 #define _HFS_RANGELIST_H_
diff --git a/bsd/i386/Makefile b/bsd/i386/Makefile
index bb3988af6..9f8e9e84b 100644
--- a/bsd/i386/Makefile
+++ b/bsd/i386/Makefile
@@ -8,14 +8,14 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 DATAFILES = \
-	endian.h param.h \
+	endian.h fasttrap_isa.h param.h \
 	profile.h setjmp.h signal.h \
-	types.h ucontext.h  vmparam.h _types.h
+	types.h vmparam.h _structs.h _types.h _param.h
 
 KERNELFILES = \
 	endian.h param.h \
 	profile.h setjmp.h signal.h \
-	types.h vmparam.h _types.h
+	types.h vmparam.h _structs.h _types.h _param.h
 
 
 INSTALL_MD_LIST = ${DATAFILES}
diff --git a/bsd/i386/_param.h b/bsd/i386/_param.h
new file mode 100644
index 000000000..b7cbc128a
--- /dev/null
+++ b/bsd/i386/_param.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _I386__PARAM_H_
+#define _I386__PARAM_H_
+
+#include <i386/_types.h>
+
+/*
+ * Round p (pointer or byte index) up to a correctly-aligned value for all
+ * data types (int, long, ...).   The result is unsigned int and must be
+ * cast to any desired pointer type.
+ */
+#define	__DARWIN_ALIGNBYTES	(sizeof(__darwin_size_t) - 1)
+#define	__DARWIN_ALIGN(p)	((__darwin_size_t)((char *)(p) + __DARWIN_ALIGNBYTES) &~ __DARWIN_ALIGNBYTES)
+
+#endif /* _I386__PARAM_H_ */
diff --git a/bsd/i386/_structs.h b/bsd/i386/_structs.h
new file mode 100644
index 000000000..9cad355eb
--- /dev/null
+++ b/bsd/i386/_structs.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/appleapiopts.h>
+
+#ifdef __need_mcontext_t
+#ifndef __need_struct_mcontext
+#define __need_struct_mcontext
+#endif /* __need_struct_mcontext */
+#endif /* __need_mcontext_t */
+
+#if defined(__need_struct_mcontext)
+#include <mach/i386/_structs.h>
+#endif /* __need_struct_mcontext */
+
+
+#ifdef __need_struct_mcontext
+#undef __need_struct_mcontext
+
+#ifndef _STRUCT_MCONTEXT32
+#if __DARWIN_UNIX03
+#define	_STRUCT_MCONTEXT32	struct __darwin_mcontext32
+_STRUCT_MCONTEXT32
+{
+	_STRUCT_X86_EXCEPTION_STATE32	__es;
+	_STRUCT_X86_THREAD_STATE32	__ss;
+	_STRUCT_X86_FLOAT_STATE32	__fs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_MCONTEXT32	struct mcontext32
+_STRUCT_MCONTEXT32
+{
+	_STRUCT_X86_EXCEPTION_STATE32	es;
+	_STRUCT_X86_THREAD_STATE32	ss;
+	_STRUCT_X86_FLOAT_STATE32	fs;
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_MCONTEXT32 */
+
+#ifndef _STRUCT_MCONTEXT64
+#if __DARWIN_UNIX03
+#define	_STRUCT_MCONTEXT64	struct __darwin_mcontext64
+_STRUCT_MCONTEXT64
+{
+	_STRUCT_X86_EXCEPTION_STATE64	__es;
+	_STRUCT_X86_THREAD_STATE64	__ss;
+	_STRUCT_X86_FLOAT_STATE64	__fs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_MCONTEXT64	struct mcontext64
+_STRUCT_MCONTEXT64
+{
+	_STRUCT_X86_EXCEPTION_STATE64	es;
+	_STRUCT_X86_THREAD_STATE64	ss;
+	_STRUCT_X86_FLOAT_STATE64	fs;
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_MCONTEXT64 */
+#endif /* __need_struct_mcontext */
+
+#ifdef __need_mcontext_t
+#undef __need_mcontext_t
+#ifndef _MCONTEXT_T
+#define _MCONTEXT_T
+#if defined(__LP64__)
+typedef _STRUCT_MCONTEXT64	*mcontext_t;
+#define _STRUCT_MCONTEXT _STRUCT_MCONTEXT64
+#else
+typedef _STRUCT_MCONTEXT32	*mcontext_t;
+#define _STRUCT_MCONTEXT	_STRUCT_MCONTEXT32
+#endif
+#endif /* _MCONTEXT_T */
+#endif /* __need_mcontext_t */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef I386_MCONTEXT_SIZE
+#define I386_MCONTEXT_SIZE	sizeof(struct mcontext)	
+#endif /* I386_MCONTEXT_SIZE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
diff --git a/bsd/i386/_types.h b/bsd/i386/_types.h
index 985c42769..67741d5d9 100644
--- a/bsd/i386/_types.h
+++ b/bsd/i386/_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_BSD_I386__TYPES_H_
 #define	_BSD_I386__TYPES_H_
@@ -86,15 +92,11 @@ typedef __SIZE_TYPE__		__darwin_size_t;	/* sizeof() */
 typedef unsigned long		__darwin_size_t;	/* sizeof() */
 #endif
 
-#ifdef KERNEL
-typedef void *			__darwin_va_list;	/* va_list */
-#else /* !KERNEL */
 #if (__GNUC__ > 2)
 typedef __builtin_va_list	__darwin_va_list;	/* va_list */
 #else
 typedef void *			__darwin_va_list;	/* va_list */
 #endif
-#endif /* KERNEL */
 
 #if defined(__GNUC__) && defined(__WCHAR_TYPE__)
 typedef __WCHAR_TYPE__		__darwin_wchar_t;	/* wchar_t */
diff --git a/bsd/i386/dis_tables.h b/bsd/i386/dis_tables.h
new file mode 100644
index 000000000..548b22b60
--- /dev/null
+++ b/bsd/i386/dis_tables.h
@@ -0,0 +1,116 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1988 AT&T	*/
+/*	  All Rights Reserved  	*/
+
+
+#ifndef _DIS_TABLES_H
+#define	_DIS_TABLES_H
+
+/* #pragma ident	"@(#)dis_tables.h	1.8	06/05/30 SMI" */
+
+/*
+ * Constants and prototypes for the IA32 disassembler backend.  See dis_tables.c
+ * for usage information and documentation.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if !defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/inttypes.h>
+#include <sys/param.h>
+#else
+#include <sys/types.h>
+#include <stdint.h>
+#include <sys/param.h>
+#endif /* __APPLE__ */
+
+/*
+ * values for cpu mode
+ */
+#define	SIZE16	1
+#define	SIZE32	2
+#define	SIZE64	3
+
+#define	OPLEN	256
+#define	PFIXLEN	  8
+#define	NCPS	12	/* number of chars per symbol	*/
+
+/*
+ * data structures that must be provided to dtrace_dis86()
+ */
+typedef struct d86opnd {
+	char		d86_opnd[OPLEN];	/* symbolic rep of operand */
+	char		d86_prefix[PFIXLEN];	/* any prefix string or "" */
+	uint_t		d86_mode;		/* mode for immediate */
+	uint_t		d86_value_size;		/* size in bytes of d86_value */
+	uint64_t	d86_value;		/* immediate value of opnd */
+} d86opnd_t;
+
+typedef struct dis86 {
+	uint_t		d86_mode;
+	uint_t		d86_error;
+	uint_t		d86_len;		/* instruction length */
+	int		d86_rmindex;		/* index of modrm byte or -1 */
+	uint_t		d86_memsize;		/* size of memory referenced */
+	char		d86_bytes[16];		/* bytes of instruction */
+	char		d86_mnem[OPLEN];
+	uint_t		d86_numopnds;
+	uint_t		d86_rex_prefix;		/* value of REX prefix if !0 */
+	char		*d86_seg_prefix;	/* segment prefix, if any */
+	uint_t		d86_opnd_size;
+	uint_t		d86_addr_size;
+	uint_t		d86_got_modrm;
+	struct d86opnd	d86_opnd[3];		/* up to 3 operands */
+	int		(*d86_check_func)(void *);
+	int		(*d86_get_byte)(void *);
+#ifdef DIS_TEXT
+	int		(*d86_sym_lookup)(void *, uint64_t, char *, size_t);
+	int		(*d86_sprintf_func)(char *, size_t, const char *, ...);
+	int		d86_flags;
+	uint_t		d86_imm_bytes;
+#endif
+	void		*d86_data;
+} dis86_t;
+
+extern int dtrace_disx86(dis86_t *x, uint_t cpu_mode);
+
+#define	DIS_F_OCTAL	0x1	/* Print all numbers in octal */
+
+#ifdef DIS_TEXT
+extern void dtrace_disx86_str(dis86_t *x, uint_t cpu_mode, uint64_t pc,
+    char *buf, size_t len);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _DIS_TABLES_H */
+
diff --git a/bsd/i386/disklabel.h b/bsd/i386/disklabel.h
index 97cab07c3..283c9174d 100644
--- a/bsd/i386/disklabel.h
+++ b/bsd/i386/disklabel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_DISKLABEL_H_
 #define _MACHINE_DISKLABEL_H_
diff --git a/bsd/i386/endian.h b/bsd/i386/endian.h
index 3e42f8a8b..9f64aa0c8 100644
--- a/bsd/i386/endian.h
+++ b/bsd/i386/endian.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -60,6 +66,7 @@
 #ifndef _I386__ENDIAN_H_
 #define	_I386__ENDIAN_H_
 
+#include <sys/cdefs.h>
 /*
  * Define _NOQUAD if the compiler does NOT support 64-bit integers.
  */
@@ -81,7 +88,7 @@
 
 #define	__DARWIN_BYTE_ORDER	__DARWIN_LITTLE_ENDIAN
 
-#if	defined(KERNEL) || !defined(_POSIX_C_SOURCE)
+#if	defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 
 #define	LITTLE_ENDIAN	__DARWIN_LITTLE_ENDIAN
 #define	BIG_ENDIAN	__DARWIN_BIG_ENDIAN
@@ -91,5 +98,5 @@
 
 #include <sys/_endian.h>
 
-#endif /* defined(KERNEL) || !defined(_POSIX_C_SOURCE) */
+#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
 #endif /* !_I386__ENDIAN_H_ */
diff --git a/bsd/i386/exec.h b/bsd/i386/exec.h
index 882e9cd79..677bd83fa 100644
--- a/bsd/i386/exec.h
+++ b/bsd/i386/exec.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
diff --git a/bsd/i386/fasttrap_isa.h b/bsd/i386/fasttrap_isa.h
new file mode 100644
index 000000000..d51c7ddfb
--- /dev/null
+++ b/bsd/i386/fasttrap_isa.h
@@ -0,0 +1,116 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_FASTTRAP_ISA_H
+#define	_FASTTRAP_ISA_H
+
+/*
+ * #pragma ident	"@(#)fasttrap_isa.h	1.6	06/09/19 SMI"
+ */
+
+#include <sys/types.h>
+#include <stdint.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#define	FASTTRAP_MAX_INSTR_SIZE		15
+
+#define	FASTTRAP_INSTR			0xcc
+
+#define	FASTTRAP_SUNWDTRACE_SIZE	64
+
+typedef	uint8_t		fasttrap_instr_t;
+
+typedef struct fasttrap_machtp {
+	uint8_t		ftmt_instr[FASTTRAP_MAX_INSTR_SIZE]; /* orig. instr. */
+	uint8_t		ftmt_size;	/* instruction size */
+#if __sol64 || defined(__APPLE__)
+	uint8_t		ftmt_ripmode;	/* %rip-relative handling mode */
+	uint8_t		ftmt_modrm;	/* saved modrm byte */
+#endif
+	uint8_t		ftmt_type;	/* emulation type */
+	uint8_t		ftmt_code;	/* branch condition */
+	uint8_t		ftmt_base;	/* branch base */
+	uint8_t		ftmt_index;	/* branch index */
+	uint8_t		ftmt_scale;	/* branch scale */
+	uint8_t		ftmt_segment;	/* segment for memory accesses */
+	user_addr_t	ftmt_dest;	/* destination of control flow */
+} fasttrap_machtp_t;
+
+#define	ftt_instr	ftt_mtp.ftmt_instr
+#if __sol64 || defined(__APPLE__)
+#define	ftt_ripmode	ftt_mtp.ftmt_ripmode
+#define	ftt_modrm	ftt_mtp.ftmt_modrm
+#endif
+#define	ftt_size	ftt_mtp.ftmt_size
+#define	ftt_type	ftt_mtp.ftmt_type
+#define	ftt_code	ftt_mtp.ftmt_code
+#define	ftt_base	ftt_mtp.ftmt_base
+#define	ftt_index	ftt_mtp.ftmt_index
+#define	ftt_scale	ftt_mtp.ftmt_scale
+#define	ftt_segment	ftt_mtp.ftmt_segment
+#define	ftt_dest	ftt_mtp.ftmt_dest
+
+#define	FASTTRAP_T_COMMON	0x00	/* common case -- no emulation */
+#define	FASTTRAP_T_JCC		0x01	/* near and far conditional jumps */
+#define	FASTTRAP_T_LOOP		0x02	/* loop instructions */
+#define	FASTTRAP_T_JCXZ		0x03	/* jump if %ecx/%rcx is zero */
+#define	FASTTRAP_T_JMP		0x04	/* relative jump */
+#define	FASTTRAP_T_CALL		0x05	/* near call (and link) */
+#define	FASTTRAP_T_RET		0x06	/* ret */
+#define	FASTTRAP_T_RET16	0x07	/* ret <imm16> */
+
+/*
+ * For performance rather than correctness.
+ */
+#define	FASTTRAP_T_PUSHL_EBP	0x10	/* pushl %ebp (for function entry) */
+#define	FASTTRAP_T_NOP		0x11	/* nop */
+
+#define	FASTTRAP_RIP_1		0x1
+#define	FASTTRAP_RIP_2		0x2
+#define	FASTTRAP_RIP_X		0x4
+
+/*
+ * Segment values.
+ */
+#define	FASTTRAP_SEG_NONE		0
+#define	FASTTRAP_SEG_CS			1
+#define	FASTTRAP_SEG_DS			2
+#define	FASTTRAP_SEG_ES			3
+#define	FASTTRAP_SEG_FS			4
+#define	FASTTRAP_SEG_GS			5
+#define	FASTTRAP_SEG_SS			6
+
+#define	FASTTRAP_RETURN_AFRAMES		6	
+#define	FASTTRAP_ENTRY_AFRAMES		5
+#define	FASTTRAP_OFFSET_AFRAMES		5
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _FASTTRAP_ISA_H */
diff --git a/bsd/i386/param.h b/bsd/i386/param.h
index dcd876782..03a38d2ce 100644
--- a/bsd/i386/param.h
+++ b/bsd/i386/param.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002, 2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1990, 1993
@@ -66,13 +72,15 @@
 #ifndef _I386_PARAM_H_
 #define _I386_PARAM_H_
 
+#include <i386/_param.h>
+
 /*
  * Round p (pointer or byte index) up to a correctly-aligned value for all
- * data types (int, long, ...).   The result is unsigned long and must be
+ * data types (int, long, ...).   The result is unsigned int and must be
  * cast to any desired pointer type.
  */
-#define	ALIGNBYTES	(sizeof(unsigned long) - 1)
-#define	ALIGN(p)	((unsigned long)((char *)(p) + ALIGNBYTES) &~ ALIGNBYTES)
+#define	ALIGNBYTES	__DARWIN_ALIGNBYTES
+#define	ALIGN(p)	__DARWIN_ALIGN(p)
 
 #define	NBPG		4096		/* bytes/page */
 #define	PGOFSET		(NBPG-1)	/* byte offset into page */
diff --git a/bsd/i386/profile.h b/bsd/i386/profile.h
index c3dd8dea3..fce3663d8 100644
--- a/bsd/i386/profile.h
+++ b/bsd/i386/profile.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997, Apple Computer, Inc. All rights reserved.
diff --git a/bsd/i386/psl.h b/bsd/i386/psl.h
index bea431e18..f11c97ef2 100644
--- a/bsd/i386/psl.h
+++ b/bsd/i386/psl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/bsd/i386/ptrace.h b/bsd/i386/ptrace.h
index 32cbfbbb4..b76395119 100644
--- a/bsd/i386/ptrace.h
+++ b/bsd/i386/ptrace.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992, 1993
diff --git a/bsd/i386/reboot.h b/bsd/i386/reboot.h
index 0724538c9..5e43cb309 100644
--- a/bsd/i386/reboot.h
+++ b/bsd/i386/reboot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #ifndef	_BSD_I386_REBOOT_H_
diff --git a/bsd/i386/reg.h b/bsd/i386/reg.h
index 7ac3d8ef9..415533afd 100644
--- a/bsd/i386/reg.h
+++ b/bsd/i386/reg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/bsd/i386/setjmp.h b/bsd/i386/setjmp.h
index a9d64e054..e28e4d679 100644
--- a/bsd/i386/setjmp.h
+++ b/bsd/i386/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved.
  *
@@ -29,8 +35,6 @@
 #define _BSD_I386_SETJMP_H
 
 #include <sys/cdefs.h>
-#include <machine/signal.h>
-
 
 #if defined(__x86_64__)
 /*
@@ -53,6 +57,10 @@ typedef int sigjmp_buf[_JBLEN + 1];
  */
 
 #if defined(KERNEL)
+
+#define __need_struct_sigcontext
+#include <i386/_structs.h>
+
 typedef struct sigcontext jmp_buf[1];
 #define _JBLEN ((sizeof(struct sigcontext)) / sizeof(int))
 typedef int sigjmp_buf[_JBLEN+1];
@@ -65,17 +73,17 @@ typedef int sigjmp_buf[_JBLEN + 1];
 #endif
 
 __BEGIN_DECLS
-extern int setjmp(jmp_buf env);
-extern void longjmp(jmp_buf env, int val);
+int	setjmp(jmp_buf);
+void	longjmp(jmp_buf, int);
 
 #ifndef _ANSI_SOURCE
-int	_setjmp(jmp_buf env);
-void	_longjmp(jmp_buf, int val);
-int sigsetjmp(sigjmp_buf env, int val);
-void siglongjmp(sigjmp_buf env, int val);
+int	_setjmp(jmp_buf);
+void	_longjmp(jmp_buf, int);
+int	sigsetjmp(sigjmp_buf, int);
+void	siglongjmp(sigjmp_buf, int);
 #endif /* _ANSI_SOURCE  */
 
-#if !defined(_ANSI_SOURCE) && !defined(_POSIX_C_SOURCE)
+#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 void	longjmperror(void);
 #endif /* neither ANSI nor POSIX */
 __END_DECLS
diff --git a/bsd/i386/signal.h b/bsd/i386/signal.h
index d1316b5df..841988884 100644
--- a/bsd/i386/signal.h
+++ b/bsd/i386/signal.h
@@ -1,36 +1,44 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
  *
  */
 
-#ifndef	_i386_SIGNAL_
-#define	_i386_SIGNAL_ 1
+#ifndef	_I386_SIGNAL_H_
+#define	_I386_SIGNAL_H_ 1
+
+#include <sys/cdefs.h>
 
 #ifndef _ANSI_SOURCE
 typedef int sig_atomic_t; 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 
 #include <sys/appleapiopts.h>
 
@@ -43,30 +51,12 @@ typedef int sig_atomic_t;
  * to the handler to allow it to properly restore state if
  * a non-standard exit is performed.
  */
-struct	sigcontext {
-    int			sc_onstack;	/* sigstack state to restore */
-    int			sc_mask;	/* signal mask to restore */
-    unsigned int	sc_eax;
-    unsigned int	sc_ebx;
-    unsigned int	sc_ecx;
-    unsigned int	sc_edx;
-    unsigned int	sc_edi;
-    unsigned int	sc_esi;
-    unsigned int	sc_ebp;
-    unsigned int	sc_esp;
-    unsigned int	sc_ss;
-    unsigned int	sc_eflags;
-    unsigned int	sc_eip;
-    unsigned int	sc_cs;
-    unsigned int	sc_ds;
-    unsigned int	sc_es;
-    unsigned int	sc_fs;
-    unsigned int	sc_gs;
-};
+#define __need_struct_sigcontext
+#include <i386/_structs.h>
 
 #endif /* __APPLE_API_OBSOLETE */
-#endif /* ! _POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #endif /* ! _ANSI_SOURCE */
 
-#endif	/* _i386_SIGNAL_ */
+#endif	/* _I386_SIGNAL_H_ */
 
diff --git a/bsd/i386/types.h b/bsd/i386/types.h
index d3841bcce..d02ab0bb2 100644
--- a/bsd/i386/types.h
+++ b/bsd/i386/types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -103,9 +109,8 @@ typedef __darwin_intptr_t	intptr_t;
 typedef unsigned long int	uintptr_t;
 #endif
 
-/* These types are used for  reserving the largest possible size. */
-// LP64todo - typedef mach_vm_address_t	user_addr_t;	/* varying length pointers from user space */ 
-// LP64todo - typedef mach_vm_size_t		user_size_t;	/* varying length values from user space (unsigned) */
+#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
+/* These types are used for reserving the largest possible size. */
 typedef u_int64_t		user_addr_t;	
 typedef u_int64_t		user_size_t;	
 typedef int64_t			user_ssize_t;
@@ -114,6 +119,7 @@ typedef u_int64_t		user_ulong_t;
 typedef int64_t			user_time_t;
 #define USER_ADDR_NULL	((user_addr_t) 0)
 #define CAST_USER_ADDR_T(a_ptr)   ((user_addr_t)((uintptr_t)(a_ptr)))
+#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /* This defines the size of syscall arguments after copying into the kernel: */
 typedef u_int64_t		syscall_arg_t;
diff --git a/bsd/i386/ucontext.h b/bsd/i386/ucontext.h
index 5f3f5cba5..acae1f055 100644
--- a/bsd/i386/ucontext.h
+++ b/bsd/i386/ucontext.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_UCONTEXT_H_
@@ -27,50 +33,31 @@
 #include <sys/appleapiopts.h>
 #include <mach/thread_status.h>
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct mcontext
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_mcontext
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 {
-#if __LP64__
-	x86_exception_state64_t	es;
-	x86_thread_state64_t 	ss;	
-	x86_float_state64_t	fs;
-#else
-	x86_exception_state32_t	es;
-	x86_thread_state32_t 	ss;	
-	x86_float_state32_t	fs;
-#endif
+	i386_exception_state_t	es;
+	i386_thread_state_t 	ss;	
+	i386_float_state_t	fs;
 };
 
-#ifndef _POSIX_C_SOURCE
-#if __LP64__
-#define I386_MCONTEXT_SIZE	(x86_THREAD_STATE64_COUNT + x86_FLOAT_STATE64_COUNT + x86_EXCEPTION_STATE64_COUNT) * sizeof(int)
-#else
-#define I386_MCONTEXT_SIZE	(x86_THREAD_STATE32_COUNT + x86_FLOAT_STATE32_COUNT + x86_EXCEPTION_STATE32_COUNT) * sizeof(int)
-#endif
-#endif /* _POSIX_C_SOURCE */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define I386_MCONTEXT_SIZE	(i386_THREAD_STATE_COUNT + i386_FLOAT_STATE_COUNT + I386_EXCEPTION_STATE_COUNT) * sizeof(int)
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifndef _MCONTEXT_T
 #define _MCONTEXT_T
-typedef __darwin_mcontext_t	mcontext_t;
+#if defined(__LP64__)
+typedef __darwin_mcontext64_t	mcontext_t;
+#else
+typedef __darwin_mcontext32_t	mcontext_t;
 #endif
-
-
-#ifdef XNU_KERNEL_PRIVATE
-struct mcontext64 {
-	x86_exception_state64_t	es;
-	x86_thread_state64_t 	ss;	
-	x86_float_state64_t	fs;
-};
-
-struct mcontext32 {
-	x86_exception_state32_t	es;
-	x86_thread_state32_t 	ss;	
-	x86_float_state32_t	fs;
-};
 #endif
 
+//#endif
+
 
 #endif /* _I386_UCONTEXT_H_ */
diff --git a/bsd/i386/vmparam.h b/bsd/i386/vmparam.h
index 4e9cc2cff..37a73b4c3 100644
--- a/bsd/i386/vmparam.h
+++ b/bsd/i386/vmparam.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_BSD_I386_VMPARAM_H_
@@ -30,6 +36,7 @@
 
 #define	USRSTACK64	VM_USRSTACK64
 
+
 /*
  * Virtual memory related constants, all in bytes
  */
diff --git a/bsd/isofs/cd9660/cd9660_bmap.c b/bsd/isofs/cd9660/cd9660_bmap.c
index 53cbb55d9..e34671f87 100644
--- a/bsd/isofs/cd9660/cd9660_bmap.c
+++ b/bsd/isofs/cd9660/cd9660_bmap.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_bmap.c,v 1.5 1994/12/13 22:33:12 mycroft Exp $	*/
 
diff --git a/bsd/isofs/cd9660/cd9660_lookup.c b/bsd/isofs/cd9660/cd9660_lookup.c
index 3498811ae..13f7ad5ed 100644
--- a/bsd/isofs/cd9660/cd9660_lookup.c
+++ b/bsd/isofs/cd9660/cd9660_lookup.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_lookup.c,v 1.13 1994/12/24 15:30:03 cgd Exp $	*/
 
@@ -146,7 +152,6 @@ cd9660_lookup(struct vnop_lookup_args *ap)
 	int flags = cnp->cn_flags;
 	int nameiop = cnp->cn_nameiop;
 	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 	size_t altlen;
 	
 	bp = NULL;
@@ -242,7 +247,7 @@ searchloop:
 		 * Get pointer to next entry.
 		 */
 		ep = (struct iso_directory_record *)
-			((char *)buf_dataptr(bp) + entryoffsetinblock);
+			((char *)0 + buf_dataptr(bp) + entryoffsetinblock);
 		
 		reclen = isonum_711(ep->length);
 		if (reclen == 0) {
@@ -297,8 +302,7 @@ searchloop:
 					if (namelen != 1
 					    || ep->name[0] != 0)
 						goto notfound;
-				} else if (imp->iso_ftype != ISO_FTYPE_JOLIET && !(res = isofncmp(name,len,
-							    ep->name,namelen))) {
+				} else if (imp->iso_ftype != ISO_FTYPE_JOLIET && !(res = isofncmp(name, len, ep->name, namelen))) {
 					if ( isoflags & directoryBit )
 						ino = isodirino(ep, imp);
 					else
@@ -338,7 +342,7 @@ foundino:
 			}
 			entryoffsetinblock = saveoffset & bmask;
 			ep = (struct iso_directory_record *)
-				((char *)buf_dataptr(bp) + entryoffsetinblock);
+				((char *)0 + buf_dataptr(bp) + entryoffsetinblock);
 			dp->i_offset = saveoffset;
 		}
 		goto found;
@@ -402,7 +406,8 @@ found:
 	 */
 	if (flags & ISDOTDOT) {
 		error = cd9660_vget_internal(vnode_mount(vdp), dp->i_ino, &tdp, NULL, NULL,
-					     dp->i_ino != ino, ep, p);
+					     dp->i_ino != ino, ep,
+					     vfs_context_proc(ctx));
 		VTOI(tdp)->i_parent = VTOI(pdp)->i_number;
 		buf_brelse(bp);
 
@@ -413,7 +418,7 @@ found:
 		*vpp = vdp;
 	} else {
 	        error = cd9660_vget_internal(vnode_mount(vdp), dp->i_ino, &tdp, vdp, cnp,
-					     dp->i_ino != ino, ep, p);
+			     dp->i_ino != ino, ep, vfs_context_proc(ctx));
 		/* save parent inode number */
 		VTOI(tdp)->i_parent = VTOI(pdp)->i_number;
 		buf_brelse(bp);
@@ -455,7 +460,7 @@ cd9660_blkatoff(vnode_t vp, off_t offset, char **res, buf_t *bpp)
 		return (error);
 	}
 	if (res)
-		*res = (char *)buf_dataptr(bp) + blkoff(imp, offset);
+		*res = (char *)0 + buf_dataptr(bp) + blkoff(imp, offset);
 	*bpp = bp;
 	
 	return (0);
diff --git a/bsd/isofs/cd9660/cd9660_mount.h b/bsd/isofs/cd9660/cd9660_mount.h
index 94318f4a0..90ef397f4 100644
--- a/bsd/isofs/cd9660/cd9660_mount.h
+++ b/bsd/isofs/cd9660/cd9660_mount.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995
@@ -90,7 +96,6 @@ struct iso_args {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with iso_args
  */
-/* LP64todo - should this move? */
 
 struct user_iso_args {
 	int	flags;			/* mounting flags, see below */
diff --git a/bsd/isofs/cd9660/cd9660_node.c b/bsd/isofs/cd9660/cd9660_node.c
index 4418c9147..0b9ddcff8 100644
--- a/bsd/isofs/cd9660/cd9660_node.c
+++ b/bsd/isofs/cd9660/cd9660_node.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_node.c,v 1.13 1994/12/24 15:30:07 cgd Exp $	*/
 
@@ -109,10 +115,17 @@ extern u_char isonullname[];
 int
 cd9660_init(__unused struct vfsconf *cp)
 {
+    return 0;
+}
 
-	isohashtbl = hashinit(desiredvnodes, M_ISOFSMNT, &isohash);
+int
+cd9660_hashinit(void)
+{
+	if (!isohashtbl)
+		isohashtbl = hashinit(desiredvnodes, M_ISOFSMNT, &isohash);
 #ifdef ISODEVMAP
-	idvhashtbl = hashinit(desiredvnodes / 8, M_ISOFSMNT, &idvhash);
+	if (!idvhashtbl)
+		idvhashtbl = hashinit(desiredvnodes / 8, M_ISOFSMNT, &idvhash);
 #endif
     return 0;
 }
@@ -174,7 +187,7 @@ iso_dunmap(dev_t device)
  * to it. If it is in core, but locked, wait for it.
  */
 struct vnode *
-cd9660_ihashget(dev_t device, ino_t inum, struct proc *p)
+cd9660_ihashget(dev_t device, ino_t inum, __unused struct proc *p)
 {
 	register struct iso_node *ip;
 	struct vnode *vp;
@@ -335,7 +348,7 @@ cd9660_defattr(struct iso_directory_record *isodir, struct iso_node *inop,
 		bp = bp2;
 	}
 	if (bp) {
-		ap = (struct iso_extended_attributes *)buf_dataptr(bp);
+		ap = (struct iso_extended_attributes *)((char *)0 + buf_dataptr(bp));
 		
 		if (isonum_711(ap->version) == 1) {
 			if (!(ap->perm[0]&0x40))
@@ -384,7 +397,7 @@ cd9660_deftstamp(struct iso_directory_record *isodir, struct iso_node *inop,
 		bp = bp2;
 	}
 	if (bp) {
-		ap = (struct iso_extended_attributes *)buf_dataptr(bp);
+		ap = (struct iso_extended_attributes *)((char *)0 + buf_dataptr(bp));
 		
 		if (isonum_711(ap->version) == 1) {
 			if (!cd9660_tstamp_conv17(ap->ftime,&inop->inode.iso_atime))
diff --git a/bsd/isofs/cd9660/cd9660_node.h b/bsd/isofs/cd9660/cd9660_node.h
index faa7450dd..80a78c2a1 100644
--- a/bsd/isofs/cd9660/cd9660_node.h
+++ b/bsd/isofs/cd9660/cd9660_node.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_node.h,v 1.10 1994/12/24 15:30:09 cgd Exp $	*/
 
@@ -168,6 +174,9 @@ int cd9660_access_internal(vnode_t, mode_t, kauth_cred_t);
 /*
  * Prototypes for ISOFS vnode operations
  */
+int cd9660_blktooff(struct vnop_blktooff_args *);
+int cd9660_offtoblk(struct vnop_offtoblk_args *);
+int cd9660_blockmap(struct vnop_blockmap_args *);
 int cd9660_lookup (struct vnop_lookup_args *);
 int cd9660_open (struct vnop_open_args *);
 int cd9660_close (struct vnop_close_args *);
@@ -188,7 +197,6 @@ int cd9660_enotsupp(void);
 int cd9660_pagein(struct vnop_pagein_args *ap);
 int cd9660_remove(struct vnop_remove_args *ap);
 int cd9660_rmdir(struct vnop_rmdir_args *ap);
-int cd9660_getattrlist(struct vnop_getattrlist_args *ap);
 
 __private_extern__ void cd9660_xa_init(struct iso_node *ip,
 				       struct iso_directory_record *isodir);
diff --git a/bsd/isofs/cd9660/cd9660_rrip.c b/bsd/isofs/cd9660/cd9660_rrip.c
index 66d8e231f..643c7ad54 100644
--- a/bsd/isofs/cd9660/cd9660_rrip.c
+++ b/bsd/isofs/cd9660/cd9660_rrip.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_rrip.c,v 1.11 1994/12/24 15:30:10 cgd Exp $	*/
 
@@ -116,7 +122,8 @@ cd9660_rrip_slink(ISO_RRIP_SLINK *p, ISO_RRIP_ANALYZE *ana)
 	register ISO_RRIP_SLINK_COMPONENT *pcomp;
 	register ISO_RRIP_SLINK_COMPONENT *pcompe;
 	int len, wlen, cont;
-	char *outbuf, *inbuf;
+	char *outbuf;
+	const char *inbuf;
 	
 	pcomp = (ISO_RRIP_SLINK_COMPONENT *)p->component;
 	pcompe = (ISO_RRIP_SLINK_COMPONENT *)((char *)p + isonum_711(p->h.length));
@@ -167,7 +174,7 @@ cd9660_rrip_slink(ISO_RRIP_SLINK *p, ISO_RRIP_ANALYZE *ana)
 			/* same as above */
 			outbuf -= len;
 			len = 0;
-			inbuf = &(vfs_statfs(ana->imp->im_mountp)->f_mntonname);
+			inbuf = vfs_statfs(ana->imp->im_mountp)->f_mntonname;
 			wlen = strlen(inbuf);
 			break;
 			
@@ -283,7 +290,7 @@ cd9660_rrip_altname(ISO_RRIP_ALTNAME *p, ISO_RRIP_ANALYZE *ana)
 static void
 cd9660_rrip_defname(struct iso_directory_record *isodir, ISO_RRIP_ANALYZE *ana)
 {
-	strcpy(ana->outbuf,"..");
+	strlcpy(ana->outbuf, "..", ana->outlen);
 	switch (*isodir->name) {
 	default:
 		isofntrans(isodir->name, isonum_711(isodir->name_len),
@@ -339,7 +346,7 @@ cd9660_rrip_tstamp(ISO_RRIP_TSTAMP *p, ISO_RRIP_ANALYZE *ana)
 			cd9660_tstamp_conv7(ptime,&ana->inop->inode.iso_mtime);
 			ptime += 7;
 		} else
-			bzero(&ana->inop->inode.iso_mtime,sizeof(struct timespec));
+			bzero(&ana->inop->inode.iso_mtime, sizeof(struct timespec));
 		
 		if (*p->flags&ISO_SUSP_TSTAMP_ACCESS) {
 			cd9660_tstamp_conv7(ptime,&ana->inop->inode.iso_atime);
@@ -360,7 +367,7 @@ cd9660_rrip_tstamp(ISO_RRIP_TSTAMP *p, ISO_RRIP_ANALYZE *ana)
 			cd9660_tstamp_conv17(ptime,&ana->inop->inode.iso_mtime);
 			ptime += 17;
 		} else
-			bzero(&ana->inop->inode.iso_mtime,sizeof(struct timespec));
+			bzero(&ana->inop->inode.iso_mtime, sizeof(struct timespec));
 		
 		if (*p->flags&ISO_SUSP_TSTAMP_ACCESS) {
 			cd9660_tstamp_conv17(ptime,&ana->inop->inode.iso_atime);
diff --git a/bsd/isofs/cd9660/cd9660_rrip.h b/bsd/isofs/cd9660/cd9660_rrip.h
index 4a3c17192..a583811bb 100644
--- a/bsd/isofs/cd9660/cd9660_rrip.h
+++ b/bsd/isofs/cd9660/cd9660_rrip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_rrip.h,v 1.6 1994/12/13 22:33:24 mycroft Exp $	*/
 
diff --git a/bsd/isofs/cd9660/cd9660_util.c b/bsd/isofs/cd9660/cd9660_util.c
index 89e0cd57a..b3ae1af55 100644
--- a/bsd/isofs/cd9660/cd9660_util.c
+++ b/bsd/isofs/cd9660/cd9660_util.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_util.c,v 1.8 1994/12/13 22:33:25 mycroft Exp $	*/
 
@@ -90,6 +96,8 @@
 #include <isofs/cd9660/cd9660_node.h>
 #include <isofs/cd9660/iso_rrip.h>
 
+#include <libkern/OSByteOrder.h>
+
 /*
  * translate and compare a filename
  * Note: Version number plus ';' may be omitted.
@@ -300,9 +308,7 @@ ucsfntrans(u_int16_t *infn, int infnlen, u_char *outfn, u_short *outfnlen,
 				fnidx = infnlen/2;
 		}
 
-		flags = UTF_NO_NULL_TERM | UTF_DECOMPOSED;
-		if (BYTE_ORDER != BIG_ENDIAN)
-			flags |= UTF_REVERSE_ENDIAN;
+		flags = UTF_NO_NULL_TERM | UTF_DECOMPOSED | UTF_BIG_ENDIAN;
 
 		(void) utf8_encodestr(infn, fnidx * 2, outfn, &outbytes, ISO_JOLIET_NAMEMAX, 0, flags);
 		*outfnlen = assoc ? outbytes + 2 : outbytes;
@@ -425,640 +431,3 @@ DerivePermissionSummary(uid_t owner, gid_t group, mode_t obj_mode, __unused stru
 Exit:
 	return permissions & ~W_OK;    	/* Write access is always impossible */
 }
-
-
-int
-attrcalcsize(struct attrlist *attrlist)
-{
-	int size;
-	attrgroup_t a;
-	boolean_t is_64_bit = proc_is64bit(current_proc());
-	
-#if ((ATTR_CMN_NAME			| ATTR_CMN_DEVID			| ATTR_CMN_FSID 			| ATTR_CMN_OBJTYPE 		| \
-      ATTR_CMN_OBJTAG		| ATTR_CMN_OBJID			| ATTR_CMN_OBJPERMANENTID	| ATTR_CMN_PAROBJID		| \
-      ATTR_CMN_SCRIPT		| ATTR_CMN_CRTIME			| ATTR_CMN_MODTIME			| ATTR_CMN_CHGTIME		| \
-      ATTR_CMN_ACCTIME		| ATTR_CMN_BKUPTIME			| ATTR_CMN_FNDRINFO			| ATTR_CMN_OWNERID		| \
-      ATTR_CMN_GRPID		| ATTR_CMN_ACCESSMASK		| ATTR_CMN_NAMEDATTRCOUNT	| ATTR_CMN_NAMEDATTRLIST| \
-      ATTR_CMN_FLAGS		| ATTR_CMN_USERACCESS) != ATTR_CMN_VALIDMASK)
-#error AttributeBlockSize: Missing bits in common mask computation!
-#endif
-	assert((attrlist->commonattr & ~ATTR_CMN_VALIDMASK) == 0);
-
-#if ((ATTR_VOL_FSTYPE		| ATTR_VOL_SIGNATURE		| ATTR_VOL_SIZE				| ATTR_VOL_SPACEFREE 	| \
-      ATTR_VOL_SPACEAVAIL	| ATTR_VOL_MINALLOCATION	| ATTR_VOL_ALLOCATIONCLUMP	| ATTR_VOL_IOBLOCKSIZE	| \
-      ATTR_VOL_OBJCOUNT		| ATTR_VOL_FILECOUNT		| ATTR_VOL_DIRCOUNT			| ATTR_VOL_MAXOBJCOUNT	| \
-      ATTR_VOL_MOUNTPOINT	| ATTR_VOL_NAME				| ATTR_VOL_MOUNTFLAGS		| ATTR_VOL_INFO 		| \
-      ATTR_VOL_MOUNTEDDEVICE| ATTR_VOL_ENCODINGSUSED	| ATTR_VOL_CAPABILITIES		| ATTR_VOL_ATTRIBUTES) != ATTR_VOL_VALIDMASK)
-#error AttributeBlockSize: Missing bits in volume mask computation!
-#endif
-	assert((attrlist->volattr & ~ATTR_VOL_VALIDMASK) == 0);
-
-#if ((ATTR_DIR_LINKCOUNT | ATTR_DIR_ENTRYCOUNT | ATTR_DIR_MOUNTSTATUS) != ATTR_DIR_VALIDMASK)
-#error AttributeBlockSize: Missing bits in directory mask computation!
-#endif
-	assert((attrlist->dirattr & ~ATTR_DIR_VALIDMASK) == 0);
-#if ((ATTR_FILE_LINKCOUNT	| ATTR_FILE_TOTALSIZE		| ATTR_FILE_ALLOCSIZE 		| ATTR_FILE_IOBLOCKSIZE 	| \
-      ATTR_FILE_CLUMPSIZE	| ATTR_FILE_DEVTYPE			| ATTR_FILE_FILETYPE		| ATTR_FILE_FORKCOUNT		| \
-      ATTR_FILE_FORKLIST	| ATTR_FILE_DATALENGTH		| ATTR_FILE_DATAALLOCSIZE	| ATTR_FILE_DATAEXTENTS		| \
-      ATTR_FILE_RSRCLENGTH	| ATTR_FILE_RSRCALLOCSIZE	| ATTR_FILE_RSRCEXTENTS) != ATTR_FILE_VALIDMASK)
-#error AttributeBlockSize: Missing bits in file mask computation!
-#endif
-	assert((attrlist->fileattr & ~ATTR_FILE_VALIDMASK) == 0);
-
-#if ((ATTR_FORK_TOTALSIZE | ATTR_FORK_ALLOCSIZE) != ATTR_FORK_VALIDMASK)
-#error AttributeBlockSize: Missing bits in fork mask computation!
-#endif
-	assert((attrlist->forkattr & ~ATTR_FORK_VALIDMASK) == 0);
-
-	size = 0;
-	
-	if ((a = attrlist->commonattr) != 0) {
-        if (a & ATTR_CMN_NAME) size += sizeof(struct attrreference);
-		if (a & ATTR_CMN_DEVID) size += sizeof(dev_t);
-		if (a & ATTR_CMN_FSID) size += sizeof(fsid_t);
-		if (a & ATTR_CMN_OBJTYPE) size += sizeof(fsobj_type_t);
-		if (a & ATTR_CMN_OBJTAG) size += sizeof(fsobj_tag_t);
-		if (a & ATTR_CMN_OBJID) size += sizeof(fsobj_id_t);
-        if (a & ATTR_CMN_OBJPERMANENTID) size += sizeof(fsobj_id_t);
-		if (a & ATTR_CMN_PAROBJID) size += sizeof(fsobj_id_t);
-		if (a & ATTR_CMN_SCRIPT) size += sizeof(text_encoding_t);
-		if (a & ATTR_CMN_CRTIME) {
-            if (is_64_bit) 
-                size += sizeof(struct user_timespec);
-            else 
-                size += sizeof(struct timespec);
-		}
-		if (a & ATTR_CMN_MODTIME) {
-            if (is_64_bit) 
-                size += sizeof(struct user_timespec);
-            else 
-                size += sizeof(struct timespec);
-		}
-		if (a & ATTR_CMN_CHGTIME) {
-            if (is_64_bit) 
-                size += sizeof(struct user_timespec);
-            else 
-                size += sizeof(struct timespec);
-		}
-		if (a & ATTR_CMN_ACCTIME) {
-            if (is_64_bit) 
-                size += sizeof(struct user_timespec);
-            else 
-                size += sizeof(struct timespec);
-		}
-		if (a & ATTR_CMN_BKUPTIME) {
-            if (is_64_bit) 
-                size += sizeof(struct user_timespec);
-            else 
-                size += sizeof(struct timespec);
-		}
-		if (a & ATTR_CMN_FNDRINFO) size += 32 * sizeof(u_int8_t);
-		if (a & ATTR_CMN_OWNERID) size += sizeof(uid_t);
-		if (a & ATTR_CMN_GRPID) size += sizeof(gid_t);
-		if (a & ATTR_CMN_ACCESSMASK) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_NAMEDATTRCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_NAMEDATTRLIST) size += sizeof(struct attrreference);
-		if (a & ATTR_CMN_FLAGS) size += sizeof(uint32_t);
-		if (a & ATTR_CMN_USERACCESS) size += sizeof(uint32_t);
-	};
-	if ((a = attrlist->volattr) != 0) {
-		if (a & ATTR_VOL_FSTYPE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_SIGNATURE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_SIZE) size += sizeof(off_t);
-		if (a & ATTR_VOL_SPACEFREE) size += sizeof(off_t);
-		if (a & ATTR_VOL_SPACEAVAIL) size += sizeof(off_t);
-		if (a & ATTR_VOL_MINALLOCATION) size += sizeof(off_t);
-		if (a & ATTR_VOL_ALLOCATIONCLUMP) size += sizeof(off_t);
-		if (a & ATTR_VOL_IOBLOCKSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_OBJCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_FILECOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_DIRCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_MAXOBJCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_VOL_MOUNTPOINT) size += sizeof(struct attrreference);
-        if (a & ATTR_VOL_NAME) size += sizeof(struct attrreference);
-		if (a & ATTR_VOL_MOUNTFLAGS) size += sizeof(uint32_t);
-        if (a & ATTR_VOL_MOUNTEDDEVICE) size += sizeof(struct attrreference);
-        if (a & ATTR_VOL_ENCODINGSUSED) size += sizeof(unsigned long long);
-        if (a & ATTR_VOL_CAPABILITIES) size += sizeof(vol_capabilities_attr_t);
-        if (a & ATTR_VOL_ATTRIBUTES) size += sizeof(vol_attributes_attr_t);
-	};
-	if ((a = attrlist->dirattr) != 0) {
-		if (a & ATTR_DIR_LINKCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_DIR_ENTRYCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_DIR_MOUNTSTATUS) size += sizeof(uint32_t);
-	};
-	if ((a = attrlist->fileattr) != 0) {
-		if (a & ATTR_FILE_LINKCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_TOTALSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_ALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_IOBLOCKSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_CLUMPSIZE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_DEVTYPE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FILETYPE) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FORKCOUNT) size += sizeof(uint32_t);
-		if (a & ATTR_FILE_FORKLIST) size += sizeof(struct attrreference);
-		if (a & ATTR_FILE_DATALENGTH) size += sizeof(off_t);
-		if (a & ATTR_FILE_DATAALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_DATAEXTENTS) size += sizeof(extentrecord);
-		if (a & ATTR_FILE_RSRCLENGTH) size += sizeof(off_t);
-		if (a & ATTR_FILE_RSRCALLOCSIZE) size += sizeof(off_t);
-		if (a & ATTR_FILE_RSRCEXTENTS) size += sizeof(extentrecord);
-	};
-	if ((a = attrlist->forkattr) != 0) {
-		if (a & ATTR_FORK_TOTALSIZE) size += sizeof(off_t);
-		if (a & ATTR_FORK_ALLOCSIZE) size += sizeof(off_t);
-	};
-
-    return size;
-}
-
-
-
-static void
-packvolattr (struct attrlist *alist,
-			 struct iso_node *ip,	/* ip for root directory */
-			 void **attrbufptrptr,
-			 void **varbufptrptr)
-{
-    void *attrbufptr;
-    void *varbufptr;
-	struct iso_mnt *imp;
-	struct mount *mp;
-	attrgroup_t a;
-	uint32_t attrlength;
-	boolean_t is_64_bit = proc_is64bit(current_proc());
-	
-	attrbufptr = *attrbufptrptr;
-	varbufptr = *varbufptrptr;
-	imp = ip->i_mnt;
-	mp = imp->im_mountp;
-
-    if ((a = alist->commonattr) != 0) {
-        if (a & ATTR_CMN_NAME) {
-            attrlength = strlen( imp->volume_id ) + 1;
-            ((struct attrreference *)attrbufptr)->attr_dataoffset = (u_int8_t *)varbufptr - (u_int8_t *)attrbufptr;
-            ((struct attrreference *)attrbufptr)->attr_length = attrlength;
-            (void) strncpy((unsigned char *)varbufptr, imp->volume_id, attrlength);
-
-            /* Advance beyond the space just allocated and round up to the next 4-byte boundary: */
-            (u_int8_t *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-            ++((struct attrreference *)attrbufptr);
-        };
-		if (a & ATTR_CMN_DEVID) *((dev_t *)attrbufptr)++ = vnode_specrdev(imp->im_devvp);
-		if (a & ATTR_CMN_FSID) *((fsid_t *)attrbufptr)++ = vfs_statfs(vnode_mount(ITOV(ip)))->f_fsid;
-		if (a & ATTR_CMN_OBJTYPE) *((fsobj_type_t *)attrbufptr)++ = 0;
-		if (a & ATTR_CMN_OBJTAG) *((fsobj_tag_t *)attrbufptr)++ = VT_ISOFS;
-		if (a & ATTR_CMN_OBJID)	{
-			((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-			((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-			++((fsobj_id_t *)attrbufptr);
-		};
-        if (a & ATTR_CMN_OBJPERMANENTID) {
-            ((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-            ((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-            ++((fsobj_id_t *)attrbufptr);
-        };
-		if (a & ATTR_CMN_PAROBJID) {
-            ((fsobj_id_t *)attrbufptr)->fid_objno = 0;
-			((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-			++((fsobj_id_t *)attrbufptr);
-		};
-        if (a & ATTR_CMN_SCRIPT) *((text_encoding_t *)attrbufptr)++ = 0;
-		if (a & ATTR_CMN_CRTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) imp->creation_date.tv_sec;
-		        tmpp->tv_nsec = imp->creation_date.tv_nsec;
-		    }
-		    else {
-		        *((struct timespec *)attrbufptr)++ = imp->creation_date;
-		    }
-		}
-		if (a & ATTR_CMN_MODTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) imp->modification_date.tv_sec;
-		        tmpp->tv_nsec = imp->modification_date.tv_nsec;
-		    }
-		    else {
-		        *((struct timespec *)attrbufptr)++ = imp->modification_date;
-		    }
-		}
-		if (a & ATTR_CMN_CHGTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) imp->modification_date.tv_sec;
-		        tmpp->tv_nsec = imp->modification_date.tv_nsec;
-		    }
-		    else {
-		        *((struct timespec *)attrbufptr)++ = imp->modification_date;
-		    }
-		}
-		if (a & ATTR_CMN_ACCTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) imp->modification_date.tv_sec;
-		        tmpp->tv_nsec = imp->modification_date.tv_nsec;
-		    }
-		    else {
-		        *((struct timespec *)attrbufptr)++ = imp->modification_date;
-		    }
-		}
-		if (a & ATTR_CMN_BKUPTIME) {
-			((struct timespec *)attrbufptr)->tv_sec = 0;
-			((struct timespec *)attrbufptr)->tv_nsec = 0;
-			++((struct timespec *)attrbufptr);
-		};
-		if (a & ATTR_CMN_FNDRINFO) {
-            bzero (attrbufptr, 32 * sizeof(u_int8_t));
-            (u_int8_t *)attrbufptr += 32 * sizeof(u_int8_t);
-		};
-		if (a & ATTR_CMN_OWNERID) *((uid_t *)attrbufptr)++ = ip->inode.iso_uid;
-		if (a & ATTR_CMN_GRPID) *((gid_t *)attrbufptr)++ = ip->inode.iso_gid;
-		if (a & ATTR_CMN_ACCESSMASK) *((uint32_t *)attrbufptr)++ = (uint32_t)ip->inode.iso_mode;
-		if (a & ATTR_CMN_FLAGS) *((uint32_t *)attrbufptr)++ = 0;
-		if (a & ATTR_CMN_USERACCESS) {
-			*((uint32_t *)attrbufptr)++ =
-				DerivePermissionSummary(ip->inode.iso_uid,
-										ip->inode.iso_gid,
-										ip->inode.iso_mode,
-										imp);
-		};
-	};
-	
-	if ((a = alist->volattr) != 0) {
-		off_t blocksize = (off_t)imp->logical_block_size;
-
-		if (a & ATTR_VOL_FSTYPE) *((uint32_t *)attrbufptr)++ = (uint32_t)vfs_typenum(mp);
-		if (a & ATTR_VOL_SIGNATURE) *((uint32_t *)attrbufptr)++ = (uint32_t)ISO9660SIGNATURE;
-        if (a & ATTR_VOL_SIZE) *((off_t *)attrbufptr)++ = (off_t)imp->volume_space_size * blocksize;
-        if (a & ATTR_VOL_SPACEFREE) *((off_t *)attrbufptr)++ = 0;
-        if (a & ATTR_VOL_SPACEAVAIL) *((off_t *)attrbufptr)++ = 0;
-        if (a & ATTR_VOL_MINALLOCATION) *((off_t *)attrbufptr)++ = blocksize;
-		if (a & ATTR_VOL_ALLOCATIONCLUMP) *((off_t *)attrbufptr)++ = blocksize;
-        if (a & ATTR_VOL_IOBLOCKSIZE) *((uint32_t *)attrbufptr)++ = (uint32_t)blocksize;
-		if (a & ATTR_VOL_OBJCOUNT) *((uint32_t *)attrbufptr)++ = 0;
-		if (a & ATTR_VOL_FILECOUNT) *((uint32_t *)attrbufptr)++ = 0;
-		if (a & ATTR_VOL_DIRCOUNT) *((uint32_t *)attrbufptr)++ = 0;
-		if (a & ATTR_VOL_MAXOBJCOUNT) *((uint32_t *)attrbufptr)++ = 0xFFFFFFFF;
-        if (a & ATTR_VOL_NAME) {
-            attrlength = strlen( imp->volume_id ) + 1;
-            ((struct attrreference *)attrbufptr)->attr_dataoffset = (u_int8_t *)varbufptr - (u_int8_t *)attrbufptr;
-            ((struct attrreference *)attrbufptr)->attr_length = attrlength;
-            (void) strncpy((unsigned char *)varbufptr, imp->volume_id, attrlength);
-
-            /* Advance beyond the space just allocated and round up to the next 4-byte boundary: */
-            (u_int8_t *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-            ++((struct attrreference *)attrbufptr);
-        };
-		if (a & ATTR_VOL_MOUNTFLAGS) {
-		    *((uint32_t *)attrbufptr)++ = (uint32_t)vfs_flags(mp);
-		}
-        if (a & ATTR_VOL_MOUNTEDDEVICE) {
-            ((struct attrreference *)attrbufptr)->attr_dataoffset = (u_int8_t *)varbufptr - (u_int8_t *)attrbufptr;
-            ((struct attrreference *)attrbufptr)->attr_length = strlen(vfs_statfs(mp)->f_mntfromname) + 1;
-			attrlength = ((struct attrreference *)attrbufptr)->attr_length;
-			attrlength = attrlength + ((4 - (attrlength & 3)) & 3);		/* round up to the next 4-byte boundary: */
-			(void) bcopy(vfs_statfs(mp)->f_mntfromname, varbufptr, attrlength);
-			
-			/* Advance beyond the space just allocated: */
-            (u_int8_t *)varbufptr += attrlength;
-            ++((struct attrreference *)attrbufptr);
-        };
-        if (a & ATTR_VOL_ENCODINGSUSED) *((unsigned long long *)attrbufptr)++ = (unsigned long long)0;
-        if (a & ATTR_VOL_CAPABILITIES) {
-        	((vol_capabilities_attr_t *)attrbufptr)->capabilities[VOL_CAPABILITIES_FORMAT] =
-        			(imp->iso_ftype == ISO_FTYPE_RRIP ? VOL_CAP_FMT_SYMBOLICLINKS : 0) |
-        			(imp->iso_ftype == ISO_FTYPE_RRIP ? VOL_CAP_FMT_HARDLINKS : 0) |
-        			(imp->iso_ftype == ISO_FTYPE_RRIP || imp->iso_ftype == ISO_FTYPE_JOLIET
-        				? VOL_CAP_FMT_CASE_SENSITIVE : 0) |
-					VOL_CAP_FMT_CASE_PRESERVING |
-        			VOL_CAP_FMT_FAST_STATFS;
-        	((vol_capabilities_attr_t *)attrbufptr)->capabilities[VOL_CAPABILITIES_INTERFACES] =
-        			VOL_CAP_INT_ATTRLIST |
-        			VOL_CAP_INT_NFSEXPORT;
-        	((vol_capabilities_attr_t *)attrbufptr)->capabilities[VOL_CAPABILITIES_RESERVED1] = 0;
-        	((vol_capabilities_attr_t *)attrbufptr)->capabilities[VOL_CAPABILITIES_RESERVED2] = 0;
-
-        	((vol_capabilities_attr_t *)attrbufptr)->valid[VOL_CAPABILITIES_FORMAT] =
-        			VOL_CAP_FMT_PERSISTENTOBJECTIDS |
-        			VOL_CAP_FMT_SYMBOLICLINKS |
-        			VOL_CAP_FMT_HARDLINKS |
-					VOL_CAP_FMT_JOURNAL |
-					VOL_CAP_FMT_JOURNAL_ACTIVE |
-					VOL_CAP_FMT_NO_ROOT_TIMES |
-					VOL_CAP_FMT_SPARSE_FILES |
-					VOL_CAP_FMT_ZERO_RUNS |
-					VOL_CAP_FMT_CASE_SENSITIVE |
-					VOL_CAP_FMT_CASE_PRESERVING |
-					VOL_CAP_FMT_FAST_STATFS | 
-					VOL_CAP_FMT_2TB_FILESIZE;
-        	((vol_capabilities_attr_t *)attrbufptr)->valid[VOL_CAPABILITIES_INTERFACES] =
-        			VOL_CAP_INT_SEARCHFS |
-        			VOL_CAP_INT_ATTRLIST |
-        			VOL_CAP_INT_NFSEXPORT |
-					VOL_CAP_INT_READDIRATTR |
-					VOL_CAP_INT_EXCHANGEDATA |
-					VOL_CAP_INT_COPYFILE |
-					VOL_CAP_INT_ALLOCATE |
-					VOL_CAP_INT_VOL_RENAME |
-					VOL_CAP_INT_ADVLOCK |
-					VOL_CAP_INT_FLOCK;
-        	((vol_capabilities_attr_t *)attrbufptr)->valid[VOL_CAPABILITIES_RESERVED1] = 0;
-        	((vol_capabilities_attr_t *)attrbufptr)->valid[VOL_CAPABILITIES_RESERVED2] = 0;
-
-            ++((vol_capabilities_attr_t *)attrbufptr);
-        };
-        if (a & ATTR_VOL_ATTRIBUTES) {
-        	((vol_attributes_attr_t *)attrbufptr)->validattr.commonattr = ATTR_CMN_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->validattr.volattr = ATTR_VOL_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->validattr.dirattr = ATTR_DIR_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->validattr.fileattr = ATTR_FILE_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->validattr.forkattr = ATTR_FORK_VALIDMASK;
-
-        	((vol_attributes_attr_t *)attrbufptr)->nativeattr.commonattr = ATTR_CMN_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->nativeattr.volattr = ATTR_VOL_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->nativeattr.dirattr = ATTR_DIR_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->nativeattr.fileattr = ATTR_FILE_VALIDMASK;
-        	((vol_attributes_attr_t *)attrbufptr)->nativeattr.forkattr = ATTR_FORK_VALIDMASK;
-
-            ++((vol_attributes_attr_t *)attrbufptr);
-        };
-	};
-
-	*attrbufptrptr = attrbufptr;
-	*varbufptrptr = varbufptr;
-}
-
-
-void
-packcommonattr (struct attrlist *alist,
-				struct iso_node *ip,
-				void **attrbufptrptr,
-				void **varbufptrptr)
-{
-	void *attrbufptr;
-	void *varbufptr;
-	attrgroup_t a;
-	uint32_t attrlength;
-	boolean_t is_64_bit = proc_is64bit(current_proc());
-	
-	attrbufptr = *attrbufptrptr;
-	varbufptr = *varbufptrptr;
-	
-    if ((a = alist->commonattr) != 0) {
-		struct iso_mnt *imp = ip->i_mnt;
-
-        if (a & ATTR_CMN_NAME) {
-			/* special case root since we know how to get it's name */
-			if (vnode_isvroot(ITOV(ip))) {
-				attrlength = strlen( imp->volume_id ) + 1;
-				(void) strncpy((unsigned char *)varbufptr, imp->volume_id, attrlength);
-        	} else {
-            	attrlength = strlen(ip->i_namep) + 1;
-				(void) strncpy((unsigned char *)varbufptr, ip->i_namep, attrlength);
-            }
-
-			((struct attrreference *)attrbufptr)->attr_dataoffset = (u_int8_t *)varbufptr - (u_int8_t *)attrbufptr;
-			((struct attrreference *)attrbufptr)->attr_length = attrlength;
-            /* Advance beyond the space just allocated and round up to the next 4-byte boundary: */
-            (u_int8_t *)varbufptr += attrlength + ((4 - (attrlength & 3)) & 3);
-            ++((struct attrreference *)attrbufptr);
-        };
-		if (a & ATTR_CMN_DEVID) *((dev_t *)attrbufptr)++ = ip->i_dev;
-		if (a & ATTR_CMN_FSID) *((fsid_t *)attrbufptr)++ = vfs_statfs(vnode_mount(ITOV(ip)))->f_fsid;
-		if (a & ATTR_CMN_OBJTYPE) *((fsobj_type_t *)attrbufptr)++ = vnode_vtype(ITOV(ip));
-		if (a & ATTR_CMN_OBJTAG) *((fsobj_tag_t *)attrbufptr)++ = vnode_tag(ITOV(ip));
-        if (a & ATTR_CMN_OBJID)	{
-			if (vnode_isvroot(ITOV(ip)))
-				((fsobj_id_t *)attrbufptr)->fid_objno = 2;	/* force root to be 2 */
-			else
-            	((fsobj_id_t *)attrbufptr)->fid_objno = ip->i_number;
-			((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-			++((fsobj_id_t *)attrbufptr);
-		};
-        if (a & ATTR_CMN_OBJPERMANENTID)	{
-			if (vnode_isvroot(ITOV(ip)))
-				((fsobj_id_t *)attrbufptr)->fid_objno = 2;	/* force root to be 2 */
-			else
-            	((fsobj_id_t *)attrbufptr)->fid_objno = ip->i_number;
-            ((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-            ++((fsobj_id_t *)attrbufptr);
-        };
-		if (a & ATTR_CMN_PAROBJID) {
-			struct iso_directory_record *dp = (struct iso_directory_record *)imp->root;
-			ino_t rootino = isodirino(dp, imp);
-
-			if (ip->i_number == rootino)
-				((fsobj_id_t *)attrbufptr)->fid_objno = 1;	/* force root parent to be 1 */
-			else if (ip->i_parent == rootino)
-				((fsobj_id_t *)attrbufptr)->fid_objno = 2;	/* force root to be 2 */
-			else
-            	((fsobj_id_t *)attrbufptr)->fid_objno = ip->i_parent;
-			((fsobj_id_t *)attrbufptr)->fid_generation = 0;
-			++((fsobj_id_t *)attrbufptr);
-		};
-        if (a & ATTR_CMN_SCRIPT) *((text_encoding_t *)attrbufptr)++ = 0;
-		if (a & ATTR_CMN_CRTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) ip->inode.iso_mtime.tv_sec;
-		        tmpp->tv_nsec = ip->inode.iso_mtime.tv_nsec;
-		    }
-		    else {
-                *((struct timespec *)attrbufptr)++ = ip->inode.iso_mtime;
-		    }
-		}
-		if (a & ATTR_CMN_MODTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) ip->inode.iso_mtime.tv_sec;
-		        tmpp->tv_nsec = ip->inode.iso_mtime.tv_nsec;
-		    }
-		    else {
-                *((struct timespec *)attrbufptr)++ = ip->inode.iso_mtime;
-		    }
-		}
-		if (a & ATTR_CMN_CHGTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) ip->inode.iso_ctime.tv_sec;
-		        tmpp->tv_nsec = ip->inode.iso_ctime.tv_nsec;
-		    }
-		    else {
-                *((struct timespec *)attrbufptr)++ = ip->inode.iso_ctime;
-		    }
-		}
-		if (a & ATTR_CMN_ACCTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) ip->inode.iso_atime.tv_sec;
-		        tmpp->tv_nsec = ip->inode.iso_atime.tv_nsec;
-		    }
-		    else {
-                *((struct timespec *)attrbufptr)++ = ip->inode.iso_atime;
-		    }
-		}
-		if (a & ATTR_CMN_BKUPTIME) {
-		    if (is_64_bit) {
-		        struct user_timespec *tmpp = ((struct user_timespec *)attrbufptr)++;
-		        tmpp->tv_sec = (user_time_t) 0;
-		        tmpp->tv_nsec = 0;
-		    }
-		    else {
-                ((struct timespec *)attrbufptr)->tv_sec = 0;
-                ((struct timespec *)attrbufptr)->tv_nsec = 0;
-                ++((struct timespec *)attrbufptr);
-                    *((struct timespec *)attrbufptr)++ = ip->inode.iso_atime;
-		    }
-		}
-		if (a & ATTR_CMN_FNDRINFO) {
-			struct finder_info finfo;
-
-			bzero(&finfo, sizeof(finfo));
-			finfo.fdFlags = ip->i_FinderFlags;
-			finfo.fdLocation.v = -1;
-			finfo.fdLocation.h = -1;
-			if (vnode_isreg(ITOV(ip))) {
-				finfo.fdType = ip->i_FileType;
-				finfo.fdCreator = ip->i_Creator;
-			}
-            bcopy (&finfo, attrbufptr, sizeof(finfo));
-            (u_int8_t *)attrbufptr += sizeof(finfo);
-            bzero (attrbufptr, EXTFNDRINFOSIZE);
-            (u_int8_t *)attrbufptr += EXTFNDRINFOSIZE;
-		};
-		if (a & ATTR_CMN_OWNERID) *((uid_t *)attrbufptr)++ = ip->inode.iso_uid;
-		if (a & ATTR_CMN_GRPID) *((gid_t *)attrbufptr)++ = ip->inode.iso_gid;
-		if (a & ATTR_CMN_ACCESSMASK) *((uint32_t *)attrbufptr)++ = (uint32_t)ip->inode.iso_mode;
-		if (a & ATTR_CMN_FLAGS) *((uint32_t *)attrbufptr)++ = 0; /* could also use ip->i_flag */
-		if (a & ATTR_CMN_USERACCESS) {
-			*((uint32_t *)attrbufptr)++ =
-				DerivePermissionSummary(ip->inode.iso_uid,
-										ip->inode.iso_gid,
-										ip->inode.iso_mode,
-										imp);
-		};
-	};
-	
-	*attrbufptrptr = attrbufptr;
-	*varbufptrptr = varbufptr;
-}
-
-
-void
-packdirattr(struct attrlist *alist,
-			struct iso_node *ip,
-			void **attrbufptrptr,
-			__unused void **varbufptrptr)
-{
-    void *attrbufptr;
-    attrgroup_t a;
-	int filcnt, dircnt;
-	
-	attrbufptr = *attrbufptrptr;
-	filcnt = dircnt = 0;
-	
-	a = alist->dirattr;
-	if (vnode_isdir(ITOV(ip)) && (a != 0)) {
-		/*
-		 * if we haven't counted our children yet, do it now...
-		 */
-		if ((ip->i_entries == 0) &&
-		    (a & (ATTR_DIR_LINKCOUNT | ATTR_DIR_ENTRYCOUNT))) {
-			(void) isochildcount(ITOV(ip), &dircnt, &filcnt);
-
-			if ((ip->inode.iso_links == 1) && (dircnt != 0))
-				ip->inode.iso_links = dircnt;
-			if ((filcnt + dircnt) > 0)
-				ip->i_entries = dircnt + filcnt;
-		}
-
-		if (a & ATTR_DIR_LINKCOUNT) {
-			*((uint32_t *)attrbufptr)++ = ip->inode.iso_links;
-		}
-		if (a & ATTR_DIR_ENTRYCOUNT) {
-			/* exclude '.' and '..' from total caount */
-			*((uint32_t *)attrbufptr)++ = ((ip->i_entries <= 2) ? 0 : (ip->i_entries - 2));
-		}
-		if (a & ATTR_DIR_MOUNTSTATUS) {
-			if (vnode_mountedhere(ITOV(ip))) {
-				*((uint32_t *)attrbufptr)++ = DIR_MNTSTATUS_MNTPOINT;
-			} else {
-				*((uint32_t *)attrbufptr)++ = 0;
-			};
-		};
-	};
-	
-	*attrbufptrptr = attrbufptr;
-}
-
-
-void
-packfileattr(struct attrlist *alist,
-			 struct iso_node *ip,
-			 void **attrbufptrptr,
-			 void **varbufptrptr)
-{
-    void *attrbufptr = *attrbufptrptr;
-    void *varbufptr = *varbufptrptr;
-    attrgroup_t a = alist->fileattr;
-	
-	if (vnode_isreg(ITOV(ip)) && (a != 0)) {
-		if (a & ATTR_FILE_LINKCOUNT)
-			*((uint32_t *)attrbufptr)++ = ip->inode.iso_links;
-		if (a & ATTR_FILE_TOTALSIZE)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_size;
-		if (a & ATTR_FILE_ALLOCSIZE)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_size;
-		if (a & ATTR_FILE_IOBLOCKSIZE)
-			*((uint32_t *)attrbufptr)++ = ip->i_mnt->logical_block_size;
-		if (a & ATTR_FILE_CLUMPSIZE)
-			*((uint32_t *)attrbufptr)++ = ip->i_mnt->logical_block_size;
-		if (a & ATTR_FILE_DEVTYPE)
-			*((uint32_t *)attrbufptr)++ = (uint32_t)ip->inode.iso_rdev;
-		if (a & ATTR_FILE_DATALENGTH)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_size;
-		if (a & ATTR_FILE_DATAALLOCSIZE)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_size;
-		if (a & ATTR_FILE_RSRCLENGTH)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_rsrcsize;
-		if (a & ATTR_FILE_RSRCALLOCSIZE)
-			*((off_t *)attrbufptr)++ = (off_t)ip->i_rsrcsize;
-	}
-	
-	*attrbufptrptr = attrbufptr;
-	*varbufptrptr = varbufptr;
-}
-
-
-void
-packattrblk(struct attrlist *alist,
-			struct vnode *vp,
-			void **attrbufptrptr,
-			void **varbufptrptr)
-{
-	struct iso_node *ip = VTOI(vp);
-
-	if (alist->volattr != 0) {
-		packvolattr(alist, ip, attrbufptrptr, varbufptrptr);
-	} else {
-		packcommonattr(alist, ip, attrbufptrptr, varbufptrptr);
-		
-		switch (vnode_vtype(ITOV(ip))) {
-		  case VDIR:
-			packdirattr(alist, ip, attrbufptrptr, varbufptrptr);
-			break;
-			
-		  case VREG:
-			packfileattr(alist, ip, attrbufptrptr, varbufptrptr);
-			break;
-		  
-		  /* Without this the compiler complains about VNON,VBLK,VCHR,VLNK,VSOCK,VFIFO,VBAD and VSTR
-		     not being handled...
-		   */
-		  default:
-			break;
-		};
-	};
-};
diff --git a/bsd/isofs/cd9660/cd9660_vfsops.c b/bsd/isofs/cd9660/cd9660_vfsops.c
index 107bdc848..acdb699a2 100644
--- a/bsd/isofs/cd9660/cd9660_vfsops.c
+++ b/bsd/isofs/cd9660/cd9660_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_vfsops.c,v 1.18 1995/03/09 12:05:36 mycroft Exp $	*/
 
@@ -131,7 +137,9 @@ struct vfsops cd9660_vfsops = {
 	cd9660_fhtovp,
 	cd9660_vptofh,
 	cd9660_init,
-	cd9660_sysctl
+	cd9660_sysctl,
+	NULL,
+	{NULL}
 };
 
 /*
@@ -183,11 +191,11 @@ cd9660_mount(mount_t mp, vnode_t devvp, user_addr_t data, vfs_context_t context)
 	struct iso_mnt *imp = NULL;
 
 	if (vfs_context_is64bit(context)) {
-		error = copyin(data, (caddr_t)&args, sizeof (args));
+		error = copyin(data, (caddr_t)&args, sizeof(args));
 	}
 	else {
 		struct iso_args temp;
-		error = copyin(data, (caddr_t)&temp, sizeof (temp));
+		error = copyin(data, (caddr_t)&temp, sizeof(temp));
 		args.flags = temp.flags;
 		args.ssector = temp.ssector;
 		args.toc_length = temp.toc_length;
@@ -253,7 +261,7 @@ cd9660_phys_device(mount_t mp, vfs_context_t context)
 		return (NULL);
 
 	/* Make a copy of the mount from name, then remove trailing "s...". */
-	strncpy(whole_path, sfs->f_mntfromname, sizeof(whole_path)-1); 
+	strlcpy(whole_path, sfs->f_mntfromname, sizeof(whole_path)); 
 	
 	/*
 	 * I would use strrchr or rindex here, but those are declared __private_extern__,
@@ -299,7 +307,6 @@ cd9660_find_video_dir(struct iso_mnt *isomp)
 	struct vnode *rvp = NULL;
 	struct vnode *videovp = NULL;
 	struct componentname cn;
-	struct vfs_context context;
 	char dirname[] = "MPEGAV";
 	
 	result = 0;		/* Assume not a video CD */
@@ -310,18 +317,15 @@ cd9660_find_video_dir(struct iso_mnt *isomp)
 		return 0;	/* couldn't find video dir */
 	}
 	
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
-
 	cn.cn_nameiop = LOOKUP;
 	cn.cn_flags = ISLASTCN;
-	cn.cn_context = &context;
+	cn.cn_context = vfs_context_current();
 	cn.cn_pnbuf = dirname;
 	cn.cn_pnlen = sizeof(dirname)-1;
 	cn.cn_nameptr = cn.cn_pnbuf;
 	cn.cn_namelen = cn.cn_pnlen;
 	
-	err = VNOP_LOOKUP(rvp, &videovp, &cn, &context);
+	err = VNOP_LOOKUP(rvp, &videovp, &cn, cn.cn_context);
 	if (err == 0) {
 		struct iso_node *ip = VTOI(videovp);
 		result = 1;		/* Looks like video CD */
@@ -340,20 +344,16 @@ cd9660_find_video_dir(struct iso_mnt *isomp)
  * Common code for mount and mountroot
  */
 static int
-iso_mountfs(devvp, mp, argp, context)
-	register struct vnode *devvp;
-	struct mount *mp;
-	struct user_iso_args *argp;
-	vfs_context_t context;
+iso_mountfs(struct vnode *devvp, struct mount *mp, struct user_iso_args *argp,
+	    vfs_context_t context)
 {
-	struct proc *p;
-	register struct iso_mnt *isomp = (struct iso_mnt *)0;
+	struct iso_mnt *isomp = (struct iso_mnt *)0;
 	struct buf *bp = NULL;
 	struct buf *pribp = NULL, *supbp = NULL;
 	dev_t dev = vnode_specrdev(devvp);
 	int error = EINVAL;
 	int breaderr = 0;
-	u_long iso_bsize;
+	u_long iso_bsize, orig_bsize;
 	int iso_blknum;
 	int joliet_level;
 	struct iso_volume_descriptor *vdp = NULL;
@@ -367,6 +367,16 @@ iso_mountfs(devvp, mp, argp, context)
 	if (vfs_isrdwr(mp))
 		return (EROFS);
 
+	/* Advisory locking should be handled at the VFS layer */
+	vfs_setlocklocal(mp);
+
+	/* Finish initializing hash tables */
+	cd9660_hashinit();
+
+	if ((error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE,
+	     (caddr_t)&orig_bsize, 0, context)))
+		return (error);
+	
 	/* This is the "logical sector size".  The standard says this
 	 * should be 2048 or the physical sector size on the device,
 	 * whichever is greater.  For now, we'll just use a constant.
@@ -391,7 +401,7 @@ iso_mountfs(devvp, mp, argp, context)
 			continue;
 		}
 
-		vdp = (struct iso_volume_descriptor *)buf_dataptr(bp);
+		vdp = (struct iso_volume_descriptor *)((char *)0 + buf_dataptr(bp));
 		if (bcmp (vdp->volume_desc_id, ISO_STANDARD_ID, sizeof(vdp->volume_desc_id)) != 0) {
 #ifdef DEBUG
 		        printf("cd9660_vfsops.c: iso_mountfs: "
@@ -474,8 +484,8 @@ iso_mountfs(devvp, mp, argp, context)
 	
 	rootp = (struct iso_directory_record *)pri->root_directory_record;
 	
-	MALLOC(isomp, struct iso_mnt *, sizeof *isomp, M_ISOFSMNT, M_WAITOK);
-	bzero((caddr_t)isomp, sizeof *isomp);
+	MALLOC(isomp, struct iso_mnt *, sizeof(*isomp), M_ISOFSMNT, M_WAITOK);
+	bzero((caddr_t)isomp, sizeof(*isomp));
 	isomp->im_sector_size = ISO_DEFAULT_BLOCK_SIZE;
 	isomp->logical_block_size = logical_block_size;
 	isomp->volume_space_size = isonum_733 (pri->volume_space_size);
@@ -488,7 +498,7 @@ iso_mountfs(devvp, mp, argp, context)
 	 * filehandle validation.
 	 */
 	isomp->volume_space_size += blkoff;
-	bcopy (rootp, isomp->root, sizeof isomp->root);
+	bcopy (rootp, isomp->root, sizeof(isomp->root));
 	isomp->root_extent = isonum_733 (rootp->extent);
 	isomp->root_size = isonum_733 (rootp->size);
 
@@ -508,7 +518,7 @@ iso_mountfs(devvp, mp, argp, context)
 	}
 
 	if (pri->volume_id[0] == 0)
-		strcpy(isomp->volume_id, ISO_DFLT_VOLUME_ID);
+		strlcpy(isomp->volume_id, ISO_DFLT_VOLUME_ID, sizeof(isomp->volume_id));
 	else
 		bcopy(pri->volume_id, isomp->volume_id, sizeof(isomp->volume_id));
 	cd9660_tstamp_conv17(pri->creation_date, &isomp->creation_date);
@@ -563,7 +573,7 @@ iso_mountfs(devvp, mp, argp, context)
 			argp->flags |= ISOFSMNT_NORRIP;
 			goto skipRRIP;
 		}
-		rootp = (struct iso_directory_record *)buf_dataptr(bp);
+		rootp = (struct iso_directory_record *)((char *)0 + buf_dataptr(bp));
 		
 		if ((isomp->rr_skip = cd9660_rrip_offset(rootp,isomp)) < 0) {
 			argp->flags  |= ISOFSMNT_NORRIP;
@@ -609,9 +619,7 @@ skipRRIP:
 		 *
 		 * This name can have up to 16 UCS-2 chars.
 		 */
-		convflags = UTF_DECOMPOSED;
-		if (BYTE_ORDER != BIG_ENDIAN)
-			convflags |= UTF_REVERSE_ENDIAN;
+		convflags = UTF_DECOMPOSED | UTF_BIG_ENDIAN;
 		uchp = (u_int16_t *)sup->volume_id;
 		for (i = 0; i < 16 && uchp[i]; ++i);
 		if ((utf8_encodestr((u_int16_t *)sup->volume_id, (i * 2), vol_id,
@@ -628,7 +636,7 @@ skipRRIP:
 
 		rootp = (struct iso_directory_record *)
 			sup->root_directory_record;
-		bcopy (rootp, isomp->root, sizeof isomp->root);
+		bcopy (rootp, isomp->root, sizeof(isomp->root));
 		isomp->root_extent = isonum_733 (rootp->extent);
 		isomp->root_size = isonum_733 (rootp->size);
 		buf_markaged(supbp);
@@ -658,6 +666,11 @@ skipRRIP:
 
 	return (0);
 out:
+	if (orig_bsize != iso_bsize) {
+		(void)VNOP_IOCTL(devvp, DKIOCSETBLOCKSIZE,
+			(caddr_t)&orig_bsize, FWRITE, context);
+	}
+
 	if (bp)
 		buf_brelse(bp);
 	if (pribp)
@@ -693,7 +706,7 @@ cd9660_start(__unused struct mount *mp, __unused int flags,
 int
 cd9660_unmount(struct mount *mp, int mntflags, vfs_context_t context)
 {
-	register struct iso_mnt *isomp;
+	struct iso_mnt *isomp;
 	int error, flags = 0;
 	int force = 0;
 	
@@ -749,10 +762,10 @@ cd9660_root(struct mount *mp, struct vnode **vpp, __unused vfs_context_t context
  */
 /* ARGSUSED */
 int
-cd9660_statfs(struct mount *mp, register struct vfsstatfs *sbp,
+cd9660_statfs(struct mount *mp, struct vfsstatfs *sbp,
 	      __unused vfs_context_t context)
 {
-	register struct iso_mnt *isomp;
+	struct iso_mnt *isomp;
 	
 	isomp = VFSTOISOFS(mp);
 
@@ -791,7 +804,7 @@ cd9660_statfs(struct mount *mp, register struct vfsstatfs *sbp,
 	return (0);
 }
 
-int cd9660_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, vfs_context_t context)
+int cd9660_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, __unused vfs_context_t context)
 {
 	struct iso_mnt *imp;
 	struct vfsstatfs *stats = vfs_statfs(mp);
@@ -847,11 +860,7 @@ int cd9660_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, vfs_context_t co
 			VOL_CAP_INT_SEARCHFS |
 			VOL_CAP_INT_ATTRLIST |
 			VOL_CAP_INT_NFSEXPORT |
-			VOL_CAP_INT_READDIRATTR |
-			VOL_CAP_INT_EXCHANGEDATA |
-			VOL_CAP_INT_COPYFILE |
 			VOL_CAP_INT_ALLOCATE |
-			VOL_CAP_INT_VOL_RENAME |
 			VOL_CAP_INT_ADVLOCK |
 			VOL_CAP_INT_FLOCK;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_RESERVED1] = 0;
@@ -867,15 +876,19 @@ int cd9660_vfs_getattr(struct mount *mp, struct vfs_attr *fsap, vfs_context_t co
 		 * VFS has implemented.
 		 */
 
-		fsap->f_attributes.validattr.commonattr = ATTR_CMN_VALIDMASK;
-        fsap->f_attributes.validattr.volattr = ATTR_VOL_VALIDMASK;
-        fsap->f_attributes.validattr.dirattr = ATTR_DIR_VALIDMASK;
+#define ISOFS_ATTR_CMN_VALIDMASK	(ATTR_CMN_VALIDMASK & ~(ATTR_CMN_PAROBJID | ATTR_CMN_CRTIME | ATTR_CMN_BKUPTIME | ATTR_CMN_PARENTID))
+#define ISOFS_ATTR_VOL_VALIDMASK	(ATTR_VOL_VALIDMASK & ~(ATTR_VOL_OBJCOUNT | ATTR_VOL_FILECOUNT | ATTR_VOL_DIRCOUNT | ATTR_VOL_MAXOBJCOUNT | ATTR_VOL_NAME))
+#define ISOFS_ATTR_DIR_VALIDMASK	(ATTR_DIR_VALIDMASK & ~(ATTR_DIR_ENTRYCOUNT))
+
+		fsap->f_attributes.validattr.commonattr = ISOFS_ATTR_CMN_VALIDMASK;
+        fsap->f_attributes.validattr.volattr = ISOFS_ATTR_VOL_VALIDMASK;
+        fsap->f_attributes.validattr.dirattr = ISOFS_ATTR_DIR_VALIDMASK;
         fsap->f_attributes.validattr.fileattr = ATTR_FILE_VALIDMASK;
         fsap->f_attributes.validattr.forkattr = ATTR_FORK_VALIDMASK;
 
-		fsap->f_attributes.nativeattr.commonattr = ATTR_CMN_VALIDMASK;
-        fsap->f_attributes.nativeattr.volattr = ATTR_VOL_VALIDMASK;
-        fsap->f_attributes.nativeattr.dirattr = ATTR_DIR_VALIDMASK;
+		fsap->f_attributes.nativeattr.commonattr = ISOFS_ATTR_CMN_VALIDMASK;
+        fsap->f_attributes.nativeattr.volattr = ISOFS_ATTR_VOL_VALIDMASK;
+        fsap->f_attributes.nativeattr.dirattr = ISOFS_ATTR_DIR_VALIDMASK;
         fsap->f_attributes.nativeattr.fileattr = ATTR_FILE_VALIDMASK;
         fsap->f_attributes.nativeattr.forkattr = ATTR_FORK_VALIDMASK;
 
@@ -919,7 +932,7 @@ int
 cd9660_fhtovp(mount_t mp, int fhlen, unsigned char *fhp, vnode_t *vpp, vfs_context_t context)
 {
 	struct ifid *ifhp = (struct ifid *)fhp;
-	register struct iso_node *ip;
+	struct iso_node *ip;
 	struct vnode *nvp;
 	int error;
 	
@@ -1050,7 +1063,7 @@ cd9660_vget_internal(mount_t mp, ino_t ino, vnode_t *vpp, vnode_t dvp,
 		     struct componentname *cnp, int relocated,
 		     struct iso_directory_record *isodir, proc_t p)
 {
-	register struct iso_mnt *imp;
+	struct iso_mnt *imp;
 	struct iso_node *ip;
 	buf_t	bp = NULL;
 	vnode_t	vp;
@@ -1144,7 +1157,7 @@ cd9660_vget_internal(mount_t mp, ino_t ino, vnode_t *vpp, vnode_t dvp,
 			struct iso_directory_record *pdp;
 
 			pdp = (struct iso_directory_record *)
-					((char *)buf_dataptr(bp) + isonum_711(isodir->length));
+					((char *)0 + buf_dataptr(bp) + isonum_711(isodir->length));
 			if ((isonum_711(pdp->flags) & directoryBit)
 					&& (pdp->name[0] == 1))
 				ip->i_parent = isodirino(pdp, imp);
@@ -1177,7 +1190,7 @@ cd9660_vget_internal(mount_t mp, ino_t ino, vnode_t *vpp, vnode_t dvp,
 		if ((error = (int)buf_bread(imp->im_devvp, lbn, imp->im_sector_size, NOCRED, &bp)))
 		        goto errout;
 
-		isodir = (struct iso_directory_record *)buf_dataptr(bp);
+		isodir = (struct iso_directory_record *)((char *)0 + buf_dataptr(bp));
 	}
 
 	/*
@@ -1583,8 +1596,8 @@ DoneLooking:
 int
 cd9660_vptofh(struct vnode *vp, int *fhlenp, unsigned char *fhp, __unused vfs_context_t context)
 {
-	register struct iso_node *ip = VTOI(vp);
-	register struct ifid *ifhp;
+	struct iso_node *ip = VTOI(vp);
+	struct ifid *ifhp;
 
 	if (*fhlenp < (int)sizeof(struct ifid))
 		return (EOVERFLOW);
diff --git a/bsd/isofs/cd9660/cd9660_vnops.c b/bsd/isofs/cd9660/cd9660_vnops.c
index b8994eb2f..a9eadab4f 100644
--- a/bsd/isofs/cd9660/cd9660_vnops.c
+++ b/bsd/isofs/cd9660/cd9660_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: cd9660_vnops.c,v 1.22 1994/12/27 19:05:12 mycroft Exp $	*/
 
@@ -240,7 +246,7 @@ cd9660_read(struct vnop_read_args *ap)
 			}
 		}
 		if (uio_resid(uio) > 0)
-			error = cluster_read(vp, uio, (off_t)ip->i_size, 0);
+			error = cluster_read(vp, uio, (off_t)ip->i_size, ap->a_ioflag);
 	} else {
 
 	do {
@@ -396,8 +402,10 @@ int
 cd9660_readdir(struct vnop_readdir_args *ap)
 {
 	register struct uio *uio = ap->a_uio;
+#if 0
 	off_t startingOffset = uio->uio_offset;
 	size_t lost = 0;
+#endif	/* 0 */
 	struct isoreaddir *idp;
 	struct vnode *vdp = ap->a_vp;
 	struct iso_node *dp;
@@ -521,7 +529,8 @@ cd9660_readdir(struct vnop_readdir_args *ap)
 			break;
 
 		default:	/* ISO_FTYPE_DEFAULT || ISO_FTYPE_9660 */
-			strcpy(idp->current.d_name,"..");
+			strlcpy(idp->current.d_name, "..",
+				__DARWIN_MAXNAMLEN + 1);
 			switch (ep->name[0]) {
 			case 0:
 				idp->current.d_namlen = 1;
@@ -762,10 +771,10 @@ cd9660_pathconf(struct vnop_pathconf_args *ap)
 		*ap->a_retval = PIPE_BUF;
 		return (0);
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		return (0);
 	case _PC_NO_TRUNC:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_NO_TRUNC */
 		return (0);
 	default:
 		return (EINVAL);
@@ -835,10 +844,6 @@ cd9660_pagein(struct vnop_pagein_args *ap)
 		}
 	} else {
 		/* check pageouts are for reg file only  and ubc info is present*/
-		if  (UBCINVALID(vp))
-			panic("cd9660_pagein: Not a  VREG");
-		UBCINFOCHECK("cd9660_pagein", vp);
-	
 		error = cluster_pagein(vp, pl, pl_offset, f_offset, size,
 				       (off_t)ip->i_size, flags);
 	}
@@ -869,96 +874,6 @@ cd9660_rmdir(struct vnop_rmdir_args *ap)
     return (EROFS);
 }   
 
-/*
-
-#
-#% getattrlist	vp	= = =
-#
- vnop_getattrlist {
-     IN struct vnode *vp;
-     IN struct attrlist *alist;
-     INOUT struct uio *uio;
-     IN vfs_context_t context;
- };
-
- */
-int
-cd9660_getattrlist(struct vnop_getattrlist_args *ap)
-{
-    struct attrlist *alist = ap->a_alist;
-    int fixedblocksize;
-    int attrblocksize;
-    int attrbufsize;
-    void *attrbufptr;
-    void *attrptr;
-    void *varptr;
-    int error = 0;
-
-	if ((alist->bitmapcount != ATTR_BIT_MAP_COUNT) ||
-        ((alist->commonattr & ~ATTR_CMN_VALIDMASK) != 0) ||
-        ((alist->volattr & ~ATTR_VOL_VALIDMASK) != 0) ||
-        ((alist->dirattr & ~ATTR_DIR_VALIDMASK) != 0) ||
-        ((alist->fileattr & ~ATTR_FILE_VALIDMASK) != 0) ||
-        ((alist->forkattr & ~ATTR_FORK_VALIDMASK) != 0)) {
-		return EINVAL;
-	};
-
-	/* 
-	 * Requesting volume information requires setting the ATTR_VOL_INFO bit and
-	 * volume info requests are mutually exclusive with all other info requests:
-	 */
-	if ((alist->volattr != 0) &&
-		(((alist->volattr & ATTR_VOL_INFO) == 0) ||
-		(alist->dirattr != 0) || 
-		(alist->fileattr != 0) || 
-		(alist->forkattr != 0) )) {
-        return EINVAL;
-	};
-
-	/*
-	 * Reject requests for unsupported options for now:
-	 */
-	if (alist->volattr & ATTR_VOL_MOUNTPOINT) return EINVAL;
-	if (alist->commonattr & (ATTR_CMN_NAMEDATTRCOUNT | ATTR_CMN_NAMEDATTRLIST)) return EINVAL;
-	if (alist->fileattr &
-		(ATTR_FILE_FILETYPE |
-		 ATTR_FILE_FORKCOUNT |
-		 ATTR_FILE_FORKLIST |
-		 ATTR_FILE_DATAEXTENTS |
-		 ATTR_FILE_RSRCEXTENTS)) {
-		return EINVAL;
-	};
-
-
-    fixedblocksize = attrcalcsize(alist);
-    attrblocksize = fixedblocksize + (sizeof(uint32_t));                    /* uint32_t for length word */
-    if (alist->commonattr & ATTR_CMN_NAME) attrblocksize += NAME_MAX;
-    if (alist->commonattr & ATTR_CMN_NAMEDATTRLIST) attrblocksize += 0;			/* XXX PPD */
-    if (alist->volattr & ATTR_VOL_MOUNTPOINT) attrblocksize += PATH_MAX;
-    if (alist->volattr & ATTR_VOL_NAME) attrblocksize += NAME_MAX;
-    if (alist->fileattr & ATTR_FILE_FORKLIST) attrblocksize += 0;				/* XXX PPD */
-
-    attrbufsize = MIN(uio_resid(ap->a_uio), attrblocksize);
-    MALLOC(attrbufptr, void *, attrblocksize, M_TEMP, M_WAITOK);
-    attrptr = attrbufptr;
-    *((uint32_t *)attrptr) = 0;									/* Set buffer length in case of errors */
-    ++((uint32_t *)attrptr);										/* Reserve space for length field */
-    varptr = ((char *)attrptr) + fixedblocksize;				/* Point to variable-length storage */
-
-	packattrblk(alist, ap->a_vp, &attrptr, &varptr);
-
-    /* Store length of fixed + var block */
-    *((uint32_t *)attrbufptr) = ((char*)varptr - (char*)attrbufptr);
-    /* Don't copy out more data than was generated */
-    attrbufsize = MIN(attrbufsize, (char*)varptr - (char*)attrbufptr);
-
-    error = uiomove((caddr_t)attrbufptr, attrbufsize, ap->a_uio);
-
-    FREE(attrbufptr, M_TEMP);
-
-    return error;
-}
-
 /*
  * Make a RIFF file header for a CD-ROM XA media file.
  */
@@ -1172,11 +1087,6 @@ cd9660_xa_pagein(struct vnop_pagein_args *ap)
     kern_return_t kret;
     vm_offset_t ioaddr;
 
-	/* check pageins are for reg file only  and ubc info is present*/
-	if  (UBCINVALID(vp))
-		panic("cd9660_xa_pagein: Not a  VREG");
-	UBCINFOCHECK("cd9660_xa_pagein", vp);
-
 	if (size <= 0)
 		panic("cd9660_xa_pagein: size = %d", size);
 
@@ -1274,7 +1184,6 @@ struct vnodeopv_entry_desc cd9660_vnodeop_entries[] = {
 	{ &vnop_bwrite_desc, (VOPFUNC)vn_bwrite },
 	{ &vnop_pagein_desc, (VOPFUNC)cd9660_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)cd9660_pageout },		/* Pageout */
-	{ &vnop_getattrlist_desc, (VOPFUNC)cd9660_getattrlist },	/* getattrlist */
 	{ &vnop_blktooff_desc, (VOPFUNC)cd9660_blktooff },	/* blktooff */
 	{ &vnop_offtoblk_desc, (VOPFUNC)cd9660_offtoblk },	/* offtoblk */
   	{ &vnop_blockmap_desc, (VOPFUNC)cd9660_blockmap },		/* blockmap */
@@ -1323,7 +1232,6 @@ struct vnodeopv_entry_desc cd9660_cdxaop_entries[] = {
 	{ &vnop_bwrite_desc, (VOPFUNC)vn_bwrite },
 	{ &vnop_pagein_desc, (VOPFUNC)cd9660_xa_pagein },		/* Pagein */
 	{ &vnop_pageout_desc, (VOPFUNC)cd9660_pageout },		/* Pageout */
-	{ &vnop_getattrlist_desc, (VOPFUNC)cd9660_getattrlist },	/* getattrlist */
 	{ (struct vnodeop_desc*)NULL, (VOPFUNC)NULL }
 };
 struct vnodeopv_desc cd9660_cdxaop_opv_desc =
diff --git a/bsd/isofs/cd9660/iso.h b/bsd/isofs/cd9660/iso.h
index 683f9f0e1..b122c4b6c 100644
--- a/bsd/isofs/cd9660/iso.h
+++ b/bsd/isofs/cd9660/iso.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: iso.h,v 1.9 1995/01/18 09:23:19 mycroft Exp $	*/
 
@@ -312,6 +318,7 @@ int cd9660_vget(struct mount *, ino64_t, struct vnode **, vfs_context_t);
 int cd9660_fhtovp(struct mount *, int, unsigned char *, struct vnode **, vfs_context_t);
 int cd9660_vptofh(struct vnode *, int *, unsigned char *, vfs_context_t);
 int cd9660_init(struct vfsconf *);
+int cd9660_hashinit(void);
 int cd9660_mountroot(mount_t, vnode_t, vfs_context_t); 
 int cd9660_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
 
@@ -393,16 +400,6 @@ void isofntrans(u_char *infn, int infnlen, u_char *outfn, u_short *outfnlen,
 		int original, int assoc);
 void ucsfntrans(u_int16_t *, int, u_char *, u_short *, int, int);
 int attrcalcsize(struct attrlist *attrlist);
-struct iso_node;
-void packcommonattr(struct attrlist *alist, struct iso_node *ip,
-		    void **attrbufptrptr, void **varbufptrptr);
-void packdirattr(struct attrlist *alist, struct iso_node *ip,
-		 void **attrbufptrptr, void **varbufptrptr);
-void packfileattr(struct attrlist *alist, struct iso_node *ip,
-		  void **attrbufptrptr, void **varbufptrptr);
-void packattrblk(struct attrlist *alist, struct vnode *vp,
-				  void **attrbufptrptr, void **varbufptrptr);
-
 
 /*
  * Associated files have a leading "._".
@@ -438,19 +435,19 @@ struct riff_header {
 #define APPLEDOUBLE_FINDERINFO   9
 
 /*
- * Note that  68k alignment is needed to make sure that the first
+ * Note that the structures are padded and aligned to 2 bytes;
+ * this is to mimic the "#pragma options align=mac68k" formerly
+ * used.  This is needed to make sure that the first
  * AppleDoubleEntry (after the numEntries below) is *immediately*
  * after the numEntries, and not padded by 2 bytes.
  *
  * Consult RFC 1740 for details on AppleSingle/AppleDouble formats.
  */
-#pragma options align=mac68k
-
 struct apple_double_entry {
 	u_int32_t  entryID;
 	u_int32_t  offset;
 	u_int32_t  length;
-};
+} __attribute__((aligned(2), packed));
 typedef struct apple_double_entry apple_double_entry_t;
 
 struct apple_double_header {
@@ -460,13 +457,11 @@ struct apple_double_header {
 	u_int16_t  count;
 	apple_double_entry_t  entries[2];	/* FinderInfo + ResourceFork */
 	struct finder_info finfo;
-};
+} __attribute__((aligned(2), packed));
 typedef struct apple_double_header apple_double_header_t;
 
 #define ADH_SIZE  4096
 #define ADH_BLKS  2
 
-#pragma options align=reset
-
 #endif /* __APPLE_API_PRIVATE */
 #endif /* ! _ISO_H_ */
diff --git a/bsd/isofs/cd9660/iso_rrip.h b/bsd/isofs/cd9660/iso_rrip.h
index cfc5f1397..1ce14ba8c 100644
--- a/bsd/isofs/cd9660/iso_rrip.h
+++ b/bsd/isofs/cd9660/iso_rrip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: iso_rrip.h,v 1.3 1994/06/29 06:32:02 cgd Exp $	*/
 
diff --git a/bsd/kern/ast.h b/bsd/kern/ast.h
index d320003a9..a8dc21932 100644
--- a/bsd/kern/ast.h
+++ b/bsd/kern/ast.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* CMU_HIST */
 
diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c
index 05b66d530..2ea3d6377 100644
--- a/bsd/kern/bsd_init.c
+++ b/bsd/kern/bsd_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1991, 1992, 1993
@@ -66,6 +72,12 @@
  * All rights reserved.  The CMU software License Agreement specifies
  * the terms and conditions for use and redistribution.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/filedesc.h>
@@ -87,6 +99,7 @@
 
 #include <sys/malloc.h>
 #include <sys/dkstat.h>
+#include <sys/codesign.h>
 
 #include <kern/startup.h>
 #include <kern/thread.h>
@@ -111,23 +124,55 @@
 #include <mach/sync_policy.h>
 #include <kern/clock.h>
 #include <mach/kern_return.h>
-
-#include <mach/shared_memory_server.h>
-#include <vm/vm_shared_memory_server.h>
+#include <mach/thread_act.h>		/* for thread_resume() */
+#include <mach/task.h>			/* for task_set_exception_ports() */
+#include <sys/ux_exception.h>		/* for ux_handler() */
+#include <sys/ubc_internal.h>		/* for ubc_init() */
+#include <sys/mcache.h>			/* for mcache_init() */
+#include <sys/mbuf.h>			/* for mbinit() */
+#include <sys/event.h>			/* for knote_init() */
+#include <sys/aio_kern.h>		/* for aio_init() */
+#include <sys/semaphore.h>		/* for psem_cache_init() */
+#include <net/dlil.h>			/* for dlil_init() */
+#include <net/kpi_protocol.h>		/* for proto_kpi_init() */
+#include <sys/pipe.h>			/* for pipeinit() */
+#include <sys/socketvar.h>		/* for socketinit() */
+#include <sys/protosw.h>		/* for domaininit() */
+#include <kern/sched_prim.h>		/* for thread_wakeup() */
+#include <net/if_ether.h>		/* for ether_family_init() */
+#include <vm/vm_protos.h>		/* for vnode_pager_bootstrap() */
+#include <miscfs/devfs/devfsdefs.h>	/* for devfs_kernel_mount() */
+#include <mach/host_priv.h>		/* for host_set_exception_ports() */
+#include <kern/host.h>			/* for host_priv_self() */
+#include <vm/vm_kern.h>			/* for kmem_suballoc() */
+#include <sys/semaphore.h>		/* for psem_lock_init() */
+#include <sys/msgbuf.h>			/* for log_setsize() */
 
 #include <net/init.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#include <security/mac_internal.h>	/* mac_init_bsd() */
+#include <security/mac_mach_internal.h>	/* mac_update_task_label() */
+#endif
+
 #include <machine/exec.h>
 
-extern int app_profile;		/* on/off switch for pre-heat cache */
+#if CONFIG_IMAGEBOOT
+#include <sys/imageboot.h>
+#endif
+
+#include <pexpert/pexpert.h>
+
+void * get_user_regs(thread_t);		/* XXX kludge for <machine/thread.h> */
+void IOKitInitializeTime(void);		/* XXX */
+void loopattach(void);			/* XXX */
 
 char    copyright[] =
 "Copyright (c) 1982, 1986, 1989, 1991, 1993\n\t"
 "The Regents of the University of California. "
 "All rights reserved.\n\n";
 
-extern void	ux_handler();
-
 /* Components of the first process -- never freed. */
 struct	proc proc0;
 struct	session session0;
@@ -136,7 +181,8 @@ struct	filedesc filedesc0;
 struct	plimit limit0;
 struct	pstats pstats0;
 struct	sigacts sigacts0;
-struct	proc *kernproc, *initproc;
+proc_t kernproc;
+proc_t initproc;
 
 long tk_cancc;
 long tk_nin;
@@ -168,7 +214,7 @@ struct exec_archhandler exec_archhandler_ppc;
 
 char rootdevice[16]; 	/* hfs device names have at least 9 chars */
 
-#ifdef  KMEMSTATS
+#if  KMEMSTATS
 struct	kmemstats kmemstats[M_LAST];
 #endif
 
@@ -176,7 +222,21 @@ int	lbolt;				/* awoken once a second */
 struct	vnode *rootvp;
 int boothowto = RB_DEBUG;
 
-#define	BSD_PAGABLE_MAP_SIZE	(16 * 512 * 1024)
+void lightning_bolt(void *);
+extern kern_return_t IOFindBSDRoot(char *, dev_t *, u_int32_t *);
+extern void IOSecureBSDRoot(const char * rootName);
+extern kern_return_t IOKitBSDInit(void );
+extern void kminit(void);
+extern void klogwakeup(void);
+extern void file_lock_init(void);
+extern void kmeminit(void);
+extern void bsd_bufferinit(void);
+
+extern int srv;
+extern int ncl;
+
+#define BSD_SIMUL_EXECS       33 /* 32 , allow for rounding */
+#define	BSD_PAGABLE_MAP_SIZE	(BSD_SIMUL_EXECS * (NCARGS + PAGE_SIZE))
 vm_map_t	bsd_pageable_map;
 vm_map_t	mb_map;
 semaphore_t execve_semaphore;
@@ -184,28 +244,44 @@ semaphore_t execve_semaphore;
 int	cmask = CMASK;
 extern int customnbuf;
 
-int parse_bsd_args(void);
-extern int bsd_hardclockinit;
+void bsd_init(void) __attribute__((section("__TEXT, initcode")));
+__private_extern__ void ubc_init(void )  __attribute__((section("__TEXT, initcode")));
+void vfsinit(void) __attribute__((section("__TEXT, initcode")));
+kern_return_t bsd_autoconf(void) __attribute__((section("__TEXT, initcode")));
+void bsd_utaskbootstrap(void) __attribute__((section("__TEXT, initcode")));
+
+static void parse_bsd_args(void);
 extern task_t bsd_init_task;
 extern char    init_task_failure_data[];
 extern void time_zone_slock_init(void);
-static void process_name(char *, struct proc *);
+static void process_name(const char *, proc_t);
 
 static void setconf(void);
 
 funnel_t *kernel_flock;
 
+#if SYSV_SHM
 extern void sysv_shm_lock_init(void);
+#endif
+#if SYSV_SEM
 extern void sysv_sem_lock_init(void);
+#endif
+#if SYSV_MSG
 extern void sysv_msg_lock_init(void);
-extern void pshm_lock_init();
-extern void psem_lock_init();
-extern int maxprocperuid;
+#endif
+extern void pthread_init(void);
 
 /* kmem access not enabled by default; can be changed with boot-args */
 int setup_kmem = 0;
 
-extern void stackshot_lock_init();
+/* size of kernel trace buffer, disabled by default */
+unsigned int new_nkdbufs = 0;
+
+/* mach leak logging */
+int log_leaks = 0;
+int turn_on_log_leaks = 0;
+
+extern void stackshot_lock_init(void);
 
 /*
  * Initialization code.
@@ -223,11 +299,9 @@ extern void stackshot_lock_init();
  *	Sets the name for the given task.
  */
 static void
-process_name(s, p)
-	char		*s;
-	struct proc *p;
+process_name(const char *s, proc_t p)
 {
-	int		length = strlen(s);
+	size_t length = strlen(s);
 
 	bcopy(s, p->p_comm,
 		length >= sizeof(p->p_comm) ? sizeof(p->p_comm) :
@@ -236,18 +310,20 @@ process_name(s, p)
 
 /* To allow these values to be patched, they're globals here */
 #include <machine/vmparam.h>
-struct rlimit vm_initial_limit_stack = { DFLSSIZ, MAXSSIZ };
+struct rlimit vm_initial_limit_stack = { DFLSSIZ, MAXSSIZ - PAGE_SIZE };
 struct rlimit vm_initial_limit_data = { DFLDSIZ, MAXDSIZ };
 struct rlimit vm_initial_limit_core = { DFLCSIZ, MAXCSIZ };
 
-extern thread_t	cloneproc(struct proc *, int);
+extern thread_t	cloneproc(proc_t, int);
 extern int 	(*mountroot)(void);
-extern int 	netboot_mountroot(); 	/* netboot.c */
-extern int	netboot_setup(struct proc * p);
+extern int 	netboot_mountroot(void); 	/* netboot.c */
+extern int	netboot_setup(void);
 
 lck_grp_t * proc_lck_grp;
 lck_grp_attr_t * proc_lck_grp_attr;
 lck_attr_t * proc_lck_attr;
+lck_mtx_t * proc_list_mlock;
+lck_mtx_t * proc_klist_mlock;
 
 /* hook called after root is mounted XXX temporary hack */
 void (*mountroot_post_hook)(void);
@@ -260,49 +336,46 @@ void (*mountroot_post_hook)(void);
  * Internally, kernel_thread_create() calls thread_create_internal(),
  * which calls uthread_alloc().  The function of uthread_alloc() is
  * normally to allocate a uthread structure, and fill out the uu_sigmask,
- * uu_act, and uu_ucred fields.  It skips filling these out in the case
- * of the "task" being "kernel_task", because the order of operation is
- * inverted.  To account for that, we need to manually fill in at least
- * the uu_cred field so that the uthread structure can be used like any
- * other.
+ * uu_context fields.  It skips filling these out in the case of the "task"
+ * being "kernel_task", because the order of operation is inverted.  To
+ * account for that, we need to manually fill in at least the contents
+ * of the uu_context.vc_ucred field so that the uthread structure can be
+ * used like any other.
  */
 void
-bsd_init()
+bsd_init(void)
 {
-	register struct proc *p;
+	proc_t p;
 	struct uthread *ut;
-	extern kauth_cred_t rootcred;
-	register int i;
-	int s;
+	unsigned int i;
+#if __i386__
 	int error;
-	thread_t	th;
+#endif	
 	struct vfs_context context;
-	void		lightning_bolt(void );
 	kern_return_t	ret;
-	boolean_t funnel_state;
 	struct ucred temp_cred;
-	extern void file_lock_init(void);
+
+#define bsd_init_kprintf(x...) /* kprintf("bsd_init: " x) */
 
 	kernel_flock = funnel_alloc(KERNEL_FUNNEL);
 	if (kernel_flock == (funnel_t *)0 ) {
 		panic("bsd_init: Failed to allocate kernel funnel");
 	}
         
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
 	printf(copyright);
 	
+	bsd_init_kprintf("calling kmeminit\n");
 	kmeminit();
 	
+	bsd_init_kprintf("calling parse_bsd_args\n");
 	parse_bsd_args();
 
-	/* Initialize the uthread zone */
-	//uthread_zone_init();	/* XXX redundant: previous uthread_alloc() */
-
 	/* Initialize kauth subsystem before instancing the first credential */
+	bsd_init_kprintf("calling kauth_init\n");
 	kauth_init();
 
 	/* Initialize process and pgrp structures. */
+	bsd_init_kprintf("calling procinit\n");
 	procinit();
 
 	kernproc = &proc0;
@@ -312,39 +385,74 @@ bsd_init()
 	/* kernel_task->proc = kernproc; */
 	set_bsdtask_info(kernel_task,(void *)p);
 	p->p_pid = 0;
+	p->p_ppid = 0;
 
 	/* give kernproc a name */
+	bsd_init_kprintf("calling process_name\n");
 	process_name("kernel_task", p);
 
 	/* allocate proc lock group attribute and group */
+	bsd_init_kprintf("calling lck_grp_attr_alloc_init\n");
 	proc_lck_grp_attr= lck_grp_attr_alloc_init();
 	
 	proc_lck_grp = lck_grp_alloc_init("proc",  proc_lck_grp_attr);
 
 	/* Allocate proc lock attribute */
 	proc_lck_attr = lck_attr_alloc_init();
+#if 0
+#if __PROC_INTERNAL_DEBUG
+	lck_attr_setdebug(proc_lck_attr);
+#endif
+#endif
 
+	proc_list_mlock = lck_mtx_alloc_init(proc_lck_grp, proc_lck_attr);
+	proc_klist_mlock = lck_mtx_alloc_init(proc_lck_grp, proc_lck_attr);
 	lck_mtx_init(&p->p_mlock, proc_lck_grp, proc_lck_attr);
 	lck_mtx_init(&p->p_fdmlock, proc_lck_grp, proc_lck_attr);
+	lck_spin_init(&p->p_slock, proc_lck_grp, proc_lck_attr);
 
 	if (current_task() != kernel_task)
 		printf("bsd_init: We have a problem, "
 				"current task is not kernel task\n");
 	
+	bsd_init_kprintf("calling get_bsdthread_info\n");
 	ut = (uthread_t)get_bsdthread_info(current_thread());
 
+#if CONFIG_MACF
+	/*
+	 * Initialize the MAC Framework
+	 */
+	mac_policy_initbsd();
+	p->p_mac_enforce = 0;
+#endif /* MAC */
+
 	/*
 	 * Create process 0.
 	 */
+	proc_list_lock();
 	LIST_INSERT_HEAD(&allproc, p, p_list);
 	p->p_pgrp = &pgrp0;
 	LIST_INSERT_HEAD(PGRPHASH(0), &pgrp0, pg_hash);
 	LIST_INIT(&pgrp0.pg_members);
+	lck_mtx_init(&pgrp0.pg_mlock, proc_lck_grp, proc_lck_attr);
+	/* There is no other bsd thread this point and is safe without pgrp lock */
 	LIST_INSERT_HEAD(&pgrp0.pg_members, p, p_pglist);
+	p->p_listflag |= P_LIST_INPGRP;
+	p->p_pgrpid = 0;
 
 	pgrp0.pg_session = &session0;
+	pgrp0.pg_membercnt = 1;
+
 	session0.s_count = 1;
 	session0.s_leader = p;
+	session0.s_listflags = 0;
+	lck_mtx_init(&session0.s_mlock, proc_lck_grp, proc_lck_attr);
+	LIST_INSERT_HEAD(SESSHASH(0), &session0, s_hash);
+	proc_list_unlock();
+
+#if CONFIG_LCTX
+	p->p_lctx = NULL;
+#endif
 
 	p->task = kernel_task;
 	
@@ -352,33 +460,46 @@ bsd_init()
 	p->p_flag = P_SYSTEM;
 	p->p_nice = NZERO;
 	p->p_pptr = p;
-	lockinit(&p->signal_lock, PVM, "signal", 0, 0);
+
 	TAILQ_INIT(&p->p_uthlist);
+	TAILQ_INSERT_TAIL(&p->p_uthlist, ut, uu_list);
+	
 	p->sigwait = FALSE;
 	p->sigwait_thread = THREAD_NULL;
 	p->exit_thread = THREAD_NULL;
+	p->p_csflags = CS_VALID;
 
 	/*
 	 * Create credential.  This also Initializes the audit information.
 	 * XXX It is not clear what the initial values should be for audit ID,
 	 * XXX session ID, etc..
 	 */
+	bsd_init_kprintf("calling bzero\n");
 	bzero(&temp_cred, sizeof(temp_cred));
 	temp_cred.cr_ngroups = 1;
 
+	bsd_init_kprintf("calling kauth_cred_create\n");
 	p->p_ucred = kauth_cred_create(&temp_cred); 
 
 	/* give the (already exisiting) initial thread a reference on it */
+	bsd_init_kprintf("calling kauth_cred_ref\n");
 	kauth_cred_ref(p->p_ucred);
-	ut->uu_ucred = p->p_ucred;
-	
+	ut->uu_context.vc_ucred = p->p_ucred;
+	ut->uu_context.vc_thread = current_thread();
+
 	TAILQ_INIT(&p->aio_activeq);
 	TAILQ_INIT(&p->aio_doneq);
 	p->aio_active_count = 0;
 	p->aio_done_count = 0;
 
+	bsd_init_kprintf("calling file_lock_init\n");
 	file_lock_init();
 
+#if CONFIG_MACF
+	mac_cred_label_associate_kernel(p->p_ucred);
+	mac_task_label_update_cred (p->p_ucred, (struct task *) p->task);
+#endif
+
 	/* Create the file descriptor table. */
 	filedesc0.fd_refcnt = 1+1;	/* +1 so shutdown will not _FREE_ZONE */
 	p->p_fd = &filedesc0;
@@ -399,7 +520,7 @@ bsd_init()
 	limit0.pl_rlimit[RLIMIT_STACK] = vm_initial_limit_stack;
 	limit0.pl_rlimit[RLIMIT_DATA] = vm_initial_limit_data;
 	limit0.pl_rlimit[RLIMIT_CORE] = vm_initial_limit_core;
-	limit0.p_refcnt = 1;
+	limit0.pl_refcnt = 1;
 
 	p->p_stats = &pstats0;
 	p->p_sigacts = &sigacts0;
@@ -407,6 +528,7 @@ bsd_init()
 	/*
 	 * Charge root for two  processes: init and mach_init.
 	 */
+	bsd_init_kprintf("calling chgproccnt\n");
 	(void)chgproccnt(0, 1);
 
 	/*
@@ -414,10 +536,11 @@ bsd_init()
 	 *	for temporary copying (execve()).
 	 */
 	{
-		vm_offset_t	min;
+		vm_offset_t	minimum;
 
+		bsd_init_kprintf("calling kmem_suballoc\n");
 		ret = kmem_suballoc(kernel_map,
-				&min,
+				&minimum,
 				(vm_size_t)BSD_PAGABLE_MAP_SIZE,
 				TRUE,
 				VM_FLAGS_ANYWHERE,
@@ -433,54 +556,97 @@ bsd_init()
 	 *		happen after a credential has been associated with
 	 *		the kernel task.
 	 */
+	bsd_init_kprintf("calling bsd_bufferinit\n");
 	bsd_bufferinit();
 
 	/* Initialize the execve() semaphore */
+	bsd_init_kprintf("calling semaphore_create\n");
 	ret = semaphore_create(kernel_task, &execve_semaphore,
-			SYNC_POLICY_FIFO, (BSD_PAGABLE_MAP_SIZE / NCARGS));
+			       SYNC_POLICY_FIFO, BSD_SIMUL_EXECS -1);
 	if (ret != KERN_SUCCESS)
 		panic("bsd_init: Failed to create execve semaphore");
 
 	/*
 	 * Initialize the calendar.
 	 */
-	IOKitResetTime();
+	IOKitInitializeTime();
+
+	if (turn_on_log_leaks && !new_nkdbufs)
+		new_nkdbufs = 200000;
+	start_kern_tracing(new_nkdbufs);
+	if (turn_on_log_leaks)
+		log_leaks = 1;
 
+	bsd_init_kprintf("calling ubc_init\n");
 	ubc_init();
 
 	/* Initialize the file systems. */
+	bsd_init_kprintf("calling vfsinit\n");
 	vfsinit();
 
+#if SOCKETS
+	/* Initialize per-CPU cache allocator */
+	mcache_init();
+
 	/* Initialize mbuf's. */
+	bsd_init_kprintf("calling mbinit\n");
 	mbinit();
+#endif /* SOCKETS */
 
 	/*
 	 * Initializes security event auditing.
 	 * XXX: Should/could this occur later?
 	 */
-	audit_init();  
+#if AUDIT
+	bsd_init_kprintf("calling audit_init\n");
+ 	audit_init();  
+#endif
 
 	/* Initialize kqueues */
+	bsd_init_kprintf("calling knote_init\n");
 	knote_init();
 
+#if CONFIG_EMBEDDED
+	/* Initialize kernel memory status notifications */
+	bsd_init_kprintf("calling kern_memorystatus_init\n");
+	kern_memorystatus_init();
+#endif
+
 	/* Initialize for async IO */
+	bsd_init_kprintf("calling aio_init\n");
 	aio_init();
 
 	/* Initialize pipes */
+	bsd_init_kprintf("calling pipeinit\n");
 	pipeinit();
 
 	/* Initialize SysV shm subsystem locks; the subsystem proper is
 	 * initialized through a sysctl.
 	 */
+#if SYSV_SHM
+	bsd_init_kprintf("calling sysv_shm_lock_init\n");
 	sysv_shm_lock_init();
+#endif
+#if SYSV_SEM
+	bsd_init_kprintf("calling sysv_sem_lock_init\n");
 	sysv_sem_lock_init();
+#endif
+#if SYSV_MSG
+	bsd_init_kprintf("sysv_msg_lock_init\n");
 	sysv_msg_lock_init();
+#endif
+	bsd_init_kprintf("calling pshm_lock_init\n");
 	pshm_lock_init();
+	bsd_init_kprintf("calling psem_lock_init\n");
 	psem_lock_init();
 
+	pthread_init();
 	/* POSIX Shm and Sem */
+	bsd_init_kprintf("calling pshm_cache_init\n");
 	pshm_cache_init();
+	bsd_init_kprintf("calling psem_cache_init\n");
 	psem_cache_init();
+	bsd_init_kprintf("calling time_zone_slock_init\n");
 	time_zone_slock_init();
 
 	/* Stack snapshot facility lock */
@@ -489,12 +655,22 @@ bsd_init()
 	 * Initialize protocols.  Block reception of incoming packets
 	 * until everything is ready.
 	 */
+	bsd_init_kprintf("calling sysctl_register_fixed\n");
 	sysctl_register_fixed(); 
+	bsd_init_kprintf("calling sysctl_mib_init\n");
 	sysctl_mib_init();
+#if NETWORKING
+	bsd_init_kprintf("calling dlil_init\n");
 	dlil_init();
+	bsd_init_kprintf("calling proto_kpi_init\n");
 	proto_kpi_init();
+#endif /* NETWORKING */
+#if SOCKETS
+	bsd_init_kprintf("calling socketinit\n");
 	socketinit();
+	bsd_init_kprintf("calling domaininit\n");
 	domaininit();
+#endif /* SOCKETS */
 
 	p->p_fd->fd_cdir = NULL;
 	p->p_fd->fd_rdir = NULL;
@@ -506,10 +682,16 @@ bsd_init()
 
 	/* kick off timeout driven events by calling first time */
 	thread_wakeup(&lbolt);
-	timeout((void (*)(void *))lightning_bolt, 0, hz);
+	timeout(lightning_bolt, 0, hz);
 
+	bsd_init_kprintf("calling bsd_autoconf\n");
 	bsd_autoconf();
 
+#if CONFIG_DTRACE
+	extern void dtrace_postinit(void);
+	dtrace_postinit();
+#endif
+
 	/*
 	 * We attach the loopback interface *way* down here to ensure
 	 * it happens after autoconf(), otherwise it becomes the
@@ -517,15 +699,23 @@ bsd_init()
 	 */
 #include <loop.h>
 #if NLOOP > 0
+	bsd_init_kprintf("calling loopattach\n");
 	loopattach();			/* XXX */
 #endif
         
-        /* Register the built-in dlil ethernet interface family */
+#if NETHER > 0
+	/* Register the built-in dlil ethernet interface family */
+	bsd_init_kprintf("calling ether_family_init\n");
 	ether_family_init();
+#endif /* ETHER */
 
+#if NETWORKING
 	/* Call any kext code that wants to run just after network init */
+	bsd_init_kprintf("calling net_init_run\n");
 	net_init_run();
+#endif /* NETWORKING */
 
+	bsd_init_kprintf("calling vnode_pager_bootstrap\n");
 	vnode_pager_bootstrap();
 #if 0
 	/* XXX Hack for early debug stop */
@@ -534,36 +724,83 @@ bsd_init()
 	/* Debugger("hello"); */
 #endif
 
+	bsd_init_kprintf("calling inittodr\n");
 	inittodr(0);
 
+#if CONFIG_EMBEDDED
+	{
+		/* print out early VM statistics */
+		kern_return_t kr1;
+		vm_statistics_data_t stat;
+		mach_msg_type_number_t count;
+
+		count = HOST_VM_INFO_COUNT;
+		kr1 = host_statistics(host_self(),
+				      HOST_VM_INFO,
+				      (host_info_t)&stat,
+				      &count);
+		kprintf("Mach Virtual Memory Statistics (page size of 4096) bytes\n"
+			"Pages free:\t\t\t%u.\n"
+			"Pages active:\t\t\t%u.\n"
+			"Pages inactive:\t\t\t%u.\n"
+			"Pages wired down:\t\t%u.\n"
+			"\"Translation faults\":\t\t%u.\n"
+			"Pages copy-on-write:\t\t%u.\n"
+			"Pages zero filled:\t\t%u.\n"
+			"Pages reactivated:\t\t%u.\n"
+			"Pageins:\t\t\t%u.\n"
+			"Pageouts:\t\t\t\%u.\n"
+			"Object cache: %u hits of %u lookups (%d%% hit rate)\n",
+
+			stat.free_count,
+			stat.active_count,
+			stat.inactive_count,
+			stat.wire_count,
+			stat.faults,
+			stat.cow_faults,
+			stat.zero_fill_count,
+			stat.reactivations,
+			stat.pageins,
+			stat.pageouts,
+			stat.hits,
+			stat.lookups,
+			(stat.hits == 0) ? 100 :
+			                   ((stat.lookups * 100) / stat.hits));
+	}
+#endif /* CONFIG_EMBEDDED */
+	
 	/* Mount the root file system. */
 	while( TRUE) {
 		int err;
 
+		bsd_init_kprintf("calling setconf\n");
 		setconf();
-		bsd_hardclockinit = -1;	/* start ticking */
 
+		bsd_init_kprintf("vfs_mountroot\n");
 		if (0 == (err = vfs_mountroot()))
 			break;
+		rootdevice[0] = '\0';
 #if NFSCLIENT
 		if (mountroot == netboot_mountroot) {
-			printf("cannot mount network root, errno = %d\n", err);
-			mountroot = NULL;
-			if (0 == (err = vfs_mountroot()))
-				break;
+			printf("bsd_init: netboot_mountroot failed,"
+			       " errno = %d\n", err);
+			panic("bsd_init: failed to mount network root: %s", PE_boot_args());
 		}
 #endif
 		printf("cannot mount root, errno = %d\n", err);
 		boothowto |= RB_ASKNAME;
 	}
 
-	context.vc_proc = p;
+	IOSecureBSDRoot(rootdevice);
+
+	context.vc_thread = current_thread();
 	context.vc_ucred = p->p_ucred;
 	mountlist.tqh_first->mnt_flag |= MNT_ROOTFS;
 
+	bsd_init_kprintf("calling VFS_ROOT\n");
 	/* Get the vnode for '/'.  Set fdp->fd_fd.fd_cdir to reference it. */
 	if (VFS_ROOT(mountlist.tqh_first, &rootvnode, &context))
-		panic("bsd_init: cannot find root vnode");
+		panic("bsd_init: cannot find root vnode: %s", PE_boot_args());
 	rootvnode->v_flag |= VROOT;
 	(void)vnode_ref(rootvnode);
 	(void)vnode_put(rootvnode);
@@ -573,97 +810,114 @@ bsd_init()
 	if (mountroot == netboot_mountroot) {
 		int err;
 		/* post mount setup */
-		if (err = netboot_setup(p)) {
-			panic("bsd_init: NetBoot could not find root, %d", err);
+		if ((err = netboot_setup()) != 0) {
+			panic("bsd_init: NetBoot could not find root, %d: %s", err, PE_boot_args());
 		}
 	}
 #endif
 	
 
-	microtime(&p->p_stats->p_start);
+#if CONFIG_IMAGEBOOT
+	/*
+	 * See if a system disk image is present. If so, mount it and
+	 * switch the root vnode to point to it
+	 */ 
+  
+	if(imageboot_needed()) {
+		int err;
+
+		/* An image was found */
+		if((err = imageboot_setup())) {
+			/*
+			 * this is not fatal. Keep trying to root
+			 * off the original media
+			 */
+			printf("%s: imageboot could not find root, %d\n",
+				__FUNCTION__, err);
+		}
+	}
+#endif /* CONFIG_IMAGEBOOT */
+  
+	microtime(&p->p_stats->p_start);	/* for compat sake */
+	microtime(&p->p_start);
 	p->p_rtime.tv_sec = p->p_rtime.tv_usec = 0;
 
 #if DEVFS
 	{
-	    extern void devfs_kernel_mount(char * str);
-	    
-	    devfs_kernel_mount("/dev");
+	    char mounthere[] = "/dev";	/* !const because of internal casting */
+
+	    bsd_init_kprintf("calling devfs_kernel_mount\n");
+	    devfs_kernel_mount(mounthere);
 	}
 #endif /* DEVFS */
 	
 	/* Initialize signal state for process 0. */
+	bsd_init_kprintf("calling siginit\n");
 	siginit(p);
 
+	bsd_init_kprintf("calling bsd_utaskbootstrap\n");
 	bsd_utaskbootstrap();
 
 #if __i386__
-	// this should be done after the root filesystem is mounted
+	/* this should be done after the root filesystem is mounted */
 	error = set_archhandler(p, CPU_TYPE_POWERPC);
-	if (error)
+	if (error) /* XXX make more generic */
 		exec_archhandler_ppc.path[0] = 0;
 #endif	
-	
+
+	bsd_init_kprintf("calling mountroot_post_hook\n");
+
 	/* invoke post-root-mount hook */
 	if (mountroot_post_hook != NULL)
 		mountroot_post_hook();
+
+#if 0 /* not yet */
+	IOKitJettisonKLD();
+	consider_zone_gc();
+#endif
 	
-	(void) thread_funnel_set(kernel_flock, funnel_state);
+	bsd_init_kprintf("done\n");
 }
 
 /* Called with kernel funnel held */
 void
 bsdinit_task(void)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
 	struct uthread *ut;
-	kern_return_t	kr;
-	thread_t th_act;
-	shared_region_mapping_t system_region;
+	thread_t thread;
 
 	process_name("init", p);
 
 	ux_handler_init();
 
-	th_act = current_thread();
+	thread = current_thread();
 	(void) host_set_exception_ports(host_priv_self(),
 					EXC_MASK_ALL & ~(EXC_MASK_SYSCALL |
 							 EXC_MASK_MACH_SYSCALL |
-							 EXC_MASK_RPC_ALERT),
-					ux_exception_port,
-					EXCEPTION_DEFAULT, 0);
-
-	(void) task_set_exception_ports(get_threadtask(th_act),
-					EXC_MASK_ALL & ~(EXC_MASK_SYSCALL |
-							 EXC_MASK_MACH_SYSCALL |
-							 EXC_MASK_RPC_ALERT),
-					ux_exception_port,
-					EXCEPTION_DEFAULT, 0);
-
+							 EXC_MASK_RPC_ALERT |
+							 EXC_MASK_CRASH),
+					(mach_port_t)ux_exception_port,
+					EXCEPTION_DEFAULT| MACH_EXCEPTION_CODES,
+					0);
 
+	ut = (uthread_t)get_bsdthread_info(thread);
 
-
-	ut = (uthread_t)get_bsdthread_info(th_act);
-
-	bsd_hardclockinit = 1;	/* Start bsd hardclock */
-	bsd_init_task = get_threadtask(th_act);
+	bsd_init_task = get_threadtask(thread);
 	init_task_failure_data[0] = 0;
-	system_region = lookup_default_shared_region(ENV_DEFAULT_ROOT, cpu_type());
-        if (system_region == NULL) {
-		shared_file_boot_time_init(ENV_DEFAULT_ROOT, cpu_type());
-	} else {
-		vm_set_shared_region(get_threadtask(th_act), system_region);
-	}
+
+#if CONFIG_MACF
+	mac_cred_label_associate_user(p->p_ucred);
+	mac_task_label_update_cred (p->p_ucred, (struct task *) p->task);
+#endif
 	load_init_program(p);
-	/* turn on app-profiling i.e. pre-heating */
-	app_profile = 1;
 	lock_trace = 1;
 }
 
 void
-lightning_bolt()
+lightning_bolt(__unused void *dummy)
 {			
 	boolean_t 	funnel_state;
-	extern void klogwakeup(void);
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
@@ -674,10 +928,10 @@ lightning_bolt()
 	(void) thread_funnel_set(kernel_flock, FALSE);
 }
 
-bsd_autoconf()
+kern_return_t
+bsd_autoconf(void)
 {
-	extern kern_return_t IOKitBSDInit( void );
-
+	kprintf("bsd_autoconf: calling kminit\n");
 	kminit();
 
 	/* 
@@ -699,8 +953,6 @@ bsd_autoconf()
 static void
 setconf(void)
 {	
-	extern kern_return_t IOFindBSDRoot( char * rootName,
-				dev_t * root, u_int32_t * flags );
 	u_int32_t	flags;
 	kern_return_t	err;
 
@@ -709,14 +961,12 @@ setconf(void)
 	 * which needs to be under network funnel. Right thing to do
 	 * here is to drop the funnel alltogether and regrab it afterwards
 	 */
-	thread_funnel_set(kernel_flock, FALSE);
 	err = IOFindBSDRoot( rootdevice, &rootdev, &flags );
-	thread_funnel_set(kernel_flock, TRUE);
 	if( err) {
 		printf("setconf: IOFindBSDRoot returned an error (%d);"
 			"setting rootdevice to 'sd0a'.\n", err); /* XXX DEBUG TEMP */
 		rootdev = makedev( 6, 0 );
-		strcpy( rootdevice, "sd0a" );
+		strlcpy(rootdevice, "sd0a", sizeof(rootdevice));
 		flags = 0;
 	}
 
@@ -734,89 +984,59 @@ setconf(void)
 
 }
 
-bsd_utaskbootstrap()
+void
+bsd_utaskbootstrap(void)
 {
-	thread_t th_act;
+	thread_t thread;
 	struct uthread *ut;
 
-	th_act = cloneproc(kernproc, 0);
-	initproc = pfind(1);				
+	thread = cloneproc(kernproc, 0);
+	/* Hold the reference as it will be dropped during shutdown */
+	initproc = proc_find(1);				
+#if __PROC_INTERNAL_DEBUG
+	if (initproc == PROC_NULL)
+		panic("bsd_utaskbootstrap: initproc not set\n");
+#endif
 	/* Set the launch time for init */
-	microtime(&initproc->p_stats->p_start);
+	microtime(&initproc->p_start);
+	microtime(&initproc->p_stats->p_start);	/* for compat sake */
+	
 
-	ut = (struct uthread *)get_bsdthread_info(th_act);
+	ut = (struct uthread *)get_bsdthread_info(thread);
 	ut->uu_sigmask = 0;
-	act_set_astbsd(th_act);
-	(void) thread_resume(th_act);
+	act_set_astbsd(thread);
+	(void) thread_resume(thread);
 }
 
-parse_bsd_args()
+static void
+parse_bsd_args(void)
 {
-	extern char init_args[];
-	char	namep[16];
-	extern int boothowto;
-	extern int srv;
-	extern int ncl;
+	char namep[16];
+	int msgbuf;
 
-	int len;
-
-	if (PE_parse_boot_arg("-s", namep)) {
+	if (PE_parse_boot_arg("-s", namep))
 		boothowto |= RB_SINGLE;
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -s");
-		else
-			strcat(init_args,"-s");
-	}
 
-	if (PE_parse_boot_arg("-b", namep)) {
+	if (PE_parse_boot_arg("-b", namep))
 		boothowto |= RB_NOBOOTRC;
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -b");
-		else
-			strcat(init_args,"-b");
-	}
 
-	if (PE_parse_boot_arg("-F", namep)) {
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -F");
-		else
-			strcat(init_args,"-F");
-	}
+	if (PE_parse_boot_arg("-x", namep)) /* safe boot */
+		boothowto |= RB_SAFEBOOT;
 
-	if (PE_parse_boot_arg("-v", namep)) {
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -v");
-		else
-			strcat(init_args,"-v");
-	}
-
-	if (PE_parse_boot_arg("-x", namep)) { /* safe boot */
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -x");
-		else
-			strcat(init_args,"-x");
-	}
-
-	if (PE_parse_boot_arg("-d", namep)) {
-		len = strlen(init_args);
-		if(len != 0)
-			strcat(init_args," -d");
-		else
-			strcat(init_args,"-d");
-	}
+	if (PE_parse_boot_arg("-l", namep)) /* leaks logging */
+		turn_on_log_leaks = 1;
 
 	PE_parse_boot_arg("srv", &srv);
 	PE_parse_boot_arg("ncl", &ncl);
-	if (PE_parse_boot_arg("nbuf", &max_nbuf_headers)) 
+	if (PE_parse_boot_arg("nbuf", &max_nbuf_headers)) {
 		customnbuf = 1;
+	}
 	PE_parse_boot_arg("kmem", &setup_kmem);
+	PE_parse_boot_arg("trace", &new_nkdbufs);
 
-	return 0;
+	if (PE_parse_boot_arg("msgbuf", &msgbuf)) {
+		log_setsize(msgbuf);
+	}
 }
 
 #if !NFSCLIENT
diff --git a/bsd/kern/bsd_stubs.c b/bsd/kern/bsd_stubs.c
index 0e885c2ae..657d9fde6 100644
--- a/bsd/kern/bsd_stubs.c
+++ b/bsd/kern/bsd_stubs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <sys/time.h>
 #include <kern/task.h>
@@ -34,6 +40,16 @@
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
 
+/* XXX these should be in a common header somwhere, but aren't */
+extern int chrtoblk_set(int, int);
+extern vm_offset_t kmem_mb_alloc(vm_map_t, int);
+
+/* XXX most of these just exist to export; there's no good header for them*/
+void	pcb_synch(void);
+int	issingleuser(void);
+void	tbeproc(void *);
+
+
 /* Just to satisfy pstat command */
 int     dmmin, dmmax, dmtext;
 
@@ -64,14 +80,14 @@ current_proc(void)
 	/* Never returns a NULL */
 	struct uthread * ut;
 	struct proc *p; 
-	thread_t thr_act = current_thread();
+	thread_t thread = current_thread();
 
-	ut = (struct uthread *)get_bsdthread_info(thr_act); 
+	ut = (struct uthread *)get_bsdthread_info(thread); 
 	if (ut &&  (ut->uu_flag & UT_VFORK) && ut->uu_proc) {
 		p = ut->uu_proc;
-		if ((p->p_flag & P_INVFORK) == 0) 
+		if ((p->p_lflag & P_LINVFORK) == 0) 
 			panic("returning child proc not under vfork");
-		if (p->p_vforkact != (void *)thr_act) 
+		if (p->p_vforkact != (void *)thread) 
 			panic("returning child proc which is not cur_act");
 		return(p);
 	}
@@ -86,8 +102,6 @@ current_proc(void)
 
 /* Device switch add delete routines */
 
-extern int nblkdev, nchrdev;
-
 struct bdevsw nobdev = NO_BDEVICE;
 struct cdevsw nocdev = NO_CDEVICE;
 /* 
@@ -205,15 +219,28 @@ cdevsw_isfree(int index)
  *	if index is -1, find a free slot to add
  *	  else see whether the slot is free
  *	return the major number that is used else -1
+ *
+ * NOTE:	In practice, -1 is unusable, since there are kernel internal
+ *		devices that call this function with absolute index values,
+ *		which will stomp on free-slot based assignments that happen
+ *		before them.  Therefore, if index is negative, we start
+ *		looking for a free slot at the absolute value of index,
+ *		instead of starting at 0 (lets out slot 1, but that's one
+ *		of the problem slots down low - the vndevice).  -12 is
+ *		currently a safe starting point.
  */
 int
 cdevsw_add(int index, struct cdevsw * csw) 
 {
 	struct cdevsw *devsw;
 
-	if (index == -1) {
-	    devsw = cdevsw;
-	    for(index=0; index < nchrdev; index++, devsw++) {
+	if (index < 0) {
+	    if (index == -1)
+	    	index = 0;	/* historical behaviour; XXX broken */
+	    else
+	        index = -index;	/* start at least this far up in the table */
+	    devsw = &cdevsw[index];
+	    for(; index < nchrdev; index++, devsw++) {
 		if(memcmp((char *)devsw, 
 			    (char *)&nocdev, 
 			    sizeof(struct cdevsw)) == 0)
@@ -253,8 +280,6 @@ cdevsw_remove(int index, struct cdevsw * csw)
 static int
 cdev_set_bdev(int cdev, int bdev)
 {
-	extern int chrtoblk_add(int cdev, int bdev);
-
 	return (chrtoblk_set(cdev, bdev));
 }
 
@@ -272,11 +297,20 @@ cdevsw_add_with_bdev(int index, struct cdevsw * csw, int bdev)
 	return (index);
 }
 
+#include <pexpert/pexpert.h>	/* for PE_parse_boot_arg */
+
+/*
+ * Notes:	This function is used solely by UFS, apparently in an effort
+ *		to work around an issue with single user mounts.
+ *
+ *		It's not technically correct to reference PE_parse_boot_arg()
+ *		from this file.
+ */
+int
 issingleuser(void)
 {
 	char namep[16];
 
-
 	if (PE_parse_boot_arg("-s", namep)) {
 		return(1);
 	} else {
@@ -284,76 +318,13 @@ issingleuser(void)
 	}
 }
 
-void *
+void
 tbeproc(void *procp)
 {
 	struct proc *p = procp;
 
 	if (p)
-		SET(p->p_flag, P_TBE);
+		OSBitOrAtomic(P_TBE, (UInt32 *)&p->p_flag);
 	return;
 }
 
-
-/* 
- * WARNING - this is a temporary workaround for binary compatibility issues
- * with anti-piracy software that relies on patching ptrace (3928003).
- * This KPI will be removed in the system release after Tiger.
- */
-uintptr_t temp_patch_ptrace(uintptr_t new_ptrace)
-{
-	struct sysent *		callp;
-	sy_call_t *			old_ptrace;
-#ifndef __ppc__
-	boolean_t	funnel_state;
-#endif
-
-	if (new_ptrace == 0)
-		return(0);
-		
-#ifdef __ppc__
-	enter_funnel_section(kernel_flock);
-#else
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-#endif
-	callp = &sysent[26];
-	old_ptrace = callp->sy_call;
-	
-	/* only allow one patcher of ptrace */
-	if (old_ptrace == (sy_call_t *) ptrace) {
-		callp->sy_call = (sy_call_t *) new_ptrace;
-	}
-	else {
-		old_ptrace = NULL;
-	}
-#ifdef __ppc__
-	exit_funnel_section( );
-#else
-	(void)thread_funnel_set(kernel_flock, funnel_state);
-#endif
-	
-	return((uintptr_t)old_ptrace);
-}
-
-void temp_unpatch_ptrace(void)
-{
-	struct sysent *		callp;
-#ifndef __ppc__
-	boolean_t	funnel_state;
-#endif
-		
-#ifdef __ppc__
-	enter_funnel_section(kernel_flock);
-#else
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-#endif
-	callp = &sysent[26];
-	callp->sy_call = (sy_call_t *) ptrace;
-#ifdef __ppc__
-	exit_funnel_section( );
-#else
-	(void)thread_funnel_set(kernel_flock, funnel_state);
-#endif
-	
-	return;
-}
diff --git a/bsd/kern/imageboot.c b/bsd/kern/imageboot.c
new file mode 100644
index 000000000..8af10ca2c
--- /dev/null
+++ b/bsd/kern/imageboot.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/proc_internal.h>
+#include <sys/systm.h>
+#include <sys/systm.h>
+#include <sys/mount_internal.h>
+#include <sys/filedesc.h>
+#include <sys/vnode_internal.h>
+#include <sys/imageboot.h>
+
+#include <pexpert/pexpert.h>
+
+extern struct filedesc filedesc0;
+
+extern int (*mountroot)(void);
+extern char rootdevice[];
+
+#define DEBUG_IMAGEBOOT 0
+
+#if DEBUG_IMAGEBOOT
+#define DBG_TRACE(...) printf(__VA_ARGS__)
+#else
+#define DBG_TRACE(...) do {} while(0)
+#endif
+
+extern int di_root_image(const char *path, char devname[], dev_t *dev_p);
+
+#define kIBFilePrefix "file://"
+
+int
+imageboot_needed(void)
+{
+	int result = 0;
+	char *root_path = NULL;
+
+	DBG_TRACE("%s: checking for presence of root path\n", __FUNCTION__);
+
+	MALLOC_ZONE(root_path, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
+	if (root_path == NULL)
+		panic("%s: M_NAMEI zone exhausted", __FUNCTION__);
+
+	if(PE_parse_boot_arg("rp", root_path) == TRUE) {
+		/* Got it, now verify scheme */
+
+		if (strncmp(root_path, kIBFilePrefix,
+					strlen(kIBFilePrefix)) == 0) {
+			DBG_TRACE("%s: Found %s\n", __FUNCTION__, root_path);
+			result = 1;
+		} else {
+			DBG_TRACE("%s: Invalid URL scheme for %s\n",
+					__FUNCTION__, root_path);
+		}
+	}
+	FREE_ZONE(root_path, MAXPATHLEN, M_NAMEI);
+
+	return (result);
+}
+
+
+/*
+ * We know there's an image. Attach it, and
+ * switch over to root off it
+ *
+ * NB: p is always kernproc
+ */
+
+int
+imageboot_setup()
+{
+	dev_t       dev;
+	int         error = 0;
+	char *root_path = NULL;
+
+	DBG_TRACE("%s: entry\n", __FUNCTION__);
+
+	MALLOC_ZONE(root_path, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
+	if (root_path == NULL)
+		return (ENOMEM);
+
+	if(PE_parse_boot_arg("rp", root_path) == FALSE) {
+		error = ENOENT;
+		goto done;
+	}
+
+	printf("%s: root image url is %s\n", __FUNCTION__, root_path);
+	error = di_root_image(root_path, rootdevice, &dev);
+	if(error) {
+		printf("%s: di_root_image failed: %d\n", __FUNCTION__, error);
+		goto done;
+	}
+
+	rootdev = dev;
+	mountroot = NULL;
+	printf("%s: root device 0x%x\n", __FUNCTION__, rootdev);
+	error = vfs_mountroot();
+
+	if (error == 0 && rootvnode != NULL) {
+		struct vnode *tvp;
+		struct vnode *newdp;
+
+		/*
+		 * Get the vnode for '/'.
+		 * Set fdp->fd_fd.fd_cdir to reference it.
+		 */
+		if (VFS_ROOT(TAILQ_LAST(&mountlist,mntlist), &newdp, vfs_context_kernel()))
+			panic("%s: cannot find root vnode", __FUNCTION__);
+
+		vnode_ref(newdp);
+		vnode_put(newdp);
+		tvp = rootvnode;
+		vnode_rele(tvp);
+		filedesc0.fd_cdir = newdp;
+		rootvnode = newdp;
+		mount_list_lock();
+		TAILQ_REMOVE(&mountlist, TAILQ_FIRST(&mountlist), mnt_list);
+		mount_list_unlock();
+		mountlist.tqh_first->mnt_flag |= MNT_ROOTFS;
+		DBG_TRACE("%s: root switched\n", __FUNCTION__);
+	}
+done:
+	FREE_ZONE(root_path, MAXPATHLEN, M_NAMEI);
+
+	DBG_TRACE("%s: exit\n", __FUNCTION__);
+
+	return (error);
+}
diff --git a/bsd/kern/init_sysent.c b/bsd/kern/init_sysent.c
index f4efae621..1f421780e 100644
--- a/bsd/kern/init_sysent.c
+++ b/bsd/kern/init_sysent.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  * 
- * @APPLE_LICENSE_HEADER_START@ 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and 
- * are subject to the Apple Public Source License Version 1.1 (the 
- * "License").  You may not use this file except in compliance with the 
- * License.  Please obtain a copy of the License at 
- * http://www.apple.com/publicsource and read it before using this file. 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are 
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the 
- * License for the specific language governing rights and limitations 
- * under the License. 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * @APPLE_LICENSE_HEADER_END@ 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  * 
  * 
  * System call switch table.
@@ -26,411 +32,582 @@
  * created from syscalls.master
  */
 
+#define __INIT_SYSENT_C__ 1
 #include <sys/appleapiopts.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/types.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
-#define AC(name) (sizeof(struct name) / sizeof(uint64_t))
+#define AC(name) (sizeof(struct name) / sizeof(syscall_arg_t))
 
 
 /* The casts are bogus but will do for now. */
 __private_extern__ struct sysent sysent[] = {
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 0 = nosys indirect syscall */
-	{AC(exit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)exit, munge_w, munge_d, _SYSCALL_RET_NONE}, /* 1 = exit */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)fork, NULL, NULL, _SYSCALL_RET_INT_T}, /* 2 = fork */
-	{AC(read_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)read, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T}, /* 3 = read */
-	{AC(write_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)write, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T}, /* 4 = write */
-	{AC(open_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)open, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 5 = open */
-	{AC(close_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)close, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 6 = close */
-	{AC(wait4_args), _SYSCALL_CANCEL_PRE, KERNEL_FUNNEL, (sy_call_t *)wait4, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 7 = wait4 */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 8 = nosys old creat */
-	{AC(link_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)link, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 9 = link */
-	{AC(unlink_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)unlink, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 10 = unlink */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 11 = nosys old execv */
-	{AC(chdir_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)chdir, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 12 = chdir */
-	{AC(fchdir_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fchdir, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 13 = fchdir */
-	{AC(mknod_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mknod, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 14 = mknod */
-	{AC(chmod_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)chmod, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 15 = chmod */
-	{AC(chown_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)chown, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 16 = chown */
-	{AC(obreak_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)obreak, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 17 = obreak old break */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 0 = nosys indirect syscall */
+	{AC(exit_args), 0, 0, (sy_call_t *)exit, munge_w, munge_d, _SYSCALL_RET_NONE, 4},           /* 1 = exit */
+	{0, 0, 0, (sy_call_t *)fork, NULL, NULL, _SYSCALL_RET_INT_T, 0},                            /* 2 = fork */
+	{AC(read_args), 0, 0, (sy_call_t *)read, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12},    /* 3 = read */
+	{AC(write_args), 0, 0, (sy_call_t *)write, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12},  /* 4 = write */
+	{AC(open_args), 0, 0, (sy_call_t *)open, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},      /* 5 = open */
+	{AC(close_args), 0, 0, (sy_call_t *)close, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 6 = close */
+	{AC(wait4_args), 0, 0, (sy_call_t *)wait4, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16},  /* 7 = wait4 */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 8 = nosys old creat */
+	{AC(link_args), 0, 0, (sy_call_t *)link, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},        /* 9 = link */
+	{AC(unlink_args), 0, 0, (sy_call_t *)unlink, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 10 = unlink */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 11 = nosys old execv */
+	{AC(chdir_args), 0, 0, (sy_call_t *)chdir, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 12 = chdir */
+	{AC(fchdir_args), 0, 0, (sy_call_t *)fchdir, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 13 = fchdir */
+	{AC(mknod_args), 0, 0, (sy_call_t *)mknod, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 14 = mknod */
+	{AC(chmod_args), 0, 0, (sy_call_t *)chmod, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 15 = chmod */
+	{AC(chown_args), 0, 0, (sy_call_t *)chown, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 16 = chown */
+	{AC(obreak_args), 0, UNSAFE_64BIT, (sy_call_t *)obreak, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 17 = obreak old break */
 #if COMPAT_GETFSSTAT
-	{AC(ogetfsstat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ogetfsstat, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 18 = ogetfsstat */
+	{AC(ogetfsstat_args), 0, 0, (sy_call_t *)ogetfsstat, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 18 = ogetfsstat */
 #else
-	{AC(getfsstat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getfsstat, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 18 = getfsstat */
+	{AC(getfsstat_args), 0, 0, (sy_call_t *)getfsstat, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 18 = getfsstat */
 #endif
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 19 = nosys old lseek */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getpid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 20 = getpid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 21 = nosys old mount */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 22 = nosys old umount */
-	{AC(setuid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setuid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 23 = setuid */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getuid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 24 = getuid */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)geteuid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 25 = geteuid */
-	{AC(ptrace_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)ptrace, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 26 = ptrace */
-	{AC(recvmsg_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)recvmsg, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 27 = recvmsg */
-	{AC(sendmsg_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)sendmsg, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 28 = sendmsg */
-	{AC(recvfrom_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)recvfrom, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 29 = recvfrom */
-	{AC(accept_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)accept, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 30 = accept */
-	{AC(getpeername_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getpeername, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 31 = getpeername */
-	{AC(getsockname_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getsockname, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 32 = getsockname */
-	{AC(access_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)access, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 33 = access */
-	{AC(chflags_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)chflags, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 34 = chflags */
-	{AC(fchflags_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fchflags, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 35 = fchflags */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sync, NULL, NULL, _SYSCALL_RET_INT_T}, /* 36 = sync */
-	{AC(kill_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)kill, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 37 = kill */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 38 = nosys old stat */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getppid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 39 = getppid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 40 = nosys old lstat */
-	{AC(dup_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)dup, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 41 = dup */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)pipe, NULL, NULL, _SYSCALL_RET_INT_T}, /* 42 = pipe */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getegid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 43 = getegid */
-	{AC(profil_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)profil, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 44 = profil */
-	{AC(ktrace_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)ktrace, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 45 = ktrace */
-	{AC(sigaction_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigaction, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 46 = sigaction */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getgid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 47 = getgid */
-	{AC(sigprocmask_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigprocmask, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 48 = sigprocmask */
-	{AC(getlogin_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getlogin, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 49 = getlogin */
-	{AC(setlogin_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setlogin, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 50 = setlogin */
-	{AC(acct_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)acct, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 51 = acct */
-	{AC(sigpending_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigpending, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 52 = sigpending */
-	{AC(sigaltstack_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigaltstack, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 53 = sigaltstack */
-	{AC(ioctl_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ioctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 54 = ioctl */
-	{AC(reboot_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)reboot, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 55 = reboot */
-	{AC(revoke_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)revoke, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 56 = revoke */
-	{AC(symlink_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)symlink, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 57 = symlink */
-	{AC(readlink_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)readlink, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 58 = readlink */
-	{AC(execve_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)execve, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 59 = execve */
-	{AC(umask_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)umask, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 60 = umask */
-	{AC(chroot_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)chroot, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 61 = chroot */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 62 = nosys old fstat */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 63 = nosys used internally , reserved */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 64 = nosys old getpagesize */
-	{AC(msync_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)msync, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 65 = msync */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)vfork, NULL, NULL, _SYSCALL_RET_INT_T}, /* 66 = vfork */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 67 = nosys old vread */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 68 = nosys old vwrite */
-	{AC(sbrk_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sbrk, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 69 = sbrk */
-	{AC(sstk_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sstk, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 70 = sstk */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 71 = nosys old mmap */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ovadvise, NULL, NULL, _SYSCALL_RET_INT_T}, /* 72 = ovadvise old vadvise */
-	{AC(munmap_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)munmap, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 73 = munmap */
-	{AC(mprotect_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mprotect, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 74 = mprotect */
-	{AC(madvise_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)madvise, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 75 = madvise */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 76 = nosys old vhangup */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 77 = nosys old vlimit */
-	{AC(mincore_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mincore, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 78 = mincore */
-	{AC(getgroups_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 79 = getgroups */
-	{AC(setgroups_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 80 = setgroups */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getpgrp, NULL, NULL, _SYSCALL_RET_INT_T}, /* 81 = getpgrp */
-	{AC(setpgid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setpgid, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 82 = setpgid */
-	{AC(setitimer_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setitimer, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 83 = setitimer */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 84 = nosys old wait */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)swapon, NULL, NULL, _SYSCALL_RET_INT_T}, /* 85 = swapon */
-	{AC(getitimer_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getitimer, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 86 = getitimer */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 87 = nosys old gethostname */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 88 = nosys old sethostname */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getdtablesize, NULL, NULL, _SYSCALL_RET_INT_T}, /* 89 = getdtablesize */
-	{AC(dup2_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)dup2, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 90 = dup2 */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 91 = nosys old getdopt */
-	{AC(fcntl_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)fcntl, munge_wws, munge_ddd, _SYSCALL_RET_INT_T}, /* 92 = fcntl */
-	{AC(select_args), _SYSCALL_CANCEL_PRE, KERNEL_FUNNEL, (sy_call_t *)select, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 93 = select */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 94 = nosys old setdopt */
-	{AC(fsync_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)fsync, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 95 = fsync */
-	{AC(setpriority_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setpriority, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 96 = setpriority */
-	{AC(socket_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)socket, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 97 = socket */
-	{AC(connect_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)connect, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 98 = connect */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 99 = nosys old accept */
-	{AC(getpriority_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getpriority, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 100 = getpriority */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 101 = nosys old send */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 102 = nosys old recv */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 103 = nosys old sigreturn */
-	{AC(bind_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)bind, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 104 = bind */
-	{AC(setsockopt_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 105 = setsockopt */
-	{AC(listen_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)listen, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 106 = listen */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 107 = nosys old vtimes */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 108 = nosys old sigvec */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 109 = nosys old sigblock */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 110 = nosys old sigsetmask */
-	{AC(sigsuspend_args), _SYSCALL_CANCEL_PRE, KERNEL_FUNNEL, (sy_call_t *)sigsuspend, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 111 = sigsuspend */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 112 = nosys old sigstack */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 113 = nosys old recvmsg */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 114 = nosys old sendmsg */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 115 = nosys old vtrace */
-	{AC(gettimeofday_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)gettimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 116 = gettimeofday */
-	{AC(getrusage_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getrusage, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 117 = getrusage */
-	{AC(getsockopt_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 118 = getsockopt */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 119 = nosys old resuba */
-	{AC(readv_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)readv, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T}, /* 120 = readv */
-	{AC(writev_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)writev, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T}, /* 121 = writev */
-	{AC(settimeofday_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)settimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 122 = settimeofday */
-	{AC(fchown_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fchown, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 123 = fchown */
-	{AC(fchmod_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fchmod, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 124 = fchmod */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 125 = nosys old recvfrom */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 126 = nosys old setreuid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 127 = nosys old setregid */
-	{AC(rename_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)rename, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 128 = rename */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 129 = nosys old truncate */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 130 = nosys old ftruncate */
-	{AC(flock_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)flock, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 131 = flock */
-	{AC(mkfifo_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mkfifo, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 132 = mkfifo */
-	{AC(sendto_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)sendto, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 133 = sendto */
-	{AC(shutdown_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shutdown, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 134 = shutdown */
-	{AC(socketpair_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)socketpair, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 135 = socketpair */
-	{AC(mkdir_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mkdir, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 136 = mkdir */
-	{AC(rmdir_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)rmdir, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 137 = rmdir */
-	{AC(utimes_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)utimes, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 138 = utimes */
-	{AC(futimes_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)futimes, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 139 = futimes */
-	{AC(adjtime_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)adjtime, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 140 = adjtime */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 141 = nosys old getpeername */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 142 = nosys old gethostid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 143 = nosys old sethostid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 144 = nosys old getrlimit */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 145 = nosys old setrlimit */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 146 = nosys old killpg */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setsid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 147 = setsid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 148 = nosys old setquota */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 149 = nosys old qquota */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 150 = nosys old getsockname */
-	{AC(getpgid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getpgid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 151 = getpgid */
-	{AC(setprivexec_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setprivexec, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 152 = setprivexec */
-	{AC(pread_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)pread, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T}, /* 153 = pread */
-	{AC(pwrite_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)pwrite, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T}, /* 154 = pwrite */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 19 = nosys old lseek */
+	{0, 0, 0, (sy_call_t *)getpid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 20 = getpid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 21 = nosys old mount */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 22 = nosys old umount */
+	{AC(setuid_args), 0, 0, (sy_call_t *)setuid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 23 = setuid */
+	{0, 0, 0, (sy_call_t *)getuid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 24 = getuid */
+	{0, 0, 0, (sy_call_t *)geteuid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                         /* 25 = geteuid */
+	{AC(ptrace_args), 0, 0, (sy_call_t *)ptrace, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 26 = ptrace */
+#if SOCKETS
+	{AC(recvmsg_args), 0, 0, (sy_call_t *)recvmsg, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 27 = recvmsg */
+	{AC(sendmsg_args), 0, 0, (sy_call_t *)sendmsg, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 28 = sendmsg */
+	{AC(recvfrom_args), 0, 0, (sy_call_t *)recvfrom, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 29 = recvfrom */
+	{AC(accept_args), 0, 0, (sy_call_t *)accept, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 30 = accept */
+	{AC(getpeername_args), 0, 0, (sy_call_t *)getpeername, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 31 = getpeername */
+	{AC(getsockname_args), 0, 0, (sy_call_t *)getsockname, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 32 = getsockname */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 27 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 28 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 29 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 30 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 31 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 32 = nosys */
+#endif /* SOCKETS */
+	{AC(access_args), 0, 0, (sy_call_t *)access, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 33 = access */
+	{AC(chflags_args), 0, 0, (sy_call_t *)chflags, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 34 = chflags */
+	{AC(fchflags_args), 0, 0, (sy_call_t *)fchflags, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 35 = fchflags */
+	{0, 0, 0, (sy_call_t *)sync, NULL, NULL, _SYSCALL_RET_INT_T, 0},                            /* 36 = sync */
+	{AC(kill_args), 0, 0, (sy_call_t *)kill, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},      /* 37 = kill */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 38 = nosys old stat */
+	{0, 0, 0, (sy_call_t *)getppid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                         /* 39 = getppid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 40 = nosys old lstat */
+	{AC(dup_args), 0, 0, (sy_call_t *)dup, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},            /* 41 = dup */
+	{0, 0, 0, (sy_call_t *)pipe, NULL, NULL, _SYSCALL_RET_INT_T, 0},                            /* 42 = pipe */
+	{0, 0, 0, (sy_call_t *)getegid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                         /* 43 = getegid */
+	{AC(profil_args), 0, 0, (sy_call_t *)profil, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 44 = profil */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 45 = nosys old ktrace */
+	{AC(sigaction_args), 0, 0, (sy_call_t *)sigaction, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 46 = sigaction */
+	{0, 0, 0, (sy_call_t *)getgid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 47 = getgid */
+	{AC(sigprocmask_args), 0, 0, (sy_call_t *)sigprocmask, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 48 = sigprocmask */
+	{AC(getlogin_args), 0, 0, (sy_call_t *)getlogin, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 49 = getlogin */
+	{AC(setlogin_args), 0, 0, (sy_call_t *)setlogin, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 50 = setlogin */
+	{AC(acct_args), 0, 0, (sy_call_t *)acct, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},          /* 51 = acct */
+	{AC(sigpending_args), 0, 0, (sy_call_t *)sigpending, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 52 = sigpending */
+	{AC(sigaltstack_args), 0, 0, (sy_call_t *)sigaltstack, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 53 = sigaltstack */
+	{AC(ioctl_args), 0, 0, (sy_call_t *)ioctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 54 = ioctl */
+	{AC(reboot_args), 0, 0, (sy_call_t *)reboot, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 55 = reboot */
+	{AC(revoke_args), 0, 0, (sy_call_t *)revoke, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 56 = revoke */
+	{AC(symlink_args), 0, 0, (sy_call_t *)symlink, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 57 = symlink */
+	{AC(readlink_args), 0, 0, (sy_call_t *)readlink, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 58 = readlink */
+	{AC(execve_args), 0, 0, (sy_call_t *)execve, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 59 = execve */
+	{AC(umask_args), 0, 0, (sy_call_t *)umask, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 60 = umask */
+	{AC(chroot_args), 0, 0, (sy_call_t *)chroot, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 61 = chroot */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 62 = nosys old fstat */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 63 = nosys used internally , reserved */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 64 = nosys old getpagesize */
+	{AC(msync_args), 0, 0, (sy_call_t *)msync, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 65 = msync */
+	{0, 0, 0, (sy_call_t *)vfork, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 66 = vfork */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 67 = nosys old vread */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 68 = nosys old vwrite */
+	{AC(sbrk_args), 0, 0, (sy_call_t *)sbrk, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},          /* 69 = sbrk */
+	{AC(sstk_args), 0, 0, (sy_call_t *)sstk, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},          /* 70 = sstk */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 71 = nosys old mmap */
+	{0, 0, 0, (sy_call_t *)ovadvise, NULL, NULL, _SYSCALL_RET_INT_T, 0},                        /* 72 = ovadvise old vadvise */
+	{AC(munmap_args), 0, 0, (sy_call_t *)munmap, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 73 = munmap */
+	{AC(mprotect_args), 0, 0, (sy_call_t *)mprotect, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 74 = mprotect */
+	{AC(madvise_args), 0, 0, (sy_call_t *)madvise, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 75 = madvise */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 76 = nosys old vhangup */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 77 = nosys old vlimit */
+	{AC(mincore_args), 0, 0, (sy_call_t *)mincore, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 78 = mincore */
+	{AC(getgroups_args), 0, 0, (sy_call_t *)getgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 79 = getgroups */
+	{AC(setgroups_args), 0, 0, (sy_call_t *)setgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 80 = setgroups */
+	{0, 0, 0, (sy_call_t *)getpgrp, NULL, NULL, _SYSCALL_RET_INT_T, 0},                         /* 81 = getpgrp */
+	{AC(setpgid_args), 0, 0, (sy_call_t *)setpgid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 82 = setpgid */
+	{AC(setitimer_args), 0, 0, (sy_call_t *)setitimer, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 83 = setitimer */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 84 = nosys old wait */
+	{0, 0, 0, (sy_call_t *)swapon, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 85 = swapon */
+	{AC(getitimer_args), 0, 0, (sy_call_t *)getitimer, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 86 = getitimer */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 87 = nosys old gethostname */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 88 = nosys old sethostname */
+	{0, 0, 0, (sy_call_t *)getdtablesize, NULL, NULL, _SYSCALL_RET_INT_T, 0},                   /* 89 = getdtablesize */
+	{AC(dup2_args), 0, 0, (sy_call_t *)dup2, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},        /* 90 = dup2 */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 91 = nosys old getdopt */
+	{AC(fcntl_args), 0, 0, (sy_call_t *)fcntl, munge_wws, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 92 = fcntl */
+	{AC(select_args), 0, 0, (sy_call_t *)select, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 93 = select */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 94 = nosys old setdopt */
+	{AC(fsync_args), 0, 0, (sy_call_t *)fsync, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 95 = fsync */
+	{AC(setpriority_args), 0, 0, (sy_call_t *)setpriority, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 96 = setpriority */
+#if SOCKETS
+	{AC(socket_args), 0, 0, (sy_call_t *)socket, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 97 = socket */
+	{AC(connect_args), 0, 0, (sy_call_t *)connect, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 98 = connect */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 97 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 98 = nosys */
+#endif /* SOCKETS */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 99 = nosys old accept */
+	{AC(getpriority_args), 0, 0, (sy_call_t *)getpriority, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 100 = getpriority */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 101 = nosys old send */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 102 = nosys old recv */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 103 = nosys old sigreturn */
+#if SOCKETS
+	{AC(bind_args), 0, 0, (sy_call_t *)bind, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},      /* 104 = bind */
+	{AC(setsockopt_args), 0, 0, (sy_call_t *)setsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 105 = setsockopt */
+	{AC(listen_args), 0, 0, (sy_call_t *)listen, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 106 = listen */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 104 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 105 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 106 = nosys */
+#endif /* SOCKETS */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 107 = nosys old vtimes */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 108 = nosys old sigvec */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 109 = nosys old sigblock */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 110 = nosys old sigsetmask */
+	{AC(sigsuspend_args), 0, 0, (sy_call_t *)sigsuspend, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 111 = sigsuspend */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 112 = nosys old sigstack */
+#if SOCKETS
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 113 = nosys old recvmsg */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 114 = nosys old sendmsg */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 113 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 114 = nosys */
+#endif /* SOCKETS */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 115 = nosys old vtrace */
+	{AC(gettimeofday_args), 0, 0, (sy_call_t *)gettimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 116 = gettimeofday */
+	{AC(getrusage_args), 0, 0, (sy_call_t *)getrusage, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 117 = getrusage */
+#if SOCKETS
+	{AC(getsockopt_args), 0, 0, (sy_call_t *)getsockopt, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 118 = getsockopt */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 118 = nosys */
+#endif /* SOCKETS */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 119 = nosys old resuba */
+	{AC(readv_args), 0, 0, (sy_call_t *)readv, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12},  /* 120 = readv */
+	{AC(writev_args), 0, 0, (sy_call_t *)writev, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12}, /* 121 = writev */
+	{AC(settimeofday_args), 0, 0, (sy_call_t *)settimeofday, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 122 = settimeofday */
+	{AC(fchown_args), 0, 0, (sy_call_t *)fchown, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 123 = fchown */
+	{AC(fchmod_args), 0, 0, (sy_call_t *)fchmod, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 124 = fchmod */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 125 = nosys old recvfrom */
+	{AC(setreuid_args), 0, 0, (sy_call_t *)setreuid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 126 = setreuid */
+	{AC(setregid_args), 0, 0, (sy_call_t *)setregid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 127 = setregid */
+	{AC(rename_args), 0, 0, (sy_call_t *)rename, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 128 = rename */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 129 = nosys old truncate */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 130 = nosys old ftruncate */
+	{AC(flock_args), 0, 0, (sy_call_t *)flock, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 131 = flock */
+	{AC(mkfifo_args), 0, 0, (sy_call_t *)mkfifo, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 132 = mkfifo */
+#if SOCKETS
+	{AC(sendto_args), 0, 0, (sy_call_t *)sendto, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 133 = sendto */
+	{AC(shutdown_args), 0, 0, (sy_call_t *)shutdown, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 134 = shutdown */
+	{AC(socketpair_args), 0, 0, (sy_call_t *)socketpair, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 135 = socketpair */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 133 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 134 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 135 = nosys */
+#endif /* SOCKETS */
+	{AC(mkdir_args), 0, 0, (sy_call_t *)mkdir, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 136 = mkdir */
+	{AC(rmdir_args), 0, 0, (sy_call_t *)rmdir, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 137 = rmdir */
+	{AC(utimes_args), 0, 0, (sy_call_t *)utimes, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 138 = utimes */
+	{AC(futimes_args), 0, 0, (sy_call_t *)futimes, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 139 = futimes */
+	{AC(adjtime_args), 0, 0, (sy_call_t *)adjtime, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 140 = adjtime */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 141 = nosys old getpeername */
+	{AC(gethostuuid_args), 0, 0, (sy_call_t *)gethostuuid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 142 = gethostuuid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 143 = nosys old sethostid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 144 = nosys old getrlimit */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 145 = nosys old setrlimit */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 146 = nosys old killpg */
+	{0, 0, 0, (sy_call_t *)setsid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 147 = setsid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 148 = nosys old setquota */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 149 = nosys old qquota */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 150 = nosys old getsockname */
+	{AC(getpgid_args), 0, 0, (sy_call_t *)getpgid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 151 = getpgid */
+	{AC(setprivexec_args), 0, 0, (sy_call_t *)setprivexec, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 152 = setprivexec */
+	{AC(pread_args), 0, 0, (sy_call_t *)pread, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T, 20}, /* 153 = pread */
+	{AC(pwrite_args), 0, 0, (sy_call_t *)pwrite, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T, 20}, /* 154 = pwrite */
 #if NFSSERVER
-	{AC(nfssvc_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)nfssvc, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 155 = nfssvc */
+	{AC(nfssvc_args), 0, 0, (sy_call_t *)nfssvc, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 155 = nfssvc */
 #else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 155 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 155 = nosys */
 #endif
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 156 = nosys old getdirentries */
-	{AC(statfs_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)statfs, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 157 = statfs */
-	{AC(fstatfs_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fstatfs, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 158 = fstatfs */
-	{AC(unmount_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)unmount, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 159 = unmount */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 160 = nosys old async_daemon */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 156 = nosys old getdirentries */
+	{AC(statfs_args), 0, 0, (sy_call_t *)statfs, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 157 = statfs */
+	{AC(fstatfs_args), 0, 0, (sy_call_t *)fstatfs, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 158 = fstatfs */
+	{AC(unmount_args), 0, 0, (sy_call_t *)unmount, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 159 = unmount */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 160 = nosys old async_daemon */
+#if NFSSERVER
+	{AC(getfh_args), 0, 0, (sy_call_t *)getfh, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 161 = getfh */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 161 = nosys */
+#endif
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 162 = nosys old getdomainname */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 163 = nosys old setdomainname */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 164 = nosys */
+	{AC(quotactl_args), 0, 0, (sy_call_t *)quotactl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 165 = quotactl */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 166 = nosys old exportfs */
+	{AC(mount_args), 0, 0, (sy_call_t *)mount, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16},  /* 167 = mount */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 168 = nosys old ustat */
+	{AC(csops_args), 0, 0, (sy_call_t *)csops, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16},  /* 169 = csops */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE, 0},                            /* 170 = table old table */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 171 = nosys old wait3 */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 172 = nosys old rpause */
+	{AC(waitid_args), 0, 0, (sy_call_t *)waitid, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 173 = waitid */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 174 = nosys old getdents */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 175 = nosys old gc_control */
+	{AC(add_profil_args), 0, 0, (sy_call_t *)add_profil, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 176 = add_profil */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 177 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 178 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 179 = nosys */
+	{AC(kdebug_trace_args), 0, 0, (sy_call_t *)kdebug_trace, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 180 = kdebug_trace */
+	{AC(setgid_args), 0, 0, (sy_call_t *)setgid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 181 = setgid */
+	{AC(setegid_args), 0, 0, (sy_call_t *)setegid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 182 = setegid */
+	{AC(seteuid_args), 0, 0, (sy_call_t *)seteuid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 183 = seteuid */
+	{AC(sigreturn_args), 0, 0, (sy_call_t *)sigreturn, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 184 = sigreturn */
+	{AC(chud_args), 0, UNSAFE_64BIT, (sy_call_t *)chud, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 185 = chud */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 186 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 187 = nosys */
+	{AC(stat_args), 0, 0, (sy_call_t *)stat, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},        /* 188 = stat */
+	{AC(fstat_args), 0, 0, (sy_call_t *)fstat, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 189 = fstat */
+	{AC(lstat_args), 0, 0, (sy_call_t *)lstat, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 190 = lstat */
+	{AC(pathconf_args), 0, 0, (sy_call_t *)pathconf, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 191 = pathconf */
+	{AC(fpathconf_args), 0, 0, (sy_call_t *)fpathconf, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 192 = fpathconf */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 193 = nosys */
+	{AC(getrlimit_args), 0, 0, (sy_call_t *)getrlimit, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 194 = getrlimit */
+	{AC(setrlimit_args), 0, 0, (sy_call_t *)setrlimit, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 195 = setrlimit */
+	{AC(getdirentries_args), 0, 0, (sy_call_t *)getdirentries, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 196 = getdirentries */
+	{AC(mmap_args), 0, 0, (sy_call_t *)mmap, munge_wwwwwl, munge_dddddd, _SYSCALL_RET_ADDR_T, 28}, /* 197 = mmap */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 198 = nosys __syscall */
+	{AC(lseek_args), 0, 0, (sy_call_t *)lseek, munge_wlw, munge_ddd, _SYSCALL_RET_OFF_T, 16},    /* 199 = lseek */
+	{AC(truncate_args), 0, 0, (sy_call_t *)truncate, munge_wl, munge_dd, _SYSCALL_RET_INT_T, 12}, /* 200 = truncate */
+	{AC(ftruncate_args), 0, 0, (sy_call_t *)ftruncate, munge_wl, munge_dd, _SYSCALL_RET_INT_T, 12}, /* 201 = ftruncate */
+	{AC(__sysctl_args), 0, 0, (sy_call_t *)__sysctl, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 202 = __sysctl */
+	{AC(mlock_args), 0, 0, (sy_call_t *)mlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 203 = mlock */
+	{AC(munlock_args), 0, 0, (sy_call_t *)munlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 204 = munlock */
+	{AC(undelete_args), 0, 0, (sy_call_t *)undelete, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 205 = undelete */
+#if NETAT
+	{AC(ATsocket_args), 0, 0, (sy_call_t *)ATsocket, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 206 = ATsocket */
+	{AC(ATgetmsg_args), 0, UNSAFE_64BIT, (sy_call_t *)ATgetmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 207 = ATgetmsg */
+	{AC(ATputmsg_args), 0, UNSAFE_64BIT, (sy_call_t *)ATputmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 208 = ATputmsg */
+	{AC(ATPsndreq_args), 0, UNSAFE_64BIT, (sy_call_t *)ATPsndreq, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 209 = ATPsndreq */
+	{AC(ATPsndrsp_args), 0, UNSAFE_64BIT, (sy_call_t *)ATPsndrsp, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 210 = ATPsndrsp */
+	{AC(ATPgetreq_args), 0, UNSAFE_64BIT, (sy_call_t *)ATPgetreq, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 211 = ATPgetreq */
+	{AC(ATPgetrsp_args), 0, UNSAFE_64BIT, (sy_call_t *)ATPgetrsp, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 212 = ATPgetrsp */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 213 = nosys Reserved for AppleTalk */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 206 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 207 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 208 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 209 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 210 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 211 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 212 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 213 = nosys Reserved for AppleTalk */
+#endif /* NETAT */
+	{AC(kqueue_from_portset_np_args), 0, 0, (sy_call_t *)kqueue_from_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 214 = kqueue_from_portset_np */
+	{AC(kqueue_portset_np_args), 0, 0, (sy_call_t *)kqueue_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 215 = kqueue_portset_np */
+	{0, 0, UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE, 12},                 /* 216 = mkcomplex soon to be obsolete */
+	{0, 0, UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE, 8},                 /* 217 = statv soon to be obsolete */
+	{0, 0, UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE, 8},                 /* 218 = lstatv soon to be obsolete */
+	{0, 0, UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE, 8},                 /* 219 = fstatv soon to be obsolete */
+	{AC(getattrlist_args), 0, 0, (sy_call_t *)getattrlist, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 220 = getattrlist */
+	{AC(setattrlist_args), 0, 0, (sy_call_t *)setattrlist, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 221 = setattrlist */
+	{AC(getdirentriesattr_args), 0, 0, (sy_call_t *)getdirentriesattr, munge_wwwwwwww, munge_dddddddd, _SYSCALL_RET_INT_T, 32}, /* 222 = getdirentriesattr */
+	{AC(exchangedata_args), 0, 0, (sy_call_t *)exchangedata, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 223 = exchangedata */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 224 = nosys was checkuseraccess */
+	{AC(searchfs_args), 0, 0, (sy_call_t *)searchfs, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 225 = searchfs */
+	{AC(delete_args), 0, 0, (sy_call_t *)delete, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 226 = delete private delete ( Carbon semantics ) */
+	{AC(copyfile_args), 0, 0, (sy_call_t *)copyfile, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 227 = copyfile */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 228 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 229 = nosys */
+	{AC(poll_args), 0, 0, (sy_call_t *)poll, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},      /* 230 = poll */
+	{AC(watchevent_args), 0, 0, (sy_call_t *)watchevent, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 231 = watchevent */
+	{AC(waitevent_args), 0, 0, (sy_call_t *)waitevent, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 232 = waitevent */
+	{AC(modwatch_args), 0, 0, (sy_call_t *)modwatch, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 233 = modwatch */
+	{AC(getxattr_args), 0, 0, (sy_call_t *)getxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_SSIZE_T, 24}, /* 234 = getxattr */
+	{AC(fgetxattr_args), 0, 0, (sy_call_t *)fgetxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_SSIZE_T, 24}, /* 235 = fgetxattr */
+	{AC(setxattr_args), 0, 0, (sy_call_t *)setxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 236 = setxattr */
+	{AC(fsetxattr_args), 0, 0, (sy_call_t *)fsetxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 237 = fsetxattr */
+	{AC(removexattr_args), 0, 0, (sy_call_t *)removexattr, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 238 = removexattr */
+	{AC(fremovexattr_args), 0, 0, (sy_call_t *)fremovexattr, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 239 = fremovexattr */
+	{AC(listxattr_args), 0, 0, (sy_call_t *)listxattr, munge_wwww, munge_dddd, _SYSCALL_RET_SSIZE_T, 16}, /* 240 = listxattr */
+	{AC(flistxattr_args), 0, 0, (sy_call_t *)flistxattr, munge_wwww, munge_dddd, _SYSCALL_RET_SSIZE_T, 16}, /* 241 = flistxattr */
+	{AC(fsctl_args), 0, 0, (sy_call_t *)fsctl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16},  /* 242 = fsctl */
+	{AC(initgroups_args), 0, 0, (sy_call_t *)initgroups, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 243 = initgroups */
+	{AC(posix_spawn_args), 0, 0, (sy_call_t *)posix_spawn, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 244 = posix_spawn */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 245 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 246 = nosys */
 #if NFSCLIENT
-	{AC(getfh_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getfh, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 161 = getfh */
+	{AC(nfsclnt_args), 0, 0, (sy_call_t *)nfsclnt, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 247 = nfsclnt */
 #else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 161 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 247 = nosys */
 #endif
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 162 = nosys old getdomainname */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 163 = nosys old setdomainname */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 164 = nosys */
-	{AC(quotactl_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)quotactl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 165 = quotactl */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 166 = nosys old exportfs */
-	{AC(mount_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mount, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 167 = mount */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 168 = nosys old ustat */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 169 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 170 = table old table */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 171 = nosys old wait3 */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 172 = nosys old rpause */
-	{AC(waitid_args), _SYSCALL_CANCEL_PRE, KERNEL_FUNNEL, (sy_call_t *)waitid, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 173 = waitid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 174 = nosys old getdents */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 175 = nosys old gc_control */
-	{AC(add_profil_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)add_profil, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 176 = add_profil */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 177 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 178 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 179 = nosys */
-	{AC(kdebug_trace_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)kdebug_trace, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 180 = kdebug_trace */
-	{AC(setgid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setgid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 181 = setgid */
-	{AC(setegid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setegid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 182 = setegid */
-	{AC(seteuid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)seteuid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 183 = seteuid */
-	{AC(sigreturn_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)sigreturn, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 184 = sigreturn */
-	{AC(chud_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)chud, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 185 = chud */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 186 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 187 = nosys */
-	{AC(stat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)stat, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 188 = stat */
-	{AC(fstat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fstat, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 189 = fstat */
-	{AC(lstat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lstat, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 190 = lstat */
-	{AC(pathconf_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)pathconf, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 191 = pathconf */
-	{AC(fpathconf_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fpathconf, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 192 = fpathconf */
-#if COMPAT_GETFSSTAT
-	{AC(getfsstat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getfsstat, munge_wsw, munge_ddd, _SYSCALL_RET_INT_T}, /* 193 = getfsstat */
+#if NFSSERVER
+	{AC(fhopen_args), 0, 0, (sy_call_t *)fhopen, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 248 = fhopen */
 #else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 193 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 248 = nosys */
 #endif
-	{AC(getrlimit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getrlimit, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 194 = getrlimit */
-	{AC(setrlimit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setrlimit, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 195 = setrlimit */
-	{AC(getdirentries_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getdirentries, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 196 = getdirentries */
-	{AC(mmap_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mmap, munge_wwwwwl, munge_dddddd, _SYSCALL_RET_ADDR_T}, /* 197 = mmap */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 198 = nosys __syscall */
-	{AC(lseek_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lseek, munge_wlw, munge_ddd, _SYSCALL_RET_OFF_T}, /* 199 = lseek */
-	{AC(truncate_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)truncate, munge_wl, munge_dd, _SYSCALL_RET_INT_T}, /* 200 = truncate */
-	{AC(ftruncate_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ftruncate, munge_wl, munge_dd, _SYSCALL_RET_INT_T}, /* 201 = ftruncate */
-	{AC(__sysctl_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)__sysctl, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 202 = __sysctl */
-	{AC(mlock_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 203 = mlock */
-	{AC(munlock_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)munlock, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 204 = munlock */
-	{AC(undelete_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)undelete, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 205 = undelete */
-	{AC(ATsocket_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)ATsocket, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 206 = ATsocket */
-	{AC(ATgetmsg_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATgetmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 207 = ATgetmsg */
-	{AC(ATputmsg_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATputmsg, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 208 = ATputmsg */
-	{AC(ATPsndreq_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPsndreq, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 209 = ATPsndreq */
-	{AC(ATPsndrsp_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPsndrsp, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 210 = ATPsndrsp */
-	{AC(ATPgetreq_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPgetreq, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 211 = ATPgetreq */
-	{AC(ATPgetrsp_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)ATPgetrsp, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 212 = ATPgetrsp */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 213 = nosys Reserved for AppleTalk */
-	{AC(kqueue_from_portset_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)kqueue_from_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 214 = kqueue_from_portset_np */
-	{AC(kqueue_portset_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)kqueue_portset_np, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 215 = kqueue_portset_np */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 216 = mkcomplex soon to be obsolete */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 217 = statv soon to be obsolete */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 218 = lstatv soon to be obsolete */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_NONE}, /* 219 = fstatv soon to be obsolete */
-	{AC(getattrlist_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getattrlist, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 220 = getattrlist */
-	{AC(setattrlist_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setattrlist, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 221 = setattrlist */
-	{AC(getdirentriesattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getdirentriesattr, munge_wwwwwwww, munge_dddddddd, _SYSCALL_RET_INT_T}, /* 222 = getdirentriesattr */
-	{AC(exchangedata_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)exchangedata, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 223 = exchangedata */
-#ifdef __APPLE_API_OBSOLETE
-	{AC(checkuseraccess_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)checkuseraccess, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 224 = checkuseraccess */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 249 = nosys */
+	{AC(minherit_args), 0, 0, (sy_call_t *)minherit, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 250 = minherit */
+#if SYSV_SEM
+	{AC(semsys_args), 0, 0, (sy_call_t *)semsys, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 251 = semsys */
 #else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 224 = nosys HFS checkuseraccess check access to a file */
-#endif /* __APPLE_API_OBSOLETE */
-	{AC(searchfs_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)searchfs, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 225 = searchfs */
-	{AC(delete_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)delete, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 226 = delete private delete ( Carbon semantics ) */
-	{AC(copyfile_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)copyfile, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 227 = copyfile */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 228 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 229 = nosys */
-	{AC(poll_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)poll, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 230 = poll */
-	{AC(watchevent_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)watchevent, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 231 = watchevent */
-	{AC(waitevent_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)waitevent, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 232 = waitevent */
-	{AC(modwatch_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL|UNSAFE_64BIT, (sy_call_t *)modwatch, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 233 = modwatch */
-	{AC(getxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_SSIZE_T}, /* 234 = getxattr */
-	{AC(fgetxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fgetxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_SSIZE_T}, /* 235 = fgetxattr */
-	{AC(setxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 236 = setxattr */
-	{AC(fsetxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fsetxattr, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 237 = fsetxattr */
-	{AC(removexattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)removexattr, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 238 = removexattr */
-	{AC(fremovexattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fremovexattr, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 239 = fremovexattr */
-	{AC(listxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)listxattr, munge_wwww, munge_dddd, _SYSCALL_RET_SSIZE_T}, /* 240 = listxattr */
-	{AC(flistxattr_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)flistxattr, munge_wwww, munge_dddd, _SYSCALL_RET_SSIZE_T}, /* 241 = flistxattr */
-	{AC(fsctl_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)fsctl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 242 = fsctl */
-	{AC(initgroups_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)initgroups, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 243 = initgroups */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 244 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 245 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 246 = nosys */
-#if NFSCLIENT
-	{AC(nfsclnt_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)nfsclnt, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 247 = nfsclnt */
-	{AC(fhopen_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)fhopen, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 248 = fhopen */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 251 = nosys */
+#endif
+#if SYSV_MSG
+	{AC(msgsys_args), 0, 0, (sy_call_t *)msgsys, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 252 = msgsys */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 252 = nosys */
+#endif
+#if SYSV_SHM
+	{AC(shmsys_args), 0, 0, (sy_call_t *)shmsys, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 253 = shmsys */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 253 = nosys */
+#endif
+#if SYSV_SEM
+	{AC(semctl_args), 0, 0, (sy_call_t *)semctl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 254 = semctl */
+	{AC(semget_args), 0, 0, (sy_call_t *)semget, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 255 = semget */
+	{AC(semop_args), 0, 0, (sy_call_t *)semop, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},    /* 256 = semop */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 257 = nosys */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 254 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 255 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 256 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 257 = nosys */
+#endif
+#if SYSV_MSG
+	{AC(msgctl_args), 0, 0, (sy_call_t *)msgctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 258 = msgctl */
+	{AC(msgget_args), 0, 0, (sy_call_t *)msgget, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 259 = msgget */
+	{AC(msgsnd_args), 0, 0, (sy_call_t *)msgsnd, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 260 = msgsnd */
+	{AC(msgrcv_args), 0, 0, (sy_call_t *)msgrcv, munge_wwwsw, munge_ddddd, _SYSCALL_RET_SSIZE_T, 20}, /* 261 = msgrcv */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 258 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 259 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 260 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 261 = nosys */
+#endif
+#if SYSV_SHM
+	{AC(shmat_args), 0, 0, (sy_call_t *)shmat, munge_www, munge_ddd, _SYSCALL_RET_ADDR_T, 12},   /* 262 = shmat */
+	{AC(shmctl_args), 0, 0, (sy_call_t *)shmctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 263 = shmctl */
+	{AC(shmdt_args), 0, 0, (sy_call_t *)shmdt, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},        /* 264 = shmdt */
+	{AC(shmget_args), 0, 0, (sy_call_t *)shmget, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 265 = shmget */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 262 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 263 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 264 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 265 = nosys */
+#endif
+	{AC(shm_open_args), 0, 0, (sy_call_t *)shm_open, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 266 = shm_open */
+	{AC(shm_unlink_args), 0, 0, (sy_call_t *)shm_unlink, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 267 = shm_unlink */
+	{AC(sem_open_args), 0, 0, (sy_call_t *)sem_open, munge_wwww, munge_dddd, _SYSCALL_RET_ADDR_T, 16}, /* 268 = sem_open */
+	{AC(sem_close_args), 0, 0, (sy_call_t *)sem_close, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 269 = sem_close */
+	{AC(sem_unlink_args), 0, 0, (sy_call_t *)sem_unlink, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 270 = sem_unlink */
+	{AC(sem_wait_args), 0, 0, (sy_call_t *)sem_wait, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 271 = sem_wait */
+	{AC(sem_trywait_args), 0, 0, (sy_call_t *)sem_trywait, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 272 = sem_trywait */
+	{AC(sem_post_args), 0, 0, (sy_call_t *)sem_post, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 273 = sem_post */
+	{AC(sem_getvalue_args), 0, 0, (sy_call_t *)sem_getvalue, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 274 = sem_getvalue */
+	{AC(sem_init_args), 0, 0, (sy_call_t *)sem_init, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 275 = sem_init */
+	{AC(sem_destroy_args), 0, 0, (sy_call_t *)sem_destroy, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 276 = sem_destroy */
+	{AC(open_extended_args), 0, 0, (sy_call_t *)open_extended, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 277 = open_extended */
+	{AC(umask_extended_args), 0, 0, (sy_call_t *)umask_extended, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 278 = umask_extended */
+	{AC(stat_extended_args), 0, 0, (sy_call_t *)stat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 279 = stat_extended */
+	{AC(lstat_extended_args), 0, 0, (sy_call_t *)lstat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 280 = lstat_extended */
+	{AC(fstat_extended_args), 0, 0, (sy_call_t *)fstat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 281 = fstat_extended */
+	{AC(chmod_extended_args), 0, 0, (sy_call_t *)chmod_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 282 = chmod_extended */
+	{AC(fchmod_extended_args), 0, 0, (sy_call_t *)fchmod_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 283 = fchmod_extended */
+	{AC(access_extended_args), 0, 0, (sy_call_t *)access_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 284 = access_extended */
+	{AC(settid_args), 0, 0, (sy_call_t *)settid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 285 = settid */
+	{AC(gettid_args), 0, 0, (sy_call_t *)gettid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 286 = gettid */
+	{AC(setsgroups_args), 0, 0, (sy_call_t *)setsgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 287 = setsgroups */
+	{AC(getsgroups_args), 0, 0, (sy_call_t *)getsgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 288 = getsgroups */
+	{AC(setwgroups_args), 0, 0, (sy_call_t *)setwgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 289 = setwgroups */
+	{AC(getwgroups_args), 0, 0, (sy_call_t *)getwgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 290 = getwgroups */
+	{AC(mkfifo_extended_args), 0, 0, (sy_call_t *)mkfifo_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 291 = mkfifo_extended */
+	{AC(mkdir_extended_args), 0, 0, (sy_call_t *)mkdir_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 292 = mkdir_extended */
+	{AC(identitysvc_args), 0, 0, (sy_call_t *)identitysvc, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 293 = identitysvc */
+	{AC(shared_region_check_np_args), 0, 0, (sy_call_t *)shared_region_check_np, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 294 = shared_region_check_np */
+	{AC(shared_region_map_np_args), 0, 0, (sy_call_t *)shared_region_map_np, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 295 = shared_region_map_np */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 296 = nosys old load_shared_file */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 297 = nosys old reset_shared_file */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 298 = nosys old new_system_shared_regions */
+	{0, 0, 0, (sy_call_t *)enosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 299 = enosys old shared_region_map_file_np */
+	{0, 0, 0, (sy_call_t *)enosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 300 = enosys old shared_region_make_private_np */
+	{AC(__pthread_mutex_destroy_args), 0, 0, (sy_call_t *)__pthread_mutex_destroy, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 301 = __pthread_mutex_destroy */
+	{AC(__pthread_mutex_init_args), 0, 0, (sy_call_t *)__pthread_mutex_init, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 302 = __pthread_mutex_init */
+	{AC(__pthread_mutex_lock_args), 0, 0, (sy_call_t *)__pthread_mutex_lock, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 303 = __pthread_mutex_lock */
+	{AC(__pthread_mutex_trylock_args), 0, 0, (sy_call_t *)__pthread_mutex_trylock, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 304 = __pthread_mutex_trylock */
+	{AC(__pthread_mutex_unlock_args), 0, 0, (sy_call_t *)__pthread_mutex_unlock, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 305 = __pthread_mutex_unlock */
+	{AC(__pthread_cond_init_args), 0, 0, (sy_call_t *)__pthread_cond_init, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 306 = __pthread_cond_init */
+	{AC(__pthread_cond_destroy_args), 0, 0, (sy_call_t *)__pthread_cond_destroy, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 307 = __pthread_cond_destroy */
+	{AC(__pthread_cond_broadcast_args), 0, 0, (sy_call_t *)__pthread_cond_broadcast, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 308 = __pthread_cond_broadcast */
+	{AC(__pthread_cond_signal_args), 0, 0, (sy_call_t *)__pthread_cond_signal, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 309 = __pthread_cond_signal */
+	{AC(getsid_args), 0, 0, (sy_call_t *)getsid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},      /* 310 = getsid */
+	{AC(settid_with_pid_args), 0, 0, (sy_call_t *)settid_with_pid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 311 = settid_with_pid */
+	{AC(__pthread_cond_timedwait_args), 0, 0, (sy_call_t *)__pthread_cond_timedwait, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 312 = __pthread_cond_timedwait */
+	{AC(aio_fsync_args), 0, 0, (sy_call_t *)aio_fsync, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 313 = aio_fsync */
+	{AC(aio_return_args), 0, 0, (sy_call_t *)aio_return, munge_w, munge_d, _SYSCALL_RET_SSIZE_T, 4}, /* 314 = aio_return */
+	{AC(aio_suspend_args), 0, 0, (sy_call_t *)aio_suspend, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 315 = aio_suspend */
+	{AC(aio_cancel_args), 0, 0, (sy_call_t *)aio_cancel, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 316 = aio_cancel */
+	{AC(aio_error_args), 0, 0, (sy_call_t *)aio_error, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 317 = aio_error */
+	{AC(aio_read_args), 0, 0, (sy_call_t *)aio_read, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 318 = aio_read */
+	{AC(aio_write_args), 0, 0, (sy_call_t *)aio_write, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 319 = aio_write */
+	{AC(lio_listio_args), 0, 0, (sy_call_t *)lio_listio, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 320 = lio_listio */
+	{AC(__pthread_cond_wait_args), 0, 0, (sy_call_t *)__pthread_cond_wait, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 321 = __pthread_cond_wait */
+	{AC(iopolicysys_args), 0, 0, (sy_call_t *)iopolicysys, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 322 = iopolicysys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 323 = nosys */
+	{AC(mlockall_args), 0, 0, (sy_call_t *)mlockall, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 324 = mlockall */
+	{AC(munlockall_args), 0, 0, (sy_call_t *)munlockall, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 325 = munlockall */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 326 = nosys */
+	{0, 0, 0, (sy_call_t *)issetugid, NULL, NULL, _SYSCALL_RET_INT_T, 0},                       /* 327 = issetugid */
+	{AC(__pthread_kill_args), 0, 0, (sy_call_t *)__pthread_kill, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 328 = __pthread_kill */
+	{AC(__pthread_sigmask_args), 0, 0, (sy_call_t *)__pthread_sigmask, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 329 = __pthread_sigmask */
+	{AC(__sigwait_args), 0, 0, (sy_call_t *)__sigwait, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 330 = __sigwait */
+	{AC(__disable_threadsignal_args), 0, 0, (sy_call_t *)__disable_threadsignal, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 331 = __disable_threadsignal */
+	{AC(__pthread_markcancel_args), 0, 0, (sy_call_t *)__pthread_markcancel, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 332 = __pthread_markcancel */
+	{AC(__pthread_canceled_args), 0, 0, (sy_call_t *)__pthread_canceled, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 333 = __pthread_canceled */
+	{AC(__semwait_signal_args), 0, 0, (sy_call_t *)__semwait_signal, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 334 = __semwait_signal */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 335 = nosys old utrace */
+	{AC(proc_info_args), 0, 0, (sy_call_t *)proc_info, munge_wwwlww, munge_dddddd, _SYSCALL_RET_INT_T, 28}, /* 336 = proc_info */
+#if SENDFILE
+	{AC(sendfile_args), 0, 0, (sy_call_t *)sendfile, munge_wwlwww, munge_dddddd, _SYSCALL_RET_INT_T, 28}, /* 337 = sendfile */
+#else /* !SENDFILE */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 337 = nosys */
+#endif /* SENDFILE */
+	{AC(stat64_args), 0, 0, (sy_call_t *)stat64, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},    /* 338 = stat64 */
+	{AC(fstat64_args), 0, 0, (sy_call_t *)fstat64, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 339 = fstat64 */
+	{AC(lstat64_args), 0, 0, (sy_call_t *)lstat64, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 340 = lstat64 */
+	{AC(stat64_extended_args), 0, 0, (sy_call_t *)stat64_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 341 = stat64_extended */
+	{AC(lstat64_extended_args), 0, 0, (sy_call_t *)lstat64_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 342 = lstat64_extended */
+	{AC(fstat64_extended_args), 0, 0, (sy_call_t *)fstat64_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 343 = fstat64_extended */
+	{AC(getdirentries64_args), 0, 0, (sy_call_t *)getdirentries64, munge_wwww, munge_dddd, _SYSCALL_RET_SSIZE_T, 16}, /* 344 = getdirentries64 */
+	{AC(statfs64_args), 0, 0, (sy_call_t *)statfs64, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 345 = statfs64 */
+	{AC(fstatfs64_args), 0, 0, (sy_call_t *)fstatfs64, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 346 = fstatfs64 */
+	{AC(getfsstat64_args), 0, 0, (sy_call_t *)getfsstat64, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 347 = getfsstat64 */
+	{AC(__pthread_chdir_args), 0, 0, (sy_call_t *)__pthread_chdir, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 348 = __pthread_chdir */
+	{AC(__pthread_fchdir_args), 0, 0, (sy_call_t *)__pthread_fchdir, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 349 = __pthread_fchdir */
+#if AUDIT
+	{AC(audit_args), 0, 0, (sy_call_t *)audit, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},      /* 350 = audit */
+	{AC(auditon_args), 0, 0, (sy_call_t *)auditon, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 351 = auditon */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 352 = nosys */
+	{AC(getauid_args), 0, 0, (sy_call_t *)getauid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 353 = getauid */
+	{AC(setauid_args), 0, 0, (sy_call_t *)setauid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 354 = setauid */
+	{AC(getaudit_args), 0, 0, (sy_call_t *)getaudit, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 355 = getaudit */
+	{AC(setaudit_args), 0, 0, (sy_call_t *)setaudit, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 356 = setaudit */
+	{AC(getaudit_addr_args), 0, 0, (sy_call_t *)getaudit_addr, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 357 = getaudit_addr */
+	{AC(setaudit_addr_args), 0, 0, (sy_call_t *)setaudit_addr, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 358 = setaudit_addr */
+	{AC(auditctl_args), 0, 0, (sy_call_t *)auditctl, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},  /* 359 = auditctl */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 350 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 351 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 352 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 353 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 354 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 355 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 356 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 357 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 358 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 359 = nosys */
+#endif
+	{AC(bsdthread_create_args), 0, 0, (sy_call_t *)bsdthread_create, munge_wwwww, munge_ddddd, _SYSCALL_RET_ADDR_T, 20}, /* 360 = bsdthread_create */
+	{AC(bsdthread_terminate_args), 0, 0, (sy_call_t *)bsdthread_terminate, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 361 = bsdthread_terminate */
+	{0, 0, 0, (sy_call_t *)kqueue, NULL, NULL, _SYSCALL_RET_INT_T, 0},                          /* 362 = kqueue */
+	{AC(kevent_args), 0, 0, (sy_call_t *)kevent, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 363 = kevent */
+	{AC(lchown_args), 0, 0, (sy_call_t *)lchown, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12},  /* 364 = lchown */
+	{AC(stack_snapshot_args), 0, 0, (sy_call_t *)stack_snapshot, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 365 = stack_snapshot */
+	{AC(bsdthread_register_args), 0, 0, (sy_call_t *)bsdthread_register, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 366 = bsdthread_register */
+	{0, 0, 0, (sy_call_t *)workq_open, NULL, NULL, _SYSCALL_RET_INT_T, 0},                      /* 367 = workq_open */
+	{AC(workq_ops_args), 0, 0, (sy_call_t *)workq_ops, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 368 = workq_ops */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 369 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 370 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 371 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 372 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 373 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 374 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 375 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 376 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 377 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 378 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 379 = nosys */
+	{AC(__mac_execve_args), 0, 0, (sy_call_t *)__mac_execve, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 380 = __mac_execve */
+	{AC(__mac_syscall_args), 0, 0, (sy_call_t *)__mac_syscall, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 381 = __mac_syscall */
+	{AC(__mac_get_file_args), 0, 0, (sy_call_t *)__mac_get_file, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 382 = __mac_get_file */
+	{AC(__mac_set_file_args), 0, 0, (sy_call_t *)__mac_set_file, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 383 = __mac_set_file */
+	{AC(__mac_get_link_args), 0, 0, (sy_call_t *)__mac_get_link, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 384 = __mac_get_link */
+	{AC(__mac_set_link_args), 0, 0, (sy_call_t *)__mac_set_link, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 385 = __mac_set_link */
+	{AC(__mac_get_proc_args), 0, 0, (sy_call_t *)__mac_get_proc, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 386 = __mac_get_proc */
+	{AC(__mac_set_proc_args), 0, 0, (sy_call_t *)__mac_set_proc, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 387 = __mac_set_proc */
+	{AC(__mac_get_fd_args), 0, 0, (sy_call_t *)__mac_get_fd, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 388 = __mac_get_fd */
+	{AC(__mac_set_fd_args), 0, 0, (sy_call_t *)__mac_set_fd, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 389 = __mac_set_fd */
+	{AC(__mac_get_pid_args), 0, 0, (sy_call_t *)__mac_get_pid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 390 = __mac_get_pid */
+	{AC(__mac_get_lcid_args), 0, 0, (sy_call_t *)__mac_get_lcid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 391 = __mac_get_lcid */
+	{AC(__mac_get_lctx_args), 0, 0, (sy_call_t *)__mac_get_lctx, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 392 = __mac_get_lctx */
+	{AC(__mac_set_lctx_args), 0, 0, (sy_call_t *)__mac_set_lctx, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 393 = __mac_set_lctx */
+	{AC(setlcid_args), 0, 0, (sy_call_t *)setlcid, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8},  /* 394 = setlcid */
+	{AC(getlcid_args), 0, 0, (sy_call_t *)getlcid, munge_w, munge_d, _SYSCALL_RET_INT_T, 4},    /* 395 = getlcid */
+	{AC(read_nocancel_args), 0, 0, (sy_call_t *)read_nocancel, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12}, /* 396 = read_nocancel */
+	{AC(write_nocancel_args), 0, 0, (sy_call_t *)write_nocancel, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12}, /* 397 = write_nocancel */
+	{AC(open_nocancel_args), 0, 0, (sy_call_t *)open_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 398 = open_nocancel */
+	{AC(close_nocancel_args), 0, 0, (sy_call_t *)close_nocancel, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 399 = close_nocancel */
+	{AC(wait4_nocancel_args), 0, 0, (sy_call_t *)wait4_nocancel, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 400 = wait4_nocancel */
+#if SOCKETS
+	{AC(recvmsg_nocancel_args), 0, 0, (sy_call_t *)recvmsg_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 401 = recvmsg_nocancel */
+	{AC(sendmsg_nocancel_args), 0, 0, (sy_call_t *)sendmsg_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 402 = sendmsg_nocancel */
+	{AC(recvfrom_nocancel_args), 0, 0, (sy_call_t *)recvfrom_nocancel, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 403 = recvfrom_nocancel */
+	{AC(accept_nocancel_args), 0, 0, (sy_call_t *)accept_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 404 = accept_nocancel */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 401 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 402 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 403 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 404 = nosys */
+#endif /* SOCKETS */
+	{AC(msync_nocancel_args), 0, 0, (sy_call_t *)msync_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 405 = msync_nocancel */
+	{AC(fcntl_nocancel_args), 0, 0, (sy_call_t *)fcntl_nocancel, munge_wws, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 406 = fcntl_nocancel */
+	{AC(select_nocancel_args), 0, 0, (sy_call_t *)select_nocancel, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 407 = select_nocancel */
+	{AC(fsync_nocancel_args), 0, 0, (sy_call_t *)fsync_nocancel, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 408 = fsync_nocancel */
+#if SOCKETS
+	{AC(connect_nocancel_args), 0, 0, (sy_call_t *)connect_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 409 = connect_nocancel */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 409 = nosys */
+#endif /* SOCKETS */
+	{AC(sigsuspend_nocancel_args), 0, 0, (sy_call_t *)sigsuspend_nocancel, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 410 = sigsuspend_nocancel */
+	{AC(readv_nocancel_args), 0, 0, (sy_call_t *)readv_nocancel, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12}, /* 411 = readv_nocancel */
+	{AC(writev_nocancel_args), 0, 0, (sy_call_t *)writev_nocancel, munge_www, munge_ddd, _SYSCALL_RET_SSIZE_T, 12}, /* 412 = writev_nocancel */
+#if SOCKETS
+	{AC(sendto_nocancel_args), 0, 0, (sy_call_t *)sendto_nocancel, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 413 = sendto_nocancel */
+#else
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 413 = nosys */
+#endif /* SOCKETS */
+	{AC(pread_nocancel_args), 0, 0, (sy_call_t *)pread_nocancel, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T, 20}, /* 414 = pread_nocancel */
+	{AC(pwrite_nocancel_args), 0, 0, (sy_call_t *)pwrite_nocancel, munge_wwwl, munge_dddd, _SYSCALL_RET_SSIZE_T, 20}, /* 415 = pwrite_nocancel */
+	{AC(waitid_nocancel_args), 0, 0, (sy_call_t *)waitid_nocancel, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 416 = waitid_nocancel */
+	{AC(poll_nocancel_args), 0, 0, (sy_call_t *)poll_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 417 = poll_nocancel */
+#if SYSV_MSG
+	{AC(msgsnd_nocancel_args), 0, 0, (sy_call_t *)msgsnd_nocancel, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T, 16}, /* 418 = msgsnd_nocancel */
+	{AC(msgrcv_nocancel_args), 0, 0, (sy_call_t *)msgrcv_nocancel, munge_wwwsw, munge_ddddd, _SYSCALL_RET_SSIZE_T, 20}, /* 419 = msgrcv_nocancel */
 #else
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 247 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 248 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 418 = nosys */
+	{0, 0, 0, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T, 0},                           /* 419 = nosys */
 #endif
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 249 = nosys */
-	{AC(minherit_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)minherit, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 250 = minherit */
-	{AC(semsys_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)semsys, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 251 = semsys */
-	{AC(msgsys_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)msgsys, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 252 = msgsys */
-	{AC(shmsys_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shmsys, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 253 = shmsys */
-	{AC(semctl_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)semctl, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 254 = semctl */
-	{AC(semget_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)semget, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 255 = semget */
-	{AC(semop_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)semop, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 256 = semop */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 257 = nosys */
-	{AC(msgctl_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)msgctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 258 = msgctl */
-	{AC(msgget_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)msgget, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 259 = msgget */
-	{AC(msgsnd_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)msgsnd, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 260 = msgsnd */
-	{AC(msgrcv_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)msgrcv, munge_wwwsw, munge_ddddd, _SYSCALL_RET_SSIZE_T}, /* 261 = msgrcv */
-	{AC(shmat_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shmat, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 262 = shmat */
-	{AC(shmctl_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shmctl, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 263 = shmctl */
-	{AC(shmdt_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shmdt, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 264 = shmdt */
-	{AC(shmget_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shmget, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 265 = shmget */
-	{AC(shm_open_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shm_open, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 266 = shm_open */
-	{AC(shm_unlink_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)shm_unlink, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 267 = shm_unlink */
-	{AC(sem_open_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_open, munge_wwww, munge_dddd, _SYSCALL_RET_ADDR_T}, /* 268 = sem_open */
-	{AC(sem_close_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_close, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 269 = sem_close */
-	{AC(sem_unlink_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_unlink, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 270 = sem_unlink */
-	{AC(sem_wait_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)sem_wait, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 271 = sem_wait */
-	{AC(sem_trywait_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_trywait, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 272 = sem_trywait */
-	{AC(sem_post_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_post, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 273 = sem_post */
-	{AC(sem_getvalue_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_getvalue, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 274 = sem_getvalue */
-	{AC(sem_init_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_init, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 275 = sem_init */
-	{AC(sem_destroy_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)sem_destroy, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 276 = sem_destroy */
-	{AC(open_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)open_extended, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 277 = open_extended */
-	{AC(umask_extended_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)umask_extended, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 278 = umask_extended */
-	{AC(stat_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)stat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 279 = stat_extended */
-	{AC(lstat_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lstat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 280 = lstat_extended */
-	{AC(fstat_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fstat_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 281 = fstat_extended */
-	{AC(chmod_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)chmod_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 282 = chmod_extended */
-	{AC(fchmod_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)fchmod_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 283 = fchmod_extended */
-	{AC(access_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)access_extended, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 284 = access_extended */
-	{AC(settid_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)settid, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 285 = settid */
-	{AC(gettid_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)gettid, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 286 = gettid */
-	{AC(setsgroups_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setsgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 287 = setsgroups */
-	{AC(getsgroups_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getsgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 288 = getsgroups */
-	{AC(setwgroups_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)setwgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 289 = setwgroups */
-	{AC(getwgroups_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)getwgroups, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 290 = getwgroups */
-	{AC(mkfifo_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mkfifo_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 291 = mkfifo_extended */
-	{AC(mkdir_extended_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mkdir_extended, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T}, /* 292 = mkdir_extended */
-	{AC(identitysvc_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)identitysvc, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 293 = identitysvc */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 294 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 295 = nosys */
-	{AC(load_shared_file_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL|UNSAFE_64BIT, (sy_call_t *)load_shared_file, munge_wwwwwww, munge_ddddddd, _SYSCALL_RET_INT_T}, /* 296 = load_shared_file */
-	{AC(reset_shared_file_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL|UNSAFE_64BIT, (sy_call_t *)reset_shared_file, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 297 = reset_shared_file */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)new_system_shared_regions, NULL, NULL, _SYSCALL_RET_INT_T}, /* 298 = new_system_shared_regions */
-	{AC(shared_region_map_file_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL|UNSAFE_64BIT, (sy_call_t *)shared_region_map_file_np, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 299 = shared_region_map_file_np */
-	{AC(shared_region_make_private_np_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL|UNSAFE_64BIT, (sy_call_t *)shared_region_make_private_np, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 300 = shared_region_make_private_np */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 301 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 302 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 303 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 304 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 305 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 306 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 307 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 308 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 309 = nosys */
-	{AC(getsid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getsid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 310 = getsid */
-	{AC(settid_with_pid_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)settid_with_pid, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 311 = settid_with_pid */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 312 = nosys */
-	{AC(aio_fsync_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_fsync, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 313 = aio_fsync */
-	{AC(aio_return_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_return, munge_w, munge_d, _SYSCALL_RET_SSIZE_T}, /* 314 = aio_return */
-	{AC(aio_suspend_args), _SYSCALL_CANCEL_PRE, NO_FUNNEL, (sy_call_t *)aio_suspend, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 315 = aio_suspend */
-	{AC(aio_cancel_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_cancel, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 316 = aio_cancel */
-	{AC(aio_error_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_error, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 317 = aio_error */
-	{AC(aio_read_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_read, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 318 = aio_read */
-	{AC(aio_write_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)aio_write, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 319 = aio_write */
-	{AC(lio_listio_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lio_listio, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 320 = lio_listio */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 321 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 322 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 323 = nosys */
-	{AC(mlockall_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)mlockall, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 324 = mlockall */
-	{AC(munlockall_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)munlockall, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 325 = munlockall */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 326 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)issetugid, NULL, NULL, _SYSCALL_RET_INT_T}, /* 327 = issetugid */
-	{AC(__pthread_kill_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)__pthread_kill, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 328 = __pthread_kill */
-	{AC(pthread_sigmask_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)pthread_sigmask, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 329 = pthread_sigmask */
-	{AC(sigwait_args), _SYSCALL_CANCEL_PRE, KERNEL_FUNNEL, (sy_call_t *)sigwait, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 330 = sigwait */
-	{AC(__disable_threadsignal_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)__disable_threadsignal, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 331 = __disable_threadsignal */
-	{AC(__pthread_markcancel_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)__pthread_markcancel, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 332 = __pthread_markcancel */
-	{AC(__pthread_canceled_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)__pthread_canceled, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 333 = __pthread_canceled */
-	{AC(__semwait_signal_args), _SYSCALL_CANCEL_POST, NO_FUNNEL, (sy_call_t *)__semwait_signal, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 334 = __semwait_signal */
-	{AC(utrace_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)utrace, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 335 = utrace */
-	{AC(proc_info_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)proc_info, munge_wwwlww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 336 = proc_info */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 337 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 338 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 339 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 340 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 341 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 342 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 343 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 344 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 345 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 346 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 347 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 348 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 349 = nosys */
-	{AC(audit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)audit, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 350 = audit */
-	{AC(auditon_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)auditon, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 351 = auditon */
-	{0, _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 352 = nosys */
-	{AC(getauid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getauid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 353 = getauid */
-	{AC(setauid_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setauid, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 354 = setauid */
-	{AC(getaudit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getaudit, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 355 = getaudit */
-	{AC(setaudit_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setaudit, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 356 = setaudit */
-	{AC(getaudit_addr_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)getaudit_addr, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 357 = getaudit_addr */
-	{AC(setaudit_addr_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)setaudit_addr, munge_ww, munge_dd, _SYSCALL_RET_INT_T}, /* 358 = setaudit_addr */
-	{AC(auditctl_args), _SYSCALL_CANCEL_NONE, KERNEL_FUNNEL, (sy_call_t *)auditctl, munge_w, munge_d, _SYSCALL_RET_INT_T}, /* 359 = auditctl */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 360 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 361 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)kqueue, NULL, NULL, _SYSCALL_RET_INT_T}, /* 362 = kqueue */
-	{AC(kevent_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)kevent, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T}, /* 363 = kevent */
-	{AC(lchown_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)lchown, munge_www, munge_ddd, _SYSCALL_RET_INT_T}, /* 364 = lchown */
-	{AC(stack_snapshot_args), _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)stack_snapshot, munge_wwww, munge_dddd, _SYSCALL_RET_INT_T}, /* 365 = stack_snapshot */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 366 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 367 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 368 = nosys */
-	{0, _SYSCALL_CANCEL_NONE, NO_FUNNEL, (sy_call_t *)nosys, NULL, NULL, _SYSCALL_RET_INT_T}, /* 369 = nosys */
+	{AC(sem_wait_nocancel_args), 0, 0, (sy_call_t *)sem_wait_nocancel, munge_w, munge_d, _SYSCALL_RET_INT_T, 4}, /* 420 = sem_wait_nocancel */
+	{AC(aio_suspend_nocancel_args), 0, 0, (sy_call_t *)aio_suspend_nocancel, munge_www, munge_ddd, _SYSCALL_RET_INT_T, 12}, /* 421 = aio_suspend_nocancel */
+	{AC(__sigwait_nocancel_args), 0, 0, (sy_call_t *)__sigwait_nocancel, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 422 = __sigwait_nocancel */
+	{AC(__semwait_signal_nocancel_args), 0, 0, (sy_call_t *)__semwait_signal_nocancel, munge_wwwwww, munge_dddddd, _SYSCALL_RET_INT_T, 24}, /* 423 = __semwait_signal_nocancel */
+	{AC(__mac_mount_args), 0, 0, (sy_call_t *)__mac_mount, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 424 = __mac_mount */
+	{AC(__mac_get_mount_args), 0, 0, (sy_call_t *)__mac_get_mount, munge_ww, munge_dd, _SYSCALL_RET_INT_T, 8}, /* 425 = __mac_get_mount */
+	{AC(__mac_getfsstat_args), 0, 0, (sy_call_t *)__mac_getfsstat, munge_wwwww, munge_ddddd, _SYSCALL_RET_INT_T, 20}, /* 426 = __mac_getfsstat */
 };
 int	nsysent = sizeof(sysent) / sizeof(sysent[0]);
+/* Verify that NUM_SYSENT reflects the latest syscall count */
+int	nsysent_size_check[((sizeof(sysent) / sizeof(sysent[0])) == NUM_SYSENT) ? 1 : -1] __unused;
diff --git a/bsd/kern/kdebug.c b/bsd/kern/kdebug.c
index 286a9d8ce..952d2b87c 100644
--- a/bsd/kern/kdebug.c
+++ b/bsd/kern/kdebug.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
  * @Apple_LICENSE_HEADER_START@
  * 
@@ -17,7 +17,7 @@
  * License for the specific language governing rights and limitations
  * under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/spl.h>
@@ -39,6 +39,7 @@
 
 #include <kern/thread.h>
 #include <kern/task.h>
+#include <kern/debug.h>
 #include <vm/vm_kern.h>
 #include <sys/lock.h>
 
@@ -67,9 +68,11 @@ void kdbg_mapinit(void);
 int kdbg_reinit(void);
 int kdbg_bootstrap(void);
 
-static int  create_buffers(void);
+static int create_buffers(void);
 static void delete_buffers(void);
 
+extern void IOSleep(int);
+
 #ifdef ppc
 extern uint32_t maxDec;
 #endif
@@ -181,7 +184,7 @@ typedef void (*kd_chudhook_fn) (unsigned int debugid, unsigned int arg1,
 
 kd_chudhook_fn kdebug_chudhook = 0;   /* pointer to CHUD toolkit function */
 
-__private_extern__ void stackshot_lock_init( void );
+__private_extern__ void stackshot_lock_init( void ) __attribute__((section("__TEXT, initcode")));
 
 /* Support syscall SYS_kdebug_trace */
 int
@@ -194,8 +197,6 @@ kdebug_trace(__unused struct proc *p, struct kdebug_trace_args *uap, __unused re
     return(0);
 }
 
-
-
 static int
 create_buffers(void)
 {
@@ -203,13 +204,15 @@ create_buffers(void)
 	int     nentries;
 
 	nentries = nkdbufs / kd_cpus;
+	nkdbufs = nentries * kd_cpus;
+
 	kd_bufsize = nentries * sizeof(kd_buf);
 
 	bzero((char *)kdbip, sizeof(struct kd_bufinfo) * kd_cpus);
 
 	if (kdcopybuf == 0) {
 	        if (kmem_alloc(kernel_map, (unsigned int *)&kdcopybuf, (vm_size_t)KDCOPYBUF_SIZE) != KERN_SUCCESS)
-		        return ENOMEM;
+		        return(ENOMEM);
 	}
 	for (cpu = 0; cpu < kd_cpus; cpu++) {
 	        if (kmem_alloc(kernel_map, (unsigned int *)&kdbip[cpu].kd_buffer, kd_bufsize) != KERN_SUCCESS)
@@ -303,7 +306,7 @@ kernel_debug_internal(unsigned int debugid, unsigned int arg1, unsigned int arg2
 	  {
 	    /* If kdebug flag is not set for current proc, return  */
 	    curproc = current_proc();
-	    if ((curproc && !(curproc->p_flag & P_KDEBUG)) &&
+	    if ((curproc && !(curproc->p_kdebug)) &&
 		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
 		goto out;
 	  }
@@ -311,7 +314,7 @@ kernel_debug_internal(unsigned int debugid, unsigned int arg1, unsigned int arg2
 	  {
 	    /* If kdebug flag is set for current proc, return  */
 	    curproc = current_proc();
-	    if ((curproc && (curproc->p_flag & P_KDEBUG)) &&
+	    if ((curproc && curproc->p_kdebug) &&
 		((debugid&0xffff0000) != (MACHDBG_CODE(DBG_MACH_SCHED, 0) | DBG_FUNC_NONE)))
 		goto out;
 	  }
@@ -393,7 +396,6 @@ kdbg_lock_init(void)
 	host_basic_info_data_t hinfo;
 	mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 
-
 	if (kdebug_flags & KDBG_LOCKINIT)
 		return;
 
@@ -402,7 +404,8 @@ kdbg_lock_init(void)
 	host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 	kd_cpus = hinfo.physical_cpu_max;
 
-	if (kmem_alloc(kernel_map, (unsigned int *)&kdbip, sizeof(struct kd_bufinfo) * kd_cpus) != KERN_SUCCESS)
+	if (kmem_alloc(kernel_map, (unsigned int *)&kdbip,
+		       sizeof(struct kd_bufinfo) * kd_cpus) != KERN_SUCCESS)
 	        return;
 
 	/*
@@ -486,11 +489,9 @@ kdbg_trace_data(struct proc *proc, long *arg_pid)
 void
 kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4)
 {
-    int i;
     char *dbg_nameptr; 
     int dbg_namelen;
     long dbg_parms[4];
-
     if (!proc)
       {
 	*arg1 = 0;
@@ -511,12 +512,7 @@ kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *a
     if(dbg_namelen > (int)sizeof(dbg_parms))
       dbg_namelen = sizeof(dbg_parms);
     
-    for(i=0;dbg_namelen > 0; i++)
-      {
-	dbg_parms[i]=*(long*)dbg_nameptr;
-	dbg_nameptr += sizeof(long);
-	dbg_namelen -= sizeof(long);
-      }
+    strncpy((char *)dbg_parms, dbg_nameptr, dbg_namelen);
 
     *arg1=dbg_parms[0];
     *arg2=dbg_parms[1];
@@ -534,9 +530,8 @@ kdbg_resolve_map(thread_t th_act, void *opaque)
     {
       mapptr=&t->map[t->count];
       mapptr->thread  = (unsigned int)th_act;
-      (void) strncpy (mapptr->command, t->atts->task_comm,
-		      sizeof(t->atts->task_comm)-1);
-      mapptr->command[sizeof(t->atts->task_comm)-1] = '\0';
+      (void) strlcpy (mapptr->command, t->atts->task_comm,
+		      sizeof(t->atts->task_comm));
 
       /*
 	Some kernel threads have no associated pid.
@@ -566,6 +561,9 @@ kdbg_mapinit(void)
         if (kdebug_flags & KDBG_MAPINIT)
 	  return;
 
+	/* need to use PROC_SCANPROCLIST with proc_iterate */
+	proc_list_lock();
+
 	/* Calculate the sizes of map buffers*/
 	for (p = allproc.lh_first, kd_mapcount=0, tts_count=0; p; 
 	     p = p->p_list.le_next)
@@ -574,6 +572,8 @@ kdbg_mapinit(void)
 	    tts_count++;
 	  }
 
+	proc_list_unlock();
+
 	/*
 	 * The proc count could change during buffer allocation,
 	 * so introduce a small fudge factor to bump up the
@@ -611,20 +611,26 @@ kdbg_mapinit(void)
 	 */
 
 	if (tts_mapptr) {
+		/* should use proc_iterate */
+		proc_list_lock();
+
 	        for (p = allproc.lh_first, i=0; p && i < tts_count; 
 		     p = p->p_list.le_next) {
-	                if (p->p_flag & P_WEXIT)
+	                if (p->p_lflag & P_LEXIT)
 		                continue;
 
 			if (p->task) {
 				task_reference(p->task);
 				tts_mapptr[i].task = p->task;
 				tts_mapptr[i].pid  = p->p_pid;
-				(void)strncpy(tts_mapptr[i].task_comm, p->p_comm, sizeof(tts_mapptr[i].task_comm) - 1);
+				(void)strlcpy(tts_mapptr[i].task_comm, p->p_comm, sizeof(tts_mapptr[i].task_comm));
 				i++;
 			}
 		}
 		tts_count = i;
+
+		proc_list_unlock();
+
 	}
 
 
@@ -695,7 +701,7 @@ kdbg_setpid(kd_regtype *kdr)
 
   if (pid > 0)
     {
-      if ((p = pfind(pid)) == NULL)
+      if ((p = proc_find(pid)) == NULL)
 	ret = ESRCH;
       else
 	{
@@ -705,14 +711,15 @@ kdbg_setpid(kd_regtype *kdr)
 	      kdebug_flags &= ~KDBG_PIDEXCLUDE;
 	      kdebug_slowcheck |= SLOW_CHECKS;
 
-	      p->p_flag |= P_KDEBUG;
+	      p->p_kdebug = 1;
 	    }
 	  else  /* turn off pid check for this pid value */
 	    {
 	      /* Don't turn off all pid checking though */
 	      /* kdebug_flags &= ~KDBG_PIDCHECK;*/   
-	      p->p_flag &= ~P_KDEBUG;
+	      p->p_kdebug = 0;
 	    }
+	proc_rele(p);
 	}
     }
   else
@@ -733,7 +740,7 @@ kdbg_setpidex(kd_regtype *kdr)
 
   if (pid > 0)
     {
-      if ((p = pfind(pid)) == NULL)
+      if ((p = proc_find(pid)) == NULL)
 	ret = ESRCH;
       else
 	{
@@ -743,14 +750,15 @@ kdbg_setpidex(kd_regtype *kdr)
 	      kdebug_flags &= ~KDBG_PIDCHECK;
 	      kdebug_slowcheck |= SLOW_CHECKS;
 
-	      p->p_flag |= P_KDEBUG;
+	      p->p_kdebug = 1;
 	    }
 	  else  /* turn off pid exclusion for this pid value */
 	    {
 	      /* Don't turn off all pid exclusion though */
 	      /* kdebug_flags &= ~KDBG_PIDEXCLUDE;*/   
-	      p->p_flag &= ~P_KDEBUG;
+	      p->p_kdebug = 0;
 	    }
+	proc_rele(p);
 	}
     }
   else
@@ -993,6 +1001,22 @@ kdbg_getentropy (user_addr_t buffer, size_t *number, int ms_timeout)
 }
 
 
+static void
+kdbg_set_nkdbufs(unsigned int value)
+{
+        /*
+	 * We allow a maximum buffer size of 25% of either ram or max mapped address, whichever is smaller
+	 * 'value' is the desired number of trace entries
+	 */
+        unsigned int max_entries = (sane_size/4) / sizeof(kd_buf);
+
+	if (value <= max_entries)
+		nkdbufs = value;
+	else
+		nkdbufs = max_entries;
+}
+
+
 /*
  * This function is provided for the CHUD toolkit only.
  *    int val:
@@ -1021,9 +1045,8 @@ kdbg_control_chud(int val, void *fn)
 int
 kdbg_control(int *name, __unused u_int namelen, user_addr_t where, size_t *sizep)
 {
-    int ret=0;
+        int ret=0;
 	size_t size=*sizep;
-	unsigned int max_entries;
 	unsigned int value = name[1];
 	kd_regtype kd_Reg;
 	kbufinfo_t kd_bufinfo;
@@ -1106,7 +1129,7 @@ kdbg_control(int *name, __unused u_int namelen, user_addr_t where, size_t *sizep
         if (global_state_pid == -1)
 	    global_state_pid = curpid;
 	else if (global_state_pid != curpid) {
-	    if ((p = pfind(global_state_pid)) == NULL) {
+	    if ((p = proc_find(global_state_pid)) == NULL) {
 	        /*
 		 * The global pid no longer exists
 		 */
@@ -1115,6 +1138,7 @@ kdbg_control(int *name, __unused u_int namelen, user_addr_t where, size_t *sizep
 	        /*
 		 * The global pid exists, deny this request
 		 */
+		proc_rele(p);
 	        lck_mtx_unlock(kd_trace_mtx_sysctl);
 
 		return(EBUSY);
@@ -1151,13 +1175,7 @@ kdbg_control(int *name, __unused u_int namelen, user_addr_t where, size_t *sizep
 		    }
 		  break;
 		case KERN_KDSETBUF:
-		  /* We allow a maximum buffer size of 25% of either ram or max mapped address, whichever is smaller */
-		  /* 'value' is the desired number of trace entries */
-		        max_entries = (sane_size/4) / sizeof(kd_buf);
-			if (value <= max_entries)
-				nkdbufs = value;
-			else
-			  nkdbufs = max_entries;
+			kdbg_set_nkdbufs(value);
 			break;
 		case KERN_KDSETUP:
 			ret=kdbg_reinit();
@@ -1254,6 +1272,8 @@ kdbg_read(user_addr_t buffer, size_t *number)
 	uint32_t tempbuf_number;
 	unsigned int old_kdebug_flags, new_kdebug_flags;
 	unsigned int old_kdebug_slowcheck, new_kdebug_slowcheck;
+	boolean_t first_event = TRUE;
+
 	count = *number/sizeof(kd_buf);
 	*number = 0;
 
@@ -1305,6 +1325,9 @@ kdbg_read(user_addr_t buffer, size_t *number)
 	if ((tempbuf_count = count) > KDCOPYBUF_COUNT)
 	        tempbuf_count = KDCOPYBUF_COUNT;
 
+	if (last_wrap_cpu == -1)
+	        first_event = FALSE;
+
 	while (count) {
 	        tempbuf = kdcopybuf;
 		tempbuf_number = 0;
@@ -1329,10 +1352,29 @@ kdbg_read(user_addr_t buffer, size_t *number)
 				 */
 			        break;
 
+                        if (first_event == TRUE) {
+			        /*
+				 * make sure we leave room for the
+				 * LAST_WRAPPER event we inject
+				 * by throwing away the first event
+				 * it's better to lose that one
+				 * than the last one
+				 */	
+			        first_event = FALSE;
+
+				kdbip[mincpu].kd_readlast++;
+
+				if (kdbip[mincpu].kd_readlast == kdbip[mincpu].kd_buflast)
+				        kdbip[mincpu].kd_readlast = kdbip[mincpu].kd_buffer;
+				if (kdbip[mincpu].kd_readlast == kdbip[mincpu].kd_stop)
+				        kdbip[mincpu].kd_stop = 0;
+
+				continue;
+			}
 			if (last_wrap_cpu == mincpu) {
 			        tempbuf->debugid = MISCDBG_CODE(DBG_BUFFER, 0) | DBG_FUNC_NONE;
-				tempbuf->arg1 = 0;
-				tempbuf->arg2 = 0;
+				tempbuf->arg1 = kd_bufsize / sizeof(kd_buf);
+				tempbuf->arg2 = kd_cpus;
 				tempbuf->arg3 = 0;
 				tempbuf->arg4 = 0;
 				tempbuf->arg5 = (int)current_thread();
@@ -1405,7 +1447,7 @@ unsigned char *getProcName(struct proc *proc) {
 #define TRAP_DEBUGGER __asm__ volatile("int3");
 #endif
 #ifdef __ppc__
-#define TRAP_DEBUGGER   __asm__ volatile("tw 4,r3,r3");
+#define TRAP_DEBUGGER __asm__ volatile("tw 4,r3,r3");
 #endif
 
 #define SANE_TRACEBUF_SIZE 2*1024*1024
@@ -1483,6 +1525,11 @@ stack_snapshot2(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_
 /* Trap to the debugger to obtain a coherent stack snapshot; this populates
  * the trace buffer
  */
+	if (panic_active()) {
+		error = ENOMEM;
+		goto error_exit;
+	}
+
 	TRAP_DEBUGGER;
 
 	bytesTraced = kdp_stack_snapshot_bytes_traced();
@@ -1513,3 +1560,16 @@ error_exit:
 	STACKSHOT_SUBSYS_UNLOCK();
 	return error;
 }
+
+void
+start_kern_tracing(unsigned int new_nkdbufs) {
+	if (!new_nkdbufs)
+		return;
+	kdbg_set_nkdbufs(new_nkdbufs);
+	kdbg_lock_init();
+	kdbg_reinit();
+	kdebug_enable |= KDEBUG_ENABLE_TRACE;
+	kdebug_slowcheck &= ~SLOW_NOLOG;
+	kdbg_mapinit();
+	printf("kernel tracing started\n");
+}
diff --git a/bsd/kern/kern_acct.c b/bsd/kern/kern_acct.c
index 427070348..1e378b1db 100644
--- a/bsd/kern/kern_acct.c
+++ b/bsd/kern/kern_acct.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -64,6 +70,12 @@
  *	Purged old history
  *  	New version based on 4.4
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 
 #include <sys/param.h>
@@ -82,6 +94,9 @@
 #include <sys/tty.h>
 #include <sys/sysproto.h>
 #include <machine/spl.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 /*
  * The routines implemented in this file are described in:
@@ -129,17 +144,16 @@ int	acctchkfreq = 15;	/* frequency (in seconds) to check space */
  * previous implementation done by Mark Tinguely.
  */
 int
-acct(struct proc *p, struct acct_args *uap, __unused int *retval)
+acct(proc_t p, struct acct_args *uap, __unused int *retval)
 {
 	struct nameidata nd;
 	int error;
-	struct vfs_context context;
+	struct vfs_context *ctx; 
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	ctx = vfs_context_current();
 
 	/* Make sure that the caller is root. */
-	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
+	if ((error = suser(vfs_context_ucred(ctx), &p->p_acflag)))
 		return (error);
 
 	/*
@@ -147,16 +161,31 @@ acct(struct proc *p, struct acct_args *uap, __unused int *retval)
 	 * writing and make sure it's a 'normal'.
 	 */
 	if (uap->path != USER_ADDR_NULL) {
-		NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_USERSPACE, uap->path, &context);
+		NDINIT(&nd, LOOKUP, NOFOLLOW, UIO_USERSPACE, uap->path, ctx);
 		if ((error = vn_open(&nd, FWRITE, 0)))
 			return (error);
+#if CONFIG_MACF
+		error = mac_system_check_acct(vfs_context_ucred(ctx), nd.ni_vp);
+		if (error) {
+			vnode_put(nd.ni_vp);
+			vn_close(nd.ni_vp, FWRITE, ctx);
+			return (error);
+		}
+#endif
 		vnode_put(nd.ni_vp);
 
 		if (nd.ni_vp->v_type != VREG) {
-			vn_close(nd.ni_vp, FWRITE, kauth_cred_get(), p);
+			vn_close(nd.ni_vp, FWRITE, ctx);
 			return (EACCES);
 		}
 	}
+#if CONFIG_MACF
+	else {
+		error = mac_system_check_acct(vfs_context_ucred(ctx), NULL);
+		if (error)
+			return (error);
+	}
+#endif
 
 	/*
 	 * If accounting was previously enabled, kill the old space-watcher,
@@ -164,8 +193,8 @@ acct(struct proc *p, struct acct_args *uap, __unused int *retval)
 	 */
 	if (acctp != NULLVP || suspend_acctp != NULLVP) {
 		untimeout(acctwatch_funnel, NULL);
-		error = vn_close((acctp != NULLVP ? acctp : suspend_acctp), FWRITE,
-		    kauth_cred_get(), p);
+		error = vn_close((acctp != NULLVP ? acctp : suspend_acctp),
+				FWRITE, vfs_context_current());
 
 		acctp = suspend_acctp = NULLVP;
 	}
@@ -188,16 +217,17 @@ acct(struct proc *p, struct acct_args *uap, __unused int *retval)
  * "acct.h" header file.)
  */
 int
-acct_process(p)
-	struct proc *p;
+acct_process(proc_t p)
 {
 	struct acct an_acct;
-	struct rusage *r;
+	struct rusage rup, *r;
 	struct timeval ut, st, tmp;
 	int t;
 	int error;
 	struct vnode *vp;
 	kauth_cred_t safecred;
+	struct session * sessp;
+	boolean_t fstate;
 
 	/* If accounting isn't enabled, don't bother */
 	vp = acctp;
@@ -217,13 +247,16 @@ acct_process(p)
 	an_acct.ac_stime = encode_comp_t(st.tv_sec, st.tv_usec);
 
 	/* (3) The elapsed time the commmand ran (and its starting time) */
-	an_acct.ac_btime = p->p_stats->p_start.tv_sec;
+	an_acct.ac_btime = p->p_start.tv_sec;
 	microtime(&tmp);
-	timevalsub(&tmp, &p->p_stats->p_start);
+	timevalsub(&tmp, &p->p_start);
 	an_acct.ac_etime = encode_comp_t(tmp.tv_sec, tmp.tv_usec);
 
 	/* (4) The average amount of memory used */
-	r = &p->p_stats->p_ru;
+	proc_lock(p);
+	rup = p->p_stats->p_ru;
+	proc_unlock(p);
+	r = &rup;
 	tmp = ut;
 	timevaladd(&tmp, &st);
 	t = tmp.tv_sec * hz + tmp.tv_usec / tick;
@@ -236,19 +269,27 @@ acct_process(p)
 	an_acct.ac_io = encode_comp_t(r->ru_inblock + r->ru_oublock, 0);
 
 	/* (6) The UID and GID of the process */
-	an_acct.ac_uid = p->p_ucred->cr_ruid;
-	an_acct.ac_gid = p->p_ucred->cr_rgid;
+	safecred = kauth_cred_proc_ref(p);
+
+	an_acct.ac_uid = safecred->cr_ruid;
+	an_acct.ac_gid = safecred->cr_rgid;
 
 	/* (7) The terminal from which the process was started */
-	if ((p->p_flag & P_CONTROLT) && p->p_pgrp->pg_session->s_ttyp)
-		an_acct.ac_tty = p->p_pgrp->pg_session->s_ttyp->t_dev;
-	else
+	
+	sessp = proc_session(p);
+	if ((p->p_flag & P_CONTROLT) && (sessp != SESSION_NULL) && (sessp->s_ttyp != TTY_NULL)) {
+		fstate = thread_funnel_set(kernel_flock, TRUE);
+		an_acct.ac_tty = sessp->s_ttyp->t_dev;
+		(void) thread_funnel_set(kernel_flock, fstate);
+	 }else
 		an_acct.ac_tty = NODEV;
 
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+
 	/* (8) The boolean flags that tell how the process terminated, etc. */
 	an_acct.ac_flag = p->p_acflag;
 
-	safecred = kauth_cred_proc_ref(p);
 	/*
 	 * Now, just write the accounting information to the file.
 	 */
@@ -258,6 +299,7 @@ acct_process(p)
 				(int *)0, p);
 		vnode_put(vp);
 	}
+
 	kauth_cred_unref(&safecred);
 	return (error);
 }
@@ -273,8 +315,7 @@ acct_process(p)
 #define	MAXFRACT	((1 << MANTSIZE) - 1)	/* Maximum fractional value. */
 
 comp_t
-encode_comp_t(s, us)
-	u_long s, us;
+encode_comp_t(u_long s, u_long us)
 {
 	int exp, rnd;
 
@@ -302,8 +343,7 @@ encode_comp_t(s, us)
 }
 
 void
-acctwatch_funnel(a)
-	void *a;
+acctwatch_funnel(void *a)
 {
         thread_funnel_set(kernel_flock, TRUE);
 	acctwatch(a);
@@ -321,14 +361,12 @@ acctwatch_funnel(a)
 void
 acctwatch(__unused void *a)
 {
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct vfs_attr va;
 
 	VFSATTR_INIT(&va);
 	VFSATTR_WANTED(&va, f_blocks);
 	VFSATTR_WANTED(&va, f_bavail);
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
 
 	if (suspend_acctp != NULLVP) {
 		/*
@@ -337,11 +375,11 @@ acctwatch(__unused void *a)
 		 * below a low watermark
 		 */
 		if (suspend_acctp->v_type == VBAD) {
-			(void) vn_close(suspend_acctp, FWRITE, NOCRED, NULL);
+			(void) vn_close(suspend_acctp, FWRITE, vfs_context_kernel());
 			suspend_acctp = NULLVP;
 			return;
 		}
-		(void)vfs_getattr(suspend_acctp->v_mount, &va, &context);
+		(void)vfs_getattr(suspend_acctp->v_mount, &va, ctx);
 		if (va.f_bavail > acctresume * va.f_blocks / 100) {
 			acctp = suspend_acctp;
 			suspend_acctp = NULLVP;
@@ -354,11 +392,11 @@ acctwatch(__unused void *a)
 		 * goes over a high watermark
 		 */
 		if (acctp->v_type == VBAD) {
-			(void) vn_close(acctp, FWRITE, NOCRED, NULL);
+			(void) vn_close(acctp, FWRITE, vfs_context_kernel());
 			acctp = NULLVP;
 			return;
 		}
-		(void)vfs_getattr(acctp->v_mount, &va, &context);
+		(void)vfs_getattr(acctp->v_mount, &va, ctx);
 		if (va.f_bavail <= acctsuspend * va.f_blocks / 100) {
 			suspend_acctp = acctp;
 			acctp = NULLVP;
diff --git a/bsd/kern/kern_aio.c b/bsd/kern/kern_aio.c
index 58f79b324..813456ab9 100644
--- a/bsd/kern/kern_aio.c
+++ b/bsd/kern/kern_aio.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -129,8 +135,8 @@ typedef struct aio_anchor_cb aio_anchor_cb;
  * us sleep channels that currently do not collide with any other kernel routines.
  * At this time, for binary compatibility reasons, we cannot create new proc fields.
  */
-#define AIO_SUSPEND_SLEEP_CHAN  p_estcpu
-#define AIO_CLEANUP_SLEEP_CHAN 	p_pctcpu
+#define AIO_SUSPEND_SLEEP_CHAN  aio_active_count
+#define AIO_CLEANUP_SLEEP_CHAN 	aio_done_count
 
 
 /*
@@ -143,20 +149,20 @@ typedef struct aio_anchor_cb aio_anchor_cb;
 /*
  *  LOCAL PROTOTYPES
  */
-static int			aio_active_requests_for_process( struct proc *procp );
+static int			aio_active_requests_for_process(proc_t procp );
 static boolean_t	aio_delay_fsync_request( aio_workq_entry *entryp );
 static int			aio_free_request( aio_workq_entry *entryp, vm_map_t the_map );
 static int			aio_get_all_queues_count( void );
-static int			aio_get_process_count( struct proc *procp );
+static int			aio_get_process_count(proc_t procp );
 static aio_workq_entry *  aio_get_some_work( void );
 static boolean_t	aio_last_group_io( aio_workq_entry *entryp );
 static void			aio_mark_requests( aio_workq_entry *entryp );
-static int			aio_queue_async_request( struct proc *procp, 
+static int			aio_queue_async_request(proc_t procp, 
 									 		 user_addr_t aiocbp,
 									   		 int kindOfIO );
 static int			aio_validate( aio_workq_entry *entryp );
 static void			aio_work_thread( void );
-static int			do_aio_cancel(	struct proc *p, 
+static int			do_aio_cancel(proc_t p, 
 									int fd, 
 									user_addr_t aiocbp, 
 									boolean_t wait_for_completion,
@@ -166,14 +172,14 @@ static int			do_aio_fsync( aio_workq_entry *entryp );
 static int			do_aio_read( aio_workq_entry *entryp );
 static int			do_aio_write( aio_workq_entry *entryp );
 static void 		do_munge_aiocb( struct aiocb *my_aiocbp, struct user_aiocb *the_user_aiocbp );
-static boolean_t	is_already_queued( 	struct proc *procp, 
+static boolean_t	is_already_queued(proc_t procp, 
 										user_addr_t aiocbp );
-static int			lio_create_async_entry( struct proc *procp, 
+static int			lio_create_async_entry(proc_t procp, 
 											 user_addr_t aiocbp, 
 						 					 user_addr_t sigp, 
 						 					 long group_tag,
 						 					 aio_workq_entry **entrypp );
-static int			lio_create_sync_entry( struct proc *procp, 
+static int			lio_create_sync_entry(proc_t procp, 
 											user_addr_t aiocbp, 
 											long group_tag,
 											aio_workq_entry **entrypp );
@@ -184,10 +190,10 @@ static int			lio_create_sync_entry( struct proc *procp,
  */
 
 /* in ...bsd/kern/sys_generic.c */
-extern int			dofileread( struct proc *p, struct fileproc *fp, int fd, 
+extern int			dofileread(vfs_context_t ctx, struct fileproc *fp,
 								user_addr_t bufp, user_size_t nbyte, 
 								off_t offset, int flags, user_ssize_t *retval );
-extern int			dofilewrite( struct proc *p, struct fileproc *fp, int fd, 
+extern int			dofilewrite(vfs_context_t ctx, struct fileproc *fp,
 								 user_addr_t bufp, user_size_t nbyte, off_t offset, 
 								 int flags, user_ssize_t *retval );
 
@@ -221,7 +227,7 @@ static struct zone  		*aio_workq_zonep;
  */
 
 int
-aio_cancel( struct proc *p, struct aio_cancel_args *uap, int *retval )
+aio_cancel(proc_t p, struct aio_cancel_args *uap, int *retval )
 {
 	struct user_aiocb		my_aiocb;
 	int							result;
@@ -234,7 +240,8 @@ aio_cancel( struct proc *p, struct aio_cancel_args *uap, int *retval )
 	result = aio_get_all_queues_count( );
 	AIO_UNLOCK;
 	if ( result < 1 ) {
-		result = EBADF;
+		result = 0;
+		*retval = AIO_ALLDONE;
 		goto ExitRoutine;
 	}
 	
@@ -289,7 +296,7 @@ ExitRoutine:
  */
 
 __private_extern__ void
-_aio_close( struct proc *p, int fd )
+_aio_close(proc_t p, int fd )
 {
 	int			error, count;
 
@@ -337,7 +344,7 @@ _aio_close( struct proc *p, int fd )
  */
 
 int
-aio_error( struct proc *p, struct aio_error_args *uap, int *retval )
+aio_error(proc_t p, struct aio_error_args *uap, int *retval )
 {
 	aio_workq_entry		 		*entryp;
 	int							error;
@@ -406,7 +413,7 @@ ExitRoutine:
  */
 
 int
-aio_fsync( struct proc *p, struct aio_fsync_args *uap, int *retval )
+aio_fsync(proc_t p, struct aio_fsync_args *uap, int *retval )
 {
 	int			error;
 	int			fsync_kind;
@@ -447,7 +454,7 @@ ExitRoutine:
  */
 
 int
-aio_read( struct proc *p, struct aio_read_args *uap, int *retval )
+aio_read(proc_t p, struct aio_read_args *uap, int *retval )
 {
 	int			error;
 
@@ -477,7 +484,7 @@ aio_read( struct proc *p, struct aio_read_args *uap, int *retval )
  */
 
 int
-aio_return( struct proc *p, struct aio_return_args *uap, user_ssize_t *retval )
+aio_return(proc_t p, struct aio_return_args *uap, user_ssize_t *retval )
 {
 	aio_workq_entry		 		*entryp;
 	int							error;
@@ -566,7 +573,7 @@ ExitRoutine:
  */
 
 __private_extern__ void
-_aio_exec( struct proc *p )
+_aio_exec(proc_t p )
 {
 
 	KERNEL_DEBUG( (BSDDBG_CODE(DBG_BSD_AIO, AIO_exec)) | DBG_FUNC_START,
@@ -590,7 +597,7 @@ _aio_exec( struct proc *p )
  */
 
 __private_extern__ void
-_aio_exit( struct proc *p )
+_aio_exit(proc_t p )
 {
 	int						error, count;
 	aio_workq_entry 		*entryp;
@@ -684,7 +691,7 @@ _aio_exit( struct proc *p )
  */
 
 static int
-do_aio_cancel( 	struct proc *p, int fd, user_addr_t aiocbp, 
+do_aio_cancel(proc_t p, int fd, user_addr_t aiocbp, 
 				boolean_t wait_for_completion, boolean_t disable_notification )
 {
 	aio_workq_entry		 	*entryp;
@@ -853,9 +860,16 @@ do_aio_cancel( 	struct proc *p, int fd, user_addr_t aiocbp,
  * set appropriately - EAGAIN if timeout elapses or EINTR if an interrupt
  * woke us up.
  */
+int
+aio_suspend(proc_t p, struct aio_suspend_args *uap, int *retval )
+{
+	__pthread_testcancel(1);
+	return(aio_suspend_nocancel(p, (struct aio_suspend_nocancel_args *)uap, retval));
+}
+
 
 int
-aio_suspend( struct proc *p, struct aio_suspend_args *uap, int *retval )
+aio_suspend_nocancel(proc_t p, struct aio_suspend_nocancel_args *uap, int *retval )
 {
 	int					error;
 	int					i, count;
@@ -902,7 +916,7 @@ aio_suspend( struct proc *p, struct aio_suspend_args *uap, int *retval )
 			goto ExitThisRoutine;
 		}
 			
-		if ( ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000 ) {
+		if ( ts.tv_sec < 0 || ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000 ) {
 			error = EINVAL;
 			goto ExitThisRoutine;
 		}
@@ -940,6 +954,7 @@ aio_suspend( struct proc *p, struct aio_suspend_args *uap, int *retval )
 	}
 	
 	/* check list of aio requests to see if any have completed */
+check_for_our_aiocbp:
 	AIO_LOCK;
 	for ( i = 0; i < uap->nent; i++ ) {
 		user_addr_t	aiocbp;  
@@ -975,9 +990,15 @@ aio_suspend( struct proc *p, struct aio_suspend_args *uap, int *retval )
 	error = thread_block( THREAD_CONTINUE_NULL );
 
 	if ( error == THREAD_AWAKENED ) {
-		/* got our wakeup call from aio_work_thread() */
-		*retval = 0;
-		error = 0;
+		/* 
+		 * got our wakeup call from aio_work_thread().
+		 * Since we can get a wakeup on this channel from another thread in the 
+		 * same process we head back up to make sure this is for the correct aiocbp.  
+		 * If it is the correct aiocbp we will return from where we do the check 
+		 * (see entryp->uaiocbp == aiocbp after check_for_our_aiocbp label)
+		 * else we will fall out and just sleep again.  
+		 */
+		goto check_for_our_aiocbp;
 	}
 	else if ( error == THREAD_TIMED_OUT ) {
 		/* our timeout expired */
@@ -1006,7 +1027,7 @@ ExitThisRoutine:
  */
 
 int
-aio_write( struct proc *p, struct aio_write_args *uap, int *retval )
+aio_write(proc_t p, struct aio_write_args *uap, int *retval )
 {
 	int			error;
 	
@@ -1036,7 +1057,7 @@ aio_write( struct proc *p, struct aio_write_args *uap, int *retval )
  */
 
 int
-lio_listio( struct proc *p, struct lio_listio_args *uap, int *retval )
+lio_listio(proc_t p, struct lio_listio_args *uap, int *retval )
 {
 	int							i;
 	int							call_result;
@@ -1442,7 +1463,7 @@ aio_delay_fsync_request( aio_workq_entry *entryp )
  */
 
 static int
-aio_queue_async_request( struct proc *procp, user_addr_t aiocbp, int kindOfIO )
+aio_queue_async_request(proc_t procp, user_addr_t aiocbp, int kindOfIO )
 {
 	aio_workq_entry		 	*entryp;
 	int						result;
@@ -1545,7 +1566,7 @@ error_exit:
  */
 
 static int
-lio_create_async_entry( struct proc *procp, user_addr_t aiocbp, 
+lio_create_async_entry(proc_t procp, user_addr_t aiocbp, 
 						 user_addr_t sigp, long group_tag,
 						 aio_workq_entry **entrypp )
 {
@@ -1681,7 +1702,7 @@ aio_mark_requests( aio_workq_entry *entryp )
  */
 
 static int
-lio_create_sync_entry( struct proc *procp, user_addr_t aiocbp, 
+lio_create_sync_entry(proc_t procp, user_addr_t aiocbp, 
 						long group_tag, aio_workq_entry **entrypp )
 {
 	aio_workq_entry		 		*entryp;
@@ -1850,7 +1871,7 @@ aio_validate( aio_workq_entry *entryp )
  */
 
 static int
-aio_get_process_count( struct proc *procp ) 
+aio_get_process_count(proc_t procp ) 
 {
 	aio_workq_entry		 		*entryp;
 	int							count;
@@ -2018,8 +2039,9 @@ aio_last_group_io( aio_workq_entry *entryp )
 static int
 do_aio_read( aio_workq_entry *entryp )
 {
-	struct fileproc 			*fp;
-	int						error;
+	struct fileproc		*fp;
+	int					error;
+	struct vfs_context	context;
 
 	if ( (error = fp_lookup(entryp->procp, entryp->aiocb.aio_fildes, &fp , 0)) )
 		return(error);
@@ -2027,18 +2049,20 @@ do_aio_read( aio_workq_entry *entryp )
 		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
 		return(EBADF);
 	}
-	if ( fp != NULL ) {
-		error = dofileread( entryp->procp, fp, entryp->aiocb.aio_fildes, 
-							entryp->aiocb.aio_buf, 
-							entryp->aiocb.aio_nbytes,
-							entryp->aiocb.aio_offset, FOF_OFFSET, 
-							&entryp->returnval );
-		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
-	}
-	else {
-		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
-		error = EBADF;
-	}
+
+	/*
+	 * <rdar://4714366>
+	 * Needs vfs_context_t from vfs_context_create() in entryp!
+	 */
+	context.vc_thread = proc_thread(entryp->procp);	/* XXX */
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
+	error = dofileread(&context, fp, 
+				entryp->aiocb.aio_buf, 
+				entryp->aiocb.aio_nbytes,
+				entryp->aiocb.aio_offset, FOF_OFFSET, 
+				&entryp->returnval);
+	fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
 			
 	return( error );
 	
@@ -2053,6 +2077,7 @@ do_aio_write( aio_workq_entry *entryp )
 {
 	struct fileproc 		*fp;
 	int						error;
+	struct vfs_context		context;
 
 	if ( (error = fp_lookup(entryp->procp, entryp->aiocb.aio_fildes, &fp , 0)) )
 		return(error);
@@ -2060,23 +2085,24 @@ do_aio_write( aio_workq_entry *entryp )
 		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
 		return(EBADF);
 	}
-	if ( fp != NULL ) {
-		/* NB: tell dofilewrite the offset, and to use the proc cred */
-		error = dofilewrite( entryp->procp,
-				     fp,
-				     entryp->aiocb.aio_fildes,
-				     entryp->aiocb.aio_buf,
-				     entryp->aiocb.aio_nbytes,
-				     entryp->aiocb.aio_offset,
-				     FOF_OFFSET | FOF_PCRED,
-				     &entryp->returnval);
-		
-		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
-	}
-	else {
-		fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
-		error = EBADF;
-	}
+
+	/*
+	 * <rdar://4714366>
+	 * Needs vfs_context_t from vfs_context_create() in entryp!
+	 */
+	context.vc_thread = proc_thread(entryp->procp);	/* XXX */
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
+	/* NB: tell dofilewrite the offset, and to use the proc cred */
+	error = dofilewrite(&context,
+				fp,
+				entryp->aiocb.aio_buf,
+				entryp->aiocb.aio_nbytes,
+				entryp->aiocb.aio_offset,
+				FOF_OFFSET | FOF_PCRED,
+				&entryp->returnval);
+	
+	fp_drop(entryp->procp, entryp->aiocb.aio_fildes, fp, 0);
 
 	return( error );
 
@@ -2090,7 +2116,7 @@ do_aio_write( aio_workq_entry *entryp )
  */
 
 static int
-aio_active_requests_for_process( struct proc *procp )
+aio_active_requests_for_process(proc_t procp )
 {
 				
 	return( procp->aio_active_count );
@@ -2122,7 +2148,7 @@ do_aio_fsync( aio_workq_entry *entryp )
 			entryp->returnval = -1;
 			return(error);
 		}
-		context.vc_proc = entryp->procp;
+		context.vc_thread = current_thread();
 		context.vc_ucred = fp->f_fglob->fg_cred;
 
 		error = VNOP_FSYNC( vp, MNT_WAIT, &context);
@@ -2147,7 +2173,7 @@ do_aio_fsync( aio_workq_entry *entryp )
  */
 
 static boolean_t
-is_already_queued( 	struct proc *procp, 
+is_already_queued(proc_t procp, 
 					user_addr_t aiocbp ) 
 {
 	aio_workq_entry		 	*entryp;
diff --git a/bsd/kern/kern_audit.c b/bsd/kern/kern_audit.c
index 6be406b5d..df9f61d2d 100644
--- a/bsd/kern/kern_audit.c
+++ b/bsd/kern/kern_audit.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 #include <sys/param.h>
 #include <sys/fcntl.h>
@@ -61,12 +73,21 @@
 #include <kern/wait_queue.h>
 #include <kern/sched_prim.h>
 
+#if CONFIG_MACF
+#include <bsm/audit_record.h>
+#include <security/mac.h>
+#include <security/mac_framework.h>
+#include <security/mac_policy.h>
+#define MAC_ARG_PREFIX "arg: "
+#define MAC_ARG_PREFIX_LEN 5
+#endif
+
 #include <net/route.h>
 
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 
-#ifdef AUDIT
+#if AUDIT
 
 /*
  * The AUDIT_EXCESSIVELY_VERBOSE define enables a number of
@@ -88,7 +109,7 @@
 #undef assert()
 #endif
 #define assert(cond)    \
-    ((void) ((cond) ? 0 : panic("%s:%d (%s)", __FILE__, __LINE__, # cond)))
+    ((void) ((cond) ? 0 : panic("Assert failed: %s", # cond)))
 #else
 #include <kern/assert.h>
 #endif /* DIAGNOSTIC */
@@ -103,7 +124,10 @@ int	audit_suspended;
  * Mutex to protect global variables shared between various threads and
  * processes.
  */
-static mutex_t				*audit_mtx;
+static lck_grp_t			*audit_grp;
+static lck_attr_t			*audit_attr;
+static lck_grp_attr_t			*audit_grp_attr;
+static lck_mtx_t			*audit_mtx;
 
 /*
  * Queue of audit records ready for delivery to disk.  We insert new
@@ -119,6 +143,9 @@ static size_t				audit_pre_q_len;
 
 static wait_queue_t			audit_wait_queue;
 static zone_t				audit_zone;
+#if CONFIG_MACF
+static zone_t				audit_mac_label_zone;
+#endif
 
 /*
  * Condition variable to signal to the worker that it has work to do:
@@ -210,6 +237,7 @@ static int audit_failure_event;
  */
 extern task_t kernel_task;
 
+extern zone_t mac_audit_data_zone;
 static void
 audit_free(struct kaudit_record *ar)
 {
@@ -234,38 +262,76 @@ audit_free(struct kaudit_record *ar)
 	}
 	if (ar->k_udata != NULL) {
 		kfree(ar->k_udata, ar->k_ulen);
+	}
 
+#if CONFIG_MACF
+	if (ar->k_ar.ar_vnode1_mac_labels != NULL) {
+		zfree(audit_mac_label_zone, ar->k_ar.ar_vnode1_mac_labels);
 	}
+	if (ar->k_ar.ar_vnode2_mac_labels != NULL) {
+		zfree(audit_mac_label_zone, ar->k_ar.ar_vnode2_mac_labels);
+	}
+	if (ar->k_ar.ar_cred_mac_labels != NULL) {
+		zfree(audit_mac_label_zone, ar->k_ar.ar_cred_mac_labels);
+	}
+	if (ar->k_ar.ar_arg_mac_string != NULL) {
+		kfree(ar->k_ar.ar_arg_mac_string,
+		    MAC_MAX_LABEL_BUF_LEN + MAC_ARG_PREFIX_LEN);
+	}
+
+	/* Free the audit data from the MAC policies. */
+	do {
+		struct mac_audit_record *head, *next;
+
+		head = LIST_FIRST(ar->k_ar.ar_mac_records);
+		while (head != NULL) {
+			next = LIST_NEXT(head, records);
+			zfree(mac_audit_data_zone, head->data);
+			kfree(head, sizeof(*head));
+			head = next;
+		}
+
+		kfree(ar->k_ar.ar_mac_records,
+		    sizeof(*ar->k_ar.ar_mac_records));
+	} while (0);
+#endif
+
 	zfree(audit_zone, ar);
 }
 
+/*
+ * Converts an audit record into the BSM format before writing out to the 
+ * audit logfile. Will perform it's own vnode iocounting.
+ *
+ * Returns: 
+ *	-1 if it could not get an ioreference on the vnode.
+ *	EINVAL if the kaudit_record ar is not a valid audit record.
+ */
 static int
-audit_write(struct vnode *vp, struct kaudit_record *ar, kauth_cred_t cred,
-    struct proc *p)
+audit_write(struct vnode *vp, struct kaudit_record *ar, vfs_context_t ctx)
 {
 	struct vfsstatfs *mnt_stat = &vp->v_mount->mnt_vfsstat;
-	int ret;
+	int ret = 0;
 	struct au_record *bsm;
-	/* KVV maybe we should take a context as a param to audit_write? */
-	struct vfs_context context;
 	off_t file_size;
 
 	mach_port_t audit_port;
 
+	if (vnode_getwithref(vp))
+		return ENOENT;
+
 	/* 
 	 * First, gather statistics on the audit log file and file system
 	 * so that we know how we're doing on space.  In both cases,
 	 * if we're unable to perform the operation, we drop the record
 	 * and return.  However, this is arguably an assertion failure.
 	 */
-	context.vc_proc = p;
-	context.vc_ucred = cred;
-	ret = vfs_update_vfsstat(vp->v_mount, &context);
+	ret = vfs_update_vfsstat(vp->v_mount, ctx, VFS_KERNEL_EVENT);
 	if (ret)
 		goto out;
 
 	/* update the global stats struct */
-	if ((ret = vnode_size(vp, &file_size, &context)) != 0)
+	if ((ret = vnode_size(vp, &file_size, ctx)) != 0)
 		goto out;
 	audit_fstat.af_currsz = file_size;
 	
@@ -308,7 +374,7 @@ audit_write(struct vnode *vp, struct kaudit_record *ar, kauth_cred_t cred,
 		 */
 		if ((audit_fstat.af_filesz != 0) &&
 		    (audit_file_rotate_wait == 0) && 
-		    (file_size >= audit_fstat.af_filesz)) {
+		    (file_size >= (off_t)audit_fstat.af_filesz)) {
 			audit_file_rotate_wait = 1;
 			ret = audit_triggers(audit_port, 
 				AUDIT_TRIGGER_FILE_FULL); 
@@ -352,15 +418,10 @@ audit_write(struct vnode *vp, struct kaudit_record *ar, kauth_cred_t cred,
 	 * we ignore errors.
 	 */
 	if (ar->k_ar_commit & AR_COMMIT_USER) {
-	        if (vnode_getwithref(vp) == 0) {
-		        ret = vn_rdwr(UIO_WRITE, vp, (void *)ar->k_udata, ar->k_ulen,
-				(off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, cred, NULL, p);
-			vnode_put(vp);
-			if (ret)
-				goto out;
-		} else {
+		ret = vn_rdwr(UIO_WRITE, vp, (void *)ar->k_udata, ar->k_ulen,
+			(off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, vfs_context_ucred(ctx), NULL, vfs_context_proc(ctx));
+		if (ret)
 			goto out;
-		}
 	}
 
 	/* 
@@ -388,17 +449,13 @@ audit_write(struct vnode *vp, struct kaudit_record *ar, kauth_cred_t cred,
 		goto out;
 	}
 	
-	/* XXX This function can be called with the kernel funnel held,
-	 * which is not optimal. We should break the write functionality
+	/* XXX:  We should break the write functionality
 	 * away from the BSM record generation and have the BSM generation
 	 * done before this function is called. This function will then
 	 * take the BSM record as a parameter.
 	 */
-	if ((ret = vnode_getwithref(vp)) == 0) {
-	        ret = (vn_rdwr(UIO_WRITE, vp, (void *)bsm->data, bsm->len,
-			       (off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, cred, NULL, p));
-		vnode_put(vp);
-	}
+	ret = (vn_rdwr(UIO_WRITE, vp, (void *)bsm->data, bsm->len,
+		       (off_t)0, UIO_SYSSPACE32, IO_APPEND|IO_UNIT, vfs_context_ucred(ctx), NULL, vfs_context_proc(ctx)));
 	kau_free(bsm);
 
 out:
@@ -410,23 +467,23 @@ out:
 	 */
 	if (audit_in_failure &&
 	    audit_q_len == 0 && audit_pre_q_len == 0) {
-		(void)VNOP_FSYNC(vp, MNT_WAIT, &context);
+		(void)VNOP_FSYNC(vp, MNT_WAIT, ctx);
 		panic("Audit store overflow; record queue drained.");
 	}
 
+	vnode_put(vp);
 	return (ret);
 }
 
 static void
 audit_worker(void)
 {
-	int do_replacement_signal, error, release_funnel;
+	int do_replacement_signal, error;
 	TAILQ_HEAD(, kaudit_record) ar_worklist;
 	struct kaudit_record *ar;
 	struct vnode *audit_vp, *old_vp;
 	kauth_cred_t audit_cred;
-	kauth_cred_t old_cred;
-	struct proc *audit_p;
+	proc_t audit_p;
 
 	AUDIT_PRINTF(("audit_worker starting\n"));
 
@@ -435,20 +492,13 @@ audit_worker(void)
 	audit_p = current_proc();
 	audit_vp = NULL;
 
-	/*
-	 * XXX: Presumably we can assume Mach threads are started without
-	 * holding the BSD kernel funnel?
-	 */
-	thread_funnel_set(kernel_flock, FALSE);
 
-	mutex_lock(audit_mtx);
+	lck_mtx_lock(audit_mtx);
 	while (1) {
+		struct vfs_context context;
+
 		/*
 		 * First priority: replace the audit log target if requested.
-		 * As we actually close the vnode in the worker thread, we
-		 * need to grab the funnel, which means releasing audit_mtx.
-		 * In case another replacement was scheduled while the mutex
-		 * we released, we loop.
 		 *
 		 * XXX It could well be we should drain existing records
 		 * first to ensure that the timestamps and ordering
@@ -456,7 +506,8 @@ audit_worker(void)
 		 */
 		do_replacement_signal = 0;
 		while (audit_replacement_flag != 0) {
-			old_cred = audit_cred;
+			kauth_cred_t old_cred = audit_cred;
+
 			old_vp = audit_vp;
 			audit_cred = audit_replacement_cred;
 			audit_vp = audit_replacement_vp;
@@ -466,30 +517,24 @@ audit_worker(void)
 
 			audit_enabled = (audit_vp != NULL);
 
-			if (old_vp != NULL || audit_vp != NULL) {
-				mutex_unlock(audit_mtx);
-				thread_funnel_set(kernel_flock, TRUE);
-				release_funnel = 1;
-			} else
-				release_funnel = 0;
 			/*
 			 * XXX: What to do about write failures here?
 			 */
 			if (old_vp != NULL) {
-				AUDIT_PRINTF(("Closing old audit file\n"));
-				vn_close(old_vp, audit_close_flags, old_cred,
-				    audit_p);
+				AUDIT_PRINTF(("Closing old audit file vnode %p\n", old_vp));
+				if (vnode_get(old_vp) == 0) {
+					vn_close(old_vp, audit_close_flags, vfs_context_kernel());
+					vnode_put(old_vp); 
+					AUDIT_PRINTF(("Audit file closed\n"));
+				}
+				else
+					printf("audit_worker(): Couldn't close audit file.\n");
 				kauth_cred_unref(&old_cred);
 				old_vp = NULL;
-				AUDIT_PRINTF(("Audit file closed\n"));
 			}
 			if (audit_vp != NULL) {
 				AUDIT_PRINTF(("Opening new audit file\n"));
 			}
-			if (release_funnel) {
-				thread_funnel_set(kernel_flock, FALSE);
-				mutex_lock(audit_mtx);
-			}
 			do_replacement_signal = 1;
 		}
 		/*
@@ -516,7 +561,7 @@ audit_worker(void)
 						     AUDIT_WORKER_EVENT,
 						     THREAD_UNINT,
 						     0);
-			mutex_unlock(audit_mtx);
+			lck_mtx_unlock(audit_mtx);
 
 			assert(ret == THREAD_WAITING);
 			ret = thread_block(THREAD_CONTINUE_NULL);
@@ -525,7 +570,7 @@ audit_worker(void)
 	AUDIT_PRINTF(("audit_worker: new vp = %p; value of flag %d\n",
 	    audit_replacement_vp, audit_replacement_flag));
 
-			mutex_lock(audit_mtx);
+			lck_mtx_lock(audit_mtx);
 			continue;
 		}
 
@@ -554,12 +599,12 @@ audit_worker(void)
 
 				TAILQ_INSERT_TAIL(&ar_worklist, ar, k_q);
 			}
-			mutex_unlock(audit_mtx);
+			lck_mtx_unlock(audit_mtx);
 			while ((ar = TAILQ_FIRST(&ar_worklist))) {
 				TAILQ_REMOVE(&ar_worklist, ar, k_q);
 				audit_free(ar);
 			}
-			mutex_lock(audit_mtx);
+			lck_mtx_lock(audit_mtx);
 			continue;
 		}
 
@@ -585,8 +630,9 @@ audit_worker(void)
 
 			TAILQ_INSERT_TAIL(&ar_worklist, ar, k_q);
 		}
-		mutex_unlock(audit_mtx);
-		release_funnel = 0;
+		lck_mtx_unlock(audit_mtx);
+		context.vc_thread = current_thread();
+		context.vc_ucred = audit_cred;
 		while ((ar = TAILQ_FIRST(&ar_worklist))) {
 			TAILQ_REMOVE(&ar_worklist, ar, k_q);
 			if (audit_vp != NULL) {
@@ -594,41 +640,24 @@ audit_worker(void)
 				 * XXX: What should happen if there's a write
 				 * error here?
 				 */
-				if (!release_funnel) {
-					thread_funnel_set(kernel_flock, TRUE);
-					release_funnel = 1;
-				}
-				error = audit_write(audit_vp, ar, audit_cred,
-				    audit_p);
+				error = audit_write(audit_vp, ar, &context);
 				if (error && audit_panic_on_write_fail) {
 					panic("audit_worker: write error %d\n",
 					    error);
 				} else if (error) {
 					printf("audit_worker: write error %d\n",
 					    error);
-			}
+				}
 			}
 			audit_free(ar);
 		}
-		if (release_funnel)
-			thread_funnel_set(kernel_flock, FALSE);
-		mutex_lock(audit_mtx);
+		lck_mtx_lock(audit_mtx);
 	}
 }
 
 void
 audit_init(void)
 {
-
-	/* Verify that the syscall to audit event table is the same
-	 * size as the system call table.
-	 */
-	if (nsys_au_event != nsysent) {
-		printf("Security auditing service initialization failed, ");
-		printf("audit event table doesn't match syscall table.\n");
-		return;
-	}
-
 	printf("Security auditing service present\n");
 	TAILQ_INIT(&audit_q);
 	audit_q_len = 0;
@@ -645,12 +674,25 @@ audit_init(void)
 	audit_qctrl.aq_bufsz = AQ_BUFSZ;
 	audit_qctrl.aq_minfree = AU_FS_MINFREE;
 
-	audit_mtx = mutex_alloc(0);
+	audit_grp_attr = lck_grp_attr_alloc_init();
+	audit_grp = lck_grp_alloc_init("audit", audit_grp_attr);
+	audit_attr = lck_attr_alloc_init();
+	audit_mtx = lck_mtx_alloc_init(audit_grp, audit_attr);
+
 	audit_wait_queue = wait_queue_alloc(SYNC_POLICY_FIFO);
 	audit_zone = zinit(sizeof(struct kaudit_record), 
 			   AQ_HIWATER*sizeof(struct kaudit_record),
 			   8192,
 			   "audit_zone");
+#if CONFIG_MACF
+	/* Assume 3 MAC labels for each audit record: two for vnodes,
+	 * one for creds.
+	 */
+	audit_mac_label_zone = zinit(MAC_AUDIT_LABEL_LEN,
+				     AQ_HIWATER * 3*MAC_AUDIT_LABEL_LEN,
+				     8192,
+				     "audit_mac_label_zone");
+#endif
 
 	/* Initialize the BSM audit subsystem. */
 	kau_init();
@@ -665,7 +707,7 @@ audit_rotate_vnode(kauth_cred_t cred, struct vnode *vp)
 	 * If other parallel log replacements have been requested, we wait
 	 * until they've finished before continuing.
 	 */
-	mutex_lock(audit_mtx);
+	lck_mtx_lock(audit_mtx);
 	while (audit_replacement_flag != 0) {
 
 		AUDIT_PRINTF(("audit_rotate_vnode: sleeping to wait for "
@@ -674,7 +716,7 @@ audit_rotate_vnode(kauth_cred_t cred, struct vnode *vp)
 					     AUDIT_REPLACEMENT_EVENT,
 					     THREAD_UNINT,
 					     0);
-		mutex_unlock(audit_mtx);
+		lck_mtx_unlock(audit_mtx);
 
 		assert(ret == THREAD_WAITING);
 		ret = thread_block(THREAD_CONTINUE_NULL);
@@ -682,7 +724,7 @@ audit_rotate_vnode(kauth_cred_t cred, struct vnode *vp)
 		AUDIT_PRINTF(("audit_rotate_vnode: woken up (flag %d)\n",
 		    audit_replacement_flag));
 
-		mutex_lock(audit_mtx);
+		lck_mtx_lock(audit_mtx);
 	}
 	audit_replacement_cred = cred;
 	audit_replacement_flag = 1;
@@ -711,7 +753,7 @@ audit_rotate_vnode(kauth_cred_t cred, struct vnode *vp)
 				     AUDIT_REPLACEMENT_EVENT,
 				     THREAD_UNINT,
 				     0);
-	mutex_unlock(audit_mtx);
+	lck_mtx_unlock(audit_mtx);
 
 	assert(ret == THREAD_WAITING);
 	ret = thread_block(THREAD_CONTINUE_NULL);
@@ -728,7 +770,8 @@ audit_rotate_vnode(kauth_cred_t cred, struct vnode *vp)
 void
 audit_shutdown(void)
 {
-	audit_rotate_vnode(NULL, NULL);
+	if (audit_mtx)
+		audit_rotate_vnode(NULL, NULL);
 }
 
 static __inline__ struct uthread *
@@ -757,7 +800,7 @@ currecord(void)
  */
 /* ARGSUSED */
 int
-audit(struct proc *p, struct audit_args *uap, __unused register_t *retval)
+audit(proc_t p, struct audit_args *uap, __unused register_t *retval)
 {
 	int error;
 	void * rec;
@@ -768,8 +811,12 @@ audit(struct proc *p, struct audit_args *uap, __unused register_t *retval)
 	if (error)
 		return (error);
 
-	if ((uap->length <= 0) || (uap->length > (int)audit_qctrl.aq_bufsz))
+	lck_mtx_lock(audit_mtx);
+	if ((uap->length <= 0) || (uap->length > (int)audit_qctrl.aq_bufsz)) {
+		lck_mtx_unlock(audit_mtx);
 		return (EINVAL);
+	}
+	lck_mtx_unlock(audit_mtx);
 
 	ar = currecord();
 
@@ -800,6 +847,12 @@ audit(struct proc *p, struct audit_args *uap, __unused register_t *retval)
 	if (error)
 		goto free_out;
 
+#if CONFIG_MACF
+	error = mac_system_check_audit(kauth_cred_get(), rec, uap->length);
+	if (error)
+		goto free_out;
+#endif
+
 	/* Verify the record */
 	if (bsm_rec_verify(rec) == 0) {
 		error = EINVAL;
@@ -828,18 +881,25 @@ free_out:
  */
 /* ARGSUSED */
 int
-auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *retval)
+auditon(proc_t p, struct auditon_args *uap, __unused register_t *retval)
 {
 	int ret;
 	int len;
 	union auditon_udata udata;
-	struct proc *tp;
+	proc_t tp = PROC_NULL;
+	kauth_cred_t my_cred;
 
 	AUDIT_ARG(cmd, uap->cmd);
 	ret = suser(kauth_cred_get(), &p->p_acflag);
 	if (ret)
 		return (ret);
 
+#if CONFIG_MACF
+	ret = mac_system_check_auditon(kauth_cred_get(), uap->cmd);
+	if (ret)
+		return (ret);
+#endif
+
 	len = uap->length;
 	if ((len <= 0) || (len > (int)sizeof(union auditon_udata)))
 		return (EINVAL);
@@ -867,11 +927,12 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 			return (ret);
 		AUDIT_ARG(auditon, &udata);
 		break;
-}
+	}
 
 	/* XXX Need to implement these commands by accessing the global
 	 * values associated with the commands.
 	 */
+	lck_mtx_lock(audit_mtx);
 	switch (uap->cmd) {
 	case A_GETPOLICY:
 		if (!audit_fail_stop)
@@ -880,11 +941,13 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 			udata.au_policy |= AUDIT_AHLT;
 		break;
 	case A_SETPOLICY:
-		if (udata.au_policy & ~(AUDIT_CNT|AUDIT_AHLT))
-			return (EINVAL);
-/*
+		if (udata.au_policy & ~(AUDIT_CNT|AUDIT_AHLT)) {
+			ret = EINVAL;
+			break;
+		}
+		/*
 		 * XXX - Need to wake up waiters if the policy relaxes?
- */
+		 */
 		audit_fail_stop = ((udata.au_policy & AUDIT_CNT) == 0);
 		audit_panic_on_write_fail = (udata.au_policy & AUDIT_AHLT);
 		break;
@@ -902,30 +965,32 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 		    (udata.au_qctrl.aq_lowater >= udata.au_qctrl.aq_hiwater) ||
 		    (udata.au_qctrl.aq_bufsz > AQ_MAXBUFSZ) ||
 		    (udata.au_qctrl.aq_minfree < 0) ||
-		    (udata.au_qctrl.aq_minfree > 100))
-			return (EINVAL);
+		    (udata.au_qctrl.aq_minfree > 100)) {
+			ret = EINVAL;
+			break;
+		}
 
 		audit_qctrl = udata.au_qctrl;
 		/* XXX The queue delay value isn't used with the kernel. */
 		audit_qctrl.aq_delay = -1;
 		break;
 	case A_GETCWD:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_GETCAR:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_GETSTAT:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_SETSTAT:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_SETUMASK:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_SETSMASK:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_GETCOND:
 		if (audit_enabled && !audit_suspended)
@@ -952,27 +1017,44 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 					udata.au_evclass.ec_class);
 		break;
 	case A_GETPINFO:
-		if (udata.au_aupinfo.ap_pid < 1) 
-			return (EINVAL);
-		if ((tp = pfind(udata.au_aupinfo.ap_pid)) == NULL)
-			return (EINVAL);
+		if (udata.au_aupinfo.ap_pid < 1) { 
+			ret = EINVAL;
+			break;
+		}
+		if ((tp = proc_find(udata.au_aupinfo.ap_pid)) == NULL) {
+			ret = EINVAL;
+			break;
+		}
 
-		udata.au_aupinfo.ap_auid = tp->p_ucred->cr_au.ai_auid;
+		lck_mtx_unlock(audit_mtx);
+		my_cred = kauth_cred_proc_ref(tp);
+
+		udata.au_aupinfo.ap_auid = my_cred->cr_au.ai_auid;
 		udata.au_aupinfo.ap_mask.am_success = 
-			tp->p_ucred->cr_au.ai_mask.am_success;
+			my_cred->cr_au.ai_mask.am_success;
 		udata.au_aupinfo.ap_mask.am_failure = 
-			tp->p_ucred->cr_au.ai_mask.am_failure;
+			my_cred->cr_au.ai_mask.am_failure;
 		udata.au_aupinfo.ap_termid.machine = 
-			tp->p_ucred->cr_au.ai_termid.machine;
+			my_cred->cr_au.ai_termid.machine;
 		udata.au_aupinfo.ap_termid.port = 
-			tp->p_ucred->cr_au.ai_termid.port;
-		udata.au_aupinfo.ap_asid = tp->p_ucred->cr_au.ai_asid;
+			my_cred->cr_au.ai_termid.port;
+		udata.au_aupinfo.ap_asid = my_cred->cr_au.ai_asid;
+
+		kauth_cred_unref(&my_cred);
+
+		proc_rele(tp);
+		tp = PROC_NULL;
+		lck_mtx_lock(audit_mtx);
 		break;
 	case A_SETPMASK:
-		if (udata.au_aupinfo.ap_pid < 1) 
-			return (EINVAL);
-		if ((tp = pfind(udata.au_aupinfo.ap_pid)) == NULL)
-			return (EINVAL);
+		if (udata.au_aupinfo.ap_pid < 1) { 
+			ret = EINVAL;
+			break;
+		}
+		if ((tp = proc_find(udata.au_aupinfo.ap_pid)) == NULL) {
+			ret = EINVAL;
+			break;
+		}
 
 		/*
 		 * we are modifying the audit info in a credential so we need a new
@@ -981,8 +1063,9 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 		 * credential is used as part of our hash key.  Get current credential 
 		 * in the target process and take a reference while we muck with it.
 		 */
+		lck_mtx_unlock(audit_mtx);
 		for (;;) {
-			kauth_cred_t my_cred, my_new_cred;
+			kauth_cred_t my_new_cred;
 			struct auditinfo temp_auditinfo;
 			
 			my_cred = kauth_cred_proc_ref(tp);
@@ -1020,11 +1103,15 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 			kauth_cred_unref(&my_cred);
 			break;
 		}
+		proc_rele(tp);
+		lck_mtx_lock(audit_mtx);
 		break;
 	case A_SETFSIZE:
 		if ((udata.au_fstat.af_filesz != 0) &&
-		   (udata.au_fstat.af_filesz < MIN_AUDIT_FILE_SIZE))
-			return (EINVAL);
+		   (udata.au_fstat.af_filesz < MIN_AUDIT_FILE_SIZE)) {
+			ret = EINVAL;
+			break;
+		}
 		audit_fstat.af_filesz = udata.au_fstat.af_filesz;
 		break;
 	case A_GETFSIZE:
@@ -1032,48 +1119,54 @@ auditon(struct proc *p, __unused struct auditon_args *uap, __unused register_t *
 		udata.au_fstat.af_currsz = audit_fstat.af_currsz;
 		break;
 	case A_GETPINFO_ADDR:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_GETKAUDIT:
-		return (ENOSYS);
+		ret = ENOSYS;
 		break;
 	case A_SETKAUDIT:
-	return (ENOSYS);
+		ret = ENOSYS;
 		break;
-}
+	}
 	/* Copy data back to userspace for the GET comands */
-	switch (uap->cmd) {
-	case A_GETPOLICY:
-	case A_GETKMASK:
-	case A_GETQCTRL:
-	case A_GETCWD:
-	case A_GETCAR:
-	case A_GETSTAT:
-	case A_GETCOND:
-	case A_GETCLASS:
-	case A_GETPINFO:
-	case A_GETFSIZE:
-	case A_GETPINFO_ADDR:
-	case A_GETKAUDIT:
-		ret = copyout((void *)&udata, uap->data, uap->length);
-		if (ret)
-			return (ret);
-		break;
+	if (ret == 0) {
+		switch (uap->cmd) {
+		case A_GETPOLICY:
+		case A_GETKMASK:
+		case A_GETQCTRL:
+		case A_GETCWD:
+		case A_GETCAR:
+		case A_GETSTAT:
+		case A_GETCOND:
+		case A_GETCLASS:
+		case A_GETPINFO:
+		case A_GETFSIZE:
+		case A_GETPINFO_ADDR:
+		case A_GETKAUDIT:
+			ret = copyout((void *)&udata, uap->data, uap->length);
+			break;
+		}
 	}
 
-	return (0);
+	lck_mtx_unlock(audit_mtx);
+	return (ret);
 }
 
 /* 
  * System calls to manage the user audit information.
- * XXXAUDIT May need to lock the proc structure.
  */
 /* ARGSUSED */
 int
-getauid(struct proc *p, struct getauid_args *uap, __unused register_t *retval)
+getauid(__unused proc_t p, struct getauid_args *uap, __unused register_t *retval)
 {
 	int error;
 
+#if CONFIG_MACF
+	error = mac_proc_check_getauid(p);
+	if (error)
+		return (error);
+#endif
+
 	error = copyout((void *)&kauth_cred_get()->cr_au.ai_auid,
 			uap->auid, sizeof(au_id_t));
 	if (error)
@@ -1084,7 +1177,7 @@ getauid(struct proc *p, struct getauid_args *uap, __unused register_t *retval)
 
 /* ARGSUSED */
 int
-setauid(struct proc *p, struct setauid_args *uap, __unused register_t *retval)
+setauid(proc_t p, struct setauid_args *uap, __unused register_t *retval)
 {
 	int error;
 	au_id_t	temp_au_id;
@@ -1098,6 +1191,11 @@ setauid(struct proc *p, struct setauid_args *uap, __unused register_t *retval)
 			sizeof(au_id_t));
 	if (error)
 		return (error);
+#if CONFIG_MACF
+	error = mac_proc_check_setauid(p, temp_au_id);
+	if (error)
+		return (error);
+#endif
 
 	/*
 	 * we are modifying the audit info in a credential so we need a new
@@ -1158,11 +1256,17 @@ setauid(struct proc *p, struct setauid_args *uap, __unused register_t *retval)
  */
 /* ARGSUSED */
 int
-getaudit(struct proc *p, struct getaudit_args *uap, __unused register_t *retval)
+getaudit(proc_t p, struct getaudit_args *uap, __unused register_t *retval)
 {
 	struct auditinfo ai;
 	int error;
 
+#if CONFIG_MACF
+	error = mac_proc_check_getaudit(p);
+	if (error)
+		return (error);
+#endif
+
 	ai = kauth_cred_get()->cr_au;
 
 	/* only superuser gets to see the real mask */
@@ -1181,7 +1285,7 @@ getaudit(struct proc *p, struct getaudit_args *uap, __unused register_t *retval)
 
 /* ARGSUSED */
 int
-setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
+setaudit(proc_t p, struct setaudit_args *uap, __unused register_t *retval)
 {
 	int error;
 	struct auditinfo temp_auditinfo;
@@ -1190,12 +1294,18 @@ setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
 	error = suser(kauth_cred_get(), &p->p_acflag);
 	if (error)
 		return (error);
-		
 	error = copyin(uap->auditinfo,
 		       (void *)&temp_auditinfo, 
 		       sizeof(temp_auditinfo));
 	if (error)
 		return (error);
+#if CONFIG_MACF
+	error = mac_proc_check_setaudit(p, &temp_auditinfo);
+	if (error)
+		return (error); 
+
+#endif
+
 
 	/*
 	 * we are modifying the audit info in a credential so we need a new
@@ -1249,14 +1359,14 @@ setaudit(struct proc *p, struct setaudit_args *uap, __unused register_t *retval)
 
 /* ARGSUSED */
 int
-getaudit_addr(struct proc *p, __unused struct getaudit_addr_args *uap, __unused register_t *retval)
+getaudit_addr(__unused proc_t p, __unused struct getaudit_addr_args *uap, __unused register_t *retval)
 {
 	return (ENOSYS);
 }
 
 /* ARGSUSED */
 int
-setaudit_addr(struct proc *p, __unused struct setaudit_addr_args *uap, __unused register_t *retval)
+setaudit_addr(proc_t p, __unused struct setaudit_addr_args *uap, __unused register_t *retval)
 {
 	int error;
 
@@ -1272,16 +1382,12 @@ setaudit_addr(struct proc *p, __unused struct setaudit_addr_args *uap, __unused
  */
 /* ARGSUSED */
 int
-auditctl(struct proc *p, struct auditctl_args *uap, __unused register_t *retval)
+auditctl(proc_t p, struct auditctl_args *uap, __unused register_t *retval)
 {
 	struct nameidata nd;
 	kauth_cred_t cred;
 	struct vnode *vp;
-	int error, flags;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	int error;
 
 	error = suser(kauth_cred_get(), &p->p_acflag);
 	if (error)
@@ -1295,24 +1401,53 @@ auditctl(struct proc *p, struct auditctl_args *uap, __unused register_t *retval)
 	 * validity checks, and grab another reference to the current
 	 * credential.
 	 */
-	if (uap->path != 0) {
+	if (uap->path != USER_ADDR_NULL) {
 		NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, 
 			(IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-			uap->path, &context);
-		flags = audit_open_flags;
-		error = vn_open(&nd, flags, 0);
+			uap->path, vfs_context_current());
+		error = vn_open(&nd, audit_open_flags, 0);
 		if (error)
 			goto out;
 		vp = nd.ni_vp;
+
 		if (vp->v_type != VREG) {
-			vn_close(vp, audit_close_flags, kauth_cred_get(), p);
+			vn_close(vp, audit_close_flags, vfs_context_current());
 			vnode_put(vp);
 			error = EINVAL;
 			goto out;
 		}
+#if CONFIG_MACF
+		/*
+		 * Accessibility of the vnode was determined in
+		 * vn_open; the mac_system_check_auditctl should only
+		 * determine whether that vnode is appropriate for
+		 * storing audit data, or that the caller was
+		 * permitted to control the auditing system at all.
+		 * For example, a confidentiality policy may want to
+		 * ensure that audit files are always high
+		 * sensitivity.
+		 */
+
+		error = mac_system_check_auditctl(kauth_cred_get(), vp);
+		if (error) {
+			vn_close(vp, audit_close_flags, vfs_context_current());
+			vnode_put(vp);
+			goto out;
+		}
+#endif
 		cred = kauth_cred_get_with_ref();
+		lck_mtx_lock(audit_mtx);
 		audit_suspended = 0;
+		lck_mtx_unlock(audit_mtx);
 	}
+#if CONFIG_MACF
+	else {
+		error = mac_system_check_auditctl(kauth_cred_get(), NULL);
+		if (error)
+			return (error);
+	}
+#endif
+	
 	/*
 	 * a vp and cred of NULL is valid at this point
 	 * and indicates we're to turn off auditing...
@@ -1332,7 +1467,7 @@ out:
  * MPSAFE
  */
 struct kaudit_record *
-audit_new(int event, struct proc *p, __unused struct uthread *uthread)
+audit_new(int event, proc_t p, __unused struct uthread *uthread)
 {
 	struct kaudit_record *ar;
 	int no_record;
@@ -1349,9 +1484,9 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 #if 0
 	if (event != AUDIT_EVENT_FILESTOP && event != AUDIT_EVENT_FILESTART) {
 #endif
-		mutex_lock(audit_mtx);
+		lck_mtx_lock(audit_mtx);
 		no_record = (audit_suspended || !audit_enabled);
-		mutex_unlock(audit_mtx);
+		lck_mtx_unlock(audit_mtx);
 		if (no_record)
 			return (NULL);
 #if 0
@@ -1370,10 +1505,6 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 	if (ar == NULL)
 		return NULL;
 
-	mutex_lock(audit_mtx);
-	audit_pre_q_len++;
-	mutex_unlock(audit_mtx);
-
 	bzero(ar, sizeof(*ar));
 	ar->k_ar.ar_magic = AUDIT_RECORD_MAGIC;
 	ar->k_ar.ar_event = event;
@@ -1395,6 +1526,44 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 	ar->k_ar.ar_subj_pid = p->p_pid;
 	bcopy(p->p_comm, ar->k_ar.ar_subj_comm, MAXCOMLEN);
 
+#if CONFIG_MACF
+	do {
+		struct mac mac;
+
+		/* Retrieve the MAC labels for the process. */
+		ar->k_ar.ar_cred_mac_labels =
+		    (char *)zalloc(audit_mac_label_zone);
+		if (ar->k_ar.ar_cred_mac_labels == NULL) {
+			zfree(audit_zone, ar);
+			return (NULL);
+		}
+		mac.m_buflen = MAC_AUDIT_LABEL_LEN;
+		mac.m_string = ar->k_ar.ar_cred_mac_labels;
+		mac_cred_label_externalize_audit(p, &mac);
+
+		/*
+		 * grab space for the reconds.
+		 */
+		ar->k_ar.ar_mac_records = (struct mac_audit_record_list_t *)
+		    kalloc(sizeof(*ar->k_ar.ar_mac_records));
+               if (ar->k_ar.ar_mac_records == NULL) {
+                       zfree(audit_mac_label_zone,
+			     ar->k_ar.ar_cred_mac_labels);
+                       zfree(audit_zone, ar);
+                       return (NULL);
+               }
+
+		LIST_INIT(ar->k_ar.ar_mac_records);
+
+		ar->k_ar.ar_forced_by_mac = 0;
+
+	} while (0);
+#endif
+
+	lck_mtx_lock(audit_mtx);
+	audit_pre_q_len++;
+	lck_mtx_unlock(audit_mtx);
+
 	return (ar);
 }
 
@@ -1405,9 +1574,9 @@ audit_new(int event, struct proc *p, __unused struct uthread *uthread)
 void
 audit_abort(struct kaudit_record *ar)
 {
-	mutex_lock(audit_mtx);
+	lck_mtx_lock(audit_mtx);
 	audit_pre_q_len--;
-	mutex_unlock(audit_mtx);
+	lck_mtx_unlock(audit_mtx);
 	audit_free(ar);
 }
 
@@ -1463,9 +1632,9 @@ audit_commit(struct kaudit_record *ar, int error, int retval)
 		ar->k_ar_commit |= AR_COMMIT_KERNEL;
 
 	if ((ar->k_ar_commit & (AR_COMMIT_USER | AR_COMMIT_KERNEL)) == 0) {
-		mutex_lock(audit_mtx);
+		lck_mtx_lock(audit_mtx);
 		audit_pre_q_len--;
-		mutex_unlock(audit_mtx);
+		lck_mtx_unlock(audit_mtx);
 		audit_free(ar);
 		return;
 	}
@@ -1483,14 +1652,14 @@ audit_commit(struct kaudit_record *ar, int error, int retval)
 	 */
 	nanotime(&ar->k_ar.ar_endtime);
 
-	mutex_lock(audit_mtx);
+	lck_mtx_lock(audit_mtx);
 	/*
 	 * Note: it could be that some records initiated while audit was
 	 * enabled should still be committed?
 	 */
 	if (audit_suspended || !audit_enabled) {
 		audit_pre_q_len--;
-		mutex_unlock(audit_mtx);
+		lck_mtx_unlock(audit_mtx);
 		audit_free(ar);
 		return;
 	}
@@ -1505,20 +1674,41 @@ audit_commit(struct kaudit_record *ar, int error, int retval)
 					     AUDIT_COMMIT_EVENT,
 					     THREAD_UNINT,
 					     0);
-		mutex_unlock(audit_mtx);
+		lck_mtx_unlock(audit_mtx);
 
 		assert(ret == THREAD_WAITING);
 
 		ret = thread_block(THREAD_CONTINUE_NULL);
 		assert(ret == THREAD_AWAKENED);
-		mutex_lock(audit_mtx);
+		lck_mtx_lock(audit_mtx);
 	}
 
 	TAILQ_INSERT_TAIL(&audit_q, ar, k_q);
 	audit_q_len++;
 	audit_pre_q_len--;
 	wait_queue_wakeup_one(audit_wait_queue, AUDIT_WORKER_EVENT, THREAD_AWAKENED);
-	mutex_unlock(audit_mtx);
+	lck_mtx_unlock(audit_mtx);
+}
+
+/*
+ * If we're out of space and need to suspend unprivileged
+ * processes, do that here rather than trying to allocate
+ * another audit record.
+ */
+static void
+audit_new_wait(int audit_event, proc_t proc, struct uthread *uthread)
+{
+	int ret;
+
+	if (audit_in_failure &&
+	    suser(kauth_cred_get(), &proc->p_acflag) != 0) {
+		ret = wait_queue_assert_wait(audit_wait_queue,
+		    AUDIT_FAILURE_EVENT, THREAD_UNINT, 0);
+		assert(ret == THREAD_WAITING);
+		(void)thread_block(THREAD_CONTINUE_NULL);
+		panic("audit_failing_stop: thread continued");
+	}
+	uthread->uu_ar = audit_new(audit_event, proc, uthread);
 }
 
 /*
@@ -1526,11 +1716,12 @@ audit_commit(struct kaudit_record *ar, int error, int retval)
  * each system call.
  */
 void
-audit_syscall_enter(unsigned short code, struct proc *proc, 
+audit_syscall_enter(unsigned short code, proc_t proc, 
 			struct uthread *uthread)
 {
 	int audit_event;
 	struct au_mask *aumask;
+	kauth_cred_t my_cred;
 
 	audit_event = sys_au_event[code];
 	if (audit_event == AUE_NULL)
@@ -1541,41 +1732,60 @@ audit_syscall_enter(unsigned short code, struct proc *proc,
 	/* Check which audit mask to use; either the kernel non-attributable
 	 * event mask or the process audit mask.
 	 */
-	if (proc->p_ucred->cr_au.ai_auid == AU_DEFAUDITID)
+	my_cred = kauth_cred_proc_ref(proc);
+
+	if (my_cred->cr_au.ai_auid == AU_DEFAUDITID)
 		aumask = &audit_nae_mask;
 	else
-		aumask = &proc->p_ucred->cr_au.ai_mask;
+		aumask = &my_cred->cr_au.ai_mask;
 
 	/*
 	 * Allocate an audit record, if preselection allows it, and store 
 	 * in the BSD thread for later use.
 	 */
-	if (au_preselect(audit_event, aumask,
-				AU_PRS_FAILURE | AU_PRS_SUCCESS)) {
-		/*
-		 * If we're out of space and need to suspend unprivileged
-		 * processes, do that here rather than trying to allocate
-		 * another audit record.
-		 */
-		if (audit_in_failure &&
-		    suser(kauth_cred_get(), &proc->p_acflag) != 0) {
-			int ret;
 
-			assert(audit_worker_thread != THREAD_NULL);
-			ret = wait_queue_assert_wait(audit_wait_queue,
-			    AUDIT_FAILURE_EVENT, THREAD_UNINT, 0);
-			assert(ret == THREAD_WAITING);
-			(void)thread_block(THREAD_CONTINUE_NULL);
-			panic("audit_failing_stop: thread continued");
-		}
+#if CONFIG_MACF
+	do {
+		int error;
+
+		error = mac_audit_check_preselect(my_cred, code,
+		    (void *) uthread->uu_arg);
+
+		if (error == MAC_AUDIT_YES) {
 			uthread->uu_ar = audit_new(audit_event, proc, uthread);
-		} else {
+			uthread->uu_ar->k_ar.ar_forced_by_mac = 1;
+			au_to_text("Forced by a MAC policy");  
+		}	
+		else if (error == MAC_AUDIT_NO) {
 			uthread->uu_ar = NULL;
 		}
+		else if (error == MAC_AUDIT_DEFAULT &&
+		    au_preselect(audit_event, &my_cred->cr_au.ai_mask,
+			AU_PRS_FAILURE | AU_PRS_SUCCESS))
+				audit_new_wait(audit_event, proc, uthread);
+	} while (0);
+#else
+	if (au_preselect(audit_event, &my_cred->cr_au.ai_mask,
+				AU_PRS_FAILURE | AU_PRS_SUCCESS)) {
+		audit_new_wait(audit_event, proc, uthread);
+	} else {
+		uthread->uu_ar = NULL;
 	}
+#endif
+	kauth_cred_unref(&my_cred);
+}
 
+/* 
+ * Note: The audit_syscall_exit() parameter list was modified to support 
+ * mac_audit_check_postselect(), which requires the Darwin syscall number.
+ */
+#if CONFIG_MACF
+void
+audit_syscall_exit(unsigned short code, int error, AUDIT_PRINTF_ONLY proc_t proc, struct uthread *uthread)
+#else
 void
-audit_syscall_exit(int error, AUDIT_PRINTF_ONLY struct proc *proc, struct uthread *uthread)
+audit_syscall_exit(int error, AUDIT_PRINTF_ONLY proc_t proc, struct uthread *uthread)
+#endif
 {
 	int retval;
 
@@ -1592,10 +1802,43 @@ audit_syscall_exit(int error, AUDIT_PRINTF_ONLY struct proc *proc, struct uthrea
 	else
 		retval = uthread->uu_rval[0];
 
+#if CONFIG_MACF
+	do {
+		int mac_error;
+		
+		if (uthread->uu_ar == NULL)  /* syscall wasn't audited */
+			goto out;
+			
+		/*  
+		 * Note, no other postselect mechanism exists.  If
+		 * mac_audit_check_postselect returns MAC_AUDIT_NO, the
+		 * record will be suppressed.  Other values at this
+		 * point result in the audit record being committed.
+		 * This suppression behavior will probably go away in
+		 * the port to 10.3.4.
+		 */
+		mac_error = mac_audit_check_postselect(kauth_cred_get(), code,
+		    (void *) uthread->uu_arg, error, retval,
+		    uthread->uu_ar->k_ar.ar_forced_by_mac);
+
+		if (mac_error == MAC_AUDIT_YES)
+			uthread->uu_ar->k_ar_commit |= AR_COMMIT_KERNEL;
+		else if (mac_error == MAC_AUDIT_NO) {
+			audit_free(uthread->uu_ar);
+			goto out;
+		}
+
+	} while (0);
+	
+#endif
 	audit_commit(uthread->uu_ar, error, retval);
 	if (uthread->uu_ar != NULL) {
 		AUDIT_PRINTF(("audit record committed by pid %d\n", proc->p_pid));
 	}
+
+#if CONFIG_MACF
+out:
+#endif
 	uthread->uu_ar = NULL;
 
 }
@@ -1608,8 +1851,9 @@ void
 audit_mach_syscall_enter(unsigned short audit_event)
 {
 	struct uthread *uthread;
-	struct proc *proc;
+	proc_t proc;
 	struct au_mask *aumask;
+	kauth_cred_t my_cred;
 
 	if (audit_event == AUE_NULL)
 		return;
@@ -1624,13 +1868,17 @@ audit_mach_syscall_enter(unsigned short audit_event)
 
 	assert(uthread->uu_ar == NULL);
 
+	my_cred = kauth_cred_proc_ref(proc);
+
 	/* Check which audit mask to use; either the kernel non-attributable
 	 * event mask or the process audit mask.
 	 */
-	if (proc->p_ucred->cr_au.ai_auid == AU_DEFAUDITID)
+	if (my_cred->cr_au.ai_auid == AU_DEFAUDITID)
 		aumask = &audit_nae_mask;
 	else
-		aumask = &proc->p_ucred->cr_au.ai_mask;
+		aumask = &my_cred->cr_au.ai_mask;
+
+	kauth_cred_unref(&my_cred);
 	
 	/*
 	 * Allocate an audit record, if desired, and store in the BSD
@@ -1773,14 +2021,7 @@ audit_arg_login(const char *login)
 	if (ar == NULL)
 		return;
 
-#if 0
-	/*
-	 * XXX: Add strlcpy() to Darwin for improved safety.
-	 */
 	strlcpy(ar->k_ar.ar_arg_login, login, MAXLOGNAME);
-#else
-	strcpy(ar->k_ar.ar_arg_login, login);
-#endif
 
 	ar->k_ar.ar_valid_arg |= ARG_LOGIN;
 }
@@ -1879,21 +2120,24 @@ audit_arg_pid(pid_t pid)
 }
 
 void
-audit_arg_process(struct proc *p)
+audit_arg_process(proc_t p)
 {
 	struct kaudit_record *ar;
+	kauth_cred_t my_cred;
 
 	ar = currecord();
 	if ((ar == NULL) || (p == NULL))
 		return;
 
-	ar->k_ar.ar_arg_auid = p->p_ucred->cr_au.ai_auid;
-	ar->k_ar.ar_arg_euid = p->p_ucred->cr_uid;
-	ar->k_ar.ar_arg_egid = p->p_ucred->cr_groups[0];
-	ar->k_ar.ar_arg_ruid = p->p_ucred->cr_ruid;
-	ar->k_ar.ar_arg_rgid = p->p_ucred->cr_rgid;
-	ar->k_ar.ar_arg_asid = p->p_ucred->cr_au.ai_asid;
-	ar->k_ar.ar_arg_termid = p->p_ucred->cr_au.ai_termid;
+	my_cred = kauth_cred_proc_ref(p);
+	ar->k_ar.ar_arg_auid = my_cred->cr_au.ai_auid;
+	ar->k_ar.ar_arg_euid = my_cred->cr_uid;
+	ar->k_ar.ar_arg_egid = my_cred->cr_groups[0];
+	ar->k_ar.ar_arg_ruid = my_cred->cr_ruid;
+	ar->k_ar.ar_arg_rgid = my_cred->cr_rgid;
+	ar->k_ar.ar_arg_asid = my_cred->cr_au.ai_asid;
+	ar->k_ar.ar_arg_termid = my_cred->cr_au.ai_termid;
+	kauth_cred_unref(&my_cred);
 
 	ar->k_ar.ar_valid_arg |= ARG_AUID | ARG_EUID | ARG_EGID | ARG_RUID | 
 		ARG_RGID | ARG_ASID | ARG_TERMID | ARG_PROCESS;
@@ -1928,13 +2172,20 @@ audit_arg_socket(int sodomain, int sotype, int soprotocol)
 	ar->k_ar.ar_valid_arg |= ARG_SOCKINFO;
 }
 
+/*
+ * Note that the current working directory vp must be supplied at the audit
+ * call site to permit per thread current working directories, and that it
+ * must take a upath starting with '/' into account for chroot if the path
+ * is absolute.  This results in the real (non-chroot) path being recorded
+ * in the audit record.
+ */
 void
-audit_arg_sockaddr(struct proc *p, struct sockaddr *so)
+audit_arg_sockaddr(struct vnode *cwd_vp, struct sockaddr *so)
 {
 	struct kaudit_record *ar;
 
 	ar = currecord();
-	if (ar == NULL || p == NULL || so == NULL)
+	if (ar == NULL || cwd_vp == NULL || so == NULL)
 		return;
 
 	bcopy(so, &ar->k_ar.ar_arg_sockaddr, sizeof(ar->k_ar.ar_arg_sockaddr));
@@ -1946,7 +2197,7 @@ audit_arg_sockaddr(struct proc *p, struct sockaddr *so)
 		ar->k_ar.ar_valid_arg |= ARG_SADDRINET6;
 		break;
 	case AF_UNIX:
-		audit_arg_upath(p, ((struct sockaddr_un *)so)->sun_path, 
+		audit_arg_upath(cwd_vp, ((struct sockaddr_un *)so)->sun_path, 
 				ARG_UPATH1);
 		ar->k_ar.ar_valid_arg |= ARG_SADDRUNIX;
 		break;
@@ -2004,7 +2255,7 @@ audit_arg_text(const char *text)
 			return;	
 	}
 
-	strncpy(ar->k_ar.ar_arg_text, text, MAXPATHLEN);
+	strlcpy(ar->k_ar.ar_arg_text, text, MAXPATHLEN);
 	ar->k_ar.ar_valid_arg |= ARG_TEXT;
 }
 
@@ -2062,7 +2313,7 @@ audit_arg_svipc_id(int id)
 }
 
 void
-audit_arg_svipc_addr(void * addr)
+audit_arg_svipc_addr(user_addr_t addr)
 {
 	struct kaudit_record *ar;
 
@@ -2108,7 +2359,7 @@ audit_arg_auditon(const union auditon_udata *udata)
  * socket address info.
  */
 void
-audit_arg_file(__unused struct proc *p, const struct fileproc *fp)
+audit_arg_file(__unused proc_t p, const struct fileproc *fp)
 {
 	struct kaudit_record *ar;
 	struct socket *so;
@@ -2153,15 +2404,19 @@ audit_arg_file(__unused struct proc *p, const struct fileproc *fp)
  * Store a path as given by the user process for auditing into the audit 
  * record stored on the user thread. This function will allocate the memory to 
  * store the path info if not already available. This memory will be 
- * freed when the audit record is freed.
+ * freed when the audit record is freed.  Note that the current working
+ * directory vp must be supplied at the audit call site to permit per thread
+ * current working directories, and that it must take a upath starting with
+ * '/' into account for chroot if the path is absolute.  This results in the
+ * real (non-chroot) path being recorded in the audit record.
  */
 void
-audit_arg_upath(struct proc *p, char *upath, u_int64_t flags)
+audit_arg_upath(struct vnode *cwd_vp, char *upath, u_int64_t flags)
 {
 	struct kaudit_record *ar;
 	char **pathp;
 
-	if (p == NULL || upath == NULL) 
+	if (cwd_vp == NULL || upath == NULL) 
 		return;		/* nothing to do! */
 
 	if ((flags & (ARG_UPATH1 | ARG_UPATH2)) == 0)
@@ -2186,7 +2441,7 @@ audit_arg_upath(struct proc *p, char *upath, u_int64_t flags)
 			return;
 	}
 
-	if (canon_path(p, upath, *pathp) == 0) {
+	if (canon_path(cwd_vp, upath, *pathp) == 0) {
 		if (flags & ARG_UPATH1)
 			ar->k_ar.ar_valid_arg |= ARG_UPATH1;
 		else
@@ -2220,8 +2475,11 @@ audit_arg_vnpath(struct vnode *vp, u_int64_t flags)
 	int len;
 	char **pathp;
 	struct vnode_au_info *vnp;
-	struct proc *p;
-	struct vfs_context context;
+	proc_t p;
+#if CONFIG_MACF
+	char **vnode_mac_labelp;
+	struct mac mac;
+#endif
 
 	if (vp == NULL)
 		return;
@@ -2240,12 +2498,18 @@ audit_arg_vnpath(struct vnode *vp, u_int64_t flags)
 		ar->k_ar.ar_valid_arg &= (ARG_ALL ^ ARG_VNODE1);
 		pathp = &ar->k_ar.ar_arg_kpath1;
 		vnp = &ar->k_ar.ar_arg_vnode1;
+#if CONFIG_MACF
+		vnode_mac_labelp = &ar->k_ar.ar_vnode1_mac_labels;
+#endif
 	}
 	else {
 		ar->k_ar.ar_valid_arg &= (ARG_ALL ^ ARG_KPATH2);
 		ar->k_ar.ar_valid_arg &= (ARG_ALL ^ ARG_VNODE2);
 		pathp = &ar->k_ar.ar_arg_kpath2;
 		vnp = &ar->k_ar.ar_arg_vnode2;
+#if CONFIG_MACF
+		vnode_mac_labelp = &ar->k_ar.ar_vnode2_mac_labels;
+#endif
 	}
 
 	if (*pathp == NULL) {
@@ -2270,9 +2534,6 @@ audit_arg_vnpath(struct vnode *vp, u_int64_t flags)
 		*pathp = NULL;
 	}
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_mode);
 	VATTR_WANTED(&va, va_uid);
@@ -2281,12 +2542,26 @@ audit_arg_vnpath(struct vnode *vp, u_int64_t flags)
 	VATTR_WANTED(&va, va_fsid);
 	VATTR_WANTED(&va, va_fileid);
 	VATTR_WANTED(&va, va_gen);
-	error = vnode_getattr(vp, &va, &context);
+	error = vnode_getattr(vp, &va, vfs_context_current());
 	if (error) {
 		/* XXX: How to handle this case? */
 		return;
 	}
 
+#if CONFIG_MACF
+	if (*vnode_mac_labelp == NULL) {
+		*vnode_mac_labelp = (char *)zalloc(audit_mac_label_zone);
+		if (*vnode_mac_labelp != NULL) {
+			mac.m_buflen = MAC_AUDIT_LABEL_LEN;
+			mac.m_string = *vnode_mac_labelp;
+			mac_vnode_label_externalize_audit(vp, &mac);
+		}
+
+
+
+	}
+#endif
+
 	/* XXX do we want to fall back here when these aren't supported? */
 	vnp->vn_mode = va.va_mode;
 	vnp->vn_uid = va.va_uid;
@@ -2343,7 +2618,7 @@ audit_arg_mach_port2(mach_port_name_t port)
  * within the system call itself.
  */
 void
-audit_sysclose(struct proc *p, int fd)
+audit_sysclose(proc_t p, int fd)
 {
 	struct fileproc *fp;
 	struct vnode *vp;
@@ -2357,6 +2632,88 @@ audit_sysclose(struct proc *p, int fd)
 	file_drop(fd);
 } 
 
+#if CONFIG_MACF
+/*
+ * This function is called by the MAC Framework to add audit data
+ * from a policy to the current audit record.
+ */
+int
+audit_mac_data(int type, int len, u_char *data) {
+	struct kaudit_record *cur;
+	struct mac_audit_record *record;
+	int ret = 0;
+
+	if (audit_enabled == 0) {
+		ret = ENOTSUP;
+		goto out_fail;
+	}
+
+	cur = currecord();
+	if (cur == NULL) {
+		ret = ENOTSUP;
+		goto out_fail;
+	}
+
+	/*
+	 * XXX: Note that we silently drop the audit data if this
+	 * allocation fails - this is consistent with the rest of the
+	 * audit implementation.
+	 */
+	record = (struct mac_audit_record *)kalloc(sizeof(*record));
+	if (record == NULL)
+		goto out_fail;
+
+	record->type = type;
+	record->length = len;
+	record->data = data;
+	LIST_INSERT_HEAD(cur->k_ar.ar_mac_records, record, records);
+
+	return (0);
+
+out_fail:
+	kfree(data, len);
+	return (ret);
+}
+
+void
+audit_arg_mac_string(const char *string)
+{
+	struct kaudit_record *ar;
+
+	ar = currecord();
+	if (ar == NULL)
+		return;
+
+	if (ar->k_ar.ar_arg_mac_string == NULL) {
+		ar->k_ar.ar_arg_mac_string =
+		    (char *)kalloc(MAC_MAX_LABEL_BUF_LEN + MAC_ARG_PREFIX_LEN);
+		/* This should be a rare event. If kalloc() returns NULL, the
+		 * system is low on kernel virtual memory. To be consistent with the
+		 * rest of audit, just return (may need to panic if required to for audit6).
+		 */
+		if (ar->k_ar.ar_arg_mac_string == NULL)
+			return;
+	}
+	strncpy(ar->k_ar.ar_arg_mac_string, MAC_ARG_PREFIX, MAC_ARG_PREFIX_LEN);
+	strncpy(ar->k_ar.ar_arg_mac_string + MAC_ARG_PREFIX_LEN, string, MAC_MAX_LABEL_BUF_LEN);
+	ar->k_ar.ar_valid_arg |= ARG_MAC_STRING;
+
+}
+#endif  /* MAC */
+
+/*
+ * kau_will_audit can be used by a security policy to determine
+ * if an audit record will be stored, reducing wasted memory allocation
+ * and string handling.
+ */
+
+int
+kau_will_audit(void)
+{
+
+	return (audit_enabled && currecord() != NULL);
+}
+
 #else /* !AUDIT */
 
 void
@@ -2372,57 +2729,70 @@ audit_shutdown(void)
 }
 
 int
-audit(struct proc *p, struct audit_args *uap, register_t *retval)
+audit(proc_t p, struct audit_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-auditon(struct proc *p, struct auditon_args *uap, register_t *retval)
+auditon(proc_t p, struct auditon_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-getauid(struct proc *p, struct getauid_args *uap, register_t *retval)
+getauid(proc_t p, struct getauid_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-setauid(struct proc *p, struct setauid_args *uap, register_t *retval)
+setauid(proc_t p, struct setauid_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-getaudit(struct proc *p, struct getaudit_args *uap, register_t *retval)
+getaudit(proc_t p, struct getaudit_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-setaudit(struct proc *p, struct setaudit_args *uap, register_t *retval)
+setaudit(proc_t p, struct setaudit_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-getaudit_addr(struct proc *p, struct getaudit_addr_args *uap, register_t *retval)
+getaudit_addr(proc_t p, struct getaudit_addr_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-setaudit_addr(struct proc *p, struct setaudit_addr_args *uap, register_t *retval)
+setaudit_addr(proc_t p, struct setaudit_addr_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-auditctl(struct proc *p, struct auditctl_args *uap, register_t *retval)
+auditctl(proc_t p, struct auditctl_args *uap, register_t *retval)
 {
 	return (ENOSYS);
 }
 
+#if CONFIG_MACF
+void
+audit_mac_data(int type, int len, u_char *data)
+{
+}
+
+int
+kau_will_audit()
+{
+	return (0);
+}
+#endif
+
 #endif /* AUDIT */
diff --git a/bsd/kern/kern_authorization.c b/bsd/kern/kern_authorization.c
index d7bf6c916..b65a828c2 100644
--- a/bsd/kern/kern_authorization.c
+++ b/bsd/kern/kern_authorization.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -111,7 +117,7 @@ struct kauth_scope {
 static TAILQ_HEAD(,kauth_scope)	kauth_scopes;
 
 static int kauth_add_callback_to_scope(kauth_scope_t sp, kauth_listener_t klp);
-static void	kauth_scope_init(void);
+static void	kauth_scope_init(void) __attribute__((section("__TEXT, initcode")));
 static kauth_scope_t kauth_alloc_scope(const char *identifier, kauth_scope_callback_t callback, void *idata);
 static kauth_listener_t kauth_alloc_listener(const char *identifier, kauth_scope_callback_t callback, void *idata);
 #if 0
@@ -126,7 +132,7 @@ static int	kauth_authorize_generic_callback(kauth_cred_t _credential, void *_ida
     uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 kauth_scope_t	kauth_scope_fileop;
 
-extern int 		cansignal(struct proc *, kauth_cred_t, struct proc *, int);
+extern int 		cansignal(struct proc *, kauth_cred_t, struct proc *, int, int);
 extern char *	get_pathbuff(void);
 extern void		release_pathbuff(char *path);
 
@@ -222,7 +228,8 @@ kauth_register_scope(const char *identifier, kauth_scope_callback_t callback, vo
 	KAUTH_SCOPELOCK();
 	TAILQ_FOREACH(tsp, &kauth_scopes, ks_link) {
 		/* duplicate! */
-		if (strcmp(tsp->ks_identifier, identifier) == 0) {
+		if (strncmp(tsp->ks_identifier, identifier, 
+					strlen(tsp->ks_identifier) + 1) == 0) {
 			KAUTH_SCOPEUNLOCK();
 			FREE(sp, M_KAUTH);
 			return(NULL);
@@ -238,7 +245,8 @@ kauth_register_scope(const char *identifier, kauth_scope_callback_t callback, vo
 	 */
 restart:
 	TAILQ_FOREACH(klp, &kauth_dangling_listeners, kl_link) {
-		if (strcmp(klp->kl_identifier, sp->ks_identifier) == 0) {
+		if (strncmp(klp->kl_identifier, sp->ks_identifier,
+					strlen(klp->kl_identifier) + 1) == 0) {
 			/* found a match on the dangling listener list.  add it to the
 			 * the active scope.
 			 */
@@ -301,7 +309,8 @@ kauth_listen_scope(const char *identifier, kauth_scope_callback_t callback, void
 	 */
 	KAUTH_SCOPELOCK();
 	TAILQ_FOREACH(sp, &kauth_scopes, ks_link) {
-		if (strcmp(sp->ks_identifier, identifier) == 0) {
+		if (strncmp(sp->ks_identifier, identifier,
+					strlen(sp->ks_identifier) + 1) == 0) {
 			/* scope exists, add it to scope listener table */
 			if (kauth_add_callback_to_scope(sp, klp) == 0) {
 				KAUTH_SCOPEUNLOCK();
@@ -376,6 +385,12 @@ kauth_unlisten_scope(kauth_listener_t listener)
 
 /*
  * Authorization requests.
+ *
+ * Returns:	0			Success
+ *		EPERM			Operation not permitted
+ *
+ * Imputed:	*arg3, modified		Callback return - depends on callback
+ *					modification of *arg3, if any
  */
 int
 kauth_authorize_action(kauth_scope_t scope, kauth_cred_t credential, kauth_action_t action,
@@ -467,7 +482,7 @@ kauth_authorize_process_callback(kauth_cred_t credential, __unused void *idata,
 		/* arg0 - process to signal
 		 * arg1 - signal to send the process
 		 */
-		if (cansignal(current_proc(), credential, (struct proc *)arg0, (int)arg1))
+		if (cansignal(current_proc(), credential, (struct proc *)arg0, (int)arg1, 0))
 			return(KAUTH_RESULT_ALLOW);
 		break;
 	case KAUTH_PROCESS_CANTRACE:
@@ -611,6 +626,7 @@ kauth_acl_evaluate(kauth_cred_t cred, kauth_acl_eval_t eval)
 	}
 
 	eval->ae_residual = eval->ae_requested;
+	eval->ae_found_deny = FALSE;
 
 	/*
 	 * Get our guid for comparison purposes.
@@ -656,6 +672,7 @@ kauth_acl_evaluate(kauth_cred_t cred, kauth_acl_eval_t eval)
 		case KAUTH_ACE_DENY:
 			if (!(eval->ae_requested & rights))
 				continue;
+			eval->ae_found_deny = TRUE;
 			break;
 		default:
 			/* we don't recognise this ACE, skip it */
@@ -765,9 +782,13 @@ kauth_acl_inherit(vnode_t dvp, kauth_acl_t initial, kauth_acl_t *product, int is
 	kauth_acl_t inherit, result;
 
 	/*
-	 * Fetch the ACL from the directory.  This should never fail.  Note that we don't
-	 * manage inheritance when the remote server is doing authorization; we just
-	 * want to compose the umask-ACL and any initial ACL.
+	 * Fetch the ACL from the directory.  This should never fail.
+	 * Note that we don't manage inheritance when the remote server is
+	 * doing authorization, since this means server enforcement of
+	 * inheritance semantics; we just want to compose the initial
+	 * ACL and any inherited ACE entries from the container object.
+	 *
+	 * XXX TODO: <rdar://3634665> wants a "umask ACL" from the process.
 	 */
 	inherit = NULL;
 	if ((dvp != NULL) && !vfs_authopaque(vnode_mount(dvp))) {
@@ -782,7 +803,8 @@ kauth_acl_inherit(vnode_t dvp, kauth_acl_t initial, kauth_acl_t *product, int is
 	}
 
 	/*
-	 * Compute the number of entries in the result ACL by scanning the input lists.
+	 * Compute the number of entries in the result ACL by scanning the
+	 * input lists.
 	 */
 	entries = 0;
 	if (inherit != NULL) {
@@ -793,16 +815,23 @@ kauth_acl_inherit(vnode_t dvp, kauth_acl_t initial, kauth_acl_t *product, int is
 	}
 
 	if (initial == NULL) {
-		/* XXX 3634665 TODO: fetch umask ACL from the process, set in initial */
+		/*
+		 * XXX 3634665 TODO: if the initial ACL is not specfied by
+		 * XXX the caller, fetch the umask ACL from the process,
+		 * and use it in place of "initial".
+		 */
 	}
 
 	if (initial != NULL) {
-		entries += initial->acl_entrycount;
+		if (initial->acl_entrycount != KAUTH_FILESEC_NOACL)
+			entries += initial->acl_entrycount;
+		else
+			initial = NULL;
 	}
 
 	/*
 	 * If there is no initial ACL, and no inheritable entries, the
-	 * object should have no ACL at all.
+	 * object should be created with no ACL at all.
 	 * Note that this differs from the case where the initial ACL
 	 * is empty, in which case the object must also have an empty ACL.
 	 */
@@ -816,7 +845,7 @@ kauth_acl_inherit(vnode_t dvp, kauth_acl_t initial, kauth_acl_t *product, int is
 	 * Allocate the result buffer.
 	 */
 	if ((result = kauth_acl_alloc(entries)) == NULL) {
-		KAUTH_DEBUG("    ERROR - could not allocate %d-entry result buffer for inherited ACL");
+		KAUTH_DEBUG("    ERROR - could not allocate %d-entry result buffer for inherited ACL", entries);
 		error = ENOMEM;
 		goto out;
 	}
@@ -834,14 +863,27 @@ kauth_acl_inherit(vnode_t dvp, kauth_acl_t initial, kauth_acl_t *product, int is
 	}
 	if (inherit != NULL) {
 		for (i = 0; i < inherit->acl_entrycount; i++) {
-			/* inherit onto this object? */
+			/*
+			 * Inherit onto this object?  We inherit only if
+			 * the target object is a container object and the
+			 * KAUTH_ACE_DIRECTORY_INHERIT bit is set, OR if
+			 * if the target object is not a container, and
+			 * the KAUTH_ACE_FILE_INHERIT bit is set.
+			 */
 			if (inherit->acl_ace[i].ace_flags & (isdir ? KAUTH_ACE_DIRECTORY_INHERIT : KAUTH_ACE_FILE_INHERIT)) {
 				result->acl_ace[index] = inherit->acl_ace[i];
 				result->acl_ace[index].ace_flags |= KAUTH_ACE_INHERITED;
-				/* don't re-inherit? */
-				if (result->acl_ace[index].ace_flags & KAUTH_ACE_LIMIT_INHERIT)
+				/*
+				 * We do not re-inherit inheritance flags
+				 * if the ACE from the container has a
+				 * KAUTH_ACE_LIMIT_INHERIT, OR if the new
+				 * object is not itself a container (since
+				 * inheritance is always container-based).
+				 */
+				if ((result->acl_ace[index].ace_flags & KAUTH_ACE_LIMIT_INHERIT) || !isdir) {
 					result->acl_ace[index].ace_flags &=
-					    ~(KAUTH_ACE_DIRECTORY_INHERIT | KAUTH_ACE_FILE_INHERIT | KAUTH_ACE_LIMIT_INHERIT);
+					    ~(KAUTH_ACE_INHERIT_CONTROL_FLAGS);
+				}
 				index++;
 			}
 		}
diff --git a/bsd/kern/kern_bsm_audit.c b/bsd/kern/kern_bsm_audit.c
index b4ddb4064..4c58ed475 100644
--- a/bsd/kern/kern_bsm_audit.c
+++ b/bsd/kern/kern_bsm_audit.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 #include <sys/types.h>
 #include <sys/vnode_internal.h>
@@ -43,6 +55,10 @@
 #include <kern/lock.h>
 #include <kern/kalloc.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 /* The number of BSM records allocated. */
 static int bsm_rec_count = 0; 
 
@@ -57,7 +73,10 @@ LIST_HEAD(, au_record) bsm_free_q;
 /*
  * Lock for serializing access to the list of audit records.
  */
-static mutex_t	*bsm_audit_mutex;
+static lck_grp_t	*bsm_audit_grp;
+static lck_attr_t	*bsm_audit_attr;
+static lck_grp_attr_t	*bsm_audit_grp_attr;
+static lck_mtx_t	*bsm_audit_mutex;
 
 static void audit_sys_auditon(struct audit_record *ar, struct au_record *rec);
 
@@ -69,7 +88,10 @@ kau_init(void)
 {
 	printf("BSM auditing present\n");
 	LIST_INIT(&bsm_free_q);
-	bsm_audit_mutex = mutex_alloc(0);
+	bsm_audit_grp_attr = lck_grp_attr_alloc_init();
+	bsm_audit_grp = lck_grp_alloc_init("BSM Audit", bsm_audit_grp_attr);
+	bsm_audit_attr = lck_attr_alloc_init();
+	bsm_audit_mutex = lck_mtx_alloc_init(bsm_audit_grp, bsm_audit_attr);
 	au_evclassmap_init();
 }
 
@@ -89,21 +111,21 @@ kau_open(void)
 	/* 
 	 * Find an unused record, remove it from the free list, mark as used
 	 */  
-	mutex_lock(bsm_audit_mutex);
+	lck_mtx_lock(bsm_audit_mutex);
 	if (!LIST_EMPTY(&bsm_free_q)) {
 		rec = LIST_FIRST(&bsm_free_q);
 		LIST_REMOVE(rec, au_rec_q);
 	}
-	mutex_unlock(bsm_audit_mutex);
+	lck_mtx_unlock(bsm_audit_mutex);
 
 	if (rec == NULL) {
-		mutex_lock(bsm_audit_mutex);
+		lck_mtx_lock(bsm_audit_mutex);
 		if (bsm_rec_count >= MAX_AUDIT_RECORDS) {
 			/* XXX We need to increase size of MAX_AUDIT_RECORDS */
-			mutex_unlock(bsm_audit_mutex);
+			lck_mtx_unlock(bsm_audit_mutex);
 			return NULL;
 		}
-		mutex_unlock(bsm_audit_mutex);
+		lck_mtx_unlock(bsm_audit_mutex);
 			
 		/*
 		 * Create a new BSM kernel record.
@@ -117,9 +139,9 @@ kau_open(void)
 			kfree(rec, sizeof(*rec));
 			return NULL;
 		}
-		mutex_lock(bsm_audit_mutex);
+		lck_mtx_lock(bsm_audit_mutex);
 		bsm_rec_count++;
-		mutex_unlock(bsm_audit_mutex);
+		lck_mtx_unlock(bsm_audit_mutex);
 	}
 	memset(rec->data, 0, MAX_AUDIT_RECORD_SIZE);
 
@@ -208,19 +230,39 @@ void kau_free(struct au_record *rec)
 	rec->used = 0;
 	rec->len = 0;	
 
-	mutex_lock(bsm_audit_mutex);
+	lck_mtx_lock(bsm_audit_mutex);
 
 	/* Add the record to the freelist */
 	LIST_INSERT_HEAD(&bsm_free_q, rec, au_rec_q);
 	
-	mutex_unlock(bsm_audit_mutex);
+	lck_mtx_unlock(bsm_audit_mutex);
 
 }
 
 /*
  * XXX May want turn some (or all) of these macros into functions in order
- * to reduce the generated code sized.
+ * to reduce the generated code size.
  */
+#if CONFIG_MACF
+#define MAC_VNODE1_LABEL_TOKEN						\
+	do {								\
+		if (ar->ar_vnode1_mac_labels != NULL) {			\
+			tok = au_to_text(ar->ar_vnode1_mac_labels);	\
+			kau_write(rec, tok);				\
+		}							\
+	} while (0)
+
+#define MAC_VNODE2_LABEL_TOKEN						\
+	do {								\
+		if (ar->ar_vnode2_mac_labels != NULL) {			\
+			tok = au_to_text(ar->ar_vnode2_mac_labels);	\
+			kau_write(rec, tok);				\
+		}							\
+	} while (0)
+#else
+#define MAC_VNODE1_LABEL_TOKEN
+#define MAC_VNODE2_LABEL_TOKEN
+#endif
 #define UPATH1_TOKENS	\
 	do { \
 		if (ar->ar_valid_arg & ARG_UPATH1) {  		\
@@ -250,6 +292,7 @@ void kau_free(struct au_record *rec)
 		if (ar->ar_valid_arg & ARG_VNODE1) {  		\
 			tok = kau_to_attr32(&ar->ar_arg_vnode1);\
 			kau_write(rec, tok);			\
+			MAC_VNODE1_LABEL_TOKEN;			\
 		}						\
 	} while (0)
 
@@ -262,6 +305,7 @@ void kau_free(struct au_record *rec)
 		if (ar->ar_valid_arg & ARG_VNODE1) {  		\
 			tok = kau_to_attr32(&ar->ar_arg_vnode1);\
 			kau_write(rec, tok);			\
+			MAC_VNODE1_LABEL_TOKEN;			\
 		}						\
 	} while (0)
 
@@ -272,8 +316,9 @@ void kau_free(struct au_record *rec)
 			kau_write(rec, tok);			\
 		}						\
 		if (ar->ar_valid_arg & ARG_VNODE2) {  		\
-			tok = kau_to_attr32(&ar->ar_arg_vnode1);\
+			tok = kau_to_attr32(&ar->ar_arg_vnode2);\
 			kau_write(rec, tok);			\
+			MAC_VNODE2_LABEL_TOKEN;			\
 		}						\
 	} while (0)
 
@@ -285,6 +330,7 @@ void kau_free(struct au_record *rec)
 			if (ar->ar_valid_arg & ARG_VNODE1) {  	\
 				tok = kau_to_attr32(&ar->ar_arg_vnode1);\
 				kau_write(rec, tok);		\
+				MAC_VNODE1_LABEL_TOKEN;		\
 			}					\
 		} else {					\
 			tok = au_to_arg32(1, "no path: fd", ar->ar_arg_fd); \
@@ -307,6 +353,14 @@ void kau_free(struct au_record *rec)
 		}							\
 	} while (0)							\
 
+#define PROCESS_MAC_TOKENS						\
+	do {								\
+		if (ar->ar_valid_arg & ARG_MAC_STRING) {		\
+			tok = au_to_text(ar->ar_arg_mac_string);	\
+			kau_write(rec, tok);				\
+		}							\
+	} while (0)							\
+
 /*
  * Implement auditing for the auditon() system call. The audit tokens
  * that are generated depend on the command that was sent into the 
@@ -416,11 +470,10 @@ audit_sys_auditon(struct audit_record *ar, struct au_record *rec)
 int
 kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 {
-	struct au_token *tok, *subj_tok;
+	struct au_token *tok = NULL, *subj_tok;
 	struct au_record *rec;
 	au_tid_t tid;
 	struct audit_record *ar;
-	int ctr;
 
 	*pau = NULL;
 	if (kar == NULL)
@@ -569,12 +622,12 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 	case AUE_GETAUDIT_ADDR:
 	case AUE_GETAUID:
 	case AUE_GETFSSTAT:
+	case AUE_MAC_GETFSSTAT:
 	case AUE_PIPE:
 	case AUE_SETPGRP:
 	case AUE_SETRLIMIT:
 	case AUE_SETSID:
 	case AUE_SETTIMEOFDAY:
-	case AUE_NEWSYSTEMSHREG:
 		/* Header, subject, and return tokens added at end */
 		break;
 
@@ -655,6 +708,10 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 	case AUE_FUTIMES:
 	case AUE_GETDIRENTRIES:
 	case AUE_GETDIRENTRIESATTR:
+	case AUE_EXTATTR_GET_FD:
+	case AUE_EXTATTR_LIST_FD:
+	case AUE_EXTATTR_SET_FD:
+	case AUE_EXTATTR_DELETE_FD:
 		FD_KPATH1_VNODE1_TOKENS;
 		break;
 	
@@ -692,6 +749,18 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		kau_write(rec, tok);
 		break;
 	
+	case AUE_GETLCID:
+		tok = au_to_arg32(1, "pid", (u_int32_t)ar->ar_arg_pid);
+		kau_write(rec, tok);
+		break;
+
+	case AUE_SETLCID:
+		tok = au_to_arg32(1, "pid", (u_int32_t)ar->ar_arg_pid);
+		kau_write(rec, tok);
+		tok = au_to_arg32(2, "lcid", (u_int32_t)ar->ar_arg_value);
+		kau_write(rec, tok);
+		break;
+
 	case AUE_IOCTL:
 		tok = au_to_arg32(2, "cmd", ar->ar_arg_cmd);
 		kau_write(rec, tok);
@@ -716,27 +785,12 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		PROCESS_PID_TOKENS(1);
 		break;
 
-	case AUE_KTRACE:
-		tok = au_to_arg32(2, "ops", ar->ar_arg_cmd);
-		kau_write(rec, tok);
-		tok = au_to_arg32(3, "trpoints", ar->ar_arg_value);
-		kau_write(rec, tok);
-		PROCESS_PID_TOKENS(4);
-		UPATH1_KPATH1_VNODE1_TOKENS;
-		break;
-
 	case AUE_LINK:
 	case AUE_RENAME:
 		UPATH1_KPATH1_VNODE1_TOKENS;
 		UPATH2_TOKENS;
 		break;
 
-	case AUE_LOADSHFILE:
-		tok = au_to_arg32(4, "base addr", (u_int32_t)ar->ar_arg_addr);
-		kau_write(rec, tok);
-		UPATH1_KPATH1_VNODE1_TOKENS;
-		break;
-
 	case AUE_MKDIR:
 		tok = au_to_arg32(2, "mode", ar->ar_arg_mode);
 		kau_write(rec, tok);
@@ -773,6 +827,11 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		}
 		break;
 
+#if CONFIG_MACF
+	case AUE_MAC_MOUNT:
+		PROCESS_MAC_TOKENS;
+		/* fall through */
+#endif
 	case AUE_MOUNT:
 		/* XXX Need to handle NFS mounts */
 		tok = au_to_arg32(3, "flags", ar->ar_arg_fflags);
@@ -806,11 +865,6 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		}
 		break;
 
-	case AUE_RESETSHFILE:
-		tok = au_to_arg32(1, "base addr", (u_int32_t)ar->ar_arg_addr);
-		kau_write(rec, tok);
-		break;
-	
 	case AUE_OPEN_RC:
 	case AUE_OPEN_RTC:
 	case AUE_OPEN_RWC:
@@ -891,6 +945,8 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		break;
 	case AUE_SETGROUPS:
 		if (ar->ar_valid_arg & ARG_GROUPSET) {
+			u_int ctr;
+
 			for(ctr = 0; ctr < ar->ar_arg_groups.gidset_size; ctr++)
 			{
 				tok = au_to_arg32(1, "setgroups", ar->ar_arg_groups.gidset[ctr]);
@@ -924,7 +980,7 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 	case AUE_SHMAT:
 		tok = au_to_arg32(1, "shmid", ar->ar_arg_svipc_id);
 		kau_write(rec, tok);
-		tok = au_to_arg32(2, "shmaddr", (int)ar->ar_arg_svipc_addr);
+		tok = au_to_arg64(2, "shmaddr", ar->ar_arg_svipc_addr);
 		kau_write(rec, tok);
 		if (ar->ar_valid_arg & ARG_SVIPC_PERM) {
 			tok = au_to_ipc(AT_IPC_SHM, ar->ar_arg_svipc_id);
@@ -970,7 +1026,7 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		break;
 
 	case AUE_SHMDT:
-		tok = au_to_arg32(1, "shmaddr", (int)ar->ar_arg_svipc_addr);
+		tok = au_to_arg64(1, "shmaddr", ar->ar_arg_svipc_addr);
 		kau_write(rec, tok);
 		break;
 
@@ -1006,8 +1062,8 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 			perm.cuid = ar->ar_arg_pipc_perm.pipc_uid;
 			perm.cgid = ar->ar_arg_pipc_perm.pipc_gid;
 			perm.mode = ar->ar_arg_pipc_perm.pipc_mode;
-			perm.seq = 0;
-			perm.key = 0;
+			perm._seq = 0;
+			perm._key = 0;
 			tok = au_to_ipc_perm(&perm);
 			kau_write(rec, tok);
 		}
@@ -1034,8 +1090,8 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 			perm.cuid = ar->ar_arg_pipc_perm.pipc_uid;
 			perm.cgid = ar->ar_arg_pipc_perm.pipc_gid;
 			perm.mode = ar->ar_arg_pipc_perm.pipc_mode;
-			perm.seq = 0;
-			perm.key = 0;
+			perm._seq = 0;
+			perm._key = 0;
 			tok = au_to_ipc_perm(&perm);
 			kau_write(rec, tok);
 		}
@@ -1057,6 +1113,8 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 	case AUE_SYSCTL:
 	case AUE_SYSCTL_NONADMIN:
 		if (ar->ar_valid_arg & (ARG_CTLNAME | ARG_LEN)) {
+			int ctr;
+
 			for (ctr = 0; ctr < ar->ar_arg_len; ctr++) {
 			  tok = au_to_arg32(1, "name", ar->ar_arg_ctlname[ctr]);
 			  kau_write(rec, tok);
@@ -1123,6 +1181,64 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		FD_KPATH1_VNODE1_TOKENS;
 		break;
 
+	case AUE_EXTATTR_GET_FILE:
+	case AUE_EXTATTR_SET_FILE:
+	case AUE_EXTATTR_LIST_FILE:
+	case AUE_EXTATTR_DELETE_FILE:
+	case AUE_EXTATTR_GET_LINK:
+	case AUE_EXTATTR_SET_LINK:
+	case AUE_EXTATTR_LIST_LINK:
+	case AUE_EXTATTR_DELETE_LINK:
+		UPATH1_KPATH1_VNODE1_TOKENS;
+		break;
+
+#if CONFIG_MACF
+	case AUE_MAC_GET_FILE:
+	case AUE_MAC_SET_FILE:
+	case AUE_MAC_GET_LINK:
+	case AUE_MAC_SET_LINK:
+	case AUE_MAC_GET_MOUNT:
+		UPATH1_KPATH1_VNODE1_TOKENS;
+		PROCESS_MAC_TOKENS;
+		break;
+
+	case AUE_MAC_GET_FD:
+	case AUE_MAC_SET_FD:
+		FD_KPATH1_VNODE1_TOKENS;
+		PROCESS_MAC_TOKENS;
+		break;
+
+	case AUE_MAC_SYSCALL:
+		PROCESS_MAC_TOKENS;
+		tok = au_to_arg32(3, "call", ar->ar_arg_value);
+		kau_write(rec, tok);
+		break;
+
+	case AUE_MAC_EXECVE:
+		UPATH1_KPATH1_VNODE1_TOKENS;
+		PROCESS_MAC_TOKENS;
+		break;
+
+	case AUE_MAC_GET_PID:
+		tok = au_to_arg32(1, "pid", (u_int32_t)ar->ar_arg_pid);
+		kau_write(rec, tok);
+		PROCESS_MAC_TOKENS;
+		break;
+
+	case AUE_MAC_GET_LCID:
+		tok = au_to_arg32(1, "lcid", (u_int32_t)ar->ar_arg_value);
+		kau_write(rec, tok);
+		PROCESS_MAC_TOKENS;
+		break;
+
+	case AUE_MAC_GET_PROC:
+	case AUE_MAC_SET_PROC:
+	case AUE_MAC_GET_LCTX:
+	case AUE_MAC_SET_LCTX:
+		PROCESS_MAC_TOKENS;
+		break;
+#endif
+
 	default: /* We shouldn't fall through to here. */
 		printf("BSM conversion requested for unknown event %d\n",
 			ar->ar_event);
@@ -1130,7 +1246,45 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		return BSM_NOAUDIT;
 	}
 
+#if CONFIG_MACF
+	do {
+		/* Convert the audit data from the MAC policies */
+		struct mac_audit_record *mar;
+		
+		LIST_FOREACH(mar, ar->ar_mac_records, records) {
+			switch (mar->type) {
+				case MAC_AUDIT_DATA_TYPE:
+					tok = au_to_data(AUP_BINARY, AUR_BYTE,
+					    mar->length, mar->data);
+					break;
+				case MAC_AUDIT_TEXT_TYPE:
+					tok = au_to_text((char*) mar->data);
+					break;
+				default:
+					/*
+					 * XXX: we can either continue,
+					 * skipping this particular entry,
+					 * or we can pre-verify the list and
+					 * abort before writing any records
+					 */
+					printf("kaudit_to_bsm(): BSM conversion requested for unknown mac_audit data type %d\n",
+					    mar->type);
+			}
+
+			kau_write(rec, tok);
+		}
+	} while (0);
+#endif
+
 	kau_write(rec, subj_tok); 
+
+#if CONFIG_MACF
+	if (ar->ar_cred_mac_labels != NULL) {
+		tok = au_to_text(ar->ar_cred_mac_labels);
+		kau_write(rec, tok);
+	}
+#endif
+
 	tok = au_to_return32((char)ar->ar_errno, ar->ar_retval);
 	kau_write(rec, tok);  /* Every record gets a return token */
 
diff --git a/bsd/kern/kern_bsm_klib.c b/bsd/kern/kern_bsm_klib.c
index d0d71faf4..f88f5bd38 100644
--- a/bsd/kern/kern_bsm_klib.c
+++ b/bsd/kern/kern_bsm_klib.c
@@ -1,26 +1,39 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #include <sys/systm.h>
+#include <sys/sysent.h>
 #include <sys/types.h>
 #include <sys/proc_internal.h>
 #include <sys/vnode_internal.h>
@@ -33,6 +46,8 @@
 #include <bsm/audit_kevents.h>
 #include <bsm/audit_klib.h>
 
+#include <kern/kalloc.h>
+
 /*
  * Initialize the system call to audit event mapping table. This table 
  * must be kept in sync with the system call table. This table is meant to
@@ -92,7 +107,7 @@ au_event_t sys_au_event[] = {
 	AUE_PIPE,			/*  42 = pipe */
 	AUE_NULL,			/*  43 = getegid */
 	AUE_NULL,			/*  44 = profil */
-	AUE_KTRACE,			/*  45 = ktrace */
+	AUE_NULL,			/*  45 = old ktrace */
 	AUE_NULL,			/*  46 = sigaction */
 	AUE_NULL,			/*  47 = getgid */
 	AUE_NULL,			/*  48 = sigprocmask */
@@ -352,9 +367,9 @@ au_event_t sys_au_event[] = {
 	AUE_NULL,			/* 293 */
 	AUE_NULL,			/* 294 */
 	AUE_NULL,			/* 295 */
-	AUE_LOADSHFILE,			/* 296 = load_shared_file */
-	AUE_RESETSHFILE,		/* 297 = reset_shared_file */
-	AUE_NEWSYSTEMSHREG,		/* 298 = new_system_shared_regions */
+	AUE_NULL,			/* 296 */
+	AUE_NULL,			/* 297 */
+	AUE_NULL,			/* 298 */
 	AUE_NULL,			/* 299 */
 	AUE_NULL,			/* 300 */
 	AUE_NULL,			/* 301 */
@@ -425,9 +440,73 @@ au_event_t sys_au_event[] = {
 	AUE_NULL,			/* 366 */
 	AUE_NULL,			/* 367 */
 	AUE_NULL,			/* 368 */
-	AUE_NULL			/* 369 */
+	AUE_NULL,			/* 369 */
+	AUE_NULL,			/* 370 */
+	AUE_NULL,			/* 371 */
+	AUE_NULL,			/* 372 */
+	AUE_NULL,			/* 373 */
+	AUE_NULL,			/* 374 */
+	AUE_NULL,			/* 375 */
+	AUE_NULL,			/* 376 */
+	AUE_NULL,			/* 377 */
+	AUE_NULL,			/* 378 */
+	AUE_NULL,			/* 379 */
+	AUE_MAC_EXECVE,			/* 380 = __mac_execve */
+	AUE_MAC_SYSCALL,       		/* 381 = __mac_syscall */
+	AUE_MAC_GET_FILE,	       	/* 382 = __mac_get_file */
+	AUE_MAC_SET_FILE,	       	/* 383 = __mac_set_file */
+	AUE_MAC_GET_LINK,	       	/* 384 = __mac_get_link */
+	AUE_MAC_SET_LINK,	       	/* 385 = __mac_set_link */
+	AUE_MAC_GET_PROC,			/* 386 = __mac_get_proc */
+	AUE_MAC_SET_PROC,      		/* 387 = __mac_set_proc */
+	AUE_MAC_GET_FD,			/* 388 = __mac_get_fd */
+	AUE_MAC_SET_FD,			/* 389 = __mac_set_fd */
+	AUE_MAC_GET_PID,		/* 390 = __mac_get_pid */
+	AUE_MAC_GET_LCID,		/* 391 = __mac_get_lcid */
+	AUE_MAC_GET_LCTX,		/* 392 = __mac_get_lctx */
+	AUE_MAC_SET_LCTX,		/* 393 = __mac_set_lctx */
+	AUE_SETLCID,			/* 394 = setlcid */
+	AUE_GETLCID,			/* 395 = getlcid */
+	AUE_NULL,			/* 396 = read_nocancel */
+	AUE_NULL,			/* 397 = write_nocancel */
+	AUE_OPEN_RWTC,			/* 398 = open_nocancel */
+	AUE_CLOSE,			/* 399 = close_nocancel */
+	AUE_NULL,			/* 400 = wait4_nocancel */
+	AUE_RECVMSG,			/* 401 = recvmsg_nocancel */
+	AUE_SENDMSG,			/* 402 = sendmsg_nocancel */
+	AUE_RECVFROM,			/* 403 = recvfrom_nocancel */
+	AUE_ACCEPT,			/* 404 = accept_nocancel */
+	AUE_NULL,			/* 405 = msync_nocancel */
+	AUE_FCNTL,			/* 406 = fcntl_nocancel */
+	AUE_NULL,			/* 407 = select_nocancel */
+	AUE_NULL,			/* 408 = fsync_nocancel */
+	AUE_CONNECT,			/* 409 = connect_nocancel */
+	AUE_NULL,			/* 410 = sigsuspend_nocancel */
+	AUE_NULL,			/* 411 = readv_nocancel */
+	AUE_NULL,			/* 412 = writev_nocancel */
+	AUE_SENDTO,			/* 413 = sendto_nocancel */
+	AUE_NULL,			/* 414 = pread_nocancel */
+	AUE_NULL,			/* 415 = pwrite_nocancel */
+	AUE_NULL,			/* 416 = waitid_nocancel */
+	AUE_NULL,			/* 417 = poll_nocancel */
+	AUE_MSGSND,			/* 418 = msgsnd_nocancel */
+	AUE_MSGRCV,			/* 419 = msgrcv_nocancel */
+	AUE_NULL,			/* 420 = sem_wait_nocancel */
+	AUE_NULL,			/* 421 = aio_suspend_nocancel */
+	AUE_NULL,			/* 422 = __sigwait_nocancel */
+	AUE_NULL,			/* 423 = __semwait_signal_nocancel */
+	AUE_MAC_MOUNT,			/* 424 = __mac_mount */
+	AUE_MAC_GET_MOUNT,		/* 425 = __mac_get_mount */
+	AUE_MAC_GETFSSTAT,		/* 426 = __mac_getfsstat */
+
 };
-int	nsys_au_event = sizeof(sys_au_event) / sizeof(sys_au_event[0]);
+
+/*
+ * Verify that sys_au_event has an entry for every syscall.
+ */
+int 	audit_sys_table_size_check[(
+		(sizeof(sys_au_event) / sizeof(sys_au_event[0])) == NUM_SYSENT)?
+			1 : -1] __unused;
 
 /*
  * Hash table functions for the audit event number to event class mask mapping.
@@ -480,7 +559,7 @@ void au_evclassmap_insert(au_event_t event, au_class_t class)
 			return;
 		}
 	}
-	kmem_alloc(kernel_map, (vm_offset_t *)&evc, sizeof(*evc));
+	evc = (struct evclass_elem *)kalloc(sizeof (*evc));
 	if (evc == NULL) {
 		return;
 	}
@@ -489,7 +568,8 @@ void au_evclassmap_insert(au_event_t event, au_class_t class)
 	LIST_INSERT_HEAD(&evcl->head, evc, entry);
 }
 
-void au_evclassmap_init() 
+void
+au_evclassmap_init(void)
 {
 	int i;
 	for (i = 0; i < EVCLASSMAP_HASH_TABLE_SIZE; i++) {
@@ -497,7 +577,7 @@ void au_evclassmap_init()
 	}
 
 	/* Set up the initial event to class mapping for system calls.  */ 
-	for (i = 0; i < nsys_au_event; i++) {
+	for (i = 0; i < NUM_SYSENT; i++) {
 		if (sys_au_event[i] != AUE_NULL) {
 			au_evclassmap_insert(sys_au_event[i], AU_NULL);
 	}
@@ -789,52 +869,43 @@ int auditon_command_event(int cmd)
 }
 
 /* 
- * Create a canonical path from given path by prefixing either the
- * root directory, or the current working directory.
- * If the process working directory is NULL, we could use 'rootvnode'
- * to obtain the root directoty, but this results in a volfs name
- * written to the audit log. So we will leave the filename starting
- * with '/' in the audit log in this case.
+ * Create a canonical path from given path by prefixing the supplied
+ * current working directory, which may be the root directory.
  */
-int canon_path(struct proc *p, char *path, char *cpath)
+int
+canon_path(struct vnode *cwd_vp, char *path, char *cpath)
 {
-	char *bufp;
 	int len;
-	struct vnode *vnp;
-	struct filedesc *fdp;
 	int ret;
+	char *bufp = path;
 
-	fdp = p->p_fd;
-	bufp = path;
+	/*
+	 * convert multiple leading '/' into a single '/' if the cwd_vp is
+	 * NULL (i.e. an absolute path), and strip them entirely if the
+	 * cwd_vp represents a chroot directory (i.e. the caller checked for
+	 * an initial '/' character itself, saw one, and passed fdp->fd_rdir).
+	 * Somewhat complicated, but it places the onus for locking structs
+	 * involved on the caller, and makes proxy operations explicit rather
+	 * than implicit.
+	 */
 	if (*(path) == '/') {
 		while (*(bufp) == '/') 
 			bufp++;			/* skip leading '/'s	     */
-		/* If no process root, or it is the same as the system root,
-		 * audit the path as passed in with a single '/'.
-		 */
-		if ((fdp->fd_rdir == NULL) ||
-		    (fdp->fd_rdir == rootvnode)) {	
-			vnp = NULL;
+		if (cwd_vp == NULL)
 			bufp--;			/* restore one '/'	     */
-		} else {
-			vnp = fdp->fd_rdir;	/* use process root	     */
-		}
-	} else {
-		vnp = fdp->fd_cdir;	/* prepend the current dir  */
-		bufp = path;
 	}
-	if (vnp != NULL) {
+	if (cwd_vp != NULL) {
 		len = MAXPATHLEN;
-		ret = vn_getpath(vnp, cpath, &len);
+		ret = vn_getpath(cwd_vp, cpath, &len);
 		if (ret != 0) {
 			cpath[0] = '\0';
 			return (ret);
 		}
 		if (len < MAXPATHLEN)
 			cpath[len-1] = '/';	
-		strncpy(cpath + len, bufp, MAXPATHLEN - len);
+		strlcpy(cpath + len, bufp, MAXPATHLEN - len);
 	} else {
-		strncpy(cpath, bufp, MAXPATHLEN);
+		strlcpy(cpath, bufp, MAXPATHLEN);
 	}
 	return (0);
 }
diff --git a/bsd/kern/kern_bsm_token.c b/bsd/kern/kern_bsm_token.c
index 7be61356e..9da0c96fa 100644
--- a/bsd/kern/kern_bsm_token.c
+++ b/bsd/kern/kern_bsm_token.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/types.h>
@@ -56,7 +62,8 @@
  * text length             2 bytes
  * text                    N bytes + 1 terminating NULL byte
  */
-token_t *au_to_arg32(char n, char *text, u_int32_t v)
+token_t *
+au_to_arg32(char n, const char *text, u_int32_t v)
 {
 	token_t *t;
 	u_char *dptr;
@@ -66,11 +73,7 @@ token_t *au_to_arg32(char n, char *text, u_int32_t v)
 		return NULL;	
 	}
 	
-	/* Make sure that text is null terminated */
 	textlen = strlen(text);
-	if(text[textlen] != '\0') {	
-		return NULL;
-	}
 	
 	GET_TOKEN_AREA(t, dptr, 9 + textlen);
 	if(t == NULL) {
@@ -89,7 +92,8 @@ token_t *au_to_arg32(char n, char *text, u_int32_t v)
 
 }
 
-token_t *au_to_arg64(char n, char *text, u_int64_t v)
+token_t *
+au_to_arg64(char n, const char *text, u_int64_t v)
 {
 	token_t *t;
 	u_char *dptr;
@@ -99,11 +103,7 @@ token_t *au_to_arg64(char n, char *text, u_int64_t v)
 		return NULL;	
 	}
 	
-	/* Make sure that text is null terminated */
 	textlen = strlen(text);
-	if(text[textlen] != '\0') {	
-		return NULL;
-	}
 	
 	GET_TOKEN_AREA(t, dptr, 13 + textlen);
 	if(t == NULL) {
@@ -122,7 +122,8 @@ token_t *au_to_arg64(char n, char *text, u_int64_t v)
 
 }
 
-token_t *au_to_arg(char n, char *text, u_int32_t v)
+token_t *
+au_to_arg(char n, char *text, u_int32_t v)
 {
 	return au_to_arg32(n, text, v);
 }
@@ -210,7 +211,7 @@ token_t *au_to_attr(struct vnode_attr *attr)
  * data items              (depends on basic unit)
  */
 token_t *au_to_data(char unit_print, char unit_type,
-                    char unit_count, char *p)
+                    char unit_count, unsigned char *p)
 {
 	token_t *t;
 	u_char *dptr;
@@ -274,7 +275,8 @@ token_t *au_to_exit(int retval, int err)
 
 /*
  */
-token_t *au_to_groups(int *groups)
+token_t *
+au_to_groups(gid_t *groups)
 {
 	return au_to_newgroups(MAX_GROUPS, groups);	
 }
@@ -443,26 +445,19 @@ token_t *au_to_ipc_perm(struct ipc_perm *perm)
 	 */
 	ADD_U_CHAR(dptr, AU_IPCPERM_TOKEN);
 
-	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->uid);
-
-	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->gid);
-
-	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->cuid);
-
-	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->cgid);
+	ADD_U_INT32(dptr, perm->uid);
+	ADD_U_INT32(dptr, perm->gid);
+	ADD_U_INT32(dptr, perm->cuid);
+	ADD_U_INT32(dptr, perm->cgid);
 
 	ADD_U_INT16(dptr, pad0);
 	ADD_U_INT16(dptr, perm->mode);
 
 	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->seq);
+	ADD_U_INT16(dptr, perm->_seq);
 
 	ADD_U_INT16(dptr, pad0);
-	ADD_U_INT16(dptr, perm->key);
+	ADD_U_INT16(dptr, perm->_key);
 
 	return t;
 }
@@ -535,11 +530,7 @@ token_t *kau_to_file(const char *file, const struct timeval *tv)
 	if(file == NULL) {
 		return NULL;
 	}
-	/* Make sure that text is null terminated */
 	filelen = strlen(file);
-	if(file[filelen] != '\0') {
-		return NULL;
-	}
  
 	GET_TOKEN_AREA(t, dptr, filelen + 12);
 	if(t == NULL) {
@@ -566,7 +557,7 @@ token_t *kau_to_file(const char *file, const struct timeval *tv)
  * text length             2 bytes
  * text                    N bytes + 1 terminating NULL byte
  */
-token_t *au_to_text(char *text)
+token_t *au_to_text(const char *text)
 {
 	token_t *t;
 	u_char *dptr;
@@ -575,11 +566,7 @@ token_t *au_to_text(char *text)
 	if(text == NULL) {
 		return NULL;
 	}
-	/* Make sure that text is null terminated */
 	textlen = strlen(text);
-	if(text[textlen] != '\0') {
-		return NULL;
-	}
 
 	GET_TOKEN_AREA(t, dptr, textlen + 4);
 	if(t == NULL) {
@@ -609,11 +596,7 @@ token_t *au_to_path(char *text)
 	if(text == NULL) {
 		return NULL;
 	}
-	/* Make sure that text is null terminated */
 	textlen = strlen(text);
-	if(text[textlen] != '\0') {
-		return NULL;
-	}
 
 	GET_TOKEN_AREA(t, dptr, textlen + 4);
 	if(t == NULL) {
@@ -812,7 +795,7 @@ token_t *au_to_return(char status, u_int32_t ret)
  * token ID                1 byte
  * sequence number         4 bytes
  */
-token_t *au_to_seq(long audit_count)
+token_t *au_to_seq(u_int32_t audit_count)
 {
 	token_t *t;
 	u_char *dptr;
@@ -1157,10 +1140,6 @@ token_t *au_to_exec_args(const char **args)
 		int nextlen;
 		
 		nextlen = strlen(nextarg);
-		if(nextarg[nextlen] != '\0') {
-			return NULL;
-		}
-		
 		totlen += nextlen + 1;
 		count++;
 		nextarg = *(args + count);
@@ -1207,10 +1186,6 @@ token_t *au_to_exec_env(const char **env)
 		int nextlen;
 		
 		nextlen = strlen(nextenv);
-		if(nextenv[nextlen] != '\0') {
-			return NULL;
-		}
-		
 		totlen += nextlen + 1;
 		count++;
 		nextenv = *(env + count);
diff --git a/bsd/kern/kern_clock.c b/bsd/kern/kern_clock.c
index 76c5353de..432a0f0e8 100644
--- a/bsd/kern/kern_clock.c
+++ b/bsd/kern/kern_clock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -87,12 +93,7 @@
 #include <kern/thread_call.h>
 
 void bsd_uprofil(struct time_value *syst, user_addr_t pc);
-void get_procrustime(time_value_t *tv);
-int sysctl_clockrate(user_addr_t where, size_t *sizep);
 int tvtohz(struct timeval *tv);
-extern void psignal_sigprof(struct proc *);
-extern void psignal_vtalarm(struct proc *);
-extern void psignal_xcpu(struct proc *);
 
 /*
  * Clock handling routines.
@@ -111,144 +112,11 @@ extern void psignal_xcpu(struct proc *);
 
 /*
  * The hz hardware interval timer.
- * We update the events relating to real time.
- * If this timer is also being used to gather statistics,
- * we run through the statistics gathering routine as well.
  */
 
 int             hz = 100;                /* GET RID OF THIS !!! */
 int             tick = (1000000 / 100);  /* GET RID OF THIS !!! */
 
-int bsd_hardclockinit = 0;
-/*ARGSUSED*/
-void
-bsd_hardclock(
-                boolean_t usermode, 
-#ifdef GPROF
-                caddr_t pc, 
-#else
-                __unused caddr_t pc, 
-#endif
-                int numticks
-             )
-{
-	register struct proc *p;
-	register thread_t	thread;
-	int nusecs = numticks * tick;
-	struct timeval		tv;
-
-	if (!bsd_hardclockinit)
-		return;
-
-	if (bsd_hardclockinit < 0) {
-	    return;
-	}
-
-	thread = current_thread();
-	/*
-	 * Charge the time out based on the mode the cpu is in.
-	 * Here again we fudge for the lack of proper interval timers
-	 * assuming that the current state has been around at least
-	 * one tick.
-	 */
-	p = (struct proc *)current_proc();
-	if (p && ((p->p_flag & P_WEXIT) == 0)) {
-		if (usermode) {		
-			if (p->p_stats && p->p_stats->p_prof.pr_scale) {
-				p->p_flag |= P_OWEUPC;
-				astbsd_on();
-			}
-
-			/*
-			 * CPU was in user state.  Increment
-			 * user time counter, and process process-virtual time
-			 * interval timer. 
-			 */
-			if (p->p_stats && 
-				timerisset(&p->p_stats->p_timer[ITIMER_VIRTUAL].it_value) &&
-				!itimerdecr(&p->p_stats->p_timer[ITIMER_VIRTUAL], nusecs)) {
-                        
-				/* does psignal(p, SIGVTALRM) in a thread context */
-				thread_call_func((thread_call_func_t)psignal_vtalarm, p, FALSE);
-			}
-		}
-
-		/*
-		 * If the cpu is currently scheduled to a process, then
-		 * charge it with resource utilization for a tick, updating
-		 * statistics which run in (user+system) virtual time,
-		 * such as the cpu time limit and profiling timers.
-		 * This assumes that the current process has been running
-		 * the entire last tick.
-		 */
-		if (!is_thread_idle(thread)) {		
-			if (p->p_limit &&
-				p->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur != RLIM_INFINITY) {
-				time_value_t	sys_time, user_time;
-
-				thread_read_times(thread, &user_time, &sys_time);
-				if ((sys_time.seconds + user_time.seconds + 1) >
-					p->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur) {
-                        
-					/* does psignal(p, SIGXCPU) in a thread context */
-					thread_call_func((thread_call_func_t)psignal_xcpu, p, FALSE);
-
-					if (p->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur <
-						p->p_limit->pl_rlimit[RLIMIT_CPU].rlim_max)
-						p->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur += 5;
-				}
-			}
-			if (timerisset(&p->p_stats->p_timer[ITIMER_PROF].it_value) &&
-				!itimerdecr(&p->p_stats->p_timer[ITIMER_PROF], nusecs)) {
-                        
-				/* does psignal(p, SIGPROF) in a thread context */
-				thread_call_func((thread_call_func_t)psignal_sigprof, p, FALSE);
-			}
-		}
-	}
-
-#ifdef GPROF
-	/*
-	 * Gather some statistics.
-	 */
-	gatherstats(usermode, pc);
-#endif
-}
-
-/*
- * Gather some statistics.
- */
-/*ARGSUSED*/
-void
-gatherstats(
-#ifdef GPROF
-                boolean_t	usermode,
-                caddr_t		pc
-#else
-                __unused boolean_t	usermode,
-                __unused caddr_t		pc
-#endif
-	       )
-	        
-{
-#ifdef GPROF
-	if (!usermode) {
-		struct gmonparam *p = &_gmonparam;
-
-		if (p->state == GMON_PROF_ON) {
-			register int s;
-
-			s = pc - p->lowpc;
-			if (s < p->textsize) {
-				s /= (HISTFRACTION * sizeof(*p->kcount));
-				p->kcount[s]++;
-			}
-		}
-	}
-#endif
-}
-
-
 /*
  * Kernel timeout services.
  */
@@ -277,8 +145,8 @@ timeout(
  */
 void
 untimeout(
-	register timeout_fcn_t		fcn,
-	register void				*param)
+	timeout_fcn_t		fcn,
+	void			*param)
 {
 	thread_call_func_cancel((thread_call_func_t)fcn, param, FALSE);
 }
@@ -311,8 +179,8 @@ bsd_timeout(
  */
 void
 bsd_untimeout(
-	register timeout_fcn_t		fcn,
-	register void				*param)
+	timeout_fcn_t		fcn,
+	void			*param)
 {
 	thread_call_func_cancel((thread_call_func_t)fcn, param, FALSE);
 }
@@ -324,12 +192,11 @@ bsd_untimeout(
  * absolute time.
  */
 int
-hzto(tv)
-	struct timeval *tv;
+hzto(struct timeval *tv)
 {
 	struct timeval now;
-	register long ticks;
-	register long sec;
+	long ticks;
+	long sec;
 
 	microtime(&now);
 	/*
@@ -357,8 +224,9 @@ hzto(tv)
 /*
  * Return information about system clocks.
  */
-int
-sysctl_clockrate(user_addr_t where, size_t *sizep)
+static int
+sysctl_clockrate
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, __unused struct sysctl_req *req)
 {
 	struct clockinfo clkinfo;
 
@@ -369,9 +237,13 @@ sysctl_clockrate(user_addr_t where, size_t *sizep)
 	clkinfo.tick = tick;
 	clkinfo.profhz = hz;
 	clkinfo.stathz = hz;
-	return sysctl_rdstruct(where, sizep, USER_ADDR_NULL, &clkinfo, sizeof(clkinfo));
+	return sysctl_io_opaque(req, &clkinfo, sizeof(clkinfo), NULL);
 }
 
+SYSCTL_PROC(_kern, KERN_CLOCKRATE, clockrate,
+		CTLTYPE_STRUCT | CTLFLAG_RD,
+		0, 0, sysctl_clockrate, "S,clockinfo", "");
+
 
 /*
  * Compute number of ticks in the specified amount of time.
@@ -379,8 +251,8 @@ sysctl_clockrate(user_addr_t where, size_t *sizep)
 int
 tvtohz(struct timeval *tv)
 {
-	register unsigned long ticks;
-	register long sec, usec;
+	unsigned long ticks;
+	long sec, usec;
 
 	/*
 	 * If the number of usecs in the whole seconds part of the time
@@ -439,31 +311,30 @@ tvtohz(struct timeval *tv)
  * keeps the profile clock running constantly.
  */
 void
-startprofclock(p)
-	register struct proc *p;
+startprofclock(struct proc *p)
 {
 	if ((p->p_flag & P_PROFIL) == 0)
-		p->p_flag |= P_PROFIL;
+		OSBitOrAtomic(P_PROFIL, (UInt32 *)&p->p_flag);
 }
 
 /*
  * Stop profiling on a process.
  */
 void
-stopprofclock(p)
-	register struct proc *p;
+stopprofclock(struct proc *p)
 {
 	if (p->p_flag & P_PROFIL)
-		p->p_flag &= ~P_PROFIL;
+		OSBitAndAtomic(~((uint32_t)P_PROFIL), (UInt32 *)&p->p_flag);
 }
 
+/* TBD locking user profiling is not resolved yet */
 void
 bsd_uprofil(struct time_value *syst, user_addr_t pc)
 {
-struct proc *p = current_proc();
-int		ticks;
-struct timeval	*tv;
-struct timeval st;
+	struct proc *p = current_proc();
+	int		ticks;
+	struct timeval	*tv;
+	struct timeval st;
 
 	if (p == NULL)
 	        return;
@@ -482,6 +353,7 @@ struct timeval st;
 		addupc_task(p, pc, ticks);
 }
 
+/* TBD locking user profiling is not resolved yet */
 void
 get_procrustime(time_value_t *tv)
 {
@@ -493,7 +365,9 @@ get_procrustime(time_value_t *tv)
 	if ( !(p->p_flag & P_PROFIL))
 	        return;
 
+	//proc_lock(p);
 	st = p->p_stats->p_ru.ru_stime;
+	//proc_unlock(p);
 	
 	tv->seconds = st.tv_sec;
 	tv->microseconds = st.tv_usec;
diff --git a/bsd/kern/kern_control.c b/bsd/kern/kern_control.c
index d4888bacb..877fe6778 100644
--- a/bsd/kern/kern_control.c
+++ b/bsd/kern/kern_control.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -42,6 +48,7 @@
 #include <sys/sys_domain.h>
 #include <sys/kern_event.h>
 #include <sys/kern_control.h>
+#include <sys/kauth.h>
 #include <net/if_var.h>
 
 #include <mach/vm_types.h>
@@ -60,8 +67,6 @@
  * Definitions and vars for we support
  */
 
-static u_int32_t		ctl_last_id = 0;
-static u_int32_t		ctl_max = 256;
 static u_int32_t		ctl_maxunit = 65536;
 static lck_grp_attr_t	*ctl_lck_grp_attr = 0;
 static lck_attr_t		*ctl_lck_attr = 0;
@@ -70,7 +75,7 @@ static lck_mtx_t 		*ctl_mtx;
 
 
 /* all the controllers are chained */
-TAILQ_HEAD(, kctl) 	ctl_head;
+TAILQ_HEAD(kctl_list, kctl) 	ctl_head;
 
 static int ctl_attach(struct socket *, int, struct proc *);
 static int ctl_detach(struct socket *);
@@ -84,7 +89,6 @@ static int ctl_send(struct socket *, int, struct mbuf *,
 static int ctl_ctloutput(struct socket *, struct sockopt *);
 static int ctl_peeraddr(struct socket *so, struct sockaddr **nam);
 
-static struct kctl *ctl_find_by_id(u_int32_t);
 static struct kctl *ctl_find_by_name(const char *);
 static struct kctl *ctl_find_by_id_unit(u_int32_t id, u_int32_t unit);
 
@@ -281,6 +285,7 @@ ctl_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
     int					error = 0;
     struct sockaddr_ctl	sa;
     struct ctl_cb		*kcb = (struct ctl_cb *)so->so_pcb;
+    struct ctl_cb		*kcb_next = NULL;
     
     if (kcb == 0)
     	panic("ctl_connect so_pcb null\n");
@@ -308,9 +313,9 @@ ctl_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
             lck_mtx_unlock(ctl_mtx);
             return(EINVAL);
         }
-        if ((error = proc_suser(p))) {
+        if (kauth_cred_issuser(kauth_cred_get()) == 0) {
             lck_mtx_unlock(ctl_mtx);
-            return error;
+            return EPERM;
         }
     }
 
@@ -320,25 +325,35 @@ ctl_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 			return EBUSY;
 		}
 	} else {
-    	u_int32_t	unit = kctl->lastunit + 1;
+		/* Find an unused ID, assumes control IDs are listed in order */
+    	u_int32_t	unit = 1;
+    	
+    	TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {
+    		if (kcb_next->unit > unit) {
+    			/* Found a gap, lets fill it in */
+    			break;
+    		}
+    		unit = kcb_next->unit + 1;
+    		if (unit == ctl_maxunit)
+    			break;
+    	}
     	
-    	while (1) {
-    	    if (unit == ctl_maxunit)
-    	    	unit = 1;
-		    if (kcb_find(kctl, unit) == NULL) {
-		    	kctl->lastunit = sa.sc_unit = unit;
-		    	break;
-		    }
-	    	if (unit++ == kctl->lastunit) {
-	    	    lck_mtx_unlock(ctl_mtx);
-    	    	return EBUSY;
-    	    }
-	    }
+		if (unit == ctl_maxunit) {
+			lck_mtx_unlock(ctl_mtx);
+			return EBUSY;
+		}
+		
+		sa.sc_unit = unit;
     }
 
 	kcb->unit = sa.sc_unit;
     kcb->kctl = kctl;
-    TAILQ_INSERT_TAIL(&kctl->kcb_head, kcb, next);
+    if (kcb_next != NULL) {
+    	TAILQ_INSERT_BEFORE(kcb_next, kcb, next);
+    }
+    else {
+		TAILQ_INSERT_TAIL(&kctl->kcb_head, kcb, next);
+	}
     lck_mtx_unlock(ctl_mtx);
 
     error = soreserve(so, kctl->sendbufsize, kctl->recvbufsize);
@@ -494,7 +509,7 @@ ctl_enqueuedata(void *kctlref, u_int32_t unit, void *data, size_t len, u_int32_t
 		return EINVAL;
 	
 	socket_lock(so, 1);
-	if ((size_t)sbspace(&so->so_rcv) < len) {
+	if (sbspace(&so->so_rcv) < (long)len) {
 		error = ENOBUFS;
 		goto bye;
 	}
@@ -534,6 +549,7 @@ ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space)
 	struct ctl_cb 	*kcb;
 	struct kctl		*kctl = (struct kctl *)kctlref;
 	struct socket 	*so;
+	long avail;
 	
 	if (kctlref == NULL || space == NULL)
 		return EINVAL;
@@ -547,7 +563,8 @@ ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space)
 		return EINVAL;
 	
 	socket_lock(so, 1);
-	*space = sbspace(&so->so_rcv);
+	avail = sbspace(&so->so_rcv);
+	*space = (avail < 0) ? 0 : avail;
 	socket_unlock(so, 1);
 
 	return 0;
@@ -576,10 +593,14 @@ ctl_ctloutput(struct socket *so, struct sockopt *sopt)
 		case SOPT_SET:
 			if (kctl->setopt == NULL)
 				return(ENOTSUP);
-			MALLOC(data, void *, sopt->sopt_valsize, M_TEMP, M_WAITOK);
-			if (data == NULL)
-				return(ENOMEM);
-			error = sooptcopyin(sopt, data, sopt->sopt_valsize, sopt->sopt_valsize);
+			if (sopt->sopt_valsize == 0) {
+				data = NULL;
+			} else {
+				MALLOC(data, void *, sopt->sopt_valsize, M_TEMP, M_WAITOK);
+				if (data == NULL)
+					return(ENOMEM);
+				error = sooptcopyin(sopt, data, sopt->sopt_valsize, sopt->sopt_valsize);
+			}
 			if (error == 0) {
 				socket_unlock(so, 0);
 				error = (*kctl->setopt)(kcb->kctl, kcb->unit, kcb->userdata, sopt->sopt_name, 
@@ -672,10 +693,10 @@ ctl_ioctl(__unused struct socket *so, u_long cmd, caddr_t data,
  */
 errno_t
 ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
-{	
-	struct kctl 	*kctl = 0;
-	u_int32_t		id = -1;
-	u_int32_t		n;
+{
+	struct kctl 	*kctl = NULL;
+	struct kctl 	*kctl_next = NULL;
+	u_int32_t		id = 1;
 	size_t			name_len;
 	
 	if (userkctl == NULL)	/* sanity check */
@@ -693,29 +714,64 @@ ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
 	
 	lck_mtx_lock(ctl_mtx);
 	
-	if ((userkctl->ctl_flags & CTL_FLAG_REG_ID_UNIT) == 0) {    
+	/*
+	 * Kernel Control IDs
+	 *
+	 * CTL_FLAG_REG_ID_UNIT indicates the control ID and unit number are
+	 * static. If they do not exist, add them to the list in order. If the
+	 * flag is not set, we must find a new unique value. We assume the
+	 * list is in order. We find the last item in the list and add one. If
+	 * this leads to wrapping the id around, we start at the front of the
+	 * list and look for a gap.
+	 */
+	
+	if ((userkctl->ctl_flags & CTL_FLAG_REG_ID_UNIT) == 0) {
+		/* Must dynamically assign an unused ID */
+		
+		/* Verify the same name isn't already registered */
 		if (ctl_find_by_name(userkctl->ctl_name) != NULL) {
 			lck_mtx_unlock(ctl_mtx);
 			FREE(kctl, M_TEMP);
 			return(EEXIST);
 		}
-		for (n = 0, id = ctl_last_id + 1; n < ctl_max; id++, n++) {
+		
+		/* Start with 1 in case the list is empty */
+		id = 1;
+		kctl_next = TAILQ_LAST(&ctl_head, kctl_list);
+		
+		if (kctl_next != NULL) {
+			/* List was not empty, add one to the last item in the list */
+			id = kctl_next->id + 1;
+			kctl_next = NULL;
+			
+			/*
+			 * If this wrapped the id number, start looking at the front
+			 * of the list for an unused id.
+			 */
 			if (id == 0) {
-				n--;
-				continue;
+				/* Find the next unused ID */
+				id = 1;
+				
+				TAILQ_FOREACH(kctl_next, &ctl_head, next) {
+					if (kctl_next->id > id) {
+						/* We found a gap */
+						break;
+					}
+					
+					id = kctl_next->id + 1;
+				}
 			}
-			if (ctl_find_by_id(id) == 0)
-				break;
 		}
-		if (id == ctl_max) {
-			lck_mtx_unlock(ctl_mtx);
-			FREE(kctl, M_TEMP);
-			return(ENOBUFS);
-		}
-		userkctl->ctl_id =id;
+		
+		userkctl->ctl_id = id;
 		kctl->id = id;
 		kctl->reg_unit = -1;
 	} else {
+		TAILQ_FOREACH(kctl_next, &ctl_head, next) {
+			if (kctl_next->id > userkctl->ctl_id)
+				break;
+		}
+		
 		if (ctl_find_by_id_unit(userkctl->ctl_id, userkctl->ctl_unit) != NULL) {
 			lck_mtx_unlock(ctl_mtx);
 			FREE(kctl, M_TEMP);
@@ -724,7 +780,7 @@ ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
 		kctl->id = userkctl->ctl_id;
 		kctl->reg_unit = userkctl->ctl_unit;
 	}
-	strcpy(kctl->name, userkctl->ctl_name);
+	strlcpy(kctl->name, userkctl->ctl_name, MAX_KCTL_NAME);
 	kctl->flags = userkctl->ctl_flags;
 
 	/* Let the caller know the default send and receive sizes */
@@ -744,8 +800,10 @@ ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
 	
 	TAILQ_INIT(&kctl->kcb_head);
 	
-	TAILQ_INSERT_TAIL(&ctl_head, kctl, next);
-	ctl_max++;
+	if (kctl_next)
+		TAILQ_INSERT_BEFORE(kctl_next, kctl, next);
+	else
+		TAILQ_INSERT_TAIL(&ctl_head, kctl, next);
 	
 	lck_mtx_unlock(ctl_mtx);
 	
@@ -778,7 +836,6 @@ ctl_deregister(void *kctlref)
 	}
 
     TAILQ_REMOVE(&ctl_head, kctl, next);
-    ctl_max--;
 
     lck_mtx_unlock(ctl_mtx);
     
@@ -787,21 +844,6 @@ ctl_deregister(void *kctlref)
     return(0);
 }
 
-/*
- * Must be called with global lock taked
- */
-static struct kctl *
-ctl_find_by_id(u_int32_t id)
-{	
-    struct kctl 	*kctl;
-
-    TAILQ_FOREACH(kctl, &ctl_head, next)
-        if (kctl->id == id)
-            return kctl;
-
-    return NULL;
-}
-
 /*
  * Must be called with global ctl_mtx lock taked
  */
@@ -811,7 +853,7 @@ ctl_find_by_name(const char *name)
     struct kctl 	*kctl;
 
     TAILQ_FOREACH(kctl, &ctl_head, next)
-        if (strcmp(kctl->name, name) == 0)
+        if (strncmp(kctl->name, name, sizeof(kctl->name)) == 0)
             return kctl;
 
     return NULL;
@@ -879,7 +921,7 @@ ctl_post_msg(u_long event_code, u_int32_t id)
 static int
 ctl_lock(struct socket *so, int refcount, int lr)
  {
-	int lr_saved;
+	uint32_t lr_saved;
 	if (lr == 0) 
 		lr_saved = (unsigned int) __builtin_return_address(0);
 	else lr_saved = lr;
@@ -887,18 +929,18 @@ ctl_lock(struct socket *so, int refcount, int lr)
 	if (so->so_pcb) {
 		lck_mtx_lock(((struct ctl_cb *)so->so_pcb)->mtx);
 	} else  {
-		panic("ctl_lock: so=%x NO PCB! lr=%x\n", so, lr_saved);
+		panic("ctl_lock: so=%p NO PCB! lr=%x\n", so, lr_saved);
 		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
 	}
 	
 	if (so->so_usecount < 0)
-		panic("ctl_lock: so=%x so_pcb=%x lr=%x ref=%x\n",
+		panic("ctl_lock: so=%p so_pcb=%p lr=%x ref=%x\n",
 		so, so->so_pcb, lr_saved, so->so_usecount);
 	
 	if (refcount)
 		so->so_usecount++;
 
-	so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = lr_saved;
 	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
@@ -906,7 +948,7 @@ ctl_lock(struct socket *so, int refcount, int lr)
 static int
 ctl_unlock(struct socket *so, int refcount, int lr)
 {
-	int lr_saved;
+	uint32_t lr_saved;
 	lck_mtx_t * mutex_held;
 	
 	if (lr == 0) 
@@ -921,15 +963,15 @@ ctl_unlock(struct socket *so, int refcount, int lr)
 		so->so_usecount--;
 	
 	if (so->so_usecount < 0)
-		panic("ctl_unlock: so=%x usecount=%x\n", so, so->so_usecount);
+		panic("ctl_unlock: so=%p usecount=%x\n", so, so->so_usecount);
 	if (so->so_pcb == NULL) {
-		panic("ctl_unlock: so=%x NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
+		panic("ctl_unlock: so=%p NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 	} else {
 		mutex_held = ((struct ctl_cb *)so->so_pcb)->mtx;
 	}
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
-	so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+	so->unlock_lr[so->next_unlock_lr] = lr_saved;
 	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(mutex_held);
 	
@@ -946,10 +988,10 @@ ctl_getlock(struct socket *so, __unused int locktype)
 	
 	if (so->so_pcb)  {
 		if (so->so_usecount < 0)
-			panic("ctl_getlock: so=%x usecount=%x\n", so, so->so_usecount);
+			panic("ctl_getlock: so=%p usecount=%x\n", so, so->so_usecount);
 		return(kcb->mtx);
 	} else {
-		panic("ctl_getlock: so=%x NULL so_pcb\n", so);
+		panic("ctl_getlock: so=%p NULL so_pcb\n", so);
 		return (so->so_proto->pr_domain->dom_mtx);
 	}
 }
diff --git a/bsd/kern/kern_core.c b/bsd/kern/kern_core.c
index 2b4b3580b..5d149c7db 100644
--- a/bsd/kern/kern_core.c
+++ b/bsd/kern/kern_core.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1991 NeXT Computer, Inc.  All rights reserved.
  *
@@ -59,18 +65,26 @@
 
 typedef struct {
 	int	flavor;			/* the number for this flavor */
-	int	count;			/* count of ints in this flavor */
+	mach_msg_type_number_t	count;	/* count of ints in this flavor */
 } mythread_state_flavor_t;
 
 #if defined (__ppc__)
+/* 64 bit */
+mythread_state_flavor_t thread_flavor_array64[]={
+		{PPC_THREAD_STATE64 , PPC_THREAD_STATE64_COUNT},
+		{PPC_FLOAT_STATE, PPC_FLOAT_STATE_COUNT}, 
+		{PPC_EXCEPTION_STATE64, PPC_EXCEPTION_STATE64_COUNT},
+		{PPC_VECTOR_STATE, PPC_VECTOR_STATE_COUNT}
+		};
 
+/* 32 bit */
 mythread_state_flavor_t thread_flavor_array[]={
 		{PPC_THREAD_STATE , PPC_THREAD_STATE_COUNT},
 		{PPC_FLOAT_STATE, PPC_FLOAT_STATE_COUNT}, 
 		{PPC_EXCEPTION_STATE, PPC_EXCEPTION_STATE_COUNT},
 		{PPC_VECTOR_STATE, PPC_VECTOR_STATE_COUNT}
 		};
-int mynum_flavors=4;
+
 #elif defined (__i386__)
 mythread_state_flavor_t thread_flavor_array [] = { 
 		{x86_THREAD_STATE, x86_THREAD_STATE_COUNT},
@@ -78,6 +92,13 @@ mythread_state_flavor_t thread_flavor_array [] = {
 		{x86_EXCEPTION_STATE, x86_EXCEPTION_STATE_COUNT},
 		};
 int mynum_flavors=3;
+#elif defined (__arm__)
+mythread_state_flavor_t thread_flavor_array[]={
+		{ARM_THREAD_STATE , ARM_THREAD_STATE_COUNT},
+		{ARM_VFP_STATE, ARM_VFP_STATE_COUNT}, 
+		{ARM_EXCEPTION_STATE, ARM_EXCEPTION_STATE_COUNT}
+		};
+int mynum_flavors=3;
 
 #else
 #error architecture not supported
@@ -89,6 +110,7 @@ typedef struct {
 	int  hoffset;
 	mythread_state_flavor_t *flavors;
 	int tstate_size;
+	int flavor_count;
 } tir_t;
 
 /* XXX should be static */
@@ -129,7 +151,7 @@ collectth_state(thread_t th_act, void *tirp)
 		 * the appropriate thread state struct for each
 		 * thread state flavor.
 		 */
-		for (i = 0; i < mynum_flavors; i++) {
+		for (i = 0; i < t->flavor_count; i++) {
 			*(mythread_state_flavor_t *)(header+hoffset) =
 			  flavors[i];
 			hoffset += sizeof(mythread_state_flavor_t);
@@ -142,18 +164,34 @@ collectth_state(thread_t th_act, void *tirp)
 		t->hoffset = hoffset;
 }
 
+
 /*
- * Create a core image on the file "core".
+ * coredump
+ *
+ * Description:	Create a core image on the file "core" for the process
+ *		indicated
+ *
+ * Parameters:	core_proc			Process to dump core [*]
+ *
+ * Returns:	0				Success
+ *		EFAULT				Failed
+ *
+ * IMPORTANT:	This function can only be called on the current process, due
+ *		to assumptions below; see variable declaration section for
+ *		details.
  */
 #define	MAX_TSTATE_FLAVORS	10
 int
-coredump(struct proc *p)
+coredump(proc_t core_proc)
 {
-	int error=0;
-	kauth_cred_t cred = kauth_cred_get();
+/* Begin assumptions that limit us to only the current process */
+	vfs_context_t ctx = vfs_context_current();
+	vm_map_t	map = current_map();
+	task_t		task = current_task();
+/* End assumptions */
+	kauth_cred_t cred = vfs_context_ucred(ctx);
+	int error = 0;
 	struct vnode_attr va;
-	struct vfs_context context;
-	vm_map_t	map;
 	int		thread_count, segment_count;
 	int		command_size, header_size, tstate_size;
 	int		hoffset;
@@ -164,21 +202,21 @@ coredump(struct proc *p)
 	vm_prot_t	prot;
 	vm_prot_t	maxprot;
 	vm_inherit_t	inherit;
-	int		error1;
-	task_t		task;
-	char		core_name[MAXCOMLEN+6];
+	int		error1 = 0;
+	char		stack_name[MAXCOMLEN+6];
+	char		*alloced_name = NULL;
 	char		*name;
 	mythread_state_flavor_t flavors[MAX_TSTATE_FLAVORS];
 	vm_size_t	mapsize;
 	int		i;
-	int nesting_depth = 0;
+	uint32_t nesting_depth = 0;
 	kern_return_t	kret;
 	struct vm_region_submap_info_64 vbr;
-	int vbrcount=0;
+	mach_msg_type_number_t vbrcount = 0;
 	tir_t tir1;
 	struct vnode * vp;
-	struct mach_header	*mh;
-	struct mach_header_64	*mh64;
+	struct mach_header	*mh = NULL;	/* protected by is_64 */
+	struct mach_header_64	*mh64 = NULL;	/* protected by is_64 */
 	int		is_64 = 0;
 	size_t		mach_header_sz = sizeof(struct mach_header);
 	size_t		segment_command_sz = sizeof(struct segment_command);
@@ -191,47 +229,46 @@ coredump(struct proc *p)
 		return (EFAULT);
 	}
 
-	if (IS_64BIT_PROCESS(p)) {
+	if (IS_64BIT_PROCESS(core_proc)) {
 		is_64 = 1;
 		mach_header_sz = sizeof(struct mach_header_64);
 		segment_command_sz = sizeof(struct segment_command_64);
 	}
 
-	task = current_task();
-	map = current_map();
 	mapsize = get_vmmap_size(map);
 
-	if (mapsize >=  p->p_rlimit[RLIMIT_CORE].rlim_cur)
+	if (mapsize >=  core_proc->p_rlimit[RLIMIT_CORE].rlim_cur)
 		return (EFAULT);
 	(void) task_suspend(task);
 
-	/* create name according to sysctl'able format string */
-	name = proc_core_name(p->p_comm, kauth_cred_getuid(cred), p->p_pid);
+	MALLOC(alloced_name, char *, MAXPATHLEN, M_TEMP, M_NOWAIT | M_ZERO);
 
+	/* create name according to sysctl'able format string */
 	/* if name creation fails, fall back to historical behaviour... */
-	if (name == NULL) {
-		sprintf(core_name, "/cores/core.%d", p->p_pid);
-		name = core_name;
-	}
-	context.vc_proc = p;
-	context.vc_ucred = cred;
+	if (proc_core_name(core_proc->p_comm, kauth_cred_getuid(cred),
+			   core_proc->p_pid, alloced_name, MAXPATHLEN)) {
+		snprintf(stack_name, sizeof(stack_name),
+			 "/cores/core.%d", core_proc->p_pid);
+		name = stack_name;
+	} else
+		name = alloced_name;
 
-	if ((error = vnode_open(name, (O_CREAT | FWRITE | O_NOFOLLOW), S_IRUSR, VNODE_LOOKUP_NOFOLLOW, &vp, &context)))
-	        return (error);
+	if ((error = vnode_open(name, (O_CREAT | FWRITE | O_NOFOLLOW), S_IRUSR, VNODE_LOOKUP_NOFOLLOW, &vp, ctx)))
+		goto out2;
 
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_nlink);
 	/* Don't dump to non-regular files or files with links. */
 	if (vp->v_type != VREG ||
-	    vnode_getattr(vp, &va, &context) || va.va_nlink != 1) {
+	    vnode_getattr(vp, &va, ctx) || va.va_nlink != 1) {
 		error = EFAULT;
 		goto out;
 	}
 
 	VATTR_INIT(&va);	/* better to do it here than waste more stack in vnode_setsize */
 	VATTR_SET(&va, va_data_size, 0);
-	vnode_setattr(vp, &va, &context);
-	p->p_acflag |= ACORE;
+	vnode_setattr(vp, &va, ctx);
+	core_proc->p_acflag |= ACORE;
 
 	/*
 	 *	If the task is modified while dumping the file
@@ -241,12 +278,21 @@ coredump(struct proc *p)
 
 	thread_count = get_task_numacts(task);
 	segment_count = get_vmmap_entries(map);	/* XXX */
-	bcopy(thread_flavor_array,flavors,sizeof(thread_flavor_array));
+#if defined (__ppc__)
+	if (is_64) {
+		tir1.flavor_count = sizeof(thread_flavor_array64)/sizeof(mythread_state_flavor_t);
+		bcopy(thread_flavor_array64, flavors,sizeof(thread_flavor_array64));
+	} else {
+#endif	/* __ppc __ */
+		tir1.flavor_count = sizeof(thread_flavor_array)/sizeof(mythread_state_flavor_t);
+		bcopy(thread_flavor_array, flavors,sizeof(thread_flavor_array));
+#if defined (__ppc__)
+	}
+#endif	/* __ppc __ */
 	tstate_size = 0;
-	for (i = 0; i < mynum_flavors; i++)
+	for (i = 0; i < tir1.flavor_count; i++)
 		tstate_size += sizeof(mythread_state_flavor_t) +
 		  (flavors[i].count * sizeof(int));
-
 	command_size = segment_count * segment_command_sz +
 	  thread_count*sizeof(struct thread_command) +
 	  tstate_size*thread_count;
@@ -384,8 +430,8 @@ coredump(struct proc *p)
 					xfer_vmsize = INT_MAX;
 				error = vn_rdwr_64(UIO_WRITE, vp,
 						vmoffset, xfer_vmsize, xfer_foffset,
-					(IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32), 
-					IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
+					(IS_64BIT_PROCESS(core_proc) ? UIO_USERSPACE64 : UIO_USERSPACE32), 
+					IO_NODELOCKED|IO_UNIT, cred, (int *) 0, core_proc);
 				tmp_vmsize -= xfer_vmsize;
 				xfer_foffset += xfer_vmsize;
 			}
@@ -406,8 +452,10 @@ coredump(struct proc *p)
 	 */
 	if (is_64) {
 		mh64->ncmds -= segment_count;
+		mh64->sizeofcmds -= segment_count * segment_command_sz;
 	} else {
 		mh->ncmds -= segment_count;
+		mh->sizeofcmds -= segment_count * segment_command_sz;
 	}
 
 	tir1.header = header;
@@ -421,10 +469,13 @@ coredump(struct proc *p)
 	 *	file.  OK to use a 32 bit write for this.
 	 */
 	error = vn_rdwr(UIO_WRITE, vp, (caddr_t)header, header_size, (off_t)0,
-			UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
+			UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, core_proc);
 	kmem_free(kernel_map, header, header_size);
 out:
-	error1 = vnode_close(vp, FWRITE, &context);
+	error1 = vnode_close(vp, FWRITE, ctx);
+out2:
+	if (alloced_name != NULL)
+		FREE(alloced_name, M_TEMP);
 	if (error == 0)
 		error = error1;
 
diff --git a/bsd/kern/kern_credential.c b/bsd/kern/kern_credential.c
index 59c1a24ee..89d2e8dbf 100644
--- a/bsd/kern/kern_credential.c
+++ b/bsd/kern/kern_credential.c
@@ -1,27 +1,40 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 /*
- * Kernel Authorization framework: Management of process/thread credentials and identity information.
+ * Kernel Authorization framework: Management of process/thread credentials
+ * and identity information.
  */
 
 
@@ -54,16 +67,68 @@
 #define MACH_ASSERT 1	/* XXX so bogus */
 #include <kern/assert.h>
 
-#define CRED_DIAGNOSTIC 1
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_framework.h>
+#endif
+
+#define CRED_DIAGNOSTIC 0
+
+# define NULLCRED_CHECK(_c)	do {if (!IS_VALID_CRED(_c)) panic("%s: bad credential %p", __FUNCTION__,_c);} while(0)
+
+/*
+ * Credential debugging; we can track entry into a function that might
+ * change a credential, and we can track actual credential changes that
+ * result.
+ *
+ * Note:	Does *NOT* currently include per-thread credential changes
+ */
+
+#if DEBUG_CRED
+#define	DEBUG_CRED_ENTER		printf
+#define	DEBUG_CRED_CHANGE		printf
+extern void kauth_cred_print(kauth_cred_t cred);
+
+#include <libkern/OSDebug.h>	/* needed for get_backtrace( ) */
+
+int is_target_cred( kauth_cred_t the_cred );
+void get_backtrace( void );
+
+static int sysctl_dump_creds( __unused struct sysctl_oid *oidp, __unused void *arg1, 
+							  __unused int arg2, struct sysctl_req *req );
+static int
+sysctl_dump_cred_backtraces( __unused struct sysctl_oid *oidp, __unused void *arg1, 
+							 __unused int arg2, struct sysctl_req *req );
+
+#define MAX_STACK_DEPTH 8
+struct cred_backtrace {
+	int				depth;
+	void *			stack[ MAX_STACK_DEPTH ];
+};
+typedef struct cred_backtrace cred_backtrace;
+
+#define MAX_CRED_BUFFER_SLOTS 200
+struct cred_debug_buffer {
+	int				next_slot;
+	cred_backtrace	stack_buffer[ MAX_CRED_BUFFER_SLOTS ];	
+};
+typedef struct cred_debug_buffer cred_debug_buffer;
+cred_debug_buffer * cred_debug_buf_p = NULL;
+
+#else	/* !DEBUG_CRED */
 
-# define NULLCRED_CHECK(_c)	do {if (!IS_VALID_CRED(_c)) panic("bad credential %p", _c);} while(0)
+#define	DEBUG_CRED_ENTER(fmt, ...)	do {} while (0)
+#define	DEBUG_CRED_CHANGE(fmt, ...)	do {} while (0)
+
+#endif	/* !DEBUG_CRED */
 
 /*
  * Interface to external identity resolver.
  *
- * The architecture of the interface is simple; the external resolver calls in to
- * get work, then calls back with completed work.  It also calls us to let us know
- * that it's (re)started, so that we can resubmit work if it times out.
+ * The architecture of the interface is simple; the external resolver calls
+ * in to get work, then calls back with completed work.  It also calls us
+ * to let us know that it's (re)started, so that we can resubmit work if it
+ * times out.
  */
 
 static lck_mtx_t *kauth_resolver_mtx;
@@ -93,9 +158,9 @@ TAILQ_HEAD(kauth_resolver_done_head, kauth_resolver_work)	kauth_resolver_done;
 static int	kauth_resolver_submit(struct kauth_identity_extlookup *lkp);
 static int	kauth_resolver_complete(user_addr_t message);
 static int	kauth_resolver_getwork(user_addr_t message);
+static int	kauth_resolver_getwork2(user_addr_t message);
 
-#define KAUTH_CRED_PRIMES_COUNT 7
-static const int kauth_cred_primes[KAUTH_CRED_PRIMES_COUNT] = {97, 241, 397, 743, 1499, 3989, 7499};
+static const int kauth_cred_primes[KAUTH_CRED_PRIMES_COUNT] = KAUTH_CRED_PRIMES;
 static int	kauth_cred_primes_index = 0;
 static int	kauth_cred_table_size = 0;
 
@@ -117,6 +182,34 @@ static void kauth_cred_hash_print(void);
 static void kauth_cred_print(kauth_cred_t cred);
 #endif
 
+
+/*
+ * kauth_resolver_init
+ *
+ * Description:	Initialize the daemon side of the credential identity resolver
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Notes:	Intialize the credential identity resolver for use; the
+ *		credential identity resolver is the KPI used by the user
+ *		space credential identity resolver daemon to communicate
+ *		with the kernel via the identitysvc() system call..
+ *
+ *		This is how membership in more than 16 groups (1 effective
+ *		and 15 supplementary) is supported, and also how UID's,
+ *		UUID's, and so on, are translated to/from POSIX credential
+ *		values.
+ *
+ *		The credential identity resolver operates by attempting to
+ *		determine identity first from the credential, then from
+ *		the kernel credential identity cache, and finally by
+ *		enqueueing a request to a user space daemon.
+ *
+ *		This function is called from kauth_init() in the file
+ *		kern_authorization.c.
+ */
 void
 kauth_resolver_init(void)
 {
@@ -127,10 +220,27 @@ kauth_resolver_init(void)
 	kauth_resolver_mtx = lck_mtx_alloc_init(kauth_lck_grp, 0/*LCK_ATTR_NULL*/);
 }
 
+
 /*
- * Allocate a work queue entry, submit the work and wait for completion.
+ * kauth_resolver_submit
  *
- * XXX do we want an 'interruptible' flag vs. always being interruptible?
+ * Description:	Submit an external credential identity resolution request to
+ *		the user space daemon.
+ *
+ * Parameters:	lkp				A pointer to an external
+ *						lookup request
+ *
+ * Returns:	0				Success
+ *		EWOULDBLOCK			No resolver registered
+ *		EINTR				Operation interrupted (e.g. by
+ *						a signal)
+ *		ENOMEM				Could not allocate work item
+ *		???				An error from the user space
+ *						daemon
+ *
+ * Notes:	Allocate a work queue entry, submit the work and wait for
+ *		the operation to either complete or time out.  Outstanding
+ *		operations may also be cancelled.
  */
 static int
 kauth_resolver_submit(struct kauth_identity_extlookup *lkp)
@@ -169,14 +279,17 @@ kauth_resolver_submit(struct kauth_identity_extlookup *lkp)
 	workp->kr_result = 0;
 
 	/*
-	 * We insert the request onto the unsubmitted queue, the call in from the
-	 * resolver will it to the submitted thread when appropriate.
+	 * We insert the request onto the unsubmitted queue, the call in from
+	 * the resolver will it to the submitted thread when appropriate.
 	 */
 	KAUTH_RESOLVER_LOCK();
 	workp->kr_seqno = workp->kr_work.el_seqno = kauth_resolver_sequence++;
 	workp->kr_work.el_result = KAUTH_EXTLOOKUP_INPROG;
 
-	/* XXX as an optimisation, we could check the queue for identical items and coalesce */
+	/*
+	 * XXX As an optimisation, we could check the queue for identical
+	 * XXX items and coalesce them
+	 */
 	TAILQ_INSERT_TAIL(&kauth_resolver_unsubmitted, workp, kr_link);
 
 	wakeup_one((caddr_t)&kauth_resolver_unsubmitted);
@@ -202,9 +315,10 @@ kauth_resolver_submit(struct kauth_identity_extlookup *lkp)
 		*lkp = workp->kr_work;
 	
 	/*
-	 * If the request timed out and was never collected, the resolver is dead and
-	 * probably not coming back anytime soon.  In this case we revert to no-resolver
-	 * behaviour, and punt all the other sleeping requests to clear the backlog.
+	 * If the request timed out and was never collected, the resolver
+	 * is dead and probably not coming back anytime soon.  In this
+	 * case we revert to no-resolver behaviour, and punt all the other
+	 * sleeping requests to clear the backlog.
 	 */
 	if ((error == EWOULDBLOCK) && (workp->kr_flags & KAUTH_REQUEST_UNSUBMITTED)) {
 		KAUTH_DEBUG("RESOLVER - request timed out without being collected for processing, resolver dead");
@@ -216,7 +330,10 @@ kauth_resolver_submit(struct kauth_identity_extlookup *lkp)
 		    wakeup(killp);
 	}
 	
-	/* drop our reference on the work item, and note whether we should free it or not */
+	/*
+	 * drop our reference on the work item, and note whether we should
+	 * free it or not
+	 */
 	if (--workp->kr_refs <= 0) {
 		/* work out which list we have to remove it from */
 		if (workp->kr_flags & KAUTH_REQUEST_DONE) {
@@ -247,8 +364,29 @@ kauth_resolver_submit(struct kauth_identity_extlookup *lkp)
 	return(error);
 }
 
+
 /*
- * System call interface for the external identity resolver.
+ * identitysvc
+ *
+ * Description:	System call interface for the external identity resolver.
+ *
+ * Parameters:	uap->message			Message from daemon to kernel
+ *
+ * Returns:	0				Successfully became resolver
+ *		EPERM				Not the resolver process
+ *	kauth_authorize_generic:EPERM		Not root user
+ *	kauth_resolver_complete:EIO
+ *	kauth_resolver_complete:EFAULT
+ *	kauth_resolver_getwork:EINTR
+ *	kauth_resolver_getwork:EFAULT
+ *
+ * Notes:	This system call blocks until there is work enqueued, at
+ *		which time the kernel wakes it up, and a message from the
+ *		kernel is copied out to the identity resolution daemon, which
+ *		proceed to attempt to resolve it.  When the resolution has
+ *		completed (successfully or not), the daemon called back into
+ *		this system call to give the result to the kernel, and wait
+ *		for the next request.
  */
 int
 identitysvc(__unused struct proc *p, struct identitysvc_args *uap, __unused register_t *retval)
@@ -315,40 +453,152 @@ identitysvc(__unused struct proc *p, struct identitysvc_args *uap, __unused regi
 	return(0);
 }
 
+
 /*
- * Get work for a caller.
+ * kauth_resolver_getwork_continue
+ *		
+ * Description:	Continuation for kauth_resolver_getwork
+ *
+ * Parameters:	result				Error code or 0 for the sleep
+ *						that got us to this function
+ *
+ * Returns:	0				Success
+ *		EINTR				Interrupted (e.g. by signal)
+ *	kauth_resolver_getwork2:EFAULT
+ *
+ * Notes:	See kauth_resolver_getwork(0 and kauth_resolver_getwork2() for
+ *		more information.
  */
 static int
-kauth_resolver_getwork(user_addr_t message)
+kauth_resolver_getwork_continue(int result)
+{
+	thread_t thread;
+	struct uthread *ut;
+	user_addr_t message;
+
+	if (result) {
+		KAUTH_RESOLVER_UNLOCK();
+		return(result);
+	}
+
+	/*
+	 * If we lost a race with another thread/memberd restarting, then we
+	 * need to go back to sleep to look for more work.  If it was memberd
+	 * restarting, then the msleep0() will error out here, as our thread
+	 * will already be "dead".
+	 */
+	if (TAILQ_FIRST(&kauth_resolver_unsubmitted) == NULL) {
+		int error;
+
+		error = msleep0(&kauth_resolver_unsubmitted, kauth_resolver_mtx, PCATCH, "GRGetWork", 0, kauth_resolver_getwork_continue);
+		KAUTH_RESOLVER_UNLOCK();
+		return(error);
+	}
+
+	thread = current_thread();
+	ut = get_bsdthread_info(thread);
+	message = ut->uu_kauth.message;
+	return(kauth_resolver_getwork2(message));
+}
+
+
+/*
+ * kauth_resolver_getwork2
+ *
+ * Decription:	Common utility function to copy out a identity resolver work
+ *		item from the kernel to user space as part of the user space
+ *		identity resolver requesting work.
+ *
+ * Parameters:	message				message to user space
+ *
+ * Returns:	0				Success
+ *		EFAULT				Bad user space message address
+ *
+ * Notes:	This common function exists to permit the use of continuations
+ *		in the identity resoultion process.  This frees up the stack
+ *		while we are waiting for the user space resolver to complete
+ *		a request.  This is specifically used so that our per thread
+ *		cost can be small, and we will therefore be willing to run a
+ *		larger number of threads in the user space identity resolver.
+ */
+static int
+kauth_resolver_getwork2(user_addr_t message)
 {
 	struct kauth_resolver_work *workp;
 	int		error;
 
-	KAUTH_RESOLVER_LOCK();
-	error = 0;
-	while ((workp = TAILQ_FIRST(&kauth_resolver_unsubmitted)) == NULL) {
-		error = msleep(&kauth_resolver_unsubmitted, kauth_resolver_mtx, PCATCH, "GRGetWork", 0);
-		if (error != 0)
-			break;
-	}
-	if (workp != NULL) {
-		if ((error = copyout(&workp->kr_work, message, sizeof(workp->kr_work))) != 0) {
-			KAUTH_DEBUG("RESOLVER - error submitting work to resolve");
-			goto out;
-		}
-		TAILQ_REMOVE(&kauth_resolver_unsubmitted, workp, kr_link);
-		workp->kr_flags &= ~KAUTH_REQUEST_UNSUBMITTED;
-		workp->kr_flags |= KAUTH_REQUEST_SUBMITTED;
-		TAILQ_INSERT_TAIL(&kauth_resolver_submitted, workp, kr_link);
+	/*
+	 * Note: We depend on the caller protecting us from a NULL work item
+	 * queue, since we must have the kauth resolver lock on entry to this
+	 * function.
+	 */
+	workp = TAILQ_FIRST(&kauth_resolver_unsubmitted);
+
+	if ((error = copyout(&workp->kr_work, message, sizeof(workp->kr_work))) != 0) {
+		KAUTH_DEBUG("RESOLVER - error submitting work to resolve");
+		goto out;
 	}
+	TAILQ_REMOVE(&kauth_resolver_unsubmitted, workp, kr_link);
+	workp->kr_flags &= ~KAUTH_REQUEST_UNSUBMITTED;
+	workp->kr_flags |= KAUTH_REQUEST_SUBMITTED;
+	TAILQ_INSERT_TAIL(&kauth_resolver_submitted, workp, kr_link);
 
 out:
 	KAUTH_RESOLVER_UNLOCK();
 	return(error);
 }
 
+
+/*
+ * kauth_resolver_getwork
+ *
+ * Description:	Get a work item from the enqueued requests from the kernel and
+ *		give it to the user space daemon.
+ *
+ * Parameters:	message				message to user space
+ *
+ * Returns:	0				Success
+ *		EINTR				Interrupted (e.g. by signal)
+ *	kauth_resolver_getwork2:EFAULT
+ *
+ * Notes:	This function blocks in a continuation if there are no work
+ *		items available for processing at the time the user space
+ *		identity resolution daemon makes a request for work.  This
+ *		permits a large number of threads to be used by the daemon,
+ *		without using a lot of wired kernel memory when there are no
+ *		acctual request outstanding.
+ */
+static int
+kauth_resolver_getwork(user_addr_t message)
+{
+	struct kauth_resolver_work *workp;
+	int		error;
+
+	KAUTH_RESOLVER_LOCK();
+	error = 0;
+	while ((workp = TAILQ_FIRST(&kauth_resolver_unsubmitted)) == NULL) {
+		thread_t thread = current_thread();
+		struct uthread *ut = get_bsdthread_info(thread);
+
+		ut->uu_kauth.message = message;
+		error = msleep0(&kauth_resolver_unsubmitted, kauth_resolver_mtx, PCATCH, "GRGetWork", 0, kauth_resolver_getwork_continue);
+		KAUTH_RESOLVER_UNLOCK();
+		return(error);
+	}
+	return kauth_resolver_getwork2(message);
+}
+
+
 /*
- * Return a result from userspace.
+ * kauth_resolver_complete
+ *
+ * Description:	Return a result from userspace.
+ *
+ * Parameters:	message				message from user space
+ *
+ * Returns:	0				Success
+ *		EIO				The resolver is dead
+ *	copyin:EFAULT				Bad message from user space
  */
 static int
 kauth_resolver_complete(user_addr_t message)
@@ -471,7 +721,7 @@ static lck_mtx_t *kauth_identity_mtx;
 
 static struct kauth_identity *kauth_identity_alloc(uid_t uid, gid_t gid, guid_t *guidp, time_t guid_expiry,
     ntsid_t *ntsidp, time_t ntsid_expiry);
-static void	kauth_identity_register(struct kauth_identity *kip);
+static void	kauth_identity_register_and_free(struct kauth_identity *kip);
 static void	kauth_identity_updatecache(struct kauth_identity_extlookup *elp, struct kauth_identity *kip);
 static void	kauth_identity_lru(struct kauth_identity *kip);
 static int	kauth_identity_guid_expired(struct kauth_identity *kip);
@@ -481,6 +731,23 @@ static int	kauth_identity_find_gid(gid_t gid, struct kauth_identity *kir);
 static int	kauth_identity_find_guid(guid_t *guidp, struct kauth_identity *kir);
 static int	kauth_identity_find_ntsid(ntsid_t *ntsid, struct kauth_identity *kir);
 
+
+/*
+ * kauth_identity_init
+ *
+ * Description:	Initialize the kernel side of the credential identity resolver
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Notes:	Intialize the credential identity resolver for use; the
+ *		credential identity resolver is the KPI used to communicate
+ *		with a user space credential identity resolver daemon.
+ *
+ *		This function is called from kauth_init() in the file
+ *		kern_authorization.c.
+ */
 void
 kauth_identity_init(void)
 {
@@ -488,12 +755,24 @@ kauth_identity_init(void)
 	kauth_identity_mtx = lck_mtx_alloc_init(kauth_lck_grp, 0/*LCK_ATTR_NULL*/);
 }
 
-static int
-kauth_identity_resolve(__unused struct kauth_identity_extlookup *el)
-{
-	return(kauth_resolver_submit(el));
-}
 
+/*
+ * kauth_identity_alloc
+ *
+ * Description:	Allocate and fill out a kauth_identity structure for
+ *		translation between {UID|GID}/GUID/NTSID
+ *
+ * Parameters:	uid
+ *
+ * Returns:	NULL				Insufficient memory to satisfy
+ *						the request
+ *		!NULL				A pointer to the applocated
+ *						structure, filled in
+ *
+ * Notes:	It is illegal to translate between UID and GID; any given UUID
+ *		or NTSID can oly refer to an NTSIDE or UUID (respectively),
+ *		and *either* a UID *or* a GID, but not both.
+ */
 static struct kauth_identity *
 kauth_identity_alloc(uid_t uid, gid_t gid, guid_t *guidp, time_t guid_expiry, ntsid_t *ntsidp, time_t ntsid_expiry)
 {
@@ -526,17 +805,29 @@ kauth_identity_alloc(uid_t uid, gid_t gid, guid_t *guidp, time_t guid_expiry, nt
 	return(kip);
 }
 
+
 /*
- * Register an association between identity tokens.
+ * kauth_identity_register_and_free
+ *
+ * Description:	Register an association between identity tokens.  The passed
+ *		'kip' is freed by this function.
+ *
+ * Parameters:	kip				Pointer to kauth_identity
+ *						structure to register
+ *
+ * Returns:	(void)
+ *
+ * Notes:	The memory pointer to by 'kip' is assumed to have been
+ *		previously allocated via kauth_identity_alloc().
  */
 static void
-kauth_identity_register(struct kauth_identity *kip)
+kauth_identity_register_and_free(struct kauth_identity *kip)
 {
 	struct kauth_identity *ip;
 
 	/*
-	 * We search the cache for the UID listed in the incoming association.  If we
-	 * already have an entry, the new information is merged.
+	 * We search the cache for the UID listed in the incoming association.
+	 * If we already have an entry, the new information is merged.
 	 */
 	ip = NULL;
 	KAUTH_IDENTITY_LOCK();
@@ -584,9 +875,22 @@ kauth_identity_register(struct kauth_identity *kip)
 		FREE(ip, M_KAUTH);
 }
 
+
 /*
- * Given a lookup result, add any associations that we don't
- * currently have.
+ * kauth_identity_updatecache
+ *
+ * Description:	Given a lookup result, add any associations that we don't
+ *		currently have.
+ *
+ * Parameters:	elp				External lookup result from
+ *						user space daemon to kernel
+ *		rkip				pointer to returned kauth
+ *						identity, or NULL
+ *
+ * Returns:	(void)
+ *
+ * Implicit returns:
+ *		*rkip				Modified (if non-NULL)
  */
 static void
 kauth_identity_updatecache(struct kauth_identity_extlookup *elp, struct kauth_identity *rkip)
@@ -631,7 +935,7 @@ kauth_identity_updatecache(struct kauth_identity_extlookup *elp, struct kauth_id
 				if (rkip != NULL)
 					*rkip = *kip;
 				KAUTH_DEBUG("CACHE - learned %d is " K_UUID_FMT, kip->ki_uid, K_UUID_ARG(kip->ki_guid));
-				kauth_identity_register(kip);
+				kauth_identity_register_and_free(kip);
 			}
 		}
 	}
@@ -671,20 +975,30 @@ kauth_identity_updatecache(struct kauth_identity_extlookup *elp, struct kauth_id
 				if (rkip != NULL)
 					*rkip = *kip;
 				KAUTH_DEBUG("CACHE - learned %d is " K_UUID_FMT, kip->ki_uid, K_UUID_ARG(kip->ki_guid));
-				kauth_identity_register(kip);
+				kauth_identity_register_and_free(kip);
 			}
 		}
 	}
 
 }
 
+
 /*
- * Promote the entry to the head of the LRU, assumes the cache is locked.
+ * kauth_identity_lru
+ *
+ * Description:	Promote the entry to the head of the LRU, assumes the cache
+ *		is locked.
  *
- * This is called even if the entry has expired; typically an expired entry
- * that's been looked up is about to be revalidated, and having it closer to
- * the head of the LRU means finding it quickly again when the revalidation
- * comes through.
+ * Parameters:	kip				kauth identity to move to the
+ *						head of the LRU list, if it's
+ *						not already there
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This is called even if the entry has expired; typically an
+ *		expired entry that's been looked up is about to be revalidated,
+ *		and having it closer to the head of the LRU means finding it
+ *		quickly again when the revalidation comes through.
  */
 static void
 kauth_identity_lru(struct kauth_identity *kip)
@@ -695,8 +1009,17 @@ kauth_identity_lru(struct kauth_identity *kip)
 	}
 }
 
+
 /*
- * Handly lazy expiration of translations.
+ * kauth_identity_guid_expired
+ *
+ * Description:	Handle lazy expiration of GUID translations.
+ *
+ * Parameters:	kip				kauth identity to check for
+ *						GUID expiration
+ *
+ * Returns:	1				Expired
+ *		0				Not expired
  */
 static int
 kauth_identity_guid_expired(struct kauth_identity *kip)
@@ -708,6 +1031,18 @@ kauth_identity_guid_expired(struct kauth_identity *kip)
 	return((kip->ki_guid_expiry <= tv.tv_sec) ? 1 : 0);
 }
 
+
+/*
+ * kauth_identity_ntsid_expired
+ *
+ * Description:	Handle lazy expiration of NTSID translations.
+ *
+ * Parameters:	kip				kauth identity to check for
+ *						NTSID expiration
+ *
+ * Returns:	1				Expired
+ *		0				Not expired
+ */
 static int
 kauth_identity_ntsid_expired(struct kauth_identity *kip)
 {
@@ -718,9 +1053,20 @@ kauth_identity_ntsid_expired(struct kauth_identity *kip)
 	return((kip->ki_ntsid_expiry <= tv.tv_sec) ? 1 : 0);
 }
 
+
 /*
- * Search for an entry by UID.  Returns a copy of the entry, ENOENT if no valid
- * association exists for the UID.
+ * kauth_identity_find_uid
+ *
+ * Description: Search for an entry by UID
+ *
+ * Parameters:	uid				UID to find
+ *		kir				Pointer to return area
+ *
+ * Returns:	0				Found
+ *		ENOENT				Not found
+ *
+ * Implicit returns:
+ *		*klr				Modified, if found
  */
 static int
 kauth_identity_find_uid(uid_t uid, struct kauth_identity *kir)
@@ -731,6 +1077,7 @@ kauth_identity_find_uid(uid_t uid, struct kauth_identity *kir)
 	TAILQ_FOREACH(kip, &kauth_identities, ki_link) {
 		if ((kip->ki_valid & KI_VALID_UID) && (uid == kip->ki_uid)) {
 			kauth_identity_lru(kip);
+			/* Copy via structure assignment */
 			*kir = *kip;
 			break;
 		}
@@ -741,8 +1088,18 @@ kauth_identity_find_uid(uid_t uid, struct kauth_identity *kir)
 
 
 /*
- * Search for an entry by GID. Returns a copy of the entry, ENOENT if no valid
- * association exists for the GID.
+ * kauth_identity_find_uid
+ *
+ * Description: Search for an entry by GID
+ *
+ * Parameters:	gid				GID to find
+ *		kir				Pointer to return area
+ *
+ * Returns:	0				Found
+ *		ENOENT				Not found
+ *
+ * Implicit returns:
+ *		*klr				Modified, if found
  */
 static int
 kauth_identity_find_gid(uid_t gid, struct kauth_identity *kir)
@@ -753,6 +1110,7 @@ kauth_identity_find_gid(uid_t gid, struct kauth_identity *kir)
 	TAILQ_FOREACH(kip, &kauth_identities, ki_link) {
 		if ((kip->ki_valid & KI_VALID_GID) && (gid == kip->ki_gid)) {
 			kauth_identity_lru(kip);
+			/* Copy via structure assignment */
 			*kir = *kip;
 			break;
 		}
@@ -763,9 +1121,21 @@ kauth_identity_find_gid(uid_t gid, struct kauth_identity *kir)
 
 
 /*
- * Search for an entry by GUID. Returns a copy of the entry, ENOENT if no valid
- * association exists for the GUID.  Note that the association may be expired,
- * in which case the caller may elect to call out to userland to revalidate.
+ * kauth_identity_find_guid
+ *
+ * Description: Search for an entry by GUID
+ *
+ * Parameters:	guidp				Pointer to GUID to find
+ *		kir				Pointer to return area
+ *
+ * Returns:	0				Found
+ *		ENOENT				Not found
+ *
+ * Implicit returns:
+ *		*klr				Modified, if found
+ *
+ * Note:	The association may be expired, in which case the caller
+ *		may elect to call out to userland to revalidate.
  */
 static int
 kauth_identity_find_guid(guid_t *guidp, struct kauth_identity *kir)
@@ -776,6 +1146,7 @@ kauth_identity_find_guid(guid_t *guidp, struct kauth_identity *kir)
 	TAILQ_FOREACH(kip, &kauth_identities, ki_link) {
 		if ((kip->ki_valid & KI_VALID_GUID) && (kauth_guid_equal(guidp, &kip->ki_guid))) {
 			kauth_identity_lru(kip);
+			/* Copy via structure assignment */
 			*kir = *kip;
 			break;
 		}
@@ -784,10 +1155,23 @@ kauth_identity_find_guid(guid_t *guidp, struct kauth_identity *kir)
 	return((kip == NULL) ? ENOENT : 0);
 }
 
+
 /*
- * Search for an entry by NT Security ID. Returns a copy of the entry, ENOENT if no valid
- * association exists for the SID.  Note that the association may be expired,
- * in which case the caller may elect to call out to userland to revalidate.
+ * kauth_identity_find_ntsid
+ *
+ * Description: Search for an entry by NTSID
+ *
+ * Parameters:	ntsid				Pointer to NTSID to find
+ *		kir				Pointer to return area
+ *
+ * Returns:	0				Found
+ *		ENOENT				Not found
+ *
+ * Implicit returns:
+ *		*klr				Modified, if found
+ *
+ * Note:	The association may be expired, in which case the caller
+ *		may elect to call out to userland to revalidate.
  */
 static int
 kauth_identity_find_ntsid(ntsid_t *ntsid, struct kauth_identity *kir)
@@ -798,6 +1182,7 @@ kauth_identity_find_ntsid(ntsid_t *ntsid, struct kauth_identity *kir)
 	TAILQ_FOREACH(kip, &kauth_identities, ki_link) {
 		if ((kip->ki_valid & KI_VALID_NTSID) && (kauth_ntsid_equal(ntsid, &kip->ki_ntsid))) {
 			kauth_identity_lru(kip);
+			/* Copy via structure assignment */
 			*kir = *kip;
 			break;
 		}
@@ -806,19 +1191,43 @@ kauth_identity_find_ntsid(ntsid_t *ntsid, struct kauth_identity *kir)
 	return((kip == NULL) ? ENOENT : 0);
 }
 
+
 /*
  * GUID handling.
  */
 guid_t kauth_null_guid;
 
+
+/*
+ * kauth_guid_equal
+ *
+ * Description:	Determine the equality of two GUIDs
+ *
+ * Parameters:	guid1				Pointer to first GUID
+ *		guid2				Pointer to second GUID
+ *
+ * Returns:	0				If GUIDs are inequal
+ *		!0				If GUIDs are equal
+ */
 int
 kauth_guid_equal(guid_t *guid1, guid_t *guid2)
 {
-	return(!bcmp(guid1, guid2, sizeof(*guid1)));
+	return(bcmp(guid1, guid2, sizeof(*guid1)) == 0);
 }
 
+
 /*
- * Look for well-known GUIDs.
+ * kauth_wellknown_guid
+ *
+ * Description:	Determine if a GUID is a well-known GUID
+ *
+ * Parameters:	guid				Pointer to GUID to check
+ *
+ * Returns:	KAUTH_WKG_NOT			Not a wel known GUID
+ *		KAUTH_WKG_EVERYBODY		"Everybody"
+ *		KAUTH_WKG_NOBODY		"Nobody"
+ *		KAUTH_WKG_OWNER			"Other"
+ *		KAUTH_WKG_GROUP			"Group"
  */
 int
 kauth_wellknown_guid(guid_t *guid)
@@ -828,11 +1237,11 @@ kauth_wellknown_guid(guid_t *guid)
 	/*
 	 * All WKGs begin with the same 12 bytes.
 	 */
-	if (!bcmp((void *)guid, fingerprint, 12)) {
+	if (bcmp((void *)guid, fingerprint, 12) == 0) {
 		/*
-		 * The final 4 bytes are our code.
+		 * The final 4 bytes are our code (in network byte order).
 		 */
-		code = *(u_int32_t *)&guid->g_guid[12];
+		code = OSSwapHostToBigInt32(*(u_int32_t *)&guid->g_guid[12]);
 		switch(code) {
 		case 0x0000000c:
 			return(KAUTH_WKG_EVERYBODY);
@@ -849,7 +1258,15 @@ kauth_wellknown_guid(guid_t *guid)
 
 
 /*
- * NT Security Identifier handling.
+ * kauth_ntsid_equal
+ *
+ * Description:	Determine the equality of two NTSIDs (NT Security Identifiers) 
+ *
+ * Paramters:	sid1				Pointer to first NTSID
+ *		sid2				Pointer to second NTSID
+ *
+ * Returns:	0				If GUIDs are inequal
+ *		!0				If GUIDs are equal
  */
 int
 kauth_ntsid_equal(ntsid_t *sid1, ntsid_t *sid2)
@@ -857,11 +1274,12 @@ kauth_ntsid_equal(ntsid_t *sid1, ntsid_t *sid2)
 	/* check sizes for equality, also sanity-check size while we're at it */
 	if ((KAUTH_NTSID_SIZE(sid1) == KAUTH_NTSID_SIZE(sid2)) &&
 	    (KAUTH_NTSID_SIZE(sid1) <= sizeof(*sid1)) &&
-	    !bcmp(sid1, sid2, KAUTH_NTSID_SIZE(sid1)))
+	    bcmp(sid1, sid2, KAUTH_NTSID_SIZE(sid1)) == 0)
 		return(1);
 	return(0);
 }
 
+
 /*
  * Identity KPI
  *
@@ -877,8 +1295,96 @@ kauth_ntsid_equal(ntsid_t *sid1, ntsid_t *sid2)
 
 static int	kauth_cred_cache_lookup(int from, int to, void *src, void *dst);
 
+
+/*
+ * kauth_cred_change_egid
+ *
+ * Description:	Set EGID by changing the first element of cr_groups for the
+ *		passed credential; if the new EGID exists in the list of
+ *		groups already, then rotate the old EGID into its position,
+ *		otherwise replace it
+ *
+ * Parameters:	cred			Pointer to the credential to modify
+ *		new_egid		The new EGID to set
+ *
+ * Returns:	0			The egid did not displace a member of
+ *					the supplementary group list
+ *		1			The egid being set displaced a member
+ *					of the supplementary groups list
+ *
+ * Note:	Utility function; internal use only because of locking.
+ *
+ *		This function operates on the credential passed; the caller
+ *		must operate either on a newly allocated credential (one for
+ *		which there is no hash cache reference and no externally
+ *		visible pointer reference), or a template credential.
+ */
+static int
+kauth_cred_change_egid(kauth_cred_t cred, gid_t new_egid)
+{
+	int	i;
+	int	displaced = 1;
+#if radar_4600026
+	int	is_member;
+#endif	/* radar_4600026 */
+	gid_t	old_egid = cred->cr_groups[0];
+
+	/* Ignoring the first entry, scan for a match for the new egid */
+	for (i = 1; i < cred->cr_ngroups; i++) {
+		/*
+		 * If we find a match, swap them so we don't lose overall
+		 * group information
+		 */
+		if (cred->cr_groups[i] == new_egid) {
+			cred->cr_groups[i] = old_egid;
+			DEBUG_CRED_CHANGE("kauth_cred_change_egid: unset displaced\n");
+			displaced = 0;
+			break;
+		}
+	}
+
+#if radar_4600026
+#error Fix radar 4600026 first!!!
+
+/*
+This is correct for memberd behaviour, but incorrect for POSIX; to address
+this, we would need to automatically opt-out any SUID/SGID binary, and force
+it to use initgroups to opt back in.  We take the approach of considering it
+opt'ed out in any group of 16 displacement instead, since it's a much more
+conservative approach (i.e. less likely to cause things to break).
+*/
+
+	/*
+	 * If we displaced a member of the supplementary groups list of the
+	 * credential, and we have not opted out of memberd, then if memberd
+	 * says that the credential is a member of the group, then it has not
+	 * actually been displaced.
+	 *
+	 * NB:	This is typically a cold code path.
+	 */
+	if (displaced && !(cred->cr_flags & CRF_NOMEMBERD) &&
+	    kauth_cred_ismember_gid(cred, new_egid, &is_member) == 0 &&
+	    is_member) {
+	    	displaced = 0;
+		DEBUG_CRED_CHANGE("kauth_cred_change_egid: reset displaced\n");
+	}
+#endif	/* radar_4600026 */
+
+	/* set the new EGID into the old spot */
+	cred->cr_groups[0] = new_egid;
+
+	return (displaced);
+}
+
+
 /*
- * Fetch UID from credential.
+ * kauth_cred_getuid
+ *
+ * Description:	Fetch UID from credential
+ *
+ * Parameters:	cred				Credential to examine
+ *
+ * Returns:	(uid_t)				UID associated with credential
  */
 uid_t
 kauth_cred_getuid(kauth_cred_t cred)
@@ -887,8 +1393,15 @@ kauth_cred_getuid(kauth_cred_t cred)
 	return(cred->cr_uid);
 }
 
+
 /*
- * Fetch GID from credential.
+ * kauth_cred_getgid
+ *
+ * Description:	Fetch GID from credential
+ *
+ * Parameters:	cred				Credential to examine
+ *
+ * Returns:	(gid_t)				GID associated with credential
  */
 uid_t
 kauth_cred_getgid(kauth_cred_t cred)
@@ -897,8 +1410,20 @@ kauth_cred_getgid(kauth_cred_t cred)
 	return(cred->cr_gid);
 }
 
+
 /*
- * Fetch UID from GUID.
+ * kauth_cred_guid2uid
+ *
+ * Description:	Fetch UID from GUID
+ *
+ * Parameters:	guidp				Pointer to GUID to examine
+ *		uidp				Pointer to buffer for UID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*uidp				Modified, if successful
  */
 int
 kauth_cred_guid2uid(guid_t *guidp, uid_t *uidp)
@@ -906,8 +1431,20 @@ kauth_cred_guid2uid(guid_t *guidp, uid_t *uidp)
 	return(kauth_cred_cache_lookup(KI_VALID_GUID, KI_VALID_UID, guidp, uidp));
 }
 
+
 /*
- * Fetch GID from GUID.
+ * kauth_cred_guid2gid
+ *
+ * Description:	Fetch GID from GUID
+ *
+ * Parameters:	guidp				Pointer to GUID to examine
+ *		gidp				Pointer to buffer for GID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*gidp				Modified, if successful
  */
 int
 kauth_cred_guid2gid(guid_t *guidp, gid_t *gidp)
@@ -915,8 +1452,20 @@ kauth_cred_guid2gid(guid_t *guidp, gid_t *gidp)
 	return(kauth_cred_cache_lookup(KI_VALID_GUID, KI_VALID_GID, guidp, gidp));
 }
 
+
 /*
- * Fetch UID from NT SID.
+ * kauth_cred_ntsid2uid
+ *
+ * Description:	Fetch UID from NTSID
+ *
+ * Parameters:	sidp				Pointer to NTSID to examine
+ *		uidp				Pointer to buffer for UID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*uidp				Modified, if successful
  */
 int
 kauth_cred_ntsid2uid(ntsid_t *sidp, uid_t *uidp)
@@ -924,8 +1473,20 @@ kauth_cred_ntsid2uid(ntsid_t *sidp, uid_t *uidp)
 	return(kauth_cred_cache_lookup(KI_VALID_NTSID, KI_VALID_UID, sidp, uidp));
 }
 
+
 /*
- * Fetch GID from NT SID.
+ * kauth_cred_ntsid2gid
+ *
+ * Description:	Fetch GID from NTSID
+ *
+ * Parameters:	sidp				Pointer to NTSID to examine
+ *		gidp				Pointer to buffer for GID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*gidp				Modified, if successful
  */
 int
 kauth_cred_ntsid2gid(ntsid_t *sidp, gid_t *gidp)
@@ -933,8 +1494,20 @@ kauth_cred_ntsid2gid(ntsid_t *sidp, gid_t *gidp)
 	return(kauth_cred_cache_lookup(KI_VALID_NTSID, KI_VALID_GID, sidp, gidp));
 }
 
+
 /*
- * Fetch GUID from NT SID.
+ * kauth_cred_ntsid2guid
+ *
+ * Description:	Fetch GUID from NTSID
+ *
+ * Parameters:	sidp				Pointer to NTSID to examine
+ *		guidp				Pointer to buffer for GUID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*guidp				Modified, if successful
  */
 int
 kauth_cred_ntsid2guid(ntsid_t *sidp, guid_t *guidp)
@@ -942,8 +1515,20 @@ kauth_cred_ntsid2guid(ntsid_t *sidp, guid_t *guidp)
 	return(kauth_cred_cache_lookup(KI_VALID_NTSID, KI_VALID_GUID, sidp, guidp));
 }
 
+
 /*
- * Fetch GUID from UID.
+ * kauth_cred_uid2guid
+ *
+ * Description:	Fetch GUID from UID
+ *
+ * Parameters:	uid				UID to examine
+ *		guidp				Pointer to buffer for GUID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*guidp				Modified, if successful
  */
 int
 kauth_cred_uid2guid(uid_t uid, guid_t *guidp)
@@ -951,8 +1536,20 @@ kauth_cred_uid2guid(uid_t uid, guid_t *guidp)
 	return(kauth_cred_cache_lookup(KI_VALID_UID, KI_VALID_GUID, &uid, guidp));
 }
 
+
 /*
- * Fetch user GUID from credential.
+ * kauth_cred_getguid
+ *
+ * Description:	Fetch GUID from credential
+ *
+ * Parameters:	cred				Credential to examine
+ *		guidp				Pointer to buffer for GUID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*guidp				Modified, if successful
  */
 int
 kauth_cred_getguid(kauth_cred_t cred, guid_t *guidp)
@@ -961,17 +1558,41 @@ kauth_cred_getguid(kauth_cred_t cred, guid_t *guidp)
 	return(kauth_cred_uid2guid(kauth_cred_getuid(cred), guidp));
 }
 
+
 /*
- * Fetch GUID from GID.
- */
-int
+ * kauth_cred_getguid
+ *
+ * Description:	Fetch GUID from GID
+ *
+ * Parameters:	gid				GID to examine
+ *		guidp				Pointer to buffer for GUID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*guidp				Modified, if successful
+ */
+int
 kauth_cred_gid2guid(gid_t gid, guid_t *guidp)
 {
 	return(kauth_cred_cache_lookup(KI_VALID_GID, KI_VALID_GUID, &gid, guidp));
 }
 
+
 /*
- * Fetch NT SID from UID.
+ * kauth_cred_uid2ntsid
+ *
+ * Description:	Fetch NTSID from UID
+ *
+ * Parameters:	uid				UID to examine
+ *		sidp				Pointer to buffer for NTSID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*sidp				Modified, if successful
  */
 int
 kauth_cred_uid2ntsid(uid_t uid, ntsid_t *sidp)
@@ -979,8 +1600,20 @@ kauth_cred_uid2ntsid(uid_t uid, ntsid_t *sidp)
 	return(kauth_cred_cache_lookup(KI_VALID_UID, KI_VALID_NTSID, &uid, sidp));
 }
 
+
 /*
- * Fetch NT SID from credential.
+ * kauth_cred_getntsid
+ *
+ * Description:	Fetch NTSID from credential
+ *
+ * Parameters:	cred				Credential to examine
+ *		sidp				Pointer to buffer for NTSID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*sidp				Modified, if successful
  */
 int
 kauth_cred_getntsid(kauth_cred_t cred, ntsid_t *sidp)
@@ -989,8 +1622,20 @@ kauth_cred_getntsid(kauth_cred_t cred, ntsid_t *sidp)
 	return(kauth_cred_uid2ntsid(kauth_cred_getuid(cred), sidp));
 }
 
+
 /*
- * Fetch NT SID from GID.
+ * kauth_cred_gid2ntsid
+ *
+ * Description:	Fetch NTSID from GID
+ *
+ * Parameters:	gid				GID to examine
+ *		sidp				Pointer to buffer for NTSID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*sidp				Modified, if successful
  */
 int
 kauth_cred_gid2ntsid(gid_t gid, ntsid_t *sidp)
@@ -998,8 +1643,20 @@ kauth_cred_gid2ntsid(gid_t gid, ntsid_t *sidp)
 	return(kauth_cred_cache_lookup(KI_VALID_GID, KI_VALID_NTSID, &gid, sidp));
 }
 
+
 /*
- * Fetch NT SID from GUID.
+ * kauth_cred_guid2ntsid
+ *
+ * Description:	Fetch NTSID from GUID
+ *
+ * Parameters:	guidp				Pointer to GUID to examine
+ *		sidp				Pointer to buffer for NTSID
+ *
+ * Returns:	0				Success
+ *	kauth_cred_cache_lookup:EINVAL
+ *
+ * Implicit returns:
+ *		*sidp				Modified, if successful
  */
 int
 kauth_cred_guid2ntsid(guid_t *guidp, ntsid_t *sidp)
@@ -1008,9 +1665,22 @@ kauth_cred_guid2ntsid(guid_t *guidp, ntsid_t *sidp)
 }
 
 
-
 /*
- * Lookup a translation in the cache.
+ * kauth_cred_cache_lookup
+ *
+ * Description:	Lookup a translation in the cache; if one is not found, and
+ *		the attempt was not fatal, submit the request to the resolver
+ *		instead, and wait for it to complete or be aborted.
+ *
+ * Parameters:	from				Identity information we have
+ *		to				Identity information we want
+ *		src				Pointer to buffer containing
+ *						the source identity
+ *		dst				Pointer to buffer to receive
+ *						the target identity
+ *
+ * Returns:	0				Success
+ *		EINVAL				Unknown source identity type
  */
 static int
 kauth_cred_cache_lookup(int from, int to, void *src, void *dst)
@@ -1047,6 +1717,7 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst)
 	if (error != 0) {
 		/* any other error is fatal */
 		if (error != ENOENT) {
+			/* XXX bogus check - this is not possible */
 			KAUTH_DEBUG("CACHE - cache search error %d", error);
 			return(error);
 		}
@@ -1087,16 +1758,19 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst)
 				goto found;
 			}
 			/*
-			 * We leave ki_valid set here; it contains a translation but the TTL has
-			 * expired.  If we can't get a result from the resolver, we will
-			 * use it as a better-than nothing alternative.
+			 * We leave ki_valid set here; it contains a
+			 * translation but the TTL has expired.  If we can't
+			 * get a result from the resolver, we will use it as
+			 * a better-than nothing alternative.
 			 */
 			KAUTH_DEBUG("CACHE - expired entry found");
 		}
 	}
 
 	/*
-	 * Call the resolver.  We ask for as much data as we can get.
+	 * We failed to find a cache entry; call the resolver.
+	 *
+	 * Note:	We ask for as much data as we can get.
 	 */
 	switch(from) {
 	case KI_VALID_UID:
@@ -1133,13 +1807,13 @@ kauth_cred_cache_lookup(int from, int to, void *src, void *dst)
 	    KAUTH_EXTLOOKUP_WANT_UGUID | KAUTH_EXTLOOKUP_WANT_GGUID |
 	    KAUTH_EXTLOOKUP_WANT_USID | KAUTH_EXTLOOKUP_WANT_GSID;
 	KAUTH_DEBUG("CACHE - calling resolver for %x", el.el_flags);
-	error = kauth_identity_resolve(&el);
+	error = kauth_resolver_submit(&el);
 	KAUTH_DEBUG("CACHE - resolver returned %d", error);
 	/* was the lookup successful? */
 	if (error == 0) {
 		/*
-		 * Save the results from the lookup - may have other information even if we didn't
-		 * get a guid.
+		 * Save the results from the lookup - may have other
+		 * information even if we didn't get a guid.
 		 */
 		kauth_identity_updatecache(&el, &ki);
 	}
@@ -1199,6 +1873,22 @@ static int	kauth_groups_expired(struct kauth_group_membership *gm);
 static void	kauth_groups_lru(struct kauth_group_membership *gm);
 static void	kauth_groups_updatecache(struct kauth_identity_extlookup *el);
 
+
+/*
+ * kauth_groups_init
+ *
+ * Description:	Initialize the groups cache
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Notes:	Intialize the groups cache for use; the group cache is used
+ *		to avoid unnecessary calls out to user space.
+ *
+ *		This function is called from kauth_init() in the file
+ *		kern_authorization.c.
+ */
 void
 kauth_groups_init(void)
 {
@@ -1206,6 +1896,18 @@ kauth_groups_init(void)
 	kauth_groups_mtx = lck_mtx_alloc_init(kauth_lck_grp, 0/*LCK_ATTR_NULL*/);
 }
 
+
+/*
+ * kauth_groups_expired
+ *
+ * Description:	Handle lazy expiration of group membership cache entries
+ *
+ * Parameters:	gm				group membership entry to
+ *						check for expiration
+ *
+ * Returns:	1				Expired
+ *		0				Not expired
+ */
 static int
 kauth_groups_expired(struct kauth_group_membership *gm)
 {
@@ -1215,6 +1917,24 @@ kauth_groups_expired(struct kauth_group_membership *gm)
 	return((gm->gm_expiry <= tv.tv_sec) ? 1 : 0);
 }
 
+
+/*
+ * kauth_groups_lru
+ *
+ * Description:	Promote the entry to the head of the LRU, assumes the cache
+ *		is locked.
+ *
+ * Parameters:	kip				group membership entry to move
+ *						to the head of the LRU list,
+ *						if it's not already there
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This is called even if the entry has expired; typically an
+ *		expired entry that's been looked up is about to be revalidated,
+ *		and having it closer to the head of the LRU means finding it
+ *		quickly again when the revalidation comes through.
+ */
 static void
 kauth_groups_lru(struct kauth_group_membership *gm)
 {
@@ -1224,6 +1944,20 @@ kauth_groups_lru(struct kauth_group_membership *gm)
 	}
 }
 
+
+/*
+ * kauth_groups_updatecache
+ *
+ * Description:	Given a lookup result, add any group cache associations that
+ *		we don't currently have.
+ *
+ * Parameters:	elp				External lookup result from
+ *						user space daemon to kernel
+ *		rkip				pointer to returned kauth
+ *						identity, or NULL
+ *
+ * Returns:	(void)
+ */
 static void
 kauth_groups_updatecache(struct kauth_identity_extlookup *el)
 {
@@ -1238,7 +1972,10 @@ kauth_groups_updatecache(struct kauth_identity_extlookup *el)
 
 	microuptime(&tv);
 
-	/* search for an existing record for this association before inserting */
+	/*
+	 * Search for an existing record for this association before inserting
+	 * a new one; if we find one, update it instead of creating a new one
+	 */
 	KAUTH_GROUPS_LOCK();
 	TAILQ_FOREACH(gm, &kauth_groups, gm_link) {
 		if ((el->el_uid == gm->gm_uid) &&
@@ -1273,10 +2010,10 @@ kauth_groups_updatecache(struct kauth_identity_extlookup *el)
 	}		
 
 	/*
-	 * Insert the new entry.  Note that it's possible to race ourselves here
-	 * and end up with duplicate entries in the list.  Wasteful, but harmless
-	 * since the first into the list will never be looked up, and thus will
-	 * eventually just fall off the end.
+	 * Insert the new entry.  Note that it's possible to race ourselves
+	 * here and end up with duplicate entries in the list.  Wasteful, but
+	 * harmless since the first into the list will never be looked up,
+	 * and thus will eventually just fall off the end.
 	 */
 	KAUTH_GROUPS_LOCK();
 	TAILQ_INSERT_HEAD(&kauth_groups, gm, gm_link);
@@ -1294,11 +2031,39 @@ kauth_groups_updatecache(struct kauth_identity_extlookup *el)
 		FREE(gm, M_KAUTH);
 }
 
+
 /*
  * Group membership KPI
  */
+
 /*
- * This function guarantees not to modify resultp when returning an error.
+ * kauth_cred_ismember_gid
+ *
+ * Description:	Given a credential and a GID, determine if the GID is a member
+ *		of one of the supplementary groups associated with the given
+ *		credential
+ *
+ * Parameters:	cred				Credential to check in
+ *		gid				GID to check for membership
+ *		resultp				Pointer to int to contain the
+ *						result of the call
+ *
+ * Returns:	0				Success
+ *		ENOENT				Could not proform lookup
+ *	kauth_resolver_submit:EWOULDBLOCK
+ *	kauth_resolver_submit:EINTR
+ *	kauth_resolver_submit:ENOMEM
+ *	kauth_resolver_submit:???		Unlikely error from user space
+ *
+ * Implicit returns:
+ *		*resultp (modified)	1	Is member
+ *					0	Is not member
+ *
+ * Notes:	This function guarantees not to modify resultp when returning
+ *		an error.
+ *
+ *		This function effectively checkes the EGID as well, since the
+ *		EGID is cr_groups[0] as an implementation detail.
  */
 int
 kauth_cred_ismember_gid(kauth_cred_t cred, gid_t gid, int *resultp)
@@ -1331,8 +2096,8 @@ kauth_cred_ismember_gid(kauth_cred_t cred, gid_t gid, int *resultp)
 		
 	
 	/*
-	 * If the resolver hasn't checked in yet, we are early in the boot phase and
-	 * the local group list is complete and authoritative.
+	 * If the resolver hasn't checked in yet, we are early in the boot
+	 * phase and the local group list is complete and authoritative.
 	 */
 	if (!kauth_resolver_registered) {
 		*resultp = 0;
@@ -1368,7 +2133,8 @@ kauth_cred_ismember_gid(kauth_cred_t cred, gid_t gid, int *resultp)
 	el.el_flags = KAUTH_EXTLOOKUP_VALID_UID | KAUTH_EXTLOOKUP_VALID_GID | KAUTH_EXTLOOKUP_WANT_MEMBERSHIP;
 	el.el_uid = cred->cr_gmuid;
 	el.el_gid = gid;
-	error = kauth_identity_resolve(&el);
+	el.el_member_valid = 0;		/* XXX set by resolver? */
+	error = kauth_resolver_submit(&el);
 	if (error != 0)
 		return(error);
 	/* save the results from the lookup */
@@ -1383,9 +2149,30 @@ kauth_cred_ismember_gid(kauth_cred_t cred, gid_t gid, int *resultp)
 	return(ENOENT);
 }
 
+
 /*
- * Determine whether the supplied credential is a member of the
- * group nominated by GUID.
+ * kauth_cred_ismember_guid
+ *
+ * Description:	Determine whether the supplied credential is a member of the
+ *		group nominated by GUID.
+ *
+ * Parameters:	cred				Credential to check in
+ *		guidp				Pointer to GUID whose group
+ *						we are testing for membership
+ *		resultp				Pointer to int to contain the
+ *						result of the call
+ *
+ * Returns:	0				Success
+ *	kauth_cred_guid2gid:EINVAL
+ *	kauth_cred_ismember_gid:ENOENT
+ *	kauth_cred_ismember_gid:EWOULDBLOCK
+ *	kauth_cred_ismember_gid:EINTR
+ *	kauth_cred_ismember_gid:ENOMEM
+ *	kauth_cred_ismember_gid:???		Unlikely error from user space
+ *
+ * Implicit returns:
+ *		*resultp (modified)	1	Is member
+ *					0	Is not member
  */
 int
 kauth_cred_ismember_guid(kauth_cred_t cred, guid_t *guidp, int *resultp)
@@ -1421,7 +2208,86 @@ kauth_cred_ismember_guid(kauth_cred_t cred, guid_t *guidp, int *resultp)
 }
 
 /*
- * Fast replacement for issuser()
+ * kauth_cred_gid_subset
+ *
+ * Description:	Given two credentials, determine if all GIDs associated with 
+ * 		the first are also associated with the second
+ *
+ * Parameters:	cred1				Credential to check for
+ * 		cred2				Credential to check in
+ *		resultp				Pointer to int to contain the
+ *						result of the call
+ *
+ * Returns:	0				Success
+ *		non-zero			See kauth_cred_ismember_gid for
+ *						error codes
+ *
+ * Implicit returns:
+ *		*resultp (modified)	1	Is subset
+ *					0	Is not subset
+ *
+ * Notes:	This function guarantees not to modify resultp when returning
+ *		an error.
+ */
+int	
+kauth_cred_gid_subset(kauth_cred_t cred1, kauth_cred_t cred2, int *resultp)
+{
+	int i, err, res = 1;
+	gid_t gid;
+
+	/* First, check the local list of groups */
+	for (i = 0; i < cred1->cr_ngroups; i++) {
+		gid = cred1->cr_groups[i];
+		if ((err = kauth_cred_ismember_gid(cred2, gid, &res)) != 0) {
+			return err;
+		}
+
+		if (!res && gid != cred2->cr_rgid && gid != cred2->cr_svgid) {
+			*resultp = 0;
+			return 0;
+		}
+	}
+
+	/* Check real gid */
+	if ((err = kauth_cred_ismember_gid(cred2, cred1->cr_rgid, &res)) != 0) {
+		return err;
+	}
+
+	if (!res && cred1->cr_rgid != cred2->cr_rgid &&
+			cred1->cr_rgid != cred2->cr_svgid) {
+		*resultp = 0;
+		return 0;
+	}
+
+	/* Finally, check saved gid */
+	if ((err = kauth_cred_ismember_gid(cred2, cred1->cr_svgid, &res)) != 0){
+		return err;
+	}
+
+	if (!res && cred1->cr_svgid != cred2->cr_rgid &&
+			cred1->cr_svgid != cred2->cr_svgid) {
+		*resultp = 0;
+		return 0;
+	}
+
+	*resultp = 1;
+	return 0;
+}
+
+
+/*
+ * kauth_cred_issuser
+ *
+ * Description:	Fast replacement for issuser()
+ *
+ * Parameters:	cred				Credential to check for super
+ *						user privileges
+ *
+ * Returns:	0				Not super user
+ *		!0				Is super user
+ *
+ * Notes:	This function uses a magic number which is not a manifest
+ *		constant; this is bad practice.
  */
 int
 kauth_cred_issuser(kauth_cred_t cred)
@@ -1429,6 +2295,7 @@ kauth_cred_issuser(kauth_cred_t cred)
 	return(cred->cr_uid == 0);
 }
 
+
 /*
  * Credential KPI
  */
@@ -1443,6 +2310,38 @@ static lck_mtx_t *kauth_cred_hash_mtx;
 #define KAUTH_CRED_HASH_LOCK_ASSERT()
 #endif	/* !KAUTH_CRED_HASH_DEBUG */
 
+
+/*
+ * kauth_cred_init
+ *
+ * Description:	Initialize the credential hash cache
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Notes:	Intialize the credential hash cache for use; the credential
+ *		hash cache is used convert duplicate credentials into a
+ *		single reference counted credential in order to save wired
+ *		kernel memory.  In practice, this generally means a desktop
+ *		system runs with a few tens of credentials, instead of one
+ *		per process, one per thread, one per vnode cache entry, and
+ *		so on.  This generally results in savings of 200K or more
+ *		(potentially much more on server systems).
+ *
+ *		The hash cache internally has a reference on the credential
+ *		for itself as a means of avoiding a reclaim race for a
+ *		credential in the process of having it's last non-hash
+ *		reference released.  This would otherwise result in the
+ *		possibility of a freed credential that was still in uses due
+ *		a race.  This use is protected by the KAUTH_CRED_HASH_LOCK.
+ *
+ *		On final release, the hash reference is droped, and the
+ *		credential is freed back to the system.
+ *
+ *		This function is called from kauth_init() in the file
+ *		kern_authorization.c.
+ */
 void
 kauth_cred_init(void)
 {
@@ -1455,13 +2354,23 @@ kauth_cred_init(void)
 	MALLOC(kauth_cred_table_anchor, struct kauth_cred_entry_head *, 
 			(sizeof(struct kauth_cred_entry_head) * kauth_cred_table_size), 
 			M_KAUTH, M_WAITOK | M_ZERO);
+	if (kauth_cred_table_anchor == NULL)
+		panic("startup: kauth_cred_init");
 	for (i = 0; i < kauth_cred_table_size; i++) {
 		TAILQ_INIT(&kauth_cred_table_anchor[i]);
 	}
 }
 
+
 /*
- * Return the current thread's effective UID.
+ * kauth_getuid
+ *
+ * Description:	Get the current thread's effective UID.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(uid_t)				The effective UID of the
+ *						current thread
  */
 uid_t
 kauth_getuid(void)
@@ -1469,8 +2378,16 @@ kauth_getuid(void)
 	return(kauth_cred_get()->cr_uid);
 }
 
+
 /*
- * Return the current thread's real UID.
+ * kauth_getruid
+ *
+ * Description:	Get the current thread's real UID.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(uid_t)				The real UID of the current
+ *						thread
  */
 uid_t
 kauth_getruid(void)
@@ -1478,8 +2395,16 @@ kauth_getruid(void)
 	return(kauth_cred_get()->cr_ruid);
 }
 
+
 /*
- * Return the current thread's effective GID.
+ * kauth_getgid
+ *
+ * Description:	Get the current thread's effective GID.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(gid_t)				The effective GID of the
+ *						current thread
  */
 gid_t
 kauth_getgid(void)
@@ -1487,8 +2412,16 @@ kauth_getgid(void)
 	return(kauth_cred_get()->cr_groups[0]);
 }
 
+
 /*
- * Return the current thread's real GID.
+ * kauth_getgid
+ *
+ * Description:	Get the current thread's real GID.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(gid_t)				The real GID of the current
+ *						thread
  */
 gid_t
 kauth_getrgid(void)
@@ -1496,10 +2429,28 @@ kauth_getrgid(void)
 	return(kauth_cred_get()->cr_rgid);
 }
 
+
 /*
- * Returns a pointer to the current thread's credential, does not take a
- * reference (so the caller must not do anything that would let the thread's
- * credential change while using the returned value).
+ * kauth_cred_get
+ *
+ * Description:	Returns a pointer to the current thread's credential
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(kauth_cred_t)			Pointer to the current thread's
+ *						credential
+ *
+ * Notes:	This function does not take a reference; because of this, the
+ *		caller MUST NOT do anything that would let the thread's
+ *		credential change while using the returned value, without
+ *		first explicitly taking their own reference.
+ *
+ *		If a caller intends to take a reference on the resulting
+ *		credential pointer from calling this function, it is strongly
+ *		recommended that the caller use kauth_cred_get_with_ref()
+ *		instead, to protect against any future changes to the cred
+ *		locking protocols; such changes could otherwise potentially
+ *		introduce race windows in the callers code.
  */
 kauth_cred_t
 kauth_cred_get(void)
@@ -1512,9 +2463,11 @@ kauth_cred_get(void)
 	if (uthread == NULL)
 		panic("thread wants credential but has no BSD thread info");
 	/*
-	 * We can lazy-bind credentials to threads, as long as their processes have them.
-	 * If we later inline this function, the code in this block should probably be
-	 * called out in a function.
+	 * We can lazy-bind credentials to threads, as long as their processes
+	 * have them.
+	 *
+	 * XXX If we later inline this function, the code in this block
+	 * XXX should probably be called out in a function.
 	 */
 	if (uthread->uu_ucred == NOCRED) {
 		if ((p = (proc_t) get_bsdtask_info(get_threadtask(current_thread()))) == NULL)
@@ -1524,8 +2477,60 @@ kauth_cred_get(void)
 	return(uthread->uu_ucred);
 }
 
+
+/*
+ * kauth_cred_uthread_update
+ *
+ * Description:	Given a uthread, a proc, and whether or not the proc is locked,
+ *		late-bind the uthread cred to the proc cred.
+ *
+ * Parameters:	uthread_t			The uthread to update
+ *		proc_t				The process to update to
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This code is common code called from system call or trap entry
+ *		in the case that the process thread may have been changed
+ *		since the last time the thread entered the kernel.  It is
+ *		generally only called with the current uthread and process as
+ *		parameters.
+ */
+void
+kauth_cred_uthread_update(uthread_t uthread, proc_t proc)
+{
+	if (uthread->uu_ucred != proc->p_ucred &&
+	    (uthread->uu_flag & UT_SETUID) == 0) {
+		kauth_cred_t old = uthread->uu_ucred;
+		uthread->uu_ucred = kauth_cred_proc_ref(proc);
+		if (IS_VALID_CRED(old))
+			kauth_cred_unref(&old);
+	}
+}
+
+
 /*
- * Returns a pointer to the current thread's credential, takes a reference.
+ * kauth_cred_get_with_ref
+ *
+ * Description:	Takes a reference on the current thread's credential, and then
+ *		returns a pointer to it to the caller.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(kauth_cred_t)			Pointer to the current thread's
+ *						newly referenced credential
+ *
+ * Notes:	This function takes a reference on the credential before
+ *		returning it to the caller.
+ *
+ *		It is the responsibility of the calling code to release this
+ *		reference when the credential is no longer in use.
+ *
+ *		Since the returned reference may be a persistent reference
+ *		(e.g. one cached in another data structure with a lifetime
+ *		longer than the calling function), this release may be delayed
+ *		until such time as the persistent reference is to be destroyed.
+ *		An example of this would be the per vnode credential cache used
+ *		to accelerate lookup operations.
  */
 kauth_cred_t
 kauth_cred_get_with_ref(void)
@@ -1541,9 +2546,11 @@ kauth_cred_get_with_ref(void)
 		panic("%s - thread wants credential but has no BSD process", __FUNCTION__);
 
 	/*
-	 * We can lazy-bind credentials to threads, as long as their processes have them.
-	 * If we later inline this function, the code in this block should probably be
-	 * called out in a function.
+	 * We can lazy-bind credentials to threads, as long as their processes
+	 * have them.
+	 *
+	 * XXX If we later inline this function, the code in this block
+	 * XXX should probably be called out in a function.
 	 */
 	if (uthread->uu_ucred == NOCRED) {
 		/* take reference for new cred in thread */
@@ -1554,8 +2561,35 @@ kauth_cred_get_with_ref(void)
 	return(uthread->uu_ucred);
 }
 
+
 /*
- * Returns a pointer to the given process's credential, takes a reference.
+ * kauth_cred_proc_ref
+ *
+ * Description:	Takes a reference on the current process's credential, and
+ *		then returns a pointer to it to the caller.
+ *
+ * Parameters:	procp				Process whose credential we
+ *						intend to take a reference on
+ *
+ * Returns:	(kauth_cred_t)			Pointer to the process's
+ *						newly referenced credential
+ *
+ * Locks:	PROC_LOCK is held before taking the reference and released
+ *		after the refeence is taken to protect the p_ucred field of
+ *		the process referred to by procp.
+ *
+ * Notes:	This function takes a reference on the credential before
+ *		returning it to the caller.
+ *
+ *		It is the responsibility of the calling code to release this
+ *		reference when the credential is no longer in use.
+ *
+ *		Since the returned reference may be a persistent reference
+ *		(e.g. one cached in another data structure with a lifetime
+ *		longer than the calling function), this release may be delayed
+ *		until such time as the persistent reference is to be destroyed.
+ *		An example of this would be the per vnode credential cache used
+ *		to accelerate lookup operations.
  */
 kauth_cred_t
 kauth_cred_proc_ref(proc_t procp)
@@ -1569,16 +2603,55 @@ kauth_cred_proc_ref(proc_t procp)
 	return(cred);
 }
 
+
 /*
- * Allocates a new credential.
+ * kauth_cred_alloc
+ *
+ * Description:	Allocate a new credential
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	!NULL				Newly allocated credential
+ *		NULL				Insufficient memory
+ *
+ * Notes:	The newly allocated credential is zero'ed as part of the
+ *		allocation process, with the exception of the reference
+ *		count, which is set to 1 to indicate a single reference
+ *		held by the caller.
+ *
+ *		Since newly allocated credentials have no external pointers
+ *		referencing them, prior to making them visible in an externally
+ *		visible pointer (e.g. by adding them to the credential hash
+ *		cache) is the only legal time in which an existing credential
+ *		can be safely iinitialized or modified directly.
+ *
+ *		After initialization, the caller is expected to call the
+ *		function kauth_cred_add() to add the credential to the hash
+ *		cache, after which time it's frozen and becomes publically
+ *		visible.
+ *
+ *		The release protocol depends on kauth_hash_add() being called
+ *		before kauth_cred_rele() (there is a diagnostic panic which
+ *		will trigger if this protocol is not observed).
+ *
+ * XXX:		This function really ought to be static, rather than being
+ *		exported as KPI, since a failure of kauth_cred_add() can only
+ *		be handled by an explicit free of the credential; such frees
+ *		depend on knowlegdge of the allocation method used, which is
+ *		permitted to change between kernel revisions.
+ *
+ * XXX:		In the insufficient resource case, this code panic's rather
+ *		than returning a NULL pointer; the code that calls this
+ *		function needs to be audited before this can be changed.
  */
 kauth_cred_t
 kauth_cred_alloc(void)
 {
 	kauth_cred_t newcred;
 	
-	MALLOC(newcred, kauth_cred_t, sizeof(*newcred), M_CRED, M_WAITOK | M_ZERO);
+	MALLOC_ZONE(newcred, kauth_cred_t, sizeof(*newcred), M_CRED, M_WAITOK);
 	if (newcred != 0) {
+		bzero(newcred, sizeof(*newcred));
 		newcred->cr_ref = 1;
 		/* must do this, or cred has same group membership as uid 0 */
 		newcred->cr_gmuid = KAUTH_UID_NONE;
@@ -1592,37 +2665,74 @@ kauth_cred_alloc(void)
 	kauth_cred_count++;
 #endif
 
+#if CONFIG_MACF
+	mac_cred_label_init(newcred);
+#endif
+
 	return(newcred);
 }
 
+
 /*
- * Looks to see if we already have a known credential and if found bumps the
- *	reference count and returns it.  If there are no credentials that match 
- *	the given credential then we allocate a new credential.
+ * kauth_cred_create
+ *
+ * Description:	Look to see if we already have a known credential in the hash
+ *		cache; if one is found, bump the reference count and return
+ *		it.  If there are no credentials that match the given
+ *		credential, then allocate a new credential.
+ *
+ * Parameters:	cred				Template for credential to
+ *						be created
+ *
+ * Returns:	(kauth_cred_t)			The credential that was found
+ *						in the hash or created
+ *		NULL				kauth_cred_add() failed, or
+ *						there was not an egid specified
+ *
+ * Notes:	The gmuid is hard-defaulted to the UID specified.  Since we
+ *		maintain this field, we can't expect callers to know how it
+ *		needs to be set.  Callers should be prepared for this field
+ *		to be overwritten.
  *
- * Note that the gmuid is hard-defaulted to the UID specified.  Since we maintain
- * this field, we can't expect callers to know how it needs to be set.  Callers
- * should be prepared for this field to be overwritten.
+ * XXX:		This code will tight-loop if memory for a new credential is
+ *		persistently unavailable; this is perhaps not the wisest way
+ *		to handle this condition, but current callers do not expect
+ *		a failure.
  */
 kauth_cred_t
 kauth_cred_create(kauth_cred_t cred)
 {
 	kauth_cred_t 	found_cred, new_cred = NULL;
 
-	cred->cr_gmuid = cred->cr_uid;
+	KAUTH_CRED_HASH_LOCK_ASSERT();
+
+	if (cred->cr_flags & CRF_NOMEMBERD)
+		cred->cr_gmuid = KAUTH_UID_NONE;
+	else
+		cred->cr_gmuid = cred->cr_uid;
+
+	/* Caller *must* specify at least the egid in cr_groups[0] */
+	if (cred->cr_ngroups < 1)
+		return(NULL);
 	
 	for (;;) {
 		KAUTH_CRED_HASH_LOCK();
 		found_cred = kauth_cred_find(cred);
 		if (found_cred != NULL) {
-			/* found an existing credential so we'll bump reference count and return */
+			/*
+			 * Found an existing credential so we'll bump
+			 * reference count and return
+			 */
 			kauth_cred_ref(found_cred);
 			KAUTH_CRED_HASH_UNLOCK();
 			return(found_cred);
 		}
 		KAUTH_CRED_HASH_UNLOCK();
 	
-		/* no existing credential found.  create one and add it to our hash table */
+		/*
+		 * No existing credential found.  Create one and add it to
+		 * our hash table.
+		 */
 		new_cred = kauth_cred_alloc();
 		if (new_cred != NULL) {
 			int		err;
@@ -1634,14 +2744,19 @@ kauth_cred_create(kauth_cred_t cred)
 			new_cred->cr_gmuid = cred->cr_gmuid;
 			new_cred->cr_ngroups = cred->cr_ngroups;	
 			bcopy(&cred->cr_groups[0], &new_cred->cr_groups[0], sizeof(new_cred->cr_groups));
+			new_cred->cr_flags = cred->cr_flags;
+			
 			KAUTH_CRED_HASH_LOCK();
 			err = kauth_cred_add(new_cred);
 			KAUTH_CRED_HASH_UNLOCK();
 			
-			/* retry if kauth_cred_add returns non zero value */
+			/* Retry if kauth_cred_add returns non zero value */
 			if (err == 0)
 				break;
-			FREE(new_cred, M_CRED);
+#if CONFIG_MACF
+			mac_cred_label_destroy(new_cred);
+#endif
+			FREE_ZONE(new_cred, sizeof(*new_cred), M_CRED);
 			new_cred = NULL;
 		}
 	}
@@ -1649,146 +2764,217 @@ kauth_cred_create(kauth_cred_t cred)
 	return(new_cred);
 }
 
+
 /*
- * Update the given credential using the uid argument.  The given uid is used
- *	set the effective user ID, real user ID, and saved user ID.  We only 
- *	allocate a new credential when the given uid actually results in changes to
- *	the existing credential.
+ * kauth_cred_setresuid
+ *
+ * Description:	Update the given credential using the UID arguments.  The given
+ *		UIDs are used to set the effective UID, real UID, saved UID,
+ *		and GMUID (used for group membership checking).
+ *
+ * Parameters:	cred				The original credential
+ *		ruid				The new real UID
+ *		euid				The new effective UID
+ *		svuid				The new saved UID
+ *		gmuid				KAUTH_UID_NONE -or- the new
+ *						group membership UID
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * Note:	gmuid is different in that a KAUTH_UID_NONE is a valid
+ *		setting, so if you don't want it to change, pass it the
+ *		previous value, explicitly.
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 kauth_cred_t
-kauth_cred_setuid(kauth_cred_t cred, uid_t uid)
+kauth_cred_setresuid(kauth_cred_t cred, uid_t ruid, uid_t euid, uid_t svuid, uid_t gmuid)
 {
 	struct ucred temp_cred;
 
 	NULLCRED_CHECK(cred);
 
-	/* don't need to do anything if the effective, real and saved user IDs are
-	 * already the same as the user ID passed in
+	/*
+	 * We don't need to do anything if the UIDs we are changing are
+	 * already the same as the UIDs passed in
 	 */
-	if (cred->cr_uid == uid && cred->cr_ruid == uid && cred->cr_svuid == uid) {
+	if ((euid == KAUTH_UID_NONE || cred->cr_uid == euid) &&
+	    (ruid == KAUTH_UID_NONE || cred->cr_ruid == ruid) &&
+	    (svuid == KAUTH_UID_NONE || cred->cr_svuid == svuid) &&
+	    (cred->cr_gmuid == gmuid)) {
 		/* no change needed */
 		return(cred);
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
+	/*
+	 * Look up in cred hash table to see if we have a matching credential
+	 * with the new values; this is done by calling kauth_cred_update().
 	 */
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
-	temp_cred.cr_uid = uid;
-	temp_cred.cr_ruid = uid;
-	temp_cred.cr_svuid = uid;
-	temp_cred.cr_gmuid = uid;
+	if (euid != KAUTH_UID_NONE) {
+		temp_cred.cr_uid = euid;
+	}
+	if (ruid != KAUTH_UID_NONE) {
+		temp_cred.cr_ruid = ruid;
+	}
+	if (svuid != KAUTH_UID_NONE) {
+		temp_cred.cr_svuid = svuid;
+	}
+
+	/*
+	 * If we are setting the gmuid to KAUTH_UID_NONE, then we want to
+	 * opt out of participation in external group resolution, unless we
+	 * unless we explicitly opt back in later.
+	 */
+	if ((temp_cred.cr_gmuid = gmuid) == KAUTH_UID_NONE) {
+		temp_cred.cr_flags |= CRF_NOMEMBERD;
+	}
 
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
- * Update the given credential using the euid argument.  The given uid is used
- *	set the effective user ID.  We only allocate a new credential when the given 
- *	uid actually results in changes to the existing credential.
+ * kauth_cred_setresgid
+ *
+ * Description:	Update the given credential using the GID arguments.  The given
+ *		GIDs are used to set the effective GID, real GID, and saved
+ *		GID.
+ *
+ * Parameters:	cred				The original credential
+ *		rgid				The new real GID
+ *		egid				The new effective GID
+ *		svgid				The new saved GID
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 kauth_cred_t
-kauth_cred_seteuid(kauth_cred_t cred, uid_t euid)
+kauth_cred_setresgid(kauth_cred_t cred, gid_t rgid, gid_t egid, gid_t svgid)
 {
-	struct ucred temp_cred;
+	struct ucred 	temp_cred;
 
 	NULLCRED_CHECK(cred);
+	DEBUG_CRED_ENTER("kauth_cred_setresgid %p %d %d %d\n", cred, rgid, egid, svgid);
 
-	/* don't need to do anything if the given effective user ID is already the 
-	 *	same as the effective user ID in the credential.
+	/*
+	 * We don't need to do anything if the given GID are already the 
+	 * same as the GIDs in the credential.
 	 */
-	if (cred->cr_uid == euid) {
+	if (cred->cr_groups[0] == egid &&
+	    cred->cr_rgid == rgid &&
+	    cred->cr_svgid == svgid) {
 		/* no change needed */
 		return(cred);
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
+	/*
+	 * Look up in cred hash table to see if we have a matching credential
+	 * with the new values; this is done by calling kauth_cred_update().
 	 */
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
-	temp_cred.cr_uid = euid;
+	if (egid != KAUTH_GID_NONE) {
+		/* displacing a supplementary group opts us out of memberd */
+		if (kauth_cred_change_egid(&temp_cred, egid)) {
+			DEBUG_CRED_CHANGE("displaced!\n");
+			temp_cred.cr_flags |= CRF_NOMEMBERD;
+			temp_cred.cr_gmuid = KAUTH_UID_NONE;
+		} else {
+			DEBUG_CRED_CHANGE("not displaced\n");
+		}
+	}
+	if (rgid != KAUTH_GID_NONE) {
+		temp_cred.cr_rgid = rgid;
+	}
+	if (svgid != KAUTH_GID_NONE) {
+		temp_cred.cr_svgid = svgid;
+	}
 
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
- * Update the given credential using the gid argument.  The given gid is used
- *	set the effective group ID, real group ID, and saved group ID.  We only 
- *	allocate a new credential when the given gid actually results in changes to
- *	the existing credential.
+ * Update the given credential with the given groups.  We only allocate a new 
+ *	credential when the given gid actually results in changes to the existing 
+ *	credential.
+ *	The gmuid argument supplies a new uid (or KAUTH_UID_NONE to opt out)
+ *	which will be used for group membership checking.
+ */
+/*
+ * kauth_cred_setgroups
+ *
+ * Description:	Update the given credential using the provide supplementary
+ *		group list and group membership UID
+ *
+ * Parameters:	cred				The original credential
+ *		groups				Pointer to gid_t array which
+ *						contains the new group list
+ *		groupcount			The cound of valid groups which
+ *						are contained in 'groups'
+ *		gmuid				KAUTH_UID_NONE -or- the new
+ *						group membership UID
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * Note:	gmuid is different in that a KAUTH_UID_NONE is a valid
+ *		setting, so if you don't want it to change, pass it the
+ *		previous value, explicitly.
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
+ *
+ * XXX:		Changes are determined in ordinal order - if the caller pasess
+ *		in the same groups list that is already present in the
+ *		credential, but the members are in a different order, even if
+ *		the EGID is not modified (i.e. cr_groups[0] is the same), it
+ *		is considered a modification to the credential, and a new
+ *		credential is created.
+ *
+ *		This should perhaps be better optimized, but it is considered
+ *		to be the caller's problem.
  */
 kauth_cred_t
-kauth_cred_setgid(kauth_cred_t cred, gid_t gid)
+kauth_cred_setgroups(kauth_cred_t cred, gid_t *groups, int groupcount, uid_t gmuid)
 {
-	struct ucred 	temp_cred;
+	int		i;
+	struct ucred temp_cred;
 
 	NULLCRED_CHECK(cred);
 
-	/* don't need to do anything if the given group ID is already the 
-	 *	same as the group ID in the credential.
-	 */
-	if (cred->cr_groups[0] == gid && cred->cr_rgid == gid && cred->cr_svgid == gid) {
-		/* no change needed */
-		return(cred);
-	}
-
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
-	 */
-	bcopy(cred, &temp_cred, sizeof(temp_cred));
-	temp_cred.cr_groups[0] = gid;
-	temp_cred.cr_rgid = gid;
-	temp_cred.cr_svgid = gid;
-
-	return(kauth_cred_update(cred, &temp_cred, TRUE));
-}
-
-
-/*
- * Update the given credential using the egid argument.  The given gid is used
- *	set the effective user ID.  We only allocate a new credential when the given 
- *	gid actually results in changes to the existing credential.
- */
-kauth_cred_t
-kauth_cred_setegid(kauth_cred_t cred, gid_t egid)
-{
-	struct ucred temp_cred;
-
-	NULLCRED_CHECK(cred);
-
-	/* don't need to do anything if the given group ID is already the 
-	 *	same as the group Id in the credential.
-	 */
-	if (cred->cr_groups[0] == egid) {
-		/* no change needed */
-		return(cred);
-	}
-
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
-	 */
-	bcopy(cred, &temp_cred, sizeof(temp_cred));
-	temp_cred.cr_groups[0] = egid;
-
-	return(kauth_cred_update(cred, &temp_cred, TRUE));
-}
-
-/*
- * Update the given credential with the given groups.  We only allocate a new 
- *	credential when the given gid actually results in changes to the existing 
- *	credential.
- *	The gmuid argument supplies a new uid (or KAUTH_UID_NONE to opt out)
- *	which will be used for group membership checking.
- */
-kauth_cred_t
-kauth_cred_setgroups(kauth_cred_t cred, gid_t *groups, int groupcount, uid_t gmuid)
-{
-	int		i;
-	struct ucred temp_cred;
-
-	NULLCRED_CHECK(cred);
-
-	/* don't need to do anything if the given list of groups does not change.
+	/*
+	 * We don't need to do anything if the given list of groups does not
+	 * change.
 	 */
 	if ((cred->cr_gmuid == gmuid) && (cred->cr_ngroups == groupcount)) {
 		for (i = 0; i < groupcount; i++) {
@@ -1801,24 +2987,60 @@ kauth_cred_setgroups(kauth_cred_t cred, gid_t *groups, int groupcount, uid_t gmu
 		}
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
+	/*
+	 * Look up in cred hash table to see if we have a matching credential
+	 * with new values.  If we are setting or clearing the gmuid, then
+	 * update the cr_flags, since clearing it is sticky.  This permits an
+	 * opt-out of memberd processing using setgroups(), and an opt-in
+	 * using initgroups().  This is required for POSIX conformance.
 	 */
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
 	temp_cred.cr_ngroups = groupcount;
 	bcopy(groups, temp_cred.cr_groups, sizeof(temp_cred.cr_groups));
 	temp_cred.cr_gmuid = gmuid;
+	if (gmuid == KAUTH_UID_NONE)
+		temp_cred.cr_flags |= CRF_NOMEMBERD;
+	else
+		temp_cred.cr_flags &= ~CRF_NOMEMBERD;
 
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
- * Update the given credential using the uid and gid arguments.  The given uid 
- *	is used set the effective user ID, real user ID, and saved user ID.  
- * 	The given gid is used set the effective group ID, real group ID, and saved 
- *	group ID.
- *	We only allocate a new credential when the given uid and gid actually results 
- *	in changes to the existing credential.
+ * kauth_cred_setuidgid
+ *
+ * Description:	Update the given credential using the UID and GID arguments.
+ *		The given UID is used to set the effective UID, real UID, and
+ *		saved UID.  The given GID is used to set the effective GID,
+ *		real GID, and saved GID.
+ *
+ * Parameters:	cred				The original credential
+ *		uid				The new UID to use
+ *		gid				The new GID to use
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * Notes:	We set the gmuid to uid if the credential we are inheriting
+ *		from has not opted out of memberd participation; otherwise
+ *		we set it to KAUTH_UID_NONE
+ *
+ *		This code is only ever called from the per-thread credential
+ *		code path in the "set per thread credential" case; and in
+ *		posix_spawn() in the case that the POSIX_SPAWN_RESETIDS
+ *		flag is set.
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 kauth_cred_t
 kauth_cred_setuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
@@ -1827,8 +3049,9 @@ kauth_cred_setuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
 
 	NULLCRED_CHECK(cred);
 
-	/* don't need to do anything if the effective, real and saved user IDs are
-	 * already the same as the user ID passed in
+	/*
+	 * We don't need to do anything if the effective, real and saved
+	 * user IDs are already the same as the user ID passed into us.
 	 */
 	if (cred->cr_uid == uid && cred->cr_ruid == uid && cred->cr_svuid == uid &&
 		cred->cr_groups[0] == gid && cred->cr_rgid == gid && cred->cr_svgid == gid) {
@@ -1836,28 +3059,61 @@ kauth_cred_setuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
 		return(cred);
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
+	/*
+	 * Look up in cred hash table to see if we have a matching credential
+	 * with the new values.
 	 */
 	bzero(&temp_cred, sizeof(temp_cred));
 	temp_cred.cr_uid = uid;
 	temp_cred.cr_ruid = uid;
 	temp_cred.cr_svuid = uid;
-	temp_cred.cr_gmuid = uid;
+	/* inherit the opt-out of memberd */
+	if (cred->cr_flags & CRF_NOMEMBERD) {
+		temp_cred.cr_gmuid = KAUTH_UID_NONE;
+		temp_cred.cr_flags |= CRF_NOMEMBERD;
+	} else {
+		temp_cred.cr_gmuid = uid;
+		temp_cred.cr_flags &= ~CRF_NOMEMBERD;
+	}
 	temp_cred.cr_ngroups = 1;
-	temp_cred.cr_groups[0] = gid;
+	/* displacing a supplementary group opts us out of memberd */
+	if (kauth_cred_change_egid(&temp_cred, gid)) {
+		temp_cred.cr_gmuid = KAUTH_UID_NONE;
+		temp_cred.cr_flags |= CRF_NOMEMBERD;
+	}
 	temp_cred.cr_rgid = gid;
 	temp_cred.cr_svgid = gid;
+#if CONFIG_MACF
+	temp_cred.cr_label = cred->cr_label;
+#endif
 
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
- * Update the given credential using the uid and gid arguments.  The given uid 
- *	is used to set the saved user ID.  The given gid is used to set the 
- *	saved group ID.
- *	We only allocate a new credential when the given uid and gid actually results 
- *	in changes to the existing credential.
+ * kauth_cred_setsvuidgid
+ *
+ * Description:	Function used by execve to set the saved uid and gid values
+ *		for suid/sgid programs
+ *
+ * Parameters:	cred				The credential to update
+ *		uid				The saved uid to set
+ *		gid				The saved gid to set
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 kauth_cred_t
 kauth_cred_setsvuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
@@ -1865,14 +3121,18 @@ kauth_cred_setsvuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
 	struct ucred temp_cred;
 
 	NULLCRED_CHECK(cred);
+	DEBUG_CRED_ENTER("kauth_cred_setsvuidgid: %p u%d->%d g%d->%d\n", cred, cred->cr_svuid, uid, cred->cr_svgid, gid);
 
-	/* don't need to do anything if the effective, real and saved user IDs are
-	 * already the same as the user ID passed in
+	/*
+	 * We don't need to do anything if the effective, real and saved
+	 * uids are already the same as the uid provided.  This check is
+	 * likely insufficient.
 	 */
 	if (cred->cr_svuid == uid && cred->cr_svgid == gid) {
 		/* no change needed */
 		return(cred);
 	}
+	DEBUG_CRED_CHANGE("kauth_cred_setsvuidgid: cred change\n");
 
 	/* look up in cred hash table to see if we have a matching credential
 	 * with new values.
@@ -1884,10 +3144,28 @@ kauth_cred_setsvuidgid(kauth_cred_t cred, uid_t uid, gid_t gid)
 	return(kauth_cred_update(cred, &temp_cred, TRUE));
 }
 
+
 /*
- * Update the given credential using the given auditinfo_t.
- *	We only allocate a new credential when the given auditinfo_t actually results 
- *	in changes to the existing credential.
+ * kauth_cred_setauditinfo
+ * 
+ * Description:	Update the given credential using the given auditinfo_t.
+ *
+ * Parameters:	cred				The original credential
+ *		auditinfo_p			Pointer to ne audit information
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 kauth_cred_t
 kauth_cred_setauditinfo(kauth_cred_t cred, auditinfo_t *auditinfo_p)
@@ -1896,25 +3174,314 @@ kauth_cred_setauditinfo(kauth_cred_t cred, auditinfo_t *auditinfo_p)
 
 	NULLCRED_CHECK(cred);
 
-	/* don't need to do anything if the audit info is already the same as the 
-	 * audit info in the credential passed in
+	/*
+	 * We don't need to do anything if the audit info is already the
+	 * same as the audit info in the credential provided.
 	 */
 	if (bcmp(&cred->cr_au, auditinfo_p, sizeof(cred->cr_au)) == 0) {
 		/* no change needed */
 		return(cred);
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
-	 */
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
 	bcopy(auditinfo_p, &temp_cred.cr_au, sizeof(temp_cred.cr_au));
 
 	return(kauth_cred_update(cred, &temp_cred, FALSE));
 }
 
+#if CONFIG_MACF
 /*
- * Add a reference to the passed credential.
+ * kauth_cred_label_update
+ * 
+ * Description:	Update the MAC label associated with a credential
+ *
+ * Parameters:	cred				The original credential
+ *		label				The MAC label to set
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
+ */
+kauth_cred_t
+kauth_cred_label_update(kauth_cred_t cred, struct label *label)
+{
+	kauth_cred_t newcred;
+	struct ucred temp_cred;
+
+	bcopy(cred, &temp_cred, sizeof(temp_cred));
+
+	mac_cred_label_init(&temp_cred);
+	mac_cred_label_associate(cred, &temp_cred);
+	mac_cred_label_update(&temp_cred, label);
+
+	newcred = kauth_cred_update(cred, &temp_cred, TRUE);
+	mac_cred_label_destroy(&temp_cred);
+	return (newcred);
+}
+
+/*
+ * kauth_cred_label_update_execve
+ * 
+ * Description:	Update the MAC label associated with a credential as
+ *		part of exec
+ *
+ * Parameters:	cred				The original credential
+ *		vp				The exec vnode
+ *		scriptl				The script MAC label
+ *		execl				The executable MAC label
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function is implemented via kauth_cred_update(), which,
+ *		if it returns a credential other than the one it is passed,
+ *		will have dropped the reference on the passed credential.  All
+ *		callers should be aware of this, and treat this function as an
+ *		unref + ref, potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
+ */
+static
+kauth_cred_t
+kauth_cred_label_update_execve(kauth_cred_t cred, vfs_context_t ctx,
+	struct vnode *vp, struct label *scriptl, struct label *execl)
+{
+	kauth_cred_t newcred;
+	struct ucred temp_cred;
+
+	bcopy(cred, &temp_cred, sizeof(temp_cred));
+
+	mac_cred_label_init(&temp_cred);
+	mac_cred_label_associate(cred, &temp_cred);
+	mac_cred_label_update_execve(ctx, &temp_cred, 
+				     vp, scriptl, execl);
+
+	newcred = kauth_cred_update(cred, &temp_cred, TRUE);
+	mac_cred_label_destroy(&temp_cred);
+	return (newcred);
+}
+
+/*
+ *  kauth_proc_label_update
+ *
+ * Description:  Update the label inside the credential associated with the process.
+ *
+ * Parameters:	p			The process to modify
+ *				label		The label to place in the process credential
+ *
+ * Notes:		The credential associated with the process may change as a result
+ *				of this call.  The caller should not assume the process reference to
+ *				the old credential still exists.
+ */
+int kauth_proc_label_update(struct proc *p, struct label *label)
+{
+	kauth_cred_t my_cred, my_new_cred;
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	DEBUG_CRED_ENTER("kauth_proc_label_update: %p\n", my_cred);
+
+	/* get current credential and take a reference while we muck with it */
+	for (;;) {
+
+  		/* 
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
+  		 */
+		my_new_cred = kauth_cred_label_update(my_cred, label);
+		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("kauth_proc_setlabel_unlocked CH(%d): %p/0x%08x -> %p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
+			proc_lock(p);
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
+			 */
+			if (p->p_ucred != my_cred) {
+				proc_unlock(p);
+				kauth_cred_unref(&my_new_cred);
+				my_cred = kauth_cred_proc_ref(p);
+				/* try again */
+				continue;
+			}
+			p->p_ucred = my_new_cred;
+			mac_proc_set_enforce(p, MAC_ALL_ENFORCE);
+			proc_unlock(p);
+		}
+		break;
+	}
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+	
+	return (0);
+}
+
+/*
+ *  kauth_proc_label_update_execve
+ *
+ * Description: Update the label inside the credential associated with the
+ *		process as part of a transitioning execve.  The label will
+ *		be updated by the policies as part of this processing, not
+ *		provided up front.
+ *
+ * Parameters:	p			The process to modify
+ *		ctx			The context of the exec
+ *		vp			The vnode being exec'ed
+ *		scriptl			The script MAC label
+ *		execl			The executable MAC label
+ *
+ * Notes:	The credential associated with the process WILL change as a
+ *		result of this call.  The caller should not assume the process
+ *		reference to the old credential still exists.
+ */
+int kauth_proc_label_update_execve(struct proc *p, vfs_context_t ctx,
+	struct vnode *vp, struct label *scriptl, struct label *execl)
+{
+	kauth_cred_t my_cred, my_new_cred;
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	DEBUG_CRED_ENTER("kauth_proc_label_update_execve: %p\n", my_cred);
+
+	/* get current credential and take a reference while we muck with it */
+	for (;;) {
+
+  		/* 
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
+  		 */
+		my_new_cred = kauth_cred_label_update_execve(my_cred, ctx, vp, scriptl, execl);
+		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("kauth_proc_label_update_execve_unlocked CH(%d): %p/0x%08x -> %p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
+			proc_lock(p);
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
+			 */
+			if (p->p_ucred != my_cred) {
+				proc_unlock(p);
+				kauth_cred_unref(&my_new_cred);
+				my_cred = kauth_cred_proc_ref(p);
+				/* try again */
+				continue;
+			}
+			p->p_ucred = my_new_cred;
+			mac_proc_set_enforce(p, MAC_ALL_ENFORCE);
+			proc_unlock(p);
+		}
+		break;
+	}
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+	
+	return (0);
+}
+
+#if 1
+/*
+ * for temporary binary compatibility
+ */
+kauth_cred_t	kauth_cred_setlabel(kauth_cred_t cred, struct label *label);
+kauth_cred_t
+kauth_cred_setlabel(kauth_cred_t cred, struct label *label)
+{
+	return kauth_cred_label_update(cred, label);
+}
+
+int kauth_proc_setlabel(struct proc *p, struct label *label);
+int
+kauth_proc_setlabel(struct proc *p, struct label *label)
+{
+	return kauth_proc_label_update(p, label);
+}
+#endif
+
+#else
+
+/* this is a temp hack to cover us when MACF is not built in a kernel configuration. 
+ * Since we cannot build our export lists based on the kernel configuration we need
+ * to define a stub. 
+ */
+kauth_cred_t
+kauth_cred_label_update(__unused kauth_cred_t cred, __unused void *label)
+{
+	return(NULL);
+}
+
+int
+kauth_proc_label_update(__unused struct proc *p, __unused void *label)
+{
+	return (0);
+}
+
+#if 1
+/*
+ * for temporary binary compatibility
+ */
+kauth_cred_t	kauth_cred_setlabel(kauth_cred_t cred, void *label);
+kauth_cred_t
+kauth_cred_setlabel(__unused kauth_cred_t cred, __unused void *label)
+{
+	return NULL;
+}
+
+int kauth_proc_setlabel(struct proc *p, void *label);
+int
+kauth_proc_setlabel(__unused struct proc *p, __unused void *label)
+{
+	return (0);
+}
+#endif
+#endif
+
+/*
+ * kauth_cred_ref
+ *
+ * Description:	Add a reference to the passed credential
+ *
+ * Parameters:	cred				The credential to reference
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This function adds a reference to the provided credential;
+ *		the existing reference on the credential is assumed to be
+ *		held stable over this operation by taking the appropriate
+ *		lock to protect the pointer from which it is being referenced,
+ *		if necessary (e.g. the proc lock is held over the call if the
+ *		credential being referenced is from p_ucred, the vnode lock
+ *		if from the per vnode name cache cred cache, and so on).
+ *
+ *		This is safe from the kauth_cred_unref() path, since an atomic
+ *		add is used, and the unref path specifically checks to see that
+ *		the value has not been changed to add a reference between the
+ *		time the credential is unreferenced by another pointer and the
+ *		time it is unreferenced from the cred hash cache.
  */
 void
 kauth_cred_ref(kauth_cred_t cred)
@@ -1923,31 +3490,49 @@ kauth_cred_ref(kauth_cred_t cred)
 	
 	NULLCRED_CHECK(cred);
 
-	old_value = OSAddAtomic(1, &cred->cr_ref);
+	// XXX SInt32 not safe for an LP64 kernel
+	old_value = OSAddAtomic(1, (SInt32 *)&cred->cr_ref);
 
 	if (old_value < 1)
 		panic("kauth_cred_ref: trying to take a reference on a cred with no references");
+
+#if 0 // use this to watch a specific credential
+	if ( is_target_cred( cred ) != 0 ) {
+ 		get_backtrace( );
+	}
+#endif
 		
 	return;
 }
 
-/*
- * Drop a reference from the passed credential, potentially destroying it.
- *
- * Note:	Assumes credential hash is NOT locked
- */
-void
-kauth_cred_unref(kauth_cred_t *credp)
-{
-	KAUTH_CRED_HASH_LOCK();
-	kauth_cred_unref_hashlocked(credp);
-	KAUTH_CRED_HASH_UNLOCK();
-}
 
 /*
- * Drop a reference from the passed credential, potentially destroying it.
+ * kauth_cred_unref_hashlocked
+ *
+ * Description:	release a credential reference; when the last reference is
+ *		released, the credential will be freed.
+ *
+ * Parameters:	credp				Pointer to address containing
+ *						credential to be freed
+ *
+ * Returns:	(void)
  *
- * Note:	Assumes credential hash IS locked
+ * Implicit returns:
+ *		*credp				Set to NOCRED
+ *
+ * Notes:	This function assumes the credential hash lock is held.
+ *
+ *		This function is internal use only, since the hash lock is
+ *		scoped to this compilation unit.
+ *
+ *		This function destroys the contents of the pointer passed by
+ *		the caller to prevent the caller accidently attempting to
+ *		release a given reference twice in error.
+ *
+ *		The last reference is considered to be released when a release
+ *		of a credential of a reference count of 2 occurs; this is an
+ *		intended effect, to take into accout the reference held by
+ *		the credential hash, which is released at the same time.
  */
 static void
 kauth_cred_unref_hashlocked(kauth_cred_t *credp)
@@ -1956,36 +3541,130 @@ kauth_cred_unref_hashlocked(kauth_cred_t *credp)
 
 	KAUTH_CRED_HASH_LOCK_ASSERT();
 	NULLCRED_CHECK(*credp);
-	old_value = OSAddAtomic(-1, &(*credp)->cr_ref);
+
+	// XXX SInt32 not safe for an LP64 kernel
+	old_value = OSAddAtomic(-1, (SInt32 *)&(*credp)->cr_ref);
 
 #if DIAGNOSTIC
 	if (old_value == 0)
-		panic("%s:0x%08x kauth_cred_rele: dropping a reference on a cred with no references", current_proc()->p_comm, *credp);
+		panic("%s:0x%08x kauth_cred_unref_hashlocked: dropping a reference on a cred with no references", current_proc()->p_comm, *credp);
 	if (old_value == 1)
-		panic("%s:0x%08x kauth_cred_rele: dropping a reference on a cred with no hash entry", current_proc()->p_comm, *credp);
+		panic("%s:0x%08x kauth_cred_unref_hashlocked: dropping a reference on a cred with no hash entry", current_proc()->p_comm, *credp);
+#endif
+
+#if 0 // use this to watch a specific credential
+	if ( is_target_cred( *credp ) != 0 ) {
+		get_backtrace( );
+	}
 #endif
 
+	/*
+	 * If the old_value is 2, then we have just released the last external
+	 * reference to this credential
+	 */
 	if (old_value < 3) {
-		/* the last reference is our credential hash table */
+		/* The last absolute reference is our credential hash table */
 		kauth_cred_remove(*credp);
 	}
 	*credp = NOCRED;
 }
 
+
+/*
+ * kauth_cred_unref
+ *
+ * Description:	Release a credential reference while holding the credential
+ *		hash lock; when the last reference is released, the credential
+ *		will be freed.
+ *
+ * Parameters:	credp				Pointer to address containing
+ *						credential to be freed
+ *
+ * Returns:	(void)
+ *
+ * Implicit returns:
+ *		*credp				Set to NOCRED
+ *
+ * Notes:	See kauth_cred_unref_hashlocked() for more information.
+ *
+ */
+void
+kauth_cred_unref(kauth_cred_t *credp)
+{
+	KAUTH_CRED_HASH_LOCK();
+	kauth_cred_unref_hashlocked(credp);
+	KAUTH_CRED_HASH_UNLOCK();
+}
+
+
+/*
+ * kauth_cred_rele
+ *
+ * Description:	release a credential reference; when the last reference is
+ *		released, the credential will be freed
+ *
+ * Parameters:	cred				Credential to release
+ *
+ * Returns:	(void)
+ *
+ * DEPRECATED:	This interface is obsolete due to a failure to clear out the
+ *		clear the pointer in the caller to avoid multiple releases of
+ *		the same credential.  The currently recommended interface is
+ *		kauth_cred_unref().
+ */
 void
 kauth_cred_rele(kauth_cred_t cred)
 {
 	kauth_cred_unref(&cred);
 }
 
+
 /*
- * Duplicate a credential.
- * 	NOTE - caller should call kauth_cred_add after any credential changes are made.
+ * kauth_cred_dup
+ *
+ * Description:	Duplicate a credential via alloc and copy; the new credential
+ *		has only it's own
+ *
+ * Parameters:	cred				The credential to duplicate
+ *
+ * Returns:	(kauth_cred_t)			The duplicate credential
+ *
+ * Notes:	The typical value to calling this routine is if you are going
+ *		to modify an existing credential, and expect to need a new one
+ *		from the hash cache.
+ *
+ *		This should probably not be used in the majority of cases;
+ *		if you are using it instead of kauth_cred_create(), you are
+ *		likely making a mistake.
+ *
+ *		The newly allocated credential is copied as part of the
+ *		allocation process, with the exception of the reference
+ *		count, which is set to 1 to indicate a single reference
+ *		held by the caller.
+ *
+ *		Since newly allocated credentials have no external pointers
+ *		referencing them, prior to making them visible in an externally
+ *		visible pointer (e.g. by adding them to the credential hash
+ *		cache) is the only legal time in which an existing credential
+ *		can be safely iinitialized or modified directly.
+ *
+ *		After initialization, the caller is expected to call the
+ *		function kauth_cred_add() to add the credential to the hash
+ *		cache, after which time it's frozen and becomes publically
+ *		visible.
+ *
+ *		The release protocol depends on kauth_hash_add() being called
+ *		before kauth_cred_rele() (there is a diagnostic panic which
+ *		will trigger if this protocol is not observed).
+ *
  */
 kauth_cred_t
 kauth_cred_dup(kauth_cred_t cred)
 {
 	kauth_cred_t newcred;
+#if CONFIG_MACF
+	struct label *temp_label;
+#endif
 	
 #if CRED_DIAGNOSTIC
 	if (cred == NOCRED || cred == FSCRED)
@@ -1993,16 +3672,35 @@ kauth_cred_dup(kauth_cred_t cred)
 #endif
 	newcred = kauth_cred_alloc();
 	if (newcred != NULL) {
+#if CONFIG_MACF
+		temp_label = newcred->cr_label;
+#endif
 		bcopy(cred, newcred, sizeof(*newcred));
+#if CONFIG_MACF
+		newcred->cr_label = temp_label;
+		mac_cred_label_associate(cred, newcred);
+#endif
 		newcred->cr_ref = 1;
 	}
 	return(newcred);
 }
 
 /*
- * Returns a credential based on the passed credential but which
- * reflects the real rather than effective UID and GID.
- * NOTE - we do NOT decrement cred reference count on passed in credential
+ * kauth_cred_copy_real
+ *
+ * Description:	Returns a credential based on the passed credential but which
+ *		reflects the real rather than effective UID and GID.
+ *
+ * Parameters:	cred				The credential from which to
+ *						derive the new credential
+ *
+ * Returns:	(kauth_cred_t)			The copied credential
+ *
+ * IMPORTANT:	This function DOES NOT utilize kauth_cred_update(); as a
+ *		result, the caller is responsible for dropping BOTH the
+ *		additional reference on the passed cred (if any), and the
+ *		credential returned by this function.  The drop should be
+ *		via the satnadr kauth_cred_unref() KPI.
  */
 kauth_cred_t
 kauth_cred_copy_real(kauth_cred_t cred)
@@ -2017,14 +3715,19 @@ kauth_cred_copy_real(kauth_cred_t cred)
 		return(cred);
 	}
 
-	/* look up in cred hash table to see if we have a matching credential
-	 * with new values.
+	/*
+	 * Look up in cred hash table to see if we have a matching credential
+	 * with the new values.
 	 */
 	bcopy(cred, &temp_cred, sizeof(temp_cred));
 	temp_cred.cr_uid = cred->cr_ruid;
-	temp_cred.cr_groups[0] = cred->cr_rgid;
+	/* displacing a supplementary group opts us out of memberd */
+	if (kauth_cred_change_egid(&temp_cred, cred->cr_rgid)) {
+		temp_cred.cr_flags |= CRF_NOMEMBERD;
+		temp_cred.cr_gmuid = KAUTH_UID_NONE;
+	}
 	/*
-	 * if the cred is not opted out, make sure we are using the r/euid
+	 * If the cred is not opted out, make sure we are using the r/euid
 	 * for group checks
 	 */
 	if (temp_cred.cr_gmuid != KAUTH_UID_NONE)
@@ -2042,47 +3745,74 @@ kauth_cred_copy_real(kauth_cred_t cred)
 		}
 		if (found_cred != NULL) {
 			/*
-			 * found a match so we bump reference count on new one.
-			 * we leave the old one alone.
+			 * Found a match so we bump reference count on new
+			 * one.  We leave the old one alone.
 			 */
 			kauth_cred_ref(found_cred);
 			KAUTH_CRED_HASH_UNLOCK();
 			return(found_cred);
 		}
 	
-		/* must allocate a new credential, copy in old credential data and update
-		 * with real user and group IDs.
+		/*
+		 * Must allocate a new credential, copy in old credential
+		 * data and update the real user and group IDs.
 		 */
 		newcred = kauth_cred_dup(&temp_cred);
 		err = kauth_cred_add(newcred);
 		KAUTH_CRED_HASH_UNLOCK();
 
-		/* retry if kauth_cred_add returns non zero value */
+		/* Retry if kauth_cred_add() fails */
 		if (err == 0)
 			break;
-		FREE(newcred, M_CRED);
+#if CONFIG_MACF
+		mac_cred_label_destroy(newcred);
+#endif
+		FREE_ZONE(newcred, sizeof(*newcred), M_CRED);
 		newcred = NULL;
 	}
 	
 	return(newcred);
 }
-	
+
+
 /*
- * Common code to update a credential.  model_cred is a temporary, non
- * reference counted credential used only for comparison and modeling
- * purposes.  old_cred is a live reference counted credential that we
- * intend to update using model_cred as our model.
+ * kauth_cred_update
  *
- * IMPORTANT:	If the old_cred ends up updated by this process, we will, as
- *		a side effect, drop the reference we held on it going in.
+ * Description:	Common code to update a credential
+ *
+ * Parameters:	old_cred			Reference counted credential
+ *						to update
+ *		model_cred			Non-reference counted model
+ *						credential to apply to the
+ *						credential to be updated
+ *		retain_auditinfo		Flag as to whether or not the
+ *						audit information should be
+ *						copied from the old_cred into
+ *						the model_cred
+ *
+ * Returns:	(kauth_cred_t)			The updated credential
+ *
+ * IMPORTANT:	This function will potentially return a credential other than
+ *		the one it is passed, and if so, it will have dropped the
+ *		reference on the passed credential.  All callers should be
+ *		aware of this, and treat this function as an unref + ref,
+ *		potentially on different credentials.
+ *
+ *		Because of this, the caller is expected to take its own
+ *		reference on the credential passed as the first parameter,
+ *		and be prepared to release the reference on the credential
+ *		that is returned to them, if it is not intended to be a
+ *		persistent reference.
  */
 static kauth_cred_t
-kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t retain_auditinfo)
+kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred,
+	boolean_t retain_auditinfo)
 {	
 	kauth_cred_t found_cred, new_cred = NULL;
 	
-	/* make sure we carry the auditinfo forward to the new credential unless
-	 * we are actually updating the auditinfo.
+	/*
+	 * Make sure we carry the auditinfo forward to the new credential
+	 * unless we are actually updating the auditinfo.
 	 */
 	if (retain_auditinfo)
 		bcopy(&old_cred->cr_au, &model_cred->cr_au, sizeof(model_cred->cr_au));
@@ -2098,8 +3828,9 @@ kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t reta
 			return(old_cred); 
 		}
 		if (found_cred != NULL) {
+			DEBUG_CRED_CHANGE("kauth_cred_update(cache hit): %p -> %p\n", old_cred, found_cred);
 			/*
-			 * found a match so we bump reference count on new
+			 * Found a match so we bump reference count on new
 			 * one and decrement reference count on the old one.
 			 */
 			kauth_cred_ref(found_cred);
@@ -2108,7 +3839,8 @@ kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t reta
 			return(found_cred);
 		}
 	
-		/* must allocate a new credential using the model.  also
+		/*
+		 * Must allocate a new credential using the model.  also
 		 * adds the new credential to the credential hash table.
 		 */
 		new_cred = kauth_cred_dup(model_cred);
@@ -2118,18 +3850,36 @@ kauth_cred_update(kauth_cred_t old_cred, kauth_cred_t model_cred, boolean_t reta
 		/* retry if kauth_cred_add returns non zero value */
 		if (err == 0)
 			break;
-		FREE(new_cred, M_CRED);
+#if CONFIG_MACF
+		mac_cred_label_destroy(new_cred);
+#endif
+		FREE_ZONE(new_cred, sizeof(*new_cred), M_CRED);
 		new_cred = NULL;
 	}
 
+	DEBUG_CRED_CHANGE("kauth_cred_update(cache miss): %p -> %p\n", old_cred, new_cred);
 	kauth_cred_unref(&old_cred);
 	return(new_cred);
 }
 
-/* 
- *	Add the given credential to our credential hash table and take an additional
- *	reference to account for our use of the credential in the hash table.
- *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
+
+/*
+ * kauth_cred_add
+ *
+ * Description:	Add the given credential to our credential hash table and
+ *		take an additional reference to account for our use of the
+ *		credential in the hash table
+ *
+ * Parameters:	new_cred			Credential to insert into cred
+ *						hash cache
+ *
+ * Returns:	0				Success
+ *		-1				Hash insertion failed: caller
+ *						should retry
+ *
+ * Locks:	Caller is expected to hold KAUTH_CRED_HASH_LOCK
+ *
+ * Notes:	The 'new_cred' MUST NOT already be in the cred hash cache
  */
 static int
 kauth_cred_add(kauth_cred_t new_cred)
@@ -2161,8 +3911,21 @@ kauth_cred_add(kauth_cred_t new_cred)
 
 
 /* 
- *	Remove the given credential from our credential hash table.
- *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
+ * kauth_cred_remove
+ *
+ * Description:	Remove the given credential from our credential hash table
+ *
+ * Parameters:	cred				Credential to remove from cred
+ *						hash cache
+ *
+ * Returns:	(void)
+ *
+ * Locks:	Caller is expected to hold KAUTH_CRED_HASH_LOCK
+ *
+ * Notes:	The check for the reference increment after entry is generally
+ *		agree to be safe, since we use atomic operations, and the
+ *		following code occurs with the hash lock held; in theory, this
+ *		protects us from the 2->1 reference that gets us here.
  */
 static void
 kauth_cred_remove(kauth_cred_t cred)
@@ -2170,24 +3933,25 @@ kauth_cred_remove(kauth_cred_t cred)
 	u_long			hash_key;
 	kauth_cred_t	found_cred;
 
-	KAUTH_CRED_HASH_LOCK_ASSERT();
-
 	hash_key = kauth_cred_get_hashkey(cred);
 	hash_key %= kauth_cred_table_size;
 
-	/* avoid race */
+	/* Avoid race */
 	if (cred->cr_ref < 1)
 		panic("cred reference underflow");
-	if (cred->cr_ref > 1) {
+	if (cred->cr_ref > 1)
 		return;		/* someone else got a ref */
-	}
-
-	/* find cred in the credential hash table */
+		
+	/* Find cred in the credential hash table */
 	TAILQ_FOREACH(found_cred, &kauth_cred_table_anchor[hash_key], cr_link) {
 		if (found_cred == cred) {
 			/* found a match, remove it from the hash table */
 			TAILQ_REMOVE(&kauth_cred_table_anchor[hash_key], found_cred, cr_link);
-			FREE(cred, M_CRED);
+#if CONFIG_MACF
+			mac_cred_label_destroy(cred);
+#endif
+			cred->cr_ref = 0;
+			FREE_ZONE(cred, sizeof(*cred), M_CRED);
 #if KAUTH_CRED_HASH_DEBUG
 			kauth_cred_count--;
 #endif
@@ -2195,22 +3959,33 @@ kauth_cred_remove(kauth_cred_t cred)
 		}
 	}
 
-	/* did not find a match.  this should not happen! */
-	printf("%s:%s - %d - %s - did not find a match for 0x%08x\n", __FILE__, __LINE__, __FUNCTION__, current_proc()->p_comm, cred);
+	/* Did not find a match... this should not happen! XXX Make panic? */
+	printf("%s:%d - %s - %s - did not find a match for %p\n", __FILE__, __LINE__, __FUNCTION__, current_proc()->p_comm, cred);
 	return;
 }
 
+
 /* 
- *	Using the given credential data, look for a match in our credential hash
- *	table.
- *	NOTE - expects caller to hold KAUTH_CRED_HASH_LOCK!
+ * kauth_cred_find
+ *
+ * Description:	Using the given credential data, look for a match in our
+ *		credential hash table
+ *
+ * Parameters:	cred				Credential to lookup in cred
+ *						hash cache
+ *
+ * Returns:	NULL				Not found
+ *		!NULL				Matching cedential already in
+ *						cred hash cache
+ *
+ * Locks:	Caller is expected to hold KAUTH_CRED_HASH_LOCK
  */
 kauth_cred_t
 kauth_cred_find(kauth_cred_t cred)
 {
 	u_long			hash_key;
 	kauth_cred_t	found_cred;
-	
+
 	KAUTH_CRED_HASH_LOCK_ASSERT();
 
 #if KAUTH_CRED_HASH_DEBUG
@@ -2225,35 +4000,52 @@ kauth_cred_find(kauth_cred_t cred)
 	hash_key = kauth_cred_get_hashkey(cred);
 	hash_key %= kauth_cred_table_size;
 
-	/* find cred in the credential hash table */
+	/* Find cred in the credential hash table */
 	TAILQ_FOREACH(found_cred, &kauth_cred_table_anchor[hash_key], cr_link) {
-		if (bcmp(&found_cred->cr_uid, &cred->cr_uid, (sizeof(struct ucred) - offsetof(struct ucred, cr_uid))) == 0) {
+		boolean_t match;
+
+		/*
+		 * don't worry about the label unless the flags in
+		 * either credential tell us to.
+		 */
+		if ((found_cred->cr_flags & CRF_MAC_ENFORCE) != 0 ||
+		    (cred->cr_flags & CRF_MAC_ENFORCE) != 0) {
+			/* include the label pointer in the compare */
+			match = (bcmp(&found_cred->cr_uid, &cred->cr_uid,
+				 (sizeof(struct ucred) -
+				  offsetof(struct ucred, cr_uid))) == 0);
+		} else {
+			/* flags have to match, but skip the label in bcmp */
+			match = (found_cred->cr_flags == cred->cr_flags &&
+				 bcmp(&found_cred->cr_uid, &cred->cr_uid,
+				      (offsetof(struct ucred, cr_label) -
+				       offsetof(struct ucred, cr_uid))) == 0);
+		}
+		if (match) {
 			/* found a match */
 			return(found_cred);
 		}
 	}
-	/* no match found */
+	/* No match found */
+
 	return(NULL);
 }
 
-/*
- * Generates a hash key using data that makes up a credential.  Based on ElfHash.
- */
-static u_long
-kauth_cred_get_hashkey(kauth_cred_t cred)
-{
-	u_long	hash_key = 0;
-	
-	hash_key = kauth_cred_hash((uint8_t *)&cred->cr_uid, 
-				   (sizeof(struct ucred) - offsetof(struct ucred, cr_uid)), 
-							   hash_key);
-	return(hash_key);
-}
 
 /*
- * Generates a hash key using data that makes up a credential.  Based on ElfHash.
+ * kauth_cred_hash
+ *
+ * Description:	Generates a hash key using data that makes up a credential;
+ *		based on ElfHash
+ *
+ * Parameters:	datap				Pointer to data to hash
+ *		data_len			Count of bytes to hash
+ *		start_key			Start key value
+ *
+ * Returns:	(u_long)			Returned hash key
  */
-static inline u_long kauth_cred_hash(const uint8_t *datap, int data_len, u_long start_key)
+static inline u_long
+kauth_cred_hash(const uint8_t *datap, int data_len, u_long start_key)
 {
 	u_long	hash_key = start_key;
 	u_long	temp;
@@ -2270,8 +4062,52 @@ static inline u_long kauth_cred_hash(const uint8_t *datap, int data_len, u_long
 	return(hash_key);
 }
 
+
+/*
+ * kauth_cred_get_hashkey
+ *
+ * Description:	Generate a hash key using data that makes up a credential;
+ *		based on ElfHash.  We hash on the entire credential data,
+ *		not including the ref count or the TAILQ, which are mutable;
+ *		everything else isn't.
+ *
+ *		We also avoid the label (if the flag is not set saying the
+ *		label is actually enforced).
+ *
+ * Parameters:	cred				Credential for which hash is
+ *						desired
+ *
+ * Returns:	(u_long)			Returned hash key
+ */
+static u_long
+kauth_cred_get_hashkey(kauth_cred_t cred)
+{
+	u_long	hash_key = 0;
+	
+	hash_key = kauth_cred_hash((uint8_t *)&cred->cr_uid, 
+			((cred->cr_flags & CRF_MAC_ENFORCE) ? 
+			    sizeof(struct ucred) : offsetof(struct ucred, cr_label)) -
+			    offsetof(struct ucred, cr_uid),
+			hash_key);
+	return(hash_key);
+}
+
+
 #if KAUTH_CRED_HASH_DEBUG
-static void kauth_cred_hash_print(void) 
+/*
+ * kauth_cred_hash_print
+ *
+ * Description:	Print out cred hash cache table information for debugging
+ *		purposes, including the credential contents
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Implicit returns:	Results in console output
+ */
+static void
+kauth_cred_hash_print(void) 
 {
 	int 			i, j;
 	kauth_cred_t	found_cred;
@@ -2294,21 +4130,289 @@ static void kauth_cred_hash_print(void)
 		}
 	}
 }
+#endif	/* KAUTH_CRED_HASH_DEBUG */
 
 
-static void kauth_cred_print(kauth_cred_t cred) 
+#if (defined(KAUTH_CRED_HASH_DEBUG) && (KAUTH_CRED_HASH_DEBUG != 0)) || defined(DEBUG_CRED)
+/*
+ * kauth_cred_print
+ *
+ * Description:	Print out an individual credential's contents for debugging
+ *		purposes
+ *
+ * Parameters:	cred				The credential to print out
+ *
+ * Returns:	(void)
+ *
+ * Implicit returns:	Results in console output
+ */
+void
+kauth_cred_print(kauth_cred_t cred) 
 {
 	int 	i;
-	
-	printf("0x%02X - refs %d uids %d %d %d ", cred, cred->cr_ref, cred->cr_uid, cred->cr_ruid, cred->cr_svuid);
+
+	printf("%p - refs %lu flags 0x%08x uids e%d r%d sv%d gm%d ", cred, cred->cr_ref, cred->cr_flags, cred->cr_uid, cred->cr_ruid, cred->cr_svuid, cred->cr_gmuid);
 	printf("group count %d gids ", cred->cr_ngroups);
 	for (i = 0; i < NGROUPS; i++) {
+		if (i == 0)
+			printf("e");
 		printf("%d ", cred->cr_groups[i]);
 	}
-	printf("%d %d %d ", cred->cr_rgid, cred->cr_svgid, cred->cr_gmuid);
-	printf("auditinfo %d %d %d %d %d %d ", 
+	printf("r%d sv%d ", cred->cr_rgid, cred->cr_svgid);
+	printf("auditinfo %d %d %d %d %d %d\n", 
 		cred->cr_au.ai_auid, cred->cr_au.ai_mask.am_success, cred->cr_au.ai_mask.am_failure, 
 		cred->cr_au.ai_termid.port, cred->cr_au.ai_termid.machine, cred->cr_au.ai_asid);
+}
+
+int is_target_cred( kauth_cred_t the_cred )
+{
+	if ( the_cred->cr_uid != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_ruid != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_svuid != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_ngroups != 11 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[0] != 11 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[1] != 81 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[2] != 63947 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[3] != 80288 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[4] != 89006 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[5] != 52173 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[6] != 84524 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[7] != 79 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[8] != 80292 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[9] != 80 ) 
+		return( 0 );
+	if ( the_cred->cr_groups[10] != 90824 ) 
+		return( 0 );
+	if ( the_cred->cr_rgid != 11 ) 
+		return( 0 );
+	if ( the_cred->cr_svgid != 11 ) 
+		return( 0 );
+	if ( the_cred->cr_gmuid != 3475 ) 
+		return( 0 );
+	if ( the_cred->cr_au.ai_auid != 3475 ) 
+		return( 0 );
+/*
+	if ( the_cred->cr_au.ai_mask.am_success != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_au.ai_mask.am_failure != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_au.ai_termid.port != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_au.ai_termid.machine != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_au.ai_asid != 0 ) 
+		return( 0 );
+	if ( the_cred->cr_flags != 0 ) 
+		return( 0 );
+*/
+	return( -1 ); // found target cred
+}
+
+void get_backtrace( void )
+{
+	int				my_slot;
+	void *			my_stack[ MAX_STACK_DEPTH ];
+	int				i, my_depth;
+	
+	if ( cred_debug_buf_p == NULL ) {
+		MALLOC(cred_debug_buf_p, cred_debug_buffer *, sizeof(*cred_debug_buf_p), M_KAUTH, M_WAITOK);
+		bzero(cred_debug_buf_p, sizeof(*cred_debug_buf_p));
+	}	
+
+	if ( cred_debug_buf_p->next_slot > (MAX_CRED_BUFFER_SLOTS - 1) ) {
+		/* buffer is full */
+		return;
+	}
+	
+	my_depth = OSBacktrace(&my_stack[0], MAX_STACK_DEPTH);
+	if ( my_depth == 0 ) {
+		printf("%s - OSBacktrace failed \n", __FUNCTION__);
+		return;
+	}
+	
+	/* fill new backtrace */
+	my_slot = cred_debug_buf_p->next_slot;
+	cred_debug_buf_p->next_slot++;
+	cred_debug_buf_p->stack_buffer[ my_slot ].depth = my_depth;
+	for ( i = 0; i < my_depth; i++ ) {
+		cred_debug_buf_p->stack_buffer[ my_slot ].stack[ i ] = my_stack[ i ];
+	}
+
+	return;
+}
+
+
+/* subset of struct ucred for use in sysctl_dump_creds */
+struct debug_ucred {
+	void	*credp;
+	u_long	cr_ref;				/* reference count */
+	uid_t	cr_uid;				/* effective user id */
+	uid_t	cr_ruid;			/* real user id */
+	uid_t	cr_svuid;			/* saved user id */
+	short	cr_ngroups;			/* number of groups in advisory list */
+	gid_t	cr_groups[NGROUPS];	/* advisory group list */
+	gid_t	cr_rgid;			/* real group id */
+	gid_t	cr_svgid;			/* saved group id */
+	uid_t	cr_gmuid;			/* UID for group membership purposes */
+	struct auditinfo cr_au;		/* user auditing data */
+	void	*cr_label;			/* MACF label */
+	int		cr_flags;			/* flags on credential */
+};
+typedef struct debug_ucred debug_ucred;
+
+SYSCTL_PROC(_kern, OID_AUTO, dump_creds, CTLFLAG_RD,
+    NULL, 0, sysctl_dump_creds, "S,debug_ucred", "List of credentials in the cred hash");
+
+/*	accessed by:      
+ *	err = sysctlbyname( "kern.dump_creds", bufp, &len, NULL, 0 );
+ */
+
+static int
+sysctl_dump_creds( __unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req )
+{
+	int 			i, j, counter = 0;
+	int				error;
+	size_t			space;
+	kauth_cred_t	found_cred;
+	debug_ucred *	cred_listp;
+	debug_ucred *	nextp;
+
+	/* This is a readonly node. */
+	if (req->newptr != USER_ADDR_NULL)
+		return (EPERM);
+
+	/* calculate space needed */
+	for (i = 0; i < kauth_cred_table_size; i++) {
+		TAILQ_FOREACH(found_cred, &kauth_cred_table_anchor[i], cr_link) {
+			counter++;
+		}
+	}
+
+	/* they are querying us so just return the space required. */
+	if (req->oldptr == USER_ADDR_NULL) {
+		counter += 10; // add in some padding;
+		req->oldidx = counter * sizeof(debug_ucred);
+		return 0;
+	}
+
+	MALLOC( cred_listp, debug_ucred *, req->oldlen, M_TEMP, M_WAITOK );
+	if ( cred_listp == NULL ) {
+		return (ENOMEM);
+	}
+	
+	/* fill in creds to send back */
+	nextp = cred_listp;
+	space = 0;
+	for (i = 0; i < kauth_cred_table_size; i++) {
+		TAILQ_FOREACH(found_cred, &kauth_cred_table_anchor[i], cr_link) {
+			nextp->credp = found_cred;
+			nextp->cr_ref = found_cred->cr_ref;
+			nextp->cr_uid = found_cred->cr_uid;
+			nextp->cr_ruid = found_cred->cr_ruid;
+			nextp->cr_svuid = found_cred->cr_svuid;
+			nextp->cr_ngroups = found_cred->cr_ngroups;
+			for ( j = 0; j < nextp->cr_ngroups; j++ ) {
+				nextp->cr_groups[ j ] = found_cred->cr_groups[ j ];
+			}
+			nextp->cr_rgid = found_cred->cr_rgid;
+			nextp->cr_svgid = found_cred->cr_svgid;
+			nextp->cr_gmuid = found_cred->cr_gmuid;
+			nextp->cr_au.ai_auid = found_cred->cr_au.ai_auid;
+			nextp->cr_au.ai_mask.am_success = found_cred->cr_au.ai_mask.am_success;
+			nextp->cr_au.ai_mask.am_failure = found_cred->cr_au.ai_mask.am_failure;
+			nextp->cr_au.ai_termid.port = found_cred->cr_au.ai_termid.port;
+			nextp->cr_au.ai_termid.machine = found_cred->cr_au.ai_termid.machine;
+			nextp->cr_au.ai_asid = found_cred->cr_au.ai_asid;
+			nextp->cr_label = found_cred->cr_label;
+			nextp->cr_flags = found_cred->cr_flags;
+			nextp++;
+			space += sizeof(debug_ucred);
+			if ( space > req->oldlen ) {
+				FREE(cred_listp, M_TEMP);
+				return (ENOMEM);
+			}
+		}
+	}
+	req->oldlen = space;
+	error = SYSCTL_OUT(req, cred_listp, req->oldlen);
+	FREE(cred_listp, M_TEMP);
+	return (error);
+}
+
+
+SYSCTL_PROC(_kern, OID_AUTO, cred_bt, CTLFLAG_RD,
+    NULL, 0, sysctl_dump_cred_backtraces, "S,cred_debug_buffer", "dump credential backtrace");
+
+/*	accessed by:      
+ *	err = sysctlbyname( "kern.cred_bt", bufp, &len, NULL, 0 );
+ */
+
+static int
+sysctl_dump_cred_backtraces( __unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req )
+{
+	int 			i, j;
+	int				error;
+	size_t			space;
+	cred_debug_buffer *	bt_bufp;
+	cred_backtrace *	nextp;
+
+	/* This is a readonly node. */
+	if (req->newptr != USER_ADDR_NULL)
+		return (EPERM);
+
+	if ( cred_debug_buf_p == NULL ) {
+		return (EAGAIN);
+	}
+
+	/* calculate space needed */
+	space = sizeof( cred_debug_buf_p->next_slot );
+	space += (sizeof( cred_backtrace ) * cred_debug_buf_p->next_slot);
+
+	/* they are querying us so just return the space required. */
+	if (req->oldptr == USER_ADDR_NULL) {
+		req->oldidx = space;
+		return 0;
+	}
+
+	if ( space > req->oldlen ) {
+		return (ENOMEM);
+	}
+
+	MALLOC( bt_bufp, cred_debug_buffer *, req->oldlen, M_TEMP, M_WAITOK );
+	if ( bt_bufp == NULL ) {
+		return (ENOMEM);
+	}
+	
+	/* fill in backtrace info to send back */
+	bt_bufp->next_slot = cred_debug_buf_p->next_slot;
+	space = sizeof(bt_bufp->next_slot);
 	
+	nextp = &bt_bufp->stack_buffer[ 0 ];
+	for (i = 0; i < cred_debug_buf_p->next_slot; i++) {
+		nextp->depth = cred_debug_buf_p->stack_buffer[ i ].depth;
+		for ( j = 0; j < nextp->depth; j++ ) {
+			nextp->stack[ j ] = cred_debug_buf_p->stack_buffer[ i ].stack[ j ];
+		}
+		space += sizeof(*nextp);
+		nextp++;
+	}
+	req->oldlen = space;
+	error = SYSCTL_OUT(req, bt_bufp, req->oldlen);
+	FREE(bt_bufp, M_TEMP);
+	return (error);
 }
-#endif
+
+#endif	/* KAUTH_CRED_HASH_DEBUG || DEBUG_CRED */
diff --git a/bsd/kern/kern_descrip.c b/bsd/kern/kern_descrip.c
index abeb1ed2d..d44744be2 100644
--- a/bsd/kern/kern_descrip.c
+++ b/bsd/kern/kern_descrip.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,12 @@
  *
  *	@(#)kern_descrip.c	8.8 (Berkeley) 2/14/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -92,28 +104,41 @@
 #include <kern/kalloc.h>
 #include <libkern/OSAtomic.h>
 
+#include <sys/ubc.h>
+
 struct psemnode;
 struct pshmnode;
 
-int fdopen(dev_t dev, int mode, int type, struct proc *p);
-int ogetdtablesize(struct proc *p, void *uap, register_t *retval);
-int finishdup(struct proc * p, struct filedesc *fdp, int old, int new, register_t *retval);
+int fdopen(dev_t dev, int mode, int type, proc_t p);
+int finishdup(proc_t p, struct filedesc *fdp, int old, int new, register_t *retval);
 
-int closef(struct fileglob *fg, struct proc *p);
-int falloc_locked(struct proc *p, struct fileproc **resultfp, int *resultfd, int locked);
-void fddrop(struct proc *p, int fd);
-int fdgetf_noref(struct proc *p, int fd, struct fileproc **resultfp);
+int falloc_locked(proc_t p, struct fileproc **resultfp, int *resultfd, vfs_context_t ctx, int locked);
+int fdgetf_noref(proc_t p, int fd, struct fileproc **resultfp);
 void fg_drop(struct fileproc * fp);
 void fg_free(struct fileglob *fg);
 void fg_ref(struct fileproc * fp);
 
-static int closef_finish(struct fileproc *fp, struct fileglob *fg, struct proc *p);
+/* flags for close_internal_locked */
+#define FD_DUP2RESV 1
+static int close_internal_locked(struct proc *p, int fd, struct fileproc *fp, int flags);
+
+static int closef_finish(struct fileproc *fp, struct fileglob *fg, proc_t p, vfs_context_t ctx);
+
+/* We don't want these exported */
+__private_extern__
+int open1(vfs_context_t, struct nameidata *, int, struct vnode_attr *, register_t *);
+
+__private_extern__
+int unlink1(vfs_context_t, struct nameidata *, int);
+
+static void _fdrelse(struct proc * p, int fd);
 
-extern void file_lock_init(void);
-extern int is_suser(void);
-extern int kqueue_stat(struct fileproc *fp, struct stat *st, struct proc *p);
-extern int soo_stat(struct socket *so, struct stat *ub);
-extern int vn_path_package_check(vnode_t vp, char *path, int pathlen, int *component);
+
+extern void file_lock_init(void) __attribute__((section("__TEXT, initcode")));
+extern int kqueue_stat(struct fileproc *fp, void *ub, int isstat4, proc_t p);
+#if SOCKETS
+extern int soo_stat(struct socket *so, void *ub, int isstat64);
+#endif /* SOCKETS */
 
 extern kauth_scope_t	kauth_scope_fileop;
 
@@ -138,14 +163,23 @@ lck_grp_t * file_lck_grp;
 lck_attr_t * file_lck_attr;
 
 lck_mtx_t * uipc_lock;
-lck_mtx_t * file_iterate_lcok;
 lck_mtx_t * file_flist_lock;
 
 
+/*
+ * file_lock_init
+ *
+ * Description:	Initialize the file lock group and the uipc and flist locks
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	void
+ *
+ * Notes:	Called at system startup from bsd_init().
+ */
 void
 file_lock_init(void)
 {
-
 	/* allocate file lock group attribute and group */
 	file_lck_grp_attr= lck_grp_attr_alloc_init();
 
@@ -155,49 +189,86 @@ file_lock_init(void)
 	file_lck_attr = lck_attr_alloc_init();
 
 	uipc_lock = lck_mtx_alloc_init(file_lck_grp, file_lck_attr);
-	file_iterate_lcok = lck_mtx_alloc_init(file_lck_grp, file_lck_attr);
 	file_flist_lock = lck_mtx_alloc_init(file_lck_grp, file_lck_attr);
+}
 
-	
 
+/*
+ * proc_fdlock, proc_fdlock_spin
+ *
+ * Description:	Lock to control access to the per process struct fileproc
+ *		and struct filedesc
+ *
+ * Parameters:	p				Process to take the lock on
+ *
+ * Returns:	void
+ *
+ * Notes:	The lock is initialized in forkproc() and destroyed in
+ *		reap_child_process().
+ */
+void
+proc_fdlock(proc_t p)
+{
+	lck_mtx_lock(&p->p_fdmlock);
 }
 
+void
+proc_fdlock_spin(proc_t p)
+{
+	lck_mtx_lock_spin(&p->p_fdmlock);
+}
 
 void
-proc_fdlock(struct proc *p)
+proc_fdlock_assert(proc_t p, int assertflags)
 {
-	lck_mtx_lock(&p->p_fdmlock);
+	lck_mtx_assert(&p->p_fdmlock, assertflags);
 }
 
+
+/*
+ * proc_fdunlock
+ *
+ * Description:	Unlock the lock previously locked by a call to proc_fdlock()
+ *
+ * Parameters:	p				Process to drop the lock on
+ *
+ * Returns:	void
+ */
 void
-proc_fdunlock(struct proc *p)
+proc_fdunlock(proc_t p)
 {
 	lck_mtx_unlock(&p->p_fdmlock);
 }
 
+
 /*
  * System calls on descriptors.
  */
 
+
+/*
+ * getdtablesize
+ *
+ * Description:	Returns the per process maximum size of the descriptor table
+ *
+ * Parameters:	p				Process being queried
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *
+ * Implicit returns:
+ *		*retval (modified)		Size of dtable
+ */
 int
-getdtablesize(struct proc *p, __unused struct getdtablesize_args *uap, register_t *retval)
+getdtablesize(proc_t p, __unused struct getdtablesize_args *uap, register_t *retval)
 {
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	*retval = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, maxfiles);
 	proc_fdunlock(p);
 
 	return (0);
 }
 
-int
-ogetdtablesize(struct proc *p, __unused void *uap, register_t *retval)
-{
-	proc_fdlock(p);
-	*retval = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, NOFILE);
-	proc_fdunlock(p);
-
-	return (0);
-}
 
 void
 procfdtbl_reservefd(struct proc * p, int fd)
@@ -245,18 +316,31 @@ procfdtbl_clearfd(struct proc * p, int fd)
 	}
 }
 
-
-static __inline__ void
-_fdrelse(struct proc *p, int fd)
+/*
+ * _fdrelse
+ *
+ * Description:	Inline utility function to free an fd in a filedesc
+ *
+ * Parameters:	fdp				Pointer to filedesc fd lies in
+ *		fd				fd to free
+ *		reserv				fd should be reserved
+ *
+ * Returns:	void
+ *
+ * Locks:	Assumes proc_fdlock for process pointing to fdp is held by
+ *		the caller
+ */
+static void
+_fdrelse(struct proc * p, int fd)
 {
-struct filedesc *fdp = p->p_fd;
-int nfd = 0;
+	struct filedesc *fdp = p->p_fd;
+	int nfd = 0;
 
 	if (fd < fdp->fd_freefile)
 		fdp->fd_freefile = fd;
 #if DIAGNOSTIC
 	if (fd > fdp->fd_lastfile)
-		panic("fdrelse: fd_lastfile inconsistent");
+ 		panic("fdrelse: fd_lastfile inconsistent");
 #endif
 	procfdtbl_clearfd(p, fd);
 
@@ -266,18 +350,27 @@ int nfd = 0;
 		fdp->fd_lastfile--;
 }
 
+
 /*
- * Duplicate a file descriptor.
+ * dup
+ *
+ * Description:	Duplicate a file descriptor.
+ *
+ * Parameters:	p				Process performing the dup
+ *		uap->fd				The fd to dup
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		!0				Errno
+ *
+ * Implicit returns:
+ *		*retval (modified)		The new descriptor
  */
-/* ARGSUSED */
 int
-dup(p, uap, retval)
-	struct proc *p;
-	struct dup_args *uap;
-	register_t *retval;
+dup(proc_t p, struct dup_args *uap, register_t *retval)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int old = uap->fd;
+	struct filedesc *fdp = p->p_fd;
+	int old = uap->fd;
 	int new, error;
 	struct fileproc *fp;
 
@@ -298,18 +391,28 @@ dup(p, uap, retval)
 	return (error);
 }
 
+
 /*
- * Duplicate a file descriptor to a particular value.
+ * dup2
+ *
+ * Description:	Duplicate a file descriptor to a particular value.
+ *
+ * Parameters:	p				Process performing the dup
+ *		uap->fd				The fd to dup
+ *		uap->to				The fd to dup it to
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		!0				Errno
+ *
+ * Implicit returns:
+ *		*retval (modified)		The new descriptor
  */
-/* ARGSUSED */
 int
-dup2(p, uap, retval)
-	struct proc *p;
-	struct dup2_args *uap;
-	register_t *retval;
+dup2(proc_t p, struct dup2_args *uap, register_t *retval)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int old = uap->from, new = uap->to;
+	struct filedesc *fdp = p->p_fd;
+	int old = uap->from, new = uap->to;
 	int i, error;
 	struct fileproc *fp, *nfp;
 
@@ -321,7 +424,7 @@ startover:
 		return(error);
 	}
 	if (new < 0 ||
-		new >= p->p_rlimit[RLIMIT_NOFILE].rlim_cur ||
+		(rlim_t)new >= p->p_rlimit[RLIMIT_NOFILE].rlim_cur ||
 	    new >= maxfiles) {
 		fp_drop(p, old, fp, 1);
 		proc_fdunlock(p);
@@ -340,7 +443,7 @@ startover:
 			return (error);
 		}
 		if (new != i) {
-			_fdrelse(p, i);
+			fdrelse(p, i);
 			goto closeit;
 		}
 	} else {
@@ -348,18 +451,39 @@ closeit:
 		while ((fdp->fd_ofileflags[new] & UF_RESERVED) == UF_RESERVED)  {
 				fp_drop(p, old, fp, 1);
 				procfdtbl_waitfd(p, new);
+#if DIAGNOSTIC
+				proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
 				goto startover;
 		}
 
 		if ((fdp->fd_ofiles[new] != NULL)  && ((error = fp_lookup(p, new, &nfp, 1)) == 0)) {
 			fp_drop(p, old, fp, 1);
-			close_internal(p, new, nfp, CLOSEINT_LOCKED | CLOSEINT_NOFDRELSE);
+			(void)close_internal_locked(p, new, nfp, FD_DUP2RESV);
+#if DIAGNOSTIC
+			proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
 			procfdtbl_clearfd(p, new);
 			goto startover;
 		} else  {
+#if DIAGNOSTIC
+			if (fdp->fd_ofiles[new] != NULL)
+				panic("dup2: unable to get ref on a fileproc %d\n", new);
+#endif
 			procfdtbl_reservefd(p, new);
 		}
+
+#if DIAGNOSTIC
+		proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
+
 	}
+#if DIAGNOSTIC
+	if (fdp->fd_ofiles[new] != 0)
+		panic("dup2-1: overwriting fd_ofiles with new %d\n", new);
+	if ((fdp->fd_ofileflags[new] & UF_RESERVED) == 0)
+		panic("dup2-1: unreserved  fileflags  with new %d\n", new);
+#endif
 	error = finishdup(p, fdp, old, new, retval);
 	fp_drop(p, old, fp, 1);
 	proc_fdunlock(p);
@@ -367,20 +491,97 @@ closeit:
 	return(error);
 }
 
+
+/*
+ * fcntl
+ *
+ * Description:	The file control system call.
+ *
+ * Parameters:	p				Process performing the fcntl
+ *		uap->fd				The fd to operate against
+ *		uap->cmd			The command to perform
+ *		uap->arg			Pointer to the command argument
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		!0				Errno (see fcntl_nocancel)
+ *
+ * Implicit returns:
+ *		*retval (modified)		fcntl return value (if any)
+ *
+ * Notes:	This system call differs from fcntl_nocancel() in that it
+ *		tests for cancellation prior to performing a potentially
+ *		blocking operation.
+ */
+int
+fcntl(proc_t p, struct fcntl_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(fcntl_nocancel(p, (struct fcntl_nocancel_args *)uap, retval));
+}
+
+
 /*
- * The file control system call.
+ * fcntl_nocancel
+ *
+ * Description:	A non-cancel-testing file control system call.
+ *
+ * Parameters:	p				Process performing the fcntl
+ *		uap->fd				The fd to operate against
+ *		uap->cmd			The command to perform
+ *		uap->arg			Pointer to the command argument
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		EINVAL
+ *	fp_lookup:EBADF				Bad file descriptor
+ * [F_DUPFD]
+ *	fdalloc:EMFILE
+ *	fdalloc:ENOMEM
+ *	finishdup:EBADF
+ *	finishdup:ENOMEM
+ * [F_SETOWN]
+ *		ESRCH
+ * [F_SETLK]
+ *		EBADF
+ *		EOVERFLOW
+ *	copyin:EFAULT
+ *	vnode_getwithref:???
+ *	VNOP_ADVLOCK:???
+ * [F_GETLK]
+ *		EBADF
+ *		EOVERFLOW
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	vnode_getwithref:???
+ *	VNOP_ADVLOCK:???
+ * [F_PREALLOCATE]
+ *		EBADF
+ *		EINVAL
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	vnode_getwithref:???
+ *	VNOP_ALLOCATE:???
+ * [F_SETSIZE,F_RDADVISE]
+ *		EBADF
+ *	copyin:EFAULT
+ *	vnode_getwithref:???
+ * [F_RDAHEAD,F_NOCACHE]
+ *		EBADF
+ *	vnode_getwithref:???
+ * [???]
+ *
+ * Implicit returns:
+ *		*retval (modified)		fcntl return value (if any)
  */
 int
-fcntl(p, uap, retval)
-	struct proc *p;
-	struct fcntl_args *uap;
-	register_t *retval;
+fcntl_nocancel(proc_t p, struct fcntl_nocancel_args *uap, register_t *retval)
 {
 	int fd = uap->fd;
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 	char *pop;
-	struct vnode *vp;
+	struct vnode *vp = NULLVP;	/* for AUDIT_ARG() at end */
 	int i, tmp, error, error2, flg = F_POSIX;
 	struct flock fl;
 	struct vfs_context context;
@@ -399,24 +600,33 @@ fcntl(p, uap, retval)
 		proc_fdunlock(p);
 		return(error);
 	}
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = fp->f_cred;
 	if (proc_is64bit(p)) {
 		argp = uap->arg;
 	}
 	else {
-		/* since the arg parameter is defined as a long but may be either
-		 * a long or a pointer we must take care to handle sign extension 
-		 * issues.  Our sys call munger will sign extend a long when we are
-		 * called from a 32-bit process.  Since we can never have an address
-		 * greater than 32-bits from a 32-bit process we lop off the top 
-		 * 32-bits to avoid getting the wrong address
+		/*
+		 * Since the arg parameter is defined as a long but may be
+		 * either a long or a pointer we must take care to handle
+		 * sign extension issues.  Our sys call munger will sign
+		 * extend a long when we are called from a 32-bit process.
+		 * Since we can never have an address greater than 32-bits
+		 * from a 32-bit process we lop off the top 32-bits to avoid
+		 * getting the wrong address
 		 */
 		argp = CAST_USER_ADDR_T(uap->arg);
 	}
 
 	pop = &fdp->fd_ofileflags[fd];
 
+#if CONFIG_MACF
+	error = mac_file_check_fcntl(proc_ucred(p), fp->f_fglob, uap->cmd,
+	    uap->arg);
+	if (error)
+		goto out;
+#endif
+
 	switch (uap->cmd) {
 
 	case F_DUPFD:
@@ -452,16 +662,16 @@ fcntl(p, uap, retval)
 		tmp = CAST_DOWN(int, uap->arg);
 		fp->f_flag |= FFLAGS(tmp) & FCNTLFLAGS;
 		tmp = fp->f_flag & FNONBLOCK;
-		error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, p);
+		error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, &context);
 		if (error)
 			goto out;
 		tmp = fp->f_flag & FASYNC;
-		error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, p);
+		error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, &context);
 		if (!error)
 			goto out;
 		fp->f_flag &= ~FNONBLOCK;
 		tmp = 0;
-		(void)fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, p);
+		(void)fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, &context);
 		goto out;
 
 	case F_GETOWN:
@@ -470,7 +680,7 @@ fcntl(p, uap, retval)
 			error = 0;
 			goto out;
 		}
-		error = fo_ioctl(fp, (int)TIOCGPGRP, (caddr_t)retval, p);
+		error = fo_ioctl(fp, (int)TIOCGPGRP, (caddr_t)retval, &context);
 		*retval = -*retval;
 		goto out;
 
@@ -482,21 +692,22 @@ fcntl(p, uap, retval)
 			goto out;
 		}
 		if (fp->f_type == DTYPE_PIPE) {
-			error =  fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, p);
+			error =  fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, &context);
 			goto out;
 		}
 
 		if (tmp <= 0) {
 			tmp = -tmp;
 		} else {
-			struct proc *p1 = pfind(tmp);
+			proc_t p1 = proc_find(tmp);
 			if (p1 == 0) {
 				error = ESRCH;
 				goto out;
 			}
-			tmp = (int)p1->p_pgrp->pg_id;
+			tmp = (int)p1->p_pgrpid;
+			proc_rele(p1);
 		}
-		error =  fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, p);
+		error =  fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, &context);
 		goto out;
 
 	case F_SETLKW:
@@ -515,16 +726,30 @@ fcntl(p, uap, retval)
 		proc_fdunlock(p);
 
 		/* Copy in the lock structure */
-		error = copyin(argp, (caddr_t)&fl, sizeof (fl));
+		error = copyin(argp, (caddr_t)&fl, sizeof(fl));
 		if (error) {
 			goto outdrop;
 		}
+
+		if ((fl.l_whence == SEEK_CUR) && (fl.l_start + offset < fl.l_start)) {
+		    error = EOVERFLOW;
+		    goto outdrop;
+		}
+
 		if ( (error = vnode_getwithref(vp)) ) {
 			goto outdrop;
 		}
 		if (fl.l_whence == SEEK_CUR)
 			fl.l_start += offset;
 
+#if CONFIG_MACF
+		error = mac_file_check_lock(proc_ucred(p), fp->f_fglob,
+		    F_SETLK, &fl);
+		if (error) {
+			(void)vnode_put(vp);
+			goto outdrop;
+		}
+#endif
 		switch (fl.l_type) {
 
 		case F_RDLCK:
@@ -533,7 +758,8 @@ fcntl(p, uap, retval)
 				error = EBADF;
 				goto outdrop;
 			}
-			OSBitOrAtomic(P_LADVLOCK, &p->p_ladvflag);
+			// XXX UInt32 unsafe for LP64 kernel
+			OSBitOrAtomic(P_LADVLOCK, (UInt32 *)&p->p_ladvflag);
 			error = VNOP_ADVLOCK(vp, (caddr_t)p, F_SETLK, &fl, flg, &context);
 			(void)vnode_put(vp);
 			goto outdrop;
@@ -544,7 +770,8 @@ fcntl(p, uap, retval)
 				error = EBADF;
 				goto outdrop;
 			}
-			OSBitOrAtomic(P_LADVLOCK, &p->p_ladvflag);
+			// XXX UInt32 unsafe for LP64 kernel
+			OSBitOrAtomic(P_LADVLOCK, (UInt32 *)&p->p_ladvflag);
 			error = VNOP_ADVLOCK(vp, (caddr_t)p, F_SETLK, &fl, flg, &context);
 			(void)vnode_put(vp);
 			goto outdrop;
@@ -572,20 +799,62 @@ fcntl(p, uap, retval)
 		proc_fdunlock(p);
 
 		/* Copy in the lock structure */
-		error = copyin(argp, (caddr_t)&fl, sizeof (fl));
+		error = copyin(argp, (caddr_t)&fl, sizeof(fl));
 		if (error)
 			goto outdrop;
 
+		/* Check starting byte and ending byte for EOVERFLOW in SEEK_CUR */
+		/* and ending byte for EOVERFLOW in SEEK_SET */
+		if (((fl.l_whence == SEEK_CUR) && 
+		     ((fl.l_start + offset < fl.l_start) ||
+		      ((fl.l_len > 0) && (fl.l_start+offset + fl.l_len - 1 < fl.l_start+offset)))) ||
+		    ((fl.l_whence == SEEK_SET) && (fl.l_len > 0) && (fl.l_start + fl.l_len - 1 < fl.l_start)))
+		{
+			/* lf_advlock doesn't check start/end for F_GETLK if file has no locks */
+			error = EOVERFLOW;
+			goto outdrop;
+		}
+
+		if ((fl.l_whence == SEEK_SET) && (fl.l_start < 0)) {
+			error = EINVAL;
+			goto outdrop;
+		}
+
+		switch (fl.l_type) {
+		case F_RDLCK:
+		case F_UNLCK:
+		case F_WRLCK:
+			break;
+		default:
+			error = EINVAL;
+			goto outdrop;
+		}
+
+		switch (fl.l_whence) {
+		case SEEK_CUR:
+		case SEEK_SET:
+		case SEEK_END:
+			break;
+		default:
+			error = EINVAL;
+			goto outdrop;
+		}
+
 		if ( (error = vnode_getwithref(vp)) == 0 ) {
 			if (fl.l_whence == SEEK_CUR)
 			        fl.l_start += offset;
 
+#if CONFIG_MACF
+			error = mac_file_check_lock(proc_ucred(p), fp->f_fglob,
+			    F_GETLK, &fl);
+			if (error == 0)
+#endif
 			error = VNOP_ADVLOCK(vp, (caddr_t)p, F_GETLK, &fl, F_POSIX, &context);
 
 			(void)vnode_put(vp);
 
 			if (error == 0)
-				error = copyout((caddr_t)&fl, argp, sizeof (fl));
+				error = copyout((caddr_t)&fl, argp, sizeof(fl));
 		}
 		goto outdrop;
 
@@ -607,7 +876,7 @@ fcntl(p, uap, retval)
 			goto outdrop;
 		}
 
-		error = copyin(argp, (caddr_t)&alloc_struct, sizeof (alloc_struct));
+		error = copyin(argp, (caddr_t)&alloc_struct, sizeof(alloc_struct));
 		if (error)
 			goto outdrop;
 
@@ -667,7 +936,7 @@ fcntl(p, uap, retval)
 					      &context);
 			(void)vnode_put(vp);
 
-			error2 = copyout((caddr_t)&alloc_struct, argp, sizeof (alloc_struct));
+			error2 = copyout((caddr_t)&alloc_struct, argp, sizeof(alloc_struct));
 
 			if (error == 0)
 				error = error2;
@@ -680,32 +949,41 @@ fcntl(p, uap, retval)
 			error = EBADF;
 			goto out;
 		}
+		vp = (struct vnode *)fp->f_data;
 		proc_fdunlock(p);
 
 		error = copyin(argp, (caddr_t)&offset, sizeof (off_t));
 		if (error)
 			goto outdrop;
 
+		error = vnode_getwithref(vp);
+		if (error)
+			goto outdrop;
+
+#if CONFIG_MACF
+		error = mac_vnode_check_truncate(&context,
+		    fp->f_fglob->fg_cred, vp);
+		if (error) {
+			(void)vnode_put(vp);
+			goto outdrop;
+		}
+#endif
 		/*
 		 * Make sure that we are root.  Growing a file
 		 * without zero filling the data is a security hole 
 		 * root would have access anyway so we'll allow it
 		 */
-
 		if (!is_suser()) {
 			error = EACCES;
-			goto outdrop;
-		}
-		vp = (struct vnode *)fp->f_data;
-
-		if ( (error = vnode_getwithref(vp)) == 0 ) {
-		        /*
+		} else {
+			/*
 			 * set the file size
 			 */
-		        error = vnode_setsize(vp, offset, IO_NOZEROFILL, &context);
-
-			(void)vnode_put(vp);
+			error = vnode_setsize(vp, offset, IO_NOZEROFILL,
+			    &context);
 		}
+
+		(void)vnode_put(vp);
 		goto outdrop;
 
 	case F_RDAHEAD:
@@ -713,29 +991,38 @@ fcntl(p, uap, retval)
 			error = EBADF;
 			goto out;
 		}
-		vp = (struct vnode *)fp->f_data;
-		proc_fdunlock(p);
-
-		if ( (error = vnode_getwithref(vp)) == 0) {
-		        if (uap->arg)
-			        vnode_clearnoreadahead(vp);
-			else
-			        vnode_setnoreadahead(vp);
+		if (uap->arg)
+		        fp->f_fglob->fg_flag &= ~FNORDAHEAD;
+		else
+		        fp->f_fglob->fg_flag |= FNORDAHEAD;
 
-			(void)vnode_put(vp);
-		}
-		goto outdrop;
+		goto out;
 
 	case F_NOCACHE:
 		if (fp->f_type != DTYPE_VNODE) {
 			error = EBADF;
 			goto out;
 		}
+		if (uap->arg)
+		        fp->f_fglob->fg_flag |= FNOCACHE;
+		else
+		        fp->f_fglob->fg_flag &= ~FNOCACHE;
+
+		goto out;
+
+	case F_GLOBAL_NOCACHE:
+	        if (fp->f_type != DTYPE_VNODE) {
+		        error = EBADF;
+			goto out;
+		}
 		vp = (struct vnode *)fp->f_data;
 		proc_fdunlock(p);
 
 		if ( (error = vnode_getwithref(vp)) == 0 ) {
-		        if (uap->arg)
+
+		        *retval = vnode_isnocache(vp);
+
+			if (uap->arg)
 			        vnode_setnocache(vp);
 			else
 			        vnode_clearnocache(vp);
@@ -744,9 +1031,9 @@ fcntl(p, uap, retval)
 		}
 		goto outdrop;
 
-	case F_GLOBAL_NOCACHE:
-		if (fp->f_type != DTYPE_VNODE) {
-			error = EBADF;
+	case F_CHECK_OPENEVT:
+	        if (fp->f_type != DTYPE_VNODE) {
+		        error = EBADF;
 			goto out;
 		}
 		vp = (struct vnode *)fp->f_data;
@@ -754,12 +1041,12 @@ fcntl(p, uap, retval)
 
 		if ( (error = vnode_getwithref(vp)) == 0 ) {
 
-		        *retval = vnode_isnocache(vp);
+		        *retval = vnode_is_openevt(vp);
 
-		        if (uap->arg)
-			        vnode_setnocache(vp);
+			if (uap->arg)
+			        vnode_set_openevt(vp);
 			else
-			        vnode_clearnocache(vp);
+			        vnode_clear_openevt(vp);
 
 			(void)vnode_put(vp);
 		}
@@ -775,7 +1062,7 @@ fcntl(p, uap, retval)
 		vp = (struct vnode *)fp->f_data;
 		proc_fdunlock(p);
 
-		if ( (error = copyin(argp, (caddr_t)&ra_struct, sizeof (ra_struct))) )
+		if ( (error = copyin(argp, (caddr_t)&ra_struct, sizeof(ra_struct))) )
 			goto outdrop;
 		if ( (error = vnode_getwithref(vp)) == 0 ) {
 		        error = VNOP_IOCTL(vp, F_RDADVISE, (caddr_t)&ra_struct, 0, &context);
@@ -824,7 +1111,8 @@ fcntl(p, uap, retval)
 				goto outdrop;
 			}
 		}
-		if (strcmp(vnode_mount(vp)->mnt_vfsstat.f_fstypename, "hfs") != 0) {
+		if (strncmp(vnode_mount(vp)->mnt_vfsstat.f_fstypename, "hfs",
+			sizeof(vnode_mount(vp)->mnt_vfsstat.f_fstypename)) != 0) {
 			error = EINVAL;
 		} else {
 			/*
@@ -868,7 +1156,7 @@ fcntl(p, uap, retval)
 			l2p_struct.l2p_contigbytes = 0;	/* for now */
 			l2p_struct.l2p_devoffset = bn * devBlockSize;
 			l2p_struct.l2p_devoffset += fp->f_offset - offset;
-			error = copyout((caddr_t)&l2p_struct, argp, sizeof (l2p_struct));
+			error = copyout((caddr_t)&l2p_struct, argp, sizeof(l2p_struct));
 		}
 		goto outdrop;
 		}
@@ -943,66 +1231,364 @@ fcntl(p, uap, retval)
 		}
 		break;
 	}
-	    
-	default:
-		if (uap->cmd < FCNTL_FS_SPECIFIC_BASE) {
-			error = EINVAL;
+
+	/*
+	 * SPI (private) for opening a file starting from a dir fd
+	 */
+	case F_OPENFROM: {
+		struct user_fopenfrom fopen;
+		struct vnode_attr va;
+		struct nameidata nd;
+		int cmode;
+
+		/* Check if this isn't a valid file descriptor */
+		if ((fp->f_type != DTYPE_VNODE) ||
+		    (fp->f_flag & FREAD) == 0) {
+			error = EBADF;
 			goto out;
 		}
+		vp = (struct vnode *)fp->f_data;
+		proc_fdunlock(p);
 
-		// if it's a fs-specific fcntl() then just pass it through
+		if (vnode_getwithref(vp)) {
+			error = ENOENT;
+			goto outdrop;
+		}
+		
+		/* Only valid for directories */
+		if (vp->v_type != VDIR) {
+			vnode_put(vp);
+			error = ENOTDIR;
+			goto outdrop;
+		}
 
-		if (fp->f_type != DTYPE_VNODE) {
+		/* Get flags, mode and pathname arguments. */
+		if (IS_64BIT_PROCESS(p)) {
+			error = copyin(argp, &fopen, sizeof(fopen));
+		} else {
+			struct fopenfrom fopen32;
+
+			error = copyin(argp, &fopen32, sizeof(fopen32));
+			fopen.o_flags = fopen32.o_flags;
+			fopen.o_mode = fopen32.o_mode;
+			fopen.o_pathname = CAST_USER_ADDR_T(fopen32.o_pathname);
+		}
+		if (error) {
+			vnode_put(vp);
+			goto outdrop;
+		}
+		VATTR_INIT(&va);
+		/* Mask off all but regular access permissions */
+		cmode = ((fopen.o_mode &~ fdp->fd_cmask) & ALLPERMS) & ~S_ISTXT;
+		VATTR_SET(&va, va_mode, cmode & ACCESSPERMS);
+
+		/* Start the lookup relative to the file descriptor's vnode. */
+		NDINIT(&nd, LOOKUP, USEDVP | FOLLOW | AUDITVNPATH1, UIO_USERSPACE,
+		       fopen.o_pathname, &context);
+		nd.ni_dvp = vp;
+
+		error = open1(&context, &nd, fopen.o_flags, &va, retval);
+
+		vnode_put(vp);
+		break;
+	}
+	/*
+	 * SPI (private) for unlinking a file starting from a dir fd
+	 */
+	case F_UNLINKFROM: {
+		struct nameidata nd;
+		user_addr_t pathname;
+
+		/* Check if this isn't a valid file descriptor */
+		if ((fp->f_type != DTYPE_VNODE) ||
+		    (fp->f_flag & FREAD) == 0) {
 			error = EBADF;
 			goto out;
 		}
 		vp = (struct vnode *)fp->f_data;
 		proc_fdunlock(p);
 
-		if ( (error = vnode_getwithref(vp)) == 0 ) {
-			error = VNOP_IOCTL(vp, uap->cmd, CAST_DOWN(caddr_t, argp), 0, &context);
+		if (vnode_getwithref(vp)) {
+			error = ENOENT;
+			goto outdrop;
+		}
+		
+		/* Only valid for directories */
+		if (vp->v_type != VDIR) {
+			vnode_put(vp);
+			error = ENOTDIR;
+			goto outdrop;
+		}
 
-			(void)vnode_put(vp);
+		/* Get flags, mode and pathname arguments. */
+		if (IS_64BIT_PROCESS(p)) {
+			pathname = (user_addr_t)argp;
+		} else {
+			pathname = CAST_USER_ADDR_T(argp);
 		}
-		break;
-	
-	}
 
-outdrop:
-	AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
-	fp_drop(p, fd, fp, 0);
-	return(error);
-out:
-	fp_drop(p, fd, fp, 1);
-	proc_fdunlock(p);
-	return(error);
-}
+		/* Start the lookup relative to the file descriptor's vnode. */
+		NDINIT(&nd, DELETE, USEDVP | AUDITVNPATH1, UIO_USERSPACE, pathname, &context);
+		nd.ni_dvp = vp;
 
-/*
- * Common code for dup, dup2, and fcntl(F_DUPFD).
- */
-int
-finishdup(struct proc * p, struct filedesc *fdp, int old, int new, register_t *retval)
-{
-	struct fileproc *nfp;
-	struct fileproc *ofp;
+		error = unlink1(&context, &nd, 0);
+		
+		vnode_put(vp);
+		break;
 
-	if ((ofp = fdp->fd_ofiles[old]) == NULL ||
-			(fdp->fd_ofileflags[old] & UF_RESERVED)) {
-		_fdrelse(p, new);
-		return (EBADF);
 	}
-	fg_ref(ofp);
-	proc_fdunlock(p);
+
+	case F_ADDSIGS: {
+		struct user_fsignatures fs;
+		kern_return_t kr;
+		vm_address_t kernel_blob_addr;
+		vm_size_t kernel_blob_size;
+
+		if (fp->f_type != DTYPE_VNODE) {
+			error = EBADF;
+			goto out;
+		}
+		vp = (struct vnode *)fp->f_data;
+		proc_fdunlock(p);
+		error = vnode_getwithref(vp);
+		if (error)
+			goto outdrop;
+
+		if (IS_64BIT_PROCESS(p)) {
+			error = copyin(argp, &fs, sizeof (fs));
+		} else {
+			struct fsignatures fs32;
+
+			error = copyin(argp, &fs32, sizeof (fs32));
+			fs.fs_file_start = fs32.fs_file_start;
+			fs.fs_blob_start = CAST_USER_ADDR_T(fs32.fs_blob_start);
+			fs.fs_blob_size = fs32.fs_blob_size;
+		}
+
+		if (error) {
+			vnode_put(vp);
+			goto outdrop;
+		}
+
+#define CS_MAX_BLOB_SIZE (1ULL * 1024 * 1024) /* XXX ? */
+		if (fs.fs_blob_size > CS_MAX_BLOB_SIZE) {
+			error = E2BIG;
+			vnode_put(vp);
+			goto outdrop;
+		}
+
+		kernel_blob_size = CAST_DOWN(vm_size_t, fs.fs_blob_size);
+		kr = kmem_alloc(kernel_map,
+				&kernel_blob_addr,
+				kernel_blob_size);
+		if (kr != KERN_SUCCESS) {
+			error = ENOMEM;
+			vnode_put(vp);
+			goto outdrop;
+		}
+
+		error = copyin(fs.fs_blob_start,
+			       (void *) kernel_blob_addr,
+			       kernel_blob_size);
+		if (error) {
+			kmem_free(kernel_map,
+				  kernel_blob_addr,
+				  kernel_blob_size);
+			vnode_put(vp);
+			goto outdrop;
+		}
+
+		error = ubc_cs_blob_add(
+			vp,
+			CPU_TYPE_ANY,	/* not for a specific architecture */
+			fs.fs_file_start,
+			kernel_blob_addr,
+			kernel_blob_size);
+		if (error) {
+			kmem_free(kernel_map,
+				  kernel_blob_addr,
+				  kernel_blob_size);
+		} else {
+			/* ubc_blob_add() was consumed "kernel_blob_addr" */
+		}
+
+		(void) vnode_put(vp);
+		break;
+	}
+
+	case F_MARKDEPENDENCY: {
+		struct vnode *root_vp;
+		struct vnode_attr va;
+		vfs_context_t ctx = vfs_context_current();
+		kauth_cred_t cred;
+
+		if ((current_proc()->p_flag & P_DEPENDENCY_CAPABLE) == 0) {
+		    error = EPERM;
+		    goto out;
+		}
+		
+		if (fp->f_type != DTYPE_VNODE) {
+			error = EBADF;
+			goto out;
+		}
+
+		vp = (struct vnode *)fp->f_data;
+		proc_fdunlock(p);
+
+		if (vnode_getwithref(vp)) {
+			error = ENOENT;
+			goto outdrop;
+		}
+
+		// the passed in vnode must be the root dir of the file system
+		if (VFS_ROOT(vp->v_mount, &root_vp, ctx) != 0 || vp != root_vp) {
+		    error = EINVAL;
+		    vnode_put(vp);
+		    goto outdrop;
+		}
+		vnode_put(root_vp);
+
+		// get the owner of the root dir
+		VATTR_INIT(&va);
+		VATTR_WANTED(&va, va_uid);
+		if (vnode_getattr(vp, &va, ctx) != 0) {
+		    error = EINVAL;
+		    vnode_put(vp);
+		    goto outdrop;
+		}
+
+		// and last, check that the caller is the super user or
+		// the owner of the mount point
+		cred = vfs_context_ucred(ctx);
+		if (!is_suser() && va.va_uid != kauth_cred_getuid(cred)) {
+			error = EACCES;
+			vnode_put(vp);
+			goto outdrop;
+		}
+
+		// if all those checks pass then we can mark the dependency
+		vfs_markdependency(vp->v_mount);
+		error = 0;
+
+		vnode_put(vp);
+		
+		break;
+	}
+
+	default:
+		if (uap->cmd < FCNTL_FS_SPECIFIC_BASE) {
+			error = EINVAL;
+			goto out;
+		}
+
+		// if it's a fs-specific fcntl() then just pass it through
+
+		if (fp->f_type != DTYPE_VNODE) {
+			error = EBADF;
+			goto out;
+		}
+		vp = (struct vnode *)fp->f_data;
+		proc_fdunlock(p);
+
+		if ( (error = vnode_getwithref(vp)) == 0 ) {
+			error = VNOP_IOCTL(vp, uap->cmd, CAST_DOWN(caddr_t, argp), 0, &context);
+
+			(void)vnode_put(vp);
+		}
+		break;
+	
+	}
+
+outdrop:
+	AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
+	fp_drop(p, fd, fp, 0);
+	return(error);
+out:
+	fp_drop(p, fd, fp, 1);
+	proc_fdunlock(p);
+	return(error);
+}
+
+
+/*
+ * finishdup
+ *
+ * Description:	Common code for dup, dup2, and fcntl(F_DUPFD).
+ *
+ * Parameters:	p				Process performing the dup
+ *		old				The fd to dup
+ *		new				The fd to dup it to
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		EBADF
+ *		ENOMEM
+ *
+ * Implicit returns:
+ *		*retval (modified)		The new descriptor
+ *
+ * Locks:	Assumes proc_fdlock for process pointing to fdp is held by
+ *		the caller
+ *
+ * Notes:	This function may drop and reacquire this lock; it is unsafe
+ *		for a caller to assume that other state protected by the lock
+ *		has not been subsequently changes out from under it.
+ */
+int
+finishdup(proc_t p, struct filedesc *fdp, int old, int new, register_t *retval)
+{
+	struct fileproc *nfp;
+	struct fileproc *ofp;
+#if CONFIG_MACF
+	int error;
+#endif
+
+#if DIAGNOSTIC
+	proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
+
+	if ((ofp = fdp->fd_ofiles[old]) == NULL ||
+			(fdp->fd_ofileflags[old] & UF_RESERVED)) {
+		fdrelse(p, new);
+		return (EBADF);
+	}
+	fg_ref(ofp);
+
+#if CONFIG_MACF
+	error = mac_file_check_dup(proc_ucred(p), ofp->f_fglob, new);
+	if (error) {
+		fg_drop(ofp);
+		fdrelse(p, new);
+		return (error);
+	}
+#endif
+
+	proc_fdunlock(p);
 
 	MALLOC_ZONE(nfp, struct fileproc *, sizeof(struct fileproc), M_FILEPROC, M_WAITOK);
-	bzero(nfp, sizeof(struct fileproc));
+	/* Failure check follows proc_fdlock() due to handling requirements */
 
 	proc_fdlock(p);
+
+	if (nfp == NULL) {
+		fg_drop(ofp);
+		fdrelse(p, new);
+		return (ENOMEM);
+	}
+
+	bzero(nfp, sizeof(struct fileproc));
+
 	nfp->f_flags = 0;
 	nfp->f_fglob = ofp->f_fglob;
 	nfp->f_iocount = 0;
 
+#if DIAGNOSTIC
+	if (fdp->fd_ofiles[new] != 0)
+		panic("finishdup: overwriting fd_ofiles with new %d\n", new);
+	if ((fdp->fd_ofileflags[new] & UF_RESERVED) == 0)
+		panic("finishdup: unreserved  fileflags  with new %d\n", new);
+#endif
+
 	if (new > fdp->fd_lastfile)
 		fdp->fd_lastfile = new;
 	procfdtbl_releasefd(p, new, nfp);
@@ -1011,8 +1597,32 @@ finishdup(struct proc * p, struct filedesc *fdp, int old, int new, register_t *r
 }
 
 
+/*
+ * close
+ *
+ * Description:	The implementation of the close(2) system call
+ *
+ * Parameters:	p			Process in whose per process file table
+ *					the close is to occur
+ *		uap->fd			fd to be closed
+ *		retval			<unused>
+ *
+ * Returns:	0			Success
+ *	fp_lookup:EBADF			Bad file descriptor
+ *	close_internal:EBADF
+ *	close_internal:??? 		Anything returnable by a per-fileops
+ *					close function
+ */
+int
+close(proc_t p, struct close_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(close_nocancel(p, (struct close_nocancel_args *)uap, retval));
+}
+
+
 int
-close(struct proc *p, struct close_args *uap, __unused register_t *retval)
+close_nocancel(proc_t p, struct close_nocancel_args *uap, __unused register_t *retval)
 {
 	struct fileproc *fp;
 	int fd = uap->fd;
@@ -1027,7 +1637,7 @@ close(struct proc *p, struct close_args *uap, __unused register_t *retval)
 		return(error);
 	}
 
-	error = close_internal(p, fd, fp, CLOSEINT_LOCKED);
+	error = close_internal_locked(p, fd, fp, 0);
 
 	proc_fdunlock(p);
 
@@ -1036,33 +1646,55 @@ close(struct proc *p, struct close_args *uap, __unused register_t *retval)
 
 
 /*
+ * close_internal_locked
+ *
  * Close a file descriptor.
+ *
+ * Parameters:	p			Process in whose per process file table
+ *					the close is to occur
+ *		fd			fd to be closed
+ *		fp			fileproc associated with the fd
+ *
+ * Returns:	0			Success
+ *		EBADF			fd already in close wait state
+ *	closef_locked:??? 		Anything returnable by a per-fileops
+ *					close function
+ *
+ * Locks:	Assumes proc_fdlock for process is held by the caller and returns
+ *		with lock held
+ *
+ * Notes:	This function may drop and reacquire this lock; it is unsafe
+ *		for a caller to assume that other state protected by the lock
+ *		has not been subsequently changes out from under it, if the
+ *		caller made the call with the lock held.
  */
-int
-close_internal(struct proc *p, int fd, struct fileproc *fp, int flags)
+static int
+close_internal_locked(proc_t p, int fd, struct fileproc *fp, int flags)
 {
 	struct filedesc *fdp = p->p_fd;
 	int error =0;
-	int locked = flags & CLOSEINT_LOCKED;
-	int waitonclose = flags & CLOSEINT_WAITONCLOSE;
-	int norelse = flags & CLOSEINT_NOFDRELSE;
-	int nofdref = flags & CLOSEINT_NOFDNOREF;
-	int slpstate = PRIBIO;
+	int resvfd = flags & FD_DUP2RESV;
 
-	if (!locked)
-		proc_fdlock(p);
+
+#if DIAGNOSTIC
+	proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
 
 	/* Keep people from using the filedesc while we are closing it */
 	procfdtbl_markclosefd(p, fd);
+
+
 	if ((fp->f_flags & FP_CLOSING) == FP_CLOSING) {
 		panic("close_internal_locked:  being called on already closing fd\n");
 	}
 
 
+#if DIAGNOSTIC
+	if ((fdp->fd_ofileflags[fd] & UF_RESERVED) == 0)
+		panic("close_internal: unreserved  fileflags  with fd %d\n", fd);
+#endif
 
 	fp->f_flags |= FP_CLOSING;
-	if (nofdref)
-		fp->f_iocount++;
 
 	if ( (fp->f_flags & FP_AIOISSUED) || kauth_authorize_fileop_has_listeners() ) {
 
@@ -1099,63 +1731,108 @@ close_internal(struct proc *p, int fd, struct fileproc *fp, int flags)
 
 	if ((fp->f_flags & FP_INCHRREAD) == 0)
 		fileproc_drain(p, fp);
-	if (norelse == 0)
+
+	if (resvfd == 0)
 		_fdrelse(p, fd);
+
 	error = closef_locked(fp, fp->f_fglob, p);
 	if ((fp->f_flags & FP_WAITCLOSE) == FP_WAITCLOSE)
 		wakeup(&fp->f_flags);
 	fp->f_flags &= ~(FP_WAITCLOSE | FP_CLOSING);
 
-	if (!locked)
-		proc_fdunlock(p);
+	proc_fdunlock(p);
+
+	FREE_ZONE(fp, sizeof(*fp), M_FILEPROC);	
+
+	proc_fdlock(p);
+
+#if DIAGNOSTIC
+	if (resvfd != 0) {
+		if ((fdp->fd_ofileflags[fd] & UF_RESERVED) == 0)
+			panic("close with reserved fd returns with freed fd:%d: proc: %x\n", fd, (unsigned int)p);
+	}
+#endif
 
-	FREE_ZONE(fp, sizeof *fp, M_FILEPROC);	
 	return(error);
 }
 
+
 /*
- * Return status information about a file descriptor.
+ * fstat1
+ *
+ * Description:	Return status information about a file descriptor.
+ *
+ * Parameters:	p				The process doing the fstat
+ *		fd				The fd to stat
+ *		ub				The user stat buffer
+ *		xsecurity			The user extended security
+ *						buffer, or 0 if none
+ *		xsecurity_size			The size of xsecurity, or 0
+ *						if no xsecurity
+ *		isstat64			Flag to indicate 64 bit version
+ *						for inode size, etc.
+ *
+ * Returns:	0				Success
+ *		EBADF
+ *		EFAULT
+ *	fp_lookup:EBADF				Bad file descriptor
+ *	vnode_getwithref:???
+ *	copyout:EFAULT
+ *	vnode_getwithref:???
+ *	vn_stat:???
+ *	soo_stat:???
+ *	pipe_stat:???
+ *	pshm_stat:???
+ *	kqueue_stat:???
+ *
+ * Notes:	Internal implementation for all other fstat() related
+ *		functions
  *
- * XXX switch on node type is bogus; need a stat in struct fileops instead.
+ *		XXX switch on node type is bogus; need a stat in struct
+ *		XXX fileops instead.
  */
 static int
-fstat1(struct proc *p, int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size)
+fstat1(proc_t p, int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size, int isstat64)
 {
 	struct fileproc *fp;
 	struct stat sb;
+	struct stat64 sb64;
 	struct user_stat user_sb;
+	struct user_stat64 user_sb64;
 	int error, my_size;
 	int funnel_state;
 	file_type_t type;
 	caddr_t data;
 	kauth_filesec_t fsec;
-	ssize_t xsecurity_bufsize;
-	int entrycount;
-	struct vfs_context context;
+	user_size_t xsecurity_bufsize;
+	vfs_context_t ctx = vfs_context_current();
+	void * sbptr;
 
 
 	AUDIT_ARG(fd, fd);
 
-	if ((error = fp_lookup(p, fd, &fp, 0)) != 0)
+	if ((error = fp_lookup(p, fd, &fp, 0)) != 0) {
 		return(error);
+	}
 	type = fp->f_type;
 	data = fp->f_data;
 	fsec = KAUTH_FILESEC_NONE;
 
+	sbptr = (isstat64 != 0) ? (void *)&sb64: (void *)&sb;
+
 	switch (type) {
 
 	case DTYPE_VNODE:
-		context.vc_proc = current_proc();
-		context.vc_ucred = kauth_cred_get();
 		if ((error = vnode_getwithref((vnode_t)data)) == 0) {
 			/*
-			 * If the caller has the file open, and is not requesting extended security,
-			 * we are going to let them get the basic stat information.
+			 * If the caller has the file open, and is not
+			 * requesting extended security information, we are
+			 * going to let them get the basic stat information.
 			 */
 			if (xsecurity == USER_ADDR_NULL) {
-				error = vn_stat_noauth((vnode_t)data, &sb, NULL, &context);
+				error = vn_stat_noauth((vnode_t)data, sbptr, NULL, isstat64, ctx);
 			} else {
-				error = vn_stat((vnode_t)data, &sb, &fsec, &context);
+				error = vn_stat((vnode_t)data, sbptr, &fsec, isstat64, ctx);
 			}
 
 			AUDIT_ARG(vnpath, (struct vnode *)data, ARG_VNODE1);
@@ -1163,21 +1840,23 @@ fstat1(struct proc *p, int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_
 		}
 		break;
 
+#if SOCKETS
 	case DTYPE_SOCKET:
-		error = soo_stat((struct socket *)data, &sb);
+		error = soo_stat((struct socket *)data, sbptr, isstat64);
 		break;
+#endif /* SOCKETS */
 
 	case DTYPE_PIPE:
-		error = pipe_stat((void *)data, &sb);
+		error = pipe_stat((void *)data, sbptr, isstat64);
 		break;
 
 	case DTYPE_PSXSHM:
-		error = pshm_stat((void *)data, &sb);
+		error = pshm_stat((void *)data, sbptr, isstat64);
 		break;
 
 	case DTYPE_KQUEUE:
 	        funnel_state = thread_funnel_set(kernel_flock, TRUE);
-		error = kqueue_stat(fp, &sb, p);
+		error = kqueue_stat(fp, sbptr, isstat64, p);
 		thread_funnel_set(kernel_flock, funnel_state);
 		break;
 
@@ -1185,21 +1864,35 @@ fstat1(struct proc *p, int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_
 		error = EBADF;
 		goto out;
 	}
-	/* Zap spare fields */
-	sb.st_lspare = 0;
-	sb.st_qspare[0] = 0LL;
-	sb.st_qspare[1] = 0LL;
 	if (error == 0) {
 		caddr_t sbp;
-		if (IS_64BIT_PROCESS(current_proc())) {
-			munge_stat(&sb, &user_sb); 
-			my_size = sizeof(user_sb);
-			sbp = (caddr_t)&user_sb;
-		}
-		else {
-			my_size = sizeof(sb);
-			sbp = (caddr_t)&sb;
+
+		if (isstat64 != 0) {
+			sb64.st_lspare = 0;
+			sb64.st_qspare[0] = 0LL;
+			sb64.st_qspare[1] = 0LL;
+			if (IS_64BIT_PROCESS(current_proc())) {
+				munge_stat64(&sb64, &user_sb64); 
+				my_size = sizeof(user_sb64);
+				sbp = (caddr_t)&user_sb64;
+			} else {
+				my_size = sizeof(sb64);
+				sbp = (caddr_t)&sb64;
+			}
+		} else {
+			sb.st_lspare = 0;
+			sb.st_qspare[0] = 0LL;
+			sb.st_qspare[1] = 0LL;
+			if (IS_64BIT_PROCESS(current_proc())) {
+				munge_stat(&sb, &user_sb); 
+				my_size = sizeof(user_sb);
+				sbp = (caddr_t)&user_sb;
+			} else {
+				my_size = sizeof(sb);
+				sbp = (caddr_t)&sb;
+			}
 		}
+
 		error = copyout(sbp, ub, my_size);
 	}
 
@@ -1234,31 +1927,117 @@ out:
 	return (error);
 }
 
+
+/*
+ * fstat_extended
+ *
+ * Description:	Extended version of fstat supporting returning extended
+ *		security information
+ *
+ * Parameters:	p				The process doing the fstat
+ *		uap->fd				The fd to stat
+ *		uap->ub				The user stat buffer
+ *		uap->xsecurity			The user extended security
+ *						buffer, or 0 if none
+ *		uap->xsecurity_size		The size of xsecurity, or 0
+ *
+ * Returns:	0				Success
+ *		!0				Errno (see fstat1)
+ */
 int
-fstat_extended(struct proc *p, struct fstat_extended_args *uap, __unused register_t *retval)
+fstat_extended(proc_t p, struct fstat_extended_args *uap, __unused register_t *retval)
 {
-	return(fstat1(p, uap->fd, uap->ub, uap->xsecurity, uap->xsecurity_size));
+	return(fstat1(p, uap->fd, uap->ub, uap->xsecurity, uap->xsecurity_size, 0));
 }
  
+
+/*
+ * fstat
+ *
+ * Description:	Get file status for the file associated with fd
+ *
+ * Parameters:	p				The process doing the fstat
+ *		uap->fd				The fd to stat
+ *		uap->ub				The user stat buffer
+ *
+ * Returns:	0				Success
+ *		!0				Errno (see fstat1)
+ */
+int
+fstat(proc_t p, register struct fstat_args *uap, __unused register_t *retval)
+{
+	return(fstat1(p, uap->fd, uap->ub, 0, 0, 0));
+}
+
+
+/*
+ * fstat64_extended
+ *
+ * Description:	Extended version of fstat64 supporting returning extended
+ *		security information
+ *
+ * Parameters:	p				The process doing the fstat
+ *		uap->fd				The fd to stat
+ *		uap->ub				The user stat buffer
+ *		uap->xsecurity			The user extended security
+ *						buffer, or 0 if none
+ *		uap->xsecurity_size		The size of xsecurity, or 0
+ *
+ * Returns:	0				Success
+ *		!0				Errno (see fstat1)
+ */
+int
+fstat64_extended(proc_t p, struct fstat64_extended_args *uap, __unused register_t *retval)
+{
+	return(fstat1(p, uap->fd, uap->ub, uap->xsecurity, uap->xsecurity_size, 1));
+}
+ 
+
+/*
+ * fstat64
+ *
+ * Description:	Get 64 bit version of the file status for the file associated
+ *		with fd
+ *
+ * Parameters:	p				The process doing the fstat
+ *		uap->fd				The fd to stat
+ *		uap->ub				The user stat buffer
+ *
+ * Returns:	0				Success
+ *		!0				Errno (see fstat1)
+ */
 int
-fstat(struct proc *p, register struct fstat_args *uap, __unused register_t *retval)
+fstat64(proc_t p, register struct fstat64_args *uap, __unused register_t *retval)
 {
-	return(fstat1(p, uap->fd, uap->ub, 0, 0));
+	return(fstat1(p, uap->fd, uap->ub, 0, 0, 1));
 }
 
+
 /*
- * Return pathconf information about a file descriptor.
+ * fpathconf
+ *
+ * Description:	Return pathconf information about a file descriptor.
+ *
+ * Parameters:	p				Process making the request
+ *		uap->fd				fd to get information about
+ *		uap->name			Name of information desired
+ *		retval				Pointer to the call return area
+ *
+ * Returns:	0				Success
+ *		EINVAL
+ *	fp_lookup:EBADF				Bad file descriptor
+ *	vnode_getwithref:???
+ *	vn_pathconf:???
+ *
+ * Implicit returns:
+ *		*retval (modified)		Returned information (numeric)
  */
 int
-fpathconf(p, uap, retval)
-	struct proc *p;
-	register struct fpathconf_args *uap;
-	register_t *retval;
+fpathconf(proc_t p, struct fpathconf_args *uap, register_t *retval)
 {
 	int fd = uap->fd;
 	struct fileproc *fp;
 	struct vnode *vp;
-	struct vfs_context context;
 	int error = 0;
 	file_type_t type;
 	caddr_t data;
@@ -1292,10 +2071,7 @@ fpathconf(p, uap, retval)
 		if ( (error = vnode_getwithref(vp)) == 0) {
 		        AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-			context.vc_proc = p;
-			context.vc_ucred = kauth_cred_get();
-
-			error = vn_pathconf(vp, uap->name, retval, &context);
+			error = vn_pathconf(vp, uap->name, retval, vfs_context_current());
 
 			(void)vnode_put(vp);
 		}
@@ -1316,18 +2092,35 @@ out:
 }
 
 /*
- * Allocate a file descriptor for the process.
+ * Statistics counter for the number of times a process calling fdalloc()
+ * has resulted in an expansion of the per process open file table.
+ *
+ * XXX This would likely be of more use if it were per process
  */
 int fdexpand;
 
+
+/*
+ * fdalloc
+ *
+ * Description:	Allocate a file descriptor for the process.
+ *
+ * Parameters:	p				Process to allocate the fd in
+ *		want				The fd we would prefer to get
+ *		result				Pointer to fd we got
+ *
+ * Returns:	0				Success
+ *		EMFILE
+ *		ENOMEM
+ *
+ * Implicit returns:
+ *		*result (modified)		The fd which was allocated
+ */
 int
-fdalloc(p, want, result)
-	struct proc *p;
-	int want;
-	int *result;
+fdalloc(proc_t p, int want, int *result)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int i;
+	struct filedesc *fdp = p->p_fd;
+	int i;
 	int lim, last, numfiles, oldnfiles;
 	struct fileproc **newofiles, **ofiles;
 	char *newofileflags;
@@ -1337,6 +2130,10 @@ fdalloc(p, want, result)
 	 * of want or fd_freefile.  If that fails, consider
 	 * expanding the ofile array.
 	 */
+#if DIAGNOSTIC
+	proc_fdlock_assert(p, LCK_MTX_ASSERT_OWNED);
+#endif
+
 	lim = min((int)p->p_rlimit[RLIMIT_NOFILE].rlim_cur, maxfiles);
 	for (;;) {
 		last = min(fdp->fd_nfiles, lim);
@@ -1384,15 +2181,15 @@ fdalloc(p, want, result)
 		 */
 		oldnfiles = fdp->fd_nfiles;
 		(void) memcpy(newofiles, fdp->fd_ofiles,
-				oldnfiles * sizeof *fdp->fd_ofiles);
+				oldnfiles * sizeof(*fdp->fd_ofiles));
 		(void) memset(&newofiles[oldnfiles], 0,
-				(numfiles - oldnfiles) * sizeof *fdp->fd_ofiles);
+				(numfiles - oldnfiles) * sizeof(*fdp->fd_ofiles));
 
 		(void) memcpy(newofileflags, fdp->fd_ofileflags,
-				oldnfiles * sizeof *fdp->fd_ofileflags);
+				oldnfiles * sizeof(*fdp->fd_ofileflags));
 		(void) memset(&newofileflags[oldnfiles], 0,
 				(numfiles - oldnfiles) *
-						sizeof *fdp->fd_ofileflags);
+						sizeof(*fdp->fd_ofileflags));
 		ofiles = fdp->fd_ofiles;
 		fdp->fd_ofiles = newofiles;
 		fdp->fd_ofileflags = newofileflags;
@@ -1402,14 +2199,26 @@ fdalloc(p, want, result)
 	}
 }
 
+
 /*
- * Check to see whether n user file descriptors
- * are available to the process p.
+ * fdavail
+ *
+ * Description:	Check to see whether n user file descriptors are available
+ *		to the process p.
+ *
+ * Parameters:	p				Process to check in
+ *		n				The number of fd's desired
+ *
+ * Returns:	0				No
+ *		1				Yes
+ *
+ * Locks:	Assumes proc_fdlock for process is held by the caller
+ *
+ * Notes:	The answer only remains valid so long as the proc_fdlock is
+ *		held by the caller.
  */
 int
-fdavail(p, n)
-	struct proc *p;
-	int n;
+fdavail(proc_t p, int n)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc **fpp;
@@ -1427,45 +2236,50 @@ fdavail(p, n)
 	return (0);
 }
 
-void
-fdrelse(p, fd)
-	struct proc *p;
-	int fd;
-{
-	_fdrelse(p, fd);
-}
 
+/*
+ * fdrelse
+ *
+ * Description:	Legacy KPI wrapper function for _fdrelse
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to free
+ *
+ * Returns:	void
+ *
+ * Locks:	Assumes proc_fdlock for process is held by the caller
+ */
 void
-fddrop(p, fd)
-	struct proc *p;
-	int fd;
+fdrelse(proc_t p, int fd)
 {
-	struct filedesc *fdp = p->p_fd;
-	struct fileproc *fp;
-
-	if (fd < fdp->fd_freefile)
-		fdp->fd_freefile = fd;
-#if DIAGNOSTIC
-	if (fd > fdp->fd_lastfile)
-		panic("fdrelse: fd_lastfile inconsistent");
-#endif
-	fp = fdp->fd_ofiles[fd];
-	fdp->fd_ofiles[fd] = NULL;
-	fdp->fd_ofileflags[fd] = 0;
-
-	while ((fd = fdp->fd_lastfile) > 0 &&
-			fdp->fd_ofiles[fd] == NULL &&
-			!(fdp->fd_ofileflags[fd] & UF_RESERVED))
-		fdp->fd_lastfile--;
-	FREE_ZONE(fp, sizeof *fp, M_FILEPROC);
+	_fdrelse(p, fd);
 }
 
 
+/*
+ * fdgetf_noref
+ *
+ * Description:	Get the fileproc pointer for the given fd from the per process
+ *		open file table without taking an explicit reference on it.
+ *
+ * Parameters:	p				Process containing fd
+ *		fd				fd to obtain fileproc for
+ *		resultfp			Pointer to pointer return area
+ *
+ * Returns:	0				Success
+ *		EBADF
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Pointer to fileproc pointer
+ *
+ * Locks:	Assumes proc_fdlock for process is held by the caller
+ *
+ * Notes:	Because there is no reference explicitly taken, the returned
+ *		fileproc pointer is only valid so long as the proc_fdlock
+ *		remains held by the caller.
+ */
 int
-fdgetf_noref(p, fd, resultfp)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
+fdgetf_noref(proc_t p, int fd, struct fileproc **resultfp)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
@@ -1481,36 +2295,40 @@ fdgetf_noref(p, fd, resultfp)
 }
 
 
-/* should be called only when proc_fdlock is held */
-void
-fp_setflags(proc_t p, struct fileproc * fp, int flags)
-{
-	proc_fdlock(p);
-	fp->f_flags |= flags;
-	proc_fdunlock(p);
-}
-
-void
-fp_clearflags(proc_t p, struct fileproc * fp, int flags)
-{
-
-	proc_fdlock(p);
-	if (fp)
-		fp->f_flags &= ~flags;
-	proc_fdunlock(p);
-}
-
+/*
+ * fp_getfvp
+ *
+ * Description:	Get fileproc and vnode pointer for a given fd from the per
+ *		process open file table of the specified process, and if
+ *		successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultvp			Pointer to result vnode pointer
+ *						area, or 0 if none
+ *
+ * Returns:	0				Success
+ *		EBADF				Bad file descriptor
+ *		ENOTSUP				fd does not refer to a vnode
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultvp (modified)		vnode pointer
+ *
+ * Notes:	The resultfp and resultvp fields are optional, and may be
+ *		independently specified as NULL to skip returning information
+ *
+ * Locks:	Internally takes and releases proc_fdlock
+ */
 int
-fp_getfvp(p, fd, resultfp, resultvp)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct vnode  **resultvp;
+fp_getfvp(proc_t p, int fd, struct fileproc **resultfp, struct vnode **resultvp)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1533,19 +2351,43 @@ fp_getfvp(p, fd, resultfp, resultvp)
 }
 
 
-
+/*
+ * fp_getfvpandvid
+ *
+ * Description:	Get fileproc, vnode pointer, and vid for a given fd from the
+ *		per process open file table of the specified process, and if
+ *		successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultvp			Pointer to result vnode pointer
+ *						area, or 0 if none
+ *		vidp				Pointer to resuld vid area
+ *
+ * Returns:	0				Success
+ *		EBADF				Bad file descriptor
+ *		ENOTSUP				fd does not refer to a vnode
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultvp (modified)		vnode pointer
+ *		*vidp				vid value
+ *
+ * Notes:	The resultfp and resultvp fields are optional, and may be
+ *		independently specified as NULL to skip returning information
+ *
+ * Locks:	Internally takes and releases proc_fdlock
+ */
 int
-fp_getfvpandvid(p, fd, resultfp, resultvp, vidp)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct vnode  **resultvp;
-	uint32_t * vidp;
+fp_getfvpandvid(proc_t p, int fd, struct fileproc **resultfp,
+		struct vnode **resultvp, uint32_t *vidp)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1569,25 +2411,40 @@ fp_getfvpandvid(p, fd, resultfp, resultvp, vidp)
 	return (0);
 }
 
+
 /*
+ * fp_getfsock
+ *
+ * Description:	Get fileproc and socket pointer for a given fd from the
+ *		per process open file table of the specified process, and if
+ *		successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		results				Pointer to result socket
+ *						pointer area, or 0 if none
+ *
  * Returns:	EBADF			The file descriptor is invalid
  *		EOPNOTSUPP		The file descriptor is not a socket
  *		0			Success
  *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*results (modified)		socket pointer
+ *
  * Notes:	EOPNOTSUPP should probably be ENOTSOCK; this function is only
  *		ever called from accept1().
  */
 int
-fp_getfsock(p, fd, resultfp, results)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct socket  **results;
+fp_getfsock(proc_t p, int fd, struct fileproc **resultfp,
+	    struct socket **results)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1610,17 +2467,39 @@ fp_getfsock(p, fd, resultfp, results)
 }
 
 
+/*
+ * fp_getfkq
+ *
+ * Description:	Get fileproc and kqueue pointer for a given fd from the
+ *		per process open file table of the specified process, and if
+ *		successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultkq			Pointer to result kqueue
+ *						pointer area, or 0 if none
+ *
+ * Returns:	EBADF			The file descriptor is invalid
+ *		EBADF			The file descriptor is not a socket
+ *		0			Success
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultkq (modified)		kqueue pointer
+ *
+ * Notes:	The second EBADF should probably be something else to make
+ *		the error condition distinct.
+ */
 int
-fp_getfkq(p, fd, resultfp, resultkq)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct kqueue  **resultkq;
+fp_getfkq(proc_t p, int fd, struct fileproc **resultfp,
+	  struct kqueue **resultkq)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if ( fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1642,17 +2521,42 @@ fp_getfkq(p, fd, resultfp, resultkq)
 	return (0);
 }
 
+
+/*
+ * fp_getfpshm
+ *
+ * Description:	Get fileproc and POSIX shared memory pointer for a given fd
+ *		from the per process open file table of the specified process
+ *		and if successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultpshm			Pointer to result POSIX
+ *						shared memory pointer
+ *						pointer area, or 0 if none
+ *
+ * Returns:	EBADF			The file descriptor is invalid
+ *		EBADF			The file descriptor is not a POSIX
+ *					shared memory area
+ *		0			Success
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultpshm (modified)		POSIX shared memory pointer
+ *
+ * Notes:	The second EBADF should probably be something else to make
+ *		the error condition distinct.
+ */
 int
-fp_getfpshm(p, fd, resultfp, resultpshm)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct pshmnode  **resultpshm;
+fp_getfpshm(proc_t p, int fd, struct fileproc **resultfp,
+	    struct pshmnode **resultpshm)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1676,17 +2580,50 @@ fp_getfpshm(p, fd, resultfp, resultpshm)
 }
 
 
+/*
+ * fp_getfsem
+ *
+ * Description:	Get fileproc and POSIX semaphore pointer for a given fd from
+ *		the per process open file table of the specified process
+ *		and if successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultpsem			Pointer to result POSIX
+ *						semaphore pointer area, or
+ *						0 if none
+ *
+ * Returns:	EBADF			The file descriptor is invalid
+ *		EBADF			The file descriptor is not a POSIX
+ *					semaphore
+ *		0			Success
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultpsem (modified)		POSIX semaphore pointer
+ *
+ * Notes:	The second EBADF should probably be something else to make
+ *		the error condition distinct.
+ *
+ *		In order to support unnamed POSIX semaphores, the named
+ *		POSIX semaphores will have to move out of the per-process
+ *		open filetable, and into a global table that is shared with
+ *		unnamed POSIX semaphores, since unnamed POSIX semaphores
+ *		are typically used by declaring instances in shared memory,
+ *		and there's no other way to do this without changing the
+ *		underlying type, which would introduce binary compatibility
+ *		issues.
+ */
 int
-fp_getfpsem(p, fd, resultfp, resultpsem)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct psemnode  **resultpsem;
+fp_getfpsem(proc_t p, int fd, struct fileproc **resultfp,
+	    struct psemnode **resultpsem)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1709,17 +2646,39 @@ fp_getfpsem(p, fd, resultfp, resultpsem)
 }
 
 
+/*
+ * fp_getfpipe
+ *
+ * Description:	Get fileproc and pipe pointer for a given fd from the
+ *		per process open file table of the specified process
+ *		and if successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultpipe			Pointer to result pipe
+ *						pointer area, or 0 if none
+ *
+ * Returns:	EBADF			The file descriptor is invalid
+ *		EBADF			The file descriptor is not a socket
+ *		0			Success
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultpipe (modified)		pipe pointer
+ *
+ * Notes:	The second EBADF should probably be something else to make
+ *		the error condition distinct.
+ */
 int
-fp_getfpipe(p, fd, resultfp, resultpipe)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct pipe  **resultpipe;
+fp_getfpipe(proc_t p, int fd, struct fileproc **resultfp,
+	    struct pipe **resultpipe)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1742,18 +2701,44 @@ fp_getfpipe(p, fd, resultfp, resultpipe)
 }
 
 
-#define DTYPE_ATALK -1
+#define DTYPE_ATALK -1		/* XXX This does not belong here */
+
+
+/*
+ * fp_getfatalk
+ *
+ * Description:	Get fileproc and atalk pointer for a given fd from the
+ *		per process open file table of the specified process
+ *		and if successful, increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		resultatalk			Pointer to result atalk
+ *						pointer area, or 0 if none
+ * Returns:	EBADF			The file descriptor is invalid
+ *		EBADF			The file descriptor is not a socket
+ *		0			Success
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *		*resultatalk (modified)		atalk pointer
+ *
+ * Notes:	The second EBADF should probably be something else to make
+ *		the error condition distinct.
+ *
+ *		XXX This code is specific to AppleTalk protocol support, and
+ *		XXX should be conditionally compiled
+ */
 int
-fp_getfatalk(p, fd, resultfp, resultatalk)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	struct atalk  **resultatalk;
+fp_getfatalk(proc_t p, int fd, struct fileproc **resultfp,
+	     struct atalk **resultatalk)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
@@ -1775,19 +2760,40 @@ fp_getfatalk(p, fd, resultfp, resultatalk)
 	return (0);
 }
 
+
+/*
+ * fp_lookup
+ *
+ * Description:	Get fileproc pointer for a given fd from the per process
+ *		open file table of the specified process and if successful,
+ *		increment the f_iocount
+ *
+ * Parameters:	p				Process in which fd lives
+ *		fd				fd to get information for
+ *		resultfp			Pointer to result fileproc
+ *						pointer area, or 0 if none
+ *		locked				!0 if the caller holds the
+ *						proc_fdlock, 0 otherwise
+ *
+ * Returns:	0			Success
+ *		EBADF			Bad file descriptor
+ *
+ * Implicit returns:
+ *		*resultfp (modified)		Fileproc pointer
+ *
+ * Locks:	If the argument 'locked' is non-zero, then the caller is
+ *		expected to have taken and held the proc_fdlock; if it is
+ *		zero, than this routine internally takes and drops this lock.
+ */
 int
-fp_lookup(p, fd, resultfp, locked)
-	struct proc *p;
-	int fd;
-	struct fileproc **resultfp;
-	int locked;
+fp_lookup(proc_t p, int fd, struct fileproc **resultfp, int locked)
 {
 	struct filedesc *fdp = p->p_fd;
 	struct fileproc *fp;
 
 	if (!locked)
-		proc_fdlock(p);
-	if (fd < 0 || fd >= fdp->fd_nfiles ||
+		proc_fdlock_spin(p);
+	if (fd < 0 || fdp == NULL || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
 		if (!locked)
@@ -1804,12 +2810,32 @@ fp_lookup(p, fd, resultfp, locked)
 	return (0);
 }
 
+
+/*
+ * fp_drop_written
+ *
+ * Description:	Set the FP_WRITTEN flag on the fileproc and drop the I/O
+ *		reference previously taken by calling fp_lookup et. al.
+ *
+ * Parameters:	p				Process in which the fd lives
+ *		fd				fd associated with the fileproc
+ *		fp				fileproc on which to set the
+ *						flag and drop the reference
+ *
+ * Returns:	0				Success
+ *	fp_drop:EBADF				Bad file descriptor
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the supplied process
+ *
+ * Notes:	The fileproc must correspond to the fd in the supplied proc
+ */
 int
 fp_drop_written(proc_t p, int fd, struct fileproc *fp)
 {
         int error;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 
 	fp->f_flags |= FP_WRITTEN;
 	
@@ -1821,12 +2847,31 @@ fp_drop_written(proc_t p, int fd, struct fileproc *fp)
 }
 
 
+/*
+ * fp_drop_event
+ *
+ * Description:	Set the FP_WAITEVENT flag on the fileproc and drop the I/O
+ *		reference previously taken by calling fp_lookup et. al.
+ *
+ * Parameters:	p				Process in which the fd lives
+ *		fd				fd associated with the fileproc
+ *		fp				fileproc on which to set the
+ *						flag and drop the reference
+ *
+ * Returns:	0				Success
+ *	fp_drop:EBADF				Bad file descriptor
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the supplied process
+ *
+ * Notes:	The fileproc must correspond to the fd in the supplied proc
+ */
 int
 fp_drop_event(proc_t p, int fd, struct fileproc *fp)
 {
         int error;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 
 	fp->f_flags |= FP_WAITEVENT;
 	
@@ -1837,17 +2882,38 @@ fp_drop_event(proc_t p, int fd, struct fileproc *fp)
 	return (error);
 }
 
+
+/*
+ * fp_drop
+ *
+ * Description:	Drop the I/O reference previously taken by calling fp_lookup
+ *		et. al.
+ *
+ * Parameters:	p				Process in which the fd lives
+ *		fd				fd associated with the fileproc
+ *		fp				fileproc on which to set the
+ *						flag and drop the reference
+ *		locked				flag to internally take and
+ *						drop proc_fdlock if it is not
+ *						already held by the caller
+ *
+ * Returns:	0				Success
+ *		EBADF				Bad file descriptor
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the supplied process if 'locked' is non-zero, and assumes that
+ *		the caller already holds this lock if 'locked' is non-zero.
+ *
+ * Notes:	The fileproc must correspond to the fd in the supplied proc
+ */
 int
-fp_drop(p, fd, fp, locked)
-	struct proc *p;
-	int fd;
-	struct fileproc *fp;
-	int locked;
+fp_drop(proc_t p, int fd, struct fileproc *fp, int locked)
 {
 	struct filedesc *fdp = p->p_fd;
+	int	needwakeup = 0;
 
 	if (!locked)
-		proc_fdlock(p);
+		proc_fdlock_spin(p);
 	 if ((fp == FILEPROC_NULL) && (fd < 0 || fd >= fdp->fd_nfiles ||
 			(fp = fdp->fd_ofiles[fd]) == NULL ||
 			((fdp->fd_ofileflags[fd] & UF_RESERVED) &&
@@ -1860,22 +2926,60 @@ fp_drop(p, fd, fp, locked)
 
 	if (p->p_fpdrainwait && fp->f_iocount == 0) {
 	        p->p_fpdrainwait = 0;
-	        wakeup(&p->p_fpdrainwait);
+		needwakeup = 1;
 	}
 	if (!locked)
 		proc_fdunlock(p);
+	if (needwakeup)
+	        wakeup(&p->p_fpdrainwait);
 		
 	return (0);
 }
 
+
+/*
+ * file_vnode
+ *
+ * Description:	Given an fd, look it up in the current process's per process
+ *		open file table, and return its internal vnode pointer.
+ *
+ * Parameters:	fd				fd to obtain vnode from
+ *		vpp				pointer to vnode return area
+ *
+ * Returns:	0				Success
+ *		EINVAL				The fd does not refer to a
+ *						vnode fileproc entry
+ *	fp_lookup:EBADF				Bad file descriptor
+ *
+ * Implicit returns:
+ *		*vpp (modified)			Returned vnode pointer
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the current process
+ *
+ * Notes:	If successful, this function increments the f_iocount on the
+ *		fd's corresponding fileproc.
+ *
+ *		The fileproc referenced is not returned; because of this, care
+ *		must be taken to not drop the last reference (e.g. by closing
+ *		the file).  This is inhernely unsafe, since the reference may
+ *		not be recoverable from the vnode, if there is a subsequent
+ *		close that destroys the associate fileproc.  The caller should
+ *		therefore retain their own reference on the fileproc so that
+ *		the f_iocount can be dropped subsequently.  Failure to do this
+ *		can result in the returned pointer immediately becoming invalid
+ *		following the call.
+ *
+ *		Use of this function is discouraged.
+ */
 int
 file_vnode(int fd, struct vnode **vpp)
 {
-	struct proc * p = current_proc();
+	proc_t p = current_proc();
 	struct fileproc *fp;
 	int error;
 	
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if ( (error = fp_lookup(p, fd, &fp, 1)) ) {
 		proc_fdunlock(p);
 		return(error);
@@ -1892,14 +2996,48 @@ file_vnode(int fd, struct vnode **vpp)
 }
 
 
+/*
+ * file_socket
+ *
+ * Description:	Given an fd, look it up in the current process's per process
+ *		open file table, and return its internal socket pointer.
+ *
+ * Parameters:	fd				fd to obtain vnode from
+ *		sp				pointer to socket return area
+ *
+ * Returns:	0				Success
+ *		ENOTSOCK			Not a socket
+ *		fp_lookup:EBADF			Bad file descriptor
+ *
+ * Implicit returns:
+ *		*sp (modified)			Returned socket pointer
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the current process
+ *
+ * Notes:	If successful, this function increments the f_iocount on the
+ *		fd's corresponding fileproc.
+ *
+ *		The fileproc referenced is not returned; because of this, care
+ *		must be taken to not drop the last reference (e.g. by closing
+ *		the file).  This is inhernely unsafe, since the reference may
+ *		not be recoverable from the socket, if there is a subsequent
+ *		close that destroys the associate fileproc.  The caller should
+ *		therefore retain their own reference on the fileproc so that
+ *		the f_iocount can be dropped subsequently.  Failure to do this
+ *		can result in the returned pointer immediately becoming invalid
+ *		following the call.
+ *
+ *		Use of this function is discouraged.
+ */
 int
 file_socket(int fd, struct socket **sp)
 {
-	struct proc * p = current_proc();
+	proc_t p = current_proc();
 	struct fileproc *fp;
 	int error;
 	
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if ( (error = fp_lookup(p, fd, &fp, 1)) ) {
 		proc_fdunlock(p);
 		return(error);
@@ -1915,15 +3053,39 @@ file_socket(int fd, struct socket **sp)
 	return(0);
 }
 
+
+/*
+ * file_flags
+ *
+ * Description:	Given an fd, look it up in the current process's per process
+ *		open file table, and return its fileproc's flags field.
+ *
+ * Parameters:	fd				fd whose flags are to be
+ *						retrieved
+ *		flags				pointer to flags data area
+ *
+ * Returns:	0				Success
+ *		ENOTSOCK			Not a socket
+ *		fp_lookup:EBADF			Bad file descriptor
+ *
+ * Implicit returns:
+ *		*flags (modified)		Returned flags field
+ *
+ * Locks:	This function internally takes and drops the proc_fdlock for
+ *		the current process
+ *
+ * Notes:	This function will internally increment and decrement the
+ *		f_iocount of the fileproc as part of its operation.
+ */
 int
-file_flags(int fd, int * flags)
+file_flags(int fd, int *flags)
 {
 
-	struct proc * p = current_proc();
+	proc_t p = current_proc();
 	struct fileproc *fp;
 	int error;
 	
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if ( (error = fp_lookup(p, fd, &fp, 1)) ) {
 		proc_fdunlock(p);
 		return(error);
@@ -1936,13 +3098,50 @@ file_flags(int fd, int * flags)
 }
 
 
+/*
+ * file_drop
+ *
+ * Description:	Drop an iocount reference on an fd, and wake up any waiters
+ *		for draining (i.e. blocked in fileproc_drain() called during
+ *		the last attempt to close a file).
+ *
+ * Parameters:	fd				fd on which an ioreference is
+ *						to be dropped
+ *
+ * Returns:	0				Success
+ *		EBADF				Bad file descriptor
+ *
+ * Description:	Given an fd, look it up in the current process's per process
+ *		open file table, and drop it's fileproc's f_iocount by one
+ *
+ * Notes:	This is intended as a corresponding operation to the functions
+ *		file_vnode() and file_socket() operations.
+ *
+ *		Technically, the close reference is supposed to be protected
+ *		by a fileproc_drain(), however, a drain will only block if
+ *		the fd refers to a character device, and that device has had
+ *		preparefileread() called on it.  If it refers to something
+ *		other than a character device, then the drain will occur and
+ *		block each close attempt, rather than merely the last close.
+ *
+ *		Since it's possible for an fd that refers to a character
+ *		device to have an intermediate close followed by an open to
+ *		cause a different file to correspond to that descriptor,
+ *		unless there was a cautionary reference taken on the fileproc,
+ *		this is an inherently unsafe function.  This happens in the
+ *		case where multiple fd's in a process refer to the same
+ *		character device (e.g. stdin/out/err pointing to a tty, etc.).
+ *
+ *		Use of this function is discouraged.
+ */
 int 
 file_drop(int fd)
 {
 	struct fileproc *fp;
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
+	int	needwakeup = 0;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	if (fd < 0 || fd >= p->p_fd->fd_nfiles ||
 			(fp = p->p_fd->fd_ofiles[fd]) == NULL ||
 			((p->p_fd->fd_ofileflags[fd] & UF_RESERVED) &&
@@ -1954,38 +3153,105 @@ file_drop(int fd)
 
 	if (p->p_fpdrainwait && fp->f_iocount == 0) {
 	        p->p_fpdrainwait = 0;
-	        wakeup(&p->p_fpdrainwait);
+		needwakeup = 1;
 	}
 	proc_fdunlock(p);
-	return(0);
-
 
+	if (needwakeup)
+	        wakeup(&p->p_fpdrainwait);
+	return(0);
 }
 
+
+/*
+ * falloc
+ *
+ * Description:	Allocate an entry in the per process open file table and
+ *		return the corresponding fileproc and fd.
+ *
+ * Parameters:	p				The process in whose open file
+ *						table the fd is to be allocated
+ *		resultfp			Pointer to fileproc pointer
+ *						return area
+ *		resultfd			Pointer to fd return area
+ *		ctx				VFS context
+ *
+ * Returns:	0				Success
+ *	falloc:ENFILE				Too many open files in system
+ *	falloc:EMFILE				Too many open files in process
+ *	falloc:ENOMEM				M_FILEPROC or M_FILEGLOB zone
+ *						exhausted
+ *
+ * Implicit returns:
+ *		*resultfd (modified)		Returned fileproc pointer
+ *		*resultfd (modified)		Returned fd
+ *
+ * Locks:	This function takes and drops the proc_fdlock; if this lock
+ *		is alread held, use falloc_locked() instead.
+ *
+ * Notes:	This function takes separate process and context arguments
+ *		solely to support kern_exec.c; otherwise, it would take
+ *		neither, and expect falloc_locked() to use the
+ *		vfs_context_current() routine internally.
+ */
 int
-falloc(p, resultfp, resultfd )
-	struct proc *p;
-	struct fileproc **resultfp;
-	int *resultfd;
+falloc(proc_t p, struct fileproc **resultfp, int *resultfd, vfs_context_t ctx)
 {
 	int error;
 
 	proc_fdlock(p);
-	error = falloc_locked(p, resultfp, resultfd, 1);
+	error = falloc_locked(p, resultfp, resultfd, ctx, 1);
 	proc_fdunlock(p);
 
 	return(error);
 }
+
+
 /*
+ * falloc_locked
+ *
  * Create a new open file structure and allocate
  * a file decriptor for the process that refers to it.
+ *
+ * Returns:	0			Success
+ *
+ * Description:	Allocate an entry in the per process open file table and
+ *		return the corresponding fileproc and fd.
+ *
+ * Parameters:	p				The process in whose open file
+ *						table the fd is to be allocated
+ *		resultfp			Pointer to fileproc pointer
+ *						return area
+ *		resultfd			Pointer to fd return area
+ *		ctx				VFS context
+ *		locked				Flag to indicate whether the
+ *						caller holds proc_fdlock
+ *
+ * Returns:	0				Success
+ *		ENFILE				Too many open files in system
+ *		fdalloc:EMFILE			Too many open files in process
+ *		ENOMEM				M_FILEPROC or M_FILEGLOB zone
+ *						exhausted
+ *	fdalloc:ENOMEM
+ *
+ * Implicit returns:
+ *		*resultfd (modified)		Returned fileproc pointer
+ *		*resultfd (modified)		Returned fd
+ *
+ * Locks:	If the parameter 'locked' is zero, this function takes and
+ *		drops the proc_fdlock; if non-zero, the caller must hold the
+ *		lock.
+ *
+ * Notes:	If you intend to use a non-zero 'locked' parameter, use the
+ *		utility function falloc() instead.
+ *
+ *		This function takes separate process and context arguments
+ *		solely to support kern_exec.c; otherwise, it would take
+ *		neither, and use the vfs_context_current() routine internally.
  */
 int
-falloc_locked(p, resultfp, resultfd, locked)
-	struct proc *p;
-	struct fileproc **resultfp;
-	int *resultfd;
-	int locked;
+falloc_locked(proc_t p, struct fileproc **resultfp, int *resultfd,
+	      vfs_context_t ctx, int locked)
 {
 	struct fileproc *fp, *fq;
 	struct fileglob *fg;
@@ -2004,6 +3270,15 @@ falloc_locked(p, resultfp, resultfd, locked)
 		tablefull("file");
 		return (ENFILE);
 	}
+#if CONFIG_MACF
+	error = mac_file_check_create(proc_ucred(p));
+	if (error) {
+		if (!locked)
+			proc_fdunlock(p);
+		return (error);
+	}
+#endif
+
 	/*
 	 * Allocate a new file descriptor.
 	 * If the process has file descriptor zero open, add to the list
@@ -2013,7 +3288,18 @@ falloc_locked(p, resultfp, resultfd, locked)
 	proc_fdunlock(p);
 
 	MALLOC_ZONE(fp, struct fileproc *, sizeof(struct fileproc), M_FILEPROC, M_WAITOK);
+	if (fp == NULL) {
+		if (locked)
+			proc_fdlock(p);
+		return (ENOMEM);
+	}
 	MALLOC_ZONE(fg, struct fileglob *, sizeof(struct fileglob), M_FILEGLOB, M_WAITOK);
+	if (fg == NULL) {
+		FREE_ZONE(fp, sizeof(*fp), M_FILEPROC);
+		if (locked)
+			proc_fdlock(p);
+		return (ENOMEM);
+	}
 	bzero(fp, sizeof(struct fileproc));
 	bzero(fg, sizeof(struct fileglob));
 	lck_mtx_init(&fg->fg_lock, file_lck_grp, file_lck_attr);
@@ -2021,12 +3307,21 @@ falloc_locked(p, resultfp, resultfd, locked)
 	fp->f_iocount = 1;
 	fg->fg_count = 1;
 	fp->f_fglob = fg;
+#if CONFIG_MACF
+	mac_file_label_init(fg);
+#endif
+
+	kauth_cred_ref(ctx->vc_ucred);
 
 	proc_fdlock(p);
 
-	fp->f_cred = kauth_cred_proc_ref(p);
+	fp->f_cred = ctx->vc_ucred;
 
-	lck_mtx_lock(file_flist_lock);
+#if CONFIG_MACF
+	mac_file_label_associate(fp->f_cred, fg);
+#endif
+
+	lck_mtx_lock_spin(file_flist_lock);
 
 	nfiles++;
 
@@ -2050,14 +3345,23 @@ falloc_locked(p, resultfp, resultfd, locked)
 	return (0);
 }
 
+
 /*
- * Free a file structure.
+ * fg_free
+ *
+ * Description:	Free a file structure; drop the global open file count, and
+ *		drop the credential reference, if the fileglob has one, and
+ *		destroy the instance mutex before freeing
+ *
+ * Parameters:	fg				Pointer to fileglob to be
+ *						freed
+ *
+ * Returns:	void
  */
 void
-fg_free(fg)
-	struct fileglob *fg;
+fg_free(struct fileglob *fg)
 {
-	lck_mtx_lock(file_flist_lock);
+	lck_mtx_lock_spin(file_flist_lock);
 	LIST_REMOVE(fg, f_list);
 	nfiles--;
 	lck_mtx_unlock(file_flist_lock);
@@ -2067,49 +3371,111 @@ fg_free(fg)
 	}
 	lck_mtx_destroy(&fg->fg_lock, file_lck_grp);
 
+#if CONFIG_MACF
+	mac_file_label_destroy(fg);
+#endif
 	FREE_ZONE(fg, sizeof *fg, M_FILEGLOB);
 }
 
+
+/*
+ * fdexec
+ *
+ * Description:	Perform close-on-exec processing for all files in a process
+ *		that are either marked as close-on-exec, or which were in the
+ *		process of being opened at the time of the execve
+ *
+ * Parameters:	p				Pointer to process calling
+ *						execve
+ *
+ * Returns:	void
+ *
+ * Locks:	This function internally takes and drops proc_fdlock()
+ *
+ * Notes:	This function drops and retakes the kernel funnel; this is
+ *		inherently unsafe, since another thread may have the
+ *		proc_fdlock.
+ *
+ * XXX:		We should likely reverse the lock and funnel drop/acquire
+ *		order to avoid the small race window; it's also possible that
+ *		if the program doing the exec has an outstanding listen socket
+ *		and a network connection is completed asyncrhonously that we
+ *		will end up with a "ghost" socket reference in the new process.
+ *
+ *		This needs reworking to make it safe to remove the funnel from
+ *		the execve and posix_spawn system calls.
+ */
 void
-fdexec(p)
-	struct proc *p;
+fdexec(proc_t p)
 {
 	struct filedesc *fdp = p->p_fd;
 	int i = fdp->fd_lastfile;
 	struct fileproc *fp;
-	int funnel_state;
 
-	funnel_state = thread_funnel_set(kernel_flock, FALSE);
 	proc_fdlock(p);
-
 	while (i >= 0) {
-		fp = fdp->fd_ofiles[i];
-		if ((fdp->fd_ofileflags[i] & (UF_RESERVED|UF_EXCLOSE)) == UF_EXCLOSE) {
-
-			if (i < fdp->fd_knlistsize)
-					knote_fdclose(p, i);
 
+		fp = fdp->fd_ofiles[i];
+		if (
+		    ((fdp->fd_ofileflags[i] & (UF_RESERVED|UF_EXCLOSE)) == UF_EXCLOSE)
+#if CONFIG_MACF
+		    || (fp && mac_file_check_inherit(proc_ucred(p), fp->f_fglob))
+#endif
+		) {
+                        if (i < fdp->fd_knlistsize)
+                                knote_fdclose(p, i);
 			procfdtbl_clearfd(p, i);
 			if (i == fdp->fd_lastfile && i > 0)
 				fdp->fd_lastfile--;
 			if (i < fdp->fd_freefile)
 				fdp->fd_freefile = i;
 			closef_locked(fp, fp->f_fglob, p);
-			FREE_ZONE(fp, sizeof *fp, M_FILEPROC);
+			FREE_ZONE(fp, sizeof(*fp), M_FILEPROC);
 		}
-
 		i--;
 	}
 	proc_fdunlock(p);
-	thread_funnel_set(kernel_flock, funnel_state);
 }
 
+
 /*
- * Copy a filedesc structure.
+ * fdcopy
+ *
+ * Description:	Copy a filedesc structure.  This is normally used as part of
+ *		forkproc() when forking a new process, to copy the per process
+ *		open file table over to the new process.
+ *
+ * Parameters:	p				Process whose open file table
+ *						is to be copied (parent)
+ *		uth_cdir			Per thread current working
+ *						cirectory, or NULL
+ *
+ * Returns:	NULL				Copy failed
+ *		!NULL				Pointer to new struct filedesc
+ *
+ * Locks:	This function internally takes and drops proc_fdlock()
+ *
+ * Notes:	Files are copied directly, ignoring the new resource limits
+ *		for the process that's being copied into.  Since the descriptor
+ *		references are just additional references, this does not count
+ *		against the number of open files on the system.
+ *
+ *		The struct filedesc includes the current working directory,
+ *		and the current root directory, if the process is chroot'ed.
+ *
+ *		If the exec was called by a thread using a per thread current
+ *		working directory, we inherit the working directory from the
+ *		thread making the call, rather than from the process.
+ *
+ *		In the case of a failure to obtain a reference, for most cases,
+ *		the file entry will be silently droppped.  There's an exception
+ *		for the case of a chroot dir, since a failure to to obtain a
+ *		reference there would constitute an "escape" from the chroot
+ *		environment, which must not be allowed.  In that case, we will
+ *		deny the execve() operation, rather than allowing the escape.
  */
 struct filedesc *
-fdcopy(p)
-	struct proc *p;
+fdcopy(proc_t p, vnode_t uth_cdir)
 {
 	struct filedesc *newfdp, *fdp = p->p_fd;
 	int i;
@@ -2129,7 +3495,15 @@ fdcopy(p)
 	(void) memcpy(newfdp, fdp, sizeof(*newfdp));
 
 	/*
-	 * for both fd_cdir and fd_rdir make sure we get
+	 * If we are running with per-thread current working directories,
+	 * inherit the new current working directory from the current thread
+	 * instead, before we take our references.
+	 */
+	if (uth_cdir != NULLVP)
+		newfdp->fd_cdir = uth_cdir;
+
+	/*
+	 * For both fd_cdir and fd_rdir make sure we get
 	 * a valid reference... if we can't, than set
 	 * set the pointer(s) to NULL in the child... this
 	 * will keep us from using a non-referenced vp
@@ -2163,21 +3537,24 @@ fdcopy(p)
 			if ( (vnode_ref(v_dir)) )
 			        newfdp->fd_rdir = NULL;
 			vnode_put(v_dir);
-		} else
+		} else {
 		        newfdp->fd_rdir = NULL;
+		}
 	}
+	/* Coming from a chroot environment and unable to get a reference... */
 	if (newfdp->fd_rdir == NULL && fdp->fd_rdir) {
 	        /*
-		 * we couldn't get a new reference on
-		 * the root directory being
-		 * inherited... we might as well drop
-		 * our reference from the parent also
-		 * since the vnode has gone DEAD making
-		 * it useless... by dropping it we'll
-		 * be that much closer to recyling it
+		 * We couldn't get a new reference on
+		 * the chroot directory being
+		 * inherited... this is fatal, since
+		 * otherwise it would constitute an
+		 * escape from a chroot environment by
+		 * the new process.
 		 */
-	        vnode_rele(fdp->fd_rdir);
-		fdp->fd_rdir = NULL;
+		if (newfdp->fd_cdir)
+		        vnode_rele(newfdp->fd_cdir);
+		FREE_ZONE(newfdp, sizeof *newfdp, M_FILEDESC);
+		return(NULL);
 	}
 	newfdp->fd_refcnt = 1;
 
@@ -2209,7 +3586,7 @@ fdcopy(p)
 		if (newfdp->fd_rdir)
 			vnode_rele(newfdp->fd_rdir);
 
-		FREE_ZONE(newfdp, sizeof *newfdp, M_FILEDESC);
+		FREE_ZONE(newfdp, sizeof(*newfdp), M_FILEDESC);
 		return(NULL);
 	}
 	(void) memset(newfdp->fd_ofiles, 0, i * OFILESIZE);
@@ -2223,9 +3600,9 @@ fdcopy(p)
 		char *flags;
 
 		(void) memcpy(newfdp->fd_ofiles, fdp->fd_ofiles,
-					(newfdp->fd_lastfile + 1) * sizeof *fdp->fd_ofiles);
+					(newfdp->fd_lastfile + 1) * sizeof(*fdp->fd_ofiles));
 		(void) memcpy(newfdp->fd_ofileflags, fdp->fd_ofileflags,
-					(newfdp->fd_lastfile + 1) * sizeof *fdp->fd_ofileflags);
+					(newfdp->fd_lastfile + 1) * sizeof(*fdp->fd_ofileflags));
 
 		/*
 		 * kq descriptors cannot be copied.
@@ -2252,13 +3629,21 @@ fdcopy(p)
 		for (i = newfdp->fd_lastfile + 1; --i >= 0; fpp++, flags++)
 			if ((ofp = *fpp) != NULL && !(*flags & UF_RESERVED)) {
 				MALLOC_ZONE(fp, struct fileproc *, sizeof(struct fileproc), M_FILEPROC, M_WAITOK);
-				bzero(fp, sizeof(struct fileproc));
-				fp->f_flags = ofp->f_flags;
-				//fp->f_iocount = ofp->f_iocount;
-				fp->f_iocount = 0;
-				fp->f_fglob = ofp->f_fglob;
-				(void)fg_ref(fp);
-				*fpp = fp;
+				if (fp == NULL) {
+					/*
+					 * XXX no room to copy, unable to
+					 * XXX safely unwind state at present
+					 */
+					*fpp = NULL;
+				} else {
+					bzero(fp, sizeof(struct fileproc));
+					fp->f_flags = ofp->f_flags;
+					//fp->f_iocount = ofp->f_iocount;
+					fp->f_iocount = 0;
+					fp->f_fglob = ofp->f_fglob;
+					(void)fg_ref(fp);
+					*fpp = fp;
+				}
 			} else {
 				if (i < newfdp->fd_freefile)
 					newfdp->fd_freefile = i;
@@ -2271,12 +3656,23 @@ fdcopy(p)
 	return (newfdp);
 }
 
+
 /*
- * Release a filedesc structure.
+ * fdfree
+ *
+ * Description:	Release a filedesc (per process open file table) structure;
+ *		this is done on process exit(), or from forkproc_free() if
+ *		the fork fails for some reason subsequent to a successful
+ *		call to fdcopy()
+ *
+ * Parameters:	p				Pointer to process going away
+ *
+ * Returns:	void
+ *
+ * Locks:	This function internally takes and drops proc_fdlock()
  */
 void
-fdfree(p)
-	struct proc *p;
+fdfree(proc_t p)
 {
 	struct filedesc *fdp;
 	struct fileproc *fp;
@@ -2301,7 +3697,7 @@ fdfree(p)
 			if ((fp = fdp->fd_ofiles[i]) != NULL) {
 			  
 			  if (fdp->fd_ofileflags[i] & UF_RESERVED)
-			    panic("fdfree: found fp with UF_RESERVED\n");
+			    	panic("fdfree: found fp with UF_RESERVED\n");
 
 				/* closef drops the iocount ... */
 				if ((fp->f_flags & FP_INCHRREAD) != 0) 
@@ -2313,7 +3709,7 @@ fdfree(p)
 				if (fp->f_flags & FP_WAITEVENT) 
 					(void)waitevent_close(p, fp);
 				(void) closef_locked(fp, fp->f_fglob, p);
-				FREE_ZONE(fp, sizeof *fp, M_FILEPROC);
+				FREE_ZONE(fp, sizeof(*fp), M_FILEPROC);
 			}
 		}
 		FREE_ZONE(fdp->fd_ofiles, fdp->fd_nfiles * OFILESIZE, M_OFILETABL);
@@ -2328,7 +3724,7 @@ fdfree(p)
 	if (fdp->fd_rdir)
 		vnode_rele(fdp->fd_rdir);
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	p->p_fd = NULL;
 	proc_fdunlock(p);
 
@@ -2337,38 +3733,48 @@ fdfree(p)
 	if (fdp->fd_knhash)
 		FREE(fdp->fd_knhash, M_KQUEUE);
 
-	FREE_ZONE(fdp, sizeof *fdp, M_FILEDESC);
+	FREE_ZONE(fdp, sizeof(*fdp), M_FILEDESC);
 }
 
+
+/*
+ * closef_finish
+ *
+ * Description:	Called on last open instance for a fileglob for a file being
+ *		closed.
+ *
+ * Parameters:	fp			Pointer to fileproc for fd
+ *		fg			Pointer to fileglob for fd
+ *		p			Pointer to proc structure
+ *
+ * Returns:	0			Success
+ *	<fo_close>:???			Anything returnable by a per-fileops
+ *					close function
+ *
+ * Note:	fp can only be non-NULL if p is also non-NULL.  If p is NULL,
+ *		then fg must eith be locked (FHASLOCK) or must not have a
+ *		type of DTYPE_VNODE.
+ *
+ *		On return, the fg is freed.
+ *
+ *		This function may block draining output to a character
+ *		device on last close of that device.
+ */
 static int
-closef_finish(fp, fg, p)
-	struct fileproc *fp;
-	struct fileglob *fg;
-	struct proc *p;
+closef_finish(struct fileproc *fp, struct fileglob *fg, proc_t p, vfs_context_t ctx)
 {
-	struct vnode *vp;
-	struct flock lf;
 	int error;
-	struct vfs_context context;
 
-	if ((fg->fg_flag & FHASLOCK) && fg->fg_type == DTYPE_VNODE) {
-		lf.l_whence = SEEK_SET;
-		lf.l_start = 0;
-		lf.l_len = 0;
-		lf.l_type = F_UNLCK;
-		vp = (struct vnode *)fg->fg_data;
-		context.vc_proc = p;
-		context.vc_ucred = fg->fg_cred;
 
-		(void) VNOP_ADVLOCK(vp, (caddr_t)fg, F_UNLCK, &lf, F_FLOCK, &context);
-	}
+	/* fg_ops completed initialization? */
 	if (fg->fg_ops)
-		error = fo_close(fg, p);
+		error = fo_close(fg, ctx);
 	else
 		error = 0;
 
+	/* if fp is non-NULL, drain it out */
 	if (((fp != (struct fileproc *)0) && ((fp->f_flags & FP_INCHRREAD) != 0))) {
-	        proc_fdlock(p);
+	        proc_fdlock_spin(p);
 		if ( ((fp->f_flags & FP_INCHRREAD) != 0) ) {
 		        fileproc_drain(p, fp);
 		}
@@ -2379,30 +3785,28 @@ closef_finish(fp, fg, p)
 	return (error);
 }
 
-int
-closef(fg, p)
-	struct fileglob *fg;
-	struct proc *p;
-{
-	int error;
-
-	proc_fdlock(p);
-	error = closef_locked((struct fileproc *)0, fg, p);
-	proc_fdunlock(p);
-
-	return(error);
-}
 /*
- * Internal form of close.
- * Decrement reference count on file structure.
- * Note: p may be NULL when closing a file
- * that was being passed in a message.
+ * closef_locked
+ *
+ * Description:	Internal form of closef; called with proc_fdlock held
+ *
+ * Parameters:	fp			Pointer to fileproc for fd
+ *		fg			Pointer to fileglob for fd
+ *		p			Pointer to proc structure
+ *
+ * Returns:	0			Success
+ *	closef_finish:??? 		Anything returnable by a per-fileops
+ *					close function
+ *
+ * Note:	Decrements reference count on file structure; if this was the
+ *		last reference, then closef_finish() is called
+ *
+ *		p and fp are allowed to  be NULL when closing a file that was
+ *		being passed in a message (but only if we are called when this
+ *		is NOT the last reference).
  */
 int
-closef_locked(fp, fg, p)
-	struct fileproc *fp;
-	struct fileglob *fg;
-	struct proc *p;
+closef_locked(struct fileproc *fp, struct fileglob *fg, proc_t p)
 {
 	struct vnode *vp;
 	struct flock lf;
@@ -2412,6 +3816,14 @@ closef_locked(fp, fg, p)
 	if (fg == NULL) {
 		return (0);
 	}
+
+	/* Set up context with cred stashed in fg */
+	if (p == current_proc())
+		context.vc_thread = current_thread();
+	else
+		context.vc_thread = NULL;
+	context.vc_ucred = fg->fg_cred;
+
 	/*
 	 * POSIX record locking dictates that any close releases ALL
 	 * locks owned by this process.  This is handled by setting
@@ -2430,23 +3842,22 @@ closef_locked(fp, fg, p)
 		vp = (struct vnode *)fg->fg_data;
 
 		if ( (error = vnode_getwithref(vp)) == 0 ) {
-		        context.vc_proc = p;
-			context.vc_ucred = fg->fg_cred;
 			(void) VNOP_ADVLOCK(vp, (caddr_t)p, F_UNLCK, &lf, F_POSIX, &context);
-
 			(void)vnode_put(vp);
 		}
 		proc_fdlock(p);
 	}
-	lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock_spin(&fg->fg_lock);
 	fg->fg_count--;
 
 	if (fg->fg_count > 0) {
 		lck_mtx_unlock(&fg->fg_lock);
 		return (0);
 	}
+#if DIAGNOSTIC
 	if (fg->fg_count != 0)
-		panic("fg: being freed with bad fg_count (%d)", fg, fg->fg_count);
+		panic("fg %p: being freed with bad fg_count (%d)", fg, fg->fg_count);
+#endif
 
 	if (fp && (fp->f_flags & FP_WRITTEN))
 	        fg->fg_flag |= FWASWRITTEN;
@@ -2455,62 +3866,128 @@ closef_locked(fp, fg, p)
 	lck_mtx_unlock(&fg->fg_lock);
 
 	proc_fdunlock(p);
-	error = closef_finish(fp, fg, p);
+	error = closef_finish(fp, fg, p, &context);
 	proc_fdlock(p);
 
 	return(error);
 }
 
 
+/* sleep address to permit wakeup of select by fileproc_drain() */
 extern int selwait;
+
+
+/*
+ * fileproc_drain
+ *
+ * Description:	Drain out pending I/O operations
+ *
+ * Parameters:	p				Process closing this file
+ *		fp				fileproc struct for the open
+ *						instance on the file
+ *
+ * Returns:	void
+ *
+ * Locks:	Assumes the caller holds the proc_fdlock
+ *
+ * Notes:	For character devices, this occurs on the last close of the
+ *		device; for all other file descriptos, this occurs on each
+ *		close to prevent fd's from being closed out from under
+ *		operations currently in progress and blocked
+ *
+ * See Also: 	file_vnode(), file_socket(), file_drop(), and the cautions
+ *		regarding their use and interaction with this function.
+ */
 void
-fileproc_drain(struct proc *p, struct fileproc * fp)
+fileproc_drain(proc_t p, struct fileproc * fp)
 {
+	struct vfs_context context;
+
+	context.vc_thread = proc_thread(p);	/* XXX */
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
 	fp->f_iocount-- ; /* (the one the close holds) */
 
 	while (fp->f_iocount) {
+
+	        lck_mtx_convert_spin(&p->p_fdmlock);
+
 		if (((fp->f_flags & FP_INSELECT)== FP_INSELECT)) {
 			wait_queue_wakeup_all((wait_queue_t)fp->f_waddr, &selwait, THREAD_INTERRUPTED);
 		} else  {
 			if (fp->f_fglob->fg_ops->fo_drain) {
-				(*fp->f_fglob->fg_ops->fo_drain)(fp, p);
+				(*fp->f_fglob->fg_ops->fo_drain)(fp, &context);
 			}
 		}
 		p->p_fpdrainwait = 1;
 
-		msleep(&p->p_fpdrainwait, &p->p_fdmlock, PRIBIO, "fpdrain",0);
+		msleep(&p->p_fpdrainwait, &p->p_fdmlock, PRIBIO, "fpdrain", NULL);
 
-		//panic("successful wait after drain\n");
 	}
 }
 
+
+/*
+ * fp_free
+ *
+ * Description:	Release the fd and free the fileproc associated with the fd
+ *		in the per process open file table of the specified process;
+ *		these values must correspond.
+ *
+ * Parameters:	p				Process containing fd
+ *		fd				fd to be released
+ *		fp				fileproc to be freed
+ *
+ * Returns:	0				Success
+ *
+ * Notes:	XXX function should be void - no one interprets the returns
+ *		XXX code
+ */
 int
-fp_free(struct proc * p, int fd, struct fileproc * fp)
+fp_free(proc_t p, int fd, struct fileproc * fp)
 {
-        proc_fdlock(p);
+        proc_fdlock_spin(p);
 	fdrelse(p, fd);
         proc_fdunlock(p);
 
 	fg_free(fp->f_fglob);
-	FREE_ZONE(fp, sizeof *fp, M_FILEPROC);
+	FREE_ZONE(fp, sizeof(*fp), M_FILEPROC);
+	return(0);
 }
 
 
 /*
- * Apply an advisory lock on a file descriptor.
+ * flock
+ *
+ * Description:	Apply an advisory lock on a file descriptor.
+ *
+ * Parameters:	p				Process making request
+ *		uap->fd				fd on which the lock is to be
+ *						attempted
+ *		uap->how			(Un)Lock bits, including type
+ *		retval				Pointer to the call return area
+ *		
+ * Returns:	0				Success
+ *	fp_getfvp:EBADF				Bad file descriptor
+ *	fp_getfvp:ENOTSUP			fd does not refer to a vnode
+ *	vnode_getwithref:???
+ *	VNOP_ADVLOCK:???
  *
- * Just attempt to get a record lock of the requested type on
- * the entire file (l_whence = SEEK_SET, l_start = 0, l_len = 0).
+ * Implicit returns:
+ *		*retval (modified)		Size of dtable
+ *
+ * Notes:	Just attempt to get a record lock of the requested type on
+ *		the entire file (l_whence = SEEK_SET, l_start = 0, l_len = 0).
  */
 int
-flock(struct proc *p, register struct flock_args *uap, __unused register_t *retval)
+flock(proc_t p, struct flock_args *uap, __unused register_t *retval)
 {
 	int fd = uap->fd;
 	int how = uap->how;
 	struct fileproc *fp;
 	struct vnode *vp;
 	struct flock lf;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error=0;
 
 	AUDIT_ARG(fd, uap->fd);
@@ -2522,16 +3999,13 @@ flock(struct proc *p, register struct flock_args *uap, __unused register_t *retv
 	}
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-	context.vc_proc = p;
-	context.vc_ucred = fp->f_cred;
-
 	lf.l_whence = SEEK_SET;
 	lf.l_start = 0;
 	lf.l_len = 0;
 	if (how & LOCK_UN) {
 		lf.l_type = F_UNLCK;
 		fp->f_flag &= ~FHASLOCK;
-		error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_UNLCK, &lf, F_FLOCK, &context);
+		error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_UNLCK, &lf, F_FLOCK, ctx);
 		goto out;
 	}
 	if (how & LOCK_EX)
@@ -2542,12 +4016,17 @@ flock(struct proc *p, register struct flock_args *uap, __unused register_t *retv
 	        error = EBADF;
 		goto out;
 	}
+#if CONFIG_MACF
+	error = mac_file_check_lock(proc_ucred(p), fp->f_fglob, F_SETLK, &lf);
+	if (error)
+		goto out;
+#endif
 	fp->f_flag |= FHASLOCK;
 	if (how & LOCK_NB) {
-		error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, F_FLOCK, &context);
+		error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, F_FLOCK, ctx);
 		goto out;	
 	}
-	error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, F_FLOCK|F_WAIT, &context);
+	error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, F_FLOCK|F_WAIT, ctx);
 out:
 	(void)vnode_put(vp);
 out1:
@@ -2557,42 +4036,34 @@ out1:
 }
 
 /*
- * File Descriptor pseudo-device driver (/dev/fd/).
+ * dupfdopen
  *
- * Opening minor device N dup()s the file (if any) connected to file
- * descriptor N belonging to the calling process.  Note that this driver
- * consists of only the ``open()'' routine, because all subsequent
- * references to this file will be direct to the other driver.
- */
-int
-fdopen(dev_t dev, __unused int mode, __unused int type, struct proc *p)
-{
-
-	/*
-	 * XXX Kludge: set curproc->p_dupfd to contain the value of the
-	 * the file descriptor being sought for duplication. The error 
-	 * return ensures that the vnode for this device will be released
-	 * by vn_open. Open will detect this special error and take the
-	 * actions in dupfdopen below. Other callers of vn_open or vnop_open
-	 * will simply report the error.
-	 */
-	p->p_dupfd = minor(dev);
-	return (ENODEV);
-}
-
-/*
- * Duplicate the specified descriptor to a free descriptor.
+ * Description:	Duplicate the specified descriptor to a free descriptor;
+ *		this is the second half of fdopen(), above.
+ *
+ * Parameters:	fdp				filedesc pointer to fill in
+ *		indx				fd to dup to
+ *		dfd				fd to dup from
+ *		mode				mode to set on new fd
+ *		error				command code
+ *
+ * Returns:	0				Success
+ *		EBADF				Source fd is bad
+ *		EACCES				Requested mode not allowed
+ *		!0				'error', if not ENODEV or
+ *						ENXIO
+ *
+ * Notes:	XXX This is not thread safe; see fdopen() above
  */
 int
-dupfdopen(fdp, indx, dfd, mode, error)
-	register struct filedesc *fdp;
-	register int indx, dfd;
-	int mode;
-	int error;
+dupfdopen(struct filedesc *fdp, int indx, int dfd, int mode, int error)
 {
 	struct fileproc *wfp;
 	struct fileproc *fp;
-	struct proc * p = current_proc();
+#if CONFIG_MACF
+	int myerror;
+#endif
+	proc_t p = current_proc();
 
 	/*
 	 * If the to-be-dup'd fd number is greater than the allowed number
@@ -2611,6 +4082,13 @@ dupfdopen(fdp, indx, dfd, mode, error)
 	        proc_fdunlock(p);
 		return (EBADF);
 	}
+#if CONFIG_MACF
+	myerror = mac_file_check_dup(proc_ucred(p), wfp->f_fglob, dfd);
+	if (myerror) {
+		proc_fdunlock(p);
+		return (myerror);
+	}
+#endif
 	/*
 	 * There are two cases of interest here.
 	 *
@@ -2653,6 +4131,19 @@ dupfdopen(fdp, indx, dfd, mode, error)
 	/* NOTREACHED */
 }
 
+
+/*
+ * fg_ref
+ *
+ * Description:	Add a reference to a fileglob by fileproc
+ *
+ * Parameters:	fp				fileproc containing fileglob
+ *						pointer
+ *
+ * Returns:	void
+ *
+ * Notes:	XXX Should use OSAddAtomic?
+ */
 void
 fg_ref(struct fileproc * fp)
 {
@@ -2660,33 +4151,67 @@ fg_ref(struct fileproc * fp)
 
 	fg = fp->f_fglob;
 
-	lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock_spin(&fg->fg_lock);
+
+#if DIAGNOSTIC
+	if ((fp->f_flags & ~((unsigned int)FP_VALID_FLAGS)) != 0)
+		panic("fg_ref: invalid bits on fp%x\n", (unsigned int)fp);
+
+	if (fg->fg_count == 0)
+		panic("fg_ref: adding fgcount to zeroed fg :fp %x, fg%x\n ", (unsigned int)fp, (unsigned int)fg);
+#endif
 	fg->fg_count++;
 	lck_mtx_unlock(&fg->fg_lock);
 }
 
+
+/*
+ * fg_drop
+ *
+ * Description:	Remove a reference to a fileglob by fileproc
+ *
+ * Parameters:	fp				fileproc containing fileglob
+ *						pointer
+ *
+ * Returns:	void
+ *
+ * Notes:	XXX Should use OSAddAtomic?
+ */
 void
 fg_drop(struct fileproc * fp)
 {
 	struct fileglob *fg;
 
 	fg = fp->f_fglob;
-	lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock_spin(&fg->fg_lock);
 	fg->fg_count--;
 	lck_mtx_unlock(&fg->fg_lock);
 }
 
 
+/*
+ * fg_insertuipc
+ *
+ * Description:	Insert fileglob onto message queue
+ *
+ * Parameters:	fg				Fileglob pointer to insert
+ *
+ * Returns:	void
+ *
+ * Locks:	Takes and drops fg_lock, potentially many times
+ */
 void
 fg_insertuipc(struct fileglob * fg)
 {
-int insertque = 0;
+	int insertque = 0;
 
-	lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock_spin(&fg->fg_lock);
 
 	while (fg->fg_lflags & FG_RMMSGQ) {
+		lck_mtx_convert_spin(&fg->fg_lock);
+
 		fg->fg_lflags |= FG_WRMMSGQ;
-		msleep(&fg->fg_lflags, &fg->fg_lock, 0, "fg_insertuipc", 0);
+		msleep(&fg->fg_lflags, &fg->fg_lock, 0, "fg_insertuipc", NULL);
 	}
 
 	fg->fg_count++;
@@ -2698,7 +4223,7 @@ int insertque = 0;
 	lck_mtx_unlock(&fg->fg_lock);
 
 	if (insertque) {
-		lck_mtx_lock(uipc_lock);
+		lck_mtx_lock_spin(uipc_lock);
 		unp_gc_wait();
 		LIST_INSERT_HEAD(&fmsghead, fg, f_msglist);
 		lck_mtx_unlock(uipc_lock);
@@ -2713,15 +4238,29 @@ int insertque = 0;
 
 }
 
+
+/*
+ * fg_removeuipc
+ *
+ * Description:	Remove fileglob from message queue
+ *
+ * Parameters:	fg				Fileglob pointer to remove
+ *
+ * Returns:	void
+ *
+ * Locks:	Takes and drops fg_lock, potentially many times
+ */
 void
 fg_removeuipc(struct fileglob * fg)
 {
-int removeque = 0;
+	int removeque = 0;
 
-	lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock_spin(&fg->fg_lock);
 	while (fg->fg_lflags & FG_INSMSGQ) {
+		lck_mtx_convert_spin(&fg->fg_lock);
+
 		fg->fg_lflags |= FG_WINSMSGQ;
-		msleep(&fg->fg_lflags, &fg->fg_lock, 0, "fg_removeuipc", 0);
+		msleep(&fg->fg_lflags, &fg->fg_lock, 0, "fg_removeuipc", NULL);
 	}
 	fg->fg_msgcount--;
 	if (fg->fg_msgcount == 0) {
@@ -2731,7 +4270,7 @@ int removeque = 0;
 	lck_mtx_unlock(&fg->fg_lock);
 
 	if (removeque) {
-		lck_mtx_lock(uipc_lock);
+		lck_mtx_lock_spin(uipc_lock);
 		unp_gc_wait();
 		LIST_REMOVE(fg, f_msglist);
 		lck_mtx_unlock(uipc_lock);
@@ -2746,44 +4285,138 @@ int removeque = 0;
 }
 
 
+/*
+ * fo_read
+ *
+ * Description:	Generic fileops read indirected through the fileops pointer
+ *		in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer
+ *		uio				user I/O structure pointer
+ *		flags				FOF_ flags
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from read
+ */
 int
-fo_read(struct fileproc *fp, struct uio *uio, kauth_cred_t cred, int flags, struct proc *p)
+fo_read(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx)
 {
-	return ((*fp->f_ops->fo_read)(fp, uio, cred, flags, p));
+	return ((*fp->f_ops->fo_read)(fp, uio, flags, ctx));
 }
 
+
+/*
+ * fo_write
+ *
+ * Description:	Generic fileops write indirected through the fileops pointer
+ *		in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer
+ *		uio				user I/O structure pointer
+ *		flags				FOF_ flags
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from write
+ */
 int
-fo_write(struct fileproc *fp, struct uio *uio, kauth_cred_t cred, int flags, struct proc *p)
+fo_write(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx)
 {
-	return((*fp->f_ops->fo_write)(fp, uio, cred, flags, p));
+	return((*fp->f_ops->fo_write)(fp, uio, flags, ctx));
 }
 
+
+/*
+ * fo_ioctl
+ *
+ * Description:	Generic fileops ioctl indirected through the fileops pointer
+ *		in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer
+ *		com				ioctl command
+ *		data				pointer to internalized copy
+ *						of user space ioctl command
+ *						parameter data in kernel space
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from ioctl
+ *
+ * Locks:	The caller is assumed to have held the proc_fdlock; this
+ *		function releases and reacquires this lock.  If the caller
+ *		accesses data protected by this lock prior to calling this
+ *		function, it will need to revalidate/reacquire any cached
+ *		protected data obtained prior to the call.
+ */
 int 
-fo_ioctl(struct fileproc *fp, u_long com, caddr_t data, struct proc *p)
+fo_ioctl(struct fileproc *fp, u_long com, caddr_t data, vfs_context_t ctx)
 {
-int error;
+	int error;
 
-	proc_fdunlock(p);
-	error = (*fp->f_ops->fo_ioctl)(fp, com, data, p);
-	proc_fdlock(p);
+	proc_fdunlock(vfs_context_proc(ctx));
+	error = (*fp->f_ops->fo_ioctl)(fp, com, data, ctx);
+	proc_fdlock(vfs_context_proc(ctx));
 	return(error);
 }       
 
+
+/*
+ * fo_select
+ *
+ * Description:	Generic fileops select indirected through the fileops pointer
+ *		in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer
+ *		which				select which
+ *		wql				pointer to wait queue list
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from select
+ */
 int
-fo_select(struct fileproc *fp, int which, void *wql, struct proc *p)
+fo_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
 {       
-	return((*fp->f_ops->fo_select)(fp, which, wql, p));
+	return((*fp->f_ops->fo_select)(fp, which, wql, ctx));
 }
 
+
+/*
+ * fo_close
+ *
+ * Description:	Generic fileops close indirected through the fileops pointer
+ *		in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer for
+ *						file to close
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from close
+ */
 int
-fo_close(struct fileglob *fg, struct proc *p)
+fo_close(struct fileglob *fg, vfs_context_t ctx)
 {       
-	return((*fg->fg_ops->fo_close)(fg, p));
+	return((*fg->fg_ops->fo_close)(fg, ctx));
 }
 
+
+/*
+ * fo_kqfilter
+ *
+ * Description:	Generic fileops kqueue filter indirected through the fileops
+ *		pointer in the fileproc structure
+ *
+ * Parameters:	fp				fileproc structure pointer
+ *		kn				pointer to knote to filter on
+ *		ctx				VFS context for operation
+ *
+ * Returns:	0				Success
+ *		!0				Errno from kqueue filter
+ */
 int
-fo_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p)
+fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx)
 {
-        return ((*fp->f_ops->fo_kqfilter)(fp, kn, p));
+        return ((*fp->f_ops->fo_kqfilter)(fp, kn, ctx));
 }
-
diff --git a/bsd/kern/kern_event.c b/bsd/kern/kern_event.c
index 0c05251df..5c940792d 100644
--- a/bsd/kern/kern_event.c
+++ b/bsd/kern/kern_event.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  */
 /*-
@@ -83,8 +89,7 @@
 #include <kern/assert.h>
 
 #include <libkern/libkern.h>
-
-extern void unix_syscall_return(int);
+#include "kpi_mbuf_internal.h"
 
 MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
 
@@ -98,16 +103,16 @@ static int	knoteuse2kqlock(struct kqueue *kq, struct knote *kn);
 
 static void 	kqueue_wakeup(struct kqueue *kq);
 static int 	kqueue_read(struct fileproc *fp, struct uio *uio,
-		    kauth_cred_t cred, int flags, struct proc *p);
+		    int flags, vfs_context_t ctx);
 static int	kqueue_write(struct fileproc *fp, struct uio *uio,
-		    kauth_cred_t cred, int flags, struct proc *p);
+		    int flags, vfs_context_t ctx);
 static int	kqueue_ioctl(struct fileproc *fp, u_long com, caddr_t data,
-		    struct proc *p);
+		    vfs_context_t ctx);
 static int 	kqueue_select(struct fileproc *fp, int which, void *wql, 
-		    struct proc *p);
-static int 	kqueue_close(struct fileglob *fp, struct proc *p);
-static int 	kqueue_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
-extern int	kqueue_stat(struct fileproc *fp, struct stat *st, struct proc *p);
+		    vfs_context_t ctx);
+static int 	kqueue_close(struct fileglob *fp, vfs_context_t ctx);
+static int 	kqueue_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx);
+extern int	kqueue_stat(struct fileproc *fp, void  *ub, int isstat64, vfs_context_t ctx);
 
 static struct fileops kqueueops = {
 	kqueue_read,
@@ -136,7 +141,6 @@ static void 	knote_enqueue(struct knote *kn);
 static void 	knote_dequeue(struct knote *kn);
 static struct 	knote *knote_alloc(void);
 static void 	knote_free(struct knote *kn);
-extern void	knote_init(void);
 
 static int	filt_fileattach(struct knote *kn);
 static struct filterops file_filtops =
@@ -183,16 +187,8 @@ static lck_mtx_t _filt_timerlock;
 static void	filt_timerlock(void);
 static void	filt_timerunlock(void);
 
-/*
- * Sentinel marker for a thread scanning through the list of
- * active knotes.
- */
-static struct filterops threadmarker_filtops =
-	{ 0, filt_badattach, 0, 0 };
-
 static zone_t	knote_zone;
 
-#define	KN_HASHSIZE		64		/* XXX should be tunable */
 #define KN_HASH(val, mask)	(((val) ^ (val >> 8)) & (mask))
 
 #if 0
@@ -366,7 +362,7 @@ static int
 filt_fileattach(struct knote *kn)
 {
 	
-	return (fo_kqfilter(kn->kn_fp, kn, current_proc()));
+	return (fo_kqfilter(kn->kn_fp, kn, vfs_context_current()));
 }
 
 #define f_flag f_fglob->fg_flag
@@ -401,28 +397,27 @@ static int
 filt_procattach(struct knote *kn)
 {
 	struct proc *p;
-	int funnel_state;
+
+	assert(PID_MAX < NOTE_PDATAMASK);
 	
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	if ((kn->kn_sfflags & (NOTE_TRACK | NOTE_TRACKERR | NOTE_CHILD)) != 0)
+		return(ENOTSUP);
 
-	if ((kn->kn_sfflags & (NOTE_TRACK | NOTE_TRACKERR | NOTE_CHILD)) != 0) {
-		thread_funnel_set(kernel_flock, funnel_state);
-		return (ENOTSUP);
-	}
-		
-	p = pfind(kn->kn_id);
+	p = proc_find(kn->kn_id);
 	if (p == NULL) {
-		thread_funnel_set(kernel_flock, funnel_state);
 		return (ESRCH);
 	}
 
-	kn->kn_flags |= EV_CLEAR;		/* automatically set */
-	kn->kn_hookid = 1;			/* mark exit not seen */
+	proc_klist_lock();
+
+	kn->kn_flags |= EV_CLEAR;	/* automatically set */
+	kn->kn_ptr.p_proc = p;		/* store the proc handle */
 
-	/* XXX lock the proc here while adding to the list? */
 	KNOTE_ATTACH(&p->p_klist, kn);
 
-	thread_funnel_set(kernel_flock, funnel_state);
+	proc_klist_unlock();
+
+	proc_rele(p);
 
 	return (0);
 }
@@ -430,40 +425,34 @@ filt_procattach(struct knote *kn)
 /*
  * The knote may be attached to a different process, which may exit,
  * leaving nothing for the knote to be attached to.  In that case,
- * we wont be able to find the process from its pid.  But the exit
- * code may still be processing the knote list for the target process.
- * We may have to wait for that processing to complete before we can
- * return (and presumably free the knote) without actually removing
- * it from the dead process' knote list.
+ * the pointer to the process will have already been nulled out.
  */
 static void
 filt_procdetach(struct knote *kn)
 {
 	struct proc *p;
-	int funnel_state;
-
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	p = pfind(kn->kn_id);
 
-	if (p != (struct proc *)NULL) {
+	proc_klist_lock();
+	
+	p = kn->kn_ptr.p_proc;
+	if (p != PROC_NULL) {
+		kn->kn_ptr.p_proc = PROC_NULL;
 		KNOTE_DETACH(&p->p_klist, kn);
-	} else if (kn->kn_hookid != 0) {	/* if not NOTE_EXIT yet */
-		kn->kn_hookid = -1;	/* we are detaching but... */
-		assert_wait(&kn->kn_hook, THREAD_UNINT); /* have to wait */
-		thread_block(THREAD_CONTINUE_NULL);
 	}
-	thread_funnel_set(kernel_flock, funnel_state);
+
+	proc_klist_unlock();
 }
 
 static int
 filt_proc(struct knote *kn, long hint)
 {
+	struct proc * p;
 
+	/* hint is 0 when called from above */
 	if (hint != 0) {
 		u_int event;
 
-		/* must hold the funnel when coming from below */
-		assert(thread_funnel_get() != (funnel_t)0);
+		/* ALWAYS CALLED WITH proc_klist_lock when (hint != 0) */
 
 		/*
 		 * mask off extra data
@@ -477,26 +466,24 @@ filt_proc(struct knote *kn, long hint)
 			kn->kn_fflags |= event;
 
 		/*
-		 * process is gone, so flag the event as finished.
-		 *
-		 * If someone was trying to detach, but couldn't
-		 * find the proc to complete the detach, wake them
-		 * up (nothing will ever need to walk the per-proc
-		 * knote list again - so its safe for them to dump
-		 * the knote now).
+		 * If this is the last possible event for the
+		 * knote, unlink this knote from the process
+		 * before the process goes away.
 		 */
-		if (event == NOTE_EXIT) {
-			boolean_t detaching = (kn->kn_hookid == -1);
-
-			kn->kn_hookid = 0;
-			kn->kn_flags |= (EV_EOF | EV_ONESHOT); 
-			if (detaching)
-				thread_wakeup(&kn->kn_hookid);
+		if (event == NOTE_REAP || (event == NOTE_EXIT && !(kn->kn_sfflags & NOTE_REAP))) {
+			kn->kn_flags |= (EV_EOF | EV_ONESHOT);
+			p = kn->kn_ptr.p_proc;
+			if (p != PROC_NULL) {
+				kn->kn_ptr.p_proc = PROC_NULL;
+				KNOTE_DETACH(&p->p_klist, kn);
+			}
 			return (1);
 		}
+
 	}
 
-	return (kn->kn_fflags != 0); /* atomic check - no funnel needed from above */
+	/* atomic check, no locking need when called from above */
+	return (kn->kn_fflags != 0); 
 }
 
 /*
@@ -765,7 +752,7 @@ kqueue_alloc(struct proc *p)
 		lck_spin_init(&kq->kq_lock, kq_lck_grp, kq_lck_attr);
 		TAILQ_INIT(&kq->kq_head);
 		TAILQ_INIT(&kq->kq_inprocess);
-		kq->kq_fdp = fdp;
+		kq->kq_p = p;
 	}
 
 	if (fdp->fd_knlistsize < 0) {
@@ -794,8 +781,9 @@ kqueue_alloc(struct proc *p)
  *	Nothing locked on entry or exit.
  */
 void
-kqueue_dealloc(struct kqueue *kq, struct proc *p)
+kqueue_dealloc(struct kqueue *kq)
 {
+	struct proc *p = kq->kq_p;
 	struct filedesc *fdp = p->p_fd;
 	struct knote *kn;
 	int i;
@@ -853,7 +841,7 @@ kqueue(struct proc *p, __unused struct kqueue_args *uap, register_t *retval)
 	struct fileproc *fp;
 	int fd, error;
 
-	error = falloc(p, &fp, &fd);
+	error = falloc(p, &fp, &fd, vfs_context_current());
 	if (error) {
 		return (error);
 	}
@@ -937,7 +925,13 @@ kevent_copyout(struct kevent *kevp, user_addr_t *addrp, struct proc *p)
 	if (IS_64BIT_PROCESS(p)) {
 		struct user_kevent kev64;
 
-		kev64.ident = (uint64_t) kevp->ident;
+		/*
+		 * deal with the special case of a user-supplied
+		 * value of (uintptr_t)-1.
+		 */
+		kev64.ident = (kevp->ident == (uintptr_t)-1) ?
+			   (uint64_t)-1LL : (uint64_t)kevp->ident;
+
 		kev64.filter = kevp->filter;
 		kev64.flags = kevp->flags;
 		kev64.fflags = kevp->fflags;
@@ -1053,7 +1047,7 @@ kevent(struct proc *p, struct kevent_args *uap, register_t *retval)
 				
 		kev.flags &= ~EV_SYSFLAGS;
 		error = kevent_register(kq, &kev, p);
-		if (error && nevents > 0) {
+		if ((error || (kev.flags & EV_RECEIPT)) && nevents > 0) {
 			kev.flags = EV_ERROR;
 			kev.data = error;
 			error = kevent_copyout(&kev, &ueventlist, p);
@@ -1067,7 +1061,7 @@ kevent(struct proc *p, struct kevent_args *uap, register_t *retval)
 
 	/* store the continuation/completion data in the uthread */
 	ut = (uthread_t)get_bsdthread_info(current_thread());
-	cont_args = (struct _kevent *)&ut->uu_state.ss_kevent;
+	cont_args = (struct _kevent *)&ut->uu_kevent.ss_kevent;
 	cont_args->fp = fp;
 	cont_args->fd = fd;
 	cont_args->retval = retval;
@@ -1099,7 +1093,7 @@ kevent_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
 	int error;
 
 	cont_args = (struct _kevent *)data;
-	assert(cont_args->eventout < cont_arg->eventcount);
+	assert(cont_args->eventout < cont_args->eventcount);
 
 	/*
 	 * Copy out the appropriate amount of event data for this user.
@@ -1130,9 +1124,10 @@ kevent_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
  */
 
 int
-kevent_register(struct kqueue *kq, struct kevent *kev, struct proc *p)
+kevent_register(struct kqueue *kq, struct kevent *kev, __unused struct proc *ctxp)
 {
-	struct filedesc *fdp = kq->kq_fdp;
+	struct proc *p = kq->kq_p;
+	struct filedesc *fdp = p->p_fd;
 	struct filterops *fops;
 	struct fileproc *fp = NULL;
 	struct knote *kn = NULL;
@@ -1443,7 +1438,7 @@ static void
 kevent_scan_continue(void *data, wait_result_t wait_result)
 {
 	uthread_t ut = (uthread_t)get_bsdthread_info(current_thread());
-	struct _kevent_scan * cont_args = &ut->uu_state.ss_kevent_scan;
+	struct _kevent_scan * cont_args = &ut->uu_kevent.ss_kevent_scan;
 	struct kqueue *kq = (struct kqueue *)data;
 	int error;
 	int count;
@@ -1527,7 +1522,6 @@ kevent_scan(struct kqueue *kq,
 			first = 0;
 			/* convert the timeout to a deadline once */
 			if (atvp->tv_sec || atvp->tv_usec) {
-				uint32_t seconds, nanoseconds;
 				uint64_t now;
 				
 				clock_get_uptime(&now);
@@ -1547,7 +1541,7 @@ kevent_scan(struct kqueue *kq,
 
 			if (continuation) {
 				uthread_t ut = (uthread_t)get_bsdthread_info(current_thread());
-				struct _kevent_scan *cont_args = &ut->uu_state.ss_kevent_scan;
+				struct _kevent_scan *cont_args = &ut->uu_kevent.ss_kevent_scan;
 				
 				cont_args->call = callback;
 				cont_args->cont = continuation;
@@ -1590,9 +1584,8 @@ kevent_scan(struct kqueue *kq,
 static int
 kqueue_read(__unused struct fileproc *fp, 
 			__unused struct uio *uio, 
-			__unused kauth_cred_t cred,
 			__unused int flags, 
-			__unused struct proc *p)
+			__unused vfs_context_t ctx)
 {
 	return (ENXIO);
 }
@@ -1601,9 +1594,8 @@ kqueue_read(__unused struct fileproc *fp,
 static int
 kqueue_write(__unused struct fileproc *fp, 
 			 __unused struct uio *uio, 
-			 __unused kauth_cred_t cred,
 	 		 __unused int flags, 
-	 		 __unused struct proc *p)
+	 		 __unused vfs_context_t ctx)
 {
 	return (ENXIO);
 }
@@ -1613,14 +1605,14 @@ static int
 kqueue_ioctl(__unused struct fileproc *fp, 
 			 __unused u_long com, 
 			 __unused caddr_t data, 
-			 __unused struct proc *p)
+			 __unused vfs_context_t ctx)
 {
 	return (ENOTTY);
 }
 
 /*ARGSUSED*/
 static int
-kqueue_select(struct fileproc *fp, int which, void *wql, struct proc *p)
+kqueue_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
 {
 	struct kqueue *kq = (struct kqueue *)fp->f_data;
 	int retnum = 0;
@@ -1630,7 +1622,7 @@ kqueue_select(struct fileproc *fp, int which, void *wql, struct proc *p)
                 if (kq->kq_count) {
 			retnum = 1;
 		} else {
-		        selrecord(p, &kq->kq_sel, wql);
+		        selrecord(vfs_context_proc(ctx), &kq->kq_sel, wql);
 			kq->kq_state |= KQ_SEL;
 		}
 		kqunlock(kq);
@@ -1643,11 +1635,11 @@ kqueue_select(struct fileproc *fp, int which, void *wql, struct proc *p)
  */
 /*ARGSUSED*/
 static int
-kqueue_close(struct fileglob *fg, struct proc *p)
+kqueue_close(struct fileglob *fg, __unused vfs_context_t ctx)
 {
 	struct kqueue *kq = (struct kqueue *)fg->fg_data;
 
-	kqueue_dealloc(kq, p);
+	kqueue_dealloc(kq);
 	fg->fg_data = NULL;
 	return (0);
 }
@@ -1659,30 +1651,73 @@ kqueue_close(struct fileglob *fg, struct proc *p)
  * that relationship is torn down.
  */
 static int
-kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused struct proc *p)
+kqueue_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx)
 {
 	struct kqueue *kq = (struct kqueue *)kn->kn_fp->f_data;
+	struct kqueue *parentkq = kn->kn_kq;
 
-	if (kn->kn_filter != EVFILT_READ)
+	if (parentkq == kq ||
+	    kn->kn_filter != EVFILT_READ)
 		return (1);
 
-	kn->kn_fop = &kqread_filtops;
-	kqlock(kq);
-	KNOTE_ATTACH(&kq->kq_sel.si_note, kn);
-	kqunlock(kq);
-	return (0);
+	/*
+	 * We have to avoid creating a cycle when nesting kqueues
+	 * inside another.  Rather than trying to walk the whole
+	 * potential DAG of nested kqueues, we just use a simple
+	 * ceiling protocol.  When a kqueue is inserted into another,
+	 * we check that the (future) parent is not already nested
+	 * into another kqueue at a lower level than the potenial
+	 * child (because it could indicate a cycle).  If that test
+	 * passes, we just mark the nesting levels accordingly.
+	 */
+
+	kqlock(parentkq);
+	if (parentkq->kq_level > 0 && 
+	    parentkq->kq_level < kq->kq_level)
+	{
+		kqunlock(parentkq);
+		return (1);
+	} else {
+		/* set parent level appropriately */
+		if (parentkq->kq_level == 0)
+			parentkq->kq_level = 2;
+		if (parentkq->kq_level < kq->kq_level + 1)
+			parentkq->kq_level = kq->kq_level + 1;
+		kqunlock(parentkq);
+
+		kn->kn_fop = &kqread_filtops;
+		kqlock(kq);
+		KNOTE_ATTACH(&kq->kq_sel.si_note, kn);
+		/* indicate nesting in child, if needed */
+		if (kq->kq_level == 0)
+			kq->kq_level = 1;
+		kqunlock(kq);
+		return (0);
+	}
 }
 
 /*ARGSUSED*/
 int
-kqueue_stat(struct fileproc *fp, struct stat *st, __unused struct proc *p)
+kqueue_stat(struct fileproc *fp, void *ub, int isstat64,  __unused vfs_context_t ctx)
 {
+	struct stat *sb = (struct stat *)0;	/* warning avoidance ; protected by isstat64 */
+	struct stat64 * sb64 = (struct stat64 *)0;  /* warning avoidance ; protected by isstat64 */
+
 	struct kqueue *kq = (struct kqueue *)fp->f_data;
+	if (isstat64 != 0) {
+		sb64 = (struct stat64 *)ub;
+		bzero((void *)sb64, sizeof(*sb64));
+		sb64->st_size = kq->kq_count;
+		sb64->st_blksize = sizeof(struct kevent);
+		sb64->st_mode = S_IFIFO;
+	} else {
+		sb = (struct stat *)ub;
+		bzero((void *)sb, sizeof(*sb));
+		sb->st_size = kq->kq_count;
+		sb->st_blksize = sizeof(struct kevent);
+		sb->st_mode = S_IFIFO;
+	}
 
-	bzero((void *)st, sizeof(*st));
-	st->st_size = kq->kq_count;
-	st->st_blksize = sizeof(struct kevent);
-	st->st_mode = S_IFIFO;
 	return (0);
 }
 
@@ -1790,6 +1825,9 @@ knote_fdclose(struct proc *p, int fd)
 	while ((kn = SLIST_FIRST(list)) != NULL) {
 		struct kqueue *kq = kn->kn_kq;
 
+		if (kq->kq_p != p)
+			panic("knote_fdclose: proc mismatch (kq->kq_p=%p != p=%p)", kq->kq_p, p);
+
 		kqlock(kq);
 		proc_fdunlock(p);
 
@@ -1820,7 +1858,7 @@ knote_fdpattach(struct knote *kn, struct filedesc *fdp, __unused struct proc *p)
 
 	if (! kn->kn_fop->f_isfd) {
 		if (fdp->fd_knhashmask == 0)
-			fdp->fd_knhash = hashinit(KN_HASHSIZE, M_KQUEUE,
+			fdp->fd_knhash = hashinit(CONFIG_KN_HASHSIZE, M_KQUEUE,
 			    &fdp->fd_knhashmask);
 		list = &fdp->fd_knhash[KN_HASH(kn->kn_id, fdp->fd_knhashmask)];
 	} else {
@@ -1858,10 +1896,11 @@ knote_fdpattach(struct knote *kn, struct filedesc *fdp, __unused struct proc *p)
  * while calling fdrop and free.
  */
 static void
-knote_drop(struct knote *kn, struct proc *p)
+knote_drop(struct knote *kn, __unused struct proc *ctxp)
 {
-        struct filedesc *fdp = p->p_fd;
 	struct kqueue *kq = kn->kn_kq;
+	struct proc *p = kq->kq_p;
+        struct filedesc *fdp = p->p_fd;
 	struct klist *list;
 
 	proc_fdlock(p);
@@ -1924,7 +1963,7 @@ knote_dequeue(struct knote *kn)
 {
 	struct kqueue *kq = kn->kn_kq;
 
-	assert((kn->kn_status & KN_DISABLED) == 0);
+	//assert((kn->kn_status & KN_DISABLED) == 0);
 	if ((kn->kn_status & KN_QUEUED) == KN_QUEUED) {
 		struct kqtailq *tq = kn->kn_tq;
 
@@ -1965,6 +2004,7 @@ knote_free(struct knote *kn)
 	zfree(knote_zone, kn);
 }
 
+#if SOCKETS
 #include <sys/param.h>
 #include <sys/socket.h>
 #include <sys/protosw.h>
@@ -2010,11 +2050,8 @@ struct kern_event_head kern_event_head;
 
 static u_long static_event_id = 0;
 struct domain *sysdom = &systemdomain;
+static lck_mtx_t *sys_mtx;
 
-static lck_grp_t		*evt_mtx_grp;
-static lck_attr_t		*evt_mtx_attr;
-static lck_grp_attr_t	*evt_mtx_grp_attr;
-lck_mtx_t				*evt_mutex;
 /*
  * Install the protosw's for the NKE manager.  Invoked at
  *  extension load time
@@ -2028,21 +2065,11 @@ kern_event_init(void)
     	    log(LOG_WARNING, "Can't install kernel events protocol (%d)\n", retval);
             return(retval);
 	}
-    
-	/*
- 	 * allocate lock group attribute and group for kern event 
-	 */
-    	evt_mtx_grp_attr = lck_grp_attr_alloc_init();
-
-	evt_mtx_grp = lck_grp_alloc_init("eventlist", evt_mtx_grp_attr);
-		
-	/*
-	 * allocate the lock attribute for mutexes
-	 */
-	evt_mtx_attr = lck_attr_alloc_init();
-  	evt_mutex = lck_mtx_alloc_init(evt_mtx_grp, evt_mtx_attr);
-	if (evt_mutex == NULL)
-			return (ENOMEM);
+   
+    /*
+     * Use the domain mutex for all system event sockets
+     */ 
+    sys_mtx = sysdom->dom_mtx;
 	
     return(KERN_SUCCESS);
 }
@@ -2065,9 +2092,9 @@ kev_attach(struct socket *so, __unused int proto, __unused struct proc *p)
      ev_pcb->vendor_code_filter = 0xffffffff;
 
      so->so_pcb = (caddr_t) ev_pcb;
-	 lck_mtx_lock(evt_mutex);
+     lck_mtx_lock(sys_mtx);
      LIST_INSERT_HEAD(&kern_event_head, ev_pcb, ev_link);
-	 lck_mtx_unlock(evt_mutex);
+     lck_mtx_unlock(sys_mtx);
 
      return 0;
 }
@@ -2079,9 +2106,7 @@ kev_detach(struct socket *so)
      struct kern_event_pcb *ev_pcb = (struct kern_event_pcb *) so->so_pcb;
 
      if (ev_pcb != 0) {
-		lck_mtx_lock(evt_mutex);
 		LIST_REMOVE(ev_pcb, ev_link);
-		lck_mtx_unlock(evt_mutex);
 		FREE(ev_pcb, M_PCB);
 		so->so_pcb = 0;
 		so->so_flags |= SOF_PCBCLEARING;
@@ -2091,27 +2116,23 @@ kev_detach(struct socket *so)
 }
 
 /*
- * For now, kev_vender_code and mbuf_tags use the same
+ * For now, kev_vendor_code and mbuf_tags use the same
  * mechanism.
  */
-extern errno_t mbuf_tag_id_find_internal(const char *string, u_long *out_id,
-										 int create);
 
 errno_t kev_vendor_code_find(
 	const char	*string,
-	u_long		*out_vender_code)
+	u_int32_t 	*out_vendor_code)
 {
 	if (strlen(string) >= KEV_VENDOR_CODE_MAX_STR_LEN) {
 		return EINVAL;
 	}
-	return mbuf_tag_id_find_internal(string, out_vender_code, 1);
+	return mbuf_tag_id_find_internal(string, out_vendor_code, 1);
 }
 
-extern void mbuf_tag_id_first_last(u_long *first, u_long *last);
-
 errno_t  kev_msg_post(struct kev_msg *event_msg)
 {
-	u_long	min_vendor, max_vendor;
+	mbuf_tag_id_t	min_vendor, max_vendor;
 	
 	mbuf_tag_id_first_last(&min_vendor, &max_vendor);
 	
@@ -2177,7 +2198,7 @@ int  kev_post_msg(struct kev_msg *event_msg)
      ev->event_code   = event_msg->event_code;
 
      m->m_len = total_size;
-     lck_mtx_lock(evt_mutex);
+     lck_mtx_lock(sys_mtx);
      for (ev_pcb = LIST_FIRST(&kern_event_head); 
 	  ev_pcb; 
 	  ev_pcb = LIST_NEXT(ev_pcb, ev_link)) {
@@ -2199,17 +2220,16 @@ int  kev_post_msg(struct kev_msg *event_msg)
 	  m2 = m_copym(m, 0, m->m_len, M_NOWAIT);
 	  if (m2 == 0) {
 	       m_free(m);
-	 	   lck_mtx_unlock(evt_mutex);
+	 	   lck_mtx_unlock(sys_mtx);
 	       return ENOBUFS;
 	  }
-	  socket_lock(ev_pcb->ev_socket, 1);
+	  /* the socket is already locked because we hold the sys_mtx here */
 	  if (sbappendrecord(&ev_pcb->ev_socket->so_rcv, m2))
 		  sorwakeup(ev_pcb->ev_socket);
-	  socket_unlock(ev_pcb->ev_socket, 1);
      }
 
      m_free(m);
-     lck_mtx_unlock(evt_mutex);
+     lck_mtx_unlock(sys_mtx);
      return 0;
 }
 
@@ -2262,20 +2282,21 @@ kev_control(struct socket *so,
 	return 0;
 }
 
+#endif /* SOCKETS */
 
 
 int
 fill_kqueueinfo(struct kqueue *kq, struct kqueue_info * kinfo)
 {
-	struct stat * st;
+	struct vinfo_stat * st;
 
 	/* No need for the funnel as fd is kept alive */
 	
 	st = &kinfo->kq_stat;
 
-	st->st_size = kq->kq_count;
-	st->st_blksize = sizeof(struct kevent);
-	st->st_mode = S_IFIFO;
+	st->vst_size = kq->kq_count;
+	st->vst_blksize = sizeof(struct kevent);
+	st->vst_mode = S_IFIFO;
 	if (kq->kq_state & KQ_SEL)
 		kinfo->kq_state |=  PROC_KQUEUE_SELECT;
 	if (kq->kq_state & KQ_SLEEP)
diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c
index 50e573131..ded4a1dcf 100644
--- a/bsd/kern/kern_exec.c
+++ b/bsd/kern/kern_exec.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -68,6 +74,12 @@
  *
  *	from: @(#)kern_exec.c	8.1 (Berkeley) 6/10/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 #include <machine/reg.h>
 
 #include <sys/param.h>
@@ -91,11 +103,18 @@
 #include <sys/signal.h>
 #include <sys/aio_kern.h>
 #include <sys/sysproto.h>
+#if SYSV_SHM
 #include <sys/shm_internal.h>		/* shmexec() */
+#endif
 #include <sys/ubc_internal.h>		/* ubc_map() */
+#include <sys/spawn.h>
+#include <sys/spawn_internal.h>
+#include <sys/codesign.h>
 
 #include <bsm/audit_kernel.h>
 
+#include <ipc/ipc_types.h>
+
 #include <mach/mach_types.h>
 #include <mach/task.h>
 #include <mach/thread_act.h>
@@ -103,12 +122,29 @@
 #include <mach/mach_vm.h>
 #include <mach/vm_param.h>
 
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_mach_internal.h>
+#endif
+
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
-#include <vm/vm_pager.h>
+#include <vm/vm_protos.h>
 #include <vm/vm_kern.h>
-#include <vm/task_working_set.h>
-#include <vm/vm_shared_memory_server.h>
+
+#if CONFIG_DTRACE
+/* Do not include dtrace.h, it redefines kmem_[alloc/free] */
+extern void (*dtrace_fasttrap_exec_ptr)(proc_t);
+extern void (*dtrace_helpers_cleanup)(proc_t);
+extern void dtrace_lazy_dofs_destroy(proc_t);
+
+#include <sys/dtrace_ptss.h>
+#endif
+
+/* support for child creation in exec after vfork */
+thread_t fork_create_child(task_t parent_task, proc_t child_proc, int inherit_memory, int is64bit);
+void vfork_exit(proc_t p, int rv);
+int setsigvec(proc_t, int, struct __user_sigaction *);
 
 /*
  * Mach things for which prototypes are unavailable from Mach headers
@@ -117,6 +153,12 @@ void		ipc_task_reset(
 			task_t		task);
 void		ipc_thread_reset(
 			thread_t	thread);
+kern_return_t ipc_object_copyin(
+	ipc_space_t		space,
+	mach_port_name_t	name,
+	mach_msg_type_name_t	msgt_name,
+	ipc_object_t		*objectp);
+void ipc_port_release_send(ipc_port_t);
 
 extern struct savearea *get_user_regs(thread_t);
 
@@ -128,11 +170,10 @@ extern struct savearea *get_user_regs(thread_t);
 #include <mach-o/fat.h>
 #include <mach-o/loader.h>
 #include <machine/vmparam.h>
-#if KTRACE   
-#include <sys/ktrace.h>
-#endif
 #include <sys/imgact.h>
 
+#include <sys/sdt.h>
+
 
 /*
  * SIZE_MAXPTR		The maximum size of a user space pointer, in bytes
@@ -144,7 +185,12 @@ extern struct savearea *get_user_regs(thread_t);
 #define	SIZE_MAXPTR		8				/* 64 bits */
 #define	SIZE_IMG_STRSPACE	(NCARGS - 2 * SIZE_MAXPTR)
 
-int	app_profile = 0;
+/*
+ * EAI_ITERLIMIT	The maximum number of times to iterate an image
+ *			activator in exec_activate_image() before treating
+ *			it as malformed/corrupt.
+ */
+#define EAI_ITERLIMIT		10
 
 extern vm_map_t bsd_pageable_map;
 extern struct fileops vnops;
@@ -154,6 +200,7 @@ extern struct fileops vnops;
 		  & ~(16 - 1) )
 
 struct image_params;	/* Forward */
+static int exec_activate_image(struct image_params *imgp);
 static int exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp);
 static int load_return_to_errno(load_return_t lrtn);
 static int execargs_alloc(struct image_params *imgp);
@@ -164,14 +211,13 @@ static int exec_handle_sugid(struct image_params *imgp);
 static int sugid_scripts = 0;
 SYSCTL_INT (_kern, OID_AUTO, sugid_scripts, CTLFLAG_RW, &sugid_scripts, 0, "");
 static kern_return_t create_unix_stack(vm_map_t map, user_addr_t user_stack,
-					int customstack, struct proc *p);
+					int customstack, proc_t p);
 static int copyoutptr(user_addr_t ua, user_addr_t ptr, int ptr_size);
+static void exec_resettextvp(proc_t, struct image_params *);
 
-/* XXX forward; should be in headers, but can't be for one reason or another */
-extern void vfork_return(thread_t th_act,
-				struct proc * p,
-				struct proc *p2,
-				register_t *retval);
+/* We don't want this one exported */
+__private_extern__
+int  open1(vfs_context_t, struct nameidata *, int, struct vnode_attr *, register_t *);
 
 /*
  * exec_add_string
@@ -180,7 +226,6 @@ extern void vfork_return(thread_t th_act,
  *
  * Parameters;	struct image_params *		image parameter block
  *		user_addr_t			string to add to strings area
- *		uio_seg				segment where string is located
  *
  * Returns:	0			Success
  *		!0			Failure errno from copyinstr()
@@ -190,7 +235,7 @@ extern void vfork_return(thread_t th_act,
  *		(imgp->ip_strspace)	updated byte count of space remaining
  */
 static int
-exec_add_string(struct image_params *imgp, user_addr_t str, /*uio_seg*/int seg)
+exec_add_string(struct image_params *imgp, user_addr_t str)
 {
         int error = 0;
 
@@ -200,7 +245,7 @@ exec_add_string(struct image_params *imgp, user_addr_t str, /*uio_seg*/int seg)
 			error = E2BIG;
 			break;
 		}
-		if (IS_UIO_SYS_SPACE(seg)) {
+		if (IS_UIO_SYS_SPACE(imgp->ip_seg)) {
 			char *kstr = CAST_DOWN(char *,str);	/* SAFE */
 			error = copystr(kstr, imgp->ip_strendp, imgp->ip_strspace, &len);
 		} else  {
@@ -228,10 +273,13 @@ exec_add_string(struct image_params *imgp, user_addr_t str, /*uio_seg*/int seg)
  *
  * Parameters;	struct image_params *		image parameter block
  *		char *				path used to invoke program
- *		uio_seg				segment where path is located
+ *		int				segment from which path comes
  *
  * Returns:	int			0	Success
- *					!0	Failure: error number
+ *		EFAULT				Bad address
+ *	copy[in]str:EFAULT			Bad address
+ *	copy[in]str:ENAMETOOLONG		Filename too long
+ *
  * Implicit returns:
  *		(imgp->ip_strings)		saved path
  *		(imgp->ip_strspace)		space remaining in ip_strings
@@ -246,7 +294,7 @@ exec_add_string(struct image_params *imgp, user_addr_t str, /*uio_seg*/int seg)
  *		unacceptable for dyld.
  */
 static int
-exec_save_path(struct image_params *imgp, user_addr_t path, /*uio_seg*/int seg)
+exec_save_path(struct image_params *imgp, user_addr_t path, int seg)
 {
 	int error;
 	size_t	len;
@@ -257,7 +305,7 @@ exec_save_path(struct image_params *imgp, user_addr_t path, /*uio_seg*/int seg)
 
 	len = MIN(MAXPATHLEN, imgp->ip_strspace);
 
-	switch( seg) {
+	switch(seg) {
 	case UIO_USERSPACE32:
 	case UIO_USERSPACE64:	/* Same for copyin()... */
 		error = copyinstr(path, imgp->ip_strings, len, &len);
@@ -316,13 +364,28 @@ exec_powerpc32_imgact(struct image_params *imgp)
 	 * Make sure it's a PowerPC binary.  If we've already redirected
 	 * from an interpreted file once, don't do it again.
 	 */
-	if (mach_header->magic != MH_CIGAM)
+	if (mach_header->magic != MH_CIGAM) {
+		/*
+		 * If it's a cross-architecture 64 bit binary, then claim
+		 * it, but refuse to run it.
+		 */
+		if (mach_header->magic == MH_CIGAM_64)
+			return (EBADARCH);
 		return (-1);
+	}
 
 	/* If there is no exec_archhandler_ppc, we can't run it */
 	if (exec_archhandler_ppc.path[0] == 0)
 		return (EBADARCH);
 
+	/* Remember the type of the original file for later grading */
+	if (!imgp->ip_origcputype) {
+		imgp->ip_origcputype = 
+			OSSwapBigToHostInt32(mach_header->cputype);
+		imgp->ip_origcpusubtype = 
+			OSSwapBigToHostInt32(mach_header->cpusubtype);
+	}
+
 	/*
 	 * The PowerPC flag will be set by the exec_check_permissions()
 	 * call anyway; however, we set this flag here so that the relookup
@@ -342,8 +405,7 @@ exec_powerpc32_imgact(struct image_params *imgp)
 	 * since the exec may fail and return to the parent.  In that case,
 	 * we would have erroneously changed the parent p->p_comm instead.
 	 */
-	strncpy(imgp->ip_p_comm, imgp->ip_ndp->ni_cnd.cn_nameptr, MAXCOMLEN);
-	imgp->ip_p_comm[MAXCOMLEN] = '\0';
+	strlcpy(imgp->ip_p_comm, imgp->ip_ndp->ni_cnd.cn_nameptr, MAXCOMLEN);
 
 	return (-3);
 }
@@ -378,6 +440,12 @@ exec_shell_imgact(struct image_params *imgp)
 	char *ihp;
 	char *line_endp;
 	char *interp;
+	char temp[16];
+	proc_t p;
+	struct fileproc *fp;
+	int fd;
+	int error;
+	size_t len;
 
 	/*
 	 * Make sure it's a shell script.  If we've already redirected
@@ -459,6 +527,34 @@ exec_shell_imgact(struct image_params *imgp)
 		imgp->ip_argc++;
 	}
 
+	/*
+	 * If we have a SUID oder SGID script, create a file descriptor
+	 * from the vnode and pass /dev/fd/%d instead of the actual
+	 * path name so that the script does not get opened twice
+	 */
+	if (imgp->ip_origvattr->va_mode & (VSUID | VSGID)) {
+		p = vfs_context_proc(imgp->ip_vfs_context);
+		error = falloc(p, &fp, &fd, imgp->ip_vfs_context);
+		if (error)
+			return(error);
+
+		fp->f_fglob->fg_flag = FREAD;
+		fp->f_fglob->fg_type = DTYPE_VNODE;
+		fp->f_fglob->fg_ops = &vnops;
+		fp->f_fglob->fg_data = (caddr_t)imgp->ip_vp;
+		
+		proc_fdlock(p);
+		procfdtbl_releasefd(p, fd, NULL);
+		fp_drop(p, fd, fp, 1);
+		proc_fdunlock(p);
+		vnode_ref(imgp->ip_vp);
+
+		snprintf(temp, sizeof(temp), "/dev/fd/%d", fd);
+		error = copyoutstr(temp, imgp->ip_user_fname, sizeof(temp), &len);
+		if (error)
+			return(error);
+	}
+
 	return (-3);
 }
 
@@ -473,9 +569,19 @@ exec_shell_imgact(struct image_params *imgp)
  * reloading the first page for the image with a first page from the
  * offset location indicated by the fat header.
  *
+ * Parameters;	struct image_params *	image parameter block
+ *
+ * Returns:	-1			not a fat binary (keep looking)
+ *		-2			Success: encapsulated binary: reread
+ *		>0			Failure: error number
+ *
  * Important:	This image activator is byte order neutral.
  *
- * Note:	If we find an encapsulated binary, we make no assertions
+ * Note:	A return value other than -1 indicates subsequent image
+ *		activators should not be given the opportunity to attempt
+ *		to activate the image.
+ *
+ * 		If we find an encapsulated binary, we make no assertions
  *		about its  validity; instead, we leave that up to a rescan
  *		for an activator to claim it, and, if it is claimed by one,
  *		that activator is responsible for determining validity.
@@ -483,9 +589,10 @@ exec_shell_imgact(struct image_params *imgp)
 static int
 exec_fat_imgact(struct image_params *imgp)
 {
-	struct proc *p = vfs_context_proc(imgp->ip_vfs_context);
+	proc_t p = vfs_context_proc(imgp->ip_vfs_context);
 	kauth_cred_t cred = kauth_cred_proc_ref(p);
 	struct fat_header *fat_header = (struct fat_header *)imgp->ip_vdata;
+	struct _posix_spawnattr *psa = NULL;
 	struct fat_arch fat_arch;
 	int resid, error;
 	load_return_t lret;
@@ -497,6 +604,49 @@ exec_fat_imgact(struct image_params *imgp)
 		goto bad;
 	}
 
+	/* If posix_spawn binprefs exist, respect those prefs. */
+	psa = (struct _posix_spawnattr *) imgp->ip_px_sa;
+	if (psa != NULL && psa->psa_binprefs[0] != 0) {
+		struct fat_arch *arches = (struct fat_arch *) (fat_header + 1);
+		int nfat_arch = 0, pr = 0, f = 0;
+
+		nfat_arch = OSSwapBigToHostInt32(fat_header->nfat_arch);
+		/* Check each preference listed against all arches in header */
+		for (pr = 0; pr < NBINPREFS; pr++) {
+			cpu_type_t pref = psa->psa_binprefs[pr];
+			if (pref == 0) {
+				/* No suitable arch in the pref list */
+				error = EBADARCH;
+				goto bad;
+			}
+
+			if (pref == CPU_TYPE_ANY) {
+				/* Fall through to regular grading */
+				break;
+			}
+
+			for (f = 0; f < nfat_arch; f++) {
+				cpu_type_t archtype = OSSwapBigToHostInt32(
+						arches[f].cputype);
+				cpu_type_t archsubtype = OSSwapBigToHostInt32(
+						arches[f].cpusubtype) & ~CPU_SUBTYPE_MASK;
+				if (pref == archtype &&
+					grade_binary(archtype, archsubtype)) {
+					/* We have a winner! */
+					fat_arch.cputype = archtype; 
+					fat_arch.cpusubtype = archsubtype; 
+					fat_arch.offset = OSSwapBigToHostInt32(
+							arches[f].offset);
+					fat_arch.size = OSSwapBigToHostInt32(
+							arches[f].size);
+					fat_arch.align = OSSwapBigToHostInt32(
+							arches[f].align);
+					goto use_arch;
+				}
+			}
+		}
+	}
+
 	/* Look up our preferred architecture in the fat file. */
 	lret = fatfile_getarch_affinity(imgp->ip_vp,
 					(vm_offset_t)fat_header,
@@ -507,7 +657,8 @@ exec_fat_imgact(struct image_params *imgp)
 		goto bad;
 	}
 
-	/* Read the Mach-O header out of it */
+use_arch:
+	/* Read the Mach-O header out of fat_arch */
 	error = vn_rdwr(UIO_READ, imgp->ip_vp, imgp->ip_vdata,
 			PAGE_SIZE, fat_arch.offset,
 			UIO_SYSSPACE32, (IO_UNIT|IO_NODELOCKED),
@@ -537,30 +688,41 @@ bad:
  *
  * Image activator for mach-o 1.0 binaries.
  *
+ * Parameters;	struct image_params *	image parameter block
+ *
+ * Returns:	-1			not a fat binary (keep looking)
+ *		-2			Success: encapsulated binary: reread
+ *		>0			Failure: error number
+ *		EBADARCH		Mach-o binary, but with an unrecognized
+ *					architecture
+ *		ENOMEM			No memory for child process after -
+ *					can only happen after vfork()
+ *
  * Important:	This image activator is NOT byte order neutral.
+ *
+ * Note:	A return value other than -1 indicates subsequent image
+ *		activators should not be given the opportunity to attempt
+ *		to activate the image.
+ *
+ * TODO:	More gracefully handle failures after vfork
  */
 static int
 exec_mach_imgact(struct image_params *imgp)
 {
 	struct mach_header *mach_header = (struct mach_header *)imgp->ip_vdata;
-	kauth_cred_t		cred = vfs_context_ucred(imgp->ip_vfs_context);
-	struct proc		*p = vfs_context_proc(imgp->ip_vfs_context);
+	proc_t			p = vfs_context_proc(imgp->ip_vfs_context);
 	int			error = 0;
 	int			vfexec = 0;
 	task_t			task;
-	task_t			new_task;
+	task_t			new_task = NULL; /* protected by vfexec */
 	thread_t		thread;
 	struct uthread		*uthread;
 	vm_map_t old_map = VM_MAP_NULL;
 	vm_map_t map;
-	boolean_t				clean_regions = FALSE;
 	load_return_t		lret;
 	load_result_t		load_result;
-	shared_region_mapping_t shared_region, initial_region;
-#ifdef IMGPF_POWERPC
-	int powerpcParent, powerpcImage;
-#endif /* IMGPF_POWERPC */
-		
+	struct _posix_spawnattr *psa = NULL;
+
 	/*
 	 * make sure it's a Mach-O 1.0 or Mach-O 2.0 binary; the difference
 	 * is a reserved field on the end, so for the most part, we can
@@ -568,10 +730,22 @@ exec_mach_imgact(struct image_params *imgp)
 	 */
 	if ((mach_header->magic != MH_MAGIC) &&
 	    (mach_header->magic != MH_MAGIC_64)) {
-	error = -1;
+		error = -1;
+		goto bad;
+	}
+
+	switch (mach_header->filetype) {
+	case MH_DYLIB:
+	case MH_BUNDLE:
+		error = -1;
 		goto bad;
 	}
 
+	if (!imgp->ip_origcputype) {
+		imgp->ip_origcputype = mach_header->cputype;
+		imgp->ip_origcpusubtype = mach_header->cpusubtype;
+	}
+
 	task = current_task();
 	thread = current_thread();
 	uthread = get_bsdthread_info(thread);
@@ -582,22 +756,43 @@ exec_mach_imgact(struct image_params *imgp)
 	if ((mach_header->cputype & CPU_ARCH_ABI64) == CPU_ARCH_ABI64)
 		imgp->ip_flags |= IMGPF_IS_64BIT;
 
-	if (!grade_binary(mach_header->cputype, mach_header->cpusubtype)) {
+	/* If posix_spawn binprefs exist, respect those prefs. */
+	psa = (struct _posix_spawnattr *) imgp->ip_px_sa;
+	if (psa != NULL && psa->psa_binprefs[0] != 0) {
+		int pr = 0;
+		for (pr = 0; pr < NBINPREFS; pr++) {
+			cpu_type_t pref = psa->psa_binprefs[pr];
+			if (pref == 0) {
+				/* No suitable arch in the pref list */
+				error = EBADARCH;
+				goto bad;
+			}
+
+			if (pref == CPU_TYPE_ANY) {
+				/* Jump to regular grading */
+				goto grade;
+			}
+
+			if (pref == imgp->ip_origcputype) {
+				/* We have a match! */
+				goto grade;
+			}
+		}
 		error = EBADARCH;
 		goto bad;
 	}
-
-	/*
-	 * Copy in arguments/environment from the old process, if the
-	 * vector is non-NULL (i.e. exec is not being called from
-	 * load_init_program(), as a special case, at system startup).
-	 */
-	if (imgp->ip_user_argv != 0LL) {
-		error = exec_extract_strings(imgp);
-		if (error)
-			goto bad;
+grade:
+	if (!grade_binary(imgp->ip_origcputype & ~CPU_SUBTYPE_LIB64, 
+				imgp->ip_origcpusubtype & ~CPU_SUBTYPE_MASK)) {
+		error = EBADARCH;
+		goto bad;
 	}
 
+	/* Copy in arguments/environment from the old process */
+	error = exec_extract_strings(imgp);
+	if (error)
+		goto bad;
+
 	/*
 	 * Hack for binary compatability; put three NULs on the end of the
 	 * string area, and round it up to the next word boundary.  This
@@ -615,7 +810,7 @@ exec_mach_imgact(struct image_params *imgp)
 	 * Should be factored out; this is here because we might be getting
 	 * invoked this way as the result of a shell script, and the check
 	 * in exec_check_permissions() is not interior to the jump back up
-	 * to the "encapsulated_binary:" label in execve().
+	 * to the "encapsulated_binary:" label in exec_activate_image().
 	 */
 	if (imgp->ip_vattr->va_fsid == exec_archhandler_ppc.fsid &&
 		imgp->ip_vattr->va_fileid == (uint64_t)((u_long)exec_archhandler_ppc.fileid)) {
@@ -624,29 +819,16 @@ exec_mach_imgact(struct image_params *imgp)
 #endif	/* IMGPF_POWERPC */
 
 	if (vfexec) {
- 		kern_return_t	result;
-
-		result = task_create_internal(task, FALSE, (imgp->ip_flags & IMGPF_IS_64BIT), &new_task);
-		if (result != KERN_SUCCESS)
-	    	printf("execve: task_create failed. Code: 0x%x\n", result);
-		p->task = new_task;
-		set_bsdtask_info(new_task, p);
-		if (p->p_nice != 0)
-			resetpriority(p);
-		map = get_task_map(new_task);
-
-		if (imgp->ip_flags & IMGPF_IS_64BIT)
-		        vm_map_set_64bit(map);
-		else
-		        vm_map_set_32bit(map);
-
-		result = thread_create(new_task, &imgp->ip_vfork_thread);
-		if (result != KERN_SUCCESS)
-	    	printf("execve: thread_create failed. Code: 0x%x\n", result);
-		/* reset local idea of task, thread, uthread */
-		task = new_task;
+		imgp->ip_vfork_thread = fork_create_child(task, p, FALSE, (imgp->ip_flags & IMGPF_IS_64BIT));
+		if (imgp->ip_vfork_thread == NULL) {
+			error = ENOMEM;
+			goto bad;
+		}
+		/* reset local idea of thread, uthread, task */
 		thread = imgp->ip_vfork_thread;
 		uthread = get_bsdthread_info(thread);
+		task = new_task = get_threadtask(thread);
+		map = get_task_map(task);
 	} else {
 		map = VM_MAP_NULL;
 	}
@@ -657,111 +839,15 @@ exec_mach_imgact(struct image_params *imgp)
 	 */
 	if (imgp->ip_flags & IMGPF_IS_64BIT) {
 		task_set_64bit(task, TRUE);
-		p->p_flag |= P_LP64;
+		OSBitOrAtomic(P_LP64, (UInt32 *)&p->p_flag);
 	} else {
 		task_set_64bit(task, FALSE);
-		p->p_flag &= ~P_LP64;
+		OSBitAndAtomic(~((uint32_t)P_LP64), (UInt32 *)&p->p_flag);
 	}
 
 	/*
 	 *	Load the Mach-O file.
 	 */
-/* LP64 - remove following "if" statement after osfmk/vm/task_working_set.c */
-if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
-	if(imgp->ip_tws_cache_name) {
-		tws_handle_startup_file(task, kauth_cred_getuid(cred), 
-			imgp->ip_tws_cache_name, imgp->ip_vp, &clean_regions);
-	}
-
-	vm_get_shared_region(task, &initial_region);
-
-#ifdef IMGPF_POWERPC
-	/*
-	 * If we are transitioning to/from powerpc, then we need to do extra
-	 * work here.
-	 */
-	powerpcParent = (p->p_flag & P_TRANSLATED) ? 1 : 0;
-	powerpcImage = (imgp->ip_flags & IMGPF_POWERPC) ? 1 : 0;
-
-	if (powerpcParent ^ powerpcImage) {
-		cpu_type_t cpu = (powerpcImage ? CPU_TYPE_POWERPC : cpu_type());
-		struct vnode *rootDir = p->p_fd->fd_rdir;
-
-		shared_region = lookup_default_shared_region((int)rootDir, cpu);
-		if (shared_region == NULL) {
-			shared_region_mapping_t old_region;
-			shared_region_mapping_t new_region;
-			vm_get_shared_region(current_task(), &old_region);
-			/* grrrr... this sets current_task(), not task
-			* -- they're different (usually)
-			*/
-			shared_file_boot_time_init((int)rootDir,cpu);
-			if ( current_task() != task ) {
-				vm_get_shared_region(current_task(),&new_region);
-				vm_set_shared_region(task,new_region);
-				vm_set_shared_region(current_task(),old_region);
-			}
-		} else {
-			vm_set_shared_region(task, shared_region);
-		}
-		shared_region_mapping_dealloc(initial_region);
-	} else 
-#endif	/* IMGPF_POWERPC */
-
-	{
-		struct shared_region_task_mappings map_info;
-		shared_region_mapping_t next;
-
-		shared_region_mapping_info(initial_region,
-					   &map_info.text_region,
-					   &map_info.text_size,
-					   &map_info.data_region,
-					   &map_info.data_size,
-					   &map_info.region_mappings,
-					   &map_info.client_base,
-					   &map_info.alternate_base,
-					   &map_info.alternate_next,
-					   &map_info.fs_base,
-					   &map_info.system,
-					   &map_info.flags,
-					   &next);
-		if (map_info.flags & SHARED_REGION_STANDALONE) {
-			/*
-			 * We were using a private shared region.
-			 * Try and get back to a system-wide shared region
-			 * with matching "fs_base" (for chroot) and "system"
-			 * (for CPU type).
-			 */
-			shared_region = lookup_default_shared_region(
-				map_info.fs_base,
-				map_info.system);
-			if (shared_region == NULL) {
-				/*
-				 * No system-wide default regions, stick to
-				 * our private region...
-				 */
-			} else {
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_INFO,
-					("shared_region: %p [%d(%s)] "
-					 "exec(\"%s\"): "
-					 "moving from private %p[%x,%x,%x] "
-					 "to default %p\n",
-					 current_thread(),
-					 p->p_pid, p->p_comm,
-					 (imgp->ip_p_comm[0] ?
-					  imgp->ip_p_comm :
-					  imgp->ip_ndp->ni_cnd.cn_nameptr),
-					 initial_region,
-					 map_info.fs_base,
-					 map_info.system,
-					 map_info.flags,
-					 shared_region));
-				vm_set_shared_region(task, shared_region);
-				shared_region_mapping_dealloc(initial_region);
-			}
-		}
-	}
 
 	/*
 	 * NOTE: An error after this point  indicates we have potentially
@@ -780,16 +866,48 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	/*
 	 * Actually load the image file we previously decided to load.
 	 */
-	lret = load_machfile(imgp, mach_header, thread, map, clean_regions, &load_result);
+	lret = load_machfile(imgp, mach_header, thread, map, &load_result);
 
 	if (lret != LOAD_SUCCESS) {
 		error = load_return_to_errno(lret);
 		goto badtoolate;
 	}
 
+	vm_map_set_user_wire_limit(get_task_map(task), p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur);
+
+	/* 
+	 * Set code-signing flags if this binary is signed, or if parent has
+	 * requested them on exec.
+	 */
+	if (load_result.csflags & CS_VALID) {
+		imgp->ip_csflags |= load_result.csflags &
+			(CS_VALID|
+			 CS_HARD|CS_KILL|CS_EXEC_SET_HARD|CS_EXEC_SET_KILL);
+	} else {
+		imgp->ip_csflags &= ~CS_VALID;
+	}
+
+	if (p->p_csflags & CS_EXEC_SET_HARD)
+		imgp->ip_csflags |= CS_HARD;
+	if (p->p_csflags & CS_EXEC_SET_KILL)
+		imgp->ip_csflags |= CS_KILL;
+
+
 	/* load_machfile() maps the vnode */
-	(void)ubc_map(imgp->ip_vp, PROT_EXEC);
- 
+	(void)ubc_map(imgp->ip_vp, PROT_READ | PROT_EXEC);
+
+	/*
+	 * Set up the system reserved areas in the new address space.
+	 */
+	vm_map_exec(get_task_map(task),
+		    task,
+		    (void *) p->p_fd->fd_rdir,
+#ifdef IMGPF_POWERPC
+		    imgp->ip_flags & IMGPF_POWERPC ?
+		    CPU_TYPE_POWERPC :
+#endif
+		    cpu_type());
+	
 	/*
 	 * Close file descriptors
 	 * which specify close-on-exec.
@@ -801,20 +919,20 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	 */
 	error = exec_handle_sugid(imgp);
 
-	KNOTE(&p->p_klist, NOTE_EXEC);
+	proc_knote(p, NOTE_EXEC);
 
-	if (!vfexec && (p->p_flag & P_TRACED))
+	if (!vfexec && (p->p_lflag & P_LTRACED))
 		psignal(p, SIGTRAP);
 
 	if (error) {
 		goto badtoolate;
 	}
-	vnode_put(imgp->ip_vp);
-	imgp->ip_vp = NULL;
 	
 	if (load_result.unixproc &&
 		create_unix_stack(get_task_map(task),
-				  load_result.user_stack, load_result.customstack, p)) {
+				  load_result.user_stack,
+				  load_result.customstack,
+				  p) != KERN_SUCCESS) {
 		error = load_return_to_errno(LOAD_NOSPACE);
 		goto badtoolate;
 	}
@@ -853,8 +971,8 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 			error = suword(ap, load_result.mach_header);
 		}
 		if (error) {
-			if (vfexec)
-				vm_map_switch(old_map);
+		        if (vfexec)
+			        vm_map_switch(old_map);
 			goto badtoolate;
 		}
 	}
@@ -879,11 +997,15 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	 */
 	_aio_exec( p );
 
+#if SYSV_SHM
 	/* FIXME: Till vmspace inherit is fixed: */
 	if (!vfexec && p->vm_shm)
 		shmexec(p);
+#endif
+#if SYSV_SEM
 	/* Clean up the semaphores */
 	semexit(p);
+#endif
 
 	/*
 	 * Remember file name for accounting.
@@ -904,28 +1026,61 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 		p->p_comm[imgp->ip_ndp->ni_cnd.cn_namelen] = '\0';
 	}
 
+#if CONFIG_DTRACE
+	/*
+	 * Invalidate any predicate evaluation already cached for this thread by DTrace.
+	 * That's because we've just stored to p_comm and DTrace refers to that when it
+	 * evaluates the "execname" special variable. uid and gid may have changed as well.
+	 */
+	dtrace_set_thread_predcache(current_thread(), 0);
+
+	/*
+	 * Free any outstanding lazy dof entries. It is imperative we
+	 * always call dtrace_lazy_dofs_destroy, rather than null check
+	 * and call if !NULL. If we NULL test, during lazy dof faulting
+	 * we can race with the faulting code and proceed from here to
+	 * beyond the helpers cleanup. The lazy dof faulting will then
+	 * install new helpers which no longer belong to this process!
+	 */
+	dtrace_lazy_dofs_destroy(p);
+
+
+	/*
+    	 * Clean up any DTrace helpers for the process.
+    	 */
+    	if (p->p_dtrace_helpers != NULL && dtrace_helpers_cleanup) {
+    		(*dtrace_helpers_cleanup)(p);
+    	}
+	
+    	/*
+    	 * Cleanup the DTrace provider associated with this process.
+    	 */
+	proc_lock(p);
+	if (p->p_dtrace_probes && dtrace_fasttrap_exec_ptr) {
+    		(*dtrace_fasttrap_exec_ptr)(p);
+    	}
+	proc_unlock(p);
+#endif
+
 	if (kdebug_enable) {
-	          long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
-
-	          /*
-		   * Collect the pathname for tracing
-		   */
-	          kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
-
-		  if (vfexec)
-		  {
-		          KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
-						 p->p_pid ,0,0,0, (unsigned int)thread);
-			  KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
-						 dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (unsigned int)thread);
-		  }
-		  else
-		  {
-		          KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
-						p->p_pid ,0,0,0,0);
-			  KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
-						dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
-		  }
+		long dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4;
+
+		/*
+		 * Collect the pathname for tracing
+		 */
+		kdbg_trace_string(p, &dbg_arg1, &dbg_arg2, &dbg_arg3, &dbg_arg4);
+
+		if (vfexec) {
+			KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
+					p->p_pid ,0,0,0, (unsigned int)thread);
+			KERNEL_DEBUG_CONSTANT1((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
+					dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, (unsigned int)thread);
+		} else {
+			KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_DATA, 2)) | DBG_FUNC_NONE,
+					p->p_pid ,0,0,0,0);
+			KERNEL_DEBUG_CONSTANT((TRACEDBG_CODE(DBG_TRACE_STRING, 2)) | DBG_FUNC_NONE,
+					dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
+		}
 	}
 
 #ifdef IMGPF_POWERPC
@@ -935,23 +1090,39 @@ if((imgp->ip_flags & IMGPF_IS_64BIT) == 0)
 	 * from the process.
 	 */
 	if (((imgp->ip_flags & IMGPF_POWERPC) != 0))
-		p->p_flag |= P_TRANSLATED;
+		OSBitOrAtomic(P_TRANSLATED, (UInt32 *)&p->p_flag);
 	else
 #endif	/* IMGPF_POWERPC */
-		p->p_flag &= ~P_TRANSLATED;
-	p->p_flag &= ~P_AFFINITY;
+		OSBitAndAtomic(~((uint32_t)P_TRANSLATED), (UInt32 *)&p->p_flag);
+	OSBitAndAtomic(~((uint32_t)P_AFFINITY), (UInt32 *)&p->p_flag);
+
+	/*
+	 * If posix_spawned with the START_SUSPENDED flag, stop the
+	 * process before it runs.
+	 */
+	if (imgp->ip_px_sa != NULL) {
+		psa = (struct _posix_spawnattr *) imgp->ip_px_sa;
+		if (psa->psa_flags & POSIX_SPAWN_START_SUSPENDED) {
+			proc_lock(p);
+			p->p_stat = SSTOP;
+			proc_unlock(p);
+			(void) task_suspend(p->task);
+		}
+	}
 
 	/*
 	 * mark as execed, wakeup the process that vforked (if any) and tell
 	 * it that it now has it's own resources back
 	 */
-	p->p_flag |= P_EXEC;
-	if (p->p_pptr && (p->p_flag & P_PPWAIT)) {
-		p->p_flag &= ~P_PPWAIT;
+	OSBitOrAtomic(P_EXEC, (UInt32 *)&p->p_flag);
+	if (p->p_pptr && (p->p_lflag & P_LPPWAIT)) {
+		proc_lock(p);
+		p->p_lflag &= ~P_LPPWAIT;
+		proc_unlock(p);
 		wakeup((caddr_t)p->p_pptr);
 	}
 
-	if (vfexec && (p->p_flag & P_TRACED)) {
+	if (vfexec && (p->p_lflag & P_LTRACED)) {
 		psignal_vfork(p, new_task, thread, SIGTRAP);
 	}
 
@@ -992,110 +1163,59 @@ struct execsw {
 
 
 /*
- * TODO:	Dynamic linker header address on stack is copied via suword()
+ * exec_activate_image
+ *
+ * Description:	Iterate through the available image activators, and activate
+ *		the image associated with the imgp structure.  We start with
+ *		the
+ *
+ * Parameters:	struct image_params *	Image parameter block
+ *
+ * Returns:	0			Success
+ *		EBADEXEC		The executable is corrupt/unknown
+ *	execargs_alloc:EINVAL		Invalid argument
+ *	execargs_alloc:EACCES		Permission denied
+ *	execargs_alloc:EINTR		Interrupted function
+ *	execargs_alloc:ENOMEM		Not enough space
+ *	exec_save_path:EFAULT		Bad address
+ *	exec_save_path:ENAMETOOLONG	Filename too long
+ *	exec_check_permissions:EACCES	Permission denied
+ *	exec_check_permissions:ENOEXEC	Executable file format error
+ *	exec_check_permissions:ETXTBSY	Text file busy [misuse of error code]
+ *	exec_check_permissions:???
+ *	namei:???
+ *	vn_rdwr:???			[anything vn_rdwr can return]
+ *	<ex_imgact>:???			[anything an imgact can return]
  */
-/* ARGSUSED */
-int
-execve(struct proc *p, struct execve_args *uap, register_t *retval)
+static int
+exec_activate_image(struct image_params *imgp)
 {
-	kauth_cred_t cred = kauth_cred_proc_ref(p);
-	struct image_params image_params, *imgp;
-	struct vnode_attr va;
-	struct vnode_attr origva;
 	struct nameidata nd;
-	struct uthread		*uthread;
-	int i;
-	int resid, error;
-	task_t  task;
-	int numthreads;
-	int vfexec=0;
+	int error;
+	int resid;
 	int once = 1;	/* save SGUID-ness for interpreted files */
-	char alt_p_comm[sizeof(p->p_comm)] = {0};	/* for PowerPC */
-	int is_64 = IS_64BIT_PROCESS(p);
-	int seg = (is_64 ? UIO_USERSPACE64 : UIO_USERSPACE32);
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = cred;	/* XXX must NOT be kauth_cred_get() */
-
-
-	imgp = &image_params;
-
-	/* Initialize the common data in the image_params structure */
-	bzero(imgp, sizeof(*imgp));
-	imgp->ip_user_fname = uap->fname;
-	imgp->ip_user_argv = uap->argp;
-	imgp->ip_user_envv = uap->envp;
-	imgp->ip_vattr = &va;
-	imgp->ip_origvattr = &origva;
-	imgp->ip_vfs_context = &context;
-	imgp->ip_flags = (is_64 ? IMGPF_WAS_64BIT : IMGPF_NONE);
-	imgp->ip_tws_cache_name = NULL;
-	imgp->ip_p_comm = alt_p_comm;		/* for PowerPC */
-
-	/*
-         * XXXAUDIT: Currently, we only audit the pathname of the binary.
-         * There may also be poor interaction with dyld.
-         */
-
-	task = current_task();
-	uthread = get_bsdthread_info(current_thread());
-
-	if (uthread->uu_flag & UT_VFORK) {
-			vfexec = 1; /* Mark in exec */
-	} else {
-		if (task != kernel_task) { 
-			numthreads = get_task_numacts(task);
-			if (numthreads <= 0 ) {
-			        kauth_cred_unref(&cred);
-				return(EINVAL);
-			}
-			if (numthreads > 1) {
-			        kauth_cred_unref(&cred);
-				return(ENOTSUP);
-			}
-		}
-	}
+	int i;
+	int iterlimit = EAI_ITERLIMIT;
 
 	error = execargs_alloc(imgp);
-	if (error) {
-	        kauth_cred_unref(&cred);
-		return(error);
-	}
+	if (error)
+		goto bad;
+	
 	/*
 	 * XXXAUDIT: Note: the double copyin introduces an audit
 	 * race.  To correct this race, we must use a single
 	 * copyin(), e.g. by passing a flag to namei to indicate an
 	 * external path buffer is being used.
 	 */
-	error = exec_save_path(imgp, uap->fname, seg);
+	error = exec_save_path(imgp, imgp->ip_user_fname, imgp->ip_seg);
 	if (error) {
-		execargs_free(imgp);
-	        kauth_cred_unref(&cred);
-		return(error);
+		goto bad;
 	}
 
-	/*
-	 * No app profiles under chroot
-	 */
-	if((p->p_fd->fd_rdir == NULLVP) && (app_profile != 0)) {
-
-		/* grab the name of the file out of its path */
-		/* we will need this for lookup within the   */
-		/* name file */
-		/* Scan backwards for the first '/' or start of string */
-		imgp->ip_tws_cache_name = imgp->ip_strendp;
-               	while (imgp->ip_tws_cache_name[0] != '/') {
-               		if(imgp->ip_tws_cache_name == imgp->ip_strings) {
-               	        	imgp->ip_tws_cache_name--;
-               	         	break;
-                      	}
-               		imgp->ip_tws_cache_name--;
-               	}
-               	imgp->ip_tws_cache_name++;
-	}
+	DTRACE_PROC1(exec, uintptr_t, imgp->ip_strings);
+
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
-		seg, uap->fname, imgp->ip_vfs_context);
+		imgp->ip_seg, imgp->ip_user_fname, imgp->ip_vfs_context);
 
 again:
 	error = namei(&nd);
@@ -1111,15 +1231,22 @@ again:
 	/* Copy; avoid invocation of an interpreter overwriting the original */
 	if (once) {
 		once = 0;
-		origva = va;
+		*imgp->ip_origvattr = *imgp->ip_vattr;
 	}
 
 	error = vn_rdwr(UIO_READ, imgp->ip_vp, imgp->ip_vdata, PAGE_SIZE, 0,
-			UIO_SYSSPACE32, IO_NODELOCKED, cred, &resid, p);
+			UIO_SYSSPACE32, IO_NODELOCKED,
+			vfs_context_ucred(imgp->ip_vfs_context),
+			&resid, vfs_context_proc(imgp->ip_vfs_context));
 	if (error)
 		goto bad;
 		
 encapsulated_binary:
+	/* Limit the number of iterations we will attempt on each binary */
+	if (--iterlimit == 0) {
+		error = EBADEXEC;
+		goto bad;
+	}
 	error = -1;
 	for(i = 0; error == -1 && execsw[i].ex_imgact != NULL; i++) {
 
@@ -1131,6 +1258,22 @@ encapsulated_binary:
 			goto encapsulated_binary;
 
 		case -3:		/* Interpreter */
+#if CONFIG_MACF
+			/*
+			 * Copy the script label for later use. Note that
+			 * the label can be different when the script is
+			 * actually read by the interpreter.
+			 */
+			if (imgp->ip_scriptlabelp)
+				mac_vnode_label_free(imgp->ip_scriptlabelp);
+			imgp->ip_scriptlabelp = mac_vnode_label_alloc();
+			if (imgp->ip_scriptlabelp == NULL) {
+				error = ENOMEM;
+				break;
+			}
+			mac_vnode_label_copy(imgp->ip_vp->v_label,
+				    imgp->ip_scriptlabelp);
+#endif
 			vnode_put(imgp->ip_vp);
 			imgp->ip_vp = NULL;	/* already put */
 			nd.ni_cnd.cn_nameiop = LOOKUP;
@@ -1155,58 +1298,780 @@ encapsulated_binary:
 			break;
 		}
 	}
-	
-	/* call out to allow 3rd party notification of exec. 
+
+	/*
+	 * Call out to allow 3rd party notification of exec. 
 	 * Ignore result of kauth_authorize_fileop call.
 	 */
 	if (error == 0 && kauth_authorize_fileop_has_listeners()) {
-		kauth_authorize_fileop(vfs_context_ucred(&context), KAUTH_FILEOP_EXEC, 
-							   (uintptr_t)nd.ni_vp, 0);
+		kauth_authorize_fileop(vfs_context_ucred(imgp->ip_vfs_context),
+					KAUTH_FILEOP_EXEC,
+					(uintptr_t)nd.ni_vp, 0);
 	}
-	
-	/* Image not claimed by any activator? */
-	if (error == -1)
-		error = ENOEXEC;
 
 bad:
-	kauth_cred_unref(&cred);
-
-	if (imgp->ip_ndp)
-		nameidone(imgp->ip_ndp);
-	if (imgp->ip_vp)
-		vnode_put(imgp->ip_vp);
 	if (imgp->ip_strings)
 		execargs_free(imgp);
-	if (!error && vfexec) {
-			vfork_return(current_thread(), p->p_pptr, p, retval);
-			(void)thread_resume(imgp->ip_vfork_thread);
-			return(0);
-	}
-	return(error);
-}
+	if (imgp->ip_ndp)
+		nameidone(imgp->ip_ndp);
 
+	return (error);
+}
 
+/*
+ * exec_handle_port_actions
+ *
+ * Description:	Go through the _posix_port_actions_t contents, 
+ * 		calling task_set_special_port and task_set_exception_ports
+ * 		for the current task.
+ *
+ * Parameters:	struct image_params *	Image parameter block
+ *
+ * Returns:	0			Success
+ * 		KERN_FAILURE		Failure
+ */
 static int
-copyinptr(user_addr_t froma, user_addr_t *toptr, int ptr_size)
+exec_handle_port_actions(struct image_params *imgp)
 {
-	int error;
+	_posix_spawn_port_actions_t pacts = imgp->ip_px_spa;
+	proc_t p = vfs_context_proc(imgp->ip_vfs_context);
+	_ps_port_action_t *act = NULL;
+	task_t task = p->task;
+	ipc_port_t port = NULL;
+	kern_return_t ret = KERN_SUCCESS;
+	int i;
 
-	if (ptr_size == 4) {
-		/* 64 bit value containing 32 bit address */
-		unsigned int i;
+	for (i = 0; i < pacts->pspa_count; i++) {
+		act = &pacts->pspa_actions[i];
 
-		error = copyin(froma, &i, 4);
-		*toptr = CAST_USER_ADDR_T(i);	/* SAFE */
-	} else {
-		error = copyin(froma, toptr, 8);
-	}
-	return (error);
-}
+		ret = ipc_object_copyin(get_task_ipcspace(current_task()),
+				(mach_port_name_t) act->new_port,
+				MACH_MSG_TYPE_COPY_SEND,
+				(ipc_object_t *) &port);
 
+		if (ret) 			
+			return ret;
 
-static int
-copyoutptr(user_addr_t ua, user_addr_t ptr, int ptr_size)
-{
+		switch (act->port_type) {
+			case PSPA_SPECIAL:
+				ret = task_set_special_port(task, 
+						act->which, 
+						port);
+				break;
+			case PSPA_EXCEPTION:
+				ret = task_set_exception_ports(task, 
+						act->mask,
+						port, 
+						act->behavior, 
+						act->flavor);
+				break;
+			default:
+				ret = KERN_FAILURE;
+		}
+		/* action failed, so release port resources */
+		if (ret) { 
+			ipc_port_release_send(port);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * exec_handle_file_actions
+ *
+ * Description:	Go through the _posix_file_actions_t contents applying the
+ *		open, close, and dup2 operations to the open file table for
+ *		the current process.
+ *
+ * Parameters:	struct image_params *	Image parameter block
+ *
+ * Returns:	0			Success
+ *		???
+ *
+ * Note:	Actions are applied in the order specified, with the credential
+ *		of the parent process.  This is done to permit the parent
+ *		process to utilize POSIX_SPAWN_RESETIDS to drop privilege in
+ *		the child following operations the child may in fact not be
+ *		normally permitted to perform.
+ */
+static int
+exec_handle_file_actions(struct image_params *imgp)
+{
+	int error = 0;
+	int action;
+	proc_t p = vfs_context_proc(imgp->ip_vfs_context);
+	_posix_spawn_file_actions_t px_sfap = imgp->ip_px_sfa;
+	register_t ival[2];		/* dummy retval for system calls) */
+
+	for (action = 0; action < px_sfap->psfa_act_count; action++) {
+		_psfa_action_t *psfa = &px_sfap->psfa_act_acts[ action];
+
+		switch(psfa->psfaa_type) {
+		case PSFA_OPEN: {
+			/*
+			 * Open is different, in that it requires the use of
+			 * a path argument, which is normally copied in from
+			 * user space; because of this, we have to support an
+			 * open from kernel space that passes an address space
+			 * context oof UIO_SYSSPACE, and casts the address
+			 * argument to a user_addr_t.
+			 */
+			struct vnode_attr va;
+			struct nameidata nd;
+			int mode = psfa->psfaa_openargs.psfao_mode;
+			struct dup2_args dup2a;
+			struct close_nocancel_args ca;
+			int origfd;
+
+			VATTR_INIT(&va);
+			/* Mask off all but regular access permissions */
+			mode = ((mode &~ p->p_fd->fd_cmask) & ALLPERMS) & ~S_ISTXT;
+			VATTR_SET(&va, va_mode, mode & ACCESSPERMS);
+
+			NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, UIO_SYSSPACE,
+			       CAST_USER_ADDR_T(psfa->psfaa_openargs.psfao_path),
+			       imgp->ip_vfs_context);
+
+			error = open1(imgp->ip_vfs_context, 
+					&nd,
+					psfa->psfaa_openargs.psfao_oflag,
+					&va,
+					ival);
+
+			/*
+			 * If there's an error, or we get the right fd by
+			 * accident, then drop out here.  This is easier that
+			 * rearchitecting all the open code to preallocate fd
+			 * slots, and internally taking one as an argument.
+			 */
+			if (error || ival[0] == psfa->psfaa_filedes)
+				break;
+
+			origfd = ival[0];
+			/*
+			 * If we didn't fall out from an error, we ended up
+			 * with the wrong fd; so now we've got to try to dup2
+			 * it to the right one.
+			 */
+			dup2a.from = origfd;
+			dup2a.to = psfa->psfaa_filedes;
+
+			/*
+			 * The dup2() system call implementation sets
+			 * ival to newfd in the success case, but we
+			 * can ignore that, since if we didn't get the
+			 * fd we wanted, the error will stop us.
+			 */
+			error = dup2(p, &dup2a, ival);
+			if (error)
+				break;
+
+			/*
+			 * Finally, close the original fd.
+			 */
+			ca.fd = origfd;
+
+			error = close_nocancel(p, &ca, ival);
+			}
+			break;
+
+		case PSFA_DUP2: {
+			struct dup2_args dup2a;
+
+			dup2a.from = psfa->psfaa_filedes;
+			dup2a.to = psfa->psfaa_openargs.psfao_oflag;
+
+			/*
+			 * The dup2() system call implementation sets
+			 * ival to newfd in the success case, but we
+			 * can ignore that, since if we didn't get the
+			 * fd we wanted, the error will stop us.
+			 */
+			error = dup2(p, &dup2a, ival);
+			}
+			break;
+
+		case PSFA_CLOSE: {
+			struct close_nocancel_args ca;
+
+			ca.fd = psfa->psfaa_filedes;
+
+			error = close_nocancel(p, &ca, ival);
+			}
+			break;
+
+		default:
+			error = EINVAL;
+			break;
+		}
+		/* All file actions failures are considered fatal, per POSIX */
+		if (error)
+			break;
+	}
+
+	return (error);
+}
+
+
+/*
+ * posix_spawn
+ *
+ * Parameters:	uap->pid		Pointer to pid return area
+ *		uap->fname		File name to exec
+ *		uap->argp		Argument list
+ *		uap->envp		Environment list
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		ENOTSUP			Not supported
+ *		ENOEXEC			Executable file format error
+ *	exec_activate_image:EINVAL	Invalid argument
+ *	exec_activate_image:EACCES	Permission denied
+ *	exec_activate_image:EINTR	Interrupted function
+ *	exec_activate_image:ENOMEM	Not enough space
+ *	exec_activate_image:EFAULT	Bad address
+ *	exec_activate_image:ENAMETOOLONG	Filename too long
+ *	exec_activate_image:ENOEXEC	Executable file format error
+ *	exec_activate_image:ETXTBSY	Text file busy [misuse of error code]
+ *	exec_activate_image:EBADEXEC	The executable is corrupt/unknown
+ *	exec_activate_image:???
+ *	mac_execve_enter:???
+ *
+ * TODO:	More gracefully handle failures after vfork
+ *		Expect to need __mac_posix_spawn() at some point...
+ *		Handle posix_spawnattr_t
+ *		Handle posix_spawn_file_actions_t
+ */
+int
+posix_spawn(proc_t ap, struct posix_spawn_args *uap, register_t *retval)
+{
+	proc_t p = ap;		/* quiet bogus GCC vfork() warning */
+	register_t ival[2];		/* dummy retval for vfork() */
+	struct image_params image_params, *imgp;
+	struct vnode_attr va;
+	struct vnode_attr origva;
+	struct uthread	*uthread = 0;	/* compiler complains if not set to 0*/
+	int error, sig;
+	task_t  task;
+	int numthreads;
+	char alt_p_comm[sizeof(p->p_comm)] = {0};	/* for PowerPC */
+	int is_64 = IS_64BIT_PROCESS(p);
+	int undo_vfork = 0;
+	struct vfs_context context;
+	struct user__posix_spawn_args_desc px_args;
+	struct _posix_spawnattr px_sa;
+	_posix_spawn_file_actions_t px_sfap = NULL;
+	_posix_spawn_port_actions_t px_spap = NULL;
+	struct __user_sigaction vec;
+
+	imgp = &image_params;
+
+	/* Initialize the common data in the image_params structure */
+	bzero(imgp, sizeof(*imgp));
+	imgp->ip_user_fname = uap->path;
+	imgp->ip_user_argv = uap->argv;
+	imgp->ip_user_envv = uap->envp;
+	imgp->ip_vattr = &va;
+	imgp->ip_origvattr = &origva;
+	imgp->ip_vfs_context = &context;
+	imgp->ip_flags = (is_64 ? IMGPF_WAS_64BIT : IMGPF_NONE);
+	imgp->ip_p_comm = alt_p_comm;		/* for PowerPC */
+	imgp->ip_seg = (is_64 ? UIO_USERSPACE64 : UIO_USERSPACE32);
+
+	if (uap->adesc != USER_ADDR_NULL) {
+		if(is_64) {
+			error = copyin(uap->adesc, &px_args, sizeof(px_args));
+		} else {
+			struct _posix_spawn_args_desc px_args32;
+
+			error = copyin(uap->adesc, &px_args32, sizeof(px_args32));
+
+			/*
+			 * Convert arguments descriptor from external 32 bit
+			 * representation to internal 64 bit representation
+			 */
+			px_args.attr_size = px_args32.attr_size;
+			px_args.attrp = CAST_USER_ADDR_T(px_args32.attrp);
+			px_args.file_actions_size = px_args32.file_actions_size;
+			px_args.file_actions = CAST_USER_ADDR_T(px_args32.file_actions);
+			px_args.port_actions_size = px_args32.port_actions_size;
+			px_args.port_actions = CAST_USER_ADDR_T(px_args32.port_actions);
+		}
+		if (error)
+			goto bad;
+
+		if (px_args.attr_size != 0) {
+			/* 
+			 * This could lose some of the port_actions pointer, 
+			 * but we already have it from px_args. 
+			 */
+			if ((error = copyin(px_args.attrp, &px_sa, sizeof(px_sa))) != 0)
+			goto bad;
+
+			imgp->ip_px_sa = &px_sa;
+		}
+		if (px_args.file_actions_size != 0) {
+			/* Limit file_actions to allowed number of open files */
+			int maxfa = (p->p_limit ? p->p_rlimit[RLIMIT_NOFILE].rlim_cur : NOFILE);
+			if (px_args.file_actions_size < PSF_ACTIONS_SIZE(1) ||
+				px_args.file_actions_size > PSF_ACTIONS_SIZE(maxfa)) {
+				error = EINVAL;
+				goto bad;
+			}
+			MALLOC(px_sfap, _posix_spawn_file_actions_t, px_args.file_actions_size, M_TEMP, M_WAITOK);
+			if (px_sfap == NULL) {
+				error = ENOMEM;
+				goto bad;
+			}
+			imgp->ip_px_sfa = px_sfap;
+
+			if ((error = copyin(px_args.file_actions, px_sfap, 
+							px_args.file_actions_size)) != 0)
+				goto bad;
+		}
+		if (px_args.port_actions_size != 0) {
+			/* Limit port_actions to one page of data */
+			if (px_args.port_actions_size < PS_PORT_ACTIONS_SIZE(1) ||
+				px_args.port_actions_size > PAGE_SIZE) {
+				error = EINVAL;
+				goto bad;
+			}
+
+			MALLOC(px_spap, _posix_spawn_port_actions_t, 
+					px_args.port_actions_size, M_TEMP, M_WAITOK);
+			if (px_spap == NULL) {
+				error = ENOMEM;
+				goto bad;
+			}
+			imgp->ip_px_spa = px_spap;
+
+			if ((error = copyin(px_args.port_actions, px_spap, 
+							px_args.port_actions_size)) != 0)
+				goto bad;
+		}
+	}
+
+	if (imgp->ip_px_sa == NULL || !(px_sa.psa_flags & POSIX_SPAWN_SETEXEC)){
+		if ((error = vfork(p, NULL, ival)) != 0)
+			goto bad;
+		undo_vfork = 1;
+	}
+
+	/* "reenter the kernel" on a new vfork()'ed process */
+	uthread = get_bsdthread_info(current_thread());
+	if (undo_vfork)
+		p = uthread->uu_proc;
+
+	context.vc_thread = current_thread();
+	context.vc_ucred = p->p_ucred;	/* XXX must NOT be kauth_cred_get() */
+
+	/*
+	 * Post fdcopy(), pre exec_handle_sugid() - this is where we want
+	 * to handle the file_actions.  Since vfork() also ends up setting
+	 * us into the parent process group, and saved off the signal flags,
+	 * this is also where we want to handle the spawn flags.
+	 */
+	/* Has spawn file actions? */
+	if (imgp->ip_px_sfa != NULL &&
+	    (error = exec_handle_file_actions(imgp)) != 0) {
+		goto bad;
+	}
+
+	/* Has spawn port actions? */
+	if (imgp->ip_px_spa != NULL) { 
+		/* Only allowed when not under vfork */
+		if (!(px_sa.psa_flags & POSIX_SPAWN_SETEXEC)) {
+			error = ENOTSUP;
+			goto bad;
+		}
+		if((error = exec_handle_port_actions(imgp)) != 0) 
+			goto bad;
+	}
+
+	/* Has spawn attr? */
+	if (imgp->ip_px_sa != NULL) {
+		/* Set the process group ID of the child process */
+		if (px_sa.psa_flags & POSIX_SPAWN_SETPGROUP) {
+			struct setpgid_args spga;
+			spga.pid = p->p_pid;
+			spga.pgid = px_sa.psa_pgroup;
+			/*
+			 * Effectively, call setpgid() system call; works
+			 * because there are no pointer arguments.
+			 */
+			if((error = setpgid(p, &spga, ival)) != 0)
+				goto bad;
+		}
+		/*
+		 * Reset UID/GID to parent's RUID/RGID; This works only
+		 * because the operation occurs *after* the vfork() and
+		 * before the call to exec_handle_sugid() by the image
+		 * activator called from exec_activate_image().
+		 *
+		 * The use of p_ucred is safe, since we are acting on the
+		 * new process, and it has no threads other than the one
+		 * we are creating for it.
+		 */
+		if (px_sa.psa_flags & POSIX_SPAWN_RESETIDS) {
+			kauth_cred_t my_cred = p->p_ucred;
+			kauth_cred_t my_new_cred = kauth_cred_setuidgid(my_cred, my_cred->cr_ruid, my_cred->cr_rgid);
+			if (my_new_cred != my_cred)
+				p->p_ucred = my_new_cred;
+		}
+		/*
+		 * Mask a list of signals, instead of them being unmasked, if
+		 * they were unmasked in the parent; note that some signals
+		 * are not maskable.
+		 */
+		if (px_sa.psa_flags & POSIX_SPAWN_SETSIGMASK)
+			uthread->uu_sigmask = (px_sa.psa_sigmask & ~sigcantmask);
+		/*
+		 * Default a list of signals instead of ignoring them, if
+		 * they were ignored in the parent.
+		 */
+		if (px_sa.psa_flags & POSIX_SPAWN_SETSIGDEF) {
+			vec.sa_handler = SIG_DFL;
+			vec.sa_tramp = 0;
+			vec.sa_mask = 0;
+			vec.sa_flags = 0;
+			for (sig = 0; sig < NSIG; sig++)
+				if (px_sa.psa_sigdefault && 1 << sig) {
+					error = setsigvec(p, sig, &vec);
+			}
+		}
+	}
+
+	/*
+         * XXXAUDIT: Currently, we only audit the pathname of the binary.
+         * There may also be poor interaction with dyld.
+         */
+
+	task = current_task();
+
+	/* If we're not in vfork, don't permit a mutithreaded task to exec */
+	if (!(uthread->uu_flag & UT_VFORK)) {
+		if (task != kernel_task) { 
+			numthreads = get_task_numacts(task);
+			if (numthreads <= 0 ) {
+				error = EINVAL;
+				goto bad;
+			}
+			if (numthreads > 1) {
+				error = ENOTSUP;
+				goto bad;
+			}
+		}
+	}
+
+#if MAC_SPAWN	/* XXX */
+	if (uap->mac_p != USER_ADDR_NULL) {
+		error = mac_execve_enter(uap->mac_p, imgp);
+		if (error)
+			goto bad;
+	}
+#endif
+
+	if ((error = exec_activate_image(imgp)) != 0) 
+		goto bad;
+bad:
+	/* Image not claimed by any activator? */
+	if (error == -1)
+		error = ENOEXEC;
+	if (error == 0) {
+		exec_resettextvp(p, imgp);
+	}	
+	if (imgp->ip_vp)
+		vnode_put(imgp->ip_vp);
+	if (imgp->ip_strings)
+		execargs_free(imgp);
+	if (imgp->ip_px_sfa != NULL)
+		FREE(imgp->ip_px_sfa, M_TEMP);
+	if (imgp->ip_px_spa != NULL)
+		FREE(imgp->ip_px_spa, M_TEMP);
+
+#if CONFIG_MACF
+	if (imgp->ip_execlabelp)
+		mac_cred_label_free(imgp->ip_execlabelp);
+	if (imgp->ip_scriptlabelp)
+		mac_vnode_label_free(imgp->ip_scriptlabelp);
+#endif
+	if (undo_vfork) {
+		if (error) {
+			DTRACE_PROC1(exec__failure, int, error);
+			vfork_exit(p, W_EXITCODE(-1, 0));
+		} else {
+			DTRACE_PROC(exec__success);
+		}
+		/*
+		 * Returning to the parent process...
+		 *
+		 * If the parent wants the pid, copy it out
+		 */
+		if (uap->pid != USER_ADDR_NULL)
+			(void)suword(uap->pid, p->p_pid);
+		retval[0] = error;
+		/*
+		 * Override inherited code signing flags with the
+		 * ones for the process that is being successfully
+		 * loaded
+		 */
+		proc_lock(p);
+		p->p_csflags = imgp->ip_csflags;
+		proc_unlock(p);
+		vfork_return(p, NULL, error);
+		(void)thread_resume(imgp->ip_vfork_thread);
+	}
+
+	if (!error) {
+		/*
+		 * Override inherited code signing flags with the
+		 * ones for the process that is being successfully
+		 * loaded
+		 */
+		proc_lock(p);
+		p->p_csflags = imgp->ip_csflags;
+		proc_unlock(p);
+		DTRACE_PROC(exec__success);
+	} else {
+		DTRACE_PROC1(exec__failure, int, error);
+	}
+
+	return(error);
+}
+
+
+/*
+ * execve
+ *
+ * Parameters:	uap->fname		File name to exec
+ *		uap->argp		Argument list
+ *		uap->envp		Environment list
+ *
+ * Returns:	0			Success
+ *	__mac_execve:EINVAL		Invalid argument
+ *	__mac_execve:ENOTSUP		Invalid argument
+ *	__mac_execve:EACCES		Permission denied
+ *	__mac_execve:EINTR		Interrupted function
+ *	__mac_execve:ENOMEM		Not enough space
+ *	__mac_execve:EFAULT		Bad address
+ *	__mac_execve:ENAMETOOLONG	Filename too long
+ *	__mac_execve:ENOEXEC		Executable file format error
+ *	__mac_execve:ETXTBSY		Text file busy [misuse of error code]
+ *	__mac_execve:???
+ *
+ * TODO:	Dynamic linker header address on stack is copied via suword()
+ */
+/* ARGSUSED */
+int
+execve(proc_t p, struct execve_args *uap, register_t *retval)
+{
+	struct __mac_execve_args muap;
+	int err;
+
+	muap.fname = uap->fname;
+	muap.argp = uap->argp;
+	muap.envp = uap->envp;
+	muap.mac_p = USER_ADDR_NULL;
+	err = __mac_execve(p, &muap, retval);
+
+	return(err);
+}
+
+/*
+ * __mac_execve
+ *
+ * Parameters:	uap->fname		File name to exec
+ *		uap->argp		Argument list
+ *		uap->envp		Environment list
+ *		uap->mac_p		MAC label supplied by caller
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		ENOTSUP			Not supported
+ *		ENOEXEC			Executable file format error
+ *	exec_activate_image:EINVAL	Invalid argument
+ *	exec_activate_image:EACCES	Permission denied
+ *	exec_activate_image:EINTR	Interrupted function
+ *	exec_activate_image:ENOMEM	Not enough space
+ *	exec_activate_image:EFAULT	Bad address
+ *	exec_activate_image:ENAMETOOLONG	Filename too long
+ *	exec_activate_image:ENOEXEC	Executable file format error
+ *	exec_activate_image:ETXTBSY	Text file busy [misuse of error code]
+ *	exec_activate_image:EBADEXEC	The executable is corrupt/unknown
+ *	exec_activate_image:???
+ *	mac_execve_enter:???
+ *
+ * TODO:	Dynamic linker header address on stack is copied via suword()
+ */
+int
+__mac_execve(proc_t p, struct __mac_execve_args *uap, register_t *retval)
+{
+	struct image_params image_params, *imgp;
+	struct vnode_attr va;
+	struct vnode_attr origva;
+	struct uthread		*uthread;
+	int error;
+	task_t  task;
+	int numthreads;
+	char alt_p_comm[sizeof(p->p_comm)] = {0};	/* for PowerPC */
+	int is_64 = IS_64BIT_PROCESS(p);
+	struct vfs_context context;
+
+	context.vc_thread = current_thread();
+	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX must NOT be kauth_cred_get() */
+
+	imgp = &image_params;
+
+	/* Initialize the common data in the image_params structure */
+	bzero(imgp, sizeof(*imgp));
+	imgp->ip_user_fname = uap->fname;
+	imgp->ip_user_argv = uap->argp;
+	imgp->ip_user_envv = uap->envp;
+	imgp->ip_vattr = &va;
+	imgp->ip_origvattr = &origva;
+	imgp->ip_vfs_context = &context;
+	imgp->ip_flags = (is_64 ? IMGPF_WAS_64BIT : IMGPF_NONE);
+	imgp->ip_p_comm = alt_p_comm;		/* for PowerPC */
+	imgp->ip_seg = (is_64 ? UIO_USERSPACE64 : UIO_USERSPACE32);
+
+	/*
+         * XXXAUDIT: Currently, we only audit the pathname of the binary.
+         * There may also be poor interaction with dyld.
+         */
+
+	task = current_task();
+	uthread = get_bsdthread_info(current_thread());
+
+	/* If we're not in vfork, don't permit a mutithreaded task to exec */
+	if (!(uthread->uu_flag & UT_VFORK)) {
+		if (task != kernel_task) { 
+			proc_lock(p);
+			numthreads = get_task_numacts(task);
+			if (numthreads <= 0 ) {
+				proc_unlock(p);
+				kauth_cred_unref(&context.vc_ucred);
+				return(EINVAL);
+			}
+			if (numthreads > 1) {
+				proc_unlock(p);
+				kauth_cred_unref(&context.vc_ucred);
+				return(ENOTSUP);
+			}
+			proc_unlock(p);
+		}
+	}
+
+#if CONFIG_MACF
+	if (uap->mac_p != USER_ADDR_NULL) {
+		error = mac_execve_enter(uap->mac_p, imgp);
+		if (error) {
+			kauth_cred_unref(&context.vc_ucred);
+			return (error);
+		}
+	}
+#endif
+
+	proc_transstart(p, 0);
+	error = exec_activate_image(imgp);
+	proc_transend(p, 0);
+
+	kauth_cred_unref(&context.vc_ucred);
+	
+	/* Image not claimed by any activator? */
+	if (error == -1)
+		error = ENOEXEC;
+
+	if (error == 0) {
+		exec_resettextvp(p, imgp);
+	}	
+	if (imgp->ip_vp != NULLVP)
+		vnode_put(imgp->ip_vp);
+	if (imgp->ip_strings)
+		execargs_free(imgp);
+#if CONFIG_MACF
+	if (imgp->ip_execlabelp)
+		mac_cred_label_free(imgp->ip_execlabelp);
+	if (imgp->ip_scriptlabelp)
+		mac_vnode_label_free(imgp->ip_scriptlabelp);
+#endif
+	if (!error) {
+		/*
+		 * Override inherited code signing flags with the
+		 * ones for the process that is being successfully
+		 * loaded
+		 */
+		proc_lock(p);
+		p->p_csflags = imgp->ip_csflags;
+		proc_unlock(p);
+		DTRACE_PROC(exec__success);
+
+		if (uthread->uu_flag & UT_VFORK) {
+			vfork_return(p, retval, p->p_pid);
+			(void)thread_resume(imgp->ip_vfork_thread);
+		}
+	} else {
+		DTRACE_PROC1(exec__failure, int, error);
+	}
+	
+	return(error);
+}
+
+
+/*
+ * copyinptr
+ *
+ * Description:	Copy a pointer in from user space to a user_addr_t in kernel
+ *		space, based on 32/64 bitness of the user space
+ *
+ * Parameters:	froma			User space address
+ *		toptr			Address of kernel space user_addr_t
+ *		ptr_size		4/8, based on 'froma' address space
+ *
+ * Returns:	0			Success
+ *		EFAULT			Bad 'froma'
+ *
+ * Implicit returns:
+ *		*ptr_size		Modified
+ */
+static int
+copyinptr(user_addr_t froma, user_addr_t *toptr, int ptr_size)
+{
+	int error;
+
+	if (ptr_size == 4) {
+		/* 64 bit value containing 32 bit address */
+		unsigned int i;
+
+		error = copyin(froma, &i, 4);
+		*toptr = CAST_USER_ADDR_T(i);	/* SAFE */
+	} else {
+		error = copyin(froma, toptr, 8);
+	}
+	return (error);
+}
+
+
+/*
+ * copyoutptr
+ *
+ * Description:	Copy a pointer out from a user_addr_t in kernel space to
+ *		user space, based on 32/64 bitness of the user space
+ *
+ * Parameters:	ua			User space address to copy to
+ *		ptr			Address of kernel space user_addr_t
+ *		ptr_size		4/8, based on 'ua' address space
+ *
+ * Returns:	0			Success
+ *		EFAULT			Bad 'ua'
+ *
+ * Implicit returns:
+ *		*ptr_size		Modified
+ */
+static int
+copyoutptr(user_addr_t ua, user_addr_t ptr, int ptr_size)
+{
 	int error;
 
 	if (ptr_size == 4) {
@@ -1289,7 +2154,7 @@ copyoutptr(user_addr_t ua, user_addr_t ptr, int ptr_size)
 static int
 exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 {
-	struct proc *p = vfs_context_proc(imgp->ip_vfs_context);
+	proc_t p = vfs_context_proc(imgp->ip_vfs_context);
 	int	ptr_size = (imgp->ip_flags & IMGPF_IS_64BIT) ? 8 : 4;
 	char	*argv = imgp->ip_argv;	/* modifiable copy of argv */
 	user_addr_t	string_area;	/* *argv[], *env[] */
@@ -1306,24 +2171,6 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 	unsigned patharea_len = imgp->ip_argv - imgp->ip_strings;
 	int envc_add = 0;
 	
-#ifdef IMGPF_POWERPC
-	/*
-	 * oah750 expects /usr/lib/dyld\0 as the start of the program name.
-	 * It also expects to have a certain environment variable set to 0.
-	 * 50 bytes for each to ensure we have enough space without having
-	 * to count every byte.
-	 */
-	char *progname, *envvar;
-	char progname_str[] = "/usr/lib/dyld";
-	char envvar_str[] = "OAH750_CFG_FU_STACK_SIZE=0";
-	
-	if (imgp->ip_flags & IMGPF_POWERPC) {
-		progname = progname_str;
-		envvar = envvar_str;
-		patharea_len += strlen(progname) + strlen(envvar) + 2;
-		envc_add = 1;
-	}
-#endif /* IMGPF_POWERPC */
 	/*
 	 * Set up pointers to the beginning of the string area, the beginning
 	 * of the path area, and the beginning of the pointer area (actually,
@@ -1341,8 +2188,10 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 	 * Record the size of the arguments area so that sysctl_procargs()
 	 * can return the argument area without having to parse the arguments.
 	 */
+	proc_lock(p);
 	p->p_argc = imgp->ip_argc;
 	p->p_argslen = (int)(stack - path_area);
+	proc_unlock(p);
 
 
 	/*
@@ -1351,18 +2200,6 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 	 * copy it just before the string area.
 	 */
 	len = 0;
-#ifdef IMGPF_POWERPC
-	if (imgp->ip_flags & IMGPF_POWERPC) {
-		error = copyoutstr(progname, path_area,
-						   patharea_len,
-						   (size_t *)&len);
-		if (error)
-			goto bad;
-		error = copyoutstr(imgp->ip_strings, path_area + strlen(progname) + 1,
-						   patharea_len,
-						   (size_t *)&len);
-	} else
-#endif /* IMGPF_POWERPC */
 	error = copyoutstr(imgp->ip_strings, path_area,
 						   patharea_len,
 				(size_t *)&len);
@@ -1392,6 +2229,9 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 		ptr_area += sizeof(int);
 	}
 
+#if CONFIG_DTRACE
+	p->p_dtrace_argv = ptr_area; /* user_addr_t &argv[0] for dtrace convenience */
+#endif /* CONFIG_DTRACE */
 
 	/*
 	 * We use (string_area - path_area) here rather than the more
@@ -1407,27 +2247,9 @@ exec_copyout_strings(struct image_params *imgp, user_addr_t *stackp)
 			/* argv[n] = NULL */
 			(void)copyoutptr(0LL, ptr_area, ptr_size);
 			ptr_area += ptr_size;
-#ifdef IMGPF_POWERPC
-			if (envc_add) {
-				(void)copyoutptr(string_area, ptr_area, ptr_size);
-				
-				do {
-					if (strspace <= 0) {
-						error = E2BIG;
-						break;
-					}
-					error = copyoutstr(envvar, string_area,
-									   (unsigned)strspace,
-									   (size_t *)&len);
-					string_area += len;
-					envvar += len;
-					strspace -= len;
-				} while (error == ENAMETOOLONG);				
-				if (error == EFAULT || error == E2BIG)
-					break;
-				ptr_area += ptr_size;
-			}
-#endif /* IMGPF_POWERPC */
+#if CONFIG_DTRACE
+			p->p_dtrace_envp = ptr_area; /* user_addr_t &env[0] for dtrace convenience */
+#endif /* CONFIG_DTRACE */
 		}
 		if (--stringc < 0)
 			break;
@@ -1477,19 +2299,24 @@ bad:
  *		(imgp->ip_envc)		Count of environment strings, updated
  *
  *
- * Notes:	The argument and environment vectors are user space pointers
+ * Note:	The argument and environment vectors are user space pointers
  *		to arrays of user space pointers.
  */
 static int
 exec_extract_strings(struct image_params *imgp)
 {
 	int error = 0;
-	struct proc *p = vfs_context_proc(imgp->ip_vfs_context);
-	int seg = (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32);
 	int	ptr_size = (imgp->ip_flags & IMGPF_WAS_64BIT) ? 8 : 4;
 	user_addr_t	argv = imgp->ip_user_argv;
 	user_addr_t	envv = imgp->ip_user_envv;
 
+	/*
+	 * If the argument vector is NULL, this is the system startup
+	 * bootstrap from load_init_program(), and there's nothing to do
+	 */
+	if (imgp->ip_user_argv == 0LL)
+		goto bad;
+
 	/* Now, get rest of arguments */
 
 	/*
@@ -1506,7 +2333,7 @@ exec_extract_strings(struct image_params *imgp)
 			goto bad;
 		if (arg != 0LL && arg != (user_addr_t)-1) {
 			argv += ptr_size;
-			error = exec_add_string(imgp, imgp->ip_user_fname, seg);
+			error = exec_add_string(imgp, imgp->ip_user_fname);
 			if (error)
 				goto bad;
 			imgp->ip_argc++;
@@ -1531,7 +2358,7 @@ exec_extract_strings(struct image_params *imgp)
 		/*
 		* av[n...] = arg[n]
 		*/
-		error = exec_add_string(imgp, arg, seg);
+		error = exec_add_string(imgp, arg);
 		if (error)
 			goto bad;
 		imgp->ip_argc++;
@@ -1555,7 +2382,7 @@ exec_extract_strings(struct image_params *imgp)
 		/*
 		* av[n...] = env[n]
 		*/
-		error = exec_add_string(imgp, env, seg);
+		error = exec_add_string(imgp, env);
 		if (error)
 			goto bad;
 		imgp->ip_envc++;
@@ -1567,12 +2394,28 @@ bad:
 
 #define	unix_stack_size(p)	(p->p_rlimit[RLIMIT_STACK].rlim_cur)
 
+/*
+ * exec_check_permissions
+ *
+ * Decription:	Verify that the file that is being attempted to be executed
+ *		is in fact allowed to be executed based on it POSIX file
+ *		permissions and other access control criteria
+ *
+ * Parameters:	struct image_params *	the image parameter block
+ *
+ * Returns:	0			Success
+ *		EACCES			Permission denied
+ *		ENOEXEC			Executable file format error
+ *		ETXTBSY			Text file busy [misuse of error code]
+ *	vnode_getattr:???
+ *	vnode_authorize:???
+ */
 static int
 exec_check_permissions(struct image_params *imgp)
 {
 	struct vnode *vp = imgp->ip_vp;
 	struct vnode_attr *vap = imgp->ip_vattr;
-	struct proc *p = vfs_context_proc(imgp->ip_vfs_context);
+	proc_t p = vfs_context_proc(imgp->ip_vfs_context);
 	int error;
 	kauth_action_t action;
 
@@ -1607,20 +2450,38 @@ exec_check_permissions(struct image_params *imgp)
 	imgp->ip_arch_size = vap->va_data_size;
 
 	/* Disable setuid-ness for traced programs or if MNT_NOSUID */
-	if ((vp->v_mount->mnt_flag & MNT_NOSUID) || (p->p_flag & P_TRACED))
+	if ((vp->v_mount->mnt_flag & MNT_NOSUID) || (p->p_lflag & P_LTRACED)) {
 		vap->va_mode &= ~(VSUID | VSGID);
+#if CONFIG_MACF
+		imgp->ip_no_trans = 1;
+#endif
+	}
+
+#if CONFIG_MACF
+	error = mac_vnode_check_exec(imgp->ip_vfs_context, vp, imgp);
+	if (error)
+		return (error);
+#endif
 
   	/* Check for execute permission */
  	action = KAUTH_VNODE_EXECUTE;
   	/* Traced images must also be readable */
- 	if (p->p_flag & P_TRACED)
+ 	if (p->p_lflag & P_LTRACED)
  		action |= KAUTH_VNODE_READ_DATA;
  	if ((error = vnode_authorize(vp, NULL, action, imgp->ip_vfs_context)) != 0)
 		return (error);
 
+#if 0
 	/* Don't let it run if anyone had it open for writing */
-	if (vp->v_writecount)
+	vnode_lock(vp);
+	if (vp->v_writecount) {
+		panic("going to return ETXTBSY %x", vp);
+		vnode_unlock(vp);
 		return (ETXTBSY);
+	}
+	vnode_unlock(vp);
+#endif
+
 
 #ifdef IMGPF_POWERPC
 	/*
@@ -1639,13 +2500,14 @@ exec_check_permissions(struct image_params *imgp)
 	return (error);
 }
 
+
 /*
  * exec_handle_sugid
  *
  * Initially clear the P_SUGID in the process flags; if an SUGID process is
  * exec'ing a non-SUGID image, then  this is the point of no return.
  *
- * If the image being activated is SUGI, then replace the credential with a
+ * If the image being activated is SUGID, then replace the credential with a
  * copy, disable tracing (unless the tracing process is root), reset the
  * mach task port to revoke it, set the P_SUGID bit,
  *
@@ -1669,39 +2531,93 @@ static int
 exec_handle_sugid(struct image_params *imgp)
 {
 	kauth_cred_t		cred = vfs_context_ucred(imgp->ip_vfs_context);
-	struct proc		*p = vfs_context_proc(imgp->ip_vfs_context);
+	proc_t			p = vfs_context_proc(imgp->ip_vfs_context);
 	int			i;
+	int			is_member = 0;
 	int			error = 0;
-	static struct vnode	*dev_null = NULLVP;
+	struct vnode	*dev_null = NULLVP;
+#if CONFIG_MACF
+	kauth_cred_t	my_cred;
+#endif
+
+#if CONFIG_MACF
+	int			mac_transition;
+	mac_transition = mac_cred_check_label_update_execve(imgp->ip_vfs_context, imgp->ip_vp,
+	    imgp->ip_scriptlabelp, imgp->ip_execlabelp, p);
+#endif
 
-	p->p_flag &= ~P_SUGID;
+	OSBitAndAtomic(~((uint32_t)P_SUGID), (UInt32 *)&p->p_flag);
 
+	/*
+	 * Order of the following is important; group checks must go last,
+	 * as we use the success of the 'is_member' check combined with the
+	 * failure of the explicit match to indicate that we will be setting
+	 * the egid of the process even though the new process did not
+	 * require VSUID/VSGID bits in order for it to set the new group as
+	 * its egid.
+	 *
+	 * Note:	Technically, by this we are implying a call to
+	 *		setegid() in the new process, rather than implying
+	 *		it used its VSGID bit to set the effective group,
+	 *		even though there is no code in that process to make
+	 *		such a call.
+	 */
 	if (((imgp->ip_origvattr->va_mode & VSUID) != 0 &&
 	     kauth_cred_getuid(cred) != imgp->ip_origvattr->va_uid) ||
+#if CONFIG_MACF
+		mac_transition ||	/* A policy wants to transition */
+#endif
 	    ((imgp->ip_origvattr->va_mode & VSGID) != 0 &&
-	     cred->cr_gid != imgp->ip_origvattr->va_gid)) {
-#if KTRACE
+		 ((kauth_cred_ismember_gid(cred, imgp->ip_origvattr->va_gid, &is_member) || !is_member) ||
+		 (cred->cr_gid != imgp->ip_origvattr->va_gid)))) {
+
 		/*
-		 * If process is being ktraced, turn off - unless
-		 * root set it.
-		 */
-		if (p->p_tracep && !(p->p_traceflag & KTRFAC_ROOT)) {
-			struct vnode *tvp = p->p_tracep;
-			p->p_tracep = NULL;
-			p->p_traceflag = 0;
-			vnode_rele(tvp);
-		}
-#endif
-	    /*
-		 * Replace the credential with a copy of itself if euid or egid change.
+		 * Replace the credential with a copy of itself if euid or
+		 * egid change.
+		 *
+		 * Note:	setuid binaries will automatically opt out of
+		 *		group resolver participation as a side effect
+		 *		of this operation.  This is an intentional
+		 *		part of the security model, which requires a
+		 *		participating credential be established by
+		 *		escalating privilege, setting up all other
+		 *		aspects of the credential including whether
+		 *		or not to participate in external group
+		 *		membership resolution, then dropping their
+		 *		effective privilege to that of the desired
+		 *		final credential state.
 		 */
 		if (imgp->ip_origvattr->va_mode & VSUID) {
-			p->p_ucred = kauth_cred_seteuid(p->p_ucred, imgp->ip_origvattr->va_uid);
+			p->p_ucred  = kauth_cred_setresuid(p->p_ucred, KAUTH_UID_NONE, imgp->ip_origvattr->va_uid, imgp->ip_origvattr->va_uid, KAUTH_UID_NONE);
 		}
 		if (imgp->ip_origvattr->va_mode & VSGID) {
-			p->p_ucred = kauth_cred_setegid(p->p_ucred, imgp->ip_origvattr->va_gid);
+			p->p_ucred = kauth_cred_setresgid(p->p_ucred, KAUTH_GID_NONE, imgp->ip_origvattr->va_gid, imgp->ip_origvattr->va_gid);
 		}
 
+#if CONFIG_MACF
+		/* 
+		 * XXXMAC: In FreeBSD, we set P_SUGID on a MAC transition
+		 * to protect against debuggers being attached by an 
+		 * insufficiently privileged process onto the result of
+		 * a transition to a more privileged credential.  This is
+		 * too conservative on FreeBSD, but we need to do
+		 * something similar here, or risk vulnerability.
+		 *
+		 * Before we make the call into the MAC policies, get a new
+		 * duplicate credential, so they can modify it without
+		 * modifying any others sharing it.
+		 */
+		if (mac_transition && !imgp->ip_no_trans) { 
+			kauth_proc_label_update_execve(p,
+				imgp->ip_vfs_context,
+				imgp->ip_vp, 
+				imgp->ip_scriptlabelp, imgp->ip_execlabelp);
+
+			my_cred = kauth_cred_proc_ref(p);
+			mac_task_label_update_cred(my_cred, p->task);
+			kauth_cred_unref(&my_cred);
+		}
+#endif
 		/*
 		 * Have mach reset the task and thread ports.
 		 * We don't want anyone who had the ports before
@@ -1713,7 +2629,15 @@ exec_handle_sugid(struct image_params *imgp)
 			ipc_thread_reset(current_thread());
 		}
 
-		p->p_flag |= P_SUGID;
+		/*
+		 * If 'is_member' is non-zero, then we passed the VSUID and
+		 * MACF checks, and successfully determined that the previous
+		 * cred was a member of the VSGID group, but that it was not
+		 * the default at the time of the execve.  So we don't set the
+		 * P_SUGID on the basis of simply running this code.
+		 */
+		if (!is_member)
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
 
 		/* Cache the vnode for /dev/null the first time around */
 		if (dev_null == NULLVP) {
@@ -1751,7 +2675,7 @@ exec_handle_sugid(struct image_params *imgp)
 				if (p->p_fd->fd_ofiles[i] != NULL)
 					continue;
 
-				if ((error = falloc(p, &fp, &indx)) != 0)
+				if ((error = falloc(p, &fp, &indx, imgp->ip_vfs_context)) != 0)
 					continue;
 
 				if ((error = vnode_ref_ext(dev_null, FREAD)) != 0) {
@@ -1792,164 +2716,217 @@ exec_handle_sugid(struct image_params *imgp)
 	return(error);
 }
 
+
+/*
+ * create_unix_stack
+ *
+ * Description:	Set the user stack address for the process to the provided
+ *		address.  If a custom stack was not set as a result of the
+ *		load process (i.e. as specified by the image file for the
+ *		executable), then allocate the stack in the provided map and
+ *		set up appropriate guard pages for enforcing administrative
+ *		limits on stack growth, if they end up being needed.
+ *
+ * Parameters:	p			Process to set stack on
+ *		user_stack		Address to set stack for process to
+ *		customstack		FALSE if no custom stack in binary
+ *		map			Address map in which to allocate the
+ *					new stack, if 'customstack' is FALSE
+ *
+ * Returns:	KERN_SUCCESS		Stack successfully created
+ *		!KERN_SUCCESS		Mach failure code
+ */
 static kern_return_t
 create_unix_stack(vm_map_t map, user_addr_t user_stack, int customstack,
-			struct proc *p)
+			proc_t p)
 {
-	mach_vm_size_t	size;
-	mach_vm_offset_t addr;
+	mach_vm_size_t		size, prot_size;
+	mach_vm_offset_t	addr, prot_addr;
+	kern_return_t		kr;
 
+	proc_lock(p);
 	p->user_stack = user_stack;
+	proc_unlock(p);
+
 	if (!customstack) {
-		size = mach_vm_round_page(unix_stack_size(p));
+		/*
+		 * Allocate enough space for the maximum stack size we
+		 * will ever authorize and an extra page to act as
+		 * a guard page for stack overflows.
+		 */
+		size = mach_vm_round_page(MAXSSIZ);
+#if STACK_GROWTH_UP
+		addr = mach_vm_trunc_page(user_stack);
+#else	/* STACK_GROWTH_UP */
 		addr = mach_vm_trunc_page(user_stack - size);
-		return (mach_vm_allocate(map, &addr, size,
+#endif	/* STACK_GROWTH_UP */
+		kr = mach_vm_allocate(map, &addr, size,
 					VM_MAKE_TAG(VM_MEMORY_STACK) |
-					VM_FLAGS_FIXED));
-	} else
-		return(KERN_SUCCESS);
+				      VM_FLAGS_FIXED);
+		if (kr != KERN_SUCCESS) {
+			return kr;
+		}
+		/*
+		 * And prevent access to what's above the current stack
+		 * size limit for this process.
+		 */
+		prot_addr = addr;
+#if STACK_GROWTH_UP
+		prot_addr += unix_stack_size(p);
+#endif /* STACK_GROWTH_UP */
+		prot_addr = mach_vm_round_page(prot_addr);
+		prot_size = mach_vm_trunc_page(size - unix_stack_size(p));
+		kr = mach_vm_protect(map,
+				     prot_addr,
+				     prot_size,
+				     FALSE,
+				     VM_PROT_NONE);
+		if (kr != KERN_SUCCESS) {
+			(void) mach_vm_deallocate(map, addr, size);
+			return kr;
+		}
+	}
+	return KERN_SUCCESS;
 }
 
 #include <sys/reboot.h>
 
 static char		init_program_name[128] = "/sbin/launchd";
-static const char *	other_init = "/sbin/mach_init";
-
-char		init_args[128] = "";
 
 struct execve_args	init_exec_args;
-int		init_attempts = 0;
-
 
+/*
+ * load_init_program
+ *
+ * Description:	Load the "init" program; in most cases, this will be "launchd"
+ *
+ * Parameters:	p			Process to call execve() to create
+ *					the "init" program
+ *
+ * Returns:	(void)
+ *
+ * Notes:	The process that is passed in is the first manufactured
+ *		process on the system, and gets here via bsd_ast() firing
+ *		for the first time.  This is done to ensure that bsd_init()
+ *		has run to completion.
+ */
 void
-load_init_program(struct proc *p)
+load_init_program(proc_t p)
 {
 	vm_offset_t	init_addr;
+	int		argc = 0;
 	char		*argv[3];
 	int			error;
 	register_t 	retval[2];
 
-	error = 0;
-
-	/* init_args are copied in string form directly from bootstrap */
-	
-	do {
-		if (boothowto & RB_INITNAME) {
-			printf("init program? ");
-#if FIXME  /* [ */
-			gets(init_program_name, init_program_name);
-#endif  /* FIXME ] */
-		}
-
-		if (error && ((boothowto & RB_INITNAME) == 0) &&
-					(init_attempts == 1)) {
-			printf("Load of %s, errno %d, trying %s\n",
-				init_program_name, error, other_init);
-			error = 0;
-			bcopy(other_init, init_program_name,
-							sizeof(other_init));
-		}
-
-		init_attempts++;
-
-		if (error) {
-			printf("Load of %s failed, errno %d\n",
-					init_program_name, error);
-			error = 0;
-			boothowto |= RB_INITNAME;
-			continue;
-		}
-
-		/*
-		 *	Copy out program name.
-		 */
+	/*
+	 * Copy out program name.
+	 */
 
-		init_addr = VM_MIN_ADDRESS;
-		(void) vm_allocate(current_map(), &init_addr,
-				   PAGE_SIZE, VM_FLAGS_ANYWHERE);
-		if (init_addr == 0)
-			init_addr++;
+	init_addr = VM_MIN_ADDRESS;
+	(void) vm_allocate(current_map(), &init_addr, PAGE_SIZE,
+				VM_FLAGS_ANYWHERE);
+	if (init_addr == 0)
+		init_addr++;
 
-		(void) copyout((caddr_t) init_program_name,
-				CAST_USER_ADDR_T(init_addr),
-				(unsigned) sizeof(init_program_name)+1);
+	(void) copyout((caddr_t) init_program_name, CAST_USER_ADDR_T(init_addr),
+			(unsigned) sizeof(init_program_name)+1);
 
-		argv[0] = (char *) init_addr;
-		init_addr += sizeof(init_program_name);
-		init_addr = (vm_offset_t)ROUND_PTR(char, init_addr);
+	argv[argc++] = (char *) init_addr;
+	init_addr += sizeof(init_program_name);
+	init_addr = (vm_offset_t)ROUND_PTR(char, init_addr);
 
-		/*
-		 *	Put out first (and only) argument, similarly.
-		 *	Assumes everything fits in a page as allocated
-		 *	above.
-		 */
+	/*
+	 * Put out first (and only) argument, similarly.
+	 * Assumes everything fits in a page as allocated
+	 * above.
+	 */
+	if (boothowto & RB_SINGLE) {
+		const char *init_args = "-s";
 
-		(void) copyout((caddr_t) init_args,
-				CAST_USER_ADDR_T(init_addr),
-				(unsigned) sizeof(init_args));
+		copyout(init_args, CAST_USER_ADDR_T(init_addr),
+			strlen(init_args));
 
-		argv[1] = (char *) init_addr;
-		init_addr += sizeof(init_args);
+		argv[argc++] = (char *)init_addr;
+		init_addr += strlen(init_args);
 		init_addr = (vm_offset_t)ROUND_PTR(char, init_addr);
 
-		/*
-		 *	Null-end the argument list
-		 */
+	}
 
-		argv[2] = (char *) 0;
-		
-		/*
-		 *	Copy out the argument list.
-		 */
-		
-		(void) copyout((caddr_t) argv,
-				CAST_USER_ADDR_T(init_addr),
-				(unsigned) sizeof(argv));
+	/*
+	 * Null-end the argument list
+	 */
+	argv[argc] = NULL;
+	
+	/*
+	 * Copy out the argument list.
+	 */
+	
+	(void) copyout((caddr_t) argv, CAST_USER_ADDR_T(init_addr),
+			(unsigned) sizeof(argv));
 
-		/*
-		 *	Set up argument block for fake call to execve.
-		 */
+	/*
+	 * Set up argument block for fake call to execve.
+	 */
 
-		init_exec_args.fname = CAST_USER_ADDR_T(argv[0]);
-		init_exec_args.argp = CAST_USER_ADDR_T((char **)init_addr);
-		init_exec_args.envp = CAST_USER_ADDR_T(0);
-		
-		/* So that mach_init task 
-		 * is set with uid,gid 0 token 
-		 */
-		set_security_token(p);
+	init_exec_args.fname = CAST_USER_ADDR_T(argv[0]);
+	init_exec_args.argp = CAST_USER_ADDR_T((char **)init_addr);
+	init_exec_args.envp = CAST_USER_ADDR_T(0);
+	
+	/*
+	 * So that mach_init task is set with uid,gid 0 token 
+	 */
+	set_security_token(p);
 
-		error = execve(p,&init_exec_args,retval);
-	} while (error);
+	error = execve(p,&init_exec_args,retval);
+	if (error)
+		panic("Process 1 exec of %s failed, errno %d\n",
+		      init_program_name, error);
 }
 
 /*
- * Convert a load_return_t to an errno.
+ * load_return_to_errno
+ *
+ * Description:	Convert a load_return_t (Mach error) to an errno (BSD error)
+ *
+ * Parameters:	lrtn			Mach error number
+ *
+ * Returns:	(int)			BSD error number
+ *		0			Success
+ *		EBADARCH		Bad architecture
+ *		EBADMACHO		Bad Mach object file
+ *		ESHLIBVERS		Bad shared library version
+ *		ENOMEM			Out of memory/resource shortage
+ *		EACCES			Access denied
+ *		ENOENT			Entry not found (usually "file does
+ *					does not exist")
+ *		EIO			An I/O error occurred
+ *		EBADEXEC		The executable is corrupt/unknown
  */
 static int 
 load_return_to_errno(load_return_t lrtn)
 {
 	switch (lrtn) {
-	    case LOAD_SUCCESS:
-			return 0;
-	    case LOAD_BADARCH:
-	    	return EBADARCH;
-	    case LOAD_BADMACHO:
-	    	return EBADMACHO;
-	    case LOAD_SHLIB:
-	    	return ESHLIBVERS;
-	    case LOAD_NOSPACE:
-	    case LOAD_RESOURCE:
-	    	return ENOMEM;
-	    case LOAD_PROTECT:
-	    	return EACCES;
-		case LOAD_ENOENT:
-			return ENOENT;
-		case LOAD_IOERROR:
-			return EIO;
-	    case LOAD_FAILURE:
-	    default:
-	    	return EBADEXEC;
+	case LOAD_SUCCESS:
+		return 0;
+	case LOAD_BADARCH:
+		return EBADARCH;
+	case LOAD_BADMACHO:
+		return EBADMACHO;
+	case LOAD_SHLIB:
+		return ESHLIBVERS;
+	case LOAD_NOSPACE:
+	case LOAD_RESOURCE:
+		return ENOMEM;
+	case LOAD_PROTECT:
+		return EACCES;
+	case LOAD_ENOENT:
+		return ENOENT;
+	case LOAD_IOERROR:
+		return EIO;
+	case LOAD_FAILURE:
+	default:
+		return EBADEXEC;
 	}
 }
 
@@ -1963,8 +2940,36 @@ load_return_to_errno(load_return_t lrtn)
 extern semaphore_t execve_semaphore;
 
 /*
- * The block of memory used by the execve arguments.  At the same time,
- * we allocate a page so that we can read in the first page of the image.
+ * execargs_alloc
+ *
+ * Description:	Allocate the block of memory used by the execve arguments.
+ *		At the same time, we allocate a page so that we can read in
+ *		the first page of the image.
+ *
+ * Parameters:	struct image_params *	the image parameter block
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		EACCES			Permission denied
+ *		EINTR			Interrupted function
+ *		ENOMEM			Not enough space
+ *
+ * Notes:	This is a temporary allocation into the kernel address space
+ *		to enable us to copy arguments in from user space.  This is
+ *		necessitated by not mapping the process calling execve() into
+ *		the kernel address space during the execve() system call.
+ *
+ *		We assemble the argument and environment, etc., into this
+ *		region before copying it as a single block into the child
+ *		process address space (at the top or bottom of the stack,
+ *		depending on which way the stack grows; see the function
+ *		exec_copyout_strings() for details).
+ *
+ *		This ends up with a second (possibly unnecessary) copy compared
+ *		with assembing the data directly into the child address space,
+ *		instead, but since we cannot be guaranteed that the parent has
+ *		not modified its environment, we can't really know that it's
+ *		really a block there as well.
  */
 static int
 execargs_alloc(struct image_params *imgp)
@@ -1993,6 +2998,19 @@ execargs_alloc(struct image_params *imgp)
 	return (0);
 }
 
+/*
+ * execargs_free
+ *
+ * Description:	Free the block of memory used by the execve arguments and the
+ *		first page of the executable by a previous call to the function
+ *		execargs_alloc().
+ *
+ * Parameters:	struct image_params *	the image parameter block
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		EINTR			Oeration interrupted
+ */
 static int
 execargs_free(struct image_params *imgp)
 {
@@ -2015,3 +3033,38 @@ execargs_free(struct image_params *imgp)
 		return (EINVAL);
 	}
 }
+
+static void
+exec_resettextvp(proc_t p, struct image_params *imgp)
+{
+	vnode_t vp;
+	off_t offset;
+	vnode_t tvp  = p->p_textvp;
+	int ret;
+
+	vp = imgp->ip_vp;
+	offset = imgp->ip_arch_offset;
+
+	if (vp == NULLVP)
+		panic("exec_resettextvp: expected valid vp");
+
+	ret = vnode_ref(vp);
+	proc_lock(p);
+	if (ret == 0) {
+		p->p_textvp = vp;
+		p->p_textoff = offset;
+	} else {
+		p->p_textvp = NULLVP;	/* this is paranoia */
+		p->p_textoff = 0;
+	}
+	proc_unlock(p);
+
+	if ( tvp != NULLVP) {
+		if (vnode_getwithref(tvp) == 0) {
+			vnode_rele(tvp);
+			vnode_put(tvp);
+		}
+	}	
+
+}
+
diff --git a/bsd/kern/kern_exit.c b/bsd/kern/kern_exit.c
index 4a318232c..7bc8b1d74 100644
--- a/bsd/kern/kern_exit.c
+++ b/bsd/kern/kern_exit.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,12 @@
  *
  *	@(#)kern_exit.c	8.7 (Berkeley) 2/12/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
  
 #include <machine/reg.h>
 #include <machine/psl.h>
@@ -69,6 +81,7 @@
 #include <sys/systm.h>
 #include <sys/ioctl.h>
 #include <sys/proc_internal.h>
+#include <sys/proc.h>
 #include <sys/kauth.h>
 #include <sys/tty.h>
 #include <sys/time.h>
@@ -86,9 +99,10 @@
 #include <sys/sysproto.h>
 #include <sys/signalvar.h>
 #include <sys/filedesc.h>	/* fdfree */
+#if SYSV_SHM
 #include <sys/shm_internal.h>	/* shmexit */
+#endif
 #include <sys/acct.h>		/* acct_process */
-#include <machine/spl.h>
 
 #include <bsm/audit_kernel.h>
 #include <bsm/audit_kevents.h>
@@ -99,10 +113,21 @@
 #include <kern/kalloc.h>
 #include <kern/task.h>
 #include <kern/thread.h>
+#include <kern/thread_call.h>
 #include <kern/sched_prim.h>
 #include <kern/assert.h>
-#if KTRACE   
-#include <sys/ktrace.h>
+#if CONFIG_DTRACE
+/* Do not include dtrace.h, it redefines kmem_[alloc/free] */
+extern void (*dtrace_fasttrap_exit_ptr)(proc_t);
+extern void (*dtrace_helpers_cleanup)(proc_t);
+extern void dtrace_lazy_dofs_destroy(proc_t);
+
+#include <sys/dtrace_ptss.h>
+#endif
+
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <sys/syscall.h>
 #endif
 
 #include <mach/mach_types.h>
@@ -110,25 +135,30 @@
 #include <mach/thread_act.h>
 #include <mach/mach_traps.h>	/* init_process */
 
+#include <sys/sdt.h>
+
 extern char init_task_failure_data[];
-int exit1(struct proc *, int, int *);
-void proc_prepareexit(struct proc *p);
-void vfork_exit(struct proc *p, int rv);
-void vproc_exit(struct proc *p);
+void proc_prepareexit(proc_t p, int rv);
+void vfork_exit(proc_t p, int rv);
+void vproc_exit(proc_t p);
 __private_extern__ void munge_rusage(struct rusage *a_rusage_p, struct user_rusage *a_user_rusage_p);
+static int reap_child_locked(proc_t parent, proc_t child, int deadparent, int locked, int droplock);
 
 /*
  * Things which should have prototypes in headers, but don't
  */
-void	unix_syscall_return(int);
 void	*get_bsduthreadarg(thread_t);
-void	proc_exit(struct proc *p);
+void	proc_exit(proc_t p);
 int	wait1continue(int result);
 int	waitidcontinue(int result);
 int	*get_bsduthreadrval(thread_t);
-kern_return_t	sys_perf_notify(struct task *task, exception_data_t code,
-			mach_msg_type_number_t codeCnt);
-
+kern_return_t sys_perf_notify(thread_t thread, int pid);
+kern_return_t abnormal_exit_notify(mach_exception_data_type_t code, 
+		mach_exception_data_type_t subcode);
+int 	in_shutdown(void);
+void workqueue_exit(struct proc *);
+void	delay(int);
+			
 /*
  * NOTE: Source and target may *NOT* overlap!
  * XXX Should share code with bsd/dev/ppc/unix_signal.c
@@ -146,7 +176,7 @@ siginfo_64to32(user_siginfo_t *in, siginfo_t *out)
 	/* following cast works for sival_int because of padding */
 	out->si_value.sival_ptr	= CAST_DOWN(void *,in->si_value.sival_ptr);
 	out->si_band	= in->si_band;			/* range reduction */
-	out->pad[0]	= in->pad[0];			/* mcontext.ss.r1 */
+	out->__pad[0]	= in->pad[0];			/* mcontext.ss.r1 */
 }
 
 /*
@@ -154,12 +184,11 @@ siginfo_64to32(user_siginfo_t *in, siginfo_t *out)
  *	Death of process.
  */
 void
-exit(struct proc *p, struct exit_args *uap, int *retval)
+exit(proc_t p, struct exit_args *uap, int *retval)
 {
 	exit1(p, W_EXITCODE(uap->rval, 0), retval);
 
 	/* drop funnel before we return */
-	thread_funnel_set(kernel_flock, FALSE);
 	thread_exception_return();
 	/* NOTREACHED */
 	while (TRUE)
@@ -173,11 +202,10 @@ exit(struct proc *p, struct exit_args *uap, int *retval)
  * status and rusage for wait().  Check for child processes and orphan them.
  */
 int
-exit1(struct proc *p, int rv, int *retval)
+exit1(proc_t p, int rv, int *retval)
 {
 	thread_t self = current_thread();
 	struct task *task = p->task;
-	register int s;
 	struct uthread *ut;
 
 	/*
@@ -189,40 +217,55 @@ exit1(struct proc *p, int rv, int *retval)
 	 ut = get_bsdthread_info(self);
 	 if (ut->uu_flag & UT_VFORK) {
 			vfork_exit(p, rv);
-			vfork_return(self, p->p_pptr, p , retval);
+			vfork_return(p , retval, p->p_pid);
 			unix_syscall_return(0);
 			/* NOT REACHED */
 	 }
-	AUDIT_SYSCALL_EXIT(0, p, ut); /* Exit is always successfull */
-        signal_lock(p);
+
+	/* 
+	 * The parameter list of audit_syscall_exit() was augmented to
+	 * take the Darwin syscall number as the first parameter,
+	 * which is currently required by mac_audit_postselect().
+	 */
+
+	AUDIT_SYSCALL_EXIT(SYS_exit, p, ut, 0); /* Exit is always successfull */
+
+	DTRACE_PROC1(exit, int, CLD_EXITED);
+
+        proc_lock(p);
 	while (p->exit_thread != self) {
 		if (sig_try_locked(p) <= 0) {
 			if (get_threadtask(self) != task) {
-                                signal_unlock(p);
+				proc_unlock(p);
 				return(0);
                         }
-			signal_unlock(p);
+			proc_unlock(p);
 			thread_terminate(self);
-			thread_funnel_set(kernel_flock, FALSE);
 			thread_exception_return();
 			/* NOTREACHED */
 		}
 		sig_lock_to_exit(p);
 	}
-        signal_unlock(p);
+#if !CONFIG_EMBEDDED /* BER_XXX */
 	if (p->p_pid == 1) {
+		proc_unlock(p);
 		printf("pid 1 exited (signal %d, exit %d)",
 		    WTERMSIG(rv), WEXITSTATUS(rv));
-		panic("init died\nState at Last Exception:\n\n%s", 
+		panic("%s died\nState at Last Exception:\n\n%s", 
+							(p->p_comm[0] != '\0' ?
+								p->p_comm :
+								"launchd"),
 							init_task_failure_data);
 	}
+#endif
 
-	s = splsched();
-	p->p_flag |= P_WEXIT;
-	splx(s);
-	proc_prepareexit(p);
+	p->p_lflag |= P_LEXIT;
 	p->p_xstat = rv;
 
+	proc_unlock(p);
+
+	proc_prepareexit(p, rv);
+
 	/* task terminate will call proc_terminate and that cleans it up */
 	task_terminate_internal(task);
 
@@ -230,16 +273,39 @@ exit1(struct proc *p, int rv, int *retval)
 }
 
 void
-proc_prepareexit(struct proc *p) 
+proc_prepareexit(proc_t p, int rv) 
 {
+	mach_exception_data_type_t code, subcode;
 	struct uthread *ut;
-	exception_data_t	code[EXCEPTION_CODE_MAX];
 	thread_t self = current_thread();
+	ut = get_bsdthread_info(self);
 
-	code[0] = (exception_data_t)0xFF000001;		/* Set terminate code */
-	code[1] = (exception_data_t)p->p_pid;		/* Pass out the pid */
+ 	/* If a core should be generated, notify crash reporter */
+	if (!in_shutdown() && hassigprop(WTERMSIG(rv), SA_CORE)) {
+		/* 
+		 * Workaround for processes checking up on PT_DENY_ATTACH:
+		 * should be backed out post-Leopard (details in 5431025).
+		 */
+		if ((SIGSEGV == WTERMSIG(rv)) && 
+				(p->p_pptr->p_lflag & P_LNOATTACH)) {
+			goto skipcheck;
+		}
+
+		/*
+		 * Crash Reporter looks for the signal value, original exception
+		 * type, and low 20 bits of the original code in code[0] 
+		 * (8, 4, and 20 bits respectively). code[1] is unmodified. 
+		 */
+		code = ((WTERMSIG(rv) & 0xff) << 24) |
+			((ut->uu_exception & 0x0f) << 20) | 
+			((int)ut->uu_code & 0xfffff);
+		subcode = ut->uu_subcode;
+		(void) abnormal_exit_notify(code, subcode);
+	}
+
+skipcheck:
 	/* Notify the perf server */
-	(void)sys_perf_notify(p->task, (exception_data_t)&code, 2);
+	(void)sys_perf_notify(self, p->p_pid);
 
 	/*
 	 * Remove proc from allproc queue and from pidhash chain.
@@ -247,46 +313,92 @@ proc_prepareexit(struct proc *p)
 	 * Not doing causes things like mount() find this on allproc
 	 * in partially cleaned state.
 	 */
+
+	proc_list_lock();
+
 	LIST_REMOVE(p, p_list);
 	LIST_INSERT_HEAD(&zombproc, p, p_list);	/* Place onto zombproc. */
-	LIST_REMOVE(p, p_hash);
+	/* will not be visible via proc_find */
+	p->p_listflag |= P_LIST_EXITED;
+
+	proc_list_unlock();
+
 
 #ifdef PGINPROF
 	vmsizmon();
 #endif
 	/*
 	 * If parent is waiting for us to exit or exec,
-	 * P_PPWAIT is set; we will wakeup the parent below.
+	 * P_LPPWAIT is set; we will wakeup the parent below.
 	 */
-	p->p_flag &= ~(P_TRACED | P_PPWAIT);
+	proc_lock(p);
+	p->p_lflag &= ~(P_LTRACED | P_LPPWAIT);
 	p->p_sigignore = ~(sigcantmask);
-	p->p_siglist = 0;
-	ut = get_bsdthread_info(self);
 	ut->uu_siglist = 0;
-	untimeout(realitexpire, (caddr_t)p->p_pid);
+	proc_unlock(p);
 }
 
 void 
-proc_exit(struct proc *p)
+proc_exit(proc_t p)
 {
-	register struct proc *q, *nq, *pp;
+	proc_t q;
+	proc_t pp;
 	struct task *task = p->task;
-	register int s;
-	boolean_t funnel_state;
+	boolean_t fstate;
+	vnode_t tvp = NULLVP;
+	struct pgrp * pg;
+	struct session *sessp;
+	struct uthread * uth;
 
 	/* This can happen if thread_terminate of the single thread
 	 * process 
 	 */
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	if( !(p->p_flag & P_WEXIT)) {
-		s = splsched();
-		p->p_flag |= P_WEXIT;
-		splx(s);
-		proc_prepareexit(p);	
+	uth = (struct uthread *)get_bsdthread_info(current_thread());
+
+	proc_lock(p);
+	if( !(p->p_lflag & P_LEXIT)) {
+		p->p_lflag |= P_LEXIT;
+		proc_unlock(p);
+		proc_prepareexit(p, 0);	
+		proc_lock(p);
 	}
 
 	p->p_lflag |= P_LPEXIT;
+	proc_unlock(p);
+
+#if CONFIG_DTRACE
+	/*
+	 * Free any outstanding lazy dof entries. It is imperative we
+	 * always call dtrace_lazy_dofs_destroy, rather than null check
+	 * and call if !NULL. If we NULL test, during lazy dof faulting
+	 * we can race with the faulting code and proceed from here to
+	 * beyond the helpers cleanup. The lazy dof faulting will then
+	 * install new helpers which will never be cleaned up, and leak.
+	 */
+	dtrace_lazy_dofs_destroy(p);
+
+	/*
+	 * Clean up any DTrace helper actions or probes for the process.
+	 */
+	if (p->p_dtrace_helpers != NULL) {
+		(*dtrace_helpers_cleanup)(p);
+	}
+
+	/*
+	 * Clean up any DTrace probes associated with this process.
+	 */
+	/*
+	 * APPLE NOTE: We release ptss pages/entries in dtrace_fasttrap_exit_ptr(),
+	 * call this after dtrace_helpers_cleanup()
+	 */
+	proc_lock(p);
+	if (p->p_dtrace_probes && dtrace_fasttrap_exit_ptr) {
+		(*dtrace_fasttrap_exit_ptr)(p);
+	}
+	proc_unlock(p);
+#endif
+
 	/* XXX Zombie allocation may fail, in which case stats get lost */
 	MALLOC_ZONE(p->p_ru, struct rusage *,
 			sizeof (*p->p_ru), M_ZOMBIE, M_WAITOK);
@@ -296,13 +408,9 @@ proc_exit(struct proc *p)
 	 * already active.  MAY BLOCK!
 	 */
 	
-	p->p_lflag |= P_LREFDRAIN;
-	while (p->p_internalref) {
-		p->p_lflag |= P_LREFDRAINWAIT;
-		msleep(&p->p_internalref, (lck_mtx_t *)0, 0, "proc_refdrain", 0) ;
-	}
-	p->p_lflag &= ~P_LREFDRAIN;
-	p->p_lflag |= P_LREFDEAD;
+	proc_refdrain(p);
+
+	workqueue_exit(p);
 
 	_aio_exit( p );
 
@@ -312,18 +420,28 @@ proc_exit(struct proc *p)
 	 */
 	fdfree(p);
 
+#if SYSV_SHM
 	/* Close ref SYSV Shared memory*/
 	if (p->vm_shm)
 		shmexit(p);
+#endif
+#if SYSV_SEM
 	/* Release SYSV semaphores */
 	semexit(p);
+#endif
 	
-	if (SESS_LEADER(p)) {
-		register struct session *sp = p->p_session;
+	sessp = proc_session(p);
+	if (SESS_LEADER(p, sessp)) {
+
+		/* Protected by funnel for tty accesses */
+		fstate = thread_funnel_set(kernel_flock, TRUE);
 
-		if (sp->s_ttyvp) {
+		if (sessp->s_ttyvp != NULLVP) {
 			struct vnode *ttyvp;
+			int ttyvid;
 			struct vfs_context context;
+			struct tty * tp;
+
 
 			/*
 			 * Controlling process.
@@ -331,79 +449,135 @@ proc_exit(struct proc *p)
 			 * drain controlling terminal
 			 * and revoke access to controlling terminal.
 			 */
-			if (sp->s_ttyp->t_session == sp) {
-				if (sp->s_ttyp->t_pgrp)
-					pgsignal(sp->s_ttyp->t_pgrp, SIGHUP, 1);
-				(void) ttywait(sp->s_ttyp);
+			tp = sessp->s_ttyp;
+
+			if ((tp != TTY_NULL) && (tp->t_session == sessp)) {
+				tty_pgsignal(tp, SIGHUP, 1);
+				(void) ttywait(tp);
 				/*
 				 * The tty could have been revoked
 				 * if we blocked.
 				 */
-				context.vc_proc = p;
-				context.vc_ucred = kauth_cred_proc_ref(p);
-				if (sp->s_ttyvp)
-					VNOP_REVOKE(sp->s_ttyvp, REVOKEALL, &context);
-				kauth_cred_unref(&context.vc_ucred);
+
+				session_lock(sessp);
+				ttyvp = sessp->s_ttyvp;
+				ttyvid = sessp->s_ttyvid;
+				sessp->s_ttyvp = NULL;
+				sessp->s_ttyvid = 0;
+				sessp->s_ttyp = NULL;
+				sessp->s_ttypgrpid = NO_PID;
+				session_unlock(sessp);
+
+				if ((ttyvp != NULLVP) && (vnode_getwithvid(ttyvp, ttyvid) == 0)) {
+					context.vc_thread = proc_thread(p); /* XXX */
+					context.vc_ucred = kauth_cred_proc_ref(p);
+					VNOP_REVOKE(ttyvp, REVOKEALL, &context);
+					vnode_put(ttyvp);
+					kauth_cred_unref(&context.vc_ucred);
+				}
+			} else {
+				session_lock(sessp);
+				ttyvp = sessp->s_ttyvp;
+				sessp->s_ttyvp = NULL;
+				sessp->s_ttyvid = 0;
+				sessp->s_ttyp = NULL;
+				sessp->s_ttypgrpid = NO_PID;
+				session_unlock(sessp);
 			}
-			ttyvp = sp->s_ttyvp;
-			sp->s_ttyvp = NULL;
-			if (ttyvp) {
+			if (ttyvp)
 				vnode_rele(ttyvp);
-			}
 			/*
 			 * s_ttyp is not zero'd; we use this to indicate
 			 * that the session once had a controlling terminal.
 			 * (for logging and informational purposes)
 			 */
 		}
-		sp->s_leader = NULL;
+		
+		(void) thread_funnel_set(kernel_flock, fstate);
+		session_lock(sessp);
+		sessp->s_leader = NULL;
+		session_unlock(sessp);
 	}
+	session_rele(sessp);
+
+	pg = proc_pgrp(p);
+	fixjobc(p, pg, 0);
+	pg_rele(pg);
 
-	fixjobc(p, p->p_pgrp, 0);
 	p->p_rlimit[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
 	(void)acct_process(p);
 
-#if KTRACE
-	/* 
-	 * release trace file
-	 */
-	p->p_traceflag = 0;	/* don't trace the vnode_put() */
-	if (p->p_tracep) {
-		struct vnode *tvp = p->p_tracep;
-		p->p_tracep = NULL;
-		vnode_rele(tvp);
-	}
-#endif
-
-	while (q = p->p_children.lh_first) {
-		proc_reparent(q, initproc);
-		/*
-		 * Traced processes are killed
-		 * since their existence means someone is messing up.
-		 */
-		if (q->p_flag & P_TRACED) {
-			q->p_flag &= ~P_TRACED;
-			if (q->sigwait_thread) {
-				/*
-				 * The sigwait_thread could be stopped at a
-				 * breakpoint. Wake it up to kill.
-				 * Need to do this as it could be a thread which is not
-				 * the first thread in the task. So any attempts to kill
-				 * the process would result into a deadlock on q->sigwait.
-				 */
-				thread_resume((thread_t)q->sigwait_thread);
-				clear_wait(q->sigwait_thread, THREAD_INTERRUPTED);
-				threadsignal((thread_t)q->sigwait_thread, SIGKILL, 0);
+	proc_list_lock();
+	/* wait till parentrefs are dropped and grant no more */
+	proc_childdrainstart(p);
+	while ((q = p->p_children.lh_first) != NULL) {
+		q->p_listflag |= P_LIST_DEADPARENT;
+		if (q->p_stat == SZOMB) {
+			if (p != q->p_pptr)
+				panic("parent child linkage broken");
+			/* check for sysctl zomb lookup */
+			while ((q->p_listflag & P_LIST_WAITING) == P_LIST_WAITING) {
+				msleep(&q->p_stat, proc_list_mlock, PWAIT, "waitcoll", 0);
+			}
+			q->p_listflag |= P_LIST_WAITING;
+			/*
+			 * This is a named reference and it is not granted
+			 * if the reap is already in progress. So we get
+			 * the reference here exclusively and their can be
+			 * no waiters. So there is no need for a wakeup
+			 * after we are done. AlsO  the reap frees the structure
+			 * and the proc struct cannot be used for wakeups as well. 
+			 * It is safe to use q here as this is system reap
+			 */
+			(void)reap_child_locked(p, q, 1, 1, 0);
+		} else {
+			proc_reparentlocked(q, initproc, 0, 1);
+			/*
+		 	* Traced processes are killed
+		 	* since their existence means someone is messing up.
+		 	*/
+			if (q->p_lflag & P_LTRACED) {
+				proc_list_unlock();
+				proc_lock(q);
+				q->p_lflag &= ~P_LTRACED;
+				if (q->sigwait_thread) {
+					proc_unlock(q);
+					/*
+				 	* The sigwait_thread could be stopped at a
+				 	* breakpoint. Wake it up to kill.
+				 	* Need to do this as it could be a thread which is not
+				 	* the first thread in the task. So any attempts to kill
+				 	* the process would result into a deadlock on q->sigwait.
+				 	*/
+					thread_resume((thread_t)q->sigwait_thread);
+					clear_wait(q->sigwait_thread, THREAD_INTERRUPTED);
+					threadsignal((thread_t)q->sigwait_thread, SIGKILL, 0);
+				} else
+					proc_unlock(q);
+				psignal(q, SIGKILL);
+				proc_list_lock();
 			}
-			psignal(q, SIGKILL);
 		}
 	}
 
+	proc_childdrainend(p);
+	proc_list_unlock();
+
+	/*
+	 * Release reference to text vnode
+	 */
+	tvp = p->p_textvp;
+	p->p_textvp = NULL;
+	if (tvp != NULLVP) {
+		vnode_rele(tvp);
+	}
+
 	/*
 	 * Save exit status and final rusage info, adding in child rusage
 	 * info and self times.  If we were unable to allocate a zombie
 	 * structure, this information is lost.
 	 */
+	/* No need for locking here as no one than this thread can access this */
 	if (p->p_ru != NULL) {
 	    *p->p_ru = p->p_stats->p_ru;
 
@@ -411,19 +585,23 @@ proc_exit(struct proc *p)
 	    timerclear(&p->p_ru->ru_stime);
 
 	    if (task) {
-		task_basic_info_data_t tinfo;
+		task_basic_info_32_data_t tinfo;
 		task_thread_times_info_data_t ttimesinfo;
-		int task_info_stuff, task_ttimes_stuff;
+		task_events_info_data_t teventsinfo;
+		mach_msg_type_number_t task_info_stuff, task_ttimes_stuff;
+		mach_msg_type_number_t task_events_stuff;
 		struct timeval ut,st;
 
-		task_info_stuff	= TASK_BASIC_INFO_COUNT;
-		task_info(task, TASK_BASIC_INFO,
+		task_info_stuff	= TASK_BASIC_INFO_32_COUNT;
+		task_info(task, TASK_BASIC2_INFO_32,
 			  (task_info_t)&tinfo, &task_info_stuff);
 		p->p_ru->ru_utime.tv_sec = tinfo.user_time.seconds;
 		p->p_ru->ru_utime.tv_usec = tinfo.user_time.microseconds;
 		p->p_ru->ru_stime.tv_sec = tinfo.system_time.seconds;
 		p->p_ru->ru_stime.tv_usec = tinfo.system_time.microseconds;
 
+		p->p_ru->ru_maxrss = tinfo.resident_size;
+
 		task_ttimes_stuff = TASK_THREAD_TIMES_INFO_COUNT;
 		task_info(task, TASK_THREAD_TIMES_INFO,
 			  (task_info_t)&ttimesinfo, &task_ttimes_stuff);
@@ -434,6 +612,18 @@ proc_exit(struct proc *p)
 		st.tv_usec = ttimesinfo.system_time.microseconds;
 		timeradd(&ut,&p->p_ru->ru_utime,&p->p_ru->ru_utime);
 		timeradd(&st,&p->p_ru->ru_stime,&p->p_ru->ru_stime);
+
+		task_events_stuff = TASK_EVENTS_INFO_COUNT;
+		task_info(task, TASK_EVENTS_INFO,
+			  (task_info_t)&teventsinfo, &task_events_stuff);
+
+		p->p_ru->ru_minflt = (teventsinfo.faults -
+				      teventsinfo.pageins);
+		p->p_ru->ru_majflt = teventsinfo.pageins;
+		p->p_ru->ru_nivcsw = (teventsinfo.csw -
+				      p->p_ru->ru_nvcsw);
+		if (p->p_ru->ru_nivcsw < 0)
+			p->p_ru->ru_nivcsw = 0;
 	    }
 
 	    ruadd(p->p_ru, &p->p_stats->p_cru);
@@ -455,71 +645,148 @@ proc_exit(struct proc *p)
 		}
 	}
 
+	proc_spinlock(p);
+	if (thread_call_cancel(p->p_rcall))
+		p->p_ractive--;
+
+	while (p->p_ractive > 0) {
+		proc_spinunlock(p);
+		
+		delay(1);
+
+		proc_spinlock(p);
+	}
+	proc_spinunlock(p);
+
+	thread_call_free(p->p_rcall);
+	p->p_rcall = NULL;
+
 	/*
 	 * Other substructures are freed from wait().
 	 */
-	FREE_ZONE(p->p_stats, sizeof *p->p_stats, M_SUBPROC);
+	FREE_ZONE(p->p_stats, sizeof *p->p_stats, M_PSTATS);
 	p->p_stats = NULL;
 
-	FREE_ZONE(p->p_sigacts, sizeof *p->p_sigacts, M_SUBPROC);
+	FREE_ZONE(p->p_sigacts, sizeof *p->p_sigacts, M_SIGACTS);
 	p->p_sigacts = NULL;
 
-	if (--p->p_limit->p_refcnt == 0)
-		FREE_ZONE(p->p_limit, sizeof *p->p_limit, M_SUBPROC);
+	proc_limitdrop(p, 1);
 	p->p_limit = NULL;
 
+
 	/*
 	 * Finish up by terminating the task
 	 * and halt this thread (only if a
 	 * member of the task exiting).
 	 */
 	p->task = TASK_NULL;
-	//task->proc = NULL;
 	set_bsdtask_info(task, NULL);
 
-	KNOTE(&p->p_klist, NOTE_EXIT);
+	proc_knote(p, NOTE_EXIT);
 
+	/* mark the thread as the one that is doing proc_exit
+	 * no need to hold proc lock in uthread_free
+	 */
+	uth->uu_flag |= UT_PROCEXIT;
 	/*
 	 * Notify parent that we're gone.
 	 */
-	if (p->p_pptr->p_flag & P_NOCLDWAIT) {
-		struct proc *opp = p->p_pptr;
+	pp = proc_parent(p);
+	if (pp->p_flag & P_NOCLDWAIT) {
 
+#if 3839178
+		/*
+		 * If the parent is ignoring SIGCHLD, then POSIX requires
+		 * us to not add the resource usage to the parent process -
+		 * we are only going to hand it off to init to get reaped.
+		 * We should contest the standard in this case on the basis
+		 * of RLIMIT_CPU.
+		 */
+#else	/* !3839178 */
 		/*
 		 * Add child resource usage to parent before giving
 		 * zombie to init.  If we were unable to allocate a
 		 * zombie structure, this information is lost.
 		 */
-		if (p->p_ru != NULL)
-			ruadd(&p->p_pptr->p_stats->p_cru, p->p_ru);
+		if (p->p_ru != NULL) {
+			proc_lock(pp);
+			ruadd(&pp->p_stats->p_cru, p->p_ru);
+			proc_unlock(pp);
+		}
+#endif	/* !3839178 */
 
-		proc_reparent(p, initproc);
-		/* If there are no more children wakeup parent */
-		if (LIST_EMPTY(&opp->p_children))
-			wakeup((caddr_t)opp);
+		/* kernel can reap this one, no need to move it to launchd */
+		proc_list_lock();
+		p->p_listflag |= P_LIST_DEADPARENT;
+		proc_list_unlock();
 	}
-	/* should be fine as parent proc would be initproc */
-	pp = p->p_pptr;
-	if (pp != initproc) {
-		pp->si_pid = p->p_pid;
-		pp->si_status = p->p_xstat;
-		pp->si_code = CLD_EXITED;
-		pp->si_uid = p->p_ucred->cr_ruid;
-	}
-	/* mark as a zombie */
-	p->p_stat = SZOMB;
+	if ((p->p_listflag & P_LIST_DEADPARENT) == 0) {
+		if (pp != initproc) {
+			proc_lock(pp);
+			pp->si_pid = p->p_pid;
+			pp->si_status = p->p_xstat;
+			pp->si_code = CLD_EXITED;
+			/*
+			 * p_ucred usage is safe as it is an exiting process
+			 * and reference is dropped in reap
+			 */
+			pp->si_uid = p->p_ucred->cr_ruid;
+			proc_unlock(pp);
+		}
+		/* mark as a zombie */
+		/* No need to take proc lock as all refs are drained and
+		 * no one except parent (reaping ) can look at this.
+		 * The write is to an int and is coherent. Also parent is
+		 *  keyed off of list lock for reaping
+		 */
+		p->p_stat = SZOMB;
+		/* 
+		 * The current process can be reaped so, no one
+		 * can depend on this
+		 */
 
-	psignal(pp, SIGCHLD);
+		psignal(pp, SIGCHLD);
+	
+		/* and now wakeup the parent */
+		proc_list_lock();
+		wakeup((caddr_t)pp);
+		proc_list_unlock();
+	} else {
+		/* should be fine as parent proc would be initproc */
+		/* mark as a zombie */
+		/* No need to take proc lock as all refs are drained and
+		 * no one except parent (reaping ) can look at this.
+		 * The write is to an int and is coherent. Also parent is
+		 *  keyed off of list lock for reaping
+		 */
+		proc_list_lock();
+		p->p_stat = SZOMB;
+		/* check for sysctl zomb lookup */
+		while ((p->p_listflag & P_LIST_WAITING) == P_LIST_WAITING) {
+			msleep(&p->p_stat, proc_list_mlock, PWAIT, "waitcoll", 0);
+		}
+		/* safe to use p as this is a system reap */
+		p->p_listflag |= P_LIST_WAITING;
+		/*
+		 * This is a named reference and it is not granted
+		 * if the reap is already in progress. So we get
+		 * the reference here exclusively and their can be
+		 * no waiters. So there is no need for a wakeup
+		 * after we are done. AlsO  the reap frees the structure
+		 * and the proc struct cannot be used for wakeups as well. 
+		 * It is safe to use p here as this is system reap
+		 */
+		(void)reap_child_locked(pp, p, 1, 1, 1);
+		/* list lock dropped by reap_child_locked */
+	}
 
-	/* and now wakeup the parent */
-	wakeup((caddr_t)p->p_pptr);
+	proc_rele(pp);
 
-	(void) thread_funnel_set(kernel_flock, funnel_state);
 }
 
 
 /*
- * reap_child_process
+ * reap_child_locked
  *
  * Description:	Given a process from which all status information needed
  *		has already been extracted, if the process is a ptrace
@@ -527,39 +794,71 @@ proc_exit(struct proc *p)
  *		parent, else recover all resources remaining associated
  *		with it.
  *
- * Parameters:	struct proc *parent	Parent of process being reaped
- *		struct proc *child	Process to reap
+ * Parameters:	proc_t parent		Parent of process being reaped
+ *		proc_t child		Process to reap
  *
  * Returns:	0			Process was not reaped because it
  *					came from an attach
  *		1			Process was reaped
  */
 static int
-reap_child_process(struct proc *parent, struct proc *child)
+reap_child_locked(proc_t parent, proc_t child, int deadparent, int locked, int droplock)
 {
-	struct proc *trace_parent;	/* Traced parent process, if tracing */
-	struct vnode *tvp;		/* Traced vnode pointer, if used */
+	proc_t trace_parent;	/* Traced parent process, if tracing */
 
 	/*
 	 * If we got the child via a ptrace 'attach',
 	 * we need to give it back to the old parent.
 	 */
-	if (child->p_oppid && (trace_parent = pfind(child->p_oppid))) {
+	if (locked == 1)
+		proc_list_unlock();
+	if (child->p_oppid && (trace_parent = proc_find(child->p_oppid))) {
+		proc_lock(child);
 		child->p_oppid = 0;
-		proc_reparent(child, trace_parent);
+		proc_unlock(child);
 		if (trace_parent != initproc) {
+			/* 
+			 * proc internal fileds  and p_ucred usage safe 
+			 * here as child is dead and is not reaped or 
+			 * reparented yet 
+			 */
+			proc_lock(trace_parent);
 			trace_parent->si_pid = child->p_pid;
 			trace_parent->si_status = child->p_xstat;
 			trace_parent->si_code = CLD_CONTINUED;
 			trace_parent->si_uid = child->p_ucred->cr_ruid;
+			proc_unlock(trace_parent);
 		}
+		proc_reparentlocked(child, trace_parent, 1, 0);
 		psignal(trace_parent, SIGCHLD);
+		proc_list_lock();
 		wakeup((caddr_t)trace_parent);
+		child->p_listflag &= ~P_LIST_WAITING;
+		wakeup(&child->p_stat);
+		proc_list_unlock();
+		proc_rele(trace_parent);
+		if ((locked == 1) && (droplock == 0))
+			proc_list_lock();
 		return (0);
 	}
+
+	proc_knote(child, NOTE_REAP);
+
 	child->p_xstat = 0;
 	if (child->p_ru) {
-		ruadd(&parent->p_stats->p_cru, child->p_ru);
+		proc_lock(parent);
+#if 3839178
+		/*
+		 * If the parent is ignoring SIGCHLD, then POSIX requires
+		 * us to not add the resource usage to the parent process -
+		 * we are only going to hand it off to init to get reaped.
+		 * We should contest the standard in this case on the basis
+		 * of RLIMIT_CPU.
+		 */
+		if (!(parent->p_flag & P_NOCLDWAIT))
+#endif	/* 3839178 */
+			ruadd(&parent->p_stats->p_cru, child->p_ru);
+		proc_unlock(parent);
 		FREE_ZONE(child->p_ru, sizeof *child->p_ru, M_ZOMBIE);
 		child->p_ru = NULL;
 	} else {
@@ -568,9 +867,18 @@ reap_child_process(struct proc *parent, struct proc *child)
 
 	/*
 	 * Decrement the count of procs running with this uid.
+	 * p_ucred usage is safe here as it is an exited process.
+	 * and refernce is dropped after these calls down below
+	 * (locking protection is provided by list lock held in chgproccnt)
 	 */
 	(void)chgproccnt(child->p_ucred->cr_ruid, -1);
 
+#if CONFIG_LCTX
+	ALLLCTX_LOCK;
+	leavelctx(child);
+	ALLLCTX_UNLOCK;
+#endif
+
 	/*
 	 * Free up credentials.
 	 */
@@ -578,28 +886,44 @@ reap_child_process(struct proc *parent, struct proc *child)
 		kauth_cred_unref(&child->p_ucred);
 	}
 
-	/*
-	 * Release reference to text vnode
-	 */
-	tvp = child->p_textvp;
-	child->p_textvp = NULL;
-	if (tvp) {
-		vnode_rele(tvp);
-	}
+	/*  XXXX Note NOT SAFE TO USE p_ucred from this point onwards */
+
 	/*
 	 * Finally finished with old proc entry.
 	 * Unlink it from its process group and free it.
 	 */
 	leavepgrp(child);
+
+	proc_list_lock();
 	LIST_REMOVE(child, p_list);	/* off zombproc */
+	parent->p_childrencnt--;
 	LIST_REMOVE(child, p_sibling);
-	child->p_lflag &= ~P_LWAITING;
+	/* If there are no more children wakeup parent */
+	if ((deadparent != 0) && (LIST_EMPTY(&parent->p_children)))
+		wakeup((caddr_t)parent);	/* with list lock held */
+	child->p_listflag &= ~P_LIST_WAITING;
 	wakeup(&child->p_stat);
 
+	/* Take it out of process hash */
+	LIST_REMOVE(child, p_hash);
+	child->p_listflag &= ~P_LIST_INHASH;
+	proc_checkdeadrefs(child);
+	nprocs--;
+
+	proc_list_unlock();
+
 	lck_mtx_destroy(&child->p_mlock, proc_lck_grp);
 	lck_mtx_destroy(&child->p_fdmlock, proc_lck_grp);
+#if CONFIG_DTRACE
+	lck_mtx_destroy(&child->p_dtrace_sprlock, proc_lck_grp);
+#endif
+	lck_spin_destroy(&child->p_slock, proc_lck_grp);
+	workqueue_destroy_lock(child);
+
 	FREE_ZONE(child, sizeof *child, M_PROC);
-	nprocs--;
+	if ((locked == 1) && (droplock == 0))
+		proc_list_lock();
+
 	return (1);
 }
 
@@ -610,7 +934,7 @@ wait1continue(int result)
 	void *vt;
 	thread_t thread;
 	int *retval;
-	struct proc *p;
+	proc_t p;
 
 	if (result)
 		return(result);
@@ -619,48 +943,62 @@ wait1continue(int result)
 	thread = current_thread();
 	vt = get_bsduthreadarg(thread);
 	retval = get_bsduthreadrval(thread);
-	return(wait4((struct proc *)p, (struct wait4_args *)vt, retval));
+	return(wait4(p, (struct wait4_args *)vt, retval));
 }
 
 int
-wait4(struct proc *q, struct wait4_args *uap, register_t *retval)
+wait4(proc_t q, struct wait4_args *uap, register_t *retval)
 {
-	register int nfound;
-	register struct proc *p;
+	__pthread_testcancel(1);
+	return(wait4_nocancel(q, (struct wait4_nocancel_args *)uap, retval));
+}
+
+int
+wait4_nocancel(proc_t q, struct wait4_nocancel_args *uap, register_t *retval)
+{
+	int nfound;
+	proc_t p;
 	int status, error;
 
 	if (uap->pid == 0)
-		uap->pid = -q->p_pgid;
+		uap->pid = -q->p_pgrpid;
 
 loop:
+	proc_list_lock();
+loop1:
 	nfound = 0;
 	for (p = q->p_children.lh_first; p != 0; p = p->p_sibling.le_next) {
 		if (uap->pid != WAIT_ANY &&
 		    p->p_pid != uap->pid &&
-		    p->p_pgid != -(uap->pid))
+		    p->p_pgrpid != -(uap->pid))
 			continue;
+
 		nfound++;
 
 		/* XXX This is racy because we don't get the lock!!!! */
 
-		if (p->p_lflag & P_LWAITING) {
-			(void)tsleep(&p->p_stat, PWAIT, "waitcoll", 0);
-			goto loop;
+		if (p->p_listflag & P_LIST_WAITING) {
+			(void)msleep(&p->p_stat, proc_list_mlock, PWAIT, "waitcoll", 0);
+			goto loop1;
 		}
-		p->p_lflag |= P_LWAITING;   /* only allow single thread to wait() */
+		p->p_listflag |= P_LIST_WAITING;   /* only allow single thread to wait() */
+
 
 		if (p->p_stat == SZOMB) {
+			proc_list_unlock();
+#if CONFIG_MACF
+			if ((error = mac_proc_check_wait(q, p)) != 0)
+				goto out;
+#endif
 			retval[0] = p->p_pid;
 			if (uap->status) {
-				status = p->p_xstat;	/* convert to int */
+				/* Legacy apps expect only 8 bits of status */
+				status = 0xffff & p->p_xstat;	/* convert to int */
 				error = copyout((caddr_t)&status,
 				   			uap->status,
 						    sizeof(status));
-				if (error) {
-					p->p_lflag &= ~P_LWAITING;
-					wakeup(&p->p_stat);
-					return (error);
-				}
+				if (error) 
+					goto out;
 			}
 			if (uap->rusage) {
 				if (p->p_ru == NULL) {
@@ -680,24 +1018,30 @@ loop:
 					}
 				}
 				/* information unavailable? */
-				if (error) {
-					p->p_lflag &= ~P_LWAITING;
-					wakeup(&p->p_stat);
-					return (error);
-				}
+				if (error) 
+					goto out;
 			}
 
 			/* Clean up */
-			if (!reap_child_process(q, p)) {
-				p->p_lflag &= ~P_LWAITING;
+			if (!reap_child_locked(q, p, 0, 0, 0)) {
+				proc_list_lock();
+				p->p_listflag &= ~P_LIST_WAITING;
 				wakeup(&p->p_stat);
+				proc_list_unlock();
 			}
 
 			return (0);
 		}
-		if (p->p_stat == SSTOP && (p->p_flag & P_WAITED) == 0 &&
-		    (p->p_flag & P_TRACED || uap->options & WUNTRACED)) {
-			p->p_flag |= P_WAITED;
+		if (p->p_stat == SSTOP && (p->p_lflag & P_LWAITED) == 0 &&
+		    (p->p_lflag & P_LTRACED || uap->options & WUNTRACED)) {
+			proc_list_unlock();
+#if CONFIG_MACF
+			if ((error = mac_proc_check_wait(q, p)) != 0)
+				goto out;
+#endif
+			proc_lock(p);
+			p->p_lflag |= P_LWAITED;
+			proc_unlock(p);
 			retval[0] = p->p_pid;
 			if (uap->status) {
 				status = W_STOPCODE(p->p_xstat);
@@ -706,25 +1050,57 @@ loop:
 				    sizeof(status));
 			} else
 				error = 0;
-			p->p_lflag &= ~P_LWAITING;
-			wakeup(&p->p_stat);
-			return (error);
+			goto out;
+		}
+		/*
+		 * If we are waiting for continued processses, and this
+		 * process was continued
+		 */
+		if ((uap->options & WCONTINUED) &&
+		    (p->p_flag & P_CONTINUED)) {
+			proc_list_unlock();
+#if CONFIG_MACF
+			if ((error = mac_proc_check_wait(q, p)) != 0)
+				goto out;
+#endif
+
+			/* Prevent other process for waiting for this event */
+			OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&p->p_flag);
+			retval[0] = p->p_pid;
+			if (uap->status) {
+				status = W_STOPCODE(SIGCONT);
+				error = copyout((caddr_t)&status,
+					uap->status,
+				    sizeof(status));
+			} else
+				error = 0;
+			goto out;
 		}
-		p->p_lflag &= ~P_LWAITING;
+		p->p_listflag &= ~P_LIST_WAITING;
 		wakeup(&p->p_stat);
 	}
-	if (nfound == 0)
+	/* list lock is held when we get here any which way */
+	if (nfound == 0) {
+		proc_list_unlock();
 		return (ECHILD);
+	}
 
 	if (uap->options & WNOHANG) {
 		retval[0] = 0;
+		proc_list_unlock();
 		return (0);
 	}
 
-	if ((error = tsleep0((caddr_t)q, PWAIT | PCATCH, "wait", 0, wait1continue)))
+	if ((error = msleep0((caddr_t)q, proc_list_mlock, PWAIT | PCATCH | PDROP, "wait", 0, wait1continue)))
 		return (error);
 
 	goto loop;
+out:
+	proc_list_lock();
+	p->p_listflag &= ~P_LIST_WAITING;
+	wakeup(&p->p_stat);
+	proc_list_unlock();
+	return (error);
 }
 
 
@@ -734,16 +1110,14 @@ waitidcontinue(int result)
 	void *vt;
 	thread_t thread;
 	int *retval;
-	struct proc *p;
 
 	if (result)
 		return(result);
 
-	p = current_proc();
 	thread = current_thread();
 	vt = get_bsduthreadarg(thread);
 	retval = get_bsduthreadrval(thread);
-	return(waitid((struct proc *)p, (struct waitid_args *)vt, retval));
+	return(waitid(current_proc(), (struct waitid_args *)vt, retval));
 }
 
 /*
@@ -760,15 +1134,48 @@ waitidcontinue(int result)
  *		!0			Error returning status to user space
  */
 int
-waitid(struct proc *q, struct waitid_args *uap, register_t *retval)
+waitid(proc_t q, struct waitid_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(waitid_nocancel(q, (struct waitid_nocancel_args *)uap, retval));
+}
+
+int
+waitid_nocancel(proc_t q, struct waitid_nocancel_args *uap, __unused register_t *retval)
 {
 	user_siginfo_t	collect64;	/* siginfo data to return to caller */
 
-	register int nfound;
-	register struct proc *p;
+	int nfound;
+	proc_t p;
 	int error;
 
+	/*
+	 * Forced validation of options for T.waitpid 21; should be a TSD!
+	 * This will pass the test, but note that we have more bits than the
+	 * standard specifies that we will allow in, in this case.  The test
+	 * passes because they light all the bits, not just the ones we allow,
+	 * and so the following check returns EINVAL like the test wants.
+	 */
+	if (((uap->options & (WNOHANG|WNOWAIT|WCONTINUED|WUNTRACED|WSTOPPED|WEXITED)) != uap->options) ||
+	    (uap->options == 0))
+		return (EINVAL);	/* bits set that aren't recognized */
+
+	/*
+	 * Overly critical options checking, per POSIX
+	 */
+	switch(uap->idtype) {
+	case P_PID:	/* child with process ID equal to... */
+	case P_PGID:	/* child with process group ID equal to... */
+		if (((int)uap->id) < 0)
+			return (EINVAL);
+		break;
+	case P_ALL:	/* any child */
+		break;
+	}
+
 loop:
+	proc_list_lock();
+loop1:
 	nfound = 0;
 	for (p = q->p_children.lh_first; p != 0; p = p->p_sibling.le_next) {
 		switch(uap->idtype) {
@@ -777,7 +1184,7 @@ loop:
 				continue;
 			break;
 		case P_PGID:	/* child with process group ID equal to... */
-			if (p->p_pgid != (pid_t)uap->id)
+			if (p->p_pgrpid != (pid_t)uap->id)
 				continue;
 			break;
 		case P_ALL:	/* any child */
@@ -790,11 +1197,11 @@ loop:
 		 * Wait collision; go to sleep and restart; used to maintain
 		 * the single return for waited process guarantee.
 		 */
-		if (p->p_lflag & P_LWAITING) {
-			(void)tsleep(&p->p_stat, PWAIT, "waitidcoll", 0);
-			goto loop;
+		if (p->p_listflag & P_LIST_WAITING) {
+			(void)msleep(&p->p_stat, proc_list_mlock, PWAIT, "waitidcoll", 0);
+			goto loop1;
 		}
-		p->p_lflag |= P_LWAITING;		/* mark busy */
+		p->p_listflag |= P_LIST_WAITING;		/* mark busy */
 
 		nfound++;
 
@@ -808,14 +1215,17 @@ loop:
 			if (!(uap->options & WEXITED))
 				break;
 
+			/* drop the lock and the thread is going to return */
+			proc_list_unlock();
+
 			/* Collect "siginfo" information for caller */
-			collect64.si_signo = 0;
+			collect64.si_signo = SIGCHLD;
 			collect64.si_code = 0;
 			collect64.si_errno = 0;
 			collect64.si_pid = 0;
 			collect64.si_uid = 0;
 			collect64.si_addr  = 0;
-			collect64.si_status = p->p_xstat;
+			collect64.si_status = WEXITSTATUS(p->p_xstat);
 			collect64.si_band = 0;
 
 			if (IS_64BIT_PROCESS(p)) {
@@ -830,19 +1240,22 @@ loop:
 					sizeof(collect));
 			}
 			/* information unavailable? */
-			if (error) {
-				p->p_lflag &= ~P_LWAITING;
-				wakeup(&p->p_stat);
-				return (error);
-			}
+			if (error) 
+				goto out;
 
 			/* Prevent other process for waiting for this event? */
 			if (!(uap->options & WNOWAIT)) {
 				/* Clean up */
-				if (!reap_child_process(q, p)) {
-					p->p_lflag &= ~P_LWAITING;
+				if (!reap_child_locked(q, p, 0, 0, 0)) {
+					proc_list_lock();
+					p->p_listflag &= ~P_LIST_WAITING;
 					wakeup(&p->p_stat);
+					proc_list_unlock();
 				}
+			} else {
+				proc_list_lock();
+				p->p_listflag &= ~P_LIST_WAITING;
+				proc_list_unlock();
 			}
 
 			return (0);
@@ -859,25 +1272,22 @@ loop:
 			 * If someone has already waited it, we lost a race
 			 * to be the one to return status.
 			 */
-			if ((p->p_flag & P_WAITED) != 0)
+			if ((p->p_lflag & P_LWAITED) != 0)
 				break;
 
-			/*
-			 * If this is not a traced process, and they haven't
-			 * indicated an interest in untraced processes, then
-			 * ignore this one.
-			 */
-			if (!(p->p_flag & P_TRACED) && !(uap->options & WUNTRACED))
-			    	break;
+			/* drop the lock and the thread is going to return */
+			proc_list_unlock();
 
 			/* Collect "siginfo" information for caller */
-			collect64.si_signo = 0;
+			collect64.si_signo = SIGCHLD;
 			collect64.si_code = 0;
 			collect64.si_errno = 0;
 			collect64.si_pid = 0;
 			collect64.si_uid = 0;
 			collect64.si_addr  = 0;
+			proc_lock(p);
 			collect64.si_status = p->p_xstat;
+			proc_unlock(p);
 			collect64.si_band = 0;
 
 			if (IS_64BIT_PROCESS(p)) {
@@ -892,20 +1302,18 @@ loop:
 					sizeof(collect));
 			}
 			/* information unavailable? */
-			if (error) {
-				p->p_lflag &= ~P_LWAITING;
-				wakeup(&p->p_stat);
-				return (error);
-			}
+			if (error)
+				goto out;
 
 			/* Prevent other process for waiting for this event? */
 			if (!(uap->options & WNOWAIT)) {
-				p->p_flag |= P_WAITED;
+				proc_lock(p);
+				p->p_lflag |= P_LWAITED;
+				proc_unlock(p);
 			}
 
-			p->p_lflag &= ~P_LWAITING;
-			wakeup(&p->p_stat);
-			return (0);
+			error = 0;
+			goto out;
 
 		default:		/* All others */
 					/* ...meaning Continued */
@@ -920,15 +1328,20 @@ loop:
 			if ((p->p_flag & P_CONTINUED) == 0)
 				break;
 
+			/* drop the lock and the thread is going to return */
+			proc_list_unlock();
+
 			/* Collect "siginfo" information for caller */
-			collect64.si_signo = 0;
-			collect64.si_code = 0;
+			proc_lock(p);
+			collect64.si_signo = SIGCHLD;
+			collect64.si_code = CLD_CONTINUED;
 			collect64.si_errno = 0;
-			collect64.si_pid = 0;
+			collect64.si_pid = p->p_contproc;
 			collect64.si_uid = 0;
 			collect64.si_addr  = 0;
 			collect64.si_status = p->p_xstat;
 			collect64.si_band = 0;
+			proc_unlock(p);
 
 			if (IS_64BIT_PROCESS(p)) {
 				error = copyout((caddr_t)&collect64,
@@ -942,61 +1355,89 @@ loop:
 					sizeof(collect));
 			}
 			/* information unavailable? */
-			if (error) {
-				p->p_lflag &= ~P_LWAITING;
-				wakeup(&p->p_stat);
-				return (error);
-			}
+			if (error)
+				goto out;
 
 			/* Prevent other process for waiting for this event? */
 			if (!(uap->options & WNOWAIT)) {
-				p->p_flag &= ~P_CONTINUED;
+				OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&p->p_flag);
 			}
 
-			p->p_lflag &= ~P_LWAITING;
-			wakeup(&p->p_stat);
-			return (0);
-
-			break;
+			error = 0;
+			goto out;
 		}
-
-
+		/* LIST LOCK IS HELD HERE */
 		/* Not a process we are interested in; go on to next child */
-		p->p_lflag &= ~P_LWAITING;
+		
+		p->p_listflag &= ~P_LIST_WAITING;
 		wakeup(&p->p_stat);
 	}
 
+	/* list lock is always held */
 	/* No child processes that could possibly satisfy the request? */
-	if (nfound == 0)
+	if (nfound == 0) {
+		proc_list_unlock();
 		return (ECHILD);
+	}
 
 	if (uap->options & WNOHANG) {
-		retval[0] = 0;
+		proc_list_unlock();
 		return (0);
 	}
 
-	if ((error = tsleep0((caddr_t)q, PWAIT | PCATCH, "waitid", 0, waitidcontinue)))
+	if ((error = msleep0((caddr_t)q, proc_list_mlock, PWAIT | PCATCH | PDROP, "waitid", 0, waitidcontinue)))
 		return (error);
 
 	goto loop;
+out:
+	proc_list_lock();
+	p->p_listflag &= ~P_LIST_WAITING;
+	wakeup(&p->p_stat);
+	proc_list_unlock();
+	return (error);
 }
 
 /*
  * make process 'parent' the new parent of process 'child'.
  */
 void
-proc_reparent(struct proc *child, struct proc *parent)
+proc_reparentlocked(proc_t child, proc_t parent, int cansignal, int locked)
 {
+	proc_t oldparent = PROC_NULL;
 
 	if (child->p_pptr == parent)
 		return;
 
+	if (locked == 0)
+		proc_list_lock();
+
+	oldparent = child->p_pptr;
+#if __PROC_INTERNAL_DEBUG
+	if (oldparent == PROC_NULL)
+		panic("proc_reparent: process %x does not have a parent\n", (unsigned int)child);
+#endif
+
 	LIST_REMOVE(child, p_sibling);
+#if __PROC_INTERNAL_DEBUG
+	if (oldparent->p_childrencnt == 0)
+		panic("process children count already 0\n");
+#endif
+	oldparent->p_childrencnt--;
+#if __PROC_INTERNAL_DEBUG1
+	if (oldparent->p_childrencnt < 0)
+		panic("process children count -ve\n");
+#endif
 	LIST_INSERT_HEAD(&parent->p_children, child, p_sibling);
+	parent->p_childrencnt++;	
 	child->p_pptr = parent;
+	child->p_ppid = parent->p_pid;
+
+	proc_list_unlock();
 
-	if (initproc == parent && child->p_stat == SZOMB)
+	if ((cansignal != 0) && (initproc == parent) && (child->p_stat == SZOMB))
 		psignal(initproc, SIGCHLD);
+	if (locked == 1)
+		proc_list_lock();
 }
 
 /*
@@ -1007,7 +1448,7 @@ proc_reparent(struct proc *child, struct proc *parent)
 kern_return_t
 init_process(__unused struct init_process_args *args)
 {
-	register struct proc *p = current_proc();
+	proc_t p = current_proc();
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_INITPROCESS);
 	if (suser(kauth_cred_get(), &p->p_acflag)) {
@@ -1015,18 +1456,22 @@ init_process(__unused struct init_process_args *args)
 		return(KERN_NO_ACCESS);
 	}
 
-	if (p->p_pid != 1 && p->p_pgid != p->p_pid)
+	if (p->p_pid != 1 && p->p_pgrpid != p->p_pid)
 		enterpgrp(p, p->p_pid, 0);
-	p->p_flag |= P_SYSTEM;
+	OSBitOrAtomic(P_SYSTEM, (UInt32 *)&p->p_flag);
 
 	/*
 	 *	Take us out of the sibling chain, and
 	 *	out of our parent's child chain.
 	 */
+	proc_list_lock();
 	LIST_REMOVE(p, p_sibling);
 	p->p_sibling.le_prev = NULL;
 	p->p_sibling.le_next = NULL;
 	p->p_pptr = kernproc;
+	p->p_ppid = 0;
+	proc_list_unlock();
+
 
 	AUDIT_MACH_SYSCALL_EXIT(KERN_SUCCESS);
 	return(KERN_SUCCESS);
@@ -1040,15 +1485,19 @@ init_process(__unused struct init_process_args *args)
  */
 
 void
-vfork_exit(struct proc *p, int rv)
+vfork_exit(proc_t p, int rv)
+{
+	vfork_exit_internal(p, rv, 0);
+}
+
+void
+vfork_exit_internal(proc_t p, int rv, int forceexit)
 {
 	thread_t self = current_thread();
 #ifdef FIXME
 	struct task *task = p->task;
 #endif
-	register int s;
 	struct uthread *ut;
-	exception_data_t	code[EXCEPTION_CODE_MAX];
 
 	/*
 	 * If a thread in this task has already
@@ -1057,39 +1506,31 @@ vfork_exit(struct proc *p, int rv)
 	 */
 
 	 ut = get_bsdthread_info(self);
-#ifdef FIXME
-        signal_lock(p);
-	while (p->exit_thread != self) {
-		if (sig_try_locked(p) <= 0) {
-			if (get_threadtask(self) != task) {
-                                signal_unlock(p);
-				return;
-                        }
-			signal_unlock(p);
-			thread_terminate(self);
-			thread_funnel_set(kernel_flock, FALSE);
-			thread_exception_return();
-			/* NOTREACHED */
-		}
-		sig_lock_to_exit(p);
-	}
-        signal_unlock(p);
-	if (p->p_pid == 1) {
-		printf("pid 1 exited (signal %d, exit %d)",
-		    WTERMSIG(rv), WEXITSTATUS(rv));
-panic("init died\nState at Last Exception:\n\n%s", init_task_failure_data);
-	}
-#endif /* FIXME */
 
-	s = splsched();
-	p->p_flag |= P_WEXIT;
-	p->p_lflag |= P_LPEXIT;
-	splx(s);
 
-	code[0] = (exception_data_t)0xFF000001;		/* Set terminate code */
-	code[1] = (exception_data_t)p->p_pid;		/* Pass out the pid */
-	/* Notify the perf server */
-	(void)sys_perf_notify(p->task, (exception_data_t)&code, 2);
+	proc_lock(p);
+	 if ((p->p_lflag & P_LPEXIT) == P_LPEXIT) {
+		/* 
+	 	* This happens when a parent exits/killed and vfork is in progress  
+		* other threads. But shutdown code for ex has already called exit1()
+	 	*/
+		proc_unlock(p);
+		return;
+	}
+	p->p_lflag |= (P_LEXIT | P_LPEXIT);
+	proc_unlock(p);
+
+	if (forceexit == 0) {
+		/*
+		 * parent of a vfork child has already called exit() and the 
+		 * thread that has vfork in proress terminates. So there is no
+		 * separate address space here and it has already been marked for
+		 * termination. This was never covered before and could cause problems
+		 * if we block here for outside code.
+		 */
+		/* Notify the perf server */
+		(void)sys_perf_notify(self, p->p_pid);
+	}
 
 	/*
 	 * Remove proc from allproc queue and from pidhash chain.
@@ -1097,49 +1538,81 @@ panic("init died\nState at Last Exception:\n\n%s", init_task_failure_data);
 	 * Not doing causes things like mount() find this on allproc
 	 * in partially cleaned state.
 	 */
+
+	proc_list_lock();
+
 	LIST_REMOVE(p, p_list);
 	LIST_INSERT_HEAD(&zombproc, p, p_list);	/* Place onto zombproc. */
-	LIST_REMOVE(p, p_hash);
-	/*
-	 * If parent is waiting for us to exit or exec,
-	 * P_PPWAIT is set; we will wakeup the parent below.
-	 */
-	p->p_flag &= ~(P_TRACED | P_PPWAIT);
-	p->p_sigignore = ~0;
-	p->p_siglist = 0;
+	/* will not be visible via proc_find */
+	p->p_listflag |= P_LIST_EXITED;
 
-	ut->uu_siglist = 0;
-	untimeout(realitexpire, (caddr_t)p->p_pid);
+	proc_list_unlock();
 
+	proc_lock(p);
 	p->p_xstat = rv;
+	p->p_lflag &= ~(P_LTRACED | P_LPPWAIT);
+	p->p_sigignore = ~0;
+	proc_unlock(p);
+
+	proc_spinlock(p);
+	if (thread_call_cancel(p->p_rcall))
+		p->p_ractive--;
+
+	while (p->p_ractive > 0) {
+		proc_spinunlock(p);
+		
+		delay(1);
+
+		proc_spinlock(p);
+	}
+	proc_spinunlock(p);
+
+	thread_call_free(p->p_rcall);
+	p->p_rcall = NULL;
+
+	ut->uu_siglist = 0;
 
 	vproc_exit(p);
 }
 
 void 
-vproc_exit(struct proc *p)
+vproc_exit(proc_t p)
 {
-	register struct proc *q, *nq, *pp;
+	proc_t q;
+	proc_t pp;
+	
+	vnode_t tvp;
 #ifdef FIXME
 	struct task *task = p->task;
 #endif
+	struct pgrp * pg;
+	struct session *sessp;
+	boolean_t fstate;
 
 	/* XXX Zombie allocation may fail, in which case stats get lost */
 	MALLOC_ZONE(p->p_ru, struct rusage *,
 			sizeof (*p->p_ru), M_ZOMBIE, M_WAITOK);
 
+
+	proc_refdrain(p);
+
 	/*
 	 * Close open files and release open-file table.
 	 * This may block!
 	 */
 	fdfree(p);
 
-	if (SESS_LEADER(p)) {
-		register struct session *sp = p->p_session;
+	sessp = proc_session(p);
+	if (SESS_LEADER(p, sessp)) {
+		
+		/* Protected by funnel for tty accesses */
+		fstate = thread_funnel_set(kernel_flock, TRUE);
 
-		if (sp->s_ttyvp) {
+		if (sessp->s_ttyvp != NULLVP) {
 			struct vnode *ttyvp;
+			int ttyvid;
 			struct vfs_context context;
+			struct tty * tp;
 
 			/*
 			 * Controlling process.
@@ -1147,78 +1620,134 @@ vproc_exit(struct proc *p)
 			 * drain controlling terminal
 			 * and revoke access to controlling terminal.
 			 */
-			if (sp->s_ttyp->t_session == sp) {
-				if (sp->s_ttyp->t_pgrp)
-					pgsignal(sp->s_ttyp->t_pgrp, SIGHUP, 1);
-				(void) ttywait(sp->s_ttyp);
+			tp = sessp->s_ttyp;
+
+			if ((tp != TTY_NULL) && (tp->t_session == sessp)) {
+				tty_pgsignal(tp, SIGHUP, 1);
+				(void) ttywait(tp);
 				/*
 				 * The tty could have been revoked
 				 * if we blocked.
 				 */
-				context.vc_proc = p;
-				context.vc_ucred = kauth_cred_proc_ref(p);
-				if (sp->s_ttyvp)
-					VNOP_REVOKE(sp->s_ttyvp, REVOKEALL, &context);
-				kauth_cred_unref(&context.vc_ucred);
+
+				session_lock(sessp);
+				ttyvp = sessp->s_ttyvp;
+				ttyvid = sessp->s_ttyvid;
+				sessp->s_ttyvp = NULL;
+				sessp->s_ttyvid = 0;
+				sessp->s_ttyp = NULL;
+				sessp->s_ttypgrpid = NO_PID;
+				session_unlock(sessp);
+
+			       if ((ttyvp != NULLVP) && (vnode_getwithvid(ttyvp, ttyvid) == 0)) {
+					context.vc_thread = proc_thread(p); /* XXX */
+					context.vc_ucred = kauth_cred_proc_ref(p);
+					VNOP_REVOKE(ttyvp, REVOKEALL, &context);
+					vnode_put(ttyvp);
+					kauth_cred_unref(&context.vc_ucred);
+				}
+			} else {
+				session_lock(sessp);
+				ttyvp = sessp->s_ttyvp;
+				sessp->s_ttyvp = NULL;
+				sessp->s_ttyvid = 0;
+				sessp->s_ttyp = NULL;
+				sessp->s_ttypgrpid = NO_PID;
+				session_unlock(sessp);
 			}
-			ttyvp = sp->s_ttyvp;
-			sp->s_ttyvp = NULL;
-			if (ttyvp) {
+			if (ttyvp) 
 				vnode_rele(ttyvp);
-			}
 			/*
 			 * s_ttyp is not zero'd; we use this to indicate
 			 * that the session once had a controlling terminal.
 			 * (for logging and informational purposes)
 			 */
 		}
-		sp->s_leader = NULL;
+		(void) thread_funnel_set(kernel_flock, fstate);
+
+		session_lock(sessp);
+		sessp->s_leader = NULL;
+		session_unlock(sessp);
 	}
+	session_rele(sessp);
 
-	fixjobc(p, p->p_pgrp, 0);
-	p->p_rlimit[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
+	pg = proc_pgrp(p);
+	fixjobc(p, pg, 0);
+	pg_rele(pg);
 
-#if KTRACE
-	/* 
-	 * release trace file
-	 */
-	p->p_traceflag = 0;	/* don't trace the vnode_rele() */
-	if (p->p_tracep) {
-		struct vnode *tvp = p->p_tracep;
-		p->p_tracep = NULL;
-		vnode_rele(tvp);
-	}
-#endif
+	p->p_rlimit[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
 
-	while (q = p->p_children.lh_first) {
-		proc_reparent(q, initproc);
-		/*
-		 * Traced processes are killed
-		 * since their existence means someone is messing up.
-		 */
-		if (q->p_flag & P_TRACED) {
-			q->p_flag &= ~P_TRACED;
-			if (q->sigwait_thread) {
-				/*
-				 * The sigwait_thread could be stopped at a
-				 * breakpoint. Wake it up to kill.
-				 * Need to do this as it could be a thread which is not
-				 * the first thread in the task. So any attempts to kill
-				 * the process would result into a deadlock on q->sigwait.
-				 */
-				thread_resume((thread_t)q->sigwait_thread);
-				clear_wait(q->sigwait_thread, THREAD_INTERRUPTED);
-				threadsignal((thread_t)q->sigwait_thread, SIGKILL, 0);
+	proc_list_lock();
+	proc_childdrainstart(p);
+	while ((q = p->p_children.lh_first) != NULL) {
+		q->p_listflag |= P_LIST_DEADPARENT;
+		if (q->p_stat == SZOMB) {
+			if (p != q->p_pptr)
+				panic("parent child linkage broken");
+			/* check for lookups by zomb sysctl */
+			while ((q->p_listflag & P_LIST_WAITING) == P_LIST_WAITING) {
+				msleep(&q->p_stat, proc_list_mlock, PWAIT, "waitcoll", 0);
+			}
+			q->p_listflag |= P_LIST_WAITING;
+			/*
+			 * This is a named reference and it is not granted
+			 * if the reap is already in progress. So we get
+			 * the reference here exclusively and their can be
+			 * no waiters. So there is no need for a wakeup
+			 * after we are done. AlsO  the reap frees the structure
+			 * and the proc struct cannot be used for wakeups as well. 
+			 * It is safe to use q here as this is system reap
+			 */
+			(void)reap_child_locked(p, q, 1, 1, 0);
+		} else {
+			proc_reparentlocked(q, initproc, 0, 1);
+			/*
+		 	* Traced processes are killed
+		 	* since their existence means someone is messing up.
+		 	*/
+			if (q->p_lflag & P_LTRACED) {
+				proc_list_unlock();
+				proc_lock(q);
+				q->p_lflag &= ~P_LTRACED;
+				if (q->sigwait_thread) {
+					proc_unlock(q);
+					/*
+				 	* The sigwait_thread could be stopped at a
+				 	* breakpoint. Wake it up to kill.
+				 	* Need to do this as it could be a thread which is not
+				 	* the first thread in the task. So any attempts to kill
+				 	* the process would result into a deadlock on q->sigwait.
+				 	*/
+					thread_resume((thread_t)q->sigwait_thread);
+					clear_wait(q->sigwait_thread, THREAD_INTERRUPTED);
+					threadsignal((thread_t)q->sigwait_thread, SIGKILL, 0);
+				} else
+					proc_unlock(q);
+					
+				psignal(q, SIGKILL);
+				proc_list_lock();
 			}
-			psignal(q, SIGKILL);
 		}
 	}
 
+	proc_childdrainend(p);
+	proc_list_unlock();
+
+	/*
+	 * Release reference to text vnode
+	 */
+	tvp = p->p_textvp;
+	p->p_textvp = NULL;
+	if (tvp != NULLVP) {
+		vnode_rele(tvp);
+	}
+
 	/*
 	 * Save exit status and final rusage info, adding in child rusage
 	 * info and self times.  If we were unable to allocate a zombie
 	 * structure, this information is lost.
 	 */
+	/* No need for locking here as no one than this thread can access this */
 	if (p->p_ru != NULL) {
 	    *p->p_ru = p->p_stats->p_ru;
 	    timerclear(&p->p_ru->ru_utime);
@@ -1274,14 +1803,13 @@ vproc_exit(struct proc *p)
 	/*
 	 * Other substructures are freed from wait().
 	 */
-	FREE_ZONE(p->p_stats, sizeof *p->p_stats, M_SUBPROC);
+	FREE_ZONE(p->p_stats, sizeof *p->p_stats, M_PSTATS);
 	p->p_stats = NULL;
 
-	FREE_ZONE(p->p_sigacts, sizeof *p->p_sigacts, M_SUBPROC);
+	FREE_ZONE(p->p_sigacts, sizeof *p->p_sigacts, M_SIGACTS);
 	p->p_sigacts = NULL;
 
-	if (--p->p_limit->p_refcnt == 0)
-		FREE_ZONE(p->p_limit, sizeof *p->p_limit, M_SUBPROC);
+	proc_limitdrop(p, 1);
 	p->p_limit = NULL;
 
 	/*
@@ -1294,20 +1822,56 @@ vproc_exit(struct proc *p)
 	/*
 	 * Notify parent that we're gone.
 	 */
-	pp = p->p_pptr;
-	if (pp != initproc) {
-		pp->si_pid = p->p_pid;
-		pp->si_status = p->p_xstat;
-		pp->si_code = CLD_EXITED;
-		pp->si_uid = p->p_ucred->cr_ruid;
-	}
-	/* mark as a zombie */
-	p->p_stat = SZOMB;
+	pp = proc_parent(p);
+	if ((p->p_listflag & P_LIST_DEADPARENT) == 0) {
+		if (pp != initproc) {
+			proc_lock(pp);
+			pp->si_pid = p->p_pid;
+			pp->si_status = p->p_xstat;
+			pp->si_code = CLD_EXITED;
+			/*
+			 * p_ucred usage is safe as it is an exiting process
+			 * and reference is dropped in reap
+			 */
+			pp->si_uid = p->p_ucred->cr_ruid;
+			proc_unlock(pp);
+		}
+		/* mark as a zombie */
+		/* mark as a zombie */
+		/* No need to take proc lock as all refs are drained and
+		 * no one except parent (reaping ) can look at this.
+		 * The write is to an int and is coherent. Also parent is
+		 *  keyed off of list lock for reaping
+		 */
+		p->p_stat = SZOMB;
 
-	psignal(p->p_pptr, SIGCHLD);
+		psignal(pp, SIGCHLD);
 
-	/* and now wakeup the parent */
-	wakeup((caddr_t)p->p_pptr);
+		/* and now wakeup the parent */
+		proc_list_lock();
+		wakeup((caddr_t)pp);
+		proc_list_unlock();
+	} else {
+		proc_list_lock();
+		p->p_stat = SZOMB;
+		/* check for lookups by zomb sysctl */
+		while ((p->p_listflag & P_LIST_WAITING) == P_LIST_WAITING) {
+			msleep(&p->p_stat, proc_list_mlock, PWAIT, "waitcoll", 0);
+		}
+		p->p_listflag |= P_LIST_WAITING;
+		/*
+		 * This is a named reference and it is not granted
+		 * if the reap is already in progress. So we get
+		 * the reference here exclusively and their can be
+		 * no waiters. So there is no need for a wakeup
+		 * after we are done. AlsO  the reap frees the structure
+		 * and the proc struct cannot be used for wakeups as well. 
+		 * It is safe to use p here as this is system reap
+		 */
+		(void)reap_child_locked(pp, p, 0, 1, 1);
+		/* list lock dropped by reap_child_locked */
+	}
+	proc_rele(pp);
 }
 
 						
diff --git a/bsd/kern/kern_fork.c b/bsd/kern/kern_fork.c
index ed4b0cabb..fa43edd2e 100644
--- a/bsd/kern/kern_fork.c
+++ b/bsd/kern/kern_fork.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,18 @@
  *
  *	@(#)kern_fork.c	8.8 (Berkeley) 2/14/95
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <kern/assert.h>
 #include <sys/param.h>
@@ -73,8 +91,15 @@
 #include <sys/vnode_internal.h>
 #include <sys/file_internal.h>
 #include <sys/acct.h>
-#if KTRACE
-#include <sys/ktrace.h>
+#include <sys/codesign.h>
+#include <sys/sysproto.h>
+#if CONFIG_DTRACE
+/* Do not include dtrace.h, it redefines kmem_[alloc/free] */
+extern void dtrace_fasttrap_fork(proc_t, proc_t);
+extern void (*dtrace_helpers_fork)(proc_t, proc_t);
+extern void dtrace_lazy_dofs_duplicate(proc_t, proc_t);
+
+#include <sys/dtrace_ptss.h>
 #endif
 
 #include <bsm/audit_kernel.h>
@@ -84,41 +109,84 @@
 #include <kern/kalloc.h>
 #include <kern/mach_param.h>
 #include <kern/task.h>
+#include <kern/thread_call.h>
 #include <kern/zalloc.h>
 
 #include <machine/spl.h>
 
-#include <vm/vm_protos.h>       // for vm_map_commpage64
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_mach_internal.h>
+#endif
+
+#include <vm/vm_map.h>
+#include <vm/vm_protos.h>
+#include <vm/vm_shared_region.h>
+
+#include <sys/shm_internal.h>	/* for shmfork() */
+#include <mach/task.h>		/* for thread_create() */
+#include <mach/thread_act.h>	/* for thread_resume() */
 
-thread_t cloneproc(struct proc *, int); 
-struct proc * forkproc(struct proc *, int);
-thread_t procdup(struct proc *child, struct proc *parent);
+#include <sys/sdt.h>
+
+/* XXX routines which should have Mach prototypes, but don't */
+void thread_set_parent(thread_t parent, int pid);
+extern void act_thread_catt(void *ctx);
+void thread_set_child(thread_t child, int pid);
+void *act_thread_csave(void);
+
+
+thread_t cloneproc(proc_t, int); 
+proc_t forkproc(proc_t, int);
+void forkproc_free(proc_t, int);
+thread_t procdup(proc_t parent, proc_t child);
+thread_t fork_create_child(task_t parent_task, proc_t child, int inherit_memory, int is64bit);
 
 #define	DOFORK	0x1	/* fork() system call */
 #define	DOVFORK	0x2	/* vfork() system call */
-static int fork1(struct proc *, long, register_t *);
 
-/*
- * fork system call.
- */
-int
-fork(struct proc *p, __unused void *uap, register_t *retval)
-{
-	return (fork1(p, (long)DOFORK, retval));
-}
 
 /*
- * vfork system call
+ * vfork
+ *
+ * Description:	vfork system call
+ *
+ * Parameters:	void			[no arguments]
+ *
+ * Retval:	0			(to child process)
+ *		!0			pid of child (to parent process)
+ *		-1			error (see "Returns:")
+ *
+ * Returns:	EAGAIN			Administrative limit reached
+ *		EINVAL			vfork() caled during vfork()
+ *		ENOMEM			Failed to allocate new process
+ *
+ * Note:	After a successful call to this function, the parent process
+ *		has its task, thread, and uthread lent to the child process,
+ *		and control is returned to the caller; if this function is
+ *		invoked as a system call, the return is to user space, and
+ *		is effectively running on the child process.
+ *
+ *		Subsequent calls that operate on process state are permitted,
+ *		though discouraged, and will operate on the child process; any
+ *		operations on the task, thread, or uthread will result in
+ *		changes in the parent state, and, if inheritable, the child
+ *		state, when a task, thread, and uthread are realized for the
+ *		child process at execve() time, will also be effected.  Given
+ *		this, it's recemmended that people use the posix_spawn() call
+ *		instead.
  */
 int
-vfork(struct proc *p, void *uap, register_t *retval)
+vfork(proc_t parent, __unused struct vfork_args *uap, register_t *retval)
 {
-	register struct proc * newproc;
-	register uid_t uid;
+	proc_t child;
+	uid_t uid;
 	thread_t cur_act = (thread_t)current_thread();
 	int count;
-	task_t t;
 	uthread_t ut;
+#if CONFIG_MACF
+	int err;
+#endif
 
 	/*
 	 * Although process entries are dynamically created, we still keep
@@ -128,50 +196,90 @@ vfork(struct proc *p, void *uap, register_t *retval)
 	 * processes, maxproc is the limit.
 	 */
 	uid = kauth_cred_get()->cr_ruid;
+	proc_list_lock();
 	if ((nprocs >= maxproc - 1 && uid != 0) || nprocs >= maxproc) {
+		proc_list_unlock();
 		tablefull("proc");
 		retval[1] = 0;
 		return (EAGAIN);
 	}
+	proc_list_unlock();
 
 	/*
 	 * Increment the count of procs running with this uid. Don't allow
-	 * a nonprivileged user to exceed their current limit.
+	 * a nonprivileged user to exceed their current limit, which is
+	 * always less than what an rlim_t can hold.
+	 * (locking protection is provided by list lock held in chgproccnt)
 	 */
 	count = chgproccnt(uid, 1);
-	if (uid != 0 && count > p->p_rlimit[RLIMIT_NPROC].rlim_cur) {
+	if (uid != 0 &&
+	    (rlim_t)count > parent->p_rlimit[RLIMIT_NPROC].rlim_cur) {
 		(void)chgproccnt(uid, -1);
 		return (EAGAIN);
 	}
 
-	ut = (struct uthread *)get_bsdthread_info(cur_act);
+	ut = (uthread_t)get_bsdthread_info(cur_act);
 	if (ut->uu_flag & UT_VFORK) {
-		printf("vfork called recursively by %s\n", p->p_comm);
+		printf("vfork called recursively by %s\n", parent->p_comm);
 		(void)chgproccnt(uid, -1);
 		return (EINVAL);
 	}
-	p->p_flag  |= P_VFORK;
-	p->p_vforkcnt++;
+
+#if CONFIG_MACF
+	/*
+	 * Determine if MAC policies applied to the process will allow
+	 * it to fork.
+	 */
+	err = mac_proc_check_fork(parent);
+	if (err  != 0) {
+		(void)chgproccnt(uid, -1);
+		return (err);
+	}
+#endif
+
+	proc_lock(parent);
+	parent->p_lflag  |= P_LVFORK;
+	parent->p_vforkcnt++;
+	proc_unlock(parent);
 
 	/* The newly created process comes with signal lock held */
-	newproc = (struct proc *)forkproc(p,1);
-
-	AUDIT_ARG(pid, newproc->p_pid);
-
-	LIST_INSERT_AFTER(p, newproc, p_pglist);
-	newproc->p_pptr = p;
-	newproc->task = p->task;
-	LIST_INSERT_HEAD(&p->p_children, newproc, p_sibling);
-	LIST_INIT(&newproc->p_children);
-	LIST_INSERT_HEAD(&allproc, newproc, p_list);
-	LIST_INSERT_HEAD(PIDHASH(newproc->p_pid), newproc, p_hash);
-	TAILQ_INIT(& newproc->p_evlist);
-	newproc->p_stat = SRUN;
-	newproc->p_flag  |= P_INVFORK;
-	newproc->p_vforkact = cur_act;
+	if ((child = forkproc(parent,1)) == NULL) {
+		/* Failed to allocate new process */
+		(void)chgproccnt(uid, -1);
+		/*
+		 * XXX kludgy, but necessary without a full flags audit...
+		 * XXX these are inherited by the child, which depends on
+		 * XXX P_VFORK being set.
+		 */
+		proc_lock(parent);
+		parent->p_lflag &= ~P_LVFORK;
+		parent->p_vforkcnt--;
+		proc_unlock(parent);
+		return (ENOMEM);
+	}
+
+#if CONFIG_MACF
+	/* allow policies to associate the credential/label  */
+	/* that we referenced from the parent ... with the child */
+	/* JMM - this really isn't safe, as we can drop that */
+	/*       association without informing the policy in other */
+	/*       situations (keep long enough to get policies changed) */
+	mac_cred_label_associate_fork(child->p_ucred, child);
+#endif
+
+	AUDIT_ARG(pid, child->p_pid);
+
+	child->task = parent->task;
+
+	/* make child visible */
+	pinsertchild(parent, child);
+
+	child->p_lflag  |= P_LINVFORK;
+	child->p_vforkact = cur_act;
+	child->p_stat = SRUN;
 
 	ut->uu_flag |= UT_VFORK;
-	ut->uu_proc = newproc;
+	ut->uu_proc = child;
 	ut->uu_userstate = (void *)act_thread_csave();
 	ut->uu_vforkmask = ut->uu_sigmask;
 
@@ -181,47 +289,66 @@ vfork(struct proc *p, void *uap, register_t *retval)
 		ut->uu_flag &= ~UT_SETUID;
 	}
 	
-	thread_set_child(cur_act, newproc->p_pid);
+	thread_set_child(cur_act, child->p_pid);
 
-	microtime(&newproc->p_stats->p_start);
-	newproc->p_acflag = AFORK;
+	microtime(&child->p_start);
+	microtime(&child->p_stats->p_start); /* for compat sake */
+	child->p_acflag = AFORK;
 
 	/*
 	 * Preserve synchronization semantics of vfork.  If waiting for
 	 * child to exec or exit, set P_PPWAIT on child, and sleep on our
 	 * proc (in case of exit).
 	 */
-	newproc->p_flag |= P_PPWAIT;
+	child->p_lflag |= P_LPPWAIT;
 
 	/* drop the signal lock on the child */
-	signal_unlock(newproc);
+	proc_signalend(child, 0);
+	proc_transend(child, 0);
 
-	retval[0] = newproc->p_pid;
-	retval[1] = 1;			/* mark child */
+	retval[0] = child->p_pid;
+	retval[1] = 1;			/* flag child return for user space */
+
+	DTRACE_PROC1(create, proc_t, child);
 
 	return (0);
 }
 
 /*
- * Return to parent vfork ehread()
+ * vfork_return
+ *
+ * Description:	"Return" to parent vfork thread() following execve/_exit;
+ *		this is done by reassociating the parent process structure
+ *		with the task, thread, and uthread.
+ *
+ * Parameters:	child			Child process
+ *		retval			System call return value array
+ *		rval			Return value to present to parent
+ *
+ * Returns:	void
+ *
+ * Note:	The caller resumes or exits the parent, as appropriate, after
+ *		callling this function.
  */
 void
-vfork_return(__unused thread_t th_act, struct proc *p, struct proc *p2,
-	register_t *retval)
+vfork_return(proc_t child, register_t *retval, int rval)
 {
+	proc_t parent = child->p_pptr;
 	thread_t cur_act = (thread_t)current_thread();
 	uthread_t ut;
 	
-	ut = (struct uthread *)get_bsdthread_info(cur_act);
+	ut = (uthread_t)get_bsdthread_info(cur_act);
 
 	act_thread_catt(ut->uu_userstate);
 
 	/* Make sure only one at this time */
-	p->p_vforkcnt--;
-	if (p->p_vforkcnt <0)
+	proc_lock(parent);
+	parent->p_vforkcnt--;
+	if (parent->p_vforkcnt <0)
 		panic("vfork cnt is -ve");
-	if (p->p_vforkcnt <=0)
-		p->p_flag  &= ~P_VFORK;
+	if (parent->p_vforkcnt <=0)
+		parent->p_lflag  &= ~P_LVFORK;
+	proc_unlock(parent);
 	ut->uu_userstate = 0;
 	ut->uu_flag &= ~UT_VFORK;
 	/* restore thread-set-id state */
@@ -231,79 +358,196 @@ vfork_return(__unused thread_t th_act, struct proc *p, struct proc *p2,
 	}
 	ut->uu_proc = 0;
 	ut->uu_sigmask = ut->uu_vforkmask;
-	p2->p_flag  &= ~P_INVFORK;
-	p2->p_vforkact = (void *)0;
+	child->p_lflag  &= ~P_LINVFORK;
+	child->p_vforkact = (void *)0;
 
-	thread_set_parent(cur_act, p2->p_pid);
+	thread_set_parent(cur_act, rval);
 
 	if (retval) {
-		retval[0] = p2->p_pid;
+		retval[0] = rval;
 		retval[1] = 0;			/* mark parent */
 	}
 
 	return;
 }
 
+
+/*
+ * fork_create_child
+ *
+ * Description:	Common operations associated with the creation of a child
+ *		process
+ *
+ * Parameters:	parent_task		parent task
+ *		child			child process
+ *		inherit_memory		TRUE, if the parents address space is
+ *					to be inherited by the child
+ *		is64bit			TRUE, if the child being created will
+ *					be associated with a 64 bit process
+ *					rather than a 32 bit process
+ *
+ * Note:	This code is called in the fork() case, from the execve() call
+ *		graph, if implementing an execve() following a vfork(), from
+ *		the posix_spawn() call graph (which implicitly includes a
+ *		vfork() equivalent call, and in the system bootstrap case.
+ *
+ *		It creates a new task and thread (and as a side effect of the
+ *		thread creation, a uthread), which is then associated with the
+ *		process 'child'.  If the parent process address space is to
+ *		be inherited, then a flag indicates that the newly created
+ *		task should inherit this from the child task.
+ *
+ *		As a special concession to bootstrapping the initial process
+ *		in the system, it's possible for 'parent_task' to be TASK_NULL;
+ *		in this case, 'inherit_memory' MUST be FALSE.
+ */
 thread_t
-procdup(struct proc *child, struct proc *parent)
+fork_create_child(task_t parent_task, proc_t child, int inherit_memory, int is64bit)
 {
-	thread_t		thread;
-	task_t			task;
- 	kern_return_t	result;
+	thread_t	child_thread = NULL;
+	task_t		child_task;
+	kern_return_t	result;
+
+	/* Create a new task for the child process */
+	result = task_create_internal(parent_task,
+					inherit_memory,
+					is64bit,
+					&child_task);
+	if (result != KERN_SUCCESS) {
+		printf("execve: task_create_internal failed.  Code: %d\n", result);
+		goto bad;
+	}
 
-	if (parent->task == kernel_task)
-		result = task_create_internal(TASK_NULL, FALSE, FALSE, &task);
+	/* Set the child task to the new task */
+	child->task = child_task;
+
+	/* Set child task proc to child proc */
+	set_bsdtask_info(child_task, child);
+
+	/* Propagate CPU limit timer from parent */
+	if (timerisset(&child->p_rlim_cpu))
+		task_vtimer_set(child_task, TASK_VTIMER_RLIM);
+
+	/* Set/clear 64 bit vm_map flag */
+	if (is64bit)
+		vm_map_set_64bit(get_task_map(child_task));
 	else
-		result = task_create_internal(parent->task, TRUE, (parent->p_flag & P_LP64), &task);
-	if (result != KERN_SUCCESS)
-	    printf("fork/procdup: task_create failed. Code: 0x%x\n", result);
-	child->task = task;
-	/* task->proc = child; */
-	set_bsdtask_info(task, child);
-	if (parent->p_flag & P_LP64) {
-		task_set_64bit(task, TRUE);
-		vm_map_set_64bit(get_task_map(task));
-		child->p_flag |= P_LP64;
-                /* LP64todo - clean up this hacked mapping of commpage */
-		pmap_map_sharedpage(task, get_map_pmap(get_task_map(task)));
-                vm_map_commpage64(get_task_map(task));
-	} else {
-		task_set_64bit(task, FALSE);
-		vm_map_set_32bit(get_task_map(task));
-		child->p_flag &= ~P_LP64;
-#ifdef __i386__
-		/*
-		 * On Intel, the comm page doesn't get mapped automatically
-		 * because it goes beyond the end of the VM map in the current
-		 * 3GB/1GB address space model.
-		 * XXX This explicit mapping will probably become unnecessary
-		 * when we switch to the new 4GB/4GB address space model.
-		 */
-		vm_map_commpage32(get_task_map(task));
-#endif	/* __i386__ */
-	}
+		vm_map_set_32bit(get_task_map(child_task));
+
+#if CONFIG_MACF
+	/* Update task for MAC framework */
+	/* valid to use p_ucred as child is still not running ... */
+	mac_task_label_update_cred(child->p_ucred, child_task);
+#endif
+
+	/* Set child scheduler priority if nice value inherited from parent */
 	if (child->p_nice != 0)
 		resetpriority(child);
-		
-	result = thread_create(task, &thread);
-	if (result != KERN_SUCCESS)
-	    printf("fork/procdup: thread_create failed. Code: 0x%x\n", result);
 
-	return(thread);
+	/* Create a new thread for the child process */
+	result = thread_create(child_task, &child_thread);
+	if (result != KERN_SUCCESS) {
+		printf("execve: thread_create failed. Code: %d\n", result);
+		task_deallocate(child_task);
+		child_task = NULL;
+	}
+bad:
+	thread_yield_internal(1);
+
+	return(child_thread);
 }
 
 
-static int
-fork1(p1, flags, retval)
-	struct proc *p1;
-	long flags;
-	register_t *retval;
+/*
+ * procdup
+ *
+ * Description:	Givben a parent process, provide a duplicate task and thread
+ *		for a child process of that parent.
+ *
+ * Parameters:	parent			Parent process to use as the template
+ *		child			Child process to duplicate into
+ *
+ * Returns:	!NULL			Child process thread pointer
+ *		NULL			Failure (unspecified)
+ *
+ * Note:	Most of the heavy lifting is done by fork_create_child(); this
+ *		function exists more or less to deal with the 64 bit commpage,
+ *		which requires explicit inheritance, the x86 commpage, which
+ *		should not need explicit mapping any more, but apparently does,
+ *		and to be variant for the bootstrap process.
+ *
+ *		There is a special case where the system is being bootstraped,
+ *		where this function will be called from cloneproc(), called in
+ *		turn from bsd_utaskbootstrap().  In this case, we are acting
+ *		to create a task and thread (and uthread) for the benefit of
+ *		the kernel process - the first process in the system (PID 0).
+ *
+ *		In that specific case, we will *not* pass a parent task, since
+ *		there is *not* parent task present to pass.
+ *
+ * XXX:		This function should go away; the variance can moved into
+ * XXX:		cloneproc(), and the 64bit commpage code can be moved into
+ * XXX:		fork_create_child(), after the x86 commpage inheritance is
+ * XXX:		corrected.
+ */
+thread_t
+procdup(proc_t parent, proc_t child)
 {
-	register struct proc *p2;
-	register uid_t uid;
+	thread_t		child_thread;
+	task_t			child_task;
+
+	if (parent->task == kernel_task)
+		child_thread = fork_create_child(TASK_NULL, child, FALSE, FALSE);
+	else
+		child_thread = fork_create_child(parent->task, child, TRUE, (parent->p_flag & P_LP64));
+
+	if (child_thread != NULL) {
+		child_task = get_threadtask(child_thread);
+		if (parent->p_flag & P_LP64) {
+			task_set_64bit(child_task, TRUE);
+			OSBitOrAtomic(P_LP64, (UInt32 *)&child->p_flag);
+#ifdef __ppc__
+			/* LP64todo - clean up hacked mapping of commpage */
+			/* 
+			 * PPC51: ppc64 is limited to 51-bit addresses.
+			 * Memory above that limit is handled specially at
+			 * the pmap level.
+			 */
+			 pmap_map_sharedpage(child_task, get_map_pmap(get_task_map(child_task)));
+#endif /* __ppc__ */
+		} else {
+			task_set_64bit(child_task, FALSE);
+			OSBitAndAtomic(~((uint32_t)P_LP64), (UInt32 *)&child->p_flag);
+		}
+	}
+
+	return(child_thread);
+}
+
+
+/*
+ * fork
+ *
+ * Description:	fork system call.
+ *
+ * Parameters:	parent			Parent process to fork
+ *		uap (void)		[unused]
+ *		retval			Return value
+ *
+ * Returns:	0			Success
+ *		EAGAIN			Resource unavailable, try again
+ */
+int
+fork(proc_t parent, __unused struct fork_args *uap, register_t *retval)
+{
+	proc_t child;
+	uid_t uid;
 	thread_t newth;
 	int count;
 	task_t t;
+#if CONFIG_MACF
+	int err;
+#endif
 
 	/*
 	 * Although process entries are dynamically created, we still keep
@@ -313,47 +557,123 @@ fork1(p1, flags, retval)
 	 * processes, maxproc is the limit.
 	 */
 	uid = kauth_cred_get()->cr_ruid;
+	proc_list_lock();
 	if ((nprocs >= maxproc - 1 && uid != 0) || nprocs >= maxproc) {
+		proc_list_unlock();
 		tablefull("proc");
 		retval[1] = 0;
 		return (EAGAIN);
 	}
+	proc_list_unlock();
 
 	/*
 	 * Increment the count of procs running with this uid. Don't allow
-	 * a nonprivileged user to exceed their current limit.
+	 * a nonprivileged user to exceed their current limit, which is
+	 * always less than what an rlim_t can hold.
+	 * (locking protection is provided by list lock held in chgproccnt)
 	 */
 	count = chgproccnt(uid, 1);
-	if (uid != 0 && count > p1->p_rlimit[RLIMIT_NPROC].rlim_cur) {
+	if (uid != 0 &&
+	    (rlim_t)count > parent->p_rlimit[RLIMIT_NPROC].rlim_cur) {
 		(void)chgproccnt(uid, -1);
 		return (EAGAIN);
 	}
 
+#if CONFIG_MACF
+	/*
+	 * Determine if MAC policies applied to the process will allow
+	 * it to fork.
+	 */
+	err = mac_proc_check_fork(parent);
+	if (err != 0) {
+		(void)chgproccnt(uid, -1);
+		return (err);
+	}
+#endif
+
 	/* The newly created process comes with signal lock held */
-	newth = cloneproc(p1, 1);
+	if ((newth = cloneproc(parent, 1)) == NULL) {
+		/* Failed to create thread */
+		(void)chgproccnt(uid, -1);
+		return (EAGAIN);
+	}
+
 	thread_dup(newth);
-	/* p2 = newth->task->proc; */
-	p2 = (struct proc *)(get_bsdtask_info(get_threadtask(newth)));
-	set_security_token(p2);         /* propagate change of PID */
+	/* child = newth->task->proc; */
+	child = (proc_t)(get_bsdtask_info(get_threadtask(newth)));
+
+#if CONFIG_MACF
+	/* inform policies of new process sharing this cred/label */
+	/* safe to use p_ucred here since child is not running */
+	/* JMM - unsafe to assume the association will stay - as */
+	/*        there are other ways it can be dropped without */
+	/*	  informing the policies. */
+	mac_cred_label_associate_fork(child->p_ucred, child);
+#endif
 
-	AUDIT_ARG(pid, p2->p_pid);
+	/* propogate change of PID - may get new cred if auditing */
+	set_security_token(child);
 
-	thread_set_child(newth, p2->p_pid);
+	AUDIT_ARG(pid, child->p_pid);
 
-	microtime(&p2->p_stats->p_start);
-	p2->p_acflag = AFORK;
+	thread_set_child(newth, child->p_pid);
 
+	microtime(&child->p_start);
+	microtime(&child->p_stats->p_start);	/* for compat sake */
+	child->p_acflag = AFORK;
+
+#if CONFIG_DTRACE
 	/*
-	 * Preserve synchronization semantics of vfork.  If waiting for
-	 * child to exec or exit, set P_PPWAIT on child, and sleep on our
-	 * proc (in case of exit).
+	 * APPLE NOTE: Solaris does a sprlock() and drops the proc_lock
+	 * here. We're cheating a bit and only taking the p_dtrace_sprlock
+	 * lock. A full sprlock would task_suspend the parent.
+	 */
+	lck_mtx_lock(&parent->p_dtrace_sprlock);
+
+	/*
+	 * Remove all DTrace tracepoints from the child process. We
+	 * need to do this _before_ duplicating USDT providers since
+	 * any associated probes may be immediately enabled.
+	 */
+	if (parent->p_dtrace_count > 0) {
+		dtrace_fasttrap_fork(parent, child);
+	}
+
+	lck_mtx_unlock(&parent->p_dtrace_sprlock);
+
+	/*
+	 * Duplicate any lazy dof(s). This must be done while NOT
+	 * holding the parent sprlock! Lock ordering is dtrace_dof_mode_lock,
+	 * then sprlock. It is imperative we always call
+	 * dtrace_lazy_dofs_duplicate, rather than null check and
+	 * call if !NULL. If we NULL test, during lazy dof faulting
+	 * we can race with the faulting code and proceed from here to
+	 * beyond the helpers copy. The lazy dof faulting will then
+	 * fail to copy the helpers to the child process.
+	 */
+	dtrace_lazy_dofs_duplicate(parent, child);
+	
+	/*
+	 * Duplicate any helper actions and providers. The SFORKING
+	 * we set above informs the code to enable USDT probes that
+	 * sprlock() may fail because the child is being forked.
 	 */
-	if (flags == DOVFORK)
-		p2->p_flag |= P_PPWAIT;
+	/*
+	 * APPLE NOTE: As best I can tell, Apple's sprlock() equivalent
+	 * never fails to find the child. We do not set SFORKING.
+	 */
+	if (parent->p_dtrace_helpers != NULL && dtrace_helpers_fork) {
+		(*dtrace_helpers_fork)(parent, child);
+	}
+
+#endif
+
 	/* drop the signal lock on the child */
-	signal_unlock(p2);
+	proc_signalend(child, 0);
+	proc_transend(child, 0);
 
-	(void) thread_resume(newth);
+	/* "Return" to the child */
+	(void)thread_resume(newth);
 
         /* drop the extra references we got during the creation */
         if ((t = (task_t)get_threadtask(newth)) != NULL) {
@@ -361,80 +681,221 @@ fork1(p1, flags, retval)
         }
         thread_deallocate(newth);
 
-	KNOTE(&p1->p_klist, NOTE_FORK | p2->p_pid);
+	proc_knote(parent, NOTE_FORK | child->p_pid);
 
-	while (p2->p_flag & P_PPWAIT)
-		tsleep(p1, PWAIT, "ppwait", 0);
+	retval[0] = child->p_pid;
+	retval[1] = 0;			/* flag parent */
 
-	retval[0] = p2->p_pid;
-	retval[1] = 0;			/* mark parent */
+	DTRACE_PROC1(create, proc_t, child);
 
 	return (0);
 }
 
 /*
- * cloneproc()
+ * cloneproc
+ *
+ * Description: Create a new process from a specified process.
+ *
+ * Parameters:	parent			The parent process of the process to
+ *					be cloned
+ *		lock			Whether or not the signal lock was held
+ *					when calling cloneproc().
  *
- * Create a new process from a specified process.
- * On return newly created child process has signal
- * lock held to block delivery of signal to it if called with
- * lock set. fork() code needs to explicity remove this lock 
- * before signals can be delivered
+ * Returns:	!NULL			pointer to new child thread
+ *		NULL			Failure (unspecified)
+ *
+ * Note:	On return newly created child process has signal lock held
+ *		to block delivery of signal to it if called with lock set.
+ *		fork() code needs to explicity remove this lock before
+ *		signals can be delivered
+ *
+ *		In the case of bootstrap, this function can be called from
+ *		bsd_utaskbootstrap() in order to bootstrap the first process;
+ *		the net effect is to provide a uthread structure for the
+ *		kernel process associated with the kernel task.  This results
+ *		in a side effect in procdup(), which is why the code is more
+ *		complicated at the top of that function.
  */
 thread_t
-cloneproc(p1, lock)
-	register struct proc *p1;
-	register int lock;
+cloneproc(proc_t parent, int lock)
 {
-	register struct proc *p2;
-	thread_t th;
+	proc_t child;
+	thread_t th = NULL;
 
-	p2 = (struct proc *)forkproc(p1,lock);
+	if ((child = forkproc(parent,lock)) == NULL) {
+		/* Failed to allocate new process */
+		goto bad;
+	}
 
+	if ((th = procdup(parent, child)) == NULL) {
+		/*
+		 * Failed to create thread; now we must deconstruct the new
+		 * process previously obtained from forkproc().
+		 */
+		forkproc_free(child, lock);
+		goto bad;
+	}
 
-	th = procdup(p2, p1);	/* child, parent */
+	/* make child visible */
+	pinsertchild(parent, child);
 
-	LIST_INSERT_AFTER(p1, p2, p_pglist);
-	p2->p_pptr = p1;
-	LIST_INSERT_HEAD(&p1->p_children, p2, p_sibling);
-	LIST_INIT(&p2->p_children);
-	LIST_INSERT_HEAD(&allproc, p2, p_list);
-	LIST_INSERT_HEAD(PIDHASH(p2->p_pid), p2, p_hash);
-	TAILQ_INIT(&p2->p_evlist);
 	/*
 	 * Make child runnable, set start time.
 	 */
-	p2->p_stat = SRUN;
+	child->p_stat = SRUN;
 
+bad:
 	return(th);
 }
 
-struct proc *
-forkproc(p1, lock)
-	register struct proc *p1;
-	register int lock;
+/*
+ * Destroy a process structure that resulted from a call to forkproc(), but
+ * which must be returned to the system because of a subsequent failure
+ * preventing it from becoming active.
+ *
+ * Parameters:	p			The incomplete process from forkproc()
+ *		lock			Whether or not the signal lock was held
+ *					when calling forkproc().
+ *
+ * Returns:	(void)
+ *
+ * Note:	This function should only be used in an error handler following
+ *		a call to forkproc().  The 'lock' paramenter should be the same
+ *		as the lock parameter passed to forkproc().
+ *
+ *		Operations occur in reverse order of those in forkproc().
+ */
+void
+forkproc_free(proc_t p, int lock)
 {
-	register struct proc *p2, *newproc;
-	static int nextpid = 0, pidchecked = 0;
-
-	/* Allocate new proc. */
-	MALLOC_ZONE(newproc, struct proc *,
-			sizeof *newproc, M_PROC, M_WAITOK);
-	if (newproc == NULL)
-		panic("forkproc: M_PROC zone exhausted");
-	MALLOC_ZONE(newproc->p_stats, struct pstats *,
-			sizeof *newproc->p_stats, M_SUBPROC, M_WAITOK);
-	if (newproc->p_stats == NULL)
-		panic("forkproc: M_SUBPROC zone exhausted (p_stats)");
-	MALLOC_ZONE(newproc->p_sigacts, struct sigacts *,
-			sizeof *newproc->p_sigacts, M_SUBPROC, M_WAITOK);
-	if (newproc->p_sigacts == NULL)
-		panic("forkproc: M_SUBPROC zone exhausted (p_sigacts)");
+
+	/* Drop the signal lock, if it was held */
+	if (lock) {
+		proc_signalend(p, 0);
+		proc_transend(p, 0);
+	}
 
 	/*
-	 * Find an unused process ID.  We remember a range of unused IDs
-	 * ready to use (from nextpid+1 through pidchecked-1).
+	 * If we have our own copy of the resource limits structure, we
+	 * need to free it.  If it's a shared copy, we need to drop our
+	 * reference on it.
 	 */
+	proc_limitdrop(p, 0);
+	p->p_limit = NULL;
+
+#if SYSV_SHM
+	/* Need to drop references to the shared memory segment(s), if any */
+	if (p->vm_shm) {
+		/*
+		 * Use shmexec(): we have no address space, so no mappings
+		 *
+		 * XXX Yes, the routine is badly named.
+		 */
+		shmexec(p);
+	}
+#endif
+
+	/* Need to undo the effects of the fdcopy(), if any */
+	fdfree(p);
+
+	/*
+	 * Drop the reference on a text vnode pointer, if any
+	 * XXX This code is broken in forkproc(); see <rdar://4256419>;
+	 * XXX if anyone ever uses this field, we will be extremely unhappy.
+	 */
+	if (p->p_textvp) {
+		vnode_rele(p->p_textvp);
+		p->p_textvp = NULL;
+	}
+
+	/* Stop the profiling clock */
+	stopprofclock(p);
+
+	/* Release the credential reference */
+	kauth_cred_unref(&p->p_ucred);
+
+	proc_list_lock();
+	/* Decrement the count of processes in the system */
+	nprocs--;
+	proc_list_unlock();
+
+	thread_call_free(p->p_rcall);
+
+	/* Free allocated memory */
+	FREE_ZONE(p->p_sigacts, sizeof *p->p_sigacts, M_SIGACTS);
+	FREE_ZONE(p->p_stats, sizeof *p->p_stats, M_PSTATS);
+	proc_checkdeadrefs(p);
+	FREE_ZONE(p, sizeof *p, M_PROC);
+}
+
+
+/*
+ * forkproc
+ *
+ * Description:	Create a new process structure, given a parent process
+ *		structure.
+ *
+ * Parameters:	parent			The parent process
+ *		lock			If the signal lock should be taken on
+ *					the newly created process.
+ *
+ * Returns:	!NULL			The new process structure
+ *		NULL			Error (insufficient free memory)
+ *
+ * Note:	When successful, the newly created process structure is
+ *		partially initialized; if a caller needs to deconstruct the
+ *		returned structure, they must call forkproc_free() to do so.
+ */
+proc_t
+forkproc(proc_t parent, int lock)
+{
+	struct proc *  child;	/* Our new process */
+	static int nextpid = 0, pidwrap = 0;
+	int error = 0;
+	struct session *sessp;
+	uthread_t uth_parent = (uthread_t)get_bsdthread_info(current_thread());
+
+	MALLOC_ZONE(child, proc_t , sizeof *child, M_PROC, M_WAITOK);
+	if (child == NULL) {
+		printf("forkproc: M_PROC zone exhausted\n");
+		goto bad;
+	}
+	/* zero it out as we need to insert in hash */
+	bzero(child, sizeof *child);
+
+	MALLOC_ZONE(child->p_stats, struct pstats *,
+			sizeof *child->p_stats, M_PSTATS, M_WAITOK);
+	if (child->p_stats == NULL) {
+		printf("forkproc: M_SUBPROC zone exhausted (p_stats)\n");
+		FREE_ZONE(child, sizeof *child, M_PROC);
+		child = NULL;
+		goto bad;
+	}
+	MALLOC_ZONE(child->p_sigacts, struct sigacts *,
+			sizeof *child->p_sigacts, M_SIGACTS, M_WAITOK);
+	if (child->p_sigacts == NULL) {
+		printf("forkproc: M_SUBPROC zone exhausted (p_sigacts)\n");
+		FREE_ZONE(child->p_stats, sizeof *child->p_stats, M_PSTATS);
+		FREE_ZONE(child, sizeof *child, M_PROC);
+		child = NULL;
+		goto bad;
+	}
+	child->p_rcall = thread_call_allocate((thread_call_func_t)realitexpire, child);
+	if (child->p_rcall == NULL) {
+		FREE_ZONE(child->p_sigacts, sizeof *child->p_sigacts, M_SIGACTS);
+		FREE_ZONE(child->p_stats, sizeof *child->p_stats, M_PSTATS);
+		FREE_ZONE(child, sizeof *child, M_PROC);
+		child = NULL;
+		goto bad;
+	}
+
+
+	/*
+	 * Find an unused PID.  
+	 */
+
+	proc_list_lock();
+
 	nextpid++;
 retry:
 	/*
@@ -444,59 +905,52 @@ retry:
 	 */
 	if (nextpid >= PID_MAX) {
 		nextpid = 100;
-		pidchecked = 0;
+		pidwrap = 1;
 	}
-	if (nextpid >= pidchecked) {
-		int doingzomb = 0;
+	if (pidwrap != 0) {
 
-		pidchecked = PID_MAX;
-		/*
-		 * Scan the active and zombie procs to check whether this pid
-		 * is in use.  Remember the lowest pid that's greater
-		 * than nextpid, so we can avoid checking for a while.
-		 */
-		p2 = allproc.lh_first;
-again:
-		for (; p2 != 0; p2 = p2->p_list.le_next) {
-			while (p2->p_pid == nextpid ||
-			    p2->p_pgrp->pg_id == nextpid ||
-				p2->p_session->s_sid == nextpid) {
-				nextpid++;
-				if (nextpid >= pidchecked)
-					goto retry;
-			}
-			if (p2->p_pid > nextpid && pidchecked > p2->p_pid)
-				pidchecked = p2->p_pid;
-			if (p2->p_pgrp && p2->p_pgrp->pg_id > nextpid && 
-			    pidchecked > p2->p_pgrp->pg_id)
-				pidchecked = p2->p_pgrp->pg_id;
-			if (p2->p_session->s_sid > nextpid &&
-				pidchecked > p2->p_session->s_sid)
-				pidchecked = p2->p_session->s_sid;
-		}
-		if (!doingzomb) {
-			doingzomb = 1;
-			p2 = zombproc.lh_first;
-			goto again;
+		/* if the pid stays in hash both for zombie and runniing state */
+		if  (pfind_locked(nextpid) != PROC_NULL) {
+			nextpid++;
+			goto retry;
 		}
-	}
 
+		if (pgfind_internal(nextpid) != PGRP_NULL) {
+			nextpid++;
+			goto retry;
+		}	
+		if (session_find_internal(nextpid) != SESSION_NULL) {
+			nextpid++;
+			goto retry;
+		}	
+	}
 	nprocs++;
-	p2 = newproc;
-	p2->p_stat = SIDL;
-	p2->p_shutdownstate = 0;
-	p2->p_pid = nextpid;
+	child->p_pid = nextpid;
+#if 1
+	if (child->p_pid != 0) {
+		if (pfind_locked(child->p_pid) != PROC_NULL)
+			panic("proc in the list already\n");
+	}
+#endif
+	/* Insert in the hash */
+	child->p_listflag |= (P_LIST_INHASH | P_LIST_INCREATE);
+	LIST_INSERT_HEAD(PIDHASH(child->p_pid), child, p_hash);
+	proc_list_unlock();
+
+
+	/*
+	 * We've identified the PID we are going to use; initialize the new
+	 * process structure.
+	 */
+	child->p_stat = SIDL;
+	child->p_pgrpid = PGRPID_DEAD;
 
 	/*
-	 * Make a proc table entry for the new process.
-	 * Start by zeroing the section of proc that is zero-initialized,
-	 * then copy the section that is copied directly from the parent.
+	 * The zero'ing of the proc was at the allocation time due to need for insertion
+	 * to hash. Copy the section that is to be copied directly from the parent.
 	 */
-	bzero(&p2->p_startzero,
-	    (unsigned) ((caddr_t)&p2->p_endzero - (caddr_t)&p2->p_startzero));
-	bcopy(&p1->p_startcopy, &p2->p_startcopy,
-	    (unsigned) ((caddr_t)&p2->p_endcopy - (caddr_t)&p2->p_startcopy));
-	p2->vm_shm = (void *)NULL; /* Make sure it is zero */
+	bcopy(&parent->p_startcopy, &child->p_startcopy,
+	    (unsigned) ((caddr_t)&child->p_endcopy - (caddr_t)&child->p_startcopy));
 
 	/*
 	 * Some flags are inherited from the parent.
@@ -504,101 +958,97 @@ again:
 	 * Increase reference counts on shared objects.
 	 * The p_stats and p_sigacts substructs are set in vm_fork.
 	 */
-	p2->p_flag = (p1->p_flag & (P_LP64 | P_TRANSLATED | P_AFFINITY));
-	if (p1->p_flag & P_PROFIL)
-		startprofclock(p2);
+	child->p_flag = (parent->p_flag & (P_LP64 | P_TRANSLATED | P_AFFINITY));
+	if (parent->p_flag & P_PROFIL)
+		startprofclock(child);
 	/*
 	 * Note that if the current thread has an assumed identity, this
 	 * credential will be granted to the new process.
 	 */
-	p2->p_ucred = kauth_cred_get_with_ref();
+	child->p_ucred = kauth_cred_get_with_ref();
 
-	lck_mtx_init(&p2->p_mlock, proc_lck_grp, proc_lck_attr);
-	lck_mtx_init(&p2->p_fdmlock, proc_lck_grp, proc_lck_attr);
-	klist_init(&p2->p_klist);
-
-	/* bump references to the text vnode */
-	p2->p_textvp = p1->p_textvp;
-	if (p2->p_textvp) {
-		vnode_rele(p2->p_textvp);
+	lck_mtx_init(&child->p_mlock, proc_lck_grp, proc_lck_attr);
+	lck_mtx_init(&child->p_fdmlock, proc_lck_grp, proc_lck_attr);
+#if CONFIG_DTRACE
+	lck_mtx_init(&child->p_dtrace_sprlock, proc_lck_grp, proc_lck_attr);
+#endif
+	lck_spin_init(&child->p_slock, proc_lck_grp, proc_lck_attr);
+	klist_init(&child->p_klist);
+
+	if (child->p_textvp != NULLVP) {
+		/* bump references to the text vnode */
+		/* Need to hold iocount across the ref call */
+		if (vnode_getwithref(child->p_textvp) == 0) {
+			error = vnode_ref(child->p_textvp);
+			vnode_put(child->p_textvp);
+			if (error != 0)
+				child->p_textvp = NULLVP;
+		}
 	}
+
 	/* XXX may fail to copy descriptors to child */
-	p2->p_fd = fdcopy(p1);
+	child->p_fd = fdcopy(parent, uth_parent->uu_cdir);
 
-	if (p1->vm_shm) {
+#if SYSV_SHM
+	if (parent->vm_shm) {
 		/* XXX may fail to attach shm to child */
-		(void)shmfork(p1,p2);
+		(void)shmfork(parent,child);
 	}
+#endif
 	/*
-	 * If p_limit is still copy-on-write, bump refcnt,
-	 * otherwise get a copy that won't be modified.
-	 * (If PL_SHAREMOD is clear, the structure is shared
-	 * copy-on-write.)
+	 * inherit the limit structure to child
 	 */
-	if (p1->p_limit->p_lflags & PL_SHAREMOD)
-		p2->p_limit = limcopy(p1->p_limit);
-	else {
-		p2->p_limit = p1->p_limit;
-		p2->p_limit->p_refcnt++;
+	proc_limitfork(parent, child);
+
+	if (child->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur != RLIM_INFINITY) {
+		uint64_t rlim_cur = child->p_limit->pl_rlimit[RLIMIT_CPU].rlim_cur;
+		child->p_rlim_cpu.tv_sec = (rlim_cur > __INT_MAX__) ? __INT_MAX__ : rlim_cur;
 	}
 
-	bzero(&p2->p_stats->pstat_startzero,
-	    (unsigned) ((caddr_t)&p2->p_stats->pstat_endzero -
-	    (caddr_t)&p2->p_stats->pstat_startzero));
-	bcopy(&p1->p_stats->pstat_startcopy, &p2->p_stats->pstat_startcopy,
-	    ((caddr_t)&p2->p_stats->pstat_endcopy -
-	     (caddr_t)&p2->p_stats->pstat_startcopy));
+	bzero(&child->p_stats->pstat_startzero,
+	    (unsigned) ((caddr_t)&child->p_stats->pstat_endzero -
+	    (caddr_t)&child->p_stats->pstat_startzero));
 
-	bzero(&p2->p_stats->user_p_prof, sizeof(struct user_uprof));
+	bzero(&child->p_stats->user_p_prof, sizeof(struct user_uprof));
 
-	if (p1->p_sigacts != NULL)
-		(void)memcpy(p2->p_sigacts,
-				p1->p_sigacts, sizeof *p2->p_sigacts);
+	if (parent->p_sigacts != NULL)
+		(void)memcpy(child->p_sigacts,
+				parent->p_sigacts, sizeof *child->p_sigacts);
 	else
-		(void)memset(p2->p_sigacts, 0, sizeof *p2->p_sigacts);
+		(void)memset(child->p_sigacts, 0, sizeof *child->p_sigacts);
 
-	if (p1->p_session->s_ttyvp != NULL && p1->p_flag & P_CONTROLT)
-		p2->p_flag |= P_CONTROLT;
+	sessp = proc_session(parent);
+	if (sessp->s_ttyvp != NULL && parent->p_flag & P_CONTROLT)
+		OSBitOrAtomic(P_CONTROLT, (UInt32 *)&child->p_flag);
+	session_rele(sessp);
 
-	p2->p_argslen = p1->p_argslen;
-	p2->p_argc = p1->p_argc;
-	p2->p_xstat = 0;
-	p2->p_ru = NULL;
-
-	p2->p_debugger = 0;	/* don't inherit */
-	lockinit(&p2->signal_lock, PVM, "signal", 0, 0);
 	/* block all signals to reach the process */
-	if (lock)
-		signal_lock(p2);
-	p2->sigwait = FALSE;
-	p2->sigwait_thread = NULL;
-	p2->exit_thread = NULL;
-	p2->user_stack = p1->user_stack;
-	p2->p_vforkcnt = 0;
-	p2->p_vforkact = 0;
-	p2->p_lflag  = 0;
-	p2->p_ladvflag  = 0;
-	p2->p_internalref  = 0;
-	TAILQ_INIT(&p2->p_uthlist);
-	TAILQ_INIT(&p2->aio_activeq);
-	TAILQ_INIT(&p2->aio_doneq);
-	p2->aio_active_count = 0;
-	p2->aio_done_count = 0;
-
-#if KTRACE
-	/*
-	 * Copy traceflag and tracefile if enabled.
-	 * If not inherited, these were zeroed above.
-	 */
-	if (p1->p_traceflag&KTRFAC_INHERIT) {
-		p2->p_traceflag = p1->p_traceflag;
-		if ((p2->p_tracep = p1->p_tracep) != NULL) {
-		        vnode_ref(p2->p_tracep);
-		}
+	if (lock) {
+		proc_signalstart(child, 0);
+		proc_transstart(child, 0);
+	}
+
+	TAILQ_INIT(&child->p_uthlist);
+	TAILQ_INIT(&child->aio_activeq);
+	TAILQ_INIT(&child->aio_doneq);
+	/* Inherit the parent flags for code sign */
+	child->p_csflags = parent->p_csflags;
+	child->p_wqthread = parent->p_wqthread;
+	child->p_threadstart = parent->p_threadstart;
+	child->p_pthsize = parent->p_pthsize;
+	workqueue_init_lock(child);
+
+#if CONFIG_LCTX
+	child->p_lctx = NULL;
+	/* Add new process to login context (if any). */
+	if (parent->p_lctx != NULL) {
+		LCTX_LOCK(parent->p_lctx);
+		enterlctx(child, parent->p_lctx, 0);
 	}
 #endif
-	return(p2);
 
+bad:
+	return(child);
 }
 
 void
@@ -613,12 +1063,36 @@ proc_unlock(proc_t p)
 	lck_mtx_unlock(&p->p_mlock);
 }
 
+void
+proc_spinlock(proc_t p)
+{
+	lck_spin_lock(&p->p_slock);
+}
+
+void
+proc_spinunlock(proc_t p)
+{
+	lck_spin_unlock(&p->p_slock);
+}
+
+void 
+proc_list_lock(void)
+{
+	lck_mtx_lock(proc_list_mlock);
+}
+
+void 
+proc_list_unlock(void)
+{
+	lck_mtx_unlock(proc_list_mlock);
+}
+
 #include <kern/zalloc.h>
 
 struct zone	*uthread_zone;
-int uthread_zone_inited = 0;
+static int uthread_zone_inited = 0;
 
-void
+static void
 uthread_zone_init(void)
 {
 	if (!uthread_zone_inited) {
@@ -631,12 +1105,12 @@ uthread_zone_init(void)
 }
 
 void *
-uthread_alloc(task_t task, thread_t thr_act )
+uthread_alloc(task_t task, thread_t thread)
 {
-	struct proc *p;
-	struct uthread *uth, *uth_parent;
+	proc_t p;
+	uthread_t uth;
+	uthread_t uth_parent;
 	void *ut;
-	boolean_t funnel_state;
 
 	if (!uthread_zone_inited)
 		uthread_zone_init();
@@ -644,8 +1118,8 @@ uthread_alloc(task_t task, thread_t thr_act )
 	ut = (void *)zalloc(uthread_zone);
 	bzero(ut, sizeof(struct uthread));
 
-	p = (struct proc *) get_bsdtask_info(task);
-	uth = (struct uthread *)ut;
+	p = (proc_t) get_bsdtask_info(task);
+	uth = (uthread_t)ut;
 
 	/*
 	 * Thread inherits credential from the creating thread, if both
@@ -655,10 +1129,10 @@ uthread_alloc(task_t task, thread_t thr_act )
 	 * task we can leave the new thread credential NULL.  If it needs
 	 * one later, it will be lazily assigned from the task's process.
 	 */
-	uth_parent = (struct uthread *)get_bsdthread_info(current_thread());
-	if ((task == current_task()) && 
-	    (uth_parent != NULL) &&
-	    (IS_VALID_CRED(uth_parent->uu_ucred))) {
+	uth_parent = (uthread_t)get_bsdthread_info(current_thread());
+	if (task == current_task() && 
+	    uth_parent != NULL &&
+	    IS_VALID_CRED(uth_parent->uu_ucred)) {
 		/*
 		 * XXX The new thread is, in theory, being created in context
 		 * XXX of parent thread, so a direct reference to the parent
@@ -672,36 +1146,42 @@ uthread_alloc(task_t task, thread_t thr_act )
 	} else {
 		uth->uu_ucred = NOCRED;
 	}
+
 	
-	if (task != kernel_task) {
+	if ((task != kernel_task) && p) {
 		
-		funnel_state = thread_funnel_set(kernel_flock, TRUE);
+		proc_lock(p);
 		if (uth_parent) {
 			if (uth_parent->uu_flag & UT_SAS_OLDMASK)
 				uth->uu_sigmask = uth_parent->uu_oldmask;
 			else
 				uth->uu_sigmask = uth_parent->uu_sigmask;
 		}
-		uth->uu_act = thr_act;
-		//signal_lock(p);
-		if (p) {
-			TAILQ_INSERT_TAIL(&p->p_uthlist, uth, uu_list);
+		uth->uu_context.vc_thread = thread;
+		TAILQ_INSERT_TAIL(&p->p_uthlist, uth, uu_list);
+		proc_unlock(p);
+
+#if CONFIG_DTRACE
+		if (p->p_dtrace_ptss_pages != NULL) {
+			uth->t_dtrace_scratch = dtrace_ptss_claim_entry(p);
 		}
-		//signal_unlock(p);
-		(void)thread_funnel_set(kernel_flock, funnel_state);
+#endif
 	}
 
 	return (ut);
 }
 
 
+/* 
+ * This routine frees all the BSD context in uthread except the credential.
+ * It does not free the uthread structure as well
+ */
 void
-uthread_free(task_t task, void *uthread, void * bsd_info)
+uthread_cleanup(task_t task, void *uthread, void * bsd_info)
 {
 	struct _select *sel;
-	struct uthread *uth = (struct uthread *)uthread;
-	struct proc * p = (struct proc *)bsd_info;
-	boolean_t funnel_state;
+	uthread_t uth = (uthread_t)uthread;
+	proc_t p = (proc_t)bsd_info;
 
 	/*
 	 * Per-thread audit state should never last beyond system
@@ -716,29 +1196,59 @@ uthread_free(task_t task, void *uthread, void * bsd_info)
 	if (sel->nbytes) {
 		FREE(sel->ibits, M_TEMP);
 		FREE(sel->obits, M_TEMP);
+		sel->nbytes = 0;
+	}
+
+	if (uth->uu_cdir) {
+		vnode_rele(uth->uu_cdir);
+		uth->uu_cdir = NULLVP;
 	}
 
-	if (sel->allocsize && sel->wqset){
-		kfree(sel->wqset, sel->allocsize);
+	if (uth->uu_allocsize && uth->uu_wqset){
+		kfree(uth->uu_wqset, uth->uu_allocsize);
 		sel->count = 0;
-		sel->allocsize = 0;
-		sel->wqset = 0;
+		uth->uu_allocsize = 0;
+		uth->uu_wqset = 0;
 		sel->wql = 0;
 	}
 
+
+	if ((task != kernel_task) && p) {
+
+		if (((uth->uu_flag & UT_VFORK) == UT_VFORK) && (uth->uu_proc != PROC_NULL))  {
+			vfork_exit_internal(uth->uu_proc, 0, 1);
+		}
+		if (get_bsdtask_info(task) == p) { 
+			proc_lock(p);
+			TAILQ_REMOVE(&p->p_uthlist, uth, uu_list);
+			proc_unlock(p);
+		}
+#if CONFIG_DTRACE
+		if (uth->t_dtrace_scratch != NULL) {
+			dtrace_ptss_release_entry(p, uth->t_dtrace_scratch);
+		}
+#endif
+	}
+}
+
+/* This routine releases the credential stored in uthread */
+void
+uthread_cred_free(void *uthread)
+{
+	uthread_t uth = (uthread_t)uthread;
+
+	/* and free the uthread itself */
 	if (IS_VALID_CRED(uth->uu_ucred)) {
 		kauth_cred_t oldcred = uth->uu_ucred;
 		uth->uu_ucred = NOCRED;
 		kauth_cred_unref(&oldcred);
 	}
+}
 
-	if ((task != kernel_task) && p) {
-		funnel_state = thread_funnel_set(kernel_flock, TRUE);
-		//signal_lock(p);
-		TAILQ_REMOVE(&p->p_uthlist, uth, uu_list);
-		//signal_unlock(p);
-		(void)thread_funnel_set(kernel_flock, funnel_state);
-	}
+/* This routine frees the uthread structure held in thread structure */
+void
+uthread_zone_free(void *uthread)
+{
 	/* and free the uthread itself */
 	zfree(uthread_zone, uthread);
 }
diff --git a/bsd/kern/kern_ktrace.c b/bsd/kern/kern_ktrace.c
deleted file mode 100644
index 5f3794b7e..000000000
--- a/bsd/kern/kern_ktrace.c
+++ /dev/null
@@ -1,622 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-/*
- * Copyright (c) 1989, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)kern_ktrace.c	8.2 (Berkeley) 9/23/93
- * $FreeBSD: src/sys/kern/kern_ktrace.c,v 1.35.2.4 2001/03/05 13:09:01 obrien Exp $
- */
-
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/proc_internal.h>
-#include <sys/kauth.h>
-#include <sys/file_internal.h>
-#include <sys/namei.h>
-#include <sys/vnode_internal.h>
-#if KTRACE
-#include <sys/ktrace.h>
-#endif
-#include <sys/malloc.h>
-#include <sys/syslog.h>
-#include <sys/sysproto.h>
-#include <sys/uio_internal.h>
-
-#include <bsm/audit_kernel.h>
-
-#if KTRACE
-static struct ktr_header *ktrgetheader(int type);
-static void ktrwrite(struct vnode *, struct ktr_header *, struct uio *);
-static int ktrcanset(struct proc *,struct proc *);
-static int ktrsetchildren(struct proc *,struct proc *,
-	int, int, struct vnode *);
-static int ktrops(struct proc *,struct proc *,int,int,struct vnode *);
-
-
-static struct ktr_header *
-ktrgetheader(type)
-	int type;
-{
-	register struct ktr_header *kth;
-	struct proc *p = current_proc();	/* XXX */
-
-	MALLOC(kth, struct ktr_header *, sizeof (struct ktr_header),
-		M_KTRACE, M_WAITOK);
-	if (kth != NULL) {
-		kth->ktr_type = type;
-		microtime(&kth->ktr_time);
-		kth->ktr_pid = p->p_pid;
-		bcopy(p->p_comm, kth->ktr_comm, MAXCOMLEN);
-	}
-	return (kth);
-}
-#endif
-
-void
-ktrsyscall(p, code, narg, args)
-	struct proc *p;
-	int code, narg;
-	syscall_arg_t args[];
-{
-#if KTRACE
-	struct vnode *vp;
-	struct	ktr_header *kth;
-	struct	ktr_syscall *ktp;
-	register int len;
-	u_int64_t *argp;
-	int i;
-
-	if (!KTRPOINT(p, KTR_SYSCALL))
-		return;
-
-	vp = p->p_tracep;
-	len = __offsetof(struct ktr_syscall, ktr_args) +
-	    (narg * sizeof(u_int64_t));
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_SYSCALL);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	MALLOC(ktp, struct ktr_syscall *, len, M_KTRACE, M_WAITOK);
-	if (ktp == NULL) {
-		FREE(kth, M_KTRACE);
-		return;
-	}
-	ktp->ktr_code = code;
-	ktp->ktr_narg = narg;
-	argp = &ktp->ktr_args[0];
-	for (i = 0; i < narg; i++)
-		*argp++ = args[i];
-	kth->ktr_buf = (caddr_t)ktp;
-	kth->ktr_len = len;
-	ktrwrite(vp, kth, NULL);
-	FREE(ktp, M_KTRACE);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-#else
-	return;
-#endif
-}
- 
-void
-ktrsysret(p, code, error, retval)
-	struct proc *p;
-	int code, error;
-	register_t retval;
-{
-#if KTRACE
-	struct vnode *vp;
-	struct ktr_header *kth;
-	struct ktr_sysret ktp;
-
-	if (!KTRPOINT(p, KTR_SYSRET))
-		return;
-
-	vp = p->p_tracep;
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_SYSRET);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	ktp.ktr_code = code;
-	ktp.ktr_error = error;
-	ktp.ktr_retval = retval;		/* what about val2 ? */
-
-	kth->ktr_buf = (caddr_t)&ktp;
-	kth->ktr_len = sizeof(struct ktr_sysret);
-
-	ktrwrite(vp, kth, NULL);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-#else
-	return;
-#endif
-}
-
-#if KTRACE
-void
-ktrnamei(vp, path)
-	struct vnode *vp;
-	char *path;
-{
-	struct ktr_header *kth;
-	struct proc *p = current_proc();	/* XXX */
-
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_NAMEI);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	kth->ktr_len = strlen(path);
-	kth->ktr_buf = path;
-
-	ktrwrite(vp, kth, NULL);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-}
-
-void
-ktrgenio(vp, fd, rw, uio, error)
-	struct vnode *vp;
-	int fd;
-	enum uio_rw rw;
-	struct uio *uio;
-	int error;
-{
-	struct ktr_header *kth;
-	struct ktr_genio ktg;
-	struct proc *p = current_proc();	/* XXX */
-
-	if (error)
-		return;
-
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_GENIO);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	ktg.ktr_fd = fd;
-	ktg.ktr_rw = rw;
-	kth->ktr_buf = (caddr_t)&ktg;
-	kth->ktr_len = sizeof(struct ktr_genio);
-	uio->uio_offset = 0;
-	uio->uio_rw = UIO_WRITE;
-
-	ktrwrite(vp, kth, uio);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-}
-
-void
-ktrpsig(vp, sig, action, mask, code)
-	struct vnode *vp;
-	int sig;
-	sig_t action;
-	sigset_t *mask;
-	int code;
-{
-	struct ktr_header *kth;
-	struct ktr_psig	kp;
-	struct proc *p = current_proc();	/* XXX */
-
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_PSIG);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	kp.signo = (char)sig;
-	kp.action = action;
-	kp.mask = *mask;
-	kp.code = code;
-	kth->ktr_buf = (caddr_t)&kp;
-	kth->ktr_len = sizeof (struct ktr_psig);
-
-	ktrwrite(vp, kth, NULL);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-}
-
-void
-ktrcsw(vp, out, user)
-	struct vnode *vp;
-	int out, user;
-{
-	struct ktr_header *kth;
-	struct	ktr_csw kc;
-	struct proc *p = current_proc();	/* XXX */
-
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_CSW);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return;
-	}
-	kc.out = out;
-	kc.user = user;
-	kth->ktr_buf = (caddr_t)&kc;
-	kth->ktr_len = sizeof (struct ktr_csw);
-
-	ktrwrite(vp, kth, NULL);
-	FREE(kth, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-}
-#endif /* KTRACE */
-
-/* Interface and common routines */
-
-/*
- * ktrace system call
- */
-/* ARGSUSED */
-int
-ktrace(struct proc *curp, register struct ktrace_args *uap, __unused register_t *retval)
-{
-#if KTRACE
-	register struct vnode *vp = NULL;
-	register struct proc *p;
-	struct pgrp *pg;
-	int facs = uap->facs & ~KTRFAC_ROOT;
-	int ops = KTROP(uap->ops);
-	int descend = uap->ops & KTRFLAG_DESCEND;
-	int ret = 0;
-	int error = 0;
-	struct nameidata nd;
-	struct vfs_context context;
-
-	AUDIT_ARG(cmd, uap->ops);
-	AUDIT_ARG(pid, uap->pid);
-	AUDIT_ARG(value, uap->facs);
-
-	context.vc_proc = curp;
-	context.vc_ucred = kauth_cred_get();
-
-	curp->p_traceflag |= KTRFAC_ACTIVE;
-	if (ops != KTROP_CLEAR) {
-		/*
-		 * an operation which requires a file argument.
-		 */
-		NDINIT(&nd, LOOKUP, (NOFOLLOW|LOCKLEAF), UIO_USERSPACE, 
-			   uap->fname, &context);
-		error = vn_open(&nd, FREAD|FWRITE|O_NOFOLLOW, 0);
-		if (error) {
-			curp->p_traceflag &= ~KTRFAC_ACTIVE;
-			return (error);
-		}
-		vp = nd.ni_vp;
-
-		if (vp->v_type != VREG) {
-			(void) vn_close(vp, FREAD|FWRITE, kauth_cred_get(), curp);
-			(void) vnode_put(vp);
-
-			curp->p_traceflag &= ~KTRFAC_ACTIVE;
-			return (EACCES);
-		}
-	}
-	/*
-	 * Clear all uses of the tracefile
-	 */
-	if (ops == KTROP_CLEARFILE) {
-		LIST_FOREACH(p, &allproc, p_list) {
-			if (p->p_tracep == vp) {
-				if (ktrcanset(curp, p)) {
-					struct vnode *tvp = p->p_tracep;
-					/* no more tracing */
-					p->p_traceflag = 0;
-					if (tvp != NULL) {
-						p->p_tracep = NULL;
-						vnode_rele(tvp);
-					}
-				} else
-					error = EPERM;
-			}
-		}
-		goto done;
-	}
-
-	/*
-	 * need something to (un)trace (XXX - why is this here?)
-	 */
-	if (!facs) {
-		error = EINVAL;
-		goto done;
-	}
-	/*
-	 * do it
-	 */
-	if (uap->pid < 0) {
-		/*
-		 * by process group
-		 */
-		pg = pgfind(-uap->pid);
-		if (pg == NULL) {
-			error = ESRCH;
-			goto done;
-		}
-		LIST_FOREACH(p, &pg->pg_members, p_pglist)
-			if (descend)
-				ret |= ktrsetchildren(curp, p, ops, facs, vp);
-			else
-				ret |= ktrops(curp, p, ops, facs, vp);
-
-	} else {
-		/*
-		 * by pid
-		 */
-		p = pfind(uap->pid);
-		if (p == NULL) {
-			error = ESRCH;
-			goto done;
-		}
-		AUDIT_ARG(process, p);
-		if (descend)
-			ret |= ktrsetchildren(curp, p, ops, facs, vp);
-		else
-			ret |= ktrops(curp, p, ops, facs, vp);
-	}
-	if (!ret)
-		error = EPERM;
-done:
-	if (vp != NULL) {
-		(void) vn_close(vp, FWRITE, kauth_cred_get(), curp);
-		(void) vnode_put(vp);
-	}
-	curp->p_traceflag &= ~KTRFAC_ACTIVE;
-	return (error);
-#else
-	return ENOSYS;
-#endif
-}
-
-/*
- * utrace system call
- */
-
-/* ARGSUSED */
-int
-utrace(__unused struct proc *curp, register struct utrace_args *uap, __unused register_t *retval)
-{
-#if KTRACE
-	struct ktr_header *kth;
-	struct proc *p = current_proc();	/* XXX */
-	register caddr_t cp;
-
-	if (!KTRPOINT(p, KTR_USER))
-		return (0);
-	if (uap->len > KTR_USER_MAXLEN)
-		return (EINVAL);
-	p->p_traceflag |= KTRFAC_ACTIVE;
-	kth = ktrgetheader(KTR_USER);
-	if (kth == NULL) {
-		p->p_traceflag &= ~KTRFAC_ACTIVE;
-		return(ENOMEM);
-	}
-	MALLOC(cp, caddr_t, uap->len, M_KTRACE, M_WAITOK);
-	if (cp == NULL) {
-		FREE(kth, M_KTRACE);
-		return(ENOMEM);
-	}
-	if (copyin(uap->addr, cp, uap->len) == 0) {
-		kth->ktr_buf = cp;
-		kth->ktr_len = uap->len;
-		ktrwrite(p->p_tracep, kth, NULL);
-	}
-	FREE(kth, M_KTRACE);
-	FREE(cp, M_KTRACE);
-	p->p_traceflag &= ~KTRFAC_ACTIVE;
-
-	return (0);
-#else
-	return (ENOSYS);
-#endif
-}
-
-#if KTRACE
-static int
-ktrops(curp, p, ops, facs, vp)
-	struct proc *p, *curp;
-	int ops, facs;
-	struct vnode *vp;
-{
-	struct vnode *tvp;
-
-	if (!ktrcanset(curp, p))
-		return (0);
-	if (ops == KTROP_SET) {
-		if (p->p_tracep != vp) {
-			tvp = p->p_tracep;
-			vnode_ref(vp);
-			p->p_tracep = vp;
-
-			if (tvp != NULL) {
-			        /*
-				 * if trace file already in use, relinquish
-				 */
-				vnode_rele(tvp);
-			}
-		}
-		p->p_traceflag |= facs;
-		if (!suser(kauth_cred_get(), NULL))
-			p->p_traceflag |= KTRFAC_ROOT;
-	} else {
-		/* KTROP_CLEAR */
-		if (((p->p_traceflag &= ~facs) & KTRFAC_MASK) == 0) {
-			/* no more tracing */
-			tvp = p->p_tracep;
-			p->p_traceflag = 0;
-			if (tvp != NULL) {
-				p->p_tracep = NULL;
-				vnode_rele(tvp);
-			}
-		}
-	}
-
-	return (1);
-}
-
-static int
-ktrsetchildren(curp, top, ops, facs, vp)
-	struct proc *curp, *top;
-	int ops, facs;
-	struct vnode *vp;
-{
-	register struct proc *p;
-	register int ret = 0;
-
-	p = top;
-	for (;;) {
-		ret |= ktrops(curp, p, ops, facs, vp);
-		/*
-		 * If this process has children, descend to them next,
-		 * otherwise do any siblings, and if done with this level,
-		 * follow back up the tree (but not past top).
-		 */
-		if (!LIST_EMPTY(&p->p_children))
-			p = LIST_FIRST(&p->p_children);
-		else for (;;) {
-			if (p == top)
-				return (ret);
-			if (LIST_NEXT(p, p_sibling)) {
-				p = LIST_NEXT(p, p_sibling);
-				break;
-			}
-			p = p->p_pptr;
-		}
-	}
-	/*NOTREACHED*/
-}
-
-static void
-ktrwrite(struct vnode *vp, struct ktr_header *kth, struct uio *uio)
-{
-	uio_t auio;
-	register struct proc *p = current_proc();	/* XXX */
-	struct vfs_context context;
-	int error;
-	char uio_buf[ UIO_SIZEOF(2) ];
-
-	if (vp == NULL)
-		return;
-
-	auio = uio_createwithbuffer(2, 0, UIO_SYSSPACE, UIO_WRITE, 
-								  &uio_buf[0], sizeof(uio_buf));
-	uio_addiov(auio, CAST_USER_ADDR_T(kth), sizeof(struct ktr_header));
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	
-	if (kth->ktr_len > 0) {
-		uio_addiov(auio, CAST_USER_ADDR_T(kth->ktr_buf), kth->ktr_len);
-		if (uio != NULL)
-			kth->ktr_len += uio_resid(uio);
-	}
-	if ((error = vnode_getwithref(vp)) == 0) {
-	        error = VNOP_WRITE(vp, auio, IO_UNIT | IO_APPEND, &context);
-		if (error == 0 && uio != NULL) {
-		        error = VNOP_WRITE(vp, uio, IO_UNIT | IO_APPEND, &context);
-		}
-		vnode_put(vp);
-	}
-	if (error) {
-	        /*
-		 * If error encountered, give up tracing on this vnode.
-		 */
-	        log(LOG_NOTICE, "ktrace write failed, errno %d, tracing stopped\n",
-		    error);
-		LIST_FOREACH(p, &allproc, p_list) {
-		        if (p->p_tracep == vp) {
-			        p->p_tracep = NULL;
-				p->p_traceflag = 0;
-				vnode_rele(vp);
-			}
-		}
-	}
-}
-
-/*
- * Return true if caller has permission to set the ktracing state
- * of target.  Essentially, the target can't possess any
- * more permissions than the caller.  KTRFAC_ROOT signifies that
- * root previously set the tracing status on the target process, and
- * so, only root may further change it.
- *
- * TODO: check groups.  use caller effective gid.
- */
-static int
-ktrcanset(__unused struct proc *callp, struct proc *targetp)
-{
-	kauth_cred_t caller = kauth_cred_get();
-	kauth_cred_t target = targetp->p_ucred;		/* XXX */
-
-#if 0
-	/* PRISON_CHECK was defined to 1 always .... */
-	if (!PRISON_CHECK(callp, targetp))
-		return (0);
-#endif
-	if ((kauth_cred_getuid(caller) == target->cr_ruid &&
-	     target->cr_ruid == target->cr_svuid &&
-	     caller->cr_rgid == target->cr_rgid &&	/* XXX */
-	     target->cr_rgid == target->cr_svgid &&
-	     (targetp->p_traceflag & KTRFAC_ROOT) == 0 &&
-	     (targetp->p_flag & P_SUGID) == 0) ||
-	     !suser(caller, NULL))
-		return (1);
-
-	return (0);
-}
-
-#endif /* KTRACE */
diff --git a/bsd/kern/kern_lock.c b/bsd/kern/kern_lock.c
deleted file mode 100644
index c69140fda..000000000
--- a/bsd/kern/kern_lock.c
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
-/* 
- * Copyright (c) 1995
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code contains ideas from software contributed to Berkeley by
- * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
- * System project at Carnegie-Mellon University.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)kern_lock.c	8.18 (Berkeley) 5/21/95
- */
-
-#include <sys/param.h>
-#include <sys/proc_internal.h>
-#include <sys/lock.h>
-#include <kern/cpu_number.h>
-#include <kern/thread.h>
-
-#include <mach/mach_types.h>
-
-/*
- * Locking primitives implementation.
- * Locks provide shared/exclusive sychronization.
- */
-
-#if 0
-#define COUNT(p, x) if (p) (p)->p_locks += (x)
-#else
-#define COUNT(p, x)
-#endif
-
-#if NCPUS > 1
-
-/*
- * For multiprocessor system, try spin lock first.
- *
- * This should be inline expanded below, but we cannot have #if
- * inside a multiline define.
- */
-int lock_wait_time = 100;
-#define PAUSE(lkp, wanted)						\
-		if (lock_wait_time > 0) {				\
-			int i;						\
-									\
-			for (i = lock_wait_time; i > 0; i--)		\
-				if (!(wanted))				\
-					break;				\
-		}							\
-		if (!(wanted))						\
-			break;
-
-#else /* NCPUS == 1 */
-
-/*
- * It is an error to spin on a uniprocessor as nothing will ever cause
- * the simple lock to clear while we are executing.
- */
-#define PAUSE(lkp, wanted)
-
-#endif /* NCPUS == 1 */
-
-/*
- * Acquire a resource.
- */
-#define ACQUIRE(lkp, error, extflags, wanted)				\
-	PAUSE(lkp, wanted);						\
-	for (error = 0; wanted; ) {					\
-		(lkp)->lk_waitcount++;					\
-		error = tsleep((void *)lkp, (lkp)->lk_prio,		\
-		    (lkp)->lk_wmesg, (lkp)->lk_timo);			\
-		(lkp)->lk_waitcount--;					\
-		if (error)						\
-			break;						\
-		if ((extflags) & LK_SLEEPFAIL) {			\
-			error = ENOLCK;					\
-			break;						\
-		}							\
-	}
-
-/*
- * Initialize a lock; required before use.
- */
-void
-lockinit(lkp, prio, wmesg, timo, flags)
-	struct lock__bsd__ *lkp;
-	int prio;
-	const char *wmesg;
-	int timo;
-	int flags;
-{
-
-	bzero(lkp, sizeof(struct lock__bsd__));
-	lkp->lk_flags = flags & LK_EXTFLG_MASK;
-	lkp->lk_prio = prio;
-	lkp->lk_timo = timo;
-	lkp->lk_wmesg = wmesg;
-	lkp->lk_lockholder = LK_NOPROC;
-	lkp->lk_lockthread = 0;
-}
-
-/*
- * Determine the status of a lock.
- */
-int
-lockstatus(lkp)
-	struct lock__bsd__ *lkp;
-{
-	int lock_type = 0;
-
-	if (lkp->lk_exclusivecount != 0)
-		lock_type = LK_EXCLUSIVE;
-	else if (lkp->lk_sharecount != 0)
-		lock_type = LK_SHARED;
-	return (lock_type);
-}
-
-/*
- * Set, change, or release a lock.
- *
- * Shared requests increment the shared count. Exclusive requests set the
- * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
- * accepted shared locks and shared-to-exclusive upgrades to go away.
- */
-int
-lockmgr(lkp, flags, interlkp, p)
-	struct lock__bsd__ *lkp;
-	u_int flags;
-	void * interlkp;
-	struct proc *p;
-{
-	int error;
-	pid_t pid;
-	int extflags;
-	void *self;
-
-	error = 0; self = current_thread();
-	if (p)
-		pid = p->p_pid;
-	else
-		pid = LK_KERNPROC;
-	extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
-#if 0
-	/*
-	 * Once a lock has drained, the LK_DRAINING flag is set and an
-	 * exclusive lock is returned. The only valid operation thereafter
-	 * is a single release of that exclusive lock. This final release
-	 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any
-	 * further requests of any sort will result in a panic. The bits
-	 * selected for these two flags are chosen so that they will be set
-	 * in memory that is freed (freed memory is filled with 0xdeadbeef).
-	 * The final release is permitted to give a new lease on life to
-	 * the lock by specifying LK_REENABLE.
-	 */
-	if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) {
-		if (lkp->lk_flags & LK_DRAINED)
-			panic("lockmgr: using decommissioned lock");
-		if ((flags & LK_TYPE_MASK) != LK_RELEASE ||
-		    (lkp->lk_lockholder != pid && lkp->lk_lockthread != self)
-			panic("lockmgr: non-release on draining lock: %d\n",
-			    flags & LK_TYPE_MASK);
-		lkp->lk_flags &= ~LK_DRAINING;
-		if ((flags & LK_REENABLE) == 0)
-			lkp->lk_flags |= LK_DRAINED;
-	}
-#endif
-
-	switch (flags & LK_TYPE_MASK) {
-
-	case LK_SHARED:
-		if (lkp->lk_lockholder != pid || lkp->lk_lockthread != self) {
-			/*
-			 * If just polling, check to see if we will block.
-			 */
-			if ((extflags & LK_NOWAIT) && (lkp->lk_flags &
-			    (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) {
-				error = EBUSY;
-				break;
-			}
-			/*
-			 * Wait for exclusive locks and upgrades to clear.
-			 */
-			ACQUIRE(lkp, error, extflags, lkp->lk_flags &
-			    (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE));
-			if (error)
-				break;
-			lkp->lk_sharecount++;
-			COUNT(p, 1);
-			break;
-		}
-		/*
-		 * We hold an exclusive lock, so downgrade it to shared.
-		 * An alternative would be to fail with EDEADLK.
-		 */
-		lkp->lk_sharecount++;
-		COUNT(p, 1);
-		/* fall into downgrade */
-
-	case LK_DOWNGRADE:
-		if (lkp->lk_lockholder != pid ||
-				lkp->lk_lockthread != self ||
-					lkp->lk_exclusivecount == 0)
-			panic("lockmgr: not holding exclusive lock");
-		lkp->lk_sharecount += lkp->lk_exclusivecount;
-		lkp->lk_exclusivecount = 0;
-		lkp->lk_flags &= ~LK_HAVE_EXCL;
-		lkp->lk_lockholder = LK_NOPROC;
-		lkp->lk_lockthread = 0;
-		if (lkp->lk_waitcount)
-			wakeup((void *)lkp);
-		break;
-
-	case LK_EXCLUPGRADE:
-		/*
-		 * If another process is ahead of us to get an upgrade,
-		 * then we want to fail rather than have an intervening
-		 * exclusive access.
-		 */
-		if (lkp->lk_flags & LK_WANT_UPGRADE) {
-			lkp->lk_sharecount--;
-			COUNT(p, -1);
-			error = EBUSY;
-			break;
-		}
-		/* fall into normal upgrade */
-
-	case LK_UPGRADE:
-		/*
-		 * Upgrade a shared lock to an exclusive one. If another
-		 * shared lock has already requested an upgrade to an
-		 * exclusive lock, our shared lock is released and an
-		 * exclusive lock is requested (which will be granted
-		 * after the upgrade). If we return an error, the file
-		 * will always be unlocked.
-		 */
-		if ((lkp->lk_lockholder == pid &&
-				lkp->lk_lockthread == self) ||
-						lkp->lk_sharecount <= 0)
-			panic("lockmgr: upgrade exclusive lock");
-		lkp->lk_sharecount--;
-		COUNT(p, -1);
-		/*
-		 * If we are just polling, check to see if we will block.
-		 */
-		if ((extflags & LK_NOWAIT) &&
-		    ((lkp->lk_flags & LK_WANT_UPGRADE) ||
-		     lkp->lk_sharecount > 1)) {
-			error = EBUSY;
-			break;
-		}
-		if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
-			/*
-			 * We are first shared lock to request an upgrade, so
-			 * request upgrade and wait for the shared count to
-			 * drop to zero, then take exclusive lock.
-			 */
-			lkp->lk_flags |= LK_WANT_UPGRADE;
-			ACQUIRE(lkp, error, extflags, lkp->lk_sharecount);
-			lkp->lk_flags &= ~LK_WANT_UPGRADE;
-			if (error)
-				break;
-			lkp->lk_flags |= LK_HAVE_EXCL;
-			lkp->lk_lockholder = pid;
-			lkp->lk_lockthread = self;
-			if (lkp->lk_exclusivecount != 0)
-				panic("lockmgr: non-zero exclusive count");
-			lkp->lk_exclusivecount = 1;
-			COUNT(p, 1);
-			break;
-		}
-		/*
-		 * Someone else has requested upgrade. Release our shared
-		 * lock, awaken upgrade requestor if we are the last shared
-		 * lock, then request an exclusive lock.
-		 */
-		if (lkp->lk_sharecount == 0 && lkp->lk_waitcount)
-			wakeup((void *)lkp);
-		/* fall into exclusive request */
-
-	case LK_EXCLUSIVE:
-		if (lkp->lk_lockholder == pid && lkp->lk_lockthread == self) {
-			/*
-			 *	Recursive lock.
-			 */
-			if ((extflags & LK_CANRECURSE) == 0)
-				panic("lockmgr: locking against myself");
-			lkp->lk_exclusivecount++;
-			COUNT(p, 1);
-			break;
-		}
-		/*
-		 * If we are just polling, check to see if we will sleep.
-		 */
-		if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
-		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
-		     lkp->lk_sharecount != 0)) {
-			error = EBUSY;
-			break;
-		}
-		/*
-		 * Try to acquire the want_exclusive flag.
-		 */
-		ACQUIRE(lkp, error, extflags, lkp->lk_flags &
-		    (LK_HAVE_EXCL | LK_WANT_EXCL));
-		if (error)
-			break;
-		lkp->lk_flags |= LK_WANT_EXCL;
-		/*
-		 * Wait for shared locks and upgrades to finish.
-		 */
-		ACQUIRE(lkp, error, extflags, lkp->lk_sharecount != 0 ||
-		       (lkp->lk_flags & LK_WANT_UPGRADE));
-		lkp->lk_flags &= ~LK_WANT_EXCL;
-		if (error)
-			break;
-		lkp->lk_flags |= LK_HAVE_EXCL;
-		lkp->lk_lockholder = pid;
-		lkp->lk_lockthread = self;
-		if (lkp->lk_exclusivecount != 0)
-			panic("lockmgr: non-zero exclusive count");
-		lkp->lk_exclusivecount = 1;
-		COUNT(p, 1);
-		break;
-
-	case LK_RELEASE:
-		if (lkp->lk_exclusivecount != 0) {
-			if (pid != lkp->lk_lockholder ||
-					lkp->lk_lockthread != self)
-				panic("lockmgr: pid %d, thread 0x%8x,"
-					" not exclusive lock holder pid %d"
-					" thread 0x%8x unlocking, exclusive count %d",
-				    pid, self, lkp->lk_lockholder,
-					lkp->lk_lockthread, lkp->lk_exclusivecount);
-			lkp->lk_exclusivecount--;
-			COUNT(p, -1);
-			if (lkp->lk_exclusivecount == 0) {
-				lkp->lk_flags &= ~LK_HAVE_EXCL;
-				lkp->lk_lockholder = LK_NOPROC;
-				lkp->lk_lockthread = 0;
-			}
-		} else if (lkp->lk_sharecount != 0) {
-			lkp->lk_sharecount--;
-			COUNT(p, -1);
-		}
-		if (lkp->lk_waitcount)
-			wakeup((void *)lkp);
-		break;
-
-	case LK_DRAIN:
-		/*
-		 * Check that we do not already hold the lock, as it can 
-		 * never drain if we do. Unfortunately, we have no way to
-		 * check for holding a shared lock, but at least we can
-		 * check for an exclusive one.
-		 */
-		if (lkp->lk_lockholder == pid && lkp->lk_lockthread == self)
-			panic("lockmgr: draining against myself");
-		/*
-		 * If we are just polling, check to see if we will sleep.
-		 */
-		if ((extflags & LK_NOWAIT) && ((lkp->lk_flags &
-		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
-		     lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) {
-			error = EBUSY;
-			break;
-		}
-		PAUSE(lkp, ((lkp->lk_flags &
-		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
-		     lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0));
-		for (error = 0; ((lkp->lk_flags &
-		     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) ||
-		     lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0); ) {
-			lkp->lk_flags |= LK_WAITDRAIN;
-			if (error = tsleep((void *)&lkp->lk_flags, lkp->lk_prio,
-			    lkp->lk_wmesg, lkp->lk_timo))
-				return (error);
-			if ((extflags) & LK_SLEEPFAIL)
-				return (ENOLCK);
-		}
-		lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL;
-		lkp->lk_lockholder = pid;
-		lkp->lk_lockthread = self;
-		lkp->lk_exclusivecount = 1;
-		COUNT(p, 1);
-		break;
-
-	default:
-		panic("lockmgr: unknown locktype request %d",
-		    flags & LK_TYPE_MASK);
-		/* NOTREACHED */
-	}
-	if ((lkp->lk_flags & LK_WAITDRAIN) && ((lkp->lk_flags &
-	     (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 &&
-	     lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) {
-		lkp->lk_flags &= ~LK_WAITDRAIN;
-		wakeup((void *)&lkp->lk_flags);
-	}
-	return (error);
-}
-
-/*
- * Print out information about state of a lock. Used by VOP_PRINT
- * routines to display ststus about contained locks.
- */
-void
-lockmgr_printinfo(lkp)
-	struct lock__bsd__ *lkp;
-{
-
-	if (lkp->lk_sharecount)
-		printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
-		    lkp->lk_sharecount);
-	else if (lkp->lk_flags & LK_HAVE_EXCL)
-		printf(" lock type %s: EXCL (count %d) by pid %d",
-		    lkp->lk_wmesg, lkp->lk_exclusivecount, lkp->lk_lockholder);
-	if (lkp->lk_waitcount > 0)
-		printf(" with %d pending", lkp->lk_waitcount);
-}
diff --git a/bsd/kern/kern_lockf.c b/bsd/kern/kern_lockf.c
index 1ef3470ce..4e61180b6 100644
--- a/bsd/kern/kern_lockf.c
+++ b/bsd/kern/kern_lockf.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -39,7 +66,9 @@
 #include <sys/lock.h>
 #include <sys/mount.h>
 #include <sys/proc.h>
+#include <sys/signalvar.h>
 #include <sys/unistd.h>
+#include <sys/user.h>
 #include <sys/vnode.h>
 #include <sys/vnode_internal.h>
 #include <sys/vnode_if.h>
@@ -47,24 +76,34 @@
 #include <sys/fcntl.h>
 #include <sys/lockf.h>
 
-#if DEAD_CODE
 /*
  * This variable controls the maximum number of processes that will
  * be checked in doing deadlock detection.
  */
 static int maxlockdepth = MAXDEPTH;
-#endif	/* DEAD_CODE */
 
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 #include <sys/sysctl.h>
-
 #include <ufs/ufs/quota.h>
 #include <ufs/ufs/inode.h>
-
-
+void lf_print(const char *tag, struct lockf *lock);
+void lf_printlist(const char *tag, struct lockf *lock);
 static int	lockf_debug = 2;
 SYSCTL_INT(_debug, OID_AUTO, lockf_debug, CTLFLAG_RW, &lockf_debug, 0, "");
-#endif
+
+/*
+ * If there is no mask bit selector, or there is on, and the selector is
+ * set, then output the debugging diagnostic.
+ */
+#define LOCKF_DEBUG(mask, ...)					\
+	do {							\
+		if( !(mask) || ((mask) & lockf_debug)) {	\
+			printf(__VA_ARGS__);			\
+		}						\
+	} while(0)
+#else	/* !LOCKF_DEBUGGING */
+#define LOCKF_DEBUG(mask, ...)		/* mask */
+#endif	/* !LOCKF_DEBUGGING */
 
 MALLOC_DEFINE(M_LOCKF, "lockf", "Byte-range locking structures");
 
@@ -72,29 +111,55 @@ MALLOC_DEFINE(M_LOCKF, "lockf", "Byte-range locking structures");
 #define SELF	0x1
 #define OTHERS	0x2
 #define OFF_MAX	0x7fffffffffffffffULL	/* max off_t */
+
+/*
+ * Overlapping lock states
+ */
+typedef enum {
+	OVERLAP_NONE = 0,
+	OVERLAP_EQUALS_LOCK,
+	OVERLAP_CONTAINS_LOCK,
+	OVERLAP_CONTAINED_BY_LOCK,
+	OVERLAP_STARTS_BEFORE_LOCK,
+	OVERLAP_ENDS_AFTER_LOCK
+} overlap_t;
+
 static int	 lf_clearlock(struct lockf *);
-static int	 lf_findoverlap(struct lockf *,
+static overlap_t lf_findoverlap(struct lockf *,
 	    struct lockf *, int, struct lockf ***, struct lockf **);
-static struct lockf *
-	 lf_getblock(struct lockf *);
+static struct lockf *lf_getblock(struct lockf *);
 static int	 lf_getlock(struct lockf *, struct flock *);
 static int	 lf_setlock(struct lockf *);
-static void	 lf_split(struct lockf *, struct lockf *);
+static int	 lf_split(struct lockf *, struct lockf *);
 static void	 lf_wakelock(struct lockf *);
 
+
 /*
- * Advisory record locking support
+ * lf_advlock
+ *
+ * Description:	Advisory record locking support
+ *
+ * Parameters:	ap			Argument pointer to a vnop_advlock_args
+ *					argument descriptor structure for the
+ *					lock operation to be attempted.
+ *
+ * Returns:	0			Success
+ *		EOVERFLOW
+ *		EINVAL
+ *		ENOLCK			Number of locked regions exceeds limit
+ *	lf_setlock:EAGAIN
+ *	lf_setlock:EDEADLK
+ *	lf_setlock:EINTR
+ *	lf_setlock:ENOLCK
+ *	lf_clearlock:ENOLCK
+ *	vnode_size:???
+ *
+ * Notes:	We return ENOLCK when we run out of memory to support locks; as
+ *		such, there is no specific expectation limit other than the
+ *		amount of available resources.
  */
 int
-lf_advlock(ap)
-	struct vnop_advlock_args /* {
-		struct vnode *a_vp;
-		caddr_t  a_id;
-		int  a_op;
-		struct flock *a_fl;
-		int  a_flags;
-		vfs_context_t a_context;
-	} */ *ap;
+lf_advlock(struct vnop_advlock_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 	struct flock *fl = ap->a_fl;
@@ -113,9 +178,7 @@ lf_advlock(ap)
 	if (*head == (struct lockf *)0) {
 		if (ap->a_op != F_SETLK) {
 			fl->l_type = F_UNLCK;
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: unlock without lock\n");
-#endif /* LOCKF_DEBUG */
+			LOCKF_DEBUG(0, "lf_advlock: '%s' unlock without lock\n", vfs_context_proc(context)->p_comm);
 			return (0);
 		}
 	}
@@ -136,67 +199,59 @@ lf_advlock(ap)
 
 	case SEEK_END:
 
-		if ((error = vnode_size(vp, &size, context)))
-{
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: vnode_getattr failed: %d\n", error);
-#endif /* LOCKF_DEBUG */
+		/*
+		 * It's OK to cast the u_quad_t to and off_t here, since they
+		 * are the same storage size, and the value of the returned
+		 * contents will never overflow into the sign bit.  We need to
+		 * do this because we will use size to force range checks.
+		 */
+		if ((error = vnode_size(vp, (off_t *)&size, context))) {
+			LOCKF_DEBUG(0, "lf_advlock: vnode_getattr failed: %d\n", error);
 			return (error);
-}
+		}
 
 		if (size > OFF_MAX ||
-		    (fl->l_start > 0 && size > OFF_MAX - fl->l_start))
+		    (fl->l_start > 0 &&
+		     size > (u_quad_t)(OFF_MAX - fl->l_start)))
 			return (EOVERFLOW);
 		start = size + fl->l_start;
 		break;
 
 	default:
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: unknown whence %d\n", fl->l_whence);
-#endif /* LOCKF_DEBUG */
+		LOCKF_DEBUG(0, "lf_advlock: unknown whence %d\n", fl->l_whence);
 		return (EINVAL);
 	}
-	if (start < 0)
-{
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: start < 0 (%qd)\n", start);
-#endif /* LOCKF_DEBUG */
+	if (start < 0) {
+		LOCKF_DEBUG(0, "lf_advlock: start < 0 (%qd)\n", start);
 		return (EINVAL);
-}
+	}
 	if (fl->l_len < 0) {
-		if (start == 0)
-{
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: len < 0 & start == 0\n");
-#endif /* LOCKF_DEBUG */
+		if (start == 0) {
+			LOCKF_DEBUG(0, "lf_advlock: len < 0 & start == 0\n");
 			return (EINVAL);
-}
+		}
 		end = start - 1;
 		start += fl->l_len;
-		if (start < 0)
-{
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: start < 0 (%qd)\n", start);
-#endif /* LOCKF_DEBUG */
+		if (start < 0) {
+			LOCKF_DEBUG(0, "lf_advlock: start < 0 (%qd)\n", start);
 			return (EINVAL);
-}
+		}
 	} else if (fl->l_len == 0)
 		end = -1;
 	else {
 		oadd = fl->l_len - 1;
-		if (oadd > (off_t)(OFF_MAX - start))
-{
-#ifdef LOCKF_DEBUG
-		printf("lf_advlock: overflow\n");
-#endif /* LOCKF_DEBUG */
+		if (oadd > (off_t)(OFF_MAX - start)) {
+			LOCKF_DEBUG(0, "lf_advlock: overflow\n");
 			return (EOVERFLOW);
-}
+		}
 		end = start + oadd;
 	}
 	/*
 	 * Create the lockf structure
 	 */
 	MALLOC(lock, struct lockf *, sizeof *lock, M_LOCKF, M_WAITOK);
+	if (lock == NULL)
+		return (ENOLCK);
 	lock->lf_start = start;
 	lock->lf_end = end;
 	lock->lf_id = ap->a_id;
@@ -233,30 +288,108 @@ lf_advlock(ap)
 	}
 	lck_mtx_unlock(&vp->v_lock);	/* done maniplulating the list */
 
-#ifdef LOCKF_DEBUG
-	printf("lf_advlock: normal exit: %d\n", error);
-#endif /* LOCKF_DEBUG */
+	LOCKF_DEBUG(0, "lf_advlock: normal exit: %d\n\n", error);
 	return (error);
 }
 
+
 /*
- * Set a byte-range lock.
+ * lf_coelesce_adjacent
+ *
+ * Description:	Helper function: when setting a lock, coelesce adjacent
+ *		locks.  Needed because adjacent locks are not overlapping,
+ *		but POSIX requires that they be coelesced.
+ *
+ * Parameters:	lock			The new lock which may be adjacent
+ *					to already locked reagions, and which
+ *					should therefore be coelesced with them
+ *
+ * Returns:	<void>
+ */
+static void
+lf_coelesce_adjacent(struct lockf *lock)
+{
+	struct lockf **lf = lock->lf_head;
+
+	while (*lf != NOLOCKF) {
+		/* reject locks that obviously could not be coelesced */
+		if ((*lf == lock) ||
+		    ((*lf)->lf_id != lock->lf_id) ||
+		    ((*lf)->lf_type != lock->lf_type)) {
+			lf = &(*lf)->lf_next;
+			continue;
+		}
+
+		/* If the lock ends adjacent to us, we can coelesce it */
+		if ((*lf)->lf_end != -1 &&
+		    ((*lf)->lf_end + 1) == lock->lf_start) {
+			struct lockf *adjacent = *lf;
+
+			LOCKF_DEBUG(0, "lf_coelesce_adjacent: coelesce adjacent previous\n");
+			lock->lf_start = (*lf)->lf_start;
+			*lf = lock;
+			lf = &(*lf)->lf_next;
+			FREE(adjacent, M_LOCKF);
+			continue;
+		}
+		/* If the lock starts adjacent to us, we can coelesce it */
+		if (lock->lf_end != -1 &&
+		    (lock->lf_end + 1) == (*lf)->lf_start) {
+			struct lockf *adjacent = *lf;
+
+			LOCKF_DEBUG(0, "lf_coelesce_adjacent: coelesce adjacent following\n");
+			lock->lf_end = (*lf)->lf_end;
+			lock->lf_next = (*lf)->lf_next;
+			lf = &lock->lf_next;
+			FREE(adjacent, M_LOCKF);
+			continue;
+		}
+
+		/* no matching conditions; go on to next lock */
+		lf = &(*lf)->lf_next;
+	}
+}
+
+
+/*
+ * lf_setlock
+ *
+ * Description:	Set a byte-range lock.
+ *
+ * Parameters:	lock			The lock structure describing the lock
+ *					to be set; allocated by the caller, it
+ *					will be linked into the lock list if
+ *					the set is successful, and freed if the
+ *					set is unsuccessful.
+ *
+ * Returns:	0			Success
+ *		EAGAIN
+ *		EDEADLK
+ *	lf_split:ENOLCK
+ *	lf_clearlock:ENOLCK
+ *	msleep:EINTR
+ *
+ * Notes:	We add the lock to the provisional lock list.  We do not
+ *		coelesce at this time; this has implications for other lock
+ *		requestors in the blocker search mechanism.
  */
 static int
-lf_setlock(lock)
-	struct lockf *lock;
+lf_setlock(struct lockf *lock)
 {
 	struct lockf *block;
 	struct lockf **head = lock->lf_head;
 	struct lockf **prev, *overlap, *ltmp;
 	static char lockstr[] = "lockf";
-	int ovcase, priority, needtolink, error;
+	int priority, needtolink, error;
 	struct vnode *vp = lock->lf_vnode;
+	overlap_t ovcase;
 
-#ifdef LOCKF_DEBUG
-	if (lockf_debug & 1)
+#ifdef LOCKF_DEBUGGING
+	if (lockf_debug & 1) {
 		lf_print("lf_setlock", lock);
-#endif /* LOCKF_DEBUG */
+		lf_printlist("lf_setlock(in)", lock);
+	}
+#endif /* LOCKF_DEBUGGING */
 
 	/*
 	 * Set the priority
@@ -276,10 +409,7 @@ lf_setlock(lock)
 			FREE(lock, M_LOCKF);
 			return (EAGAIN);
 		}
-#if DEAD_CODE
-/*
- * XXX This is dead code on MacOS X; it shouldn't be.
- */
+
 		/*
 		 * We are blocked. Since flock style locks cover
 		 * the whole file, there is no chance for deadlock.
@@ -292,35 +422,66 @@ lf_setlock(lock)
 		 */
 		if ((lock->lf_flags & F_POSIX) &&
 		    (block->lf_flags & F_POSIX)) {
-			struct proc *wproc;
-			struct thread *td;
+			struct proc *wproc, *bproc;
+			struct uthread *ut;
 			struct lockf *waitblock;
 			int i = 0;
 
 			/* The block is waiting on something */
-			/* XXXKSE this is not complete under threads */
 			wproc = (struct proc *)block->lf_id;
-			mtx_lock_spin(&sched_lock);
-			FOREACH_THREAD_IN_PROC(wproc, td) {
-				while (td->td_wchan &&
-				    (td->td_wmesg == lockstr) &&
+			proc_lock(wproc);
+			TAILQ_FOREACH(ut, &wproc->p_uthlist, uu_list) {
+				/*
+				 * While the thread is asleep (uu_wchan != 0)
+				 * in this code (uu_wmesg == lockstr)
+				 * and we have not exceeded the maximum cycle
+				 * depth (i < maxlockdepth), then check for a
+				 * cycle to see if the lock is blocked behind
+				 * someone blocked behind us.
+				 */
+				while (((waitblock = (struct lockf *)ut->uu_wchan) != NULL) &&
+				    ut->uu_wmesg == lockstr &&
 				    (i++ < maxlockdepth)) {
-					waitblock = (struct lockf *)td->td_wchan;
-					/* Get the owner of the blocking lock */
+					waitblock = (struct lockf *)ut->uu_wchan;
+					/*
+					 * Get the lock blocking the lock
+					 * which would block us, and make
+					 * certain it hasn't come unblocked
+					 * (been granted, e.g. between the time
+					 * we called lf_getblock, and the time
+					 * we successfully acquired the
+					 * proc_lock).
+					 */
 					waitblock = waitblock->lf_next;
+					if (waitblock == NULL)
+						break;
+
+					/*
+					 * Make sure it's an advisory range
+					 * lock and not an overall file lock;
+					 * if we mix lock types, it's our own
+					 * fault.
+					 */
 					if ((waitblock->lf_flags & F_POSIX) == 0)
 						break;
-					wproc = (struct proc *)waitblock->lf_id;
-					if (wproc == (struct proc *)lock->lf_id) {
-						mtx_unlock_spin(&sched_lock);
+
+					/*
+					 * If the owner of the lock that's
+					 * blocking a lock that's blocking us
+					 * getting the requested lock, then we
+					 * would deadlock, so error out.
+					 */
+					bproc = (struct proc *)waitblock->lf_id;
+					if (bproc == (struct proc *)lock->lf_id) {
+						proc_unlock(wproc);
 						FREE(lock, M_LOCKF);
 						return (EDEADLK);
 					}
 				}
 			}
-			mtx_unlock_spin(&sched_lock);
+			proc_unlock(wproc);
 		}
-#endif	/* DEAD_CODE */
+
 		/*
 		 * For flock type locks, we must first remove
 		 * any shared locks that we hold before we sleep
@@ -329,7 +490,10 @@ lf_setlock(lock)
 		if ((lock->lf_flags & F_FLOCK) &&
 		    lock->lf_type == F_WRLCK) {
 			lock->lf_type = F_UNLCK;
-			(void) lf_clearlock(lock);
+			if ((error = lf_clearlock(lock)) != 0) {
+				FREE(lock, M_LOCKF);
+				return (error);
+			}
 			lock->lf_type = F_WRLCK;
 		}
 		/*
@@ -338,12 +502,12 @@ lf_setlock(lock)
 		 */
 		lock->lf_next = block;
 		TAILQ_INSERT_TAIL(&block->lf_blkhd, lock, lf_block);
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 		if (lockf_debug & 1) {
 			lf_print("lf_setlock: blocking on", block);
-			lf_printlist("lf_setlock", block);
+			lf_printlist("lf_setlock(block)", block);
 		}
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 		error = msleep(lock, &vp->v_lock, priority, lockstr, 0);
 		if (error) {	/* XXX */
 			/*
@@ -387,14 +551,14 @@ lf_setlock(lock)
 		 *	5) overlap ends after lock
 		 */
 		switch (ovcase) {
-		case 0: /* no overlap */
+		case OVERLAP_NONE:
 			if (needtolink) {
 				*prev = lock;
 				lock->lf_next = overlap;
 			}
 			break;
 
-		case 1: /* overlap == lock */
+		case OVERLAP_EQUALS_LOCK:
 			/*
 			 * If downgrading lock, others may be
 			 * able to acquire it.
@@ -404,28 +568,37 @@ lf_setlock(lock)
 				lf_wakelock(overlap);
 			overlap->lf_type = lock->lf_type;
 			FREE(lock, M_LOCKF);
-			lock = overlap; /* for debug output below */
+			lock = overlap; /* for lf_coelesce_adjacent() */
 			break;
 
-		case 2: /* overlap contains lock */
+		case OVERLAP_CONTAINS_LOCK:
 			/*
 			 * Check for common starting point and different types.
 			 */
 			if (overlap->lf_type == lock->lf_type) {
 				FREE(lock, M_LOCKF);
-				lock = overlap; /* for debug output below */
+				lock = overlap; /* for lf_coelesce_adjacent() */
 				break;
 			}
 			if (overlap->lf_start == lock->lf_start) {
 				*prev = lock;
 				lock->lf_next = overlap;
 				overlap->lf_start = lock->lf_end + 1;
-			} else
-				lf_split(overlap, lock);
+			} else {
+				/*
+				 * If we can't split the lock, we can't
+				 * grant it.  Claim a system limit for the
+				 * resource shortage.
+				 */
+				if (lf_split(overlap, lock)) {
+					FREE(lock, M_LOCKF);
+					return (ENOLCK);
+				}
+			}
 			lf_wakelock(overlap);
 			break;
 
-		case 3: /* lock contains overlap */
+		case OVERLAP_CONTAINED_BY_LOCK:
 			/*
 			 * If downgrading lock, others may be able to
 			 * acquire it, otherwise take the list.
@@ -456,7 +629,7 @@ lf_setlock(lock)
 			FREE(overlap, M_LOCKF);
 			continue;
 
-		case 4: /* overlap starts before lock */
+		case OVERLAP_STARTS_BEFORE_LOCK:
 			/*
 			 * Add lock after overlap on the list.
 			 */
@@ -468,7 +641,7 @@ lf_setlock(lock)
 			needtolink = 0;
 			continue;
 
-		case 5: /* overlap ends after lock */
+		case OVERLAP_ENDS_AFTER_LOCK:
 			/*
 			 * Add the new lock before overlap.
 			 */
@@ -482,101 +655,136 @@ lf_setlock(lock)
 		}
 		break;
 	}
-#ifdef LOCKF_DEBUG
+	/* Coelesce adjacent locks with identical attributes */
+	lf_coelesce_adjacent(lock);
+#ifdef LOCKF_DEBUGGING
 	if (lockf_debug & 1) {
 		lf_print("lf_setlock: got the lock", lock);
-		lf_printlist("lf_setlock", lock);
+		lf_printlist("lf_setlock(out)", lock);
 	}
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 	return (0);
 }
 
+
 /*
- * Remove a byte-range lock on an inode.
+ * lf_clearlock
+ *
+ * Description:	Remove a byte-range lock on an vnode.  Generally, find the
+ *		lock (or an overlap to that lock) and remove it (or shrink
+ *		it), then wakeup anyone we can.
+ *
+ * Parameters:	unlock			The lock to clear
  *
- * Generally, find the lock (or an overlap to that lock)
- * and remove it (or shrink it), then wakeup anyone we can.
+ * Returns:	0			Success
+ *	lf_split:ENOLCK
+ *
+ * Notes:	A caller may unlock all the locks owned by the caller by
+ *		specifying the entire file range; locks owned by other
+ *		callers are not effected by this operation.
  */
 static int
-lf_clearlock(unlock)
-	struct lockf *unlock;
+lf_clearlock(struct lockf *unlock)
 {
 	struct lockf **head = unlock->lf_head;
 	struct lockf *lf = *head;
 	struct lockf *overlap, **prev;
-	int ovcase;
+	overlap_t ovcase;
 
 	if (lf == NOLOCKF)
 		return (0);
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 	if (unlock->lf_type != F_UNLCK)
 		panic("lf_clearlock: bad type");
 	if (lockf_debug & 1)
 		lf_print("lf_clearlock", unlock);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 	prev = head;
-	while ((ovcase = lf_findoverlap(lf, unlock, SELF, &prev, &overlap))) {
+	while ((ovcase = lf_findoverlap(lf, unlock, SELF, &prev, &overlap)) != OVERLAP_NONE) {
 		/*
 		 * Wakeup the list of locks to be retried.
 		 */
 		lf_wakelock(overlap);
 
 		switch (ovcase) {
+		case OVERLAP_NONE:	/* satisfy compiler enum/switch */
+			break;
 
-		case 1: /* overlap == lock */
+		case OVERLAP_EQUALS_LOCK:
 			*prev = overlap->lf_next;
 			FREE(overlap, M_LOCKF);
 			break;
 
-		case 2: /* overlap contains lock: split it */
+		case OVERLAP_CONTAINS_LOCK: /* split it */
 			if (overlap->lf_start == unlock->lf_start) {
 				overlap->lf_start = unlock->lf_end + 1;
 				break;
 			}
-			lf_split(overlap, unlock);
+			/*
+			 * If we can't split the lock, we can't grant it.
+			 * Claim a system limit for the resource shortage.
+			 */
+			if (lf_split(overlap, unlock))
+				return (ENOLCK);
 			overlap->lf_next = unlock->lf_next;
 			break;
 
-		case 3: /* lock contains overlap */
+		case OVERLAP_CONTAINED_BY_LOCK:
 			*prev = overlap->lf_next;
 			lf = overlap->lf_next;
 			FREE(overlap, M_LOCKF);
 			continue;
 
-		case 4: /* overlap starts before lock */
+		case OVERLAP_STARTS_BEFORE_LOCK:
 			overlap->lf_end = unlock->lf_start - 1;
 			prev = &overlap->lf_next;
 			lf = overlap->lf_next;
 			continue;
 
-		case 5: /* overlap ends after lock */
+		case OVERLAP_ENDS_AFTER_LOCK:
 			overlap->lf_start = unlock->lf_end + 1;
 			break;
 		}
 		break;
 	}
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 	if (lockf_debug & 1)
 		lf_printlist("lf_clearlock", unlock);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 	return (0);
 }
 
+
 /*
- * Check whether there is a blocking lock,
- * and if so return its process identifier.
+ * lf_getlock
+ *
+ * Description:	Check whether there is a blocking lock, and if so return
+ *		its process identifier into the lock being requested.
+ *
+ * Parameters:	lock			Pointer to lock to test for blocks
+ *		fl			Pointer to flock structure to receive
+ *					the blocking lock information, if a
+ *					blocking lock is found.
+ *
+ * Returns:	0			Success
+ *
+ * Implicit Returns:
+ *		*fl			Contents modified to reflect the
+ *					blocking lock, if one is found; not
+ *					modified otherwise
+ *
+ * Notes:	fl->l_pid will be (-1) for file locks and will only be set to
+ *		the blocking process ID for advisory record locks.
  */
 static int
-lf_getlock(lock, fl)
-	struct lockf *lock;
-	struct flock *fl;
+lf_getlock(struct lockf *lock, struct flock *fl)
 {
 	struct lockf *block;
 
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 	if (lockf_debug & 1)
 		lf_print("lf_getlock", lock);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 
 	if ((block = lf_getblock(lock))) {
 		fl->l_type = block->lf_type;
@@ -596,19 +804,30 @@ lf_getlock(lock, fl)
 	return (0);
 }
 
+
 /*
- * Walk the list of locks for an inode and
- * return the first blocking lock.
+ * lf_getblock
+ *
+ * Description:	Walk the list of locks for an inode and return the first
+ *		blocking lock.  A lock is considered blocking if we are not
+ *		the lock owner; otherwise, we are permitted to upgrade or
+ *		downgrade it, and it's not considered blocking.
+ *
+ * Parameters:	lock			The lock for which we are interested
+ *					in obtaining the blocking lock, if any
+ *
+ * Returns:	NOLOCKF			No blocking lock exists
+ *		!NOLOCKF		The address of the blocking lock's
+ *					struct lockf.
  */
 static struct lockf *
-lf_getblock(lock)
-	struct lockf *lock;
+lf_getblock(struct lockf *lock)
 {
 	struct lockf **prev, *overlap, *lf = *(lock->lf_head);
 	int ovcase;
 
 	prev = lock->lf_head;
-	while ((ovcase = lf_findoverlap(lf, lock, OTHERS, &prev, &overlap))) {
+	while ((ovcase = lf_findoverlap(lf, lock, OTHERS, &prev, &overlap)) != OVERLAP_NONE) {
 		/*
 		 * We've found an overlap, see if it blocks us
 		 */
@@ -623,30 +842,64 @@ lf_getblock(lock)
 	return (NOLOCKF);
 }
 
+
 /*
- * Walk the list of locks to
- * find an overlapping lock (if any).
+ * lf_findoverlap
+ *
+ * Description:	Walk the list of locks to find an overlapping lock (if any).
+ *
+ * Parameters:	lf			First lock on lock list
+ *		lock			The lock we are checking for an overlap
+ *		check			Check type
+ *		prev			pointer to pointer pointer to contain
+ *					address of pointer to previous lock
+ *					pointer to overlapping lock, if overlap
+ *		overlap			pointer to pointer to contain address
+ *					of overlapping lock
+ *
+ * Returns:	OVERLAP_NONE
+ *		OVERLAP_EQUALS_LOCK
+ *		OVERLAP_CONTAINS_LOCK
+ *		OVERLAP_CONTAINED_BY_LOCK
+ *		OVERLAP_STARTS_BEFORE_LOCK
+ *		OVERLAP_ENDS_AFTER_LOCK
  *
- * NOTE: this returns only the FIRST overlapping lock.  There
- *	 may be more than one.
+ * Implicit Returns:
+ *		*prev			The address of the next pointer in the
+ *					lock previous to the overlapping lock;
+ *					this is generally used to relink the
+ *					lock list, avoiding a second iteration.
+ *		*overlap		The pointer to the overlapping lock
+ *					itself; this is ussed to return data in
+ *					the check == OTHERS case, and for the
+ *					caller to modify the overlapping lock,
+ *					in the check == SELF case
+ *
+ * Note:	This returns only the FIRST overlapping lock.  There may be
+ *		more than one.  lf_getlock will return the first blocking lock,
+ *		while lf_setlock will iterate over all overlapping locks to
+ *
+ *		The check parameter can be SELF, meaning we are looking for
+ *		overelapping locks owned by us, or it can be OTHERS, meaning
+ *		we are looking for overlapping locks owned by someone else so
+ *		we can report a blocking lock on an F_GETLK request.
+ *
+ *		The value of *overlap and *prev are modified, even if there is
+ *		no overlapping lock found; always check the return code.
  */
-static int
-lf_findoverlap(lf, lock, type, prev, overlap)
-	struct lockf *lf;
-	struct lockf *lock;
-	int type;
-	struct lockf ***prev;
-	struct lockf **overlap;
+static overlap_t
+lf_findoverlap(struct lockf *lf, struct lockf *lock, int type,
+	       struct lockf ***prev, struct lockf **overlap)
 {
 	off_t start, end;
 
 	*overlap = lf;
 	if (lf == NOLOCKF)
 		return (0);
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 	if (lockf_debug & 2)
 		lf_print("lf_findoverlap: looking for overlap in", lock);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 	start = lock->lf_start;
 	end = lock->lf_end;
 	while (lf != NOLOCKF) {
@@ -656,122 +909,111 @@ lf_findoverlap(lf, lock, type, prev, overlap)
 			*overlap = lf = lf->lf_next;
 			continue;
 		}
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 		if (lockf_debug & 2)
 			lf_print("\tchecking", lf);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 		/*
 		 * OK, check for overlap
-		 *
-		 * Six cases:
-		 *	0) no overlap
-		 *	1) overlap == lock
-		 *	2) overlap contains lock
-		 *	3) lock contains overlap
-		 *	4) overlap starts before lock
-		 *	5) overlap ends after lock
 		 */
 		if ((lf->lf_end != -1 && start > lf->lf_end) ||
 		    (end != -1 && lf->lf_start > end)) {
 			/* Case 0 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("no overlap\n");
-#endif /* LOCKF_DEBUG */
+			LOCKF_DEBUG(2, "no overlap\n");
 			if ((type & SELF) && end != -1 && lf->lf_start > end)
-				return (0);
+				return (OVERLAP_NONE);
 			*prev = &lf->lf_next;
 			*overlap = lf = lf->lf_next;
 			continue;
 		}
 		if ((lf->lf_start == start) && (lf->lf_end == end)) {
-			/* Case 1 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("overlap == lock\n");
-#endif /* LOCKF_DEBUG */
-			return (1);
+			LOCKF_DEBUG(2, "overlap == lock\n");
+			return (OVERLAP_EQUALS_LOCK);
 		}
 		if ((lf->lf_start <= start) &&
 		    (end != -1) &&
 		    ((lf->lf_end >= end) || (lf->lf_end == -1))) {
-			/* Case 2 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("overlap contains lock\n");
-#endif /* LOCKF_DEBUG */
-			return (2);
+			LOCKF_DEBUG(2, "overlap contains lock\n");
+			return (OVERLAP_CONTAINS_LOCK);
 		}
 		if (start <= lf->lf_start &&
 		           (end == -1 ||
 			   (lf->lf_end != -1 && end >= lf->lf_end))) {
-			/* Case 3 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("lock contains overlap\n");
-#endif /* LOCKF_DEBUG */
-			return (3);
+			LOCKF_DEBUG(2, "lock contains overlap\n");
+			return (OVERLAP_CONTAINED_BY_LOCK);
 		}
 		if ((lf->lf_start < start) &&
 			((lf->lf_end >= start) || (lf->lf_end == -1))) {
-			/* Case 4 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("overlap starts before lock\n");
-#endif /* LOCKF_DEBUG */
-			return (4);
+			LOCKF_DEBUG(2, "overlap starts before lock\n");
+			return (OVERLAP_STARTS_BEFORE_LOCK);
 		}
 		if ((lf->lf_start > start) &&
 			(end != -1) &&
 			((lf->lf_end > end) || (lf->lf_end == -1))) {
-			/* Case 5 */
-#ifdef LOCKF_DEBUG
-			if (lockf_debug & 2)
-				printf("overlap ends after lock\n");
-#endif /* LOCKF_DEBUG */
-			return (5);
+			LOCKF_DEBUG(2, "overlap ends after lock\n");
+			return (OVERLAP_ENDS_AFTER_LOCK);
 		}
 		panic("lf_findoverlap: default");
 	}
-	return (0);
+	return (OVERLAP_NONE);
 }
 
+
 /*
- * Split a lock and a contained region into
- * two or three locks as necessary.
+ * lf_split
+ *
+ * Description:	Split a lock and a contained region into two or three locks
+ *		as necessary.
+ *
+ * Parameters:	lock1			Lock to split
+ *		lock2			Overlapping lock region requiring the
+ *					split (upgrade/downgrade/unlock)
+ *
+ * Returns:	0			Success
+ *		ENOLCK			No memory for new lock
+ *
+ * Implicit Returns:
+ *		*lock1			Modified original lock
+ *		*lock2			Overlapping lock (inserted into list)
+ *		(new lock)		Potential new lock inserted into list
+ *					if split results in 3 locks
+ *
+ * Notes:	This operation can only fail if the split would result in three
+ *		locks, and there is insufficient memory to allocate the third
+ *		lock; in that case, neither of the locks will be modified.
  */
-static void
-lf_split(lock1, lock2)
-	struct lockf *lock1;
-	struct lockf *lock2;
+static int
+lf_split(struct lockf *lock1, struct lockf *lock2)
 {
 	struct lockf *splitlock;
 
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 	if (lockf_debug & 2) {
 		lf_print("lf_split", lock1);
 		lf_print("splitting from", lock2);
 	}
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 	/*
 	 * Check to see if spliting into only two pieces.
 	 */
 	if (lock1->lf_start == lock2->lf_start) {
 		lock1->lf_start = lock2->lf_end + 1;
 		lock2->lf_next = lock1;
-		return;
+		return (0);
 	}
 	if (lock1->lf_end == lock2->lf_end) {
 		lock1->lf_end = lock2->lf_start - 1;
 		lock2->lf_next = lock1->lf_next;
 		lock1->lf_next = lock2;
-		return;
+		return (0);
 	}
 	/*
 	 * Make a new lock consisting of the last part of
 	 * the encompassing lock
 	 */
 	MALLOC(splitlock, struct lockf *, sizeof *splitlock, M_LOCKF, M_WAITOK);
+	if (splitlock == NULL)
+		return (ENOLCK);
 	bcopy(lock1, splitlock, sizeof *splitlock);
 	splitlock->lf_start = lock2->lf_end + 1;
 	TAILQ_INIT(&splitlock->lf_blkhd);
@@ -782,14 +1024,31 @@ lf_split(lock1, lock2)
 	splitlock->lf_next = lock1->lf_next;
 	lock2->lf_next = splitlock;
 	lock1->lf_next = lock2;
+
+	return (0);
 }
 
+
 /*
- * Wakeup a blocklist
+ * lf_wakelock
+ *
+ * Wakeup a blocklist in the case of a downgrade or unlock, since others
+ * waiting on the lock may now be able to acquire it.
+ *
+ * Parameters:	listhead		Lock list head on which waiters may
+ *					have pending locks
+ *
+ * Returns:	<void>
+ *
+ * Notes:	This function iterates a list of locks and wakes all waiters,
+ *		rather than only waiters for the contended regions.  Because
+ *		of this, for heavily contended files, this can result in a
+ *		"thundering herd" situation.  Refactoring the code could make
+ *		this operation more efficient, if heavy contention ever results
+ *		in a real-world performance problem.
  */
 static void
-lf_wakelock(listhead)
-	struct lockf *listhead;
+lf_wakelock(struct lockf *listhead)
 {
 	struct lockf *wakelock;
 
@@ -797,38 +1056,44 @@ lf_wakelock(listhead)
 		wakelock = TAILQ_FIRST(&listhead->lf_blkhd);
 		TAILQ_REMOVE(&listhead->lf_blkhd, wakelock, lf_block);
 		wakelock->lf_next = NOLOCKF;
-#ifdef LOCKF_DEBUG
+#ifdef LOCKF_DEBUGGING
 		if (lockf_debug & 2)
 			lf_print("lf_wakelock: awakening", wakelock);
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
 		wakeup(wakelock);
 	}
 }
 
-#ifdef LOCKF_DEBUG
+
+#ifdef LOCKF_DEBUGGING
 /*
- * Print out a lock.
+ * lf_print DEBUG
+ *
+ * Print out a lock; lock information is prefixed by the string in 'tag'
+ *
+ * Parameters:	tag			A string tag for debugging
+ *		lock			The lock whose information should be
+ *					displayed
+ *
+ * Returns:	<void>
  */
 void
-lf_print(tag, lock)
-	char *tag;
-	struct lockf *lock;
+lf_print(const char *tag, struct lockf *lock)
 {
-
 	printf("%s: lock %p for ", tag, (void *)lock);
 	if (lock->lf_flags & F_POSIX)
 		printf("proc %ld", (long)((struct proc *)lock->lf_id)->p_pid);
 	else
 		printf("id %p", (void *)lock->lf_id);
 	if (lock->lf_vnode != 0)
-		printf(" in vno 0x%08x, %s, start %jd, end %jd",
+		printf(" in vno %p, %s, start 0x%016llx, end 0x%016llx",
 		    lock->lf_vnode,
 		    lock->lf_type == F_RDLCK ? "shared" :
 		    lock->lf_type == F_WRLCK ? "exclusive" :
 		    lock->lf_type == F_UNLCK ? "unlock" : "unknown",
 		    (intmax_t)lock->lf_start, (intmax_t)lock->lf_end);
 	else
-		printf(" %s, start %jd, end %jd",
+		printf(" %s, start 0x%016llx, end 0x%016llx",
 		    lock->lf_type == F_RDLCK ? "shared" :
 		    lock->lf_type == F_WRLCK ? "exclusive" :
 		    lock->lf_type == F_UNLCK ? "unlock" : "unknown",
@@ -839,17 +1104,28 @@ lf_print(tag, lock)
 		printf("\n");
 }
 
+
+/*
+ * lf_printlist DEBUG
+ *
+ * Print out a lock list for the vnode associated with 'lock'; lock information
+ * is prefixed by the string in 'tag'
+ *
+ * Parameters:	tag			A string tag for debugging
+ *		lock			The lock whose vnode's lock list should
+ *					be displayed
+ *
+ * Returns:	<void>
+ */
 void
-lf_printlist(tag, lock)
-	char *tag;
-	struct lockf *lock;
+lf_printlist(const char *tag, struct lockf *lock)
 {
 	struct lockf *lf, *blk;
 
 	if (lock->lf_vnode == 0)
 		return;
 
-	printf("%s: Lock list for vno 0x%08x:\n",
+	printf("%s: Lock list for vno %p:\n",
 	    tag, lock->lf_vnode);
 	for (lf = lock->lf_vnode->v_lockf; lf; lf = lf->lf_next) {
 		printf("\tlock %p for ",(void *)lf);
@@ -858,7 +1134,7 @@ lf_printlist(tag, lock)
 			    (long)((struct proc *)lf->lf_id)->p_pid);
 		else
 			printf("id %p", (void *)lf->lf_id);
-		printf(", %s, start %jd, end %jd",
+		printf(", %s, start 0x%016llx, end 0x%016llx",
 		    lf->lf_type == F_RDLCK ? "shared" :
 		    lf->lf_type == F_WRLCK ? "exclusive" :
 		    lf->lf_type == F_UNLCK ? "unlock" :
@@ -870,7 +1146,7 @@ lf_printlist(tag, lock)
 				    (long)((struct proc *)blk->lf_id)->p_pid);
 			else
 				printf("id %p", (void *)blk->lf_id);
-			printf(", %s, start %jd, end %jd",
+			printf(", %s, start 0x%016llx, end 0x%016llx",
 			    blk->lf_type == F_RDLCK ? "shared" :
 			    blk->lf_type == F_WRLCK ? "exclusive" :
 			    blk->lf_type == F_UNLCK ? "unlock" :
@@ -882,4 +1158,4 @@ lf_printlist(tag, lock)
 		printf("\n");
 	}
 }
-#endif /* LOCKF_DEBUG */
+#endif /* LOCKF_DEBUGGING */
diff --git a/bsd/kern/kern_malloc.c b/bsd/kern/kern_malloc.c
index 3a61d1338..a1c8f1b50 100644
--- a/bsd/kern/kern_malloc.c
+++ b/bsd/kern/kern_malloc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)kern_malloc.c	8.4 (Berkeley) 5/20/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -82,13 +94,13 @@
 #include <sys/quota.h>
 #include <sys/uio_internal.h>
 #include <sys/resourcevar.h>
+#include <sys/signalvar.h>
 
 #include <ufs/ufs/inode.h>
 
 #include <hfs/hfs_cnode.h>
 #include <isofs/cd9660/cd9660_node.h>
 
-#include <miscfs/volfs/volfs.h>
 #include <miscfs/specfs/specdev.h>
 
 #include <nfs/rpcv2.h>
@@ -103,128 +115,370 @@
 #include <kern/zalloc.h>
 #include <kern/kalloc.h>
 
-struct kmemstats kmemstats[M_LAST];
-char *memname[] = INITKMEMNAMES;
+void kmeminit(void) __attribute__((section("__TEXT, initcode")));
 
-struct kmzones {
-	size_t		kz_elemsize;
-	void		*kz_zalloczone;
+/* Strings corresponding to types of memory.
+ * Must be in synch with the #defines is sys/malloc.h 
+ * NOTE - the reason we pass null strings in some cases is to reduce of foot
+ * print as much as possible for systems where a tiny kernel is needed.
+ * todo - We should probably redsign this and use enums for our types and only
+ * include types needed for that configuration of the kernel.  This can't be
+ * done without some kind of kpi since several types are hardwired and exported
+ * (for example see types M_HFSMNT, M_UDFMNT, M_TEMP, etc in sys/malloc.h)
+ */
+const char *memname[] = {
+	"free",		/* 0 M_FREE */
+	"mbuf",		/* 1 M_MBUF */
+	"devbuf",	/* 2 M_DEVBUF */ 
+	"socket",	/* 3 M_SOCKET */ 
+	"pcb",		/* 4 M_PCB */ 
+	"routetbl",	/* 5 M_RTABLE */ 
+	"hosttbl",	/* 6 M_HTABLE */ 
+	"fragtbl",	/* 7 M_FTABLE */ 
+	"zombie",	/* 8 M_ZOMBIE */ 
+	"ifaddr",	/* 9 M_IFADDR */ 
+	"soopts",	/* 10 M_SOOPTS */ 
+	"soname",	/* 11 M_SONAME */ 
+	"namei",	/* 12 M_NAMEI */ 
+	"gprof",	/* 13 M_GPROF */ 
+	"ioctlops",	/* 14 M_IOCTLOPS */ 
+	"mapmem",	/* 15 M_MAPMEM */ 
+	"cred",		/* 16 M_CRED */ 
+	"pgrp",		/* 17 M_PGRP */ 
+	"session",	/* 18 M_SESSION */ 
+	"iov32",	/* 19 M_IOV32 */ 
+	"mount",	/* 20 M_MOUNT */ 
+	"fhandle",		/* 21 M_FHANDLE */ 
+#if (NFSCLIENT || NFSSERVER)
+	"NFS req",		/* 22 M_NFSREQ */ 
+	"NFS mount",	/* 23 M_NFSMNT */ 
+	"NFS node",		/* 24 M_NFSNODE */ 
+#else
+	"",				/* 22 M_NFSREQ */ 
+	"",				/* 23 M_NFSMNT */ 
+	"",				/* 24 M_NFSNODE */ 
+#endif
+	"vnodes",		/* 25 M_VNODE */ 
+	"namecache",	/* 26 M_CACHE */ 
+#if QUOTA
+	"UFS quota",	/* 27 M_DQUOT */ 
+#else
+	"",				/* 27 M_DQUOT */ 
+#endif
+#if FFS
+	"UFS mount",	/* 28 M_UFSMNT */ 
+#else
+	"",				/* 28 M_UFSMNT */ 
+#endif
+#if (SYSV_SEM || SYSV_MSG || SYSV_SHM)
+	"shm",			/* 29 M_SHM */ 
+#else
+	"",				/* 29 M_SHM */ 
+#endif
+	"plimit",		/* 30 M_VMMAP */ 
+	"sigacts",	/* 31 M_VMMAPENT */ 
+	"VM object",	/* 32 M_VMOBJ */ 
+	"VM objhash",	/* 33 M_VMOBJHASH */ 
+	"VM pmap",		/* 34 M_VMPMAP */ 
+	"VM pvmap",		/* 35 M_VMPVENT */ 
+	"VM pager",		/* 36 M_VMPAGER */ 
+	"VM pgdata",	/* 37 M_VMPGDATA */ 
+	"fileproc",		/* 38 M_FILEPROC */ 
+	"file desc",	/* 39 M_FILEDESC */ 
+	"lockf",		/* 40 M_LOCKF */ 
+	"proc",			/* 41 M_PROC */ 
+	"pstats",		/* 42 M_SUBPROC */ 
+	"LFS segment",	/* 43 M_SEGMENT */ 
+	"LFS node",		/* 44 M_LFSNODE */ 
+#if FFS
+	"FFS node",		/* 45 M_FFSNODE */ 
+#else
+	"",				/* 45 M_FFSNODE */ 
+#endif
+	"MFS node",		/* 46 M_MFSNODE */ 
+	"NQNFS Lease",	/* 47 M_NQLEASE */ 
+	"NQNFS Host",	/* 48 M_NQMHOST */ 
+	"Export Host",	/* 49 M_NETADDR */ 
+#if (NFSCLIENT || NFSSERVER)
+	"NFS srvsock",	/* 50 M_NFSSVC */ 
+	"NFS uid",		/* 51 M_NFSUID */ 
+	"NFS daemon",	/* 52 M_NFSD */ 
+#else
+	"",				/* 50 M_NFSSVC */ 
+	"",				/* 51 M_NFSUID */ 
+	"",				/* 52 M_NFSD */ 
+#endif
+	"ip_moptions",	/* 53 M_IPMOPTS */ 
+	"in_multi",		/* 54 M_IPMADDR */ 
+	"ether_multi",	/* 55 M_IFMADDR */ 
+	"mrt",			/* 56 M_MRTABLE */ 
+#if CD9660
+	"ISOFS mount",	/* 57 M_ISOFSMNT */ 
+	"ISOFS node",	/* 58 M_ISOFSNODE */ 
+#else
+	"",				/* 57 M_ISOFSMNT */ 
+	"",				/* 58 M_ISOFSNODE */ 
+#endif
+#if (NFSCLIENT || NFSSERVER)
+	"NFSV3 srvdesc",/* 59 M_NFSRVDESC */ 
+	"NFSV3 diroff",	/* 60 M_NFSDIROFF */ 
+	"NFSV3 bigfh",	/* 61 M_NFSBIGFH */ 
+#else
+	"",				/* 59 M_NFSRVDESC */ 
+	"",				/* 60 M_NFSDIROFF */ 
+	"",				/* 61 M_NFSBIGFH */ 
+#endif
+	"MSDOSFS mount",/* 62 M_MSDOSFSMNT */ 
+	"MSDOSFS fat",	/* 63 M_MSDOSFSFAT */ 
+	"MSDOSFS node",	/* 64 M_MSDOSFSNODE */ 
+	"ttys",			/* 65 M_TTYS */ 
+	"exec",			/* 66 M_EXEC */ 
+	"miscfs mount",	/* 67 M_MISCFSMNT */ 
+	"miscfs node",	/* 68 M_MISCFSNODE */ 
+	"adosfs mount",	/* 69 M_ADOSFSMNT */ 
+	"adosfs node",	/* 70 M_ADOSFSNODE */ 
+	"adosfs anode",	/* 71 M_ANODE */ 
+	"buf hdrs",		/* 72 M_BUFHDR */ 
+	"ofile tabl",	/* 73 M_OFILETABL */ 
+	"mbuf clust",	/* 74 M_MCLUST */ 
+#if HFS
+	"HFS mount",	/* 75 M_HFSMNT */ 
+	"HFS node",		/* 76 M_HFSNODE */ 
+	"HFS fork",		/* 77 M_HFSFORK */ 
+#else
+	"",				/* 75 M_HFSMNT */ 
+	"",				/* 76 M_HFSNODE */ 
+	"",				/* 77 M_HFSFORK */ 
+#endif
+	"ZFS mount", 	/* 78 M_ZFSFSMNT */ 
+	"ZFS node", 	/* 79 M_ZFSNODE */ 
+	"temp",			/* 80 M_TEMP */ 
+	"key mgmt",		/* 81 M_SECA */ 
+	"DEVFS",		/* 82 M_DEVFS */ 
+	"IpFw/IpAcct",	/* 83 M_IPFW */ 
+	"UDF node",		/* 84 M_UDFNODE */ 
+	"UDF mount",	/* 85 M_UDFMNT */ 
+#if INET6
+	"IPv6 NDP",		/* 86 M_IP6NDP */ 
+	"IPv6 options",	/* 87 M_IP6OPT */ 
+	"IPv6 Misc",	/* 88 M_IP6MISC */
+#else
+	"",				/* 86 M_IP6NDP */ 
+	"",				/* 87 M_IP6OPT */ 
+	"",				/* 88 M_IP6MISC */
+#endif
+	"TCP Segment Q",/* 89 M_TSEGQ */
+	"IGMP state",	/* 90 M_IGMP */
+#if JOURNALING
+	"Journal",		/* 91 M_JNL_JNL */
+	"Transaction",	/* 92 M_JNL_TR */
+#else
+	"",    			/* 91 M_JNL_JNL */
+	"",    			/* 92 M_JNL_TR */
+#endif
+	"specinfo",		/* 93 M_SPECINFO */
+	"kqueue",		/* 94 M_KQUEUE */
+#if HFS
+	"HFS dirhint",	/* 95 M_HFSDIRHINT */ 
+#else
+	"",				/* 95 M_HFSDIRHINT */ 
+#endif
+	"cluster_read",	/* 96 M_CLRDAHEAD */ 
+	"cluster_write",/* 97 M_CLWRBEHIND */ 
+	"iov64",		/* 98 M_IOV64 */ 
+	"fileglob",		/* 99 M_FILEGLOB */ 
+	"kauth",		/* 100 M_KAUTH */ 
+	"dummynet",		/* 101 M_DUMMYNET */ 
+	"unsafe_fsnode",	/* 102 M_UNSAFEFS */ 
+	"macpipelabel", /* 103 M_MACPIPELABEL */
+	"mactemp",      /* 104 M_MACTEMP */
+	"sbuf",         /* 105 M_SBUF */
+	"extattr",      /* 106 M_EXTATTR */
+	"lctx",         /* 107 M_LCTX */
+#if TRAFFIC_MGT
+	"traffic_mgt",   /* 108 M_TRAFFIC_MGT */
+#else
+	"", /* 108 M_TRAFFIC_MGT */
+#endif
+};
+
+/* for use with kmzones.kz_zalloczone */
 #define	KMZ_CREATEZONE		((void *)-2)
 #define KMZ_LOOKUPZONE		((void *)-1)
-#define KMZ_MALLOC		((void *)0)
+#define KMZ_MALLOC			((void *)0)
 #define	KMZ_SHAREZONE		((void *)1)
+
+struct kmzones {
+	size_t		kz_elemsize;
+	void		*kz_zalloczone;
 } kmzones[M_LAST] = {
 #define	SOS(sname)	sizeof (struct sname)
 #define SOX(sname)	-1
-	-1,		0,			/* 0 M_FREE */
-	MSIZE,		KMZ_CREATEZONE,		/* 1 M_MBUF */
-	0,		KMZ_MALLOC,		/* 2 M_DEVBUF */
-	SOS(socket),	KMZ_CREATEZONE,		/* 3 M_SOCKET */
-	SOS(inpcb),	KMZ_LOOKUPZONE,		/* 4 M_PCB */
-	M_MBUF,		KMZ_SHAREZONE,		/* 5 M_RTABLE */
-	M_MBUF,		KMZ_SHAREZONE,		/* 6 M_HTABLE */
-	M_MBUF,		KMZ_SHAREZONE,		/* 7 M_FTABLE */
-	SOS(rusage),	KMZ_CREATEZONE,		/* 8 M_ZOMBIE */
-	0,		KMZ_MALLOC,		/* 9 M_IFADDR */
-	M_MBUF,		KMZ_SHAREZONE,		/* 10 M_SOOPTS */
-	0,		KMZ_MALLOC,		/* 11 M_SONAME */
-	MAXPATHLEN,	KMZ_CREATEZONE,		/* 12 M_NAMEI */
-	0,		KMZ_MALLOC,		/* 13 M_GPROF */
-	0,		KMZ_MALLOC,		/* 14 M_IOCTLOPS */
-	0,		KMZ_MALLOC,		/* 15 M_MAPMEM */
-	SOS(ucred),	KMZ_CREATEZONE,		/* 16 M_CRED */
-	SOS(pgrp),	KMZ_CREATEZONE,		/* 17 M_PGRP */
-	SOS(session),	KMZ_CREATEZONE,		/* 18 M_SESSION */
-	SOS(iovec_32),	KMZ_LOOKUPZONE,		/* 19 M_IOV32 */
-	SOS(mount),	KMZ_CREATEZONE,		/* 20 M_MOUNT */
-	0,		KMZ_MALLOC,		/* 21 M_FHANDLE */
-	SOS(nfsreq),	KMZ_CREATEZONE,		/* 22 M_NFSREQ */
-	SOS(nfsmount),	KMZ_CREATEZONE,		/* 23 M_NFSMNT */
-	SOS(nfsnode),	KMZ_CREATEZONE,		/* 24 M_NFSNODE */
-	SOS(vnode),	KMZ_CREATEZONE,		/* 25 M_VNODE */
-	SOS(namecache),	KMZ_CREATEZONE,		/* 26 M_CACHE */
-	SOX(dquot),	KMZ_LOOKUPZONE,		/* 27 M_DQUOT */
-	SOX(ufsmount),	KMZ_LOOKUPZONE,		/* 28 M_UFSMNT */
-	0,		KMZ_MALLOC,		/* 29 M_CGSUM */
-	0,		KMZ_MALLOC,		/* 30 M_VMMAP */
-	0,		KMZ_MALLOC,		/* 31 M_VMMAPENT */
-	0,		KMZ_MALLOC,		/* 32 M_VMOBJ */
-	0,		KMZ_MALLOC,		/* 33 M_VMOBJHASH */
-	0,		KMZ_MALLOC,		/* 34 M_VMPMAP */
-	0,		KMZ_MALLOC,		/* 35 M_VMPVENT */
-	0,		KMZ_MALLOC,		/* 36 M_VMPAGER */
-	0,		KMZ_MALLOC,		/* 37 M_VMPGDATA */
-	SOS(fileproc),	KMZ_CREATEZONE,		/* 38 M_FILEPROC */
-	SOS(filedesc),	KMZ_CREATEZONE,		/* 39 M_FILEDESC */
-	SOX(lockf),	KMZ_CREATEZONE,		/* 40 M_LOCKF */
-	SOS(proc),	KMZ_CREATEZONE,		/* 41 M_PROC */
-	SOS(pstats),	KMZ_CREATEZONE,		/* 42 M_SUBPROC */
-	0,		KMZ_MALLOC,		/* 43 M_SEGMENT */
-	M_FFSNODE,	KMZ_SHAREZONE,		/* 44 M_LFSNODE */
-	SOS(inode),	KMZ_CREATEZONE,		/* 45 M_FFSNODE */
-	M_FFSNODE,	KMZ_SHAREZONE,		/* 46 M_MFSNODE */
-	0,		KMZ_MALLOC,		/* 47 M_NQLEASE */
-	0,		KMZ_MALLOC,		/* 48 M_NQMHOST */
-	0,		KMZ_MALLOC,		/* 49 M_NETADDR */
-	SOX(nfssvc_sock),
-			KMZ_CREATEZONE,		/* 50 M_NFSSVC */
-	SOS(nfsuid),	KMZ_CREATEZONE,		/* 51 M_NFSUID */
-	SOX(nfsrvcache),
-			KMZ_CREATEZONE,		/* 52 M_NFSD */
-	SOX(ip_moptions),
-			KMZ_LOOKUPZONE,		/* 53 M_IPMOPTS */
-	SOX(in_multi),	KMZ_LOOKUPZONE,		/* 54 M_IPMADDR */
-	SOX(ether_multi),
-			KMZ_LOOKUPZONE,		/* 55 M_IFMADDR */
-	SOX(mrt),	KMZ_CREATEZONE,		/* 56 M_MRTABLE */
-	SOX(iso_mnt),	KMZ_LOOKUPZONE,		/* 57 M_ISOFSMNT */
-	SOS(iso_node),	KMZ_CREATEZONE,		/* 58 M_ISOFSNODE */
-	SOS(nfsrv_descript),
-			KMZ_CREATEZONE,		/* 59 M_NFSRVDESC */
-	SOS(nfsdmap),	KMZ_CREATEZONE,		/* 60 M_NFSDIROFF */
-	SOS(fhandle),	KMZ_LOOKUPZONE,		/* 61 M_NFSBIGFH */
-	0,		KMZ_MALLOC,		/* 62 M_MSDOSFSMNT */
-	0,		KMZ_MALLOC,		/* 63 M_MSDOSFSFAT */
-	0,		KMZ_MALLOC,		/* 64 M_MSDOSFSNODE */
-	SOS(tty),	KMZ_CREATEZONE,		/* 65 M_TTYS */
-	0,		KMZ_MALLOC,		/* 66 M_EXEC */
-	0,		KMZ_MALLOC,		/* 67 M_MISCFSMNT */
-	0,		KMZ_MALLOC,		/* 68 M_MISCFSNODE */
-	0,		KMZ_MALLOC,		/* 69 M_ADOSFSMNT */
-	0,		KMZ_MALLOC,		/* 70 M_ADOSFSNODE */
-	0,		KMZ_MALLOC,		/* 71 M_ANODE */
-	SOX(buf),	KMZ_CREATEZONE,		/* 72 M_BUFHDR */
-	(NDFILE * OFILESIZE),
-			KMZ_CREATEZONE,		/* 73 M_OFILETABL */
-	MCLBYTES,	KMZ_CREATEZONE,		/* 74 M_MCLUST */
-	SOX(hfsmount),	KMZ_LOOKUPZONE,		/* 75 M_HFSMNT */
-	SOS(cnode),	KMZ_CREATEZONE,		/* 76 M_HFSNODE */
-	SOS(filefork),	KMZ_CREATEZONE,		/* 77 M_HFSFORK */
-	SOX(volfs_mntdata),	KMZ_LOOKUPZONE,		/* 78 M_VOLFSMNT */
-	SOS(volfs_vndata),	KMZ_CREATEZONE,		/* 79 M_VOLFSNODE */
-	0,		KMZ_MALLOC,		/* 80 M_TEMP */
-	0,		KMZ_MALLOC,		/* 81 M_SECA */
-	0,		KMZ_MALLOC,		/* 82 M_DEVFS */
-	0,		KMZ_MALLOC,		/* 83 M_IPFW */
-	0,		KMZ_MALLOC,		/* 84 M_UDFNODE */
-	0,		KMZ_MALLOC,		/* 85 M_UDFMOUNT */
-	0,		KMZ_MALLOC,		/* 86 M_IP6NDP */
-	0,		KMZ_MALLOC,		/* 87 M_IP6OPT */
-	0,		KMZ_MALLOC,		/* 88 M_IP6MISC */
-	0,		KMZ_MALLOC,		/* 89 M_TSEGQ */
-	0,		KMZ_MALLOC,		/* 90 M_IGMP */
-	SOS(journal), KMZ_CREATEZONE,     /* 91 M_JNL_JNL */
-	SOS(transaction), KMZ_CREATEZONE,     /* 92 M_JNL_TR */
-	SOS(specinfo), KMZ_CREATEZONE,		/* 93 M_SPECINFO */
-	SOS(kqueue), KMZ_CREATEZONE,		/* 94 M_KQUEUE */
-	SOS(directoryhint), KMZ_CREATEZONE,	/* 95 M_HFSDIRHINT */
-	SOS(cl_readahead),  KMZ_CREATEZONE,	/* 96 M_CLRDAHEAD */
-	SOS(cl_writebehind),KMZ_CREATEZONE,	/* 97 M_CLWRBEHIND */
-	SOS(iovec_64),	KMZ_LOOKUPZONE,		/* 98 M_IOV64 */
-	SOS(fileglob),	KMZ_CREATEZONE,		/* 99 M_FILEGLOB */
-	0,		KMZ_MALLOC,		/* 100 M_KAUTH */
-	0,		KMZ_MALLOC,		/* 101 M_DUMMYNET */
-	SOS(unsafe_fsnode),KMZ_CREATEZONE,	/* 102 M_UNSAFEFS */
+	{ -1,		0 },			/* 0 M_FREE */
+	{ MSIZE,	KMZ_CREATEZONE },	/* 1 M_MBUF */
+	{ 0,		KMZ_MALLOC },		/* 2 M_DEVBUF */
+	{ SOS(socket),	KMZ_CREATEZONE },	/* 3 M_SOCKET */
+	{ SOS(inpcb),	KMZ_LOOKUPZONE },	/* 4 M_PCB */
+	{ M_MBUF,	KMZ_SHAREZONE },	/* 5 M_RTABLE */
+	{ M_MBUF,	KMZ_SHAREZONE },	/* 6 M_HTABLE */
+	{ M_MBUF,	KMZ_SHAREZONE },	/* 7 M_FTABLE */
+	{ SOS(rusage),	KMZ_CREATEZONE },	/* 8 M_ZOMBIE */
+	{ 0,		KMZ_MALLOC },		/* 9 M_IFADDR */
+	{ M_MBUF,	KMZ_SHAREZONE },		/* 10 M_SOOPTS */
+	{ 0,		KMZ_MALLOC },		/* 11 M_SONAME */
+	{ MAXPATHLEN,	KMZ_CREATEZONE },		/* 12 M_NAMEI */
+	{ 0,		KMZ_MALLOC },		/* 13 M_GPROF */
+	{ 0,		KMZ_MALLOC },		/* 14 M_IOCTLOPS */
+	{ 0,		KMZ_MALLOC },		/* 15 M_MAPMEM */
+	{ SOS(ucred),	KMZ_CREATEZONE },	/* 16 M_CRED */
+	{ SOS(pgrp),	KMZ_CREATEZONE },	/* 17 M_PGRP */
+	{ SOS(session),	KMZ_CREATEZONE },	/* 18 M_SESSION */
+	{ SOS(iovec_32),	KMZ_LOOKUPZONE },	/* 19 M_IOV32 */
+	{ SOS(mount),	KMZ_CREATEZONE },	/* 20 M_MOUNT */
+	{ 0,		KMZ_MALLOC },		/* 21 M_FHANDLE */
+#if (NFSCLIENT || NFSSERVER)
+	{ SOS(nfsreq),	KMZ_CREATEZONE },	/* 22 M_NFSREQ */
+	{ SOS(nfsmount),	KMZ_CREATEZONE },	/* 23 M_NFSMNT */
+	{ SOS(nfsnode),	KMZ_CREATEZONE },	/* 24 M_NFSNODE */
+#else
+	{ 0,		KMZ_MALLOC },		/* 22 M_NFSREQ */
+	{ 0,		KMZ_MALLOC },		/* 23 M_NFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 24 M_NFSNODE */
+#endif
+	{ SOS(vnode),	KMZ_CREATEZONE },	/* 25 M_VNODE */
+	{ SOS(namecache),	KMZ_CREATEZONE },	/* 26 M_CACHE */
+#if QUOTA
+	{ SOX(dquot),	KMZ_LOOKUPZONE },	/* 27 M_DQUOT */
+#else
+	{ 0,		KMZ_MALLOC },		/* 27 M_DQUOT */
+#endif
+#if FFS
+	{ SOX(ufsmount),	KMZ_LOOKUPZONE },	/* 28 M_UFSMNT */
+#else
+	{ 0,		KMZ_MALLOC },		/* 28 M_UFSMNT */
+#endif
+	{ 0,		KMZ_MALLOC },		/* 29 M_CGSUM */
+	{ SOS(plimit),	KMZ_CREATEZONE },	/* 30 M_PLIMIT */
+	{ SOS(sigacts),	KMZ_CREATEZONE },	/* 31 M_SIGACTS */
+	{ 0,		KMZ_MALLOC },		/* 32 M_VMOBJ */
+	{ 0,		KMZ_MALLOC },		/* 33 M_VMOBJHASH */
+	{ 0,		KMZ_MALLOC },		/* 34 M_VMPMAP */
+	{ 0,		KMZ_MALLOC },		/* 35 M_VMPVENT */
+	{ 0,		KMZ_MALLOC },		/* 36 M_VMPAGER */
+	{ 0,		KMZ_MALLOC },		/* 37 M_VMPGDATA */
+	{ SOS(fileproc),	KMZ_CREATEZONE },	/* 38 M_FILEPROC */
+	{ SOS(filedesc),	KMZ_CREATEZONE },	/* 39 M_FILEDESC */
+	{ SOX(lockf),	KMZ_CREATEZONE },	/* 40 M_LOCKF */
+	{ SOS(proc),	KMZ_CREATEZONE },	/* 41 M_PROC */
+	{ SOS(pstats),	KMZ_CREATEZONE },	/* 42 M_PSTATS */
+	{ 0,		KMZ_MALLOC },		/* 43 M_SEGMENT */
+	{ M_FFSNODE,	KMZ_SHAREZONE },	/* 44 M_LFSNODE */
+#if FFS
+	{ SOS(inode),	KMZ_CREATEZONE },	/* 45 M_FFSNODE */
+#else
+	{ 0,		KMZ_MALLOC },		/* 45 M_FFSNODE */
+#endif
+	{ M_FFSNODE,	KMZ_SHAREZONE },	/* 46 M_MFSNODE */
+	{ 0,		KMZ_MALLOC },		/* 47 M_NQLEASE */
+	{ 0,		KMZ_MALLOC },		/* 48 M_NQMHOST */
+	{ 0,		KMZ_MALLOC },		/* 49 M_NETADDR */
+#if (NFSCLIENT || NFSSERVER)
+	{ SOX(nfsrv_sock),
+			KMZ_CREATEZONE },	/* 50 M_NFSSVC */
+	{ 0,		KMZ_MALLOC },		/* 51 M_NFSUID */
+	{ SOX(nfsrvcache),
+			KMZ_CREATEZONE },	/* 52 M_NFSD */
+#else
+	{ 0,		KMZ_MALLOC },		/* 50 M_NFSSVC */
+	{ 0,		KMZ_MALLOC },		/* 51 M_NFSUID */
+	{ 0,		KMZ_MALLOC },		/* 52 M_NFSD */
+#endif
+	{ SOX(ip_moptions),
+			KMZ_LOOKUPZONE },	/* 53 M_IPMOPTS */
+	{ SOX(in_multi),	KMZ_LOOKUPZONE },	/* 54 M_IPMADDR */
+	{ SOX(ether_multi),
+			KMZ_LOOKUPZONE },	/* 55 M_IFMADDR */
+	{ SOX(mrt),	KMZ_CREATEZONE },	/* 56 M_MRTABLE */
+#if CD9660
+	{ SOX(iso_mnt),	KMZ_LOOKUPZONE },	/* 57 M_ISOFSMNT */
+	{ SOS(iso_node),	KMZ_CREATEZONE },	/* 58 M_ISOFSNODE */
+#else
+	{ 0,		KMZ_MALLOC },		/* 57 M_ISOFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 58 M_ISOFSNODE */
+#endif
+#if (NFSCLIENT || NFSSERVER)
+	{ SOS(nfsrv_descript),
+			KMZ_CREATEZONE },	/* 59 M_NFSRVDESC */
+	{ SOS(nfsdmap),	KMZ_CREATEZONE },	/* 60 M_NFSDIROFF */
+	{ SOS(fhandle),	KMZ_LOOKUPZONE },	/* 61 M_NFSBIGFH */
+#else
+	{ 0,		KMZ_MALLOC },		/* 59 M_NFSRVDESC */
+	{ 0,		KMZ_MALLOC },		/* 60 M_NFSDIROFF */
+	{ 0,		KMZ_MALLOC },		/* 61 M_NFSBIGFH */
+#endif
+	{ 0,		KMZ_MALLOC },		/* 62 M_MSDOSFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 63 M_MSDOSFSFAT */
+	{ 0,		KMZ_MALLOC },		/* 64 M_MSDOSFSNODE */
+	{ SOS(tty),	KMZ_CREATEZONE },	/* 65 M_TTYS */
+	{ 0,		KMZ_MALLOC },		/* 66 M_EXEC */
+	{ 0,		KMZ_MALLOC },		/* 67 M_MISCFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 68 M_MISCFSNODE */
+	{ 0,		KMZ_MALLOC },		/* 69 M_ADOSFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 70 M_ADOSFSNODE */
+	{ 0,		KMZ_MALLOC },		/* 71 M_ANODE */
+	{ SOX(buf),	KMZ_CREATEZONE },	/* 72 M_BUFHDR */
+	{ (NDFILE * OFILESIZE),
+			KMZ_CREATEZONE },	/* 73 M_OFILETABL */
+	{ MCLBYTES,	KMZ_CREATEZONE },	/* 74 M_MCLUST */
+#if HFS
+	{ SOX(hfsmount),	KMZ_LOOKUPZONE },	/* 75 M_HFSMNT */
+	{ SOS(cnode),	KMZ_CREATEZONE },	/* 76 M_HFSNODE */
+	{ SOS(filefork),	KMZ_CREATEZONE },	/* 77 M_HFSFORK */
+#else
+	{ 0,		KMZ_MALLOC },		/* 75 M_HFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 76 M_HFSNODE */
+	{ 0,		KMZ_MALLOC },		/* 77 M_HFSFORK */
+#endif
+	{ 0,		KMZ_MALLOC },		/* 78 M_ZFSMNT */
+	{ 0,		KMZ_MALLOC },		/* 79 M_ZFSNODE */
+	{ 0,		KMZ_MALLOC },		/* 80 M_TEMP */
+	{ 0,		KMZ_MALLOC },		/* 81 M_SECA */
+	{ 0,		KMZ_MALLOC },		/* 82 M_DEVFS */
+	{ 0,		KMZ_MALLOC },		/* 83 M_IPFW */
+	{ 0,		KMZ_MALLOC },		/* 84 M_UDFNODE */
+	{ 0,		KMZ_MALLOC },		/* 85 M_UDFMOUNT */
+	{ 0,		KMZ_MALLOC },		/* 86 M_IP6NDP */
+	{ 0,		KMZ_MALLOC },		/* 87 M_IP6OPT */
+	{ 0,		KMZ_MALLOC },		/* 88 M_IP6MISC */
+	{ 0,		KMZ_MALLOC },		/* 89 M_TSEGQ */
+	{ 0,		KMZ_MALLOC },		/* 90 M_IGMP */
+#if JOURNALING
+	{ SOS(journal), KMZ_CREATEZONE },	/* 91 M_JNL_JNL */
+	{ SOS(transaction), KMZ_CREATEZONE },	/* 92 M_JNL_TR */
+#else
+	{ 0,	 KMZ_MALLOC },		/* 91 M_JNL_JNL */
+	{ 0,	 KMZ_MALLOC },		/* 92 M_JNL_TR */
+#endif
+	{ SOS(specinfo), KMZ_CREATEZONE },	/* 93 M_SPECINFO */
+	{ SOS(kqueue), KMZ_CREATEZONE },	/* 94 M_KQUEUE */
+#if HFS
+	{ SOS(directoryhint), KMZ_CREATEZONE },	/* 95 M_HFSDIRHINT */
+#else
+	{ 0,	KMZ_MALLOC },		/* 95 M_HFSDIRHINT */
+#endif
+	{ SOS(cl_readahead),  KMZ_CREATEZONE },	/* 96 M_CLRDAHEAD */
+	{ SOS(cl_writebehind),KMZ_CREATEZONE },	/* 97 M_CLWRBEHIND */
+	{ SOS(iovec_64),	KMZ_LOOKUPZONE },	/* 98 M_IOV64 */
+	{ SOS(fileglob),	KMZ_CREATEZONE },	/* 99 M_FILEGLOB */
+	{ 0,		KMZ_MALLOC },		/* 100 M_KAUTH */
+	{ 0,		KMZ_MALLOC },		/* 101 M_DUMMYNET */
+	{ SOS(unsafe_fsnode),KMZ_CREATEZONE },	/* 102 M_UNSAFEFS */
+	{ 0,		KMZ_MALLOC },		/* 103 M_MACPIPELABEL */
+	{ 0,		KMZ_MALLOC },		/* 104 M_MACTEMP */
+	{ 0,		KMZ_MALLOC },		/* 105 M_SBUF */
+	{ 0,		KMZ_MALLOC },		/* 106 M_HFS_EXTATTR */
+	{ 0,		KMZ_MALLOC },		/* 107 M_LCTX */
+	{ 0,		KMZ_MALLOC },		/* 108 M_TRAFFIC_MGT */
 #undef	SOS
 #undef	SOX
 };
@@ -240,14 +494,14 @@ kmeminit(void)
 	struct kmzones	*kmz;
 
 	if ((sizeof(kmzones)/sizeof(kmzones[0])) != (sizeof(memname)/sizeof(memname[0]))) {
-		panic("kmeminit: kmzones has %d elements but memname has %d\n",
+		panic("kmeminit: kmzones has %lu elements but memname has %lu\n",
 			  (sizeof(kmzones)/sizeof(kmzones[0])), (sizeof(memname)/sizeof(memname[0])));
 	}
 
 	kmz = kmzones;
 	while (kmz < &kmzones[M_LAST]) {
 /* XXX */
-		if (kmz->kz_elemsize == -1)
+		if (kmz->kz_elemsize == (size_t)(-1))
 			;
 		else
 /* XXX */
@@ -265,7 +519,7 @@ kmeminit(void)
 	kmz = kmzones;
 	while (kmz < &kmzones[M_LAST]) {
 /* XXX */
-		if (kmz->kz_elemsize == -1)
+		if (kmz->kz_elemsize == (size_t)(-1))
 			;
 		else
 /* XXX */
@@ -291,8 +545,6 @@ struct _mhead {
 	char	dat[0];
 };
 
-#define ZEROSIZETOKEN (void *)0xFADEDFAD
-
 void *
 _MALLOC(
 	size_t		size,
@@ -305,12 +557,8 @@ _MALLOC(
 	if (type >= M_LAST)
 		panic("_malloc TYPE");
 
-	/*
-	 * On zero request we do not return zero as that
-	 * could be mistaken for ENOMEM.
-	 */
 	if (size == 0)
-		return (ZEROSIZETOKEN);
+		return (NULL);
 
 	if (flags & M_NOWAIT) {
 		mem = (void *)kalloc_noblock(memsize);
@@ -338,8 +586,6 @@ _FREE(
 	if (type >= M_LAST)
 		panic("_free TYPE");
 
-	if (addr == (void *)ZEROSIZETOKEN)
-		return;
 	if (!addr)
 		return; /* correct (convenient bsd kernel legacy) */
 
@@ -364,7 +610,7 @@ _MALLOC_ZONE(
 		panic("_malloc_zone ZONE: type = %d", type);
 
 /* XXX */
-	if (kmz->kz_elemsize == -1)
+	if (kmz->kz_elemsize == (size_t)(-1))
 		panic("_malloc_zone XXX");
 /* XXX */
 	if (size == kmz->kz_elemsize)
@@ -399,7 +645,7 @@ _FREE_ZONE(
 		panic("free_zone ZONE");
 
 /* XXX */
-	if (kmz->kz_elemsize == -1)
+	if (kmz->kz_elemsize == (size_t)(-1))
 		panic("FREE_SIZE XXX");
 /* XXX */
 	if (size == kmz->kz_elemsize)
diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c
new file mode 100644
index 000000000..78380edcf
--- /dev/null
+++ b/bsd/kern/kern_memorystatus.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ *
+ */
+/*-
+ * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/kern_event.h>
+#include <sys/kern_memorystatus.h>
+
+#include <kern/sched_prim.h>
+#include <kern/task.h>
+#include <kern/thread.h>
+#include <libkern/libkern.h>
+#include <sys/sysctl.h>
+
+static void kern_memorystatus_thread(void);
+
+int kern_memorystatus_wakeup = 0;
+int kern_memorystatus_pause = 0;
+int kern_memorystatus_level = 0;
+int kern_memorystatus_last_level = 0;
+unsigned int kern_memorystatus_kev_failure_count = 0;
+
+SYSCTL_INT(_kern, OID_AUTO, memorystatus_level, CTLFLAG_RD, &kern_memorystatus_level, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, memorystatus_kev_failure_count, CTLFLAG_RD, &kern_memorystatus_kev_failure_count, 0, "");
+
+__private_extern__ void
+kern_memorystatus_init(void)
+{
+	(void)kernel_thread(kernel_task, kern_memorystatus_thread);
+}
+
+static void
+kern_memorystatus_thread(void)
+{
+	struct kev_msg ev_msg;
+	int ret;
+
+	while(1) {
+		
+		kern_memorystatus_last_level = kern_memorystatus_level;
+
+		ev_msg.vendor_code    = KEV_VENDOR_APPLE;
+		ev_msg.kev_class      = KEV_SYSTEM_CLASS;
+		ev_msg.kev_subclass   = KEV_MEMORYSTATUS_SUBCLASS;
+
+		/* pass the memory status level in the event code (as percent used) */
+		ev_msg.event_code     = 100 - kern_memorystatus_last_level;
+
+		ev_msg.dv[0].data_length = 0;
+
+		ret = kev_post_msg(&ev_msg);
+		if (ret) {
+			kern_memorystatus_kev_failure_count++;
+			printf("%s: kev_post_msg() failed, err %d\n", __func__, ret);
+		}
+
+		assert_wait_timeout((event_t)&kern_memorystatus_pause, THREAD_UNINT, 1, 250*1000*NSEC_PER_USEC);
+		(void)thread_block(THREAD_CONTINUE_NULL);
+
+		if (kern_memorystatus_level >= kern_memorystatus_last_level + 5 ||
+		    kern_memorystatus_level <= kern_memorystatus_last_level - 5)
+			continue;
+
+		assert_wait(&kern_memorystatus_wakeup, THREAD_UNINT);
+		(void)thread_block((thread_continue_t)kern_memorystatus_thread);
+	}
+}
diff --git a/bsd/kern/kern_mib.c b/bsd/kern/kern_mib.c
index adbd53608..e5a10c5b2 100644
--- a/bsd/kern/kern_mib.c
+++ b/bsd/kern/kern_mib.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -105,33 +111,42 @@ extern vm_map_t bsd_pageable_map;
 #include <machine/machine_routines.h>
 #include <machine/cpu_capabilities.h>
 
+#include <mach/mach_host.h>		/* for host_info() */
+
 #ifdef __i386__
 #include <i386/cpuid.h>		/* for cpuid_info() */
 #endif
 
+
+
 #ifndef MAX
 #define MAX(a,b) (a >= b ? a : b)
 #endif
 
-static int	cputype, cpusubtype, cputhreadtype, cpufamily, cacheconfig[10];;
+/* XXX This should be in a BSD accessible Mach header, but isn't. */
+extern unsigned int vm_page_wire_count;
+
+static int	cputype, cpusubtype, cputhreadtype, cpufamily, cpu64bit;
+static uint64_t	cacheconfig[10], cachesize[10];
+static int	packages;
 
-SYSCTL_NODE(, 0,	  sysctl, CTLFLAG_RW, 0,
+SYSCTL_NODE(, 0,	  sysctl, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"Sysctl internal magic");
-SYSCTL_NODE(, CTL_KERN,	  kern,   CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_KERN,	  kern,   CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"High kernel, proc, limits &c");
-SYSCTL_NODE(, CTL_VM,	  vm,     CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_VM,	  vm,     CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"Virtual memory");
-SYSCTL_NODE(, CTL_VFS,	  vfs,     CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_VFS,	  vfs,     CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"File system");
-SYSCTL_NODE(, CTL_NET,	  net,    CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_NET,	  net,    CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"Network, (see socket.h)");
-SYSCTL_NODE(, CTL_DEBUG,  debug,  CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_DEBUG,  debug,  CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"Debugging");
-SYSCTL_NODE(, CTL_HW,	  hw,     CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_HW,	  hw,     CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"hardware");
-SYSCTL_NODE(, CTL_MACHDEP, machdep, CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_MACHDEP, machdep, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"machine dependent");
-SYSCTL_NODE(, CTL_USER,	  user,   CTLFLAG_RW, 0,
+SYSCTL_NODE(, CTL_USER,	  user,   CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"user-level");
 
 #define SYSCTL_RETURN(r, x)	SYSCTL_OUT(r, &x, sizeof(x))
@@ -155,17 +170,17 @@ SYSCTL_NODE(, CTL_USER,	  user,   CTLFLAG_RW, 0,
  * gather some of that here.
  */
 static int
-sysctl_hw_generic SYSCTL_HANDLER_ARGS
+sysctl_hw_generic(__unused struct sysctl_oid *oidp, __unused void *arg1,
+	int arg2, struct sysctl_req *req)
 {
 	char dummy[65];
 	int  epochTemp;
-	extern int vm_page_wire_count;
 	ml_cpu_info_t cpu_info;
 	int val, doquad;
 	long long qval;
 	host_basic_info_data_t hinfo;
 	kern_return_t kret;
-	int count = HOST_BASIC_INFO_COUNT;
+	mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 
 	/*
 	 * Test and mask off the 'return quad' flag.
@@ -177,7 +192,7 @@ sysctl_hw_generic SYSCTL_HANDLER_ARGS
 	ml_cpu_get_info(&cpu_info);
 
 #define BSD_HOST 1
-	kret = host_info(BSD_HOST, HOST_BASIC_INFO, &hinfo, &count);
+	kret = host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
 
 	/*
 	 * Handle various OIDs.
@@ -298,6 +313,23 @@ sysctl_hw_generic SYSCTL_HANDLER_ARGS
 	return(SYSCTL_RETURN(req, val));
 }
 
+/* hw.pagesize and hw.tbfrequency are expected as 64 bit values */
+static int
+sysctl_pagesize
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	long long l = page_size;
+	return sysctl_io_number(req, l, sizeof(l), NULL, NULL);
+}
+
+static int
+sysctl_tbfrequency
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	long long l = gPEClockFrequencyInfo.timebase_frequency_hz;
+	return sysctl_io_number(req, l, sizeof(l), NULL, NULL);
+}
+
 /*
  * hw.* MIB variables.
  */
@@ -310,9 +342,11 @@ SYSCTL_PROC    (_hw, OID_AUTO, logicalcpu_max, CTLTYPE_INT | CTLFLAG_RD | CTLFLA
 SYSCTL_INT     (_hw, HW_BYTEORDER, byteorder, CTLFLAG_RD | CTLFLAG_KERN, NULL, BYTE_ORDER, "");
 SYSCTL_INT     (_hw, OID_AUTO, cputype, CTLFLAG_RD | CTLFLAG_KERN, &cputype, 0, "");
 SYSCTL_INT     (_hw, OID_AUTO, cpusubtype, CTLFLAG_RD | CTLFLAG_KERN, &cpusubtype, 0, "");
+SYSCTL_INT     (_hw, OID_AUTO, cpu64bit_capable, CTLFLAG_RD | CTLFLAG_KERN, &cpu64bit, 0, "");
 SYSCTL_INT     (_hw, OID_AUTO, cpufamily, CTLFLAG_RD | CTLFLAG_KERN, &cpufamily, 0, "");
-SYSCTL_OPAQUE  (_hw, OID_AUTO, cacheconfig, CTLFLAG_RD, &cacheconfig, sizeof(cacheconfig), "I", "");
-SYSCTL_INT2QUAD(_hw, OID_AUTO, pagesize, CTLFLAG_RD | CTLFLAG_KERN, &page_size, "");
+SYSCTL_OPAQUE  (_hw, OID_AUTO, cacheconfig, CTLFLAG_RD, &cacheconfig, sizeof(cacheconfig), "Q", "");
+SYSCTL_OPAQUE  (_hw, OID_AUTO, cachesize, CTLFLAG_RD, &cachesize, sizeof(cachesize), "Q", "");
+SYSCTL_PROC	   (_hw, OID_AUTO, pagesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN, 0, 0, sysctl_pagesize, "Q", "");
 SYSCTL_QUAD    (_hw, OID_AUTO, busfrequency, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.bus_frequency_hz, "");
 SYSCTL_QUAD    (_hw, OID_AUTO, busfrequency_min, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.bus_frequency_min_hz, "");
 SYSCTL_QUAD    (_hw, OID_AUTO, busfrequency_max, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.bus_frequency_max_hz, "");
@@ -324,8 +358,9 @@ SYSCTL_PROC    (_hw, OID_AUTO, l1icachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG
 SYSCTL_PROC    (_hw, OID_AUTO, l1dcachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN, 0, HW_L1DCACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", "");
 SYSCTL_PROC    (_hw, OID_AUTO, l2cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN, 0, HW_L2CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", "");
 SYSCTL_PROC    (_hw, OID_AUTO, l3cachesize, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN, 0, HW_L3CACHESIZE | CTLHW_RETQUAD, sysctl_hw_generic, "Q", "");
-SYSCTL_INT2QUAD(_hw, OID_AUTO, tbfrequency, CTLFLAG_RD | CTLFLAG_KERN, &gPEClockFrequencyInfo.timebase_frequency_hz, "");
+SYSCTL_PROC(_hw, OID_AUTO, tbfrequency, CTLTYPE_QUAD | CTLFLAG_RD | CTLFLAG_KERN, 0, 0, sysctl_tbfrequency, "Q", "");
 SYSCTL_QUAD    (_hw, HW_MEMSIZE, memsize, CTLFLAG_RD | CTLFLAG_KERN, &max_mem, "");
+SYSCTL_INT     (_hw, OID_AUTO, packages, CTLFLAG_RD | CTLFLAG_KERN, &packages, 0, "");
 
 /*
  * Optional features can register nodes below hw.optional.
@@ -335,7 +370,7 @@ SYSCTL_QUAD    (_hw, HW_MEMSIZE, memsize, CTLFLAG_RD | CTLFLAG_KERN, &max_mem, "
  * 0.  If the feature is present and its use is advised, the node should 
  * return 1.
  */
-SYSCTL_NODE(_hw, OID_AUTO, optional, CTLFLAG_RW, NULL, "optional features");
+SYSCTL_NODE(_hw, OID_AUTO, optional, CTLFLAG_RW|CTLFLAG_LOCKED, NULL, "optional features");
 
 SYSCTL_INT(_hw_optional, OID_AUTO, floatingpoint, CTLFLAG_RD | CTLFLAG_KERN, 0, 1, "");	/* always set */
 
@@ -368,12 +403,53 @@ SYSCTL_PROC(_hw, HW_EPOCH,        epoch, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASK
 SYSCTL_PROC(_hw, HW_VECTORUNIT,   vectorunit, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, 0, HW_VECTORUNIT, sysctl_hw_generic, "I", "");
 SYSCTL_PROC(_hw, HW_L2SETTINGS,   l2settings, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, 0, HW_L2SETTINGS, sysctl_hw_generic, "I", "");
 SYSCTL_PROC(_hw, HW_L3SETTINGS,   l3settings, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MASKED, 0, HW_L3SETTINGS, sysctl_hw_generic, "I", "");
+SYSCTL_INT (_hw, OID_AUTO, cputhreadtype, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &cputhreadtype, 0, "");
+
+#ifdef __ppc__
+int altivec_flag = -1;
+int graphicsops_flag = -1;
+int x64bitops_flag = -1;
+int fsqrt_flag = -1;
+int stfiwx_flag = -1;
+int dcba_flag = -1;
+int datastreams_flag = -1;
+int dcbtstreams_flag = -1;
+
+SYSCTL_INT(_hw_optional, OID_AUTO, altivec, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &altivec_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, graphicsops, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &graphicsops_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, 64bitops, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &x64bitops_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, fsqrt, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &fsqrt_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, stfiwx, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &stfiwx_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, dcba, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &dcba_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, datastreams, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &datastreams_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, dcbtstreams, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &dcbtstreams_flag, 0, "");
+#elif defined (__i386__)
+int mmx_flag = -1;
+int sse_flag = -1;
+int sse2_flag = -1;
+int sse3_flag = -1;
+int sse4_1_flag = -1;
+int sse4_2_flag = -1;
+int x86_64_flag = -1;
+int supplementalsse3_flag = -1;
+
+SYSCTL_INT(_hw_optional, OID_AUTO, mmx, CTLFLAG_RD | CTLFLAG_KERN, &mmx_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, sse, CTLFLAG_RD | CTLFLAG_KERN, &sse_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, sse2, CTLFLAG_RD | CTLFLAG_KERN, &sse2_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, sse3, CTLFLAG_RD | CTLFLAG_KERN, &sse3_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, supplementalsse3, CTLFLAG_RD | CTLFLAG_KERN, &supplementalsse3_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, sse4_1, CTLFLAG_RD | CTLFLAG_KERN, &sse4_1_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, sse4_2, CTLFLAG_RD | CTLFLAG_KERN, &sse4_2_flag, 0, "");
+SYSCTL_INT(_hw_optional, OID_AUTO, x86_64, CTLFLAG_RD | CTLFLAG_KERN, &x86_64_flag, 0, "");
+#endif /* __ppc__ */
 
 /*
  * Debugging interface to the CPU power management code.
  */
-static void pmsSysctl SYSCTL_HANDLER_ARGS {
-
+static int
+pmsSysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
+	  __unused int arg2, struct sysctl_req *req)
+{
 	pmsctl_t	ctl;
 	int		error;
 	boolean_t	intr;
@@ -382,7 +458,7 @@ static void pmsSysctl SYSCTL_HANDLER_ARGS {
 		return(error);
 
 	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
-	error = pmsControl(ctl.request, (user_addr_t)ctl.reqaddr, ctl.reqsize);
+	error = pmsControl(ctl.request, (user_addr_t)(unsigned long)ctl.reqaddr, ctl.reqsize);
 	(void)ml_set_interrupts_enabled(intr);			/* Restore interruptions */
 
 	return(error);
@@ -404,6 +480,13 @@ sysctl_mib_init(void)
 	cputype = cpu_type();
 	cpusubtype = cpu_subtype();
 	cputhreadtype = cpu_threadtype();
+#if defined(__ppc__)
+    cpu64bit = (_cpu_capabilities & k64Bit) == k64Bit;
+#elif defined(__i386__)
+    cpu64bit = (_get_cpu_capabilities() & k64Bit) == k64Bit;
+#elif defined(__arm__)
+    cpu64bit = 0; // FIXME make this not hard-coded
+#endif
 
 	/*
 	 * Populate the optional portion of the hw.* MIB.
@@ -414,89 +497,57 @@ sysctl_mib_init(void)
 	 */
 
 	if (cputhreadtype != CPU_THREADTYPE_NONE) {
-		static SYSCTL_INT(_hw, OID_AUTO, cputhreadtype, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &cputhreadtype, 0, "");
 		sysctl_register_oid(&sysctl__hw_cputhreadtype);
 	}
 
 #ifdef __ppc__
-	{
-		static int altivec_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, altivec, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &altivec_flag, 0, "");
-
-		if (_cpu_capabilities & kHasAltivec) {
-			altivec_flag = 1;
-			sysctl_register_oid(&sysctl__hw_optional_altivec);
-		}
-	}
-	{
-		static int graphicsops_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, graphicsops, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &graphicsops_flag, 0, "");
-
-		if (_cpu_capabilities & kHasGraphicsOps) {
-			graphicsops_flag = 1;
-			sysctl_register_oid(&sysctl__hw_optional_graphicsops);
-		}		
-	}
-	{
-		static int x64bitops_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, 64bitops, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &x64bitops_flag, 0, "");
-
-		if (_cpu_capabilities & k64Bit) {
-			x64bitops_flag = 1;
-			sysctl_register_oid(&sysctl__hw_optional_64bitops);
-		}		
-	}
-	{
-		static int fsqrt_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, fsqrt, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &fsqrt_flag, 0, "");
-
-		if (_cpu_capabilities & kHasFsqrt) {
-			fsqrt_flag = 1;
-			sysctl_register_oid(&sysctl__hw_optional_fsqrt);
-		}		
-	}
-	{
-		static int stfiwx_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, stfiwx, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &stfiwx_flag, 0, "");
+/*
+ * The convention for these is as follows:
+ * If the sysctl does not exist, the functionality is not present in the CPU.
+ * If the sysctl exists, it will not crash, and should otherwise function
+ *   corectly.
+ * If the sysctl exists and returns 0, we advise against using this feature.
+ * If the sysctl exists and returns 1, we advise it's use.
+ */
 
-		if (_cpu_capabilities & kHasStfiwx) {
-			stfiwx_flag = 1;
-			sysctl_register_oid(&sysctl__hw_optional_stfiwx);
-		}		
-	}
-	{
-		static int dcba_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, dcba, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &dcba_flag, 0, "");
-
-		if (_cpu_capabilities & kDcbaAvailable)
-			dcba_flag = 0;
-		if (_cpu_capabilities & kDcbaRecommended)
-			dcba_flag = 1;
-		if (dcba_flag >= 0)
-			sysctl_register_oid(&sysctl__hw_optional_dcba);
-	}
-	{
-		static int datastreams_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, datastreams, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &datastreams_flag, 0, "");
-
-		if (_cpu_capabilities & kDataStreamsAvailable)
-			datastreams_flag = 0;
-		if (_cpu_capabilities & kDataStreamsRecommended)
-			datastreams_flag = 1;
-		if (datastreams_flag >= 0)
-			sysctl_register_oid(&sysctl__hw_optional_datastreams);
-	}
-	{
-		static int dcbtstreams_flag = -1;
-		static SYSCTL_INT(_hw_optional, OID_AUTO, dcbtstreams, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, &dcbtstreams_flag, 0, "");
-
-		if (_cpu_capabilities & kDcbtStreamsAvailable)
-			dcbtstreams_flag = 0;
-		if (_cpu_capabilities & kDcbtStreamsRecommended)
-			dcbtstreams_flag = 1;
-		if (dcbtstreams_flag >= 0)
-			sysctl_register_oid(&sysctl__hw_optional_dcbtstreams);
+	if (_cpu_capabilities & kHasAltivec) {
+		altivec_flag = 1;
+		sysctl_register_oid(&sysctl__hw_optional_altivec);
 	}
+	if (_cpu_capabilities & kHasGraphicsOps) {
+		graphicsops_flag = 1;
+		sysctl_register_oid(&sysctl__hw_optional_graphicsops);
+	}		
+	if (_cpu_capabilities & k64Bit) {
+		x64bitops_flag = 1;
+		sysctl_register_oid(&sysctl__hw_optional_64bitops);
+	}		
+	if (_cpu_capabilities & kHasFsqrt) {
+		fsqrt_flag = 1;
+		sysctl_register_oid(&sysctl__hw_optional_fsqrt);
+	}		
+	if (_cpu_capabilities & kHasStfiwx) {
+		stfiwx_flag = 1;
+		sysctl_register_oid(&sysctl__hw_optional_stfiwx);
+	}		
+	if (_cpu_capabilities & kDcbaAvailable)
+		dcba_flag = 0;
+	if (_cpu_capabilities & kDcbaRecommended)
+		dcba_flag = 1;
+	if (dcba_flag >= 0)
+		sysctl_register_oid(&sysctl__hw_optional_dcba);
+	if (_cpu_capabilities & kDataStreamsAvailable)
+		datastreams_flag = 0;
+	if (_cpu_capabilities & kDataStreamsRecommended)
+		datastreams_flag = 1;
+	if (datastreams_flag >= 0)
+		sysctl_register_oid(&sysctl__hw_optional_datastreams);
+	if (_cpu_capabilities & kDcbtStreamsAvailable)
+		dcbtstreams_flag = 0;
+	if (_cpu_capabilities & kDcbtStreamsRecommended)
+		dcbtstreams_flag = 1;
+	if (dcbtstreams_flag >= 0)
+		sysctl_register_oid(&sysctl__hw_optional_dcbtstreams);
 
 	/* hw.cpufamily */
 	switch (cpusubtype) {
@@ -514,45 +565,67 @@ sysctl_mib_init(void)
 		cpufamily = CPUFAMILY_UNKNOWN;
 	}
 
-	/* hw.cacheconfig */
-	cacheconfig[0] = 0; /* XXX not supported on PowerPC */
+	ml_cpu_info_t cpu_info;
+	ml_cpu_get_info(&cpu_info);
 
-#elif defined (__i386__)
+	host_basic_info_data_t hinfo;
+	mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+	kern_return_t kret = host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+	if(kret != KERN_SUCCESS)
+	{
+		hinfo.max_cpus = 1;
+	}
 
+	/* hw.cachesize */
+	cachesize[0] = max_mem;
+	cachesize[1] = cpu_info.l1_dcache_size;
+	cachesize[2] = cpu_info.l2_settings ? cpu_info.l2_cache_size : 0;
+	cachesize[3] = cpu_info.l3_settings ? cpu_info.l3_cache_size : 0;
+	cachesize[4] = 0;
+	
+	/* hw.cacheconfig */
+	cacheconfig[0] = hinfo.max_cpus;
+	cacheconfig[1] = 1;
+	cacheconfig[2] = cachesize[2] ? 1 : 0;
+	cacheconfig[3] = cachesize[3] ? 1 : 0;
+	cacheconfig[4] = 0;
 
-#define DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, flags) { \
-	static int named##_flag = -1;	\
-	static SYSCTL_INT(_hw_optional, OID_AUTO, named, CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | flags, &named##_flag, 0, "");	\
-	named##_flag = ((_get_cpu_capabilities() & BITS) == BITS)? 1 : 0;	\
-	sysctl_register_oid(&sysctl__hw_optional_##named);	\
-	}
+	/* hw.packages */
+	if (cpusubtype == CPU_SUBTYPE_POWERPC_970 && 
+	    cpu_info.l2_cache_size == 1 * 1024 * 1024)
+		/* The signature of the dual-core G5 */
+		packages = hinfo.max_cpus / 2;
+	else
+		packages = hinfo.max_cpus;
 
-#define DECLARE_X86_HW_OPTIONAL(named, BITS) \
-	DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, 0)
-
-#define DECLARE_X86_HW_OPTIONAL_MASKED(named, BITS) \
-	DECLARE_X86_HW_OPTIONAL_FLAGS(named, BITS, CTLFLAG_MASKED)
-
-	DECLARE_X86_HW_OPTIONAL(mmx, kHasMMX);
-	DECLARE_X86_HW_OPTIONAL(sse, kHasSSE);
-	DECLARE_X86_HW_OPTIONAL(sse2, kHasSSE2);
-	DECLARE_X86_HW_OPTIONAL(sse3, kHasSSE3);
-	if (_get_cpu_capabilities() & k64Bit)
-		DECLARE_X86_HW_OPTIONAL(x86_64, k64Bit);
-	if (_get_cpu_capabilities() & kHasSupplementalSSE3) {
-		DECLARE_X86_HW_OPTIONAL_MASKED(mni, kHasSupplementalSSE3); /* XXX */
-		DECLARE_X86_HW_OPTIONAL(supplementalsse3, kHasSupplementalSSE3);
-	}
+#elif defined (__i386__)
+	mmx_flag = ((_get_cpu_capabilities() & kHasMMX) == kHasMMX)? 1 : 0;
+	sse_flag = ((_get_cpu_capabilities() & kHasSSE) == kHasSSE)? 1 : 0;
+	sse2_flag = ((_get_cpu_capabilities() & kHasSSE2) == kHasSSE2)? 1 : 0;
+	sse3_flag = ((_get_cpu_capabilities() & kHasSSE3) == kHasSSE3)? 1 : 0;
+	supplementalsse3_flag = ((_get_cpu_capabilities() & kHasSupplementalSSE3) == kHasSupplementalSSE3)? 1 : 0;
+	sse4_1_flag = ((_get_cpu_capabilities() & kHasSSE4_1) == kHasSSE4_1)? 1 : 0;
+	sse4_2_flag = ((_get_cpu_capabilities() & kHasSSE4_2) == kHasSSE4_2)? 1 : 0;
+	x86_64_flag = ((_get_cpu_capabilities() & k64Bit) == k64Bit)? 1 : 0;
 
 	/* hw.cpufamily */
 	switch (cpuid_info()->cpuid_family) {
 	case 6:
 		switch (cpuid_info()->cpuid_model) {
+		case 13:
+			cpufamily = CPUFAMILY_INTEL_6_13;
+			break;
 		case 14:
 			cpufamily = CPUFAMILY_INTEL_6_14; /* Core Solo/Duo */
 			break;
 		case 15:
-			cpufamily = CPUFAMILY_INTEL_6_15;
+			cpufamily = CPUFAMILY_INTEL_6_15; /* Core 2 */
+			break;
+		case 23:
+			cpufamily = CPUFAMILY_INTEL_6_23;
+			break;
+		case 26:
+			cpufamily = CPUFAMILY_INTEL_6_26;
 			break;
 		default:
 			cpufamily = CPUFAMILY_UNKNOWN;
@@ -562,15 +635,49 @@ sysctl_mib_init(void)
 		cpufamily = CPUFAMILY_UNKNOWN;
 	}
 	/* hw.cacheconfig */
-	cacheconfig[0] = cpuid_info()->cpuid_cores_per_package;
-	cacheconfig[1] = MAX(cpuid_info()->cache_sharing[L1I], cpuid_info()->cache_sharing[L1D]);
-	cacheconfig[2] = cpuid_info()->cache_sharing[L2U];
-	cacheconfig[3] = cpuid_info()->cache_sharing[L3U];
+	cacheconfig[0] = ml_cpu_cache_sharing(0);
+	cacheconfig[1] = ml_cpu_cache_sharing(1);
+	cacheconfig[2] = ml_cpu_cache_sharing(2);
+	cacheconfig[3] = ml_cpu_cache_sharing(3);
 	cacheconfig[4] = 0;
 
-#else /* end __i386 */
+	/* hw.cachesize */
+	cachesize[0] = ml_cpu_cache_size(0);
+	cachesize[1] = ml_cpu_cache_size(1);
+	cachesize[2] = ml_cpu_cache_size(2);
+	cachesize[3] = ml_cpu_cache_size(3);
+	cachesize[4] = 0;
+
+	/* hw.packages */
+	packages = ml_cpu_cache_sharing(0) /
+			cpuid_info()->cpuid_cores_per_package;
+	
+#elif defined(__arm__) /* end __i386 */
+	switch (cpuid_info()->arm_info.arm_part) {
+		case CPU_PART_1136JFS:
+		case CPU_PART_1176JZFS:
+			cpufamily = CPUFAMILY_ARM_11;
+			break;
+		case CPU_PART_920T:
+			cpufamily = CPUFAMILY_ARM_9;
+			break;
+		default:
+			cpufamily = CPUFAMILY_UNKNOWN;
+	}
+
+	cacheconfig[0] = cache_info()->c_unified;
+	cacheconfig[1] = cache_info()->c_isize;
+	cacheconfig[2] = cache_info()->c_dsize;
+	cacheconfig[3] = cache_info()->c_type;
+	cacheconfig[4] = cache_info()->c_linesz;
+	cacheconfig[5] = cache_info()->c_assoc;
+	cacheconfig[6] = 0;
+
+	packages = 1;
+#else /* end __arm__ */
 # warning we do not support this platform yet
 #endif /* __ppc__ */
 
 
 }
+
diff --git a/bsd/kern/kern_mman.c b/bsd/kern/kern_mman.c
index 27d9cc1f1..499d6f61b 100644
--- a/bsd/kern/kern_mman.c
+++ b/bsd/kern/kern_mman.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
+ * Copyright (c) 2007 Apple Inc. All Rights Reserved.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 University of Utah.
@@ -60,6 +66,12 @@
  *
  *	@(#)vm_mmap.c	8.10 (Berkeley) 2/19/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * Mapped file (mmap) interface to VM
@@ -81,8 +93,12 @@
 #include <sys/conf.h>
 #include <sys/stat.h>
 #include <sys/ubc.h>
+#include <sys/ubc_internal.h>
 #include <sys/sysproto.h>
 
+#include <sys/syscall.h>
+#include <sys/kdebug.h>
+
 #include <bsm/audit_kernel.h>
 #include <bsm/audit_kevents.h>
 
@@ -103,34 +119,42 @@
 #include <vm/vm_kern.h>
 #include <vm/vm_pager.h>
 
+struct osmmap_args {
+		caddr_t	addr;
+		int	len;
+		int	prot;
+		int	share;
+		int	fd;
+		long	pos;
+};
+
+/* XXX the following function should probably be static */
+kern_return_t map_fd_funneled(int, vm_object_offset_t, vm_offset_t *,
+				boolean_t, vm_size_t);
+
+/* XXX the following two functions aren't used anywhere */
+int osmmap(proc_t , struct osmmap_args *, register_t *);
+int mremap(void);
+
 int
-sbrk(__unused struct proc *p, __unused struct sbrk_args *uap, __unused register_t *retval)
+sbrk(__unused proc_t p, __unused struct sbrk_args *uap, __unused register_t *retval)
 {
 	/* Not yet implemented */
 	return (ENOTSUP);
 }
 
 int
-sstk(__unused struct proc *p, __unused struct sstk_args *uap, __unused register_t *retval)
+sstk(__unused proc_t p, __unused struct sstk_args *uap, __unused register_t *retval)
 {
 	/* Not yet implemented */
 	return (ENOTSUP);
 }
 
 
-struct osmmap_args {
-		caddr_t	addr;
-		int	len;
-		int	prot;
-		int	share;
-		int	fd;
-		long	pos;
-};
-
 int
 osmmap(
-	struct proc *curp,
-	register struct osmmap_args *uap,
+	proc_t curp,
+	struct osmmap_args *uap,
 	register_t *retval)
 {
 	struct mmap_args newargs;
@@ -153,8 +177,13 @@ osmmap(
 }
 
 
+/*
+ * XXX Internally, we use VM_PROT_* somewhat interchangeably, but the correct
+ * XXX usage is PROT_* from an interface perspective.  Thus the values of
+ * XXX VM_PROT_* and PROT_* need to correspond.
+ */
 int
-mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
+mmap(proc_t p, struct mmap_args *uap, user_addr_t *retval)
 {
 	/*
 	 *	Map in special device (must be SHARED) or file
@@ -162,7 +191,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 	struct fileproc *fp;
 	register struct		vnode *vp;
 	int			flags;
-	int			prot;
+	int			prot, file_prot;
 	int			err=0;
 	vm_map_t		user_map;
 	kern_return_t		result;
@@ -170,7 +199,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 	mach_vm_size_t		user_size;
 	vm_object_offset_t	pageoff;
 	vm_object_offset_t	file_pos;
-	int			alloc_flags;
+	int			alloc_flags=0;
 	boolean_t		docow;
 	vm_prot_t		maxprot;
 	void 			*handle;
@@ -188,6 +217,17 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 	AUDIT_ARG(fd, uap->fd);
 
 	prot = (uap->prot & VM_PROT_ALL);
+#if 3777787
+	/*
+	 * Since the hardware currently does not support writing without
+	 * read-before-write, or execution-without-read, if the request is
+	 * for write or execute access, we must imply read access as well;
+	 * otherwise programs expecting this to work will fail to operate.
+	 */
+	if (prot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+		prot |= VM_PROT_READ;
+#endif	/* radar 3777787 */
+
 	flags = uap->flags;
 	vp = NULLVP;
 
@@ -200,8 +240,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 
 
 	/* make sure mapping fits into numeric range etc */
-	if ((file_pos + user_size > (vm_object_offset_t)-PAGE_SIZE_64) ||
-	    ((flags & MAP_ANON) && fd != -1))
+	if (file_pos + user_size > (vm_object_offset_t)-PAGE_SIZE_64)
 		return (EINVAL);
 
 	/*
@@ -229,7 +268,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 		 */
 		user_addr -= pageoff;
 		if (user_addr & PAGE_MASK)
-			return (EINVAL);
+		return (EINVAL);
 	}
 #ifdef notyet
 	/* DO not have apis to get this info, need to wait till then*/
@@ -246,18 +285,34 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 
 #endif
 
+	alloc_flags = 0;
 
 	if (flags & MAP_ANON) {
 		/*
-		 * Mapping blank space is trivial.
+		 * Mapping blank space is trivial.  Use positive fds as the alias
+		 * value for memory tracking. 
 		 */
+		if (fd != -1) {
+			/*
+			 * Use "fd" to pass (some) Mach VM allocation flags,
+			 * (see the VM_FLAGS_* definitions).
+			 */
+			alloc_flags = fd & (VM_FLAGS_ALIAS_MASK |
+					    VM_FLAGS_PURGABLE);
+			if (alloc_flags != fd) {
+				/* reject if there are any extra flags */
+				return EINVAL;
+			}
+		}
+			
 		handle = NULL;
 		maxprot = VM_PROT_ALL;
 		file_pos = 0;
 		mapanon = 1;
 	} else {
 		struct vnode_attr va;
-		struct vfs_context context;
+		vfs_context_t ctx = vfs_context_current();
+
 		/*
 		 * Mapping file, get fp for validation. Obtain vnode and make
 		 * sure it is of appropriate type.
@@ -293,16 +348,16 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 
 		AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 		
-		/* conformance change - mmap needs to update access time for mapped
-		 * files
+		/*
+		 * POSIX: mmap needs to update access time for mapped files
 		 */
-		VATTR_INIT(&va);
-		nanotime(&va.va_access_time);
-		VATTR_SET_ACTIVE(&va, va_access_time);
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
-		vnode_setattr(vp, &va, &context);
-
+		if ((vnode_vfsvisflags(vp) & MNT_NOATIME) == 0) {
+			VATTR_INIT(&va);
+			nanotime(&va.va_access_time);
+			VATTR_SET_ACTIVE(&va, va_access_time);
+			vnode_setattr(vp, &va, ctx);
+		}
+		
 		/*
 		 * XXX hack to handle use of /dev/zero to map anon memory (ala
 		 * SunOS).
@@ -345,7 +400,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
  					 * Note that we already made this check when granting FWRITE
  					 * against the file, so it seems redundant here.
  					 */
- 					error = vnode_authorize(vp, NULL, KAUTH_VNODE_CHECKIMMUTABLE, &context);
+ 					error = vnode_authorize(vp, NULL, KAUTH_VNODE_CHECKIMMUTABLE, ctx);
  
  					/* if not granted for any reason, but we wanted it, bad */
  					if ((prot & PROT_WRITE) && (error != 0)) {
@@ -366,6 +421,14 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 				maxprot |= VM_PROT_WRITE;
 
 			handle = (void *)vp;
+#if CONFIG_MACF
+			error = mac_file_check_mmap(vfs_context_ucred(ctx),
+			    fp->f_fglob, prot, flags, &maxprot);
+			if (error) {
+				(void)vnode_put(vp);
+				goto bad;
+			}
+#endif /* MAC */
 		}
 	}
 
@@ -392,7 +455,7 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 	user_map = current_map();
 
 	if ((flags & MAP_FIXED) == 0) {
-		alloc_flags = VM_FLAGS_ANYWHERE;
+		alloc_flags |= VM_FLAGS_ANYWHERE;
 		user_addr = mach_vm_round_page(user_addr);
 	} else {
 		if (user_addr != mach_vm_trunc_page(user_addr)) {
@@ -412,9 +475,11 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 		 * has to deallocate the existing mappings and establish the
 		 * new ones atomically.
 		 */
-		alloc_flags = VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE;
+		alloc_flags |= VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE;
 	}
 
+	if (flags & MAP_NOCACHE)
+		alloc_flags |= VM_FLAGS_NO_CACHE;
 
 	/*
 	 * Lookup/allocate object.
@@ -426,20 +491,29 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 #if defined(VM_PROT_READ_IS_EXEC)
 		if (prot & VM_PROT_READ)
 			prot |= VM_PROT_EXECUTE;
-
 		if (maxprot & VM_PROT_READ)
 			maxprot |= VM_PROT_EXECUTE;
 #endif
 #endif
-		result = mach_vm_map(user_map, &user_addr, user_size, 0,
-				alloc_flags, IPC_PORT_NULL, 0,
-				FALSE, prot, maxprot,
-				(flags & MAP_SHARED) ? VM_INHERIT_SHARE : 
-				                       VM_INHERIT_DEFAULT);
+
+#if 3777787
+		if (prot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+			prot |= VM_PROT_READ;
+		if (maxprot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+			maxprot |= VM_PROT_READ;
+#endif	/* radar 3777787 */
+
+		result = vm_map_enter_mem_object(user_map,
+						 &user_addr, user_size,
+						 0, alloc_flags,
+						 IPC_PORT_NULL, 0, FALSE,
+						 prot, maxprot,
+						 (flags & MAP_SHARED) ?
+						 VM_INHERIT_SHARE : 
+						 VM_INHERIT_DEFAULT);
 		if (result != KERN_SUCCESS) 
 				goto out;
 	} else {
-		UBCINFOCHECK("mmap", vp);
 		pager = (vm_pager_t)ubc_getpager(vp);
 		
 		if (pager == NULL) {
@@ -466,24 +540,38 @@ mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval)
 #if defined(VM_PROT_READ_IS_EXEC)
 		if (prot & VM_PROT_READ)
 			prot |= VM_PROT_EXECUTE;
-
 		if (maxprot & VM_PROT_READ)
 			maxprot |= VM_PROT_EXECUTE;
 #endif
 #endif /* notyet */
 
-		result = mach_vm_map(user_map, &user_addr, user_size,
-				0, alloc_flags, (ipc_port_t)pager, file_pos,
-				docow, prot, maxprot, 
-				(flags & MAP_SHARED) ? VM_INHERIT_SHARE : 
-				                       VM_INHERIT_DEFAULT);
+#if 3777787
+		if (prot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+			prot |= VM_PROT_READ;
+		if (maxprot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+			maxprot |= VM_PROT_READ;
+#endif	/* radar 3777787 */
+
+		result = vm_map_enter_mem_object(user_map,
+						 &user_addr, user_size,
+						 0, alloc_flags,
+						 (ipc_port_t)pager, file_pos,
+						 docow, prot, maxprot, 
+						 (flags & MAP_SHARED) ?
+						 VM_INHERIT_SHARE : 
+						 VM_INHERIT_DEFAULT);
 
 		if (result != KERN_SUCCESS)  {
 				(void)vnode_put(vp);
 				goto out;
 		}
 
-		(void)ubc_map(vp,(prot & ( PROT_EXEC | PROT_READ | PROT_WRITE | PROT_EXEC)));
+		file_prot = prot & (PROT_READ | PROT_WRITE | PROT_EXEC);
+		if (docow) {
+			/* private mapping: won't write to the file */
+			file_prot &= ~PROT_WRITE;
+		}
+		(void) ubc_map(vp, file_prot);
 	}
 
 	if (!mapanon)
@@ -509,11 +597,23 @@ out:
 bad:
 	if (fpref)
 		fp_drop(p, fd, fp, 0);
+
+	KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_mmap) | DBG_FUNC_NONE), fd, (uint32_t)(*retval), (uint32_t)user_size, error, 0);
+	KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO2, SYS_mmap) | DBG_FUNC_NONE), (uint32_t)(*retval >> 32), (uint32_t)(user_size >> 32),
+			      (uint32_t)(file_pos >> 32), (uint32_t)file_pos, 0);
+
 	return(error);
 }
 
 int
-msync(__unused struct proc *p, struct msync_args *uap, __unused register_t *retval)
+msync(__unused proc_t p, struct msync_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(msync_nocancel(p, (struct msync_nocancel_args *)uap, retval));
+}
+
+int
+msync_nocancel(__unused proc_t p, struct msync_nocancel_args *uap, __unused register_t *retval)
 {
 	mach_vm_offset_t addr;
 	mach_vm_size_t size;
@@ -525,6 +625,8 @@ msync(__unused struct proc *p, struct msync_args *uap, __unused register_t *retv
 	addr = (mach_vm_offset_t) uap->addr;
 	size = (mach_vm_size_t)uap->len;
 
+	KERNEL_DEBUG_CONSTANT((BSDDBG_CODE(DBG_BSD_SC_EXTENDED_INFO, SYS_msync) | DBG_FUNC_NONE), (uint32_t)(addr >> 32), (uint32_t)(size >> 32), 0, 0, 0);
+
 	if (addr & PAGE_MASK_64) {
 		/* UNIX SPEC: user address is not page-aligned, return EINVAL */
 		return EINVAL;
@@ -542,8 +644,7 @@ msync(__unused struct proc *p, struct msync_args *uap, __unused register_t *retv
 
 	flags = uap->flags;
 	/* disallow contradictory flags */
-	if ((flags & (MS_SYNC|MS_ASYNC)) == (MS_SYNC|MS_ASYNC) ||
-	    (flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
+	if ((flags & (MS_SYNC|MS_ASYNC)) == (MS_SYNC|MS_ASYNC))
 		return (EINVAL);
 
 	if (flags & MS_KILLPAGES)
@@ -587,7 +688,7 @@ mremap(void)
 }
 
 int
-munmap(__unused struct proc *p, struct munmap_args *uap, __unused register_t *retval)
+munmap(__unused proc_t p, struct munmap_args *uap, __unused register_t *retval)
 {
 	mach_vm_offset_t	user_addr;
 	mach_vm_size_t	user_size;
@@ -620,13 +721,16 @@ munmap(__unused struct proc *p, struct munmap_args *uap, __unused register_t *re
 }
 
 int
-mprotect(__unused struct proc *p, struct mprotect_args *uap, __unused register_t *retval)
+mprotect(__unused proc_t p, struct mprotect_args *uap, __unused register_t *retval)
 {
 	register vm_prot_t prot;
 	mach_vm_offset_t	user_addr;
 	mach_vm_size_t	user_size;
 	kern_return_t	result;
 	vm_map_t	user_map;
+#if CONFIG_MACF
+	int error;
+#endif
 
 	AUDIT_ARG(addr, uap->addr);
 	AUDIT_ARG(len, uap->len);
@@ -649,8 +753,29 @@ mprotect(__unused struct proc *p, struct mprotect_args *uap, __unused register_t
 #endif
 #endif /* notyet */
 
+#if 3936456
+	if (prot & (VM_PROT_EXECUTE | VM_PROT_WRITE))
+		prot |= VM_PROT_READ;
+#endif	/* 3936456 */
+
 	user_map = current_map();
 
+#if CONFIG_MACF
+	/*
+	 * The MAC check for mprotect is of limited use for 2 reasons:
+	 * Without mmap revocation, the caller could have asked for the max
+	 * protections initially instead of a reduced set, so a mprotect
+	 * check would offer no new security.
+	 * It is not possible to extract the vnode from the pager object(s)
+	 * of the target memory range.
+	 * However, the MAC check may be used to prevent a process from,
+	 * e.g., making the stack executable.
+	 */
+	error = mac_proc_check_mprotect(p, user_addr,
+	    		user_size, prot);
+	if (error)
+		return (error);
+#endif
 	result = mach_vm_protect(user_map, user_addr, user_size,
 				 FALSE, prot);
 	switch (result) {
@@ -667,7 +792,7 @@ mprotect(__unused struct proc *p, struct mprotect_args *uap, __unused register_t
 
 
 int
-minherit(__unused struct proc *p, struct minherit_args *uap, __unused register_t *retval)
+minherit(__unused proc_t p, struct minherit_args *uap, __unused register_t *retval)
 {
 	mach_vm_offset_t addr;
 	mach_vm_size_t size;
@@ -696,7 +821,7 @@ minherit(__unused struct proc *p, struct minherit_args *uap, __unused register_t
 }
 
 int
-madvise(__unused struct proc *p, struct madvise_args *uap, __unused register_t *retval)
+madvise(__unused proc_t p, struct madvise_args *uap, __unused register_t *retval)
 {
 	vm_map_t user_map;
 	mach_vm_offset_t start;
@@ -745,7 +870,7 @@ madvise(__unused struct proc *p, struct madvise_args *uap, __unused register_t *
 }
 
 int
-mincore(__unused struct proc *p, struct mincore_args *uap, __unused register_t *retval)
+mincore(__unused proc_t p, struct mincore_args *uap, __unused register_t *retval)
 {
 	mach_vm_offset_t addr, first_addr, end;
 	vm_map_t map;
@@ -786,7 +911,7 @@ mincore(__unused struct proc *p, struct mincore_args *uap, __unused register_t *
 	lastvecindex = -1;
 	for( ; addr < end; addr += PAGE_SIZE ) {
 		pqueryinfo = 0;
-		ret = vm_map_page_query(map, addr, &pqueryinfo, &numref);
+		ret = mach_vm_page_query(map, addr, &pqueryinfo, &numref);
 		if (ret != KERN_SUCCESS) 
 			pqueryinfo = 0;
 		mincoreinfo = 0;
@@ -845,7 +970,7 @@ mincore(__unused struct proc *p, struct mincore_args *uap, __unused register_t *
 }
 
 int
-mlock(__unused struct proc *p, struct mlock_args *uap, __unused register_t *retvalval)
+mlock(__unused proc_t p, struct mlock_args *uap, __unused register_t *retvalval)
 {
 	vm_map_t user_map;
 	vm_map_offset_t addr;
@@ -868,31 +993,21 @@ mlock(__unused struct proc *p, struct mlock_args *uap, __unused register_t *retv
 	pageoff = (addr & PAGE_MASK);
 	addr -= pageoff;
 	size = vm_map_round_page(size+pageoff);
-
-#ifdef notyet 
-/* Hmm.. What am I going to do with this? */
-	if (atop(size) + cnt.v_wire_count > vm_page_max_wired)
-		return (EAGAIN);
-#ifdef pmap_wired_count
-	if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) >
-	    p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur)
-		return (ENOMEM);
-#else
-	error = suser(kauth_cred_get(), &p->p_acflag);
-	if (error)
-		return (error);
-#endif
-#endif /* notyet */
-
 	user_map = current_map();
 
 	/* have to call vm_map_wire directly to pass "I don't know" protections */
 	result = vm_map_wire(user_map, addr, addr+size, VM_PROT_NONE, TRUE);
-	return (result == KERN_SUCCESS ? 0 : ENOMEM);
+
+	if (result == KERN_RESOURCE_SHORTAGE)
+		return EAGAIN;
+	else if (result != KERN_SUCCESS)
+		return ENOMEM;
+
+	return 0;	/* KERN_SUCCESS */
 }
 
 int
-munlock(__unused struct proc *p, struct munlock_args *uap, __unused register_t *retval)
+munlock(__unused proc_t p, struct munlock_args *uap, __unused register_t *retval)
 {
 	mach_vm_offset_t addr;
 	mach_vm_size_t size;
@@ -904,17 +1019,6 @@ munlock(__unused struct proc *p, struct munlock_args *uap, __unused register_t *
 
 	addr = (mach_vm_offset_t) uap->addr;
 	size = (mach_vm_size_t)uap->len;
-
-
-#ifdef notyet 
-/* Hmm.. What am I going to do with this? */
-#ifndef pmap_wired_count
-	error = suser(kauth_cred_get(), &p->p_acflag);
-	if (error)
-		return (error);
-#endif
-#endif /* notyet */
-
 	user_map = current_map();
 
 	/* JMM - need to remove all wirings by spec - this just removes one */
@@ -924,13 +1028,13 @@ munlock(__unused struct proc *p, struct munlock_args *uap, __unused register_t *
 
 
 int
-mlockall(__unused struct proc *p, __unused struct mlockall_args *uap, __unused register_t *retval)
+mlockall(__unused proc_t p, __unused struct mlockall_args *uap, __unused register_t *retval)
 {
 	return (ENOSYS);
 }
 
 int
-munlockall(__unused struct proc *p, __unused struct munlockall_args *uap, __unused register_t *retval)
+munlockall(__unused proc_t p, __unused struct munlockall_args *uap, __unused register_t *retval)
 {
 	return(ENOSYS);
 }
@@ -938,7 +1042,7 @@ munlockall(__unused struct proc *p, __unused struct munlockall_args *uap, __unus
 
 /* BEGIN DEFUNCT */
 int
-obreak(__unused struct proc *p, __unused struct obreak_args *uap, __unused register_t *retval)
+obreak(__unused proc_t p, __unused struct obreak_args *uap, __unused register_t *retval)
 {
 	/* Not implemented, obsolete */
 	return (ENOMEM);
@@ -947,7 +1051,7 @@ obreak(__unused struct proc *p, __unused struct obreak_args *uap, __unused regis
 int	both;
 
 int
-ovadvise(__unused struct proc *p, __unused struct ovadvise_args *uap, __unused register_t *retval)
+ovadvise(__unused proc_t p, __unused struct ovadvise_args *uap, __unused register_t *retval)
 {
 
 #ifdef lint
@@ -994,9 +1098,8 @@ map_fd_funneled(
 	vm_size_t	map_size;
 	int		err=0;
 	vm_map_t	my_map;
-	struct proc	*p =(struct proc *)current_proc();
+	proc_t		p = current_proc();
 	struct vnode_attr vattr;
-	struct vfs_context context;
 
 	/*
 	 *	Find the inode; verify that it's a regular file.
@@ -1029,16 +1132,16 @@ map_fd_funneled(
 
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-	/* conformance change - mmap needs to update access time for mapped
-	 * files
+	/*
+	 * POSIX: mmap needs to update access time for mapped files
 	 */
-	VATTR_INIT(&vattr);
-	nanotime(&vattr.va_access_time);
-	VATTR_SET_ACTIVE(&vattr, va_access_time);
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	vnode_setattr(vp, &vattr, &context);
-
+	if ((vnode_vfsvisflags(vp) & MNT_NOATIME) == 0) {
+		VATTR_INIT(&vattr);
+		nanotime(&vattr.va_access_time);
+		VATTR_SET_ACTIVE(&vattr, va_access_time);
+		vnode_setattr(vp, &vattr, vfs_context_current());
+	}
+	
 	if (offset & PAGE_MASK_64) {
 		printf("map_fd: file offset not page aligned(%d : %s)\n",p->p_pid, p->p_comm);
 		(void)vnode_put(vp);
@@ -1058,8 +1161,7 @@ map_fd_funneled(
 	/*
 	 *	Map in the file.
 	 */
-	UBCINFOCHECK("map_fd_funneled", vp);
-	pager = (void *) ubc_getpager(vp);
+	pager = (void *)ubc_getpager(vp);
 	if (pager == NULL) {
 		(void)vnode_put(vp);
 		err = KERN_FAILURE;
@@ -1129,7 +1231,7 @@ map_fd_funneled(
 	}
 
 	ubc_setthreadcred(vp, current_proc(), current_thread());
-	(void)ubc_map(vp, (PROT_READ | PROT_WRITE | PROT_EXEC));
+	(void)ubc_map(vp, (PROT_READ | PROT_EXEC));
 	(void)vnode_put(vp);
 	err = 0;
 bad:
diff --git a/bsd/kern/kern_newsysctl.c b/bsd/kern/kern_newsysctl.c
index 63524f2c4..54ce269fa 100644
--- a/bsd/kern/kern_newsysctl.c
+++ b/bsd/kern/kern_newsysctl.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -80,7 +86,7 @@ struct sysctl_oid_list sysctl__sysctl_children;
 
 extern struct sysctl_oid *newsysctl_list[];
 extern struct sysctl_oid *machdep_sysctl_list[];
-
+lck_rw_t * sysctl_geometry_lock = NULL;
 
 static void
 sysctl_sysctl_debug_dump_node(struct sysctl_oid_list *l, int i);
@@ -90,11 +96,7 @@ sysctl_sysctl_debug_dump_node(struct sysctl_oid_list *l, int i);
 /*
  * Locking and stats
  */
-static struct sysctl_lock {
-	int	sl_lock;
-	int	sl_want;
-	int	sl_locked;
-} memlock;
+static struct sysctl_lock memlock;
 
 /*
  * XXX this does not belong here
@@ -143,15 +145,27 @@ void sysctl_register_oid(struct sysctl_oid *oidp)
 
 	fnl = spl_kernel_funnel();
 
+	if(sysctl_geometry_lock == NULL)
+	{
+		/* Initialise the geometry lock for reading/modifying the sysctl tree
+		 * This is done here because IOKit registers some sysctls before bsd_init()
+		 * calls sysctl_register_fixed().
+		 */
+
+		lck_grp_t* lck_grp  = lck_grp_alloc_init("sysctl", NULL);
+		sysctl_geometry_lock = lck_rw_alloc_init(lck_grp, NULL);
+	}
+	/* Get the write lock to modify the geometry */
+	lck_rw_lock_exclusive(sysctl_geometry_lock);
+
 	/*
 	 * If this oid has a number OID_AUTO, give it a number which
 	 * is greater than any current oid.  Make sure it is at least
-	 * 100 to leave space for pre-assigned oid numbers.
+	 * OID_AUTO_START to leave space for pre-assigned oid numbers.
 	 */
-/*	sysctl_sysctl_debug_dump_node(parent, 3); */
 	if (oidp->oid_number == OID_AUTO) {
-		/* First, find the highest oid in the parent list >99 */
-		n = 99;
+		/* First, find the highest oid in the parent list >OID_AUTO_START-1 */
+		n = OID_AUTO_START;
 		SLIST_FOREACH(p, parent, oid_link) {
 			if (p->oid_number > n)
 				n = p->oid_number;
@@ -173,6 +187,9 @@ void sysctl_register_oid(struct sysctl_oid *oidp)
 	else
 		SLIST_INSERT_HEAD(parent, oidp, oid_link);
 
+	/* Release the write lock */
+	lck_rw_unlock_exclusive(sysctl_geometry_lock);
+
 	splx_kernel_funnel(fnl);
 }
 
@@ -181,57 +198,191 @@ void sysctl_unregister_oid(struct sysctl_oid *oidp)
 	funnel_t *fnl;
 
 	fnl = spl_kernel_funnel();
+
+	/* Get the write lock to modify the geometry */
+	lck_rw_lock_exclusive(sysctl_geometry_lock);
+
 	SLIST_REMOVE(oidp->oid_parent, oidp, sysctl_oid, oid_link);
+
+	/* Release the write lock */
+	lck_rw_unlock_exclusive(sysctl_geometry_lock);
+
 	splx_kernel_funnel(fnl);
 }
 
 /*
  * Bulk-register all the oids in a linker_set.
  */
-void sysctl_register_set(struct linker_set *lsp)
+void sysctl_register_set(const char *set)
 {
-	int count = lsp->ls_length;
-	int i;
-	for (i = 0; i < count; i++)
-		sysctl_register_oid((struct sysctl_oid *) lsp->ls_items[i]);
+	struct sysctl_oid **oidpp, *oidp;
+
+	LINKER_SET_FOREACH(oidpp, set) {
+		oidp = *oidpp;
+		if (!(oidp->oid_kind & CTLFLAG_NOAUTO)) {
+		    sysctl_register_oid(oidp);
+		}
+	}
 }
 
-void sysctl_unregister_set(struct linker_set *lsp)
+void sysctl_unregister_set(const char *set)
 {
-	int count = lsp->ls_length;
-	int i;
-	for (i = 0; i < count; i++)
-		sysctl_unregister_oid((struct sysctl_oid *) lsp->ls_items[i]);
+	struct sysctl_oid **oidpp, *oidp;
+
+	LINKER_SET_FOREACH(oidpp, set) {
+		oidp = *oidpp;
+		if (!(oidp->oid_kind & CTLFLAG_NOAUTO)) {
+		    sysctl_unregister_oid(oidp);
+		}
+	}
 }
 
 
 /*
- * Register OID's from fixed list
+ * Register the kernel's oids on startup.
  */
 
-void sysctl_register_fixed()
+void
+sysctl_register_all()
+{
+	sysctl_register_set("__sysctl_set");
+}
+
+void
+sysctl_register_fixed(void)
 {
-    int i;
-
-    for (i=0; newsysctl_list[i]; i++) {
-	sysctl_register_oid(newsysctl_list[i]);
-    }
-    for (i=0; machdep_sysctl_list[i]; i++) {
-	sysctl_register_oid(machdep_sysctl_list[i]);
-    }
+	sysctl_register_all();
 }
 
 /*
- * Register the kernel's oids on startup.
+ * New handler interface
+ *   If the sysctl caller (user mode or kernel mode) is interested in the
+ *   value (req->oldptr != NULL), we copy the data (bigValue etc.) out,
+ *   if the caller wants to set the value (req->newptr), we copy
+ *   the data in (*pValue etc.).
  */
-struct linker_set sysctl_set;
 
-void sysctl_register_all(void *arg)
+int
+sysctl_io_number(struct sysctl_req *req, long long bigValue, size_t valueSize, void *pValue, int *changed) {
+	int		smallValue;
+	int		error;
+
+	if (changed) *changed = 0;
+
+	/*
+	 * Handle the various combinations of caller buffer size and
+	 * data value size.  We are generous in the case where the
+	 * caller has specified a 32-bit buffer but the value is 64-bit
+	 * sized.
+	 */
+
+	/* 32 bit value expected or 32 bit buffer offered */
+	if ((valueSize == sizeof(int)) ||
+	    ((req->oldlen == sizeof(int)) && (valueSize == sizeof(long long)))) {
+		smallValue = (int)bigValue;
+		if ((long long)smallValue != bigValue)
+			return(ERANGE);
+		error = SYSCTL_OUT(req, &smallValue, sizeof(smallValue));
+	} else {
+		/* any other case is either size-equal or a bug */
+		error = SYSCTL_OUT(req, &bigValue, valueSize);
+	}
+	/* error or nothing to set */
+	if (error || !req->newptr)
+		return(error);
+
+	/* set request for constant */
+	if (pValue == NULL)
+		return(EPERM);
+
+	/* set request needs to convert? */
+	if ((req->newlen == sizeof(int)) && (valueSize == sizeof(long long))) {
+		/* new value is 32 bits, upconvert to 64 bits */
+		error = SYSCTL_IN(req, &smallValue, sizeof(smallValue));
+		if (!error)
+			*(long long *)pValue = (long long)smallValue;
+	} else if ((req->newlen == sizeof(long long)) && (valueSize == sizeof(int))) {
+		/* new value is 64 bits, downconvert to 32 bits and range check */
+		error = SYSCTL_IN(req, &bigValue, sizeof(bigValue));
+		if (!error) {
+			smallValue = (int)bigValue;
+			if ((long long)smallValue != bigValue)
+				return(ERANGE);
+			*(int *)pValue = smallValue;
+		}
+	} else {
+		/* sizes match, just copy in */
+		error = SYSCTL_IN(req, pValue, valueSize);
+	}
+	if (!error && changed)
+		*changed = 1;
+	return(error);
+}
+
+int
+sysctl_io_string(struct sysctl_req *req, char *pValue, size_t valueSize, int trunc, int *changed)
 {
-	sysctl_register_set(&sysctl_set);
+	int error;
+
+	if (changed) *changed = 0;
+
+	if (trunc && req->oldptr && req->oldlen && (req->oldlen<strlen(pValue) + 1)) {
+		/* If trunc != 0, if you give it a too small (but larger than
+		 * 0 bytes) buffer, instead of returning ENOMEM, it truncates the
+		 * returned string to the buffer size.  This preserves the semantics
+		 * of some library routines implemented via sysctl, which truncate
+		 * their returned data, rather than simply returning an error. The
+		 * returned string is always NUL terminated. */
+		error = SYSCTL_OUT(req, pValue, req->oldlen-1);
+		if (!error) {
+			char c = 0;
+			error = SYSCTL_OUT(req, &c, 1);
+		}
+	} else {
+		/* Copy string out */
+		error = SYSCTL_OUT(req, pValue, strlen(pValue) + 1);
+	}
+
+	/* error or no new value */
+	if (error || !req->newptr)
+		return(error);
+
+	/* attempt to set read-only value */
+	if (valueSize == 0)
+		return(EPERM);
+
+	/* make sure there's room for the new string */
+	if (req->newlen >= valueSize)
+		return(EINVAL);
+
+	/* copy the string in and force NUL termination */
+	error = SYSCTL_IN(req, pValue, req->newlen);
+	pValue[req->newlen] = '\0';
+
+	if (!error && changed)
+		*changed = 1;
+	return(error);
 }
 
-SYSINIT(sysctl, SI_SUB_KMEM, SI_ORDER_ANY, sysctl_register_all, 0);
+int sysctl_io_opaque(struct sysctl_req *req,void *pValue, size_t valueSize, int *changed)
+{
+	int error;
+
+	if (changed) *changed = 0;
+
+	/* Copy blob out */
+	error = SYSCTL_OUT(req, pValue, valueSize);
+
+	/* error or nothing to set */
+	if (error || !req->newptr)
+		return(error);
+
+	error = SYSCTL_IN(req, pValue, valueSize);
+
+	if (!error && changed)
+		*changed = 1;
+	return(error);
+}
 
 /*
  * "Staff-functions"
@@ -290,7 +441,8 @@ sysctl_sysctl_debug_dump_node(struct sysctl_oid_list *l, int i)
 }
 
 static int
-sysctl_sysctl_debug SYSCTL_HANDLER_ARGS
+sysctl_sysctl_debug(__unused struct sysctl_oid *oidp, __unused void *arg1,
+	__unused int arg2, __unused struct sysctl_req *req)
 {
 	sysctl_sysctl_debug_dump_node(&sysctl__children, 0);
 	return ENOENT;
@@ -300,7 +452,8 @@ SYSCTL_PROC(_sysctl, 0, debug, CTLTYPE_STRING|CTLFLAG_RD,
 	0, 0, sysctl_sysctl_debug, "-", "");
 
 static int
-sysctl_sysctl_name SYSCTL_HANDLER_ARGS
+sysctl_sysctl_name(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
 	int *name = (int *) arg1;
 	u_int namelen = arg2;
@@ -372,6 +525,11 @@ sysctl_sysctl_next_ls (struct sysctl_oid_list *lsp, int *name, u_int namelen,
 				/* We really should call the handler here...*/
 				return 0;
 			lsp = (struct sysctl_oid_list *)oidp->oid_arg1;
+
+			if (!SLIST_FIRST(lsp))
+				/* This node had no children - skip it! */
+				continue;
+
 			if (!sysctl_sysctl_next_ls (lsp, 0, 0, next+1, 
 				len, level+1, oidpp))
 				return 0;
@@ -410,7 +568,8 @@ sysctl_sysctl_next_ls (struct sysctl_oid_list *lsp, int *name, u_int namelen,
 }
 
 static int
-sysctl_sysctl_next SYSCTL_HANDLER_ARGS
+sysctl_sysctl_next(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
 	int *name = (int *) arg1;
 	u_int namelen = arg2;
@@ -486,10 +645,12 @@ name2oid (char *name, int *oid, int *len, struct sysctl_oid **oidpp)
 }
 
 static int
-sysctl_sysctl_name2oid SYSCTL_HANDLER_ARGS
+sysctl_sysctl_name2oid(__unused struct sysctl_oid *oidp, __unused void *arg1,
+	__unused int arg2, struct sysctl_req *req)
 {
 	char *p;
-	int error, oid[CTL_MAXNAME], len;
+	int error, oid[CTL_MAXNAME];
+	int len = 0;		/* set by name2oid() */
 	struct sysctl_oid *op = 0;
 
 	if (!req->newlen) 
@@ -524,11 +685,12 @@ SYSCTL_PROC(_sysctl, 3, name2oid, CTLFLAG_RW|CTLFLAG_ANYBODY|CTLFLAG_KERN, 0, 0,
 	sysctl_sysctl_name2oid, "I", "");
 
 static int
-sysctl_sysctl_oidfmt SYSCTL_HANDLER_ARGS
+sysctl_sysctl_oidfmt(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
 	int *name = (int *) arg1, error;
 	u_int namelen = arg2;
-	int indx;
+	u_int indx;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *lsp = &sysctl__children;
 
@@ -581,26 +743,10 @@ SYSCTL_NODE(_sysctl, 4, oidfmt, CTLFLAG_RD, sysctl_sysctl_oidfmt, "");
  */
 
 int
-sysctl_handle_int SYSCTL_HANDLER_ARGS
+sysctl_handle_int(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
-	int error = 0;
-
-	if (arg1)
-		error = SYSCTL_OUT(req, arg1, sizeof(int));
-	else
-		error = SYSCTL_OUT(req, &arg2, sizeof(int));
-
-	if (error || !req->newptr)
-		return (error);
-
-	if (!arg1)
-		error = EPERM;
-	else
-		error = SYSCTL_IN(req, arg1, sizeof(int));
-
-	if (error == 0)
-		AUDIT_ARG(value, *(int *)arg1);
-	return (error);
+	return sysctl_io_number(req, arg1? *(int*)arg1: arg2, sizeof(int), arg1, NULL);
 }
 
 /*
@@ -608,19 +754,12 @@ sysctl_handle_int SYSCTL_HANDLER_ARGS
  */
 
 int
-sysctl_handle_long SYSCTL_HANDLER_ARGS
+sysctl_handle_long(__unused struct sysctl_oid *oidp, void *arg1,
+	__unused int arg2, struct sysctl_req *req)
 {
-	int error = 0;
-
 	if (!arg1)
 		return (EINVAL);
-	error = SYSCTL_OUT(req, arg1, sizeof(long));
-
-	if (error || !req->newptr)
-		return (error);
-
-	error = SYSCTL_IN(req, arg1, sizeof(long));
-	return (error);
+	return sysctl_io_number(req, *(long*)arg1, sizeof(long), arg1, NULL);
 }
 
 /*
@@ -628,19 +767,12 @@ sysctl_handle_long SYSCTL_HANDLER_ARGS
  */
 
 int
-sysctl_handle_quad SYSCTL_HANDLER_ARGS
+sysctl_handle_quad(__unused struct sysctl_oid *oidp, void *arg1,
+	__unused int arg2, struct sysctl_req *req)
 {
-	int error = 0;
-
 	if (!arg1)
 		return (EINVAL);
-	error = SYSCTL_OUT(req, arg1, sizeof(long long));
-
-	if (error || !req->newptr)
-		return (error);
-
-	error = SYSCTL_IN(req, arg1, sizeof(long long));
-	return (error);
+	return sysctl_io_number(req, *(long long*)arg1, sizeof(long long), arg1, NULL);
 }
 
 /*
@@ -651,7 +783,8 @@ sysctl_handle_quad SYSCTL_HANDLER_ARGS
  * using ints.
  */
 int
-sysctl_handle_int2quad SYSCTL_HANDLER_ARGS
+sysctl_handle_int2quad(__unused struct sysctl_oid *oidp, void *arg1,
+	__unused int arg2, struct sysctl_req *req)
 {
 	int error = 0;
 	long long val;
@@ -689,24 +822,10 @@ sysctl_handle_int2quad SYSCTL_HANDLER_ARGS
  */
 
 int
-sysctl_handle_string SYSCTL_HANDLER_ARGS
+sysctl_handle_string( __unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
-	int error=0;
-
-	error = SYSCTL_OUT(req, arg1, strlen((char *)arg1)+1);
-
-	if (error || !req->newptr)
-		return (error);
-
-	if ((req->newlen - req->newidx) >= arg2) {
-		error = EINVAL;
-	} else {
-		arg2 = (req->newlen - req->newidx);
-		error = SYSCTL_IN(req, arg1, arg2);
-		((char *)arg1)[arg2] = '\0';
-	}
-
-	return (error);
+	return sysctl_io_string(req, arg1, arg2, 0, NULL);
 }
 
 /*
@@ -715,18 +834,10 @@ sysctl_handle_string SYSCTL_HANDLER_ARGS
  */
 
 int
-sysctl_handle_opaque SYSCTL_HANDLER_ARGS
+sysctl_handle_opaque(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
-	int error;
-
-	error = SYSCTL_OUT(req, arg1, arg2);
-
-	if (error || !req->newptr)
-		return (error);
-
-	error = SYSCTL_IN(req, arg1, arg2);
-
-	return (error);
+	return sysctl_io_opaque(req, arg1, arg2, NULL);
 }
 
 /*
@@ -742,7 +853,7 @@ sysctl_old_kernel(struct sysctl_req *req, const void *p, size_t l)
 		if (i > req->oldlen - req->oldidx)
 			i = req->oldlen - req->oldidx;
 		if (i > 0)
-			bcopy((void*)p, CAST_DOWN(char *, (req->oldptr + req->oldidx)), i);
+			bcopy((const void*)p, CAST_DOWN(char *, (req->oldptr + req->oldidx)), i);
 	}
 	req->oldidx += l;
 	if (req->oldptr && i != l)
@@ -767,7 +878,6 @@ kernel_sysctl(struct proc *p, int *name, u_int namelen, void *old, size_t *oldle
 {
 	int error = 0;
 	struct sysctl_req req;
-	funnel_t *fnl;
 
 	/*
 	 * Construct request.
@@ -786,19 +896,6 @@ kernel_sysctl(struct proc *p, int *name, u_int namelen, void *old, size_t *oldle
 	req.newfunc = sysctl_new_kernel;
 	req.lock = 1;
 
-	/*
-	 * Locking.  Tree traversal always begins with the kernel funnel held.
-	 */
-	fnl = spl_kernel_funnel();
-
-	/* XXX this should probably be done in a general way */
-	while (memlock.sl_lock) {
-		memlock.sl_want = 1;
-		(void) tsleep((caddr_t)&memlock, PRIBIO+1, "sysctl", 0);
-		memlock.sl_locked++;
-	}
-	memlock.sl_lock = 1;
-
 	/* make the request */
 	error = sysctl_root(0, name, namelen, &req);
 
@@ -806,18 +903,6 @@ kernel_sysctl(struct proc *p, int *name, u_int namelen, void *old, size_t *oldle
 	if (req.lock == 2)
 		vsunlock(req.oldptr, (user_size_t)req.oldlen, B_WRITE);
 
-	memlock.sl_lock = 0;
-
-	if (memlock.sl_want) {
-		memlock.sl_want = 0;
-		wakeup((caddr_t)&memlock);
-	}
-
-	/*
-	 * Undo locking.
-	 */
-	splx_kernel_funnel(fnl);
-
 	if (error && error != ENOMEM)
 		return (error);
 
@@ -843,7 +928,7 @@ sysctl_old_user(struct sysctl_req *req, const void *p, size_t l)
 		if (i > req->oldlen - req->oldidx)
 			i = req->oldlen - req->oldidx;
 		if (i > 0)
-			error = copyout((void*)p, (req->oldptr + req->oldidx), i);
+			error = copyout((const void*)p, (req->oldptr + req->oldidx), i);
 	}
 	req->oldidx += l;
 	if (error)
@@ -873,14 +958,21 @@ sysctl_new_user(struct sysctl_req *req, void *p, size_t l)
  */
 
 int
-sysctl_root SYSCTL_HANDLER_ARGS
+sysctl_root(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+        struct sysctl_req *req)
 {
 	int *name = (int *) arg1;
 	u_int namelen = arg2;
-	int indx, i;
+	u_int indx;
+	int i;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *lsp = &sysctl__children;
 	int error;
+	funnel_t *fnl = NULL;
+	boolean_t funnel_held = FALSE;
+
+	/* Get the read lock on the geometry */
+	lck_rw_lock_shared(sysctl_geometry_lock);
 
 	oid = SLIST_FIRST(lsp);
 
@@ -888,30 +980,43 @@ sysctl_root SYSCTL_HANDLER_ARGS
 	while (oid && indx < CTL_MAXNAME) {
 		if (oid->oid_number == name[indx]) {
 			indx++;
+			if (!(oid->oid_kind & CTLFLAG_LOCKED))
+			{
+				funnel_held = TRUE;
+			}
 			if (oid->oid_kind & CTLFLAG_NOLOCK)
 				req->lock = 0;
 			if ((oid->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
 				if (oid->oid_handler)
 					goto found;
 				if (indx == namelen)
-					return ENOENT;
+				{
+					error = ENOENT;
+					goto err;
+				}
+
 				lsp = (struct sysctl_oid_list *)oid->oid_arg1;
 				oid = SLIST_FIRST(lsp);
 			} else {
 				if (indx != namelen)
-					return EISDIR;
+				{
+					error = EISDIR;
+					goto err;
+				}
 				goto found;
 			}
 		} else {
 			oid = SLIST_NEXT(oid, oid_link);
 		}
 	}
-	return ENOENT;
+	error = ENOENT;
+	goto err;
 found:
 	/* If writing isn't allowed */
 	if (req->newptr && (!(oid->oid_kind & CTLFLAG_WR) ||
 			    ((oid->oid_kind & CTLFLAG_SECURE) && securelevel > 0))) {
-		return (EPERM);
+		error = (EPERM);
+		goto err;
 	}
 
 	/*
@@ -919,18 +1024,27 @@ found:
 	 * XXX This mechanism for testing is bad.
 	 */
 	if ((req->oldfunc == sysctl_old_kernel) && !(oid->oid_kind & CTLFLAG_KERN))
-		return(EPERM);
+	{
+		error = (EPERM);
+		goto err;
+	}
 
 	/* Most likely only root can write */
 	if (!(oid->oid_kind & CTLFLAG_ANYBODY) &&
 	    req->newptr && req->p &&
-	    (error = suser(req->p->p_ucred, &req->p->p_acflag)))
-		return (error);
+	    (error = proc_suser(req->p)))
+		goto err;
 
 	if (!oid->oid_handler) {
-	    return EINVAL;
+	    error = EINVAL;
+		goto err;
 	}
 
+	if (funnel_held)
+	{
+		fnl = spl_kernel_funnel();
+		MEMLOCK_LOCK();
+	}
 
 	if ((oid->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
 		i = (oid->oid_handler) (oid,
@@ -941,8 +1055,17 @@ found:
 					oid->oid_arg1, oid->oid_arg2,
 					req);
 	}
+	error = i;
+
+	if (funnel_held)
+	{
+		MEMLOCK_UNLOCK();
+		splx_kernel_funnel(fnl);
+	}
 
-	return (i);
+err:
+	lck_rw_done(sysctl_geometry_lock);
+	return (error);
 }
 
 #ifndef _SYS_SYSPROTO_H_
diff --git a/bsd/kern/kern_panicinfo.c b/bsd/kern/kern_panicinfo.c
index 9ad8549f1..dc9042401 100644
--- a/bsd/kern/kern_panicinfo.c
+++ b/bsd/kern/kern_panicinfo.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>
@@ -51,14 +57,8 @@ extern int sysctl_dopanicinfo(int *, u_int, user_addr_t, size_t *, user_addr_t,
 static int image_size_limit = PANIC_IMAGE_SIZE_LIMIT;
 
 __private_extern__ int
-sysctl_dopanicinfo(name, namelen, oldp, oldlenp, newp, newlen, p)
-	int *name;
-	u_int namelen;
-	user_addr_t oldp;
-	size_t *oldlenp;
-	user_addr_t newp;
-	size_t newlen;
-	struct proc *p;
+sysctl_dopanicinfo(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
+		   user_addr_t newp, size_t newlen, struct proc *p)
 {
 	int error = 0;
 	vm_offset_t newimage = (vm_offset_t )NULL;
@@ -66,7 +66,6 @@ sysctl_dopanicinfo(name, namelen, oldp, oldlenp, newp, newlen, p)
 	unsigned char * prev_image_ptr;
 	unsigned int prev_image_size;
 
-
 	/* all sysctl names at this level are terminal */
 	if (namelen != 1)
 		return (ENOTDIR);		/* overloaded */
diff --git a/bsd/kern/kern_pcsamples.c b/bsd/kern/kern_pcsamples.c
index f231dd6cb..36757c2af 100644
--- a/bsd/kern/kern_pcsamples.c
+++ b/bsd/kern/kern_pcsamples.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/kdebug.h>
@@ -50,7 +56,7 @@ unsigned int pcsample_end    = 0;
 
 static pid_t global_state_pid = -1;       /* Used to control exclusive use of pc_buffer */
 
-extern int pc_trace_buf[];
+extern unsigned int pc_trace_buf[];
 extern int pc_trace_cnt;
 
 void add_pcbuffer(void);
@@ -69,9 +75,10 @@ enable_branch_tracing(void)
 #ifndef i386
   struct proc *p;
   if (-1 != pc_sample_pid) {
-    p = pfind(pc_sample_pid);
+    p = proc_find(pc_sample_pid);
     if (p) {
-      p->p_flag |= P_BTRACE;
+      p->p_btrace = 1;
+	 proc_rele(p);
     } 
   }
   else {
@@ -96,9 +103,10 @@ disable_branch_tracing(void)
     case 0:
         break;
     default:
-        p = pfind(pc_sample_pid);
+        p = proc_find(pc_sample_pid);
         if (p) {
-            p->p_flag &= ~P_BTRACE;
+            p->p_btrace = 0;
+			proc_rele(p);
         }
         break;
     }
@@ -116,7 +124,7 @@ branch_tracing_enabled(void)
   struct proc *p = current_proc();
   if (TRUE == pc_trace_frameworks) return TRUE;
   if (p) {
-    return (P_BTRACE == (p->p_flag & P_BTRACE));
+    return (p->p_btrace);
   }
   return 0;
 }
@@ -242,13 +250,14 @@ pcsamples_control(int *name, __unused u_int namelen, user_addr_t where, size_t *
 	      global_state_pid = curpid;
 	    else if (global_state_pid != curpid)
 	      {
-		if((p = pfind(global_state_pid)) == NULL)
+		if((p = proc_find(global_state_pid)) == NULL)
 		  {
 		    /* The global pid no longer exists */
 		    global_state_pid = curpid;
 		  }
 		else
 		  {
+			proc_rele(p);
 		    /* The global pid exists, deny this request */
 		    return(EBUSY);
 		  }
diff --git a/bsd/kern/kern_physio.c b/bsd/kern/kern_physio.c
index 2b6ba9062..2255daf1d 100644
--- a/bsd/kern/kern_physio.c
+++ b/bsd/kern/kern_physio.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1982, 1986, 1990, 1993
@@ -70,19 +76,23 @@
 #include <sys/conf.h>
 #include <sys/proc_internal.h>
 #include <sys/uio_internal.h>
+#include <kern/assert.h>
+
+int rawread(dev_t dev, struct uio *uio);
+int rawwrite(dev_t dev, struct uio *uio);
 
 int
-physio(strategy, bp, dev, flags, minphys, uio, blocksize)
-	void (*strategy)(); 
-	buf_t bp;
-	dev_t dev;
-	int flags;
-	u_int (*minphys)();
-	struct uio *uio;
-	int blocksize;
+physio( void (*f_strategy)(buf_t), 
+	buf_t bp,
+	dev_t dev,
+	int flags,
+	u_int (*f_minphys)(buf_t),
+	struct uio *uio,
+	int blocksize)
 {
 	struct proc *p = current_proc();
-	int error, i, nobuf, todo, iosize;
+	int error, i, buf_allocated, todo, iosize;
+	int orig_bflags = 0;
 #if LP64KERN
 	int64_t done;
 #else
@@ -91,6 +101,7 @@ physio(strategy, bp, dev, flags, minphys, uio, blocksize)
 
 	error = 0;
 	flags &= B_READ | B_WRITE;
+	buf_allocated = 0;
 
 	/*
 	 * [check user read/write access to the data buffer]
@@ -100,32 +111,46 @@ physio(strategy, bp, dev, flags, minphys, uio, blocksize)
 	 * we're doing a read, that's a *write* to user-space.
 	 */
 	for (i = 0; i < uio->uio_iovcnt; i++) {
-		if(UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
+		if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
 			if (!useracc(uio_iov_base_at(uio, i),
 					uio_iov_len_at(uio, i),
 		    		(flags == B_READ) ? B_WRITE : B_READ))
 			return (EFAULT);
 		}
 	}
-	/* Make sure we have a buffer, creating one if necessary. */
-	if (nobuf = (bp == NULL)) {
-			bp = buf_alloc((vnode_t)0);
-		}
-
-	/* [while the buffer is marked busy] */
-	while (((error = (int)buf_acquire(bp, 0, 0, 0)) == EAGAIN));
-	
-	if (error) {
-			if (nobuf)
-				buf_free(bp);
-	        return (error);
-	}
+	/*
+	 * Make sure we have a buffer, creating one if necessary.
+	 */
+	if (bp == NULL) {
+		bp = buf_alloc((vnode_t)0);
+		buf_allocated = 1;
+	} else
+	        orig_bflags = buf_flags(bp);
+	/*
+	 * at this point we should have a buffer
+	 * that is marked BL_BUSY... we either 
+	 * acquired it via buf_alloc, or it was
+	 * passed into us... if it was passed
+	 * in, it needs to already be owned by
+	 * the caller (i.e. BL_BUSY is set)
+	 */
+	assert(bp->b_lflags & BL_BUSY);
 
-	/* [set up the fixed part of the buffer for a transfer] */
+	/*
+	 * [set up the fixed part of the buffer for a transfer]
+	 */
 	bp->b_dev = dev;
 	bp->b_proc = p;
 
-	buf_seterror(bp, 0);
+	/*
+	 * [mark the buffer busy for physical I/O]
+	 * (i.e. set B_PHYS (because it's an I/O to user
+	 * memory, and B_RAW, because B_RAW is to be
+	 * "Set by physio for raw transfers.", in addition
+	 * to the read/write flag.)
+	 */
+        buf_setflags(bp, B_PHYS | B_RAW);
+
 	/*
 	 * [while there is data to transfer and no I/O error]
 	 * Note that I/O errors are handled with a 'goto' at the bottom
@@ -133,30 +158,27 @@ physio(strategy, bp, dev, flags, minphys, uio, blocksize)
 	 */
 	for (i = 0; i < uio->uio_iovcnt; i++) {
 		while (uio_iov_len_at(uio, i) > 0) {
-			/*
-			 * [mark the buffer busy for physical I/O]
-			 * (i.e. set B_PHYS (because it's an I/O to user
-			 * memory, and B_RAW, because B_RAW is to be
-			 * "Set by physio for raw transfers.", in addition
-			 * to the read/write flag.)
-			 */
-		        buf_setflags(bp, B_PHYS | B_RAW | flags);
 			
 			if ( (iosize = uio_iov_len_at(uio, i)) > MAXPHYSIO_WIRED)
 			        iosize = MAXPHYSIO_WIRED;
+			/*
+			 * make sure we're set to issue a fresh I/O
+			 * in the right direction
+			 */
+			buf_reset(bp, flags);
 
 			/* [set up the buffer for a maximum-sized transfer] */
  			buf_setblkno(bp, uio->uio_offset / blocksize);
 			buf_setcount(bp, iosize);
 			// LP64todo - fix this!
-			buf_setdataptr(bp, CAST_DOWN(caddr_t, uio_iov_base_at(uio, i)));
+			buf_setdataptr(bp, (uintptr_t)CAST_DOWN(caddr_t, uio_iov_base_at(uio, i)));
 			
 			/*
-			 * [call minphys to bound the tranfer size]
+			 * [call f_minphys to bound the tranfer size]
 			 * and remember the amount of data to transfer,
 			 * for later comparison.
 			 */
-			(*minphys)(bp);
+			(*f_minphys)(bp);
 			todo = buf_count(bp);
 
 			/*
@@ -164,12 +186,15 @@ physio(strategy, bp, dev, flags, minphys, uio, blocksize)
 			 *    in the transfer]
 			 */
 
-			if(UIO_SEG_IS_USER_SPACE(uio->uio_segflg))
-				vslock(CAST_USER_ADDR_T(buf_dataptr(bp)),
-				       (user_size_t)todo);
+			if(UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
+				error = vslock(CAST_USER_ADDR_T(buf_dataptr(bp)),
+					       (user_size_t)todo);
+				if (error)
+					goto done;
+			}
 			
-			/* [call strategy to start the transfer] */
-			(*strategy)(bp);
+			/* [call f_strategy to start the transfer] */
+			(*f_strategy)(bp);
 
 
 			/* [wait for the transfer to complete] */
@@ -204,19 +229,10 @@ physio(strategy, bp, dev, flags, minphys, uio, blocksize)
 	}
 
 done:
-	/*
-	 * [clean up the state of the buffer]
-	 * Remember if somebody wants it, so we can wake them up below.
-	 * Also, if we had to steal it, give it back.
-	 */
-
-	buf_clearflags(bp, B_PHYS | B_RAW);
-	if (nobuf)
-		buf_free(bp);
-	else 
-	        {
-		        buf_drop(bp);
-		}
+	if (buf_allocated)
+	        buf_free(bp);
+	else
+		buf_setflags(bp, orig_bflags);
 
 	return (error);
 }
@@ -231,8 +247,7 @@ done:
  * and return the new count;
  */
 u_int
-minphys(bp)
-	struct buf *bp;
+minphys(struct buf *bp)
 {
 
 	buf_setcount(bp, min(MAXPHYS, buf_count(bp)));
@@ -242,9 +257,8 @@ minphys(bp)
 /*
  * Do a read on a device for a user process.
  */
-rawread(dev, uio)
-	dev_t dev;
-	struct uio *uio;
+int
+rawread(dev_t dev, struct uio *uio)
 {
 	return (physio(cdevsw[major(dev)].d_strategy, (struct buf *)NULL,
 	    dev, B_READ, minphys, uio, DEV_BSIZE));
@@ -253,9 +267,8 @@ rawread(dev, uio)
 /*
  * Do a write on a device for a user process.
  */
-rawwrite(dev, uio)
-	dev_t dev;
-	struct uio *uio;
+int
+rawwrite(dev_t dev, struct uio *uio)
 {
 	return (physio(cdevsw[major(dev)].d_strategy, (struct buf *)NULL,
 	    dev, B_WRITE, minphys, uio, DEV_BSIZE));
diff --git a/bsd/kern/kern_proc.c b/bsd/kern/kern_proc.c
index f1a0cc880..3a4441434 100644
--- a/bsd/kern/kern_proc.c
+++ b/bsd/kern/kern_proc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)kern_proc.c	8.4 (Berkeley) 1/4/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /* HISTORY
  *  04-Aug-97  Umesh Vaishampayan (umeshv@apple.com)
  *	Added current_proc_EXTERNAL() function for the use of kernel
@@ -74,12 +86,27 @@
 #include <ufs/ufs/quota.h>
 #include <sys/uio.h>
 #include <sys/malloc.h>
+#include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/ioctl.h>
 #include <sys/tty.h>
 #include <sys/signalvar.h>
 #include <sys/syslog.h>
+#include <sys/sysctl.h>
+#include <sys/sysproto.h>
+#include <sys/kauth.h>
+#include <sys/codesign.h>
 #include <sys/kernel_types.h>
+#include <kern/kalloc.h>
+#include <kern/task.h>
+#include <kern/assert.h>
+#include <vm/vm_protos.h>
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
+#include <libkern/crypto/sha1.h>
 
 /*
  * Structure associated with user cacheing.
@@ -100,192 +127,440 @@ struct pidhashhead *pidhashtbl;
 u_long pidhash;
 struct pgrphashhead *pgrphashtbl;
 u_long pgrphash;
+struct sesshashhead *sesshashtbl;
+u_long sesshash;
+
 struct proclist allproc;
 struct proclist zombproc;
 extern struct tty cons;
 
+#if CONFIG_LCTX
+/*
+ * Login Context
+ */
+static pid_t	lastlcid = 1;
+static int	alllctx_cnt;
+
+#define	LCID_MAX	8192	/* Does this really need to be large? */
+static int	maxlcid = LCID_MAX;
+
+LIST_HEAD(lctxlist, lctx);
+static struct lctxlist alllctx;
+
+lck_mtx_t alllctx_lock;
+lck_grp_t * lctx_lck_grp;
+lck_grp_attr_t * lctx_lck_grp_attr;
+lck_attr_t * lctx_lck_attr;
+
+static	void	lctxinit(void);
+#endif
+
+#define __PROC_INTERNAL_DEBUG 1
 /* Name to give to core files */
 __private_extern__ char corefilename[MAXPATHLEN+1] = {"/cores/core.%P"};
 
 static void orphanpg(struct pgrp *pg);
+void 	proc_name_kdp(task_t t, char * buf, int size);
+char	*proc_name_address(void *p);
+
+static proc_t proc_refinternal_locked(proc_t p);
+static void  pgrp_add(struct pgrp * pgrp, proc_t parent, proc_t child);
+static void pgrp_remove(proc_t p);
+static void pgrp_replace(proc_t p, struct pgrp *pgrp);
+static void pgdelete_dropref(struct pgrp *pgrp);
+static proc_t proc_find_zombref(int pid);
+static void proc_drop_zombref(proc_t p);
+extern void pg_rele_dropref(struct pgrp * pgrp);
+
+struct fixjob_iterargs {
+	struct pgrp * pg;
+	struct session * mysession;
+	int entering;
+};
+
+int fixjob_callback(proc_t, void *);
 
 /*
  * Initialize global process hashing structures.
  */
 void
-procinit()
+procinit(void)
 {
-
 	LIST_INIT(&allproc);
 	LIST_INIT(&zombproc);
 	pidhashtbl = hashinit(maxproc / 4, M_PROC, &pidhash);
 	pgrphashtbl = hashinit(maxproc / 4, M_PROC, &pgrphash);
+	sesshashtbl = hashinit(maxproc / 4, M_PROC, &sesshash);
 	uihashtbl = hashinit(maxproc / 16, M_PROC, &uihash);
+#if CONFIG_LCTX
+	lctxinit();
+#endif
 }
 
 /*
  * Change the count associated with number of processes
- * a given user is using.
+ * a given user is using. This routine protects the uihash
+ * with the list lock
  */
 int
-chgproccnt(uid, diff)
-	uid_t	uid;
-	int	diff;
+chgproccnt(uid_t uid, int diff)
 {
-	register struct uidinfo *uip;
-	register struct uihashhead *uipp;
+	struct uidinfo *uip;
+	struct uidinfo *newuip = NULL;
+	struct uihashhead *uipp;
+	int retval;
 
+again:
+	proc_list_lock();
 	uipp = UIHASH(uid);
 	for (uip = uipp->lh_first; uip != 0; uip = uip->ui_hash.le_next)
 		if (uip->ui_uid == uid)
 			break;
 	if (uip) {
 		uip->ui_proccnt += diff;
-		if (uip->ui_proccnt > 0)
-			return (uip->ui_proccnt);
+		if (uip->ui_proccnt > 0) {
+			retval = uip->ui_proccnt;
+			proc_list_unlock();
+			goto out;
+		}
 		if (uip->ui_proccnt < 0)
 			panic("chgproccnt: procs < 0");
 		LIST_REMOVE(uip, ui_hash);
-		FREE_ZONE(uip, sizeof *uip, M_PROC);
-		return (0);
+		retval = 0;
+		proc_list_unlock();
+		FREE_ZONE(uip, sizeof(*uip), M_PROC);
+		goto out;
 	}
 	if (diff <= 0) {
-		if (diff == 0)
-			return(0);
+		if (diff == 0) {
+			retval = 0;
+			proc_list_unlock();
+			goto out;
+		}
 		panic("chgproccnt: lost user");
 	}
-	MALLOC_ZONE(uip, struct uidinfo *, sizeof(*uip), M_PROC, M_WAITOK);
-	if (uip == NULL)
+	if (newuip != NULL) {
+		uip = newuip;
+		newuip = NULL;
+		LIST_INSERT_HEAD(uipp, uip, ui_hash);
+		uip->ui_uid = uid;
+		uip->ui_proccnt = diff;
+		retval = diff;
+		proc_list_unlock();
+		goto out;
+	}
+	proc_list_unlock();
+	MALLOC_ZONE(newuip, struct uidinfo *, sizeof(*uip), M_PROC, M_WAITOK);
+	if (newuip == NULL)
 		panic("chgproccnt: M_PROC zone depleted");
-	LIST_INSERT_HEAD(uipp, uip, ui_hash);
-	uip->ui_uid = uid;
-	uip->ui_proccnt = diff;
-	return (diff);
+	goto again;
+out:
+	if (newuip != NULL) 
+		FREE_ZONE(newuip, sizeof(*uip), M_PROC);
+	return(retval);
 }
 
 /*
  * Is p an inferior of the current process?
  */
 int
-inferior(p)
-	register struct proc *p;
+inferior(proc_t p)
 {
+	int retval = 0;
 
+	proc_list_lock();
 	for (; p != current_proc(); p = p->p_pptr)
-		if (p->p_pid == 0)
-			return (0);
-	return (1);
+		if (p->p_pid == 0) 
+			goto out;
+	retval = 1;
+out:
+	proc_list_unlock();
+	return(retval);
 }
+
 /*
  * Is p an inferior of t ?
  */
 int
-isinferior(struct proc *p, struct proc *t)
+isinferior(proc_t p, proc_t t)
 {
+int retval = 0;
 
 	/* if p==t they are not inferior */
 	if (p == t)
 		return(0);
+
+	proc_list_lock();
 	for (; p != t; p = p->p_pptr)
 		if (p->p_pid == 0)
-			return (0);
-	return (1);
+			goto out;
+	retval = 1;
+out:
+	proc_list_unlock();
+	return(retval);
 }
 
 int
 proc_isinferior(int pid1, int pid2)
 {
-	proc_t p;
-	proc_t t;
+	proc_t p = PROC_NULL;
+	proc_t t = PROC_NULL;
+	int retval = 0;
+	
+	if (((p = proc_find(pid1)) != (proc_t)0 ) && ((t = proc_find(pid2)) != (proc_t)0))
+		retval = isinferior(p, t);
 
-	if (((p = pfind(pid1)) != (struct proc *)0 ) && ((t = pfind(pid2)) != (struct proc *)0))
-		return (isinferior(p, t));
-	return(0);
+	if (p != PROC_NULL)
+		proc_rele(p);
+	if (t != PROC_NULL)
+		proc_rele(t);
+
+	return(retval);
 }
 
 proc_t
 proc_find(int pid)
 {
-	return(pfind(pid));
+	return(proc_findinternal(pid, 0));
 }
 
-int 
-proc_rele(__unused proc_t p)
+proc_t
+proc_findinternal(int pid, int locked)
 {
-	return(0);
+	proc_t p = PROC_NULL;
+
+	if (locked == 0) {
+		proc_list_lock();
+	}
+
+	p = pfind_locked(pid);
+	if ((p == PROC_NULL) || (p != proc_refinternal_locked(p)))
+		p = PROC_NULL;
+
+	if (locked == 0) {
+		proc_list_unlock();
+	}
+
+	return(p);
 }
 
-proc_t
-proc_self()
+int 
+proc_rele(proc_t p)
 {
-	return(current_proc());
+	proc_list_lock();
+	proc_rele_locked(p);
+	proc_list_unlock();
+
+	return(0);
 }
 
 proc_t
-proc_findref(int pid)
+proc_self(void)
 {
-	boolean_t funnel_state;
-	proc_t p;
+	struct proc * p;
 
-	funnel_state = thread_funnel_set(kernel_flock,TRUE);
-	p = pfind(pid);
-	
-	if (p != proc_refinternal(p, 1))
+	p = current_proc();
+
+	proc_list_lock();
+	if (p != proc_refinternal_locked(p))
 		p = PROC_NULL;
-		
-	thread_funnel_set(kernel_flock, funnel_state);
+	proc_list_unlock();
 	return(p);
 }
 
-void
-proc_dropref(proc_t p)
+
+static proc_t
+proc_refinternal_locked(proc_t p)
 {
+	proc_t p1 = p;
+	
+	/* if process still in creation return failure */
+	if ((p == PROC_NULL) || ((p->p_listflag & P_LIST_INCREATE) != 0))
+			return (PROC_NULL);
+	/* do not return process marked for termination */
+	if ((p->p_stat != SZOMB) && ((p->p_listflag & P_LIST_EXITED) == 0) && ((p->p_listflag & (P_LIST_DRAINWAIT | P_LIST_DRAIN | P_LIST_DEAD)) == 0))
+		p->p_refcount++;
+	else 
+		p1 = PROC_NULL;
 
-	proc_dropinternal(p, 0);
+	return(p1);
 }
 
+void
+proc_rele_locked(proc_t p)
+{
 
-proc_t
-proc_refinternal(proc_t p, int funneled)
+	if (p->p_refcount > 0) {
+		p->p_refcount--;
+		if ((p->p_refcount == 0) && ((p->p_listflag & P_LIST_DRAINWAIT) == P_LIST_DRAINWAIT)) {
+			p->p_listflag &= ~P_LIST_DRAINWAIT;
+			wakeup(&p->p_refcount);
+		}
+	} else
+		panic("proc_rele_locked  -ve ref\n");
+
+}
+
+static proc_t
+proc_find_zombref(int pid)
 {
+	proc_t p1 = PROC_NULL;
+	proc_t p = PROC_NULL;
 
-	proc_t p1 = p;
-	boolean_t funnel_state = TRUE; 	/* need to init just to avoid warnings and build failure */
+	proc_list_lock();
 
-	if (funneled == 0)
-		funnel_state = thread_funnel_set(kernel_flock,TRUE);
-	
-	if ((p != PROC_NULL) &&(p->p_stat != SZOMB) && ((p->p_lflag & (P_LREFDRAINWAIT | P_LREFDRAIN | P_LREFDEAD)) == 0))
-		p->p_internalref++;
-	else 
+	p = pfind_locked(pid);
+
+	/* if process still in creation return NULL */
+	if ((p == PROC_NULL) || ((p->p_listflag & P_LIST_INCREATE) != 0)) {
+		proc_list_unlock();
+		return (p1);
+	}
+
+	/* if process has not started exit or  is being reaped, return NULL */
+	if (((p->p_listflag & P_LIST_EXITED) != 0) && ((p->p_listflag & P_LIST_WAITING) == 0)) {
+		p->p_listflag |=  P_LIST_WAITING;
+		p1 = p;
+	} else 
 		p1 = PROC_NULL;
 
-	if (funneled == 0)
-		thread_funnel_set(kernel_flock,funnel_state);
+	proc_list_unlock();
+
 	return(p1);
 }
 
+static void
+proc_drop_zombref(proc_t p)
+{
+	proc_list_lock();
+	if ((p->p_listflag & P_LIST_WAITING) ==  P_LIST_WAITING) {
+		p->p_listflag &= ~P_LIST_WAITING;
+		wakeup(&p->p_stat);
+	}
+	proc_list_unlock();
+}
+
+
 void
-proc_dropinternal(proc_t p, int funneled)
+proc_refdrain(proc_t p)
+{
+
+	proc_list_lock();
+
+	p->p_listflag |= P_LIST_DRAIN;
+	while (p->p_refcount) {
+		p->p_listflag |= P_LIST_DRAINWAIT;
+		msleep(&p->p_refcount, proc_list_mlock, 0, "proc_refdrain", 0) ;
+	}
+	p->p_listflag &= ~P_LIST_DRAIN;
+	p->p_listflag |= P_LIST_DEAD;
+
+	proc_list_unlock();
+
+
+}
+
+proc_t 
+proc_parentholdref(proc_t p)
 {
-	boolean_t funnel_state = TRUE; /* need to init just to avoid warnings and build failure */
+	proc_t parent = PROC_NULL;
+	proc_t pp;
+	int loopcnt = 0;
+
 
-	if (funneled == 0)
-		funnel_state = thread_funnel_set(kernel_flock,TRUE);
+	proc_list_lock();
+loop:
+	pp = p->p_pptr;
+	if ((pp == PROC_NULL) || (pp->p_stat == SZOMB) || ((pp->p_listflag & (P_LIST_CHILDDRSTART | P_LIST_CHILDDRAINED)) == (P_LIST_CHILDDRSTART | P_LIST_CHILDDRAINED))) {
+		parent = PROC_NULL;
+		goto out;
+	}
+		
+	if ((pp->p_listflag & (P_LIST_CHILDDRSTART | P_LIST_CHILDDRAINED)) == P_LIST_CHILDDRSTART) {
+		pp->p_listflag |= P_LIST_CHILDDRWAIT;
+		msleep(&pp->p_childrencnt, proc_list_mlock, 0, "proc_parent", 0);
+		loopcnt++;
+		if (loopcnt == 5) {
+			parent = PROC_NULL;
+			goto out;
+		}
+		goto loop;
+	}
 
-	if (p->p_internalref > 0) {
-		p->p_internalref--;
-		if ((p->p_internalref == 0) && ((p->p_lflag & P_LREFDRAINWAIT) == P_LREFDRAINWAIT)) {
-			p->p_lflag &= ~P_LREFDRAINWAIT;
-			wakeup(&p->p_internalref);
+	if ((pp->p_listflag & (P_LIST_CHILDDRSTART | P_LIST_CHILDDRAINED)) == 0) {
+		pp->p_parentref++;
+		parent = pp;
+		goto out;
+	}
+	
+out:
+	proc_list_unlock();
+	return(parent);
+}
+int 
+proc_parentdropref(proc_t p, int listlocked)
+{
+	if (listlocked == 0)
+		proc_list_lock();
+
+	if (p->p_parentref > 0) {
+		p->p_parentref--;
+		if ((p->p_parentref == 0) && ((p->p_listflag & P_LIST_PARENTREFWAIT) == P_LIST_PARENTREFWAIT)) {
+			p->p_listflag &= ~P_LIST_PARENTREFWAIT;
+			wakeup(&p->p_parentref);
 		}
 	} else
-		printf("proc_dropreg -ve ref\n");
+		panic("proc_parentdropref  -ve ref\n");
+	if (listlocked == 0)
+		proc_list_unlock();
+
+	return(0);
+}
 
-	if (funneled == 0)
-		thread_funnel_set(kernel_flock,funnel_state);
+void
+proc_childdrainstart(proc_t p)
+{
+#if __PROC_INTERNAL_DEBUG
+	if ((p->p_listflag & P_LIST_CHILDDRSTART) == P_LIST_CHILDDRSTART)
+		panic("proc_childdrainstart: childdrain already started\n");
+#endif
+	p->p_listflag |= P_LIST_CHILDDRSTART;
+	/* wait for all that hold parentrefs to drop */
+	while (p->p_parentref > 0) {
+		p->p_listflag |= P_LIST_PARENTREFWAIT;
+		msleep(&p->p_parentref, proc_list_mlock, 0, "proc_childdrainstart", 0) ;
+	}
+}
+
+
+void
+proc_childdrainend(proc_t p)
+{
+#if __PROC_INTERNAL_DEBUG
+	if (p->p_childrencnt > 0)
+		panic("exiting: children stil hanging around\n");
+#endif
+	p->p_listflag |= P_LIST_CHILDDRAINED;
+	if ((p->p_listflag & (P_LIST_CHILDLKWAIT |P_LIST_CHILDDRWAIT)) != 0) {
+		p->p_listflag &= ~(P_LIST_CHILDLKWAIT |P_LIST_CHILDDRWAIT);
+		wakeup(&p->p_childrencnt);
+	}
 }
 
+void
+proc_checkdeadrefs(proc_t p)
+{
+//#if __PROC_INTERNAL_DEBUG
+	if ((p->p_listflag  & P_LIST_INHASH) != 0)
+		panic("proc being freed and still in hash %x: %x\n", (unsigned int)p, (unsigned int)p->p_listflag);
+	if (p->p_childrencnt != 0)
+		panic("proc being freed and pending children cnt %x:%x\n", (unsigned int)p, (unsigned int)p->p_childrencnt);
+	if (p->p_refcount != 0)
+		panic("proc being freed and pending refcount %x:%x\n", (unsigned int)p, (unsigned int)p->p_refcount);
+	if (p->p_parentref != 0)
+		panic("proc being freed and pending parentrefs %x:%x\n", (unsigned int)p, (unsigned int)p->p_parentref);
+//#endif
+}
 
 int
 proc_pid(proc_t p)
@@ -296,48 +571,76 @@ proc_pid(proc_t p)
 int 
 proc_ppid(proc_t p)
 {
-	if (p->p_pptr != (struct proc *)0) 
-		return(p->p_pptr->p_pid);
-	return(0);
+		return(p->p_ppid);
 }
 
 int 
 proc_selfpid(void)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
 	return(p->p_pid);
 }
 
-
 int 
 proc_selfppid(void)
 {
-	struct proc *p = current_proc();
-	if (p->p_pptr)
-		return(p->p_pptr->p_pid);
-	else
-		return(0);
+	proc_t p = current_proc();
+	return(p->p_ppid);
+}
+
+proc_t 
+proc_parent(proc_t p)
+{
+	proc_t parent;
+	proc_t pp;
+
+	proc_list_lock();
+loop:
+	pp = p->p_pptr;
+	parent =  proc_refinternal_locked(pp);
+	if ((parent == PROC_NULL) && (pp != PROC_NULL) && (pp->p_stat != SZOMB) && ((pp->p_listflag & P_LIST_EXITED) != 0) && ((pp->p_listflag & P_LIST_CHILDDRAINED)== 0)){
+		pp->p_listflag |= P_LIST_CHILDLKWAIT;
+		msleep(&pp->p_childrencnt, proc_list_mlock, 0, "proc_parent", 0);
+		goto loop;
+	}
+	proc_list_unlock();
+	return(parent);
 }
 
+
 void
 proc_name(int pid, char * buf, int size)
 {
-	struct proc  *p;
+	proc_t p;
 
-	if ((p = pfind(pid))!= (struct proc *)0) {
-		strncpy(buf, &p->p_comm[0], size);
-		buf[size-1] = 0;
+	if ((p = proc_find(pid)) != PROC_NULL) {
+		strlcpy(buf, &p->p_comm[0], size);
+		proc_rele(p);
 	}
 }
 
+void
+proc_name_kdp(task_t t, char * buf, int size)
+{
+	proc_t p = get_bsdtask_info(t);
+
+	if (p != PROC_NULL)
+		strlcpy(buf, &p->p_comm[0], size);
+}
+
+char *
+proc_name_address(void *p)
+{
+	return &((proc_t)p)->p_comm[0];
+}
+
 void
 proc_selfname(char * buf, int  size)
 {
-	struct proc  *p;
+	proc_t p;
 
-	if ((p = current_proc())!= (struct proc *)0) {
-		strncpy(buf, &p->p_comm[0], size);
-		buf[size-1] = 0;
+	if ((p = current_proc())!= (proc_t)0) {
+		strlcpy(buf, &p->p_comm[0], size);
 	}
 }
 
@@ -346,8 +649,9 @@ proc_signal(int pid, int signum)
 {
 	proc_t p;
 
-	if ((p = pfind(pid))!= (struct proc *)0) {
+	if ((p = proc_find(pid)) != PROC_NULL) {
 			psignal(p, signum);
+			proc_rele(p);
 	}	
 }
 
@@ -355,11 +659,14 @@ int
 proc_issignal(int pid, sigset_t mask)
 {
 	proc_t p;
+	int error=0;
 
-	if ((p = pfind(pid))!= (struct proc *)0) {
-		return(proc_pendingsignals(p, mask));
+	if ((p = proc_find(pid)) != PROC_NULL) {
+		error = proc_pendingsignals(p, mask);
+		proc_rele(p);
 	}	
-	return(0);
+
+	return(error);
 }
 
 int
@@ -379,11 +686,10 @@ proc_exiting(proc_t p)
 	int retval = 0;
 
 	if (p)
-		retval = p->p_flag & P_WEXIT;
+		retval = p->p_lflag & P_LEXIT;
 	return(retval? 1: 0);
 }
 
-
 int
 proc_forcequota(proc_t p)
 {
@@ -409,23 +715,57 @@ proc_tbe(proc_t p)
 int
 proc_suser(proc_t p)
 {
-	return(suser(p->p_ucred, NULL));
-	
+	kauth_cred_t my_cred;
+	int error;
+
+	my_cred = kauth_cred_proc_ref(p);
+	error = suser(my_cred, &p->p_acflag);
+	kauth_cred_unref(&my_cred);
+	return(error);
 }
 
+/*      
+ * Obtain the first thread in a process
+ *
+ * XXX This is a bad thing to do; it exists predominantly to support the
+ * XXX use of proc_t's in places that should really be using
+ * XXX thread_t's instead.  This maintains historical behaviour, but really
+ * XXX needs an audit of the context (proxy vs. not) to clean up.
+ */
+thread_t
+proc_thread(proc_t proc)                                                
+{           
+        uthread_t uth = TAILQ_FIRST(&proc->p_uthlist);
+
+        if (uth != NULL)
+                return(uth->uu_context.vc_thread);
+
+	return(NULL);
+}       
+
 kauth_cred_t
 proc_ucred(proc_t p)
 {
 	return(p->p_ucred);
 }
 
-
 int
 proc_is64bit(proc_t p)
 {
 	return(IS_64BIT_PROCESS(p));
 }
 
+void
+bsd_set_dependency_capable(task_t task)
+{
+    proc_t p = get_bsdtask_info(task);
+
+    if (p) {
+	OSBitOrAtomic(P_DEPENDENCY_CAPABLE, (UInt32 *)&p->p_flag);
+    }
+}
+
+
 /* LP64todo - figure out how to identify 64-bit processes if NULL procp */
 int
 IS_64BIT_PROCESS(proc_t p)
@@ -436,48 +776,79 @@ IS_64BIT_PROCESS(proc_t p)
 		return(0);
 }
 
-
 /*
  * Locate a process by number
  */
-struct proc *
-pfind(pid)
-	register pid_t pid;
+proc_t
+pfind_locked(pid_t pid)
 {
-	register struct proc *p;
+	proc_t p;
+#ifdef DEBUG
+	proc_t q;
+#endif
 
 	if (!pid)
 		return (kernproc);
 
-	for (p = PIDHASH(pid)->lh_first; p != 0; p = p->p_hash.le_next)
-		if (p->p_pid == pid)
+	for (p = PIDHASH(pid)->lh_first; p != 0; p = p->p_hash.le_next) {
+		if (p->p_pid == pid) {
+#ifdef DEBUG
+			for (q = p->p_hash.le_next; q != 0; q = q->p_hash.le_next) {
+				if ((p !=q) && (q->p_pid == pid))	
+					panic("two procs with same pid %x:%x:%d:%d\n", (unsigned int)p, (unsigned int)q, p->p_pid, q->p_pid);
+			}
+#endif
 			return (p);
+		}
+	}
 	return (NULL);
 }
 
 /*
  * Locate a zombie by PID
  */
-__private_extern__ struct proc *
-pzfind(pid)
-	register pid_t pid;
+__private_extern__ proc_t
+pzfind(pid_t pid)
 {
-	register struct proc *p;
+	proc_t p;
+
+
+	proc_list_lock();
 
 	for (p = zombproc.lh_first; p != 0; p = p->p_list.le_next)
 		if (p->p_pid == pid)
-			return (p);
-	return (NULL);
+			break;
+
+	proc_list_unlock();
+
+	return (p);
 }
 
 /*
  * Locate a process group by number
  */
+
 struct pgrp *
-pgfind(pgid)
-	register pid_t pgid;
+pgfind(pid_t pgid)
 {
-	register struct pgrp *pgrp;
+	struct pgrp * pgrp;
+
+	proc_list_lock();
+	pgrp = pgfind_internal(pgid);
+	if ((pgrp == NULL) || ((pgrp->pg_listflags & PGRP_FLAG_TERMINATE) != 0))
+		pgrp = PGRP_NULL;
+	else
+		pgrp->pg_refcount++;
+	proc_list_unlock();
+	return(pgrp);
+}
+
+
+
+struct pgrp *
+pgfind_internal(pid_t pgid)
+{
+	struct pgrp *pgrp;
 
 	for (pgrp = PGRPHASH(pgid)->lh_first; pgrp != 0; pgrp = pgrp->pg_hash.le_next)
 		if (pgrp->pg_id == pgid)
@@ -485,27 +856,108 @@ pgfind(pgid)
 	return (NULL);
 }
 
+void
+pg_rele(struct pgrp * pgrp)
+{
+	if(pgrp == PGRP_NULL)
+		return;
+	pg_rele_dropref(pgrp);
+}
+
+void
+pg_rele_dropref(struct pgrp * pgrp)
+{
+	proc_list_lock();
+	if ((pgrp->pg_refcount == 1) && ((pgrp->pg_listflags & PGRP_FLAG_TERMINATE) == PGRP_FLAG_TERMINATE)) {
+		proc_list_unlock();
+		pgdelete_dropref(pgrp);
+		return;
+	}
+
+	pgrp->pg_refcount--;
+	proc_list_unlock();
+}
+
+struct session *
+session_find_internal(pid_t sessid)
+{
+	struct session *sess;
+
+	for (sess = SESSHASH(sessid)->lh_first; sess != 0; sess = sess->s_hash.le_next)
+		if (sess->s_sid == sessid)
+			return (sess);
+	return (NULL);
+}
+
+
+/*
+ * Make a new process ready to become a useful member of society by making it
+ * visible in all the right places and initialize its own lists to empty.
+ *
+ * Parameters:	parent			The parent of the process to insert
+ *		child			The child process to insert
+ *
+ * Returns:	(void)
+ *
+ * Notes:	Insert a child process into the parents process group, assign
+ *		the child the parent process pointer and PPID of the parent,
+ *		place it on the parents p_children list as a sibling,
+ *		initialize its own child list, place it in the allproc list,
+ *		insert it in the proper hash bucket, and initialize its
+ *		event list.
+ */
+void
+pinsertchild(proc_t parent, proc_t child)
+{
+	struct pgrp * pg;
+
+	LIST_INIT(&child->p_children);
+	TAILQ_INIT(&child->p_evlist);
+	child->p_pptr = parent;
+	child->p_ppid = parent->p_pid;
+
+	pg = proc_pgrp(parent);
+	pgrp_add(pg, parent, child);
+	pg_rele(pg);
+
+	proc_list_lock();
+	parent->p_childrencnt++;
+	LIST_INSERT_HEAD(&parent->p_children, child, p_sibling);
+
+	LIST_INSERT_HEAD(&allproc, child, p_list);
+	/* mark the completion of proc creation */
+	child->p_listflag &= ~P_LIST_INCREATE;
+
+	proc_list_unlock();
+
+}
 
 /*
  * Move p to a new or existing process group (and session)
+ *
+ * Returns:	0			Success
+ *		ESRCH			No such process
  */
 int
-enterpgrp(p, pgid, mksess)
-	register struct proc *p;
-	pid_t pgid;
-	int mksess;
+enterpgrp(proc_t p, pid_t pgid, int mksess)
 {
-	register struct pgrp *pgrp = pgfind(pgid);
+	struct pgrp *pgrp;
+	struct pgrp *mypgrp;
+	struct session * procsp;
+
+	pgrp = pgfind(pgid);
+	mypgrp = proc_pgrp(p);
+	procsp = proc_session(p);
 
 #if DIAGNOSTIC
 	if (pgrp != NULL && mksess)	/* firewalls */
 		panic("enterpgrp: setsid into non-empty pgrp");
-	if (SESS_LEADER(p))
+	if (SESS_LEADER(p, procsp))
 		panic("enterpgrp: session leader attempted setpgrp");
 #endif
-	if (pgrp == NULL) {
+	if (pgrp == PGRP_NULL) {
 		pid_t savepid = p->p_pid;
-		struct proc *np;
+		proc_t np = PROC_NULL;
 		/*
 		 * new process group
 		 */
@@ -517,12 +969,19 @@ enterpgrp(p, pgid, mksess)
 		    M_WAITOK);
 		if (pgrp == NULL)
 			panic("enterpgrp: M_PGRP zone depleted");
-		if ((np = pfind(savepid)) == NULL || np != p) {
+		if ((np = proc_find(savepid)) == NULL || np != p) {
+			if (np != PROC_NULL)
+				proc_rele(np);
+			if (mypgrp != PGRP_NULL)
+				pg_rele(mypgrp);
+			if (procsp != SESSION_NULL)
+				session_rele(procsp);
 			FREE_ZONE(pgrp, sizeof(struct pgrp), M_PGRP);
 			return (ESRCH);
 		}
+		proc_rele(np);
 		if (mksess) {
-			register struct session *sess;
+			struct session *sess;
 
 			/*
 			 * new session
@@ -536,85 +995,153 @@ enterpgrp(p, pgid, mksess)
 			sess->s_count = 1;
 			sess->s_ttyvp = NULL;
 			sess->s_ttyp = NULL;
-			bcopy(p->p_session->s_login, sess->s_login,
+			sess->s_flags = 0;
+			sess->s_listflags = 0;
+			sess->s_ttypgrpid = NO_PID;
+			lck_mtx_init(&sess->s_mlock, proc_lck_grp, proc_lck_attr);
+			bcopy(procsp->s_login, sess->s_login,
 			    sizeof(sess->s_login));
-			p->p_flag &= ~P_CONTROLT;
+			OSBitAndAtomic(~((uint32_t)P_CONTROLT), (UInt32 *)&p->p_flag);
+			proc_list_lock();
+			LIST_INSERT_HEAD(SESSHASH(sess->s_sid), sess, s_hash);
+			proc_list_unlock();
 			pgrp->pg_session = sess;
 #if DIAGNOSTIC
 			if (p != current_proc())
 				panic("enterpgrp: mksession and p != curproc");
 #endif
 		} else {
-			pgrp->pg_session = p->p_session;
+			proc_list_lock();
+			pgrp->pg_session = procsp;
+			
+			if ((pgrp->pg_session->s_listflags & (S_LIST_TERM | S_LIST_DEAD)) != 0)
+				panic("enterpgrp:  providing ref to terminating session ");	
 			pgrp->pg_session->s_count++;
+			proc_list_unlock();
 		}
 		pgrp->pg_id = pgid;
+		lck_mtx_init(&pgrp->pg_mlock, proc_lck_grp, proc_lck_attr);
 		LIST_INIT(&pgrp->pg_members);
-		LIST_INSERT_HEAD(PGRPHASH(pgid), pgrp, pg_hash);
+		pgrp->pg_membercnt = 0;
 		pgrp->pg_jobc = 0;
-	} else if (pgrp == p->p_pgrp)
+		proc_list_lock();
+		pgrp->pg_refcount = 1;
+		pgrp->pg_listflags = 0;
+		LIST_INSERT_HEAD(PGRPHASH(pgid), pgrp, pg_hash);
+		proc_list_unlock();
+	} else if (pgrp == mypgrp) {
+		pg_rele(pgrp);
+		if (mypgrp != NULL)
+			pg_rele(mypgrp);
+		if (procsp != SESSION_NULL)
+			session_rele(procsp);
 		return (0);
+	}
 
+	if (procsp != SESSION_NULL)
+		session_rele(procsp);
 	/*
 	 * Adjust eligibility of affected pgrps to participate in job control.
 	 * Increment eligibility counts before decrementing, otherwise we
 	 * could reach 0 spuriously during the first call.
 	 */
 	fixjobc(p, pgrp, 1);
-	fixjobc(p, p->p_pgrp, 0);
+	fixjobc(p, mypgrp, 0);
 
-	LIST_REMOVE(p, p_pglist);
-	if (p->p_pgrp->pg_members.lh_first == 0)
-		pgdelete(p->p_pgrp);
-	p->p_pgrp = pgrp;
-	LIST_INSERT_HEAD(&pgrp->pg_members, p, p_pglist);
-	return (0);
+	if(mypgrp != PGRP_NULL)
+		pg_rele(mypgrp);
+	pgrp_replace(p, pgrp);
+	pg_rele(pgrp);
+
+	return(0);
 }
 
 /*
  * remove process from process group
  */
 int
-leavepgrp(p)
-	register struct proc *p;
+leavepgrp(proc_t p)
 {
 
-	LIST_REMOVE(p, p_pglist);
-	if (p->p_pgrp->pg_members.lh_first == 0)
-		pgdelete(p->p_pgrp);
-	p->p_pgrp = 0;
+	pgrp_remove(p);
 	return (0);
 }
 
 /*
  * delete a process group
  */
-void
-pgdelete(pgrp)
-	register struct pgrp *pgrp;
+static void
+pgdelete_dropref(struct pgrp *pgrp)
 {
 	struct tty * ttyp;
+	boolean_t fstate;
+	int emptypgrp  = 1;
+	struct session *sessp;
 
-	ttyp = pgrp->pg_session->s_ttyp;
-	if (ttyp != NULL && pgrp->pg_session->s_ttyp->t_pgrp == pgrp) {
-		pgrp->pg_session->s_ttyp->t_pgrp = NULL;
+
+	pgrp_lock(pgrp);
+	if (pgrp->pg_membercnt != 0) {
+		emptypgrp = 0;
+	}
+	pgrp_unlock(pgrp);
+
+	proc_list_lock();
+	pgrp->pg_refcount--;
+	if ((emptypgrp == 0) || (pgrp->pg_membercnt != 0)) {
+		proc_list_unlock();
+		return;
+	}
+
+	pgrp->pg_listflags |= PGRP_FLAG_TERMINATE;
+	
+	if (pgrp->pg_refcount > 0) {
+		proc_list_unlock();
+		return;
 	}
+
+	pgrp->pg_listflags |= PGRP_FLAG_DEAD;
 	LIST_REMOVE(pgrp, pg_hash);
-	if (--pgrp->pg_session->s_count == 0) {
-		if (ttyp != NULL && (ttyp->t_session == pgrp->pg_session))
-			ttyp->t_session = 0;
-		FREE_ZONE(pgrp->pg_session, sizeof(struct session), M_SESSION);
+
+	proc_list_unlock();
+	
+	fstate = thread_funnel_set(kernel_flock, TRUE);
+	
+	ttyp = pgrp->pg_session->s_ttyp;
+	if ((ttyp != NULL) && (pgrp->pg_session->s_ttyp->t_pgrp == pgrp)) {
+		pgrp->pg_session->s_ttyp->t_pgrp = NULL;
+		pgrp->pg_session->s_ttypgrpid = NO_PID;
 	}
-	FREE_ZONE(pgrp, sizeof *pgrp, M_PGRP);
+	(void) thread_funnel_set(kernel_flock, fstate);
+
+	proc_list_lock();
+
+	sessp = pgrp->pg_session;
+	if ((sessp->s_listflags & (S_LIST_TERM | S_LIST_DEAD)) != 0)
+			panic("pg_deleteref: manipulating refs of already terminating session");
+	if (--sessp->s_count == 0) {
+		if ((sessp->s_listflags & (S_LIST_TERM | S_LIST_DEAD)) != 0)
+			panic("pg_deleteref: terminating already terminated session");
+		sessp->s_listflags |= S_LIST_TERM;
+		ttyp = sessp->s_ttyp;
+		LIST_REMOVE(sessp, s_hash);
+		proc_list_unlock();
+		fstate = thread_funnel_set(kernel_flock, TRUE);
+		if (ttyp != NULL && ttyp->t_session == sessp)
+			ttyp->t_session = NULL;
+		(void) thread_funnel_set(kernel_flock, fstate);
+		proc_list_lock();
+		sessp->s_listflags |= S_LIST_DEAD;
+		if (sessp->s_count != 0)
+			panic("pg_deleteref: freeing session in use");	
+		proc_list_unlock();
+		lck_mtx_destroy(&sessp->s_mlock, proc_lck_grp);
+		FREE_ZONE(sessp, sizeof(struct session), M_SESSION);
+	} else
+		proc_list_unlock();
+	lck_mtx_destroy(&pgrp->pg_mlock, proc_lck_grp);
+	FREE_ZONE(pgrp, sizeof(*pgrp), M_PGRP);
 }
 
-void
-sessrele(sess)
-	struct session *sess;
-{
-	if (--sess->s_count == 0)
-		FREE_ZONE(sess, sizeof (struct session), M_SESSION);
-}
 
 /*
  * Adjust pgrp jobc counters when specified process changes process group.
@@ -626,38 +1153,90 @@ sessrele(sess)
  * entering == 0 => p is leaving specified group.
  * entering == 1 => p is entering specified group.
  */
-void
-fixjobc(struct proc *p, struct pgrp *pgrp, int entering)
+int
+fixjob_callback(proc_t p, void * arg)
 {
-	register struct pgrp *hispgrp;
-	register struct session *mysession = pgrp->pg_session;
+	struct fixjob_iterargs *fp;
+	struct pgrp * pg, *hispg;
+	struct session * mysession, *hissess;
+	int entering;
+
+	fp = (struct fixjob_iterargs *)arg;
+	pg = fp->pg;
+	mysession = fp->mysession;
+	entering = fp->entering;
+
+	hispg = proc_pgrp(p);
+	hissess = proc_session(p);
+
+	if ((hispg  != pg) &&
+	    (hissess == mysession)) {
+		pgrp_lock(hispg);
+		if (entering) {
+			hispg->pg_jobc++;
+			pgrp_unlock(hispg);
+		} else if (--hispg->pg_jobc == 0) {
+			pgrp_unlock(hispg);
+			orphanpg(hispg);
+		} else
+			pgrp_unlock(hispg);
+	}
+	if (hissess != SESSION_NULL)
+		session_rele(hissess);
+	if (hispg != PGRP_NULL)
+		pg_rele(hispg);
 
-	/*
-	 * Check p's parent to see whether p qualifies its own process
+	return(PROC_RETURNED);
+}
+
+void
+fixjobc(proc_t p, struct pgrp *pgrp, int entering)
+{
+	struct pgrp *hispgrp = PGRP_NULL;
+	struct session *hissess = SESSION_NULL;
+	struct session *mysession = pgrp->pg_session;
+	proc_t parent;
+	struct fixjob_iterargs fjarg;
+
+	parent = proc_parent(p);
+	if (parent != PROC_NULL) {
+		hispgrp = proc_pgrp(parent);	
+		hissess = proc_session(parent);
+		proc_rele(parent);
+	}
+
+
+	/*
+	 * Check p's parent to see whether p qualifies its own process
 	 * group; if so, adjust count for p's process group.
 	 */
-	if ((hispgrp = p->p_pptr->p_pgrp) != pgrp &&
-	    hispgrp->pg_session == mysession) {
-		if (entering)
+	if ((hispgrp != pgrp) &&
+	    (hissess == mysession)) {
+		pgrp_lock(pgrp);
+		if (entering) {
 			pgrp->pg_jobc++;
-		else if (--pgrp->pg_jobc == 0)
+			pgrp_unlock(pgrp);
+		 }else if (--pgrp->pg_jobc == 0) {
+			pgrp_unlock(pgrp);
 			orphanpg(pgrp);
+		} else
+			pgrp_unlock(pgrp);
 	}
 
+	if (hissess != SESSION_NULL)
+		session_rele(hissess);
+	if (hispgrp != PGRP_NULL)
+		pg_rele(hispgrp);
+
 	/*
 	 * Check this process' children to see whether they qualify
 	 * their process groups; if so, adjust counts for children's
 	 * process groups.
 	 */
-	for (p = p->p_children.lh_first; p != 0; p = p->p_sibling.le_next)
-		if ((hispgrp = p->p_pgrp) != pgrp &&
-		    hispgrp->pg_session == mysession &&
-		    p->p_stat != SZOMB) {
-			if (entering)
-				hispgrp->pg_jobc++;
-			else if (--hispgrp->pg_jobc == 0)
-				orphanpg(hispgrp);
-		}
+	fjarg.pg = pgrp;
+	fjarg.mysession = mysession;
+	fjarg.entering = entering;
+	proc_childrenwalk(p, fixjob_callback, &fjarg);
 }
 
 /* 
@@ -666,61 +1245,83 @@ fixjobc(struct proc *p, struct pgrp *pgrp, int entering)
  * hang-up all process in that group.
  */
 static void
-orphanpg(struct pgrp *pg)
+orphanpg(struct pgrp * pgrp)
 {
-	register struct proc *p;
-
-	for (p = pg->pg_members.lh_first; p != 0; p = p->p_pglist.le_next) {
+	proc_t p;
+	pid_t * pid_list;
+	int count, pidcount, i, alloc_count;
+
+	if (pgrp == PGRP_NULL)
+		return;
+	count = 0;
+	pgrp_lock(pgrp);
+	for (p = pgrp->pg_members.lh_first; p != 0; p = p->p_pglist.le_next) {
 		if (p->p_stat == SSTOP) {
-			for (p = pg->pg_members.lh_first; p != 0;
-			    p = p->p_pglist.le_next) {
-				pt_setrunnable(p);
-				psignal(p, SIGHUP);
-				psignal(p, SIGCONT);
-			}
-			return;
+			for (p = pgrp->pg_members.lh_first; p != 0;
+				p = p->p_pglist.le_next) 
+				count++;
+			break;	/* ??? stops after finding one.. */
 		}
 	}
-}
-
-#ifdef DEBUG
-void pgrpdump(void);	/* forward declare here (called from debugger) */
-
-void
-pgrpdump(void)
-{
-	struct pgrp *pgrp;
-	struct proc *p;
-	u_long i;
-
-	for (i = 0; i <= pgrphash; i++) {
-		if ((pgrp = pgrphashtbl[i].lh_first) != NULL) {
-			printf("\tindx %d\n", i);
-			for (; pgrp != 0; pgrp = pgrp->pg_hash.le_next) {
-				printf("\tpgrp 0x%08x, pgid %d, sess %p, sesscnt %d, mem %p\n",
-				    pgrp, pgrp->pg_id, pgrp->pg_session,
-				    pgrp->pg_session->s_count,
-				    pgrp->pg_members.lh_first);
-				for (p = pgrp->pg_members.lh_first; p != 0;
-				    p = p->p_pglist.le_next) {
-					printf("\t\tpid %d addr 0x%08x pgrp 0x%08x\n", 
-					    p->p_pid, p, p->p_pgrp);
-				}
+	pgrp_unlock(pgrp);
+
+	count += 20;
+	if (count > hard_maxproc)
+		count = hard_maxproc;
+	alloc_count = count * sizeof(pid_t);
+	pid_list = (pid_t *)kalloc(alloc_count);
+	bzero(pid_list, alloc_count);
+	
+	pidcount = 0;
+	pgrp_lock(pgrp);
+	for (p = pgrp->pg_members.lh_first; p != 0;
+	     p = p->p_pglist.le_next) {
+		if (p->p_stat == SSTOP) {
+			for (p = pgrp->pg_members.lh_first; p != 0;
+				p = p->p_pglist.le_next) {
+				pid_list[pidcount] = p->p_pid;
+				pidcount++;
+				if (pidcount >= count)
+					break;
 			}
+			break; /* ??? stops after finding one.. */
 		}
 	}
+	pgrp_unlock(pgrp);
+		
+	if (pidcount == 0)
+		goto out;
+
+
+	for (i = 0; i< pidcount; i++) {
+		/* No handling or proc0 */
+		if (pid_list[i] == 0)
+			continue;
+		p = proc_find(pid_list[i]);
+		if (p) {
+			proc_transwait(p, 0);
+			pt_setrunnable(p);
+			psignal(p, SIGHUP);
+			psignal(p, SIGCONT);
+			proc_rele(p);
+		}
+	}
+out:
+	kfree(pid_list, alloc_count);
+	return;
 }
-#endif /* DEBUG */
+
+
 
 /* XXX should be __private_extern__ */
 int
-proc_is_classic(struct proc *p)
+proc_is_classic(proc_t p)
 {
     return (p->p_flag & P_TRANSLATED) ? 1 : 0;
 }
 
 /* XXX Why does this function exist?  Need to kill it off... */
-struct proc *
+proc_t
 current_proc_EXTERNAL(void)
 {
 	return (current_proc());
@@ -737,19 +1338,19 @@ current_proc_EXTERNAL(void)
  * by using "/dev/null", or all core files can be stored in "/cores/%U/%N-%P".
  * This is controlled by the sysctl variable kern.corefile (see above).
  */
-__private_extern__ char *
-proc_core_name(const char *name, uid_t uid, pid_t pid)
+__private_extern__ int
+proc_core_name(const char *name, uid_t uid, pid_t pid, char *cf_name,
+		size_t cf_name_len)
 {
 	const char *format, *appendstr;
-	char *temp;
 	char id_buf[11];		/* Buffer for pid/uid -- max 4B */
 	size_t i, l, n;
 
+	if (cf_name == NULL)
+		goto toolong;
+
 	format = corefilename;
-	MALLOC(temp, char *, MAXPATHLEN, M_TEMP, M_NOWAIT | M_ZERO);
-	if (temp == NULL)
-		return (NULL);
-	for (i = 0, n = 0; n < MAXPATHLEN && format[i]; i++) {
+	for (i = 0, n = 0; n < cf_name_len && format[i]; i++) {
 		switch (format[i]) {
 		case '%':	/* Format character */
 			i++;
@@ -761,11 +1362,11 @@ proc_core_name(const char *name, uid_t uid, pid_t pid)
 				appendstr = name;
 				break;
 			case 'P':	/* process id */
-				sprintf(id_buf, "%u", pid);
+				snprintf(id_buf, sizeof(id_buf), "%u", pid);
 				appendstr = id_buf;
 				break;
 			case 'U':	/* user id */
-				sprintf(id_buf, "%u", uid);
+				snprintf(id_buf, sizeof(id_buf), "%u", uid);
 				appendstr = id_buf;
 				break;
 			default:
@@ -775,21 +1376,1149 @@ proc_core_name(const char *name, uid_t uid, pid_t pid)
 				    format[i], format);
 			}
 			l = strlen(appendstr);
-			if ((n + l) >= MAXPATHLEN)
+			if ((n + l) >= cf_name_len)
 				goto toolong;
-			bcopy(appendstr, temp + n, l);
+			bcopy(appendstr, cf_name + n, l);
 			n += l;
 			break;
 		default:
-			temp[n++] = format[i];
+			cf_name[n++] = format[i];
 		}
 	}
 	if (format[i] != '\0')
 		goto toolong;
-	return (temp);
+	return (0);
 toolong:
 	log(LOG_ERR, "pid %ld (%s), uid (%lu): corename is too long\n",
 	    (long)pid, name, (u_long)uid);
-	FREE(temp, M_TEMP);
-	return (NULL);
+	return (1);
+}
+
+#if CONFIG_LCTX
+
+static void
+lctxinit(void)
+{
+	LIST_INIT(&alllctx);
+	alllctx_cnt = 0;
+
+	/* allocate lctx lock group attribute and group */
+	lctx_lck_grp_attr = lck_grp_attr_alloc_init();
+	lck_grp_attr_setstat(lctx_lck_grp_attr);
+
+	lctx_lck_grp = lck_grp_alloc_init("lctx", lctx_lck_grp_attr);
+	/* Allocate lctx lock attribute */
+	lctx_lck_attr = lck_attr_alloc_init();
+
+	lck_mtx_init(&alllctx_lock, lctx_lck_grp, lctx_lck_attr);
+}
+
+/*
+ * Locate login context by number.
+ */
+struct lctx *
+lcfind(pid_t lcid)
+{
+	struct lctx *l;
+
+	ALLLCTX_LOCK;
+	LIST_FOREACH(l, &alllctx, lc_list) {
+		if (l->lc_id == lcid) {
+			LCTX_LOCK(l);
+			break;
+		}
+	}
+	ALLLCTX_UNLOCK;
+	return (l);
+}
+
+#define	LCID_INC				\
+	do {					\
+		lastlcid++;			\
+		if (lastlcid > maxlcid)	\
+			lastlcid = 1;		\
+	} while (0)				\
+
+struct lctx *
+lccreate(void)
+{
+	struct lctx *l;
+	pid_t newlcid;
+
+	/* Not very efficient but this isn't a common operation. */
+	while ((l = lcfind(lastlcid)) != NULL) {
+		LCTX_UNLOCK(l);
+		LCID_INC;
+	}
+	newlcid = lastlcid;
+	LCID_INC;
+
+	MALLOC(l, struct lctx *, sizeof(struct lctx), M_LCTX, M_WAITOK|M_ZERO);
+	l->lc_id = newlcid;
+	LIST_INIT(&l->lc_members);
+	lck_mtx_init(&l->lc_mtx, lctx_lck_grp, lctx_lck_attr);
+#if CONFIG_MACF
+	l->lc_label = mac_lctx_label_alloc();
+#endif
+	ALLLCTX_LOCK;
+	LIST_INSERT_HEAD(&alllctx, l, lc_list);
+	alllctx_cnt++;
+	ALLLCTX_UNLOCK;
+
+	return (l);
+}
+
+/*
+ * Call with proc protected (either by being invisible
+ * or by having the all-login-context lock held) and
+ * the lctx locked.
+ *
+ * Will unlock lctx on return.
+ */
+void
+enterlctx (proc_t p, struct lctx *l, __unused int create)
+{
+	if (l == NULL)
+		return;
+
+	p->p_lctx = l;
+	LIST_INSERT_HEAD(&l->lc_members, p, p_lclist);
+	l->lc_mc++;
+
+#if CONFIG_MACF
+	if (create)
+		mac_lctx_notify_create(p, l);
+	else
+		mac_lctx_notify_join(p, l);
+#endif
+	LCTX_UNLOCK(l);
+
+	return;
+}
+
+/*
+ * Remove process from login context (if any). Called with p protected by
+ * the alllctx lock.
+ */
+void
+leavelctx (proc_t p)
+{
+	struct lctx *l;
+
+	if (p->p_lctx == NULL) {
+		return;
+	}
+
+	LCTX_LOCK(p->p_lctx);
+	l = p->p_lctx;
+	p->p_lctx = NULL;
+	LIST_REMOVE(p, p_lclist);
+	l->lc_mc--;
+#if CONFIG_MACF
+	mac_lctx_notify_leave(p, l);
+#endif
+	if (LIST_EMPTY(&l->lc_members)) {
+		LIST_REMOVE(l, lc_list);
+		alllctx_cnt--;
+		LCTX_UNLOCK(l);
+		lck_mtx_destroy(&l->lc_mtx, lctx_lck_grp);
+#if CONFIG_MACF
+		mac_lctx_label_free(l->lc_label);
+		l->lc_label = NULL;
+#endif
+		FREE(l, M_LCTX);
+	} else {
+		LCTX_UNLOCK(l);
+	}
+	return;
+}
+
+static int
+sysctl_kern_lctx SYSCTL_HANDLER_ARGS
+{
+	int *name = (int*) arg1;
+	u_int namelen = arg2;
+	struct kinfo_lctx kil;
+	struct lctx *l;
+	int error;
+
+	error = 0;
+
+	switch (oidp->oid_number) {
+	case KERN_LCTX_ALL:
+		ALLLCTX_LOCK;
+		/* Request for size. */
+		if (!req->oldptr) {
+			error = SYSCTL_OUT(req, 0,
+				sizeof(struct kinfo_lctx) * (alllctx_cnt + 1));
+			goto out;
+		}
+		break;
+
+	case KERN_LCTX_LCID:
+		/* No space */
+		if (req->oldlen < sizeof(struct kinfo_lctx))
+			return (ENOMEM);
+		/* No argument */
+		if (namelen != 1)
+			return (EINVAL);
+		/* No login context */
+		l = lcfind((pid_t)name[0]);
+		if (l == NULL)
+			return (ENOENT);
+		kil.id = l->lc_id;
+		kil.mc = l->lc_mc;
+		LCTX_UNLOCK(l);
+		return (SYSCTL_OUT(req, (caddr_t)&kil, sizeof(kil)));
+
+	default:
+		return (EINVAL);
+	}
+
+	/* Provided buffer is too small. */
+	if (req->oldlen < (sizeof(struct kinfo_lctx) * alllctx_cnt)) {
+		error = ENOMEM;
+		goto out;
+	}
+
+	LIST_FOREACH(l, &alllctx, lc_list) {
+		LCTX_LOCK(l);
+		kil.id = l->lc_id;
+		kil.mc = l->lc_mc;
+		LCTX_UNLOCK(l);
+		error = SYSCTL_OUT(req, (caddr_t)&kil, sizeof(kil));
+		if (error)
+			break;
+	}
+out:
+	ALLLCTX_UNLOCK;
+
+	return (error);
+}
+
+SYSCTL_NODE(_kern, KERN_LCTX, lctx, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Login Context");
+
+SYSCTL_PROC(_kern_lctx, KERN_LCTX_ALL, all, CTLFLAG_RD|CTLTYPE_STRUCT,
+	    0, 0, sysctl_kern_lctx, "S,lctx",
+	    "Return entire login context table");
+SYSCTL_NODE(_kern_lctx, KERN_LCTX_LCID, lcid, CTLFLAG_RD,
+	    sysctl_kern_lctx, "Login Context Table");
+SYSCTL_INT(_kern_lctx, OID_AUTO, last,  CTLFLAG_RD, &lastlcid, 0, ""); 
+SYSCTL_INT(_kern_lctx, OID_AUTO, count, CTLFLAG_RD, &alllctx_cnt, 0, "");
+SYSCTL_INT(_kern_lctx, OID_AUTO, max, CTLFLAG_RW, &maxlcid, 0, "");
+
+#endif	/* LCTX */
+
+/* Code Signing related routines */
+
+int 
+csops(__unused proc_t p, struct csops_args *uap, __unused register_t *retval)
+{
+	int ops = uap->ops;
+	pid_t pid = uap->pid;
+	user_addr_t uaddr = uap->useraddr;
+	size_t usize = (size_t)CAST_DOWN(size_t, uap->usersize);
+	proc_t pt;
+	uint32_t retflags;
+	int vid, forself;
+	int error;
+	vnode_t tvp;
+	off_t toff;
+	char * buf;
+	unsigned char cdhash[SHA1_RESULTLEN];
+	
+	forself = error = 0;
+
+	if (pid == 0) 
+		pid = proc_selfpid();
+	if (pid == proc_selfpid())
+		forself = 1;
+
+
+	/* Pre flight checks for CS_OPS_PIDPATH */
+	if (ops == CS_OPS_PIDPATH) {
+		/* usize is unsigned.. */
+	 	if (usize > 4 * PATH_MAX)
+			return(EOVERFLOW);
+		if (kauth_cred_issuser(kauth_cred_get()) != TRUE) 
+			return(EPERM);
+	} else if ((forself == 0) && ((ops != CS_OPS_STATUS) && (ops != CS_OPS_CDHASH) && (kauth_cred_issuser(kauth_cred_get()) != TRUE))) {
+		return(EPERM);
+	}
+
+	pt = proc_find(pid);
+	if (pt == PROC_NULL)
+		return(ESRCH);
+
+
+
+	switch (ops) {
+
+		case CS_OPS_STATUS:
+			retflags = pt->p_csflags;
+			if (uaddr != USER_ADDR_NULL)
+				error = copyout(&retflags, uaddr, sizeof(uint32_t));
+			break;
+		
+		case CS_OPS_MARKINVALID:
+			proc_lock(pt);
+			if ((pt->p_csflags & CS_VALID) == CS_VALID) {	/* is currently valid */
+				pt->p_csflags &= ~CS_VALID;	/* set invalid */
+				if ((pt->p_csflags & CS_KILL) == CS_KILL) {
+					proc_unlock(pt);
+					psignal(pt, SIGKILL);
+				} else
+					proc_unlock(pt);
+			} else
+				proc_unlock(pt);
+				
+			break;
+
+		case CS_OPS_MARKHARD:
+			proc_lock(pt);
+			pt->p_csflags |= CS_HARD;
+			if ((pt->p_csflags & CS_VALID) == 0) {
+				/* @@@ allow? reject? kill? @@@ */
+				proc_unlock(pt);
+				error = EINVAL;
+				goto out;
+			} else
+				proc_unlock(pt);
+			break;
+
+		case CS_OPS_MARKKILL:
+			proc_lock(pt);
+			pt->p_csflags |= CS_KILL;
+			if ((pt->p_csflags & CS_VALID) == 0) {
+				proc_unlock(pt);
+				psignal(pt, SIGKILL);
+			} else
+				proc_unlock(pt);
+			break;
+
+		case CS_OPS_PIDPATH:
+			tvp = pt->p_textvp;
+			vid = vnode_vid(tvp);
+
+			proc_rele(pt);
+
+			buf = (char *)kalloc(usize);
+			if (buf == NULL) 
+				return(ENOMEM);
+
+
+			error = vnode_getwithvid(tvp, vid);
+			if (error == 0) {
+				int len; 
+				len = usize;
+				error = vn_getpath(tvp, buf, &len);
+				vnode_put(tvp);
+				if (error == 0) {
+					error = copyout(buf, uaddr, usize);
+				}
+				kfree(buf, usize);
+			}
+			return(error);
+
+		case CS_OPS_CDHASH:
+			if (usize != SHA1_RESULTLEN) {
+				proc_rele(pt);
+				return EINVAL;
+			}
+
+			/* pt already holds a reference on its p_textvp */
+			tvp = pt->p_textvp;
+			toff = pt->p_textoff;
+
+			error = vn_getcdhash(tvp, toff, cdhash);
+			proc_rele(pt);
+
+			if (error == 0) {
+				error = copyout(cdhash, uaddr, sizeof (cdhash));
+			}
+
+			return error;
+		
+		default:
+			error = EINVAL;
+			break;
+	}
+out:
+	proc_rele(pt);
+	return(error);
 }
+
+
+int
+proc_iterate(flags, callout, arg, filterfn, filterarg)
+	int flags;
+	int (*callout)(proc_t, void *);
+	void * arg;
+	int (*filterfn)(proc_t, void *);
+	void * filterarg;
+{
+	proc_t p;
+	pid_t * pid_list;
+	int count, pidcount, alloc_count, i, retval;
+
+	count = nprocs+ 10;
+	if (count > hard_maxproc)
+		count = hard_maxproc;
+	alloc_count = count * sizeof(pid_t);
+	pid_list = (pid_t *)kalloc(alloc_count);
+	bzero(pid_list, alloc_count);
+
+
+	proc_list_lock();
+
+
+	pidcount = 0;
+	if (flags & PROC_ALLPROCLIST) {
+		for (p = allproc.lh_first; (p != 0); p = p->p_list.le_next) {
+			if (p->p_stat == SIDL)
+				continue;
+			if ( (filterfn == 0 ) || (filterfn(p, filterarg) != 0)) {
+				pid_list[pidcount] = p->p_pid;
+				pidcount++;
+				if (pidcount >= count)
+					break;
+			}
+		}
+	}
+	if ((pidcount <  count ) && (flags & PROC_ZOMBPROCLIST)) {
+		for (p = zombproc.lh_first; p != 0; p = p->p_list.le_next) {
+			if ( (filterfn == 0 ) || (filterfn(p, filterarg) != 0)) {
+				pid_list[pidcount] = p->p_pid;
+				pidcount++;
+				if (pidcount >= count)
+					break;
+			}
+		}
+	}
+		
+
+	proc_list_unlock();
+
+
+	for (i = 0; i< pidcount; i++) {
+		p = proc_find(pid_list[i]);
+		if (p) {
+			if ((flags & PROC_NOWAITTRANS) == 0)
+				proc_transwait(p, 0);
+			retval = callout(p, arg);
+
+			switch (retval) {
+		  		case PROC_RETURNED:
+		  		case PROC_RETURNED_DONE:
+			  		proc_rele(p);
+			  		if (retval == PROC_RETURNED_DONE) {
+						goto out;
+			  		}
+			  		break;
+
+		  		case PROC_CLAIMED_DONE:
+					goto out;
+		  		case PROC_CLAIMED:
+		  		default:
+					break;
+			}
+		} else if (flags & PROC_ZOMBPROCLIST) {
+			p = proc_find_zombref(pid_list[i]);
+			if (p != PROC_NULL) {
+				retval = callout(p, arg);
+		
+				switch (retval) {
+		  			case PROC_RETURNED:
+		  			case PROC_RETURNED_DONE:
+						proc_drop_zombref(p);
+			  			if (retval == PROC_RETURNED_DONE) {
+							goto out;
+			  			}
+			  			break;
+
+		  			case PROC_CLAIMED_DONE:
+						goto out;
+		  			case PROC_CLAIMED:
+		  			default:
+						break;
+				}
+			}
+		}
+	}
+
+out: 
+	kfree(pid_list, alloc_count);
+	return(0);
+
+}
+
+
+#if 0
+/* This is for iteration in case of trivial non blocking callouts */
+int
+proc_scanall(flags, callout, arg)
+	int flags;
+	int (*callout)(proc_t, void *);
+	void * arg;
+{
+	proc_t p;
+	int retval;
+
+
+	proc_list_lock();
+
+
+	if (flags & PROC_ALLPROCLIST) {
+		for (p = allproc.lh_first; (p != 0); p = p->p_list.le_next) {
+			retval = callout(p, arg);
+			if (retval == PROC_RETURNED_DONE)
+				goto out;
+		}
+	}
+	if (flags & PROC_ZOMBPROCLIST) {
+		for (p = zombproc.lh_first; p != 0; p = p->p_list.le_next) {
+			retval = callout(p, arg);
+			if (retval == PROC_RETURNED_DONE)
+				goto out;
+		}
+	}
+out:
+
+	proc_list_unlock();
+
+	return(0);
+}
+#endif
+
+
+int
+proc_rebootscan(callout, arg, filterfn, filterarg)
+	int (*callout)(proc_t, void *);
+	void * arg;
+	int (*filterfn)(proc_t, void *);
+	void * filterarg;
+{
+	proc_t p;
+	int lockheld = 0, retval;
+
+ps_allprocscan:
+
+	proc_list_lock();
+	lockheld = 1;
+
+	for (p = allproc.lh_first; (p != 0); p = p->p_list.le_next) {
+		if ( (filterfn == 0 ) || (filterfn(p, filterarg) != 0)) {
+			p = proc_refinternal_locked(p);
+
+			proc_list_unlock();
+			lockheld = 0;
+
+			if (p) {
+				proc_transwait(p, 0);
+				retval = callout(p, arg);
+				proc_rele(p);
+	
+				switch (retval) {
+					case PROC_RETURNED_DONE:
+					case PROC_CLAIMED_DONE:
+						goto out;
+				}
+			}
+			goto ps_allprocscan;	
+		} /* filter pass */
+	} /* allproc walk thru */
+
+	if (lockheld == 1) {
+		proc_list_unlock();
+		lockheld = 0;
+	}
+
+out: 
+	return(0);
+
+}
+
+
+int
+proc_childrenwalk(parent, callout, arg)
+	struct proc * parent;
+	int (*callout)(proc_t, void *);
+	void * arg;
+{
+	register struct proc *p;
+	pid_t * pid_list;
+	int count, pidcount, alloc_count, i, retval;
+
+	count = nprocs+ 10;
+	if (count > hard_maxproc)
+		count = hard_maxproc;
+	alloc_count = count * sizeof(pid_t);
+	pid_list = (pid_t *)kalloc(alloc_count);
+	bzero(pid_list, alloc_count);
+
+
+	proc_list_lock();
+
+
+	pidcount = 0;
+	for (p = parent->p_children.lh_first; (p != 0); p = p->p_sibling.le_next) {
+		if (p->p_stat == SIDL)
+			continue;
+		pid_list[pidcount] = p->p_pid;
+		pidcount++;
+		if (pidcount >= count)
+			break;
+	}
+	proc_list_unlock();
+
+
+	for (i = 0; i< pidcount; i++) {
+		p = proc_find(pid_list[i]);
+		if (p) {
+			proc_transwait(p, 0);
+			retval = callout(p, arg);
+
+			switch (retval) {
+		  		case PROC_RETURNED:
+		  		case PROC_RETURNED_DONE:
+			  		proc_rele(p);
+			  		if (retval == PROC_RETURNED_DONE) {
+						goto out;
+			  		}
+			  		break;
+
+		  		case PROC_CLAIMED_DONE:
+					goto out;
+		  		case PROC_CLAIMED:
+		  		default:
+					break;
+			}
+		}
+	}
+
+out: 
+	kfree(pid_list, alloc_count);
+	return(0);
+
+}
+
+/*
+ */
+/* PGRP_BLOCKITERATE is not implemented yet */
+int
+pgrp_iterate(pgrp, flags, callout, arg, filterfn, filterarg)
+	struct pgrp *pgrp;
+	int flags;
+	int (*callout)(proc_t, void *);
+	void * arg;
+	int (*filterfn)(proc_t, void *);
+	void * filterarg;
+{
+	proc_t p;
+	pid_t * pid_list;
+	int count, pidcount, i, alloc_count;
+	int retval;
+	pid_t pgid;
+	int dropref = flags & PGRP_DROPREF;
+#if 0
+	int serialize = flags & PGRP_BLOCKITERATE;
+#else
+	int serialize = 0;
+#endif
+
+	if (pgrp == 0)
+		return(0);
+	count = pgrp->pg_membercnt + 10;
+	if (count > hard_maxproc)
+		count = hard_maxproc;
+	alloc_count = count * sizeof(pid_t);
+	pid_list = (pid_t *)kalloc(alloc_count);
+	bzero(pid_list, alloc_count);
+	
+	pgrp_lock(pgrp);
+	if (serialize  != 0) {
+		while ((pgrp->pg_listflags & PGRP_FLAG_ITERABEGIN) == PGRP_FLAG_ITERABEGIN) {
+			pgrp->pg_listflags |= PGRP_FLAG_ITERWAIT;
+			msleep(&pgrp->pg_listflags, &pgrp->pg_mlock, 0, "pgrp_iterate", 0);
+		}
+		pgrp->pg_listflags |= PGRP_FLAG_ITERABEGIN;
+	}
+
+	pgid = pgrp->pg_id;
+
+	pidcount = 0;
+	for (p = pgrp->pg_members.lh_first; p != 0;
+	     p = p->p_pglist.le_next) {
+		if ( (filterfn == 0 ) || (filterfn(p, filterarg) != 0)) {
+			pid_list[pidcount] = p->p_pid;
+			pidcount++;
+			if (pidcount >= count)
+				break;
+		}
+	}
+		
+
+	pgrp_unlock(pgrp);
+	if ((serialize == 0) && (dropref != 0))
+		pg_rele(pgrp);
+
+
+	for (i = 0; i< pidcount; i++) {
+		/* No handling or proc0 */
+		if (pid_list[i] == 0)
+			continue;
+		p = proc_find(pid_list[i]);
+		if (p) {
+			if (p->p_pgrpid != pgid) {
+				proc_rele(p);
+				continue;
+			}
+			proc_transwait(p, 0);
+			retval = callout(p, arg);
+
+			switch (retval) {
+		  		case PROC_RETURNED:
+		  		case PROC_RETURNED_DONE:
+			  		proc_rele(p);
+			  		if (retval == PROC_RETURNED_DONE) {
+						goto out;
+			  		}
+			  		break;
+
+		  		case PROC_CLAIMED_DONE:
+					goto out;
+		  		case PROC_CLAIMED:
+		  		default:
+					break;
+			}
+		}
+	}
+out:
+	if (serialize != 0) {
+		pgrp_lock(pgrp);
+		pgrp->pg_listflags &= ~PGRP_FLAG_ITERABEGIN;
+		if ((pgrp->pg_listflags & PGRP_FLAG_ITERWAIT) == PGRP_FLAG_ITERWAIT) {
+			pgrp->pg_listflags &= ~PGRP_FLAG_ITERWAIT;
+			wakeup(&pgrp->pg_listflags);
+		}
+		pgrp_unlock(pgrp);
+		if (dropref != 0)
+			pg_rele(pgrp);
+	}
+	kfree(pid_list, alloc_count);
+	return(0);
+}
+
+static void
+pgrp_add(struct pgrp * pgrp, struct proc * parent, struct proc * child)
+{
+	proc_list_lock();
+	child->p_pgrp = pgrp;
+	child->p_pgrpid = pgrp->pg_id;
+	child->p_listflag |= P_LIST_INPGRP;
+	/*
+	 * When pgrp is being freed , a process can still 
+	 * request addition using setpgid from bash when 
+ 	 * login is terminated (login cycler) return ESRCH
+	 * Safe to hold lock due to refcount on pgrp 
+	 */
+	if ((pgrp->pg_listflags & (PGRP_FLAG_TERMINATE | PGRP_FLAG_DEAD)) == PGRP_FLAG_TERMINATE) {
+		pgrp->pg_listflags &= ~PGRP_FLAG_TERMINATE;	
+	}
+
+	if ((pgrp->pg_listflags & PGRP_FLAG_DEAD) == PGRP_FLAG_DEAD)
+		panic("pgrp_add : pgrp is dead adding process");
+	proc_list_unlock();
+
+	pgrp_lock(pgrp);
+	pgrp->pg_membercnt++;
+	if ( parent != PROC_NULL) {
+		LIST_INSERT_AFTER(parent, child, p_pglist);
+	 }else {
+		LIST_INSERT_HEAD(&pgrp->pg_members, child, p_pglist);
+	}
+	pgrp_unlock(pgrp);
+
+	proc_list_lock();
+	if (((pgrp->pg_listflags & (PGRP_FLAG_TERMINATE | PGRP_FLAG_DEAD)) == PGRP_FLAG_TERMINATE) && (pgrp->pg_membercnt != 0)) {
+		pgrp->pg_listflags &= ~PGRP_FLAG_TERMINATE;	
+	}
+	proc_list_unlock();
+}
+
+static void
+pgrp_remove(struct proc * p)
+{
+	struct pgrp * pg;
+
+	pg = proc_pgrp(p);
+
+	proc_list_lock();
+#if __PROC_INTERNAL_DEBUG
+	if ((p->p_listflag & P_LIST_INPGRP) == 0)
+		panic("removing from pglist but no named ref\n");
+#endif
+	p->p_pgrpid = PGRPID_DEAD;
+	p->p_listflag &= ~P_LIST_INPGRP;
+	p->p_pgrp = NULL;
+	proc_list_unlock();
+
+	if (pg == PGRP_NULL)
+		panic("pgrp_remove: pg is NULL");
+	pgrp_lock(pg);
+	pg->pg_membercnt--;
+
+	if (pg->pg_membercnt < 0)
+		panic("pgprp: -ve membercnt pgprp:%x p:%x\n",(unsigned int)pg, (unsigned int)p);
+
+	LIST_REMOVE(p, p_pglist);
+	if (pg->pg_members.lh_first == 0) {
+		pgrp_unlock(pg);
+		pgdelete_dropref(pg);
+	} else {
+		pgrp_unlock(pg);
+		pg_rele(pg);
+	}
+}
+
+
+/* cannot use proc_pgrp as it maybe stalled */
+static void
+pgrp_replace(struct proc * p, struct pgrp * newpg)
+{
+        struct pgrp * oldpg;
+
+
+
+       proc_list_lock();
+
+	while ((p->p_listflag & P_LIST_PGRPTRANS) == P_LIST_PGRPTRANS) {
+		p->p_listflag |= P_LIST_PGRPTRWAIT;
+		(void)msleep(&p->p_pgrpid, proc_list_mlock, 0, "proc_pgrp", 0);
+	}
+
+	p->p_listflag |= P_LIST_PGRPTRANS;
+
+	oldpg = p->p_pgrp;
+	if (oldpg == PGRP_NULL)
+		panic("pgrp_replace: oldpg NULL");
+	oldpg->pg_refcount++;
+#if __PROC_INTERNAL_DEBUG
+        if ((p->p_listflag & P_LIST_INPGRP) == 0)
+                panic("removing from pglist but no named ref\n");
+#endif
+        p->p_pgrpid = PGRPID_DEAD;
+        p->p_listflag &= ~P_LIST_INPGRP;
+        p->p_pgrp = NULL;
+ 
+       proc_list_unlock();
+
+       pgrp_lock(oldpg);
+       oldpg->pg_membercnt--;
+       if (oldpg->pg_membercnt < 0)
+                panic("pgprp: -ve membercnt pgprp:%x p:%x\n",(unsigned int)oldpg, (unsigned int)p);
+       LIST_REMOVE(p, p_pglist);
+        if (oldpg->pg_members.lh_first == 0) {
+                pgrp_unlock(oldpg);
+                pgdelete_dropref(oldpg);
+        } else {
+                pgrp_unlock(oldpg);
+                pg_rele(oldpg);
+        }
+
+        proc_list_lock();
+        p->p_pgrp = newpg;
+        p->p_pgrpid = newpg->pg_id;
+        p->p_listflag |= P_LIST_INPGRP;
+        /*
+         * When pgrp is being freed , a process can still
+         * request addition using setpgid from bash when 
+         * login is terminated (login cycler) return ESRCH
+         * Safe to hold lock due to refcount on pgrp 
+         */
+        if ((newpg->pg_listflags & (PGRP_FLAG_TERMINATE | PGRP_FLAG_DEAD)) == PGRP_FLAG_TERMINATE) {
+                newpg->pg_listflags &= ~PGRP_FLAG_TERMINATE;
+        }
+
+        if ((newpg->pg_listflags & PGRP_FLAG_DEAD) == PGRP_FLAG_DEAD)
+                panic("pgrp_add : pgrp is dead adding process");
+        proc_list_unlock();
+
+        pgrp_lock(newpg);
+        newpg->pg_membercnt++;
+	LIST_INSERT_HEAD(&newpg->pg_members, p, p_pglist);
+        pgrp_unlock(newpg);
+
+        proc_list_lock();
+        if (((newpg->pg_listflags & (PGRP_FLAG_TERMINATE | PGRP_FLAG_DEAD)) == PGRP_FLAG_TERMINATE) && (newpg->pg_membercnt != 0)) {
+                newpg->pg_listflags &= ~PGRP_FLAG_TERMINATE;
+        }
+
+	p->p_listflag &= ~P_LIST_PGRPTRANS;
+	if ((p->p_listflag & P_LIST_PGRPTRWAIT) == P_LIST_PGRPTRWAIT) {
+		p->p_listflag &= ~P_LIST_PGRPTRWAIT;
+		wakeup(&p->p_pgrpid);
+		
+	}
+        proc_list_unlock();
+}
+
+void
+pgrp_lock(struct pgrp * pgrp)
+{
+	lck_mtx_lock(&pgrp->pg_mlock);
+}
+
+void
+pgrp_unlock(struct pgrp * pgrp)
+{
+	lck_mtx_unlock(&pgrp->pg_mlock);
+}
+
+void
+session_lock(struct session * sess)
+{
+	lck_mtx_lock(&sess->s_mlock);
+}
+
+
+void
+session_unlock(struct session * sess)
+{
+	lck_mtx_unlock(&sess->s_mlock);
+}
+
+struct pgrp *
+proc_pgrp(proc_t p)
+{
+	struct pgrp * pgrp;
+
+	if (p == PROC_NULL)
+		return(PGRP_NULL);
+	proc_list_lock();
+
+	while ((p->p_listflag & P_LIST_PGRPTRANS) == P_LIST_PGRPTRANS) {
+		p->p_listflag |= P_LIST_PGRPTRWAIT;
+		(void)msleep(&p->p_pgrpid, proc_list_mlock, 0, "proc_pgrp", 0);
+	}
+		
+	pgrp = p->p_pgrp;
+
+	assert(pgrp != NULL);
+
+	if ((pgrp->pg_listflags & (PGRP_FLAG_TERMINATE | PGRP_FLAG_DEAD)) != 0)
+		panic("proc_pgrp: ref being povided for dead pgrp");
+		
+	if (pgrp != PGRP_NULL) 
+		pgrp->pg_refcount++;
+	proc_list_unlock();
+	
+	return(pgrp);
+}
+
+struct pgrp *
+tty_pgrp(struct tty * tp)
+{
+	struct pgrp * pg = PGRP_NULL;
+
+	proc_list_lock();
+	pg = tp->t_pgrp;
+
+	if (pg != PGRP_NULL) {
+		if ((pg->pg_listflags & PGRP_FLAG_DEAD) != 0)
+			panic("tty_pgrp: ref being povided for dead pgrp");
+		pg->pg_refcount++;
+	}
+	proc_list_unlock();
+
+	return(pg);
+}
+
+struct session *
+proc_session(proc_t p)
+{
+	struct session * sess = SESSION_NULL;
+	
+	if (p == PROC_NULL)
+		return(SESSION_NULL);
+
+	proc_list_lock();
+
+	/* wait during transitions */
+	while ((p->p_listflag & P_LIST_PGRPTRANS) == P_LIST_PGRPTRANS) {
+		p->p_listflag |= P_LIST_PGRPTRWAIT;
+		(void)msleep(&p->p_pgrpid, proc_list_mlock, 0, "proc_pgrp", 0);
+	}
+
+	if ((p->p_pgrp != PGRP_NULL) && ((sess = p->p_pgrp->pg_session) != SESSION_NULL)) {
+		if ((sess->s_listflags & (S_LIST_TERM | S_LIST_DEAD)) != 0)
+			panic("proc_session:returning sesssion ref on terminating session");
+		sess->s_count++;
+	}
+	proc_list_unlock();
+	return(sess);
+}
+
+void
+session_rele(struct session *sess)
+{
+	proc_list_lock();
+	if (--sess->s_count == 0) {
+		if ((sess->s_listflags & (S_LIST_TERM | S_LIST_DEAD)) != 0)
+			panic("session_rele: terminating already terminated session");
+		sess->s_listflags |= S_LIST_TERM;
+		LIST_REMOVE(sess, s_hash);
+		sess->s_listflags |= S_LIST_DEAD;
+		if (sess->s_count != 0)
+			panic("session_rele: freeing session in use");	
+		proc_list_unlock();
+		lck_mtx_destroy(&sess->s_mlock, proc_lck_grp);
+		FREE_ZONE(sess, sizeof(struct session), M_SESSION);
+	} else
+		proc_list_unlock();
+}
+
+void
+proc_transstart(proc_t p, int locked)
+{
+	if (locked == 0)
+		proc_lock(p);
+	while ((p->p_lflag & P_LINTRANSIT) == P_LINTRANSIT) {
+		p->p_lflag |= P_LTRANSWAIT;
+		msleep(&p->p_lflag, &p->p_mlock, 0, "proc_signstart", NULL);
+	}
+	p->p_lflag |= P_LINTRANSIT;
+	p->p_transholder = current_thread();
+	if (locked == 0)
+		proc_unlock(p);
+
+}
+
+
+void
+proc_transend(proc_t p, int locked)
+{
+	if (locked == 0)
+		proc_lock(p);
+	p->p_lflag &= ~P_LINTRANSIT;
+
+	if ((p->p_lflag & P_LTRANSWAIT) == P_LTRANSWAIT) {
+		p->p_lflag &= ~P_LTRANSWAIT;
+		wakeup(&p->p_lflag);
+	}
+	p->p_transholder = NULL;
+	if (locked == 0)
+		proc_unlock(p);
+}
+
+void
+proc_transwait(proc_t p, int locked)
+{
+	if (locked == 0)
+		proc_lock(p);
+	while ((p->p_lflag & P_LINTRANSIT) == P_LINTRANSIT) {
+		p->p_lflag |= P_LTRANSWAIT;
+		msleep(&p->p_lflag, &p->p_mlock, 0, "proc_signstart", NULL);
+	}
+	if (locked == 0)
+		proc_unlock(p);
+}
+
+void
+proc_klist_lock(void)
+{
+	lck_mtx_lock(proc_klist_mlock);
+}
+
+void
+proc_klist_unlock(void)
+{
+	lck_mtx_unlock(proc_klist_mlock);
+}
+
+void
+proc_knote(struct proc * p, long hint)
+{
+	proc_klist_lock();
+	KNOTE(&p->p_klist, hint);
+	proc_klist_unlock();
+}
+
+
+unsigned long cs_procs_killed = 0;
+unsigned long cs_procs_invalidated = 0;
+int cs_force_kill = 0;
+int cs_force_hard = 0;
+int cs_debug = 0;
+SYSCTL_INT(_vm, OID_AUTO, cs_force_kill, CTLFLAG_RW, &cs_force_kill, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, cs_force_hard, CTLFLAG_RW, &cs_force_hard, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, cs_debug, CTLFLAG_RW, &cs_debug, 0, "");
+
+int
+cs_invalid_page(void)
+{
+	struct proc	*p;
+	int		retval;
+
+	p = current_proc();
+
+	/*
+	 * XXX revisit locking when proc is no longer protected
+	 * by the kernel funnel...
+	 */
+
+	/* XXX for testing */
+	proc_lock(p);
+	if (cs_force_kill)
+		p->p_csflags |= CS_KILL;
+	if (cs_force_hard)
+		p->p_csflags |= CS_HARD;
+
+	if (p->p_csflags & CS_VALID) {
+		p->p_csflags &= ~CS_VALID;
+
+		proc_unlock(p);
+		cs_procs_invalidated++;
+		printf("CODE SIGNING: cs_invalid_page: "
+		       "p=%d[%s] clearing CS_VALID\n",
+		       p->p_pid, p->p_comm);
+		proc_lock(p);
+
+
+		if (p->p_csflags & CS_KILL) {
+			proc_unlock(p);
+			if (cs_debug) {
+				printf("CODE SIGNING: cs_invalid_page: "
+				       "p=%d[%s] honoring CS_KILL\n",
+				       p->p_pid, p->p_comm);
+			}
+			cs_procs_killed++;
+			psignal(p, SIGKILL);
+			proc_lock(p);
+		}
+		
+		if (p->p_csflags & CS_HARD) {
+			proc_unlock(p);
+			if (cs_debug) {
+				printf("CODE SIGNING: cs_invalid_page: "
+				       "p=%d[%s] honoring CS_HARD\n",
+				       p->p_pid, p->p_comm);
+			}
+			retval = 1;
+		} else {
+			proc_unlock(p);
+			retval = 0;
+		}
+	} else {
+		proc_unlock(p);
+		if (cs_debug) {
+			printf("CODE SIGNING: cs_invalid_page: "
+			       "p=%d[%s] ignored...\n",
+			       p->p_pid, p->p_comm);
+		}
+		retval = 0;
+	}
+
+	return retval;
+}
+
diff --git a/bsd/kern/kern_prot.c b/bsd/kern/kern_prot.c
index 3823a912c..47c7983ca 100644
--- a/bsd/kern/kern_prot.c
+++ b/bsd/kern/kern_prot.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,18 @@
  *
  *	@(#)kern_prot.c	8.9 (Berkeley) 2/14/95
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * System calls related to processes and protection
@@ -77,25 +95,64 @@
 
 #include <bsm/audit_kernel.h>
 
+#if CONFIG_LCTX
+#include <sys/lctx.h>
+#endif
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#if CONFIG_MACF_MACH
+#include <secuity/mac_mach_internal.h>
+#endif
+#endif
+
 #include <sys/mount_internal.h>
 #include <sys/sysproto.h>
 #include <mach/message.h>
 #include <mach/host_security.h>
 
 #include <kern/host.h>
+#include <kern/task.h>		/* for current_task() */
+#include <kern/assert.h>
+
 
 int groupmember(gid_t gid, kauth_cred_t cred);
-int is_suser(void);
-int is_suser1(void);
 
-extern int prepare_profile_database(int user);
+/*
+ * Credential debugging; we can track entry into a function that might
+ * change a credential, and we can track actual credential changes that
+ * result.
+ *
+ * Note:	Does *NOT* currently include per-thread credential changes
+ *
+ *		We don't use kauth_cred_print() in current debugging, but it
+ *		can be used if needed when debugging is active.
+ */
+#if DEBUG_CRED
+#define	DEBUG_CRED_ENTER		printf
+#define	DEBUG_CRED_CHANGE		printf
+extern void kauth_cred_print(kauth_cred_t cred);
+#else	/* !DEBUG_CRED */
+#define	DEBUG_CRED_ENTER(fmt, ...)	do {} while (0)
+#define	DEBUG_CRED_CHANGE(fmt, ...)	do {} while (0)
+#endif	/* !DEBUG_CRED */
+
+
 
 /*
- * setprivexec:  (dis)allow this process to hold
- * task, thread, or execption ports of processes about to exec.
+ * setprivexec
+ *
+ * Description:	(dis)allow this process to hold task, thread, or execption
+ *		ports of processes about to exec.
+ *
+ * Parameters:	uap->flag			New value for flag
+ *
+ * Returns:	int				Previous value of flag
+ *
+ * XXX:		Belongs in kern_proc.c
  */
 int
-setprivexec(struct proc *p, struct setprivexec_args *uap, register_t *retval)
+setprivexec(proc_t p, struct setprivexec_args *uap, register_t *retval)
 {
 	AUDIT_ARG(value, uap->flag);
 	*retval = p->p_debugger;
@@ -103,93 +160,194 @@ setprivexec(struct proc *p, struct setprivexec_args *uap, register_t *retval)
 	return(0);
 }
 
-/* ARGSUSED */
+
+/*
+ * getpid
+ *
+ * Description:	get the process ID
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	pid_t				Current process ID
+ *
+ * XXX:		Belongs in kern_proc.c
+ */
 int
-getpid(struct proc *p, __unused struct getpid_args *uap, register_t *retval)
+getpid(proc_t p, __unused struct getpid_args *uap, register_t *retval)
 {
 
 	*retval = p->p_pid;
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * getppid
+ *
+ * Description: get the parent process ID
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	pid_t				Parent process ID
+ *
+ * XXX:		Belongs in kern_proc.c
+ */
 int
-getppid(struct proc *p, __unused struct getppid_args *uap, register_t *retval)
+getppid(proc_t p, __unused struct getppid_args *uap, register_t *retval)
 {
 
-	*retval = p->p_pptr->p_pid;
+	*retval = p->p_ppid;
 	return (0);
 }
 
-/* Get process group ID; note that POSIX getpgrp takes no parameter */
+
+/*
+ * getpgrp
+ *
+ * Description:	get the process group ID of the calling process
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	pid_t				Process group ID
+ *
+ * XXX:		Belongs in kern_proc.c
+ */
 int
-getpgrp(struct proc *p, __unused struct getpgrp_args *uap, register_t *retval)
+getpgrp(proc_t p, __unused struct getpgrp_args *uap, register_t *retval)
 {
 
-	*retval = p->p_pgrp->pg_id;
+	*retval = p->p_pgrpid;
 	return (0);
 }
 
-/* Get an arbitary pid's process group id */
+
+/*
+ * getpgid
+ *
+ * Description: Get an arbitary pid's process group id
+ *
+ * Parameters:	uap->pid			The target pid
+ *
+ * Returns:	0				Success
+ *		ESRCH				No such process
+ *
+ * Notes:	We are permitted to return EPERM in the case that the target
+ *		process is not in the same session as the calling process,
+ *		which could be a security consideration
+ *
+ * XXX:		Belongs in kern_proc.c
+ */
 int
-getpgid(struct proc *p, struct getpgid_args *uap, register_t *retval)
+getpgid(proc_t p, struct getpgid_args *uap, register_t *retval)
 {
-	struct proc *pt;
+	proc_t pt;
+	int refheld = 0;
 
 	pt = p;
 	if (uap->pid == 0)
 		goto found;
 
-	if ((pt = pfind(uap->pid)) == 0)
+	if ((pt = proc_find(uap->pid)) == 0)
 		return (ESRCH);
+	refheld = 1;
 found:
-	*retval = pt->p_pgrp->pg_id;
+	*retval = pt->p_pgrpid;
+	if (refheld != 0)
+		proc_rele(pt);
 	return (0);
 }
 
+
 /*
- * Get an arbitary pid's session id.
+ * getsid
+ *
+ * Description:	Get an arbitary pid's session leaders process group ID
+ *
+ * Parameters:	uap->pid			The target pid
+ *
+ * Returns:	0				Success
+ *		ESRCH				No such process
+ *
+ * Notes:	We are permitted to return EPERM in the case that the target
+ *		process is not in the same session as the calling process,
+ *		which could be a security consideration
+ *
+ * XXX:		Belongs in kern_proc.c
  */
-
 int
-getsid(struct proc *p, struct getsid_args *uap, register_t *retval)
+getsid(proc_t p, struct getsid_args *uap, register_t *retval)
 {
-	struct proc *pt;
+	proc_t pt;
+	int refheld = 0;
+	struct session * sessp;
 
 	pt = p;
 	if (uap->pid == 0)
 		goto found;
 
-	if ((pt = pfind(uap->pid)) == 0)
+	if ((pt = proc_find(uap->pid)) == 0)
 		return (ESRCH);
+	refheld = 1;
 found:
-	*retval = pt->p_session->s_sid;
+	sessp = proc_session(pt);
+	*retval = sessp->s_sid;
+	session_rele(sessp);
+
+	if (refheld != 0)
+		proc_rele(pt);
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * getuid
+ *
+ * Description:	get real user ID for caller
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	uid_t				The real uid of the caller
+ */
 int
-getuid(__unused struct proc *p, __unused struct getuid_args *uap, register_t *retval)
+getuid(__unused proc_t p, __unused struct getuid_args *uap, register_t *retval)
 {
 
  	*retval = kauth_getruid();
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * geteuid
+ *
+ * Description:	get effective user ID for caller
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	uid_t				The effective uid of the caller
+ */
 int
-geteuid(__unused struct proc *p, __unused struct geteuid_args *uap, register_t *retval)
+geteuid(__unused proc_t p, __unused struct geteuid_args *uap, register_t *retval)
 {
 
  	*retval = kauth_getuid();
 	return (0);
 }
 
+
 /*
- * Return the per-thread override identity.
+ * gettid
+ *
+ * Description:	Return the per-thread override identity.
+ *
+ * Parameters:	uap->uidp			Address of uid_t to get uid
+ *		uap->gidp			Address of gid_t to get gid
+ *
+ * Returns:	0				Success
+ *		ESRCH				No per thread identity active
  */
 int
-gettid(__unused struct proc *p, struct gettid_args *uap, register_t *retval)
+gettid(__unused proc_t p, struct gettid_args *uap, register_t *retval)
 {
 	struct uthread *uthread = get_bsdthread_info(current_thread());
 	int	error;
@@ -210,33 +368,83 @@ gettid(__unused struct proc *p, struct gettid_args *uap, register_t *retval)
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * getgid
+ *
+ * Description:	get the real group ID for the calling process
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	gid_t				The real gid of the caller
+ */
 int
-getgid(__unused struct proc *p, __unused struct getgid_args *uap, register_t *retval)
+getgid(__unused proc_t p, __unused struct getgid_args *uap, register_t *retval)
 {
 
 	*retval = kauth_getrgid();
 	return (0);
 }
 
+
 /*
- * Get effective group ID.  The "egid" is groups[0], and could be obtained
- * via getgroups.  This syscall exists because it is somewhat painful to do
- * correctly in a library function.
+ * getegid
+ *
+ * Description:	get the effective group ID for the calling process
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	gid_t				The effective gid of the caller
+ *
+ * Notes:	As an implementation detail, the effective gid is stored as
+ *		the first element of the supplementary group list.
+ *
+ *		This could be implemented in Libc instead because of the above
+ *		detail.
  */
-/* ARGSUSED */
 int
-getegid(struct proc *p, __unused struct getegid_args *uap, register_t *retval)
+getegid(__unused proc_t p, __unused struct getegid_args *uap, register_t *retval)
 {
 
 	*retval = kauth_getgid();
 	return (0);
 }
 
+
+/*
+ * getgroups
+ *
+ * Description:	get the list of supplementary groups for the calling process
+ *
+ * Parameters:	uap->gidsetsize			# of gid_t's in user buffer
+ *		uap->gidset			Pointer to user buffer
+ *
+ * Returns:	0				Success
+ *		EINVAL				User buffer too small
+ *	copyout:EFAULT				User buffer invalid
+ *
+ * Retval:	-1				Error
+ *		!0				# of groups
+ *
+ * Notes:	The caller may specify a 0 value for gidsetsize, and we will
+ *		then return how large a buffer is required (in gid_t's) to
+ *		contain the answer at the time of the call.  Otherwise, we
+ *		return the number of gid_t's catually copied to user space.
+ *
+ *		When called with a 0 gidsetsize from a multithreaded program,
+ *		there is no guarantee that another thread may not change the
+ *		number of supplementary groups, and therefore a subsequent
+ *		call could still fail, unless the maximum possible buffer
+ *		size is supplied by the user.
+ *
+ *		As an implementation detail, the effective gid is stored as
+ *		the first element of the supplementary group list, and will
+ *		be returned by this call.
+ */
 int
-getgroups(__unused struct proc *p, struct getgroups_args *uap, register_t *retval)
+getgroups(__unused proc_t p, struct getgroups_args *uap, register_t *retval)
 {
-	register int ngrp;
+	int ngrp;
 	int error;
 	kauth_cred_t cred;
 
@@ -264,12 +472,13 @@ getgroups(__unused struct proc *p, struct getgroups_args *uap, register_t *retva
 	return (0);
 }
 
+
 /*
  * Return the per-thread/per-process supplementary groups list.
  */
-#warning XXX implement
+#warning XXX implement getsgroups
 int
-getsgroups(__unused struct proc *p, __unused struct getsgroups_args *uap, __unused register_t *retval)
+getsgroups(__unused proc_t p, __unused struct getsgroups_args *uap, __unused register_t *retval)
 {
 	/* XXX implement */
 	return(ENOTSUP);
@@ -278,73 +487,186 @@ getsgroups(__unused struct proc *p, __unused struct getsgroups_args *uap, __unus
 /*
  * Return the per-thread/per-process whiteout groups list.
  */
-#warning XXX implement
+#warning XXX implement getwgroups
 int
-getwgroups(__unused struct proc *p, __unused struct getwgroups_args *uap, __unused register_t *retval)
+getwgroups(__unused proc_t p, __unused struct getwgroups_args *uap, __unused register_t *retval)
 {
 	/* XXX implement */
 	return(ENOTSUP);
 }
 
-/* ARGSUSED */
+
+/*
+ * setsid
+ *
+ * Description:	Create a new session and set the process group ID to the
+ *		session ID
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	0				Success
+ *		EPERM				Permission denied
+ *
+ * Notes:	If the calling process is not the process group leader; there
+ *		is no existing process group with its ID, and we are not
+ *		currently in vfork, then this function will create a new
+ *		session, a new process group, and put the caller in the
+ *		process group (as the sole member) and make it the session
+ *		leader (as the sole process in the session).
+ *
+ *		The existing controlling tty (if any) will be dissociated
+ *		from the process, and the next non-O_NOCTTY open of a tty
+ *		will establish a new controlling tty.
+ *
+ * XXX:		Belongs in kern_proc.c
+ */
 int
-setsid(struct proc *p, __unused struct setsid_args *uap, register_t *retval)
+setsid(proc_t p, __unused struct setsid_args *uap, register_t *retval)
 {
+	struct pgrp * pg = PGRP_NULL;
 
-	if (p->p_pgid == p->p_pid || pgfind(p->p_pid) || p->p_flag & P_INVFORK) {
+	if (p->p_pgrpid == p->p_pid || (pg = pgfind(p->p_pid)) || p->p_lflag & P_LINVFORK) {
+		if (pg != PGRP_NULL)
+			pg_rele(pg);
 		return (EPERM);
 	} else {
+		/* enter pgrp works with its own pgrp refcount */
 		(void)enterpgrp(p, p->p_pid, 1);
 		*retval = p->p_pid;
 		return (0);
 	}
 }
 
+
 /*
- * set process group (setpgid/old setpgrp)
- *
- * caller does setpgid(targpid, targpgid)
- *
- * pid must be caller or child of caller (ESRCH)
- * if a child
- *	pid must be in same session (EPERM)
- *	pid can't have done an exec (EACCES)
- * ig pgid is -ve return EINVAL (as per SUV spec)
- * if pgid != pid
- * 	there must exist some pid in same session having pgid (EPERM)
- * pid must not be session leader (EPERM)
+ * setpgid
+ *
+ * Description: set process group ID for job control
+ *
+ * Parameters:	uap->pid			Process to change
+ *		uap->pgid			Process group to join or create
+ *
+ * Returns:	0			Success
+ *		ESRCH			pid is not the caller or a child of
+ *					the caller
+ *	enterpgrp:ESRCH			No such process
+ *		EACCES			Permission denied due to exec
+ *		EINVAL			Invalid argument
+ *		EPERM			The target process is not in the same
+ *					session as the calling process
+ *		EPERM			The target process is a session leader
+ *		EPERM			pid and pgid are not the same, and
+ *					there is no process in the calling
+ *					process whose process group ID matches
+ *					pgid
+ *
+ * Notes:	This function will cause the target process to either join
+ *		an existing process process group, or create a new process
+ *		group in the session of the calling process.  It cannot be
+ *		used to change the process group ID of a process which is
+ *		already a session leader.
+ *
+ *		If the target pid is 0, the pid of the calling process is
+ *		substituted as the new target; if pgid is 0, the target pid
+ *		is used as the target process group ID.
+ *
+ * Legacy:	This system call entry point is also used to implement the
+ *		legacy library routine setpgrp(), which under POSIX 
+ *
+ * XXX:		Belongs in kern_proc.c
  */
-/* ARGSUSED */
 int
-setpgid(struct proc *curp, register struct setpgid_args *uap, __unused register_t *retval)
+setpgid(proc_t curp, register struct setpgid_args *uap, __unused register_t *retval)
 {
-	register struct proc *targp;		/* target process */
-	register struct pgrp *pgrp;		/* target pgrp */
+	proc_t targp = PROC_NULL;	/* target process */
+	struct pgrp *pg = PGRP_NULL;	/* target pgrp */
+	int error = 0;
+	int refheld = 0;
+	int samesess = 0;
+	struct session * curp_sessp = SESSION_NULL;
+	struct session * targp_sessp = SESSION_NULL;
+
+	curp_sessp = proc_session(curp);
 
 	if (uap->pid != 0 && uap->pid != curp->p_pid) {
-		if ((targp = pfind(uap->pid)) == 0 || !inferior(targp))
-			return (ESRCH);
-		if (targp->p_session != curp->p_session)
-			return (EPERM);
-		if (targp->p_flag & P_EXEC)
-			return (EACCES);
-	} else
+		if ((targp = proc_find(uap->pid)) == 0 || !inferior(targp)) {
+			if (targp != PROC_NULL)
+				refheld = 1;
+			error = ESRCH;
+			goto out;
+		}
+		refheld = 1;
+		targp_sessp = proc_session(targp);
+		if (targp_sessp != curp_sessp) {
+			error = EPERM;
+			goto out;
+		}
+		if (targp->p_flag & P_EXEC) {
+			error = EACCES;
+			goto out;
+		}
+	} else {
 		targp = curp;
-	if (SESS_LEADER(targp))
-		return (EPERM);
-	if (uap->pgid < 0)
-		return(EINVAL);
+		targp_sessp = proc_session(targp);
+	}
+
+	if (SESS_LEADER(targp, targp_sessp)) {
+		error = EPERM;
+		goto out;
+	}
+	if (targp_sessp != SESSION_NULL) {
+		session_rele(targp_sessp);
+		targp_sessp = SESSION_NULL;
+	}
+
+	if (uap->pgid < 0) {
+		error = EINVAL;
+		goto out;
+	}
 	if (uap->pgid == 0)
 		uap->pgid = targp->p_pid;
-	else if (uap->pgid != targp->p_pid)
-		if ((pgrp = pgfind(uap->pgid)) == 0 ||
-		    pgrp->pg_session != curp->p_session)
-			return (EPERM);
-	return (enterpgrp(targp, uap->pgid, 0));
+	else if (uap->pgid != targp->p_pid) {
+		if ((pg = pgfind(uap->pgid)) == 0){
+			error = EPERM;
+			goto out;
+		}
+		samesess = (pg->pg_session != curp_sessp); 
+		pg_rele(pg);
+		if (samesess != 0) {
+			error = EPERM;
+			goto out;
+		}
+	}
+	error = enterpgrp(targp, uap->pgid, 0);
+out:
+	if (targp_sessp != SESSION_NULL)
+		session_rele(targp_sessp);
+	if (curp_sessp != SESSION_NULL)
+		session_rele(curp_sessp);
+	if (refheld != 0)
+		proc_rele(targp);
+	return(error);
 }
 
+
+/*
+ * issetugid
+ *
+ * Description:	Is current process tainted by uid or gid changes system call
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	0				Not tainted
+ *		1				Tainted
+ *
+ * Notes:	A process is considered tainted if it was created as a retult
+ *		of an execve call from an imnage that had either the SUID or
+ *		SGID bit set on the executable, or if it has changed any of its
+ *		real, effective, or saved user or group IDs since beginning
+ *		execution.
+ */
 int
-issetugid(struct proc *p, __unused struct issetugid_args *uap, register_t *retval)
+issetugid(proc_t p, __unused struct issetugid_args *uap, register_t *retval)
 {
 	/*
 	 * Note: OpenBSD sets a P_SUGIDEXEC flag set at execve() time,
@@ -359,227 +681,676 @@ issetugid(struct proc *p, __unused struct issetugid_args *uap, register_t *retva
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * setuid
+ *
+ * Description:	Set user ID system call
+ *
+ * Parameters:	uap->uid			uid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *
+ * Notes:	If called by a privileged process, this function will set the
+ *		real, effective, and saved uid to the requested value.
+ *
+ *		If called from an unprivileged process, but uid is equal to the
+ *		real or saved uid, then the effective uid will be set to the
+ *		requested value, but the real and saved uid will not change.
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ */
 int
-setuid(struct proc *p, struct setuid_args *uap, __unused register_t *retval)
+setuid(proc_t p, struct setuid_args *uap, __unused register_t *retval)
 {
-	register uid_t uid;
+	uid_t uid;
+	uid_t svuid = KAUTH_UID_NONE;
+	uid_t ruid = KAUTH_UID_NONE;
+	uid_t gmuid = KAUTH_UID_NONE;
 	int error;
 	kauth_cred_t my_cred, my_new_cred;
 
+
 	uid = uap->uid;
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	DEBUG_CRED_ENTER("setuid (%d/%d): %p %d\n", p->p_pid, (p->p_pptr ? p->p_pptr->p_pid : 0), my_cred, uap->uid);
 	AUDIT_ARG(uid, uid, 0, 0, 0);
-	if (uid != p->p_ucred->cr_ruid &&
-	    (error = suser(p->p_ucred, &p->p_acflag)))
+
+	if (uid != my_cred->cr_ruid &&	/* allow setuid(getuid()) */
+	    uid != my_cred->cr_svuid &&	/* allow setuid(saved uid) */
+	    (error = suser(my_cred, &p->p_acflag))) {
+		kauth_cred_unref(&my_cred);
 		return (error);
+	}
 	/*
 	 * Everything's okay, do it.
-	 * Transfer proc count to new user.
-	 * Copy credentials so other references do not see our changes.
 	 */
 
-	/* prepare app access profile files */
-	prepare_profile_database(uap->uid);
-	(void)chgproccnt(kauth_getruid(), -1);
-	(void)chgproccnt(uid, 1);
+	/*
+	 * If we are priviledged, then set the saved and real UID too;
+	 * otherwise, just set the effective UID
+	 */
+	if (suser(my_cred, &p->p_acflag) == 0) {
+		svuid = uid;
+		ruid = uid;
+		/*
+		 * Transfer proc count to new user.
+		 * chgproccnt uses list lock for protection
+		 */
+		(void)chgproccnt(uid, 1);
+		(void)chgproccnt(kauth_getruid(), -1);
+	}
 
 	/* get current credential and take a reference while we muck with it */
 	for (;;) {
-		my_cred = kauth_cred_proc_ref(p);
-		
-		/* 
+		/*
+		 * Only set the gmuid if the current cred has not opt'ed out;
+		 * this normally only happens when calling setgroups() instead
+		 * of initgroups() to set an explicit group list, or one of the
+		 * other group manipulation functions is invoked and results in
+		 * a dislocation (i.e. the credential group membership changes
+		 * to something other than the default list for the user, as
+		 * in entering a group or leaving an exclusion group).
+		 */
+		if (!(my_cred->cr_flags & CRF_NOMEMBERD))
+			gmuid = uid;
+
+  		/* 
 		 * Set the credential with new info.  If there is no change,
 		 * we get back the same credential we passed in; if there is
 		 * a change, we drop the reference on the credential we
 		 * passed in.  The subsequent compare is safe, because it is
 		 * a pointer compare rather than a contents compare.
-		 */
-		my_new_cred = kauth_cred_setuid(my_cred, uid);
+  		 */
+		my_new_cred = kauth_cred_setresuid(my_cred, ruid, uid, svuid, gmuid);
 		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("setuid CH(%d): %p/0x%08x -> %p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
 			proc_lock(p);
-			/* need to protect for a race where another thread also changed
-			 * the credential after we took our reference.  If p_ucred has 
-			 * changed then we should restart this again with the new cred.
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
 				kauth_cred_unref(&my_new_cred);
+				my_cred = kauth_cred_proc_ref(p);
 				/* try again */
 				continue;
 			}
 			p->p_ucred = my_new_cred;
-			p->p_flag |= P_SUGID;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
 			proc_unlock(p);
 		}
-		/* drop old proc reference or our extra reference */
+		break;
+	}
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+	
+	set_security_token(p);
+	return (0);
+}
+
+
+/*
+ * seteuid
+ *
+ * Description:	Set effective user ID system call
+ *
+ * Parameters:	uap->euid			effective uid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *
+ * Notes:	If called by a privileged process, or called from an
+ *		unprivileged process but euid is equal to the real or saved
+ *		uid, then the effective uid will be set to the requested
+ *		value, but the real and saved uid will not change.
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ */
+int
+seteuid(proc_t p, struct seteuid_args *uap, __unused register_t *retval)
+{
+	uid_t euid;
+	int error;
+	kauth_cred_t my_cred, my_new_cred;
+
+	DEBUG_CRED_ENTER("seteuid: %d\n", uap->euid);
+
+	euid = uap->euid;
+	AUDIT_ARG(uid, 0, euid, 0, 0);
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	if (euid != my_cred->cr_ruid && euid != my_cred->cr_svuid &&
+	    (error = suser(my_cred, &p->p_acflag))) {
 		kauth_cred_unref(&my_cred);
+		return (error);
+	}
+
+	/*
+	 * Everything's okay, do it.  Copy credentials so other references do
+	 * not see our changes.  get current credential and take a reference 
+	 * while we muck with it
+	 */
+	for (;;) {
+  		/* 
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
+  		 */
+		my_new_cred = kauth_cred_setresuid(my_cred, KAUTH_UID_NONE, euid, KAUTH_UID_NONE, my_cred->cr_gmuid);
+	
+		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("seteuid CH(%d): %p/0x%08x -> %p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
+			proc_lock(p);
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we
+			 * should restart this again with the new cred.
+			 */
+			if (p->p_ucred != my_cred) {
+				proc_unlock(p);
+				kauth_cred_unref(&my_new_cred);
+				my_cred = kauth_cred_proc_ref(p);
+				/* try again */
+				continue;
+			}
+			p->p_ucred = my_new_cred;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+			proc_unlock(p);
+		}
 		break;
 	}
+	/* drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+
+	set_security_token(p);
+	return (0);
+}
+
+
+/*
+ * setreuid
+ *
+ * Description:	Set real and effective user ID system call
+ *
+ * Parameters:	uap->ruid			real uid to set
+ *		uap->euid			effective uid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *
+ * Notes:	A value of -1 is a special case indicating that the uid for
+ *		which that value is specified not be changed.  If both values
+ *		are specified as -1, no action is taken.
+ *
+ *		If called by a privileged process, the real and effective uid
+ *		will be set to the new value(s) specified.
+ *
+ *		If called from an unprivileged process, the real uid may be
+ *		set to the current value of the real uid, or to the current
+ *		value of the saved uid.  The effective uid may be set to the
+ *		current value of any of the effective, real, or saved uid.
+ *
+ *		If the newly requested real uid or effective uid does not
+ *		match the saved uid, then set the saved uid to the new
+ *		effective uid (potentially unrecoverably dropping saved
+ *		privilege).
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ */
+int
+setreuid(proc_t p, struct setreuid_args *uap, __unused register_t *retval)
+{
+	uid_t ruid, euid;
+	int error;
+	kauth_cred_t my_cred, my_new_cred;
+
+	DEBUG_CRED_ENTER("setreuid %d %d\n", uap->ruid, uap->euid);
+
+	ruid = uap->ruid;
+	euid = uap->euid;
+	if (ruid == (uid_t)-1)
+		ruid = KAUTH_UID_NONE;
+	if (euid == (uid_t)-1)
+		euid = KAUTH_UID_NONE;
+	AUDIT_ARG(uid, euid, ruid, 0, 0);
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	if (((ruid != KAUTH_UID_NONE &&		/* allow no change of ruid */
+	      ruid != my_cred->cr_ruid &&	/* allow ruid = ruid */
+	      ruid != my_cred->cr_uid &&	/* allow ruid = euid */
+	      ruid != my_cred->cr_svuid) ||	/* allow ruid = svuid */
+	     (euid != KAUTH_UID_NONE &&		/* allow no change of euid */
+	      euid != my_cred->cr_uid &&	/* allow euid = euid */
+	      euid != my_cred->cr_ruid &&	/* allow euid = ruid */
+	      euid != my_cred->cr_svuid)) &&	/* allow euid = svui */
+	    (error = suser(my_cred, &p->p_acflag))) { /* allow root user any */
+		kauth_cred_unref(&my_cred);
+		return (error);
+	}
+
+	/*
+	 * Everything's okay, do it.  Copy credentials so other references do
+	 * not see our changes.  get current credential and take a reference 
+	 * while we muck with it
+	 */
+	for (;;) {
+		uid_t new_euid;
+		uid_t new_ruid;
+		uid_t svuid = KAUTH_UID_NONE;
+
+		new_euid = my_cred->cr_uid;
+		new_ruid = my_cred->cr_ruid;
 	
+  		/* 
+		 * Set the credential with new info.  If there is no change,
+		 * we get back the same credential we passed in; if there is
+		 * a change, we drop the reference on the credential we
+		 * passed in.  The subsequent compare is safe, because it is
+		 * a pointer compare rather than a contents compare.
+  		 */
+		if (euid == KAUTH_UID_NONE && my_cred->cr_uid != euid) {
+			/* changing the effective UID */
+			new_euid = euid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+		if (ruid != KAUTH_UID_NONE && my_cred->cr_ruid != ruid) {
+			/* changing the real UID; must do user accounting */
+		 	/* chgproccnt uses list lock for protection */
+			(void)chgproccnt(ruid, 1);
+			(void)chgproccnt(my_cred->cr_ruid, -1);
+			new_ruid = ruid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+		/*
+		 * If the newly requested real uid or effective uid does
+		 * not match the saved uid, then set the saved uid to the
+		 * new effective uid.  We are protected from escalation
+		 * by the prechecking.
+		 */
+		if (my_cred->cr_svuid != uap->ruid &&
+		    my_cred->cr_svuid != uap->euid) {
+		    	svuid = new_euid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+
+		my_new_cred = kauth_cred_setresuid(my_cred, ruid, euid, svuid, my_cred->cr_gmuid);
+	
+		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("setreuid CH(%d): %p/0x%08x -> %p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
+			proc_lock(p);
+			/*
+			 * We need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we should
+			 * restart this again with the new cred.
+			 */
+			if (p->p_ucred != my_cred) {
+				proc_unlock(p);
+				kauth_cred_unref(&my_new_cred);
+				my_cred = kauth_cred_proc_ref(p);
+				/* try again */
+				continue;
+			}
+			p->p_ucred = my_new_cred;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag); /* XXX redundant? */
+			proc_unlock(p);
+		}
+		break;
+	}
+	/* drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+
 	set_security_token(p);
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * setgid
+ *
+ * Description:	Set group ID system call
+ *
+ * Parameters:	uap->gid			gid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *
+ * Notes:	If called by a privileged process, this function will set the
+ *		real, effective, and saved gid to the requested value.
+ *
+ *		If called from an unprivileged process, but gid is equal to the
+ *		real or saved gid, then the effective gid will be set to the
+ *		requested value, but the real and saved gid will not change.
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ *
+ *		As an implementation detail, the effective gid is stored as
+ *		the first element of the supplementary group list, and
+ *		therefore the effective group list may be reordered to keep
+ *		the supplementary group list unchanged.
+ */
 int
-seteuid(struct proc *p, struct seteuid_args *uap, __unused register_t *retval)
+setgid(proc_t p, struct setgid_args *uap, __unused register_t *retval)
 {
-	register uid_t euid;
+	gid_t gid;
+	gid_t rgid = KAUTH_GID_NONE;
+	gid_t svgid = KAUTH_GID_NONE;
 	int error;
 	kauth_cred_t my_cred, my_new_cred;
 
-	euid = uap->euid;
-	AUDIT_ARG(uid, 0, euid, 0, 0);
-	if (euid != p->p_ucred->cr_ruid && euid != p->p_ucred->cr_svuid &&
-	    (error = suser(p->p_ucred, &p->p_acflag)))
+	DEBUG_CRED_ENTER("setgid(%d/%d): %d\n", p->p_pid, (p->p_pptr ? p->p_pptr->p_pid : 0), uap->gid);
+
+	gid = uap->gid;
+	AUDIT_ARG(gid, gid, 0, 0, 0);
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	if (gid != my_cred->cr_rgid &&	/* allow setgid(getgid()) */
+	    gid != my_cred->cr_svgid &&	/* allow setgid(saved gid) */
+	    (error = suser(my_cred, &p->p_acflag))) {
+		kauth_cred_unref(&my_cred);
 		return (error);
+	}
+
 	/*
-	 * Everything's okay, do it.  Copy credentials so other references do
-	 * not see our changes.  get current credential and take a reference 
-	 * while we muck with it
+	 * If we are priviledged, then set the saved and real GID too;
+	 * otherwise, just set the effective GID
 	 */
+	if (suser(my_cred,  &p->p_acflag) == 0) {
+		svgid = gid;
+		rgid = gid;
+	}
+
+	/* get current credential and take a reference while we muck with it */
 	for (;;) {
-		my_cred = kauth_cred_proc_ref(p);
-	
-		/* 
+		
+  		/* 
 		 * Set the credential with new info.  If there is no change,
 		 * we get back the same credential we passed in; if there is
 		 * a change, we drop the reference on the credential we
 		 * passed in.  The subsequent compare is safe, because it is
 		 * a pointer compare rather than a contents compare.
-		 */
-		my_new_cred = kauth_cred_seteuid(my_cred, euid);
-	
+  		 */
+		my_new_cred = kauth_cred_setresgid(my_cred, rgid, gid, svgid);
 		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("setgid(CH)%d: %p/0x%08x->%p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
 			proc_lock(p);
 			/*
 			 * We need to protect for a race where another thread
 			 * also changed the credential after we took our
-			 * reference.  If p_ucred has changed then we should
-			 * restart this again with the new cred.
+			 * reference.  If p_ucred has changed then we
+			 * should restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
 				kauth_cred_unref(&my_new_cred);
 				/* try again */
+				my_cred = kauth_cred_proc_ref(p);
 				continue;
 			}
 			p->p_ucred = my_new_cred;
-			p->p_flag |= P_SUGID;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
 			proc_unlock(p);
 		}
-		/* drop old proc reference or our extra reference */
-		kauth_cred_unref(&my_cred);
 		break;
 	}
-
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+	
 	set_security_token(p);
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * setegid
+ *
+ * Description:	Set effective group ID system call
+ *
+ * Parameters:	uap->egid			effective gid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM
+ *
+ * Notes:	If called by a privileged process, or called from an
+ *		unprivileged process but egid is equal to the real or saved
+ *		gid, then the effective gid will be set to the requested
+ *		value, but the real and saved gid will not change.
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ *
+ *		As an implementation detail, the effective gid is stored as
+ *		the first element of the supplementary group list, and
+ *		therefore the effective group list may be reordered to keep
+ *		the supplementary group list unchanged.
+ */
 int
-setgid(struct proc *p, struct setgid_args *uap, __unused register_t *retval)
+setegid(proc_t p, struct setegid_args *uap, __unused register_t *retval)
 {
-	register gid_t gid;
+	gid_t egid;
 	int error;
 	kauth_cred_t my_cred, my_new_cred;
 
-	gid = uap->gid;
-	AUDIT_ARG(gid, gid, 0, 0, 0);
-	if (gid != p->p_ucred->cr_rgid && (error = suser(p->p_ucred, &p->p_acflag)))
+	DEBUG_CRED_ENTER("setegid %d\n", uap->egid);
+
+	egid = uap->egid;
+	AUDIT_ARG(gid, 0, egid, 0, 0);
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	if (egid != my_cred->cr_rgid &&
+	    egid != my_cred->cr_svgid &&
+	    (error = suser(my_cred, &p->p_acflag))) {
+		kauth_cred_unref(&my_cred);
 		return (error);
+	}
 
 	/* get current credential and take a reference while we muck with it */
 	for (;;) {
-		my_cred = kauth_cred_proc_ref(p);
-		
-		/* 
+  		/* 
 		 * Set the credential with new info.  If there is no change,
 		 * we get back the same credential we passed in; if there is
 		 * a change, we drop the reference on the credential we
 		 * passed in.  The subsequent compare is safe, because it is
 		 * a pointer compare rather than a contents compare.
-		 */
-		my_new_cred = kauth_cred_setgid(my_cred, gid);
+  		 */
+		my_new_cred = kauth_cred_setresgid(my_cred, KAUTH_GID_NONE, egid, KAUTH_GID_NONE);
 		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("setegid(CH)%d: %p/0x%08x->%p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
 			proc_lock(p);
 			/*
 			 * We need to protect for a race where another thread
 			 * also changed the credential after we took our
-			 * reference.  If p_ucred has changed then we should
-			 * restart this again with the new cred.
+			 * reference.  If p_ucred has changed then we
+			 * should restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
 				kauth_cred_unref(&my_new_cred);
 				/* try again */
+				my_cred = kauth_cred_proc_ref(p);
 				continue;
 			}
 			p->p_ucred = my_new_cred;
-			p->p_flag |= P_SUGID;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
 			proc_unlock(p);
 		}
-		/* drop old proc reference or our extra reference */
-		kauth_cred_unref(&my_cred);
 		break;
 	}
-	
+
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
+
 	set_security_token(p);
 	return (0);
 }
 
-/* ARGSUSED */
+/*
+ * setregid
+ *
+ * Description:	Set real and effective group ID system call
+ *
+ * Parameters:	uap->rgid			real gid to set
+ *		uap->egid			effective gid to set
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *
+ * Notes:	A value of -1 is a special case indicating that the gid for
+ *		which that value is specified not be changed.  If both values
+ *		are specified as -1, no action is taken.
+ *
+ *		If called by a privileged process, the real and effective gid
+ *		will be set to the new value(s) specified.
+ *
+ *		If called from an unprivileged process, the real gid may be
+ *		set to the current value of the real gid, or to the current
+ *		value of the saved gid.  The effective gid may be set to the
+ *		current value of any of the effective, real, or saved gid.
+ *
+ *		If the new real and effective gid will not be equal, or the
+ *		new real or effective gid is not the same as the saved gid,
+ *		then the saved gid will be updated to reflect the new
+ *		effective gid (potentially unrecoverably dropping saved
+ *		privilege).
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ *
+ *		As an implementation detail, the effective gid is stored as
+ *		the first element of the supplementary group list, and
+ *		therefore the effective group list may be reordered to keep
+ *		the supplementary group list unchanged.
+ */
 int
-setegid(struct proc *p, struct setegid_args *uap, __unused register_t *retval)
+setregid(proc_t p, struct setregid_args *uap, __unused register_t *retval)
 {
-	register gid_t egid;
+	gid_t rgid, egid;
 	int error;
 	kauth_cred_t my_cred, my_new_cred;
 
+	DEBUG_CRED_ENTER("setregid %d %d\n", uap->rgid, uap->egid);
+
+	rgid = uap->rgid;
 	egid = uap->egid;
-	AUDIT_ARG(gid, 0, egid, 0, 0);
-	if (egid != p->p_ucred->cr_rgid && egid != p->p_ucred->cr_svgid &&
-	    (error = suser(p->p_ucred, &p->p_acflag)))
+
+	if (rgid == (uid_t)-1)
+		rgid = KAUTH_GID_NONE;
+	if (egid == (uid_t)-1)
+		egid = KAUTH_GID_NONE;
+	AUDIT_ARG(gid, egid, rgid, 0, 0);
+
+	my_cred = kauth_cred_proc_ref(p);
+
+	if (((rgid != KAUTH_UID_NONE &&		/* allow no change of rgid */
+	      rgid != my_cred->cr_rgid &&	/* allow rgid = rgid */
+	      rgid != my_cred->cr_gid &&	/* allow rgid = egid */
+	      rgid != my_cred->cr_svgid) ||	/* allow rgid = svgid */
+	     (egid != KAUTH_UID_NONE &&		/* allow no change of egid */
+	      egid != my_cred->cr_groups[0] &&	/* allow no change of egid */
+	      egid != my_cred->cr_gid &&	/* allow egid = egid */
+	      egid != my_cred->cr_rgid &&	/* allow egid = rgid */
+	      egid != my_cred->cr_svgid)) &&	/* allow egid = svgid */
+	    (error = suser(my_cred, &p->p_acflag))) { /* allow root user any */
+		kauth_cred_unref(&my_cred);
 		return (error);
+	}
 
 	/* get current credential and take a reference while we muck with it */
 	for (;;) {
-		my_cred = kauth_cred_proc_ref(p);
+		uid_t new_egid = my_cred->cr_gid;
+		uid_t new_rgid = my_cred->cr_rgid;
+		uid_t svgid = KAUTH_UID_NONE;
+
 		
-		/* 
+  		/* 
 		 * Set the credential with new info.  If there is no change,
 		 * we get back the same credential we passed in; if there is
 		 * a change, we drop the reference on the credential we
 		 * passed in.  The subsequent compare is safe, because it is
 		 * a pointer compare rather than a contents compare.
+  		 */
+		if (egid == KAUTH_UID_NONE && my_cred->cr_groups[0] != egid) {
+			/* changing the effective GID */
+			new_egid = egid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+		if (rgid != KAUTH_UID_NONE && my_cred->cr_rgid != rgid) {
+			/* changing the real GID */
+			new_rgid = rgid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+		/*
+		 * If the newly requested real gid or effective gid does
+		 * not match the saved gid, then set the saved gid to the
+		 * new effective gid.  We are protected from escalation
+		 * by the prechecking.
 		 */
-		my_new_cred = kauth_cred_setegid(my_cred, egid);
+		if (my_cred->cr_svgid != uap->rgid &&
+		    my_cred->cr_svgid != uap->egid) {
+		    	svgid = new_egid;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
+		}
+
+		my_new_cred = kauth_cred_setresgid(my_cred, rgid, egid, svgid);
 		if (my_cred != my_new_cred) {
+
+			DEBUG_CRED_CHANGE("setregid(CH)%d: %p/0x%08x->%p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
 			proc_lock(p);
-			/* need to protect for a race where another thread also changed
-			 * the credential after we took our reference.  If p_ucred has 
-			 * changed then we should restart this again with the new cred.
+			/* need to protect for a race where another thread
+			 * also changed the credential after we took our
+			 * reference.  If p_ucred has changed then we
+			 * should restart this again with the new cred.
 			 */
 			if (p->p_ucred != my_cred) {
 				proc_unlock(p);
 				kauth_cred_unref(&my_new_cred);
 				/* try again */
+				my_cred = kauth_cred_proc_ref(p);
 				continue;
 			}
 			p->p_ucred = my_new_cred;
-			p->p_flag |= P_SUGID;
+			OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag); /* XXX redundant? */
 			proc_unlock(p);
 		}
-		/* drop old proc reference or our extra reference */
-		kauth_cred_unref(&my_cred);
 		break;
 	}
+	/* Drop old proc reference or our extra reference */
+	kauth_cred_unref(&my_cred);
 
 	set_security_token(p);
 	return (0);
 }
 
+
 /*
  * Set the per-thread override identity.  The first parameter can be the
  * current real UID, KAUTH_UID_NONE, or, if the caller is priviledged, it
@@ -589,20 +1360,19 @@ setegid(struct proc *p, struct setegid_args *uap, __unused register_t *retval)
  * thread to the requested UID and single GID, and clears all other GIDs.
  */
 int
-settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
+settid(proc_t p, struct settid_args *uap, __unused register_t *retval)
 {
 	kauth_cred_t uc;
 	struct uthread *uthread = get_bsdthread_info(current_thread());
-	register uid_t uid;
-	register gid_t gid;
+	uid_t uid;
+	gid_t gid;
 
 	uid = uap->uid;
 	gid = uap->gid;
 	AUDIT_ARG(uid, uid, gid, gid, 0);
 
-	if (suser(p->p_ucred, &p->p_acflag) != 0) {
+	if (proc_suser(p) != 0) 
 		return (EPERM);
-	}
 	
 	if (uid == KAUTH_UID_NONE) {
 
@@ -624,9 +1394,11 @@ settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
 		}
 
 		/*
-		 * get a new credential instance from the old if this one changes else
-		 * kauth_cred_setuidgid returns the same credential.  we take an extra 
-		 * reference on the current credential while we muck wit it here.
+		 * Get a new credential instance from the old if this one
+		 * changes; otherwise kauth_cred_setuidgid() returns the
+		 * same credential.  We take an extra reference on the
+		 * current credential while we muck with it, so we can do
+		 * the post-compare for changes by pointer.
 		 */
 		kauth_cred_ref(uthread->uu_ucred); 
 		my_cred = uthread->uu_ucred;
@@ -635,7 +1407,7 @@ settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
 			uthread->uu_ucred = my_new_cred;
 		uthread->uu_flag |= UT_SETUID;
 
-		/* drop our extra reference */
+		/* Drop old uthread reference or our extra reference */
 		kauth_cred_unref(&my_cred);
 	}
 	/*
@@ -647,16 +1419,19 @@ settid(struct proc *p, struct settid_args *uap, __unused register_t *retval)
 	return (0);
 }
 
+
 /*
  * Set the per-thread override identity.  Use this system call for a thread to
- * assume the identity of another process or to revert back to normal identity 
+ * assume the identity of another process or to revert back to normal identity
  * of the current process.
- * When the "assume" argument is non zero the current thread will assume the 
+ *
+ * When the "assume" argument is non zero the current thread will assume the
  * identity of the process represented by the pid argument.
+ *
  * When the assume argument is zero we revert back to our normal identity.
  */
 int
-settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused register_t *retval)
+settid_with_pid(proc_t p, struct settid_with_pid_args *uap, __unused register_t *retval)
 {
 	proc_t target_proc;
 	struct uthread *uthread = get_bsdthread_info(current_thread());
@@ -665,7 +1440,7 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 	AUDIT_ARG(pid, uap->pid);
 	AUDIT_ARG(value, uap->assume);
 
-	if (suser(p->p_ucred, &p->p_acflag) != 0) {
+	if (proc_suser(p) != 0) {
 		return (EPERM);
 	}
 
@@ -685,15 +1460,25 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 		if ((uthread->uu_flag & UT_SETUID) != 0)
 			return (EPERM);
 	
-		target_proc = pfind(uap->pid);
+		target_proc = proc_find(uap->pid);
 		/* can't assume the identity of the kernel process */
 		if (target_proc == NULL || target_proc == kernproc) {
+			if (target_proc!= NULL)
+				proc_rele(target_proc);
 			return (ESRCH);
 		}
 	
 		/*
-		 * take a reference on the credential used in our target process then use
-		 * it as the identity for our current thread.
+		 * Take a reference on the credential used in our target
+		 * process then use it as the identity for our current
+		 * thread.  We take an extra reference on the current
+		 * credential while we muck with it, so we can do the
+		 * post-compare for changes by pointer.
+		 *
+		 * The post-compare is needed for the case that our process
+		 * credential has been changed to be identical to our thread
+		 * credential following our assumption of a per-thread one,
+		 * since the credential cache will maintain a unique instance.
 		 */
 		kauth_cred_ref(uthread->uu_ucred); 
 		my_cred = uthread->uu_ucred;
@@ -704,15 +1489,18 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 	
 		uthread->uu_flag |= UT_SETUID;
 		
-		/* drop our extra references */
+		/* Drop old uthread reference or our extra reference */
+		proc_rele(target_proc);
 		kauth_cred_unref(&my_cred);
 		kauth_cred_unref(&my_target_cred);
 
 		return (0);
 	}
 	
-	/* we are reverting back to normal mode of operation where delayed binding
-	 * of the process credential sets the credential in the thread (uu_ucred)
+	/*
+	 * Otherwise, we are reverting back to normal mode of operation where
+	 * delayed binding of the process credential sets the credential in
+	 * the thread (uu_ucred)
 	 */
 	if ((uthread->uu_flag & UT_SETUID) == 0)
 		return (EPERM);
@@ -726,26 +1514,60 @@ settid_with_pid(struct proc *p, struct settid_with_pid_args *uap, __unused regis
 	return (0);
 }
 
-/* ARGSUSED */
+
+/*
+ * setgroups1
+ *
+ * Description: Internal implementation for both the setgroups and initgroups
+ *		system calls
+ *
+ * Parameters:	gidsetsize			Number of groups in set
+ *		gidset				Pointer to group list
+ *		gmuid				Base gid (initgroups only!)
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permision denied
+ *		EINVAL				Invalid gidsetsize value
+ *	copyin:EFAULT				Bad gidset or gidsetsize is
+ *						too large
+ *
+ * Notes:	When called from a thread running under an assumed per-thread
+ *		identity, this function will operate against the per-thread
+ *		credential, rather than against the process credential.  In
+ *		this specific case, the process credential is verified to
+ *		still be privileged at the time of the call, rather than the
+ *		per-thread credential for this operation to be permitted.
+ *
+ *		This effectively means that setgroups/initigroups calls in
+ *		a thread running a per-thread credential should occur *after*
+ *		the settid call that created it, not before (unlike setuid,
+ *		which must be called after, since it will result in privilege
+ *		being dropped).
+ *
+ *		When called normally (i.e. no per-thread assumed identity),
+ *		the per process credential is updated per POSIX.
+ *
+ *		If the credential is changed as a result of this call, then we
+ *		flag the process as having set privilege since the last exec.
+ */
 static int
-setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __unused register_t *retval)
+setgroups1(proc_t p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __unused register_t *retval)
 {
-	register u_int ngrp;
+	u_int ngrp;
 	gid_t	newgroups[NGROUPS] = { 0 };
 	int 	error;
 	kauth_cred_t my_cred, my_new_cred;
 	struct uthread *uthread = get_bsdthread_info(current_thread());
 
-	if ((error = suser(p->p_ucred, &p->p_acflag)))
-		return (error);
+	DEBUG_CRED_ENTER("setgroups1 (%d/%d): %d 0x%016x %d\n", p->p_pid, (p->p_pptr ? p->p_pptr->p_pid : 0), gidsetsize, gidset, gmuid);
+
 	ngrp = gidsetsize;
 	if (ngrp > NGROUPS)
 		return (EINVAL);
 
 	if ( ngrp < 1 ) {
 		ngrp = 1;
-	}
-	else {
+	} else {
 		error = copyin(gidset,
 			(caddr_t)newgroups, ngrp * sizeof(gid_t));
 		if (error) {
@@ -753,7 +1575,18 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 		}
 	}
 
+	my_cred = kauth_cred_proc_ref(p);
+	if ((error = suser(my_cred, &p->p_acflag))) {
+		kauth_cred_unref(&my_cred);
+		return (error);
+	}
+
 	if ((uthread->uu_flag & UT_SETUID) != 0) {
+#if DEBUG_CRED
+		int my_cred_flags = uthread->uu_ucred->cr_flags;
+#endif	/* DEBUG_CRED */
+		kauth_cred_unref(&my_cred);
+
 		/*
 		 * If this thread is under an assumed identity, set the
 		 * supplementary grouplist on the thread credential instead
@@ -762,13 +1595,14 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 		 * is dropped and we get back a different cred with a reference
 		 * already held on it.  Because this is per-thread, we don't
 		 * need the referencing/locking/retry required for per-process.
-		 *
-		 * Hack: this opts into memberd to avoid needing to use a per
-		 * thread credential initgroups() instead of setgroups() in
-		 * AFP server to address <rdar://4561060>
 		 */
 		my_cred = uthread->uu_ucred;
-		uthread->uu_ucred = kauth_cred_setgroups(my_cred, &newgroups[0], ngrp, my_cred->cr_gmuid);
+		uthread->uu_ucred = kauth_cred_setgroups(my_cred, &newgroups[0], ngrp, gmuid);
+#if DEBUG_CRED
+		if (my_cred != uthread->uu_ucred) {
+			DEBUG_CRED_CHANGE("setgroups1(CH)%d: %p/0x%08x->%p/0x%08x\n", p->p_pid, my_cred, my_cred_flags, uthread->uu_ucred , uthread->uu_ucred ->cr_flags);
+		}
+#endif	/* DEBUG_CRED */
 	} else {
 
 		/*
@@ -776,8 +1610,6 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 		 * with it
 		 */
 		for (;;) {
-			my_cred = kauth_cred_proc_ref(p);
-
 			/* 
 			 * Set the credential with new info.  If there is no
 			 * change, we get back the same credential we passed
@@ -788,9 +1620,12 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 			 */
 			my_new_cred = kauth_cred_setgroups(my_cred, &newgroups[0], ngrp, gmuid);
 			if (my_cred != my_new_cred) {
+
+				DEBUG_CRED_CHANGE("setgroups1(CH)%d: %p/0x%08x->%p/0x%08x\n", p->p_pid, my_cred, my_cred->cr_flags, my_new_cred, my_new_cred->cr_flags);
+
 				proc_lock(p);
 				/*
-				 * need to protect for a race where another
+				 * We need to protect for a race where another
 				 * thread also changed the credential after we
 				 * took our reference.  If p_ucred has 
 				 * changed then we should restart this again
@@ -799,43 +1634,106 @@ setgroups1(struct proc *p, u_int gidsetsize, user_addr_t gidset, uid_t gmuid, __
 				if (p->p_ucred != my_cred) {
 					proc_unlock(p);
 					kauth_cred_unref(&my_new_cred);
+					my_cred = kauth_cred_proc_ref(p);
 					/* try again */
 					continue;
 				}
 				p->p_ucred = my_new_cred;
-				p->p_flag |= P_SUGID;
+				OSBitOrAtomic(P_SUGID, (UInt32 *)&p->p_flag);
 				proc_unlock(p);
 			}
-			/* drop old proc reference or our extra reference */
-			kauth_cred_unref(&my_cred);
 			break;
 		}
+		/* Drop old proc reference or our extra reference */
+		AUDIT_ARG(groupset, my_cred->cr_groups, ngrp);
+		kauth_cred_unref(&my_cred);
+
 
-		AUDIT_ARG(groupset, p->p_ucred->cr_groups, ngrp);
 		set_security_token(p);
 	}
 
 	return (0);
 }
 
+
+/*
+ * initgroups
+ *
+ * Description: Initialize the default supplementary groups list and set the
+ *		gmuid for use by the external group resolver (if any)
+ *
+ * Parameters:	uap->gidsetsize			Number of groups in set
+ *		uap->gidset			Pointer to group list
+ *		uap->gmuid			Base gid
+ *
+ * Returns:	0				Success
+ *	setgroups1:EPERM			Permision denied
+ *	setgroups1:EINVAL			Invalid gidsetsize value
+ *	setgroups1:EFAULT			Bad gidset or gidsetsize is
+ *
+ * Notes:	This function opts *IN* to memberd participation
+ *
+ *		The normal purpose of this function is for a privileged
+ *		process to indicate supplementary groups and identity for
+ *		participation in extended group membership resolution prior
+ *		to dropping privilege by assuming a specific user identity.
+ *
+ *		It is the first half of the primary mechanism whereby user
+ *		identity is established to the system by programs such as
+ *		/usr/bin/login.  The second half is the drop of uid privilege
+ *		for a specific uid corresponding to the user.
+ *
+ * See also:	setgroups1()
+ */
 int
-initgroups(struct proc *p, struct initgroups_args *uap, __unused register_t *retval)
+initgroups(proc_t p, struct initgroups_args *uap, __unused register_t *retval)
 {
+	DEBUG_CRED_ENTER("initgroups\n");
+
 	return(setgroups1(p, uap->gidsetsize, uap->gidset, uap->gmuid, retval));
 }
 
+
+/*
+ * setgroups
+ *
+ * Description: Initialize the default supplementary groups list
+ *
+ * Parameters:	gidsetsize			Number of groups in set
+ *		gidset				Pointer to group list
+ *
+ * Returns:	0				Success
+ *	setgroups1:EPERM			Permision denied
+ *	setgroups1:EINVAL			Invalid gidsetsize value
+ *	setgroups1:EFAULT			Bad gidset or gidsetsize is
+ *
+ * Notes:	This functions opts *OUT* of memberd participation.
+ *
+ *		This function exists for compatibility with POSIX.  Most user
+ *		programs should use initgroups() instead to ensure correct
+ *		participation in group membership resolution when utilizing
+ *		a directory service for authentication.
+ *
+ *		It is identical to an initgroups() call with a gmuid argument
+ *		of KAUTH_UID_NONE.
+ *
+ * See also:	setgroups1()
+ */
 int
-setgroups(struct proc *p, struct setgroups_args *uap, __unused register_t *retval)
+setgroups(proc_t p, struct setgroups_args *uap, __unused register_t *retval)
 {
+	DEBUG_CRED_ENTER("setgroups\n");
+
 	return(setgroups1(p, uap->gidsetsize, uap->gidset, KAUTH_UID_NONE, retval));
 }
 
+
 /*
  * Set the per-thread/per-process supplementary groups list.
  */
-#warning XXX implement
+#warning XXX implement setsgroups
 int
-setsgroups(__unused struct proc *p, __unused struct setsgroups_args *uap, __unused register_t *retval)
+setsgroups(__unused proc_t p, __unused struct setsgroups_args *uap, __unused register_t *retval)
 {
 	return(ENOTSUP);
 }
@@ -843,17 +1741,19 @@ setsgroups(__unused struct proc *p, __unused struct setsgroups_args *uap, __unus
 /*
  * Set the per-thread/per-process whiteout groups list.
  */
-#warning XXX implement
+#warning XXX implement setwgroups
 int
-setwgroups(__unused struct proc *p, __unused struct setwgroups_args *uap, __unused register_t *retval)
+setwgroups(__unused proc_t p, __unused struct setwgroups_args *uap, __unused register_t *retval)
 {
 	return(ENOTSUP);
 }
 
+
 /*
  * Check if gid is a member of the group set.
  *
- * XXX This interface is going away
+ * XXX This interface is going away; use kauth_cred_ismember_gid() directly
+ * XXX instead.
  */
 int
 groupmember(gid_t gid, kauth_cred_t cred)
@@ -865,13 +1765,20 @@ groupmember(gid_t gid, kauth_cred_t cred)
 	return (0);
 }
 
+
 /*
  * Test whether the specified credentials imply "super-user"
  * privilege; if so, and we have accounting info, set the flag
  * indicating use of super-powers.
  * Returns 0 or error.
  *
- * XXX This interface is going away
+ * XXX This interface is going away; use kauth_cred_issuser() directly
+ * XXX instead.
+ *
+ * Note:	This interface exists to implement the "has used privilege"
+ *		bit (ASU) in the p_acflags field of the process, which is
+ *		only externalized via private sysctl and in process accounting
+ *		records.  The flag is technically not required in either case.
  */
 int
 suser(kauth_cred_t cred, u_short *acflag)
@@ -888,62 +1795,140 @@ suser(kauth_cred_t cred, u_short *acflag)
 	return (EPERM);
 }
 
+
+/*
+ * XXX This interface is going away; use kauth_cred_issuser() directly
+ * XXX instead.
+ */
 int
 is_suser(void)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
 
 	if (!p)
 		return (0);
 
-	return (suser(p->p_ucred, &p->p_acflag) == 0);
+	return (proc_suser(p) == 0);
 }
 
+
+/*
+ * XXX This interface is going away; use kauth_cred_issuser() directly
+ * XXX instead.
+ */
 int
 is_suser1(void)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
+	kauth_cred_t my_cred;
+	int err;
 
 	if (!p)
 		return (0);
 
-	return (suser(p->p_ucred, &p->p_acflag) == 0 ||
-			p->p_ucred->cr_ruid == 0 || p->p_ucred->cr_svuid == 0);
+	my_cred = kauth_cred_proc_ref(p);
+
+	err =  (suser(my_cred, &p->p_acflag) == 0 ||
+			my_cred->cr_ruid == 0 || my_cred->cr_svuid == 0);
+	kauth_cred_unref(&my_cred);
+	return(err);
 }
 
+
 /*
- * Get login name, if available.
+ * getlogin
+ *
+ * Description:	Get login name, if available.
+ *
+ * Parameters:	uap->namebuf			User buffer for return
+ *		uap->namelen			User buffer length
+ *
+ * Returns:	0				Success
+ *	copyout:EFAULT
+ *
+ * Notes:	Intended to obtain a string containing the user name of the
+ *		user associated with the controlling terminal for the calling
+ *		process.
+ *
+ *		Not very useful on modern systems, due to inherent length
+ *		limitations for the static array in the session structure
+ *		which is used to store the login name.
+ *
+ *		Permitted to return NULL
+ *
+ * XXX:		Belongs in kern_proc.c
  */
-/* ARGSUSED */
 int
-getlogin(struct proc *p, struct getlogin_args *uap, __unused register_t *retval)
+getlogin(proc_t p, struct getlogin_args *uap, __unused register_t *retval)
 {
+	char buffer[MAXLOGNAME+1];
+	struct session * sessp;
+
+	bzero(buffer, MAXLOGNAME+1);
+
+	sessp = proc_session(p);
 
-	if (uap->namelen > sizeof (p->p_pgrp->pg_session->s_login))
-		uap->namelen = sizeof (p->p_pgrp->pg_session->s_login);
-	return (copyout((caddr_t) p->p_pgrp->pg_session->s_login,
-	   				uap->namebuf, uap->namelen));
+	if (uap->namelen > MAXLOGNAME)
+		uap->namelen = MAXLOGNAME;
+
+	if(sessp != SESSION_NULL) {
+		session_lock(sessp);
+		bcopy( sessp->s_login, buffer, uap->namelen);
+		session_unlock(sessp);
+	}
+	session_rele(sessp);
+
+	return (copyout((caddr_t)buffer, uap->namebuf, uap->namelen));
 }
 
+
 /*
- * Set login name.
+ * setlogin
+ *
+ * Description:	Set login name.
+ *
+ * Parameters:	uap->namebuf			User buffer containing name
+ *
+ * Returns:	0				Success
+ *	suser:EPERM				Permission denied
+ *	copyinstr:EFAULT			User buffer invalid
+ *	copyinstr:EINVAL			Supplied name was too long
+ *
+ * Notes:	This is a utility system call to support getlogin().
+ *
+ * XXX:		Belongs in kern_proc.c
  */
-/* ARGSUSED */
 int
-setlogin(struct proc *p, struct setlogin_args *uap, __unused register_t *retval)
+setlogin(proc_t p, struct setlogin_args *uap, __unused register_t *retval)
 {
 	int error;
 	int dummy=0;
+	char buffer[MAXLOGNAME+1];
+	struct session * sessp;
 
-	if ((error = suser(p->p_ucred, &p->p_acflag)))
+	if ((error = proc_suser(p)))
 		return (error);
-	 
+
+	bzero(&buffer[0], MAXLOGNAME+1);
+
+
 	error = copyinstr(uap->namebuf,
-	    (caddr_t) p->p_pgrp->pg_session->s_login,
-	    sizeof (p->p_pgrp->pg_session->s_login) - 1, (size_t *)&dummy);
-	if (!error)
-		AUDIT_ARG(text, p->p_pgrp->pg_session->s_login);
-	else if (error == ENAMETOOLONG)
+	    (caddr_t) &buffer[0],
+	    MAXLOGNAME - 1, (size_t *)&dummy);
+
+	sessp = proc_session(p);
+
+	if (sessp != SESSION_NULL) {
+		session_lock(sessp);
+		bcopy(buffer, sessp->s_login, MAXLOGNAME);
+		session_unlock(sessp);
+		session_rele(sessp);
+	}
+
+
+	if (!error) {
+		AUDIT_ARG(text, buffer);
+	 } else if (error == ENAMETOOLONG)
 		error = EINVAL;
 	return (error);
 }
@@ -955,10 +1940,12 @@ setlogin(struct proc *p, struct setlogin_args *uap, __unused register_t *retval)
  * XXX identifier.
  */
 int
-set_security_token(struct proc * p)
+set_security_token(proc_t p)
 {
 	security_token_t sec_token;
 	audit_token_t    audit_token;
+	kauth_cred_t my_cred;
+	host_priv_t host_priv;
 
 	/*
 	 * Don't allow a vfork child to override the parent's token settings
@@ -973,10 +1960,11 @@ set_security_token(struct proc * p)
 			return (1);
 	}
 		
+	my_cred = kauth_cred_proc_ref(p);
 	/* XXX mach_init doesn't have a p_ucred when it calls this function */
-	if (IS_VALID_CRED(p->p_ucred)) {
-		sec_token.val[0] = kauth_cred_getuid(p->p_ucred);
-		sec_token.val[1] = p->p_ucred->cr_gid;
+	if (IS_VALID_CRED(my_cred)) {
+		sec_token.val[0] = kauth_cred_getuid(my_cred);
+		sec_token.val[1] = my_cred->cr_gid;
 	} else {
 		sec_token.val[0] = 0;
 		sec_token.val[1] = 0;
@@ -991,22 +1979,31 @@ set_security_token(struct proc * p)
 	 * the user of the trailer from future representation
 	 * changes.
 	 */
-	audit_token.val[0] = p->p_ucred->cr_au.ai_auid;
-	audit_token.val[1] = p->p_ucred->cr_uid;
-	audit_token.val[2] = p->p_ucred->cr_gid;
-	audit_token.val[3] = p->p_ucred->cr_ruid;
-	audit_token.val[4] = p->p_ucred->cr_rgid;
+	audit_token.val[0] = my_cred->cr_au.ai_auid;
+	audit_token.val[1] = my_cred->cr_uid;
+	audit_token.val[2] = my_cred->cr_gid;
+	audit_token.val[3] = my_cred->cr_ruid;
+	audit_token.val[4] = my_cred->cr_rgid;
 	audit_token.val[5] = p->p_pid;
-	audit_token.val[6] = p->p_ucred->cr_au.ai_asid;
-	audit_token.val[7] = p->p_ucred->cr_au.ai_termid.port;
+	audit_token.val[6] = my_cred->cr_au.ai_asid;
+	audit_token.val[7] = my_cred->cr_au.ai_termid.port;
+
+#if CONFIG_MACF_MACH
+	mac_task_label_update_cred(my_cred, p->task);
+#endif
+	
+	host_priv = (sec_token.val[0]) ? HOST_PRIV_NULL : host_priv_self();
+#if CONFIG_MACF
+	if (host_priv != HOST_PRIV_NULL && mac_system_check_host_priv(my_cred))
+		host_priv = HOST_PRIV_NULL;
+#endif
+	kauth_cred_unref(&my_cred);
 
 	return (host_security_set_task_token(host_security_self(),
 					   p->task,
 					   sec_token,
 					   audit_token,
-					   (sec_token.val[0]) ?
-						HOST_PRIV_NULL :
-						host_priv_self()) != KERN_SUCCESS);
+					   host_priv) != KERN_SUCCESS);
 }
 
 
@@ -1024,3 +2021,170 @@ cru2x(kauth_cred_t cr, struct xucred *xcr)
 	xcr->cr_ngroups = cr->cr_ngroups;
 	bcopy(cr->cr_groups, xcr->cr_groups, sizeof(xcr->cr_groups));
 }
+
+#if CONFIG_LCTX
+
+/*
+ * Set Login Context ID
+ */
+/*
+ * MPSAFE - assignment of (visible) process to context protected by ALLLCTX_LOCK,
+ *	    LCTX by its own locks.
+ */
+int
+setlcid(proc_t p0, struct setlcid_args *uap, __unused register_t *retval)
+{
+	proc_t p;
+	struct lctx *l;
+	int error = 0;
+	int refheld = 0;
+
+	AUDIT_ARG(pid, uap->pid);
+	AUDIT_ARG(value, uap->lcid);
+	if (uap->pid == LCID_PROC_SELF) {	/* Create/Join/Leave */
+		p = p0;
+	} else {				/* Adopt/Orphan */
+		p = proc_find(uap->pid);
+		if (p == NULL)
+			return (ESRCH);
+		refheld = 1;
+	}
+
+#if CONFIG_MACF
+	error = mac_proc_check_setlcid(p0, p, uap->pid, uap->lcid);
+	if (error)
+		goto out;
+#endif
+
+	switch (uap->lcid) {
+	/* Leave/Orphan */
+	case LCID_REMOVE:
+
+		/* Only root may Leave/Orphan. */
+		if (!is_suser1()) {
+			error = EPERM;
+			goto out;
+		}
+
+		/* Process not in login context. */
+		if (p->p_lctx == NULL) {
+			error = ENOATTR;
+			goto out;
+		}
+
+		l = NULL;
+
+		break;
+
+	/* Create */
+	case LCID_CREATE:
+
+		/* Create only valid for self! */
+		if (uap->pid != LCID_PROC_SELF) {
+			error = EPERM;
+			goto out;
+		}
+
+		/* Already in a login context. */
+		if (p->p_lctx != NULL) {
+			error = EPERM;
+			goto out;
+		}
+
+		l = lccreate();
+		if (l == NULL) {
+			error = ENOMEM;
+			goto out;
+		}
+
+		LCTX_LOCK(l);
+
+		break;
+
+	/* Join/Adopt */
+	default:
+
+		/* Only root may Join/Adopt. */
+		if (!is_suser1()) {
+			error = EPERM;
+			goto out;
+		}
+
+		l = lcfind(uap->lcid);
+		if (l == NULL) {
+			error = ENOATTR;
+			goto out;
+		}
+
+		break;
+	}
+
+	ALLLCTX_LOCK;
+	leavelctx(p);
+	enterlctx(p, l, (uap->lcid == LCID_CREATE) ? 1 : 0);
+	ALLLCTX_UNLOCK;
+
+out:
+	if (refheld != 0)
+		proc_rele(p);
+	return (error);
+}
+
+/*
+ * Get Login Context ID
+ */
+/*
+ * MPSAFE - membership of (visible) process in a login context
+ *	    protected by the all-context lock.
+ */
+int
+getlcid(proc_t p0, struct getlcid_args *uap, register_t *retval)
+{
+	proc_t p;
+	int error = 0;
+	int refheld = 0;
+
+	AUDIT_ARG(pid, uap->pid);
+	if (uap->pid == LCID_PROC_SELF) {
+		p = p0;
+	} else {
+		p = proc_find(uap->pid);
+		if (p == NULL)
+			return (ESRCH);
+		refheld = 1;
+	}
+
+#if CONFIG_MACF
+	error = mac_proc_check_getlcid(p0, p, uap->pid);
+	if (error)
+		goto out;
+#endif
+	ALLLCTX_LOCK;
+	if (p->p_lctx == NULL) {
+		error = ENOATTR;
+		ALLLCTX_UNLOCK;
+		goto out;
+	}
+	*retval = p->p_lctx->lc_id;
+	ALLLCTX_UNLOCK;
+ out:
+	if (refheld != 0)
+		proc_rele(p);
+
+	return (error);
+}
+#else	/* LCTX */
+int
+setlcid(proc_t p0, struct setlcid_args *uap, register_t *retval)
+{
+
+	return (ENOSYS);
+}
+
+int
+getlcid(proc_t p0, struct getlcid_args *uap, register_t *retval)
+{
+
+	return (ENOSYS);
+}
+#endif	/* !LCTX */
diff --git a/bsd/kern/kern_resource.c b/bsd/kern/kern_resource.c
index 3a8419566..5e2f8c171 100644
--- a/bsd/kern/kern_resource.c
+++ b/bsd/kern/kern_resource.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -59,6 +65,12 @@
  *
  *	@(#)kern_resource.c	8.5 (Berkeley) 1/21/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -80,16 +92,27 @@
 
 #include <mach/mach_types.h>
 #include <mach/time_value.h>
+#include <mach/task.h>
 #include <mach/task_info.h>
 #include <mach/vm_map.h>
+#include <mach/mach_vm.h>
+#include <mach/thread_act.h>  /* for thread_policy_set( ) */
+#include <kern/lock.h>
+#include <kern/thread.h>
+
+#include <kern/task.h>
+#include <kern/clock.h>		/* for absolutetime_to_microtime() */
+#include <netinet/in.h>		/* for TRAFFIC_MGT_SO_BACKGROUND */
+#include <sys/socketvar.h>	/* for struct socket */
 
 #include <vm/vm_map.h>
 
 int	donice(struct proc *curp, struct proc *chgp, int n);
 int	dosetrlimit(struct proc *p, u_int which, struct rlimit *limp);
+static int do_background_thread(struct proc *curp, int priority);
 
 rlim_t maxdmap = MAXDSIZ;	/* XXX */ 
-rlim_t maxsmap = MAXSSIZ;	/* XXX */ 
+rlim_t maxsmap = MAXSSIZ - PAGE_SIZE;	/* XXX */ 
 
 /*
  * Limits on the number of open files per process, and the number
@@ -97,7 +120,6 @@ rlim_t maxsmap = MAXSSIZ;	/* XXX */
  *
  * Note: would be in kern/subr_param.c in FreeBSD.
  */
-extern int maxprocperuid;		/* max # of procs per user */
 int maxfilesperproc = OPEN_MAX;		/* per-proc open files limit */
 
 SYSCTL_INT( _kern, KERN_MAXPROCPERUID, maxprocperuid, CTLFLAG_RW,
@@ -106,6 +128,25 @@ SYSCTL_INT( _kern, KERN_MAXPROCPERUID, maxprocperuid, CTLFLAG_RW,
 SYSCTL_INT( _kern, KERN_MAXFILESPERPROC, maxfilesperproc, CTLFLAG_RW,       
     		&maxfilesperproc, 0, "Maximum files allowed open per process" );
 
+/* Args and fn for proc_iteration callback used in setpriority */
+struct puser_nice_args {
+	proc_t curp;
+	int	prio;
+	id_t	who;
+	int *	foundp;
+	int *	errorp;
+};
+static int puser_donice_callback(proc_t p, void * arg);
+
+
+/* Args and fn for proc_iteration callback used in setpriority */
+struct ppgrp_nice_args {
+	proc_t curp;
+	int	prio;
+	int *	foundp;
+	int *	errorp;
+};
+static int ppgrp_donice_callback(proc_t p, void * arg);
 
 /*
  * Resource controls and accounting.
@@ -113,46 +154,86 @@ SYSCTL_INT( _kern, KERN_MAXFILESPERPROC, maxfilesperproc, CTLFLAG_RW,
 int
 getpriority(struct proc *curp, struct getpriority_args *uap, register_t *retval)
 {
-	register struct proc *p;
-	register int low = PRIO_MAX + 1;
+	struct proc *p;
+	int low = PRIO_MAX + 1;
+	kauth_cred_t my_cred;
 
-	if (uap->who < 0)
+	/* would also test (uap->who < 0), but id_t is unsigned */
+	if (uap->who > 0x7fffffff)
 		return (EINVAL);
 
 	switch (uap->which) {
 
 	case PRIO_PROCESS:
-		if (uap->who == 0)
+		if (uap->who == 0) {
 			p = curp;
-		else
-			p = pfind(uap->who);
-		if (p == 0)
-			break;
-		low = p->p_nice;
+			low = p->p_nice;
+		} else {
+			p = proc_find(uap->who);
+			if (p == 0)
+				break;
+			low = p->p_nice;
+			proc_rele(p);
+
+		}
 		break;
 
 	case PRIO_PGRP: {
-		register struct pgrp *pg;
+		struct pgrp *pg = PGRP_NULL;
 
-		if (uap->who == 0)
-			pg = curp->p_pgrp;
-		else if ((pg = pgfind(uap->who)) == NULL)
+		if (uap->who == 0) {
+			/* returns the pgrp to ref */
+			pg = proc_pgrp(curp);
+		 } else if ((pg = pgfind(uap->who)) == PGRP_NULL) {
 			break;
+		}
+		/* No need for iteration as it is a simple scan */
+		pgrp_lock(pg);
 		for (p = pg->pg_members.lh_first; p != 0; p = p->p_pglist.le_next) {
 			if (p->p_nice < low)
 				low = p->p_nice;
 		}
+		pgrp_unlock(pg);
+		pg_rele(pg);
 		break;
 	}
 
 	case PRIO_USER:
 		if (uap->who == 0)
 			uap->who = kauth_cred_getuid(kauth_cred_get());
-		for (p = allproc.lh_first; p != 0; p = p->p_list.le_next)
-			if (kauth_cred_getuid(p->p_ucred) == uap->who &&
+
+		proc_list_lock();
+
+		for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
+			my_cred = kauth_cred_proc_ref(p);
+			if (kauth_cred_getuid(my_cred) == uap->who &&
 			    p->p_nice < low)
 				low = p->p_nice;
+			kauth_cred_unref(&my_cred);
+		}
+
+		proc_list_unlock();
+
+		break;
+
+	case PRIO_DARWIN_THREAD: {
+		thread_t			thread;
+		struct uthread		*ut;
+
+		/* we currently only support the current thread */
+		if (uap->who != 0) {
+			return (EINVAL);
+		}
+	
+		thread = current_thread();
+		ut = get_bsdthread_info(thread);
+
+		low = 0;
+		if ( (ut->uu_flag & UT_BACKGROUND) != 0 ) {
+			low = 1;
+		}
 		break;
+	}
 
 	default:
 		return (EINVAL);
@@ -163,18 +244,69 @@ getpriority(struct proc *curp, struct getpriority_args *uap, register_t *retval)
 	return (0);
 }
 
+/* call back function used for proc iteration in PRIO_USER */
+static int
+puser_donice_callback(proc_t p, void * arg)
+{
+	int error, n;
+	struct puser_nice_args * pun = (struct puser_nice_args *)arg;
+	kauth_cred_t my_cred;
+
+	my_cred = kauth_cred_proc_ref(p);
+	if (kauth_cred_getuid(my_cred) == pun->who) {
+		error = donice(pun->curp, p, pun->prio);
+		if (pun->errorp != NULL)
+			*pun->errorp = error;
+		if (pun->foundp != NULL) {
+			n = *pun->foundp;
+			*pun->foundp = n+1;
+		}
+	}
+	kauth_cred_unref(&my_cred);
+
+	return(PROC_RETURNED);
+}
+
+/* call back function used for proc iteration in PRIO_PGRP */
+static int
+ppgrp_donice_callback(proc_t p, void * arg)
+{
+	int error;
+	struct ppgrp_nice_args * pun = (struct ppgrp_nice_args *)arg;
+	int n;
+
+	error = donice(pun->curp, p, pun->prio);
+	if (pun->errorp != NULL)
+		*pun->errorp = error;
+	if (pun->foundp!= NULL) {
+		n = *pun->foundp;
+		*pun->foundp = n+1;
+	}
+
+	return(PROC_RETURNED);
+}
+
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		ESRCH
+ *	donice:EPERM
+ *	donice:EACCES
+ */
 /* ARGSUSED */
 int
 setpriority(struct proc *curp, struct setpriority_args *uap, __unused register_t *retval)
 {
-	register struct proc *p;
+	struct proc *p;
 	int found = 0, error = 0;
+	int refheld = 0;
 
 	AUDIT_ARG(cmd, uap->which);
 	AUDIT_ARG(owner, uap->who, 0);
 	AUDIT_ARG(value, uap->prio);
 
-	if (uap->who < 0)
+	/* would also test (uap->who < 0), but id_t is unsigned */
+	if (uap->who > 0x7fffffff)
 		return (EINVAL);
 
 	switch (uap->which) {
@@ -182,38 +314,64 @@ setpriority(struct proc *curp, struct setpriority_args *uap, __unused register_t
 	case PRIO_PROCESS:
 		if (uap->who == 0)
 			p = curp;
-		else
-			p = pfind(uap->who);
-		if (p == 0)
-			break;
+		else {
+			p = proc_find(uap->who);
+			if (p == 0)
+				break;
+			refheld = 1;
+		}
 		error = donice(curp, p, uap->prio);
 		found++;
+		if (refheld != 0)
+			proc_rele(p);
 		break;
 
 	case PRIO_PGRP: {
-		register struct pgrp *pg;
+		struct pgrp *pg = PGRP_NULL;
+		struct ppgrp_nice_args ppgrp;
 		 
-		if (uap->who == 0)
-			pg = curp->p_pgrp;
-		else if ((pg = pgfind(uap->who)) == NULL)
+		if (uap->who == 0) {
+			pg = proc_pgrp(curp);
+		 } else if ((pg = pgfind(uap->who)) == PGRP_NULL)
 			break;
-		for (p = pg->pg_members.lh_first; p != 0;
-		    p = p->p_pglist.le_next) {
-			error = donice(curp, p, uap->prio);
-			found++;
-		}
+
+		ppgrp.curp = curp;
+		ppgrp.prio = uap->prio;
+		ppgrp.foundp = &found;
+		ppgrp.errorp = &error;
+		
+		/* PGRP_DROPREF drops the reference on process group */
+		pgrp_iterate(pg, PGRP_DROPREF, ppgrp_donice_callback, (void *)&ppgrp, NULL, NULL);
+
 		break;
 	}
 
-	case PRIO_USER:
+	case PRIO_USER: {
+		struct puser_nice_args punice;
+
 		if (uap->who == 0)
 			uap->who = kauth_cred_getuid(kauth_cred_get());
-		for (p = allproc.lh_first; p != 0; p = p->p_list.le_next)
-			if (kauth_cred_getuid(p->p_ucred) == uap->who) {
-				error = donice(curp, p, uap->prio);
-				found++;
-			}
+
+		punice.curp = curp;
+		punice.prio = uap->prio;
+		punice.who = uap->who;
+		punice.foundp = &found;
+		error = 0;
+		punice.errorp = &error;
+		proc_iterate(PROC_ALLPROCLIST, puser_donice_callback, (void *)&punice, NULL, NULL);
+
 		break;
+	}
+
+	case PRIO_DARWIN_THREAD: {
+		/* we currently only support the current thread */
+		if (uap->who != 0) {
+			return (EINVAL);
+		}
+		error = do_background_thread(curp, uap->prio);
+		found++;
+		break;
+	}
 
 	default:
 		return (EINVAL);
@@ -223,32 +381,145 @@ setpriority(struct proc *curp, struct setpriority_args *uap, __unused register_t
 	return (error);
 }
 
+
+/*
+ * Returns:	0			Success
+ *		EPERM
+ *		EACCES
+ *	mac_check_proc_sched:???
+ */
 int
-donice(curp, chgp, n)
-	register struct proc *curp, *chgp;
-	register int n;
+donice(struct proc *curp, struct proc *chgp, int n)
 {
-	kauth_cred_t ucred = curp->p_ucred;
+	int error = 0;
+	kauth_cred_t ucred;
+	kauth_cred_t my_cred;
+
+	ucred = kauth_cred_proc_ref(curp);
+	my_cred = kauth_cred_proc_ref(chgp);
 
 	if (suser(ucred, NULL) && ucred->cr_ruid &&
-	    kauth_cred_getuid(ucred) != kauth_cred_getuid(chgp->p_ucred) &&
-	    ucred->cr_ruid != kauth_cred_getuid(chgp->p_ucred))
-		return (EPERM);
+	    kauth_cred_getuid(ucred) != kauth_cred_getuid(my_cred) &&
+	    ucred->cr_ruid != kauth_cred_getuid(my_cred)) {
+		error = EPERM;
+		goto out;
+	}
 	if (n > PRIO_MAX)
 		n = PRIO_MAX;
 	if (n < PRIO_MIN)
 		n = PRIO_MIN;
-	if (n < chgp->p_nice && suser(ucred, &curp->p_acflag))
-		return (EACCES);
+	if (n < chgp->p_nice && suser(ucred, &curp->p_acflag)) {
+		error = EACCES;
+		goto out;
+	}
+#if CONFIG_MACF
+	error = mac_proc_check_sched(curp, chgp);
+	if (error) 
+		goto out;
+#endif
+	proc_lock(chgp);
 	chgp->p_nice = n;
+	proc_unlock(chgp);
 	(void)resetpriority(chgp);
-	return (0);
+out:
+	kauth_cred_unref(&ucred);
+	kauth_cred_unref(&my_cred);
+	return (error);
+}
+
+/*
+ * do_background_thread
+ * Returns:	0			Success
+ * XXX - todo - does this need a MACF hook?
+ */
+static int
+do_background_thread(struct proc *curp, int priority)
+{
+	int									i;
+	thread_t							thread;
+	struct uthread						*ut;
+	thread_precedence_policy_data_t		policy;
+	struct filedesc						*fdp;
+	struct fileproc						*fp;
+	
+	thread = current_thread();
+	ut = get_bsdthread_info(thread);
+
+	if ( (priority & PRIO_DARWIN_BG) == 0 ) {
+		/* turn off backgrounding of thread */
+		if ( (ut->uu_flag & UT_BACKGROUND) == 0 ) {
+			/* already off */
+			return(0);
+		}
+
+		/* clear background bit in thread and disable disk IO throttle */
+		ut->uu_flag &= ~UT_BACKGROUND;
+		ut->uu_iopol_disk = IOPOL_NORMAL;
+
+		/* reset thread priority (we did not save previous value) */
+		policy.importance = 0;
+		thread_policy_set( thread, THREAD_PRECEDENCE_POLICY,
+						   (thread_policy_t)&policy,
+						   THREAD_PRECEDENCE_POLICY_COUNT );
+
+		/* disable networking IO throttle.
+		 * NOTE - It is a known limitation of the current design that we 
+		 * could potentially clear TRAFFIC_MGT_SO_BACKGROUND bit for 
+		 * sockets created by other threads within this process.  
+		 */
+		proc_fdlock(curp);
+		fdp = curp->p_fd;
+		for ( i = 0; i < fdp->fd_nfiles; i++ ) {
+			struct socket		*sockp;
+			
+			fp = fdp->fd_ofiles[ i ];
+			if ( fp == NULL || (fdp->fd_ofileflags[ i ] & UF_RESERVED) != 0 || 
+				 fp->f_fglob->fg_type != DTYPE_SOCKET ) {
+				continue;
+			}
+			sockp = (struct socket *)fp->f_fglob->fg_data;
+			if ( sockp->so_background_thread != thread ) {
+				continue;
+			}
+			sockp->so_traffic_mgt_flags &= ~TRAFFIC_MGT_SO_BACKGROUND;
+			sockp->so_background_thread = NULL;
+		}
+		proc_fdunlock(curp);
+
+		return(0);
+	}
+	
+	/* background this thread */
+	if ( (ut->uu_flag & UT_BACKGROUND) != 0 ) {
+		/* already backgrounded */
+		return(0);
+	}
+
+	/* tag thread as background and throttle disk IO */
+	ut->uu_flag |= UT_BACKGROUND;
+	ut->uu_iopol_disk = IOPOL_THROTTLE;
+
+	policy.importance = INT_MIN;
+	thread_policy_set( thread, THREAD_PRECEDENCE_POLICY,
+					   (thread_policy_t)&policy,
+					   THREAD_PRECEDENCE_POLICY_COUNT );
+	
+	/* throttle networking IO happens in socket( ) syscall.
+	 * If UT_BACKGROUND is set in the current thread then
+	 * TRAFFIC_MGT_SO_BACKGROUND socket option is set.
+	 */
+	return(0);
 }
 
 
+/*
+ * Returns:	0			Success
+ *	copyin:EFAULT
+ *	dosetrlimit:
+ */
 /* ARGSUSED */
 int
-setrlimit(struct proc *p, register struct setrlimit_args *uap, __unused register_t *retval)
+setrlimit(struct proc *p, struct setrlimit_args *uap, __unused register_t *retval)
 {
 	struct rlimit alim;
 	int error;
@@ -256,36 +527,88 @@ setrlimit(struct proc *p, register struct setrlimit_args *uap, __unused register
 	if ((error = copyin(uap->rlp, (caddr_t)&alim,
 	    sizeof (struct rlimit))))
 		return (error);
+
 	return (dosetrlimit(p, uap->which, &alim));
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		ENOMEM			Cannot copy limit structure
+ *	suser:EPERM
+ *
+ * Notes:	EINVAL is returned both for invalid arguments, and in the
+ *		case that the current usage (e.g. RLIMIT_STACK) is already
+ *		in excess of the requested limit.
+ */
 int
-dosetrlimit(p, which, limp)
-	struct proc *p;
-	u_int which;
-	struct rlimit *limp;
+dosetrlimit(struct proc *p, u_int which, struct rlimit *limp)
 {
-	register struct rlimit *alimp;
+	struct rlimit *alimp;
 	int error;
+	kern_return_t	kr;
+	int posix = (which & _RLIMIT_POSIX_FLAG) ? 1 : 0;
+
+	/* Mask out POSIX flag, saved above */
+	which &= ~_RLIMIT_POSIX_FLAG;
 
 	if (which >= RLIM_NLIMITS)
 		return (EINVAL);
+
 	alimp = &p->p_rlimit[which];
+	if (limp->rlim_cur > limp->rlim_max)
+		return EINVAL;
+
 	if (limp->rlim_cur > alimp->rlim_max || 
 	    limp->rlim_max > alimp->rlim_max)
-		if ((error = suser(kauth_cred_get(), &p->p_acflag)))
+		if ((error = suser(kauth_cred_get(), &p->p_acflag))) {
 			return (error);
-	if (limp->rlim_cur > limp->rlim_max)
-		limp->rlim_cur = limp->rlim_max;
-	if (p->p_limit->p_refcnt > 1 &&
-	    (p->p_limit->p_lflags & PL_SHAREMOD) == 0) {
-		p->p_limit->p_refcnt--;
-		p->p_limit = limcopy(p->p_limit);
-		alimp = &p->p_rlimit[which];
 	}
 
+	proc_limitblock(p);
+
+	if ((error = proc_limitreplace(p)) != 0) {
+		proc_limitunblock(p);
+		return(error);
+	}
+
+	alimp = &p->p_rlimit[which];
+	
 	switch (which) {
 
+	case RLIMIT_CPU:
+		if (limp->rlim_cur == RLIM_INFINITY) {
+			task_vtimer_clear(p->task, TASK_VTIMER_RLIM);
+			timerclear(&p->p_rlim_cpu);
+		}
+		else {
+			task_absolutetime_info_data_t	tinfo;
+			mach_msg_type_number_t			count;
+			struct timeval					ttv, tv;
+
+			count = TASK_ABSOLUTETIME_INFO_COUNT;
+			task_info(p->task, TASK_ABSOLUTETIME_INFO,
+							  	(task_info_t)&tinfo, &count);
+			absolutetime_to_microtime(tinfo.total_user + tinfo.total_system,
+								(uint32_t *) &ttv.tv_sec, (uint32_t *) &ttv.tv_usec);
+
+			tv.tv_sec = (limp->rlim_cur > __INT_MAX__ ? __INT_MAX__ : limp->rlim_cur);
+			tv.tv_usec = 0;
+			timersub(&tv, &ttv, &p->p_rlim_cpu);
+
+			timerclear(&tv);
+			if (timercmp(&p->p_rlim_cpu, &tv, >))
+				task_vtimer_set(p->task, TASK_VTIMER_RLIM);
+			else {
+				task_vtimer_clear(p->task, TASK_VTIMER_RLIM);
+
+				timerclear(&p->p_rlim_cpu);
+
+				psignal(p, SIGXCPU);
+			}
+		}
+		break;
+
 	case RLIMIT_DATA:
 		if (limp->rlim_cur > maxdmap)
 			limp->rlim_cur = maxdmap;
@@ -294,55 +617,162 @@ dosetrlimit(p, which, limp)
 		break;
 
 	case RLIMIT_STACK:
-		if (limp->rlim_cur > maxsmap)
-			limp->rlim_cur = maxsmap;
-		if (limp->rlim_max > maxsmap)
-			limp->rlim_max = maxsmap;
+		/* Disallow illegal stack size instead of clipping */
+		if (limp->rlim_cur > maxsmap ||
+		    limp->rlim_max > maxsmap) {
+			if (posix) {
+				error = EINVAL;
+				goto out;
+			}
+			else {
+				/* 
+				 * 4797860 - workaround poorly written installers by 
+				 * doing previous implementation (< 10.5) when caller 
+				 * is non-POSIX conforming.
+				 */
+				if (limp->rlim_cur > maxsmap) 
+					limp->rlim_cur = maxsmap;
+				if (limp->rlim_max > maxsmap) 
+					limp->rlim_max = maxsmap;
+			}
+		}
+
 		/*
 		 * Stack is allocated to the max at exec time with only
 		 * "rlim_cur" bytes accessible.  If stack limit is going
 		 * up make more accessible, if going down make inaccessible.
 		 */
-		if (limp->rlim_cur != alimp->rlim_cur) {
+		if (limp->rlim_cur > alimp->rlim_cur) {
 			user_addr_t addr;
 			user_size_t size;
 			
-			if (limp->rlim_cur > alimp->rlim_cur) {
 				/* grow stack */
 				size = round_page_64(limp->rlim_cur);
 				size -= round_page_64(alimp->rlim_cur);
 
 #if STACK_GROWTH_UP
 				/* go to top of current stack */
-				addr = p->user_stack + alimp->rlim_cur;
-#else STACK_GROWTH_UP
-				addr = p->user_stack - alimp->rlim_cur;
-				addr -= size;
+			addr = p->user_stack + round_page_64(alimp->rlim_cur);
+#else	/* STACK_GROWTH_UP */
+			addr = p->user_stack - round_page_64(limp->rlim_cur);
 #endif /* STACK_GROWTH_UP */
-				if (mach_vm_allocate(current_map(), 
-						     &addr, size,
-						     VM_FLAGS_FIXED) != KERN_SUCCESS)
-					return(EINVAL);
-			} else {
+			kr = mach_vm_protect(current_map(), 
+					     addr, size,
+					     FALSE, VM_PROT_DEFAULT);
+			if (kr != KERN_SUCCESS) {
+				error =  EINVAL;
+				goto out;
+			}
+		} else if (limp->rlim_cur < alimp->rlim_cur) {
+			user_addr_t addr;
+			user_size_t size;
+			user_addr_t cur_sp;
+
 				/* shrink stack */
+
+			/*
+			 * First check if new stack limit would agree
+			 * with current stack usage.
+			 * Get the current thread's stack pointer...
+			 */
+			cur_sp = thread_adjuserstack(current_thread(),
+						     0);
+#if STACK_GROWTH_UP
+			if (cur_sp >= p->user_stack &&
+			    cur_sp < (p->user_stack +
+				      round_page_64(alimp->rlim_cur))) {
+				/* current stack pointer is in main stack */
+				if (cur_sp >= (p->user_stack +
+					       round_page_64(limp->rlim_cur))) {
+					/*
+					 * New limit would cause
+					 * current usage to be invalid:
+					 * reject new limit.
+					 */
+					error =  EINVAL;
+					goto out;
 			}
+			} else {
+				/* not on the main stack: reject */
+				error =  EINVAL;
+				goto out;
+		}
+				 
+#else	/* STACK_GROWTH_UP */
+			if (cur_sp <= p->user_stack &&
+			    cur_sp > (p->user_stack -
+				      round_page_64(alimp->rlim_cur))) {
+				/* stack pointer is in main stack */
+				if (cur_sp <= (p->user_stack -
+					       round_page_64(limp->rlim_cur))) {
+					/*
+					 * New limit would cause
+					 * current usage to be invalid:
+					 * reject new limit.
+					 */
+					error =  EINVAL;
+					goto out;
+				}
+			} else {
+				/* not on the main stack: reject */
+				error =  EINVAL;
+				goto out;
+			}
+#endif	/* STACK_GROWTH_UP */
+				
+			size = round_page_64(alimp->rlim_cur);
+			size -= round_page_64(limp->rlim_cur);
+
+#if STACK_GROWTH_UP
+			addr = p->user_stack + round_page_64(limp->rlim_cur);
+#else	/* STACK_GROWTH_UP */
+			addr = p->user_stack - round_page_64(alimp->rlim_cur);
+#endif /* STACK_GROWTH_UP */
+
+			kr = mach_vm_protect(current_map(),
+					     addr, size,
+					     FALSE, VM_PROT_NONE);
+			if (kr != KERN_SUCCESS) {
+				error =  EINVAL;
+				goto out;
+			}
+		} else {
+			/* no change ... */
 		}
 		break;
 
 	case RLIMIT_NOFILE:
 		/* 
-		* Only root can set the maxfiles limits, as it is systemwide resource
+		 * Only root can set the maxfiles limits, as it is
+		 * systemwide resource.  If we are expecting POSIX behavior,
+		 * instead of clamping the value, return EINVAL.  We do this
+		 * because historically, people have been able to attempt to
+		 * set RLIM_INFINITY to get "whatever the maximum is".
 		*/
 		if ( is_suser() ) {
-			if (limp->rlim_cur > maxfiles)
+			if (limp->rlim_cur != alimp->rlim_cur &&
+			    limp->rlim_cur > (rlim_t)maxfiles) {
+			    	if (posix) {
+					error =  EINVAL;
+					goto out;
+				}
 				limp->rlim_cur = maxfiles;
-			if (limp->rlim_max > maxfiles)
+			}
+			if (limp->rlim_max != alimp->rlim_max &&
+			    limp->rlim_max > (rlim_t)maxfiles)
 				limp->rlim_max = maxfiles;
 		}
 		else {
-			if (limp->rlim_cur > maxfilesperproc)
+			if (limp->rlim_cur != alimp->rlim_cur &&
+			    limp->rlim_cur > (rlim_t)maxfilesperproc) {
+			    	if (posix) {
+					error =  EINVAL;
+					goto out;
+				}
 				limp->rlim_cur = maxfilesperproc;
-			if (limp->rlim_max > maxfilesperproc)
+			}
+			if (limp->rlim_max != alimp->rlim_max &&
+			    limp->rlim_max > (rlim_t)maxfilesperproc)
 				limp->rlim_max = maxfilesperproc;
 		}
 		break;
@@ -354,31 +784,53 @@ dosetrlimit(p, which, limp)
 		 * maxprocperuid (presumably less than maxproc).
 		 */
 		if ( is_suser() ) {
-			if (limp->rlim_cur > maxproc)
+			if (limp->rlim_cur > (rlim_t)maxproc)
 				limp->rlim_cur = maxproc;
-			if (limp->rlim_max > maxproc)
+			if (limp->rlim_max > (rlim_t)maxproc)
 				limp->rlim_max = maxproc;
 		} 
 		else {
-			if (limp->rlim_cur > maxprocperuid)
+			if (limp->rlim_cur > (rlim_t)maxprocperuid)
 				limp->rlim_cur = maxprocperuid;
-			if (limp->rlim_max > maxprocperuid)
+			if (limp->rlim_max > (rlim_t)maxprocperuid)
 				limp->rlim_max = maxprocperuid;
 		}
 		break;
+
+	case RLIMIT_MEMLOCK:
+		/*
+		 * Tell the Mach VM layer about the new limit value.
+		 */
+
+		vm_map_set_user_wire_limit(current_map(), limp->rlim_cur);
+		break;
 		
 	} /* switch... */
+	proc_lock(p);
 	*alimp = *limp;
-	return (0);
+	proc_unlock(p);
+	error = 0;
+out:
+	proc_limitunblock(p);
+	return (error);
 }
 
 /* ARGSUSED */
 int
-getrlimit(struct proc *p, register struct getrlimit_args *uap, __unused register_t *retval)
+getrlimit(struct proc *p, struct getrlimit_args *uap, __unused register_t *retval)
 {
+	struct rlimit lim;
+
+	/*
+	 * Take out flag now in case we need to use it to trigger variant
+	 * behaviour later.
+	 */
+	uap->which &= ~_RLIMIT_POSIX_FLAG;
+
 	if (uap->which >= RLIM_NLIMITS)
 		return (EINVAL);
-	return (copyout((caddr_t)&p->p_rlimit[uap->which],
+	proc_limitget(p, uap->which, &lim);
+	return (copyout((caddr_t)&lim,
 	    		uap->rlp, sizeof (struct rlimit)));
 }
 
@@ -386,12 +838,9 @@ getrlimit(struct proc *p, register struct getrlimit_args *uap, __unused register
  * Transform the running time and tick information in proc p into user,
  * system, and interrupt time usage.
  */
+/* No lock on proc is held for this.. */
 void
-calcru(p, up, sp, ip)
-	register struct proc *p;
-	register struct timeval *up;
-	register struct timeval *sp;
-	register struct timeval *ip;
+calcru(struct proc *p, struct timeval *up, struct timeval *sp, struct timeval *ip)
 {
 	task_t			task;
 
@@ -404,12 +853,12 @@ calcru(p, up, sp, ip)
 	if (task) {
 		task_basic_info_data_t tinfo;
 		task_thread_times_info_data_t ttimesinfo;
-		int task_info_stuff, task_ttimes_stuff;
+		mach_msg_type_number_t task_info_stuff, task_ttimes_stuff;
 		struct timeval ut,st;
 
 		task_info_stuff	= TASK_BASIC_INFO_COUNT;
 		task_info(task, TASK_BASIC_INFO,
-			  &tinfo, &task_info_stuff);
+			  (task_info_t)&tinfo, &task_info_stuff);
 		ut.tv_sec = tinfo.user_time.seconds;
 		ut.tv_usec = tinfo.user_time.microseconds;
 		st.tv_sec = tinfo.system_time.seconds;
@@ -419,7 +868,7 @@ calcru(p, up, sp, ip)
 
 		task_ttimes_stuff = TASK_THREAD_TIMES_INFO_COUNT;
 		task_info(task, TASK_THREAD_TIMES_INFO,
-			  &ttimesinfo, &task_ttimes_stuff);
+			  (task_info_t)&ttimesinfo, &task_ttimes_stuff);
 
 		ut.tv_sec = ttimesinfo.user_time.seconds;
 		ut.tv_usec = ttimesinfo.user_time.microseconds;
@@ -434,24 +883,35 @@ __private_extern__ void munge_rusage(struct rusage *a_rusage_p, struct user_rusa
 
 /* ARGSUSED */
 int
-getrusage(register struct proc *p, register struct getrusage_args *uap, __unused register_t *retval)
+getrusage(struct proc *p, struct getrusage_args *uap, __unused register_t *retval)
 {
 	struct rusage *rup, rubuf;
 	struct user_rusage rubuf64;
 	size_t retsize = sizeof(rubuf);			/* default: 32 bits */
 	caddr_t retbuf = (caddr_t)&rubuf;		/* default: 32 bits */
+	struct timeval utime;
+	struct timeval stime;
+
 
 	switch (uap->who) {
 	case RUSAGE_SELF:
-		rup = &p->p_stats->p_ru;
-		calcru(p, &rup->ru_utime, &rup->ru_stime, NULL);
+		calcru(p, &utime, &stime, NULL);
 		// LP64todo: proc struct should have 64 bit version of struct
+		proc_lock(p);
+		rup = &p->p_stats->p_ru;
+		rup->ru_utime = utime;
+		rup->ru_stime = stime;
+
 		rubuf = *rup;
+		proc_unlock(p);
+
 		break;
 
 	case RUSAGE_CHILDREN:
+		proc_lock(p);
 		rup = &p->p_stats->p_cru;
 		rubuf = *rup;
+		proc_unlock(p);
 		break;
 
 	default:
@@ -466,11 +926,10 @@ getrusage(register struct proc *p, register struct getrusage_args *uap, __unused
 }
 
 void
-ruadd(ru, ru2)
-	register struct rusage *ru, *ru2;
+ruadd(struct rusage *ru, struct rusage *ru2)
 {
-	register long *ip, *ip2;
-	register int i;
+	long *ip, *ip2;
+	int i;
 
 	timeradd(&ru->ru_utime, &ru2->ru_utime, &ru->ru_utime);
 	timeradd(&ru->ru_stime, &ru2->ru_stime, &ru->ru_stime);
@@ -481,24 +940,195 @@ ruadd(ru, ru2)
 		*ip++ += *ip2++;
 }
 
-/*
- * Make a copy of the plimit structure.
- * We share these structures copy-on-write after fork,
- * and copy when a limit is changed.
- */
-struct plimit *
-limcopy(lim)
-	struct plimit *lim;
+void
+proc_limitget(proc_t p, int which, struct rlimit * limp)
+{
+	proc_list_lock();
+	limp->rlim_cur = p->p_rlimit[which].rlim_cur;
+	limp->rlim_max = p->p_rlimit[which].rlim_max;
+	proc_list_unlock();
+}
+
+
+void
+proc_limitdrop(proc_t p, int exiting)
 {
-	register struct plimit *copy;
+	struct  plimit * freelim = NULL;
+	struct  plimit * freeoldlim = NULL;
+
+	proc_list_lock();
+
+	if (--p->p_limit->pl_refcnt == 0) { 
+		freelim = p->p_limit;
+		p->p_limit = NULL;
+	}
+	if ((exiting != 0) && (p->p_olimit != NULL) && (--p->p_olimit->pl_refcnt == 0)) {
+		freeoldlim =  p->p_olimit;
+		p->p_olimit = NULL;
+	}
+
+	proc_list_unlock();
+	if (freelim != NULL)
+		FREE_ZONE(freelim, sizeof *p->p_limit, M_PLIMIT);
+	if (freeoldlim != NULL)
+		FREE_ZONE(freeoldlim, sizeof *p->p_olimit, M_PLIMIT);
+}
+
+
+void
+proc_limitfork(proc_t parent, proc_t child)
+{
+	proc_list_lock();
+	child->p_limit = parent->p_limit;
+	child->p_limit->pl_refcnt++;
+	child->p_olimit = NULL;
+	proc_list_unlock();
+}
+
+void
+proc_limitblock(proc_t p)
+{
+	proc_lock(p);
+	while (p->p_lflag & P_LLIMCHANGE) {
+		p->p_lflag |= P_LLIMWAIT;
+		msleep(&p->p_olimit, &p->p_mlock, 0, "proc_limitblock", NULL);
+	}
+	p->p_lflag |= P_LLIMCHANGE;
+	proc_unlock(p);
+
+}
+
+
+void
+proc_limitunblock(proc_t p)
+{
+	proc_lock(p);
+	p->p_lflag &= ~P_LLIMCHANGE;
+	if (p->p_lflag & P_LLIMWAIT) {
+		p->p_lflag &= ~P_LLIMWAIT;
+		wakeup(&p->p_olimit);
+	}
+	proc_unlock(p);
+}
+
+/* This is called behind serialization provided by proc_limitblock/unlbock */
+int
+proc_limitreplace(proc_t p)
+{
+	struct plimit *copy;
+
+
+	proc_list_lock();
+
+	if (p->p_limit->pl_refcnt == 1) {
+		proc_list_unlock();
+		return(0);
+	}
+		
+	proc_list_unlock();
 
 	MALLOC_ZONE(copy, struct plimit *,
-			sizeof(struct plimit), M_SUBPROC, M_WAITOK);
-	if (copy == NULL)
-		panic("limcopy");
-	bcopy(lim->pl_rlimit, copy->pl_rlimit,
+			sizeof(struct plimit), M_PLIMIT, M_WAITOK);
+	if (copy == NULL) {
+		return(ENOMEM);
+	}
+
+	proc_list_lock();
+	bcopy(p->p_limit->pl_rlimit, copy->pl_rlimit,
 	    sizeof(struct rlimit) * RLIM_NLIMITS);
-	copy->p_lflags = 0;
-	copy->p_refcnt = 1;
-	return (copy);
+	copy->pl_refcnt = 1;
+	/* hang on to reference to old till process exits */
+	p->p_olimit = p->p_limit;
+	p->p_limit = copy;
+	proc_list_unlock();
+
+	return(0);
+}
+
+
+/*
+ * iopolicysys
+ *
+ * Description:	System call MUX for use in manipulating I/O policy attributes of the current process or thread
+ *
+ * Parameters:	cmd				Policy command
+ *		arg				Pointer to policy arguments
+ *
+ * Returns:	0				Success
+ *		EINVAL				Invalid command or invalid policy arguments
+ *
+ */
+int
+iopolicysys(__unused struct proc *p, __unused struct iopolicysys_args *uap, __unused register_t *retval)
+{
+	int	error = 0;
+	thread_t thread = THREAD_NULL;
+	int *policy;
+	struct uthread	*ut = NULL;
+	struct _iopol_param_t iop_param;
+
+	if ((error = copyin(uap->arg, &iop_param, sizeof(iop_param))) != 0)
+		goto exit;
+
+	if (iop_param.iop_iotype != IOPOL_TYPE_DISK) {
+		error = EINVAL;
+		goto exit;
+	}
+
+	switch (iop_param.iop_scope) {
+	case IOPOL_SCOPE_PROCESS:
+		policy = &p->p_iopol_disk;
+		break;
+	case IOPOL_SCOPE_THREAD:
+		thread = current_thread();
+		ut = get_bsdthread_info(thread);
+		policy = &ut->uu_iopol_disk;
+		break;
+	default:
+		error = EINVAL;
+		goto exit;
+	}
+		
+	switch(uap->cmd) {
+	case IOPOL_CMD_SET:
+		switch (iop_param.iop_policy) {
+		case IOPOL_DEFAULT:
+		case IOPOL_NORMAL:
+		case IOPOL_THROTTLE:
+		case IOPOL_PASSIVE:
+			proc_lock(p);
+			*policy = iop_param.iop_policy;
+			proc_unlock(p);
+			break;
+		default:
+			error = EINVAL;
+			goto exit;
+		}
+		break;
+	case IOPOL_CMD_GET:
+		switch (*policy) {
+		case IOPOL_DEFAULT:
+		case IOPOL_NORMAL:
+		case IOPOL_THROTTLE:
+		case IOPOL_PASSIVE:
+			iop_param.iop_policy = *policy;
+			break;
+		default: // in-kernel 
+			// this should never happen
+			printf("%s: unknown I/O policy %d\n", __func__, *policy);
+			// restore to default value
+			*policy = IOPOL_DEFAULT;
+			iop_param.iop_policy = *policy;
+		}
+		
+		error = copyout((caddr_t)&iop_param, uap->arg, sizeof(iop_param));
+		break;
+	default:
+		error = EINVAL; // unknown command
+		break;
+	}
+
+  exit:
+	*retval = error;
+	return (error);
 }
diff --git a/bsd/kern/kern_shutdown.c b/bsd/kern/kern_shutdown.c
index 931b27c3d..4b59526ed 100644
--- a/bsd/kern/kern_shutdown.c
+++ b/bsd/kern/kern_shutdown.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File:	bsd/kern/kern_shutdown.c
@@ -59,25 +65,60 @@
 
 #include <bsm/audit_kernel.h>
 
+#include <kern/sched_prim.h>		/* for thread_block() */
+#include <kern/host.h>			/* for host_priv_self() */
+#include <net/if_var.h>			/* for if_down_all() */
+#include <sys/buf_internal.h>		/* for count_busy_buffers() */
+#include <sys/mount_internal.h>		/* for vfs_unmountall() */
+#include <mach/task.h>			/* for task_suspend() */
+#include <sys/sysproto.h>		/* abused for sync() */
+#include <kern/clock.h>			/* for delay_for_interval() */
+
+/* XXX should be in a header file somewhere, but isn't */
+extern void md_prepare_for_shutdown(int, int, char *);
+
 int	waittime = -1;
-static void proc_shutdown();
+static int shutting_down = 0;
+
+static void proc_shutdown(void);
+int in_shutdown(void);
+
+extern void IOSystemShutdownNotification(void);
+
+struct sd_filterargs{
+	int delayterm;
+	int shutdownstate;
+};
+
+
+struct sd_iterargs {
+	int signo;	/* the signal to be posted */
+	int setsdstate;  /* shutdown state to be set */
+};
+
+static int sd_filt1(proc_t, void *);
+static int sd_filt2(proc_t, void *);
+static int  sd_callback1(proc_t p, void * arg);
+static int  sd_callback2(proc_t p, void * arg);
+static int  sd_callback3(proc_t p, void * arg);
 
 void
-boot(paniced, howto, command)
-	int paniced, howto;
-	char *command;
+boot(int paniced, int howto, char *command)
 {
-	register int i;
-	int s;
 	struct proc *p = current_proc();	/* XXX */
 	int hostboot_option=0;
 	int funnel_state;
-	struct proc  *launchd_proc;
-
-    extern void md_prepare_for_shutdown(int paniced, int howto, char * command);
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
+       /*
+	* Temporary hack to notify the power management root domain
+	* that the system will shut down.
+	*/
+	IOSystemShutdownNotification();
+
+	shutting_down = 1;
+	    
 	md_prepare_for_shutdown(paniced, howto, command);
 
 	if ((howto&RB_NOSYNC)==0 && waittime < 0) {
@@ -94,19 +135,19 @@ boot(paniced, howto, command)
 		/* handle live procs (deallocate their root and current directories). */		
 		proc_shutdown();
 
-		audit_shutdown();
+#if AUDIT
+ 		audit_shutdown();
+#endif
 
 		sync(p, (void *)NULL, (int *)NULL);
 
 		/*
-		 * Now that all processes have been  termianted and system is sync'ed up, 
-		 * suspend launchd
+		 * Now that all processes have been terminated and system is
+		 * sync'ed up, suspend init
 		 */
 
-		launchd_proc = pfind(1);
-		if (launchd_proc && p != launchd_proc) {
-			task_suspend(launchd_proc->task);
-		}
+		if (initproc && p != initproc)
+			task_suspend(initproc->task);
 
 		/*
 		 * Unmount filesystems
@@ -119,7 +160,7 @@ boot(paniced, howto, command)
 			if (nbusy == 0)
 				break;
 			printf("%d ", nbusy);
-			IOSleep( 1 * nbusy );
+			delay_for_interval( 1 * nbusy, 1000 * 1000);
 		}
 		if (nbusy)
 			printf("giving up\n");
@@ -127,12 +168,14 @@ boot(paniced, howto, command)
 			printf("done\n");
 	}
 
+#if NETWORKING
 	/*
 	 * Can't just use an splnet() here to disable the network
 	 * because that will lock out softints which the disk
 	 * drivers depend on to finish DMAs.
 	 */
 	if_down_all();
+#endif /* NETWORKING */
 
 	if (howto & RB_POWERDOWN)
 		hostboot_option = HOST_REBOOT_HALT;
@@ -145,24 +188,119 @@ boot(paniced, howto, command)
         hostboot_option = HOST_REBOOT_UPSDELAY;
     }
 
-	/*
-	 * if we're going to power down due to a halt,
-	 * give the disks a chance to finish getting
-	 * the track cache flushed to the media... 
-	 * unfortunately, some of our earlier drives
-	 * don't properly hold off on returning 
-	 * from the track flush command (issued by
-	 * the unmounts) until it's actully fully
-	 * committed.
-	 */
-	if (hostboot_option == HOST_REBOOT_HALT)
-	        IOSleep( 1 * 1000 );
-
 	host_reboot(host_priv_self(), hostboot_option);
 
 	thread_funnel_set(kernel_flock, FALSE);
 }
 
+static int
+sd_filt1(proc_t p, void * args)
+{
+	proc_t self = current_proc();
+	struct sd_filterargs * sf = (struct sd_filterargs *)args;
+	int delayterm = sf-> delayterm;
+	int shutdownstate = sf->shutdownstate;
+
+	if (((p->p_flag&P_SYSTEM) != 0) || (p->p_ppid == 0) 
+		||(p == self) || (p->p_stat == SZOMB) 
+		|| (p->p_shutdownstate != shutdownstate) 
+		||((delayterm == 0) && ((p->p_lflag& P_LDELAYTERM) == P_LDELAYTERM))
+		|| ((p->p_sigcatch & sigmask(SIGTERM))== 0)) {
+			return(0);
+		}
+        else 
+                return(1);
+}
+
+
+static int  
+sd_callback1(proc_t p, void * args)
+{
+	struct sd_iterargs * sd = (struct sd_iterargs *)args;
+	int signo = sd->signo;
+	int setsdstate = sd->setsdstate;
+
+	proc_lock(p);
+	p->p_shutdownstate = setsdstate;
+	if (p->p_stat != SZOMB) {
+		proc_unlock(p);
+		psignal(p, signo);
+	} else
+		proc_unlock(p);
+	return(PROC_RETURNED);
+
+}
+
+static int
+sd_filt2(proc_t p, void * args)
+{
+	proc_t self = current_proc();
+	struct sd_filterargs * sf = (struct sd_filterargs *)args;
+	int delayterm = sf-> delayterm;
+	int shutdownstate = sf->shutdownstate;
+
+	if (((p->p_flag&P_SYSTEM) != 0) || (p->p_ppid == 0) 
+		||(p == self) || (p->p_stat == SZOMB) 
+		|| (p->p_shutdownstate == shutdownstate) 
+		||((delayterm == 0) && ((p->p_lflag& P_LDELAYTERM) == P_LDELAYTERM))) {
+			return(0);
+		}
+        else
+                return(1);
+}
+
+static int  
+sd_callback2(proc_t p, void * args)
+{
+	struct sd_iterargs * sd = (struct sd_iterargs *)args;
+	int signo = sd->signo;
+	int setsdstate = sd->setsdstate;
+
+	proc_lock(p);
+	p->p_shutdownstate = setsdstate;
+	if (p->p_stat != SZOMB) {
+		proc_unlock(p);
+		psignal(p, signo);
+	} else
+		proc_unlock(p);
+
+	return(PROC_RETURNED);
+
+}
+
+static int  
+sd_callback3(proc_t p, void * args)
+{
+	struct sd_iterargs * sd = (struct sd_iterargs *)args;
+	int setsdstate = sd->setsdstate;
+
+	proc_lock(p);
+	p->p_shutdownstate = setsdstate;
+	if (p->p_stat != SZOMB) {
+	       /*
+		* NOTE: following code ignores sig_lock and plays
+		* with exit_thread correctly.  This is OK unless we
+		* are a multiprocessor, in which case I do not
+		* understand the sig_lock.  This needs to be fixed.
+		* XXX
+		*/
+		if (p->exit_thread) {	/* someone already doing it */
+			proc_unlock(p);
+			/* give him a chance */
+			thread_block(THREAD_CONTINUE_NULL);
+		} else {
+			p->exit_thread = current_thread();
+			printf(".");
+			proc_unlock(p);
+			exit1(p, 1, (int *)NULL);
+		}
+	} else
+		proc_unlock(p);
+
+	return(PROC_RETURNED);
+}
+
+
 /*
  * proc_shutdown()
  *
@@ -176,12 +314,13 @@ boot(paniced, howto, command)
  */
 
 static void
-proc_shutdown()
+proc_shutdown(void)
 {
 	struct proc	*p, *self;
-	struct vnode	**cdirp, **rdirp, *vp;
-	int		restart, i, TERM_catch;
+	int		i, TERM_catch;
 	int delayterm = 0;
+	struct sd_filterargs sfargs;
+	struct sd_iterargs sdargs;
 
 	/*
 	 *	Kill as many procs as we can.  (Except ourself...)
@@ -192,33 +331,26 @@ proc_shutdown()
 	 * Signal the init with SIGTERM so that he does not launch
 	 * new processes 
 	 */
-	p = pfind(1);
+	p = proc_find(1);
 	if (p && p != self) {
 		psignal(p, SIGTERM);
 	}
+	proc_rele(p);
 
 	printf("Killing all processes ");
 
+sigterm_loop:
 	/*
 	 * send SIGTERM to those procs interested in catching one
 	 */
-sigterm_loop:
-	for (p = allproc.lh_first; p; p = p->p_list.le_next) {
-	        if (((p->p_flag&P_SYSTEM) == 0) && (p->p_pptr->p_pid != 0) && (p != self) && (p->p_stat != SZOMB) && (p->p_shutdownstate == 0)) {
+	sfargs.delayterm = delayterm;
+	sfargs.shutdownstate = 0;
+	sdargs.signo = SIGTERM;
+	sdargs.setsdstate = 1;
+
+	/* post a SIGTERM to all that catch SIGTERM and not marked for delay */
+	proc_rebootscan(sd_callback1, (void *)&sdargs, sd_filt1, (void *)&sfargs);
 
-			if ((delayterm == 0) && ((p->p_lflag& P_LDELAYTERM) == P_LDELAYTERM)) {
-				continue;
-			}
-		        if (p->p_sigcatch & sigmask(SIGTERM)) {
-					p->p_shutdownstate = 1;
-					if (proc_refinternal(p, 1) == p) {
-			        	psignal(p, SIGTERM);
-						proc_dropinternal(p, 1);
-					}
-				goto sigterm_loop;
-		}
-	}
-	}
 	/*
 	 * now wait for up to 30 seconds to allow those procs catching SIGTERM
 	 * to digest it
@@ -230,14 +362,20 @@ sigterm_loop:
 		 * and then check to see if the tasks that were sent a
 		 * SIGTERM have exited
 		 */
-		IOSleep(100);   
+		delay_for_interval(100, 1000 * 1000);
 		TERM_catch = 0;
 
+
+		proc_list_lock();
+
 		for (p = allproc.lh_first; p; p = p->p_list.le_next) {
 			if (p->p_shutdownstate == 1) {
 				TERM_catch++;
 			}
 		}
+
+		proc_list_unlock();
+
 		if (TERM_catch == 0)
 		        break;
 	}
@@ -246,42 +384,50 @@ sigterm_loop:
 		 * log the names of the unresponsive tasks
 		 */
 
+
+		proc_list_lock();
+
 	        for (p = allproc.lh_first; p; p = p->p_list.le_next) {
 			if (p->p_shutdownstate == 1) {
 				  printf("%s[%d]: didn't act on SIGTERM\n", p->p_comm, p->p_pid);
 			}
 		}
-		IOSleep(1000 * 5);
+
+		proc_list_unlock();
+
+		delay_for_interval(1000 * 5, 1000 * 1000);
 	}
 
 	/*
 	 * send a SIGKILL to all the procs still hanging around
 	 */
-sigkill_loop:
-	for (p = allproc.lh_first; p; p = p->p_list.le_next) {
-	        if (((p->p_flag&P_SYSTEM) == 0) && (p->p_pptr->p_pid != 0) && (p != self) && (p->p_stat != SZOMB) && (p->p_shutdownstate != 2)) {
+	sfargs.delayterm = delayterm;
+	sfargs.shutdownstate = 2;
+	sdargs.signo = SIGKILL;
+	sdargs.setsdstate = 2;
+
+	/* post a SIGTERM to all that catch SIGTERM and not marked for delay */
+	proc_rebootscan(sd_callback2, (void *)&sdargs, sd_filt2, (void *)&sfargs);
 
-			if ((delayterm == 0) && ((p->p_lflag& P_LDELAYTERM) == P_LDELAYTERM)) {
-				continue;
-			}
-			if (proc_refinternal(p, 1) == p) {
-				psignal(p, SIGKILL);
-				proc_dropinternal(p, 1);
-			}
-			p->p_shutdownstate = 2;
-			goto sigkill_loop;
-		}
-	}
 	/*
 	 * wait for up to 60 seconds to allow these procs to exit normally
+	 *
+	 * History:	The delay interval was changed from 100 to 200
+	 *		for NFS requests in particular.
 	 */
 	for (i = 0; i < 300; i++) {
-		IOSleep(200);  /* double the time from 100 to 200 for NFS requests in particular */
+		delay_for_interval(200, 1000 * 1000);
+
+
+		proc_list_lock();
 
 	        for (p = allproc.lh_first; p; p = p->p_list.le_next) {
 				if (p->p_shutdownstate == 2)
 			        break;
 		}
+
+		proc_list_unlock();
+
 		if (!p)
 		        break;
 	}
@@ -289,43 +435,30 @@ sigkill_loop:
 	/*
 	 * if we still have procs that haven't exited, then brute force 'em
 	 */
-	p = allproc.lh_first;
-	while (p) {
-	        if ((p->p_shutdownstate == 3) || (p->p_flag&P_SYSTEM) || (!delayterm && ((p->p_lflag& P_LDELAYTERM))) 
-				|| (p->p_pptr->p_pid == 0) || (p == self)) {
-		        p = p->p_list.le_next;
-		}
-		else {
-			p->p_shutdownstate = 3;
-		        /*
-			 * NOTE: following code ignores sig_lock and plays
-			 * with exit_thread correctly.  This is OK unless we
-			 * are a multiprocessor, in which case I do not
-			 * understand the sig_lock.  This needs to be fixed.
-			 * XXX
-			 */
-			if (p->exit_thread) {	/* someone already doing it */
-				/* give him a chance */
-				thread_block(THREAD_CONTINUE_NULL);
-			} else {
-				p->exit_thread = current_thread();
-				printf(".");
-				if (proc_refinternal(p, 1) == p) {
-					exit1(p, 1, (int *)NULL);
-					proc_dropinternal(p, 1);
-				}
-			}
-			p = allproc.lh_first;
-		}
-	}
-	printf("\n");
+	sfargs.delayterm = delayterm;
+	sfargs.shutdownstate = 3;
+	sdargs.signo = 0;
+	sdargs.setsdstate = 3;
 
+	/* post a SIGTERM to all that catch SIGTERM and not marked for delay */
+	proc_rebootscan(sd_callback3, (void *)&sdargs, sd_filt2, (void *)&sfargs);
+	printf("\n");
 
 	/* Now start the termination of processes that are marked for delayed termn */
 	if (delayterm == 0) {
 		delayterm = 1;
 		goto  sigterm_loop;
 	}
+	/* drop the ref on initproc */
+	proc_rele(initproc);
 	printf("continuing\n");
 }
 
+/*
+ * Check whether the system has begun its shutdown sequence. 
+ */
+int
+in_shutdown(void)
+{
+	return shutting_down;
+}
diff --git a/bsd/kern/kern_sig.c b/bsd/kern/kern_sig.c
index 479b1f5e1..105ece427 100644
--- a/bsd/kern/kern_sig.c
+++ b/bsd/kern/kern_sig.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1995-1998 Apple Computer, Inc. All Rights Reserved */
 /*
  * Copyright (c) 1982, 1986, 1989, 1991, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -59,6 +64,12 @@
  *
  *	@(#)kern_sig.c	8.7 (Berkeley) 4/18/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #define	SIGPROP		/* include signal properties table */
 #include <sys/param.h>
@@ -73,9 +84,6 @@
 #include <sys/kernel.h>
 #include <sys/wait.h>
 #include <sys/signalvar.h>
-#if KTRACE
-#include <sys/ktrace.h>
-#endif
 #include <sys/syslog.h>
 #include <sys/stat.h>
 #include <sys/lock.h>
@@ -101,6 +109,9 @@
 #include <mach/exception.h>
 #include <mach/task.h>
 #include <mach/thread_act.h>
+#include <libkern/OSAtomic.h>
+
+#include <sys/sdt.h>
 
 /*
  * Missing prototypes that Mach should export
@@ -108,9 +119,7 @@
  * +++
  */
 extern int thread_enable_fpe(thread_t act, int onoff);
-extern void unix_syscall_return(int error);
 extern thread_t	port_name_to_thread(mach_port_name_t port_name);
-extern kern_return_t check_actforsig(task_t task, thread_t thread, int setast);
 extern kern_return_t get_signalact(task_t , thread_t *, int);
 extern boolean_t thread_should_abort(thread_t);
 extern unsigned int get_useraddr(void);
@@ -119,23 +128,22 @@ extern unsigned int get_useraddr(void);
  * ---
  */
 
-extern void doexception(int exc, int code, int sub);
+extern void doexception(int exc, mach_exception_code_t code, 
+		mach_exception_subcode_t sub);
 
-void stop(struct proc *p);
-int cansignal(struct proc *, kauth_cred_t, struct proc *, int);
-int killpg1(struct proc *, int, int, int);
-void sigexit_locked(struct proc *, int);
-int setsigvec(struct proc *, int, struct __user_sigaction *);
-void exit1(struct proc *, int, int *);
-void psignal_uthread(thread_t, int);
+static void stop(proc_t, proc_t);
+int cansignal(proc_t, kauth_cred_t, proc_t, int, int);
+int killpg1(proc_t, int, int, int, int);
+int setsigvec(proc_t, int, struct __user_sigaction *);
+static void psignal_uthread(thread_t, int);
 kern_return_t do_bsdexception(int, int, int);
 void __posix_sem_syscall_return(kern_return_t);
 
 /* implementations in osfmk/kern/sync_sema.c. We do not want port.h in this scope, so void * them  */
-kern_return_t semaphore_timedwait_signal_trap_internal(void *, void *,time_t, int32_t, void (*)(int));
-kern_return_t semaphore_timedwait_trap_internal(void *, time_t, int32_t, void (*)(int));
-kern_return_t semaphore_wait_signal_trap_internal(void *, void *, void (*)(int));
-kern_return_t semaphore_wait_trap_internal(void *, void (*)(int));
+kern_return_t semaphore_timedwait_signal_trap_internal(mach_port_name_t, mach_port_name_t, unsigned int, clock_res_t, void (*)(kern_return_t));
+kern_return_t semaphore_timedwait_trap_internal(mach_port_name_t, unsigned int, clock_res_t, void (*)(kern_return_t));
+kern_return_t semaphore_wait_signal_trap_internal(mach_port_name_t, mach_port_name_t, void (*)(kern_return_t));
+kern_return_t semaphore_wait_trap_internal(mach_port_name_t, void (*)(kern_return_t));
 
 static int	filt_sigattach(struct knote *kn);
 static void	filt_sigdetach(struct knote *kn);
@@ -144,6 +152,35 @@ static int	filt_signal(struct knote *kn, long hint);
 struct filterops sig_filtops =
 	{ 0, filt_sigattach, filt_sigdetach, filt_signal };
 
+/* structures  and fns for killpg1 iterartion callback and filters */
+struct killpg1_filtargs {
+	int  posix;
+	proc_t cp;
+};
+
+struct killpg1_iterargs {
+	proc_t cp;
+	kauth_cred_t uc;
+	int signum;
+	int * nfoundp;
+};
+
+static int killpg1_filt(proc_t p, void * arg);
+static int killpg1_pgrpfilt(proc_t p, __unused void * arg);
+static int killpg1_callback(proc_t p, void * arg);
+
+static int pgsignal_filt(proc_t p, void * arg);
+static int pgsignal_callback(proc_t p, void * arg);
+static kern_return_t get_signalthread(proc_t, int, thread_t *);
+
+
+/* flags for psignal_internal */
+#define PSIG_LOCKED     0x1
+#define PSIG_VFORK      0x2
+#define PSIG_THREAD     0x4
+
+
+static void psignal_internal(proc_t p, task_t task, thread_t thread, int flavor, int signum);
 
 /*
  * NOTE: Source and target may *NOT* overlap! (target is smaller)
@@ -200,81 +237,32 @@ ram_printf(int x)
 }
 #endif /* SIGNAL_DEBUG */
 
-int
-signal_lock(struct proc *p)
-{
-int error = 0;
-#if DIAGNOSTIC
-#if SIGNAL_DEBUG
-#ifdef __ppc__
-        {
-            int register sp, *fp, numsaved; 
- 
-            __asm__ volatile("mr %0,r1" : "=r" (sp));
-
-            fp = (int *)*((int *)sp);
-            for (numsaved = 0; numsaved < 3; numsaved++) {
-                p->lockpc[numsaved] = fp[2];
-                if ((int)fp <= 0)
-                        break;
-                fp = (int *)*fp;
-            }
-        }
-#endif /* __ppc__ */       
-#endif /* SIGNAL_DEBUG */
-#endif /* DIAGNOSTIC */
-
-siglock_retry:
-	error = lockmgr((struct lock__bsd__ *)&p->signal_lock[0], LK_EXCLUSIVE, 0, (struct proc *)0);
-	if (error == EINTR)
-		goto siglock_retry;
-	return(error);
-}
-
-int
-signal_unlock(struct proc *p)
-{
-#if DIAGNOSTIC
-#if SIGNAL_DEBUG
-#ifdef __ppc__
-        {
-            int register sp, *fp, numsaved; 
- 
-            __asm__ volatile("mr %0,r1" : "=r" (sp));
-
-            fp = (int *)*((int *)sp);
-            for (numsaved = 0; numsaved < 3; numsaved++) {
-                p->unlockpc[numsaved] = fp[2];
-                if ((int)fp <= 0)
-                        break;
-                fp = (int *)*fp;
-            }
-        }
-#endif /* __ppc__ */       
-#endif /* SIGNAL_DEBUG */
-#endif /* DIAGNOSTIC */
-
-	/* TBD:  check p last arg */
-	return(lockmgr((struct lock__bsd__ *)&p->signal_lock[0], LK_RELEASE, (simple_lock_t)0, (struct proc *)0));
-}
 
 void
-signal_setast(sig_actthread)
-thread_t sig_actthread;
+signal_setast(thread_t sig_actthread)
 {
 	act_set_astbsd(sig_actthread);
 }
 
 /*
  * Can process p, with ucred uc, send the signal signum to process q?
+ * uc is refcounted  by the caller so internal fileds can be used safely
+ * when called with zombie arg, list lock is held
  */
 int
-cansignal(p, uc, q, signum)
-	struct proc *p;
-	kauth_cred_t uc;
-	struct proc *q;
-	int signum;
+cansignal(proc_t p, kauth_cred_t uc, proc_t q, int signum, int zombie)
 {
+	kauth_cred_t my_cred;
+	struct session * p_sessp = SESSION_NULL;
+	struct session * q_sessp = SESSION_NULL;
+#if CONFIG_MACF
+	int error;
+
+	error = mac_proc_check_signal(p, q, signum);
+	if (error)
+		return (0);
+#endif
+
 	/* you can signal yourself */
 	if (p == q)
 		return(1);
@@ -282,61 +270,67 @@ cansignal(p, uc, q, signum)
 	if (!suser(uc, NULL))
 		return (1);		/* root can always signal */
 
-	if (signum == SIGCONT && q->p_session == p->p_session)
+	if (zombie == 0)
+		proc_list_lock();
+	if (p->p_pgrp != PGRP_NULL)
+		p_sessp = p->p_pgrp->pg_session;
+	if (q->p_pgrp != PGRP_NULL)
+		q_sessp = q->p_pgrp->pg_session;
+
+	if (signum == SIGCONT && q_sessp == p_sessp) {
+		if (zombie == 0)
+			proc_list_unlock();
 		return (1);		/* SIGCONT in session */
+	}
+
+	if (zombie == 0) 
+		proc_list_unlock();
 
 	/*
-	 * Using kill(), only certain signals can be sent to setugid
-	 * child processes
+	 * If the real or effective UID of the sender matches the real,
+	 * effective, or ssaved UID of the target, permit the signal to
+	 * be sent.
 	 */
-	if (q->p_flag & P_SUGID) {
-		switch (signum) {
-		case 0:
-		case SIGKILL:
-		case SIGINT:
-		case SIGTERM:
-		case SIGSTOP:
-		case SIGTTIN:
-		case SIGTTOU:
-		case SIGTSTP:
-		case SIGHUP:
-		case SIGUSR1:
-		case SIGUSR2:
-			if (uc->cr_ruid == q->p_ucred->cr_ruid ||
-			    kauth_cred_getuid(uc) == q->p_ucred->cr_ruid ||
-			    uc->cr_ruid == kauth_cred_getuid(q->p_ucred) ||
-			    kauth_cred_getuid(uc) == kauth_cred_getuid(q->p_ucred))
-				return (1);
-		}
-		return (0);
+	if (zombie == 0)
+		my_cred = kauth_cred_proc_ref(q);
+	else
+		my_cred = proc_ucred(q);
+
+	if (uc->cr_ruid == my_cred->cr_ruid ||
+	    uc->cr_ruid == my_cred->cr_svuid ||
+	    kauth_cred_getuid(uc) == my_cred->cr_ruid ||
+	    kauth_cred_getuid(uc) == my_cred->cr_svuid ||
+	    uc->cr_ruid == kauth_cred_getuid(my_cred) ||
+	    kauth_cred_getuid(uc) == kauth_cred_getuid(my_cred)) {
+		if (zombie == 0)
+			kauth_cred_unref(&my_cred);
+		return (1);
 	}
 
-	/* XXX
-	 * because the P_SUGID test exists, this has extra tests which
-	 * could be removed.
-	 */
-	if (uc->cr_ruid == q->p_ucred->cr_ruid ||
-	    uc->cr_ruid == q->p_ucred->cr_svuid ||
-	    kauth_cred_getuid(uc) == q->p_ucred->cr_ruid ||
-	    kauth_cred_getuid(uc) == q->p_ucred->cr_svuid ||
-	    uc->cr_ruid == kauth_cred_getuid(q->p_ucred) ||
-	    kauth_cred_getuid(uc) == kauth_cred_getuid(q->p_ucred))
-		return (1);
+	if (zombie == 0)
+		kauth_cred_unref(&my_cred);
+
 	return (0);
 }
 
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	copyout:EFAULT
+ *	copyin:EFAULT
+ */
 /* ARGSUSED */
 int
-sigaction(struct proc *p, register struct sigaction_args *uap, __unused register_t *retval)
+sigaction(proc_t p, struct sigaction_args *uap, __unused register_t *retval)
 {
 	struct user_sigaction vec;
 	struct __user_sigaction __vec;
 
 	struct user_sigaction *sa = &vec;
-	register struct sigacts *ps = p->p_sigacts;
+	struct sigacts *ps = p->p_sigacts;
 
-	register int signum;
+	int signum;
 	int bit, error=0;
 
 	signum = uap->signum;
@@ -384,6 +378,7 @@ sigaction(struct proc *p, register struct sigaction_args *uap, __unused register
 		}
 		if (error)
 			return (error);
+		__vec.sa_flags &= SA_USERSPACE_MASK; /* Only pass on valid sa_flags */
 		error = setsigvec(p, signum, &__vec);
 	}
 	return (error);
@@ -391,115 +386,132 @@ sigaction(struct proc *p, register struct sigaction_args *uap, __unused register
 
 /* Routines to manipulate bits on all threads */
 int
-clear_procsiglist(struct proc *p,  int bit)
+clear_procsiglist(proc_t p,  int bit)
 {
 	struct uthread * uth;
 	thread_t thact;
 
-	signal_lock(p);
+	proc_lock(p);
+	proc_signalstart(p, 1);
 
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
 		thact = p->p_vforkact;	
 		uth = (struct uthread *)get_bsdthread_info(thact);
 		if (uth) {
 			uth->uu_siglist &= ~bit;
 		}
-		p->p_siglist &= ~bit;
-		signal_unlock(p);
+		proc_signalend(p, 1);
+		proc_unlock(p);
 		return(0);
 	} 
 
 	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
 		uth->uu_siglist &= ~bit;
 	}
-	p->p_siglist &= ~bit;
-	signal_unlock(p);
+
+	proc_signalend(p, 1);
+	proc_unlock(p);
+
 	return(0);
 }
 
 
 static int
-unblock_procsigmask(struct proc *p,  int bit)
+unblock_procsigmask(proc_t p,  int bit)
 {
 	struct uthread * uth;
 	thread_t thact;
 
-	signal_lock(p);
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
+	proc_lock(p);
+	proc_signalstart(p, 1);
+
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
 		thact = p->p_vforkact;	
 		uth = (struct uthread *)get_bsdthread_info(thact);
 		if (uth) {
 			uth->uu_sigmask &= ~bit;
 		}
 		p->p_sigmask &= ~bit;
-		signal_unlock(p);
+		proc_signalend(p, 1);
+		proc_unlock(p);
 		return(0);
 	} 
 	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
 		uth->uu_sigmask &= ~bit;
 	}
 	p->p_sigmask &= ~bit;
-	signal_unlock(p);
+
+	proc_signalend(p, 1);
+	proc_unlock(p);
 	return(0);
 }
 
-
 static int
-block_procsigmask(struct proc *p,  int bit)
+block_procsigmask(proc_t p,  int bit)
 {
 	struct uthread * uth;
 	thread_t thact;
 
-	signal_lock(p);
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
+	proc_lock(p);
+	proc_signalstart(p, 1);
+
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
 		thact = p->p_vforkact;	
 		uth = (struct uthread *)get_bsdthread_info(thact);
 		if (uth) {
 			uth->uu_sigmask |= bit;
 		}
 		p->p_sigmask |=  bit;
-		signal_unlock(p);
+		proc_signalend(p, 1);
+		proc_unlock(p);
 		return(0);
 	} 
 	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
 		uth->uu_sigmask |= bit;
 	}
 	p->p_sigmask |=  bit;
-	signal_unlock(p);
+
+	proc_signalend(p, 1);
+	proc_unlock(p);
 	return(0);
 }
 
 int
-set_procsigmask(struct proc *p,  int bit)
+set_procsigmask(proc_t p,  int bit)
 {
 	struct uthread * uth;
 	thread_t thact;
 
-	signal_lock(p);
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
+	proc_lock(p);
+	proc_signalstart(p, 1);
+
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
 		thact = p->p_vforkact;	
 		uth = (struct uthread *)get_bsdthread_info(thact);
 		if (uth) {
 			uth->uu_sigmask = bit;
 		}
 		p->p_sigmask =  bit;
-		signal_unlock(p);
+		proc_signalend(p, 1);
+		proc_unlock(p);
 		return(0);
 	} 
 	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
 		uth->uu_sigmask = bit;
 	}
 	p->p_sigmask =  bit;
-	signal_unlock(p);
+	proc_signalend(p, 1);
+	proc_unlock(p);
+
 	return(0);
 }
 
 /* XXX should be static? */
 int
-setsigvec(struct proc *p, int signum, struct __user_sigaction *sa)
+setsigvec(proc_t p, int signum, struct __user_sigaction *sa)
 {
-	register struct sigacts *ps = p->p_sigacts;
-	register int bit;
+	struct sigacts *ps = p->p_sigacts;
+	int bit;
 
 	if ((signum == SIGKILL || signum == SIGSTOP) &&
 		sa->sa_handler != SIG_DFL)
@@ -541,13 +553,13 @@ setsigvec(struct proc *p, int signum, struct __user_sigaction *sa)
 		ps->ps_signodefer &= ~bit;
 	if (signum == SIGCHLD) {
 		if (sa->sa_flags & SA_NOCLDSTOP)
-			p->p_flag |= P_NOCLDSTOP;
+			OSBitOrAtomic(P_NOCLDSTOP, (UInt32 *)&p->p_flag);
 		else
-			p->p_flag &= ~P_NOCLDSTOP;
+			OSBitAndAtomic(~((uint32_t)P_NOCLDSTOP), (UInt32 *)&p->p_flag);
 		if ((sa->sa_flags & SA_NOCLDWAIT) || (sa->sa_handler == SIG_IGN))
-			p->p_flag |= P_NOCLDWAIT;
+			OSBitOrAtomic(P_NOCLDWAIT, (UInt32 *)&p->p_flag);
 		else
-			p->p_flag &= ~P_NOCLDWAIT;
+			OSBitAndAtomic(~((uint32_t)P_NOCLDWAIT), (UInt32 *)&p->p_flag);
 	}
 
 #ifdef __ppc__ 
@@ -586,10 +598,9 @@ setsigvec(struct proc *p, int signum, struct __user_sigaction *sa)
  * set to ignore signals that are ignored by default.
  */
 void
-siginit(p)
-	struct proc *p;
+siginit(proc_t p)
 {
-	register int i;
+	int i;
 
 	for (i = 0; i < NSIG; i++)
 		if (sigprop[i] & SA_IGNORE && i != SIGCONT)
@@ -600,17 +611,13 @@ siginit(p)
  * Reset signals for an exec of the specified process.
  */
 void
-execsigs(p, thr_act)
-	register struct proc *p;
-	register thread_t thr_act;
+execsigs(proc_t p, thread_t thread)
 {
-	register struct sigacts *ps = p->p_sigacts;
-	register int nc, mask;
-	struct uthread *ut = (struct uthread *)0;
+	struct sigacts *ps = p->p_sigacts;
+	int nc, mask;
+	struct uthread *ut;
 
-	if (thr_act){
-		ut = (struct uthread *)get_bsdthread_info(thr_act);
-	}
+	ut = (struct uthread *)get_bsdthread_info(thread);
 	/*
 	 * Reset caught signals.  Held signals remain held
 	 * through p_sigmask (unless they were caught,
@@ -623,9 +630,8 @@ execsigs(p, thr_act)
 		if (sigprop[nc] & SA_IGNORE) {
 			if (nc != SIGCONT)
 				p->p_sigignore |= mask;
-			if (thr_act){
+			if (thread){
 				ut->uu_siglist &= ~mask;
-				p->p_siglist &= ~mask;
 			} else
 				clear_procsiglist(p, mask);
 		}
@@ -635,16 +641,12 @@ execsigs(p, thr_act)
 	 * Reset stack state to the user stack.
 	 * Clear set of signals caught on the signal stack.
 	 */
-	ps->ps_sigstk.ss_flags = SA_DISABLE;
-	ps->ps_sigstk.ss_size = 0;
-	ps->ps_sigstk.ss_sp = USER_ADDR_NULL;
-	ps->ps_flags = 0;
-	if (thr_act) {
-		ut->uu_sigstk.ss_flags = SA_DISABLE;
-		ut->uu_sigstk.ss_size = 0;
-		ut->uu_sigstk.ss_sp = USER_ADDR_NULL;
-		ut->uu_flag &= ~UT_ALTSTACK;
-	}
+	/* thread */
+	ut->uu_sigstk.ss_flags = SA_DISABLE;
+	ut->uu_sigstk.ss_size = 0;
+	ut->uu_sigstk.ss_sp = USER_ADDR_NULL;
+	ut->uu_flag &= ~UT_ALTSTACK;
+	/* process */
 	ps->ps_sigonstack = 0;
 }
 
@@ -655,7 +657,7 @@ execsigs(p, thr_act)
  * the library stub does the rest.
  */
 int
-sigprocmask(register struct proc *p, struct sigprocmask_args *uap, __unused register_t *retval)
+sigprocmask(proc_t p, struct sigprocmask_args *uap, __unused register_t *retval)
 {
 	int error = 0;
 	sigset_t oldmask, nmask;
@@ -700,7 +702,7 @@ out:
 }
 
 int
-sigpending(__unused struct proc *p, register struct sigpending_args *uap, __unused register_t *retval)
+sigpending(__unused proc_t p, struct sigpending_args *uap, __unused register_t *retval)
 {
 	struct uthread *ut;
 	sigset_t pendlist;
@@ -713,7 +715,6 @@ sigpending(__unused struct proc *p, register struct sigpending_args *uap, __unus
 	return(0);
 }
 
-
 /*
  * Suspend process until signal, providing mask to be set
  * in the meantime.  Note nonstandard calling convention:
@@ -724,11 +725,18 @@ static int
 sigcontinue(__unused int error)
 {
 //	struct uthread *ut = get_bsdthread_info(current_thread());
-  unix_syscall_return(EINTR);
+	unix_syscall_return(EINTR);
+}
+
+int
+sigsuspend(proc_t p, struct sigsuspend_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(sigsuspend_nocancel(p, (struct sigsuspend_nocancel_args *)uap, retval));
 }
 
 int
-sigsuspend(register struct proc *p, struct sigsuspend_args *uap, __unused register_t *retval)
+sigsuspend_nocancel(proc_t p, struct sigsuspend_nocancel_args *uap, __unused register_t *retval)
 {
 	struct uthread *ut;
 
@@ -751,29 +759,47 @@ sigsuspend(register struct proc *p, struct sigsuspend_args *uap, __unused regist
 
 
 int
-__disable_threadsignal(struct proc *p, 
-					   __unused register struct __disable_threadsignal_args *uap, 
-					   __unused register_t *retval)
+__disable_threadsignal(__unused proc_t p,
+		       __unused struct __disable_threadsignal_args *uap,
+		       __unused register_t *retval)
 {
 	struct uthread *uth;
 
 	uth = (struct uthread *)get_bsdthread_info(current_thread());
 
 	/* No longer valid to have any signal delivered */
-	signal_lock(p);
-	uth->uu_flag |= UT_NO_SIGMASK;
-	signal_unlock(p);
+	uth->uu_flag |= (UT_NO_SIGMASK | UT_CANCELDISABLE);
 
 	return(0);
 
 }
 
+void
+__pthread_testcancel(int presyscall)
+{
+
+	thread_t self = current_thread();
+	struct uthread * uthread;
+
+	uthread = (struct uthread *)get_bsdthread_info(self);
+
+	
+	uthread->uu_flag &= ~UT_NOTCANCELPT;
+
+	if ((uthread->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL) {
+		if(presyscall != 0) {
+			unix_syscall_return(EINTR);
+			/* NOTREACHED */
+		} else 
+			thread_abort_safely(self);
+	}
+}
+
+
 
 int
-__pthread_markcancel(p, uap, retval)
-	struct proc *p;
-	register struct __pthread_markcancel_args *uap;
-	register_t *retval;
+__pthread_markcancel(__unused proc_t p,
+	struct __pthread_markcancel_args *uap, __unused register_t *retval)
 {
 	thread_act_t target_act;
 	int error = 0;
@@ -787,7 +813,7 @@ __pthread_markcancel(p, uap, retval)
 	uth = (struct uthread *)get_bsdthread_info(target_act);
 
 	/* if the thread is in vfork do not cancel */
-	if ((uth->uu_flag & (P_VFORK | UT_CANCEL | UT_CANCELED )) == 0) {
+	if ((uth->uu_flag & (UT_VFORK | UT_CANCEL | UT_CANCELED )) == 0) {
 		uth->uu_flag |= (UT_CANCEL | UT_NO_SIGMASK);
 		if (((uth->uu_flag & UT_NOTCANCELPT) == 0) 
 			&& ((uth->uu_flag & UT_CANCELDISABLE) == 0))
@@ -804,17 +830,15 @@ __pthread_markcancel(p, uap, retval)
  * if action = 2; Disable the cancel handling
  */
 int
-__pthread_canceled(p, uap, retval)
-	struct proc *p;
-	register struct __pthread_canceled_args *uap;
-	register_t *retval;
+__pthread_canceled(__unused proc_t p,
+	struct __pthread_canceled_args *uap, __unused register_t *retval)
 {
-	thread_act_t thr_act;
+	thread_act_t thread;
 	struct uthread *uth;
 	int action = uap->action;
 
-	thr_act = current_act();
-	uth = (struct uthread *)get_bsdthread_info(thr_act);
+	thread = current_thread();
+	uth = (struct uthread *)get_bsdthread_info(thread);
 
 	switch (action) {
 		case 1:
@@ -853,12 +877,23 @@ __posix_sem_syscall_return(kern_return_t kern_result)
 	/* does not return */
 }
 
+/*
+ * Returns:	0			Success
+ *		EINTR
+ *		ETIMEDOUT
+ *		EINVAL
+ */
+int
+__semwait_signal(__unused proc_t p, struct __semwait_signal_args *uap,
+		register_t *retval)
+{
+	__pthread_testcancel(0);
+	return(__semwait_signal_nocancel(p, (struct __semwait_signal_nocancel_args *)uap, retval));
+}
 
 int
-__semwait_signal(p, uap, retval)
-	struct proc *p;
-	register struct __semwait_signal_args *uap;
-	register_t *retval;
+__semwait_signal_nocancel(__unused proc_t p, struct __semwait_signal_nocancel_args *uap,
+		__unused register_t *retval)
 {
 
 	kern_return_t kern_result;
@@ -878,23 +913,27 @@ __semwait_signal(p, uap, retval)
 				then.tv_nsec += NSEC_PER_SEC;
 				then.tv_sec--; 
 			}
+			/* if time has elapsed, set time to null timepsec to bailout rightaway */
+			if ((int)then.tv_sec < 0) {
+				then.tv_sec = 0;
+				then.tv_nsec = 0;
+			}
 		}
 
-		if (uap->mutex_sem == (void *)NULL)
-			kern_result = semaphore_timedwait_trap_internal(uap->cond_sem, then.tv_sec, then.tv_nsec, __posix_sem_syscall_return);
+		if (uap->mutex_sem == 0)
+			kern_result = semaphore_timedwait_trap_internal((mach_port_name_t)uap->cond_sem, then.tv_sec, then.tv_nsec, __posix_sem_syscall_return);
 		else
 			kern_result = semaphore_timedwait_signal_trap_internal(uap->cond_sem, uap->mutex_sem, then.tv_sec, then.tv_nsec, __posix_sem_syscall_return);
 
 	} else {
 
-		if (uap->mutex_sem == (void *)NULL)
+		if (uap->mutex_sem == 0)
 			kern_result = semaphore_wait_trap_internal(uap->cond_sem, __posix_sem_syscall_return);
 		else
 
 			kern_result = semaphore_wait_signal_trap_internal(uap->cond_sem, uap->mutex_sem, __posix_sem_syscall_return);
 	}
 
-out:
 	if (kern_result == KERN_SUCCESS)
 		return(0);
 	else if (kern_result == KERN_ABORTED)
@@ -905,11 +944,9 @@ out:
 		return(EINVAL);
 }
 
-
 int 
-__pthread_kill(__unused struct proc *p, 
-			   register struct __pthread_kill_args *uap, 
-			   __unused register_t *retval) 
+__pthread_kill(__unused proc_t p, struct __pthread_kill_args *uap,
+	       __unused register_t *retval) 
 {
 	thread_t target_act;
 	int error = 0;
@@ -941,9 +978,8 @@ out:
 
 
 int 
-pthread_sigmask(__unused register struct proc *p, 
-				register struct pthread_sigmask_args *uap, 
-				__unused register_t *retval)
+__pthread_sigmask(__unused proc_t p, struct __pthread_sigmask_args *uap,
+		  __unused register_t *retval)
 {
 	user_addr_t set = uap->set;
 	user_addr_t oset = uap->oset;
@@ -990,9 +1026,21 @@ out:
 	return(error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ */
+int 
+__sigwait(proc_t p, struct __sigwait_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(__sigwait_nocancel(p, (struct __sigwait_nocancel_args *)uap, retval));
+}
 
 int 
-sigwait(register struct proc *p, register struct sigwait_args *uap, __unused register_t *retval)
+__sigwait_nocancel(proc_t p, struct __sigwait_nocancel_args *uap, __unused register_t *retval)
 {
 	struct uthread *ut;
 	struct uthread *uth;
@@ -1016,18 +1064,20 @@ sigwait(register struct proc *p, register struct sigwait_args *uap, __unused reg
 	if (siglist == 0)
 		return(EINVAL);
 
-	signal_lock(p);
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
-		signal_unlock(p);
+	proc_lock(p);
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
+		proc_unlock(p);
 		return(EINVAL);
 	} else {
+		proc_signalstart(p, 1);
 		TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
 			if ( (sigw = uth->uu_siglist & siglist) ) {
 				break;
 			}
 		}
+		proc_signalend(p, 1);
 	}
-	signal_unlock(p);
+
 	if (sigw) {
 		/* The signal was pending on a thread */
 		goto sigwait1;
@@ -1039,15 +1089,19 @@ sigwait(register struct proc *p, register struct sigwait_args *uap, __unused reg
 	 * save it here and mark the sigacts structure
 	 * to indicate this.
 	 */
+	uth = ut;		/* wait for it to be delivered to us */
 	ut->uu_oldmask = ut->uu_sigmask;
 	ut->uu_flag |= UT_SAS_OLDMASK;
-	if (siglist == (sigset_t)0)
+	if (siglist == (sigset_t)0) {
+		proc_unlock(p);
 		return(EINVAL);
+	}
 	/* SIGKILL and SIGSTOP are not maskable as well */
 	ut->uu_sigmask = ~(siglist|sigcantmask);
 	ut->uu_sigwait = siglist; 
+
 	/* No Continuations for now */
-	error =  tsleep((caddr_t)&ut->uu_sigwait, PPAUSE|PCATCH, "pause", 0);
+	error =  msleep((caddr_t)&ut->uu_sigwait, &p->p_mlock, PPAUSE|PCATCH, "pause", 0);
 
 	if ((error == EINTR) || (error == ERESTART))
 		error = 0;
@@ -1062,39 +1116,33 @@ sigwait1:
 		signum = ffs((unsigned int)sigw);
 		if (!signum)
 			panic("sigwait with no signal wakeup");
-		ut->uu_siglist &= ~(sigmask(signum));
+		/* Clear the pending signal in the thread it was delivered */
+		uth->uu_siglist &= ~(sigmask(signum));
+		proc_unlock(p);
 		if (uap->sig != USER_ADDR_NULL)
 				error = copyout(&signum, uap->sig, sizeof(int));
-	}
+	} else
+		proc_unlock(p);
 
 	return(error);
 
 }
 
-
 int
-sigaltstack(struct proc *p, register struct sigaltstack_args *uap, __unused register_t *retval)
+sigaltstack(__unused proc_t p, struct sigaltstack_args *uap, __unused register_t *retval)
 {
-	struct sigacts *psp;
-	struct user_sigaltstack *pstk;
 	struct user_sigaltstack ss;
-	struct uthread *uth;
-	int uthsigaltstack = 0;
+	struct user_sigaltstack *pstk;
 	int error;
+	struct uthread *uth;
+	int onstack;
 
 	uth = (struct uthread *)get_bsdthread_info(current_thread());
-	uthsigaltstack = p->p_lflag & P_LTHSIGSTACK;
 
-	psp = p->p_sigacts;
-	if (uthsigaltstack != 0)  {
-		pstk = &uth->uu_sigstk;
-		if ((uth->uu_flag & UT_ALTSTACK) == 0)
-			uth->uu_sigstk.ss_flags |= SA_DISABLE;
-	} else {
-		pstk = &psp->ps_sigstk;
-		if ((psp->ps_flags & SAS_ALTSTACK) == 0)
-			psp->ps_sigstk.ss_flags |= SA_DISABLE;
-	}
+	pstk = &uth->uu_sigstk;
+	if ((uth->uu_flag & UT_ALTSTACK) == 0)
+		uth->uu_sigstk.ss_flags |= SA_DISABLE;
+	onstack = pstk->ss_flags & SA_ONSTACK;
 	if (uap->oss) {
 		if (IS_64BIT_PROCESS(p)) {
 			error = copyout(pstk, uap->oss, sizeof(struct user_sigaltstack));
@@ -1122,40 +1170,30 @@ sigaltstack(struct proc *p, register struct sigaltstack_args *uap, __unused regi
 	}
 
 	if (ss.ss_flags & SA_DISABLE) {
-		if (uthsigaltstack != 0)  {
-			/* if we are here we are not in the signal handler ;so no need to check */
-			if (uth->uu_sigstk.ss_flags & SA_ONSTACK)
-				return (EINVAL);
-			uth->uu_flag &= ~UT_ALTSTACK;
-			uth->uu_sigstk.ss_flags = ss.ss_flags;
-		} else {
-			if (psp->ps_sigstk.ss_flags & SA_ONSTACK)
-				return (EINVAL);
-			psp->ps_flags &= ~SAS_ALTSTACK;
-			psp->ps_sigstk.ss_flags = ss.ss_flags;
-		}
-		
+		/* if we are here we are not in the signal handler ;so no need to check */
+		if (uth->uu_sigstk.ss_flags & SA_ONSTACK)
+			return (EINVAL);
+		uth->uu_flag &= ~UT_ALTSTACK;
+		uth->uu_sigstk.ss_flags = ss.ss_flags;
 		return (0);
 	}
+	if (onstack)
+		return (EPERM);
 /* The older stacksize was 8K, enforce that one so no compat problems */
 #define OLDMINSIGSTKSZ 8*1024
 	if (ss.ss_size < OLDMINSIGSTKSZ)
 		return (ENOMEM);
-	if (uthsigaltstack != 0)  {
-		uth->uu_flag |= UT_ALTSTACK;
-		uth->uu_sigstk= ss;
-	} else {
-		psp->ps_flags |= SAS_ALTSTACK;
-		psp->ps_sigstk= ss;
-	}
+	uth->uu_flag |= UT_ALTSTACK;
+	uth->uu_sigstk= ss;
 	return (0);
 }
 
 int
-kill(struct proc *cp, struct kill_args *uap, __unused register_t *retval)
+kill(proc_t cp, struct kill_args *uap, __unused register_t *retval)
 {
-	register struct proc *p;
+	proc_t p;
 	kauth_cred_t uc = kauth_cred_get();
+	int posix = uap->posix;		/* !0 if posix behaviour desired */
 
        AUDIT_ARG(pid, uap->pid);
        AUDIT_ARG(signum, uap->signum);
@@ -1164,7 +1202,7 @@ kill(struct proc *cp, struct kill_args *uap, __unused register_t *retval)
 		return (EINVAL);
 	if (uap->pid > 0) {
 		/* kill single process */
-		if ((p = proc_findref(uap->pid)) == NULL) {
+		if ((p = proc_find(uap->pid)) == NULL) {
 			if ((p = pzfind(uap->pid)) != NULL) {
 				/*
 				 * IEEE Std 1003.1-2001: return success
@@ -1175,139 +1213,226 @@ kill(struct proc *cp, struct kill_args *uap, __unused register_t *retval)
 			return (ESRCH);
 		}
 		AUDIT_ARG(process, p);
-		if (!cansignal(cp, uc, p, uap->signum)) {
-			proc_dropref(p);
+		if (!cansignal(cp, uc, p, uap->signum, 0)) {
+			proc_rele(p);
 			return(EPERM);
 		}
 		if (uap->signum)
 			psignal(p, uap->signum);
-		proc_dropref(p);
+		proc_rele(p);
 		return (0);
 	}
 	switch (uap->pid) {
 	case -1:		/* broadcast signal */
-		return (killpg1(cp, uap->signum, 0, 1));
+		return (killpg1(cp, uap->signum, 0, 1, posix));
 	case 0:			/* signal own process group */
-		return (killpg1(cp, uap->signum, 0, 0));
+		return (killpg1(cp, uap->signum, 0, 0, posix));
 	default:		/* negative explicit process group */
-		return (killpg1(cp, uap->signum, -(uap->pid), 0));
+		return (killpg1(cp, uap->signum, -(uap->pid), 0, posix));
 	}
 	/* NOTREACHED */
 }
 
+static int
+killpg1_filt(proc_t p, void * arg)
+{
+	struct killpg1_filtargs * kfargp = (struct killpg1_filtargs *)arg;
+	proc_t cp = kfargp->cp;
+	int posix = kfargp->posix;
+
+
+	if (p->p_pid <= 1 || p->p_flag & P_SYSTEM ||
+		(!posix && p == cp))
+		return(0);
+	else
+		return(1);
+}
+
+
+static int
+killpg1_pgrpfilt(proc_t p, __unused void * arg)
+{
+        if (p->p_pid <= 1 || p->p_flag & P_SYSTEM ||
+                (p->p_stat == SZOMB))
+                return(0);
+        else
+                return(1);
+}
+
+
+
+static int
+killpg1_callback(proc_t p, void * arg)
+{
+	struct killpg1_iterargs * kargp = (struct killpg1_iterargs *)arg;
+        proc_t cp = kargp->cp;
+        kauth_cred_t uc = kargp->uc;   /* refcounted by the caller safe to use internal fields */
+        int signum = kargp->signum;
+	int * nfoundp = kargp->nfoundp;
+	int n;
+
+
+	if (cansignal(cp, uc, p, signum, 0) == 0)
+		return(PROC_RETURNED);
+
+	if (nfoundp != NULL) {
+		n = *nfoundp;
+		*nfoundp = n+1;
+	}
+	if (signum != 0)
+		psignal(p, signum);
+
+	return(PROC_RETURNED);
+
+}
 
 /*
  * Common code for kill process group/broadcast kill.
  * cp is calling process.
  */
 int
-killpg1(cp, signum, pgid, all)
-	register struct proc *cp;
-	int signum, pgid, all;
+killpg1(proc_t cp, int signum, int pgid, int all, int posix)
 {
-	register struct proc *p;
-	kauth_cred_t uc = cp->p_ucred;
+	proc_t p;
+	kauth_cred_t uc;
 	struct pgrp *pgrp;
 	int nfound = 0;
+	struct killpg1_iterargs karg;
+	struct killpg1_filtargs kfarg;
+	int error = 0;
 	
+	uc = kauth_cred_proc_ref(cp);
 	if (all) {
 		/* 
 		 * broadcast 
 		 */
-		for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
+		kfarg.posix = posix;
+		kfarg.cp = cp;
+
+		karg.cp = cp;
+		karg.uc = uc;
+		karg.nfoundp = &nfound;
+		karg.signum = signum;
+
+		proc_iterate(PROC_ALLPROCLIST, killpg1_callback, &karg, killpg1_filt, (void *)&kfarg);
+		/*
+		 * Signalling zombies is a no-op, but they must be counted
+		 * among those processes which have been signalled, since
+		 * they are still members of the process group.
+		 */
+
+		proc_list_lock();
+
+		for (p = zombproc.lh_first; p != 0; p = p->p_list.le_next) {
 			if (p->p_pid <= 1 || p->p_flag & P_SYSTEM || 
-			    p == cp || !cansignal(cp, uc, p, signum))
+			    (!posix && p == cp) || !cansignal(cp, uc, p, signum, 1))
 				continue;
 			nfound++;
-			if (signum)
-				psignal(p, signum);
 		}
+
+		proc_list_unlock();
+
 	} else {
-		if (pgid == 0)		
+		if (pgid == 0) {
 			/* 
 			 * zero pgid means send to my process group.
 			 */
-			pgrp = cp->p_pgrp;
-		else {
+			pgrp = proc_pgrp(cp);
+		 } else {
 			pgrp = pgfind(pgid);
-			if (pgrp == NULL)
-				return (ESRCH);
-		}
-		for (p = pgrp->pg_members.lh_first; p != 0;
-		     p = p->p_pglist.le_next) {
-			if (p->p_pid <= 1 || p->p_flag & P_SYSTEM ||
-			    p->p_stat == SZOMB ||
-			    !cansignal(cp, uc, p, signum))
-				continue;
-			nfound++;
-			if (signum)
-				psignal(p, signum);
+			if (pgrp == NULL) {
+				error = ESRCH;
+				goto out;
+			}
 		}
+
+                karg.nfoundp = &nfound;
+                karg.uc = uc;
+                karg.signum = signum;
+		karg.cp = cp;
+
+
+		/* PGRP_DROPREF drops the pgrp refernce */
+		pgrp_iterate(pgrp, PGRP_BLOCKITERATE | PGRP_DROPREF, killpg1_callback, &karg,
+			killpg1_pgrpfilt, NULL);
 	}
-	return (nfound ? 0 : ESRCH);
+	error =  (nfound ? 0 : (posix ? EPERM : ESRCH));
+out:
+	kauth_cred_unref(&uc);
+	return (error);
 }
 
+
 /*
  * Send a signal to a process group.
  */
 void
-gsignal(pgid, signum)
-	int pgid, signum;
+gsignal(int pgid, int signum)
 {
 	struct pgrp *pgrp;
 
-	if (pgid && (pgrp = pgfind(pgid)))
+	if (pgid && (pgrp = pgfind(pgid))) {
 		pgsignal(pgrp, signum, 0);
+		pg_rele(pgrp);
+	}
 }
 
 /*
- * Send a signal to a process group.  If checktty is 1,
+ * Send a signal to a process group.  If checkctty is 1,
  * limit to members which have a controlling terminal.
  */
-void
-pgsignal(pgrp, signum, checkctty)
-	struct pgrp *pgrp;
-	int signum, checkctty;
+
+static int
+pgsignal_filt(proc_t p, void * arg)
 {
-	register struct proc *p;
+	int checkctty = (int)arg;
 
-	if (pgrp)
-		for (p = pgrp->pg_members.lh_first; p != 0;
-		     p = p->p_pglist.le_next)
-			if (checkctty == 0 || p->p_flag & P_CONTROLT)
-				psignal(p, signum);
+	if ((checkctty == 0) || p->p_flag & P_CONTROLT)
+		return(1);
+	else 
+		return(0);
 }
 
-/*
- * Send signal to a backgrounded process blocked due to tty access
- * In FreeBSD, the backgrounded process wakes up every second and
- * discovers whether it is foregounded or not. In our case, we block 
- * the thread in tsleep as we want to avoid storm of processes as well 
- * as the suspend is only at AST level
- */
-void
-tty_pgsignal(pgrp, signum)
-	struct pgrp *pgrp;
-	int signum;
+
+static int
+pgsignal_callback(proc_t p, void * arg)
 {
-	register struct proc *p;
+        int  signum = (int)arg;
 
-	if (pgrp)
-		for (p = pgrp->pg_members.lh_first; p != 0;
-		     p = p->p_pglist.le_next)
-			if ((p->p_flag & P_TTYSLEEP) && (p->p_flag & P_CONTROLT))
-				psignal(p, signum);
+	psignal(p, signum);
+	return(PROC_RETURNED);
+}
+
+
+void
+pgsignal(struct pgrp *pgrp, int signum, int checkctty)
+{
+	if (pgrp != PGRP_NULL) {
+		pgrp_iterate(pgrp, PGRP_BLOCKITERATE, pgsignal_callback, (void *)signum, pgsignal_filt, (void *)checkctty);
+	}
 }
 
+
+void
+tty_pgsignal(struct tty *tp, int signum, int checkctty)
+{
+	struct pgrp * pg;
+
+	pg = tty_pgrp(tp);
+	if (pg != PGRP_NULL) {
+		pgrp_iterate(pg, PGRP_BLOCKITERATE, pgsignal_callback, (void *)signum, pgsignal_filt, (void *)checkctty);
+		pg_rele(pg);
+	}
+}
 /*
  * Send a signal caused by a trap to a specific thread.
  */
 void
-threadsignal(thread_t sig_actthread, int signum, u_long code)
+threadsignal(thread_t sig_actthread, int signum, mach_exception_code_t code)
 {
-	register struct uthread *uth;
-	register struct task * sig_task;
-	register struct proc *p ;
+	struct uthread *uth;
+	struct task * sig_task;
+	proc_t p;
 	int mask;
 
 	if ((u_int)signum >= NSIG || signum == 0)
@@ -1317,38 +1442,93 @@ threadsignal(thread_t sig_actthread, int signum, u_long code)
 	if ((mask & threadmask) == 0)
 		return;
 	sig_task = get_threadtask(sig_actthread);
-	p = (struct proc *)(get_bsdtask_info(sig_task));
+	p = (proc_t)(get_bsdtask_info(sig_task));
 
 	uth = get_bsdthread_info(sig_actthread);
 	if (uth && (uth->uu_flag & UT_VFORK))
 		p = uth->uu_proc;
 
-	if (!(p->p_flag & P_TRACED) && (p->p_sigignore & mask))
+	proc_lock(p);
+	if (!(p->p_lflag & P_LTRACED) && (p->p_sigignore & mask)) {
+		proc_unlock(p);
 		return;
+	}
 
 	uth->uu_siglist |= mask;
-	p->p_siglist |= mask;	/* just for lame ones looking here */
 	uth->uu_code = code;
+	proc_unlock(p);
+
 	/* mark on process as well */
 	signal_setast(sig_actthread);
 }
 
- 
-void
-psignal(p, signum)
-	register struct proc *p;
-	register int signum;
+static kern_return_t
+get_signalthread(proc_t p, int signum, thread_t * thr)
 {
-	psignal_lock(p, signum, 1);
+	struct uthread *uth;
+	sigset_t mask = sigmask(signum);
+	thread_t sig_thread;
+	struct task * sig_task = p->task;
+	kern_return_t kret;
+	
+	*thr = THREAD_NULL;
+
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
+		sig_thread = p->p_vforkact;	
+		kret = check_actforsig(sig_task, sig_thread, 1);
+		if (kret == KERN_SUCCESS)  {
+			*thr = sig_thread;
+			return(KERN_SUCCESS);
+		}else
+			return(KERN_FAILURE);
+	} 
+
+	proc_lock(p);
+	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
+		if(((uth->uu_flag & UT_NO_SIGMASK)== 0) && 
+			(((uth->uu_sigmask & mask) == 0) || (uth->uu_sigwait & mask))) {
+			if (check_actforsig(p->task, uth->uu_context.vc_thread, 1) == KERN_SUCCESS) {
+				*thr = uth->uu_context.vc_thread;
+				proc_unlock(p);
+				return(KERN_SUCCESS);
+			}
+		}
+	}
+	proc_unlock(p);
+	if (get_signalact(p->task, thr, 1) == KERN_SUCCESS) {
+		return(KERN_SUCCESS);
+	}
+
+	return(KERN_FAILURE);
 }
 
-void
-psignal_vfork(struct proc *p, task_t new_task, thread_t thr_act, int signum)
+/*
+ * Send the signal to the process.  If the signal has an action, the action
+ * is usually performed by the target process rather than the caller; we add
+ * the signal to the set of pending signals for the process.
+ *
+ * Exceptions:
+ *   o When a stop signal is sent to a sleeping process that takes the
+ *     default action, the process is stopped without awakening it.
+ *   o SIGCONT restarts stopped processes (or puts them back to sleep)
+ *     regardless of the signal action (eg, blocked or ignored).
+ *
+ * Other ignored signals are discarded immediately.
+ */
+static void
+psignal_internal(proc_t p, task_t task, thread_t thread, int flavor, int signum)
 {
-	register int prop;
-	register sig_t action;
+	int prop;
+	sig_t action = NULL;
+	proc_t 		sig_proc;
+	thread_t	sig_thread;
+	register task_t		sig_task;
 	int mask;
 	struct uthread *uth;
+	kern_return_t kret;
+	uid_t r_uid;
+	proc_t pp;
+	kauth_cred_t my_cred;
 
 	if ((u_int)signum >= NSIG || signum == 0)
 		panic("psignal signal number");
@@ -1356,195 +1536,49 @@ psignal_vfork(struct proc *p, task_t new_task, thread_t thr_act, int signum)
 	prop = sigprop[signum];
 
 #if SIGNAL_DEBUG
-        if(rdebug_proc && (p == rdebug_proc)) {
+        if(rdebug_proc && (p != PROC_NULL) && (p == rdebug_proc)) {
                 ram_printf(3);
         }
 #endif /* SIGNAL_DEBUG */
 
-	if ((new_task == TASK_NULL) || (thr_act == (thread_t)NULL)  || is_kerneltask(new_task))
-		return;
-
-
-	uth = get_bsdthread_info(thr_act);
-	signal_lock(p);
-
-	/*
-	 * proc is traced, always give parent a chance.
-	 */
-	action = SIG_DFL;
-
-	if (p->p_nice > NZERO && action == SIG_DFL && (prop & SA_KILL) &&
-		(p->p_flag & P_TRACED) == 0)
-		p->p_nice = NZERO;
-
-	if (prop & SA_CONT) {
-		p->p_siglist &= ~stopsigmask;
-		uth->uu_siglist &= ~stopsigmask;
-	}
-
-	if (prop & SA_STOP) {
-		/*
-		 * If sending a tty stop signal to a member of an orphaned
-		 * process group, discard the signal here if the action
-		 * is default; don't stop the process below if sleeping,
-		 * and don't clear any pending SIGCONT.
-		 */
-		if (prop & SA_TTYSTOP && p->p_pgrp->pg_jobc == 0 &&
-			action == SIG_DFL)
-			goto psigout;
-		uth->uu_siglist &= ~contsigmask;
-		p->p_siglist &= ~contsigmask;
-	}
-	uth->uu_siglist |= mask;
-	p->p_siglist |= mask;   /* just for lame ones looking here */
-	
-	/* Deliver signal to the activation passed in */
-	act_set_astbsd(thr_act);
-
-	/*
-	 *	SIGKILL priority twiddling moved here from above because
-	 *	it needs sig_thread.  Could merge it into large switch
-	 *	below if we didn't care about priority for tracing
-	 *	as SIGKILL's action is always SIG_DFL.
-	 */
-	if ((signum == SIGKILL) && (p->p_nice > NZERO)) {
-		p->p_nice = NZERO;
-	}
-
-	/*
-	 *	This Process is traced - wake it up (if not already
-	 *	stopped) so that it can discover the signal in
-	 *	issig() and stop for the parent.
-	 */
-	  if (p->p_flag & P_TRACED) {
-		if (p->p_stat != SSTOP)
-			goto run;
-		else
-			goto psigout;
-	}
-run:
-	/*
-	 * If we're being traced (possibly because someone attached us
-	 * while we were stopped), check for a signal from the debugger.
-	 */
-	if (p->p_stat == SSTOP) {
-		if ((p->p_flag & P_TRACED) != 0 && p->p_xstat != 0) {
-			uth->uu_siglist |= sigmask(p->p_xstat); 
-			p->p_siglist |= mask;   /* just for lame ones looking here */
-		}
-	}
-
-	/*
-	 * setrunnable(p) in BSD
-	 */
-	p->p_stat = SRUN;
-
-psigout:
-	signal_unlock(p);
-}
-
-static thread_t
-get_signalthread(struct proc *p, int signum)
-{
-	struct uthread *uth;
-	thread_t thr_act;
-	sigset_t mask = sigmask(signum);
-	thread_t sig_thread_act;
-	struct task * sig_task = p->task;
-	kern_return_t kret;
-	
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
-		sig_thread_act = p->p_vforkact;	
-		kret = check_actforsig(sig_task, sig_thread_act, 1);
-		if (kret == KERN_SUCCESS) 
-			return(sig_thread_act);
-		else
-			return(THREAD_NULL);
-	} 
-
-	TAILQ_FOREACH(uth, &p->p_uthlist, uu_list) {
-		if(((uth->uu_flag & UT_NO_SIGMASK)== 0) && 
-			(((uth->uu_sigmask & mask) == 0) || (uth->uu_sigwait & mask))) {
-			if (check_actforsig(p->task, uth->uu_act, 1) == KERN_SUCCESS)
-				return(uth->uu_act);
-		}
-	}
-	if (get_signalact(p->task, &thr_act, 1) == KERN_SUCCESS) {
-		return(thr_act);
-	}
-
-	return(THREAD_NULL);
-}
-
-/*
- * Send the signal to the process.  If the signal has an action, the action
- * is usually performed by the target process rather than the caller; we add
- * the signal to the set of pending signals for the process.
- *
- * Exceptions:
- *   o When a stop signal is sent to a sleeping process that takes the
- *     default action, the process is stopped without awakening it.
- *   o SIGCONT restarts stopped processes (or puts them back to sleep)
- *     regardless of the signal action (eg, blocked or ignored).
- *
- * Other ignored signals are discarded immediately.
- */
-void
-psignal_lock(p, signum, withlock)
-	register struct proc *p;
-	register int signum;
-	register int withlock;
-{
-	register int prop;
-	register sig_t action;
-	thread_t	sig_thread_act;
-	register task_t		sig_task;
-	int mask;
-	struct uthread *uth;
-	boolean_t funnel_state = FALSE;
-	int sw_funnel = 0;
-
-	if ((u_int)signum >= NSIG || signum == 0)
-		panic("psignal signal number");
-	mask = sigmask(signum);
-	prop = sigprop[signum];
-
-#if SIGNAL_DEBUG
-        if(rdebug_proc && (p == rdebug_proc)) {
-                ram_printf(3);
-        }
-#endif /* SIGNAL_DEBUG */
-
-	if (thread_funnel_get() == (funnel_t *)0) {
-		sw_funnel = 1;
-		funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	}
 	/*
 	 *	We will need the task pointer later.  Grab it now to
 	 *	check for a zombie process.  Also don't send signals
 	 *	to kernel internal tasks.
 	 */
-	if (((sig_task = p->task) == TASK_NULL)  || is_kerneltask(sig_task)) {
-		if (sw_funnel)
-			thread_funnel_set(kernel_flock, funnel_state);
+	if (flavor & PSIG_VFORK) {
+		sig_task = task;
+		sig_thread = thread;
+		sig_proc= p;
+	} else if (flavor & PSIG_THREAD) {
+		sig_task = get_threadtask(thread);
+		sig_thread = thread;
+		sig_proc = (proc_t)get_bsdtask_info(sig_task);
+	} else {
+		sig_task = p->task;
+		sig_proc = p;
+		sig_thread = (struct thread *)0;
+	}
+	if (((sig_task == TASK_NULL)  || is_kerneltask(sig_task))) {
 		return;
 	}
 
-        KNOTE(&p->p_klist, NOTE_SIGNAL | signum);
-
 	/*
 	 * do not send signals to the process that has the thread
 	 * doing a reboot(). Not doing so will mark that thread aborted
 	 * and can cause IO failures wich will cause data loss.
 	 */
-	if (ISSET(p->p_flag, P_REBOOT)) {
-		if (sw_funnel)
-			thread_funnel_set(kernel_flock, funnel_state);
+	if (ISSET(sig_proc->p_flag, P_REBOOT)) {
 		return;
 	}
 
-	if (withlock)
-		signal_lock(p);
+	if( (flavor & (PSIG_VFORK | PSIG_THREAD)) == 0) {
+		proc_knote(sig_proc, NOTE_SIGNAL | signum);
+	}
+
+
+	if ((flavor & PSIG_LOCKED)== 0)
+		proc_signalstart(sig_proc, 0);
 
 	/*
 	 *	Deliver the signal to the first thread in the task. This
@@ -1556,82 +1590,107 @@ psignal_lock(p, signum, withlock)
 	 *	reference is released by thread_deallocate.
 	 */
 	
-	if (((p->p_flag & P_TRACED) == 0) && (p->p_sigignore & mask))
-		goto psigout;
 
-	/* If successful return with ast set */
-	sig_thread_act = get_signalthread(p, signum);
+	if (((flavor & PSIG_VFORK) == 0) && ((sig_proc->p_lflag & P_LTRACED) == 0) && (sig_proc->p_sigignore & mask)) {
+		DTRACE_PROC3(signal__discard, thread_t, sig_thread, proc_t, sig_proc, int, signum);
+		goto psigout;
+	}
 
-	if (sig_thread_act == THREAD_NULL) {
-		/* XXXX FIXME
-		 * if it is sigkill, may be we should
-	 	 * inject a thread to terminate
-	 	 */
+	if (flavor & PSIG_VFORK) {
+		action = SIG_DFL;
+		act_set_astbsd(sig_thread);
+		kret = KERN_SUCCESS;
+	} else if (flavor & PSIG_THREAD) {
+		/* If successful return with ast set */
+		kret = check_actforsig(sig_task, sig_thread, 1);
+	} else {
+		/* If successful return with ast set */
+		kret = get_signalthread(sig_proc, signum, &sig_thread);
+	}
+	if (kret != KERN_SUCCESS) {
 #if SIGNAL_DEBUG
        		ram_printf(1);
 #endif /* SIGNAL_DEBUG */
 		goto psigout;
 	}
 
-	uth = get_bsdthread_info(sig_thread_act);
+
+	uth = get_bsdthread_info(sig_thread);
 
 	/*
 	 * If proc is traced, always give parent a chance.
 	 */
-	if (p->p_flag & P_TRACED)
-		action = SIG_DFL;
-	else {
-		/*
-		 * If the signal is being ignored,
-		 * then we forget about it immediately.
-		 * (Note: we don't set SIGCONT in p_sigignore,
-		 * and if it is set to SIG_IGN,
-		 * action will be SIG_DFL here.)
-		 */
-		if (p->p_sigignore & mask)
-			goto psigout;
-		/* sigwait takes precedence */
-		if (uth->uu_sigwait & mask)
-			action = KERN_SIG_WAIT;
-		else if (uth->uu_sigmask & mask)
-			action = KERN_SIG_HOLD;
-		else if (p->p_sigcatch & mask)
-			action = KERN_SIG_CATCH;
-		else
+
+	if ((flavor & PSIG_VFORK) == 0) {
+		if (sig_proc->p_lflag & P_LTRACED)
 			action = SIG_DFL;
+		else {
+			/*
+			 * If the signal is being ignored,
+			 * then we forget about it immediately.
+			 * (Note: we don't set SIGCONT in p_sigignore,
+			 * and if it is set to SIG_IGN,
+			 * action will be SIG_DFL here.)
+			 */
+			if (sig_proc->p_sigignore & mask)
+				goto psigout;
+			if (uth->uu_sigwait & mask)
+				action = KERN_SIG_WAIT;
+			else if (uth->uu_sigmask & mask)
+				action = KERN_SIG_HOLD;
+			else if (sig_proc->p_sigcatch & mask)
+				action = KERN_SIG_CATCH;
+			else
+				action = SIG_DFL;
+		}
 	}
 
-	if (p->p_nice > NZERO && action == SIG_DFL && (prop & SA_KILL) &&
-		(p->p_flag & P_TRACED) == 0)
-		p->p_nice = NZERO;
 
-	if (prop & SA_CONT) {
+	proc_lock(sig_proc);
+
+	if (sig_proc->p_nice > NZERO && action == SIG_DFL && (prop & SA_KILL) &&
+		(sig_proc->p_lflag & P_LTRACED) == 0)
+			sig_proc->p_nice = NZERO;
+
+	if (prop & SA_CONT)
 		uth->uu_siglist &= ~stopsigmask;
-		p->p_siglist &= ~stopsigmask;
-	}
 
 	if (prop & SA_STOP) {
+		struct pgrp *pg;
 		/*
 		 * If sending a tty stop signal to a member of an orphaned
 		 * process group, discard the signal here if the action
 		 * is default; don't stop the process below if sleeping,
 		 * and don't clear any pending SIGCONT.
 		 */
-		if (prop & SA_TTYSTOP && p->p_pgrp->pg_jobc == 0 &&
-			action == SIG_DFL)
+		proc_unlock(sig_proc);
+		pg = proc_pgrp(sig_proc);
+		if (prop & SA_TTYSTOP && pg->pg_jobc == 0 &&
+			action == SIG_DFL) {
+			pg_rele(pg);
 			goto psigout;
+		}
+		pg_rele(pg);
+		proc_lock(sig_proc);
 		uth->uu_siglist &= ~contsigmask;
-		p->p_siglist &= ~contsigmask;
 	}
+
 	uth->uu_siglist |= mask;
-	p->p_siglist |= mask;   /* just for lame ones looking here */
+	/* 
+	 * Repost AST incase sigthread has processed 
+	 * ast and missed signal post.
+	 */
+	if (action == KERN_SIG_CATCH)
+		act_set_astbsd(sig_thread);
 
 	
 	/*
 	 * Defer further processing for signals which are held,
 	 * except that stopped processes must be continued by SIGCONT.
 	 */
-	if (action == KERN_SIG_HOLD && ((prop & SA_CONT) == 0 || p->p_stat != SSTOP)) {
+	/* vfork will not go thru as action is SIG_DFL */
+	if ((action == KERN_SIG_HOLD) && ((prop & SA_CONT) == 0 || sig_proc->p_stat != SSTOP)) {
+		proc_unlock(sig_proc);
 		goto psigout;
 	}
 	/*
@@ -1640,8 +1699,8 @@ psignal_lock(p, signum, withlock)
 	 *	below if we didn't care about priority for tracing
 	 *	as SIGKILL's action is always SIG_DFL.
 	 */
-	if ((signum == SIGKILL) && (p->p_nice > NZERO)) {
-		p->p_nice = NZERO;
+	if ((signum == SIGKILL) && (sig_proc->p_nice > NZERO)) {
+		sig_proc->p_nice = NZERO;
 	}
 
 	/*
@@ -1649,23 +1708,31 @@ psignal_lock(p, signum, withlock)
 	 *	stopped) so that it can discover the signal in
 	 *	issig() and stop for the parent.
 	 */
-	if (p->p_flag & P_TRACED) {
-	   	if (p->p_stat != SSTOP)
-			goto run;
-		else
+	if (sig_proc->p_lflag & P_LTRACED) {
+	   	if (sig_proc->p_stat != SSTOP)
+			goto runlocked;
+		else {
+			proc_unlock(sig_proc);
 			goto psigout;
+		}
 	}
+	if ((flavor & PSIG_VFORK) != 0)
+		goto runlocked;
 
 	if (action == KERN_SIG_WAIT) {
 		uth->uu_sigwait = mask;
 		uth->uu_siglist &= ~mask;
-		p->p_siglist &= ~mask;
 		wakeup(&uth->uu_sigwait);
 		/* if it is SIGCONT resume whole process */
 		if (prop & SA_CONT) {
-			p->p_flag |= P_CONTINUED;
+			OSBitOrAtomic(P_CONTINUED, (UInt32 *)&sig_proc->p_flag);
+			sig_proc->p_contproc = current_proc()->p_pid;
+
+			proc_unlock(sig_proc);
 			(void) task_resume(sig_task);
+			goto psigout;
 		}
+		proc_unlock(sig_proc);
 		goto psigout;
 	}
 
@@ -1676,307 +1743,39 @@ psignal_lock(p, signum, withlock)
 		 *	(except for SIGCONT).
 		 */
 		if (prop & SA_CONT) {
-			if (p->p_flag & P_TTYSLEEP) {
-				p->p_flag &= ~P_TTYSLEEP;
-				wakeup(&p->p_siglist);
-			} else {
-				p->p_flag |= P_CONTINUED;
-				(void) task_resume(sig_task);
-			}
-			p->p_stat = SRUN;
-		} else if (p->p_stat == SSTOP)
-			goto psigout;
-		goto run;
-	} else {
-		/*	Default action - varies */
-		if (mask & stopsigmask) {
-			/*
-			 * These are the signals which by default
-			 * stop a process.
-			 *
-			 * Don't clog system with children of init
-			 * stopped from the keyboard.
-			 */
-			if (!(prop & SA_STOP) && p->p_pptr == initproc) {
-				psignal_lock(p, SIGKILL, 0);
-				uth->uu_siglist &= ~mask;
-				p->p_siglist &= ~mask;
-				goto psigout;
-			}
-                        
-			/*
-			 *	Stop the task
-			 *	if task hasn't already been stopped by
-			 *	a signal.
-			 */
-			uth->uu_siglist &= ~mask;
-			p->p_siglist &= ~mask;
-			if (p->p_stat != SSTOP) {
-				p->p_xstat = signum;
-				stop(p);
-				if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0) {
-					struct proc *pp = p->p_pptr;
-
-					pp->si_pid = p->p_pid;
-					pp->si_status = p->p_xstat;
-					pp->si_code = CLD_STOPPED;
-					pp->si_uid = p->p_ucred->cr_ruid;
-					psignal(pp, SIGCHLD);
-				}
-			}
-			goto psigout;
-		}
-
-		switch (signum) {
-			/*
-			 * Signals ignored by default have been dealt
-			 * with already, since their bits are on in
-			 * p_sigignore.
-			 */
-
-		case SIGKILL:
-			/*
-			 * Kill signal always sets process running and
-			 * unsuspends it.
-			 */
-			/*
-			 *	Process will be running after 'run'
-			 */
-			p->p_stat = SRUN;
-
-			thread_abort(sig_thread_act);
-
-			goto psigout;
-
-		case SIGCONT:
-			/*
-			 * Let the process run.  If it's sleeping on an
-			 * event, it remains so.
-			 */
-			if (p->p_flag & P_TTYSLEEP) {
-				p->p_flag &= ~P_TTYSLEEP;
-				wakeup(&p->p_siglist);
-			} else {
-				p->p_flag |= P_CONTINUED;
-				(void) task_resume(sig_task);
-			}
-			uth->uu_siglist &= ~mask;
-			p->p_siglist &= ~mask;
-			p->p_stat = SRUN;
-
+			OSBitOrAtomic(P_CONTINUED, (UInt32 *)&sig_proc->p_flag);
+			proc_unlock(sig_proc);
+			(void) task_resume(sig_task);
+			proc_lock(sig_proc);
+			sig_proc->p_stat = SRUN;
+		}  else if (sig_proc->p_stat == SSTOP) {
+			proc_unlock(sig_proc);
 			goto psigout;
-
-		default:
-			/*
-			 * All other signals wake up the process, but don't
-			 * resume it.
-			 */
-			if (p->p_stat == SSTOP)
-				goto psigout;
-			goto run;
 		}
-	}
-	/*NOTREACHED*/
-run:
-	/*
-	 * If we're being traced (possibly because someone attached us
-	 * while we were stopped), check for a signal from the debugger.
-	 */
-	if (p->p_stat == SSTOP) {
-		if ((p->p_flag & P_TRACED) != 0 && p->p_xstat != 0)
-			uth->uu_siglist |= sigmask(p->p_xstat); 
-	} else {
 		/*
-	 	 * setrunnable(p) in BSD and
-	 	 * Wake up the thread if it is interruptible.
-	 	 */
-		p->p_stat = SRUN;
-		thread_abort_safely(sig_thread_act);
-	}
-psigout:
-	if (withlock) 
-		signal_unlock(p);
-	if (sw_funnel)
-			thread_funnel_set(kernel_flock, funnel_state);
-}
-
-
-/* psignal_lock(p, signum, withlock ) */
-void
-psignal_uthread(thr_act, signum)
-	thread_t thr_act;
-	int signum;
-{
-	struct proc *p;
-	register int prop;
-	register sig_t action;
-	thread_t	sig_thread_act;
-	register task_t		sig_task;
-	int mask;
-	struct uthread *uth;
-	kern_return_t kret;
-	int error = 0;
-
-	p = (struct proc *)get_bsdtask_info(get_threadtask(thr_act));
-	if ((u_int)signum >= NSIG || signum == 0)
-		panic("Invalid signal number in psignal_uthread"); 
-	mask = sigmask(signum);
-	prop = sigprop[signum];
-
-#if SIGNAL_DEBUG
-        if(rdebug_proc && (p == rdebug_proc)) {
-                ram_printf(3);
-        }
-#endif /* SIGNAL_DEBUG */
-
-	/*
-	 *	We will need the task pointer later.  Grab it now to
-	 *	check for a zombie process.  Also don't send signals
-	 *	to kernel internal tasks.
-	 */
-	if (((sig_task = p->task) == TASK_NULL)  || is_kerneltask(sig_task)) {
-		return;
-	}
-
-	sig_thread_act = thr_act;
-	/*
-	 * do not send signals to the process that has the thread
-	 * doing a reboot(). Not doing so will mark that thread aborted
-	 * and can cause IO failures wich will cause data loss.
-	 */
-	if (ISSET(p->p_flag, P_REBOOT)) {
-		return;
-	}
-
-	signal_lock(p);
-
-	/*
-	 *	Deliver the signal to the first thread in the task. This
-	 *	allows single threaded applications which use signals to
-	 *	be able to be linked with multithreaded libraries.  We have
-	 *	an implicit reference to the current thread, but need
-	 *	an explicit one otherwise.  The thread reference keeps
-	 *	the corresponding task data structures around too.  This
-	 *	reference is released by thread_deallocate.
-	 */
-	
-	if (((p->p_flag & P_TRACED) == 0) && (p->p_sigignore & mask))
-		goto puthout;
-
-	kret = check_actforsig(sig_task, sig_thread_act, 1);
-
-	if (kret != KERN_SUCCESS) {
-		error = EINVAL;
-		goto puthout;
-	}
-
-
-	uth = get_bsdthread_info(sig_thread_act);
-
-	/*
-	 * If proc is traced, always give parent a chance.
-	 */
-	if (p->p_flag & P_TRACED)
-		action = SIG_DFL;
-	else {
-		/*
-		 * If the signal is being ignored,
-		 * then we forget about it immediately.
-		 * (Note: we don't set SIGCONT in p_sigignore,
-		 * and if it is set to SIG_IGN,
-		 * action will be SIG_DFL here.)
-		 */
-		if (p->p_sigignore & mask)
-			goto puthout;
-		/* sigwait takes precedence */
-		if (uth->uu_sigwait & mask)
-			action = KERN_SIG_WAIT;
-		else if (uth->uu_sigmask & mask)
-			action = KERN_SIG_HOLD;
-		else if (p->p_sigcatch & mask)
-			action = KERN_SIG_CATCH;
-		else
-			action = SIG_DFL;
-	}
-
-	if (p->p_nice > NZERO && action == SIG_DFL && (prop & SA_KILL) &&
-		(p->p_flag & P_TRACED) == 0)
-		p->p_nice = NZERO;
-
-	if (prop & SA_CONT) {
-		uth->uu_siglist &= ~stopsigmask;
-		p->p_siglist &= ~stopsigmask;
-	}
-
-	if (prop & SA_STOP) {
-		/*
-		 * If sending a tty stop signal to a member of an orphaned
-		 * process group, discard the signal here if the action
-		 * is default; don't stop the process below if sleeping,
-		 * and don't clear any pending SIGCONT.
+		 * Fill out siginfo structure information to pass to the
+		 * signalled process/thread sigaction handler, when it
+		 * wakes up.  si_code is 0 because this is an ordinary
+		 * signal, not a SIGCHLD, and so si_status is the signal
+		 * number itself, instead of the child process exit status.
+		 * We shift this left because it will be shifted right before
+		 * it is passed to user space.  kind of ugly to use W_EXITCODE
+		 * this way, but it beats defining a new macro.
+		 *
+		 * Note:	Avoid the SIGCHLD recursion case!
 		 */
-		if (prop & SA_TTYSTOP && p->p_pgrp->pg_jobc == 0 &&
-			action == SIG_DFL)
-			goto puthout;
-		uth->uu_siglist &= ~contsigmask;
-		p->p_siglist &= ~contsigmask;
-	}
-	uth->uu_siglist |= mask;
-	p->p_siglist |= mask;   /* just for lame ones looking here */
-
-	/*
-	 * Defer further processing for signals which are held,
-	 * except that stopped processes must be continued by SIGCONT.
-	 */
-	if (action == KERN_SIG_HOLD && ((prop & SA_CONT) == 0 || p->p_stat != SSTOP))
-		goto puthout;
-		
-	/*
-	 *	SIGKILL priority twiddling moved here from above because
-	 *	it needs sig_thread.  Could merge it into large switch
-	 *	below if we didn't care about priority for tracing
-	 *	as SIGKILL's action is always SIG_DFL.
-	 */
-	if ((signum == SIGKILL) && (p->p_nice > NZERO)) {
-		p->p_nice = NZERO;
-	}
-
-	/*
-	 *	Process is traced - wake it up (if not already
-	 *	stopped) so that it can discover the signal in
-	 *	issig() and stop for the parent.
-	 */
-	if (p->p_flag & P_TRACED) {
-	   	if (p->p_stat != SSTOP)
-			goto psurun;
-		else
-			goto puthout;
-	}
-
-	if (action == KERN_SIG_WAIT) {
-		uth->uu_sigwait = mask;
-		uth->uu_siglist &= ~mask;
-		p->p_siglist &= ~mask;
-		wakeup(&uth->uu_sigwait);
-		/* if it is SIGCONT resume whole process */
-		if (prop & SA_CONT) {
-			p->p_flag |= P_CONTINUED;
-			(void) task_resume(sig_task);
+		if (signum != SIGCHLD) {
+			proc_unlock(sig_proc);
+			r_uid = kauth_getruid();
+			proc_lock(sig_proc);
+
+			sig_proc->si_pid = current_proc()->p_pid;
+			sig_proc->si_status = W_EXITCODE(signum, 0);
+			sig_proc->si_uid = r_uid;
+			sig_proc->si_code = 0;
 		}
-		goto puthout;
-	}
 
-	if (action != SIG_DFL) {
-		/*
-		 *	User wants to catch the signal.
-		 *	Wake up the thread, but don't un-suspend it
-		 *	(except for SIGCONT).
-		 */
-		if (prop & SA_CONT) {
-			p->p_flag |= P_CONTINUED;
-			(void) task_resume(sig_task);
-		}
-		goto psurun;
+		goto runlocked;
 	} else {
 		/*	Default action - varies */
 		if (mask & stopsigmask) {
@@ -1987,11 +1786,13 @@ psignal_uthread(thr_act, signum)
 			 * Don't clog system with children of init
 			 * stopped from the keyboard.
 			 */
-			if (!(prop & SA_STOP) && p->p_pptr == initproc) {
-				psignal_lock(p, SIGKILL, 0);
+			if (!(prop & SA_STOP) && sig_proc->p_pptr == initproc) {
+				proc_unlock(sig_proc);
+				psignal_locked(sig_proc, SIGKILL);
+				proc_lock(sig_proc);
 				uth->uu_siglist &= ~mask;
-				p->p_siglist &= ~mask;
-				goto puthout;
+				proc_unlock(sig_proc);
+				goto psigout;
 			}
                         
 			/*
@@ -1999,24 +1800,52 @@ psignal_uthread(thr_act, signum)
 			 *	if task hasn't already been stopped by
 			 *	a signal.
 			 */
-			uth->uu_siglist &= ~mask;
-			p->p_siglist &= ~mask;
-			if (p->p_stat != SSTOP) {
-				p->p_xstat = signum;
-				if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0) {
-					struct proc *pp = p->p_pptr;
-
-					pp->si_pid = p->p_pid;
-					pp->si_status = p->p_xstat;
+			uth->uu_siglist &= ~mask;
+			if (sig_proc->p_stat != SSTOP) {
+				sig_proc->p_xstat = signum;
+				sig_proc->p_stat = SSTOP;
+				OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&sig_proc->p_flag);
+				sig_proc->p_lflag &= ~P_LWAITED;
+				proc_unlock(sig_proc);
+
+				pp = proc_parentholdref(sig_proc);
+				stop(sig_proc, pp);
+				if (( pp != PROC_NULL) && ((pp->p_flag & P_NOCLDSTOP) == 0)) {
+
+					my_cred = kauth_cred_proc_ref(sig_proc);
+					r_uid = my_cred->cr_ruid;
+					kauth_cred_unref(&my_cred);
+
+					proc_lock(sig_proc);
+					pp->si_pid = sig_proc->p_pid;
+					/*
+					 * POSIX: sigaction for a stopped child
+					 * when sent to the parent must set the
+					 * child's signal number into si_status.
+					 */
+					if (signum != SIGSTOP)
+						pp->si_status = WEXITSTATUS(sig_proc->p_xstat);
+					else
+						pp->si_status = W_EXITCODE(signum, signum);
 					pp->si_code = CLD_STOPPED;
-					pp->si_uid = p->p_ucred->cr_ruid;
+					pp->si_uid = r_uid;
+					proc_unlock(sig_proc);
+
 					psignal(pp, SIGCHLD);
 				}
-				stop(p);
-			}
-			goto puthout;
+				if (pp != PROC_NULL)
+					proc_parentdropref(pp, 0);
+			} else
+				proc_unlock(sig_proc);
+			goto psigout;
 		}
 
+		DTRACE_PROC3(signal__send, thread_t, sig_thread, proc_t, p, int, signum);
+
+		/*
+		 * enters switch with sig_proc lock held but dropped when
+		 * gets out of switch
+		 */
 		switch (signum) {
 			/*
 			 * Signals ignored by default have been dealt
@@ -2032,103 +1861,123 @@ psignal_uthread(thr_act, signum)
 			/*
 			 *	Process will be running after 'run'
 			 */
-			p->p_stat = SRUN;
+			sig_proc->p_stat = SRUN;
+			proc_unlock(sig_proc);
+			thread_abort(sig_thread);
 
-			thread_abort(sig_thread_act);
-
-			goto puthout;
+			goto psigout;
 
 		case SIGCONT:
 			/*
 			 * Let the process run.  If it's sleeping on an
 			 * event, it remains so.
 			 */
-			if (p->p_flag & P_TTYSLEEP) {
-				p->p_flag &= ~P_TTYSLEEP;
-				wakeup(&p->p_siglist);
-			} else {
-				p->p_flag |= P_CONTINUED;
-				(void) task_resume(sig_task);
+			OSBitOrAtomic(P_CONTINUED, (UInt32 *)&sig_proc->p_flag);
+			sig_proc->p_contproc = sig_proc->p_pid;
+
+			proc_unlock(sig_proc);
+			(void) task_resume(sig_task);
+			proc_lock(sig_proc);
+			/*
+			 * When processing a SIGCONT, we need to check
+			 * to see if there are signals pending that
+			 * were not delivered because we had been
+			 * previously stopped.  If that's the case,
+			 * we need to thread_abort_safely() to trigger
+			 * interruption of the current system call to
+			 * cause their handlers to fire.  If it's only
+			 * the SIGCONT, then don't wake up.
+			 */
+			if (((flavor & (PSIG_VFORK|PSIG_THREAD)) == 0) && (((uth->uu_siglist & ~uth->uu_sigmask) & ~sig_proc->p_sigignore) & ~mask)) {
+				uth->uu_siglist &= ~mask;
+				sig_proc->p_stat = SRUN;
+				goto runlocked;
 			}
+
 			uth->uu_siglist &= ~mask;
-			p->p_siglist &= ~mask;
-			p->p_stat = SRUN;
-			goto puthout;
+			sig_proc->p_stat = SRUN;
+			proc_unlock(sig_proc);
+			goto psigout;
 
 		default:
+			/*
+			 * A signal which has a default action of killing
+			 * the process, and for which there is no handler,
+			 * needs to act like SIGKILL
+			 */
+			if (((flavor & (PSIG_VFORK|PSIG_THREAD)) == 0) && (action == SIG_DFL) && (prop & SA_KILL)) {
+				sig_proc->p_stat = SRUN;
+				proc_unlock(sig_proc);
+				thread_abort(sig_thread);
+				goto psigout;
+			}
+
 			/*
 			 * All other signals wake up the process, but don't
 			 * resume it.
 			 */
-			goto psurun;
+			if (sig_proc->p_stat == SSTOP) {
+				proc_unlock(sig_proc);
+				goto psigout;
+			}
+			goto runlocked;
 		}
 	}
 	/*NOTREACHED*/
-psurun:
+
+runlocked:
 	/*
 	 * If we're being traced (possibly because someone attached us
 	 * while we were stopped), check for a signal from the debugger.
 	 */
-	if (p->p_stat == SSTOP) {
-		if ((p->p_flag & P_TRACED) != 0 && p->p_xstat != 0) {
-			uth->uu_siglist |= sigmask(p->p_xstat); 
-			p->p_siglist |= sigmask(p->p_xstat); 
+	if (sig_proc->p_stat == SSTOP) {
+		if ((sig_proc->p_lflag & P_LTRACED) != 0 && sig_proc->p_xstat != 0)
+			uth->uu_siglist |= sigmask(sig_proc->p_xstat); 
+		if ((flavor & PSIG_VFORK) != 0) {
+			sig_proc->p_stat = SRUN;
 		}
+		proc_unlock(sig_proc);	
 	} else {
 		/*
 	 	 * setrunnable(p) in BSD and
 	 	 * Wake up the thread if it is interruptible.
 	 	 */
-		p->p_stat = SRUN;
-		thread_abort_safely(sig_thread_act);
+		sig_proc->p_stat = SRUN;
+		proc_unlock(sig_proc);	
+		if ((flavor & PSIG_VFORK) == 0)
+			thread_abort_safely(sig_thread);
+	}
+psigout:
+	if ((flavor & PSIG_LOCKED)== 0) {
+		proc_signalend(sig_proc, 0);
 	}
-
-puthout:
-		signal_unlock(p);
 }
 
-
-__inline__ void
-sig_lock_to_exit(struct proc *p)
+void
+psignal(proc_t p, int signum)
 {
-	thread_t	self = current_thread();
+	psignal_internal(p, NULL, NULL, 0, signum);
+}
 
-	p->exit_thread = self;
-	(void) task_suspend(p->task);
+void
+psignal_locked(proc_t p, int signum)
+{
+	psignal_internal(p, NULL, NULL, PSIG_LOCKED, signum);
 }
 
-__inline__ int
-sig_try_locked(struct proc *p)
+void
+psignal_vfork(proc_t p, task_t new_task, thread_t thread, int signum)
 {
-	thread_t	self = current_thread();
+	psignal_internal(p, new_task, thread, PSIG_VFORK, signum);
+}
 
-	while (p->sigwait || p->exit_thread) {
-		if (p->exit_thread) {
-			if (p->exit_thread != self) {
-				/*
-				 * Already exiting - no signals.
-				 */
-				thread_abort(self);
-			}
-			return(0);
-		}
-		if(assert_wait_possible()) {
-			assert_wait((caddr_t)&p->sigwait_thread, 
-					(THREAD_INTERRUPTIBLE));
-		}
-		signal_unlock(p);
-		thread_block(THREAD_CONTINUE_NULL);
-		signal_lock(p);
-		if (thread_should_abort(self)) {
-			/*
-			 * Terminate request - clean up.
-			 */
-			return -1;
-		}
-	}
-	return 1;
+static void
+psignal_uthread(thread_t thread, int signum)
+{
+	psignal_internal(PROC_NULL, TASK_NULL, thread, PSIG_THREAD, signum);
 }
 
+
 /*
  * If the current process has received a signal (should be caught or cause
  * termination, should interrupt current syscall), return the signal number.
@@ -2142,13 +1991,15 @@ sig_try_locked(struct proc *p)
  *		postsig(signum);
  */
 int
-issignal(p)
-	register struct proc *p;
+issignal(proc_t p)
 {
-	register int signum, mask, prop, sigbits;
+	int signum, mask, prop, sigbits;
 	thread_t cur_act;
 	struct uthread * ut;
-	struct proc *pp;
+	proc_t pp;
+	kauth_cred_t my_cred;
+	int retval = 0;
+	uid_t r_uid;
 
 	cur_act = current_thread();
 
@@ -2157,65 +2008,80 @@ issignal(p)
                 ram_printf(3);
         }
 #endif /* SIGNAL_DEBUG */
-	signal_lock(p);
+	proc_lock(p);
 
 	/*
 	 * Try to grab the signal lock.
 	 */
 	if (sig_try_locked(p) <= 0) {
-		signal_unlock(p);
-		return (0);
+		proc_unlock(p);
+		return(0);
 	}
 
+	proc_signalstart(p, 1);
+
 	ut = get_bsdthread_info(cur_act);
 	for(;;) {
 		sigbits = ut->uu_siglist  & ~ut->uu_sigmask;
 
-		if (p->p_flag & P_PPWAIT)
+		if (p->p_lflag & P_LPPWAIT)
 			sigbits &= ~stopsigmask;
 		if (sigbits == 0) {	 	/* no signal to send */
-			signal_unlock(p);
-			return (0);
+			retval = 0;
+			goto out;
 		}
+
 		signum = ffs((long)sigbits);
 		mask = sigmask(signum);
 		prop = sigprop[signum];
 
 		/*
 		 * We should see pending but ignored signals
-		 * only if P_TRACED was on when they were posted.
+		 * only if P_LTRACED was on when they were posted.
 		 */
-		if (mask & p->p_sigignore && (p->p_flag & P_TRACED) == 0) {
+		if (mask & p->p_sigignore && (p->p_lflag & P_LTRACED) == 0) {
 			ut->uu_siglist &= ~mask;		/* take the signal! */
-			p->p_siglist &= ~mask;		/* take the signal! */
 			continue;
 		}
-		if (p->p_flag & P_TRACED && (p->p_flag & P_PPWAIT) == 0)  {
-			register task_t	task;
+		if (p->p_lflag & P_LTRACED && (p->p_lflag & P_LPPWAIT) == 0)  {
+			task_t	task;
 			/*
 			 * If traced, always stop, and stay
 			 * stopped until released by the debugger.
 			 */
 			/* ptrace debugging */
 			p->p_xstat = signum;
-			pp = p->p_pptr;
-			if (p->p_flag & P_SIGEXC) {
+	
+			if (p->p_lflag & P_LSIGEXC) {
 				p->sigwait = TRUE;
 				p->sigwait_thread = cur_act;
 				p->p_stat = SSTOP;
-				p->p_flag &= ~(P_WAITED|P_CONTINUED);
+				OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&p->p_flag);
+				p->p_lflag &= ~P_LWAITED;
 				ut->uu_siglist &= ~mask;	/* clear the old signal */
-				p->p_siglist &= ~mask;	/* clear the old signal */
-				signal_unlock(p);
+				proc_signalend(p, 1);
+				proc_unlock(p);
 				do_bsdexception(EXC_SOFTWARE, EXC_SOFT_SIGNAL, signum);
-				signal_lock(p);
+				proc_lock(p);
+				proc_signalstart(p, 1);
 			} else {
-//				panic("Unsupportef gdb option \n");;
-				pp->si_pid = p->p_pid;
-				pp->si_status = p->p_xstat;
-				pp->si_code = CLD_TRAPPED;
-				pp->si_uid = p->p_ucred->cr_ruid;
-				psignal(pp, SIGCHLD);
+				proc_unlock(p);
+				my_cred = kauth_cred_proc_ref(p);
+				r_uid = my_cred->cr_ruid;
+				kauth_cred_unref(&my_cred);
+
+				pp = proc_parentholdref(p);
+				if (pp != PROC_NULL) {
+					proc_lock(pp);
+
+					pp->si_pid = p->p_pid;
+					pp->si_status = p->p_xstat;
+					pp->si_code = CLD_TRAPPED;
+					pp->si_uid = r_uid;
+
+					proc_unlock(pp);
+				}
+
 				/*
 			 	*	XXX Have to really stop for debuggers;
 			 	*	XXX stop() doesn't do the right thing.
@@ -2224,19 +2090,31 @@ issignal(p)
 			 	*	XXX middle of it.
 			 	*/
 				task = p->task;
-				task_hold(task);
+				task_suspend(task);
+
+				proc_lock(p);
 				p->sigwait = TRUE;
 				p->sigwait_thread = cur_act;
 				p->p_stat = SSTOP;
-				p->p_flag &= ~(P_WAITED|P_CONTINUED);
+				OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&p->p_flag);
+				p->p_lflag &= ~P_LWAITED;
 				ut->uu_siglist &= ~mask;	/* clear the old signal */
-				p->p_siglist &= ~mask;	/* clear the old signal */
 
-				wakeup((caddr_t)p->p_pptr);
-				signal_unlock(p);
+				proc_signalend(p, 1);
+				proc_unlock(p);
+
+				if (pp != PROC_NULL) {
+					psignal(pp, SIGCHLD);
+					proc_list_lock();
+					wakeup((caddr_t)pp);
+					proc_parentdropref(pp, 1);
+					proc_list_unlock();
+				}
+
 				assert_wait((caddr_t)&p->sigwait, (THREAD_INTERRUPTIBLE));
 				thread_block(THREAD_CONTINUE_NULL);
-				signal_lock(p);
+				proc_lock(p);
+				proc_signalstart(p, 1);
 			}
 
 			p->sigwait = FALSE;
@@ -2264,8 +2142,9 @@ issignal(p)
 			 	* before calling exit().  (Since exit()
 			 	* calls closef() which can trash u_qsave.)
 			 	*/
-				signal_unlock(p);
-				exit1(p,signum, (int *)NULL);
+				proc_signalend(p, 1);
+				proc_unlock(p);
+				exit1(p, W_EXITCODE(0, SIGKILL), (int *)NULL);
 				return(0);
 			}
 
@@ -2273,8 +2152,8 @@ issignal(p)
 			 *	We may have to quit
 			 */
 			if (thread_should_abort(current_thread())) {
-				signal_unlock(p);
-				return(0);
+				retval = 0;
+				goto out;
 			}
 			/*
 			 * If parent wants us to take the signal,
@@ -2290,7 +2169,6 @@ issignal(p)
 			 */
 			mask = sigmask(signum);
 			ut->uu_siglist |= mask;
-			p->p_siglist |= mask;   /* just for lame ones looking here */
 			if (ut->uu_sigmask & mask)
 				continue;
 		}
@@ -2307,7 +2185,7 @@ issignal(p)
 			/*
 			 * Don't take default actions on system processes.
 			 */
-			if (p->p_pptr->p_pid == 0) {
+			if (p->p_ppid == 0) {
 #if DIAGNOSTIC
 				/*
 				 * Are you sure you want to ignore SIGSEGV
@@ -2327,22 +2205,46 @@ issignal(p)
 			 * process group, ignore tty stop signals.
 			 */
 			if (prop & SA_STOP) {
-				if (p->p_flag & P_TRACED ||
-					(p->p_pgrp->pg_jobc == 0 &&
-					prop & SA_TTYSTOP))
+				struct pgrp * pg;
+
+				proc_unlock(p);
+				pg = proc_pgrp(p);
+				if (p->p_lflag & P_LTRACED ||
+					(pg->pg_jobc == 0 &&
+					prop & SA_TTYSTOP)) {
+					proc_lock(p);
+					pg_rele(pg);
 					break;	/* == ignore */
+				}
+				pg_rele(pg);
 				if (p->p_stat != SSTOP) {
+					proc_lock(p);
 					p->p_xstat = signum;
-					stop(p);
-					if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0) {
-						pp = p->p_pptr;
+				
+					p->p_stat = SSTOP;
+					p->p_lflag &= ~P_LWAITED;
+					proc_unlock(p);
+
+					pp = proc_parentholdref(p);
+					stop(p, pp);
+					if ((pp != PROC_NULL) && ((pp->p_flag & P_NOCLDSTOP) == 0)) {
+						my_cred = kauth_cred_proc_ref(p);
+						r_uid = my_cred->cr_ruid;
+						kauth_cred_unref(&my_cred);
+
+						proc_lock(pp);
 						pp->si_pid = p->p_pid;
-						pp->si_status = p->p_xstat;
+						pp->si_status = WEXITSTATUS(p->p_xstat);
 						pp->si_code = CLD_STOPPED;
-						pp->si_uid = p->p_ucred->cr_ruid;
+						pp->si_uid = r_uid;
+						proc_unlock(pp);
+
 						psignal(pp, SIGCHLD);
 					}
+					if (pp != PROC_NULL)
+						proc_parentdropref(pp, 0);
 				}
+				proc_lock(p);
 				break;
 			} else if (prop & SA_IGNORE) {
 				/*
@@ -2352,11 +2254,12 @@ issignal(p)
 				break;		/* == ignore */
 			} else {
 				ut->uu_siglist &= ~mask;	/* take the signal! */
-				p->p_siglist &= ~mask;	/* take the signal! */
-				signal_unlock(p);
-				return (signum);
+				retval = signum;
+				goto out;
 			}
+
 			/*NOTREACHED*/
+			break;
 
 		case (long)SIG_IGN:
 			/*
@@ -2365,7 +2268,7 @@ issignal(p)
 			 * than SIGCONT, unless process is traced.
 			 */
 			if ((prop & SA_CONT) == 0 &&
-				(p->p_flag & P_TRACED) == 0)
+				(p->p_lflag & P_LTRACED) == 0)
 				printf("issignal\n");
 			break;		/* == ignore */
 
@@ -2375,22 +2278,23 @@ issignal(p)
 			 * postsig() process it.
 			 */
 			ut->uu_siglist &= ~mask;		/* take the signal! */
-			p->p_siglist &= ~mask;		/* take the signal! */
-			signal_unlock(p);
-			return (signum);
+			retval = signum;
+			goto out;
 		}
 		ut->uu_siglist &= ~mask;		/* take the signal! */
-		p->p_siglist &= ~mask;		/* take the signal! */
 		}
 	/* NOTREACHED */
+out:
+	proc_signalend(p,1);
+	proc_unlock(p);
+	return(retval);
 }
 
 /* called from _sleep */
 int
-CURSIG(p)
-    register struct proc *p;
+CURSIG(proc_t p)
 {
-	register int signum, mask, prop, sigbits;
+	int signum, mask, prop, sigbits;
 	thread_t cur_act;
 	struct uthread * ut;
 	int retnum = 0;
@@ -2403,13 +2307,13 @@ CURSIG(p)
 	if (ut->uu_siglist == 0)
 		return (0);
 
-	if (((ut->uu_siglist & ~ut->uu_sigmask) == 0) && ((p->p_flag & P_TRACED) == 0))
+	if (((ut->uu_siglist & ~ut->uu_sigmask) == 0) && ((p->p_lflag & P_LTRACED) == 0))
 		return (0);
 
 	sigbits = ut->uu_siglist & ~ut->uu_sigmask;
 
 	for(;;) {
-		if (p->p_flag & P_PPWAIT)
+		if (p->p_lflag & P_LPPWAIT)
 			sigbits &= ~stopsigmask;
 		if (sigbits == 0) {	 	/* no signal to send */
 			return (retnum);
@@ -2421,12 +2325,12 @@ CURSIG(p)
 
 		/*
 		 * We should see pending but ignored signals
-		 * only if P_TRACED was on when they were posted.
+		 * only if P_LTRACED was on when they were posted.
 		 */
-		if (mask & p->p_sigignore && (p->p_flag & P_TRACED) == 0) {
+		if (mask & p->p_sigignore && (p->p_lflag & P_LTRACED) == 0) {
 			continue;
 		}
-		if (p->p_flag & P_TRACED && (p->p_flag & P_PPWAIT) == 0) {
+		if (p->p_lflag & P_LTRACED && (p->p_lflag & P_LPPWAIT) == 0) {
 			/*
 			 * Put the new signal into p_siglist.  If the
 			 * signal is being masked, look for other signals.
@@ -2449,7 +2353,7 @@ CURSIG(p)
 			/*
 			 * Don't take default actions on system processes.
 			 */
-			if (p->p_pptr->p_pid == 0) {
+			if (p->p_ppid == 0) {
 #if DIAGNOSTIC
 				/*
 				 * Are you sure you want to ignore SIGSEGV
@@ -2469,10 +2373,17 @@ CURSIG(p)
 			 * process group, ignore tty stop signals.
 			 */
 			if (prop & SA_STOP) {
-				if (p->p_flag & P_TRACED ||
-					(p->p_pgrp->pg_jobc == 0 &&
-					prop & SA_TTYSTOP))
+				struct pgrp *pg;
+
+				pg = proc_pgrp(p);
+
+				if (p->p_lflag & P_LTRACED ||
+					(pg->pg_jobc == 0 &&
+					prop & SA_TTYSTOP)) {
+					pg_rele(pg);
 					break;	/* == ignore */
+				}
+				pg_rele(pg);
 				retnum = signum;
 				break;
 			} else if (prop & SA_IGNORE) {
@@ -2493,7 +2404,7 @@ CURSIG(p)
 			 * than SIGCONT, unless process is traced.
 			 */
 			if ((prop & SA_CONT) == 0 &&
-				(p->p_flag & P_TRACED) == 0)
+				(p->p_lflag & P_LTRACED) == 0)
 				printf("issignal\n");
 			break;		/* == ignore */
 
@@ -2514,14 +2425,15 @@ CURSIG(p)
  * via wakeup.  Signals are handled elsewhere.  The process must not be
  * on the run queue.
  */
-void
-stop(p)
-	register struct proc *p;
+static void
+stop(proc_t p, proc_t parent)
 {
-	p->p_stat = SSTOP;
-	p->p_flag &= ~(P_WAITED|P_CONTINUED);
-	if (p->p_pptr->p_stat != SSTOP)
-		wakeup((caddr_t)p->p_pptr);
+	OSBitAndAtomic(~((uint32_t)P_CONTINUED), (UInt32 *)&p->p_flag);
+	if ((parent != PROC_NULL) && (parent->p_stat != SSTOP)) {
+		proc_list_lock();
+		wakeup((caddr_t)parent);
+		proc_list_unlock();
+	}
 	(void) task_suspend(p->task);	/*XXX*/
 }
 
@@ -2532,7 +2444,7 @@ stop(p)
 void
 postsig(int signum)
 {
-	struct proc *p = current_proc();
+	proc_t p = current_proc();
 	struct sigacts *ps = p->p_sigacts;
 	user_addr_t catcher;
 	u_long code;
@@ -2549,36 +2461,52 @@ postsig(int signum)
 		panic("psig not on master");
 #endif
 
-	signal_lock(p);
+	proc_lock(p);
 	/*
 	 * Try to grab the signal lock.
 	 */
 	if (sig_try_locked(p) <= 0) {
-		signal_unlock(p);
+		proc_unlock(p);
 		return;
 	}
 
+	proc_signalstart(p, 1);
+
 	ut = (struct uthread *)get_bsdthread_info(current_thread());
 	mask = sigmask(signum);
 	ut->uu_siglist &= ~mask;
-	p->p_siglist &= ~mask;
 	catcher = ps->ps_sigact[signum];
-#if KTRACE
-	//LP64: catcher argument is a 64 bit user space handler address
-	if (KTRPOINT(p, KTR_PSIG))
-		ktrpsig(p->p_tracep,
-		    signum, CAST_DOWN(void *,catcher), ut->uu_flag & UT_SAS_OLDMASK ?
-		    &ut->uu_oldmask : &ut->uu_sigmask, 0);
-#endif
 	if (catcher == SIG_DFL) {
 		/*
 		 * Default catcher, where the default is to kill
 		 * the process.  (Other cases were ignored above.)
 		 */
-		/* called with signal_lock() held */
-		sigexit_locked(p, signum);
+		siginfo_t sinfo;
+		bzero((caddr_t)&sinfo, sizeof(siginfo_t));
+
+		sig_lock_to_exit(p);
+		p->p_acflag |= AXSIG;
+		if (sigprop[signum] & SA_CORE) {
+			p->p_sigacts->ps_sig = signum;
+			proc_signalend(p, 1);
+			proc_unlock(p);
+			if (coredump(p) == 0)
+				signum |= WCOREFLAG;
+		} else  {
+			proc_signalend(p, 1);
+			proc_unlock(p);
+		}
+		
+		sinfo.si_signo = signum;
+		sinfo.si_pid = p->si_pid;
+		sinfo.si_uid = p->si_uid;
+		sinfo.si_status = WEXITSTATUS(p->si_status);
+
+		DTRACE_PROC3(signal__handle, int, signum, siginfo_t *, &sinfo,
+					void (*)(void), SIG_DFL);
+
+		exit1(p, W_EXITCODE(0, signum), (int *)NULL);
 		return;
-		/* NOTREACHED */
 	} else {
 		/*
 		 * If we get here, the signal must be caught.
@@ -2588,6 +2516,7 @@ postsig(int signum)
 			log(LOG_WARNING,
 				"postsig: processing masked or ignored signal\n");
 #endif
+
 		/*
 		 * Set the new mask value and also defer further
 		 * occurences of this signal.
@@ -2626,75 +2555,62 @@ postsig(int signum)
 			code = ps->ps_code;
 			ps->ps_code = 0;
 		}
-		p->p_stats->p_ru.ru_nsignals++;
+		OSIncrementAtomic(&p->p_stats->p_ru.ru_nsignals);
 		sendsig(p, catcher, signum, returnmask, code);
 	}
-	signal_unlock(p);
+	proc_signalend(p, 1);
+	proc_unlock(p);
 }
 
 /*
- * Force the current process to exit with the specified signal, dumping core
- * if appropriate.  We bypass the normal tests for masked and caught signals,
- * allowing unrecoverable failures to terminate the process without changing
- * signal state.  Mark the accounting record with the signal termination.
- * If dumping core, save the signal number for the debugger.  Calls exit and
- * does not return.
+ * Attach a signal knote to the list of knotes for this process.
+ *
+ * Signal knotes share the knote list with proc knotes.  This
+ * could be avoided by using a signal-specific knote list, but
+ * probably isn't worth the trouble.
  */
- /* called with signal lock */
-void
-sigexit_locked(p, signum)
-	register struct proc *p;
-	int signum;
-{
-
-	sig_lock_to_exit(p);
-	p->p_acflag |= AXSIG;
-	if (sigprop[signum] & SA_CORE) {
-		p->p_sigacts->ps_sig = signum;
-		signal_unlock(p);
-		if (coredump(p) == 0)
-			signum |= WCOREFLAG;
-	} else 
-		signal_unlock(p);
-		
-	exit1(p, W_EXITCODE(0, signum), (int *)NULL);
-	/* NOTREACHED */
-}
-
 
 static int
 filt_sigattach(struct knote *kn)
 {
-	struct proc *p = current_proc();
-	boolean_t funnel_state;
+	proc_t p = current_proc();  /* can attach only to oneself */
+
+	proc_klist_lock();
 
 	kn->kn_ptr.p_proc = p;
 	kn->kn_flags |= EV_CLEAR;		/* automatically set */
 
-	/* Take the funnel to protect the proc while adding to the list */
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	KNOTE_ATTACH(&p->p_klist, kn);
-	thread_funnel_set(kernel_flock, funnel_state);
+
+	proc_klist_unlock();
 
 	return (0);
 }
 
+/*
+ * remove the knote from the process list, if it hasn't already
+ * been removed by exit processing.  
+ */
+	   
 static void
 filt_sigdetach(struct knote *kn)
 {
-	struct proc *p = kn->kn_ptr.p_proc;
-	boolean_t funnel_state;
+	proc_t p = kn->kn_ptr.p_proc;
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	proc_klist_lock();
+	kn->kn_ptr.p_proc = NULL;
 	KNOTE_DETACH(&p->p_klist, kn);
-	thread_funnel_set(kernel_flock, funnel_state);
+	proc_klist_unlock();
 }
 
 /*
- * signal knotes are shared with proc knotes, so we apply a mask to 
- * the hint in order to differentiate them from process hints.  This
- * could be avoided by using a signal-specific knote list, but probably
- * isn't worth the trouble.
+ * Post an event to the signal filter.  Because we share the same list
+ * as process knotes, we have to filter out and handle only signal events.
+ *
+ * We assume that we process fdfree() before we post the NOTE_EXIT for
+ * a process during exit.  Therefore, since signal filters can only be
+ * set up "in-process", we should have already torn down the kqueue
+ * hosting the EVFILT_SIGNAL knote and should never see NOTE_EXIT.
  */
 static int
 filt_signal(struct knote *kn, long hint)
@@ -2705,94 +2621,106 @@ filt_signal(struct knote *kn, long hint)
 
 		if (kn->kn_id == (unsigned int)hint)
 			kn->kn_data++;
+	} else if (hint & NOTE_EXIT) {
+		panic("filt_signal: detected NOTE_EXIT event");
 	}
+
 	return (kn->kn_data != 0);
 }
 
-
 void
-bsd_ast(thread_t thr_act)
+bsd_ast(thread_t thread)
 {
-	struct proc *p = current_proc();
-	struct uthread *ut = get_bsdthread_info(thr_act);
+	proc_t p = current_proc();
+	struct uthread *ut = get_bsdthread_info(thread);
 	int	signum;
 	user_addr_t pc;
-	boolean_t funnel_state;
 	static int bsd_init_done = 0;
 
 	if (p == NULL)
 		return;
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
 	if ((p->p_flag & P_OWEUPC) && (p->p_flag & P_PROFIL)) {
 		pc = get_useraddr();
 		addupc_task(p, pc, 1);
-		p->p_flag &= ~P_OWEUPC;
+		OSBitAndAtomic(~((uint32_t)P_OWEUPC), (UInt32 *)&p->p_flag);
 	}
 
-	if (CHECK_SIGNALS(p, current_thread(), ut)) {
-		while ( (signum = issignal(p)) )
-			postsig(signum);
+	if (timerisset(&p->p_vtimer_user.it_value)) {
+		uint32_t	microsecs;
+
+		task_vtimer_update(p->task, TASK_VTIMER_USER, &microsecs);
+
+		if (!itimerdecr(p, &p->p_vtimer_user, microsecs)) {
+			if (timerisset(&p->p_vtimer_user.it_value))
+				task_vtimer_set(p->task, TASK_VTIMER_USER);
+			else
+				task_vtimer_clear(p->task, TASK_VTIMER_USER);
+
+			psignal(p, SIGVTALRM);
 	}
-	if (!bsd_init_done) {
-		bsd_init_done = 1;
-		bsdinit_task();
 	}
 
-	(void) thread_funnel_set(kernel_flock, FALSE);
-}
+	if (timerisset(&p->p_vtimer_prof.it_value)) {
+		uint32_t	microsecs;
 
-/*
- * Follwing routines are called using callout from bsd_hardclock
- * so that psignals are called in a thread context and are funneled
- */
-void
-psignal_vtalarm(struct proc *p)
-{
-	boolean_t funnel_state;
+		task_vtimer_update(p->task, TASK_VTIMER_PROF, &microsecs);
 
-	if (p == NULL)
-		return;
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	psignal_lock(p, SIGVTALRM, 1);
-	(void) thread_funnel_set(kernel_flock, FALSE);
+		if (!itimerdecr(p, &p->p_vtimer_prof, microsecs)) {
+			if (timerisset(&p->p_vtimer_prof.it_value))
+				task_vtimer_set(p->task, TASK_VTIMER_PROF);
+			else
+				task_vtimer_clear(p->task, TASK_VTIMER_PROF);
+
+			psignal(p, SIGPROF);
+		}
 }
 
-void
-psignal_xcpu(struct proc *p)
-{
-	boolean_t funnel_state;
+	if (timerisset(&p->p_rlim_cpu)) {
+		struct timeval		tv;
 
-	if (p == NULL)
-		return;
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	psignal_lock(p, SIGXCPU, 1);
-	(void) thread_funnel_set(kernel_flock, FALSE);
-}
+		task_vtimer_update(p->task, TASK_VTIMER_RLIM, (uint32_t *) &tv.tv_usec);
 
-void
-psignal_sigprof(struct proc *p)
-{
-	boolean_t funnel_state;
+		proc_spinlock(p);
+		if (p->p_rlim_cpu.tv_sec > 0 || p->p_rlim_cpu.tv_usec > tv.tv_usec) {
+			tv.tv_sec = 0;
+			timersub(&p->p_rlim_cpu, &tv, &p->p_rlim_cpu);
+			proc_spinunlock(p);
+		} else {
+
+			timerclear(&p->p_rlim_cpu);
+			proc_spinunlock(p);
+
+			task_vtimer_clear(p->task, TASK_VTIMER_RLIM);
+
+			psignal(p, SIGXCPU);
+		}
+	}
+
+	if (CHECK_SIGNALS(p, current_thread(), ut)) {
+		while ( (signum = issignal(p)) )
+			postsig(signum);
+	}
+
+	if (!bsd_init_done) {
+		bsd_init_done = 1;
+		bsdinit_task();
+	}
 
-	if (p == NULL)
-		return;
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	psignal_lock(p, SIGPROF, 1);
-	(void) thread_funnel_set(kernel_flock, FALSE);
 }
 
 /* ptrace set runnalbe */
 void
-pt_setrunnable(struct proc *p)
+pt_setrunnable(proc_t p)
 {
-task_t task;
+	task_t task;
 
 	task = p->task;
 
-	if (p->p_flag & P_TRACED) {
+	if (p->p_lflag & P_LTRACED) {
+		proc_lock(p);
 		p->p_stat = SRUN;
+		proc_unlock(p);
 		if (p->sigwait) {
 			wakeup((caddr_t)&(p->sigwait));
 			task_release(task);
@@ -2800,14 +2728,13 @@ task_t task;
 	}
 }
 
-
 kern_return_t
 do_bsdexception(
 	    int exc,
 	    int code,
 	    int sub)
 {
-	exception_data_type_t   codes[EXCEPTION_CODE_MAX];
+	mach_exception_data_type_t   codes[EXCEPTION_CODE_MAX];
 
 	codes[0] = code;	
 	codes[1] = sub;
@@ -2815,28 +2742,19 @@ do_bsdexception(
 }
 
 int
-proc_pendingsignals(struct proc *p, sigset_t mask)
+proc_pendingsignals(proc_t p, sigset_t mask)
 {
 	struct uthread * uth;
 	thread_t th;
 	sigset_t bits = 0;
-	int error;
 
+	proc_lock(p);
 	/* If the process is in proc exit return no signal info */
-	if (p->p_lflag & P_LPEXIT) 
-		return(0);
-
-	/* duplicate the signal lock code to enable recursion; as exit
-	 * holds the lock too long. All this code is being reworked
-	 * this is just a workaround for regressions till new code
-	 * arrives.
-	 */
-ppend_retry:
-		error = lockmgr((struct lock__bsd__ *)&p->signal_lock[0], (LK_EXCLUSIVE | LK_CANRECURSE), 0, (struct proc *)0);
-		if (error == EINTR)
-			goto ppend_retry;
+	if (p->p_lflag & P_LPEXIT)  {
+		goto out;
+	}
 
-	if ((p->p_flag & P_INVFORK) && p->p_vforkact) {
+	if ((p->p_lflag & P_LINVFORK) && p->p_vforkact) {
 		th = p->p_vforkact;	
 		uth = (struct uthread *)get_bsdthread_info(th);
 		if (uth) {
@@ -2850,7 +2768,7 @@ ppend_retry:
 		bits |= (((uth->uu_siglist & ~uth->uu_sigmask) & ~p->p_sigignore) & mask);
 	}
 out:
-	signal_unlock(p);
+	proc_unlock(p);
 	return(bits);
 }
 
@@ -2860,11 +2778,140 @@ thread_issignal(proc_t p, thread_t th, sigset_t mask)
 	struct uthread * uth;
 	sigset_t  bits=0;
 
+	proc_lock(p);
+	uth = (struct uthread *)get_bsdthread_info(th);
+	if (uth) {
+		bits = (((uth->uu_siglist & ~uth->uu_sigmask) & ~p->p_sigignore) & mask);
+	}
+	proc_unlock(p);
+	return(bits);
+}
 
-		uth = (struct uthread *)get_bsdthread_info(th);
-		if (uth) {
-			bits = (((uth->uu_siglist & ~uth->uu_sigmask) & ~p->p_sigignore) & mask);
-		}
-		return(bits);
+/*
+ * Allow external reads of the sigprop array.
+ */
+int
+hassigprop(int sig, int prop)
+{
+	return (sigprop[sig] & prop);
+}
+
+void
+pgsigio(pid_t pgid, int sig)
+{ 
+	proc_t p = PROC_NULL;
+
+	if (pgid < 0) 
+		gsignal(-(pgid), sig);
+
+	else if (pgid > 0 && (p = proc_find(pgid)) != 0) 
+		psignal(p, sig);
+	if (p != PROC_NULL)
+		proc_rele(p);
+}
+
+
+void
+proc_signalstart(proc_t p, int locked)
+{
+	if (locked == 0)
+		proc_lock(p);
+	while ((p->p_lflag & P_LINSIGNAL) == P_LINSIGNAL) {
+		p->p_lflag |= P_LSIGNALWAIT;
+		msleep(&p->p_sigmask, &p->p_mlock, 0, "proc_signstart", NULL);
+	}
+	p->p_lflag |= P_LINSIGNAL;
+#if DIAGNOSTIC
+#if SIGNAL_DEBUG
+#ifdef __ppc__
+        {
+            int  sp, *fp, numsaved; 
+ 
+            __asm__ volatile("mr %0,r1" : "=r" (sp));
+
+            fp = (int *)*((int *)sp);
+            for (numsaved = 0; numsaved < 3; numsaved++) {
+                p->lockpc[numsaved] = fp[2];
+                if ((int)fp <= 0)
+                        break;
+                fp = (int *)*fp;
+            }
+        }
+#endif /* __ppc__ */       
+#endif /* SIGNAL_DEBUG */
+#endif /* DIAGNOSTIC */
+	p->p_signalholder = current_thread();
+	if (locked == 0)
+		proc_unlock(p);
+
+}
+
+void
+proc_signalend(proc_t p, int locked)
+{
+	if (locked == 0)
+		proc_lock(p);
+	p->p_lflag &= ~P_LINSIGNAL;
+
+#if DIAGNOSTIC
+#if SIGNAL_DEBUG
+#ifdef __ppc__
+        {
+            int sp, *fp, numsaved; 
+ 
+            __asm__ volatile("mr %0,r1" : "=r" (sp));
+
+            fp = (int *)*((int *)sp);
+            for (numsaved = 0; numsaved < 3; numsaved++) {
+                p->unlockpc[numsaved] = fp[2];
+                if ((int)fp <= 0)
+                        break;
+                fp = (int *)*fp;
+            }
+        }
+#endif /* __ppc__ */       
+#endif /* SIGNAL_DEBUG */
+#endif /* DIAGNOSTIC */
+
+	if ((p->p_lflag & P_LSIGNALWAIT) == P_LSIGNALWAIT) {
+		p->p_lflag &= ~P_LSIGNALWAIT;
+		wakeup(&p->p_sigmask);
+	}
+	p->p_signalholder = NULL;
+	if (locked == 0)
+		proc_unlock(p);
+}
+
+
+void
+sig_lock_to_exit(proc_t p)
+{
+	thread_t	self = current_thread();
+
+	p->exit_thread = self;
+	proc_unlock(p);
+	(void) task_suspend(p->task);
+	proc_lock(p);
 }
 
+int
+sig_try_locked(proc_t p)
+{
+	thread_t	self = current_thread();
+
+	while (p->sigwait || p->exit_thread) {
+		if (p->exit_thread) {
+			return(0);
+		}
+		msleep((caddr_t)&p->sigwait_thread, &p->p_mlock, PCATCH | PDROP, 0, 0);
+		if (thread_should_abort(self)) {
+			/*
+			 * Terminate request - clean up.
+			 */
+			proc_lock(p);
+			return -1;
+		}
+		proc_lock(p);
+	}
+	return 1;
+}
diff --git a/bsd/kern/kern_subr.c b/bsd/kern/kern_subr.c
index ab64b242c..711dc5fe7 100644
--- a/bsd/kern/kern_subr.c
+++ b/bsd/kern/kern_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -78,27 +84,40 @@
 #if DEBUG
 #include <kern/simple_lock.h>
 
-static int				uio_t_count = 0;
+static uint32_t				uio_t_count = 0;
 #endif /* DEBUG */
 
 
+/*
+ * Returns:	0			Success
+ *	uiomove64:EFAULT
+ *
+ * Notes:	The first argument should be a caddr_t, but const poisoning
+ *		for typedef'ed types doesn't work in gcc.
+ */
 int
-uiomove(cp, n, uio)
-	register caddr_t cp;
-	register int n;
-	register uio_t uio;
+uiomove(const char * cp, int n, uio_t uio)
 {
-	return uiomove64((addr64_t)((unsigned int)cp), n, uio);
+	return uiomove64((const addr64_t)((const unsigned int)cp), n, uio);
 }
 
+/*
+ * Returns:	0			Success
+ *		EFAULT
+ *	copyout:EFAULT
+ *	copyin:EFAULT
+ *	copywithin:EFAULT
+ *	copypv:EFAULT
+ */
 	// LP64todo - fix this! 'n' should be int64_t?
 int
-uiomove64(addr64_t cp, int n, register struct uio *uio)
+uiomove64(const addr64_t c_cp, int n, struct uio *uio)
 {
+	addr64_t cp = c_cp;
 #if LP64KERN
-	register uint64_t acnt;
+	uint64_t acnt;
 #else
-	register u_int acnt;
+	u_int acnt;
 #endif
 	int error = 0;
 
@@ -301,9 +320,7 @@ uiomove64(addr64_t cp, int n, register struct uio *uio)
  * Give next character to user as result of read.
  */
 int
-ureadc(c, uio)
-	register int c;
-	register struct uio *uio;
+ureadc(int c, struct uio *uio)
 {
 	if (uio_resid(uio) <= 0)
 		panic("ureadc: non-positive resid");
@@ -355,10 +372,9 @@ again:
  * Get next character written in by user from uio.
  */
 int
-uwritec(uio)
-	uio_t uio;
+uwritec(uio_t uio)
 {
-	register int c = 0;
+	int c = 0;
 
 	if (uio_resid(uio) <= 0)
 		return (-1);
@@ -412,9 +428,7 @@ again:
  * General routine to allocate a hash table.
  */
 void *
-hashinit(elements, type, hashmask)
-	int elements, type;
-	u_long *hashmask;
+hashinit(int elements, int type, u_long *hashmask)
 {
 	long hashsize;
 	LIST_HEAD(generic, generic) *hashtbl;
@@ -442,7 +456,7 @@ user_ssize_t uio_resid( uio_t a_uio )
 {
 #if DEBUG
 	if (a_uio == NULL) {
-		panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); 
+		printf("%s :%d - invalid uio_t\n", __FILE__, __LINE__); 
 	}
 /* 	if (IS_VALID_UIO_SEGFLG(a_uio->uio_segflg) == 0) { */
 /* 		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);  */
@@ -495,44 +509,6 @@ void uio_setresid( uio_t a_uio, user_ssize_t a_value )
 	return;
 }
 
-#if 0 // obsolete
-/*
- * uio_proc_t - return the proc_t for the given uio_t
- * WARNING - This call is going away.  Find another way to get the proc_t!!
- */
-__private_extern__ proc_t uio_proc_t( uio_t a_uio )
-{
-#if LP64_DEBUG
-	if (a_uio == NULL) {
-		panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); 
-	}
-#endif /* LP64_DEBUG */
-
-	/* return 0 if there are no active iovecs */
-	if (a_uio == NULL) {
-		return( NULL );
-	}
-	return( a_uio->uio_procp );
-}
-
-/*
- * uio_setproc_t - set the residual IO value for the given uio_t
- * WARNING - This call is going away. 
- */
-__private_extern__ void uio_setproc_t( uio_t a_uio, proc_t a_proc_t )
-{
-	if (a_uio == NULL) {
-#if LP64_DEBUG
-		panic("%s :%d - invalid uio_t\n", __FILE__, __LINE__); 
-#endif /* LP64_DEBUG */
-		return;
-	}
-
-	a_uio->uio_procp = a_proc_t;
-	return;
-}
-#endif // obsolete
-
 /*
  * uio_curriovbase - return the base address of the current iovec associated 
  *	with the given uio_t.  May return 0.
@@ -737,7 +713,7 @@ uio_t uio_create( int a_iovcount,		/* number of iovecs */
 	int					my_size;
 	uio_t				my_uio;
 	
-	my_size = sizeof(struct uio) + (sizeof(struct user_iovec) * a_iovcount);
+	my_size = UIO_SIZEOF(a_iovcount);
 	my_buf_p = kalloc(my_size);
 	my_uio = uio_createwithbuffer( a_iovcount, 
 									 a_offset,
@@ -749,7 +725,7 @@ uio_t uio_create( int a_iovcount,		/* number of iovecs */
 		/* leave a note that we allocated this uio_t */
 		my_uio->uio_flags |= UIO_FLAGS_WE_ALLOCED;
 #if DEBUG
-		hw_atomic_add(&uio_t_count, 1);
+		(void)hw_atomic_add(&uio_t_count, 1);
 #endif
 	}
 	
@@ -941,9 +917,8 @@ void uio_free( uio_t a_uio )
 
 	if (a_uio != NULL && (a_uio->uio_flags & UIO_FLAGS_WE_ALLOCED) != 0) {
 #if DEBUG
-		if ((int)(hw_atomic_sub(&uio_t_count, 1)) < 0) {
-			panic("%s :%d - uio_t_count has gone negative\n", __FILE__, __LINE__); 
-		}
+		if (hw_atomic_sub(&uio_t_count, 1) == UINT_MAX)
+			panic("%s :%d - uio_t_count underflow\n", __FILE__, __LINE__); 
 #endif
 		kfree(a_uio, a_uio->uio_size);
 	}
@@ -1257,6 +1232,8 @@ uio_t uio_duplicate( uio_t a_uio )
 		}
 	}
 
+	my_uio->uio_flags = UIO_FLAGS_WE_ALLOCED | UIO_FLAGS_INITED;
+
 	return(my_uio);
 }
 
diff --git a/bsd/kern/kern_symfile.c b/bsd/kern/kern_symfile.c
index 70e60fc43..98e4e7771 100644
--- a/bsd/kern/kern_symfile.c
+++ b/bsd/kern/kern_symfile.c
@@ -1,48 +1,35 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
  *
  *	File:	bsd/kern/kern_symfile.c
  *
- *	This file contains creates a dummy symbol file for mach_kernel
- *	based on the symbol table information passed by the
- *	SecondaryLoader/PlatformExpert.  This allows us to correctly
- *	link other executables (drivers, etc) against the the kernel in
- *	cases where the kernel image on the root device does not match
- *      the live kernel. This can occur during net-booting where the
- *	actual kernel image is obtained from the network via tftp rather
- *	than the root device.
- *
- *      If a symbol table is available, then the file /mach.sym will be
- *	created containing a Mach Header and a LC_SYMTAB load command
- *	followed by the the symbol table data for mach_kernel.
- *
- * NOTE:	This file supports only 32 bit kernels at the present time;
- *		adding support for 64 bit kernels is possible, but is not
- *		necessary at the present time.
- *
  * HISTORY
- * 
- *	.
  */
 
 #include <mach/vm_param.h>
@@ -73,279 +60,19 @@
 #include <pexpert/pexpert.h>
 #include <IOKit/IOHibernatePrivate.h>
 
-extern unsigned char 	rootdevice[];
-extern struct mach_header _mh_execute_header;
-
-static int kernel_symfile_opened = 0;
-static int error_code = 0;
-
-extern int  IODTGetLoaderInfo(char *key, void **infoAddr, int *infoSize);
-extern void IODTFreeLoaderInfo(char *key, void *infoAddr, int infoSize);
-
-/*
- * Can only operate against currently running 32 bit mach_kernel
+/* This function is called from kern_sysctl in the current process context;
+ * it is exported with the System6.0.exports, but this appears to be a legacy
+ * export, as there are no internal consumers.
  */
-static int
-output_kernel_symbols(struct proc *p)
-{
-    struct vnode		*vp;
-    kauth_cred_t		cred = p->p_ucred;	/* XXX unsafe */
-    struct vnode_attr		va;
-    struct vfs_context		context;
-    struct load_command		*cmd;
-    struct mach_header		*orig_mh, *mh;
-    struct segment_command	*orig_ds, *orig_ts, *orig_le, *sg;
-    struct section		*se, *const_text;
-    struct symtab_command	*st, *orig_st;
-    struct nlist		*sym;
-    vm_size_t			orig_mhsize, orig_st_size;
-    vm_offset_t			header;
-    vm_size_t			header_size = 0; /* out: protected by header */
-    int				error, error1;
-    unsigned int		i, j;
-    caddr_t			addr;
-    vm_offset_t			offset;
-    int				rc_mh, rc_sc;
-
-    error = EFAULT;
-
-    vp		= NULL;
-    header	= NULL;
-    orig_mh	= NULL;
-    orig_st	= NULL;
-    
-    context.vc_proc = p;
-    context.vc_ucred = kauth_cred_proc_ref(p);
-
-    // Dispose of unnecessary gumf, the booter doesn't need to load these
-    rc_mh = IODTGetLoaderInfo("Kernel-__HEADER",
-				(void **)&orig_mh, &orig_mhsize);
-    if (rc_mh == 0 && orig_mh)
-	IODTFreeLoaderInfo("Kernel-__HEADER",
-			    (void *)orig_mh, round_page_32(orig_mhsize));
-
-    rc_sc = IODTGetLoaderInfo("Kernel-__SYMTAB",
-				(void **) &orig_st, &orig_st_size);
-    if (rc_sc == 0 && orig_st)
-	IODTFreeLoaderInfo("Kernel-__SYMTAB",
-			    (void *)orig_st, round_page_32(orig_st_size));
-
-    if (cred->cr_svuid != cred->cr_ruid || cred->cr_svgid != cred->cr_rgid)
-	goto out;
-
-    // Check to see if the root is 'e' or 'n', is this a test for network?
-    if (rootdevice[0] == 'e' && rootdevice[1] == 'n')
-	goto out;
-
-    if ((error = vnode_open("mach.sym", (O_CREAT | FWRITE), (S_IRUSR | S_IRGRP | S_IROTH), 0, &vp, &context)))
-        goto out;
-
-    /* Don't dump to non-regular files or files with links. */
-    error = EFAULT;
-    VATTR_INIT(&va);
-    VATTR_WANTED(&va, va_nlink);
-    if ((vp->v_type != VREG) || vnode_getattr(vp, &va, &context) || (va.va_nlink != 1))
-	goto out;
-
-    VATTR_INIT(&va);			/* better to do it here than waste more stack in vnode_getsize */
-    VATTR_SET(&va, va_data_size, 0);
-    vnode_setattr(vp, &va, &context);
-    p->p_acflag |= ACORE;
-
-    // If the file type is MH_EXECUTE then this must be a kernel
-    // as all Kernel extensions must be of type MH_OBJECT
-    orig_ds = orig_ts = orig_le = NULL;
-    orig_st = NULL;
-    orig_mh = &_mh_execute_header;
-    cmd = (struct load_command *) &orig_mh[1];
-    for (i = 0; i < orig_mh->ncmds; i++) {
-	if (cmd->cmd == LC_SEGMENT) {
-	    struct segment_command *orig_sg = (struct segment_command *) cmd;
-    
-	    if (!strcmp(SEG_TEXT, orig_sg->segname))
-		orig_ts = orig_sg;
-	    else if (!strcmp(SEG_DATA, orig_sg->segname))
-		orig_ds = orig_sg;
-	    else if (!strcmp(SEG_LINKEDIT, orig_sg->segname))
-		orig_le = orig_sg;
-	}
-	else if (cmd->cmd == LC_SYMTAB)
-	    orig_st = (struct symtab_command *) cmd;
-    
-	cmd = (struct load_command *) ((caddr_t) cmd + cmd->cmdsize);
-    }
-
-    if (!orig_ts || !orig_ds || !orig_le || !orig_st) 
-	goto out;
-
-    const_text = NULL;
-    se = (struct section *) &orig_ts[1];
-    for (i = 0; i < orig_ts->nsects; i++, se++) {
-	if (!strcmp("__const", se->sectname)) {
-	    const_text = se;
-	    break;
-	}
-    }
-    if (!const_text)
-	goto out;
-
-    header_size =   sizeof(struct mach_header) 
-		    + orig_ts->cmdsize
-		    + orig_ds->cmdsize
-		    + sizeof(struct symtab_command);
-
-    (void) kmem_alloc(kernel_map,
-			    (vm_offset_t *) &header,
-			    (vm_size_t) header_size);
-    if (header)
-	bzero((void *) header, header_size);
-    else
-	goto out;
-
-    /*
-     *	Set up Mach-O header.
-     */
-    mh = (struct mach_header *) header;
-    mh->magic      = orig_mh->magic;
-    mh->cputype    = orig_mh->cputype;
-    mh->cpusubtype = orig_mh->cpusubtype;
-    mh->filetype   = orig_mh->filetype;
-    mh->ncmds      = 3;
-    mh->sizeofcmds = header_size - sizeof(struct mach_header);
-    mh->flags      = orig_mh->flags;
-
-    // Initialise the current file offset and addr
-    offset = round_page(header_size);
-    addr = (caddr_t) const_text->addr;	// Load address of __TEXT,__const
-
-    /*
-     * Construct a TEXT segment load command
-     * the only part of the TEXT segment we keep is the __TEXT,__const
-     * which contains the kernel vtables. 
-     */
-    sg = (struct segment_command *) &mh[1];
-    bcopy(orig_ts, sg, orig_ts->cmdsize);
-    sg->vmaddr   = (unsigned long) addr;
-    sg->vmsize   = const_text->size;
-    sg->fileoff  = 0;
-    sg->filesize = const_text->size + round_page(header_size);
-    sg->maxprot  = 0;
-    sg->initprot = 0;
-    sg->flags    = 0;
-    se = (struct section *)(sg+1);
-    for ( j = 0; j < sg->nsects; j++, se++ ) {
-	se->addr  = (unsigned long) addr;
-	se->size  = 0;
-	se->offset = offset;
-	se->nreloc = 0;
-	if (!strcmp("__const", se->sectname)) {
-	    se->size = const_text->size;
-	    addr    += const_text->size;
-	    offset  += const_text->size;
-	    const_text = se;
-	}
-    }
-    offset = round_page(offset);
-
-    // Now copy of the __DATA segment load command, the image need
-    // not be stored to disk nobody needs it, yet!
-    sg = (struct segment_command *)((int)sg + sg->cmdsize);
-    bcopy(orig_ds, sg, orig_ds->cmdsize);
-
-    sg->vmaddr   = (unsigned long) addr;
-    sg->vmsize   = 0x1000;	// One page for some reason?
-    sg->fileoff  = offset;
-    sg->filesize = 0;
-    sg->maxprot  = 0;
-    sg->initprot = 0;
-    sg->flags    = 0;
-    se = (struct section *)(sg+1);
-    for ( j = 0; j < sg->nsects; j++, se++ ) {
-	se->addr  = (unsigned long) addr;
-	se->size  = 0;
-	se->offset = offset;
-	se->nreloc = 0;
-    }
-    offset = round_page(offset);
-
-
-    /*
-     *	Set up LC_SYMTAB command
-     */
-    st          = (struct symtab_command *)((int)sg + sg->cmdsize);
-    st->cmd     = LC_SYMTAB;
-    st->cmdsize = sizeof(struct symtab_command);
-    st->symoff  = offset;
-    st->nsyms   = orig_st->nsyms;
-    st->strsize = orig_st->strsize;
-    st->stroff =  offset + st->nsyms * sizeof(struct nlist);    
-
-    /*
-     * Convert the symbol table in place from section references
-     * to absolute references.
-     */
-    sym = (struct nlist *) orig_le->vmaddr;
-    for (i = 0; i < st->nsyms; i++, sym++ ) {
-	if ( (sym->n_type & N_TYPE) == N_SECT) {
-	    sym->n_sect = NO_SECT;
-	    sym->n_type = (sym->n_type & ~N_TYPE) | N_ABS;
-	}
-    }
-
-    /*
-     *	Write out the load commands at the beginning of the file.
-     */
-    error = vn_rdwr(UIO_WRITE, vp, (caddr_t) mh, header_size, (off_t) 0,
-		    UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
-    if (error)
-	goto out;
-
-    /*
-     *	Write out the __TEXT,__const data segment.
-     */
-    error = vn_rdwr(UIO_WRITE, vp, (caddr_t) const_text->addr,
-		    const_text->size, const_text->offset,
-		    UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
-    if (error)
-	goto out;
-
-    /*
-     * 	Write out kernel symbols
-     */
-    offset = st->nsyms * sizeof(struct nlist) + st->strsize;	// symtab size
-    error = vn_rdwr(UIO_WRITE, vp,
-		   (caddr_t) orig_le->vmaddr, offset, st->symoff,
-		    UIO_SYSSPACE32, IO_NODELOCKED|IO_UNIT, cred, (int *) 0, p);
-out:
-    if (header)
-	kmem_free(kernel_map, header, header_size);
-
-    if (vp) {
-	error1 = vnode_close(vp, FWRITE, &context);
-	if (!error) error = error1;
-    }
-
-    kauth_cred_unref(&context.vc_ucred);
-    return(error);
-}
-/*
- * 
- */
-int get_kernel_symfile(struct proc *p, char **symfile)
+int
+get_kernel_symfile(__unused proc_t p, __unused char const **symfile)
 {
-    if (!kernel_symfile_opened) {
-        kernel_symfile_opened = 1;
-        error_code = output_kernel_symbols(p);
-    }
-    if (!error_code)
-	*symfile = "\\mach.sym";
-
-    return error_code;
+    return KERN_FAILURE;
 }
 
 struct kern_direct_file_io_ref_t
 {
-    struct vfs_context		context;
+    vfs_context_t		ctx;
     struct vnode		*vp;
 };
 
@@ -373,7 +100,7 @@ kern_open_file_for_direct_io(const char * name,
 {
     struct kern_direct_file_io_ref_t * ref;
 
-    struct proc 		*p;
+    proc_t			p;
     struct vnode_attr		va;
     int				error;
     off_t			f_offset;
@@ -397,10 +124,9 @@ kern_open_file_for_direct_io(const char * name,
 
     ref->vp = NULL;
     p = current_proc();		// kernproc;
-    ref->context.vc_proc = p;
-    ref->context.vc_ucred = kauth_cred_proc_ref(p);
+    ref->ctx = vfs_context_create(vfs_context_current());
 
-    if ((error = vnode_open(name, (O_CREAT | FWRITE), (0), 0, &ref->vp, &ref->context)))
+    if ((error = vnode_open(name, (O_CREAT | FWRITE), (0), 0, &ref->vp, ref->ctx)))
         goto out;
 
     VATTR_INIT(&va);
@@ -409,7 +135,7 @@ kern_open_file_for_direct_io(const char * name,
     VATTR_WANTED(&va, va_data_size);
     VATTR_WANTED(&va, va_nlink);
     error = EFAULT;
-    if (vnode_getattr(ref->vp, &va, &ref->context))
+    if (vnode_getattr(ref->vp, &va, ref->ctx))
     	goto out;
 
     kprintf("vp va_rdev major %d minor %d\n", major(va.va_rdev), minor(va.va_rdev));
@@ -433,7 +159,7 @@ kern_open_file_for_direct_io(const char * name,
         device = va.va_rdev;
 
         p1 = ref->vp;
-        p2 = &ref->context;
+        p2 = ref->ctx;
         do_ioctl = &device_ioctl;
     }
     else
@@ -535,14 +261,14 @@ out:
     kprintf("kern_open_file_for_direct_io(%d)\n", error);
 
     if (error && ref) {
-		if (ref->vp) {
-			vnode_close(ref->vp, FWRITE, &ref->context);
-			ref->vp = NULLVP;
-		}
-
-		kauth_cred_unref(&ref->context.vc_ucred);
-		kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
-		ref = NULL;
+	if (ref->vp) {
+	    vnode_close(ref->vp, FWRITE, ref->ctx);
+	    ref->vp = NULLVP;
+	}
+
+	vfs_context_rele(ref->ctx);
+	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
+	ref = NULL;
     }
 
     return(ref);
@@ -554,7 +280,8 @@ kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, caddr_t ad
     return (vn_rdwr(UIO_WRITE, ref->vp,
 			addr, len, offset,
 			UIO_SYSSPACE32, IO_SYNC|IO_NODELOCKED|IO_UNIT, 
-                        ref->context.vc_ucred, (int *) 0, ref->context.vc_proc));
+                        vfs_context_ucred(ref->ctx), (int *) 0,
+			vfs_context_proc(ref->ctx)));
 }
 
 void
@@ -566,11 +293,13 @@ kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref)
 	int                error;
 
 	if (ref->vp) {
-	    error = vnode_close(ref->vp, FWRITE, &ref->context);
+	    error = vnode_close(ref->vp, FWRITE, ref->ctx);
+	    ref->vp = NULLVP;
 	    kprintf("vnode_close(%d)\n", error);
-		ref->vp = NULLVP;
 	}
-	kauth_cred_unref(&ref->context.vc_ucred);
+	vfs_context_rele(ref->ctx);
+	ref->ctx = NULL;
 	kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
     }
 }
+
diff --git a/bsd/kern/kern_synch.c b/bsd/kern/kern_synch.c
index 9f33c4ce1..8cfe29bfc 100644
--- a/bsd/kern/kern_synch.c
+++ b/bsd/kern/kern_synch.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -49,19 +55,19 @@
 #include <mach/time_value.h>
 #include <kern/lock.h>
 
+#include <sys/systm.h>			/* for unix_syscall_return() */
+#include <libkern/OSAtomic.h>
+
+extern boolean_t thread_should_abort(thread_t);	/* XXX */
+extern void compute_averunnable(void *);	/* XXX */
+
 
-#if KTRACE
-#include <sys/uio.h>
-#include <sys/ktrace.h>
-#endif 
 
 static void
-_sleep_continue(
-	void			*parameter,
-	wait_result_t	wresult)
+_sleep_continue( __unused void *parameter, wait_result_t wresult)
 {
-	register struct proc *p = current_proc();
-	register thread_t self  = current_thread();
+	struct proc *p = current_proc();
+	thread_t self  = current_thread();
 	struct uthread * ut;
 	int sig, catch;
 	int error = 0;
@@ -89,7 +95,7 @@ _sleep_continue(
 				if (thread_should_abort(self)) {
 					error = EINTR;
 				} else if (SHOULDissignal(p,ut)) {
-					if (sig = CURSIG(p)) {
+					if ((sig = CURSIG(p)) != 0) {
 						if (p->p_sigacts->ps_sigintr & sigmask(sig))
 							error = EINTR;
 						else
@@ -110,13 +116,12 @@ _sleep_continue(
 	if (error == EINTR || error == ERESTART)
 		act_set_astbsd(self);
 
-#if KTRACE
-	if (KTRPOINT(p, KTR_CSW))
-		ktrcsw(p->p_tracep, 0, 0);
-#endif
 	if (ut->uu_mtx && !dropmutex)
 	        lck_mtx_lock(ut->uu_mtx);
 
+	ut->uu_wchan = NULL;
+	ut->uu_wmesg = NULL;
+
 	unix_syscall_return((*ut->uu_continuation)(error));
 }
 
@@ -150,8 +155,8 @@ _sleep(
 	int		(*continuation)(int),
         lck_mtx_t	*mtx)
 {
-	register struct proc *p;
-	register thread_t self = current_thread();
+	struct proc *p;
+	thread_t self = current_thread();
 	struct uthread * ut;
 	int sig, catch = pri & PCATCH;
 	int dropmutex  = pri & PDROP;
@@ -161,12 +166,14 @@ _sleep(
 	ut = get_bsdthread_info(self);
 
 	p = current_proc();
-#if KTRACE
-	if (KTRPOINT(p, KTR_CSW))
-		ktrcsw(p->p_tracep, 1, 0);
-#endif	
 	p->p_priority = pri & PRIMASK;
-	p->p_stats->p_ru.ru_nvcsw++;
+	/* It can still block in proc_exit() after the teardown. */
+	if (p->p_stats != NULL)
+		OSIncrementAtomic(&p->p_stats->p_ru.ru_nvcsw);
+
+	/* set wait message & channel */
+	ut->uu_wchan = chan;
+	ut->uu_wmesg = wmsg ? wmsg : "unknown";
 
 	if (mtx != NULL && chan != NULL && (thread_continue_t)continuation == THREAD_CONTINUE_NULL) {
 
@@ -184,27 +191,9 @@ _sleep(
 			lck_mtx_unlock(mtx);
 		if (catch) {
 			if (SHOULDissignal(p,ut)) {
-				if (sig = CURSIG(p)) {
+				if ((sig = CURSIG(p)) != 0) {
 					if (clear_wait(self, THREAD_INTERRUPTED) == KERN_FAILURE)
 						goto block;
-					/* if SIGTTOU or SIGTTIN then block till SIGCONT */
-					if ((pri & PTTYBLOCK) && ((sig == SIGTTOU) || (sig == SIGTTIN))) {
-						p->p_flag |= P_TTYSLEEP;
-						/* reset signal bits */
-						clear_procsiglist(p, sig);
-						assert_wait(&p->p_siglist, THREAD_ABORTSAFE);
-						/* assert wait can block and SIGCONT should be checked */
-						if (p->p_flag & P_TTYSLEEP) {
-							thread_block(THREAD_CONTINUE_NULL);
-							
-							if (mtx && !dropmutex)
-		        					lck_mtx_lock(mtx);
-						}
-
-						/* return with success */
-						error = 0;
-						goto out;
-					}
 					if (p->p_sigacts->ps_sigintr & sigmask(sig))
 						error = EINTR;
 					else
@@ -260,7 +249,7 @@ block:
 				if (thread_should_abort(self)) {
 					error = EINTR;
 				} else if (SHOULDissignal(p, ut)) {
-					if (sig = CURSIG(p)) {
+					if ((sig = CURSIG(p)) != 0) {
 						if (p->p_sigacts->ps_sigintr & sigmask(sig))
 							error = EINTR;
 						else
@@ -277,11 +266,9 @@ block:
 out:
 	if (error == EINTR || error == ERESTART)
 		act_set_astbsd(self);
+	ut->uu_wchan = NULL;
+	ut->uu_wmesg = NULL;
 
-#if KTRACE
-	if (KTRPOINT(p, KTR_CSW))
-		ktrcsw(p->p_tracep, 0, 0);
-#endif
 	return (error);
 }
 
@@ -383,8 +370,7 @@ tsleep1(
  * Wake up all processes sleeping on chan.
  */
 void
-wakeup(chan)
-	register void *chan;
+wakeup(void *chan)
 {
 	thread_wakeup_prim((caddr_t)chan, FALSE, THREAD_AWAKENED);
 }
@@ -396,8 +382,7 @@ wakeup(chan)
  * the right one to wakeup.
  */
 void
-wakeup_one(chan)
-	register caddr_t chan;
+wakeup_one(caddr_t chan)
 {
 	thread_wakeup_prim((caddr_t)chan, TRUE, THREAD_AWAKENED);
 }
@@ -408,8 +393,7 @@ wakeup_one(chan)
  * than that of the current process.
  */
 void
-resetpriority(p)
-	register struct proc *p;
+resetpriority(struct proc *p)
 {
 	(void)task_importance(p->task, -p->p_nice);
 }
@@ -427,12 +411,11 @@ static fixpt_t cexp[3] = {
 };
 
 void
-compute_averunnable(
-	void 			*arg)
+compute_averunnable(void *arg)
 {
 	unsigned int		nrun = *(unsigned int *)arg;
 	struct loadavg		*avg = &averunnable;
-	register int		i;
+	int		i;
 
     for (i = 0; i < 3; i++)
         avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
diff --git a/bsd/kern/kern_sysctl.c b/bsd/kern/kern_sysctl.c
index 2b6a8b4a8..029fddc8e 100644
--- a/bsd/kern/kern_sysctl.c
+++ b/bsd/kern/kern_sysctl.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -57,9 +63,24 @@
  *
  *	@(#)kern_sysctl.c	8.4 (Berkeley) 4/14/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
- * sysctl system call.
+* DEPRECATED sysctl system call code
+ *
+ * Everything in this file is deprecated. Sysctls should be handled
+ * by the code in kern_newsysctl.c.
+ * The remaining "case" sections are supposed to be converted into
+ * SYSCTL_*-style definitions, and as soon as all of them are gone,
+ * this source file is supposed to die.
+ *
+ * DO NOT ADD ANY MORE "case" SECTIONS TO THIS FILE, instead define
+ * your sysctl with SYSCTL_INT, SYSCTL_PROC etc. in your source file.
  */
 
 #include <sys/param.h>
@@ -80,6 +101,7 @@
 #include <sys/sysctl.h>
 #include <sys/user.h>
 #include <sys/aio_kern.h>
+#include <sys/reboot.h>
 
 #include <bsm/audit_kernel.h>
 
@@ -92,8 +114,6 @@
 #include <vm/vm_map.h>
 #include <mach/host_info.h>
 
-extern vm_map_t bsd_pageable_map;
-
 #include <sys/mount_internal.h>
 #include <sys/kdebug.h>
 #include <sys/sysproto.h>
@@ -114,90 +134,70 @@ sysctlfn kern_sysctl;
 #ifdef DEBUG
 sysctlfn debug_sysctl;
 #endif
-extern sysctlfn vm_sysctl;
-extern sysctlfn vfs_sysctl;
 extern sysctlfn net_sysctl;
 extern sysctlfn cpu_sysctl;
 extern int aio_max_requests;  				
 extern int aio_max_requests_per_process;	
 extern int aio_worker_threads;				
-extern int maxfilesperproc;
 extern int lowpri_IO_window_msecs;
 extern int lowpri_IO_delay_msecs;
 extern int nx_enabled;
+extern int speculative_reads_disabled;
 
 static void
-fill_eproc(struct proc *p, struct eproc *ep);
+fill_eproc(proc_t p, struct eproc *ep);
 static void
-fill_externproc(struct proc *p, struct extern_proc *exp);
+fill_externproc(proc_t p, struct extern_proc *exp);
 static void
-fill_user_eproc(struct proc *p, struct user_eproc *ep);
+fill_user_eproc(proc_t p, struct user_eproc *ep);
 static void
-fill_user_proc(struct proc *p, struct user_kinfo_proc *kp);
+fill_user_proc(proc_t p, struct user_kinfo_proc *kp);
 static void
-fill_user_externproc(struct proc *p, struct user_extern_proc *exp);
+fill_user_externproc(proc_t p, struct user_extern_proc *exp);
 extern int 
 kdbg_control(int *name, u_int namelen, user_addr_t where, size_t * sizep);
 int
-kdebug_ops(int *name, u_int namelen, user_addr_t where, size_t *sizep, struct proc *p);
+kdebug_ops(int *name, u_int namelen, user_addr_t where, size_t *sizep, proc_t p);
 #if NFSCLIENT
 extern int 
 netboot_root(void);
 #endif
 int
 pcsamples_ops(int *name, u_int namelen, user_addr_t where, size_t *sizep, 
-              struct proc *p);
+              proc_t p);
 __private_extern__ kern_return_t
 reset_vmobjectcache(unsigned int val1, unsigned int val2);
-extern int
-resize_namecache(u_int newsize);
-static int
-sysctl_aiomax(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen);
-static int
-sysctl_aioprocmax(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen);
-static int
-sysctl_aiothreads(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen);
-extern int
-sysctl_clockrate(user_addr_t where, size_t *sizep);
 int
 sysctl_doproc(int *name, u_int namelen, user_addr_t where, size_t *sizep);
 int 
 sysctl_doprof(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
 			  user_addr_t newp, size_t newlen);
-int
-sysctl_file(user_addr_t where, size_t *sizep);
 static void
-fill_proc(struct proc *p, struct kinfo_proc *kp);
-static int
-sysctl_maxfilesperproc(user_addr_t oldp, size_t *oldlenp, 
-                       user_addr_t newp, size_t newlen);
-static int
-sysctl_maxprocperuid(user_addr_t oldp, size_t *oldlenp, 
-                     user_addr_t newp, size_t newlen);
-static int
-sysctl_maxproc(user_addr_t oldp, size_t *oldlenp, 
-               user_addr_t newp, size_t newlen);
+fill_proc(proc_t p, struct kinfo_proc *kp);
 int
 sysctl_procargs(int *name, u_int namelen, user_addr_t where, 
-				size_t *sizep, struct proc *cur_proc);
+				size_t *sizep, proc_t cur_proc);
 static int
 sysctl_procargs2(int *name, u_int namelen, user_addr_t where, size_t *sizep, 
-                 struct proc *cur_proc);
+                 proc_t cur_proc);
 static int
 sysctl_procargsx(int *name, u_int namelen, user_addr_t where, size_t *sizep, 
-                 struct proc *cur_proc, int argc_yes);
+                 proc_t cur_proc, int argc_yes);
 int
 sysctl_struct(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, 
               size_t newlen, void *sp, int len);
-extern int
-sysctl_vnode(user_addr_t where, size_t *sizep);
 
+static int sysdoproc_filt_KERN_PROC_PID(proc_t p, void * arg);
+static int sysdoproc_filt_KERN_PROC_PGRP(proc_t p, void * arg);
+static int sysdoproc_filt_KERN_PROC_TTY(proc_t p, void * arg);
+static int  sysdoproc_filt_KERN_PROC_UID(proc_t p, void * arg);
+static int  sysdoproc_filt_KERN_PROC_RUID(proc_t p, void * arg);
+static int  sysdoproc_filt_KERN_PROC_LCID(proc_t p, void * arg);
+int sysdoproc_callback(proc_t p, void *arg);
 
-/*
- * temporary location for vm_sysctl.  This should be machine independant
- */
- 
-extern uint32_t mach_factor[3];
+static int __sysctl_funneled(proc_t p, struct __sysctl_args *uap, register_t *retval);
+
+extern void IORegistrySetOSBuildVersion(char * build_version); 
 
 static void
 loadavg32to64(struct loadavg *la32, struct user_loadavg *la64)
@@ -208,89 +208,37 @@ loadavg32to64(struct loadavg *la32, struct user_loadavg *la64)
 	la64->fscale	= (user_long_t)la32->fscale;
 }
 
-int
-vm_sysctl(int *name, __unused u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-          user_addr_t newp, size_t newlen, __unused struct proc *p)
-{
-	struct loadavg loadinfo;
-
-	switch (name[0]) {
-	case VM_LOADAVG:
-		if (proc_is64bit(p)) {
-			struct user_loadavg loadinfo64;
-			loadavg32to64(&averunnable, &loadinfo64);
-			return (sysctl_struct(oldp, oldlenp, newp, newlen,
-					&loadinfo64, sizeof(loadinfo64)));
-		} else {
-			return (sysctl_struct(oldp, oldlenp, newp, newlen,
-					&averunnable, sizeof(struct loadavg)));
-		}
-	case VM_MACHFACTOR:
-		loadinfo.ldavg[0] = mach_factor[0];
-		loadinfo.ldavg[1] = mach_factor[1];
-		loadinfo.ldavg[2] = mach_factor[2];
-		loadinfo.fscale = LSCALE;
-		if (proc_is64bit(p)) {
-			struct user_loadavg loadinfo64;
-			loadavg32to64(&loadinfo, &loadinfo64);
-			return (sysctl_struct(oldp, oldlenp, newp, newlen,
-					&loadinfo64, sizeof(loadinfo64)));
-		} else {
-			return (sysctl_struct(oldp, oldlenp, newp, newlen,
-					&loadinfo, sizeof(struct loadavg)));
-		}
-	case VM_SWAPUSAGE: {
-		int			error;
-		uint64_t		swap_total;
-		uint64_t		swap_avail;
-		uint32_t		swap_pagesize;
-		boolean_t		swap_encrypted;
-		struct xsw_usage	xsu;
-
-		error = macx_swapinfo(&swap_total,
-				      &swap_avail,
-				      &swap_pagesize,
-				      &swap_encrypted);
-		if (error)
-			return error;
-
-		xsu.xsu_total = swap_total;
-		xsu.xsu_avail = swap_avail;
-		xsu.xsu_used = swap_total - swap_avail;
-		xsu.xsu_pagesize = swap_pagesize;
-		xsu.xsu_encrypted = swap_encrypted;
-		return sysctl_struct(oldp, oldlenp, newp, newlen,
-				     &xsu, sizeof (struct xsw_usage));
-	}
-	case VM_METER:
-		return (ENOTSUP);
-	case VM_MAXID:
-		return (ENOTSUP);
-	default:
-		return (ENOTSUP);
-	}
-	/* NOTREACHED */
-	return (ENOTSUP);
-}
-
 /*
  * Locking and stats
  */
-static struct sysctl_lock {
-	int	sl_lock;
-	int	sl_want;
-	int	sl_locked;
-} memlock;
+static struct sysctl_lock memlock;
 
+/* sysctl() syscall */
 int
-__sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
+__sysctl(proc_t p, struct __sysctl_args *uap, register_t *retval)
+{
+	boolean_t funnel_state;
+	int error;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	error = __sysctl_funneled(p, uap, retval);
+	thread_funnel_set(kernel_flock, funnel_state);
+	return(error);
+}
+
+static int
+__sysctl_funneled(proc_t p, struct __sysctl_args *uap, __unused register_t *retval)
 {
 	int error, dolock = 1;
 	size_t savelen = 0, oldlen = 0, newlen;
 	sysctlfn *fnp = NULL;
 	int name[CTL_MAXNAME];
-	int i;
 	int error1;
+	boolean_t memlock_taken = FALSE;
+	boolean_t vslock_taken = FALSE;
+#if CONFIG_MACF
+	kauth_cred_t my_cred;
+#endif
 
 	/*
 	 * all top-level sysctl names are non-terminal
@@ -318,13 +266,15 @@ __sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
 	if (uap->new != USER_ADDR_NULL
 	    && ((name[0] == CTL_KERN
 		&& !(name[1] == KERN_IPC || name[1] == KERN_PANICINFO || name[1] == KERN_PROCDELAYTERM || 
-		     name[1] == KERN_PROC_LOW_PRI_IO || name[1] == KERN_PROCNAME || name[1] == KERN_THALTSTACK))
+		     name[1] == KERN_PROC_LOW_PRI_IO || name[1] == KERN_PROCNAME || name[1] == KERN_RAGEVNODE || name[1] == KERN_CHECKOPENEVT))
 	    || (name[0] == CTL_HW)
-	    || (name[0] == CTL_VM)
-		|| (name[0] == CTL_VFS))
+	    || (name[0] == CTL_VM))
 	    && (error = suser(kauth_cred_get(), &p->p_acflag)))
 		return (error);
 
+/* XXX: KERN, VFS and DEBUG are handled by their respective functions,
+ * but there is a fallback for all sysctls other than VFS to
+ * userland_sysctl() - KILL THIS! */
 	switch (name[0]) {
 	case CTL_KERN:
 		fnp = kern_sysctl;
@@ -332,10 +282,6 @@ __sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
 			&& (name[1] != KERN_PROC))
 			dolock = 0;
 		break;
-	case CTL_VM:
-		fnp = vm_sysctl; 
-		break;
-                
 	case CTL_VFS:
 		fnp = vfs_sysctl;
 		break;
@@ -363,68 +309,82 @@ __sysctl(struct proc *p, struct __sysctl_args *uap, __unused register_t *retval)
 	if (uap->old != USER_ADDR_NULL) {
 		if (!useracc(uap->old, (user_size_t)oldlen, B_WRITE))
 			return (EFAULT);
-
-		/* The pc sampling mechanism does not need to take this lock */
-		if ((name[1] != KERN_PCSAMPLES) &&
-		    (!((name[1] == KERN_KDEBUG) && (name[2] == KERN_KDGETENTROPY)))) {
-		  while (memlock.sl_lock) {
-			memlock.sl_want = 1;
-			sleep((caddr_t)&memlock, PRIBIO+1);
-			memlock.sl_locked++;
-		  }
-		  memlock.sl_lock = 1;
-		}
-
-		if (dolock && oldlen &&
-		    (error = vslock(uap->old, (user_size_t)oldlen))) {
-			if ((name[1] != KERN_PCSAMPLES) &&
-			   (! ((name[1] == KERN_KDEBUG) && (name[2] == KERN_KDGETENTROPY)))) {
-		  		memlock.sl_lock = 0;
-		  		if (memlock.sl_want) {
-		        	memlock.sl_want = 0;
-					wakeup((caddr_t)&memlock);
-		  		}
+		/*
+		 * The kernel debug mechanism does not need to take this lock, and
+		 * we don't grab the memlock around calls to KERN_PROC because it is reentrant.
+		 * Grabbing the lock for a KERN_PROC sysctl makes a deadlock possible 5024049.
+		 */
+		if (!((name[1] == KERN_KDEBUG) && (name[2] == KERN_KDGETENTROPY)) &&
+		    !(name[1] == KERN_PROC)) {
+		        MEMLOCK_LOCK();
+			memlock_taken = TRUE;
+                }
+
+		if (dolock && oldlen) {
+		        if ((error = vslock(uap->old, (user_size_t)oldlen))) {
+			        if (memlock_taken == TRUE)
+				        MEMLOCK_UNLOCK();
+				return(error);
 			}
-			return(error);
+			savelen = oldlen;
+			vslock_taken = TRUE;
 		}
-		savelen = oldlen;
 	}
 
+#if CONFIG_MACF
+	my_cred = kauth_cred_proc_ref(p);
+	error = mac_system_check_sysctl(
+	    my_cred, 
+	    (int *) name,
+	    uap->namelen,
+  	    uap->old,
+	    uap->oldlenp,
+	    fnp == kern_sysctl ? 1 : 0,
+	    uap->new,
+	    newlen
+   	);
+	kauth_cred_unref(&my_cred);
+	if (!error) {
+#endif
 	if (fnp) {
-        error = (*fnp)(name + 1, uap->namelen - 1, uap->old,
+	        error = (*fnp)(name + 1, uap->namelen - 1, uap->old,
                        &oldlen, uap->new, newlen, p);
 	}
 	else
-	    error = ENOTSUP;
+	        error = ENOTSUP;
+#if CONFIG_MACF
+	}
+#endif
+
+	if (vslock_taken == TRUE) {
+	        error1 = vsunlock(uap->old, (user_size_t)savelen, B_WRITE);
+		if (!error)
+		        error = error1;
+        }
+	if (memlock_taken == TRUE)
+	        MEMLOCK_UNLOCK();
 
 	if ( (name[0] != CTL_VFS) && (error == ENOTSUP)) {
-	    size_t  tmp = oldlen;
+	        size_t  tmp = oldlen;
+		boolean_t funnel_state;
+
+		/*
+		 * Drop the funnel when calling new sysctl code, which will conditionally
+		 * grab the funnel if it really needs to.
+		 */
+		funnel_state = thread_funnel_set(kernel_flock, FALSE);
+		
 		error = userland_sysctl(p, name, uap->namelen, uap->old, &tmp, 
 		                        1, uap->new, newlen, &oldlen);
-	}
 
-	if (uap->old != USER_ADDR_NULL) {
-		if (dolock && savelen) {
-			error1 = vsunlock(uap->old, (user_size_t)savelen, B_WRITE);
-			if (!error &&  error1)
-				error = error1;
-		}
-		if (name[1] != KERN_PCSAMPLES) {
-		  memlock.sl_lock = 0;
-		  if (memlock.sl_want) {
-		        memlock.sl_want = 0;
-			wakeup((caddr_t)&memlock);
-		  }
-		}
+		thread_funnel_set(kernel_flock, funnel_state);
 	}
+
 	if ((error) && (error != ENOMEM))
 		return (error);
 
-	if (uap->oldlenp != USER_ADDR_NULL) {
-	    i =	suulong(uap->oldlenp, oldlen);
-		if (i) 
-		    return i;
-	}
+	if (uap->oldlenp != USER_ADDR_NULL)
+	        error = suulong(uap->oldlenp, oldlen);
 
 	return (error);
 }
@@ -436,6 +396,9 @@ __private_extern__ char corefilename[MAXPATHLEN+1];
 __private_extern__ int do_coredump;
 __private_extern__ int sugid_coredump;
 
+#if COUNT_SYSCALLS
+__private_extern__ int do_count_syscalls;
+#endif
 
 #ifdef INSECURE
 int securelevel = -1;
@@ -451,7 +414,7 @@ sysctl_affinity(
 	size_t *oldSize,
 	user_addr_t newBuf,
 	__unused size_t newSize,
-	struct proc *cur_proc)
+	proc_t cur_proc)
 {
 	if (namelen < 1)
 		return (ENOTSUP);
@@ -461,16 +424,15 @@ sysctl_affinity(
 			                (cur_proc->p_flag & P_AFFINITY) ? 1 : 0);
 	} else if (name[0] == 1 && 2 == namelen) {
 		if (name[1] == 0) {
-			cur_proc->p_flag &= ~P_AFFINITY;
+			OSBitAndAtomic(~((uint32_t)P_AFFINITY), (UInt32 *)&cur_proc->p_flag);
 		} else {
-			cur_proc->p_flag |= P_AFFINITY;
+			OSBitOrAtomic(P_AFFINITY, (UInt32 *)&cur_proc->p_flag);
 		}
 		return 0;
 	}
 	return (ENOTSUP);
 }
 
-
 static int
 sysctl_translate(
 	int *name,
@@ -479,47 +441,54 @@ sysctl_translate(
 	size_t *oldSize,
 	user_addr_t newBuf,
 	__unused size_t newSize,
-	struct proc *cur_proc)
+	proc_t cur_proc)
 {
-	struct proc *p;
+	proc_t p;
+	int istranslated = 0;
+	kauth_cred_t my_cred;
+	uid_t uid;
 
 	if (namelen != 1)
 		return (ENOTSUP);
 
-	p = pfind(name[0]);
+	p = proc_find(name[0]);
 	if (p == NULL)
 		return (EINVAL);
 
-	if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
-		&& suser(kauth_cred_get(), &cur_proc->p_acflag))
+	my_cred = kauth_cred_proc_ref(p);
+	uid = kauth_cred_getuid(my_cred);
+	kauth_cred_unref(&my_cred);
+	if ((uid != kauth_cred_getuid(kauth_cred_get())) 
+		&& suser(kauth_cred_get(), &cur_proc->p_acflag)) {
+		proc_rele(p);
 		return (EPERM);
+	}
 
+	istranslated = (p->p_flag & P_TRANSLATED);
+	proc_rele(p);
 	return sysctl_rdint(oldBuf, oldSize, newBuf,
-		                (p->p_flag & P_TRANSLATED) ? 1 : 0);
+		                (istranslated != 0) ? 1 : 0);
 }
 
 int
-set_archhandler(struct proc *p, int arch)
+set_archhandler(__unused proc_t p, int arch)
 {
 	int error;
 	struct nameidata nd;
 	struct vnode_attr va;
-	struct vfs_context context;
-	char *archhandler;
+	vfs_context_t ctx = vfs_context_current();
+	struct exec_archhandler *archhandler;
 
 	switch(arch) {
 	case CPU_TYPE_POWERPC:
-		archhandler = exec_archhandler_ppc.path;
+		archhandler = &exec_archhandler_ppc;
 		break;
 	default:
 		return (EBADARCH);
 	}
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
-		   CAST_USER_ADDR_T(archhandler), &context);
+		   CAST_USER_ADDR_T(archhandler->path), ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -535,18 +504,20 @@ set_archhandler(struct proc *p, int arch)
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_fsid);
 	VATTR_WANTED(&va, va_fileid);
-	error = vnode_getattr(nd.ni_vp, &va, &context);
+	error = vnode_getattr(nd.ni_vp, &va, ctx);
 	if (error) {
 		vnode_put(nd.ni_vp);
 		return (error);
 	}
 	vnode_put(nd.ni_vp);
 	
-	exec_archhandler_ppc.fsid = va.va_fsid;
-	exec_archhandler_ppc.fileid = (u_long)va.va_fileid;
+	archhandler->fsid = va.va_fsid;
+	archhandler->fileid = (u_long)va.va_fileid;
 	return 0;
 }
 
+/* XXX remove once Rosetta is rev'ed */
+/*****************************************************************************/
 static int
 sysctl_exec_archhandler_ppc(
 	__unused int *name,
@@ -555,17 +526,12 @@ sysctl_exec_archhandler_ppc(
 	size_t *oldSize,
 	user_addr_t newBuf,
 	size_t newSize,
-	struct proc *p)
+	proc_t p)
 {
 	int error;
 	size_t len;
-	struct nameidata nd;
-	struct vnode_attr va;
 	char handler[sizeof(exec_archhandler_ppc.path)];
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	if (oldSize) {
 		len = strlen(exec_archhandler_ppc.path) + 1;
@@ -579,7 +545,7 @@ sysctl_exec_archhandler_ppc(
 		*oldSize = len - 1;
 	}
 	if (newBuf) {
-		error = suser(context.vc_ucred, &p->p_acflag);
+		error = suser(vfs_context_ucred(ctx), &p->p_acflag);
 		if (error)
 			return (error);
 		if (newSize >= sizeof(exec_archhandler_ppc.path))
@@ -588,36 +554,54 @@ sysctl_exec_archhandler_ppc(
 		if (error)
 			return (error);
 		handler[newSize] = 0;
-		strcpy(exec_archhandler_ppc.path, handler);
+		strlcpy(exec_archhandler_ppc.path, handler, MAXPATHLEN);
 		error = set_archhandler(p, CPU_TYPE_POWERPC);
 		if (error)
 			return (error);
 	}
 	return 0;
 }
+/*****************************************************************************/
+
+static int
+sysctl_handle_exec_archhandler_ppc(struct sysctl_oid *oidp, void *arg1,
+		int arg2, struct sysctl_req *req)
+{
+	int error = 0;
+
+	error = sysctl_handle_string(oidp, arg1, arg2, req);
+
+	if (error)
+		goto done;
+
+	if (req->newptr)
+		error = set_archhandler(req->p, CPU_TYPE_POWERPC);
+
+done:
+	return error;
+
+}
 
-SYSCTL_NODE(_kern, KERN_EXEC, exec, CTLFLAG_RD, 0, "");
+SYSCTL_NODE(_kern, KERN_EXEC, exec, CTLFLAG_RD|CTLFLAG_LOCKED, 0, "");
 
-SYSCTL_NODE(_kern_exec, OID_AUTO, archhandler, CTLFLAG_RD, 0, "");
+SYSCTL_NODE(_kern_exec, OID_AUTO, archhandler, CTLFLAG_RD|CTLFLAG_LOCKED, 0, "");
 
-SYSCTL_STRING(_kern_exec_archhandler, OID_AUTO, powerpc, CTLFLAG_RD,
-		exec_archhandler_ppc.path, 0, "");
+SYSCTL_PROC(_kern_exec_archhandler, OID_AUTO, powerpc,
+	    CTLTYPE_STRING | CTLFLAG_RW, exec_archhandler_ppc.path, 0,
+	    sysctl_handle_exec_archhandler_ppc, "A", "");
 
-extern int get_kernel_symfile( struct proc *, char **);
+extern int get_kernel_symfile(proc_t, char **);
 __private_extern__ int 
 sysctl_dopanicinfo(int *, u_int, user_addr_t, size_t *, user_addr_t, 
-                   size_t, struct proc *);
+                   size_t, proc_t);
 
 /*
  * kernel related system variables.
  */
 int
 kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-            user_addr_t newp, size_t newlen, struct proc *p)
+            user_addr_t newp, size_t newlen, proc_t p)
 {
-	int error, level, inthostid, tmp;
-	unsigned int oldval=0;
-	char *str;
 	/* all sysctl names not listed below are terminal at this level */
 	if (namelen != 1
 		&& !(name[0] == KERN_PROC
@@ -625,7 +609,6 @@ kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 			|| name[0] == KERN_KDEBUG
 			|| name[0] == KERN_PROCARGS
 			|| name[0] == KERN_PROCARGS2
-			|| name[0] == KERN_PCSAMPLES
 			|| name[0] == KERN_IPC
 			|| name[0] == KERN_SYSV
 			|| name[0] == KERN_AFFINITY
@@ -633,324 +616,88 @@ kern_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 			|| name[0] == KERN_EXEC
 			|| name[0] == KERN_PANICINFO
 			|| name[0] == KERN_POSIX
-			|| name[0] == KERN_TFP)
+			|| name[0] == KERN_TFP
+			|| name[0] == KERN_TTY
+#if CONFIG_LCTX
+			|| name[0] == KERN_LCTX
+#endif
+						)
 		)
 		return (ENOTDIR);		/* overloaded */
 
 	switch (name[0]) {
-	case KERN_OSTYPE:
-		return (sysctl_rdstring(oldp, oldlenp, newp, ostype));
-	case KERN_OSRELEASE:
-		return (sysctl_rdstring(oldp, oldlenp, newp, osrelease));
-	case KERN_OSREV:
-		return (sysctl_rdint(oldp, oldlenp, newp, BSD));
-	case KERN_VERSION:
-		return (sysctl_rdstring(oldp, oldlenp, newp, version));
-	case KERN_MAXVNODES:
-		oldval = desiredvnodes;
-		error = sysctl_int(oldp, oldlenp, newp, 
-				newlen, &desiredvnodes);
-		reset_vmobjectcache(oldval, desiredvnodes);
-		resize_namecache(desiredvnodes);
-		return(error);
-	case KERN_MAXPROC:
-		return (sysctl_maxproc(oldp, oldlenp, newp, newlen));
-	case KERN_MAXFILES:
-		return (sysctl_int(oldp, oldlenp, newp, newlen, &maxfiles));
-	case KERN_MAXPROCPERUID:
-		return( sysctl_maxprocperuid( oldp, oldlenp, newp, newlen ) );
-	case KERN_MAXFILESPERPROC:
-		return( sysctl_maxfilesperproc( oldp, oldlenp, newp, newlen ) );
-	case KERN_ARGMAX:
-		return (sysctl_rdint(oldp, oldlenp, newp, ARG_MAX));
-	case KERN_SECURELVL:
-		level = securelevel;
-		if ((error = sysctl_int(oldp, oldlenp, newp, newlen, &level)) ||
-		    newp == USER_ADDR_NULL)
-			return (error);
-		if (level < securelevel && p->p_pid != 1)
-			return (EPERM);
-		securelevel = level;
-		return (0);
-	case KERN_HOSTNAME:
-		error = sysctl_trstring(oldp, oldlenp, newp, newlen,
-		    hostname, sizeof(hostname));
-		if (newp && !error)
-			hostnamelen = newlen;
-		return (error);
-	case KERN_DOMAINNAME:
-		error = sysctl_string(oldp, oldlenp, newp, newlen,
-		    domainname, sizeof(domainname));
-		if (newp && !error)
-			domainnamelen = newlen;
-		return (error);
-	case KERN_HOSTID:
-		inthostid = hostid;  /* XXX assumes sizeof long <= sizeof int */
-		error =  sysctl_int(oldp, oldlenp, newp, newlen, &inthostid);
-		hostid = inthostid;
-		return (error);
-	case KERN_CLOCKRATE:
-		return (sysctl_clockrate(oldp, oldlenp));
-	case KERN_BOOTTIME:
-	{
-		struct timeval	t;
-
-		t.tv_sec = boottime_sec();
-		t.tv_usec = 0;
-
-		return (sysctl_rdstruct(oldp, oldlenp, newp, &t,
-		    sizeof(struct timeval)));
-	}
-	case KERN_VNODE:
-		return (sysctl_vnode(oldp, oldlenp));
 	case KERN_PROC:
 		return (sysctl_doproc(name + 1, namelen - 1, oldp, oldlenp));
-	case KERN_FILE:
-		return (sysctl_file(oldp, oldlenp));
 #ifdef GPROF
 	case KERN_PROF:
 		return (sysctl_doprof(name + 1, namelen - 1, oldp, oldlenp,
 		    newp, newlen));
-#endif
-	case KERN_POSIX1:
-		return (sysctl_rdint(oldp, oldlenp, newp, _POSIX_VERSION));
-	case KERN_NGROUPS:
-		return (sysctl_rdint(oldp, oldlenp, newp, NGROUPS_MAX));
-	case KERN_JOB_CONTROL:
-		return (sysctl_rdint(oldp, oldlenp, newp, 1));
-	case KERN_SAVED_IDS:
-#ifdef _POSIX_SAVED_IDS
-		return (sysctl_rdint(oldp, oldlenp, newp, 1));
-#else
-		return (sysctl_rdint(oldp, oldlenp, newp, 0));
 #endif
 	case KERN_KDEBUG:
 		return (kdebug_ops(name + 1, namelen - 1, oldp, oldlenp, p));
-	case KERN_PCSAMPLES:
-		return (pcsamples_ops(name + 1, namelen - 1, oldp, oldlenp, p));
 	case KERN_PROCARGS:
 		/* new one as it does not use kinfo_proc */
 		return (sysctl_procargs(name + 1, namelen - 1, oldp, oldlenp, p));
 	case KERN_PROCARGS2:
 		/* new one as it does not use kinfo_proc */
 		return (sysctl_procargs2(name + 1, namelen - 1, oldp, oldlenp, p));
-	case KERN_SYMFILE:
-		error = get_kernel_symfile( p, &str );
-		if ( error )
-			return error;
-		return (sysctl_rdstring(oldp, oldlenp, newp, str));
-#if NFSCLIENT
-	case KERN_NETBOOT:
-		return (sysctl_rdint(oldp, oldlenp, newp, netboot_root()));
-#endif
+#if PANIC_INFO
 	case KERN_PANICINFO:
 		return(sysctl_dopanicinfo(name + 1, namelen - 1, oldp, oldlenp,
 			newp, newlen, p));
+#endif
 	case KERN_AFFINITY:
 		return sysctl_affinity(name+1, namelen-1, oldp, oldlenp,
 									newp, newlen, p);
 	case KERN_TRANSLATE:
 		return sysctl_translate(name+1, namelen-1, oldp, oldlenp, newp,
 				      newlen, p);
-	case KERN_CLASSICHANDLER:
+
+		/* XXX remove once Rosetta has rev'ed */
+	case KERN_EXEC:
 		return sysctl_exec_archhandler_ppc(name+1, namelen-1, oldp,
 						   oldlenp, newp, newlen, p);
-	case KERN_AIOMAX:
-		return( sysctl_aiomax( oldp, oldlenp, newp, newlen ) );
-	case KERN_AIOPROCMAX:
-		return( sysctl_aioprocmax( oldp, oldlenp, newp, newlen ) );
-	case KERN_AIOTHREADS:
-		return( sysctl_aiothreads( oldp, oldlenp, newp, newlen ) );
-	case KERN_USRSTACK:
-		return (sysctl_rdint(oldp, oldlenp, newp, (uintptr_t)p->user_stack));
-	case KERN_USRSTACK64:
-		return (sysctl_rdquad(oldp, oldlenp, newp, p->user_stack));
-	case KERN_COREFILE:
-		error = sysctl_string(oldp, oldlenp, newp, newlen,
-		    corefilename, sizeof(corefilename));
-		return (error);
-	case KERN_COREDUMP:
-		tmp = do_coredump;
-		error = sysctl_int(oldp, oldlenp, newp, newlen, &do_coredump);
-		if (!error && ((do_coredump < 0) || (do_coredump > 1))) {
-			do_coredump = tmp;
-			error = EINVAL;
-		}
-		return (error);
-	case KERN_SUGID_COREDUMP:
-		tmp = sugid_coredump;
-		error = sysctl_int(oldp, oldlenp, newp, newlen, &sugid_coredump);
-		if (!error && ((sugid_coredump < 0) || (sugid_coredump > 1))) {
-			sugid_coredump = tmp;
-			error = EINVAL;
-		}
-		return (error);
-	case KERN_PROCDELAYTERM:
+#if COUNT_SYSCALLS
+	case KERN_COUNT_SYSCALLS:
 	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(int))
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(int) )
-			return(EINVAL);
-		*oldlenp = sizeof(int);
-		old_value = (p->p_lflag & P_LDELAYTERM)? 1: 0;
-		if (oldp && (error = copyout( &old_value, oldp, sizeof(int))))
-			return(error);
-        	if (error == 0 && newp )
-                	error = copyin( newp, &new_value, sizeof(int) );
-        	if (error == 0 && newp) {
-                	if (new_value)
-                        	p->p_lflag |=  P_LDELAYTERM;
-                	else
-                        	p->p_lflag &=  ~P_LDELAYTERM;
-        	}
-		return(error);
-	}
-	case KERN_PROC_LOW_PRI_IO:
-	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(int))
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(int) )
-			return(EINVAL);
-		*oldlenp = sizeof(int);
-
-		old_value = (p->p_lflag & P_LLOW_PRI_IO)? 0x01: 0;
-		if (p->p_lflag & P_LBACKGROUND_IO)
-		        old_value |= 0x02;
-
-		if (oldp && (error = copyout( &old_value, oldp, sizeof(int))))
-			return(error);
-        	if (error == 0 && newp )
-                	error = copyin( newp, &new_value, sizeof(int) );
-        	if (error == 0 && newp) {
-                	if (new_value & 0x01)
-                        	p->p_lflag |= P_LLOW_PRI_IO;
-			else if (new_value & 0x02)
-                        	p->p_lflag |= P_LBACKGROUND_IO;
-                	else if (new_value == 0)
-                        	p->p_lflag &= ~(P_LLOW_PRI_IO | P_LBACKGROUND_IO);
-        	}
-		return(error);
-	}
-	case KERN_LOW_PRI_WINDOW:
-	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(old_value) )
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(new_value) )
-			return(EINVAL);
-		*oldlenp = sizeof(old_value);
-
-		old_value = lowpri_IO_window_msecs;
-
-		if (oldp && (error = copyout( &old_value, oldp, *oldlenp)))
-			return(error);
-        	if (error == 0 && newp )
-                	error = copyin( newp, &new_value, sizeof(newlen) );
-        	if (error == 0 && newp) {
-		        lowpri_IO_window_msecs = new_value;
-        	}
-		return(error);
-	}
-	case KERN_LOW_PRI_DELAY:
-	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(old_value) )
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(new_value) )
-			return(EINVAL);
-		*oldlenp = sizeof(old_value);
-
-		old_value = lowpri_IO_delay_msecs;
-
-		if (oldp && (error = copyout( &old_value, oldp, *oldlenp)))
-			return(error);
-        	if (error == 0 && newp )
-                	error = copyin( newp, &new_value, sizeof(newlen) );
-        	if (error == 0 && newp) {
-		        lowpri_IO_delay_msecs = new_value;
-        	}
-		return(error);
-	}
-	case KERN_NX_PROTECTION:
-	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(old_value) )
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(new_value) )
-			return(EINVAL);
-		*oldlenp = sizeof(old_value);
-
-		old_value = nx_enabled;
-
-		if (oldp && (error = copyout( &old_value, oldp, *oldlenp)))
-			return(error);
-#ifdef __i386__
-		/*
-		 * Only allow setting if NX is supported on the chip
+		/* valid values passed in:
+		 * = 0 means don't keep called counts for each bsd syscall
+		 * > 0 means keep called counts for each bsd syscall
+		 * = 2 means dump current counts to the system log
+		 * = 3 means reset all counts
+		 * for example, to dump current counts:  
+		 *		sysctl -w kern.count_calls=2
 		 */
-		if (cpuid_extfeatures() & CPUID_EXTFEATURE_XD) {
-#endif
-        		if (error == 0 && newp)
-                		error = copyin(newp, &new_value,
-					       sizeof(newlen));
-        		if (error == 0 && newp)
-		        	nx_enabled = new_value;
-#ifdef __i386__
-        	} else if (newp) {
-			error = ENOTSUP;
+		error = sysctl_int(oldp, oldlenp, newp, newlen, &tmp);
+		if ( error != 0 ) {
+			return (error);
 		}
-#endif
-		return(error);
-	}
-	case KERN_SHREG_PRIVATIZABLE:
-		/* this kernel does implement shared_region_make_private_np() */
-		return (sysctl_rdint(oldp, oldlenp, newp, 1));
-	case KERN_PROCNAME:
-		error = sysctl_trstring(oldp, oldlenp, newp, newlen,
-		    &p->p_name[0], (2*MAXCOMLEN+1));
-		return (error);
-	case KERN_THALTSTACK:
-	{
-		int	 old_value, new_value;
-
-		error = 0;
-		if (oldp && *oldlenp < sizeof(int))
-			return (ENOMEM);
-		if ( newp && newlen != sizeof(int) )
-			return(EINVAL);
-		*oldlenp = sizeof(int);
-		old_value = (p->p_lflag & P_LTHSIGSTACK)? 1: 0;
-		if (oldp && (error = copyout( &old_value, oldp, sizeof(int))))
-			return(error);
-        	if (error == 0 && newp )
-                	error = copyin( newp, &new_value, sizeof(int) );
-        	if (error == 0 && newp) {
-                	if (new_value) {
-							/* we cannot swich midstream if inuse */
-							if ((p->p_sigacts->ps_flags & SAS_ALTSTACK) == SAS_ALTSTACK)
-								return(EPERM);
-                        	p->p_lflag |=  P_LTHSIGSTACK;
-                	} else {
-							/* we cannot swich midstream */
-							if ((p->p_lflag & P_LTHSIGSTACK) == P_LTHSIGSTACK)
-								return(EPERM);
-							p->p_lflag &=  ~P_LTHSIGSTACK;
+			
+		if ( tmp == 1 ) {
+			do_count_syscalls = 1;
+		}
+		else if ( tmp == 0 || tmp == 2 || tmp == 3 ) {
+			extern int 			nsysent;
+			extern int			syscalls_log[];
+			extern const char *	syscallnames[];
+			int			i;
+			for ( i = 0; i < nsysent; i++ ) {
+				if ( syscalls_log[i] != 0 ) {
+					if ( tmp == 2 ) {
+						printf("%d calls - name %s \n", syscalls_log[i], syscallnames[i]);
 					}
-        	}
-		return(error);
+					else {
+						syscalls_log[i] = 0;
+					}
+				}
+			}
+			if ( tmp != 0 ) {
+				do_count_syscalls = 1;
+			}
+		}
+		return (0);
 	}
+#endif
 	default:
 		return (ENOTSUP);
 	}
@@ -977,7 +724,7 @@ static struct ctldebug *debugvars[CTL_DEBUG_MAXID] = {
 };
 int
 debug_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-             user_addr_t newp, size_t newlen, struct proc *p)
+             user_addr_t newp, size_t newlen, __unused proc_t p)
 {
 	struct ctldebug *cdp;
 
@@ -999,6 +746,16 @@ debug_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 }
 #endif /* DEBUG */
 
+/*
+ * The following sysctl_* functions should not be used
+ * any more, as they can only cope with callers in
+ * user mode: Use new-style
+ *  sysctl_io_number()
+ *  sysctl_io_string()
+ *  sysctl_io_opaque()
+ * instead.
+ */
+
 /*
  * Validate parameters and get old / set new parameters
  * for an integer-valued sysctl function.
@@ -1073,11 +830,7 @@ sysctl_quad(user_addr_t oldp, size_t *oldlenp,
  * As above, but read-only.
  */
 int
-sysctl_rdquad(oldp, oldlenp, newp, val)
-	void *oldp;
-	size_t *oldlenp;
-	void *newp;
-	quad_t val;
+sysctl_rdquad(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, quad_t val)
 {
 	int error = 0;
 
@@ -1089,7 +842,7 @@ sysctl_rdquad(oldp, oldlenp, newp, val)
 		return (EPERM);
 	*oldlenp = sizeof(quad_t);
 	if (oldp)
-		error = copyout((caddr_t)&val, CAST_USER_ADDR_T(oldp), sizeof(quad_t));
+		error = copyout((caddr_t)&val, oldp, sizeof(quad_t));
 	return (error);
 }
 
@@ -1237,44 +990,33 @@ sysctl_rdstruct(user_addr_t oldp, size_t *oldlenp,
 /*
  * Get file structures.
  */
-int
-sysctl_file(user_addr_t where, size_t *sizep)
+static int
+sysctl_file
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int buflen, error;
+	int error;
 	struct fileglob *fg;
-	user_addr_t start = where;
 	struct extern_file nef;
 
-	buflen = *sizep;
-	if (where == USER_ADDR_NULL) {
+	if (req->oldptr == USER_ADDR_NULL) {
 		/*
 		 * overestimate by 10 files
 		 */
-		*sizep = sizeof(filehead) + (nfiles + 10) * sizeof(struct extern_file);
+		req->oldidx = sizeof(filehead) + (nfiles + 10) * sizeof(struct extern_file);
 		return (0);
 	}
 
 	/*
 	 * first copyout filehead
 	 */
-	if (buflen < 0 || (size_t)buflen < sizeof(filehead)) {
-		*sizep = 0;
-		return (0);
-	}
-    error = copyout((caddr_t)&filehead, where, sizeof(filehead));
+	error = SYSCTL_OUT(req, &filehead, sizeof(filehead));
 	if (error)
 		return (error);
-	buflen -= sizeof(filehead);
-	where += sizeof(filehead);
 
 	/*
 	 * followed by an array of file structures
 	 */
 	for (fg = filehead.lh_first; fg != 0; fg = fg->f_list.le_next) {
-		if (buflen < 0 || (size_t)buflen < sizeof(struct extern_file)) {
-			*sizep = where - start;
-			return (ENOMEM);
-		}
         nef.f_list.le_next =  (struct extern_file *)fg->f_list.le_next;
         nef.f_list.le_prev =  (struct extern_file **)fg->f_list.le_prev;
 		nef.f_flag = (fg->fg_flag & FMASK);
@@ -1285,40 +1027,176 @@ sysctl_file(user_addr_t where, size_t *sizep)
 		nef.f_ops = fg->fg_ops;
 		nef.f_offset = fg->fg_offset;
 		nef.f_data = fg->fg_data;
-        error = copyout((caddr_t)&nef, where, sizeof (struct extern_file));
+		error = SYSCTL_OUT(req, &nef, sizeof(nef));
 		if (error)
 			return (error);
-		buflen -= sizeof(struct extern_file);
-		where += sizeof(struct extern_file);
 	}
-    *sizep = where - start;
 	return (0);
 }
 
+SYSCTL_PROC(_kern, KERN_FILE, file,
+		CTLTYPE_STRUCT | CTLFLAG_RW,
+		0, 0, sysctl_file, "S,filehead", "");
+
+static int
+sysdoproc_filt_KERN_PROC_PID(proc_t p, void * arg)
+{
+	if (p->p_pid != (pid_t)arg)
+		return(0);
+	else
+		return(1);
+}
+
+static int
+sysdoproc_filt_KERN_PROC_PGRP(proc_t p, void * arg)
+{
+	if (p->p_pgrpid != (pid_t)arg)
+		return(0);
+	else
+	  return(1);
+}
+
+static int
+sysdoproc_filt_KERN_PROC_TTY(proc_t p, void * arg)
+{
+	boolean_t funnel_state;
+	int retval;
+
+	
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	/* This is very racy but list lock is held.. Hmmm. */
+	if ((p->p_flag & P_CONTROLT) == 0 ||
+		(p->p_pgrp == NULL) || (p->p_pgrp->pg_session == NULL) ||
+			p->p_pgrp->pg_session->s_ttyp == NULL ||
+			p->p_pgrp->pg_session->s_ttyp->t_dev != (dev_t)arg)
+				retval = 0;
+	else
+		retval = 1;
+
+	thread_funnel_set(kernel_flock, funnel_state);
+
+	return(retval);
+}
+
+static int
+sysdoproc_filt_KERN_PROC_UID(proc_t p, void * arg)
+{
+	kauth_cred_t my_cred;
+	uid_t uid;
+
+	if (p->p_ucred == NULL)
+		return(0);
+	my_cred = kauth_cred_proc_ref(p);
+	uid = kauth_cred_getuid(my_cred);
+	kauth_cred_unref(&my_cred);
+
+	if (uid != (uid_t)arg)
+		return(0);
+	else
+		return(1);
+}
+
+
+static int
+sysdoproc_filt_KERN_PROC_RUID(proc_t p, void * arg)
+{
+	kauth_cred_t my_cred;
+	uid_t ruid;
+
+	if (p->p_ucred == NULL)
+		return(0);
+	my_cred = kauth_cred_proc_ref(p);
+	ruid = my_cred->cr_ruid;
+	kauth_cred_unref(&my_cred);
+
+	if (ruid != (uid_t)arg)
+		return(0);
+	else
+		return(1);
+}
+
+static int
+sysdoproc_filt_KERN_PROC_LCID(proc_t p, void * arg)
+{
+	if ((p->p_lctx == NULL) ||
+		(p->p_lctx->lc_id != (pid_t)arg))
+		return(0);
+	else
+		return(1);
+}
+
 /*
  * try over estimating by 5 procs
  */
 #define KERN_PROCSLOP	(5 * sizeof (struct kinfo_proc))
+struct sysdoproc_args {
+	int	buflen;
+	caddr_t	kprocp;
+	boolean_t is_64_bit;
+	user_addr_t	dp;
+	size_t needed;
+	int sizeof_kproc;
+	int * errorp;
+	int uidcheck;
+	int ruidcheck;
+	int ttycheck;
+	int uidval;
+};
+
+int
+sysdoproc_callback(proc_t p, void * arg)
+{
+	struct sysdoproc_args * args = (struct sysdoproc_args *)arg;
+	int error=0;
+
+	if (args->buflen >= args->sizeof_kproc) {
+		if ((args->ruidcheck != 0)  && (sysdoproc_filt_KERN_PROC_RUID(p, (void *)args->uidval) == 0))
+			return(PROC_RETURNED);
+		if ((args->uidcheck != 0)  && (sysdoproc_filt_KERN_PROC_UID(p, (void *)args->uidval) == 0))
+			return(PROC_RETURNED);
+		if ((args->ttycheck != 0)  && (sysdoproc_filt_KERN_PROC_TTY(p, (void *)args->uidval) == 0))
+			return(PROC_RETURNED);
+
+		bzero(args->kprocp, args->sizeof_kproc);
+		if (args->is_64_bit) {
+			fill_user_proc(p, (struct user_kinfo_proc *) args->kprocp);
+		}
+		else {
+			fill_proc(p, (struct kinfo_proc *) args->kprocp);
+		}
+		error = copyout(args->kprocp, args->dp, args->sizeof_kproc);
+		if (error) {
+			*args->errorp = error;
+			return(PROC_RETURNED_DONE);
+			return (error);
+		}
+		args->dp += args->sizeof_kproc;
+		args->buflen -= args->sizeof_kproc;
+	}
+	args->needed += args->sizeof_kproc;
+	return(PROC_RETURNED);
+}
 
 int
 sysctl_doproc(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 {
-	struct proc *p;
 	user_addr_t dp = where;
 	size_t needed = 0;
 	int buflen = where != USER_ADDR_NULL ? *sizep : 0;
-	int doingzomb;
 	int error = 0;
 	boolean_t is_64_bit = FALSE;
 	struct kinfo_proc       kproc;
 	struct user_kinfo_proc  user_kproc;
 	int sizeof_kproc;
 	caddr_t kprocp;
+	int (*filterfn)(proc_t, void *) = 0;
+	struct sysdoproc_args args;
+	int uidcheck = 0;
+	int ruidcheck = 0;
+	int ttycheck = 0;
 
 	if (namelen != 2 && !(namelen == 1 && name[0] == KERN_PROC_ALL))
 		return (EINVAL);
-	p = allproc.lh_first;
-	doingzomb = 0;
 	is_64_bit = proc_is64bit(current_proc()); 
 	if (is_64_bit) {
 		sizeof_kproc = sizeof(user_kproc);
@@ -1328,72 +1206,58 @@ sysctl_doproc(int *name, u_int namelen, user_addr_t where, size_t *sizep)
 		sizeof_kproc = sizeof(kproc);
 		kprocp = (caddr_t) &kproc;
 	}
-again:
-	for (; p != 0; p = p->p_list.le_next) {
-		/*
-		 * Skip embryonic processes.
-		 */
-		if (p->p_stat == SIDL)
-			continue;
-		/*
-		 * TODO - make more efficient (see notes below).
-		 * do by session.
-		 */
-		switch (name[0]) {
+
+
+	switch (name[0]) {
 
 		case KERN_PROC_PID:
-			/* could do this with just a lookup */
-			if (p->p_pid != (pid_t)name[1])
-				continue;
+			filterfn = sysdoproc_filt_KERN_PROC_PID;
 			break;
 
 		case KERN_PROC_PGRP:
-			/* could do this by traversing pgrp */
-			if (p->p_pgrp->pg_id != (pid_t)name[1])
-				continue;
+			filterfn = sysdoproc_filt_KERN_PROC_PGRP;
 			break;
-
+	
 		case KERN_PROC_TTY:
-			if ((p->p_flag & P_CONTROLT) == 0 ||
-				(p->p_session == NULL) ||
-			    p->p_session->s_ttyp == NULL ||
-			    p->p_session->s_ttyp->t_dev != (dev_t)name[1])
-				continue;
+			ttycheck = 1;
 			break;
 
 		case KERN_PROC_UID:
-			if ((p->p_ucred == NULL) ||
-				(kauth_cred_getuid(p->p_ucred) != (uid_t)name[1]))
-				continue;
+			uidcheck = 1;
 			break;
 
 		case KERN_PROC_RUID:
-			if ((p->p_ucred == NULL) ||
-				(p->p_ucred->cr_ruid != (uid_t)name[1]))
-				continue;
+			ruidcheck = 1;
 			break;
-		}
-		if (buflen >= sizeof_kproc) {
-			bzero(kprocp, sizeof_kproc);
-			if (is_64_bit) {
-				fill_user_proc(p, (struct user_kinfo_proc *) kprocp);
-			}
-			else {
-				fill_proc(p, (struct kinfo_proc *) kprocp);
-			}
-			error = copyout(kprocp, dp, sizeof_kproc);
-			if (error)
-				return (error);
-			dp += sizeof_kproc;
-			buflen -= sizeof_kproc;
-		}
-		needed += sizeof_kproc;
-	}
-	if (doingzomb == 0) {
-		p = zombproc.lh_first;
-		doingzomb++;
-		goto again;
+
+#if CONFIG_LCTX
+		case KERN_PROC_LCID:
+			filterfn = sysdoproc_filt_KERN_PROC_LCID;
+			break;
+#endif
 	}
+
+	error = 0;
+	args.buflen = buflen;
+	args.kprocp = kprocp;
+	args.is_64_bit = is_64_bit;
+	args.dp = dp;
+	args.needed = needed;
+	args.errorp = &error;
+	args.uidcheck = uidcheck;
+	args.ruidcheck = ruidcheck;
+	args.ttycheck = ttycheck;
+	args.sizeof_kproc = sizeof_kproc;
+	args.uidval = name[1];
+
+	proc_iterate((PROC_ALLPROCLIST | PROC_ZOMBPROCLIST), sysdoproc_callback, &args, filterfn, (void *)name[1]);
+
+	if (error)
+		return(error);
+
+	dp = args.dp;
+	needed = args.needed;
+	
 	if (where != USER_ADDR_NULL) {
 		*sizep = dp - where;
 		if (needed > *sizep)
@@ -1409,42 +1273,54 @@ again:
  * Fill in an eproc structure for the specified process.
  */
 static void
-fill_eproc(p, ep)
-	register struct proc *p;
-	register struct eproc *ep;
+fill_eproc(proc_t p, struct eproc *ep)
 {
-	register struct tty *tp;
+	struct tty *tp;
+	kauth_cred_t my_cred;
+	struct pgrp * pg;
+	struct session * sessp;
+
+	pg = proc_pgrp(p);
+	sessp = proc_session(p);
 
 	ep->e_paddr = p;
-	if (p->p_pgrp) {
-		ep->e_sess = p->p_pgrp->pg_session;
-		ep->e_pgid = p->p_pgrp->pg_id;
-		ep->e_jobc = p->p_pgrp->pg_jobc;
-		if (ep->e_sess && ep->e_sess->s_ttyvp)
+
+	if (pg != PGRP_NULL) {
+		ep->e_sess = sessp;
+		ep->e_pgid = p->p_pgrpid;
+		ep->e_jobc = pg->pg_jobc;
+		if ((sessp != SESSION_NULL) && sessp->s_ttyvp)
 			ep->e_flag = EPROC_CTTY;
 	} else {
 		ep->e_sess = (struct session *)0;
 		ep->e_pgid = 0;
 		ep->e_jobc = 0;
 	}
-	ep->e_ppid = (p->p_pptr) ? p->p_pptr->p_pid : 0;
+#if CONFIG_LCTX
+	if (p->p_lctx) {
+		ep->e_lcid = p->p_lctx->lc_id;
+	} else {
+		ep->e_lcid = 0;
+	}
+#endif
+	ep->e_ppid = p->p_ppid;
 	/* Pre-zero the fake historical pcred */
 	bzero(&ep->e_pcred, sizeof(struct _pcred));
 	if (p->p_ucred) {
-		/* XXX not ref-counted */
+		my_cred = kauth_cred_proc_ref(p);
 
 		/* A fake historical pcred */
-		ep->e_pcred.p_ruid = p->p_ucred->cr_ruid;
-		ep->e_pcred.p_svuid = p->p_ucred->cr_svuid;
-		ep->e_pcred.p_rgid = p->p_ucred->cr_rgid;
-		ep->e_pcred.p_svgid = p->p_ucred->cr_svgid;
-
+		ep->e_pcred.p_ruid = my_cred->cr_ruid;
+		ep->e_pcred.p_svuid = my_cred->cr_svuid;
+		ep->e_pcred.p_rgid = my_cred->cr_rgid;
+		ep->e_pcred.p_svgid = my_cred->cr_svgid;
 		/* A fake historical *kauth_cred_t */
-		ep->e_ucred.cr_ref = p->p_ucred->cr_ref;
-		ep->e_ucred.cr_uid = kauth_cred_getuid(p->p_ucred);
-		ep->e_ucred.cr_ngroups = p->p_ucred->cr_ngroups;
-		bcopy(p->p_ucred->cr_groups, ep->e_ucred.cr_groups, NGROUPS*sizeof(gid_t));
+		ep->e_ucred.cr_ref = my_cred->cr_ref;
+		ep->e_ucred.cr_uid = kauth_cred_getuid(my_cred);
+		ep->e_ucred.cr_ngroups = my_cred->cr_ngroups;
+		bcopy(my_cred->cr_groups, ep->e_ucred.cr_groups, NGROUPS*sizeof(gid_t));
 
+		kauth_cred_unref(&my_cred);
 	}
 	if (p->p_stat == SIDL || p->p_stat == SZOMB) {
 		ep->e_vm.vm_tsize = 0;
@@ -1453,39 +1329,46 @@ fill_eproc(p, ep)
 	}
 	ep->e_vm.vm_rssize = 0;
 
-	if ((p->p_flag & P_CONTROLT) && (ep->e_sess) &&
-	     (tp = ep->e_sess->s_ttyp)) {
+	if ((p->p_flag & P_CONTROLT) && (sessp != SESSION_NULL) &&
+	     (tp = sessp->s_ttyp)) {
 		ep->e_tdev = tp->t_dev;
-		ep->e_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PID;
+		ep->e_tpgid = sessp->s_ttypgrpid;
 		ep->e_tsess = tp->t_session;
 	} else
 		ep->e_tdev = NODEV;
 
-	if (SESS_LEADER(p))
+	if (SESS_LEADER(p, sessp))
 		ep->e_flag |= EPROC_SLEADER;
-	if (p->p_wmesg)
-		strncpy(ep->e_wmesg, p->p_wmesg, WMESGLEN);
+	bzero(&ep->e_wmesg[0], WMESGLEN+1);
 	ep->e_xsize = ep->e_xrssize = 0;
 	ep->e_xccount = ep->e_xswrss = 0;
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+	if(pg != PGRP_NULL)
+		pg_rele(pg);
 }
 
 /*
  * Fill in an LP64 version of eproc structure for the specified process.
  */
 static void
-fill_user_eproc(register struct proc *p, register struct user_eproc *ep)
+fill_user_eproc(proc_t p, struct user_eproc *ep)
 {
-	register struct tty *tp;
-	struct	session *sessionp = NULL;
+	struct tty *tp;
+	struct	session *sessp = NULL;
+	struct pgrp * pg;
+	kauth_cred_t my_cred;
+	
+	pg = proc_pgrp(p);
+	sessp = proc_session(p);
 
 	ep->e_paddr = CAST_USER_ADDR_T(p);
-	if (p->p_pgrp) {
-	    sessionp = p->p_pgrp->pg_session;
-		ep->e_sess = CAST_USER_ADDR_T(sessionp);
-		ep->e_pgid = p->p_pgrp->pg_id;
-		ep->e_jobc = p->p_pgrp->pg_jobc;
-		if (sessionp) {
-            if (sessionp->s_ttyvp)
+	if (pg != PGRP_NULL) {
+		ep->e_sess = CAST_USER_ADDR_T(sessp);
+		ep->e_pgid = p->p_pgrpid;
+		ep->e_jobc = pg->pg_jobc;
+		if (sessp != SESSION_NULL) {
+            		if (sessp->s_ttyvp)
 			    ep->e_flag = EPROC_CTTY;
 		}
 	} else {
@@ -1493,24 +1376,32 @@ fill_user_eproc(register struct proc *p, register struct user_eproc *ep)
 		ep->e_pgid = 0;
 		ep->e_jobc = 0;
 	}
-	ep->e_ppid = (p->p_pptr) ? p->p_pptr->p_pid : 0;
+#if CONFIG_LCTX
+	if (p->p_lctx) {
+		ep->e_lcid = p->p_lctx->lc_id;
+	} else {
+		ep->e_lcid = 0;
+	}
+#endif
+	ep->e_ppid = p->p_ppid;
 	/* Pre-zero the fake historical pcred */
 	bzero(&ep->e_pcred, sizeof(ep->e_pcred));
 	if (p->p_ucred) {
-		/* XXX not ref-counted */
+		my_cred = kauth_cred_proc_ref(p);
 
 		/* A fake historical pcred */
-		ep->e_pcred.p_ruid = p->p_ucred->cr_ruid;
-		ep->e_pcred.p_svuid = p->p_ucred->cr_svuid;
-		ep->e_pcred.p_rgid = p->p_ucred->cr_rgid;
-		ep->e_pcred.p_svgid = p->p_ucred->cr_svgid;
+		ep->e_pcred.p_ruid = my_cred->cr_ruid;
+		ep->e_pcred.p_svuid = my_cred->cr_svuid;
+		ep->e_pcred.p_rgid = my_cred->cr_rgid;
+		ep->e_pcred.p_svgid = my_cred->cr_svgid;
 
 		/* A fake historical *kauth_cred_t */
-		ep->e_ucred.cr_ref = p->p_ucred->cr_ref;
-		ep->e_ucred.cr_uid = kauth_cred_getuid(p->p_ucred);
-		ep->e_ucred.cr_ngroups = p->p_ucred->cr_ngroups;
-		bcopy(p->p_ucred->cr_groups, ep->e_ucred.cr_groups, NGROUPS*sizeof(gid_t));
+		ep->e_ucred.cr_ref = my_cred->cr_ref;
+		ep->e_ucred.cr_uid = kauth_cred_getuid(my_cred);
+		ep->e_ucred.cr_ngroups = my_cred->cr_ngroups;
+		bcopy(my_cred->cr_groups, ep->e_ucred.cr_groups, NGROUPS*sizeof(gid_t));
 
+		kauth_cred_unref(&my_cred);
 	}
 	if (p->p_stat == SIDL || p->p_stat == SZOMB) {
 		ep->e_vm.vm_tsize = 0;
@@ -1519,60 +1410,71 @@ fill_user_eproc(register struct proc *p, register struct user_eproc *ep)
 	}
 	ep->e_vm.vm_rssize = 0;
 
-	if ((p->p_flag & P_CONTROLT) && (sessionp) &&
-	     (tp = sessionp->s_ttyp)) {
+	if ((p->p_flag & P_CONTROLT) && (sessp != SESSION_NULL) &&
+	     (tp = sessp->s_ttyp)) {
 		ep->e_tdev = tp->t_dev;
-		ep->e_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PID;
+		ep->e_tpgid = sessp->s_ttypgrpid;
 		ep->e_tsess = CAST_USER_ADDR_T(tp->t_session);
 	} else
 		ep->e_tdev = NODEV;
 
-	if (SESS_LEADER(p))
+	if (SESS_LEADER(p, sessp))
 		ep->e_flag |= EPROC_SLEADER;
-	if (p->p_wmesg)
-		strncpy(ep->e_wmesg, p->p_wmesg, WMESGLEN);
+	bzero(&ep->e_wmesg[0], WMESGLEN+1);
 	ep->e_xsize = ep->e_xrssize = 0;
 	ep->e_xccount = ep->e_xswrss = 0;
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+	if (pg != PGRP_NULL)
+		pg_rele(pg);
 }
 
 /*
  * Fill in an eproc structure for the specified process.
  */
 static void
-fill_externproc(p, exp)
-	register struct proc *p;
-	register struct extern_proc *exp;
+fill_externproc(proc_t p, struct extern_proc *exp)
 {
 	exp->p_forw = exp->p_back = NULL;
-	if (p->p_stats)
-		exp->p_starttime = p->p_stats->p_start;
+	exp->p_starttime = p->p_start;
 	exp->p_vmspace = NULL;
 	exp->p_sigacts = p->p_sigacts;
 	exp->p_flag  = p->p_flag;
+	if (p->p_lflag & P_LTRACED)
+		exp->p_flag |= P_TRACED;
+	if (p->p_lflag & P_LPPWAIT)
+		exp->p_flag |= P_PPWAIT;
+	if (p->p_lflag & P_LEXIT)
+		exp->p_flag |= P_WEXIT;
 	exp->p_stat  = p->p_stat ;
 	exp->p_pid  = p->p_pid ;
 	exp->p_oppid  = p->p_oppid ;
-	exp->p_dupfd  = p->p_dupfd ;
 	/* Mach related  */
 	exp->user_stack  = CAST_DOWN(caddr_t, p->user_stack);
 	exp->exit_thread  = p->exit_thread ;
 	exp->p_debugger  = p->p_debugger ;
 	exp->sigwait  = p->sigwait ;
 	/* scheduling */
+#ifdef _PROC_HAS_SCHEDINFO_
 	exp->p_estcpu  = p->p_estcpu ;
-	exp->p_cpticks  = p->p_cpticks ;
 	exp->p_pctcpu  = p->p_pctcpu ;
-	exp->p_wchan  = p->p_wchan ;
-	exp->p_wmesg  = p->p_wmesg ;
-	exp->p_swtime  = p->p_swtime ;
 	exp->p_slptime  = p->p_slptime ;
+#else
+	exp->p_estcpu  = 0 ;
+	exp->p_pctcpu  = 0 ;
+	exp->p_slptime = 0 ;
+#endif
+	exp->p_cpticks  = 0 ;
+	exp->p_wchan  = 0 ;
+	exp->p_wmesg  = 0 ;
+	exp->p_swtime  = 0 ;
 	bcopy(&p->p_realtimer, &exp->p_realtimer,sizeof(struct itimerval));
 	bcopy(&p->p_rtime, &exp->p_rtime,sizeof(struct timeval));
-	exp->p_uticks  = p->p_uticks ;
-	exp->p_sticks  = p->p_sticks ;
-	exp->p_iticks  = p->p_iticks ;
-	exp->p_traceflag  = p->p_traceflag ;
-	exp->p_tracep  = p->p_tracep ;
+	exp->p_uticks  = 0 ;
+	exp->p_sticks  = 0 ;
+	exp->p_iticks  = 0 ;
+	exp->p_traceflag  = 0;
+	exp->p_tracep  = 0 ;
 	exp->p_siglist  = 0 ;	/* No longer relevant */
 	exp->p_textvp  = p->p_textvp ;
 	exp->p_holdcnt = 0 ;
@@ -1580,7 +1482,7 @@ fill_externproc(p, exp)
 	exp->p_sigignore  = p->p_sigignore ;
 	exp->p_sigcatch  = p->p_sigcatch ;
 	exp->p_priority  = p->p_priority ;
-	exp->p_usrpri  = p->p_usrpri ;
+	exp->p_usrpri  = 0 ;
 	exp->p_nice  = p->p_nice ;
 	bcopy(&p->p_comm, &exp->p_comm,MAXCOMLEN);
 	exp->p_comm[MAXCOMLEN] = '\0';
@@ -1595,44 +1497,53 @@ fill_externproc(p, exp)
  * Fill in an LP64 version of extern_proc structure for the specified process.
  */
 static void
-fill_user_externproc(register struct proc *p, register struct user_extern_proc *exp)
+fill_user_externproc(proc_t p, struct user_extern_proc *exp)
 {
 	exp->p_forw = exp->p_back = USER_ADDR_NULL;
-	if (p->p_stats) {
-		exp->p_starttime.tv_sec = p->p_stats->p_start.tv_sec;
-		exp->p_starttime.tv_usec = p->p_stats->p_start.tv_usec;
-	}
+	exp->p_starttime.tv_sec = p->p_start.tv_sec;
+	exp->p_starttime.tv_usec = p->p_start.tv_usec;
 	exp->p_vmspace = USER_ADDR_NULL;
 	exp->p_sigacts = CAST_USER_ADDR_T(p->p_sigacts);
 	exp->p_flag  = p->p_flag;
+	if (p->p_lflag & P_LTRACED)
+		exp->p_flag |= P_TRACED;
+	if (p->p_lflag & P_LPPWAIT)
+		exp->p_flag |= P_PPWAIT;
+	if (p->p_lflag & P_LEXIT)
+		exp->p_flag |= P_WEXIT;
 	exp->p_stat  = p->p_stat ;
 	exp->p_pid  = p->p_pid ;
 	exp->p_oppid  = p->p_oppid ;
-	exp->p_dupfd  = p->p_dupfd ;
 	/* Mach related  */
 	exp->user_stack  = p->user_stack;
 	exp->exit_thread  = CAST_USER_ADDR_T(p->exit_thread);
 	exp->p_debugger  = p->p_debugger ;
 	exp->sigwait  = p->sigwait ;
 	/* scheduling */
+#ifdef _PROC_HAS_SCHEDINFO_
 	exp->p_estcpu  = p->p_estcpu ;
-	exp->p_cpticks  = p->p_cpticks ;
 	exp->p_pctcpu  = p->p_pctcpu ;
-	exp->p_wchan  = CAST_USER_ADDR_T(p->p_wchan);
-	exp->p_wmesg  = CAST_USER_ADDR_T(p->p_wmesg);
-	exp->p_swtime  = p->p_swtime ;
 	exp->p_slptime  = p->p_slptime ;
+#else
+	exp->p_estcpu  = 0 ;
+	exp->p_pctcpu  = 0 ;
+	exp->p_slptime = 0 ;
+#endif
+	exp->p_cpticks  = 0 ;
+	exp->p_wchan  = 0;
+	exp->p_wmesg  = 0;
+	exp->p_swtime  = 0 ;
 	exp->p_realtimer.it_interval.tv_sec = p->p_realtimer.it_interval.tv_sec;
 	exp->p_realtimer.it_interval.tv_usec = p->p_realtimer.it_interval.tv_usec;
 	exp->p_realtimer.it_value.tv_sec = p->p_realtimer.it_value.tv_sec;
 	exp->p_realtimer.it_value.tv_usec = p->p_realtimer.it_value.tv_usec;
 	exp->p_rtime.tv_sec = p->p_rtime.tv_sec;
 	exp->p_rtime.tv_usec = p->p_rtime.tv_usec;
-	exp->p_uticks  = p->p_uticks ;
-	exp->p_sticks  = p->p_sticks ;
-	exp->p_iticks  = p->p_iticks ;
-	exp->p_traceflag  = p->p_traceflag ;
-	exp->p_tracep  = CAST_USER_ADDR_T(p->p_tracep);
+	exp->p_uticks  = 0 ;
+	exp->p_sticks  = 0 ;
+	exp->p_iticks  = 0 ;
+	exp->p_traceflag  = 0 ;
+	exp->p_tracep  = 0;
 	exp->p_siglist  = 0 ;	/* No longer relevant */
 	exp->p_textvp  = CAST_USER_ADDR_T(p->p_textvp);
 	exp->p_holdcnt = 0 ;
@@ -1640,7 +1551,7 @@ fill_user_externproc(register struct proc *p, register struct user_extern_proc *
 	exp->p_sigignore  = p->p_sigignore ;
 	exp->p_sigcatch  = p->p_sigcatch ;
 	exp->p_priority  = p->p_priority ;
-	exp->p_usrpri  = p->p_usrpri ;
+	exp->p_usrpri  = 0 ;
 	exp->p_nice  = p->p_nice ;
 	bcopy(&p->p_comm, &exp->p_comm,MAXCOMLEN);
 	exp->p_comm[MAXCOMLEN] = '\0';
@@ -1652,16 +1563,14 @@ fill_user_externproc(register struct proc *p, register struct user_extern_proc *
 }
 
 static void
-fill_proc(p, kp)
-	register struct proc *p;
-	register struct kinfo_proc *kp;
+fill_proc(proc_t p, struct kinfo_proc *kp)
 {
 	fill_externproc(p, &kp->kp_proc);
 	fill_eproc(p, &kp->kp_eproc);
 }
 
 static void
-fill_user_proc(register struct proc *p, register struct user_kinfo_proc *kp)
+fill_user_proc(proc_t p, struct user_kinfo_proc *kp)
 {
 	fill_user_externproc(p, &kp->kp_proc);
 	fill_user_eproc(p, &kp->kp_eproc);
@@ -1669,7 +1578,7 @@ fill_user_proc(register struct proc *p, register struct user_kinfo_proc *kp)
 
 int
 kdebug_ops(int *name, u_int namelen, user_addr_t where, 
-           size_t *sizep, struct proc *p)
+           size_t *sizep, proc_t p)
 {
 	int ret=0;
 
@@ -1702,36 +1611,6 @@ kdebug_ops(int *name, u_int namelen, user_addr_t where,
 	return(ret);
 }
 
-extern int pcsamples_control(int *name, u_int namelen, user_addr_t where,
-                             size_t * sizep);
-
-int
-pcsamples_ops(int *name, u_int namelen, user_addr_t where, 
-			  size_t *sizep, struct proc *p)
-{
-	int ret=0;
-
-    ret = suser(kauth_cred_get(), &p->p_acflag);
-	if (ret)
-		return(ret);
-
-	switch(name[0]) {
-	case KERN_PCDISABLE:
-	case KERN_PCGETBUF:
-	case KERN_PCSETUP:
-	case KERN_PCREMOVE:
-	case KERN_PCREADBUF:
-	case KERN_PCSETREG:
-	case KERN_PCSETBUF:
-	case KERN_PCCOMM:
-	        ret = pcsamples_control(name, namelen, where, sizep);
-	        break;
-	default:
-		ret= ENOTSUP;
-		break;
-	}
-	return(ret);
-}
 
 /*
  * Return the top *sizep bytes of the user stack, or the entire area of the
@@ -1739,35 +1618,38 @@ pcsamples_ops(int *name, u_int namelen, user_addr_t where,
  */
 int
 sysctl_procargs(int *name, u_int namelen, user_addr_t where, 
-                size_t *sizep, struct proc *cur_proc)
+                size_t *sizep, proc_t cur_proc)
 {
 	return sysctl_procargsx( name, namelen, where, sizep, cur_proc, 0);
 }
 
 static int
 sysctl_procargs2(int *name, u_int namelen, user_addr_t where, 
-                 size_t *sizep, struct proc *cur_proc)
+                 size_t *sizep, proc_t cur_proc)
 {
 	return sysctl_procargsx( name, namelen, where, sizep, cur_proc, 1);
 }
 
 static int
 sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where, 
-                 size_t *sizep, struct proc *cur_proc, int argc_yes)
+                 size_t *sizep, proc_t cur_proc, int argc_yes)
 {
-	struct proc *p;
+	proc_t p;
 	int buflen = where != USER_ADDR_NULL ? *sizep : 0;
 	int error = 0;
-	struct vm_map *proc_map;
+	struct _vm_map *proc_map;
 	struct task * task;
 	vm_map_copy_t	tmp;
 	user_addr_t	arg_addr;
 	size_t		arg_size;
 	caddr_t data;
+	size_t argslen=0;
 	int size;
 	vm_offset_t	copy_start, copy_end;
 	kern_return_t ret;
 	int pid;
+	kauth_cred_t my_cred;
+	uid_t uid;
 
 	if (argc_yes)
 		buflen -= sizeof(int);		/* reserve first word to return argc */
@@ -1785,7 +1667,7 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 	 *	Lookup process by pid
 	 */
 	pid = name[0];
-	p = pfind(pid);
+	p = proc_find(pid);
 	if (p == NULL) {
 		return(EINVAL);
 	}
@@ -1800,15 +1682,20 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 	 *	size.
 	 */
 
-	if (!p->user_stack)
+	if (!p->user_stack) {
+		proc_rele(p);
 		return(EINVAL);
+	}
 
 	if (where == USER_ADDR_NULL) {
 		/* caller only wants to know length of proc args data */
-		if (sizep == NULL)
+		if (sizep == NULL) {
+			proc_rele(p);
 			return(EFAULT);
+		}
 			
 		 size = p->p_argslen;
+		proc_rele(p);
 		 if (argc_yes) {
 		 	size += sizeof(int);
 		 }
@@ -1824,9 +1711,15 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 		return (0);
 	}
 	
-	if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
-		&& suser(kauth_cred_get(), &cur_proc->p_acflag))
+	my_cred = kauth_cred_proc_ref(p);
+	uid = kauth_cred_getuid(my_cred);
+	kauth_cred_unref(&my_cred);
+
+	if ((uid != kauth_cred_getuid(kauth_cred_get())) 
+		&& suser(kauth_cred_get(), &cur_proc->p_acflag)) {
+		proc_rele(p);
 		return (EINVAL);
+	}
 
 	if ((u_int)arg_size > p->p_argslen)
 	        arg_size = round_page(p->p_argslen);
@@ -1840,9 +1733,12 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 	 *	that by getting a reference on the task itself.
 	 */
 	task = p->task;
-	if (task == NULL)
+	if (task == NULL) {
+		proc_rele(p);
 		return(EINVAL);
+	}
 	
+	argslen = p->p_argslen;
 	/*
 	 * Once we have a task reference we can convert that into a
 	 * map reference, which we will use in the calls below.  The
@@ -1851,8 +1747,10 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 	 * of stale info (which is always a possibility).
 	 */
 	task_reference(task);
+	proc_rele(p);
 	proc_map = get_task_map_reference(task);
 	task_deallocate(task);
+	
 	if (proc_map == NULL)
 		return(EINVAL);
 
@@ -1887,9 +1785,9 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 			return (EIO);
 	}
 
-	if (arg_size > p->p_argslen) {
-		data = (caddr_t) (copy_end - p->p_argslen);
-		size = p->p_argslen;
+	if (arg_size > argslen) {
+		data = (caddr_t) (copy_end - argslen);
+		size = argslen;
 	} else {
 		data = (caddr_t) (copy_end - arg_size);
 		size = arg_size;
@@ -1911,7 +1809,7 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 		 * 
 		 * Note: we keep all pointers&sizes aligned to word boundries
 		 */
-		if ( (! error) && (buflen > 0 && (u_int)buflen > p->p_argslen) )
+		if ( (! error) && (buflen > 0 && (u_int)buflen > argslen) )
 		{
 			int binPath_sz, alignedBinPath_sz = 0;
 			int extraSpaceNeeded, addThis;
@@ -1949,7 +1847,7 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 			extraSpaceNeeded = alignedBinPath_sz + addThis + binPath_sz + (4 * sizeof(int));
 
 			/* is there is room to tack on argv[0]? */
-			if ( (buflen & ~(sizeof(int)-1)) >= ( p->p_argslen + extraSpaceNeeded ))
+			if ( (buflen & ~(sizeof(int)-1)) >= ( argslen + extraSpaceNeeded ))
 			{
 				placeHere += addThis;
 				suword(placeHere, 0);
@@ -1983,221 +1881,628 @@ sysctl_procargsx(int *name, __unused u_int namelen, user_addr_t where,
 
 
 /*
- * Validate parameters and get old / set new parameters
- * for max number of concurrent aio requests.  Makes sure
- * the system wide limit is greater than the per process
- * limit.
+ * Max number of concurrent aio requests
  */
 static int
-sysctl_aiomax(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen)
+sysctl_aiomax
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int		new_value;
-
-	if ( oldp && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp )
-		error = copyout( &aio_max_requests, oldp, sizeof(int) );
-	if ( error == 0 && newp )
-		error = copyin( newp, &new_value, sizeof(int) );
-	if ( error == 0 && newp ) {
-		if ( new_value >= aio_max_requests_per_process )
+	int new_value, changed;
+	int error = sysctl_io_number(req, aio_max_requests, sizeof(int), &new_value, &changed);
+	if (changed) {
+		 /* make sure the system-wide limit is greater than the per process limit */
+		if (new_value >= aio_max_requests_per_process)
 			aio_max_requests = new_value;
 		else
 			error = EINVAL;
 	}
-	return( error );
-	
-} /* sysctl_aiomax */
+	return(error);
+}
 
 
 /*
- * Validate parameters and get old / set new parameters
- * for max number of concurrent aio requests per process.  
- * Makes sure per process limit is less than the system wide
- * limit.
+ * Max number of concurrent aio requests per process
  */
 static int
-sysctl_aioprocmax(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen )
+sysctl_aioprocmax
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int		new_value = 0;
-
-	if ( oldp && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp )
-		error = copyout( &aio_max_requests_per_process, oldp, sizeof(int) );
-	if ( error == 0 && newp )
-		error = copyin( newp, &new_value, sizeof(int) );
-	if ( error == 0 && newp ) {
-		if ( new_value <= aio_max_requests && new_value >= AIO_LISTIO_MAX )
+	int new_value, changed;
+	int error = sysctl_io_number(req, aio_max_requests_per_process, sizeof(int), &new_value, &changed);
+	if (changed) {
+		/* make sure per process limit is less than the system-wide limit */
+		if (new_value <= aio_max_requests && new_value >= AIO_LISTIO_MAX)
 			aio_max_requests_per_process = new_value;
 		else
 			error = EINVAL;
 	}
-	return( error );
-	
-} /* sysctl_aioprocmax */
+	return(error);
+}
 
 
 /*
- * Validate parameters and get old / set new parameters
- * for max number of async IO worker threads.  
- * We only allow an increase in the number of worker threads.
+ * Max number of async IO worker threads
  */
 static int
-sysctl_aiothreads(user_addr_t oldp, size_t *oldlenp, user_addr_t newp, size_t newlen)
+sysctl_aiothreads
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int		new_value;
-
-	if ( oldp && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp )
-		error = copyout( &aio_worker_threads, oldp, sizeof(int) );
-	if ( error == 0 && newp )
-		error = copyin( newp, &new_value, sizeof(int) );
-	if ( error == 0 && newp ) {
+	int new_value, changed;
+	int error = sysctl_io_number(req, aio_worker_threads, sizeof(int), &new_value, &changed);
+	if (changed) {
+		/* we only allow an increase in the number of worker threads */
 	        if (new_value > aio_worker_threads ) {
-		        _aio_create_worker_threads( (new_value - aio_worker_threads) );
+		        _aio_create_worker_threads((new_value - aio_worker_threads));
 			aio_worker_threads = new_value;
 		}
 		else
 		        error = EINVAL;
 	}
-	return( error );
-	
-} /* sysctl_aiothreads */
+	return(error);
+}
 
 
 /*
- * Validate parameters and get old / set new parameters
- * for max number of processes per UID.
- * Makes sure per UID limit is less than the system wide limit.
+ * System-wide limit on the max number of processes
  */
 static int
-sysctl_maxprocperuid(user_addr_t oldp, size_t *oldlenp, 
-                     user_addr_t newp, size_t newlen)
+sysctl_maxproc
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int		new_value;
-
-	if ( oldp != USER_ADDR_NULL && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp != USER_ADDR_NULL && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp != USER_ADDR_NULL )
-		error = copyout( &maxprocperuid, oldp, sizeof(int) );
-	if ( error == 0 && newp != USER_ADDR_NULL ) {
-		error = copyin( newp, &new_value, sizeof(int) );
-		if ( error == 0 ) {
-			AUDIT_ARG(value, new_value);
-			if ( new_value <= maxproc && new_value > 0 )
-				maxprocperuid = new_value;
-			else
-				error = EINVAL;
-		}
-		else 
+	int new_value, changed;
+	int error = sysctl_io_number(req, maxproc, sizeof(int), &new_value, &changed);
+	if (changed) {
+		AUDIT_ARG(value, new_value);
+		/* make sure the system-wide limit is less than the configured hard
+		   limit set at kernel compilation */
+		if (new_value <= hard_maxproc && new_value > 0)
+			maxproc = new_value;
+		else
 			error = EINVAL;
 	}
-	return( error );
-	
-} /* sysctl_maxprocperuid */
+	return(error);
+}
 
+SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		ostype, 0, "");
+SYSCTL_STRING(_kern, KERN_OSRELEASE, osrelease, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		osrelease, 0, "");
+SYSCTL_INT(_kern, KERN_OSREV, osrevision, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, BSD, "");
+SYSCTL_STRING(_kern, KERN_VERSION, version, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		version, 0, "");
+
+/* PR-5293665: need to use a callback function for kern.osversion to set
+ * osversion in IORegistry */
 
-/*
- * Validate parameters and get old / set new parameters
- * for max number of files per process.
- * Makes sure per process limit is less than the system-wide limit.
- */
 static int
-sysctl_maxfilesperproc(user_addr_t oldp, size_t *oldlenp, 
-                       user_addr_t newp, size_t newlen)
+sysctl_osversion(__unused struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int		new_value;
+    int rval = 0;
 
-	if ( oldp != USER_ADDR_NULL && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp != USER_ADDR_NULL && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp != USER_ADDR_NULL )
-		error = copyout( &maxfilesperproc, oldp, sizeof(int) );
-	if ( error == 0 && newp != USER_ADDR_NULL ) {
-		error = copyin( newp, &new_value, sizeof(int) );
-		if ( error == 0 ) {
-			AUDIT_ARG(value, new_value);
-			if ( new_value < maxfiles && new_value > 0 )
-				maxfilesperproc = new_value;
-			else
-				error = EINVAL;
+    rval = sysctl_handle_string(oidp, arg1, arg2, req);
+
+    if (req->newptr) {
+        IORegistrySetOSBuildVersion((char *)arg1); 
+    }
+
+    return rval;
+}
+
+SYSCTL_PROC(_kern, KERN_OSVERSION, osversion,
+        CTLFLAG_RW | CTLFLAG_KERN | CTLTYPE_STRING,
+        osversion, 256 /* OSVERSIZE*/, 
+        sysctl_osversion, "A", "");
+
+static int
+sysctl_sysctl_bootargs
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int error;
+	char buf[256];
+
+	strlcpy(buf, PE_boot_args(), 256);
+	error = sysctl_io_string(req, buf, 256, 0, NULL);
+	return(error);
+}
+
+SYSCTL_PROC(_kern, OID_AUTO, bootargs,
+	CTLFLAG_LOCKED | CTLFLAG_RD | CTLFLAG_KERN | CTLTYPE_STRING,
+	NULL, 0,
+	sysctl_sysctl_bootargs, "A", "bootargs");
+
+SYSCTL_INT(_kern, KERN_MAXFILES, maxfiles, 
+		CTLFLAG_RW | CTLFLAG_KERN, 
+		&maxfiles, 0, "");
+SYSCTL_INT(_kern, KERN_ARGMAX, argmax, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, ARG_MAX, "");
+SYSCTL_INT(_kern, KERN_POSIX1, posix1version, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, _POSIX_VERSION, "");
+SYSCTL_INT(_kern, KERN_NGROUPS, ngroups, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, NGROUPS_MAX, "");
+SYSCTL_INT(_kern, KERN_JOB_CONTROL, job_control, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, 1, "");
+#if 1	/* _POSIX_SAVED_IDS from <unistd.h> */
+SYSCTL_INT(_kern, KERN_SAVED_IDS, saved_ids, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, 1, "");
+#else
+SYSCTL_INT(_kern, KERN_SAVED_IDS, saved_ids, 
+		CTLFLAG_RD | CTLFLAG_KERN, 
+		NULL, 0, "");
+#endif
+
+static int
+sysctl_maxvnodes (__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	unsigned int oldval = desiredvnodes;
+	int error = sysctl_io_number(req, desiredvnodes, sizeof(int), &desiredvnodes, NULL);
+	reset_vmobjectcache(oldval, desiredvnodes);
+	resize_namecache(desiredvnodes);
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_maxvnodes, "I", "");
+
+SYSCTL_PROC(_kern, KERN_MAXPROC, maxproc,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_maxproc, "I", "");
+
+SYSCTL_PROC(_kern, KERN_AIOMAX, aiomax,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_aiomax, "I", "");
+
+SYSCTL_PROC(_kern, KERN_AIOPROCMAX, aioprocmax,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_aioprocmax, "I", "");
+
+SYSCTL_PROC(_kern, KERN_AIOTHREADS, aiothreads,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_aiothreads, "I", "");
+
+static int
+sysctl_securelvl
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int new_value, changed;
+	int error = sysctl_io_number(req, securelevel, sizeof(int), &new_value, &changed);
+	if (changed) {
+		if (!(new_value < securelevel && req->p->p_pid != 1)) {
+			proc_list_lock();
+			securelevel = new_value;
+			proc_list_unlock();
+		} else {
+			error = EPERM;
 		}
+	}
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_SECURELVL, securelevel,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_securelvl, "I", "");
+
+
+static int
+sysctl_domainname
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int error, changed;
+	error = sysctl_io_string(req, domainname, sizeof(domainname), 0, &changed);
+	if (changed) {
+		domainnamelen = strlen(domainname);
+	}
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_DOMAINNAME, nisdomainname,
+		CTLTYPE_STRING | CTLFLAG_RW,
+		0, 0, sysctl_domainname, "A", "");
+
+SYSCTL_INT(_kern, KERN_HOSTID, hostid, 
+		CTLFLAG_RW | CTLFLAG_KERN, 
+		&hostid, 0, "");
+
+static int
+sysctl_hostname
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int error, changed;
+	error = sysctl_io_string(req, hostname, sizeof(hostname), 1, &changed);
+	if (changed) {
+		hostnamelen = req->newlen;
+	}
+	return(error);
+}
+
+
+SYSCTL_PROC(_kern, KERN_HOSTNAME, hostname,
+		CTLTYPE_STRING | CTLFLAG_RW,
+		0, 0, sysctl_hostname, "A", "");
+
+static int
+sysctl_procname
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	/* Original code allowed writing, I'm copying this, although this all makes
+	   no sense to me. Besides, this sysctl is never used. */
+	return sysctl_io_string(req, &req->p->p_name[0], (2*MAXCOMLEN+1), 1, NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_PROCNAME, procname,
+		CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_ANYBODY,
+		0, 0, sysctl_procname, "A", "");
+
+SYSCTL_INT(_kern, KERN_SPECULATIVE_READS, speculative_reads_disabled, 
+		CTLFLAG_RW | CTLFLAG_KERN, 
+		&speculative_reads_disabled, 0, "");
+
+static int
+sysctl_boottime
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	struct timeval t;
+
+	t.tv_sec = boottime_sec();
+	t.tv_usec = 0;
+
+	return sysctl_io_opaque(req, &t, sizeof(t), NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime,
+		CTLTYPE_STRUCT | CTLFLAG_RD,
+		0, 0, sysctl_boottime, "S,timeval", "");
+
+static int
+sysctl_symfile
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	char *str;
+	int error = get_kernel_symfile(req->p, &str);
+	if (error)
+		return (error);
+	return sysctl_io_string(req, str, 0, 0, NULL);
+}
+
+
+SYSCTL_PROC(_kern, KERN_SYMFILE, symfile,
+		CTLTYPE_STRING | CTLFLAG_RD,
+		0, 0, sysctl_symfile, "A", "");
+
+#if NFSCLIENT
+static int
+sysctl_netboot
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	return sysctl_io_number(req, netboot_root(), sizeof(int), NULL, NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_NETBOOT, netboot,
+		CTLTYPE_INT | CTLFLAG_RD,
+		0, 0, sysctl_netboot, "I", "");
+#endif
+
+static int
+sysctl_usrstack
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	return sysctl_io_number(req, (int)req->p->user_stack, sizeof(int), NULL, NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_USRSTACK, usrstack,
+		CTLTYPE_INT | CTLFLAG_RD,
+		0, 0, sysctl_usrstack, "I", "");
+
+static int
+sysctl_usrstack64
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	return sysctl_io_number(req, req->p->user_stack, sizeof(req->p->user_stack), NULL, NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_USRSTACK64, usrstack64,
+		CTLTYPE_QUAD | CTLFLAG_RD,
+		0, 0, sysctl_usrstack64, "Q", "");
+
+SYSCTL_STRING(_kern, KERN_COREFILE, corefile, 
+		CTLFLAG_RW | CTLFLAG_KERN, 
+		corefilename, sizeof(corefilename), "");
+
+static int
+sysctl_coredump
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int new_value, changed;
+	int error = sysctl_io_number(req, do_coredump, sizeof(int), &new_value, &changed);
+	if (changed) {
+		if ((new_value == 0) || (new_value == 1))
+			do_coredump = new_value;
+		else
+			error = EINVAL;
+	}
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_COREDUMP, coredump,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_coredump, "I", "");
+
+static int
+sysctl_suid_coredump
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int new_value, changed;
+	int error = sysctl_io_number(req, sugid_coredump, sizeof(int), &new_value, &changed);
+	if (changed) {
+		if ((new_value == 0) || (new_value == 1))
+			sugid_coredump = new_value;
 		else
 			error = EINVAL;
 	}
-	return( error );
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_SUGID_COREDUMP, sugid_coredump,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_suid_coredump, "I", "");
+
+static int
+sysctl_delayterm
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	struct proc *p = req->p;
+	int new_value, changed;
+	int error = sysctl_io_number(req, (req->p->p_lflag & P_LDELAYTERM)? 1: 0, sizeof(int), &new_value, &changed);
+	if (changed) {
+		proc_lock(p);
+		if (new_value)
+			req->p->p_lflag |=  P_LDELAYTERM;
+		else
+			req->p->p_lflag &=  ~P_LDELAYTERM;
+		proc_unlock(p);
+	}
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_PROCDELAYTERM, delayterm,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_delayterm, "I", "");
+
+static int
+sysctl_proc_low_pri_io
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	struct proc *p = req->p;
+	int new_value, old_value, changed;
+	int error;
+
+	proc_lock(p);
+    switch (req->p->p_iopol_disk) {
+    case IOPOL_DEFAULT:
+    case IOPOL_NORMAL:
+        old_value = 0;
+        break;
+    case IOPOL_THROTTLE:
+        old_value = 1;
+        break;
+    case IOPOL_PASSIVE:
+        old_value = 2;
+        break;
+    default:
+        /* this should never happen, but to be robust, return the default value */
+        old_value = 0;
+        break;
+    }
+	proc_unlock(p);
 	
-} /* sysctl_maxfilesperproc */
+	error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
+	if (changed) {
+		proc_lock(p);
+        if (new_value & 0x01)
+            req->p->p_iopol_disk = IOPOL_THROTTLE;
+        else if (new_value & 0x02)
+            req->p->p_iopol_disk = IOPOL_PASSIVE;
+        else if (new_value == 0)
+            req->p->p_iopol_disk = IOPOL_NORMAL;
+		proc_unlock(p);
+	}
+	return(error);
+}
 
+SYSCTL_PROC(_kern, KERN_PROC_LOW_PRI_IO, proc_low_pri_io,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_proc_low_pri_io, "I", "");
 
-/*
- * Validate parameters and get old / set new parameters
- * for the system-wide limit on the max number of processes.
- * Makes sure the system-wide limit is less than the configured hard
- * limit set at kernel compilation.
- */
 static int
-sysctl_maxproc(user_addr_t oldp, size_t *oldlenp, 
-               user_addr_t newp, size_t newlen )
+sysctl_rage_vnode
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
-	int 	error = 0;
-	int	new_value;
+	struct proc *p = req->p;
+        struct  uthread *ut;
+	int new_value, old_value, changed;
+	int error;
 
-	if ( oldp != USER_ADDR_NULL && *oldlenp < sizeof(int) )
-		return (ENOMEM);
-	if ( newp != USER_ADDR_NULL && newlen != sizeof(int) )
-		return (EINVAL);
-		
-	*oldlenp = sizeof(int);
-	if ( oldp != USER_ADDR_NULL )
-		error = copyout( &maxproc, oldp, sizeof(int) );
-	if ( error == 0 && newp != USER_ADDR_NULL ) {
-		error = copyin( newp, &new_value, sizeof(int) );
-		if ( error == 0 ) {
-			AUDIT_ARG(value, new_value);
-			if ( new_value <= hard_maxproc && new_value > 0 )
-			maxproc = new_value;
-			else
-				error = EINVAL;
+	ut = get_bsdthread_info(current_thread());
+
+	if (ut->uu_flag & UT_RAGE_VNODES)
+	        old_value = KERN_RAGE_THREAD;
+	else if (p->p_lflag & P_LRAGE_VNODES)
+	        old_value = KERN_RAGE_PROC;
+	else
+	        old_value = 0;
+
+	error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
+
+	if (error == 0) {
+	        switch (new_value) {
+		case KERN_RAGE_PROC:
+		        proc_lock(p);
+			p->p_lflag |= P_LRAGE_VNODES;
+			proc_unlock(p);
+			break;
+		case KERN_UNRAGE_PROC:
+		        proc_lock(p);
+			p->p_lflag &= ~P_LRAGE_VNODES;
+			proc_unlock(p);
+			break;
+
+		case KERN_RAGE_THREAD:
+			ut->uu_flag |= UT_RAGE_VNODES;
+			break;
+		case KERN_UNRAGE_THREAD:
+		        ut = get_bsdthread_info(current_thread());
+			ut->uu_flag &= ~UT_RAGE_VNODES;
+			break;
 		}
-		else
+	}
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_RAGEVNODE, rage_vnode,
+		CTLTYPE_INT | CTLFLAG_RW,
+		0, 0, sysctl_rage_vnode, "I", "");
+
+
+static int
+sysctl_kern_check_openevt
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	struct proc *p = req->p;
+	int new_value, old_value, changed;
+	int error;
+
+	if (p->p_flag & P_CHECKOPENEVT) {
+		old_value = KERN_OPENEVT_PROC;
+	} else {
+	        old_value = 0;
+	}
+
+	error = sysctl_io_number(req, old_value, sizeof(int), &new_value, &changed);
+
+	if (error == 0) {
+	        switch (new_value) {
+		case KERN_OPENEVT_PROC:
+			OSBitOrAtomic(P_CHECKOPENEVT, (UInt32 *)&p->p_flag);
+			break;
+
+		case KERN_UNOPENEVT_PROC:
+			OSBitAndAtomic(~((uint32_t)P_CHECKOPENEVT), (UInt32 *)&p->p_flag);
+			break;
+
+		default:
 			error = EINVAL;
+		}
 	}
-	return( error );
-	
-} /* sysctl_maxproc */
+	return(error);
+}
+
+SYSCTL_PROC(_kern, KERN_CHECKOPENEVT, check_openevt, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_ANYBODY,
+            0, 0, sysctl_kern_check_openevt, "I", "set the per-process check-open-evt flag");
+
+
+
+static int
+sysctl_nx
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	int new_value, changed;
+	int error;
+
+	error = sysctl_io_number(req, nx_enabled, sizeof(nx_enabled), &new_value, &changed);
+    if (error)
+        return error;
+
+    if (changed) {
+#ifdef __i386__
+		/*
+		 * Only allow setting if NX is supported on the chip
+		 */
+		if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
+            return ENOTSUP;
+#endif
+        nx_enabled = new_value;
+    }
+	return(error);
+}
+
+
+
+SYSCTL_PROC(_kern, KERN_NX_PROTECTION, nx, 
+		CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_KERN, 
+		0, 0, sysctl_nx, "I", "");
+
+static int
+sysctl_loadavg
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+		if (proc_is64bit(req->p)) {
+			struct user_loadavg loadinfo64;
+			loadavg32to64(&averunnable, &loadinfo64);
+			return sysctl_io_opaque(req, &loadinfo64, sizeof(loadinfo64), NULL);
+		} else {
+			return sysctl_io_opaque(req, &averunnable, sizeof(averunnable), NULL);
+		}
+}
+
+SYSCTL_PROC(_vm, VM_LOADAVG, loadavg,
+		CTLTYPE_STRUCT | CTLFLAG_RD,
+		0, 0, sysctl_loadavg, "S,loadavg", "");
+
+static int
+sysctl_swapusage
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+		int			error;
+		uint64_t		swap_total;
+		uint64_t		swap_avail;
+		uint32_t		swap_pagesize;
+		boolean_t		swap_encrypted;
+		struct xsw_usage	xsu;
+
+		error = macx_swapinfo(&swap_total,
+				      &swap_avail,
+				      &swap_pagesize,
+				      &swap_encrypted);
+		if (error)
+			return error;
+
+		xsu.xsu_total = swap_total;
+		xsu.xsu_avail = swap_avail;
+		xsu.xsu_used = swap_total - swap_avail;
+		xsu.xsu_pagesize = swap_pagesize;
+		xsu.xsu_encrypted = swap_encrypted;
+		return sysctl_io_opaque(req, &xsu, sizeof(xsu), NULL);
+}
+
+
+
+SYSCTL_PROC(_vm, VM_SWAPUSAGE, swapusage,
+		CTLTYPE_STRUCT | CTLFLAG_RD,
+		0, 0, sysctl_swapusage, "S,xsw_usage", "");
+
+
+/* this kernel does NOT implement shared_region_make_private_np() */
+SYSCTL_INT(_kern, KERN_SHREG_PRIVATIZABLE, shreg_private, 
+		CTLFLAG_RD, 
+		NULL, 0, "");
 
 #if __i386__
 static int
-sysctl_sysctl_exec_affinity SYSCTL_HANDLER_ARGS
+sysctl_sysctl_exec_affinity(__unused struct sysctl_oid *oidp,
+			   __unused void *arg1, __unused int arg2,
+			   struct sysctl_req *req)
 {
-	struct proc *cur_proc = req->p;
+	proc_t cur_proc = req->p;
 	int error;
 	
 	if (req->oldptr != USER_ADDR_NULL) {
@@ -2211,9 +2516,9 @@ sysctl_sysctl_exec_affinity SYSCTL_HANDLER_ARGS
 		if ((error = SYSCTL_IN(req, &newcputype, sizeof(newcputype))))
 			return error;
 		if (newcputype == CPU_TYPE_I386)
-			cur_proc->p_flag &= ~P_AFFINITY;
+			OSBitAndAtomic(~((uint32_t)P_AFFINITY), (UInt32 *)&cur_proc->p_flag);
 		else if (newcputype == CPU_TYPE_POWERPC)
-			cur_proc->p_flag |= P_AFFINITY;
+			OSBitOrAtomic(P_AFFINITY, (UInt32 *)&cur_proc->p_flag);
 		else
 			return (EINVAL);
 	}
@@ -2225,25 +2530,26 @@ SYSCTL_PROC(_sysctl, OID_AUTO, proc_exec_affinity, CTLTYPE_INT|CTLFLAG_RW|CTLFLA
 
 static int
 fetch_process_cputype(
-	struct proc *cur_proc,
+	proc_t cur_proc,
 	int *name,
 	u_int namelen,
 	cpu_type_t *cputype)
 {
-	struct proc *p = NULL;
+	proc_t p = PROC_NULL;
+	int refheld = 0;
 	cpu_type_t ret = 0;
+	int error = 0;
 	
 	if (namelen == 0)
 		p = cur_proc;
 	else if (namelen == 1) {
-		p = pfind(name[0]);
+		p = proc_find(name[0]);
 		if (p == NULL)
 			return (EINVAL);
-		if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
-			&& suser(kauth_cred_get(), &cur_proc->p_acflag))
-			return (EPERM);
+		refheld = 1;
 	} else {
-		return EINVAL;
+		error = EINVAL;
+		goto out;
 	}
 
 #if __i386__
@@ -2259,11 +2565,15 @@ fetch_process_cputype(
 	}
 	*cputype = ret;
 	
-	return 0;
+	if (refheld != 0)
+		proc_rele(p);
+out:
+	return (error);
 }
 
 static int
-sysctl_sysctl_native SYSCTL_HANDLER_ARGS
+sysctl_sysctl_native(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+		    struct sysctl_req *req)
 {
 	int error;
 	cpu_type_t proc_cputype = 0;
@@ -2277,7 +2587,8 @@ sysctl_sysctl_native SYSCTL_HANDLER_ARGS
 SYSCTL_PROC(_sysctl, OID_AUTO, proc_native, CTLTYPE_NODE|CTLFLAG_RD, 0, 0, sysctl_sysctl_native ,"I","proc_native");
 
 static int
-sysctl_sysctl_cputype SYSCTL_HANDLER_ARGS
+sysctl_sysctl_cputype(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+		     struct sysctl_req *req)
 {
 	int error;
 	cpu_type_t proc_cputype = 0;
@@ -2287,3 +2598,54 @@ sysctl_sysctl_cputype SYSCTL_HANDLER_ARGS
 }
 SYSCTL_PROC(_sysctl, OID_AUTO, proc_cputype, CTLTYPE_NODE|CTLFLAG_RD, 0, 0, sysctl_sysctl_cputype ,"I","proc_cputype");
 
+static int
+sysctl_safeboot
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	return sysctl_io_number(req, boothowto & RB_SAFEBOOT ? 1 : 0, sizeof(int), NULL, NULL);
+}
+
+SYSCTL_PROC(_kern, KERN_SAFEBOOT, safeboot,
+		CTLTYPE_INT | CTLFLAG_RD,
+		0, 0, sysctl_safeboot, "I", "");
+
+static int
+sysctl_singleuser
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, struct sysctl_req *req)
+{
+	return sysctl_io_number(req, boothowto & RB_SINGLE ? 1 : 0, sizeof(int), NULL, NULL);
+}
+
+SYSCTL_PROC(_kern, OID_AUTO, singleuser,
+		CTLTYPE_INT | CTLFLAG_RD,
+		0, 0, sysctl_singleuser, "I", "");
+
+/*
+ * Controls for debugging affinity sets - see osfmk/kern/affinity.c
+ */
+extern boolean_t	affinity_sets_enabled;
+extern int		affinity_sets_mapping;
+
+SYSCTL_INT (_kern, OID_AUTO, affinity_sets_enabled,
+	    CTLFLAG_RW, (int *) &affinity_sets_enabled, 0, "hinting enabled");
+SYSCTL_INT (_kern, OID_AUTO, affinity_sets_mapping,
+	    CTLFLAG_RW, &affinity_sets_mapping, 0, "mapping policy");
+
+/*
+ * Limit on total memory users can wire.
+ *
+ * vm_global_user_wire_limit - system wide limit on wired memory from all processes combined. 
+ *
+ * vm_user_wire_limit - per address space limit on wired memory.  This puts a cap on the process's rlimit value.
+ *
+ * These values are initialized to reasonable defaults at boot time based on the available physical memory in
+ * kmem_init().
+ *
+ * All values are in bytes.
+ */
+
+vm_map_size_t	vm_global_user_wire_limit;
+vm_map_size_t	vm_user_wire_limit;
+
+SYSCTL_QUAD(_vm, OID_AUTO, global_user_wire_limit, CTLFLAG_RW, &vm_global_user_wire_limit, "");
+SYSCTL_QUAD(_vm, OID_AUTO, user_wire_limit, CTLFLAG_RW, &vm_user_wire_limit, "");
diff --git a/bsd/kern/kern_time.c b/bsd/kern/kern_time.c
index cc413684d..30b0a651c 100644
--- a/bsd/kern/kern_time.c
+++ b/bsd/kern/kern_time.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)kern_time.c	8.4 (Berkeley) 5/26/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/resourcevar.h>
@@ -68,7 +80,11 @@
 #include <sys/signalvar.h>
 
 #include <kern/clock.h>
+#include <kern/task.h>
 #include <kern/thread_call.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 #define HZ	100	/* XXX */
 
@@ -81,7 +97,7 @@ lck_grp_attr_t	*tz_slock_grp_attr;
 static void		setthetime(
 					struct timeval	*tv);
 
-void time_zone_slock_init(void);
+void time_zone_slock_init(void) __attribute__((section("__TEXT, initcode")));
 
 /* 
  * Time of day and interval timer support.
@@ -103,7 +119,7 @@ __unused	struct proc	*p,
 	struct timezone ltz; /* local copy */
 
 	if (uap->tp)
-		clock_gettimeofday(&retval[0], &retval[1]);
+		clock_gettimeofday((uint32_t *)&retval[0], (uint32_t *)&retval[1]);
 	
 	if (uap->tzp) {
 		lck_spin_lock(tz_slock);
@@ -121,14 +137,21 @@ __unused	struct proc	*p,
  */
 /* ARGSUSED */
 int
-settimeofday(struct proc *p, struct settimeofday_args  *uap, __unused register_t *retval)
+settimeofday(__unused struct proc *p, struct settimeofday_args  *uap, __unused register_t *retval)
 {
 	struct timeval atv;
 	struct timezone atz;
 	int error;
 
+#if CONFIG_MACF
+	error = mac_system_check_settime(kauth_cred_get());
+	if (error)
+		return (error);
+#endif
+#ifndef CONFIG_EMBEDDED
 	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
 		return (error);
+#endif
 	/* Verify all parameters before changing time */
 	if (uap->tv) {
 		if (IS_64BIT_PROCESS(p)) {
@@ -170,11 +193,16 @@ setthetime(
  */
 /* ARGSUSED */
 int
-adjtime(struct proc *p, register struct adjtime_args *uap, __unused register_t *retval)
+adjtime(struct proc *p, struct adjtime_args *uap, __unused register_t *retval)
 {
 	struct timeval atv;
 	int error;
 
+#if CONFIG_MACF
+	error = mac_system_check_settime(kauth_cred_get());
+	if (error)
+		return (error);
+#endif
 	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
 		return (error);
 	if (IS_64BIT_PROCESS(p)) {
@@ -255,26 +283,32 @@ uint64_t tvtoabstime(struct timeval *tvp);
  * is kept as an absolute time rather than as a delta, so that
  * it is easy to keep periodic real-time signals from drifting.
  *
- * Virtual time timers are processed in the hardclock() routine of
- * kern_clock.c.  The real time timer is processed by a callout
- * routine.  Since a callout may be delayed in real time due to
+ * The real time timer is processed by a callout routine.
+ * Since a callout may be delayed in real time due to
  * other processing in the system, it is possible for the real
  * time callout routine (realitexpire, given below), to be delayed
  * in real time past when it is supposed to occur.  It does not
  * suffice, therefore, to reload the real time .it_value from the
  * real time .it_interval.  Rather, we compute the next time in
  * absolute time when the timer should go off.
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *	copyout:EFAULT			Bad address
  */
- 
 /* ARGSUSED */
 int
-getitimer(struct proc *p, register struct getitimer_args *uap, __unused register_t *retval)
+getitimer(struct proc *p, struct getitimer_args *uap, __unused register_t *retval)
 {
 	struct itimerval aitv;
 
 	if (uap->which > ITIMER_PROF)
 		return(EINVAL);
-	if (uap->which == ITIMER_REAL) {
+
+	proc_spinlock(p);
+	switch (uap->which) {
+
+	case ITIMER_REAL:
 		/*
 		 * If time for real time timer has passed return 0,
 		 * else return difference between current time and
@@ -294,9 +328,18 @@ getitimer(struct proc *p, register struct getitimer_args *uap, __unused register
 		}
 		else
 			timerclear(&aitv.it_value);
+		break;
+
+	case ITIMER_VIRTUAL:
+		aitv = p->p_vtimer_user;
+		break;
+
+	case ITIMER_PROF:
+		aitv = p->p_vtimer_prof;
+		break;
 	}
-	else
-		aitv = p->p_stats->p_timer[uap->which];
+
+	proc_spinunlock(p);
 
 	if (IS_64BIT_PROCESS(p)) {
 		struct user_itimerval user_itv;
@@ -310,12 +353,16 @@ getitimer(struct proc *p, register struct getitimer_args *uap, __unused register
 	}
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *	copyin:EFAULT			Bad address
+ *	getitimer:EINVAL		Invalid argument
+ *	getitimer:EFAULT		Bad address
+ */
 /* ARGSUSED */
 int
-setitimer(p, uap, retval)
-	struct proc *p;
-	register struct setitimer_args *uap;
-	register_t *retval;
+setitimer(struct proc *p, struct setitimer_args *uap, register_t *retval)
 {
 	struct itimerval aitv;
 	user_addr_t itvp;
@@ -343,22 +390,50 @@ setitimer(p, uap, retval)
 		return (0);
 	if (itimerfix(&aitv.it_value) || itimerfix(&aitv.it_interval))
 		return (EINVAL);
-	if (uap->which == ITIMER_REAL) {
-		thread_call_func_cancel((thread_call_func_t)realitexpire, (void *)p->p_pid, FALSE);
+
+	switch (uap->which) {
+
+	case ITIMER_REAL:
+		proc_spinlock(p);
 		if (timerisset(&aitv.it_value)) {
 			microuptime(&p->p_rtime);
 			timevaladd(&p->p_rtime, &aitv.it_value);
-			thread_call_func_delayed(
-								(thread_call_func_t)realitexpire, (void *)p->p_pid,
-										tvtoabstime(&p->p_rtime));
+			p->p_realtimer = aitv;
+			if (!thread_call_enter_delayed(p->p_rcall, tvtoabstime(&p->p_rtime)))
+				p->p_ractive++;
+		} else  {
+			timerclear(&p->p_rtime);
+			p->p_realtimer = aitv;
+			if (thread_call_cancel(p->p_rcall))
+				p->p_ractive--;
 		}
+		proc_spinunlock(p);
+
+		break;
+
+
+	case ITIMER_VIRTUAL:
+		if (timerisset(&aitv.it_value))
+			task_vtimer_set(p->task, TASK_VTIMER_USER);
+	else
+			task_vtimer_clear(p->task, TASK_VTIMER_USER);
+
+		proc_spinlock(p);
+		p->p_vtimer_user = aitv;
+		proc_spinunlock(p);
+		break;
+
+	case ITIMER_PROF:
+		if (timerisset(&aitv.it_value))
+			task_vtimer_set(p->task, TASK_VTIMER_PROF);
 		else
-			timerclear(&p->p_rtime);
+			task_vtimer_clear(p->task, TASK_VTIMER_PROF);
 
-		p->p_realtimer = aitv;
+		proc_spinlock(p);
+		p->p_vtimer_prof = aitv;
+		proc_spinunlock(p);
+		break;
 	}
-	else
-		p->p_stats->p_timer[uap->which] = aitv;
 
 	return (0);
 }
@@ -373,66 +448,68 @@ setitimer(p, uap, retval)
  */
 void
 realitexpire(
-	void		*pid)
+	struct proc	*p)
 {
-	register struct proc *p;
-	struct timeval	now;
-	boolean_t		funnel_state;
-
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	p = pfind((pid_t)pid);
-	if (p == NULL) {
-		(void) thread_funnel_set(kernel_flock, FALSE);
+	struct proc *r;
+	struct timeval	t;
+
+	r = proc_find(p->p_pid);
+
+	proc_spinlock(p);
+
+	if (--p->p_ractive > 0 || r != p) {
+		proc_spinunlock(p);
+
+		if (r != NULL)
+			proc_rele(r);
 		return;
 	}
-
+	
 	if (!timerisset(&p->p_realtimer.it_interval)) {
 		timerclear(&p->p_rtime);
-		psignal(p, SIGALRM);
+		proc_spinunlock(p);
 
-		(void) thread_funnel_set(kernel_flock, FALSE);
+		psignal(p, SIGALRM);
+		proc_rele(p);
 		return;
 	}
 
-	microuptime(&now);
+	microuptime(&t);
 	timevaladd(&p->p_rtime, &p->p_realtimer.it_interval);
-	if (timercmp(&p->p_rtime, &now, <=)) {
-		if ((p->p_rtime.tv_sec + 2) >= now.tv_sec) {
+	if (timercmp(&p->p_rtime, &t, <=)) {
+		if ((p->p_rtime.tv_sec + 2) >= t.tv_sec) {
 			for (;;) {
 				timevaladd(&p->p_rtime, &p->p_realtimer.it_interval);
-				if (timercmp(&p->p_rtime, &now, >))
+				if (timercmp(&p->p_rtime, &t, >))
 					break;
 			}
 		}
 		else {
 			p->p_rtime = p->p_realtimer.it_interval;
-			timevaladd(&p->p_rtime, &now);
+			timevaladd(&p->p_rtime, &t);
 		}
 	}
 
-	psignal(p, SIGALRM);
-
-	thread_call_func_delayed((thread_call_func_t)realitexpire, pid, tvtoabstime(&p->p_rtime));
+	if (!thread_call_enter_delayed(p->p_rcall, tvtoabstime(&p->p_rtime)))
+		p->p_ractive++;
+	proc_spinunlock(p);
 
-	(void) thread_funnel_set(kernel_flock, FALSE);
+	psignal(p, SIGALRM);
+	proc_rele(p);
 }
 
 /*
  * Check that a proposed value to load into the .it_value or
- * .it_interval part of an interval timer is acceptable, and
- * fix it to have at least minimal value (i.e. if it is less
- * than the resolution of the clock, round it up.)
+ * .it_interval part of an interval timer is acceptable.
  */
 int
-itimerfix(tv)
-	struct timeval *tv;
+itimerfix(
+	struct timeval *tv)
 {
 
 	if (tv->tv_sec < 0 || tv->tv_sec > 100000000 ||
 	    tv->tv_usec < 0 || tv->tv_usec >= 1000000)
 		return (EINVAL);
-	if (tv->tv_sec == 0 && tv->tv_usec != 0 && tv->tv_usec < tick)
-		tv->tv_usec = tick;
 	return (0);
 }
 
@@ -441,17 +518,18 @@ itimerfix(tv)
  * of microseconds, which must be less than a second,
  * i.e. < 1000000.  If the timer expires, then reload
  * it.  In this case, carry over (usec - old value) to
- * reducint the value reloaded into the timer so that
+ * reduce the value reloaded into the timer so that
  * the timer does not drift.  This routine assumes
  * that it is called in a context where the timers
  * on which it is operating cannot change in value.
  */
 int
-itimerdecr(itp, usec)
-	register struct itimerval *itp;
-	int usec;
+itimerdecr(proc_t p,
+	struct itimerval *itp, int usec)
 {
 
+	proc_spinlock(p);
+	
 	if (itp->it_value.tv_usec < usec) {
 		if (itp->it_value.tv_sec == 0) {
 			/* expired, and already in next interval */
@@ -463,19 +541,24 @@ itimerdecr(itp, usec)
 	}
 	itp->it_value.tv_usec -= usec;
 	usec = 0;
-	if (timerisset(&itp->it_value))
+	if (timerisset(&itp->it_value)) {
+		proc_spinunlock(p);
 		return (1);
+	}
 	/* expired, exactly at end of interval */
 expire:
 	if (timerisset(&itp->it_interval)) {
 		itp->it_value = itp->it_interval;
+		if (itp->it_value.tv_sec > 0) {
 		itp->it_value.tv_usec -= usec;
 		if (itp->it_value.tv_usec < 0) {
 			itp->it_value.tv_usec += 1000000;
 			itp->it_value.tv_sec--;
+			}
 		}
 	} else
 		itp->it_value.tv_usec = 0;		/* sec is already 0 */
+	proc_spinunlock(p);
 	return (0);
 }
 
@@ -529,14 +612,14 @@ void
 microtime(
 	struct timeval	*tvp)
 {
-	clock_get_calendar_microtime((uint32_t *)&tvp->tv_sec, &tvp->tv_usec);
+	clock_get_calendar_microtime((uint32_t *)&tvp->tv_sec, (uint32_t *)&tvp->tv_usec);
 }
 
 void
 microuptime(
 	struct timeval	*tvp)
 {
-	clock_get_system_microtime((uint32_t *)&tvp->tv_sec, &tvp->tv_usec);
+	clock_get_system_microtime((uint32_t *)&tvp->tv_sec, (uint32_t *)&tvp->tv_usec);
 }
 
 /*
@@ -583,4 +666,3 @@ time_zone_slock_init(void)
 	/* Allocate the spin lock */
 	tz_slock = lck_spin_alloc_init(tz_slock_grp, tz_slock_attr);
 }
-
diff --git a/bsd/kern/kern_xxx.c b/bsd/kern/kern_xxx.c
index 470a220e8..7884fbd3c 100644
--- a/bsd/kern/kern_xxx.c
+++ b/bsd/kern/kern_xxx.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)kern_xxx.c	8.2 (Berkeley) 11/14/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <cputypes.h> 
 
@@ -71,28 +83,42 @@
 
 #include <sys/mount_internal.h>
 #include <sys/sysproto.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 int
 reboot(struct proc *p, register struct reboot_args *uap, __unused register_t *retval)
 {
 	char command[64];
-	int error;
+	int error=0;
 	int dummy=0;
+#if CONFIG_MACF
+	kauth_cred_t my_cred;
+#endif
 
 	AUDIT_ARG(cmd, uap->opt);
 
 	command[0] = '\0';
 
+#ifndef CONFIG_EMBEDDED
 	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
 		return(error);	
+#endif	
 	
 	if (uap->opt & RB_COMMAND)
 		error = copyinstr(uap->command,
 					(void *)command, sizeof(command), (size_t *)&dummy);
+#if CONFIG_MACF
+	if (error)
+		return (error);
+	my_cred = kauth_cred_proc_ref(p);
+	error = mac_system_check_reboot(my_cred, uap->opt);
+	kauth_cred_unref(&my_cred);
+#endif
 	if (!error) {
-		SET(p->p_flag, P_REBOOT);	/* No more signals for this proc */
+		OSBitOrAtomic(P_REBOOT, (UInt32 *)&p->p_flag);  /* No more signals for this proc */
 		boot(RB_BOOT, uap->opt, command);
 	}
 	return(error);
 }
-
diff --git a/bsd/kern/kpi_mbuf.c b/bsd/kern/kpi_mbuf.c
index b1062b566..ea47e2f6c 100644
--- a/bsd/kern/kpi_mbuf.c
+++ b/bsd/kern/kpi_mbuf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #define __KPI__
@@ -30,9 +36,8 @@
 #include <libkern/OSAtomic.h>
 #include <kern/kalloc.h>
 #include <string.h>
-
-void mbuf_tag_id_first_last(u_long *first, u_long *last);
-errno_t mbuf_tag_id_find_internal(const char *string, u_long *out_id, int create);
+#include <netinet/in.h>
+#include "kpi_mbuf_internal.h"
 
 static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
 				MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG |
@@ -40,7 +45,7 @@ static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR |
 
 void* mbuf_data(mbuf_t mbuf)
 {
-	return m_mtod(mbuf);
+	return mbuf->m_data;
 }
 
 void* mbuf_datastart(mbuf_t mbuf)
@@ -77,7 +82,7 @@ errno_t mbuf_align_32(mbuf_t mbuf, size_t len)
 
 addr64_t mbuf_data_to_physical(void* ptr)
 {
-	return (addr64_t)mcl_to_paddr(ptr);
+	return (addr64_t)(intptr_t)mcl_to_paddr(ptr);
 }
 
 errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
@@ -96,13 +101,73 @@ errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf)
 	return (*mbuf == NULL) ? ENOMEM : 0;
 }
 
-extern struct mbuf * m_mbigget(struct mbuf *m, int nowait);
+errno_t
+mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
+    caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t),
+    size_t extsize, caddr_t extarg)
+{
+	if (extbuf == NULL || extfree == NULL || extsize == 0)
+		return (EINVAL);
+
+	if ((*mbuf = m_clattach(mbuf != NULL ? *mbuf : NULL, type, extbuf,
+	    extfree, extsize, extarg, how)) == NULL)
+		return (ENOMEM);
+
+	return (0);
+}
 
-errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
+errno_t
+mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr)
+{
+	if (size == NULL || *size == 0 || addr == NULL)
+		return (EINVAL);
+
+	*addr = NULL;
+
+	/* Jumbo cluster pool not available? */
+	if (*size > NBPG && njcl == 0)
+		return (ENOTSUP);
+
+	if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL)
+		*size = MCLBYTES;
+	else if (*size > MCLBYTES && *size <= NBPG &&
+	    (*addr = m_bigalloc(how)) != NULL)
+		*size = NBPG;
+	else if (*size > NBPG && *size <= M16KCLBYTES &&
+	    (*addr = m_16kalloc(how)) != NULL)
+		*size = M16KCLBYTES;
+	else
+		*size = 0;
+
+	if (*addr == NULL)
+		return (ENOMEM);
+
+	return (0);
+}
+
+void
+mbuf_freecluster(caddr_t addr, size_t size)
+{
+	if (size != MCLBYTES && size != NBPG && size != M16KCLBYTES)
+		panic("%s: invalid size (%ld) for cluster %p", __func__,
+		    size, (void *)addr);
+
+	if (size == MCLBYTES)
+		m_mclfree(addr);
+	else if (size == NBPG)
+		m_bigfree(addr, NBPG, NULL);
+	else if (njcl > 0)
+		m_16kfree(addr, M16KCLBYTES, NULL);
+	else
+		panic("%s: freeing jumbo cluster to an empty pool", __func__);
+}
+
+errno_t
+mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf)
 {
 	/* Must set *mbuf to NULL in failure case */
 	errno_t	error = 0;
-	int		created = 0;
+	int	created = 0;
 
 	if (mbuf == NULL)
 		return EINVAL;
@@ -113,13 +178,21 @@ errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* m
 		created = 1;
 	}
 	/*
-	 * At the time this code was written, m_mclget and m_mbigget would always
-	 * return the same value that was passed in to it.
+	 * At the time this code was written, m_{mclget,mbigget,m16kget}
+	 * would always return the same value that was passed in to it.
 	 */
 	if (size == MCLBYTES) {
 		*mbuf = m_mclget(*mbuf, how);
 	} else if (size == NBPG) {
 		*mbuf = m_mbigget(*mbuf, how);
+	} else if (size == M16KCLBYTES) {
+		if (njcl > 0) {
+			*mbuf = m_m16kget(*mbuf, how);
+		} else {
+			/* Jumbo cluster pool not available? */
+			error = ENOTSUP;
+			goto out;
+		}
 	} else {
 		error = EINVAL;
 		goto out;
@@ -128,7 +201,6 @@ errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* m
 		error = ENOMEM;
 out:
 	if (created && error != 0) {
-		error = ENOMEM;
 		mbuf_free(*mbuf);
 		*mbuf = NULL;
 	}
@@ -196,18 +268,18 @@ int	mbuf_freem_list(mbuf_t mbuf)
 	return m_freem_list(mbuf);
 }
 
-size_t mbuf_leadingspace(mbuf_t mbuf)
+size_t mbuf_leadingspace(const mbuf_t mbuf)
 {
 	return m_leadingspace(mbuf);
 }
 
-size_t mbuf_trailingspace(mbuf_t mbuf)
+size_t mbuf_trailingspace(const mbuf_t mbuf)
 {
 	return m_trailingspace(mbuf);
 }
 
 /* Manipulation */
-errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len,
+errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len,
 				   mbuf_how_t how, mbuf_t *new_mbuf)
 {
 	/* Must set *mbuf to NULL in failure case */
@@ -216,7 +288,7 @@ errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len,
 	return (*new_mbuf == NULL) ? ENOMEM : 0;
 }
 
-errno_t	mbuf_dup(mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
+errno_t	mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf)
 {
 	/* Must set *new_mbuf to NULL in failure case */
 	*new_mbuf = m_dup(src, how);
@@ -264,10 +336,29 @@ void mbuf_adj(mbuf_t mbuf, int len)
 	m_adj(mbuf, len);
 }
 
-errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data)
+errno_t mbuf_adjustlen(mbuf_t m, int amount)
+{
+	/* Verify m_len will be valid after adding amount */
+	if (amount > 0) {
+		int		used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) +
+					   m->m_len;
+		
+		if ((size_t)(amount + used) > mbuf_maxlen(m))
+			return EINVAL;
+	}
+	else if (-amount > m->m_len) {
+		return EINVAL;
+	}
+	
+	m->m_len += amount;
+	return 0;
+}
+
+errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data)
 {
 	/* Copied m_copydata, added error handling (don't just panic) */
 	int count;
+	mbuf_t	m = m0;
 
 	while (off > 0) {
 		if (m == 0)
@@ -291,16 +382,6 @@ errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data)
 	return 0;
 }
 
-int mbuf_mclref(mbuf_t mbuf)
-{
-	return m_mclref(mbuf);
-}
-
-int mbuf_mclunref(mbuf_t mbuf)
-{
-	return m_mclunref(mbuf);
-}
-
 int mbuf_mclhasreference(mbuf_t mbuf)
 {
 	if ((mbuf->m_flags & M_EXT))
@@ -311,7 +392,7 @@ int mbuf_mclhasreference(mbuf_t mbuf)
 
 
 /* mbuf header */
-mbuf_t mbuf_next(mbuf_t mbuf)
+mbuf_t mbuf_next(const mbuf_t mbuf)
 {
 	return mbuf->m_next;
 }
@@ -325,7 +406,7 @@ errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next)
 	return 0;
 }
 
-mbuf_t mbuf_nextpkt(mbuf_t mbuf)
+mbuf_t mbuf_nextpkt(const mbuf_t mbuf)
 {
 	return mbuf->m_nextpkt;
 }
@@ -335,7 +416,7 @@ void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt)
 	mbuf->m_nextpkt = nextpkt;
 }
 
-size_t mbuf_len(mbuf_t mbuf)
+size_t mbuf_len(const mbuf_t mbuf)
 {
 	return mbuf->m_len;
 }
@@ -345,14 +426,14 @@ void mbuf_setlen(mbuf_t mbuf, size_t len)
 	mbuf->m_len = len;
 }
 
-size_t mbuf_maxlen(mbuf_t mbuf)
+size_t mbuf_maxlen(const mbuf_t mbuf)
 {
 	if (mbuf->m_flags & M_EXT)
 		return mbuf->m_ext.ext_size;
 	return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf));
 }
 
-mbuf_type_t mbuf_type(mbuf_t mbuf)
+mbuf_type_t mbuf_type(const mbuf_t mbuf)
 {
 	return mbuf->m_type;
 }
@@ -366,7 +447,7 @@ errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type)
 	return 0;
 }
 
-mbuf_flags_t mbuf_flags(mbuf_t mbuf)
+mbuf_flags_t mbuf_flags(const mbuf_t mbuf)
 {
 	return mbuf->m_flags & mbuf_flags_mask;
 }
@@ -389,7 +470,7 @@ errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask)
 	return 0;
 }
 
-errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src)
+errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src)
 {
 	if (((src)->m_flags & M_PKTHDR) == 0)
 		return EINVAL;
@@ -399,7 +480,7 @@ errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src)
 	return 0;
 }
 
-size_t mbuf_pkthdr_len(mbuf_t mbuf)
+size_t mbuf_pkthdr_len(const mbuf_t mbuf)
 {
 	return mbuf->m_pkthdr.len;
 }
@@ -409,7 +490,12 @@ void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len)
 	mbuf->m_pkthdr.len = len;
 }
 
-ifnet_t mbuf_pkthdr_rcvif(mbuf_t mbuf)
+void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount)
+{
+	mbuf->m_pkthdr.len += amount;
+}
+
+ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf)
 {
 	// If we reference count ifnets, we should take a reference here before returning
 	return mbuf->m_pkthdr.rcvif;
@@ -422,7 +508,7 @@ errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet)
 	return 0;
 }
 
-void* mbuf_pkthdr_header(mbuf_t mbuf)
+void* mbuf_pkthdr_header(const mbuf_t mbuf)
 {
 	return mbuf->m_pkthdr.header;
 }
@@ -432,23 +518,6 @@ void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header)
 	mbuf->m_pkthdr.header = (void*)header;
 }
 
-/* mbuf aux data */
-errno_t mbuf_aux_add(mbuf_t mbuf, int family, mbuf_type_t type, mbuf_t *aux_mbuf)
-{
-	*aux_mbuf = m_aux_add(mbuf, family, type);
-	return (*aux_mbuf == NULL) ? ENOMEM : 0;
-}
-
-mbuf_t mbuf_aux_find(mbuf_t mbuf, int family, mbuf_type_t type)
-{
-	return m_aux_find(mbuf, family, type);
-}
-
-void mbuf_aux_delete(mbuf_t mbuf, mbuf_t aux)
-{
-	m_aux_delete(mbuf, aux);
-}
-
 void
 mbuf_inbound_modified(mbuf_t mbuf)
 {
@@ -499,7 +568,7 @@ mbuf_outbound_finalize(mbuf_t mbuf, u_long protocol_family, size_t protocol_offs
 			 * Hardware checksum code looked pretty IPv4 specific.
 			 */
 			if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP)) != 0)
-				panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%d)!\n", protocol_family);
+				panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%lu)!\n", protocol_family);
 	}
 }
 
@@ -617,6 +686,70 @@ mbuf_clear_csum_performed(
 	return 0;
 }
 
+errno_t
+mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
+    u_int16_t *csum)
+{
+	if (mbuf == NULL || length == 0 || csum == NULL ||
+	   (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
+		return (EINVAL);
+
+	*csum = inet_cksum(mbuf, protocol, offset, length);
+	return (0);
+}
+
+#if INET6
+errno_t
+mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length,
+    u_int16_t *csum)
+{
+	if (mbuf == NULL || length == 0 || csum == NULL ||
+	   (u_int32_t)mbuf->m_pkthdr.len < (offset + length))
+		return (EINVAL);
+
+	*csum = inet6_cksum(mbuf, protocol, offset, length);
+	return (0);
+}
+#else /* INET6 */
+errno_t
+mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol,
+		__unused u_int32_t offset, __unused u_int32_t length,
+		__unused u_int16_t *csum)
+{
+	panic("mbuf_inet6_cksum() doesn't exist on this platform\n");
+	return (0);
+}
+
+u_int16_t
+inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt,
+		__unused unsigned int off, __unused unsigned int len)
+{
+	panic("inet6_cksum() doesn't exist on this platform\n");
+	return (0);
+}
+
+void nd6_lookup_ipv6(void);
+void
+nd6_lookup_ipv6(void)
+{
+	panic("nd6_lookup_ipv6() doesn't exist on this platform\n");
+}
+
+int
+in6addr_local(__unused struct in6_addr *a)
+{
+	panic("in6addr_local() doesn't exist on this platform\n");
+	return (0);
+}
+
+void nd6_storelladdr(void);
+void
+nd6_storelladdr(void)
+{
+	panic("nd6_storelladdr() doesn't exist on this platform\n");
+}
+#endif /* INET6 */
+
 /*
  * Mbuf tag KPIs
  */
@@ -632,14 +765,14 @@ struct mbuf_tag_id_entry {
 	 strlen(__str) + 1)
 
 #define	MTAG_FIRST_ID					1000
-static u_long							mtag_id_next = MTAG_FIRST_ID;
+static mbuf_tag_id_t					mtag_id_next = MTAG_FIRST_ID;
 static SLIST_HEAD(,mbuf_tag_id_entry)	mtag_id_list = {NULL};
 static lck_mtx_t						*mtag_id_lock = NULL;
 
 __private_extern__ void
 mbuf_tag_id_first_last(
-	u_long	*first,
-	u_long	*last)
+	mbuf_tag_id_t * first,
+	mbuf_tag_id_t * last)
 {
 	*first = MTAG_FIRST_ID;
 	*last = mtag_id_next - 1;
@@ -647,11 +780,11 @@ mbuf_tag_id_first_last(
 
 __private_extern__ errno_t
 mbuf_tag_id_find_internal(
-	const char	*string,
-	u_long		*out_id,
-	int			create)
+	const char		*string,
+	mbuf_tag_id_t	*out_id,
+	int				create)
 {
-	struct mbuf_tag_id_entry				*entry = NULL;
+	struct mbuf_tag_id_entry			*entry = NULL;
 	
 	
 	*out_id = 0;
@@ -691,7 +824,7 @@ mbuf_tag_id_find_internal(
 	/* Look for an existing entry */
 	lck_mtx_lock(mtag_id_lock);
 	SLIST_FOREACH(entry, &mtag_id_list, next) {
-		if (strcmp(string, entry->string) == 0) {
+		if (strncmp(string, entry->string, strlen(string) + 1) == 0) {
 			break;
 		}
 	}
@@ -708,7 +841,7 @@ mbuf_tag_id_find_internal(
 			return ENOMEM;
 		}
 		
-		strcpy(entry->string, string);
+		strlcpy(entry->string, string, strlen(string)+1);
 		entry->id = mtag_id_next;
 		mtag_id_next++;
 		SLIST_INSERT_HEAD(&mtag_id_list, entry, next);
@@ -725,7 +858,7 @@ mbuf_tag_id_find(
 	const char		*string,
 	mbuf_tag_id_t	*out_id)
 {
-	return mbuf_tag_id_find_internal(string, (u_long*)out_id, 1);
+	return mbuf_tag_id_find_internal(string, out_id, 1);
 }
 
 errno_t
@@ -864,6 +997,39 @@ mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf
 		else
 			error = ENOMEM;
 	} else {
+		if (maxchunks)
+			*maxchunks = numchunks;
+		error = 0;
+		*mbuf = m;
+	}
+out:
+	return error;
+}
+
+errno_t
+mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf)
+{
+	errno_t error;
+	struct mbuf *m;
+	unsigned int numchunks = maxchunks ? *maxchunks : 0;
+
+	if (numpkts == 0) {
+		error = EINVAL;
+		goto out;
+	}
+	if (packetlen == 0) {
+		error = EINVAL;
+		goto out;
+	}
+	m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0);
+	if (m == 0) {
+		if (maxchunks && *maxchunks && numchunks > *maxchunks)
+			error = ENOBUFS;
+		else
+			error = ENOMEM;
+	} else {
+		if (maxchunks)
+			*maxchunks = numchunks;
 		error = 0;
 		*mbuf = m;
 	}
@@ -872,6 +1038,7 @@ out:
 }
 
 
+
 /*
  * mbuf_copyback differs from m_copyback in a few ways:
  * 1) mbuf_copyback will allocate clusters for new mbufs we append
@@ -949,3 +1116,12 @@ out:
 	
 	return result;
 }
+
+#if !INET6
+void inet6_unsupported(void);
+
+void inet6_unsupported(void)
+{
+	*((int *)0) = 0x1;
+}
+#endif /* !INET6 */
diff --git a/bsd/kern/kpi_mbuf_internal.h b/bsd/kern/kpi_mbuf_internal.h
new file mode 100644
index 000000000..8b424e2d9
--- /dev/null
+++ b/bsd/kern/kpi_mbuf_internal.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef __KPI_MBUF_INTERNAL_
+#define __KPI_MBUF_INTERNAL_
+
+#include <sys/kpi_mbuf.h>
+
+void mbuf_tag_id_first_last(mbuf_tag_id_t * first, mbuf_tag_id_t * last);
+errno_t mbuf_tag_id_find_internal(const char *string, 
+				  mbuf_tag_id_t * out_id, int create);
+#endif __KPI_MBUF_INTERNAL_
diff --git a/bsd/kern/kpi_socket.c b/bsd/kern/kpi_socket.c
index d589b2ab2..f7658fee6 100644
--- a/bsd/kern/kpi_socket.c
+++ b/bsd/kern/kpi_socket.c
@@ -1,26 +1,33 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #define __KPI__
+#include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -37,8 +44,8 @@
 #include <sys/uio_internal.h>
 #include <kern/lock.h>
 
-extern void	*memcpy(void *, const void *, size_t);
 extern int soclose_locked(struct socket *so);
+extern void soclose_wait_locked(struct socket *so);
 
 errno_t sock_send_internal(
 	socket_t			sock,
@@ -47,6 +54,7 @@ errno_t sock_send_internal(
 	int					flags,
 	size_t				*sentlen);
 
+typedef	void	(*so_upcall)(struct socket *, caddr_t , int );
 
 
 errno_t
@@ -95,7 +103,7 @@ sock_accept(
 			sock->so_error = ECONNABORTED;
 			break;
 		}
-		error = msleep((caddr_t)&sock->so_timeo, mutex_held, PSOCK | PCATCH, "sock_accept", 0);
+		error = msleep((caddr_t)&sock->so_timeo, mutex_held, PSOCK | PCATCH, "sock_accept", NULL);
 		if (error) {
 			socket_unlock(sock, 1);
 			return (error);
@@ -111,7 +119,26 @@ sock_accept(
 	new_so = TAILQ_FIRST(&sock->so_comp);
 	TAILQ_REMOVE(&sock->so_comp, new_so, so_list);
 	sock->so_qlen--;
-	socket_unlock(sock, 1);	/* release the head */
+
+	/*
+	 * Pass the pre-accepted socket to any interested socket filter(s).
+	 * Upon failure, the socket would have been closed by the callee.
+	 */
+	if (new_so->so_filt != NULL) {
+		/*
+		 * Temporarily drop the listening socket's lock before we
+		 * hand off control over to the socket filter(s), but keep
+		 * a reference so that it won't go away.  We'll grab it
+		 * again once we're done with the filter(s).
+		 */
+		socket_unlock(sock, 0);
+		if ((error = soacceptfilter(new_so)) != 0) {
+			/* Drop reference on listening socket */
+			sodereference(sock);
+			return (error);
+		}
+		socket_lock(sock, 0);
+	}
 
 	if (dosocklock)	{
 		lck_mtx_assert(new_so->so_proto->pr_getlock(new_so, 0),
@@ -121,12 +148,17 @@ sock_accept(
 	
 	new_so->so_state &= ~SS_COMP;
 	new_so->so_head = NULL;
-	soacceptlock(new_so, &sa, 0);
+	(void) soacceptlock(new_so, &sa, 0);
 	
+	socket_unlock(sock, 1);	/* release the head */
+
 	if (callback) {
-		new_so->so_upcall = callback;
+		new_so->so_upcall = (so_upcall) callback;
 		new_so->so_upcallarg = cookie;
 		new_so->so_rcv.sb_flags |= SB_UPCALL;
+#if CONFIG_SOWUPCALL
+		new_so->so_snd.sb_flags |= SB_UPCALL;
+#endif
 	}
 	
 	if (sa && from)
@@ -135,6 +167,20 @@ sock_accept(
 		memcpy(from, sa, fromlen);
 	}
 	if (sa) FREE(sa, M_SONAME);
+
+	/*
+	 * If the socket has been marked as inactive by soacceptfilter(),
+	 * disallow further operations on it.  We explicitly call shutdown
+	 * on both data directions to ensure that SS_CANT{RCV,SEND}MORE
+	 * states are set for the socket.  This would also flush out data
+	 * hanging off the receive list of this socket.
+	 */
+	if (new_so->so_flags & SOF_DEFUNCT) {
+		(void) soshutdownlock(new_so, SHUT_RD);
+		(void) soshutdownlock(new_so, SHUT_WR);
+		(void) sodisconnectlocked(new_so);
+	}
+
 	*new_sock = new_so;
 	if (dosocklock)	
 		socket_unlock(new_so, 1);
@@ -185,7 +231,7 @@ sock_connect(
 
 		while ((sock->so_state & SS_ISCONNECTING) && sock->so_error == 0) {
 			error = msleep((caddr_t)&sock->so_timeo, mutex_held, PSOCK | PCATCH,
-				"sock_connect", 0);
+				"sock_connect", NULL);
 			if (error)
 				break;
 		}
@@ -287,51 +333,77 @@ sock_nointerrupt(
 }
 
 errno_t
-sock_getpeername(
-	socket_t		sock,
-	struct sockaddr	*peername,
-	int				peernamelen)
+sock_getpeername(socket_t sock, struct sockaddr	*peername, int peernamelen)
 {
-	int				error = 0;
+	int error;
 	struct sockaddr	*sa = NULL;
-	
-	if (sock == NULL || peername == NULL || peernamelen < 0) return EINVAL;
+
+	if (sock == NULL || peername == NULL || peernamelen < 0)
+		return (EINVAL);
+
 	socket_lock(sock, 1);
-	if ((sock->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
+	if (!(sock->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING))) {
 		socket_unlock(sock, 1);
-		return ENOTCONN;
+		return (ENOTCONN);
 	}
-	error = sock->so_proto->pr_usrreqs->pru_peeraddr(sock, &sa);
-	if (!error)
-	{
-		if (peernamelen > sa->sa_len) peernamelen = sa->sa_len;
+	error = sock_getaddr(sock, &sa, 1);
+	socket_unlock(sock, 1);
+	if (error == 0) {
+		if (peernamelen > sa->sa_len)
+			peernamelen = sa->sa_len;
 		memcpy(peername, sa, peernamelen);
+		FREE(sa, M_SONAME);
 	}
-	if (sa) FREE(sa, M_SONAME);
-	socket_unlock(sock, 1);
-	return error;
+	return (error);
 }
 
 errno_t
-sock_getsockname(
-	socket_t		sock,
-	struct sockaddr	*sockname,
-	int				socknamelen)
+sock_getsockname(socket_t sock, struct sockaddr	*sockname, int socknamelen)
 {
-	int				error = 0;
+	int error;
 	struct sockaddr	*sa = NULL;
-	
-	if (sock == NULL || sockname == NULL || socknamelen < 0) return EINVAL;
+
+	if (sock == NULL || sockname == NULL || socknamelen < 0)
+		return (EINVAL);
+
 	socket_lock(sock, 1);
-	error = sock->so_proto->pr_usrreqs->pru_sockaddr(sock, &sa);
-	if (!error)
-	{
-		if (socknamelen > sa->sa_len) socknamelen = sa->sa_len;
+	error = sock_getaddr(sock, &sa, 0);
+	socket_unlock(sock, 1);
+	if (error == 0) {
+		if (socknamelen > sa->sa_len)
+			socknamelen = sa->sa_len;
 		memcpy(sockname, sa, socknamelen);
+		FREE(sa, M_SONAME);
 	}
-	if (sa) FREE(sa, M_SONAME);
-	socket_unlock(sock, 1);
-	return error;
+	return (error);
+}
+
+errno_t
+sock_getaddr(socket_t sock, struct sockaddr **psa, int peer)
+{
+	int error;
+
+	if (sock == NULL || psa == NULL)
+		return (EINVAL);
+
+	*psa = NULL;
+	error = peer ? sock->so_proto->pr_usrreqs->pru_peeraddr(sock, psa) :
+	    sock->so_proto->pr_usrreqs->pru_sockaddr(sock, psa);
+
+	if (error == 0 && *psa == NULL) {
+		error = ENOMEM;
+	} else if (error != 0 && *psa != NULL) {
+		FREE(*psa, M_SONAME);
+		*psa = NULL;
+	}
+	return (error);
+}
+
+void
+sock_freeaddr(struct sockaddr *sa)
+{
+	if (sa != NULL)
+		FREE(sa, M_SONAME);
 }
 
 errno_t
@@ -662,7 +734,6 @@ sock_shutdown(
 	return soshutdown(sock, how);
 }
 
-typedef	void	(*so_upcall)(struct socket *sock, void* arg, int waitf);
 
 errno_t
 sock_socket(
@@ -680,6 +751,9 @@ sock_socket(
 	if (error == 0 && callback)
 	{
 		(*new_so)->so_rcv.sb_flags |= SB_UPCALL;
+#if CONFIG_SOWUPCALL
+		(*new_so)->so_snd.sb_flags |= SB_UPCALL;
+#endif
 		(*new_so)->so_upcall = (so_upcall)callback;
 		(*new_so)->so_upcallarg = context;
 	}
@@ -708,19 +782,26 @@ sock_retain(
 
 /* Do we want this to be APPLE_PRIVATE API? */
 void
-sock_release(
-	socket_t	sock)
+sock_release(socket_t sock)
 {
-	if (sock == NULL) return;
+	if (sock == NULL)
+		return;
 	socket_lock(sock, 1);
+
+	if (sock->so_flags & SOF_UPCALLINUSE)
+		soclose_wait_locked(sock);
+
 	sock->so_retaincnt--;
 	if (sock->so_retaincnt < 0)
-		panic("sock_release: negative retain count for sock=%x cnt=%x\n",
-			sock, sock->so_retaincnt);
-	if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2))
-		soclose_locked(sock); /* close socket only if the FD is not holding it */
-	else
-		sock->so_usecount--;	/* remove extra reference holding the socket */
+		panic("sock_release: negative retain count for sock=%p "
+		    "cnt=%x\n", sock, sock->so_retaincnt);
+	if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2)) {
+		/* close socket only if the FD is not holding it */
+		soclose_locked(sock);
+	} else {
+		/* remove extra reference holding the socket */
+		sock->so_usecount--;
+	}
 	socket_unlock(sock, 1);
 }
 
@@ -782,3 +863,17 @@ sock_gettype(
 	socket_unlock(sock, 1);
 	return 0;
 }
+
+/*
+ * Return the listening socket of a pre-accepted socket.  It returns the
+ * listener (so_head) value of a given socket.  This is intended to be
+ * called by a socket filter during a filter attach (sf_attach) callback.
+ * The value returned by this routine is safe to be used only in the
+ * context of that callback, because we hold the listener's lock across
+ * the sflt_initsock() call.
+ */
+socket_t
+sock_getlistener(socket_t sock)
+{
+	return (sock->so_head);
+}
diff --git a/bsd/kern/kpi_socketfilter.c b/bsd/kern/kpi_socketfilter.c
index 10b73c86d..377bb2e84 100644
--- a/bsd/kern/kpi_socketfilter.c
+++ b/bsd/kern/kpi_socketfilter.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/kpi_socketfilter.h>
@@ -30,6 +36,8 @@
 #include <kern/locks.h>
 #include <net/kext_net.h>
 
+#include <string.h>
+
 static struct socket_filter_list	sock_filter_head;
 static lck_mtx_t					*sock_filter_lock = 0;
 
@@ -273,7 +281,6 @@ sflt_detach_private(
 	struct socket_filter_entry *entry,
 	int	unregistering)
 {
-	struct socket *so = entry->sfe_socket;
 	struct socket_filter_entry **next_ptr;
 	int				detached = 0;
 	int				found = 0;
@@ -407,53 +414,74 @@ sflt_detach(
 errno_t
 sflt_register(
 	const struct sflt_filter	*filter,
-	int							domain,
-	int							type,
-	int							protocol)
+	int				domain,
+	int				type,
+	int				protocol)
 {
 	struct socket_filter *sock_filt = NULL;
 	struct socket_filter *match = NULL;
 	int error = 0;
 	struct protosw *pr = pffindproto(domain, protocol, type);
-	
-	if (pr == NULL) return ENOENT;
-	
-	if (filter->sf_attach == NULL || filter->sf_detach == NULL) return EINVAL;
-	if (filter->sf_handle == 0) return EINVAL;
-	if (filter->sf_name == NULL) return EINVAL;
+	unsigned int len;
+
+	if (pr == NULL)
+		return ENOENT;
+
+	if (filter->sf_attach == NULL || filter->sf_detach == NULL ||
+	    filter->sf_handle == 0 || filter->sf_name == NULL)
+		return EINVAL;
 
 	/* Allocate the socket filter */
-	MALLOC(sock_filt, struct socket_filter*, sizeof(*sock_filt), M_IFADDR, M_WAITOK);
+	MALLOC(sock_filt, struct socket_filter *, sizeof (*sock_filt),
+	    M_IFADDR, M_WAITOK);
 	if (sock_filt == NULL) {
 		return ENOBUFS;
 	}
-	
-	bzero(sock_filt, sizeof(*sock_filt));
-	sock_filt->sf_filter = *filter;
-	
+
+	bzero(sock_filt, sizeof (*sock_filt));
+
+	/* Legacy sflt_filter length; current structure minus extended */
+	len = sizeof (*filter) - sizeof (struct sflt_filter_ext);
+	/*
+	 * Include extended fields if filter defines SFLT_EXTENDED.
+	 * We've zeroed out our internal sflt_filter placeholder,
+	 * so any unused portion would have been taken care of.
+	 */
+	if (filter->sf_flags & SFLT_EXTENDED) {
+		unsigned int ext_len = filter->sf_len;
+
+		if (ext_len > sizeof (struct sflt_filter_ext))
+			ext_len = sizeof (struct sflt_filter_ext);
+
+		len += ext_len;
+	}
+	bcopy(filter, &sock_filt->sf_filter, len);
+
 	lck_mtx_lock(sock_filter_lock);
 	/* Look for an existing entry */
 	TAILQ_FOREACH(match, &sock_filter_head, sf_global_next) {
-		if (match->sf_filter.sf_handle == sock_filt->sf_filter.sf_handle) {
+		if (match->sf_filter.sf_handle ==
+		    sock_filt->sf_filter.sf_handle) {
 			break;
 		}
 	}
-	
+
 	/* Add the entry only if there was no existing entry */
 	if (match == NULL) {
 		TAILQ_INSERT_TAIL(&sock_filter_head, sock_filt, sf_global_next);
 		if ((sock_filt->sf_filter.sf_flags & SFLT_GLOBAL) != 0) {
-			TAILQ_INSERT_TAIL(&pr->pr_filter_head, sock_filt, sf_protosw_next);
+			TAILQ_INSERT_TAIL(&pr->pr_filter_head, sock_filt,
+			    sf_protosw_next);
 			sock_filt->sf_proto = pr;
 		}
 	}
 	lck_mtx_unlock(sock_filter_lock);
-	
+
 	if (match != NULL) {
 		FREE(sock_filt, M_IFADDR);
 		return EEXIST;
 	}
-	
+
 	return error;
 }
 
@@ -566,7 +594,7 @@ sock_inject_data_out(
 {
 	int	sosendflags = 0;
 	if (flags & sock_data_filt_flag_oob) sosendflags = MSG_OOB;
-	return sosend(so, (const struct sockaddr*)to, NULL,
+	return sosend(so, (struct sockaddr*)to, NULL,
 				  data, control, sosendflags);
 }
 
diff --git a/bsd/kern/mach_fat.c b/bsd/kern/mach_fat.c
index 7002441de..9a75e6861 100644
--- a/bsd/kern/mach_fat.c
+++ b/bsd/kern/mach_fat.c
@@ -1,33 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1991-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1991 NeXT Computer, Inc.  All rights reserved.
- *
- *	File:	kern/mach_fat.c
- *	Author:	Peter King
- *
- *	Fat file support routines.
- *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/uio.h>
@@ -79,7 +76,9 @@ fatfile_getarch2(
 	int			grade;
 	int			best_grade;
 	int			nfat_arch;
-	off_t		end_of_archs;
+	off_t			end_of_archs;
+	cpu_type_t		testtype;
+	cpu_type_t		testsubtype;
 	struct fat_header	*header;
 #if 0
 	off_t filesize;
@@ -97,8 +96,8 @@ fatfile_getarch2(
 	 */
 	nfat_arch = OSSwapBigToHostInt32(header->nfat_arch);
 
-	end_of_archs = (off_t)nfat_arch * sizeof(struct fat_arch) + 
-		sizeof(struct fat_header);
+	end_of_archs = (off_t)nfat_arch * sizeof(struct fat_arch) +
+			sizeof(struct fat_header);
 #if 0
 	filesize = ubc_getsize(vp);
 	if (end_of_archs > (int)filesize) {
@@ -111,7 +110,7 @@ fatfile_getarch2(
 	 * only PAGE_SIZE bytes
 	 */
 	if (end_of_archs > PAGE_SIZE ||
-		end_of_archs < (sizeof(struct fat_header)+sizeof(struct fat_arch)))
+	    end_of_archs < (sizeof(struct fat_header)+sizeof(struct fat_arch)))
 		return(LOAD_BADMACHO);
 
 	/*
@@ -122,26 +121,41 @@ fatfile_getarch2(
 		return(LOAD_BADMACHO);
 
 	/*
-	 * Scan the fat_arch's looking for the best one.
+	 * Ignore LIB64 flag so that binary slices with the flag set
+	 * don't choke in grade_binary.
 	 */
+	mask_bits |= CPU_SUBTYPE_LIB64;
+
+	/*
+	 * Scan the fat_arch's looking for the best one.  */
 	addr = data_ptr;
 	best_arch = NULL;
 	best_grade = 0;
 	arch = (struct fat_arch *) (addr + sizeof(struct fat_header));
 	for (; nfat_arch-- > 0; arch++) {
 
+		/* 
+		 *	Collect flags from both cputype and cpusubtype 
+		 */
+ 		testtype = OSSwapBigToHostInt32(arch->cputype) |
+ 				(OSSwapBigToHostInt32(arch->cpusubtype) & 
+ 					CPU_SUBTYPE_MASK);
+ 		testsubtype = OSSwapBigToHostInt32(arch->cpusubtype) 
+ 			& ~CPU_SUBTYPE_MASK;
+
 		/*
 		 *	Check to see if right cpu type.
 		 */
-		if(((cpu_type_t)OSSwapBigToHostInt32(arch->cputype) & ~mask_bits) != req_cpu_type)
+ 		if((testtype & ~mask_bits) != req_cpu_type) {
 			continue;
+		}
 
 		/*
-		 * 	Get the grade of the cpu subtype.
+		 * 	Get the grade of the cpu subtype (without feature flags)
 		 */
-		grade = grade_binary(
-			    OSSwapBigToHostInt32(arch->cputype),
-			    OSSwapBigToHostInt32(arch->cpusubtype));
+ 		grade = grade_binary(
+				(testtype & ~CPU_SUBTYPE_LIB64), 
+				testsubtype);
 
 		/*
 		 *	Remember it if it's the best we've seen.
@@ -197,13 +211,14 @@ fatfile_getarch_affinity(
 				fallback_type = CPU_TYPE_POWERPC;
 		}
 		/*
-		 * Ignore the architectural bits when determining if an image
+		 * Ignore all architectural bits when determining if an image
 		 * in a fat file should be skipped or graded.
 		 */
-		lret = fatfile_getarch2(vp, data_ptr, primary_type, CPU_ARCH_MASK, archret);
+		lret = fatfile_getarch2(vp, data_ptr, primary_type, 
+				CPU_ARCH_MASK, archret);
 		if ((lret != 0) && handler) {
 			lret = fatfile_getarch2(vp, data_ptr, fallback_type,
-						0, archret);
+						CPU_SUBTYPE_LIB64, archret);
 		}
 		return lret;
 }
@@ -228,7 +243,8 @@ fatfile_getarch(
 	vm_offset_t 	data_ptr,
 	struct fat_arch		*archret)
 {
-	return fatfile_getarch2(vp, data_ptr, cpu_type(), 0, archret);
+	return fatfile_getarch2(vp, data_ptr, cpu_type(), 
+			CPU_SUBTYPE_LIB64, archret);
 }
 
 /**********************************************************************
@@ -253,6 +269,7 @@ fatfile_getarch_with_bits(
 	vm_offset_t 	data_ptr,
 	struct fat_arch		*archret)
 {
-	return fatfile_getarch2(vp, data_ptr, archbits | cpu_type(), 0, archret);
+	return fatfile_getarch2(vp, data_ptr, archbits | cpu_type(), 
+			CPU_SUBTYPE_LIB64, archret);
 }
 
diff --git a/bsd/kern/mach_header.c b/bsd/kern/mach_header.c
index 9071eaa48..86efc1f82 100644
--- a/bsd/kern/mach_header.c
+++ b/bsd/kern/mach_header.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.c
@@ -44,6 +50,9 @@
 #if !defined(KERNEL_PRELOAD)
 #include <kern/mach_header.h>
 #include <string.h>		// from libsa
+#if DEBUG
+#include <libkern/libkern.h>
+#endif
 
 extern struct mach_header _mh_execute_header;
 
@@ -353,7 +362,8 @@ getsegbyname(const char *seg_name)
 	 * For the kernel's header add on the faked segment for the
 	 * USER boot code identified by a FVMFILE_COMMAND in the mach header.
 	 */
-	if (!this && strcmp(seg_name, fvm_seg->segname) == 0)
+	if (!this && strncmp(seg_name, fvm_seg->segname,
+				sizeof(fvm_seg->segname)) == 0)
 		this = fvm_seg;
 
 	return this;
@@ -451,7 +461,7 @@ getfakefvmseg(void)
 	sgp->vmaddr = fvp->header_addr;
 	sgp->vmsize = getsizeofmacho((struct mach_header *)(sgp->vmaddr));
 
-	strcpy(sp->sectname, fvp->name.ptr);
+	strlcpy(sp->sectname, fvp->name.ptr, sizeof(sp->sectname));
 	sp->addr = sgp->vmaddr;
 	sp->size = sgp->vmsize;
 
diff --git a/bsd/kern/mach_header.h b/bsd/kern/mach_header.h
index ff667a6f4..28cdd5b2e 100644
--- a/bsd/kern/mach_header.h
+++ b/bsd/kern/mach_header.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.h
diff --git a/bsd/kern/mach_loader.c b/bsd/kern/mach_loader.c
index 00dd19683..9a4f954b9 100644
--- a/bsd/kern/mach_loader.c
+++ b/bsd/kern/mach_loader.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (C) 1988, 1989,  NeXT, Inc.
@@ -48,11 +54,11 @@
 #include <mach/vm_map.h>	/* vm_allocate() */
 #include <mach/mach_vm.h>	/* mach_vm_allocate() */
 #include <mach/vm_statistics.h>
-#include <mach/shared_memory_server.h>
 #include <mach/task.h>
 #include <mach/thread_act.h>
 
 #include <machine/vmparam.h>
+#include <machine/exec.h>
 
 #include <kern/kern_types.h>
 #include <kern/cpu_number.h>
@@ -69,7 +75,6 @@
 #include <vm/vm_kern.h>
 #include <vm/vm_pager.h>
 #include <vm/vnode_pager.h>
-#include <vm/vm_shared_memory_server.h>
 #include <vm/vm_protos.h> 
 
 /*
@@ -78,7 +83,6 @@
  */
 extern pmap_t	pmap_create(vm_map_size_t size, boolean_t is_64bit);
 extern void	pmap_switch(pmap_t);
-extern void	pmap_map_sharedpage(task_t task, pmap_t pmap);
 
 /*
  * XXX kern/thread.h should not treat these prototypes as MACH_KERNEL_PRIVATE
@@ -98,16 +102,19 @@ extern kern_return_t	thread_userstack(thread_t, int, thread_state_t,
 extern kern_return_t	thread_entrypoint(thread_t, int, thread_state_t,
 				unsigned int, mach_vm_offset_t *);
 
+extern kern_return_t	memory_object_signed(memory_object_control_t control,
+					     boolean_t is_signed);
 
 /* An empty load_result_t */
 static load_result_t load_result_null = {
-	MACH_VM_MIN_ADDRESS,
-	MACH_VM_MIN_ADDRESS,
-	MACH_VM_MIN_ADDRESS,
-	0,
-	0,
-	0,
-	0
+	.mach_header = MACH_VM_MIN_ADDRESS,
+	.entry_point = MACH_VM_MIN_ADDRESS,
+	.user_stack = MACH_VM_MIN_ADDRESS,
+	.thread_count = 0,
+	.unixproc = 0,
+	.dynlinker = 0,
+	.customstack = 0,
+	.csflags = 0
 };
 
 /*
@@ -117,12 +124,10 @@ static load_return_t
 parse_machfile(
 	struct vnode		*vp,
 	vm_map_t		map,
-	thread_t		thr_act,
+	thread_t		thread,
 	struct mach_header	*header,
 	off_t			file_offset,
 	off_t			macho_size,
-	boolean_t		shared_regions,
-	boolean_t		clean_regions,
 	int			depth,
 	load_result_t		*result
 );
@@ -149,17 +154,25 @@ load_segment_64(
 	load_result_t			*result
 );
 
+int load_code_signature(
+	struct linkedit_data_command	*lcp,
+	struct vnode			*vp,
+	off_t				macho_offset,
+	off_t				macho_size,
+	cpu_type_t			cputype,
+	load_result_t			*result);
+	
 static load_return_t
 load_unixthread(
 	struct thread_command	*tcp,
-	thread_t			thr_act,
+	thread_t			thread,
 	load_result_t			*result
 );
 
 static load_return_t
 load_thread(
 	struct thread_command	*tcp,
-	thread_t			thr_act,
+	thread_t			thread,
 	load_result_t			*result
 );
 
@@ -175,7 +188,7 @@ load_threadstack(
 	thread_t		thread,
 	unsigned long	*ts,
 	unsigned long	total_size,
-	mach_vm_offset_t	*user_stack,
+	user_addr_t	*user_stack,
 	int				*customstack
 );
 
@@ -192,10 +205,9 @@ load_dylinker(
 	struct dylinker_command	*lcp,
 	integer_t		archbits,
 	vm_map_t				map,
-	thread_t			thr_act,
+	thread_t			thread,
 	int						depth,
 	load_result_t			*result,
-	boolean_t			clean_regions,
 	boolean_t			is_64bit
 );
 
@@ -213,9 +225,8 @@ load_return_t
 load_machfile(
 	struct image_params	*imgp,
 	struct mach_header	*header,
-	thread_t 		thr_act,
+	thread_t 		thread,
 	vm_map_t 		new_map,
-	boolean_t		clean_regions,
 	load_result_t		*result
 )
 {
@@ -236,12 +247,7 @@ load_machfile(
 
 	if (create_map) {
 		old_map = current_map();
-#ifdef NO_NESTED_PMAP
-		pmap = get_task_pmap(current_task());
-		pmap_reference(pmap);
-#else	/* NO_NESTED_PMAP */
 		pmap = pmap_create((vm_map_size_t) 0, (imgp->ip_flags & IMGPF_IS_64BIT));
-#endif	/* NO_NESTED_PMAP */
 		map = vm_map_create(pmap,
 				0,
 				vm_compute_max_offset((imgp->ip_flags & IMGPF_IS_64BIT)),
@@ -251,15 +257,14 @@ load_machfile(
 
 	if ( (header->flags & MH_ALLOW_STACK_EXECUTION) )
 	        vm_map_disable_NX(map);
-
+		
 	if (!result)
 		result = &myresult;
 
 	*result = load_result_null;
 
-	lret = parse_machfile(vp, map, thr_act, header, file_offset, macho_size,
-			      ((imgp->ip_flags & IMGPF_IS_64BIT) == 0), /* shared regions? */
-			      clean_regions, 0, result);
+	lret = parse_machfile(vp, map, thread, header, file_offset, macho_size,
+			      0, result);
 
 	if (lret != LOAD_SUCCESS) {
 		if (create_map) {
@@ -294,16 +299,12 @@ load_machfile(
 
 		old_map = swap_task_map(current_task(), map);
 		vm_map_clear_4GB_pagezero(old_map);
-#ifndef NO_NESTED_PMAP
 		pmap_switch(pmap);	/* Make sure we are using the new pmap */
-#endif	/* !NO_NESTED_PMAP */
 		vm_map_deallocate(old_map);
 	}
 	return(LOAD_SUCCESS);
 }
 
-int	dylink_test = 1;
-
 /*
  * The file size of a mach-o file is limited to 32 bits; this is because
  * this is the limit on the kalloc() of enough bytes for a mach_header and
@@ -318,12 +319,10 @@ load_return_t
 parse_machfile(
 	struct vnode 		*vp,       
 	vm_map_t		map,
-	thread_t		thr_act,
+	thread_t		thread,
 	struct mach_header	*header,
 	off_t			file_offset,
 	off_t			macho_size,
-	boolean_t		shared_regions,
-	boolean_t		clean_regions,
 	int			depth,
 	load_result_t		*result
 )
@@ -340,12 +339,13 @@ parse_machfile(
 	size_t			offset;
 	size_t			oldoffset;	/* for overflow check */
 	int			pass;
-	struct proc *p = current_proc();		/* XXXX */
+	proc_t			p = current_proc();		/* XXXX */
 	int			error;
 	int resid=0;
 	task_t task;
 	size_t			mach_header_sz = sizeof(struct mach_header);
 	boolean_t		abi64;
+	boolean_t		got_code_signatures = FALSE;
 
 	if (header->magic == MH_MAGIC_64 ||
 	    header->magic == MH_CIGAM_64) {
@@ -355,10 +355,11 @@ parse_machfile(
 	/*
 	 *	Break infinite recursion
 	 */
-	if (depth > 6)
+	if (depth > 6) {
 		return(LOAD_FAILURE);
+	}
 
-	task = (task_t)get_threadtask(thr_act);
+	task = (task_t)get_threadtask(thread);
 
 	depth++;
 
@@ -366,7 +367,8 @@ parse_machfile(
 	 *	Check to see if right machine type.
 	 */
 	if (((cpu_type_t)(header->cputype & ~CPU_ARCH_MASK) != cpu_type()) ||
-	    !grade_binary(header->cputype, header->cpusubtype))
+	    !grade_binary(header->cputype, 
+	    	header->cpusubtype & ~CPU_SUBTYPE_MASK))
 		return(LOAD_BADARCH);
 		
 	abi64 = ((header->cputype & CPU_ARCH_ABI64) == CPU_ARCH_ABI64);
@@ -376,19 +378,22 @@ parse_machfile(
 	case MH_OBJECT:
 	case MH_EXECUTE:
 	case MH_PRELOAD:
-		if (depth != 1)
+		if (depth != 1) {
 			return (LOAD_FAILURE);
+		}
 		break;
 		
 	case MH_FVMLIB:
 	case MH_DYLIB:
-		if (depth == 1)
+		if (depth == 1) {
 			return (LOAD_FAILURE);
+		}
 		break;
 
 	case MH_DYLINKER:
-		if (depth != 2)
+		if (depth != 2) {
 			return (LOAD_FAILURE);
+		}
 		break;
 		
 	default:
@@ -398,7 +403,6 @@ parse_machfile(
 	/*
 	 *	Get the pager for the file.
 	 */
-	UBCINFOCHECK("parse_machfile", vp);
 	pager = (void *) ubc_getpager(vp);
 
 	/*
@@ -502,7 +506,7 @@ parse_machfile(
 				if (pass != 2)
 					break;
 				ret = load_thread((struct thread_command *)lcp,
-						   thr_act,
+						   thread,
 						  result);
 				break;
 			case LC_UNIXTHREAD:
@@ -510,7 +514,7 @@ parse_machfile(
 					break;
 				ret = load_unixthread(
 						 (struct thread_command *) lcp,
-						   thr_act,
+						   thread,
 						 result);
 				break;
 			case LC_LOAD_DYLINKER:
@@ -523,6 +527,30 @@ parse_machfile(
 					ret = LOAD_FAILURE;
 				}
 				break;
+			case LC_CODE_SIGNATURE:
+				/* CODE SIGNING */
+				if (pass != 2)
+					break;
+				/* pager -> uip ->
+				   load signatures & store in uip
+				   set VM object "signed_pages"
+				*/
+				ret = load_code_signature(
+					(struct linkedit_data_command *) lcp,
+					vp,
+					file_offset,
+					macho_size,
+					header->cputype,
+					(depth == 1) ? result : NULL);
+				if (ret != LOAD_SUCCESS) {
+					printf("proc %d: load code signature error %d "
+					       "for file \"%s\"\n",
+					       p->p_pid, ret, vp->v_name);
+					ret = LOAD_SUCCESS; /* ignore error */
+				} else {
+					got_code_signatures = TRUE;
+				}
+				break;
 			default:
 				/* Other commands are ignored by the kernel */
 				ret = LOAD_SUCCESS;
@@ -535,129 +563,33 @@ parse_machfile(
 			break;
 	}
 	if (ret == LOAD_SUCCESS) { 
+	    if (! got_code_signatures) {
+		    struct cs_blob *blob;
+		    /* no embedded signatures: look for detached ones */
+		    blob = ubc_cs_blob_get(vp, -1, file_offset);
+		    if (blob != NULL) {
+			    /* get flags to be applied to the process */
+			    result->csflags |= blob->csb_flags;
+		    }
+	    }
 
-	    if (shared_regions) {
-		vm_offset_t vmaddr;
-		shared_region_mapping_t shared_region;
-		struct shared_region_task_mappings	map_info;
-		shared_region_mapping_t next;
-
-RedoLookup:
-		vm_get_shared_region(task, &shared_region);
-		map_info.self = (vm_offset_t)shared_region;
-		shared_region_mapping_info(shared_region,
-			&(map_info.text_region),   
-			&(map_info.text_size),
-			&(map_info.data_region),
-			&(map_info.data_size),
-			&(map_info.region_mappings),
-			&(map_info.client_base),
-			&(map_info.alternate_base),
-			&(map_info.alternate_next), 
-			&(map_info.fs_base),
-			&(map_info.system),
-			&(map_info.flags), &next);
-
-		if((map_info.flags & SHARED_REGION_FULL) ||
-			(map_info.flags & SHARED_REGION_STALE)) {
-			shared_region_mapping_t system_region;
-			system_region = lookup_default_shared_region(
-				map_info.fs_base, map_info.system);
-			if((map_info.self != (vm_offset_t)system_region) &&
-				(map_info.flags & SHARED_REGION_SYSTEM)) {
-			   if(system_region == NULL) {
-				shared_file_boot_time_init(
-					map_info.fs_base, map_info.system);
-			   } else {
-			   	vm_set_shared_region(task, system_region);
-			   }
-			   shared_region_mapping_dealloc(
-					(shared_region_mapping_t)map_info.self);
-			   goto RedoLookup;
-			} else if (map_info.flags & SHARED_REGION_SYSTEM) {
-			      shared_region_mapping_dealloc(system_region);
-			      shared_file_boot_time_init(
-					map_info.fs_base, map_info.system);
-			      shared_region_mapping_dealloc(
-				     (shared_region_mapping_t)map_info.self);
-			} else {
-			      shared_region_mapping_dealloc(system_region);
-			}
-		}
-
-		if (dylink_test) {
-			p->p_flag |=  P_NOSHLIB; /* no shlibs in use */
-			vmaddr = map_info.client_base;
-			if(clean_regions) {
-			   vm_map(map, &vmaddr, map_info.text_size, 
-				0, SHARED_LIB_ALIAS|VM_FLAGS_FIXED,
-				map_info.text_region, 0, FALSE,
-				VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE);
-			} else {
-			   vm_map(map, &vmaddr, map_info.text_size, 0, 
-				(VM_MEMORY_SHARED_PMAP << 24) 
-				  | SHARED_LIB_ALIAS | VM_FLAGS_FIXED,
-				map_info.text_region, 0, FALSE,
-				VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE);
-			}
-			vmaddr = map_info.client_base + map_info.text_size;
-			vm_map(map, &vmaddr, map_info.data_size, 
-				0, SHARED_LIB_ALIAS | VM_FLAGS_FIXED,
-				map_info.data_region, 0, TRUE,
-				VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE);
-	
-			while (next) {
-		           /* this should be fleshed out for the general case */
-			   /* but this is not necessary for now.  Indeed we   */
-			   /* are handling the com page inside of the         */
-			   /* shared_region mapping create calls for now for  */
-			   /* simplicities sake.  If more general support is  */
-			   /* needed the code to manipulate the shared range  */
-			   /* chain can be pulled out and moved to the callers*/
-			   shared_region_mapping_info(next,
-				&(map_info.text_region),   
-				&(map_info.text_size),
-				&(map_info.data_region),
-				&(map_info.data_size),
-				&(map_info.region_mappings),
-				&(map_info.client_base),
-				&(map_info.alternate_base),
-				&(map_info.alternate_next), 
-				&(map_info.fs_base),
-				&(map_info.system),
-				&(map_info.flags), &next);
-
-			   vmaddr = map_info.client_base;
-			   vm_map(map, &vmaddr, map_info.text_size, 
-				0, SHARED_LIB_ALIAS | VM_FLAGS_FIXED,
-				map_info.text_region, 0, FALSE,
-				VM_PROT_READ, VM_PROT_READ, VM_INHERIT_SHARE);
-			}
-		}
-            }
 	    if (dlp != 0)
-			ret = load_dylinker(dlp, dlarchbits, map, thr_act, depth, result, clean_regions, abi64);
+			ret = load_dylinker(dlp, dlarchbits, map, thread, depth, result, abi64);
 
 	    if(depth == 1) {
-		if (result->thread_count == 0)
+		if (result->thread_count == 0) {
 			ret = LOAD_FAILURE;
-		else if ( abi64 ) {
+		} else if ( abi64 ) {
+#ifdef __ppc__
 			/* Map in 64-bit commpage */
 			/* LP64todo - make this clean */
-			pmap_map_sharedpage(current_task(), get_map_pmap(map));
-                        vm_map_commpage64(map);
-		} else {
-#ifdef __i386__
 			/*
-			 * On Intel, the comm page doesn't get mapped
-			 * automatically because it goes beyond the current end
-			 * of the VM map in the current 3GB/1GB address space
-			 * model.
-			 * XXX This will probably become unnecessary when we
-			 * switch to the 4GB/4GB address space model.
+			 * PPC51: ppc64 is limited to 51-bit addresses.
+			 * Memory above that limit is handled specially
+			 * at the pmap level.
 			 */
-			vm_map_commpage32(map);
-#endif	/* __i386__ */
+			pmap_map_sharedpage(current_task(), get_map_pmap(map));
+#endif /* __ppc__ */
 		}
 	    }
 	}
@@ -666,15 +598,11 @@ RedoLookup:
 		kfree(kl_addr, kl_size);
 
 	if (ret == LOAD_SUCCESS)
-		(void)ubc_map(vp, PROT_EXEC);
+		(void)ubc_map(vp, PROT_READ | PROT_EXEC);
 		
 	return(ret);
 }
 
-#ifndef SG_PROTECTED_VERSION_1
-#define SG_PROTECTED_VERSION_1 0x8
-#endif /* SG_PROTECTED_VERSION_1 */
-
 #ifdef __i386__
 
 #define	APPLE_UNPROTECTED_HEADER_SIZE	(3 * PAGE_SIZE_64)
@@ -751,6 +679,11 @@ load_segment(
 	 */
 	if (scp->fileoff + scp->filesize > macho_size)
 		return (LOAD_BADMACHO);
+	/*
+	 * Make sure the segment is page-aligned in the file.
+	 */
+	if ((scp->fileoff & PAGE_MASK) != 0)
+		return LOAD_BADMACHO;
 
 	seg_size = round_page(scp->vmsize);
 	if (seg_size == 0)
@@ -881,6 +814,11 @@ load_segment_64(
 	 */
 	if (scp64->fileoff + scp64->filesize > (uint64_t)macho_size)
 		return (LOAD_BADMACHO);
+	/*
+	 * Make sure the segment is page-aligned in the file.
+	 */
+	if ((scp64->fileoff & PAGE_MASK_64) != 0)
+		return LOAD_BADMACHO;
 
 	seg_size = round_page_64(scp64->vmsize);
 	if (seg_size == 0)
@@ -944,7 +882,7 @@ load_segment_64(
 								delta_size)) {
 				(void) vm_deallocate(
 						kernel_map, tmp, delta_size);
-		return (LOAD_FAILURE);
+				return (LOAD_FAILURE);
 			}
 	
 			(void) vm_deallocate(kernel_map, tmp, delta_size);
@@ -1000,7 +938,7 @@ load_thread(
 
 	task = get_threadtask(thread);
 
-	/* if count is 0; same as thr_act */
+	/* if count is 0; same as thread */
 	if (result->thread_count != 0) {
 		kret = thread_create(task, &thread);
 		if (kret != KERN_SUCCESS)
@@ -1062,8 +1000,10 @@ load_unixthread(
 	load_return_t	ret;
 	int customstack =0;
 	
-	if (result->thread_count != 0)
+	if (result->thread_count != 0) {
+printf("load_unixthread: already have a thread!");
 		return (LOAD_FAILURE);
+	}
 	
 	ret = load_threadstack(thread,
 		       (unsigned long *)(((vm_offset_t)tcp) + 
@@ -1113,8 +1053,9 @@ load_threadstate(
 	unsigned long	thread_size;
 
     ret = thread_state_initialize( thread );
-    if (ret != KERN_SUCCESS)
+    if (ret != KERN_SUCCESS) {
         return(LOAD_FAILURE);
+    }
     
 	/*
 	 *	Set the new thread state; iterate through the state flavors in
@@ -1133,8 +1074,9 @@ load_threadstate(
 		 * based on the value of flavor.
 		 */
 		ret = thread_setstatus(thread, flavor, (thread_state_t)ts, size);
-		if (ret != KERN_SUCCESS)
+		if (ret != KERN_SUCCESS) {
 			return(LOAD_FAILURE);
+		}
 		ts += size;	/* ts is a (unsigned long *) */
 	}
 	return(LOAD_SUCCESS);
@@ -1169,8 +1111,9 @@ load_threadstack(
 		 * the value of flavor.
 		 */
 		ret = thread_userstack(thread, flavor, (thread_state_t)ts, size, user_stack, customstack);
-		if (ret != KERN_SUCCESS)
+		if (ret != KERN_SUCCESS) {
 			return(LOAD_FAILURE);
+		}
 		ts += size;	/* ts is a (unsigned long *) */
 	}
 	return(LOAD_SUCCESS);
@@ -1207,8 +1150,9 @@ load_threadentry(
 		 * the value of flavor.
 		 */
 		ret = thread_entrypoint(thread, flavor, (thread_state_t)ts, size, entry_point);
-		if (ret != KERN_SUCCESS)
+		if (ret != KERN_SUCCESS) {
 			return(LOAD_FAILURE);
+		}
 		ts += size;	/* ts is a (unsigned long *) */
 	}
 	return(LOAD_SUCCESS);
@@ -1221,19 +1165,18 @@ load_dylinker(
 	struct dylinker_command	*lcp,
 	integer_t		archbits,
 	vm_map_t		map,
-	thread_t	thr_act,
+	thread_t	thread,
 	int			depth,
 	load_result_t		*result,
-	boolean_t		clean_regions,
 	boolean_t		is_64bit
 )
 {
 	char			*name;
 	char			*p;
-	struct vnode		*vp;
+	struct vnode		*vp = NULLVP;	/* set by get_macho_vnode() */
 	struct mach_header	header;
-	off_t			file_offset;
-	off_t			macho_size;
+	off_t			file_offset = 0; /* set by get_macho_vnode() */
+	off_t			macho_size = 0;	/* set by get_macho_vnode() */
 	vm_map_t		copy_map;
 	load_result_t		myresult;
 	kern_return_t		ret;
@@ -1255,85 +1198,198 @@ load_dylinker(
 	if (ret)
 		return (ret);
 			
-	/*
-	 *	Load the Mach-O.
-	 *	Use a temporary map to do the work.
-	 */
-	copy_map = vm_map_create(pmap_create(vm_map_round_page(macho_size),
-					     is_64bit),
-				 get_map_min(map), get_map_max(map), TRUE);
-	if (VM_MAP_NULL == copy_map) {
-		ret = LOAD_RESOURCE;
-		goto out;
-	}
-
 	myresult = load_result_null;
 
-	ret = parse_machfile(vp, copy_map, thr_act, &header,
-				file_offset, macho_size,
-				FALSE, clean_regions, depth, &myresult);
-
-	if (ret)
-		goto out;
+	/*
+	 *	First try to map dyld in directly.  This should work most of
+	 *	the time since there shouldn't normally be something already
+	 *	mapped to its address.
+	 */
 
-	if (get_map_nentries(copy_map) > 0) {
+	ret = parse_machfile(vp, map, thread, &header, file_offset, macho_size,
+				depth, &myresult);
 
-		dyl_start = mach_get_vm_start(copy_map);
-		dyl_length = mach_get_vm_end(copy_map) - dyl_start;
+	/*
+	 *	If it turned out something was in the way, then we'll take
+	 *	take this longer path to map dyld into a temporary map and
+	 *	copy it into destination map at a different address.
+	 */
 
-		map_addr = dyl_start;
-		ret = mach_vm_allocate(map, &map_addr, dyl_length, VM_FLAGS_FIXED);
-		if (ret != KERN_SUCCESS)  {
-			ret = mach_vm_allocate(map, &map_addr, dyl_length, VM_FLAGS_ANYWHERE);
-		}
+	if (ret == LOAD_NOSPACE) {
 
-		if (ret != KERN_SUCCESS) {
-			ret = LOAD_NOSPACE;
+		/*
+		 *	Load the Mach-O.
+		 *	Use a temporary map to do the work.
+		 */
+		copy_map = vm_map_create(pmap_create(vm_map_round_page(macho_size),
+						     is_64bit),
+					 get_map_min(map), get_map_max(map), TRUE);
+		if (VM_MAP_NULL == copy_map) {
+			ret = LOAD_RESOURCE;
 			goto out;
-		
 		}
-		ret = vm_map_copyin(copy_map,
-				    (vm_map_address_t)dyl_start,
-				    (vm_map_size_t)dyl_length,
-				    TRUE, &tmp);
-		if (ret != KERN_SUCCESS) {
-			(void) vm_map_remove(map,
-				     vm_map_trunc_page(map_addr),
-				     vm_map_round_page(map_addr + dyl_length),
-				     VM_MAP_NO_FLAGS);
+	
+		myresult = load_result_null;
+
+		ret = parse_machfile(vp, copy_map, thread, &header,
+					file_offset, macho_size,
+					depth, &myresult);
+	
+		if (ret) {
+			vm_map_deallocate(copy_map);
 			goto out;
 		}
+	
+		if (get_map_nentries(copy_map) > 0) {
+	
+			dyl_start = mach_get_vm_start(copy_map);
+			dyl_length = mach_get_vm_end(copy_map) - dyl_start;
+	
+			map_addr = dyl_start;
+			ret = mach_vm_allocate(map, &map_addr, dyl_length, VM_FLAGS_ANYWHERE);
+	
+			if (ret != KERN_SUCCESS) {
+				vm_map_deallocate(copy_map);
+				ret = LOAD_NOSPACE;
+				goto out;
+			
+			}
 
-		ret = vm_map_copy_overwrite(map,
-				     (vm_map_address_t)map_addr,
-				     tmp, FALSE);
-		if (ret != KERN_SUCCESS) {
-			vm_map_copy_discard(tmp);
-			(void) vm_map_remove(map,
-				     vm_map_trunc_page(map_addr),
-				     vm_map_round_page(map_addr + dyl_length),
-				     VM_MAP_NO_FLAGS);
-			goto out;
+			ret = vm_map_copyin(copy_map,
+					    (vm_map_address_t)dyl_start,
+					    (vm_map_size_t)dyl_length,
+					    TRUE, &tmp);
+			if (ret != KERN_SUCCESS) {
+				(void) vm_map_remove(map,
+					     vm_map_trunc_page(map_addr),
+					     vm_map_round_page(map_addr + dyl_length),
+					     VM_MAP_NO_FLAGS);
+				vm_map_deallocate(copy_map);
+				goto out;
+			}
+	
+			ret = vm_map_copy_overwrite(map,
+					     (vm_map_address_t)map_addr,
+					     tmp, FALSE);
+			if (ret != KERN_SUCCESS) {
+				vm_map_copy_discard(tmp);
+				(void) vm_map_remove(map,
+					     vm_map_trunc_page(map_addr),
+					     vm_map_round_page(map_addr + dyl_length),
+					     VM_MAP_NO_FLAGS);
+				vm_map_deallocate(copy_map);
+				goto out;
+			}
+	
+			if (map_addr != dyl_start)
+				myresult.entry_point += (map_addr - dyl_start);
+		} else {
+			ret = LOAD_FAILURE;
 		}
 
-		if (map_addr != dyl_start)
-			myresult.entry_point += (map_addr - dyl_start);
-	} else
-		ret = LOAD_FAILURE;
+		vm_map_deallocate(copy_map);
+	}
 	
 	if (ret == LOAD_SUCCESS) {		
 		result->dynlinker = TRUE;
 		result->entry_point = myresult.entry_point;
-		(void)ubc_map(vp, PROT_EXEC);
+		(void)ubc_map(vp, PROT_READ | PROT_EXEC);
 	}
 out:
-	vm_map_deallocate(copy_map);
-	
 	vnode_put(vp);
 	return (ret);
 
 }
 
+int
+load_code_signature(
+	struct linkedit_data_command	*lcp,
+	struct vnode			*vp,
+	off_t				macho_offset,
+	off_t				macho_size,
+	cpu_type_t			cputype,
+	load_result_t			*result)
+{
+	int		ret;
+	kern_return_t	kr;
+	vm_offset_t	addr;
+	int		resid;
+	struct cs_blob	*blob;
+	int		error;
+
+	addr = 0;
+	blob = NULL;
+
+	if (lcp->cmdsize != sizeof (struct linkedit_data_command) ||
+	    lcp->dataoff + lcp->datasize > macho_size) {
+		ret = LOAD_BADMACHO;
+		goto out;
+	}
+
+	blob = ubc_cs_blob_get(vp, cputype, -1);
+	if (blob != NULL) {
+		/* we already have a blob for this vnode and cputype */
+		if (blob->csb_cpu_type == cputype &&
+		    blob->csb_base_offset == macho_offset &&
+		    blob->csb_mem_size == lcp->datasize) {
+			/* it matches the blob we want here: we're done */
+			ret = LOAD_SUCCESS;
+		} else {
+			/* the blob has changed for this vnode: fail ! */
+			ret = LOAD_BADMACHO;
+		}
+		goto out;
+	}
+
+	kr = kmem_alloc(kernel_map, &addr, round_page(lcp->datasize));
+	if (kr != KERN_SUCCESS) {
+		ret = LOAD_NOSPACE;
+		goto out;
+	}
+	
+	resid = 0;
+	error = vn_rdwr(UIO_READ,
+			vp,
+			(caddr_t) addr,
+			lcp->datasize,
+			macho_offset + lcp->dataoff,
+			UIO_SYSSPACE32,
+			0,
+			kauth_cred_get(),
+			&resid,
+			current_proc());
+	if (error || resid != 0) {
+		ret = LOAD_IOERROR;
+		goto out;
+	}
+
+	if (ubc_cs_blob_add(vp,
+			    cputype,
+			    macho_offset,
+			    addr,
+			    lcp->datasize)) {
+		ret = LOAD_FAILURE;
+		goto out;
+	} else {
+		/* ubc_cs_blob_add() has consumed "addr" */
+		addr = 0;
+	}
+		
+	blob = ubc_cs_blob_get(vp, cputype, -1);
+
+	ret = LOAD_SUCCESS;
+out:
+	if (result && ret == LOAD_SUCCESS) {
+		result->csflags |= blob->csb_flags;
+	}
+	if (addr != 0) {
+		kmem_free(kernel_map, addr, round_page(lcp->datasize));
+		addr = 0;
+	}
+
+	return ret;
+}
+
 /*
  * This routine exists to support the load_dylinker().
  *
@@ -1352,9 +1408,10 @@ get_macho_vnode(
 )
 {
 	struct vnode		*vp;
-	struct vfs_context context;
+	vfs_context_t		ctx = vfs_context_current();
+	proc_t			p = vfs_context_proc(ctx);
+	kauth_cred_t		kerncred;
 	struct nameidata nid, *ndp;
-	struct proc *p = current_proc();		/* XXXX */
 	boolean_t		is_fat;
 	struct fat_arch		fat_arch;
 	int			error = LOAD_SUCCESS;
@@ -1365,22 +1422,29 @@ get_macho_vnode(
 		char	pad[512];
 	} header;
 	off_t fsize = (off_t)0;
-	struct	ucred *cred = kauth_cred_get();
 	int err2;
 	
-	context.vc_proc = p;
-	context.vc_ucred = cred;
+	/*
+	 * Capture the kernel credential for use in the actual read of the
+	 * file, since the user doing the execution may have execute rights
+	 * but not read rights, but to exec something, we have to either map
+	 * or read it into the new process address space, which requires
+	 * read rights.  This is to deal with lack of common credential
+	 * serialization code which would treat NOCRED as "serialize 'root'".
+	 */
+	kerncred = vfs_context_ucred(vfs_context_kernel());
 
 	ndp = &nid;
 	
 	/* init the namei data to point the file user's program name */
-	NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32, CAST_USER_ADDR_T(path), &context);
+	NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32, CAST_USER_ADDR_T(path), ctx);
 
 	if ((error = namei(ndp)) != 0) {
-		if (error == ENOENT)
+		if (error == ENOENT) {
 			error = LOAD_ENOENT;
-		else
+		} else {
 			error = LOAD_FAILURE;
+		}
 		return(error);
 	}
 	nameidone(ndp);
@@ -1393,7 +1457,7 @@ get_macho_vnode(
 	}
 
 	/* get size */
-	if ((error = vnode_size(vp, &fsize, &context)) != 0) {
+	if ((error = vnode_size(vp, &fsize, ctx)) != 0) {
 		error = LOAD_FAILURE;
 		goto bad1;
 	}
@@ -1405,19 +1469,19 @@ get_macho_vnode(
 	}
 
 	/* check access */
-	if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, &context)) != 0) {
+	if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, ctx)) != 0) {
 		error = LOAD_PROTECT;
 		goto bad1;
 	}
 
 	/* try to open it */
-	if ((error = VNOP_OPEN(vp, FREAD, &context)) != 0) {
+	if ((error = VNOP_OPEN(vp, FREAD, ctx)) != 0) {
 		error = LOAD_PROTECT;
 		goto bad1;
 	}
 
 	if ((error = vn_rdwr(UIO_READ, vp, (caddr_t)&header, sizeof(header), 0,
-	    UIO_SYSSPACE32, IO_NODELOCKED, cred, &resid, p)) != 0) {
+	    UIO_SYSSPACE32, IO_NODELOCKED, kerncred, &resid, p)) != 0) {
 		error = LOAD_IOERROR;
 		goto bad2;
 	}
@@ -1442,7 +1506,7 @@ get_macho_vnode(
 		/* Read the Mach-O header out of it */
 		error = vn_rdwr(UIO_READ, vp, (caddr_t)&header.mach_header,
 				sizeof(header.mach_header), fat_arch.offset,
-				UIO_SYSSPACE32, IO_NODELOCKED, cred, &resid, p);
+				UIO_SYSSPACE32, IO_NODELOCKED, kerncred, &resid, p);
 		if (error) {
 			error = LOAD_IOERROR;
 			goto bad2;
@@ -1456,7 +1520,7 @@ get_macho_vnode(
 		}
 
 		*file_offset = fat_arch.offset;
-		*macho_size = fsize = fat_arch.size;
+		*macho_size = fat_arch.size;
 	} else {
 		/*
 		 * Force get_macho_vnode() to fail if the architecture bits
@@ -1483,7 +1547,7 @@ get_macho_vnode(
 	return (error);
 
 bad2:
-	err2 = VNOP_CLOSE(vp, FREAD, &context);
+	err2 = VNOP_CLOSE(vp, FREAD, ctx);
 	vnode_put(vp);
 	return (error);
 
diff --git a/bsd/kern/mach_loader.h b/bsd/kern/mach_loader.h
index 75713b125..a2cf6dd99 100644
--- a/bsd/kern/mach_loader.h
+++ b/bsd/kern/mach_loader.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (C) 1992, NeXT, Inc.
@@ -53,15 +59,15 @@ typedef struct _load_result {
 			dynlinker	:1,
 			customstack	:1,
 					:0;
+	unsigned int	csflags;
 } load_result_t;
 
 struct image_params;
 load_return_t load_machfile(
 	struct image_params	*imgp,
 	struct mach_header	*header,
-	thread_t		thr_act,
+	thread_t		thread,
 	vm_map_t		map,
-	boolean_t		clean_regions,
 	load_result_t		*result);
 
 #define LOAD_SUCCESS		0
diff --git a/bsd/kern/mach_process.c b/bsd/kern/mach_process.c
index e07e0300a..46d527f05 100644
--- a/bsd/kern/mach_process.c
+++ b/bsd/kern/mach_process.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -82,6 +88,11 @@
 #include <kern/task.h>
 #include <kern/thread.h>
 
+#include <mach/task.h>			/* for task_resume() */
+#include <kern/sched_prim.h>		/* for thread_exception_return() */
+
+/* XXX ken/bsd_kern.c - prototype should be in common header */
+int get_task_userstop(task_t);
 
 /* Macros to clear/set/test flags. */
 #define	SET(t, f)	(t) |= (f)
@@ -97,16 +108,15 @@ extern thread_t get_firstthread(task_t);
  */
 
 int
-ptrace(p, uap, retval)
-	struct proc *p;
-	struct ptrace_args *uap;
-	register_t *retval;
+ptrace(struct proc *p, struct ptrace_args *uap, register_t *retval)
 {
 	struct proc *t = current_proc();	/* target process */
 	task_t		task;
 	thread_t	th_act;
 	struct uthread 	*ut;
 	int tr_sigexc = 0;
+	int error = 0;
+	int stopped = 0;
 
 	AUDIT_ARG(cmd, uap->req);
 	AUDIT_ARG(pid, uap->pid);
@@ -114,21 +124,23 @@ ptrace(p, uap, retval)
 	AUDIT_ARG(value, uap->data);
 
 	if (uap->req == PT_DENY_ATTACH) {
-		if (ISSET(p->p_flag, P_TRACED)) {
+		proc_lock(p);
+		if (ISSET(p->p_lflag, P_LTRACED)) {
+			proc_unlock(p);
 			exit1(p, W_EXITCODE(ENOTSUP, 0), retval);
 			/* drop funnel before we return */
-			thread_funnel_set(kernel_flock, FALSE);
 			thread_exception_return();
 			/* NOTREACHED */
 		}
-		SET(p->p_flag, P_NOATTACH);
+		SET(p->p_lflag, P_LNOATTACH);
+		proc_unlock(p);
 
 		return(0);
 	}
 
 	if (uap->req == PT_FORCEQUOTA) {
 		if (is_suser()) {
-			SET(t->p_flag, P_FORCEQUOTA);
+			OSBitOrAtomic(P_FORCEQUOTA, (UInt32 *)&t->p_flag);
 			return (0);
 		} else
 			return (EPERM);
@@ -138,33 +150,40 @@ ptrace(p, uap, retval)
 	 *	Intercept and deal with "please trace me" request.
 	 */	 
 	if (uap->req == PT_TRACE_ME) {
-		SET(p->p_flag, P_TRACED);
+		proc_lock(p);
+		SET(p->p_lflag, P_LTRACED);
 		/* Non-attached case, our tracer is our parent. */
-		t->p_oppid = t->p_pptr->p_pid;
+		p->p_oppid = p->p_ppid;
+		proc_unlock(p);
 		return(0);
 	}
 	if (uap->req == PT_SIGEXC) {
-		if (ISSET(p->p_flag, P_TRACED)) {
-			SET(p->p_flag, P_SIGEXC);
+		proc_lock(p);
+		if (ISSET(p->p_lflag, P_LTRACED)) {
+			SET(p->p_lflag, P_LSIGEXC);
+			proc_unlock(p);
 			return(0);
-		} else
+		} else {
+			proc_unlock(p);
 			return(EINVAL);
+		}
+	}
+
+	/* 
+	 * We do not want ptrace to do anything with kernel or launchd 
+	 */
+	if (uap->pid < 2) {
+		return(EPERM);
 	}
 
 	/*
 	 *	Locate victim, and make sure it is traceable.
 	 */
-	if ((t = pfind(uap->pid)) == NULL)
+	if ((t = proc_find(uap->pid)) == NULL)
 			return (ESRCH);
 
 	AUDIT_ARG(process, t);
 
-	/* We do not want ptrace to do anything with kernel, init 
-	 * and mach_init
-	 */
-	if (uap->pid <=2 )
-		return (EPERM);
-
 	task = t->task;
 	if (uap->req == PT_ATTACHEXC) {
 		uap->req = PT_ATTACH;
@@ -176,29 +195,41 @@ ptrace(p, uap, retval)
 		if ( kauth_authorize_process(proc_ucred(p), KAUTH_PROCESS_CANTRACE, 
 									 t, (uintptr_t)&err, 0, 0) == 0 ) {
 			/* it's OK to attach */
-			SET(t->p_flag, P_TRACED);
+			proc_lock(t);
+			SET(t->p_lflag, P_LTRACED);
 			if (tr_sigexc) 
-				SET(t->p_flag, P_SIGEXC);
+				SET(t->p_lflag, P_LSIGEXC);
 	
-			t->p_oppid = t->p_pptr->p_pid;
+			t->p_oppid = t->p_ppid;
+			proc_unlock(t);
 			if (t->p_pptr != p)
-				proc_reparent(t, p);
+				proc_reparentlocked(t, p, 1, 0);
 	
-			if (get_task_userstop(task) == 0 ) {
-				t->p_xstat = 0;
-				psignal(t, SIGSTOP);
-			} else {
-				t->p_xstat = SIGSTOP; 
-				task_resume(task);       
+			proc_lock(t);
+			if (get_task_userstop(task) > 0 ) {
+				stopped = 1;
 			}
-			return(0);
+			t->p_xstat = 0;
+			proc_unlock(t);
+			psignal(t, SIGSTOP);
+			/*
+			 * If the process was stopped, wake up and run through
+			 * issignal() again to properly connect to the tracing
+			 * process.
+			 */
+			if (stopped)
+				task_resume(task);       
+			error = 0;
+			goto out;
 		}
 		else {
 			/* not allowed to attach, proper error code returned by kauth_authorize_process */
-			if (ISSET(t->p_flag, P_NOATTACH)) {
+			if (ISSET(t->p_lflag, P_LNOATTACH)) {
 				psignal(p, SIGSEGV);
 			}
-			return (err);
+			
+			error = err;
+			goto out;
 		}
 	}
 
@@ -206,38 +237,55 @@ ptrace(p, uap, retval)
 	 * You can't do what you want to the process if:
 	 *	(1) It's not being traced at all,
 	 */
-	if (!ISSET(t->p_flag, P_TRACED))
-		return (EPERM);
+	proc_lock(t);
+	if (!ISSET(t->p_lflag, P_LTRACED)) {
+		proc_unlock(t);
+		error = EPERM;
+		goto out;
+	}
 
 	/*
 	 *	(2) it's not being traced by _you_, or
 	 */
-	if (t->p_pptr != p)
-		return (EBUSY);
+	if (t->p_pptr != p) {
+		proc_unlock(t);
+		error = EBUSY;
+		goto out;
+	}
 
 	/*
 	 *	(3) it's not currently stopped.
 	 */
-	if (t->p_stat != SSTOP)
-		return (EBUSY);
+	if (t->p_stat != SSTOP) {
+		proc_unlock(t);
+		error = EBUSY;
+		goto out;
+	}
 
 	/*
 	 *	Mach version of ptrace executes request directly here,
 	 *	thus simplifying the interaction of ptrace and signals.
 	 */
+	/* proc lock is held here */
 	switch (uap->req) {
 
 	case PT_DETACH:
-		if (t->p_oppid != t->p_pptr->p_pid) {
+		if (t->p_oppid != t->p_ppid) {
 			struct proc *pp;
 
-			pp = pfind(t->p_oppid);
-			proc_reparent(t, pp ? pp : initproc);
+			proc_unlock(t);
+			pp = proc_find(t->p_oppid);
+			proc_reparentlocked(t, pp ? pp : initproc, 1, 0);
+			if (pp != PROC_NULL)
+				proc_rele(pp);
+			proc_lock(t);
+			
 		}
 
 		t->p_oppid = 0;
-		CLR(t->p_flag, P_TRACED);
-		CLR(t->p_flag, P_SIGEXC);
+		CLR(t->p_lflag, P_LTRACED);
+		CLR(t->p_lflag, P_LSIGEXC);
+		proc_unlock(t);
 		goto resume;
 		
 	case PT_KILL:
@@ -245,74 +293,100 @@ ptrace(p, uap, retval)
 		 *	Tell child process to kill itself after it
 		 *	is resumed by adding NSIG to p_cursig. [see issig]
 		 */
-		psignal_lock(t, SIGKILL, 0);
+		proc_unlock(t);
+		psignal(t, SIGKILL);
 		goto resume;
 
 	case PT_STEP:			/* single step the child */
 	case PT_CONTINUE:		/* continue the child */
+		proc_unlock(t);
 		th_act = (thread_t)get_firstthread(task);
-		if (th_act == THREAD_NULL)
-			goto errorLabel;
+		if (th_act == THREAD_NULL) {
+			error = EINVAL;
+			goto out;
+		}
 
 		if (uap->addr != (user_addr_t)1) {
 #if defined(ppc)
 #define ALIGNED(addr,size)	(((unsigned)(addr)&((size)-1))==0)
-		        if (!ALIGNED((int)uap->addr, sizeof(int)))
-			        return (ERESTART);
+			if (!ALIGNED((int)uap->addr, sizeof(int)))
+				return (ERESTART);
 #undef 	ALIGNED
 #endif
-		        thread_setentrypoint(th_act, uap->addr);
+			thread_setentrypoint(th_act, uap->addr);
 		}
 
-		if ((unsigned)uap->data >= NSIG)
-			goto errorLabel;
+		if ((unsigned)uap->data >= NSIG) {
+			error = EINVAL;
+			goto out;
+		}
 
 		if (uap->data != 0) {
-			psignal_lock(t, uap->data, 0);
+			psignal(t, uap->data);
                 }
 
 		if (uap->req == PT_STEP) {
 		        /*
 			 * set trace bit
 			 */
-		        thread_setsinglestep(th_act, 1);
+			if (thread_setsinglestep(th_act, 1) != KERN_SUCCESS) {
+				error = ENOTSUP;
+				goto out;
+			}
 		} else {
 		        /*
 			 * clear trace bit if on
 			 */
-		        thread_setsinglestep(th_act, 0);
-		}
+			if (thread_setsinglestep(th_act, 0) != KERN_SUCCESS) {
+				error = ENOTSUP;
+				goto out;
+			}
+		}	
 	resume:
+		proc_lock(t);
 		t->p_xstat = uap->data;
 		t->p_stat = SRUN;
 		if (t->sigwait) {
 			wakeup((caddr_t)&(t->sigwait));
-			if ((t->p_flag & P_SIGEXC) == 0)
-				task_release(task);
-		}
+			proc_unlock(t);
+			if ((t->p_lflag & P_LSIGEXC) == 0) {
+				task_resume(task);
+			}
+		} else
+			proc_unlock(t);
+			
 		break;
 		
 	case PT_THUPDATE:  {
-		if ((unsigned)uap->data >= NSIG)
-			goto errorLabel;
+		proc_unlock(t);
+		if ((unsigned)uap->data >= NSIG) {
+			error = EINVAL;
+			goto out;
+		}
 		th_act = port_name_to_thread(CAST_DOWN(mach_port_name_t, uap->addr));
 		if (th_act == THREAD_NULL)
 			return (ESRCH);
 		ut = (uthread_t)get_bsdthread_info(th_act);
 		if (uap->data)
 			ut->uu_siglist |= sigmask(uap->data);
+		proc_lock(t);
 		t->p_xstat = uap->data;
 		t->p_stat = SRUN;
+		proc_unlock(t);
 		thread_deallocate(th_act);
-		return(0);
+		error = 0;
 		}
 		break;
-errorLabel:
 	default:
-		return(EINVAL);
+		proc_unlock(t);
+		error = EINVAL;
+		goto out;
 	}
 
-	return(0);
+	error = 0;
+out:
+	proc_rele(t);
+	return(error);
 }
 
 
@@ -336,7 +410,7 @@ cantrace(proc_t cur_procp, kauth_cred_t creds, proc_t traced_procp, int *errp)
 	/*
 	 *	(2) it's already being traced, or
 	 */
-	if (ISSET(traced_procp->p_flag, P_TRACED)) {
+	if (ISSET(traced_procp->p_lflag, P_LTRACED)) {
 		*errp = EBUSY;
 		return (0);
 	}
@@ -352,12 +426,12 @@ cantrace(proc_t cur_procp, kauth_cred_t creds, proc_t traced_procp, int *errp)
 		return (0);
 	}
 
-	if ((cur_procp->p_flag & P_TRACED) && isinferior(cur_procp, traced_procp)) {
+	if ((cur_procp->p_lflag & P_LTRACED) && isinferior(cur_procp, traced_procp)) {
 		*errp = EPERM;
 		return (0);
 	}
 
-	if (ISSET(traced_procp->p_flag, P_NOATTACH)) {
+	if (ISSET(traced_procp->p_lflag, P_LNOATTACH)) {
 		*errp = EBUSY;
 		return (0);
 	}
diff --git a/bsd/kern/makesyscalls.sh b/bsd/kern/makesyscalls.sh
index e2e5a7f95..a6f88584d 100755
--- a/bsd/kern/makesyscalls.sh
+++ b/bsd/kern/makesyscalls.sh
@@ -2,25 +2,26 @@
 #	@(#)makesyscalls.sh	8.1 (Berkeley) 6/10/93
 # $FreeBSD: src/sys/kern/makesyscalls.sh,v 1.60 2003/04/01 01:12:24 jeff Exp $
 #
-# Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+# Copyright (c) 2004-2007 Apple Inc. All rights reserved.
 #
-# @APPLE_LICENSE_HEADER_START@
+# @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 # 
-# The contents of this file constitute Original Code as defined in and
-# are subject to the Apple Public Source License Version 1.1 (the
-# "License").  You may not use this file except in compliance with the
-# License.  Please obtain a copy of the License at
-# http://www.apple.com/publicsource and read it before using this file.
+# This file contains Original Code and/or Modifications of Original Code
+# as defined in and that are subject to the Apple Public Source License
+# Version 2.0 (the 'License'). You may not use this file except in
+# compliance with the License. Please obtain a copy of the License at
+# http://www.opensource.apple.com/apsl/ and read it before using this
+# file.
 # 
-# This Original Code and all software distributed under the License are
-# distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+# The Original Code and all software distributed under the License are
+# distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 # EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 # INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
-# License for the specific language governing rights and limitations
-# under the License.
+# FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+# Please see the License for the specific language governing rights and
+# limitations under the License.
 # 
-# @APPLE_LICENSE_HEADER_END@
+# @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 #
 
 set -e
@@ -69,7 +70,7 @@ s/\$//g
 	b join
 	}
 2,${
-	/^#/!s/\([{}()*,]\)/ \1 /g
+	/^#/!s/\([{}()*,;]\)/ \1 /g
 }
 ' < $1 | awk "
 	BEGIN {
@@ -93,25 +94,31 @@ s/\$//g
 		"'
 
 		printf "/*\n" > syslegal
-		printf " * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.\n" > syslegal
+		printf " * Copyright (c) 2004-2007 Apple Inc. All rights reserved.\n" > syslegal
 		printf " * \n" > syslegal
-		printf " * @APPLE_LICENSE_HEADER_START@ \n" > syslegal
+		printf " * @APPLE_OSREFERENCE_LICENSE_HEADER_START@\n" > syslegal
 		printf " * \n" > syslegal
-		printf " * The contents of this file constitute Original Code as defined in and \n" > syslegal
-		printf " * are subject to the Apple Public Source License Version 1.1 (the \n" > syslegal
-		printf " * \"License\").  You may not use this file except in compliance with the \n" > syslegal
-		printf " * License.  Please obtain a copy of the License at \n" > syslegal
-		printf " * http://www.apple.com/publicsource and read it before using this file. \n" > syslegal
+		printf " * This file contains Original Code and/or Modifications of Original Code\n" > syslegal
+		printf " * as defined in and that are subject to the Apple Public Source License\n" > syslegal
+		printf " * Version 2.0 (the \047License\047). You may not use this file except in\n" > syslegal
+		printf " * compliance with the License. The rights granted to you under the License\n" > syslegal
+		printf " * may not be used to create, or enable the creation or redistribution of,\n" > syslegal
+		printf " * unlawful or unlicensed copies of an Apple operating system, or to\n" > syslegal
+		printf " * circumvent, violate, or enable the circumvention or violation of, any\n" > syslegal
+		printf " * terms of an Apple operating system software license agreement.\n" > syslegal
 		printf " * \n" > syslegal
-		printf " * This Original Code and all software distributed under the License are \n" > syslegal
-		printf " * distributed on an \"AS IS\" basis, WITHOUT WARRANTY OF ANY KIND, EITHER \n" > syslegal
-		printf " * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, \n" > syslegal
-		printf " * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, \n" > syslegal
-		printf " * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the \n" > syslegal
-		printf " * License for the specific language governing rights and limitations \n" > syslegal
-		printf " * under the License. \n" > syslegal
+		printf " * Please obtain a copy of the License at\n" > syslegal
+		printf " * http://www.opensource.apple.com/apsl/ and read it before using this file.\n" > syslegal
 		printf " * \n" > syslegal
-		printf " * @APPLE_LICENSE_HEADER_END@ \n" > syslegal
+		printf " * The Original Code and all software distributed under the License are\n" > syslegal
+		printf " * distributed on an \047AS IS\047 basis, WITHOUT WARRANTY OF ANY KIND, EITHER\n" > syslegal
+		printf " * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,\n" > syslegal
+		printf " * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,\n" > syslegal
+		printf " * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.\n" > syslegal
+		printf " * Please see the License for the specific language governing rights and\n" > syslegal
+		printf " * limitations under the License.\n" > syslegal
+		printf " * \n" > syslegal
+		printf " * @APPLE_OSREFERENCE_LICENSE_HEADER_END@\n" > syslegal
 		printf " * \n" > syslegal
 		printf " * \n" > syslegal
 		printf " * System call switch table.\n *\n" > syslegal
@@ -135,11 +142,16 @@ s/\$//g
 		printf "#include <sys/sem_internal.h>\n" > sysarg
 		printf "#include <sys/semaphore.h>\n" > sysarg
 		printf "#include <sys/wait.h>\n" > sysarg
-		printf "#include <mach/shared_memory_server.h>\n" > sysarg
+		printf "#include <mach/shared_region.h>\n" > sysarg
 		printf "\n#ifdef KERNEL\n" > sysarg
 		printf "#ifdef __APPLE_API_PRIVATE\n" > sysarg
+		printf "#ifndef __arm__\n" > sysarg
 		printf "#define\tPAD_(t)\t(sizeof(uint64_t) <= sizeof(t) \\\n " > sysarg
 		printf "\t\t? 0 : sizeof(uint64_t) - sizeof(t))\n" > sysarg
+		printf "#else\n" > sysarg
+		printf "#define\tPAD_(t)\t(sizeof(register_t) <= sizeof(t) \\\n" > sysarg
+		printf " 		? 0 : sizeof(register_t) - sizeof(t))\n" > sysarg
+		printf "#endif\n" > sysarg
 		printf "#if BYTE_ORDER == LITTLE_ENDIAN\n"> sysarg
 		printf "#define\tPADL_(t)\t0\n" > sysarg
 		printf "#define\tPADR_(t)\tPAD_(t)\n" > sysarg
@@ -150,6 +162,7 @@ s/\$//g
 		printf "\n__BEGIN_DECLS\n" > sysarg
 		printf "#ifndef __MUNGE_ONCE\n" > sysarg
 		printf "#define __MUNGE_ONCE\n" > sysarg
+		printf "#ifndef __arm__\n" > sysarg
 		printf "void munge_w(const void *, void *);  \n" > sysarg
 		printf "void munge_ww(const void *, void *);  \n" > sysarg
 		printf "void munge_www(const void *, void *);  \n" > sysarg
@@ -161,12 +174,33 @@ s/\$//g
 		printf "void munge_wl(const void *, void *);  \n" > sysarg
 		printf "void munge_wlw(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwl(const void *, void *);  \n" > sysarg
-		printf "void munge_wwwlww(const void *, void *); \n" > sysarg
+		printf "void munge_wwwlww(const void *, void *);  \n" > sysarg
+		printf "void munge_wwlwww(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwwl(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwwwl(const void *, void *);  \n" > sysarg
 		printf "void munge_wsw(const void *, void *);  \n" > sysarg
 		printf "void munge_wws(const void *, void *);  \n" > sysarg
 		printf "void munge_wwwsw(const void *, void *);  \n" > sysarg
+		printf "#else \n" > sysarg
+		printf "#define munge_w  NULL \n" > sysarg
+		printf "#define munge_ww  NULL \n" > sysarg
+		printf "#define munge_www  NULL \n" > sysarg
+		printf "#define munge_wwww  NULL \n" > sysarg
+		printf "#define munge_wwwww  NULL \n" > sysarg
+		printf "#define munge_wwwwww  NULL \n" > sysarg
+		printf "#define munge_wwwwwww  NULL \n" > sysarg
+		printf "#define munge_wwwwwwww  NULL \n" > sysarg
+		printf "#define munge_wl  NULL \n" > sysarg
+		printf "#define munge_wlw  NULL \n" > sysarg
+		printf "#define munge_wwwl  NULL \n" > sysarg
+		printf "#define munge_wwwlww  NULL\n" > sysarg
+		printf "#define munge_wwlwww  NULL \n" > sysarg
+		printf "#define munge_wwwwl  NULL \n" > sysarg
+		printf "#define munge_wwwwwl  NULL \n" > sysarg
+		printf "#define munge_wsw  NULL \n" > sysarg
+		printf "#define munge_wws  NULL \n" > sysarg
+		printf "#define munge_wwwsw  NULL \n" > sysarg
+		printf "#endif // ! __arm__\n" > sysarg
 		printf "#ifdef __ppc__\n" > sysarg
 		printf "void munge_d(const void *, void *);  \n" > sysarg
 		printf "void munge_dd(const void *, void *);  \n" > sysarg
@@ -204,31 +238,31 @@ s/\$//g
 		print > sysent
 		print > sysarg
 		print > syscallnamestempfile
-		print > syshdrtempfile
 		print > sysprotoend
-		savesyscall = syscall
+		savesyscall = syscall_num
+		skip_for_header = 0
 		next
 	}
 	$1 ~ /^#[ 	]*else/ {
 		print > sysent
 		print > sysarg
 		print > syscallnamestempfile
-		print > syshdrtempfile
 		print > sysprotoend
-		syscall = savesyscall
+		syscall_num = savesyscall
+		skip_for_header = 1
 		next
 	}
 	$1 ~ /^#/ {
 		print > sysent
 		print > sysarg
 		print > syscallnamestempfile
-		print > syshdrtempfile
 		print > sysprotoend
+		skip_for_header = 0
 		next
 	}
-	syscall != $1 {
+	syscall_num != $1 {
 		printf "%s: line %d: syscall number out of sync at %d\n",
-		    infile, NR, syscall
+		    infile, NR, syscall_num
 		printf "line is:\n"
 		print
 		exit 1
@@ -248,7 +282,7 @@ s/\$//g
 	
 	function parseline() {
 		funcname = ""
-		current_field = 5
+		current_field = 3
 		args_start = 0
 		args_end = 0
 		comments_start = 0
@@ -299,6 +333,11 @@ s/\$//g
 		if ($args_end != ";")
 			parserr($args_end, ";")
 		args_end--
+
+		# skip any NO_SYSCALL_STUB qualifier
+		if ($args_end == "NO_SYSCALL_STUB")
+			args_end--
+
 		if ($args_end != ")")
 			parserr($args_end, ")")
 		args_end--
@@ -367,70 +406,46 @@ s/\$//g
 		add_sysproto_entry = 1
 		add_64bit_unsafe = 0
 		add_64bit_fakesafe = 0
-		add_cancel_enable = "0"
+		add_resv = "0"
+		my_flags = "0"
 
-		if ($2 == "NONE") {
-			add_cancel_enable = "_SYSCALL_CANCEL_NONE"
-		}
-		else if ($2 == "PRE") {
-			add_cancel_enable = "_SYSCALL_CANCEL_PRE"
-		}
-		else if ($2 == "POST") {
-			add_cancel_enable = "_SYSCALL_CANCEL_POST"
-		} 
-		else {
-			printf "%s: line %d: unrecognized keyword %s\n", infile, NR, $2
-			exit 1
 
-		}
-
-		if ($3 == "KERN") {
-			my_funnel = "KERNEL_FUNNEL"
-		}
-		else if ($3 == "NONE") {
-			my_funnel = "NO_FUNNEL"
-		}
-		else {
-			printf "%s: line %d: unrecognized keyword %s\n", infile, NR, $3
-			exit 1
-		}
-		
-		if ($4 != "ALL" && $4 != "UALL") {
+		if ($2 != "ALL" && $2 != "UALL") {
 			files_keyword_OK = 0
 			add_sysent_entry = 0
 			add_sysnames_entry = 0
 			add_sysheader_entry = 0
 			add_sysproto_entry = 0
 			
-			if (match($4, "[T]") != 0) {
+			if (match($2, "[T]") != 0) {
 				add_sysent_entry = 1
 				files_keyword_OK = 1
 			}
-			if (match($4, "[N]") != 0) {
+			if (match($2, "[N]") != 0) {
 				add_sysnames_entry = 1
 				files_keyword_OK = 1
 			}
-			if (match($4, "[H]") != 0) {
+			if (match($2, "[H]") != 0) {
 				add_sysheader_entry = 1
 				files_keyword_OK = 1
 			}
-			if (match($4, "[P]") != 0) {
+			if (match($2, "[P]") != 0) {
 				add_sysproto_entry = 1
 				files_keyword_OK = 1
 			}
-			if (match($4, "[U]") != 0) {
+			if (match($2, "[U]") != 0) {
 				add_64bit_unsafe = 1
 			}
-			if (match($4, "[F]") != 0) {
+			if (match($2, "[F]") != 0) {
 				add_64bit_fakesafe = 1
 			}
 			
 			if (files_keyword_OK == 0) {
-				printf "%s: line %d: unrecognized keyword %s\n", infile, NR, $4
+				printf "%s: line %d: unrecognized keyword %s\n", infile, NR, $2
 				exit 1
 			}
 		}
-		else if ($4 == "UALL") {
+		else if ($2 == "UALL") {
 			add_64bit_unsafe = 1;
 		}
 		
@@ -441,7 +456,9 @@ s/\$//g
 		# name of the appropriate argument mungers
 		munge32 = "NULL"
 		munge64 = "NULL"
-		if (funcname != "nosys" || (syscall == 0 && funcname == "nosys")) {
+		size32 = 0
+
+		if ((funcname != "nosys" && funcname != "enosys") || (syscall_num == 0 && funcname == "nosys")) {
 			if (argc != 0) {
 				if (add_sysproto_entry == 1) {
 					printf("struct %s {\n", argalias) > sysarg
@@ -459,33 +476,39 @@ s/\$//g
 							ext_argtype[i] = "user_long_t";
 						munge32 = munge32 "s"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "u_long") {
 						if (add_64bit_unsafe == 0)
 							ext_argtype[i] = "user_ulong_t";
 						munge32 = munge32 "w"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "size_t") {
 						if (add_64bit_unsafe == 0)
 							ext_argtype[i] = "user_size_t";
 						munge32 = munge32 "w"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "ssize_t") {
 						if (add_64bit_unsafe == 0)
 							ext_argtype[i] = "user_ssize_t";
 						munge32 = munge32 "s"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "user_ssize_t" || argtype[i] == "user_long_t") {
 						munge32 = munge32 "s"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "user_addr_t" || argtype[i] == "user_size_t" ||
 						argtype[i] == "user_ulong_t") {
 						munge32 = munge32 "w"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "caddr_t" || argtype[i] == "semun_t" ||
   						match(argtype[i], "[\*]") != 0) {
@@ -493,6 +516,7 @@ s/\$//g
 							ext_argtype[i] = "user_addr_t";
 						munge32 = munge32 "w"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "int" || argtype[i] == "u_int" ||
 							 argtype[i] == "uid_t" || argtype[i] == "pid_t" ||
@@ -502,10 +526,12 @@ s/\$//g
 							 argtype[i] == "mode_t" || argtype[i] == "key_t" || argtype[i] == "time_t") {
 						munge32 = munge32 "w"
 						munge64 = munge64 "d"
+						size32 += 4
 					}
 					else if (argtype[i] == "off_t" || argtype[i] == "int64_t" || argtype[i] == "uint64_t") {
 						munge32 = munge32 "l"
 						munge64 = munge64 "d"
+						size32 += 8
 					}
 					else {
 						printf "%s: line %d: invalid type \"%s\" \n", 
@@ -538,7 +564,9 @@ s/\$//g
 			munge32 = "NULL"
 			munge64 = "NULL"
 			munge_ret = "_SYSCALL_RET_NONE"
-			tempname = "nosys"
+			if (tempname != "enosys") {
+				tempname = "nosys"
+			}
 		}
 		else {
 			# figure out which return value type to munge
@@ -573,56 +601,56 @@ s/\$//g
 		}
 
 		if (add_64bit_unsafe == 1  && add_64bit_fakesafe == 0)
-			my_funnel = my_funnel "|UNSAFE_64BIT";
+			my_flags = "UNSAFE_64BIT";
 
-		printf("\t{%s, %s, %s, \(sy_call_t *\)%s, %s, %s, %s},", 
-				argssize, add_cancel_enable, my_funnel, tempname, munge32, munge64, munge_ret) > sysent
-		linesize = length(argssize) + length(add_cancel_enable) + length(my_funnel) + length(tempname) + \
+		printf("\t{%s, %s, %s, \(sy_call_t *\)%s, %s, %s, %s, %s},", 
+				argssize, add_resv, my_flags, tempname, munge32, munge64, munge_ret, size32) > sysent
+		linesize = length(argssize) + length(add_resv) + length(my_flags) + length(tempname) + \
 				length(munge32) + length(munge64) + length(munge_ret) + 28
 		align_comment(linesize, 88, sysent)
-		printf("/* %d = %s%s*/\n", syscall, funcname, additional_comments) > sysent
+		printf("/* %d = %s%s*/\n", syscall_num, funcname, additional_comments) > sysent
 		
 		# output to syscalls.c
 		if (add_sysnames_entry == 1) {
 			tempname = funcname
-			if (funcname == "nosys") {
-				if (syscall == 0)
+			if (funcname == "nosys" || funcname == "enosys") {
+				if (syscall_num == 0)
 					tempname = "syscall"
 				else
-					tempname = "#" syscall
+					tempname = "#" syscall_num
 			}
 			printf("\t\"%s\", ", tempname) > syscallnamestempfile
 			linesize = length(tempname) + 8
 			align_comment(linesize, 25, syscallnamestempfile)
 			if (substr(tempname,1,1) == "#") {
-				printf("/* %d =%s*/\n", syscall, additional_comments) > syscallnamestempfile
+				printf("/* %d =%s*/\n", syscall_num, additional_comments) > syscallnamestempfile
 			}
 			else {
-				printf("/* %d = %s%s*/\n", syscall, tempname, additional_comments) > syscallnamestempfile
+				printf("/* %d = %s%s*/\n", syscall_num, tempname, additional_comments) > syscallnamestempfile
 			}
 		}
 
 		# output to syscalls.h
 		if (add_sysheader_entry == 1) {
 			tempname = funcname
-			if (syscall == 0) {
+			if (syscall_num == 0) {
 				tempname = "syscall"
 			}
-			if (tempname != "nosys") {
+			if (tempname != "nosys" && tempname != "enosys") {
 				printf("#define\t%s%s", syscallprefix, tempname) > syshdrtempfile
 				linesize = length(syscallprefix) + length(tempname) + 12
 				align_comment(linesize, 30, syshdrtempfile)
-				printf("%d\n", syscall) > syshdrtempfile
+				printf("%d\n", syscall_num) > syshdrtempfile
 				# special case for gettimeofday on ppc - cctools project uses old name
 				if (tempname == "ppc_gettimeofday") {
 					printf("#define\t%s%s", syscallprefix, "gettimeofday") > syshdrtempfile
 					linesize = length(syscallprefix) + length(tempname) + 12
 					align_comment(linesize, 30, syshdrtempfile)
-					printf("%d\n", syscall) > syshdrtempfile
+					printf("%d\n", syscall_num) > syshdrtempfile
 				}
 			}
-			else {
-				printf("\t\t\t/* %d %s*/\n", syscall, additional_comments) > syshdrtempfile
+			else if (skip_for_header == 0) {
+				printf("\t\t\t/* %d %s*/\n", syscall_num, additional_comments) > syshdrtempfile
 			}
 		}
 		
@@ -632,18 +660,18 @@ s/\$//g
 				printf("void %s(struct proc *, struct %s *, int *);\n", 
 						funcname, argalias) > sysprotoend
 			}
-			else if (funcname != "nosys" || (syscall == 0 && funcname == "nosys")) {
+			else if ((funcname != "nosys" && funcname != "enosys") || (syscall_num == 0 && funcname == "nosys")) {
 				printf("int %s(struct proc *, struct %s *, %s *);\n", 
 						funcname, argalias, returntype) > sysprotoend
 			}
 		}
 		
-		syscall++
+		syscall_num++
 		next
 	}
 
 	END {
-		printf "#define AC(name) (sizeof(struct name) / sizeof(uint64_t))\n" > sysinc
+		printf "#define AC(name) (sizeof(struct name) / sizeof(syscall_arg_t))\n" > sysinc
 		printf "\n" > sysinc
 
 		printf("\n__END_DECLS\n") > sysprotoend
@@ -656,15 +684,21 @@ s/\$//g
 
 		printf("};\n") > sysent
 		printf("int	nsysent = sizeof(sysent) / sizeof(sysent[0]);\n") > sysent
+		printf("/* Verify that NUM_SYSENT reflects the latest syscall count */\n") > sysent
+		printf("int	nsysent_size_check[((sizeof(sysent) / sizeof(sysent[0])) == NUM_SYSENT) ? 1 : -1] __unused;\n") > sysent
 
 		printf("};\n") > syscallnamestempfile
-		printf("#define\t%sMAXSYSCALL\t%d\n", syscallprefix, syscall) \
+		printf("#define\t%sMAXSYSCALL\t%d\n", syscallprefix, syscall_num) \
 		    > syshdrtempfile
 		printf("\n#endif /* __APPLE_API_PRIVATE */\n") > syshdrtempfile
 		printf("#endif /* !%s */\n", syscall_h) > syshdrtempfile
 	} '
 
-cat $syslegal $sysinc $sysent > $syscalltablefile
+# define value in syscall table file to permit redifintion because of the way
+# __private_extern__ (doesn't) work.
+cat $syslegal > $syscalltablefile
+printf "#define __INIT_SYSENT_C__ 1\n" >> $syscalltablefile
+cat $sysinc $sysent >> $syscalltablefile
 cat $syslegal $sysarg $sysprotoend > $sysprotofile
 cat $syslegal $syscallnamestempfile > $syscallnamesfile
 cat $syslegal $syshdrtempfile > $syshdrfile
diff --git a/bsd/kern/mcache.c b/bsd/kern/mcache.c
new file mode 100644
index 000000000..9d5fb63d6
--- /dev/null
+++ b/bsd/kern/mcache.c
@@ -0,0 +1,1560 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * Memory allocator with per-CPU caching, derived from the kmem magazine
+ * concept and implementation as described in the following paper:
+ * http://www.usenix.org/events/usenix01/full_papers/bonwick/bonwick.pdf
+ * That implementation is Copyright 2006 Sun Microsystems, Inc.  All rights
+ * reserved.  Use is subject to license terms.
+ *
+ * There are several major differences between this and the original kmem
+ * magazine: this derivative implementation allows for multiple objects to
+ * be allocated and freed from/to the object cache in one call; in addition,
+ * it provides for better flexibility where the user is allowed to define
+ * its own slab allocator (instead of the default zone allocator).  Finally,
+ * no object construction/destruction takes place at the moment, although
+ * this could be added in future to improve efficiency.
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/queue.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+
+#include <kern/debug.h>
+#include <kern/zalloc.h>
+#include <kern/cpu_number.h>
+#include <kern/locks.h>
+
+#include <libkern/libkern.h>
+#include <libkern/OSAtomic.h>
+#include <libkern/OSDebug.h>
+
+#include <mach/vm_param.h>
+#include <machine/limits.h>
+#include <machine/machine_routines.h>
+
+#include <string.h>
+
+#include <sys/mcache.h>
+
+#define	MCACHE_SIZE(n) \
+	((size_t)(&((mcache_t *)0)->mc_cpu[n]))
+
+/* Allocate extra in case we need to manually align the pointer */
+#define	MCACHE_ALLOC_SIZE \
+	(sizeof (void *) + MCACHE_SIZE(ncpu) + CPU_CACHE_SIZE)
+
+#define	MCACHE_CPU(c) \
+	(mcache_cpu_t *)((char *)(c) + MCACHE_SIZE(cpu_number()))
+
+/*
+ * MCACHE_LIST_LOCK() and MCACHE_LIST_UNLOCK() are macros used
+ * to serialize accesses to the global list of caches in the system.
+ * They also record the thread currently running in the critical
+ * section, so that we can avoid recursive requests to reap the
+ * caches when memory runs low.
+ */
+#define	MCACHE_LIST_LOCK() {				\
+	lck_mtx_lock(mcache_llock);			\
+	mcache_llock_owner = current_thread();		\
+}
+
+#define	MCACHE_LIST_UNLOCK() {				\
+	mcache_llock_owner = NULL;			\
+	lck_mtx_unlock(mcache_llock);			\
+}
+
+#define	MCACHE_LOCK(l)		lck_mtx_lock(l)
+#define	MCACHE_UNLOCK(l)	lck_mtx_unlock(l)
+#define	MCACHE_LOCK_TRY(l)	lck_mtx_try_lock(l)
+
+/* This should be in a header file */
+#define	atomic_add_32(a, n)	((void) OSAddAtomic(n, (volatile SInt32 *)a))
+
+static int ncpu;
+static lck_mtx_t *mcache_llock;
+static struct thread *mcache_llock_owner;
+static lck_attr_t *mcache_llock_attr;
+static lck_grp_t *mcache_llock_grp;
+static lck_grp_attr_t *mcache_llock_grp_attr;
+static struct zone *mcache_zone;
+static unsigned int mcache_reap_interval;
+static UInt32 mcache_reaping;
+static int mcache_ready;
+static int mcache_updating;
+
+static int mcache_bkt_contention = 3;
+#if DEBUG
+static unsigned int mcache_flags = MCF_DEBUG;
+#else
+static unsigned int mcache_flags = 0;
+#endif
+
+#define	DUMP_MCA_BUF_SIZE	512
+static char *mca_dump_buf;
+
+static mcache_bkttype_t mcache_bkttype[] = {
+	{ 1,	4096,	32768,	NULL },
+	{ 3,	2048,	16384,	NULL },
+	{ 7,	1024,	12288,	NULL },
+	{ 15,	256,	8192,	NULL },
+	{ 31,	64,	4096,	NULL },
+	{ 47,	0,	2048,	NULL },
+	{ 63,	0,	1024,	NULL },
+	{ 95,	0,	512,	NULL },
+	{ 143,	0,	256,	NULL },
+	{ 165,	0,	0,	NULL },
+};
+
+static mcache_t *mcache_create_common(const char *, size_t, size_t,
+    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t,
+    void *, u_int32_t, int, int);
+static unsigned int mcache_slab_alloc(void *, mcache_obj_t ***,
+    unsigned int, int);
+static void mcache_slab_free(void *, mcache_obj_t *, boolean_t);
+static void mcache_slab_audit(void *, mcache_obj_t *, boolean_t);
+static void mcache_cpu_refill(mcache_cpu_t *, mcache_bkt_t *, int);
+static mcache_bkt_t *mcache_bkt_alloc(mcache_t *, mcache_bktlist_t *,
+    mcache_bkttype_t **);
+static void mcache_bkt_free(mcache_t *, mcache_bktlist_t *, mcache_bkt_t *);
+static void mcache_cache_bkt_enable(mcache_t *);
+static void mcache_bkt_purge(mcache_t *);
+static void mcache_bkt_destroy(mcache_t *, mcache_bkttype_t *,
+    mcache_bkt_t *, int);
+static void mcache_bkt_ws_update(mcache_t *);
+static void mcache_bkt_ws_reap(mcache_t *);
+static void mcache_dispatch(void (*)(void *), void *);
+static void mcache_cache_reap(mcache_t *);
+static void mcache_cache_update(mcache_t *);
+static void mcache_cache_bkt_resize(void *);
+static void mcache_cache_enable(void *);
+static void mcache_update(void *);
+static void mcache_update_timeout(void *);
+static void mcache_applyall(void (*)(mcache_t *));
+static void mcache_reap_start(void *);
+static void mcache_reap_done(void *);
+static void mcache_reap_timeout(void *);
+static void mcache_notify(mcache_t *, u_int32_t);
+static void mcache_purge(void *);
+
+static LIST_HEAD(, mcache) mcache_head;
+mcache_t *mcache_audit_cache;
+
+/*
+ * Initialize the framework; this is currently called as part of BSD init.
+ */
+__private_extern__ void
+mcache_init(void)
+{
+	mcache_bkttype_t *btp;
+	unsigned int i;
+	char name[32];
+
+	ncpu = ml_get_max_cpus();
+
+	mcache_llock_grp_attr = lck_grp_attr_alloc_init();
+	mcache_llock_grp = lck_grp_alloc_init("mcache.list",
+	    mcache_llock_grp_attr);
+	mcache_llock_attr = lck_attr_alloc_init();
+	mcache_llock = lck_mtx_alloc_init(mcache_llock_grp, mcache_llock_attr);
+
+	mcache_zone = zinit(MCACHE_ALLOC_SIZE, 256 * MCACHE_ALLOC_SIZE,
+	    PAGE_SIZE, "mcache");
+	if (mcache_zone == NULL)
+		panic("mcache_init: failed to allocate mcache zone\n");
+
+	LIST_INIT(&mcache_head);
+
+	for (i = 0; i < sizeof (mcache_bkttype) / sizeof (*btp); i++) {
+		btp = &mcache_bkttype[i];
+		(void) snprintf(name, sizeof (name), "bkt_%d",
+		    btp->bt_bktsize);
+		btp->bt_cache = mcache_create(name,
+		    (btp->bt_bktsize + 1) * sizeof (void *), 0, 0, MCR_SLEEP);
+	}
+
+	PE_parse_boot_arg("mcache_flags", &mcache_flags);
+	mcache_flags &= MCF_FLAGS_MASK;
+
+	mcache_audit_cache = mcache_create("audit", sizeof (mcache_audit_t),
+	    0, 0, MCR_SLEEP);
+
+	mcache_reap_interval = 15 * hz;
+	mcache_applyall(mcache_cache_bkt_enable);
+	mcache_ready = 1;
+}
+
+/*
+ * Return the global mcache flags.
+ */
+__private_extern__ unsigned int
+mcache_getflags(void)
+{
+	return (mcache_flags);
+}
+
+/*
+ * Create a cache using the zone allocator as the backend slab allocator.
+ * The caller may specify any alignment for the object; if it specifies 0
+ * the default alignment (MCACHE_ALIGN) will be used.
+ */
+__private_extern__ mcache_t *
+mcache_create(const char *name, size_t bufsize, size_t align,
+    u_int32_t flags, int wait)
+{
+	return (mcache_create_common(name, bufsize, align, mcache_slab_alloc,
+	    mcache_slab_free, mcache_slab_audit, NULL, NULL, flags, 1, wait));
+}
+
+/*
+ * Create a cache using a custom backend slab allocator.  Since the caller
+ * is responsible for allocation, no alignment guarantee will be provided
+ * by this framework.
+ */
+__private_extern__ mcache_t *
+mcache_create_ext(const char *name, size_t bufsize,
+    mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
+    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int wait)
+{
+	return (mcache_create_common(name, bufsize, 0, allocfn,
+	    freefn, auditfn, notifyfn, arg, flags, 0, wait));
+}
+
+/*
+ * Common cache creation routine.
+ */
+static mcache_t *
+mcache_create_common(const char *name, size_t bufsize, size_t align,
+    mcache_allocfn_t allocfn, mcache_freefn_t freefn, mcache_auditfn_t auditfn,
+    mcache_notifyfn_t notifyfn, void *arg, u_int32_t flags, int need_zone,
+    int wait)
+{
+	mcache_bkttype_t *btp;
+	mcache_t *cp = NULL;
+	size_t chunksize;
+	void *buf, **pbuf;
+	int c;
+	char lck_name[64];
+
+	/* If auditing is on and print buffer is NULL, allocate it now */
+	if ((flags & MCF_AUDIT) && mca_dump_buf == NULL) {
+		int malloc_wait = (wait & MCR_NOSLEEP) ? M_NOWAIT : M_WAITOK;
+		MALLOC(mca_dump_buf, char *, DUMP_MCA_BUF_SIZE, M_TEMP,
+		    malloc_wait | M_ZERO);
+		if (mca_dump_buf == NULL)
+			return (NULL);
+	}
+
+	if (!(wait & MCR_NOSLEEP))
+		buf = zalloc(mcache_zone);
+	else
+		buf = zalloc_noblock(mcache_zone);
+
+	if (buf == NULL)
+		goto fail;
+
+	bzero(buf, MCACHE_ALLOC_SIZE);
+
+	/*
+	 * In case we didn't get a cache-aligned memory, round it up
+	 * accordingly.  This is needed in order to get the rest of
+	 * structure members aligned properly.  It also means that
+	 * the memory span gets shifted due to the round up, but it
+	 * is okay since we've allocated extra space for this.
+	 */
+	cp = (mcache_t *)
+	    P2ROUNDUP((intptr_t)buf + sizeof (void *), CPU_CACHE_SIZE);
+	pbuf = (void **)((intptr_t)cp - sizeof (void *));
+	*pbuf = buf;
+
+	/*
+	 * Guaranteed alignment is valid only when we use the internal
+	 * slab allocator (currently set to use the zone allocator).
+	 */
+	if (!need_zone)
+		align = 1;
+	else if (align == 0)
+		align = MCACHE_ALIGN;
+
+	if ((align & (align - 1)) != 0)
+		panic("mcache_create: bad alignment %lu", align);
+
+	cp->mc_align = align;
+	cp->mc_slab_alloc = allocfn;
+	cp->mc_slab_free = freefn;
+	cp->mc_slab_audit = auditfn;
+	cp->mc_slab_notify = notifyfn;
+	cp->mc_private = need_zone ? cp : arg;
+	cp->mc_bufsize = bufsize;
+	cp->mc_flags = (flags & MCF_FLAGS_MASK) | mcache_flags;
+
+	(void) snprintf(cp->mc_name, sizeof (cp->mc_name), "mcache.%s", name);
+
+	(void) snprintf(lck_name, sizeof (lck_name), "%s.cpu", cp->mc_name);
+	cp->mc_cpu_lock_grp_attr = lck_grp_attr_alloc_init();
+	cp->mc_cpu_lock_grp = lck_grp_alloc_init(lck_name,
+	    cp->mc_cpu_lock_grp_attr);
+	cp->mc_cpu_lock_attr = lck_attr_alloc_init();
+
+	/*
+	 * Allocation chunk size is the object's size plus any extra size
+	 * needed to satisfy the object's alignment.  It is enforced to be
+	 * at least the size of an LP64 pointer to simplify auditing and to
+	 * handle multiple-element allocation requests, where the elements
+	 * returned are linked together in a list.
+	 */
+	chunksize = MAX(bufsize, sizeof (u_int64_t));
+	if (need_zone) {
+		/* Enforce 64-bit minimum alignment for zone-based buffers */
+		align = MAX(align, sizeof (u_int64_t));
+		chunksize += sizeof (void *) + align;
+		chunksize = P2ROUNDUP(chunksize, align);
+		if ((cp->mc_slab_zone = zinit(chunksize, 64 * 1024 * ncpu,
+		    PAGE_SIZE, cp->mc_name)) == NULL)
+			goto fail;
+		zone_change(cp->mc_slab_zone, Z_EXPAND, TRUE);
+	}
+	cp->mc_chunksize = chunksize;
+
+	/*
+	 * Initialize the bucket layer.
+	 */
+	(void) snprintf(lck_name, sizeof (lck_name), "%s.bkt", cp->mc_name);
+	cp->mc_bkt_lock_grp_attr = lck_grp_attr_alloc_init();
+	cp->mc_bkt_lock_grp = lck_grp_alloc_init(lck_name,
+	    cp->mc_bkt_lock_grp_attr);
+	cp->mc_bkt_lock_attr = lck_attr_alloc_init();
+	lck_mtx_init(&cp->mc_bkt_lock, cp->mc_bkt_lock_grp,
+	    cp->mc_bkt_lock_attr);
+
+	(void) snprintf(lck_name, sizeof (lck_name), "%s.sync", cp->mc_name);
+	cp->mc_sync_lock_grp_attr = lck_grp_attr_alloc_init();
+	cp->mc_sync_lock_grp = lck_grp_alloc_init(lck_name,
+	    cp->mc_sync_lock_grp_attr);
+	cp->mc_sync_lock_attr = lck_attr_alloc_init();
+	lck_mtx_init(&cp->mc_sync_lock, cp->mc_sync_lock_grp,
+	    cp->mc_sync_lock_attr);
+
+	for (btp = mcache_bkttype; chunksize <= btp->bt_minbuf; btp++)
+		continue;
+
+	cp->cache_bkttype = btp;
+
+	/*
+	 * Initialize the CPU layer.  Each per-CPU structure is aligned
+	 * on the CPU cache line boundary to prevent false sharing.
+	 */
+	for (c = 0; c < ncpu; c++) {
+		mcache_cpu_t *ccp = &cp->mc_cpu[c];
+
+		VERIFY(IS_P2ALIGNED(ccp, CPU_CACHE_SIZE));
+		lck_mtx_init(&ccp->cc_lock, cp->mc_cpu_lock_grp,
+		    cp->mc_cpu_lock_attr);
+		ccp->cc_objs = -1;
+		ccp->cc_pobjs = -1;
+	}
+
+	if (mcache_ready)
+		mcache_cache_bkt_enable(cp);
+
+	/* TODO: dynamically create sysctl for stats */
+
+	MCACHE_LIST_LOCK();
+	LIST_INSERT_HEAD(&mcache_head, cp, mc_list);
+	MCACHE_LIST_UNLOCK();
+
+	/*
+	 * If cache buckets are enabled and this is the first cache
+	 * created, start the periodic cache update.
+	 */
+	if (!(mcache_flags & MCF_NOCPUCACHE) && !mcache_updating) {
+		mcache_updating = 1;
+		mcache_update_timeout(NULL);
+	}
+	if (cp->mc_flags & MCF_DEBUG) {
+		printf("mcache_create: %s (%s) arg %p bufsize %lu align %lu "
+		    "chunksize %lu bktsize %d\n", name, need_zone ? "i" : "e",
+		    arg, bufsize, cp->mc_align, chunksize, btp->bt_bktsize);
+	}
+	return (cp);
+
+fail:
+	if (buf != NULL)
+		zfree(mcache_zone, buf);
+	return (NULL);
+}
+
+/*
+ * Allocate one or more objects from a cache.
+ */
+__private_extern__ unsigned int
+mcache_alloc_ext(mcache_t *cp, mcache_obj_t **list, unsigned int num, int wait)
+{
+	mcache_cpu_t *ccp;
+	mcache_obj_t **top = &(*list);
+	mcache_bkt_t *bkt;
+	unsigned int need = num;
+	boolean_t nwretry = FALSE;
+
+	/* MCR_NOSLEEP and MCR_FAILOK are mutually exclusive */
+	VERIFY((wait & (MCR_NOSLEEP|MCR_FAILOK)) != (MCR_NOSLEEP|MCR_FAILOK));
+
+	ASSERT(list != NULL);
+	*list = NULL;
+
+	if (num == 0)
+		return (0);
+
+retry_alloc:
+	/* We may not always be running in the same CPU in case of retries */
+	ccp = MCACHE_CPU(cp);
+
+	MCACHE_LOCK(&ccp->cc_lock);
+	for (;;) {
+		/*
+		 * If we have an object in the current CPU's filled bucket,
+		 * chain the object to any previous objects and return if
+		 * we've satisfied the number of requested objects.
+		 */
+		if (ccp->cc_objs > 0) {
+			mcache_obj_t *tail;
+			int objs;
+
+			/*
+			 * Objects in the bucket are already linked together
+			 * with the most recently freed object at the head of
+			 * the list; grab as many objects as we can.
+			 */
+			objs = MIN((unsigned int)ccp->cc_objs, need);
+			*list = ccp->cc_filled->bkt_obj[ccp->cc_objs - 1];
+			ccp->cc_objs -= objs;
+			ccp->cc_alloc += objs;
+
+			tail = ccp->cc_filled->bkt_obj[ccp->cc_objs];
+			list = &tail->obj_next;
+			*list = NULL;
+
+			/* If we got them all, return to caller */
+			if ((need -= objs) == 0) {
+				MCACHE_UNLOCK(&ccp->cc_lock);
+				if (cp->mc_flags & MCF_DEBUG)
+					goto debug_alloc;
+
+				return (num);
+			}
+		}
+
+		/*
+		 * The CPU's filled bucket is empty.  If the previous filled
+		 * bucket was full, exchange and try again.
+		 */
+		if (ccp->cc_pobjs > 0) {
+			mcache_cpu_refill(ccp, ccp->cc_pfilled, ccp->cc_pobjs);
+			continue;
+		}
+
+		/*
+		 * If the bucket layer is disabled, allocate from slab.  This
+		 * can happen either because MCF_NOCPUCACHE is set, or because
+		 * the bucket layer is currently being resized.
+		 */
+		if (ccp->cc_bktsize == 0)
+			break;
+
+		/*
+		 * Both of the CPU's buckets are empty; try to get a full
+		 * bucket from the bucket layer.  Upon success, refill this
+		 * CPU and place any empty bucket into the empty list.
+		 */
+		bkt = mcache_bkt_alloc(cp, &cp->mc_full, NULL);
+		if (bkt != NULL) {
+			if (ccp->cc_pfilled != NULL)
+				mcache_bkt_free(cp, &cp->mc_empty,
+				    ccp->cc_pfilled);
+			mcache_cpu_refill(ccp, bkt, ccp->cc_bktsize);
+			continue;
+		}
+
+		/*
+		 * The bucket layer has no full buckets; allocate the
+		 * object(s) directly from the slab layer.
+		 */
+		break;
+	}
+	MCACHE_UNLOCK(&ccp->cc_lock);
+
+	need -= (*cp->mc_slab_alloc)(cp->mc_private, &list, need, wait);
+
+	/*
+	 * If this is a blocking allocation, or if it is non-blocking and
+	 * the cache's full bucket is non-empty, then retry the allocation.
+	 */
+	if (need > 0) {
+		if (!(wait & MCR_NONBLOCKING)) {
+			atomic_add_32(&cp->mc_wretry_cnt, 1);
+			goto retry_alloc;
+		} else if ((wait & (MCR_NOSLEEP | MCR_TRYHARD)) &&
+		    !mcache_bkt_isempty(cp)) {
+			if (!nwretry)
+				nwretry = TRUE;
+			atomic_add_32(&cp->mc_nwretry_cnt, 1);
+			goto retry_alloc;
+		} else if (nwretry) {
+			atomic_add_32(&cp->mc_nwfail_cnt, 1);
+		}
+	}
+
+	if (!(cp->mc_flags & MCF_DEBUG))
+		return (num - need);
+
+debug_alloc:
+	if (cp->mc_flags & MCF_VERIFY) {
+		mcache_obj_t **o = top;
+		unsigned int n;
+
+		n = 0;
+		/*
+		 * Verify that the chain of objects have the same count as
+		 * what we are about to report to the caller.  Any mismatch
+		 * here means that the object list is insanely broken and
+		 * therefore we must panic.
+		 */
+		while (*o != NULL) {
+			o = &(*o)->obj_next;
+			++n;
+		}
+		if (n != (num - need)) {
+			panic("mcache_alloc_ext: %s cp %p corrupted list "
+			    "(got %d actual %d)\n", cp->mc_name,
+			    (void *)cp, num - need, n);
+		}
+	}
+
+	/* Invoke the slab layer audit callback if auditing is enabled */
+	if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+		(*cp->mc_slab_audit)(cp->mc_private, *top, TRUE);
+
+	return (num - need);
+}
+
+/*
+ * Allocate a single object from a cache.
+ */
+__private_extern__ void *
+mcache_alloc(mcache_t *cp, int wait)
+{
+	mcache_obj_t *buf;
+
+	(void) mcache_alloc_ext(cp, &buf, 1, wait);
+	return (buf);
+}
+
+__private_extern__ void
+mcache_waiter_inc(mcache_t *cp)
+{
+	atomic_add_32(&cp->mc_waiter_cnt, 1);
+}
+
+__private_extern__ void
+mcache_waiter_dec(mcache_t *cp)
+{
+	atomic_add_32(&cp->mc_waiter_cnt, -1);
+}
+
+__private_extern__ boolean_t
+mcache_bkt_isempty(mcache_t *cp)
+{
+	/*
+	 * This isn't meant to accurately tell whether there are
+	 * any full buckets in the cache; it is simply a way to
+	 * obtain "hints" about the state of the cache.
+	 */
+	return (cp->mc_full.bl_total == 0);
+}
+
+/*
+ * Notify the slab layer about an event.
+ */
+static void
+mcache_notify(mcache_t *cp, u_int32_t event)
+{
+	if (cp->mc_slab_notify != NULL)
+		(*cp->mc_slab_notify)(cp->mc_private, event);
+}
+
+/*
+ * Purge the cache and disable its buckets.
+ */
+static void
+mcache_purge(void *arg)
+{
+	mcache_t *cp = arg;
+
+	mcache_bkt_purge(cp);
+	/*
+	 * We cannot simply call mcache_cache_bkt_enable() from here as
+	 * a bucket resize may be in flight and we would cause the CPU
+	 * layers of the cache to point to different sizes.  Therefore,
+	 * we simply increment the enable count so that during the next
+	 * periodic cache update the buckets can be reenabled.
+	 */
+	lck_mtx_lock_spin(&cp->mc_sync_lock);
+	cp->mc_enable_cnt++;
+	lck_mtx_unlock(&cp->mc_sync_lock);
+
+}
+
+__private_extern__ boolean_t
+mcache_purge_cache(mcache_t *cp)
+{
+	/*
+	 * Purging a cache that has no per-CPU caches or is already
+	 * in the process of being purged is rather pointless.
+	 */
+	if (cp->mc_flags & MCF_NOCPUCACHE)
+		return (FALSE);
+
+	lck_mtx_lock_spin(&cp->mc_sync_lock);
+	if (cp->mc_purge_cnt > 0) {
+		lck_mtx_unlock(&cp->mc_sync_lock);
+		return (FALSE);
+	}
+	cp->mc_purge_cnt++;
+	lck_mtx_unlock(&cp->mc_sync_lock);
+
+	mcache_dispatch(mcache_purge, cp);
+
+	return (TRUE);
+}
+
+/*
+ * Free a single object to a cache.
+ */
+__private_extern__ void
+mcache_free(mcache_t *cp, void *buf)
+{
+	((mcache_obj_t *)buf)->obj_next = NULL;
+	mcache_free_ext(cp, (mcache_obj_t *)buf);
+}
+
+/*
+ * Free one or more objects to a cache.
+ */
+__private_extern__ void
+mcache_free_ext(mcache_t *cp, mcache_obj_t *list)
+{
+	mcache_cpu_t *ccp = MCACHE_CPU(cp);
+	mcache_bkttype_t *btp;
+	mcache_obj_t *nlist;
+	mcache_bkt_t *bkt;
+
+	/* Invoke the slab layer audit callback if auditing is enabled */
+	if ((cp->mc_flags & MCF_AUDIT) && cp->mc_slab_audit != NULL)
+		(*cp->mc_slab_audit)(cp->mc_private, list, FALSE);
+
+	MCACHE_LOCK(&ccp->cc_lock);
+	for (;;) {
+		/*
+		 * If there is space in the current CPU's filled bucket, put
+		 * the object there and return once all objects are freed.
+		 * Note the cast to unsigned integer takes care of the case
+		 * where the bucket layer is disabled (when cc_objs is -1).
+		 */
+		if ((unsigned int)ccp->cc_objs <
+		    (unsigned int)ccp->cc_bktsize) {
+			/*
+			 * Reverse the list while we place the object into the
+			 * bucket; this effectively causes the most recently
+			 * freed object(s) to be reused during allocation.
+			 */
+			nlist = list->obj_next;
+			list->obj_next = (ccp->cc_objs == 0) ? NULL :
+			    ccp->cc_filled->bkt_obj[ccp->cc_objs - 1];
+			ccp->cc_filled->bkt_obj[ccp->cc_objs++] = list;
+			ccp->cc_free++;
+
+			if ((list = nlist) != NULL)
+				continue;
+
+			/* We are done; return to caller */
+			MCACHE_UNLOCK(&ccp->cc_lock);
+
+			/* If there is a waiter below, notify it */
+			if (cp->mc_waiter_cnt > 0)
+				mcache_notify(cp, MCN_RETRYALLOC);
+			return;
+		}
+
+		/*
+		 * The CPU's filled bucket is full.  If the previous filled
+		 * bucket was empty, exchange and try again.
+		 */
+		if (ccp->cc_pobjs == 0) {
+			mcache_cpu_refill(ccp, ccp->cc_pfilled, ccp->cc_pobjs);
+			continue;
+		}
+
+		/*
+		 * If the bucket layer is disabled, free to slab.  This can
+		 * happen either because MCF_NOCPUCACHE is set, or because
+		 * the bucket layer is currently being resized.
+		 */
+		if (ccp->cc_bktsize == 0)
+			break;
+
+		/*
+		 * Both of the CPU's buckets are full; try to get an empty
+		 * bucket from the bucket layer.  Upon success, empty this
+		 * CPU and place any full bucket into the full list.
+		 */
+		bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp);
+		if (bkt != NULL) {
+			if (ccp->cc_pfilled != NULL)
+				mcache_bkt_free(cp, &cp->mc_full,
+				    ccp->cc_pfilled);
+			mcache_cpu_refill(ccp, bkt, 0);
+			continue;
+		}
+
+		/*
+		 * We need an empty bucket to put our freed objects into
+		 * but couldn't get an empty bucket from the bucket layer;
+		 * attempt to allocate one.  We do not want to block for
+		 * allocation here, and if the bucket allocation fails
+		 * we will simply fall through to the slab layer.
+		 */
+		MCACHE_UNLOCK(&ccp->cc_lock);
+		bkt = mcache_alloc(btp->bt_cache, MCR_NOSLEEP);
+		MCACHE_LOCK(&ccp->cc_lock);
+
+		if (bkt != NULL) {
+			/*
+			 * We have an empty bucket, but since we drop the
+			 * CPU lock above, the cache's bucket size may have
+			 * changed.  If so, free the bucket and try again.
+			 */
+			if (ccp->cc_bktsize != btp->bt_bktsize) {
+				MCACHE_UNLOCK(&ccp->cc_lock);
+				mcache_free(btp->bt_cache, bkt);
+				MCACHE_LOCK(&ccp->cc_lock);
+				continue;
+			}
+
+			/*
+			 * We have an empty bucket of the right size;
+			 * add it to the bucket layer and try again.
+			 */
+			mcache_bkt_free(cp, &cp->mc_empty, bkt);
+			continue;
+		}
+
+		/*
+		 * The bucket layer has no empty buckets; free the
+		 * object(s) directly to the slab layer.
+		 */
+		break;
+	}
+	MCACHE_UNLOCK(&ccp->cc_lock);
+
+	/* If there is a waiter below, notify it */
+	if (cp->mc_waiter_cnt > 0)
+		mcache_notify(cp, MCN_RETRYALLOC);
+
+	/* Advise the slab layer to purge the object(s) */
+	(*cp->mc_slab_free)(cp->mc_private, list,
+	    (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
+}
+
+/*
+ * Cache destruction routine.
+ */
+__private_extern__ void
+mcache_destroy(mcache_t *cp)
+{
+	void **pbuf;
+
+	MCACHE_LIST_LOCK();
+	LIST_REMOVE(cp, mc_list);
+	MCACHE_LIST_UNLOCK();
+
+	mcache_bkt_purge(cp);
+
+	/*
+	 * This cache is dead; there should be no further transaction.
+	 * If it's still invoked, make sure that it induces a fault.
+	 */
+	cp->mc_slab_alloc = NULL;
+	cp->mc_slab_free = NULL;
+	cp->mc_slab_audit = NULL;
+
+	lck_attr_free(cp->mc_bkt_lock_attr);
+	lck_grp_free(cp->mc_bkt_lock_grp);
+	lck_grp_attr_free(cp->mc_bkt_lock_grp_attr);
+
+	lck_attr_free(cp->mc_cpu_lock_attr);
+	lck_grp_free(cp->mc_cpu_lock_grp);
+	lck_grp_attr_free(cp->mc_cpu_lock_grp_attr);
+
+	lck_attr_free(cp->mc_sync_lock_attr);
+	lck_grp_free(cp->mc_sync_lock_grp);
+	lck_grp_attr_free(cp->mc_sync_lock_grp_attr);
+
+	/*
+	 * TODO: We need to destroy the zone here, but cannot do it
+	 * because there is no such way to achieve that.  Until then
+	 * the memory allocated for the zone structure is leaked.
+	 * Once it is achievable, uncomment these lines:
+	 *
+	 *	if (cp->mc_slab_zone != NULL) {
+	 *		zdestroy(cp->mc_slab_zone);
+	 *		cp->mc_slab_zone = NULL;
+	 *	}
+	 */
+
+	/* Get the original address since we're about to free it */
+	pbuf = (void **)((intptr_t)cp - sizeof (void *));
+
+	zfree(mcache_zone, *pbuf);
+}
+
+/*
+ * Internal slab allocator used as a backend for simple caches.  The current
+ * implementation uses the zone allocator for simplicity reasons.
+ */
+static unsigned int
+mcache_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
+{
+	mcache_t *cp = arg;
+	unsigned int need = num;
+	size_t offset = 0;
+	size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
+	u_int32_t flags = cp->mc_flags;
+	void *buf, *base, **pbuf;
+	mcache_obj_t **list = *plist;
+
+	*list = NULL;
+
+	/*
+	 * The address of the object returned to the caller is an
+	 * offset from the 64-bit aligned base address only if the
+	 * cache's alignment requirement is neither 1 nor 8 bytes.
+	 */
+	if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
+		offset = cp->mc_align;
+
+	for (;;) {
+		if (!(wait & MCR_NOSLEEP))
+			buf = zalloc(cp->mc_slab_zone);
+		else
+			buf = zalloc_noblock(cp->mc_slab_zone);
+
+		if (buf == NULL)
+			break;
+
+		/* Get the 64-bit aligned base address for this object */
+		base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
+		    sizeof (u_int64_t));
+
+		/*
+		 * Wind back a pointer size from the aligned base and
+		 * save the original address so we can free it later.
+		 */
+		pbuf = (void **)((intptr_t)base - sizeof (void *));
+		*pbuf = buf;
+
+		/*
+		 * If auditing is enabled, patternize the contents of
+		 * the buffer starting from the 64-bit aligned base to
+		 * the end of the buffer; the length is rounded up to
+		 * the nearest 64-bit multiply; this is because we use
+		 * 64-bit memory access to set/check the pattern.
+		 */
+		if (flags & MCF_AUDIT) {
+			VERIFY(((intptr_t)base + rsize) <=
+			    ((intptr_t)buf + cp->mc_chunksize));
+			mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize);
+		}
+
+		/*
+		 * Fix up the object's address to fulfill the cache's
+		 * alignment requirement (if needed) and return this
+		 * to the caller.
+		 */
+		VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <=
+		    ((intptr_t)buf + cp->mc_chunksize));
+		*list = (mcache_obj_t *)((intptr_t)base + offset);
+
+		(*list)->obj_next = NULL;
+		list = *plist = &(*list)->obj_next;
+
+		/* If we got them all, return to mcache */
+		if (--need == 0)
+			break;
+	}
+
+	return (num - need);
+}
+
+/*
+ * Internal slab deallocator used as a backend for simple caches.
+ */
+static void
+mcache_slab_free(void *arg, mcache_obj_t *list, __unused boolean_t purged)
+{
+	mcache_t *cp = arg;
+	mcache_obj_t *nlist;
+	size_t offset = 0;
+	size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
+	u_int32_t flags = cp->mc_flags;
+	void *base;
+	void **pbuf;
+
+	/*
+	 * The address of the object is an offset from a 64-bit
+	 * aligned base address only if the cache's alignment
+	 * requirement is neither 1 nor 8 bytes.
+	 */
+	if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
+		offset = cp->mc_align;
+
+	for (;;) {
+		nlist = list->obj_next;
+		list->obj_next = NULL;
+
+		/* Get the 64-bit aligned base address of this object */
+		base = (void *)((intptr_t)list - offset);
+		VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t)));
+
+		/* Get the original address since we're about to free it */
+		pbuf = (void **)((intptr_t)base - sizeof (void *));
+
+		if (flags & MCF_AUDIT) {
+			VERIFY(((intptr_t)base + rsize) <=
+			    ((intptr_t)*pbuf + cp->mc_chunksize));
+			mcache_audit_free_verify(NULL, base, offset, rsize);
+		}
+
+		/* Free it to zone */
+		VERIFY(((intptr_t)base + offset + cp->mc_bufsize) <=
+		    ((intptr_t)*pbuf + cp->mc_chunksize));
+		zfree(cp->mc_slab_zone, *pbuf);
+
+		/* No more objects to free; return to mcache */
+		if ((list = nlist) == NULL)
+			break;
+	}
+}
+
+/*
+ * Internal slab auditor for simple caches.
+ */
+static void
+mcache_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
+{
+	mcache_t *cp = arg;
+	size_t offset = 0;
+	size_t rsize = P2ROUNDUP(cp->mc_bufsize, sizeof (u_int64_t));
+	void *base, **pbuf;
+
+	/*
+	 * The address of the object returned to the caller is an
+	 * offset from the 64-bit aligned base address only if the
+	 * cache's alignment requirement is neither 1 nor 8 bytes.
+	 */
+	if (cp->mc_align != 1 && cp->mc_align != sizeof (u_int64_t))
+		offset = cp->mc_align;
+
+	while (list != NULL) {
+		mcache_obj_t *next = list->obj_next;
+
+		/* Get the 64-bit aligned base address of this object */
+		base = (void *)((intptr_t)list - offset);
+		VERIFY(IS_P2ALIGNED(base, sizeof (u_int64_t)));
+
+		/* Get the original address */
+		pbuf = (void **)((intptr_t)base - sizeof (void *));
+
+		VERIFY(((intptr_t)base + rsize) <=
+		    ((intptr_t)*pbuf + cp->mc_chunksize));
+
+		if (!alloc)
+			mcache_set_pattern(MCACHE_FREE_PATTERN, base, rsize);
+		else
+			mcache_audit_free_verify_set(NULL, base, offset, rsize);
+
+		list = list->obj_next = next;
+	}
+}
+
+/*
+ * Refill the CPU's filled bucket with bkt and save the previous one.
+ */
+static void
+mcache_cpu_refill(mcache_cpu_t *ccp, mcache_bkt_t *bkt, int objs)
+{
+	ASSERT((ccp->cc_filled == NULL && ccp->cc_objs == -1) ||
+	    (ccp->cc_filled && ccp->cc_objs + objs == ccp->cc_bktsize));
+	ASSERT(ccp->cc_bktsize > 0);
+
+	ccp->cc_pfilled = ccp->cc_filled;
+	ccp->cc_pobjs = ccp->cc_objs;
+	ccp->cc_filled = bkt;
+	ccp->cc_objs = objs;
+}
+
+/*
+ * Allocate a bucket from the bucket layer.
+ */
+static mcache_bkt_t *
+mcache_bkt_alloc(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkttype_t **btp)
+{
+	mcache_bkt_t *bkt;
+
+	if (!MCACHE_LOCK_TRY(&cp->mc_bkt_lock)) {
+		/*
+		 * The bucket layer lock is held by another CPU; increase
+		 * the contention count so that we can later resize the
+		 * bucket size accordingly.
+		 */
+		MCACHE_LOCK(&cp->mc_bkt_lock);
+		cp->mc_bkt_contention++;
+	}
+
+	if ((bkt = blp->bl_list) != NULL) {
+		blp->bl_list = bkt->bkt_next;
+		if (--blp->bl_total < blp->bl_min)
+			blp->bl_min = blp->bl_total;
+		blp->bl_alloc++;
+	}
+
+	if (btp != NULL)
+		*btp = cp->cache_bkttype;
+
+	MCACHE_UNLOCK(&cp->mc_bkt_lock);
+
+	return (bkt);
+}
+
+/*
+ * Free a bucket to the bucket layer.
+ */
+static void
+mcache_bkt_free(mcache_t *cp, mcache_bktlist_t *blp, mcache_bkt_t *bkt)
+{
+	MCACHE_LOCK(&cp->mc_bkt_lock);
+
+	bkt->bkt_next = blp->bl_list;
+	blp->bl_list = bkt;
+	blp->bl_total++;
+
+	MCACHE_UNLOCK(&cp->mc_bkt_lock);
+}
+
+/*
+ * Enable the bucket layer of a cache.
+ */
+static void
+mcache_cache_bkt_enable(mcache_t *cp)
+{
+	mcache_cpu_t *ccp;
+	int cpu;
+
+	if (cp->mc_flags & MCF_NOCPUCACHE)
+		return;
+
+	for (cpu = 0; cpu < ncpu; cpu++) {
+		ccp = &cp->mc_cpu[cpu];
+		MCACHE_LOCK(&ccp->cc_lock);
+		ccp->cc_bktsize = cp->cache_bkttype->bt_bktsize;
+		MCACHE_UNLOCK(&ccp->cc_lock);
+	}
+}
+
+/*
+ * Purge all buckets from a cache and disable its bucket layer.
+ */
+static void
+mcache_bkt_purge(mcache_t *cp)
+{
+	mcache_cpu_t *ccp;
+	mcache_bkt_t *bp, *pbp;
+	mcache_bkttype_t *btp;
+	int cpu, objs, pobjs;
+
+	for (cpu = 0; cpu < ncpu; cpu++) {
+		ccp = &cp->mc_cpu[cpu];
+
+		MCACHE_LOCK(&ccp->cc_lock);
+
+		btp = cp->cache_bkttype;
+		bp = ccp->cc_filled;
+		pbp = ccp->cc_pfilled;
+		objs = ccp->cc_objs;
+		pobjs = ccp->cc_pobjs;
+		ccp->cc_filled = NULL;
+		ccp->cc_pfilled = NULL;
+		ccp->cc_objs = -1;
+		ccp->cc_pobjs = -1;
+		ccp->cc_bktsize = 0;
+
+		MCACHE_UNLOCK(&ccp->cc_lock);
+
+		if (bp != NULL)
+			mcache_bkt_destroy(cp, btp, bp, objs);
+		if (pbp != NULL)
+			mcache_bkt_destroy(cp, btp, pbp, pobjs);
+	}
+
+	/*
+	 * Updating the working set back to back essentially sets
+	 * the working set size to zero, so everything is reapable.
+	 */
+	mcache_bkt_ws_update(cp);
+	mcache_bkt_ws_update(cp);
+
+	mcache_bkt_ws_reap(cp);
+}
+
+/*
+ * Free one or more objects in the bucket to the slab layer,
+ * and also free the bucket itself.
+ */
+static void
+mcache_bkt_destroy(mcache_t *cp, mcache_bkttype_t *btp, mcache_bkt_t *bkt,
+    int nobjs)
+{
+	if (nobjs > 0) {
+		mcache_obj_t *top = bkt->bkt_obj[nobjs - 1];
+
+		if (cp->mc_flags & MCF_VERIFY) {
+			mcache_obj_t *o = top;
+			int cnt = 0;
+
+			/*
+			 * Verify that the chain of objects in the bucket is
+			 * valid.  Any mismatch here means a mistake when the
+			 * object(s) were freed to the CPU layer, so we panic.
+			 */
+			while (o != NULL) {
+				o = o->obj_next;
+				++cnt;
+			}
+			if (cnt != nobjs) {
+				panic("mcache_bkt_destroy: %s cp %p corrupted "
+				    "list in bkt %p (nobjs %d actual %d)\n",
+				    cp->mc_name, (void *)cp, (void *)bkt,
+				    nobjs, cnt);
+			}
+		}
+
+		/* Advise the slab layer to purge the object(s) */
+		(*cp->mc_slab_free)(cp->mc_private, top,
+		    (cp->mc_flags & MCF_DEBUG) || cp->mc_purge_cnt);
+	}
+	mcache_free(btp->bt_cache, bkt);
+}
+
+/*
+ * Update the bucket layer working set statistics.
+ */
+static void
+mcache_bkt_ws_update(mcache_t *cp)
+{
+	MCACHE_LOCK(&cp->mc_bkt_lock);
+
+	cp->mc_full.bl_reaplimit = cp->mc_full.bl_min;
+	cp->mc_full.bl_min = cp->mc_full.bl_total;
+	cp->mc_empty.bl_reaplimit = cp->mc_empty.bl_min;
+	cp->mc_empty.bl_min = cp->mc_empty.bl_total;
+
+	MCACHE_UNLOCK(&cp->mc_bkt_lock);
+}
+
+/*
+ * Reap all buckets that are beyond the working set.
+ */
+static void
+mcache_bkt_ws_reap(mcache_t *cp)
+{
+	long reap;
+	mcache_bkt_t *bkt;
+	mcache_bkttype_t *btp;
+
+	reap = MIN(cp->mc_full.bl_reaplimit, cp->mc_full.bl_min);
+	while (reap-- &&
+	    (bkt = mcache_bkt_alloc(cp, &cp->mc_full, &btp)) != NULL)
+		mcache_bkt_destroy(cp, btp, bkt, btp->bt_bktsize);
+
+	reap = MIN(cp->mc_empty.bl_reaplimit, cp->mc_empty.bl_min);
+	while (reap-- &&
+	    (bkt = mcache_bkt_alloc(cp, &cp->mc_empty, &btp)) != NULL)
+		mcache_bkt_destroy(cp, btp, bkt, 0);
+}
+
+static void
+mcache_reap_timeout(void *arg)
+{
+	volatile UInt32 *flag = arg;
+
+	ASSERT(flag == &mcache_reaping);
+
+	*flag = 0;
+}
+
+static void
+mcache_reap_done(void *flag)
+{
+	timeout(mcache_reap_timeout, flag, mcache_reap_interval);
+}
+
+static void
+mcache_reap_start(void *arg)
+{
+	UInt32 *flag = arg;
+
+	ASSERT(flag == &mcache_reaping);
+
+	mcache_applyall(mcache_cache_reap);
+	mcache_dispatch(mcache_reap_done, flag);
+}
+
+__private_extern__ void
+mcache_reap(void)
+{
+	UInt32 *flag = &mcache_reaping;
+
+	if (mcache_llock_owner == current_thread() ||
+	    !OSCompareAndSwap(0, 1, flag))
+		return;
+
+	mcache_dispatch(mcache_reap_start, flag);
+}
+
+static void
+mcache_cache_reap(mcache_t *cp)
+{
+	mcache_bkt_ws_reap(cp);
+}
+
+/*
+ * Performs period maintenance on a cache.
+ */
+static void
+mcache_cache_update(mcache_t *cp)
+{
+	int need_bkt_resize = 0;
+	int need_bkt_reenable = 0;
+
+	lck_mtx_assert(mcache_llock, LCK_MTX_ASSERT_OWNED);
+
+	mcache_bkt_ws_update(cp);
+
+	/*
+	 * Cache resize and post-purge reenable are mutually exclusive.
+	 * If the cache was previously purged, there is no point of
+	 * increasing the bucket size as there was an indication of
+	 * memory pressure on the system.
+	 */
+	lck_mtx_lock_spin(&cp->mc_sync_lock);
+	if (!(cp->mc_flags & MCF_NOCPUCACHE) && cp->mc_enable_cnt)
+		need_bkt_reenable = 1;
+	lck_mtx_unlock(&cp->mc_sync_lock);
+
+	MCACHE_LOCK(&cp->mc_bkt_lock);
+	/*
+	 * If the contention count is greater than the threshold, and if
+	 * we are not already at the maximum bucket size, increase it.
+	 * Otherwise, if this cache was previously purged by the user
+	 * then we simply reenable it.
+	 */
+	if ((unsigned int)cp->mc_chunksize < cp->cache_bkttype->bt_maxbuf &&
+	    (int)(cp->mc_bkt_contention - cp->mc_bkt_contention_prev) >
+	    mcache_bkt_contention && !need_bkt_reenable)
+		need_bkt_resize = 1;
+
+	cp ->mc_bkt_contention_prev = cp->mc_bkt_contention;
+	MCACHE_UNLOCK(&cp->mc_bkt_lock);
+
+	if (need_bkt_resize)
+		mcache_dispatch(mcache_cache_bkt_resize, cp);
+	else if (need_bkt_reenable)
+		mcache_dispatch(mcache_cache_enable, cp);
+}
+
+/*
+ * Recompute a cache's bucket size.  This is an expensive operation
+ * and should not be done frequently; larger buckets provide for a
+ * higher transfer rate with the bucket while smaller buckets reduce
+ * the memory consumption.
+ */
+static void
+mcache_cache_bkt_resize(void *arg)
+{
+	mcache_t *cp = arg;
+	mcache_bkttype_t *btp = cp->cache_bkttype;
+
+	if ((unsigned int)cp->mc_chunksize < btp->bt_maxbuf) {
+		mcache_bkt_purge(cp);
+
+		/*
+		 * Upgrade to the next bucket type with larger bucket size;
+		 * temporarily set the previous contention snapshot to a
+		 * negative number to prevent unnecessary resize request.
+		 */
+		MCACHE_LOCK(&cp->mc_bkt_lock);
+		cp->cache_bkttype = ++btp;
+		cp ->mc_bkt_contention_prev = cp->mc_bkt_contention + INT_MAX;
+		MCACHE_UNLOCK(&cp->mc_bkt_lock);
+
+		mcache_cache_enable(cp);
+	}
+}
+
+/*
+ * Reenable a previously disabled cache due to purge.
+ */
+static void
+mcache_cache_enable(void *arg)
+{
+	mcache_t *cp = arg;
+
+	lck_mtx_lock_spin(&cp->mc_sync_lock);
+	cp->mc_purge_cnt = 0;
+	cp->mc_enable_cnt = 0;
+	lck_mtx_unlock(&cp->mc_sync_lock);
+
+	mcache_cache_bkt_enable(cp);
+}
+
+static void
+mcache_update_timeout(__unused void *arg)
+{
+	timeout(mcache_update, NULL, mcache_reap_interval);
+}
+
+static void
+mcache_update(__unused void *arg)
+{
+	mcache_applyall(mcache_cache_update);
+	mcache_dispatch(mcache_update_timeout, NULL);
+}
+
+static void
+mcache_applyall(void (*func)(mcache_t *))
+{
+	mcache_t *cp;
+
+	MCACHE_LIST_LOCK();
+	LIST_FOREACH(cp, &mcache_head, mc_list) {
+		func(cp);
+	}
+	MCACHE_LIST_UNLOCK();
+}
+
+static void
+mcache_dispatch(void (*func)(void *), void *arg)
+{
+	ASSERT(func != NULL);
+	timeout(func, arg, hz/1000);
+}
+
+__private_extern__ void
+mcache_buffer_log(mcache_audit_t *mca, void *addr, mcache_t *cp)
+{
+	mca->mca_addr = addr;
+	mca->mca_cache = cp;
+	mca->mca_pthread = mca->mca_thread;
+	mca->mca_thread = current_thread();
+	bcopy(mca->mca_stack, mca->mca_pstack, sizeof (mca->mca_pstack));
+	mca->mca_pdepth = mca->mca_depth;
+	bzero(mca->mca_stack, sizeof (mca->mca_stack));
+	mca->mca_depth = OSBacktrace(mca->mca_stack, MCACHE_STACK_DEPTH);
+}
+
+__private_extern__ void
+mcache_set_pattern(u_int64_t pattern, void *buf_arg, size_t size)
+{
+	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf = (u_int64_t *)buf_arg;
+
+	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
+	VERIFY(IS_P2ALIGNED(size, sizeof (u_int64_t)));
+
+	while (buf < buf_end)
+		*buf++ = pattern;
+}
+
+__private_extern__ void *
+mcache_verify_pattern(u_int64_t pattern, void *buf_arg, size_t size)
+{
+	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf;
+
+	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
+	VERIFY(IS_P2ALIGNED(size, sizeof (u_int64_t)));
+
+	for (buf = buf_arg; buf < buf_end; buf++) {
+		if (*buf != pattern)
+			return (buf);
+	}
+	return (NULL);
+}
+
+__private_extern__ void *
+mcache_verify_set_pattern(u_int64_t old, u_int64_t new, void *buf_arg,
+    size_t size)
+{
+	u_int64_t *buf_end = (u_int64_t *)((char *)buf_arg + size);
+	u_int64_t *buf;
+
+	VERIFY(IS_P2ALIGNED(buf_arg, sizeof (u_int64_t)));
+	VERIFY(IS_P2ALIGNED(size, sizeof (u_int64_t)));
+
+	for (buf = buf_arg; buf < buf_end; buf++) {
+		if (*buf != old) {
+			mcache_set_pattern(old, buf_arg,
+			    (uintptr_t)buf - (uintptr_t)buf_arg);
+			return (buf);
+		}
+		*buf = new;
+	}
+	return (NULL);
+}
+
+__private_extern__ void
+mcache_audit_free_verify(mcache_audit_t *mca, void *base, size_t offset,
+    size_t size)
+{
+	void *addr;
+	u_int64_t *oaddr64;
+	mcache_obj_t *next;
+
+	addr = (void *)((uintptr_t)base + offset);
+	next = ((mcache_obj_t *)addr)->obj_next;
+
+	/* For the "obj_next" pointer in the buffer */
+	oaddr64 = (u_int64_t *)P2ROUNDDOWN(addr, sizeof (u_int64_t));
+	*oaddr64 = MCACHE_FREE_PATTERN;
+
+	if ((oaddr64 = mcache_verify_pattern(MCACHE_FREE_PATTERN,
+	    (caddr_t)base, size)) != NULL) {
+		mcache_audit_panic(mca, addr, (caddr_t)oaddr64 - (caddr_t)base,
+		    (int64_t)MCACHE_FREE_PATTERN, (int64_t)*oaddr64);
+		/* NOTREACHED */
+	}
+	((mcache_obj_t *)addr)->obj_next = next;
+}
+
+__private_extern__ void
+mcache_audit_free_verify_set(mcache_audit_t *mca, void *base, size_t offset,
+    size_t size)
+{
+	void *addr;
+	u_int64_t *oaddr64;
+	mcache_obj_t *next;
+
+	addr = (void *)((uintptr_t)base + offset);
+	next = ((mcache_obj_t *)addr)->obj_next;
+
+	/* For the "obj_next" pointer in the buffer */
+	oaddr64 = (u_int64_t *)P2ROUNDDOWN(addr, sizeof (u_int64_t));
+	*oaddr64 = MCACHE_FREE_PATTERN;
+
+	if ((oaddr64 = mcache_verify_set_pattern(MCACHE_FREE_PATTERN,
+	    MCACHE_UNINITIALIZED_PATTERN, (caddr_t)base, size)) != NULL) {
+		mcache_audit_panic(mca, addr, (caddr_t)oaddr64 - (caddr_t)base,
+		    (int64_t)MCACHE_FREE_PATTERN, (int64_t)*oaddr64);
+		/* NOTREACHED */
+	}
+	((mcache_obj_t *)addr)->obj_next = next;
+}
+
+#undef panic(...)
+
+__private_extern__ char *
+mcache_dump_mca(mcache_audit_t *mca)
+{
+	if (mca_dump_buf == NULL)
+		return (NULL);
+
+	snprintf(mca_dump_buf, DUMP_MCA_BUF_SIZE,
+	    "mca %p: addr %p, cache %p (%s)\n"
+	    "last transaction; thread %p, saved PC stack (%d deep):\n"
+	    "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
+	    "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
+	    "previous transaction; thread %p, saved PC stack (%d deep):\n"
+	    "\t%p, %p, %p, %p, %p, %p, %p, %p\n"
+	    "\t%p, %p, %p, %p, %p, %p, %p, %p\n",
+	    mca, mca->mca_addr, mca->mca_cache,
+	    mca->mca_cache ? mca->mca_cache->mc_name : "?",
+	    mca->mca_thread, mca->mca_depth,
+	    mca->mca_stack[0], mca->mca_stack[1], mca->mca_stack[2],
+	    mca->mca_stack[3], mca->mca_stack[4], mca->mca_stack[5],
+	    mca->mca_stack[6], mca->mca_stack[7], mca->mca_stack[8],
+	    mca->mca_stack[9], mca->mca_stack[10], mca->mca_stack[11],
+	    mca->mca_stack[12], mca->mca_stack[13], mca->mca_stack[14],
+	    mca->mca_stack[15],
+	    mca->mca_pthread, mca->mca_pdepth,
+	    mca->mca_pstack[0], mca->mca_pstack[1], mca->mca_pstack[2],
+	    mca->mca_pstack[3], mca->mca_pstack[4], mca->mca_pstack[5],
+	    mca->mca_pstack[6], mca->mca_pstack[7], mca->mca_pstack[8],
+	    mca->mca_pstack[9], mca->mca_pstack[10], mca->mca_pstack[11],
+	    mca->mca_pstack[12], mca->mca_pstack[13], mca->mca_pstack[14],
+	    mca->mca_pstack[15]);
+
+	return (mca_dump_buf);
+}
+
+__private_extern__ void
+mcache_audit_panic(mcache_audit_t *mca, void *addr, size_t offset,
+    int64_t expected, int64_t got)
+{
+	if (mca == NULL) {
+		panic("mcache_audit: buffer %p modified after free at "
+		    "offset 0x%lx (0x%llx instead of 0x%llx)\n", addr,
+		    offset, got, expected);
+		/* NOTREACHED */
+	}
+
+	panic("mcache_audit: buffer %p modified after free at offset 0x%lx "
+	    "(0x%llx instead of 0x%llx)\n%s\n",
+	    addr, offset, got, expected, mcache_dump_mca(mca));
+	/* NOTREACHED */
+}
+
+__private_extern__ int
+assfail(const char *a, const char *f, int l)
+{
+	panic("assertion failed: %s, file: %s, line: %d", a, f, l);
+	return (0);
+}
diff --git a/bsd/kern/md5c.c b/bsd/kern/md5c.c
deleted file mode 100644
index 616779655..000000000
--- a/bsd/kern/md5c.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm
- *
- * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
- * rights reserved.
- *
- * License to copy and use this software is granted provided that it
- * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
- * Algorithm" in all material mentioning or referencing this software
- * or this function.
- *
- * License is also granted to make and use derivative works provided
- * that such works are identified as "derived from the RSA Data
- * Security, Inc. MD5 Message-Digest Algorithm" in all material
- * mentioning or referencing the derived work.
- *
- * RSA Data Security, Inc. makes no representations concerning either
- * the merchantability of this software or the suitability of this
- * software for any particular purpose. It is provided "as is"
- * without express or implied warranty of any kind.
- *
- * These notices must be retained in any copies of any part of this
- * documentation and/or software.
- *
- * $Id: md5c.c,v 1.2.4880.1 2005/06/24 01:47:07 lindak Exp $
- *
- * This code is the same as the code published by RSA Inc.  It has been
- * edited for clarity and style only.
- */
-
-#include <sys/types.h>
-
-#ifdef KERNEL
-#include <sys/systm.h>
-#else
-#include <string.h>
-#endif
-
-#include <sys/md5.h>
-
-
-#ifdef KERNEL
-#define memset(x,y,z)	bzero(x,z);
-#define memcpy(x,y,z)	bcopy(y, x, z)
-#endif
-
-#if defined(__i386__) || defined(__alpha__)
-#define Encode memcpy
-#define Decode memcpy
-#else /* __i386__ */
-
-/*
- * Encodes input (u_int32_t) into output (unsigned char). Assumes len is
- * a multiple of 4.
- */
-
-/* XXX not prototyped, and not compatible with memcpy(). */
-static void
-Encode (output, input, len)
-	unsigned char *output;
-	u_int32_t *input;
-	unsigned int len;
-{
-	unsigned int i, j;
-
-	for (i = 0, j = 0; j < len; i++, j += 4) {
-		output[j] = (unsigned char)(input[i] & 0xff);
-		output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
-		output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
-		output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
-	}
-}
-
-/*
- * Decodes input (unsigned char) into output (u_int32_t). Assumes len is
- * a multiple of 4.
- */
-
-static void
-Decode (output, input, len)
-	u_int32_t *output;
-	const unsigned char *input;
-	unsigned int len;
-{
-	unsigned int i, j;
-
-	for (i = 0, j = 0; j < len; i++, j += 4)
-		output[i] = ((u_int32_t)input[j]) | (((u_int32_t)input[j+1]) << 8) |
-		    (((u_int32_t)input[j+2]) << 16) | (((u_int32_t)input[j+3]) << 24);
-}
-#endif /* i386 */
-
-static unsigned char PADDING[64] = {
-  0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-};
-
-/* F, G, H and I are basic MD5 functions. */
-#define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
-#define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
-#define H(x, y, z) ((x) ^ (y) ^ (z))
-#define I(x, y, z) ((y) ^ ((x) | (~z)))
-
-/* ROTATE_LEFT rotates x left n bits. */
-#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
-
-/*
- * FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
- * Rotation is separate from addition to prevent recomputation.
- */
-#define FF(a, b, c, d, x, s, ac) { \
-	(a) += F ((b), (c), (d)) + (x) + (u_int32_t)(ac); \
-	(a) = ROTATE_LEFT ((a), (s)); \
-	(a) += (b); \
-	}
-#define GG(a, b, c, d, x, s, ac) { \
-	(a) += G ((b), (c), (d)) + (x) + (u_int32_t)(ac); \
-	(a) = ROTATE_LEFT ((a), (s)); \
-	(a) += (b); \
-	}
-#define HH(a, b, c, d, x, s, ac) { \
-	(a) += H ((b), (c), (d)) + (x) + (u_int32_t)(ac); \
-	(a) = ROTATE_LEFT ((a), (s)); \
-	(a) += (b); \
-	}
-#define II(a, b, c, d, x, s, ac) { \
-	(a) += I ((b), (c), (d)) + (x) + (u_int32_t)(ac); \
-	(a) = ROTATE_LEFT ((a), (s)); \
-	(a) += (b); \
-	}
-
-/* MD5 initialization. Begins an MD5 operation, writing a new context. */
-
-void
-MD5Init (context)
-	MD5_CTX *context;
-{
-
-	context->count[0] = context->count[1] = 0;
-
-	/* Load magic initialization constants.  */
-	context->state[0] = 0x67452301;
-	context->state[1] = 0xefcdab89;
-	context->state[2] = 0x98badcfe;
-	context->state[3] = 0x10325476;
-}
-
-/* 
- * MD5 block update operation. Continues an MD5 message-digest
- * operation, processing another message block, and updating the
- * context.
- */
-
-void
-MD5Update (context, input, inputLen)
-	MD5_CTX *context;
-	const unsigned char *input;
-	unsigned int inputLen;
-{
-	unsigned int i, index, partLen;
-
-	/* Compute number of bytes mod 64 */
-	index = (unsigned int)((context->count[0] >> 3) & 0x3F);
-
-	/* Update number of bits */
-	if ((context->count[0] += ((u_int32_t)inputLen << 3))
-	    < ((u_int32_t)inputLen << 3))
-		context->count[1]++;
-	context->count[1] += ((u_int32_t)inputLen >> 29);
-
-	partLen = 64 - index;
-
-	/* Transform as many times as possible. */
-	if (inputLen >= partLen) {
-		memcpy((void *)&context->buffer[index], (const void *)input,
-		    partLen);
-		MD5Transform (context->state, context->buffer);
-
-		for (i = partLen; i + 63 < inputLen; i += 64)
-			MD5Transform (context->state, &input[i]);
-
-		index = 0;
-	}
-	else
-		i = 0;
-
-	/* Buffer remaining input */
-	memcpy ((void *)&context->buffer[index], (const void *)&input[i],
-	    inputLen-i);
-}
-
-/*
- * MD5 padding. Adds padding followed by original length.
- */
-
-void
-MD5Pad (context)
-	MD5_CTX *context;
-{
-	unsigned char bits[8];
-	unsigned int index, padLen;
-
-	/* Save number of bits */
-	Encode (bits, context->count, 8);
-
-	/* Pad out to 56 mod 64. */
-	index = (unsigned int)((context->count[0] >> 3) & 0x3f);
-	padLen = (index < 56) ? (56 - index) : (120 - index);
-	MD5Update (context, PADDING, padLen);
-
-	/* Append length (before padding) */
-	MD5Update (context, bits, 8);
-}
-
-/*
- * MD5 finalization. Ends an MD5 message-digest operation, writing the
- * the message digest and zeroizing the context.
- */
-
-void
-MD5Final (digest, context)
-	unsigned char digest[16];
-	MD5_CTX *context;
-{
-	/* Do padding. */
-	MD5Pad (context);
-
-	/* Store state in digest */
-	Encode (digest, context->state, 16);
-
-	/* Zeroize sensitive information. */
-	memset ((void *)context, 0, sizeof (*context));
-}
-
-/* MD5 basic transformation. Transforms state based on block. */
-
-void
-MD5Transform (state, block)
-	u_int32_t state[4];
-	const unsigned char block[64];
-{
-	u_int32_t a = state[0], b = state[1], c = state[2], d = state[3], x[16];
-
-	Decode (x, block, 64);
-
-	/* Round 1 */
-#define S11 7
-#define S12 12
-#define S13 17
-#define S14 22
-	FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
-	FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
-	FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
-	FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
-	FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
-	FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
-	FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
-	FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
-	FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
-	FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
-	FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
-	FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
-	FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
-	FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
-	FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
-	FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
-
-	/* Round 2 */
-#define S21 5
-#define S22 9
-#define S23 14
-#define S24 20
-	GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
-	GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
-	GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
-	GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
-	GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
-	GG (d, a, b, c, x[10], S22,  0x2441453); /* 22 */
-	GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
-	GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
-	GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
-	GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
-	GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
-	GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
-	GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
-	GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
-	GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
-	GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
-
-	/* Round 3 */
-#define S31 4
-#define S32 11
-#define S33 16
-#define S34 23
-	HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
-	HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
-	HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
-	HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
-	HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
-	HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
-	HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
-	HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
-	HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
-	HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
-	HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
-	HH (b, c, d, a, x[ 6], S34,  0x4881d05); /* 44 */
-	HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
-	HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
-	HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
-	HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
-
-	/* Round 4 */
-#define S41 6
-#define S42 10
-#define S43 15
-#define S44 21
-	II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
-	II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
-	II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
-	II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
-	II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
-	II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
-	II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
-	II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
-	II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
-	II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
-	II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
-	II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
-	II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
-	II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
-	II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
-	II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
-
-	state[0] += a;
-	state[1] += b;
-	state[2] += c;
-	state[3] += d;
-
-	/* Zeroize sensitive information. */
-	memset ((void *)x, 0, sizeof (x));
-}
diff --git a/bsd/kern/netboot.c b/bsd/kern/netboot.c
index b034b9e87..3f2c11a86 100644
--- a/bsd/kern/netboot.c
+++ b/bsd/kern/netboot.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2001-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2001-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -39,6 +45,7 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/reboot.h>
+#include <sys/kauth.h>
 #include <net/if.h>
 #include <net/if_dl.h>
 #include <net/if_types.h>
@@ -46,18 +53,16 @@
 #include <netinet/in.h>
 #include <netinet/if_ether.h>
 #include <netinet/dhcp_options.h>
-#include <pexpert/pexpert.h>
+#include <netinet/in_dhcp.h>
 
 #include <kern/kern_types.h>
 #include <kern/kalloc.h>
 
+#include <pexpert/pexpert.h>
+
 //#include <libkern/libkern.h>
 extern struct filedesc 	filedesc0;
 
-extern int	strncmp(const char *,const char *, size_t);
-extern unsigned long strtoul(const char *, char **, int);
-extern char *	strchr(const char *str, int ch);
-
 extern int 	nfs_mountroot(void); 	/* nfs_vfsops.c */
 extern int (*mountroot)(void);
 
@@ -84,12 +89,6 @@ extern int di_root_image(const char *path, char devname[], dev_t *dev_p);
 #define BSDP_RESPONSE	"bsdp-response"
 #define DHCP_RESPONSE	"dhcp-response"
 
-extern int 
-bootp(struct ifnet * ifp, struct in_addr * iaddr_p, int max_retry,
-      struct in_addr * netmask_p, struct in_addr * router_p,
-      struct proc * procp);
-
-
 /* forward declarations */
 int	inet_aton(char * cp, struct in_addr * pin);
 
@@ -97,7 +96,7 @@ boolean_t	netboot_iaddr(struct in_addr * iaddr_p);
 boolean_t	netboot_rootpath(struct in_addr * server_ip,
 				 char * name, int name_len, 
 				 char * path, int path_len);
-int	netboot_setup(struct proc * p);
+int	netboot_setup(void);
 int	netboot_mountroot(void);
 int	netboot_root(void);
 
@@ -132,7 +131,7 @@ struct netboot_info {
 int
 inet_aton(char * cp, struct in_addr * pin)
 {
-    u_char * b = (char *)pin;
+    u_char * b = (u_char *)pin;
     int	   i;
     char * p;
 
@@ -162,7 +161,7 @@ inet_aton(char * cp, struct in_addr * pin)
  *   "17.202.16.17:seaport:/release/.images/Image9/CurrentHera"
  */
 static __inline__ boolean_t
-parse_booter_path(char * path, struct in_addr * iaddr_p, char * * host,
+parse_booter_path(char * path, struct in_addr * iaddr_p, char const * * host,
 		  char * * mount_dir, char * * image_path)
 {
     char *	start;
@@ -249,7 +248,7 @@ find_colon(char * str)
  * nfs:17.202.42.112:/Volumes/Foo\:/Library/NetBoot/NetBootSP0:Jaguar/Jaguar.dmg
  */
 static __inline__ boolean_t
-parse_netboot_path(char * path, struct in_addr * iaddr_p, char * * host,
+parse_netboot_path(char * path, struct in_addr * iaddr_p, char const * * host,
 		   char * * mount_dir, char * * image_path)
 {
 	static char	tmp[MAX_IPv4_STR_LEN];	/* Danger - not thread safe */
@@ -291,7 +290,7 @@ parse_netboot_path(char * path, struct in_addr * iaddr_p, char * * host,
 }
 
 static boolean_t
-parse_image_path(char * path, struct in_addr * iaddr_p, char * * host,
+parse_image_path(char * path, struct in_addr * iaddr_p, char const * * host,
 		 char * * mount_dir, char * * image_path)
 {
     if (path[0] >= '0' && path[0] <= '9') {
@@ -328,16 +327,16 @@ get_root_path(char * root_path)
     if (pkt != NULL) {
 	int			len;
 	dhcpol_t 		options;
-	char *			path;
-	struct dhcp *		reply;
+	const char *		path;
+	const struct dhcp *	reply;
 
-	reply = (struct dhcp *)pkt;
-	(void)dhcpol_parse_packet(&options, reply, pkt_len, NULL);
+	reply = (const struct dhcp *)pkt;
+	(void)dhcpol_parse_packet(&options, reply, pkt_len);
 
-	path = (char *)dhcpol_find(&options, 
-				   dhcptag_root_path_e, &len, NULL);
+	path = (const char *)dhcpol_find(&options, 
+					 dhcptag_root_path_e, &len, NULL);
 	if (path) {
-	    bcopy(path, root_path, len);
+	    memcpy(root_path, path, len);
 	    root_path[len] = '\0';
 	    found = TRUE;
 	}
@@ -358,7 +357,7 @@ netboot_info_init(struct in_addr iaddr)
     MALLOC_ZONE(vndevice, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
     if (vndevice == NULL)
     	panic("netboot_info_init: M_NAMEI zone exhausted");
-    if (PE_parse_boot_arg("vndevice", vndevice) == TRUE) {
+    if (PE_parse_boot_argn("vndevice", vndevice, MAXPATHLEN) == TRUE) {
 	use_hdix = FALSE;
     }
     FREE_ZONE(vndevice, MAXPATHLEN, M_NAMEI);
@@ -373,10 +372,10 @@ netboot_info_init(struct in_addr iaddr)
     MALLOC_ZONE(root_path, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
     if (root_path  == NULL)
     	panic("netboot_info_init: M_NAMEI zone exhausted");
-    if (PE_parse_boot_arg("rp", root_path) == TRUE
-	|| PE_parse_boot_arg("rootpath", root_path) == TRUE
+    if (PE_parse_boot_argn("rp", root_path, MAXPATHLEN) == TRUE
+	|| PE_parse_boot_argn("rootpath", root_path, MAXPATHLEN) == TRUE
 	|| get_root_path(root_path) == TRUE) {
-	char * server_name = NULL;
+	const char * server_name = NULL;
 	char * mount_point = NULL;
 	char * image_path = NULL;
 	struct in_addr 	server_ip;
@@ -389,8 +388,8 @@ netboot_info_init(struct in_addr iaddr)
 	    info->server_name = (char *)kalloc(info->server_name_length);
 	    info->mount_point_length = strlen(mount_point) + 1;
 	    info->mount_point = (char *)kalloc(info->mount_point_length);
-	    strcpy(info->server_name, server_name);
-	    strcpy(info->mount_point, mount_point);
+	    strlcpy(info->server_name, server_name, info->server_name_length);
+	    strlcpy(info->mount_point, mount_point, info->mount_point_length);
 	    
 	    printf("Server %s Mount %s", 
 		   server_name, info->mount_point);
@@ -404,11 +403,12 @@ netboot_info_init(struct in_addr iaddr)
 		}
 		info->image_path = (char *)kalloc(info->image_path_length);
 		if (needs_slash) {
-		    info->image_path[0] = '/';
-		    strcpy(info->image_path + 1, image_path);
-		}
-		else {
-		    strcpy(info->image_path, image_path);
+			info->image_path[0] = '/';
+			strlcpy(info->image_path + 1, image_path,
+					info->image_path_length - 1);
+		} else {
+			strlcpy(info->image_path, image_path,
+					info->image_path_length);
 		}
 		printf(" Image %s", info->image_path);
 	    }
@@ -421,7 +421,7 @@ netboot_info_init(struct in_addr iaddr)
 	    info->use_hdix = TRUE;
 	    info->image_path_length = strlen(root_path) + 1;
 	    info->image_path = (char *)kalloc(info->image_path_length);
-	    strcpy(info->image_path, root_path);
+	    strlcpy(info->image_path, root_path, info->image_path_length);
 	}	    
 	else {
 	    printf("netboot: root path uses unrecognized format\n");
@@ -481,8 +481,8 @@ netboot_rootpath(struct in_addr * server_ip,
 	       path_len, S_netboot_info_p->mount_point_length);
 	return (FALSE);
     }
-    strcpy(path, S_netboot_info_p->mount_point);
-    strncpy(name, S_netboot_info_p->server_name, name_len);
+    strlcpy(path, S_netboot_info_p->mount_point, path_len);
+    strlcpy(name, S_netboot_info_p->server_name, name_len);
     *server_ip = S_netboot_info_p->server_ip;
     return (TRUE);
 }
@@ -512,21 +512,21 @@ get_ip_parameters(struct in_addr * iaddr_p, struct in_addr * netmask_p,
 	}
     }
     if (pkt != NULL) {
-	struct in_addr *	ip;
+	const struct in_addr *	ip;
 	int			len;
 	dhcpol_t 		options;
-	struct dhcp *		reply;
+	const struct dhcp *	reply;
 
-	reply = (struct dhcp *)pkt;
-	(void)dhcpol_parse_packet(&options, reply, pkt_len, NULL);
+	reply = (const struct dhcp *)pkt;
+	(void)dhcpol_parse_packet(&options, reply, pkt_len);
 	*iaddr_p = reply->dp_yiaddr;
-	ip = (struct in_addr *)
+	ip = (const struct in_addr *)
 	    dhcpol_find(&options, 
 			dhcptag_subnet_mask_e, &len, NULL);
 	if (ip) {
 	    *netmask_p = *ip;
 	}
-	ip = (struct in_addr *)
+	ip = (const struct in_addr *)
 	    dhcpol_find(&options, dhcptag_router_e, &len, NULL);
 	if (ip) {
 	    *router_p = *ip;
@@ -536,35 +536,6 @@ get_ip_parameters(struct in_addr * iaddr_p, struct in_addr * netmask_p,
     return (pkt != NULL);
 }
 
-static int
-inet_aifaddr(struct socket * so, char * name, const struct in_addr * addr, 
-	     const struct in_addr * mask,
-	     const struct in_addr * broadcast)
-{
-    struct sockaddr	blank_sin;
-    struct ifaliasreq	ifra;
-
-    bzero(&blank_sin, sizeof(blank_sin));
-    blank_sin.sa_len = sizeof(blank_sin);
-    blank_sin.sa_family = AF_INET;
-
-    bzero(&ifra, sizeof(ifra));
-    strncpy(ifra.ifra_name, name, sizeof(ifra.ifra_name));
-    if (addr) {
-	ifra.ifra_addr = blank_sin;
-	((struct sockaddr_in *)&ifra.ifra_addr)->sin_addr = *addr;
-    }
-    if (mask) {
-	ifra.ifra_mask = blank_sin;
-	((struct sockaddr_in *)&ifra.ifra_mask)->sin_addr = *mask;
-    }
-    if (broadcast) {
-	ifra.ifra_broadaddr = blank_sin;
-	((struct sockaddr_in *)&ifra.ifra_broadaddr)->sin_addr = *broadcast;
-    }
-    return (ifioctl(so, SIOCAIFADDR, (caddr_t)&ifra, current_proc()));
-}
-
 static int
 route_cmd(int cmd, struct in_addr d, struct in_addr g, 
 	  struct in_addr m, u_long more_flags)
@@ -624,7 +595,7 @@ find_interface(void)
     struct ifnet *		ifp = NULL;
 
     if (rootdevice[0]) {
-		ifp = ifunit(rootdevice);
+		ifp = ifunit((char *)rootdevice);
     }
     if (ifp == NULL) {
 		ifnet_head_lock_shared();
@@ -644,14 +615,13 @@ netboot_mountroot(void)
     struct ifreq 		ifr;
     struct ifnet *		ifp;
     struct in_addr		netmask = { 0 };
-    struct proc *		procp = current_proc();
+    proc_t			procp = current_proc();
     struct in_addr		router = { 0 };
     struct socket *		so = NULL;
     unsigned int		try;
 
     bzero(&ifr, sizeof(ifr));
 
-
     /* find the interface */
     ifp = find_interface();
     if (ifp == NULL) {
@@ -659,7 +629,8 @@ netboot_mountroot(void)
 	error = ENXIO;
 	goto failed;
     }
-    sprintf(ifr.ifr_name, "%s%d", ifp->if_name, ifp->if_unit);
+    snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", ifp->if_name,
+	     ifp->if_unit);
     printf("netboot: using network interface '%s'\n", ifr.ifr_name);
 
     /* bring it up */
@@ -676,10 +647,10 @@ netboot_mountroot(void)
 
     /* grab information from the registry */
     if (get_ip_parameters(&iaddr, &netmask, &router) == FALSE) {
-	/* use BOOTP to retrieve IP address, netmask and router */
-	error = bootp(ifp, &iaddr, 32, &netmask, &router, procp);
+	/* use DHCP to retrieve IP address, netmask and router */
+	error = dhcp(ifp, &iaddr, 64, &netmask, &router, procp);
 	if (error) {
-	    printf("netboot: BOOTP failed %d\n", error);
+	    printf("netboot: DHCP failed %d\n", error);
 	    goto failed;
 	}
     }
@@ -752,7 +723,7 @@ netboot_mountroot(void)
 	}
 	break;
     case kNetBootImageTypeHTTP:
-	error = netboot_setup(procp);
+	error = netboot_setup();
 	break;
     }
     if (error == 0) {
@@ -770,7 +741,7 @@ failed:
 }
 
 int
-netboot_setup(struct proc * p)
+netboot_setup()
 {
     dev_t 	dev;
     int 	error = 0;
@@ -782,7 +753,7 @@ netboot_setup(struct proc * p)
     if (S_netboot_info_p->use_hdix) {
 	printf("netboot_setup: calling di_root_image\n");
 	error = di_root_image(S_netboot_info_p->image_path, 
-			      rootdevice, &dev);
+			      (char *)rootdevice, &dev);
 	if (error) {
 	    printf("netboot_setup: di_root_image: failed %d\n", error);
 	    goto done;
@@ -791,7 +762,7 @@ netboot_setup(struct proc * p)
     else {
 	printf("netboot_setup: calling vndevice_root_image\n");
 	error = vndevice_root_image(S_netboot_info_p->image_path, 
-				    rootdevice, &dev);
+				    (char *)rootdevice, &dev);
 	if (error) {
 	    printf("netboot_setup: vndevice_root_image: failed %d\n", error);
 	    goto done;
@@ -804,13 +775,9 @@ netboot_setup(struct proc * p)
     if (error == 0 && rootvnode != NULL) {
         struct vnode *tvp;
         struct vnode *newdp;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
 	/* Get the vnode for '/'.  Set fdp->fd_fd.fd_cdir to reference it. */
-	if (VFS_ROOT(mountlist.tqh_last, &newdp, &context))
+	if (VFS_ROOT(TAILQ_LAST(&mountlist,mntlist), &newdp, vfs_context_kernel()))
 		panic("netboot_setup: cannot find root vnode");
 	vnode_ref(newdp);
 	vnode_put(newdp);
@@ -822,7 +789,6 @@ netboot_setup(struct proc * p)
 	TAILQ_REMOVE(&mountlist, TAILQ_FIRST(&mountlist), mnt_list);
 	mount_list_unlock();
 	mountlist.tqh_first->mnt_flag |= MNT_ROOTFS;
-	kauth_cred_unref(&context.vc_ucred);
     }
  done:
     netboot_info_free(&S_netboot_info_p);
diff --git a/bsd/kern/posix_sem.c b/bsd/kern/posix_sem.c
index 4f0811968..63bbd878e 100644
--- a/bsd/kern/posix_sem.c
+++ b/bsd/kern/posix_sem.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
@@ -34,6 +40,12 @@
  *	Created for MacOSX
  *
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/cdefs.h>
 #include <sys/param.h>
@@ -54,6 +66,11 @@
 #include <sys/sysproto.h>
 #include <sys/proc_info.h>
 
+#if CONFIG_MACF
+#include <sys/vnode_internal.h>
+#include <security/mac_framework.h>
+#endif
+
 #include <bsm/audit_kernel.h>
 
 #include <mach/mach_types.h>
@@ -66,9 +83,6 @@
 #include <kern/clock.h>
 #include <mach/kern_return.h>
 
-#if KTRACE
-#include <sys/ktrace.h>
-#endif
 
 #define f_flag f_fglob->fg_flag
 #define f_type f_fglob->fg_type
@@ -87,7 +101,8 @@ struct pseminfo {
 	gid_t		psem_gid;
 	char		psem_name[PSEMNAMLEN + 1];	/* segment name */
 	semaphore_t	psem_semobject;
-	struct proc *	sem_proc;
+	proc_t		sem_proc;
+	struct label *	psem_label;
 };
 #define PSEMINFO_NULL (struct pseminfo *)0
 
@@ -140,8 +155,8 @@ u_long	psemhash;				/* size of hash table - 1 */
 long	psemnument;			/* number of cache entries allocated */
 long	posix_sem_max = 10000;		/* tunable for max POSIX semaphores */
 					/* 10000 limits to ~1M of memory */
-SYSCTL_NODE(_kern, KERN_POSIX, posix, CTLFLAG_RW,  0, "Posix");
-SYSCTL_NODE(_kern_posix, OID_AUTO, sem, CTLFLAG_RW, 0, "Semaphores");
+SYSCTL_NODE(_kern, KERN_POSIX, posix, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Posix");
+SYSCTL_NODE(_kern_posix, OID_AUTO, sem, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Semaphores");
 SYSCTL_INT (_kern_posix_sem, OID_AUTO, max, CTLFLAG_RW, &posix_sem_max, 0, "max");
 
 struct psemstats psemstats;		/* cache effectiveness statistics */
@@ -152,18 +167,18 @@ static int psem_cache_search(struct pseminfo **,
 static int psem_delete(struct pseminfo * pinfo);
 
 static int psem_read (struct fileproc *fp, struct uio *uio,
-			    kauth_cred_t cred, int flags, struct proc *p);
+			    int flags, vfs_context_t ctx);
 static int psem_write (struct fileproc *fp, struct uio *uio,
-			    kauth_cred_t cred, int flags, struct proc *p);
+			    int flags, vfs_context_t ctx);
 static int psem_ioctl (struct fileproc *fp, u_long com,
-			    caddr_t data, struct proc *p);
-static int psem_select (struct fileproc *fp, int which, void *wql, struct proc *p);
-static int psem_closefile (struct fileglob *fp, struct proc *p);
+			    caddr_t data, vfs_context_t ctx);
+static int psem_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx);
+static int psem_closefile (struct fileglob *fp, vfs_context_t ctx);
 
-static int psem_kqfilter (struct fileproc *fp, struct knote *kn, struct proc *p);
+static int psem_kqfilter (struct fileproc *fp, struct knote *kn, vfs_context_t ctx);
 
 struct 	fileops psemops =
-	{ psem_read, psem_write, psem_ioctl, psem_select, psem_closefile, psem_kqfilter, 0 };
+	{ psem_read, psem_write, psem_ioctl, psem_select, psem_closefile, psem_kqfilter, NULL };
 
 
 static lck_grp_t       *psx_sem_subsys_lck_grp;
@@ -200,10 +215,8 @@ psem_lock_init( void )
  */
 
 static int
-psem_cache_search(psemp, pnp, pcache)
-	struct pseminfo **psemp;
-	struct psemname *pnp;
-	struct psemcache **pcache;
+psem_cache_search(struct pseminfo **psemp, struct psemname *pnp,
+		  struct psemcache **pcache)
 {
 	struct psemcache *pcp, *nnp;
 	struct psemhashhead *pcpp;
@@ -254,7 +267,7 @@ psem_cache_add(struct pseminfo *psemp, struct psemname *pnp, struct psemcache *p
 	struct psemcache *dpcp;
 
 #if DIAGNOSTIC
-	if (pnp->psem_namelen > NCHNAMLEN)
+	if (pnp->psem_namelen > PSEMNAMLEN)
 		panic("cache_enter: name too long");
 #endif
 
@@ -295,7 +308,7 @@ psem_cache_add(struct pseminfo *psemp, struct psemname *pnp, struct psemcache *p
 void
 psem_cache_init(void)
 {
-	psemhashtbl = hashinit(desiredvnodes, M_SHM, &psemhash);
+	psemhashtbl = hashinit(posix_sem_max / 2, M_SHM, &psemhash);
 }
 
 static void
@@ -308,7 +321,7 @@ psem_cache_delete(struct psemcache *pcp)
 		panic("namecache purge le_next");
 #endif /* DIAGNOSTIC */
 	LIST_REMOVE(pcp, psem_hash);
-	pcp->psem_hash.le_prev = 0;	
+	pcp->psem_hash.le_prev = NULL;	
 	psemnument--;
 }
 
@@ -335,7 +348,7 @@ psem_cache_purge(void)
 #endif	/* NOT_USED */
 
 int
-sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
+sem_open(proc_t p, struct sem_open_args *uap, user_addr_t *retval)
 {
 	struct fileproc *fp;
 	size_t i;
@@ -343,7 +356,7 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 	int indx, error;
 	struct psemname nd;
 	struct pseminfo *pinfo;
-	struct psemcache *pcp;
+	struct psemcache *pcp = NULL;	/* protected by !incache */
 	char * pnbuf;
 	char * nameptr;
 	char * cp;
@@ -396,17 +409,12 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 	nameptr = pnbuf;
 	nd.psem_nameptr = nameptr;
 	nd.psem_namelen = plen;
-	nd. psem_hash =0;
+	nd.psem_hash = 0;
 
         for (cp = nameptr, i=1; *cp != 0 && i <= plen; i++, cp++) {
                nd.psem_hash += (unsigned char)*cp * i;
 	}
 
-#if KTRACE
-	if (KTRPOINT(p, KTR_NAMEI))
-		ktrnamei(p->p_tracep, nameptr);
-#endif
-	
 	PSEM_SUBSYS_LOCK();
 	error = psem_cache_search(&pinfo, &nd, &pcache);
 
@@ -423,7 +431,7 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 	fmode = FFLAGS(uap->oflag);
 
 	PSEM_SUBSYS_UNLOCK();
-	error = falloc(p, &nfp, &indx);
+	error = falloc(p, &nfp, &indx, vfs_context_current());
 	if (error)
 		goto bad;
 
@@ -449,7 +457,7 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 	}
 
 	if ( (fmode & O_CREAT) ) {
-		if((value < 0) && (value > SEM_VALUE_MAX)) {
+		if((value < 0) || (value > SEM_VALUE_MAX)) {
 			PSEM_SUBSYS_UNLOCK();
 			error = EINVAL;
 			goto bad1;
@@ -460,6 +468,9 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 			error = ENOSPC;
 			goto bad1;
 		}
+#if CONFIG_MACF
+		mac_posixsem_label_init(pinfo);
+#endif
 		PSEM_SUBSYS_LOCK();
 
 		pinfo_alloc = 1;
@@ -479,6 +490,14 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 		pinfo->psem_flags &= ~PSEM_DEFINED;
 		pinfo->psem_flags |= PSEM_ALLOCATED;
 		pinfo->sem_proc = p;
+#if CONFIG_MACF
+		error = mac_posixsem_check_create(kauth_cred_get(), nameptr);
+		if (error) {
+			PSEM_SUBSYS_UNLOCK();
+			goto bad2;
+		}
+		mac_posixsem_label_associate(kauth_cred_get(), pinfo, nameptr);
+#endif
 	} else {
 		/* semaphore should exist as it is without  O_CREAT */
 		if (!incache) {
@@ -493,6 +512,13 @@ sem_open(struct proc *p, struct sem_open_args *uap, user_addr_t *retval)
 		}	
 		AUDIT_ARG(posix_ipc_perm, pinfo->psem_uid,
 			pinfo->psem_gid, pinfo->psem_mode);
+#if CONFIG_MACF
+		error = mac_posixsem_check_open(kauth_cred_get(), pinfo);
+		if (error) {
+			PSEM_SUBSYS_UNLOCK();
+			goto bad1;
+		}
+#endif
 		if ( (error = psem_access(pinfo, fmode, kauth_cred_get())) ) {
 			PSEM_SUBSYS_UNLOCK();
 			goto bad1;
@@ -547,8 +573,10 @@ bad3:
 	switch (kret) {
 	case KERN_RESOURCE_SHORTAGE:
 		error = ENOMEM;
+		break;
 	case KERN_PROTECTION_FAILURE:
 		error = EACCES;
+		break;
 	default:
 		error = EINVAL;
 	}
@@ -556,8 +584,12 @@ bad3:
 bad2:
 	FREE(pnode, M_SHM);
 bad1:
-	if (pinfo_alloc)
+	if (pinfo_alloc) {
+#if CONFIG_MACF
+		mac_posixsem_label_destroy(pinfo);
+#endif
 		FREE(pinfo, M_SHM);
+	}
 	fp_free(p, indx, nfp);
 bad:
 	FREE_ZONE(pnbuf, MAXPATHLEN, M_NAMEI);
@@ -606,7 +638,7 @@ psem_access(struct pseminfo *pinfo, int mode, kauth_cred_t cred)
 }
 
 int
-sem_unlink(__unused struct proc *p, struct sem_unlink_args *uap, __unused register_t *retval)
+sem_unlink(__unused proc_t p, struct sem_unlink_args *uap, __unused register_t *retval)
 {
 	size_t i;
 	int error=0;
@@ -676,6 +708,13 @@ sem_unlink(__unused struct proc *p, struct sem_unlink_args *uap, __unused regist
 		goto bad;
 	} else
 		incache = 1;
+#if CONFIG_MACF
+	error = mac_posixsem_check_unlink(kauth_cred_get(), pinfo, nameptr);
+	if (error) {
+		PSEM_SUBSYS_UNLOCK();
+		goto bad;
+	}
+#endif
 	if ( (error = psem_access(pinfo, pinfo->psem_mode, kauth_cred_get())) ) {
 		PSEM_SUBSYS_UNLOCK();
 		goto bad;
@@ -683,7 +722,8 @@ sem_unlink(__unused struct proc *p, struct sem_unlink_args *uap, __unused regist
 
 	if ((pinfo->psem_flags & (PSEM_DEFINED | PSEM_ALLOCATED))==0) {
 		PSEM_SUBSYS_UNLOCK();
-		return (EINVAL);
+		error = EINVAL;
+		goto bad;
 	}
 
 	if ( (pinfo->psem_flags & PSEM_INDELETE) ) {
@@ -714,7 +754,7 @@ bad:
 }
 
 int
-sem_close(struct proc *p, struct sem_close_args *uap, __unused register_t *retval)
+sem_close(proc_t p, struct sem_close_args *uap, __unused register_t *retval)
 {
 	int fd = CAST_DOWN(int,uap->sem);
 	struct fileproc *fp;
@@ -728,6 +768,7 @@ sem_close(struct proc *p, struct sem_close_args *uap, __unused register_t *retva
 		proc_fdunlock(p);
 		return(error);
 	}
+	fileproc_drain(p, fp);
 	fdrelse(p, fd);
 	error = closef_locked(fp, fp->f_fglob, p);
 	FREE_ZONE(fp, sizeof *fp, M_FILEPROC);
@@ -736,7 +777,14 @@ sem_close(struct proc *p, struct sem_close_args *uap, __unused register_t *retva
 }
 
 int
-sem_wait(struct proc *p, struct sem_wait_args *uap, __unused register_t *retval)
+sem_wait(proc_t p, struct sem_wait_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(sem_wait_nocancel(p, (struct sem_wait_nocancel_args *)uap, retval));
+}
+
+int
+sem_wait_nocancel(proc_t p, struct sem_wait_nocancel_args *uap, __unused register_t *retval)
 {
 	int fd = CAST_DOWN(int,uap->sem);
 	struct fileproc *fp;
@@ -764,7 +812,13 @@ sem_wait(struct proc *p, struct sem_wait_args *uap, __unused register_t *retval)
 		error = EINVAL;
 		goto out;
 	}
-
+#if CONFIG_MACF
+	error = mac_posixsem_check_wait(kauth_cred_get(), pinfo);
+	if (error) {
+		PSEM_SUBSYS_UNLOCK();
+		goto out;
+	}
+#endif
 	PSEM_SUBSYS_UNLOCK();
 	kret = semaphore_wait(pinfo->psem_semobject);
 	switch (kret) {
@@ -790,7 +844,7 @@ out:
 }
 
 int
-sem_trywait(struct proc *p, struct sem_trywait_args *uap, __unused register_t *retval)
+sem_trywait(proc_t p, struct sem_trywait_args *uap, __unused register_t *retval)
 {
 	int fd = CAST_DOWN(int,uap->sem);
 	struct fileproc *fp;
@@ -819,7 +873,13 @@ sem_trywait(struct proc *p, struct sem_trywait_args *uap, __unused register_t *r
 		error = EINVAL;
 		goto out;
 	}
-
+#if CONFIG_MACF
+	error = mac_posixsem_check_wait(kauth_cred_get(), pinfo);
+	if (error) {
+		PSEM_SUBSYS_UNLOCK();
+		goto out;
+	}
+#endif
 	PSEM_SUBSYS_UNLOCK();
 	wait_time.tv_sec = 0;
 	wait_time.tv_nsec = 0;
@@ -849,7 +909,7 @@ out:
 }
 
 int
-sem_post(struct proc *p, struct sem_post_args *uap, __unused register_t *retval)
+sem_post(proc_t p, struct sem_post_args *uap, __unused register_t *retval)
 {
 	int fd = CAST_DOWN(int,uap->sem);
 	struct fileproc *fp;
@@ -877,7 +937,13 @@ sem_post(struct proc *p, struct sem_post_args *uap, __unused register_t *retval)
 		error = EINVAL;
 		goto out;
 	}
-
+#if CONFIG_MACF
+	error = mac_posixsem_check_post(kauth_cred_get(), pinfo);
+	if (error) {
+		PSEM_SUBSYS_UNLOCK();
+		goto out;
+	}
+#endif
 	PSEM_SUBSYS_UNLOCK();
 	kret = semaphore_signal(pinfo->psem_semobject);
 	switch (kret) {
@@ -902,29 +968,28 @@ out:
 }
 
 int
-sem_init(__unused struct proc *p, __unused struct sem_init_args *uap, __unused register_t *retval)
+sem_init(__unused proc_t p, __unused struct sem_init_args *uap, __unused register_t *retval)
 {
 	return(ENOSYS);
 }
 
 int
-sem_destroy(__unused struct proc *p, __unused struct sem_destroy_args *uap, __unused register_t *retval)
+sem_destroy(__unused proc_t p, __unused struct sem_destroy_args *uap, __unused register_t *retval)
 {
 	return(ENOSYS);
 }
 
 int
-sem_getvalue(__unused struct proc *p, __unused struct sem_getvalue_args *uap, __unused register_t *retval)
+sem_getvalue(__unused proc_t p, __unused struct sem_getvalue_args *uap, __unused register_t *retval)
 {
 	return(ENOSYS);
 }
 
 static int
-psem_close(struct psemnode *pnode, __unused int flags, 
-		__unused kauth_cred_t cred, __unused struct proc *p)
+psem_close(struct psemnode *pnode, __unused int flags)
 {
 	int error=0;
-	register struct pseminfo *pinfo;
+	struct pseminfo *pinfo;
 
 	PSEM_SUBSYS_LOCK();
 	if ((pinfo = pnode->pinfo) == PSEMINFO_NULL){
@@ -957,15 +1022,15 @@ psem_close(struct psemnode *pnode, __unused int flags,
 }
 
 static int
-psem_closefile(fg, p)
-	struct fileglob *fg;
-	struct proc *p;
+psem_closefile(struct fileglob *fg, __unused vfs_context_t ctx)
 {
 	int error;
 
-	/* Not locked as psem_close is called only from here and is locked properly */
-	error =  psem_close(((struct psemnode *)fg->fg_data), fg->fg_flag,
-		fg->fg_cred, p);
+	/*
+	 * Not locked as psem_close is called only from here and is locked
+	 * properly
+	 */
+	error =  psem_close(((struct psemnode *)fg->fg_data), fg->fg_flag);
 
 	return(error);
 }
@@ -976,6 +1041,9 @@ psem_delete(struct pseminfo * pinfo)
 	kern_return_t kret;
 
 	kret = semaphore_destroy(kernel_task, pinfo->psem_semobject);
+#if CONFIG_MACF
+	mac_posixsem_label_destroy(pinfo);
+#endif
 
 	switch (kret) {
 	case KERN_INVALID_ADDRESS:
@@ -993,37 +1061,35 @@ psem_delete(struct pseminfo * pinfo)
 
 static int
 psem_read(__unused struct fileproc *fp, __unused struct uio *uio, 
-		  __unused kauth_cred_t cred, __unused int flags, 
-		  __unused struct proc *p)
+		  __unused int flags, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 psem_write(__unused struct fileproc *fp, __unused struct uio *uio, 
-		   __unused kauth_cred_t cred, __unused int flags, 
-		   __unused struct proc *p)
+		   __unused int flags, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 psem_ioctl(__unused struct fileproc *fp, __unused u_long com, 
-			__unused caddr_t data, __unused struct proc *p)
+			__unused caddr_t data, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 psem_select(__unused struct fileproc *fp, __unused int which, 
-			__unused void *wql, __unused struct proc *p)
+			__unused void *wql, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 psem_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, 
-				__unused struct proc *p)
+				__unused vfs_context_t ctx)
 {
 	return (ENOTSUP);
 }
@@ -1031,8 +1097,8 @@ psem_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn,
 int
 fill_pseminfo(struct psemnode *pnode, struct psem_info * info)
 {
-	register struct pseminfo *pinfo;
-	struct stat *sb;
+	struct pseminfo *pinfo;
+	struct vinfo_stat  *sb;
 
 	PSEM_SUBSYS_LOCK();
 	if ((pinfo = pnode->pinfo) == PSEMINFO_NULL){
@@ -1048,15 +1114,35 @@ fill_pseminfo(struct psemnode *pnode, struct psem_info * info)
 #endif
 
 	sb = &info->psem_stat;
-	bzero(sb, sizeof(struct stat));
+	bzero(sb, sizeof(struct vinfo_stat));
 
-    	sb->st_mode = pinfo->psem_mode;
-    	sb->st_uid = pinfo->psem_uid;
-    	sb->st_gid = pinfo->psem_gid;
-    	sb->st_size = pinfo->psem_usecount;
+    	sb->vst_mode = pinfo->psem_mode;
+    	sb->vst_uid = pinfo->psem_uid;
+    	sb->vst_gid = pinfo->psem_gid;
+    	sb->vst_size = pinfo->psem_usecount;
 	bcopy(&pinfo->psem_name[0], &info->psem_name[0], PSEMNAMLEN+1);
 
 	PSEM_SUBSYS_UNLOCK();
 	return(0);
 }
 
+#if CONFIG_MACF
+void
+psem_label_associate(struct fileproc *fp, struct vnode *vp, vfs_context_t ctx)
+{
+	struct psemnode *pnode;
+	struct pseminfo *psem;
+
+	PSEM_SUBSYS_LOCK();
+	pnode = (struct psemnode *)fp->f_fglob->fg_data;
+	if (pnode != NULL) {
+		psem = pnode->pinfo;
+		if (psem != NULL)
+			mac_posixsem_vnode_label_associate(
+				vfs_context_ucred(ctx), psem, psem->psem_label,
+				vp, vp->v_label);
+	}
+	PSEM_SUBSYS_UNLOCK();
+}
+#endif
+      
diff --git a/bsd/kern/posix_shm.c b/bsd/kern/posix_shm.c
index 4681099c2..7ad07ace0 100644
--- a/bsd/kern/posix_shm.c
+++ b/bsd/kern/posix_shm.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
@@ -34,6 +40,12 @@
  *	Created for MacOSX
  *
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/cdefs.h>
 #include <sys/param.h>
@@ -47,6 +59,7 @@
 #include <sys/mount.h>
 #include <sys/namei.h>
 #include <sys/vnode.h>
+#include <sys/vnode_internal.h>
 #include <sys/ioctl.h>
 #include <sys/tty.h>
 #include <sys/malloc.h>
@@ -54,9 +67,12 @@
 #include <sys/stat.h>
 #include <sys/sysproto.h>
 #include <sys/proc_info.h>
-
 #include <bsm/audit_kernel.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 #include <mach/mach_types.h>
 #include <mach/mach_vm.h>
 #include <mach/vm_map.h>
@@ -67,11 +83,6 @@
 
 #include <vm/vm_map.h>
 #include <vm/vm_protos.h>
-#include <vm/vm_shared_memory_server.h>
-
-#if KTRACE
-#include <sys/ktrace.h>
-#endif
 
 #define f_flag f_fglob->fg_flag
 #define f_type f_fglob->fg_type
@@ -82,7 +93,6 @@
 #define f_data f_fglob->fg_data
 #define	PSHMNAMLEN	31	/* maximum name segment length we bother with */
 
-
 struct pshminfo {
 	unsigned int	pshm_flags;
 	unsigned int	pshm_usecount;
@@ -95,8 +105,9 @@ struct pshminfo {
 #if DIAGNOSTIC
 	unsigned int 	pshm_readcount;
 	unsigned int 	pshm_writecount;
-	struct proc *	pshm_proc;
+	proc_t		pshm_proc;
 #endif /* DIAGNOSTIC */
+	struct label*	pshm_label;
 };
 #define PSHMINFO_NULL (struct pshminfo *)0
 
@@ -154,18 +165,18 @@ long	pshmnument;			/* number of cache entries allocated */
 struct pshmstats pshmstats;		/* cache effectiveness statistics */
 
 static int pshm_read (struct fileproc *fp, struct uio *uio,
-		    kauth_cred_t cred, int flags, struct proc *p);
+		    int flags, vfs_context_t ctx);
 static int pshm_write (struct fileproc *fp, struct uio *uio,
-		    kauth_cred_t cred, int flags, struct proc *p);
+		    int flags, vfs_context_t ctx);
 static int pshm_ioctl (struct fileproc *fp, u_long com,
-		    caddr_t data, struct proc *p);
-static int pshm_select (struct fileproc *fp, int which, void *wql, struct proc *p);
+		    caddr_t data, vfs_context_t ctx);
+static int pshm_select (struct fileproc *fp, int which, void *wql, vfs_context_t ctx);
 static int pshm_close(struct pshmnode *pnode);
-static int pshm_closefile (struct fileglob *fg, struct proc *p);
+static int pshm_closefile (struct fileglob *fg, vfs_context_t ctx);
 
-static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
+static int pshm_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx);
 
-int pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, struct proc *p);
+int pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, proc_t p);
 static int pshm_cache_add(struct pshminfo *pshmp, struct pshmname *pnp, struct pshmcache *pcp);
 static void pshm_cache_delete(struct pshmcache *pcp);
 #if NOT_USED
@@ -263,7 +274,7 @@ pshm_cache_add(struct pshminfo *pshmp, struct pshmname *pnp, struct pshmcache *p
 	struct pshmcache *dpcp;
 
 #if DIAGNOSTIC
-	if (pnp->pshm_namelen > NCHNAMLEN)
+	if (pnp->pshm_namelen > PSHMNAMLEN)
 		panic("cache_enter: name too long");
 #endif
 
@@ -303,7 +314,7 @@ pshm_cache_add(struct pshminfo *pshmp, struct pshmname *pnp, struct pshmcache *p
 void
 pshm_cache_init(void)
 {
-	pshmhashtbl = hashinit(desiredvnodes, M_SHM, &pshmhash);
+	pshmhashtbl = hashinit(desiredvnodes / 8, M_SHM, &pshmhash);
 }
 
 #if NOT_USED
@@ -344,7 +355,7 @@ pshm_cache_delete(struct pshmcache *pcp)
 
 
 int
-shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
+shm_open(proc_t p, struct shm_open_args *uap, register_t *retval)
 {
 	struct fileproc *fp;
 	size_t  i;
@@ -361,7 +372,7 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	int incache = 0;
 	struct pshmnode * pnode = PSHMNODE_NULL;
 	struct pshmcache * pcache = PSHMCACHE_NULL;
-	struct pshmcache *pcp;
+	struct pshmcache *pcp = NULL;	/* protected by !incache */
 	int pinfo_alloc=0;
 
 	AUDIT_ARG(fflags, uap->oflag);
@@ -394,7 +405,7 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 			error = EINVAL;
 			goto bad;
 		}
-        } else {
+	} else {
 		error = EINVAL;
 		goto bad;
 	}
@@ -406,15 +417,10 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	nd.pshm_namelen = plen;
 	nd. pshm_hash =0;
 
-        for (cp = nameptr, i=1; *cp != 0 && i <= plen; i++, cp++) {
-               nd.pshm_hash += (unsigned char)*cp * i;
+	for (cp = nameptr, i=1; *cp != 0 && i <= plen; i++, cp++) {
+		nd.pshm_hash += (unsigned char)*cp * i;
 	}
 
-#if KTRACE
-	if (KTRPOINT(p, KTR_NAMEI))
-		ktrnamei(p->p_tracep, nameptr);
-#endif
-	
 	PSHM_SUBSYS_LOCK();
 	error = pshm_cache_search(&pinfo, &nd, &pcache);
 
@@ -438,10 +444,10 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	/*
 	 * XXXXXXXXXX TBD XXXXXXXXXX
 	 * There is a race that existed with the funnels as well.
-     * Need to be fixed later
+	 * Need to be fixed later
 	 */
 	PSHM_SUBSYS_UNLOCK();
-	error = falloc(p, &nfp, &indx);
+	error = falloc(p, &nfp, &indx, vfs_context_current());
 	if (error ) 
 		goto bad;
 	PSHM_SUBSYS_LOCK();
@@ -453,48 +459,66 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	if (fmode & O_CREAT) {
 		if ((fmode & O_EXCL) && incache) {
 			AUDIT_ARG(posix_ipc_perm, pinfo->pshm_uid,
-				  pinfo->pshm_gid, pinfo->pshm_mode);
+					pinfo->pshm_gid, pinfo->pshm_mode);
 
 			/* shm obj exists and opened O_EXCL */
 #if notyet
-                        if (pinfo->pshm_flags & PSHM_INDELETE) {
-                        }
+			if (pinfo->pshm_flags & PSHM_INDELETE) {
+			}
 #endif 
-                        error = EEXIST;
-						PSHM_SUBSYS_UNLOCK();
-                        goto bad1;
-                } 
-                if (!incache) {
-					PSHM_SUBSYS_UNLOCK();
-                    /*  create a new one */
-                    MALLOC(pinfo, struct pshminfo *, sizeof(struct pshminfo), M_SHM, M_WAITOK|M_ZERO);
-		    		if (pinfo == NULL) {
-		    			error = ENOSPC;
-		    			goto bad1;
-		    		}
-					PSHM_SUBSYS_LOCK();
-					pinfo_alloc = 1;
-                    pinfo->pshm_flags = PSHM_DEFINED | PSHM_INCREATE;
-                    pinfo->pshm_usecount = 1; /* existence reference */
-                    pinfo->pshm_mode = cmode;
-                    pinfo->pshm_uid = kauth_cred_getuid(kauth_cred_get());
-                    pinfo->pshm_gid = kauth_cred_get()->cr_gid;
+			error = EEXIST;
+			PSHM_SUBSYS_UNLOCK();
+			goto bad1;
+		} 
+		if (!incache) {
+			PSHM_SUBSYS_UNLOCK();
+			/*  create a new one */
+			MALLOC(pinfo, struct pshminfo *, sizeof(struct pshminfo), M_SHM, M_WAITOK|M_ZERO);
+			if (pinfo == NULL) {
+				error = ENOSPC;
+				goto bad1;
+			}
+			PSHM_SUBSYS_LOCK();
+			pinfo_alloc = 1;
+			pinfo->pshm_flags = PSHM_DEFINED | PSHM_INCREATE;
+			pinfo->pshm_usecount = 1; /* existence reference */
+			pinfo->pshm_mode = cmode;
+			pinfo->pshm_uid = kauth_cred_getuid(kauth_cred_get());
+			pinfo->pshm_gid = kauth_cred_get()->cr_gid;
 			bcopy(pnbuf, &pinfo->pshm_name[0], PSHMNAMLEN);
 			pinfo->pshm_name[PSHMNAMLEN]=0;
-                } else {
-                    /*  already exists */
-                        if( pinfo->pshm_flags & PSHM_INDELETE) {
-							PSHM_SUBSYS_UNLOCK();
-                            error = ENOENT;
-                            goto bad1;
-                        }	
-						AUDIT_ARG(posix_ipc_perm, pinfo->pshm_uid,
-						pinfo->pshm_gid, pinfo->pshm_mode);
-                        if ( (error = pshm_access(pinfo, fmode, kauth_cred_get(), p)) ) {
-							PSHM_SUBSYS_UNLOCK();
-                            goto bad1;
-						}
-                }
+#if CONFIG_MACF
+			PSHM_SUBSYS_UNLOCK();
+			mac_posixshm_label_init(pinfo);
+			PSHM_SUBSYS_LOCK();
+			error = mac_posixshm_check_create(kauth_cred_get(), nameptr);
+			if (error) {
+				PSHM_SUBSYS_UNLOCK();
+				goto bad2;
+			}
+			mac_posixshm_label_associate(kauth_cred_get(), pinfo, nameptr);
+#endif
+		} else {
+			/*  already exists */
+			if( pinfo->pshm_flags & PSHM_INDELETE) {
+				PSHM_SUBSYS_UNLOCK();
+				error = ENOENT;
+				goto bad1;
+			}	
+			AUDIT_ARG(posix_ipc_perm, pinfo->pshm_uid,
+					pinfo->pshm_gid, pinfo->pshm_mode);
+#if CONFIG_MACF	
+			if ((error = mac_posixshm_check_open(
+							kauth_cred_get(), pinfo))) {
+				PSHM_SUBSYS_UNLOCK();
+				goto bad1;
+			}
+#endif
+			if ( (error = pshm_access(pinfo, fmode, kauth_cred_get(), p)) ) {
+				PSHM_SUBSYS_UNLOCK();
+				goto bad1;
+			}
+		}
 	} else {
 		if (!incache) {
 			/* O_CREAT  is not set and the shm obecj does not exist */
@@ -507,6 +531,14 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 			error = ENOENT;
 			goto bad1;
 		}	
+#if CONFIG_MACF	
+		if ((error = mac_posixshm_check_open(
+						kauth_cred_get(), pinfo))) {
+			PSHM_SUBSYS_UNLOCK();
+			goto bad1;
+		}
+#endif
+
 		if ( (error = pshm_access(pinfo, fmode, kauth_cred_get(), p)) ) {
 			PSHM_SUBSYS_UNLOCK();
 			goto bad1;
@@ -531,10 +563,10 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	}
 	if (!incache) {
 		/*
-	 	* We allocate a new entry if we are less than the maximum
-	 	* allowed and the one at the front of the LRU list is in use.
-	 	* Otherwise we use the one at the front of the LRU list.
-	 	*/
+		 * We allocate a new entry if we are less than the maximum
+		 * allowed and the one at the front of the LRU list is in use.
+		 * Otherwise we use the one at the front of the LRU list.
+		 */
 		MALLOC(pcp, struct pshmcache *, sizeof(struct pshmcache), M_SHM, M_WAITOK|M_ZERO);
 		if (pcp == NULL) {
 			error = ENOSPC;
@@ -561,6 +593,7 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	fp->f_type = DTYPE_PSXSHM;
 	fp->f_ops = &pshmops;
 	fp->f_data = (caddr_t)pnode;
+	*fdflags(p, indx) |= UF_EXCLOSE;
 	procfdtbl_releasefd(p, indx, NULL);
 	fp_drop(p, indx, fp, 1);
 	proc_fdunlock(p);
@@ -570,10 +603,14 @@ shm_open(struct proc *p, struct shm_open_args *uap, register_t *retval)
 	return (0);
 bad3:
 	FREE(pnode, M_SHM);
-		
+
 bad2:
-	if (pinfo_alloc)
+	if (pinfo_alloc) {
+#if CONFIG_MACF
+		mac_posixshm_label_destroy(pinfo);
+#endif
 		FREE(pinfo, M_SHM);
+	}
 bad1:
 	fp_free(p, indx, fp);
 bad:
@@ -583,7 +620,7 @@ bad:
 
 
 int
-pshm_truncate(__unused struct proc *p, struct fileproc *fp, __unused int fd, 
+pshm_truncate(__unused proc_t p, struct fileproc *fp, __unused int fd, 
 				off_t length, __unused register_t *retval)
 {
 	struct pshminfo * pinfo;
@@ -592,6 +629,9 @@ pshm_truncate(__unused struct proc *p, struct fileproc *fp, __unused int fd,
 	mach_vm_offset_t user_addr;
 	mem_entry_name_port_t mem_object;
 	mach_vm_size_t size;
+#if CONFIG_MACF
+	int error;
+#endif
 
 	if (fp->f_type != DTYPE_PSXSHM) {
 		return(EINVAL);
@@ -611,7 +651,13 @@ pshm_truncate(__unused struct proc *p, struct fileproc *fp, __unused int fd,
 		PSHM_SUBSYS_UNLOCK();
 		return(EINVAL);
 	}
-
+#if CONFIG_MACF
+	error = mac_posixshm_check_truncate(kauth_cred_get(), pinfo, size);
+	if (error) {
+		PSHM_SUBSYS_UNLOCK();
+		return(error);
+	}
+#endif
 	PSHM_SUBSYS_UNLOCK();
 	size = round_page_64(length);
 	kret = mach_vm_allocate(current_map(), &user_addr, size, VM_FLAGS_ANYWHERE);
@@ -648,9 +694,14 @@ out:
 }
 
 int
-pshm_stat(struct pshmnode *pnode, struct stat *sb)
+pshm_stat(struct pshmnode *pnode, void *ub, int isstat64)
 {
+	struct stat *sb = (struct stat *)0;	/* warning avoidance ; protected by isstat64 */
+	struct stat64 * sb64 = (struct stat64 *)0;  /* warning avoidance ; protected by isstat64 */
 	struct pshminfo *pinfo;
+#if CONFIG_MACF
+	int error;
+#endif
 	
 	PSHM_SUBSYS_LOCK();
 	if ((pinfo = pnode->pinfo) == PSHMINFO_NULL){
@@ -658,11 +709,29 @@ pshm_stat(struct pshmnode *pnode, struct stat *sb)
 		return(EINVAL);
 	}
 
-	bzero(sb, sizeof(struct stat)); 
-	sb->st_mode = pinfo->pshm_mode;
-	sb->st_uid = pinfo->pshm_uid;
-	sb->st_gid = pinfo->pshm_gid;
-	sb->st_size = pinfo->pshm_length;
+#if CONFIG_MACF
+	error = mac_posixshm_check_stat(kauth_cred_get(), pinfo);
+	if (error) {
+		PSHM_SUBSYS_UNLOCK();
+		return(error);
+	}
+#endif
+
+	if (isstat64 != 0) {
+		sb64 = (struct stat64 *)ub;
+		bzero(sb64, sizeof(struct stat64)); 
+		sb64->st_mode = pinfo->pshm_mode;
+		sb64->st_uid = pinfo->pshm_uid;
+		sb64->st_gid = pinfo->pshm_gid;
+		sb64->st_size = pinfo->pshm_length;
+	} else {
+		sb = (struct stat *)ub;
+		bzero(sb, sizeof(struct stat)); 
+		sb->st_mode = pinfo->pshm_mode;
+		sb->st_uid = pinfo->pshm_uid;
+		sb->st_gid = pinfo->pshm_gid;
+		sb->st_size = pinfo->pshm_length;
+	}
 	PSHM_SUBSYS_UNLOCK();
 
 	return(0);
@@ -673,7 +742,7 @@ pshm_stat(struct pshmnode *pnode, struct stat *sb)
  * XXX This code is repeated many times
  */
 int
-pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, __unused struct proc *p)
+pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, __unused proc_t p)
 {
 	mode_t mask;
 	int is_member;
@@ -711,14 +780,13 @@ pshm_access(struct pshminfo *pinfo, int mode, kauth_cred_t cred, __unused struct
 }
 
 int
-pshm_mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval, struct fileproc *fp, off_t pageoff) 
+pshm_mmap(__unused proc_t p, struct mmap_args *uap, user_addr_t *retval, struct fileproc *fp, off_t pageoff) 
 {
 	mach_vm_offset_t	user_addr = (mach_vm_offset_t)uap->addr;
 	mach_vm_size_t		user_size = (mach_vm_size_t)uap->len ;
 	int prot = uap->prot;
 	int flags = uap->flags;
 	vm_object_offset_t file_pos = (vm_object_offset_t)uap->pos;
-	int fd = uap->fd;
 	vm_map_t	user_map;
 	int		alloc_flags;
 	boolean_t 	docow;
@@ -726,6 +794,9 @@ pshm_mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval, struct fil
 	struct pshminfo * pinfo;
 	struct pshmnode * pnode;
 	void * mem_object;
+#if CONFIG_MACF
+	int error;
+#endif
 
 	if (user_size == 0) 
 		return(0);
@@ -764,6 +835,13 @@ pshm_mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval, struct fil
 		return(EINVAL);
 	}
 
+#if CONFIG_MACF
+	error = mac_posixshm_check_mmap(kauth_cred_get(), pinfo, prot, flags);
+	if (error) {
+		PSHM_SUBSYS_UNLOCK();
+		return(error);
+	}
+#endif
 
 	PSHM_SUBSYS_UNLOCK();
 	user_map = current_map();
@@ -784,10 +862,11 @@ pshm_mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval, struct fil
 	}
 	docow = FALSE;	
 
-	kret = mach_vm_map(user_map, &user_addr, user_size,
-			0, alloc_flags, pinfo->pshm_memobject, file_pos, docow,
-      	                prot, VM_PROT_DEFAULT, 
-			VM_INHERIT_SHARE);
+	kret = vm_map_enter_mem_object(user_map, &user_addr, user_size,
+				       0, alloc_flags,
+				       pinfo->pshm_memobject, file_pos, docow,
+				       prot, VM_PROT_DEFAULT, 
+				       VM_INHERIT_SHARE);
 	if (kret != KERN_SUCCESS) 
 			goto out;
 	/* LP64todo - this should be superfluous at this point */
@@ -819,7 +898,7 @@ out:
 }
 
 int
-shm_unlink(__unused struct proc *p, struct shm_unlink_args *uap, 
+shm_unlink(__unused proc_t p, struct shm_unlink_args *uap, 
 			__unused register_t *retval)
 {
 	size_t i;
@@ -893,7 +972,8 @@ shm_unlink(__unused struct proc *p, struct shm_unlink_args *uap,
 
 	if ((pinfo->pshm_flags & (PSHM_DEFINED | PSHM_ALLOCATED))==0) {
 		PSHM_SUBSYS_UNLOCK();
-		return (EINVAL);
+		error = EINVAL;
+		goto bad;
 	}
 
 	if (pinfo->pshm_flags & PSHM_INDELETE) {
@@ -901,6 +981,13 @@ shm_unlink(__unused struct proc *p, struct shm_unlink_args *uap,
 		error = 0;
 		goto bad;
 	}
+#if CONFIG_MACF
+	error = mac_posixshm_check_unlink(kauth_cred_get(), pinfo, nameptr);
+	if (error) {
+		PSHM_SUBSYS_UNLOCK();
+		goto bad;
+	}
+#endif
 
 	AUDIT_ARG(posix_ipc_perm, pinfo->pshm_uid, pinfo->pshm_gid,
 		  pinfo->pshm_mode);
@@ -964,15 +1051,18 @@ pshm_close(struct pshmnode *pnode)
 		if (pinfo->pshm_memobject != NULL)
 			mach_memory_entry_port_release(pinfo->pshm_memobject);
 		PSHM_SUBSYS_LOCK();
+#if CONFIG_MACF
+		mac_posixshm_label_destroy(pinfo);
+#endif
 		FREE(pinfo,M_SHM);
 	}
 	FREE(pnode, M_SHM);
 	return (error);
 }
 
-/* struct proc passed to match prototype for struct fileops */
+/* vfs_context_t passed to match prototype for struct fileops */
 static int
-pshm_closefile(struct fileglob *fg, __unused struct proc *p)
+pshm_closefile(struct fileglob *fg, __unused vfs_context_t ctx)
 {
 	int error;
 
@@ -984,37 +1074,35 @@ pshm_closefile(struct fileglob *fg, __unused struct proc *p)
 
 static int
 pshm_read(__unused struct fileproc *fp, __unused struct uio *uio, 
-			__unused kauth_cred_t cred, __unused int flags, 
-			__unused struct proc *p)
+			__unused int flags, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 pshm_write(__unused struct fileproc *fp, __unused struct uio *uio, 
-			__unused kauth_cred_t cred, __unused int flags, 
-			__unused struct proc *p)
+			__unused int flags, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 pshm_ioctl(__unused struct fileproc *fp, __unused u_long com, 
-			__unused caddr_t data, __unused struct proc *p)
+			__unused caddr_t data, __unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 pshm_select(__unused struct fileproc *fp, __unused int which, __unused void *wql, 
-			__unused struct proc *p)
+			__unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
 
 static int
 pshm_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, 
-				__unused struct proc *p)
+				__unused vfs_context_t ctx)
 {
 	return(ENOTSUP);
 }
@@ -1023,7 +1111,7 @@ int
 fill_pshminfo(struct pshmnode * pshm, struct pshm_info * info)
 {
 	struct pshminfo *pinfo;
-	struct stat *sb;
+	struct vinfo_stat *sb;
 	
 	PSHM_SUBSYS_LOCK();
 	if ((pinfo = pshm->pinfo) == PSHMINFO_NULL){
@@ -1033,11 +1121,11 @@ fill_pshminfo(struct pshmnode * pshm, struct pshm_info * info)
 
 	sb = &info->pshm_stat;
 
-	bzero(sb, sizeof(struct stat)); 
-	sb->st_mode = pinfo->pshm_mode;
-	sb->st_uid = pinfo->pshm_uid;
-	sb->st_gid = pinfo->pshm_gid;
-	sb->st_size = pinfo->pshm_length;
+	bzero(sb, sizeof(struct vinfo_stat)); 
+	sb->vst_mode = pinfo->pshm_mode;
+	sb->vst_uid = pinfo->pshm_uid;
+	sb->vst_gid = pinfo->pshm_gid;
+	sb->vst_size = pinfo->pshm_length;
 
 	info->pshm_mappaddr = pshm->mapp_addr;
 	bcopy(&pinfo->pshm_name[0], &info->pshm_name[0], PSHMNAMLEN+1); 
@@ -1046,4 +1134,22 @@ fill_pshminfo(struct pshmnode * pshm, struct pshm_info * info)
 	return(0);
 }
 
+#if CONFIG_MACF
+void
+pshm_label_associate(struct fileproc *fp, struct vnode *vp, vfs_context_t ctx)
+{
+	struct pshmnode *pnode;
+	struct pshminfo *pshm;
 
+	PSHM_SUBSYS_LOCK();
+	pnode = (struct pshmnode *)fp->f_fglob->fg_data;
+	if (pnode != NULL) {
+		pshm = pnode->pinfo;
+		if (pshm != NULL)
+			mac_posixshm_vnode_label_associate(
+				vfs_context_ucred(ctx), pshm, pshm->pshm_label,
+				vp, vp->v_label);
+	}
+	PSHM_SUBSYS_UNLOCK();
+}
+#endif
diff --git a/bsd/kern/preload.h b/bsd/kern/preload.h
index 7db056c43..283db76d2 100644
--- a/bsd/kern/preload.h
+++ b/bsd/kern/preload.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_BSD_KERN_PRELOAD_H_
 #define _BSD_KERN_PRELOAD_H_
diff --git a/bsd/kern/proc_info.c b/bsd/kern/proc_info.c
index 621886c82..f06b40da2 100644
--- a/bsd/kern/proc_info.c
+++ b/bsd/kern/proc_info.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -65,6 +71,7 @@
 #include <sys/sysproto.h>
 #include <sys/msgbuf.h>
 
+#include <sys/msgbuf.h>
 
 #include <machine/machine_routines.h>
 
@@ -90,29 +97,31 @@ int proc_pidbsdinfo(proc_t p, struct proc_bsdinfo *pbsd);
 int proc_pidtaskinfo(proc_t p, struct proc_taskinfo *ptinfo);
 int proc_pidallinfo(proc_t p, int flavor, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
 int proc_pidthreadinfo(proc_t p, uint64_t arg,  struct proc_threadinfo *pthinfo);
+int proc_pidthreadpathinfo(proc_t p, uint64_t arg,  struct proc_threadwithpathinfo *pinfo);
 int proc_pidlistthreads(proc_t p,  user_addr_t buffer, uint32_t buffersize, register_t *retval);
 int proc_pidregioninfo(proc_t p, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
 int proc_pidregionpathinfo(proc_t p,  uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
 int proc_pidvnodepathinfo(proc_t p,  uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
+int proc_pidpathinfo(proc_t p, uint64_t arg, user_addr_t buffer, uint32_t buffersize, register_t *retval);
 
 
 /* protos for proc_pidfdinfo calls */
-int pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_socketinfo(socket_t  so, struct fileproc *fp, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_pseminfo(struct psemnode * psem, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_pshminfo(struct pshmnode * pshm, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_pipeinfo(struct pipe * p, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_kqueueinfo(struct kqueue * kq, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
-int pid_atalkinfo(struct atalk  * at, struct fileproc * fp,  user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_socketinfo(socket_t  so, struct fileproc *fp, int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pseminfo(struct psemnode * psem, struct fileproc * fp,  int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pshminfo(struct pshmnode * pshm, struct fileproc * fp,  int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_pipeinfo(struct pipe * p, struct fileproc * fp,  int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_kqueueinfo(struct kqueue * kq, struct fileproc * fp,  int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
+int pid_atalkinfo(struct atalk  * at, struct fileproc * fp,  int closeonexec, user_addr_t  buffer, uint32_t buffersize, register_t * retval);
 
 
 /* protos for misc */
 
 int fill_vnodeinfo(vnode_t vp, struct vnode_info *vinfo);
-void  fill_fileinfo(struct fileproc * fp, struct proc_fileinfo * finfo);
+void  fill_fileinfo(struct fileproc * fp, int closeonexec, struct proc_fileinfo * finfo);
 static int proc_security_policy(proc_t p);
-
+static void munge_vinfo_stat(struct stat64 *sbp, struct vinfo_stat *vsbp);
 
 /***************************** proc_info ********************/
 
@@ -136,7 +145,7 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b
 		case 3: /* proc_pidfdinfo */
 			return(proc_pidfdinfo(pid, flavor, (int)arg, buffer, buffersize, retval));
 		case 4: /* proc_kernmsgbuf */
-			return(proc_kernmsgbuf( buffer, buffersize, retval));
+			return(proc_kernmsgbuf(buffer, buffersize, retval));
 		default:
 				return(EINVAL);
 	}
@@ -148,7 +157,6 @@ proc_info_internal(int callnum, int pid, int flavor, uint64_t arg, user_addr_t b
 int
 proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t  buffersize, register_t * retval)
 {
-	boolean_t funnel_state;
 	int numprocs, wantpids;
 	char * kbuf;
 	int * ptr;
@@ -171,9 +179,12 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t  bu
 		numprocs = wantpids;
 
 	kbuf = (char *)kalloc((vm_size_t)(numprocs * sizeof(int)));
+	if (kbuf == NULL)
+		return(ENOMEM);
 	bzero(kbuf, sizeof(int));
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	proc_list_lock();
+
 	
 	n = 0;
 	ptr = (int *)kbuf;
@@ -181,27 +192,47 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t  bu
 		skip = 0;
 		switch (type) {
 			case PROC_PGRP_ONLY:
-				if (p->p_pgrp->pg_id != (pid_t)typeinfo)
+				if (p->p_pgrpid != (pid_t)typeinfo)
 					skip = 1;
 			  	break;
 			case PROC_ALL_PIDS:
 				skip = 0;
 			  	break;
 			case PROC_TTY_ONLY:
+				/* racy but list lock is held */
 				if ((p->p_flag & P_CONTROLT) == 0 ||
-					(p->p_session == NULL) ||
-			    	p->p_session->s_ttyp == NULL ||
-			    	p->p_session->s_ttyp->t_dev != (dev_t)typeinfo)
+					(p->p_pgrp == NULL) || (p->p_pgrp->pg_session == NULL) ||
+			    	p->p_pgrp->pg_session->s_ttyp == NULL ||
+			    	p->p_pgrp->pg_session->s_ttyp->t_dev != (dev_t)typeinfo)
 					skip = 1;
 			  	break;
 			case PROC_UID_ONLY:
-				if ((p->p_ucred == NULL) ||
-					(kauth_cred_getuid(p->p_ucred) != (uid_t)typeinfo))
+				if (p->p_ucred == NULL)
 					skip = 1;
+				else {
+					kauth_cred_t my_cred;
+					uid_t uid;
+			
+					my_cred = kauth_cred_proc_ref(p);
+					uid = kauth_cred_getuid(my_cred);
+					kauth_cred_unref(&my_cred);
+					if (uid != (uid_t)typeinfo)
+						skip = 1;
+				}
 			  	break;
 			case PROC_RUID_ONLY:
-				if ((p->p_ucred == NULL) ||
-					(p->p_ucred->cr_ruid != (uid_t)typeinfo))
+				if (p->p_ucred == NULL)
+					skip = 1;
+				else {
+					kauth_cred_t my_cred;
+					uid_t uid;
+			
+					my_cred = kauth_cred_proc_ref(p);
+					uid = my_cred->cr_ruid;
+					kauth_cred_unref(&my_cred);
+					if (uid != (uid_t)typeinfo)
+						skip = 1;
+				}
 			  	break;
 			default:
 			  skip = 1;
@@ -230,7 +261,8 @@ proc_listpids(uint32_t type, uint32_t typeinfo, user_addr_t buffer, uint32_t  bu
 		}
 	}
 	
-	thread_funnel_set(kernel_flock, funnel_state);
+
+	proc_list_unlock();
 
 	ptr = (int *)kbuf;
 	error = copyout((caddr_t)ptr, buffer, n * sizeof(int));
@@ -270,6 +302,8 @@ proc_pidfdlist(proc_t p, user_addr_t buffer, uint32_t  buffersize, register_t *r
 			numfds = needfds;
 
 		kbuf = (char *)kalloc((vm_size_t)(numfds * sizeof(struct proc_fdinfo)));
+		if (kbuf == NULL)
+			return(ENOMEM);
 		bzero(kbuf, numfds * sizeof(struct proc_fdinfo));
 
 		proc_fdlock(p);
@@ -300,54 +334,66 @@ proc_pidbsdinfo(proc_t p, struct proc_bsdinfo * pbsd)
 {
 	register struct tty *tp;
 	struct  session *sessionp = NULL;
+	struct pgrp * pg;
+	kauth_cred_t my_cred;
 
+	pg = proc_pgrp(p);
+	sessionp = proc_session(p);
+
+	my_cred = kauth_cred_proc_ref(p);
 	bzero(pbsd, sizeof(struct proc_bsdinfo));
 	pbsd->pbi_status = p->p_stat;
 	pbsd->pbi_xstatus = p->p_xstat;
 	pbsd->pbi_pid = p->p_pid;
-	pbsd->pbi_ppid = p->p_pptr->p_pid;
-	pbsd->pbi_uid = p->p_ucred->cr_uid;
-	pbsd->pbi_gid = p->p_ucred->cr_uid; /* XXX */
-	pbsd->pbi_ruid =  p->p_ucred->cr_ruid;
-	pbsd->pbi_rgid = p->p_ucred->cr_rgid;
-	pbsd->pbi_svuid =  p->p_ucred->cr_svuid;
-	pbsd->pbi_svgid = p->p_ucred->cr_svgid;
+	pbsd->pbi_ppid = p->p_ppid;
+	pbsd->pbi_uid = my_cred->cr_uid;
+	pbsd->pbi_gid = my_cred->cr_gid; 
+	pbsd->pbi_ruid =  my_cred->cr_ruid;
+	pbsd->pbi_rgid = my_cred->cr_rgid;
+	pbsd->pbi_svuid =  my_cred->cr_svuid;
+	pbsd->pbi_svgid = my_cred->cr_svgid;
+	kauth_cred_unref(&my_cred);
 	
 	pbsd->pbi_nice = p->p_nice;
-	if (p->p_stats)
-		pbsd->pbi_start = p->p_stats->p_start;
+	pbsd->pbi_start = p->p_start;
 	bcopy(&p->p_comm, &pbsd->pbi_comm[0], MAXCOMLEN);
 	bcopy(&p->p_name, &pbsd->pbi_name[0], 2* MAXCOMLEN);
 
 	pbsd->pbi_flags = 0;	
 	if ((p->p_flag & P_SYSTEM) == P_SYSTEM) 
 		pbsd->pbi_flags |= PROC_FLAG_SYSTEM;
-	if ((p->p_flag & P_TRACED) == P_TRACED) 
+	if ((p->p_lflag & P_LTRACED) == P_LTRACED) 
 		pbsd->pbi_flags |= PROC_FLAG_TRACED;
-	if ((p->p_flag & P_WEXIT) == P_WEXIT) 
+	if ((p->p_lflag & P_LEXIT) == P_LEXIT) 
 		pbsd->pbi_flags |= PROC_FLAG_INEXIT;
-	if ((p->p_flag & P_PPWAIT) == P_PPWAIT) 
+	if ((p->p_lflag & P_LPPWAIT) == P_LPPWAIT) 
 		pbsd->pbi_flags |= PROC_FLAG_PPWAIT;
 	if ((p->p_flag & P_LP64) == P_LP64) 
 		pbsd->pbi_flags |= PROC_FLAG_LP64;
 	if ((p->p_flag & P_CONTROLT) == P_CONTROLT) 
 		pbsd->pbi_flags |= PROC_FLAG_CONTROLT;
+	if ((p->p_flag & P_THCWD) == P_THCWD) 
+		pbsd->pbi_flags |= PROC_FLAG_THCWD;
 
-	if (SESS_LEADER(p))
+	if (SESS_LEADER(p, sessionp))
 		pbsd->pbi_flags |= PROC_FLAG_SLEADER;
-	if (p->p_pgrp->pg_session && p->p_pgrp->pg_session->s_ttyvp)
+	if ((sessionp != SESSION_NULL) && sessionp->s_ttyvp)
 		pbsd->pbi_flags |= PROC_FLAG_CTTY;
 		
 	pbsd->pbi_nfiles = p->p_fd->fd_nfiles;
-	if (p->p_pgrp) {
-		sessionp = p->p_pgrp->pg_session;
-		pbsd->pbi_pgid = p->p_pgrp->pg_id;
-		pbsd->pbi_pjobc = p->p_pgrp->pg_jobc;
-		if ((p->p_flag & P_CONTROLT) && (sessionp) && (tp = sessionp->s_ttyp)) {
+	if (pg != PGRP_NULL) {
+		pbsd->pbi_pgid = p->p_pgrpid;
+		pbsd->pbi_pjobc = pg->pg_jobc;
+		if ((p->p_flag & P_CONTROLT) && (sessionp != SESSION_NULL) && (tp = sessionp->s_ttyp)) {
 			pbsd->e_tdev = tp->t_dev;
-			pbsd->e_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : 0;
+			pbsd->e_tpgid = sessionp->s_ttypgrpid;
 		}
 	} 
+	if (sessionp != SESSION_NULL)
+		session_rele(sessionp);
+	if (pg != PGRP_NULL)
+		pg_rele(pg);
+
 	return(0);
 }
 
@@ -375,7 +421,7 @@ proc_pidthreadinfo(proc_t p, uint64_t arg,  struct proc_threadinfo *pthinfo)
 
 	bzero(pthinfo, sizeof(struct proc_threadinfo));
 
-	error = fill_taskthreadinfo(p->task, threadaddr, (struct proc_threadinfo_internal *)pthinfo);
+	error = fill_taskthreadinfo(p->task, threadaddr, (struct proc_threadinfo_internal *)pthinfo, NULL, NULL);
 	if (error)
 		return(ESRCH);
 	else
@@ -384,6 +430,53 @@ proc_pidthreadinfo(proc_t p, uint64_t arg,  struct proc_threadinfo *pthinfo)
 }
 
 
+void
+bsd_threadcdir(void * uth, void *vptr, int *vidp)
+{
+	struct uthread * ut = (struct uthread *)uth;
+	vnode_t vp;
+	vnode_t *vpp = (vnode_t *)vptr;
+
+	vp = ut->uu_cdir;
+	if (vp  != NULLVP) {
+		if (vpp != NULL) {
+			*vpp = vp;
+			if (vidp != NULL)
+				*vidp = vp->v_id;
+		}
+	}
+}
+
+
+int 
+proc_pidthreadpathinfo(proc_t p, uint64_t arg,  struct proc_threadwithpathinfo *pinfo)
+{
+	vnode_t vp = NULLVP;
+	int vid;
+	int error = 0;
+	uint64_t threadaddr = (uint64_t)arg;
+	int count;
+
+	bzero(pinfo, sizeof(struct proc_threadwithpathinfo));
+
+	error = fill_taskthreadinfo(p->task, threadaddr, (struct proc_threadinfo_internal *)&pinfo->pt, (void *)&vp, &vid);
+	if (error)
+		return(ESRCH);
+
+	if ((vp != NULLVP) && ((vnode_getwithvid(vp, vid)) == 0)) {
+		error = fill_vnodeinfo(vp, &pinfo->pvip.vip_vi) ;
+		if (error == 0) {
+			count = MAXPATHLEN;
+			vn_getpath(vp, &pinfo->pvip.vip_path[0], &count);
+			pinfo->pvip.vip_path[MAXPATHLEN-1] = 0;
+		}
+		vnode_put(vp);
+	}	
+	return(error);
+}
+
+
+
 int 
 proc_pidlistthreads(proc_t p,  user_addr_t buffer, uint32_t  buffersize, register_t *retval)
 {
@@ -403,6 +496,8 @@ proc_pidlistthreads(proc_t p,  user_addr_t buffer, uint32_t  buffersize, registe
 		numthreads = count;
 
 	kbuf = (void *)kalloc(numthreads * sizeof(uint64_t));
+	if (kbuf == NULL)
+		return(ENOMEM);
 	bzero(kbuf, numthreads * sizeof(uint64_t));
 	
 	ret = fill_taskthreadlist(p->task, kbuf, numthreads);
@@ -466,6 +561,10 @@ proc_pidregionpathinfo(proc_t p, uint64_t arg, user_addr_t buffer, __unused uint
 	return(error);
 }
 
+/*
+ * Path is relative to current process directory; may different from current
+ * thread directory.
+ */
 int 
 proc_pidvnodepathinfo(proc_t p, __unused uint64_t arg, user_addr_t buffer, __unused uint32_t  buffersize, register_t *retval)
 {
@@ -528,6 +627,44 @@ out:
 	return(error);
 }
 
+int 
+proc_pidpathinfo(proc_t p, __unused uint64_t arg, user_addr_t buffer, uint32_t buffersize, __unused register_t *retval)
+{
+	int vid, error;
+	vnode_t tvp;
+	vnode_t nvp = NULLVP;
+	int len = buffersize; 
+	char * buf;
+
+	tvp = p->p_textvp;
+
+	if (tvp == NULLVP)
+		return(ESRCH);
+
+	buf = (char *)kalloc(buffersize);
+	if (buf == NULL) 
+		return(ENOMEM);
+
+
+	vid = vnode_vid(tvp);
+	error = vnode_getwithvid(tvp, vid);
+	if (error == 0) {
+		error = vn_getpath(tvp, buf, &len);
+		vnode_put(tvp);
+		if (error == 0) {
+			error = vnode_lookup(buf, 0, &nvp, vfs_context_current()); 
+			if ((error == 0) && ( nvp != NULLVP))
+				vnode_put(nvp);
+			if (error == 0) {
+				error = copyout(buf, buffer, len);
+			}
+		}
+	}
+	kfree(buf, buffersize);
+	return(error);
+}
+
+
 /********************************** proc_pidinfo ********************************/
 
 
@@ -538,7 +675,7 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  bu
 	int error = ENOTSUP;
 	int gotref = 0;
 	int findzomb = 0;
-	boolean_t funnel_state;
+	int refheld = 0;
 	uint32_t size;
 
 	switch (flavor) {
@@ -571,6 +708,12 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  bu
 		case PROC_PIDVNODEPATHINFO:
 			size = PROC_PIDVNODEPATHINFO_SIZE;
 			break;
+		case PROC_PIDTHREADPATHINFO:
+			size = PROC_PIDTHREADPATHINFO_SIZE;
+			break;
+		case PROC_PIDPATHINFO:
+			size = MAXPATHLEN;
+			break;
 		default:
 			return(EINVAL);
 	}
@@ -578,8 +721,12 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  bu
 	if (buffersize < size) 
 		return(ENOMEM);
 
-	if (flavor != PROC_PIDTBSDINFO) {
-		if ((p = proc_findref(pid)) == PROC_NULL) {
+	if ((flavor == PROC_PIDPATHINFO) && (buffersize > PROC_PIDPATHINFO_MAXSIZE)) {
+		return(EOVERFLOW);
+	}
+
+	if ((flavor != PROC_PIDTBSDINFO) && (flavor != PROC_PIDPATHINFO)) {
+		if ((p = proc_find(pid)) == PROC_NULL) {
 				error = ESRCH;
 				goto out;
 			} else {
@@ -602,24 +749,25 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  bu
 
 			if (arg)
 				findzomb = 1;
-			funnel_state = thread_funnel_set(kernel_flock, TRUE);
-			p = pfind(pid);
+			p = proc_find(pid);
 			if (p == PROC_NULL) {
 				if (findzomb) 
 					p = pzfind(pid);
 				if (p == NULL) {
 					error = ESRCH;
-					thread_funnel_set(kernel_flock, funnel_state);
 					goto out;	
 				}
-			}
+			} else 
+				refheld = 1;
 			/* Do we have permission to look into this ? */
 			if ((error = proc_security_policy(p)) != 0) {
-				thread_funnel_set(kernel_flock, funnel_state);
+				if (refheld != 0)
+					proc_rele(p);
 				goto out;
 			}
 			error = proc_pidbsdinfo(p, &pbsd);
-			thread_funnel_set(kernel_flock, funnel_state);	
+			if (refheld != 0)
+				proc_rele(p);
 			if (error == 0) {
 				error = copyout(&pbsd, buffer, sizeof(struct proc_bsdinfo));
 				if (error == 0)
@@ -686,19 +834,43 @@ proc_pidinfo(int pid, int flavor, uint64_t arg, user_addr_t buffer, uint32_t  bu
 		}
 		break;
 
+
+		case PROC_PIDTHREADPATHINFO:{
+		struct proc_threadwithpathinfo pinfo;
+
+			error  = proc_pidthreadpathinfo(p,  arg, &pinfo);
+			if (error == 0) {
+				error = copyout((caddr_t)&pinfo, buffer, sizeof(struct proc_threadwithpathinfo));
+				if (error == 0)
+						*retval = sizeof(struct proc_threadwithpathinfo);
+			}
+		}
+		break;
+
+		case PROC_PIDPATHINFO: {
+			p = proc_find(pid);
+			if (p == PROC_NULL) {
+				error = ESRCH;
+				goto out;
+			}
+			gotref = 1;
+			error =  proc_pidpathinfo(p, arg, buffer, buffersize, retval);
+		}
+		break;
+
 		default:
 			error = ENOTSUP;
 	}
 	
 out:
 	if (gotref)
-		proc_dropref(p);
+		proc_rele(p);
 	return(error);
 }
 
 
 int 
-pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
+pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
 {
 	struct vnode_fdinfo vfi;
 	int error= 0;
@@ -707,7 +879,7 @@ pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffe
 		return(error);
 	}
 	bzero(&vfi, sizeof(struct vnode_fdinfo));
-	fill_fileinfo(fp, &vfi.pfi);
+	fill_fileinfo(fp, closeonexec, &vfi.pfi);
 	error = fill_vnodeinfo(vp, &vfi.pvi);
 	vnode_put(vp);
 	if (error == 0) {
@@ -719,7 +891,7 @@ pid_vnodeinfo(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffe
 }
 
 int 
-pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
+pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval) 
 {
 	struct vnode_fdinfowithpath vfip;
 	int count, error= 0;
@@ -728,7 +900,7 @@ pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  b
 		return(error);
 	}
 	bzero(&vfip, sizeof(struct vnode_fdinfowithpath));
-	fill_fileinfo(fp, &vfip.pfi);
+	fill_fileinfo(fp, closeonexec, &vfip.pfi);
 	error = fill_vnodeinfo(vp, &vfip.pvip.vip_vi) ;
 	if (error == 0) {
 		count = MAXPATHLEN;
@@ -744,12 +916,16 @@ pid_vnodeinfopath(vnode_t vp, uint32_t vid, struct fileproc * fp, user_addr_t  b
 }
 
 void  
-fill_fileinfo(struct fileproc * fp, struct proc_fileinfo * fproc)
+fill_fileinfo(struct fileproc * fp, int closeonexec, struct proc_fileinfo * fproc)
 {
 		fproc->fi_openflags = fp->f_fglob->fg_flag;
-		fproc->fi_status = fp->f_flags;
+		fproc->fi_status = 0;
 		fproc->fi_offset = fp->f_fglob->fg_offset;
 		fproc->fi_type = fp->f_fglob->fg_type;
+		if (fp->f_fglob->fg_count)
+			fproc->fi_status |= PROC_FP_SHARED;
+		if (closeonexec != 0)
+			fproc->fi_status |= PROC_FP_CLEXEC;
 }
 
 
@@ -758,15 +934,15 @@ int
 fill_vnodeinfo(vnode_t vp, struct vnode_info *vinfo)
 {
 		vfs_context_t context;
-		struct stat * sb;
+		struct stat64 sb;
 		int error = 0;
 
-		sb = &vinfo->vi_stat;
-
 		context = vfs_context_create((vfs_context_t)0);
-		error = vn_stat(vp, sb, NULL, context);
+		error = vn_stat(vp, &sb, NULL, 1, context);
 		(void)vfs_context_rele(context);
 
+		munge_vinfo_stat(&sb, &vinfo->vi_stat);
+
 		if (error != 0)
 			goto out;
 
@@ -782,29 +958,33 @@ out:
 }
 
 int
-pid_socketinfo(socket_t so, struct fileproc *fp, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+pid_socketinfo(socket_t so, struct fileproc *fp, int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
 {
+#if SOCKETS
 	struct socket_fdinfo s;
 	int error = 0;
 
 	bzero(&s, sizeof(struct socket_fdinfo));
-	fill_fileinfo(fp, &s.pfi);
+	fill_fileinfo(fp, closeonexec, &s.pfi);
 	if ((error = fill_socketinfo(so, &s.psi)) == 0) {
 		if ((error = copyout(&s, buffer, sizeof(struct socket_fdinfo))) == 0)
 				*retval = sizeof(struct socket_fdinfo);
 	}
-	
 	return (error);
+#else
+	*retval = 0;
+	return (ENOTSUP);
+#endif
 }
 
 int
-pid_pseminfo(struct psemnode *psem, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+pid_pseminfo(struct psemnode *psem, struct fileproc *fp,  int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
 {
 	struct psem_fdinfo pseminfo;
 	int error = 0;
  
 	bzero(&pseminfo, sizeof(struct psem_fdinfo));
-	fill_fileinfo(fp, &pseminfo.pfi);
+	fill_fileinfo(fp, closeonexec, &pseminfo.pfi);
 
 	if ((error = fill_pseminfo(psem, &pseminfo.pseminfo)) == 0) {
 		if ((error = copyout(&pseminfo, buffer, sizeof(struct psem_fdinfo))) == 0)
@@ -815,13 +995,13 @@ pid_pseminfo(struct psemnode *psem, struct fileproc *fp,  user_addr_t  buffer, _
 }
 
 int
-pid_pshminfo(struct pshmnode *pshm, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+pid_pshminfo(struct pshmnode *pshm, struct fileproc *fp,  int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
 {
 	struct pshm_fdinfo pshminfo;
 	int error = 0;
  
 	bzero(&pshminfo, sizeof(struct pshm_fdinfo));
-	fill_fileinfo(fp, &pshminfo.pfi);
+	fill_fileinfo(fp, closeonexec, &pshminfo.pfi);
 
 	if ((error = fill_pshminfo(pshm, &pshminfo.pshminfo)) == 0) {
 		if ((error = copyout(&pshminfo, buffer, sizeof(struct pshm_fdinfo))) == 0)
@@ -832,13 +1012,13 @@ pid_pshminfo(struct pshmnode *pshm, struct fileproc *fp,  user_addr_t  buffer, _
 }
 
 int
-pid_pipeinfo(struct pipe *  p, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+pid_pipeinfo(struct pipe *  p, struct fileproc *fp,  int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
 {
 	struct pipe_fdinfo pipeinfo;
 	int error = 0;
 
 	bzero(&pipeinfo, sizeof(struct pipe_fdinfo));
-	fill_fileinfo(fp, &pipeinfo.pfi);
+	fill_fileinfo(fp, closeonexec, &pipeinfo.pfi);
 	if ((error = fill_pipeinfo(p, &pipeinfo.pipeinfo)) == 0) {
 		if ((error = copyout(&pipeinfo, buffer, sizeof(struct pipe_fdinfo))) == 0)
 				*retval = sizeof(struct pipe_fdinfo);
@@ -848,14 +1028,14 @@ pid_pipeinfo(struct pipe *  p, struct fileproc *fp,  user_addr_t  buffer, __unus
 }
 
 int
-pid_kqueueinfo(struct kqueue * kq, struct fileproc *fp,  user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
+pid_kqueueinfo(struct kqueue * kq, struct fileproc *fp,  int closeonexec, user_addr_t  buffer, __unused uint32_t buffersize, register_t * retval)
 {
 	struct kqueue_fdinfo kqinfo;
 	int error = 0;
 	
 	bzero(&kqinfo, sizeof(struct kqueue_fdinfo));
  
-	fill_fileinfo(fp, &kqinfo.pfi);
+	fill_fileinfo(fp, closeonexec, &kqinfo.pfi);
 
 	if ((error = fill_kqueueinfo(kq, &kqinfo.kqueueinfo)) == 0) {
 		if ((error = copyout(&kqinfo, buffer, sizeof(struct kqueue_fdinfo))) == 0)
@@ -866,7 +1046,7 @@ pid_kqueueinfo(struct kqueue * kq, struct fileproc *fp,  user_addr_t  buffer, __
 }
 
 int
-pid_atalkinfo(__unused struct atalk * at, __unused struct fileproc *fp,  __unused user_addr_t  buffer, __unused uint32_t buffersize, __unused register_t * retval)
+pid_atalkinfo(__unused struct atalk * at, __unused struct fileproc *fp,  __unused int closeonexec, __unused user_addr_t  buffer, __unused uint32_t buffersize, __unused register_t * retval)
 {
 	return ENOTSUP;
 }
@@ -881,7 +1061,7 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 	int error = ENOTSUP;
 	struct fileproc * fp;
 	uint32_t size;
-	
+	int closeonexec = 0;
 
 	switch (flavor) {
 		case PROC_PIDFDVNODEINFO:
@@ -917,7 +1097,7 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 	if (buffersize < size)
 		return(ENOMEM);
 
-	if ((p = proc_findref(pid)) == PROC_NULL) {
+	if ((p = proc_find(pid)) == PROC_NULL) {
 		error = ESRCH;
 		goto out;
 	}
@@ -934,7 +1114,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfvpandvid(p, fd, &fp,  &vp, &vid)) !=0) {
 				goto out1;
 			}
-			error =  pid_vnodeinfo(vp, vid, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_vnodeinfo(vp, vid, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -946,7 +1128,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 				goto out1;
 			}
 
-			error =  pid_vnodeinfopath(vp, vid, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_vnodeinfopath(vp, vid, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -956,7 +1140,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfsock(p, fd, &fp,  &so)) !=0) {
 				goto out1;
 			}
-			error =  pid_socketinfo(so, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_socketinfo(so, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -966,7 +1152,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfpsem(p, fd, &fp,  &psem)) !=0) {
 				goto out1;
 			}
-			error =  pid_pseminfo(psem, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_pseminfo(psem, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -976,7 +1164,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfpshm(p, fd, &fp,  &pshm)) !=0) {
 				goto out1;
 			}
-			error =  pid_pshminfo(pshm, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_pshminfo(pshm, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -986,7 +1176,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfpipe(p, fd, &fp,  &cpipe)) !=0) {
 				goto out1;
 			}
-			error =  pid_pipeinfo(cpipe, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_pipeinfo(cpipe, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -996,7 +1188,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 			if ((error = fp_getfkq(p, fd, &fp,  &kq)) !=0) {
 				goto out1;
 			}
-			error =  pid_kqueueinfo(kq, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_kqueueinfo(kq, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -1007,7 +1201,9 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 				goto out1;
 			}
 
-			error =  pid_atalkinfo(at, fp, buffer, buffersize, retval);
+			/* no need to be under the fdlock */
+			closeonexec = p->p_fd->fd_ofileflags[fd] & UF_EXCLOSE;
+			error =  pid_atalkinfo(at, fp, closeonexec, buffer, buffersize, retval);
 		}
 		break;
 
@@ -1020,7 +1216,7 @@ proc_pidfdinfo(int pid, int flavor,  int fd, user_addr_t buffer, uint32_t buffer
 
 	fp_drop(p, fd, fp , 0); 	
 out1 :
-	proc_dropref(p);
+	proc_rele(p);
 out:
 	return(error);
 }
@@ -1029,7 +1225,14 @@ out:
 static int
 proc_security_policy(proc_t p)
 {
-	if ((kauth_cred_getuid(p->p_ucred) != kauth_cred_getuid(kauth_cred_get())) 
+	kauth_cred_t my_cred;
+	uid_t uid;
+
+	my_cred = kauth_cred_proc_ref(p);
+	uid = kauth_cred_getuid(my_cred) ;
+	kauth_cred_unref(&my_cred);
+	
+	if ((uid != kauth_cred_getuid(kauth_cred_get())) 
 		&& suser(kauth_cred_get(), (u_short *)0)) {
 			return(EPERM);
 		}
@@ -1037,22 +1240,43 @@ proc_security_policy(proc_t p)
 	return(0);
 }
 
-
-/* Temporary hack to get dmesg to work. In Leopard this will disappear */
 int 
 proc_kernmsgbuf(user_addr_t buffer, uint32_t buffersize, register_t * retval)
 {
-	int error;
-
-	if (buffersize  < sizeof(struct msgbuf))
-		return(ENOMEM);
-
 	if (suser(kauth_cred_get(), (u_short *)0) == 0) {
-		error = copyout(msgbufp, buffer, sizeof(struct msgbuf));
-		if (error ==  0)
-			*retval = sizeof(struct msgbuf);
-		return(error);
+		return(log_dmesg(buffer, buffersize, retval));
 	} else
 		return(EPERM);
 }
 
+/*
+ * copy stat64 structure into vinfo_stat structure.
+ */
+static void
+munge_vinfo_stat(struct stat64 *sbp, struct vinfo_stat *vsbp)
+{
+        bzero(vsbp, sizeof(struct vinfo_stat));
+
+	vsbp->vst_dev = sbp->st_dev;
+	vsbp->vst_mode = sbp->st_mode;
+	vsbp->vst_nlink = sbp->st_nlink;
+	vsbp->vst_ino = sbp->st_ino;
+	vsbp->vst_uid = sbp->st_uid;
+	vsbp->vst_gid = sbp->st_gid;
+	vsbp->vst_atime = sbp->st_atimespec.tv_sec;
+	vsbp->vst_atimensec = sbp->st_atimespec.tv_nsec;
+	vsbp->vst_mtime = sbp->st_mtimespec.tv_sec;
+	vsbp->vst_mtimensec = sbp->st_mtimespec.tv_nsec;
+	vsbp->vst_ctime = sbp->st_ctimespec.tv_sec;
+	vsbp->vst_ctimensec = sbp->st_ctimespec.tv_nsec;
+	vsbp->vst_birthtime = sbp->st_birthtimespec.tv_sec;
+	vsbp->vst_birthtimensec = sbp->st_birthtimespec.tv_nsec;
+	vsbp->vst_size = sbp->st_size;
+	vsbp->vst_blocks = sbp->st_blocks;
+	vsbp->vst_blksize = sbp->st_blksize;
+	vsbp->vst_flags = sbp->st_flags;
+	vsbp->vst_gen = sbp->st_gen;
+	vsbp->vst_rdev = sbp->st_rdev;
+	vsbp->vst_qspare[0] = sbp->st_qspare[0];
+	vsbp->vst_qspare[1] = sbp->st_qspare[1];
+}
diff --git a/bsd/kern/pthread_support.c b/bsd/kern/pthread_support.c
new file mode 100644
index 000000000..6cbe8e0df
--- /dev/null
+++ b/bsd/kern/pthread_support.c
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1995-2005 Apple Computer, Inc. All Rights Reserved */
+/*
+ *	pthread_support.c
+ */
+
+
+#define  _PTHREAD_CONDATTR_T
+#define  _PTHREAD_COND_T
+#define _PTHREAD_MUTEXATTR_T
+#define _PTHREAD_MUTEX_T
+#define _PTHREAD_RWLOCKATTR_T
+#define _PTHREAD_RWLOCK_T
+
+#undef pthread_mutexattr_t
+#undef pthread_mutex_t
+#undef pthread_condattr_t
+#undef pthread_cond_t
+#undef pthread_rwlockattr_t
+#undef pthread_rwlock_t
+
+#include <sys/param.h>
+#include <sys/resourcevar.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/systm.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/acct.h>
+#include <sys/file_internal.h>
+#include <sys/kernel.h>
+#include <sys/wait.h>
+#include <sys/signalvar.h>
+#include <sys/syslog.h>
+#include <sys/stat.h>
+#include <sys/lock.h>
+#include <sys/kdebug.h>
+
+#include <sys/mount.h>
+#include <sys/sysproto.h>
+
+#include <sys/vm.h>
+#include <kern/task.h>	
+#include <kern/thread.h>
+
+#include <sys/pthread_internal.h>
+
+#define PTHREAD_SYNCH_MAX 256
+static pthread_mutex_t * pmutex_trans_array[PTHREAD_SYNCH_MAX];
+static pthread_cond_t * pcond_trans_array[PTHREAD_SYNCH_MAX];
+//static pthread_rwlock_t * prwlock_trans_array[PTHREAD_SYNCH_MAX];
+
+pthread_mutex_t * 
+pthread_id_to_mutex(int mutexid)
+{
+	pthread_mutex_t * mtx = NULL;
+
+
+	if (mutexid >= 0 && mutexid < PTHREAD_SYNCH_MAX) {
+			pthread_list_lock();
+			mtx = pmutex_trans_array[mutexid];
+			if (mtx) {
+				MTX_LOCK(mtx->lock);
+				mtx->refcount++;
+				MTX_UNLOCK(mtx->lock);
+			} 
+			pthread_list_unlock();
+	}
+	return(mtx);
+}
+
+
+int	
+pthread_id_mutex_add(pthread_mutex_t * mutex)
+{
+	int i;
+
+	pthread_list_lock();
+	for(i = 1; i < PTHREAD_SYNCH_MAX; i++) {
+		if (pmutex_trans_array[i] == 0) {
+			pmutex_trans_array[i]  = mutex;
+			break;
+		}
+	}
+	pthread_list_unlock();
+	if (i == PTHREAD_SYNCH_MAX)
+		return(0);
+	return(i);
+}
+
+
+void
+pthread_id_mutex_remove(int mutexid)
+{
+	pthread_list_lock();
+	if (pmutex_trans_array[mutexid]) {
+		pmutex_trans_array[mutexid] = 0;
+	}
+	pthread_list_unlock();
+}
+
+
+void 
+pthread_mutex_release(pthread_mutex_t * mutex)
+{
+	MTX_LOCK(mutex->lock);
+	mutex->refcount --;
+	MTX_UNLOCK(mutex->lock);
+}
+
+
+pthread_cond_t * 
+pthread_id_to_cond(int condid)
+{
+	pthread_cond_t * cond = NULL;
+
+
+	if (condid >= 0 && condid < PTHREAD_SYNCH_MAX) {
+			pthread_list_lock();
+			cond = pcond_trans_array[condid];
+			if (cond) {
+				COND_LOCK(cond->lock);
+				cond->refcount++;
+				COND_UNLOCK(cond->lock);
+			} 
+			pthread_list_unlock();
+	}
+	return(cond);
+}
+
+
+int	
+pthread_id_cond_add(pthread_cond_t * cond)
+{
+	int i;
+
+	pthread_list_lock();
+	for(i = 1; i < PTHREAD_SYNCH_MAX; i++) {
+		if (pcond_trans_array[i] == 0) {
+			pcond_trans_array[i]  = cond;
+			break;
+		}
+	}
+	pthread_list_unlock();
+	if (i == PTHREAD_SYNCH_MAX)
+		return(0);
+	return(i);
+}
+
+
+void
+pthread_id_cond_remove(int condid)
+{
+	pthread_list_lock();
+	if (pcond_trans_array[condid]) {
+		pcond_trans_array[condid] = 0;
+	}
+	pthread_list_unlock();
+}
+
+
+void 
+pthread_cond_release(pthread_cond_t * cond)
+{
+	COND_LOCK(cond->lock);
+	cond->refcount --;
+	COND_UNLOCK(cond->lock);
+}
+
diff --git a/bsd/kern/pthread_synch.c b/bsd/kern/pthread_synch.c
new file mode 100644
index 000000000..153a2fa81
--- /dev/null
+++ b/bsd/kern/pthread_synch.c
@@ -0,0 +1,2087 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1995-2005 Apple Computer, Inc. All Rights Reserved */
+/*
+ *	pthread_synch.c
+ */
+
+#define  _PTHREAD_CONDATTR_T
+#define  _PTHREAD_COND_T
+#define _PTHREAD_MUTEXATTR_T
+#define _PTHREAD_MUTEX_T
+#define _PTHREAD_RWLOCKATTR_T
+#define _PTHREAD_RWLOCK_T
+
+#undef pthread_mutexattr_t
+#undef pthread_mutex_t
+#undef pthread_condattr_t
+#undef pthread_cond_t
+#undef pthread_rwlockattr_t
+#undef pthread_rwlock_t
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/resourcevar.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/systm.h>
+#include <sys/timeb.h>
+#include <sys/times.h>
+#include <sys/acct.h>
+#include <sys/kernel.h>
+#include <sys/wait.h>
+#include <sys/signalvar.h>
+#include <sys/syslog.h>
+#include <sys/stat.h>
+#include <sys/lock.h>
+#include <sys/kdebug.h>
+#include <sys/sysproto.h>
+#include <sys/pthread_internal.h>
+#include <sys/vm.h>
+#include <sys/user.h>		/* for coredump */
+
+
+#include <mach/mach_types.h>
+#include <mach/vm_prot.h>
+#include <mach/semaphore.h>
+#include <mach/sync_policy.h>
+#include <mach/task.h>
+#include <kern/kern_types.h>
+#include <kern/task.h>
+#include <kern/clock.h>
+#include <mach/kern_return.h>
+#include <kern/thread.h>
+#include <kern/sched_prim.h>
+#include <kern/kalloc.h>
+#include <kern/sched_prim.h>	/* for thread_exception_return */
+#include <kern/processor.h>
+#include <kern/affinity.h>
+#include <mach/mach_vm.h>
+#include <mach/mach_param.h>
+#include <mach/thread_status.h>
+#include <mach/thread_policy.h>
+#include <mach/message.h>
+#include <mach/port.h>
+#include <vm/vm_protos.h>
+#include <vm/vm_map.h>`	/* for current_map() */
+#include <mach/thread_act.h> /* for thread_resume */
+#include <machine/machine_routines.h>
+#if defined(__i386__)
+#include <i386/machine_routines.h>
+#include <i386/eflags.h>
+#include <i386/psl.h>   
+#include <i386/seg.h>   
+#endif
+
+#include <libkern/OSAtomic.h>
+
+#if 0
+#undef KERNEL_DEBUG
+#define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT
+#undef KERNEL_DEBUG1
+#define KERNEL_DEBUG1 KERNEL_DEBUG_CONSTANT1
+#endif
+
+
+#if defined(__ppc__) || defined(__ppc64__)
+#include <architecture/ppc/cframe.h>
+#endif
+
+
+lck_grp_attr_t   *pthread_lck_grp_attr;
+lck_grp_t    *pthread_lck_grp;
+lck_attr_t   *pthread_lck_attr;
+lck_mtx_t * pthread_list_mlock;
+extern void pthread_init(void);
+
+extern kern_return_t thread_getstatus(register thread_t act, int flavor,
+			thread_state_t tstate, mach_msg_type_number_t *count);
+extern kern_return_t thread_setstatus(thread_t thread, int flavor,
+			thread_state_t tstate, mach_msg_type_number_t count);
+extern void thread_set_cthreadself(thread_t thread, uint64_t pself, int isLP64);
+extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t);
+extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t);
+
+static int workqueue_additem(struct workqueue *wq, int prio, user_addr_t item);
+static int workqueue_removeitem(struct workqueue *wq, int prio, user_addr_t item);
+static void workqueue_run_nextitem(proc_t p, thread_t th);
+static void wq_runitem(proc_t p, user_addr_t item, thread_t th, struct threadlist *tl,
+		       int reuse_thread, int wake_thread, int return_directly);
+static int setup_wqthread(proc_t p, thread_t th, user_addr_t item, int reuse_thread, struct threadlist *tl);
+static int  workqueue_addnewthread(struct workqueue *wq);
+static void workqueue_removethread(struct workqueue *wq);
+static void workqueue_lock(proc_t);
+static void workqueue_lock_spin(proc_t);
+static void workqueue_unlock(proc_t);
+
+#define C_32_STK_ALIGN          16
+#define C_64_STK_ALIGN          16
+#define C_64_REDZONE_LEN        128
+#define TRUNC_DOWN32(a,c)       ((((uint32_t)a)-(c)) & ((uint32_t)(-(c))))
+#define TRUNC_DOWN64(a,c)       ((((uint64_t)a)-(c)) & ((uint64_t)(-(c))))
+
+
+/*
+ * Flags filed passed to bsdthread_create and back in pthread_start 
+31  <---------------------------------> 0
+_________________________________________
+| flags(8) | policy(8) | importance(16) |
+-----------------------------------------
+*/
+void _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
+
+#define PTHREAD_START_CUSTOM	0x01000000
+#define PTHREAD_START_SETSCHED	0x02000000
+#define PTHREAD_START_DETACHED	0x04000000
+#define PTHREAD_START_POLICY_BITSHIFT 16
+#define PTHREAD_START_POLICY_MASK 0xffff
+#define PTHREAD_START_IMPORTANCE_MASK 0xffff
+
+#define SCHED_OTHER      POLICY_TIMESHARE
+#define SCHED_FIFO       POLICY_FIFO
+#define SCHED_RR         POLICY_RR
+
+void
+pthread_init(void)
+{
+
+	pthread_lck_grp_attr = lck_grp_attr_alloc_init();
+	pthread_lck_grp = lck_grp_alloc_init("pthread", pthread_lck_grp_attr);
+
+	/*
+	 * allocate the lock attribute for pthread synchronizers
+	 */
+	pthread_lck_attr = lck_attr_alloc_init();
+
+	pthread_list_mlock = lck_mtx_alloc_init(pthread_lck_grp, pthread_lck_attr);
+
+}
+
+void
+pthread_list_lock(void)
+{
+	lck_mtx_lock(pthread_list_mlock);
+}
+
+void
+pthread_list_unlock(void)
+{
+	lck_mtx_unlock(pthread_list_mlock);
+}
+
+
+int
+__pthread_mutex_destroy(__unused struct proc *p, struct __pthread_mutex_destroy_args *uap, __unused register_t *retval)
+{
+	int res;
+	int mutexid = uap->mutexid;
+	pthread_mutex_t * mutex;
+	lck_mtx_t * lmtx;
+	lck_mtx_t * lmtx1;
+	
+	
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0)
+		return(EINVAL);
+
+	MTX_LOCK(mutex->lock);
+	if (mutex->sig == _PTHREAD_KERN_MUTEX_SIG)
+	{
+		if (mutex->owner == (thread_t)NULL &&
+		    mutex->refcount == 1)
+		{
+			mutex->sig = _PTHREAD_NO_SIG;
+			lmtx = mutex->mutex;
+			lmtx1 = mutex->lock;
+			mutex->mutex = NULL;
+			pthread_id_mutex_remove(mutexid);
+			mutex->refcount --;
+			MTX_UNLOCK(mutex->lock);
+			lck_mtx_free(lmtx, pthread_lck_grp);
+			lck_mtx_free(lmtx1, pthread_lck_grp);
+			kfree((void *)mutex, sizeof(struct _pthread_mutex));
+			return(0);
+		}
+		else
+			res = EBUSY;
+	}
+	else
+		res = EINVAL;
+	MTX_UNLOCK(mutex->lock);
+	pthread_mutex_release(mutex);
+	return (res);
+}
+
+/*
+ * Initialize a mutex variable, possibly with additional attributes.
+ */
+static void
+pthread_mutex_init_internal(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
+{
+	mutex->prioceiling = attr->prioceiling;
+	mutex->protocol = attr->protocol;
+	mutex->type = attr->type;
+	mutex->pshared = attr->pshared;
+	mutex->refcount = 0;
+	mutex->owner = (thread_t)NULL;
+	mutex->owner_proc = current_proc();
+	mutex->sig = _PTHREAD_KERN_MUTEX_SIG;
+	mutex->lock = lck_mtx_alloc_init(pthread_lck_grp, pthread_lck_attr);
+	mutex->mutex = lck_mtx_alloc_init(pthread_lck_grp, pthread_lck_attr);
+}
+
+/*
+ * Initialize a mutex variable, possibly with additional attributes.
+ * Public interface - so don't trust the lock - initialize it first.
+ */
+int
+__pthread_mutex_init(__unused struct proc *p, struct __pthread_mutex_init_args *uap, __unused register_t *retval)
+{
+	user_addr_t umutex = uap->mutex;
+	pthread_mutex_t * mutex;
+	user_addr_t uattr = uap->attr;
+	pthread_mutexattr_t attr;
+	unsigned int addr = (unsigned int)((uintptr_t)uap->mutex);
+	int pmutex_sig;
+	int mutexid;
+	int error = 0;
+	
+	if ((umutex == 0) || (uattr == 0))
+		return(EINVAL);
+
+	if ((error = copyin(uattr, &attr, sizeof(pthread_mutexattr_t))))
+		return(error);
+
+	if (attr.sig != _PTHREAD_MUTEX_ATTR_SIG)
+			return (EINVAL);
+
+	if ((error = copyin(umutex, &pmutex_sig, sizeof(int))))
+		return(error);
+	
+	if (pmutex_sig == _PTHREAD_KERN_MUTEX_SIG)
+		return(EBUSY);
+	mutex = (pthread_mutex_t *)kalloc(sizeof(pthread_mutex_t));
+		
+	 pthread_mutex_init_internal(mutex, &attr);
+	
+	
+	addr += 8;
+	mutexid = pthread_id_mutex_add(mutex);
+	if (mutexid) {
+		if ((error = copyout(&mutexid, ((user_addr_t)((uintptr_t)(addr))), 4)))
+			goto cleanup;
+		return(0);
+	}  else	
+		error = ENOMEM;
+cleanup:
+	if(mutexid)			
+		pthread_id_mutex_remove(mutexid);
+	lck_mtx_free(mutex->lock, pthread_lck_grp);
+	lck_mtx_free(mutex->mutex, pthread_lck_grp);
+	kfree(mutex, sizeof(struct _pthread_mutex));
+	return(error);
+}
+
+/*
+ * Lock a mutex.
+ * TODO: Priority inheritance stuff
+ */
+int
+__pthread_mutex_lock(struct proc *p, struct __pthread_mutex_lock_args *uap, __unused register_t *retval)
+{
+	int mutexid = uap->mutexid;
+	pthread_mutex_t  * mutex;
+	int error;
+
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0)
+		return(EINVAL);
+
+	MTX_LOCK(mutex->lock);
+
+	if (mutex->sig != _PTHREAD_KERN_MUTEX_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != mutex->owner_proc) && (mutex->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	MTX_UNLOCK(mutex->lock);
+	
+	lck_mtx_lock(mutex->mutex);
+
+	MTX_LOCK(mutex->lock);
+	mutex->owner = current_thread();
+	error = 0;
+out:
+	MTX_UNLOCK(mutex->lock);
+	pthread_mutex_release(mutex);
+	return (error);
+}
+
+/*
+ * Attempt to lock a mutex, but don't block if this isn't possible.
+ */
+int
+__pthread_mutex_trylock(struct proc *p, struct __pthread_mutex_trylock_args *uap, __unused register_t *retval)
+{
+	int mutexid = uap->mutexid;
+	pthread_mutex_t  * mutex;
+	boolean_t state;
+	int error;
+
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0)
+		return(EINVAL);
+
+	MTX_LOCK(mutex->lock);
+
+	if (mutex->sig != _PTHREAD_KERN_MUTEX_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != mutex->owner_proc) && (mutex->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	MTX_UNLOCK(mutex->lock);
+	
+	state = lck_mtx_try_lock(mutex->mutex);
+	if (state) {
+		MTX_LOCK(mutex->lock);
+		mutex->owner = current_thread();
+		MTX_UNLOCK(mutex->lock);
+		error = 0;
+	} else
+		error = EBUSY;
+
+	pthread_mutex_release(mutex);
+	return (error);
+out:
+	MTX_UNLOCK(mutex->lock);
+	pthread_mutex_release(mutex);
+	return (error);
+}
+
+/*
+ * Unlock a mutex.
+ * TODO: Priority inheritance stuff
+ */
+int
+__pthread_mutex_unlock(struct proc *p, struct __pthread_mutex_unlock_args *uap, __unused register_t *retval)
+{
+	int mutexid = uap->mutexid;
+	pthread_mutex_t  * mutex;
+	int error;
+
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0)
+		return(EINVAL);
+
+	MTX_LOCK(mutex->lock);
+
+	if (mutex->sig != _PTHREAD_KERN_MUTEX_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != mutex->owner_proc) && (mutex->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	MTX_UNLOCK(mutex->lock);
+	
+	lck_mtx_unlock(mutex->mutex);
+
+	MTX_LOCK(mutex->lock);
+	mutex->owner = NULL;
+	error = 0;
+out:
+	MTX_UNLOCK(mutex->lock);
+	pthread_mutex_release(mutex);
+	return (error);
+}
+
+
+int       
+__pthread_cond_init(__unused struct proc *p, struct __pthread_cond_init_args *uap, __unused register_t *retval)
+{
+	pthread_cond_t * cond;
+	pthread_condattr_t attr;
+	user_addr_t ucond = uap->cond;
+	user_addr_t uattr = uap->attr;
+	unsigned int addr = (unsigned int)((uintptr_t)uap->cond);
+	int condid, error, cond_sig;
+	semaphore_t sem;
+	kern_return_t kret;
+	int value = 0;
+
+	if ((ucond == 0) || (uattr == 0))
+		return(EINVAL);
+
+	if ((error = copyin(uattr, &attr, sizeof(pthread_condattr_t))))
+		return(error);
+
+	if (attr.sig != _PTHREAD_COND_ATTR_SIG)
+			return (EINVAL);
+
+	if ((error = copyin(ucond, &cond_sig, sizeof(int))))
+		return(error);
+	
+	if (cond_sig == _PTHREAD_KERN_COND_SIG)
+		return(EBUSY);
+	kret = semaphore_create(kernel_task, &sem, SYNC_POLICY_FIFO, value);
+	if (kret != KERN_SUCCESS)
+		return(ENOMEM);
+
+	cond = (pthread_cond_t *)kalloc(sizeof(pthread_cond_t));
+		
+	cond->lock = lck_mtx_alloc_init(pthread_lck_grp, pthread_lck_attr);
+	cond->pshared = attr.pshared;
+	cond->sig = _PTHREAD_KERN_COND_SIG;
+	cond->sigpending = 0;
+	cond->waiters = 0;
+	cond->refcount = 0;
+	cond->mutex = (pthread_mutex_t *)0;
+	cond->owner_proc = current_proc();
+	cond->sem = sem;
+	
+	addr += 8;
+	condid = pthread_id_cond_add(cond);
+	if (condid) {
+		if ((error = copyout(&condid, ((user_addr_t)((uintptr_t)(addr))), 4)))
+			goto cleanup;
+		return(0);
+	}  else	
+		error = ENOMEM;
+cleanup:
+	if(condid)			
+		pthread_id_cond_remove(condid);
+	semaphore_destroy(kernel_task, cond->sem);
+	kfree(cond, sizeof(pthread_cond_t));
+	return(error);
+}
+
+
+/*
+ * Destroy a condition variable.
+ */
+int       
+__pthread_cond_destroy(__unused struct proc *p, struct __pthread_cond_destroy_args  *uap, __unused register_t *retval)
+{
+	pthread_cond_t *cond;
+	int condid = uap->condid;
+	semaphore_t sem;
+	lck_mtx_t * lmtx;
+	int res;
+	
+	cond = pthread_id_to_cond(condid);
+	if (cond == 0)
+		return(EINVAL);
+
+	COND_LOCK(cond->lock);
+	if (cond->sig == _PTHREAD_KERN_COND_SIG)
+	{
+		if (cond->refcount == 1)
+		{
+			cond->sig = _PTHREAD_NO_SIG;
+			sem = cond->sem;
+			cond->sem = NULL;
+			lmtx = cond->lock;
+			pthread_id_cond_remove(condid);
+			cond->refcount --;
+			COND_UNLOCK(cond->lock);
+			lck_mtx_free(lmtx, pthread_lck_grp);
+			(void)semaphore_destroy(kernel_task, sem);
+			kfree((void *)cond, sizeof(pthread_cond_t));
+			return(0);
+		}
+		else
+			res = EBUSY;
+	}
+	else
+		res = EINVAL;
+	COND_UNLOCK(cond->lock);
+	pthread_cond_release(cond);
+	return (res);
+}
+
+
+/*
+ * Signal a condition variable, waking up all threads waiting for it.
+ */
+int       
+__pthread_cond_broadcast(__unused struct proc *p, struct __pthread_cond_broadcast_args  *uap, __unused register_t *retval)
+{
+	int condid = uap->condid;
+	pthread_cond_t  * cond;
+	int error;
+	kern_return_t kret;
+
+	cond = pthread_id_to_cond(condid);
+	if (cond == 0)
+		return(EINVAL);
+
+	COND_LOCK(cond->lock);
+
+	if (cond->sig != _PTHREAD_KERN_COND_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != cond->owner_proc) && (cond->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	COND_UNLOCK(cond->lock);
+	
+	kret = semaphore_signal_all(cond->sem);
+    switch (kret) {
+    case KERN_INVALID_ADDRESS:
+    case KERN_PROTECTION_FAILURE:
+        error = EINVAL;
+        break;
+    case KERN_ABORTED:
+    case KERN_OPERATION_TIMED_OUT:
+        error = EINTR;
+        break;
+    case KERN_SUCCESS:
+        error = 0;
+        break;
+    default:
+        error = EINVAL;
+        break;
+    }
+
+	COND_LOCK(cond->lock);
+out:
+	COND_UNLOCK(cond->lock);
+	pthread_cond_release(cond);
+	return (error);
+}
+
+
+/*
+ * Signal a condition variable, waking only one thread.
+ */
+int
+__pthread_cond_signal(__unused struct proc *p, struct __pthread_cond_signal_args  *uap, __unused register_t *retval)
+{
+	int condid = uap->condid;
+	pthread_cond_t  * cond;
+	int error;
+	kern_return_t kret;
+
+	cond = pthread_id_to_cond(condid);
+	if (cond == 0)
+		return(EINVAL);
+
+	COND_LOCK(cond->lock);
+
+	if (cond->sig != _PTHREAD_KERN_COND_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != cond->owner_proc) && (cond->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	COND_UNLOCK(cond->lock);
+	
+	kret = semaphore_signal(cond->sem);
+    switch (kret) {
+    case KERN_INVALID_ADDRESS:
+    case KERN_PROTECTION_FAILURE:
+        error = EINVAL;
+        break;
+    case KERN_ABORTED:
+    case KERN_OPERATION_TIMED_OUT:
+        error = EINTR;
+        break;
+    case KERN_SUCCESS:
+        error = 0;
+        break;
+    default:
+        error = EINVAL;
+        break;
+    }
+
+	COND_LOCK(cond->lock);
+out:
+	COND_UNLOCK(cond->lock);
+	pthread_cond_release(cond);
+	return (error);
+}
+
+
+int       
+__pthread_cond_wait(__unused struct proc *p, struct __pthread_cond_wait_args  *uap, __unused register_t *retval)
+{
+	int condid = uap->condid;
+	pthread_cond_t  * cond;
+	int mutexid = uap->mutexid;
+	pthread_mutex_t  * mutex;
+	int error;
+	kern_return_t kret;
+
+	cond = pthread_id_to_cond(condid);
+	if (cond == 0)
+		return(EINVAL);
+
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0) {
+		pthread_cond_release(cond);
+		return(EINVAL);
+	}
+	COND_LOCK(cond->lock);
+
+	if (cond->sig != _PTHREAD_KERN_COND_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != cond->owner_proc) && (cond->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	COND_UNLOCK(cond->lock);
+	
+	kret = semaphore_wait(cond->sem);
+    switch (kret) {
+    case KERN_INVALID_ADDRESS:
+    case KERN_PROTECTION_FAILURE:
+        error = EACCES;
+        break;
+    case KERN_ABORTED:
+    case KERN_OPERATION_TIMED_OUT:
+        error = EINTR;
+        break;
+    case KERN_SUCCESS:
+        error = 0;
+        break;
+    default:
+        error = EINVAL;
+        break;
+    }
+
+	COND_LOCK(cond->lock);
+out:
+	COND_UNLOCK(cond->lock);
+	pthread_cond_release(cond);
+	pthread_mutex_release(mutex);
+	return (error);
+}
+
+int       
+__pthread_cond_timedwait(__unused struct proc *p, struct __pthread_cond_timedwait_args  *uap, __unused register_t *retval)
+{
+	int condid = uap->condid;
+	pthread_cond_t  * cond;
+	int mutexid = uap->mutexid;
+	pthread_mutex_t  * mutex;
+	mach_timespec_t absts;
+	int error;
+	kern_return_t kret;
+
+	absts.tv_sec = 0;
+	absts.tv_nsec = 0;
+
+	if (uap->abstime)
+		if ((error = copyin(uap->abstime, &absts, sizeof(mach_timespec_t ))))
+			return(error);
+	cond = pthread_id_to_cond(condid);
+	if (cond == 0)
+		return(EINVAL);
+
+	mutex = pthread_id_to_mutex(mutexid);
+	if (mutex == 0) {
+		pthread_cond_release(cond);
+		return(EINVAL);
+	}
+	COND_LOCK(cond->lock);
+
+	if (cond->sig != _PTHREAD_KERN_COND_SIG)
+	{
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((p != cond->owner_proc) && (cond->pshared != PTHREAD_PROCESS_SHARED)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	COND_UNLOCK(cond->lock);
+	
+	kret = semaphore_timedwait(cond->sem, absts);
+    switch (kret) {
+    case KERN_INVALID_ADDRESS:
+    case KERN_PROTECTION_FAILURE:
+        error = EACCES;
+        break;
+    case KERN_ABORTED:
+    case KERN_OPERATION_TIMED_OUT:
+        error = EINTR;
+        break;
+    case KERN_SUCCESS:
+        error = 0;
+        break;
+    default:
+        error = EINVAL;
+        break;
+    }
+
+	COND_LOCK(cond->lock);
+out:
+	COND_UNLOCK(cond->lock);
+	pthread_cond_release(cond);
+	pthread_mutex_release(mutex);
+	return (error);
+}
+
+int
+bsdthread_create(__unused struct proc *p, struct bsdthread_create_args  *uap, user_addr_t *retval)
+{
+	kern_return_t kret;
+	void * sright;
+	int error = 0;
+	int allocated = 0;
+	mach_vm_offset_t stackaddr;
+        mach_vm_size_t th_allocsize = 0;
+        mach_vm_size_t user_stacksize;
+        mach_vm_size_t th_stacksize;
+        mach_vm_offset_t th_stackaddr;
+        mach_vm_offset_t th_stack;
+        mach_vm_offset_t th_pthread;
+        mach_port_t th_thport;
+	thread_t th;
+	user_addr_t user_func = uap->func;
+	user_addr_t user_funcarg = uap->func_arg;
+	user_addr_t user_stack = uap->stack;
+	user_addr_t user_pthread = uap->pthread;
+	unsigned int  flags = (unsigned int)uap->flags;
+	vm_map_t vmap = current_map();
+	task_t ctask = current_task();
+	unsigned int policy, importance;
+	
+	int isLP64 = 0;
+
+
+#if 0
+	KERNEL_DEBUG_CONSTANT(0x9000080 | DBG_FUNC_START, flags, 0, 0, 0, 0);
+#endif
+
+	isLP64 = IS_64BIT_PROCESS(p);
+
+
+#if defined(__ppc__)
+	stackaddr = 0xF0000000;
+#elif defined(__i386__)
+	stackaddr = 0xB0000000;
+#elif defined(__arm__)
+	stackaddr = 0xB0000000; /* XXX ARM */
+#else
+#error Need to define a stack address hint for this architecture
+#endif
+	kret = thread_create(ctask, &th);
+	if (kret != KERN_SUCCESS)
+		return(ENOMEM);
+	thread_reference(th);
+
+	sright = (void *) convert_thread_to_port(th);
+	th_thport = (void *)ipc_port_copyout_send(sright, get_task_ipcspace(ctask));
+
+	if ((flags & PTHREAD_START_CUSTOM) == 0) {
+		th_stacksize = (mach_vm_size_t)user_stack;		/* if it is custom them it is stacksize */
+		th_allocsize = th_stacksize + PTH_DEFAULT_GUARDSIZE + p->p_pthsize;
+
+		kret = mach_vm_map(vmap, &stackaddr,
+    				th_allocsize,
+    				page_size-1,
+    				VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE , NULL,
+    				0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
+    				VM_INHERIT_DEFAULT);
+    		if (kret != KERN_SUCCESS)
+    			kret = mach_vm_allocate(vmap,
+    					&stackaddr, th_allocsize,
+    					VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE);
+    		if (kret != KERN_SUCCESS) {
+			error = ENOMEM;
+			goto out;
+    		}
+#if 0
+		KERNEL_DEBUG_CONSTANT(0x9000080 |DBG_FUNC_NONE, th_allocsize, stackaddr, 0, 2, 0);
+#endif
+		th_stackaddr = stackaddr;
+		allocated = 1;
+     		/*
+		 * The guard page is at the lowest address
+     		 * The stack base is the highest address
+		 */
+		kret = mach_vm_protect(vmap,  stackaddr, PTH_DEFAULT_GUARDSIZE, FALSE, VM_PROT_NONE);
+
+    		if (kret != KERN_SUCCESS) { 
+			error = ENOMEM;
+			goto out1;
+    		}
+		th_stack = (stackaddr + th_stacksize + PTH_DEFAULT_GUARDSIZE);
+		th_pthread = (stackaddr + th_stacksize + PTH_DEFAULT_GUARDSIZE);
+		user_stacksize = th_stacksize;
+	} else {
+		th_stack = user_stack;
+		user_stacksize = user_stack;
+		th_pthread = user_pthread;
+#if 0
+		KERNEL_DEBUG_CONSTANT(0x9000080 |DBG_FUNC_NONE, 0, 0, 0, 3, 0);
+#endif
+	}
+	
+#if defined(__ppc__)
+	/*
+	 * Set up PowerPC registers...
+	 * internally they are always kept as 64 bit and
+	 * since the register set is the same between 32 and 64bit modes
+	 * we don't need 2 different methods for setting the state
+	 */
+	{
+	        ppc_thread_state64_t state64;
+		ppc_thread_state64_t *ts64 = &state64;
+
+		ts64->srr0 = (uint64_t)p->p_threadstart;
+		ts64->r1 = (uint64_t)(th_stack - C_ARGSAVE_LEN - C_RED_ZONE);
+		ts64->r3 = (uint64_t)th_pthread;
+		ts64->r4 = (uint64_t)((unsigned int)th_thport);
+		ts64->r5 = (uint64_t)user_func;
+		ts64->r6 = (uint64_t)user_funcarg;
+		ts64->r7 = (uint64_t)user_stacksize;
+		ts64->r8 = (uint64_t)uap->flags;
+
+		thread_set_wq_state64(th, (thread_state_t)ts64);
+
+		thread_set_cthreadself(th, (uint64_t)th_pthread, isLP64);
+	}
+#elif defined(__i386__)
+	{
+        /*
+         * Set up i386 registers & function call.
+         */
+	if (isLP64 == 0) {
+		x86_thread_state32_t state;
+		x86_thread_state32_t *ts = &state;
+
+        	ts->eip = (int)p->p_threadstart;
+		ts->eax = (unsigned int)th_pthread;
+		ts->ebx = (unsigned int)th_thport;
+		ts->ecx = (unsigned int)user_func;
+		ts->edx = (unsigned int)user_funcarg;
+		ts->edi = (unsigned int)user_stacksize;
+		ts->esi = (unsigned int)uap->flags;
+		/*
+		 * set stack pointer
+		 */
+        	ts->esp = (int)((vm_offset_t)(th_stack-C_32_STK_ALIGN));
+
+		thread_set_wq_state32(th, (thread_state_t)ts);
+
+	} else {
+	        x86_thread_state64_t state64;
+		x86_thread_state64_t *ts64 = &state64;
+
+        	ts64->rip = (uint64_t)p->p_threadstart;
+		ts64->rdi = (uint64_t)th_pthread;
+		ts64->rsi = (uint64_t)((unsigned int)(th_thport));
+		ts64->rdx = (uint64_t)user_func;
+		ts64->rcx = (uint64_t)user_funcarg;
+		ts64->r8 = (uint64_t)user_stacksize;
+		ts64->r9 = (uint64_t)uap->flags;
+		/*
+		 * set stack pointer aligned to 16 byte boundary
+		 */
+		ts64->rsp = (uint64_t)(th_stack - C_64_REDZONE_LEN);
+
+		thread_set_wq_state64(th, (thread_state_t)ts64);
+	}
+	}
+#elif defined(__arm__)
+	{
+	int flavor=0, count=0;
+	void * state;
+
+	kret = thread_getstatus(th, flavor, (thread_state_t)&state, &count);
+	if (kret != KERN_SUCCESS) {
+		error = EINVAL;
+		goto out1;
+	}
+
+	/* XXX ARM TODO */
+
+	kret = thread_setstatus(th, flavor, (thread_state_t)&state, count);
+	if (kret != KERN_SUCCESS)
+		error = EINVAL;
+		goto out1;
+	}
+#else
+#error bsdthread_create  not defined for this architecture
+#endif
+	/* Set scheduling parameters if needed */
+	if ((flags & PTHREAD_START_SETSCHED) != 0) {
+		thread_extended_policy_data_t    extinfo;
+		thread_precedence_policy_data_t   precedinfo;
+
+		importance = (flags & PTHREAD_START_IMPORTANCE_MASK);
+		policy = (flags >> PTHREAD_START_POLICY_BITSHIFT) & PTHREAD_START_POLICY_MASK;
+
+		if (policy == SCHED_OTHER)
+			extinfo.timeshare = 1;
+		else
+			extinfo.timeshare = 0;
+		thread_policy_set(th, THREAD_EXTENDED_POLICY, (thread_policy_t)&extinfo, THREAD_EXTENDED_POLICY_COUNT);
+
+		precedinfo.importance = importance;
+		thread_policy_set(th, THREAD_PRECEDENCE_POLICY, (thread_policy_t)&precedinfo, THREAD_PRECEDENCE_POLICY_COUNT);
+	}
+
+	kret = thread_resume(th);
+	if (kret != KERN_SUCCESS) {
+		error = EINVAL;
+		goto out1;
+	}
+	thread_deallocate(th);	/* drop the creator reference */
+#if 0
+	KERNEL_DEBUG_CONSTANT(0x9000080 |DBG_FUNC_END, error, (unsigned int)th_pthread, 0, 0, 0);
+#endif
+	*retval = th_pthread;
+
+	return(0);
+
+out1:
+	if (allocated != 0)
+		(void)mach_vm_deallocate(vmap,  stackaddr, th_allocsize);
+out:
+	(void)mach_port_deallocate(get_task_ipcspace(ctask), (mach_port_name_t)th_thport);
+	(void)thread_terminate(th);
+	(void)thread_deallocate(th);
+	return(error);
+}
+
+int       
+bsdthread_terminate(__unused struct proc *p, struct bsdthread_terminate_args  *uap, __unused register_t *retval)
+{
+	mach_vm_offset_t  freeaddr;
+	mach_vm_size_t freesize;
+	kern_return_t kret;
+	mach_port_name_t kthport = (mach_port_name_t)uap->port;
+	mach_port_name_t sem = (mach_port_name_t)uap->sem;
+
+	freeaddr = (mach_vm_offset_t)uap->stackaddr;
+	freesize = uap->freesize;
+	
+#if 0
+	KERNEL_DEBUG_CONSTANT(0x9000084 |DBG_FUNC_START, (unsigned int)freeaddr, (unsigned int)freesize, (unsigned int)kthport, 0xff, 0);
+#endif
+	if (sem != MACH_PORT_NULL) {
+		 kret = semaphore_signal_internal_trap(sem);
+		if (kret != KERN_SUCCESS) {
+			return(EINVAL);
+		}
+	}
+	if ((freesize != (mach_vm_size_t)0) && (freeaddr != (mach_vm_offset_t)0)) {
+		kret = mach_vm_deallocate(current_map(), freeaddr, freesize);
+		if (kret != KERN_SUCCESS) {
+			return(EINVAL);
+		}
+	}
+	
+	(void) thread_terminate(current_thread());
+	if (kthport != MACH_PORT_NULL)
+			mach_port_deallocate(get_task_ipcspace(current_task()), kthport);
+	thread_exception_return();
+	panic("bsdthread_terminate: still running\n");
+#if 0
+	KERNEL_DEBUG_CONSTANT(0x9000084 |DBG_FUNC_END, 0, 0, 0, 0xff, 0);
+#endif
+	return(0);
+}
+
+
+int
+bsdthread_register(struct proc *p, struct bsdthread_register_args  *uap, __unused register_t *retval)
+{
+	/* syscall randomizer test can pass bogus values */
+	if (uap->pthsize > MAX_PTHREAD_SIZE) {
+		return(EINVAL);
+	}
+	p->p_threadstart = uap->threadstart;
+	p->p_wqthread = uap->wqthread;
+	p->p_pthsize = uap->pthsize;
+
+	return(0);
+}
+
+
+
+
+int wq_stalled_window_usecs	= WQ_STALLED_WINDOW_USECS;
+int wq_reduce_pool_window_usecs	= WQ_REDUCE_POOL_WINDOW_USECS;
+int wq_max_run_latency_usecs	= WQ_MAX_RUN_LATENCY_USECS;
+int wq_timer_interval_msecs	= WQ_TIMER_INTERVAL_MSECS;
+
+
+SYSCTL_INT(_kern, OID_AUTO, wq_stalled_window_usecs, CTLFLAG_RW,
+	   &wq_stalled_window_usecs, 0, "");
+
+SYSCTL_INT(_kern, OID_AUTO, wq_reduce_pool_window_usecs, CTLFLAG_RW,
+	   &wq_reduce_pool_window_usecs, 0, "");
+
+SYSCTL_INT(_kern, OID_AUTO, wq_max_run_latency_usecs, CTLFLAG_RW,
+	   &wq_max_run_latency_usecs, 0, "");
+
+SYSCTL_INT(_kern, OID_AUTO, wq_timer_interval_msecs, CTLFLAG_RW,
+	   &wq_timer_interval_msecs, 0, "");
+
+
+
+
+void
+workqueue_init_lock(proc_t p)
+{
+        lck_mtx_init(&p->p_wqlock, pthread_lck_grp, pthread_lck_attr);
+}
+
+void
+workqueue_destroy_lock(proc_t p)
+{
+	lck_mtx_destroy(&p->p_wqlock, pthread_lck_grp);
+}
+
+static void
+workqueue_lock(proc_t p)
+{
+	lck_mtx_lock(&p->p_wqlock);
+}
+
+static void
+workqueue_lock_spin(proc_t p)
+{
+	lck_mtx_lock_spin(&p->p_wqlock);
+}
+
+static void
+workqueue_unlock(proc_t p)
+{
+	lck_mtx_unlock(&p->p_wqlock);
+}
+
+
+
+static void
+workqueue_interval_timer_start(thread_call_t call, int interval_in_ms)
+{
+        uint64_t deadline;
+
+	clock_interval_to_deadline(interval_in_ms, 1000 * 1000, &deadline);
+
+	thread_call_enter_delayed(call, deadline);
+}
+
+
+static void
+workqueue_timer(struct workqueue *wq, __unused int param1)
+{
+        struct timeval tv, dtv;
+        uint32_t i;
+	boolean_t added_more_threads = FALSE;
+	boolean_t reset_maxactive = FALSE;
+	boolean_t restart_timer = FALSE;
+		
+        microuptime(&tv);
+
+        KERNEL_DEBUG(0xefffd108, (int)wq, 0, 0, 0, 0);
+
+	/*
+	 * check to see if the stall frequency was beyond our tolerance
+	 * or we have work on the queue, but haven't scheduled any 
+	 * new work within our acceptable time interval because
+	 * there were no idle threads left to schedule
+	 *
+	 * WQ_TIMER_WATCH will only be set if we have 1 or more affinity
+	 * groups that have stalled (no active threads and no idle threads)...
+	 * it will not be set if all affinity groups have at least 1 thread
+	 * that is currently runnable... if all processors have a runnable
+	 * thread, there is no need to add more threads even if we're not
+	 * scheduling new work within our allowed window... it just means
+	 * that the work items are taking a long time to complete.
+	 */
+	if (wq->wq_flags & (WQ_ADD_TO_POOL | WQ_TIMER_WATCH)) {
+
+		if (wq->wq_flags & WQ_ADD_TO_POOL)
+		        added_more_threads = TRUE;
+		else {
+		        timersub(&tv, &wq->wq_lastran_ts, &dtv);
+
+			if (((dtv.tv_sec * 1000000) + dtv.tv_usec) > wq_stalled_window_usecs)
+			        added_more_threads = TRUE;
+		}
+		if (added_more_threads == TRUE) {
+		        for (i = 0; i < wq->wq_affinity_max && wq->wq_nthreads < WORKQUEUE_MAXTHREADS; i++) {
+			        (void)workqueue_addnewthread(wq);
+			}
+		}
+	}
+	timersub(&tv, &wq->wq_reduce_ts, &dtv);
+
+	if (((dtv.tv_sec * 1000000) + dtv.tv_usec) > wq_reduce_pool_window_usecs)
+	        reset_maxactive = TRUE;
+
+	/*
+	 * if the pool size has grown beyond the minimum number
+	 * of threads needed to keep all of the processors busy, and
+	 * the maximum number of threads scheduled concurrently during
+	 * the last sample period didn't exceed half the current pool
+	 * size, then its time to trim the pool size back
+	 */
+	if (added_more_threads == FALSE &&
+	    reset_maxactive == TRUE &&
+	    wq->wq_nthreads > wq->wq_affinity_max &&
+	    wq->wq_max_threads_scheduled <= (wq->wq_nthreads / 2)) {
+		uint32_t nthreads_to_remove;
+		
+		if ((nthreads_to_remove = (wq->wq_nthreads / 4)) == 0)
+			nthreads_to_remove = 1;
+
+	        for (i = 0; i < nthreads_to_remove && wq->wq_nthreads > wq->wq_affinity_max; i++)
+		        workqueue_removethread(wq);
+	}
+	workqueue_lock_spin(wq->wq_proc);
+
+	if (reset_maxactive == TRUE) {
+	        wq->wq_max_threads_scheduled = 0;
+		microuptime(&wq->wq_reduce_ts);
+	}
+	if (added_more_threads) {
+	        wq->wq_flags &= ~(WQ_ADD_TO_POOL | WQ_TIMER_WATCH);
+
+	        /*
+		 * since we added more threads, we should be
+		 * able to run some work if its still available
+		 */
+	        workqueue_run_nextitem(wq->wq_proc, THREAD_NULL);
+		workqueue_lock_spin(wq->wq_proc);
+	}
+	if ((wq->wq_nthreads > wq->wq_affinity_max) ||
+	    (wq->wq_flags & WQ_TIMER_WATCH)) {
+	        restart_timer = TRUE;
+	} else
+	        wq->wq_flags &= ~WQ_TIMER_RUNNING;
+
+	workqueue_unlock(wq->wq_proc);
+
+	/*
+	 * we needed to knock down the WQ_TIMER_RUNNING flag while behind
+	 * the workqueue lock... however, we don't want to hold the lock
+	 * while restarting the timer and we certainly don't want 2 or more
+	 * instances of the timer... so set a local to indicate the need
+	 * for a restart since the state of wq_flags may change once we
+	 * drop the workqueue lock...
+	 */
+	if (restart_timer == TRUE)
+	        workqueue_interval_timer_start(wq->wq_timer_call, wq_timer_interval_msecs);
+}
+
+
+static void
+workqueue_callback(
+		   int		type,
+		   thread_t	thread)
+{
+	struct uthread    *uth;
+	struct threadlist *tl;
+	struct workqueue  *wq;
+
+	uth = get_bsdthread_info(thread);
+	tl  = uth->uu_threadlist;
+	wq  = tl->th_workq;
+
+        switch (type) {
+
+	      case SCHED_CALL_BLOCK:
+	        {
+		uint32_t	old_activecount;
+
+		old_activecount = OSAddAtomic(-1, (SInt32 *)&wq->wq_thactivecount[tl->th_affinity_tag]);
+
+		if (old_activecount == 1 && wq->wq_itemcount) {
+		        /*
+			 * we were the last active thread on this affinity set
+			 * and we've got work to do
+			 */
+		        workqueue_lock_spin(wq->wq_proc);
+			/*
+			 * if this thread is blocking (not parking)
+			 * and the idle list is empty for this affinity group
+			 * we'll count it as a 'stall'
+			 */
+			if ((tl->th_flags & TH_LIST_RUNNING) &&
+			    TAILQ_EMPTY(&wq->wq_thidlelist[tl->th_affinity_tag]))
+			        wq->wq_stalled_count++;
+
+			workqueue_run_nextitem(wq->wq_proc, THREAD_NULL);
+			/*
+			 * workqueue_run_nextitem will drop the workqueue
+			 * lock before it returns
+			 */
+		}
+	        KERNEL_DEBUG(0xefffd020, (int)thread, wq->wq_threads_scheduled, tl->th_affinity_tag, 0, 0);
+	        }
+		break;
+
+	      case SCHED_CALL_UNBLOCK:
+		/*
+		 * we cannot take the workqueue_lock here...
+		 * an UNBLOCK can occur from a timer event which
+		 * is run from an interrupt context... if the workqueue_lock
+		 * is already held by this processor, we'll deadlock...
+		 * the thread lock for the thread being UNBLOCKED
+		 * is also held
+		 */
+		if (tl->th_unparked)
+		        OSAddAtomic(-1, (SInt32 *)&tl->th_unparked);
+		else
+		        OSAddAtomic(1, (SInt32 *)&wq->wq_thactivecount[tl->th_affinity_tag]);
+
+		KERNEL_DEBUG(0xefffd024, (int)thread, wq->wq_threads_scheduled, tl->th_affinity_tag, 0, 0);
+		break;
+	}
+}
+
+static void
+workqueue_removethread(struct workqueue *wq)
+{
+        struct threadlist *tl;
+	uint32_t	i, affinity_tag = 0;
+
+	tl = NULL;
+
+	workqueue_lock_spin(wq->wq_proc);
+	
+	for (i = 0; i < wq->wq_affinity_max; i++) {
+
+	        affinity_tag = wq->wq_nextaffinitytag;
+
+		if (affinity_tag == 0)
+		        affinity_tag = wq->wq_affinity_max - 1;
+		else
+		        affinity_tag--;
+		wq->wq_nextaffinitytag = affinity_tag;
+
+		/*
+		 * look for an idle thread to steal from this affinity group
+		 * but don't grab the only thread associated with it
+		 */
+		if (!TAILQ_EMPTY(&wq->wq_thidlelist[affinity_tag]) && wq->wq_thcount[affinity_tag] > 1) {
+		        tl = TAILQ_FIRST(&wq->wq_thidlelist[affinity_tag]);
+			TAILQ_REMOVE(&wq->wq_thidlelist[affinity_tag], tl, th_entry);
+
+			wq->wq_nthreads--;
+			wq->wq_thcount[affinity_tag]--;
+
+			break;
+		}
+	}
+	workqueue_unlock(wq->wq_proc);
+
+	if (tl != NULL) {
+		thread_sched_call(tl->th_thread, NULL);
+	
+		if ( (tl->th_flags & TH_LIST_BLOCKED) )
+		        wakeup(tl);
+		else {
+			/*
+			 * thread was created, but never used... 
+			 * need to clean up the stack and port ourselves
+			 * since we're not going to spin up through the
+			 * normal exit path triggered from Libc
+			 */
+		        (void)mach_vm_deallocate(wq->wq_map, tl->th_stackaddr, tl->th_allocsize);
+			(void)mach_port_deallocate(get_task_ipcspace(wq->wq_task), (mach_port_name_t)tl->th_thport);
+
+		        thread_terminate(tl->th_thread);
+		}
+		KERNEL_DEBUG(0xefffd030, (int)tl->th_thread, wq->wq_nthreads, tl->th_flags & TH_LIST_BLOCKED, 0, 0);
+		/*
+		 * drop our ref on the thread
+		 */
+		thread_deallocate(tl->th_thread);
+
+		kfree(tl, sizeof(struct threadlist));
+	}
+}
+
+
+static int
+workqueue_addnewthread(struct workqueue *wq)
+{
+	struct threadlist *tl;
+	struct uthread	*uth;
+	kern_return_t	kret;
+	thread_t	th;
+	proc_t		p;
+	void 	 	*sright;
+	mach_vm_offset_t stackaddr;
+	uint32_t	affinity_tag;
+
+	p = wq->wq_proc;
+
+	kret = thread_create(wq->wq_task, &th);
+
+ 	if (kret != KERN_SUCCESS)
+	        return(EINVAL);
+
+	tl = kalloc(sizeof(struct threadlist));
+	bzero(tl, sizeof(struct threadlist));
+
+#if defined(__ppc__)
+	stackaddr = 0xF0000000;
+#elif defined(__i386__)
+	stackaddr = 0xB0000000;
+#elif defined(__arm__)
+	stackaddr = 0xB0000000; /* XXX ARM */
+#else
+#error Need to define a stack address hint for this architecture
+#endif
+	tl->th_allocsize = PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE + p->p_pthsize;
+
+	kret = mach_vm_map(wq->wq_map, &stackaddr,
+    			tl->th_allocsize,
+    			page_size-1,
+    			VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE , NULL,
+    			0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
+    			VM_INHERIT_DEFAULT);
+
+	if (kret != KERN_SUCCESS) {
+	        kret = mach_vm_allocate(wq->wq_map,
+    					&stackaddr, tl->th_allocsize,
+    					VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_ANYWHERE);
+	}
+	if (kret == KERN_SUCCESS) {
+	        /*
+		 * The guard page is at the lowest address
+		 * The stack base is the highest address
+		 */
+	        kret = mach_vm_protect(wq->wq_map, stackaddr, PTH_DEFAULT_GUARDSIZE, FALSE, VM_PROT_NONE);
+
+		if (kret != KERN_SUCCESS)
+		        (void) mach_vm_deallocate(wq->wq_map, stackaddr, tl->th_allocsize);
+	}
+	if (kret != KERN_SUCCESS) {
+		(void) thread_terminate(th);
+
+		kfree(tl, sizeof(struct threadlist));
+
+	        return(EINVAL);
+	}
+	thread_reference(th);
+
+	sright = (void *) convert_thread_to_port(th);
+	tl->th_thport = (void *)ipc_port_copyout_send(sright, get_task_ipcspace(wq->wq_task));
+
+	thread_static_param(th, TRUE);
+
+        workqueue_lock_spin(p);
+
+	affinity_tag = wq->wq_nextaffinitytag;
+	wq->wq_nextaffinitytag = (affinity_tag + 1) % wq->wq_affinity_max;
+
+        workqueue_unlock(p);
+
+	tl->th_flags = TH_LIST_INITED | TH_LIST_SUSPENDED;
+
+	tl->th_thread = th;
+	tl->th_workq = wq;
+	tl->th_stackaddr = stackaddr;
+	tl->th_affinity_tag = affinity_tag;
+
+#if defined(__ppc__)
+	//ml_fp_setvalid(FALSE);
+	thread_set_cthreadself(th, (uint64_t)(tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE), IS_64BIT_PROCESS(p));
+#endif /* __ppc__ */
+	/*
+	 * affinity tag of 0 means no affinity...
+	 * but we want our tags to be 0 based because they
+	 * are used to index arrays, so...
+	 * keep it 0 based internally and bump by 1 when
+	 * calling out to set it
+	 */
+	(void)thread_affinity_set(th, affinity_tag + 1);
+	thread_sched_call(th, workqueue_callback);
+
+	uth = get_bsdthread_info(tl->th_thread);
+	uth->uu_threadlist = (void *)tl;
+
+        workqueue_lock_spin(p);
+
+	TAILQ_INSERT_TAIL(&wq->wq_thidlelist[tl->th_affinity_tag], tl, th_entry);
+	wq->wq_nthreads++;
+	wq->wq_thcount[affinity_tag]++;
+
+	KERNEL_DEBUG1(0xefffd014 | DBG_FUNC_START, (int)current_thread(), affinity_tag, wq->wq_nthreads, 0, (int)tl->th_thread);
+
+	/*
+	 * work may have come into the queue while
+	 * no threads were available to run... since
+	 * we're adding a new thread, go evaluate the
+	 * current state
+	 */
+	workqueue_run_nextitem(p, THREAD_NULL);
+	/*
+	 * workqueue_run_nextitem is responsible for
+	 * dropping the workqueue lock in all cases
+	 */
+
+	return(0);
+}
+
+int
+workq_open(__unused struct proc *p, __unused struct workq_open_args  *uap, __unused register_t *retval)
+{
+	struct workqueue * wq;
+	int size;
+	char * ptr;
+	int j;
+	uint32_t i;
+	int error = 0;
+	int num_cpus;
+	struct workitem * witem;
+	struct workitemlist *wl;
+
+	workqueue_lock(p);	
+
+	if (p->p_wqptr == NULL) {
+	        num_cpus = ml_get_max_cpus();
+
+		size = (sizeof(struct workqueue)) +
+		       (num_cpus * sizeof(int *)) +
+		       (num_cpus * sizeof(TAILQ_HEAD(, threadlist)));
+
+		ptr = (char *)kalloc(size);
+		bzero(ptr, size);
+
+		wq = (struct workqueue *)ptr;
+		wq->wq_flags = WQ_LIST_INITED;
+		wq->wq_proc = p;
+		wq->wq_affinity_max = num_cpus;
+		wq->wq_task = current_task();
+		wq->wq_map  = current_map();
+
+		for (i = 0; i < WORKQUEUE_NUMPRIOS; i++) {
+		        wl = (struct workitemlist *)&wq->wq_list[i];
+			TAILQ_INIT(&wl->wl_itemlist);
+			TAILQ_INIT(&wl->wl_freelist);
+
+			for (j = 0; j < WORKITEM_SIZE; j++) {
+			        witem = &wq->wq_array[(i*WORKITEM_SIZE) + j];
+				TAILQ_INSERT_TAIL(&wl->wl_freelist, witem, wi_entry);
+			}
+		}
+		wq->wq_thactivecount = (uint32_t *)((char *)ptr + sizeof(struct workqueue));
+		wq->wq_thcount       = (uint32_t *)&wq->wq_thactivecount[wq->wq_affinity_max];
+		wq->wq_thidlelist    = (struct wq_thidlelist *)&wq->wq_thcount[wq->wq_affinity_max];
+
+		for (i = 0; i < wq->wq_affinity_max; i++)
+		        TAILQ_INIT(&wq->wq_thidlelist[i]);
+
+		TAILQ_INIT(&wq->wq_thrunlist);
+
+		p->p_wqptr = (void *)wq;
+		p->p_wqsize = size;
+
+		workqueue_unlock(p);
+
+		wq->wq_timer_call = thread_call_allocate((thread_call_func_t)workqueue_timer, (thread_call_param_t)wq);
+
+		for (i = 0; i < wq->wq_affinity_max; i++) {
+			(void)workqueue_addnewthread(wq);
+		}
+		/* If unable to create any threads, return error */
+		if (wq->wq_nthreads == 0)
+			error = EINVAL;
+		workqueue_lock_spin(p);
+
+		microuptime(&wq->wq_reduce_ts);
+		microuptime(&wq->wq_lastran_ts);
+		wq->wq_max_threads_scheduled = 0;
+		wq->wq_stalled_count = 0;
+	}
+	workqueue_unlock(p);
+
+	return(error);
+}
+
+int
+workq_ops(struct proc *p, struct workq_ops_args  *uap, __unused register_t *retval)
+{
+	int options	 = uap->options;
+	int prio	 = uap->prio;	/* should  be used to find the right workqueue */
+	user_addr_t item = uap->item;
+	int error = 0;
+	thread_t th = THREAD_NULL;
+        struct workqueue *wq;
+
+	prio += 2;	/* normalize prio -2 to +2 to 0 -4 */
+
+	switch (options) {
+
+		case WQOPS_QUEUE_ADD: {
+
+		        KERNEL_DEBUG(0xefffd008 | DBG_FUNC_NONE, (int)item, 0, 0, 0, 0);
+
+			workqueue_lock_spin(p);
+
+			if ((wq = (struct workqueue *)p->p_wqptr) == NULL) {
+			        workqueue_unlock(p);
+			        return (EINVAL);
+			}
+			error = workqueue_additem(wq, prio, item);
+			
+		        }
+			break;
+		case WQOPS_QUEUE_REMOVE: {
+
+			workqueue_lock_spin(p);
+
+			if ((wq = (struct workqueue *)p->p_wqptr) == NULL) {
+			        workqueue_unlock(p);
+			        return (EINVAL);
+			}
+		        error = workqueue_removeitem(wq, prio, item);
+			}
+			break;
+		case WQOPS_THREAD_RETURN: {
+
+		        th = current_thread();
+
+		        KERNEL_DEBUG(0xefffd004 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+			workqueue_lock_spin(p);
+
+			if ((wq = (struct workqueue *)p->p_wqptr) == NULL) {
+			        workqueue_unlock(p);
+			        return (EINVAL);
+			}
+		        }
+			break;
+		default:
+		        return (EINVAL);
+	}
+	workqueue_run_nextitem(p, th);
+	/*
+	 * workqueue_run_nextitem is responsible for
+	 * dropping the workqueue lock in all cases
+	 */
+	return(error);
+}
+
+void
+workqueue_exit(struct proc *p)
+{
+	struct workqueue  * wq;
+	struct threadlist  * tl, *tlist;
+	uint32_t i;
+
+	if (p->p_wqptr != NULL) {
+
+	        workqueue_lock_spin(p);
+
+	        wq = (struct workqueue *)p->p_wqptr;
+		p->p_wqptr = NULL;
+
+		workqueue_unlock(p);
+
+		if (wq == NULL)
+		        return;
+		
+		if (wq->wq_flags & WQ_TIMER_RUNNING)
+		        thread_call_cancel(wq->wq_timer_call);
+		thread_call_free(wq->wq_timer_call);
+
+		TAILQ_FOREACH_SAFE(tl, &wq->wq_thrunlist, th_entry, tlist) {
+		        /*
+			 * drop our last ref on the thread
+			 */
+		        thread_sched_call(tl->th_thread, NULL);
+		        thread_deallocate(tl->th_thread);
+
+			TAILQ_REMOVE(&wq->wq_thrunlist, tl, th_entry);
+			kfree(tl, sizeof(struct threadlist));
+		}
+		for (i = 0; i < wq->wq_affinity_max; i++) {
+		        TAILQ_FOREACH_SAFE(tl, &wq->wq_thidlelist[i], th_entry, tlist) {
+			        /*
+				 * drop our last ref on the thread
+				 */
+			        thread_sched_call(tl->th_thread, NULL);
+				thread_deallocate(tl->th_thread);
+
+				TAILQ_REMOVE(&wq->wq_thidlelist[i], tl, th_entry);
+				kfree(tl, sizeof(struct threadlist));
+			}
+		}
+		kfree(wq, p->p_wqsize);
+	}
+}
+
+static int 
+workqueue_additem(struct workqueue *wq, int prio, user_addr_t item)
+{
+	struct workitem	*witem;
+	struct workitemlist *wl;
+
+	wl = (struct workitemlist *)&wq->wq_list[prio];
+
+	if (TAILQ_EMPTY(&wl->wl_freelist))
+		return (ENOMEM);
+
+	witem = (struct workitem *)TAILQ_FIRST(&wl->wl_freelist);
+	TAILQ_REMOVE(&wl->wl_freelist, witem, wi_entry);
+
+	witem->wi_item = item;
+	TAILQ_INSERT_TAIL(&wl->wl_itemlist, witem, wi_entry);
+
+	if (wq->wq_itemcount == 0) {
+	        microuptime(&wq->wq_lastran_ts);
+		wq->wq_stalled_count = 0;
+	}
+	wq->wq_itemcount++;
+
+	return (0);
+}
+
+static int 
+workqueue_removeitem(struct workqueue *wq, int prio, user_addr_t item)
+{
+	struct workitem *witem;
+	struct workitemlist *wl;
+	int error = ESRCH;
+
+	wl = (struct workitemlist *)&wq->wq_list[prio];
+
+	TAILQ_FOREACH(witem, &wl->wl_itemlist, wi_entry) {
+		if (witem->wi_item == item) {
+			TAILQ_REMOVE(&wl->wl_itemlist, witem, wi_entry);
+			wq->wq_itemcount--;
+
+			witem->wi_item = (user_addr_t)0;
+			TAILQ_INSERT_HEAD(&wl->wl_freelist, witem, wi_entry);
+
+			error = 0;
+			break;
+		}
+	}
+	if (wq->wq_itemcount == 0)
+	        wq->wq_flags &= ~(WQ_ADD_TO_POOL | WQ_TIMER_WATCH);
+
+	return (error);
+}
+
+/*
+ * workqueue_run_nextitem:
+ *   called with the workqueue lock held...
+ *   responsible for dropping it in all cases
+ */
+static void
+workqueue_run_nextitem(proc_t p, thread_t thread)
+{
+        struct workqueue *wq;
+	struct workitem *witem = NULL;
+	user_addr_t item = 0;
+	thread_t th_to_run = THREAD_NULL;
+	thread_t th_to_park = THREAD_NULL;
+	int wake_thread = 0;
+	int reuse_thread = 1;
+	uint32_t stalled_affinity_count = 0;
+	int i;
+	uint32_t affinity_tag;
+	struct threadlist *tl = NULL;
+	struct uthread *uth = NULL;
+	struct workitemlist *wl;
+	boolean_t start_timer = FALSE;
+	struct timeval tv, lat_tv;
+
+	wq = (struct workqueue *)p->p_wqptr;
+
+	KERNEL_DEBUG(0xefffd000 | DBG_FUNC_START, (int)thread, wq->wq_threads_scheduled, wq->wq_stalled_count, 0, 0);
+
+	if (wq->wq_itemcount == 0) {
+	        if ((th_to_park = thread) == THREAD_NULL)
+		        goto out;
+	        goto parkit;
+	}
+	if (thread != THREAD_NULL) {
+	        /*
+		 * we're a worker thread from the pool... currently we
+		 * are considered 'active' which means we're counted
+		 * in "wq_thactivecount"
+		 */
+	        uth = get_bsdthread_info(thread);
+		tl = uth->uu_threadlist;
+
+		if (wq->wq_thactivecount[tl->th_affinity_tag] == 1) {
+		        /*
+			 * we're the only active thread associated with our
+			 * affinity group, so pick up some work and keep going
+			 */
+		        th_to_run = thread;
+			goto pick_up_work;
+		}
+	}
+	for (affinity_tag = 0; affinity_tag < wq->wq_affinity_max; affinity_tag++) {
+	        /*
+		 * look for first affinity group that is currently not active
+		 * and has at least 1 idle thread
+		 */
+	        if (wq->wq_thactivecount[affinity_tag] == 0) {
+			if (!TAILQ_EMPTY(&wq->wq_thidlelist[affinity_tag]))
+			        break;
+		        stalled_affinity_count++;
+		}
+	}
+	if (thread == THREAD_NULL) {
+	        /*
+		 * we're not one of the 'worker' threads
+		 */
+	        if (affinity_tag >= wq->wq_affinity_max) {
+		        /*
+			 * we've already got at least 1 thread per
+			 * affinity group in the active state... or
+			 * we've got no idle threads to play with
+			 */
+		        if (stalled_affinity_count) {
+
+				if ( !(wq->wq_flags & WQ_TIMER_RUNNING) ) {
+				        wq->wq_flags |= WQ_TIMER_RUNNING;
+					start_timer = TRUE;
+				}
+				wq->wq_flags |= WQ_TIMER_WATCH;
+			}
+			goto out;
+		}
+	} else {
+	        /*
+		 * we're overbooked on the affinity group we're associated with,
+		 * so park this thread 
+		 */
+	        th_to_park = thread;
+
+		if (affinity_tag >= wq->wq_affinity_max) {
+		        /*
+			 * all the affinity groups have active threads
+			 * running, or there are no idle threads to 
+			 * schedule
+			 */
+		        if (stalled_affinity_count) {
+
+				if ( !(wq->wq_flags & WQ_TIMER_RUNNING) ) {
+				        wq->wq_flags |= WQ_TIMER_RUNNING;
+					start_timer = TRUE;
+				}
+				wq->wq_flags |= WQ_TIMER_WATCH;
+			}
+		        goto parkit;
+		}
+		/*
+		 * we've got a candidate (affinity group with no currently
+		 * active threads) to start a new thread on...
+		 * we already know there is both work available
+		 * and an idle thread with the correct affinity tag, so
+		 * fall into the code that pulls a new thread and workitem...
+		 * once we've kicked that thread off, we'll park this one
+		 */
+	}
+	tl = TAILQ_FIRST(&wq->wq_thidlelist[affinity_tag]);
+	TAILQ_REMOVE(&wq->wq_thidlelist[affinity_tag], tl, th_entry);
+	
+	th_to_run = tl->th_thread;
+	TAILQ_INSERT_TAIL(&wq->wq_thrunlist, tl, th_entry);
+
+	if ((tl->th_flags & TH_LIST_SUSPENDED) == TH_LIST_SUSPENDED) {
+	        tl->th_flags &= ~TH_LIST_SUSPENDED;
+		reuse_thread = 0;
+	} else if ((tl->th_flags & TH_LIST_BLOCKED) == TH_LIST_BLOCKED) {
+	        tl->th_flags &= ~TH_LIST_BLOCKED;
+		wake_thread = 1;
+	}
+	tl->th_flags |= TH_LIST_RUNNING;
+
+        wq->wq_threads_scheduled++;
+
+	if (wq->wq_threads_scheduled > wq->wq_max_threads_scheduled)
+	        wq->wq_max_threads_scheduled = wq->wq_threads_scheduled;
+
+pick_up_work:
+	for (i = 0; i < WORKQUEUE_NUMPRIOS; i++) {
+	        wl = (struct workitemlist *)&wq->wq_list[i];
+
+		if (!(TAILQ_EMPTY(&wl->wl_itemlist))) {
+
+		        witem = TAILQ_FIRST(&wl->wl_itemlist);
+			TAILQ_REMOVE(&wl->wl_itemlist, witem, wi_entry);
+			wq->wq_itemcount--;
+
+			item = witem->wi_item;
+			witem->wi_item = (user_addr_t)0;
+			TAILQ_INSERT_HEAD(&wl->wl_freelist, witem, wi_entry);
+
+			break;
+		}
+	}
+	if (witem == NULL)
+	        panic("workq_run_nextitem: NULL witem");
+
+	if (thread != th_to_run) {
+	        /*
+		 * we're starting up a thread from a parked/suspended condition
+		 */
+	        OSAddAtomic(1, (SInt32 *)&wq->wq_thactivecount[tl->th_affinity_tag]);
+		OSAddAtomic(1, (SInt32 *)&tl->th_unparked);
+	}
+	if (wq->wq_itemcount == 0)
+		wq->wq_flags &= ~WQ_TIMER_WATCH;
+	else {
+	        microuptime(&tv);
+		/*
+		 * if we had any affinity groups stall (no threads runnable)
+		 * since we last scheduled an item... and
+		 * the elapsed time since we last scheduled an item
+		 * exceeds the latency tolerance...
+		 * we ask the timer thread (which should already be running)
+		 * to add some more threads to the pool
+		 */
+		if (wq->wq_stalled_count && !(wq->wq_flags & WQ_ADD_TO_POOL)) {
+		        timersub(&tv, &wq->wq_lastran_ts, &lat_tv);
+
+			if (((lat_tv.tv_sec * 1000000) + lat_tv.tv_usec) > wq_max_run_latency_usecs)
+			        wq->wq_flags |= WQ_ADD_TO_POOL;
+
+			KERNEL_DEBUG(0xefffd10c, wq->wq_stalled_count, lat_tv.tv_sec, lat_tv.tv_usec, wq->wq_flags, 0);
+		}
+		wq->wq_lastran_ts = tv;
+	}
+	wq->wq_stalled_count = 0;
+        workqueue_unlock(p);
+
+        KERNEL_DEBUG(0xefffd02c, wq->wq_thactivecount[0], wq->wq_thactivecount[1],
+		     wq->wq_thactivecount[2], wq->wq_thactivecount[3], 0);
+
+        KERNEL_DEBUG(0xefffd02c, wq->wq_thactivecount[4], wq->wq_thactivecount[5],
+		     wq->wq_thactivecount[6], wq->wq_thactivecount[7], 0);
+
+	/*
+	 * if current thread is reused for workitem, does not return via unix_syscall
+	 */
+	wq_runitem(p, item, th_to_run, tl, reuse_thread, wake_thread, (thread == th_to_run));
+	
+	if (th_to_park == THREAD_NULL) {
+
+	        KERNEL_DEBUG(0xefffd000 | DBG_FUNC_END, (int)thread, (int)item, wq->wq_flags, 1, 0);
+
+		return;
+	}
+	workqueue_lock_spin(p);
+
+parkit:
+	wq->wq_threads_scheduled--;
+	/*
+	 * this is a workqueue thread with no more
+	 * work to do... park it for now
+	 */
+	uth = get_bsdthread_info(th_to_park);
+	tl = uth->uu_threadlist;
+	if (tl == 0) 
+	        panic("wq thread with no threadlist ");
+	
+	TAILQ_REMOVE(&wq->wq_thrunlist, tl, th_entry);
+	tl->th_flags &= ~TH_LIST_RUNNING;
+
+	tl->th_flags |= TH_LIST_BLOCKED;
+	TAILQ_INSERT_HEAD(&wq->wq_thidlelist[tl->th_affinity_tag], tl, th_entry);
+
+	assert_wait((caddr_t)tl, (THREAD_INTERRUPTIBLE));
+
+	workqueue_unlock(p);
+
+	if (start_timer)
+		workqueue_interval_timer_start(wq->wq_timer_call, wq_timer_interval_msecs);
+
+	KERNEL_DEBUG1(0xefffd018 | DBG_FUNC_START, (int)current_thread(), wq->wq_threads_scheduled, 0, 0, (int)th_to_park);
+
+	thread_block((thread_continue_t)thread_exception_return);
+
+	panic("unexpected return from thread_block");
+
+out:
+	workqueue_unlock(p);
+
+	if (start_timer)
+		workqueue_interval_timer_start(wq->wq_timer_call, wq_timer_interval_msecs);
+
+	KERNEL_DEBUG(0xefffd000 | DBG_FUNC_END, (int)thread, 0, wq->wq_flags, 2, 0);
+
+	return;
+}
+
+static void 
+wq_runitem(proc_t p, user_addr_t item, thread_t th, struct threadlist *tl,
+	   int reuse_thread, int wake_thread, int return_directly)
+{
+	int ret = 0;
+
+	KERNEL_DEBUG1(0xefffd004 | DBG_FUNC_START, (int)current_thread(), (int)item, wake_thread, tl->th_affinity_tag, (int)th);
+
+	ret = setup_wqthread(p, th, item, reuse_thread, tl);
+
+	if (ret != 0)
+		panic("setup_wqthread failed  %x\n", ret);
+
+	if (return_directly) {
+		thread_exception_return();
+
+		panic("wq_runitem: thread_exception_return returned ...\n");
+	}
+	if (wake_thread) {
+		KERNEL_DEBUG1(0xefffd018 | DBG_FUNC_END, (int)current_thread(), 0, 0, 0, (int)th);
+	
+		wakeup(tl);
+	} else {
+	        KERNEL_DEBUG1(0xefffd014 | DBG_FUNC_END, (int)current_thread(), 0, 0, 0, (int)th);
+
+		thread_resume(th);
+	}
+}
+
+
+int
+setup_wqthread(proc_t p, thread_t th, user_addr_t item, int reuse_thread, struct threadlist *tl)
+{
+
+#if defined(__ppc__)
+	/*
+	 * Set up PowerPC registers...
+	 * internally they are always kept as 64 bit and
+	 * since the register set is the same between 32 and 64bit modes
+	 * we don't need 2 different methods for setting the state
+	 */
+	{
+	        ppc_thread_state64_t state64;
+		ppc_thread_state64_t *ts64 = &state64;
+
+		ts64->srr0 = (uint64_t)p->p_wqthread;
+		ts64->r1 = (uint64_t)((tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE) - C_ARGSAVE_LEN - C_RED_ZONE);
+		ts64->r3 = (uint64_t)(tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE);
+		ts64->r4 = (uint64_t)((unsigned int)tl->th_thport);
+		ts64->r5 = (uint64_t)(tl->th_stackaddr + PTH_DEFAULT_GUARDSIZE);
+		ts64->r6 = (uint64_t)item;
+		ts64->r7 = (uint64_t)reuse_thread;
+		ts64->r8 = (uint64_t)0;
+
+		thread_set_wq_state64(th, (thread_state_t)ts64);
+	}
+#elif defined(__i386__)
+	int isLP64 = 0;
+
+	isLP64 = IS_64BIT_PROCESS(p);
+        /*
+         * Set up i386 registers & function call.
+         */
+	if (isLP64 == 0) {
+		x86_thread_state32_t state;
+		x86_thread_state32_t *ts = &state;
+
+        	ts->eip = (int)p->p_wqthread;
+		ts->eax = (unsigned int)(tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE);
+		ts->ebx = (unsigned int)tl->th_thport;
+		ts->ecx = (unsigned int)(tl->th_stackaddr + PTH_DEFAULT_GUARDSIZE);
+		ts->edx = (unsigned int)item;
+		ts->edi = (unsigned int)reuse_thread;
+		ts->esi = (unsigned int)0;
+		/*
+		 * set stack pointer
+		 */
+        	ts->esp = (int)((vm_offset_t)((tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE) - C_32_STK_ALIGN));
+
+		thread_set_wq_state32(th, (thread_state_t)ts);
+
+	} else {
+	        x86_thread_state64_t state64;
+		x86_thread_state64_t *ts64 = &state64;
+
+        	ts64->rip = (uint64_t)p->p_wqthread;
+		ts64->rdi = (uint64_t)(tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE);
+		ts64->rsi = (uint64_t)((unsigned int)(tl->th_thport));
+		ts64->rdx = (uint64_t)(tl->th_stackaddr + PTH_DEFAULT_GUARDSIZE);
+		ts64->rcx = (uint64_t)item;
+		ts64->r8 = (uint64_t)reuse_thread;
+		ts64->r9 = (uint64_t)0;
+
+		/*
+		 * set stack pointer aligned to 16 byte boundary
+		 */
+		ts64->rsp = (uint64_t)((tl->th_stackaddr + PTH_DEFAULT_STACKSIZE + PTH_DEFAULT_GUARDSIZE) - C_64_REDZONE_LEN);
+
+		thread_set_wq_state64(th, (thread_state_t)ts64);
+	}
+#elif defined(__arm__)
+	arm_thread_state_t state;
+	arm_thread_state_t *ts = &state;
+
+	/* XXX ARM add more */
+	ts->pc = p->p_wqthread;
+	ts->sp = tl->th_stackaddr + PTH_DEFAULT_GUARDSIZE;
+
+	thread_set_wq_state32(th, (thread_state_t)ts);
+#else
+#error setup_wqthread  not defined for this architecture
+#endif
+	return(0);
+}
+
diff --git a/bsd/kern/qsort.c b/bsd/kern/qsort.c
index 9ac15c01b..8910df65e 100644
--- a/bsd/kern/qsort.c
+++ b/bsd/kern/qsort.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved
@@ -59,8 +65,9 @@
 
 #include <sys/types.h>
 //#include <stdlib.h>
+#include <libsa/stdlib.h>	/* qsort() */
 
-static inline char	*med3(char *, char *, char *, int (*)());
+static inline char	*med3(char *, char *, char *, int (*)(const void *, const void *));
 static inline void	 swapfunc(char *, char *, int, int);
 
 #define min(a, b)	(a) < (b) ? a : b
@@ -70,10 +77,10 @@ static inline void	 swapfunc(char *, char *, int, int);
  */
 #define swapcode(TYPE, parmi, parmj, n) { 		\
 	long i = (n) / sizeof (TYPE); 			\
-	register TYPE *pi = (TYPE *) (parmi); 		\
-	register TYPE *pj = (TYPE *) (parmj); 		\
+	TYPE *pi = (TYPE *) (parmi); 			\
+	TYPE *pj = (TYPE *) (parmj); 			\
 	do { 						\
-		register TYPE	t = *pi;		\
+		TYPE	t = *pi;			\
 		*pi++ = *pj;				\
 		*pj++ = t;				\
         } while (--i > 0);				\
@@ -83,9 +90,7 @@ static inline void	 swapfunc(char *, char *, int, int);
 	es % sizeof(long) ? 2 : es == sizeof(long)? 0 : 1;
 
 static inline void
-swapfunc(a, b, n, swaptype)
-	char *a, *b;
-	int n, swaptype;
+swapfunc(char *a, char *b, int n, int swaptype)
 {
 	if(swaptype <= 1) 
 		swapcode(long, a, b, n)
@@ -104,9 +109,7 @@ swapfunc(a, b, n, swaptype)
 #define vecswap(a, b, n) 	if ((n) > 0) swapfunc(a, b, n, swaptype)
 
 static inline char *
-med3(a, b, c, cmp)
-	char *a, *b, *c;
-	int (*cmp)();
+med3(char *a, char *b, char *c, int (*cmp)(const void *, const void *))
 {
 	return cmp(a, b) < 0 ?
 	       (cmp(b, c) < 0 ? b : (cmp(a, c) < 0 ? c : a ))
@@ -115,13 +118,11 @@ med3(a, b, c, cmp)
 
 __private_extern__
 void
-qsort(a, n, es, cmp)
-	void *a;
-	size_t n, es;
-	int (*cmp)();
+qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *))
 {
 	char *pa, *pb, *pc, *pd, *pl, *pm, *pn;
-	int d, r, swaptype, swap_cnt;
+	int d, swaptype, swap_cnt;
+	int r;
 
 loop:	SWAPINIT(a, es);
 	swap_cnt = 0;
@@ -183,11 +184,11 @@ loop:	SWAPINIT(a, es);
 	pn = (char *)a + n * es;
 	r = min(pa - (char *)a, pb - pa);
 	vecswap(a, pb - r, r);
-	r = min(pd - pc, pn - pd - es);
+	r = min((size_t)(pd - pc), pn - pd - es);
 	vecswap(pb, pn - r, r);
-	if ((r = pb - pa) > es)
+	if ((size_t)(r = pb - pa) > es)
 		qsort(a, r / es, es, cmp);
-	if ((r = pd - pc) > es) { 
+	if ((size_t)(r = pd - pc) > es) { 
 		/* Iterate rather than recurse to save stack space */
 		a = pn - r;
 		n = r / es;
diff --git a/bsd/kern/socket_info.c b/bsd/kern/socket_info.c
index 179f95104..e6e41f92b 100644
--- a/bsd/kern/socket_info.c
+++ b/bsd/kern/socket_info.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/types.h>
@@ -185,7 +191,7 @@ fill_socketinfo(struct socket *so, struct socket_info *si)
 				
 				ndrvsi->ndrvsi_if_family = ifp->if_family;
 				ndrvsi->ndrvsi_if_unit = ifp->if_unit;
-				strncpy(ndrvsi->ndrvsi_if_name, ifp->if_name, IFNAMSIZ);
+				strlcpy(ndrvsi->ndrvsi_if_name, ifp->if_name, IFNAMSIZ);
 			}
 
 			break;
@@ -217,7 +223,7 @@ fill_socketinfo(struct socket *so, struct socket_info *si)
 				kcsi->kcsi_recvbufsize = kctl->recvbufsize;
 				kcsi->kcsi_sendbufsize = kctl->sendbufsize;
 				kcsi->kcsi_unit = kcb->unit;
-				strncpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
+				strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
 			}
 			break;
 
diff --git a/bsd/kern/spl.c b/bsd/kern/spl.c
index 52768d76b..1078cdc8a 100644
--- a/bsd/kern/spl.c
+++ b/bsd/kern/spl.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <machine/machine_routines.h>
 #include <machine/spl.h>
@@ -111,19 +117,19 @@ spl0(void)
 }
 
 void
-spln(unsigned t)
+spln(__unused unsigned t)
 {
         return;
 }
 
 void
-splx(unsigned l)
+splx(__unused unsigned l)
 {
        return;
 }
 
 void
-splon(unsigned l)
+splon(__unused unsigned l)
 {
        return;
 }
diff --git a/bsd/kern/subr_log.c b/bsd/kern/subr_log.c
index 73dc5cc51..af0e42341 100644
--- a/bsd/kern/subr_log.c
+++ b/bsd/kern/subr_log.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -71,6 +77,14 @@
 #include <sys/kernel.h>
 #include <kern/thread.h>
 #include <sys/lock.h>
+#include <sys/signalvar.h>
+#include <sys/conf.h>
+#include <sys/sysctl.h>
+#include <kern/kalloc.h>
+
+/* XXX should be in a common header somewhere */
+extern void klogwakeup(void);
+extern void logwakeup(void);
 
 #define LOG_RDPRI	(PZERO + 1)
 
@@ -85,15 +99,23 @@ struct logsoftc {
 } logsoftc;
 
 int	log_open;			/* also used in log() */
-struct msgbuf temp_msgbuf;
+char smsg_bufc[MSG_BSIZE]; /* static buffer */
+struct msgbuf temp_msgbuf = {0,MSG_BSIZE,0,0,smsg_bufc};
 struct msgbuf *msgbufp;
 static int _logentrypend = 0;
 static int log_inited = 0;
-void bsd_log_lock(void);
-/* the following two are implemented in osfmk/kern/printf.c  */
+/* the following are implemented in osfmk/kern/printf.c  */
+extern void bsd_log_lock(void);
 extern void bsd_log_unlock(void);
 extern void bsd_log_init(void);
 
+/* XXX wants a linker set so these can be static */
+extern d_open_t         logopen;
+extern d_close_t        logclose;
+extern d_read_t         logread;
+extern d_ioctl_t        logioctl;
+extern d_select_t       logselect;
+
 /*
  * Serialize log access.  Note that the log can be written at interrupt level,
  * so any log manipulations that can be done from, or affect, another processor
@@ -105,10 +127,8 @@ extern void bsd_log_init(void);
 
 
 /*ARGSUSED*/
-logopen(dev, flags, mode, p)
-	dev_t dev;
-	int flags, mode;
-	struct proc *p;
+int
+logopen(__unused dev_t dev, __unused int flags, __unused int mode, struct proc *p)
 {
 	LOG_LOCK();
 	if (log_open) {
@@ -137,10 +157,8 @@ logopen(dev, flags, mode, p)
 
 /*ARGSUSED*/
 int
-logclose(dev, flag)
-	dev_t dev;
+logclose(__unused dev_t dev, __unused int flag, __unused int devtype, __unused struct proc *p)
 {
-	int oldpri;
 	LOG_LOCK();
 	log_open = 0;
 	selwakeup(&logsoftc.sc_selp);
@@ -151,16 +169,10 @@ logclose(dev, flag)
 
 /*ARGSUSED*/
 int
-logread(dev, uio, flag)
-	dev_t dev;
-	struct uio *uio;
-	int flag;
+logread(__unused dev_t dev, struct uio *uio, int flag)
 {
 	register long l;
-	register int s;
 	int error = 0;
-	char localbuff[MSG_BSIZE];
-	int copybytes;
 
 	LOG_LOCK();
 	while (msgbufp->msg_bufr == msgbufp->msg_bufx) {
@@ -179,8 +191,8 @@ logread(dev, uio, flag)
 		 * if there are any new characters. If that doesn't do it
 		 * then wait for 5 sec and reevaluate 
 		 */
-		if (error = tsleep((caddr_t)msgbufp, LOG_RDPRI | PCATCH,
-				"klog", 5 * hz)) {
+		if ((error = tsleep((caddr_t)msgbufp, LOG_RDPRI | PCATCH,
+				"klog", 5 * hz)) != 0) {
 			/* if it times out; ignore */
 			if (error != EWOULDBLOCK)
 				return (error);
@@ -189,23 +201,21 @@ logread(dev, uio, flag)
 	}
 	logsoftc.sc_state &= ~LOG_RDWAIT;
 
-
 	while (uio_resid(uio) > 0) {
 		l = msgbufp->msg_bufx - msgbufp->msg_bufr;
 		if (l < 0)
-			l = MSG_BSIZE - msgbufp->msg_bufr;
+			l = msgbufp->msg_size - msgbufp->msg_bufr;
 		l = min(l, uio_resid(uio));
 		if (l == 0)
 			break;
-		bcopy(&msgbufp->msg_bufc[msgbufp->msg_bufr], &localbuff[0], l);
 		LOG_UNLOCK();
-		error = uiomove((caddr_t)&localbuff[0],
+		error = uiomove((caddr_t)&msgbufp->msg_bufc[msgbufp->msg_bufr],
 			(int)l, uio);
 		LOG_LOCK();
 		if (error)
 			break;
 		msgbufp->msg_bufr += l;
-		if (msgbufp->msg_bufr < 0 || msgbufp->msg_bufr >= MSG_BSIZE)
+		if (msgbufp->msg_bufr < 0 || msgbufp->msg_bufr >= msgbufp->msg_size)
 			msgbufp->msg_bufr = 0;
 	}
 out:
@@ -215,13 +225,8 @@ out:
 
 /*ARGSUSED*/
 int
-logselect(dev, rw, wql, p)
-	dev_t dev;
-	int rw;
-	void * wql;
-	struct proc *p;
+logselect(__unused dev_t dev, int rw, void * wql, struct proc *p)
 {
-
 	switch (rw) {
 
 	case FREAD:
@@ -238,11 +243,9 @@ logselect(dev, rw, wql, p)
 }
 
 void
-logwakeup()
+logwakeup(void)
 {
-	struct proc *p;
 	int pgid;
-	boolean_t funnel_state;
 
 	LOG_LOCK();	
 	if (!log_open) {
@@ -255,8 +258,8 @@ logwakeup()
 		LOG_UNLOCK();
 		if (pgid < 0)
 			gsignal(-pgid, SIGIO); 
-		else if (p = pfind(pgid))
-			psignal(p, SIGIO);
+		else 
+			proc_signal(pgid, SIGIO);
 		LOG_LOCK();
 	}
 	if (logsoftc.sc_state & LOG_RDWAIT) {
@@ -267,9 +270,8 @@ logwakeup()
 }
 
 void
-klogwakeup()
+klogwakeup(void)
 {
-
 	if (_logentrypend) {
 		_logentrypend = 0;
 		logwakeup();
@@ -278,20 +280,18 @@ klogwakeup()
 
 /*ARGSUSED*/
 int
-logioctl(dev, com, data, flag)
-	caddr_t data;
+logioctl(__unused dev_t dev, u_long com, caddr_t data, __unused int flag, __unused struct proc *p)
 {
 	long l;
-	int s;
 
-	LOG_LOCK();	
+	LOG_LOCK();
 	switch (com) {
 
 	/* return number of characters immediately available */
 	case FIONREAD:
 		l = msgbufp->msg_bufx - msgbufp->msg_bufr;
 		if (l < 0)
-			l += MSG_BSIZE;
+			l += msgbufp->msg_size;
 		*(off_t *)data = l;
 		break;
 
@@ -326,7 +326,7 @@ logioctl(dev, com, data, flag)
 }
 
 void
-bsd_log_init()
+bsd_log_init(void)
 {
 	if (!log_inited) { 
 		msgbufp = &temp_msgbuf;
@@ -334,18 +334,33 @@ bsd_log_init()
 	}
 }
 
+
+/*
+ * log_putc_locked
+ *
+ * Decription:	Output a character to the log; assumes the LOG_LOCK() is held
+ *		by the caller.
+ *
+ * Parameters:	c				Character to output
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This functions is used for multibyte output to the log; it
+ *		should be used preferrentially where possible to ensure that
+ *		log entries do not end up interspersed due to preemption or
+ *		SMP reentrancy.
+ */
 void
-log_putc(char c)
+log_putc_locked(char c)
 {
 	register struct msgbuf *mbp;
 
 	if (!log_inited) {
 		panic("bsd log is not inited");
 	}
-	LOG_LOCK();
 
 	mbp = msgbufp; 
-	if (mbp-> msg_magic != MSG_MAGIC) { 
+	if (mbp-> msg_magic != MSG_MAGIC) {
 		register int i;
 
 		mbp->msg_magic = MSG_MAGIC;
@@ -355,8 +370,139 @@ log_putc(char c)
 	}
 	mbp->msg_bufc[mbp->msg_bufx++] = c;
 	_logentrypend = 1;
-	if (mbp->msg_bufx < 0 || mbp->msg_bufx >= MSG_BSIZE)
+	if (mbp->msg_bufx < 0 || mbp->msg_bufx >= msgbufp->msg_size)
 		mbp->msg_bufx = 0;
+}
+
+
+/*
+ * log_putc
+ *
+ * Decription:	Output a character to the log; assumes the LOG_LOCK() is NOT
+ *		held by the caller.
+ *
+ * Parameters:	c				Character to output
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This function is used for syingle byte output to the log.  It
+ *		primarily exists to maintain binary backward compatibility.
+ */
+void
+log_putc(char c)
+{
+	if (!log_inited) {
+		panic("bsd log is not inited");
+	}
+	LOG_LOCK();
+	log_putc_locked(c);
 	LOG_UNLOCK();
 }
 
+
+/*
+ * it is possible to increase the kernel log buffer size by adding
+ *   msgbuf=n
+ * to the kernel command line, and to read the current size using
+ *   sysctl kern.msgbuf
+ * If there is no parameter on the kernel command line, the buffer is
+ * allocated statically and is MSG_BSIZE characters in size, otherwise
+ * memory is dynamically allocated.
+ * This function may only be called once, during kernel initialization.
+ * Memory management must already be up. The buffer must not have
+ * overflown yet.
+ */
+void
+log_setsize(long size) {
+	char *new_logdata;
+	if (msgbufp->msg_size!=MSG_BSIZE) {
+		printf("log_setsize: attempt to change size more than once\n");
+		return;
+	}
+	if (size==MSG_BSIZE)
+		return;
+	if (size<MSG_BSIZE) { /* we don't support reducing the log size */
+		printf("log_setsize: can't decrease log size\n");
+		return;
+	}
+	if (!(new_logdata = (char*)kalloc(size))) {
+		printf("log_setsize: unable to allocate memory\n");
+		return;
+	}
+	LOG_LOCK();
+	bcopy(smsg_bufc, new_logdata, MSG_BSIZE);
+	bzero(new_logdata+MSG_BSIZE, size - MSG_BSIZE);
+	/* this memory is now dead - clear it so that it compresses better
+	   in case of suspend to disk etc. */
+	bzero(smsg_bufc, MSG_BSIZE);
+	msgbufp->msg_size = size;
+	msgbufp->msg_bufc = new_logdata;
+	LOG_UNLOCK();
+	printf("set system log size to %ld bytes\n", msgbufp->msg_size);
+}
+
+SYSCTL_LONG(_kern, OID_AUTO, msgbuf, CTLFLAG_RD, &temp_msgbuf.msg_size, "");
+
+/*
+ * This should be called by single user mode /sbin/dmesg only.
+ * It returns as much data still in the buffer as possible.
+ */
+int
+log_dmesg(user_addr_t buffer, uint32_t buffersize, register_t * retval) {
+	unsigned long i;
+	int error = 0, newl, skip;
+	char *localbuff, *p, *copystart, ch;
+	long localbuff_size = msgbufp->msg_size+2, copysize;
+
+	if (!(localbuff = (char *)kalloc(localbuff_size))) {
+		printf("log_dmesg: unable to allocate memory\n");
+		return (ENOMEM);
+	}
+	/* in between here, the log could become bigger, but that's fine */
+	LOG_LOCK();
+
+	/*
+	 * The message buffer is circular; start at the write pointer, and
+	 * make one loop up to write pointer - 1.
+	 */
+	p = msgbufp->msg_bufc + msgbufp->msg_bufx;
+	for (i = newl = skip = 0; p != msgbufp->msg_bufc + msgbufp->msg_bufx - 1; ++p) {
+		if (p >= msgbufp->msg_bufc + msgbufp->msg_size)
+			p = msgbufp->msg_bufc;
+		ch = *p;
+		/* Skip "\n<.*>" syslog sequences. */
+		if (skip) {
+			if (ch == '>')
+				newl = skip = 0;
+			continue;
+		}
+		if (newl && ch == '<') {
+			skip = 1;
+			continue;
+		}
+		if (ch == '\0')
+			continue;
+		newl = ch == '\n';
+		localbuff[i++] = ch;
+	}
+	if (!newl)
+		localbuff[i++] = '\n';
+	localbuff[i++] = 0;
+
+	if (buffersize >= i) {
+		copystart = localbuff;
+		copysize = i;
+	} else {
+		copystart = localbuff + i - buffersize;
+		copysize = buffersize;
+	}
+
+	LOG_UNLOCK();
+
+	error = copyout(copystart, buffer, copysize);
+	if (!error)
+		*retval = copysize;
+
+	kfree(localbuff, localbuff_size);
+	return (error);
+}
diff --git a/bsd/kern/subr_prf.c b/bsd/kern/subr_prf.c
index 3b77fb394..4a1860496 100644
--- a/bsd/kern/subr_prf.c
+++ b/bsd/kern/subr_prf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -95,6 +101,13 @@
 #include <machine/spl.h>
 #include <libkern/libkern.h>
 
+/* for vaddlog(): the following are implemented in osfmk/kern/printf.c  */
+extern void bsd_log_lock(void);
+extern void bsd_log_unlock(void);
+
+/* Keep this around only because it's exported */
+void _printf(int, struct tty *, const char *, ...);
+
 struct snprintf_arg {
 	char *str;
 	size_t remain;
@@ -108,32 +121,28 @@ struct snprintf_arg {
  */
 extern const char	*panicstr;
 
-extern	cnputc();			/* standard console putc */
-int	(*v_putc)() = cnputc;		/* routine to putc on virtual console */
+extern	void cnputc(char);		/* standard console putc */
+void	(*v_putc)(char) = cnputc;	/* routine to putc on virtual console */
 
 extern	struct tty cons;		/* standard console tty */
 extern struct	tty *constty;		/* pointer to console "window" tty */
 extern int  __doprnt(const char *fmt,
 					 va_list    *argp,
-					 void       (*putc)(int, void *arg),
+					 void       (*)(int, void *),
 					 void       *arg,
 					 int        radix);
 
 /*
  *	Record cpu that panic'd and lock around panic data
  */
-
-static void puts(const char *s, int flags, struct tty *ttyp);
 static void printn(u_long n, int b, int flags, struct tty *ttyp, int zf, int fld_size);
 
 #if	NCPUS > 1
 boolean_t new_printf_cpu_number;  /* do we need to output who we are */
 #endif
 
-extern	void logwakeup();
-extern	void halt_cpu();
-extern	boot();
-
+extern	void logwakeup(void);
+extern	void halt_cpu(void);
 
 static void
 snprintf_func(int ch, void *arg);
@@ -153,37 +162,48 @@ static void putchar(int c, void *arg);
 void
 uprintf(const char *fmt, ...)
 {
-	register struct proc *p = current_proc();
+	struct proc *p = current_proc();
 	struct putchar_args pca;
 	va_list ap;
+	struct session * sessp;
+	boolean_t fstate;
 	
+	sessp = proc_session(p);
+
+	fstate = thread_funnel_set(kernel_flock, TRUE);
 	pca.flags = TOTTY;
-	pca.tty   = (struct tty *)p->p_session->s_ttyp;
+	pca.tty   = (struct tty *)sessp->s_ttyp;
 
-	if (p->p_flag & P_CONTROLT && p->p_session->s_ttyvp) {
+	if (p->p_flag & P_CONTROLT && sessp->s_ttyvp) {
 		va_start(ap, fmt);
 		__doprnt(fmt, &ap, putchar, &pca, 10);
 		va_end(ap);
 	}
+	(void) thread_funnel_set(kernel_flock, fstate);
+	session_rele(sessp);
 }
 
 tpr_t
-tprintf_open(p)
-	register struct proc *p;
+tprintf_open(struct proc *p)
 {
-	if (p->p_flag & P_CONTROLT && p->p_session->s_ttyvp) {
-		SESSHOLD(p->p_session);
-		return ((tpr_t) p->p_session);
+	struct session * sessp;
+
+	sessp = proc_session(p);
+
+	if (p->p_flag & P_CONTROLT && sessp->s_ttyvp) {
+		return ((tpr_t)sessp);
 	}
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+
 	return ((tpr_t) NULL);
 }
 
 void
-tprintf_close(sess)
-	tpr_t sess;
+tprintf_close(tpr_t sessp)
 {
-	if (sess)
-		SESSRELE((struct session *) sess);
+	if (sessp)
+		session_rele((struct session *) sessp);
 }
 
 /*
@@ -193,13 +213,17 @@ tprintf_close(sess)
 void
 tprintf(tpr_t tpr, const char *fmt, ...)
 {
-	register struct session *sess = (struct session *)tpr;
+	struct session *sess = (struct session *)tpr;
 	struct tty *tp = NULL;
 	int flags = TOLOG;
 	va_list ap;
 	struct putchar_args pca;
+	boolean_t fstate;
 
 	logpri(LOG_INFO);
+	/* to protect tty */
+	fstate = thread_funnel_set(kernel_flock, TRUE);
+
 	if (sess && sess->s_ttyvp && ttycheckoutq(sess->s_ttyp, 0)) {
 		flags |= TOTTY;
 		tp = sess->s_ttyp;
@@ -211,6 +235,8 @@ tprintf(tpr_t tpr, const char *fmt, ...)
 	__doprnt(fmt, &ap, putchar, &pca, 10);
 	va_end(ap);
 
+	(void) thread_funnel_set(kernel_flock, fstate);
+
 	logwakeup();
 }
 
@@ -223,8 +249,10 @@ void
 ttyprintf(struct tty *tp, const char *fmt, ...)
 {
 	va_list ap;
+	boolean_t fstate;
 
 	if (tp != NULL) {
+		fstate = thread_funnel_set(kernel_flock, TRUE);
 		struct putchar_args pca;
 		pca.flags = TOTTY;
 		pca.tty   = tp;
@@ -232,6 +260,7 @@ ttyprintf(struct tty *tp, const char *fmt, ...)
 		va_start(ap, fmt);
 		__doprnt(fmt, &ap, putchar, &pca, 10);
 		va_end(ap);
+		(void) thread_funnel_set(kernel_flock, fstate);
 	}
 }
 
@@ -239,8 +268,7 @@ extern	int log_open;
 
 
 void
-logpri(level)
-	int level;
+logpri(int level)
 {
 	struct putchar_args pca;
 	pca.flags = TOLOG;
@@ -251,38 +279,56 @@ logpri(level)
 	putchar('>', &pca);
 }
 
-void
-addlog(const char *fmt, ...)
+static void
+_logtime(const char *fmt, ...)
 {
-	register s = splhigh();
 	va_list ap;
+	va_start(ap, fmt);
+	vaddlog(fmt, ap);
+	va_end(ap);
+}
+
+void
+logtime(time_t secs)
+{
+	_logtime("         0 [Time %ld] [Message ", secs);
+}
+
+int
+vaddlog(const char *fmt, va_list ap)
+{
 	struct putchar_args pca;
 
-	pca.flags = TOLOG;
+	pca.flags = TOLOGLOCKED;
 	pca.tty   = NULL;
 
-	va_start(ap, fmt);
-	__doprnt(fmt, &ap, putchar, &pca, 10);
-	
-	splx(s);
 	if (!log_open) {
-		pca.flags = TOCONS;
-		__doprnt(fmt, &ap, putchar, &pca, 10);
+		pca.flags |= TOCONS;
 	}
-	va_end(ap);
+
+	bsd_log_lock();
+	__doprnt(fmt, &ap, putchar, &pca, 10);
+	bsd_log_unlock();
+	
 	logwakeup();
+	return 0;
 }
-void _printf(int flags, struct tty *ttyp, const char *format, ...)
+
+void
+_printf(int flags, struct tty *ttyp, const char *format, ...)
 {
 	va_list ap;
 	struct putchar_args pca;
+	boolean_t fstate;
 
 	pca.flags = flags;
 	pca.tty   = ttyp;
+	fstate = thread_funnel_set(kernel_flock, TRUE);
 	
 	va_start(ap, format);
 	__doprnt(format, &ap, putchar, &pca, 10);
 	va_end(ap);
+	(void) thread_funnel_set(kernel_flock, fstate);
 }
 
 int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp)
@@ -320,18 +366,6 @@ int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp)
 	return 0;
 }
 
-static void puts(const char *s, int flags, struct tty *ttyp)
-{
-	register char c;
-	struct putchar_args pca;
-
-	pca.flags = flags;
-	pca.tty   = ttyp;
-
-	while ((c = *s++))
-		putchar(c, &pca);
-}
-
 /*
  * Printn prints a number n in base b.
  * We don't use recursion to avoid deep kernel stacks.
@@ -339,7 +373,7 @@ static void puts(const char *s, int flags, struct tty *ttyp)
 static void printn(u_long n, int b, int flags, struct tty *ttyp, int zf, int fld_size)
 {
 	char prbuf[11];
-	register char *cp;
+	char *cp;
 	struct putchar_args pca;
 
 	pca.flags = flags;
@@ -386,7 +420,6 @@ void
 putchar(int c, void *arg)
 {
 	struct putchar_args *pca = arg;
-	register struct msgbuf *mbp;
 	char **sp = (char**) pca->tty;
 
 	if (panicstr)
@@ -400,6 +433,8 @@ putchar(int c, void *arg)
 		constty = 0;
 	if ((pca->flags & TOLOG) && c != '\0' && c != '\r' && c != 0177)
 		log_putc(c);
+	if ((pca->flags & TOLOGLOCKED) && c != '\0' && c != '\r' && c != 0177)
+		log_putc_locked(c);
 	if ((pca->flags & TOCONS) && constty == 0 && c != '\0')
 		(*v_putc)(c);
 	if (pca->flags & TOSTR) {
@@ -408,10 +443,22 @@ putchar(int c, void *arg)
 	}
 }
 
+int
+vprintf(const char *fmt, va_list ap)
+{
+	struct putchar_args pca;
 
+	pca.flags = TOLOG | TOCONS;
+	pca.tty   = NULL;
+	__doprnt(fmt, &ap, putchar, &pca, 10);
+	return 0;
+}
 
 /*
  * Scaled down version of vsprintf(3).
+ *
+ * Deprecation Warning:
+ *	vsprintf() is being deprecated. Please use vsnprintf() instead. 
  */
 int
 vsprintf(char *buf, const char *cfmt, va_list ap)
diff --git a/bsd/kern/subr_prof.c b/bsd/kern/subr_prof.c
index f0151a07d..cff03ee0a 100644
--- a/bsd/kern/subr_prof.c
+++ b/bsd/kern/subr_prof.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -55,6 +61,10 @@
  *	@(#)subr_prof.c	8.3 (Berkeley) 9/23/93
  */
 
+#ifdef GPROF
+#include <kern/mach_header.h>
+#endif
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -71,13 +81,14 @@
 #include <kern/cpu_number.h>
 #include <kern/kalloc.h>
 
-extern boolean_t ml_set_interrupts_enabled(boolean_t enable);
-
 #ifdef GPROF
 #include <sys/malloc.h>
 #include <sys/gmon.h>
-#include <kern/mach_header.h>
-#include <machine/profile.h>
+
+extern int sysctl_doprof(int *, u_int, user_addr_t, size_t *, 
+		user_addr_t, size_t newlen);
+extern int sysctl_struct(user_addr_t, size_t *,
+		user_addr_t, size_t, void *, int);
 
 lck_spin_t * mcount_lock;
 lck_grp_t * mcount_lock_grp;
@@ -86,7 +97,7 @@ lck_attr_t * mcount_lock_attr;
 /*
  * Froms is actually a bunch of unsigned shorts indexing tos
  */
-struct gmonparam _gmonparam = { GMON_PROF_OFF };
+struct gmonparam _gmonparam = { .state = GMON_PROF_OFF };
 
 /*
  * This code uses 32 bit mach object segment information from the currently
@@ -96,7 +107,6 @@ void
 kmstartup(void)
 {
 	char *cp;
-	u_long	fromssize, tossize;
 	struct segment_command	*sgp;	/* 32 bit mach object file segment */
 	struct gmonparam *p = &_gmonparam;
 	
@@ -111,7 +121,7 @@ kmstartup(void)
 	p->lowpc = ROUNDDOWN(p->lowpc, HISTFRACTION * sizeof(HISTCOUNTER));
 	p->highpc = ROUNDUP(p->highpc, HISTFRACTION * sizeof(HISTCOUNTER));
 	p->textsize = p->highpc - p->lowpc;
-	printf("Profiling kernel, textsize=%d [0x%08x..0x%08x]\n",
+	printf("Profiling kernel, textsize=%lu [0x%016lx..0x%016lx]\n",
 	       p->textsize, p->lowpc, p->highpc);
 	p->kcountsize = p->textsize / HISTFRACTION;
 	p->hashfraction = HASHFRACTION;
@@ -188,14 +198,14 @@ sysctl_doprof(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
  */
 void
 mcount(
-    register u_long frompc,
-    register u_long selfpc
+    u_long frompc,
+    u_long selfpc
 )
 {
     unsigned short *frompcindex;
-	register struct tostruct *top, *prevtop;
+	struct tostruct *top, *prevtop;
 	struct gmonparam *p = &_gmonparam;
-	register long toindex;
+	long toindex;
 
     /*
      * check that we are profiling
@@ -302,110 +312,177 @@ overflow:
 #define PROFILE_LOCK(x)
 #define PROFILE_UNLOCK(x)
 
+static int profil_funneled(struct proc *p, struct profil_args *uap, register_t *retval);
+static int add_profil_funneled(struct proc *p, struct add_profil_args *uap, register_t *retval);
+
+
 int
-profil(struct proc *p, register struct profil_args *uap, __unused register_t *retval)
+profil(struct proc *p, struct profil_args *uap, register_t *retval)
+{
+	boolean_t funnel_state;
+	int error;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	error = profil_funneled(p, uap, retval);
+	thread_funnel_set(kernel_flock, funnel_state);
+	return(error);
+}
+
+static int
+profil_funneled(struct proc *p, struct profil_args *uap, __unused register_t *retval)
 {
-    struct uprof *upp = &p->p_stats->p_prof;
-    int s;
+        struct uprof *upp = &p->p_stats->p_prof;
+	int s;
 
 	if (uap->pcscale > (1 << 16))
 		return (EINVAL);
+
 	if (uap->pcscale == 0) {
 		stopprofclock(p);
 		return (0);
 	}
-
-	/* Block profile interrupts while changing state. */
-    s = ml_set_interrupts_enabled(FALSE);	
+	/*
+	 * Block profile interrupts while changing state.
+	 */
+	s = ml_set_interrupts_enabled(FALSE);	
 
 	if (proc_is64bit(p)) {
-        struct user_uprof *user_upp = &p->p_stats->user_p_prof;
-    	struct user_uprof *upc, *nupc;
+	        struct user_uprof *user_upp = &p->p_stats->user_p_prof;
+		struct user_uprof *upc, *nupc;
 	
-	    PROFILE_LOCK(&user_upp->pr_lock);
-        user_upp->pr_base = uap->bufbase;
-        user_upp->pr_size = uap->bufsize;
-        user_upp->pr_off = uap->pcoffset;
-	    user_upp->pr_scale = uap->pcscale;
-        upp->pr_base = NULL;
-        upp->pr_size = 0;
-        upp->pr_scale = 0;
-
-        /* remove buffers previously allocated with add_profil() */
-        for (upc = user_upp->pr_next; upc; upc = nupc) {
-            nupc = upc->pr_next;
-            kfree(upc, sizeof (*upc));
-        }
-        user_upp->pr_next = 0;
-	    PROFILE_UNLOCK(&user_upp->pr_lock);
-	}
-	else {
-    	struct uprof *upc, *nupc;
+		PROFILE_LOCK(&user_upp->pr_lock);
+
+		user_upp->pr_base = uap->bufbase;
+		user_upp->pr_size = uap->bufsize;
+		user_upp->pr_off = uap->pcoffset;
+		user_upp->pr_scale = uap->pcscale;
+		upp->pr_base = NULL;
+		upp->pr_size = 0;
+		upp->pr_scale = 0;
+
+		/*
+		 * remove buffers previously allocated with add_profil()
+		 * don't do the kfree's while interrupts disabled
+		 */
+		upc = user_upp->pr_next;
+		user_upp->pr_next = 0;
+
+		PROFILE_UNLOCK(&user_upp->pr_lock);
+
+		startprofclock(p);
+		ml_set_interrupts_enabled(s);
+
+		while (upc) {
+		        nupc = upc->pr_next;
+			kfree(upc, sizeof (*upc));
+			upc = nupc;
+		}
+
+	} else {
+	        struct uprof *upc, *nupc;
 	    
-	    PROFILE_LOCK(&upp->pr_lock);
-        upp->pr_base = CAST_DOWN(caddr_t, uap->bufbase);
-        upp->pr_size = uap->bufsize;
-        upp->pr_off = uap->pcoffset;
-	    upp->pr_scale = uap->pcscale;
-
-        /* remove buffers previously allocated with add_profil() */
-        for (upc = upp->pr_next; upc; upc = nupc) {
-            nupc = upc->pr_next;
-            kfree(upc, sizeof (struct uprof));
-        }
-        upp->pr_next = 0;
-	    PROFILE_UNLOCK(&upp->pr_lock);
-	}
+		PROFILE_LOCK(&upp->pr_lock);
+
+		upp->pr_base = CAST_DOWN(caddr_t, uap->bufbase);
+		upp->pr_size = uap->bufsize;
+		upp->pr_off = uap->pcoffset;
+		upp->pr_scale = uap->pcscale;
+
+		/*
+		 * remove buffers previously allocated with add_profil()
+		 * don't do the kfree's while interrupts disabled
+		 */
+		upc = upp->pr_next;
+		upp->pr_next = 0;
+
+		PROFILE_UNLOCK(&upp->pr_lock);
+
+		startprofclock(p);
+		ml_set_interrupts_enabled(s);
 
-	startprofclock(p);
-	ml_set_interrupts_enabled(s);
+		while (upc) {
+		        nupc = upc->pr_next;
+			kfree(upc, sizeof (struct uprof));
+			upc = nupc;
+		}
+	}
 	return(0);
 }
 
 int
-add_profil(struct proc *p, register struct add_profil_args *uap, __unused register_t *retval)
+add_profil(struct proc *p, struct add_profil_args *uap, register_t *retval)
+{
+	boolean_t funnel_state;
+	int error;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	error = add_profil_funneled(p, uap, retval);
+	thread_funnel_set(kernel_flock, funnel_state);
+	return(error);
+}
+
+
+static int
+add_profil_funneled(struct proc *p, struct add_profil_args *uap, __unused register_t *retval)
 {
 	struct uprof *upp = &p->p_stats->p_prof, *upc;
 	struct user_uprof *user_upp = NULL, *user_upc;
 	int s;
 	boolean_t is64bit = proc_is64bit(p);
 
+
+	upc = NULL;
+	user_upc = NULL;
+
 	if (is64bit) {
-       user_upp = &p->p_stats->user_p_prof;
-       if (user_upp->pr_scale == 0)
-            return (0);
-    }
-    else {
-        if (upp->pr_scale == 0)
-            return (0);
-    }
-
-    s = ml_set_interrupts_enabled(FALSE);	
-    
+	        user_upp = &p->p_stats->user_p_prof;
+
+		if (user_upp->pr_scale == 0)
+		        return (0);
+	}
+	else {
+	        if (upp->pr_scale == 0)
+		        return (0);
+	}
 	if (is64bit) {
-        user_upc = (struct user_uprof *) kalloc(sizeof (struct user_uprof));
-        user_upc->pr_base = uap->bufbase;
-        user_upc->pr_size = uap->bufsize;
-        user_upc->pr_off = uap->pcoffset;
-        user_upc->pr_scale = uap->pcscale;
-        PROFILE_LOCK(&user_upp->pr_lock);
-        user_upc->pr_next = user_upp->pr_next;
-        user_upp->pr_next = user_upc;
-        PROFILE_UNLOCK(&user_upp->pr_lock);
-    }
-    else {
-        upc = (struct uprof *) kalloc(sizeof (struct uprof));
-        upc->pr_base = CAST_DOWN(caddr_t, uap->bufbase);
-        upc->pr_size = uap->bufsize;
-        upc->pr_off = uap->pcoffset;
-        upc->pr_scale = uap->pcscale;
-        PROFILE_LOCK(&upp->pr_lock);
-        upc->pr_next = upp->pr_next;
-        upp->pr_next = upc;
-        PROFILE_UNLOCK(&upp->pr_lock);
-    }
+	        user_upc = (struct user_uprof *) kalloc(sizeof (struct user_uprof));
+	        user_upc->pr_base = uap->bufbase;
+		user_upc->pr_size = uap->bufsize;
+		user_upc->pr_off = uap->pcoffset;
+		user_upc->pr_scale = uap->pcscale;
+	} else {
+	        upc = (struct uprof *) kalloc(sizeof (struct uprof));
+	        upc->pr_base = CAST_DOWN(caddr_t, uap->bufbase);
+		upc->pr_size = uap->bufsize;
+		upc->pr_off = uap->pcoffset;
+		upc->pr_scale = uap->pcscale;
+	}
+	s = ml_set_interrupts_enabled(FALSE);	
     
+	if (is64bit) {
+		PROFILE_LOCK(&user_upp->pr_lock);
+		if (user_upp->pr_scale) {
+		        user_upc->pr_next = user_upp->pr_next;
+			user_upp->pr_next = user_upc;
+			user_upc = NULL;
+		}
+		PROFILE_UNLOCK(&user_upp->pr_lock);
+	} else {
+		PROFILE_LOCK(&upp->pr_lock);
+	        if (upp->pr_scale) {
+		        upc->pr_next = upp->pr_next;
+			upp->pr_next = upc;
+			upc = NULL;
+		}
+		PROFILE_UNLOCK(&upp->pr_lock);
+	}
 	ml_set_interrupts_enabled(s);		
+
+	if (upc)
+	        kfree(upc, sizeof(struct uprof));
+	if (user_upc)
+	        kfree(user_upc, sizeof(struct user_uprof));
+
 	return(0);
 }
 
@@ -433,12 +510,9 @@ add_profil(struct proc *p, register struct add_profil_args *uap, __unused regist
  * update fails, we simply turn off profiling.
  */
 void
-addupc_task(p, pc, ticks)
-	register struct proc *p;
-	user_addr_t pc;
-	u_int ticks;
+addupc_task(struct proc *p, user_addr_t pc, u_int ticks)
 {
-	register u_int off;
+	u_int off;
 	u_short count;
 
 	/* Testing P_PROFIL may be unnecessary, but is certainly safe. */
diff --git a/bsd/kern/subr_sbuf.c b/bsd/kern/subr_sbuf.c
new file mode 100644
index 000000000..e966704b8
--- /dev/null
+++ b/bsd/kern/subr_sbuf.c
@@ -0,0 +1,584 @@
+/*-
+ * Copyright (c) 2000 Poul-Henning Kamp and Dag-Erling Codan Sm¿rgrav
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+
+#ifdef KERNEL
+/* #include <ctype.h> */
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+#include <sys/uio_internal.h>
+#include <sys/systm.h>
+#include <stdarg.h>
+#else /* KERNEL */
+#include <ctype.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#endif /* KERNEL */
+
+#include <sys/sbuf.h>
+
+#ifdef KERNEL
+/* MALLOC_DEFINE(M_SBUF, "sbuf", "string buffers"); */
+#define	SBMALLOC(size)		_MALLOC(size, M_SBUF, M_WAITOK)
+#define	SBFREE(buf)		FREE(buf, M_SBUF)
+#else /* KERNEL */
+#define	KASSERT(e, m)
+#define	SBMALLOC(size)		malloc(size)
+#define	SBFREE(buf)		free(buf)
+#define	min(x,y)		MIN(x,y)
+	
+#endif /* KERNEL */
+
+/*
+ * Predicates
+ */
+#define	SBUF_ISDYNAMIC(s)	((s)->s_flags & SBUF_DYNAMIC)
+#define	SBUF_ISDYNSTRUCT(s)	((s)->s_flags & SBUF_DYNSTRUCT)
+#define	SBUF_ISFINISHED(s)	((s)->s_flags & SBUF_FINISHED)
+#define	SBUF_HASOVERFLOWED(s)	((s)->s_flags & SBUF_OVERFLOWED)
+#define	SBUF_HASROOM(s)		((s)->s_len < (s)->s_size - 1)
+#define	SBUF_FREESPACE(s)	((s)->s_size - (s)->s_len - 1)
+#define	SBUF_CANEXTEND(s)	((s)->s_flags & SBUF_AUTOEXTEND)
+
+/*
+ * Set / clear flags
+ */
+#define	SBUF_SETFLAG(s, f)	do { (s)->s_flags |= (f); } while (0)
+#define	SBUF_CLEARFLAG(s, f)	do { (s)->s_flags &= ~(f); } while (0)
+
+#define	SBUF_MINEXTENDSIZE	16		/* Should be power of 2. */
+#define	SBUF_MAXEXTENDSIZE	PAGE_SIZE
+#define	SBUF_MAXEXTENDINCR	PAGE_SIZE
+
+/*
+ * Debugging support
+ */
+#if defined(KERNEL) && defined(INVARIANTS)
+static void
+_assert_sbuf_integrity(const char *fun, struct sbuf *s)
+{
+	KASSERT(s != NULL,
+	    ("%s called with a NULL sbuf pointer", fun));
+	KASSERT(s->s_buf != NULL,
+	    ("%s called with uninitialized or corrupt sbuf", fun));
+	KASSERT(s->s_len < s->s_size,
+	    ("wrote past end of sbuf (%d >= %d)", s->s_len, s->s_size));
+}
+
+static void
+_assert_sbuf_state(const char *fun, struct sbuf *s, int state)
+{
+	KASSERT((s->s_flags & SBUF_FINISHED) == state,
+	    ("%s called with %sfinished or corrupt sbuf", fun,
+	    (state ? "un" : "")));
+}
+#define	assert_sbuf_integrity(s) _assert_sbuf_integrity(__func__, (s))
+#define	assert_sbuf_state(s, i)	 _assert_sbuf_state(__func__, (s), (i))
+#else /* KERNEL && INVARIANTS */
+#define	assert_sbuf_integrity(s) do { } while (0)
+#define	assert_sbuf_state(s, i)	 do { } while (0)
+#endif /* KERNEL && INVARIANTS */
+
+static int
+sbuf_extendsize(int size)
+{
+	int newsize;
+
+	newsize = SBUF_MINEXTENDSIZE;
+	while (newsize < size) {
+		if (newsize < (int)SBUF_MAXEXTENDSIZE)
+			newsize *= 2;
+		else
+			newsize += SBUF_MAXEXTENDINCR;
+	}
+
+	return (newsize);
+}
+
+
+/*
+ * Extend an sbuf.
+ */
+static int
+sbuf_extend(struct sbuf *s, int addlen)
+{
+	char *newbuf;
+	int newsize;
+
+	if (!SBUF_CANEXTEND(s))
+		return (-1);
+
+	newsize = sbuf_extendsize(s->s_size + addlen);
+	newbuf = (char *)SBMALLOC(newsize);
+	if (newbuf == NULL)
+		return (-1);
+	bcopy(s->s_buf, newbuf, s->s_size);
+	if (SBUF_ISDYNAMIC(s))
+		SBFREE(s->s_buf);
+	else
+		SBUF_SETFLAG(s, SBUF_DYNAMIC);
+	s->s_buf = newbuf;
+	s->s_size = newsize;
+	return (0);
+}
+
+/*
+ * Initialize an sbuf.
+ * If buf is non-NULL, it points to a static or already-allocated string
+ * big enough to hold at least length characters.
+ */
+struct sbuf *
+sbuf_new(struct sbuf *s, char *buf, int length, int flags)
+{
+	KASSERT(length >= 0,
+	    ("attempt to create an sbuf of negative length (%d)", length));
+	KASSERT((flags & ~SBUF_USRFLAGMSK) == 0,
+	    ("%s called with invalid flags", __func__));
+
+	flags &= SBUF_USRFLAGMSK;
+	if (s == NULL) {
+		s = (struct sbuf *)SBMALLOC(sizeof *s);
+		if (s == NULL)
+			return (NULL);
+		bzero(s, sizeof *s);
+		s->s_flags = flags;
+		SBUF_SETFLAG(s, SBUF_DYNSTRUCT);
+	} else {
+		bzero(s, sizeof *s);
+		s->s_flags = flags;
+	}
+	s->s_size = length;
+	if (buf) {
+		s->s_buf = buf;
+		return (s);
+	}
+	if (flags & SBUF_AUTOEXTEND)
+		s->s_size = sbuf_extendsize(s->s_size);
+	s->s_buf = (char *)SBMALLOC(s->s_size);
+	if (s->s_buf == NULL) {
+		if (SBUF_ISDYNSTRUCT(s))
+			SBFREE(s);
+		return (NULL);
+	}
+	SBUF_SETFLAG(s, SBUF_DYNAMIC);
+	return (s);
+}
+
+#ifdef KERNEL
+/*
+ * Create an sbuf with uio data
+ */
+struct sbuf *
+sbuf_uionew(struct sbuf *s, struct uio *uio, int *error)
+{
+	KASSERT(uio != NULL,
+	    ("%s called with NULL uio pointer", __func__));
+	KASSERT(error != NULL,
+	    ("%s called with NULL error pointer", __func__));
+
+	s = sbuf_new(s, NULL, uio->uio_resid + 1, 0);
+	if (s == NULL) {
+		*error = ENOMEM;
+		return (NULL);
+	}
+	*error = uiomove(s->s_buf, uio->uio_resid, uio);
+	if (*error != 0) {
+		sbuf_delete(s);
+		return (NULL);
+	}
+	s->s_len = s->s_size - 1;
+	*error = 0;
+	return (s);
+}
+#endif
+
+/*
+ * Clear an sbuf and reset its position.
+ */
+void
+sbuf_clear(struct sbuf *s)
+{
+	assert_sbuf_integrity(s);
+	/* don't care if it's finished or not */
+
+	SBUF_CLEARFLAG(s, SBUF_FINISHED);
+	SBUF_CLEARFLAG(s, SBUF_OVERFLOWED);
+	s->s_len = 0;
+}
+
+/*
+ * Set the sbuf's end position to an arbitrary value.
+ * Effectively truncates the sbuf at the new position.
+ */
+int
+sbuf_setpos(struct sbuf *s, int pos)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	KASSERT(pos >= 0,
+	    ("attempt to seek to a negative position (%d)", pos));
+	KASSERT(pos < s->s_size,
+	    ("attempt to seek past end of sbuf (%d >= %d)", pos, s->s_size));
+
+	if (pos < 0 || pos > s->s_len)
+		return (-1);
+	s->s_len = pos;
+	return (0);
+}
+
+/*
+ * Append a byte string to an sbuf.
+ */
+int
+sbuf_bcat(struct sbuf *s, const void *buf, size_t len)
+{
+	const char *str = buf;
+
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	for (; len; len--) {
+		if (!SBUF_HASROOM(s) && sbuf_extend(s, len) < 0)
+			break;
+		s->s_buf[s->s_len++] = *str++;
+	}
+	if (len) {
+		SBUF_SETFLAG(s, SBUF_OVERFLOWED);
+		return (-1);
+	}
+	return (0);
+}
+
+#ifdef KERNEL
+/*
+ * Copy a byte string from userland into an sbuf.
+ */
+int
+sbuf_bcopyin(struct sbuf *s, const void *uaddr, size_t len)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	if (len == 0)
+		return (0);
+	if (len > (unsigned) SBUF_FREESPACE(s)) {
+		sbuf_extend(s, len - SBUF_FREESPACE(s));
+		len = min(len, SBUF_FREESPACE(s));
+	}
+	if (copyin(CAST_USER_ADDR_T(uaddr), s->s_buf + s->s_len, len) != 0)
+		return (-1);
+	s->s_len += len;
+
+	return (0);
+}
+#endif
+
+/*
+ * Copy a byte string into an sbuf.
+ */
+int
+sbuf_bcpy(struct sbuf *s, const void *buf, size_t len)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	sbuf_clear(s);
+	return (sbuf_bcat(s, buf, len));
+}
+
+/*
+ * Append a string to an sbuf.
+ */
+int
+sbuf_cat(struct sbuf *s, const char *str)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	while (*str) {
+		if (!SBUF_HASROOM(s) && sbuf_extend(s, strlen(str)) < 0)
+			break;
+		s->s_buf[s->s_len++] = *str++;
+	}
+	if (*str) {
+		SBUF_SETFLAG(s, SBUF_OVERFLOWED);
+		return (-1);
+	}
+	return (0);
+}
+
+#ifdef KERNEL
+/*
+ * Append a string from userland to an sbuf.
+ */
+int
+sbuf_copyin(struct sbuf *s, const void *uaddr, size_t len)
+{
+	size_t done;
+
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	if (len == 0)
+		len = SBUF_FREESPACE(s);	/* XXX return 0? */
+	if (len > (unsigned) SBUF_FREESPACE(s)) {
+		sbuf_extend(s, len);
+		len = min(len, SBUF_FREESPACE(s));
+	}
+	switch (copyinstr(CAST_USER_ADDR_T(uaddr), s->s_buf + s->s_len, len + 1, &done)) {
+	case ENAMETOOLONG:
+		SBUF_SETFLAG(s, SBUF_OVERFLOWED);
+		/* fall through */
+	case 0:
+		s->s_len += done - 1;
+		break;
+	default:
+		return (-1);	/* XXX */
+	}
+
+	return (done);
+}
+#endif
+
+/*
+ * Copy a string into an sbuf.
+ */
+int
+sbuf_cpy(struct sbuf *s, const char *str)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	sbuf_clear(s);
+	return (sbuf_cat(s, str));
+}
+
+/*
+ * Format the given argument list and append the resulting string to an sbuf.
+ */
+int
+sbuf_vprintf(struct sbuf *s, const char *fmt, va_list ap)
+{
+	__builtin_va_list ap_copy; /* XXX tduffy - blame on him */
+	int len;
+
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	KASSERT(fmt != NULL,
+	    ("%s called with a NULL format string", __func__));
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	do {
+		va_copy(ap_copy, ap);
+		len = vsnprintf(&s->s_buf[s->s_len], SBUF_FREESPACE(s) + 1,
+		    fmt, ap_copy);
+		va_end(ap_copy);
+	} while (len > SBUF_FREESPACE(s) &&
+	    sbuf_extend(s, len - SBUF_FREESPACE(s)) == 0);
+
+	/*
+	 * s->s_len is the length of the string, without the terminating nul.
+	 * When updating s->s_len, we must subtract 1 from the length that
+	 * we passed into vsnprintf() because that length includes the
+	 * terminating nul.
+	 *
+	 * vsnprintf() returns the amount that would have been copied,
+	 * given sufficient space, hence the min() calculation below.
+	 */
+	s->s_len += min(len, SBUF_FREESPACE(s));
+	if (!SBUF_HASROOM(s) && !SBUF_CANEXTEND(s))
+		SBUF_SETFLAG(s, SBUF_OVERFLOWED);
+
+	KASSERT(s->s_len < s->s_size,
+	    ("wrote past end of sbuf (%d >= %d)", s->s_len, s->s_size));
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+	return (0);
+}
+
+/*
+ * Format the given arguments and append the resulting string to an sbuf.
+ */
+int
+sbuf_printf(struct sbuf *s, const char *fmt, ...)
+{
+	va_list ap;
+	int result;
+
+	va_start(ap, fmt);
+	result = sbuf_vprintf(s, fmt, ap);
+	va_end(ap);
+	return(result);
+}
+
+/*
+ * Append a character to an sbuf.
+ */
+int
+sbuf_putc(struct sbuf *s, int c)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	if (!SBUF_HASROOM(s) && sbuf_extend(s, 1) < 0) {
+		SBUF_SETFLAG(s, SBUF_OVERFLOWED);
+		return (-1);
+	}
+	if (c != '\0')
+	    s->s_buf[s->s_len++] = c;
+	return (0);
+}
+
+static inline int
+isspace(char ch)
+{
+  return (ch == ' ' || ch == '\n' || ch == '\t');
+}
+
+/*
+ * Trim whitespace characters from end of an sbuf.
+ */
+int
+sbuf_trim(struct sbuf *s)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+
+	while (s->s_len && isspace(s->s_buf[s->s_len-1]))
+		--s->s_len;
+
+	return (0);
+}
+
+/*
+ * Check if an sbuf overflowed
+ */
+int
+sbuf_overflowed(struct sbuf *s)
+{
+    return SBUF_HASOVERFLOWED(s);
+}
+
+/*
+ * Finish off an sbuf.
+ */
+void
+sbuf_finish(struct sbuf *s)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, 0);
+
+	s->s_buf[s->s_len] = '\0';
+	SBUF_CLEARFLAG(s, SBUF_OVERFLOWED);
+	SBUF_SETFLAG(s, SBUF_FINISHED);
+}
+
+/*
+ * Return a pointer to the sbuf data.
+ */
+char *
+sbuf_data(struct sbuf *s)
+{
+	assert_sbuf_integrity(s);
+	assert_sbuf_state(s, SBUF_FINISHED);
+
+	return s->s_buf;
+}
+
+/*
+ * Return the length of the sbuf data.
+ */
+int
+sbuf_len(struct sbuf *s)
+{
+	assert_sbuf_integrity(s);
+	/* don't care if it's finished or not */
+
+	if (SBUF_HASOVERFLOWED(s))
+		return (-1);
+	return s->s_len;
+}
+
+/*
+ * Clear an sbuf, free its buffer if necessary.
+ */
+void
+sbuf_delete(struct sbuf *s)
+{
+	int isdyn;
+
+	assert_sbuf_integrity(s);
+	/* don't care if it's finished or not */
+
+	if (SBUF_ISDYNAMIC(s))
+		SBFREE(s->s_buf);
+	isdyn = SBUF_ISDYNSTRUCT(s);
+	bzero(s, sizeof *s);
+	if (isdyn)
+		SBFREE(s);
+}
+
+/*
+ * Check if an sbuf has been finished.
+ */
+int
+sbuf_done(struct sbuf *s)
+{
+
+	return(SBUF_ISFINISHED(s));
+}
diff --git a/bsd/kern/subr_xxx.c b/bsd/kern/subr_xxx.c
index 393c07142..cc73e7389 100644
--- a/bsd/kern/subr_xxx.c
+++ b/bsd/kern/subr_xxx.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -61,12 +67,21 @@
 #include <sys/proc_internal.h>
 #include <sys/vnode.h>
 #include <sys/uio.h>
+#include <sys/sysproto.h>
+
+#include <sys/signalvar.h>		/* for psignal() */
+
+
+#ifdef GPROF
+#include <sys/gmon.h>
+#endif
+
 
 /*
  * Unsupported device function (e.g. writing to read-only device).
  */
 int
-enodev()
+enodev(void)
 {
 	return (ENODEV);
 }
@@ -75,7 +90,7 @@ enodev()
  * Unsupported strategy function.
  */
 void
-enodev_strat()
+enodev_strat(void)
 {
 	return;
 }
@@ -84,7 +99,7 @@ enodev_strat()
  * Unconfigured device function; driver not configured.
  */
 int
-enxio()
+enxio(void)
 {
 	return (ENXIO);
 }
@@ -93,7 +108,7 @@ enxio()
  * Unsupported ioctl function.
  */
 int
-enoioctl()
+enoioctl(void)
 {
 	return (ENOTTY);
 }
@@ -105,7 +120,7 @@ enoioctl()
  * that is not supported by the current system binary.
  */
 int
-enosys()
+enosys(void)
 {
 	return (ENOSYS);
 }
@@ -117,7 +132,7 @@ enosys()
  * XXX Name of this routine is wrong.
  */
 int
-eopnotsupp()
+eopnotsupp(void)
 {
 	return (ENOTSUP);
 }
@@ -126,7 +141,7 @@ eopnotsupp()
  * Generic null operation, always returns success.
  */
 int
-nullop()
+nullop(void)
 {
 	return (0);
 }
@@ -137,7 +152,7 @@ nullop()
  * in the bdevsw and cdevsw tables.
  */
 int
-nulldev()
+nulldev(void)
 {
 	return (0);
 }
@@ -146,13 +161,13 @@ nulldev()
  * Null system calls. Not invalid, just not configured.
  */
 int
-errsys()
+errsys(void)
 {
 	return(EINVAL);
 }
 
 void
-nullsys()
+nullsys(void)
 {
 }
 
@@ -163,10 +178,7 @@ nullsys()
  */
 /* ARGSUSED */
 int
-nosys(p, args, retval)
-	struct proc *p;
-	void *args;
-	register_t *retval;
+nosys(struct proc *p, __unused struct nosys_args *args, __unused register_t *retval)
 {
 	psignal(p, SIGSYS);
 	return (ENOSYS);
@@ -177,11 +189,9 @@ nosys(p, args, retval)
  * Stub routine in case it is ever possible to free space.
  */
 void
-cfreemem(cp, size)
-	caddr_t cp;
-	int size;
+cfreemem(caddr_t cp, int size)
 {
-	printf("freeing %x, size %d\n", cp, size);
+	printf("freeing %p, size %d\n", cp, size);
 }
 #endif
 
diff --git a/bsd/kern/sys_domain.c b/bsd/kern/sys_domain.c
index 244d2ed3c..66af22826 100644
--- a/bsd/kern/sys_domain.c
+++ b/bsd/kern/sys_domain.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  */
 /*
@@ -33,13 +39,14 @@
 
 
 /* domain init function */
-void systemdomain_init(void);
+void systemdomain_init(void) __attribute__((section("__TEXT, initcode")));
 
 struct domain systemdomain =
     { PF_SYSTEM, "system", systemdomain_init, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, { 0, 0} };
 
 
-void systemdomain_init()
+void
+systemdomain_init(void)
 {
     /* add system domain built in protocol initializers here */
 
diff --git a/bsd/kern/sys_generic.c b/bsd/kern/sys_generic.c
index 1b92b6f00..0fe948aae 100644
--- a/bsd/kern/sys_generic.c
+++ b/bsd/kern/sys_generic.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,12 @@
  *
  *	@(#)sys_generic.c	8.9 (Berkeley) 2/14/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -67,11 +79,7 @@
 #include <sys/file_internal.h>
 #include <sys/proc_internal.h>
 #include <sys/socketvar.h>
-#if KTRACE
 #include <sys/uio_internal.h>
-#else
-#include <sys/uio.h>
-#endif
 #include <sys/kernel.h>
 #include <sys/stat.h>
 #include <sys/malloc.h>
@@ -121,20 +129,24 @@
 /* for wait queue based select */
 #include <kern/wait_queue.h>
 #include <kern/kalloc.h>
-#if KTRACE 
-#include <sys/ktrace.h>
-#endif
 #include <sys/vnode_internal.h>
 
+/* XXX should be in a header file somewhere */
+void evsofree(struct socket *);
+void evpipefree(struct pipe *);
+void postpipeevent(struct pipe *, int);
+void postevent(struct socket *, struct sockbuf *, int);
+extern kern_return_t IOBSDGetPlatformUUID(__darwin_uuid_t uuid, mach_timespec_t timeoutp);
+
 int rd_uio(struct proc *p, int fdes, uio_t uio, user_ssize_t *retval);
 int wr_uio(struct proc *p, int fdes, uio_t uio, user_ssize_t *retval);
 extern void	*get_bsduthreadarg(thread_t);
 extern int	*get_bsduthreadrval(thread_t);
 
-__private_extern__ int	dofileread(struct proc *p, struct fileproc *fp, int fd, 
+__private_extern__ int	dofileread(vfs_context_t ctx, struct fileproc *fp,
 								   user_addr_t bufp, user_size_t nbyte, 
 								   off_t offset, int flags, user_ssize_t *retval);
-__private_extern__ int	dofilewrite(struct proc *p, struct fileproc *fp, int fd, 
+__private_extern__ int	dofilewrite(vfs_context_t ctx, struct fileproc *fp,
 									user_addr_t bufp, user_size_t nbyte, 
 									off_t offset, int flags, user_ssize_t *retval);
 __private_extern__ int	preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_vnode);
@@ -151,14 +163,26 @@ extern int appletalk_inited;
 #define f_ops f_fglob->fg_ops
 #define f_offset f_fglob->fg_offset
 #define f_data f_fglob->fg_data
+
 /*
  * Read system call.
+ *
+ * Returns:	0			Success
+ *	preparefileread:EBADF
+ *	preparefileread:ESPIPE
+ *	preparefileread:ENXIO
+ *	preparefileread:EBADF
+ *	dofileread:???
  */
 int
-read(p, uap, retval)
-	struct proc *p;
-	register struct read_args *uap;
-	user_ssize_t *retval;
+read(struct proc *p, struct read_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(read_nocancel(p, (struct read_nocancel_args *)uap, retval));
+}
+
+int
+read_nocancel(struct proc *p, struct read_nocancel_args *uap, user_ssize_t *retval)
 {
 	struct fileproc *fp;
 	int error;
@@ -167,7 +191,7 @@ read(p, uap, retval)
 	if ( (error = preparefileread(p, &fp, fd, 0)) )
 	        return (error);
 
-	error = dofileread(p, fp, uap->fd, uap->cbuf, uap->nbyte,
+	error = dofileread(vfs_context_current(), fp, uap->cbuf, uap->nbyte,
 			   (off_t)-1, 0, retval);
 
 	donefileread(p, fp, fd);
@@ -177,21 +201,32 @@ read(p, uap, retval)
 
 /* 
  * Pread system call
+ *
+ * Returns:	0			Success
+ *	preparefileread:EBADF
+ *	preparefileread:ESPIPE
+ *	preparefileread:ENXIO
+ *	preparefileread:EBADF
+ *	dofileread:???
  */
 int
-pread(p, uap, retval)
-	struct proc *p;
-	register struct pread_args *uap;
-	user_ssize_t *retval;
+pread(struct proc *p, struct pread_args *uap, user_ssize_t *retval)
 {
-	struct fileproc *fp;
+	__pthread_testcancel(1);
+	return(pread_nocancel(p, (struct pread_nocancel_args *)uap, retval));
+}
+
+int
+pread_nocancel(struct proc *p, struct pread_nocancel_args *uap, user_ssize_t *retval)
+{
+	struct fileproc *fp = NULL;	/* fp set by preparefileread() */
 	int fd = uap->fd;
 	int error;
 
 	if ( (error = preparefileread(p, &fp, fd, 1)) )
 	        return (error);
 
-	error = dofileread(p, fp, uap->fd, uap->buf, uap->nbyte,
+	error = dofileread(vfs_context_current(), fp, uap->buf, uap->nbyte,
 			uap->offset, FOF_OFFSET, retval);
 	
 	donefileread(p, fp, fd);
@@ -210,7 +245,7 @@ pread(p, uap, retval)
 void
 donefileread(struct proc *p, struct fileproc *fp, int fd)
 {
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 
 	fp->f_flags &= ~FP_INCHRREAD;
 
@@ -218,6 +253,14 @@ donefileread(struct proc *p, struct fileproc *fp, int fd)
         proc_fdunlock(p);
 }
 
+/*
+ * Returns:	0			Success
+ *		EBADF
+ *		ESPIPE
+ *		ENXIO
+ *	fp_lookup:EBADF
+ *	fo_read:???
+ */
 int
 preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_pread)
 {
@@ -225,7 +268,7 @@ preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_
 	int 	error;
 	struct fileproc *fp;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 
 	error = fp_lookup(p, fd, &fp, 1);
 
@@ -244,6 +287,14 @@ preparefileread(struct proc *p, struct fileproc **fp_ret, int fd, int check_for_
 	if (fp->f_type == DTYPE_VNODE) {
 		vp = (struct vnode *)fp->f_fglob->fg_data;
 
+		if (check_for_pread && (vnode_isfifo(vp))) {
+			error = ESPIPE;
+			goto out;
+		} 
+		if (check_for_pread && (vp->v_flag & VISTTY)) {
+			error = ENXIO;
+			goto out;
+		}
 		if (vp->v_type == VCHR)
 			fp->f_flags |= FP_INCHRREAD;
 	}
@@ -260,31 +311,26 @@ out:
 }
 
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	fo_read:???
+ */
 __private_extern__ int
-dofileread(p, fp, fd, bufp, nbyte, offset, flags, retval)
-	struct proc *p;
-	struct fileproc *fp;
-	int fd, flags;
-	user_addr_t bufp;
-	user_size_t nbyte;
-	off_t offset;
-	user_ssize_t *retval;
+dofileread(vfs_context_t ctx, struct fileproc *fp,
+	   user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
+	   user_ssize_t *retval)
 {
 	uio_t auio;
 	user_ssize_t bytecnt;
 	long error = 0;
 	char uio_buf[ UIO_SIZEOF(1) ];
-#if KTRACE
-	uio_t ktruio = NULL;
-	char ktr_uio_buf[ UIO_SIZEOF(1) ];
-	int didktr = 0;
-#endif
 
 	// LP64todo - do we want to raise this?
 	if (nbyte > INT_MAX)
 		return (EINVAL);
 
-	if (IS_64BIT_PROCESS(p)) {
+	if (IS_64BIT_PROCESS(vfs_context_proc(ctx))) {
 		auio = uio_createwithbuffer(1, offset, UIO_USERSPACE64, UIO_READ, 
 									  &uio_buf[0], sizeof(uio_buf));
 	} else {
@@ -293,37 +339,14 @@ dofileread(p, fp, fd, bufp, nbyte, offset, flags, retval)
 	}
 	uio_addiov(auio, bufp, nbyte);
 
-#if KTRACE
-	/*
-	* if tracing, save a copy of iovec
-	*/
-	if (KTRPOINT(p, KTR_GENIO)) {
-		didktr = 1;
-
-		if (IS_64BIT_PROCESS(p)) {
-			ktruio = uio_createwithbuffer(1, offset, UIO_USERSPACE64, UIO_READ, 
-									  &ktr_uio_buf[0], sizeof(ktr_uio_buf));
-		} else {
-			ktruio = uio_createwithbuffer(1, offset, UIO_USERSPACE32, UIO_READ, 
-									  &ktr_uio_buf[0], sizeof(ktr_uio_buf));
-		}
-		uio_addiov(ktruio, bufp, nbyte);
-	}
-#endif
 	bytecnt = nbyte;
 
-	if ((error = fo_read(fp, auio, fp->f_cred, flags, p))) {
+	if ((error = fo_read(fp, auio, flags, ctx))) {
 		if (uio_resid(auio) != bytecnt && (error == ERESTART ||
 			error == EINTR || error == EWOULDBLOCK))
 			error = 0;
 	}
 	bytecnt -= uio_resid(auio);
-#if KTRACE
-	if (didktr && error == 0) {
-		uio_setresid(ktruio, bytecnt);
-		ktrgenio(p->p_tracep, fd, UIO_READ, ktruio, error);
-	}
-#endif  
 
 	*retval = bytecnt;
 
@@ -332,12 +355,22 @@ dofileread(p, fp, fd, bufp, nbyte, offset, flags, retval)
 
 /*      
  * Scatter read system call.
+ *
+ * Returns:	0			Success
+ *		EINVAL
+ *		ENOMEM
+ *	copyin:EFAULT
+ *	rd_uio:???
  */
 int
-readv(p, uap, retval)
-	struct proc *p;
-	register struct readv_args *uap;
-	user_ssize_t *retval;
+readv(struct proc *p, struct readv_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(readv_nocancel(p, (struct readv_nocancel_args *)uap, retval));
+}
+
+int
+readv_nocancel(struct proc *p, struct readv_nocancel_args *uap, user_ssize_t *retval)
 {
 	uio_t auio = NULL;
 	int error;
@@ -381,12 +414,22 @@ ExitThisRoutine:
 
 /*
  * Write system call
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *	fp_lookup:EBADF
+ *	dofilewrite:???
  */
 int
-write(p, uap, retval)
-	struct proc *p;
-	register struct write_args *uap;
-	user_ssize_t *retval;	
+write(struct proc *p, struct write_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(write_nocancel(p, (struct write_nocancel_args *)uap, retval));
+
+}
+
+int
+write_nocancel(struct proc *p, struct write_nocancel_args *uap, user_ssize_t *retval)
 {
 	struct fileproc *fp;
 	int error;      
@@ -398,7 +441,10 @@ write(p, uap, retval)
 	if ((fp->f_flag & FWRITE) == 0) {
 		error = EBADF;
 	} else {
-		error = dofilewrite(p, fp, uap->fd, uap->cbuf, uap->nbyte,
+		struct vfs_context context = *(vfs_context_current());
+		context.vc_ucred = fp->f_fglob->fg_cred;
+
+		error = dofilewrite(&context, fp, uap->cbuf, uap->nbyte,
 			(off_t)-1, 0, retval);
 	}
 	if (error == 0)
@@ -410,16 +456,29 @@ write(p, uap, retval)
 
 /*                          
  * pwrite system call
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *		ESPIPE
+ *		ENXIO
+ *		EINVAL
+ *	fp_lookup:EBADF
+ *	dofilewrite:???
  */
 int
-pwrite(p, uap, retval)
-	struct proc *p;
-	register struct pwrite_args *uap;
-	user_ssize_t *retval;	
+pwrite(struct proc *p, struct pwrite_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(pwrite_nocancel(p, (struct pwrite_nocancel_args *)uap, retval));
+}
+
+int
+pwrite_nocancel(struct proc *p, struct pwrite_nocancel_args *uap, user_ssize_t *retval)
 {
         struct fileproc *fp;
         int error; 
 	int fd = uap->fd;
+	vnode_t vp  = (vnode_t)0;
 
 	error = fp_lookup(p,fd,&fp,0);
 	if (error)
@@ -428,13 +487,31 @@ pwrite(p, uap, retval)
 	if ((fp->f_flag & FWRITE) == 0) {
 		error = EBADF;
 	} else {
+		struct vfs_context context = *vfs_context_current();
+		context.vc_ucred = fp->f_fglob->fg_cred;
+
 		if (fp->f_type != DTYPE_VNODE) {
 			error = ESPIPE;
-		} else {
-		    error = dofilewrite(p, fp, uap->fd, uap->buf, uap->nbyte,
-			uap->offset, FOF_OFFSET, retval);
+			goto errout;
+		}
+		vp = (vnode_t)fp->f_fglob->fg_data;
+		if (vnode_isfifo(vp)) {
+			error = ESPIPE;
+			goto errout;
+		} 
+		if ((vp->v_flag & VISTTY)) {
+			error = ENXIO;
+			goto errout;
 		}
+		if (uap->offset == (off_t)-1) {
+			error = EINVAL;
+			goto errout;
+		}
+
+		    error = dofilewrite(&context, fp, uap->buf, uap->nbyte,
+			uap->offset, FOF_OFFSET, retval);
         }
+errout:
 	if (error == 0)
 	        fp_drop_written(p, fd, fp);
 	else
@@ -447,31 +524,27 @@ pwrite(p, uap, retval)
         return(error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	<fo_write>:EPIPE
+ *	<fo_write>:???			[indirect through struct fileops]
+ */
 __private_extern__ int                  
-dofilewrite(p, fp, fd, bufp, nbyte, offset, flags, retval)
-	struct proc *p;
-	struct fileproc *fp; 
-	int fd, flags;
-	user_addr_t bufp;
-	user_size_t nbyte;   
-	off_t offset; 
-	user_ssize_t *retval;
+dofilewrite(vfs_context_t ctx, struct fileproc *fp,
+	    user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
+	    user_ssize_t *retval)
 {       
 	uio_t auio;
 	long error = 0;
 	user_ssize_t bytecnt;
 	char uio_buf[ UIO_SIZEOF(1) ];
-#if KTRACE
-	uio_t ktruio;
-	int didktr = 0; 
-	char ktr_uio_buf[ UIO_SIZEOF(1) ];
-#endif
 
 	// LP64todo - do we want to raise this?
 	if (nbyte > INT_MAX)   
 		return (EINVAL);
 
-	if (IS_64BIT_PROCESS(p)) {
+	if (IS_64BIT_PROCESS(vfs_context_proc(ctx))) {
 		auio = uio_createwithbuffer(1, offset, UIO_USERSPACE64, UIO_WRITE, 
 									  &uio_buf[0], sizeof(uio_buf));
 	} else {
@@ -480,39 +553,18 @@ dofilewrite(p, fp, fd, bufp, nbyte, offset, flags, retval)
 	}
 	uio_addiov(auio, bufp, nbyte);
 
-#if KTRACE
-	/*
-	* if tracing, save a copy of iovec and uio
-	*/
-	if (KTRPOINT(p, KTR_GENIO)) {
-		didktr = 1;
-
-		if (IS_64BIT_PROCESS(p)) {
-			ktruio = uio_createwithbuffer(1, offset, UIO_USERSPACE64, UIO_WRITE, 
-										  &ktr_uio_buf[0], sizeof(ktr_uio_buf));
-		} else {
-			ktruio = uio_createwithbuffer(1, offset, UIO_USERSPACE32, UIO_WRITE, 
-										  &ktr_uio_buf[0], sizeof(ktr_uio_buf));
-		}
-		uio_addiov(ktruio, bufp, nbyte);
-	}
-#endif  
 	bytecnt = nbyte; 
-	if ((error = fo_write(fp, auio, fp->f_cred, flags, p))) {
+	if ((error = fo_write(fp, auio, flags, ctx))) {
 		if (uio_resid(auio) != bytecnt && (error == ERESTART ||
 			error == EINTR || error == EWOULDBLOCK))
 			error = 0;
 		/* The socket layer handles SIGPIPE */
-		if (error == EPIPE && fp->f_type != DTYPE_SOCKET)
-			psignal(p, SIGPIPE);
+		if (error == EPIPE && fp->f_type != DTYPE_SOCKET) {
+			/* XXX Raise the signal on the thread? */
+			psignal(vfs_context_proc(ctx), SIGPIPE);
+		}
 	}
 	bytecnt -= uio_resid(auio);
-#if KTRACE 
-	if (didktr && error == 0) {
-		uio_setresid(ktruio, bytecnt);
-		ktrgenio(p->p_tracep, fd, UIO_WRITE, ktruio, error);
-	}
-#endif  
 	*retval = bytecnt;
 
 	return (error); 
@@ -522,10 +574,14 @@ dofilewrite(p, fp, fd, bufp, nbyte, offset, flags, retval)
  * Gather write system call  
  */     
 int
-writev(p, uap, retval)
-	struct proc *p;
-	register struct writev_args *uap;
-	user_ssize_t *retval;
+writev(struct proc *p, struct writev_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(writev_nocancel(p, (struct writev_nocancel_args *)uap, retval));
+}
+
+int
+writev_nocancel(struct proc *p, struct writev_nocancel_args *uap, user_ssize_t *retval)
 {
 	uio_t auio = NULL;
 	int error;
@@ -569,21 +625,12 @@ ExitThisRoutine:
 
 
 int
-wr_uio(p, fdes, uio, retval)
-	struct proc *p;
-	int fdes;
-	register uio_t uio;
-	user_ssize_t *retval;
+wr_uio(struct proc *p, int fdes, uio_t uio, user_ssize_t *retval)
 {
 	struct fileproc *fp;
 	int error;
 	user_ssize_t count;
-#if KTRACE
-	struct iovec_64 *ktriov = NULL;
-	struct uio ktruio;
-	int didktr = 0;
-	u_int iovlen;
-#endif
+	struct vfs_context context = *vfs_context_current();
 
 	error = fp_lookup(p,fdes,&fp,0);
 	if (error)
@@ -594,22 +641,9 @@ wr_uio(p, fdes, uio, retval)
 		goto out;
 	}
 	count = uio_resid(uio);
-#if KTRACE
-	/*
-	 * if tracing, save a copy of iovec
-	 */
-	if (KTRPOINT(p, KTR_GENIO)) {
-		iovlen = uio->uio_iovcnt *
-			(IS_64BIT_PROCESS(p) ? sizeof (struct iovec_64) : sizeof (struct iovec_32));
-		MALLOC(ktriov, struct iovec_64 *, iovlen, M_TEMP, M_WAITOK);
-		if (ktriov != NULL) {
-			bcopy((caddr_t)uio->uio_iovs.iov64p, (caddr_t)ktriov, iovlen);
-			ktruio = *uio;
-			didktr = 1;
-		}
-	} 
-#endif  
-	error = fo_write(fp, uio, fp->f_cred, 0, p);
+
+	context.vc_ucred = fp->f_cred;
+	error = fo_write(fp, uio, 0, &context);
 	if (error) {
 		if (uio_resid(uio) != count && (error == ERESTART ||
 						error == EINTR || error == EWOULDBLOCK))
@@ -620,17 +654,6 @@ wr_uio(p, fdes, uio, retval)
 	}
 	*retval = count - uio_resid(uio);
 
-#if KTRACE
-	if (didktr) {
-		if (error == 0) {
-			ktruio.uio_iovs.iov64p = ktriov; 
-			uio_setresid(&ktruio, *retval);
-			ktrgenio(p->p_tracep, fdes, UIO_WRITE, &ktruio, error);
-		}
-		FREE(ktriov, M_TEMP);
-	}
-#endif
-
 out:
 	if ( (error == 0) )
 	        fp_drop_written(p, fdes, fp);
@@ -641,42 +664,21 @@ out:
 
 
 int
-rd_uio(p, fdes, uio, retval)
-	struct proc *p;
-	int fdes;
-	register uio_t uio;
-	user_ssize_t *retval;
+rd_uio(struct proc *p, int fdes, uio_t uio, user_ssize_t *retval)
 {
 	struct fileproc *fp;
 	int error;
 	user_ssize_t count;
-#if KTRACE
-	struct iovec_64 *ktriov = NULL;
-	struct uio ktruio;
-	int didktr = 0;
-	u_int iovlen;
-#endif
+	struct vfs_context context = *vfs_context_current();
 
 	if ( (error = preparefileread(p, &fp, fdes, 0)) )
 	        return (error);
 
 	count = uio_resid(uio);
-#if KTRACE
-	/*
-	 * if tracing, save a copy of iovec
-	 */
-	if (KTRPOINT(p, KTR_GENIO)) {
-		iovlen = uio->uio_iovcnt *
-			(IS_64BIT_PROCESS(p) ? sizeof (struct iovec_64) : sizeof (struct iovec_32));
-		MALLOC(ktriov, struct iovec_64 *, iovlen, M_TEMP, M_WAITOK);
-		if (ktriov != NULL) {
-			bcopy((caddr_t)uio->uio_iovs.iov64p, (caddr_t)ktriov, iovlen);
-			ktruio = *uio;
-			didktr = 1;
-		}
-	} 
-#endif  
-	error = fo_read(fp, uio, fp->f_cred, 0, p);
+
+	context.vc_ucred = fp->f_cred;
+
+	error = fo_read(fp, uio, 0, &context);
 
 	if (error) {
 	        if (uio_resid(uio) != count && (error == ERESTART ||
@@ -685,16 +687,6 @@ rd_uio(p, fdes, uio, retval)
 	}
 	*retval = count - uio_resid(uio);
 
-#if KTRACE
-	if (didktr) {
-		if (error == 0) {
-			ktruio.uio_iovs.iov64p = ktriov; 
-			uio_setresid(&ktruio, *retval);
-			ktrgenio(p->p_tracep, fdes, UIO_READ, &ktruio, error);
-		}
-		FREE(ktriov, M_TEMP);
-	}
-#endif
 	donefileread(p, fp, fdes);
 
 	return (error);
@@ -703,20 +695,30 @@ rd_uio(p, fdes, uio, retval)
 /*
  * Ioctl system call
  *
+ * Returns:	0			Success
+ *		EBADF
+ *		ENOTTY
+ *		ENOMEM
+ *		ESRCH
+ *	copyin:EFAULT
+ *	copyoutEFAULT
+ *	fp_lookup:EBADF			Bad file descriptor
+ *	fo_ioctl:???
  */
 int
-ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retval)
+ioctl(struct proc *p, struct ioctl_args *uap, __unused register_t *retval)
 {
 	struct fileproc *fp;
-	register u_long com;
+	u_long com;
 	int error = 0;
-	register u_int size;
+	u_int size;
 	caddr_t datap, memp;
 	boolean_t is64bit;
 	int tmp;
 #define STK_PARAMS	128
 	char stkbuf[STK_PARAMS];
 	int fd = uap->fd;
+	struct vfs_context context = *vfs_context_current();
 
 	AUDIT_ARG(fd, uap->fd);
 	AUDIT_ARG(cmd, CAST_DOWN(int, uap->com)); /* LP64todo: uap->com is a user-land long */
@@ -737,6 +739,14 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 			error = EBADF;
 			goto out;
 	}
+
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
+#if CONFIG_MACF
+	error = mac_file_check_ioctl(context.vc_ucred, fp->f_fglob, uap->com);
+	if (error)
+		goto out;
+#endif
 		
 #if NETAT
 	/*
@@ -746,13 +756,14 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 	{
 		if (appletalk_inited && ((uap->com & 0x0000FFFF) == 0xff99)) {
 			u_long  fixed_command;
+
 #ifdef APPLETALK_DEBUG
 			kprintf("ioctl: special AppleTalk \n");
 #endif
 			datap = &stkbuf[0];
 			*(user_addr_t *)datap = uap->data;
 			fixed_command = _IOW(0, 0xff99, uap->data);
-			error = fo_ioctl(fp, fixed_command, datap, p);
+			error = fo_ioctl(fp, fixed_command, datap, &context);
 			goto out;
 		}
 	}
@@ -835,7 +846,7 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 			fp->f_flag |= FNONBLOCK;
 		else
 			fp->f_flag &= ~FNONBLOCK;
-		error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, p);
+		error = fo_ioctl(fp, FIONBIO, (caddr_t)&tmp, &context);
 		break;
 
 	case FIOASYNC:
@@ -843,7 +854,7 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 			fp->f_flag |= FASYNC;
 		else
 			fp->f_flag &= ~FASYNC;
-		error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, p);
+		error = fo_ioctl(fp, FIOASYNC, (caddr_t)&tmp, &context);
 		break;
 
 	case FIOSETOWN:
@@ -854,20 +865,21 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 			break;
 		}
 		if (fp->f_type == DTYPE_PIPE) {
-		        error = fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, p);
+		        error = fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, &context);
 			break;
 		}
 		if (tmp <= 0) {
 			tmp = -tmp;
 		} else {
-			struct proc *p1 = pfind(tmp);
+			struct proc *p1 = proc_find(tmp);
 			if (p1 == 0) {
 				error = ESRCH;
 				break;
 			}
-			tmp = p1->p_pgrp->pg_id;
+			tmp = p1->p_pgrpid;
+			proc_rele(p1);
 		}
-		error = fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, p);
+		error = fo_ioctl(fp, (int)TIOCSPGRP, (caddr_t)&tmp, &context);
 		break;
 
 	case FIOGETOWN:
@@ -876,12 +888,12 @@ ioctl(struct proc *p, register struct ioctl_args *uap, __unused register_t *retv
 			*(int *)datap = ((struct socket *)fp->f_data)->so_pgid;
 			break;
 		}
-		error = fo_ioctl(fp, TIOCGPGRP, datap, p);
+		error = fo_ioctl(fp, TIOCGPGRP, datap, &context);
 		*(int *)datap = -*(int *)datap;
 		break;
 
 	default:
-		error = fo_ioctl(fp, com, datap, p);
+		error = fo_ioctl(fp, com, datap, &context);
 		/*
 		 * Copy any data to user, size was
 		 * already set and checked above.
@@ -908,15 +920,27 @@ extern int selprocess(int error, int sel_pass);
 static int selscan(struct proc *p, struct _select * sel,
 			int nfd, register_t *retval, int sel_pass, wait_queue_sub_t wqsub);
 static int selcount(struct proc *p, u_int32_t *ibits, u_int32_t *obits,
-			int nfd, int * count);
+			int nfd, int * count, int *kfcount);
 static int seldrop(struct proc *p, u_int32_t *ibits, int nfd);
 extern uint64_t	tvtoabstime(struct timeval	*tvp);
 
 /*
  * Select system call.
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		EAGAIN			Nonconformant error if allocation fails
+ *	selprocess:???
  */
 int
 select(struct proc *p, struct select_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(select_nocancel(p, (struct select_nocancel_args *)uap, retval));
+}
+
+int
+select_nocancel(struct proc *p, struct select_nocancel_args *uap, register_t *retval)
 {
 	int error = 0;
 	u_int ni, nw, size;
@@ -925,6 +949,7 @@ select(struct proc *p, struct select_args *uap, register_t *retval)
 	struct _select *sel;
 	int needzerofill = 1;
 	int count = 0;
+	int kfcount = 0;
 
 	th_act = current_thread();
 	uth = get_bsdthread_info(th_act);
@@ -936,6 +961,11 @@ select(struct proc *p, struct select_args *uap, register_t *retval)
 		return (EINVAL);
 	}
 
+	/* select on thread of process that already called proc_exit() */
+	if (p->p_fd == NULL) {
+		return (EBADF);
+	}
+
 	if (uap->nd > p->p_fd->fd_nfiles)
 		uap->nd = p->p_fd->fd_nfiles; /* forgiving; slightly wrong */
 
@@ -943,32 +973,38 @@ select(struct proc *p, struct select_args *uap, register_t *retval)
 	ni = nw * sizeof(fd_mask);
 
 	/*
-	 * if this is the first select by the thread 
-	 * allocate the space for bits.
-	 */
-	if (sel->nbytes == 0) {
-		sel->nbytes = 3 * ni;
-		MALLOC(sel->ibits, u_int32_t *, sel->nbytes, M_TEMP, M_WAITOK | M_ZERO);
-		MALLOC(sel->obits, u_int32_t *, sel->nbytes, M_TEMP, M_WAITOK | M_ZERO);
-		if ((sel->ibits == NULL) || (sel->obits == NULL))
-			panic("select out of memory");
-		needzerofill = 0;
-	}
-
-	/*
-	 * if the previously allocated space for the bits
-	 * is smaller than what is requested. Reallocate.
+	 * if the previously allocated space for the bits is smaller than
+	 * what is requested or no space has yet been allocated for this
+	 * thread, allocate enough space now.
+	 *
+	 * Note: If this process fails, select() will return EAGAIN; this
+	 * is the same thing pool() returns in a no-memory situation, but
+	 * it is not a POSIX compliant error code for select().
 	 */
 	if (sel->nbytes < (3 * ni)) {
-		sel->nbytes = (3 * ni);
-		FREE(sel->ibits, M_TEMP);
-		FREE(sel->obits, M_TEMP);
-		MALLOC(sel->ibits, u_int32_t *, sel->nbytes, M_TEMP, M_WAITOK | M_ZERO);
-		MALLOC(sel->obits, u_int32_t *, sel->nbytes, M_TEMP, M_WAITOK | M_ZERO);
-		if ((sel->ibits == NULL) || (sel->obits == NULL))
-			panic("select out of memory");
+		int nbytes = 3 * ni;
+
+		/* Free previous allocation, if any */
+		if (sel->ibits != NULL)
+			FREE(sel->ibits, M_TEMP);
+		if (sel->obits != NULL) {
+			FREE(sel->obits, M_TEMP);
+			/* NULL out; subsequent ibits allocation may fail */
+			sel->obits = NULL;
+		}
+
+		MALLOC(sel->ibits, u_int32_t *, nbytes, M_TEMP, M_WAITOK | M_ZERO);
+		if (sel->ibits == NULL)
+			return (EAGAIN);
+		MALLOC(sel->obits, u_int32_t *, nbytes, M_TEMP, M_WAITOK | M_ZERO);
+		if (sel->obits == NULL) {
+			FREE(sel->ibits, M_TEMP);
+			sel->ibits = NULL;
+			return (EAGAIN);
+		}
+		sel->nbytes = nbytes;
 		needzerofill = 0;
-	}   
+	}
 
 	if (needzerofill) {
 		bzero((caddr_t)sel->ibits, sel->nbytes);
@@ -1014,33 +1050,34 @@ select(struct proc *p, struct select_args *uap, register_t *retval)
 	else
 		sel->abstime = 0;
 
-	if ( (error = selcount(p, sel->ibits, sel->obits, uap->nd, &count)) ) {
+	sel->kfcount = 0;
+	if ( (error = selcount(p, sel->ibits, sel->obits, uap->nd, &count, &kfcount)) ) {
 			goto continuation;
 	}
-
 	sel->count = count;
+	sel->kfcount = kfcount;
 	size = SIZEOF_WAITQUEUE_SET + (count * SIZEOF_WAITQUEUE_LINK);
-	if (sel->allocsize) {
-		if (sel->wqset == 0)
+	if (uth->uu_allocsize) {
+		if (uth->uu_wqset == 0)
 			panic("select: wql memory smashed");
 		/* needed for the select now */
-		if (size > sel->allocsize) {
-			kfree(sel->wqset,  sel->allocsize);
-			sel->allocsize = size;
-			sel->wqset = (wait_queue_set_t)kalloc(size);
-			if (sel->wqset == (wait_queue_set_t)NULL)
+		if (size > uth->uu_allocsize) {
+			kfree(uth->uu_wqset,  uth->uu_allocsize);
+			uth->uu_allocsize = size;
+			uth->uu_wqset = (wait_queue_set_t)kalloc(size);
+			if (uth->uu_wqset == (wait_queue_set_t)NULL)
 				panic("failed to allocate memory for waitqueue\n");
 		}
 	} else {
 		sel->count = count;
-		sel->allocsize = size;
-		sel->wqset = (wait_queue_set_t)kalloc(sel->allocsize);
-		if (sel->wqset == (wait_queue_set_t)NULL)
+		uth->uu_allocsize = size;
+		uth->uu_wqset = (wait_queue_set_t)kalloc(uth->uu_allocsize);
+		if (uth->uu_wqset == (wait_queue_set_t)NULL)
 			panic("failed to allocate memory for waitqueue\n");
 	}
-	bzero(sel->wqset, size);
-	sel->wql = (char *)sel->wqset + SIZEOF_WAITQUEUE_SET;
-	wait_queue_set_init(sel->wqset, (SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST));
+	bzero(uth->uu_wqset, size);
+	sel->wql = (char *)uth->uu_wqset + SIZEOF_WAITQUEUE_SET;
+	wait_queue_set_init(uth->uu_wqset, (SYNC_POLICY_FIFO | SYNC_POLICY_PREPOST));
 
 continuation:
 	return selprocess(error, SEL_FIRSTPASS);
@@ -1087,13 +1124,13 @@ retry:
 	}
 
 	ncoll = nselcoll;
-	p->p_flag |= P_SELECT;
+	OSBitOrAtomic(P_SELECT, (UInt32 *)&p->p_flag);
 	/* skip scans if the select is just for timeouts */
 	if (sel->count) {
 		if (sel_pass == SEL_FIRSTPASS)
-			wait_queue_sub_clearrefs(sel->wqset);
+			wait_queue_sub_clearrefs(uth->uu_wqset);
 
-		error = selscan(p, sel, uap->nd, retval, sel_pass, sel->wqset);
+		error = selscan(p, sel, uap->nd, retval, sel_pass, (wait_queue_sub_t)uth->uu_wqset);
 		if (error || *retval) {
 			goto done;
 		}
@@ -1139,14 +1176,14 @@ retry:
 		goto retry;
 	}
 
-	p->p_flag &= ~P_SELECT;
+	OSBitAndAtomic(~((uint32_t)P_SELECT), (UInt32 *)&p->p_flag);
 
 	/* if the select is just for timeout skip check */
 	if (sel->count &&(sel_pass == SEL_SECONDPASS))
 		panic("selprocess: 2nd pass assertwaiting");
 
 	/* Wait Queue Subordinate has waitqueue as first element */
-	wait_result = wait_queue_assert_wait((wait_queue_t)sel->wqset,
+	wait_result = wait_queue_assert_wait((wait_queue_t)uth->uu_wqset,
 					     &selwait, THREAD_ABORTSAFE, sel->abstime);
 	if (wait_result != THREAD_AWAKENED) {
 		/* there are no preposted events */
@@ -1165,10 +1202,10 @@ retry:
 	}
 done:
 	if (unwind) {
-		wait_subqueue_unlink_all(sel->wqset);
+		wait_subqueue_unlink_all(uth->uu_wqset);
 		seldrop(p, sel->ibits, uap->nd);
 	}
-	p->p_flag &= ~P_SELECT;
+	OSBitAndAtomic(~((uint32_t)P_SELECT), (UInt32 *)&p->p_flag);
 	/* select is not restarted after signals... */
 	if (error == ERESTART)
 		error = EINTR;
@@ -1196,17 +1233,12 @@ done:
 }
 
 static int
-selscan(p, sel, nfd, retval, sel_pass, wqsub)
-	struct proc *p;
-	struct _select *sel;
-	int nfd;
-	register_t *retval;
-	int sel_pass;
-	wait_queue_sub_t wqsub;
+selscan(struct proc *p, struct _select *sel, int nfd, register_t *retval,
+	int sel_pass, wait_queue_sub_t wqsub)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int msk, i, j, fd;
-	register u_int32_t bits;
+	struct filedesc *fdp = p->p_fd;
+	int msk, i, j, fd;
+	u_int32_t bits;
 	struct fileproc *fp;
 	int n = 0;
 	int nc = 0;
@@ -1216,6 +1248,10 @@ selscan(p, sel, nfd, retval, sel_pass, wqsub)
 	u_int32_t *ibits, *obits;
 	char * wql;
 	char * wql_ptr;
+	int count, kfcount;
+	boolean_t funnel_state;
+	vnode_t vp;
+	struct vfs_context context = *vfs_context_current();
 
 	/*
 	 * Problems when reboot; due to MacOSX signal probs
@@ -1231,10 +1267,16 @@ selscan(p, sel, nfd, retval, sel_pass, wqsub)
 
 	nw = howmany(nfd, NFDBITS);
 
-	nc = 0;
-	proc_fdlock(p);
+	count = sel->count;
+	kfcount = sel->kfcount;
 
-	if (sel->count) {
+	if (kfcount > count)
+		panic("selscan: count < kfcount");
+
+	if (kfcount != 0) {
+		funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+		proc_fdlock(p);
 		for (msk = 0; msk < 3; msk++) {
 			iptr = (u_int32_t *)&ibits[msk * nw];
 			optr = (u_int32_t *)&obits[msk * nw];
@@ -1249,6 +1291,7 @@ selscan(p, sel, nfd, retval, sel_pass, wqsub)
 					if (fp == NULL ||
 						(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
 						proc_fdunlock(p);
+						thread_funnel_set(kernel_flock, funnel_state);
 						return(EBADF);
 					}
 					if (sel_pass == SEL_SECONDPASS) {
@@ -1260,7 +1303,13 @@ selscan(p, sel, nfd, retval, sel_pass, wqsub)
 						fp->f_flags |= FP_INSELECT;
 						fp->f_waddr = (void *)wqsub;
 					}
-					if (fp->f_ops && fo_select(fp, flag[msk], wql_ptr, p)) {
+
+					context.vc_ucred = fp->f_cred;
+
+					if (fp->f_ops && (fp->f_type == DTYPE_VNODE)
+							&& ((vp = (struct vnode *)fp->f_data)  != NULLVP)
+							&& (vp->v_type == VCHR)
+						&& fo_select(fp, flag[msk], wql_ptr, &context)) {
 						optr[fd/NFDBITS] |= (1 << (fd % NFDBITS));
 						n++;
 					}
@@ -1268,13 +1317,61 @@ selscan(p, sel, nfd, retval, sel_pass, wqsub)
 				}
 			}
 		}
+		proc_fdunlock(p);
+		thread_funnel_set(kernel_flock, funnel_state);
+	}
+
+	nc = 0;
+	if (kfcount != count) {
+		proc_fdlock(p);
+		for (msk = 0; msk < 3; msk++) {
+			iptr = (u_int32_t *)&ibits[msk * nw];
+			optr = (u_int32_t *)&obits[msk * nw];
+
+			for (i = 0; i < nfd; i += NFDBITS) {
+				bits = iptr[i/NFDBITS];
+
+				while ((j = ffs(bits)) && (fd = i + --j) < nfd) {
+					bits &= ~(1 << j);
+					fp = fdp->fd_ofiles[fd];
+
+					if (fp == NULL ||
+						(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
+						proc_fdunlock(p);
+						return(EBADF);
+					}
+					if (sel_pass == SEL_SECONDPASS) {
+						wql_ptr = (char *)0;
+						fp->f_flags &= ~FP_INSELECT;
+						fp->f_waddr = (void *)0;
+					} else {
+					        wql_ptr = (wql + nc * SIZEOF_WAITQUEUE_LINK);
+						fp->f_flags |= FP_INSELECT;
+						fp->f_waddr = (void *)wqsub;
+					}
+
+					context.vc_ucred = fp->f_cred;
+
+					if ((fp->f_ops && 
+						((fp->f_type != DTYPE_VNODE)
+						|| (((vp = (struct vnode *)fp->f_data)  != NULLVP)
+							&& (vp->v_type != VCHR))
+						)
+						&& fo_select(fp, flag[msk], wql_ptr, &context))) {
+						optr[fd/NFDBITS] |= (1 << (fd % NFDBITS));
+						n++;
+					}
+					nc++;
+				}
+			}
+		}
+		proc_fdunlock(p);
 	}
-	proc_fdunlock(p);
 	*retval = n;
 	return (0);
 }
 
-static int poll_callback(struct kqueue *, struct kevent *, void *);
+int poll_callback(struct kqueue *, struct kevent *, void *);
 
 struct poll_continue_args {
 	user_addr_t pca_fds;
@@ -1284,6 +1381,14 @@ struct poll_continue_args {
 
 int
 poll(struct proc *p, struct poll_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(poll_nocancel(p, (struct poll_nocancel_args *)uap, retval));
+}
+
+
+int
+poll_nocancel(struct proc *p, struct poll_nocancel_args *uap, register_t *retval)
 {
 	struct poll_continue_args *cont;
 	struct pollfd *fds;
@@ -1303,7 +1408,7 @@ poll(struct proc *p, struct poll_args *uap, register_t *retval)
 	 * safe, but not overly restrictive.
 	 */
 	if (nfds > OPEN_MAX ||
-	    (nfds > p->p_rlimit[RLIMIT_NOFILE].rlim_cur && nfds > FD_SETSIZE))
+	    (nfds > p->p_rlimit[RLIMIT_NOFILE].rlim_cur && (proc_suser(p) || nfds > FD_SETSIZE)))
 		return (EINVAL);
 
 	kq = kqueue_alloc(p);
@@ -1340,8 +1445,7 @@ poll(struct proc *p, struct poll_args *uap, register_t *retval)
 
 	/* JMM - all this P_SELECT stuff is bogus */
 	ncoll = nselcoll;
-	p->p_flag |= P_SELECT;
-
+	OSBitOrAtomic(P_SELECT, (UInt32 *)&p->p_flag);
 	for (i = 0; i < nfds; i++) {
 		short events = fds[i].events;
 		struct kevent kev;
@@ -1361,7 +1465,7 @@ poll(struct proc *p, struct poll_args *uap, register_t *retval)
 		kev.udata = CAST_USER_ADDR_T(&fds[i]);
 
 		/* Handle input events */
-		if (events & ( POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND )) {
+		if (events & ( POLLIN | POLLRDNORM | POLLPRI | POLLRDBAND | POLLHUP )) {
 			kev.filter = EVFILT_READ;
 			if (!(events & ( POLLIN | POLLRDNORM )))
 				kev.flags |= EV_OOBAND;
@@ -1410,7 +1514,7 @@ poll(struct proc *p, struct poll_args *uap, register_t *retval)
 	rfds = cont->pca_rfds;
 
  done:
-	p->p_flag &= ~P_SELECT;
+	OSBitAndAtomic(~((uint32_t)P_SELECT), (UInt32 *)&p->p_flag);
 	/* poll is not restarted after signals... */
 	if (error == ERESTART)
 		error = EINTR;
@@ -1425,11 +1529,11 @@ poll(struct proc *p, struct poll_args *uap, register_t *retval)
 	if (NULL != cont)
 		FREE(cont, M_TEMP);
 
-	kqueue_dealloc(kq, p);
+	kqueue_dealloc(kq);
 	return (error);
 }
 
-static int
+int
 poll_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
 {
 	struct poll_continue_args *cont = (struct poll_continue_args *)data;
@@ -1441,7 +1545,6 @@ poll_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
 		fds->revents |= POLLHUP;
 	if (kevp->flags & EV_ERROR)
 		fds->revents |= POLLERR;
-	cont->pca_rfds++;
 
 	switch (kevp->filter) {
 	case EVFILT_READ:
@@ -1462,7 +1565,7 @@ poll_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
 			fds->revents |= (fds->events & ( POLLOUT | POLLWRNORM | POLLWRBAND ));
 		break;
 
-	case EVFILT_PROC:
+	case EVFILT_VNODE:
 		if (kevp->fflags & NOTE_EXTEND)
 			fds->revents |= (fds->events & POLLEXTEND);
 		if (kevp->fflags & NOTE_ATTRIB)
@@ -1473,6 +1576,10 @@ poll_callback(__unused struct kqueue *kq, struct kevent *kevp, void *data)
 			fds->revents |= (fds->events & POLLWRITE);
 		break;
 	}
+
+	if (fds->revents)
+		cont->pca_rfds++;
+
 	return 0;
 }
 	
@@ -1485,24 +1592,27 @@ seltrue(__unused dev_t dev, __unused int flag, __unused struct proc *p)
 
 static int
 selcount(struct proc *p, u_int32_t *ibits, __unused u_int32_t *obits, 
-		 int nfd, int *count)
+		 int nfd, int *countp, int * kfcountp)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int msk, i, j, fd;
-	register u_int32_t bits;
+	struct filedesc *fdp = p->p_fd;
+	int msk, i, j, fd;
+	u_int32_t bits;
 	struct fileproc *fp;
 	int n = 0;
 	u_int32_t *iptr;
 	u_int nw;
 	int error=0; 
+	int kfc = 0;
 	int dropcount;
+	vnode_t vp;
 
 	/*
 	 * Problems when reboot; due to MacOSX signal probs
 	 * in Beaker1C ; verify that the p->p_fd is valid
 	 */
 	if (fdp == NULL) {
-		*count=0;
+		*countp = 0;
+		*kfcountp = 0;
 		return(EIO);
 	}
 	nw = howmany(nfd, NFDBITS);
@@ -1517,18 +1627,25 @@ selcount(struct proc *p, u_int32_t *ibits, __unused u_int32_t *obits,
 				fp = fdp->fd_ofiles[fd];
 				if (fp == NULL ||
 					(fdp->fd_ofileflags[fd] & UF_RESERVED)) {
-						*count=0;
+						*countp = 0;
+						*kfcountp = 0;
 						error = EBADF;
 						goto bad;
 				}
 				fp->f_iocount++;
+				if ((fp->f_type == DTYPE_VNODE)
+						&& ((vp = (struct vnode *)fp->f_data)  != NULLVP)
+						&& (vp->v_type == VCHR) )
+					kfc++;
+
 				n++;
 			}
 		}
 	}
 	proc_fdunlock(p);
 
-	*count = n;
+	*countp = n;
+	*kfcountp = kfc;
 	return (0);
 bad:
 	dropcount = 0;
@@ -1561,14 +1678,11 @@ out:
 }
 
 static int
-seldrop(p, ibits, nfd)
-	struct proc *p;
-	u_int32_t *ibits;
-	int nfd;
+seldrop(struct proc *p, u_int32_t *ibits, int nfd)
 {
-	register struct filedesc *fdp = p->p_fd;
-	register int msk, i, j, fd;
-	register u_int32_t bits;
+	struct filedesc *fdp = p->p_fd;
+	int msk, i, j, fd;
+	u_int32_t bits;
 	struct fileproc *fp;
 	int n = 0;
 	u_int32_t *iptr;
@@ -1649,16 +1763,15 @@ selrecord(__unused struct proc *selector, struct selinfo *sip, void * p_wql)
 		sip->si_flags &= ~SI_COLL;
 
 	sip->si_flags |= SI_RECORDED;
-	if (!wait_queue_member(&sip->si_wait_queue, ut->uu_select.wqset))
-		wait_queue_link_noalloc(&sip->si_wait_queue, ut->uu_select.wqset,
+	if (!wait_queue_member(&sip->si_wait_queue, ut->uu_wqset))
+		wait_queue_link_noalloc(&sip->si_wait_queue, ut->uu_wqset,
 					(wait_queue_link_t)p_wql);
 
 	return;
 }
 
 void
-selwakeup(sip)
-	register struct selinfo *sip;
+selwakeup(struct selinfo *sip)
 {
 	
 	if ((sip->si_flags & SI_INITED) == 0) {
@@ -1682,8 +1795,7 @@ selwakeup(sip)
 }
 
 void 
-selthreadclear(sip)
-	register struct selinfo *sip;
+selthreadclear(struct selinfo *sip)
 {
 
 	if ((sip->si_flags & SI_INITED) == 0) {
@@ -1789,7 +1901,7 @@ evprocenque(struct eventqelt *evq)
 	assert(evq);
 	p = evq->ee_proc;
 
-	KERNEL_DEBUG(DBG_MISC_ENQUEUE|DBG_FUNC_START, evq, evq->ee_flags, evq->ee_eventmask,0,0);
+	KERNEL_DEBUG(DBG_MISC_ENQUEUE|DBG_FUNC_START, (uint32_t)evq, evq->ee_flags, evq->ee_eventmask,0,0);
 
 	proc_lock(p);
 
@@ -1881,7 +1993,7 @@ postpipeevent(struct pipe *pipep, int event)
 			 */
 		        evq->ee_req.er_eventbits |= mask;
 
-			KERNEL_DEBUG(DBG_MISC_POST, evq, evq->ee_req.er_eventbits, mask, 1,0);
+			KERNEL_DEBUG(DBG_MISC_POST, (uint32_t)evq, evq->ee_req.er_eventbits, mask, 1,0);
 
 			evprocenque(evq);
 		}
@@ -1889,7 +2001,7 @@ postpipeevent(struct pipe *pipep, int event)
 	KERNEL_DEBUG(DBG_MISC_POST|DBG_FUNC_END, 0,0,0,1,0);
 }
 
-
+#if SOCKETS
 /*
  * given either a sockbuf or a socket run down the
  * event list and queue ready events found...
@@ -2056,6 +2168,7 @@ postevent(struct socket *sp, struct sockbuf *sb, int event)
 	}
 	KERNEL_DEBUG(DBG_MISC_POST|DBG_FUNC_END, (int)sp, 0, 0, 0, 0);
 }
+#endif /* SOCKETS */
 
 
 /*
@@ -2066,13 +2179,17 @@ postevent(struct socket *sp, struct sockbuf *sb, int event)
  * via waitevent().
  *
  * should this prevent duplicate events on same socket?
+ *
+ * Returns:
+ *		ENOMEM			No memory for operation
+ *	copyin:EFAULT
  */
 int
 watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
 {
 	struct eventqelt *evq = (struct eventqelt *)0;
 	struct eventqelt *np = NULL;
-	struct eventreq *erp;
+	struct eventreq64 *erp;
 	struct fileproc *fp = NULL;
 	int error;
 
@@ -2082,18 +2199,36 @@ watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
 	MALLOC(evq, struct eventqelt *, sizeof(struct eventqelt), M_TEMP, M_WAITOK);
 
 	if (evq == NULL)
-	        panic("can't MALLOC evq");
+		return (ENOMEM);
 	erp = &evq->ee_req;
 
 	// get users request pkt
-	if ( (error = copyin(CAST_USER_ADDR_T(uap->u_req), (caddr_t)erp,
-			   sizeof(struct eventreq))) ) {
-		FREE(evq, M_TEMP);
 
+	if (IS_64BIT_PROCESS(p)) {
+	        error = copyin(uap->u_req, (caddr_t)erp, sizeof(struct eventreq64));
+	} else {
+	        struct eventreq32 er32;
+
+	        error = copyin(uap->u_req, (caddr_t)&er32, sizeof(struct eventreq32));
+		if (error == 0) {
+		       /*
+			* the user only passes in the
+			* er_type, er_handle and er_data...
+			* the other fields are initialized
+			* below, so don't bother to copy
+			*/
+		        erp->er_type = er32.er_type;
+		        erp->er_handle = er32.er_handle;
+		        erp->er_data = (user_addr_t)er32.er_data;
+		}
+	}
+	if (error) {
+	        FREE(evq, M_TEMP);
 		KERNEL_DEBUG(DBG_MISC_WATCH|DBG_FUNC_END, error,0,0,0,0);
-		return(error);
+
+		return(error);		
 	}
-	KERNEL_DEBUG(DBG_MISC_WATCH, erp->er_handle,uap->u_eventmask,evq,0,0);
+	KERNEL_DEBUG(DBG_MISC_WATCH, erp->er_handle,uap->u_eventmask,(uint32_t)evq,0,0);
 
 	// validate, freeing qelt if errors
 	error = 0;
@@ -2103,9 +2238,11 @@ watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
 		error = EINVAL;
 	} else if ((error = fp_lookup(p, erp->er_handle, &fp, 1)) != 0) {
 		error = EBADF;
+#if SOCKETS
 	} else if (fp->f_type == DTYPE_SOCKET) {
 		socket_lock((struct socket *)fp->f_data, 1);
 		np = ((struct socket *)fp->f_data)->so_evlist.tqh_first;
+#endif /* SOCKETS */
 	} else if (fp->f_type == DTYPE_PIPE) {
 		PIPE_LOCK((struct pipe *)fp->f_data);
 		np = ((struct pipe *)fp->f_data)->pipe_evlist.tqh_first;
@@ -2127,9 +2264,11 @@ watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
 	 */
 	for ( ; np != NULL; np = np->ee_slist.tqe_next) {
 		if (np->ee_proc == p) {
+#if SOCKETS
 			if (fp->f_type == DTYPE_SOCKET)
 				socket_unlock((struct socket *)fp->f_data, 1);
 			else 
+#endif /* SOCKETS */
 				PIPE_UNLOCK((struct pipe *)fp->f_data);
 			fp_drop(p, erp->er_handle, fp, 0);
 			FREE(evq, M_TEMP);
@@ -2143,12 +2282,15 @@ watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
 	evq->ee_eventmask = uap->u_eventmask & EV_MASK;
 	evq->ee_flags = 0;
 
+#if SOCKETS
 	if (fp->f_type == DTYPE_SOCKET) {
 		TAILQ_INSERT_TAIL(&((struct socket *)fp->f_data)->so_evlist, evq, ee_slist);
 		postevent((struct socket *)fp->f_data, 0, EV_RWBYTES); // catch existing events
 
 		socket_unlock((struct socket *)fp->f_data, 1);
-	} else {
+	} else
+#endif /* SOCKETS */
+	{
 		TAILQ_INSERT_TAIL(&((struct pipe *)fp->f_data)->pipe_evlist, evq, ee_slist);
 		postpipeevent((struct pipe *)fp->f_data, EV_RWBYTES);
 
@@ -2166,20 +2308,49 @@ watchevent(proc_t p, struct watchevent_args *uap, __unused int *retval)
  * waitevent system call.
  * grabs the next waiting event for this proc and returns
  * it. if no events, user can request to sleep with timeout
- * or poll mode (tv=NULL);
+ * or without or poll mode
+ *    ((tv != NULL && interval == 0) || tv == -1)
  */
 int
 waitevent(proc_t p, struct waitevent_args *uap, int *retval)
 {
         int error = 0;
 	struct eventqelt *evq;
-	struct eventreq   er;
+	struct eventreq64 *erp;
 	uint64_t abstime, interval;
+	boolean_t fast_poll = FALSE;
+	union {
+	        struct eventreq64 er64;
+	        struct eventreq32 er32;
+	} uer;
+
+	interval = 0;
 
 	if (uap->tv) {
 		struct timeval atv;
+		/*
+		 * check for fast poll method
+		 */
+		if (IS_64BIT_PROCESS(p)) {
+		        if (uap->tv == (user_addr_t)-1)
+			        fast_poll = TRUE;
+		} else if (uap->tv == (user_addr_t)((uint32_t)-1))
+		        fast_poll = TRUE;
+
+		if (fast_poll == TRUE) {
+		        if (p->p_evlist.tqh_first == NULL) {
+				KERNEL_DEBUG(DBG_MISC_WAIT|DBG_FUNC_NONE, -1,0,0,0,0);
+				/*
+				 * poll failed
+				 */
+			        *retval = 1;
+				return (0);
+			}
+			proc_lock(p);
+			goto retry;
+		}
+		error = copyin(uap->tv, (caddr_t)&atv, sizeof (atv));
 
-		error = copyin(CAST_USER_ADDR_T(uap->tv), (caddr_t)&atv, sizeof (atv));
 		if (error)
 			return(error);
 		if (itimerfix(&atv)) {
@@ -2187,9 +2358,7 @@ waitevent(proc_t p, struct waitevent_args *uap, int *retval)
 			return(error);
 		}
 		interval = tvtoabstime(&atv);
-	} else
-		interval = 0;
-
+	}
 	KERNEL_DEBUG(DBG_MISC_WAIT|DBG_FUNC_START, 0,0,0,0,0);
 
 	proc_lock(p);
@@ -2201,18 +2370,32 @@ retry:
 		 * don't want to hold the proc lock across a copyout because
 		 * it might block on a page fault at the target in user space
 		 */
-	        bcopy((caddr_t)&evq->ee_req, (caddr_t)&er, sizeof (struct eventreq));
+	        erp = &evq->ee_req;
 
+		if (IS_64BIT_PROCESS(p))
+		        bcopy((caddr_t)erp, (caddr_t)&uer.er64, sizeof (struct eventreq64));
+		else {
+		        uer.er32.er_type  = erp->er_type;
+		        uer.er32.er_handle  = erp->er_handle;
+		        uer.er32.er_data  = (uint32_t)erp->er_data;
+		        uer.er32.er_ecnt  = erp->er_ecnt;
+		        uer.er32.er_rcnt  = erp->er_rcnt;
+		        uer.er32.er_wcnt  = erp->er_wcnt;
+		        uer.er32.er_eventbits = erp->er_eventbits;
+		}
 	        TAILQ_REMOVE(&p->p_evlist, evq, ee_plist);
 
 		evq->ee_flags &= ~EV_QUEUED;
 
 		proc_unlock(p);
 
-		error = copyout((caddr_t)&er, CAST_USER_ADDR_T(uap->u_req), sizeof(struct eventreq));
+		if (IS_64BIT_PROCESS(p))
+		        error = copyout((caddr_t)&uer.er64, uap->u_req, sizeof(struct eventreq64));
+		else
+		        error = copyout((caddr_t)&uer.er32, uap->u_req, sizeof(struct eventreq32));
 
 		KERNEL_DEBUG(DBG_MISC_WAIT|DBG_FUNC_END, error,
-			     evq->ee_req.er_handle,evq->ee_req.er_eventbits,evq,0);
+			     evq->ee_req.er_handle,evq->ee_req.er_eventbits,(uint32_t)evq,0);
 		return (error);
 	}
 	else {
@@ -2228,11 +2411,11 @@ retry:
 		else
 		        abstime = 0;
 
-		KERNEL_DEBUG(DBG_MISC_WAIT, 1,&p->p_evlist,0,0,0);
+		KERNEL_DEBUG(DBG_MISC_WAIT, 1,(uint32_t)&p->p_evlist,0,0,0);
 
 		error = msleep1(&p->p_evlist, &p->p_mlock, (PSOCK | PCATCH), "waitevent", abstime);
 
-		KERNEL_DEBUG(DBG_MISC_WAIT, 2,&p->p_evlist,0,0,0);
+		KERNEL_DEBUG(DBG_MISC_WAIT, 2,(uint32_t)&p->p_evlist,0,0,0);
 
 		if (error == 0)
 			goto retry;
@@ -2258,9 +2441,9 @@ retry:
 int
 modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 {
-	struct eventreq er;
-	struct eventreq *erp = &er;
-	struct eventqelt *evq;
+	struct eventreq64 er;
+	struct eventreq64 *erp = &er;
+	struct eventqelt *evq = NULL;	/* protected by error return */
 	int error;
 	struct fileproc *fp;
 	int flag;
@@ -2269,10 +2452,12 @@ modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 
 	/*
 	 * get user's request pkt
+	 * just need the er_type and er_handle which sit above the
+	 * problematic er_data (32/64 issue)... so only copy in
+	 * those 2 fields
 	 */
-	if ((error = copyin(CAST_USER_ADDR_T(uap->u_req), (caddr_t)erp,
-			     sizeof(struct eventreq)))) {
-			KERNEL_DEBUG(DBG_MISC_MOD|DBG_FUNC_END, error,0,0,0,0);
+	if ((error = copyin(uap->u_req, (caddr_t)erp, sizeof(er.er_type) + sizeof(er.er_handle)))) {
+	        KERNEL_DEBUG(DBG_MISC_MOD|DBG_FUNC_END, error,0,0,0,0);
 	        return(error);
 	}
 	proc_fdlock(p);
@@ -2281,9 +2466,11 @@ modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 		error = EINVAL;
 	} else if ((error = fp_lookup(p, erp->er_handle, &fp, 1)) != 0) {
 		error = EBADF;
+#if SOCKETS
 	} else if (fp->f_type == DTYPE_SOCKET) {
 		socket_lock((struct socket *)fp->f_data, 1);
 		evq = ((struct socket *)fp->f_data)->so_evlist.tqh_first;
+#endif /* SOCKETS */
 	} else if (fp->f_type == DTYPE_PIPE) {
 		PIPE_LOCK((struct pipe *)fp->f_data);
 		evq = ((struct pipe *)fp->f_data)->pipe_evlist.tqh_first;
@@ -2309,23 +2496,28 @@ modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 		        break;
 	}
 	if (evq == NULL) {
+#if SOCKETS
 		if (fp->f_type == DTYPE_SOCKET) 
 			socket_unlock((struct socket *)fp->f_data, 1);
-		else 
+		else
+#endif /* SOCKETS */
 			PIPE_UNLOCK((struct pipe *)fp->f_data);
 		fp_drop(p, erp->er_handle, fp, 0);
 		KERNEL_DEBUG(DBG_MISC_MOD|DBG_FUNC_END, EINVAL,0,0,0,0);
 		return(EINVAL);
 	}
-	KERNEL_DEBUG(DBG_MISC_MOD, erp->er_handle,uap->u_eventmask,evq,0,0);
+	KERNEL_DEBUG(DBG_MISC_MOD, erp->er_handle,uap->u_eventmask,(uint32_t)evq,0,0);
 
 	if (uap->u_eventmask == EV_RM) {
 		EVPROCDEQUE(p, evq);
 
+#if SOCKETS
 		if (fp->f_type == DTYPE_SOCKET) {
 			TAILQ_REMOVE(&((struct socket *)fp->f_data)->so_evlist, evq, ee_slist);
 			socket_unlock((struct socket *)fp->f_data, 1);
-		} else {
+		} else
+#endif /* SOCKETS */
+		{
 			TAILQ_REMOVE(&((struct pipe *)fp->f_data)->pipe_evlist, evq, ee_slist);
 			PIPE_UNLOCK((struct pipe *)fp->f_data);
 		}
@@ -2357,9 +2549,11 @@ modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 		break;
 
 	default:
+#if SOCKETS
 		if (fp->f_type == DTYPE_SOCKET) 
 			socket_unlock((struct socket *)fp->f_data, 1);
 		else 
+#endif /* SOCKETS */
 			PIPE_UNLOCK((struct pipe *)fp->f_data);
 		fp_drop(p, erp->er_handle, fp, 0);
 		KERNEL_DEBUG(DBG_MISC_WATCH|DBG_FUNC_END, EINVAL,0,0,0,0);
@@ -2388,16 +2582,18 @@ modwatch(proc_t p, struct modwatch_args *uap, __unused int *retval)
 	evq->ee_req.er_eventbits = 0;
 	evq->ee_eventmask = uap->u_eventmask & EV_MASK;
 
+#if SOCKETS
 	if (fp->f_type == DTYPE_SOCKET) {
 		postevent((struct socket *)fp->f_data, 0, flag);
 		socket_unlock((struct socket *)fp->f_data, 1);
-	}
-	else {
+	} else
+#endif /* SOCKETS */
+	{
 		postpipeevent((struct pipe *)fp->f_data, flag);
 		PIPE_UNLOCK((struct pipe *)fp->f_data);
 	}
 	fp_drop(p, erp->er_handle, fp, 0);
-	KERNEL_DEBUG(DBG_MISC_MOD|DBG_FUNC_END, evq->ee_req.er_handle,evq->ee_eventmask,fp->f_data,flag,0);
+	KERNEL_DEBUG(DBG_MISC_MOD|DBG_FUNC_END, evq->ee_req.er_handle,evq->ee_eventmask,(uint32_t)fp->f_data,flag,0);
 	return(0);
 }
 
@@ -2410,11 +2606,13 @@ waitevent_close(struct proc *p, struct fileproc *fp)
 
 	fp->f_flags &= ~FP_WAITEVENT;
 
+#if SOCKETS
 	if (fp->f_type == DTYPE_SOCKET) {
 	        socket_lock((struct socket *)fp->f_data, 1);
 		evq = ((struct socket *)fp->f_data)->so_evlist.tqh_first;
-	}
-	else if (fp->f_type == DTYPE_PIPE) {
+	} else
+#endif /* SOCKETS */
+	if (fp->f_type == DTYPE_PIPE) {
 	        PIPE_LOCK((struct pipe *)fp->f_data);
 		evq = ((struct pipe *)fp->f_data)->pipe_evlist.tqh_first;
 	}
@@ -2430,9 +2628,11 @@ waitevent_close(struct proc *p, struct fileproc *fp)
 		        break;
 	}
 	if (evq == NULL) {
+#if SOCKETS
 	        if (fp->f_type == DTYPE_SOCKET) 
 		        socket_unlock((struct socket *)fp->f_data, 1);
 		else 
+#endif /* SOCKETS */
 		        PIPE_UNLOCK((struct pipe *)fp->f_data);
 
 		proc_fdlock(p);
@@ -2441,10 +2641,13 @@ waitevent_close(struct proc *p, struct fileproc *fp)
 	}
 	EVPROCDEQUE(p, evq);
 
+#if SOCKETS
 	if (fp->f_type == DTYPE_SOCKET) {
 		TAILQ_REMOVE(&((struct socket *)fp->f_data)->so_evlist, evq, ee_slist);
 		socket_unlock((struct socket *)fp->f_data, 1);
-	} else {
+	} else
+#endif /* SOCKETS */
+	{
 		TAILQ_REMOVE(&((struct pipe *)fp->f_data)->pipe_evlist, evq, ee_slist);
 		PIPE_UNLOCK((struct pipe *)fp->f_data);
 	}
@@ -2455,3 +2658,60 @@ waitevent_close(struct proc *p, struct fileproc *fp)
 	return(0);
 }
 
+
+/*
+ * gethostuuid
+ *
+ * Description:	Get the host UUID from IOKit and return it to user space.
+ *
+ * Parameters:	uuid_buf		Pointer to buffer to receive UUID
+ *		timeout			Timespec for timout
+ *
+ * Returns:	0			Success
+ *		EWOULDBLOCK		Timeout is too short
+ *		copyout:EFAULT		Bad user buffer
+ *
+ * Notes:	A timeout seems redundant, since if it's tolerable to not
+ *		have a system UUID in hand, then why ask for one?
+ */
+int
+gethostuuid(struct proc *p, struct gethostuuid_args *uap, __unused register_t *retval)
+{
+	kern_return_t kret;
+	int error;
+	mach_timespec_t mach_ts;	/* for IOKit call */
+	__darwin_uuid_t uuid_kern;	/* for IOKit call */
+
+	/* Convert the 32/64 bit timespec into a mach_timespec_t */
+	if ( proc_is64bit(p) ) {
+		struct user_timespec ts;
+		error = copyin(uap->timeoutp, &ts, sizeof(ts));
+		if (error)
+			return (error);
+		mach_ts.tv_sec = ts.tv_sec;
+		mach_ts.tv_nsec = ts.tv_nsec;
+	} else {
+		struct timespec ts;
+		error = copyin(uap->timeoutp, &ts, sizeof(ts) );
+		if (error)
+			return (error);
+		mach_ts.tv_sec = ts.tv_sec;
+		mach_ts.tv_nsec = ts.tv_nsec;
+	}
+
+	/* Call IOKit with the stack buffer to get the UUID */
+	kret = IOBSDGetPlatformUUID(uuid_kern, mach_ts);
+
+	/*
+	 * If we get it, copy out the data to the user buffer; note that a
+	 * uuid_t is an array of characters, so this is size invariant for
+	 * 32 vs. 64 bit.
+	 */
+	if (kret == KERN_SUCCESS) {
+		error = copyout(uuid_kern, uap->uuid_buf, sizeof(uuid_kern));
+	} else {
+		error = EWOULDBLOCK;
+	}
+
+	return (error);
+}
diff --git a/bsd/kern/sys_pipe.c b/bsd/kern/sys_pipe.c
index 0b5355183..3190f00f1 100644
--- a/bsd/kern/sys_pipe.c
+++ b/bsd/kern/sys_pipe.c
@@ -17,25 +17,37 @@
  *    are met.
  */
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 /*
@@ -151,18 +163,21 @@
  * interfaces to the outside world
  */
 static int pipe_read(struct fileproc *fp, struct uio *uio,
-                kauth_cred_t cred, int flags, struct proc *p);
+                int flags, vfs_context_t ctx);
 
 static int pipe_write(struct fileproc *fp, struct uio *uio,
-                kauth_cred_t cred, int flags, struct proc *p);
+                int flags, vfs_context_t ctx);
 
-static int pipe_close(struct fileglob *fg, struct proc *p);
+static int pipe_close(struct fileglob *fg, vfs_context_t ctx);
 
-static int pipe_select(struct fileproc *fp, int which, void * wql, struct proc *p);
+static int pipe_select(struct fileproc *fp, int which, void * wql,
+		vfs_context_t ctx);
 
-static int pipe_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
+static int pipe_kqfilter(struct fileproc *fp, struct knote *kn,
+		vfs_context_t ctx);
 
-static int pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p);
+static int pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data,
+		vfs_context_t ctx);
 
 
 struct  fileops pipeops =
@@ -172,7 +187,7 @@ struct  fileops pipeops =
     pipe_select,
     pipe_close,
     pipe_kqfilter,
-    0 };
+    NULL };
 
 
 static void	filt_pipedetach(struct knote *kn);
@@ -223,7 +238,6 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, pipekvawired, CTLFLAG_RD,
 	   &amountpipekvawired, 0, "Pipe wired KVA usage");
 #endif
 
-void pipeinit(void *dummy __unused);
 static void pipeclose(struct pipe *cpipe);
 static void pipe_free_kmem(struct pipe *cpipe);
 static int pipe_create(struct pipe **cpipep);
@@ -253,7 +267,7 @@ static zone_t pipe_zone;
 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
 
 void
-pipeinit(void *dummy __unused)
+pipeinit(void)
 {
         pipe_zone = (zone_t)zinit(sizeof(struct pipe), 8192 * sizeof(struct pipe), 4096, "pipe zone");
 
@@ -269,6 +283,34 @@ pipeinit(void *dummy __unused)
 	pipe_mtx_attr = lck_attr_alloc_init();
 }
 
+/* Bitmap for things to touch in pipe_touch() */
+#define	PIPE_ATIME	0x00000001	/* time of last access */
+#define	PIPE_MTIME	0x00000002	/* time of last modification */
+#define	PIPE_CTIME	0x00000004	/* time of last status change */
+
+static void
+pipe_touch(struct pipe *tpipe, int touch)
+{
+	struct timeval now;
+
+	microtime(&now);
+
+	if (touch & PIPE_ATIME) {
+		tpipe->st_atimespec.tv_sec  = now.tv_sec;
+		tpipe->st_atimespec.tv_nsec = now.tv_usec * 1000;
+	}
+
+	if (touch & PIPE_MTIME) {
+		tpipe->st_mtimespec.tv_sec  = now.tv_sec;
+		tpipe->st_mtimespec.tv_nsec = now.tv_usec * 1000;
+	}
+
+	if (touch & PIPE_CTIME) {
+		tpipe->st_ctimespec.tv_sec  = now.tv_sec;
+		tpipe->st_ctimespec.tv_nsec = now.tv_usec * 1000;
+	}
+}
+
 
 
 /*
@@ -277,7 +319,7 @@ pipeinit(void *dummy __unused)
 
 /* ARGSUSED */
 int
-pipe(struct proc *p, __unused struct pipe_args *uap, register_t *retval)
+pipe(proc_t p, __unused struct pipe_args *uap, register_t *retval)
 {
 	struct fileproc *rf, *wf;
 	struct pipe *rpipe, *wpipe;
@@ -314,7 +356,7 @@ pipe(struct proc *p, __unused struct pipe_args *uap, register_t *retval)
 	TAILQ_INIT(&rpipe->pipe_evlist);
 	TAILQ_INIT(&wpipe->pipe_evlist);
 
-	error = falloc(p, &rf, &fd);
+	error = falloc(p, &rf, &fd, vfs_context_current());
 	if (error) {
 	        goto freepipes;
 	}
@@ -330,7 +372,7 @@ pipe(struct proc *p, __unused struct pipe_args *uap, register_t *retval)
 	rf->f_data = (caddr_t)rpipe;
 	rf->f_ops = &pipeops;
 
-	error = falloc(p, &wf, &fd);
+	error = falloc(p, &wf, &fd, vfs_context_current());
 	if (error) {
 		fp_free(p, retval[0], rf);
 	        goto freepipes;
@@ -342,22 +384,23 @@ pipe(struct proc *p, __unused struct pipe_args *uap, register_t *retval)
 
 	rpipe->pipe_peer = wpipe;
 	wpipe->pipe_peer = rpipe;
-
 	rpipe->pipe_mtxp = wpipe->pipe_mtxp = pmtx;
+
 	retval[1] = fd;
-#ifdef MAC
+#if CONFIG_MACF
 	/*
 	 * XXXXXXXX SHOULD NOT HOLD FILE_LOCK() XXXXXXXXXXXX
 	 *
 	 * struct pipe represents a pipe endpoint.  The MAC label is shared
-	 * between the connected endpoints.  As a result mac_init_pipe() and
-	 * mac_create_pipe() should only be called on one of the endpoints
+	 * between the connected endpoints.  As a result mac_pipe_label_init() and
+	 * mac_pipe_label_associate() should only be called on one of the endpoints
 	 * after they have been connected.
 	 */
-	mac_init_pipe(rpipe);
-	mac_create_pipe(td->td_ucred, rpipe);
+	mac_pipe_label_init(rpipe);
+	mac_pipe_label_associate(kauth_cred_get(), rpipe);
+	wpipe->pipe_label = rpipe->pipe_label;
 #endif
-	proc_fdlock(p);
+	proc_fdlock_spin(p);
 	procfdtbl_releasefd(p, retval[0], NULL);
 	procfdtbl_releasefd(p, retval[1], NULL);
 	fp_drop(p, retval[0], rf, 1);
@@ -375,56 +418,116 @@ freepipes:
 	return (error);
 }
 
-
 int
-pipe_stat(struct pipe *cpipe, struct stat *ub)
+pipe_stat(struct pipe *cpipe, void *ub, int isstat64)
 {
-#ifdef MAC
+#if CONFIG_MACF
         int error;
 #endif
-	struct timeval now;
+	int	pipe_size = 0;
+	int	pipe_count;
+	struct stat *sb = (struct stat *)0;	/* warning avoidance ; protected by isstat64 */
+	struct stat64 * sb64 = (struct stat64 *)0;  /* warning avoidance ; protected by isstat64 */
 
 	if (cpipe == NULL)
 	        return (EBADF);
-#ifdef MAC
 	PIPE_LOCK(cpipe);
-	error = mac_check_pipe_stat(active_cred, cpipe);
-	PIPE_UNLOCK(cpipe);
-	if (error)
+
+#if CONFIG_MACF
+	error = mac_pipe_check_stat(kauth_cred_get(), cpipe);
+	if (error) {
+		PIPE_UNLOCK(cpipe);
 	        return (error);
+	}
 #endif
 	if (cpipe->pipe_buffer.buffer == 0) {
 	        /*
 		 * must be stat'ing the write fd
 		 */
-	        cpipe = cpipe->pipe_peer;
-
-		if (cpipe == NULL)
-		        return (EBADF);
+	        if (cpipe->pipe_peer) {
+		        /*
+			 * the peer still exists, use it's info
+			 */
+		        pipe_size  = cpipe->pipe_peer->pipe_buffer.size;
+			pipe_count = cpipe->pipe_peer->pipe_buffer.cnt;
+		} else {
+			pipe_count = 0;
+		}
+	} else {
+	        pipe_size  = cpipe->pipe_buffer.size;
+		pipe_count = cpipe->pipe_buffer.cnt;
 	}
-	bzero(ub, sizeof(*ub));
-	ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
-	ub->st_blksize = cpipe->pipe_buffer.size;
-	ub->st_size = cpipe->pipe_buffer.cnt;
-	ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
-	ub->st_nlink = 1;
+	/*
+	 * since peer's buffer is setup ouside of lock
+	 * we might catch it in transient state
+	 */
+	if (pipe_size == 0)
+		pipe_size  = PIPE_SIZE;
 
-	ub->st_uid = kauth_getuid();
-	ub->st_gid = kauth_getgid();
+	if (isstat64 != 0) {
+		sb64 = (struct stat64 *)ub;	
 
-	microtime(&now);
-	ub->st_atimespec.tv_sec  = now.tv_sec;
-	ub->st_atimespec.tv_nsec = now.tv_usec * 1000;
+		bzero(sb64, sizeof(*sb64));
+		sb64->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
+		sb64->st_blksize = pipe_size;
+		sb64->st_size = pipe_count;
+		sb64->st_blocks = (sb64->st_size + sb64->st_blksize - 1) / sb64->st_blksize;
+	
+		sb64->st_uid = kauth_getuid();
+		sb64->st_gid = kauth_getgid();
+	
+		sb64->st_atimespec.tv_sec  = cpipe->st_atimespec.tv_sec;
+		sb64->st_atimespec.tv_nsec = cpipe->st_atimespec.tv_nsec;
+	
+		sb64->st_mtimespec.tv_sec  = cpipe->st_mtimespec.tv_sec;
+		sb64->st_mtimespec.tv_nsec = cpipe->st_mtimespec.tv_nsec;
 
-	ub->st_mtimespec.tv_sec  = now.tv_sec;
-	ub->st_mtimespec.tv_nsec = now.tv_usec * 1000;
+		sb64->st_ctimespec.tv_sec  = cpipe->st_ctimespec.tv_sec;
+		sb64->st_ctimespec.tv_nsec = cpipe->st_ctimespec.tv_nsec;
 
-	ub->st_ctimespec.tv_sec  = now.tv_sec;
-	ub->st_ctimespec.tv_nsec = now.tv_usec * 1000;
+		/*
+	 	* Return a relatively unique inode number based on the current
+	 	* address of this pipe's struct pipe.  This number may be recycled
+	 	* relatively quickly.
+	 	*/
+		sb64->st_ino = (ino64_t)((uint32_t)cpipe);
+	} else {
+		sb = (struct stat *)ub;	
+
+		bzero(sb, sizeof(*sb));
+		sb->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
+		sb->st_blksize = pipe_size;
+		sb->st_size = pipe_count;
+		sb->st_blocks = (sb->st_size + sb->st_blksize - 1) / sb->st_blksize;
+	
+		sb->st_uid = kauth_getuid();
+		sb->st_gid = kauth_getgid();
+	
+		sb->st_atimespec.tv_sec  = cpipe->st_atimespec.tv_sec;
+		sb->st_atimespec.tv_nsec = cpipe->st_atimespec.tv_nsec;
+	
+		sb->st_mtimespec.tv_sec  = cpipe->st_mtimespec.tv_sec;
+		sb->st_mtimespec.tv_nsec = cpipe->st_mtimespec.tv_nsec;
+
+		sb->st_ctimespec.tv_sec  = cpipe->st_ctimespec.tv_sec;
+		sb->st_ctimespec.tv_nsec = cpipe->st_ctimespec.tv_nsec;
+
+		/*
+	 	* Return a relatively unique inode number based on the current
+	 	* address of this pipe's struct pipe.  This number may be recycled
+	 	* relatively quickly.
+	 	*/
+		sb->st_ino = (ino_t)cpipe;
+	}
+	PIPE_UNLOCK(cpipe);
 
 	/*
-	 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid.
-	 * XXX (st_dev, st_ino) should be unique.
+	 * POSIX: Left as 0: st_dev, st_nlink, st_rdev, st_flags, st_gen,
+	 * st_uid, st_gid.
+	 *
+	 * XXX (st_dev) should be unique, but there is no device driver that
+	 * XXX is associated with pipes, since they are implemented via a
+	 * XXX struct fileops indirection rather than as FS objects.
 	 */
 	return (0);
 }
@@ -479,6 +582,9 @@ pipe_create(struct pipe **cpipep)
 	 */
 	bzero(cpipe, sizeof *cpipe);
 
+	/* Initial times are all the time of creation of the pipe */
+	pipe_touch(cpipe, PIPE_ATIME | PIPE_MTIME | PIPE_CTIME);
+
 	return (0);
 }
 
@@ -486,10 +592,8 @@ pipe_create(struct pipe **cpipep)
 /*
  * lock a pipe for I/O, blocking other access
  */
-static __inline int
-pipelock(cpipe, catch)
-	struct pipe *cpipe;
-	int catch;
+static inline int
+pipelock(struct pipe *cpipe, int catch)
 {
 	int error;
 
@@ -509,11 +613,9 @@ pipelock(cpipe, catch)
 /*
  * unlock a pipe I/O lock
  */
-static __inline void
-pipeunlock(cpipe)
-	struct pipe *cpipe;
+static inline void
+pipeunlock(struct pipe *cpipe)
 {
-
 	cpipe->pipe_state &= ~PIPE_LOCKFL;
 
 	if (cpipe->pipe_state & PIPE_LWANT) {
@@ -523,11 +625,8 @@ pipeunlock(cpipe)
 }
 
 static void
-pipeselwakeup(cpipe, spipe)
-	struct pipe *cpipe;
-	struct pipe *spipe;
+pipeselwakeup(struct pipe *cpipe, struct pipe *spipe)
 {
-
 	if (cpipe->pipe_state & PIPE_SEL) {
 		cpipe->pipe_state &= ~PIPE_SEL;
 		selwakeup(&cpipe->pipe_sel);
@@ -538,18 +637,17 @@ pipeselwakeup(cpipe, spipe)
 	postpipeevent(cpipe, EV_RWBYTES);
 
 	if (spipe && (spipe->pipe_state & PIPE_ASYNC) && spipe->pipe_pgid) {
-	        struct proc *p;
-
 	        if (spipe->pipe_pgid < 0)
 		        gsignal(-spipe->pipe_pgid, SIGIO);
-		else if ((p = pfind(spipe->pipe_pgid)) != (struct proc *)0)
-		        psignal(p, SIGIO);
+		else 
+		        proc_signal(spipe->pipe_pgid, SIGIO);
         }
 }
 
 /* ARGSUSED */
 static int
-pipe_read(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cred, __unused int flags, __unused struct proc *p)
+pipe_read(struct fileproc *fp, struct uio *uio, __unused int flags,
+	__unused vfs_context_t ctx)
 {
 	struct pipe *rpipe = (struct pipe *)fp->f_data;
 	int error;
@@ -563,8 +661,8 @@ pipe_read(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cre
 	if (error)
 		goto unlocked_error;
 
-#ifdef MAC
-	error = mac_check_pipe_read(active_cred, rpipe);
+#if CONFIG_MACF
+	error = mac_pipe_check_read(kauth_cred_get(), rpipe);
 	if (error)
 		goto locked_error;
 #endif
@@ -678,7 +776,7 @@ pipe_read(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cre
 				goto unlocked_error;
 		}
 	}
-#ifdef MAC
+#if CONFIG_MACF
 locked_error:
 #endif
 	pipeunlock(rpipe);
@@ -705,6 +803,9 @@ unlocked_error:
 	if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) >= PIPE_BUF)
 		pipeselwakeup(rpipe, rpipe->pipe_peer);
 
+	/* update last read time */
+	pipe_touch(rpipe, PIPE_ATIME);
+
 	PIPE_UNLOCK(rpipe);
 
 	return (error);
@@ -955,7 +1056,8 @@ error1:
 
 
 static int
-pipe_write(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cred, __unused int flags, __unused struct proc *p)
+pipe_write(struct fileproc *fp, struct uio *uio, __unused int flags,
+	__unused vfs_context_t ctx)
 {
 	int error = 0;
 	int orig_resid;
@@ -974,8 +1076,8 @@ pipe_write(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cr
 		PIPE_UNLOCK(rpipe);
 		return (EPIPE);
 	}
-#ifdef MAC
-	error = mac_check_pipe_write(active_cred, wpipe);
+#if CONFIG_MACF
+	error = mac_pipe_check_write(kauth_cred_get(), wpipe);
 	if (error) {
 		PIPE_UNLOCK(rpipe);
 		return (error);
@@ -1264,6 +1366,10 @@ pipe_write(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cr
 		 */
 		pipeselwakeup(wpipe, wpipe);
 	}
+
+	/* Update modification, status change (# of bytes in pipe) times */
+	pipe_touch(rpipe, PIPE_MTIME | PIPE_CTIME);
+	pipe_touch(wpipe, PIPE_MTIME | PIPE_CTIME);
 	PIPE_UNLOCK(rpipe);
 
 	return (error);
@@ -1274,17 +1380,18 @@ pipe_write(struct fileproc *fp, struct uio *uio, __unused kauth_cred_t active_cr
  */
 /* ARGSUSED 3 */
 static int
-pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, __unused struct proc *p)
+pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data,
+	__unused vfs_context_t ctx)
 {
 	struct pipe *mpipe = (struct pipe *)fp->f_data;
-#ifdef MAC
+#if CONFIG_MACF
 	int error;
 #endif
 
 	PIPE_LOCK(mpipe);
 
-#ifdef MAC
-	error = mac_check_pipe_ioctl(active_cred, mpipe, cmd, data);
+#if CONFIG_MACF
+	error = mac_pipe_check_ioctl(kauth_cred_get(), mpipe, cmd);
 	if (error) {
 		PIPE_UNLOCK(mpipe);
 
@@ -1336,7 +1443,7 @@ pipe_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, __unused struct proc *
 
 
 static int
-pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
+pipe_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
 {
 	struct pipe *rpipe = (struct pipe *)fp->f_data;
 	struct pipe *wpipe;
@@ -1349,6 +1456,17 @@ pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
 
 	wpipe = rpipe->pipe_peer;
 
+#if CONFIG_MACF
+	/*
+	 * XXX We should use a per thread credential here; minimally, the
+	 * XXX process credential should have a persistent reference on it
+	 * XXX before being passed in here.
+	 */
+	if (mac_pipe_check_select(vfs_context_ucred(ctx), rpipe, which)) {
+		PIPE_UNLOCK(rpipe);
+		return (0);
+	}
+#endif
         switch (which) {
 
         case FREAD:
@@ -1359,7 +1477,7 @@ pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
 		        retnum = 1;
 		} else {
 		        rpipe->pipe_state |= PIPE_SEL;
-		        selrecord(p, &rpipe->pipe_sel, wql);
+		        selrecord(vfs_context_proc(ctx), &rpipe->pipe_sel, wql);
 		}
 		break;
 
@@ -1371,12 +1489,12 @@ pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
 		        retnum = 1;
 		} else {
 		        wpipe->pipe_state |= PIPE_SEL;
-			selrecord(p, &wpipe->pipe_sel, wql);
+			selrecord(vfs_context_proc(ctx), &wpipe->pipe_sel, wql);
 		}
 		break;
         case 0:
 	        rpipe->pipe_state |= PIPE_SEL;
-		selrecord(p, &rpipe->pipe_sel, wql);
+		selrecord(vfs_context_proc(ctx), &rpipe->pipe_sel, wql);
 		break;
         }
 	PIPE_UNLOCK(rpipe);
@@ -1387,14 +1505,14 @@ pipe_select(struct fileproc *fp, int which, void *wql, struct proc *p)
 
 /* ARGSUSED 1 */
 static int
-pipe_close(struct fileglob *fg, __unused struct proc *p)
+pipe_close(struct fileglob *fg, __unused vfs_context_t ctx)
 {
         struct pipe *cpipe;
 
-	proc_fdlock(p);
+	proc_fdlock_spin(vfs_context_proc(ctx));
 	cpipe = (struct pipe *)fg->fg_data;
 	fg->fg_data = NULL;
-	proc_fdunlock(p);
+	proc_fdunlock(vfs_context_proc(ctx));
 
 	if (cpipe)
 	        pipeclose(cpipe);
@@ -1446,23 +1564,27 @@ pipeclose(struct pipe *cpipe)
 	if (PIPE_MTX(cpipe) != NULL)
 		PIPE_LOCK(cpipe);
 		
-	pipeselwakeup(cpipe, cpipe);
 
 	/*
 	 * If the other side is blocked, wake it up saying that
 	 * we want to close it down.
 	 */
+	cpipe->pipe_state |= PIPE_EOF;
+	pipeselwakeup(cpipe, cpipe);
+	
 	while (cpipe->pipe_busy) {
-		cpipe->pipe_state |= PIPE_WANT | PIPE_EOF;
+		cpipe->pipe_state |= PIPE_WANT;
 
 		wakeup(cpipe);
-
  		msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
 	}
 
-#ifdef MAC
+#if CONFIG_MACF
+	/*
+	 * Free the shared pipe label only after the two ends are disconnected.
+	 */
 	if (cpipe->pipe_label != NULL && cpipe->pipe_peer == NULL)
-		mac_destroy_pipe(cpipe);
+		mac_pipe_label_destroy(cpipe);
 #endif
 
 	/*
@@ -1507,20 +1629,31 @@ pipeclose(struct pipe *cpipe)
 	zfree(pipe_zone, cpipe);
 }
 
-
 /*ARGSUSED*/
 static int
-pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused struct proc *p)
+pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused vfs_context_t ctx)
 {
 	struct pipe *cpipe;
 
 	cpipe = (struct pipe *)kn->kn_fp->f_data;
 
 	PIPE_LOCK(cpipe);
+#if CONFIG_MACF
+	/*
+	 * XXX We should use a per thread credential here; minimally, the
+	 * XXX process credential should have a persistent reference on it
+	 * XXX before being passed in here.
+	 */
+	if (mac_pipe_check_kqfilter(vfs_context_ucred(ctx), kn, cpipe) != 0) {
+		PIPE_UNLOCK(cpipe);
+		return (1);
+	}
+#endif
 
 	switch (kn->kn_filter) {
 	case EVFILT_READ:
 		kn->kn_fop = &pipe_rfiltops;
+
 		break;
 	case EVFILT_WRITE:
 		kn->kn_fop = &pipe_wfiltops;
@@ -1532,6 +1665,7 @@ pipe_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused struct pr
 		        PIPE_UNLOCK(cpipe);
 			return (EPIPE);
 		}
+		if (cpipe->pipe_peer)
 		cpipe = cpipe->pipe_peer;
 		break;
 	default:
@@ -1595,9 +1729,10 @@ filt_piperead(struct knote *kn, long hint)
 	    (wpipe == NULL) || (wpipe->pipe_state & PIPE_EOF)) {
 		kn->kn_flags |= EV_EOF;
 		retval = 1;
-	} else
+	} else {
 		retval = (kn->kn_sfflags & NOTE_LOWAT) ?
 		         (kn->kn_data >= kn->kn_sdata) : (kn->kn_data > 0);
+	}
 
 	if (hint == 0)
 	        PIPE_UNLOCK(rpipe);
@@ -1632,6 +1767,8 @@ filt_pipewrite(struct knote *kn, long hint)
 		return (1);
 	}
 	kn->kn_data = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
+	if (!kn->kn_data && wpipe->pipe_buffer.size == 0)
+		kn->kn_data = 1; /* unwritten pipe is ready for write */
 
 #ifndef PIPE_NODIRECT
 	if (wpipe->pipe_state & PIPE_DIRECTW)
@@ -1647,53 +1784,71 @@ filt_pipewrite(struct knote *kn, long hint)
 int
 fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo)
 {
-#ifdef MAC
+#if CONFIG_MACF
         int error;
 #endif
 	struct timeval now;
-	struct stat * ub;
+	struct vinfo_stat * ub;
+	int pipe_size = 0;
+	int pipe_count;
 
 	if (cpipe == NULL)
 	        return (EBADF);
-#ifdef MAC
 	PIPE_LOCK(cpipe);
-	error = mac_check_pipe_stat(active_cred, cpipe);
-	PIPE_UNLOCK(cpipe);
-	if (error)
+
+#if CONFIG_MACF
+	error = mac_pipe_check_stat(kauth_cred_get(), cpipe);
+	if (error) {
+		PIPE_UNLOCK(cpipe);
 	        return (error);
+	}
 #endif
 	if (cpipe->pipe_buffer.buffer == 0) {
 	        /*
 		 * must be stat'ing the write fd
 		 */
-	        cpipe = cpipe->pipe_peer;
-
-		if (cpipe == NULL)
-		        return (EBADF);
+	        if (cpipe->pipe_peer) {
+		        /*
+			 * the peer still exists, use it's info
+			 */
+		        pipe_size  = cpipe->pipe_peer->pipe_buffer.size;
+			pipe_count = cpipe->pipe_peer->pipe_buffer.cnt;
+		} else {
+			pipe_count = 0;
+		}
+	} else {
+	        pipe_size  = cpipe->pipe_buffer.size;
+		pipe_count = cpipe->pipe_buffer.cnt;
 	}
+	/*
+	 * since peer's buffer is setup ouside of lock
+	 * we might catch it in transient state
+	 */
+	if (pipe_size == 0)
+		pipe_size  = PIPE_SIZE;
 
 	ub = &pinfo->pipe_stat;
 
 	bzero(ub, sizeof(*ub));
-	ub->st_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
-	ub->st_blksize = cpipe->pipe_buffer.size;
-	ub->st_size = cpipe->pipe_buffer.cnt;
-	if (ub->st_blksize != 0);
-		ub->st_blocks = (ub->st_size + ub->st_blksize - 1) / ub->st_blksize;
-	ub->st_nlink = 1;
+	ub->vst_mode = S_IFIFO | S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
+	ub->vst_blksize = pipe_size;
+	ub->vst_size = pipe_count;
+	if (ub->vst_blksize != 0)
+		ub->vst_blocks = (ub->vst_size + ub->vst_blksize - 1) / ub->vst_blksize;
+	ub->vst_nlink = 1;
 
-	ub->st_uid = kauth_getuid();
-	ub->st_gid = kauth_getgid();
+	ub->vst_uid = kauth_getuid();
+	ub->vst_gid = kauth_getgid();
 
 	microtime(&now);
-	ub->st_atimespec.tv_sec  = now.tv_sec;
-	ub->st_atimespec.tv_nsec = now.tv_usec * 1000;
+	ub->vst_atime  = now.tv_sec;
+	ub->vst_atimensec = now.tv_usec * 1000;
 
-	ub->st_mtimespec.tv_sec  = now.tv_sec;
-	ub->st_mtimespec.tv_nsec = now.tv_usec * 1000;
+	ub->vst_mtime  = now.tv_sec;
+	ub->vst_mtimensec = now.tv_usec * 1000;
 
-	ub->st_ctimespec.tv_sec  = now.tv_sec;
-	ub->st_ctimespec.tv_nsec = now.tv_usec * 1000;
+	ub->vst_ctime  = now.tv_sec;
+	ub->vst_ctimensec = now.tv_usec * 1000;
 
 	/*
 	 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid.
@@ -1703,6 +1858,8 @@ fill_pipeinfo(struct pipe * cpipe, struct pipe_info * pinfo)
 	pinfo->pipe_handle = (uint64_t)((uintptr_t)cpipe);
 	pinfo->pipe_peerhandle = (uint64_t)((uintptr_t)(cpipe->pipe_peer));
 	pinfo->pipe_status = cpipe->pipe_state;
+
+	PIPE_UNLOCK(cpipe);
+
 	return (0);
 }
-
diff --git a/bsd/kern/sys_socket.c b/bsd/kern/sys_socket.c
index 1e7b7d3c0..a508b87ac 100644
--- a/bsd/kern/sys_socket.c
+++ b/bsd/kern/sys_socket.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993
@@ -53,6 +59,12 @@
  *
  *	@(#)sys_socket.c	8.1 (Berkeley) 6/10/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -68,105 +80,150 @@
 #include <sys/filedesc.h>
 #include <sys/kauth.h>
 #include <sys/signalvar.h>
+#include <sys/vnode.h>
 
 #include <net/if.h>
 #include <net/route.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 /*
  * File operations on sockets.
  */
-int	soo_read(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
-		int flags, struct proc *p);
-int	soo_write(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
-		int flags, struct proc *p);
-int soo_close(struct fileglob *fp, struct proc *p);
-int	soo_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p);
-int	soo_stat(struct socket *so, struct stat *ub);
-int	soo_select(struct fileproc *fp, int which, void * wql, struct proc *p);
-int     soo_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
-int	soo_drain(struct fileproc *fp, struct proc *p);
-
-struct	fileops socketops =
-    { soo_read, soo_write, soo_ioctl, soo_select, soo_close, soo_kqfilter, soo_drain };
+static int soo_read(struct fileproc *, struct uio *, int, vfs_context_t ctx);
+static int soo_write(struct fileproc *, struct uio *, int, vfs_context_t ctx);
+static int soo_close(struct fileglob *, vfs_context_t ctx);
+static int soo_drain(struct fileproc *, vfs_context_t ctx);
+
+/* TODO: these should be in header file */
+extern int soo_ioctl(struct fileproc *, u_long, caddr_t, vfs_context_t ctx);
+extern int soo_stat(struct socket *, void *, int);
+extern int soo_select(struct fileproc *, int, void *, vfs_context_t ctx);
+extern int soo_kqfilter(struct fileproc *, struct knote *, vfs_context_t ctx);
+
+struct fileops socketops = {
+	soo_read, soo_write, soo_ioctl, soo_select, soo_close,
+	soo_kqfilter, soo_drain
+};
 
 /* ARGSUSED */
-int
-soo_read(
-	struct fileproc *fp,
-	struct uio *uio,
-	__unused kauth_cred_t cred,
-	__unused int flags,
-	__unused struct proc *p)
+static int
+soo_read(struct fileproc *fp, struct uio *uio, __unused int flags,
+#if !CONFIG_MACF_SOCKET
+	__unused
+#endif
+	vfs_context_t ctx)
 {
 	struct socket *so;
 	int stat;
-	int (*fsoreceive)(struct socket *so2, 
-			       struct sockaddr **paddr,
-			       struct uio *uio2, struct mbuf **mp0,
-			       struct mbuf **controlp, int *flagsp);
+#if CONFIG_MACF_SOCKET
+	int error;
+#endif
 
+	int (*fsoreceive)(struct socket *so2, struct sockaddr **paddr,
+	    struct uio *uio2, struct mbuf **mp0, struct mbuf **controlp,
+	    int *flagsp);
 
+	if ((so = (struct socket *)fp->f_fglob->fg_data) == NULL) {
+		/* This is not a valid open file descriptor */
+		return (EBADF);
+	}
+
+#if CONFIG_MACF_SOCKET
+	error = mac_socket_check_receive(vfs_context_ucred(ctx), so);
+	if (error)
+		return (error);
+#endif /* CONFIG_MACF_SOCKET */
 
-        if ((so = (struct socket *)fp->f_fglob->fg_data) == NULL) {
-                /* This is not a valid open file descriptor */
-		return(EBADF);
-        }
 //###LD will have to change
 	fsoreceive = so->so_proto->pr_usrreqs->pru_soreceive;
-	
+
 	stat = (*fsoreceive)(so, 0, uio, 0, 0, 0);
-	return stat;
+	return (stat);
 }
 
 /* ARGSUSED */
-int
-soo_write(
-	struct fileproc *fp,
-	struct uio *uio,
-	__unused kauth_cred_t cred,
-	__unused int flags,
-	struct proc *procp)
+static int
+soo_write(struct fileproc *fp, struct uio *uio, __unused int flags,
+	vfs_context_t ctx)
 {
 	struct socket *so;
-	int	(*fsosend)(struct socket *so2, struct sockaddr *addr,
-				struct uio *uio2, struct mbuf *top,
-				struct mbuf *control, int flags2);
-	int           stat;
+	int stat;
+	int (*fsosend)(struct socket *so2, struct sockaddr *addr,
+	    struct uio *uio2, struct mbuf *top, struct mbuf *control,
+	    int flags2);
+	proc_t procp;
+
+#if CONFIG_MACF_SOCKET
+	int error;
+#endif
 
 	if ((so = (struct socket *)fp->f_fglob->fg_data) == NULL) {
 		/* This is not a valid open file descriptor */
 		return (EBADF);
 	}
 
+#if CONFIG_MACF_SOCKET
+	/* JMM - have to fetch the socket's remote addr */
+	error = mac_socket_check_send(vfs_context_ucred(ctx), so, NULL);
+	if (error)
+		return (error);
+#endif /* CONFIG_MACF_SOCKET */
+
 	fsosend = so->so_proto->pr_usrreqs->pru_sosend;
 
 	stat = (*fsosend)(so, 0, uio, 0, 0, 0);
 
 	/* Generation of SIGPIPE can be controlled per socket */
+	procp = vfs_context_proc(ctx);
 	if (stat == EPIPE && procp && !(so->so_flags & SOF_NOSIGPIPE))
 		psignal(procp, SIGPIPE);
 
-	return stat;
+	return (stat);
 }
 
 __private_extern__ int
-soioctl(
-	struct socket *so,
-	u_long cmd,
-	caddr_t data,
-	struct proc *p)
+soioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 {
 	struct sockopt sopt;
-	int    error = 0;
+	int error = 0;
 	int dropsockref = -1;
-
-
+	
 	socket_lock(so, 1);
 
 	sopt.sopt_level = cmd;
 	sopt.sopt_name = (int)data;
 	sopt.sopt_p = p;
 
+	/* Call the socket filter's ioctl handler for most ioctls */
+	if (IOCGROUP(cmd) != 'i' && IOCGROUP(cmd) != 'r') {
+		int filtered = 0;
+		struct socket_filter_entry *filter;
+
+		for (filter = so->so_filt; filter && error == 0;
+		    filter = filter->sfe_next_onsocket) {
+			if (filter->sfe_filter->sf_filter.sf_ioctl) {
+				if (filtered == 0) {
+					sflt_use(so);
+					socket_unlock(so, 0);
+					filtered = 1;
+				}
+				error = filter->sfe_filter->sf_filter.
+				    sf_ioctl(filter->sfe_cookie, so, cmd, data);
+			}
+		}
+
+		if (filtered) {
+			socket_lock(so, 0);
+			sflt_unuse(so);
+		}
+
+		if (error != 0)
+			goto out;
+	}
+
 	switch (cmd) {
 
 	case FIONBIO:
@@ -202,131 +259,141 @@ soioctl(
 		goto out;
 
 	case SIOCATMARK:
-	     *(int *)data = (so->so_state&SS_RCVATMARK) != 0;
+		*(int *)data = (so->so_state&SS_RCVATMARK) != 0;
 		goto out;
 
 	case SIOCSETOT: {
-	     /*
-	      * Set socket level options here and then call protocol
-	      * specific routine.
-	      */
-	     struct socket	*cloned_so = NULL;
-	     int				cloned_fd = *(int *)data;
-
-	     /* let's make sure it's either -1 or a valid file descriptor */
-	     if (cloned_fd != -1) {
-		  error = file_socket(cloned_fd, &cloned_so);
-		  if (error) {
-			goto out;
-		  }
-		 dropsockref = cloned_fd;
-	     }
-
-	     /* Always set socket non-blocking for OT */
-	     so->so_state |= SS_NBIO;
-	     so->so_options |= SO_DONTTRUNC | SO_WANTMORE;
-          so->so_flags |= SOF_NOSIGPIPE;
-
-	     if (cloned_so && so != cloned_so) {
-		  /* Flags options */
-		  so->so_options |= cloned_so->so_options & ~SO_ACCEPTCONN;
-
-            /* SO_LINGER */
-            if (so->so_options & SO_LINGER)
-                so->so_linger = cloned_so->so_linger;
-
-            /* SO_SNDBUF, SO_RCVBUF */
-		  if (cloned_so->so_snd.sb_hiwat > 0) {
-		       if (sbreserve(&so->so_snd, cloned_so->so_snd.sb_hiwat) == 0) {
-			    error = ENOBUFS;
-			    goto out;
-		       }
-		  }
-		  if (cloned_so->so_rcv.sb_hiwat > 0) {
-		       if (sbreserve(&so->so_rcv, cloned_so->so_rcv.sb_hiwat) == 0) {
-			    error = ENOBUFS;
-			    goto out;
-		       }
-		  }
-
-		  /* SO_SNDLOWAT, SO_RCVLOWAT */
-		  so->so_snd.sb_lowat =
-		       (cloned_so->so_snd.sb_lowat > so->so_snd.sb_hiwat) ?
-		       so->so_snd.sb_hiwat : cloned_so->so_snd.sb_lowat;
-		  so->so_rcv.sb_lowat =
-		       (cloned_so->so_rcv.sb_lowat > so->so_rcv.sb_hiwat) ?
-		       so->so_rcv.sb_hiwat : cloned_so->so_rcv.sb_lowat;
-
-            	  /* SO_SNDTIMEO, SO_RCVTIMEO */
-		  so->so_snd.sb_timeo = cloned_so->so_snd.sb_timeo;
-		  so->so_rcv.sb_timeo = cloned_so->so_rcv.sb_timeo;
-	     }
-
-	     error = (*so->so_proto->pr_usrreqs->pru_control)(so, cmd, data, 0, p);
-	     /* Just ignore protocols that do not understand it */
-	     if (error == EOPNOTSUPP)
-		  error = 0;
+		/*
+		 * Set socket level options here and then call protocol
+		 * specific routine.
+		 */
+		struct socket *cloned_so = NULL;
+		int cloned_fd = *(int *)data;
+
+		/* let's make sure it's either -1 or a valid file descriptor */
+		if (cloned_fd != -1) {
+			error = file_socket(cloned_fd, &cloned_so);
+			if (error) {
+				goto out;
+			}
+			dropsockref = cloned_fd;
+		}
+
+		/* Always set socket non-blocking for OT */
+		so->so_state |= SS_NBIO;
+		so->so_options |= SO_DONTTRUNC | SO_WANTMORE;
+		so->so_flags |= SOF_NOSIGPIPE;
+
+		if (cloned_so && so != cloned_so) {
+			/* Flags options */
+			so->so_options |=
+			    cloned_so->so_options & ~SO_ACCEPTCONN;
+
+			/* SO_LINGER */
+			if (so->so_options & SO_LINGER)
+				so->so_linger = cloned_so->so_linger;
+
+			/* SO_SNDBUF, SO_RCVBUF */
+			if (cloned_so->so_snd.sb_hiwat > 0) {
+				if (sbreserve(&so->so_snd,
+				    cloned_so->so_snd.sb_hiwat) == 0) {
+					error = ENOBUFS;
+					goto out;
+				}
+			}
+			if (cloned_so->so_rcv.sb_hiwat > 0) {
+				if (sbreserve(&so->so_rcv,
+				    cloned_so->so_rcv.sb_hiwat) == 0) {
+					error = ENOBUFS;
+					goto out;
+				}
+			}
+
+			/* SO_SNDLOWAT, SO_RCVLOWAT */
+			so->so_snd.sb_lowat =
+			    (cloned_so->so_snd.sb_lowat > so->so_snd.sb_hiwat) ?
+			    so->so_snd.sb_hiwat : cloned_so->so_snd.sb_lowat;
+			so->so_rcv.sb_lowat =
+			    (cloned_so->so_rcv.sb_lowat > so->so_rcv.sb_hiwat) ?
+			    so->so_rcv.sb_hiwat : cloned_so->so_rcv.sb_lowat;
+
+			/* SO_SNDTIMEO, SO_RCVTIMEO */
+			so->so_snd.sb_timeo = cloned_so->so_snd.sb_timeo;
+			so->so_rcv.sb_timeo = cloned_so->so_rcv.sb_timeo;
+		}
+
+		error = (*so->so_proto->pr_usrreqs->pru_control)(so, cmd,
+		    data, 0, p);
+		/* Just ignore protocols that do not understand it */
+		if (error == EOPNOTSUPP)
+			error = 0;
 
 		goto out;
-        }
+	}
 	}
 	/*
 	 * Interface/routing/protocol specific ioctls:
 	 * interface and routing ioctls should have a
 	 * different entry since a socket's unnecessary
 	 */
-	if (IOCGROUP(cmd) == 'i')
-	     error = ifioctllocked(so, cmd, data, p);
-	else 
-	     if (IOCGROUP(cmd) == 'r')
-		  error = rtioctl(cmd, data, p);
-	     else
-		  error = (*so->so_proto->pr_usrreqs->pru_control)(so, cmd, data, 0, p);
+	if (IOCGROUP(cmd) == 'i') {
+		error = ifioctllocked(so, cmd, data, p);
+	} else {
+		if (IOCGROUP(cmd) == 'r')
+			error = rtioctl(cmd, data, p);
+		else
+			error = (*so->so_proto->pr_usrreqs->pru_control)(so,
+			    cmd, data, 0, p);
+	}
 
 out:
 	if (dropsockref != -1)
 		file_drop(dropsockref);
 	socket_unlock(so, 1);
 
-	return error;
+	if (error == EJUSTRETURN)
+		error = 0;
+
+	return (error);
 }
 
 int
-soo_ioctl(fp, cmd, data, p)
-	struct fileproc *fp;
-	u_long cmd;
-	register caddr_t data;
-	struct proc *p;
+soo_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, vfs_context_t ctx)
 {
-	register struct socket *so;
+	struct socket *so;
 	int error;
-
+	proc_t procp = vfs_context_proc(ctx);
 
 	if ((so = (struct socket *)fp->f_fglob->fg_data) == NULL) {
 		/* This is not a valid open file descriptor */
 		return (EBADF);
 	}
-	
-	error = soioctl(so, cmd, data, p);
-	
+
+	error = soioctl(so, cmd, data, procp);
+
 	if (error == 0 && cmd == SIOCSETOT)
 		fp->f_fglob->fg_flag |= FNONBLOCK;
 
-	return error;
+	return (error);
 }
 
 int
-soo_select(fp, which, wql, p)
-	struct fileproc *fp;
-	int which;
-	void * wql;
-	struct proc *p;
+soo_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx)
 {
-	register struct socket *so = (struct socket *)fp->f_fglob->fg_data;
-	int retnum=0;
+	struct socket *so = (struct socket *)fp->f_fglob->fg_data;
+	int retnum = 0;
+	proc_t procp;
+
+	if (so == NULL || so == (struct socket *)-1)
+		return (0);
+	
+	procp = vfs_context_proc(ctx);
 
-	if (so == NULL || so == (struct socket*)-1)
+#if CONFIG_MACF_SOCKET
+	if (mac_socket_check_select(vfs_context_ucred(ctx), so, which) != 0);
 		return (0);
+#endif /* CONFIG_MACF_SOCKET */
+
 
 	socket_lock(so, 1);
 	switch (which) {
@@ -338,7 +405,7 @@ soo_select(fp, which, wql, p)
 			so->so_rcv.sb_flags &= ~SB_SEL;
 			goto done;
 		}
-		selrecord(p, &so->so_rcv.sb_sel, wql);
+		selrecord(procp, &so->so_rcv.sb_sel, wql);
 		break;
 
 	case FWRITE:
@@ -348,7 +415,7 @@ soo_select(fp, which, wql, p)
 			so->so_snd.sb_flags &= ~SB_SEL;
 			goto done;
 		}
-		selrecord(p, &so->so_snd.sb_sel, wql);
+		selrecord(procp, &so->so_snd.sb_sel, wql);
 		break;
 
 	case 0:
@@ -358,34 +425,69 @@ soo_select(fp, which, wql, p)
 			so->so_rcv.sb_flags &= ~SB_SEL;
 			goto done;
 		}
-		selrecord(p, &so->so_rcv.sb_sel, wql);
+		selrecord(procp, &so->so_rcv.sb_sel, wql);
 		break;
 	}
-	
+
 done:
 	socket_unlock(so, 1);
 	return (retnum);
 }
 
-
 int
-soo_stat(so, ub)
-	register struct socket *so;
-	register struct stat *ub;
+soo_stat(struct socket *so, void *ub, int isstat64)
 {
-	int stat;
+	int ret;
+	/* warning avoidance ; protected by isstat64 */
+	struct stat *sb = (struct stat *)0;
+	/* warning avoidance ; protected by isstat64 */
+	struct stat64 *sb64 = (struct stat64 *)0;
+
+#if CONFIG_MACF_SOCKET
+	ret = mac_socket_check_stat(kauth_cred_get(), so);
+	if (ret)
+		return (ret);
+#endif
+
+	if (isstat64 != 0) {
+		sb64 = (struct stat64 *)ub;
+		bzero((caddr_t)sb64, sizeof (*sb64));
+	} else {
+		sb = (struct stat *)ub;
+		bzero((caddr_t)sb, sizeof (*sb));
+	}
 
-	bzero((caddr_t)ub, sizeof (*ub));
 	socket_lock(so, 1);
-	ub->st_mode = S_IFSOCK;
-	stat = (*so->so_proto->pr_usrreqs->pru_sense)(so, ub);
+	if (isstat64 != 0) {
+		sb64->st_mode = S_IFSOCK;
+		if ((so->so_state & SS_CANTRCVMORE) == 0 ||
+		    so->so_rcv.sb_cc != 0)
+			sb64->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
+		if ((so->so_state & SS_CANTSENDMORE) == 0)
+			sb64->st_mode |= S_IWUSR | S_IWGRP | S_IWOTH;
+		sb64->st_size = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
+		sb64->st_uid = so->so_uid;
+		sb64->st_gid = -1;	/* XXX -- what else to do? */
+	} else {
+		sb->st_mode = S_IFSOCK;
+		if ((so->so_state & SS_CANTRCVMORE) == 0 ||
+		    so->so_rcv.sb_cc != 0)
+			sb->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
+		if ((so->so_state & SS_CANTSENDMORE) == 0)
+			sb->st_mode |= S_IWUSR | S_IWGRP | S_IWOTH;
+		sb->st_size = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
+		sb->st_uid = so->so_uid;
+		sb->st_gid = -1;	/* XXX -- what else to do? */
+	}
+
+	ret = (*so->so_proto->pr_usrreqs->pru_sense)(so, ub, isstat64);
 	socket_unlock(so, 1);
-	return stat;
+	return (ret);
 }
 
 /* ARGSUSED */
-int
-soo_close(struct fileglob *fg, __unused proc_t p)
+static int
+soo_close(struct fileglob *fg, __unused vfs_context_t ctx)
 {
 	int error = 0;
 	struct socket *sp;
@@ -393,16 +495,14 @@ soo_close(struct fileglob *fg, __unused proc_t p)
 	sp = (struct socket *)fg->fg_data;
 	fg->fg_data = NULL;
 
-
 	if (sp)
-	     error = soclose(sp);
-
+		error = soclose(sp);
 
 	return (error);
 }
 
-int
-soo_drain(struct fileproc *fp, __unused struct proc *p)
+static int
+soo_drain(struct fileproc *fp, __unused vfs_context_t ctx)
 {
 	int error = 0;
 	struct socket *so = (struct socket *)fp->f_fglob->fg_data;
@@ -414,10 +514,9 @@ soo_drain(struct fileproc *fp, __unused struct proc *p)
 		wakeup((caddr_t)&so->so_timeo);
 		sorwakeup(so);
 		sowwakeup(so);
-	
+
 		socket_unlock(so, 1);
 	}
 
-	return error;
+	return (error);
 }
-
diff --git a/bsd/kern/syscalls.c b/bsd/kern/syscalls.c
index 503f43f00..82ded92dd 100644
--- a/bsd/kern/syscalls.c
+++ b/bsd/kern/syscalls.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  * 
- * @APPLE_LICENSE_HEADER_START@ 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and 
- * are subject to the Apple Public Source License Version 1.1 (the 
- * "License").  You may not use this file except in compliance with the 
- * License.  Please obtain a copy of the License at 
- * http://www.apple.com/publicsource and read it before using this file. 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are 
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the 
- * License for the specific language governing rights and limitations 
- * under the License. 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * @APPLE_LICENSE_HEADER_END@ 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  * 
  * 
  * System call switch table.
@@ -58,12 +64,21 @@ const char *syscallnames[] = {
 	"getuid",             /* 24 = getuid */
 	"geteuid",            /* 25 = geteuid */
 	"ptrace",             /* 26 = ptrace */
+#if SOCKETS
 	"recvmsg",            /* 27 = recvmsg */
 	"sendmsg",            /* 28 = sendmsg */
 	"recvfrom",           /* 29 = recvfrom */
 	"accept",             /* 30 = accept */
 	"getpeername",        /* 31 = getpeername */
 	"getsockname",        /* 32 = getsockname */
+#else
+	"#27",                /* 27 = */
+	"#28",                /* 28 = */
+	"#29",                /* 29 = */
+	"#30",                /* 30 = */
+	"#31",                /* 31 = */
+	"#32",                /* 32 = */
+#endif /* SOCKETS */
 	"access",             /* 33 = access */
 	"chflags",            /* 34 = chflags */
 	"fchflags",           /* 35 = fchflags */
@@ -76,7 +91,7 @@ const char *syscallnames[] = {
 	"pipe",               /* 42 = pipe */
 	"getegid",            /* 43 = getegid */
 	"profil",             /* 44 = profil */
-	"ktrace",             /* 45 = ktrace */
+	"#45",                /* 45 = old ktrace */
 	"sigaction",          /* 46 = sigaction */
 	"getgid",             /* 47 = getgid */
 	"sigprocmask",        /* 48 = sigprocmask */
@@ -128,28 +143,48 @@ const char *syscallnames[] = {
 	"#94",                /* 94 = old setdopt */
 	"fsync",              /* 95 = fsync */
 	"setpriority",        /* 96 = setpriority */
+#if SOCKETS
 	"socket",             /* 97 = socket */
 	"connect",            /* 98 = connect */
+#else
+	"#97",                /* 97 = */
+	"#98",                /* 98 = */
+#endif /* SOCKETS */
 	"#99",                /* 99 = old accept */
 	"getpriority",        /* 100 = getpriority */
 	"#101",               /* 101 = old send */
 	"#102",               /* 102 = old recv */
 	"#103",               /* 103 = old sigreturn */
+#if SOCKETS
 	"bind",               /* 104 = bind */
 	"setsockopt",         /* 105 = setsockopt */
 	"listen",             /* 106 = listen */
+#else
+	"#104",               /* 104 = */
+	"#105",               /* 105 = */
+	"#106",               /* 106 = */
+#endif /* SOCKETS */
 	"#107",               /* 107 = old vtimes */
 	"#108",               /* 108 = old sigvec */
 	"#109",               /* 109 = old sigblock */
 	"#110",               /* 110 = old sigsetmask */
 	"sigsuspend",         /* 111 = sigsuspend */
 	"#112",               /* 112 = old sigstack */
+#if SOCKETS
 	"#113",               /* 113 = old recvmsg */
 	"#114",               /* 114 = old sendmsg */
+#else
+	"#113",               /* 113 = */
+	"#114",               /* 114 = */
+#endif /* SOCKETS */
 	"#115",               /* 115 = old vtrace */
 	"gettimeofday",       /* 116 = gettimeofday */
 	"getrusage",          /* 117 = getrusage */
+#if SOCKETS
 	"getsockopt",         /* 118 = getsockopt */
+#else
+	"#118",               /* 118 = */
+#endif /* SOCKETS */
 	"#119",               /* 119 = old resuba */
 	"readv",              /* 120 = readv */
 	"writev",             /* 121 = writev */
@@ -157,23 +192,29 @@ const char *syscallnames[] = {
 	"fchown",             /* 123 = fchown */
 	"fchmod",             /* 124 = fchmod */
 	"#125",               /* 125 = old recvfrom */
-	"#126",               /* 126 = old setreuid */
-	"#127",               /* 127 = old setregid */
+	"setreuid",           /* 126 = setreuid */
+	"setregid",           /* 127 = setregid */
 	"rename",             /* 128 = rename */
 	"#129",               /* 129 = old truncate */
 	"#130",               /* 130 = old ftruncate */
 	"flock",              /* 131 = flock */
 	"mkfifo",             /* 132 = mkfifo */
+#if SOCKETS
 	"sendto",             /* 133 = sendto */
 	"shutdown",           /* 134 = shutdown */
 	"socketpair",         /* 135 = socketpair */
+#else
+	"#133",               /* 133 = */
+	"#134",               /* 134 = */
+	"#135",               /* 135 = */
+#endif /* SOCKETS */
 	"mkdir",              /* 136 = mkdir */
 	"rmdir",              /* 137 = rmdir */
 	"utimes",             /* 138 = utimes */
 	"futimes",            /* 139 = futimes */
 	"adjtime",            /* 140 = adjtime */
 	"#141",               /* 141 = old getpeername */
-	"#142",               /* 142 = old gethostid */
+	"gethostuuid",        /* 142 = gethostuuid */
 	"#143",               /* 143 = old sethostid */
 	"#144",               /* 144 = old getrlimit */
 	"#145",               /* 145 = old setrlimit */
@@ -196,7 +237,7 @@ const char *syscallnames[] = {
 	"fstatfs",            /* 158 = fstatfs */
 	"unmount",            /* 159 = unmount */
 	"#160",               /* 160 = old async_daemon */
-#if NFSCLIENT
+#if NFSSERVER
 	"getfh",              /* 161 = getfh */
 #else
 	"#161",               /* 161 = */
@@ -208,7 +249,7 @@ const char *syscallnames[] = {
 	"#166",               /* 166 = old exportfs */
 	"mount",              /* 167 = mount */
 	"#168",               /* 168 = old ustat */
-	"#169",               /* 169 = */
+	"csops",              /* 169 = csops */
 	"table",              /* 170 = table old table */
 	"#171",               /* 171 = old wait3 */
 	"#172",               /* 172 = old rpause */
@@ -232,11 +273,7 @@ const char *syscallnames[] = {
 	"lstat",              /* 190 = lstat */
 	"pathconf",           /* 191 = pathconf */
 	"fpathconf",          /* 192 = fpathconf */
-#if COMPAT_GETFSSTAT
-	"getfsstat",          /* 193 = getfsstat */
-#else
 	"#193",               /* 193 = */
-#endif
 	"getrlimit",          /* 194 = getrlimit */
 	"setrlimit",          /* 195 = setrlimit */
 	"getdirentries",      /* 196 = getdirentries */
@@ -249,6 +286,7 @@ const char *syscallnames[] = {
 	"mlock",              /* 203 = mlock */
 	"munlock",            /* 204 = munlock */
 	"undelete",           /* 205 = undelete */
+#if NETAT
 	"ATsocket",           /* 206 = ATsocket */
 	"ATgetmsg",           /* 207 = ATgetmsg */
 	"ATputmsg",           /* 208 = ATputmsg */
@@ -257,6 +295,16 @@ const char *syscallnames[] = {
 	"ATPgetreq",          /* 211 = ATPgetreq */
 	"ATPgetrsp",          /* 212 = ATPgetrsp */
 	"#213",               /* 213 = Reserved for AppleTalk */
+#else
+	"#206",               /* 206 = */
+	"#207",               /* 207 = */
+	"#208",               /* 208 = */
+	"#209",               /* 209 = */
+	"#210",               /* 210 = */
+	"#211",               /* 211 = */
+	"#212",               /* 212 = */
+	"#213",               /* 213 = Reserved for AppleTalk */
+#endif /* NETAT */
 	"kqueue_from_portset_np",  /* 214 = kqueue_from_portset_np */
 	"kqueue_portset_np",  /* 215 = kqueue_portset_np */
 	"mkcomplex",          /* 216 = mkcomplex soon to be obsolete */
@@ -267,11 +315,7 @@ const char *syscallnames[] = {
 	"setattrlist",        /* 221 = setattrlist */
 	"getdirentriesattr",  /* 222 = getdirentriesattr */
 	"exchangedata",       /* 223 = exchangedata */
-#ifdef __APPLE_API_OBSOLETE
-	"checkuseraccess",    /* 224 = checkuseraccess */
-#else
-	"#224",               /* 224 = HFS checkuseraccess check access to a file */
-#endif /* __APPLE_API_OBSOLETE */
+	"#224",               /* 224 = was checkuseraccess */
 	"searchfs",           /* 225 = searchfs */
 	"delete",             /* 226 = delete private delete ( Carbon semantics ) */
 	"copyfile",           /* 227 = copyfile */
@@ -291,33 +335,69 @@ const char *syscallnames[] = {
 	"flistxattr",         /* 241 = flistxattr */
 	"fsctl",              /* 242 = fsctl */
 	"initgroups",         /* 243 = initgroups */
-	"#244",               /* 244 = */
+	"posix_spawn",        /* 244 = posix_spawn */
 	"#245",               /* 245 = */
 	"#246",               /* 246 = */
 #if NFSCLIENT
 	"nfsclnt",            /* 247 = nfsclnt */
-	"fhopen",             /* 248 = fhopen */
 #else
 	"#247",               /* 247 = */
+#endif
+#if NFSSERVER
+	"fhopen",             /* 248 = fhopen */
+#else
 	"#248",               /* 248 = */
 #endif
 	"#249",               /* 249 = */
 	"minherit",           /* 250 = minherit */
+#if SYSV_SEM
 	"semsys",             /* 251 = semsys */
+#else
+	"#251",               /* 251 = */
+#endif
+#if SYSV_MSG
 	"msgsys",             /* 252 = msgsys */
+#else
+	"#252",               /* 252 = */
+#endif
+#if SYSV_SHM
 	"shmsys",             /* 253 = shmsys */
+#else
+	"#253",               /* 253 = */
+#endif
+#if SYSV_SEM
 	"semctl",             /* 254 = semctl */
 	"semget",             /* 255 = semget */
 	"semop",              /* 256 = semop */
 	"#257",               /* 257 = */
+#else
+	"#254",               /* 254 = */
+	"#255",               /* 255 = */
+	"#256",               /* 256 = */
+	"#257",               /* 257 = */
+#endif
+#if SYSV_MSG
 	"msgctl",             /* 258 = msgctl */
 	"msgget",             /* 259 = msgget */
 	"msgsnd",             /* 260 = msgsnd */
 	"msgrcv",             /* 261 = msgrcv */
+#else
+	"#258",               /* 258 = */
+	"#259",               /* 259 = */
+	"#260",               /* 260 = */
+	"#261",               /* 261 = */
+#endif
+#if SYSV_SHM
 	"shmat",              /* 262 = shmat */
 	"shmctl",             /* 263 = shmctl */
 	"shmdt",              /* 264 = shmdt */
 	"shmget",             /* 265 = shmget */
+#else
+	"#262",               /* 262 = */
+	"#263",               /* 263 = */
+	"#264",               /* 264 = */
+	"#265",               /* 265 = */
+#endif
 	"shm_open",           /* 266 = shm_open */
 	"shm_unlink",         /* 267 = shm_unlink */
 	"sem_open",           /* 268 = sem_open */
@@ -346,25 +426,25 @@ const char *syscallnames[] = {
 	"mkfifo_extended",    /* 291 = mkfifo_extended */
 	"mkdir_extended",     /* 292 = mkdir_extended */
 	"identitysvc",        /* 293 = identitysvc */
-	"#294",               /* 294 = */
-	"#295",               /* 295 = */
-	"load_shared_file",   /* 296 = load_shared_file */
-	"reset_shared_file",  /* 297 = reset_shared_file */
-	"new_system_shared_regions",  /* 298 = new_system_shared_regions */
-	"shared_region_map_file_np",  /* 299 = shared_region_map_file_np */
-	"shared_region_make_private_np",  /* 300 = shared_region_make_private_np */
-	"#301",               /* 301 = */
-	"#302",               /* 302 = */
-	"#303",               /* 303 = */
-	"#304",               /* 304 = */
-	"#305",               /* 305 = */
-	"#306",               /* 306 = */
-	"#307",               /* 307 = */
-	"#308",               /* 308 = */
-	"#309",               /* 309 = */
+	"shared_region_check_np",  /* 294 = shared_region_check_np */
+	"shared_region_map_np",  /* 295 = shared_region_map_np */
+	"#296",               /* 296 = old load_shared_file */
+	"#297",               /* 297 = old reset_shared_file */
+	"#298",               /* 298 = old new_system_shared_regions */
+	"#299",               /* 299 = old shared_region_map_file_np */
+	"#300",               /* 300 = old shared_region_make_private_np */
+	"__pthread_mutex_destroy",  /* 301 = __pthread_mutex_destroy */
+	"__pthread_mutex_init",  /* 302 = __pthread_mutex_init */
+	"__pthread_mutex_lock",  /* 303 = __pthread_mutex_lock */
+	"__pthread_mutex_trylock",  /* 304 = __pthread_mutex_trylock */
+	"__pthread_mutex_unlock",  /* 305 = __pthread_mutex_unlock */
+	"__pthread_cond_init",  /* 306 = __pthread_cond_init */
+	"__pthread_cond_destroy",  /* 307 = __pthread_cond_destroy */
+	"__pthread_cond_broadcast",  /* 308 = __pthread_cond_broadcast */
+	"__pthread_cond_signal",  /* 309 = __pthread_cond_signal */
 	"getsid",             /* 310 = getsid */
 	"settid_with_pid",    /* 311 = settid_with_pid */
-	"#312",               /* 312 = */
+	"__pthread_cond_timedwait",  /* 312 = __pthread_cond_timedwait */
 	"aio_fsync",          /* 313 = aio_fsync */
 	"aio_return",         /* 314 = aio_return */
 	"aio_suspend",        /* 315 = aio_suspend */
@@ -373,35 +453,40 @@ const char *syscallnames[] = {
 	"aio_read",           /* 318 = aio_read */
 	"aio_write",          /* 319 = aio_write */
 	"lio_listio",         /* 320 = lio_listio */
-	"#321",               /* 321 = */
-	"#322",               /* 322 = */
+	"__pthread_cond_wait",  /* 321 = __pthread_cond_wait */
+	"iopolicysys",        /* 322 = iopolicysys */
 	"#323",               /* 323 = */
 	"mlockall",           /* 324 = mlockall */
 	"munlockall",         /* 325 = munlockall */
 	"#326",               /* 326 = */
 	"issetugid",          /* 327 = issetugid */
 	"__pthread_kill",     /* 328 = __pthread_kill */
-	"pthread_sigmask",    /* 329 = pthread_sigmask */
-	"sigwait",            /* 330 = sigwait */
+	"__pthread_sigmask",  /* 329 = __pthread_sigmask */
+	"__sigwait",          /* 330 = __sigwait */
 	"__disable_threadsignal",  /* 331 = __disable_threadsignal */
 	"__pthread_markcancel",  /* 332 = __pthread_markcancel */
 	"__pthread_canceled",  /* 333 = __pthread_canceled */
 	"__semwait_signal",   /* 334 = __semwait_signal */
-	"utrace",             /* 335 = utrace */
+	"#335",               /* 335 = old utrace */
 	"proc_info",          /* 336 = proc_info */
+#if SENDFILE
+	"sendfile",           /* 337 = sendfile */
+#else /* !SENDFILE */
 	"#337",               /* 337 = */
-	"#338",               /* 338 = */
-	"#339",               /* 339 = */
-	"#340",               /* 340 = */
-	"#341",               /* 341 = */
-	"#342",               /* 342 = */
-	"#343",               /* 343 = */
-	"#344",               /* 344 = */
-	"#345",               /* 345 = */
-	"#346",               /* 346 = */
-	"#347",               /* 347 = */
-	"#348",               /* 348 = */
-	"#349",               /* 349 = */
+#endif /* SENDFILE */
+	"stat64",             /* 338 = stat64 */
+	"fstat64",            /* 339 = fstat64 */
+	"lstat64",            /* 340 = lstat64 */
+	"stat64_extended",    /* 341 = stat64_extended */
+	"lstat64_extended",   /* 342 = lstat64_extended */
+	"fstat64_extended",   /* 343 = fstat64_extended */
+	"getdirentries64",    /* 344 = getdirentries64 */
+	"statfs64",           /* 345 = statfs64 */
+	"fstatfs64",          /* 346 = fstatfs64 */
+	"getfsstat64",        /* 347 = getfsstat64 */
+	"__pthread_chdir",    /* 348 = __pthread_chdir */
+	"__pthread_fchdir",   /* 349 = __pthread_fchdir */
+#if AUDIT
 	"audit",              /* 350 = audit */
 	"auditon",            /* 351 = auditon */
 	"#352",               /* 352 = */
@@ -412,14 +497,103 @@ const char *syscallnames[] = {
 	"getaudit_addr",      /* 357 = getaudit_addr */
 	"setaudit_addr",      /* 358 = setaudit_addr */
 	"auditctl",           /* 359 = auditctl */
-	"#360",               /* 360 = */
-	"#361",               /* 361 = */
+#else
+	"#350",               /* 350 = */
+	"#351",               /* 351 = */
+	"#352",               /* 352 = */
+	"#353",               /* 353 = */
+	"#354",               /* 354 = */
+	"#355",               /* 355 = */
+	"#356",               /* 356 = */
+	"#357",               /* 357 = */
+	"#358",               /* 358 = */
+	"#359",               /* 359 = */
+#endif
+	"bsdthread_create",   /* 360 = bsdthread_create */
+	"bsdthread_terminate",  /* 361 = bsdthread_terminate */
 	"kqueue",             /* 362 = kqueue */
 	"kevent",             /* 363 = kevent */
 	"lchown",             /* 364 = lchown */
 	"stack_snapshot",     /* 365 = stack_snapshot */
-	"#366",               /* 366 = */
-	"#367",               /* 367 = */
-	"#368",               /* 368 = */
+	"bsdthread_register",  /* 366 = bsdthread_register */
+	"workq_open",         /* 367 = workq_open */
+	"workq_ops",          /* 368 = workq_ops */
 	"#369",               /* 369 = */
+	"#370",               /* 370 = */
+	"#371",               /* 371 = */
+	"#372",               /* 372 = */
+	"#373",               /* 373 = */
+	"#374",               /* 374 = */
+	"#375",               /* 375 = */
+	"#376",               /* 376 = */
+	"#377",               /* 377 = */
+	"#378",               /* 378 = */
+	"#379",               /* 379 = */
+	"__mac_execve",       /* 380 = __mac_execve */
+	"__mac_syscall",      /* 381 = __mac_syscall */
+	"__mac_get_file",     /* 382 = __mac_get_file */
+	"__mac_set_file",     /* 383 = __mac_set_file */
+	"__mac_get_link",     /* 384 = __mac_get_link */
+	"__mac_set_link",     /* 385 = __mac_set_link */
+	"__mac_get_proc",     /* 386 = __mac_get_proc */
+	"__mac_set_proc",     /* 387 = __mac_set_proc */
+	"__mac_get_fd",       /* 388 = __mac_get_fd */
+	"__mac_set_fd",       /* 389 = __mac_set_fd */
+	"__mac_get_pid",      /* 390 = __mac_get_pid */
+	"__mac_get_lcid",     /* 391 = __mac_get_lcid */
+	"__mac_get_lctx",     /* 392 = __mac_get_lctx */
+	"__mac_set_lctx",     /* 393 = __mac_set_lctx */
+	"setlcid",            /* 394 = setlcid */
+	"getlcid",            /* 395 = getlcid */
+	"read_nocancel",      /* 396 = read_nocancel */
+	"write_nocancel",     /* 397 = write_nocancel */
+	"open_nocancel",      /* 398 = open_nocancel */
+	"close_nocancel",     /* 399 = close_nocancel */
+	"wait4_nocancel",     /* 400 = wait4_nocancel */
+#if SOCKETS
+	"recvmsg_nocancel",   /* 401 = recvmsg_nocancel */
+	"sendmsg_nocancel",   /* 402 = sendmsg_nocancel */
+	"recvfrom_nocancel",  /* 403 = recvfrom_nocancel */
+	"accept_nocancel",    /* 404 = accept_nocancel */
+#else
+	"#401",               /* 401 = */
+	"#402",               /* 402 = */
+	"#403",               /* 403 = */
+	"#404",               /* 404 = */
+#endif /* SOCKETS */
+	"msync_nocancel",     /* 405 = msync_nocancel */
+	"fcntl_nocancel",     /* 406 = fcntl_nocancel */
+	"select_nocancel",    /* 407 = select_nocancel */
+	"fsync_nocancel",     /* 408 = fsync_nocancel */
+#if SOCKETS
+	"connect_nocancel",   /* 409 = connect_nocancel */
+#else
+	"#409",               /* 409 = */
+#endif /* SOCKETS */
+	"sigsuspend_nocancel",  /* 410 = sigsuspend_nocancel */
+	"readv_nocancel",     /* 411 = readv_nocancel */
+	"writev_nocancel",    /* 412 = writev_nocancel */
+#if SOCKETS
+	"sendto_nocancel",    /* 413 = sendto_nocancel */
+#else
+	"#413",               /* 413 = */
+#endif /* SOCKETS */
+	"pread_nocancel",     /* 414 = pread_nocancel */
+	"pwrite_nocancel",    /* 415 = pwrite_nocancel */
+	"waitid_nocancel",    /* 416 = waitid_nocancel */
+	"poll_nocancel",      /* 417 = poll_nocancel */
+#if SYSV_MSG
+	"msgsnd_nocancel",    /* 418 = msgsnd_nocancel */
+	"msgrcv_nocancel",    /* 419 = msgrcv_nocancel */
+#else
+	"#418",               /* 418 = */
+	"#419",               /* 419 = */
+#endif
+	"sem_wait_nocancel",  /* 420 = sem_wait_nocancel */
+	"aio_suspend_nocancel",  /* 421 = aio_suspend_nocancel */
+	"__sigwait_nocancel",  /* 422 = __sigwait_nocancel */
+	"__semwait_signal_nocancel",  /* 423 = __semwait_signal_nocancel */
+	"__mac_mount",        /* 424 = __mac_mount */
+	"__mac_get_mount",    /* 425 = __mac_get_mount */
+	"__mac_getfsstat",    /* 426 = __mac_getfsstat */
 };
diff --git a/bsd/kern/syscalls.master b/bsd/kern/syscalls.master
index 0c0008058..10d694d4c 100644
--- a/bsd/kern/syscalls.master
+++ b/bsd/kern/syscalls.master
@@ -7,18 +7,23 @@
 ;	.../xnu/bsd/sys/syscall.h 
 ;	.../xnu/bsd/sys/sysproto.h 
 
-; Columns -> | Number | Cancel | Funnel | Files | { Name and Args } | { Comments }
+; Columns -> | Number Files | { Name and Args } | { Comments }
 ;	Number:  	system call number, must be in order
-;	Cancel:  	type of thread cancel - "PRE", "POST" or "NONE"
-;	Funnel:  	type of funnel - "KERN" or  "NONE"
 ;	Files:		with files to generate - "ALL" or any combo of:
 ;					"T" for syscall table (in init_sysent.c)
 ;					"N" for syscall names (in syscalls.c)
 ;					"H" for syscall headers (in syscall.h)
 ;					"P" for syscall prototypes (in sysproto.h)
+;	Name and Args:	function prototype, optionally followed by
+;			NO_SYSCALL_STUB (which mean no system call stub will
+;			be generated in libSystem) and ending with a semicolon.
+;			(Note: functions prefixed by double-underbar are
+;			automatically given the NO_SYSCALL_STUB attribute.)
 ;	Comments:  	additional comments about the sys call copied to output files
 
 ; #ifdef's, #include's, #if's etc. are copied to all output files.
+; N.B.: makesyscalls.sh and createsyscalls.pl must be updated to account
+; for any new argument types.
 
 #include <sys/appleapiopts.h>
 #include <sys/param.h>
@@ -27,419 +32,587 @@
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
 
-0	NONE	NONE	ALL	{ int nosys(void); }   { indirect syscall }
-1	NONE	KERN	ALL	{ void exit(int rval); } 
-2	NONE	KERN	ALL	{ int fork(void); } 
-3	PRE 	NONE	ALL	{ user_ssize_t read(int fd, user_addr_t cbuf, user_size_t nbyte); } 
-4	PRE 	NONE	ALL	{ user_ssize_t write(int fd, user_addr_t cbuf, user_size_t nbyte); } 
-5	PRE 	NONE	ALL	{ int open(user_addr_t path, int flags, int mode); } 
-6	PRE 	NONE	ALL	{ int close(int fd); } 
-7	PRE 	KERN	ALL	{ int wait4(int pid, user_addr_t status, int options, user_addr_t rusage); } 
-8	NONE	NONE	ALL	{ int nosys(void); }   { old creat }
-9	NONE	NONE	ALL	{ int link(user_addr_t path, user_addr_t link); } 
-10	NONE	NONE    ALL	{ int unlink(user_addr_t path); } 
-11	NONE	NONE	ALL	{ int nosys(void); }   { old execv }
-12	NONE	NONE	ALL	{ int chdir(user_addr_t path); } 
-13	NONE	NONE	ALL	{ int fchdir(int fd); } 
-14	NONE	NONE	ALL	{ int mknod(user_addr_t path, int mode, int dev); } 
-15	NONE	NONE	ALL	{ int chmod(user_addr_t path, int mode); } 
-16	NONE	NONE	ALL	{ int chown(user_addr_t path, int uid, int gid); } 
-17	NONE	NONE	UALL	{ int obreak(char *nsize); }   { old break }
+0	ALL	{ int nosys(void); }   { indirect syscall }
+1	ALL	{ void exit(int rval); } 
+2	ALL	{ int fork(void); } 
+3	ALL	{ user_ssize_t read(int fd, user_addr_t cbuf, user_size_t nbyte); } 
+4	ALL	{ user_ssize_t write(int fd, user_addr_t cbuf, user_size_t nbyte); } 
+5	ALL	{ int open(user_addr_t path, int flags, int mode); } 
+6	ALL	{ int close(int fd); } 
+7	ALL	{ int wait4(int pid, user_addr_t status, int options, user_addr_t rusage); } 
+8	ALL	{ int nosys(void); }   { old creat }
+9	ALL	{ int link(user_addr_t path, user_addr_t link); } 
+10	ALL	{ int unlink(user_addr_t path); } 
+11	ALL	{ int nosys(void); }   { old execv }
+12	ALL	{ int chdir(user_addr_t path); } 
+13	ALL	{ int fchdir(int fd); } 
+14	ALL	{ int mknod(user_addr_t path, int mode, int dev); } 
+15	ALL	{ int chmod(user_addr_t path, int mode); } 
+16	ALL	{ int chown(user_addr_t path, int uid, int gid); } 
+17	UALL	{ int obreak(char *nsize) NO_SYSCALL_STUB; }   { old break }
 
 #if COMPAT_GETFSSTAT
-18	NONE	NONE	ALL	{ int ogetfsstat(user_addr_t buf, int bufsize, int flags); } 
+18	ALL	{ int ogetfsstat(user_addr_t buf, int bufsize, int flags); } 
 #else
-18	NONE	NONE	ALL	{ int getfsstat(user_addr_t buf, int bufsize, int flags); } 
+18	ALL	{ int getfsstat(user_addr_t buf, int bufsize, int flags); } 
 #endif
 
-19	NONE	NONE	ALL	{ int nosys(void); }   { old lseek }
-20	NONE	NONE	ALL	{ int getpid(void); } 
-21	NONE	NONE	ALL	{ int nosys(void); }   { old mount }
-22	NONE	NONE	ALL	{ int nosys(void); }   { old umount }
-23	NONE	KERN	ALL	{ int setuid(uid_t uid); } 
-24	NONE	KERN	ALL	{ int getuid(void); } 
-25	NONE	KERN	ALL	{ int geteuid(void); } 
-26	NONE	KERN	ALL	{ int ptrace(int req, pid_t pid, caddr_t addr, int data); } 
-27	PRE 	NONE	ALL	{ int recvmsg(int s, struct msghdr *msg, int flags); } 
-28	PRE 	NONE	ALL	{ int sendmsg(int s, caddr_t msg, int flags); } 
-29	PRE 	NONE	ALL	{ int recvfrom(int s, void *buf, size_t len, int flags, struct sockaddr *from, int *fromlenaddr); } 
-30	PRE 	NONE	ALL	{ int accept(int s, caddr_t name, socklen_t	*anamelen); } 
-31	NONE	NONE	ALL	{ int getpeername(int fdes, caddr_t asa, socklen_t *alen); } 
-32	NONE	NONE	ALL	{ int getsockname(int fdes, caddr_t asa, socklen_t *alen); } 
-33	NONE	NONE	ALL	{ int access(user_addr_t path, int flags); } 
-34	NONE	NONE	ALL	{ int chflags(char *path, int flags); } 
-35	NONE	NONE	ALL	{ int fchflags(int fd, int flags); } 
-36	NONE	NONE	ALL	{ int sync(void); } 
-37	NONE	KERN	ALL	{ int kill(int pid, int signum); } 
-38	NONE	NONE	ALL	{ int nosys(void); }   { old stat  }
-39	NONE	KERN	ALL	{ int getppid(void); } 
-40	NONE	NONE	ALL	{ int nosys(void); }   { old lstat }
-41	NONE	NONE	ALL	{ int dup(u_int fd); } 
-42	NONE	NONE	ALL	{ int pipe(void); } 
-43	NONE	KERN	ALL	{ int getegid(void); } 
-44	NONE	KERN	ALL	{ int profil(short *bufbase, size_t bufsize, u_long pcoffset, u_int pcscale); } 
-45	NONE	KERN	ALL	{ int ktrace(const char *fname, int ops, int facs, int pid); } 
-46	NONE	KERN	ALL	{ int sigaction(int signum, struct __sigaction *nsa, struct sigaction *osa); } 
-47	NONE	KERN	ALL	{ int getgid(void); } 
-48	NONE	KERN	ALL	{ int sigprocmask(int how, user_addr_t mask, user_addr_t omask); } 
-49	NONE	KERN	ALL	{ int getlogin(char *namebuf, u_int namelen); } 
-50	NONE	KERN	ALL	{ int setlogin(char *namebuf); } 
-51	NONE	KERN	ALL	{ int acct(char *path); } 
-52	NONE	KERN	ALL	{ int sigpending(struct sigvec *osv); } 
-53	NONE	KERN	ALL	{ int sigaltstack(struct sigaltstack *nss, struct sigaltstack *oss); } 
-54	NONE	NONE	ALL	{ int ioctl(int fd, u_long com, caddr_t data); } 
-55	NONE	KERN	ALL	{ int reboot(int opt, char *command); } 
-56	NONE	NONE	ALL	{ int revoke(char *path); } 
-57	NONE	NONE	ALL	{ int symlink(char *path, char *link); } 
-58	NONE	NONE	ALL	{ int readlink(char *path, char *buf, int count); } 
-59	NONE	KERN	ALL	{ int execve(char *fname, char **argp, char **envp); } 
-60	NONE	KERN	ALL	{ int umask(int newmask); } 
-61	NONE	KERN	ALL	{ int chroot(user_addr_t path); } 
-62	NONE	NONE	ALL	{ int nosys(void); }   { old fstat }
-63	NONE	NONE	ALL	{ int nosys(void); }   { used internally, reserved }
-64	NONE	NONE	ALL	{ int nosys(void); }   { old getpagesize }
-65	PRE 	NONE	ALL	{ int msync(caddr_t addr, size_t len, int flags); } 
-66	NONE	KERN	ALL	{ int vfork(void); } 
-67	NONE	NONE	ALL	{ int nosys(void); }   { old vread }
-68	NONE	NONE	ALL	{ int nosys(void); }   { old vwrite }
-69	NONE	NONE	ALL	{ int sbrk(int incr); } 
-70	NONE	NONE	ALL	{ int sstk(int incr); } 
-71	NONE	NONE	ALL	{ int nosys(void); }   { old mmap }
-72	NONE	NONE	ALL	{ int ovadvise(void); }   { old vadvise }
-73	NONE	NONE	ALL	{ int munmap(caddr_t addr, size_t len); } 
-74	NONE	NONE	ALL	{ int mprotect(caddr_t addr, size_t len, int prot); } 
-75	NONE	NONE	ALL	{ int madvise(caddr_t addr, size_t len, int behav); } 
-76	NONE	NONE	ALL	{ int nosys(void); }   { old vhangup }
-77	NONE	NONE	ALL	{ int nosys(void); }   { old vlimit }
-78	NONE	NONE	ALL	{ int mincore(user_addr_t addr, user_size_t len, user_addr_t vec); } 
-79	NONE	KERN	ALL	{ int getgroups(u_int gidsetsize, gid_t *gidset); } 
-80	NONE	KERN	ALL	{ int setgroups(u_int gidsetsize, gid_t *gidset); } 
-81	NONE	KERN	ALL	{ int getpgrp(void); } 
-82	NONE	KERN	ALL	{ int setpgid(int pid, int pgid); } 
-83	NONE	KERN	ALL	{ int setitimer(u_int which, struct itimerval *itv, struct itimerval *oitv); } 
-84	NONE	NONE	ALL	{ int nosys(void); }   { old wait }
-85	NONE	NONE	ALL	{ int swapon(void); } 
-86	NONE	KERN	ALL	{ int getitimer(u_int which, struct itimerval *itv); } 
-87	NONE	NONE	ALL	{ int nosys(void); }   { old gethostname }
-88	NONE	NONE	ALL	{ int nosys(void); }   { old sethostname }
-89	NONE	NONE	ALL	{ int getdtablesize(void); } 
-90	NONE	NONE	ALL	{ int dup2(u_int from, u_int to); } 
-91	NONE	NONE	ALL	{ int nosys(void); }   { old getdopt }
-92	PRE 	NONE	ALL	{ int fcntl(int fd, int cmd, long arg); } 
-93	PRE 	KERN	ALL	{ int select(int nd, u_int32_t *in, u_int32_t *ou, u_int32_t *ex, struct timeval *tv); } 
-94	NONE	NONE	ALL	{ int nosys(void); }   { old setdopt }
-95	PRE 	NONE	ALL	{ int fsync(int fd); } 
-96	NONE	KERN	ALL	{ int setpriority(int which, int who, int prio); } 
-97	NONE	NONE	ALL	{ int socket(int domain, int type, int protocol); } 
-98	PRE 	NONE	ALL	{ int connect(int s, caddr_t name, socklen_t namelen); } 
-99	NONE	NONE	ALL	{ int nosys(void); }   { old accept }
-100	NONE	KERN	ALL	{ int getpriority(int which, int who); } 
-101	NONE	NONE	ALL	{ int nosys(void); }   { old send }
-102	NONE	NONE	ALL	{ int nosys(void); }   { old recv }
-103	NONE	NONE	ALL	{ int nosys(void); }   { old sigreturn }
-104	NONE	NONE	ALL	{ int bind(int s, caddr_t name, socklen_t namelen); } 
-105	NONE	NONE	ALL	{ int setsockopt(int s, int level, int name, caddr_t val, socklen_t valsize); } 
-106	NONE	NONE	ALL	{ int listen(int s, int backlog); } 
-107	NONE	NONE	ALL	{ int nosys(void); }   { old vtimes }
-108	NONE	NONE	ALL	{ int nosys(void); }   { old sigvec }
-109	NONE	NONE	ALL	{ int nosys(void); }   { old sigblock }
-110	NONE	NONE	ALL	{ int nosys(void); }   { old sigsetmask }
-111	PRE 	KERN	ALL	{ int sigsuspend(sigset_t mask); } 
-112	NONE	NONE	ALL	{ int nosys(void); }   { old sigstack }
-113	NONE	NONE	ALL	{ int nosys(void); }   { old recvmsg }
-114	NONE	NONE	ALL	{ int nosys(void); }   { old sendmsg }	
-115	NONE	NONE	ALL	{ int nosys(void); }   { old vtrace }
-116	NONE	NONE	ALL	{ int gettimeofday(struct timeval *tp, struct timezone *tzp); } 
-117	NONE	KERN	ALL	{ int getrusage(int who, struct rusage *rusage); } 
-118	NONE	NONE	ALL	{ int getsockopt(int s, int level, int name, caddr_t val, socklen_t *avalsize); } 
-119	NONE	NONE	ALL	{ int nosys(void); }   { old resuba }
-120	PRE 	NONE	ALL	{ user_ssize_t readv(int fd, struct iovec *iovp, u_int iovcnt); } 
-121	PRE 	NONE	ALL	{ user_ssize_t writev(int fd, struct iovec *iovp, u_int iovcnt); } 
-122	NONE	KERN	ALL	{ int settimeofday(struct timeval *tv, struct timezone *tzp); } 
-123	NONE	NONE	ALL	{ int fchown(int fd, int uid, int gid); } 
-124	NONE	NONE	ALL	{ int fchmod(int fd, int mode); } 
-125	NONE	NONE	ALL	{ int nosys(void); }   { old recvfrom }
-126	NONE	NONE	ALL	{ int nosys(void); }   { old setreuid }
-127	NONE	NONE	ALL	{ int nosys(void); }   { old setregid }
-128	NONE	NONE	ALL	{ int rename(char *from, char *to); } 
-129	NONE	NONE	ALL	{ int nosys(void); }   { old truncate }
-130	NONE	NONE	ALL	{ int nosys(void); }   { old ftruncate }
-131	NONE	NONE	ALL	{ int flock(int fd, int how); } 
-132	NONE	NONE	ALL	{ int mkfifo(user_addr_t path, int mode); } 
-133	PRE 	NONE	ALL	{ int sendto(int s, caddr_t buf, size_t len, int flags, caddr_t to, socklen_t tolen); } 
-134	NONE	NONE	ALL	{ int shutdown(int s, int how); } 
-135	NONE	NONE	ALL	{ int socketpair(int domain, int type, int protocol, int *rsv); } 
-136	NONE	NONE	ALL	{ int mkdir(user_addr_t path, int mode); } 
-137	NONE	NONE	ALL	{ int rmdir(char *path); } 
-138	NONE	NONE	ALL	{ int utimes(char *path, struct timeval *tptr); } 
-139	NONE	NONE	ALL	{ int futimes(int fd, struct timeval *tptr); } 
-140	NONE	KERN	ALL	{ int adjtime(struct timeval *delta, struct timeval *olddelta); } 
-141	NONE	NONE	ALL	{ int nosys(void); }   { old getpeername }
-142	NONE	NONE	ALL	{ int nosys(void); }   { old gethostid }
-143	NONE	NONE	ALL	{ int nosys(void); }   { old sethostid 	}
-144	NONE	NONE	ALL	{ int nosys(void); }   { old getrlimit }
-145	NONE	NONE	ALL	{ int nosys(void); }   { old setrlimit }
-146	NONE	NONE	ALL	{ int nosys(void); }   { old killpg }
-147	NONE	KERN	ALL	{ int setsid(void); } 
-148	NONE	NONE	ALL	{ int nosys(void); }   { old setquota }
-149	NONE	NONE	ALL	{ int nosys(void); }   { old qquota }
-150	NONE	NONE	ALL	{ int nosys(void); }   { old getsockname }
-151	NONE	KERN	ALL	{ int getpgid(pid_t pid); } 
-152	NONE	KERN	ALL	{ int setprivexec(int flag); } 
-153	PRE 	NONE	ALL	{ user_ssize_t pread(int fd, user_addr_t buf, user_size_t nbyte, off_t offset); } 
-154	PRE 	NONE	ALL	{ user_ssize_t pwrite(int fd, user_addr_t buf, user_size_t nbyte, off_t offset); } 
+19	ALL	{ int nosys(void); }   { old lseek }
+20	ALL	{ int getpid(void); } 
+21	ALL	{ int nosys(void); }   { old mount }
+22	ALL	{ int nosys(void); }   { old umount }
+23	ALL	{ int setuid(uid_t uid); } 
+24	ALL	{ int getuid(void); } 
+25	ALL	{ int geteuid(void); } 
+26	ALL	{ int ptrace(int req, pid_t pid, caddr_t addr, int data); } 
+#if SOCKETS
+27	ALL	{ int recvmsg(int s, struct msghdr *msg, int flags); } 
+28	ALL	{ int sendmsg(int s, caddr_t msg, int flags); } 
+29	ALL	{ int recvfrom(int s, void *buf, size_t len, int flags, struct sockaddr *from, int *fromlenaddr); } 
+30	ALL	{ int accept(int s, caddr_t name, socklen_t	*anamelen); } 
+31	ALL	{ int getpeername(int fdes, caddr_t asa, socklen_t *alen); } 
+32	ALL	{ int getsockname(int fdes, caddr_t asa, socklen_t *alen); } 
+#else
+27	ALL	{ int nosys(void); }
+28	ALL	{ int nosys(void); }
+29	ALL	{ int nosys(void); }
+30	ALL	{ int nosys(void); }
+31	ALL	{ int nosys(void); }
+32	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+33	ALL	{ int access(user_addr_t path, int flags); } 
+34	ALL	{ int chflags(char *path, int flags); } 
+35	ALL	{ int fchflags(int fd, int flags); } 
+36	ALL	{ int sync(void); } 
+37	ALL	{ int kill(int pid, int signum, int posix); } 
+38	ALL	{ int nosys(void); }   { old stat  }
+39	ALL	{ int getppid(void); } 
+40	ALL	{ int nosys(void); }   { old lstat }
+41	ALL	{ int dup(u_int fd); } 
+42	ALL	{ int pipe(void); } 
+43	ALL	{ int getegid(void); } 
+44	ALL	{ int profil(short *bufbase, size_t bufsize, u_long pcoffset, u_int pcscale); } 
+45	ALL	{ int nosys(void); } { old ktrace }
+46	ALL	{ int sigaction(int signum, struct __sigaction *nsa, struct sigaction *osa); } 
+47	ALL	{ int getgid(void); } 
+48	ALL	{ int sigprocmask(int how, user_addr_t mask, user_addr_t omask); } 
+49	ALL	{ int getlogin(char *namebuf, u_int namelen); } 
+50	ALL	{ int setlogin(char *namebuf); } 
+51	ALL	{ int acct(char *path); } 
+52	ALL	{ int sigpending(struct sigvec *osv); } 
+53	ALL	{ int sigaltstack(struct sigaltstack *nss, struct sigaltstack *oss); } 
+54	ALL	{ int ioctl(int fd, u_long com, caddr_t data); } 
+55	ALL	{ int reboot(int opt, char *command); } 
+56	ALL	{ int revoke(char *path); } 
+57	ALL	{ int symlink(char *path, char *link); } 
+58	ALL	{ int readlink(char *path, char *buf, int count); } 
+59	ALL	{ int execve(char *fname, char **argp, char **envp); } 
+60	ALL	{ int umask(int newmask); } 
+61	ALL	{ int chroot(user_addr_t path); } 
+62	ALL	{ int nosys(void); }   { old fstat }
+63	ALL	{ int nosys(void); }   { used internally, reserved }
+64	ALL	{ int nosys(void); }   { old getpagesize }
+65	ALL	{ int msync(caddr_t addr, size_t len, int flags); } 
+66	ALL	{ int vfork(void); } 
+67	ALL	{ int nosys(void); }   { old vread }
+68	ALL	{ int nosys(void); }   { old vwrite }
+69	ALL	{ int sbrk(int incr) NO_SYSCALL_STUB; } 
+70	ALL	{ int sstk(int incr) NO_SYSCALL_STUB; } 
+71	ALL	{ int nosys(void); }   { old mmap }
+72	ALL	{ int ovadvise(void) NO_SYSCALL_STUB; }   { old vadvise }
+73	ALL	{ int munmap(caddr_t addr, size_t len); } 
+74	ALL	{ int mprotect(caddr_t addr, size_t len, int prot); } 
+75	ALL	{ int madvise(caddr_t addr, size_t len, int behav); } 
+76	ALL	{ int nosys(void); }   { old vhangup }
+77	ALL	{ int nosys(void); }   { old vlimit }
+78	ALL	{ int mincore(user_addr_t addr, user_size_t len, user_addr_t vec); } 
+79	ALL	{ int getgroups(u_int gidsetsize, gid_t *gidset); } 
+80	ALL	{ int setgroups(u_int gidsetsize, gid_t *gidset); } 
+81	ALL	{ int getpgrp(void); } 
+82	ALL	{ int setpgid(int pid, int pgid); } 
+83	ALL	{ int setitimer(u_int which, struct itimerval *itv, struct itimerval *oitv); } 
+84	ALL	{ int nosys(void); }   { old wait }
+85	ALL	{ int swapon(void); } 
+86	ALL	{ int getitimer(u_int which, struct itimerval *itv); } 
+87	ALL	{ int nosys(void); }   { old gethostname }
+88	ALL	{ int nosys(void); }   { old sethostname }
+89	ALL	{ int getdtablesize(void); } 
+90	ALL	{ int dup2(u_int from, u_int to); } 
+91	ALL	{ int nosys(void); }   { old getdopt }
+92	ALL	{ int fcntl(int fd, int cmd, long arg); } 
+93	ALL	{ int select(int nd, u_int32_t *in, u_int32_t *ou, u_int32_t *ex, struct timeval *tv); } 
+94	ALL	{ int nosys(void); }   { old setdopt }
+95	ALL	{ int fsync(int fd); } 
+96	ALL	{ int setpriority(int which, id_t who, int prio); } 
+#if SOCKETS
+97	ALL	{ int socket(int domain, int type, int protocol); } 
+98	ALL	{ int connect(int s, caddr_t name, socklen_t namelen); } 
+#else
+97	ALL	{ int nosys(void); }
+98	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+99	ALL	{ int nosys(void); }   { old accept }
+100	ALL	{ int getpriority(int which, id_t who); } 
+101	ALL	{ int nosys(void); }   { old send }
+102	ALL	{ int nosys(void); }   { old recv }
+103	ALL	{ int nosys(void); }   { old sigreturn }
+#if SOCKETS
+104	ALL	{ int bind(int s, caddr_t name, socklen_t namelen); } 
+105	ALL	{ int setsockopt(int s, int level, int name, caddr_t val, socklen_t valsize); } 
+106	ALL	{ int listen(int s, int backlog); } 
+#else
+104	ALL	{ int nosys(void); }
+105	ALL	{ int nosys(void); }
+106	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+107	ALL	{ int nosys(void); }   { old vtimes }
+108	ALL	{ int nosys(void); }   { old sigvec }
+109	ALL	{ int nosys(void); }   { old sigblock }
+110	ALL	{ int nosys(void); }   { old sigsetmask }
+111	ALL	{ int sigsuspend(sigset_t mask); } 
+112	ALL	{ int nosys(void); }   { old sigstack }
+#if SOCKETS
+113	ALL	{ int nosys(void); }   { old recvmsg }
+114	ALL	{ int nosys(void); }   { old sendmsg }	
+#else
+113	ALL	{ int nosys(void); }
+114	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+115	ALL	{ int nosys(void); }   { old vtrace }
+116	ALL	{ int gettimeofday(struct timeval *tp, struct timezone *tzp); } 
+117	ALL	{ int getrusage(int who, struct rusage *rusage); } 
+#if SOCKETS
+118	ALL	{ int getsockopt(int s, int level, int name, caddr_t val, socklen_t *avalsize); } 
+#else
+118	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+119	ALL	{ int nosys(void); }   { old resuba }
+120	ALL	{ user_ssize_t readv(int fd, struct iovec *iovp, u_int iovcnt); } 
+121	ALL	{ user_ssize_t writev(int fd, struct iovec *iovp, u_int iovcnt); } 
+122	ALL	{ int settimeofday(struct timeval *tv, struct timezone *tzp); } 
+123	ALL	{ int fchown(int fd, int uid, int gid); } 
+124	ALL	{ int fchmod(int fd, int mode); } 
+125	ALL	{ int nosys(void); }   { old recvfrom }
+126	ALL	{ int setreuid(uid_t ruid, uid_t euid); }
+127	ALL	{ int setregid(gid_t rgid, gid_t egid); }
+128	ALL	{ int rename(char *from, char *to); } 
+129	ALL	{ int nosys(void); }   { old truncate }
+130	ALL	{ int nosys(void); }   { old ftruncate }
+131	ALL	{ int flock(int fd, int how); } 
+132	ALL	{ int mkfifo(user_addr_t path, int mode); } 
+#if SOCKETS
+133	ALL	{ int sendto(int s, caddr_t buf, size_t len, int flags, caddr_t to, socklen_t tolen); } 
+134	ALL	{ int shutdown(int s, int how); } 
+135	ALL	{ int socketpair(int domain, int type, int protocol, int *rsv); } 
+#else
+133	ALL	{ int nosys(void); }
+134	ALL	{ int nosys(void); }
+135	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+136	ALL	{ int mkdir(user_addr_t path, int mode); } 
+137	ALL	{ int rmdir(char *path); } 
+138	ALL	{ int utimes(char *path, struct timeval *tptr); } 
+139	ALL	{ int futimes(int fd, struct timeval *tptr); } 
+140	ALL	{ int adjtime(struct timeval *delta, struct timeval *olddelta); } 
+141	ALL	{ int nosys(void); }   { old getpeername }
+142	ALL	{ int gethostuuid(unsigned char *uuid_buf, const struct timespec *timeoutp); }
+143	ALL	{ int nosys(void); }   { old sethostid 	}
+144	ALL	{ int nosys(void); }   { old getrlimit }
+145	ALL	{ int nosys(void); }   { old setrlimit }
+146	ALL	{ int nosys(void); }   { old killpg }
+147	ALL	{ int setsid(void); } 
+148	ALL	{ int nosys(void); }   { old setquota }
+149	ALL	{ int nosys(void); }   { old qquota }
+150	ALL	{ int nosys(void); }   { old getsockname }
+151	ALL	{ int getpgid(pid_t pid); } 
+152	ALL	{ int setprivexec(int flag); } 
+153	ALL	{ user_ssize_t pread(int fd, user_addr_t buf, user_size_t nbyte, off_t offset); } 
+154	ALL	{ user_ssize_t pwrite(int fd, user_addr_t buf, user_size_t nbyte, off_t offset); } 
 
 #if NFSSERVER
-155	NONE	KERN	ALL	{ int nfssvc(int flag, caddr_t argp); } 
+155	ALL	{ int nfssvc(int flag, caddr_t argp); } 
 #else
-155	NONE	NONE	ALL	{ int nosys(void); } 
+155	ALL	{ int nosys(void); } 
 #endif
 
-156	NONE	NONE	ALL	{ int nosys(void); }   { old getdirentries }
-157	NONE	NONE	ALL	{ int statfs(char *path, struct statfs *buf); } 
-158	NONE	NONE	ALL	{ int fstatfs(int fd, struct statfs *buf); } 
-159	NONE	NONE	ALL	{ int unmount(user_addr_t path, int flags); } 
-160	NONE	NONE	ALL	{ int nosys(void); }   { old async_daemon }
+156	ALL	{ int nosys(void); }   { old getdirentries }
+157	ALL	{ int statfs(char *path, struct statfs *buf); } 
+158	ALL	{ int fstatfs(int fd, struct statfs *buf); } 
+159	ALL	{ int unmount(user_addr_t path, int flags); } 
+160	ALL	{ int nosys(void); }   { old async_daemon }
 
-#if NFSCLIENT
-161	NONE	KERN	ALL	{ int getfh(char *fname, fhandle_t *fhp); } 
+#if NFSSERVER
+161	ALL	{ int getfh(char *fname, fhandle_t *fhp); } 
 #else
-161	NONE	NONE	ALL	{ int nosys(void); } 
+161	ALL	{ int nosys(void); } 
 #endif
 
-162	NONE	NONE	ALL	{ int nosys(void); }   { old getdomainname }
-163	NONE	NONE	ALL	{ int nosys(void); }   { old setdomainname }
-164	NONE	NONE	ALL	{ int nosys(void); } 
-165	NONE	KERN	ALL	{ int quotactl(char *path, int cmd, int uid, caddr_t arg); } 
-166	NONE	NONE	ALL	{ int nosys(void); }   { old exportfs }
-167	NONE	NONE	ALL	{ int mount(char *type, char *path, int flags, caddr_t data); } 
-168	NONE	NONE	ALL	{ int nosys(void); }   { old ustat }
-169	NONE	NONE	ALL	{ int nosys(void); } 
-170	NONE	NONE	HN	{ int table(void); }   { old table }
-171	NONE	NONE	ALL	{ int nosys(void); }   { old wait3 }
-172	NONE	NONE	ALL	{ int nosys(void); }   { old rpause	}	
-173	PRE	KERN	ALL	{ int waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); } 
-174	NONE	NONE	ALL	{ int nosys(void); }   { old getdents }
-175	NONE	NONE	ALL	{ int nosys(void); }   { old gc_control }
-176	NONE	KERN	ALL	{ int add_profil(short *bufbase, size_t bufsize, u_long pcoffset, u_int pcscale); } 
-177	NONE	NONE	ALL	{ int nosys(void); } 
-178	NONE	NONE	ALL	{ int nosys(void); } 
-179	NONE	NONE	ALL	{ int nosys(void); } 
-180	NONE	NONE	UALL { int kdebug_trace(int code, int arg1, int arg2, int arg3, int arg4, int arg5); } 
-181	NONE	KERN	ALL	{ int setgid(gid_t gid); } 
-182	NONE	KERN	ALL	{ int setegid(gid_t egid); } 
-183	NONE	KERN	ALL	{ int seteuid(uid_t euid); } 
-184	NONE	KERN	ALL	{ int sigreturn(struct ucontext *uctx, int infostyle); } 
-185	NONE	NONE	UALL { int chud(int code, int arg1, int arg2, int arg3, int arg4, int arg5); } 
-186	NONE	NONE	ALL	{ int nosys(void); } 
-187	NONE	NONE	ALL	{ int nosys(void); } 
-188	NONE	NONE	ALL	{ int stat(user_addr_t path, user_addr_t ub); } 
-189	NONE	NONE	ALL	{ int fstat(int fd, user_addr_t ub); } 
-190	NONE	NONE	ALL	{ int lstat(user_addr_t path, user_addr_t ub); } 
-191	NONE	NONE	ALL	{ int pathconf(char *path, int name); } 
-192	NONE	NONE	ALL	{ int fpathconf(int fd, int name); } 
+162	ALL	{ int nosys(void); }   { old getdomainname }
+163	ALL	{ int nosys(void); }   { old setdomainname }
+164	ALL	{ int nosys(void); } 
+165	ALL	{ int quotactl(const char *path, int cmd, int uid, caddr_t arg); } 
+166	ALL	{ int nosys(void); }   { old exportfs }
+167	ALL	{ int mount(char *type, char *path, int flags, caddr_t data); } 
+168	ALL	{ int nosys(void); }   { old ustat }
+169	ALL	{ int csops(pid_t pid, uint32_t ops, user_addr_t useraddr, user_size_t usersize); } 
+170	HN	{ int table(void); }   { old table }
+171	ALL	{ int nosys(void); }   { old wait3 }
+172	ALL	{ int nosys(void); }   { old rpause	}	
+173	ALL	{ int waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options); } 
+174	ALL	{ int nosys(void); }   { old getdents }
+175	ALL	{ int nosys(void); }   { old gc_control }
+176	ALL	{ int add_profil(short *bufbase, size_t bufsize, u_long pcoffset, u_int pcscale); } 
+177	ALL	{ int nosys(void); } 
+178	ALL	{ int nosys(void); } 
+179	ALL	{ int nosys(void); } 
+180	ALL 	{ int kdebug_trace(int code, int arg1, int arg2, int arg3, int arg4, int arg5) NO_SYSCALL_STUB; } 
+181	ALL	{ int setgid(gid_t gid); } 
+182	ALL	{ int setegid(gid_t egid); } 
+183	ALL	{ int seteuid(uid_t euid); } 
+184	ALL	{ int sigreturn(struct ucontext *uctx, int infostyle); } 
+185	UALL 	{ int chud(int code, int arg1, int arg2, int arg3, int arg4, int arg5) NO_SYSCALL_STUB; } 
+186	ALL	{ int nosys(void); } 
+187	ALL	{ int nosys(void); } 
+188	ALL	{ int stat(user_addr_t path, user_addr_t ub); } 
+189	ALL	{ int fstat(int fd, user_addr_t ub); } 
+190	ALL	{ int lstat(user_addr_t path, user_addr_t ub); } 
+191	ALL	{ int pathconf(char *path, int name); } 
+192	ALL	{ int fpathconf(int fd, int name); } 
+193	ALL	{ int nosys(void); } 
+194	ALL	{ int getrlimit(u_int which, struct rlimit *rlp); } 
+195	ALL	{ int setrlimit(u_int which, struct rlimit *rlp); } 
+196	ALL	{ int getdirentries(int fd, char *buf, u_int count, long *basep); } 
+197	ALL	{ user_addr_t mmap(caddr_t addr, size_t len, int prot, int flags, int fd, off_t pos); } 
+198	ALL	{ int nosys(void); } 	{ __syscall }
+199	ALL	{ off_t lseek(int fd, off_t offset, int whence); } 
+200	ALL	{ int truncate(char *path, off_t length); } 
+201	ALL	{ int ftruncate(int fd, off_t length); } 
+202	ALL 	{ int __sysctl(int *name, u_int namelen, void *old, size_t *oldlenp, void *new, size_t newlen); } 
+203	ALL	{ int mlock(caddr_t addr, size_t len); } 
+204	ALL	{ int munlock(caddr_t addr, size_t len); } 
+205	ALL	{ int undelete(user_addr_t path); } 
 
-#if COMPAT_GETFSSTAT
-193	NONE	NONE	ALL	{ int getfsstat(user_addr_t buf, user_long_t bufsize, int flags); } 
+#if NETAT
+206	ALL	{ int ATsocket(int proto); } 
+207	UALL	{ int ATgetmsg(int fd, void *ctlptr, void *datptr, int *flags); } 
+208	UALL	{ int ATputmsg(int fd, void *ctlptr, void *datptr, int flags); } 
+209	UALL	{ int ATPsndreq(int fd, unsigned char *buf, int len, int nowait); } 
+210	UALL	{ int ATPsndrsp(int fd, unsigned char *respbuff, int resplen, int datalen); } 
+211	UALL	{ int ATPgetreq(int fd, unsigned char *buf, int buflen); } 
+212	UALL	{ int ATPgetrsp(int fd, unsigned char *bdsp); } 
+213	ALL	{ int nosys(void); } 	{ Reserved for AppleTalk }
 #else
-193	NONE	NONE	ALL	{ int nosys(void); } 
-#endif
+206	ALL	{ int nosys(void); }
+207	ALL	{ int nosys(void); }
+208	ALL	{ int nosys(void); }
+209	ALL	{ int nosys(void); }
+210	ALL	{ int nosys(void); }
+211	ALL	{ int nosys(void); }
+212	ALL	{ int nosys(void); }
+213	ALL	{ int nosys(void); } 	{ Reserved for AppleTalk }
+#endif /* NETAT */
 
-194	NONE	KERN	ALL	{ int getrlimit(u_int which, struct rlimit *rlp); } 
-195	NONE	KERN	ALL	{ int setrlimit(u_int which, struct rlimit *rlp); } 
-196	NONE	NONE	ALL	{ int getdirentries(int fd, char *buf, u_int count, long *basep); } 
-197	NONE	NONE	ALL	{ user_addr_t mmap(caddr_t addr, size_t len, int prot, int flags, int fd, off_t pos); } 
-198	NONE	NONE	ALL	{ int nosys(void); } 	{ __syscall }
-199	NONE	NONE	ALL	{ off_t lseek(int fd, off_t offset, int whence); } 
-200	NONE	NONE	ALL	{ int truncate(char *path, off_t length); } 
-201	NONE	NONE	ALL	{ int ftruncate(int fd, off_t length); } 
-202	NONE	KERN    ALL { int __sysctl(int *name, u_int namelen, void *old, size_t *oldlenp, void *new, size_t newlen); } 
-203	NONE	NONE	ALL	{ int mlock(caddr_t addr, size_t len); } 
-204	NONE	NONE	ALL	{ int munlock(caddr_t addr, size_t len); } 
-205	NONE	NONE	ALL	{ int undelete(user_addr_t path); } 
-
-206	NONE	NONE	ALL	{ int ATsocket(int proto); } 
-207	NONE	NONE	UALL	{ int ATgetmsg(int fd, void *ctlptr, void *datptr, int *flags); } 
-208	NONE	NONE	UALL	{ int ATputmsg(int fd, void *ctlptr, void *datptr, int flags); } 
-209	NONE	NONE	UALL	{ int ATPsndreq(int fd, unsigned char *buf, int len, int nowait); } 
-210	NONE	NONE	UALL	{ int ATPsndrsp(int fd, unsigned char *respbuff, int resplen, int datalen); } 
-211	NONE	NONE	UALL	{ int ATPgetreq(int fd, unsigned char *buf, int buflen); } 
-212	NONE	NONE	UALL	{ int ATPgetrsp(int fd, unsigned char *bdsp); } 
-213	NONE	NONE	ALL	{ int nosys(void); } 	{ Reserved for AppleTalk }
-
-214	NONE	KERN	ALL	{ int kqueue_from_portset_np(int portset); } 
-215	NONE	KERN	ALL	{ int kqueue_portset_np(int fd); } 
+214	ALL	{ int kqueue_from_portset_np(int portset); } 
+215	ALL	{ int kqueue_portset_np(int fd); } 
 
 ; System Calls 216 - 230 are reserved for calls to support HFS/HFS Plus
 ; file system semantics. Currently, we only use 215-227.  The rest is 
 ; for future expansion in anticipation of new MacOS APIs for HFS Plus.
-; These calls are not conditionalized becuase while they are specific
+; These calls are not conditionalized because while they are specific
 ; to HFS semantics, they are not specific to the HFS filesystem.
 ; We expect all filesystems to recognize the call and report that it is
 ; not supported or to actually implement it.
-216	NONE	NONE	UHN	{ int mkcomplex(const char *path, mode_t mode, u_long type); } 	{ soon to be obsolete }
-217	NONE	NONE	UHN	{ int statv(const char *path, struct vstat *vsb); } 	{ soon to be obsolete }
-218	NONE	NONE	UHN	{ int lstatv(const char *path, struct vstat *vsb); } 	{ soon to be obsolete }
-219	NONE	NONE	UHN	{ int fstatv(int fd, struct vstat *vsb); } 	{ soon to be obsolete }
-220	NONE	NONE	ALL	{ int getattrlist(const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, u_long options); } 
-221	NONE	NONE	ALL	{ int setattrlist(const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, u_long options); } 
-222	NONE	NONE	ALL	{ int getdirentriesattr(int fd, struct attrlist *alist, void *buffer, size_t buffersize, u_long *count, u_long *basep, u_long *newstate, u_long options); } 
-223	NONE	NONE	ALL	{ int exchangedata(const char *path1, const char *path2, u_long options); } 
+216	UHN	{ int mkcomplex(const char *path, mode_t mode, u_long type); } 	{ soon to be obsolete }
+217	UHN	{ int statv(const char *path, struct vstat *vsb); } 	{ soon to be obsolete }
+218	UHN	{ int lstatv(const char *path, struct vstat *vsb); } 	{ soon to be obsolete }
+219	UHN	{ int fstatv(int fd, struct vstat *vsb); } 	{ soon to be obsolete }
+220	ALL	{ int getattrlist(const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, u_long options); } 
+221	ALL	{ int setattrlist(const char *path, struct attrlist *alist, void *attributeBuffer, size_t bufferSize, u_long options); } 
+222	ALL	{ int getdirentriesattr(int fd, struct attrlist *alist, void *buffer, size_t buffersize, u_long *count, u_long *basep, u_long *newstate, u_long options); } 
+223	ALL	{ int exchangedata(const char *path1, const char *path2, u_long options); } 
+224	ALL	{ int nosys(void); } { was checkuseraccess }
+225	ALL	{ int searchfs(const char *path, struct fssearchblock *searchblock, u_long *nummatches, u_long scriptcode, u_long options, struct searchstate *state); } 
+226	ALL	{ int delete(user_addr_t path) NO_SYSCALL_STUB; }       { private delete (Carbon semantics) }
+227	ALL	{ int copyfile(char *from, char *to, int mode, int flags) NO_SYSCALL_STUB; } 
+228	ALL	{ int nosys(void); } 
+229	ALL	{ int nosys(void); } 
+230	ALL	{ int poll(struct pollfd *fds, u_int nfds, int timeout); } 
+231	ALL	{ int watchevent(struct eventreq *u_req, int u_eventmask); } 
+232	ALL	{ int waitevent(struct eventreq *u_req, struct timeval *tv); } 
+233	ALL	{ int modwatch(struct eventreq *u_req, int u_eventmask); } 
+234	ALL	{ user_ssize_t getxattr(user_addr_t path, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
+235	ALL	{ user_ssize_t fgetxattr(int fd, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
+236	ALL	{ int setxattr(user_addr_t path, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
+237	ALL	{ int fsetxattr(int fd, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
+238	ALL	{ int removexattr(user_addr_t path, user_addr_t attrname, int options); } 
+239	ALL	{ int fremovexattr(int fd, user_addr_t attrname, int options); } 
+240	ALL	{ user_ssize_t listxattr(user_addr_t path, user_addr_t namebuf, size_t bufsize, int options); } 
+241	ALL	{ user_ssize_t flistxattr(int fd, user_addr_t namebuf, size_t bufsize, int options); } 
+242	ALL	{ int fsctl(const char *path, u_long cmd, caddr_t data, u_long options); } 
+243	ALL	{ int initgroups(u_int gidsetsize, gid_t *gidset, int gmuid); } 
+244	ALL	{ int posix_spawn(pid_t *pid, const char *path, const struct _posix_spawn_args_desc *adesc, char **argv, char **envp); } 
+245	ALL	{ int nosys(void); } 
+246	ALL	{ int nosys(void); } 
 
-#ifdef __APPLE_API_OBSOLETE
-224	NONE	NONE	UALL	{ int checkuseraccess(const char *path, uid_t userid, gid_t *groups, int ngroups, int accessrequired, u_long options); } 
+#if NFSCLIENT
+247	ALL	{ int nfsclnt(int flag, caddr_t argp); } 
 #else
-224	NONE	NONE	ALL	{ int nosys(void); } 	{ HFS checkuseraccess check access to a file }
-#endif /* __APPLE_API_OBSOLETE */
-225	NONE	KERN	ALL	{ int searchfs(const char *path, struct fssearchblock *searchblock, u_long *nummatches, u_long scriptcode, u_long options, struct searchstate *state); } 
-226	NONE	NONE    ALL	{ int delete(user_addr_t path); }       { private delete (Carbon semantics) }
-227	NONE	NONE	ALL	{ int copyfile(char *from, char *to, int mode, int flags); } 
-228	NONE	NONE	ALL	{ int nosys(void); } 
-229	NONE	NONE	ALL	{ int nosys(void); } 
-230	PRE 	NONE	ALL	{ int poll(struct pollfd *fds, u_int nfds, int timeout); } 
-231	NONE	NONE	UALL	{ int watchevent(struct eventreq *u_req, int u_eventmask); } 
-232	NONE	NONE	UALL	{ int waitevent(struct eventreq *u_req, struct timeval *tv); } 
-233	NONE	NONE	UALL	{ int modwatch(struct eventreq *u_req, int u_eventmask); } 
-234	NONE	NONE	ALL	{ user_ssize_t getxattr(user_addr_t path, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
-235	NONE	NONE	ALL	{ user_ssize_t fgetxattr(int fd, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
-236	NONE	NONE	ALL	{ int setxattr(user_addr_t path, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
-237	NONE	NONE	ALL	{ int fsetxattr(int fd, user_addr_t attrname, user_addr_t value, size_t size, uint32_t position, int options); } 
-238	NONE	NONE	ALL	{ int removexattr(user_addr_t path, user_addr_t attrname, int options); } 
-239	NONE	NONE	ALL	{ int fremovexattr(int fd, user_addr_t attrname, int options); } 
-240	NONE	NONE	ALL	{ user_ssize_t listxattr(user_addr_t path, user_addr_t namebuf, size_t bufsize, int options); } 
-241	NONE	NONE	ALL	{ user_ssize_t flistxattr(int fd, user_addr_t namebuf, size_t bufsize, int options); } 
-242	NONE	KERN	ALL	{ int fsctl(const char *path, u_long cmd, caddr_t data, u_long options); } 
-243	NONE	KERN	ALL	{ int initgroups(u_int gidsetsize, gid_t *gidset, int gmuid); } 
-244	NONE	NONE	ALL	{ int nosys(void); } 
-245	NONE	NONE	ALL	{ int nosys(void); } 
-246	NONE	NONE	ALL	{ int nosys(void); } 
+247	ALL	{ int nosys(void); } 
+#endif
+#if NFSSERVER
+248	ALL	{ int fhopen(const struct fhandle *u_fhp, int flags); } 
+#else
+248	ALL	{ int nosys(void); } 
+#endif
 
-#if NFSCLIENT
-247	NONE	KERN	ALL	{ int nfsclnt(int flag, caddr_t argp); } 
-248	NONE	KERN	ALL	{ int fhopen(const struct fhandle *u_fhp, int flags); } 
+249	ALL	{ int nosys(void); } 
+250	ALL	{ int minherit(void *addr, size_t len, int inherit); } 
+#if SYSV_SEM
+251	ALL	{ int semsys(u_int which, int a2, int a3, int a4, int a5); } 
+#else
+251	ALL	{ int nosys(void); } 
+#endif
+#if SYSV_MSG
+252	ALL	{ int msgsys(u_int which, int a2, int a3, int a4, int a5); }
+#else
+252	ALL	{ int nosys(void); } 
+#endif
+#if SYSV_SHM
+253	ALL	{ int shmsys(u_int which, int a2, int a3, int a4); } 
+#else
+253	ALL	{ int nosys(void); } 
+#endif
+#if SYSV_SEM
+254	ALL	{ int semctl(int semid, int semnum, int cmd, semun_t arg); } 
+255	ALL	{ int semget(key_t key, int	nsems, int semflg); } 
+256	ALL	{ int semop(int semid, struct sembuf *sops, int nsops); } 
+257	ALL	{ int nosys(void); } 
+#else
+254	ALL	{ int nosys(void); } 
+255	ALL	{ int nosys(void); } 
+256	ALL	{ int nosys(void); } 
+257	ALL	{ int nosys(void); } 
+#endif
+#if SYSV_MSG
+258	ALL	{ int msgctl(int msqid, int cmd, struct	msqid_ds *buf); } 
+259	ALL	{ int msgget(key_t key, int msgflg); } 
+260	ALL	{ int msgsnd(int msqid, void *msgp, size_t msgsz, int msgflg); } 
+261	ALL	{ user_ssize_t msgrcv(int msqid, void *msgp, size_t msgsz, long msgtyp, int msgflg); } 
+#else
+258	ALL	{ int nosys(void); } 
+259	ALL	{ int nosys(void); } 
+260	ALL	{ int nosys(void); } 
+261	ALL	{ int nosys(void); } 
+#endif
+#if SYSV_SHM
+262	ALL	{ user_addr_t shmat(int shmid, void *shmaddr, int shmflg); } 
+263	ALL	{ int shmctl(int shmid, int cmd, struct shmid_ds *buf); } 
+264	ALL	{ int shmdt(void *shmaddr); } 
+265	ALL	{ int shmget(key_t key, size_t size, int shmflg); } 
+#else
+262	ALL	{ int nosys(void); } 
+263	ALL	{ int nosys(void); } 
+264	ALL	{ int nosys(void); } 
+265	ALL	{ int nosys(void); } 
+#endif
+266	ALL	{ int shm_open(const char *name, int oflag, int mode); } 
+267	ALL	{ int shm_unlink(const char *name); } 
+268	ALL	{ user_addr_t sem_open(const char *name, int oflag, int mode, int value); } 
+269	ALL	{ int sem_close(sem_t *sem); } 
+270	ALL	{ int sem_unlink(const char *name); } 
+271	ALL	{ int sem_wait(sem_t *sem); } 
+272	ALL	{ int sem_trywait(sem_t *sem); } 
+273	ALL	{ int sem_post(sem_t *sem); } 
+274	ALL	{ int sem_getvalue(sem_t *sem, int *sval); } 
+275	ALL	{ int sem_init(sem_t *sem, int phsared, u_int value); } 
+276	ALL	{ int sem_destroy(sem_t *sem); } 
+277	ALL	{ int open_extended(user_addr_t path, int flags, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+278	ALL	{ int umask_extended(int newmask, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+279	ALL	{ int stat_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+280	ALL	{ int lstat_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+281	ALL	{ int fstat_extended(int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+282	ALL	{ int chmod_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+283	ALL	{ int fchmod_extended(int fd, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+284	ALL	{ int access_extended(user_addr_t entries, size_t size, user_addr_t results, uid_t uid) NO_SYSCALL_STUB; } 
+285	ALL	{ int settid(uid_t uid, gid_t gid) NO_SYSCALL_STUB; } 
+286	ALL	{ int gettid(uid_t *uidp, gid_t *gidp) NO_SYSCALL_STUB; } 
+287	ALL	{ int setsgroups(int setlen, user_addr_t guidset) NO_SYSCALL_STUB; } 
+288	ALL	{ int getsgroups(user_addr_t setlen, user_addr_t guidset) NO_SYSCALL_STUB; } 
+289	ALL	{ int setwgroups(int setlen, user_addr_t guidset) NO_SYSCALL_STUB; } 
+290	ALL	{ int getwgroups(user_addr_t setlen, user_addr_t guidset) NO_SYSCALL_STUB; }
+291	ALL	{ int mkfifo_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+292	ALL	{ int mkdir_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity) NO_SYSCALL_STUB; } 
+293	ALL	{ int identitysvc(int opcode, user_addr_t message) NO_SYSCALL_STUB; } 
+294	ALL	{ int shared_region_check_np(uint64_t *start_address) NO_SYSCALL_STUB; }
+295	ALL	{ int shared_region_map_np(int fd, uint32_t count, const struct shared_file_mapping_np *mappings) NO_SYSCALL_STUB; }
+296	ALL	{ int nosys(void); } { old load_shared_file }
+297	ALL	{ int nosys(void); } { old reset_shared_file }
+298	ALL	{ int nosys(void); } { old new_system_shared_regions }
+299	ALL	{ int enosys(void); } { old shared_region_map_file_np }
+300	ALL	{ int enosys(void); } { old shared_region_make_private_np }
+301	ALL	{ int __pthread_mutex_destroy(int mutexid); } 
+302	ALL	{ int __pthread_mutex_init(user_addr_t  mutex, user_addr_t attr); } 
+303	ALL	{ int __pthread_mutex_lock(int mutexid); } 
+304	ALL	{ int __pthread_mutex_trylock(int mutexid); } 
+305	ALL	{ int __pthread_mutex_unlock(int mutexid); } 
+306	ALL	{ int __pthread_cond_init(user_addr_t cond, user_addr_t attr); } 
+307	ALL	{ int __pthread_cond_destroy(int condid); } 
+308	ALL	{ int __pthread_cond_broadcast(int condid); } 
+309	ALL	{ int __pthread_cond_signal(int condid); } 
+310	ALL	{ int getsid(pid_t pid); } 
+311	ALL	{ int settid_with_pid(pid_t pid, int assume) NO_SYSCALL_STUB; } 
+312	ALL	{ int __pthread_cond_timedwait(int condid, int mutexid, user_addr_t abstime); } 
+313	ALL	{ int aio_fsync(int op, user_addr_t aiocbp); } 
+314 	ALL	{ user_ssize_t aio_return(user_addr_t aiocbp); } 
+315	ALL	{ int aio_suspend(user_addr_t aiocblist, int nent, user_addr_t timeoutp); } 
+316	ALL	{ int aio_cancel(int fd, user_addr_t aiocbp); } 
+317	ALL	{ int aio_error(user_addr_t aiocbp); } 
+318	ALL	{ int aio_read(user_addr_t aiocbp); } 
+319	ALL	{ int aio_write(user_addr_t aiocbp); } 
+320	ALL	{ int lio_listio(int mode, user_addr_t aiocblist, int nent, user_addr_t sigp); } 
+321	ALL	{ int __pthread_cond_wait(int condid, int mutexid); } 
+322	ALL	{ int iopolicysys(int cmd, void *arg) NO_SYSCALL_STUB; } 
+323	ALL	{ int nosys(void); } 
+324	ALL	{ int mlockall(int how); } 
+325	ALL	{ int munlockall(int how); } 
+326	ALL	{ int nosys(void); } 
+327	ALL	{ int issetugid(void); } 
+328	ALL	{ int __pthread_kill(int thread_port, int sig); } 
+329	ALL	{ int __pthread_sigmask(int how, user_addr_t set, user_addr_t oset); } 
+330	ALL	{ int __sigwait(user_addr_t set, user_addr_t sig); } 
+331	ALL	{ int __disable_threadsignal(int value); } 
+332	ALL	{ int __pthread_markcancel(int thread_port); } 
+333	ALL	{ int __pthread_canceled(int  action); } 
+334	ALL	{ int __semwait_signal(int cond_sem, int mutex_sem, int timeout, int relative, time_t tv_sec, int32_t tv_nsec); } 
+335	ALL	{ int nosys(void); }   { old utrace }
+336	ALL	{ int proc_info(int32_t callnum,int32_t pid,uint32_t flavor, uint64_t arg,user_addr_t buffer,int32_t buffersize) NO_SYSCALL_STUB; } 
+#if SENDFILE
+337	ALL	{ int sendfile(int fd, int s, off_t offset, off_t *nbytes, struct sf_hdtr *hdtr, int flags); } 
+#else /* !SENDFILE */
+337	ALL	{ int nosys(void); }
+#endif /* SENDFILE */
+338	ALL	{ int stat64(user_addr_t path, user_addr_t ub); } 
+339	ALL	{ int fstat64(int fd, user_addr_t ub); } 
+340	ALL	{ int lstat64(user_addr_t path, user_addr_t ub); } 
+341	ALL	{ int stat64_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+342	ALL	{ int lstat64_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+343	ALL	{ int fstat64_extended(int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size) NO_SYSCALL_STUB; } 
+344	ALL	{ user_ssize_t getdirentries64(int fd, void *buf, user_size_t bufsize, off_t *position) NO_SYSCALL_STUB; } 
+345	ALL	{ int statfs64(char *path, struct statfs64 *buf); } 
+346	ALL	{ int fstatfs64(int fd, struct statfs64 *buf); } 
+347	ALL	{ int getfsstat64(user_addr_t buf, int bufsize, int flags); } 
+348	ALL	{ int __pthread_chdir(user_addr_t path); } 
+349	ALL	{ int __pthread_fchdir(int fd); } 
+
+#if AUDIT
+350	ALL	{ int audit(void *record, int length); } 
+351	ALL	{ int auditon(int cmd, void *data, int length); } 
+352	ALL	{ int nosys(void); } 
+353	ALL	{ int getauid(au_id_t *auid); } 
+354	ALL	{ int setauid(au_id_t *auid); } 
+355	ALL	{ int getaudit(struct auditinfo *auditinfo); } 
+356	ALL	{ int setaudit(struct auditinfo *auditinfo); } 
+357	ALL	{ int getaudit_addr(struct auditinfo_addr *auditinfo_addr, int length); } 
+358	ALL	{ int setaudit_addr(struct auditinfo_addr *auditinfo_addr, int length); } 
+359	ALL	{ int auditctl(char *path); } 
+#else
+350	ALL	{ int nosys(void); } 
+351	ALL	{ int nosys(void); } 
+352	ALL	{ int nosys(void); } 
+353	ALL	{ int nosys(void); } 
+354	ALL	{ int nosys(void); } 
+355	ALL	{ int nosys(void); } 
+356	ALL	{ int nosys(void); } 
+357	ALL	{ int nosys(void); } 
+358	ALL	{ int nosys(void); } 
+359	ALL	{ int nosys(void); } 
+#endif
+
+360	ALL	{ user_addr_t bsdthread_create(user_addr_t func, user_addr_t func_arg, user_addr_t stack, user_addr_t pthread, uint32_t flags) NO_SYSCALL_STUB; } 
+361	ALL	{ int bsdthread_terminate(user_addr_t stackaddr, size_t freesize, uint32_t port, uint32_t sem) NO_SYSCALL_STUB; } 
+362	ALL	{ int kqueue(void); } 
+363	ALL	{ int kevent(int fd, const struct kevent *changelist, int nchanges, struct kevent *eventlist, int nevents, const struct timespec *timeout); } 
+364	ALL	{ int lchown(user_addr_t path, uid_t owner, gid_t group); }
+365	ALL	{ int stack_snapshot(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t options) NO_SYSCALL_STUB; }
+366	ALL	{ int bsdthread_register(user_addr_t threadstart, user_addr_t wqthread, int pthsize) NO_SYSCALL_STUB; } 
+367	ALL	{ int workq_open(void) NO_SYSCALL_STUB; }
+368	ALL	{ int workq_ops(int options, user_addr_t item, int prio) NO_SYSCALL_STUB; }
+369	ALL	{ int nosys(void); } 
+370	ALL	{ int nosys(void); } 
+371	ALL	{ int nosys(void); } 
+372	ALL	{ int nosys(void); } 
+373	ALL	{ int nosys(void); } 
+374	ALL	{ int nosys(void); } 
+375	ALL	{ int nosys(void); } 
+376	ALL	{ int nosys(void); } 
+377	ALL	{ int nosys(void); } 
+378	ALL	{ int nosys(void); } 
+379	ALL	{ int nosys(void); } 
+380	ALL	{ int __mac_execve(char *fname, char **argp, char **envp, struct mac *mac_p); } 
+381	ALL	{ int __mac_syscall(char *policy, int call, user_addr_t arg); } 
+382	ALL	{ int __mac_get_file(char *path_p, struct mac *mac_p); } 
+383	ALL	{ int __mac_set_file(char *path_p, struct mac *mac_p); } 
+384	ALL	{ int __mac_get_link(char *path_p, struct mac *mac_p); } 
+385	ALL	{ int __mac_set_link(char *path_p, struct mac *mac_p); } 
+386	ALL	{ int __mac_get_proc(struct mac *mac_p); } 
+387	ALL	{ int __mac_set_proc(struct mac *mac_p); } 
+388	ALL	{ int __mac_get_fd(int fd, struct mac *mac_p); } 
+389	ALL	{ int __mac_set_fd(int fd, struct mac *mac_p); } 
+390	ALL	{ int __mac_get_pid(pid_t pid, struct mac *mac_p); } 
+391	ALL	{ int __mac_get_lcid(pid_t lcid, struct mac *mac_p); }
+392	ALL	{ int __mac_get_lctx(struct mac *mac_p); }
+393	ALL	{ int __mac_set_lctx(struct mac *mac_p); }
+394	ALL	{ int setlcid(pid_t pid, pid_t lcid) NO_SYSCALL_STUB; }
+395	ALL	{ int getlcid(pid_t pid) NO_SYSCALL_STUB; }
+396	ALL	{ user_ssize_t read_nocancel(int fd, user_addr_t cbuf, user_size_t nbyte) NO_SYSCALL_STUB; } 
+397	ALL	{ user_ssize_t write_nocancel(int fd, user_addr_t cbuf, user_size_t nbyte) NO_SYSCALL_STUB; } 
+398	ALL	{ int open_nocancel(user_addr_t path, int flags, int mode) NO_SYSCALL_STUB; } 
+399	ALL	{ int close_nocancel(int fd) NO_SYSCALL_STUB; } 
+400	ALL	{ int wait4_nocancel(int pid, user_addr_t status, int options, user_addr_t rusage) NO_SYSCALL_STUB; } 
+#if SOCKETS
+401	ALL	{ int recvmsg_nocancel(int s, struct msghdr *msg, int flags) NO_SYSCALL_STUB; } 
+402	ALL	{ int sendmsg_nocancel(int s, caddr_t msg, int flags) NO_SYSCALL_STUB; } 
+403	ALL	{ int recvfrom_nocancel(int s, void *buf, size_t len, int flags, struct sockaddr *from, int *fromlenaddr) NO_SYSCALL_STUB; } 
+404	ALL	{ int accept_nocancel(int s, caddr_t name, socklen_t	*anamelen) NO_SYSCALL_STUB; } 
+#else
+401	ALL	{ int nosys(void); }
+402	ALL	{ int nosys(void); }
+403	ALL	{ int nosys(void); }
+404	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+405	ALL	{ int msync_nocancel(caddr_t addr, size_t len, int flags) NO_SYSCALL_STUB; } 
+406	ALL	{ int fcntl_nocancel(int fd, int cmd, long arg) NO_SYSCALL_STUB; } 
+407	ALL	{ int select_nocancel(int nd, u_int32_t *in, u_int32_t *ou, u_int32_t *ex, struct timeval *tv) NO_SYSCALL_STUB; } 
+408	ALL	{ int fsync_nocancel(int fd) NO_SYSCALL_STUB; } 
+#if SOCKETS
+409	ALL	{ int connect_nocancel(int s, caddr_t name, socklen_t namelen) NO_SYSCALL_STUB; } 
+#else
+409	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+410	ALL	{ int sigsuspend_nocancel(sigset_t mask) NO_SYSCALL_STUB; } 
+411	ALL	{ user_ssize_t readv_nocancel(int fd, struct iovec *iovp, u_int iovcnt) NO_SYSCALL_STUB; } 
+412	ALL	{ user_ssize_t writev_nocancel(int fd, struct iovec *iovp, u_int iovcnt) NO_SYSCALL_STUB; } 
+#if SOCKETS
+413	ALL	{ int sendto_nocancel(int s, caddr_t buf, size_t len, int flags, caddr_t to, socklen_t tolen) NO_SYSCALL_STUB; } 
+#else
+413	ALL	{ int nosys(void); }
+#endif /* SOCKETS */
+414	ALL	{ user_ssize_t pread_nocancel(int fd, user_addr_t buf, user_size_t nbyte, off_t offset) NO_SYSCALL_STUB; } 
+415	ALL	{ user_ssize_t pwrite_nocancel(int fd, user_addr_t buf, user_size_t nbyte, off_t offset) NO_SYSCALL_STUB; } 
+416	ALL	{ int waitid_nocancel(idtype_t idtype, id_t id, siginfo_t *infop, int options) NO_SYSCALL_STUB; } 
+417	ALL	{ int poll_nocancel(struct pollfd *fds, u_int nfds, int timeout) NO_SYSCALL_STUB; } 
+#if SYSV_MSG
+418	ALL	{ int msgsnd_nocancel(int msqid, void *msgp, size_t msgsz, int msgflg) NO_SYSCALL_STUB; } 
+419	ALL	{ user_ssize_t msgrcv_nocancel(int msqid, void *msgp, size_t msgsz, long msgtyp, int msgflg) NO_SYSCALL_STUB; } 
 #else
-247	NONE	NONE	ALL	{ int nosys(void); } 
-248	NONE	NONE	ALL	{ int nosys(void); } 
+418	ALL	{ int nosys(void); } 
+419	ALL	{ int nosys(void); } 
 #endif
+420	ALL	{ int sem_wait_nocancel(sem_t *sem) NO_SYSCALL_STUB; } 
+421	ALL	{ int aio_suspend_nocancel(user_addr_t aiocblist, int nent, user_addr_t timeoutp) NO_SYSCALL_STUB; } 
+422	ALL	{ int __sigwait_nocancel(user_addr_t set, user_addr_t sig) NO_SYSCALL_STUB; } 
+423	ALL	{ int __semwait_signal_nocancel(int cond_sem, int mutex_sem, int timeout, int relative, time_t tv_sec, int32_t tv_nsec) NO_SYSCALL_STUB; } 
+424	ALL	{ int __mac_mount(char *type, char *path, int flags, caddr_t data, struct mac *mac_p); } 
+425	ALL	{ int __mac_get_mount(char *path, struct mac *mac_p); } 
+426	ALL	{ int __mac_getfsstat(user_addr_t buf, int bufsize, user_addr_t mac, int macsize, int flags); } 
 
-249	NONE	NONE	ALL	{ int nosys(void); } 
-250	NONE	NONE	ALL	{ int minherit(void *addr, size_t len, int inherit); } 
-251	NONE	NONE	ALL	{ int semsys(u_int which, int a2, int a3, int a4, int a5); } 
-252	NONE	NONE	ALL	{ int msgsys(u_int which, int a2, int a3, int a4, int a5); }
-253	NONE	NONE	ALL	{ int shmsys(u_int which, int a2, int a3, int a4); } 
-254	NONE	NONE	ALL	{ int semctl(int semid, int semnum, int cmd, semun_t arg); } 
-255	NONE	NONE	ALL	{ int semget(key_t key, int	nsems, int semflg); } 
-256	NONE	NONE	ALL	{ int semop(int semid, struct sembuf *sops, int nsops); } 
-257	NONE	NONE	ALL	{ int nosys(void); } 
-258	NONE	NONE	ALL	{ int msgctl(int msqid, int cmd, struct	msqid_ds *buf); } 
-259	NONE	NONE	ALL	{ int msgget(key_t key, int msgflg); } 
-260	PRE 	NONE	ALL	{ int msgsnd(int msqid, void *msgp, size_t msgsz, int msgflg); } 
-261	PRE 	NONE	ALL	{ user_ssize_t msgrcv(int msqid, void *msgp, size_t msgsz, long msgtyp, int msgflg); } 
-262	NONE	NONE	ALL	{ int shmat(int shmid, void *shmaddr, int shmflg); } 
-263	NONE	NONE	ALL	{ int shmctl(int shmid, int cmd, struct shmid_ds *buf); } 
-264	NONE	NONE	ALL	{ int shmdt(void *shmaddr); } 
-265	NONE	NONE	ALL	{ int shmget(key_t key, size_t size, int shmflg); } 
-266	NONE	NONE	ALL	{ int shm_open(const char *name, int oflag, int mode); } 
-267	NONE	NONE	ALL	{ int shm_unlink(const char *name); } 
-268	NONE	NONE	ALL	{ user_addr_t sem_open(const char *name, int oflag, int mode, int value); } 
-269	NONE	NONE	ALL	{ int sem_close(sem_t *sem); } 
-270	NONE	NONE	ALL	{ int sem_unlink(const char *name); } 
-271	PRE 	NONE	ALL	{ int sem_wait(sem_t *sem); } 
-272	NONE	NONE	ALL	{ int sem_trywait(sem_t *sem); } 
-273	NONE	NONE	ALL	{ int sem_post(sem_t *sem); } 
-274	NONE	NONE	ALL	{ int sem_getvalue(sem_t *sem, int *sval); } 
-275	NONE	NONE 	ALL	{ int sem_init(sem_t *sem, int phsared, u_int value); } 
-276	NONE	NONE	ALL	{ int sem_destroy(sem_t *sem); } 
-277	NONE	NONE	ALL	{ int open_extended(user_addr_t path, int flags, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity); } 
-278	NONE	KERN	ALL	{ int umask_extended(int newmask, user_addr_t xsecurity); } 
-279	NONE	NONE	ALL	{ int stat_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size); } 
-280	NONE	NONE	ALL	{ int lstat_extended(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size); } 
-281	NONE	NONE	ALL	{ int fstat_extended(int fd, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size); } 
-282	NONE	NONE	ALL	{ int chmod_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity); } 
-283	NONE	NONE	ALL	{ int fchmod_extended(int fd, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity); } 
-284	NONE	NONE	ALL	{ int access_extended(user_addr_t entries, size_t size, user_addr_t results, uid_t uid); } 
-285	NONE	NONE	ALL	{ int settid(uid_t uid, gid_t gid); } 
-286	NONE	NONE	ALL	{ int gettid(uid_t *uidp, gid_t *gidp); } 
-287	NONE	NONE	ALL	{ int setsgroups(int setlen, user_addr_t guidset); } 
-288	NONE	NONE	ALL	{ int getsgroups(user_addr_t setlen, user_addr_t guidset); } 
-289	NONE	NONE	ALL	{ int setwgroups(int setlen, user_addr_t guidset); } 
-290	NONE	NONE	ALL	{ int getwgroups(user_addr_t setlen, user_addr_t guidset); }
-291	NONE	NONE	ALL	{ int mkfifo_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity); } 
-292	NONE	NONE	ALL	{ int mkdir_extended(user_addr_t path, uid_t uid, gid_t gid, int mode, user_addr_t xsecurity); } 
-293	NONE	NONE	ALL	{ int identitysvc(int opcode, user_addr_t message); } 
-294	NONE	NONE	ALL	{ int nosys(void); } 
-295	NONE	NONE	ALL	{ int nosys(void); } 
-296	NONE	KERN	UALL	{ int load_shared_file(char *filename, caddr_t mfa, u_long mfs, caddr_t *ba, int map_cnt, sf_mapping_t *mappings, int *flags); } 
-297	NONE	KERN	UALL	{ int reset_shared_file(caddr_t *ba, int map_cnt, sf_mapping_t *mappings); } 
-298	NONE	KERN	ALL	{ int new_system_shared_regions(void); } 
-299	NONE	KERN	UALL	{ int shared_region_map_file_np(int fd, uint32_t mappingCount, user_addr_t mappings, user_addr_t slide_p); } 
-300	NONE	KERN	UALL	{ int shared_region_make_private_np(uint32_t rangeCount, user_addr_t ranges); } 
-301	NONE	NONE	ALL	{ int nosys(void); } 
-302	NONE	NONE	ALL	{ int nosys(void); } 
-303	NONE	NONE	ALL	{ int nosys(void); } 
-304	NONE	NONE	ALL	{ int nosys(void); } 
-305	NONE	NONE	ALL	{ int nosys(void); } 
-306	NONE	NONE	ALL	{ int nosys(void); } 
-307	NONE	NONE	ALL	{ int nosys(void); } 
-308	NONE	NONE	ALL	{ int nosys(void); } 
-309	NONE	NONE	ALL	{ int nosys(void); } 
-310	NONE	KERN	ALL	{ int getsid(pid_t pid); } 
-311	NONE	NONE	ALL	{ int settid_with_pid(pid_t pid, int assume); } 
-312	NONE	NONE	ALL	{ int nosys(void); } 
-313	NONE	NONE	ALL	{ int aio_fsync(int op, user_addr_t aiocbp); } 
-314 NONE	NONE	ALL	{ user_ssize_t aio_return(user_addr_t aiocbp); } 
-315	PRE 	NONE	ALL	{ int aio_suspend(user_addr_t aiocblist, int nent, user_addr_t timeoutp); } 
-316 NONE	NONE	ALL	{ int aio_cancel(int fd, user_addr_t aiocbp); } 
-317	NONE	NONE	ALL	{ int aio_error(user_addr_t aiocbp); } 
-318	NONE	NONE	ALL	{ int aio_read(user_addr_t aiocbp); } 
-319 NONE	NONE	ALL	{ int aio_write(user_addr_t aiocbp); } 
-320 NONE	NONE	ALL	{ int lio_listio(int mode, user_addr_t aiocblist, int nent, user_addr_t sigp); } 
-321	NONE	NONE	ALL	{ int nosys(void); } 
-322	NONE	NONE	ALL	{ int nosys(void); } 
-323	NONE	NONE	ALL	{ int nosys(void); } 
-324	NONE	NONE	ALL	{ int mlockall(int how); } 
-325	NONE	NONE	ALL	{ int munlockall(int how); } 
-326	NONE	NONE	ALL	{ int nosys(void); } 
-327	NONE	KERN	ALL	{ int issetugid(void); } 
-328	NONE	KERN	ALL	{ int __pthread_kill(int thread_port, int sig); } 
-329	NONE	KERN	ALL	{ int pthread_sigmask(int how, user_addr_t set, user_addr_t oset); } 
-330	PRE 	KERN	ALL	{ int sigwait(user_addr_t set, user_addr_t sig); } 
-331	NONE	KERN	ALL	{ int __disable_threadsignal(int value); } 
-332	NONE	NONE	ALL	{ int __pthread_markcancel(int thread_port); } 
-333	NONE	NONE	ALL	{ int __pthread_canceled(int  action); } 
-334	POST	NONE	ALL	{ int __semwait_signal(int cond_sem, int mutex_sem, int timeout, int relative, time_t tv_sec, int32_t tv_nsec); } 
-335	NONE	KERN	ALL	{ int utrace(const void *addr, size_t len); } 
-336	NONE	NONE	ALL	{ int proc_info(int32_t callnum,int32_t pid,uint32_t flavor, uint64_t arg,user_addr_t buffer,int32_t buffersize); } 
-337	NONE	NONE	ALL	{ int nosys(void); } 
-338	NONE	NONE	ALL	{ int nosys(void); } 
-339	NONE	NONE	ALL	{ int nosys(void); } 
-340	NONE	NONE	ALL	{ int nosys(void); } 
-341	NONE	NONE	ALL	{ int nosys(void); } 
-342	NONE	NONE	ALL	{ int nosys(void); } 
-343	NONE	NONE	ALL	{ int nosys(void); } 
-344	NONE	NONE	ALL	{ int nosys(void); } 
-345	NONE	NONE	ALL	{ int nosys(void); } 
-346	NONE	NONE	ALL	{ int nosys(void); } 
-347	NONE	NONE	ALL	{ int nosys(void); } 
-348	NONE	NONE	ALL	{ int nosys(void); } 
-349	NONE	NONE	ALL	{ int nosys(void); } 
-350	NONE	KERN	ALL	{ int audit(void *record, int length); } 
-351	NONE	KERN	ALL	{ int auditon(int cmd, void *data, int length); } 
-352	NONE	KERN	ALL	{ int nosys(void); } 
-353	NONE	KERN	ALL	{ int getauid(au_id_t *auid); } 
-354	NONE	KERN	ALL	{ int setauid(au_id_t *auid); } 
-355	NONE	KERN	ALL	{ int getaudit(struct auditinfo *auditinfo); } 
-356	NONE	KERN	ALL	{ int setaudit(struct auditinfo *auditinfo); } 
-357	NONE	KERN	ALL	{ int getaudit_addr(struct auditinfo_addr *auditinfo_addr, int length); } 
-358	NONE	KERN	ALL	{ int setaudit_addr(struct auditinfo_addr *auditinfo_addr, int length); } 
-359	NONE	KERN	ALL	{ int auditctl(char *path); } 
-360	NONE	NONE	ALL	{ int nosys(void); } 
-361	NONE	NONE	ALL	{ int nosys(void); } 
-362	NONE	NONE	ALL	{ int kqueue(void); } 
-363	NONE	NONE	ALL	{ int kevent(int fd, const struct kevent *changelist, int nchanges, struct kevent *eventlist, int nevents, const struct timespec *timeout); } 
-364	NONE	NONE	ALL	{ int lchown(user_addr_t path, uid_t owner, gid_t group); }
-365	NONE	NONE	ALL	{ int stack_snapshot(pid_t pid, user_addr_t tracebuf, uint32_t tracebuf_size, uint32_t options); }
-366	NONE	NONE	ALL	{ int nosys(void); } 
-367	NONE	NONE	ALL	{ int nosys(void); } 
-368	NONE	NONE	ALL	{ int nosys(void); } 
-369	NONE	NONE	ALL	{ int nosys(void); } 
diff --git a/bsd/kern/sysctl_init.c b/bsd/kern/sysctl_init.c
index 82817a457..1319ea15b 100644
--- a/bsd/kern/sysctl_init.c
+++ b/bsd/kern/sysctl_init.c
@@ -1,961 +1,33 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
-
-#include <sys/param.h>
-#include <sys/kernel.h>
-#include <sys/sysctl.h>
-
-extern struct sysctl_oid sysctl__debug_bpf_bufsize;
-extern struct sysctl_oid sysctl__debug_bpf_maxbufsize;
-extern struct sysctl_oid sysctl__debug_bpf_maxdevices;
-extern struct sysctl_oid sysctl__debug_iokit;
-
-#if TUN
-extern struct sysctl_oid sysctl__debug_if_tun_debug;
-#endif
-
-#if COMPAT_43_TTY
-#ifndef NeXT
-extern struct sysctl_oid sysctl__debug_ttydebug;
-#endif
-#endif
-
-extern struct sysctl_oid sysctl__hw_machine;
-extern struct sysctl_oid sysctl__hw_model;
-extern struct sysctl_oid sysctl__hw_ncpu;
-extern struct sysctl_oid sysctl__hw_activecpu;
-extern struct sysctl_oid sysctl__hw_physicalcpu;
-extern struct sysctl_oid sysctl__hw_physicalcpu_max;
-extern struct sysctl_oid sysctl__hw_logicalcpu;
-extern struct sysctl_oid sysctl__hw_logicalcpu_max;
-extern struct sysctl_oid sysctl__hw_byteorder;
-extern struct sysctl_oid sysctl__hw_cputype;
-extern struct sysctl_oid sysctl__hw_cpusubtype;
-extern struct sysctl_oid sysctl__hw_cpufamily;
-extern struct sysctl_oid sysctl__hw_cacheconfig;
-extern struct sysctl_oid sysctl__hw_physmem;
-extern struct sysctl_oid sysctl__hw_usermem;
-extern struct sysctl_oid sysctl__hw_pagesize;
-extern struct sysctl_oid sysctl__hw_pms;
-extern struct sysctl_oid sysctl__hw_epoch;
-extern struct sysctl_oid sysctl__hw_vectorunit;
-extern struct sysctl_oid sysctl__hw_busfrequency;
-extern struct sysctl_oid sysctl__hw_busfrequency_min;
-extern struct sysctl_oid sysctl__hw_busfrequency_max;
-extern struct sysctl_oid sysctl__hw_cpufrequency;
-extern struct sysctl_oid sysctl__hw_cpufrequency_min;
-extern struct sysctl_oid sysctl__hw_cpufrequency_max;
-extern struct sysctl_oid sysctl__hw_cachelinesize;
-extern struct sysctl_oid sysctl__hw_l1icachesize;
-extern struct sysctl_oid sysctl__hw_l1dcachesize;
-extern struct sysctl_oid sysctl__hw_l2settings;
-extern struct sysctl_oid sysctl__hw_l2cachesize;
-extern struct sysctl_oid sysctl__hw_l3settings;
-extern struct sysctl_oid sysctl__hw_l3cachesize;
-extern struct sysctl_oid sysctl__hw_tbfrequency;
-extern struct sysctl_oid sysctl__hw_memsize;
-
-extern struct sysctl_oid sysctl__hw_optional;
-extern struct sysctl_oid sysctl__hw_optional_floatingpoint;
-
-extern struct sysctl_oid sysctl__hw_pagesize_compat;
-extern struct sysctl_oid sysctl__hw_busfrequency_compat;
-extern struct sysctl_oid sysctl__hw_cpufrequency_compat;
-extern struct sysctl_oid sysctl__hw_cachelinesize_compat;
-extern struct sysctl_oid sysctl__hw_l1icachesize_compat;
-extern struct sysctl_oid sysctl__hw_l1dcachesize_compat;
-extern struct sysctl_oid sysctl__hw_l2cachesize_compat;
-extern struct sysctl_oid sysctl__hw_l3cachesize_compat;
-extern struct sysctl_oid sysctl__hw_tbfrequency_compat;
-
-#if __i386__
-extern struct sysctl_oid sysctl__sysctl_proc_exec_affinity;
-#endif
-extern struct sysctl_oid sysctl__sysctl_proc_cputype;
-extern struct sysctl_oid sysctl__sysctl_proc_native;
-
-extern struct sysctl_oid sysctl__kern_sysv_shmmax;
-extern struct sysctl_oid sysctl__kern_sysv_shmmin;
-extern struct sysctl_oid sysctl__kern_sysv_shmmni;
-extern struct sysctl_oid sysctl__kern_sysv_shmseg;
-extern struct sysctl_oid sysctl__kern_sysv_shmall;
-extern struct sysctl_oid sysctl__kern_sysv_ipcs;
-extern struct sysctl_oid sysctl__kern_sysv_ipcs_shm;
-extern struct sysctl_oid sysctl__kern_sysv_ipcs_sem;
-extern struct sysctl_oid sysctl__kern_sysv_ipcs_msg;
-
-extern struct sysctl_oid sysctl__kern_sysv_semmni;
-extern struct sysctl_oid sysctl__kern_sysv_semmns;
-extern struct sysctl_oid sysctl__kern_sysv_semmnu;
-extern struct sysctl_oid sysctl__kern_sysv_semmsl;
-extern struct sysctl_oid sysctl__kern_sysv_semume;
-
-extern struct sysctl_oid sysctl__kern_tfp_policy;
-extern struct sysctl_oid sysctl__kern_tfp_read_group;
-extern struct sysctl_oid sysctl__kern_tfp_rw_group;
-
-extern struct sysctl_oid sysctl__kern_dummy;
-extern struct sysctl_oid sysctl__kern_ipc_maxsockbuf;
-extern struct sysctl_oid sysctl__kern_ipc_mbstat;
-extern struct sysctl_oid sysctl__kern_ipc_nmbclusters;
-extern struct sysctl_oid sysctl__kern_ipc_sockbuf_waste_factor;
-extern struct sysctl_oid sysctl__kern_ipc_somaxconn;
-extern struct sysctl_oid sysctl__kern_ipc_sosendminchain;
-extern struct sysctl_oid sysctl__kern_ipc_sorecvmincopy;
-extern struct sysctl_oid sysctl__kern_ipc_maxsockets;
-extern struct sysctl_oid sysctl__kern_posix;
-extern struct sysctl_oid sysctl__kern_posix_sem;
-extern struct sysctl_oid sysctl__kern_posix_sem_max;
-extern struct sysctl_oid sysctl__kern_sugid_scripts;
-extern struct sysctl_oid sysctl__kern_nbuf;
-extern struct sysctl_oid sysctl__kern_maxnbuf;
-extern struct sysctl_oid sysctl__kern_always_do_fullfsync;
-extern struct sysctl_oid sysctl__kern_flush_cache_on_write;
-extern struct sysctl_oid sysctl__kern_exec;
-extern struct sysctl_oid sysctl__kern_exec_archhandler;
-extern struct sysctl_oid sysctl__kern_exec_archhandler_powerpc;
-extern struct sysctl_oid sysctl__net_inet_icmp_icmplim;
-extern struct sysctl_oid sysctl__net_inet_icmp_maskrepl;
-extern struct sysctl_oid sysctl__net_inet_icmp_timestamp;
-extern struct sysctl_oid sysctl__net_inet_icmp_bmcastecho;
-extern struct sysctl_oid sysctl__net_inet_icmp_log_redirect;
-extern struct sysctl_oid sysctl__net_inet_icmp_drop_redirect;
-extern struct sysctl_oid sysctl__net_inet_ip_accept_sourceroute;
-
-#if IPCTL_DEFMTU
-extern struct sysctl_oid sysctl__net_inet_ip_mtu;
-#endif
-
-extern struct sysctl_oid sysctl__net_inet_ip_ttl;
-extern struct sysctl_oid sysctl__net_inet_ip_fastforwarding;
-extern struct sysctl_oid sysctl__net_inet_ip_forwarding;
-extern struct sysctl_oid sysctl__net_inet_ip_intr_queue_drops;
-extern struct sysctl_oid sysctl__net_inet_ip_intr_queue_maxlen;
-extern struct sysctl_oid sysctl__net_inet_ip_rtexpire;
-extern struct sysctl_oid sysctl__net_inet_ip_rtmaxcache;
-extern struct sysctl_oid sysctl__net_inet_ip_rtminexpire;
-extern struct sysctl_oid sysctl__net_inet_ip_redirect;
-extern struct sysctl_oid sysctl__net_inet_ip_sourceroute;
-extern struct sysctl_oid sysctl__net_inet_ip_subnets_are_local;
-extern struct sysctl_oid sysctl__net_inet_ip_keepfaith;
-extern struct sysctl_oid sysctl__net_inet_ip_maxfragpackets;
-extern struct sysctl_oid sysctl__net_inet_ip_maxfragsperpacket;
-extern struct sysctl_oid sysctl__net_inet_ip_maxfrags;
-extern struct sysctl_oid sysctl__net_inet_ip_check_interface;
-extern struct sysctl_oid sysctl__net_inet_ip_check_route_selfref;
-extern struct sysctl_oid sysctl__net_inet_ip_use_route_genid;
-#if NGIF > 0
-extern struct sysctl_oid sysctl__net_inet_ip_gifttl;
-#endif
-
-#if DUMMYNET
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_hash_size;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_curr_time;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_ready_heap;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_extract_heap;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_searches;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_search_steps;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_expire;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_max_chain_len;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_red_lookup_depth;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_red_avg_pkt_size;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet_red_max_pkt_size;
-extern struct sysctl_oid sysctl__net_inet_ip_dummynet;
-#endif
-
-#if IPFIREWALL && !IPFIREWALL_KEXT
-extern struct sysctl_oid sysctl__net_inet_ip_fw_enable;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_debug;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_verbose;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_verbose_limit;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_one_pass;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_autoinc_step;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_buckets;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_curr_dyn_buckets;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_count;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_max;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_static_count;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_ack_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_syn_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_fin_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_rst_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_udp_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_short_lifetime;
-extern struct sysctl_oid sysctl__net_inet_ip_fw_dyn_keepalive;
-extern struct sysctl_oid sysctl__net_inet_ip_fw;
-#endif
-
-extern struct sysctl_oid sysctl__net_inet_ip_linklocal;
-extern struct sysctl_oid sysctl__net_inet_ip_linklocal_stat;
-extern struct sysctl_oid sysctl__net_inet_ip_linklocal_in;
-extern struct sysctl_oid sysctl__net_inet_ip_linklocal_in_allowbadttl;
-extern struct sysctl_oid sysctl__net_inet_ip_maxchainsent;
-
-extern struct sysctl_oid sysctl__net_inet_raw_maxdgram;
-extern struct sysctl_oid sysctl__net_inet_raw_recvspace;
-extern struct sysctl_oid sysctl__net_inet_tcp_always_keepalive;
-extern struct sysctl_oid sysctl__net_inet_tcp_delayed_ack;
-extern struct sysctl_oid sysctl__net_inet_tcp_log_in_vain;
-extern struct sysctl_oid sysctl__net_inet_tcp_pcbcount;
-extern struct sysctl_oid sysctl__net_inet_tcp_rfc1323;
-extern struct sysctl_oid sysctl__net_inet_tcp_rfc1644;
-extern struct sysctl_oid sysctl__net_inet_tcp_keepidle;
-extern struct sysctl_oid sysctl__net_inet_tcp_keepinit;
-extern struct sysctl_oid sysctl__net_inet_tcp_keepintvl;
-extern struct sysctl_oid sysctl__net_inet_tcp_mssdflt;
-extern struct sysctl_oid sysctl__net_inet_tcp_minmss;
-extern struct sysctl_oid sysctl__net_inet_tcp_minmssoverload;
-extern struct sysctl_oid sysctl__net_inet_tcp_recvspace;
-extern struct sysctl_oid sysctl__net_inet_tcp_sendspace;
-extern struct sysctl_oid sysctl__net_inet_tcp_slowlink_wsize;
-extern struct sysctl_oid sysctl__net_inet_tcp_blackhole;
-extern struct sysctl_oid sysctl__net_inet_tcp_tcp_lq_overflow;
-extern struct sysctl_oid sysctl__net_inet_tcp_path_mtu_discovery;
-extern struct sysctl_oid sysctl__net_inet_tcp_slowstart_flightsize;
-extern struct sysctl_oid sysctl__net_inet_tcp_local_slowstart_flightsize;
-extern struct sysctl_oid sysctl__net_inet_tcp_newreno;
-extern struct sysctl_oid sysctl__net_inet_tcp_packetchain;
-extern struct sysctl_oid sysctl__net_inet_tcp_tcbhashsize;
-extern struct sysctl_oid sysctl__net_inet_tcp_do_tcpdrain;
-extern struct sysctl_oid sysctl__net_inet_tcp_icmp_may_rst;
-extern struct sysctl_oid sysctl__net_inet_tcp_strict_rfc1948;
-extern struct sysctl_oid sysctl__net_inet_tcp_delacktime;
-extern struct sysctl_oid sysctl__net_inet_tcp_isn_reseed_interval;
-extern struct sysctl_oid sysctl__net_inet_tcp_msl;
-extern struct sysctl_oid sysctl__net_inet_tcp_sack;
-extern struct sysctl_oid sysctl__net_inet_tcp_sack_maxholes;
-extern struct sysctl_oid sysctl__net_inet_tcp_sack_globalmaxholes;
-extern struct sysctl_oid sysctl__net_inet_tcp_sack_globalholes;
-#if TCP_DROP_SYNFIN
-extern struct sysctl_oid sysctl__net_inet_tcp_drop_synfin;
-#endif
-#if TCPDEBUG
-extern struct sysctl_oid sysctl__net_inet_tcp_tcpconsdebug;
-#endif
-extern struct sysctl_oid sysctl__net_inet_tcp_sockthreshold;
-extern struct sysctl_oid sysctl__net_inet_tcp_reass_maxsegments;
-extern struct sysctl_oid sysctl__net_inet_tcp_reass_cursegments;
-extern struct sysctl_oid sysctl__net_inet_tcp_reass_overflows;
-extern struct sysctl_oid sysctl__net_inet_tcp_reass;
-
-extern struct sysctl_oid sysctl__net_inet_udp_log_in_vain;
-extern struct sysctl_oid sysctl__net_inet_udp_checksum;
-extern struct sysctl_oid sysctl__net_inet_udp_maxdgram;
-extern struct sysctl_oid sysctl__net_inet_udp_recvspace;
-extern struct sysctl_oid sysctl__net_inet_udp_blackhole;
-extern struct sysctl_oid sysctl__net_inet_udp_pcbcount;
-
-#if NETAT
-extern struct sysctl_oid sysctl__net_appletalk_debug;
-extern struct sysctl_oid sysctl__net_appletalk_routermix;
-extern struct sysctl_oid sysctl__net_appletalk_ddpstats;
-#endif /* NETAT */
-
-#if BRIDGE
-extern struct sysctl_oid sysctl__net_link_ether_bdgfwc;
-extern struct sysctl_oid sysctl__net_link_ether_bdgfwt;
-extern struct sysctl_oid sysctl__net_link_ether_bdginc;
-extern struct sysctl_oid sysctl__net_link_ether_bdgint;
-extern struct sysctl_oid sysctl__net_link_ether_bridge_ipfw;
-extern struct sysctl_oid sysctl__net_link_ethe_bdgstats;
-#endif
-
-extern struct sysctl_oid sysctl__net_link_ether_inet_host_down_time;
-extern struct sysctl_oid sysctl__net_link_ether_inet_max_age;
-extern struct sysctl_oid sysctl__net_link_ether_inet_maxtries;
-extern struct sysctl_oid sysctl__net_link_ether_inet_proxyall;
-extern struct sysctl_oid sysctl__net_link_ether_inet_prune_intvl;
-extern struct sysctl_oid sysctl__net_link_ether_inet_useloopback;
-extern struct sysctl_oid sysctl__net_link_ether_inet_log_arp_warnings;
-extern struct sysctl_oid sysctl__net_link_ether_inet_apple_hwcksum_tx;
-extern struct sysctl_oid sysctl__net_link_ether_inet_apple_hwcksum_rx;
-
-#if NETMIBS
-extern struct sysctl_oid sysctl__net_link_generic_system_ifcount;
-extern struct sysctl_oid sysctl__net_link_generic;
-extern struct sysctl_oid sysctl__net_link_generic_ifdata;
-extern struct sysctl_oid sysctl__net_link_generic_ifalldata;
-extern struct sysctl_oid sysctl__net_link_generic_system;
-#endif
-
-#if VLAN
-extern struct sysctl_oid sysctl__net_link_vlan_link_proto;
-extern struct sysctl_oid sysctl__net_link_vlan;
-extern struct sysctl_oid sysctl__net_link_vlan_link;
-#endif
-
-extern struct sysctl_oid sysctl__net_local_inflight;
-extern struct sysctl_oid sysctl__net_local_dgram_maxdgram;
-extern struct sysctl_oid sysctl__net_local_dgram_recvspace;
-extern struct sysctl_oid sysctl__net_local_stream_recvspace;
-extern struct sysctl_oid sysctl__net_local_stream_sendspace;
-
-#if 0
-extern struct sysctl_oid sysctl__vfs_nfs_nfs_privport;
-extern struct sysctl_oid sysctl__vfs_nfs_async;
-extern struct sysctl_oid sysctl__vfs_nfs_debug;
-extern struct sysctl_oid sysctl__vfs_nfs_defect;
-extern struct sysctl_oid sysctl__vfs_nfs_diskless_valid;
-extern struct sysctl_oid sysctl__vfs_nfs_gatherdelay;
-extern struct sysctl_oid sysctl__vfs_nfs_gatherdelay_v3;
-extern struct sysctl_oid sysctl__vfs_nfs;
-extern struct sysctl_oid sysctl__vfs_nfs_diskless_rootaddr;
-extern struct sysctl_oid sysctl__vfs_nfs_diskless_swapaddr;
-extern struct sysctl_oid sysctl__vfs_nfs_diskless_rootpath;
-extern struct sysctl_oid sysctl__vfs_nfs_diskless_swappath;
-extern struct sysctl_oid sysctl__vfs_nfs_nfsstats;
-#endif
-#if NFSCLIENT
-extern struct sysctl_oid sysctl__vfs_generic_nfs_client_initialdowndelay;
-extern struct sysctl_oid sysctl__vfs_generic_nfs_client_nextdowndelay;
-extern struct sysctl_oid sysctl__vfs_generic_nfs_client;
-extern struct sysctl_oid sysctl__vfs_generic_nfs;
-#endif
-
-extern struct sysctl_oid sysctl__vfs_generic;
-extern struct sysctl_oid sysctl__vfs_generic_vfsidlist;
-extern struct sysctl_oid sysctl__vfs_generic_ctlbyfsid;
-extern struct sysctl_oid sysctl__vfs_generic_noremotehang;
-
-extern struct sysctl_oid sysctl__kern_ipc;
-extern struct sysctl_oid sysctl__kern_sysv;
-extern struct sysctl_oid sysctl__kern_tfp;
-
-extern struct sysctl_oid sysctl__net_inet;
-
-#if NETAT
-extern struct sysctl_oid sysctl__net_appletalk;
-#endif /* NETAT */
-
-extern struct sysctl_oid sysctl__net_link;
-extern struct sysctl_oid sysctl__net_local;
-extern struct sysctl_oid sysctl__net_routetable;
-
-#if IPDIVERT
-extern struct sysctl_oid sysctl__net_inet_div;
-#endif
-
-extern struct sysctl_oid sysctl__net_inet_icmp;
-extern struct sysctl_oid sysctl__net_inet_igmp;
-extern struct sysctl_oid sysctl__net_inet_ip;
-extern struct sysctl_oid sysctl__net_inet_raw;
-extern struct sysctl_oid sysctl__net_inet_tcp;
-extern struct sysctl_oid sysctl__net_inet_udp;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange;
-
-extern struct sysctl_oid sysctl__net_link_ether;
-extern struct sysctl_oid sysctl__net_link_ether_inet;
-
-extern struct sysctl_oid sysctl__net_local_dgram;
-extern struct sysctl_oid sysctl__net_local_stream;
-extern struct sysctl_oid sysctl__sysctl_name;
-extern struct sysctl_oid sysctl__sysctl_next;
-extern struct sysctl_oid sysctl__sysctl_oidfmt;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_first;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_hifirst;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_hilast;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_last;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_lowfirst;
-extern struct sysctl_oid sysctl__net_inet_ip_portrange_lowlast;
-extern struct sysctl_oid sysctl__net_inet_raw_pcblist;
-extern struct sysctl_oid sysctl__net_inet_tcp_pcblist;
-extern struct sysctl_oid sysctl__net_inet_udp_pcblist;
-extern struct sysctl_oid sysctl__net_link_ether_bridge;
-extern struct sysctl_oid sysctl__net_local_dgram_pcblist;
-extern struct sysctl_oid sysctl__net_local_stream_pcblist;
-extern struct sysctl_oid sysctl__sysctl_debug;
-extern struct sysctl_oid sysctl__sysctl_name2oid;
-extern struct sysctl_oid sysctl__net_inet_icmp_stats;
-extern struct sysctl_oid sysctl__net_inet_igmp_stats;
-extern struct sysctl_oid sysctl__net_inet_ip_stats;
-extern struct sysctl_oid sysctl__net_inet_tcp_stats;
-extern struct sysctl_oid sysctl__net_inet_udp_stats;
-extern struct sysctl_oid sysctl__kern;
-extern struct sysctl_oid sysctl__hw;
-extern struct sysctl_oid sysctl__machdep;
-extern struct sysctl_oid sysctl__net;
-extern struct sysctl_oid sysctl__debug;
-extern struct sysctl_oid sysctl__vfs;
-extern struct sysctl_oid sysctl__vm;
-extern struct sysctl_oid sysctl__sysctl;
-
-#if INET6
-extern struct sysctl_oid sysctl__net_inet_tcp_v6mssdflt;
-extern struct sysctl_oid sysctl__net_inet6;
-extern struct sysctl_oid sysctl__net_inet6_ip6;
-extern struct sysctl_oid sysctl__net_inet6_ip6_stats;
-extern struct sysctl_oid sysctl__net_inet6_icmp6;
-extern struct sysctl_oid sysctl__net_inet6_ip6_forwarding;
-extern struct sysctl_oid sysctl__net_inet6_ip6_redirect;
-extern struct sysctl_oid sysctl__net_inet6_ip6_hlim;
-extern struct sysctl_oid sysctl__net_inet6_ip6_maxfragpackets;
-extern struct sysctl_oid sysctl__net_inet6_ip6_maxfrags;
-extern struct sysctl_oid sysctl__net_inet6_ip6_accept_rtadv;
-extern struct sysctl_oid sysctl__net_inet6_ip6_keepfaith;
-extern struct sysctl_oid sysctl__net_inet6_ip6_log_interval;
-extern struct sysctl_oid sysctl__net_inet6_ip6_hdrnestlimit;
-extern struct sysctl_oid sysctl__net_inet6_ip6_dad_count;
-extern struct sysctl_oid sysctl__net_inet6_ip6_auto_flowlabel;
-extern struct sysctl_oid sysctl__net_inet6_ip6_defmcasthlim;
-extern struct sysctl_oid sysctl__net_inet6_ip6_gifhlim;
-extern struct sysctl_oid sysctl__net_inet6_ip6_kame_version;
-extern struct sysctl_oid sysctl__net_inet6_ip6_use_deprecated;
-extern struct sysctl_oid sysctl__net_inet6_ip6_rr_prune;
-extern struct sysctl_oid sysctl__net_inet6_ip6_use_tempaddr;
-extern struct sysctl_oid sysctl__net_inet6_ip6_v6only;
-extern struct sysctl_oid sysctl__net_inet6_ip6_auto_linklocal;
-extern struct sysctl_oid sysctl__net_inet6_ip6_rip6stats;
-extern struct sysctl_oid sysctl__net_inet6_ip6_mrt6stat;
-extern struct sysctl_oid sysctl__net_inet6_ip6_rtexpire;
-extern struct sysctl_oid sysctl__net_inet6_ip6_rtminexpire;
-extern struct sysctl_oid sysctl__net_inet6_ip6_rtmaxcache;
-extern struct sysctl_oid sysctl__net_inet6_ip6_temppltime;
-extern struct sysctl_oid sysctl__net_inet6_ip6_tempvltime;
-#if IPV6FIREWALL
-extern struct sysctl_oid sysctl__net_inet6_ip6_fw;
-extern struct sysctl_oid sysctl__net_inet6_ip6_fw_debug;
-extern struct sysctl_oid sysctl__net_inet6_ip6_fw_verbose;
-extern struct sysctl_oid sysctl__net_inet6_ip6_fw_verbose_limit;
-#endif
-extern struct sysctl_oid sysctl__net_inet6_icmp6_rediraccept;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_redirtimeout;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_stats;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nodeinfo;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_errppslimit;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_maxnudhint;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_debug;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_prune;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_delay;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_umaxtries;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_mmaxtries;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nd6_useloopback;
-extern struct sysctl_oid sysctl__net_inet6_icmp6_nodeinfo;
-#if IPSEC
-extern struct sysctl_oid sysctl__net_inet6_ipsec6;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_stats;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_def_policy;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_esp_trans_deflev;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_esp_net_deflev;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_ah_trans_deflev;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_ah_net_deflev;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_ecn;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_debug;
-extern struct sysctl_oid sysctl__net_inet6_ipsec6_esp_randpad;
-#endif
-#endif
-#if IPSEC
-extern struct sysctl_oid sysctl__net_inet_ipsec;
-extern struct sysctl_oid sysctl__net_inet_ipsec_esp_port;
-extern struct sysctl_oid sysctl__net_inet_ipsec_bypass;
-extern struct sysctl_oid sysctl__net_inet_ipsec_def_policy;
-extern struct sysctl_oid sysctl__net_inet_ipsec_esp_randpad;
-extern struct sysctl_oid sysctl__net_inet_ipsec_esp_trans_deflev;
-extern struct sysctl_oid sysctl__net_inet_ipsec_esp_net_deflev;
-extern struct sysctl_oid sysctl__net_inet_ipsec_ah_trans_deflev;
-extern struct sysctl_oid sysctl__net_inet_ipsec_ah_net_deflev;
-extern struct sysctl_oid sysctl__net_inet_ipsec_ah_cleartos;
-extern struct sysctl_oid sysctl__net_inet_ipsec_ah_offsetmask;
-extern struct sysctl_oid sysctl__net_inet_ipsec_dfbit;
-extern struct sysctl_oid sysctl__net_inet_ipsec_ecn;
-extern struct sysctl_oid sysctl__net_inet_ipsec_debug;
-extern struct sysctl_oid sysctl__net_inet_ipsec_stats;
-extern struct sysctl_oid sysctl__net_key;
-extern struct sysctl_oid sysctl__net_key_debug;
-extern struct sysctl_oid sysctl__net_key_prefered_oldsa;
-extern struct sysctl_oid sysctl__net_key_spi_trycnt;
-extern struct sysctl_oid sysctl__net_key_spi_minval;
-extern struct sysctl_oid sysctl__net_key_spi_maxval;
-extern struct sysctl_oid sysctl__net_key_int_random;
-extern struct sysctl_oid sysctl__net_key_larval_lifetime;
-extern struct sysctl_oid sysctl__net_key_blockacq_count;
-extern struct sysctl_oid sysctl__net_key_blockacq_lifetime;
-extern struct sysctl_oid sysctl__net_key_esp_keymin;
-extern struct sysctl_oid sysctl__net_key_esp_auth;
-extern struct sysctl_oid sysctl__net_key_ah_keymin;
-extern struct sysctl_oid sysctl__net_key_natt_keepalive_interval;
-extern struct sysctl_oid sysctl__net_key_pfkeystat;
-#endif
-
-extern struct sysctl_oid sysctl__vm_shared_region_trace_level;
-
-struct sysctl_oid *newsysctl_list[] =
-{
-    &sysctl__kern,
-    &sysctl__hw,
-    &sysctl__machdep,
-    &sysctl__net,
-    &sysctl__debug,
-    &sysctl__vfs,
-    &sysctl__vm,
-    &sysctl__sysctl,
-    &sysctl__debug_bpf_bufsize,
-    &sysctl__debug_bpf_maxbufsize,
-    &sysctl__debug_bpf_maxdevices,
-    &sysctl__debug_iokit
-#if TUN
-    ,&sysctl__debug_if_tun_debug
-#endif
-
-#if COMPAT_43_TTY
-#ifndef NeXT
-    ,&sysctl__debug_ttydebug
-#endif
-#endif
-
-    ,&sysctl__kern_posix
-    ,&sysctl__kern_posix_sem
-    ,&sysctl__kern_posix_sem_max
-
-#if __i386__
-    ,&sysctl__sysctl_proc_exec_affinity
-#endif
-    ,&sysctl__sysctl_proc_cputype
-    ,&sysctl__sysctl_proc_native
-	
-    ,&sysctl__kern_sysv_shmmax
-    ,&sysctl__kern_sysv_shmmin
-    ,&sysctl__kern_sysv_shmmni
-    ,&sysctl__kern_sysv_shmseg
-    ,&sysctl__kern_sysv_shmall
-    ,&sysctl__kern_sysv_ipcs
-    ,&sysctl__kern_sysv_ipcs_shm
-    ,&sysctl__kern_sysv_ipcs_sem
-    ,&sysctl__kern_sysv_ipcs_msg
-    ,&sysctl__kern_sysv_semmni
-    ,&sysctl__kern_sysv_semmns
-    ,&sysctl__kern_sysv_semmnu
-    ,&sysctl__kern_sysv_semmsl
-    ,&sysctl__kern_sysv_semume
-    ,&sysctl__kern_dummy
-
-    ,&sysctl__kern_tfp
-    ,&sysctl__kern_tfp_policy
-    ,&sysctl__kern_tfp_read_group
-    ,&sysctl__kern_tfp_rw_group
-
-    ,&sysctl__kern_ipc_maxsockbuf
-    ,&sysctl__kern_ipc_mbstat
-    ,&sysctl__kern_ipc_nmbclusters
-    ,&sysctl__kern_ipc_sockbuf_waste_factor
-    ,&sysctl__kern_ipc_somaxconn
-    ,&sysctl__kern_ipc_sosendminchain
-    ,&sysctl__kern_ipc_sorecvmincopy
-    ,&sysctl__kern_ipc_maxsockets
-
-    ,&sysctl__kern_sugid_scripts
-    ,&sysctl__kern_always_do_fullfsync
-    ,&sysctl__kern_flush_cache_on_write
-    ,&sysctl__kern_exec
-    ,&sysctl__kern_exec_archhandler
-    ,&sysctl__kern_exec_archhandler_powerpc
-    ,&sysctl__kern_nbuf
-    ,&sysctl__kern_maxnbuf
-
-    ,&sysctl__hw_machine
-    ,&sysctl__hw_model
-    ,&sysctl__hw_ncpu
-    ,&sysctl__hw_activecpu
-    ,&sysctl__hw_physicalcpu
-    ,&sysctl__hw_physicalcpu_max
-    ,&sysctl__hw_logicalcpu
-    ,&sysctl__hw_logicalcpu_max
-    ,&sysctl__hw_byteorder
-    ,&sysctl__hw_cputype
-    ,&sysctl__hw_cpusubtype
-    ,&sysctl__hw_cpufamily
-    ,&sysctl__hw_cacheconfig
-    ,&sysctl__hw_physmem
-    ,&sysctl__hw_usermem
-    ,&sysctl__hw_pagesize
-    ,&sysctl__hw_epoch
-    ,&sysctl__hw_vectorunit
-    ,&sysctl__hw_busfrequency
-    ,&sysctl__hw_busfrequency_min
-    ,&sysctl__hw_busfrequency_max
-    ,&sysctl__hw_cpufrequency
-    ,&sysctl__hw_cpufrequency_min
-    ,&sysctl__hw_cpufrequency_max
-    ,&sysctl__hw_cachelinesize
-    ,&sysctl__hw_l1icachesize
-    ,&sysctl__hw_l1dcachesize
-    ,&sysctl__hw_l2settings
-    ,&sysctl__hw_l2cachesize
-    ,&sysctl__hw_l3settings
-    ,&sysctl__hw_l3cachesize
-    ,&sysctl__hw_tbfrequency
-    ,&sysctl__hw_memsize
-    ,&sysctl__hw_optional
-    ,&sysctl__hw_optional_floatingpoint
-
-    ,&sysctl__hw_pagesize_compat
-    ,&sysctl__hw_pms    
-    ,&sysctl__hw_busfrequency_compat
-    ,&sysctl__hw_cpufrequency_compat
-    ,&sysctl__hw_cachelinesize_compat
-    ,&sysctl__hw_l1icachesize_compat
-    ,&sysctl__hw_l1dcachesize_compat
-    ,&sysctl__hw_l2cachesize_compat
-    ,&sysctl__hw_l3cachesize_compat
-    ,&sysctl__hw_tbfrequency_compat
-
-    ,&sysctl__net_inet_icmp_icmplim
-    ,&sysctl__net_inet_icmp_maskrepl
-    ,&sysctl__net_inet_icmp_timestamp
-    ,&sysctl__net_inet_icmp_bmcastecho
-    ,&sysctl__net_inet_icmp_drop_redirect
-    ,&sysctl__net_inet_icmp_log_redirect
-    ,&sysctl__net_inet_ip_accept_sourceroute
-#if IPCTL_DEFMTU
-    ,&sysctl__net_inet_ip_mtu
-#endif
-    ,&sysctl__net_inet_ip_ttl
-    ,&sysctl__net_inet_ip_fastforwarding
-    ,&sysctl__net_inet_ip_forwarding
-    ,&sysctl__net_inet_ip_intr_queue_drops
-    ,&sysctl__net_inet_ip_intr_queue_maxlen
-    ,&sysctl__net_inet_ip_rtexpire
-    ,&sysctl__net_inet_ip_rtmaxcache
-    ,&sysctl__net_inet_ip_rtminexpire
-    ,&sysctl__net_inet_ip_redirect
-    ,&sysctl__net_inet_ip_sourceroute
-    ,&sysctl__net_inet_ip_subnets_are_local
-    ,&sysctl__net_inet_ip_keepfaith
-    ,&sysctl__net_inet_ip_maxfragpackets
-    ,&sysctl__net_inet_ip_maxfragsperpacket
-    ,&sysctl__net_inet_ip_maxfrags
-    ,&sysctl__net_inet_ip_check_interface
-    ,&sysctl__net_inet_ip_check_route_selfref
-    ,&sysctl__net_inet_ip_use_route_genid
-#if NGIF > 0
-    ,&sysctl__net_inet_ip_gifttl
-#endif
-#if DUMMYNET
-    ,&sysctl__net_inet_ip_dummynet_hash_size
-    ,&sysctl__net_inet_ip_dummynet_curr_time
-    ,&sysctl__net_inet_ip_dummynet_ready_heap
-    ,&sysctl__net_inet_ip_dummynet_extract_heap
-    ,&sysctl__net_inet_ip_dummynet_searches
-    ,&sysctl__net_inet_ip_dummynet_search_steps
-    ,&sysctl__net_inet_ip_dummynet_expire
-    ,&sysctl__net_inet_ip_dummynet_max_chain_len
-    ,&sysctl__net_inet_ip_dummynet_red_lookup_depth
-    ,&sysctl__net_inet_ip_dummynet_red_avg_pkt_size
-    ,&sysctl__net_inet_ip_dummynet_red_max_pkt_size
-    ,&sysctl__net_inet_ip_dummynet
-#endif
-
-#if IPFIREWALL && !IPFIREWALL_KEXT
-    ,&sysctl__net_inet_ip_fw_enable
-    ,&sysctl__net_inet_ip_fw_debug
-    ,&sysctl__net_inet_ip_fw_verbose
-    ,&sysctl__net_inet_ip_fw_verbose_limit
-    ,&sysctl__net_inet_ip_fw_one_pass
-    ,&sysctl__net_inet_ip_fw_autoinc_step
-    ,&sysctl__net_inet_ip_fw_dyn_buckets
-    ,&sysctl__net_inet_ip_fw_curr_dyn_buckets
-    ,&sysctl__net_inet_ip_fw_dyn_count
-    ,&sysctl__net_inet_ip_fw_dyn_max
-    ,&sysctl__net_inet_ip_fw_static_count
-    ,&sysctl__net_inet_ip_fw_dyn_ack_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_syn_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_fin_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_rst_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_udp_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_short_lifetime
-    ,&sysctl__net_inet_ip_fw_dyn_keepalive
-    ,&sysctl__net_inet_ip_fw
-#endif
-    ,&sysctl__net_inet_ip_linklocal
-    ,&sysctl__net_inet_ip_linklocal_stat
-    ,&sysctl__net_inet_ip_linklocal_in
-    ,&sysctl__net_inet_ip_linklocal_in_allowbadttl
-    ,&sysctl__net_inet_ip_maxchainsent
-    ,&sysctl__net_inet_raw_maxdgram
-    ,&sysctl__net_inet_raw_recvspace
-    ,&sysctl__net_inet_tcp_always_keepalive
-    ,&sysctl__net_inet_tcp_delayed_ack
-    ,&sysctl__net_inet_tcp_log_in_vain
-    ,&sysctl__net_inet_tcp_pcbcount
-    ,&sysctl__net_inet_tcp_rfc1323
-    ,&sysctl__net_inet_tcp_rfc1644
-    ,&sysctl__net_inet_tcp_keepidle
-    ,&sysctl__net_inet_tcp_keepinit
-    ,&sysctl__net_inet_tcp_keepintvl
-    ,&sysctl__net_inet_tcp_mssdflt
-    ,&sysctl__net_inet_tcp_minmss
-    ,&sysctl__net_inet_tcp_minmssoverload
-    ,&sysctl__net_inet_tcp_recvspace
-    ,&sysctl__net_inet_tcp_sendspace
-    ,&sysctl__net_inet_tcp_slowlink_wsize
-    ,&sysctl__net_inet_tcp_blackhole
-    ,&sysctl__net_inet_tcp_tcp_lq_overflow
-    ,&sysctl__net_inet_tcp_path_mtu_discovery
-    ,&sysctl__net_inet_tcp_slowstart_flightsize
-    ,&sysctl__net_inet_tcp_local_slowstart_flightsize
-    ,&sysctl__net_inet_tcp_newreno
-    ,&sysctl__net_inet_tcp_packetchain
-    ,&sysctl__net_inet_tcp_tcbhashsize
-    ,&sysctl__net_inet_tcp_do_tcpdrain
-    ,&sysctl__net_inet_tcp_icmp_may_rst
-    ,&sysctl__net_inet_tcp_strict_rfc1948
-    ,&sysctl__net_inet_tcp_delacktime
-    ,&sysctl__net_inet_tcp_isn_reseed_interval
-    ,&sysctl__net_inet_tcp_msl
-    ,&sysctl__net_inet_tcp_sack
-    ,&sysctl__net_inet_tcp_sack_maxholes
-    ,&sysctl__net_inet_tcp_sack_globalmaxholes
-    ,&sysctl__net_inet_tcp_sack_globalholes
-#if TCP_DROP_SYNFIN
-    ,&sysctl__net_inet_tcp_drop_synfin
-#endif
-#if TCPDEBUG
-    ,&sysctl__net_inet_tcp_tcpconsdebug
-#endif
-    ,&sysctl__net_inet_tcp_sockthreshold
-    ,&sysctl__net_inet_tcp_reass_maxsegments
-    ,&sysctl__net_inet_tcp_reass_cursegments
-    ,&sysctl__net_inet_tcp_reass_overflows
-    ,&sysctl__net_inet_tcp_reass
-
-    ,&sysctl__net_inet_udp_log_in_vain 
-    ,&sysctl__net_inet_udp_checksum
-    ,&sysctl__net_inet_udp_maxdgram
-    ,&sysctl__net_inet_udp_recvspace
-    ,&sysctl__net_inet_udp_blackhole
-    ,&sysctl__net_inet_udp_pcbcount
-
-#if NETAT
-    ,&sysctl__net_appletalk_debug
-    ,&sysctl__net_appletalk_routermix
-    ,&sysctl__net_appletalk_ddpstats
-#endif /* NETAT */
-
-#if BRIDGE
-    ,&sysctl__net_link_ether_bdgfwc
-    ,&sysctl__net_link_ether_bdgfwt
-    ,&sysctl__net_link_ether_bdginc
-    ,&sysctl__net_link_ether_bdgint
-    ,&sysctl__net_link_ether_bridge_ipfw
-    ,&sysctl__net_link_ethe_bdgstats
-    ,&sysctl__net_link_ether_bridge
-#endif
-
-    ,&sysctl__net_link_ether_inet_host_down_time
-    ,&sysctl__net_link_ether_inet_max_age
-    ,&sysctl__net_link_ether_inet_maxtries
-    ,&sysctl__net_link_ether_inet_proxyall
-    ,&sysctl__net_link_ether_inet_prune_intvl
-    ,&sysctl__net_link_ether_inet_useloopback
-    ,&sysctl__net_link_ether_inet_log_arp_warnings
-    ,&sysctl__net_link_ether_inet_apple_hwcksum_tx
-    ,&sysctl__net_link_ether_inet_apple_hwcksum_rx
-#if NETMIBS
-    ,&sysctl__net_link_generic_system_ifcount
-    ,&sysctl__net_link_generic
-    ,&sysctl__net_link_generic_ifdata
-    ,&sysctl__net_link_generic_ifalldata
-    ,&sysctl__net_link_generic_system
-#endif
-
-#if 0
-#if VLAN
-    ,&sysctl__net_link_vlan_link_proto
-    ,&sysctl__net_link_vlan
-    ,&sysctl__net_link_vlan_link
-#endif
-#endif 0
-
-    ,&sysctl__net_local_inflight
-    ,&sysctl__net_local_dgram_maxdgram
-    ,&sysctl__net_local_dgram_recvspace
-    ,&sysctl__net_local_stream_recvspace
-    ,&sysctl__net_local_stream_sendspace
-#if 0
-    ,&sysctl__vfs_nfs_nfs_privport
-    ,&sysctl__vfs_nfs_async
-    ,&sysctl__vfs_nfs_debug
-    ,&sysctl__vfs_nfs_defect
-    ,&sysctl__vfs_nfs_diskless_valid
-    ,&sysctl__vfs_nfs_gatherdelay
-    ,&sysctl__vfs_nfs_gatherdelay_v3
-    ,&sysctl__vfs_nfs
-    ,&sysctl__vfs_nfs_diskless_rootaddr
-    ,&sysctl__vfs_nfs_diskless_swapaddr
-    ,&sysctl__vfs_nfs_diskless_rootpath
-    ,&sysctl__vfs_nfs_diskless_swappath
-    ,&sysctl__vfs_nfs_nfsstats
-#endif
-    ,&sysctl__vfs_generic
-    ,&sysctl__vfs_generic_vfsidlist
-    ,&sysctl__vfs_generic_ctlbyfsid
-    ,&sysctl__vfs_generic_noremotehang
-#if NFSCLIENT
-    ,&sysctl__vfs_generic_nfs
-    ,&sysctl__vfs_generic_nfs_client
-    ,&sysctl__vfs_generic_nfs_client_initialdowndelay
-    ,&sysctl__vfs_generic_nfs_client_nextdowndelay
-#endif
-    ,&sysctl__kern_ipc
-    ,&sysctl__kern_sysv
-    ,&sysctl__net_inet
-#if NETAT
-    ,&sysctl__net_appletalk
-#endif /* NETAT */
-    ,&sysctl__net_link
-    ,&sysctl__net_local
-    ,&sysctl__net_routetable
-#if IPDIVERT
-    ,&sysctl__net_inet_div
-#endif
-    ,&sysctl__net_inet_icmp
-    ,&sysctl__net_inet_igmp
-    ,&sysctl__net_inet_ip
-    ,&sysctl__net_inet_raw
-    ,&sysctl__net_inet_tcp
-    ,&sysctl__net_inet_udp
-    ,&sysctl__net_inet_ip_portrange
-    ,&sysctl__net_link_ether
-    ,&sysctl__net_link_ether_inet
-    ,&sysctl__net_local_dgram
-    ,&sysctl__net_local_stream
-    ,&sysctl__sysctl_name
-    ,&sysctl__sysctl_next
-    ,&sysctl__sysctl_oidfmt
-    ,&sysctl__net_inet_ip_portrange_first
-    ,&sysctl__net_inet_ip_portrange_hifirst
-    ,&sysctl__net_inet_ip_portrange_hilast
-    ,&sysctl__net_inet_ip_portrange_last
-    ,&sysctl__net_inet_ip_portrange_lowfirst
-    ,&sysctl__net_inet_ip_portrange_lowlast
-    ,&sysctl__net_inet_raw_pcblist
-    ,&sysctl__net_inet_tcp_pcblist
-    ,&sysctl__net_inet_udp_pcblist
-    ,&sysctl__net_local_dgram_pcblist
-    ,&sysctl__net_local_stream_pcblist
-    ,&sysctl__sysctl_debug
-    ,&sysctl__sysctl_name2oid
-    ,&sysctl__net_inet_icmp_stats
-    ,&sysctl__net_inet_igmp_stats
-    ,&sysctl__net_inet_ip_stats
-    ,&sysctl__net_inet_tcp_stats
-    ,&sysctl__net_inet_udp_stats
-#if INET6
-    ,&sysctl__net_inet6
-    ,&sysctl__net_inet6_ip6
-    ,&sysctl__net_inet6_icmp6
-    ,&sysctl__net_inet6_ip6_stats
-    ,&sysctl__net_inet6_ip6_forwarding
-    ,&sysctl__net_inet6_ip6_redirect
-    ,&sysctl__net_inet6_ip6_hlim
-    ,&sysctl__net_inet6_ip6_maxfragpackets
-    ,&sysctl__net_inet6_ip6_maxfrags
-    ,&sysctl__net_inet6_ip6_accept_rtadv
-    ,&sysctl__net_inet6_ip6_keepfaith
-    ,&sysctl__net_inet6_ip6_log_interval
-    ,&sysctl__net_inet6_ip6_hdrnestlimit
-    ,&sysctl__net_inet6_ip6_dad_count
-    ,&sysctl__net_inet6_ip6_auto_flowlabel
-    ,&sysctl__net_inet6_ip6_defmcasthlim
-    ,&sysctl__net_inet6_ip6_gifhlim
-    ,&sysctl__net_inet6_ip6_kame_version
-    ,&sysctl__net_inet6_ip6_use_deprecated
-    ,&sysctl__net_inet6_ip6_rr_prune
-    ,&sysctl__net_inet6_ip6_use_tempaddr
-    ,&sysctl__net_inet6_ip6_v6only
-    ,&sysctl__net_inet6_ip6_auto_linklocal
-    ,&sysctl__net_inet6_ip6_rip6stats
-    ,&sysctl__net_inet6_ip6_mrt6stat
-    ,&sysctl__net_inet6_ip6_rtexpire
-    ,&sysctl__net_inet6_ip6_rtminexpire
-    ,&sysctl__net_inet6_ip6_rtmaxcache
-    ,&sysctl__net_inet6_ip6_temppltime
-    ,&sysctl__net_inet6_ip6_tempvltime
-   ,&sysctl__net_inet6_icmp6_rediraccept
-   ,&sysctl__net_inet6_icmp6_redirtimeout
-   ,&sysctl__net_inet6_icmp6_nd6_prune
-   ,&sysctl__net_inet6_icmp6_nd6_delay
-   ,&sysctl__net_inet6_icmp6_nd6_umaxtries
-   ,&sysctl__net_inet6_icmp6_nd6_mmaxtries
-   ,&sysctl__net_inet6_icmp6_nd6_useloopback
-   ,&sysctl__net_inet6_icmp6_nodeinfo
-   ,&sysctl__net_inet6_icmp6_stats
-   ,&sysctl__net_inet6_icmp6_errppslimit
-   ,&sysctl__net_inet6_icmp6_nd6_maxnudhint
-   ,&sysctl__net_inet6_icmp6_nd6_debug
-    ,&sysctl__net_inet_tcp_v6mssdflt
-#if IPV6FIREWALL
-   ,&sysctl__net_inet6_ip6_fw
-   ,&sysctl__net_inet6_ip6_fw_debug
-   ,&sysctl__net_inet6_ip6_fw_verbose
-   ,&sysctl__net_inet6_ip6_fw_verbose_limit
-#endif
-#if IPSEC
-   ,&sysctl__net_inet6_ipsec6
-   ,&sysctl__net_inet6_ipsec6_stats
-   ,&sysctl__net_inet6_ipsec6_def_policy
-   ,&sysctl__net_inet6_ipsec6_esp_trans_deflev
-   ,&sysctl__net_inet6_ipsec6_esp_net_deflev
-   ,&sysctl__net_inet6_ipsec6_ah_trans_deflev
-   ,&sysctl__net_inet6_ipsec6_ah_net_deflev
-   ,&sysctl__net_inet6_ipsec6_ecn
-   ,&sysctl__net_inet6_ipsec6_debug
-   ,&sysctl__net_inet6_ipsec6_esp_randpad
-#endif
-#endif
-#if IPSEC
-   ,&sysctl__net_key
-   ,&sysctl__net_key_debug
-   ,&sysctl__net_key_prefered_oldsa
-   ,&sysctl__net_key_spi_trycnt
-   ,&sysctl__net_key_spi_minval
-   ,&sysctl__net_key_spi_maxval
-   ,&sysctl__net_key_int_random
-   ,&sysctl__net_key_larval_lifetime
-   ,&sysctl__net_key_blockacq_count
-   ,&sysctl__net_key_blockacq_lifetime
-   ,&sysctl__net_key_esp_keymin
-   ,&sysctl__net_key_esp_auth
-   ,&sysctl__net_key_ah_keymin
-   ,&sysctl__net_key_natt_keepalive_interval
-   ,&sysctl__net_key_pfkeystat
-   ,&sysctl__net_inet_ipsec
-   ,&sysctl__net_inet_ipsec_stats
-   ,&sysctl__net_inet_ipsec_def_policy
-   ,&sysctl__net_inet_ipsec_esp_trans_deflev
-   ,&sysctl__net_inet_ipsec_esp_net_deflev
-   ,&sysctl__net_inet_ipsec_ah_trans_deflev
-   ,&sysctl__net_inet_ipsec_ah_net_deflev
-   ,&sysctl__net_inet_ipsec_ah_cleartos
-   ,&sysctl__net_inet_ipsec_ah_offsetmask
-   ,&sysctl__net_inet_ipsec_dfbit
-   ,&sysctl__net_inet_ipsec_ecn
-   ,&sysctl__net_inet_ipsec_debug
-   ,&sysctl__net_inet_ipsec_esp_randpad
-   ,&sysctl__net_inet_ipsec_bypass
-   ,&sysctl__net_inet_ipsec_esp_port
-#endif
-
-    ,&sysctl__vm_shared_region_trace_level
-
-    ,(struct sysctl_oid *) 0
-};
-
diff --git a/bsd/kern/sysv_ipc.c b/bsd/kern/sysv_ipc.c
index 0a13e17fd..95c23d418 100644
--- a/bsd/kern/sysv_ipc.c
+++ b/bsd/kern/sysv_ipc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: sysv_ipc.c,v 1.7 1994/06/29 06:33:11 cgd Exp $	*/
 
@@ -65,6 +71,11 @@
  * XXX: proc->p_acflag to suser()
  */
 
+/*
+ * Returns:	0			Success
+ *		EPERM
+ *		EACCES
+ */
 int
 ipcperm(kauth_cred_t cred, struct ipc_perm *perm, int mode)
 {
@@ -81,12 +92,14 @@ ipcperm(kauth_cred_t cred, struct ipc_perm *perm, int mode)
 		/* Check for group match. */
 		mode >>= 3;
 		if ((kauth_cred_ismember_gid(cred, perm->gid, &is_member) || !is_member) &&
-		    (kauth_cred_ismember_gid(cred, perm->cgid, &is_member) || !is_member))
+		    (kauth_cred_ismember_gid(cred, perm->cgid, &is_member) || !is_member)) {
 			/* Check for `other' match. */
 			mode >>= 3;
 	}
+	}
 
 	if (mode & IPC_M)
 		return (0);
+
 	return ((mode & perm->mode) == mode ? 0 : EACCES);
 }
diff --git a/bsd/kern/sysv_msg.c b/bsd/kern/sysv_msg.c
index 9d34b4d7c..3b85639bd 100644
--- a/bsd/kern/sysv_msg.c
+++ b/bsd/kern/sysv_msg.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Implementation of SVID messages
@@ -37,6 +43,12 @@
  *
  * This software is provided ``AS IS'' without any warranties of any kind.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -55,11 +67,20 @@
 #include <sys/sysproto.h>
 #include <sys/ipcs.h>
 
-static void msginit(void *);
+#if SYSV_MSG
+
+static int msginit(void *);
 
 #define MSG_DEBUG
 #undef MSG_DEBUG_OK
 
+/* Uncomment this line to see MAC debugging output. */
+/* #define	MAC_DEBUG */
+#if CONFIG_MACF_DEBUG
+#define	MPRINTF(a)	printf(a)
+#else
+#define	MPRINTF(a)
+#endif
 static void msg_freehdr(struct msg *msghdr);
 
 typedef int     sy_call_t(struct proc *, void *, int *);
@@ -76,7 +97,7 @@ static struct msg	*free_msghdrs;	/* list of free msg headers */
 char			*msgpool;	/* MSGMAX byte long msg buffer pool */
 struct msgmap		*msgmaps;	/* MSGSEG msgmap structures */
 struct msg		*msghdrs;	/* MSGTQL msg headers */
-struct user_msqid_ds	*msqids;	/* MSGMNI user_msqid_ds struct's */
+struct msqid_kernel	*msqids;	/* MSGMNI msqid_kernel structs (wrapping user_msqid_ds structs) */
 
 static lck_grp_t       *sysv_msg_subsys_lck_grp;
 static lck_grp_attr_t  *sysv_msg_subsys_lck_grp_attr;
@@ -90,6 +111,12 @@ void sysv_msg_lock_init(void);
 
 
 #ifdef __APPLE_API_PRIVATE
+	int	msgmax,		/* max chars in a message */
+		msgmni,		/* max message queue identifiers */
+		msgmnb,		/* max chars in a queue */
+		msgtql,		/* max messages in system */
+		msgssz,		/* size of a message segment (see notes above) */
+		msgseg;		/* number of message segments */
 struct msginfo msginfo = {
 		MSGMAX,		/* = (MSGSSZ*MSGSEG) : max chars in a message */
 		MSGMNI,		/* = 40 : max message queue identifiers */
@@ -157,7 +184,7 @@ msqid_ds_32to64(struct msqid_ds *in, struct user_msqid_ds *out)
 }
 
 /* This routine assumes the system is locked prior to calling this routine */
-void 
+static int
 msginit(__unused void *dummy)
 {
 	static int initted = 0;
@@ -165,43 +192,63 @@ msginit(__unused void *dummy)
 
 	/* Lazy initialization on first system call; we don't have SYSINIT(). */
 	if (initted)
-		return;
-	initted = 1;
-
-	msgpool = (char *)_MALLOC(msginfo.msgmax, M_SHM, M_WAITOK);
-	MALLOC(msgmaps, struct msgmap *,
-			sizeof(struct msgmap) * msginfo.msgseg, 
-			M_SHM, M_WAITOK);
-	MALLOC(msghdrs, struct msg *,
-			sizeof(struct msg) * msginfo.msgtql, 
-			M_SHM, M_WAITOK);
-	MALLOC(msqids, struct user_msqid_ds *,
-			sizeof(struct user_msqid_ds) * msginfo.msgmni, 
-			M_SHM, M_WAITOK);
+		return (initted);
 
 	/*
 	 * msginfo.msgssz should be a power of two for efficiency reasons.
 	 * It is also pretty silly if msginfo.msgssz is less than 8
 	 * or greater than about 256 so ...
 	 */
-
 	i = 8;
 	while (i < 1024 && i != msginfo.msgssz)
 		i <<= 1;
     	if (i != msginfo.msgssz) {
-		printf("msginfo.msgssz=%d (0x%x)\n", msginfo.msgssz,
-		    msginfo.msgssz);
-		panic("msginfo.msgssz not a small power of 2");
+		printf("msginfo.msgssz=%d (0x%x) not a small power of 2; resetting to %d\n", msginfo.msgssz, msginfo.msgssz, MSGSSZ);
+		msginfo.msgssz = MSGSSZ;
 	}
 
 	if (msginfo.msgseg > 32767) {
-		printf("msginfo.msgseg=%d\n", msginfo.msgseg);
-		panic("msginfo.msgseg > 32767");
+		printf("msginfo.msgseg=%d (> 32767); resetting to %d\n", msginfo.msgseg, MSGSEG);
+		msginfo.msgseg = MSGSEG;
 	}
 
-	if (msgmaps == NULL)
-		panic("msgmaps is NULL");
 
+	/*
+	 * Allocate memory for message pool, maps, headers, and queue IDs;
+	 * if this fails, fail safely and leave it uninitialized (related
+	 * system calls will fail).
+	 */
+	msgpool = (char *)_MALLOC(msginfo.msgmax, M_SHM, M_WAITOK);
+	if (msgpool == NULL) {
+		printf("msginit: can't allocate msgpool");
+		goto bad;
+	}
+	MALLOC(msgmaps, struct msgmap *,
+			sizeof(struct msgmap) * msginfo.msgseg, 
+			M_SHM, M_WAITOK);
+	if (msgmaps == NULL) {
+		printf("msginit: can't allocate msgmaps");
+		goto bad;
+	}
+
+	MALLOC(msghdrs, struct msg *,
+			sizeof(struct msg) * msginfo.msgtql, 
+			M_SHM, M_WAITOK);
+	if (msghdrs == NULL) {
+		printf("msginit: can't allocate msghdrs");
+		goto bad;
+	}
+
+	MALLOC(msqids, struct msqid_kernel *,
+			sizeof(struct user_msqid_ds) * msginfo.msgmni, 
+			M_SHM, M_WAITOK);
+	if (msqids == NULL) {
+		printf("msginit: can't allocate msqids");
+		goto bad;
+	}
+
+
+	/* init msgmaps */
 	for (i = 0; i < msginfo.msgseg; i++) {
 		if (i > 0)
 			msgmaps[i-1].next = i;
@@ -210,24 +257,42 @@ msginit(__unused void *dummy)
 	free_msgmaps = 0;
 	nfree_msgmaps = msginfo.msgseg;
 
-	if (msghdrs == NULL)
-		panic("msghdrs is NULL");
 
+	/* init msghdrs */
 	for (i = 0; i < msginfo.msgtql; i++) {
 		msghdrs[i].msg_type = 0;
 		if (i > 0)
 			msghdrs[i-1].msg_next = &msghdrs[i];
 		msghdrs[i].msg_next = NULL;
+#if CONFIG_MACF
+		mac_sysvmsg_label_init(&msghdrs[i]);
+#endif
     	}
 	free_msghdrs = &msghdrs[0];
 
-	if (msqids == NULL)
-		panic("msqids is NULL");
-
+	/* init msqids */
 	for (i = 0; i < msginfo.msgmni; i++) {
-		msqids[i].msg_qbytes = 0;	/* implies entry is available */
-		msqids[i].msg_perm.seq = 0;	/* reset to a known value */
+		msqids[i].u.msg_qbytes = 0;	/* implies entry is available */
+		msqids[i].u.msg_perm._seq = 0;	/* reset to a known value */
+		msqids[i].u.msg_perm.mode = 0;
+#if CONFIG_MACF
+		mac_sysvmsq_label_init(&msqids[i]);
+#endif
+	}
+
+	initted = 1;
+bad:
+	if (!initted) {
+		if (msgpool != NULL)
+			_FREE(msgpool, M_SHM);
+		if (msgmaps != NULL)
+			FREE(msgmaps, M_SHM);
+		if (msghdrs != NULL)
+			FREE(msghdrs, M_SHM);
+		if (msqids != NULL)
+			FREE(msqids, M_SHM);
 	}
+	return (initted);
 }
 
 /*
@@ -263,6 +328,14 @@ msg_freehdr(struct msg *msghdr)
 		panic("msghdr->msg_spot != -1");
 	msghdr->msg_next = free_msghdrs;
 	free_msghdrs = msghdr;
+#if CONFIG_MACF
+	mac_sysvmsg_label_recycle(msghdr);
+#endif
+	/*
+	 * Notify waiters that there are free message headers and segments
+	 * now available.
+	 */
+	wakeup((caddr_t)&free_msghdrs);
 }
 
 int
@@ -273,12 +346,15 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 	kauth_cred_t cred = kauth_cred_get();
 	int rval, eval;
 	struct user_msqid_ds msqbuf;
-	struct user_msqid_ds *msqptr;
+	struct msqid_kernel *msqptr;
 	struct user_msqid_ds umsds;
 
 	SYSV_MSG_SUBSYS_LOCK();
 
-	msginit( 0);
+	if (!msginit(0)) {
+		eval =  ENOMEM;
+		goto msgctlout;
+	}
 
 #ifdef MSG_DEBUG_OK
 	printf("call to msgctl(%d, %d, 0x%qx)\n", msqid, cmd, uap->buf);
@@ -299,20 +375,25 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 
 	msqptr = &msqids[msqid];
 
-	if (msqptr->msg_qbytes == 0) {
+	if (msqptr->u.msg_qbytes == 0) {
 #ifdef MSG_DEBUG_OK
 		printf("no such msqid\n");
 #endif
 		eval = EINVAL;
 		goto msgctlout;
 	}
-	if (msqptr->msg_perm.seq != IPCID_TO_SEQ(uap->msqid)) {
+	if (msqptr->u.msg_perm._seq != IPCID_TO_SEQ(uap->msqid)) {
 #ifdef MSG_DEBUG_OK
 		printf("wrong sequence number\n");
 #endif
 		eval = EINVAL;
 		goto msgctlout;
 	}
+#if CONFIG_MACF
+	eval = mac_sysvmsq_check_msqctl(kauth_cred_get(), msqptr, cmd);
+	if (eval) 
+		goto msgctlout;
+#endif
 
 	eval = 0;
 	rval = 0;
@@ -322,28 +403,47 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 	case IPC_RMID:
 	{
 		struct msg *msghdr;
-		if ((eval = ipcperm(cred, &msqptr->msg_perm, IPC_M)))
+		if ((eval = ipcperm(cred, &msqptr->u.msg_perm, IPC_M)))
 			goto msgctlout;
-
+#if CONFIG_MACF
+		/*
+		 * Check that the thread has MAC access permissions to
+		 * individual msghdrs.  Note: We need to do this in a
+		 * separate loop because the actual loop alters the
+		 * msq/msghdr info as it progresses, and there is no going
+		 * back if half the way through we discover that the
+		 * thread cannot free a certain msghdr.  The msq will get
+		 * into an inconsistent state.
+		 */
+		for (msghdr = msqptr->u.msg_first; msghdr != NULL;
+		    msghdr = msghdr->msg_next) {
+			eval = mac_sysvmsq_check_msgrmid(kauth_cred_get(), msghdr);
+			if (eval) 
+				goto msgctlout;
+		}
+#endif
 		/* Free the message headers */
-		msghdr = msqptr->msg_first;
+		msghdr = msqptr->u.msg_first;
 		while (msghdr != NULL) {
 			struct msg *msghdr_tmp;
 
 			/* Free the segments of each message */
-			msqptr->msg_cbytes -= msghdr->msg_ts;
-			msqptr->msg_qnum--;
+			msqptr->u.msg_cbytes -= msghdr->msg_ts;
+			msqptr->u.msg_qnum--;
 			msghdr_tmp = msghdr;
 			msghdr = msghdr->msg_next;
 			msg_freehdr(msghdr_tmp);
 		}
 
-		if (msqptr->msg_cbytes != 0)
+		if (msqptr->u.msg_cbytes != 0)
 			panic("msg_cbytes is messed up");
-		if (msqptr->msg_qnum != 0)
+		if (msqptr->u.msg_qnum != 0)
 			panic("msg_qnum is messed up");
 
-		msqptr->msg_qbytes = 0;	/* Mark it as free */
+		msqptr->u.msg_qbytes = 0;	/* Mark it as free */
+#if CONFIG_MACF
+		mac_sysvmsq_label_recycle(msqptr);
+#endif
 
 		wakeup((caddr_t)msqptr);
 	}
@@ -351,7 +451,7 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 		break;
 
 	case IPC_SET:
-		if ((eval = ipcperm(cred, &msqptr->msg_perm, IPC_M)))
+		if ((eval = ipcperm(cred, &msqptr->u.msg_perm, IPC_M)))
 			goto msgctlout;
 
 		SYSV_MSG_SUBSYS_UNLOCK();
@@ -368,7 +468,7 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 
 		SYSV_MSG_SUBSYS_LOCK();
 
-		if (msqbuf.msg_qbytes > msqptr->msg_qbytes) {
+		if (msqbuf.msg_qbytes > msqptr->u.msg_qbytes) {
 			eval = suser(cred, &p->p_acflag);
 			if (eval)
 				goto msgctlout;
@@ -390,16 +490,16 @@ msgctl(struct proc *p, struct msgctl_args *uap, register_t *retval)
 			eval = EINVAL;
 			goto msgctlout;
 		}
-		msqptr->msg_perm.uid = msqbuf.msg_perm.uid;	/* change the owner */
-		msqptr->msg_perm.gid = msqbuf.msg_perm.gid;	/* change the owner */
-		msqptr->msg_perm.mode = (msqptr->msg_perm.mode & ~0777) |
+		msqptr->u.msg_perm.uid = msqbuf.msg_perm.uid;	/* change the owner */
+		msqptr->u.msg_perm.gid = msqbuf.msg_perm.gid;	/* change the owner */
+		msqptr->u.msg_perm.mode = (msqptr->u.msg_perm.mode & ~0777) |
 		    (msqbuf.msg_perm.mode & 0777);
-		msqptr->msg_qbytes = msqbuf.msg_qbytes;
-		msqptr->msg_ctime = sysv_msgtime();
+		msqptr->u.msg_qbytes = msqbuf.msg_qbytes;
+		msqptr->u.msg_ctime = sysv_msgtime();
 		break;
 
 	case IPC_STAT:
-		if ((eval = ipcperm(cred, &msqptr->msg_perm, IPC_R))) {
+		if ((eval = ipcperm(cred, &msqptr->u.msg_perm, IPC_R))) {
 #ifdef MSG_DEBUG_OK
 			printf("requester doesn't have read access\n");
 #endif
@@ -441,10 +541,14 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 	int key = uap->key;
 	int msgflg = uap->msgflg;
 	kauth_cred_t cred = kauth_cred_get();
-	struct user_msqid_ds *msqptr = NULL;
+	struct msqid_kernel *msqptr = NULL;
 
 	SYSV_MSG_SUBSYS_LOCK();
-	msginit( 0);
+
+	if (!msginit(0)) {
+		eval =  ENOMEM;
+		goto msggetout;
+	}
 
 #ifdef MSG_DEBUG_OK
 	printf("msgget(0x%x, 0%o)\n", key, msgflg);
@@ -453,8 +557,8 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 	if (key != IPC_PRIVATE) {
 		for (msqid = 0; msqid < msginfo.msgmni; msqid++) {
 			msqptr = &msqids[msqid];
-			if (msqptr->msg_qbytes != 0 &&
-			    msqptr->msg_perm.key == key)
+			if (msqptr->u.msg_qbytes != 0 &&
+			    msqptr->u.msg_perm._key == key)
 				break;
 		}
 		if (msqid < msginfo.msgmni) {
@@ -468,13 +572,18 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 				eval = EEXIST;
 				goto msggetout;
 			}
-			if ((eval = ipcperm(cred, &msqptr->msg_perm, msgflg & 0700 ))) {
+			if ((eval = ipcperm(cred, &msqptr->u.msg_perm, msgflg & 0700 ))) {
 #ifdef MSG_DEBUG_OK
 				printf("requester doesn't have 0%o access\n",
 				    msgflg & 0700);
 #endif
 				goto msggetout;
 			}
+#if CONFIG_MACF
+			eval = mac_sysvmsq_check_msqget(cred, msqptr);
+			if (eval) 
+				goto msggetout;
+#endif
 			goto found;
 		}
 	}
@@ -491,8 +600,8 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 			 * can't re-use the entry until they release it.
 			 */
 			msqptr = &msqids[msqid];
-			if (msqptr->msg_qbytes == 0 &&
-			    (msqptr->msg_perm.mode & MSG_LOCKED) == 0)
+			if (msqptr->u.msg_qbytes == 0 &&
+			    (msqptr->u.msg_perm.mode & MSG_LOCKED) == 0)
 				break;
 		}
 		if (msqid == msginfo.msgmni) {
@@ -505,24 +614,27 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 #ifdef MSG_DEBUG_OK
 		printf("msqid %d is available\n", msqid);
 #endif
-		msqptr->msg_perm.key = key;
-		msqptr->msg_perm.cuid = kauth_cred_getuid(cred);
-		msqptr->msg_perm.uid = kauth_cred_getuid(cred);
-		msqptr->msg_perm.cgid = cred->cr_gid;
-		msqptr->msg_perm.gid = cred->cr_gid;
-		msqptr->msg_perm.mode = (msgflg & 0777);
+		msqptr->u.msg_perm._key = key;
+		msqptr->u.msg_perm.cuid = kauth_cred_getuid(cred);
+		msqptr->u.msg_perm.uid = kauth_cred_getuid(cred);
+		msqptr->u.msg_perm.cgid = cred->cr_gid;
+		msqptr->u.msg_perm.gid = cred->cr_gid;
+		msqptr->u.msg_perm.mode = (msgflg & 0777);
 		/* Make sure that the returned msqid is unique */
-		msqptr->msg_perm.seq++;
-		msqptr->msg_first = NULL;
-		msqptr->msg_last = NULL;
-		msqptr->msg_cbytes = 0;
-		msqptr->msg_qnum = 0;
-		msqptr->msg_qbytes = msginfo.msgmnb;
-		msqptr->msg_lspid = 0;
-		msqptr->msg_lrpid = 0;
-		msqptr->msg_stime = 0;
-		msqptr->msg_rtime = 0;
-		msqptr->msg_ctime = sysv_msgtime();
+		msqptr->u.msg_perm._seq++;
+		msqptr->u.msg_first = NULL;
+		msqptr->u.msg_last = NULL;
+		msqptr->u.msg_cbytes = 0;
+		msqptr->u.msg_qnum = 0;
+		msqptr->u.msg_qbytes = msginfo.msgmnb;
+		msqptr->u.msg_lspid = 0;
+		msqptr->u.msg_lrpid = 0;
+		msqptr->u.msg_stime = 0;
+		msqptr->u.msg_rtime = 0;
+		msqptr->u.msg_ctime = sysv_msgtime();
+#if CONFIG_MACF
+		mac_sysvmsq_label_associate(cred, msqptr);
+#endif
 	} else {
 #ifdef MSG_DEBUG_OK
 		printf("didn't find it and wasn't asked to create it\n");
@@ -533,7 +645,7 @@ msgget(__unused struct proc *p, struct msgget_args *uap, register_t *retval)
 
 found:
 	/* Construct the unique msqid */
-	*retval = IXSEQ_TO_IPCID(msqid, msqptr->msg_perm);
+	*retval = IXSEQ_TO_IPCID(msqid, msqptr->u.msg_perm);
 	AUDIT_ARG(svipc_id, *retval);
 	eval = 0;
 msggetout:
@@ -544,20 +656,31 @@ msggetout:
 
 int
 msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(msgsnd_nocancel(p, (struct msgsnd_nocancel_args *)uap, retval));
+}
+
+int
+msgsnd_nocancel(struct proc *p, struct msgsnd_nocancel_args *uap, register_t *retval)
 {
 	int msqid = uap->msqid;
 	user_addr_t user_msgp = uap->msgp;
 	size_t msgsz = (size_t)uap->msgsz;	/* limit to 4G */
 	int msgflg = uap->msgflg;
 	int segs_needed, eval;
-	struct user_msqid_ds *msqptr;
+	struct msqid_kernel *msqptr;
 	struct msg *msghdr;
 	short next;
 	user_long_t msgtype;
 
 
 	SYSV_MSG_SUBSYS_LOCK();
-	msginit( 0);
+
+	if (!msginit(0)) {
+		eval =  ENOMEM;
+		goto msgsndout;
+	}
 
 #ifdef MSG_DEBUG_OK
 	printf("call to msgsnd(%d, 0x%qx, %d, %d)\n", msqid, user_msgp, msgsz,
@@ -577,14 +700,14 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 	}
 
 	msqptr = &msqids[msqid];
-	if (msqptr->msg_qbytes == 0) {
+	if (msqptr->u.msg_qbytes == 0) {
 #ifdef MSG_DEBUG_OK
 		printf("no such message queue id\n");
 #endif
 		eval = EINVAL;
 		goto msgsndout;
 	}
-	if (msqptr->msg_perm.seq != IPCID_TO_SEQ(uap->msqid)) {
+	if (msqptr->u.msg_perm._seq != IPCID_TO_SEQ(uap->msqid)) {
 #ifdef MSG_DEBUG_OK
 		printf("wrong sequence number\n");
 #endif
@@ -592,27 +715,39 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 		goto msgsndout;
 	}
 
-	if ((eval = ipcperm(kauth_cred_get(), &msqptr->msg_perm, IPC_W))) {
+	if ((eval = ipcperm(kauth_cred_get(), &msqptr->u.msg_perm, IPC_W))) {
 #ifdef MSG_DEBUG_OK
 		printf("requester doesn't have write access\n");
 #endif
 		goto msgsndout;
 	}
 
+#if CONFIG_MACF
+	eval = mac_sysvmsq_check_msqsnd(kauth_cred_get(), msqptr);
+	if (eval) 
+		goto msgsndout;
+#endif
 	segs_needed = (msgsz + msginfo.msgssz - 1) / msginfo.msgssz;
 #ifdef MSG_DEBUG_OK
 	printf("msgsz=%d, msgssz=%d, segs_needed=%d\n", msgsz, msginfo.msgssz,
 	    segs_needed);
 #endif
+
+	/*
+	 * If we suffer resource starvation, we will sleep in this loop and
+	 * wait for more resources to become available.  This is a loop to
+	 * ensure reacquisition of the mutex following any sleep, since there
+	 * are multiple resources under contention.
+	 */
 	for (;;) {
-		int need_more_resources = 0;
+		void *blocking_resource = NULL;
 
 		/*
-		 * check msgsz
-		 * (inside this loop in case msg_qbytes changes while we sleep)
+		 * Check that we have not had the maximum message size change
+		 * out from under us and render our message invalid while we
+		 * slept waiting for some resource.
 		 */
-
-		if (msgsz > msqptr->msg_qbytes) {
+		if (msgsz > msqptr->u.msg_qbytes) {
 #ifdef MSG_DEBUG_OK
 			printf("msgsz > msqptr->msg_qbytes\n");
 #endif
@@ -620,32 +755,49 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 			goto msgsndout;
 		}
 
-		if (msqptr->msg_perm.mode & MSG_LOCKED) {
+		/*
+		 * If the user_msqid_ds is already locked, we need to sleep on
+		 * the queue until it's unlocked.
+		 */
+		if (msqptr->u.msg_perm.mode & MSG_LOCKED) {
 #ifdef MSG_DEBUG_OK
 			printf("msqid is locked\n");
 #endif
-			need_more_resources = 1;
+			blocking_resource = msqptr;
 		}
-		if (msgsz + msqptr->msg_cbytes > msqptr->msg_qbytes) {
+
+		/*
+		 * If our message plus the messages already in the queue would
+		 * cause us to exceed the maximum number of bytes wer are
+		 * permitted to queue, then block on the queue until it drains.
+		 */
+		if (msgsz + msqptr->u.msg_cbytes > msqptr->u.msg_qbytes) {
 #ifdef MSG_DEBUG_OK
 			printf("msgsz + msg_cbytes > msg_qbytes\n");
 #endif
-			need_more_resources = 1;
+			blocking_resource = msqptr;
 		}
+
+		/*
+		 * Both message maps and message headers are protected by
+		 * sleeping on the address of the pointer to the list of free
+		 * message headers, since they are allocated and freed in
+		 * tandem.
+		 */
 		if (segs_needed > nfree_msgmaps) {
 #ifdef MSG_DEBUG_OK
 			printf("segs_needed > nfree_msgmaps\n");
 #endif
-			need_more_resources = 1;
+			blocking_resource = &free_msghdrs;
 		}
 		if (free_msghdrs == NULL) {
 #ifdef MSG_DEBUG_OK
 			printf("no more msghdrs\n");
 #endif
-			need_more_resources = 1;
+			blocking_resource = &free_msghdrs;
 		}
 
-		if (need_more_resources) {
+		if (blocking_resource != NULL) {
 			int we_own_it;
 
 			if ((msgflg & IPC_NOWAIT) != 0) {
@@ -656,7 +808,7 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 				goto msgsndout;
 			}
 
-			if ((msqptr->msg_perm.mode & MSG_LOCKED) != 0) {
+			if ((msqptr->u.msg_perm.mode & MSG_LOCKED) != 0) {
 #ifdef MSG_DEBUG_OK
 				printf("we don't own the user_msqid_ds\n");
 #endif
@@ -667,19 +819,19 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 #ifdef MSG_DEBUG_OK
 				printf("we own the user_msqid_ds\n");
 #endif
-				msqptr->msg_perm.mode |= MSG_LOCKED;
+				msqptr->u.msg_perm.mode |= MSG_LOCKED;
 				we_own_it = 1;
 			}
 #ifdef MSG_DEBUG_OK
 			printf("goodnight\n");
 #endif
-			eval = msleep((caddr_t)msqptr, &sysv_msg_subsys_mutex, (PZERO - 4) | PCATCH,
+			eval = msleep(blocking_resource, &sysv_msg_subsys_mutex, (PZERO - 4) | PCATCH,
 			    "msgwait", 0);
 #ifdef MSG_DEBUG_OK
 			printf("good morning, eval=%d\n", eval);
 #endif
 			if (we_own_it)
-				msqptr->msg_perm.mode &= ~MSG_LOCKED;
+				msqptr->u.msg_perm.mode &= ~MSG_LOCKED;
 			if (eval != 0) {
 #ifdef MSG_DEBUG_OK
 				printf("msgsnd:  interrupted system call\n");
@@ -692,18 +844,11 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 			 * Make sure that the msq queue still exists
 			 */
 
-			if (msqptr->msg_qbytes == 0) {
+			if (msqptr->u.msg_qbytes == 0) {
 #ifdef MSG_DEBUG_OK
 				printf("msqid deleted\n");
 #endif
-				/* The SVID says to return EIDRM. */
-#ifdef EIDRM
 				eval = EIDRM;
-#else
-				/* Unfortunately, BSD doesn't define that code
-				   yet! */
-				eval = EINVAL;
-#endif
 				goto msgsndout;
 			
 			}
@@ -721,11 +866,11 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 	 * Make sure!
 	 */
 
-	if (msqptr->msg_perm.mode & MSG_LOCKED)
+	if (msqptr->u.msg_perm.mode & MSG_LOCKED)
 		panic("msg_perm.mode & MSG_LOCKED");
 	if (segs_needed > nfree_msgmaps)
 		panic("segs_needed > nfree_msgmaps");
-	if (msgsz + msqptr->msg_cbytes > msqptr->msg_qbytes)
+	if (msgsz + msqptr->u.msg_cbytes > msqptr->u.msg_qbytes)
 		panic("msgsz + msg_cbytes > msg_qbytes");
 	if (free_msghdrs == NULL)
 		panic("no more msghdrs");
@@ -734,20 +879,21 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 	 * Re-lock the user_msqid_ds in case we page-fault when copying in
 	 * the message
 	 */
-
-	if ((msqptr->msg_perm.mode & MSG_LOCKED) != 0)
+	if ((msqptr->u.msg_perm.mode & MSG_LOCKED) != 0)
 		panic("user_msqid_ds is already locked");
-	msqptr->msg_perm.mode |= MSG_LOCKED;
+	msqptr->u.msg_perm.mode |= MSG_LOCKED;
 
 	/*
 	 * Allocate a message header
 	 */
-
 	msghdr = free_msghdrs;
 	free_msghdrs = msghdr->msg_next;
 	msghdr->msg_spot = -1;
 	msghdr->msg_ts = msgsz;
 
+#if CONFIG_MACF
+	mac_sysvmsg_label_associate(kauth_cred_get(), msqptr, msghdr);
+#endif
 	/*
 	 * Allocate space for the message
 	 */
@@ -794,7 +940,7 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 		printf("error %d copying the message type\n", eval);
 #endif
 		msg_freehdr(msghdr);
-		msqptr->msg_perm.mode &= ~MSG_LOCKED;
+		msqptr->u.msg_perm.mode &= ~MSG_LOCKED;
 		wakeup((caddr_t)msqptr);
 		goto msgsndout;
 	}
@@ -805,7 +951,7 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 	 */
 	if (msghdr->msg_type < 1) {
 		msg_freehdr(msghdr);
-		msqptr->msg_perm.mode &= ~MSG_LOCKED;
+		msqptr->u.msg_perm.mode &= ~MSG_LOCKED;
 		wakeup((caddr_t)msqptr);
 #ifdef MSG_DEBUG_OK
 		printf("mtype (%d) < 1\n", msghdr->msg_type);
@@ -839,7 +985,7 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 			printf("error %d copying in message segment\n", eval);
 #endif
 			msg_freehdr(msghdr);
-			msqptr->msg_perm.mode &= ~MSG_LOCKED;
+			msqptr->u.msg_perm.mode &= ~MSG_LOCKED;
 			wakeup((caddr_t)msqptr);
 
 			goto msgsndout;
@@ -855,13 +1001,13 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 	 * We've got the message.  Unlock the user_msqid_ds.
 	 */
 
-	msqptr->msg_perm.mode &= ~MSG_LOCKED;
+	msqptr->u.msg_perm.mode &= ~MSG_LOCKED;
 
 	/*
 	 * Make sure that the user_msqid_ds is still allocated.
 	 */
 
-	if (msqptr->msg_qbytes == 0) {
+	if (msqptr->u.msg_qbytes == 0) {
 		msg_freehdr(msghdr);
 		wakeup((caddr_t)msqptr);
 		/* The SVID says to return EIDRM. */
@@ -874,23 +1020,42 @@ msgsnd(struct proc *p, struct msgsnd_args *uap, register_t *retval)
 		goto msgsndout;
 	}
 
+#if CONFIG_MACF
+	/*
+	 * Note: Since the task/thread allocates the msghdr and usually
+	 * primes it with its own MAC label, for a majority of policies, it
+	 * won't be necessary to check whether the msghdr has access
+	 * permissions to the msgq.  The mac_sysvmsq_check_msqsnd check would
+	 * suffice in that case.  However, this hook may be required where
+	 * individual policies derive a non-identical label for the msghdr
+	 * from the current thread label and may want to check the msghdr
+	 * enqueue permissions, along with read/write permissions to the
+	 * msgq.
+	 */
+	eval = mac_sysvmsq_check_enqueue(kauth_cred_get(), msghdr, msqptr);
+	if (eval) {
+		msg_freehdr(msghdr);
+		wakeup((caddr_t) msqptr);
+		goto msgsndout;
+	}
+#endif
 	/*
 	 * Put the message into the queue
 	 */
 
-	if (msqptr->msg_first == NULL) {
-		msqptr->msg_first = msghdr;
-		msqptr->msg_last = msghdr;
+	if (msqptr->u.msg_first == NULL) {
+		msqptr->u.msg_first = msghdr;
+		msqptr->u.msg_last = msghdr;
 	} else {
-		msqptr->msg_last->msg_next = msghdr;
-		msqptr->msg_last = msghdr;
+		msqptr->u.msg_last->msg_next = msghdr;
+		msqptr->u.msg_last = msghdr;
 	}
-	msqptr->msg_last->msg_next = NULL;
+	msqptr->u.msg_last->msg_next = NULL;
 
-	msqptr->msg_cbytes += msghdr->msg_ts;
-	msqptr->msg_qnum++;
-	msqptr->msg_lspid = p->p_pid;
-	msqptr->msg_stime = sysv_msgtime();
+	msqptr->u.msg_cbytes += msghdr->msg_ts;
+	msqptr->u.msg_qnum++;
+	msqptr->u.msg_lspid = p->p_pid;
+	msqptr->u.msg_stime = sysv_msgtime();
 
 	wakeup((caddr_t)msqptr);
 	*retval = 0;
@@ -904,6 +1069,13 @@ msgsndout:
 
 int
 msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
+{
+	__pthread_testcancel(1);
+	return(msgrcv_nocancel(p, (struct msgrcv_nocancel_args *)uap, retval));
+}
+
+int
+msgrcv_nocancel(struct proc *p, struct msgrcv_nocancel_args *uap, user_ssize_t *retval)
 {
 	int msqid = uap->msqid;
 	user_addr_t user_msgp = uap->msgp;
@@ -911,7 +1083,7 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 	long msgtyp = (long)uap->msgtyp;	/* limit to 32 bits */
 	int msgflg = uap->msgflg;
 	size_t len;
-	struct user_msqid_ds *msqptr;
+	struct msqid_kernel *msqptr;
 	struct msg *msghdr;
 	int eval;
 	short next;
@@ -919,7 +1091,11 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 	long msg_type_long;
 
 	SYSV_MSG_SUBSYS_LOCK();
-	msginit( 0);
+
+	if (!msginit(0)) {
+		eval =  ENOMEM;
+		goto msgrcvout;
+	}
 
 #ifdef MSG_DEBUG_OK
 	printf("call to msgrcv(%d, 0x%qx, %d, %ld, %d)\n", msqid, user_msgp,
@@ -939,14 +1115,14 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 	}
 
 	msqptr = &msqids[msqid];
-	if (msqptr->msg_qbytes == 0) {
+	if (msqptr->u.msg_qbytes == 0) {
 #ifdef MSG_DEBUG_OK
 		printf("no such message queue id\n");
 #endif
 		eval = EINVAL;
 		goto msgrcvout;
 	}
-	if (msqptr->msg_perm.seq != IPCID_TO_SEQ(uap->msqid)) {
+	if (msqptr->u.msg_perm._seq != IPCID_TO_SEQ(uap->msqid)) {
 #ifdef MSG_DEBUG_OK
 		printf("wrong sequence number\n");
 #endif
@@ -954,17 +1130,22 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 		goto msgrcvout;
 	}
 
-	if ((eval = ipcperm(kauth_cred_get(), &msqptr->msg_perm, IPC_R))) {
+	if ((eval = ipcperm(kauth_cred_get(), &msqptr->u.msg_perm, IPC_R))) {
 #ifdef MSG_DEBUG_OK
 		printf("requester doesn't have read access\n");
 #endif
 		goto msgrcvout;
 	}
 
+#if CONFIG_MACF
+	eval = mac_sysvmsq_check_msqrcv(kauth_cred_get(), msqptr);
+	if (eval) 
+		goto msgrcvout;
+#endif
 	msghdr = NULL;
 	while (msghdr == NULL) {
 		if (msgtyp == 0) {
-			msghdr = msqptr->msg_first;
+			msghdr = msqptr->u.msg_first;
 			if (msghdr != NULL) {
 				if (msgsz < msghdr->msg_ts &&
 				    (msgflg & MSG_NOERROR) == 0) {
@@ -975,12 +1156,18 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 					eval = E2BIG;
 					goto msgrcvout;
 				}
-				if (msqptr->msg_first == msqptr->msg_last) {
-					msqptr->msg_first = NULL;
-					msqptr->msg_last = NULL;
+#if CONFIG_MACF
+				eval = mac_sysvmsq_check_msgrcv(kauth_cred_get(),
+				    msghdr);
+				if (eval) 
+					goto msgrcvout;
+#endif
+				if (msqptr->u.msg_first == msqptr->u.msg_last) {
+					msqptr->u.msg_first = NULL;
+					msqptr->u.msg_last = NULL;
 				} else {
-					msqptr->msg_first = msghdr->msg_next;
-					if (msqptr->msg_first == NULL)
+					msqptr->u.msg_first = msghdr->msg_next;
+					if (msqptr->u.msg_first == NULL)
 						panic("msg_first/last messed up #1");
 				}
 			}
@@ -989,7 +1176,7 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 			struct msg **prev;
 
 			previous = NULL;
-			prev = &(msqptr->msg_first);
+			prev = &(msqptr->u.msg_first);
 			while ((msghdr = *prev) != NULL) {
 				/*
 				 * Is this message's type an exact match or is
@@ -1015,21 +1202,27 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 						eval = E2BIG;
 						goto msgrcvout;
 					}
+#if CONFIG_MACF
+					eval = mac_sysvmsq_check_msgrcv(
+					    kauth_cred_get(), msghdr);
+					if (eval) 
+						goto msgrcvout;
+#endif
 					*prev = msghdr->msg_next;
-					if (msghdr == msqptr->msg_last) {
+					if (msghdr == msqptr->u.msg_last) {
 						if (previous == NULL) {
 							if (prev !=
-							    &msqptr->msg_first)
+							    &msqptr->u.msg_first)
 								panic("msg_first/last messed up #2");
-							msqptr->msg_first =
+							msqptr->u.msg_first =
 							    NULL;
-							msqptr->msg_last =
+							msqptr->u.msg_last =
 							    NULL;
 						} else {
 							if (prev ==
-							    &msqptr->msg_first)
+							    &msqptr->u.msg_first)
 								panic("msg_first/last messed up #3");
-							msqptr->msg_last =
+							msqptr->u.msg_last =
 							    previous;
 						}
 					}
@@ -1093,8 +1286,8 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 		 * Make sure that the msq queue still exists
 		 */
 
-		if (msqptr->msg_qbytes == 0 ||
-		    msqptr->msg_perm.seq != IPCID_TO_SEQ(uap->msqid)) {
+		if (msqptr->u.msg_qbytes == 0 ||
+		    msqptr->u.msg_perm._seq != IPCID_TO_SEQ(uap->msqid)) {
 #ifdef MSG_DEBUG_OK
 			printf("msqid deleted\n");
 #endif
@@ -1115,10 +1308,10 @@ msgrcv(struct proc *p, struct msgrcv_args *uap, user_ssize_t *retval)
 	 * First, do the bookkeeping (before we risk being interrupted).
 	 */
 
-	msqptr->msg_cbytes -= msghdr->msg_ts;
-	msqptr->msg_qnum--;
-	msqptr->msg_lrpid = p->p_pid;
-	msqptr->msg_rtime = sysv_msgtime();
+	msqptr->u.msg_cbytes -= msghdr->msg_ts;
+	msqptr->u.msg_qnum--;
+	msqptr->u.msg_lrpid = p->p_pid;
+	msqptr->u.msg_rtime = sysv_msgtime();
 
 	/*
 	 * Make msgsz the actual amount that we'll be returning.
@@ -1266,7 +1459,10 @@ IPCS_msg_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 
 	case IPCS_MSG_ITER:	/* Iterate over existing segments */
 		/* Not done up top so we can set limits via sysctl (later) */
-		msginit( 0);
+		if (!msginit(0)) {
+			error =  ENOMEM;
+			break;
+		}
 
 		cursor = ipcs.u64.ipcs_cursor;
 		if (cursor < 0 || cursor >= msginfo.msgmni) {
@@ -1274,11 +1470,11 @@ IPCS_msg_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 			break;
 		}
 		if (ipcs.u64.ipcs_datalen != (int)msqid_ds_sz) {
-			error = ENOMEM;
+			error = EINVAL;
 			break;
 		}
 		for( ; cursor < msginfo.msgmni; cursor++) {
-			if (msqids[cursor].msg_qbytes != 0)	/* allocated */
+			if (msqids[cursor].u.msg_qbytes != 0)	/* allocated */
 				break;
 			continue;
 		}
@@ -1324,3 +1520,5 @@ SYSCTL_PROC(_kern_sysv_ipcs, OID_AUTO, msg, CTLFLAG_RW|CTLFLAG_ANYBODY,
 	0, 0, IPCS_msg_sysctl,
 	"S,IPCS_msg_command",
 	"ipcs msg command interface");
+
+#endif /* SYSV_MSG */
diff --git a/bsd/kern/sysv_sem.c b/bsd/kern/sysv_sem.c
index 199cdd161..d244876ab 100644
--- a/bsd/kern/sysv_sem.c
+++ b/bsd/kern/sysv_sem.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Implementation of SVID semaphores
@@ -29,6 +35,13 @@
 /*
  * John Bellardo modified the implementation for Darwin. 12/2000
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -45,13 +58,26 @@
 #include <sys/ipcs.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 #include <bsm/audit_kernel.h>
 
+#if SYSV_SEM
+
 
 /* Uncomment this line to see the debugging output */
 /* #define SEM_DEBUG */
 
+/* Uncomment this line to see MAC debugging output. */
+/* #define	MAC_DEBUG */
+#if CONFIG_MACF_DEBUG
+#define	MPRINTF(a)	printf(a)
+#else
+#define	MPRINTF(a)
+#endif
+
 #define M_SYSVSEM	M_TEMP
 
 
@@ -89,8 +115,8 @@ struct seminfo seminfo = {
 };
 
 
-static struct sem_undo *semu_alloc(struct proc *p);
-static int semundo_adjust(struct proc *p, struct sem_undo **supptr, 
+static int semu_alloc(struct proc *p);
+static int semundo_adjust(struct proc *p, int *supidx, 
 		int semid, int semnum, int adjval);
 static void semundo_clear(int semid, int semnum);
 
@@ -101,9 +127,9 @@ static sy_call_t *semcalls[] = {
 };
 
 static int		semtot = 0;		/* # of used semaphores */
-struct user_semid_ds	*sema = NULL;		/* semaphore id pool */
+struct semid_kernel	*sema = NULL;		/* semaphore id pool */
 struct sem		*sem_pool =  NULL;	/* semaphore pool */
-static struct sem_undo	*semu_list = NULL;	/* active undo structures */
+static int	 	semu_list_idx = -1;	/* active undo structures */
 struct sem_undo		*semu = NULL;		/* semaphore undo pool */
 
 
@@ -265,7 +291,7 @@ grow_semu_array(int newSize)
 static int
 grow_sema_array(int newSize)
 {
-	register struct user_semid_ds *newSema;
+	register struct semid_kernel *newSema;
 	register int i;
 
 	if (newSize <= seminfo.semmni)
@@ -284,8 +310,8 @@ grow_sema_array(int newSize)
 #ifdef SEM_DEBUG
 	printf("growing sema[] from %d to %d\n", seminfo.semmni, newSize);
 #endif
-	MALLOC(newSema, struct user_semid_ds *,
-	       sizeof (struct user_semid_ds) * newSize,
+	MALLOC(newSema, struct semid_kernel *,
+	       sizeof (struct semid_kernel) * newSize,
 	       M_SYSVSEM, M_WAITOK | M_ZERO);
 	if (NULL == newSema)
 	{
@@ -305,11 +331,19 @@ grow_sema_array(int newSize)
 		 * this with the existing code, so we wake up the
 		 * process and let it do a lot of work to determine the
 		 * semaphore set is really not available yet, and then
-		 * sleep on the correct, reallocated user_semid_ds pointer.
+		 * sleep on the correct, reallocated semid_kernel pointer.
 		 */
-		if (sema[i].sem_perm.mode & SEM_ALLOC)
+		if (sema[i].u.sem_perm.mode & SEM_ALLOC)
 			wakeup((caddr_t)&sema[i]);
 	}
+
+#if CONFIG_MACF
+	for (i = seminfo.semmni; i < newSize; i++)
+	{
+		mac_sysvsem_label_init(&newSema[i]);
+	}
+#endif
+	
 	/*
 	 * The new elements (from newSema[i] to newSema[newSize-1]) have their
 	 * "sem_base" and "sem_perm.mode" set to 0 (i.e. NULL) by the M_ZERO
@@ -374,8 +408,8 @@ grow_sem_pool(int new_pool_size)
 
 	/* Update our id structures to point to the new semaphores */
 	for(i = 0; i < seminfo.semmni; i++) {
-		if (sema[i].sem_perm.mode & SEM_ALLOC)  /* ID in use */
-			sema[i].sem_base += (new_sem_pool - sem_pool);
+		if (sema[i].u.sem_perm.mode & SEM_ALLOC)  /* ID in use */
+			sema[i].u.sem_base += (new_sem_pool - sem_pool);
 	}
 
 	sem_free = sem_pool;
@@ -399,12 +433,12 @@ grow_sem_pool(int new_pool_size)
  * Assumes we already hold the subsystem lock.
  */
 
-static struct sem_undo *
+static int
 semu_alloc(struct proc *p)
 {
 	register int i;
 	register struct sem_undo *suptr;
-	register struct sem_undo **supptr;
+	int *supidx;
 	int attempt;
 
 	/*
@@ -422,12 +456,12 @@ semu_alloc(struct proc *p)
 		for (i = 0; i < seminfo.semmnu; i++) {
 			suptr = SEMU(i);
 			if (suptr->un_proc == NULL) {
-				suptr->un_next = semu_list;
-				semu_list = suptr;
+				suptr->un_next_idx = semu_list_idx;
+				semu_list_idx = i;
 				suptr->un_cnt = 0;
 				suptr->un_ent = NULL;
 				suptr->un_proc = p;
-				return(suptr);
+				return i;
 			}
 		}
 
@@ -440,14 +474,15 @@ semu_alloc(struct proc *p)
 			/* All the structures are in use - try to free some */
 			int did_something = 0;
 
-			supptr = &semu_list;
-			while ((suptr = *supptr) != NULL) {
+			supidx = &semu_list_idx;
+			while (*supidx != -1) {
+				suptr = SEMU(*supidx);
 				if (suptr->un_cnt == 0)  {
 					suptr->un_proc = NULL;
-					*supptr = suptr->un_next;
+					*supidx = suptr->un_next_idx;
 					did_something = 1;
 				} else
-					supptr = &(suptr->un_next);
+					supidx = &(suptr->un_next_idx);
 			}
 
 			/* If we didn't free anything. Try expanding
@@ -457,7 +492,7 @@ semu_alloc(struct proc *p)
 			 */
 			if (!did_something)
 				if (!grow_semu_array(seminfo.semmnu + 1))
-					return(NULL);
+					return -1;
 		} else {
 			/*
 			 * The second pass failed even though we freed
@@ -467,7 +502,7 @@ semu_alloc(struct proc *p)
 			panic("semu_alloc - second attempt failed");
 		}
 	}
-	return (NULL);
+	return -1;
 }
 
 /*
@@ -476,10 +511,11 @@ semu_alloc(struct proc *p)
  * Assumes we already hold the subsystem lock.
  */
 static int
-semundo_adjust(struct proc *p, struct sem_undo **supptr, int semid,
+semundo_adjust(struct proc *p, int *supidx, int semid,
 	int semnum, int adjval)
 {
 	register struct sem_undo *suptr;
+	int suidx;
 	register struct undo *sueptr, **suepptr, *new_sueptr;
 	int i;
 
@@ -487,22 +523,23 @@ semundo_adjust(struct proc *p, struct sem_undo **supptr, int semid,
 	 * Look for and remember the sem_undo if the caller doesn't provide it
 	 */
 
-	suptr = *supptr;
-	if (suptr == NULL) {
-		for (suptr = semu_list; suptr != NULL;
-		    suptr = suptr->un_next) {
+	suidx = *supidx;
+	if (suidx == -1) {
+		for (suidx = semu_list_idx; suidx != -1;
+		    suidx = suptr->un_next_idx) {
+			suptr = SEMU(suidx);
 			if (suptr->un_proc == p) {
-				*supptr = suptr;
+				*supidx = suidx;
 				break;
 			}
 		}
-		if (suptr == NULL) {
+		if (suidx == -1) {
 			if (adjval == 0)
 				return(0);
-			suptr = semu_alloc(p);
-			if (suptr == NULL)
+			suidx = semu_alloc(p);
+			if (suidx == -1)
 				return(ENOSPC);
-			*supptr = suptr;
+			*supidx = suidx;
 		}
 	}
 
@@ -510,6 +547,7 @@ semundo_adjust(struct proc *p, struct sem_undo **supptr, int semid,
 	 * Look for the requested entry and adjust it (delete if adjval becomes
 	 * 0).
 	 */
+	suptr = SEMU(suidx);
 	new_sueptr = NULL;
 	for (i = 0, suepptr = &suptr->un_ent, sueptr = suptr->un_ent;
 	     i < suptr->un_cnt;
@@ -564,12 +602,14 @@ static void
 semundo_clear(int semid, int semnum)
 {
 	struct sem_undo *suptr;
+	int suidx;
 
-	for (suptr = semu_list; suptr != NULL; suptr = suptr->un_next) {
+	for (suidx = semu_list_idx; suidx != -1; suidx = suptr->un_next_idx) {
 		struct undo *sueptr;
 		struct undo **suepptr;
 		int i = 0;
 
+		suptr = SEMU(suidx);
 		sueptr = suptr->un_ent;
 		suepptr = &suptr->un_ent;
 		while (i < suptr->un_cnt) {
@@ -608,7 +648,7 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 	kauth_cred_t cred = kauth_cred_get();
 	int i, rval, eval;
 	struct user_semid_ds sbuf;
-	struct user_semid_ds *semaptr;
+	struct semid_kernel *semakptr;
 	struct user_semid_ds uds;
 	
 
@@ -631,38 +671,46 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 		goto semctlout;
 	}
 
-	semaptr = &sema[semid];
-	if ((semaptr->sem_perm.mode & SEM_ALLOC) == 0 ||
-	    semaptr->sem_perm.seq != IPCID_TO_SEQ(uap->semid)) {
+	semakptr = &sema[semid];
+	if ((semakptr->u.sem_perm.mode & SEM_ALLOC) == 0 ||
+	    semakptr->u.sem_perm._seq != IPCID_TO_SEQ(uap->semid)) {
 		eval = EINVAL;
 		goto semctlout;
 	}
+#if CONFIG_MACF
+	eval = mac_sysvsem_check_semctl(cred, semakptr, cmd);
+	if (eval)
+		goto semctlout;
+#endif
 
 	eval = 0;
 	rval = 0;
 
 	switch (cmd) {
 	case IPC_RMID:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_M))) 
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_M))) 
 			goto semctlout;
 
-		semaptr->sem_perm.cuid = kauth_cred_getuid(cred);
-		semaptr->sem_perm.uid = kauth_cred_getuid(cred);
-		semtot -= semaptr->sem_nsems;
-		for (i = semaptr->sem_base - sem_pool; i < semtot; i++)
-			sem_pool[i] = sem_pool[i + semaptr->sem_nsems];
+		semakptr->u.sem_perm.cuid = kauth_cred_getuid(cred);
+		semakptr->u.sem_perm.uid = kauth_cred_getuid(cred);
+		semtot -= semakptr->u.sem_nsems;
+		for (i = semakptr->u.sem_base - sem_pool; i < semtot; i++)
+			sem_pool[i] = sem_pool[i + semakptr->u.sem_nsems];
 		for (i = 0; i < seminfo.semmni; i++) {
-			if ((sema[i].sem_perm.mode & SEM_ALLOC) &&
-			    sema[i].sem_base > semaptr->sem_base)
-				sema[i].sem_base -= semaptr->sem_nsems;
+			if ((sema[i].u.sem_perm.mode & SEM_ALLOC) &&
+			    sema[i].u.sem_base > semakptr->u.sem_base)
+				sema[i].u.sem_base -= semakptr->u.sem_nsems;
 		}
-		semaptr->sem_perm.mode = 0;
+		semakptr->u.sem_perm.mode = 0;
+#if CONFIG_MACF
+		mac_sysvsem_label_recycle(semakptr);
+#endif
 		semundo_clear(semid, -1);
-		wakeup((caddr_t)semaptr);
+		wakeup((caddr_t)semakptr);
 		break;
 
 	case IPC_SET:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_M)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_M)))
 				goto semctlout;
 
 		if (IS_64BIT_PROCESS(p)) {
@@ -677,17 +725,17 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 			goto semctlout;
 		}
 
-		semaptr->sem_perm.uid = sbuf.sem_perm.uid;
-		semaptr->sem_perm.gid = sbuf.sem_perm.gid;
-		semaptr->sem_perm.mode = (semaptr->sem_perm.mode & ~0777) |
-		    (sbuf.sem_perm.mode & 0777);
-		semaptr->sem_ctime = sysv_semtime();
+		semakptr->u.sem_perm.uid = sbuf.sem_perm.uid;
+		semakptr->u.sem_perm.gid = sbuf.sem_perm.gid;
+		semakptr->u.sem_perm.mode = (semakptr->u.sem_perm.mode &
+		    ~0777) | (sbuf.sem_perm.mode & 0777);
+		semakptr->u.sem_ctime = sysv_semtime();
 		break;
 
 	case IPC_STAT:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
-		bcopy(semaptr, &uds, sizeof(struct user_semid_ds));
+		bcopy((caddr_t)&semakptr->u, &uds, sizeof(struct user_semid_ds));
 		if (IS_64BIT_PROCESS(p)) {
 			eval = copyout(&uds, user_arg.buf, sizeof(struct user_semid_ds));
 		} else {
@@ -698,42 +746,42 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 		break;
 
 	case GETNCNT:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
-		if (semnum < 0 || semnum >= semaptr->sem_nsems) {
+		if (semnum < 0 || semnum >= semakptr->u.sem_nsems) {
 			eval = EINVAL;
 			goto semctlout;
 		}
-		rval = semaptr->sem_base[semnum].semncnt;
+		rval = semakptr->u.sem_base[semnum].semncnt;
 		break;
 
 	case GETPID:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
-		if (semnum < 0 || semnum >= semaptr->sem_nsems) {
+		if (semnum < 0 || semnum >= semakptr->u.sem_nsems) {
 			eval = EINVAL;
 			goto semctlout;
 		}
-		rval = semaptr->sem_base[semnum].sempid;
+		rval = semakptr->u.sem_base[semnum].sempid;
 		break;
 
 	case GETVAL:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
-		if (semnum < 0 || semnum >= semaptr->sem_nsems) {
+		if (semnum < 0 || semnum >= semakptr->u.sem_nsems) {
 			eval = EINVAL;
 			goto semctlout;
 		}
-		rval = semaptr->sem_base[semnum].semval;
+		rval = semakptr->u.sem_base[semnum].semval;
 		break;
 
 	case GETALL:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
 /* XXXXXXXXXXXXXXXX TBD XXXXXXXXXXXXXXXX */
-		for (i = 0; i < semaptr->sem_nsems; i++) {
+		for (i = 0; i < semakptr->u.sem_nsems; i++) {
 			/* XXX could be done in one go... */
-			eval = copyout((caddr_t)&semaptr->sem_base[i].semval,
+			eval = copyout((caddr_t)&semakptr->u.sem_base[i].semval,
 			    user_arg.array + (i * sizeof(unsigned short)),
 			    sizeof(unsigned short));
 			if (eval != 0)
@@ -742,24 +790,24 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 		break;
 
 	case GETZCNT:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_R)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_R)))
 				goto semctlout;
-		if (semnum < 0 || semnum >= semaptr->sem_nsems) {
+		if (semnum < 0 || semnum >= semakptr->u.sem_nsems) {
 			eval = EINVAL;
 			goto semctlout;
 		}
-		rval = semaptr->sem_base[semnum].semzcnt;
+		rval = semakptr->u.sem_base[semnum].semzcnt;
 		break;
 
 	case SETVAL:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_W)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_W)))
                 {
 #ifdef SEM_DEBUG
 			printf("Invalid credentials for write\n");
 #endif
 				goto semctlout;
 		}
-		if (semnum < 0 || semnum >= semaptr->sem_nsems)
+		if (semnum < 0 || semnum >= semakptr->u.sem_nsems)
 		{
 #ifdef SEM_DEBUG
 			printf("Invalid number out of range for set\n");
@@ -772,25 +820,29 @@ semctl(struct proc *p, struct semctl_args *uap, register_t *retval)
 		 * to avoid introducing endieness and a pad field into the
 		 * header file.  Ugly, but it works.
 		 */
-		semaptr->sem_base[semnum].semval = CAST_DOWN(int,user_arg.buf);
+		semakptr->u.sem_base[semnum].semval = CAST_DOWN(int,user_arg.buf);
+		semakptr->u.sem_base[semnum].sempid = p->p_pid;
+		/* XXX scottl Should there be a MAC call here? */
 		semundo_clear(semid, semnum);
-		wakeup((caddr_t)semaptr);
+		wakeup((caddr_t)semakptr);
 		break;
 
 	case SETALL:
-		if ((eval = ipcperm(cred, &semaptr->sem_perm, IPC_W)))
+		if ((eval = ipcperm(cred, &semakptr->u.sem_perm, IPC_W)))
 				goto semctlout;
 /*** XXXXXXXXXXXX TBD ********/
-		for (i = 0; i < semaptr->sem_nsems; i++) {
+		for (i = 0; i < semakptr->u.sem_nsems; i++) {
 			/* XXX could be done in one go... */
 			eval = copyin(user_arg.array + (i * sizeof(unsigned short)),
-			    (caddr_t)&semaptr->sem_base[i].semval,
+			    (caddr_t)&semakptr->u.sem_base[i].semval,
 			    sizeof(unsigned short));
 			if (eval != 0)
 				break;
+			semakptr->u.sem_base[i].sempid = p->p_pid;
 		}
+		/* XXX scottl Should there be a MAC call here? */
 		semundo_clear(semid, -1);
-		wakeup((caddr_t)semaptr);
+		wakeup((caddr_t)semakptr);
 		break;
 
 	default:
@@ -827,18 +879,18 @@ semget(__unused struct proc *p, struct semget_args *uap, register_t *retval)
     
 	if (key != IPC_PRIVATE) {
 		for (semid = 0; semid < seminfo.semmni; semid++) {
-			if ((sema[semid].sem_perm.mode & SEM_ALLOC) &&
-			    sema[semid].sem_perm.key == key)
+			if ((sema[semid].u.sem_perm.mode & SEM_ALLOC) &&
+			    sema[semid].u.sem_perm._key == key)
 				break;
 		}
 		if (semid < seminfo.semmni) {
 #ifdef SEM_DEBUG
 			printf("found public key\n");
 #endif
-			if ((eval = ipcperm(cred, &sema[semid].sem_perm,
+			if ((eval = ipcperm(cred, &sema[semid].u.sem_perm,
 			    semflg & 0700)))
 				goto semgetout;
-			if (nsems < 0 || sema[semid].sem_nsems < nsems) {
+			if (nsems < 0 || sema[semid].u.sem_nsems < nsems) {
 #ifdef SEM_DEBUG
 				printf("too small\n");
 #endif
@@ -852,6 +904,11 @@ semget(__unused struct proc *p, struct semget_args *uap, register_t *retval)
 				eval = EEXIST;
 				goto semgetout;
 			}
+#if CONFIG_MACF
+			eval = mac_sysvsem_check_semget(cred, &sema[semid]);
+			if (eval) 
+				goto semgetout;
+#endif
 			goto found;
 		}
 	}
@@ -882,7 +939,7 @@ semget(__unused struct proc *p, struct semget_args *uap, register_t *retval)
 			}
 		}
 		for (semid = 0; semid < seminfo.semmni; semid++) {
-			if ((sema[semid].sem_perm.mode & SEM_ALLOC) == 0)
+			if ((sema[semid].u.sem_perm.mode & SEM_ALLOC) == 0)
 				break;
 		}
 		if (semid == seminfo.semmni) {
@@ -901,23 +958,26 @@ semget(__unused struct proc *p, struct semget_args *uap, register_t *retval)
 #ifdef SEM_DEBUG
 		printf("semid %d is available\n", semid);
 #endif
-		sema[semid].sem_perm.key = key;
-		sema[semid].sem_perm.cuid = kauth_cred_getuid(cred);
-		sema[semid].sem_perm.uid = kauth_cred_getuid(cred);
-		sema[semid].sem_perm.cgid = cred->cr_gid;
-		sema[semid].sem_perm.gid = cred->cr_gid;
-		sema[semid].sem_perm.mode = (semflg & 0777) | SEM_ALLOC;
-		sema[semid].sem_perm.seq =
-		    (sema[semid].sem_perm.seq + 1) & 0x7fff;
-		sema[semid].sem_nsems = nsems;
-		sema[semid].sem_otime = 0;
-		sema[semid].sem_ctime = sysv_semtime();
-		sema[semid].sem_base = &sem_pool[semtot];
+		sema[semid].u.sem_perm._key = key;
+		sema[semid].u.sem_perm.cuid = kauth_cred_getuid(cred);
+		sema[semid].u.sem_perm.uid = kauth_cred_getuid(cred);
+		sema[semid].u.sem_perm.cgid = cred->cr_gid;
+		sema[semid].u.sem_perm.gid = cred->cr_gid;
+		sema[semid].u.sem_perm.mode = (semflg & 0777) | SEM_ALLOC;
+		sema[semid].u.sem_perm._seq =
+		    (sema[semid].u.sem_perm._seq + 1) & 0x7fff;
+		sema[semid].u.sem_nsems = nsems;
+		sema[semid].u.sem_otime = 0;
+		sema[semid].u.sem_ctime = sysv_semtime();
+		sema[semid].u.sem_base = &sem_pool[semtot];
 		semtot += nsems;
-		bzero(sema[semid].sem_base,
-		    sizeof(sema[semid].sem_base[0])*nsems);
+		bzero(sema[semid].u.sem_base,
+		    sizeof(sema[semid].u.sem_base[0])*nsems);
+#if CONFIG_MACF
+		mac_sysvsem_label_associate(cred, &sema[semid]);
+#endif
 #ifdef SEM_DEBUG
-		printf("sembase = 0x%x, next = 0x%x\n", sema[semid].sem_base,
+		printf("sembase = 0x%x, next = 0x%x\n", sema[semid].u.sem_base,
 		    &sem_pool[semtot]);
 #endif
 	} else {
@@ -929,7 +989,7 @@ semget(__unused struct proc *p, struct semget_args *uap, register_t *retval)
 	}
 
 found:
-	*retval = IXSEQ_TO_IPCID(semid, sema[semid].sem_perm);
+	*retval = IXSEQ_TO_IPCID(semid, sema[semid].u.sem_perm);
 	AUDIT_ARG(svipc_id, *retval);
 #ifdef SEM_DEBUG
 	printf("semget is done, returning %d\n", *retval);
@@ -947,10 +1007,10 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 	int semid = uap->semid;
 	int nsops = uap->nsops;
 	struct sembuf sops[MAX_SOPS];
-	register struct user_semid_ds *semaptr;
+	register struct semid_kernel *semakptr;
 	register struct sembuf *sopptr = NULL;	/* protected by 'semptr' */
 	register struct sem *semptr = NULL;	/* protected by 'if' */
-	struct sem_undo *suptr = NULL;
+	int supidx = -1;
 	int i, j, eval;
 	int do_wakeup, do_undos;
 
@@ -969,23 +1029,41 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		goto semopout;
 	}
 
-	semaptr = &sema[semid];
-	if ((semaptr->sem_perm.mode & SEM_ALLOC) == 0) {
+	semakptr = &sema[semid];
+	if ((semakptr->u.sem_perm.mode & SEM_ALLOC) == 0) {
 		eval = EINVAL;
 		goto semopout;
 	}
-	if (semaptr->sem_perm.seq != IPCID_TO_SEQ(uap->semid)) {
+	if (semakptr->u.sem_perm._seq != IPCID_TO_SEQ(uap->semid)) {
 		eval = EINVAL;
 		goto semopout;
 	}
 
-	if ((eval = ipcperm(kauth_cred_get(), &semaptr->sem_perm, IPC_W))) {
+	if ((eval = ipcperm(kauth_cred_get(), &semakptr->u.sem_perm, IPC_W))) {
 #ifdef SEM_DEBUG
 		printf("eval = %d from ipaccess\n", eval);
 #endif
 		goto semopout;
 	}
 
+#if CONFIG_MACF
+	/*
+	 * Initial pass thru sops to see what permissions are needed.
+	 */
+	j = 0;		/* permission needed */
+	for (i = 0; i < nsops; i++)
+		j |= (sops[i].sem_op == 0) ? SEM_R : SEM_A;
+
+	/*
+	 * The MAC hook checks whether the thread has read (and possibly
+	 * write) permissions to the semaphore array based on the
+	 * sopptr->sem_op value.
+	 */
+	eval = mac_sysvsem_check_semop(kauth_cred_get(), semakptr, j);
+	if (eval)
+		goto semopout;
+#endif
+
 	if (nsops < 0 || nsops > MAX_SOPS) {
 #ifdef SEM_DEBUG
 		printf("too many sops (max=%d, nsops=%d)\n", MAX_SOPS, nsops);
@@ -1020,16 +1098,16 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		for (i = 0; i < nsops; i++) {
 			sopptr = &sops[i];
 
-			if (sopptr->sem_num >= semaptr->sem_nsems) {
+			if (sopptr->sem_num >= semakptr->u.sem_nsems) {
 				eval = EFBIG;
 				goto semopout;
 			}
 
-			semptr = &semaptr->sem_base[sopptr->sem_num];
+			semptr = &semakptr->u.sem_base[sopptr->sem_num];
 
 #ifdef SEM_DEBUG
-			printf("semop:  semaptr=%x, sem_base=%x, semptr=%x, sem[%d]=%d : op=%d, flag=%s\n",
-			    semaptr, semaptr->sem_base, semptr,
+			printf("semop:  semakptr=%x, sem_base=%x, semptr=%x, sem[%d]=%d : op=%d, flag=%s\n",
+			    semakptr, semakptr->u.sem_base, semptr,
 			    sopptr->sem_num, semptr->semval, sopptr->sem_op,
 			    (sopptr->sem_flg & IPC_NOWAIT) ? "nowait" : "wait");
 #endif
@@ -1077,7 +1155,7 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		printf("semop:  rollback 0 through %d\n", i-1);
 #endif
 		for (j = 0; j < i; j++)
-			semaptr->sem_base[sops[j].sem_num].semval -=
+			semakptr->u.sem_base[sops[j].sem_num].semval -=
 			    sops[j].sem_op;
 
 		/*
@@ -1102,7 +1180,7 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		 * waiting for. We will get the lock back after we
 		 * wake up.
 		 */
-		eval = msleep((caddr_t)semaptr, &sysv_sem_subsys_mutex , (PZERO - 4) | PCATCH,
+		eval = msleep((caddr_t)semakptr, &sysv_sem_subsys_mutex , (PZERO - 4) | PCATCH,
 		    "semwait", 0);
                 
 #ifdef SEM_DEBUG
@@ -1119,15 +1197,20 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		 * structures and re-validate them.
 		 */
 
-		suptr = NULL;	/* sem_undo may have been reallocated */
-	 	semaptr = &sema[semid];	   /* sema may have been reallocated */
+		semptr = NULL;
 
 		/*
 		 * Make sure that the semaphore still exists
+		 *
+		 * XXX POSIX: Third test this 'if' and 'EINTR' precedence may
+		 * fail testing; if so, we will need to revert this code.
 		 */
-		if ((semaptr->sem_perm.mode & SEM_ALLOC) == 0 ||
-		    semaptr->sem_perm.seq != IPCID_TO_SEQ(uap->semid) ||
-		    sopptr->sem_num >= semaptr->sem_nsems) {
+	 	semakptr = &sema[semid];   /* sema may have been reallocated */
+		if ((semakptr->u.sem_perm.mode & SEM_ALLOC) == 0 ||
+		    semakptr->u.sem_perm._seq != IPCID_TO_SEQ(uap->semid) ||
+		    sopptr->sem_num >= semakptr->u.sem_nsems) {
+			/* The man page says to return EIDRM. */
+			/* Unfortunately, BSD doesn't define that code! */
 			if (eval == EINTR) {
 				/*
 				 * EINTR takes precedence over the fact that
@@ -1135,14 +1218,10 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 				 * sleeping...
 				 */
 			} else {
-				/*
-				 * The man page says to return EIDRM.
-				 * Unfortunately, BSD doesn't define that code!
-				 */
 #ifdef EIDRM
 				eval = EIDRM;
 #else
-				eval = EINVAL;
+				eval = EINVAL;		/* Ancient past */
 #endif
 			}
 			goto semopout;
@@ -1154,7 +1233,7 @@ semop(struct proc *p, struct semop_args *uap, register_t *retval)
 		 * because the sem[] may have been reallocated while
 		 * we were sleeping, updating our sem_base pointer.
 		 */
-		semptr = &semaptr->sem_base[sopptr->sem_num];
+		semptr = &semakptr->u.sem_base[sopptr->sem_num];
 		if (sopptr->sem_op == 0)
 			semptr->semzcnt--;
 		else
@@ -1182,7 +1261,7 @@ done:
 			adjval = sops[i].sem_op;
 			if (adjval == 0)
 				continue;
-			eval = semundo_adjust(p, &suptr, semid,
+			eval = semundo_adjust(p, &supidx, semid,
 			    sops[i].sem_num, -adjval);
 			if (eval == 0)
 				continue;
@@ -1202,13 +1281,13 @@ done:
 				adjval = sops[j].sem_op;
 				if (adjval == 0)
 					continue;
-				if (semundo_adjust(p, &suptr, semid,
+				if (semundo_adjust(p, &supidx, semid,
 				    sops[j].sem_num, adjval) != 0)
 					panic("semop - can't undo undos");
 			}
 
 			for (j = 0; j < nsops; j++)
-				semaptr->sem_base[sops[j].sem_num].semval -=
+				semakptr->u.sem_base[sops[j].sem_num].semval -=
 				    sops[j].sem_op;
 
 #ifdef SEM_DEBUG
@@ -1221,21 +1300,22 @@ done:
 	/* We're definitely done - set the sempid's */
 	for (i = 0; i < nsops; i++) {
 		sopptr = &sops[i];
-		semptr = &semaptr->sem_base[sopptr->sem_num];
+		semptr = &semakptr->u.sem_base[sopptr->sem_num];
 		semptr->sempid = p->p_pid;
 	}
+	semakptr->u.sem_otime = sysv_semtime();
 
 	if (do_wakeup) {
 #ifdef SEM_DEBUG
 		printf("semop:  doing wakeup\n");
 #ifdef SEM_WAKEUP
-		sem_wakeup((caddr_t)semaptr);
+		sem_wakeup((caddr_t)semakptr);
 #else
-		wakeup((caddr_t)semaptr);
+		wakeup((caddr_t)semakptr);
 #endif
 		printf("semop:  back from wakeup\n");
 #else
-		wakeup((caddr_t)semaptr);
+		wakeup((caddr_t)semakptr);
 #endif
 	}
 #ifdef SEM_DEBUG
@@ -1255,8 +1335,9 @@ semopout:
 void
 semexit(struct proc *p)
 {
-	register struct sem_undo *suptr;
-	register struct sem_undo **supptr;
+	register struct sem_undo *suptr = NULL;
+	int suidx;
+	int *supidx;
 	int did_something;
 
 	/* If we have not allocated our semaphores yet there can't be
@@ -1277,13 +1358,14 @@ semexit(struct proc *p)
 	 * associated with this process.
 	 */
 
-	for (supptr = &semu_list; (suptr = *supptr) != NULL;
-	    supptr = &suptr->un_next) {
+	for (supidx = &semu_list_idx; (suidx = *supidx) != -1;
+	    supidx = &suptr->un_next_idx) {
+		suptr = SEMU(suidx);
 		if (suptr->un_proc == p)
 			break;
 	}
 
-	if (suptr == NULL)
+	if (suidx == -1)
 		goto unlock;
 
 #ifdef SEM_DEBUG
@@ -1300,17 +1382,17 @@ semexit(struct proc *p)
 			int semid;
 			int semnum;
 			int adjval;
-			struct user_semid_ds *semaptr;
+			struct semid_kernel *semakptr;
 
 			sueptr = suptr->un_ent;
 			semid = sueptr->une_id;
 			semnum = sueptr->une_num;
 			adjval = sueptr->une_adjval;
 
-			semaptr = &sema[semid];
-			if ((semaptr->sem_perm.mode & SEM_ALLOC) == 0)
+			semakptr = &sema[semid];
+			if ((semakptr->u.sem_perm.mode & SEM_ALLOC) == 0)
 				panic("semexit - semid not allocated");
-			if (semnum >= semaptr->sem_nsems)
+			if (semnum >= semakptr->u.sem_nsems)
 				panic("semexit - semnum out of range");
 
 #ifdef SEM_DEBUG
@@ -1319,19 +1401,19 @@ semexit(struct proc *p)
 			       semid,
 			       semnum,
 			       adjval,
-			       semaptr->sem_base[semnum].semval);
+			       semakptr->u.sem_base[semnum].semval);
 #endif
 
 			if (adjval < 0) {
-				if (semaptr->sem_base[semnum].semval < -adjval)
-					semaptr->sem_base[semnum].semval = 0;
+				if (semakptr->u.sem_base[semnum].semval < -adjval)
+					semakptr->u.sem_base[semnum].semval = 0;
 				else
-					semaptr->sem_base[semnum].semval +=
+					semakptr->u.sem_base[semnum].semval +=
 					    adjval;
 			} else
-				semaptr->sem_base[semnum].semval += adjval;
+				semakptr->u.sem_base[semnum].semval += adjval;
 
-		/* Maybe we should build a list of semaptr's to wake
+		/* Maybe we should build a list of semakptr's to wake
 		 * up, finish all access to data structures, release the
 		 * subsystem lock, and wake all the processes.  Something
 		 * to think about.  It wouldn't buy us anything unless
@@ -1340,9 +1422,9 @@ semexit(struct proc *p)
 		 * in the BSD code at once.
 		 */
 #ifdef SEM_WAKEUP
-			sem_wakeup((caddr_t)semaptr);
+			sem_wakeup((caddr_t)semakptr);
 #else
-			wakeup((caddr_t)semaptr);
+			wakeup((caddr_t)semakptr);
 #endif
 #ifdef SEM_DEBUG
 			printf("semexit:  back from wakeup\n");
@@ -1361,7 +1443,7 @@ semexit(struct proc *p)
 	printf("removing vector\n");
 #endif
 	suptr->un_proc = NULL;
-	*supptr = suptr->un_next;
+	*supidx = suptr->un_next_idx;
 
 unlock:
 	/*
@@ -1405,7 +1487,7 @@ sysctl_seminfo(__unused struct sysctl_oid *oidp, void *arg1,
 	if ((sem_pool == NULL) && 
 		(sema == NULL) && 
 		(semu == NULL) && 
-		(semu_list == NULL)) {
+		(semu_list_idx == -1)) {
 			if ((error = SYSCTL_IN(req, arg1, sizeof(int)))) {
 				goto out;
 			}
@@ -1419,19 +1501,19 @@ out:
 
 /* SYSCTL_NODE(_kern, KERN_SYSV, sysv, CTLFLAG_RW, 0, "SYSV"); */
 extern struct sysctl_oid_list sysctl__kern_sysv_children;
-SYSCTL_PROC(_kern_sysv, KSYSV_SEMMNI, semmni, CTLTYPE_INT | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, semmni, CTLTYPE_INT | CTLFLAG_RW,
     &limitseminfo.semmni, 0, &sysctl_seminfo ,"I","semmni");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SEMMNS, semmns, CTLTYPE_INT | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, semmns, CTLTYPE_INT | CTLFLAG_RW,
     &limitseminfo.semmns, 0, &sysctl_seminfo ,"I","semmns");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SEMMNU, semmnu, CTLTYPE_INT | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, semmnu, CTLTYPE_INT | CTLFLAG_RW,
     &limitseminfo.semmnu, 0, &sysctl_seminfo ,"I","semmnu");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SEMMSL, semmsl, CTLTYPE_INT | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, semmsl, CTLTYPE_INT | CTLFLAG_RW,
     &limitseminfo.semmsl, 0, &sysctl_seminfo ,"I","semmsl");
     
-SYSCTL_PROC(_kern_sysv, KSYSV_SEMUNE, semume, CTLTYPE_INT | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, semume, CTLTYPE_INT | CTLFLAG_RW,
     &limitseminfo.semume, 0, &sysctl_seminfo ,"I","semume");
 
 
@@ -1451,15 +1533,18 @@ IPCS_sem_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 	size_t semid_ds_sz = sizeof(struct user_semid_ds);
 	struct proc *p = current_proc();
 
+	if (!IS_64BIT_PROCESS(p)) {
+		ipcs_sz = sizeof(struct IPCS_command);
+		semid_ds_sz = sizeof(struct semid_ds);
+	}
+
 	/* Copy in the command structure */
 	if ((error = SYSCTL_IN(req, &ipcs, ipcs_sz)) != 0) {
 		return(error);
 	}
 
-	if (!IS_64BIT_PROCESS(p)) {
-		ipcs_sz = sizeof(struct IPCS_command);
-		semid_ds_sz = sizeof(struct semid_ds);
-	}
+	if (!IS_64BIT_PROCESS(p)) /* convert in place */
+		ipcs.u64.ipcs_data = CAST_USER_ADDR_T(ipcs.u32.ipcs_data);
 
 	/* Let us version this interface... */
 	if (ipcs.u64.ipcs_magic != IPCS_MAGIC) {
@@ -1491,7 +1576,7 @@ IPCS_sem_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 			break;
 		}
 		for( ; cursor < seminfo.semmni; cursor++) {
-			if (sema[cursor].sem_perm.mode & SEM_ALLOC)
+			if (sema[cursor].u.sem_perm.mode & SEM_ALLOC)
 				break;
 			continue;
 		}
@@ -1500,7 +1585,7 @@ IPCS_sem_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 			break;
 		}
 
-		semid_dsp = &sema[cursor];	/* default: 64 bit */
+		semid_dsp = &sema[cursor].u;	/* default: 64 bit */
 
 		/*
 		 * If necessary, convert the 64 bit kernel segment
@@ -1531,3 +1616,5 @@ SYSCTL_PROC(_kern_sysv_ipcs, OID_AUTO, sem, CTLFLAG_RW|CTLFLAG_ANYBODY,
 	0, 0, IPCS_sem_sysctl,
 	"S,IPCS_sem_command",
 	"ipcs sem command interface");
+
+#endif /* SYSV_SEM */
diff --git a/bsd/kern/sysv_shm.c b/bsd/kern/sysv_shm.c
index 09f12c6d5..dfe108c62 100644
--- a/bsd/kern/sysv_shm.c
+++ b/bsd/kern/sysv_shm.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: sysv_shm.c,v 1.23 1994/07/04 23:25:12 glass Exp $	*/
 
@@ -50,6 +56,13 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+*/
 
 
 #include <sys/appleapiopts.h>
@@ -66,6 +79,9 @@
 #include <sys/ipcs.h>
 #include <sys/sysent.h>
 #include <sys/sysproto.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 #include <bsm/audit_kernel.h>
 
@@ -76,11 +92,19 @@
 #include <mach/mach_vm.h>
 
 #include <vm/vm_map.h>
-#include <vm/vm_shared_memory_server.h>
 #include <vm/vm_protos.h>
 
 #include <kern/locks.h>
 
+/* Uncomment this line to see MAC debugging output. */
+/* #define MAC_DEBUG */
+#if CONFIG_MACF_DEBUG
+#define	MPRINTF(a)	printf a
+#else
+#define	MPRINTF(a)     
+#endif
+
+#if SYSV_SHM
 static void shminit(void *);
 #if 0
 SYSINIT(sysv_shm, SI_SUB_SYSV_SHM, SI_ORDER_FIRST, shminit, NULL)
@@ -113,7 +137,7 @@ static sy_call_t *shmcalls[] = {
 #define	SHMSEG_WANTED		0x1000
 
 static int shm_last_free, shm_nused, shm_committed;
-struct user_shmid_ds	*shmsegs;	/* 64 bit version */
+struct shmid_kernel	*shmsegs;	/* 64 bit version */
 static int shm_inited = 0;
 
 struct shm_handle {
@@ -125,18 +149,23 @@ struct shmmap_state {
 	int shmid;			/* segment id */
 };
 
-static void shm_deallocate_segment(struct user_shmid_ds *);
+static void shm_deallocate_segment(struct shmid_kernel *);
 static int shm_find_segment_by_key(key_t);
-static struct user_shmid_ds *shm_find_segment_by_shmid(int);
+static struct shmid_kernel *shm_find_segment_by_shmid(int);
 static int shm_delete_mapping(struct proc *, struct shmmap_state *, int);
 
 #ifdef __APPLE_API_PRIVATE
+#define DEFAULT_SHMMAX	(4 * 1024 * 1024)
+#define DEFAULT_SHMMIN	1
+#define DEFAULT_SHMMNI	32
+#define DEFAULT_SHMSEG	8
+#define DEFAULT_SHMALL	1024
 struct  shminfo shminfo = {
-        -1,	/* SHMMAX 4096 *1024 */
-        -1,	/* SHMMIN = 1 */
-        -1,	/* SHMMNI = 1 */
-        -1,	/* SHMSEG = 8 */
-        -1	/* SHMALL = 1024 */
+        DEFAULT_SHMMAX,
+        DEFAULT_SHMMIN,
+        DEFAULT_SHMMNI,
+	DEFAULT_SHMSEG,
+	DEFAULT_SHMALL
 };
 #endif /* __APPLE_API_PRIVATE */
 
@@ -196,66 +225,70 @@ shm_find_segment_by_key(key_t key)
 	int i;
 
 	for (i = 0; i < shminfo.shmmni; i++)
-		if ((shmsegs[i].shm_perm.mode & SHMSEG_ALLOCATED) &&
-		    shmsegs[i].shm_perm.key == key)
+		if ((shmsegs[i].u.shm_perm.mode & SHMSEG_ALLOCATED) &&
+		    shmsegs[i].u.shm_perm._key == key)
 			return i;
 	return -1;
 }
 
-static struct user_shmid_ds *
+static struct shmid_kernel *
 shm_find_segment_by_shmid(int shmid)
 {
 	int segnum;
-	struct user_shmid_ds *shmseg;
+	struct shmid_kernel *shmseg;
 
 	segnum = IPCID_TO_IX(shmid);
 	if (segnum < 0 || segnum >= shminfo.shmmni)
 		return NULL;
 	shmseg = &shmsegs[segnum];
-	if ((shmseg->shm_perm.mode & (SHMSEG_ALLOCATED | SHMSEG_REMOVED))
+	if ((shmseg->u.shm_perm.mode & (SHMSEG_ALLOCATED | SHMSEG_REMOVED))
 	    != SHMSEG_ALLOCATED ||
-	    shmseg->shm_perm.seq != IPCID_TO_SEQ(shmid))
+	    shmseg->u.shm_perm._seq != IPCID_TO_SEQ(shmid))
 		return NULL;
 	return shmseg;
 }
 
 static void
-shm_deallocate_segment(struct user_shmid_ds *shmseg)
+shm_deallocate_segment(struct shmid_kernel *shmseg)
 {
 	struct shm_handle *shm_handle;
 	mach_vm_size_t size;
 
-	shm_handle = CAST_DOWN(void *,shmseg->shm_internal);	/* tunnel */
-	size = mach_vm_round_page(shmseg->shm_segsz);
+	shm_handle = CAST_DOWN(void *,shmseg->u.shm_internal);	/* tunnel */
+	size = mach_vm_round_page(shmseg->u.shm_segsz);
 	mach_memory_entry_port_release(shm_handle->shm_object);
 	shm_handle->shm_object = NULL;
 	FREE((caddr_t)shm_handle, M_SHM);
-	shmseg->shm_internal = USER_ADDR_NULL;		/* tunnel */
+	shmseg->u.shm_internal = USER_ADDR_NULL;		/* tunnel */
 	shm_committed -= btoc(size);
 	shm_nused--;
-	shmseg->shm_perm.mode = SHMSEG_FREE;
+	shmseg->u.shm_perm.mode = SHMSEG_FREE;
+#if CONFIG_MACF
+	/* Reset the MAC label */
+	mac_sysvshm_label_recycle(shmseg);
+#endif
 }
 
 static int
 shm_delete_mapping(__unused struct proc *p, struct shmmap_state *shmmap_s,
 	int deallocate)
 {
-	struct user_shmid_ds *shmseg;
+	struct shmid_kernel *shmseg;
 	int segnum, result;
 	mach_vm_size_t size;
 
 	segnum = IPCID_TO_IX(shmmap_s->shmid);
 	shmseg = &shmsegs[segnum];
-	size = mach_vm_round_page(shmseg->shm_segsz);	/* XXX done for us? */
+	size = mach_vm_round_page(shmseg->u.shm_segsz);	/* XXX done for us? */
 	if (deallocate) {
 	result = mach_vm_deallocate(current_map(), shmmap_s->va, size);
 	if (result != KERN_SUCCESS)
 		return EINVAL;
 	}
 	shmmap_s->shmid = -1;
-	shmseg->shm_dtime = sysv_shmtime();
-	if ((--shmseg->shm_nattch <= 0) &&
-	    (shmseg->shm_perm.mode & SHMSEG_REMOVED)) {
+	shmseg->u.shm_dtime = sysv_shmtime();
+	if ((--shmseg->u.shm_nattch <= 0) &&
+	    (shmseg->u.shm_perm.mode & SHMSEG_REMOVED)) {
 		shm_deallocate_segment(shmseg);
 		shm_last_free = segnum;
 	}
@@ -265,18 +298,19 @@ shm_delete_mapping(__unused struct proc *p, struct shmmap_state *shmmap_s,
 int
 shmdt(struct proc *p, struct shmdt_args *uap, register_t *retval)
 {
+#if CONFIG_MACF
+	struct shmid_kernel *shmsegptr;
+#endif
 	struct shmmap_state *shmmap_s;
 	int i;
 	int shmdtret = 0;
 
-	// LP64todo - fix this
-	AUDIT_ARG(svipc_addr, CAST_DOWN(void *,uap->shmaddr));
+	AUDIT_ARG(svipc_addr, uap->shmaddr);
 
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		shmdtret = EINVAL;
-		goto shmdt_out;
+		shminit(NULL);
 	}
 	shmmap_s = (struct shmmap_state *)p->vm_shm;
  	if (shmmap_s == NULL) {
@@ -292,6 +326,16 @@ shmdt(struct proc *p, struct shmdt_args *uap, register_t *retval)
 		shmdtret = EINVAL;
 		goto shmdt_out;
 	}
+#if CONFIG_MACF
+	/*
+	 * XXX: It might be useful to move this into the shm_delete_mapping
+	 * function
+	 */
+	shmsegptr = &shmsegs[IPCID_TO_IX(shmmap_s->shmid)];
+	shmdtret = mac_sysvshm_check_shmdt(kauth_cred_get(), shmsegptr);
+	if (shmdtret)
+		goto shmdt_out;
+#endif
 	i = shm_delete_mapping(p, shmmap_s, 1);
 
 	if (i == 0)
@@ -303,10 +347,10 @@ shmdt_out:
 }
 
 int
-shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
+shmat(struct proc *p, struct shmat_args *uap, user_addr_t *retval)
 {
 	int error, i, flags;
-	struct user_shmid_ds	*shmseg;
+	struct shmid_kernel	*shmseg;
 	struct shmmap_state	*shmmap_s = NULL;
 	struct shm_handle	*shm_handle;
 	mach_vm_address_t	attach_va;	/* attach address in/out */
@@ -317,14 +361,12 @@ shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
 	int shmat_ret = 0;
 
 	AUDIT_ARG(svipc_id, uap->shmid);
-	// LP64todo - fix this
-	AUDIT_ARG(svipc_addr, CAST_DOWN(void *,uap->shmaddr));
+	AUDIT_ARG(svipc_addr, uap->shmaddr);
 
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		shmat_ret = EINVAL;
-		goto shmat_out;
+		shminit(NULL);
 	}
 
 	shmmap_s = (struct shmmap_state *)p->vm_shm;
@@ -346,14 +388,21 @@ shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
 		goto shmat_out;
 	}
 
-	AUDIT_ARG(svipc_perm, &shmseg->shm_perm);
-	error = ipcperm(kauth_cred_get(), &shmseg->shm_perm,
+	AUDIT_ARG(svipc_perm, &shmseg->u.shm_perm);
+	error = ipcperm(kauth_cred_get(), &shmseg->u.shm_perm,
 	    (uap->shmflg & SHM_RDONLY) ? IPC_R : IPC_R|IPC_W);
 	if (error) {
 		shmat_ret = error;
 		goto shmat_out;
 	}
 
+#if CONFIG_MACF
+	error = mac_sysvshm_check_shmat(kauth_cred_get(), shmseg, uap->shmflg);
+	if (error) {
+		shmat_ret = error;
+		goto shmat_out;
+	}
+#endif
 	for (i = 0; i < shminfo.shmseg; i++) {
 		if (shmmap_s->shmid == -1)
 			break;
@@ -364,7 +413,7 @@ shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
 		goto shmat_out;
 	}
 
-	map_size = mach_vm_round_page(shmseg->shm_segsz);
+	map_size = mach_vm_round_page(shmseg->u.shm_segsz);
 	prot = VM_PROT_READ;
 	if ((uap->shmflg & SHM_RDONLY) == 0)
 		prot |= VM_PROT_WRITE;
@@ -380,13 +429,13 @@ shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
 		goto shmat_out;
 	}
 
-	shm_handle = CAST_DOWN(void *, shmseg->shm_internal);	/* tunnel */
+	shm_handle = CAST_DOWN(void *, shmseg->u.shm_internal);	/* tunnel */
 
 	rv = mach_vm_map(current_map(),			/* process map */
 			&attach_va,			/* attach address */
 			map_size,			/* segment size */
 			(mach_vm_offset_t)0,		/* alignment mask */
-		(flags & MAP_FIXED)? VM_FLAGS_FIXED: VM_FLAGS_ANYWHERE,
+			(flags & MAP_FIXED)? VM_FLAGS_FIXED: VM_FLAGS_ANYWHERE,
 			shm_handle->shm_object,
 			(mach_vm_offset_t)0,
 			FALSE,
@@ -404,9 +453,9 @@ shmat(struct proc *p, struct shmat_args *uap, register_t *retval)
 
 	shmmap_s->va = attach_va;
 	shmmap_s->shmid = uap->shmid;
-	shmseg->shm_lpid = p->p_pid;
-	shmseg->shm_atime = sysv_shmtime();
-	shmseg->shm_nattch++;
+	shmseg->u.shm_lpid = p->p_pid;
+	shmseg->u.shm_atime = sysv_shmtime();
+	shmseg->u.shm_nattch++;
 	*retval = attach_va;	/* XXX return -1 on error */
 	shmat_ret = 0;
 	goto shmat_out;
@@ -431,13 +480,21 @@ oshmctl(__unused void *p, __unused void *uap, __unused void *retval)
 	return EINVAL;
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	copyout:EFAULT
+ *	copyin:EFAULT
+ *	ipcperm:EPERM
+ *	ipcperm:EACCES
+ */
 int
 shmctl(__unused struct proc *p, struct shmctl_args *uap, register_t *retval)
 {
 	int error;
 	kauth_cred_t cred = kauth_cred_get();
 	struct user_shmid_ds inbuf;
-	struct user_shmid_ds *shmseg;
+	struct shmid_kernel *shmseg;
 	size_t shmid_ds_sz = sizeof(struct user_shmid_ds);
 
 	int shmctl_ret = 0;
@@ -448,8 +505,7 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, register_t *retval)
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		shmctl_ret = EINVAL;
-		goto shmctl_out;
+		shminit(NULL);
 	}
 
 	if (!IS_64BIT_PROCESS(p))
@@ -464,21 +520,28 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, register_t *retval)
 	/* XXAUDIT: This is the perms BEFORE any change by this call. This 
 	 * may not be what is desired.
 	 */
-	AUDIT_ARG(svipc_perm, &shmseg->shm_perm);
+	AUDIT_ARG(svipc_perm, &shmseg->u.shm_perm);
 
+#if CONFIG_MACF
+	error = mac_sysvshm_check_shmctl(cred, shmseg, uap->cmd);
+	if (error) {
+		shmctl_ret = error;
+		goto shmctl_out;
+	}
+#endif
 	switch (uap->cmd) {
 	case IPC_STAT:
-		error = ipcperm(cred, &shmseg->shm_perm, IPC_R);
+		error = ipcperm(cred, &shmseg->u.shm_perm, IPC_R);
 		if (error) {
 			shmctl_ret = error;
 			goto shmctl_out;
 		}
 
 		if (IS_64BIT_PROCESS(p)) {
-			error = copyout(shmseg, uap->buf, sizeof(struct user_shmid_ds));
+			error = copyout((caddr_t)&shmseg->u, uap->buf, sizeof(struct user_shmid_ds));
 		} else {
 			struct shmid_ds shmid_ds32;
-			shmid_ds_64to32(shmseg, &shmid_ds32);
+			shmid_ds_64to32(&shmseg->u, &shmid_ds32);
 			error = copyout(&shmid_ds32, uap->buf, sizeof(struct shmid_ds));
 		}
 		if (error) {
@@ -487,7 +550,7 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, register_t *retval)
 		}
 		break;
 	case IPC_SET:
-		error = ipcperm(cred, &shmseg->shm_perm, IPC_M);
+		error = ipcperm(cred, &shmseg->u.shm_perm, IPC_M);
 		if (error) {
 			shmctl_ret = error;
 			goto shmctl_out;
@@ -503,22 +566,22 @@ shmctl(__unused struct proc *p, struct shmctl_args *uap, register_t *retval)
 			shmctl_ret = error;
 			goto shmctl_out;
 		}
-		shmseg->shm_perm.uid = inbuf.shm_perm.uid;
-		shmseg->shm_perm.gid = inbuf.shm_perm.gid;
-		shmseg->shm_perm.mode =
-		    (shmseg->shm_perm.mode & ~ACCESSPERMS) |
+		shmseg->u.shm_perm.uid = inbuf.shm_perm.uid;
+		shmseg->u.shm_perm.gid = inbuf.shm_perm.gid;
+		shmseg->u.shm_perm.mode =
+		    (shmseg->u.shm_perm.mode & ~ACCESSPERMS) |
 		    (inbuf.shm_perm.mode & ACCESSPERMS);
-		shmseg->shm_ctime = sysv_shmtime();
+		shmseg->u.shm_ctime = sysv_shmtime();
 		break;
 	case IPC_RMID:
-		error = ipcperm(cred, &shmseg->shm_perm, IPC_M);
+		error = ipcperm(cred, &shmseg->u.shm_perm, IPC_M);
 		if (error) {
 			shmctl_ret = error;
 			goto shmctl_out;
 		}
-		shmseg->shm_perm.key = IPC_PRIVATE;
-		shmseg->shm_perm.mode |= SHMSEG_REMOVED;
-		if (shmseg->shm_nattch <= 0) {
+		shmseg->u.shm_perm._key = IPC_PRIVATE;
+		shmseg->u.shm_perm.mode |= SHMSEG_REMOVED;
+		if (shmseg->u.shm_nattch <= 0) {
 			shm_deallocate_segment(shmseg);
 			shm_last_free = IPCID_TO_IX(uap->shmid);
 		}
@@ -541,30 +604,47 @@ shmctl_out:
 static int
 shmget_existing(struct shmget_args *uap, int mode, int segnum, int *retval)
 {
-	struct user_shmid_ds *shmseg;
-	int error;
+	struct shmid_kernel *shmseg;
+	int error = 0;
 
 	shmseg = &shmsegs[segnum];
-	if (shmseg->shm_perm.mode & SHMSEG_REMOVED) {
+	if (shmseg->u.shm_perm.mode & SHMSEG_REMOVED) {
 		/*
 		 * This segment is in the process of being allocated.  Wait
 		 * until it's done, and look the key up again (in case the
 		 * allocation failed or it was freed).
 		 */
-		shmseg->shm_perm.mode |= SHMSEG_WANTED;
+		shmseg->u.shm_perm.mode |= SHMSEG_WANTED;
 		error = tsleep((caddr_t)shmseg, PLOCK | PCATCH, "shmget", 0);
 		if (error)
 			return error;
 		return EAGAIN;
 	}
-	error = ipcperm(kauth_cred_get(), &shmseg->shm_perm, mode);
-	if (error)
-		return error;
-	if (uap->size && uap->size > shmseg->shm_segsz)
+
+	/*
+	 * The low 9 bits of shmflag are the mode bits being requested, which
+	 * are the actual mode bits desired on the segment, and not in IPC_R
+	 * form; therefore it would be incorrect to call ipcperm() to validate
+	 * them; instead, we AND the existing mode with the requested mode, and
+	 * verify that it matches the requested mode; otherwise, we fail with
+	 * EACCES (access denied).
+	 */
+	if ((shmseg->u.shm_perm.mode & mode) != mode)
+		return EACCES;
+
+#if CONFIG_MACF
+	error = mac_sysvshm_check_shmget(kauth_cred_get(), shmseg, uap->shmflg);
+	if (error) 
+		return (error);
+#endif
+
+	if (uap->size && uap->size > shmseg->u.shm_segsz)
 		return EINVAL;
+
        if ((uap->shmflg & (IPC_CREAT | IPC_EXCL)) == (IPC_CREAT | IPC_EXCL))
 		return EEXIST;
-	*retval = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm);
+
+	*retval = IXSEQ_TO_IPCID(segnum, shmseg->u.shm_perm);
 	return 0;
 }
 
@@ -572,12 +652,13 @@ static int
 shmget_allocate_segment(struct proc *p, struct shmget_args *uap, int mode,
 	int *retval)
 {
-	int i, segnum, shmid, size;
+	int i, segnum, shmid;
 	kauth_cred_t cred = kauth_cred_get();
-	struct user_shmid_ds *shmseg;
+	struct shmid_kernel *shmseg;
 	struct shm_handle *shm_handle;
 	kern_return_t kret;
-	vm_offset_t user_addr;
+	mach_vm_offset_t user_addr;
+	mach_vm_size_t size;
 	void * mem_object;
 
 	if (uap->size < (user_size_t)shminfo.shmmin ||
@@ -586,11 +667,11 @@ shmget_allocate_segment(struct proc *p, struct shmget_args *uap, int mode,
 	if (shm_nused >= shminfo.shmmni) /* any shmids left? */
 		return ENOSPC;
 	size = mach_vm_round_page(uap->size);
-	if (shm_committed + btoc(size) > shminfo.shmall)
+	if ((user_ssize_t)(shm_committed + btoc(size)) > shminfo.shmall)
 		return ENOMEM;
 	if (shm_last_free < 0) {
 		for (i = 0; i < shminfo.shmmni; i++)
-			if (shmsegs[i].shm_perm.mode & SHMSEG_FREE)
+			if (shmsegs[i].u.shm_perm.mode & SHMSEG_FREE)
 				break;
 		if (i == shminfo.shmmni)
 			panic("shmseg free count inconsistent");
@@ -604,21 +685,24 @@ shmget_allocate_segment(struct proc *p, struct shmget_args *uap, int mode,
 	 * In case we sleep in malloc(), mark the segment present but deleted
 	 * so that noone else tries to create the same key.
 	 */
-	kret = vm_allocate(current_map(), &user_addr, size, VM_FLAGS_ANYWHERE);
+	kret = mach_vm_allocate(current_map(), &user_addr, size, VM_FLAGS_ANYWHERE);
 	if (kret != KERN_SUCCESS) 
 		goto out;
 
-	kret = mach_make_memory_entry (current_map(), &size, user_addr, 
-		VM_PROT_DEFAULT, (mem_entry_name_port_t *)&mem_object, 0);
+	kret = mach_make_memory_entry_64(current_map(),
+					(memory_object_size_t *)&size,
+					(memory_object_offset_t)user_addr, 
+					VM_PROT_DEFAULT,
+					(ipc_port_t *)&mem_object, 0);
 
 	if (kret != KERN_SUCCESS) 
 		goto out;
 
-	vm_deallocate(current_map(), user_addr, size);
+	mach_vm_deallocate(current_map(), user_addr, size);
 
-	shmseg->shm_perm.mode = SHMSEG_ALLOCATED | SHMSEG_REMOVED;
-	shmseg->shm_perm.key = uap->key;
-	shmseg->shm_perm.seq = (shmseg->shm_perm.seq + 1) & 0x7fff;
+	shmseg->u.shm_perm.mode = SHMSEG_ALLOCATED | SHMSEG_REMOVED;
+	shmseg->u.shm_perm._key = uap->key;
+	shmseg->u.shm_perm._seq = (shmseg->u.shm_perm._seq + 1) & 0x7fff;
 	MALLOC(shm_handle, struct shm_handle *, sizeof(struct shm_handle), M_SHM, M_WAITOK);
 	if (shm_handle == NULL) {
 		kret = KERN_NO_SPACE;
@@ -627,27 +711,30 @@ shmget_allocate_segment(struct proc *p, struct shmget_args *uap, int mode,
 		goto out;
 	}
 	shm_handle->shm_object = mem_object;
-	shmid = IXSEQ_TO_IPCID(segnum, shmseg->shm_perm);
+	shmid = IXSEQ_TO_IPCID(segnum, shmseg->u.shm_perm);
 
-	shmseg->shm_internal = CAST_USER_ADDR_T(shm_handle);	/* tunnel */
-	shmseg->shm_perm.cuid = shmseg->shm_perm.uid = kauth_cred_getuid(cred);
-	shmseg->shm_perm.cgid = shmseg->shm_perm.gid = cred->cr_gid;
-	shmseg->shm_perm.mode = (shmseg->shm_perm.mode & SHMSEG_WANTED) |
+	shmseg->u.shm_internal = CAST_USER_ADDR_T(shm_handle);	/* tunnel */
+	shmseg->u.shm_perm.cuid = shmseg->u.shm_perm.uid = kauth_cred_getuid(cred);
+	shmseg->u.shm_perm.cgid = shmseg->u.shm_perm.gid = cred->cr_gid;
+	shmseg->u.shm_perm.mode = (shmseg->u.shm_perm.mode & SHMSEG_WANTED) |
 	    (mode & ACCESSPERMS) | SHMSEG_ALLOCATED;
-	shmseg->shm_segsz = uap->size;
-	shmseg->shm_cpid = p->p_pid;
-	shmseg->shm_lpid = shmseg->shm_nattch = 0;
-	shmseg->shm_atime = shmseg->shm_dtime = 0;
-	shmseg->shm_ctime = sysv_shmtime();
+	shmseg->u.shm_segsz = uap->size;
+	shmseg->u.shm_cpid = p->p_pid;
+	shmseg->u.shm_lpid = shmseg->u.shm_nattch = 0;
+	shmseg->u.shm_atime = shmseg->u.shm_dtime = 0;
+#if CONFIG_MACF
+	mac_sysvshm_label_associate(cred, shmseg);
+#endif
+	shmseg->u.shm_ctime = sysv_shmtime();
 	shm_committed += btoc(size);
 	shm_nused++;
-	AUDIT_ARG(svipc_perm, &shmseg->shm_perm);
-	if (shmseg->shm_perm.mode & SHMSEG_WANTED) {
+	AUDIT_ARG(svipc_perm, &shmseg->u.shm_perm);
+	if (shmseg->u.shm_perm.mode & SHMSEG_WANTED) {
 		/*
 		 * Somebody else wanted this key while we were asleep.  Wake
 		 * them up now.
 		 */
-		shmseg->shm_perm.mode &= ~SHMSEG_WANTED;
+		shmseg->u.shm_perm.mode &= ~SHMSEG_WANTED;
 		wakeup((caddr_t)shmseg);
 	}
 	*retval = shmid;
@@ -677,8 +764,7 @@ shmget(struct proc *p, struct shmget_args *uap, register_t *retval)
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		shmget_ret = EINVAL;
-		goto shmget_out;
+		shminit(NULL);
 	}
 
 	mode = uap->shmflg & ACCESSPERMS;
@@ -731,8 +817,7 @@ shmfork(struct proc *p1, struct proc *p2)
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		shmfork_ret = 0;
-		goto shmfork_out;
+		shminit(NULL);
 	}
 		
 	size = shminfo.shmseg * sizeof(struct shmmap_state);
@@ -742,7 +827,7 @@ shmfork(struct proc *p1, struct proc *p2)
 		p2->vm_shm = (caddr_t)shmmap_s;
 		for (i = 0; i < shminfo.shmseg; i++, shmmap_s++)
 			if (shmmap_s->shmid != -1)
-				shmsegs[IPCID_TO_IX(shmmap_s->shmid)].shm_nattch++;
+				shmsegs[IPCID_TO_IX(shmmap_s->shmid)].u.shm_nattch++;
 		shmfork_ret = 0;
 		goto shmfork_out;
 	}
@@ -764,6 +849,10 @@ shmexit(struct proc *p)
 	SYSV_SHM_SUBSYS_LOCK();
 	for (i = 0; i < shminfo.shmseg; i++, shmmap_s++)
 		if (shmmap_s->shmid != -1)
+			/*
+			 * XXX: Should the MAC framework enforce
+			 * check here as well.
+			 */
 			shm_delete_mapping(p, shmmap_s, 1);
 	FREE((caddr_t)p->vm_shm, M_SHM);
 	p->vm_shm = NULL;
@@ -806,16 +895,19 @@ shminit(__unused void *dummy)
 		 * dictates this filed be a size_t, which is 64 bits when
 		 * running 64 bit binaries.
 		 */
-		s = sizeof(struct user_shmid_ds) * shminfo.shmmni;
+		s = sizeof(struct shmid_kernel) * shminfo.shmmni;
 
-		MALLOC(shmsegs, struct user_shmid_ds *, s, M_SHM, M_WAITOK);
+		MALLOC(shmsegs, struct shmid_kernel *, s, M_SHM, M_WAITOK);
 		if (shmsegs == NULL) {
 			/* XXX fail safely: leave shared memory uninited */
 			return;
 		}
 		for (i = 0; i < shminfo.shmmni; i++) {
-			shmsegs[i].shm_perm.mode = SHMSEG_FREE;
-			shmsegs[i].shm_perm.seq = 0;
+			shmsegs[i].u.shm_perm.mode = SHMSEG_FREE;
+			shmsegs[i].u.shm_perm._seq = 0;
+#if CONFIG_MACF
+			mac_sysvshm_label_init(&shmsegs[i]);
+#endif
 		}
 		shm_last_free = 0;
 		shm_nused = 0;
@@ -844,35 +936,32 @@ sysctl_shminfo(__unused struct sysctl_oid *oidp, void *arg1,
 {
 	int error = 0;
 	int sysctl_shminfo_ret = 0;
+	uint64_t	saved_shmmax;
 
 	error = SYSCTL_OUT(req, arg1, sizeof(int64_t));
 	if (error || req->newptr == USER_ADDR_NULL)
 		return(error);
 
 	SYSV_SHM_SUBSYS_LOCK();
-	/* Set the values only if shared memory is not initialised */
-	if (!shm_inited) {
-		if ((error = SYSCTL_IN(req, arg1, sizeof(int64_t))) 
-		    != 0) {
-			sysctl_shminfo_ret = error;
-			goto sysctl_shminfo_out;
-		}
 
-		if (arg1 == &shminfo.shmmax) {
-			if (shminfo.shmmax & PAGE_MASK_64) {
-				shminfo.shmmax = (int64_t)-1;
-				sysctl_shminfo_ret = EINVAL;
-				goto sysctl_shminfo_out;
-			}
-		}
+	/* shmmni can not be changed after SysV SHM has been initialized */
+	if (shm_inited && arg1 == &shminfo.shmmni) {
+		sysctl_shminfo_ret = EPERM;
+		goto sysctl_shminfo_out;
+	}
+	saved_shmmax = shminfo.shmmax;
 
-		/* Initialize only when all values are set */
-		if ((shminfo.shmmax != (int64_t)-1) &&
-			(shminfo.shmmin != (int64_t)-1) &&	
-			(shminfo.shmmni != (int64_t)-1) &&
-			(shminfo.shmseg != (int64_t)-1) &&
-			(shminfo.shmall != (int64_t)-1)) {
-				shminit(NULL);
+	if ((error = SYSCTL_IN(req, arg1, sizeof(int64_t))) != 0) {
+		sysctl_shminfo_ret = error;
+		goto sysctl_shminfo_out;
+	}
+
+	if (arg1 == &shminfo.shmmax) {
+		/* shmmax needs to be page-aligned */
+		if (shminfo.shmmax & PAGE_MASK_64) {
+			shminfo.shmmax = saved_shmmax;
+			sysctl_shminfo_ret = EINVAL;
+			goto sysctl_shminfo_out;
 		}
 	}
 	sysctl_shminfo_ret = 0;
@@ -897,13 +986,10 @@ IPCS_shm_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 	size_t shmid_ds_sz = sizeof(struct user_shmid_ds);
 	struct proc *p = current_proc();
 
-	int ipcs__shminfo_ret = 0;
-
 	SYSV_SHM_SUBSYS_LOCK();
 
 	if (!shm_inited) {
-		error = EINVAL;
-		goto ipcs_shm_sysctl_out;
+		shminit(NULL);
 	}
 
 	if (!IS_64BIT_PROCESS(p)) {
@@ -945,11 +1031,11 @@ IPCS_shm_sysctl(__unused struct sysctl_oid *oidp, __unused void *arg1,
 			break;
 		}
 		if (ipcs.u64.ipcs_datalen != (int)shmid_ds_sz) {
-			error = ENOMEM;
+			error = EINVAL;
 			break;
 		}
 		for( ; cursor < shminfo.shmmni; cursor++) {
-			if (shmsegs[cursor].shm_perm.mode & SHMSEG_ALLOCATED)
+			if (shmsegs[cursor].u.shm_perm.mode & SHMSEG_ALLOCATED)
 				break;
 			continue;
 		}
@@ -988,26 +1074,29 @@ ipcs_shm_sysctl_out:
 	return(error);
 }
 
-SYSCTL_NODE(_kern, KERN_SYSV, sysv, CTLFLAG_RW, 0, "SYSV");
+SYSCTL_NODE(_kern, KERN_SYSV, sysv, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "SYSV");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SHMMAX, shmmax, CTLTYPE_QUAD | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, shmmax, CTLTYPE_QUAD | CTLFLAG_RW,
     &shminfo.shmmax, 0, &sysctl_shminfo ,"Q","shmmax");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SHMMIN, shmmin, CTLTYPE_QUAD | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, shmmin, CTLTYPE_QUAD | CTLFLAG_RW,
     &shminfo.shmmin, 0, &sysctl_shminfo ,"Q","shmmin");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SHMMNI, shmmni, CTLTYPE_QUAD | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, shmmni, CTLTYPE_QUAD | CTLFLAG_RW,
     &shminfo.shmmni, 0, &sysctl_shminfo ,"Q","shmmni");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SHMSEG, shmseg, CTLTYPE_QUAD | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, shmseg, CTLTYPE_QUAD | CTLFLAG_RW,
     &shminfo.shmseg, 0, &sysctl_shminfo ,"Q","shmseg");
 
-SYSCTL_PROC(_kern_sysv, KSYSV_SHMALL, shmall, CTLTYPE_QUAD | CTLFLAG_RW,
+SYSCTL_PROC(_kern_sysv, OID_AUTO, shmall, CTLTYPE_QUAD | CTLFLAG_RW,
     &shminfo.shmall, 0, &sysctl_shminfo ,"Q","shmall");
 
-SYSCTL_NODE(_kern_sysv, OID_AUTO, ipcs, CTLFLAG_RW, 0, "SYSVIPCS");
+SYSCTL_NODE(_kern_sysv, OID_AUTO, ipcs, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "SYSVIPCS");
 
 SYSCTL_PROC(_kern_sysv_ipcs, OID_AUTO, shm, CTLFLAG_RW|CTLFLAG_ANYBODY,
 	0, 0, IPCS_shm_sysctl,
 	"S,IPCS_shm_command",
 	"ipcs shm command interface");
+#endif /* SYSV_SHM */
+
+/* DSEP Review Done pl-20051108-v02 @2743,@2908,@2913,@3009 */
diff --git a/bsd/kern/tty.c b/bsd/kern/tty.c
index 0798dcd98..59205429a 100644
--- a/bsd/kern/tty.c
+++ b/bsd/kern/tty.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
  * Copyright (c) 1982, 1986, 1990, 1991, 1993
  *      The Regents of the University of California.  All rights reserved.
@@ -86,13 +91,6 @@
  *	o Restore TS_WOPEN since it is useful in pstat.  It must be cleared
  *	  only when _all_ openers leave open().
  */
-#ifdef NeXT
-#define NSNP		0
-#else
-#include "snp.h"
-#include "opt_uconsole.h"
-#endif
-
 #include <sys/param.h>
 #define	TTYDEFCHARS 1
 #include <sys/systm.h>
@@ -110,37 +108,15 @@
 #include <sys/user.h>
 #include <sys/signalvar.h>
 #include <sys/signalvar.h>
-#ifndef NeXT
-#include <sys/resourcevar.h>
-#endif
 #include <sys/malloc.h>
-#if NSNP > 0
-#include <sys/snoop.h>
-#endif
 
-#ifndef NeXT
-#include <vm/vm.h>
-#include <vm/vm_param.h>
-#include <vm/vm_prot.h>
-#include <vm/lock.h>
-#include <vm/pmap.h>
-#include <vm/vm_map.h>
-#else
 #include <dev/kmreg_com.h>
 #include <machine/cons.h>
-#include <machine/spl.h>
-#if 0 /* [ */
-#include <machdep/machine/pmap.h>
-#endif  /* 0 ] */
-#endif /* !NeXT */
 #include <sys/resource.h>	/* averunnable */
 
-#ifndef NeXT
-static int	proc_compare(struct proc *p1, struct proc *p2);
-#endif /* NeXT */
 static int	ttnread(struct tty *tp);
 static void	ttyecho(int c, struct tty *tp);
-static int	ttyoutput(int c, register struct tty *tp);
+static int	ttyoutput(int c, struct tty *tp);
 static void	ttypend(struct tty *tp);
 static void	ttyretype(struct tty *tp);
 static void	ttyrub(int c, struct tty *tp);
@@ -148,7 +124,10 @@ static void	ttyrubo(struct tty *tp, int count);
 static void	ttystop(struct tty *tp, int rw);
 static void	ttyunblock(struct tty *tp);
 static int	ttywflush(struct tty *tp);
-static int	proc_compare(struct proc *p1, struct proc *p2);
+static int	proc_compare(proc_t p1, proc_t p2);
+
+static int isctty(proc_t p, struct tty  *tp);
+static int isctty_sp(proc_t p, struct tty  *tp, struct session *sessp);
 
 /*
  * Table with character classes and parity. The 8th bit indicates parity,
@@ -166,6 +145,8 @@ static int	proc_compare(struct proc *p1, struct proc *p2);
 
 #define	CCLASSMASK	0x3f
 #define	CCLASS(c)	(char_type[c] & CCLASSMASK)
+/* 0b10xxxxxx is the mask for UTF-8 continuations */
+#define	CCONT(c)	((c & 0xc0) == 0x80)
 
 #define	BS	BACKSPACE
 #define	CC	CONTROL
@@ -236,9 +217,6 @@ static u_char const char_type[] = {
 #define	I_HIGH_WATER	(TTYHOG - 2 * 256)	/* XXX */
 #define	I_LOW_WATER	((TTYHOG - 2 * 256) * 7 / 8)	/* XXX */
 
-#undef MAX_INPUT		/* XXX wrong in <sys/syslimits.h> */
-#define	MAX_INPUT	TTYHOG
-
 static void
 termios32to64(struct termios *in, struct user_termios *out)
 {
@@ -274,16 +252,17 @@ termios64to32(struct user_termios *in, struct termios *out)
  * Initial open of tty, or (re)entry to standard tty line discipline.
  */
 int
-ttyopen(device, tp)
-	dev_t device;
-	register struct tty *tp;
+ttyopen(dev_t device, struct tty *tp)
 {
-	int s;
 	boolean_t funnel_state;
+	proc_t p = current_proc();
+	struct pgrp * pg, * oldpg;
+	struct session *sessp, *oldsess;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-	s = spltty();
+
 	tp->t_dev = device;
+
 	if (!ISSET(tp->t_state, TS_ISOPEN)) {
 		SET(tp->t_state, TS_ISOPEN);
 		if (ISSET(tp->t_cflag, CLOCAL)) {
@@ -291,19 +270,49 @@ ttyopen(device, tp)
 		bzero(&tp->t_winsize, sizeof(tp->t_winsize));
 	}
 
-#ifndef NeXT
+	pg = proc_pgrp(p);
+	sessp = proc_session(p);
+
 	/*
-	 * Initialize or restore a cblock allocation policy suitable for
-	 * the standard line discipline.
+	 * First tty open affter setsid() call makes this tty its controlling
+	 * tty, if the tty does not already have a session associated with it.
+	 * Only do this if the process
 	 */
-	clist_alloc_cblocks(&tp->t_canq, TTYHOG, 512);
-	clist_alloc_cblocks(&tp->t_outq, TTMAXHIWAT + OBUFSIZ + 100,
-			    TTMAXHIWAT + OBUFSIZ + 100);
-	clist_alloc_cblocks(&tp->t_rawq, TTYHOG, TTYHOG);
-#endif /* !NeXT */
+	if (SESS_LEADER(p, sessp) &&			/* process is session leader */
+	    sessp->s_ttyvp == NULL &&	/* but has no controlling tty */
+	    tp->t_session == NULL ) {		/* and tty not controlling */
+		session_lock(sessp);
+	    	if ((sessp->s_flags & S_NOCTTY) == 0) {	/* and no O_NOCTTY */
+			/* Hold on to the reference */
+			sessp->s_ttyp = tp;
+			OSBitOrAtomic(P_CONTROLT, (UInt32 *)&p->p_flag);
+			session_unlock(sessp);
+			proc_list_lock();
+			oldpg = tp->t_pgrp;
+			oldsess = tp->t_session;
+			if (oldsess != SESSION_NULL)
+				oldsess->s_ttypgrpid = NO_PID;
+			tp->t_session = sessp;
+			tp->t_pgrp = pg;
+			sessp->s_ttypgrpid = pg->pg_id;
+			proc_list_unlock();
+			if (oldpg != PGRP_NULL)
+				pg_rele(oldpg);
+			if (oldsess != SESSION_NULL)
+				session_rele(oldsess);	
+			goto out;
+	     	}
+		session_unlock(sessp);
+	}
 
-	splx(s);
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+	if (pg != PGRP_NULL)
+		pg_rele(pg);
+
+out:
 	thread_funnel_set(kernel_flock, funnel_state);
+
 	return (0);
 }
 
@@ -316,50 +325,43 @@ ttyopen(device, tp)
  * the flush in case there are buggy callers.
  */
 int
-ttyclose(tp)
-	register struct tty *tp;
+ttyclose(struct tty *tp)
 {
-	int s;
+	struct pgrp * oldpg;
+	struct session * oldsessp;
 
-	s = spltty();
 	if (constty == tp) {
 		constty = NULL;
 
-splx(s);
-spltty();
 
-#ifdef NeXT
 		/*
 		 * Closing current console tty; disable printing of console
 		 * messages at bottom-level driver. 
 		 */
 		(*cdevsw[major(tp->t_dev)].d_ioctl)
 			(tp->t_dev, KMIOCDISABLCONS, NULL, 0, current_proc());
-#endif	/* NeXT */
 	}
 
 	ttyflush(tp, FREAD | FWRITE);
-#ifndef NeXT
-	clist_free_cblocks(&tp->t_canq);
-	clist_free_cblocks(&tp->t_outq);
-	clist_free_cblocks(&tp->t_rawq);
-#endif
-
-#if NSNP > 0
-	if (ISSET(tp->t_state, TS_SNOOP) && tp->t_sc != NULL)
-		snpdown((struct snoop *)tp->t_sc);
-#endif
 
 	tp->t_gen++;
 	tp->t_line = TTYDISC;
+	proc_list_lock();
+	oldpg = tp->t_pgrp;
+	oldsessp = tp->t_session;
 	tp->t_pgrp = NULL;
 	tp->t_session = NULL;
+	if (oldsessp != SESSION_NULL)
+		oldsessp->s_ttypgrpid = NO_PID;
+	proc_list_unlock();
+	/* drop the reference on prev session and pgrp */
+	if (oldsessp != SESSION_NULL)
+		session_rele(oldsessp);
+	if (oldpg != PGRP_NULL)
+		pg_rele(oldpg);
 	tp->t_state = 0;
-#if NeXT
 	selthreadclear(&tp->t_wsel);
 	selthreadclear(&tp->t_rsel);
-#endif
-	splx(s);
 	return (0);
 }
 
@@ -378,12 +380,10 @@ spltty();
  * Process input of a single character received on a tty.
  */
 int
-ttyinput(c, tp)
-	register int c;
-	register struct tty *tp;
+ttyinput(int c, struct tty *tp)
 {
-	register tcflag_t iflag, lflag;
-	register cc_t *cc;
+	tcflag_t iflag, lflag;
+	cc_t *cc;
 	int i, err, retval;
 	boolean_t funnel_state;
 
@@ -432,7 +432,7 @@ ttyinput(c, tp)
                         }
 			if (ISSET(iflag, BRKINT)) {
 				ttyflush(tp, FREAD | FWRITE);
-				pgsignal(tp->t_pgrp, SIGINT, 1);
+				tty_pgsignal(tp, SIGINT, 1);
 				goto endcase;
 			}
 			if (ISSET(iflag, PARMRK))
@@ -512,7 +512,7 @@ parmrk:
 				if (!ISSET(lflag, NOFLSH))
 					ttyflush(tp, FREAD | FWRITE);
 				ttyecho(c, tp);
-				pgsignal(tp->t_pgrp,
+				tty_pgsignal(tp,
 				    CCEQ(cc[VINTR], c) ? SIGINT : SIGQUIT, 1);
 				goto endcase;
 			}
@@ -520,7 +520,7 @@ parmrk:
 				if (!ISSET(lflag, NOFLSH))
 					ttyflush(tp, FREAD);
 				ttyecho(c, tp);
-				pgsignal(tp->t_pgrp, SIGTSTP, 1);
+				tty_pgsignal(tp, SIGTSTP, 1);
 				goto endcase;
 			}
 		}
@@ -569,8 +569,15 @@ parmrk:
 		 * erase (^H / ^?)
 		 */
 		if (CCEQ(cc[VERASE], c)) {
-			if (tp->t_rawq.c_cc)
-				ttyrub(unputc(&tp->t_rawq), tp);
+			if (tp->t_rawq.c_cc) {
+				if (ISSET(iflag, IUTF8)) {
+					do {
+						ttyrub((c = unputc(&tp->t_rawq)), tp);
+					} while(tp->t_rawq.c_cc && CCONT(c));
+				} else {
+					ttyrub(unputc(&tp->t_rawq), tp);
+				}
+			}
 			goto endcase;
 		}
 		/*
@@ -644,7 +651,7 @@ parmrk:
 		 */
 		if (CCEQ(cc[VSTATUS], c) && ISSET(lflag, IEXTEN)) {
 			if (ISSET(lflag, ISIG))
-				pgsignal(tp->t_pgrp, SIGINFO, 1);
+				tty_pgsignal(tp, SIGINFO, 1);
 			if (!ISSET(lflag, NOKERNINFO))
 				ttyinfo(tp);
 			goto endcase;
@@ -727,12 +734,10 @@ startoutput:
  * Must be recursive.
  */
 static int
-ttyoutput(c, tp)
-	register int c;
-	register struct tty *tp;
+ttyoutput(int c, struct tty *tp)
 {
-	register tcflag_t oflag;
-	register int col, s;
+	tcflag_t oflag;
+	int col;
 
 	oflag = tp->t_oflag;
 	if (!ISSET(oflag, OPOST)) {
@@ -753,16 +758,14 @@ ttyoutput(c, tp)
 	CLR(c, ~TTY_CHARMASK);
 	if (c == '\t' &&
 	    ISSET(oflag, OXTABS) && !ISSET(tp->t_lflag, EXTPROC)) {
-		c = 8 - (tp->t_column & 7);
+		col = c = 8 - (tp->t_column & 7);
 		if (!ISSET(tp->t_lflag, FLUSHO)) {
-			s = spltty();		/* Don't interrupt tabs. */
-			c -= b_to_q("        ", c, &tp->t_outq);
+			c -= b_to_q((const u_char *)"        ", c, &tp->t_outq);
 			tk_nout += c;
 			tp->t_outcc += c;
-			splx(s);
 		}
 		tp->t_column += c;
-		return (c ? -1 : '\t');
+		return (c == col ? -1 : '\t');
 	}
 	if (c == CEOT && ISSET(oflag, ONOEOT))
 		return (-1);
@@ -777,6 +780,12 @@ ttyoutput(c, tp)
 		if (putc('\r', &tp->t_outq))
 			return (c);
 	}
+        /* If OCRNL is set, translate "\r" into "\n". */
+        else if (c == '\r' && ISSET(tp->t_oflag, OCRNL))
+                c = '\n';
+        /* If ONOCR is set, don't transmit CRs when on column 0. */
+        else if (c == '\r' && ISSET(tp->t_oflag, ONOCR) && tp->t_column == 0)
+                return (-1);
 	tk_nout++;
 	tp->t_outcc++;
 	if (!ISSET(tp->t_lflag, FLUSHO) && putc(c, &tp->t_outq))
@@ -810,19 +819,25 @@ ttyoutput(c, tp)
  * has been called to do discipline-specific functions and/or reject any
  * of these ioctl commands.
  */
-/* ARGSUSED */
 int
-ttioctl(register struct tty *tp,
+ttioctl(struct tty *tp,
 	u_long cmd, caddr_t data, int flag,
-	struct proc *p)
+	proc_t p)
 {
-	int s, error;
+	int error = 0;
 	struct uthread *ut;
+	struct pgrp * pg, *oldpg;
+	struct session *sessp, * oldsessp;
 
 	ut = (struct uthread *)get_bsdthread_info(current_thread());
-	/* If the ioctl involves modification, hang if in the background. */
+	/* If the ioctl involves modification, signal if in the background. */
 	switch (cmd) {
+	case TIOCIXON:
+	case TIOCIXOFF:
+	case  TIOCDRAIN:
 	case  TIOCFLUSH:
+	case TIOCSTOP:
+	case TIOCSTART:
 	case  TIOCSETA:
 	case  TIOCSETA_64:
 	case  TIOCSETD:
@@ -830,9 +845,7 @@ ttioctl(register struct tty *tp,
 	case  TIOCSETAF_64:
 	case  TIOCSETAW:
 	case  TIOCSETAW_64:
-#ifdef notdef
 	case  TIOCSPGRP:
-#endif
 	case  TIOCSTAT:
 	case  TIOCSTI:
 	case  TIOCSWINSZ:
@@ -847,43 +860,52 @@ ttioctl(register struct tty *tp,
 	case  TIOCSLTC:
 #endif
 		while (isbackground(p, tp) &&
-		    (p->p_flag & P_PPWAIT) == 0 &&
+		    (p->p_lflag & P_LPPWAIT) == 0 &&
 		    (p->p_sigignore & sigmask(SIGTTOU)) == 0 &&
 		    (ut->uu_sigmask & sigmask(SIGTTOU)) == 0) {
-			if (p->p_pgrp->pg_jobc == 0)
-				return (EIO);
-			pgsignal(p->p_pgrp, SIGTTOU, 1);
-			error = ttysleep(tp, &lbolt, TTOPRI | PCATCH | PTTYBLOCK, "ttybg1",
-					 0);
-			if (error)
-				return (error);
+			pg = proc_pgrp(p);
+			if (pg == PGRP_NULL) {
+				error = EIO;
+				goto out;
+			}
+			if (pg->pg_jobc == 0) {
+				pg_rele(pg);
+				error = EIO;
+				goto out;
+			}
+			pgsignal(pg, SIGTTOU, 1);
+			pg_rele(pg);
+
+
+			/*
+			 * We signalled ourself, so we need to act as if we
+			 * have been "interrupted" from a "sleep" to act on
+			 * the signal.  If it's a signal that stops the
+			 * process, that's handled in the signal sending code.
+			 */
+			error = EINTR;
+			goto out;
 		}
 		break;
 	}
 
 	switch (cmd) {			/* Process the ioctl. */
 	case FIOASYNC:			/* set/clear async i/o */
-		s = spltty();
 		if (*(int *)data)
 			SET(tp->t_state, TS_ASYNC);
 		else
 			CLR(tp->t_state, TS_ASYNC);
-		splx(s);
 		break;
 	case FIONBIO:			/* set/clear non-blocking i/o */
 		break;			/* XXX: delete. */
 	case FIONREAD:			/* get # bytes to read */
-		s = spltty();
 		*(int *)data = ttnread(tp);
-		splx(s);
 		break;
 	case TIOCEXCL:			/* set exclusive use of tty */
-		s = spltty();
 		SET(tp->t_state, TS_XCLUDE);
-		splx(s);
 		break;
 	case TIOCFLUSH: {		/* flush buffers */
-		register int flags = *(int *)data;
+		int flags = *(int *)data;
 
 		if (flags == 0)
 			flags = FREAD | FWRITE;
@@ -892,7 +914,6 @@ ttioctl(register struct tty *tp,
 		ttyflush(tp, flags);
 		break;
 	}
-#ifdef	NeXT
 	case TIOCSCONS: {
 		/* Set current console device to this line */
 		int bogusData = 1;
@@ -900,22 +921,19 @@ ttioctl(register struct tty *tp,
 
 		/* No break - Fall through to BSD code */
 	}
-#endif /* NeXT */
 	case TIOCCONS: {			/* become virtual console */
 		if (*(int *)data) {
 			if (constty && constty != tp &&
 			    ISSET(constty->t_state, TS_CONNECTED)) {
-				return (EBUSY);
+				error = EBUSY;
+				goto out;
 			}
-#if defined(NeXT) || !defined(UCONSOLE)
-			if ( (error = suser(kauth_cred_get(), &p->p_acflag)) )
-				return (error);
-#endif
+			if ( (error = suser(kauth_cred_get(), &p->p_acflag)) ) 
+				goto out;
 			constty = tp;
 		} else if (tp == constty) {
 			constty = NULL;
 		}
-#ifdef	NeXT
 		if (constty) {
 			(*cdevsw[major(cons.t_dev)].d_ioctl)
 				(cons.t_dev, KMIOCDISABLCONS, NULL, 0, p);
@@ -923,13 +941,12 @@ ttioctl(register struct tty *tp,
 			(*cdevsw[major(tp->t_dev)].d_ioctl)
 				(tp->t_dev, KMIOCDISABLCONS, NULL, 0, p);
 		}
-#endif /* NeXT */
 		break;
 	}
 	case TIOCDRAIN:			/* wait till output drained */
 		error = ttywait(tp);
 		if (error)
-			return (error);
+			goto out;
 		break;
 	case TIOCGETA:		/* get termios struct */
 	case TIOCGETA_64: {		/* get termios struct */
@@ -947,21 +964,19 @@ ttioctl(register struct tty *tp,
 		*(struct winsize *)data = tp->t_winsize;
 		break;
 	case TIOCGPGRP:			/* get pgrp of tty */
-		if (!isctty(p, tp))
-			return (ENOTTY);
+		if (!isctty(p, tp)) {
+			error = ENOTTY;
+			goto out;
+		}
 		*(int *)data = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PID;
 		break;
 #ifdef TIOCHPCL
 	case TIOCHPCL:			/* hang up on last close */
-		s = spltty();
 		SET(tp->t_cflag, HUPCL);
-		splx(s);
 		break;
 #endif
 	case TIOCNXCL:			/* reset exclusive use of tty */
-		s = spltty();
 		CLR(tp->t_state, TS_XCLUDE);
-		splx(s);
 		break;
 	case TIOCOUTQ:			/* output queue size */
 		*(int *)data = tp->t_outq.c_cc;
@@ -972,22 +987,27 @@ ttioctl(register struct tty *tp,
 	case TIOCSETAW_64:
 	case TIOCSETAF:		/* drn out, fls in, set */
 	case TIOCSETAF_64: {		/* drn out, fls in, set */
-		register struct termios *t = (struct termios *)data;
+		struct termios *t = (struct termios *)data;
 		struct termios lcl_termios;
 
 		if (IS_64BIT_PROCESS(p)) {
 			termios64to32((struct user_termios *)data, &lcl_termios);
 			t = &lcl_termios;
 		}
-		if (t->c_ispeed < 0 || t->c_ospeed < 0)
-			return (EINVAL);
-		s = spltty();
+#if 0
+	/* XXX bogus test; always false */
+		if (t->c_ispeed < 0 || t->c_ospeed < 0) {
+			error = EINVAL;
+			goto out;
+		}
+#endif	/* 0 - leave in; may end up being a conformance issue */
+		if (t->c_ispeed == 0)
+			t->c_ispeed = t->c_ospeed;
 		if (cmd == TIOCSETAW || cmd == TIOCSETAF ||
 		    cmd == TIOCSETAW_64 || cmd == TIOCSETAF_64) {
 			error = ttywait(tp);
 			if (error) {
-				splx(s);
-				return (error);
+				goto out;
 			}
 			if (cmd == TIOCSETAF || cmd == TIOCSETAF_64)
 				ttyflush(tp, FREAD);
@@ -997,8 +1017,7 @@ ttioctl(register struct tty *tp,
 			 * Set device hardware.
 			 */
 			if (tp->t_param && (error = (*tp->t_param)(tp, t))) {
-				splx(s);
-				return (error);
+				goto out;
 			}
 			if (ISSET(t->c_cflag, CLOCAL) &&
 			    !ISSET(tp->t_cflag, CLOCAL)) {
@@ -1036,18 +1055,6 @@ ttioctl(register struct tty *tp,
 				 * discipline.  Now we have to worry about
 				 * panicing for a null queue.
 				 */
-#ifndef NeXT
-				if (tp->t_canq.c_cbreserved > 0 &&
-				    tp->t_rawq.c_cbreserved > 0) {
-					catq(&tp->t_rawq, &tp->t_canq);
-					/*
-					 * XXX the queue limits may be
-					 * different, so the old queue
-					 * swapping method no longer works.
-					 */
-					catq(&tp->t_canq, &tp->t_rawq);
-				}
-#else
 				if (tp->t_rawq.c_cs && tp->t_canq.c_cs) {
 				    struct clist tq;
 
@@ -1056,7 +1063,6 @@ ttioctl(register struct tty *tp,
 				    tp->t_rawq = tp->t_canq;
 				    tp->t_canq = tq;
 				}
-#endif /* !NeXT */
 				CLR(tp->t_lflag, PENDIN);
 			}
 			ttwakeup(tp);
@@ -1075,153 +1081,183 @@ ttioctl(register struct tty *tp,
 		    t->c_cc[VTIME] != tp->t_cc[VTIME])
 			ttwakeup(tp);
 		bcopy(t->c_cc, tp->t_cc, sizeof(t->c_cc));
-		splx(s);
 		break;
 	}
 	case TIOCSETD: {		/* set line discipline */
-		register int t = *(int *)data;
+		int t = *(int *)data;
 		dev_t device = tp->t_dev;
 
-		if (t >= nlinesw || t < 0)
-			return (ENXIO);
+		if (t >= nlinesw || t < 0) {
+			error = ENXIO;
+			goto out;
+		}
 		if (t != tp->t_line) {
-			s = spltty();
 			(*linesw[tp->t_line].l_close)(tp, flag);
 			error = (*linesw[t].l_open)(device, tp);
 			if (error) {
 				(void)(*linesw[tp->t_line].l_open)(device, tp);
-				splx(s);
-				return (error);
+				goto out;
 			}
 			tp->t_line = t;
-			splx(s);
 		}
 		break;
 	}
 	case TIOCSTART:			/* start output, like ^Q */
-		s = spltty();
 		if (ISSET(tp->t_state, TS_TTSTOP) ||
 		    ISSET(tp->t_lflag, FLUSHO)) {
 			CLR(tp->t_lflag, FLUSHO);
 			CLR(tp->t_state, TS_TTSTOP);
 			ttstart(tp);
 		}
-		splx(s);
 		break;
 	case TIOCSTI:			/* simulate terminal input */
-		if (suser(kauth_cred_get(), NULL) && (flag & FREAD) == 0)
-			return (EPERM);
-		if (suser(kauth_cred_get(), NULL) && !isctty(p, tp))
-			return (EACCES);
-		s = spltty();
+		if (suser(kauth_cred_get(), NULL) && (flag & FREAD) == 0) {
+			error = EPERM;
+			goto out;
+		}
+		if (suser(kauth_cred_get(), NULL) && !isctty(p, tp)) {
+			error = EACCES;
+			goto out;
+		}
 		(*linesw[tp->t_line].l_rint)(*(u_char *)data, tp);
-		splx(s);
 		break;
 	case TIOCSTOP:			/* stop output, like ^S */
-		s = spltty();
 		if (!ISSET(tp->t_state, TS_TTSTOP)) {
 			SET(tp->t_state, TS_TTSTOP);
                         ttystop(tp, 0);
 		}
-		splx(s);
+		break;
+	case TIOCIXON:
+		ttyunblock(tp);
+		break;
+	case TIOCIXOFF:
+		ttyblock(tp);
 		break;
 	case TIOCSCTTY:			/* become controlling tty */
 		/* Session ctty vnode pointer set in vnode layer. */
-		if (!SESS_LEADER(p) ||
-		    ((p->p_session->s_ttyvp || tp->t_session) &&
-		    (tp->t_session != p->p_session)))
-			return (EPERM);
-		tp->t_session = p->p_session;
-		tp->t_pgrp = p->p_pgrp;
-		p->p_session->s_ttyp = tp;
-		p->p_flag |= P_CONTROLT;
-		/* The backgrounded process blocking on tty now 
-		 * could be foregound process. Wake such processes
-		 */
-		tty_pgsignal(tp->t_pgrp, SIGCONT);
+		pg = proc_pgrp(p);
+		sessp = proc_session(p);
+		if (!SESS_LEADER(p, sessp) ||
+		    ((sessp->s_ttyvp || tp->t_session) &&
+		    (tp->t_session != sessp))) {
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			if (pg != PGRP_NULL)
+				pg_rele(pg);
+			error = EPERM;
+			goto out;
+		}
+		proc_list_lock();
+		oldsessp = tp->t_session;
+		oldpg = tp->t_pgrp;
+		if (oldsessp != SESSION_NULL)
+			oldsessp->s_ttypgrpid = NO_PID;
+		/* do not drop refs on sessp and pg as tp holds them */
+		tp->t_session = sessp;
+		sessp->s_ttypgrpid = pg->pg_id;
+		tp->t_pgrp = pg;
+		proc_list_unlock();
+		session_lock(sessp);
+		sessp->s_ttyp = tp;
+		session_unlock(sessp);
+		OSBitOrAtomic(P_CONTROLT, (UInt32 *)&p->p_flag);
+		/* drop the reference on prev session and pgrp */
+		if (oldsessp != SESSION_NULL)
+			session_rele(oldsessp);
+		if (oldpg != PGRP_NULL)
+			pg_rele(oldpg);
 		break;
+
 	case TIOCSPGRP: {		/* set pgrp of tty */
-		register struct pgrp *pgrp = pgfind(*(int *)data);
+		struct pgrp *pgrp = PGRP_NULL;
 
-		if (!isctty(p, tp))
-			return (ENOTTY);
-		else if (pgrp == NULL || pgrp->pg_session != p->p_session)
-			return (EPERM);
+		sessp = proc_session(p);
+		if (!isctty_sp(p, tp, sessp)) {
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			error = ENOTTY;
+			goto out;
+		}
+		else if ((pgrp = pgfind(*(int *)data)) == PGRP_NULL) {
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			error = EINVAL;
+			goto out;
+		 } else if (pgrp->pg_session != sessp) {
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			pg_rele(pgrp);
+			error = EPERM;
+			goto out;
+		}
+		proc_list_lock();
+		oldpg = tp->t_pgrp;
 		tp->t_pgrp = pgrp;
-		/* The backgrounded process blocking on tty now 
-		 * could be foregound process. Wake such processes
-		 */
-		tty_pgsignal(tp->t_pgrp, SIGCONT);
+		sessp->s_ttypgrpid = pgrp->pg_id;
+		proc_list_unlock();
+		if (oldpg != PGRP_NULL)
+			pg_rele(oldpg);
+		if (sessp != SESSION_NULL)
+			session_rele(sessp);
 		break;
 	}
 	case TIOCSTAT:			/* simulate control-T */
-		s = spltty();
 		ttyinfo(tp);
-		splx(s);
 		break;
 	case TIOCSWINSZ:		/* set window size */
 		if (bcmp((caddr_t)&tp->t_winsize, data,
 		    sizeof (struct winsize))) {
 			tp->t_winsize = *(struct winsize *)data;
-			pgsignal(tp->t_pgrp, SIGWINCH, 1);
+			tty_pgsignal(tp, SIGWINCH, 1);
 		}
 		break;
 	case TIOCSDRAINWAIT:
 		error = suser(kauth_cred_get(), &p->p_acflag);
-		if (error)
-			return (error);
+		if (error) {
+			goto out;
+		}
 		tp->t_timeout = *(int *)data * hz;
 		wakeup(TSA_OCOMPLETE(tp));
 		wakeup(TSA_OLOWAT(tp));
 		break;
 	case TIOCGDRAINWAIT:
 		*(int *)data = tp->t_timeout / hz;
-		break;
+		break; 
 	default:
 #if COMPAT_43_TTY || defined(COMPAT_SUNOS)
-#ifdef NeXT
-		return (ttcompat(tp, cmd, data, flag, p));
-#else
-		return (ttcompat(tp, cmd, data, flag));
-#endif /* NeXT */
+		error = ttcompat(tp, cmd, data, flag, p);
 #else
-		return (ENOTTY);
+		error = ENOTTY;
 #endif
+		goto out;
 	}
 
-	return (0);
+	error = 0;
+out:
+	return(error);
 }
 
 int
-ttyselect(tp, rw, wql, p)
-	struct tty *tp;
-	int rw;
-	void * wql;
-	struct proc *p;
+ttyselect(struct tty *tp, int rw, void *wql, proc_t p)
 {
-	int s;
-
 	if (tp == NULL)
 		return (ENXIO);
 
-	s = spltty();
 	switch (rw) {
 	case FREAD:
 		if (ttnread(tp) > 0 || ISSET(tp->t_state, TS_ZOMBIE))
-			goto win;
+			return(1);
 		selrecord(p, &tp->t_rsel, wql);
 		break;
 	case FWRITE:
 		if ((tp->t_outq.c_cc <= tp->t_lowat &&
 		     ISSET(tp->t_state, TS_CONNECTED))
 		    || ISSET(tp->t_state, TS_ZOMBIE)) {
-win:			splx(s);
 			return (1);
 		}
 		selrecord(p, &tp->t_wsel, wql);
 		break;
 	}
-	splx(s);
 	return (0);
 }
 
@@ -1230,25 +1266,16 @@ win:			splx(s);
  * cdevsw.  It relies on a proper xxxdevtotty routine.
  */
 int
-ttselect(dev, rw, wql, p)
-	dev_t dev;
-	int rw;
-	void * wql;
-	struct proc *p;
+ttselect(dev_t dev, int rw, void *wql, proc_t p)
 {
-#ifndef NeXT
-	return ttyselect((*cdevsw[major(dev)]->d_devtotty)(dev), rw, wql, p);
-#else
 	return ttyselect(cdevsw[major(dev)].d_ttys[minor(dev)], rw, wql, p);
-#endif
 }
 
 /*
  * Must be called at spltty().
  */
 static int
-ttnread(tp)
-	struct tty *tp;
+ttnread(struct tty *tp)
 {
 	int nread;
 
@@ -1267,13 +1294,11 @@ ttnread(tp)
  * Wait for output to drain.
  */
 int
-ttywait(tp)
-	register struct tty *tp;
+ttywait(struct tty *tp)
 {
-	int error, s;
+	int error;
 
 	error = 0;
-	s = spltty();
 	while ((tp->t_outq.c_cc || ISSET(tp->t_state, TS_BUSY)) &&
 	       ISSET(tp->t_state, TS_CONNECTED) && tp->t_oproc) {
 		(*tp->t_oproc)(tp);
@@ -1293,30 +1318,20 @@ ttywait(tp)
 	}
 	if (!error && (tp->t_outq.c_cc || ISSET(tp->t_state, TS_BUSY)))
 		error = EIO;
-	splx(s);
 	return (error);
 }
 
 static void
-ttystop(tp, rw)
-        struct tty *tp;
-        int rw;
+ttystop(struct tty *tp, int rw)
 {
-#ifdef sun4c						/* XXX */
-	(*tp->t_stop)(tp, rw);
-#elif defined(NeXT)
 	(*cdevsw[major(tp->t_dev)].d_stop)(tp, rw);
-#else
-	(*cdevsw[major(tp->t_dev)]->d_stop)(tp, rw);
-#endif
 }
 
 /*
  * Flush if successfully wait.
  */
 static int
-ttywflush(tp)
-	struct tty *tp;
+ttywflush(struct tty *tp)
 {
 	int error;
 
@@ -1329,13 +1344,8 @@ ttywflush(tp)
  * Flush tty read and/or write queues, notifying anyone waiting.
  */
 void
-ttyflush(tp, rw)
-	register struct tty *tp;
-	int rw;
+ttyflush(struct tty *tp, int rw)
 {
-	register int s;
-
-	s = spltty();
 #if 0
 again:
 #endif
@@ -1393,17 +1403,14 @@ again:
 		FLUSHQ(&tp->t_outq);
 		ttwwakeup(tp);
 	}
-	splx(s);
 }
 
 /*
  * Copy in the default termios characters.
  */
 void
-termioschars(t)
-	struct termios *t;
+termioschars(struct termios *t)
 {
-
 	bcopy(ttydefchars, t->c_cc, sizeof t->c_cc);
 }
 
@@ -1411,10 +1418,8 @@ termioschars(t)
  * Old interface.
  */
 void
-ttychars(tp)
-	struct tty *tp;
+ttychars(struct tty *tp)
 {
-
 	termioschars(&tp->t_termios);
 }
 
@@ -1424,10 +1429,8 @@ ttychars(tp)
  * XXX the stop character should be put in a special high priority queue.
  */
 void
-ttyblock(tp)
-	struct tty *tp;
+ttyblock(struct tty *tp)
 {
-
 	SET(tp->t_state, TS_TBLOCK);
 	if (ISSET(tp->t_iflag, IXOFF) && tp->t_cc[VSTOP] != _POSIX_VDISABLE &&
 	    putc(tp->t_cc[VSTOP], &tp->t_outq) != 0)
@@ -1441,10 +1444,8 @@ ttyblock(tp)
  * XXX the start character should be put in a special high priority queue.
  */
 static void
-ttyunblock(tp)
-	struct tty *tp;
+ttyunblock(struct tty *tp)
 {
-
 	CLR(tp->t_state, TS_TBLOCK);
 	if (ISSET(tp->t_iflag, IXOFF) && tp->t_cc[VSTART] != _POSIX_VDISABLE &&
 	    putc(tp->t_cc[VSTART], &tp->t_outq) != 0)
@@ -1452,35 +1453,28 @@ ttyunblock(tp)
 	ttstart(tp);
 }
 
-#if defined(NeXT) || defined(notyet)
 /* FreeBSD: Not used by any current (i386) drivers. */
 /*
  * Restart after an inter-char delay.
  */
 void
-ttrstrt(tp_arg)
-	void *tp_arg;
+ttrstrt(void *tp_arg)
 {
 	struct tty *tp;
-	int s;
 
 #if DIAGNOSTIC
 	if (tp_arg == NULL)
 		panic("ttrstrt");
 #endif
 	tp = tp_arg;
-	s = spltty();
 
 	CLR(tp->t_state, TS_TIMEOUT);
 	ttstart(tp);
 
-	splx(s);
 }
-#endif /* NeXT || notyet */
 
 int
-ttstart(tp)
-	struct tty *tp;
+ttstart(struct tty *tp)
 {
 	boolean_t funnel_state;
 
@@ -1496,9 +1490,7 @@ ttstart(tp)
  * "close" a line discipline
  */
 int
-ttylclose(tp, flag)
-	struct tty *tp;
-	int flag;
+ttylclose(struct tty *tp, int flag)
 {
 	boolean_t funnel_state;
 
@@ -1515,9 +1507,7 @@ ttylclose(tp, flag)
  * Returns 0 if the line should be turned off, otherwise 1.
  */
 int
-ttymodem(tp, flag)
-	register struct tty *tp;
-	int flag;
+ttymodem(struct tty *tp, int flag)
 {
 	boolean_t funnel_state;
 
@@ -1573,30 +1563,16 @@ ttymodem(tp, flag)
  * call at spltty().
  */
 static void
-ttypend(tp)
-	register struct tty *tp;
+ttypend(struct tty *tp)
 {
 	struct clist tq;
-	register int c;
+	int c;
 
 	CLR(tp->t_lflag, PENDIN);
 	SET(tp->t_state, TS_TYPEN);
-#ifndef NeXT
-	/*
-	 * XXX this assumes too much about clist internals.  It may even
-	 * fail if the cblock slush pool is empty.  We can't allocate more
-	 * cblocks here because we are called from an interrupt handler
-	 * and clist_alloc_cblocks() can wait.
-	 */
-	tq = tp->t_rawq;
-	bzero(&tp->t_rawq, sizeof tp->t_rawq);
-	tp->t_rawq.c_cbmax = tq.c_cbmax;
-	tp->t_rawq.c_cbreserved = tq.c_cbreserved;
-#else
 	tq = tp->t_rawq;
 	tp->t_rawq.c_cc = 0;
-	tp->t_rawq.c_cf = tp->t_rawq.c_cl = 0;
-#endif /* !NeXT */
+	tp->t_rawq.c_cf = tp->t_rawq.c_cl = NULL;
 	while ((c = getc(&tq)) >= 0)
 		ttyinput(c, tp);
 	CLR(tp->t_state, TS_TYPEN);
@@ -1606,61 +1582,68 @@ ttypend(tp)
  * Process a read call on a tty device.
  */
 int
-ttread(tp, uio, flag)
-	register struct tty *tp;
-	struct uio *uio;
-	int flag;
+ttread(struct tty *tp, struct uio *uio, int flag)
 {
-	register struct clist *qp;
-	register int c;
-	register tcflag_t lflag;
-	register cc_t *cc = tp->t_cc;
-	register struct proc *p = current_proc();
-	int s, first, error = 0;
+	struct clist *qp;
+	int c;
+	tcflag_t lflag;
+	cc_t *cc = tp->t_cc;
+	proc_t p = current_proc();
+	int first, error = 0;
 	int has_etime = 0, last_cc = 0;
 	long slp = 0;		/* XXX this should be renamed `timo'. */
 	boolean_t funnel_state;
 	struct uthread *ut;
+	struct pgrp * pg;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
 	ut = (struct uthread *)get_bsdthread_info(current_thread());
 
 loop:
-	s = spltty();
 	lflag = tp->t_lflag;
 	/*
 	 * take pending input first
 	 */
 	if (ISSET(lflag, PENDIN)) {
 		ttypend(tp);
-		splx(s);	/* reduce latency */
-		s = spltty();
 		lflag = tp->t_lflag;	/* XXX ttypend() clobbers it */
 	}
 
 	/*
-	 * Hang process if it's in the background.
+	 * Signal the process if it's in the background.
 	 */
 	if (isbackground(p, tp)) {
-		splx(s);
 		if ((p->p_sigignore & sigmask(SIGTTIN)) ||
 		   (ut->uu_sigmask & sigmask(SIGTTIN)) ||
-		    p->p_flag & P_PPWAIT || p->p_pgrp->pg_jobc == 0) {
+		    p->p_lflag & P_LPPWAIT) {
 			thread_funnel_set(kernel_flock, funnel_state);
 			return (EIO);
 		}
-		pgsignal(p->p_pgrp, SIGTTIN, 1);
-		error = ttysleep(tp, &lbolt, TTIPRI | PCATCH | PTTYBLOCK, "ttybg2", 0);
-		if (error){
+		pg = proc_pgrp(p);
+		if (pg == PGRP_NULL) {
 			thread_funnel_set(kernel_flock, funnel_state);
-			return (error);
+			return (EIO);
 		}
-		goto loop;
+		if (pg->pg_jobc == 0) {
+			pg_rele(pg);
+			thread_funnel_set(kernel_flock, funnel_state);
+			return (EIO);
+		}
+		pgsignal(pg, SIGTTIN, 1);
+		pg_rele(pg);
+
+		/*
+		 * We signalled ourself, so we need to act as if we
+		 * have been "interrupted" from a "sleep" to act on
+		 * the signal.  If it's a signal that stops the
+		 * process, that's handled in the signal sending code.
+		 */
+			thread_funnel_set(kernel_flock, funnel_state);
+		return (EINTR);
 	}
 
 	if (ISSET(tp->t_state, TS_ZOMBIE)) {
-		splx(s);
 		thread_funnel_set(kernel_flock, funnel_state);
 		return (0);	/* EOF */
 	}
@@ -1677,18 +1660,17 @@ loop:
 		if (qp->c_cc > 0)
 			goto read;
 		if (!ISSET(lflag, ICANON) && cc[VMIN] == 0) {
-			splx(s);
 			thread_funnel_set(kernel_flock, funnel_state);
 			return (0);
 		}
-		splx(s);
 		thread_funnel_set(kernel_flock, funnel_state);
 		return (EWOULDBLOCK);
 	}
 	if (!ISSET(lflag, ICANON)) {
 		int m = cc[VMIN];
 		long t = cc[VTIME];
-		struct timeval etime, timecopy;
+		struct timeval timecopy;
+		struct timeval etime = {0, 0};	/* protected by !has_etime */
 
 		/*
 		 * Check each of the four combinations.
@@ -1705,7 +1687,6 @@ loop:
 				goto read;
 
 			/* m, t and qp->c_cc are all 0.  0 is enough input. */
-			splx(s);
 			thread_funnel_set(kernel_flock, funnel_state);
 			return (0);
 		}
@@ -1758,7 +1739,6 @@ loop:
 			} else {
 			        if (timercmp(&etime, &timecopy, <=)) {
 					/* Timed out, but 0 is enough input. */
-					splx(s);
 					thread_funnel_set(kernel_flock, funnel_state);
 					return (0);
 				}
@@ -1786,7 +1766,6 @@ sleep:
 		error = ttysleep(tp, TSA_HUP_OR_INPUT(tp), TTIPRI | PCATCH,
 				 ISSET(tp->t_state, TS_CONNECTED) ?
 				 "ttyin" : "ttyhup", (int)slp);
-		splx(s);
 		if (error == EWOULDBLOCK)
 			error = 0;
 		else if (error) {
@@ -1803,24 +1782,19 @@ sleep:
 		goto loop;
 	}
 read:
-	splx(s);
 	/*
 	 * Input present, check for input mapping and processing.
 	 */
 	first = 1;
-#ifdef NeXT
 	if (ISSET(lflag, ICANON)
 	|| (ISSET(lflag, IEXTEN | ISIG) == (IEXTEN | ISIG)) )
-#else
-	if (ISSET(lflag, ICANON | ISIG))
-#endif
 		goto slowcase;
 	for (;;) {
 		char ibuf[IBUFSIZ];
 		int icc;
 
 		icc = min(uio_resid(uio), IBUFSIZ);
-		icc = q_to_b(qp, ibuf, icc);
+		icc = q_to_b(qp, (u_char *)ibuf, icc);
 		if (icc <= 0) {
 			if (first)
 				goto loop;
@@ -1831,11 +1805,6 @@ read:
 		 * XXX if there was an error then we should ungetc() the
 		 * unmoved chars and reduce icc here.
 		 */
-#if NSNP > 0
-		if (ISSET(tp->t_lflag, ECHO) &&
-		    ISSET(tp->t_state, TS_SNOOP) && tp->t_sc != NULL)
-			snpin((struct snoop *)tp->t_sc, ibuf, icc);
-#endif
 		if (error)
 			break;
  		if (uio_resid(uio) == 0)
@@ -1856,7 +1825,7 @@ slowcase:
 		 */
 		if (CCEQ(cc[VDSUSP], c) &&
 		    ISSET(lflag, IEXTEN | ISIG) == (IEXTEN | ISIG)) {
-			pgsignal(tp->t_pgrp, SIGTSTP, 1);
+			tty_pgsignal(tp, SIGTSTP, 1);
 			if (first) {
 				error = ttysleep(tp, &lbolt, TTIPRI | PCATCH,
 						 "ttybg3", 0);
@@ -1878,15 +1847,6 @@ slowcase:
 		if (error)
 			/* XXX should ungetc(c, qp). */
 			break;
-#if NSNP > 0
-		/*
-		 * Only snoop directly on input in echo mode.  Non-echoed
-		 * input will be snooped later iff the application echoes it.
-		 */
-		if (ISSET(tp->t_lflag, ECHO) &&
-		    ISSET(tp->t_state, TS_SNOOP) && tp->t_sc != NULL)
-			snpinc((struct snoop *)tp->t_sc, (char)c);
-#endif
  		if (uio_resid(uio) == 0)
 			break;
 		/*
@@ -1903,11 +1863,9 @@ out:
 	 * Look to unblock input now that (presumably)
 	 * the input queue has gone down.
 	 */
-	s = spltty();
 	if (ISSET(tp->t_state, TS_TBLOCK) &&
 	    tp->t_rawq.c_cc + tp->t_canq.c_cc <= I_LOW_WATER)
 		ttyunblock(tp);
-	splx(s);
 
 	thread_funnel_set(kernel_flock, funnel_state);
 	return (error);
@@ -1921,18 +1879,15 @@ out:
  * arrive.
  */
 int
-ttycheckoutq(tp, wait)
-	register struct tty *tp;
-	int wait;
+ttycheckoutq(struct tty *tp, int wait)
 {
-	int hiwat, s;
+	int hiwat;
 	sigset_t oldsig;
 	struct uthread *ut;
 
 	ut = (struct uthread *)get_bsdthread_info(current_thread());
 
 	hiwat = tp->t_hiwat;
-	s = spltty();
 	oldsig = wait ? ut->uu_siglist : 0;
 	if (tp->t_outq.c_cc > hiwat + OBUFSIZ + 100)
 		while (tp->t_outq.c_cc > hiwat) {
@@ -1940,13 +1895,11 @@ ttycheckoutq(tp, wait)
 			if (tp->t_outq.c_cc <= hiwat)
 				break;
 			if (wait == 0 || ut->uu_siglist != oldsig) {
-				splx(s);
 				return (0);
 			}
 			SET(tp->t_state, TS_SO_OLOWAT);
 			tsleep(TSA_OLOWAT(tp), PZERO - 1, "ttoutq", hz);
 		}
-	splx(s);
 	return (1);
 }
 
@@ -1954,18 +1907,16 @@ ttycheckoutq(tp, wait)
  * Process a write call on a tty device.
  */
 int
-ttwrite(tp, uio, flag)
-	register struct tty *tp;
-	register struct uio *uio;
-	int flag;
+ttwrite(struct tty *tp, struct uio *uio, int flag)
 {
-	register char *cp = NULL;
-	register int cc, ce;
-	register struct proc *p;
-	int i, hiwat, count, error, s;
+	char *cp = NULL;
+	int cc, ce;
+	proc_t p;
+	int i, hiwat, count, error;
 	char obuf[OBUFSIZ];
 	boolean_t funnel_state;
 	struct uthread *ut;
+	struct pgrp * pg;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
@@ -1976,44 +1927,51 @@ ttwrite(tp, uio, flag)
 	error = 0;
 	cc = 0;
 loop:
-	s = spltty();
 	if (ISSET(tp->t_state, TS_ZOMBIE)) {
-		splx(s);
 		if (uio_resid(uio) == count)
 			error = EIO;
 		goto out;
 	}
 	if (!ISSET(tp->t_state, TS_CONNECTED)) {
 		if (flag & IO_NDELAY) {
-			splx(s);
 			error = EWOULDBLOCK;
 			goto out;
 		}
 		error = ttysleep(tp, TSA_CARR_ON(tp), TTIPRI | PCATCH,
 				 "ttydcd", 0);
-		splx(s);
 		if (error) {
 			goto out; }
 		goto loop;
 	}
-	splx(s);
 	/*
-	 * Hang the process if it's in the background.
+	 * Signal the process if it's in the background.
 	 */
 	p = current_proc();
 	if (isbackground(p, tp) &&
-	    ISSET(tp->t_lflag, TOSTOP) && (p->p_flag & P_PPWAIT) == 0 &&
+	    ISSET(tp->t_lflag, TOSTOP) && (p->p_lflag & P_LPPWAIT) == 0 &&
 	    (p->p_sigignore & sigmask(SIGTTOU)) == 0 &&
 	    (ut->uu_sigmask & sigmask(SIGTTOU)) == 0) {
-		if (p->p_pgrp->pg_jobc == 0) {
+
+		pg = proc_pgrp(p);
+		if (pg == PGRP_NULL) {
 			error = EIO;
 			goto out;
 		}
-		pgsignal(p->p_pgrp, SIGTTOU, 1);
-		error = ttysleep(tp, &lbolt, TTIPRI | PCATCH | PTTYBLOCK, "ttybg4", 0);
-		if (error)
+		if (pg->pg_jobc == 0) {
+			pg_rele(pg);
+			error = EIO;
+			goto out;
+		}
+		pgsignal(pg, SIGTTOU, 1);
+		pg_rele(pg);
+		/*
+		 * We signalled ourself, so we need to act as if we
+		 * have been "interrupted" from a "sleep" to act on
+		 * the signal.  If it's a signal that stops the
+		 * process, that's handled in the signal sending code.
+		 */
+		error = EINTR;
 			goto out;
-		goto loop;
 	}
 	/*
 	 * Process the user's data in at most OBUFSIZ chunks.  Perform any
@@ -2040,10 +1998,6 @@ loop:
 				cc = 0;
 				break;
 			}
-#if NSNP > 0
-			if (ISSET(tp->t_state, TS_SNOOP) && tp->t_sc != NULL)
-				snpin((struct snoop *)tp->t_sc, cp, cc);
-#endif
 		}
 		/*
 		 * If nothing fancy need be done, grab those characters we
@@ -2067,23 +2021,8 @@ loop:
 				if (ce == 0) {
 					tp->t_rocount = 0;
 					if (ttyoutput(*cp, tp) >= 0) {
-#ifdef NeXT
 						/* out of space */
 						goto overfull;
-#else
-						/* No Clists, wait a bit. */
-						ttstart(tp);
-						if (flag & IO_NDELAY) {
-							error = EWOULDBLOCK;
-							goto out;
-						}
-						error = ttysleep(tp, &lbolt,
-								 TTOPRI|PCATCH,
-								 "ttybf1", 0);
-						if (error)
-							goto out;
-						goto loop;
-#endif /* NeXT */
 					}
 					cp++;
 					cc--;
@@ -2102,28 +2041,14 @@ loop:
 			 * requiring special handling by ttyoutput.
 			 */
 			tp->t_rocount = 0;
-			i = b_to_q(cp, ce, &tp->t_outq);
+			i = b_to_q((u_char *)cp, ce, &tp->t_outq);
 			ce -= i;
 			tp->t_column += ce;
 			cp += ce, cc -= ce, tk_nout += ce;
 			tp->t_outcc += ce;
 			if (i > 0) {
-#ifdef NeXT
 				/* out of space */
 				goto overfull;
-#else
-				/* No Clists, wait a bit. */
-				ttstart(tp);
-				if (flag & IO_NDELAY) {
-					error = EWOULDBLOCK;
-					goto out;
-				}
-				error = ttysleep(tp, &lbolt, TTOPRI | PCATCH,
-						 "ttybf2", 0);
-				if (error)
-					goto out;
-				goto loop;
-#endif /* NeXT */
 			}
 			if (ISSET(tp->t_lflag, FLUSHO) ||
 			    tp->t_outq.c_cc > hiwat)
@@ -2141,7 +2066,6 @@ out:
 	thread_funnel_set(kernel_flock, funnel_state);
 	return (error);
 
-#ifdef NeXT
 overfull:
 
 	/*
@@ -2151,21 +2075,17 @@ overfull:
 	 * proceed as normal.
 	 */
 	hiwat = tp->t_outq.c_cc - 1;
-#endif
 
 ovhiwat:
 	ttstart(tp);
-	s = spltty();
 	/*
 	 * This can only occur if FLUSHO is set in t_lflag,
 	 * or if ttstart/oproc is synchronous (or very fast).
 	 */
 	if (tp->t_outq.c_cc <= hiwat) {
-		splx(s);
 		goto loop;
 	}
 	if (flag & IO_NDELAY) {
-		splx(s);
 		uio_setresid(uio, (uio_resid(uio) + cc));
 		thread_funnel_set(kernel_flock, funnel_state);
 		return (uio_resid(uio) == count ? EWOULDBLOCK : 0);
@@ -2173,7 +2093,6 @@ ovhiwat:
 	SET(tp->t_state, TS_SO_OLOWAT);
 	error = ttysleep(tp, TSA_OLOWAT(tp), TTOPRI | PCATCH, "ttywri",
 			 tp->t_timeout);
-	splx(s);
 	if (error == EWOULDBLOCK)
 		error = EIO;
 	if (error)
@@ -2186,13 +2105,11 @@ ovhiwat:
  * as cleanly as possible.
  */
 static void
-ttyrub(c, tp)
-	register int c;
-	register struct tty *tp;
+ttyrub(int c, struct tty *tp)
 {
-	register u_char *cp;
-	register int savecol;
-	int tabc, s;
+	u_char *cp;
+	int savecol;
+	int tabc;
 
 	if (!ISSET(tp->t_lflag, ECHO) || ISSET(tp->t_lflag, EXTPROC))
 		return;
@@ -2211,7 +2128,9 @@ ttyrub(c, tp)
 			CLR(c, ~TTY_CHARMASK);
 			switch (CCLASS(c)) {
 			case ORDINARY:
-				ttyrubo(tp, 1);
+				if(!(ISSET(tp->t_iflag, IUTF8) && CCONT(c))) {
+					ttyrubo(tp, 1);
+				}
 				break;
 			case BACKSPACE:
 			case CONTROL:
@@ -2226,25 +2145,15 @@ ttyrub(c, tp)
 					ttyretype(tp);
 					return;
 				}
-				s = spltty();
 				savecol = tp->t_column;
 				SET(tp->t_state, TS_CNTTB);
 				SET(tp->t_lflag, FLUSHO);
 				tp->t_column = tp->t_rocol;
-#ifndef NeXT
-				cp = tp->t_rawq.c_cf;
-				if (cp)
-					tabc = *cp;	/* XXX FIX NEXTC */
-				for (; cp; cp = nextc(&tp->t_rawq, cp, &tabc))
-					ttyecho(tabc, tp);
-#else
 				for (cp = firstc(&tp->t_rawq, &tabc); cp;
 				    cp = nextc(&tp->t_rawq, cp, &tabc))
 					ttyecho(tabc, tp);
-#endif /* !NeXT */
 				CLR(tp->t_lflag, FLUSHO);
 				CLR(tp->t_state, TS_CNTTB);
-				splx(s);
 
 				/* savecol will now be length of the tab. */
 				savecol -= tp->t_column;
@@ -2256,7 +2165,7 @@ ttyrub(c, tp)
 				break;
 			default:			/* XXX */
 #define	PANICSTR	"ttyrub: would panic c = %d, val = %d\n"
-				(void)printf(PANICSTR, c, CCLASS(c));
+				printf(PANICSTR, c, CCLASS(c));
 #ifdef notdef
 				panic(PANICSTR, c, CCLASS(c));
 #endif
@@ -2279,7 +2188,6 @@ ttyrub(c, tp)
 static void
 ttyrubo(struct tty *tp, int count)
 {
-
 	while (count-- > 0) {
 		(void)ttyoutput('\b', tp);
 		(void)ttyoutput(' ', tp);
@@ -2293,11 +2201,10 @@ ttyrubo(struct tty *tp, int count)
  *	been checked.
  */
 static void
-ttyretype(tp)
-	register struct tty *tp;
+ttyretype(struct tty *tp)
 {
-	register u_char *cp;
-	int s, c;
+	u_char *cp;
+	int c;
 
 	/* Echo the reprint character. */
 	if (tp->t_cc[VREPRINT] != _POSIX_VDISABLE)
@@ -2310,22 +2217,11 @@ ttyretype(tp)
 	 * FIX: NEXTC IS BROKEN - DOESN'T CHECK QUOTE
 	 * BIT OF FIRST CHAR.
 	 */
-	s = spltty();
-#ifndef NeXT
-	for (cp = tp->t_canq.c_cf, c = (cp != NULL ? *cp : 0);
-	    cp != NULL; cp = nextc(&tp->t_canq, cp, &c))
-		ttyecho(c, tp);
-	for (cp = tp->t_rawq.c_cf, c = (cp != NULL ? *cp : 0);
-	    cp != NULL; cp = nextc(&tp->t_rawq, cp, &c))
-		ttyecho(c, tp);
-#else NeXT
 	for (cp = firstc(&tp->t_canq, &c); cp; cp = nextc(&tp->t_canq, cp, &c))
 		ttyecho(c, tp);
 	for (cp = firstc(&tp->t_rawq, &c); cp; cp = nextc(&tp->t_rawq, cp, &c))
 		ttyecho(c, tp);
-#endif /* !NeXT */
 	CLR(tp->t_state, TS_ERASE);
-	splx(s);
 
 	tp->t_rocount = tp->t_rawq.c_cc;
 	tp->t_rocol = 0;
@@ -2335,11 +2231,8 @@ ttyretype(tp)
  * Echo a typed character to the terminal.
  */
 static void
-ttyecho(c, tp)
-	register int c;
-	register struct tty *tp;
+ttyecho(int c, struct tty *tp)
 {
-
 	if (!ISSET(tp->t_state, TS_CNTTB))
 		CLR(tp->t_lflag, FLUSHO);
 	if ((!ISSET(tp->t_lflag, ECHO) &&
@@ -2363,16 +2256,11 @@ ttyecho(c, tp)
  * Wake up any readers on a tty.
  */
 void
-ttwakeup(tp)
-	register struct tty *tp;
+ttwakeup(struct tty *tp)
 {
-
-#ifndef NeXT
-	if (tp->t_rsel.si_pid != 0)
-#endif
-		selwakeup(&tp->t_rsel);
+	selwakeup(&tp->t_rsel);
 	if (ISSET(tp->t_state, TS_ASYNC))
-		pgsignal(tp->t_pgrp, SIGIO, 1);
+		tty_pgsignal(tp, SIGIO, 1);
 	wakeup(TSA_HUP_OR_INPUT(tp));
 }
 
@@ -2380,14 +2268,9 @@ ttwakeup(tp)
  * Wake up any writers on a tty.
  */
 void
-ttwwakeup(tp)
-	register struct tty *tp;
+ttwwakeup(struct tty *tp)
 {
-#ifndef NeXT
-	if (tp->t_wsel.si_pid != 0 && tp->t_outq.c_cc <= tp->t_lowat)
-#else
 	if (tp->t_outq.c_cc <= tp->t_lowat)
-#endif
 		selwakeup(&tp->t_wsel);
 	if (ISSET(tp->t_state, TS_BUSY | TS_SO_OCOMPLETE) ==
 	    TS_SO_OCOMPLETE && tp->t_outq.c_cc == 0) {
@@ -2406,11 +2289,8 @@ ttwwakeup(tp)
  * used by drivers to map software speed values to hardware parameters.
  */
 int
-ttspeedtab(speed, table)
-	int speed;
-	register struct speedtab *table;
+ttspeedtab(int speed, struct speedtab *table)
 {
-
 	for ( ; table->sp_speed != -1; table++)
 		if (table->sp_speed == speed)
 			return (table->sp_code);
@@ -2440,7 +2320,7 @@ ttsetwater(struct tty *tp)
 #undef	CLAMP
 }
 
-/* NeXT ttyinfo has been converted to the MACH kernel */
+/* ttyinfo has been converted to the MACH kernel */
 #include <mach/thread_info.h>
 
 /* XXX Should be in Mach header <kern/thread.h>, but doesn't work */
@@ -2458,13 +2338,16 @@ ttyinfo(struct tty *tp)
 	int		load;
 	thread_t	thread;
 	uthread_t	uthread;
-	struct proc	*p;
-	struct proc	*pick;
+	proc_t		p;
+	proc_t		pick;
+	pid_t pickpid;
 	const char	*state;
 	struct timeval	utime;
 	struct timeval	stime;
 	thread_basic_info_data_t	basic_info;
 	mach_msg_type_number_t		mmtn = THREAD_BASIC_INFO_COUNT;
+	struct pgrp * pg;
+
 
 	if (ttycheckoutq(tp,0) == 0)
 		return;
@@ -2488,6 +2371,7 @@ ttyinfo(struct tty *tp)
 		return;
 	}
 	/* first process in process group */
+	/* XXX is there a need for pgrp lock ? */
 	if ((p = tp->t_pgrp->pg_members.lh_first) == NULL) {
 		ttyprintf(tp, "empty foreground process group\n");
 		tp->t_rocount = 0;
@@ -2498,17 +2382,31 @@ ttyinfo(struct tty *tp)
 	 * Pick the most interesting process and copy some of its
 	 * state for printing later.
 	 */
+	pg = proc_pgrp(p);
+	pgrp_lock(pg);
+	/* the proc_compare is non blocking fn, no need to use iterator */
 	for (pick = NULL; p != NULL; p = p->p_pglist.le_next) {
-		if (proc_compare(pick, p))
+		if (proc_compare(pick, p)) {
 			pick = p;
+			pickpid = p->p_pid;
+		} else {
+			pickpid = pick->p_pid;
+		}
 	}
+	pgrp_unlock(pg);
+	pg_rele(pg);
+
+	pick = proc_find(pickpid);
+	if (pick == PROC_NULL)
+		return;
 
 	if (TAILQ_EMPTY(&pick->p_uthlist) ||
 	    (uthread = TAILQ_FIRST(&pick->p_uthlist)) == NULL ||
-	    (thread = uthread->uu_act) == NULL ||
+	    (thread = vfs_context_thread(&uthread->uu_context)) == NULL ||
 	    (thread_info_internal(thread, THREAD_BASIC_INFO, (thread_info_t)&basic_info, &mmtn) != KERN_SUCCESS)) {
 		ttyprintf(tp, "foreground process without thread\n");
 		tp->t_rocount = 0;
+		proc_rele(pick);
 		return;
 	}
 
@@ -2533,6 +2431,7 @@ ttyinfo(struct tty *tp)
 		break;
 	}
 	calcru(pick, &utime, &stime, NULL);
+	proc_rele(pick);
 
 	/* Print command, pid, state, utime, and stime */
 	ttyprintf(tp, " cmd: %s %d %s %ld.%02ldu %ld.%02lds\n",
@@ -2563,9 +2462,9 @@ ttyinfo(struct tty *tp)
 #define BOTH    3
 
 static int
-proc_compare(p1, p2)
-	register struct proc *p1, *p2;
+proc_compare(proc_t p1, proc_t p2)
 {
+	/* NOTE THIS FN needs to be NON BLOCKING */
 
 	if (p1 == NULL)
 		return (1);
@@ -2581,10 +2480,13 @@ proc_compare(p1, p2)
 		/*
 		 * tie - favor one with highest recent cpu utilization
 		 */
+#ifdef _PROC_HAS_SCHEDINFO_
+		/* Without the support the fields are always zero */
 		if (p2->p_estcpu > p1->p_estcpu)
 			return (1);
 		if (p1->p_estcpu > p2->p_estcpu)
 			return (0);
+#endif /* _PROC_HAS_SCHEDINFO_ */
 		return (p2->p_pid > p1->p_pid);	/* tie - return highest pid */
 	}
 	/*
@@ -2601,10 +2503,13 @@ proc_compare(p1, p2)
 	/*
 	 * pick the one with the smallest sleep time
 	 */
+#ifdef _PROC_HAS_SCHEDINFO_
+	/* Without the support the fields are always zero */
 	if (p2->p_slptime > p1->p_slptime)
 		return (0);
 	if (p1->p_slptime > p2->p_slptime)
 		return (1);
+#endif /* _PROC_HAS_SCHEDINFO_ */
 	return (p2->p_pid > p1->p_pid);		/* tie - return highest pid */
 }
 
@@ -2612,22 +2517,15 @@ proc_compare(p1, p2)
  * Output char to tty; console putchar style.
  */
 int
-tputchar(c, tp)
-	int c;
-	struct tty *tp;
+tputchar(int c, struct tty *tp)
 {
-	register int s;
-
-	s = spltty();
 	if (!ISSET(tp->t_state, TS_CONNECTED)) {
-		splx(s);
 		return (-1);
 	}
 	if (c == '\n')
 		(void)ttyoutput('\r', tp);
 	(void)ttyoutput(c, tp);
 	ttstart(tp);
-	splx(s);
 	return (0);
 }
 
@@ -2650,9 +2548,13 @@ ttysleep(struct tty *tp, void *chan, int pri, const char *wmesg, int timo)
 	return (tp->t_gen == gen ? 0 : ERESTART);
 }
 
-#ifdef NeXT
 /*
  * Allocate a tty structure and its associated buffers.
+ *
+ * Parameters:	void
+ *
+ * Returns:	!NULL				Address of new struct tty
+ *		NULL				Error ("ENOMEM")
  */
 struct tty *
 ttymalloc(void)
@@ -2674,8 +2576,7 @@ ttymalloc(void)
  * Free a tty structure and its buffers.
  */
 void
-ttyfree(tp)
-struct tty *tp;
+ttyfree(struct tty *tp)
 {
 	clfree(&tp->t_rawq);
 	clfree(&tp->t_canq);
@@ -2683,37 +2584,28 @@ struct tty *tp;
 	FREE(tp, M_TTYS);
 }
 
-#else /* !NeXT */
+int
+isbackground(proc_t p, struct tty  *tp)
+{
+	return (isctty(p, tp) && (p->p_pgrp != tp->t_pgrp));
+}
 
-#ifdef notyet
-/*
- * XXX this is usable not useful or used.  Most tty drivers have
- * ifdefs for using ttymalloc() but assume a different interface.
- */
-/*
- * Allocate a tty struct.  Clists in the struct will be allocated by
- * ttyopen().
- */
-struct tty *
-ttymalloc()
+static int 
+isctty(proc_t p, struct tty  *tp)
 {
-        struct tty *tp;
+	int retval;
+	struct session * sessp;
 
-        MALLOC(tp, struct tty *, sizeof *tp, M_TTYS, M_WAITOK|M_ZERO);
-        return (tp);
+	sessp = proc_session(p);
+	retval = (sessp == tp->t_session && p->p_flag & P_CONTROLT);
+	session_rele(sessp);
+	return(retval);
 }
-#endif
 
-#if 0 /* XXX not yet usable: session leader holds a ref (see kern_exit.c). */
-/*
- * Free a tty struct.  Clists in the struct should have been freed by
- * ttyclose().
- */
-void
-ttyfree(tp)
-	struct tty *tp;
+static int
+isctty_sp(proc_t p, struct tty  *tp, struct session *sessp)
 {
-        FREE(tp, M_TTYS);
+	return(sessp == tp->t_session && p->p_flag & P_CONTROLT);
+
 }
-#endif /* 0 */
-#endif /* NeXT */
+
diff --git a/bsd/kern/tty_compat.c b/bsd/kern/tty_compat.c
index 346ac762b..e5a53b56c 100644
--- a/bsd/kern/tty_compat.c
+++ b/bsd/kern/tty_compat.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/kern/tty_conf.c b/bsd/kern/tty_conf.c
index 5bf3647b1..44f6b12d5 100644
--- a/bsd/kern/tty_conf.c
+++ b/bsd/kern/tty_conf.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
  * Copyright (c) 1982, 1986, 1991, 1993
  *      The Regents of the University of California.  All rights reserved.
@@ -69,17 +74,11 @@
 #define MAXLDISC 8
 #endif
 
-#ifndef NeXT
-static l_open_t		l_noopen;
-static l_close_t	l_noclose;
-static l_rint_t		l_norint;
-#else /* NeXT */
 #define	l_noopen	((l_open_t *)  &enodev)
 #define	l_noclose	((l_close_t *) &enodev)
 #define	l_noread	((l_read_t *)  &enodev)
 #define	l_nowrite	((l_write_t *) &enodev)
 #define	l_norint	((l_rint_t *)  &enodev)
-#endif /* !NeXT */
 
 static l_ioctl_t	l_noioctl;
 static l_start_t	l_nostart;
@@ -128,9 +127,7 @@ static struct linesw nodisc = NODISC(0);
  * Returns: Index used or -1 on failure.
  */
 int
-ldisc_register(discipline, linesw_p)
-	int discipline;
-	struct linesw *linesw_p;
+ldisc_register(int discipline, struct linesw *linesw_p)
 {
 	int slot = -1;
 
@@ -158,63 +155,13 @@ ldisc_register(discipline, linesw_p)
  * discipline: Index for discipline to unload.
  */
 void
-ldisc_deregister(discipline)
-	int discipline;
+ldisc_deregister(int discipline)
 {
 	if (discipline >= LOADABLE_LDISC && discipline < MAXLDISC) {
 		linesw[discipline] = nodisc;
 	}
 }
 
-#ifndef NeXT
-static int
-l_noopen(dev, tp)
-	dev_t dev;
-	struct tty *tp;
-{
-
-	return (ENODEV);
-}
-
-static int
-l_noclose(tp, flag)
-	struct tty *tp;
-	int flag;
-{
-
-	return (ENODEV);
-}
-
-int
-l_noread(tp, uio, flag)
-	struct tty *tp;
-	struct uio *uio;
-	int flag;
-{
-
-	return (ENODEV);
-}
-
-int
-l_nowrite(tp, uio, flag)
-	struct tty *tp;
-	struct uio *uio;
-	int flag;
-{
-
-	return (ENODEV);
-}
-
-static int
-l_norint(c, tp)
-	int c;
-	struct tty *tp;
-{
-
-	return (ENODEV);
-}
-#endif /* !NeXT */
-
 /*
  * Do nothing specific version of line
  * discipline specific ioctl command.
@@ -223,10 +170,10 @@ static int
 l_noioctl(__unused struct tty *tp, __unused u_long cmd, __unused caddr_t data,
 	  __unused int flags, __unused struct proc *p)
 {
-
 	return ENOTTY;
 }
 
 static void
 l_nostart(__unused struct tty *tp)
-    { }
+{
+}
diff --git a/bsd/kern/tty_ptmx.c b/bsd/kern/tty_ptmx.c
new file mode 100644
index 000000000..4bd839801
--- /dev/null
+++ b/bsd/kern/tty_ptmx.c
@@ -0,0 +1,1454 @@
+/*
+ * Copyright (c) 1997-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1982, 1986, 1989, 1993
+ *      The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by the University of
+ *      California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)tty_pty.c	8.4 (Berkeley) 2/20/95
+ */
+
+/*
+ * Pseudo-teletype Driver
+ * (Actually two drivers, requiring two entries in 'cdevsw')
+ */
+#include "pty.h"		/* XXX */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/ioctl.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/tty.h>
+#include <sys/conf.h>
+#include <sys/file_internal.h>
+#include <sys/uio.h>
+#include <sys/kernel.h>
+#include <sys/vnode.h>
+#include <sys/vnode_internal.h>		/* _devfs_setattr() */
+#include <sys/stat.h>			/* _devfs_setattr() */
+#include <sys/user.h>
+#include <sys/signalvar.h>
+#include <sys/sysctl.h>
+#include <miscfs/devfs/devfs.h>
+#include <miscfs/devfs/devfsdefs.h>	/* DEVFS_LOCK()/DEVFS_UNLOCK() */
+
+/* XXX belongs in devfs somewhere - LATER */
+int _devfs_setattr(void *, unsigned short, uid_t, gid_t);
+
+
+#define FREE_BSDSTATIC __private_extern__
+#define d_devtotty_t    struct tty **
+
+/*
+ * Forward declarations
+ */
+int ptmx_init(int n_ptys);
+static void ptsd_start(struct tty *tp);
+static void ptmx_wakeup(struct tty *tp, int flag);
+FREE_BSDSTATIC	d_open_t	ptsd_open;
+FREE_BSDSTATIC	d_close_t	ptsd_close;
+FREE_BSDSTATIC	d_read_t	ptsd_read;
+FREE_BSDSTATIC	d_write_t	ptsd_write;
+FREE_BSDSTATIC	d_ioctl_t	cptyioctl;	/* common ioctl */
+FREE_BSDSTATIC	d_stop_t	ptsd_stop;
+FREE_BSDSTATIC	d_reset_t	ptsd_reset;
+FREE_BSDSTATIC	d_devtotty_t	ptydevtotty;
+FREE_BSDSTATIC	d_open_t	ptmx_open;
+FREE_BSDSTATIC	d_close_t	ptmx_close;
+FREE_BSDSTATIC	d_read_t	ptmx_read;
+FREE_BSDSTATIC	d_write_t	ptmx_write;
+FREE_BSDSTATIC	d_stop_t	ptmx_stop;	/* NO-OP */
+FREE_BSDSTATIC	d_reset_t	ptmx_reset;
+FREE_BSDSTATIC	d_select_t	ptmx_select;
+FREE_BSDSTATIC	d_select_t	ptsd_select;
+
+static int ptmx_major;		/* dynamically assigned major number */
+static struct cdevsw ptmx_cdev = {
+	ptmx_open,	ptmx_close,	ptmx_read,	ptmx_write,
+	cptyioctl,	ptmx_stop,	ptmx_reset,	0,
+	ptmx_select,	eno_mmap,	eno_strat,	eno_getc,
+	eno_putc,	D_TTY
+};
+
+static int ptsd_major;		/* dynamically assigned major number */
+static struct cdevsw ptsd_cdev = {
+	ptsd_open,	ptsd_close,	ptsd_read,	ptsd_write,
+	cptyioctl,	ptsd_stop,	ptsd_reset,	0,
+	ptsd_select,	eno_mmap,	eno_strat,	eno_getc,
+	eno_putc,	D_TTY
+};
+
+/*
+ * XXX Should be devfs function... and use VATTR mechanisms, per
+ * XXX vnode_setattr2(); only we maybe can't really get back to the
+ * XXX vnode here for cloning devices (but it works for *cloned* devices
+ * XXX that are not themselves cloning).
+ *
+ * Returns:	0			Success
+ *	namei:???
+ *	vnode_setattr:???
+ */
+int
+_devfs_setattr(void * handle, unsigned short mode, uid_t uid, gid_t gid)
+{
+	devdirent_t		*direntp = (devdirent_t *)handle;
+	devnode_t		*devnodep;
+	int			error = EACCES;
+	vfs_context_t		ctx = vfs_context_current();;
+	struct vnode_attr	va;
+
+	VATTR_INIT(&va);
+	VATTR_SET(&va, va_uid, uid);
+	VATTR_SET(&va, va_gid, gid);
+	VATTR_SET(&va, va_mode, mode & ALLPERMS);
+
+	/*
+	 * If the TIOCPTYGRANT loses the race with the clone operation because
+	 * this function is not part of devfs, and therefore can't take the
+	 * devfs lock to protect the direntp update, then force user space to
+	 * redrive the grant request.
+	 */
+	if (direntp == NULL || (devnodep = direntp->de_dnp) == NULL) {
+		error = ERESTART;
+		goto out;
+	}
+
+	/*
+	 * Only do this if we are operating on device that doesn't clone
+	 * each time it's referenced.  We perform a lookup on the device
+	 * to insure we get the right instance.  We can't just use the call
+	 * to devfs_dntovn() to get the vp for the operation, because
+	 * dn_dvm may not have been initialized.
+	 */
+	if (devnodep->dn_clone == NULL) {
+		struct nameidata nd;
+		char name[128];
+
+		snprintf(name, sizeof(name), "/dev/%s", direntp->de_name);
+		NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ctx);
+		error = namei(&nd);
+		if (error)
+			goto out;
+		error = vnode_setattr(nd.ni_vp, &va, ctx);
+		vnode_put(nd.ni_vp);
+		nameidone(&nd);
+		goto out;
+	}
+
+out:
+	return(error);
+}
+
+
+
+#define BUFSIZ 100		/* Chunk size iomoved to/from user */
+
+/*
+ * ptmx == /dev/ptmx
+ * ptsd == /dev/pts[0123456789]{3}
+ */
+#define	PTMX_TEMPLATE	"ptmx"
+#define PTSD_TEMPLATE	"ttys%03d"
+
+/*
+ * System-wide limit on the max number of cloned ptys
+ */
+#define	PTMX_MAX_DEFAULT	127	/* 128 entries */
+#define	PTMX_MAX_HARD		999	/* 1000 entries, due to PTSD_TEMPLATE */
+
+static int ptmx_max = PTMX_MAX_DEFAULT;	/* default # of clones we allow */
+
+/* Range enforcement for the sysctl */
+static int
+sysctl_ptmx_max(__unused struct sysctl_oid *oidp, __unused void *arg1,
+		__unused int arg2, struct sysctl_req *req)
+{
+	int new_value, changed;
+	int error = sysctl_io_number(req, ptmx_max, sizeof(int), &new_value, &changed);
+	if (changed) {
+		if (new_value > 0 && new_value <= PTMX_MAX_HARD)
+			ptmx_max = new_value;
+		else
+			error = EINVAL;
+	}
+	return(error);
+}
+
+SYSCTL_NODE(_kern, KERN_TTY, tty, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "TTY");
+SYSCTL_PROC(_kern_tty, OID_AUTO, ptmx_max,
+		CTLTYPE_INT | CTLFLAG_RW,
+		&ptmx_max, 0, &sysctl_ptmx_max, "I", "ptmx_max");
+
+
+/*
+ * ptmx_ioctl is a pointer to a list of pointers to tty structures which is
+ * grown, as necessary, copied, and replaced, but never shrunk.  The ioctl
+ * structures themselves pointed to from this list come and go as needed.
+ */
+struct ptmx_ioctl {
+	struct tty	*pt_tty;	/* pointer to ttymalloc()'ed data */
+	int		pt_flags;
+	struct selinfo	pt_selr;
+	struct selinfo	pt_selw;
+	u_char		pt_send;
+	u_char		pt_ucntl;
+	void		*pt_devhandle;	/* cloned slave device handle */
+};
+
+#define	PF_PKT		0x0008		/* packet mode */
+#define	PF_STOPPED	0x0010		/* user told stopped */
+#define	PF_REMOTE	0x0020		/* remote and flow controlled input */
+#define	PF_NOSTOP	0x0040
+#define PF_UCNTL	0x0080		/* user control mode */
+#define	PF_UNLOCKED	0x0100		/* slave unlock (master open resets) */
+#define	PF_OPEN_M	0x0200		/* master is open */
+#define	PF_OPEN_S	0x0400		/* slave is open */
+
+static int	ptmx_clone(dev_t dev, int minor);
+
+int
+ptmx_init( __unused int config_count)
+{
+	/*
+	 * We start looking at slot 10, since there are inits that will
+	 * stomp explicit slots (e.g. vndevice stomps 1) below that.
+	 */
+
+	/* Get a major number for /dev/ptmx */
+	if((ptmx_major = cdevsw_add(-15, &ptmx_cdev)) == -1) {
+		printf("ptmx_init: failed to obtain /dev/ptmx major number\n");
+		return (ENOENT);
+	}
+
+	/* Get a major number for /dev/pts/nnn */
+	if ((ptsd_major = cdevsw_add(-15, &ptsd_cdev)) == -1) {
+		(void)cdevsw_remove(ptmx_major, &ptmx_cdev);
+		printf("ptmx_init: failed to obtain /dev/ptmx major number\n");
+		return (ENOENT);
+	}
+
+	/* Create the /dev/ptmx device {<major>,0} */
+	(void)devfs_make_node_clone(makedev(ptmx_major, 0),
+				DEVFS_CHAR, UID_ROOT, GID_TTY, 0666,
+				ptmx_clone, PTMX_TEMPLATE);
+    return (0);
+}
+
+
+static struct _ptmx_ioctl_state {
+	struct ptmx_ioctl	**pis_ioctl_list;	/* pointer vector */
+	int			pis_total;		/* total slots */
+	int			pis_free;		/* free slots */
+} _state;
+#define	PTMX_GROW_VECTOR	16	/* Grow by this many slots at a time */
+
+/*
+ * Given a minor number, return the corresponding structure for that minor
+ * number.  If there isn't one, and the create flag is specified, we create
+ * one if possible.
+ *
+ * Parameters:	minor			Minor number of ptmx device
+ *		open_flag		PF_OPEN_M	First open of master
+ *					PF_OPEN_S	First open of slave
+ *					0		Just want ioctl struct
+ *
+ * Returns:	NULL			Did not exist/could not create
+ *		!NULL			structure corresponding minor number
+ */
+static struct ptmx_ioctl *
+ptmx_get_ioctl(int minor, int open_flag)
+{
+	struct ptmx_ioctl *new_ptmx_ioctl;
+
+	if (open_flag & PF_OPEN_M) {
+
+		/*
+		 * If we are about to allocate more memory, but we have
+		 * already hit the administrative limit, then fail the
+		 * operation.
+		 *
+		 * Note:	Subtract free from total when making this
+		 *		check to allow unit increments, rather than
+		 *		snapping to the nearest PTMX_GROW_VECTOR...
+		 */
+		if ((_state.pis_total - _state.pis_free) >= ptmx_max) {
+			return (NULL);
+		}
+
+		MALLOC(new_ptmx_ioctl, struct ptmx_ioctl *, sizeof(struct ptmx_ioctl), M_TTYS, M_WAITOK|M_ZERO);
+		if (new_ptmx_ioctl == NULL) {
+			return (NULL);
+		}
+
+		if ((new_ptmx_ioctl->pt_tty = ttymalloc()) == NULL) {
+			FREE(new_ptmx_ioctl, M_TTYS);
+			return (NULL);
+		}
+	
+		/*
+		 * Hold the DEVFS_LOCK() over this whole operation; devfs
+		 * itself does this over malloc/free as well, so this should
+		 * be safe to do.  We hold it longer than we want to, but
+		 * doing so avoids a reallocation race on the minor number.
+		 */
+		DEVFS_LOCK();
+		/* Need to allocate a larger vector? */
+		if (_state.pis_free == 0) {
+			struct ptmx_ioctl **new_pis_ioctl_list;
+			struct ptmx_ioctl **old_pis_ioctl_list = NULL;
+
+			/* Yes. */
+			MALLOC(new_pis_ioctl_list, struct ptmx_ioctl **, sizeof(struct ptmx_ioctl *) * (_state.pis_total + PTMX_GROW_VECTOR), M_TTYS, M_WAITOK|M_ZERO);
+			if (new_pis_ioctl_list == NULL) {
+				ttyfree(new_ptmx_ioctl->pt_tty);
+				DEVFS_UNLOCK();
+				FREE(new_ptmx_ioctl, M_TTYS);
+				return (NULL);
+			}
+
+			/* If this is not the first time, copy the old over */
+			bcopy(_state.pis_ioctl_list, new_pis_ioctl_list, sizeof(struct ptmx_ioctl *) * _state.pis_total);
+			old_pis_ioctl_list = _state.pis_ioctl_list;
+			_state.pis_ioctl_list = new_pis_ioctl_list;
+			_state.pis_free += PTMX_GROW_VECTOR;
+			_state.pis_total += PTMX_GROW_VECTOR;
+			if (old_pis_ioctl_list)
+				FREE(old_pis_ioctl_list, M_TTYS);
+		}
+
+		/* Vector is large enough; grab a new ptmx_ioctl */
+
+		/* Now grab a free slot... */
+		_state.pis_ioctl_list[minor] = new_ptmx_ioctl;
+
+		/* reduce free count */
+		_state.pis_free--;
+
+		_state.pis_ioctl_list[minor]->pt_flags |= PF_OPEN_M;
+		DEVFS_UNLOCK();
+
+		/* Create the /dev/ttysXXX device {<major>,XXX} */
+		_state.pis_ioctl_list[minor]->pt_devhandle = devfs_make_node(
+				makedev(ptsd_major, minor),
+				DEVFS_CHAR, UID_ROOT, GID_TTY, 0620,
+				PTSD_TEMPLATE, minor);
+	} else if (open_flag & PF_OPEN_S) {
+		DEVFS_LOCK();
+		_state.pis_ioctl_list[minor]->pt_flags |= PF_OPEN_S;
+		DEVFS_UNLOCK();
+	}
+	return (_state.pis_ioctl_list[minor]);
+}
+
+static int
+ptmx_free_ioctl(int minor, int open_flag)
+{
+	struct ptmx_ioctl *old_ptmx_ioctl = NULL;
+
+	DEVFS_LOCK();
+#if 5161374
+	/*
+	 * We have to check after taking the DEVFS_LOCK, since the pointer
+	 * is protected by the lock
+	 */
+	if (_state.pis_ioctl_list[minor] == NULL) {
+		DEVFS_UNLOCK();
+		return (ENXIO);
+	}
+#endif	/* 5161374 */
+	_state.pis_ioctl_list[minor]->pt_flags &= ~(open_flag);
+
+	/*
+	 * Was this the last close?  We will recognize it because we only get
+	 * a notification on the last close of a device, and we will have
+	 * cleared both the master and the slave open bits in the flags.
+	 */
+	if (!(_state.pis_ioctl_list[minor]->pt_flags & (PF_OPEN_M|PF_OPEN_S))) {
+		/* Mark as free so it can be reallocated later */
+		old_ptmx_ioctl = _state.pis_ioctl_list[ minor];
+		_state.pis_ioctl_list[ minor] = NULL;
+		_state.pis_free++;
+	}
+	DEVFS_UNLOCK();
+
+	/* Free old after dropping lock */
+	if (old_ptmx_ioctl != NULL) {
+		/*
+		 * XXX See <rdar://5348651> and <rdar://4854638>
+		 *
+		 * XXX Conditional to be removed when/if tty/pty reference
+		 * XXX counting and mutex implemented.
+		 */
+		if (old_ptmx_ioctl->pt_devhandle != NULL)
+			devfs_remove(old_ptmx_ioctl->pt_devhandle);
+		ttyfree(old_ptmx_ioctl->pt_tty);
+		FREE(old_ptmx_ioctl, M_TTYS);
+	}
+
+	return (0);	/* Success */
+}
+
+
+
+
+/*
+ * Given the dev entry that's being opened, we clone the device.  This driver
+ * doesn't actually use the dev entry, since we alreaqdy know who we are by
+ * being called from this code.  This routine is a callback registered from
+ * devfs_make_node_clone() in ptmx_init(); it's purpose is to provide a new
+ * minor number, or to return -1, if one can't be provided.
+ *
+ * Parameters:	dev			The device we are cloning from
+ *
+ * Returns:	>= 0			A new minor device number
+ *		-1			Error: ENOMEM ("Can't alloc device")
+ *
+ * NOTE:	Called with DEVFS_LOCK() held
+ */
+static int
+ptmx_clone(__unused dev_t dev, int action)
+{
+	int i;
+
+	if (action == DEVFS_CLONE_ALLOC) {
+		/* First one */
+		if (_state.pis_total == 0)
+			return (0);
+
+		/*
+		 * Note: We can add hinting on free slots, if this linear search
+		 * ends up being a performance bottleneck...
+		 */
+		for(i = 0; i < _state.pis_total; i++) {
+			if (_state.pis_ioctl_list[ i] == NULL)
+				break;
+		}
+
+		/*
+		 * XXX We fall off the end here; if we did this twice at the
+		 * XXX same time, we could return the same minor to two
+		 * XXX callers; we should probably exand the pointer vector
+		 * XXX here, but I need more information on the MALLOC/FREE
+		 * XXX locking to ensure against a deadlock.  Maybe we can
+		 * XXX just high watermark it at 1/2 of PTMX_GROW_VECTOR?
+		 * XXX That would require returning &minor as implict return
+		 * XXX and an error code ("EAGAIN/ERESTART") or 0 as our
+		 * XXX explicit return.
+		 */
+
+		return (i);	/* empty slot or next slot */
+	}
+	return(-1);
+}
+
+FREE_BSDSTATIC int
+ptsd_open(dev_t dev, int flag, __unused int devtype, __unused proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int error;
+	boolean_t   funnel_state;
+
+	if ((pti = ptmx_get_ioctl(minor(dev), 0)) == NULL) {
+	        return (ENXIO);
+	}
+	tp = pti->pt_tty;
+
+	if (!(pti->pt_flags & PF_UNLOCKED)) {
+		return (EAGAIN);
+	}
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	if ((tp->t_state & TS_ISOPEN) == 0) {
+		ttychars(tp);		/* Set up default chars */
+		tp->t_iflag = TTYDEF_IFLAG;
+		tp->t_oflag = TTYDEF_OFLAG;
+		tp->t_lflag = TTYDEF_LFLAG;
+		tp->t_cflag = TTYDEF_CFLAG;
+		tp->t_ispeed = tp->t_ospeed = TTYDEF_SPEED;
+		ttsetwater(tp);		/* would be done in xxparam() */
+	} else if (tp->t_state&TS_XCLUDE && suser(kauth_cred_get(), NULL)) {
+	        error = EBUSY;
+		goto out;
+	}
+	if (tp->t_oproc)			/* Ctrlr still around. */
+		(void)(*linesw[tp->t_line].l_modem)(tp, 1);
+	while ((tp->t_state & TS_CARR_ON) == 0) {
+		if (flag&FNONBLOCK)
+			break;
+		error = ttysleep(tp, TSA_CARR_ON(tp), TTIPRI | PCATCH,
+				 "ptsd_opn", 0);
+		if (error)
+			goto out;
+	}
+	error = (*linesw[tp->t_line].l_open)(dev, tp);
+	/* Successful open; mark as open by the slave */
+	pti->pt_flags |= PF_OPEN_S;
+	if (error == 0)
+		ptmx_wakeup(tp, FREAD|FWRITE);
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
+
+FREE_BSDSTATIC int
+ptsd_close(dev_t dev, int flag, __unused int mode, __unused proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int err;
+	boolean_t   funnel_state;
+
+	/*
+	 * This is temporary until the VSX conformance tests
+	 * are fixed.  They are hanging with a deadlock
+	 * where close(ptsd) will not complete without t_timeout set
+	 */
+#define	FIX_VSX_HANG	1
+#ifdef	FIX_VSX_HANG
+	int save_timeout;
+#endif
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+#ifdef	FIX_VSX_HANG
+	save_timeout = tp->t_timeout;
+	tp->t_timeout = 60;
+#endif
+	err = (*linesw[tp->t_line].l_close)(tp, flag);
+	ptsd_stop(tp, FREAD|FWRITE);
+	(void) ttyclose(tp);
+#ifdef	FIX_VSX_HANG
+	tp->t_timeout = save_timeout;
+#endif
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+
+	/* unconditional, just like ttyclose() */
+	ptmx_free_ioctl(minor(dev), PF_OPEN_S);
+
+	return (err);
+}
+
+FREE_BSDSTATIC int
+ptsd_read(dev_t dev, struct uio *uio, int flag)
+{
+	proc_t p = current_proc();
+
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int error = 0;
+	struct uthread *ut;
+	boolean_t   funnel_state;
+	struct pgrp * pg;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+
+	ut = (struct uthread *)get_bsdthread_info(current_thread());
+again:
+	if (pti->pt_flags & PF_REMOTE) {
+		while (isbackground(p, tp)) {
+			if ((p->p_sigignore & sigmask(SIGTTIN)) ||
+			    (ut->uu_sigmask & sigmask(SIGTTIN)) ||
+			    p->p_lflag & P_LPPWAIT) {
+				error = EIO;
+				goto out;
+			}
+			pg = proc_pgrp(p);
+			if (pg == PGRP_NULL) {
+				error = EIO;
+				goto out;
+			}
+			if (pg->pg_jobc == 0) {
+				pg_rele(pg);
+				error = EIO;
+				goto out;
+			}
+			pgsignal(pg, SIGTTIN, 1);
+			pg_rele(pg);
+
+			error = ttysleep(tp, &lbolt, TTIPRI | PCATCH | PTTYBLOCK, "ptsd_bg",
+					 0);
+			if (error)
+			        goto out;
+		}
+		if (tp->t_canq.c_cc == 0) {
+			if (flag & IO_NDELAY)
+				return (EWOULDBLOCK);
+			error = ttysleep(tp, TSA_PTS_READ(tp), TTIPRI | PCATCH,
+					 "ptsd_in", 0);
+			if (error)
+			        goto out;
+			goto again;
+		}
+		while (tp->t_canq.c_cc > 1 && uio_resid(uio) > 0) {
+			int cc;
+			char buf[BUFSIZ];
+
+			cc = min(uio_resid(uio), BUFSIZ);
+			// Don't copy the very last byte
+			cc = min(cc, tp->t_canq.c_cc - 1);
+			cc = q_to_b(&tp->t_canq, (u_char *)buf, cc);
+			error = uiomove(buf, cc, uio);
+			if (error)
+				break;
+		}
+		if (tp->t_canq.c_cc == 1)
+			(void) getc(&tp->t_canq);
+		if (tp->t_canq.c_cc)
+		        goto out;
+	} else
+		if (tp->t_oproc)
+			error = (*linesw[tp->t_line].l_read)(tp, uio, flag);
+	ptmx_wakeup(tp, FWRITE);
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
+
+/*
+ * Write to pseudo-tty.
+ * Wakeups of controlling tty will happen
+ * indirectly, when tty driver calls ptsd_start.
+ */
+FREE_BSDSTATIC int
+ptsd_write(dev_t dev, struct uio *uio, int flag)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int error;
+	boolean_t   funnel_state;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	if (tp->t_oproc == 0)
+		error = EIO;
+	else
+	        error = (*linesw[tp->t_line].l_write)(tp, uio, flag);
+
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
+
+/*
+ * Start output on pseudo-tty.
+ * Wake up process selecting or sleeping for input from controlling tty.
+ */
+static void
+ptsd_start(struct tty *tp)
+{
+	struct ptmx_ioctl *pti;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(tp->t_dev), 0);
+#if 5161374
+	if (pti == NULL)
+		return;		/* XXX ENXIO, but this function is void! */
+#endif	/* 5161374 */
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	if (tp->t_state & TS_TTSTOP)
+	        goto out;
+	if (pti->pt_flags & PF_STOPPED) {
+		pti->pt_flags &= ~PF_STOPPED;
+		pti->pt_send = TIOCPKT_START;
+	}
+	ptmx_wakeup(tp, FREAD);
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return;
+}
+
+static void
+ptmx_wakeup(struct tty *tp, int flag)
+{
+	struct ptmx_ioctl *pti;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(tp->t_dev), 0);
+#if 5161374
+	if (pti == NULL)
+		return;		/* XXX ENXIO, but this function is void! */
+#endif	/* 5161374 */
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	if (flag & FREAD) {
+		selwakeup(&pti->pt_selr);
+		wakeup(TSA_PTC_READ(tp));
+	}
+	if (flag & FWRITE) {
+		selwakeup(&pti->pt_selw);
+		wakeup(TSA_PTC_WRITE(tp));
+	}
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+}
+
+FREE_BSDSTATIC int
+ptmx_open(dev_t dev, __unused int flag, __unused int devtype, __unused proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int error = 0;
+	boolean_t   funnel_state;
+
+
+	if ((pti = ptmx_get_ioctl(minor(dev), PF_OPEN_M)) == NULL) {
+	        return (ENXIO);
+	}
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	/* If master is open OR slave is still draining, pty is still busy */
+	if (tp->t_oproc || (tp->t_state & TS_ISOPEN)) {
+		/*
+		 * If master is closed, we are the only reference, so we
+		 * need to clear the master open bit
+		 */
+		if (!tp->t_oproc)
+			ptmx_free_ioctl(minor(dev), PF_OPEN_M);
+		error = EBUSY;
+		goto out;
+	}
+	tp->t_oproc = ptsd_start;
+	CLR(tp->t_state, TS_ZOMBIE);
+#ifdef sun4c
+	tp->t_stop = ptsd_stop;
+#endif
+	(void)(*linesw[tp->t_line].l_modem)(tp, 1);
+	tp->t_lflag &= ~EXTPROC;
+
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
+
+FREE_BSDSTATIC int
+ptmx_close(dev_t dev, __unused int flags, __unused int fmt, __unused proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	(void)(*linesw[tp->t_line].l_modem)(tp, 0);
+
+	/*
+	 * XXX MDMBUF makes no sense for ptys but would inhibit the above
+	 * l_modem().  CLOCAL makes sense but isn't supported.   Special
+	 * l_modem()s that ignore carrier drop make no sense for ptys but
+	 * may be in use because other parts of the line discipline make
+	 * sense for ptys.  Recover by doing everything that a normal
+	 * ttymodem() would have done except for sending a SIGHUP.
+	 */
+	if (tp->t_state & TS_ISOPEN) {
+		tp->t_state &= ~(TS_CARR_ON | TS_CONNECTED);
+		tp->t_state |= TS_ZOMBIE;
+		ttyflush(tp, FREAD | FWRITE);
+	}
+
+	tp->t_oproc = 0;		/* mark closed */
+
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+
+	ptmx_free_ioctl(minor(dev), PF_OPEN_M);
+
+	return (0);
+}
+
+FREE_BSDSTATIC int
+ptmx_read(dev_t dev, struct uio *uio, int flag)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	char buf[BUFSIZ];
+	int error = 0, cc;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	/*
+	 * We want to block until the slave
+	 * is open, and there's something to read;
+	 * but if we lost the slave or we're NBIO,
+	 * then return the appropriate error instead.
+	 */
+	for (;;) {
+		if (tp->t_state&TS_ISOPEN) {
+			if (pti->pt_flags & PF_PKT && pti->pt_send) {
+				error = ureadc((int)pti->pt_send, uio);
+				if (error)
+					goto out;
+				if (pti->pt_send & TIOCPKT_IOCTL) {
+					cc = min(uio_resid(uio),
+						sizeof(tp->t_termios));
+					uiomove((caddr_t)&tp->t_termios, cc,
+						uio);
+				}
+				pti->pt_send = 0;
+				goto out;
+			}
+			if (pti->pt_flags & PF_UCNTL && pti->pt_ucntl) {
+				error = ureadc((int)pti->pt_ucntl, uio);
+				if (error)
+					goto out;
+				pti->pt_ucntl = 0;
+				goto out;
+			}
+			if (tp->t_outq.c_cc && (tp->t_state&TS_TTSTOP) == 0)
+				break;
+		}
+		if ((tp->t_state & TS_CONNECTED) == 0)
+			goto out;	/* EOF */
+		if (flag & IO_NDELAY) {
+			error = EWOULDBLOCK;
+			goto out;
+		}
+		error = tsleep(TSA_PTC_READ(tp), TTIPRI | PCATCH, "ptmx_in", 0);
+		if (error)
+		        goto out;
+	}
+	if (pti->pt_flags & (PF_PKT|PF_UCNTL))
+		error = ureadc(0, uio);
+	while (uio_resid(uio) > 0 && error == 0) {
+		cc = q_to_b(&tp->t_outq, (u_char *)buf, min(uio_resid(uio), BUFSIZ));
+		if (cc <= 0)
+			break;
+		error = uiomove(buf, cc, uio);
+	}
+	(*linesw[tp->t_line].l_start)(tp);
+
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
+
+FREE_BSDSTATIC int
+ptsd_stop(struct tty *tp, int flush)
+{
+	struct ptmx_ioctl *pti;
+	int flag;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(tp->t_dev), 0);
+#if 5161374
+	if (pti == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	/* note: FLUSHREAD and FLUSHWRITE already ok */
+	if (flush == 0) {
+		flush = TIOCPKT_STOP;
+		pti->pt_flags |= PF_STOPPED;
+	} else
+		pti->pt_flags &= ~PF_STOPPED;
+	pti->pt_send |= flush;
+	/* change of perspective */
+	flag = 0;
+	if (flush & FREAD)
+		flag |= FWRITE;
+	if (flush & FWRITE)
+		flag |= FREAD;
+	ptmx_wakeup(tp, flag);
+
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+
+	return (0);
+}
+
+FREE_BSDSTATIC int
+ptsd_reset(__unused int uban)
+{
+	return (0);
+}
+
+/*
+ * Reinput pending characters after state switch
+ * call at spltty().
+ *
+ * XXX Code duplication: static function, should be inlined
+ */
+static void
+ttypend(struct tty *tp)
+{
+	struct clist tq;
+	int c;
+
+	CLR(tp->t_lflag, PENDIN);
+	SET(tp->t_state, TS_TYPEN);
+	tq = tp->t_rawq;
+	tp->t_rawq.c_cc = 0;
+	tp->t_rawq.c_cf = tp->t_rawq.c_cl = 0;
+	while ((c = getc(&tq)) >= 0)
+		ttyinput(c, tp);
+	CLR(tp->t_state, TS_TYPEN);
+}
+
+/*
+ * Must be called at spltty().
+ *
+ * XXX Code duplication: static function, should be inlined
+ */
+static int
+ttnread(struct tty *tp)
+{
+	int nread;
+
+	if (ISSET(tp->t_lflag, PENDIN))
+		ttypend(tp);
+	nread = tp->t_canq.c_cc;
+	if (!ISSET(tp->t_lflag, ICANON)) {
+		nread += tp->t_rawq.c_cc;
+		if (nread < tp->t_cc[VMIN] && tp->t_cc[VTIME] == 0)
+			nread = 0;
+	}
+	return (nread);
+}
+
+int
+ptsd_select(dev_t dev, int rw, void *wql, proc_t p)
+{
+	struct ptmx_ioctl *pti;
+	struct tty *tp;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	if (tp == NULL)
+		return (ENXIO);
+
+	switch (rw) {
+	case FREAD:
+		if (ttnread(tp) > 0 || ISSET(tp->t_state, TS_ZOMBIE))
+			return(1);
+		selrecord(p, &tp->t_rsel, wql);
+		break;
+	case FWRITE:
+		if ((tp->t_outq.c_cc <= tp->t_lowat &&
+		     ISSET(tp->t_state, TS_CONNECTED))
+		    || ISSET(tp->t_state, TS_ZOMBIE)) {
+			return (1);
+		}
+		selrecord(p, &tp->t_wsel, wql);
+		break;
+	}
+	return (0);
+}
+
+FREE_BSDSTATIC int
+ptmx_select(dev_t dev, int rw, void *wql, proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	int retval = 0;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	if ((tp->t_state & TS_CONNECTED) == 0) {
+		retval = 1;
+		goto out;
+	}
+	switch (rw) {
+
+	case FREAD:
+		/*
+		 * Need to block timeouts (ttrstart).
+		 */
+		if ((tp->t_state&TS_ISOPEN) &&
+		     tp->t_outq.c_cc && (tp->t_state&TS_TTSTOP) == 0) {
+			retval = 1;
+			goto out;
+		}
+		/* FALLTHROUGH */
+
+	case 0:					/* exceptional */
+		if ((tp->t_state&TS_ISOPEN) &&
+		    ((pti->pt_flags & PF_PKT && pti->pt_send) ||
+		     (pti->pt_flags & PF_UCNTL && pti->pt_ucntl))) {
+			retval = 1;
+			goto out;
+		}
+		selrecord(p, &pti->pt_selr, wql);
+		break;
+
+
+	case FWRITE:
+		if (tp->t_state&TS_ISOPEN) {
+			if (pti->pt_flags & PF_REMOTE) {
+			    if (tp->t_canq.c_cc == 0) {
+				retval = 1;
+				goto out;
+			    }
+			} else {
+			    if (tp->t_rawq.c_cc + tp->t_canq.c_cc < TTYHOG-2) {
+				    retval = 1;
+				    goto out;
+			    }
+			    if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) {
+				    retval = 1;
+				    goto out;
+			    }
+			}
+		}
+		selrecord(p, &pti->pt_selw, wql);
+		break;
+
+	}
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (retval);
+}
+
+FREE_BSDSTATIC int
+ptmx_stop(__unused struct tty *tp, __unused int flush)
+{
+	return (0);
+}
+
+FREE_BSDSTATIC int
+ptmx_reset(__unused int uban)
+{
+	return (0);
+}
+
+FREE_BSDSTATIC int
+ptmx_write(dev_t dev, struct uio *uio, int flag)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	u_char *cp = NULL;
+	int cc = 0;
+	u_char locbuf[BUFSIZ];
+	int wcnt = 0;
+	int error = 0;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+again:
+	if ((tp->t_state&TS_ISOPEN) == 0)
+		goto block;
+	if (pti->pt_flags & PF_REMOTE) {
+		if (tp->t_canq.c_cc)
+			goto block;
+		while ((uio_resid(uio) > 0 || cc > 0) &&
+		       tp->t_canq.c_cc < TTYHOG - 1) {
+			if (cc == 0) {
+				cc = min(uio_resid(uio), BUFSIZ);
+				cc = min(cc, TTYHOG - 1 - tp->t_canq.c_cc);
+				cp = locbuf;
+				error = uiomove((caddr_t)cp, cc, uio);
+				if (error)
+					goto out;
+				/* check again for safety */
+				if ((tp->t_state & TS_ISOPEN) == 0) {
+					/* adjust as usual */
+					uio_setresid(uio, (uio_resid(uio) + cc));
+					error = EIO;
+					goto out;
+				}
+			}
+			if (cc > 0) {
+				cc = b_to_q((u_char *)cp, cc, &tp->t_canq);
+				/*
+				 * XXX we don't guarantee that the canq size
+				 * is >= TTYHOG, so the above b_to_q() may
+				 * leave some bytes uncopied.  However, space
+				 * is guaranteed for the null terminator if
+				 * we don't fail here since (TTYHOG - 1) is
+				 * not a multiple of CBSIZE.
+				 */
+				if (cc > 0)
+					break;
+			}
+		}
+		/* adjust for data copied in but not written */
+		uio_setresid(uio, (uio_resid(uio) + cc));
+		(void) putc(0, &tp->t_canq);
+		ttwakeup(tp);
+		wakeup(TSA_PTS_READ(tp));
+		goto out;
+	}
+	while (uio_resid(uio) > 0 || cc > 0) {
+		if (cc == 0) {
+			cc = min(uio_resid(uio), BUFSIZ);
+			cp = locbuf;
+			error = uiomove((caddr_t)cp, cc, uio);
+			if (error)
+				goto out;
+			/* check again for safety */
+			if ((tp->t_state & TS_ISOPEN) == 0) {
+				/* adjust for data copied in but not written */
+				uio_setresid(uio, (uio_resid(uio) + cc));
+				error = EIO;
+				goto out;
+			}
+		}
+		while (cc > 0) {
+			if ((tp->t_rawq.c_cc + tp->t_canq.c_cc) >= TTYHOG - 2 &&
+			   (tp->t_canq.c_cc > 0 || !(tp->t_lflag&ICANON))) {
+				wakeup(TSA_HUP_OR_INPUT(tp));
+				goto block;
+			}
+			(*linesw[tp->t_line].l_rint)(*cp++, tp);
+			wcnt++;
+			cc--;
+		}
+		cc = 0;
+	}
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+block:
+	/*
+	 * Come here to wait for slave to open, for space
+	 * in outq, or space in rawq, or an empty canq.
+	 */
+	if ((tp->t_state & TS_CONNECTED) == 0) {
+		/* adjust for data copied in but not written */
+		uio_setresid(uio, (uio_resid(uio) + cc));
+		error = EIO;
+		goto out;
+	}
+	if (flag & IO_NDELAY) {
+		/* adjust for data copied in but not written */
+		uio_setresid(uio, (uio_resid(uio) + cc));
+		if (wcnt == 0)
+			error = EWOULDBLOCK;
+		goto out;
+	}
+	error = tsleep(TSA_PTC_WRITE(tp), TTOPRI | PCATCH, "ptmx_out", 0);
+	if (error) {
+		/* adjust for data copied in but not written */
+		uio_setresid(uio, (uio_resid(uio) + cc));
+		goto out;
+	}
+	goto again;
+}
+
+
+FREE_BSDSTATIC int
+cptyioctl(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p)
+{
+	struct tty *tp;
+	struct ptmx_ioctl *pti;
+	u_char *cc;
+	int stop, error = 0;
+	boolean_t   funnel_state;
+
+	pti = ptmx_get_ioctl(minor(dev), 0);
+#if 5161374
+	if (pti == NULL || pti->pt_tty == NULL)
+		return(ENXIO);
+#endif	/* 5161374 */
+	tp = pti->pt_tty;
+	cc = tp->t_cc;
+
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+
+	/*
+	 * IF CONTROLLER STTY THEN MUST FLUSH TO PREVENT A HANG.
+	 * ttywflush(tp) will hang if there are characters in the outq.
+	 */
+	if (cmd == TIOCEXT) {
+		/*
+		 * When the EXTPROC bit is being toggled, we need
+		 * to send an TIOCPKT_IOCTL if the packet driver
+		 * is turned on.
+		 */
+		if (*(int *)data) {
+			if (pti->pt_flags & PF_PKT) {
+				pti->pt_send |= TIOCPKT_IOCTL;
+				ptmx_wakeup(tp, FREAD);
+			}
+			tp->t_lflag |= EXTPROC;
+		} else {
+			if ((tp->t_lflag & EXTPROC) &&
+			    (pti->pt_flags & PF_PKT)) {
+				pti->pt_send |= TIOCPKT_IOCTL;
+				ptmx_wakeup(tp, FREAD);
+			}
+			tp->t_lflag &= ~EXTPROC;
+		}
+		goto out;
+	} else
+	if (cdevsw[major(dev)].d_open == ptmx_open)
+		switch (cmd) {
+
+		case TIOCGPGRP:
+			/*
+			 * We aviod calling ttioctl on the controller since,
+			 * in that case, tp must be the controlling terminal.
+			 */
+			*(int *)data = tp->t_pgrp ? tp->t_pgrp->pg_id : 0;
+			goto out;
+
+		case TIOCPKT:
+			if (*(int *)data) {
+			        if (pti->pt_flags & PF_UCNTL) {
+					error = EINVAL;
+					goto out;
+				}
+				pti->pt_flags |= PF_PKT;
+			} else
+				pti->pt_flags &= ~PF_PKT;
+			goto out;
+
+		case TIOCUCNTL:
+			if (*(int *)data) {
+			        if (pti->pt_flags & PF_PKT) {
+					error = EINVAL;
+					goto out;
+				}
+				pti->pt_flags |= PF_UCNTL;
+			} else
+				pti->pt_flags &= ~PF_UCNTL;
+			goto out;
+
+		case TIOCREMOTE:
+			if (*(int *)data)
+				pti->pt_flags |= PF_REMOTE;
+			else
+				pti->pt_flags &= ~PF_REMOTE;
+			ttyflush(tp, FREAD|FWRITE);
+			goto out;
+
+#if COMPAT_43_TTY
+		case TIOCSETP:
+		case TIOCSETN:
+#endif
+		case TIOCSETD:
+		case TIOCSETA:
+		case TIOCSETAW:
+		case TIOCSETAF:
+			ndflush(&tp->t_outq, tp->t_outq.c_cc);
+			break;
+
+		case TIOCSIG:
+			if (*(unsigned int *)data >= NSIG ||
+			    *(unsigned int *)data == 0) {
+				error = EINVAL;
+				goto out;
+			}
+			if ((tp->t_lflag&NOFLSH) == 0)
+				ttyflush(tp, FREAD|FWRITE);
+			tty_pgsignal(tp, *(unsigned int *)data, 1);
+			if ((*(unsigned int *)data == SIGINFO) &&
+			    ((tp->t_lflag&NOKERNINFO) == 0))
+				ttyinfo(tp);
+			goto out;
+
+		case TIOCPTYGRANT:	/* grantpt(3) */
+			/*
+			 * Change the uid of the slave to that of the calling
+			 * thread, change the gid of the slave to GID_TTY,
+			 * change the mode to 0620 (rw--w----).
+			 */
+			{
+				error = _devfs_setattr(pti->pt_devhandle, 0620, kauth_getuid(), GID_TTY);
+				goto out;
+			}
+
+		case TIOCPTYGNAME:	/* ptsname(3) */
+			/*
+			 * Report the name of the slave device in *data
+			 * (128 bytes max.).  Use the same template string
+			 * used for calling devfs_make_node() to create it.
+			 */
+			snprintf(data, 128, "/dev/" PTSD_TEMPLATE, minor(dev));
+			error = 0;
+			goto out;
+		
+		case TIOCPTYUNLK:	/* unlockpt(3) */
+			/*
+			 * Unlock the slave device so that it can be opened.
+			 */
+			pti->pt_flags |= PF_UNLOCKED;
+			error = 0;
+			goto out;
+		}
+	error = (*linesw[tp->t_line].l_ioctl)(tp, cmd, data, flag, p);
+	if (error == ENOTTY) {
+		error = ttioctl(tp, cmd, data, flag, p);
+		if (error == ENOTTY) {
+			if (pti->pt_flags & PF_UCNTL && (cmd & ~0xff) == UIOCCMD(0)) {
+				/* Process the UIOCMD ioctl group */
+				if (cmd & 0xff) {
+					pti->pt_ucntl = (u_char)cmd;
+					ptmx_wakeup(tp, FREAD);
+				}
+				error = 0;
+				goto out;
+			} else if (cmd == TIOCSBRK || cmd == TIOCCBRK) {
+				/*
+				 * POSIX conformance; rdar://3936338
+				 *
+				 * Clear ENOTTY in the case of setting or
+				 * clearing a break failing because pty's
+				 * don't support break like real serial
+				 * ports.
+				 */
+				error = 0;
+				goto out;
+			}
+		}
+	}
+
+	/*
+	 * If external processing and packet mode send ioctl packet.
+	 */
+	if ((tp->t_lflag&EXTPROC) && (pti->pt_flags & PF_PKT)) {
+		switch(cmd) {
+		case TIOCSETA:
+		case TIOCSETAW:
+		case TIOCSETAF:
+#if COMPAT_43_TTY
+		case TIOCSETP:
+		case TIOCSETN:
+#endif
+#if COMPAT_43_TTY || defined(COMPAT_SUNOS)
+		case TIOCSETC:
+		case TIOCSLTC:
+		case TIOCLBIS:
+		case TIOCLBIC:
+		case TIOCLSET:
+#endif
+			pti->pt_send |= TIOCPKT_IOCTL;
+			ptmx_wakeup(tp, FREAD);
+		default:
+			break;
+		}
+	}
+	stop = (tp->t_iflag & IXON) && CCEQ(cc[VSTOP], CTRL('s'))
+		&& CCEQ(cc[VSTART], CTRL('q'));
+	if (pti->pt_flags & PF_NOSTOP) {
+		if (stop) {
+			pti->pt_send &= ~TIOCPKT_NOSTOP;
+			pti->pt_send |= TIOCPKT_DOSTOP;
+			pti->pt_flags &= ~PF_NOSTOP;
+			ptmx_wakeup(tp, FREAD);
+		}
+	} else {
+		if (!stop) {
+			pti->pt_send &= ~TIOCPKT_DOSTOP;
+			pti->pt_send |= TIOCPKT_NOSTOP;
+			pti->pt_flags |= PF_NOSTOP;
+			ptmx_wakeup(tp, FREAD);
+		}
+	}
+out:
+	(void) thread_funnel_set(kernel_flock, funnel_state);
+	return (error);
+}
diff --git a/bsd/kern/tty_pty.c b/bsd/kern/tty_pty.c
index b396f4b09..ed2042b7f 100644
--- a/bsd/kern/tty_pty.c
+++ b/bsd/kern/tty_pty.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
  *      The Regents of the University of California.  All rights reserved.
@@ -75,11 +80,6 @@
 #include <sys/user.h>
 #include <sys/signalvar.h>
 
-#ifndef NeXT
-
-#define FREE_BSDSTATIC	static
-#else
-#define FREE_BSDSTATIC __private_extern__
 #define d_devtotty_t    struct tty **
 
 #ifdef d_stop_t
@@ -87,44 +87,27 @@
 #endif
 typedef void d_stop_t(struct tty *tp, int rw);
 
-#endif /* NeXT */
-
 /* XXX function should be removed??? */
 int pty_init(int n_ptys);
 
-#ifdef notyet
-static void ptyattach(int n);
-#endif
+/* XXX should be a devfs function */
+int _devfs_setattr(void * handle, unsigned short mode, uid_t uid, gid_t gid);
+
 static void ptsstart(struct tty *tp);
 static void ptcwakeup(struct tty *tp, int flag);
 
-FREE_BSDSTATIC	d_open_t	ptsopen;
-FREE_BSDSTATIC	d_close_t	ptsclose;
-FREE_BSDSTATIC	d_read_t	ptsread;
-FREE_BSDSTATIC	d_write_t	ptswrite;
-FREE_BSDSTATIC	d_ioctl_t	ptyioctl;
-FREE_BSDSTATIC	d_stop_t	ptsstop;
-FREE_BSDSTATIC	d_devtotty_t	ptydevtotty;
-FREE_BSDSTATIC	d_open_t	ptcopen;
-FREE_BSDSTATIC	d_close_t	ptcclose;
-FREE_BSDSTATIC	d_read_t	ptcread;
-FREE_BSDSTATIC	d_write_t	ptcwrite;
-FREE_BSDSTATIC	d_select_t	ptcselect;
-
-#ifndef NeXT
-#define CDEV_MAJOR_S 5
-#define CDEV_MAJOR_C 6
-static struct cdevsw pts_cdevsw = 
-	{ ptsopen,	ptsclose,	ptsread,	ptswrite,	/*5*/
-	  ptyioctl,	ptsstop,	nullreset,	ptydevtotty,/* ttyp */
-	  ttselect,	nommap,		NULL,	"pts",	NULL,	-1 };
-
-static struct cdevsw ptc_cdevsw = 
-	{ ptcopen,	ptcclose,	ptcread,	ptcwrite,	/*6*/
-	  ptyioctl,	nullstop,	nullreset,	ptydevtotty,/* ptyp */
-	  ptcselect,	nommap,		NULL,	"ptc",	NULL,	-1 };
-#endif /* !NeXT */
-
+__private_extern__	d_open_t	ptsopen;
+__private_extern__	d_close_t	ptsclose;
+__private_extern__	d_read_t	ptsread;
+__private_extern__	d_write_t	ptswrite;
+__private_extern__	d_ioctl_t	ptyioctl;
+__private_extern__	d_stop_t	ptsstop;
+__private_extern__	d_devtotty_t	ptydevtotty;
+__private_extern__	d_open_t	ptcopen;
+__private_extern__	d_close_t	ptcclose;
+__private_extern__	d_read_t	ptcread;
+__private_extern__	d_write_t	ptcwrite;
+__private_extern__	d_select_t	ptcselect;
 
 #if NPTY == 1
 #undef NPTY
@@ -132,38 +115,21 @@ static struct cdevsw ptc_cdevsw =
 #warning	You have only one pty defined, redefining to 32.
 #endif
 
-#ifndef NeXT
-#ifdef DEVFS
-#define MAXUNITS (8 * 32)
-static	void	*devfs_token_pts[MAXUNITS];
-static	void	*devfs_token_ptc[MAXUNITS];
-static  const	char jnames[] = "pqrsPQRS";
-#if NPTY > MAXUNITS
-#undef NPTY
-#define NPTY MAXUNITS
-#warning	Can't have more than 256 pty's with DEVFS defined.
-#endif /* NPTY > MAXUNITS */
-#endif /* DEVFS */
-#endif /* !NeXT */
-
 #define BUFSIZ 100		/* Chunk size iomoved to/from user */
 
 /*
  * pts == /dev/tty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv]
  * ptc == /dev/pty[pqrsPQRS][0123456789abcdefghijklmnopqrstuv]
  */
-#ifndef NeXT
-FREE_BSDSTATIC struct	tty pt_tty[NPTY];	/* XXX */
-#else /* NeXT */
-/* NeXT All references to have been changed to indirections in the file */
-FREE_BSDSTATIC struct	tty *pt_tty[NPTY] = { NULL };
-#endif /* ! NeXT */
+/* All references to have been changed to indirections in the file */
+__private_extern__ struct	tty *pt_tty[NPTY] = { NULL };
 
 static struct	pt_ioctl {
 	int	pt_flags;
 	struct	selinfo pt_selr, pt_selw;
 	u_char	pt_send;
 	u_char	pt_ucntl;
+	void	*pt_devhandle;	/* slave device handle for grantpt() */
 } pt_ioctl[NPTY];		/* XXX */
 static int	npty = NPTY;		/* for pstat -t */
 
@@ -173,38 +139,6 @@ static int	npty = NPTY;		/* for pstat -t */
 #define	PF_NOSTOP	0x40
 #define PF_UCNTL	0x80		/* user control mode */
 
-#ifdef notyet
-/*
- * Establish n (or default if n is 1) ptys in the system.
- *
- * XXX cdevsw & pstat require the array `pty[]' to be an array
- */
-FREEBSD_STATIC void
-ptyattach(n)
-	int n;
-{
-	char *mem;
-	register u_long ntb;
-#define	DEFAULT_NPTY	32
-
-	/* maybe should allow 0 => none? */
-	if (n <= 1)
-		n = DEFAULT_NPTY;
-	ntb = n * sizeof(struct tty);
-#ifndef NeXT
-	mem = malloc(ntb + ALIGNBYTES + n * sizeof(struct pt_ioctl),
-	    M_DEVBUF, M_WAITOK);
-#else
-	MALLOC(mem, char *, ntb + ALIGNBYTES + n * sizeof(struct pt_ioctl),
-			M_DEVBUF, M_WAITOK);
-#endif /* !NeXT */
-	pt_tty = (struct tty *)mem;
-	mem = (char *)ALIGN(mem + ntb);
-	pt_ioctl = (struct pt_ioctl *)mem;
-	npty = n;
-}
-#endif
-
 #ifndef DEVFS
 int
 pty_init(__unused int n_ptys)
@@ -227,7 +161,7 @@ pty_init(int n_ptys)
 	    int m = j * HEX_BASE + i;
 	    if (m == n_ptys)
 		goto done;
-	    (void)devfs_make_node(makedev(4, m), 
+	    pt_ioctl[m].pt_devhandle = devfs_make_node(makedev(4, m), 
 				  DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0666, 
 				  "tty%c%x", j + START_CHAR, i);
 	    (void)devfs_make_node(makedev(5, m), 
@@ -240,18 +174,14 @@ pty_init(int n_ptys)
 }
 #endif /* DEVFS */
 
-/*ARGSUSED*/
-FREE_BSDSTATIC int
+__private_extern__ int
 ptsopen(dev_t dev, int flag, __unused int devtype, __unused struct proc *p)
 {
-	register struct tty *tp;
+	struct tty *tp;
 	int error;
 	boolean_t   funnel_state;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-#ifndef NeXT
-	tp = &pt_tty[minor(dev)];
-#else
 	/*
 	 * You will see this sort of code coming up in diffs later both
 	 * the ttymalloc and the tp indirection.
@@ -261,10 +191,16 @@ ptsopen(dev_t dev, int flag, __unused int devtype, __unused struct proc *p)
 		goto out;
 	}
 	if (!pt_tty[minor(dev)]) {
-		tp = pt_tty[minor(dev)] = ttymalloc();
+		/*
+		 * If we can't allocate a new one, act as if we had run out
+		 * of device nodes.
+		 */
+		if ((tp = pt_tty[minor(dev)] = ttymalloc()) == NULL) {
+			error = ENXIO;
+			goto out;
+		}
 	} else
 		tp = pt_tty[minor(dev)];
-#endif
 	if ((tp->t_state & TS_ISOPEN) == 0) {
 		ttychars(tp);		/* Set up default chars */
 		tp->t_iflag = TTYDEF_IFLAG;
@@ -295,40 +231,48 @@ out:
 	return (error);
 }
 
-FREE_BSDSTATIC int
+__private_extern__ int
 ptsclose(dev_t dev, int flag, __unused int mode, __unused proc_t p)
 {
-	register struct tty *tp;
+	struct tty *tp;
 	int err;
 	boolean_t   funnel_state;
-
+	/*
+	 * This is temporary until the VSX conformance tests
+	 * are fixed.  They are hanging with a deadlock
+	 * where close(pts) will not complete without t_timeout set
+	 */
+#define	FIX_VSX_HANG	1
+#ifdef	FIX_VSX_HANG
+	int save_timeout;
+#endif
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
 	tp = pt_tty[minor(dev)];
+#ifdef	FIX_VSX_HANG
+	save_timeout = tp->t_timeout;
+	tp->t_timeout = 60;
+#endif
 	err = (*linesw[tp->t_line].l_close)(tp, flag);
 	ptsstop(tp, FREAD|FWRITE);
 	(void) ttyclose(tp);
-
+#ifdef	FIX_VSX_HANG
+	tp->t_timeout = save_timeout;
+#endif
 	(void) thread_funnel_set(kernel_flock, funnel_state);
 	return (err);
 }
 
-FREE_BSDSTATIC int
-ptsread(dev, uio, flag)
-	dev_t dev;
-	struct uio *uio;
-	int flag;
+__private_extern__ int
+ptsread(dev_t dev, struct uio *uio, int flag)
 {
-#ifndef NeXT
-	struct proc *p = curproc;
-#else
 	struct proc *p = current_proc();
-#endif /* NeXT */
-	register struct tty *tp = pt_tty[minor(dev)];
-	register struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
+	struct tty *tp = pt_tty[minor(dev)];
+	struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
 	int error = 0;
 	struct uthread *ut;
 	boolean_t   funnel_state;
+	struct pgrp *pg;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
@@ -339,12 +283,25 @@ again:
 		while (isbackground(p, tp)) {
 			if ((p->p_sigignore & sigmask(SIGTTIN)) ||
 			    (ut->uu_sigmask & sigmask(SIGTTIN)) ||
-			    p->p_pgrp->pg_jobc == 0 ||
-			    p->p_flag & P_PPWAIT) {
+			    p->p_lflag & P_LPPWAIT) {
 				error = EIO;
 				goto out;
 			}
-			pgsignal(p->p_pgrp, SIGTTIN, 1);
+
+
+			pg = proc_pgrp(p);
+			if (pg == PGRP_NULL) {
+				error = EIO;
+				goto out;
+			}
+			if (pg->pg_jobc == 0) {
+				pg_rele(pg);
+				error = EIO;
+				goto out;
+			}
+			pgsignal(pg, SIGTTIN, 1);
+			pg_rele(pg);
+
 			error = ttysleep(tp, &lbolt, TTIPRI | PCATCH | PTTYBLOCK, "ptsbg",
 					 0);
 			if (error)
@@ -366,7 +323,7 @@ again:
 			cc = min(uio_resid(uio), BUFSIZ);
 			// Don't copy the very last byte
 			cc = min(cc, tp->t_canq.c_cc - 1);
-			cc = q_to_b(&tp->t_canq, buf, cc);
+			cc = q_to_b(&tp->t_canq, (u_char *)buf, cc);
 			error = uiomove(buf, cc, uio);
 			if (error)
 				break;
@@ -389,13 +346,10 @@ out:
  * Wakeups of controlling tty will happen
  * indirectly, when tty driver calls ptsstart.
  */
-FREE_BSDSTATIC int
-ptswrite(dev, uio, flag)
-	dev_t dev;
-	struct uio *uio;
-	int flag;
+__private_extern__ int
+ptswrite(dev_t dev, struct uio *uio, int flag)
 {
-	register struct tty *tp;
+	struct tty *tp;
 	int error;
 	boolean_t   funnel_state;
 
@@ -416,10 +370,9 @@ ptswrite(dev, uio, flag)
  * Wake up process selecting or sleeping for input from controlling tty.
  */
 static void
-ptsstart(tp)
-	struct tty *tp;
+ptsstart(struct tty *tp)
 {
-	register struct pt_ioctl *pti = &pt_ioctl[minor(tp->t_dev)];
+	struct pt_ioctl *pti = &pt_ioctl[minor(tp->t_dev)];
 	boolean_t   funnel_state;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
@@ -437,9 +390,7 @@ out:
 }
 
 static void
-ptcwakeup(tp, flag)
-	struct tty *tp;
-	int flag;
+ptcwakeup(struct tty *tp, int flag)
 {
 	struct pt_ioctl *pti = &pt_ioctl[minor(tp->t_dev)];
 	boolean_t   funnel_state;
@@ -457,10 +408,10 @@ ptcwakeup(tp, flag)
 	(void) thread_funnel_set(kernel_flock, funnel_state);
 }
 
-FREE_BSDSTATIC int
+__private_extern__ int
 ptcopen(dev_t dev, __unused int flag, __unused int devtype, __unused proc_t p)
 {
-	register struct tty *tp;
+	struct tty *tp;
 	struct pt_ioctl *pti;
 	int error = 0;
 	boolean_t   funnel_state;
@@ -475,8 +426,9 @@ ptcopen(dev_t dev, __unused int flag, __unused int devtype, __unused proc_t p)
 		tp = pt_tty[minor(dev)] = ttymalloc();
 	} else
 		tp = pt_tty[minor(dev)];
-	if (tp->t_oproc) {
-		error = EIO;
+	/* If master is open OR slave is still draining, pty is still busy */
+	if (tp->t_oproc || (tp->t_state & TS_ISOPEN)) {
+		error = EBUSY;
 		goto out;
 	}
 	tp->t_oproc = ptsstart;
@@ -495,10 +447,10 @@ out:
 	return (error);
 }
 
-FREE_BSDSTATIC int
+__private_extern__ int
 ptcclose(dev_t dev, __unused int flags, __unused int fmt, __unused proc_t p)
 {
-	register struct tty *tp;
+	struct tty *tp;
 	boolean_t   funnel_state;
 
 	funnel_state = thread_funnel_set(kernel_flock, TRUE);
@@ -526,13 +478,10 @@ ptcclose(dev_t dev, __unused int flags, __unused int fmt, __unused proc_t p)
 	return (0);
 }
 
-FREE_BSDSTATIC int
-ptcread(dev, uio, flag)
-	dev_t dev;
-	struct uio *uio;
-	int flag;
+__private_extern__ int
+ptcread(dev_t dev, struct uio *uio, int flag)
 {
-	register struct tty *tp = pt_tty[minor(dev)];
+	struct tty *tp = pt_tty[minor(dev)];
 	struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
 	char buf[BUFSIZ];
 	int error = 0, cc;
@@ -584,7 +533,7 @@ ptcread(dev, uio, flag)
 	if (pti->pt_flags & (PF_PKT|PF_UCNTL))
 		error = ureadc(0, uio);
 	while (uio_resid(uio) > 0 && error == 0) {
-		cc = q_to_b(&tp->t_outq, buf, min(uio_resid(uio), BUFSIZ));
+		cc = q_to_b(&tp->t_outq, (u_char *)buf, min(uio_resid(uio), BUFSIZ));
 		if (cc <= 0)
 			break;
 		error = uiomove(buf, cc, uio);
@@ -596,10 +545,8 @@ out:
 	return (error);
 }
 
-FREE_BSDSTATIC void
-ptsstop(tp, flush)
-	register struct tty *tp;
-	int flush;
+__private_extern__ void
+ptsstop(struct tty *tp, int flush)
 {
 	struct pt_ioctl *pti = &pt_ioctl[minor(tp->t_dev)];
 	int flag;
@@ -625,14 +572,10 @@ ptsstop(tp, flush)
 	(void) thread_funnel_set(kernel_flock, funnel_state);
 }
 
-FREE_BSDSTATIC int
-ptcselect(dev, rw, wql, p)
-	dev_t dev;
-	int rw;
-	void * wql;
-	struct proc *p;
+__private_extern__ int
+ptcselect(dev_t dev, int rw, void *wql, struct proc *p)
 {
-	register struct tty *tp = pt_tty[minor(dev)];
+	struct tty *tp = pt_tty[minor(dev)];
 	struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
 	int retval = 0;
 	boolean_t   funnel_state;
@@ -679,7 +622,7 @@ ptcselect(dev, rw, wql, p)
 				    retval = 1;
 				    goto out;
 			    }
-			    if (tp->t_canq.c_cc == 0 && (tp->t_iflag&ICANON)) {
+			    if (tp->t_canq.c_cc == 0 && (tp->t_lflag&ICANON)) {
 				    retval = 1;
 				    goto out;
 			    }
@@ -694,15 +637,12 @@ out:
 	return (retval);
 }
 
-FREE_BSDSTATIC int
-ptcwrite(dev, uio, flag)
-	dev_t dev;
-	register struct uio *uio;
-	int flag;
+__private_extern__ int
+ptcwrite(dev_t dev, struct uio *uio, int flag)
 {
-	register struct tty *tp = pt_tty[minor(dev)];
-	register u_char *cp = NULL;
-	register int cc = 0;
+	struct tty *tp = pt_tty[minor(dev)];
+	u_char *cp = NULL;
+	int cc = 0;
 	u_char locbuf[BUFSIZ];
 	int wcnt = 0;
 	struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
@@ -735,7 +675,7 @@ again:
 				}
 			}
 			if (cc > 0) {
-				cc = b_to_q((char *)cp, cc, &tp->t_canq);
+				cc = b_to_q((u_char *)cp, cc, &tp->t_canq);
 				/*
 				 * XXX we don't guarantee that the canq size
 				 * is >= TTYHOG, so the above b_to_q() may
@@ -772,7 +712,7 @@ again:
 		}
 		while (cc > 0) {
 			if ((tp->t_rawq.c_cc + tp->t_canq.c_cc) >= TTYHOG - 2 &&
-			   (tp->t_canq.c_cc > 0 || !(tp->t_iflag&ICANON))) {
+			   (tp->t_canq.c_cc > 0 || !(tp->t_lflag&ICANON))) {
 				wakeup(TSA_HUP_OR_INPUT(tp));
 				goto block;
 			}
@@ -812,40 +752,12 @@ block:
 	goto again;
 }
 
-#ifndef NeXT
-/* XXX we eventually want to go to this model,
- * but premier can't change the cdevsw */
-static	struct tty *
-ptydevtotty(dev)
-	dev_t		dev;
+__private_extern__ int
+ptyioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
 {
-	if (minor(dev) >= npty)
-		return (NULL);
-
-	return &pt_tty[minor(dev)];
-}
-#endif /* !NeXT */
-
-/*ARGSUSED*/
-FREE_BSDSTATIC int
-#ifndef NeXT
-ptyioctl(dev, cmd, data, flag)
-	dev_t dev;
-	int cmd;
-	caddr_t data;
-	int flag;
-#else
-ptyioctl(dev, cmd, data, flag, p)
-	dev_t dev;
-	u_long cmd;
-	caddr_t data;
-	int flag;
-	struct proc *p;
-#endif
-{
-	register struct tty *tp = pt_tty[minor(dev)];
-	register struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
-	register u_char *cc = tp->t_cc;
+	struct tty *tp = pt_tty[minor(dev)];
+	struct pt_ioctl *pti = &pt_ioctl[minor(dev)];
+	u_char *cc = tp->t_cc;
 	int stop, error = 0;
 	boolean_t   funnel_state;
 
@@ -877,11 +789,7 @@ ptyioctl(dev, cmd, data, flag, p)
 		}
 		goto out;
 	} else
-#ifndef NeXT
-	if (cdevsw[major(dev)]->d_open == ptcopen)
-#else
 	if (cdevsw[major(dev)].d_open == ptcopen)
-#endif
 		switch (cmd) {
 
 		case TIOCGPGRP:
@@ -941,24 +849,66 @@ ptyioctl(dev, cmd, data, flag, p)
 			}
 			if ((tp->t_lflag&NOFLSH) == 0)
 				ttyflush(tp, FREAD|FWRITE);
-			pgsignal(tp->t_pgrp, *(unsigned int *)data, 1);
+			tty_pgsignal(tp, *(unsigned int *)data, 1);
 			if ((*(unsigned int *)data == SIGINFO) &&
 			    ((tp->t_lflag&NOKERNINFO) == 0))
 				ttyinfo(tp);
 			goto out;
+
+		case TIOCPTYGRANT:	/* grantpt(3) */
+			/*
+			 * Change the uid of the slave to that of the calling
+			 * thread, change the gid of the slave to GID_TTY,
+			 * change the mode to 0620 (rw--w----).
+			 */
+			{
+				_devfs_setattr(pti->pt_devhandle, 0620, kauth_getuid(), GID_TTY);
+				goto out;
+			}
+
+		case TIOCPTYGNAME:	/* ptsname(3) */
+			/*
+			 * Report the name of the slave device in *data
+			 * (128 bytes max.).  Use the same derivation method
+			 * used for calling devfs_make_node() to create it.
+			 */
+			snprintf(data, 128, "/dev/tty%c%x",
+				START_CHAR + (minor(dev) / HEX_BASE),
+				minor(dev) % HEX_BASE);
+			error = 0;
+			goto out;
+		
+		case TIOCPTYUNLK:	/* unlockpt(3) */
+			/*
+			 * Unlock the slave device so that it can be opened.
+			 */
+			error = 0;
+			goto out;
 		}
 	error = (*linesw[tp->t_line].l_ioctl)(tp, cmd, data, flag, p);
 	if (error == ENOTTY) {
 		error = ttioctl(tp, cmd, data, flag, p);
-		if (error == ENOTTY
-		&&  pti->pt_flags & PF_UCNTL && (cmd & ~0xff) == UIOCCMD(0)) {
-			/* Process the UIOCMD ioctl group */
-			if (cmd & 0xff) {
-				pti->pt_ucntl = (u_char)cmd;
-				ptcwakeup(tp, FREAD);
+		if (error == ENOTTY) {
+			if (pti->pt_flags & PF_UCNTL && (cmd & ~0xff) == UIOCCMD(0)) {
+				/* Process the UIOCMD ioctl group */
+				if (cmd & 0xff) {
+					pti->pt_ucntl = (u_char)cmd;
+					ptcwakeup(tp, FREAD);
+				}
+				error = 0;
+				goto out;
+			} else if (cmd == TIOCSBRK || cmd == TIOCCBRK) {
+				/*
+				 * POSIX conformance; rdar://3936338
+				 *
+				 * Clear ENOTTY in the case of setting or
+				 * clearing a break failing because pty's
+				 * don't support break like real serial
+				 * ports.
+				 */
+				error = 0;
+				goto out;
 			}
-			error = 0;
-			goto out;
 		}
 	}
 
@@ -1008,40 +958,3 @@ out:
 	(void) thread_funnel_set(kernel_flock, funnel_state);
 	return (error);
 }
-
-#ifndef NeXT
-static ptc_devsw_installed = 0;
-
-static void
-ptc_drvinit(void *unused)
-{
-#ifdef DEVFS
-	int i,j,k;
-#endif
-	dev_t dev;
-
-	if( ! ptc_devsw_installed ) {
-		dev = makedev(CDEV_MAJOR_S, 0);
-		cdevsw_add(&dev, &pts_cdevsw, NULL);
-		dev = makedev(CDEV_MAJOR_C, 0);
-		cdevsw_add(&dev, &ptc_cdevsw, NULL);
-		ptc_devsw_installed = 1;
-#ifdef DEVFS
-		for ( i = 0 ; i<NPTY ; i++ ) {
-			j = i / 32;
-			k = i % 32;
-			devfs_token_pts[i] = 
-				devfs_add_devswf(&pts_cdevsw,i,
-						DV_CHR,0,0,0666,
-						"tty%c%n",jnames[j],k);
-			devfs_token_ptc[i] =
-				devfs_add_devswf(&ptc_cdevsw,i,
-						DV_CHR,0,0,0666,
-						"pty%c%n",jnames[j],k);
-		}
-#endif
-    	}
-}
-
-SYSINIT(ptcdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE+CDEV_MAJOR_C,ptc_drvinit,NULL)
-#endif /* !NeXT */
diff --git a/bsd/kern/tty_subr.c b/bsd/kern/tty_subr.c
index 1fcda738d..c2abc010d 100644
--- a/bsd/kern/tty_subr.c
+++ b/bsd/kern/tty_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -48,7 +54,6 @@
  *
  */
 
-#ifdef NeXT
 /*
  * We use the NetBSD based clist system, it is much more efficient than the
  * old style clist stuff used by free bsd.
@@ -60,7 +65,6 @@
 #include <sys/tty.h>
 #include <sys/malloc.h>
 
-#include <machine/spl.h>
 
 /*
  * At compile time, choose:
@@ -97,12 +101,8 @@ cinit(void)
  * of the specified length, with/without quoting support.
  */
 int
-clalloc(clp, size, quot)
-	struct clist *clp;
-	int size;
-	int quot;
+clalloc(struct clist *clp, int size, int quot)
 {
-
 	MALLOC_ZONE(clp->c_cs, u_char *, size, M_TTYS, M_WAITOK);
 	if (!clp->c_cs)
 		return (-1);
@@ -126,8 +126,7 @@ clalloc(clp, size, quot)
 }
 
 void
-clfree(clp)
-	struct clist *clp;
+clfree(struct clist *clp)
 {
 	if(clp->c_cs)
 		FREE_ZONE(clp->c_cs, clp->c_cn, M_TTYS);
@@ -141,13 +140,10 @@ clfree(clp)
  * Get a character from a clist.
  */
 int
-getc(clp)
-	struct clist *clp;
+getc(struct clist *clp)
 {
-	register int c = -1;
-	int s;
+	int c = -1;
 
-	s = spltty();
 	if (clp->c_cc == 0)
 		goto out;
 
@@ -166,7 +162,6 @@ getc(clp)
 	if (--clp->c_cc == 0)
 		clp->c_cf = clp->c_cl = (u_char *)0;
 out:
-	splx(s);
 	return c;
 }
 
@@ -175,16 +170,11 @@ out:
  * Return number of bytes moved.
  */
 int
-q_to_b(clp, cp, count)
-	struct clist *clp;
-	u_char *cp;
-	int count;
+q_to_b(struct clist *clp, u_char *cp, int count)
 {
-	register int cc;
+	int cc;
 	u_char *p = cp;
-	int s;
 
-	s = spltty();
 	/* optimize this while loop */
 	while (count > 0 && clp->c_cc > 0) {
 		cc = clp->c_cl - clp->c_cf;
@@ -202,7 +192,6 @@ q_to_b(clp, cp, count)
 	}
 	if (clp->c_cc == 0)
 		clp->c_cf = clp->c_cl = (u_char *)0;
-	splx(s);
 	return p - cp;
 }
 
@@ -211,16 +200,12 @@ q_to_b(clp, cp, count)
  * Stop counting if flag&character is non-null.
  */
 int
-ndqb(clp, flag)
-	struct clist *clp;
-	int flag;
+ndqb(struct clist *clp, int flag)
 {
 	int count = 0;
 	register int i;
 	register int cc;
-	int s;
 
-	s = spltty();
 	if ((cc = clp->c_cc) == 0)
 		goto out;
 
@@ -247,7 +232,6 @@ ndqb(clp, flag)
 		}
 	}
 out:
-	splx(s);
 	return count;
 }
 
@@ -255,18 +239,14 @@ out:
  * Flush count bytes from clist.
  */
 void
-ndflush(clp, count)
-	struct clist *clp;
-	int count;
+ndflush(struct clist *clp, int count)
 {
-	register int cc;
-	int s;
+	int cc;
 
-	s = spltty();
 	if (count == clp->c_cc) {
 		clp->c_cc = 0;
 		clp->c_cf = clp->c_cl = (u_char *)0;
-		goto out;
+		return;
 	}
 	/* optimize this while loop */
 	while (count > 0 && clp->c_cc > 0) {
@@ -283,22 +263,16 @@ ndflush(clp, count)
 	}
 	if (clp->c_cc == 0)
 		clp->c_cf = clp->c_cl = (u_char *)0;
-out:
-	splx(s);
 }
 
 /*
  * Put a character into the output queue.
  */
 int
-putc(c, clp)
-	int c;
-	struct clist *clp;
+putc(int c, struct clist *clp)
 {
 	register int i;
-	int s;
 
-	s = spltty();
 	if (clp->c_cc == 0) {
 		if (!clp->c_cs) {
 #if DIAGNOSTIC
@@ -306,7 +280,6 @@ putc(c, clp)
 #endif
 			if(clalloc(clp, 1024, 1)) {
 out:
-				splx(s);
 				return -1;
 			}
 		}
@@ -333,7 +306,6 @@ out:
 	clp->c_cl++;
 	if (clp->c_cl == clp->c_ce)
 		clp->c_cl = clp->c_cs;
-	splx(s);
 	return 0;
 }
 
@@ -385,12 +357,10 @@ b_to_q(const u_char *cp, int count, struct clist *clp)
 {
 	int cc;
 	const u_char *p = cp;
-	int s;
 
 	if (count <= 0)
 		return 0;
 
-	s = spltty();
 
 	if (clp->c_cc == 0) {
 		if (!clp->c_cs) {
@@ -429,7 +399,6 @@ b_to_q(const u_char *cp, int count, struct clist *clp)
 			clp->c_cl = clp->c_cs;
 	}
 out:
-	splx(s);
 	return count;
 }
 
@@ -444,12 +413,8 @@ static int cc;
  * masked.
  */
 u_char *
-nextc(clp, cp, c)
-	struct clist *clp;
-	register u_char *cp;
-	int *c;
+nextc(struct clist *clp, u_char *cp, int *c)
 {
-
 	if (clp->c_cf == cp) {
 		/*
 		 * First time initialization.
@@ -486,11 +451,9 @@ nextc(clp, cp, c)
  * *c is set to the NEXT character
  */
 u_char *
-firstc(clp, c)
-	struct clist *clp;
-	int *c;
+firstc(struct clist *clp, int *c)
 {
-	register u_char *cp;
+	u_char *cp;
 
 	cc = clp->c_cc;
 	if (cc == 0)
@@ -513,13 +476,10 @@ firstc(clp, c)
  * Remove the last character in the clist and return it.
  */
 int
-unputc(clp)
-	struct clist *clp;
+unputc(struct clist *clp)
 {
 	unsigned int c = -1;
-	int s;
 
-	s = spltty();
 	if (clp->c_cc == 0)
 		goto out;
 
@@ -542,7 +502,6 @@ unputc(clp)
 	if (clp->c_cc == 0)
 		clp->c_cf = clp->c_cl = (u_char *)0;
 out:
-	splx(s);
 	return c;
 }
 
@@ -550,13 +509,10 @@ out:
  * Put the chars in the from queue on the end of the to queue.
  */
 void
-catq(from, to)
-	struct clist *from, *to;
+catq(struct clist *from, struct clist *to)
 {
 	int c;
 
 	while ((c = getc(from)) != -1)
 		putc(c, to);
 }
-
-#endif /* NeXT */
diff --git a/bsd/kern/tty_tb.c b/bsd/kern/tty_tb.c
deleted file mode 100644
index d4b8e4d62..000000000
--- a/bsd/kern/tty_tb.c
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
-/*-
- * Copyright (c) 1982, 1986, 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- *	@(#)tty_tb.c	8.1 (Berkeley) 6/10/93
- */
-
-#include "tb.h"
-#if NTB > 0
-
-/*
- * Line discipline for RS232 tablets;
- * supplies binary coordinate data.
- */
-#include <sys/param.h>
-#include <sys/tablet.h>
-#include <sys/tty.h>
-#if NeXT
-#include <sys/proc_internal.h>
-#endif
-
-/*
- * Tablet configuration table.
- */
-struct	tbconf {
-	short	tbc_recsize;	/* input record size in bytes */
-	short	tbc_uiosize;	/* size of data record returned user */
-	int	tbc_sync;	/* mask for finding sync byte/bit */
-	int	(*tbc_decode)();/* decoding routine */
-	char	*tbc_run;	/* enter run mode sequence */
-	char	*tbc_point;	/* enter point mode sequence */
-	char	*tbc_stop;	/* stop sequence */
-	char	*tbc_start;	/* start/restart sequence */
-	int	tbc_flags;
-#define	TBF_POL		0x1	/* polhemus hack */
-#define	TBF_INPROX	0x2	/* tablet has proximity info */
-};
-
-static	int tbdecode(), gtcodecode(), poldecode();
-static	int tblresdecode(), tbhresdecode();
-
-struct	tbconf tbconf[TBTYPE] = {
-{ 0 },
-{ 5, sizeof (struct tbpos), 0200, tbdecode, "6", "4" },
-{ 5, sizeof (struct tbpos), 0200, tbdecode, "\1CN", "\1RT", "\2", "\4" },
-{ 8, sizeof (struct gtcopos), 0200, gtcodecode },
-{17, sizeof (struct polpos), 0200, poldecode, 0, 0, "\21", "\5\22\2\23",
-  TBF_POL },
-{ 5, sizeof (struct tbpos), 0100, tblresdecode, "\1CN", "\1PT", "\2", "\4",
-  TBF_INPROX },
-{ 6, sizeof (struct tbpos), 0200, tbhresdecode, "\1CN", "\1PT", "\2", "\4",
-  TBF_INPROX },
-{ 5, sizeof (struct tbpos), 0100, tblresdecode, "\1CL\33", "\1PT\33", 0, 0},
-{ 6, sizeof (struct tbpos), 0200, tbhresdecode, "\1CL\33", "\1PT\33", 0, 0},
-};
-
-/*
- * Tablet state
- */
-struct tb {
-	int	tbflags;		/* mode & type bits */
-#define	TBMAXREC	17	/* max input record size */
-	char	cbuf[TBMAXREC];		/* input buffer */
-	union {
-		struct	tbpos tbpos;
-		struct	gtcopos gtcopos;
-		struct	polpos polpos;
-	} rets;				/* processed state */
-#define NTBS	16
-} tb[NTBS];
-
-/*
- * Open as tablet discipline; called on discipline change.
- */
-/*ARGSUSED*/
-tbopen(dev, tp)
-	dev_t dev;
-	register struct tty *tp;
-{
-	register struct tb *tbp;
-
-	if (tp->t_line == TABLDISC)
-		return (ENODEV);
-	ttywflush(tp);
-	for (tbp = tb; tbp < &tb[NTBS]; tbp++)
-		if (tbp->tbflags == 0)
-			break;
-	if (tbp >= &tb[NTBS])
-		return (EBUSY);
-	tbp->tbflags = TBTIGER|TBPOINT;		/* default */
-	tp->t_cp = tbp->cbuf;
-	tp->t_inbuf = 0;
-	bzero((caddr_t)&tbp->rets, sizeof (tbp->rets));
-	tp->T_LINEP = (caddr_t)tbp;
-	tp->t_flags |= LITOUT;
-	return (0);
-}
-
-/*
- * Line discipline change or last device close.
- */
-tbclose(tp)
-	register struct tty *tp;
-{
-	register int s;
-	int modebits = TBPOINT|TBSTOP;
-
-#ifndef NeXT
-	tbioctl(tp, BIOSMODE, &modebits, 0);
-#else
-	tbioctl(tp, BIOSMODE, &modebits, 0, current_proc());
-#endif
-	s = spltty();
-	((struct tb *)tp->T_LINEP)->tbflags = 0;
-	tp->t_cp = 0;
-	tp->t_inbuf = 0;
-	tp->t_rawq.c_cc = 0;		/* clear queues -- paranoid */
-	tp->t_canq.c_cc = 0;
-	tp->t_line = 0;			/* paranoid: avoid races */
-	splx(s);
-}
-
-/*
- * Read from a tablet line.
- * Characters have been buffered in a buffer and decoded.
- */
-tbread(tp, uio)
-	register struct tty *tp;
-	struct uio *uio;
-{
-	register struct tb *tbp = (struct tb *)tp->T_LINEP;
-	register struct tbconf *tc = &tbconf[tbp->tbflags & TBTYPE];
-	int ret;
-
-	if ((tp->t_state&TS_CARR_ON) == 0)
-		return (EIO);
-	ret = uiomove(&tbp->rets, tc->tbc_uiosize, uio);
-	if (tc->tbc_flags&TBF_POL)
-		tbp->rets.polpos.p_key = ' ';
-	return (ret);
-}
-
-/*
- * Low level character input routine.
- * Stuff the character in the buffer, and decode
- * if all the chars are there.
- *
- * This routine could be expanded in-line in the receiver
- * interrupt routine to make it run as fast as possible.
- */
-tbinput(c, tp)
-	register int c;
-	register struct tty *tp;
-{
-	register struct tb *tbp = (struct tb *)tp->T_LINEP;
-	register struct tbconf *tc = &tbconf[tbp->tbflags & TBTYPE];
-
-	if (tc->tbc_recsize == 0 || tc->tbc_decode == 0)	/* paranoid? */
-		return;
-	/*
-	 * Locate sync bit/byte or reset input buffer.
-	 */
-	if (c&tc->tbc_sync || tp->t_inbuf == tc->tbc_recsize) {
-		tp->t_cp = tbp->cbuf;
-		tp->t_inbuf = 0;
-	}
-	*tp->t_cp++ = c&0177;
-	/*
-	 * Call decode routine only if a full record has been collected.
-	 */
-	if (++tp->t_inbuf == tc->tbc_recsize)
-		(*tc->tbc_decode)(tc, tbp->cbuf, &tbp->rets);
-}
-
-/*
- * Decode GTCO 8 byte format (high res, tilt, and pressure).
- */
-static
-gtcodecode(tc, cp, tbpos)
-	struct tbconf *tc;
-	register char *cp;
-	register struct gtcopos *tbpos;
-{
-
-	tbpos->pressure = *cp >> 2;
-	tbpos->status = (tbpos->pressure > 16) | TBINPROX; /* half way down */
-	tbpos->xpos = (*cp++ & 03) << 14;
-	tbpos->xpos |= *cp++ << 7;
-	tbpos->xpos |= *cp++;
-	tbpos->ypos = (*cp++ & 03) << 14;
-	tbpos->ypos |= *cp++ << 7;
-	tbpos->ypos |= *cp++;
-	tbpos->xtilt = *cp++;
-	tbpos->ytilt = *cp++;
-	tbpos->scount++;
-}
-
-/*
- * Decode old Hitachi 5 byte format (low res).
- */
-static
-tbdecode(tc, cp, tbpos)
-	struct tbconf *tc;
-	register char *cp;
-	register struct tbpos *tbpos;
-{
-	register char byte;
-
-	byte = *cp++;
-	tbpos->status = (byte&0100) ? TBINPROX : 0;
-	byte &= ~0100;
-	if (byte > 036)
-		tbpos->status |= 1 << ((byte-040)/2);
-	tbpos->xpos = *cp++ << 7;
-	tbpos->xpos |= *cp++;
-	if (tbpos->xpos < 256)			/* tablet wraps around at 256 */
-		tbpos->status &= ~TBINPROX;	/* make it out of proximity */
-	tbpos->ypos = *cp++ << 7;
-	tbpos->ypos |= *cp++;
-	tbpos->scount++;
-}
-
-/*
- * Decode new Hitach 5-byte format (low res).
- */
-static
-tblresdecode(tc, cp, tbpos)
-	struct tbconf *tc;
-	register char *cp;
-	register struct tbpos *tbpos;
-{
-
-	*cp &= ~0100;		/* mask sync bit */
-	tbpos->status = (*cp++ >> 2) | TBINPROX;
-	if (tc->tbc_flags&TBF_INPROX && tbpos->status&020)
-		tbpos->status &= ~(020|TBINPROX);
-	tbpos->xpos = *cp++;
-	tbpos->xpos |= *cp++ << 6;
-	tbpos->ypos = *cp++;
-	tbpos->ypos |= *cp++ << 6;
-	tbpos->scount++;
-}
-
-/*
- * Decode new Hitach 6-byte format (high res).
- */
-static
-tbhresdecode(tc, cp, tbpos)
-	struct tbconf *tc;
-	register char *cp;
-	register struct tbpos *tbpos;
-{
-	char byte;
-
-	byte = *cp++;
-	tbpos->xpos = (byte & 03) << 14;
-	tbpos->xpos |= *cp++ << 7;
-	tbpos->xpos |= *cp++;
-	tbpos->ypos = *cp++ << 14;
-	tbpos->ypos |= *cp++ << 7;
-	tbpos->ypos |= *cp++;
-	tbpos->status = (byte >> 2) | TBINPROX;
-	if (tc->tbc_flags&TBF_INPROX && tbpos->status&020)
-		tbpos->status &= ~(020|TBINPROX);
-	tbpos->scount++;
-}
-
-/*
- * Polhemus decode.
- */
-static
-poldecode(tc, cp, polpos)
-	struct tbconf *tc;
-	register char *cp;
-	register struct polpos *polpos;
-{
-
-	polpos->p_x = cp[4] | cp[3]<<7 | (cp[9] & 0x03) << 14;
-	polpos->p_y = cp[6] | cp[5]<<7 | (cp[9] & 0x0c) << 12;
-	polpos->p_z = cp[8] | cp[7]<<7 | (cp[9] & 0x30) << 10;
-	polpos->p_azi = cp[11] | cp[10]<<7 | (cp[16] & 0x03) << 14;
-	polpos->p_pit = cp[13] | cp[12]<<7 | (cp[16] & 0x0c) << 12;
-	polpos->p_rol = cp[15] | cp[14]<<7 | (cp[16] & 0x30) << 10;
-	polpos->p_stat = cp[1] | cp[0]<<7;
-	if (cp[2] != ' ')
-		polpos->p_key = cp[2];
-}
-
-/*ARGSUSED*/
-#ifndef NeXT
-tbioctl(tp, cmd, data, flag)
-	struct tty *tp;
-	caddr_t data;
-#else
-tbtioctl(tp, cmd, data, flag, p)
-	struct tty *tp;
-	u_long cmd;
-	caddr_t data;
-	int flag;
-	struct proc *p;
-#endif /* !NeXT */
-{
-	register struct tb *tbp = (struct tb *)tp->T_LINEP;
-
-	switch (cmd) {
-
-	case BIOGMODE:
-		*(int *)data = tbp->tbflags & TBMODE;
-		break;
-
-	case BIOSTYPE:
-		if (tbconf[*(int *)data & TBTYPE].tbc_recsize == 0 ||
-		    tbconf[*(int *)data & TBTYPE].tbc_decode == 0)
-			return (EINVAL);
-		tbp->tbflags &= ~TBTYPE;
-		tbp->tbflags |= *(int *)data & TBTYPE;
-		/* fall thru... to set mode bits */
-
-	case BIOSMODE: {
-		register struct tbconf *tc;
-
-		tbp->tbflags &= ~TBMODE;
-		tbp->tbflags |= *(int *)data & TBMODE;
-		tc = &tbconf[tbp->tbflags & TBTYPE];
-		if (tbp->tbflags&TBSTOP) {
-			if (tc->tbc_stop)
-				ttyout(tc->tbc_stop, tp);
-		} else if (tc->tbc_start)
-			ttyout(tc->tbc_start, tp);
-		if (tbp->tbflags&TBPOINT) {
-			if (tc->tbc_point)
-				ttyout(tc->tbc_point, tp);
-		} else if (tc->tbc_run)
-			ttyout(tc->tbc_run, tp);
-		ttstart(tp);
-		break;
-	}
-
-	case BIOGTYPE:
-		*(int *)data = tbp->tbflags & TBTYPE;
-		break;
-
-	case TIOCSETD:
-	case TIOCGETD:
-	case TIOCGETP:
-	case TIOCGETC:
-		return (-1);		/* pass thru... */
-
-	default:
-		return (ENOTTY);
-	}
-	return (0);
-}
-#endif /* NTB > 0 */
diff --git a/bsd/kern/tty_tty.c b/bsd/kern/tty_tty.c
index fa2a62135..1c9882d96 100644
--- a/bsd/kern/tty_tty.c
+++ b/bsd/kern/tty_tty.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
  * Copyright (c) 1982, 1986, 1991, 1993
  *      The Regents of the University of California.  All rights reserved.
@@ -66,174 +71,153 @@
 #include <sys/tty.h>
 #include <sys/vnode_internal.h>
 #include <sys/file_internal.h>
-#ifndef NeXT
-#include <sys/kernel.h>
-#ifdef DEVFS
-#include <sys/devfsext.h>
-#endif /*DEVFS*/
-
-static	d_open_t	cttyopen;
-static	d_read_t	cttyread;
-static	d_write_t	cttywrite;
-static	d_ioctl_t	cttyioctl;
-static	d_select_t	cttyselect;
-
-#endif /* !NeXT */
+#include <sys/kauth.h>
 
 /* Forward declarations for cdevsw[] entry */
 /* XXX we should consider making these static */
-int cttyopen(dev_t dev, int flag, int mode, struct proc *p);
+int cttyopen(dev_t dev, int flag, int mode, proc_t p);
 int cttyread(dev_t dev, struct uio *uio, int flag);
 int cttywrite(dev_t dev, struct uio *uio, int flag);
-int cttyioctl(dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p);
-int cttyselect(dev_t dev, int flag, void* wql, struct proc *p);
-
-#ifndef NeXT
-
-#define CDEV_MAJOR 1
-/* Don't make static, fdesc_vnops uses this. */
-struct cdevsw ctty_cdevsw = 
-	{ cttyopen,	nullclose,	cttyread,	cttywrite,	/*1*/
-	  cttyioctl,	nullstop,	nullreset,	nodevtotty,/* tty */
-	  cttyselect,	nommap,		NULL,	"ctty",	NULL,	-1 };
+int cttyioctl(dev_t dev, u_long cmd, caddr_t addr, int flag, proc_t p);
+int cttyselect(dev_t dev, int flag, void* wql, proc_t p);
+static vnode_t cttyvp(proc_t p);
 
-#endif /* !NeXT */
 
-#define cttyvp(p) ((p)->p_flag & P_CONTROLT ? (p)->p_session->s_ttyvp : NULL)
-
-/*ARGSUSED*/
 int
-cttyopen(__unused dev_t dev, int flag, __unused int mode, struct proc *p)
+cttyopen(__unused dev_t dev, int flag, __unused int mode, proc_t p)
 {
-	struct vnode *ttyvp = cttyvp(p);
+	vnode_t ttyvp = cttyvp(p);
 	struct vfs_context context;
 	int error;
 
 	if (ttyvp == NULL)
 		return (ENXIO);
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = kauth_cred_proc_ref(p);
+
 	error = VNOP_OPEN(ttyvp, flag, &context);
+	vnode_put(ttyvp);
 	kauth_cred_unref(&context.vc_ucred);
 
 	return (error);
 }
 
-/*ARGSUSED*/
 int
 cttyread(__unused dev_t dev, struct uio *uio, int flag)
 {
-	struct proc *p = current_proc();
-	register struct vnode *ttyvp = cttyvp(p);
+	vnode_t ttyvp = cttyvp(current_proc());
 	struct vfs_context context;
 	int error;
 
 	if (ttyvp == NULL)
 		return (EIO);
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = NOCRED;
 
 	error = VNOP_READ(ttyvp, uio, flag, &context);
+	vnode_put(ttyvp);
 
 	return (error);
 }
 
-/*ARGSUSED*/
 int
 cttywrite(__unused dev_t dev, struct uio *uio, int flag)
 {
-	struct proc *p = current_proc();
-	register struct vnode *ttyvp = cttyvp(p);
+	vnode_t ttyvp = cttyvp(current_proc());
 	struct vfs_context context;
 	int error;
 
 	if (ttyvp == NULL)
 		return (EIO);
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = NOCRED;
 
 	error = VNOP_WRITE(ttyvp, uio, flag, &context);
+	vnode_put(ttyvp);
 
 	return (error);
 }
 
-/*ARGSUSED*/
-#ifndef NeXT
-static	int
-cttyioctl(dev, cmd, addr, flag, p)
-	dev_t dev;
-	int cmd;
-	caddr_t addr;
-	int flag;
-	struct proc *p;
-#else
 int
-cttyioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flag,
-	struct proc *p)
-#endif /* !NeXT */
+cttyioctl(__unused dev_t dev, u_long cmd, caddr_t addr, int flag, proc_t p)
 {
-	struct vnode *ttyvp = cttyvp(p);
+	vnode_t ttyvp = cttyvp(current_proc());
 	struct vfs_context context;
+	struct session *sessp;
+	int error = 0;
 
 	if (ttyvp == NULL)
 		return (EIO);
-	if (cmd == TIOCSCTTY)  /* don't allow controlling tty to be set    */
-		return EINVAL; /* to controlling tty -- infinite recursion */
+	if (cmd == TIOCSCTTY)  { /* don't allow controlling tty to be set    */
+		error = EINVAL; /* to controlling tty -- infinite recursion */
+		goto out;
+	}
 	if (cmd == TIOCNOTTY) {
-		if (!SESS_LEADER(p)) {
-			p->p_flag &= ~P_CONTROLT;
-			return (0);
-		} else
-			return (EINVAL);
+		sessp = proc_session(p);
+		if (!SESS_LEADER(p, sessp)) {
+			OSBitAndAtomic(~((uint32_t)P_CONTROLT), (UInt32 *)&p->p_flag);
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			error = 0;
+			goto out;
+		} else {
+			if (sessp != SESSION_NULL)
+				session_rele(sessp);
+			error = EINVAL;
+			goto out;
+		}
 	}
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = NOCRED;
 
-	return (VNOP_IOCTL(ttyvp, cmd, addr, flag, &context));
+	error = VNOP_IOCTL(ttyvp, cmd, addr, flag, &context);
+out:
+	vnode_put(ttyvp);
+	return (error);
 }
 
-/*ARGSUSED*/
 int
-cttyselect(__unused dev_t dev, int flag, void* wql, struct proc *p)
+cttyselect(__unused dev_t dev, int flag, void* wql, __unused proc_t p)
 {
-	struct vnode *ttyvp = cttyvp(p);
+	vnode_t ttyvp = cttyvp(current_proc());
 	struct vfs_context context;
+	int error;
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = NOCRED;
 
 	if (ttyvp == NULL)
 		return (1);	/* try operation to get EOF/failure */
-	return (VNOP_SELECT(ttyvp, flag, FREAD|FWRITE, wql, &context));
+	error = VNOP_SELECT(ttyvp, flag, FREAD|FWRITE, wql, &context);
+	vnode_put(ttyvp);
+	return (error);
 }
 
-#ifndef NeXT
-static ctty_devsw_installed = 0;
-#ifdef DEVFS
-static 	void	*ctty_devfs_token;
-#endif
-
-static void
-ctty_drvinit(void *unused)
+/* This returns vnode with ioref */
+static vnode_t
+cttyvp(proc_t p)
 {
-	dev_t dev;
-
-	if( ! ctty_devsw_installed ) {
-		dev = makedev(CDEV_MAJOR,0);
-		cdevsw_add(&dev,&ctty_cdevsw,NULL);
-		ctty_devsw_installed = 1;
-#ifdef DEVFS
-		ctty_devfs_token = 
-			devfs_add_devswf(&ctty_cdevsw, 0, DV_CHR, 0, 0, 
-					0666, "tty");
-#endif
-    	}
-}
+	vnode_t vp;
+	int vid;
+	struct session *sessp;
 
-SYSINIT(cttydev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE+CDEV_MAJOR,ctty_drvinit,NULL)
+	sessp = proc_session(p);
 
+	session_lock(sessp);
+	vp = (p->p_flag & P_CONTROLT ? sessp->s_ttyvp : NULLVP);
+	vid = sessp->s_ttyvid;	
+	session_unlock(sessp);
+
+	session_rele(sessp);
+
+	if (vp != NULLVP) {
+		/* cannot get an IO reference, return NULLVP */
+		if (vnode_getwithvid(vp, vid) != 0)
+			vp = NULLVP;
+	}
+	return(vp);
+}
 
-#endif /* !NeXT */
diff --git a/bsd/kern/ubc_subr.c b/bsd/kern/ubc_subr.c
index 1ef210f5c..75abf3d39 100644
--- a/bsd/kern/ubc_subr.c
+++ b/bsd/kern/ubc_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  *	File:	ubc_subr.c
@@ -30,9 +36,6 @@
  *
  */ 
 
-#undef DIAGNOSTIC
-#define DIAGNOSTIC 1
-
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -46,6 +49,7 @@
 #include <sys/kauth.h>
 #include <sys/buf.h>
 #include <sys/user.h>
+#include <sys/codesign.h>
 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
@@ -54,68 +58,322 @@
 #include <mach/upl.h>
 
 #include <kern/kern_types.h>
+#include <kern/kalloc.h>
 #include <kern/zalloc.h>
 #include <kern/thread.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_protos.h> /* last */
 
+#include <libkern/crypto/sha1.h>
+
+/* XXX These should be in a BSD accessible Mach header, but aren't. */
+extern kern_return_t memory_object_pages_resident(memory_object_control_t,
+							boolean_t *);
+extern kern_return_t	memory_object_signed(memory_object_control_t control,
+					     boolean_t is_signed);
+extern void Debugger(const char *message);
+
+
+/* XXX no one uses this interface! */
+kern_return_t ubc_page_op_with_control(
+	memory_object_control_t	 control,
+	off_t		         f_offset,
+	int		         ops,
+	ppnum_t	                 *phys_entryp,
+	int		         *flagsp);
+
+
 #if DIAGNOSTIC
 #if defined(assert)
 #undef assert()
 #endif
 #define assert(cond)    \
-    ((void) ((cond) ? 0 : panic("%s:%d (%s)", __FILE__, __LINE__, # cond)))
+    ((void) ((cond) ? 0 : panic("Assert failed: %s", # cond)))
 #else
 #include <kern/assert.h>
 #endif /* DIAGNOSTIC */
 
-int ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize);
+static int ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize);
 static int ubc_umcallback(vnode_t, void *);
-int ubc_isinuse_locked(vnode_t, int, int);
 static int ubc_msync_internal(vnode_t, off_t, off_t, off_t *, int, int *);
+static void ubc_cs_free(struct ubc_info *uip);
 
 struct zone	*ubc_info_zone;
 
+
+/*
+ * CODESIGNING
+ * Routines to navigate code signing data structures in the kernel...
+ */
+static boolean_t
+cs_valid_range(
+	const void *start,
+	const void *end,
+	const void *lower_bound,
+	const void *upper_bound)
+{
+	if (upper_bound < lower_bound ||
+	    end < start) {
+		return FALSE;
+	}
+
+	if (start < lower_bound ||
+	    end > upper_bound) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+/*
+ * Magic numbers used by Code Signing
+ */
+enum {
+	CSMAGIC_REQUIREMENT = 0xfade0c00,		/* single Requirement blob */
+	CSMAGIC_REQUIREMENTS = 0xfade0c01,		/* Requirements vector (internal requirements) */
+	CSMAGIC_CODEDIRECTORY = 0xfade0c02,		/* CodeDirectory blob */
+	CSMAGIC_EMBEDDED_SIGNATURE = 0xfade0cc0, /* embedded form of signature data */
+	CSMAGIC_EMBEDDED_SIGNATURE_OLD = 0xfade0b02,	/* XXX */
+	CSMAGIC_DETACHED_SIGNATURE = 0xfade0cc1, /* multi-arch collection of embedded signatures */
+	
+	CSSLOT_CODEDIRECTORY = 0,				/* slot index for CodeDirectory */
+};
+
+
+/*
+ * Structure of an embedded-signature SuperBlob
+ */
+typedef struct __BlobIndex {
+	uint32_t type;					/* type of entry */
+	uint32_t offset;				/* offset of entry */
+} CS_BlobIndex;
+
+typedef struct __SuperBlob {
+	uint32_t magic;					/* magic number */
+	uint32_t length;				/* total length of SuperBlob */
+	uint32_t count;					/* number of index entries following */
+	CS_BlobIndex index[];			/* (count) entries */
+	/* followed by Blobs in no particular order as indicated by offsets in index */
+} CS_SuperBlob;
+
+
+/*
+ * C form of a CodeDirectory.
+ */
+typedef struct __CodeDirectory {
+	uint32_t magic;					/* magic number (CSMAGIC_CODEDIRECTORY) */
+	uint32_t length;				/* total length of CodeDirectory blob */
+	uint32_t version;				/* compatibility version */
+	uint32_t flags;					/* setup and mode flags */
+	uint32_t hashOffset;			/* offset of hash slot element at index zero */
+	uint32_t identOffset;			/* offset of identifier string */
+	uint32_t nSpecialSlots;			/* number of special hash slots */
+	uint32_t nCodeSlots;			/* number of ordinary (code) hash slots */
+	uint32_t codeLimit;				/* limit to main image signature range */
+	uint8_t hashSize;				/* size of each hash in bytes */
+	uint8_t hashType;				/* type of hash (cdHashType* constants) */
+	uint8_t spare1;					/* unused (must be zero) */
+	uint8_t	pageSize;				/* log2(page size in bytes); 0 => infinite */
+	uint32_t spare2;				/* unused (must be zero) */
+	/* followed by dynamic content as located by offset fields above */
+} CS_CodeDirectory;
+
+
+/*
+ * Locate the CodeDirectory from an embedded signature blob
+ */
+static const 
+CS_CodeDirectory *findCodeDirectory(
+	const CS_SuperBlob *embedded,
+	char *lower_bound,
+	char *upper_bound)
+{
+	const CS_CodeDirectory *cd = NULL;
+
+	if (embedded &&
+	    cs_valid_range(embedded, embedded + 1, lower_bound, upper_bound) &&
+	    ntohl(embedded->magic) == CSMAGIC_EMBEDDED_SIGNATURE) {
+		const CS_BlobIndex *limit;
+		const CS_BlobIndex *p;
+
+		limit = &embedded->index[ntohl(embedded->count)];
+		if (!cs_valid_range(&embedded->index[0], limit,
+				    lower_bound, upper_bound)) {
+			return NULL;
+		}
+		for (p = embedded->index; p < limit; ++p) {
+			if (ntohl(p->type) == CSSLOT_CODEDIRECTORY) {
+				const unsigned char *base;
+
+				base = (const unsigned char *)embedded;
+				cd = (const CS_CodeDirectory *)(base + ntohl(p->offset));
+				break;
+			}
+		}
+	} else {
+		/*
+		 * Detached signatures come as a bare CS_CodeDirectory,
+		 * without a blob.
+		 */
+		cd = (const CS_CodeDirectory *) embedded;
+	}
+
+	if (cd &&
+	    cs_valid_range(cd, cd + 1, lower_bound, upper_bound) &&
+	    cs_valid_range(cd, (const char *) cd + ntohl(cd->length),
+			   lower_bound, upper_bound) &&
+	    ntohl(cd->magic) == CSMAGIC_CODEDIRECTORY) {
+		return cd;
+	}
+
+	// not found or not a valid code directory
+	return NULL;
+}
+
+
+/*
+ * Locating a page hash
+ */
+static const unsigned char *
+hashes(
+	const CS_CodeDirectory *cd,
+	unsigned page,
+	char *lower_bound,
+	char *upper_bound)
+{
+	const unsigned char *base, *top, *hash;
+	uint32_t nCodeSlots;
+
+	assert(cs_valid_range(cd, cd + 1, lower_bound, upper_bound));
+
+	base = (const unsigned char *)cd + ntohl(cd->hashOffset);
+	nCodeSlots = ntohl(cd->nCodeSlots);
+	top = base + nCodeSlots * SHA1_RESULTLEN;
+	if (!cs_valid_range(base, top, 
+			    lower_bound, upper_bound) ||
+	    page > nCodeSlots) {
+		return NULL;
+	}
+	assert(page < nCodeSlots);
+
+	hash = base + page * SHA1_RESULTLEN;
+	if (!cs_valid_range(hash, hash + SHA1_RESULTLEN,
+			    lower_bound, upper_bound)) {
+		hash = NULL;
+	}
+
+	return hash;
+}
+/*
+ * CODESIGNING
+ * End of routines to navigate code signing data structures in the kernel.
+ */
+
+
 /*
- *	Initialization of the zone for Unified Buffer Cache.
+ * ubc_init
+ * 
+ * Initialization of the zone for Unified Buffer Cache.
+ *
+ * Parameters:	(void)
+ *
+ * Returns:	(void)
+ *
+ * Implicit returns:
+ *		ubc_info_zone(global)	initialized for subsequent allocations
  */
 __private_extern__ void
-ubc_init()
+ubc_init(void)
 {
 	int	i;
 
 	i = (vm_size_t) sizeof (struct ubc_info);
-	/* XXX  the number of elements should be tied in to maxvnodes */
+
 	ubc_info_zone = zinit (i, 10000*i, 8192, "ubc_info zone");
-	return;
 }
 
+
 /*
- *	Initialize a ubc_info structure for a vnode.
+ * ubc_info_init
+ *
+ * Allocate and attach an empty ubc_info structure to a vnode
+ *
+ * Parameters:	vp			Pointer to the vnode
+ *
+ * Returns:	0			Success
+ *	vnode_size:ENOMEM		Not enough space
+ *	vnode_size:???			Other error from vnode_getattr
+ *
  */
 int
 ubc_info_init(struct vnode *vp)
 {
 	return(ubc_info_init_internal(vp, 0, 0));
 }
+
+
+/*
+ * ubc_info_init_withsize
+ *
+ * Allocate and attach a sized ubc_info structure to a vnode
+ *
+ * Parameters:	vp			Pointer to the vnode
+ *		filesize		The size of the file
+ *
+ * Returns:	0			Success
+ *	vnode_size:ENOMEM		Not enough space
+ *	vnode_size:???			Other error from vnode_getattr
+ */
 int
 ubc_info_init_withsize(struct vnode *vp, off_t filesize)
 {
 	return(ubc_info_init_internal(vp, 1, filesize));
 }
 
-int
-ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize)
+
+/*
+ * ubc_info_init_internal
+ *
+ * Allocate and attach a ubc_info structure to a vnode
+ *
+ * Parameters:	vp			Pointer to the vnode
+ *		withfsize{0,1}		Zero if the size should be obtained
+ *					from the vnode; otherwise, use filesize
+ *		filesize		The size of the file, if withfsize == 1
+ *
+ * Returns:	0			Success
+ *	vnode_size:ENOMEM		Not enough space
+ *	vnode_size:???			Other error from vnode_getattr
+ *
+ * Notes:	We call a blocking zalloc(), and the zone was created as an
+ *		expandable and collectable zone, so if no memory is available,
+ *		it is possible for zalloc() to block indefinitely.  zalloc()
+ *		may also panic if the zone of zones is exhausted, since it's
+ *		NOT expandable.
+ *
+ *		We unconditionally call vnode_pager_setup(), even if this is
+ *		a reuse of a ubc_info; in that case, we should probably assert
+ *		that it does not already have a pager association, but do not.
+ *
+ *		Since memory_object_create_named() can only fail from receiving
+ *		an invalid pager argument, the explicit check and panic is
+ *		merely precautionary.
+ */
+static int
+ubc_info_init_internal(vnode_t vp, int withfsize, off_t filesize)
 {
 	register struct ubc_info	*uip;
 	void *  pager;
-	struct proc *p = current_proc();
 	int error = 0;
 	kern_return_t kret;
 	memory_object_control_t control;
 
 	uip = vp->v_ubcinfo;
 
+	/*
+	 * If there is not already a ubc_info attached to the vnode, we
+	 * attach one; otherwise, we will reuse the one that's there.
+	 */
 	if (uip == UBC_INFO_NULL) {
 
 		uip = (struct ubc_info *) zalloc(ubc_info_zone);
@@ -125,28 +383,34 @@ ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize)
 		uip->ui_flags = UI_INITED;
 		uip->ui_ucred = NOCRED;
 	}
-#if DIAGNOSTIC
-	else
-		Debugger("ubc_info_init: already");
-#endif /* DIAGNOSTIC */
-	
 	assert(uip->ui_flags != UI_NONE);
 	assert(uip->ui_vnode == vp);
 
 	/* now set this ubc_info in the vnode */
 	vp->v_ubcinfo = uip;
 
+	/*
+	 * Allocate a pager object for this vnode
+	 *
+	 * XXX The value of the pager parameter is currently ignored.
+	 * XXX Presumably, this API changed to avoid the race between
+	 * XXX setting the pager and the UI_HASPAGER flag.
+	 */
 	pager = (void *)vnode_pager_setup(vp, uip->ui_pager);
 	assert(pager);
 
+	/*
+	 * Explicitly set the pager into the ubc_info, after setting the
+	 * UI_HASPAGER flag.
+	 */
 	SET(uip->ui_flags, UI_HASPAGER);
 	uip->ui_pager = pager;
 
 	/*
 	 * Note: We can not use VNOP_GETATTR() to get accurate
-	 * value of ui_size. Thanks to NFS.
-	 * nfs_getattr() can call vinvalbuf() and in this case
-	 * ubc_info is not set up to deal with that.
+	 * value of ui_size because this may be an NFS vnode, and
+	 * nfs_getattr() can call vinvalbuf(); if this happens,
+	 * ubc_info is not set up to deal with that event.
 	 * So use bogus size.
 	 */
 
@@ -167,29 +431,37 @@ ubc_info_init_internal(struct vnode *vp, int withfsize, off_t filesize)
 	assert(control);
 	uip->ui_control = control;	/* cache the value of the mo control */
 	SET(uip->ui_flags, UI_HASOBJREF);	/* with a named reference */
-#if 0
-	/* create a pager reference on the vnode */
-	error = vnode_pager_vget(vp);
-	if (error)
-		panic("ubc_info_init: vnode_pager_vget error = %d", error);
-#endif
+
 	if (withfsize == 0) {
-		struct vfs_context context;
 		/* initialize the size */
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
-		error = vnode_size(vp, &uip->ui_size, &context);
+		error = vnode_size(vp, &uip->ui_size, vfs_context_current());
 		if (error)
 			uip->ui_size = 0;
 	} else {
 		uip->ui_size = filesize;
 	}
-	vp->v_lflag |= VNAMED_UBC;
+	vp->v_lflag |= VNAMED_UBC;	/* vnode has a named ubc reference */
 
 	return (error);
 }
 
-/* Free the ubc_info */
+
+/*
+ * ubc_info_free
+ *
+ * Free a ubc_info structure
+ *
+ * Parameters:	uip			A pointer to the ubc_info to free
+ *
+ * Returns:	(void)
+ *
+ * Notes:	If there is a credential that has subsequently been associated
+ *		with the ubc_info via a call to ubc_setcred(), the reference
+ *		to the credential is dropped.
+ *
+ *		It's actually impossible for a ubc_info.ui_control to take the
+ *		value MEMORY_OBJECT_CONTROL_NULL.
+ */
 static void
 ubc_info_free(struct ubc_info *uip)
 {
@@ -201,20 +473,49 @@ ubc_info_free(struct ubc_info *uip)
 		memory_object_control_deallocate(uip->ui_control);
 	
 	cluster_release(uip);
+	ubc_cs_free(uip);
 
-	zfree(ubc_info_zone, (vm_offset_t)uip);
+	zfree(ubc_info_zone, uip);
 	return;
 }
 
+
 void
 ubc_info_deallocate(struct ubc_info *uip)
 {
         ubc_info_free(uip);
 }
 
+
 /*
- * Communicate with VM the size change of the file
- * returns 1 on success, 0 on failure
+ * ubc_setsize
+ *
+ * Tell the  VM that the the size of the file represented by the vnode has
+ * changed
+ *
+ * Parameters:	vp			The vp whose backing file size is
+ *					being changed
+ *		nsize			The new size of the backing file
+ *
+ * Returns:	1			Success
+ *		0			Failure
+ *
+ * Notes:	This function will indicate failure if the new size that's
+ *		being attempted to be set is negative.
+ *
+ *		This function will fail if there is no ubc_info currently
+ *		associated with the vnode.
+ *
+ *		This function will indicate success it the new size is the
+ *		same or larger than the old size (in this case, the remainder
+ *		of the file will require modification or use of an existing upl
+ *		to access successfully).
+ *
+ *		This function will fail if the new file size is smaller, and
+ *		the memory region being invalidated was unable to actually be
+ *		invalidated and/or the last page could not be flushed, if the
+ *		new size is not aligned to a page boundary.  This is usually
+ *		indicative of an I/O error.
  */
 int
 ubc_setsize(struct vnode *vp, off_t nsize)
@@ -223,7 +524,7 @@ ubc_setsize(struct vnode *vp, off_t nsize)
 	off_t lastpg, olastpgend, lastoff;
 	struct ubc_info *uip;
 	memory_object_control_t control;
-	kern_return_t kret;
+	kern_return_t kret = KERN_SUCCESS;
 
 	if (nsize < (off_t)0)
 		return (0);
@@ -232,8 +533,10 @@ ubc_setsize(struct vnode *vp, off_t nsize)
 		return (0);
 
 	uip = vp->v_ubcinfo;
-	osize = uip->ui_size;	/* call ubc_getsize() ??? */
-	/* Update the size before flushing the VM */
+	osize = uip->ui_size;
+	/*
+	 * Update the size before flushing the VM
+	 */
 	uip->ui_size = nsize;
 
 	if (nsize >= osize)	/* Nothing more to do */
@@ -242,58 +545,71 @@ ubc_setsize(struct vnode *vp, off_t nsize)
 	/*
 	 * When the file shrinks, invalidate the pages beyond the
 	 * new size. Also get rid of garbage beyond nsize on the
-	 * last page. The ui_size already has the nsize. This
-	 * insures that the pageout would not write beyond the new
-	 * end of the file.
+	 * last page. The ui_size already has the nsize, so any
+	 * subsequent page-in will zero-fill the tail properly
 	 */
-
 	lastpg = trunc_page_64(nsize);
 	olastpgend = round_page_64(osize);
 	control = uip->ui_control;
 	assert(control);
 	lastoff = (nsize & PAGE_MASK_64);
 
-	/*
-	 * If length is multiple of page size, we should not flush
-	 * invalidating is sufficient
-	 */
-	 if (!lastoff) {
-        /* invalidate last page and old contents beyond nsize */
-        kret = memory_object_lock_request(control,
-                    (memory_object_offset_t)lastpg,
-		    (memory_object_size_t)(olastpgend - lastpg), NULL, NULL,
-                    MEMORY_OBJECT_RETURN_NONE, MEMORY_OBJECT_DATA_FLUSH,
-                    VM_PROT_NO_CHANGE);
-        if (kret != KERN_SUCCESS)
-            printf("ubc_setsize: invalidate failed (error = %d)\n", kret);
-
-		return ((kret == KERN_SUCCESS) ? 1 : 0);
-	 }
-
-	/* flush the last page */
-	kret = memory_object_lock_request(control,
-				(memory_object_offset_t)lastpg,
-			        PAGE_SIZE_64, NULL, NULL,
-				MEMORY_OBJECT_RETURN_DIRTY, FALSE,
-				VM_PROT_NO_CHANGE);
-
-	if (kret == KERN_SUCCESS) {
-		/* invalidate last page and old contents beyond nsize */
-		kret = memory_object_lock_request(control,
-					(memory_object_offset_t)lastpg,
-				        (memory_object_size_t)(olastpgend - lastpg), NULL, NULL,
-					MEMORY_OBJECT_RETURN_NONE, MEMORY_OBJECT_DATA_FLUSH,
-					VM_PROT_NO_CHANGE);
+	if (lastoff) {
+	        upl_t		upl;
+		upl_page_info_t	*pl;
+
+
+	        /*
+		 * new EOF ends up in the middle of a page
+		 * zero the tail of this page if its currently
+		 * present in the cache
+		 */
+	        kret = ubc_create_upl(vp, lastpg, PAGE_SIZE, &upl, &pl, UPL_SET_LITE);
+		
 		if (kret != KERN_SUCCESS)
-			printf("ubc_setsize: invalidate failed (error = %d)\n", kret);
-	} else
-		printf("ubc_setsize: flush failed (error = %d)\n", kret);
+		        panic("ubc_setsize: ubc_create_upl (error = %d)\n", kret);
+
+		if (upl_valid_page(pl, 0))
+		        cluster_zero(upl, (uint32_t)lastoff, PAGE_SIZE - (uint32_t)lastoff, NULL);
+
+		ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
 
+		lastpg += PAGE_SIZE_64;
+	}
+	if (olastpgend > lastpg) {
+	        /*
+		 * invalidate the pages beyond the new EOF page
+		 *
+		 */
+	        kret = memory_object_lock_request(control,
+						  (memory_object_offset_t)lastpg,
+						  (memory_object_size_t)(olastpgend - lastpg), NULL, NULL,
+						  MEMORY_OBJECT_RETURN_NONE, MEMORY_OBJECT_DATA_FLUSH,
+						  VM_PROT_NO_CHANGE);
+		if (kret != KERN_SUCCESS)
+		        printf("ubc_setsize: invalidate failed (error = %d)\n", kret);
+	}
 	return ((kret == KERN_SUCCESS) ? 1 : 0);
 }
 
+
 /*
- * Get the size of the file
+ * ubc_getsize
+ *
+ * Get the size of the file assocated with the specified vnode
+ *
+ * Parameters:	vp			The vnode whose size is of interest
+ *
+ * Returns:	0			There is no ubc_info associated with
+ *					this vnode, or the size is zero
+ *		!0			The size of the file
+ *
+ * Notes:	Using this routine, it is not possible for a caller to
+ *		successfully distinguish between a vnode associate with a zero
+ *		length file, and a vnode with no associated ubc_info.  The
+ *		caller therefore needs to not care, or needs to ensure that
+ *		they have previously successfully called ubc_info_init() or
+ *		ubc_info_init_withsize().
  */
 off_t
 ubc_getsize(struct vnode *vp)
@@ -306,12 +622,30 @@ ubc_getsize(struct vnode *vp)
 	return (vp->v_ubcinfo->ui_size);
 }
 
+
 /*
- * call ubc_sync_range(vp, 0, EOF, UBC_PUSHALL) on all the vnodes
- * for this mount point.
- * returns 1 on success, 0 on failure
+ * ubc_umount
+ *
+ * Call ubc_sync_range(vp, 0, EOF, UBC_PUSHALL) on all the vnodes for this
+ * mount point
+ *
+ * Parameters:	mp			The mount point
+ *
+ * Returns:	0			Success
+ *
+ * Notes:	There is no failure indication for this function.
+ *
+ *		This function is used in the unmount path; since it may block
+ *		I/O indefinitely, it should not be used in the forced unmount
+ *		path, since a device unavailability could also block that
+ *		indefinitely.
+ *
+ *		Because there is no device ejection interlock on USB, FireWire,
+ *		or similar devices, it's possible that an ejection that begins
+ *		subsequent to the vnode_iterate() completing, either on one of
+ *		those devices, or a network mount for which the server quits
+ *		responding, etc., may cause the caller to block indefinitely.
  */
-
 __private_extern__ int
 ubc_umount(struct mount *mp)
 {
@@ -319,22 +653,40 @@ ubc_umount(struct mount *mp)
 	return(0);
 }
 
+
+/*
+ * ubc_umcallback
+ *
+ * Used by ubc_umount() as an internal implementation detail; see ubc_umount()
+ * and vnode_iterate() for details of implementation.
+ */
 static int
 ubc_umcallback(vnode_t vp, __unused void * args)
 {
 
 	if (UBCINFOEXISTS(vp)) {
 
-		cluster_push(vp, 0);
-
 		(void) ubc_msync(vp, (off_t)0, ubc_getsize(vp), NULL, UBC_PUSHALL);
 	}
 	return (VNODE_RETURNED);
 }
 
 
-
-/* Get the credentials */
+/*
+ * ubc_getcred
+ *
+ * Get the credentials currently active for the ubc_info associated with the
+ * vnode.
+ *
+ * Parameters:	vp			The vnode whose ubc_info credentials
+ *					are to be retrieved
+ *
+ * Returns:	!NOCRED			The credentials
+ *		NOCRED			If there is no ubc_info for the vnode,
+ *					or if there is one, but it has not had
+ *					any credentials associated with it via
+ *					a call to ubc_setcred()
+ */
 kauth_cred_t
 ubc_getcred(struct vnode *vp)
 {
@@ -344,15 +696,73 @@ ubc_getcred(struct vnode *vp)
 	return (NOCRED);
 }
 
+
+/*
+ * ubc_setthreadcred
+ *
+ * If they are not already set, set the credentials of the ubc_info structure
+ * associated with the vnode to those of the supplied thread; otherwise leave
+ * them alone.
+ *
+ * Parameters:	vp			The vnode whose ubc_info creds are to
+ *					be set
+ *		p			The process whose credentials are to
+ *					be used, if not running on an assumed
+ *					credential
+ *		thread			The thread whose credentials are to
+ *					be used
+ *
+ * Returns:	1			This vnode has no associated ubc_info
+ *		0			Success
+ *
+ * Notes:	This function takes a proc parameter to account for bootstrap
+ *		issues where a task or thread may call this routine, either
+ *		before credentials have been initialized by bsd_init(), or if
+ *		there is no BSD info asscoiate with a mach thread yet.  This
+ *		is known to happen in both the initial swap and memory mapping
+ *		calls.
+ *
+ *		This function is generally used only in the following cases:
+ *
+ *		o	a memory mapped file via the mmap() system call
+ *		o	a memory mapped file via the deprecated map_fd() call
+ *		o	a swap store backing file
+ *		o	subsequent to a successful write via vn_write()
+ *
+ *		The information is then used by the NFS client in order to
+ *		cons up a wire message in either the page-in or page-out path.
+ *
+ *		There are two potential problems with the use of this API:
+ *
+ *		o	Because the write path only set it on a successful
+ *			write, there is a race window between setting the
+ *			credential and its use to evict the pages to the
+ *			remote file server
+ *
+ *		o	Because a page-in may occur prior to a write, the
+ *			credential may not be set at this time, if the page-in
+ *			is not the result of a mapping established via mmap()
+ *			or map_fd().
+ *
+ *		In both these cases, this will be triggered from the paging
+ *		path, which will instead use the credential of the current
+ *		process, which in this case is either the dynamic_pager or
+ *		the kernel task, both of which utilize "root" credentials.
+ *
+ *		This may potentially permit operations to occur which should
+ *		be denied, or it may cause to be denied operations which
+ *		should be permitted, depending on the configuration of the NFS
+ *		server.
+ */
 int
-ubc_setthreadcred(struct vnode *vp, struct proc *p, thread_t thread)
+ubc_setthreadcred(struct vnode *vp, proc_t p, thread_t thread)
 {
 	struct ubc_info *uip;
 	kauth_cred_t credp;
-	struct uthread *uthread = get_bsdthread_info(thread);
+	struct uthread  *uthread = get_bsdthread_info(thread);
 
 	if (!UBCINFOEXISTS(vp))
-		return (1);
+		return (1); 
 
 	vnode_lock(vp);
 
@@ -367,31 +777,57 @@ ubc_setthreadcred(struct vnode *vp, struct proc *p, thread_t thread)
 			uip->ui_ucred = uthread->uu_ucred;
 			kauth_cred_ref(uip->ui_ucred);
 		}
-	}
+	} 
 	vnode_unlock(vp);
 
 	return (0);
 }
 
+
 /*
- * Set the credentials
- * existing credentials are not changed
- * returns 1 on success and 0 on failure
+ * ubc_setcred
+ *
+ * If they are not already set, set the credentials of the ubc_info structure
+ * associated with the vnode to those of the process; otherwise leave them
+ * alone.
+ *
+ * Parameters:	vp			The vnode whose ubc_info creds are to
+ *					be set
+ *		p			The process whose credentials are to
+ *					be used
+ *
+ * Returns:	0			This vnode has no associated ubc_info
+ *		1			Success
+ *
+ * Notes:	The return values for this function are inverted from nearly
+ *		all other uses in the kernel.
+ *
+ *		See also ubc_setthreadcred(), above.
+ *
+ *		This function is considered deprecated, and generally should
+ *		not be used, as it is incompatible with per-thread credentials;
+ *		it exists for legacy KPI reasons.
+ *
+ * DEPRECATION:	ubc_setcred() is being deprecated. Please use 
+ *		ubc_setthreadcred() instead.
  */
 int
-ubc_setcred(struct vnode *vp, struct proc *p)
+ubc_setcred(struct vnode *vp, proc_t p)
 {
 	struct ubc_info *uip;
 	kauth_cred_t credp;
 
-        if ( !UBCINFOEXISTS(vp))
+	/* If there is no ubc_info, deny the operation */
+	if ( !UBCINFOEXISTS(vp))
 		return (0); 
 
+	/*
+	 * Check to see if there is already a credential reference in the
+	 * ubc_info; if there is not, take one on the supplied credential.
+	 */
 	vnode_lock(vp);
-
 	uip = vp->v_ubcinfo;
 	credp = uip->ui_ucred;
-
 	if (!IS_VALID_CRED(credp)) {
 		uip->ui_ucred = kauth_cred_proc_ref(p);
 	} 
@@ -400,7 +836,22 @@ ubc_setcred(struct vnode *vp, struct proc *p)
 	return (1);
 }
 
-/* Get the pager */
+
+/*
+ * ubc_getpager
+ *
+ * Get the pager associated with the ubc_info associated with the vnode.
+ *
+ * Parameters:	vp			The vnode to obtain the pager from
+ *
+ * Returns:	!VNODE_PAGER_NULL	The memory_object_t for the pager
+ *		VNODE_PAGER_NULL	There is no ubc_info for this vnode
+ *
+ * Notes:	For each vnode that has a ubc_info associated with it, that
+ *		ubc_info SHALL have a pager associated with it, so in the
+ *		normal case, it's impossible to return VNODE_PAGER_NULL for
+ *		a vnode with an associated ubc_info.
+ */
 __private_extern__ memory_object_t
 ubc_getpager(struct vnode *vp)
 {
@@ -410,65 +861,150 @@ ubc_getpager(struct vnode *vp)
 	return (0);
 }
 
+
 /*
- * Get the memory object associated with this vnode
- * If the vnode was reactivated, memory object would not exist.
- * Unless "do not rectivate" was specified, look it up using the pager.
- * If hold was requested create an object reference of one does not
- * exist already.
+ * ubc_getobject
+ *
+ * Get the memory object control associated with the ubc_info associated with
+ * the vnode
+ *
+ * Parameters:	vp			The vnode to obtain the memory object
+ *					from
+ *		flags			DEPRECATED
+ *
+ * Returns:	!MEMORY_OBJECT_CONTROL_NULL
+ *		MEMORY_OBJECT_CONTROL_NULL
+ *
+ * Notes:	Historically, if the flags were not "do not reactivate", this
+ *		function would look up the memory object using the pager if
+ *		it did not exist (this could be the case if the vnode had
+ *		been previously reactivated).  The flags would also permit a
+ *		hold to be requested, which would have created an object
+ *		reference, if one had not already existed.  This usage is
+ *		deprecated, as it would permit a race between finding and
+ *		taking the reference vs. a single reference being dropped in
+ *		another thread.
  */
-
 memory_object_control_t
 ubc_getobject(struct vnode *vp, __unused int flags)
 {
         if (UBCINFOEXISTS(vp))
 	        return((vp->v_ubcinfo->ui_control));
 
-	return (0);
+	return (MEMORY_OBJECT_CONTROL_NULL);
 }
 
 
+/*
+ * ubc_blktooff
+ *
+ * Convert a given block number to a memory backing object (file) offset for a
+ * given vnode
+ *
+ * Parameters:	vp			The vnode in which the block is located
+ *		blkno			The block number to convert
+ *
+ * Returns:	!-1			The offset into the backing object
+ *		-1			There is no ubc_info associated with
+ *					the vnode
+ *		-1			An error occurred in the underlying VFS
+ *					while translating the block to an
+ *					offset; the most likely cause is that
+ *					the caller specified a block past the
+ *					end of the file, but this could also be
+ *					any other error from VNOP_BLKTOOFF().
+ *
+ * Note:	Representing the error in band loses some information, but does
+ *		not occlude a valid offset, since an off_t of -1 is normally
+ *		used to represent EOF.  If we had a more reliable constant in
+ *		our header files for it (i.e. explicitly cast to an off_t), we
+ *		would use it here instead.
+ */
 off_t
 ubc_blktooff(vnode_t vp, daddr64_t blkno)
 {
-	off_t file_offset;
+	off_t file_offset = -1;
 	int error;
 
-	if (UBCINVALID(vp))
-	        return ((off_t)-1);
-
-	error = VNOP_BLKTOOFF(vp, blkno, &file_offset);
-	if (error)
-		file_offset = -1;
+	if (UBCINFOEXISTS(vp)) {
+		error = VNOP_BLKTOOFF(vp, blkno, &file_offset);
+		if (error)
+			file_offset = -1;
+	}
 
 	return (file_offset);
 }
 
+
+/*
+ * ubc_offtoblk
+ *
+ * Convert a given offset in a memory backing object into a block number for a
+ * given vnode
+ *
+ * Parameters:	vp			The vnode in which the offset is
+ *					located
+ *		offset			The offset into the backing object
+ *
+ * Returns:	!-1			The returned block number
+ *		-1			There is no ubc_info associated with
+ *					the vnode
+ *		-1			An error occurred in the underlying VFS
+ *					while translating the block to an
+ *					offset; the most likely cause is that
+ *					the caller specified a block past the
+ *					end of the file, but this could also be
+ *					any other error from VNOP_OFFTOBLK().
+ *
+ * Note:	Representing the error in band loses some information, but does
+ *		not occlude a valid block number, since block numbers exceed
+ *		the valid range for offsets, due to their relative sizes.  If
+ *		we had a more reliable constant than -1 in our header files
+ *		for it (i.e. explicitly cast to an daddr64_t), we would use it
+ *		here instead.
+ */
 daddr64_t
 ubc_offtoblk(vnode_t vp, off_t offset)
 {
-	daddr64_t blkno;
+	daddr64_t blkno = -1;
 	int error = 0;
 
-	if (UBCINVALID(vp))
-	        return ((daddr64_t)-1);
-
-	error = VNOP_OFFTOBLK(vp, offset, &blkno);
-	if (error)
-		blkno = -1;
+	if (UBCINFOEXISTS(vp)) {
+		error = VNOP_OFFTOBLK(vp, offset, &blkno);
+		if (error)
+			blkno = -1;
+	}
 
 	return (blkno);
 }
 
+
+/*
+ * ubc_pages_resident
+ *
+ * Determine whether or not a given vnode has pages resident via the memory
+ * object control associated with the ubc_info associated with the vnode
+ *
+ * Parameters:	vp			The vnode we want to know about
+ *
+ * Returns:	1			Yes
+ *		0			No
+ */
 int
 ubc_pages_resident(vnode_t vp)
 {
 	kern_return_t		kret;
 	boolean_t			has_pages_resident;
 	
-	if ( !UBCINFOEXISTS(vp))
+	if (!UBCINFOEXISTS(vp))
 		return (0);
 			
+	/*
+	 * The following call may fail if an invalid ui_control is specified,
+	 * or if there is no VM object associated with the control object.  In
+	 * either case, reacting to it as if there were no pages resident will
+	 * result in correct behavior.
+	 */
 	kret = memory_object_pages_resident(vp->v_ubcinfo->ui_control, &has_pages_resident);
 	
 	if (kret != KERN_SUCCESS)
@@ -481,15 +1017,26 @@ ubc_pages_resident(vnode_t vp)
 }
 
 
-
 /*
- * This interface will eventually be deprecated
+ * ubc_sync_range
+ *
+ * Clean and/or invalidate a range in the memory object that backs this vnode
+ *
+ * Parameters:	vp			The vnode whose associated ubc_info's
+ *					associated memory object is to have a
+ *					range invalidated within it
+ *		beg_off			The start of the range, as an offset
+ *		end_off			The end of the range, as an offset
+ *		flags			See ubc_msync_internal()
  *
- * clean and/or invalidate  a range in the memory object that backs this
- * vnode. The start offset is truncated to the page boundary and the
- * size is adjusted to include the last page in the range.
+ * Returns:	1			Success
+ *		0			Failure
  *
- * returns 1 for success,  0 for failure
+ * Notes:	see ubc_msync_internal() for more detailed information.
+ *
+ * DEPRECATED:	This interface is obsolete due to a failure to return error
+ *		information needed in order to correct failures.  The currently
+ *		recommended interface is ubc_msync().
  */
 int
 ubc_sync_range(vnode_t vp, off_t beg_off, off_t end_off, int flags)
@@ -499,10 +1046,37 @@ ubc_sync_range(vnode_t vp, off_t beg_off, off_t end_off, int flags)
 
 
 /*
- * clean and/or invalidate  a range in the memory object that backs this
- * vnode. The start offset is truncated to the page boundary and the
- * size is adjusted to include the last page in the range.
- * if a
+ * ubc_msync
+ *
+ * Clean and/or invalidate a range in the memory object that backs this vnode
+ *
+ * Parameters:	vp			The vnode whose associated ubc_info's
+ *					associated memory object is to have a
+ *					range invalidated within it
+ *		beg_off			The start of the range, as an offset
+ *		end_off			The end of the range, as an offset
+ *		resid_off		The address of an off_t supplied by the
+ *					caller; may be set to NULL to ignore
+ *		flags			See ubc_msync_internal()
+ *
+ * Returns:	0			Success
+ *		!0			Failure; an errno is returned
+ *
+ * Implicit Returns:
+ *		*resid_off, modified	If non-NULL, the  contents are ALWAYS
+ *					modified; they are initialized to the
+ *					beg_off, and in case of an I/O error,
+ *					the difference between beg_off and the
+ *					current value will reflect what was
+ *					able to be written before the error
+ *					occurred.  If no error is returned, the
+ *					value of the resid_off is undefined; do
+ *					NOT use it in place of end_off if you
+ *					intend to increment from the end of the
+ *					last call and call iteratively.
+ *
+ * Notes:	see ubc_msync_internal() for more detailed information.
+ *
  */
 errno_t
 ubc_msync(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags)
@@ -521,11 +1095,66 @@ ubc_msync(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags)
 }
 
 
-
 /*
- * clean and/or invalidate  a range in the memory object that backs this
- * vnode. The start offset is truncated to the page boundary and the
- * size is adjusted to include the last page in the range.
+ * Clean and/or invalidate a range in the memory object that backs this vnode
+ *
+ * Parameters:	vp			The vnode whose associated ubc_info's
+ *					associated memory object is to have a
+ *					range invalidated within it
+ *		beg_off			The start of the range, as an offset
+ *		end_off			The end of the range, as an offset
+ *		resid_off		The address of an off_t supplied by the
+ *					caller; may be set to NULL to ignore
+ *		flags			MUST contain at least one of the flags
+ *					UBC_INVALIDATE, UBC_PUSHDIRTY, or
+ *					UBC_PUSHALL; if UBC_PUSHDIRTY is used,
+ *					UBC_SYNC may also be specified to cause
+ *					this function to block until the
+ *					operation is complete.  The behavior
+ *					of UBC_SYNC is otherwise undefined.
+ *		io_errno		The address of an int to contain the
+ *					errno from a failed I/O operation, if
+ *					one occurs; may be set to NULL to
+ *					ignore
+ *
+ * Returns:	1			Success
+ *		0			Failure
+ *
+ * Implicit Returns:
+ *		*resid_off, modified	The contents of this offset MAY be
+ *					modified; in case of an I/O error, the
+ *					difference between beg_off and the
+ *					current value will reflect what was
+ *					able to be written before the error
+ *					occurred.
+ *		*io_errno, modified	The contents of this offset are set to
+ *					an errno, if an error occurs; if the
+ *					caller supplies an io_errno parameter,
+ *					they should be careful to initialize it
+ *					to 0 before calling this function to
+ *					enable them to distinguish an error
+ *					with a valid *resid_off from an invalid
+ *					one, and to avoid potentially falsely
+ *					reporting an error, depending on use.
+ *
+ * Notes:	If there is no ubc_info associated with the vnode supplied,
+ *		this function immediately returns success.
+ *
+ *		If the value of end_off is less than or equal to beg_off, this
+ *		function immediately returns success; that is, end_off is NOT
+ *		inclusive.
+ *
+ *		IMPORTANT: one of the flags UBC_INVALIDATE, UBC_PUSHDIRTY, or
+ *		UBC_PUSHALL MUST be specified; that is, it is NOT possible to
+ *		attempt to block on in-progress I/O by calling this function
+ *		with UBC_PUSHDIRTY, and then later call it with just UBC_SYNC
+ *		in order to block pending on the I/O already in progress.
+ *
+ *		The start offset is truncated to the page boundary and the
+ *		size is adjusted to include the last page in the range; that
+ *		is, end_off on exactly a page boundary will not change if it
+ *		is rounded, and the range of bytes written will be from the
+ *		truncate beg_off to the rounded (end_off - 1).
  */
 static int
 ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, int flags, int *io_errno)
@@ -537,10 +1166,10 @@ ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, i
 	
 	if ( !UBCINFOEXISTS(vp))
 	        return (0);
-	if (end_off <= beg_off)
-	        return (0);
 	if ((flags & (UBC_INVALIDATE | UBC_PUSHDIRTY | UBC_PUSHALL)) == 0)
 	        return (0);
+	if (end_off <= beg_off)
+	        return (1);
 
 	if (flags & UBC_INVALIDATE)
 	        /*
@@ -562,8 +1191,8 @@ ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, i
 
 	if (flags & UBC_PUSHALL)
 	        /*
-		 * then return all the interesting pages in the range (both dirty and precious)
-		 * to the pager
+		 * then return all the interesting pages in the range (both
+		 * dirty and precious) to the pager
 		 */
 	        flush_flags = MEMORY_OBJECT_RETURN_ALL;
 
@@ -573,15 +1202,65 @@ ubc_msync_internal(vnode_t vp, off_t beg_off, off_t end_off, off_t *resid_off, i
 
 	/* flush and/or invalidate pages in the range requested */
 	kret = memory_object_lock_request(vp->v_ubcinfo->ui_control,
-					  beg_off, tsize, resid_off, io_errno,
-					  flush_flags, request_flags, VM_PROT_NO_CHANGE);
+					  beg_off, tsize,
+					  (memory_object_offset_t *)resid_off,
+					  io_errno, flush_flags, request_flags,
+					  VM_PROT_NO_CHANGE);
 	
 	return ((kret == KERN_SUCCESS) ? 1 : 0);
 }
 
 
 /*
- * The vnode is mapped explicitly, mark it so.
+ * ubc_msync_internal
+ *
+ * Explicitly map a vnode that has an associate ubc_info, and add a reference
+ * to it for the ubc system, if there isn't one already, so it will not be
+ * recycled while it's in use, and set flags on the ubc_info to indicate that
+ * we have done this
+ *
+ * Parameters:	vp			The vnode to map
+ *		flags			The mapping flags for the vnode; this
+ *					will be a combination of one or more of
+ *					PROT_READ, PROT_WRITE, and PROT_EXEC
+ *
+ * Returns:	0			Success
+ *		EPERM			Permission was denied
+ *
+ * Notes:	An I/O reference on the vnode must already be held on entry
+ *
+ *		If there is no ubc_info associated with the vnode, this function
+ *		will return success.
+ *
+ *		If a permission error occurs, this function will return
+ *		failure; all other failures will cause this function to return
+ *		success.
+ *
+ *		IMPORTANT: This is an internal use function, and its symbols
+ *		are not exported, hence its error checking is not very robust.
+ *		It is primarily used by:
+ *
+ *		o	mmap(), when mapping a file
+ *		o	The deprecated map_fd() interface, when mapping a file
+ *		o	When mapping a shared file (a shared library in the
+ *			shared segment region)
+ *		o	When loading a program image during the exec process
+ *
+ *		...all of these uses ignore the return code, and any fault that
+ *		results later because of a failure is handled in the fix-up path
+ *		of the fault handler.  The interface exists primarily as a
+ *		performance hint.
+ *
+ *		Given that third party implementation of the type of interfaces
+ *		that would use this function, such as alternative executable
+ *		formats, etc., are unsupported, this function is not exported
+ *		for general use.
+ *
+ *		The extra reference is held until the VM system unmaps the
+ *		vnode from its own context to maintain a vnode reference in
+ *		cases like open()/mmap()/close(), which leave the backing
+ *		object referenced by a mapped memory region in a process
+ *		address space.
  */
 __private_extern__ int
 ubc_map(vnode_t vp, int flags)
@@ -589,103 +1268,119 @@ ubc_map(vnode_t vp, int flags)
 	struct ubc_info *uip;
 	int error = 0;
 	int need_ref = 0;
-	struct vfs_context context;
-
-	if (vnode_getwithref(vp))
-	        return (0);
+	int need_wakeup = 0;
 
 	if (UBCINFOEXISTS(vp)) {
-		context.vc_proc = current_proc();
-		context.vc_ucred = kauth_cred_get();
 
-		error = VNOP_MMAP(vp, flags, &context);
+		vnode_lock(vp);
+		uip = vp->v_ubcinfo;
+
+		while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
+			SET(uip->ui_flags, UI_MAPWAITING);
+			(void) msleep(&uip->ui_flags, &vp->v_lock,
+				      PRIBIO, "ubc_map", NULL);
+		}
+		SET(uip->ui_flags, UI_MAPBUSY);
+		vnode_unlock(vp);
+
+		error = VNOP_MMAP(vp, flags, vfs_context_current());
 
 		if (error != EPERM)
 		        error = 0;
 
-		if (error == 0) {
-		        vnode_lock(vp);
-			
-			uip = vp->v_ubcinfo;
+		vnode_lock_spin(vp);
 
+		if (error == 0) {
 			if ( !ISSET(uip->ui_flags, UI_ISMAPPED))
 			        need_ref = 1;
 			SET(uip->ui_flags, (UI_WASMAPPED | UI_ISMAPPED));
+		}
+		CLR(uip->ui_flags, UI_MAPBUSY);
 
-			vnode_unlock(vp);
-			
-			if (need_ref)
-			        vnode_ref(vp);
+		if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
+			CLR(uip->ui_flags, UI_MAPWAITING);
+			need_wakeup = 1;
 		}
-	}
-	vnode_put(vp);
+		vnode_unlock(vp);
 
+		if (need_wakeup)
+			wakeup(&uip->ui_flags);
+
+		if (need_ref)
+			vnode_ref(vp);
+	}
 	return (error);
 }
 
+
 /*
- * destroy the named reference for a given vnode
+ * ubc_destroy_named
+ *
+ * Destroy the named memory object associated with the ubc_info control object
+ * associated with the designated vnode, if there is a ubc_info associated
+ * with the vnode, and a control object is associated with it
+ *
+ * Parameters:	vp			The designated vnode
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This function is called on vnode termination for all vnodes,
+ *		and must therefore not assume that there is a ubc_info that is
+ *		associated with the vnode, nor that there is a control object
+ *		associated with the ubc_info.
+ *
+ *		If all the conditions necessary are present, this function
+ *		calls memory_object_destory(), which will in turn end up
+ *		calling ubc_unmap() to release any vnode references that were
+ *		established via ubc_map().
+ *
+ *		IMPORTANT: This is an internal use function that is used
+ *		exclusively by the internal use function vclean().
  */
-__private_extern__ int
-ubc_destroy_named(struct vnode	*vp)
+__private_extern__ void
+ubc_destroy_named(vnode_t vp)
 {
 	memory_object_control_t control;
 	struct ubc_info *uip;
 	kern_return_t kret;
 
-	/*
-	 * We may already have had the object terminated
-	 * and the ubcinfo released as a side effect of
-	 * some earlier processing.  If so, pretend we did
-	 * it, because it probably was a result of our
-	 * efforts.
-	 */
-	if (!UBCINFOEXISTS(vp))
-		return (1);
-
-	uip = vp->v_ubcinfo;
-
-	/* 
-	 * Terminate the memory object.
-	 * memory_object_destroy() will result in
-	 * vnode_pager_no_senders(). 
-	 * That will release the pager reference
-	 * and the vnode will move to the free list.
-	 */
-	control = ubc_getobject(vp, UBC_HOLDOBJECT);
-	if (control != MEMORY_OBJECT_CONTROL_NULL) {
-
-	  /*
-	   * XXXXX - should we hold the vnode lock here?
-	   */
-		if (ISSET(vp->v_flag, VTERMINATE))
-			panic("ubc_destroy_named: already teminating");
-		SET(vp->v_flag, VTERMINATE);
-
-		kret = memory_object_destroy(control, 0);
-		if (kret != KERN_SUCCESS)
-			return (0);
-
-		/* 
-		 * memory_object_destroy() is asynchronous
-		 * with respect to vnode_pager_no_senders().
-		 * wait for vnode_pager_no_senders() to clear
-		 * VTERMINATE
-		 */
-		vnode_lock(vp);
-		while (ISSET(vp->v_lflag, VNAMED_UBC)) {
-			(void)msleep((caddr_t)&vp->v_lflag, &vp->v_lock,
-						 PINOD, "ubc_destroy_named", 0);
+	if (UBCINFOEXISTS(vp)) {
+	        uip = vp->v_ubcinfo;
+
+		/* Terminate the memory object  */
+		control = ubc_getobject(vp, UBC_HOLDOBJECT);
+		if (control != MEMORY_OBJECT_CONTROL_NULL) {
+		        kret = memory_object_destroy(control, 0);
+			if (kret != KERN_SUCCESS)
+			        panic("ubc_destroy_named: memory_object_destroy failed");
 		}
-		vnode_unlock(vp);
 	}
-	return (1);
 }
 
 
 /*
- * Find out whether a vnode is in use by UBC
- * Returns 1 if file is in use by UBC, 0 if not
+ * ubc_isinuse
+ *
+ * Determine whether or not a vnode is currently in use by ubc at a level in
+ * excess of the requested busycount
+ *
+ * Parameters:	vp			The vnode to check
+ *		busycount		The threshold busy count, used to bias
+ *					the count usually already held by the
+ *					caller to avoid races
+ *
+ * Returns:	1			The vnode is in use over the threshold
+ *		0			The vnode is not in use over the
+ *					threshold
+ *
+ * Notes:	Because the vnode is only held locked while actually asking
+ *		the use count, this function only represents a snapshot of the
+ *		current state of the vnode.  If more accurate information is
+ *		required, an additional busycount should be held by the caller
+ *		and a non-zero busycount used.
+ *
+ *		If there is no ubc_info associated with the vnode, this
+ *		function will report that the vnode is not in use by ubc.
  */
 int
 ubc_isinuse(struct vnode *vp, int busycount)
@@ -696,6 +1391,34 @@ ubc_isinuse(struct vnode *vp, int busycount)
 }
 
 
+/*
+ * ubc_isinuse_locked
+ *
+ * Determine whether or not a vnode is currently in use by ubc at a level in
+ * excess of the requested busycount
+ *
+ * Parameters:	vp			The vnode to check
+ *		busycount		The threshold busy count, used to bias
+ *					the count usually already held by the
+ *					caller to avoid races
+ *		locked			True if the vnode is already locked by
+ *					the caller
+ *
+ * Returns:	1			The vnode is in use over the threshold
+ *		0			The vnode is not in use over the
+ *					threshold
+ *
+ * Notes:	If the vnode is not locked on entry, it is locked while
+ *		actually asking the use count.  If this is the case, this
+ *		function only represents a snapshot of the current state of
+ *		the vnode.  If more accurate information is required, the
+ *		vnode lock should be held by the caller, otherwise an
+ *		additional busycount should be held by the caller and a
+ *		non-zero busycount used.
+ *
+ *		If there is no ubc_info associated with the vnode, this
+ *		function will report that the vnode is not in use by ubc.
+ */
 int
 ubc_isinuse_locked(struct vnode *vp, int busycount, int locked)
 {
@@ -715,22 +1438,56 @@ ubc_isinuse_locked(struct vnode *vp, int busycount, int locked)
 
 
 /*
- * MUST only be called by the VM
+ * ubc_unmap
+ *
+ * Reverse the effects of a ubc_map() call for a given vnode
+ *
+ * Parameters:	vp			vnode to unmap from ubc
+ *
+ * Returns:	(void)
+ *
+ * Notes:	This is an internal use function used by vnode_pager_unmap().
+ *		It will attempt to obtain a reference on the supplied vnode,
+ *		and if it can do so, and there is an associated ubc_info, and
+ *		the flags indicate that it was mapped via ubc_map(), then the
+ *		flag is cleared, the mapping removed, and the reference taken
+ *		by ubc_map() is released.
+ *
+ *		IMPORTANT: This MUST only be called by the VM
+ *		to prevent race conditions.
  */
 __private_extern__ void
 ubc_unmap(struct vnode *vp)
 {
-	struct vfs_context context;
 	struct ubc_info *uip;
 	int	need_rele = 0;
+	int	need_wakeup = 0;
+#if NAMEDRSRCFORK 
+	int	named_fork = 0;
+#endif
 
 	if (vnode_getwithref(vp))
 	        return;
 
 	if (UBCINFOEXISTS(vp)) {
 		vnode_lock(vp);
-
 		uip = vp->v_ubcinfo;
+
+		while (ISSET(uip->ui_flags, UI_MAPBUSY)) {
+			SET(uip->ui_flags, UI_MAPWAITING);
+			(void) msleep(&uip->ui_flags, &vp->v_lock,
+				      PRIBIO, "ubc_unmap", NULL);
+		}
+		SET(uip->ui_flags, UI_MAPBUSY);
+
+#if NAMEDRSRCFORK
+		if ((vp->v_flag & VISNAMEDSTREAM) &&
+		    (vp->v_parent != NULLVP) && 
+		    !(vp->v_parent->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)) {
+		    	named_fork = 1;
+		}
+#endif
+
 		if (ISSET(uip->ui_flags, UI_ISMAPPED)) {
 		        CLR(uip->ui_flags, UI_ISMAPPED);
 			need_rele = 1;
@@ -738,12 +1495,29 @@ ubc_unmap(struct vnode *vp)
 		vnode_unlock(vp);
 		
 		if (need_rele) {
-		        context.vc_proc = current_proc();
-			context.vc_ucred = kauth_cred_get();
-		        (void)VNOP_MNOMAP(vp, &context);
+		        (void)VNOP_MNOMAP(vp, vfs_context_current());
+
+#if NAMEDRSRCFORK
+			if (named_fork) {
+				vnode_relenamedstream(vp->v_parent, vp, vfs_context_current());
+			}
+#endif
 
 		        vnode_rele(vp);
 		}
+
+		vnode_lock_spin(vp);
+
+		CLR(uip->ui_flags, UI_MAPBUSY);
+		if (ISSET(uip->ui_flags, UI_MAPWAITING)) {
+			CLR(uip->ui_flags, UI_MAPWAITING);
+			need_wakeup = 1;
+		}
+		vnode_unlock(vp);
+
+		if (need_wakeup)
+		        wakeup(&uip->ui_flags);
+
 	}
 	/*
 	 * the drop of the vnode ref will cleanup
@@ -751,6 +1525,82 @@ ubc_unmap(struct vnode *vp)
 	vnode_put(vp);
 }
 
+
+/*
+ * ubc_page_op
+ *
+ * Manipulate individual page state for a vnode with an associated ubc_info
+ * with an associated memory object control.
+ *
+ * Parameters:	vp			The vnode backing the page
+ *		f_offset		A file offset interior to the page
+ *		ops			The operations to perform, as a bitmap
+ *					(see below for more information)
+ *		phys_entryp		The address of a ppnum_t; may be NULL
+ *					to ignore
+ *		flagsp			A pointer to an int to contain flags;
+ *					may be NULL to ignore
+ *
+ * Returns:	KERN_SUCCESS		Success
+ *		KERN_INVALID_ARGUMENT	If the memory object control has no VM
+ *					object associated
+ *		KERN_INVALID_OBJECT	If UPL_POP_PHYSICAL and the object is
+ *					not physically contiguous
+ *		KERN_INVALID_OBJECT	If !UPL_POP_PHYSICAL and the object is
+ *					physically contiguous
+ *		KERN_FAILURE		If the page cannot be looked up
+ *
+ * Implicit Returns:
+ *		*phys_entryp (modified)	If phys_entryp is non-NULL and
+ *					UPL_POP_PHYSICAL
+ *		*flagsp (modified)	If flagsp is non-NULL and there was
+ *					!UPL_POP_PHYSICAL and a KERN_SUCCESS
+ *
+ * Notes:	For object boundaries, it is considerably more efficient to
+ *		ensure that f_offset is in fact on a page boundary, as this
+ *		will avoid internal use of the hash table to identify the
+ *		page, and would therefore skip a number of early optimizations.
+ *		Since this is a page operation anyway, the caller should try
+ *		to pass only a page aligned offset because of this.
+ *
+ *		*flagsp may be modified even if this function fails.  If it is
+ *		modified, it will contain the condition of the page before the
+ *		requested operation was attempted; these will only include the
+ *		bitmap flags, and not the PL_POP_PHYSICAL, UPL_POP_DUMP,
+ *		UPL_POP_SET, or UPL_POP_CLR bits.
+ *
+ *		The flags field may contain a specific operation, such as
+ *		UPL_POP_PHYSICAL or UPL_POP_DUMP:
+ *
+ *		o	UPL_POP_PHYSICAL	Fail if not contiguous; if
+ *						*phys_entryp and successful, set
+ *						*phys_entryp
+ *		o	UPL_POP_DUMP		Dump the specified page
+ *
+ *		Otherwise, it is treated as a bitmap of one or more page
+ *		operations to perform on the final memory object; allowable
+ *		bit values are:
+ *
+ *		o	UPL_POP_DIRTY		The page is dirty
+ *		o	UPL_POP_PAGEOUT		The page is paged out
+ *		o	UPL_POP_PRECIOUS	The page is precious
+ *		o	UPL_POP_ABSENT		The page is absent
+ *		o	UPL_POP_BUSY		The page is busy
+ *
+ *		If the page status is only being queried and not modified, then
+ *		not other bits should be specified.  However, if it is being
+ *		modified, exactly ONE of the following bits should be set:
+ *
+ *		o	UPL_POP_SET		Set the current bitmap bits
+ *		o	UPL_POP_CLR		Clear the current bitmap bits
+ *
+ *		Thus to effect a combination of setting an clearing, it may be
+ *		necessary to call this function twice.  If this is done, the
+ *		set should be used before the clear, since clearing may trigger
+ *		a wakeup on the destination page, and if the page is backed by
+ *		an encrypted swap file, setting will trigger the decryption
+ *		needed before the wakeup occurs.
+ */
 kern_return_t
 ubc_page_op(
 	struct vnode 	*vp,
@@ -771,22 +1621,66 @@ ubc_page_op(
 				      phys_entryp,
 				      flagsp));
 }
-				      
-__private_extern__ kern_return_t
-ubc_page_op_with_control(
-	memory_object_control_t	 control,
-	off_t		         f_offset,
-	int		         ops,
-	ppnum_t	                 *phys_entryp,
-	int		         *flagsp)
-{
-	return (memory_object_page_op(control,
-				      (memory_object_offset_t)f_offset,
-				      ops,
-				      phys_entryp,
-				      flagsp));
-}
-				      
+
+
+/*
+ * ubc_range_op
+ *
+ * Manipulate page state for a range of memory for a vnode with an associated
+ * ubc_info with an associated memory object control, when page level state is
+ * not required to be returned from the call (i.e. there are no phys_entryp or
+ * flagsp parameters to this call, and it takes a range which may contain
+ * multiple pages, rather than an offset interior to a single page).
+ *
+ * Parameters:	vp			The vnode backing the page
+ *		f_offset_beg		A file offset interior to the start page
+ *		f_offset_end		A file offset interior to the end page
+ *		ops			The operations to perform, as a bitmap
+ *					(see below for more information)
+ *		range			The address of an int; may be NULL to
+ *					ignore
+ *
+ * Returns:	KERN_SUCCESS		Success
+ *		KERN_INVALID_ARGUMENT	If the memory object control has no VM
+ *					object associated
+ *		KERN_INVALID_OBJECT	If the object is physically contiguous
+ *
+ * Implicit Returns:
+ *		*range (modified)	If range is non-NULL, its contents will
+ *					be modified to contain the number of
+ *					bytes successfully operated upon.
+ *
+ * Notes:	IMPORTANT: This function cannot be used on a range that
+ *		consists of physically contiguous pages.
+ *
+ *		For object boundaries, it is considerably more efficient to
+ *		ensure that f_offset_beg and f_offset_end are in fact on page
+ *		boundaries, as this will avoid internal use of the hash table
+ *		to identify the page, and would therefore skip a number of
+ *		early optimizations.  Since this is an operation on a set of
+ *		pages anyway, the caller should try to pass only a page aligned
+ *		offsets because of this.
+ *
+ *		*range will be modified only if this function succeeds.
+ *
+ *		The flags field MUST contain a specific operation; allowable
+ *		values are:
+ *
+ *		o	UPL_ROP_ABSENT	Returns the extent of the range
+ *					presented which is absent, starting
+ *					with the start address presented
+ *
+ *		o	UPL_ROP_PRESENT	Returns the extent of the range
+ *					presented which is present (resident),
+ *					starting with the start address
+ *					presented
+ *		o	UPL_ROP_DUMP	Dump the pages which are found in the
+ *					target object for the target range.
+ *
+ *		IMPORTANT: For UPL_ROP_ABSENT and UPL_ROP_PRESENT; if there are
+ *		multiple regions in the range, only the first matching region
+ *		is returned.
+ */
 kern_return_t
 ubc_range_op(
 	struct vnode 	*vp,
@@ -807,18 +1701,57 @@ ubc_range_op(
 				      ops,
 				      range));
 }
-				      
+
+
+/*
+ * ubc_create_upl
+ *
+ * Given a vnode, cause the population of a portion of the vm_object; based on
+ * the nature of the request, the pages returned may contain valid data, or
+ * they may be uninitialized.
+ *
+ * Parameters:	vp			The vnode from which to create the upl
+ *		f_offset		The start offset into the backing store
+ *					represented by the vnode
+ *		bufsize			The size of the upl to create
+ *		uplp			Pointer to the upl_t to receive the
+ *					created upl; MUST NOT be NULL
+ *		plp			Pointer to receive the internal page
+ *					list for the created upl; MAY be NULL
+ *					to ignore
+ *
+ * Returns:	KERN_SUCCESS		The requested upl has been created
+ *		KERN_INVALID_ARGUMENT	The bufsize argument is not an even
+ *					multiple of the page size
+ *		KERN_INVALID_ARGUMENT	There is no ubc_info associated with
+ *					the vnode, or there is no memory object
+ *					control associated with the ubc_info 
+ *	memory_object_upl_request:KERN_INVALID_VALUE
+ *					The supplied upl_flags argument is
+ *					invalid
+ * Implicit Returns:
+ *		*uplp (modified)	
+ *		*plp (modified)		If non-NULL, the value of *plp will be
+ *					modified to point to the internal page
+ *					list; this modification may occur even
+ *					if this function is unsuccessful, in
+ *					which case the contents may be invalid
+ *
+ * Note:	If successful, the returned *uplp MUST subsequently be freed
+ *		via a call to ubc_upl_commit(), ubc_upl_commit_range(),
+ *		ubc_upl_abort(), or ubc_upl_abort_range().
+ */
 kern_return_t
 ubc_create_upl(
 	struct vnode	*vp,
-	off_t 			f_offset,
-	long			bufsize,
-	upl_t			*uplp,
+	off_t 		f_offset,
+	long		bufsize,
+	upl_t		*uplp,
 	upl_page_info_t	**plp,
-	int				uplflags)
+	int		uplflags)
 {
 	memory_object_control_t		control;
-	int				count;
+	mach_msg_type_number_t		count;
 	int                             ubcflags;
 	kern_return_t			kr;
 	
@@ -841,14 +1774,47 @@ ubc_create_upl(
 	} else
 	        uplflags |= (UPL_NO_SYNC|UPL_CLEAN_IN_PLACE|UPL_SET_INTERNAL);
 	count = 0;
-	kr = memory_object_upl_request(control, f_offset, bufsize,
-								   uplp, NULL, &count, uplflags);
+
+	kr = memory_object_upl_request(control, f_offset, bufsize, uplp, NULL, &count, uplflags);
 	if (plp != NULL)
 			*plp = UPL_GET_INTERNAL_PAGE_LIST(*uplp);
 	return kr;
 }
-				      
+		
+					  		      
+/*
+ * ubc_upl_maxbufsize
+ *
+ * Return the maximum bufsize ubc_create_upl( ) will take.
+ *
+ * Parameters:	none
+ *
+ * Returns:	maximum size buffer (in bytes) ubc_create_upl( ) will take.
+ */
+upl_size_t 
+ubc_upl_maxbufsize(
+	void)
+{
+	return(MAX_UPL_TRANSFER * PAGE_SIZE);
+}
 
+/*
+ * ubc_upl_map
+ *
+ * Map the page list assocated with the supplied upl into the kernel virtual
+ * address space at the virtual address indicated by the dst_addr argument;
+ * the entire upl is mapped
+ *
+ * Parameters:	upl			The upl to map
+ *		dst_addr		The address at which to map the upl
+ *
+ * Returns:	KERN_SUCCESS		The upl has been mapped
+ *		KERN_INVALID_ARGUMENT	The upl is UPL_NULL
+ *		KERN_FAILURE		The upl is already mapped
+ *	vm_map_enter:KERN_INVALID_ARGUMENT
+ *					A failure code from vm_map_enter() due
+ *					to an invalid argument
+ */
 kern_return_t
 ubc_upl_map(
 	upl_t		upl,
@@ -858,6 +1824,18 @@ ubc_upl_map(
 }
 
 
+/*
+ * ubc_upl_unmap
+ *
+ * Unmap the page list assocated with the supplied upl from the kernel virtual
+ * address space; the entire upl is unmapped.
+ *
+ * Parameters:	upl			The upl to unmap
+ *
+ * Returns:	KERN_SUCCESS		The upl has been unmapped
+ *		KERN_FAILURE		The upl is not currently mapped
+ *		KERN_INVALID_ARGUMENT	If the upl is UPL_NULL
+ */
 kern_return_t
 ubc_upl_unmap(
 	upl_t	upl)
@@ -865,6 +1843,30 @@ ubc_upl_unmap(
 	return(vm_upl_unmap(kernel_map, upl));
 }
 
+
+/*
+ * ubc_upl_commit
+ *
+ * Commit the contents of the upl to the backing store
+ *
+ * Parameters:	upl			The upl to commit
+ *
+ * Returns:	KERN_SUCCESS		The upl has been committed
+ *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
+ *		KERN_FAILURE		The supplied upl does not represent
+ *					device memory, and the offset plus the
+ *					size would exceed the actual size of
+ *					the upl
+ *
+ * Notes:	In practice, the only return value for this function should be
+ *		KERN_SUCCESS, unless there has been data structure corruption;
+ *		since the upl is deallocated regardless of success or failure,
+ *		there's really nothing to do about this other than panic.
+ *
+ *		IMPORTANT: Use of this function should not be mixed with use of
+ *		ubc_upl_commit_range(), due to the unconditional deallocation
+ *		by this function.
+ */
 kern_return_t
 ubc_upl_commit(
 	upl_t 			upl)
@@ -879,6 +1881,58 @@ ubc_upl_commit(
 }
 
 
+/*
+ * ubc_upl_commit
+ *
+ * Commit the contents of the specified range of the upl to the backing store
+ *
+ * Parameters:	upl			The upl to commit
+ *		offset			The offset into the upl
+ *		size			The size of the region to be committed,
+ *					starting at the specified offset
+ *		flags			commit type (see below)
+ *
+ * Returns:	KERN_SUCCESS		The range has been committed
+ *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
+ *		KERN_FAILURE		The supplied upl does not represent
+ *					device memory, and the offset plus the
+ *					size would exceed the actual size of
+ *					the upl
+ *
+ * Notes:	IMPORTANT: If the commit is successful, and the object is now
+ *		empty, the upl will be deallocated.  Since the caller cannot
+ *		check that this is the case, the UPL_COMMIT_FREE_ON_EMPTY flag
+ *		should generally only be used when the offset is 0 and the size
+ *		is equal to the upl size.
+ *
+ *		The flags argument is a bitmap of flags on the rage of pages in
+ *		the upl to be committed; allowable flags are:
+ *
+ *		o	UPL_COMMIT_FREE_ON_EMPTY	Free the upl when it is
+ *							both empty and has been
+ *							successfully committed
+ *		o	UPL_COMMIT_CLEAR_DIRTY		Clear each pages dirty
+ *							bit; will prevent a
+ *							later pageout
+ *		o	UPL_COMMIT_SET_DIRTY		Set each pages dirty
+ *							bit; will cause a later
+ *							pageout
+ *		o	UPL_COMMIT_INACTIVATE		Clear each pages
+ *							reference bit; the page
+ *							will not be accessed
+ *		o	UPL_COMMIT_ALLOW_ACCESS		Unbusy each page; pages
+ *							become busy when an
+ *							IOMemoryDescriptor is
+ *							mapped or redirected,
+ *							and we have to wait for
+ *							an IOKit driver
+ *
+ *		The flag UPL_COMMIT_NOTIFY_EMPTY is used internally, and should
+ *		not be specified by the caller.
+ *
+ *		The UPL_COMMIT_CLEAR_DIRTY and UPL_COMMIT_SET_DIRTY flags are
+ *		mutually exclusive, and should not be combined.
+ */
 kern_return_t
 ubc_upl_commit_range(
 	upl_t 			upl,
@@ -903,7 +1957,53 @@ ubc_upl_commit_range(
 
 	return kr;
 }
-	
+
+
+/*
+ * ubc_upl_abort_range
+ *
+ * Abort the contents of the specified range of the specified upl
+ *
+ * Parameters:	upl			The upl to abort
+ *		offset			The offset into the upl
+ *		size			The size of the region to be aborted,
+ *					starting at the specified offset
+ *		abort_flags		abort type (see below)
+ *
+ * Returns:	KERN_SUCCESS		The range has been aborted
+ *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
+ *		KERN_FAILURE		The supplied upl does not represent
+ *					device memory, and the offset plus the
+ *					size would exceed the actual size of
+ *					the upl
+ *
+ * Notes:	IMPORTANT: If the abort is successful, and the object is now
+ *		empty, the upl will be deallocated.  Since the caller cannot
+ *		check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
+ *		should generally only be used when the offset is 0 and the size
+ *		is equal to the upl size.
+ *
+ *		The abort_flags argument is a bitmap of flags on the range of
+ *		pages in the upl to be aborted; allowable flags are:
+ *
+ *		o	UPL_ABORT_FREE_ON_EMPTY	Free the upl when it is both
+ *						empty and has been successfully
+ *						aborted
+ *		o	UPL_ABORT_RESTART	The operation must be restarted
+ *		o	UPL_ABORT_UNAVAILABLE	The pages are unavailable
+ *		o	UPL_ABORT_ERROR		An I/O error occurred
+ *		o	UPL_ABORT_DUMP_PAGES	Just free the pages
+ *		o	UPL_ABORT_NOTIFY_EMPTY	RESERVED
+ *		o	UPL_ABORT_ALLOW_ACCESS	RESERVED
+ *
+ *		The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
+ *		not be specified by the caller.  It is intended to fulfill the
+ *		same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
+ *		ubc_upl_commit_range(), but is never referenced internally.
+ *
+ *		The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
+ *		referenced; do not use it.
+ */
 kern_return_t
 ubc_upl_abort_range(
 	upl_t			upl,
@@ -925,6 +2025,49 @@ ubc_upl_abort_range(
 	return kr;
 }
 
+
+/*
+ * ubc_upl_abort
+ *
+ * Abort the contents of the specified upl
+ *
+ * Parameters:	upl			The upl to abort
+ *		abort_type		abort type (see below)
+ *
+ * Returns:	KERN_SUCCESS		The range has been aborted
+ *		KERN_INVALID_ARGUMENT	The supplied upl was UPL_NULL
+ *		KERN_FAILURE		The supplied upl does not represent
+ *					device memory, and the offset plus the
+ *					size would exceed the actual size of
+ *					the upl
+ *
+ * Notes:	IMPORTANT: If the abort is successful, and the object is now
+ *		empty, the upl will be deallocated.  Since the caller cannot
+ *		check that this is the case, the UPL_ABORT_FREE_ON_EMPTY flag
+ *		should generally only be used when the offset is 0 and the size
+ *		is equal to the upl size.
+ *
+ *		The abort_type is a bitmap of flags on the range of
+ *		pages in the upl to be aborted; allowable flags are:
+ *
+ *		o	UPL_ABORT_FREE_ON_EMPTY	Free the upl when it is both
+ *						empty and has been successfully
+ *						aborted
+ *		o	UPL_ABORT_RESTART	The operation must be restarted
+ *		o	UPL_ABORT_UNAVAILABLE	The pages are unavailable
+ *		o	UPL_ABORT_ERROR		An I/O error occurred
+ *		o	UPL_ABORT_DUMP_PAGES	Just free the pages
+ *		o	UPL_ABORT_NOTIFY_EMPTY	RESERVED
+ *		o	UPL_ABORT_ALLOW_ACCESS	RESERVED
+ *
+ *		The UPL_ABORT_NOTIFY_EMPTY is an internal use flag and should
+ *		not be specified by the caller.  It is intended to fulfill the
+ *		same role as UPL_COMMIT_NOTIFY_EMPTY does in the function
+ *		ubc_upl_commit_range(), but is never referenced internally.
+ *
+ *		The UPL_ABORT_ALLOW_ACCESS is defined, but neither set nor
+ *		referenced; do not use it.
+ */
 kern_return_t
 ubc_upl_abort(
 	upl_t			upl,
@@ -937,6 +2080,25 @@ ubc_upl_abort(
 	return kr;
 }
 
+
+/*
+ * ubc_upl_pageinfo
+ *
+ *  Retrieve the internal page list for the specified upl
+ *
+ * Parameters:	upl			The upl to obtain the page list from
+ *
+ * Returns:	!NULL			The (upl_page_info_t *) for the page
+ *					list internal to the upl
+ *		NULL			Error/no page list associated
+ *
+ * Notes:	IMPORTANT: The function is only valid on internal objects
+ *		where the list request was made with the UPL_INTERNAL flag.
+ *
+ *		This function is a utility helper function, since some callers
+ *		may not have direct access to the header defining the macro,
+ *		due to abstraction layering constraints.
+ */
 upl_page_info_t *
 ubc_upl_pageinfo(
 	upl_t			upl)
@@ -944,45 +2106,570 @@ ubc_upl_pageinfo(
 	return (UPL_GET_INTERNAL_PAGE_LIST(upl));
 }
 
-/************* UBC APIS **************/
 
 int 
-UBCINFOMISSING(struct vnode * vp)
+UBCINFOEXISTS(struct vnode * vp)
 {
-	return((vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo == UBC_INFO_NULL));
+        return((vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo != UBC_INFO_NULL));
 }
 
-int 
-UBCINFORECLAIMED(struct vnode * vp)
+
+/*
+ * CODE SIGNING
+ */
+#define CS_BLOB_KEEP_IN_KERNEL 1
+static volatile SInt32 cs_blob_size = 0;
+static volatile SInt32 cs_blob_count = 0;
+static SInt32 cs_blob_size_peak = 0;
+static UInt32 cs_blob_size_max = 0;
+static SInt32 cs_blob_count_peak = 0;
+extern int cs_debug;
+
+int cs_validation = 1;
+
+SYSCTL_INT(_vm, OID_AUTO, cs_validation, CTLFLAG_RW, &cs_validation, 0, "Do validate code signatures");
+SYSCTL_INT(_vm, OID_AUTO, cs_blob_count, CTLFLAG_RD, &cs_blob_count, 0, "Current number of code signature blobs");
+SYSCTL_INT(_vm, OID_AUTO, cs_blob_size, CTLFLAG_RD, &cs_blob_size, 0, "Current size of all code signature blobs");
+SYSCTL_INT(_vm, OID_AUTO, cs_blob_count_peak, CTLFLAG_RD, &cs_blob_count_peak, 0, "Peak number of code signature blobs");
+SYSCTL_INT(_vm, OID_AUTO, cs_blob_size_peak, CTLFLAG_RD, &cs_blob_size_peak, 0, "Peak size of code signature blobs");
+SYSCTL_INT(_vm, OID_AUTO, cs_blob_size_max, CTLFLAG_RD, &cs_blob_size_max, 0, "Size of biggest code signature blob");
+
+int
+ubc_cs_blob_add(
+	struct vnode	*vp,
+	cpu_type_t	cputype,
+	off_t		base_offset,
+	vm_address_t	addr,
+	vm_size_t	size)
 {
-	return((vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo == UBC_INFO_NULL));
+	kern_return_t		kr;
+	struct ubc_info		*uip;
+	struct cs_blob		*blob, *oblob;
+	int			error;
+	ipc_port_t		blob_handle;
+	memory_object_size_t	blob_size;
+	const CS_CodeDirectory *cd;
+	off_t			blob_start_offset, blob_end_offset;
+	SHA1_CTX		sha1ctxt;
+
+	blob_handle = IPC_PORT_NULL;
+
+	blob = (struct cs_blob *) kalloc(sizeof (struct cs_blob));
+	if (blob == NULL) {
+		return ENOMEM;
+	}
+
+	/* get a memory entry on the blob */
+	blob_size = (memory_object_size_t) size;
+	kr = mach_make_memory_entry_64(kernel_map,
+				       &blob_size,
+				       addr,
+				       VM_PROT_READ,
+				       &blob_handle,
+				       IPC_PORT_NULL);
+	if (kr != KERN_SUCCESS) {
+		error = ENOMEM;
+		goto out;
+	}
+	if (memory_object_round_page(blob_size) !=
+	    (memory_object_size_t) round_page(size)) {
+		printf("ubc_cs_blob_add: size mismatch 0x%llx 0x%x !?\n",
+		       blob_size, size);
+		panic("XXX FBDP size mismatch 0x%llx 0x%x\n", blob_size, size);
+		error = EINVAL;
+		goto out;
+	}
+
+
+	/* fill in the new blob */
+	blob->csb_cpu_type = cputype;
+	blob->csb_base_offset = base_offset;
+	blob->csb_mem_size = size;
+	blob->csb_mem_offset = 0;
+	blob->csb_mem_handle = blob_handle;
+	blob->csb_mem_kaddr = addr;
+
+	
+	/*
+	 * Validate the blob's contents
+	 */
+	cd = findCodeDirectory(
+		(const CS_SuperBlob *) addr, 
+		(char *) addr, 
+		(char *) addr + blob->csb_mem_size);
+	if (cd == NULL) {
+		/* no code directory => useless blob ! */
+		blob->csb_flags = 0;
+		blob->csb_start_offset = 0;
+		blob->csb_end_offset = 0;
+	} else {
+		unsigned char *sha1_base;
+		int sha1_size;
+
+		blob->csb_flags = ntohl(cd->flags) | CS_VALID;
+		blob->csb_end_offset = round_page(ntohl(cd->codeLimit));
+		blob->csb_start_offset = (blob->csb_end_offset -
+					  (ntohl(cd->nCodeSlots) * PAGE_SIZE));
+		/* compute the blob's SHA1 hash */
+		sha1_base = (const unsigned char *) cd;
+		sha1_size = ntohl(cd->length);
+		SHA1Init(&sha1ctxt);
+		SHA1Update(&sha1ctxt, sha1_base, sha1_size);
+		SHA1Final(blob->csb_sha1, &sha1ctxt);
+	}
+
+
+	/*
+	 * Validate the blob's coverage
+	 */
+	blob_start_offset = blob->csb_base_offset + blob->csb_start_offset;
+	blob_end_offset = blob->csb_base_offset + blob->csb_end_offset;
+
+	if (blob_start_offset >= blob_end_offset) {
+		/* reject empty or backwards blob */
+		error = EINVAL;
+		goto out;
+	}
+
+	vnode_lock(vp);
+	if (! UBCINFOEXISTS(vp)) {
+		vnode_unlock(vp);
+		error = ENOENT;
+		goto out;
+	}
+	uip = vp->v_ubcinfo;
+
+	/* check if this new blob overlaps with an existing blob */
+	for (oblob = uip->cs_blobs;
+	     oblob != NULL;
+	     oblob = oblob->csb_next) {
+		 off_t oblob_start_offset, oblob_end_offset;
+
+		 oblob_start_offset = (oblob->csb_base_offset +
+				       oblob->csb_start_offset);
+		 oblob_end_offset = (oblob->csb_base_offset +
+				     oblob->csb_end_offset);
+		 if (blob_start_offset >= oblob_end_offset ||
+		     blob_end_offset <= oblob_start_offset) {
+			 /* no conflict with this existing blob */
+		 } else {
+			 /* conflict ! */
+			 if (blob_start_offset == oblob_start_offset &&
+			     blob_end_offset == oblob_end_offset &&
+			     blob->csb_mem_size == oblob->csb_mem_size &&
+			     blob->csb_flags == oblob->csb_flags &&
+			     (blob->csb_cpu_type == CPU_TYPE_ANY ||
+			      oblob->csb_cpu_type == CPU_TYPE_ANY ||
+			      blob->csb_cpu_type == oblob->csb_cpu_type) &&
+			     !bcmp(blob->csb_sha1,
+				   oblob->csb_sha1,
+				   SHA1_RESULTLEN)) {
+				 /*
+				  * We already have this blob:
+				  * we'll return success but
+				  * throw away the new blob.
+				  */
+				 if (oblob->csb_cpu_type == CPU_TYPE_ANY) {
+					 /*
+					  * The old blob matches this one
+					  * but doesn't have any CPU type.
+					  * Update it with whatever the caller
+					  * provided this time.
+					  */
+					 oblob->csb_cpu_type = cputype;
+				 }
+				 vnode_unlock(vp);
+				 error = EAGAIN;
+				 goto out;
+			 } else {
+				 /* different blob: reject the new one */
+				 vnode_unlock(vp);
+				 error = EALREADY;
+				 goto out;
+			 }
+		 }
+
+	}
+
+
+	/* mark this vnode's VM object as having "signed pages" */
+	kr = memory_object_signed(uip->ui_control, TRUE);
+	if (kr != KERN_SUCCESS) {
+		vnode_unlock(vp);
+		error = ENOENT;
+		goto out;
+	}
+
+	/*
+	 * Add this blob to the list of blobs for this vnode.
+	 * We always add at the front of the list and we never remove a
+	 * blob from the list, so ubc_cs_get_blobs() can return whatever
+	 * the top of the list was and that list will remain valid
+	 * while we validate a page, even after we release the vnode's lock.
+	 */
+	blob->csb_next = uip->cs_blobs;
+	uip->cs_blobs = blob;
+
+	OSAddAtomic(+1, &cs_blob_count);
+	if (cs_blob_count > cs_blob_count_peak) {
+		cs_blob_count_peak = cs_blob_count; /* XXX atomic ? */
+	}
+	OSAddAtomic(+blob->csb_mem_size, &cs_blob_size);
+	if (cs_blob_size > cs_blob_size_peak) {
+		cs_blob_size_peak = cs_blob_size; /* XXX atomic ? */
+	}
+	if (blob->csb_mem_size > cs_blob_size_max) {
+		cs_blob_size_max = blob->csb_mem_size;
+	}
+
+	if (cs_debug) {
+		proc_t p;
+
+		p = current_proc();
+		printf("CODE SIGNING: proc %d(%s) "
+		       "loaded %s signatures for file (%s) "
+		       "range 0x%llx:0x%llx flags 0x%x\n",
+		       p->p_pid, p->p_comm,
+		       blob->csb_cpu_type == -1 ? "detached" : "embedded",
+		       vnode_name(vp),
+		       blob->csb_base_offset + blob->csb_start_offset,
+		       blob->csb_base_offset + blob->csb_end_offset,
+		       blob->csb_flags);
+	}
+
+#if !CS_BLOB_KEEP_IN_KERNEL
+	blob->csb_mem_kaddr = 0;
+#endif /* CS_BLOB_KEEP_IN_KERNEL */
+
+	vnode_unlock(vp);
+
+	error = 0;	/* success ! */
+
+out:
+	if (error) {
+		/* we failed; release what we allocated */
+		if (blob) {
+			kfree(blob, sizeof (*blob));
+			blob = NULL;
+		}
+		if (blob_handle != IPC_PORT_NULL) {
+			mach_memory_entry_port_release(blob_handle);
+			blob_handle = IPC_PORT_NULL;
+		}
+	} else {
+#if !CS_BLOB_KEEP_IN_KERNEL
+		kmem_free(kernel_map, addr, size);
+#endif /* CS_BLOB_KEEP_IN_KERNEL */
+	}
+
+	if (error == EAGAIN) {
+		/*
+		 * See above:  error is EAGAIN if we were asked 
+		 * to add an existing blob again.  We cleaned the new
+		 * blob and we want to return success.
+		 */
+		error = 0;
+		/*
+		 * Since we're not failing, consume the data we received.
+		 */
+		kmem_free(kernel_map, addr, size);
+	}
+
+	return error;
 }
 
 
-int 
-UBCINFOEXISTS(struct vnode * vp)
+struct cs_blob *
+ubc_cs_blob_get(
+	struct vnode	*vp,
+	cpu_type_t	cputype,
+	off_t		offset)
 {
-        return((vp) && ((vp)->v_type == VREG) && ((vp)->v_ubcinfo != UBC_INFO_NULL));
+	struct ubc_info	*uip;
+	struct cs_blob	*blob;
+	off_t offset_in_blob;
+
+	vnode_lock_spin(vp);
+
+	if (! UBCINFOEXISTS(vp)) {
+		blob = NULL;
+		goto out;
+	}
+
+	uip = vp->v_ubcinfo;
+	for (blob = uip->cs_blobs;
+	     blob != NULL;
+	     blob = blob->csb_next) {
+		if (cputype != -1 && blob->csb_cpu_type == cputype) {
+			break;
+		}
+		if (offset != -1) {
+			offset_in_blob = offset - blob->csb_base_offset;
+			if (offset_in_blob >= blob->csb_start_offset &&
+			    offset_in_blob < blob->csb_end_offset) {
+				/* our offset is covered by this blob */
+				break;
+			}
+		}
+	}
+
+out:
+	vnode_unlock(vp);
+
+	return blob;
 }
-int 
-UBCISVALID(struct vnode * vp)
+
+static void
+ubc_cs_free(
+	struct ubc_info	*uip)
 {
-	return((vp) && ((vp)->v_type == VREG) && !((vp)->v_flag & VSYSTEM));
+	struct cs_blob	*blob, *next_blob;
+
+	for (blob = uip->cs_blobs;
+	     blob != NULL;
+	     blob = next_blob) {
+		next_blob = blob->csb_next;
+		if (blob->csb_mem_kaddr != 0) {
+			kmem_free(kernel_map,
+				  blob->csb_mem_kaddr,
+				  blob->csb_mem_size);
+			blob->csb_mem_kaddr = 0;
+		}
+		mach_memory_entry_port_release(blob->csb_mem_handle);
+		blob->csb_mem_handle = IPC_PORT_NULL;
+		OSAddAtomic(-1, &cs_blob_count);
+		OSAddAtomic(-blob->csb_mem_size, &cs_blob_size);
+		kfree(blob, sizeof (*blob));
+	}
+	uip->cs_blobs = NULL;
 }
-int 
-UBCINVALID(struct vnode * vp)
+
+struct cs_blob *
+ubc_get_cs_blobs(
+	struct vnode	*vp)
 {
-	return(((vp) == NULL) || ((vp) && ((vp)->v_type != VREG))
-		|| ((vp) && ((vp)->v_flag & VSYSTEM)));
+	struct ubc_info	*uip;
+	struct cs_blob	*blobs;
+
+	vnode_lock_spin(vp);
+
+	if (! UBCINFOEXISTS(vp)) {
+		blobs = NULL;
+		goto out;
+	}
+
+	uip = vp->v_ubcinfo;
+	blobs = uip->cs_blobs;
+
+out:
+	vnode_unlock(vp);
+
+	return blobs;
 }
-int 
-UBCINFOCHECK(const char * fun, struct vnode * vp)
+
+unsigned long cs_validate_page_no_hash = 0;
+unsigned long cs_validate_page_bad_hash = 0;
+boolean_t
+cs_validate_page(
+	void			*_blobs,
+	memory_object_offset_t	page_offset,
+	const void		*data,
+	boolean_t		*tainted)
 {
-	if ((vp) && ((vp)->v_type == VREG) &&
-		((vp)->v_ubcinfo == UBC_INFO_NULL)) {
-		panic("%s: lost ubc_info", (fun));
-		return(1);
-	} else
-		return(0);
+	SHA1_CTX		sha1ctxt;
+	unsigned char		actual_hash[SHA1_RESULTLEN];
+	unsigned char		expected_hash[SHA1_RESULTLEN];
+	boolean_t		found_hash;
+	struct cs_blob		*blobs, *blob;
+	const CS_CodeDirectory	*cd;
+	const CS_SuperBlob	*embedded;
+	off_t			start_offset, end_offset;
+	const unsigned char	*hash;
+	boolean_t		validated;
+	off_t			offset;	/* page offset in the file */
+	size_t			size;
+	off_t			codeLimit = 0;
+	char			*lower_bound, *upper_bound;
+	vm_offset_t		kaddr, blob_addr;
+	vm_size_t		ksize;
+	kern_return_t		kr;
+
+	offset = page_offset;
+
+	/* retrieve the expected hash */
+	found_hash = FALSE;
+	blobs = (struct cs_blob *) _blobs;
+
+	for (blob = blobs;
+	     blob != NULL;
+	     blob = blob->csb_next) {
+		offset = page_offset - blob->csb_base_offset;
+		if (offset < blob->csb_start_offset ||
+		    offset >= blob->csb_end_offset) {
+			/* our page is not covered by this blob */
+			continue;
+		}
+
+		/* map the blob in the kernel address space */
+		kaddr = blob->csb_mem_kaddr;
+		if (kaddr == 0) {
+			ksize = (vm_size_t) (blob->csb_mem_size +
+					     blob->csb_mem_offset);
+			kr = vm_map(kernel_map,
+				    &kaddr,
+				    ksize,
+				    0,
+				    VM_FLAGS_ANYWHERE,
+				    blob->csb_mem_handle,
+				    0,
+				    TRUE,
+				    VM_PROT_READ,
+				    VM_PROT_READ,
+				    VM_INHERIT_NONE);
+			if (kr != KERN_SUCCESS) {
+				/* XXX FBDP what to do !? */
+				printf("cs_validate_page: failed to map blob, "
+				       "size=0x%x kr=0x%x\n",
+				       blob->csb_mem_size, kr);
+				break;
+			}
+		}
+		blob_addr = kaddr + blob->csb_mem_offset;
+		
+		lower_bound = CAST_DOWN(char *, blob_addr);
+		upper_bound = lower_bound + blob->csb_mem_size;
+
+		embedded = (const CS_SuperBlob *) blob_addr;
+		cd = findCodeDirectory(embedded, lower_bound, upper_bound);
+		if (cd != NULL) {
+			if (cd->pageSize != PAGE_SHIFT ||
+			    cd->hashType != 0x1 ||
+			    cd->hashSize != SHA1_RESULTLEN) {
+				/* bogus blob ? */
+#if !CS_BLOB_KEEP_IN_KERNEL
+				kmem_free(kernel_map, kaddr, ksize);
+#endif /* CS_BLOB_KEEP_IN_KERNEL */
+				continue;
+			}
+			    
+			end_offset = round_page(ntohl(cd->codeLimit));
+			start_offset = end_offset - (ntohl(cd->nCodeSlots) * PAGE_SIZE);
+			offset = page_offset - blob->csb_base_offset;
+			if (offset < start_offset ||
+			    offset >= end_offset) {
+				/* our page is not covered by this blob */
+#if !CS_BLOB_KEEP_IN_KERNEL
+				kmem_free(kernel_map, kaddr, ksize);
+#endif /* CS_BLOB_KEEP_IN_KERNEL */
+				continue;
+			}
+
+			codeLimit = ntohl(cd->codeLimit);
+			hash = hashes(cd, atop(offset),
+				      lower_bound, upper_bound);
+			bcopy(hash, expected_hash, sizeof (expected_hash));
+			found_hash = TRUE;
+
+#if !CS_BLOB_KEEP_IN_KERNEL
+			/* we no longer need that blob in the kernel map */
+			kmem_free(kernel_map, kaddr, ksize);
+#endif /* CS_BLOB_KEEP_IN_KERNEL */
+
+			break;
+		}
+	}
+
+	if (found_hash == FALSE) {
+		/*
+		 * We can't verify this page because there is no signature
+		 * for it (yet).  It's possible that this part of the object
+		 * is not signed, or that signatures for that part have not
+		 * been loaded yet.
+		 * Report that the page has not been validated and let the
+		 * caller decide if it wants to accept it or not.
+		 */
+		cs_validate_page_no_hash++;
+		if (cs_debug > 1) {
+			printf("CODE SIGNING: cs_validate_page: "
+			       "off 0x%llx: no hash to validate !?\n",
+			       page_offset);
+		}
+		validated = FALSE;
+		*tainted = FALSE;
+	} else {
+		const uint32_t *asha1, *esha1;
+
+		size = PAGE_SIZE;
+		if (offset + size > codeLimit) {
+			/* partial page at end of segment */
+			assert(offset < codeLimit);
+			size = codeLimit & PAGE_MASK;
+		}
+		/* compute the actual page's SHA1 hash */
+		SHA1Init(&sha1ctxt);
+		SHA1Update(&sha1ctxt, data, size);
+		SHA1Final(actual_hash, &sha1ctxt);
+
+		asha1 = (const uint32_t *) actual_hash;
+		esha1 = (const uint32_t *) expected_hash;
+
+		if (bcmp(expected_hash, actual_hash, SHA1_RESULTLEN) != 0) {
+			if (cs_debug) {
+				printf("CODE SIGNING: cs_validate_page: "
+				       "off 0x%llx size 0x%lx: "
+				       "actual [0x%x 0x%x 0x%x 0x%x 0x%x] != "
+				       "expected [0x%x 0x%x 0x%x 0x%x 0x%x]\n",
+				       page_offset, size,
+				       asha1[0], asha1[1], asha1[2],
+				       asha1[3], asha1[4],
+				       esha1[0], esha1[1], esha1[2],
+				       esha1[3], esha1[4]);
+			}
+			cs_validate_page_bad_hash++;
+			*tainted = TRUE;
+		} else {
+			if (cs_debug > 1) {
+				printf("CODE SIGNING: cs_validate_page: "
+				       "off 0x%llx size 0x%lx: SHA1 OK\n",
+				       page_offset, size);
+			}
+			*tainted = FALSE;
+		}
+		validated = TRUE;
+	}
+	
+	return validated;
 }
 
+int
+ubc_cs_getcdhash(
+	vnode_t		vp,
+	off_t		offset,
+	unsigned char	*cdhash)
+{
+	struct cs_blob *blobs, *blob;
+	off_t rel_offset;
+
+	blobs = ubc_get_cs_blobs(vp);
+	for (blob = blobs;
+	     blob != NULL;
+	     blob = blob->csb_next) {
+		/* compute offset relative to this blob */
+		rel_offset = offset - blob->csb_base_offset;
+		if (rel_offset >= blob->csb_start_offset &&
+		    rel_offset < blob->csb_end_offset) {
+			/* this blob does cover our "offset" ! */
+			break;
+		}
+	}
+
+	if (blob == NULL) {
+		/* we didn't find a blob covering "offset" */
+		return EBADEXEC; /* XXX any better error ? */
+	}
+
+	/* get the SHA1 hash of that blob */
+	bcopy(blob->csb_sha1, cdhash, sizeof (blob->csb_sha1));
+
+	return 0;
+}
diff --git a/bsd/kern/uipc_domain.c b/bsd/kern/uipc_domain.c
index 2f63a9203..c8a1df41d 100644
--- a/bsd/kern/uipc_domain.c
+++ b/bsd/kern/uipc_domain.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -69,9 +74,22 @@
 #include <sys/syslog.h>
 #include <sys/queue.h>
 
+#include <pexpert/pexpert.h>
+
+void domaininit(void) __attribute__((section("__TEXT, initcode")));
+void init_domain(struct domain *dp) __attribute__((section("__TEXT, initcode")));
+void concat_domain(struct domain *dp) __attribute__((section("__TEXT, initcode")));
+
+
 void	pffasttimo(void *);
 void	pfslowtimo(void *);
 
+struct protosw *pffindprotonotype(int, int);
+struct protosw *pffindprotonotype_locked(int , int , int);
+struct domain *pffinddomain(int);
+void concat_domain(struct domain *);
+void init_domain(struct domain *);
+
 /*
  * Add/delete 'domain': Link structure into system list,
  *  invoke the domain init, and then the proto inits.
@@ -84,7 +102,19 @@ static lck_grp_attr_t	*domain_proto_mtx_grp_attr;
 lck_mtx_t		*domain_proto_mtx;
 extern int		do_reclaim;
 
-void init_domain(register struct domain *dp)
+static void
+init_proto(struct protosw *pr)
+{
+	TAILQ_INIT(&pr->pr_filter_head);
+	if (pr->pr_init)
+		(*pr->pr_init)();
+
+	/* Make sure pr_init isn't called again!! */
+	pr->pr_init = 0;
+}
+
+void
+init_domain(struct domain *dp)
 {
 	struct protosw  *pr;
 	
@@ -104,8 +134,8 @@ void init_domain(register struct domain *dp)
 			      dp->dom_name, 
 			      (int)(pr - dp->dom_protosw));
 
-		if (pr->pr_init)
-			(*pr->pr_init)();
+		init_proto(pr);
+
 	}
 
 	/* Recompute for new protocol */
@@ -117,7 +147,8 @@ void init_domain(register struct domain *dp)
 	max_datalen = MHLEN - max_hdr;
 }
 
-void	concat_domain(struct domain *dp) 
+void
+concat_domain(struct domain *dp) 
 {
 	lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_OWNED);
 	dp->dom_next = domains; 
@@ -125,9 +156,8 @@ void	concat_domain(struct domain *dp)
 }
 
 void
-net_add_domain(register struct domain *dp)
-{	register struct protosw *pr;
-
+net_add_domain(struct domain *dp)
+{
 	kprintf("Adding domain %s (family %d)\n", dp->dom_name,
 		dp->dom_family);
 	/* First, link in the domain */
@@ -141,7 +171,7 @@ net_add_domain(register struct domain *dp)
 }
 
 int
-net_del_domain(register struct domain *dp)
+net_del_domain(struct domain *dp)
 {	register struct domain *dp1, *dp2;
 	register int retval = 0;
 
@@ -174,8 +204,7 @@ net_del_domain(register struct domain *dp)
  * note: protocols must use their own domain lock before calling net_add_proto
  */
 int
-net_add_proto(register struct protosw *pp,
-	      register struct domain *dp)
+net_add_proto(struct protosw *pp, struct domain *dp)
 {	register struct protosw *pp1, *pp2;
 
 	for (pp2 = NULL, pp1 = dp->dom_protosw; pp1; pp1 = pp1->pr_next)
@@ -189,13 +218,9 @@ net_add_proto(register struct protosw *pp,
 		dp->dom_protosw = pp;
 	else
 		pp2->pr_next = pp;
-	pp->pr_next = NULL;
-	TAILQ_INIT(&pp->pr_filter_head);
-	if (pp->pr_init)
-		(*pp->pr_init)();
 
-	/* Make sure pr_init isn't called again!! */
-	pp->pr_init = 0;
+	init_proto(pp);
+
 	return(0);
 }
 
@@ -207,10 +232,9 @@ net_add_proto(register struct protosw *pp,
  * note: protocols must use their own domain lock before calling net_del_proto
  */
 int
-net_del_proto(register int type,
-	      register int protocol,
-	      register struct domain *dp)
-{	register struct protosw *pp1, *pp2;
+net_del_proto(int type, int protocol, struct domain *dp)
+{
+	register struct protosw *pp1, *pp2;
 
 	for (pp2 = NULL, pp1 = dp->dom_protosw; pp1; pp1 = pp1->pr_next)
 	{	if (pp1->pr_type == type &&
@@ -229,32 +253,34 @@ net_del_proto(register int type,
 }
 
 
-void
-domaininit()
-{	register struct domain *dp;
-	register struct protosw *pr;
-	extern struct domain localdomain, routedomain, ndrvdomain, inetdomain;
-	extern struct domain systemdomain;
 #if NS
-	extern struct domain nsdomain;
+extern struct domain nsdomain;
 #endif
 #if ISO
-	extern struct domain isodomain;
+extern struct domain isodomain;
 #endif
 #if CCITT
-	extern struct domain ccittdomain;
+extern struct domain ccittdomain;
 #endif
 
 #if NETAT
-	extern struct domain atalkdomain;
+extern struct domain atalkdomain;
 #endif
 #if INET6
-	extern struct domain inet6domain;
+extern struct domain inet6domain;
 #endif
 #if IPSEC
-	extern struct domain keydomain;
+extern struct domain keydomain;
 #endif
 
+extern struct domain routedomain, ndrvdomain, inetdomain;
+extern struct domain systemdomain;
+
+void
+domaininit(void)
+{
+	register struct domain *dp;
+
 	/*
 	 * allocate lock group attribute and group for domain mutexes
 	 */
@@ -315,21 +341,34 @@ domaininit()
 	timeout(pfslowtimo, NULL, 1);
 }
 
+static __inline__ struct domain *
+pffinddomain_locked(int pf)
+{
+	struct domain *dp;
+
+	dp = domains;
+	while (dp != NULL)
+	{	if (dp->dom_family == pf) {
+			break;
+		}
+		dp = dp->dom_next;
+	}
+	return (dp);
+}
+
 struct protosw *
-pffindtype(family, type)
-	int family, type;
+pffindtype(int family, int type)
 {
 	register struct domain *dp;
 	register struct protosw *pr;
 
 	lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(domain_proto_mtx);
-	for (dp = domains; dp; dp = dp->dom_next)
-		if (dp->dom_family == family)
-			goto found;
+	dp = pffinddomain_locked(family);
+	if (dp == NULL) {
 	lck_mtx_unlock(domain_proto_mtx);
-	return (0);
-found:
+		return (NULL);
+	}
 	for (pr = dp->dom_protosw; pr; pr = pr->pr_next)
 		if (pr->pr_type && pr->pr_type == type) {
 			lck_mtx_unlock(domain_proto_mtx);
@@ -341,25 +380,18 @@ found:
 
 struct domain *
 pffinddomain(int pf)
-{	struct domain *dp;
+{
+	struct domain *dp;
 
 	lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(domain_proto_mtx);
-	dp = domains;
-	while (dp)
-	{	if (dp->dom_family == pf) {
+	dp = pffinddomain_locked(pf);
 			lck_mtx_unlock(domain_proto_mtx);
 			return(dp);
 		}
-		dp = dp->dom_next;
-	}
-	lck_mtx_unlock(domain_proto_mtx);
-	return(NULL);
-}
 
 struct protosw *
-pffindproto(family, protocol, type)
-	int family, protocol, type;
+pffindproto(int family, int protocol, int type)
 {
 	register struct protosw *pr;
 	lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
@@ -370,8 +402,7 @@ pffindproto(family, protocol, type)
 }
 
 struct protosw *
-pffindproto_locked(family, protocol, type)
-	int family, protocol, type;
+pffindproto_locked(int family, int protocol, int type)
 {
 	register struct domain *dp;
 	register struct protosw *pr;
@@ -379,11 +410,10 @@ pffindproto_locked(family, protocol, type)
 
 	if (family == 0)
 		return (0);
-	for (dp = domains; dp; dp = dp->dom_next)
-		if (dp->dom_family == family)
-			goto found;
-	return (0);
-found:
+	dp = pffinddomain_locked(family);
+	if (dp == NULL) {
+		return (NULL);
+	}
 	for (pr = dp->dom_protosw; pr; pr = pr->pr_next) {
 		if ((pr->pr_protocol == protocol) && (pr->pr_type == type))
 			return (pr);
@@ -395,9 +425,43 @@ found:
 	return (maybe);
 }
 
+struct protosw *
+pffindprotonotype_locked(int family, int protocol, __unused int type)
+{
+	register struct domain *dp;
+	register struct protosw *pr;
+
+	if (family == 0)
+		return (0);
+	dp = pffinddomain_locked(family);
+	if (dp == NULL) {
+		return (NULL);
+	}
+	for (pr = dp->dom_protosw; pr; pr = pr->pr_next) {
+		if (pr->pr_protocol == protocol) {
+			return (pr);
+		}
+	}
+	return (NULL);
+}
+
+struct protosw *
+pffindprotonotype(int family, int protocol)
+{
+	register struct protosw *pr;
+	if (protocol == 0) {
+		return (NULL);
+	}
+	lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
+	lck_mtx_lock(domain_proto_mtx);
+	pr = pffindprotonotype_locked(family, protocol, 0);
+	lck_mtx_unlock(domain_proto_mtx);
+	return (pr);
+}
+
 int
 net_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-           user_addr_t newp, size_t newlen, struct proc *p)
+           user_addr_t newp, size_t newlen, __unused struct proc *p)
 {
 	register struct domain *dp;
 	register struct protosw *pr;
@@ -434,18 +498,13 @@ found:
 }
 
 void
-pfctlinput(cmd, sa)
-	int cmd;
-	struct sockaddr *sa;
+pfctlinput(int cmd, struct sockaddr *sa)
 {
 	pfctlinput2(cmd, sa, (void*)0);
 }
 
 void
-pfctlinput2(cmd, sa, ctlparam)
-	int cmd;
-	struct sockaddr *sa;
-	void *ctlparam;
+pfctlinput2(int cmd, struct sockaddr *sa, void *ctlparam)
 {
 	struct domain *dp;
 	struct protosw *pr;
@@ -462,8 +521,7 @@ pfctlinput2(cmd, sa, ctlparam)
 }
 
 void
-pfslowtimo(arg)
-	void *arg;
+pfslowtimo(__unused void *arg)
 {
 	register struct domain *dp;
 	register struct protosw *pr;
@@ -478,13 +536,12 @@ pfslowtimo(arg)
 		}
 	do_reclaim = 0;
 	lck_mtx_unlock(domain_proto_mtx);
-	timeout(pfslowtimo, NULL, hz/2);
+	timeout(pfslowtimo, NULL, hz/PR_SLOWHZ);
         
 }
 
 void
-pffasttimo(arg)
-	void *arg;
+pffasttimo(__unused void *arg)
 {
 	register struct domain *dp;
 	register struct protosw *pr;
@@ -495,5 +552,5 @@ pffasttimo(arg)
 			if (pr->pr_fasttimo)
 				(*pr->pr_fasttimo)();
 	lck_mtx_unlock(domain_proto_mtx);
-	timeout(pffasttimo, NULL, hz/5);
+	timeout(pffasttimo, NULL, hz/PR_FASTHZ);
 }
diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c
index aca7fcc51..38fe262e6 100644
--- a/bsd/kern/uipc_mbuf.c
+++ b/bsd/kern/uipc_mbuf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,15 +60,11 @@
  *
  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
  */
-/* HISTORY
- *
- *	10/15/97 Annette DeSchon (deschon@apple.com)
- *		Fixed bug in which all cluster mbufs were broken up 
- *		into regular mbufs: Some clusters are now reserved.
- *		When a cluster is needed, regular mbufs are no longer
- *		used.  (Radar 1683621)
- *	20-May-95 Mac Gillon (mgillon) at NeXT
- *		New version based on 4.4
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #include <sys/param.h>
@@ -74,737 +76,2945 @@
 #include <sys/syslog.h>
 #include <sys/protosw.h>
 #include <sys/domain.h>
+#include <sys/queue.h>
 
-#include <kern/queue.h>
 #include <kern/kern_types.h>
+#include <kern/simple_lock.h>
+#include <kern/queue.h>
 #include <kern/sched_prim.h>
+#include <kern/cpu_number.h>
+
+#include <libkern/OSAtomic.h>
+#include <libkern/libkern.h>
 
 #include <IOKit/IOMapper.h>
 
-extern vm_offset_t kmem_mb_alloc(vm_map_t  , int );
-extern boolean_t PE_parse_boot_arg(const char *, void *);
-
-#define _MCLREF(p)       (++mclrefcnt[mtocl(p)])
-#define _MCLUNREF(p)     (--mclrefcnt[mtocl(p)] == 0)
-#define _M_CLEAR_PKTHDR(mbuf_ptr)	(mbuf_ptr)->m_pkthdr.rcvif = NULL; \
-									(mbuf_ptr)->m_pkthdr.len = 0; \
-									(mbuf_ptr)->m_pkthdr.header = NULL; \
-									(mbuf_ptr)->m_pkthdr.csum_flags = 0; \
-									(mbuf_ptr)->m_pkthdr.csum_data = 0; \
-									(mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
-									(mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
-									(mbuf_ptr)->m_pkthdr.socket_id = 0; \
-									SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
+#include <machine/limits.h>
+#include <machine/machine_routines.h>
 
-/* kernel translater */
-extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
+
+#include <sys/mcache.h>
 
-lck_mtx_t			* mbuf_mlock;
-lck_grp_t			* mbuf_mlock_grp;
-lck_grp_attr_t	* mbuf_mlock_grp_attr;
-lck_attr_t 		* mbuf_mlock_attr;
-extern lck_mtx_t 	*domain_proto_mtx;
+/*
+ * MBUF IMPLEMENTATION NOTES.
+ *
+ * There is a total of 5 per-CPU caches:
+ *
+ * MC_MBUF:
+ *	This is a cache of rudimentary objects of MSIZE in size; each
+ *	object represents an mbuf structure.  This cache preserves only
+ *	the m_type field of the mbuf during its transactions.
+ *
+ * MC_CL:
+ *	This is a cache of rudimentary objects of MCLBYTES in size; each
+ *	object represents a mcluster structure.  This cache does not
+ *	preserve the contents of the objects during its transactions.
+ *
+ * MC_BIGCL:
+ *	This is a cache of rudimentary objects of NBPG in size; each
+ *	object represents a mbigcluster structure.  This cache does not
+ *	preserve the contents of the objects during its transaction.
+ *
+ * MC_MBUF_CL:
+ *	This is a cache of mbufs each having a cluster attached to it.
+ *	It is backed by MC_MBUF and MC_CL rudimentary caches.  Several
+ *	fields of the mbuf related to the external cluster are preserved
+ *	during transactions.
+ *
+ * MC_MBUF_BIGCL:
+ *	This is a cache of mbufs each having a big cluster attached to it.
+ *	It is backed by MC_MBUF and MC_BIGCL rudimentary caches.  Several
+ *	fields of the mbuf related to the external cluster are preserved
+ *	during transactions.
+ *
+ * OBJECT ALLOCATION:
+ *
+ * Allocation requests are handled first at the per-CPU (mcache) layer
+ * before falling back to the slab layer.  Performance is optimal when
+ * the request is satisfied at the CPU layer because global data/lock
+ * never gets accessed.  When the slab layer is entered for allocation,
+ * the slab freelist will be checked first for available objects before
+ * the VM backing store is invoked.  Slab layer operations are serialized
+ * for all of the caches as the mbuf global lock is held most of the time.
+ * Allocation paths are different depending on the class of objects:
+ *
+ * a. Rudimentary object:
+ *
+ *	{ m_get_common(), m_clattach(), m_mclget(),
+ *	  m_mclalloc(), m_bigalloc(), m_copym_with_hdrs(),
+ *	  composite object allocation }
+ *			|	^
+ *			|	|
+ *			|	+-----------------------+
+ *			v				|
+ *	   mcache_alloc/mcache_alloc_ext()	mbuf_slab_audit()
+ *			|				^
+ *			v				|
+ *		   [CPU cache] ------->	(found?) -------+
+ *			|				|
+ *			v				|
+ *		 mbuf_slab_alloc()			|
+ *			|				|
+ *			v				|
+ *	+---------> [freelist] ------->	(found?) -------+
+ *	|		|
+ *	|		v
+ *	|	    m_clalloc()
+ *	|		|
+ *	|		v
+ *	+---<<---- kmem_mb_alloc()
+ *
+ * b. Composite object:
+ *
+ *	{ m_getpackets_internal(), m_allocpacket_internal() }
+ *			|	^
+ *			|	|
+ *			|	+------	(done) ---------+
+ *			v				|
+ *	   mcache_alloc/mcache_alloc_ext()	mbuf_cslab_audit()
+ *			|				^
+ *			v				|
+ *		   [CPU cache] ------->	(found?) -------+
+ *			|				|
+ *			v				|
+ *		 mbuf_cslab_alloc()			|
+ *			|				|
+ *			v				|
+ *		    [freelist] ------->	(found?) -------+
+ *			|				|
+ *			v				|
+ *		(rudimentary object)			|
+ *	   mcache_alloc/mcache_alloc_ext() ------>>-----+
+ *
+ * Auditing notes: If auditing is enabled, buffers will be subjected to
+ * integrity checks by the audit routine.  This is done by verifying their
+ * contents against DEADBEEF (free) pattern before returning them to caller.
+ * As part of this step, the routine will also record the transaction and
+ * pattern-fill the buffers with BADDCAFE (uninitialized) pattern.  It will
+ * also restore any constructed data structure fields if necessary.
+ *
+ * OBJECT DEALLOCATION:
+ *
+ * Freeing an object simply involves placing it into the CPU cache; this
+ * pollutes the cache to benefit subsequent allocations.  The slab layer
+ * will only be entered if the object is to be purged out of the cache.
+ * During normal operations, this happens only when the CPU layer resizes
+ * its bucket while it's adjusting to the allocation load.  Deallocation
+ * paths are different depending on the class of objects:
+ *
+ * a. Rudimentary object:
+ *
+ *	{ m_free(), m_freem_list(), composite object deallocation }
+ *			|	^
+ *			|	|
+ *			|	+------	(done) ---------+
+ *			v				|
+ *	   mcache_free/mcache_free_ext()		|
+ *			|				|
+ *			v				|
+ *		mbuf_slab_audit()			|
+ *			|				|
+ *			v				|
+ *		   [CPU cache] ---> (not purging?) -----+
+ *			|				|
+ *			v				|
+ *		 mbuf_slab_free()			|
+ *			|				|
+ *			v				|
+ *		    [freelist] ----------->>------------+
+ *	 (objects never get purged to VM)
+ *
+ * b. Composite object:
+ *
+ *	{ m_free(), m_freem_list() }
+ *			|	^
+ *			|	|
+ *			|	+------	(done) ---------+
+ *			v				|
+ *	   mcache_free/mcache_free_ext()		|
+ *			|				|
+ *			v				|
+ *		mbuf_cslab_audit()			|
+ *			|				|
+ *			v				|
+ *		   [CPU cache] ---> (not purging?) -----+
+ *			|				|
+ *			v				|
+ *		 mbuf_cslab_free()			|
+ *			|				|
+ *			v				|
+ *		    [freelist] ---> (not purging?) -----+
+ *			|				|
+ *			v				|
+ *		(rudimentary object)			|
+ *	   mcache_free/mcache_free_ext() ------->>------+
+ *
+ * Auditing notes: If auditing is enabled, the audit routine will save
+ * any constructed data structure fields (if necessary) before filling the
+ * contents of the buffers with DEADBEEF (free) pattern and recording the
+ * transaction.  Buffers that are freed (whether at CPU or slab layer) are
+ * expected to contain the free pattern.
+ *
+ * DEBUGGING:
+ *
+ * Debugging can be enabled by adding "mbuf_debug=0x3" to boot-args; this
+ * translates to the mcache flags (MCF_VERIFY | MCF_AUDIT).  Additionally,
+ * the CPU layer cache can be disabled by setting the MCF_NOCPUCACHE flag,
+ * i.e. modify the boot argument parameter to "mbuf_debug=0x13".  Note
+ * that debugging consumes more CPU and memory.
+ *
+ * Each object is associated with exactly one mcache_audit_t structure that
+ * contains the information related to its last buffer transaction.  Given
+ * an address of an object, the audit structure can be retrieved by finding
+ * the position of the object relevant to the base address of the cluster:
+ *
+ *	+------------+			+=============+
+ *	| mbuf addr  |			| mclaudit[i] |
+ *	+------------+			+=============+
+ *	      |				| cl_audit[0] |
+ *	i = MTOCL(addr)			+-------------+
+ *	      |			+----->	| cl_audit[1] | -----> mcache_audit_t
+ *	b = CLTOM(i)		|	+-------------+
+ *	      |			|	|     ...     |
+ *	x = MCLIDX(b, addr)	|	+-------------+
+ *	      |			|	| cl_audit[7] |
+ *	      +-----------------+	+-------------+
+ *		 (e.g. x == 1)
+ *
+ * The mclaudit[] array is allocated at initialization time, but its contents
+ * get populated when the corresponding cluster is created.  Because a cluster
+ * can be turned into NMBPCL number of mbufs, we preserve enough space for the
+ * mbufs so that there is a 1-to-1 mapping between them.  A cluster that never
+ * gets (or has not yet) turned into mbufs will use only cl_audit[0] with the
+ * remaining entries unused.  For big clusters, only one entry is allocated
+ * and used for the entire cluster pair.
+ */
 
-struct mbuf 	*mfree;		/* mbuf free list */
-struct	mbuf *mfreelater;	/* mbuf deallocation list */
+/* TODO: should be in header file */
+/* kernel translater */
+extern vm_offset_t kmem_mb_alloc(vm_map_t, int);
+extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 extern vm_map_t mb_map;		/* special map */
-int		m_want;		/* sleepers on mbufs */
-short		*mclrefcnt; 	/* mapped cluster reference counts */
-int             *mcl_paddr;
+
+/* Global lock */
+static lck_mtx_t *mbuf_mlock;
+static lck_attr_t *mbuf_mlock_attr;
+static lck_grp_t *mbuf_mlock_grp;
+static lck_grp_attr_t *mbuf_mlock_grp_attr;
+
+/* Back-end (common) layer */
+static void *mbuf_worker_run;	/* wait channel for worker thread */
+static int mbuf_worker_ready;	/* worker thread is runnable */
+static int mbuf_expand_mcl;	/* number of cluster creation requets */
+static int mbuf_expand_big;	/* number of big cluster creation requests */
+static int mbuf_expand_16k;	/* number of 16K cluster creation requests */
+static int ncpu;		/* number of CPUs */
+static int *mcl_paddr;		/* Array of cluster physical addresses */
 static ppnum_t mcl_paddr_base;	/* Handle returned by IOMapper::iovmAlloc() */
-union mcluster 	*mclfree;	/* mapped cluster free list */
-union mbigcluster 	*mbigfree;	/* mapped cluster free list */
-int		max_linkhdr;	/* largest link-level header */
-int		max_protohdr;	/* largest protocol header */
-int		max_hdr;	/* largest link+protocol header */
-int		max_datalen;	/* MHLEN - max_hdr */
-struct mbstat 	mbstat;		/* statistics */
-union mcluster 	*mbutl;		/* first mapped cluster address */
-union mcluster  *embutl;	/* ending virtual address of mclusters */
-
-static int 	nclpp;		/* # clusters per physical page */
-
-static int m_howmany(int, size_t );
-void m_reclaim(void);
-static int m_clalloc(const int , const int, const size_t, int);
-int do_reclaim = 0;
+static mcache_t *ref_cache;	/* Cache of cluster reference & flags */
+static mcache_t *mcl_audit_con_cache; /* Audit contents cache */
+static unsigned int mbuf_debug;	/* patchable mbuf mcache flags */
+static unsigned int mb_normalized; /* number of packets "normalized" */
+
+typedef enum {
+	MC_MBUF = 0,	/* Regular mbuf */
+	MC_CL,		/* Cluster */
+	MC_BIGCL,	/* Large (4K) cluster */
+	MC_16KCL,	/* Jumbo (16K) cluster */
+	MC_MBUF_CL,	/* mbuf + cluster */
+	MC_MBUF_BIGCL,	/* mbuf + large (4K) cluster */
+	MC_MBUF_16KCL	/* mbuf + jumbo (16K) cluster */
+} mbuf_class_t;
+
+#define	MBUF_CLASS_MIN		MC_MBUF
+#define	MBUF_CLASS_MAX		MC_MBUF_16KCL
+#define	MBUF_CLASS_LAST		MC_16KCL
+#define	MBUF_CLASS_VALID(c) \
+	((int)(c) >= MBUF_CLASS_MIN && (int)(c) <= MBUF_CLASS_MAX)
+#define	MBUF_CLASS_COMPOSITE(c) \
+	((int)(c) > MBUF_CLASS_LAST)
 
-#define MF_NOWAIT	0x1
-#define MF_BIG		0x2
-
-/* The number of cluster mbufs that are allocated, to start. */
-#define MINCL	max(16, 2)
-
-static int mbuf_expand_thread_wakeup = 0;
-static int mbuf_expand_mcl = 0;
-static int mbuf_expand_big = 0;
-static int mbuf_expand_thread_initialized = 0;
-
-static void mbuf_expand_thread_init(void);
-static void mbuf_expand_thread(void);
-static int m_expand(int );
-static caddr_t m_bigalloc(int );
-static void m_bigfree(caddr_t ,  u_int ,  caddr_t );
-__private_extern__ struct mbuf * m_mbigget(struct mbuf *, int );
-void mbinit(void);
-static void m_range_check(void *addr);
-
-
-#if 0
-static int mfree_munge = 0;
-#if 0
-#define _MFREE_MUNGE(m) {                                               \
-    if (mfree_munge)                                                    \
-        {   int i;                                                      \
-            vm_offset_t *element = (vm_offset_t *)(m);                  \
-            for (i = 0;                                                 \
-                 i < sizeof(struct mbuf)/sizeof(vm_offset_t);           \
-                 i++)                                                   \
-                    (element)[i] = 0xdeadbeef;                          \
-        }                                                               \
-}
-#else
-void
-munge_mbuf(struct mbuf *m)
-{
-    int i;
-    vm_offset_t *element = (vm_offset_t *)(m);
-    for (i = 0;
-            i < sizeof(struct mbuf)/sizeof(vm_offset_t);
-            i++)
-            (element)[i] = 0xdeadbeef;
-}
-#define _MFREE_MUNGE(m) {  \
-    if (mfree_munge)       \
-        munge_mbuf(m);     \
-}
-#endif
-#else
-#define _MFREE_MUNGE(m)
-#endif
 
+/*
+ * mbuf specific mcache allocation request flags.
+ */
+#define	MCR_COMP	MCR_USR1 /* for MC_MBUF_{CL,BIGCL,16KCL} caches */
 
-#define _MINTGET(m, type) { 						\
-	MBUF_LOCK();							\
-	if (((m) = mfree) != 0) {					\
-		MCHECK(m);                                              \
-		++mclrefcnt[mtocl(m)]; 					\
-		mbstat.m_mtypes[MT_FREE]--;				\
-		mbstat.m_mtypes[(type)]++;				\
-		mfree = (m)->m_next;					\
-	}								\
-	MBUF_UNLOCK();							\
-}
-	
+/*
+ * Per-cluster slab structure.
+ *
+ * A slab is a cluster control structure that contains one or more object
+ * chunks; the available chunks are chained in the slab's freelist (sl_head).
+ * Each time a chunk is taken out of the slab, the slab's reference count
+ * gets incremented.  When all chunks have been taken out, the empty slab
+ * gets removed (SLF_DETACHED) from the class's slab list.  A chunk that is
+ * returned to a slab causes the slab's reference count to be decremented;
+ * it also causes the slab to be reinserted back to class's slab list, if
+ * it's not already done.
+ *
+ * Compartmentalizing of the object chunks into slabs allows us to easily
+ * merge one or more slabs together when the adjacent slabs are idle, as
+ * well as to convert or move a slab from one class to another; e.g. the
+ * mbuf cluster slab can be converted to a regular cluster slab when all
+ * mbufs in the slab have been freed.
+ *
+ * A slab may also span across multiple clusters for chunks larger than
+ * a cluster's size.  In this case, only the slab of the first cluster is
+ * used.  The rest of the slabs are marked with SLF_PARTIAL to indicate
+ * that they are part of the larger slab.
+ */
+typedef struct mcl_slab {
+	struct mcl_slab	*sl_next;	/* neighboring slab */
+	u_int8_t	sl_class;	/* controlling mbuf class */
+	int8_t		sl_refcnt;	/* outstanding allocations */
+	int8_t		sl_chunks;	/* chunks (bufs) in this slab */
+	u_int16_t	sl_flags;	/* slab flags (see below) */
+	u_int16_t	sl_len;		/* slab length */
+	void		*sl_base;	/* base of allocated memory */
+	void		*sl_head;	/* first free buffer */
+	TAILQ_ENTRY(mcl_slab) sl_link;	/* next/prev slab on freelist */
+} mcl_slab_t;
+
+#define	SLF_MAPPED	0x0001		/* backed by a mapped page */
+#define	SLF_PARTIAL	0x0002		/* part of another slab */
+#define	SLF_DETACHED	0x0004		/* not in slab freelist */
 
-static void
-m_range_check(void *addr)
-{
-	if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
-		panic("mbuf address out of range 0x%x", addr);
-}
+/*
+ * The array of slabs are broken into groups of arrays per 1MB of kernel
+ * memory to reduce the footprint.  Each group is allocated on demand
+ * whenever a new piece of memory mapped in from the VM crosses the 1MB
+ * boundary.
+ */
+#define	MBSHIFT		20				/* 1MB */
+#define	NSLABSPMB	((1 << MBSHIFT) >> MCLSHIFT)	/* 512 slabs/grp */
 
-__private_extern__ void
-mbinit(void)
-{
-	int m;
-	int initmcl = 32;
-	int mcl_pages;
+typedef struct mcl_slabg {
+	mcl_slab_t	slg_slab[NSLABSPMB];	/* group of slabs */
+} mcl_slabg_t;
 
-	if (nclpp)
-		return;
-	nclpp = round_page_32(MCLBYTES) / MCLBYTES;	/* see mbufgc() */
-	if (nclpp < 1) nclpp = 1;
-	mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
+/*
+ * Per-cluster audit structure.
+ */
+typedef struct {
+	mcache_audit_t	*cl_audit[NMBPCL];	/* array of audits */
+} mcl_audit_t;
 
-	mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
-	mbuf_mlock_attr = lck_attr_alloc_init();
+#if CONFIG_MBUF_NOEXPAND
+static unsigned int maxmbufcl;
+#endif /* CONFIG_MBUF_NOEXPAND */
 
-	mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
+/*
+ * Size of data from the beginning of an mbuf that covers m_hdr, pkthdr
+ * and m_ext structures.  If auditing is enabled, we allocate a shadow
+ * mbuf structure of this size inside each audit structure, and the
+ * contents of the real mbuf gets copied into it when the mbuf is freed.
+ * This allows us to pattern-fill the mbuf for integrity check, and to
+ * preserve any constructed mbuf fields (e.g. mbuf + cluster cache case).
+ * Note that we don't save the contents of clusters when they are freed;
+ * we simply pattern-fill them.
+ */
+#if defined(__LP64__)
+#define	AUDIT_CONTENTS_SIZE	160
+#else
+#define	AUDIT_CONTENTS_SIZE	80
+#endif /* __LP64__ */
 
-	mbstat.m_msize = MSIZE;
-	mbstat.m_mclbytes = MCLBYTES;
-	mbstat.m_minclsize = MINCLSIZE;
-	mbstat.m_mlen = MLEN;
-	mbstat.m_mhlen = MHLEN;
-	mbstat.m_bigmclbytes = NBPG;
+/*
+ * mbuf specific mcache audit flags
+ */
+#define	MB_INUSE	0x01	/* object has not been returned to slab */
+#define	MB_COMP_INUSE	0x02	/* object has not been returned to cslab */
+#define	MB_SCVALID	0x04	/* object has valid saved contents */
 
-	if (nmbclusters == 0)
-		nmbclusters = NMBCLUSTERS;
-	MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
-					M_TEMP, M_WAITOK);
-	if (mclrefcnt == 0)
-		panic("mbinit");
-	for (m = 0; m < nmbclusters; m++)
-		mclrefcnt[m] = -1;
+/*
+ * Each of the following two arrays hold up to nmbclusters elements.
+ */
+static mcl_audit_t *mclaudit;	/* array of cluster audit information */
+static mcl_slabg_t **slabstbl;	/* cluster slabs table */
+static unsigned int maxslabgrp;	/* max # of entries in slabs table */
+static unsigned int slabgrp;	/* # of entries in slabs table */
+
+/* Globals */
+int nclusters;			/* # of clusters for non-jumbo (legacy) sizes */
+int njcl;			/* # of clusters for jumbo sizes */
+int njclbytes;			/* size of a jumbo cluster */
+union mcluster *mbutl;		/* first mapped cluster address */
+union mcluster *embutl;		/* ending virtual address of mclusters */
+int max_linkhdr;		/* largest link-level header */
+int max_protohdr;		/* largest protocol header */
+int max_hdr;			/* largest link+protocol header */
+int max_datalen;		/* MHLEN - max_hdr */
+
+/* TODO: should be in header file */
+int do_reclaim = 0;
 
-	/* Calculate the number of pages assigned to the cluster pool */
-	mcl_pages = nmbclusters/(NBPG/CLBYTES);
-	MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
-	if (mcl_paddr == 0)
-		panic("mbinit1");
-	/* Register with the I/O Bus mapper */
-	mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
-	bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
+/* The minimum number of objects that are allocated, to start. */
+#define	MINCL		32
+#define	MINBIGCL	(MINCL >> 1)
+#define	MIN16KCL	(MINCL >> 2)
+
+/* Low watermarks (only map in pages once free counts go below) */
+#define	MCL_LOWAT	MINCL
+#define	MBIGCL_LOWAT	MINBIGCL
+#define	M16KCL_LOWAT	MIN16KCL
+
+typedef struct {
+	mbuf_class_t	mtbl_class;	/* class type */
+	mcache_t	*mtbl_cache;	/* mcache for this buffer class */
+	TAILQ_HEAD(mcl_slhead, mcl_slab) mtbl_slablist; /* slab list */
+	mcache_obj_t	*mtbl_cobjlist;	/* composite objects freelist */
+	mb_class_stat_t	*mtbl_stats;	/* statistics fetchable via sysctl */
+	u_int32_t	mtbl_maxsize;	/* maximum buffer size */
+	int		mtbl_minlimit;	/* minimum allowed */
+	int		mtbl_maxlimit;	/* maximum allowed */
+	u_int32_t	mtbl_wantpurge;	/* purge during next reclaim */
+} mbuf_table_t;
+
+#define	m_class(c)	mbuf_table[c].mtbl_class
+#define	m_cache(c)	mbuf_table[c].mtbl_cache
+#define	m_slablist(c)	mbuf_table[c].mtbl_slablist
+#define	m_cobjlist(c)	mbuf_table[c].mtbl_cobjlist
+#define	m_maxsize(c)	mbuf_table[c].mtbl_maxsize
+#define	m_minlimit(c)	mbuf_table[c].mtbl_minlimit
+#define	m_maxlimit(c)	mbuf_table[c].mtbl_maxlimit
+#define	m_wantpurge(c)	mbuf_table[c].mtbl_wantpurge
+#define	m_cname(c)	mbuf_table[c].mtbl_stats->mbcl_cname
+#define	m_size(c)	mbuf_table[c].mtbl_stats->mbcl_size
+#define	m_total(c)	mbuf_table[c].mtbl_stats->mbcl_total
+#define	m_active(c)	mbuf_table[c].mtbl_stats->mbcl_active
+#define	m_infree(c)	mbuf_table[c].mtbl_stats->mbcl_infree
+#define	m_slab_cnt(c)	mbuf_table[c].mtbl_stats->mbcl_slab_cnt
+#define	m_alloc_cnt(c)	mbuf_table[c].mtbl_stats->mbcl_alloc_cnt
+#define	m_free_cnt(c)	mbuf_table[c].mtbl_stats->mbcl_free_cnt
+#define	m_notified(c)	mbuf_table[c].mtbl_stats->mbcl_notified
+#define	m_purge_cnt(c)	mbuf_table[c].mtbl_stats->mbcl_purge_cnt
+#define	m_fail_cnt(c)	mbuf_table[c].mtbl_stats->mbcl_fail_cnt
+#define	m_ctotal(c)	mbuf_table[c].mtbl_stats->mbcl_ctotal
+
+static mbuf_table_t mbuf_table[] = {
+	/*
+	 * The caches for mbufs, regular clusters and big clusters.
+	 */
+	{ MC_MBUF, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_MBUF)),
+	    NULL, NULL, 0, 0, 0, 0 },
+	{ MC_CL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_CL)),
+	    NULL, NULL, 0, 0, 0, 0 },
+	{ MC_BIGCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_BIGCL)),
+	    NULL, NULL, 0, 0, 0, 0 },
+	{ MC_16KCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_16KCL)),
+	    NULL, NULL, 0, 0, 0, 0 },
+	/*
+	 * The following are special caches; they serve as intermediate
+	 * caches backed by the above rudimentary caches.  Each object
+	 * in the cache is an mbuf with a cluster attached to it.  Unlike
+	 * the above caches, these intermediate caches do not directly
+	 * deal with the slab structures; instead, the constructed
+	 * cached elements are simply stored in the freelists.
+	 */
+	{ MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0 },
+	{ MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0 },
+	{ MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0 },
+};
+
+#define	NELEM(a)	(sizeof (a) / sizeof ((a)[0]))
+
+static void *mb_waitchan = &mbuf_table;	/* wait channel for all caches */
+static int mb_waiters;			/* number of sleepers */
+
+/* The following are used to serialize m_clalloc() */
+static boolean_t mb_clalloc_busy;
+static void *mb_clalloc_waitchan = &mb_clalloc_busy;
+static int mb_clalloc_waiters;
+
+static int mbstat_sysctl SYSCTL_HANDLER_ARGS;
+static int mb_stat_sysctl SYSCTL_HANDLER_ARGS;
+static void mbuf_table_init(void);
+static inline void m_incref(struct mbuf *);
+static inline u_int32_t m_decref(struct mbuf *);
+static int m_clalloc(const u_int32_t, const int, const u_int32_t);
+static void mbuf_worker_thread_init(void);
+static mcache_obj_t *slab_alloc(mbuf_class_t, int);
+static void slab_free(mbuf_class_t, mcache_obj_t *);
+static unsigned int mbuf_slab_alloc(void *, mcache_obj_t ***,
+    unsigned int, int);
+static void mbuf_slab_free(void *, mcache_obj_t *, int);
+static void mbuf_slab_audit(void *, mcache_obj_t *, boolean_t);
+static void mbuf_slab_notify(void *, u_int32_t);
+static unsigned int cslab_alloc(mbuf_class_t, mcache_obj_t ***,
+    unsigned int);
+static unsigned int cslab_free(mbuf_class_t, mcache_obj_t *, int);
+static unsigned int mbuf_cslab_alloc(void *, mcache_obj_t ***,
+    unsigned int, int);
+static void mbuf_cslab_free(void *, mcache_obj_t *, int);
+static void mbuf_cslab_audit(void *, mcache_obj_t *, boolean_t);
+static int freelist_populate(mbuf_class_t, unsigned int, int);
+static boolean_t mbuf_cached_above(mbuf_class_t, int);
+static boolean_t mbuf_steal(mbuf_class_t, unsigned int);
+static void m_reclaim(mbuf_class_t, unsigned int, boolean_t);
+static int m_howmany(int, size_t);
+static void mbuf_worker_thread(void);
+static boolean_t mbuf_sleep(mbuf_class_t, unsigned int, int);
+
+static void mcl_audit_init(void *, mcache_audit_t **, mcache_obj_t **,
+    size_t, unsigned int);
+static mcache_audit_t *mcl_audit_buf2mca(mbuf_class_t, mcache_obj_t *);
+static void mcl_audit_mbuf(mcache_audit_t *, void *, boolean_t, boolean_t);
+static void mcl_audit_cluster(mcache_audit_t *, void *, size_t, boolean_t,
+    boolean_t);
+static void mcl_audit_restore_mbuf(struct mbuf *, mcache_audit_t *, boolean_t);
+static void mcl_audit_save_mbuf(struct mbuf *, mcache_audit_t *);
+static void mcl_audit_mcheck_panic(struct mbuf *);
+static void mcl_audit_verify_nextptr(void *, mcache_audit_t *);
+
+static mcl_slab_t *slab_get(void *);
+static void slab_init(mcl_slab_t *, mbuf_class_t, u_int32_t,
+    void *, void *, unsigned int, int, int);
+static void slab_insert(mcl_slab_t *, mbuf_class_t);
+static void slab_remove(mcl_slab_t *, mbuf_class_t);
+static boolean_t slab_inrange(mcl_slab_t *, void *);
+static void slab_nextptr_panic(mcl_slab_t *, void *);
+static void slab_detach(mcl_slab_t *);
+static boolean_t slab_is_detached(mcl_slab_t *);
+
+/*
+ * This flag is set for all mbufs that come out of and into the composite
+ * mbuf + cluster caches, i.e. MC_MBUF_CL and MC_MBUF_BIGCL.  mbufs that
+ * are marked with such a flag have clusters attached to them, and will be
+ * treated differently when they are freed; instead of being placed back
+ * into the mbuf and cluster freelists, the composite mbuf + cluster objects
+ * are placed back into the appropriate composite cache's freelist, and the
+ * actual freeing is deferred until the composite objects are purged.  At
+ * such a time, this flag will be cleared from the mbufs and the objects
+ * will be freed into their own separate freelists.
+ */
+#define	EXTF_COMPOSITE	0x1
 
-	embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
+#define	MEXT_RFA(m)		((m)->m_ext.ext_refflags)
+#define	MEXT_REF(m)		(MEXT_RFA(m)->refcnt)
+#define	MEXT_FLAGS(m)		(MEXT_RFA(m)->flags)
+#define	MBUF_IS_COMPOSITE(m)	\
+	(MEXT_REF(m) == 0 && (MEXT_FLAGS(m) & EXTF_COMPOSITE))
 
-	PE_parse_boot_arg("initmcl", &initmcl);
-	
-	if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
-		goto bad;
-	MBUF_UNLOCK();
+/*
+ * Macros used to verify the integrity of the mbuf.
+ */
+#define	_MCHECK(m) {							\
+	if ((m)->m_type != MT_FREE) {					\
+		if (mclaudit == NULL)					\
+			panic("MCHECK: m_type=%d m=%p",			\
+			    (u_int16_t)(m)->m_type, m);			\
+		else							\
+			mcl_audit_mcheck_panic(m);			\
+	}								\
+}
 
-    (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
+#define	MBUF_IN_MAP(addr)						\
+	((void *)(addr) >= (void *)mbutl && (void *)(addr) < (void *)embutl)
 
-	return;
-bad:
-		panic("mbinit");
+#define	MRANGE(addr) {							\
+	if (!MBUF_IN_MAP(addr))						\
+		panic("MRANGE: address out of range 0x%p", addr);	\
 }
 
 /*
- * Allocate some number of mbuf clusters
- * and place on cluster free list.
- * Take the mbuf lock (if not already locked) and do not release it
+ * Macro version of mtod.
  */
-/* ARGSUSED */
-static int
-m_clalloc(
-	const int num,
-	const int nowait,
-	const size_t bufsize,
-	int locked)
-{
-	int i;
-	vm_size_t size = 0;
-	int numpages = 0;
-	vm_offset_t page = 0;
-
-	if (locked == 0)
-		MBUF_LOCK();
-	/*
-	 * Honor the caller's wish to block or not block.
-	 * We have a way to grow the pool asynchronously,
-	 * by kicking the dlil_input_thread.
-	 */
-	i = m_howmany(num, bufsize);
-	if (i == 0 || nowait == M_DONTWAIT)
-		goto out;
+#define	MTOD(m, t)	((t)((m)->m_data))
 
-	MBUF_UNLOCK();	
-	size = round_page_32(i * bufsize);
-	page = kmem_mb_alloc(mb_map, size);
+/*
+ * Macros to obtain cluster index and base cluster address.
+ */
+#define	MTOCL(x)	(((char *)(x) - (char *)mbutl) >> MCLSHIFT)
+#define	CLTOM(x)	((union mcluster *)(mbutl + (x)))
 
-	if (page == 0) {
-		size = NBPG; /* Try for 1 if failed */
-		page = kmem_mb_alloc(mb_map, size);
-	}
-	MBUF_LOCK();
-
-	if (page) {
-		numpages = size / NBPG;
-		for (i = 0; i < numpages; i++, page += NBPG) {
-			if (((int)page & PGOFSET) == 0) {
-				ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
-				ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
-				
-				/*
-				 * In the case of no mapper being available
-				 * the following code nops and returns the
-				 * input page, if there is a mapper the I/O
-				 * page appropriate is returned.
-				 */
-				new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
-				mcl_paddr[offset] = new_page << 12;
-			}
-			if (bufsize == MCLBYTES) {
-				union mcluster *mcl = (union mcluster *)page;
-
-				if (++mclrefcnt[mtocl(mcl)] != 0)
-					panic("m_clalloc already there");
-				mcl->mcl_next = mclfree;
-				mclfree = mcl++;
-				if (++mclrefcnt[mtocl(mcl)] != 0)
-					panic("m_clalloc already there");
-				mcl->mcl_next = mclfree;
-				mclfree = mcl++;
-			} else {
-				union mbigcluster *mbc = (union mbigcluster *)page;
+/*
+ * Macro to find the mbuf index relative to the cluster base.
+ */
+#define	MCLIDX(c, m)	(((char *)(m) - (char *)(c)) >> 8)
 
-				if (++mclrefcnt[mtocl(mbc)] != 0)
-					panic("m_clalloc already there");
-				if (++mclrefcnt[mtocl(mbc) + 1] != 0)
-					panic("m_clalloc already there");
+/*
+ * Macros used during mbuf and cluster initialization.
+ */
+#define	MBUF_INIT(m, pkthdr, type) {					\
+	_MCHECK(m);							\
+	(m)->m_next = (m)->m_nextpkt = NULL;				\
+	(m)->m_len = 0;							\
+	(m)->m_type = type;						\
+	if ((pkthdr) == 0) {						\
+		(m)->m_data = (m)->m_dat;				\
+		(m)->m_flags = 0;					\
+	} else {							\
+		(m)->m_data = (m)->m_pktdat;				\
+		(m)->m_flags = M_PKTHDR;				\
+		(m)->m_pkthdr.rcvif = NULL;				\
+		(m)->m_pkthdr.len = 0;					\
+		(m)->m_pkthdr.header = NULL;				\
+		(m)->m_pkthdr.csum_flags = 0;				\
+		(m)->m_pkthdr.csum_data = 0;				\
+		(m)->m_pkthdr.reserved0 = NULL;				\
+		(m)->m_pkthdr.vlan_tag = 0;				\
+		(m)->m_pkthdr.socket_id = 0;				\
+		m_tag_init(m);						\
+	}								\
+}
 
-				mbc->mbc_next = mbigfree;
-				mbigfree = mbc;
-			}
-		}
-		if (bufsize == MCLBYTES) {
-			int numcl = numpages << 1;
-			mbstat.m_clfree += numcl;
-			mbstat.m_clusters += numcl;
-			return (numcl);
-		} else {
-			mbstat.m_bigclfree += numpages;
-			mbstat.m_bigclusters += numpages;
-			return (numpages);
-		}
-	} /* else ... */
-out:
-	/* 
-	 * When non-blocking we kick a thread if we havve to grow the 
-	 * pool or if the number of free clusters is less than requested.
-	 */
-	if (bufsize == MCLBYTES) {
-		if (i > 0) {
-			/* Remember total number of clusters needed at this time */
-			i += mbstat.m_clusters;
-			if (i > mbuf_expand_mcl) {
-				mbuf_expand_mcl = i;
-				if (mbuf_expand_thread_initialized)
-					wakeup((caddr_t)&mbuf_expand_thread_wakeup);
-			}
-		}
-	
-		if (mbstat.m_clfree >= num) 
-			 return 1;
-	} else {
-		if (i > 0) {
-			/* Remember total number of 4KB clusters needed at this time */
-			i += mbstat.m_bigclusters;
-			if (i > mbuf_expand_big) {
-				mbuf_expand_big = i; 
-				if (mbuf_expand_thread_initialized)
-					wakeup((caddr_t)&mbuf_expand_thread_wakeup);
-			}
-		}
-	
-		if (mbstat.m_bigclfree >= num) 
-			 return 1;
-	}
-	return 0;
+#define	MEXT_INIT(m, buf, size, free, arg, rfa, ref, flag) {		\
+	(m)->m_data = (m)->m_ext.ext_buf = (buf);			\
+	(m)->m_flags |= M_EXT;						\
+	(m)->m_ext.ext_size = (size);					\
+	(m)->m_ext.ext_free = (free);					\
+	(m)->m_ext.ext_arg = (arg);					\
+	(m)->m_ext.ext_refs.forward = (m)->m_ext.ext_refs.backward =	\
+	    &(m)->m_ext.ext_refs;					\
+	MEXT_RFA(m) = (rfa);						\
+	MEXT_REF(m) = (ref);						\
+	MEXT_FLAGS(m) = (flag);						\
 }
 
+#define	MBUF_CL_INIT(m, buf, rfa, ref, flag)	\
+	MEXT_INIT(m, buf, m_maxsize(MC_CL), NULL, NULL, rfa, ref, flag)
+
+#define	MBUF_BIGCL_INIT(m, buf, rfa, ref, flag)	\
+	MEXT_INIT(m, buf, m_maxsize(MC_BIGCL), m_bigfree, NULL, rfa, ref, flag)
+
+#define	MBUF_16KCL_INIT(m, buf, rfa, ref, flag)	\
+	MEXT_INIT(m, buf, m_maxsize(MC_16KCL), m_16kfree, NULL, rfa, ref, flag)
+
 /*
- * Add more free mbufs by cutting up a cluster.
+ * Macro to convert BSD malloc sleep flag to mcache's
  */
-static int
-m_expand(int canwait)
-{
-	caddr_t mcl;
+#define	MSLEEPF(f)	((!((f) & M_DONTWAIT)) ? MCR_SLEEP : MCR_NOSLEEP)
 
-	if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
-		/* 
-		 * 1/16th of the total number of cluster mbufs allocated is
-		 * reserved for large packets.  The number reserved must
-		 * always be < 1/2, or future allocation will be prevented.
-		 */
-		(void)m_clalloc(1, canwait, MCLBYTES, 0);
-		MBUF_UNLOCK();
-		if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
-			return 0;
-	}
+/*
+ * The structure that holds all mbuf class statistics exportable via sysctl.
+ * Similar to mbstat structure, the mb_stat structure is protected by the
+ * global mbuf lock.  It contains additional information about the classes
+ * that allows for a more accurate view of the state of the allocator.
+ */
+struct mb_stat *mb_stat;
 
-	MCLALLOC(mcl, canwait);
-	if (mcl) {
-		struct mbuf *m = (struct mbuf *)mcl;
-		int i = NMBPCL;
-		MBUF_LOCK();
-		mbstat.m_mtypes[MT_FREE] += i;
-		mbstat.m_mbufs += i;
-		while (i--) {
-			_MFREE_MUNGE(m);
-			m->m_type = MT_FREE;
-			m->m_next = mfree;
-			mfree = m++;
-		}
-		i = m_want;
-		m_want = 0;
-		MBUF_UNLOCK();
-		if (i) wakeup((caddr_t)&mfree);
-		return 1;
-	}
-	return 0;
-}
+#define	MB_STAT_SIZE(n) \
+	((size_t)(&((mb_stat_t *)0)->mbs_class[n]))
 
 /*
- * When MGET failes, ask protocols to free space when short of memory,
- * then re-attempt to allocate an mbuf.
+ * The legacy structure holding all of the mbuf allocation statistics.
+ * The actual statistics used by the kernel are stored in the mbuf_table
+ * instead, and are updated atomically while the global mbuf lock is held.
+ * They are mirrored in mbstat to support legacy applications (e.g. netstat).
+ * Unlike before, the kernel no longer relies on the contents of mbstat for
+ * its operations (e.g. cluster expansion) because the structure is exposed
+ * to outside and could possibly be modified, therefore making it unsafe.
+ * With the exception of the mbstat.m_mtypes array (see below), all of the
+ * statistics are updated as they change.
  */
-struct mbuf *
-m_retry(
-	int canwait, 
-	int type)
-{
-	struct mbuf *m;
-	int wait;
+struct mbstat mbstat;
 
-	for (;;) {
-		(void) m_expand(canwait);
-        	_MINTGET(m, type);
-        	if (m) {
-                	(m)->m_next = (m)->m_nextpkt = 0;
-                	(m)->m_type = (type);
-                	(m)->m_data = (m)->m_dat;
-                	(m)->m_flags = 0;
-                	(m)->m_len = 0;
-        	}
-		if (m || canwait == M_DONTWAIT)
-			break;
-		MBUF_LOCK();
-		wait = m_want++;
-		mbuf_expand_mcl++;
-		if (wait == 0)
-			mbstat.m_drain++;
-		else
-			mbstat.m_wait++;
-		MBUF_UNLOCK();
-        
-		if (mbuf_expand_thread_initialized)
-			wakeup((caddr_t)&mbuf_expand_thread_wakeup);
-
-		if (wait == 0) {
-			m_reclaim();
-		} else {
-			struct timespec ts;
-			ts.tv_sec = 1;
-			ts.tv_nsec = 0;
-			(void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
-		}
-	}
-	if (m == 0)
-		mbstat.m_drops++;
-	return (m);
-}
+#define	MBSTAT_MTYPES_MAX \
+	(sizeof (mbstat.m_mtypes) / sizeof (mbstat.m_mtypes[0]))
 
 /*
- * As above; retry an MGETHDR.
+ * Allocation statistics related to mbuf types (up to MT_MAX-1) are updated
+ * atomically and stored in a per-CPU structure which is lock-free; this is
+ * done in order to avoid writing to the global mbstat data structure which
+ * would cause false sharing.  During sysctl request for kern.ipc.mbstat,
+ * the statistics across all CPUs will be converged into the mbstat.m_mtypes
+ * array and returned to the application.  Any updates for types greater or
+ * equal than MT_MAX would be done atomically to the mbstat; this slows down
+ * performance but is okay since the kernel uses only up to MT_MAX-1 while
+ * anything beyond that (up to type 255) is considered a corner case.
  */
-struct mbuf *
-m_retryhdr(
-	int canwait, 
-	int type)
-{
-	struct mbuf *m;
+typedef struct {
+	unsigned int	cpu_mtypes[MT_MAX];
+} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mtypes_cpu_t;
 
-	if ((m = m_retry(canwait, type))) {
-		m->m_next = m->m_nextpkt = 0;
-		m->m_flags |= M_PKTHDR;
-		m->m_data = m->m_pktdat;
-		_M_CLEAR_PKTHDR(m);
-	}
-	return (m);
-}
+typedef struct {
+	mtypes_cpu_t	mbs_cpu[1];
+} mbuf_mtypes_t;
 
-void
-m_reclaim(void)
-{
-	do_reclaim = 1;	/* drain is performed in pfslowtimo(), to avoid deadlocks */
-	mbstat.m_drain++;
+static mbuf_mtypes_t *mbuf_mtypes;	/* per-CPU statistics */
+
+#define	MBUF_MTYPES_SIZE(n) \
+	((size_t)(&((mbuf_mtypes_t *)0)->mbs_cpu[n]))
+
+#define	MTYPES_CPU(p) \
+	((mtypes_cpu_t *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number())))
+
+/* This should be in a header file */
+#define	atomic_add_32(a, n)	((void) OSAddAtomic(n, (volatile SInt32 *)a))
+
+#define	mtype_stat_add(type, n) {					\
+	if ((unsigned)(type) < MT_MAX) {				\
+		mtypes_cpu_t *mbs = MTYPES_CPU(mbuf_mtypes);		\
+		atomic_add_32(&mbs->cpu_mtypes[type], n);		\
+	} else if ((unsigned)(type) < MBSTAT_MTYPES_MAX) {		\
+		atomic_add_32(&mbstat.m_mtypes[type], n);		\
+	}								\
 }
 
-/*
- * Space allocation routines.
- * These are also available as macros
- * for critical paths.
- */
-struct mbuf *
-m_get(
-	int nowait, 
-	int type)
-{
-	struct mbuf *m;
+#define	mtype_stat_sub(t, n)	mtype_stat_add(t, -(n))
+#define	mtype_stat_inc(t)	mtype_stat_add(t, 1)
+#define	mtype_stat_dec(t)	mtype_stat_sub(t, 1)
 
-	m_range_check(mfree);
-	m_range_check(mclfree);
-	m_range_check(mbigfree);
+static int
+mbstat_sysctl SYSCTL_HANDLER_ARGS
+{
+#pragma unused(oidp, arg1, arg2)
+	int m, n;
+	mtypes_cpu_t mtc;
 
-	_MINTGET(m, type);
-	if (m) {
-		m->m_next = m->m_nextpkt = 0;
-		m->m_type = type;
-		m->m_data = m->m_dat;
-		m->m_flags = 0;
-		m->m_len = 0;
-	} else
-		(m) = m_retry(nowait, type);
+	bzero(&mtc, sizeof (mtc));
+	for (m = 0; m < ncpu; m++) {
+		mtypes_cpu_t *scp = &mbuf_mtypes->mbs_cpu[m];
+		mtypes_cpu_t temp;
 
-	m_range_check(mfree);
-	m_range_check(mclfree);
-	m_range_check(mbigfree);
+		bcopy(&scp->cpu_mtypes, &temp.cpu_mtypes,
+		    sizeof (temp.cpu_mtypes));
 
+		for (n = 0; n < MT_MAX; n++)
+			mtc.cpu_mtypes[n] += temp.cpu_mtypes[n];
+	}
+	lck_mtx_lock(mbuf_mlock);
+	for (n = 0; n < MT_MAX; n++)
+		mbstat.m_mtypes[n] = mtc.cpu_mtypes[n];
+	lck_mtx_unlock(mbuf_mlock);
 
-	return (m);
+	return (SYSCTL_OUT(req, &mbstat, sizeof (mbstat)));
 }
 
-struct mbuf *
-m_gethdr(
-	int nowait, 
-	int type)
+static int
+mb_stat_sysctl SYSCTL_HANDLER_ARGS
 {
-	struct mbuf *m;
+#pragma unused(oidp, arg1, arg2)
+	mcache_t *cp;
+	mcache_cpu_t *ccp;
+	mb_class_stat_t *sp;
+	int k, m, bktsize;
+
+	lck_mtx_lock(mbuf_mlock);
+	for (k = 0; k < NELEM(mbuf_table); k++) {
+		cp = m_cache(k);
+		ccp = &cp->mc_cpu[0];
+		bktsize = ccp->cc_bktsize;
+		sp = mbuf_table[k].mtbl_stats;
+
+		if (cp->mc_flags & MCF_NOCPUCACHE)
+			sp->mbcl_mc_state = MCS_DISABLED;
+		else if (cp->mc_purge_cnt > 0)
+			sp->mbcl_mc_state = MCS_PURGING;
+		else if (bktsize == 0)
+			sp->mbcl_mc_state = MCS_OFFLINE;
+		else
+			sp->mbcl_mc_state = MCS_ONLINE;
+
+		sp->mbcl_mc_cached = 0;
+		for (m = 0; m < ncpu; m++) {
+			ccp = &cp->mc_cpu[m];
+			if (ccp->cc_objs > 0)
+				sp->mbcl_mc_cached += ccp->cc_objs;
+			if (ccp->cc_pobjs > 0)
+				sp->mbcl_mc_cached += ccp->cc_pobjs;
+		}
+		sp->mbcl_mc_cached += (cp->mc_full.bl_total * bktsize);
+		sp->mbcl_active = sp->mbcl_total - sp->mbcl_mc_cached -
+		    sp->mbcl_infree;
+
+		sp->mbcl_mc_waiter_cnt = cp->mc_waiter_cnt;
+		sp->mbcl_mc_wretry_cnt = cp->mc_wretry_cnt;
+		sp->mbcl_mc_nwretry_cnt = cp->mc_nwretry_cnt;
+
+		/* Calculate total count specific to each class */
+		sp->mbcl_ctotal = sp->mbcl_total;
+		switch (m_class(k)) {
+		case MC_MBUF:
+			/* Deduct mbufs used in composite caches */
+			sp->mbcl_ctotal -= (m_total(MC_MBUF_CL) +
+			    m_total(MC_MBUF_BIGCL));
+			break;
 
-	m_range_check(mfree);
-	m_range_check(mclfree);
-	m_range_check(mbigfree);
+		case MC_CL:
+			/* Deduct clusters used in composite cache and mbufs */
+			sp->mbcl_ctotal -= (m_total(MC_MBUF_CL) +
+			    (P2ROUNDUP(m_total(MC_MBUF), NMBPCL)/NMBPCL));
+			break;
 
+		case MC_BIGCL:
+			/* Deduct clusters used in composite cache */
+			sp->mbcl_ctotal -= m_total(MC_MBUF_BIGCL);
+			break;
 
-	_MINTGET(m, type);
-	if (m) {
-		m->m_next = m->m_nextpkt = 0;
-		m->m_type = type;
-		m->m_data = m->m_pktdat;
-		m->m_flags = M_PKTHDR;
-		m->m_len = 0;
-		_M_CLEAR_PKTHDR(m)
-	} else
-		m = m_retryhdr(nowait, type);
+		case MC_16KCL:
+			/* Deduct clusters used in composite cache */
+			sp->mbcl_ctotal -= m_total(MC_MBUF_16KCL);
+			break;
+
+		default:
+			break;
+		}
+	}
+	lck_mtx_unlock(mbuf_mlock);
 
-	m_range_check(mfree);
-	m_range_check(mclfree);
-	m_range_check(mbigfree);
+	return (SYSCTL_OUT(req, mb_stat, MB_STAT_SIZE(NELEM(mbuf_table))));
+}
 
+static inline void
+m_incref(struct mbuf *m)
+{
+	UInt32 old, new;
+	volatile UInt32 *addr = (volatile UInt32 *)&MEXT_REF(m);
 
-	return m;
+	do {
+		old = *addr;
+		new = old + 1;
+		ASSERT(new != 0);
+	} while (!OSCompareAndSwap(old, new, addr));
 }
 
-struct mbuf *
-m_getclr(
-	int nowait, 
-	int type)
+static inline u_int32_t
+m_decref(struct mbuf *m)
 {
-	struct mbuf *m;
+	UInt32 old, new;
+	volatile UInt32 *addr = (volatile UInt32 *)&MEXT_REF(m);
 
-	MGET(m, nowait, type);
-	if (m == 0)
-		return (0);
-	bzero(mtod(m, caddr_t), MLEN);
-	return (m);
+	do {
+		old = *addr;
+		new = old - 1;
+		ASSERT(old != 0);
+	} while (!OSCompareAndSwap(old, new, addr));
+
+	return (new);
 }
 
-struct mbuf *
-m_free(
-	struct mbuf *m)
+static void
+mbuf_table_init(void)
 {
-	struct mbuf *n = m->m_next;
-	int i;
+	int m;
 
-	m_range_check(m);
-	m_range_check(mfree);
-	m_range_check(mclfree);
+	MALLOC(mb_stat, mb_stat_t *, MB_STAT_SIZE(NELEM(mbuf_table)),
+	    M_TEMP, M_WAITOK | M_ZERO);
+	VERIFY(mb_stat != NULL);
 
-	if (m->m_type == MT_FREE)
-		panic("freeing free mbuf");
+	mb_stat->mbs_cnt = NELEM(mbuf_table);
+	for (m = 0; m < NELEM(mbuf_table); m++)
+		mbuf_table[m].mtbl_stats = &mb_stat->mbs_class[m];
 
-	/* Free the aux data if there is any */
-	if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
-	{
-		m_freem(m->m_pkthdr.aux);
-	}
-	if ((m->m_flags & M_PKTHDR) != 0)
-		m_tag_delete_chain(m, NULL);
+#if CONFIG_MBUF_JUMBO
+	/*
+	 * Set aside 1/3 of the mbuf cluster map for jumbo clusters; we do
+	 * this only on platforms where jumbo cluster pool is enabled.
+	 */
+	njcl = nmbclusters / 3;
+	njclbytes = M16KCLBYTES;
+#endif /* CONFIG_MBUF_JUMBO */
 
-	MBUF_LOCK();
-	if ((m->m_flags & M_EXT)) 
-    {
-		if (MCLHASREFERENCE(m)) {
-			remque((queue_t)&m->m_ext.ext_refs);
-		} else if (m->m_ext.ext_free == NULL) {
-			union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
-			
-			m_range_check(mcl);
-			
-			if (_MCLUNREF(mcl)) {
-				mcl->mcl_next = mclfree;
-				mclfree = mcl;
-				++mbstat.m_clfree;
-			} 
-#ifdef COMMENT_OUT
-/* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]", 
-       and AppleTalk ADSP uses m_split(), this incorrect sanity check
-       caused a panic.  
-*** */
-			else	/* sanity check - not referenced this way */
-				panic("m_free m_ext cluster not free");
-#endif
-		} else {
-			(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
-			    m->m_ext.ext_size, m->m_ext.ext_arg);
-		}
+	/*
+	 * nclusters is going to be split in 2 to hold both the 2K
+	 * and the 4K pools, so make sure each half is even.
+	 */
+	nclusters = P2ROUNDDOWN(nmbclusters - njcl, 4);
+	if (njcl > 0) {
+		/*
+		 * Each jumbo cluster takes 8 2K clusters, so make
+		 * sure that the pool size is evenly divisible by 8.
+		 */
+		njcl = P2ROUNDDOWN(nmbclusters - nclusters, 8);
 	}
-	mbstat.m_mtypes[m->m_type]--;
-	(void) _MCLUNREF(m);
-	_MFREE_MUNGE(m);
-	m->m_type = MT_FREE;
-	mbstat.m_mtypes[m->m_type]++;
-	m->m_flags = 0;
-	m->m_next = mfree;
-	m->m_len = 0;
-	mfree = m;
-	i = m_want;
-	m_want = 0;
-	MBUF_UNLOCK();
-	if (i) wakeup((caddr_t)&mfree);
-	return (n);
-}
 
-/* m_mclget() add an mbuf cluster to a normal mbuf */
-struct mbuf *
-m_mclget(
-        struct mbuf *m,
-        int nowait)
-{
-	MCLALLOC(m->m_ext.ext_buf, nowait);
-	if (m->m_ext.ext_buf) {
-		m->m_data = m->m_ext.ext_buf;
-		m->m_flags |= M_EXT;
-		m->m_ext.ext_size = MCLBYTES;
-		m->m_ext.ext_free = 0;
-		m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
-			&m->m_ext.ext_refs;
+#if CONFIG_MBUF_NOEXPAND
+	/* Only use 4k clusters if we're setting aside more than 256k */
+	if (nmbclusters <= 128) {
+		maxmbufcl = nmbclusters / 4;
+	} else {
+		/* Half to big clusters, half to small */
+		maxmbufcl = (nmbclusters / 4) * 3;
 	}
-        
-    return m;
-}
+#endif /* CONFIG_MBUF_NOEXPAND */
+
+	/*
+	 * 1/2 of the map is reserved for 2K clusters.  Out of this, 1/16th
+	 * of the total number of 2K clusters allocated is reserved and cannot
+	 * be turned into mbufs.  It can only be used for pure cluster objects.
+	 */
+	m_minlimit(MC_CL) = (nclusters >> 5);
+	m_maxlimit(MC_CL) = (nclusters >> 1);
+	m_maxsize(MC_CL) = m_size(MC_CL) = MCLBYTES;
+	(void) snprintf(m_cname(MC_CL), MAX_MBUF_CNAME, "cl");
+
+	/*
+	 * The remaining (15/16th) can be turned into mbufs.
+	 */
+	m_minlimit(MC_MBUF) = 0;
+	m_maxlimit(MC_MBUF) = (m_maxlimit(MC_CL) - m_minlimit(MC_CL)) * NMBPCL;
+	m_maxsize(MC_MBUF) = m_size(MC_MBUF) = MSIZE;
+	(void) snprintf(m_cname(MC_MBUF), MAX_MBUF_CNAME, "mbuf");
+
+	/*
+	 * The other 1/2 of the map is reserved for 4K clusters.
+	 */
+	m_minlimit(MC_BIGCL) = 0;
+	m_maxlimit(MC_BIGCL) = m_maxlimit(MC_CL) >> 1;
+	m_maxsize(MC_BIGCL) = m_size(MC_BIGCL) = NBPG;
+	(void) snprintf(m_cname(MC_BIGCL), MAX_MBUF_CNAME, "bigcl");
+
+	/*
+	 * Set limits for the composite classes.
+	 */
+	m_minlimit(MC_MBUF_CL) = 0;
+	m_maxlimit(MC_MBUF_CL) = m_maxlimit(MC_CL) - m_minlimit(MC_CL);
+	m_maxsize(MC_MBUF_CL) = MCLBYTES;
+	m_size(MC_MBUF_CL) = m_size(MC_MBUF) + m_size(MC_CL);
+	(void) snprintf(m_cname(MC_MBUF_CL), MAX_MBUF_CNAME, "mbuf_cl");
+
+	m_minlimit(MC_MBUF_BIGCL) = 0;
+	m_maxlimit(MC_MBUF_BIGCL) = m_maxlimit(MC_BIGCL);
+	m_maxsize(MC_MBUF_BIGCL) = NBPG;
+	m_size(MC_MBUF_BIGCL) = m_size(MC_MBUF) + m_size(MC_BIGCL);
+	(void) snprintf(m_cname(MC_MBUF_BIGCL), MAX_MBUF_CNAME, "mbuf_bigcl");
+
+	/*
+	 * And for jumbo classes.
+	 */
+	m_minlimit(MC_16KCL) = 0;
+	m_maxlimit(MC_16KCL) = (njcl >> 3);
+	m_maxsize(MC_16KCL) = m_size(MC_16KCL) = M16KCLBYTES;
+	(void) snprintf(m_cname(MC_16KCL), MAX_MBUF_CNAME, "16kcl");
+
+	m_minlimit(MC_MBUF_16KCL) = 0;
+	m_maxlimit(MC_MBUF_16KCL) = m_maxlimit(MC_16KCL);
+	m_maxsize(MC_MBUF_16KCL) = M16KCLBYTES;
+	m_size(MC_MBUF_16KCL) = m_size(MC_MBUF) + m_size(MC_16KCL);
+	(void) snprintf(m_cname(MC_MBUF_16KCL), MAX_MBUF_CNAME, "mbuf_16kcl");
+
+	/*
+	 * Initialize the legacy mbstat structure.
+	 */
+	bzero(&mbstat, sizeof (mbstat));
+	mbstat.m_msize = m_maxsize(MC_MBUF);
+	mbstat.m_mclbytes = m_maxsize(MC_CL);
+	mbstat.m_minclsize = MINCLSIZE;
+	mbstat.m_mlen = MLEN;
+	mbstat.m_mhlen = MHLEN;
+	mbstat.m_bigmclbytes = m_maxsize(MC_BIGCL);
+}
+
+__private_extern__ void
+mbinit(void)
+{
+	unsigned int m;
+	int initmcl = MINCL;
+	int mcl_pages;
+	void *buf;
+
+	if (nmbclusters == 0)
+		nmbclusters = NMBCLUSTERS;
+
+	/* Setup the mbuf table */
+	mbuf_table_init();
+
+	/* Global lock for common layer */
+	mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
+	mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
+	mbuf_mlock_attr = lck_attr_alloc_init();
+	mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
+
+	/* Allocate cluster slabs table */
+	maxslabgrp = P2ROUNDUP(nmbclusters, NSLABSPMB) / NSLABSPMB;
+	MALLOC(slabstbl, mcl_slabg_t **, maxslabgrp * sizeof (mcl_slabg_t *),
+	    M_TEMP, M_WAITOK | M_ZERO);
+	VERIFY(slabstbl != NULL);
+
+	/* Allocate audit structures if needed */
+	PE_parse_boot_arg("mbuf_debug", &mbuf_debug);
+	mbuf_debug |= mcache_getflags();
+	if (mbuf_debug & MCF_AUDIT) {
+		MALLOC(mclaudit, mcl_audit_t *,
+		    nmbclusters * sizeof (*mclaudit), M_TEMP,
+		    M_WAITOK | M_ZERO);
+		VERIFY(mclaudit != NULL);
+
+		mcl_audit_con_cache = mcache_create("mcl_audit_contents",
+		    AUDIT_CONTENTS_SIZE, 0, 0, MCR_SLEEP);
+		VERIFY(mcl_audit_con_cache != NULL);
+	}
+
+	/* Calculate the number of pages assigned to the cluster pool */
+	mcl_pages = nmbclusters/(NBPG/CLBYTES);
+	MALLOC(mcl_paddr, int *, mcl_pages * sizeof (int), M_TEMP, M_WAITOK);
+	VERIFY(mcl_paddr != NULL);
+
+	/* Register with the I/O Bus mapper */
+	mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
+	bzero((char *)mcl_paddr, mcl_pages * sizeof (int));
+
+	embutl = (union mcluster *)
+	    ((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
+
+	PE_parse_boot_arg("initmcl", &initmcl);
+
+	lck_mtx_lock(mbuf_mlock);
+
+	if (m_clalloc(MAX(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES) == 0)
+		panic("mbinit: m_clalloc failed\n");
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	(void) kernel_thread(kernel_task, mbuf_worker_thread_init);
+
+	ref_cache = mcache_create("mext_ref", sizeof (struct ext_ref),
+	    0, 0, MCR_SLEEP);
+
+	/* Create the cache for each class */
+	for (m = 0; m < NELEM(mbuf_table); m++) {
+		void *allocfunc, *freefunc, *auditfunc;
+		u_int32_t flags;
+
+		flags = mbuf_debug;
+		if (m_class(m) == MC_MBUF_CL || m_class(m) == MC_MBUF_BIGCL ||
+		    m_class(m) == MC_MBUF_16KCL) {
+			allocfunc = mbuf_cslab_alloc;
+			freefunc = mbuf_cslab_free;
+			auditfunc = mbuf_cslab_audit;
+		} else {
+			allocfunc = mbuf_slab_alloc;
+			freefunc = mbuf_slab_free;
+			auditfunc = mbuf_slab_audit;
+		}
+
+		/*
+		 * Disable per-CPU caches for jumbo classes if there
+		 * is no jumbo cluster pool available in the system.
+		 * The cache itself is still created (but will never
+		 * be populated) since it simplifies the code.
+		 */
+		if ((m_class(m) == MC_MBUF_16KCL || m_class(m) == MC_16KCL) &&
+		    njcl == 0)
+			flags |= MCF_NOCPUCACHE;
+
+		m_cache(m) = mcache_create_ext(m_cname(m), m_maxsize(m),
+		    allocfunc, freefunc, auditfunc, mbuf_slab_notify,
+		    (void *)m, flags, MCR_SLEEP);
+	}
+
+	/*
+	 * Allocate structure for per-CPU statistics that's aligned
+	 * on the CPU cache boundary; this code assumes that we never
+	 * uninitialize this framework, since the original address
+	 * before alignment is not saved.
+	 */
+	ncpu = ml_get_max_cpus();
+	MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_SIZE,
+	    M_TEMP, M_WAITOK);
+	VERIFY(buf != NULL);
+
+	mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, CPU_CACHE_SIZE);
+	bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu));
+
+	printf("mbinit: done\n");
+}
+
+/*
+ * Obtain a slab of object(s) from the class's freelist.
+ */
+static mcache_obj_t *
+slab_alloc(mbuf_class_t class, int wait)
+{
+	mcl_slab_t *sp;
+	mcache_obj_t *buf;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	VERIFY(class != MC_16KCL || njcl > 0);
+
+	/* This should always be NULL for us */
+	VERIFY(m_cobjlist(class) == NULL);
+
+	/*
+	 * Treat composite objects as having longer lifespan by using
+	 * a slab from the reverse direction, in hoping that this could
+	 * reduce the probability of fragmentation for slabs that hold
+	 * more than one buffer chunks (e.g. mbuf slabs).  For other
+	 * slabs, this probably doesn't make much of a difference.
+	 */
+	if (class == MC_MBUF && (wait & MCR_COMP))
+		sp = (mcl_slab_t *)TAILQ_LAST(&m_slablist(class), mcl_slhead);
+	else
+		sp = (mcl_slab_t *)TAILQ_FIRST(&m_slablist(class));
+
+	if (sp == NULL) {
+		VERIFY(m_infree(class) == 0 && m_slab_cnt(class) == 0);
+		/* The slab list for this class is empty */
+		return (NULL);
+	}
+
+	VERIFY(m_infree(class) > 0);
+	VERIFY(!slab_is_detached(sp));
+	VERIFY(sp->sl_class == class &&
+	    (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
+	buf = sp->sl_head;
+	VERIFY(slab_inrange(sp, buf) && sp == slab_get(buf));
+
+	if (class == MC_MBUF) {
+		sp->sl_head = buf->obj_next;
+		VERIFY(sp->sl_head != NULL || sp->sl_refcnt == (NMBPCL - 1));
+	} else {
+		sp->sl_head = NULL;
+	}
+	if (sp->sl_head != NULL && !slab_inrange(sp, sp->sl_head)) {
+		slab_nextptr_panic(sp, sp->sl_head);
+		/* In case sl_head is in the map but not in the slab */
+		VERIFY(slab_inrange(sp, sp->sl_head));
+		/* NOTREACHED */
+	}
+
+	/* Increment slab reference */
+	sp->sl_refcnt++;
+
+	if (mclaudit != NULL) {
+		mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
+		mca->mca_uflags = 0;
+		/* Save contents on mbuf objects only */
+		if (class == MC_MBUF)
+			mca->mca_uflags |= MB_SCVALID;
+	}
+
+	if (class == MC_CL) {
+		mbstat.m_clfree = (--m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
+		/*
+		 * A 2K cluster slab can have at most 1 reference.
+		 */
+		VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 &&
+		    sp->sl_len == m_maxsize(MC_CL) && sp->sl_head == NULL);
+	} else if (class == MC_BIGCL) {
+		mcl_slab_t *nsp = sp->sl_next;
+		mbstat.m_bigclfree = (--m_infree(MC_BIGCL)) +
+		    m_infree(MC_MBUF_BIGCL);
+		/*
+		 * Increment 2nd slab.  A 4K big cluster takes
+		 * 2 slabs, each having at most 1 reference.
+		 */
+		VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 &&
+		    sp->sl_len == m_maxsize(MC_BIGCL) && sp->sl_head == NULL);
+		/* Next slab must already be present */
+		VERIFY(nsp != NULL);
+		nsp->sl_refcnt++;
+		VERIFY(!slab_is_detached(nsp));
+		VERIFY(nsp->sl_class == MC_BIGCL &&
+		    nsp->sl_flags == (SLF_MAPPED | SLF_PARTIAL) &&
+		    nsp->sl_refcnt == 1 && nsp->sl_chunks == 0 &&
+		    nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
+		    nsp->sl_head == NULL);
+	} else if (class == MC_16KCL) {
+		mcl_slab_t *nsp;
+		int k;
+
+		--m_infree(MC_16KCL);
+		VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 &&
+		    sp->sl_len == m_maxsize(MC_16KCL) && sp->sl_head == NULL);
+		/*
+		 * Increment 2nd-8th slab.  A 16K big cluster takes
+		 * 8 cluster slabs, each having at most 1 reference.
+		 */
+		for (nsp = sp, k = 1; k < (M16KCLBYTES / MCLBYTES); k++) {
+			nsp = nsp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			nsp->sl_refcnt++;
+			VERIFY(!slab_is_detached(nsp));
+			VERIFY(nsp->sl_class == MC_16KCL &&
+			    nsp->sl_flags == (SLF_MAPPED | SLF_PARTIAL) &&
+			    nsp->sl_refcnt == 1 && nsp->sl_chunks == 0 &&
+			    nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
+			    nsp->sl_head == NULL);
+		}
+	} else {
+		ASSERT(class == MC_MBUF);
+		--m_infree(MC_MBUF);
+		/*
+		 * If auditing is turned on, this check is
+		 * deferred until later in mbuf_slab_audit().
+		 */
+		if (mclaudit == NULL)
+			_MCHECK((struct mbuf *)buf);
+		/*
+		 * Since we have incremented the reference count above,
+		 * an mbuf slab (formerly a 2K cluster slab that was cut
+		 * up into mbufs) must have a reference count between 1
+		 * and NMBPCL at this point.
+		 */
+		VERIFY(sp->sl_refcnt >= 1 &&
+		    (unsigned short)sp->sl_refcnt <= NMBPCL &&
+		    sp->sl_chunks == NMBPCL && sp->sl_len == m_maxsize(MC_CL));
+		VERIFY((unsigned short)sp->sl_refcnt < NMBPCL ||
+		    sp->sl_head == NULL);
+	}
+
+	/* If empty, remove this slab from the class's freelist */
+	if (sp->sl_head == NULL) {
+		VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPCL);
+		slab_remove(sp, class);
+	}
+
+	return (buf);
+}
+
+/*
+ * Place a slab of object(s) back into a class's slab list.
+ */
+static void
+slab_free(mbuf_class_t class, mcache_obj_t *buf)
+{
+	mcl_slab_t *sp;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	VERIFY(class != MC_16KCL || njcl > 0);
+	VERIFY(buf->obj_next == NULL);
+	sp = slab_get(buf);
+	VERIFY(sp->sl_class == class && slab_inrange(sp, buf) &&
+	    (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
+
+	/* Decrement slab reference */
+	sp->sl_refcnt--;
+
+	if (class == MC_CL || class == MC_BIGCL) {
+		VERIFY(IS_P2ALIGNED(buf, MCLBYTES));
+		/*
+		 * A 2K cluster slab can have at most 1 reference
+		 * which must be 0 at this point.
+		 */
+		VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 &&
+		    sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
+		VERIFY(slab_is_detached(sp));
+		if (class == MC_BIGCL) {
+			mcl_slab_t *nsp = sp->sl_next;
+			VERIFY(IS_P2ALIGNED(buf, NBPG));
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			/* Decrement 2nd slab reference */
+			nsp->sl_refcnt--;
+			/*
+			 * A 4K big cluster takes 2 slabs, both
+			 * must now have 0 reference.
+			 */
+			VERIFY(slab_is_detached(nsp));
+			VERIFY(nsp->sl_class == MC_BIGCL &&
+			    (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) &&
+			    nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 &&
+			    nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
+			    nsp->sl_head == NULL);
+		}
+	} else if (class == MC_16KCL) {
+		mcl_slab_t *nsp;
+		int k;
+		/*
+		 * A 16K cluster takes 8 cluster slabs, all must
+		 * now have 0 reference.
+		 */
+		VERIFY(IS_P2ALIGNED(buf, NBPG));
+		VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 &&
+		    sp->sl_len == m_maxsize(MC_16KCL) && sp->sl_head == NULL);
+		VERIFY(slab_is_detached(sp));
+		for (nsp = sp, k = 1; k < (M16KCLBYTES / MCLBYTES); k++) {
+			nsp = nsp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			nsp->sl_refcnt--;
+			VERIFY(slab_is_detached(nsp));
+			VERIFY(nsp->sl_class == MC_16KCL &&
+			    (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) &&
+			    nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 &&
+			    nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
+			    nsp->sl_head == NULL);
+		}
+	} else {
+		/*
+		 * An mbuf slab has a total of NMBPL reference counts.
+		 * Since we have decremented the reference above, it
+		 * must now be between 0 and NMBPCL-1.
+		 */
+		VERIFY(sp->sl_refcnt >= 0 &&
+		    (unsigned short)sp->sl_refcnt <= (NMBPCL - 1) &&
+		    sp->sl_chunks == NMBPCL && sp->sl_len == m_maxsize(MC_CL));
+		VERIFY(sp->sl_refcnt < (NMBPCL - 1) ||
+		    (slab_is_detached(sp) && sp->sl_head == NULL));
+	}
+
+	/*
+	 * When auditing is enabled, ensure that the buffer still
+	 * contains the free pattern.  Otherwise it got corrupted
+	 * while at the CPU cache layer.
+	 */
+	if (mclaudit != NULL) {
+		mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
+		mcache_audit_free_verify(mca, buf, 0, m_maxsize(class));
+		mca->mca_uflags &= ~MB_SCVALID;
+	}
+
+	if (class == MC_CL) {
+		mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
+	} else if (class == MC_BIGCL) {
+		mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) +
+		    m_infree(MC_MBUF_BIGCL);
+	} else if (class == MC_16KCL) {
+		++m_infree(MC_16KCL);
+	} else {
+		++m_infree(MC_MBUF);
+		buf->obj_next = sp->sl_head;
+	}
+	sp->sl_head = buf;
+
+	/* All mbufs are freed; return the cluster that we stole earlier */
+	if (sp->sl_refcnt == 0 && class == MC_MBUF) {
+		int i = NMBPCL;
+
+		m_total(MC_MBUF) -= NMBPCL;
+		mbstat.m_mbufs = m_total(MC_MBUF);
+		m_infree(MC_MBUF) -= NMBPCL;
+		mtype_stat_add(MT_FREE, -NMBPCL);
+
+		while (i--) {
+			struct mbuf *m = sp->sl_head;
+			VERIFY(m != NULL);
+			sp->sl_head = m->m_next;
+			m->m_next = NULL;
+		}
+		VERIFY(sp->sl_head == NULL);
+
+		/* Remove the slab from the mbuf class's slab list */
+		slab_remove(sp, class);
+
+		/* Reinitialize it as a 2K cluster slab */
+		slab_init(sp, MC_CL, sp->sl_flags, sp->sl_base, sp->sl_base,
+		    sp->sl_len, 0, 1);
+
+		if (mclaudit != NULL)
+			mcache_set_pattern(MCACHE_FREE_PATTERN,
+			    (caddr_t)sp->sl_head, m_maxsize(MC_CL));
+
+		mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
+
+		VERIFY(slab_is_detached(sp));
+		/* And finally switch class */
+		class = MC_CL;
+	}
+
+	/* Reinsert the slab to the class's slab list */
+	if (slab_is_detached(sp))
+		slab_insert(sp, class);
+}
+
+/*
+ * Common allocator for rudimentary objects called by the CPU cache layer
+ * during an allocation request whenever there is no available element in the
+ * bucket layer.  It returns one or more elements from the appropriate global
+ * freelist.  If the freelist is empty, it will attempt to populate it and
+ * retry the allocation.
+ */
+static unsigned int
+mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	unsigned int need = num;
+	mcache_obj_t **list = *plist;
+
+	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+	ASSERT(need > 0);
+
+	lck_mtx_lock(mbuf_mlock);
+
+	for (;;) {
+		if ((*list = slab_alloc(class, wait)) != NULL) {
+			(*list)->obj_next = NULL;
+			list = *plist = &(*list)->obj_next;
+
+			if (--need == 0) {
+				/*
+				 * If the number of elements in freelist has
+				 * dropped below low watermark, asynchronously
+				 * populate the freelist now rather than doing
+				 * it later when we run out of elements.
+				 */
+				if (!mbuf_cached_above(class, wait) &&
+				    m_infree(class) < m_total(class) >> 5) {
+					(void) freelist_populate(class, 1,
+					    M_DONTWAIT);
+				}
+				break;
+			}
+		} else {
+			VERIFY(m_infree(class) == 0 || class == MC_CL);
+
+			(void) freelist_populate(class, 1,
+			    (wait & MCR_NOSLEEP) ? M_DONTWAIT : M_WAIT);
+
+			if (m_infree(class) > 0)
+				continue;
+
+			/* Check if there's anything at the cache layer */
+			if (mbuf_cached_above(class, wait))
+				break;
+
+			/* We have nothing and cannot block; give up */
+			if (wait & MCR_NOSLEEP) {
+				if (!(wait & MCR_TRYHARD)) {
+					m_fail_cnt(class)++;
+					mbstat.m_drops++;
+					break;
+				}
+			}
+
+			/*
+			 * If the freelist is still empty and the caller is
+			 * willing to be blocked, sleep on the wait channel
+			 * until an element is available.  Otherwise, if
+			 * MCR_TRYHARD is set, do our best to satisfy the
+			 * request without having to go to sleep.
+			 */
+			if (mbuf_worker_ready &&
+			    mbuf_sleep(class, need, wait))
+				break;
+
+			lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+		}
+	}
+
+	m_alloc_cnt(class) += num - need;
+	lck_mtx_unlock(mbuf_mlock);
+
+	return (num - need);
+}
+
+/*
+ * Common de-allocator for rudimentary objects called by the CPU cache
+ * layer when one or more elements need to be returned to the appropriate
+ * global freelist.
+ */
+static void
+mbuf_slab_free(void *arg, mcache_obj_t *list, __unused int purged)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	mcache_obj_t *nlist;
+	unsigned int num = 0;
+	int w;
+
+	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+
+	lck_mtx_lock(mbuf_mlock);
+
+	for (;;) {
+		nlist = list->obj_next;
+		list->obj_next = NULL;
+		slab_free(class, list);
+		++num;
+		if ((list = nlist) == NULL)
+			break;
+	}
+	m_free_cnt(class) += num;
+
+	if ((w = mb_waiters) > 0)
+		mb_waiters = 0;
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	if (w != 0)
+		wakeup(mb_waitchan);
+}
+
+/*
+ * Common auditor for rudimentary objects called by the CPU cache layer
+ * during an allocation or free request.  For the former, this is called
+ * after the objects are obtained from either the bucket or slab layer
+ * and before they are returned to the caller.  For the latter, this is
+ * called immediately during free and before placing the objects into
+ * the bucket or slab layer.
+ */
+static void
+mbuf_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	mcache_audit_t *mca;
+
+	ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+
+	while (list != NULL) {
+		lck_mtx_lock(mbuf_mlock);
+		mca = mcl_audit_buf2mca(class, list);
+
+		/* Do the sanity checks */
+		if (class == MC_MBUF) {
+			mcl_audit_mbuf(mca, list, FALSE, alloc);
+			ASSERT(mca->mca_uflags & MB_SCVALID);
+		} else {
+			mcl_audit_cluster(mca, list, m_maxsize(class),
+			    alloc, TRUE);
+			ASSERT(!(mca->mca_uflags & MB_SCVALID));
+		}
+		/* Record this transaction */
+		mcache_buffer_log(mca, list, m_cache(class));
+		if (alloc)
+			mca->mca_uflags |= MB_INUSE;
+		else
+			mca->mca_uflags &= ~MB_INUSE;
+		/* Unpair the object (unconditionally) */
+		mca->mca_uptr = NULL;
+		lck_mtx_unlock(mbuf_mlock);
+
+		list = list->obj_next;
+	}
+}
+
+/*
+ * Common notify routine for all caches.  It is called by mcache when
+ * one or more objects get freed.  We use this indication to trigger
+ * the wakeup of any sleeping threads so that they can retry their
+ * allocation requests.
+ */
+static void
+mbuf_slab_notify(void *arg, u_int32_t reason)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	int w;
+
+	ASSERT(MBUF_CLASS_VALID(class));
+
+	if (reason != MCN_RETRYALLOC)
+		return;
+
+	lck_mtx_lock(mbuf_mlock);
+	if ((w = mb_waiters) > 0) {
+		m_notified(class)++;
+		mb_waiters = 0;
+	}
+	lck_mtx_unlock(mbuf_mlock);
+
+	if (w != 0)
+		wakeup(mb_waitchan);
+}
+
+/*
+ * Obtain object(s) from the composite class's freelist.
+ */
+static unsigned int
+cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num)
+{
+	unsigned int need = num;
+	mcl_slab_t *sp, *clsp, *nsp;
+	struct mbuf *m;
+	mcache_obj_t **list = *plist;
+	void *cl;
+
+	VERIFY(need > 0);
+	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	/* Get what we can from the freelist */
+	while ((*list = m_cobjlist(class)) != NULL) {
+		MRANGE(*list);
+
+		m = (struct mbuf *)*list;
+		sp = slab_get(m);
+		cl = m->m_ext.ext_buf;
+		clsp = slab_get(cl);
+		VERIFY(m->m_flags == M_EXT && cl != NULL);
+		VERIFY(MEXT_RFA(m) != NULL && MBUF_IS_COMPOSITE(m));
+		VERIFY(clsp->sl_refcnt == 1);
+		if (class == MC_MBUF_BIGCL) {
+			nsp = clsp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			VERIFY(nsp->sl_refcnt == 1);
+		} else if (class == MC_MBUF_16KCL) {
+			int k;
+			for (nsp = clsp, k = 1;
+			    k < (M16KCLBYTES / MCLBYTES); k++) {
+				nsp = nsp->sl_next;
+				/* Next slab must already be present */
+				VERIFY(nsp != NULL);
+				VERIFY(nsp->sl_refcnt == 1);
+			}
+		}
+
+		if ((m_cobjlist(class) = (*list)->obj_next) != NULL &&
+		    !MBUF_IN_MAP(m_cobjlist(class))) {
+			slab_nextptr_panic(sp, m_cobjlist(class));
+			/* NOTREACHED */
+		}
+		(*list)->obj_next = NULL;
+		list = *plist = &(*list)->obj_next;
+
+		if (--need == 0)
+			break;
+	}
+	m_infree(class) -= (num - need);
+
+	return (num - need);
+}
+
+/*
+ * Place object(s) back into a composite class's freelist.
+ */
+static unsigned int
+cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged)
+{
+	mcache_obj_t *o, *tail;
+	unsigned int num = 0;
+	struct mbuf *m, *ms;
+	mcache_audit_t *mca = NULL;
+	mcache_obj_t *ref_list = NULL;
+	mcl_slab_t *clsp, *nsp;
+	void *cl;
+
+	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	o = tail = list;
+
+	while ((m = ms = (struct mbuf *)o) != NULL) {
+		mcache_obj_t *rfa, *nexto = o->obj_next;
+
+		/* Do the mbuf sanity checks */
+		if (mclaudit != NULL) {
+			mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+			mcache_audit_free_verify(mca, m, 0, m_maxsize(MC_MBUF));
+			ms = (struct mbuf *)mca->mca_contents;
+		}
+
+		/* Do the cluster sanity checks */
+		cl = ms->m_ext.ext_buf;
+		clsp = slab_get(cl);
+		if (mclaudit != NULL) {
+			size_t size;
+			if (class == MC_MBUF_CL)
+				size = m_maxsize(MC_CL);
+			else if (class == MC_MBUF_BIGCL)
+				size = m_maxsize(MC_BIGCL);
+			else
+				size = m_maxsize(MC_16KCL);
+			mcache_audit_free_verify(mcl_audit_buf2mca(MC_CL,
+			    (mcache_obj_t *)cl), cl, 0, size);
+		}
+		VERIFY(ms->m_type == MT_FREE);
+		VERIFY(ms->m_flags == M_EXT);
+		VERIFY(MEXT_RFA(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+		VERIFY(clsp->sl_refcnt == 1);
+		if (class == MC_MBUF_BIGCL) {
+			nsp = clsp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			VERIFY(nsp->sl_refcnt == 1);
+		} else if (class == MC_MBUF_16KCL) {
+			int k;
+			for (nsp = clsp, k = 1;
+			    k < (M16KCLBYTES / MCLBYTES); k++) {
+				nsp = nsp->sl_next;
+				/* Next slab must already be present */
+				VERIFY(nsp != NULL);
+				VERIFY(nsp->sl_refcnt == 1);
+			}
+		}
+
+		/*
+		 * If we're asked to purge, restore the actual mbuf using
+		 * contents of the shadow structure (if auditing is enabled)
+		 * and clear EXTF_COMPOSITE flag from the mbuf, as we are
+		 * about to free it and the attached cluster into their caches.
+		 */
+		if (purged) {
+			/* Restore constructed mbuf fields */
+			if (mclaudit != NULL)
+				mcl_audit_restore_mbuf(m, mca, TRUE);
+
+			MEXT_REF(m) = 0;
+			MEXT_FLAGS(m) = 0;
+
+			rfa = (mcache_obj_t *)MEXT_RFA(m);
+			rfa->obj_next = ref_list;
+			ref_list = rfa;
+			MEXT_RFA(m) = NULL;
+
+			m->m_type = MT_FREE;
+			m->m_flags = m->m_len = 0;
+			m->m_next = m->m_nextpkt = NULL;
+
+			/* Save mbuf fields and make auditing happy */
+			if (mclaudit != NULL)
+				mcl_audit_mbuf(mca, o, FALSE, FALSE);
+
+			VERIFY(m_total(class) > 0);
+			m_total(class)--;
+
+			/* Free the mbuf */
+			o->obj_next = NULL;
+			slab_free(MC_MBUF, o);
+
+			/* And free the cluster */
+			((mcache_obj_t *)cl)->obj_next = NULL;
+			if (class == MC_MBUF_CL)
+				slab_free(MC_CL, cl);
+			else if (class == MC_MBUF_BIGCL)
+				slab_free(MC_BIGCL, cl);
+			else
+				slab_free(MC_16KCL, cl);
+		}
+
+		++num;
+		tail = o;
+		o = nexto;
+	}
+
+	if (!purged) {
+		tail->obj_next = m_cobjlist(class);
+		m_cobjlist(class) = list;
+		m_infree(class) += num;
+	} else if (ref_list != NULL) {
+		mcache_free_ext(ref_cache, ref_list);
+	}
+
+	return (num);
+}
+
+/*
+ * Common allocator for composite objects called by the CPU cache layer
+ * during an allocation request whenever there is no available element in
+ * the bucket layer.  It returns one or more composite elements from the
+ * appropriate global freelist.  If the freelist is empty, it will attempt
+ * to obtain the rudimentary objects from their caches and construct them
+ * into composite mbuf + cluster objects.
+ */
+static unsigned int
+mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed,
+    int wait)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	mcache_t *cp = NULL;
+	unsigned int num = 0, cnum = 0, want = needed;
+	mcache_obj_t *ref_list = NULL;
+	mcache_obj_t *mp_list = NULL;
+	mcache_obj_t *clp_list = NULL;
+	mcache_obj_t **list;
+	struct ext_ref *rfa;
+	struct mbuf *m;
+	void *cl;
+
+	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+	ASSERT(needed > 0);
+
+	VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+
+	/* There should not be any slab for this class */
+	VERIFY(m_slab_cnt(class) == 0 &&
+	    m_slablist(class).tqh_first == NULL &&
+	    m_slablist(class).tqh_last == NULL);
+
+	lck_mtx_lock(mbuf_mlock);
+
+	/* Try using the freelist first */
+	num = cslab_alloc(class, plist, needed);
+	list = *plist;
+	if (num == needed) {
+		m_alloc_cnt(class) += num;
+		lck_mtx_unlock(mbuf_mlock);
+		return (needed);
+	}
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	/*
+	 * We could not satisfy the request using the freelist alone;
+	 * allocate from the appropriate rudimentary caches and use
+	 * whatever we can get to construct the composite objects.
+	 */
+	needed -= num;
+
+	/*
+	 * Mark these allocation requests as coming from a composite cache.
+	 * Also, if the caller is willing to be blocked, mark the request
+	 * with MCR_FAILOK such that we don't end up sleeping at the mbuf
+	 * slab layer waiting for the individual object when one or more
+	 * of the already-constructed composite objects are available.
+	 */
+	wait |= MCR_COMP;
+	if (!(wait & MCR_NOSLEEP))
+		wait |= MCR_FAILOK;
+
+	needed = mcache_alloc_ext(m_cache(MC_MBUF), &mp_list, needed, wait);
+	if (needed == 0) {
+		ASSERT(mp_list == NULL);
+		goto fail;
+	}
+	if (class == MC_MBUF_CL)
+		cp = m_cache(MC_CL);
+	else if (class == MC_MBUF_BIGCL)
+		cp = m_cache(MC_BIGCL);
+	else
+		cp = m_cache(MC_16KCL);
+	needed = mcache_alloc_ext(cp, &clp_list, needed, wait);
+	if (needed == 0) {
+		ASSERT(clp_list == NULL);
+		goto fail;
+	}
+	needed = mcache_alloc_ext(ref_cache, &ref_list, needed, wait);
+	if (needed == 0) {
+		ASSERT(ref_list == NULL);
+		goto fail;
+	}
+
+	/*
+	 * By this time "needed" is MIN(mbuf, cluster, ref).  Any left
+	 * overs will get freed accordingly before we return to caller.
+	 */
+	for (cnum = 0; cnum < needed; cnum++) {
+		struct mbuf *ms;
+
+		m = ms = (struct mbuf *)mp_list;
+		mp_list = mp_list->obj_next;
+
+		cl = clp_list;
+		clp_list = clp_list->obj_next;
+		((mcache_obj_t *)cl)->obj_next = NULL;
+
+		rfa = (struct ext_ref *)ref_list;
+		ref_list = ref_list->obj_next;
+		((mcache_obj_t *)rfa)->obj_next = NULL;
+
+		/*
+		 * If auditing is enabled, construct the shadow mbuf
+		 * in the audit structure instead of in the actual one.
+		 * mbuf_cslab_audit() will take care of restoring the
+		 * contents after the integrity check.
+		 */
+		if (mclaudit != NULL) {
+			mcache_audit_t *mca, *cl_mca;
+			size_t size;
+
+			lck_mtx_lock(mbuf_mlock);
+			mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+			ms = ((struct mbuf *)mca->mca_contents);
+			cl_mca = mcl_audit_buf2mca(MC_CL, (mcache_obj_t *)cl);
+
+			/*
+			 * Pair them up.  Note that this is done at the time
+			 * the mbuf+cluster objects are constructed.  This
+			 * information should be treated as "best effort"
+			 * debugging hint since more than one mbufs can refer
+			 * to a cluster.  In that case, the cluster might not
+			 * be freed along with the mbuf it was paired with.
+			 */
+			mca->mca_uptr = cl_mca;
+			cl_mca->mca_uptr = mca;
+
+			ASSERT(mca->mca_uflags & MB_SCVALID);
+			ASSERT(!(cl_mca->mca_uflags & MB_SCVALID));
+			lck_mtx_unlock(mbuf_mlock);
+
+			/* Technically, they are in the freelist */
+			mcache_set_pattern(MCACHE_FREE_PATTERN, m,
+			    m_maxsize(MC_MBUF));
+			if (class == MC_MBUF_CL)
+				size = m_maxsize(MC_CL);
+			else if (class == MC_MBUF_BIGCL)
+				size = m_maxsize(MC_BIGCL);
+			else
+				size = m_maxsize(MC_16KCL);
+			mcache_set_pattern(MCACHE_FREE_PATTERN, cl, size);
+		}
+
+		MBUF_INIT(ms, 0, MT_FREE);
+		if (class == MC_MBUF_16KCL) {
+			MBUF_16KCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+		} else if (class == MC_MBUF_BIGCL) {
+			MBUF_BIGCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+		} else {
+			MBUF_CL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+		}
+		VERIFY(ms->m_flags == M_EXT);
+		VERIFY(MEXT_RFA(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+
+		*list = (mcache_obj_t *)m;
+		(*list)->obj_next = NULL;
+		list = *plist = &(*list)->obj_next;
+	}
+
+fail:
+	/*
+	 * Free up what's left of the above.
+	 */
+	if (mp_list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF), mp_list);
+	if (clp_list != NULL)
+		mcache_free_ext(cp, clp_list);
+	if (ref_list != NULL)
+		mcache_free_ext(ref_cache, ref_list);
+
+	lck_mtx_lock(mbuf_mlock);
+	if (num > 0 || cnum > 0) {
+		m_total(class) += cnum;
+		VERIFY(m_total(class) <= m_maxlimit(class));
+		m_alloc_cnt(class) += num + cnum;
+	}
+	if ((num + cnum) < want)
+		m_fail_cnt(class) += (want - (num + cnum));
+	lck_mtx_unlock(mbuf_mlock);
+
+	return (num + cnum);
+}
+
+/*
+ * Common de-allocator for composite objects called by the CPU cache
+ * layer when one or more elements need to be returned to the appropriate
+ * global freelist.
+ */
+static void
+mbuf_cslab_free(void *arg, mcache_obj_t *list, int purged)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	unsigned int num;
+	int w;
+
+	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+
+	lck_mtx_lock(mbuf_mlock);
+
+	num = cslab_free(class, list, purged);
+	m_free_cnt(class) += num;
+
+	if ((w = mb_waiters) > 0)
+		mb_waiters = 0;
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	if (w != 0)
+		wakeup(mb_waitchan);
+}
+
+/*
+ * Common auditor for composite objects called by the CPU cache layer
+ * during an allocation or free request.  For the former, this is called
+ * after the objects are obtained from either the bucket or slab layer
+ * and before they are returned to the caller.  For the latter, this is
+ * called immediately during free and before placing the objects into
+ * the bucket or slab layer.
+ */
+static void
+mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
+{
+	mbuf_class_t class = (mbuf_class_t)arg;
+	mcache_audit_t *mca;
+	struct mbuf *m, *ms;
+	mcl_slab_t *clsp, *nsp;
+	size_t size;
+	void *cl;
+
+	ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+
+	while ((m = ms = (struct mbuf *)list) != NULL) {
+		lck_mtx_lock(mbuf_mlock);
+		/* Do the mbuf sanity checks and record its transaction */
+		mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+		mcl_audit_mbuf(mca, m, TRUE, alloc);
+		mcache_buffer_log(mca, m, m_cache(class));
+		if (alloc)
+			mca->mca_uflags |= MB_COMP_INUSE;
+		else
+			mca->mca_uflags &= ~MB_COMP_INUSE;
+
+		/*
+		 * Use the shadow mbuf in the audit structure if we are
+		 * freeing, since the contents of the actual mbuf has been
+		 * pattern-filled by the above call to mcl_audit_mbuf().
+		 */
+		if (!alloc)
+			ms = (struct mbuf *)mca->mca_contents;
+
+		/* Do the cluster sanity checks and record its transaction */
+		cl = ms->m_ext.ext_buf;
+		clsp = slab_get(cl);
+		VERIFY(ms->m_flags == M_EXT && cl != NULL);
+		VERIFY(MEXT_RFA(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+		VERIFY(clsp->sl_refcnt == 1);
+		if (class == MC_MBUF_BIGCL) {
+			nsp = clsp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(nsp != NULL);
+			VERIFY(nsp->sl_refcnt == 1);
+		} else if (class == MC_MBUF_16KCL) {
+			int k;
+			for (nsp = clsp, k = 1;
+			    k < (M16KCLBYTES / MCLBYTES); k++) {
+				nsp = nsp->sl_next;
+				/* Next slab must already be present */
+				VERIFY(nsp != NULL);
+				VERIFY(nsp->sl_refcnt == 1);
+			}
+		}
+
+		mca = mcl_audit_buf2mca(MC_CL, cl);
+		if (class == MC_MBUF_CL)
+			size = m_maxsize(MC_CL);
+		else if (class == MC_MBUF_BIGCL)
+			size = m_maxsize(MC_BIGCL);
+		else
+			size = m_maxsize(MC_16KCL);
+		mcl_audit_cluster(mca, cl, size, alloc, FALSE);
+		mcache_buffer_log(mca, cl, m_cache(class));
+		if (alloc)
+			mca->mca_uflags |= MB_COMP_INUSE;
+		else
+			mca->mca_uflags &= ~MB_COMP_INUSE;
+		lck_mtx_unlock(mbuf_mlock);
+
+		list = list->obj_next;
+	}
+}
+
+/*
+ * Allocate some number of mbuf clusters and place on cluster freelist.
+ */
+static int
+m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
+{
+	int i;
+	vm_size_t size = 0;
+	int numpages = 0;
+	vm_offset_t page = 0;
+	mcache_audit_t *mca_list = NULL;
+	mcache_obj_t *con_list = NULL;
+	mcl_slab_t *sp;
+
+	VERIFY(bufsize == m_maxsize(MC_CL) ||
+	    bufsize == m_maxsize(MC_BIGCL) || bufsize == m_maxsize(MC_16KCL));
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	/*
+	 * Multiple threads may attempt to populate the cluster map one
+	 * after another.  Since we drop the lock below prior to acquiring
+	 * the physical page(s), our view of the cluster map may no longer
+	 * be accurate, and we could end up over-committing the pages beyond
+	 * the maximum allowed for each class.  To prevent it, this entire
+	 * operation (including the page mapping) is serialized.
+	 */
+	while (mb_clalloc_busy) {
+		mb_clalloc_waiters++;
+		(void) msleep(mb_clalloc_waitchan, mbuf_mlock,
+		    (PZERO-1), "m_clalloc", NULL);
+		lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+	}
+
+	/* We are busy now; tell everyone else to go away */
+	mb_clalloc_busy = TRUE;
+
+	/*
+	 * Honor the caller's wish to block or not block.  We have a way
+	 * to grow the pool asynchronously using the mbuf worker thread.
+	 */
+	i = m_howmany(num, bufsize);
+	if (i == 0 || (wait & M_DONTWAIT))
+		goto out;
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	size = round_page_32(i * bufsize);
+	page = kmem_mb_alloc(mb_map, size);
+
+	if (page == 0) {
+		if (bufsize <= m_maxsize(MC_BIGCL)) {
+			/* Try for 1 page if failed, only for 2KB/4KB request */
+			size = NBPG;
+			page = kmem_mb_alloc(mb_map, size);
+		}
+
+		if (page == 0) {
+			lck_mtx_lock(mbuf_mlock);
+			goto out;
+		}
+	}
+
+	VERIFY(IS_P2ALIGNED(page, NBPG));
+	numpages = size / NBPG;
+
+	/* If auditing is enabled, allocate the audit structures now */
+	if (mclaudit != NULL) {
+		int needed;
+
+		/*
+		 * Yes, I realize this is a waste of memory for clusters
+		 * that never get transformed into mbufs, as we may end
+		 * up with NMBPCL-1 unused audit structures per cluster.
+		 * But doing so tremendously simplifies the allocation
+		 * strategy, since at this point we are not holding the
+		 * mbuf lock and the caller is okay to be blocked.  For
+		 * the case of big clusters, we allocate one structure
+		 * for each as we never turn them into mbufs.
+		 */
+		if (bufsize == m_maxsize(MC_CL)) {
+			needed = numpages * 2 * NMBPCL;
+
+			i = mcache_alloc_ext(mcl_audit_con_cache,
+			    &con_list, needed, MCR_SLEEP);
+
+			VERIFY(con_list != NULL && i == needed);
+		} else if (bufsize == m_maxsize(MC_BIGCL)) {
+			needed = numpages;
+		} else {
+			needed = numpages / (M16KCLBYTES / NBPG);
+		}
+
+		i = mcache_alloc_ext(mcache_audit_cache,
+		    (mcache_obj_t **)&mca_list, needed, MCR_SLEEP);
+
+		VERIFY(mca_list != NULL && i == needed);
+	}
+
+	lck_mtx_lock(mbuf_mlock);
+
+	for (i = 0; i < numpages; i++, page += NBPG) {
+		ppnum_t offset = ((char *)page - (char *)mbutl) / NBPG;
+		ppnum_t new_page = pmap_find_phys(kernel_pmap,
+		    (vm_address_t)page);
+
+		/*
+		 * In the case of no mapper being available the following
+		 * code noops and returns the input page; if there is a
+		 * mapper the appropriate I/O page is returned.
+		 */
+		new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
+		mcl_paddr[offset] = new_page << PGSHIFT;
+
+		/* Pattern-fill this fresh page */
+		if (mclaudit != NULL)
+			mcache_set_pattern(MCACHE_FREE_PATTERN,
+			    (caddr_t)page, NBPG);
+
+		if (bufsize == m_maxsize(MC_CL)) {
+			union mcluster *mcl = (union mcluster *)page;
+
+			/* 1st cluster in the page */
+			sp = slab_get(mcl);
+			if (mclaudit != NULL)
+				mcl_audit_init(mcl, &mca_list, &con_list,
+				    AUDIT_CONTENTS_SIZE, NMBPCL);
+
+			VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+			slab_init(sp, MC_CL, SLF_MAPPED,
+			    mcl, mcl, bufsize, 0, 1);
+
+			/* Insert this slab */
+			slab_insert(sp, MC_CL);
+
+			/* Update stats now since slab_get() drops the lock */
+			mbstat.m_clfree = ++m_infree(MC_CL) +
+			    m_infree(MC_MBUF_CL);
+			mbstat.m_clusters = ++m_total(MC_CL);
+			VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
+
+			/* 2nd cluster in the page */
+			sp = slab_get(++mcl);
+			if (mclaudit != NULL)
+				mcl_audit_init(mcl, &mca_list, &con_list,
+				    AUDIT_CONTENTS_SIZE, NMBPCL);
+
+			VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+			slab_init(sp, MC_CL, SLF_MAPPED,
+			    mcl, mcl, bufsize, 0, 1);
+
+			/* Insert this slab */
+			slab_insert(sp, MC_CL);
+
+			/* Update stats now since slab_get() drops the lock */
+			mbstat.m_clfree = ++m_infree(MC_CL) +
+			    m_infree(MC_MBUF_CL);
+			mbstat.m_clusters = ++m_total(MC_CL);
+			VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
+		} else if (bufsize == m_maxsize(MC_BIGCL)) {
+			union mbigcluster *mbc = (union mbigcluster *)page;
+			mcl_slab_t *nsp;
+
+			/* One for the entire page */
+			sp = slab_get(mbc);
+			if (mclaudit != NULL)
+				mcl_audit_init(mbc, &mca_list, NULL, 0, 1);
+
+			VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+			slab_init(sp, MC_BIGCL, SLF_MAPPED,
+			    mbc, mbc, bufsize, 0, 1);
+
+			/* 2nd cluster's slab is part of the previous one */
+			nsp = slab_get(((union mcluster *)page) + 1);
+			slab_init(nsp, MC_BIGCL, SLF_MAPPED | SLF_PARTIAL,
+			    mbc, NULL, 0, 0, 0);
+
+			/* Insert this slab */
+			slab_insert(sp, MC_BIGCL);
+
+			/* Update stats now since slab_get() drops the lock */
+			mbstat.m_bigclfree = ++m_infree(MC_BIGCL) +
+			    m_infree(MC_MBUF_BIGCL);
+			mbstat.m_bigclusters = ++m_total(MC_BIGCL);
+			VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL));
+		} else if ((i % (M16KCLBYTES / NBPG)) == 0) {
+			union m16kcluster *m16kcl = (union m16kcluster *)page;
+			mcl_slab_t *nsp;
+			int k;
+
+			VERIFY(njcl > 0);
+			/* One for the entire 16KB */
+			sp = slab_get(m16kcl);
+			if (mclaudit != NULL)
+				mcl_audit_init(m16kcl, &mca_list, NULL, 0, 1);
+
+			VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+			slab_init(sp, MC_16KCL, SLF_MAPPED,
+			    m16kcl, m16kcl, bufsize, 0, 1);
+
+			/* 2nd-8th cluster's slab is part of the first one */
+			for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) {
+				nsp = slab_get(((union mcluster *)page) + k);
+				VERIFY(nsp->sl_refcnt == 0 &&
+				    nsp->sl_flags == 0);
+				slab_init(nsp, MC_16KCL,
+				    SLF_MAPPED | SLF_PARTIAL,
+				    m16kcl, NULL, 0, 0, 0);
+			}
+
+			/* Insert this slab */
+			slab_insert(sp, MC_16KCL);
+
+			/* Update stats now since slab_get() drops the lock */
+			m_infree(MC_16KCL)++;
+			m_total(MC_16KCL)++;
+			VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
+		}
+	}
+	VERIFY(mca_list == NULL && con_list == NULL);
+
+	/* We're done; let others enter */
+	mb_clalloc_busy = FALSE;
+	if (mb_clalloc_waiters > 0) {
+		mb_clalloc_waiters = 0;
+		wakeup(mb_clalloc_waitchan);
+	}
+
+	if (bufsize == m_maxsize(MC_CL))
+		return (numpages << 1);
+	else if (bufsize == m_maxsize(MC_BIGCL))
+		return (numpages);
+
+	VERIFY(bufsize == m_maxsize(MC_16KCL));
+	return (numpages / (M16KCLBYTES / NBPG));
+
+out:
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	/* We're done; let others enter */
+	mb_clalloc_busy = FALSE;
+	if (mb_clalloc_waiters > 0) {
+		mb_clalloc_waiters = 0;
+		wakeup(mb_clalloc_waitchan);
+	}
+
+	/*
+	 * When non-blocking we kick a thread if we have to grow the
+	 * pool or if the number of free clusters is less than requested.
+	 */
+	if (bufsize == m_maxsize(MC_CL)) {
+		if (i > 0) {
+			/*
+			 * Remember total number of clusters needed
+			 * at this time.
+			 */
+			i += m_total(MC_CL);
+			if (i > mbuf_expand_mcl) {
+				mbuf_expand_mcl = i;
+				if (mbuf_worker_ready)
+					wakeup((caddr_t)&mbuf_worker_run);
+			}
+		}
+
+		if (m_infree(MC_CL) >= num)
+			return (1);
+	} else if (bufsize == m_maxsize(MC_BIGCL)) {
+		if (i > 0) {
+			/*
+			 * Remember total number of 4KB clusters needed
+			 * at this time.
+			 */
+			i += m_total(MC_BIGCL);
+			if (i > mbuf_expand_big) {
+				mbuf_expand_big = i;
+				if (mbuf_worker_ready)
+					wakeup((caddr_t)&mbuf_worker_run);
+			}
+		}
+
+		if (m_infree(MC_BIGCL) >= num)
+			return (1);
+	} else {
+		if (i > 0) {
+			/*
+			 * Remember total number of 16KB clusters needed
+			 * at this time.
+			 */
+			i += m_total(MC_16KCL);
+			if (i > mbuf_expand_16k) {
+				mbuf_expand_16k = i;
+				if (mbuf_worker_ready)
+					wakeup((caddr_t)&mbuf_worker_run);
+			}
+		}
+
+		if (m_infree(MC_16KCL) >= num)
+			return (1);
+	}
+	return (0);
+}
+
+/*
+ * Populate the global freelist of the corresponding buffer class.
+ */
+static int
+freelist_populate(mbuf_class_t class, unsigned int num, int wait)
+{
+	mcache_obj_t *o = NULL;
+	int i;
+
+	VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL ||
+	    class == MC_16KCL);
+
+#if CONFIG_MBUF_NOEXPAND
+	if ((mbstat.m_mbufs / NMBPCL) >= maxmbufcl) {
+#if DEBUG
+		static int printonce = 1;
+		if (printonce == 1) {
+			printonce = 0;
+			printf("m_expand failed, allocated %ld out of %d "
+			    "clusters\n", mbstat.m_mbufs / NMBPCL,
+			    nmbclusters);
+		}
+#endif /* DEBUG */
+		return (0);
+	}
+#endif /* CONFIG_MBUF_NOEXPAND */
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	switch (class) {
+	case MC_MBUF:
+	case MC_CL:
+		i = m_clalloc(num, wait, m_maxsize(MC_CL));
+
+		/* Respect the 2K clusters minimum limit */
+		if (m_total(MC_CL) == m_maxlimit(MC_CL) &&
+		    m_infree(MC_CL) <= m_minlimit(MC_CL)) {
+			if (class != MC_CL || (wait & MCR_COMP))
+				return (0);
+		}
+		if (class == MC_CL)
+			return (i != 0);
+		break;
+
+	case MC_BIGCL:
+	case MC_16KCL:
+		return (m_clalloc(num, wait, m_maxsize(class)) != 0);
+		/* NOTREACHED */
+
+	default:
+		VERIFY(0);
+		/* NOTREACHED */
+	}
+
+	/* Steal a cluster and cut it up to create NMBPCL mbufs */
+	if ((o = slab_alloc(MC_CL, wait)) != NULL) {
+		struct mbuf *m = (struct mbuf *)o;
+		mcache_audit_t *mca = NULL;
+		mcl_slab_t *sp = slab_get(o);
+
+		VERIFY(slab_is_detached(sp) &&
+		    (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
+
+		/* Make sure that the cluster is unmolested while in freelist */
+		if (mclaudit != NULL) {
+			mca = mcl_audit_buf2mca(MC_CL, o);
+			mcache_audit_free_verify(mca, o, 0, m_maxsize(MC_CL));
+		}
+
+		/* Reinitialize it as an mbuf slab */
+		slab_init(sp, MC_MBUF, sp->sl_flags, sp->sl_base, NULL,
+		    sp->sl_len, 0, NMBPCL);
+
+		VERIFY(m == (struct mbuf *)sp->sl_base);
+		VERIFY(sp->sl_head == NULL);
+
+		m_total(MC_MBUF) += NMBPCL;
+		mbstat.m_mbufs = m_total(MC_MBUF);
+		m_infree(MC_MBUF) += NMBPCL;
+		mtype_stat_add(MT_FREE, NMBPCL);
+
+		i = NMBPCL;
+		while (i--) {
+			/*
+			 * If auditing is enabled, construct the shadow mbuf
+			 * in the audit structure instead of the actual one.
+			 * mbuf_slab_audit() will take care of restoring the
+			 * contents after the integrity check.
+			 */
+			if (mclaudit != NULL) {
+				struct mbuf *ms;
+				mca = mcl_audit_buf2mca(MC_MBUF,
+				    (mcache_obj_t *)m);
+				ms = ((struct mbuf *)mca->mca_contents);
+				ms->m_type = MT_FREE;
+			} else {
+				m->m_type = MT_FREE;
+			}
+			m->m_next = sp->sl_head;
+			sp->sl_head = (void *)m++;
+		}
+
+		/* Insert it into the mbuf class's slab list */
+		slab_insert(sp, MC_MBUF);
+
+		if ((i = mb_waiters) > 0)
+			mb_waiters = 0;
+		if (i != 0)
+			wakeup(mb_waitchan);
+
+		return (1);
+	}
+
+	return (0);
+}
+
+/*
+ * (Inaccurately) check if it might be worth a trip back to the
+ * mcache layer due the availability of objects there.  We'll
+ * end up back here if there's nothing up there.
+ */
+static boolean_t
+mbuf_cached_above(mbuf_class_t class, int wait)
+{
+	switch (class) {
+	case MC_MBUF:
+		if (wait & MCR_COMP)
+			return (!mcache_bkt_isempty(m_cache(MC_MBUF_CL)) ||
+			    !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL)));
+		break;
+
+	case MC_CL:
+		if (wait & MCR_COMP)
+			return (!mcache_bkt_isempty(m_cache(MC_MBUF_CL)));
+		break;
+
+	case MC_BIGCL:
+		if (wait & MCR_COMP)
+			return (!mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL)));
+		break;
+
+	case MC_16KCL:
+		if (wait & MCR_COMP)
+			return (!mcache_bkt_isempty(m_cache(MC_MBUF_16KCL)));
+		break;
+
+	case MC_MBUF_CL:
+	case MC_MBUF_BIGCL:
+	case MC_MBUF_16KCL:
+		break;
+
+	default:
+		VERIFY(0);
+		/* NOTREACHED */
+	}
+
+	return (!mcache_bkt_isempty(m_cache(class)));
+}
+
+/*
+ * If possible, convert constructed objects to raw ones.
+ */
+static boolean_t
+mbuf_steal(mbuf_class_t class, unsigned int num)
+{
+	mcache_obj_t *top = NULL;
+	mcache_obj_t **list = &top;
+	unsigned int tot = 0;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	switch (class) {
+	case MC_MBUF:
+	case MC_CL:
+	case MC_BIGCL:
+	case MC_16KCL:
+		return (FALSE);
+
+	case MC_MBUF_CL:
+	case MC_MBUF_BIGCL:
+	case MC_MBUF_16KCL:
+		/* Get the required number of constructed objects if possible */
+		if (m_infree(class) > m_minlimit(class)) {
+			tot = cslab_alloc(class, &list,
+			    MIN(num, m_infree(class)));
+		}
+
+		/* And destroy them to get back the raw objects */
+		if (top != NULL)
+			(void) cslab_free(class, top, 1);
+		break;
+
+	default:
+		VERIFY(0);
+		/* NOTREACHED */
+	}
+
+	return (tot == num);
+}
+
+static void
+m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp)
+{
+	int m, bmap = 0;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
+	VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL));
+	VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
+
+	/*
+	 * This logic can be made smarter; for now, simply mark
+	 * all other related classes as potential victims.
+	 */
+	switch (class) {
+	case MC_MBUF:
+		m_wantpurge(MC_CL)++;
+		m_wantpurge(MC_MBUF_CL)++;
+		m_wantpurge(MC_MBUF_BIGCL)++;
+		break;
+
+	case MC_CL:
+		m_wantpurge(MC_MBUF)++;
+		if (!comp)
+			m_wantpurge(MC_MBUF_CL)++;
+		break;
+
+	case MC_BIGCL:
+		if (!comp)
+			m_wantpurge(MC_MBUF_BIGCL)++;
+		break;
+
+	case MC_16KCL:
+		if (!comp)
+			m_wantpurge(MC_MBUF_16KCL)++;
+		break;
+
+	default:
+		VERIFY(0);
+		/* NOTREACHED */
+	}
+
+	/*
+	 * Run through each marked class and check if we really need to
+	 * purge (and therefore temporarily disable) the per-CPU caches
+	 * layer used by the class.  If so, remember the classes since
+	 * we are going to drop the lock below prior to purging.
+	 */
+	for (m = 0; m < NELEM(mbuf_table); m++) {
+		if (m_wantpurge(m) > 0) {
+			m_wantpurge(m) = 0;
+			/*
+			 * Try hard to steal the required number of objects
+			 * from the freelist of other mbuf classes.  Only
+			 * purge and disable the per-CPU caches layer when
+			 * we don't have enough; it's the last resort.
+			 */
+			if (!mbuf_steal(m, num))
+				bmap |= (1 << m);
+		}
+	}
+
+	lck_mtx_unlock(mbuf_mlock);
+
+	if (bmap != 0) {
+		/* drain is performed in pfslowtimo(), to avoid deadlocks */
+		do_reclaim = 1;
+
+		/* Sigh; we have no other choices but to ask mcache to purge */
+		for (m = 0; m < NELEM(mbuf_table); m++) {
+			if ((bmap & (1 << m)) &&
+			    mcache_purge_cache(m_cache(m))) {
+				lck_mtx_lock(mbuf_mlock);
+				m_purge_cnt(m)++;
+				mbstat.m_drain++;
+				lck_mtx_unlock(mbuf_mlock);
+			}
+		}
+	} else {
+		/*
+		 * Request mcache to reap extra elements from all of its caches;
+		 * note that all reaps are serialized and happen only at a fixed
+		 * interval.
+		 */
+		mcache_reap();
+	}
+	lck_mtx_lock(mbuf_mlock);
+}
+
+static inline struct mbuf *
+m_get_common(int wait, short type, int hdr)
+{
+	struct mbuf *m;
+	int mcflags = MSLEEPF(wait);
+
+	/* Is this due to a non-blocking retry?  If so, then try harder */
+	if (mcflags & MCR_NOSLEEP)
+		mcflags |= MCR_TRYHARD;
+
+	m = mcache_alloc(m_cache(MC_MBUF), mcflags);
+	if (m != NULL) {
+		MBUF_INIT(m, hdr, type);
+		mtype_stat_inc(type);
+		mtype_stat_dec(MT_FREE);
+#if CONFIG_MACF_NET
+		if (hdr && mac_init_mbuf(m, wait) != 0) {
+			m_free(m);
+			return (NULL);
+		}
+#endif /* MAC_NET */
+	}
+	return (m);
+}
+
+/*
+ * Space allocation routines; these are also available as macros
+ * for critical paths.
+ */
+#define	_M_GET(wait, type)	m_get_common(wait, type, 0)
+#define	_M_GETHDR(wait, type)	m_get_common(wait, type, 1)
+#define	_M_RETRY(wait, type)	_M_GET(wait, type)
+#define	_M_RETRYHDR(wait, type)	_M_GETHDR(wait, type)
+#define	_MGET(m, how, type)	((m) = _M_GET(how, type))
+#define	_MGETHDR(m, how, type)	((m) = _M_GETHDR(how, type))
+
+struct mbuf *
+m_get(int wait, int type)
+{
+	return (_M_GET(wait, type));
+}
+
+struct mbuf *
+m_gethdr(int wait, int type)
+{
+	return (_M_GETHDR(wait, type));
+}
+
+struct mbuf *
+m_retry(int wait, int type)
+{
+	return (_M_RETRY(wait, type));
+}
+
+struct mbuf *
+m_retryhdr(int wait, int type)
+{
+	return (_M_RETRYHDR(wait, type));
+}
+
+struct mbuf *
+m_getclr(int wait, int type)
+{
+	struct mbuf *m;
+
+	_MGET(m, wait, type);
+	if (m != NULL)
+		bzero(MTOD(m, caddr_t), MLEN);
+	return (m);
+}
+
+struct mbuf *
+m_free(struct mbuf *m)
+{
+	struct mbuf *n = m->m_next;
+
+	if (m->m_type == MT_FREE)
+		panic("m_free: freeing an already freed mbuf");
+
+	/* Free the aux data and tags if there is any */
+	if (m->m_flags & M_PKTHDR) {
+		m_tag_delete_chain(m, NULL);
+	}
+
+	if (m->m_flags & M_EXT) {
+		u_int32_t refcnt;
+		u_int32_t flags;
+
+		refcnt = m_decref(m);
+		flags = MEXT_FLAGS(m);
+		if (refcnt == 0 && flags == 0) {
+			if (m->m_ext.ext_free == NULL) {
+				mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
+			} else if (m->m_ext.ext_free == m_bigfree) {
+				mcache_free(m_cache(MC_BIGCL),
+				    m->m_ext.ext_buf);
+			} else if (m->m_ext.ext_free == m_16kfree) {
+				mcache_free(m_cache(MC_16KCL),
+				    m->m_ext.ext_buf);
+			} else {
+				(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
+				    m->m_ext.ext_size, m->m_ext.ext_arg);
+			}
+			mcache_free(ref_cache, MEXT_RFA(m));
+			MEXT_RFA(m) = NULL;
+		} else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) {
+			VERIFY(m->m_type != MT_FREE);
+
+			mtype_stat_dec(m->m_type);
+			mtype_stat_inc(MT_FREE);
+
+			m->m_type = MT_FREE;
+			m->m_flags = M_EXT;
+			m->m_len = 0;
+			m->m_next = m->m_nextpkt = NULL;
+
+			/* "Free" into the intermediate cache */
+			if (m->m_ext.ext_free == NULL) {
+				mcache_free(m_cache(MC_MBUF_CL), m);
+			} else if (m->m_ext.ext_free == m_bigfree) {
+				mcache_free(m_cache(MC_MBUF_BIGCL), m);
+			} else {
+				VERIFY(m->m_ext.ext_free == m_16kfree);
+				mcache_free(m_cache(MC_MBUF_16KCL), m);
+			}
+			return (n);
+		}
+	}
+
+	if (m->m_type != MT_FREE) {
+		mtype_stat_dec(m->m_type);
+		mtype_stat_inc(MT_FREE);
+	}
+
+	m->m_type = MT_FREE;
+	m->m_flags = m->m_len = 0;
+	m->m_next = m->m_nextpkt = NULL;
+
+	mcache_free(m_cache(MC_MBUF), m);
+
+	return (n);
+}
+
+__private_extern__ struct mbuf *
+m_clattach(struct mbuf *m, int type, caddr_t extbuf,
+    void (*extfree)(caddr_t, u_int, caddr_t), u_int extsize, caddr_t extarg,
+    int wait)
+{
+	struct ext_ref *rfa = NULL;
+
+	if (m == NULL && (m = _M_GETHDR(wait, type)) == NULL)
+		return (NULL);
+
+	if (m->m_flags & M_EXT) {
+		u_int32_t refcnt;
+		u_int32_t flags;
+
+		refcnt = m_decref(m);
+		flags = MEXT_FLAGS(m);
+		if (refcnt == 0 && flags == 0) {
+			if (m->m_ext.ext_free == NULL) {
+				mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
+			} else if (m->m_ext.ext_free == m_bigfree) {
+				mcache_free(m_cache(MC_BIGCL),
+				    m->m_ext.ext_buf);
+			} else if (m->m_ext.ext_free == m_16kfree) {
+				mcache_free(m_cache(MC_16KCL),
+				    m->m_ext.ext_buf);
+			} else {
+				(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
+				    m->m_ext.ext_size, m->m_ext.ext_arg);
+			}
+			/* Re-use the reference structure */
+			rfa = MEXT_RFA(m);
+		} else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) {
+			VERIFY(m->m_type != MT_FREE);
+
+			mtype_stat_dec(m->m_type);
+			mtype_stat_inc(MT_FREE);
+
+			m->m_type = MT_FREE;
+			m->m_flags = M_EXT;
+			m->m_len = 0;
+			m->m_next = m->m_nextpkt = NULL;
+			/* "Free" into the intermediate cache */
+			if (m->m_ext.ext_free == NULL) {
+				mcache_free(m_cache(MC_MBUF_CL), m);
+			} else if (m->m_ext.ext_free == m_bigfree) {
+				mcache_free(m_cache(MC_MBUF_BIGCL), m);
+			} else {
+				VERIFY(m->m_ext.ext_free == m_16kfree);
+				mcache_free(m_cache(MC_MBUF_16KCL), m);
+			}
+			/*
+			 * Allocate a new mbuf, since we didn't divorce
+			 * the composite mbuf + cluster pair above.
+			 */
+			if ((m = _M_GETHDR(wait, type)) == NULL)
+				return (NULL);
+		}
+	}
+
+	if (rfa == NULL &&
+	    (rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
+		m_free(m);
+		return (NULL);
+	}
+
+	MEXT_INIT(m, extbuf, extsize, extfree, extarg, rfa, 1, 0);
+
+	return (m);
+}
+
+/* m_mclget() add an mbuf cluster to a normal mbuf */
+struct mbuf *
+m_mclget(struct mbuf *m, int wait)
+{
+	struct ext_ref *rfa;
+
+	if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
+		return (m);
+
+	m->m_ext.ext_buf = m_mclalloc(wait);
+	if (m->m_ext.ext_buf != NULL) {
+		MBUF_CL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
+	} else {
+		mcache_free(ref_cache, rfa);
+	}
+	return (m);
+}
+
+/* Allocate an mbuf cluster */
+caddr_t
+m_mclalloc(int wait)
+{
+	int mcflags = MSLEEPF(wait);
+
+	/* Is this due to a non-blocking retry?  If so, then try harder */
+	if (mcflags & MCR_NOSLEEP)
+		mcflags |= MCR_TRYHARD;
+
+	return (mcache_alloc(m_cache(MC_CL), mcflags));
+}
+
+/* Free an mbuf cluster */
+void
+m_mclfree(caddr_t p)
+{
+	mcache_free(m_cache(MC_CL), p);
+}
+
+/*
+ * mcl_hasreference() checks if a cluster of an mbuf is referenced by
+ * another mbuf
+ */
+int
+m_mclhasreference(struct mbuf *m)
+{
+	if (!(m->m_flags & M_EXT))
+		return (0);
 
-/* m_mclalloc() allocate an mbuf cluster */
-caddr_t
-m_mclalloc(
-    int nowait)
-{
-    caddr_t p;
-        
-	(void)m_clalloc(1, nowait, MCLBYTES, 0);
-	if ((p = (caddr_t)mclfree)) {
-		++mclrefcnt[mtocl(p)];
-		mbstat.m_clfree--;
-		mclfree = ((union mcluster *)p)->mcl_next;
-	} else {
-		mbstat.m_drops++;
-	}
-	MBUF_UNLOCK();
-        
-    return p;
+	ASSERT(MEXT_RFA(m) != NULL);
+
+	return (MEXT_REF(m) > 1);
 }
 
-/* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
- * freeing the cluster if the reference count has reached 0. */
-void
-m_mclfree(
-    caddr_t p)
+__private_extern__ caddr_t
+m_bigalloc(int wait)
 {
-	MBUF_LOCK();
+	int mcflags = MSLEEPF(wait);
 
-	m_range_check(p);
+	/* Is this due to a non-blocking retry?  If so, then try harder */
+	if (mcflags & MCR_NOSLEEP)
+		mcflags |= MCR_TRYHARD;
 
-	if (--mclrefcnt[mtocl(p)] == 0) {
-		((union mcluster *)(p))->mcl_next = mclfree;
-		mclfree = (union mcluster *)(p);
-		mbstat.m_clfree++;
-	}
-	MBUF_UNLOCK();
+	return (mcache_alloc(m_cache(MC_BIGCL), mcflags));
 }
 
-/* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
-int
-m_mclhasreference(
-    struct mbuf *m)
+__private_extern__ void
+m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
 {
-    return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
+	mcache_free(m_cache(MC_BIGCL), p);
 }
 
-__private_extern__ caddr_t
-m_bigalloc(int nowait)
-{
-    caddr_t p;
-        
-	(void)m_clalloc(1, nowait, NBPG, 0);
-	if ((p = (caddr_t)mbigfree)) {
-		if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
-			panic("m_bigalloc mclrefcnt %x mismatch %d != %d",  
-				p, mclrefcnt[mtocl(p)],  mclrefcnt[mtocl(p) + 1]);
-		if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
-			panic("m_bigalloc mclrefcnt %x not null %d != %d",  
-				p, mclrefcnt[mtocl(p)],  mclrefcnt[mtocl(p) + 1]);
-		++mclrefcnt[mtocl(p)];
-		++mclrefcnt[mtocl(p) + 1];
-		mbstat.m_bigclfree--;
-		mbigfree = ((union mbigcluster *)p)->mbc_next;
+/* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
+__private_extern__ struct mbuf *
+m_mbigget(struct mbuf *m, int wait)
+{
+	struct ext_ref *rfa;
+
+	if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
+		return (m);
+
+	m->m_ext.ext_buf =  m_bigalloc(wait);
+	if (m->m_ext.ext_buf != NULL) {
+		MBUF_BIGCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
 	} else {
-		mbstat.m_drops++;
+		mcache_free(ref_cache, rfa);
 	}
-	MBUF_UNLOCK();
-	return p;
+	return (m);
+}
+
+__private_extern__ caddr_t
+m_16kalloc(int wait)
+{
+	int mcflags = MSLEEPF(wait);
+
+	/* Is this due to a non-blocking retry?  If so, then try harder */
+	if (mcflags & MCR_NOSLEEP)
+		mcflags |= MCR_TRYHARD;
+
+	return (mcache_alloc(m_cache(MC_16KCL), mcflags));
 }
 
 __private_extern__ void
-m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
+m_16kfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
 {
-	m_range_check(p);
-	
-	if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
-		panic("m_bigfree mclrefcnt %x mismatch %d != %d",  
-			p, mclrefcnt[mtocl(p)],  mclrefcnt[mtocl(p) + 1]);
-	--mclrefcnt[mtocl(p)];
-	--mclrefcnt[mtocl(p) + 1];
-	if (mclrefcnt[mtocl(p)] == 0) {
-		((union mbigcluster *)(p))->mbc_next = mbigfree;
-		mbigfree = (union mbigcluster *)(p);
-		mbstat.m_bigclfree++;
-	}
+	mcache_free(m_cache(MC_16KCL), p);
 }
 
-/* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
+/* m_m16kget() add a 16KB mbuf cluster to a normal mbuf */
 __private_extern__ struct mbuf *
-m_mbigget(struct mbuf *m, int nowait)
+m_m16kget(struct mbuf *m, int wait)
 {
-	m->m_ext.ext_buf =  m_bigalloc(nowait);
-	if (m->m_ext.ext_buf) {
-		m->m_data = m->m_ext.ext_buf;
-		m->m_flags |= M_EXT;
-		m->m_ext.ext_size = NBPG;
-		m->m_ext.ext_free = m_bigfree;
-		m->m_ext.ext_arg = 0;
-		m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
-			&m->m_ext.ext_refs;
+	struct ext_ref *rfa;
+
+	if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL)
+		return (m);
+
+	m->m_ext.ext_buf =  m_16kalloc(wait);
+	if (m->m_ext.ext_buf != NULL) {
+		MBUF_16KCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
+	} else {
+		mcache_free(ref_cache, rfa);
 	}
-        
-    return m;
+	return (m);
 }
 
-
 /* */
 void
-m_copy_pkthdr(
-    struct mbuf *to, 
-    struct mbuf *from)
+m_copy_pkthdr(struct mbuf *to, struct mbuf *from)
 {
-	to->m_pkthdr = from->m_pkthdr;
-	from->m_pkthdr.aux = (struct mbuf *)NULL;
-	SLIST_INIT(&from->m_pkthdr.tags);       /* purge tags from src */
+#if CONFIG_MACF_NET
+	/* We will be taking over the tags of 'to' */
+	if (to->m_flags & M_PKTHDR)
+		m_tag_delete_chain(to, NULL);
+#endif /* MAC_NET */
+	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
+	m_tag_init(from);			/* purge tags from src */
 	to->m_flags = from->m_flags & M_COPYFLAGS;
 	to->m_data = (to)->m_pktdat;
 }
 
-/*
- * "Move" mbuf pkthdr from "from" to "to".
- * "from" must have M_PKTHDR set, and "to" must be empty.
- */
-#ifndef __APPLE__
-void
-m_move_pkthdr(struct mbuf *to, struct mbuf *from)
-{
-        KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
-
-        to->m_flags = from->m_flags & M_COPYFLAGS;
-        to->m_data = to->m_pktdat;
-        to->m_pkthdr = from->m_pkthdr;          /* especially tags */
-        SLIST_INIT(&from->m_pkthdr.tags);       /* purge tags from src */
-        from->m_flags &= ~M_PKTHDR;
-}
-#endif
-
 /*
  * Duplicate "from"'s mbuf pkthdr in "to".
  * "from" must have M_PKTHDR set, and "to" must be empty.
@@ -813,353 +3023,501 @@ m_move_pkthdr(struct mbuf *to, struct mbuf *from)
 static int
 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
 {
-        to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
-        if ((to->m_flags & M_EXT) == 0)
-                to->m_data = to->m_pktdat;
-	if (to->m_pkthdr.aux != NULL)
-		m_freem(to->m_pkthdr.aux);
-        to->m_pkthdr = from->m_pkthdr;
-	to->m_pkthdr.aux = NULL;
-	(void) m_aux_copy(to, from);
-        SLIST_INIT(&to->m_pkthdr.tags);
-        return (m_tag_copy_chain(to, from, how));
+#if CONFIG_MACF_NET
+	if (to->m_flags & M_PKTHDR)
+		m_tag_delete_chain(to, NULL);
+#endif /* MAC_NET */
+	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
+	if ((to->m_flags & M_EXT) == 0)
+		to->m_data = to->m_pktdat;
+	to->m_pkthdr = from->m_pkthdr;
+	m_tag_init(to);
+	return (m_tag_copy_chain(to, from, how));
 }
 
 /*
- * return a list of mbuf hdrs that point to clusters...
- * try for num_needed, if wantall is not set, return whatever
- * number were available... set up the first num_with_pkthdrs
- * with mbuf hdrs configured as packet headers... these are
- * chained on the m_nextpkt field... any packets requested beyond
- * this are chained onto the last packet header's m_next field.
- * The size of the cluster is controlled by the paramter bufsize.
+ * Return a list of mbuf hdrs that point to clusters.  Try for num_needed;
+ * if wantall is not set, return whatever number were available.  Set up the
+ * first num_with_pkthdrs with mbuf hdrs configured as packet headers; these
+ * are chained on the m_nextpkt field.  Any packets requested beyond this
+ * are chained onto the last packet header's m_next field.  The size of
+ * the cluster is controlled by the parameter bufsize.
  */
 __private_extern__ struct mbuf *
-m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
+m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs,
+    int wait, int wantall, size_t bufsize)
 {
 	struct mbuf *m;
 	struct mbuf **np, *top;
-	unsigned int num, needed = *num_needed;
-	
-	if (bufsize != MCLBYTES && bufsize != NBPG)
-		return 0;
-	
+	unsigned int pnum, needed = *num_needed;
+	mcache_obj_t *mp_list = NULL;
+	int mcflags = MSLEEPF(wait);
+	u_int32_t flag;
+	struct ext_ref *rfa;
+	mcache_t *cp;
+	void *cl;
+
+	ASSERT(bufsize == m_maxsize(MC_CL) ||
+	    bufsize == m_maxsize(MC_BIGCL) ||
+	    bufsize == m_maxsize(MC_16KCL));
+
+	/*
+	 * Caller must first check for njcl because this
+	 * routine is internal and not exposed/used via KPI.
+	 */
+	VERIFY(bufsize != m_maxsize(MC_16KCL) || njcl > 0);
+
 	top = NULL;
 	np = &top;
-	
-	(void)m_clalloc(needed, how, bufsize, 0);		/* takes the MBUF_LOCK, but doesn't release it... */
-	
-	for (num = 0; num < needed; num++) {
-		m_range_check(mfree);
-		m_range_check(mclfree);
-		m_range_check(mbigfree);
-		
-		if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
-			/* mbuf + cluster are available */
-			m = mfree;
-			MCHECK(m);
-			mfree = m->m_next;
-			++mclrefcnt[mtocl(m)];
-			mbstat.m_mtypes[MT_FREE]--;
-			mbstat.m_mtypes[MT_DATA]++;
-			if (bufsize == NBPG) {
-				m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
-				++mclrefcnt[mtocl(m->m_ext.ext_buf)];
-				++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
-				mbstat.m_bigclfree--;
-				mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
-				m->m_ext.ext_free = m_bigfree;
-				m->m_ext.ext_size = NBPG;
-			} else {
-				m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
-				++mclrefcnt[mtocl(m->m_ext.ext_buf)];
-				mbstat.m_clfree--;
-				mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
-				m->m_ext.ext_free = 0;
-				m->m_ext.ext_size = MCLBYTES;
-			}
-			m->m_ext.ext_arg = 0;
-			m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
-			m->m_next = m->m_nextpkt = 0;
-			m->m_type = MT_DATA;
-			m->m_data = m->m_ext.ext_buf;
-			m->m_len = 0;
+	pnum = 0;
 
-			if (num_with_pkthdrs == 0)
-				m->m_flags = M_EXT;
-			else {
-				m->m_flags = M_PKTHDR | M_EXT;
-				_M_CLEAR_PKTHDR(m);
-			
-				num_with_pkthdrs--;
-			}
+	/*
+	 * The caller doesn't want all the requested buffers; only some.
+	 * Try hard to get what we can, but don't block.  This effectively
+	 * overrides MCR_SLEEP, since this thread will not go to sleep
+	 * if we can't get all the buffers.
+	 */
+	if (!wantall || (mcflags & MCR_NOSLEEP))
+		mcflags |= MCR_TRYHARD;
+
+	/* Allocate the composite mbuf + cluster elements from the cache */
+	if (bufsize == m_maxsize(MC_CL))
+		cp = m_cache(MC_MBUF_CL);
+	else if (bufsize == m_maxsize(MC_BIGCL))
+		cp = m_cache(MC_MBUF_BIGCL);
+	else
+		cp = m_cache(MC_MBUF_16KCL);
+	needed = mcache_alloc_ext(cp, &mp_list, needed, mcflags);
+
+	for (pnum = 0; pnum < needed; pnum++) {
+		m = (struct mbuf *)mp_list;
+		mp_list = mp_list->obj_next;
+
+		VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
+		cl = m->m_ext.ext_buf;
+		rfa = MEXT_RFA(m);
+
+		ASSERT(cl != NULL && rfa != NULL);
+		VERIFY(MBUF_IS_COMPOSITE(m));
+
+		flag = MEXT_FLAGS(m);
+
+		MBUF_INIT(m, num_with_pkthdrs, MT_DATA);
+		if (bufsize == m_maxsize(MC_16KCL)) {
+			MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
+		} else if (bufsize == m_maxsize(MC_BIGCL)) {
+			MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
 		} else {
-			MBUF_UNLOCK();
-			
-			if (num_with_pkthdrs == 0) {
-				MGET(m, how, MT_DATA );
-			} else {
-				MGETHDR(m, how, MT_DATA);
-			
-				num_with_pkthdrs--;
-			}
-			if (m == 0)
-				goto fail;
-			
-			if (bufsize == NBPG)
-				m = m_mbigget(m, how);
-			else
-				m = m_mclget(m, how);
-			if ((m->m_flags & M_EXT) == 0) {
+			MBUF_CL_INIT(m, cl, rfa, 1, flag);
+		}
+
+		if (num_with_pkthdrs > 0) {
+			--num_with_pkthdrs;
+#if CONFIG_MACF_NET
+			if (mac_mbuf_label_init(m, wait) != 0) {
 				m_free(m);
-				goto fail;
+				break;
 			}
-			MBUF_LOCK();
+#endif /* MAC_NET */
 		}
-		*np = m; 
-		
-		if (num_with_pkthdrs)
+
+		*np = m;
+		if (num_with_pkthdrs > 0)
 			np = &m->m_nextpkt;
 		else
 			np = &m->m_next;
 	}
-	MBUF_UNLOCK();
-	
-	*num_needed = num;
-	return (top);
-fail:
-	if (wantall && top) {
-		m_freem(top);
-		return 0;
+	ASSERT(pnum != *num_needed || mp_list == NULL);
+	if (mp_list != NULL)
+		mcache_free_ext(cp, mp_list);
+
+	if (pnum > 0) {
+		mtype_stat_add(MT_DATA, pnum);
+		mtype_stat_sub(MT_FREE, pnum);
+	}
+
+	if (wantall && (pnum != *num_needed)) {
+		if (top != NULL)
+			m_freem_list(top);
+		return (NULL);
 	}
-	return top;
-}
 
+	*num_needed = pnum;
+	return (top);
+}
 
 /*
- * Return list of mbuf linked by m_nextpkt
- * Try for num_needed, and if wantall is not set, return whatever
- * number were available
- * The size of each mbuf in the list is controlled by the parameter packetlen.
- * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in 
- * the chain is called a segment.
- * If maxsegments is not null and the value pointed to is not null, this specify 
- * the maximum number of segments for a chain of mbufs.
- * If maxsegments is zero or the value pointed to is zero the 
- * caller does not have any restriction on the number of segments. 
- * The actual  number of segments of a mbuf chain is return in the value pointed 
- * to by maxsegments.
- * When possible the allocation is done under a single lock.
+ * Return list of mbuf linked by m_nextpkt.  Try for numlist, and if
+ * wantall is not set, return whatever number were available.  The size of
+ * each mbuf in the list is controlled by the parameter packetlen.  Each
+ * mbuf of the list may have a chain of mbufs linked by m_next.  Each mbuf
+ * in the chain is called a segment.  If maxsegments is not null and the
+ * value pointed to is not null, this specify the maximum number of segments
+ * for a chain of mbufs.  If maxsegments is zero or the value pointed to
+ * is zero the caller does not have any restriction on the number of segments.
+ * The actual  number of segments of a mbuf chain is return in the value
+ * pointed to by maxsegments.
  */
-
 __private_extern__ struct mbuf *
-m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments, 
-			int how, int wantall, size_t wantsize)
+m_allocpacket_internal(unsigned int *numlist, size_t packetlen,
+    unsigned int *maxsegments, int wait, int wantall, size_t wantsize)
 {
-	struct mbuf **np, *top;
-	size_t bufsize;
-	unsigned int num;
-	unsigned int numchunks = 0;
+	struct mbuf **np, *top, *first = NULL;
+	size_t bufsize, r_bufsize;
+	unsigned int num = 0;
+	unsigned int nsegs = 0;
+	unsigned int needed, resid;
+	int mcflags = MSLEEPF(wait);
+	mcache_obj_t *mp_list = NULL, *rmp_list = NULL;
+	mcache_t *cp = NULL, *rcp = NULL;
+
+	if (*numlist == 0)
+		return (NULL);
 
 	top = NULL;
 	np = &top;
-	
+
 	if (wantsize == 0) {
-		if (packetlen <= MINCLSIZE)
+		if (packetlen <= MINCLSIZE) {
 			bufsize = packetlen;
-		else if (packetlen > MCLBYTES)
-			bufsize = NBPG;
-		else
-			bufsize = MCLBYTES;
-	} else if (wantsize == MCLBYTES || wantsize == NBPG)
+		} else if (packetlen > m_maxsize(MC_CL)) {
+			/* Use 4KB if jumbo cluster pool isn't available */
+			if (packetlen <= m_maxsize(MC_BIGCL) || njcl == 0)
+				bufsize = m_maxsize(MC_BIGCL);
+			else
+				bufsize = m_maxsize(MC_16KCL);
+		} else {
+			bufsize = m_maxsize(MC_CL);
+		}
+	} else if (wantsize == m_maxsize(MC_CL) ||
+	    wantsize == m_maxsize(MC_BIGCL) ||
+	    (wantsize == m_maxsize(MC_16KCL) && njcl > 0)) {
 		bufsize = wantsize;
-	else
-		return 0;
+	} else {
+		return (NULL);
+	}
 
 	if (bufsize <= MHLEN) {
-			numchunks = 1;
+		nsegs = 1;
 	} else if (bufsize <= MINCLSIZE) {
 		if (maxsegments != NULL && *maxsegments == 1) {
-			bufsize = MCLBYTES;
-			numchunks = 1;
+			bufsize = m_maxsize(MC_CL);
+			nsegs = 1;
 		} else {
-			numchunks = 2;
+			nsegs = 2;
 		}
-	} else if (bufsize == NBPG) {
-		numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
+	} else if (bufsize == m_maxsize(MC_16KCL)) {
+		VERIFY(njcl > 0);
+		nsegs = ((packetlen - 1) >> (PGSHIFT + 2)) + 1;
+	} else if (bufsize == m_maxsize(MC_BIGCL)) {
+		nsegs = ((packetlen - 1) >> PGSHIFT) + 1;
 	} else {
-		numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
+		nsegs = ((packetlen - 1) >> MCLSHIFT) + 1;
 	}
 	if (maxsegments != NULL) {
-	 	if (*maxsegments && numchunks > *maxsegments) {
-			*maxsegments = numchunks;
-			return 0;
+		if (*maxsegments && nsegs > *maxsegments) {
+			*maxsegments = nsegs;
+			return (NULL);
 		}
-		*maxsegments = numchunks;
+		*maxsegments = nsegs;
 	}
-	/* m_clalloc takes the MBUF_LOCK, but do not release it */	
-	(void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
-	for (num = 0; num < *num_needed; num++) {
-		struct mbuf **nm, *pkt = 0;
-		size_t len;
 
-		nm = &pkt;
+	/*
+	 * The caller doesn't want all the requested buffers; only some.
+	 * Try hard to get what we can, but don't block.  This effectively
+	 * overrides MCR_SLEEP, since this thread will not go to sleep
+	 * if we can't get all the buffers.
+	 */
+	if (!wantall || (mcflags & MCR_NOSLEEP))
+		mcflags |= MCR_TRYHARD;
+
+	/*
+	 * Simple case where all elements in the lists/chains are mbufs.
+	 * Unless bufsize is greater than MHLEN, each segment chain is made
+	 * up of exactly 1 mbuf.  Otherwise, each segment chain is made up
+	 * of 2 mbufs; the second one is used for the residual data, i.e.
+	 * the remaining data that cannot fit into the first mbuf.
+	 */
+	if (bufsize <= MINCLSIZE) {
+		/* Allocate the elements in one shot from the mbuf cache */
+		ASSERT(bufsize <= MHLEN || nsegs == 2);
+		cp = m_cache(MC_MBUF);
+		needed = mcache_alloc_ext(cp, &mp_list,
+		    (*numlist) * nsegs, mcflags);
+
+		/*
+		 * The number of elements must be even if we are to use an
+		 * mbuf (instead of a cluster) to store the residual data.
+		 * If we couldn't allocate the requested number of mbufs,
+		 * trim the number down (if it's odd) in order to avoid
+		 * creating a partial segment chain.
+		 */
+		if (bufsize > MHLEN && (needed & 0x1))
+			needed--;
 
-		m_range_check(mfree);
-		m_range_check(mclfree);
-		m_range_check(mbigfree);
+		while (num < needed) {
+			struct mbuf *m;
 
-		for (len = 0; len < packetlen; ) {
-			struct mbuf *m = NULL;
+			m = (struct mbuf *)mp_list;
+			mp_list = mp_list->obj_next;
+			ASSERT(m != NULL);
 
-			if (wantsize == 0 && packetlen > MINCLSIZE) {
-				if (packetlen - len > MCLBYTES)
-					bufsize = NBPG;
-				else
-					bufsize = MCLBYTES;
+			MBUF_INIT(m, 1, MT_DATA);
+#if CONFIG_MACF_NET
+			if (mac_init_mbuf(m, wait) != 0) {
+				m_free(m);
+				break;
 			}
-			len += bufsize;
-			
-			if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
-				/* mbuf + cluster are available */
-				m = mfree;
-				MCHECK(m);
-				mfree = m->m_next;
-				++mclrefcnt[mtocl(m)];
-				mbstat.m_mtypes[MT_FREE]--;
-				mbstat.m_mtypes[MT_DATA]++;
-				if (bufsize == NBPG) {
-					m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
-					++mclrefcnt[mtocl(m->m_ext.ext_buf)];
-					++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
-					mbstat.m_bigclfree--;
-					mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
-					m->m_ext.ext_free = m_bigfree;
-					m->m_ext.ext_size = NBPG;
-				} else {
-					m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
-					++mclrefcnt[mtocl(m->m_ext.ext_buf)];
-					mbstat.m_clfree--;
-					mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
-					m->m_ext.ext_free = 0;
-					m->m_ext.ext_size = MCLBYTES;
-				}
-				m->m_ext.ext_arg = 0;
-				m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
-				m->m_next = m->m_nextpkt = 0;
-				m->m_type = MT_DATA;
-				m->m_data = m->m_ext.ext_buf;
-				m->m_len = 0;
-	
-				if (pkt == 0) {
-					pkt = m;
-					m->m_flags = M_PKTHDR | M_EXT;
-					_M_CLEAR_PKTHDR(m);
-				} else {
-					m->m_flags = M_EXT;
-				}
-			} else {
-				MBUF_UNLOCK();
-				
-				if (pkt == 0) {
-					MGETHDR(m, how, MT_DATA);
-				} else {
-					MGET(m, how, MT_DATA );
-				}
-				if (m == 0) {
-					m_freem(pkt);
-					goto fail;
-				}
-				if (bufsize <= MINCLSIZE) {
-					if (bufsize > MHLEN) {
-						MGET(m->m_next, how, MT_DATA);
-						if (m->m_next == 0) {
-							m_free(m);
-							m_freem(pkt);
-							goto fail;
-						}
-					}
-				} else {
-					if (bufsize == NBPG)
-						m = m_mbigget(m, how);
-					else
-						m = m_mclget(m, how);
-					if ((m->m_flags & M_EXT) == 0) {
-						m_free(m);
-						m_freem(pkt);
-						goto fail;
-					}
-				}
-				MBUF_LOCK();
+#endif /* MAC_NET */
+			num++;
+			if (bufsize > MHLEN) {
+				/* A second mbuf for this segment chain */
+				m->m_next = (struct mbuf *)mp_list;
+				mp_list = mp_list->obj_next;
+				ASSERT(m->m_next != NULL);
+
+				MBUF_INIT(m->m_next, 0, MT_DATA);
+				num++;
 			}
-			*nm = m;
-			nm = &m->m_next;
+			*np = m;
+			np = &m->m_nextpkt;
+		}
+		ASSERT(num != *numlist || mp_list == NULL);
+
+		if (num > 0) {
+			mtype_stat_add(MT_DATA, num);
+			mtype_stat_sub(MT_FREE, num);
+		}
+		num /= nsegs;
+
+		/* We've got them all; return to caller */
+		if (num == *numlist)
+			return (top);
+
+		goto fail;
+	}
+
+	/*
+	 * Complex cases where elements are made up of one or more composite
+	 * mbufs + cluster, depending on packetlen.  Each N-segment chain can
+	 * be illustrated as follows:
+	 *
+	 * [mbuf + cluster 1] [mbuf + cluster 2] ... [mbuf + cluster N]
+	 *
+	 * Every composite mbuf + cluster element comes from the intermediate
+	 * cache (either MC_MBUF_CL or MC_MBUF_BIGCL).  For space efficiency,
+	 * the last composite element will come from the MC_MBUF_CL cache,
+	 * unless the residual data is larger than 2KB where we use the
+	 * big cluster composite cache (MC_MBUF_BIGCL) instead.  Residual
+	 * data is defined as extra data beyond the first element that cannot
+	 * fit into the previous element, i.e. there is no residual data if
+	 * the chain only has 1 segment.
+	 */
+	r_bufsize = bufsize;
+	resid = packetlen > bufsize ? packetlen % bufsize : 0;
+	if (resid > 0) {
+		/* There is residual data; figure out the cluster size */
+		if (wantsize == 0 && packetlen > MINCLSIZE) {
+			/*
+			 * Caller didn't request that all of the segments
+			 * in the chain use the same cluster size; use the
+			 * smaller of the cluster sizes.
+			 */
+			if (njcl > 0 && resid > m_maxsize(MC_BIGCL))
+				r_bufsize = m_maxsize(MC_16KCL);
+			else if (resid > m_maxsize(MC_CL))
+				r_bufsize = m_maxsize(MC_BIGCL);
+			else
+				r_bufsize = m_maxsize(MC_CL);
+		} else {
+			/* Use the same cluster size as the other segments */
+			resid = 0;
+		}
+	}
+
+	needed = *numlist;
+	if (resid > 0) {
+		/*
+		 * Attempt to allocate composite mbuf + cluster elements for
+		 * the residual data in each chain; record the number of such
+		 * elements that can be allocated so that we know how many
+		 * segment chains we can afford to create.
+		 */
+		if (r_bufsize <= m_maxsize(MC_CL))
+			rcp = m_cache(MC_MBUF_CL);
+		else if (r_bufsize <= m_maxsize(MC_BIGCL))
+			rcp = m_cache(MC_MBUF_BIGCL);
+		else
+			rcp = m_cache(MC_MBUF_16KCL);
+		needed = mcache_alloc_ext(rcp, &rmp_list, *numlist, mcflags);
+
+		if (needed == 0)
+			goto fail;
+
+		/* This is temporarily reduced for calculation */
+		ASSERT(nsegs > 1);
+		nsegs--;
+	}
+
+	/*
+	 * Attempt to allocate the rest of the composite mbuf + cluster
+	 * elements for the number of segment chains that we need.
+	 */
+	if (bufsize <= m_maxsize(MC_CL))
+		cp = m_cache(MC_MBUF_CL);
+	else if (bufsize <= m_maxsize(MC_BIGCL))
+		cp = m_cache(MC_MBUF_BIGCL);
+	else
+		cp = m_cache(MC_MBUF_16KCL);
+	needed = mcache_alloc_ext(cp, &mp_list, needed * nsegs, mcflags);
+
+	/* Round it down to avoid creating a partial segment chain */
+	needed = (needed / nsegs) * nsegs;
+	if (needed == 0)
+		goto fail;
+
+	if (resid > 0) {
+		/*
+		 * We're about to construct the chain(s); take into account
+		 * the number of segments we have created above to hold the
+		 * residual data for each chain, as well as restore the
+		 * original count of segments per chain.
+		 */
+		ASSERT(nsegs > 0);
+		needed += needed / nsegs;
+		nsegs++;
+	}
+
+	for (;;) {
+		struct mbuf *m;
+		u_int32_t flag;
+		struct ext_ref *rfa;
+		void *cl;
+		int pkthdr;
+
+		++num;
+		if (nsegs == 1 || (num % nsegs) != 0 || resid == 0) {
+			m = (struct mbuf *)mp_list;
+			mp_list = mp_list->obj_next;
+		} else {
+			m = (struct mbuf *)rmp_list;
+			rmp_list = rmp_list->obj_next;
+		}
+		ASSERT(m != NULL);
+		VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
+		VERIFY(m->m_ext.ext_free == NULL ||
+		    m->m_ext.ext_free == m_bigfree ||
+		    m->m_ext.ext_free == m_16kfree);
+
+		cl = m->m_ext.ext_buf;
+		rfa = MEXT_RFA(m);
+
+		ASSERT(cl != NULL && rfa != NULL);
+		VERIFY(MBUF_IS_COMPOSITE(m));
+
+		flag = MEXT_FLAGS(m);
+
+		pkthdr = (nsegs == 1 || (num % nsegs) == 1);
+		if (pkthdr)
+			first = m;
+		MBUF_INIT(m, pkthdr, MT_DATA);
+		if (m->m_ext.ext_free == m_16kfree) {
+			MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
+		} else if (m->m_ext.ext_free == m_bigfree) {
+			MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
+		} else {
+			MBUF_CL_INIT(m, cl, rfa, 1, flag);
+		}
+#if CONFIG_MACF_NET
+		if (pkthdr && mac_init_mbuf(m, wait) != 0) {
+			--num;
+			m_free(m);
+			break;
 		}
-		*np = pkt; 		
-		np = &pkt->m_nextpkt;
+#endif /* MAC_NET */
+
+		*np = m;
+		if ((num % nsegs) == 0)
+			np = &first->m_nextpkt;
+		else
+			np = &m->m_next;
+
+		if (num == needed)
+			break;
+	}
+
+	if (num > 0) {
+		mtype_stat_add(MT_DATA, num);
+		mtype_stat_sub(MT_FREE, num);
 	}
-	MBUF_UNLOCK();
-	*num_needed = num;
-	
-	return top;
+
+	num /= nsegs;
+
+	/* We've got them all; return to caller */
+	if (num == *numlist) {
+		ASSERT(mp_list == NULL && rmp_list == NULL);
+		return (top);
+	}
+
 fail:
-	if (wantall && top) {
+	/* Free up what's left of the above */
+	if (mp_list != NULL)
+		mcache_free_ext(cp, mp_list);
+	if (rmp_list != NULL)
+		mcache_free_ext(rcp, rmp_list);
+	if (wantall && top != NULL) {
 		m_freem(top);
-		return 0;
+		return (NULL);
 	}
-	*num_needed = num;
-	
-	return top;
+	*numlist = num;
+	return (top);
 }
 
-
-/* Best effort to get a mbuf cluster + pkthdr under one lock.
- * If we don't have them avail, just bail out and use the regular
- * path.
- * Used by drivers to allocated packets on receive ring.
+/*
+ * Best effort to get a mbuf cluster + pkthdr.  Used by drivers to allocated
+ * packets on receive ring.
  */
 __private_extern__ struct mbuf *
-m_getpacket_how(int how)
+m_getpacket_how(int wait)
 {
 	unsigned int num_needed = 1;
-	
-	return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
+
+	return (m_getpackets_internal(&num_needed, 1, wait, 1,
+	    m_maxsize(MC_CL)));
 }
 
-/* Best effort to get a mbuf cluster + pkthdr under one lock.
- * If we don't have them avail, just bail out and use the regular
- * path.
- * Used by drivers to allocated packets on receive ring.
+/*
+ * Best effort to get a mbuf cluster + pkthdr.  Used by drivers to allocated
+ * packets on receive ring.
  */
 struct mbuf *
 m_getpacket(void)
 {
 	unsigned int num_needed = 1;
 
-	return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
+	return (m_getpackets_internal(&num_needed, 1, M_WAIT, 1,
+	    m_maxsize(MC_CL)));
 }
 
-
 /*
- * return a list of mbuf hdrs that point to clusters...
- * try for num_needed, if this can't be met, return whatever
- * number were available... set up the first num_with_pkthdrs
- * with mbuf hdrs configured as packet headers... these are
- * chained on the m_nextpkt field... any packets requested beyond
- * this are chained onto the last packet header's m_next field.
+ * Return a list of mbuf hdrs that point to clusters.  Try for num_needed;
+ * if this can't be met, return whatever number were available.  Set up the
+ * first num_with_pkthdrs with mbuf hdrs configured as packet headers.  These
+ * are chained on the m_nextpkt field.  Any packets requested beyond this are
+ * chained onto the last packet header's m_next field.
  */
 struct mbuf *
 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
 {
 	unsigned int n = num_needed;
-	
-	return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
-}
 
+	return (m_getpackets_internal(&n, num_with_pkthdrs, how, 0,
+	    m_maxsize(MC_CL)));
+}
 
 /*
- * return a list of mbuf hdrs set up as packet hdrs
- * chained together on the m_nextpkt field
+ * Return a list of mbuf hdrs set up as packet hdrs chained together
+ * on the m_nextpkt field
  */
 struct mbuf *
 m_getpackethdrs(int num_needed, int how)
@@ -1170,168 +3528,202 @@ m_getpackethdrs(int num_needed, int how)
 	top = NULL;
 	np = &top;
 
-	MBUF_LOCK();
-
 	while (num_needed--) {
-		m_range_check(mfree);
-		m_range_check(mclfree);
-		m_range_check(mbigfree);
-
-	    if ((m = mfree)) {	/* mbufs are available */
-                MCHECK(m);
-                mfree = m->m_next;
-                ++mclrefcnt[mtocl(m)];
-                mbstat.m_mtypes[MT_FREE]--;
-                mbstat.m_mtypes[MT_DATA]++;
-
-                m->m_next = m->m_nextpkt = 0;
-                m->m_type = MT_DATA;
-                m->m_flags = M_PKTHDR;
-                m->m_len = 0;
-                m->m_data = m->m_pktdat;
-                _M_CLEAR_PKTHDR(m);
-
-	    } else {
-
-			MBUF_UNLOCK();
-			m = m_retryhdr(how, MT_DATA);
-			if (m == 0)
-				return(top);
-			MBUF_LOCK();
-	    }
-	    *np = m; 
-	    np = &m->m_nextpkt;
-	}
-	MBUF_UNLOCK();
+		m = _M_RETRYHDR(how, MT_DATA);
+		if (m == NULL)
+			break;
+
+		*np = m;
+		np = &m->m_nextpkt;
+	}
 
 	return (top);
 }
 
-
-/* free and mbuf list (m_nextpkt) while following m_next under one lock.
- * returns the count for mbufs packets freed. Used by the drivers.
+/*
+ * Free an mbuf list (m_nextpkt) while following m_next.  Returns the count
+ * for mbufs packets freed.  Used by the drivers.
  */
-int 
-m_freem_list(
-	struct mbuf *m)
+int
+m_freem_list(struct mbuf *m)
 {
 	struct mbuf *nextpkt;
-	int i, count=0;
-
-	MBUF_LOCK();
+	mcache_obj_t *mp_list = NULL;
+	mcache_obj_t *mcl_list = NULL;
+	mcache_obj_t *mbc_list = NULL;
+	mcache_obj_t *m16k_list = NULL;
+	mcache_obj_t *m_mcl_list = NULL;
+	mcache_obj_t *m_mbc_list = NULL;
+	mcache_obj_t *m_m16k_list = NULL;
+	mcache_obj_t *ref_list = NULL;
+	int pktcount = 0;
+	int mt_free = 0, mt_data = 0, mt_header = 0, mt_soname = 0, mt_tag = 0;
+
+	while (m != NULL) {
+		pktcount++;
+
+		nextpkt = m->m_nextpkt;
+		m->m_nextpkt = NULL;
+
+		while (m != NULL) {
+			struct mbuf *next = m->m_next;
+			mcache_obj_t *o, *rfa;
+			u_int32_t refcnt, flags;
 
-	while (m) {
-		if (m) 
-		        nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
-		else 
-		        nextpkt = 0;
-
-		count++;
-
-		while (m) { /* free the mbuf chain (like mfreem) */
-			
-			struct mbuf *n;
+			if (m->m_type == MT_FREE)
+				panic("m_free: freeing an already freed mbuf");
 
-			m_range_check(m);
-			m_range_check(mfree);
-			m_range_check(mclfree);
-			m_range_check(mbigfree);
-	
+			if (m->m_type != MT_FREE)
+				mt_free++;
 
-			/* Free the aux data if there is any */
-			if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
-				/*
-				 * Treat the current m as the nextpkt and set m
-				 * to the aux data. Preserve nextpkt in m->m_nextpkt.
-				 * This lets us free the aux data in this loop
-				 * without having to call m_freem recursively,
-				 * which wouldn't work because we've still got
-				 * the lock.
-				 */
-				m->m_nextpkt = nextpkt;
-				nextpkt = m;
-				m = nextpkt->m_pkthdr.aux;
-				nextpkt->m_pkthdr.aux = NULL;
-			}
-			
-			if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
-				/* A quick (albeit inefficient) expedient */
-				MBUF_UNLOCK();
+			if (m->m_flags & M_PKTHDR) {
 				m_tag_delete_chain(m, NULL);
-				MBUF_LOCK();
 			}
 
-			n = m->m_next;
-
-			if (n && n->m_nextpkt)
-				panic("m_freem_list: m_nextpkt of m_next != NULL");
-			if (m->m_type == MT_FREE)
-				panic("freeing free mbuf");
+			if (!(m->m_flags & M_EXT))
+				goto simple_free;
+
+			o = (mcache_obj_t *)m->m_ext.ext_buf;
+			refcnt = m_decref(m);
+			flags = MEXT_FLAGS(m);
+			if (refcnt == 0 && flags == 0) {
+				if (m->m_ext.ext_free == NULL) {
+					o->obj_next = mcl_list;
+					mcl_list = o;
+				} else if (m->m_ext.ext_free == m_bigfree) {
+					o->obj_next = mbc_list;
+					mbc_list = o;
+				} else if (m->m_ext.ext_free == m_16kfree) {
+					o->obj_next = m16k_list;
+					m16k_list = o;
+				} else {
+					(*(m->m_ext.ext_free))((caddr_t)o,
+					    m->m_ext.ext_size,
+					    m->m_ext.ext_arg);
+				}
+				rfa = (mcache_obj_t *)MEXT_RFA(m);
+				rfa->obj_next = ref_list;
+				ref_list = rfa;
+				MEXT_RFA(m) = NULL;
+			} else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) {
+				VERIFY(m->m_type != MT_FREE);
+				/*
+				 * Amortize the costs of atomic operations
+				 * by doing them at the end, if possible.
+				 */
+				if (m->m_type == MT_DATA)
+					mt_data++;
+				else if (m->m_type == MT_HEADER)
+					mt_header++;
+				else if (m->m_type == MT_SONAME)
+					mt_soname++;
+				else if (m->m_type == MT_TAG)
+					mt_tag++;
+				else
+					mtype_stat_dec(m->m_type);
 
-			if (m->m_flags & M_EXT) {
-				if (MCLHASREFERENCE(m)) {
-					remque((queue_t)&m->m_ext.ext_refs);
-				} else if (m->m_ext.ext_free == NULL) {
-					union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
-					
-					m_range_check(mcl);
-
-					if (_MCLUNREF(mcl)) {
-						mcl->mcl_next = mclfree;
-						mclfree = mcl;
-						++mbstat.m_clfree;
-					} 
+				m->m_type = MT_FREE;
+				m->m_flags = M_EXT;
+				m->m_len = 0;
+				m->m_next = m->m_nextpkt = NULL;
+
+				/* "Free" into the intermediate cache */
+				o = (mcache_obj_t *)m;
+				if (m->m_ext.ext_free == NULL) {
+					o->obj_next = m_mcl_list;
+					m_mcl_list = o;
+				} else if (m->m_ext.ext_free == m_bigfree) {
+					o->obj_next = m_mbc_list;
+					m_mbc_list = o;
 				} else {
-					(*(m->m_ext.ext_free))(m->m_ext.ext_buf,
-					    m->m_ext.ext_size, m->m_ext.ext_arg);
+					VERIFY(m->m_ext.ext_free == m_16kfree);
+					o->obj_next = m_m16k_list;
+					m_m16k_list = o;
 				}
+				m = next;
+				continue;
 			}
-			mbstat.m_mtypes[m->m_type]--;
-			(void) _MCLUNREF(m);
-              _MFREE_MUNGE(m);
-			mbstat.m_mtypes[MT_FREE]++;
+simple_free:
+			/*
+			 * Amortize the costs of atomic operations
+			 * by doing them at the end, if possible.
+			 */
+			if (m->m_type == MT_DATA)
+				mt_data++;
+			else if (m->m_type == MT_HEADER)
+				mt_header++;
+			else if (m->m_type == MT_SONAME)
+				mt_soname++;
+			else if (m->m_type == MT_TAG)
+				mt_tag++;
+			else if (m->m_type != MT_FREE)
+				mtype_stat_dec(m->m_type);
+
 			m->m_type = MT_FREE;
-			m->m_flags = 0;
-			m->m_len = 0;
-			m->m_next = mfree;
-			mfree = m;
-			m = n;
-		}
-		m = nextpkt; /* bump m with saved nextpkt if any */
-	}
-	if ((i = m_want))
-		m_want = 0;
+			m->m_flags = m->m_len = 0;
+			m->m_next = m->m_nextpkt = NULL;
 
-	MBUF_UNLOCK();
+			((mcache_obj_t *)m)->obj_next = mp_list;
+			mp_list = (mcache_obj_t *)m;
+
+			m = next;
+		}
 
-	if (i)
-		wakeup((caddr_t)&mfree);
+		m = nextpkt;
+	}
 
-	return (count);
+	if (mt_free > 0)
+		mtype_stat_add(MT_FREE, mt_free);
+	if (mt_data > 0)
+		mtype_stat_sub(MT_DATA, mt_data);
+	if (mt_header > 0)
+		mtype_stat_sub(MT_HEADER, mt_header);
+	if (mt_soname > 0)
+		mtype_stat_sub(MT_SONAME, mt_soname);
+	if (mt_tag > 0)
+		mtype_stat_sub(MT_TAG, mt_tag);
+
+	if (mp_list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF), mp_list);
+	if (mcl_list != NULL)
+		mcache_free_ext(m_cache(MC_CL), mcl_list);
+	if (mbc_list != NULL)
+		mcache_free_ext(m_cache(MC_BIGCL), mbc_list);
+	if (m16k_list != NULL)
+		mcache_free_ext(m_cache(MC_16KCL), m16k_list);
+	if (m_mcl_list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF_CL), m_mcl_list);
+	if (m_mbc_list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF_BIGCL), m_mbc_list);
+	if (m_m16k_list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF_16KCL), m_m16k_list);
+	if (ref_list != NULL)
+		mcache_free_ext(ref_cache, ref_list);
+
+	return (pktcount);
 }
 
 void
-m_freem(
-	struct mbuf *m)
+m_freem(struct mbuf *m)
 {
-	while (m)
+	while (m != NULL)
 		m = m_free(m);
 }
 
 /*
  * Mbuffer utility routines.
  */
+
 /*
- * Compute the amount of space available
- * before the current start of data in an mbuf.
+ * Compute the amount of space available before the current start
+ * of data in an mbuf.
  */
 int
-m_leadingspace(
-	struct mbuf *m)
+m_leadingspace(struct mbuf *m)
 {
 	if (m->m_flags & M_EXT) {
 		if (MCLHASREFERENCE(m))
-			return(0);
+			return (0);
 		return (m->m_data - m->m_ext.ext_buf);
 	}
 	if (m->m_flags & M_PKTHDR)
@@ -1340,40 +3732,33 @@ m_leadingspace(
 }
 
 /*
- * Compute the amount of space available
- * after the end of data in an mbuf.
+ * Compute the amount of space available after the end of data in an mbuf.
  */
 int
-m_trailingspace(
-	struct mbuf *m)
+m_trailingspace(struct mbuf *m)
 {
 	if (m->m_flags & M_EXT) {
 		if (MCLHASREFERENCE(m))
-			return(0);
+			return (0);
 		return (m->m_ext.ext_buf + m->m_ext.ext_size -
-			(m->m_data + m->m_len));
+		    (m->m_data + m->m_len));
 	}
 	return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
 }
 
 /*
- * Lesser-used path for M_PREPEND:
- * allocate new mbuf to prepend to chain,
- * copy junk along.
- * Does not adjust packet header length.
+ * Lesser-used path for M_PREPEND: allocate new mbuf to prepend to chain,
+ * copy junk along.  Does not adjust packet header length.
  */
 struct mbuf *
-m_prepend(
-	struct mbuf *m,
-	int len, 
-	int how)
+m_prepend(struct mbuf *m, int len, int how)
 {
 	struct mbuf *mn;
 
-	MGET(mn, how, m->m_type);
-	if (mn == (struct mbuf *)NULL) {
+	_MGET(mn, how, m->m_type);
+	if (mn == NULL) {
 		m_freem(m);
-		return ((struct mbuf *)NULL);
+		return (NULL);
 	}
 	if (m->m_flags & M_PKTHDR) {
 		M_COPY_PKTHDR(mn, m);
@@ -1388,26 +3773,21 @@ m_prepend(
 }
 
 /*
- * Replacement for old M_PREPEND macro:
- * allocate new mbuf to prepend to chain,
- * copy junk along, and adjust length.
- * 
+ * Replacement for old M_PREPEND macro: allocate new mbuf to prepend to
+ * chain, copy junk along, and adjust length.
  */
 struct mbuf *
-m_prepend_2(
-        struct mbuf *m,
-        int len,
-        int how)
-{
-        if (M_LEADINGSPACE(m) >= len) {
-                m->m_data -= len;
-                m->m_len += len;
-        } else {
+m_prepend_2(struct mbuf *m, int len, int how)
+{
+	if (M_LEADINGSPACE(m) >= len) {
+		m->m_data -= len;
+		m->m_len += len;
+	} else {
 		m = m_prepend(m, len, how);
-        }
-        if ((m) && (m->m_flags & M_PKTHDR))
-                m->m_pkthdr.len += len;
-        return (m);
+	}
+	if ((m) && (m->m_flags & M_PKTHDR))
+		m->m_pkthdr.len += len;
+	return (m);
 }
 
 /*
@@ -1418,64 +3798,45 @@ m_prepend_2(
 int MCFail;
 
 struct mbuf *
-m_copym(
-	struct mbuf *m,
-	int off0,
-	int len,
-	int wait)
+m_copym(struct mbuf *m, int off0, int len, int wait)
 {
-	struct mbuf *n, **np;
+	struct mbuf *n, *mhdr = NULL, **np;
 	int off = off0;
 	struct mbuf *top;
 	int copyhdr = 0;
 
 	if (off < 0 || len < 0)
-		panic("m_copym");
-	if (off == 0 && m->m_flags & M_PKTHDR)
+		panic("m_copym: invalid offset %d or len %d", off, len);
+
+	if (off == 0 && (m->m_flags & M_PKTHDR)) {
+		mhdr = m;
 		copyhdr = 1;
+	}
 
 	while (off >= m->m_len) {
-		if (m == 0)
-			panic("m_copym");
+		if (m->m_next == NULL)
+			panic("m_copym: invalid mbuf chain");
 		off -= m->m_len;
 		m = m->m_next;
 	}
 	np = &top;
-	top = 0;
-
-	MBUF_LOCK();
+	top = NULL;
 
 	while (len > 0) {
-		m_range_check(mfree);
-		m_range_check(mclfree);
-		m_range_check(mbigfree);
-
-		if (m == 0) {
+		if (m == NULL) {
 			if (len != M_COPYALL)
-				panic("m_copym");
+				panic("m_copym: len != M_COPYALL");
 			break;
 		}
-		if ((n = mfree)) {
-			MCHECK(n);
-			++mclrefcnt[mtocl(n)];
-			mbstat.m_mtypes[MT_FREE]--;
-			mbstat.m_mtypes[m->m_type]++;
-			mfree = n->m_next;
-			n->m_next = n->m_nextpkt = 0;
-			n->m_type = m->m_type;
-			n->m_data = n->m_dat;
-			n->m_flags = 0;
-		} else {
-		        MBUF_UNLOCK();
-		        n = m_retry(wait, m->m_type);
-		        MBUF_LOCK();
-		}
+
+		n = _M_RETRY(wait, m->m_type);
 		*np = n;
 
-		if (n == 0)
+		if (n == NULL)
 			goto nospace;
-		if (copyhdr) {
-			M_COPY_PKTHDR(n, m);
+
+		if (copyhdr != 0) {
+			M_COPY_PKTHDR(n, mhdr);
 			if (len == M_COPYALL)
 				n->m_pkthdr.len -= off0;
 			else
@@ -1483,24 +3844,24 @@ m_copym(
 			copyhdr = 0;
 		}
 		if (len == M_COPYALL) {
-		    if (min(len, (m->m_len - off)) == len) {
-			printf("m->m_len %d - off %d = %d, %d\n", 
-			       m->m_len, off, m->m_len - off,
-			       min(len, (m->m_len - off)));
-		    }
+			if (MIN(len, (m->m_len - off)) == len) {
+				printf("m->m_len %ld - off %d = %ld, %ld\n",
+				    m->m_len, off, m->m_len - off,
+				    MIN(len, (m->m_len - off)));
+			}
 		}
-		n->m_len = min(len, (m->m_len - off));
+		n->m_len = MIN(len, (m->m_len - off));
 		if (n->m_len == M_COPYALL) {
-		    printf("n->m_len == M_COPYALL, fixing\n");
-		    n->m_len = MHLEN;
+			printf("n->m_len == M_COPYALL, fixing\n");
+			n->m_len = MHLEN;
 		}
 		if (m->m_flags & M_EXT) {
 			n->m_ext = m->m_ext;
-			insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
+			m_incref(m);
 			n->m_data = m->m_data + off;
 			n->m_flags |= M_EXT;
 		} else {
-			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
+			bcopy(MTOD(m, caddr_t)+off, MTOD(n, caddr_t),
 			    (unsigned)n->m_len);
 		}
 		if (len != M_COPYALL)
@@ -1509,176 +3870,172 @@ m_copym(
 		m = m->m_next;
 		np = &n->m_next;
 	}
-	MBUF_UNLOCK();
 
-	if (top == 0)
+	if (top == NULL)
 		MCFail++;
 
 	return (top);
 nospace:
-	MBUF_UNLOCK();
 
 	m_freem(top);
 	MCFail++;
-	return (0);
+	return (NULL);
 }
 
-
 /*
- * equivilent to m_copym except that all necessary
- * mbuf hdrs are allocated within this routine
- * also, the last mbuf and offset accessed are passed
- * out and can be passed back in to avoid having to
- * rescan the entire mbuf list (normally hung off of the socket)
+ * Equivalent to m_copym except that all necessary mbuf hdrs are allocated
+ * within this routine also, the last mbuf and offset accessed are passed
+ * out and can be passed back in to avoid having to rescan the entire mbuf
+ * list (normally hung off of the socket)
  */
 struct mbuf *
-m_copym_with_hdrs(
-	struct mbuf *m,
-	int off0, 
-	int len,
-	int wait,
-	struct mbuf **m_last,
-	int *m_off)
-{
-	struct mbuf *n, **np = 0;
-	int off = off0;
-	struct mbuf *top = 0;
+m_copym_with_hdrs(struct mbuf *m, int off0, int len0, int wait,
+    struct mbuf **m_last, int *m_off)
+{
+	struct mbuf *n, **np = NULL;
+	int off = off0, len = len0;
+	struct mbuf *top = NULL;
+	int mcflags = MSLEEPF(wait);
 	int copyhdr = 0;
-	int type;
+	int type = 0;
+	mcache_obj_t *list = NULL;
+	int needed = 0;
 
-	if (off == 0 && m->m_flags & M_PKTHDR)
+	if (off == 0 && (m->m_flags & M_PKTHDR))
 		copyhdr = 1;
 
-	if (*m_last) {
-	        m   = *m_last;
+	if (*m_last != NULL) {
+		m = *m_last;
 		off = *m_off;
 	} else {
-	        while (off >= m->m_len) {
-		        off -= m->m_len;
+		while (off >= m->m_len) {
+			off -= m->m_len;
 			m = m->m_next;
 		}
 	}
 
-	MBUF_LOCK();
+	n = m;
+	while (len > 0) {
+		needed++;
+		ASSERT(n != NULL);
+		len -= MIN(len, (n->m_len - ((needed == 1) ? off : 0)));
+		n = n->m_next;
+	}
+	needed++;
+	len = len0;
+
+	/*
+	 * If the caller doesn't want to be put to sleep, mark it with
+	 * MCR_TRYHARD so that we may reclaim buffers from other places
+	 * before giving up.
+	 */
+	if (mcflags & MCR_NOSLEEP)
+		mcflags |= MCR_TRYHARD;
+
+	if (mcache_alloc_ext(m_cache(MC_MBUF), &list, needed,
+	    mcflags) != needed)
+		goto nospace;
 
+	needed = 0;
 	while (len > 0) {
-		m_range_check(mfree);
-		m_range_check(mclfree);
-		m_range_check(mbigfree);
-
-		if (top == 0)
-		        type = MT_HEADER;
-		else {
-		        if (m == 0)
-			        panic("m_gethdr_and_copym");
-		        type = m->m_type;
-		}
-		if ((n = mfree)) {
-		        MCHECK(n);
-		        ++mclrefcnt[mtocl(n)];
-			mbstat.m_mtypes[MT_FREE]--;
-			mbstat.m_mtypes[type]++;
-			mfree = n->m_next;
-			n->m_next = n->m_nextpkt = 0;
-			n->m_type = type;
-
-			if (top) {
-			        n->m_data = n->m_dat;
-				n->m_flags = 0;
-			} else {
-			        n->m_data = n->m_pktdat;
-				n->m_flags = M_PKTHDR;
-				_M_CLEAR_PKTHDR(n);
-			}
-		} else {
-		        MBUF_UNLOCK();
-			if (top)
-			        n = m_retry(wait, type);
-			else
-			        n = m_retryhdr(wait, type);
-		        MBUF_LOCK();
-		}
-		if (n == 0)
+		n = (struct mbuf *)list;
+		list = list->obj_next;
+		ASSERT(n != NULL && m != NULL);
+
+		type = (top == NULL) ? MT_HEADER : m->m_type;
+		MBUF_INIT(n, (top == NULL), type);
+#if CONFIG_MACF_NET
+		if (top == NULL && mac_mbuf_label_init(n, wait) != 0) {
+			mtype_stat_inc(MT_HEADER);
+			mtype_stat_dec(MT_FREE);
+			m_free(n);
 			goto nospace;
-		if (top == 0) {
-		        top = n;
+		}
+#endif /* MAC_NET */
+
+		if (top == NULL) {
+			top = n;
 			np = &top->m_next;
 			continue;
-		} else
-		        *np = n;
+		} else {
+			needed++;
+			*np = n;
+		}
 
 		if (copyhdr) {
 			M_COPY_PKTHDR(n, m);
 			n->m_pkthdr.len = len;
 			copyhdr = 0;
 		}
-		n->m_len = min(len, (m->m_len - off));
+		n->m_len = MIN(len, (m->m_len - off));
 
 		if (m->m_flags & M_EXT) {
 			n->m_ext = m->m_ext;
-			insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
+			m_incref(m);
 			n->m_data = m->m_data + off;
 			n->m_flags |= M_EXT;
 		} else {
-			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
+			bcopy(MTOD(m, caddr_t)+off, MTOD(n, caddr_t),
 			    (unsigned)n->m_len);
 		}
 		len -= n->m_len;
-		
+
 		if (len == 0) {
-		        if ((off + n->m_len) == m->m_len) {
-			       *m_last = m->m_next;
-			       *m_off  = 0;
+			if ((off + n->m_len) == m->m_len) {
+				*m_last = m->m_next;
+				*m_off  = 0;
 			} else {
-			       *m_last = m;
-			       *m_off  = off + n->m_len;
+				*m_last = m;
+				*m_off  = off + n->m_len;
 			}
-		        break;
+			break;
 		}
 		off = 0;
 		m = m->m_next;
 		np = &n->m_next;
 	}
-	MBUF_UNLOCK();
 
+	mtype_stat_inc(MT_HEADER);
+	mtype_stat_add(type, needed);
+	mtype_stat_sub(MT_FREE, needed + 1);
+
+	ASSERT(list == NULL);
 	return (top);
-nospace:
-	MBUF_UNLOCK();
 
-	if (top)
-	        m_freem(top);
+nospace:
+	if (list != NULL)
+		mcache_free_ext(m_cache(MC_MBUF), list);
+	if (top != NULL)
+		m_freem(top);
 	MCFail++;
-	return (0);
+	return (NULL);
 }
 
-
 /*
  * Copy data from an mbuf chain starting "off" bytes from the beginning,
  * continuing for "len" bytes, into the indicated buffer.
  */
-void m_copydata(
-	struct mbuf *m,
-	int off,
-	int len,
-	caddr_t cp)
+void
+m_copydata(struct mbuf *m, int off, int len, caddr_t cp)
 {
 	unsigned count;
 
 	if (off < 0 || len < 0)
-		panic("m_copydata");
+		panic("m_copydata: invalid offset %d or len %d", off, len);
+
 	while (off > 0) {
-		if (m == 0)
-			panic("m_copydata");
+		if (m == NULL)
+			panic("m_copydata: invalid mbuf chain");
 		if (off < m->m_len)
 			break;
 		off -= m->m_len;
 		m = m->m_next;
 	}
 	while (len > 0) {
-		if (m == 0)
-			panic("m_copydata");
-		count = min(m->m_len - off, len);
-		bcopy(mtod(m, caddr_t) + off, cp, count);
+		if (m == NULL)
+			panic("m_copydata: invalid mbuf chain");
+		count = MIN(m->m_len - off, len);
+		bcopy(MTOD(m, caddr_t) + off, cp, count);
 		len -= count;
 		cp += count;
 		off = 0;
@@ -1687,24 +4044,23 @@ void m_copydata(
 }
 
 /*
- * Concatenate mbuf chain n to m.
- * Both chains must be of the same type (e.g. MT_DATA).
- * Any m_pkthdr is not updated.
+ * Concatenate mbuf chain n to m.  Both chains must be of the same type
+ * (e.g. MT_DATA).  Any m_pkthdr is not updated.
  */
-void m_cat(
-	struct mbuf *m, struct mbuf *n)
+void
+m_cat(struct mbuf *m, struct mbuf *n)
 {
 	while (m->m_next)
 		m = m->m_next;
 	while (n) {
-		if (m->m_flags & M_EXT ||
+		if ((m->m_flags & M_EXT) ||
 		    m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
 			/* just join the two chains */
 			m->m_next = n;
 			return;
 		}
 		/* splat the data from one into the other */
-		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
+		bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
 		    (u_int)n->m_len);
 		m->m_len += n->m_len;
 		n = m_free(n);
@@ -1712,9 +4068,7 @@ void m_cat(
 }
 
 void
-m_adj(
-	struct mbuf *mp,
-	int req_len)
+m_adj(struct mbuf *mp, int req_len)
 {
 	int len = req_len;
 	struct mbuf *m;
@@ -1797,9 +4151,7 @@ m_adj(
 int MPFail;
 
 struct mbuf *
-m_pullup(
-	struct mbuf *n,
-	int len)
+m_pullup(struct mbuf *n, int len)
 {
 	struct mbuf *m;
 	int count;
@@ -1820,7 +4172,7 @@ m_pullup(
 	} else {
 		if (len > MHLEN)
 			goto bad;
-		MGET(m, M_DONTWAIT, n->m_type);
+		_MGET(m, M_DONTWAIT, n->m_type);
 		if (m == 0)
 			goto bad;
 		m->m_len = 0;
@@ -1831,9 +4183,9 @@ m_pullup(
 	}
 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
 	do {
-		count = min(min(max(len, max_protohdr), space), n->m_len);
-		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
-		  (unsigned)count);
+		count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len);
+		bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
+		    (unsigned)count);
 		len -= count;
 		m->m_len += count;
 		n->m_len -= count;
@@ -1861,23 +4213,20 @@ bad:
  * attempts to restore the chain to its original state.
  */
 struct mbuf *
-m_split(
-	struct mbuf *m0,
-	int len0, 
-	int wait)
+m_split(struct mbuf *m0, int len0, int wait)
 {
 	struct mbuf *m, *n;
 	unsigned len = len0, remain;
 
 	for (m = m0; m && len > m->m_len; m = m->m_next)
 		len -= m->m_len;
-	if (m == 0)
-		return (0);
+	if (m == NULL)
+		return (NULL);
 	remain = m->m_len - len;
 	if (m0->m_flags & M_PKTHDR) {
-		MGETHDR(n, wait, m0->m_type);
-		if (n == 0)
-			return (0);
+		_MGETHDR(n, wait, m0->m_type);
+		if (n == NULL)
+			return (NULL);
 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
 		m0->m_pkthdr.len = len0;
@@ -1887,53 +4236,48 @@ m_split(
 			/* m can't be the lead packet */
 			MH_ALIGN(n, 0);
 			n->m_next = m_split(m, len, wait);
-			if (n->m_next == 0) {
+			if (n->m_next == NULL) {
 				(void) m_free(n);
-				return (0);
+				return (NULL);
 			} else
 				return (n);
 		} else
 			MH_ALIGN(n, remain);
 	} else if (remain == 0) {
 		n = m->m_next;
-		m->m_next = 0;
+		m->m_next = NULL;
 		return (n);
 	} else {
-		MGET(n, wait, m->m_type);
-		if (n == 0)
-			return (0);
+		_MGET(n, wait, m->m_type);
+		if (n == NULL)
+			return (NULL);
 		M_ALIGN(n, remain);
 	}
 extpacket:
 	if (m->m_flags & M_EXT) {
 		n->m_flags |= M_EXT;
-		MBUF_LOCK();
 		n->m_ext = m->m_ext;
-                insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
-		MBUF_UNLOCK();
+		m_incref(m);
 		n->m_data = m->m_data + len;
 	} else {
-		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
+		bcopy(MTOD(m, caddr_t) + len, MTOD(n, caddr_t), remain);
 	}
 	n->m_len = remain;
 	m->m_len = len;
 	n->m_next = m->m_next;
-	m->m_next = 0;
+	m->m_next = NULL;
 	return (n);
 }
+
 /*
  * Routine to copy from device local memory into mbufs.
  */
 struct mbuf *
-m_devget(
-	char *buf,
-	int totlen, 
-	int off0,
-	struct ifnet *ifp,
-	void (*copy)(const void *, void *, size_t))
+m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
+    void (*copy)(const void *, void *, size_t))
 {
 	struct mbuf *m;
-	struct mbuf *top = 0, **mp = &top;
+	struct mbuf *top = NULL, **mp = &top;
 	int off = off0, len;
 	char *cp;
 	char *epkt;
@@ -1945,52 +4289,54 @@ m_devget(
 		 * If 'off' is non-zero, packet is trailer-encapsulated,
 		 * so we have to skip the type and length fields.
 		 */
-		cp += off + 2 * sizeof(u_int16_t);
-		totlen -= 2 * sizeof(u_int16_t);
+		cp += off + 2 * sizeof (u_int16_t);
+		totlen -= 2 * sizeof (u_int16_t);
 	}
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
-	if (m == 0)
-		return (0);
+	_MGETHDR(m, M_DONTWAIT, MT_DATA);
+	if (m == NULL)
+		return (NULL);
 	m->m_pkthdr.rcvif = ifp;
 	m->m_pkthdr.len = totlen;
 	m->m_len = MHLEN;
 
 	while (totlen > 0) {
-		if (top) {
-			MGET(m, M_DONTWAIT, MT_DATA);
-			if (m == 0) {
+		if (top != NULL) {
+			_MGET(m, M_DONTWAIT, MT_DATA);
+			if (m == NULL) {
 				m_freem(top);
-				return (0);
+				return (NULL);
 			}
 			m->m_len = MLEN;
 		}
-		len = min(totlen, epkt - cp);
+		len = MIN(totlen, epkt - cp);
 		if (len >= MINCLSIZE) {
 			MCLGET(m, M_DONTWAIT);
-			if (m->m_flags & M_EXT)
-				m->m_len = len = min(len, MCLBYTES);
-			else {
-			  	/* give up when it's out of cluster mbufs */
-			        if (top)
-				   m_freem(top);
+			if (m->m_flags & M_EXT) {
+				m->m_len = len = MIN(len, m_maxsize(MC_CL));
+			} else {
+				/* give up when it's out of cluster mbufs */
+				if (top != NULL)
+					m_freem(top);
 				m_freem(m);
-				return (0);
+				return (NULL);
 			}
 		} else {
 			/*
 			 * Place initial small packet/header at end of mbuf.
 			 */
 			if (len < m->m_len) {
-				if (top == 0 && len + max_linkhdr <= m->m_len)
+				if (top == NULL &&
+				    len + max_linkhdr <= m->m_len)
 					m->m_data += max_linkhdr;
 				m->m_len = len;
-			} else
+			} else {
 				len = m->m_len;
+			}
 		}
 		if (copy)
-			copy(cp, mtod(m, caddr_t), (unsigned)len);
+			copy(cp, MTOD(m, caddr_t), (unsigned)len);
 		else
-			bcopy(cp, mtod(m, caddr_t), (unsigned)len);
+			bcopy(cp, MTOD(m, caddr_t), (unsigned)len);
 		cp += len;
 		*mp = m;
 		mp = &m->m_next;
@@ -2002,49 +4348,93 @@ m_devget(
 }
 
 /*
- * Cluster freelist allocation check. The mbuf lock  must be held.
- * Ensure hysteresis between hi/lo.
+ * Cluster freelist allocation check.
  */
 static int
 m_howmany(int num, size_t bufsize)
 {
-	int i = 0;
-	
+	int i = 0, j = 0;
+	u_int32_t m_clusters, m_bigclusters, m_16kclusters;
+	u_int32_t m_clfree, m_bigclfree, m_16kclfree;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	m_clusters = m_total(MC_CL);
+	m_bigclusters = m_total(MC_BIGCL);
+	m_16kclusters = m_total(MC_16KCL);
+	m_clfree = m_infree(MC_CL);
+	m_bigclfree = m_infree(MC_BIGCL);
+	m_16kclfree = m_infree(MC_16KCL);
+
 	/* Bail if we've maxed out the mbuf memory map */
-	if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
-		int j = 0;
-		
-		if (bufsize == MCLBYTES) {
-			/* Under minimum */
-			if (mbstat.m_clusters < MINCL)
-				return (MINCL - mbstat.m_clusters);
-			/* Too few (free < 1/2 total) and not over maximum */
-			if (mbstat.m_clusters < (nmbclusters >> 1)) {
-				if (num >= mbstat.m_clfree)
-					i = num - mbstat.m_clfree;
-				if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
-					j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
-				i = max(i, j);
-				if (i + mbstat.m_clusters >= (nmbclusters >> 1))
-					i = (nmbclusters >> 1) - mbstat.m_clusters;
-			}
-		} else {
-			/* Under minimum */
-			if (mbstat.m_bigclusters < MINCL)
-				return (MINCL - mbstat.m_bigclusters);
-			/* Too few (free < 1/2 total) and not over maximum */
-			if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
-				if (num >= mbstat.m_bigclfree)
-					i = num - mbstat.m_bigclfree;
-				if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
-					j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
-				i = max(i, j);
-				if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
-					i = (nmbclusters >> 2) - mbstat.m_bigclusters;
-			}
+	if ((bufsize != m_maxsize(MC_16KCL) &&
+	    (m_clusters + (m_bigclusters << 1) >= nclusters)) ||
+	    (njcl > 0 && bufsize == m_maxsize(MC_16KCL) &&
+	    (m_16kclusters << 3) >= njcl)) {
+#if DEBUG
+		if (bufsize == MCLBYTES && num > m_clfree) {
+			printf("m_howmany - out of small clusters, "
+			    "%d short\n", num - mbstat.m_clfree);
+		}
+#endif /* DEBUG */
+		return (0);
+	}
+
+	if (bufsize == m_maxsize(MC_CL)) {
+		/* Under minimum */
+		if (m_clusters < MINCL)
+			return (MINCL - m_clusters);
+		/* Too few (free < 1/16 total) and not over maximum */
+		if (m_clusters < m_maxlimit(MC_CL)) {
+			if (m_clfree >= MCL_LOWAT)
+				return (0);
+			if (num >= m_clfree)
+				i = num - m_clfree;
+			if (((m_clusters + num) >> 4) > m_clfree)
+				j = ((m_clusters + num) >> 4) - m_clfree;
+			i = MAX(i, j);
+			if (i + m_clusters >= m_maxlimit(MC_CL))
+				i = m_maxlimit(MC_CL) - m_clusters;
 		}
+		VERIFY((m_total(MC_CL) + i) <= m_maxlimit(MC_CL));
+	} else if (bufsize == m_maxsize(MC_BIGCL)) {
+		/* Under minimum */
+		if (m_bigclusters < MINBIGCL)
+			return (MINBIGCL - m_bigclusters);
+		/* Too few (free < 1/16 total) and not over maximum */
+		if (m_bigclusters < m_maxlimit(MC_BIGCL)) {
+			if (m_bigclfree >= MBIGCL_LOWAT)
+				return (0);
+			if (num >= m_bigclfree)
+				i = num - m_bigclfree;
+			if (((m_bigclusters + num) >> 4) > m_bigclfree)
+				j = ((m_bigclusters + num) >> 4) - m_bigclfree;
+			i = MAX(i, j);
+			if (i + m_bigclusters >= m_maxlimit(MC_BIGCL))
+				i = m_maxlimit(MC_BIGCL) - m_bigclusters;
+		}
+		VERIFY((m_total(MC_BIGCL) + i) <= m_maxlimit(MC_BIGCL));
+	} else {
+		VERIFY(njcl > 0);
+		/* Under minimum */
+		if (m_16kclusters < MIN16KCL)
+			return (MIN16KCL - m_16kclusters);
+		/* Too few (free < 1/16 total) and not over maximum */
+		if (m_16kclusters < m_maxlimit(MC_16KCL)) {
+			if (m_16kclfree >= M16KCL_LOWAT)
+				return (0);
+			if (num >= m_16kclfree)
+				i = num - m_16kclfree;
+			if (((m_16kclusters + num) >> 4) > m_16kclfree)
+				j = ((m_16kclusters + num) >> 4) - m_16kclfree;
+			i = MAX(i, j);
+			if (i + m_16kclusters >= m_maxlimit(MC_16KCL))
+				i = m_maxlimit(MC_16KCL) - m_16kclusters;
+		}
+		VERIFY((m_total(MC_16KCL) + i) <= m_maxlimit(MC_16KCL));
 	}
-	return i;
+
+	return (i);
 }
 
 /*
@@ -2053,33 +4443,29 @@ m_howmany(int num, size_t bufsize)
  * chain if necessary.
  */
 void
-m_copyback(
-	struct	mbuf *m0,
-	int off,
-	int len,
-	caddr_t cp)
+m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
 {
 	int mlen;
 	struct mbuf *m = m0, *n;
 	int totlen = 0;
 
-	if (m0 == 0)
+	if (m0 == NULL)
 		return;
 	while (off > (mlen = m->m_len)) {
 		off -= mlen;
 		totlen += mlen;
-		if (m->m_next == 0) {
+		if (m->m_next == NULL) {
 			n = m_getclr(M_DONTWAIT, m->m_type);
-			if (n == 0)
+			if (n == NULL)
 				goto out;
-			n->m_len = min(MLEN, len + off);
+			n->m_len = MIN(MLEN, len + off);
 			m->m_next = n;
 		}
 		m = m->m_next;
 	}
 	while (len > 0) {
-		mlen = min (m->m_len - off, len);
-		bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
+		mlen = MIN(m->m_len - off, len);
+		bcopy(cp, off + MTOD(m, caddr_t), (unsigned)mlen);
 		cp += mlen;
 		len -= mlen;
 		mlen += off;
@@ -2087,51 +4473,52 @@ m_copyback(
 		totlen += mlen;
 		if (len == 0)
 			break;
-		if (m->m_next == 0) {
-			n = m_get(M_DONTWAIT, m->m_type);
-			if (n == 0)
+		if (m->m_next == NULL) {
+			n = _M_GET(M_DONTWAIT, m->m_type);
+			if (n == NULL)
 				break;
-			n->m_len = min(MLEN, len);
+			n->m_len = MIN(MLEN, len);
 			m->m_next = n;
 		}
 		m = m->m_next;
 	}
-out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
+out:
+	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
 		m->m_pkthdr.len = totlen;
 }
 
+char *
+mcl_to_paddr(char *addr)
+{
+	int base_phys;
 
-char *mcl_to_paddr(char *addr) {
-        int base_phys;
-  
-	if (addr < (char *)mbutl || addr >= (char *)embutl)
-	        return (0);
+	if (!MBUF_IN_MAP(addr))
+		return (NULL);
 	base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
 
 	if (base_phys == 0)
-	        return (0);
+		return (NULL);
 	return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
 }
 
 /*
  * Dup the mbuf chain passed in.  The whole thing.  No cute additional cruft.
  * And really copy the thing.  That way, we don't "precompute" checksums
- *  for unsuspecting consumers.
- * Assumption: m->m_nextpkt == 0.
- * Trick: for small packets, don't dup into a cluster.  That way received
- *  packets don't take up too much room in the sockbuf (cf. sbspace()).
+ * for unsuspecting consumers.  Assumption: m->m_nextpkt == 0.  Trick: for
+ * small packets, don't dup into a cluster.  That way received  packets
+ * don't take up too much room in the sockbuf (cf. sbspace()).
  */
 int MDFail;
 
 struct mbuf *
 m_dup(struct mbuf *m, int how)
-{	
+{
 	struct mbuf *n, **np;
 	struct mbuf *top;
 	int copyhdr = 0;
 
 	np = &top;
-	top = 0;
+	top = NULL;
 	if (m->m_flags & M_PKTHDR)
 		copyhdr = 1;
 
@@ -2139,172 +4526,766 @@ m_dup(struct mbuf *m, int how)
 	 * Quick check: if we have one mbuf and its data fits in an
 	 *  mbuf with packet header, just copy and go.
 	 */
-	if (m->m_next == NULL)
-	{	/* Then just move the data into an mbuf and be done... */
-		if (copyhdr)
-		{	if (m->m_pkthdr.len <= MHLEN)
-			{	if ((n = m_gethdr(how, m->m_type)) == NULL)
-					return(NULL);
+	if (m->m_next == NULL) {
+		/* Then just move the data into an mbuf and be done... */
+		if (copyhdr) {
+			if (m->m_pkthdr.len <= MHLEN && m->m_len <= MHLEN) {
+				if ((n = _M_GETHDR(how, m->m_type)) == NULL)
+					return (NULL);
 				n->m_len = m->m_len;
 				m_dup_pkthdr(n, m, how);
 				bcopy(m->m_data, n->m_data, m->m_len);
-				return(n);
+				return (n);
 			}
-		} else if (m->m_len <= MLEN)
-		{	if ((n = m_get(how, m->m_type)) == NULL)
-				return(NULL);
+		} else if (m->m_len <= MLEN) {
+			if ((n = _M_GET(how, m->m_type)) == NULL)
+				return (NULL);
 			bcopy(m->m_data, n->m_data, m->m_len);
 			n->m_len = m->m_len;
-			return(n);
+			return (n);
 		}
 	}
-	while (m)
-	{
+	while (m != NULL) {
 #if BLUE_DEBUG
 		kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
-			m->m_data);
+		    m->m_data);
 #endif
 		if (copyhdr)
-			n = m_gethdr(how, m->m_type);
+			n = _M_GETHDR(how, m->m_type);
 		else
-			n = m_get(how, m->m_type);
-		if (n == 0)
+			n = _M_GET(how, m->m_type);
+		if (n == NULL)
 			goto nospace;
-		if (m->m_flags & M_EXT)
-		{	MCLGET(n, how);
-			if ((n->m_flags & M_EXT) == 0)
+		if (m->m_flags & M_EXT) {
+			if (m->m_len <= m_maxsize(MC_CL))
+				MCLGET(n, how);
+			else if (m->m_len <= m_maxsize(MC_BIGCL))
+				n = m_mbigget(n, how);
+			else if (m->m_len <= m_maxsize(MC_16KCL) && njcl > 0)
+				n = m_m16kget(n, how);
+			if (!(n->m_flags & M_EXT)) {
+				(void) m_free(n);
 				goto nospace;
+			}
 		}
 		*np = n;
-		if (copyhdr)
-		{	/* Don't use M_COPY_PKTHDR: preserve m_data */
+		if (copyhdr) {
+			/* Don't use M_COPY_PKTHDR: preserve m_data */
 			m_dup_pkthdr(n, m, how);
 			copyhdr = 0;
-			if ((n->m_flags & M_EXT) == 0)
+			if (!(n->m_flags & M_EXT))
 				n->m_data = n->m_pktdat;
 		}
 		n->m_len = m->m_len;
 		/*
 		 * Get the dup on the same bdry as the original
 		 * Assume that the two mbufs have the same offset to data area
-		 *  (up to word bdries)
+		 * (up to word boundaries)
 		 */
-		bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
+		bcopy(MTOD(m, caddr_t), MTOD(n, caddr_t), (unsigned)n->m_len);
 		m = m->m_next;
 		np = &n->m_next;
 #if BLUE_DEBUG
 		kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
-			n->m_data);
+		    n->m_data);
 #endif
 	}
 
-	if (top == 0)
+	if (top == NULL)
 		MDFail++;
 	return (top);
- nospace:
+
+nospace:
 	m_freem(top);
 	MDFail++;
-	return (0);
+	return (NULL);
 }
 
-int
-m_mclref(struct mbuf *p)
+#define	MBUF_MULTIPAGES(m)						\
+	(((m)->m_flags & M_EXT) &&					\
+	((IS_P2ALIGNED((m)->m_data, NBPG) && (m)->m_len > NBPG) ||	\
+	(!IS_P2ALIGNED((m)->m_data, NBPG) &&				\
+	P2ROUNDUP((m)->m_data, NBPG) < ((uintptr_t)(m)->m_data + (m)->m_len))))
+
+static struct mbuf *
+m_expand(struct mbuf *m, struct mbuf **last)
 {
-	return (_MCLREF(p));
+	struct mbuf *top = NULL;
+	struct mbuf **nm = &top;
+	uintptr_t data0, data;
+	unsigned int len0, len;
+
+	VERIFY(MBUF_MULTIPAGES(m));
+	VERIFY(m->m_next == NULL);
+	data0 = (uintptr_t)m->m_data;
+	len0 = m->m_len;
+	*last = top;
+
+	for (;;) {
+		struct mbuf *n;
+
+		data = data0;
+		if (IS_P2ALIGNED(data, NBPG) && len0 > NBPG)
+			len = NBPG;
+		else if (!IS_P2ALIGNED(data, NBPG) &&
+		    P2ROUNDUP(data, NBPG) < (data + len0))
+			len = P2ROUNDUP(data, NBPG) - data;
+		else
+			len = len0;
+
+		VERIFY(len > 0);
+		VERIFY(m->m_flags & M_EXT);
+		m->m_data = (void *)data;
+		m->m_len = len;
+
+		*nm = *last = m;
+		nm = &m->m_next;
+		m->m_next = NULL;
+
+		data0 += len;
+		len0 -= len;
+		if (len0 == 0)
+			break;
+
+		n = _M_RETRY(M_DONTWAIT, MT_DATA);
+		if (n == NULL) {
+			m_freem(top);
+			top = *last = NULL;
+			break;
+		}
+
+		n->m_ext = m->m_ext;
+		m_incref(m);
+		n->m_flags |= M_EXT;
+		m = n;
+	}
+	return (top);
 }
 
-int
-m_mclunref(struct mbuf *p)
+struct mbuf *
+m_normalize(struct mbuf *m)
 {
-	return (_MCLUNREF(p));
+	struct mbuf *top = NULL;
+	struct mbuf **nm = &top;
+	boolean_t expanded = FALSE;
+
+	while (m != NULL) {
+		struct mbuf *n;
+
+		n = m->m_next;
+		m->m_next = NULL;
+
+		/* Does the data cross one or more page boundaries? */
+		if (MBUF_MULTIPAGES(m)) {
+			struct mbuf *last;
+			if ((m = m_expand(m, &last)) == NULL) {
+				m_freem(n);
+				m_freem(top);
+				top = NULL;
+				break;
+			}
+			*nm = m;
+			nm = &last->m_next;
+			expanded = TRUE;
+		} else {
+			*nm = m;
+			nm = &m->m_next;
+		}
+		m = n;
+	}
+	if (expanded)
+		atomic_add_32(&mb_normalized, 1);
+	return (top);
 }
 
-/* change mbuf to new type */
 void
 m_mchtype(struct mbuf *m, int t)
 {
-        MBUF_LOCK();
-        mbstat.m_mtypes[(m)->m_type]--;
-        mbstat.m_mtypes[t]++;
-        (m)->m_type = t;
-        MBUF_UNLOCK();
+	mtype_stat_inc(t);
+	mtype_stat_dec(m->m_type);
+	(m)->m_type = t;
 }
 
-void *m_mtod(struct mbuf *m)
+void *
+m_mtod(struct mbuf *m)
 {
-	return ((m)->m_data);
+	return (MTOD(m, void *));
 }
 
-struct mbuf *m_dtom(void *x)
+struct mbuf *
+m_dtom(void *x)
 {
 	return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
 }
 
-int m_mtocl(void *x)
+void
+m_mcheck(struct mbuf *m)
 {
-	return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
+	_MCHECK(m);
 }
 
-union mcluster *m_cltom(int x)
+/*
+ * Inform the corresponding mcache(s) that there's a waiter below.
+ */
+static void
+mbuf_waiter_inc(mbuf_class_t class, boolean_t comp)
 {
-	return ((union mcluster *)(mbutl + (x)));
+	mcache_waiter_inc(m_cache(class));
+	if (comp) {
+		if (class == MC_CL) {
+			mcache_waiter_inc(m_cache(MC_MBUF_CL));
+		} else if (class == MC_BIGCL) {
+			mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
+		} else if (class == MC_16KCL) {
+			mcache_waiter_inc(m_cache(MC_MBUF_16KCL));
+		} else {
+			mcache_waiter_inc(m_cache(MC_MBUF_CL));
+			mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
+		}
+	}
 }
 
+/*
+ * Inform the corresponding mcache(s) that there's no more waiter below.
+ */
+static void
+mbuf_waiter_dec(mbuf_class_t class, boolean_t comp)
+{
+	mcache_waiter_dec(m_cache(class));
+	if (comp) {
+		if (class == MC_CL) {
+			mcache_waiter_dec(m_cache(MC_MBUF_CL));
+		} else if (class == MC_BIGCL) {
+			mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
+		} else if (class == MC_16KCL) {
+			mcache_waiter_dec(m_cache(MC_MBUF_16KCL));
+		} else {
+			mcache_waiter_dec(m_cache(MC_MBUF_CL));
+			mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
+		}
+	}
+}
 
-void m_mcheck(struct mbuf *m)
+/*
+ * Called during blocking allocation.  Returns TRUE if one or more objects
+ * are available at the per-CPU caches layer and that allocation should be
+ * retried at that level.
+ */
+static boolean_t
+mbuf_sleep(mbuf_class_t class, unsigned int num, int wait)
 {
-	if (m->m_type != MT_FREE) 
-		panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
+	boolean_t mcache_retry = FALSE;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	/* Check if there's anything at the cache layer */
+	if (mbuf_cached_above(class, wait)) {
+		mcache_retry = TRUE;
+		goto done;
+	}
+
+	/* Nothing?  Then try hard to get it from somewhere */
+	m_reclaim(class, num, (wait & MCR_COMP));
+
+	/* We tried hard and got something? */
+	if (m_infree(class) > 0) {
+		mbstat.m_wait++;
+		goto done;
+	} else if (mbuf_cached_above(class, wait)) {
+		mbstat.m_wait++;
+		mcache_retry = TRUE;
+		goto done;
+	} else if (wait & MCR_TRYHARD) {
+		mcache_retry = TRUE;
+		goto done;
+	}
+
+	/*
+	 * There's really nothing for us right now; inform the
+	 * cache(s) that there is a waiter below and go to sleep.
+	 */
+	mbuf_waiter_inc(class, (wait & MCR_COMP));
+
+	VERIFY(!(wait & MCR_NOSLEEP));
+	mb_waiters++;
+	(void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL);
+
+	/* We are now up; stop getting notified until next round */
+	mbuf_waiter_dec(class, (wait & MCR_COMP));
+
+	/* We waited and got something */
+	if (m_infree(class) > 0) {
+		mbstat.m_wait++;
+		goto done;
+	} else if (mbuf_cached_above(class, wait)) {
+		mbstat.m_wait++;
+		mcache_retry = TRUE;
+	}
+done:
+	return (mcache_retry);
 }
 
 static void
-mbuf_expand_thread(void)
+mbuf_worker_thread(void)
 {
+	int mbuf_expand;
+
 	while (1) {
-		MBUF_LOCK();
+		lck_mtx_lock(mbuf_mlock);
+
+		mbuf_expand = 0;
 		if (mbuf_expand_mcl) {
 			int n;
-		        
-		    /* Adjust to the current number of cluster in use */
-			n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
+
+			/* Adjust to current number of cluster in use */
+			n = mbuf_expand_mcl -
+			    (m_total(MC_CL) - m_infree(MC_CL));
+			if ((n + m_total(MC_CL)) > m_maxlimit(MC_CL))
+				n = m_maxlimit(MC_CL) - m_total(MC_CL);
 			mbuf_expand_mcl = 0;
-		    
-		    if (n > 0)
-				(void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
+
+			if (n > 0 && freelist_populate(MC_CL, n, M_WAIT) > 0)
+				mbuf_expand++;
 		}
 		if (mbuf_expand_big) {
 			int n;
-		        
-		    /* Adjust to the current number of 4 KB cluster in use */
-			n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
+
+			/* Adjust to current number of 4 KB cluster in use */
+			n = mbuf_expand_big -
+			    (m_total(MC_BIGCL) - m_infree(MC_BIGCL));
+			if ((n + m_total(MC_BIGCL)) > m_maxlimit(MC_BIGCL))
+				n = m_maxlimit(MC_BIGCL) - m_total(MC_BIGCL);
 			mbuf_expand_big = 0;
-		        
-		    if (n > 0)
-				(void)m_clalloc(n, M_WAIT, NBPG, 1);
-        }
-		MBUF_UNLOCK();
-		/* 
-		 * Because we can run out of memory before filling the mbuf map, we 
-		 * should not allocate more clusters than they are mbufs -- otherwise
-		 * we could have a large number of useless clusters allocated.
+
+			if (n > 0 && freelist_populate(MC_BIGCL, n, M_WAIT) > 0)
+				mbuf_expand++;
+		}
+		if (mbuf_expand_16k) {
+			int n;
+
+			/* Adjust to current number of 16 KB cluster in use */
+			n = mbuf_expand_16k -
+			    (m_total(MC_16KCL) - m_infree(MC_16KCL));
+			if ((n + m_total(MC_16KCL)) > m_maxlimit(MC_16KCL))
+				n = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
+			mbuf_expand_16k = 0;
+
+			if (n > 0)
+				(void) freelist_populate(MC_16KCL, n, M_WAIT);
+		}
+
+		/*
+		 * Because we can run out of memory before filling the mbuf
+		 * map, we should not allocate more clusters than they are
+		 * mbufs -- otherwise we could have a large number of useless
+		 * clusters allocated.
 		 */
-		while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
-			if (m_expand(M_WAIT) == 0)
-				break;
+		if (mbuf_expand) {
+			while (m_total(MC_MBUF) <
+			    (m_total(MC_BIGCL) + m_total(MC_CL))) {
+				if (freelist_populate(MC_MBUF, 1, M_WAIT) == 0)
+					break;
+			}
 		}
-	
-		assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
-		(void) thread_block((thread_continue_t)mbuf_expand_thread);
+
+		lck_mtx_unlock(mbuf_mlock);
+
+		assert_wait(&mbuf_worker_run, THREAD_UNINT);
+		(void) thread_block((thread_continue_t)mbuf_worker_thread);
 	}
 }
 
 static void
-mbuf_expand_thread_init(void)
+mbuf_worker_thread_init(void)
 {
-	mbuf_expand_thread_initialized++;
-	mbuf_expand_thread();
+	mbuf_worker_ready++;
+	mbuf_worker_thread();
 }
 
-SYSCTL_DECL(_kern_ipc);
-SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
+static mcl_slab_t *
+slab_get(void *buf)
+{
+	mcl_slabg_t *slg;
+	unsigned int ix, k;
+
+	lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+	VERIFY(MBUF_IN_MAP(buf));
+	ix = ((char *)buf - (char *)mbutl) >> MBSHIFT;
+	VERIFY(ix < maxslabgrp);
+
+	if ((slg = slabstbl[ix]) == NULL) {
+		/*
+		 * In the current implementation, we never shrink the memory
+		 * pool (hence the cluster map); if we attempt to reallocate
+		 * a cluster group when it's already allocated, panic since
+		 * this is a sign of a memory corruption (slabstbl[ix] got
+		 * nullified).  This also means that there shouldn't be any
+		 * hole in the kernel sub-map for the mbuf pool.
+		 */
+		++slabgrp;
+		VERIFY(ix < slabgrp);
+		/*
+		 * Slabs expansion can only be done single threaded; when
+		 * we get here, it must be as a result of m_clalloc() which
+		 * is serialized and therefore mb_clalloc_busy must be set.
+		 */
+		VERIFY(mb_clalloc_busy);
+		lck_mtx_unlock(mbuf_mlock);
+
+		/* This is a new buffer; create the slabs group for it */
+		MALLOC(slg, mcl_slabg_t *, sizeof (*slg), M_TEMP,
+		    M_WAITOK | M_ZERO);
+		VERIFY(slg != NULL);
+
+		lck_mtx_lock(mbuf_mlock);
+		/*
+		 * No other thread could have gone into m_clalloc() after
+		 * we dropped the lock above, so verify that it's true.
+		 */
+		VERIFY(mb_clalloc_busy);
+
+		slabstbl[ix] = slg;
+
+		/* Chain each slab in the group to its forward neighbor */
+		for (k = 1; k < NSLABSPMB; k++)
+			slg->slg_slab[k - 1].sl_next = &slg->slg_slab[k];
+		VERIFY(slg->slg_slab[NSLABSPMB - 1].sl_next == NULL);
+
+		/* And chain the last slab in the previous group to this */
+		if (ix > 0) {
+			VERIFY(slabstbl[ix - 1]->
+			    slg_slab[NSLABSPMB - 1].sl_next == NULL);
+			slabstbl[ix - 1]->slg_slab[NSLABSPMB - 1].sl_next =
+			    &slg->slg_slab[0];
+		}
+	}
+
+	ix = MTOCL(buf) % NSLABSPMB;
+	VERIFY(ix < NSLABSPMB);
+
+	return (&slg->slg_slab[ix]);
+}
+
+static void
+slab_init(mcl_slab_t *sp, mbuf_class_t class, u_int32_t flags,
+    void *base, void *head, unsigned int len, int refcnt, int chunks)
+{
+	sp->sl_class = class;
+	sp->sl_flags = flags;
+	sp->sl_base = base;
+	sp->sl_head = head;
+	sp->sl_len = len;
+	sp->sl_refcnt = refcnt;
+	sp->sl_chunks = chunks;
+	slab_detach(sp);
+}
+
+static void
+slab_insert(mcl_slab_t *sp, mbuf_class_t class)
+{
+	VERIFY(slab_is_detached(sp));
+	m_slab_cnt(class)++;
+	TAILQ_INSERT_TAIL(&m_slablist(class), sp, sl_link);
+	sp->sl_flags &= ~SLF_DETACHED;
+	if (class == MC_BIGCL) {
+		sp = sp->sl_next;
+		/* Next slab must already be present */
+		VERIFY(sp != NULL);
+		VERIFY(slab_is_detached(sp));
+		sp->sl_flags &= ~SLF_DETACHED;
+	} else if (class == MC_16KCL) {
+		int k;
+		for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) {
+			sp = sp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(sp != NULL);
+			VERIFY(slab_is_detached(sp));
+			sp->sl_flags &= ~SLF_DETACHED;
+		}
+	}
+}
+
+static void
+slab_remove(mcl_slab_t *sp, mbuf_class_t class)
+{
+	VERIFY(!slab_is_detached(sp));
+	VERIFY(m_slab_cnt(class) > 0);
+	m_slab_cnt(class)--;
+	TAILQ_REMOVE(&m_slablist(class), sp, sl_link);
+	slab_detach(sp);
+	if (class == MC_BIGCL) {
+		sp = sp->sl_next;
+		/* Next slab must already be present */
+		VERIFY(sp != NULL);
+		VERIFY(!slab_is_detached(sp));
+		slab_detach(sp);
+	} else if (class == MC_16KCL) {
+		int k;
+		for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) {
+			sp = sp->sl_next;
+			/* Next slab must already be present */
+			VERIFY(sp != NULL);
+			VERIFY(!slab_is_detached(sp));
+			slab_detach(sp);
+		}
+	}
+}
+
+static boolean_t
+slab_inrange(mcl_slab_t *sp, void *buf)
+{
+	return ((uintptr_t)buf >= (uintptr_t)sp->sl_base &&
+	    (uintptr_t)buf < ((uintptr_t)sp->sl_base + sp->sl_len));
+}
+
+#undef panic(...)
+
+static void
+slab_nextptr_panic(mcl_slab_t *sp, void *addr)
+{
+	int i;
+	unsigned int chunk_len = sp->sl_len / sp->sl_chunks;
+	uintptr_t buf = (uintptr_t)sp->sl_base;
+
+	for (i = 0; i < sp->sl_chunks; i++, buf += chunk_len) {
+		void *next = ((mcache_obj_t *)buf)->obj_next;
+		if (next != addr)
+			continue;
+		if (mclaudit == NULL) {
+			if (next != NULL && !MBUF_IN_MAP(next)) {
+				mcache_t *cp = m_cache(sp->sl_class);
+				panic("%s: %s buffer %p in slab %p modified "
+				    "after free at offset 0: %p out of range "
+				    "[%p-%p)\n", __func__, cp->mc_name,
+				    (void *)buf, sp, next, mbutl, embutl);
+				/* NOTREACHED */
+			}
+		} else {
+			mcache_audit_t *mca = mcl_audit_buf2mca(sp->sl_class,
+			    (mcache_obj_t *)buf);
+			mcl_audit_verify_nextptr(next, mca);
+		}
+	}
+}
+
+static void
+slab_detach(mcl_slab_t *sp)
+{
+	sp->sl_link.tqe_next = (mcl_slab_t *)-1;
+	sp->sl_link.tqe_prev = (mcl_slab_t **)-1;
+	sp->sl_flags |= SLF_DETACHED;
+}
+
+static boolean_t
+slab_is_detached(mcl_slab_t *sp)
+{
+	return ((intptr_t)sp->sl_link.tqe_next == -1 &&
+	    (intptr_t)sp->sl_link.tqe_prev == -1 &&
+	    (sp->sl_flags & SLF_DETACHED));
+}
+
+static void
+mcl_audit_init(void *buf, mcache_audit_t **mca_list,
+    mcache_obj_t **con_list, size_t con_size, unsigned int num)
+{
+	mcache_audit_t *mca, *mca_tail;
+	mcache_obj_t *con = NULL;
+	boolean_t save_contents = (con_list != NULL);
+	unsigned int i, ix;
+
+	ASSERT(num <= NMBPCL);
+	ASSERT(con_list == NULL || con_size != 0);
+
+	ix = MTOCL(buf);
+	/* Make sure we haven't been here before */
+	for (i = 0; i < NMBPCL; i++)
+		VERIFY(mclaudit[ix].cl_audit[i] == NULL);
+
+	mca = mca_tail = *mca_list;
+	if (save_contents)
+		con = *con_list;
+
+	for (i = 0; i < num; i++) {
+		mcache_audit_t *next;
+
+		next = mca->mca_next;
+		bzero(mca, sizeof (*mca));
+		mca->mca_next = next;
+		mclaudit[ix].cl_audit[i] = mca;
+
+		/* Attach the contents buffer if requested */
+		if (save_contents) {
+			VERIFY(con != NULL);
+			mca->mca_contents_size = con_size;
+			mca->mca_contents = con;
+			con = con->obj_next;
+			bzero(mca->mca_contents, mca->mca_contents_size);
+		}
+
+		mca_tail = mca;
+		mca = mca->mca_next;
+	}
 
+	if (save_contents)
+		*con_list = con;
+
+	*mca_list = mca_tail->mca_next;
+	mca_tail->mca_next = NULL;
+}
+
+/*
+ * Given an address of a buffer (mbuf/cluster/big cluster), return
+ * the corresponding audit structure for that buffer.
+ */
+static mcache_audit_t *
+mcl_audit_buf2mca(mbuf_class_t class, mcache_obj_t *o)
+{
+	mcache_audit_t *mca = NULL;
+	int ix = MTOCL(o);
+
+	VERIFY(IS_P2ALIGNED(o, MIN(m_maxsize(class), NBPG)));
+
+	switch (class) {
+	case MC_MBUF:
+		/*
+		 * For the mbuf case, find the index of the cluster
+		 * used by the mbuf and use that index to locate the
+		 * base address of the cluster.  Then find out the
+		 * mbuf index relative to the cluster base and use
+		 * it to locate the audit structure.
+		 */
+		VERIFY(MCLIDX(CLTOM(ix), o) < (int)NMBPCL);
+		mca = mclaudit[ix].cl_audit[MCLIDX(CLTOM(ix), o)];
+		break;
+
+	case MC_CL:
+	case MC_BIGCL:
+	case MC_16KCL:
+		/*
+		 * Same as above, but only return the first element.
+		 */
+		mca = mclaudit[ix].cl_audit[0];
+		break;
+
+	default:
+		VERIFY(0);
+		/* NOTREACHED */
+	}
+
+	return (mca);
+}
+
+static void
+mcl_audit_mbuf(mcache_audit_t *mca, void *addr, boolean_t composite,
+    boolean_t alloc)
+{
+	struct mbuf *m = addr;
+	mcache_obj_t *next = ((mcache_obj_t *)m)->obj_next;
+
+	VERIFY(mca->mca_contents != NULL &&
+	    mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
+
+	mcl_audit_verify_nextptr(next, mca);
+
+	if (!alloc) {
+		/* Save constructed mbuf fields */
+		mcl_audit_save_mbuf(m, mca);
+		mcache_set_pattern(MCACHE_FREE_PATTERN, m, m_maxsize(MC_MBUF));
+		((mcache_obj_t *)m)->obj_next = next;
+		return;
+	}
+
+	/* Check if the buffer has been corrupted while in freelist */
+	mcache_audit_free_verify_set(mca, addr, 0, m_maxsize(MC_MBUF));
+
+	/* Restore constructed mbuf fields */
+	mcl_audit_restore_mbuf(m, mca, composite);
+}
+
+static void
+mcl_audit_restore_mbuf(struct mbuf *m, mcache_audit_t *mca, boolean_t composite)
+{
+	struct mbuf *ms = (struct mbuf *)mca->mca_contents;
+
+	if (composite) {
+		struct mbuf *next = m->m_next;
+		VERIFY(ms->m_flags == M_EXT && MEXT_RFA(ms) != NULL &&
+		    MBUF_IS_COMPOSITE(ms));
+		/*
+		 * We could have hand-picked the mbuf fields and restore
+		 * them individually, but that will be a maintenance
+		 * headache.  Instead, restore everything that was saved;
+		 * the mbuf layer will recheck and reinitialize anyway.
+		 */
+		bcopy(ms, m, mca->mca_contents_size);
+		m->m_next = next;
+	} else {
+		/*
+		 * For a regular mbuf (no cluster attached) there's nothing
+		 * to restore other than the type field, which is expected
+		 * to be MT_FREE.
+		 */
+		m->m_type = ms->m_type;
+	}
+	_MCHECK(m);
+}
+
+static void
+mcl_audit_save_mbuf(struct mbuf *m, mcache_audit_t *mca)
+{
+	_MCHECK(m);
+	bcopy(m, mca->mca_contents, mca->mca_contents_size);
+}
+
+static void
+mcl_audit_cluster(mcache_audit_t *mca, void *addr, size_t size, boolean_t alloc,
+    boolean_t save_next)
+{
+	mcache_obj_t *next = ((mcache_obj_t *)addr)->obj_next;
+
+	if (!alloc) {
+		mcache_set_pattern(MCACHE_FREE_PATTERN, addr, size);
+		if (save_next) {
+			mcl_audit_verify_nextptr(next, mca);
+			((mcache_obj_t *)addr)->obj_next = next;
+		}
+	} else {
+		/* Check if the buffer has been corrupted while in freelist */
+		mcl_audit_verify_nextptr(next, mca);
+		mcache_audit_free_verify_set(mca, addr, 0, size);
+	}
+}
+
+static void
+mcl_audit_mcheck_panic(struct mbuf *m)
+{
+	mcache_audit_t *mca;
+
+	MRANGE(m);
+	mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+
+	panic("mcl_audit: freed mbuf %p with type 0x%x (instead of 0x%x)\n%s\n",
+	    m, (u_int16_t)m->m_type, MT_FREE, mcache_dump_mca(mca));
+	/* NOTREACHED */
+}
+
+static void
+mcl_audit_verify_nextptr(void *next, mcache_audit_t *mca)
+{
+	if (next != NULL && next != (void *)MCACHE_FREE_PATTERN &&
+	    !MBUF_IN_MAP(next)) {
+		panic("mcl_audit: buffer %p modified after free at offset 0: "
+		    "%p out of range [%p-%p)\n%s\n",
+		    mca->mca_addr, next, mbutl, embutl, mcache_dump_mca(mca));
+		/* NOTREACHED */
+	}
+}
+
+SYSCTL_DECL(_kern_ipc);
+SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RD | CTLFLAG_LOCKED,
+    0, 0, mbstat_sysctl, "S,mbstat", "");
+SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_stat, CTLFLAG_RD | CTLFLAG_LOCKED,
+    0, 0, mb_stat_sysctl, "S,mb_stat", "");
+SYSCTL_INT(_kern_ipc, OID_AUTO, mb_normalized, CTLFLAG_RD | CTLFLAG_LOCKED,
+    &mb_normalized, 0, "");
diff --git a/bsd/kern/uipc_mbuf2.c b/bsd/kern/uipc_mbuf2.c
index 3a4a3a14f..d5ea69c86 100644
--- a/bsd/kern/uipc_mbuf2.c
+++ b/bsd/kern/uipc_mbuf2.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $	*/
 
@@ -84,6 +90,12 @@
  *
  *	@(#)uipc_mbuf.c	8.4 (Berkeley) 2/14/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 
 /*#define PULLDOWN_DEBUG*/
@@ -99,6 +111,10 @@
 #include <netinet6/ip6_var.h>
 #endif
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 /*
  * ensure that [off, off + len) is contiguous on the mbuf chain "m".
  * packet chain before "off" is kept untouched.
@@ -110,10 +126,7 @@
  * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster)
  */
 struct mbuf *
-m_pulldown(m, off, len, offp)
-	struct mbuf *m;
-	int off, len;
-	int *offp;
+m_pulldown(struct mbuf *m, int off, int len, int *offp)
 {
 	struct mbuf *n, *o;
 	int hlen, tlen, olen;
@@ -352,127 +365,6 @@ ok:
 	return n;
 }
 
-/*
- * pkthdr.aux chain manipulation.
- * we don't allow clusters at this moment. 
- */
-struct mbuf *
-m_aux_add(m, af, type)
-	struct mbuf *m;
-	int af, type;
-{
-	struct mbuf *n;
-	struct mauxtag *t;
-
-	if ((m->m_flags & M_PKTHDR) == 0)
-		return NULL;
-
-	n = m_aux_find(m, af, type);
-	if (n)
-		return n;
-
-	MGET(n, M_DONTWAIT, m->m_type);
-	if (n == NULL)
-		return NULL;
-
-	t = mtod(n, struct mauxtag *);
-	t->af = af;
-	t->type = type;
-	n->m_data += sizeof(struct mauxtag);
-	n->m_len = 0;
-	n->m_next = m->m_pkthdr.aux;
-	m->m_pkthdr.aux = n;
-	return n;
-}
-
-struct mbuf *
-m_aux_find(m, af, type)
-	struct mbuf *m;
-	int af, type;
-{
-	struct mbuf *n;
-	struct mauxtag *t;
-
-	if ((m->m_flags & M_PKTHDR) == 0)
-		return NULL;
-
-	for (n = m->m_pkthdr.aux; n; n = n->m_next) {
-		t = (struct mauxtag *)n->m_dat;
-		if (t->af == af && t->type == type)
-			return n;
-	}
-	return NULL;
-}
-
-void
-m_aux_delete(m, victim)
-	struct mbuf *m;
-	struct mbuf *victim;
-{
-	struct mbuf *n, *prev, *next;
-	struct mauxtag *t;
-
-	if ((m->m_flags & M_PKTHDR) == 0)
-		return;
-
-	prev = NULL;
-	n = m->m_pkthdr.aux;
-	while (n) {
-		t = (struct mauxtag *)n->m_dat;
-		next = n->m_next;
-		if (n == victim) {
-			if (prev)
-				prev->m_next = n->m_next;
-			else
-				m->m_pkthdr.aux = n->m_next;
-			n->m_next = NULL;
-			m_free(n);
-		} else
-			prev = n;
-		n = next;
-	}
-}
-
-struct mbuf *
-m_aux_copy(struct mbuf *to, struct mbuf *from)
-{
-	struct mbuf *m;
-	struct mbuf *top = NULL, **np = &top;
-
-	if (!(to->m_flags & M_PKTHDR) || !(from->m_flags & M_PKTHDR))
-		return (NULL);
-
-	if ((m = from->m_pkthdr.aux) == NULL)
-		return (NULL);
-
-	while (m != NULL) {
-		struct mbuf *n;
-
-		MGET(n, M_DONTWAIT, m->m_type);
-		if (n == NULL) {
-			m_freem(top);
-			return (NULL);
-		}
-
-		/* Set length and data offset accordingly */
-		n->m_len = m->m_len;
-		n->m_data += (m->m_data - m->m_dat);
-
-		/* Copy databuf (mauxtag + possible aux data) */
-		bcopy(m->m_dat, n->m_dat, sizeof (m->m_dat));
-
-		*np = n;
-		np = &n->m_next;
-		m = m->m_next;
-	}
-
-	if (to->m_pkthdr.aux != NULL)
-		m_freem(to->m_pkthdr.aux);
-
-	to->m_pkthdr.aux = top;
-	return (top);
-}
-
 /* Get a packet tag structure along with specified data following. */
 struct m_tag *
 m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
@@ -489,9 +381,9 @@ m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
 		struct mbuf *m = m_get(wait, MT_TAG);
 		if (m == NULL)
 			return NULL;
-		t = (struct m_tag *) m->m_dat;
+		t = mtod(m, struct m_tag *);
         } else if (len + sizeof(struct m_tag) <= MCLBYTES) {
-		MCLALLOC((caddr_t)t, wait);
+        	t = (struct m_tag *) m_mclalloc(wait);
         } else
                 t = NULL;
 #endif
@@ -508,13 +400,19 @@ m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
 void
 m_tag_free(struct m_tag *t)
 {
+#if CONFIG_MACF_NET
+	if (t != NULL && 
+	    t->m_tag_id   == KERNEL_MODULE_TAG_ID &&
+	    t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL)
+		mac_mbuf_tag_destroy(t);
+#endif
 #ifndef __APPLE__
 	free(t, M_PACKET_TAGS);
 #else
 	/* FREE(t, M_TEMP); */
 	if (t == NULL)
 		return;
-	if (t->m_tag_len <= MLEN) {
+	if (t->m_tag_len + sizeof(struct m_tag) <= MLEN) {
 		struct mbuf * m = m_dtom(t);
 		m_free(m);
 	} else {
@@ -595,6 +493,22 @@ m_tag_copy(struct m_tag *t, int how)
 	p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how);
 	if (p == NULL)
 		return (NULL);
+#if CONFIG_MACF_NET
+	/*
+	 * XXXMAC: we should probably pass off the initialization, and
+	 * copying here?  can we hid that KERNEL_TAG_TYPE_MACLABEL is
+	 * special from the mbuf code?
+	 */
+	if (t != NULL &&
+	    t->m_tag_id   == KERNEL_MODULE_TAG_ID &&
+	    t->m_tag_type == KERNEL_TAG_TYPE_MACLABEL) {
+		if (mac_mbuf_tag_init(p, how) != 0) {
+			m_tag_free(p);
+			return (NULL);
+		}
+		mac_mbuf_tag_copy(t, p);
+	} else
+#endif
 	bcopy(t + 1, p + 1, t->m_tag_len); /* Copy the data */
 	return p;
 }
diff --git a/bsd/kern/uipc_proto.c b/bsd/kern/uipc_proto.c
index 6fd419ddd..a77e126d9 100644
--- a/bsd/kern/uipc_proto.c
+++ b/bsd/kern/uipc_proto.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -69,59 +75,54 @@
  * Definitions of protocols supported in the UNIX domain.
  */
 
-int	uipc_usrreq(), raw_usrreq();
-void	raw_init(), raw_input(), raw_ctlinput();
-extern	struct domain localdomain;		/* or at least forward */
+int	raw_usrreq(void);
+static void pre_unp_init(void) __attribute__((section("__TEXT, initcode")));
 
 static struct protosw localsw[] = {
-{ SOCK_STREAM,	&localdomain,	0,	PR_CONNREQUIRED|PR_WANTRCVD|PR_RIGHTS,
-  0,		0,		0,		uipc_ctloutput,
-  0,
-  0,		0,		0,		0,
-  0,	
-  &uipc_usrreqs,
-  0,		0,		0
-
-},
-{ SOCK_DGRAM,	&localdomain,	0,		PR_ATOMIC|PR_ADDR|PR_RIGHTS,
-  0,		0,		0,		uipc_ctloutput,
-  0,
-  0,		0,		0,		0,
-  0,	
-  &uipc_usrreqs,
-  0,		0,		0
-},
-{ 0,		0,		0,		0,
-  0,		0,		raw_ctlinput,	0,
-  0,
-  0,		0,		0,		0,
-  0,
-  &raw_usrreqs,
-  0,			0,		0
-}
+	{
+		.pr_type = SOCK_STREAM,
+		.pr_domain = &localdomain,
+		.pr_flags = PR_CONNREQUIRED|PR_WANTRCVD|PR_RIGHTS,
+		.pr_ctloutput = uipc_ctloutput,
+		.pr_usrreqs = &uipc_usrreqs,
+	},
+	{
+		.pr_type = SOCK_DGRAM,
+		.pr_domain = &localdomain,
+		.pr_flags = PR_ATOMIC|PR_ADDR|PR_RIGHTS,
+		.pr_ctloutput = uipc_ctloutput,
+		.pr_usrreqs = &uipc_usrreqs,
+	},
+	{
+		.pr_ctlinput  = raw_ctlinput,
+		.pr_usrreqs = &raw_usrreqs,
+	},
 };
 
 int local_proto_count = (sizeof (localsw) / sizeof (struct protosw));
 
-static void pre_unp_init()
-{	static int localdomain_initted = 0;
-	register int i;
-	register struct protosw *pr;
-	register struct domain *dp = &localdomain;
+static void
+pre_unp_init(void)
+{
+	static int localdomain_initted = 0;
+	int i;
+	struct protosw *pr;
+	struct domain *dp = &localdomain;
 
 	for (i=0, pr = &localsw[0]; i<local_proto_count; i++, pr++)
 		net_add_proto(pr, dp);
 	unp_init();
 }
 
-struct domain localdomain =
-    { AF_LOCAL, "unix", pre_unp_init, unp_externalize, unp_dispose,
-      localsw};
-
-
-
-
-SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW, 0, "Local domain");
-SYSCTL_NODE(_net_local, SOCK_STREAM, stream, CTLFLAG_RW, 0, "SOCK_STREAM");
-SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, CTLFLAG_RW, 0, "SOCK_DGRAM");
+struct domain localdomain = {
+	.dom_family = AF_LOCAL,
+	.dom_name = "unix",
+	.dom_init = pre_unp_init,
+	.dom_externalize = unp_externalize,
+	.dom_dispose = unp_dispose,
+	.dom_protosw = localsw
+};
 
+SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW|CTLFLAG_LOCKED, NULL, "Local domain");
+SYSCTL_NODE(_net_local, SOCK_STREAM, stream, CTLFLAG_RW|CTLFLAG_LOCKED, NULL, "SOCK_STREAM");
+SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, CTLFLAG_RW|CTLFLAG_LOCKED, NULL, "SOCK_DGRAM");
diff --git a/bsd/kern/uipc_socket.c b/bsd/kern/uipc_socket.c
index 090953d65..e178c29b4 100644
--- a/bsd/kern/uipc_socket.c
+++ b/bsd/kern/uipc_socket.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993
@@ -56,10 +61,17 @@
  *	@(#)uipc_socket.c	8.3 (Berkeley) 4/15/94
  * $FreeBSD: src/sys/kern/uipc_socket.c,v 1.68.2.16 2001/06/14 20:46:06 ume Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/filedesc.h>
+#include <sys/proc.h>
 #include <sys/proc_internal.h>
 #include <sys/kauth.h>
 #include <sys/file_internal.h>
@@ -79,12 +91,26 @@
 #include <sys/uio.h>
 #include <sys/ev.h>
 #include <sys/kdebug.h>
+#include <sys/un.h>
 #include <net/route.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <kern/zalloc.h>
 #include <kern/locks.h>
 #include <machine/limits.h>
+#include <libkern/OSAtomic.h>
+#include <pexpert/pexpert.h>
+
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_framework.h>
+#endif /* MAC */
+
+/* how a timeval looks to a 64-bit process */
+struct timeval64 {
+	int64_t		tv_sec;
+	int32_t		tv_usec;
+};
 
 int			so_cache_hw = 0;
 int			so_cache_timeouts = 0;
@@ -95,8 +121,6 @@ struct socket		*socket_cache_tail = 0;
 u_long			so_cache_time = 0;
 int			so_cache_init_done = 0;
 struct zone		*so_cache_zone;
-extern int		get_inpcb_str_size();
-extern int		get_tcp_str_size();
 
 static lck_grp_t		*so_cache_mtx_grp;
 static lck_attr_t		*so_cache_mtx_attr;
@@ -105,20 +129,26 @@ lck_mtx_t				*so_cache_mtx;
 
 #include <machine/limits.h>
 
-static void     filt_sordetach(struct knote *kn);
-static int      filt_soread(struct knote *kn, long hint);
-static void     filt_sowdetach(struct knote *kn);
-static int      filt_sowrite(struct knote *kn, long hint);
-static int      filt_solisten(struct knote *kn, long hint);
+static void	filt_sordetach(struct knote *kn);
+static int	filt_soread(struct knote *kn, long hint);
+static void	filt_sowdetach(struct knote *kn);
+static int	filt_sowrite(struct knote *kn, long hint);
+static int	filt_solisten(struct knote *kn, long hint);
+
+static int
+sooptcopyin_timeval(struct sockopt *sopt, struct timeval * tv_p);
+
+static int
+sooptcopyout_timeval(struct sockopt *sopt, const struct timeval * tv_p);
 
 static struct filterops solisten_filtops =
-  { 1, NULL, filt_sordetach, filt_solisten };
+	{ 1, NULL, filt_sordetach, filt_solisten };
 static struct filterops soread_filtops =
-  { 1, NULL, filt_sordetach, filt_soread };
+	{ 1, NULL, filt_sordetach, filt_soread };
 static struct filterops sowrite_filtops =
-  { 1, NULL, filt_sowdetach, filt_sowrite };
+	{ 1, NULL, filt_sowdetach, filt_sowrite };
 
-#define EVEN_MORE_LOCKING_DEBUG 0
+#define	EVEN_MORE_LOCKING_DEBUG 0
 int socket_debug = 0;
 int socket_zone = M_SOCKET;
 so_gen_t	so_gencnt;	/* generation count for sockets */
@@ -126,33 +156,52 @@ so_gen_t	so_gencnt;	/* generation count for sockets */
 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
 
-#define DBG_LAYER_IN_BEG	NETDBG_CODE(DBG_NETSOCK, 0)
-#define DBG_LAYER_IN_END	NETDBG_CODE(DBG_NETSOCK, 2)
-#define DBG_LAYER_OUT_BEG	NETDBG_CODE(DBG_NETSOCK, 1)
-#define DBG_LAYER_OUT_END	NETDBG_CODE(DBG_NETSOCK, 3)
-#define DBG_FNC_SOSEND		NETDBG_CODE(DBG_NETSOCK, (4 << 8) | 1)
-#define DBG_FNC_SORECEIVE	NETDBG_CODE(DBG_NETSOCK, (8 << 8))
-#define DBG_FNC_SOSHUTDOWN      NETDBG_CODE(DBG_NETSOCK, (9 << 8))
+#define	DBG_LAYER_IN_BEG	NETDBG_CODE(DBG_NETSOCK, 0)
+#define	DBG_LAYER_IN_END	NETDBG_CODE(DBG_NETSOCK, 2)
+#define	DBG_LAYER_OUT_BEG	NETDBG_CODE(DBG_NETSOCK, 1)
+#define	DBG_LAYER_OUT_END	NETDBG_CODE(DBG_NETSOCK, 3)
+#define	DBG_FNC_SOSEND		NETDBG_CODE(DBG_NETSOCK, (4 << 8) | 1)
+#define	DBG_FNC_SORECEIVE	NETDBG_CODE(DBG_NETSOCK, (8 << 8))
+#define	DBG_FNC_SOSHUTDOWN	NETDBG_CODE(DBG_NETSOCK, (9 << 8))
 
-#define MAX_SOOPTGETM_SIZE	(128 * MCLBYTES)
+#define	MAX_SOOPTGETM_SIZE	(128 * MCLBYTES)
 
 
 SYSCTL_DECL(_kern_ipc);
 
-static int somaxconn = SOMAXCONN;
-SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW, &somaxconn,
-	   0, "");
+int somaxconn = SOMAXCONN;
+SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW, &somaxconn, 0, "");
 
 /* Should we get a maximum also ??? */
 static int sosendmaxchain = 65536;
 static int sosendminchain = 16384;
 static int sorecvmincopy  = 16384;
 SYSCTL_INT(_kern_ipc, OID_AUTO, sosendminchain, CTLFLAG_RW, &sosendminchain,
-           0, "");
+    0, "");
 SYSCTL_INT(_kern_ipc, OID_AUTO, sorecvmincopy, CTLFLAG_RW, &sorecvmincopy,
-           0, "");
+    0, "");
+
+/*
+ * Set to enable jumbo clusters (if available) for large writes when
+ * the socket is marked with SOF_MULTIPAGES; see below.
+ */
+int sosendjcl = 1;
+SYSCTL_INT(_kern_ipc, OID_AUTO, sosendjcl, CTLFLAG_RW, &sosendjcl, 0, "");
 
-void  so_cache_timer();
+/*
+ * Set this to ignore SOF_MULTIPAGES and use jumbo clusters for large
+ * writes on the socket for all protocols on any network interfaces,
+ * depending upon sosendjcl above.  Be extra careful when setting this
+ * to 1, because sending down packets that cross physical pages down to
+ * broken drivers (those that falsely assume that the physical pages
+ * are contiguous) might lead to system panics or silent data corruption.
+ * When set to 0, the system will respect SOF_MULTIPAGES, which is set
+ * only for TCP sockets whose outgoing interface is IFNET_MULTIPAGES
+ * capable.  Set this to 1 only for testing/debugging purposes.
+ */
+int sosendjcl_ignore_capab = 0;
+SYSCTL_INT(_kern_ipc, OID_AUTO, sosendjcl_ignore_capab, CTLFLAG_RW,
+    &sosendjcl_ignore_capab, 0, "");
 
 /*
  * Socket operation routines.
@@ -162,141 +211,155 @@ void  so_cache_timer();
  * switching out to the protocol specific routines.
  */
 
+/* sys_generic.c */
+extern void postevent(struct socket *, struct sockbuf *, int);
+extern void evsofree(struct socket *);
+
+/* TODO: these should be in header file */
+extern int get_inpcb_str_size(void);
+extern int get_tcp_str_size(void);
+extern struct domain *pffinddomain(int);
+extern struct protosw *pffindprotonotype(int, int);
+extern int soclose_locked(struct socket *);
+extern int soo_kqfilter(struct fileproc *, struct knote *, struct proc *);
+
 #ifdef __APPLE__
 
 vm_size_t	so_cache_zone_element_size;
 
-static int sodelayed_copy(struct socket *so, struct uio *uio, struct mbuf **free_list, int *resid);
+static int sodelayed_copy(struct socket *, struct uio *, struct mbuf **, int *);
+static void cached_sock_alloc(struct socket **, int);
+static void cached_sock_free(struct socket *);
+static void so_cache_timer(void *);
+
+void soclose_wait_locked(struct socket *so);
 
 
-void socketinit()
+void
+socketinit(void)
 {
-    vm_size_t	str_size;
+	vm_size_t str_size;
 
 	if (so_cache_init_done) {
 		printf("socketinit: already called...\n");
 		return;
 	}
 
+	PE_parse_boot_arg("socket_debug", &socket_debug);
+
 	/*
 	 * allocate lock group attribute and group for socket cache mutex
 	 */
 	so_cache_mtx_grp_attr = lck_grp_attr_alloc_init();
 
-	so_cache_mtx_grp = lck_grp_alloc_init("so_cache", so_cache_mtx_grp_attr);
-		
+	so_cache_mtx_grp = lck_grp_alloc_init("so_cache",
+	    so_cache_mtx_grp_attr);
+
 	/*
 	 * allocate the lock attribute for socket cache mutex
 	 */
 	so_cache_mtx_attr = lck_attr_alloc_init();
 
-    so_cache_init_done = 1;
+	so_cache_init_done = 1;
+
+	/* cached sockets mutex */
+	so_cache_mtx = lck_mtx_alloc_init(so_cache_mtx_grp, so_cache_mtx_attr);
 
-    so_cache_mtx = lck_mtx_alloc_init(so_cache_mtx_grp, so_cache_mtx_attr);	/* cached sockets mutex */
-	
-    if (so_cache_mtx == NULL)
+	if (so_cache_mtx == NULL)
 		return; /* we're hosed... */
 
-    str_size = (vm_size_t)( sizeof(struct socket) + 4 +
-			    get_inpcb_str_size()  + 4 +
-			    get_tcp_str_size());
-    so_cache_zone = zinit (str_size, 120000*str_size, 8192, "socache zone");
+	str_size = (vm_size_t)(sizeof (struct socket) + 4 +
+	    get_inpcb_str_size() + 4 + get_tcp_str_size());
+
+	so_cache_zone = zinit(str_size, 120000*str_size, 8192, "socache zone");
 #if TEMPDEBUG
-    printf("cached_sock_alloc -- so_cache_zone size is %x\n", str_size);
+	printf("cached_sock_alloc -- so_cache_zone size is %x\n", str_size);
 #endif
-    timeout(so_cache_timer, NULL, (SO_CACHE_FLUSH_INTERVAL * hz));
-
-    so_cache_zone_element_size = str_size;
+	timeout(so_cache_timer, NULL, (SO_CACHE_FLUSH_INTERVAL * hz));
 
-    sflt_init();
+	so_cache_zone_element_size = str_size;
 
+	sflt_init();
 }
 
-void   cached_sock_alloc(so, waitok)
-struct socket **so;
-int           waitok;
-
+static void
+cached_sock_alloc(struct socket **so, int waitok)
 {
-    caddr_t	temp;
-    register u_long  offset;
-
+	caddr_t	temp;
+	register u_long offset;
 
 	lck_mtx_lock(so_cache_mtx);
 
-    if (cached_sock_count) {
-	    cached_sock_count--;
-	    *so = socket_cache_head;
-	    if (*so == 0)
-		    panic("cached_sock_alloc: cached sock is null");
+	if (cached_sock_count) {
+		cached_sock_count--;
+		*so = socket_cache_head;
+		if (*so == 0)
+			panic("cached_sock_alloc: cached sock is null");
 
-	    socket_cache_head = socket_cache_head->cache_next;
-	    if (socket_cache_head)
-		    socket_cache_head->cache_prev = 0;
-	    else
-		    socket_cache_tail = 0;
+		socket_cache_head = socket_cache_head->cache_next;
+		if (socket_cache_head)
+			socket_cache_head->cache_prev = 0;
+		else
+			socket_cache_tail = 0;
 
 		lck_mtx_unlock(so_cache_mtx);
 
-	    temp = (*so)->so_saved_pcb;
-	    bzero((caddr_t)*so, sizeof(struct socket));
+		temp = (*so)->so_saved_pcb;
+		bzero((caddr_t)*so, sizeof (struct socket));
 #if TEMPDEBUG
-	    kprintf("cached_sock_alloc - retreiving cached sock %x - count == %d\n", *so,
-		   cached_sock_count);
+		kprintf("cached_sock_alloc - retreiving cached sock %p - "
+		    "count == %d\n", *so, cached_sock_count);
 #endif
-	    (*so)->so_saved_pcb = temp;
-	    (*so)->cached_in_sock_layer = 1;
-
-    }
-    else {
+		(*so)->so_saved_pcb = temp;
+		(*so)->cached_in_sock_layer = 1;
+	} else {
 #if TEMPDEBUG
-	    kprintf("Allocating cached sock %x from memory\n", *so);
+		kprintf("Allocating cached sock %p from memory\n", *so);
 #endif
 
-	    lck_mtx_unlock(so_cache_mtx);
-
-	    if (waitok)
-		 *so = (struct socket *) zalloc(so_cache_zone);
-	    else
-		 *so = (struct socket *) zalloc_noblock(so_cache_zone);
-
-	    if (*so == 0)
-		 return;
+		lck_mtx_unlock(so_cache_mtx);
 
-	    bzero((caddr_t)*so, sizeof(struct socket));
+		if (waitok)
+			*so = (struct socket *)zalloc(so_cache_zone);
+		else
+			*so = (struct socket *)zalloc_noblock(so_cache_zone);
 
-	    /*
-	     * Define offsets for extra structures into our single block of
-	     * memory. Align extra structures on longword boundaries.
-	     */
+		if (*so == 0)
+			return;
 
+		bzero((caddr_t)*so, sizeof (struct socket));
 
-	    offset = (u_long) *so;
-	    offset += sizeof(struct socket);
-	    if (offset & 0x3) {
-		offset += 4;
-		offset &= 0xfffffffc;
-	    }
-	    (*so)->so_saved_pcb = (caddr_t) offset;
-	    offset += get_inpcb_str_size();
-	    if (offset & 0x3) {
-		offset += 4;
-		offset &= 0xfffffffc;
-	    }
+		/*
+		 * Define offsets for extra structures into our single block of
+		 * memory. Align extra structures on longword boundaries.
+		 */
+		offset = (u_long) *so;
+		offset += sizeof (struct socket);
+		if (offset & 0x3) {
+			offset += 4;
+			offset &= 0xfffffffc;
+		}
+		(*so)->so_saved_pcb = (caddr_t)offset;
+		offset += get_inpcb_str_size();
+		if (offset & 0x3) {
+			offset += 4;
+			offset &= 0xfffffffc;
+		}
 
-	    ((struct inpcb *) (*so)->so_saved_pcb)->inp_saved_ppcb = (caddr_t) offset;
+		((struct inpcb *)(*so)->so_saved_pcb)->inp_saved_ppcb =
+		    (caddr_t)offset;
 #if TEMPDEBUG
-	    kprintf("Allocating cached socket - %x, pcb=%x tcpcb=%x\n", *so,
-		    (*so)->so_saved_pcb,
+		kprintf("Allocating cached socket - %p, pcb=%p tcpcb=%p\n",
+		    *so, (*so)->so_saved_pcb,
 		    ((struct inpcb *)(*so)->so_saved_pcb)->inp_saved_ppcb);
 #endif
-    }
+	}
 
-    (*so)->cached_in_sock_layer = 1;
+	(*so)->cached_in_sock_layer = 1;
 }
 
-
-void cached_sock_free(so) 
-struct socket *so;
+static void
+cached_sock_free(struct socket *so)
 {
 
 	lck_mtx_lock(so_cache_mtx);
@@ -305,13 +368,12 @@ struct socket *so;
 		--cached_sock_count;
 		lck_mtx_unlock(so_cache_mtx);
 #if TEMPDEBUG
-		kprintf("Freeing overflowed cached socket %x\n", so);
+		kprintf("Freeing overflowed cached socket %p\n", so);
 #endif
 		zfree(so_cache_zone, so);
-	}
-	else {
+	} else {
 #if TEMPDEBUG
-		kprintf("Freeing socket %x into cache\n", so);
+		kprintf("Freeing socket %p into cache\n", so);
 #endif
 		if (so_cache_hw < cached_sock_count)
 			so_cache_hw = cached_sock_count;
@@ -329,49 +391,42 @@ struct socket *so;
 	}
 
 #if TEMPDEBUG
-	kprintf("Freed cached sock %x into cache - count is %d\n", so, cached_sock_count);
+	kprintf("Freed cached sock %p into cache - count is %d\n",
+	    so, cached_sock_count);
 #endif
-
-
 }
 
-
-void so_cache_timer()
+static void
+so_cache_timer(__unused void *dummy)
 {
 	register struct socket	*p;
 	register int		n_freed = 0;
 
-
 	lck_mtx_lock(so_cache_mtx);
 
 	++so_cache_time;
 
-	while ( (p = socket_cache_tail) )
-	{
+	while ((p = socket_cache_tail)) {
 		if ((so_cache_time - p->cache_timestamp) < SO_CACHE_TIME_LIMIT)
-		        break;
+			break;
 
 		so_cache_timeouts++;
-		
-		if ( (socket_cache_tail = p->cache_prev) )
-		        p->cache_prev->cache_next = 0;
-		if (--cached_sock_count == 0)
-		        socket_cache_head = 0;
 
+		if ((socket_cache_tail = p->cache_prev))
+			p->cache_prev->cache_next = 0;
+		if (--cached_sock_count == 0)
+			socket_cache_head = 0;
 
 		zfree(so_cache_zone, p);
-		
-		if (++n_freed >= SO_CACHE_MAX_FREE_BATCH)
-		{
-		        so_cache_max_freed++;
+
+		if (++n_freed >= SO_CACHE_MAX_FREE_BATCH) {
+			so_cache_max_freed++;
 			break;
 		}
 	}
 	lck_mtx_unlock(so_cache_mtx);
 
 	timeout(so_cache_timer, NULL, (SO_CACHE_FLUSH_INTERVAL * hz));
-
-
 }
 #endif /* __APPLE__ */
 
@@ -383,37 +438,49 @@ void so_cache_timer()
  * the protocols can be easily modified to do this.
  */
 struct socket *
-soalloc(waitok, dom, type)
-	int waitok;
-	int dom;
-	int type;
+soalloc(int waitok, int dom, int type)
 {
 	struct socket *so;
 
-	if ((dom == PF_INET) && (type == SOCK_STREAM)) 
-	    cached_sock_alloc(&so, waitok);
-	else
-	{
-	     MALLOC_ZONE(so, struct socket *, sizeof(*so), socket_zone, M_WAITOK);
-	     if (so) 
-		  bzero(so, sizeof *so);
+	if ((dom == PF_INET) && (type == SOCK_STREAM)) {
+		cached_sock_alloc(&so, waitok);
+	} else {
+		MALLOC_ZONE(so, struct socket *, sizeof (*so), socket_zone,
+		    M_WAITOK);
+		if (so != NULL)
+			bzero(so, sizeof (*so));
 	}
 	/* XXX race condition for reentrant kernel */
 //###LD Atomic add for so_gencnt
-	if (so) {
-	     so->so_gencnt = ++so_gencnt;
-	     so->so_zone = socket_zone;
+	if (so != NULL) {
+		so->so_gencnt = ++so_gencnt;
+		so->so_zone = socket_zone;
+#if CONFIG_MACF_SOCKET
+	     /* Convert waitok to  M_WAITOK/M_NOWAIT for MAC Framework. */
+	     if (mac_socket_label_init(so, !waitok) != 0) {
+			sodealloc(so);
+			return (NULL);
+		}
+#endif /* MAC_SOCKET */
 	}
 
-	return so;
+	return (so);
 }
 
+/*
+ * Returns:	0			Success
+ *		EAFNOSUPPORT
+ *		EPROTOTYPE
+ *		EPROTONOSUPPORT
+ *		ENOBUFS
+ *	<pru_attach>:ENOBUFS[AF_UNIX]
+ *	<pru_attach>:ENOBUFS[TCP]
+ *	<pru_attach>:ENOMEM[TCP]
+ *	<pru_attach>:EISCONN[TCP]
+ *	<pru_attach>:???		[other protocol families, IPSEC]
+ */
 int
-socreate(dom, aso, type, proto)
-	int dom;
-	struct socket **aso;
-	register int type;
-	int proto;
+socreate(int dom, struct socket **aso, int type, int proto)
 {
 	struct proc *p = current_proc();
 	register struct protosw *prp;
@@ -427,18 +494,17 @@ socreate(dom, aso, type, proto)
 	else
 		prp = pffindtype(dom, type);
 
-	if (prp == 0 || prp->pr_usrreqs->pru_attach == 0)
-		return (EPROTONOSUPPORT);
-#ifndef __APPLE__
-
-	if (p->p_prison && jail_socket_unixiproute_only &&
-	    prp->pr_domain->dom_family != PF_LOCAL &&
-	    prp->pr_domain->dom_family != PF_INET &&
-	    prp->pr_domain->dom_family != PF_ROUTE) {
+	if (prp == 0 || prp->pr_usrreqs->pru_attach == 0) {
+		if (pffinddomain(dom) == NULL) {
+			return (EAFNOSUPPORT);
+		}
+		if (proto != 0) {
+			if (pffindprotonotype(dom, proto) != NULL) {
+				return (EPROTOTYPE);
+			}
+		}
 		return (EPROTONOSUPPORT);
 	}
-	
-#endif
 	if (prp->pr_type != type)
 		return (EPROTOTYPE);
 	so = soalloc(p != 0, dom, type);
@@ -449,15 +515,11 @@ socreate(dom, aso, type, proto)
 	TAILQ_INIT(&so->so_comp);
 	so->so_type = type;
 
-#ifdef __APPLE__
 	if (p != 0) {
 		so->so_uid = kauth_cred_getuid(kauth_cred_get());
-		if (!suser(kauth_cred_get(),NULL))
+		if (!suser(kauth_cred_get(), NULL))
 			so->so_state = SS_PRIV;
 	}
-#else
-	so->so_cred = kauth_cred_get_with_ref();
-#endif
 	so->so_proto = prp;
 #ifdef __APPLE__
 	so->so_rcv.sb_flags |= SB_RECV;	/* XXX */
@@ -465,17 +527,24 @@ socreate(dom, aso, type, proto)
 #endif
 	so->next_lock_lr = 0;
 	so->next_unlock_lr = 0;
-	
-	
+
+#if CONFIG_MACF_SOCKET
+	mac_socket_label_associate(kauth_cred_get(), so);
+#endif /* MAC_SOCKET */
+
 //### Attachement will create the per pcb lock if necessary and increase refcount
-	so->so_usecount++;	/* for creation, make sure it's done before socket is inserted in lists */
+	/*
+	 * for creation, make sure it's done before
+	 * socket is inserted in lists
+	 */
+	so->so_usecount++;
 
 	error = (*prp->pr_usrreqs->pru_attach)(so, proto, p);
 	if (error) {
-		/* 
-		 * Warning: 
-		 * If so_pcb is not zero, the socket will be leaked, 
-		 * so protocol attachment handler must be coded carefuly 
+		/*
+		 * Warning:
+		 * If so_pcb is not zero, the socket will be leaked,
+		 * so protocol attachment handler must be coded carefuly
 		 */
 		so->so_state |= SS_NOFDREF;
 		so->so_usecount--;
@@ -493,36 +562,64 @@ socreate(dom, aso, type, proto)
 		so->so_options |= SO_DEBUG;
 #endif
 #endif
-
 	*aso = so;
 	return (0);
 }
 
+/*
+ * Returns:	0			Success
+ *	<pru_bind>:EINVAL		Invalid argument [COMMON_START]
+ *	<pru_bind>:EAFNOSUPPORT		Address family not supported
+ *	<pru_bind>:EADDRNOTAVAIL	Address not available.
+ *	<pru_bind>:EINVAL		Invalid argument
+ *	<pru_bind>:EAFNOSUPPORT		Address family not supported [notdef]
+ *	<pru_bind>:EACCES		Permission denied
+ *	<pru_bind>:EADDRINUSE		Address in use
+ *	<pru_bind>:EAGAIN		Resource unavailable, try again
+ *	<pru_bind>:EPERM		Operation not permitted
+ *	<pru_bind>:???
+ *	<sf_bind>:???
+ *
+ * Notes:	It's not possible to fully enumerate the return codes above,
+ *		since socket filter authors and protocol family authors may
+ *		not choose to limit their error returns to those listed, even
+ *		though this may result in some software operating incorrectly.
+ *
+ *		The error codes which are enumerated above are those known to
+ *		be returned by the tcp_usr_bind function supplied.
+ */
 int
-sobind(so, nam)
-	struct socket *so;
-	struct sockaddr *nam;
-
+sobind(struct socket *so, struct sockaddr *nam)
 {
 	struct proc *p = current_proc();
 	int error = 0;
-	struct socket_filter_entry	*filter;
-	int						 	filtered = 0;
+	struct socket_filter_entry *filter;
+	int filtered = 0;
 
 	socket_lock(so, 1);
 
+	/*
+	 * If this is a bind request on a previously-accepted socket
+	 * that has been marked as inactive, reject it now before
+	 * we go any further.
+	 */
+	if (so->so_flags & SOF_DEFUNCT) {
+		error = EINVAL;
+		goto out;
+	}
+
 	/* Socket filter */
 	error = 0;
 	for (filter = so->so_filt; filter && (error == 0);
-		 filter = filter->sfe_next_onsocket) {
+	    filter = filter->sfe_next_onsocket) {
 		if (filter->sfe_filter->sf_filter.sf_bind) {
 			if (filtered == 0) {
 				filtered = 1;
 				sflt_use(so);
 				socket_unlock(so, 0);
 			}
-			error = filter->sfe_filter->sf_filter.sf_bind(
-						filter->sfe_cookie, so, nam);
+			error = filter->sfe_filter->sf_filter.
+			    sf_bind(filter->sfe_cookie, so, nam);
 		}
 	}
 	if (filtered != 0) {
@@ -530,124 +627,155 @@ sobind(so, nam)
 		sflt_unuse(so);
 	}
 	/* End socket filter */
-	
+
 	if (error == 0)
 		error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, p);
-	
+out:
 	socket_unlock(so, 1);
-	
+
 	if (error == EJUSTRETURN)
 		error = 0;
-	
+
 	return (error);
 }
 
 void
-sodealloc(so)
-	struct socket *so;
+sodealloc(struct socket *so)
 {
 	so->so_gencnt = ++so_gencnt;
 
-#ifndef __APPLE__
-	if (so->so_rcv.sb_hiwat)
-		(void)chgsbsize(so->so_cred->cr_uidinfo,
-		    &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY);
-	if (so->so_snd.sb_hiwat)
-		(void)chgsbsize(so->so_cred->cr_uidinfo,
-		    &so->so_snd.sb_hiwat, 0, RLIM_INFINITY);
-#ifdef INET
-	if (so->so_accf != NULL) {
-		if (so->so_accf->so_accept_filter != NULL && 
-			so->so_accf->so_accept_filter->accf_destroy != NULL) {
-			so->so_accf->so_accept_filter->accf_destroy(so);
-		}
-		if (so->so_accf->so_accept_filter_str != NULL)
-			FREE(so->so_accf->so_accept_filter_str, M_ACCF);
-		FREE(so->so_accf, M_ACCF);
-	}
-#endif /* INET */
-	kauth_cred_unref(&so->so_cred);
-	zfreei(so->so_zone, so);
-#else
-	if (so->cached_in_sock_layer == 1) 
-	     cached_sock_free(so);
-	else {
-	     if (so->cached_in_sock_layer == -1)
-			panic("sodealloc: double dealloc: so=%x\n", so);
-	     so->cached_in_sock_layer = -1;
-	     FREE_ZONE(so, sizeof(*so), so->so_zone);
+#if CONFIG_MACF_SOCKET
+	mac_socket_label_destroy(so);
+#endif /* MAC_SOCKET */
+	if (so->cached_in_sock_layer == 1) {
+		cached_sock_free(so);
+	} else {
+		if (so->cached_in_sock_layer == -1)
+			panic("sodealloc: double dealloc: so=%p\n", so);
+		so->cached_in_sock_layer = -1;
+		FREE_ZONE(so, sizeof (*so), so->so_zone);
 	}
-#endif /* __APPLE__ */
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		EOPNOTSUPP
+ *	<pru_listen>:EINVAL[AF_UNIX]
+ *	<pru_listen>:EINVAL[TCP]
+ *	<pru_listen>:EADDRNOTAVAIL[TCP]	Address not available.
+ *	<pru_listen>:EINVAL[TCP]	Invalid argument
+ *	<pru_listen>:EAFNOSUPPORT[TCP]	Address family not supported [notdef]
+ *	<pru_listen>:EACCES[TCP]	Permission denied
+ *	<pru_listen>:EADDRINUSE[TCP]	Address in use
+ *	<pru_listen>:EAGAIN[TCP]	Resource unavailable, try again
+ *	<pru_listen>:EPERM[TCP]		Operation not permitted
+ *	<sf_listen>:???
+ *
+ * Notes:	Other <pru_listen> returns depend on the protocol family; all
+ *		<sf_listen> returns depend on what the filter author causes
+ *		their filter to return.
+ */
 int
-solisten(so, backlog)
-	register struct socket *so;
-	int backlog;
-
+solisten(struct socket *so, int backlog)
 {
 	struct proc *p = current_proc();
-	int error;
+	int error = 0;
+	struct socket_filter_entry *filter;
+	int filtered = 0;
 
 	socket_lock(so, 1);
-	
-	{
-		struct socket_filter_entry	*filter;
-		int						 	filtered = 0;
-		error = 0;
-		for (filter = so->so_filt; filter && (error == 0);
-			 filter = filter->sfe_next_onsocket) {
-			if (filter->sfe_filter->sf_filter.sf_listen) {
-				if (filtered == 0) {
-					filtered = 1;
-					sflt_use(so);
-					socket_unlock(so, 0);
-				}
-				error = filter->sfe_filter->sf_filter.sf_listen(
-							filter->sfe_cookie, so);
+	if (so->so_proto == NULL) {
+		error = EINVAL;
+		goto out;
+	}
+	if ((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0) {
+		error = EOPNOTSUPP;
+		goto out;
+	}
+
+	/*
+	 * If the listen request is made on a socket that is not fully
+	 * disconnected, or on a previously-accepted socket that has
+	 * been marked as inactive, reject the request now.
+	 */
+	if ((so->so_state &
+	    (SS_ISCONNECTED|SS_ISCONNECTING|SS_ISDISCONNECTING)) ||
+	    (so->so_flags & SOF_DEFUNCT)) {
+		error = EINVAL;
+		goto out;
+	}
+
+	if ((so->so_restrictions & SO_RESTRICT_DENYIN) != 0) {
+		error = EPERM;
+		goto out;
+	}
+
+	error = 0;
+	for (filter = so->so_filt; filter && (error == 0);
+	    filter = filter->sfe_next_onsocket) {
+		if (filter->sfe_filter->sf_filter.sf_listen) {
+			if (filtered == 0) {
+				filtered = 1;
+				sflt_use(so);
+				socket_unlock(so, 0);
 			}
+			error = filter->sfe_filter->sf_filter.
+			    sf_listen(filter->sfe_cookie, so);
 		}
-		if (filtered != 0) {
-			socket_lock(so, 0);
-			sflt_unuse(so);
-		}
+	}
+	if (filtered != 0) {
+		socket_lock(so, 0);
+		sflt_unuse(so);
 	}
 
 	if (error == 0) {
 		error = (*so->so_proto->pr_usrreqs->pru_listen)(so, p);
 	}
-	
+
 	if (error) {
-		socket_unlock(so, 1);
 		if (error == EJUSTRETURN)
 			error = 0;
-		return (error);
+		goto out;
 	}
-	
+
 	if (TAILQ_EMPTY(&so->so_comp))
 		so->so_options |= SO_ACCEPTCONN;
-	if (backlog < 0 || backlog > somaxconn)
+	/*
+	 * POSIX: The implementation may have an upper limit on the length of
+	 * the listen queue-either global or per accepting socket. If backlog
+	 * exceeds this limit, the length of the listen queue is set to the
+	 * limit.
+	 *
+	 * If listen() is called with a backlog argument value that is less
+	 * than 0, the function behaves as if it had been called with a backlog
+	 * argument value of 0.
+	 *
+	 * A backlog argument of 0 may allow the socket to accept connections,
+	 * in which case the length of the listen queue may be set to an
+	 * implementation-defined minimum value.
+	 */
+	if (backlog <= 0 || backlog > somaxconn)
 		backlog = somaxconn;
-	so->so_qlimit = backlog;
 
+	so->so_qlimit = backlog;
+out:
 	socket_unlock(so, 1);
-	return (0);
+	return (error);
 }
 
 void
-sofreelastref(so, dealloc)
-	register struct socket *so;
-	int dealloc;
+sofreelastref(struct socket *so, int dealloc)
 {
-	int error;
 	struct socket *head = so->so_head;
 
-	/*### Assume socket is locked */
+	/* Assume socket is locked */
 
 	/* Remove any filters - may be called more than once */
 	sflt_termsock(so);
-	
-	if ((!(so->so_flags & SOF_PCBCLEARING)) || ((so->so_state & SS_NOFDREF) == 0)) {
+
+	if ((!(so->so_flags & SOF_PCBCLEARING)) ||
+	    ((so->so_state & SS_NOFDREF) == 0)) {
 #ifdef __APPLE__
 		selthreadclear(&so->so_snd.sb_sel);
 		selthreadclear(&so->so_rcv.sb_sel);
@@ -689,60 +817,104 @@ sofreelastref(so, dealloc)
 	sbrelease(&so->so_snd);
 #endif
 	sorflush(so);
-	
+
 	/* 3932268: disable upcall */
 	so->so_rcv.sb_flags &= ~SB_UPCALL;
 	so->so_snd.sb_flags &= ~SB_UPCALL;
-	
+
 	if (dealloc)
 		sodealloc(so);
 }
 
+void
+soclose_wait_locked(struct socket *so)
+{
+	lck_mtx_t *mutex_held;
+
+	if (so->so_proto->pr_getlock != NULL)
+		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
+	else
+		mutex_held = so->so_proto->pr_domain->dom_mtx;
+	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
+
+	/* Double check here and return if there's no outstanding upcall */
+	if (!(so->so_flags & SOF_UPCALLINUSE))
+		return;
+
+	so->so_flags |= SOF_CLOSEWAIT;
+	(void) msleep((caddr_t)&so->so_upcall, mutex_held, (PZERO - 1),
+	    "soclose_wait_locked", NULL);
+	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
+	so->so_flags &= ~SOF_CLOSEWAIT;
+}
+
 /*
  * Close a socket on last file table reference removal.
  * Initiate disconnect if connected.
  * Free socket when disconnect complete.
  */
 int
-soclose_locked(so)
-	register struct socket *so;
+soclose_locked(struct socket *so)
 {
 	int error = 0;
-	lck_mtx_t * mutex_held;
+	lck_mtx_t *mutex_held;
 	struct timespec ts;
 
 	if (so->so_usecount == 0) {
-		panic("soclose: so=%x refcount=0\n", so);
+		panic("soclose: so=%p refcount=0\n", so);
 	}
 
 	sflt_notify(so, sock_evt_closing, NULL);
-	
+
 	if ((so->so_options & SO_ACCEPTCONN)) {
-		struct socket *sp;
-		
-		/* We do not want new connection to be added to the connection queues */
+		struct socket *sp, *sonext;
+		int socklock = 0;
+
+		/*
+		 * We do not want new connection to be added
+		 * to the connection queues
+		 */
 		so->so_options &= ~SO_ACCEPTCONN;
-		
-		while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) {
-			/* A bit tricky here. We need to keep
-			 * a lock if it's a protocol global lock
-			 * but we want the head, not the socket locked
-			 * in the case of per-socket lock...
+
+		for (sp = TAILQ_FIRST(&so->so_incomp); sp != NULL; sp = sonext) {
+			sonext = TAILQ_NEXT(sp, so_list);
+
+			/* Radar 5350314
+			 * skip sockets thrown away by tcpdropdropblreq
+			 * they will get cleanup by the garbage collection.
+			 * otherwise, remove the incomp socket from the queue
+			 * and let soabort trigger the appropriate cleanup.
 			 */
+			if (sp->so_flags & SOF_OVERFLOW) 
+				continue;
+
 			if (so->so_proto->pr_getlock != NULL) {
+				/* lock ordering for consistency with the rest of the stack,
+				 * we lock the socket first and then grabb the head.
+				 */
 				socket_unlock(so, 0);
 				socket_lock(sp, 1);
-			}
-			(void) soabort(sp);
-			if (so->so_proto->pr_getlock != NULL) {
-				socket_unlock(sp, 1);
 				socket_lock(so, 0);
+				socklock = 1; 
+			}
+
+			TAILQ_REMOVE(&so->so_incomp, sp, so_list);
+			so->so_incqlen--;
+
+			if (sp->so_state & SS_INCOMP) {
+				sp->so_state &= ~SS_INCOMP;
+				sp->so_head = NULL;
+
+				(void) soabort(sp);
 			}
+
+			if (socklock) 
+				socket_unlock(sp, 1);
 		}
 
 		while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) {
 			/* Dequeue from so_comp since sofree() won't do it */
-			TAILQ_REMOVE(&so->so_comp, sp, so_list);			
+			TAILQ_REMOVE(&so->so_comp, sp, so_list);
 			so->so_qlen--;
 
 			if (so->so_proto->pr_getlock != NULL) {
@@ -750,10 +922,13 @@ soclose_locked(so)
 				socket_lock(sp, 1);
 			}
 
-			sp->so_state &= ~SS_COMP;
-			sp->so_head = NULL;
+			if (sp->so_state & SS_COMP) {
+				sp->so_state &= ~SS_COMP;
+				sp->so_head = NULL;
+
+				(void) soabort(sp);
+			}
 
-			(void) soabort(sp);
 			if (so->so_proto->pr_getlock != NULL) {
 				socket_unlock(sp, 1);
 				socket_lock(so, 0);
@@ -775,17 +950,21 @@ soclose_locked(so)
 			if ((so->so_state & SS_ISDISCONNECTING) &&
 			    (so->so_state & SS_NBIO))
 				goto drop;
-			if (so->so_proto->pr_getlock != NULL) 
+			if (so->so_proto->pr_getlock != NULL)
 				mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-			else 
+			else
 				mutex_held = so->so_proto->pr_domain->dom_mtx;
 			while (so->so_state & SS_ISCONNECTED) {
 				ts.tv_sec = (so->so_linger/100);
-				ts.tv_nsec = (so->so_linger % 100) * NSEC_PER_USEC * 1000 * 10;
-				error = msleep((caddr_t)&so->so_timeo, mutex_held,
-				    PSOCK | PCATCH, "soclos", &ts);
+				ts.tv_nsec = (so->so_linger % 100) *
+				    NSEC_PER_USEC * 1000 * 10;
+				error = msleep((caddr_t)&so->so_timeo,
+				    mutex_held, PSOCK | PCATCH, "soclose", &ts);
 				if (error) {
-					/* It's OK when the time fires, don't report an error */
+					/*
+					 * It's OK when the time fires,
+					 * don't report an error
+					 */
 					if (error == EWOULDBLOCK)
 						error = 0;
 					break;
@@ -795,14 +974,14 @@ soclose_locked(so)
 	}
 drop:
 	if (so->so_usecount == 0)
-		panic("soclose: usecount is zero so=%x\n", so);
+		panic("soclose: usecount is zero so=%p\n", so);
 	if (so->so_pcb && !(so->so_flags & SOF_PCBCLEARING)) {
 		int error2 = (*so->so_proto->pr_usrreqs->pru_detach)(so);
 		if (error == 0)
 			error = error2;
 	}
 	if (so->so_usecount <= 0)
-		panic("soclose: usecount is zero so=%x\n", so);
+		panic("soclose: usecount is zero so=%p\n", so);
 discard:
 	if (so->so_pcb && so->so_state & SS_NOFDREF)
 		panic("soclose: NOFDREF");
@@ -817,94 +996,204 @@ discard:
 }
 
 int
-soclose(so)
-	register struct socket *so;
+soclose(struct socket *so)
 {
 	int error = 0;
 	socket_lock(so, 1);
-	if (so->so_retaincnt == 0)
+
+	if (so->so_flags & SOF_UPCALLINUSE)
+		soclose_wait_locked(so);
+
+	if (so->so_retaincnt == 0) {
 		error = soclose_locked(so);
-	else {	/* if the FD is going away, but socket is retained in kernel remove its reference */
+	} else {
+		/*
+		 * if the FD is going away, but socket is
+		 * retained in kernel remove its reference
+		 */
 		so->so_usecount--;
 		if (so->so_usecount < 2)
-			panic("soclose: retaincnt non null and so=%x usecount=%x\n", so->so_usecount);
+			panic("soclose: retaincnt non null and so=%p "
+			    "usecount=%d\n", so, so->so_usecount);
 	}
 	socket_unlock(so, 1);
 	return (error);
 }
 
-
 /*
  * Must be called at splnet...
  */
-//#### Should already be locked
+/* Should already be locked */
 int
-soabort(so)
-	struct socket *so;
+soabort(struct socket *so)
 {
 	int error;
 
 #ifdef MORE_LOCKING_DEBUG
-	lck_mtx_t * mutex_held;
+	lck_mtx_t *mutex_held;
 
-	if (so->so_proto->pr_getlock != NULL) 
+	if (so->so_proto->pr_getlock != NULL)
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-	else 
+	else
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 #endif
 
-	error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
-	if (error) {
-		sofree(so);
-		return error;
+	if ((so->so_flags & SOF_ABORTED) == 0) {
+		so->so_flags |= SOF_ABORTED;
+		error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
+		if (error) {
+			sofree(so);
+			return (error);
+		}
 	}
 	return (0);
 }
 
 int
-soacceptlock(so, nam, dolock)
-	register struct socket *so;
-	struct sockaddr **nam;
-	int dolock;
+soacceptlock(struct socket *so, struct sockaddr **nam, int dolock)
 {
 	int error;
 
-	if (dolock) socket_lock(so, 1);
+	if (dolock)
+		socket_lock(so, 1);
 
 	if ((so->so_state & SS_NOFDREF) == 0)
 		panic("soaccept: !NOFDREF");
 	so->so_state &= ~SS_NOFDREF;
 	error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam);
-    
-	if (dolock) socket_unlock(so, 1);
+
+	if (dolock)
+		socket_unlock(so, 1);
 	return (error);
 }
+
 int
-soaccept(so, nam)
-	register struct socket *so;
-	struct sockaddr **nam;
+soaccept(struct socket *so, struct sockaddr **nam)
 {
 	return (soacceptlock(so, nam, 1));
 }
 
 int
-soconnectlock(so, nam, dolock)
-	register struct socket *so;
-	struct sockaddr *nam;
-	int dolock;
+soacceptfilter(struct socket *so)
+{
+	struct sockaddr *local = NULL, *remote = NULL;
+	struct socket_filter_entry *filter;
+	int error = 0, filtered = 0;
+	struct socket *head = so->so_head;
+
+	/*
+	 * There's no need to hold the lock; this socket
+	 * has not been made visible to the filter(s).
+	 */
+	if ((sock_getaddr(so, &remote, 1) != 0) ||
+	    sock_getaddr(so, &local, 0) != 0) {
+		so->so_state &= ~(SS_NOFDREF | SS_COMP);
+		so->so_head = NULL;
+		soclose(so);
+		/* Out of resources; try it again next time */
+		error = ECONNABORTED;
+		goto done;
+	}
+
+	/*
+	 * At this point, we have a reference on the listening socket
+	 * so we know it won't be going away.  Do the same for the newly
+	 * accepted socket while we invoke the accept callback routine.
+	 */
+	socket_lock(so, 1);
+	for (filter = so->so_filt; filter != NULL && error == 0;
+	    filter = filter->sfe_next_onsocket) {
+		if (filter->sfe_filter->sf_filter.sf_accept != NULL) {
+			if (!filtered) {
+				filtered = 1;
+				sflt_use(so);
+				socket_unlock(so, 0);
+			}
+			error = filter->sfe_filter->sf_filter.
+			    sf_accept(filter->sfe_cookie,
+			    head, so, local, remote);
+		}
+	}
+
+	if (filtered) {
+		socket_lock(so, 0);
+		sflt_unuse(so);
+	}
+
+	/*
+	 * If we get EJUSTRETURN from one of the filters, mark this socket
+	 * as inactive and return it anyway.  This newly accepted socket
+	 * will be disconnected later before we hand it off to the caller.
+	 */
+	if (error == EJUSTRETURN) {
+		error = 0;
+		so->so_flags |= SOF_DEFUNCT;
+		/* Prevent data from being appended to the socket buffers */
+		so->so_snd.sb_flags |= SB_DROP;
+		so->so_rcv.sb_flags |= SB_DROP;
+	}
+
+	if (error != 0) {
+		/*
+		 * This may seem like a duplication to the above error
+		 * handling part when we return ECONNABORTED, except
+		 * the following is done while holding the lock since
+		 * the socket has been exposed to the filter(s) earlier.
+		 */
+		so->so_state &= ~(SS_NOFDREF | SS_COMP);
+		so->so_head = NULL;
+		socket_unlock(so, 1);
+		soclose(so);
+		/* Propagate socket filter's error code to the caller */
+	} else {
+		socket_unlock(so, 1);
+	}
+done:
+	/* Callee checks for NULL pointer */
+	sock_freeaddr(remote);
+	sock_freeaddr(local);
+	return (error);
+}
 
+/*
+ * Returns:	0			Success
+ *		EOPNOTSUPP		Operation not supported on socket
+ *		EISCONN			Socket is connected
+ *	<pru_connect>:EADDRNOTAVAIL	Address not available.
+ *	<pru_connect>:EINVAL		Invalid argument
+ *	<pru_connect>:EAFNOSUPPORT	Address family not supported [notdef]
+ *	<pru_connect>:EACCES		Permission denied
+ *	<pru_connect>:EADDRINUSE	Address in use
+ *	<pru_connect>:EAGAIN		Resource unavailable, try again
+ *	<pru_connect>:EPERM		Operation not permitted
+ *	<sf_connect_out>:???		[anything a filter writer might set]
+ */
+int
+soconnectlock(struct socket *so, struct sockaddr *nam, int dolock)
 {
-	int s;
 	int error;
 	struct proc *p = current_proc();
 
-	if (dolock) socket_lock(so, 1);
+	if (dolock)
+		socket_lock(so, 1);
 
-	if (so->so_options & SO_ACCEPTCONN) {
-		if (dolock) socket_unlock(so, 1);
+	/*
+	 * If this is a listening socket or if this is a previously-accepted
+	 * socket that has been marked as inactive, reject the connect request.
+	 */
+	if ((so->so_options & SO_ACCEPTCONN) || (so->so_flags & SOF_DEFUNCT)) {
+		if (dolock)
+			socket_unlock(so, 1);
 		return (EOPNOTSUPP);
 	}
+
+	if ((so->so_restrictions & SO_RESTRICT_DENYOUT) != 0) {
+		if (dolock)
+			socket_unlock(so, 1);
+		return (EPERM);
+	}
+
 	/*
 	 * If protocol is connection-based, can only connect once.
 	 * Otherwise, if connected, try to disconnect first.
@@ -913,78 +1202,82 @@ soconnectlock(so, nam, dolock)
 	 */
 	if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
 	    ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
-	    (error = sodisconnectlocked(so))))
+	    (error = sodisconnectlocked(so)))) {
 		error = EISCONN;
-	else {
+	} else {
 		/*
 		 * Run connect filter before calling protocol:
 		 *  - non-blocking connect returns before completion;
 		 */
-		{
-			struct socket_filter_entry	*filter;
-			int						 	filtered = 0;
-			error = 0;
-			for (filter = so->so_filt; filter && (error == 0);
-				 filter = filter->sfe_next_onsocket) {
-				if (filter->sfe_filter->sf_filter.sf_connect_out) {
-					if (filtered == 0) {
-						filtered = 1;
-						sflt_use(so);
-						socket_unlock(so, 0);
-					}
-					error = filter->sfe_filter->sf_filter.sf_connect_out(
-								filter->sfe_cookie, so, nam);
+		struct socket_filter_entry *filter;
+		int filtered = 0;
+
+		error = 0;
+		for (filter = so->so_filt; filter && (error == 0);
+		    filter = filter->sfe_next_onsocket) {
+			if (filter->sfe_filter->sf_filter.sf_connect_out) {
+				if (filtered == 0) {
+					filtered = 1;
+					sflt_use(so);
+					socket_unlock(so, 0);
 				}
+				error = filter->sfe_filter->sf_filter.
+				    sf_connect_out(filter->sfe_cookie, so, nam);
 			}
-			if (filtered != 0) {
-				socket_lock(so, 0);
-				sflt_unuse(so);
-			}
 		}
+		if (filtered != 0) {
+			socket_lock(so, 0);
+			sflt_unuse(so);
+		}
+
 		if (error) {
 			if (error == EJUSTRETURN)
 				error = 0;
-			if (dolock) socket_unlock(so, 1);
-			return error;
+			if (dolock)
+				socket_unlock(so, 1);
+			return (error);
 		}
-		
+
 		error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, p);
 	}
-	if (dolock) socket_unlock(so, 1);
+	if (dolock)
+		socket_unlock(so, 1);
 	return (error);
 }
 
 int
-soconnect(so, nam)
-	register struct socket *so;
-	struct sockaddr *nam;
+soconnect(struct socket *so, struct sockaddr *nam)
 {
 	return (soconnectlock(so, nam, 1));
 }
 
+/*
+ * Returns:	0			Success
+ *	<pru_connect2>:EINVAL[AF_UNIX]
+ *	<pru_connect2>:EPROTOTYPE[AF_UNIX]
+ *	<pru_connect2>:???		[other protocol families]
+ *
+ * Notes:	<pru_connect2> is not supported by [TCP].
+ */
 int
-soconnect2(so1, so2)
-	register struct socket *so1;
-	struct socket *so2;
+soconnect2(struct socket *so1, struct socket *so2)
 {
 	int error;
 
 	socket_lock(so1, 1);
-	if (so2->so_proto->pr_lock) 
+	if (so2->so_proto->pr_lock)
 		socket_lock(so2, 1);
 
 	error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2);
-	
+
 	socket_unlock(so1, 1);
-	if (so2->so_proto->pr_lock) 
+	if (so2->so_proto->pr_lock)
 		socket_unlock(so2, 1);
 	return (error);
 }
 
-
 int
-sodisconnectlocked(so)
-	register struct socket *so;
+sodisconnectlocked(struct socket *so)
 {
 	int error;
 
@@ -996,27 +1289,26 @@ sodisconnectlocked(so)
 		error = EALREADY;
 		goto bad;
 	}
-	
+
 	error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so);
-	
+
 	if (error == 0) {
 		sflt_notify(so, sock_evt_disconnected, NULL);
 	}
-
 bad:
 	return (error);
 }
-//### Locking version
+
+/* Locking version */
 int
-sodisconnect(so)
-	register struct socket *so;
+sodisconnect(struct socket *so)
 {
-	int error; 
+	int error;
 
 	socket_lock(so, 1);
 	error = sodisconnectlocked(so);
 	socket_unlock(so, 1);
-	return(error);
+	return (error);
 }
 
 #define	SBLOCKWAIT(f)	(((f) & MSG_DONTWAIT) ? M_DONTWAIT : M_WAIT)
@@ -1024,17 +1316,18 @@ sodisconnect(so)
 /*
  * sosendcheck will lock the socket buffer if it isn't locked and
  * verify that there is space for the data being inserted.
+ *
+ * Returns:	0			Success
+ *		EPIPE
+ *	sblock:EWOULDBLOCK
+ *	sblock:EINTR
+ *	sbwait:EBADF
+ *	sbwait:EINTR
+ *	[so_error]:???
  */
-
 static int
-sosendcheck(
-	struct socket *so,
-	struct sockaddr *addr,
-	long resid,
-	long clen,
-	long atomic,
-	int flags,
-	int *sblocked)
+sosendcheck(struct socket *so, struct sockaddr *addr, long resid, long clen,
+    long atomic, int flags, int *sblocked)
 {
 	int error = 0;
 	long space;
@@ -1043,8 +1336,8 @@ sosendcheck(
 restart:
 	if (*sblocked == 0) {
 		if ((so->so_snd.sb_flags & SB_LOCK) != 0 &&
-			so->so_send_filt_thread != 0 &&
-			so->so_send_filt_thread == current_thread()) {
+		    so->so_send_filt_thread != 0 &&
+		    so->so_send_filt_thread == current_thread()) {
 			/*
 			 * We're being called recursively from a filter,
 			 * allow this to continue. Radar 4150520.
@@ -1052,60 +1345,63 @@ restart:
 			 * to perform an unlock later.
 			 */
 			assumelock = 1;
-		}
-		else {
+		} else {
 			error = sblock(&so->so_snd, SBLOCKWAIT(flags));
 			if (error) {
-				return error;
+				return (error);
 			}
 			*sblocked = 1;
 		}
 	}
-	
-	if (so->so_state & SS_CANTSENDMORE) 
-		return EPIPE;
-	
+
+	/*
+	 * If a send attempt is made on a previously-accepted socket
+	 * that has been marked as inactive (disconnected), reject
+	 * the request.
+	 */
+	if (so->so_flags & SOF_DEFUNCT)
+		return (ENOTCONN);
+
+	if (so->so_state & SS_CANTSENDMORE)
+		return (EPIPE);
+
 	if (so->so_error) {
 		error = so->so_error;
 		so->so_error = 0;
-		return error;
+		return (error);
 	}
-	
+
 	if ((so->so_state & SS_ISCONNECTED) == 0) {
-		/*
-		 * `sendto' and `sendmsg' is allowed on a connection-
-		 * based socket if it supports implied connect.
-		 * Return ENOTCONN if not connected and no address is
-		 * supplied.
-		 */
-		if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
-			(so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
+		if ((so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
 			if ((so->so_state & SS_ISCONFIRMING) == 0 &&
-				!(resid == 0 && clen != 0))
-				return ENOTCONN;
-		} else if (addr == 0 && !(flags&MSG_HOLD))
-			return (so->so_proto->pr_flags & PR_CONNREQUIRED) ? ENOTCONN : EDESTADDRREQ;
+			    !(resid == 0 && clen != 0))
+				return (ENOTCONN);
+		} else if (addr == 0 && !(flags&MSG_HOLD)) {
+			return ((so->so_proto->pr_flags & PR_CONNREQUIRED) ?
+			    ENOTCONN : EDESTADDRREQ);
+		}
 	}
 	space = sbspace(&so->so_snd);
 	if (flags & MSG_OOB)
 		space += 1024;
 	if ((atomic && resid > so->so_snd.sb_hiwat) ||
-		clen > so->so_snd.sb_hiwat)
-		return EMSGSIZE;
-	if (space < resid + clen && 
-		(atomic || space < so->so_snd.sb_lowat || space < clen)) {
-		if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO) || assumelock) {
-			return EWOULDBLOCK;
+	    clen > so->so_snd.sb_hiwat)
+		return (EMSGSIZE);
+	if (space < resid + clen &&
+	    (atomic || space < (long)so->so_snd.sb_lowat || space < clen)) {
+		if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO) ||
+		    assumelock) {
+			return (EWOULDBLOCK);
 		}
 		sbunlock(&so->so_snd, 1);
 		error = sbwait(&so->so_snd);
 		if (error) {
-			return error;
+			return (error);
 		}
 		goto restart;
 	}
-	
-	return 0;
+
+	return (0);
 }
 
 /*
@@ -1128,16 +1424,49 @@ restart:
  * MSG_HOLD: go thru most of sosend(), but just enqueue the mbuf
  * MSG_SEND: go thru as for MSG_HOLD on current fragment, then
  *  point at the mbuf chain being constructed and go from there.
+ *
+ * Returns:	0			Success
+ *		EOPNOTSUPP
+ *		EINVAL
+ *		ENOBUFS
+ *	uiomove:EFAULT
+ *	sosendcheck:EPIPE
+ *	sosendcheck:EWOULDBLOCK
+ *	sosendcheck:EINTR
+ *	sosendcheck:EBADF
+ *	sosendcheck:EINTR
+ *	sosendcheck:???			[value from so_error]
+ *	<pru_send>:ECONNRESET[TCP]
+ *	<pru_send>:EINVAL[TCP]
+ *	<pru_send>:ENOBUFS[TCP]
+ *	<pru_send>:EADDRINUSE[TCP]
+ *	<pru_send>:EADDRNOTAVAIL[TCP]
+ *	<pru_send>:EAFNOSUPPORT[TCP]
+ *	<pru_send>:EACCES[TCP]
+ *	<pru_send>:EAGAIN[TCP]
+ *	<pru_send>:EPERM[TCP]
+ *	<pru_send>:EMSGSIZE[TCP]
+ *	<pru_send>:EHOSTUNREACH[TCP]
+ *	<pru_send>:ENETUNREACH[TCP]
+ *	<pru_send>:ENETDOWN[TCP]
+ *	<pru_send>:ENOMEM[TCP]
+ *	<pru_send>:ENOBUFS[TCP]
+ *	<pru_send>:???[TCP]		[ignorable: mostly IPSEC/firewall/DLIL]
+ *	<pru_send>:EINVAL[AF_UNIX]
+ *	<pru_send>:EOPNOTSUPP[AF_UNIX]
+ *	<pru_send>:EPIPE[AF_UNIX]
+ *	<pru_send>:ENOTCONN[AF_UNIX]
+ *	<pru_send>:EISCONN[AF_UNIX]
+ *	<pru_send>:???[AF_UNIX]		[whatever a filter author chooses]
+ *	<sf_data_out>:???		[whatever a filter author chooses]
+ *
+ * Notes:	Other <pru_send> returns depend on the protocol family; all
+ *		<sf_data_out> returns depend on what the filter author causes
+ *		their filter to return.
  */
 int
-sosend(so, addr, uio, top, control, flags)
-	register struct socket *so;
-	struct sockaddr *addr;
-	struct uio *uio;
-	struct mbuf *top;
-	struct mbuf *control;
-	int flags;
-
+sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
+    struct mbuf *top, struct mbuf *control, int flags)
 {
 	struct mbuf **mp;
 	register struct mbuf *m, *freelist = NULL;
@@ -1147,20 +1476,21 @@ sosend(so, addr, uio, top, control, flags)
 	int sblocked = 0;
 	struct proc *p = current_proc();
 
-	if (uio)
+	if (uio) {
 		// LP64todo - fix this!
 		resid = uio_resid(uio);
-	else
+	} else {
 		resid = top->m_pkthdr.len;
-
-	KERNEL_DEBUG((DBG_FNC_SOSEND | DBG_FUNC_START),
-		     so,
-		     resid,
-		     so->so_snd.sb_cc,
-		     so->so_snd.sb_lowat,
-		     so->so_snd.sb_hiwat);
+	}
+	KERNEL_DEBUG((DBG_FNC_SOSEND | DBG_FUNC_START), so, resid,
+	    so->so_snd.sb_cc, so->so_snd.sb_lowat, so->so_snd.sb_hiwat);
 
 	socket_lock(so, 1);
+	if (so->so_type != SOCK_STREAM && (flags & MSG_OOB) != 0) {
+		error = EOPNOTSUPP;
+		socket_unlock(so, 1);
+		goto out;
+	}
 
 	/*
 	 * In theory resid should be unsigned.
@@ -1182,21 +1512,26 @@ sosend(so, addr, uio, top, control, flags)
 	    (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
 	    (so->so_proto->pr_flags & PR_ATOMIC);
 	if (p)
-		p->p_stats->p_ru.ru_msgsnd++;
+		OSIncrementAtomic(&p->p_stats->p_ru.ru_msgsnd);	
 	if (control)
 		clen = control->m_len;
 
 	do {
-		error = sosendcheck(so, addr, resid, clen, atomic, flags, &sblocked);
+		error = sosendcheck(so, addr, resid, clen, atomic, flags,
+		    &sblocked);
 		if (error) {
 			goto release;
 		}
 		mp = &top;
-		space = sbspace(&so->so_snd) - clen + ((flags & MSG_OOB) ? 1024 : 0);
+		space = sbspace(&so->so_snd) - clen + ((flags & MSG_OOB) ?
+		    1024 : 0);
 
 		do {
-			
-		    if (uio == NULL) {
+			struct socket_filter_entry *filter;
+			int filtered;
+			boolean_t recursive;
+
+			if (uio == NULL) {
 				/*
 				 * Data is prepackaged in "top".
 				 */
@@ -1204,55 +1539,119 @@ sosend(so, addr, uio, top, control, flags)
 				if (flags & MSG_EOR)
 					top->m_flags |= M_EOR;
 			} else {
-				int             chainlength;
-				int             bytes_to_copy;
-	
+				int chainlength;
+				int bytes_to_copy;
+				boolean_t jumbocl;
+
 				bytes_to_copy = min(resid, space);
-	
+
 				if (sosendminchain > 0) {
 					chainlength = 0;
-				} else
+				} else {
 					chainlength = sosendmaxchain;
-	
+				}
+
+				/*
+				 * Attempt to use larger than system page-size
+				 * clusters for large writes only if there is
+				 * a jumbo cluster pool and if the socket is
+				 * marked accordingly.
+				 */
+				jumbocl = sosendjcl && njcl > 0 &&
+				    ((so->so_flags & SOF_MULTIPAGES) ||
+				    sosendjcl_ignore_capab);
+
 				socket_unlock(so, 0);
-	
+
 				do {
 					int num_needed;
 					int hdrs_needed = (top == 0) ? 1 : 0;
-					
+
 					/*
-					 * try to maintain a local cache of mbuf clusters needed to complete this write
-					 * the list is further limited to the number that are currently needed to fill the socket
-					 * this mechanism allows a large number of mbufs/clusters to be grabbed under a single 
-					 * mbuf lock... if we can't get any clusters, than fall back to trying for mbufs
-					 * if we fail early (or miscalcluate the number needed) make sure to release any clusters
-					 * we haven't yet consumed.
+					 * try to maintain a local cache of mbuf
+					 * clusters needed to complete this
+					 * write the list is further limited to
+					 * the number that are currently needed
+					 * to fill the socket this mechanism
+					 * allows a large number of mbufs/
+					 * clusters to be grabbed under a single
+					 * mbuf lock... if we can't get any
+					 * clusters, than fall back to trying
+					 * for mbufs if we fail early (or
+					 * miscalcluate the number needed) make
+					 * sure to release any clusters we
+					 * haven't yet consumed.
 					 */
-					if (freelist == NULL && bytes_to_copy > MCLBYTES) {
-						num_needed = bytes_to_copy / NBPG;
+					if (freelist == NULL &&
+					    bytes_to_copy > NBPG && jumbocl) {
+						num_needed =
+						    bytes_to_copy / M16KCLBYTES;
+
+						if ((bytes_to_copy -
+						    (num_needed * M16KCLBYTES))
+						    >= MINCLSIZE)
+							num_needed++;
 
-						if ((bytes_to_copy - (num_needed * NBPG)) >= MINCLSIZE)
+						freelist =
+						    m_getpackets_internal(
+						    (unsigned int *)&num_needed,
+						    hdrs_needed, M_WAIT, 0,
+						    M16KCLBYTES);
+						/*
+						 * Fall back to 4K cluster size
+						 * if allocation failed
+						 */
+					}
+
+					if (freelist == NULL &&
+					    bytes_to_copy > MCLBYTES) {
+						num_needed =
+						    bytes_to_copy / NBPG;
+
+						if ((bytes_to_copy -
+						    (num_needed * NBPG)) >=
+						    MINCLSIZE)
 							num_needed++;
-						
-						freelist = m_getpackets_internal(&num_needed, hdrs_needed, M_WAIT, 0, NBPG);
-						/* Fall back to cluster size if allocation failed */
+
+						freelist =
+						    m_getpackets_internal(
+						    (unsigned int *)&num_needed,
+						    hdrs_needed, M_WAIT, 0,
+						    NBPG);
+						/*
+						 * Fall back to cluster size
+						 * if allocation failed
+						 */
 					}
-					
-					if (freelist == NULL && bytes_to_copy > MINCLSIZE) {
-						num_needed = bytes_to_copy / MCLBYTES;
-					
-						if ((bytes_to_copy - (num_needed * MCLBYTES)) >= MINCLSIZE)
+
+					if (freelist == NULL &&
+					    bytes_to_copy > MINCLSIZE) {
+						num_needed =
+						    bytes_to_copy / MCLBYTES;
+
+						if ((bytes_to_copy -
+						    (num_needed * MCLBYTES)) >=
+						    MINCLSIZE)
 							num_needed++;
-						
-						freelist = m_getpackets_internal(&num_needed, hdrs_needed, M_WAIT, 0, MCLBYTES);
-						/* Fall back to a single mbuf if allocation failed */
+
+						freelist =
+						    m_getpackets_internal(
+						    (unsigned int *)&num_needed,
+						    hdrs_needed, M_WAIT, 0,
+						    MCLBYTES);
+						/*
+						 * Fall back to a single mbuf
+						 * if allocation failed
+						 */
 					}
-					
+
 					if (freelist == NULL) {
 						if (top == 0)
-							MGETHDR(freelist, M_WAIT, MT_DATA);
+							MGETHDR(freelist,
+							    M_WAIT, MT_DATA);
 						else
-							MGET(freelist, M_WAIT, MT_DATA);
+							MGET(freelist,
+							    M_WAIT, MT_DATA);
 
 						if (freelist == NULL) {
 							error = ENOBUFS;
@@ -1260,37 +1659,43 @@ sosend(so, addr, uio, top, control, flags)
 							goto release;
 						}
 						/*
-						 * For datagram protocols, leave room
-						 * for protocol headers in first mbuf.
+						 * For datagram protocols,
+						 * leave room for protocol
+						 * headers in first mbuf.
 						 */
-						if (atomic && top == 0 && bytes_to_copy < MHLEN)
-							MH_ALIGN(freelist, bytes_to_copy);
+						if (atomic && top == 0 &&
+						    bytes_to_copy < MHLEN) {
+							MH_ALIGN(freelist,
+							    bytes_to_copy);
+						}
 					}
 					m = freelist;
 					freelist = m->m_next;
 					m->m_next = NULL;
-					
+
 					if ((m->m_flags & M_EXT))
 						mlen = m->m_ext.ext_size;
 					else if ((m->m_flags & M_PKTHDR))
-						mlen = MHLEN - m_leadingspace(m);
+						mlen =
+						    MHLEN - m_leadingspace(m);
 					else
 						mlen = MLEN;
 					len = min(mlen, bytes_to_copy);
 
 					chainlength += len;
-					
+
 					space -= len;
 
-					error = uiomove(mtod(m, caddr_t), (int)len, uio);
-		
+					error = uiomove(mtod(m, caddr_t),
+					    (int)len, uio);
+
 					// LP64todo - fix this!
 					resid = uio_resid(uio);
-					
+
 					m->m_len = len;
 					*mp = m;
 					top->m_pkthdr.len += len;
-					if (error) 
+					if (error)
 						break;
 					mp = &m->m_next;
 					if (resid <= 0) {
@@ -1299,21 +1704,21 @@ sosend(so, addr, uio, top, control, flags)
 						break;
 					}
 					bytes_to_copy = min(resid, space);
-				
-				} while (space > 0 && (chainlength < sosendmaxchain || atomic || resid < MINCLSIZE));
-		
+
+				} while (space > 0 &&
+				    (chainlength < sosendmaxchain || atomic ||
+				    resid < MINCLSIZE));
+
 				socket_lock(so, 0);
-	
+
 				if (error)
 					goto release;
 			}
-            
-		    if (flags & (MSG_HOLD|MSG_SEND))
-		    {
+
+			if (flags & (MSG_HOLD|MSG_SEND)) {
 				/* Enqueue for later, go away if HOLD */
 				register struct mbuf *mb1;
-				if (so->so_temp && (flags & MSG_FLUSH))
-				{
+				if (so->so_temp && (flags & MSG_FLUSH)) {
 					m_freem(so->so_temp);
 					so->so_temp = NULL;
 				}
@@ -1323,100 +1728,106 @@ sosend(so, addr, uio, top, control, flags)
 					so->so_temp = top;
 				mb1 = top;
 				while (mb1->m_next)
-						mb1 = mb1->m_next;
+					mb1 = mb1->m_next;
 				so->so_tail = mb1;
-				if (flags & MSG_HOLD)
-				{
+				if (flags & MSG_HOLD) {
 					top = NULL;
 					goto release;
 				}
 				top = so->so_temp;
-		    }
-		    if (dontroute)
-			    so->so_options |= SO_DONTROUTE;
-		    /* Compute flags here, for pru_send and NKEs */
-		    sendflags = (flags & MSG_OOB) ? PRUS_OOB :
-			/*
-			 * If the user set MSG_EOF, the protocol
-			 * understands this flag and nothing left to
-			 * send then use PRU_SEND_EOF instead of PRU_SEND.
-			 */
-			((flags & MSG_EOF) &&
-			 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
-			 (resid <= 0)) ?
+			}
+			if (dontroute)
+				so->so_options |= SO_DONTROUTE;
+
+			/* Compute flags here, for pru_send and NKEs */
+			sendflags = (flags & MSG_OOB) ? PRUS_OOB :
+			    /*
+			     * If the user set MSG_EOF, the protocol
+			     * understands this flag and nothing left to
+			     * send then use PRU_SEND_EOF instead of PRU_SEND.
+			     */
+			    ((flags & MSG_EOF) &&
+			     (so->so_proto->pr_flags & PR_IMPLOPCL) &&
+			     (resid <= 0)) ?
 				PRUS_EOF :
-			/* If there is more to send set PRUS_MORETOCOME */
-			(resid > 0 && space > 0) ? PRUS_MORETOCOME : 0;
-			
+			    /* If there is more to send set PRUS_MORETOCOME */
+			    (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0;
+
 			/*
 			 * Socket filter processing
 			 */
-			{
-				struct socket_filter_entry *filter;
-				int							filtered;
-				
-				filtered = 0;
-				error = 0;
-				for (filter = so->so_filt; filter && (error == 0);
-					 filter = filter->sfe_next_onsocket) {
-					if (filter->sfe_filter->sf_filter.sf_data_out) {
-						int so_flags = 0;
-						if (filtered == 0) {
-							filtered = 1;
-							so->so_send_filt_thread = current_thread();
-							sflt_use(so);
-							socket_unlock(so, 0);
-							so_flags = (sendflags & MSG_OOB) ? sock_data_filt_flag_oob : 0;
-						}
-						error = filter->sfe_filter->sf_filter.sf_data_out(
-									filter->sfe_cookie, so, addr, &top, &control, so_flags);
+			recursive = (so->so_send_filt_thread != NULL);
+			filtered = 0;
+			error = 0;
+			for (filter = so->so_filt; filter && (error == 0);
+			    filter = filter->sfe_next_onsocket) {
+				if (filter->sfe_filter->sf_filter.sf_data_out) {
+					int so_flags = 0;
+					if (filtered == 0) {
+						filtered = 1;
+						so->so_send_filt_thread =
+						    current_thread();
+						sflt_use(so);
+						socket_unlock(so, 0);
+						so_flags =
+						    (sendflags & MSG_OOB) ?
+						    sock_data_filt_flag_oob : 0;
 					}
+					error = filter->sfe_filter->sf_filter.
+					    sf_data_out(filter->sfe_cookie, so,
+					    addr, &top, &control, so_flags);
 				}
-				
-				if (filtered) {
-					/*
-					 * At this point, we've run at least one filter.
-					 * The socket is unlocked as is the socket buffer.
-					 */
-					socket_lock(so, 0);
-					sflt_unuse(so);
+			}
+
+			if (filtered) {
+				/*
+				 * At this point, we've run at least one
+				 * filter.  The socket is unlocked as is
+				 * the socket buffer.  Clear the recorded
+				 * filter thread only when we are outside
+				 * of a filter's context.  This allows for
+				 * a filter to issue multiple inject calls
+				 * from its sf_data_out callback routine.
+				 */
+				socket_lock(so, 0);
+				sflt_unuse(so);
+				if (!recursive)
 					so->so_send_filt_thread = 0;
-					if (error) {
-						if (error == EJUSTRETURN) {
-							error = 0;
-							clen = 0;
-							control = 0;
-							top = 0;
-						}
-						
-						goto release;
+				if (error) {
+					if (error == EJUSTRETURN) {
+						error = 0;
+						clen = 0;
+						control = 0;
+						top = 0;
 					}
+
+					goto release;
 				}
 			}
 			/*
 			 * End Socket filter processing
 			 */
-			
+
 			if (error == EJUSTRETURN) {
 				/* A socket filter handled this data */
 				error = 0;
-			}
-			else {
-				error = (*so->so_proto->pr_usrreqs->pru_send)(so,
-							sendflags, top, addr, control, p);
+			} else {
+				error = (*so->so_proto->pr_usrreqs->pru_send)
+				    (so, sendflags, top, addr, control, p);
 			}
 #ifdef __APPLE__
-		    if (flags & MSG_SEND)
-		    	so->so_temp = NULL;
+			if (flags & MSG_SEND)
+				so->so_temp = NULL;
 #endif
-		    if (dontroute)
-			    so->so_options &= ~SO_DONTROUTE;
-		    clen = 0;
-		    control = 0;
-		    top = 0;
-		    mp = &top;
-		    if (error)
-			goto release;
+			if (dontroute)
+				so->so_options &= ~SO_DONTROUTE;
+
+			clen = 0;
+			control = 0;
+			top = 0;
+			mp = &top;
+			if (error)
+				goto release;
 		} while (resid && space > 0);
 	} while (resid);
 
@@ -1431,14 +1842,10 @@ out:
 	if (control)
 		m_freem(control);
 	if (freelist)
-	        m_freem_list(freelist);     
+		m_freem_list(freelist);
 
-	KERNEL_DEBUG(DBG_FNC_SOSEND | DBG_FUNC_END,
-		     so,
-		     resid,
-		     so->so_snd.sb_cc,
-		     space,
-		     error);
+	KERNEL_DEBUG(DBG_FNC_SOSEND | DBG_FUNC_END, so, resid, so->so_snd.sb_cc,
+	    space, error);
 
 	return (error);
 }
@@ -1458,15 +1865,31 @@ out:
  * The caller may receive the data as a single mbuf chain by supplying
  * an mbuf **mp0 for use in returning the chain.  The uio is then used
  * only for the count in uio_resid.
+ *
+ * Returns:	0			Success
+ *		ENOBUFS
+ *		ENOTCONN
+ *		EWOULDBLOCK
+ *	uiomove:EFAULT
+ *	sblock:EWOULDBLOCK
+ *	sblock:EINTR
+ *	sbwait:EBADF
+ *	sbwait:EINTR
+ *	sodelayed_copy:EFAULT
+ *	<pru_rcvoob>:EINVAL[TCP]
+ *	<pru_rcvoob>:EWOULDBLOCK[TCP]
+ *	<pru_rcvoob>:???
+ *	<pr_domain->dom_externalize>:EMSGSIZE[AF_UNIX]
+ *	<pr_domain->dom_externalize>:ENOBUFS[AF_UNIX]
+ *	<pr_domain->dom_externalize>:???
+ *
+ * Notes:	Additional return values from calls through <pru_rcvoob> and
+ *		<pr_domain->dom_externalize> depend on protocols other than
+ *		TCP or AF_UNIX, which are documented above.
  */
 int
-soreceive(so, psa, uio, mp0, controlp, flagsp)
-	register struct socket *so;
-	struct sockaddr **psa;
-	struct uio *uio;
-	struct mbuf **mp0;
-	struct mbuf **controlp;
-	int *flagsp;
+soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
+    struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
 {
 	register struct mbuf *m, **mp, *ml = NULL;
 	register int flags, len, error, offset;
@@ -1475,20 +1898,15 @@ soreceive(so, psa, uio, mp0, controlp, flagsp)
 	int moff, type = 0;
 		// LP64todo - fix this!
 	int orig_resid = uio_resid(uio);
-	volatile struct mbuf *free_list;
-	volatile int delayed_copy_len;
+	struct mbuf *free_list;
+	int delayed_copy_len;
 	int can_delay;
 	int need_event;
 	struct proc *p = current_proc();
 
-
-		// LP64todo - fix this!
-	KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_START,
-		     so,
-		     uio_resid(uio),
-		     so->so_rcv.sb_cc,
-		     so->so_rcv.sb_lowat,
-		     so->so_rcv.sb_hiwat);
+	// LP64todo - fix this!
+	KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_START, so, uio_resid(uio),
+	    so->so_rcv.sb_cc, so->so_rcv.sb_lowat, so->so_rcv.sb_hiwat);
 
 	socket_lock(so, 1);
 
@@ -1505,19 +1923,39 @@ soreceive(so, psa, uio, mp0, controlp, flagsp)
 		flags = *flagsp &~ MSG_EOR;
 	else
 		flags = 0;
-        /*
-         * When SO_WANTOOBFLAG is set we try to get out-of-band data 
-         * regardless of the flags argument. Here is the case were 
-         * out-of-band data is not inline.
-         */
-	if ((flags & MSG_OOB) || 
-            ((so->so_options & SO_WANTOOBFLAG) != 0 && 
-             (so->so_options & SO_OOBINLINE) == 0 &&
-             (so->so_oobmark || (so->so_state & SS_RCVATMARK)))) {
+
+	/*
+	 * If a recv attempt is made on a previously-accepted socket
+	 * that has been marked as inactive (disconnected), reject
+	 * the request.
+	 */
+	if (so->so_flags & SOF_DEFUNCT) {
+		struct sockbuf *sb = &so->so_rcv;
+
+		/*
+		 * This socket should have been disconnected and flushed
+		 * prior to being returned from accept; there should be
+		 * no data on its receive list, so panic otherwise.
+		 */
+		sb_empty_assert(sb, __func__);
+		socket_unlock(so, 1);
+		return (ENOTCONN);
+	}
+
+	/*
+	 * When SO_WANTOOBFLAG is set we try to get out-of-band data
+	 * regardless of the flags argument. Here is the case were
+	 * out-of-band data is not inline.
+	 */
+	if ((flags & MSG_OOB) ||
+	    ((so->so_options & SO_WANTOOBFLAG) != 0 &&
+	    (so->so_options & SO_OOBINLINE) == 0 &&
+	    (so->so_oobmark || (so->so_state & SS_RCVATMARK)))) {
 		m = m_get(M_WAIT, MT_DATA);
 		if (m == NULL) {
 			socket_unlock(so, 1);
-			KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, ENOBUFS,0,0,0,0);
+			KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END,
+			    ENOBUFS, 0, 0, 0, 0);
 			return (ENOBUFS);
 		}
 		error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK);
@@ -1527,7 +1965,7 @@ soreceive(so, psa, uio, mp0, controlp, flagsp)
 		do {
 		// LP64todo - fix this!
 			error = uiomove(mtod(m, caddr_t),
-			    (int) min(uio_resid(uio), m->m_len), uio);
+			    (int)min(uio_resid(uio), m->m_len), uio);
 			m = m_free(m);
 		} while (uio_resid(uio) && error == 0 && m);
 		socket_lock(so, 0);
@@ -1537,18 +1975,21 @@ bad:
 #ifdef __APPLE__
 		if ((so->so_options & SO_WANTOOBFLAG) != 0) {
 			if (error == EWOULDBLOCK || error == EINVAL) {
-				/* 
+				/*
 				 * Let's try to get normal data:
-				 *  EWOULDBLOCK: out-of-band data not receive yet;
-				 *  EINVAL: out-of-band data already read.
+				 * EWOULDBLOCK: out-of-band data not
+				 * receive yet. EINVAL: out-of-band data
+				 * already read.
 				 */
 				error = 0;
 				goto nooob;
-			} else if (error == 0 && flagsp)
+			} else if (error == 0 && flagsp) {
 				*flagsp |= MSG_OOB;
-		}	
+			}
+		}
 		socket_unlock(so, 1);
-		KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,0,0,0,0);
+		KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,
+		    0, 0, 0, 0);
 #endif
 		return (error);
 	}
@@ -1564,7 +2005,8 @@ nooob:
 restart:
 #ifdef MORE_LOCKING_DEBUG
 	if (so->so_usecount <= 1)
-		printf("soreceive: sblock so=%x ref=%d on socket\n", so, so->so_usecount);
+		printf("soreceive: sblock so=%p ref=%d on socket\n",
+		    so, so->so_usecount);
 #endif
 	/*
 	 * See if the socket has been closed (SS_NOFDREF|SS_CANTRCVMORE)
@@ -1587,7 +2029,8 @@ restart:
 	error = sblock(&so->so_rcv, SBLOCKWAIT(flags));
 	if (error) {
 		socket_unlock(so, 1);
-		KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,0,0,0,0);
+		KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,
+		    0, 0, 0, 0);
 		return (error);
 	}
 
@@ -1605,11 +2048,20 @@ restart:
 	 */
 	if (m == 0 || (((flags & MSG_DONTWAIT) == 0 &&
 	    so->so_rcv.sb_cc < uio_resid(uio)) &&
-           (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
+	    (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
 	    ((flags & MSG_WAITALL) && uio_resid(uio) <= so->so_rcv.sb_hiwat)) &&
 	    m->m_nextpkt == 0 && (pr->pr_flags & PR_ATOMIC) == 0)) {
+		/*
+		 * Panic if we notice inconsistencies in the socket's
+		 * receive list; both sb_mb and sb_cc should correctly
+		 * reflect the contents of the list, otherwise we may
+		 * end up with false positives during select() or poll()
+		 * which could put the application in a bad state.
+		 */
+		if (m == NULL && so->so_rcv.sb_cc != 0)
+			panic("soreceive corrupted so_rcv: m %p cc %lu",
+			    m, so->so_rcv.sb_cc);
 
-		KASSERT(m != 0 || !so->so_rcv.sb_cc, ("receive 1"));
 		if (so->so_error) {
 			if (m)
 				goto dontblock;
@@ -1625,7 +2077,7 @@ restart:
 				goto release;
 		}
 		for (; m; m = m->m_next)
-			if (m->m_type == MT_OOBDATA  || (m->m_flags & M_EOR)) {
+			if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
 				m = so->so_rcv.sb_mb;
 				goto dontblock;
 			}
@@ -1636,27 +2088,32 @@ restart:
 		}
 		if (uio_resid(uio) == 0)
 			goto release;
-		if ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO))) {
+		if ((so->so_state & SS_NBIO) ||
+		    (flags & (MSG_DONTWAIT|MSG_NBIO))) {
 			error = EWOULDBLOCK;
 			goto release;
 		}
+		SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
+		SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
 		sbunlock(&so->so_rcv, 1);
-#ifdef EVEN_MORE_LOCKING_DEBUG
+#if EVEN_MORE_LOCKING_DEBUG
 		if (socket_debug)
-		    printf("Waiting for socket data\n");
+			printf("Waiting for socket data\n");
 #endif
 
 		error = sbwait(&so->so_rcv);
-#ifdef EVEN_MORE_LOCKING_DEBUG
+#if EVEN_MORE_LOCKING_DEBUG
 		if (socket_debug)
-		    printf("SORECEIVE - sbwait returned %d\n", error);
+			printf("SORECEIVE - sbwait returned %d\n", error);
 #endif
 		if (so->so_usecount < 1)
-			panic("soreceive: after 2nd sblock so=%x ref=%d on socket\n", so, so->so_usecount);
+			panic("soreceive: after 2nd sblock so=%p ref=%d on "
+			    "socket\n", so, so->so_usecount);
 		if (error) {
 			socket_unlock(so, 1);
-		    KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,0,0,0,0);
-		    return (error);
+			KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, error,
+			    0, 0, 0, 0);
+			return (error);
 		}
 		goto restart;
 	}
@@ -1672,15 +2129,84 @@ dontblock:
 	 * fill out the uio structure properly.
 	 */
 	if (p)
-		p->p_stats->p_ru.ru_msgrcv++;
+		OSIncrementAtomic(&p->p_stats->p_ru.ru_msgrcv);
 #endif	/* __APPLE__ */
+	SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
+	SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
 	nextrecord = m->m_nextpkt;
 	if ((pr->pr_flags & PR_ADDR) && m->m_type == MT_SONAME) {
 		KASSERT(m->m_type == MT_SONAME, ("receive 1a"));
+#if CONFIG_MACF_SOCKET_SUBSET
+		/*
+		 * Call the MAC framework for policy checking if we're in
+		 * the user process context and the socket isn't connected.
+		 */
+		if (p != kernproc && !(so->so_state & SS_ISCONNECTED)) {
+			struct mbuf *m0 = m;
+			/*
+			 * Dequeue this record (temporarily) from the receive
+			 * list since we're about to drop the socket's lock
+			 * where a new record may arrive and be appended to
+			 * the list.  Upon MAC policy failure, the record
+			 * will be freed.  Otherwise, we'll add it back to
+			 * the head of the list.  We cannot rely on SB_LOCK
+			 * because append operation uses the socket's lock.
+			 */
+			do {
+				m->m_nextpkt = NULL;
+				sbfree(&so->so_rcv, m);
+				m = m->m_next;
+			} while (m != NULL);
+			m = m0;
+			so->so_rcv.sb_mb = nextrecord;
+			SB_EMPTY_FIXUP(&so->so_rcv);
+			SBLASTRECORDCHK(&so->so_rcv, "soreceive 1a");
+			SBLASTMBUFCHK(&so->so_rcv, "soreceive 1a");
+			socket_unlock(so, 0);
+			if (mac_socket_check_received(proc_ucred(p), so,
+			    mtod(m, struct sockaddr *)) != 0) {
+				/*
+				 * MAC policy failure; free this record and
+				 * process the next record (or block until
+				 * one is available).  We have adjusted sb_cc
+				 * and sb_mbcnt above so there is no need to
+				 * call sbfree() again.
+				 */
+				do {
+					m = m_free(m);
+				} while (m != NULL);
+				/*
+				 * Clear SB_LOCK but don't unlock the socket.
+				 * Process the next record or wait for one.
+				 */
+				socket_lock(so, 0);
+				sbunlock(&so->so_rcv, 1);
+				goto restart;
+			}
+			socket_lock(so, 0);
+			/*
+			 * Re-adjust the socket receive list and re-enqueue
+			 * the record in front of any packets which may have
+			 * been appended while we dropped the lock.
+			 */
+			for (m = m0; m->m_next != NULL; m = m->m_next)
+				sballoc(&so->so_rcv, m);
+			sballoc(&so->so_rcv, m);
+			if (so->so_rcv.sb_mb == NULL) {
+				so->so_rcv.sb_lastrecord = m0;
+				so->so_rcv.sb_mbtail = m;
+			}
+			m = m0;
+			nextrecord = m->m_nextpkt = so->so_rcv.sb_mb;
+			so->so_rcv.sb_mb = m;
+			SBLASTRECORDCHK(&so->so_rcv, "soreceive 1b");
+			SBLASTMBUFCHK(&so->so_rcv, "soreceive 1b");
+		}
+#endif /* CONFIG_MACF_SOCKET_SUBSET */
 		orig_resid = 0;
 		if (psa) {
 			*psa = dup_sockaddr(mtod(m, struct sockaddr *),
-					    mp0 == 0);
+			    mp0 == 0);
 			if ((*psa == 0) && (flags & MSG_NEEDSA)) {
 				error = EWOULDBLOCK;
 				goto release;
@@ -1691,56 +2217,152 @@ dontblock:
 		} else {
 			sbfree(&so->so_rcv, m);
 			if (m->m_next == 0 && so->so_rcv.sb_cc != 0)
-				panic("soreceive: about to create invalid socketbuf");
+				panic("soreceive: about to create invalid "
+				    "socketbuf");
 			MFREE(m, so->so_rcv.sb_mb);
 			m = so->so_rcv.sb_mb;
+			if (m != NULL) {
+				m->m_nextpkt = nextrecord;
+			} else {
+				so->so_rcv.sb_mb = nextrecord;
+				SB_EMPTY_FIXUP(&so->so_rcv);
+			}
 		}
 	}
-	while (m && m->m_type == MT_CONTROL && error == 0) {
-		if (flags & MSG_PEEK) {
-			if (controlp)
-				*controlp = m_copy(m, 0, m->m_len);
-			m = m->m_next;
-		} else {
-			sbfree(&so->so_rcv, m);
-			if (controlp) {
-				if (pr->pr_domain->dom_externalize &&
-				    mtod(m, struct cmsghdr *)->cmsg_type ==
-				    SCM_RIGHTS) {
-				   socket_unlock(so, 0); /* release socket lock: see 3903171 */
-				   error = (*pr->pr_domain->dom_externalize)(m);
-				   socket_lock(so, 0);
+
+	/*
+	 * Process one or more MT_CONTROL mbufs present before any data mbufs
+	 * in the first mbuf chain on the socket buffer.  If MSG_PEEK, we
+	 * just copy the data; if !MSG_PEEK, we call into the protocol to
+	 * perform externalization.
+	 */
+	if (m != NULL && m->m_type == MT_CONTROL) {
+		struct mbuf *cm = NULL, *cmn;
+		struct mbuf **cme = &cm;
+		struct sockbuf *sb_rcv = &so->so_rcv;
+
+		/*
+		 * Externalizing the control messages would require us to
+		 * drop the socket's lock below.  Once we re-acquire the
+		 * lock, the mbuf chain might change.  In order to preserve
+		 * consistency, we unlink all control messages from the
+		 * first mbuf chain in one shot and link them separately
+		 * onto a different chain.
+		 */
+		do {
+			if (flags & MSG_PEEK) {
+				if (controlp != NULL) {
+					*controlp = m_copy(m, 0, m->m_len);
+					controlp = &(*controlp)->m_next;
 				}
-				*controlp = m;
-				if (m->m_next == 0 && so->so_rcv.sb_cc != 0)
-					panic("soreceive: so->so_rcv.sb_mb->m_next == 0 && so->so_rcv.sb_cc != 0");
-				so->so_rcv.sb_mb = m->m_next;
-				m->m_next = 0;
-				m = so->so_rcv.sb_mb;
+				m = m->m_next;
 			} else {
-				MFREE(m, so->so_rcv.sb_mb);
-				m = so->so_rcv.sb_mb;
+				m->m_nextpkt = NULL;
+				sbfree(sb_rcv, m);
+				sb_rcv->sb_mb = m->m_next;
+				m->m_next = NULL;
+				*cme = m;
+				cme = &(*cme)->m_next;
+				m = sb_rcv->sb_mb;
+			}
+		} while (m != NULL && m->m_type == MT_CONTROL);
+
+		if (!(flags & MSG_PEEK)) {
+			if (sb_rcv->sb_mb != NULL) {
+				sb_rcv->sb_mb->m_nextpkt = nextrecord;
+			} else {
+				sb_rcv->sb_mb = nextrecord;
+				SB_EMPTY_FIXUP(sb_rcv);
 			}
+			if (nextrecord == NULL)
+				sb_rcv->sb_lastrecord = m;
 		}
-		if (controlp) {
-			orig_resid = 0;
-			controlp = &(*controlp)->m_next;
+
+		SBLASTRECORDCHK(&so->so_rcv, "soreceive ctl");
+		SBLASTMBUFCHK(&so->so_rcv, "soreceive ctl");
+
+		while (cm != NULL) {
+			int cmsg_type;
+
+			cmn = cm->m_next;
+			cm->m_next = NULL;
+			cmsg_type = mtod(cm, struct cmsghdr *)->cmsg_type;
+
+			/*
+			 * Call the protocol to externalize SCM_RIGHTS message
+			 * and return the modified message to the caller upon
+			 * success.  Otherwise, all other control messages are
+			 * returned unmodified to the caller.  Note that we
+			 * only get into this loop if MSG_PEEK is not set.
+			 */
+			if (pr->pr_domain->dom_externalize != NULL &&
+			    cmsg_type == SCM_RIGHTS) {
+				/*
+				 * Release socket lock: see 3903171.  This
+				 * would also allow more records to be appended
+				 * to the socket buffer.  We still have SB_LOCK
+				 * set on it, so we can be sure that the head
+				 * of the mbuf chain won't change.
+				 */
+				socket_unlock(so, 0);
+				error = (*pr->pr_domain->dom_externalize)(cm);
+				socket_lock(so, 0);
+			} else {
+				error = 0;
+			}
+
+			if (controlp != NULL && error == 0) {
+				*controlp = cm;
+				controlp = &(*controlp)->m_next;
+				orig_resid = 0;
+			} else {
+				(void) m_free(cm);
+			}
+			cm = cmn;
 		}
+		orig_resid = 0;
+		if (sb_rcv->sb_mb != NULL)
+			nextrecord = sb_rcv->sb_mb->m_nextpkt;
+		else
+			nextrecord = NULL;
 	}
-	if (m) {
-		if ((flags & MSG_PEEK) == 0)
-			m->m_nextpkt = nextrecord;
+
+	if (m != NULL) {
+		if (!(flags & MSG_PEEK)) {
+			/*
+			 * We get here because m points to an mbuf following
+			 * any MT_SONAME or MT_CONTROL mbufs which have been
+			 * processed above.  In any case, m should be pointing
+			 * to the head of the mbuf chain, and the nextrecord
+			 * should be either NULL or equal to m->m_nextpkt.
+			 * See comments above about SB_LOCK.
+			 */
+			if (m != so->so_rcv.sb_mb || m->m_nextpkt != nextrecord)
+				panic("soreceive: post-control !sync so=%p "
+				    "m=%p nextrecord=%p\n", so, m, nextrecord);
+
+			if (nextrecord == NULL)
+				so->so_rcv.sb_lastrecord = m;
+		}
 		type = m->m_type;
 		if (type == MT_OOBDATA)
 			flags |= MSG_OOB;
+	} else {
+		if (!(flags & MSG_PEEK)) {
+			so->so_rcv.sb_mb = nextrecord;
+			SB_EMPTY_FIXUP(&so->so_rcv);
+		}
 	}
+	SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
+	SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
+
 	moff = 0;
 	offset = 0;
 
 	if (!(flags & MSG_PEEK) && uio_resid(uio) > sorecvmincopy)
-	        can_delay = 1;
+		can_delay = 1;
 	else
-	        can_delay = 0;
+		can_delay = 0;
 
 	need_event = 0;
 
@@ -1748,28 +2370,18 @@ dontblock:
 		if (m->m_type == MT_OOBDATA) {
 			if (type != MT_OOBDATA)
 				break;
-		} else if (type == MT_OOBDATA)
+		} else if (type == MT_OOBDATA) {
 			break;
-#ifndef __APPLE__
-/*
- * This assertion needs rework.  The trouble is Appletalk is uses many
- * mbuf types (NOT listed in mbuf.h!) which will trigger this panic.
- * For now just remove the assertion...  CSM 9/98
- */
-		else
-		    KASSERT(m->m_type == MT_DATA || m->m_type == MT_HEADER,
-			("receive 3"));
-#else
+		}
 		/*
-		 * Make sure to allways set MSG_OOB event when getting 
+		 * Make sure to allways set MSG_OOB event when getting
 		 * out of band data inline.
 		 */
 		if ((so->so_options & SO_WANTOOBFLAG) != 0 &&
-			(so->so_options & SO_OOBINLINE) != 0 && 
-			(so->so_state & SS_RCVATMARK) != 0) {
+		    (so->so_options & SO_OOBINLINE) != 0 &&
+		    (so->so_state & SS_RCVATMARK) != 0) {
 			flags |= MSG_OOB;
 		}
-#endif
 		so->so_state &= ~SS_RCVATMARK;
 		// LP64todo - fix this!
 		len = uio_resid(uio) - delayed_copy_len;
@@ -1786,47 +2398,55 @@ dontblock:
 		 * block interrupts again.
 		 */
 		if (mp == 0) {
+			SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
+			SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
 			if (can_delay && len == m->m_len) {
-			        /*
+				/*
 				 * only delay the copy if we're consuming the
 				 * mbuf and we're NOT in MSG_PEEK mode
 				 * and we have enough data to make it worthwile
-				 * to drop and retake the funnel... can_delay
-				 * reflects the state of the 2 latter constraints
-				 * moff should always be zero in these cases
+				 * to drop and retake the lock... can_delay
+				 * reflects the state of the 2 latter
+				 * constraints moff should always be zero
+				 * in these cases
 				 */
-			        delayed_copy_len += len;
+				delayed_copy_len += len;
 			} else {
-
-  			        if (delayed_copy_len) {
-				        error = sodelayed_copy(so, uio, &free_list, &delayed_copy_len);
+				if (delayed_copy_len) {
+					error = sodelayed_copy(so, uio,
+					    &free_list, &delayed_copy_len);
 
 					if (error) {
 						goto release;
 					}
+					/*
+					 * can only get here if MSG_PEEK is not
+					 * set therefore, m should point at the
+					 * head of the rcv queue; if it doesn't,
+					 * it means something drastically
+					 * changed while we were out from behind
+					 * the lock in sodelayed_copy. perhaps
+					 * a RST on the stream. in any event,
+					 * the stream has been interrupted. it's
+					 * probably best just to return whatever
+					 * data we've moved and let the caller
+					 * sort it out...
+					 */
 					if (m != so->so_rcv.sb_mb) {
-					        /*
-						 * can only get here if MSG_PEEK is not set
-						 * therefore, m should point at the head of the rcv queue...
-						 * if it doesn't, it means something drastically changed
-						 * while we were out from behind the funnel in sodelayed_copy...
-						 * perhaps a RST on the stream... in any event, the stream has
-						 * been interrupted... it's probably best just to return 
-						 * whatever data we've moved and let the caller sort it out...
-						 */
-					        break;
+						break;
 					}
 				}
 				socket_unlock(so, 0);
-				error = uiomove(mtod(m, caddr_t) + moff, (int)len, uio);
+				error = uiomove(mtod(m, caddr_t) + moff,
+				    (int)len, uio);
 				socket_lock(so, 0);
 
 				if (error)
-				        goto release;
+					goto release;
 			}
-		} else
+		} else {
 			uio_setresid(uio, (uio_resid(uio) - len));
-
+		}
 		if (len == m->m_len - moff) {
 			if (m->m_flags & M_EOR)
 				flags |= MSG_EOR;
@@ -1845,20 +2465,28 @@ dontblock:
 					*mp = (struct mbuf *)0;
 				} else {
 					if (free_list == NULL)
-					    free_list = m;
-				        else
-                                            ml->m_next = m;
-                                        ml = m;
+						free_list = m;
+					else
+						ml->m_next = m;
+					ml = m;
 					so->so_rcv.sb_mb = m = m->m_next;
-                                        ml->m_next = 0;
+					ml->m_next = 0;
 				}
-				if (m)
+				if (m != NULL) {
 					m->m_nextpkt = nextrecord;
+					if (nextrecord == NULL)
+						so->so_rcv.sb_lastrecord = m;
+				} else {
+					so->so_rcv.sb_mb = nextrecord;
+					SB_EMPTY_FIXUP(&so->so_rcv);
+				}
+				SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
+				SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
 			}
 		} else {
-			if (flags & MSG_PEEK)
+			if (flags & MSG_PEEK) {
 				moff += len;
-			else {
+			} else {
 				if (mp)
 					*mp = m_copym(m, 0, len, M_WAIT);
 				m->m_data += len;
@@ -1870,13 +2498,14 @@ dontblock:
 			if ((flags & MSG_PEEK) == 0) {
 				so->so_oobmark -= len;
 				if (so->so_oobmark == 0) {
-				    so->so_state |= SS_RCVATMARK;
-				    /*
-				     * delay posting the actual event until after
-				     * any delayed copy processing has finished
-				     */
-				    need_event = 1;
-				    break;
+					so->so_state |= SS_RCVATMARK;
+					/*
+					 * delay posting the actual event until
+					 * after any delayed copy processing
+					 * has finished
+					 */
+					need_event = 1;
+					break;
 				}
 			} else {
 				offset += len;
@@ -1884,45 +2513,68 @@ dontblock:
 					break;
 			}
 		}
-		if (flags & MSG_EOR) 
+		if (flags & MSG_EOR)
 			break;
 		/*
-		 * If the MSG_WAITALL or MSG_WAITSTREAM flag is set (for non-atomic socket),
-		 * we must not quit until "uio->uio_resid == 0" or an error
-		 * termination.  If a signal/timeout occurs, return
-		 * with a short count but without error.
-		 * Keep sockbuf locked against other readers.
+		 * If the MSG_WAITALL or MSG_WAITSTREAM flag is set
+		 * (for non-atomic socket), we must not quit until
+		 * "uio->uio_resid == 0" or an error termination.
+		 * If a signal/timeout occurs, return with a short
+		 * count but without error.  Keep sockbuf locked
+		 * against other readers.
 		 */
-		while (flags & (MSG_WAITALL|MSG_WAITSTREAM) && m == 0 && (uio_resid(uio) - delayed_copy_len) > 0 &&
+		while (flags & (MSG_WAITALL|MSG_WAITSTREAM) && m == 0 &&
+		    (uio_resid(uio) - delayed_copy_len) > 0 &&
 		    !sosendallatonce(so) && !nextrecord) {
 			if (so->so_error || so->so_state & SS_CANTRCVMORE)
-			        goto release;
+				goto release;
 
-		        if (pr->pr_flags & PR_WANTRCVD && so->so_pcb && (((struct inpcb *)so->so_pcb)->inp_state != INPCB_STATE_DEAD))
-			        (*pr->pr_usrreqs->pru_rcvd)(so, flags);
-			if (sbwait(&so->so_rcv)) {
-			        error = 0;
+			/*
+			 * Depending on the protocol (e.g. TCP), the following
+			 * might cause the socket lock to be dropped and later
+			 * be reacquired, and more data could have arrived and
+			 * have been appended to the receive socket buffer by
+			 * the time it returns.  Therefore, we only sleep in
+			 * sbwait() below if and only if the socket buffer is
+			 * empty, in order to avoid a false sleep.
+			 */
+			if (pr->pr_flags & PR_WANTRCVD && so->so_pcb &&
+			    (((struct inpcb *)so->so_pcb)->inp_state !=
+			    INPCB_STATE_DEAD))
+				(*pr->pr_usrreqs->pru_rcvd)(so, flags);
+
+			SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
+			SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
+
+			if (so->so_rcv.sb_mb == NULL && sbwait(&so->so_rcv)) {
+				error = 0;
 				goto release;
 			}
 			/*
-			 * have to wait until after we get back from the sbwait to do the copy because
-			 * we will drop the funnel if we have enough data that has been delayed... by dropping
-			 * the funnel we open up a window allowing the netisr thread to process the incoming packets
-			 * and to change the state of this socket... we're issuing the sbwait because
-			 * the socket is empty and we're expecting the netisr thread to wake us up when more
-			 * packets arrive... if we allow that processing to happen and then sbwait, we
-			 * could stall forever with packets sitting in the socket if no further packets
-			 * arrive from the remote side.
+			 * have to wait until after we get back from the sbwait
+			 * to do the copy because we will drop the lock if we
+			 * have enough data that has been delayed... by dropping
+			 * the lock we open up a window allowing the netisr
+			 * thread to process the incoming packets and to change
+			 * the state of this socket... we're issuing the sbwait
+			 * because the socket is empty and we're expecting the
+			 * netisr thread to wake us up when more packets arrive;
+			 * if we allow that processing to happen and then sbwait
+			 * we could stall forever with packets sitting in the
+			 * socket if no further packets arrive from the remote
+			 * side.
 			 *
-			 * we want to copy before we've collected all the data to satisfy this request to 
-			 * allow the copy to overlap the incoming packet processing on an MP system
+			 * we want to copy before we've collected all the data
+			 * to satisfy this request to allow the copy to overlap
+			 * the incoming packet processing on an MP system
 			 */
-			if (delayed_copy_len > sorecvmincopy && (delayed_copy_len > (so->so_rcv.sb_hiwat / 2))) {
-
-			        error = sodelayed_copy(so, uio, &free_list, &delayed_copy_len);
+			if (delayed_copy_len > sorecvmincopy &&
+			    (delayed_copy_len > (so->so_rcv.sb_hiwat / 2))) {
+				error = sodelayed_copy(so, uio,
+				    &free_list, &delayed_copy_len);
 
 				if (error)
-				        goto release;
+					goto release;
 			}
 			m = so->so_rcv.sb_mb;
 			if (m) {
@@ -1932,14 +2584,15 @@ dontblock:
 	}
 #ifdef MORE_LOCKING_DEBUG
 	if (so->so_usecount <= 1)
-		panic("soreceive: after big while so=%x ref=%d on socket\n", so, so->so_usecount);
+		panic("soreceive: after big while so=%p ref=%d on socket\n",
+		    so, so->so_usecount);
 #endif
 
 	if (m && pr->pr_flags & PR_ATOMIC) {
 #ifdef __APPLE__
-		if (so->so_options & SO_DONTTRUNC)
+		if (so->so_options & SO_DONTTRUNC) {
 			flags |= MSG_RCVMORE;
-		else {
+		} else {
 #endif
 			flags |= MSG_TRUNC;
 			if ((flags & MSG_PEEK) == 0)
@@ -1948,28 +2601,51 @@ dontblock:
 		}
 #endif
 	}
+
+	/*
+	 * pru_rcvd below (for TCP) may cause more data to be received
+	 * if the socket lock is dropped prior to sending the ACK; some
+	 * legacy OpenTransport applications don't handle this well
+	 * (if it receives less data than requested while MSG_HAVEMORE
+	 * is set), and so we set the flag now based on what we know
+	 * prior to calling pru_rcvd.
+	 */
+	if ((so->so_options & SO_WANTMORE) && so->so_rcv.sb_cc > 0)
+		flags |= MSG_HAVEMORE;
+
 	if ((flags & MSG_PEEK) == 0) {
-		if (m == 0)
+		if (m == 0) {
 			so->so_rcv.sb_mb = nextrecord;
+			/*
+			 * First part is an inline SB_EMPTY_FIXUP().  Second
+			 * part makes sure sb_lastrecord is up-to-date if
+			 * there is still data in the socket buffer.
+			 */
+			if (so->so_rcv.sb_mb == NULL) {
+				so->so_rcv.sb_mbtail = NULL;
+				so->so_rcv.sb_lastrecord = NULL;
+			} else if (nextrecord->m_nextpkt == NULL) {
+				so->so_rcv.sb_lastrecord = nextrecord;
+			}
+		}
+		SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
+		SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
 		if (pr->pr_flags & PR_WANTRCVD && so->so_pcb)
 			(*pr->pr_usrreqs->pru_rcvd)(so, flags);
 	}
 #ifdef __APPLE__
-	if ((so->so_options & SO_WANTMORE) && so->so_rcv.sb_cc > 0)
-		flags |= MSG_HAVEMORE;
-
 	if (delayed_copy_len) {
 		error = sodelayed_copy(so, uio, &free_list, &delayed_copy_len);
 
 		if (error)
-		        goto release;
+			goto release;
 	}
 	if (free_list) {
-	        m_freem_list((struct mbuf *)free_list);
+		m_freem_list((struct mbuf *)free_list);
 		free_list = (struct mbuf *)0;
 	}
 	if (need_event)
-	        postevent(so, 0, EV_OOB);
+		postevent(so, 0, EV_OOB);
 #endif
 	if (orig_resid == uio_resid(uio) && orig_resid &&
 	    (flags & MSG_EOR) == 0 && (so->so_state & SS_CANTRCVMORE) == 0) {
@@ -1982,97 +2658,144 @@ dontblock:
 release:
 #ifdef MORE_LOCKING_DEBUG
 	if (so->so_usecount <= 1)
-		panic("soreceive: release so=%x ref=%d on socket\n", so, so->so_usecount);
+		panic("soreceive: release so=%p ref=%d on socket\n",
+		    so, so->so_usecount);
 #endif
 	if (delayed_copy_len) {
-	        error = sodelayed_copy(so, uio, &free_list, &delayed_copy_len);
+		error = sodelayed_copy(so, uio, &free_list, &delayed_copy_len);
 	}
 	if (free_list) {
-	        m_freem_list((struct mbuf *)free_list);
+		m_freem_list((struct mbuf *)free_list);
 	}
 	sbunlock(&so->so_rcv, 0);	/* will unlock socket */
 
-		// LP64todo - fix this!
-	KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END,
-		     so,
-		     uio_resid(uio),
-		     so->so_rcv.sb_cc,
-		     0,
-		     error);
+	// LP64todo - fix this!
+	KERNEL_DEBUG(DBG_FNC_SORECEIVE | DBG_FUNC_END, so, uio_resid(uio),
+	    so->so_rcv.sb_cc, 0, error);
 
 	return (error);
 }
 
-
-static int sodelayed_copy(struct socket *so, struct uio *uio, struct mbuf **free_list, int *resid)
+/*
+ * Returns:	0			Success
+ *	uiomove:EFAULT
+ */
+static int
+sodelayed_copy(struct socket *so, struct uio *uio, struct mbuf **free_list,
+    int *resid)
 {
-        int         error  = 0;
+	int error = 0;
 	struct mbuf *m;
 
 	m = *free_list;
 
 	socket_unlock(so, 0);
 
-        while (m && error == 0) {
+	while (m && error == 0) {
+
+		error = uiomove(mtod(m, caddr_t), (int)m->m_len, uio);
+
+		m = m->m_next;
+	}
+	m_freem_list(*free_list);
+
+	*free_list = (struct mbuf *)NULL;
+	*resid = 0;
+
+	socket_lock(so, 0);
 
-	        error = uiomove(mtod(m, caddr_t), (int)m->m_len, uio);
+	return (error);
+}
+
+
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		ENOTCONN
+ *	<pru_shutdown>:EINVAL
+ *	<pru_shutdown>:EADDRNOTAVAIL[TCP]
+ *	<pru_shutdown>:ENOBUFS[TCP]
+ *	<pru_shutdown>:EMSGSIZE[TCP]
+ *	<pru_shutdown>:EHOSTUNREACH[TCP]
+ *	<pru_shutdown>:ENETUNREACH[TCP]
+ *	<pru_shutdown>:ENETDOWN[TCP]
+ *	<pru_shutdown>:ENOMEM[TCP]
+ *	<pru_shutdown>:EACCES[TCP]
+ *	<pru_shutdown>:EMSGSIZE[TCP]
+ *	<pru_shutdown>:ENOBUFS[TCP]
+ *	<pru_shutdown>:???[TCP]		[ignorable: mostly IPSEC/firewall/DLIL]
+ *	<pru_shutdown>:???		[other protocol families]
+ */
+int
+soshutdown(struct socket *so, int how)
+{
+	int error;
 
-		m = m->m_next;
+	switch (how) {
+	case SHUT_RD:
+	case SHUT_WR:
+	case SHUT_RDWR:
+		socket_lock(so, 1);
+		if ((so->so_state &
+		    (SS_ISCONNECTED|SS_ISCONNECTING|SS_ISDISCONNECTING)) == 0) {
+			error = ENOTCONN;
+		} else {
+			error = soshutdownlock(so, how);
+		}
+		socket_unlock(so, 1);
+		break;
+	default:
+		error = EINVAL;
+		break;
 	}
-	m_freem_list(*free_list);
-
-	*free_list = (struct mbuf *)NULL;
-	*resid = 0;
-
-	socket_lock(so, 0);
 
 	return (error);
 }
 
-
 int
-soshutdown(so, how)
-	register struct socket *so;
-	int how;
+soshutdownlock(struct socket *so, int how)
 {
-	register struct protosw *pr = so->so_proto;
-	int ret;
+	struct protosw *pr = so->so_proto;
+	int error = 0;
 
-	socket_lock(so, 1);
-	
 	sflt_notify(so, sock_evt_shutdown, &how);
 
 	if (how != SHUT_WR) {
+		if ((so->so_state & SS_CANTRCVMORE) != 0) {
+			/* read already shut down */
+			error = ENOTCONN;
+			goto done;
+		}
 		sorflush(so);
 		postevent(so, 0, EV_RCLOSED);
 	}
 	if (how != SHUT_RD) {
-	    ret = ((*pr->pr_usrreqs->pru_shutdown)(so));
-	    postevent(so, 0, EV_WCLOSED);
-	    KERNEL_DEBUG(DBG_FNC_SOSHUTDOWN | DBG_FUNC_END, 0,0,0,0,0);
-		socket_unlock(so, 1);
-	    return(ret);
+		if ((so->so_state & SS_CANTSENDMORE) != 0) {
+			/* write already shut down */
+			error = ENOTCONN;
+			goto done;
+		}
+		error = (*pr->pr_usrreqs->pru_shutdown)(so);
+		postevent(so, 0, EV_WCLOSED);
 	}
-
-	KERNEL_DEBUG(DBG_FNC_SOSHUTDOWN | DBG_FUNC_END, 0,0,0,0,0);
-	socket_unlock(so, 1);
-	return (0);
+done:
+	KERNEL_DEBUG(DBG_FNC_SOSHUTDOWN | DBG_FUNC_END, 0, 0, 0, 0, 0);
+	return (error);
 }
 
 void
-sorflush(so)
-	register struct socket *so;
+sorflush(struct socket *so)
 {
 	register struct sockbuf *sb = &so->so_rcv;
 	register struct protosw *pr = so->so_proto;
 	struct sockbuf asb;
 
 #ifdef MORE_LOCKING_DEBUG
-	lck_mtx_t * mutex_held;
+	lck_mtx_t *mutex_held;
 
-	if (so->so_proto->pr_getlock != NULL) 
+	if (so->so_proto->pr_getlock != NULL)
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-	else 
+	else
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 #endif
@@ -2093,8 +2816,24 @@ sorflush(so)
 		sb->sb_sel.si_note = asb.sb_sel.si_note;
 		sb->sb_flags = SB_KNOTE;
 	}
-	if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose)
+	if (asb.sb_flags & SB_DROP)
+		sb->sb_flags |= SB_DROP;
+	if (asb.sb_flags & SB_UNIX)
+		sb->sb_flags |= SB_UNIX;
+	if ((pr->pr_flags & PR_RIGHTS) && pr->pr_domain->dom_dispose) {
+		boolean_t unp = (pr->pr_domain->dom_dispose == unp_dispose);
+		/*
+		 * Currently AF_UNIX domain uses a global domain mutex;
+		 * unp_dispose() may end up calling soclose() on another
+		 * AF_UNIX socket and therefore the lock must not be held
+		 * across the call.
+		 */
+		if (unp)
+			socket_unlock(so, 0);
 		(*pr->pr_domain->dom_dispose)(asb.sb_mb);
+		if (unp)
+			socket_lock(so, 0);
+	}
 	sbrelease(&asb);
 }
 
@@ -2104,13 +2843,13 @@ sorflush(so)
  * to be some kind of integer, but not a specific size.
  * In addition to their use here, these functions are also called by the
  * protocol-level pr_ctloutput() routines.
+ *
+ * Returns:	0			Success
+ *		EINVAL
+ *	copyin:EFAULT
  */
 int
-sooptcopyin(sopt, buf, len, minlen)
-	struct	sockopt *sopt;
-	void	*buf;
-	size_t	len;
-	size_t	minlen;
+sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen)
 {
 	size_t	valsize;
 
@@ -2121,7 +2860,7 @@ sooptcopyin(sopt, buf, len, minlen)
 	 * is set to however much we actually retrieved.
 	 */
 	if ((valsize = sopt->sopt_valsize) < minlen)
-		return EINVAL;
+		return (EINVAL);
 	if (valsize > len)
 		sopt->sopt_valsize = valsize = len;
 
@@ -2129,59 +2868,133 @@ sooptcopyin(sopt, buf, len, minlen)
 		return (copyin(sopt->sopt_val, buf, valsize));
 
 	bcopy(CAST_DOWN(caddr_t, sopt->sopt_val), buf, valsize);
-	return 0;
+	return (0);
+}
+
+/*
+ * sooptcopyin_timeval
+ *   Copy in a timeval value into tv_p, and take into account whether the
+ *   the calling process is 64-bit or 32-bit.  Moved the sanity checking
+ *   code here so that we can verify the 64-bit tv_sec value before we lose
+ *   the top 32-bits assigning tv64.tv_sec to tv_p->tv_sec.
+ */
+static int
+sooptcopyin_timeval(struct sockopt *sopt, struct timeval * tv_p)
+{
+	int			error;
+		
+	if (proc_is64bit(sopt->sopt_p)) {
+		struct timeval64	tv64;
+
+		if (sopt->sopt_valsize < sizeof(tv64)) {
+			return (EINVAL);
+		}
+		sopt->sopt_valsize = sizeof(tv64);
+		error = copyin(sopt->sopt_val, &tv64, sizeof(tv64));
+		if (error != 0) {
+			return (error);
+		}
+		if (tv64.tv_sec < 0 || tv64.tv_sec > LONG_MAX 
+		    || tv64.tv_usec < 0 || tv64.tv_usec >= 1000000) {
+			return (EDOM);
+		}
+		tv_p->tv_sec = tv64.tv_sec;
+		tv_p->tv_usec = tv64.tv_usec;
+	} else {
+		if (sopt->sopt_valsize < sizeof(*tv_p)) {
+			return (EINVAL);
+		}
+		sopt->sopt_valsize = sizeof(*tv_p);
+		if (sopt->sopt_p != 0) {
+			error = copyin(sopt->sopt_val, tv_p, sizeof(*tv_p));
+			if (error != 0) {
+				return (error);
+			}
+		} else {
+			bcopy(CAST_DOWN(caddr_t, sopt->sopt_val), tv_p,
+			      sizeof(*tv_p));
+		}
+		if (tv_p->tv_sec < 0 || tv_p->tv_sec > LONG_MAX 
+		    || tv_p->tv_usec < 0 || tv_p->tv_usec >= 1000000) {
+			return (EDOM);
+		}
+	}
+	return (0);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		ENOPROTOOPT
+ *		ENOBUFS
+ *		EDOM
+ *	sooptcopyin:EINVAL
+ *	sooptcopyin:EFAULT
+ *	sooptcopyin_timeval:EINVAL
+ *	sooptcopyin_timeval:EFAULT
+ *	sooptcopyin_timeval:EDOM
+ *	<pr_ctloutput>:EOPNOTSUPP[AF_UNIX]
+ *	<pr_ctloutput>:???w
+ *	sflt_attach_private:???		[whatever a filter author chooses]
+ *	<sf_setoption>:???		[whatever a filter author chooses]
+ *
+ * Notes:	Other <pru_listen> returns depend on the protocol family; all
+ *		<sf_listen> returns depend on what the filter author causes
+ *		their filter to return.
+ */
 int
-sosetopt(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+sosetopt(struct socket *so, struct sockopt *sopt)
 {
 	int	error, optval;
 	struct	linger l;
 	struct	timeval tv;
-	short	val;
+	struct socket_filter_entry *filter;
+	int filtered = 0;
+#if CONFIG_MACF_SOCKET
+	struct mac extmac;
+#endif /* MAC_SOCKET */
 
 	socket_lock(so, 1);
+	if ((so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE))
+	    == (SS_CANTRCVMORE | SS_CANTSENDMORE)) {
+		/* the socket has been shutdown, no more sockopt's */
+		error = EINVAL;
+		goto bad;
+	}
 
 	if (sopt->sopt_dir != SOPT_SET) {
 		sopt->sopt_dir = SOPT_SET;
 	}
 
-	{
-		struct socket_filter_entry	*filter;
-		int						 	filtered = 0;
-		error = 0;
-		for (filter = so->so_filt; filter && (error == 0);
-			 filter = filter->sfe_next_onsocket) {
-			if (filter->sfe_filter->sf_filter.sf_setoption) {
-				if (filtered == 0) {
-					filtered = 1;
-					sflt_use(so);
-					socket_unlock(so, 0);
-				}
-				error = filter->sfe_filter->sf_filter.sf_setoption(
-							filter->sfe_cookie, so, sopt);
+	error = 0;
+	for (filter = so->so_filt; filter && (error == 0);
+	    filter = filter->sfe_next_onsocket) {
+		if (filter->sfe_filter->sf_filter.sf_setoption) {
+			if (filtered == 0) {
+				filtered = 1;
+				sflt_use(so);
+				socket_unlock(so, 0);
 			}
+			error = filter->sfe_filter->sf_filter.
+			    sf_setoption(filter->sfe_cookie, so, sopt);
 		}
-		
-		if (filtered != 0) {
-			socket_lock(so, 0);
-			sflt_unuse(so);
-			
-			if (error) {
-				if (error == EJUSTRETURN)
-					error = 0;
-				goto bad;
-			}
+	}
+
+	if (filtered != 0) {
+		socket_lock(so, 0);
+		sflt_unuse(so);
+
+		if (error) {
+			if (error == EJUSTRETURN)
+				error = 0;
+			goto bad;
 		}
 	}
 
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
 		if (so->so_proto && so->so_proto->pr_ctloutput) {
-			error = (*so->so_proto->pr_ctloutput)
-				  (so, sopt);
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
 			socket_unlock(so, 1);
 			return (error);
 		}
@@ -2190,11 +3003,12 @@ sosetopt(so, sopt)
 		switch (sopt->sopt_name) {
 		case SO_LINGER:
 		case SO_LINGER_SEC:
-			error = sooptcopyin(sopt, &l, sizeof l, sizeof l);
+			error = sooptcopyin(sopt, &l, sizeof (l), sizeof (l));
 			if (error)
 				goto bad;
 
-			so->so_linger = (sopt->sopt_name == SO_LINGER) ? l.l_linger : l.l_linger * hz;
+			so->so_linger = (sopt->sopt_name == SO_LINGER) ?
+			    l.l_linger : l.l_linger * hz;
 			if (l.l_onoff)
 				so->so_options |= SO_LINGER;
 			else
@@ -2215,8 +3029,8 @@ sosetopt(so, sopt)
 		case SO_WANTMORE:
 		case SO_WANTOOBFLAG:
 #endif
-			error = sooptcopyin(sopt, &optval, sizeof optval,
-					    sizeof optval);
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
 			if (error)
 				goto bad;
 			if (optval)
@@ -2229,8 +3043,8 @@ sosetopt(so, sopt)
 		case SO_RCVBUF:
 		case SO_SNDLOWAT:
 		case SO_RCVLOWAT:
-			error = sooptcopyin(sopt, &optval, sizeof optval,
-					    sizeof optval);
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
 			if (error)
 				goto bad;
 
@@ -2247,11 +3061,15 @@ sosetopt(so, sopt)
 			case SO_SNDBUF:
 			case SO_RCVBUF:
 				if (sbreserve(sopt->sopt_name == SO_SNDBUF ?
-					      &so->so_snd : &so->so_rcv,
-					      (u_long) optval) == 0) {
+				    &so->so_snd : &so->so_rcv,
+				    (u_long) optval) == 0) {
 					error = ENOBUFS;
 					goto bad;
 				}
+				if (sopt->sopt_name == SO_SNDBUF)
+					so->so_snd.sb_flags |= SB_USRSIZE;
+				else
+					so->so_rcv.sb_flags |= SB_USRSIZE;
 				break;
 
 			/*
@@ -2273,17 +3091,10 @@ sosetopt(so, sopt)
 
 		case SO_SNDTIMEO:
 		case SO_RCVTIMEO:
-			error = sooptcopyin(sopt, &tv, sizeof tv,
-					    sizeof tv);
+			error = sooptcopyin_timeval(sopt, &tv);
 			if (error)
 				goto bad;
 
-			if (tv.tv_sec < 0 || tv.tv_sec > LONG_MAX ||
-			    tv.tv_usec < 0 || tv.tv_usec >= 1000000) {
-				error = EDOM;
-				goto bad;
-			}
-			
 			switch (sopt->sopt_name) {
 			case SO_SNDTIMEO:
 				so->so_snd.sb_timeo = tv;
@@ -2298,37 +3109,90 @@ sosetopt(so, sopt)
 		{
 			struct so_nke nke;
 
-			error = sooptcopyin(sopt, &nke,
-								sizeof nke, sizeof nke);
+			error = sooptcopyin(sopt, &nke, sizeof (nke),
+			    sizeof (nke));
 			if (error)
-			  goto bad;
+				goto bad;
 
-			error = sflt_attach_private(so, NULL, nke.nke_handle, 1);
+			error = sflt_attach_private(so, NULL,
+			    nke.nke_handle, 1);
 			break;
 		}
 
 		case SO_NOSIGPIPE:
-                        error = sooptcopyin(sopt, &optval, sizeof optval,
-                                            sizeof optval);
-                        if (error)
-                                goto bad;
-                        if (optval)
-                                so->so_flags |= SOF_NOSIGPIPE;
-                        else
-                                so->so_flags &= ~SOF_NOSIGPIPE;
-			
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
+			if (error)
+				goto bad;
+			if (optval)
+				so->so_flags |= SOF_NOSIGPIPE;
+			else
+				so->so_flags &= ~SOF_NOSIGPIPE;
+
 			break;
 
 		case SO_NOADDRERR:
-                        error = sooptcopyin(sopt, &optval, sizeof optval,
-                                            sizeof optval);
-                        if (error)
-                                goto bad;
-                        if (optval)
-                                so->so_flags |= SOF_NOADDRAVAIL;
-                        else
-                                so->so_flags &= ~SOF_NOADDRAVAIL;
-			
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
+			if (error)
+				goto bad;
+			if (optval)
+				so->so_flags |= SOF_NOADDRAVAIL;
+			else
+				so->so_flags &= ~SOF_NOADDRAVAIL;
+
+			break;
+
+		case SO_REUSESHAREUID:
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
+			if (error)
+				goto bad;
+			if (optval)
+				so->so_flags |= SOF_REUSESHAREUID;
+			else
+				so->so_flags &= ~SOF_REUSESHAREUID;
+			break;
+#ifdef __APPLE_API_PRIVATE
+		case SO_NOTIFYCONFLICT:
+			if (kauth_cred_issuser(kauth_cred_get()) == 0) {
+				error = EPERM;
+				goto bad;
+			}
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
+			if (error)
+				goto bad;
+			if (optval)
+				so->so_flags |= SOF_NOTIFYCONFLICT;
+			else
+				so->so_flags &= ~SOF_NOTIFYCONFLICT;
+			break;
+#endif
+		case SO_RESTRICTIONS:
+			if (kauth_cred_issuser(kauth_cred_get()) == 0) {
+				error = EPERM;
+				goto bad;
+			}
+			error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval));
+			if (error)
+				goto bad;
+			so->so_restrictions = (optval & (SO_RESTRICT_DENYIN |
+			    SO_RESTRICT_DENYOUT | SO_RESTRICT_DENYSET));
+			break;
+
+		case SO_LABEL:
+#if CONFIG_MACF_SOCKET
+			if ((error = sooptcopyin(sopt, &extmac, sizeof (extmac),
+			    sizeof (extmac))) != 0)
+				goto bad;
+
+			error = mac_setsockopt_label(proc_ucred(sopt->sopt_p),
+			    so, &extmac);
+#else
+			error = EOPNOTSUPP;
+#endif /* MAC_SOCKET */
 			break;
 
 		default:
@@ -2336,8 +3200,7 @@ sosetopt(so, sopt)
 			break;
 		}
 		if (error == 0 && so->so_proto && so->so_proto->pr_ctloutput) {
-			(void) ((*so->so_proto->pr_ctloutput)
-				  (so, sopt));
+			(void) ((*so->so_proto->pr_ctloutput)(so, sopt));
 		}
 	}
 bad:
@@ -2345,12 +3208,9 @@ bad:
 	return (error);
 }
 
-/* Helper routine for getsockopt */
+/* Helper routines for getsockopt */
 int
-sooptcopyout(sopt, buf, len)
-	struct	sockopt *sopt;
-	void	*buf;
-	size_t	len;
+sooptcopyout(struct sockopt *sopt, void *buf, size_t len)
 {
 	int	error;
 	size_t	valsize;
@@ -2374,58 +3234,93 @@ sooptcopyout(sopt, buf, len)
 		else
 			bcopy(buf, CAST_DOWN(caddr_t, sopt->sopt_val), valsize);
 	}
-	return error;
+	return (error);
+}
+
+static int
+sooptcopyout_timeval(struct sockopt *sopt, const struct timeval * tv_p)
+{
+	int			error;
+	size_t			len;
+	struct timeval64	tv64;
+	const void *		val;
+	size_t			valsize;
+		
+	error = 0;
+	if (proc_is64bit(sopt->sopt_p)) {
+		len = sizeof(struct timeval64);
+		tv64.tv_sec = tv_p->tv_sec;
+		tv64.tv_usec = tv_p->tv_usec;
+		val = &tv64;
+	} else {
+		len = sizeof(struct timeval);
+		val = tv_p;
+	}
+	valsize = min(len, sopt->sopt_valsize);
+	sopt->sopt_valsize = valsize;
+	if (sopt->sopt_val != USER_ADDR_NULL) {
+		if (sopt->sopt_p != 0)
+			error = copyout(val, sopt->sopt_val, valsize);
+		else
+			bcopy(val, CAST_DOWN(caddr_t, sopt->sopt_val), valsize);
+	}
+	return (error);
 }
 
+/*
+ * Return:	0			Success
+ *		ENOPROTOOPT
+ *	<pr_ctloutput>:EOPNOTSUPP[AF_UNIX]
+ *	<pr_ctloutput>:???
+ *	<sf_getoption>:???
+ */
 int
-sogetopt(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+sogetopt(struct socket *so, struct sockopt *sopt)
 {
 	int	error, optval;
 	struct	linger l;
 	struct	timeval tv;
+	struct	socket_filter_entry *filter;
+	int	filtered = 0;
+#if CONFIG_MACF_SOCKET
+	struct mac extmac;
+#endif /* MAC_SOCKET */
 
-        if (sopt->sopt_dir != SOPT_GET) {
-                sopt->sopt_dir = SOPT_GET;
-        }
+	if (sopt->sopt_dir != SOPT_GET) {
+		sopt->sopt_dir = SOPT_GET;
+	}
 
 	socket_lock(so, 1);
-	
-	{
-		struct socket_filter_entry	*filter;
-		int						 	filtered = 0;
-		error = 0;
-		for (filter = so->so_filt; filter && (error == 0);
-			 filter = filter->sfe_next_onsocket) {
-			if (filter->sfe_filter->sf_filter.sf_getoption) {
-				if (filtered == 0) {
-					filtered = 1;
-					sflt_use(so);
-					socket_unlock(so, 0);
-				}
-				error = filter->sfe_filter->sf_filter.sf_getoption(
-							filter->sfe_cookie, so, sopt);
+
+	error = 0;
+	for (filter = so->so_filt; filter && (error == 0);
+	    filter = filter->sfe_next_onsocket) {
+		if (filter->sfe_filter->sf_filter.sf_getoption) {
+			if (filtered == 0) {
+				filtered = 1;
+				sflt_use(so);
+				socket_unlock(so, 0);
 			}
+			error = filter->sfe_filter->sf_filter.
+			    sf_getoption(filter->sfe_cookie, so, sopt);
 		}
-		if (filtered != 0) {
-			socket_lock(so, 0);
-			sflt_unuse(so);
-			
-			if (error) {
-				if (error == EJUSTRETURN)
-					error = 0;
-				socket_unlock(so, 1);
-				return error;
-			}
+	}
+	if (filtered != 0) {
+		socket_lock(so, 0);
+		sflt_unuse(so);
+
+		if (error) {
+			if (error == EJUSTRETURN)
+				error = 0;
+			socket_unlock(so, 1);
+			return (error);
 		}
 	}
 
 	error = 0;
 	if (sopt->sopt_level != SOL_SOCKET) {
 		if (so->so_proto && so->so_proto->pr_ctloutput) {
-			error = (*so->so_proto->pr_ctloutput)
-				  (so, sopt);
+			error = (*so->so_proto->pr_ctloutput)(so, sopt);
 			socket_unlock(so, 1);
 			return (error);
 		} else {
@@ -2437,9 +3332,9 @@ sogetopt(so, sopt)
 		case SO_LINGER:
 		case SO_LINGER_SEC:
 			l.l_onoff = so->so_options & SO_LINGER;
-			l.l_linger = (sopt->sopt_name == SO_LINGER) ? so->so_linger : 
-				so->so_linger / hz;
-			error = sooptcopyout(sopt, &l, sizeof l);
+			l.l_linger = (sopt->sopt_name == SO_LINGER) ?
+			    so->so_linger : so->so_linger / hz;
+			error = sooptcopyout(sopt, &l, sizeof (l));
 			break;
 
 		case SO_USELOOPBACK:
@@ -2458,7 +3353,7 @@ sogetopt(so, sopt)
 #endif
 			optval = so->so_options & sopt->sopt_name;
 integer:
-			error = sooptcopyout(sopt, &optval, sizeof optval);
+			error = sooptcopyout(sopt, &optval, sizeof (optval));
 			break;
 
 		case SO_TYPE:
@@ -2467,27 +3362,27 @@ integer:
 
 #ifdef __APPLE__
 		case SO_NREAD:
-		{
-			int pkt_total;
-			struct mbuf *m1;
+			if (so->so_proto->pr_flags & PR_ATOMIC) {
+				int pkt_total;
+				struct mbuf *m1;
 
-			pkt_total = 0;
-			m1 = so->so_rcv.sb_mb;
-		  	if (so->so_proto->pr_flags & PR_ATOMIC)
-			{
+				pkt_total = 0;
+				m1 = so->so_rcv.sb_mb;
 				while (m1) {
-					if (m1->m_type == MT_DATA)
+					if (m1->m_type == MT_DATA || m1->m_type == MT_HEADER ||
+						m1->m_type == MT_OOBDATA)
 						pkt_total += m1->m_len;
 					m1 = m1->m_next;
 				}
 				optval = pkt_total;
-			} else
-				optval = so->so_rcv.sb_cc;
+			} else {
+				optval = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
+			}
 			goto integer;
-		}
+		
 		case SO_NWRITE:
 			optval = so->so_snd.sb_cc;
-			goto integer;		
+			goto integer;
 #endif
 		case SO_ERROR:
 			optval = so->so_error;
@@ -2513,10 +3408,10 @@ integer:
 		case SO_SNDTIMEO:
 		case SO_RCVTIMEO:
 			tv = (sopt->sopt_name == SO_SNDTIMEO ?
-				  so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
+			    so->so_snd.sb_timeo : so->so_rcv.sb_timeo);
 
-			error = sooptcopyout(sopt, &tv, sizeof tv);
-			break;			
+			error = sooptcopyout_timeval(sopt, &tv);
+			break;
 
 		case SO_NOSIGPIPE:
 			optval = (so->so_flags & SOF_NOSIGPIPE);
@@ -2526,6 +3421,48 @@ integer:
 			optval = (so->so_flags & SOF_NOADDRAVAIL);
 			goto integer;
 
+		case SO_REUSESHAREUID:
+			optval = (so->so_flags & SOF_REUSESHAREUID);
+			goto integer;
+
+#ifdef __APPLE_API_PRIVATE
+		case SO_NOTIFYCONFLICT:
+			optval = (so->so_flags & SOF_NOTIFYCONFLICT);
+			goto integer;
+#endif
+		case SO_RESTRICTIONS:
+			optval = so->so_restrictions & (SO_RESTRICT_DENYIN |
+			    SO_RESTRICT_DENYOUT | SO_RESTRICT_DENYSET);
+			goto integer;
+
+		case SO_LABEL:
+#if CONFIG_MACF_SOCKET
+			if ((error = sooptcopyin(sopt, &extmac, sizeof (extmac),
+			    sizeof (extmac))) != 0 ||
+			    (error = mac_socket_label_get(proc_ucred(
+			    sopt->sopt_p), so, &extmac)) != 0)
+				break;
+
+			error = sooptcopyout(sopt, &extmac, sizeof (extmac));
+#else
+			error = EOPNOTSUPP;
+#endif /* MAC_SOCKET */
+			break;
+
+		case SO_PEERLABEL:
+#if CONFIG_MACF_SOCKET
+			if ((error = sooptcopyin(sopt, &extmac, sizeof (extmac),
+			    sizeof (extmac))) != 0 ||
+			    (error = mac_socketpeer_label_get(proc_ucred(
+			    sopt->sopt_p), so, &extmac)) != 0)
+				break;
+
+			error = sooptcopyout(sopt, &extmac, sizeof (extmac));
+#else
+			error = EOPNOTSUPP;
+#endif /* MAC_SOCKET */
+			break;
+
 		default:
 			error = ENOPROTOOPT;
 			break;
@@ -2543,16 +3480,16 @@ soopt_getm(struct sockopt *sopt, struct mbuf **mp)
 	int sopt_size = sopt->sopt_valsize;
 
 	if (sopt_size > MAX_SOOPTGETM_SIZE)
-		return EMSGSIZE;
+		return (EMSGSIZE);
 
 	MGET(m, sopt->sopt_p ? M_WAIT : M_DONTWAIT, MT_DATA);
 	if (m == 0)
-		return ENOBUFS;
+		return (ENOBUFS);
 	if (sopt_size > MLEN) {
 		MCLGET(m, sopt->sopt_p ? M_WAIT : M_DONTWAIT);
 		if ((m->m_flags & M_EXT) == 0) {
 			m_free(m);
-			return ENOBUFS;
+			return (ENOBUFS);
 		}
 		m->m_len = min(MCLBYTES, sopt_size);
 	} else {
@@ -2566,13 +3503,13 @@ soopt_getm(struct sockopt *sopt, struct mbuf **mp)
 		MGET(m, sopt->sopt_p ? M_WAIT : M_DONTWAIT, MT_DATA);
 		if (m == 0) {
 			m_freem(*mp);
-			return ENOBUFS;
+			return (ENOBUFS);
 		}
 		if (sopt_size > MLEN) {
 			MCLGET(m, sopt->sopt_p ? M_WAIT : M_DONTWAIT);
 			if ((m->m_flags & M_EXT) == 0) {
 				m_freem(*mp);
-				return ENOBUFS;
+				return (ENOBUFS);
 			}
 			m->m_len = min(MCLBYTES, sopt_size);
 		} else {
@@ -2582,7 +3519,7 @@ soopt_getm(struct sockopt *sopt, struct mbuf **mp)
 		m_prev->m_next = m;
 		m_prev = m;
 	}
-	return 0;
+	return (0);
 }
 
 /* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */
@@ -2592,25 +3529,28 @@ soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
 	struct mbuf *m0 = m;
 
 	if (sopt->sopt_val == USER_ADDR_NULL)
-		return 0;
+		return (0);
 	while (m != NULL && sopt->sopt_valsize >= m->m_len) {
 		if (sopt->sopt_p != NULL) {
 			int error;
 
-			error = copyin(sopt->sopt_val, mtod(m, char *), m->m_len);
+			error = copyin(sopt->sopt_val, mtod(m, char *),
+			    m->m_len);
 			if (error != 0) {
 				m_freem(m0);
-				return(error);
+				return (error);
 			}
-		} else
-			bcopy(CAST_DOWN(caddr_t, sopt->sopt_val), mtod(m, char *), m->m_len);
+		} else {
+			bcopy(CAST_DOWN(caddr_t, sopt->sopt_val),
+			    mtod(m, char *), m->m_len);
+		}
 		sopt->sopt_valsize -= m->m_len;
-		sopt->sopt_val += m->m_len; 
+		sopt->sopt_val += m->m_len;
 		m = m->m_next;
 	}
 	if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */
 		panic("soopt_mcopyin");
-	return 0;
+	return (0);
 }
 
 /* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */
@@ -2621,42 +3561,43 @@ soopt_mcopyout(struct sockopt *sopt, struct mbuf *m)
 	size_t valsize = 0;
 
 	if (sopt->sopt_val == USER_ADDR_NULL)
-		return 0;
+		return (0);
 	while (m != NULL && sopt->sopt_valsize >= m->m_len) {
 		if (sopt->sopt_p != NULL) {
 			int error;
 
-			error = copyout(mtod(m, char *), sopt->sopt_val, m->m_len);
+			error = copyout(mtod(m, char *), sopt->sopt_val,
+			    m->m_len);
 			if (error != 0) {
 				m_freem(m0);
-				return(error);
+				return (error);
 			}
-		} else
-			bcopy(mtod(m, char *), CAST_DOWN(caddr_t, sopt->sopt_val), m->m_len);
-	       sopt->sopt_valsize -= m->m_len;
-	       sopt->sopt_val += m->m_len;
-	       valsize += m->m_len;
-	       m = m->m_next;
+		} else {
+			bcopy(mtod(m, char *),
+			    CAST_DOWN(caddr_t, sopt->sopt_val), m->m_len);
+		}
+		sopt->sopt_valsize -= m->m_len;
+		sopt->sopt_val += m->m_len;
+		valsize += m->m_len;
+		m = m->m_next;
 	}
 	if (m != NULL) {
 		/* enough soopt buffer should be given from user-land */
 		m_freem(m0);
-		return(EINVAL);
+		return (EINVAL);
 	}
 	sopt->sopt_valsize = valsize;
-	return 0;
+	return (0);
 }
 
 void
-sohasoutofband(so)
-	register struct socket *so;
+sohasoutofband(struct socket *so)
 {
-	struct proc *p;
 
 	if (so->so_pgid < 0)
 		gsignal(-so->so_pgid, SIGURG);
-	else if (so->so_pgid > 0 && (p = pfind(so->so_pgid)) != 0)
-		psignal(p, SIGURG);
+	else if (so->so_pgid > 0)
+		proc_signal(so->so_pgid, SIGURG);
 	selwakeup(&so->so_rcv.sb_sel);
 }
 
@@ -2682,13 +3623,19 @@ sopoll(struct socket *so, int events, __unused kauth_cred_t cred, void * wql)
 
 	if (revents == 0) {
 		if (events & (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) {
-			/* Darwin sets the flag first, BSD calls selrecord first */
+			/*
+			 * Darwin sets the flag first,
+			 * BSD calls selrecord first
+			 */
 			so->so_rcv.sb_flags |= SB_SEL;
 			selrecord(p, &so->so_rcv.sb_sel, wql);
 		}
 
 		if (events & (POLLOUT | POLLWRNORM)) {
-			/* Darwin sets the flag first, BSD calls selrecord first */
+			/*
+			 * Darwin sets the flag first,
+			 * BSD calls selrecord first
+			 */
 			so->so_snd.sb_flags |= SB_SEL;
 			selrecord(p, &so->so_snd.sb_sel, wql);
 		}
@@ -2698,15 +3645,22 @@ sopoll(struct socket *so, int events, __unused kauth_cred_t cred, void * wql)
 	return (revents);
 }
 
-int     soo_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p);
-
 int
-soo_kqfilter(__unused struct fileproc *fp, struct knote *kn, __unused struct proc *p)
+soo_kqfilter(__unused struct fileproc *fp, struct knote *kn,
+    __unused struct proc *p)
 {
 	struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data;
 	struct sockbuf *sb;
+
 	socket_lock(so, 1);
 
+#if CONFIG_MACF_SOCKET
+	if (mac_socket_check_kqfilter(proc_ucred(p), kn, so) != 0) {
+		socket_unlock(so, 1);
+		return (1);
+	}
+#endif /* MAC_SOCKET */
+
 	switch (kn->kn_filter) {
 	case EVFILT_READ:
 		if (so->so_options & SO_ACCEPTCONN)
@@ -2751,9 +3705,11 @@ filt_soread(struct knote *kn, long hint)
 	if ((hint & SO_FILT_HINT_LOCKED) == 0)
 		socket_lock(so, 1);
 
+	kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl;
+
 	if (so->so_oobmark) {
 		if (kn->kn_flags & EV_OOBAND) {
-			kn->kn_data = so->so_rcv.sb_cc - so->so_oobmark;
+			kn->kn_data -= so->so_oobmark;
 			if ((hint & SO_FILT_HINT_LOCKED) == 0)
 				socket_unlock(so, 1);
 			return (1);
@@ -2761,7 +3717,6 @@ filt_soread(struct knote *kn, long hint)
 		kn->kn_data = so->so_oobmark;
 		kn->kn_flags |= EV_OOBAND;
 	} else {
-		kn->kn_data = so->so_rcv.sb_cc;
 		if (so->so_state & SS_CANTRCVMORE) {
 			kn->kn_flags |= EV_EOF;
 			kn->kn_fflags = so->so_error;
@@ -2794,9 +3749,9 @@ filt_soread(struct knote *kn, long hint)
 	if ((hint & SO_FILT_HINT_LOCKED) == 0)
 		socket_unlock(so, 1);
 
-	return( kn->kn_flags & EV_OOBAND ||
-	        kn->kn_data >= ((kn->kn_sfflags & NOTE_LOWAT) ? 
-	                        kn->kn_sdata : so->so_rcv.sb_lowat));
+	return ((kn->kn_flags & EV_OOBAND) ||
+	    kn->kn_data >= ((kn->kn_sfflags & NOTE_LOWAT) ?
+	    kn->kn_sdata : so->so_rcv.sb_lowat));
 }
 
 static void
@@ -2805,7 +3760,7 @@ filt_sowdetach(struct knote *kn)
 	struct socket *so = (struct socket *)kn->kn_fp->f_fglob->fg_data;
 	socket_lock(so, 1);
 
-	if(so->so_snd.sb_flags & SB_KNOTE)
+	if (so->so_snd.sb_flags & SB_KNOTE)
 		if (KNOTE_DETACH(&so->so_snd.sb_sel.si_note, kn))
 			so->so_snd.sb_flags &= ~SB_KNOTE;
 	socket_unlock(so, 1);
@@ -2822,7 +3777,7 @@ filt_sowrite(struct knote *kn, long hint)
 
 	kn->kn_data = sbspace(&so->so_snd);
 	if (so->so_state & SS_CANTSENDMORE) {
-		kn->kn_flags |= EV_EOF; 
+		kn->kn_flags |= EV_EOF;
 		kn->kn_fflags = so->so_error;
 		if ((hint & SO_FILT_HINT_LOCKED) == 0)
 			socket_unlock(so, 1);
@@ -2864,9 +3819,7 @@ filt_solisten(struct knote *kn, long hint)
 
 
 int
-socket_lock(so, refcount)
-	struct socket *so;
-	int refcount;
+socket_lock(struct socket *so, int refcount)
 {
 	int error = 0, lr_saved;
 
@@ -2874,48 +3827,46 @@ socket_lock(so, refcount)
 
 	if (so->so_proto->pr_lock) {
 		error = (*so->so_proto->pr_lock)(so, refcount, lr_saved);
-	}
-	else {
+	} else {
 #ifdef MORE_LOCKING_DEBUG
-		lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
+		lck_mtx_assert(so->so_proto->pr_domain->dom_mtx,
+		    LCK_MTX_ASSERT_NOTOWNED);
 #endif
 		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
 		if (refcount)
 			so->so_usecount++;
-		so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+		so->lock_lr[so->next_lock_lr] = (u_int32_t)lr_saved;
 		so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	}
 
-	return(error);
-
+	return (error);
 }
 
 int
-socket_unlock(so, refcount)
-	struct socket *so;
-	int refcount;
+socket_unlock(struct socket *so, int refcount)
 {
 	int error = 0, lr_saved;
-	lck_mtx_t * mutex_held;
+	lck_mtx_t *mutex_held;
 
 	lr_saved = (unsigned int) __builtin_return_address(0);
 
 	if (so->so_proto == NULL)
-		panic("socket_unlock null so_proto so=%x\n", so);
+		panic("socket_unlock null so_proto so=%p\n", so);
 
-	if (so && so->so_proto->pr_unlock) 
+	if (so && so->so_proto->pr_unlock) {
 		error = (*so->so_proto->pr_unlock)(so, refcount, lr_saved);
-	else {
+	} else {
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 #ifdef MORE_LOCKING_DEBUG
 		lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 #endif
-		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+		so->unlock_lr[so->next_unlock_lr] = (u_int32_t)lr_saved;
 		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 
 		if (refcount) {
 			if (so->so_usecount <= 0)
-				panic("socket_unlock: bad refcount so=%x value=%d\n", so, so->so_usecount);
+				panic("socket_unlock: bad refcount so=%p "
+				    "value=%d\n", so, so->so_usecount);
 			so->so_usecount--;
 			if (so->so_usecount == 0) {
 				sofreelastref(so, 1);
@@ -2924,36 +3875,48 @@ socket_unlock(so, refcount)
 		lck_mtx_unlock(mutex_held);
 	}
 
-	return(error);
+	return (error);
 }
-//### Called with socket locked, will unlock socket
+
+/* Called with socket locked, will unlock socket */
 void
-sofree(so) 
-	struct socket *so;
+sofree(struct socket *so)
 {
 
-	lck_mtx_t * mutex_held;
-	if (so->so_proto->pr_getlock != NULL)  
+	lck_mtx_t *mutex_held;
+	if (so->so_proto->pr_getlock != NULL)
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-	else  
+	else
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
-	
+
 	sofreelastref(so, 0);
 }
 
 void
-soreference(so)
-	struct socket *so;
+soreference(struct socket *so)
 {
 	socket_lock(so, 1);	/* locks & take one reference on socket */
 	socket_unlock(so, 0);	/* unlock only */
 }
 
 void
-sodereference(so)
-	struct socket *so;
+sodereference(struct socket *so)
 {
 	socket_lock(so, 0);
 	socket_unlock(so, 1);
 }
+
+/*
+ * Set or clear SOF_MULTIPAGES on the socket to enable or disable the
+ * possibility of using jumbo clusters.  Caller must ensure to hold
+ * the socket lock.
+ */
+void
+somultipages(struct socket *so, boolean_t set)
+{
+	if (set)
+		so->so_flags |= SOF_MULTIPAGES;
+	else
+		so->so_flags &= ~SOF_MULTIPAGES;
+}
diff --git a/bsd/kern/uipc_socket2.c b/bsd/kern/uipc_socket2.c
index 723f797c4..379b9afd6 100644
--- a/bsd/kern/uipc_socket2.c
+++ b/bsd/kern/uipc_socket2.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993
@@ -56,6 +61,12 @@
  *	@(#)uipc_socket2.c	8.1 (Berkeley) 6/10/93
  * $FreeBSD: src/sys/kern/uipc_socket2.c,v 1.55.2.9 2001/07/26 18:53:02 peter Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -77,19 +88,37 @@
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <sys/kdebug.h>
+#include <libkern/OSAtomic.h>
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
+/* TODO: this should be in a header file somewhere */
+extern void postevent(struct socket *, struct sockbuf *, int);
 
 #define DBG_FNC_SBDROP	NETDBG_CODE(DBG_NETSOCK, 4)
 #define DBG_FNC_SBAPPEND	NETDBG_CODE(DBG_NETSOCK, 5)
 
-static int sbcompress(struct sockbuf *, struct mbuf *, struct mbuf *);
+static inline void sbcompress(struct sockbuf *, struct mbuf *, struct mbuf *);
+static struct socket *sonewconn_internal(struct socket *, int);
+static int sbappendaddr_internal(struct sockbuf *, struct sockaddr *,
+    struct mbuf *, struct mbuf *);
+static int sbappendcontrol_internal(struct sockbuf *, struct mbuf *,
+    struct mbuf *);
 
 /*
  * Primitive routines for operating on sockets and socket buffers
  */
+static int soqlimitcompat = 1;
+static int soqlencomp = 0;
 
 u_long	sb_max = SB_MAX;		/* XXX should be static */
 
 static	u_long sb_efficiency = 8;	/* parameter for sbreserve() */
+__private_extern__ unsigned int total_mb_cnt = 0;
+__private_extern__ unsigned int total_cl_cnt = 0;
+__private_extern__ int sbspace_factor = 8;
 
 /*
  * Procedures to manipulate state flags of socket
@@ -121,27 +150,25 @@ static	u_long sb_efficiency = 8;	/* parameter for sbreserve() */
  * cause software-interrupt process scheduling.
  */
 void
-soisconnecting(so)
-	register struct socket *so;
+soisconnecting(struct socket *so)
 {
 
 	so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= SS_ISCONNECTING;
-	
+
 	sflt_notify(so, sock_evt_connecting, NULL);
 }
 
 void
-soisconnected(so)
-	struct socket *so;
+soisconnected(struct socket *so)
 {
 	struct socket *head = so->so_head;
 
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
-	
+
 	sflt_notify(so, sock_evt_connected, NULL);
-	
+
 	if (head && (so->so_state & SS_INCOMP)) {
 		so->so_state &= ~SS_INCOMP;
 		so->so_state |= SS_COMP;
@@ -168,8 +195,7 @@ soisconnected(so)
 }
 
 void
-soisdisconnecting(so)
-	register struct socket *so;
+soisdisconnecting(struct socket *so)
 {
 	so->so_state &= ~SS_ISCONNECTING;
 	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
@@ -180,8 +206,7 @@ soisdisconnecting(so)
 }
 
 void
-soisdisconnected(so)
-	register struct socket *so;
+soisdisconnected(struct socket *so)
 {
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
@@ -191,59 +216,6 @@ soisdisconnected(so)
 	sorwakeup(so);
 }
 
-/*
- * Return a random connection that hasn't been serviced yet and
- * is eligible for discard.  There is a one in qlen chance that
- * we will return a null, saying that there are no dropable
- * requests.  In this case, the protocol specific code should drop
- * the new request.  This insures fairness.
- *
- * This may be used in conjunction with protocol specific queue
- * congestion routines.
- */
-struct socket *
-sodropablereq(head)
-	register struct socket *head;
-{
-	struct socket *so, *sonext = NULL;
-	unsigned int i, j, qlen;
-	static int rnd;
-	static struct timeval old_runtime;
-	static unsigned int cur_cnt, old_cnt;
-	struct timeval tv;
-
-	microtime(&tv);
-	if ((i = (tv.tv_sec - old_runtime.tv_sec)) != 0) {
-		old_runtime = tv;
-		old_cnt = cur_cnt / i;
-		cur_cnt = 0;
-	}
-
-	so = TAILQ_FIRST(&head->so_incomp);
-	if (!so)
-		return (NULL);
-
-	qlen = head->so_incqlen;
-	if (++cur_cnt > qlen || old_cnt > qlen) {
-		rnd = (314159 * rnd + 66329) & 0xffff;
-		j = ((qlen + 1) * rnd) >> 16;
-//###LD To clean up
-		while (j-- && so) {
-//			if (in_pcb_checkstate(so->so_pcb, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
-				socket_lock(so, 1);
-		    		sonext = TAILQ_NEXT(so, so_list);
-//				in_pcb_check_state(so->so_pcb, WNT_RELEASE, 0);
-				socket_unlock(so, 1);	
-		    		so = sonext;
-		}
-	}
-
-//	if (in_pcb_checkstate(so->so_pcb, WNT_ACQUIRE, 0) == WNT_STOPUSING) 
-//		return (NULL);
-//	else
-		return (so);
-}
-
 /*
  * When an attempt at a new connection is noted on a socket
  * which accepts connections, sonewconn is called.  If the
@@ -253,29 +225,49 @@ sodropablereq(head)
  * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
  */
 static struct socket *
-sonewconn_internal(head, connstatus)
-	register struct socket *head;
-	int connstatus;
+sonewconn_internal(struct socket *head, int connstatus)
 {
-	int error = 0;
-	register struct socket *so;
+	int so_qlen, error = 0;
+	struct socket *so;
 	lck_mtx_t *mutex_held;
 
-	if (head->so_proto->pr_getlock != NULL) 
+	if (head->so_proto->pr_getlock != NULL)
 		mutex_held = (*head->so_proto->pr_getlock)(head, 0);
-	else 
+	else
 		mutex_held = head->so_proto->pr_domain->dom_mtx;
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 
-	if (head->so_qlen > 3 * head->so_qlimit / 2)
+	if (!soqlencomp) {
+		/*
+		 * This is the default case; so_qlen represents the
+		 * sum of both incomplete and completed queues.
+		 */
+		so_qlen = head->so_qlen;
+	} else {
+		/*
+		 * When kern.ipc.soqlencomp is set to 1, so_qlen
+		 * represents only the completed queue.  Since we
+		 * cannot let the incomplete queue goes unbounded
+		 * (in case of SYN flood), we cap the incomplete
+		 * queue length to at most somaxconn, and use that
+		 * as so_qlen so that we fail immediately below.
+		 */
+		so_qlen = head->so_qlen - head->so_incqlen;
+		if (head->so_incqlen > somaxconn)
+			so_qlen = somaxconn;
+	}
+
+	if (so_qlen >=
+	    (soqlimitcompat ? head->so_qlimit : (3 * head->so_qlimit / 2)))
 		return ((struct socket *)0);
-	so = soalloc(1, head->so_proto->pr_domain->dom_family, head->so_type);
+	so = soalloc(M_NOWAIT, head->so_proto->pr_domain->dom_family,
+	    head->so_type);
 	if (so == NULL)
 		return ((struct socket *)0);
 	/* check if head was closed during the soalloc */
 	if (head->so_proto == NULL) {
-	  sodealloc(so);
-	  return ((struct socket *)0);
+		sodealloc(so);
+		return ((struct socket *)0);
 	}
 
 	so->so_head = head;
@@ -287,6 +279,7 @@ sonewconn_internal(head, connstatus)
 	so->so_timeo = head->so_timeo;
 	so->so_pgid  = head->so_pgid;
 	so->so_uid = head->so_uid;
+	so->so_flags = head->so_flags & (SOF_REUSESHAREUID|SOF_NOTIFYCONFLICT); /* inherit SO_REUSESHAREUID and SO_NOTIFYCONFLICT ocket options */
 	so->so_usecount = 1;
 	so->next_lock_lr = 0;
 	so->next_unlock_lr = 0;
@@ -297,6 +290,10 @@ sonewconn_internal(head, connstatus)
 	TAILQ_INIT(&so->so_evlist);
 #endif
 
+#if CONFIG_MACF_SOCKET
+	mac_socket_label_associate_accept(head, so);
+#endif
+
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
 		sflt_termsock(so);
 		sodealloc(so);
@@ -304,11 +301,13 @@ sonewconn_internal(head, connstatus)
 	}
 
 	/*
-	 * Must be done with head unlocked to avoid deadlock for protocol with per socket mutexes.
+	 * Must be done with head unlocked to avoid deadlock
+	 * for protocol with per socket mutexes.
 	 */
 	if (head->so_proto->pr_unlock)
 		socket_unlock(head, 0);
-	if (((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL) != 0) || error) {
+	if (((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL) != 0) ||
+	    error) {
 		sflt_termsock(so);
 		sodealloc(so);
 		if (head->so_proto->pr_unlock)
@@ -335,6 +334,7 @@ sonewconn_internal(head, connstatus)
 	/* Attach socket filters for this protocol */
 	sflt_initsock(so);
 #endif
+
 	if (connstatus) {
 		so->so_state |= connstatus;
 		sorwakeup(head);
@@ -345,38 +345,34 @@ sonewconn_internal(head, connstatus)
 
 
 struct socket *
-sonewconn(
-	struct socket *head,
-	int connstatus,
-	const struct sockaddr *from)
+sonewconn(struct socket *head, int connstatus, const struct sockaddr *from)
 {
 	int error = 0;
-	struct socket_filter_entry	*filter;
-	int						 	filtered = 0;
-	
-	error = 0;
+	struct socket_filter_entry *filter;
+	int filtered = 0;
+
 	for (filter = head->so_filt; filter && (error == 0);
-		 filter = filter->sfe_next_onsocket) {
+	    filter = filter->sfe_next_onsocket) {
 		if (filter->sfe_filter->sf_filter.sf_connect_in) {
 			if (filtered == 0) {
 				filtered = 1;
 				sflt_use(head);
 				socket_unlock(head, 0);
 			}
-			error = filter->sfe_filter->sf_filter.sf_connect_in(
-						filter->sfe_cookie, head, from);
+			error = filter->sfe_filter->sf_filter.
+			    sf_connect_in(filter->sfe_cookie, head, from);
 		}
 	}
 	if (filtered != 0) {
 		socket_lock(head, 0);
 		sflt_unuse(head);
 	}
-	
+
 	if (error) {
-		return NULL;
+		return (NULL);
 	}
-	
-	return sonewconn_internal(head, connstatus);
+
+	return (sonewconn_internal(head, connstatus));
 }
 
 /*
@@ -390,8 +386,7 @@ sonewconn(
  */
 
 void
-socantsendmore(so)
-	struct socket *so;
+socantsendmore(struct socket *so)
 {
 	so->so_state |= SS_CANTSENDMORE;
 	sflt_notify(so, sock_evt_cantsendmore, NULL);
@@ -399,8 +394,7 @@ socantsendmore(so)
 }
 
 void
-socantrcvmore(so)
-	struct socket *so;
+socantrcvmore(struct socket *so)
 {
 	so->so_state |= SS_CANTRCVMORE;
 	sflt_notify(so, sock_evt_cantrecvmore, NULL);
@@ -409,10 +403,13 @@ socantrcvmore(so)
 
 /*
  * Wait for data to arrive at/drain from a socket buffer.
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *	msleep:EINTR
  */
 int
-sbwait(sb)
-	struct sockbuf *sb;
+sbwait(struct sockbuf *sb)
 {
 	int error = 0, lr_saved;
 	struct socket *so = sb->sb_so;
@@ -420,26 +417,25 @@ sbwait(sb)
 	struct timespec ts;
 
 	lr_saved = (unsigned int) __builtin_return_address(0);
-	
-	if (so->so_proto->pr_getlock != NULL) 
+
+	if (so->so_proto->pr_getlock != NULL)
 		mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-	else 
+	else
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 
 	sb->sb_flags |= SB_WAIT;
 
 	if (so->so_usecount < 1)
-		panic("sbwait: so=%x refcount=%d\n", so, so->so_usecount);
+		panic("sbwait: so=%p refcount=%d\n", so, so->so_usecount);
 	ts.tv_sec = sb->sb_timeo.tv_sec;
 	ts.tv_nsec = sb->sb_timeo.tv_usec * 1000;
 	error = msleep((caddr_t)&sb->sb_cc, mutex_held,
-    		(sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait",
-    		&ts);
+	    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", &ts);
 
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 
 	if (so->so_usecount < 1)
-		panic("sbwait: so=%x refcount=%d\n", so, so->so_usecount);
+		panic("sbwait: so=%p refcount=%d\n", so, so->so_usecount);
 
 	if ((so->so_state & SS_DRAINING)) {
 		error = EBADF;
@@ -451,32 +447,37 @@ sbwait(sb)
 /*
  * Lock a sockbuf already known to be locked;
  * return any error returned from sleep (EINTR).
+ *
+ * Returns:	0			Success
+ *		EINTR
  */
 int
-sb_lock(sb)
-	register struct sockbuf *sb;
+sb_lock(struct sockbuf *sb)
 {
 	struct socket *so = sb->sb_so;
-	lck_mtx_t * mutex_held;
+	lck_mtx_t *mutex_held;
 	int error = 0;
-	
+
 	if (so == NULL)
-		panic("sb_lock: null so back pointer sb=%x\n", sb);
+		panic("sb_lock: null so back pointer sb=%p\n", sb);
 
 	while (sb->sb_flags & SB_LOCK) {
 		sb->sb_flags |= SB_WANT;
-		if (so->so_proto->pr_getlock != NULL) 
+		if (so->so_proto->pr_getlock != NULL)
 			mutex_held = (*so->so_proto->pr_getlock)(so, 0);
 		else
 			mutex_held = so->so_proto->pr_domain->dom_mtx;
 		if (so->so_usecount < 1)
-			panic("sb_lock: so=%x refcount=%d\n", so, so->so_usecount);
+			panic("sb_lock: so=%p refcount=%d\n", so,
+			    so->so_usecount);
 
 		error = msleep((caddr_t)&sb->sb_flags, mutex_held,
-	    		(sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sblock", 0);
+		    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH,
+		    "sb_lock", 0);
 		if (so->so_usecount < 1)
-			panic("sb_lock: 2 so=%x refcount=%d\n", so, so->so_usecount);
-		if (error) 
+			panic("sb_lock: 2 so=%p refcount=%d\n", so,
+			    so->so_usecount);
+		if (error)
 			return (error);
 	}
 	sb->sb_flags |= SB_LOCK;
@@ -489,11 +490,8 @@ sb_lock(sb)
  * if the socket has the SS_ASYNC flag set.
  */
 void
-sowakeup(so, sb)
-	register struct socket *so;
-	register struct sockbuf *sb;
+sowakeup(struct socket *so, struct sockbuf *sb)
 {
-	struct proc *p = current_proc();
 	sb->sb_flags &= ~SB_SEL;
 	selwakeup(&sb->sb_sel);
 	if (sb->sb_flags & SB_WAIT) {
@@ -503,16 +501,29 @@ sowakeup(so, sb)
 	if (so->so_state & SS_ASYNC) {
 		if (so->so_pgid < 0)
 			gsignal(-so->so_pgid, SIGIO);
-		else if (so->so_pgid > 0 && (p = pfind(so->so_pgid)) != 0)
-			psignal(p, SIGIO);
+		else if (so->so_pgid > 0)
+			proc_signal(so->so_pgid, SIGIO);
 	}
 	if (sb->sb_flags & SB_KNOTE) {
 		KNOTE(&sb->sb_sel.si_note, SO_FILT_HINT_LOCKED);
 	}
 	if (sb->sb_flags & SB_UPCALL) {
+		void (*so_upcall)(struct socket *, caddr_t, int);
+		caddr_t so_upcallarg;
+
+		so_upcall = so->so_upcall;
+		so_upcallarg = so->so_upcallarg;
+		/* Let close know that we're about to do an upcall */
+		so->so_flags |= SOF_UPCALLINUSE;
+
 		socket_unlock(so, 0);
-		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
+		(*so_upcall)(so, so_upcallarg, M_DONTWAIT);
 		socket_lock(so, 0);
+
+		so->so_flags &= ~SOF_UPCALLINUSE;
+		/* Tell close that it's safe to proceed */
+		if (so->so_flags & SOF_CLOSEWAIT)
+			wakeup((caddr_t)&so->so_upcall);
 	}
 }
 
@@ -548,10 +559,12 @@ sowakeup(so, sb)
  * should be released by calling sbrelease() when the socket is destroyed.
  */
 
+/*
+ * Returns:	0			Success
+ *		ENOBUFS
+ */
 int
-soreserve(so, sndcc, rcvcc)
-	register struct socket *so;
-	u_long sndcc, rcvcc;
+soreserve(struct socket *so, u_long sndcc, u_long rcvcc)
 {
 
 	if (sbreserve(&so->so_snd, sndcc) == 0)
@@ -580,9 +593,7 @@ bad:
  * if buffering efficiency is near the normal case.
  */
 int
-sbreserve(sb, cc)
-	struct sockbuf *sb;
-	u_long cc;
+sbreserve(struct sockbuf *sb, u_long cc)
 {
 	if ((u_quad_t)cc > (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES))
 		return (0);
@@ -596,16 +607,13 @@ sbreserve(sb, cc)
 /*
  * Free mbufs held by a socket, and reserved mbuf space.
  */
- /*  WARNING needs to do selthreadclear() before calling this */
+/*  WARNING needs to do selthreadclear() before calling this */
 void
-sbrelease(sb)
-	struct sockbuf *sb;
+sbrelease(struct sockbuf *sb)
 {
-
 	sbflush(sb);
 	sb->sb_hiwat = 0;
 	sb->sb_mbmax = 0;
-	
 }
 
 /*
@@ -640,73 +648,88 @@ sbrelease(sb)
  * discarded and mbufs are compacted where possible.
  */
 int
-sbappend(sb, m)
-	struct sockbuf *sb;
-	struct mbuf *m;
+sbappend(struct sockbuf *sb, struct mbuf *m)
 {
-	register struct mbuf *n, *sb_first;
-	int result = 0;
-	int error = 0;
-	int	filtered = 0;
+	struct socket *so = sb->sb_so;
 
+	if (m == NULL || (sb->sb_flags & SB_DROP)) {
+		if (m != NULL)
+			m_freem(m);
+		return (0);
+	}
 
-	KERNEL_DEBUG((DBG_FNC_SBAPPEND | DBG_FUNC_START), sb, m->m_len, 0, 0, 0);
+	SBLASTRECORDCHK(sb, "sbappend 1");
 
-	if (m == 0)
-		return 0;
-	
-again:
-	sb_first = n = sb->sb_mb;
-	if (n) {
-		while (n->m_nextpkt)
-			n = n->m_nextpkt;
-		do {
-			if (n->m_flags & M_EOR) {
-				result = sbappendrecord(sb, m); /* XXXXXX!!!! */
-				KERNEL_DEBUG((DBG_FNC_SBAPPEND | DBG_FUNC_END), sb, sb->sb_cc, 0, 0, 0);
-				return result;
-			}
-		} while (n->m_next && (n = n->m_next));
-	}
-	
-	if (!filtered && (sb->sb_flags & SB_RECV) != 0) {
-		error = sflt_data_in(sb->sb_so, NULL, &m, NULL, 0, &filtered);
-		if (error) {
-			/* no data was appended, caller should not call sowakeup */
-			return 0;
+	if (sb->sb_lastrecord != NULL && (sb->sb_mbtail->m_flags & M_EOR))
+		return (sbappendrecord(sb, m));
+
+	if (sb->sb_flags & SB_RECV) {
+		int error = sflt_data_in(so, NULL, &m, NULL, 0, NULL);
+		SBLASTRECORDCHK(sb, "sbappend 2");
+		if (error != 0) {
+			if (error != EJUSTRETURN)
+				m_freem(m);
+			return (0);
 		}
-		
-		/*
-		  If we any filters, the socket lock was dropped. n and sb_first
-		  cached data from the socket buffer. This cache is not valid
-		  since we dropped the lock. We must start over. Since filtered
-		  is set we won't run through the filters a second time. We just
-		  set n and sb_start again.
-		*/
-		if (filtered)
-			goto again;
 	}
 
-	result = sbcompress(sb, m, n);
+	/* If this is the first record, it's also the last record */
+	if (sb->sb_lastrecord == NULL)
+		sb->sb_lastrecord = m;
 
-	KERNEL_DEBUG((DBG_FNC_SBAPPEND | DBG_FUNC_END), sb, sb->sb_cc, 0, 0, 0);
-	
-	return result;
+	sbcompress(sb, m, sb->sb_mbtail);
+	SBLASTRECORDCHK(sb, "sbappend 3");
+	return (1);
+}
+
+/*
+ * Similar to sbappend, except that this is optimized for stream sockets.
+ */
+int
+sbappendstream(struct sockbuf *sb, struct mbuf *m)
+{
+	struct socket *so = sb->sb_so;
+
+	if (m->m_nextpkt != NULL || (sb->sb_mb != sb->sb_lastrecord))
+		panic("sbappendstream: nexpkt %p || mb %p != lastrecord %p\n",
+		    m->m_nextpkt, sb->sb_mb, sb->sb_lastrecord);
+
+	SBLASTMBUFCHK(sb, __func__);
+
+	if (m == NULL || (sb->sb_flags & SB_DROP)) {
+		if (m != NULL)
+			m_freem(m);
+		return (0);
+	}
+
+	if (sb->sb_flags & SB_RECV) {
+		int error = sflt_data_in(so, NULL, &m, NULL, 0, NULL);
+		SBLASTRECORDCHK(sb, "sbappendstream 1");
+		if (error != 0) {
+			if (error != EJUSTRETURN)
+				m_freem(m);
+			return (0);
+		}
+	}
+
+	sbcompress(sb, m, sb->sb_mbtail);
+	sb->sb_lastrecord = sb->sb_mb;
+	SBLASTRECORDCHK(sb, "sbappendstream 2");
+	return (1);
 }
 
 #ifdef SOCKBUF_DEBUG
 void
-sbcheck(sb)
-	register struct sockbuf *sb;
+sbcheck(struct sockbuf *sb)
 {
-	register struct mbuf *m;
-	register struct mbuf *n = 0;
-	register u_long len = 0, mbcnt = 0;
+	struct mbuf *m;
+	struct mbuf *n = 0;
+	u_long len = 0, mbcnt = 0;
 	lck_mtx_t *mutex_held;
 
-	if (sb->sb_so->so_proto->pr_getlock != NULL) 
+	if (sb->sb_so->so_proto->pr_getlock != NULL)
 		mutex_held = (*sb->sb_so->so_proto->pr_getlock)(sb->sb_so, 0);
-	else 
+	else
 		mutex_held = sb->sb_so->so_proto->pr_domain->dom_mtx;
 
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
@@ -715,65 +738,121 @@ sbcheck(sb)
 		return;
 
 	for (m = sb->sb_mb; m; m = n) {
-	    n = m->m_nextpkt;
-	    for (; m; m = m->m_next) {
-                len += m->m_len;
-                mbcnt += MSIZE;
-                if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */
-                    mbcnt += m->m_ext.ext_size;
-            }
+		n = m->m_nextpkt;
+		for (; m; m = m->m_next) {
+			len += m->m_len;
+			mbcnt += MSIZE;
+			/* XXX pretty sure this is bogus */
+			if (m->m_flags & M_EXT)
+				mbcnt += m->m_ext.ext_size;
+		}
+	}
+	if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
+		panic("cc %ld != %ld || mbcnt %ld != %ld\n", len, sb->sb_cc,
+		    mbcnt, sb->sb_mbcnt);
 	}
-        if (len != sb->sb_cc || mbcnt != sb->sb_mbcnt) {
-                panic("cc %ld != %ld || mbcnt %ld != %ld\n", len, sb->sb_cc,
-                    mbcnt, sb->sb_mbcnt);
-        }
 }
 #endif
 
+void
+sblastrecordchk(struct sockbuf *sb, const char *where)
+{
+	struct mbuf *m = sb->sb_mb;
+
+	while (m && m->m_nextpkt)
+		m = m->m_nextpkt;
+
+	if (m != sb->sb_lastrecord) {
+		printf("sblastrecordchk: mb %p lastrecord %p last %p\n",
+		    sb->sb_mb, sb->sb_lastrecord, m);
+		printf("packet chain:\n");
+		for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt)
+			printf("\t%p\n", m);
+		panic("sblastrecordchk from %s", where);
+	}
+}
+
+void
+sblastmbufchk(struct sockbuf *sb, const char *where)
+{
+	struct mbuf *m = sb->sb_mb;
+	struct mbuf *n;
+
+	while (m && m->m_nextpkt)
+		m = m->m_nextpkt;
+
+	while (m && m->m_next)
+		m = m->m_next;
+
+	if (m != sb->sb_mbtail) {
+		printf("sblastmbufchk: mb %p mbtail %p last %p\n",
+		    sb->sb_mb, sb->sb_mbtail, m);
+		printf("packet tree:\n");
+		for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) {
+			printf("\t");
+			for (n = m; n != NULL; n = n->m_next)
+				printf("%p ", n);
+			printf("\n");
+		}
+		panic("sblastmbufchk from %s", where);
+	}
+}
+
 /*
- * As above, except the mbuf chain
- * begins a new record.
+ * Similar to sbappend, except the mbuf chain begins a new record.
  */
 int
-sbappendrecord(sb, m0)
-	register struct sockbuf *sb;
-	struct mbuf *m0;
+sbappendrecord(struct sockbuf *sb, struct mbuf *m0)
 {
-	register struct mbuf *m;
-	int result = 0;
+	struct mbuf *m;
+	int space = 0;
 
-	if (m0 == 0)
-		return 0;
-    
-	if ((sb->sb_flags & SB_RECV) != 0) {
-		int error = sflt_data_in(sb->sb_so, NULL, &m0, NULL, sock_data_filt_flag_record, NULL);
+	if (m0 == NULL || (sb->sb_flags & SB_DROP)) {
+		if (m0 != NULL)
+			m_freem(m0);
+		return (0);
+	}
+
+	for (m = m0; m != NULL; m = m->m_next)
+		space += m->m_len;
+
+	if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX)) {
+		m_freem(m0);
+		return (0);
+	}
+
+	if (sb->sb_flags & SB_RECV) {
+		int error = sflt_data_in(sb->sb_so, NULL, &m0, NULL,
+		    sock_data_filt_flag_record, NULL);
 		if (error != 0) {
+			SBLASTRECORDCHK(sb, "sbappendrecord 1");
 			if (error != EJUSTRETURN)
 				m_freem(m0);
-			return 0;
+			return (0);
 		}
 	}
-    
-	m = sb->sb_mb;
-	if (m)
-		while (m->m_nextpkt)
-			m = m->m_nextpkt;
+
 	/*
-	 * Put the first mbuf on the queue.
 	 * Note this permits zero length records.
 	 */
 	sballoc(sb, m0);
-	if (m)
-		m->m_nextpkt = m0;
-	else
+	SBLASTRECORDCHK(sb, "sbappendrecord 2");
+	if (sb->sb_lastrecord != NULL) {
+		sb->sb_lastrecord->m_nextpkt = m0;
+	}  else {
 		sb->sb_mb = m0;
+	}
+	sb->sb_lastrecord = m0;
+
 	m = m0->m_next;
 	m0->m_next = 0;
 	if (m && (m0->m_flags & M_EOR)) {
 		m0->m_flags &= ~M_EOR;
 		m->m_flags |= M_EOR;
 	}
-	return sbcompress(sb, m, m0);
+	sbcompress(sb, m, m0);
+	SBLASTRECORDCHK(sb, "sbappendrecord 3");
+	return (1);
 }
 
 /*
@@ -782,31 +861,32 @@ sbappendrecord(sb, m0)
  * but after any other OOB data.
  */
 int
-sbinsertoob(sb, m0)
-	struct sockbuf *sb;
-	struct mbuf *m0;
+sbinsertoob(struct sockbuf *sb, struct mbuf *m0)
 {
 	struct mbuf *m;
 	struct mbuf **mp;
 
 	if (m0 == 0)
-		return 0;
-	
+		return (0);
+
+	SBLASTRECORDCHK(sb, "sbinsertoob 1");
+
 	if ((sb->sb_flags & SB_RECV) != 0) {
 		int error = sflt_data_in(sb->sb_so, NULL, &m0, NULL,
-								 sock_data_filt_flag_oob, NULL);
-		
+		    sock_data_filt_flag_oob, NULL);
+
+		SBLASTRECORDCHK(sb, "sbinsertoob 2");
 		if (error) {
 			if (error != EJUSTRETURN) {
 				m_freem(m0);
 			}
-			return 0;
+			return (0);
 		}
 	}
-    
-	for (mp = &sb->sb_mb; *mp ; mp = &((*mp)->m_nextpkt)) {
-	    m = *mp;
-	    again:
+
+	for (mp = &sb->sb_mb; *mp; mp = &((*mp)->m_nextpkt)) {
+		m = *mp;
+again:
 		switch (m->m_type) {
 
 		case MT_OOBDATA:
@@ -825,6 +905,10 @@ sbinsertoob(sb, m0)
 	 */
 	sballoc(sb, m0);
 	m0->m_nextpkt = *mp;
+	if (*mp == NULL) {
+		/* m0 is actually the new tail */
+		sb->sb_lastrecord = m0;
+	}
 	*mp = m0;
 	m = m0->m_next;
 	m0->m_next = 0;
@@ -832,7 +916,9 @@ sbinsertoob(sb, m0)
 		m0->m_flags &= ~M_EOR;
 		m->m_flags |= M_EOR;
 	}
-	return sbcompress(sb, m, m0);
+	sbcompress(sb, m, m0);
+	SBLASTRECORDCHK(sb, "sbinsertoob 3");
+	return (1);
 }
 
 /*
@@ -840,14 +926,15 @@ sbinsertoob(sb, m0)
  * to the receive queue of a socket.  If present,
  * m0 must include a packet header with total length.
  * Returns 0 if no space in sockbuf or insufficient mbufs.
+ *
+ * Returns:	0			No space/out of mbufs
+ *		1			Success
  */
 static int
-sbappendaddr_internal(sb, asa, m0, control)
-	register struct sockbuf *sb;
-	struct sockaddr *asa;
-	struct mbuf *m0, *control;
+sbappendaddr_internal(struct sockbuf *sb, struct sockaddr *asa,
+    struct mbuf *m0, struct mbuf *control)
 {
-	register struct mbuf *m, *n;
+	struct mbuf *m, *n, *nlast;
 	int space = asa->sa_len;
 
 	if (m0 && (m0->m_flags & M_PKTHDR) == 0)
@@ -874,64 +961,96 @@ sbappendaddr_internal(sb, asa, m0, control)
 	else
 		control = m0;
 	m->m_next = control;
-	for (n = m; n; n = n->m_next)
+
+	SBLASTRECORDCHK(sb, "sbappendadddr 1");
+
+	for (n = m; n->m_next != NULL; n = n->m_next)
 		sballoc(sb, n);
-	n = sb->sb_mb;
-	if (n) {
-		while (n->m_nextpkt)
-			n = n->m_nextpkt;
-		n->m_nextpkt = m;
-	} else
+	sballoc(sb, n);
+	nlast = n;
+
+	if (sb->sb_lastrecord != NULL) {
+		sb->sb_lastrecord->m_nextpkt = m;
+	} else {
 		sb->sb_mb = m;
-	postevent(0,sb,EV_RWBYTES);
+	}
+	sb->sb_lastrecord = m;
+	sb->sb_mbtail = nlast;
+
+	SBLASTMBUFCHK(sb, __func__);
+	SBLASTRECORDCHK(sb, "sbappendadddr 2");
+
+	postevent(0, sb, EV_RWBYTES);
 	return (1);
 }
 
+/*
+ * Returns:	0			Error: No space/out of mbufs/etc.
+ *		1			Success
+ *
+ * Imputed:	(*error_out)		errno for error
+ *		ENOBUFS
+ *	sflt_data_in:???		[whatever a filter author chooses]
+ */
 int
-sbappendaddr(
-	struct sockbuf* sb,
-	struct sockaddr* asa,
-	struct mbuf *m0,
-	struct mbuf *control,
-	int	*error_out)
+sbappendaddr(struct sockbuf *sb, struct sockaddr *asa, struct mbuf *m0,
+    struct mbuf *control, int *error_out)
 {
 	int result = 0;
-	
-	if (error_out) *error_out = 0;
-	
+	boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
+
+	if (error_out)
+		*error_out = 0;
+
 	if (m0 && (m0->m_flags & M_PKTHDR) == 0)
 		panic("sbappendaddrorfree");
-	
+
+	if (sb->sb_flags & SB_DROP) {
+		if (m0 != NULL)
+			m_freem(m0);
+		if (control != NULL && !sb_unix)
+			m_freem(control);
+		if (error_out != NULL)
+			*error_out = EINVAL;
+		return (0);
+	}
+
 	/* Call socket data in filters */
 	if ((sb->sb_flags & SB_RECV) != 0) {
 		int error;
 		error = sflt_data_in(sb->sb_so, asa, &m0, &control, 0, NULL);
+		SBLASTRECORDCHK(sb, __func__);
 		if (error) {
 			if (error != EJUSTRETURN) {
-				if (m0) m_freem(m0);
-				if (control) m_freem(control);
-				if (error_out) *error_out = error;
+				if (m0)
+					m_freem(m0);
+				if (control != NULL && !sb_unix)
+					m_freem(control);
+				if (error_out)
+					*error_out = error;
 			}
-			return 0;
+			return (0);
 		}
 	}
-	
+
 	result = sbappendaddr_internal(sb, asa, m0, control);
 	if (result == 0) {
-		if (m0) m_freem(m0);
-		if (control) m_freem(control);
-		if (error_out) *error_out = ENOBUFS;
+		if (m0)
+			m_freem(m0);
+		if (control != NULL && !sb_unix)
+			m_freem(control);
+		if (error_out)
+			*error_out = ENOBUFS;
 	}
-	
-	return result;
+
+	return (result);
 }
 
 static int
-sbappendcontrol_internal(sb, m0, control)
-	struct sockbuf *sb;
-	struct mbuf *control, *m0;
+sbappendcontrol_internal(struct sockbuf *sb, struct mbuf *m0,
+    struct mbuf *control)
 {
-	register struct mbuf *m, *n;
+	struct mbuf *m, *mlast, *n;
 	int space = 0;
 
 	if (control == 0)
@@ -945,54 +1064,81 @@ sbappendcontrol_internal(sb, m0, control)
 	n = m;			/* save pointer to last control buffer */
 	for (m = m0; m; m = m->m_next)
 		space += m->m_len;
-	if (space > sbspace(sb))
+	if (space > sbspace(sb) && !(sb->sb_flags & SB_UNIX))
 		return (0);
 	n->m_next = m0;			/* concatenate data to control */
-	for (m = control; m; m = m->m_next)
+
+	SBLASTRECORDCHK(sb, "sbappendcontrol 1");
+
+	for (m = control; m->m_next != NULL; m = m->m_next)
 		sballoc(sb, m);
-	n = sb->sb_mb;
-	if (n) {
-		while (n->m_nextpkt)
-			n = n->m_nextpkt;
-		n->m_nextpkt = control;
-	} else
+	sballoc(sb, m);
+	mlast = m;
+
+	if (sb->sb_lastrecord != NULL) {
+		sb->sb_lastrecord->m_nextpkt = control;
+	} else {
 		sb->sb_mb = control;
-	postevent(0,sb,EV_RWBYTES);
+	}
+	sb->sb_lastrecord = control;
+	sb->sb_mbtail = mlast;
+
+	SBLASTMBUFCHK(sb, __func__);
+	SBLASTRECORDCHK(sb, "sbappendcontrol 2");
+
+	postevent(0, sb, EV_RWBYTES);
 	return (1);
 }
 
 int
-sbappendcontrol(
-	struct sockbuf	*sb,
-	struct mbuf		*m0,
-	struct mbuf		*control,
-	int				*error_out)
+sbappendcontrol(struct sockbuf *sb, struct mbuf	*m0, struct mbuf *control,
+    int *error_out)
 {
 	int result = 0;
-	
-	if (error_out) *error_out = 0;
-	
+	boolean_t sb_unix = (sb->sb_flags & SB_UNIX);
+
+	if (error_out)
+		*error_out = 0;
+
+	if (sb->sb_flags & SB_DROP) {
+		if (m0 != NULL)
+			m_freem(m0);
+		if (control != NULL && !sb_unix)
+			m_freem(control);
+		if (error_out != NULL)
+			*error_out = EINVAL;
+		return (0);
+	}
+
 	if (sb->sb_flags & SB_RECV) {
 		int error;
+
 		error = sflt_data_in(sb->sb_so, NULL, &m0, &control, 0, NULL);
+		SBLASTRECORDCHK(sb, __func__);
 		if (error) {
 			if (error != EJUSTRETURN) {
-				if (m0) m_freem(m0);
-				if (control) m_freem(control);
-				if (error_out) *error_out = error;
+				if (m0)
+					m_freem(m0);
+				if (control != NULL && !sb_unix)
+					m_freem(control);
+				if (error_out)
+					*error_out = error;
 			}
-			return 0;
+			return (0);
 		}
 	}
-	
+
 	result = sbappendcontrol_internal(sb, m0, control);
 	if (result == 0) {
-		if (m0) m_freem(m0);
-		if (control) m_freem(control);
-		if (error_out) *error_out = ENOBUFS;
+		if (m0)
+			m_freem(m0);
+		if (control != NULL && !sb_unix)
+			m_freem(control);
+		if (error_out)
+			*error_out = ENOBUFS;
 	}
-	
-	return result;
+
+	return (result);
 }
 
 /*
@@ -1000,20 +1146,26 @@ sbappendcontrol(
  * buffer sb following mbuf n.  If n
  * is null, the buffer is presumed empty.
  */
-static int
-sbcompress(sb, m, n)
-	register struct sockbuf *sb;
-	register struct mbuf *m, *n;
-{
-	register int eor = 0;
-	register struct mbuf *o;
+static inline void
+sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n)
+{
+	int eor = 0;
+	struct mbuf *o;
+
+	if (m == NULL) {
+		/* There is nothing to compress; just update the tail */
+		for (; n->m_next != NULL; n = n->m_next)
+			;
+		sb->sb_mbtail = n;
+		goto done;
+	}
 
 	while (m) {
 		eor |= m->m_flags & M_EOR;
-		if (m->m_len == 0 &&
-		    (eor == 0 ||
-		     (((o = m->m_next) || (o = n)) &&
-		      o->m_type == m->m_type))) {
+		if (m->m_len == 0 && (eor == 0 ||
+		    (((o = m->m_next) || (o = n)) && o->m_type == m->m_type))) {
+			if (sb->sb_lastrecord == m)
+				sb->sb_lastrecord = m->m_next;
 			m = m_free(m);
 			continue;
 		}
@@ -1028,6 +1180,10 @@ sbcompress(sb, m, n)
 			    (unsigned)m->m_len);
 			n->m_len += m->m_len;
 			sb->sb_cc += m->m_len;
+			if (m->m_type != MT_DATA && m->m_type != MT_HEADER &&
+				m->m_type != MT_OOBDATA)
+				/* XXX: Probably don't need.*/
+				sb->sb_ctl += m->m_len;
 			m = m_free(m);
 			continue;
 		}
@@ -1035,6 +1191,7 @@ sbcompress(sb, m, n)
 			n->m_next = m;
 		else
 			sb->sb_mb = m;
+		sb->sb_mbtail = m;
 		sballoc(sb, m);
 		n = m;
 		m->m_flags &= ~M_EOR;
@@ -1047,8 +1204,21 @@ sbcompress(sb, m, n)
 		else
 			printf("semi-panic: sbcompress\n");
 	}
-	postevent(0,sb, EV_RWBYTES);
-	return 1;
+done:
+	SBLASTMBUFCHK(sb, __func__);
+	postevent(0, sb, EV_RWBYTES);
+}
+
+void
+sb_empty_assert(struct sockbuf *sb, const char *where)
+{
+	if (!(sb->sb_cc == 0 && sb->sb_mb == NULL && sb->sb_mbcnt == 0 &&
+	    sb->sb_mbtail == NULL && sb->sb_lastrecord == NULL)) {
+		panic("%s: sb %p so %p cc %ld mbcnt %ld mb %p mbtail %p "
+		    "lastrecord %p\n", where, sb, sb->sb_so, sb->sb_cc,
+		    sb->sb_mbcnt, sb->sb_mb, sb->sb_mbtail, sb->sb_lastrecord);
+		/* NOTREACHED */
+	}
 }
 
 /*
@@ -1056,12 +1226,11 @@ sbcompress(sb, m, n)
  * Check that all resources are reclaimed.
  */
 void
-sbflush(sb)
-	register struct sockbuf *sb;
+sbflush(struct sockbuf *sb)
 {
 	if (sb->sb_so == NULL)
-		panic ("sbflush sb->sb_so already null sb=%x\n", sb);
-	(void)sblock(sb, M_WAIT);
+		panic("sbflush sb->sb_so already null sb=%p\n", sb);
+	(void) sblock(sb, M_WAIT);
 	while (sb->sb_mbcnt) {
 		/*
 		 * Don't call sbdrop(sb, 0) if the leading mbuf is non-empty:
@@ -1071,9 +1240,7 @@ sbflush(sb)
 			break;
 		sbdrop(sb, (int)sb->sb_cc);
 	}
-	if (sb->sb_cc || sb->sb_mb || sb->sb_mbcnt || sb->sb_so == NULL)
-		panic("sbflush: cc %ld || mb %p || mbcnt %ld sb_so=%x", sb->sb_cc, (void *)sb->sb_mb, sb->sb_mbcnt, sb->sb_so);
-
+	sb_empty_assert(sb, __func__);
 	postevent(0, sb, EV_RWBYTES);
 	sbunlock(sb, 1);	/* keep socket locked */
 
@@ -1091,11 +1258,9 @@ sbflush(sb)
  * m_freem_list.
  */
 void
-sbdrop(sb, len)
-	register struct sockbuf *sb;
-	register int len;
+sbdrop(struct sockbuf *sb, int len)
 {
-	register struct mbuf *m, *free_list, *ml;
+	struct mbuf *m, *free_list, *ml;
 	struct mbuf *next, *last;
 
 	KERNEL_DEBUG((DBG_FNC_SBDROP | DBG_FUNC_START), sb, len, 0, 0, 0);
@@ -1106,28 +1271,36 @@ sbdrop(sb, len)
 
 	while (len > 0) {
 		if (m == 0) {
-		  if (next == 0) {
-		    /* temporarily replacing this panic with printf because
-		     * it occurs occasionally when closing a socket when there
-		     * is no harm in ignoring it.  This problem will be investigated
-		     * further.
-		     */
-		    /* panic("sbdrop"); */
-		    printf("sbdrop - count not zero\n");
-		    len = 0;
-		    /* zero the counts. if we have no mbufs, we have no data (PR-2986815) */
-		    sb->sb_cc = 0;
-		    sb->sb_mbcnt = 0;
-		    break;
-		  }
-		  m = last = next;
-		  next = m->m_nextpkt;
-		  continue;
+			if (next == 0) {
+				/*
+				 * temporarily replacing this panic with printf
+				 * because it occurs occasionally when closing
+				 * a socket when there is no harm in ignoring
+				 * it. This problem will be investigated
+				 * further.
+				 */
+				/* panic("sbdrop"); */
+				printf("sbdrop - count not zero\n");
+				len = 0;
+				/*
+				 * zero the counts. if we have no mbufs,
+				 * we have no data (PR-2986815)
+				 */
+				sb->sb_cc = 0;
+				sb->sb_mbcnt = 0;
+				break;
+			}
+			m = last = next;
+			next = m->m_nextpkt;
+			continue;
 		}
 		if (m->m_len > len) {
 			m->m_len -= len;
 			m->m_data += len;
 			sb->sb_cc -= len;
+			if (m->m_type != MT_DATA && m->m_type != MT_HEADER &&
+				m->m_type != MT_OOBDATA)
+				sb->sb_ctl -= len;
 			break;
 		}
 		len -= m->m_len;
@@ -1143,15 +1316,29 @@ sbdrop(sb, len)
 		m = m->m_next;
 	}
 	if (ml) {
-	        ml->m_next = (struct mbuf *)0;
+		ml->m_next = (struct mbuf *)0;
 		last->m_nextpkt = (struct mbuf *)0;
-	        m_freem_list(free_list);
+		m_freem_list(free_list);
 	}
 	if (m) {
 		sb->sb_mb = m;
 		m->m_nextpkt = next;
-	} else
+	} else {
 		sb->sb_mb = next;
+	}
+
+	/*
+	 * First part is an inline SB_EMPTY_FIXUP().  Second part
+	 * makes sure sb_lastrecord is up-to-date if we dropped
+	 * part of the last record.
+	 */
+	m = sb->sb_mb;
+	if (m == NULL) {
+		sb->sb_mbtail = NULL;
+		sb->sb_lastrecord = NULL;
+	} else if (m->m_nextpkt == NULL) {
+		sb->sb_lastrecord = m;
+	}
 
 	postevent(0, sb, EV_RWBYTES);
 
@@ -1163,10 +1350,9 @@ sbdrop(sb, len)
  * and move the next record to the front.
  */
 void
-sbdroprecord(sb)
-	register struct sockbuf *sb;
+sbdroprecord(struct sockbuf *sb)
 {
-	register struct mbuf *m, *mn;
+	struct mbuf *m, *mn;
 
 	m = sb->sb_mb;
 	if (m) {
@@ -1177,6 +1363,7 @@ sbdroprecord(sb)
 			m = mn;
 		} while (m);
 	}
+	SB_EMPTY_FIXUP(sb);
 	postevent(0, sb, EV_RWBYTES);
 }
 
@@ -1185,21 +1372,18 @@ sbdroprecord(sb)
  * with the specified type for presentation on a socket buffer.
  */
 struct mbuf *
-sbcreatecontrol(p, size, type, level)
-	caddr_t p;
-	register int size;
-	int type, level;
+sbcreatecontrol(caddr_t p, int size, int type, int level)
 {
-	register struct cmsghdr *cp;
+	struct cmsghdr *cp;
 	struct mbuf *m;
 
 	if (CMSG_SPACE((u_int)size) > MLEN)
-		return ((struct mbuf *) NULL);
+		return ((struct mbuf *)NULL);
 	if ((m = m_get(M_DONTWAIT, MT_CONTROL)) == NULL)
-		return ((struct mbuf *) NULL);
+		return ((struct mbuf *)NULL);
 	cp = mtod(m, struct cmsghdr *);
 	/* XXX check size? */
-	(void)memcpy(CMSG_DATA(cp), p, size);
+	(void) memcpy(CMSG_DATA(cp), p, size);
 	m->m_len = CMSG_SPACE(size);
 	cp->cmsg_len = CMSG_LEN(size);
 	cp->cmsg_level = level;
@@ -1212,92 +1396,95 @@ sbcreatecontrol(p, size, type, level)
  * supported by a protocol.  Fill in as needed.
  */
 int
-pru_abort_notsupp(struct socket *so)
+pru_abort_notsupp(__unused struct socket *so)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
-
 int
-pru_accept_notsupp(struct socket *so, struct sockaddr **nam)
+pru_accept_notsupp(__unused struct socket *so, __unused struct sockaddr **nam)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_attach_notsupp(struct socket *so, int proto, struct proc *p)
+pru_attach_notsupp(__unused struct socket *so, __unused int proto,
+    __unused struct proc *p)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_bind_notsupp(struct socket *so, struct sockaddr *nam, struct proc *p)
+pru_bind_notsupp(__unused struct socket *so, __unused struct sockaddr *nam,
+    __unused struct proc *p)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_connect_notsupp(struct socket *so, struct sockaddr *nam, struct proc *p)
+pru_connect_notsupp(__unused struct socket *so, __unused struct sockaddr *nam,
+    __unused struct proc *p)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_connect2_notsupp(struct socket *so1, struct socket *so2)
+pru_connect2_notsupp(__unused struct socket *so1, __unused struct socket *so2)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data,
-		    struct ifnet *ifp, struct proc *p)
+pru_control_notsupp(__unused struct socket *so, __unused u_long cmd,
+    __unused caddr_t data, __unused struct ifnet *ifp, __unused struct proc *p)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_detach_notsupp(struct socket *so)
+pru_detach_notsupp(__unused struct socket *so)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_disconnect_notsupp(struct socket *so)
+pru_disconnect_notsupp(__unused struct socket *so)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_listen_notsupp(struct socket *so, struct proc *p)
+pru_listen_notsupp(__unused struct socket *so, __unused struct proc *p)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam)
+pru_peeraddr_notsupp(__unused struct socket *so, __unused struct sockaddr **nam)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_rcvd_notsupp(struct socket *so, int flags)
+pru_rcvd_notsupp(__unused struct socket *so, __unused int flags)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags)
+pru_rcvoob_notsupp(__unused struct socket *so, __unused struct mbuf *m,
+    __unused int flags)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_send_notsupp(struct socket *so, int flags, struct mbuf *m,
-		 struct sockaddr *addr, struct mbuf *control,
-		 struct proc *p)
+pru_send_notsupp(__unused struct socket *so, __unused int flags,
+    __unused struct mbuf *m, __unused struct sockaddr *addr,
+    __unused struct mbuf *control, __unused struct proc *p)
 
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 
@@ -1306,63 +1493,59 @@ pru_send_notsupp(struct socket *so, int flags, struct mbuf *m,
  * and doesn't do anything destructive.
  */
 int
-pru_sense_null(struct socket *so, struct stat *sb)
+pru_sense_null(struct socket *so, void *ub, int isstat64)
 {
-	sb->st_blksize = so->so_snd.sb_hiwat;
-	return 0;
-}
+	if (isstat64 != 0) {
+		struct stat64 *sb64;
 
+		sb64 = (struct stat64 *)ub;
+		sb64->st_blksize = so->so_snd.sb_hiwat;
+	} else {
+		struct stat *sb;
 
-int	pru_sosend_notsupp(struct socket *so, struct sockaddr *addr,
-		   struct uio *uio, struct mbuf *top,
-		   struct mbuf *control, int flags)
+		sb = (struct stat *)ub;
+		sb->st_blksize = so->so_snd.sb_hiwat;
+	}
 
-{
-    return EOPNOTSUPP;
+	return (0);
 }
 
-int	pru_soreceive_notsupp(struct socket *so, 
-		      struct sockaddr **paddr,
-		      struct uio *uio, struct mbuf **mp0,
-		      struct mbuf **controlp, int *flagsp)
-{
-    return EOPNOTSUPP;
-}
 
 int
+pru_sosend_notsupp(__unused struct socket *so, __unused struct sockaddr *addr,
+    __unused struct uio *uio, __unused struct mbuf *top,
+    __unused struct mbuf *control, __unused int flags)
 
-pru_shutdown_notsupp(struct socket *so)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 int
-pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam)
+pru_soreceive_notsupp(__unused struct socket *so,
+    __unused struct sockaddr **paddr,
+    __unused struct uio *uio, __unused struct mbuf **mp0,
+    __unused struct mbuf **controlp, __unused int *flagsp)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
-int     pru_sosend(struct socket *so, struct sockaddr *addr,
-		   struct uio *uio, struct mbuf *top,
-		   struct mbuf *control, int flags)
+int
+pru_shutdown_notsupp(__unused struct socket *so)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
-int     pru_soreceive(struct socket *so,
-		      struct sockaddr **paddr,
-		      struct uio *uio, struct mbuf **mp0,
-		      struct mbuf **controlp, int *flagsp)
+int
+pru_sockaddr_notsupp(__unused struct socket *so, __unused struct sockaddr **nam)
 {
-	return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
-
 int
 pru_sopoll_notsupp(__unused struct socket *so, __unused int events,
-		   __unused kauth_cred_t cred, __unused void *wql)
+    __unused kauth_cred_t cred, __unused void *wql)
 {
-    return EOPNOTSUPP;
+	return (EOPNOTSUPP);
 }
 
 
@@ -1375,10 +1558,11 @@ pru_sopoll_notsupp(__unused struct socket *so, __unused int events,
  * Do we need to notify the other side when I/O is possible?
  */
 
-int 
+int
 sb_notify(struct sockbuf *sb)
 {
-	return ((sb->sb_flags & (SB_WAIT|SB_SEL|SB_ASYNC|SB_UPCALL|SB_KNOTE)) != 0); 
+	return ((sb->sb_flags &
+	    (SB_WAIT|SB_SEL|SB_ASYNC|SB_UPCALL|SB_KNOTE)) != 0);
 }
 
 /*
@@ -1390,24 +1574,24 @@ sb_notify(struct sockbuf *sb)
 long
 sbspace(struct sockbuf *sb)
 {
-    return ((long) imin((int)(sb->sb_hiwat - sb->sb_cc), 
-	 (int)(sb->sb_mbmax - sb->sb_mbcnt)));
+	return ((long)imin((int)(sb->sb_hiwat - sb->sb_cc),
+	    (int)(sb->sb_mbmax - sb->sb_mbcnt)));
 }
 
 /* do we have to send all at once on a socket? */
 int
 sosendallatonce(struct socket *so)
 {
-    return (so->so_proto->pr_flags & PR_ATOMIC);
+	return (so->so_proto->pr_flags & PR_ATOMIC);
 }
 
 /* can we read something from so? */
 int
 soreadable(struct socket *so)
 {
-    return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat || 
-	(so->so_state & SS_CANTRCVMORE) || 
-	so->so_comp.tqh_first || so->so_error);
+	return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat ||
+	    (so->so_state & SS_CANTRCVMORE) ||
+	    so->so_comp.tqh_first || so->so_error);
 }
 
 /* can we write something to so? */
@@ -1415,11 +1599,11 @@ soreadable(struct socket *so)
 int
 sowriteable(struct socket *so)
 {
-    return ((sbspace(&(so)->so_snd) >= (so)->so_snd.sb_lowat && 
-	((so->so_state&SS_ISCONNECTED) || 
-	  (so->so_proto->pr_flags&PR_CONNREQUIRED)==0)) || 
-     (so->so_state & SS_CANTSENDMORE) || 
-     so->so_error);
+	return ((sbspace(&(so)->so_snd) >= (long)(so)->so_snd.sb_lowat &&
+	    ((so->so_state&SS_ISCONNECTED) ||
+	    (so->so_proto->pr_flags&PR_CONNREQUIRED) == 0)) ||
+	    (so->so_state & SS_CANTSENDMORE) ||
+	    so->so_error);
 }
 
 /* adjust counters in sb reflecting allocation of m */
@@ -1427,26 +1611,45 @@ sowriteable(struct socket *so)
 void
 sballoc(struct sockbuf *sb, struct mbuf *m)
 {
+	int cnt = 1;
 	sb->sb_cc += m->m_len; 
+	if (m->m_type != MT_DATA && m->m_type != MT_HEADER && 
+		m->m_type != MT_OOBDATA)
+		sb->sb_ctl += m->m_len;
 	sb->sb_mbcnt += MSIZE; 
-	if (m->m_flags & M_EXT) 
+	
+	if (m->m_flags & M_EXT) {
 		sb->sb_mbcnt += m->m_ext.ext_size; 
+		cnt += m->m_ext.ext_size / MSIZE ;
+	}
+	OSAddAtomic(cnt, (SInt32*)&total_mb_cnt);
 }
 
 /* adjust counters in sb reflecting freeing of m */
 void
 sbfree(struct sockbuf *sb, struct mbuf *m)
 {
-	sb->sb_cc -= m->m_len; 
+	int cnt = -1;
+	sb->sb_cc -= m->m_len;
+	if (m->m_type != MT_DATA && m->m_type != MT_HEADER &&     
+		m->m_type != MT_OOBDATA)
+		sb->sb_ctl -= m->m_len;
 	sb->sb_mbcnt -= MSIZE; 
-	if (m->m_flags & M_EXT) 
+	if (m->m_flags & M_EXT) {
 		sb->sb_mbcnt -= m->m_ext.ext_size; 
+		cnt -= m->m_ext.ext_size / MSIZE ;
+	}
+	OSAddAtomic(cnt, (SInt32*)&total_mb_cnt);
 }
 
 /*
  * Set lock on sockbuf sb; sleep if lock is already held.
  * Unless SB_NOINTR is set on sockbuf, sleep is interruptible.
  * Returns error without lock if sleep is interrupted.
+ *
+ * Returns:	0			Success
+ *		EWOULDBLOCK
+ *	sb_lock:EINTR
  */
 int
 sblock(struct sockbuf *sb, int wf)
@@ -1471,62 +1674,64 @@ sbunlock(struct sockbuf *sb, int keeplocked)
 
 	lr_saved = (unsigned int) __builtin_return_address(0);
 
-	sb->sb_flags &= ~SB_LOCK; 
+	sb->sb_flags &= ~SB_LOCK;
 
-	if (sb->sb_flags & SB_WANT) { 
-		sb->sb_flags &= ~SB_WANT; 
+	if (sb->sb_flags & SB_WANT) {
+		sb->sb_flags &= ~SB_WANT;
 		if (so->so_usecount < 0)
-			panic("sbunlock: b4 wakeup so=%x ref=%d lr=%x sb_flags=%x\n", sb->sb_so, so->so_usecount, lr_saved, sb->sb_flags);
+			panic("sbunlock: b4 wakeup so=%p ref=%d lr=%x "
+			    "sb_flags=%x\n", sb->sb_so, so->so_usecount,
+			    lr_saved, sb->sb_flags);
 
-		wakeup((caddr_t)&(sb)->sb_flags); 
-	} 
+		wakeup((caddr_t)&(sb)->sb_flags);
+	}
 	if (keeplocked == 0) {	/* unlock on exit */
 		if (so->so_proto->pr_getlock != NULL) 
 			mutex_held = (*so->so_proto->pr_getlock)(so, 0);
 		else 
 			mutex_held = so->so_proto->pr_domain->dom_mtx;
-
+	
 		lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
 
 		so->so_usecount--;
 		if (so->so_usecount < 0)
-			panic("sbunlock: unlock on exit so=%x lr=%x sb_flags=%x\n", so, so->so_usecount,lr_saved, sb->sb_flags);
-		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+			panic("sbunlock: unlock on exit so=%p ref=%d lr=%x "
+			    "sb_flags=%x\n", so, so->so_usecount, lr_saved,
+			    sb->sb_flags);
+		so->unlock_lr[so->next_unlock_lr] = (u_int32_t)lr_saved;
 		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(mutex_held);
 	}
 }
 
 void
-sorwakeup(struct socket * so)
+sorwakeup(struct socket *so)
 {
-  if (sb_notify(&so->so_rcv)) 
-	sowakeup(so, &so->so_rcv); 
+	if (sb_notify(&so->so_rcv))
+		sowakeup(so, &so->so_rcv);
 }
 
 void
-sowwakeup(struct socket * so)
+sowwakeup(struct socket *so)
 {
-  if (sb_notify(&so->so_snd)) 
-	sowakeup(so, &so->so_snd); 
+	if (sb_notify(&so->so_snd))
+		sowakeup(so, &so->so_snd);
 }
-#endif __APPLE__
+#endif /* __APPLE__ */
 
 /*
  * Make a copy of a sockaddr in a malloced buffer of type M_SONAME.
  */
 struct sockaddr *
-dup_sockaddr(sa, canwait)
-	struct sockaddr *sa;
-	int canwait;
+dup_sockaddr(struct sockaddr *sa, int canwait)
 {
 	struct sockaddr *sa2;
 
-	MALLOC(sa2, struct sockaddr *, sa->sa_len, M_SONAME, 
-	       canwait ? M_WAITOK : M_NOWAIT);
+	MALLOC(sa2, struct sockaddr *, sa->sa_len, M_SONAME,
+	    canwait ? M_WAITOK : M_NOWAIT);
 	if (sa2)
 		bcopy(sa, sa2, sa->sa_len);
-	return sa2;
+	return (sa2);
 }
 
 /*
@@ -1540,7 +1745,7 @@ dup_sockaddr(sa, canwait)
 void
 sotoxsocket(struct socket *so, struct xsocket *xso)
 {
-	xso->xso_len = sizeof *xso;
+	xso->xso_len = sizeof (*xso);
 	xso->xso_so = so;
 	xso->so_type = so->so_type;
 	xso->so_options = so->so_options;
@@ -1550,9 +1755,9 @@ sotoxsocket(struct socket *so, struct xsocket *xso)
 	if (so->so_proto) {
 		xso->xso_protocol = so->so_proto->pr_protocol;
 		xso->xso_family = so->so_proto->pr_domain->dom_family;
-	}
-	else 
+	} else {
 		xso->xso_protocol = xso->xso_family = 0;
+	}
 	xso->so_qlen = so->so_qlen;
 	xso->so_incqlen = so->so_incqlen;
 	xso->so_qlimit = so->so_qlimit;
@@ -1580,7 +1785,8 @@ sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
 	xsb->sb_mbmax = sb->sb_mbmax;
 	xsb->sb_lowat = sb->sb_lowat;
 	xsb->sb_flags = sb->sb_flags;
-	xsb->sb_timeo = (u_long)(sb->sb_timeo.tv_sec * hz) + sb->sb_timeo.tv_usec / tick;
+	xsb->sb_timeo = (u_long)
+	    (sb->sb_timeo.tv_sec * hz) + sb->sb_timeo.tv_usec / tick;
 	if (xsb->sb_timeo == 0 && sb->sb_timeo.tv_usec != 0)
 		xsb->sb_timeo = 1;
 }
@@ -1589,17 +1795,25 @@ sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb)
  * Here is the definition of some of the basic objects in the kern.ipc
  * branch of the MIB.
  */
-SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC");
+SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "IPC");
 
 /* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */
 static int dummy;
 SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, "");
 
-SYSCTL_INT(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLFLAG_RW, 
+SYSCTL_INT(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLFLAG_RW,
     &sb_max, 0, "Maximum socket buffer size");
-SYSCTL_INT(_kern_ipc, OID_AUTO, maxsockets, CTLFLAG_RD, 
+SYSCTL_INT(_kern_ipc, OID_AUTO, maxsockets, CTLFLAG_RD,
     &maxsockets, 0, "Maximum number of sockets avaliable");
 SYSCTL_INT(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW,
-	   &sb_efficiency, 0, "");
-SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD, &nmbclusters, 0, "");
-
+    &sb_efficiency, 0, "");
+SYSCTL_INT(_kern_ipc, OID_AUTO, sbspace_factor, CTLFLAG_RW,
+    &sbspace_factor, 0, "Ratio of mbuf/cluster use for socket layers");
+SYSCTL_INT(_kern_ipc, KIPC_NMBCLUSTERS, nmbclusters, CTLFLAG_RD,
+    &nmbclusters, 0, "");
+SYSCTL_INT(_kern_ipc, OID_AUTO, njcl, CTLFLAG_RD, &njcl, 0, "");
+SYSCTL_INT(_kern_ipc, OID_AUTO, njclbytes, CTLFLAG_RD, &njclbytes, 0, "");
+SYSCTL_INT(_kern_ipc, KIPC_SOQLIMITCOMPAT, soqlimitcompat, CTLFLAG_RW,
+    &soqlimitcompat, 1, "Enable socket queue limit compatibility");
+SYSCTL_INT(_kern_ipc, OID_AUTO, soqlencomp, CTLFLAG_RW,
+    &soqlencomp, 0, "Listen backlog represents only complete queue");
diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c
index b3a9fcac2..1126e7955 100644
--- a/bsd/kern/uipc_syscalls.c
+++ b/bsd/kern/uipc_syscalls.c
@@ -1,30 +1,36 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
  * sendfile(2) and related extensions:
- * Copyright (c) 1998, David Greenman. All rights reserved. 
+ * Copyright (c) 1998, David Greenman. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -56,14 +62,19 @@
  *
  *	@(#)uipc_syscalls.c	8.4 (Berkeley) 2/21/94
  */
-
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/filedesc.h>
 #include <sys/proc_internal.h>
 #include <sys/file_internal.h>
+#include <sys/vnode_internal.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <kern/lock.h>
@@ -72,104 +83,67 @@
 #include <sys/signalvar.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
-#if KTRACE
-#include <sys/ktrace.h>
-#endif
 #include <sys/kernel.h>
 #include <sys/uio_internal.h>
+#include <sys/kauth.h>
 
 #include <bsm/audit_kernel.h>
 
 #include <sys/kdebug.h>
 #include <sys/sysproto.h>
-
-#define f_flag f_fglob->fg_flag
-#define f_type f_fglob->fg_type
-#define f_msgcount f_fglob->fg_msgcount
-#define f_cred f_fglob->fg_cred
-#define f_ops f_fglob->fg_ops
-#define f_offset f_fglob->fg_offset
-#define f_data f_fglob->fg_data
-#if KDEBUG
-
-#define DBG_LAYER_IN_BEG	NETDBG_CODE(DBG_NETSOCK, 0)
-#define DBG_LAYER_IN_END	NETDBG_CODE(DBG_NETSOCK, 2)
-#define DBG_LAYER_OUT_BEG	NETDBG_CODE(DBG_NETSOCK, 1)
-#define DBG_LAYER_OUT_END	NETDBG_CODE(DBG_NETSOCK, 3)
-#define DBG_FNC_SENDMSG		NETDBG_CODE(DBG_NETSOCK, (1 << 8) | 1)
-#define DBG_FNC_SENDTO		NETDBG_CODE(DBG_NETSOCK, (2 << 8) | 1)
-#define DBG_FNC_SENDIT		NETDBG_CODE(DBG_NETSOCK, (3 << 8) | 1)
-#define DBG_FNC_RECVFROM	NETDBG_CODE(DBG_NETSOCK, (5 << 8))
-#define DBG_FNC_RECVMSG		NETDBG_CODE(DBG_NETSOCK, (6 << 8))
-#define DBG_FNC_RECVIT		NETDBG_CODE(DBG_NETSOCK, (7 << 8))
-
-#endif
-
-
-#define HACK_FOR_4056224 1
+#include <netinet/in.h>
+#include <net/route.h>
+#include <netinet/in_pcb.h>
+
+#if CONFIG_MACF_SOCKET_SUBSET
+#include <security/mac_framework.h>
+#endif /* MAC_SOCKET_SUBSET */
+
+#define	f_flag f_fglob->fg_flag
+#define	f_type f_fglob->fg_type
+#define	f_msgcount f_fglob->fg_msgcount
+#define	f_cred f_fglob->fg_cred
+#define	f_ops f_fglob->fg_ops
+#define	f_offset f_fglob->fg_offset
+#define	f_data f_fglob->fg_data
+
+
+#define	DBG_LAYER_IN_BEG	NETDBG_CODE(DBG_NETSOCK, 0)
+#define	DBG_LAYER_IN_END	NETDBG_CODE(DBG_NETSOCK, 2)
+#define	DBG_LAYER_OUT_BEG	NETDBG_CODE(DBG_NETSOCK, 1)
+#define	DBG_LAYER_OUT_END	NETDBG_CODE(DBG_NETSOCK, 3)
+#define	DBG_FNC_SENDMSG		NETDBG_CODE(DBG_NETSOCK, (1 << 8) | 1)
+#define	DBG_FNC_SENDTO		NETDBG_CODE(DBG_NETSOCK, (2 << 8) | 1)
+#define	DBG_FNC_SENDIT		NETDBG_CODE(DBG_NETSOCK, (3 << 8) | 1)
+#define	DBG_FNC_RECVFROM	NETDBG_CODE(DBG_NETSOCK, (5 << 8))
+#define	DBG_FNC_RECVMSG		NETDBG_CODE(DBG_NETSOCK, (6 << 8))
+#define	DBG_FNC_RECVIT		NETDBG_CODE(DBG_NETSOCK, (7 << 8))
+#define	DBG_FNC_SENDFILE	NETDBG_CODE(DBG_NETSOCK, (10 << 8))
+#define	DBG_FNC_SENDFILE_WAIT	NETDBG_CODE(DBG_NETSOCK, ((10 << 8) | 1))
+#define	DBG_FNC_SENDFILE_READ	NETDBG_CODE(DBG_NETSOCK, ((10 << 8) | 2))
+#define	DBG_FNC_SENDFILE_SEND	NETDBG_CODE(DBG_NETSOCK, ((10 << 8) | 3))
+
+
+#define	HACK_FOR_4056224 1
 #if HACK_FOR_4056224
 static pid_t last_pid_4056224 = 0;
 #endif /* HACK_FOR_4056224 */
 
-
+/* TODO: should be in header file */
+int falloc_locked(proc_t, struct fileproc **, int *, vfs_context_t, int);
+
+static int sendit(struct proc *, int, struct user_msghdr *, uio_t, int,
+    register_t *);
+static int recvit(struct proc *, int, struct user_msghdr *, uio_t, user_addr_t,
+    register_t *);
+static int getsockaddr(struct socket *, struct sockaddr **, user_addr_t,
+    size_t);
+static int getsockaddr_s(struct socket *, struct sockaddr_storage *,
+    user_addr_t, size_t);
 #if SENDFILE
-static void sf_buf_init(void *arg);
-SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL)
-static struct sf_buf *sf_buf_alloc(void);
-static void sf_buf_ref(caddr_t addr, u_int size);
-static void sf_buf_free(caddr_t addr, u_int size);
-
-static SLIST_HEAD(, sf_buf) sf_freelist;
-static vm_offset_t sf_base;
-static struct sf_buf *sf_bufs;
-static int sf_buf_alloc_want;
-#endif
-
-static int sendit(struct proc *p, int s, struct user_msghdr *mp, uio_t uiop, 
-					int flags, register_t *retval);
-static int recvit(struct proc *p, int s, struct user_msghdr *mp, uio_t uiop,
-					user_addr_t namelenp, register_t *retval);
-  
-static int accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat);
-static int getsockname1(struct proc *p, struct getsockname_args *uap,
-			     register_t *retval, int compat);
-static int getpeername1(struct proc *p, struct getpeername_args *uap,
-			     register_t *retval, int compat);
-
-
-#if COMPAT_43_SOCKET
-struct orecvmsg_args  {
-	int	s;
-	struct	omsghdr *msg;
-	int	flags;
-};
-struct osendmsg_args {
-	int s;
-	caddr_t msg;
-	int flags;
-};
-struct osend_args {
-	int s;
-	caddr_t buf;
-	int len;
-	int flags;
-};
-struct	orecv_args {
-	int	s;
-	caddr_t	buf;
-	int	len;
-	int	flags;
-};
-
-int oaccept(struct proc *p, struct accept_args *uap, register_t *retval);
-int ogetpeername(struct proc *p, struct getpeername_args *uap, register_t *retval);
-int ogetsockname(struct proc *p, struct getsockname_args *uap, register_t *retval);
-int orecv(struct proc *p, struct orecv_args	*uap, register_t *retval);
-int orecvfrom(struct proc *p, struct recvfrom_args *uap, register_t *retval);
-int orecvmsg(struct proc *p, struct orecvmsg_args *uap, register_t *retval);
-int	osend(struct proc *p, struct osend_args *uap, register_t *retval);
-int osendmsg(struct proc *p, struct osendmsg_args *uap, register_t *retval);
-#endif // COMPAT_43_SOCKET
+static void alloc_sendpkt(int, size_t, unsigned int *, struct mbuf **,
+    boolean_t);
+#endif /* SENDFILE */
 
 /*
  * System call interface to the socket abstraction.
@@ -177,19 +151,35 @@ int osendmsg(struct proc *p, struct osendmsg_args *uap, register_t *retval);
 
 extern	struct fileops socketops;
 
+/*
+ * Returns:	0			Success
+ *		EACCES			Mandatory Access Control failure
+ *	falloc:ENFILE
+ *	falloc:EMFILE
+ *	falloc:ENOMEM
+ *	socreate:EAFNOSUPPORT
+ *	socreate:EPROTOTYPE
+ *	socreate:EPROTONOSUPPORT
+ *	socreate:ENOBUFS
+ *	socreate:ENOMEM
+ *	socreate:EISCONN
+ *	socreate:???			[other protocol families, IPSEC]
+ */
 int
-socket(p, uap, retval)
-	struct proc *p;
-	register struct socket_args *uap;
-	register_t *retval;
+socket(struct proc *p, struct socket_args *uap, register_t *retval)
 {
 	struct socket *so;
 	struct fileproc *fp;
 	int fd, error;
 
 	AUDIT_ARG(socket, uap->domain, uap->type, uap->protocol);
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_create(kauth_cred_get(), uap->domain,
+	    uap->type, uap->protocol)) != 0)
+		return (error);
+#endif /* MAC_SOCKET_SUBSET */
 
-	error = falloc(p, &fp, &fd);
+	error = falloc(p, &fp, &fd, vfs_context_current());
 	if (error) {
 		return (error);
 	}
@@ -201,11 +191,22 @@ socket(p, uap, retval)
 	if (error) {
 		fp_free(p, fd, fp);
 	} else {
+		thread_t			thread;
+		struct uthread		*ut;
+		
+		thread = current_thread();
+		ut = get_bsdthread_info(thread);
+			
+		/* if this is a backgrounded thread then throttle all new sockets */
+		if ( (ut->uu_flag & UT_BACKGROUND) != 0 ) {
+			so->so_traffic_mgt_flags |= TRAFFIC_MGT_SO_BACKGROUND;
+			so->so_background_thread = thread;
+		}
 		fp->f_data = (caddr_t)so;
 
 		proc_fdlock(p);
 		procfdtbl_releasefd(p, fd, NULL);
-		
+
 		fp_drop(p, fd, fp, 1);
 		proc_fdunlock(p);
 
@@ -214,77 +215,144 @@ socket(p, uap, retval)
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EDESTADDRREQ		Destination address required
+ *		EBADF			Bad file descriptor
+ *		EACCES			Mandatory Access Control failure
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	getsockaddr:ENAMETOOLONG	Filename too long
+ *	getsockaddr:EINVAL		Invalid argument
+ *	getsockaddr:ENOMEM		Not enough space
+ *	getsockaddr:EFAULT		Bad address
+ *	sobind:???
+ */
 /* ARGSUSED */
 int
-bind(struct proc *p, struct bind_args *uap, __unused register_t *retval)
+bind(__unused proc_t p, struct bind_args *uap, __unused register_t *retval)
 {
-	struct sockaddr *sa;
+	struct sockaddr_storage ss;
+	struct sockaddr *sa = NULL;
 	struct socket *so;
+	boolean_t want_free = TRUE;
 	int error;
 
 	AUDIT_ARG(fd, uap->s);
 	error = file_socket(uap->s, &so);
-	if (error)
+	if (error != 0)
 		return (error);
-	error = getsockaddr(&sa, uap->name, uap->namelen);
-	if (error) 
+	if (so == NULL) {
+		error = EBADF;
+		goto out;
+	}
+	if (uap->name == USER_ADDR_NULL) {
+		error = EDESTADDRREQ;
+		goto out;
+	}
+	if (uap->namelen > sizeof (ss)) {
+		error = getsockaddr(so, &sa, uap->name, uap->namelen);
+	} else {
+		error = getsockaddr_s(so, &ss, uap->name, uap->namelen);
+		if (error == 0) {
+			sa = (struct sockaddr *)&ss;
+			want_free = FALSE;
+		}
+	}
+	if (error != 0)
 		goto out;
-	AUDIT_ARG(sockaddr, p, sa);
-	if (so != NULL)	
+	AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa);
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_bind(kauth_cred_get(), so, sa)) == 0)
 		error = sobind(so, sa);
-	else
-		error = EBADF;
-	FREE(sa, M_SONAME);
+#else
+		error = sobind(so, sa);
+#endif /* MAC_SOCKET_SUBSET */
+	if (want_free)
+		FREE(sa, M_SONAME);
 out:
 	file_drop(uap->s);
 	return (error);
 }
 
-
+/*
+ * Returns:	0			Success
+ *		EBADF
+ *		EACCES			Mandatory Access Control failure
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	solisten:EINVAL
+ *	solisten:EOPNOTSUPP
+ *	solisten:???
+ */
 int
-listen(__unused struct proc *p, register struct listen_args *uap, 
-		__unused register_t *retval)
+listen(__unused struct proc *p, struct listen_args *uap,
+    __unused register_t *retval)
 {
 	int error;
-	struct socket * so;
+	struct socket *so;
 
 	AUDIT_ARG(fd, uap->s);
 	error = file_socket(uap->s, &so);
 	if (error)
 		return (error);
 	if (so != NULL)
+#if CONFIG_MACF_SOCKET_SUBSET
+	{
+		error = mac_socket_check_listen(kauth_cred_get(), so);
+		if (error == 0)
+			error = solisten(so, uap->backlog);
+	}
+#else
 		error =  solisten(so, uap->backlog);
+#endif /* MAC_SOCKET_SUBSET */
 	else
 		error = EBADF;
+
 	file_drop(uap->s);
 	return (error);
 }
 
-#if !COMPAT_43_SOCKET
-#define	accept1	accept
-#endif
-
-
-
+/*
+ * Returns:	fp_getfsock:EBADF	Bad file descriptor
+ *		fp_getfsock:EOPNOTSUPP	...
+ *		xlate => :ENOTSOCK	Socket operation on non-socket
+ *		:EFAULT			Bad address on copyin/copyout
+ *		:EBADF			Bad file descriptor
+ *		:EOPNOTSUPP		Operation not supported on socket
+ *		:EINVAL			Invalid argument
+ *		:EWOULDBLOCK		Operation would block
+ *		:ECONNABORTED		Connection aborted
+ *		:EINTR			Interrupted function
+ *		:EACCES			Mandatory Access Control failure
+ *		falloc_locked:ENFILE	Too many files open in system
+ *		falloc_locked::EMFILE	Too many open files
+ *		falloc_locked::ENOMEM	Not enough space
+ *		0			Success
+ */
 int
-accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat)
+accept_nocancel(struct proc *p, struct accept_nocancel_args *uap,
+    register_t *retval)
 {
 	struct fileproc *fp;
-	struct sockaddr *sa;
+	struct sockaddr *sa = NULL;
 	socklen_t namelen;
 	int error;
 	struct socket *head, *so = NULL;
 	lck_mtx_t *mutex_held;
 	int fd = uap->s;
-	int newfd;;
+	int newfd;
 	short fflag;		/* type must match fp->f_flag */
 	int dosocklock = 0;
 
+	*retval = -1;
+
 	AUDIT_ARG(fd, uap->s);
+
 	if (uap->name) {
 		error = copyin(uap->anamelen, (caddr_t)&namelen,
-			sizeof(socklen_t));
-		if(error)
+		    sizeof (socklen_t));
+		if (error)
 			return (error);
 	}
 	error = fp_getfsock(p, fd, &fp, &head);
@@ -297,22 +365,29 @@ accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat)
 		error = EBADF;
 		goto out;
 	}
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_accept(kauth_cred_get(), head)) != 0)
+		goto out;
+#endif /* MAC_SOCKET_SUBSET */
 
 	socket_lock(head, 1);
 
 	if (head->so_proto->pr_getlock != NULL)  {
 		mutex_held = (*head->so_proto->pr_getlock)(head, 0);
 		dosocklock = 1;
-	}
-	else {
+	} else {
 		mutex_held = head->so_proto->pr_domain->dom_mtx;
 		dosocklock = 0;
 	}
 
-
 	if ((head->so_options & SO_ACCEPTCONN) == 0) {
+		if ((head->so_proto->pr_flags & PR_CONNREQUIRED) == 0) {
+			error = EOPNOTSUPP;
+		} else {
+			/* POSIX: The socket is not accepting connections */
+			error = EINVAL;
+		}
 		socket_unlock(head, 1);
-		error = EINVAL;
 		goto out;
 	}
 	if ((head->so_state & SS_NBIO) && head->so_comp.tqh_first == NULL) {
@@ -320,17 +395,19 @@ accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat)
 		error = EWOULDBLOCK;
 		goto out;
 	}
-        while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
+	while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) {
 		if (head->so_state & SS_CANTRCVMORE) {
 			head->so_error = ECONNABORTED;
 			break;
 		}
 		if (head->so_usecount < 1)
-			panic("accept1: head=%x refcount=%d\n", head, head->so_usecount);
-		error = msleep((caddr_t)&head->so_timeo, mutex_held, PSOCK | PCATCH,
-		    "accept", 0);
+			panic("accept: head=%p refcount=%d\n", head,
+			    head->so_usecount);
+		error = msleep((caddr_t)&head->so_timeo, mutex_held,
+		    PSOCK | PCATCH, "accept", 0);
 		if (head->so_usecount < 1)
-			panic("accept1: 2 head=%x refcount=%d\n", head, head->so_usecount);
+			panic("accept: 2 head=%p refcount=%d\n", head,
+			    head->so_usecount);
 		if ((head->so_state & SS_DRAINING)) {
 			error = ECONNABORTED;
 		}
@@ -358,10 +435,39 @@ accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat)
 	so = TAILQ_FIRST(&head->so_comp);
 	TAILQ_REMOVE(&head->so_comp, so, so_list);
 	head->so_qlen--;
-	socket_unlock(head, 0); /* unlock head to avoid deadlock with select, keep a ref on head */
+	/* unlock head to avoid deadlock with select, keep a ref on head */
+	socket_unlock(head, 0);
+
+#if CONFIG_MACF_SOCKET_SUBSET
+	/*
+	 * Pass the pre-accepted socket to the MAC framework. This is
+	 * cheaper than allocating a file descriptor for the socket,
+	 * calling the protocol accept callback, and possibly freeing
+	 * the file descriptor should the MAC check fails.
+	 */
+	if ((error = mac_socket_check_accepted(kauth_cred_get(), so)) != 0) {
+		so->so_state &= ~(SS_NOFDREF | SS_COMP);
+		so->so_head = NULL;
+		soclose(so);
+		/* Drop reference on listening socket */
+		sodereference(head);
+		goto out;
+	}
+#endif /* MAC_SOCKET_SUBSET */
+
+	/*
+	 * Pass the pre-accepted socket to any interested socket filter(s).
+	 * Upon failure, the socket would have been closed by the callee.
+	 */
+	if (so->so_filt != NULL && (error = soacceptfilter(so)) != 0) {
+		/* Drop reference on listening socket */
+		sodereference(head);
+		/* Propagate socket filter's error code to the caller */
+		goto out;
+	}
+
 	fflag = fp->f_flag;
-	proc_fdlock(p);
-	error = falloc_locked(p, &fp, &newfd, 1);
+	error = falloc(p, &fp, &newfd, vfs_context_current());
 	if (error) {
 		/*
 		 * Probably ran out of file descriptors. Put the
@@ -369,58 +475,74 @@ accept1(struct proc *p, struct accept_args *uap, register_t *retval, int compat)
 		 * do another wakeup so some other process might
 		 * have a chance at it.
 		 */
-		proc_fdunlock(p);
 		socket_lock(head, 0);
 		TAILQ_INSERT_HEAD(&head->so_comp, so, so_list);
 		head->so_qlen++;
 		wakeup_one((caddr_t)&head->so_timeo);
 		socket_unlock(head, 1);
 		goto out;
-	} 
-	procfdtbl_releasefd(p, newfd, NULL);
+	}
 	*retval = newfd;
 	fp->f_type = DTYPE_SOCKET;
 	fp->f_flag = fflag;
 	fp->f_ops = &socketops;
 	fp->f_data = (caddr_t)so;
-	fp_drop(p, newfd, fp, 1);
-	proc_fdunlock(p);
 	socket_lock(head, 0);
 	if (dosocklock)
 		socket_lock(so, 1);
 	so->so_state &= ~SS_COMP;
 	so->so_head = NULL;
-	sa = 0;
 	(void) soacceptlock(so, &sa, 0);
 	socket_unlock(head, 1);
-	if (sa == 0) {
+	if (sa == NULL) {
 		namelen = 0;
 		if (uap->name)
 			goto gotnoname;
 		if (dosocklock)
 			socket_unlock(so, 1);
 		error = 0;
-		goto out;
+		goto releasefd;
 	}
-	AUDIT_ARG(sockaddr, p, sa);
+	AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa);
+
 	if (uap->name) {
-		/* check sa_len before it is destroyed */
-		if (namelen > sa->sa_len)
-			namelen = sa->sa_len;
-#if COMPAT_43_SOCKET
-		if (compat)
-			((struct osockaddr *)sa)->sa_family =
-			    sa->sa_family;
-#endif
+		socklen_t	sa_len;
+
+		/* save sa_len before it is destroyed */
+		sa_len = sa->sa_len;
+		namelen = MIN(namelen, sa_len);
 		error = copyout(sa, uap->name, namelen);
 		if (!error)
+			/* return the actual, untruncated address length */
+			namelen = sa_len;
 gotnoname:
-			error = copyout((caddr_t)&namelen, uap->anamelen, 
-			    			sizeof(socklen_t));
+		error = copyout((caddr_t)&namelen, uap->anamelen,
+		    sizeof (socklen_t));
 	}
 	FREE(sa, M_SONAME);
+
+	/*
+	 * If the socket has been marked as inactive by soacceptfilter(),
+	 * disallow further operations on it.  We explicitly call shutdown
+	 * on both data directions to ensure that SS_CANT{RCV,SEND}MORE
+	 * states are set for the socket.  This would also flush out data
+	 * hanging off the receive list of this socket.
+	 */
+	if (so->so_flags & SOF_DEFUNCT) {
+		(void) soshutdownlock(so, SHUT_RD);
+		(void) soshutdownlock(so, SHUT_WR);
+		(void) sodisconnectlocked(so);
+	}
+
 	if (dosocklock)
 		socket_unlock(so, 1);
+
+releasefd:
+	proc_fdlock(p);
+	procfdtbl_releasefd(p, newfd, NULL);
+	fp_drop(p, newfd, fp, 1);
+	proc_fdunlock(p);
+
 out:
 	file_drop(fd);
 	return (error);
@@ -429,67 +551,107 @@ out:
 int
 accept(struct proc *p, struct accept_args *uap, register_t *retval)
 {
-
-	return (accept1(p, uap, retval, 0));
+	__pthread_testcancel(1);
+	return(accept_nocancel(p, (struct accept_nocancel_args *)uap, retval));
 }
 
-#if COMPAT_43_SOCKET
+/*
+ * Returns:	0			Success
+ *		EBADF			Bad file descriptor
+ *		EALREADY		Connection already in progress
+ *		EINPROGRESS		Operation in progress
+ *		ECONNABORTED		Connection aborted
+ *		EINTR			Interrupted function
+ *		EACCES			Mandatory Access Control failure
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	getsockaddr:ENAMETOOLONG	Filename too long
+ *	getsockaddr:EINVAL		Invalid argument
+ *	getsockaddr:ENOMEM		Not enough space
+ *	getsockaddr:EFAULT		Bad address
+ *	soconnectlock:EOPNOTSUPP
+ *	soconnectlock:EISCONN
+ *	soconnectlock:???		[depends on protocol, filters]
+ *	msleep:EINTR
+ *
+ * Imputed:	so_error		error may be set from so_error, which
+ *					may have been set by soconnectlock.
+ */
+/* ARGSUSED */
 int
-oaccept(struct proc *p, struct accept_args *uap, register_t *retval)
+connect(struct proc *p, struct connect_args *uap, register_t *retval)
 {
-
-	return (accept1(p, uap, retval, 1));
+	__pthread_testcancel(1);
+	return(connect_nocancel(p, (struct connect_nocancel_args *)uap, retval));
 }
-#endif /* COMPAT_43_SOCKET */
 
-/* ARGSUSED */
 int
-connect(struct proc *p, struct connect_args *uap, __unused register_t *retval)
+connect_nocancel(__unused proc_t p, struct connect_nocancel_args *uap, __unused register_t *retval)
 {
 	struct socket *so;
-	struct sockaddr *sa;
+	struct sockaddr_storage ss;
+	struct sockaddr *sa = NULL;
 	lck_mtx_t *mutex_held;
+	boolean_t want_free = TRUE;
 	int error;
 	int fd = uap->s;
 
 	AUDIT_ARG(fd, uap->s);
-	error = file_socket( fd, &so);
-	if (error)
+	error = file_socket(fd, &so);
+	if (error != 0)
 		return (error);
 	if (so == NULL) {
 		error = EBADF;
 		goto out;
 	}
 
+	/* Get socket address now before we obtain socket lock */
+	if (uap->namelen > sizeof (ss)) {
+		error = getsockaddr(so, &sa, uap->name, uap->namelen);
+	} else {
+		error = getsockaddr_s(so, &ss, uap->name, uap->namelen);
+		if (error == 0) {
+			sa = (struct sockaddr *)&ss;
+			want_free = FALSE;
+		}
+	}
+	if (error != 0)
+		goto out;
+
+	AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), sa);
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_connect(kauth_cred_get(), so, sa)) != 0) {
+		if (want_free)
+			FREE(sa, M_SONAME);
+		goto out;
+	}
+#endif /* MAC_SOCKET_SUBSET */
 	socket_lock(so, 1);
 
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
+		if (want_free)
+			FREE(sa, M_SONAME);
 		socket_unlock(so, 1);
 		error = EALREADY;
 		goto out;
 	}
-	error = getsockaddr(&sa, uap->name, uap->namelen);
-	if (error)  {
-		socket_unlock(so, 1);
-		goto out;
-	}
-	AUDIT_ARG(sockaddr, p, sa);
 	error = soconnectlock(so, sa, 0);
 	if (error)
 		goto bad;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
-		FREE(sa, M_SONAME);
+		if (want_free)
+			FREE(sa, M_SONAME);
 		socket_unlock(so, 1);
 		error = EINPROGRESS;
 		goto out;
 	}
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		if (so->so_proto->pr_getlock != NULL) 
+		if (so->so_proto->pr_getlock != NULL)
 			mutex_held = (*so->so_proto->pr_getlock)(so, 0);
-		else 
+		else
 			mutex_held = so->so_proto->pr_domain->dom_mtx;
-		error = msleep((caddr_t)&so->so_timeo, mutex_held, PSOCK | PCATCH,
-		    "connec", 0);
+		error = msleep((caddr_t)&so->so_timeo, mutex_held,
+		    PSOCK | PCATCH, "connect", 0);
 		if ((so->so_state & SS_DRAINING)) {
 			error = ECONNABORTED;
 		}
@@ -503,7 +665,8 @@ connect(struct proc *p, struct connect_args *uap, __unused register_t *retval)
 bad:
 	so->so_state &= ~SS_ISCONNECTING;
 	socket_unlock(so, 1);
-	FREE(sa, M_SONAME);
+	if (want_free)
+		FREE(sa, M_SONAME);
 	if (error == ERESTART)
 		error = EINTR;
 out:
@@ -511,8 +674,26 @@ out:
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *	socreate:EAFNOSUPPORT
+ *	socreate:EPROTOTYPE
+ *	socreate:EPROTONOSUPPORT
+ *	socreate:ENOBUFS
+ *	socreate:ENOMEM
+ *	socreate:EISCONN
+ *	socreate:???			[other protocol families, IPSEC]
+ *	falloc:ENFILE
+ *	falloc:EMFILE
+ *	falloc:ENOMEM
+ *	copyout:EFAULT
+ *	soconnect2:EINVAL
+ *	soconnect2:EPROTOTYPE
+ *	soconnect2:???			[other protocol families[
+ */
 int
-socketpair(struct proc *p, struct socketpair_args *uap, __unused register_t *retval)
+socketpair(struct proc *p, struct socketpair_args *uap,
+    __unused register_t *retval)
 {
 	struct fileproc *fp1, *fp2;
 	struct socket *so1, *so2;
@@ -526,7 +707,7 @@ socketpair(struct proc *p, struct socketpair_args *uap, __unused register_t *ret
 	if (error)
 		goto free1;
 
-	error = falloc(p, &fp1, &fd);
+	error = falloc(p, &fp1, &fd, vfs_context_current());
 	if (error) {
 		goto free2;
 	}
@@ -536,7 +717,7 @@ socketpair(struct proc *p, struct socketpair_args *uap, __unused register_t *ret
 	fp1->f_data = (caddr_t)so1;
 	sv[0] = fd;
 
-	error = falloc(p, &fp2, &fd);
+	error = falloc(p, &fp2, &fd, vfs_context_current());
 	if (error) {
 		goto free3;
 	}
@@ -554,10 +735,10 @@ socketpair(struct proc *p, struct socketpair_args *uap, __unused register_t *ret
 		/*
 		 * Datagram socket connection is asymmetric.
 		 */
-		 error = soconnect2(so2, so1);
-		 if (error) {
-			 goto free4;
-		 }
+		error = soconnect2(so2, so1);
+		if (error) {
+			goto free4;
+		}
 	}
 
 	proc_fdlock(p);
@@ -567,142 +748,168 @@ socketpair(struct proc *p, struct socketpair_args *uap, __unused register_t *ret
 	fp_drop(p, sv[1], fp2, 1);
 	proc_fdunlock(p);
 
-	error = copyout((caddr_t)sv, uap->rsv, 2 * sizeof(int));
-#if 0   /* old pipe(2) syscall compatability, unused these days */
-	retval[0] = sv[0];		/* XXX ??? */
-	retval[1] = sv[1];		/* XXX ??? */
-#endif /* 0 */
+	error = copyout((caddr_t)sv, uap->rsv, 2 * sizeof (int));
 	return (error);
 free4:
 	fp_free(p, sv[1], fp2);
 free3:
 	fp_free(p, sv[0], fp1);
 free2:
-	(void)soclose(so2);
+	(void) soclose(so2);
 free1:
-	(void)soclose(so1);
+	(void) soclose(so1);
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		ENOBUFS
+ *		EBADF
+ *		EPIPE
+ *		EACCES			Mandatory Access Control failure
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	getsockaddr:ENAMETOOLONG	Filename too long
+ *	getsockaddr:EINVAL		Invalid argument
+ *	getsockaddr:ENOMEM		Not enough space
+ *	getsockaddr:EFAULT		Bad address
+ *	<pru_sosend>:EACCES[TCP]
+ *	<pru_sosend>:EADDRINUSE[TCP]
+ *	<pru_sosend>:EADDRNOTAVAIL[TCP]
+ *	<pru_sosend>:EAFNOSUPPORT[TCP]
+ *	<pru_sosend>:EAGAIN[TCP]
+ *	<pru_sosend>:EBADF
+ *	<pru_sosend>:ECONNRESET[TCP]
+ *	<pru_sosend>:EFAULT
+ *	<pru_sosend>:EHOSTUNREACH[TCP]
+ *	<pru_sosend>:EINTR
+ *	<pru_sosend>:EINVAL
+ *	<pru_sosend>:EISCONN[AF_INET]
+ *	<pru_sosend>:EMSGSIZE[TCP]
+ *	<pru_sosend>:ENETDOWN[TCP]
+ *	<pru_sosend>:ENETUNREACH[TCP]
+ *	<pru_sosend>:ENOBUFS
+ *	<pru_sosend>:ENOMEM[TCP]
+ *	<pru_sosend>:ENOTCONN[AF_INET]
+ *	<pru_sosend>:EOPNOTSUPP
+ *	<pru_sosend>:EPERM[TCP]
+ *	<pru_sosend>:EPIPE
+ *	<pru_sosend>:EWOULDBLOCK
+ *	<pru_sosend>:???[TCP]		[ignorable: mostly IPSEC/firewall/DLIL]
+ *	<pru_sosend>:???[AF_INET]	[whatever a filter author chooses]
+ *	<pru_sosend>:???		[value from so_error]
+ *	sockargs:???
+ */
 static int
-sendit(struct proc *p, int s, struct user_msghdr *mp, uio_t uiop, 
-		int flags, register_t *retval)
+sendit(struct proc *p, int s, struct user_msghdr *mp, uio_t uiop,
+    int flags, register_t *retval)
 {
-	struct mbuf *control;
-	struct sockaddr *to;
+	struct mbuf *control = NULL;
+	struct sockaddr_storage ss;
+	struct sockaddr *to = NULL;
+	boolean_t want_free = TRUE;
 	int error;
 	struct socket *so;
 	user_ssize_t len;
-#if KTRACE
-	uio_t ktruio = NULL;
-#endif
-	
-	KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_START, 0,0,0,0,0);
+
+	KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_START, 0, 0, 0, 0, 0);
 
 	error = file_socket(s, &so);
-	if (error )
-	{
-	    KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_END, error,0,0,0,0);
-	    return (error);
+	if (error) {
+		KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_END, error, 0, 0, 0, 0);
+		return (error);
 	}
-	
-	if (mp->msg_name) {
-		error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
-		if (error) {
-		    KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_END, error,0,0,0,0);
-			goto out;
+	if (so == NULL) {
+		error = EBADF;
+		goto out;
+	}
+	if (mp->msg_name != USER_ADDR_NULL) {
+		if (mp->msg_namelen > sizeof (ss)) {
+			error = getsockaddr(so, &to, mp->msg_name,
+			    mp->msg_namelen);
+		} else {
+			error = getsockaddr_s(so, &ss, mp->msg_name,
+			    mp->msg_namelen);
+			if (error == 0) {
+				to = (struct sockaddr *)&ss;
+				want_free = FALSE;
+			}
 		}
-		AUDIT_ARG(sockaddr, p, to);
-	} else {
-		to = 0;
+		if (error != 0)
+			goto out;
+		AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), to);
 	}
-	if (mp->msg_control) {
-		if (mp->msg_controllen < ((socklen_t)sizeof(struct cmsghdr))
-#if COMPAT_43_SOCKET
-		    && !(mp->msg_flags & MSG_COMPAT)
-#endif
-		) {
+	if (mp->msg_control != USER_ADDR_NULL) {
+		if (mp->msg_controllen < sizeof (struct cmsghdr)) {
 			error = EINVAL;
 			goto bad;
 		}
 		error = sockargs(&control, mp->msg_control,
 		    mp->msg_controllen, MT_CONTROL);
-		if (error)
+		if (error != 0)
 			goto bad;
-#if COMPAT_43_SOCKET
-		if (mp->msg_flags & MSG_COMPAT) {
-			register struct cmsghdr *cm;
-
-			M_PREPEND(control, sizeof(*cm), M_WAIT);
-			if (control == 0) {
-				error = ENOBUFS;
-				goto bad;
-			} else {
-				cm = mtod(control, struct cmsghdr *);
-				cm->cmsg_len = control->m_len;
-				cm->cmsg_level = SOL_SOCKET;
-				cm->cmsg_type = SCM_RIGHTS;
-			}
-		}
-#endif
-	} else {
-		control = 0;
 	}
 
-#if KTRACE    
-	if (KTRPOINT(p, KTR_GENIO)) {
-		ktruio = uio_duplicate(uiop);
-	}
-#endif
+#if CONFIG_MACF_SOCKET_SUBSET
+	/*
+	 * We check the state without holding the socket lock;
+	 * if a race condition occurs, it would simply result
+	 * in an extra call to the MAC check function.
+	 */
+	if (!(so->so_state & SS_ISCONNECTED) &&
+	    (error = mac_socket_check_send(kauth_cred_get(), so, to)) != 0)
+		goto bad;
+#endif /* MAC_SOCKET_SUBSET */
 
 	len = uio_resid(uiop);
-	if (so == NULL)
-		error = EBADF;
-	else
-		error = so->so_proto->pr_usrreqs->pru_sosend(so, to, uiop, 0, control,
-							     flags);
-	if (error) {
+	error = so->so_proto->pr_usrreqs->pru_sosend(so, to, uiop, 0, control,
+	    flags);
+	if (error != 0) {
 		if (uio_resid(uiop) != len && (error == ERESTART ||
 		    error == EINTR || error == EWOULDBLOCK))
 			error = 0;
-                /* Generation of SIGPIPE can be controlled per socket */
+		/* Generation of SIGPIPE can be controlled per socket */
 		if (error == EPIPE && !(so->so_flags & SOF_NOSIGPIPE))
 			psignal(p, SIGPIPE);
 	}
 	if (error == 0)
 		*retval = (int)(len - uio_resid(uiop));
 bad:
-#if KTRACE
-	if (ktruio != NULL) {
-		if (error == 0) {
-			uio_setresid(ktruio, retval[0]);
-			ktrgenio(p->p_tracep, s, UIO_WRITE, ktruio, error);
-		}
-		uio_free(ktruio);
-	}
-#endif
-	if (to)
+	if (to != NULL && want_free)
 		FREE(to, M_SONAME);
-	KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_END, error,0,0,0,0);
 out:
+	KERNEL_DEBUG(DBG_FNC_SENDIT | DBG_FUNC_END, error, 0, 0, 0, 0);
 	file_drop(s);
 	return (error);
 }
 
-
+/*
+ * Returns:	0			Success
+ *		ENOMEM
+ *	sendit:???			[see sendit definition in this file]
+ *	write:???			[4056224: applicable for pipes]
+ */
 int
 sendto(struct proc *p, struct sendto_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(sendto_nocancel(p, (struct sendto_nocancel_args *)uap, retval));
+}
+
+int
+sendto_nocancel(struct proc *p, struct sendto_nocancel_args *uap, register_t *retval)
 {
 	struct user_msghdr msg;
 	int error;
 	uio_t auio = NULL;
 
-	KERNEL_DEBUG(DBG_FNC_SENDTO | DBG_FUNC_START, 0,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_SENDTO | DBG_FUNC_START, 0, 0, 0, 0, 0);
 	AUDIT_ARG(fd, uap->s);
 
 	auio = uio_create(1, 0,
-				  (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-				  UIO_WRITE);
+	    (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
+	    UIO_WRITE);
 	if (auio == NULL) {
 		return (ENOMEM);
 	}
@@ -717,39 +924,40 @@ sendto(struct proc *p, struct sendto_args *uap, register_t *retval)
 	msg.msg_flags = 0;
 
 	error = sendit(p, uap->s, &msg, auio, uap->flags, retval);
-	
+
 	if (auio != NULL) {
 		uio_free(auio);
 	}
-	
+
 #if HACK_FOR_4056224
-	/* 
-	 * Radar 4056224 
-	 * Temporary workaround to let send() and recv() work over a pipe for binary compatibility
+	/*
+	 * Radar 4056224
+	 * Temporary workaround to let send() and recv() work over
+	 * a pipe for binary compatibility
 	 * This will be removed in the release following Tiger
 	 */
 	if (error == ENOTSOCK) {
 		struct fileproc *fp;
-		
-        if (fp_lookup(p, uap->s, &fp, 0) == 0) {
-			(void) fp_drop(p, uap->s, fp,0);
-			
+
+		if (fp_lookup(p, uap->s, &fp, 0) == 0) {
+			(void) fp_drop(p, uap->s, fp, 0);
+
 			if (fp->f_type == DTYPE_PIPE) {
 				struct write_args write_uap;
 				user_ssize_t write_retval;
-				
+
 				if (p->p_pid > last_pid_4056224) {
 					last_pid_4056224 = p->p_pid;
 
-					printf("%s[%d] uses send/recv on a pipe\n", 
-						p->p_comm, p->p_pid);
+					printf("%s[%d] uses send/recv "
+					    "on a pipe\n", p->p_comm, p->p_pid);
 				}
-				
-				bzero(&write_uap, sizeof(struct write_args));
+
+				bzero(&write_uap, sizeof (struct write_args));
 				write_uap.fd = uap->s;
 				write_uap.cbuf = uap->buf;
 				write_uap.nbyte = uap->len;
-	
+
 				error = write(p, &write_uap, &write_retval);
 				*retval = (int)write_retval;
 			}
@@ -757,40 +965,26 @@ sendto(struct proc *p, struct sendto_args *uap, register_t *retval)
 	}
 #endif /* HACK_FOR_4056224 */
 
-	KERNEL_DEBUG(DBG_FNC_SENDTO | DBG_FUNC_END, error, *retval,0,0,0);
-	
-	return(error);
-}
+	KERNEL_DEBUG(DBG_FNC_SENDTO | DBG_FUNC_END, error, *retval, 0, 0, 0);
 
-#if COMPAT_43_SOCKET
-int
-osend(__unused struct proc *p, 
-	  __unused struct osend_args *uap, 
-	  __unused register_t *retval)
-{
-	/* these are no longer supported and in fact 
-	 * there is no way to call it directly.
-	 * LP64todo - remove this once we're sure there are no clients 
-	 */
-	return (ENOTSUP);
+	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOBUFS
+ *	copyin:EFAULT
+ *	sendit:???			[see sendit definition in this file]
+ */
 int
-osendmsg(__unused struct proc *p, 
-	  	 __unused struct osendmsg_args *uap, 
-		 __unused register_t *retval)
+sendmsg(struct proc *p, struct sendmsg_args *uap, register_t *retval)
 {
-	/* these are no longer supported and in fact 
-	 * there is no way to call it directly.
-	 * LP64todo - remove this once we're sure there are no clients 
-	 */
-	return (ENOTSUP);
+	__pthread_testcancel(1);
+	return(sendmsg_nocancel(p, (struct sendmsg_nocancel_args *)uap, retval));
 }
-#endif
-
 
 int
-sendmsg(struct proc *p, register struct sendmsg_args *uap, register_t *retval)
+sendmsg_nocancel(struct proc *p, struct sendmsg_nocancel_args *uap, register_t *retval)
 {
 	struct msghdr msg;
 	struct user_msghdr user_msg;
@@ -801,23 +995,21 @@ sendmsg(struct proc *p, register struct sendmsg_args *uap, register_t *retval)
 	uio_t auio = NULL;
 	struct user_iovec *iovp;
 
-	KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_START, 0,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_START, 0, 0, 0, 0, 0);
 	AUDIT_ARG(fd, uap->s);
 	if (IS_64BIT_PROCESS(p)) {
-		msghdrp = (caddr_t) &user_msg;
-		size_of_msghdr = sizeof(user_msg);
-		size_of_iovec = sizeof(struct user_iovec);
-	}
-	else {
-		msghdrp = (caddr_t) &msg;
-		size_of_msghdr = sizeof(msg);
-		size_of_iovec = sizeof(struct iovec);
+		msghdrp = (caddr_t)&user_msg;
+		size_of_msghdr = sizeof (user_msg);
+		size_of_iovec = sizeof (struct user_iovec);
+	} else {
+		msghdrp = (caddr_t)&msg;
+		size_of_msghdr = sizeof (msg);
+		size_of_iovec = sizeof (struct iovec);
 	}
 	error = copyin(uap->msg, msghdrp, size_of_msghdr);
-	if (error)
-	{
-	    KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, error,0,0,0,0);
-	    return (error);
+	if (error) {
+		KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, error, 0, 0, 0, 0);
+		return (error);
 	}
 
 	/* only need to copy if user process is not 64-bit */
@@ -832,61 +1024,80 @@ sendmsg(struct proc *p, register struct sendmsg_args *uap, register_t *retval)
 	}
 
 	if (user_msg.msg_iovlen <= 0 || user_msg.msg_iovlen > UIO_MAXIOV) {
-		KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, EMSGSIZE,0,0,0,0);
+		KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, EMSGSIZE,
+		    0, 0, 0, 0);
 		return (EMSGSIZE);
 	}
 
 	/* allocate a uio large enough to hold the number of iovecs passed */
 	auio = uio_create(user_msg.msg_iovlen, 0,
-				  (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-				  UIO_WRITE);
+	    (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
+	    UIO_WRITE);
 	if (auio == NULL) {
 		error = ENOBUFS;
 		goto done;
 	}
-		
+
 	if (user_msg.msg_iovlen) {
-		/* get location of iovecs within the uio.  then copyin the iovecs from
-		 * user space.
+		/*
+		 * get location of iovecs within the uio.
+		 * then copyin the iovecs from user space.
 		 */
 		iovp = uio_iovsaddr(auio);
 		if (iovp == NULL) {
 			error = ENOBUFS;
 			goto done;
 		}
-		error = copyin(user_msg.msg_iov, (caddr_t)iovp, (user_msg.msg_iovlen * size_of_iovec));
+		error = copyin(user_msg.msg_iov, (caddr_t)iovp,
+		    (user_msg.msg_iovlen * size_of_iovec));
 		if (error)
 			goto done;
 		user_msg.msg_iov = CAST_USER_ADDR_T(iovp);
-	
-		/* finish setup of uio_t */ 
+
+		/* finish setup of uio_t */
 		uio_calculateresid(auio);
-	}
-	else {
+	} else {
 		user_msg.msg_iov = 0;
 	}
-	
-#if COMPAT_43_SOCKET
+
+	/* msg_flags is ignored for send */
 	user_msg.msg_flags = 0;
-#endif
+
 	error = sendit(p, uap->s, &user_msg, auio, uap->flags, retval);
 done:
 	if (auio != NULL) {
 		uio_free(auio);
 	}
-	KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, error,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_SENDMSG | DBG_FUNC_END, error, 0, 0, 0, 0);
 
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOTSOCK
+ *		EINVAL
+ *		EBADF
+ *		EACCES			Mandatory Access Control failure
+ *	copyout:EFAULT
+ *	fp_lookup:EBADF
+ *	<pru_soreceive>:ENOBUFS
+ *	<pru_soreceive>:ENOTCONN
+ *	<pru_soreceive>:EWOULDBLOCK
+ *	<pru_soreceive>:EFAULT
+ *	<pru_soreceive>:EINTR
+ *	<pru_soreceive>:EBADF
+ *	<pru_soreceive>:EINVAL
+ *	<pru_soreceive>:EMSGSIZE
+ *	<pru_soreceive>:???
+ *
+ * Notes:	Additional return values from calls through <pru_soreceive>
+ *		depend on protocols other than TCP or AF_UNIX, which are
+ *		documented above.
+ */
 static int
-recvit(p, s, mp, uiop, namelenp, retval)
-	register struct proc *p;
-	int s;
-	register struct user_msghdr *mp;
-	uio_t uiop;
-	user_addr_t namelenp;
-	register_t *retval;
+recvit(struct proc *p, int s, struct user_msghdr *mp, uio_t uiop,
+    user_addr_t namelenp, register_t *retval)
 {
 	int len, error;
 	struct mbuf *m, *control = 0;
@@ -894,114 +1105,85 @@ recvit(p, s, mp, uiop, namelenp, retval)
 	struct socket *so;
 	struct sockaddr *fromsa = 0;
 	struct fileproc *fp;
-#if KTRACE
-	uio_t ktruio = NULL;
-#endif
 
-	KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_START, 0,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_START, 0, 0, 0, 0, 0);
 	proc_fdlock(p);
-	if ( (error = fp_lookup(p, s, &fp, 1)) ) {
-	    KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, error,0,0,0,0);
+	if ((error = fp_lookup(p, s, &fp, 1))) {
+		KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, error, 0, 0, 0, 0);
 		proc_fdunlock(p);
-	    return (error);
+		return (error);
 	}
 	if (fp->f_type != DTYPE_SOCKET) {
-		fp_drop(p, s, fp,1);
+		fp_drop(p, s, fp, 1);
 		proc_fdunlock(p);
-		return(ENOTSOCK);
+		return (ENOTSOCK);
 	}
 
-	so = (struct socket *)fp->f_data;	
+	so = (struct socket *)fp->f_data;
+	if (so == NULL) {
+		fp_drop(p, s, fp, 1);
+		proc_fdunlock(p);
+		return (EBADF);
+	}
 
 	proc_fdunlock(p);
+
+#if CONFIG_MACF_SOCKET_SUBSET
+	/*
+	 * We check the state without holding the socket lock;
+	 * if a race condition occurs, it would simply result
+	 * in an extra call to the MAC check function.
+	 */
+	if (!(so->so_state & SS_ISCONNECTED) &&
+	    (error = mac_socket_check_receive(kauth_cred_get(), so)) != 0)
+		goto out1;
+#endif /* MAC_SOCKET_SUBSET */
 	if (uio_resid(uiop) < 0) {
-		KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, EINVAL,0,0,0,0);
+		KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, EINVAL, 0, 0, 0, 0);
 		error = EINVAL;
 		goto out1;
 	}
-#if KTRACE
-	if (KTRPOINT(p, KTR_GENIO)) {
-		ktruio = uio_duplicate(uiop);
-	}
-#endif
 
 	len = uio_resid(uiop);
-	if (so == NULL)
-		error = EBADF;
-	else {
-		error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, uiop,
-			(struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
-			&mp->msg_flags);
-	}
-	AUDIT_ARG(sockaddr, p, fromsa);
+	error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, uiop,
+	    (struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
+	    &mp->msg_flags);
+	AUDIT_ARG(sockaddr, vfs_context_cwd(vfs_context_current()), fromsa);
 	if (error) {
 		if (uio_resid(uiop) != len && (error == ERESTART ||
 		    error == EINTR || error == EWOULDBLOCK))
 			error = 0;
 	}
-#if KTRACE
-	if (ktruio != NULL) {
-		if (error == 0) {
-			uio_setresid(ktruio, len - uio_resid(uiop));
-			ktrgenio(p->p_tracep, s, UIO_WRITE, ktruio, error);
-		}
-		uio_free(ktruio);
-	}
-#endif
+
 	if (error)
 		goto out;
+
 	*retval = len - uio_resid(uiop);
 	if (mp->msg_name) {
+		socklen_t sa_len = 0;
+
 		len = mp->msg_namelen;
-		if (len <= 0 || fromsa == 0)
+		if (len <= 0 || fromsa == 0) {
 			len = 0;
-		else {
+		} else {
 #ifndef MIN
-#define MIN(a,b) ((a)>(b)?(b):(a))
-#endif
-			/* save sa_len before it is destroyed by MSG_COMPAT */
-			len = MIN(len, fromsa->sa_len);
-#if COMPAT_43_SOCKET
-			if (mp->msg_flags & MSG_COMPAT)
-				((struct osockaddr *)fromsa)->sa_family =
-				    fromsa->sa_family;
+#define	MIN(a, b) ((a) > (b) ? (b) : (a))
 #endif
+			sa_len = fromsa->sa_len;
+			len = MIN((unsigned int)len, sa_len);
 			error = copyout(fromsa, mp->msg_name, (unsigned)len);
 			if (error)
 				goto out;
 		}
-		mp->msg_namelen = len;
+		mp->msg_namelen = sa_len;
+		/* return the actual, untruncated address length */
 		if (namelenp &&
-		    (error = copyout((caddr_t)&len, namelenp, sizeof (int)))) {
-#if COMPAT_43_SOCKET
-			if (mp->msg_flags & MSG_COMPAT)
-				error = 0;	/* old recvfrom didn't check */
-			else
-#endif
+		    (error = copyout((caddr_t)&sa_len, namelenp,
+		    sizeof (int)))) {
 			goto out;
 		}
 	}
 	if (mp->msg_control) {
-#if COMPAT_43_SOCKET
-		/*
-		 * We assume that old recvmsg calls won't receive access
-		 * rights and other control info, esp. as control info
-		 * is always optional and those options didn't exist in 4.3.
-		 * If we receive rights, trim the cmsghdr; anything else
-		 * is tossed.
-		 */
-		if (control && mp->msg_flags & MSG_COMPAT) {
-			if (mtod(control, struct cmsghdr *)->cmsg_level !=
-			    SOL_SOCKET ||
-			    mtod(control, struct cmsghdr *)->cmsg_type !=
-			    SCM_RIGHTS) {
-				mp->msg_controllen = 0;
-				goto out;
-			}
-			control->m_len -= sizeof (struct cmsghdr);
-			control->m_data += sizeof (struct cmsghdr);
-		}
-#endif
 		len = mp->msg_controllen;
 		m = control;
 		mp->msg_controllen = 0;
@@ -1010,14 +1192,15 @@ recvit(p, s, mp, uiop, namelenp, retval)
 		while (m && len > 0) {
 			unsigned int tocopy;
 
-			if (len >= m->m_len) 
+			if (len >= m->m_len) {
 				tocopy = m->m_len;
-			else {
+			} else {
 				mp->msg_flags |= MSG_CTRUNC;
 				tocopy = len;
 			}
-		
-			error = copyout((caddr_t)mtod(m, caddr_t), ctlbuf, tocopy);
+
+			error = copyout((caddr_t)mtod(m, caddr_t), ctlbuf,
+			    tocopy);
 			if (error)
 				goto out;
 
@@ -1032,31 +1215,43 @@ out:
 		FREE(fromsa, M_SONAME);
 	if (control)
 		m_freem(control);
-	KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, error,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_RECVIT | DBG_FUNC_END, error, 0, 0, 0, 0);
 out1:
 	fp_drop(p, s, fp, 0);
 	return (error);
 }
 
 
+/*
+ * Returns:	0			Success
+ *		ENOMEM
+ *	copyin:EFAULT
+ *	recvit:???
+ *	read:???			[4056224: applicable for pipes]
+ *
+ * Notes:	The read entry point is only called as part of support for
+ *		binary backward compatability; new code should use read
+ *		instead of recv or recvfrom when attempting to read data
+ *		from pipes.
+ *
+ *		For full documentation of the return codes from recvit, see
+ *		the block header for the recvit function.
+ */
+int
+recvfrom(struct proc *p, struct recvfrom_args *uap, register_t *retval)
+{
+	__pthread_testcancel(1);
+	return(recvfrom_nocancel(p, (struct recvfrom_nocancel_args *)uap, retval));
+}
+
 int
-recvfrom(p, uap, retval)
-	struct proc *p;
-	register struct recvfrom_args /* {
-		int	s;
-		caddr_t	buf;
-		size_t	len;
-		int	flags;
-		caddr_t	from;
-		int	*fromlenaddr;
-	} */ *uap;
-	register_t *retval;
+recvfrom_nocancel(struct proc *p, struct recvfrom_nocancel_args *uap, register_t *retval)
 {
 	struct user_msghdr msg;
 	int error;
 	uio_t auio = NULL;
 
-	KERNEL_DEBUG(DBG_FNC_RECVFROM | DBG_FUNC_START, 0,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_RECVFROM | DBG_FUNC_START, 0, 0, 0, 0, 0);
 	AUDIT_ARG(fd, uap->s);
 
 	if (uap->fromlenaddr) {
@@ -1064,16 +1259,17 @@ recvfrom(p, uap, retval)
 		    (caddr_t)&msg.msg_namelen, sizeof (msg.msg_namelen));
 		if (error)
 			return (error);
-	} else
+	} else {
 		msg.msg_namelen = 0;
+	}
 	msg.msg_name = uap->from;
 	auio = uio_create(1, 0,
-				  (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-				  UIO_READ);
+	    (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
+	    UIO_READ);
 	if (auio == NULL) {
 		return (ENOMEM);
 	}
-	
+
 	uio_addiov(auio, uap->buf, uap->len);
 	/* no need to set up msg_iov.  recvit uses uio_t we send it */
 	msg.msg_iov = 0;
@@ -1085,35 +1281,36 @@ recvfrom(p, uap, retval)
 	if (auio != NULL) {
 		uio_free(auio);
 	}
-	
+
 #if HACK_FOR_4056224
-	/* 
-	 * Radar 4056224 
-	 * Temporary workaround to let send() and recv() work over a pipe for binary compatibility
+	/*
+	 * Radar 4056224
+	 * Temporary workaround to let send() and recv() work over
+	 * a pipe for binary compatibility
 	 * This will be removed in the release following Tiger
 	 */
 	if (error == ENOTSOCK && proc_is64bit(p) == 0) {
 		struct fileproc *fp;
-		
-        if (fp_lookup(p, uap->s, &fp, 0) == 0) {
-			(void) fp_drop(p, uap->s, fp,0);
-			
+
+		if (fp_lookup(p, uap->s, &fp, 0) == 0) {
+			(void) fp_drop(p, uap->s, fp, 0);
+
 			if (fp->f_type == DTYPE_PIPE) {
 				struct read_args read_uap;
 				user_ssize_t read_retval;
-				
+
 				if (p->p_pid > last_pid_4056224) {
 					last_pid_4056224 = p->p_pid;
 
-					printf("%s[%d] uses send/recv on a pipe\n", 
-						p->p_comm, p->p_pid);
+					printf("%s[%d] uses send/recv on "
+					    "a pipe\n", p->p_comm, p->p_pid);
 				}
-				
-				bzero(&read_uap, sizeof(struct read_args));
+
+				bzero(&read_uap, sizeof (struct read_args));
 				read_uap.fd = uap->s;
 				read_uap.cbuf = uap->buf;
 				read_uap.nbyte = uap->len;
-	
+
 				error = read(p, &read_uap, &read_retval);
 				*retval = (int)read_retval;
 			}
@@ -1121,86 +1318,56 @@ recvfrom(p, uap, retval)
 	}
 #endif /* HACK_FOR_4056224 */
 
-	KERNEL_DEBUG(DBG_FNC_RECVFROM | DBG_FUNC_END, error,0,0,0,0);
-	
-	return (error);
-}
-
-#if COMPAT_43_SOCKET
-int
-orecvfrom(struct proc *p, struct recvfrom_args *uap, register_t *retval)
-{
-
-	uap->flags |= MSG_COMPAT;
-	return (recvfrom(p, uap, retval));
-}
-#endif
-
+	KERNEL_DEBUG(DBG_FNC_RECVFROM | DBG_FUNC_END, error, 0, 0, 0, 0);
 
-#if COMPAT_43_SOCKET
-int
-orecv(__unused struct proc *p, __unused struct orecv_args	*uap, 
-		__unused register_t *retval)
-{
-	/* these are no longer supported and in fact 
-	 * there is no way to call it directly.
-	 * LP64todo - remove this once we're sure there are no clients 
-	 */
-
-	return (ENOTSUP);
+	return (error);
 }
 
 /*
- * Old recvmsg.  This code takes advantage of the fact that the old msghdr
- * overlays the new one, missing only the flags, and with the (old) access
- * rights where the control fields are now.
+ * Returns:	0			Success
+ *		EMSGSIZE
+ *		ENOMEM
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	recvit:???
+ *
+ * Notes:	For full documentation of the return codes from recvit, see
+ *		the block header for the recvit function.
  */
 int
-orecvmsg(__unused struct proc *p, __unused struct orecvmsg_args *uap, 
-		__unused register_t *retval)
+recvmsg(struct proc *p, struct recvmsg_args *uap, register_t *retval)
 {
-	/* these are no longer supported and in fact 
-	 * there is no way to call it directly.
-	 * LP64todo - remove this once we're sure there are no clients 
-	 */
-
-	return (ENOTSUP);
-
+	__pthread_testcancel(1);
+	return(recvmsg_nocancel(p, (struct recvmsg_nocancel_args *)uap, retval));
 }
-#endif
 
 int
-recvmsg(p, uap, retval)
-	struct proc *p;
-	struct recvmsg_args *uap;
-	register_t *retval;
+recvmsg_nocancel(struct proc *p, struct recvmsg_nocancel_args *uap, register_t *retval)
 {
 	struct msghdr msg;
 	struct user_msghdr user_msg;
 	caddr_t msghdrp;
 	int	size_of_msghdr;
 	user_addr_t uiov;
-	register int error;
+	int error;
 	int size_of_iovec;
 	uio_t auio = NULL;
 	struct user_iovec *iovp;
 
-	KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_START, 0,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_START, 0, 0, 0, 0, 0);
 	AUDIT_ARG(fd, uap->s);
 	if (IS_64BIT_PROCESS(p)) {
-		msghdrp = (caddr_t) &user_msg;
-		size_of_msghdr = sizeof(user_msg);
-		size_of_iovec = sizeof(struct user_iovec);
-	}
-	else {
-		msghdrp = (caddr_t) &msg;
-		size_of_msghdr = sizeof(msg);
-		size_of_iovec = sizeof(struct iovec);
+		msghdrp = (caddr_t)&user_msg;
+		size_of_msghdr = sizeof (user_msg);
+		size_of_iovec = sizeof (struct user_iovec);
+	} else {
+		msghdrp = (caddr_t)&msg;
+		size_of_msghdr = sizeof (msg);
+		size_of_iovec = sizeof (struct iovec);
 	}
 	error = copyin(uap->msg, msghdrp, size_of_msghdr);
-	if (error)
-	{
-	    	KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, error,0,0,0,0);
+	if (error) {
+		KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, error, 0, 0, 0, 0);
 		return (error);
 	}
 
@@ -1216,26 +1383,24 @@ recvmsg(p, uap, retval)
 	}
 
 	if (user_msg.msg_iovlen <= 0 || user_msg.msg_iovlen > UIO_MAXIOV) {
-		KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, EMSGSIZE,0,0,0,0);
+		KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, EMSGSIZE,
+		    0, 0, 0, 0);
 		return (EMSGSIZE);
 	}
 
-#if COMPAT_43_SOCKET
-	user_msg.msg_flags = uap->flags &~ MSG_COMPAT;
-#else
 	user_msg.msg_flags = uap->flags;
-#endif
 
 	/* allocate a uio large enough to hold the number of iovecs passed */
 	auio = uio_create(user_msg.msg_iovlen, 0,
-				  (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-				  UIO_READ);
+	    (IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
+	    UIO_READ);
 	if (auio == NULL) {
 		error = ENOMEM;
 		goto done;
 	}
 
-	/* get location of iovecs within the uio.  then copyin the iovecs from
+	/*
+	 * get location of iovecs within the uio.  then copyin the iovecs from
 	 * user space.
 	 */
 	iovp = uio_iovsaddr(auio);
@@ -1245,13 +1410,14 @@ recvmsg(p, uap, retval)
 	}
 	uiov = user_msg.msg_iov;
 	user_msg.msg_iov = CAST_USER_ADDR_T(iovp);
-	error = copyin(uiov, (caddr_t)iovp, (user_msg.msg_iovlen * size_of_iovec));
+	error = copyin(uiov, (caddr_t)iovp,
+	    (user_msg.msg_iovlen * size_of_iovec));
 	if (error)
 		goto done;
 
-	/* finish setup of uio_t */ 
+	/* finish setup of uio_t */
 	uio_calculateresid(auio);
-		
+
 	error = recvit(p, uap->s, &user_msg, auio, 0, retval);
 	if (!error) {
 		user_msg.msg_iov = uiov;
@@ -1260,9 +1426,11 @@ recvmsg(p, uap, retval)
 			// LP64todo - do all these change?  if not, then no need to copy all of them!
 			msg.msg_flags = user_msg.msg_flags;
 			msg.msg_controllen = user_msg.msg_controllen;
-			msg.msg_control = CAST_DOWN(caddr_t, user_msg.msg_control);
+			msg.msg_control =
+			    CAST_DOWN(caddr_t, user_msg.msg_control);
 			msg.msg_iovlen = user_msg.msg_iovlen;
-			msg.msg_iov = (struct iovec *) CAST_DOWN(caddr_t, user_msg.msg_iov);
+			msg.msg_iov = (struct iovec *)
+			    CAST_DOWN(caddr_t, user_msg.msg_iov);
 			msg.msg_namelen = user_msg.msg_namelen;
 			msg.msg_name = CAST_DOWN(caddr_t, user_msg.msg_name);
 		}
@@ -1272,15 +1440,36 @@ done:
 	if (auio != NULL) {
 		uio_free(auio);
 	}
-	KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, error,0,0,0,0);
+	KERNEL_DEBUG(DBG_FNC_RECVMSG | DBG_FUNC_END, error, 0, 0, 0, 0);
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EBADF
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	soshutdown:EINVAL
+ *	soshutdown:ENOTCONN
+ *	soshutdown:EADDRNOTAVAIL[TCP]
+ *	soshutdown:ENOBUFS[TCP]
+ *	soshutdown:EMSGSIZE[TCP]
+ *	soshutdown:EHOSTUNREACH[TCP]
+ *	soshutdown:ENETUNREACH[TCP]
+ *	soshutdown:ENETDOWN[TCP]
+ *	soshutdown:ENOMEM[TCP]
+ *	soshutdown:EACCES[TCP]
+ *	soshutdown:EMSGSIZE[TCP]
+ *	soshutdown:ENOBUFS[TCP]
+ *	soshutdown:???[TCP]		[ignorable: mostly IPSEC/firewall/DLIL]
+ *	soshutdown:???			[other protocol families]
+ */
 /* ARGSUSED */
 int
-shutdown(__unused struct proc *p, struct shutdown_args *uap, __unused register_t *retval)
+shutdown(__unused struct proc *p, struct shutdown_args *uap,
+    __unused register_t *retval)
 {
-	struct socket * so;
+	struct socket *so;
 	int error;
 
 	AUDIT_ARG(fd, uap->s);
@@ -1294,26 +1483,37 @@ shutdown(__unused struct proc *p, struct shutdown_args *uap, __unused register_t
 	error =  soshutdown((struct socket *)so, uap->how);
 out:
 	file_drop(uap->s);
-	return(error);
+	return (error);
 }
 
-
-
-
-
+/*
+ * Returns:	0			Success
+ *		EFAULT
+ *		EINVAL
+ *		EACCES			Mandatory Access Control failure
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	sosetopt:EINVAL
+ *	sosetopt:ENOPROTOOPT
+ *	sosetopt:ENOBUFS
+ *	sosetopt:EDOM
+ *	sosetopt:EFAULT
+ *	sosetopt:EOPNOTSUPP[AF_UNIX]
+ *	sosetopt:???
+ */
 /* ARGSUSED */
 int
-setsockopt(struct proc *p, struct setsockopt_args *uap, __unused register_t *retval)
+setsockopt(struct proc *p, struct setsockopt_args *uap,
+    __unused register_t *retval)
 {
-	struct socket * so;
+	struct socket *so;
 	struct sockopt sopt;
 	int error;
 
 	AUDIT_ARG(fd, uap->s);
 	if (uap->val == 0 && uap->valsize != 0)
 		return (EFAULT);
-	if (uap->valsize < 0)
-		return (EINVAL);
+	/* No bounds checking on size (it's unsigned) */
 
 	error = file_socket(uap->s, &so);
 	if (error)
@@ -1330,36 +1530,51 @@ setsockopt(struct proc *p, struct setsockopt_args *uap, __unused register_t *ret
 		error = EINVAL;
 		goto out;
 	}
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_setsockopt(kauth_cred_get(), so,
+	    &sopt)) != 0)
+		goto out;
+#endif /* MAC_SOCKET_SUBSET */
 	error = sosetopt(so, &sopt);
 out:
 	file_drop(uap->s);
-	return(error);
+	return (error);
 }
 
 
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		EBADF
+ *		EACCES			Mandatory Access Control failure
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	sogetopt:???
+ */
 int
-getsockopt(struct proc *p, struct getsockopt_args  *uap, __unused register_t *retval)
+getsockopt(struct proc *p, struct getsockopt_args  *uap,
+    __unused register_t *retval)
 {
 	int		error;
 	socklen_t	valsize;
 	struct sockopt	sopt;
-	struct socket *	so;
+	struct socket *so;
 
 	error = file_socket(uap->s, &so);
 	if (error)
 		return (error);
 	if (uap->val) {
-		error = copyin(uap->avalsize, (caddr_t)&valsize, sizeof (valsize));
+		error = copyin(uap->avalsize, (caddr_t)&valsize,
+		    sizeof (valsize));
 		if (error)
 			goto out;
-		if (valsize < 0) {
-			error = EINVAL;
-			goto out;
-		}
-	} else
+		/* No bounds checking on size (it's unsigned) */
+	} else {
 		valsize = 0;
-
+	}
 	sopt.sopt_dir = SOPT_GET;
 	sopt.sopt_level = uap->level;
 	sopt.sopt_name = uap->name;
@@ -1371,10 +1586,16 @@ getsockopt(struct proc *p, struct getsockopt_args  *uap, __unused register_t *re
 		error = EBADF;
 		goto out;
 	}
+#if CONFIG_MACF_SOCKET_SUBSET
+	if ((error = mac_socket_check_getsockopt(kauth_cred_get(), so,
+	    &sopt)) != 0)
+		goto out;
+#endif /* MAC_SOCKET_SUBSET */
 	error = sogetopt((struct socket *)so, &sopt);
 	if (error == 0) {
 		valsize = sopt.sopt_valsize;
-		error = copyout((caddr_t)&valsize, uap->avalsize, sizeof (valsize));
+		error = copyout((caddr_t)&valsize, uap->avalsize,
+		    sizeof (valsize));
 	}
 out:
 	file_drop(uap->s);
@@ -1384,21 +1605,33 @@ out:
 
 /*
  * Get socket name.
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	<pru_sockaddr>:ENOBUFS[TCP]
+ *	<pru_sockaddr>:ECONNRESET[TCP]
+ *	<pru_sockaddr>:EINVAL[AF_UNIX]
+ *	<sf_getsockname>:???
  */
 /* ARGSUSED */
-static int
-getsockname1(__unused struct proc *p, struct getsockname_args *uap, __unused register_t *retval,
-	int compat)
+int
+getsockname(__unused struct proc *p, struct getsockname_args *uap,
+    __unused register_t *retval)
 {
 	struct socket *so;
 	struct sockaddr *sa;
 	socklen_t len;
+	socklen_t sa_len;
 	int error;
 
 	error = file_socket(uap->fdes, &so);
 	if (error)
 		return (error);
-	error = copyin(uap->alen, (caddr_t)&len, sizeof(socklen_t));
+	error = copyin(uap->alen, (caddr_t)&len, sizeof (socklen_t));
 	if (error)
 		goto out;
 	if (so == NULL) {
@@ -1408,26 +1641,25 @@ getsockname1(__unused struct proc *p, struct getsockname_args *uap, __unused reg
 	sa = 0;
 	socket_lock(so, 1);
 	error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &sa);
-	if (error == 0)
-	{
+	if (error == 0) {
 		struct socket_filter_entry *filter;
 		int	filtered = 0;
 		for (filter = so->so_filt; filter && error == 0;
-			 filter = filter->sfe_next_onsocket) {
+		    filter = filter->sfe_next_onsocket) {
 			if (filter->sfe_filter->sf_filter.sf_getsockname) {
 				if (!filtered) {
 					filtered = 1;
 					sflt_use(so);
 					socket_unlock(so, 0);
 				}
-				error = filter->sfe_filter->sf_filter.sf_getsockname(filter->sfe_cookie,
-							so, &sa);
+				error = filter->sfe_filter->sf_filter.
+				    sf_getsockname(filter->sfe_cookie, so, &sa);
 			}
 		}
-		
+
 		if (error == EJUSTRETURN)
 			error = 0;
-		
+
 		if (filtered) {
 			socket_lock(so, 0);
 			sflt_unuse(so);
@@ -1441,15 +1673,15 @@ getsockname1(__unused struct proc *p, struct getsockname_args *uap, __unused reg
 		goto gotnothing;
 	}
 
-	len = MIN(len, sa->sa_len);
-#if COMPAT_43_SOCKET
-	if (compat)
-		((struct osockaddr *)sa)->sa_family = sa->sa_family;
-#endif
+	sa_len = sa->sa_len;
+	len = MIN(len, sa_len);
 	error = copyout((caddr_t)sa, uap->asa, len);
-	if (error == 0)
+	if (error)
+		goto bad;
+	/* return the actual, untruncated address length */
+	len = sa_len;
 gotnothing:
-		error = copyout((caddr_t)&len, uap->alen, sizeof(socklen_t));
+		error = copyout((caddr_t)&len, uap->alen, sizeof (socklen_t));
 bad:
 	if (sa)
 		FREE(sa, M_SONAME);
@@ -1458,31 +1690,29 @@ out:
 	return (error);
 }
 
-int
-getsockname(struct proc *p, struct getsockname_args *uap, register_t *retval)
-{
-	return (getsockname1(p, uap, retval, 0));
-}
-
-#if COMPAT_43_SOCKET
-int
-ogetsockname(struct proc *p, struct getsockname_args *uap, register_t *retval)
-{
-	return (getsockname1(p, uap, retval, 1));
-}
-#endif /* COMPAT_43_SOCKET */
-
 /*
  * Get name of peer for connected socket.
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *		EINVAL
+ *		ENOTCONN
+ *	file_socket:ENOTSOCK
+ *	file_socket:EBADF
+ *	copyin:EFAULT
+ *	copyout:EFAULT
+ *	<pru_peeraddr>:???
+ *	<sf_getpeername>:???
  */
 /* ARGSUSED */
 int
-getpeername1(__unused struct proc *p, struct getpeername_args *uap, __unused register_t *retval,
-	int compat)
+getpeername(__unused struct proc *p, struct getpeername_args *uap,
+    __unused register_t *retval)
 {
 	struct socket *so;
 	struct sockaddr *sa;
 	socklen_t len;
+	socklen_t sa_len;
 	int error;
 
 	error = file_socket(uap->fdes, &so);
@@ -1495,38 +1725,45 @@ getpeername1(__unused struct proc *p, struct getpeername_args *uap, __unused reg
 
 	socket_lock(so, 1);
 
+	if ((so->so_state & (SS_CANTRCVMORE | SS_CANTSENDMORE)) ==
+	    (SS_CANTRCVMORE | SS_CANTSENDMORE)) {
+		/* the socket has been shutdown, no more getpeername's */
+		socket_unlock(so, 1);
+		error = EINVAL;
+		goto out;
+	}
+
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
 		socket_unlock(so, 1);
 		error = ENOTCONN;
 		goto out;
 	}
-	error = copyin(uap->alen, (caddr_t)&len, sizeof(socklen_t));
+	error = copyin(uap->alen, (caddr_t)&len, sizeof (socklen_t));
 	if (error) {
 		socket_unlock(so, 1);
 		goto out;
 	}
 	sa = 0;
 	error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, &sa);
-	if (error == 0)
-	{
+	if (error == 0) {
 		struct socket_filter_entry *filter;
 		int	filtered = 0;
 		for (filter = so->so_filt; filter && error == 0;
-			 filter = filter->sfe_next_onsocket) {
+		    filter = filter->sfe_next_onsocket) {
 			if (filter->sfe_filter->sf_filter.sf_getpeername) {
 				if (!filtered) {
 					filtered = 1;
 					sflt_use(so);
 					socket_unlock(so, 0);
 				}
-				error = filter->sfe_filter->sf_filter.sf_getpeername(filter->sfe_cookie,
-							so, &sa);
+				error = filter->sfe_filter->sf_filter.
+				    sf_getpeername(filter->sfe_cookie, so, &sa);
 			}
 		}
-		
+
 		if (error == EJUSTRETURN)
 			error = 0;
-		
+
 		if (filtered) {
 			socket_lock(so, 0);
 			sflt_unuse(so);
@@ -1539,17 +1776,15 @@ getpeername1(__unused struct proc *p, struct getpeername_args *uap, __unused reg
 		len = 0;
 		goto gotnothing;
 	}
-	len = MIN(len, sa->sa_len);
-#if COMPAT_43_SOCKET
-	if (compat)
-		((struct osockaddr *)sa)->sa_family =
-		    sa->sa_family;
-#endif
+	sa_len = sa->sa_len;
+	len = MIN(len, sa_len);
 	error = copyout(sa, uap->asa, len);
 	if (error)
 		goto bad;
+	/* return the actual, untruncated address length */
+	len = sa_len;
 gotnothing:
-	error = copyout((caddr_t)&len, uap->alen, sizeof(socklen_t));
+	error = copyout((caddr_t)&len, uap->alen, sizeof (socklen_t));
 bad:
 	if (sa) FREE(sa, M_SONAME);
 out:
@@ -1558,38 +1793,16 @@ out:
 }
 
 int
-getpeername(struct proc *p, struct getpeername_args *uap, register_t *retval)
+sockargs(struct mbuf **mp, user_addr_t data, int buflen, int type)
 {
-
-	return (getpeername1(p, uap, retval, 0));
-}
-
-#if COMPAT_43_SOCKET
-int
-ogetpeername(struct proc *p, struct getpeername_args *uap, register_t *retval)
-{
-
-	return (getpeername1(p, uap, retval, 1));
-}
-#endif /* COMPAT_43_SOCKET */
-
-int
-sockargs(mp, data, buflen, type)
-	struct mbuf **mp;
-	user_addr_t data;
-	int buflen, type;
-{
-	register struct sockaddr *sa;
-	register struct mbuf *m;
+	struct sockaddr *sa;
+	struct mbuf *m;
 	int error;
 
 	if ((u_int)buflen > MLEN) {
-#if COMPAT_43_SOCKET
 		if (type == MT_SONAME && (u_int)buflen <= 112)
 			buflen = MLEN;		/* unix domain compat. hack */
-		else
-#endif
-		if ((u_int)buflen > MCLBYTES)
+		else if ((u_int)buflen > MCLBYTES)
 			return (EINVAL);
 	}
 	m = m_get(M_WAIT, type);
@@ -1599,22 +1812,17 @@ sockargs(mp, data, buflen, type)
 		MCLGET(m, M_WAIT);
 		if ((m->m_flags & M_EXT) == 0) {
 			m_free(m);
-			return ENOBUFS;
+			return (ENOBUFS);
 		}
 	}
 	m->m_len = buflen;
 	error = copyin(data, mtod(m, caddr_t), (u_int)buflen);
-	if (error)
+	if (error) {
 		(void) m_free(m);
-	else {
+	} else {
 		*mp = m;
 		if (type == MT_SONAME) {
 			sa = mtod(m, struct sockaddr *);
-
-#if COMPAT_43_SOCKET && BYTE_ORDER != BIG_ENDIAN
-			if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
-				sa->sa_family = sa->sa_len;
-#endif
 			sa->sa_len = buflen;
 		}
 	}
@@ -1623,372 +1831,440 @@ sockargs(mp, data, buflen, type)
 
 /*
  * Given a user_addr_t of length len, allocate and fill out a *sa.
+ *
+ * Returns:	0			Success
+ *		ENAMETOOLONG		Filename too long
+ *		EINVAL			Invalid argument
+ *		ENOMEM			Not enough space
+ *		copyin:EFAULT		Bad address
  */
-int
-getsockaddr(struct sockaddr **namp, user_addr_t uaddr, size_t len)
+static int
+getsockaddr(struct socket *so, struct sockaddr **namp, user_addr_t uaddr,
+    size_t len)
 {
 	struct sockaddr *sa;
 	int error;
 
 	if (len > SOCK_MAXADDRLEN)
-		return ENAMETOOLONG;
+		return (ENAMETOOLONG);
 
-	if (len == 0)
-	     return EINVAL;
+	if (len < offsetof(struct sockaddr, sa_data[0]))
+		return (EINVAL);
 
-	MALLOC(sa, struct sockaddr *, len, M_SONAME, M_WAITOK);
+	MALLOC(sa, struct sockaddr *, len, M_SONAME, M_WAITOK | M_ZERO);
 	if (sa == NULL) {
-		return ENOMEM;
+		return (ENOMEM);
 	}
 	error = copyin(uaddr, (caddr_t)sa, len);
 	if (error) {
 		FREE(sa, M_SONAME);
 	} else {
-#if COMPAT_43_SOCKET && BYTE_ORDER != BIG_ENDIAN
-		if (sa->sa_family == 0 && sa->sa_len < AF_MAX)
-			sa->sa_family = sa->sa_len;
-#endif
+		/*
+		 * Force sa_family to AF_INET on AF_INET sockets to handle
+		 * legacy applications that use AF_UNSPEC (0).  On all other
+		 * sockets we leave it unchanged and let the lower layer
+		 * handle it.
+		 */
+		if (sa->sa_family == AF_UNSPEC &&
+		    INP_CHECK_SOCKAF(so, AF_INET) &&
+		    len == sizeof (struct sockaddr_in))
+			sa->sa_family = AF_INET;
+
 		sa->sa_len = len;
 		*namp = sa;
 	}
-	return error;
+	return (error);
 }
 
-
-#if SENDFILE
-/*
- * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
- * XXX - The sf_buf functions are currently private to sendfile(2), so have
- * been made static, but may be useful in the future for doing zero-copy in
- * other parts of the networking code. 
- */
-static void
-sf_buf_init(void *arg)
+static int
+getsockaddr_s(struct socket *so, struct sockaddr_storage *ss,
+    user_addr_t uaddr, size_t len)
 {
-	int i;
+	int error;
+
+	if (ss == NULL || uaddr == USER_ADDR_NULL ||
+	    len < offsetof(struct sockaddr, sa_data[0]))
+		return (EINVAL);
+
+	/*
+	 * sockaddr_storage size is less than SOCK_MAXADDRLEN,
+	 * so the check here is inclusive.
+	 */
+	if (len > sizeof (*ss))
+		return (ENAMETOOLONG);
 
-	SLIST_INIT(&sf_freelist);
-	kmem_alloc_pageable(kernel_map, &sf_base, nsfbufs * PAGE_SIZE);
-	MALLOC(sf_bufs, struct sf_buf *, nsfbufs * sizeof(struct sf_buf), M_TEMP, M_NOWAIT|M_ZERO);
-	if (sf_bufs == NULL)
-		return;		/* XXX silently fail leaving sf_bufs NULL */
+	bzero(ss, sizeof (*ss));
+	error = copyin(uaddr, (caddr_t)ss, len);
+	if (error == 0) {
+		/*
+		 * Force sa_family to AF_INET on AF_INET sockets to handle
+		 * legacy applications that use AF_UNSPEC (0).  On all other
+		 * sockets we leave it unchanged and let the lower layer
+		 * handle it.
+		 */
+		if (ss->ss_family == AF_UNSPEC &&
+		    INP_CHECK_SOCKAF(so, AF_INET) &&
+		    len == sizeof (struct sockaddr_in))
+			ss->ss_family = AF_INET;
 
-	for (i = 0; i < nsfbufs; i++) {
-		sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
-		SLIST_INSERT_HEAD(&sf_freelist, &sf_bufs[i], free_list);
+		ss->ss_len = len;
 	}
+	return (error);
 }
 
-/*
- * Get an sf_buf from the freelist. Will block if none are available.
- */
-static struct sf_buf *
-sf_buf_alloc()
+#if SENDFILE
+
+SYSCTL_DECL(_kern_ipc);
+
+#define	SFUIOBUFS 64
+static int sendfileuiobufs = SFUIOBUFS;
+SYSCTL_INT(_kern_ipc, OID_AUTO, sendfileuiobufs, CTLFLAG_RW, &sendfileuiobufs,
+    0, "");
+
+/* Macros to compute the number of mbufs needed depending on cluster size */
+#define	HOWMANY_16K(n)	((((unsigned int)(n) - 1) >> (PGSHIFT + 2)) + 1)
+#define	HOWMANY_4K(n)	((((unsigned int)(n) - 1) >> PGSHIFT) + 1)
+
+/* Upper send limit in bytes (sendfileuiobufs * PAGESIZE) */
+#define SENDFILE_MAX_BYTES	(sendfileuiobufs << PGSHIFT)
+
+/* Upper send limit in the number of mbuf clusters */
+#define	SENDFILE_MAX_16K	HOWMANY_16K(SENDFILE_MAX_BYTES)
+#define	SENDFILE_MAX_4K		HOWMANY_4K(SENDFILE_MAX_BYTES)
+
+size_t mbuf_pkt_maxlen(mbuf_t m);
+
+__private_extern__ size_t
+mbuf_pkt_maxlen(mbuf_t m)
 {
-	struct sf_buf *sf;
+	size_t maxlen = 0;
 
-	while ((sf = SLIST_FIRST(&sf_freelist)) == NULL) {
-		sf_buf_alloc_want = 1;
-		tsleep(&sf_freelist, PVM, "sfbufa", 0);
+	while (m) {
+		maxlen += mbuf_maxlen(m);
+		m = mbuf_next(m);
 	}
-	SLIST_REMOVE_HEAD(&sf_freelist, free_list);
-	sf->refcnt = 1;
-	return (sf);
+	return (maxlen);
 }
 
-#define dtosf(x)	(&sf_bufs[((uintptr_t)(x) - (uintptr_t)sf_base) >> PAGE_SHIFT])
 static void
-sf_buf_ref(caddr_t addr, u_int size)
+alloc_sendpkt(int how, size_t pktlen, unsigned int *maxchunks,
+    struct mbuf **m, boolean_t jumbocl)
 {
-	struct sf_buf *sf;
+	unsigned int needed;
 
-	sf = dtosf(addr);
-	if (sf->refcnt == 0)
-		panic("sf_buf_ref: referencing a free sf_buf");
-	sf->refcnt++;
-}
+	if (pktlen == 0)
+		panic("%s: pktlen (%ld) must be non-zero\n", __func__, pktlen);
 
-/*
- * Lose a reference to an sf_buf. When none left, detach mapped page
- * and release resources back to the system.
- *
- * Must be called at splimp.
- */
-static void
-sf_buf_free(caddr_t addr, u_int size)
-{
-	struct sf_buf *sf;
-	struct vm_page *m;
-
-	sf = dtosf(addr);
-	if (sf->refcnt == 0)
-		panic("sf_buf_free: freeing free sf_buf");
-	sf->refcnt--;
-	if (sf->refcnt == 0) {
-		pmap_qremove((vm_offset_t)addr, 1);
-		m = sf->m;
-		vm_page_unwire(m, 0);
-		/*
-		 * Check for the object going away on us. This can
-		 * happen since we don't hold a reference to it.
-		 * If so, we're responsible for freeing the page.
-		 */
-		if (m->wire_count == 0 && m->object == NULL)
-			vm_page_lock_queues();
-			vm_page_free(m);
-			vm_page_unlock_queues();
-		sf->m = NULL;
-		SLIST_INSERT_HEAD(&sf_freelist, sf, free_list);
-		if (sf_buf_alloc_want) {
-			sf_buf_alloc_want = 0;
-			wakeup(&sf_freelist);
-		}
+	/*
+	 * Try to allocate for the whole thing.  Since we want full control
+	 * over the buffer size and be able to accept partial result, we can't
+	 * use mbuf_allocpacket().  The logic below is similar to sosend().
+	 */
+	*m = NULL;
+	if (pktlen > NBPG && jumbocl) {
+		needed = MIN(SENDFILE_MAX_16K, HOWMANY_16K(pktlen));
+		*m = m_getpackets_internal(&needed, 1, how, 0, M16KCLBYTES);
+	}
+	if (*m == NULL) {
+		needed = MIN(SENDFILE_MAX_4K, HOWMANY_4K(pktlen));
+		*m = m_getpackets_internal(&needed, 1, how, 0, NBPG);
+	}
+
+	/*
+	 * Our previous attempt(s) at allocation had failed; the system
+	 * may be short on mbufs, and we want to block until they are
+	 * available.  This time, ask just for 1 mbuf and don't return
+	 * until we get it.
+	 */
+	if (*m == NULL) {
+		needed = 1;
+		*m = m_getpackets_internal(&needed, 1, M_WAIT, 1, NBPG);
 	}
+	if (*m == NULL)
+		panic("%s: blocking allocation returned NULL\n", __func__);
+
+	*maxchunks = needed;
 }
 
 /*
  * sendfile(2).
- * int sendfile(int fd, int s, off_t offset, size_t nbytes,
- *	 struct sf_hdtr *hdtr, off_t *sbytes, int flags)
+ * int sendfile(int fd, int s, off_t offset, off_t *nbytes,
+ *	 struct sf_hdtr *hdtr, int flags)
  *
  * Send a file specified by 'fd' and starting at 'offset' to a socket
- * specified by 's'. Send only 'nbytes' of the file or until EOF if
- * nbytes == 0. Optionally add a header and/or trailer to the socket
- * output. If specified, write the total number of bytes sent into *sbytes.
+ * specified by 's'. Send only '*nbytes' of the file or until EOF if
+ * *nbytes == 0. Optionally add a header and/or trailer to the socket
+ * output. If specified, write the total number of bytes sent into *nbytes.
  */
 int
-sendfile(struct proc *p, struct sendfile_args *uap)
+sendfile(struct proc *p, struct sendfile_args *uap, __unused int *retval)
 {
 	struct fileproc *fp;
 	struct vnode *vp;
-	struct vm_object *obj;
 	struct socket *so;
-	struct mbuf *m;
-	struct sf_buf *sf;
-	struct vm_page *pg;
-	struct writev_args nuap;
+	struct writev_nocancel_args nuap;
+	user_ssize_t writev_retval;
 	struct sf_hdtr hdtr;
-	off_t off, xfsize, sbytes = 0;
-	int error = 0, s;
-	kauth_cred_t safecred;
-
-	if (sf_bufs == NULL) {
-		/* Fail if initialization failed */
-		return ENOSYS;
-	}
-
+	struct user_sf_hdtr user_hdtr;
+	off_t off, xfsize;
+	off_t nbytes = 0, sbytes = 0;
+	int error = 0;
+	size_t sizeof_hdtr;
+	size_t size_of_iovec;
+	off_t file_size;
+	struct vfs_context context = *vfs_context_current();
+
+	KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE | DBG_FUNC_START), uap->s,
+	    0, 0, 0, 0);
 	/*
 	 * Do argument checking. Must be a regular file in, stream
 	 * type and connected socket out, positive offset.
 	 */
-	if (error = fp_getfvp(p, uap->fd, &fp, &vp))
+	if ((error = fp_getfvp(p, uap->fd, &fp, &vp))) {
 		goto done;
-	if (fp->f_flag & FREAD) == 0) {
+	}
+	if ((fp->f_flag & FREAD) == 0) {
 		error = EBADF;
 		goto done1;
 	}
-	obj = vp->v_object;
-	if (vp->v_type != VREG || obj == NULL) {
-		error = EINVAL;
+	if (vnode_isreg(vp) == 0) {
+		error = ENOTSUP;
 		goto done1;
 	}
 	error = file_socket(uap->s, &so);
-	if (error)
+	if (error) {
 		goto done1;
+	}
 	if (so == NULL) {
 		error = EBADF;
 		goto done2;
 	}
-
-	socket_lock(so, 1);
-
 	if (so->so_type != SOCK_STREAM) {
 		error = EINVAL;
-		goto done3;
+		goto done2;
 	}
 	if ((so->so_state & SS_ISCONNECTED) == 0) {
 		error = ENOTCONN;
-		goto done3;
+		goto done2;
 	}
 	if (uap->offset < 0) {
 		error = EINVAL;
-		goto done3;
+		goto done2;
 	}
+	if (uap->nbytes == USER_ADDR_NULL) {
+		error = EINVAL;
+		goto done2;
+	}
+	if (uap->flags != 0) {
+		error = EINVAL;
+		goto done2;
+	}
+
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
+#if CONFIG_MACF_SOCKET_SUBSET
+	/* JMM - fetch connected sockaddr? */
+	error = mac_socket_check_send(context.vc_ucred, so, NULL);
+	if (error)
+		goto done2;
+#endif
+
+	/*
+	 * Get number of bytes to send
+	 * Should it applies to size of header and trailer?
+	 * JMM - error handling?
+	 */
+	copyin(uap->nbytes, &nbytes, sizeof (off_t));
 
 	/*
 	 * If specified, get the pointer to the sf_hdtr struct for
 	 * any headers/trailers.
 	 */
-	if (uap->hdtr != NULL) {
-		error = copyin(CAST_USER_ADDR_T(uap->hdtr), &hdtr, sizeof(hdtr));
+	if (uap->hdtr != USER_ADDR_NULL) {
+		caddr_t hdtrp;
+
+		bzero(&user_hdtr, sizeof (user_hdtr));
+		if (IS_64BIT_PROCESS(p)) {
+			hdtrp = (caddr_t)&user_hdtr;
+			sizeof_hdtr = sizeof (user_hdtr);
+			size_of_iovec = sizeof (struct user_iovec);
+		} else {
+			hdtrp = (caddr_t)&hdtr;
+			sizeof_hdtr = sizeof (hdtr);
+			size_of_iovec = sizeof (struct iovec);
+		}
+		error = copyin(uap->hdtr, hdtrp, sizeof_hdtr);
 		if (error)
-			goto done3;
+			goto done2;
+		/*  need to copy if user process is not 64-bit */
+		if (!IS_64BIT_PROCESS(p)) {
+			user_hdtr.headers = CAST_USER_ADDR_T(hdtr.headers);
+			user_hdtr.hdr_cnt = hdtr.hdr_cnt;
+			user_hdtr.trailers = CAST_USER_ADDR_T(hdtr.trailers);
+			user_hdtr.trl_cnt = hdtr.trl_cnt;
+		}
+
 		/*
 		 * Send any headers. Wimp out and use writev(2).
 		 */
-		if (hdtr.headers != NULL) {
+		if (user_hdtr.headers != USER_ADDR_NULL) {
+			bzero(&nuap, sizeof (struct writev_args));
 			nuap.fd = uap->s;
-			nuap.iovp = hdtr.headers;
-			nuap.iovcnt = hdtr.hdr_cnt;
-			error = writev(p, &nuap);
+			nuap.iovp = user_hdtr.headers;
+			nuap.iovcnt = user_hdtr.hdr_cnt;
+			error = writev_nocancel(p, &nuap, &writev_retval);
 			if (error)
-				goto done3;
-			sbytes += p->p_retval[0];
+				goto done2;
+			sbytes += writev_retval;
 		}
 	}
 
 	/*
-	 * Protect against multiple writers to the socket.
+	 * Get the file size for 2 reasons:
+	 *  1. We don't want to allocate more mbufs than necessary
+	 *  2. We don't want to read past the end of file
 	 */
-	(void) sblock(&so->so_snd, M_WAIT);
+	if ((error = vnode_size(vp, &file_size, vfs_context_current())) != 0)
+		goto done2;
 
 	/*
-	 * Loop through the pages in the file, starting with the requested
-	 * offset. Get a file page (do I/O if necessary), map the file page
-	 * into an sf_buf, attach an mbuf header to the sf_buf, and queue
-	 * it on the socket.
+	 * Simply read file data into a chain of mbufs that used with scatter
+	 * gather reads. We're not (yet?) setup to use zero copy external
+	 * mbufs that point to the file pages.
 	 */
+	socket_lock(so, 1);
+	error = sblock(&so->so_snd, M_WAIT);
+	if (error) {
+		socket_unlock(so, 1);
+		goto done2;
+	}
 	for (off = uap->offset; ; off += xfsize, sbytes += xfsize) {
-		vm_object_offset_t pindex;
-		vm_object_offset_t pgoff;
+		mbuf_t	m0 = NULL, m;
+		unsigned int	nbufs = sendfileuiobufs, i;
+		uio_t	auio;
+		char	uio_buf[UIO_SIZEOF(sendfileuiobufs)]; /* 1 KB !!! */
+		size_t	uiolen;
+		user_ssize_t	rlen;
+		off_t	pgoff;
+		size_t	pktlen;
+		boolean_t jumbocl;
 
-		pindex = OFF_TO_IDX(off);
-retry_lookup:
 		/*
-		 * Calculate the amount to transfer. Not to exceed a page,
+		 * Calculate the amount to transfer.
+		 * Align to round number of pages.
+		 * Not to exceed send socket buffer,
 		 * the EOF, or the passed in nbytes.
 		 */
-		xfsize = obj->un_pager.vnp.vnp_size - off;
-		if (xfsize > PAGE_SIZE_64)
-			xfsize = PAGE_SIZE;
-		pgoff = (vm_object_offset_t)(off & PAGE_MASK_64);
-		if (PAGE_SIZE - pgoff < xfsize)
+		xfsize = sbspace(&so->so_snd);
+
+		if (xfsize <= 0) {
+			if (so->so_state & SS_CANTSENDMORE) {
+				error = EPIPE;
+				goto done3;
+			} else if ((so->so_state & SS_NBIO)) {
+				error = EAGAIN;
+				goto done3;
+			} else {
+				xfsize = PAGE_SIZE;
+			}
+		}
+
+		if (xfsize > SENDFILE_MAX_BYTES)
+			xfsize = SENDFILE_MAX_BYTES;
+		else if (xfsize > PAGE_SIZE)
+			xfsize = trunc_page(xfsize);
+		pgoff = off & PAGE_MASK_64;
+		if (pgoff > 0 && PAGE_SIZE - pgoff < xfsize)
 			xfsize = PAGE_SIZE_64 - pgoff;
-		if (uap->nbytes && xfsize > (uap->nbytes - sbytes))
-			xfsize = uap->nbytes - sbytes;
+		if (nbytes && xfsize > (nbytes - sbytes))
+			xfsize = nbytes - sbytes;
+		if (xfsize <= 0)
+			break;
+		if (off + xfsize > file_size)
+			xfsize = file_size - off;
 		if (xfsize <= 0)
 			break;
+
 		/*
-		 * Optimize the non-blocking case by looking at the socket space
-		 * before going to the extra work of constituting the sf_buf.
+		 * Attempt to use larger than system page-size clusters for
+		 * large writes only if there is a jumbo cluster pool and
+		 * if the socket is marked accordingly.
 		 */
-		if ((so->so_state & SS_NBIO) && sbspace(&so->so_snd) <= 0) {
-			if (so->so_state & SS_CANTSENDMORE)
-				error = EPIPE;
-			else
-				error = EAGAIN;
-			sbunlock(&so->so_snd, 0); /* will release lock */
-			goto done2;
+		jumbocl = sosendjcl && njcl > 0 &&
+		    ((so->so_flags & SOF_MULTIPAGES) || sosendjcl_ignore_capab);
+
+		socket_unlock(so, 0);
+		alloc_sendpkt(M_WAIT, xfsize, &nbufs, &m0, jumbocl);
+		pktlen = mbuf_pkt_maxlen(m0);
+		if (pktlen < xfsize)
+			xfsize = pktlen;
+
+		auio = uio_createwithbuffer(nbufs, off, UIO_SYSSPACE,
+		    UIO_READ, &uio_buf[0], sizeof (uio_buf));
+		if (auio == NULL) {
+			//printf("sendfile: uio_createwithbuffer failed\n");
+			mbuf_freem(m0);
+			error = ENXIO;
+			socket_lock(so, 0);
+			goto done3;
 		}
-		/*
-		 * Attempt to look up the page. If the page doesn't exist or the
-		 * part we're interested in isn't valid, then read it from disk.
-		 * If some other part of the kernel has this page (i.e. it's busy),
-		 * then disk I/O may be occuring on it, so wait and retry.
-		 */
-		pg = vm_page_lookup(obj, pindex);
-		if (pg == NULL || (!(pg->flags & PG_BUSY) && !pg->busy &&
-		    !vm_page_is_valid(pg, pgoff, xfsize))) {
-			struct uio auio;
-			struct iovec aiov;
-			int bsize;
-
-			if (pg == NULL) {
-				pg = vm_page_alloc(obj, pindex, VM_ALLOC_NORMAL);
-				if (pg == NULL) {
-					VM_WAIT;
-					goto retry_lookup;
-				}
-				/*
-				 * don't just clear PG_BUSY manually -
-				 * vm_page_alloc() should be considered opaque,
-				 * use the VM routine provided to clear
-				 * PG_BUSY.
-				 */
-				vm_page_wakeup(pg);
 
+		for (i = 0, m = m0, uiolen = 0;
+		    i < nbufs && m != NULL && uiolen < xfsize;
+		    i++, m = mbuf_next(m)) {
+			size_t mlen = mbuf_maxlen(m);
+
+			if (mlen + uiolen > xfsize)
+				mlen = xfsize - uiolen;
+			mbuf_setlen(m, mlen);
+			uio_addiov(auio, CAST_USER_ADDR_T(mbuf_datastart(m)),
+			    mlen);
+			uiolen += mlen;
+		}
+
+		if (xfsize != uio_resid(auio))
+			printf("sendfile: xfsize: %lld != uio_resid(auio): "
+			    "%lld\n", xfsize, uio_resid(auio));
+
+		KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_READ | DBG_FUNC_START),
+		    uap->s, (unsigned int)((xfsize >> 32) & 0x0ffffffff),
+		    (unsigned int)(xfsize & 0x0ffffffff), 0, 0);
+		error = fo_read(fp, auio, FOF_OFFSET, &context);
+		socket_lock(so, 0);
+		if (error != 0) {
+			if (uio_resid(auio) != xfsize && (error == ERESTART ||
+			    error == EINTR || error == EWOULDBLOCK)) {
+				error = 0;
+			} else {
+				mbuf_freem(m0);
+				goto done3;
 			}
-			/*
-			 * Ensure that our page is still around when the I/O completes.
-			 */
-			vm_page_io_start(pg);
-			vm_page_wire(pg);
-			/*
-			 * Get the page from backing store.
-			 */
-			bsize = vp->v_mount->mnt_vfsstat.f_iosize;
-			auio.uio_iov = &aiov;
-			auio.uio_iovcnt = 1;
-			aiov.iov_base = 0;
-			aiov.iov_len = MAXBSIZE;
-			auio.uio_offset = trunc_page(off);
-			auio.uio_segflg = UIO_NOCOPY;
-			auio.uio_rw = UIO_READ;
-			uio_setresid(&auio, MAXBSIZE);
-			safecred = kauth_cred_proc_ref(p);
-			error = VOP_READ(vp, &auio, IO_VMIO | ((MAXBSIZE / bsize) << 16),
-			        safecred);
-			kauth_cred_unref(&safecred);
-			vm_page_flag_clear(pg, PG_ZERO);
-			vm_page_io_finish(pg);
-			if (error) {
-				vm_page_unwire(pg, 0);
-				/*
-				 * See if anyone else might know about this page.
-				 * If not and it is not valid, then free it.
-				 */
-				if (pg->wire_count == 0 && pg->valid == 0 &&
-				    pg->busy == 0 && !(pg->flags & PG_BUSY) &&
-				    pg->hold_count == 0)
-					vm_page_lock_queues();
-					vm_page_free(pg);
-					vm_page_unlock_queues();
-				sbunlock(&so->so_snd, 0); /* will release socket lock */
-				goto done2;
-			}
-		} else {
-			if ((pg->flags & PG_BUSY) || pg->busy)  {
-				s = splvm();
-				if ((pg->flags & PG_BUSY) || pg->busy) {
-					/*
-					 * Page is busy. Wait and retry.
-					 */
-					vm_page_flag_set(pg, PG_WANTED);
-					tsleep(pg, PVM, "sfpbsy", 0);
-					goto retry_lookup;
-				}
-			}
-			/*
-			 * Protect from having the page ripped out from beneath us.
-			 */
-			vm_page_wire(pg);
 		}
-		/*
-		 * Allocate a kernel virtual page and insert the physical page
-		 * into it.
-		 */
-		sf = sf_buf_alloc();
-		sf->m = pg;
-		pmap_qenter(sf->kva, &pg, 1);
-		/*
-		 * Get an mbuf header and set it up as having external storage.
-		 */
-		MGETHDR(m, M_WAIT, MT_DATA);
-		if (m == NULL) {
-			error = ENOBUFS;
-			sbunlock(&so->so_snd, 0); /* will release socket lock */
-			goto done2;
+		xfsize -= uio_resid(auio);
+		KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_READ | DBG_FUNC_END),
+		    uap->s, (unsigned int)((xfsize >> 32) & 0x0ffffffff),
+		    (unsigned int)(xfsize & 0x0ffffffff), 0, 0);
+
+		if (xfsize == 0) {
+			//printf("sendfile: fo_read 0 bytes, EOF\n");
+			break;
 		}
-		m->m_ext.ext_free = sf_buf_free;
-		m->m_ext.ext_ref = sf_buf_ref;
-		m->m_ext.ext_buf = (void *)sf->kva;
-		m->m_ext.ext_size = PAGE_SIZE;
-		m->m_data = (char *) sf->kva + pgoff;
-		m->m_flags |= M_EXT;
-		m->m_pkthdr.len = m->m_len = xfsize;
-		/*
-		 * Add the buffer to the socket buffer chain.
-		 */
+		if (xfsize + off > file_size)
+			printf("sendfile: xfsize: %lld + off: %lld > file_size:"
+			    "%lld\n", xfsize, off, file_size);
+		for (i = 0, m = m0, rlen = 0;
+		    i < nbufs && m != NULL && rlen < xfsize;
+		    i++, m = mbuf_next(m)) {
+			size_t mlen = mbuf_maxlen(m);
+
+			if (rlen + mlen > xfsize)
+				mlen = xfsize - rlen;
+			mbuf_setlen(m, mlen);
+
+			rlen += mlen;
+		}
+		mbuf_pkthdr_setlen(m0, xfsize);
+
 retry_space:
 		/*
 		 * Make sure that the socket is still able to take more data.
@@ -2008,69 +2284,130 @@ retry_space:
 				error = so->so_error;
 				so->so_error = 0;
 			}
-			m_freem(m);
-			sbunlock(&so->so_snd, 0); /* will release socket lock */
-			goto done2;
+			m_freem(m0);
+			goto done3;
 		}
 		/*
 		 * Wait for socket space to become available. We do this just
 		 * after checking the connection state above in order to avoid
 		 * a race condition with sbwait().
 		 */
-		if (sbspace(&so->so_snd) < so->so_snd.sb_lowat) {
+		if (sbspace(&so->so_snd) < (long)so->so_snd.sb_lowat) {
 			if (so->so_state & SS_NBIO) {
-				m_freem(m);
-				sbunlock(&so->so_snd, 0); /* will release socket lock */
+				m_freem(m0);
 				error = EAGAIN;
-				goto done2;
+				goto done3;
 			}
+			KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_WAIT |
+			    DBG_FUNC_START), uap->s, 0, 0, 0, 0);
 			error = sbwait(&so->so_snd);
+			KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_WAIT|
+			    DBG_FUNC_END), uap->s, 0, 0, 0, 0);
 			/*
 			 * An error from sbwait usually indicates that we've
 			 * been interrupted by a signal. If we've sent anything
 			 * then return bytes sent, otherwise return the error.
 			 */
 			if (error) {
-				m_freem(m);
-				sbunlock(&so->so_snd, 0);
-				goto done2;
+				m_freem(m0);
+				goto done3;
 			}
 			goto retry_space;
 		}
-		error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, m, 0, 0, p);
-		splx(s);
+		{
+			/*
+			 * Socket filter processing
+			 */
+			struct socket_filter_entry *filter;
+			int filtered = 0;
+			struct mbuf *control = NULL;
+			boolean_t recursive = (so->so_send_filt_thread != NULL);
+
+			error = 0;
+			for (filter = so->so_filt; filter && (error == 0);
+			    filter = filter->sfe_next_onsocket) {
+				if (filter->sfe_filter->sf_filter.sf_data_out) {
+					if (filtered == 0) {
+						filtered = 1;
+						so->so_send_filt_thread =
+						    current_thread();
+						sflt_use(so);
+						socket_unlock(so, 0);
+					}
+					error = filter->sfe_filter->sf_filter.
+					    sf_data_out(filter->sfe_cookie, so,
+					    NULL, &m0, &control, 0);
+				}
+			}
+
+			if (filtered) {
+				/*
+				 * At this point, we've run at least one filter.
+				 * The socket is unlocked as is the socket
+				 * buffer.  Clear the recorded filter thread
+				 * only when we are outside of a filter's
+				 * context.  This allows for a filter to issue
+				 * multiple inject calls from its sf_data_out
+				 * callback routine.
+				 */
+				socket_lock(so, 0);
+				sflt_unuse(so);
+				if (!recursive)
+					so->so_send_filt_thread = 0;
+				if (error) {
+					if (error == EJUSTRETURN) {
+						error = 0;
+						continue;
+					}
+					goto done3;
+				}
+			}
+			/*
+			 * End Socket filter processing
+			 */
+		}
+		KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_SEND | DBG_FUNC_START),
+		    uap->s, 0, 0, 0, 0);
+		error = (*so->so_proto->pr_usrreqs->pru_send)(so, 0, m0,
+		    0, 0, p);
+		KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE_SEND | DBG_FUNC_START),
+		    uap->s, 0, 0, 0, 0);
 		if (error) {
-			sbunlock(&so->so_snd, 0); /* will release socket lock */
-			goto done2;
+			goto done3;
 		}
 	}
-	sbunlock(&so->so_snd, 0); /* will release socket lock */
-
+	sbunlock(&so->so_snd, 0);	/* will unlock socket */
 	/*
 	 * Send trailers. Wimp out and use writev(2).
 	 */
-	if (uap->hdtr != NULL && hdtr.trailers != NULL) {
-			nuap.fd = uap->s;
-			nuap.iovp = hdtr.trailers;
-			nuap.iovcnt = hdtr.trl_cnt;
-			error = writev(p, &nuap);
-			if (error)
-				goto done2;
-			sbytes += p->p_retval[0];
+	if (uap->hdtr != USER_ADDR_NULL &&
+	    user_hdtr.trailers != USER_ADDR_NULL) {
+		bzero(&nuap, sizeof (struct writev_args));
+		nuap.fd = uap->s;
+		nuap.iovp = user_hdtr.trailers;
+		nuap.iovcnt = user_hdtr.trl_cnt;
+		error = writev_nocancel(p, &nuap, &writev_retval);
+		if (error)
+			goto done2;
+		sbytes += writev_retval;
 	}
 done2:
 	file_drop(uap->s);
 done1:
 	file_drop(uap->fd);
 done:
-	if (uap->sbytes != NULL) {
+	if (uap->nbytes != USER_ADDR_NULL) {
 		/* XXX this appears bogus for some early failure conditions */
-		copyout(&sbytes, CAST_USER_ADDR_T(uap->sbytes), sizeof(off_t));
+		copyout(&sbytes, uap->nbytes, sizeof (off_t));
 	}
+	KERNEL_DEBUG_CONSTANT((DBG_FNC_SENDFILE | DBG_FUNC_END), uap->s,
+	    (unsigned int)((sbytes >> 32) & 0x0ffffffff),
+	    (unsigned int)(sbytes & 0x0ffffffff), error, 0);
 	return (error);
 done3:
-	socket_unlock(so, 1);
+	sbunlock(&so->so_snd, 0);	/* will unlock socket */
 	goto done2;
 }
 
-#endif
+
+#endif /* SENDFILE */
diff --git a/bsd/kern/uipc_usrreq.c b/bsd/kern/uipc_usrreq.c
index 4555e7e1e..9b568d24c 100644
--- a/bsd/kern/uipc_usrreq.c
+++ b/bsd/kern/uipc_usrreq.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1991, 1993
@@ -53,6 +59,12 @@
  *
  *	From: @(#)uipc_usrreq.c	8.3 (Berkeley) 1/4/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -80,21 +92,25 @@
 #include <kern/zalloc.h>
 #include <kern/locks.h>
 
-#define f_msgcount f_fglob->fg_msgcount
-#define f_cred f_fglob->fg_cred
-#define f_ops f_fglob->fg_ops
-#define f_offset f_fglob->fg_offset
-#define f_data f_fglob->fg_data
+#if CONFIG_MACF_SOCKET
+#include <security/mac_framework.h>
+#endif /* MAC_SOCKET */
+
+#define	f_msgcount f_fglob->fg_msgcount
+#define	f_cred f_fglob->fg_cred
+#define	f_ops f_fglob->fg_ops
+#define	f_offset f_fglob->fg_offset
+#define	f_data f_fglob->fg_data
 struct	zone *unp_zone;
 static	unp_gen_t unp_gencnt;
 static	u_int unp_count;
 
-static	lck_attr_t 		*unp_mtx_attr;
-static	lck_grp_t 		*unp_mtx_grp;
-static	lck_grp_attr_t 		*unp_mtx_grp_attr;
-static	lck_rw_t 		*unp_list_mtx;
+static	lck_attr_t		*unp_mtx_attr;
+static	lck_grp_t		*unp_mtx_grp;
+static	lck_grp_attr_t		*unp_mtx_grp_attr;
+static	lck_rw_t		*unp_list_mtx;
 
-extern lck_mtx_t * uipc_lock;
+extern lck_mtx_t *uipc_lock;
 static	struct unp_head unp_shead, unp_dhead;
 
 /*
@@ -106,24 +122,26 @@ static	struct unp_head unp_shead, unp_dhead;
  *	need a proper out-of-band
  *	lock pushdown
  */
-static struct	sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL, { 0 } };
+static struct	sockaddr sun_noname = { sizeof (sun_noname), AF_LOCAL, { 0 } };
 static ino_t	unp_ino;		/* prototype for fake inode numbers */
 
-static int     unp_attach(struct socket *);
-static void    unp_detach(struct unpcb *);
-static int     unp_bind(struct unpcb *,struct sockaddr *, struct proc *);
-static int     unp_connect(struct socket *,struct sockaddr *, struct proc *);
-static void    unp_disconnect(struct unpcb *);
-static void    unp_shutdown(struct unpcb *);
-static void    unp_drop(struct unpcb *, int);
-static void    unp_gc(void);
-static void    unp_scan(struct mbuf *, void (*)(struct fileglob *));
-static void    unp_mark(struct fileglob *);
-static void    unp_discard(struct fileglob *);
-static void    unp_discard_fdlocked(struct fileglob *, struct proc *);
-static int     unp_internalize(struct mbuf *, struct proc *);
-static int     unp_listen(struct unpcb *, struct proc *);
-
+static int	unp_attach(struct socket *);
+static void	unp_detach(struct unpcb *);
+static int	unp_bind(struct unpcb *, struct sockaddr *, proc_t);
+static int	unp_connect(struct socket *, struct sockaddr *, proc_t);
+static void	unp_disconnect(struct unpcb *);
+static void	unp_shutdown(struct unpcb *);
+static void	unp_drop(struct unpcb *, int);
+static void	unp_gc(void);
+static void	unp_scan(struct mbuf *, void (*)(struct fileglob *));
+static void	unp_mark(struct fileglob *);
+static void	unp_discard(struct fileglob *);
+static void	unp_discard_fdlocked(struct fileglob *, proc_t);
+static int	unp_internalize(struct mbuf *, proc_t);
+static int	unp_listen(struct unpcb *, proc_t);
+
+/* TODO: this should be in header file */
+extern int fdgetf_noref(proc_t, int, struct fileproc **);
 
 static int
 uipc_abort(struct socket *so)
@@ -131,11 +149,11 @@ uipc_abort(struct socket *so)
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 	unp_drop(unp, ECONNABORTED);
 	unp_detach(unp);
 	sofree(so);
-	return 0;
+	return (0);
 }
 
 static int
@@ -144,7 +162,7 @@ uipc_accept(struct socket *so, struct sockaddr **nam)
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 
 	/*
 	 * Pass back name of connected socket,
@@ -152,54 +170,70 @@ uipc_accept(struct socket *so, struct sockaddr **nam)
 	 * (our peer may have closed already!).
 	 */
 	if (unp->unp_conn && unp->unp_conn->unp_addr) {
-		*nam = dup_sockaddr((struct sockaddr *)unp->unp_conn->unp_addr,
-				    1);
+		*nam = dup_sockaddr((struct sockaddr *)
+		    unp->unp_conn->unp_addr, 1);
 	} else {
 		*nam = dup_sockaddr((struct sockaddr *)&sun_noname, 1);
 	}
-	return 0;
+	return (0);
 }
 
+/*
+ * Returns:	0			Success
+ *		EISCONN
+ *	unp_attach:
+ */
 static int
-uipc_attach(struct socket *so, __unused int proto, __unused struct proc *p)
+uipc_attach(struct socket *so, __unused int proto, __unused proc_t p)
 {
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp != 0)
-		return EISCONN;
-	return unp_attach(so);
+		return (EISCONN);
+	return (unp_attach(so));
 }
 
 static int
-uipc_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
+uipc_bind(struct socket *so, struct sockaddr *nam, proc_t p)
 {
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 
-	return unp_bind(unp, nam, p);
+	return (unp_bind(unp, nam, p));
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	unp_connect:???			[See elsewhere in this file]
+ */
 static int
-uipc_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
+uipc_connect(struct socket *so, struct sockaddr *nam, proc_t p)
 {
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
-	return unp_connect(so, nam, p);
+		return (EINVAL);
+	return (unp_connect(so, nam, p));
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	unp_connect2:EPROTOTYPE		Protocol wrong type for socket
+ *	unp_connect2:EINVAL		Invalid argument
+ */
 static int
 uipc_connect2(struct socket *so1, struct socket *so2)
 {
 	struct unpcb *unp = sotounpcb(so1);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 
-	return unp_connect2(so1, so2);
+	return (unp_connect2(so1, so2));
 }
 
 /* control is EOPNOTSUPP */
@@ -210,10 +244,10 @@ uipc_detach(struct socket *so)
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 
 	unp_detach(unp);
-	return 0;
+	return (0);
 }
 
 static int
@@ -222,19 +256,23 @@ uipc_disconnect(struct socket *so)
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 	unp_disconnect(unp);
-	return 0;
+	return (0);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ */
 static int
-uipc_listen(struct socket *so, __unused struct proc *p)
+uipc_listen(struct socket *so, __unused proc_t p)
 {
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0 || unp->unp_vnode == 0)
-		return EINVAL;
-	return unp_listen(unp, p);
+		return (EINVAL);
+	return (unp_listen(unp, p));
 }
 
 static int
@@ -242,12 +280,15 @@ uipc_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	struct unpcb *unp = sotounpcb(so);
 
-	if (unp == 0)
-		return EINVAL;
-	if (unp->unp_conn && unp->unp_conn->unp_addr)
-		*nam = dup_sockaddr((struct sockaddr *)unp->unp_conn->unp_addr,
-				    1);
-	return 0;
+	if (unp == NULL)
+		return (EINVAL);
+	if (unp->unp_conn != NULL && unp->unp_conn->unp_addr != NULL) {
+		*nam = dup_sockaddr((struct sockaddr *)
+		    unp->unp_conn->unp_addr, 1);
+	} else {
+		*nam = dup_sockaddr((struct sockaddr *)&sun_noname, 1);
+	}
+	return (0);
 }
 
 static int
@@ -257,7 +298,7 @@ uipc_rcvd(struct socket *so, __unused int flags)
 	struct socket *so2;
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 	switch (so->so_type) {
 	case SOCK_DGRAM:
 		panic("uipc_rcvd DGRAM?");
@@ -265,7 +306,7 @@ uipc_rcvd(struct socket *so, __unused int flags)
 
 	case SOCK_STREAM:
 #define	rcv (&so->so_rcv)
-#define snd (&so2->so_snd)
+#define	snd (&so2->so_snd)
 		if (unp->unp_conn == 0)
 			break;
 		so2 = unp->unp_conn->unp_socket;
@@ -285,14 +326,33 @@ uipc_rcvd(struct socket *so, __unused int flags)
 	default:
 		panic("uipc_rcvd unknown socktype");
 	}
-	return 0;
+	return (0);
 }
 
 /* pru_rcvoob is EOPNOTSUPP */
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *		EOPNOTSUPP
+ *		EPIPE
+ *		ENOTCONN
+ *		EISCONN
+ *	unp_internalize:EINVAL
+ *	unp_internalize:EBADF
+ *	unp_connect:EAFNOSUPPORT	Address family not supported
+ *	unp_connect:EINVAL		Invalid argument
+ *	unp_connect:ENOTSOCK		Not a socket
+ *	unp_connect:ECONNREFUSED	Connection refused
+ *	unp_connect:EISCONN		Socket is connected
+ *	unp_connect:EPROTOTYPE		Protocol wrong type for socket
+ *	unp_connect:???
+ *	sbappendaddr:ENOBUFS		[5th argument, contents modified]
+ *	sbappendaddr:???		[whatever a filter author chooses]
+ */
 static int
 uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
-	  struct mbuf *control, struct proc *p)
+    struct mbuf *control, proc_t p)
 {
 	int error = 0;
 	struct unpcb *unp = sotounpcb(so);
@@ -308,7 +368,8 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	}
 
 	if (control) {
-		socket_unlock(so, 0); /* release global lock to avoid deadlock (4436174) */ 
+		/* release global lock to avoid deadlock (4436174) */
+		socket_unlock(so, 0);
 		error = unp_internalize(control, p);
 		socket_lock(so, 0);
 		if (error)
@@ -316,7 +377,7 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	}
 
 	switch (so->so_type) {
-	case SOCK_DGRAM: 
+	case SOCK_DGRAM:
 	{
 		struct sockaddr *from;
 
@@ -339,11 +400,22 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 			from = (struct sockaddr *)unp->unp_addr;
 		else
 			from = &sun_noname;
+		/*
+		 * sbappendaddr() will fail when the receiver runs out of
+		 * space; in contrast to SOCK_STREAM, we will lose messages
+		 * for the SOCK_DGRAM case when the receiver's queue overflows.
+		 * SB_UNIX on the socket buffer implies that the callee will
+		 * not free the control message, if any, because we would need
+		 * to call unp_dispose() on it.
+		 */
 		if (sbappendaddr(&so2->so_rcv, from, m, control, &error)) {
+			control = NULL;
 			sorwakeup(so2);
+		} else if (control != NULL && error == 0) {
+			/* A socket filter took control; don't touch it */
+			control = NULL;
 		}
-		m = 0;
-		control = 0;
+		m = NULL;
 		if (nam)
 			unp_disconnect(unp);
 		break;
@@ -377,23 +449,29 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 			panic("uipc_send connected but no connection?");
 		so2 = unp->unp_conn->unp_socket;
 		/*
-		 * Send to paired receive port, and then reduce
-		 * send buffer hiwater marks to maintain backpressure.
-		 * Wake up readers.
+		 * Send to paired receive port, and then reduce send buffer
+		 * hiwater marks to maintain backpressure.  Wake up readers.
+		 * SB_UNIX flag will allow new record to be appended to the
+		 * receiver's queue even when it is already full.  It is
+		 * possible, however, that append might fail.  In that case,
+		 * we will need to call unp_dispose() on the control message;
+		 * the callee will not free it since SB_UNIX is set.
 		 */
-		if ((control && sbappendcontrol(rcv, m, control, NULL)) ||
-			sbappend(rcv, m)) {
-			didreceive = 1;
-		}
-		snd->sb_mbmax -=
-			rcv->sb_mbcnt - unp->unp_conn->unp_mbcnt;
+		didreceive = control ?
+		    sbappendcontrol(rcv, m, control, &error) : sbappend(rcv, m);
+
+		snd->sb_mbmax -= rcv->sb_mbcnt - unp->unp_conn->unp_mbcnt;
 		unp->unp_conn->unp_mbcnt = rcv->sb_mbcnt;
 		snd->sb_hiwat -= rcv->sb_cc - unp->unp_conn->unp_cc;
 		unp->unp_conn->unp_cc = rcv->sb_cc;
-		if (didreceive)
+		if (didreceive) {
+			control = NULL;
 			sorwakeup(so2);
-		m = 0;
-		control = 0;
+		} else if (control != NULL && error == 0) {
+			/* A socket filter took control; don't touch it */
+			control = NULL;
+		}
+		m = NULL;
 #undef snd
 #undef rcv
 		}
@@ -412,59 +490,95 @@ uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 		unp_shutdown(unp);
 	}
 
-	if (control && error != 0)
+	if (control && error != 0) {
+		socket_unlock(so, 0);
 		unp_dispose(control);
+		socket_lock(so, 0);
+	}
 
 release:
 	if (control)
 		m_freem(control);
 	if (m)
 		m_freem(m);
-	return error;
+	return (error);
 }
 
 static int
-uipc_sense(struct socket *so, struct stat *sb)
+uipc_sense(struct socket *so, void *ub, int isstat64)
 {
 	struct unpcb *unp = sotounpcb(so);
 	struct socket *so2;
+	blksize_t blksize;
 
 	if (unp == 0)
-		return EINVAL;
-	sb->st_blksize = so->so_snd.sb_hiwat;
+		return (EINVAL);
+
+	blksize = so->so_snd.sb_hiwat;
 	if (so->so_type == SOCK_STREAM && unp->unp_conn != 0) {
 		so2 = unp->unp_conn->unp_socket;
-		sb->st_blksize += so2->so_rcv.sb_cc;
+		blksize += so2->so_rcv.sb_cc;
 	}
-	sb->st_dev = NODEV;
 	if (unp->unp_ino == 0)
 		unp->unp_ino = unp_ino++;
-	sb->st_ino = unp->unp_ino;
+
+	if (isstat64 != 0) {
+		struct stat64  *sb64;
+
+		sb64 = (struct stat64 *)ub;
+		sb64->st_blksize = blksize;
+		sb64->st_dev = NODEV;
+		sb64->st_ino = (ino64_t)unp->unp_ino;
+	} else {
+		struct stat *sb;
+
+		sb = (struct stat *)ub;
+		sb->st_blksize = blksize;
+		sb->st_dev = NODEV;
+		sb->st_ino = (ino_t)unp->unp_ino;
+	}
+
 	return (0);
 }
 
+/*
+ * Returns:	0		Success
+ *		EINVAL
+ *
+ * Notes:	This is not strictly correct, as unp_shutdown() also calls
+ *		socantrcvmore().  These should maybe both be conditionalized
+ *		on the 'how' argument in soshutdown() as called from the
+ *		shutdown() system call.
+ */
 static int
 uipc_shutdown(struct socket *so)
 {
 	struct unpcb *unp = sotounpcb(so);
 
 	if (unp == 0)
-		return EINVAL;
+		return (EINVAL);
 	socantsendmore(so);
 	unp_shutdown(unp);
-	return 0;
+	return (0);
 }
 
+/*
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ */
 static int
 uipc_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	struct unpcb *unp = sotounpcb(so);
 
-	if (unp == 0)
-		return EINVAL;
-	if (unp->unp_addr)
+	if (unp == NULL)
+		return (EINVAL);
+	if (unp->unp_addr != NULL) {
 		*nam = dup_sockaddr((struct sockaddr *)unp->unp_addr, 1);
-	return 0;
+	} else {
+		*nam = dup_sockaddr((struct sockaddr *)&sun_noname, 1);
+	}
+	return (0);
 }
 
 struct pr_usrreqs uipc_usrreqs = {
@@ -476,9 +590,7 @@ struct pr_usrreqs uipc_usrreqs = {
 };
 
 int
-uipc_ctloutput(
-	struct socket *so,
-	struct sockopt *sopt)
+uipc_ctloutput(struct socket *so, struct sockopt *sopt)
 {
 	struct unpcb *unp = sotounpcb(so);
 	int error;
@@ -487,10 +599,10 @@ uipc_ctloutput(
 	case SOPT_GET:
 		switch (sopt->sopt_name) {
 		case LOCAL_PEERCRED:
-			if (unp->unp_flags & UNP_HAVEPC)
+			if (unp->unp_flags & UNP_HAVEPC) {
 				error = sooptcopyout(sopt, &unp->unp_peercred,
-				    sizeof(unp->unp_peercred));
-			else {
+				    sizeof (unp->unp_peercred));
+			} else {
 				if (so->so_type == SOCK_STREAM)
 					error = ENOTCONN;
 				else
@@ -509,7 +621,7 @@ uipc_ctloutput(
 	}
 	return (error);
 }
-	
+
 /*
  * Both send and receive buffers are allocated PIPSIZ bytes of buffering
  * for stream sockets, although the total for sender and receiver is
@@ -527,20 +639,26 @@ static u_long	unpdg_sendspace = 2*1024;	/* really max datagram size */
 static u_long	unpdg_recvspace = 4*1024;
 
 static int	unp_rights;			/* file descriptors in flight */
+static int	unp_disposed;			/* discarded file descriptors */
 
 SYSCTL_DECL(_net_local_stream);
-SYSCTL_INT(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW, 
-	   &unpst_sendspace, 0, "");
+SYSCTL_INT(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW,
+   &unpst_sendspace, 0, "");
 SYSCTL_INT(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW,
-	   &unpst_recvspace, 0, "");
+   &unpst_recvspace, 0, "");
 SYSCTL_DECL(_net_local_dgram);
 SYSCTL_INT(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW,
-	   &unpdg_sendspace, 0, "");
+   &unpdg_sendspace, 0, "");
 SYSCTL_INT(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW,
-	   &unpdg_recvspace, 0, "");
+   &unpdg_recvspace, 0, "");
 SYSCTL_DECL(_net_local);
 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, "");
 
+/*
+ * Returns:	0			Success
+ *		ENOBUFS
+ *	soreserve:ENOBUFS
+ */
 static int
 unp_attach(struct socket *so)
 {
@@ -564,18 +682,38 @@ unp_attach(struct socket *so)
 		if (error)
 			return (error);
 	}
-	unp = (struct unpcb*)zalloc(unp_zone);
+	unp = (struct unpcb *)zalloc(unp_zone);
 	if (unp == NULL)
 		return (ENOBUFS);
-	bzero(unp, sizeof *unp);
+	bzero(unp, sizeof (*unp));
 	lck_rw_lock_exclusive(unp_list_mtx);
 	LIST_INIT(&unp->unp_refs);
 	unp->unp_socket = so;
 	unp->unp_gencnt = ++unp_gencnt;
 	unp_count++;
-	LIST_INSERT_HEAD(so->so_type == SOCK_DGRAM ? &unp_dhead
-			 : &unp_shead, unp, unp_link);
+	LIST_INSERT_HEAD(so->so_type == SOCK_DGRAM ?
+	    &unp_dhead : &unp_shead, unp, unp_link);
 	so->so_pcb = (caddr_t)unp;
+	/*
+	 * Mark AF_UNIX socket buffers accordingly so that:
+	 *
+	 * a. In the SOCK_STREAM case, socket buffer append won't fail due to
+	 *    the lack of space; this essentially loosens the sbspace() check,
+	 *    since there is disconnect between sosend() and uipc_send() with
+	 *    respect to flow control that might result in our dropping the
+	 *    data in uipc_send().  By setting this, we allow for slightly
+	 *    more records to be appended to the receiving socket to avoid
+	 *    losing data (which we can't afford in the SOCK_STREAM case).
+	 *    Flow control still takes place since we adjust the sender's
+	 *    hiwat during each send.  This doesn't affect the SOCK_DGRAM
+	 *    case and append would still fail when the queue overflows.
+	 *
+	 * b. In the presence of control messages containing internalized
+	 *    file descriptors, the append routines will not free them since
+	 *    we'd need to undo the work first via unp_dispose().
+	 */
+	so->so_rcv.sb_flags |= SB_UNIX;
+	so->so_snd.sb_flags |= SB_UNIX;
 	lck_rw_done(unp_list_mtx);
 	return (0);
 }
@@ -590,8 +728,8 @@ unp_detach(struct unpcb *unp)
 	--unp_count;
 	if (unp->unp_vnode) {
 		struct vnode *tvp = unp->unp_vnode;
-		unp->unp_vnode->v_socket = 0;
-		unp->unp_vnode = 0;
+		unp->unp_vnode->v_socket = NULL;
+		unp->unp_vnode = NULL;
 		vnode_rele(tvp);		/* drop the usecount */
 	}
 	if (unp->unp_conn)
@@ -599,8 +737,9 @@ unp_detach(struct unpcb *unp)
 	while (unp->unp_refs.lh_first)
 		unp_drop(unp->unp_refs.lh_first, ECONNRESET);
 	soisdisconnected(unp->unp_socket);
-	unp->unp_socket->so_flags |= SOF_PCBCLEARING; /* makes sure we're getting dealloced */
-	unp->unp_socket->so_pcb = 0;
+	/* makes sure we're getting dealloced */
+	unp->unp_socket->so_flags |= SOF_PCBCLEARING;
+	unp->unp_socket->so_pcb = NULL;
 	if (unp_rights) {
 		/*
 		 * Normally the receive buffer is flushed later,
@@ -617,47 +756,54 @@ unp_detach(struct unpcb *unp)
 	zfree(unp_zone, unp);
 }
 
+/*
+ * Returns:	0			Success
+ *		EAFNOSUPPORT
+ *		EINVAL
+ *		EADDRINUSE
+ *		namei:???		[anything namei can return]
+ *		vnode_authorize:???	[anything vnode_authorize can return]
+ *
+ * Notes:	p at this point is the current process, as this function is
+ *		only called by sobind().
+ */
 static int
 unp_bind(
 	struct unpcb *unp,
 	struct sockaddr *nam,
-	struct proc *p)
+	proc_t p)
 {
 	struct sockaddr_un *soun = (struct sockaddr_un *)nam;
 	struct vnode *vp, *dvp;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error, namelen;
 	struct nameidata nd;
 	char buf[SOCK_MAXADDRLEN];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
+	if (nam->sa_family != 0 && nam->sa_family != AF_UNIX) {
+		return (EAFNOSUPPORT);
+	}
 
-	if (unp->unp_vnode != NULL) {
-		kauth_cred_unref(&context.vc_ucred);
+	if (unp->unp_vnode != NULL)
 		return (EINVAL);
-	}
 	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
-	if (namelen <= 0) {
-		kauth_cred_unref(&context.vc_ucred);
-		return EINVAL;
-	}
-	strncpy(buf, soun->sun_path, namelen);
-	buf[namelen] = 0;	/* null-terminate the string */
+	if (namelen <= 0)
+		return (EINVAL);
+
+	strlcpy(buf, soun->sun_path, namelen+1);
 	NDINIT(&nd, CREATE, FOLLOW | LOCKPARENT, UIO_SYSSPACE32,
-	    CAST_USER_ADDR_T(buf), &context);
-/* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
+	    CAST_USER_ADDR_T(buf), ctx);
+	/* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
 	error = namei(&nd);
 	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	dvp = nd.ni_dvp;
 	vp = nd.ni_vp;
 
 	if (vp != NULL) {
-	        /*
+		/*
 		 * need to do this before the vnode_put of dvp
 		 * since we may have to release an fs_nodelock
 		 */
@@ -666,27 +812,35 @@ unp_bind(
 		vnode_put(dvp);
 		vnode_put(vp);
 
-		kauth_cred_unref(&context.vc_ucred);
 		return (EADDRINUSE);
 	}
 
+	VATTR_INIT(&va);
+	VATTR_SET(&va, va_type, VSOCK);
+	VATTR_SET(&va, va_mode, (ACCESSPERMS & ~p->p_fd->fd_cmask));
+
+#if CONFIG_MACF_SOCKET
+	/*
+	 * This is #if MAC_SOCKET, because it affects the connection rate
+	 * of Unix domain dockets that is critical for server performance
+	 */
+	error = mac_vnode_check_create(ctx,
+	    nd.ni_dvp, &nd.ni_cnd, &va);
+
+	if (error == 0)
+#endif /* MAC_SOCKET */
 	/* authorize before creating */
-	error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context);
+	error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx);
 
 	if (!error) {
-		VATTR_INIT(&va);
-		VATTR_SET(&va, va_type, VSOCK);
-		VATTR_SET(&va, va_mode, (ACCESSPERMS & ~p->p_fd->fd_cmask));
-
 		/* create the socket */
-		error = vn_create(dvp, &vp, &nd.ni_cnd, &va, 0, &context);
+		error = vn_create(dvp, &vp, &nd.ni_cnd, &va, 0, ctx);
 	}
-	
+
 	nameidone(&nd);
 	vnode_put(dvp);
 
 	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	vnode_ref(vp);	/* gain a longterm reference */
@@ -695,41 +849,54 @@ unp_bind(
 	unp->unp_addr = (struct sockaddr_un *)dup_sockaddr(nam, 1);
 	vnode_put(vp);		/* drop the iocount */
 
-	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 }
 
+
+/*
+ * Returns:	0			Success
+ *		EAFNOSUPPORT		Address family not supported
+ *		EINVAL			Invalid argument
+ *		ENOTSOCK		Not a socket
+ *		ECONNREFUSED		Connection refused
+ *		EPROTOTYPE		Protocol wrong type for socket
+ *		EISCONN			Socket is connected
+ *	unp_connect2:EPROTOTYPE		Protocol wrong type for socket
+ *	unp_connect2:EINVAL		Invalid argument
+ *	namei:???			[anything namei can return]
+ *	vnode_authorize:????		[anything vnode_authorize can return]
+ *
+ * Notes:	p at this point is the current process, as this function is
+ *		only called by sosend(), sendfile(), and soconnectlock().
+ */
 static int
-unp_connect(
-	struct socket *so,
-	struct sockaddr *nam,
-	struct proc *p)
+unp_connect(struct socket *so, struct sockaddr *nam, __unused proc_t p)
 {
 	struct sockaddr_un *soun = (struct sockaddr_un *)nam;
 	struct vnode *vp;
 	struct socket *so2, *so3;
 	struct unpcb *unp, *unp2, *unp3;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error, len;
 	struct nameidata nd;
 	char buf[SOCK_MAXADDRLEN];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
+	if (nam->sa_family != 0 && nam->sa_family != AF_UNIX) {
+		return (EAFNOSUPPORT);
+	}
+
 	so2 = so3 = NULL;
 
 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
-	if (len <= 0) {
-		kauth_cred_unref(&context.vc_ucred);
-		return EINVAL;
-	}
-	strncpy(buf, soun->sun_path, len);
-	buf[len] = 0;
+	if (len <= 0)
+		return (EINVAL);
+
+	strlcpy(buf, soun->sun_path, len+1);
 
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32, CAST_USER_ADDR_T(buf), &context);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
+	    CAST_USER_ADDR_T(buf), ctx);
 	error = namei(&nd);
 	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 	nameidone(&nd);
@@ -739,11 +906,11 @@ unp_connect(
 		goto bad;
 	}
 
-	error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, &context);
+	error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, ctx);
 	if (error)
 		goto bad;
 	so2 = vp->v_socket;
-	if (so2 == 0 || so2->so_pcb == NULL ) {
+	if (so2 == 0 || so2->so_pcb == NULL) {
 		error = ECONNREFUSED;
 		goto bad;
 	}
@@ -755,7 +922,7 @@ unp_connect(
 		error = EPROTOTYPE;
 		goto bad;
 	}
-	
+
 	/*
 	 * Check if socket was connected while we were trying to
 	 * acquire the funnel.
@@ -765,7 +932,7 @@ unp_connect(
 		error = EISCONN;
 		goto bad;
 	}
-	
+
 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
 		if ((so2->so_options & SO_ACCEPTCONN) == 0 ||
 		    (so3 = sonewconn(so2, 0, nam)) == 0) {
@@ -777,8 +944,7 @@ unp_connect(
 		unp3 = sotounpcb(so3);
 		if (unp2->unp_addr)
 			unp3->unp_addr = (struct sockaddr_un *)
-				dup_sockaddr((struct sockaddr *)
-					     unp2->unp_addr, 1);
+			    dup_sockaddr((struct sockaddr *)unp2->unp_addr, 1);
 
 		/*
 		 * unp_peercred management:
@@ -787,7 +953,7 @@ unp_connect(
 		 * from its process structure at the time of connect()
 		 * (which is now).
 		 */
-		cru2x(context.vc_ucred, &unp3->unp_peercred);
+		cru2x(vfs_context_ucred(ctx), &unp3->unp_peercred);
 		unp3->unp_flags |= UNP_HAVEPC;
 		/*
 		 * The receiver's (server's) credentials are copied
@@ -799,28 +965,34 @@ unp_connect(
 		KASSERT(unp2->unp_flags & UNP_HAVEPCCACHED,
 		    ("unp_connect: listener without cached peercred"));
 		memcpy(&unp->unp_peercred, &unp2->unp_peercred,
-		    sizeof(unp->unp_peercred));
+		    sizeof (unp->unp_peercred));
 		unp->unp_flags |= UNP_HAVEPC;
 
+#if CONFIG_MACF_SOCKET
+		/* XXXMAC: recursive lock: SOCK_LOCK(so); */
+		mac_socketpeer_label_associate_socket(so, so3);
+		mac_socketpeer_label_associate_socket(so3, so);
+		/* XXXMAC: SOCK_UNLOCK(so); */
+#endif /* MAC_SOCKET */
 		so2->so_usecount--; /* drop reference taken on so2 */
 		so2 = so3;
 		so3->so_usecount++; /* make sure we keep it around */
 	}
 	error = unp_connect2(so, so2);
 bad:
-	 
-	if (so2 != NULL) 
-			so2->so_usecount--;	/* release count on socket */
-	
+	if (so2 != NULL)
+		so2->so_usecount--; /* release count on socket */
 	vnode_put(vp);
-	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		EPROTOTYPE		Protocol wrong type for socket
+ *		EINVAL			Invalid argument
+ */
 int
-unp_connect2(
-	struct socket *so,
-	struct socket *so2)
+unp_connect2(struct socket *so, struct socket *so2)
 {
 	struct unpcb *unp = sotounpcb(so);
 	struct unpcb *unp2;
@@ -842,14 +1014,15 @@ unp_connect2(
 		break;
 
 	case SOCK_STREAM:
-                /* This takes care of socketpair */
-                if (!(unp->unp_flags & UNP_HAVEPC) && !(unp2->unp_flags & UNP_HAVEPC)) {
-                        cru2x(kauth_cred_get(), &unp->unp_peercred);
-                        unp->unp_flags |= UNP_HAVEPC;
-                        
-                        cru2x(kauth_cred_get(), &unp2->unp_peercred);
-                        unp2->unp_flags |= UNP_HAVEPC;
-                }
+		/* This takes care of socketpair */
+		if (!(unp->unp_flags & UNP_HAVEPC) &&
+		    !(unp2->unp_flags & UNP_HAVEPC)) {
+			cru2x(kauth_cred_get(), &unp->unp_peercred);
+			unp->unp_flags |= UNP_HAVEPC;
+
+			cru2x(kauth_cred_get(), &unp2->unp_peercred);
+			unp2->unp_flags |= UNP_HAVEPC;
+		}
 		unp2->unp_conn = unp;
 		soisconnected(so);
 		soisconnected(so2);
@@ -868,7 +1041,7 @@ unp_disconnect(struct unpcb *unp)
 
 	if (unp2 == 0)
 		return;
-	unp->unp_conn = 0;
+	unp->unp_conn = NULL;
 	switch (unp->unp_socket->so_type) {
 
 	case SOCK_DGRAM:
@@ -880,7 +1053,7 @@ unp_disconnect(struct unpcb *unp)
 
 	case SOCK_STREAM:
 		soisdisconnected(unp->unp_socket);
-		unp2->unp_conn = 0;
+		unp2->unp_conn = NULL;
 		soisdisconnected(unp2->unp_socket);
 		break;
 	}
@@ -898,6 +1071,7 @@ unp_abort(struct unpcb *unp)
 static int
 unp_pcblist SYSCTL_HANDLER_ARGS
 {
+#pragma unused(oidp,arg2)
 	int error, i, n;
 	struct unpcb *unp, **unp_list;
 	unp_gen_t gencnt;
@@ -913,15 +1087,15 @@ unp_pcblist SYSCTL_HANDLER_ARGS
 	 */
 	if (req->oldptr == USER_ADDR_NULL) {
 		n = unp_count;
-		req->oldidx = 2 * (sizeof xug)
-			+ (n + n/8) * sizeof(struct xunpcb);
+		req->oldidx = 2 * sizeof (xug) + (n + n / 8) *
+		    sizeof (struct xunpcb);
 		lck_rw_done(unp_list_mtx);
-		return 0;
+		return (0);
 	}
 
 	if (req->newptr != USER_ADDR_NULL) {
 		lck_rw_done(unp_list_mtx);
-		return EPERM;
+		return (EPERM);
 	}
 
 	/*
@@ -930,33 +1104,34 @@ unp_pcblist SYSCTL_HANDLER_ARGS
 	gencnt = unp_gencnt;
 	n = unp_count;
 
-	bzero(&xug, sizeof(xug));
-	xug.xug_len = sizeof xug;
+	bzero(&xug, sizeof (xug));
+	xug.xug_len = sizeof (xug);
 	xug.xug_count = n;
 	xug.xug_gen = gencnt;
 	xug.xug_sogen = so_gencnt;
-	error = SYSCTL_OUT(req, &xug, sizeof xug);
+	error = SYSCTL_OUT(req, &xug, sizeof (xug));
 	if (error) {
 		lck_rw_done(unp_list_mtx);
-		return error;
+		return (error);
 	}
 
 	/*
 	 * We are done if there is no pcb
 	 */
 	if (n == 0)  {
-	    lck_rw_done(unp_list_mtx);
-	    return 0;
+		lck_rw_done(unp_list_mtx);
+		return (0);
 	}
 
-	MALLOC(unp_list, struct unpcb **, n * sizeof *unp_list, M_TEMP, M_WAITOK);
+	MALLOC(unp_list, struct unpcb **, n * sizeof (*unp_list),
+	    M_TEMP, M_WAITOK);
 	if (unp_list == 0) {
 		lck_rw_done(unp_list_mtx);
-		return ENOMEM;
+		return (ENOMEM);
 	}
-	
+
 	for (unp = head->lh_first, i = 0; unp && i < n;
-	     unp = unp->unp_link.le_next) {
+	    unp = unp->unp_link.le_next) {
 		if (unp->unp_gencnt <= gencnt)
 			unp_list[i++] = unp;
 	}
@@ -968,23 +1143,23 @@ unp_pcblist SYSCTL_HANDLER_ARGS
 		if (unp->unp_gencnt <= gencnt) {
 			struct xunpcb xu;
 
-			bzero(&xu, sizeof(xu));
-			xu.xu_len = sizeof xu;
+			bzero(&xu, sizeof (xu));
+			xu.xu_len = sizeof (xu);
 			xu.xu_unpp = (struct  unpcb_compat *)unp;
 			/*
 			 * XXX - need more locking here to protect against
 			 * connect/disconnect races for SMP.
 			 */
 			if (unp->unp_addr)
-				bcopy(unp->unp_addr, &xu.xu_addr, 
-				      unp->unp_addr->sun_len);
+				bcopy(unp->unp_addr, &xu.xu_addr,
+				    unp->unp_addr->sun_len);
 			if (unp->unp_conn && unp->unp_conn->unp_addr)
 				bcopy(unp->unp_conn->unp_addr,
-				      &xu.xu_caddr,
-				      unp->unp_conn->unp_addr->sun_len);
-			bcopy(unp, &xu.xu_unp, sizeof(xu.xu_unp));
+				    &xu.xu_caddr,
+				    unp->unp_conn->unp_addr->sun_len);
+			bcopy(unp, &xu.xu_unp, sizeof (xu.xu_unp));
 			sotoxsocket(unp->unp_socket, &xu.xu_socket);
-			error = SYSCTL_OUT(req, &xu, sizeof xu);
+			error = SYSCTL_OUT(req, &xu, sizeof (xu));
 		}
 	}
 	if (!error) {
@@ -995,22 +1170,22 @@ unp_pcblist SYSCTL_HANDLER_ARGS
 		 * while we were processing this request, and it
 		 * might be necessary to retry.
 		 */
-		bzero(&xug, sizeof(xug));
-		xug.xug_len = sizeof xug;
+		bzero(&xug, sizeof (xug));
+		xug.xug_len = sizeof (xug);
 		xug.xug_gen = unp_gencnt;
 		xug.xug_sogen = so_gencnt;
 		xug.xug_count = unp_count;
-		error = SYSCTL_OUT(req, &xug, sizeof xug);
+		error = SYSCTL_OUT(req, &xug, sizeof (xug));
 	}
 	FREE(unp_list, M_TEMP);
 	lck_rw_done(unp_list_mtx);
-	return error;
+	return (error);
 }
 
-SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, CTLFLAG_RD, 
+SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, CTLFLAG_RD,
 	    (caddr_t)(long)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb",
 	    "List of active local datagram sockets");
-SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, CTLFLAG_RD, 
+SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, CTLFLAG_RD,
 	    (caddr_t)(long)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb",
 	    "List of active local stream sockets");
 
@@ -1025,9 +1200,7 @@ unp_shutdown(struct unpcb *unp)
 }
 
 static void
-unp_drop(
-	struct unpcb *unp,
-	int errno)
+unp_drop(struct unpcb *unp, int errno)
 {
 	struct socket *so = unp->unp_socket;
 
@@ -1043,16 +1216,21 @@ unp_drain()
 }
 #endif
 
+/*
+ * Returns:	0			Success
+ *		EMSGSIZE		The new fd's will not fit
+ *		ENOBUFS			Cannot alloc struct fileproc
+ */
 int
 unp_externalize(struct mbuf *rights)
 {
-	struct proc *p = current_proc();		/* XXX */
+	proc_t p = current_proc();		/* XXX */
 	int i;
 	struct cmsghdr *cm = mtod(rights, struct cmsghdr *);
 	struct fileglob **rp = (struct fileglob **)(cm + 1);
 	struct fileproc *fp;
 	struct fileglob *fg;
-	int newfds = (cm->cmsg_len - sizeof(*cm)) / sizeof (int);
+	int newfds = (cm->cmsg_len - sizeof (*cm)) / sizeof (int);
 	int f;
 
 	proc_fdlock(p);
@@ -1064,29 +1242,45 @@ unp_externalize(struct mbuf *rights)
 		for (i = 0; i < newfds; i++) {
 			fg = *rp;
 			unp_discard_fdlocked(fg, p);
-			*rp++ = 0;
+			*rp++ = NULL;
 		}
 		proc_fdunlock(p);
 
 		return (EMSGSIZE);
 	}
 	/*
-	 * now change each pointer to an fd in the global table to 
+	 * now change each pointer to an fd in the global table to
 	 * an integer that is the index to the local fd table entry
 	 * that we set up to point to the global one we are transferring.
-	 * XXX this assumes a pointer and int are the same size...!
+	 * XXX (1) this assumes a pointer and int are the same size...!
+	 * XXX (2) allocation failures should be non-fatal
 	 */
 	for (i = 0; i < newfds; i++) {
+#if CONFIG_MACF_SOCKET
+		/*
+		 * If receive access is denied, don't pass along
+		 * and error message, just discard the descriptor.
+		 */
+		if (mac_file_check_receive(kauth_cred_get(), *rp)) {
+			fg = *rp;
+			*rp++ = 0;
+			unp_discard_fdlocked(fg, p);
+			continue;
+		}
+#endif
 		if (fdalloc(p, 0, &f))
-			panic("unp_externalize");
+			panic("unp_externalize:fdalloc");
 		fg = *rp;
-		MALLOC_ZONE(fp, struct fileproc *, sizeof(struct fileproc), M_FILEPROC, M_WAITOK);
-		bzero(fp, sizeof(struct fileproc));
+		MALLOC_ZONE(fp, struct fileproc *, sizeof (struct fileproc),
+		    M_FILEPROC, M_WAITOK);
+		if (fp == NULL)
+			panic("unp_externalize: MALLOC_ZONE");
+		bzero(fp, sizeof (struct fileproc));
 		fp->f_iocount = 0;
 		fp->f_fglob = fg;
 		fg_removeuipc(fg);
 		procfdtbl_releasefd(p, f, fp);
-		unp_rights--;
+		(void) OSAddAtomic(-1, (volatile SInt32 *)&unp_rights);
 		*(int *)rp++ = f;
 	}
 	proc_fdunlock(p);
@@ -1097,47 +1291,51 @@ unp_externalize(struct mbuf *rights)
 void
 unp_init(void)
 {
-	unp_zone = zinit(sizeof(struct unpcb), 
-			 (nmbclusters * sizeof(struct unpcb)), 
-			  4096, "unpzone");
+	unp_zone = zinit(sizeof (struct unpcb),
+	    (nmbclusters * sizeof (struct unpcb)), 4096, "unpzone");
+
 	if (unp_zone == 0)
 		panic("unp_init");
 	LIST_INIT(&unp_dhead);
 	LIST_INIT(&unp_shead);
-	
+
 	/*
 	 * allocate lock group attribute and group for udp pcb mutexes
 	 */
 	unp_mtx_grp_attr = lck_grp_attr_alloc_init();
 
 	unp_mtx_grp = lck_grp_alloc_init("unp_list", unp_mtx_grp_attr);
-		
+
 	unp_mtx_attr = lck_attr_alloc_init();
 
-	if ((unp_list_mtx = lck_rw_alloc_init(unp_mtx_grp, unp_mtx_attr)) == NULL)
+	if ((unp_list_mtx = lck_rw_alloc_init(unp_mtx_grp,
+	    unp_mtx_attr)) == NULL)
 		return;	/* pretty much dead if this fails... */
 
 }
 
 #ifndef MIN
-#define	MIN(a,b) (((a)<(b))?(a):(b))
+#define	MIN(a, b) (((a) < (b)) ? (a) : (b))
 #endif
 
+/*
+ * Returns:	0			Success
+ *		EINVAL
+ *	fdgetf_noref:EBADF
+ */
 static int
-unp_internalize(
-	struct mbuf *control,
-	struct proc *p)
+unp_internalize(struct mbuf *control, proc_t p)
 {
 	struct cmsghdr *cm = mtod(control, struct cmsghdr *);
 	struct fileglob **rp;
 	struct fileproc *fp;
-	register int i, error;
+	int i, error;
 	int oldfds;
-	int fdgetf_noref(proc_t, struct fileglob **, struct fileproc **);
 
+	/* 64bit: cmsg_len is 'uint32_t', m_len is 'long' */
 	if (cm->cmsg_type != SCM_RIGHTS || cm->cmsg_level != SOL_SOCKET ||
-	    cm->cmsg_len != control->m_len) {
-	     return (EINVAL);
+	    (unsigned long)cm->cmsg_len != (unsigned long)control->m_len) {
+		return (EINVAL);
 	}
 	oldfds = (cm->cmsg_len - sizeof (*cm)) / sizeof (int);
 
@@ -1145,10 +1343,10 @@ unp_internalize(
 	rp = (struct fileglob **)(cm + 1);
 
 	for (i = 0; i < oldfds; i++) {
-	     if (error = fdgetf_noref(p, *(int *)rp++, (struct fileglob **)0)) {
-	             proc_fdunlock(p);
-		     return (error);
-	     }
+		if ((error = fdgetf_noref(p, *(int *)rp++, NULL)) != 0) {
+			proc_fdunlock(p);
+			return (error);
+		}
 	}
 	rp = (struct fileglob **)(cm + 1);
 
@@ -1156,7 +1354,7 @@ unp_internalize(
 		(void) fdgetf_noref(p, *(int *)rp, &fp);
 		fg_insertuipc(fp->f_fglob);
 		*rp++ = fp->f_fglob;
-		unp_rights++;
+		(void) OSAddAtomic(1, (volatile SInt32 *)&unp_rights);
 	}
 	proc_fdunlock(p);
 
@@ -1164,6 +1362,7 @@ unp_internalize(
 }
 
 static int	unp_defer, unp_gcing, unp_gcwait;
+
 /* always called under uipc_lock */
 void
 unp_gc_wait(void)
@@ -1174,15 +1373,16 @@ unp_gc_wait(void)
 	}
 }
 
+
 static void
-unp_gc()
+unp_gc(void)
 {
-	register struct fileglob *fg, *nextfg;
-	register struct socket *so;
+	struct fileglob *fg, *nextfg;
+	struct socket *so;
 	struct fileglob **extra_ref, **fpp;
 	int nunref, i;
 	int need_gcwakeup = 0;
-	
+
 	lck_mtx_lock(uipc_lock);
 	if (unp_gcing) {
 		lck_mtx_unlock(uipc_lock);
@@ -1191,8 +1391,8 @@ unp_gc()
 	unp_gcing = 1;
 	unp_defer = 0;
 	lck_mtx_unlock(uipc_lock);
-	/* 
-	 * before going through all this, set all FDs to 
+	/*
+	 * before going through all this, set all FDs to
 	 * be NOT defered and NOT externally accessible
 	 */
 	for (fg = fmsghead.lh_first; fg != 0; fg = fg->f_msglist.le_next) {
@@ -1201,7 +1401,8 @@ unp_gc()
 		lck_mtx_unlock(&fg->fg_lock);
 	}
 	do {
-		for (fg = fmsghead.lh_first; fg != 0; fg = fg->f_msglist.le_next) {
+		for (fg = fmsghead.lh_first; fg != 0;
+		    fg = fg->f_msglist.le_next) {
 			lck_mtx_lock(&fg->fg_lock);
 			/*
 			 * If the file is not open, skip it
@@ -1223,30 +1424,30 @@ unp_gc()
 				 * if it's not defered, then check if it's
 				 * already marked.. if so skip it
 				 */
-				if (fg->fg_flag & FMARK){
+				if (fg->fg_flag & FMARK) {
 					lck_mtx_unlock(&fg->fg_lock);
 					continue;
 				}
-				/* 
+				/*
 				 * If all references are from messages
-				 * in transit, then skip it. it's not 
+				 * in transit, then skip it. it's not
 				 * externally accessible.
-				 */ 
+				 */
 				if (fg->fg_count == fg->fg_msgcount) {
 					lck_mtx_unlock(&fg->fg_lock);
 					continue;
 				}
-				/* 
+				/*
 				 * If it got this far then it must be
 				 * externally accessible.
 				 */
 				fg->fg_flag |= FMARK;
 			}
 			/*
-			 * either it was defered, or it is externally 
+			 * either it was defered, or it is externally
 			 * accessible and not already marked so.
 			 * Now check if it is possibly one of OUR sockets.
-			 */ 
+			 */
 			if (fg->fg_type != DTYPE_SOCKET ||
 			    (so = (struct socket *)fg->fg_data) == 0) {
 				lck_mtx_unlock(&fg->fg_lock);
@@ -1258,7 +1459,10 @@ unp_gc()
 				continue;
 			}
 #ifdef notdef
-			/* if this code is enabled need to run under network funnel */
+			/*
+			 * if this code is enabled need to run
+			 * under network funnel
+			 */
 			if (so->so_rcv.sb_flags & SB_LOCK) {
 				/*
 				 * This is problematical; it's not clear
@@ -1278,7 +1482,7 @@ unp_gc()
 			 * So, Ok, it's one of our sockets and it IS externally
 			 * accessible (or was defered). Now we look
 			 * to see if we hold any file descriptors in its
-			 * message buffers. Follow those links and mark them 
+			 * message buffers. Follow those links and mark them
 			 * as accessible too.
 			 */
 			unp_scan(so->so_rcv.sb_mb, unp_mark);
@@ -1324,20 +1528,21 @@ unp_gc()
 	 *
 	 * 91/09/19, bsy@cs.cmu.edu
 	 */
-	extra_ref = _MALLOC(nfiles * sizeof(struct fileglob *), M_FILEGLOB, M_WAITOK);
+	extra_ref = _MALLOC(nfiles * sizeof (struct fileglob *),
+	    M_FILEGLOB, M_WAITOK);
 	for (nunref = 0, fg = fmsghead.lh_first, fpp = extra_ref; fg != 0;
 	    fg = nextfg) {
 		lck_mtx_lock(&fg->fg_lock);
 
 		nextfg = fg->f_msglist.le_next;
-		/* 
+		/*
 		 * If it's not open, skip it
 		 */
 		if (fg->fg_count == 0) {
 			lck_mtx_unlock(&fg->fg_lock);
 			continue;
 		}
-		/* 
+		/*
 		 * If all refs are from msgs, and it's not marked accessible
 		 * then it must be referenced from some unreachable cycle
 		 * of (shut-down) FDs, so include it in our
@@ -1350,7 +1555,7 @@ unp_gc()
 		}
 		lck_mtx_unlock(&fg->fg_lock);
 	}
-	/* 
+	/*
 	 * for each FD on our hit list, do the following two things
 	 */
 	for (i = nunref, fpp = extra_ref; --i >= 0; ++fpp) {
@@ -1359,12 +1564,26 @@ unp_gc()
 		tfg = *fpp;
 
 		if (tfg->fg_type == DTYPE_SOCKET && tfg->fg_data != NULL) {
-			sorflush((struct socket *)(tfg->fg_data));
+			int locked = 0;
+
+			so = (struct socket *)(tfg->fg_data);
+
+			/* XXXX */
+			/* Assume local sockets use a global lock */
+			if (so->so_proto->pr_domain->dom_family != PF_LOCAL) {
+				socket_lock(so, 0);
+				locked = 1;
+			}
+			sorflush(so);
+
+			if (locked)
+				socket_unlock(so, 0);
 		}
 	}
 	for (i = nunref, fpp = extra_ref; --i >= 0; ++fpp)
-		closef_locked((struct fileproc *)0, *fpp, (struct proc *) NULL);
-	lck_mtx_lock(uipc_lock);
+		closef_locked((struct fileproc *)0, *fpp, (proc_t)NULL);
+
+        lck_mtx_lock(uipc_lock);
 	unp_gcing = 0;
 
 	if (unp_gcwait != 0) {
@@ -1376,22 +1595,21 @@ unp_gc()
 	if (need_gcwakeup != 0)
 		wakeup(&unp_gcing);
 	FREE((caddr_t)extra_ref, M_FILEGLOB);
-
 }
 
 void
 unp_dispose(struct mbuf *m)
 {
-
 	if (m) {
 		unp_scan(m, unp_discard);
 	}
 }
 
+/*
+ * Returns:	0			Success
+ */
 static int
-unp_listen(
-	struct unpcb *unp,
-	struct proc *p)
+unp_listen(struct unpcb *unp, proc_t p)
 {
 	kauth_cred_t safecred = kauth_cred_proc_ref(p);
 	cru2x(safecred, &unp->unp_peercred);
@@ -1400,11 +1618,9 @@ unp_listen(
 	return (0);
 }
 
-/* should run under kernel funnel */ 
+/* should run under kernel funnel */
 static void
-unp_scan(
-	struct mbuf *m0,
-	void (*op)(struct fileglob *))
+unp_scan(struct mbuf *m0, void (*op)(struct fileglob *))
 {
 	struct mbuf *m;
 	struct fileglob **rp;
@@ -1415,13 +1631,13 @@ unp_scan(
 	while (m0) {
 		for (m = m0; m; m = m->m_next)
 			if (m->m_type == MT_CONTROL &&
-			    (size_t) m->m_len >= sizeof(*cm)) {
+			    (size_t)m->m_len >= sizeof (*cm)) {
 				cm = mtod(m, struct cmsghdr *);
 				if (cm->cmsg_level != SOL_SOCKET ||
 				    cm->cmsg_type != SCM_RIGHTS)
 					continue;
-				qfds = (cm->cmsg_len - sizeof *cm)
-						/ sizeof (struct fileglob *);
+				qfds = (cm->cmsg_len - sizeof (*cm)) /
+				    sizeof (struct fileglob *);
 				rp = (struct fileglob **)(cm + 1);
 				for (i = 0; i < qfds; i++)
 					(*op)(*rp++);
@@ -1435,38 +1651,36 @@ unp_scan(
 static void
 unp_mark(struct fileglob *fg)
 {
-        lck_mtx_lock(&fg->fg_lock);
+	lck_mtx_lock(&fg->fg_lock);
 
 	if (fg->fg_flag & FMARK) {
-	        lck_mtx_unlock(&fg->fg_lock);
+		lck_mtx_unlock(&fg->fg_lock);
 		return;
 	}
 	fg->fg_flag |= (FMARK|FDEFER);
 
-        lck_mtx_unlock(&fg->fg_lock);
+	lck_mtx_unlock(&fg->fg_lock);
 
 	unp_defer++;
 }
 
 /* should run under kernel funnel */
 static void
-unp_discard(fg)
-	struct fileglob *fg;
+unp_discard(struct fileglob *fg)
 {
-	struct proc *p = current_proc();		/* XXX */
+	proc_t p = current_proc();		/* XXX */
+
+	(void) OSAddAtomic(1, (volatile SInt32 *)&unp_disposed);
 
 	proc_fdlock(p);
 	unp_discard_fdlocked(fg, p);
 	proc_fdunlock(p);
 }
 static void
-unp_discard_fdlocked(fg, p)
-	struct fileglob *fg;
-	struct proc *p;
+unp_discard_fdlocked(struct fileglob *fg, proc_t p)
 {
-
 	fg_removeuipc(fg);
 
-	unp_rights--;
+	(void) OSAddAtomic(-1, (volatile SInt32 *)&unp_rights);
 	(void) closef_locked((struct fileproc *)0, fg, p);
 }
diff --git a/bsd/libkern/bcd.c b/bsd/libkern/bcd.c
index 02f3fde8b..a8cbec8ee 100644
--- a/bsd/libkern/bcd.c
+++ b/bsd/libkern/bcd.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Some data-tables that are often used.
diff --git a/bsd/libkern/bcmp.c b/bsd/libkern/bcmp.c
index e315fee7d..ea4d1d54f 100644
--- a/bsd/libkern/bcmp.c
+++ b/bsd/libkern/bcmp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1987, 1993
diff --git a/bsd/libkern/inet_ntop.c b/bsd/libkern/inet_ntop.c
index 03d64504a..dc0727318 100644
--- a/bsd/libkern/inet_ntop.c
+++ b/bsd/libkern/inet_ntop.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -57,7 +63,7 @@
 static const char *hexchars = "0123456789abcdef";
 
 static const char *
-inet_ntop4(const struct in_addr *addr, char *buf, size_t len)
+inet_ntop4(const struct in_addr *addr, char *buf, socklen_t len)
 {
 	const u_int8_t *ap = (const u_int8_t *)&addr->s_addr;
 	char tmp[MAX_IPv4_STR_LEN]; /* max length of ipv4 addr string */
@@ -80,7 +86,7 @@ inet_ntop4(const struct in_addr *addr, char *buf, size_t len)
 }
 
 static const char *
-inet_ntop6(const struct in6_addr *addr, char *dst, size_t size)
+inet_ntop6(const struct in6_addr *addr, char *dst, socklen_t size)
 {
 	char hexa[8][5], tmp[MAX_IPv6_STR_LEN];
 	int zr[8];
@@ -198,7 +204,7 @@ inet_ntop6(const struct in6_addr *addr, char *dst, size_t size)
 }
 
 const char *
-inet_ntop(int af, const void *addr, char *buf, size_t len)
+inet_ntop(int af, const void *addr, char *buf, socklen_t len)
 {
 	if(af==AF_INET6)
 		return inet_ntop6(addr, buf, len);
diff --git a/bsd/libkern/libkern.h b/bsd/libkern/libkern.h
index 6eee3e08c..c7588d418 100644
--- a/bsd/libkern/libkern.h
+++ b/bsd/libkern/libkern.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
@@ -53,16 +59,26 @@
  *
  *	@(#)libkern.h	8.1 (Berkeley) 6/10/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _LIBKERN_LIBKERN_H_
 #define _LIBKERN_LIBKERN_H_
 
 #include <sys/appleapiopts.h>
 #include <stdint.h>
+#include <stdarg.h>	/* for platform-specific va_list */
 #include <string.h>
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <mach/vm_param.h>
+#if __arm__
+#include <arm/arch.h> /* for _ARM_ARCH_* */
+#endif
 
 #ifdef __APPLE_API_OBSOLETE
 /* BCD conversions. */
@@ -121,6 +137,8 @@ ulmin(u_long a, u_long b)
 	return (a < b ? a : b);
 }
 
+
+
 /* Prototypes for non-quad routines. */
 extern int	ffs(int);
 extern int	locc(int, char *, u_int);
@@ -132,11 +150,18 @@ extern long	strtol(const char*, char **, int);
 extern u_long	strtoul(const char *, char **, int);
 extern quad_t	strtoq(const char *, char **, int);
 extern u_quad_t strtouq(const char *, char **, int);
+extern char	*strsep(char **stringp, const char *delim);
 
-int	snprintf(char *, size_t, const char *, ...);
-int	sprintf(char *bufp, const char *, ...);
-int	sscanf(const char *, char const *, ...);
-void printf(const char *, ...);
+int	snprintf(char *, size_t, const char *, ...) __printflike(3,4);
+
+/* sprintf() is being deprecated. Please use snprintf() instead. */
+int	sprintf(char *bufp, const char *, ...) __deprecated;
+int	sscanf(const char *, char const *, ...) __scanflike(2,3);
+int	printf(const char *, ...) __printflike(1,2);
+
+#if CONFIG_NO_PRINTF_STRINGS
+#define printf(x, ...)  do {} while (0)
+#endif
 
 uint32_t	crc32(uint32_t crc, const void *bufp, size_t len);
 
@@ -146,15 +171,62 @@ int	copyoutstr(const void *kaddr, user_addr_t udaddr, size_t len, size_t *done);
 int	copyin(const user_addr_t uaddr, void *kaddr, size_t len);
 int	copyout(const void *kaddr, user_addr_t udaddr, size_t len);
 
-int vsscanf(const char *, char const *, __darwin_va_list);
-extern int	vsnprintf(char *, size_t, const char *, __darwin_va_list);
-extern int	vsprintf(char *bufp, const char *, __darwin_va_list); 
+int vsscanf(const char *, char const *, va_list);
+
+extern int	vprintf(const char *, va_list);
+extern int	vsnprintf(char *, size_t, const char *, va_list);
+
+/* vsprintf() is being deprecated. Please use vsnprintf() instead. */
+extern int	vsprintf(char *bufp, const char *, va_list) __deprecated;
 
 extern void invalidate_icache(vm_offset_t, unsigned, int);
 extern void flush_dcache(vm_offset_t, unsigned, int);
 extern void invalidate_icache64(addr64_t, unsigned, int);
 extern void flush_dcache64(addr64_t, unsigned, int);
 
+
+/*
+ * assembly versions of clz... ideally we would just call
+ * __builtin_clz(num), unfortunately this one is ill defined
+ * by gcc for num=0
+ */
+static __inline__ unsigned int
+clz(unsigned int num)
+{
+#if __ppc__
+	unsigned int result;
+	__asm__ volatile(
+		"cntlzw %0, %1"
+		: "=r" (result)
+		: "r" (num)
+	);
+	return result;
+
+#elif __i386__
+	unsigned int result;
+	__asm__ volatile(
+		"bsrl   %1, %0\n\t"
+		"cmovel %2, %0"
+		: "=r" (result)
+		: "rm" (num), "r" (63)
+	);
+	return 31 ^ result;
+
+#elif __arm__ && !__thumb__ && defined(_ARM_ARCH_5)
+	unsigned int result;
+	__asm__ volatile(
+		"clz %0, %1"
+		: "=r" (result)
+		: "r" (num)
+	);
+
+	return result;
+#else
+	return num?__builtin_clz(num):__builtin_clz(0);
+#endif
+}
+
+
 __END_DECLS
 
 #endif /* _LIBKERN_LIBKERN_H_ */
diff --git a/bsd/libkern/locc.c b/bsd/libkern/locc.c
index 29572e4ee..2f84e6976 100644
--- a/bsd/libkern/locc.c
+++ b/bsd/libkern/locc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
diff --git a/bsd/libkern/random.c b/bsd/libkern/random.c
index 8a002af2c..920ae5c60 100644
--- a/bsd/libkern/random.c
+++ b/bsd/libkern/random.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
@@ -56,6 +62,8 @@
 
 #include <sys/types.h>
 #include <sys/random.h>
+#include <libkern/libkern.h>
+#include <dev/random/randomdev.h>
 
 /*
  * Pseudo-random number generator for randomizing the profiling clock,
@@ -63,8 +71,10 @@
  * [0, 2^31 - 1].
  */
 u_long
-random()
+random(void)
 {
-	return RandomULong();
+	/* Zero all but bottom 31 bits, also works for 64-bit longs */
+	u_long mask = (u_long)-1 >> ((sizeof(u_long) * 8) - 31);
+	return (mask & RandomULong());
 }
 
diff --git a/bsd/libkern/rindex.c b/bsd/libkern/rindex.c
index a8d4044a0..47a881199 100644
--- a/bsd/libkern/rindex.c
+++ b/bsd/libkern/rindex.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1993
diff --git a/bsd/libkern/scanc.c b/bsd/libkern/scanc.c
index 8eb93c384..4d2ec8f8b 100644
--- a/bsd/libkern/scanc.c
+++ b/bsd/libkern/scanc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
@@ -59,10 +65,10 @@
 int
 scanc(u_int size, u_char *cp, const u_char table[], int mask0)
 {
-	register u_char *end;
-	register u_char mask;
+	u_char *end;
+	u_char mask;
 
 	mask = mask0;
-	for (end = &cp[size]; cp < end && (table[*cp] & mask) == 0; ++cp);
+	for (end = &cp[size]; cp != end && (table[*cp] & mask) == 0; ++cp);
 	return (end - cp);
 }
diff --git a/bsd/libkern/skpc.c b/bsd/libkern/skpc.c
index 68110e8a5..78d886bb1 100644
--- a/bsd/libkern/skpc.c
+++ b/bsd/libkern/skpc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
@@ -57,10 +63,7 @@
 #include <libkern/libkern.h>
 
 int
-skpc(mask0, size, cp0)
-	int mask0;
-	int size;
-	char *cp0;
+skpc(int mask0, int size, char *cp0)
 {
 	register u_char *cp, *end, mask;
 
diff --git a/bsd/libkern/strsep.c b/bsd/libkern/strsep.c
new file mode 100644
index 000000000..75ef7cf1b
--- /dev/null
+++ b/bsd/libkern/strsep.c
@@ -0,0 +1,77 @@
+/*-
+ * Copyright (c) 1990, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strsep.c	8.1 (Berkeley) 6/4/93";
+#endif /* LIBC_SCCS and not lint */
+#include <sys/cdefs.h>
+/* __FBSDID("$FreeBSD: src/sys/libkern/strsep.c,v 1.7 2004/04/07 20:46:10 imp Exp $"); */
+
+#include <sys/param.h>
+#include <libkern/libkern.h>
+
+/*
+ * Get next token from string *stringp, where tokens are possibly-empty
+ * strings separated by characters from delim.
+ *
+ * Writes NULs into the string at *stringp to end tokens.
+ * delim need not remain constant from call to call.
+ * On return, *stringp points past the last NUL written (if there might
+ * be further tokens), or is NULL (if there are definitely no more tokens).
+ *
+ * If *stringp is NULL, strsep returns NULL.
+ */
+char *
+strsep(stringp, delim)
+	char **stringp;
+	const char *delim;
+{
+	char *s;
+	const char *spanp;
+	int c, sc;
+	char *tok;
+
+	if ((s = *stringp) == NULL)
+		return (NULL);
+	for (tok = s;;) {
+		c = *s++;
+		spanp = delim;
+		do {
+			if ((sc = *spanp++) == c) {
+				if (c == 0)
+					s = NULL;
+				else
+					s[-1] = 0;
+				*stringp = s;
+				return (tok);
+			}
+		} while (sc != 0);
+	}
+	/* NOTREACHED */
+}
diff --git a/bsd/libkern/strtol.c b/bsd/libkern/strtol.c
index 440d7e587..7fc24e1aa 100644
--- a/bsd/libkern/strtol.c
+++ b/bsd/libkern/strtol.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*      Copyright (c) 1995 NeXT Computer, Inc.  All rights reserved.
  *
diff --git a/bsd/machine/Makefile b/bsd/machine/Makefile
index 0304d6d01..0c7accd47 100644
--- a/bsd/machine/Makefile
+++ b/bsd/machine/Makefile
@@ -9,15 +9,17 @@ include $(MakeInc_def)
 
 
 DATAFILES = \
-	byte_order.h  endian.h \
+	byte_order.h  endian.h fasttrap_isa.h \
 	param.h  profile.h \
 	setjmp.h signal.h  types.h\
-	ucontext.h vmparam.h _types.h _limits.h
+	vmparam.h _structs.h _types.h _limits.h _param.h
+
 KERNELFILES = \
+	disklabel.h \
 	byte_order.h  endian.h \
 	param.h   profile.h \
 	signal.h spl.h types.h \
-	 vmparam.h _types.h _limits.h
+	vmparam.h _structs.h _types.h _limits.h _param.h
 
 
 INSTALL_MI_LIST = ${DATAFILES}
diff --git a/bsd/machine/_limits.h b/bsd/machine/_limits.h
index b200f635b..373e726f0 100644
--- a/bsd/machine/_limits.h
+++ b/bsd/machine/_limits.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE__LIMITS_H_
 #define _BSD_MACHINE__LIMITS_H_
@@ -26,6 +32,8 @@
 #include "ppc/_limits.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/_limits.h"
+#elif defined (__arm__)
+#include "arm/_limits.h"
 #else
 #error architecture not supported
 #endif
diff --git a/bsd/machine/_param.h b/bsd/machine/_param.h
new file mode 100644
index 000000000..4171fd03c
--- /dev/null
+++ b/bsd/machine/_param.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#if defined (__ppc__) || defined (__ppc64__)
+#include "ppc/_param.h"
+#elif defined (__i386__) || defined (__x86_64__)
+#include "i386/_param.h"
+#elif defined (__arm__)
+#include "arm/_param.h"
+#else
+#error architecture not supported
+#endif
diff --git a/bsd/machine/_structs.h b/bsd/machine/_structs.h
new file mode 100644
index 000000000..0e359387e
--- /dev/null
+++ b/bsd/machine/_structs.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#if defined (__ppc__) || defined (__ppc64__)
+#include "ppc/_structs.h"
+#elif defined (__i386__) || defined (__x86_64__)
+#include "i386/_structs.h"
+#elif defined (__arm__)
+#include "arm/_structs.h"
+#else
+#error architecture not supported
+#endif
diff --git a/bsd/machine/_types.h b/bsd/machine/_types.h
index 0f2de3369..9f2d6d255 100644
--- a/bsd/machine/_types.h
+++ b/bsd/machine/_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE__TYPES_H_
 #define _BSD_MACHINE__TYPES_H_
@@ -26,6 +32,8 @@
 #include "ppc/_types.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/_types.h"
+#elif defined (__arm__)
+#include "arm/_types.h"
 #else
 #error architecture not supported
 #endif
diff --git a/bsd/machine/byte_order.h b/bsd/machine/byte_order.h
index c100caa6f..4008142fe 100644
--- a/bsd/machine/byte_order.h
+++ b/bsd/machine/byte_order.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995 NeXT Computer, Inc.
diff --git a/bsd/machine/cons.h b/bsd/machine/cons.h
index 39f932b5e..38b771d86 100644
--- a/bsd/machine/cons.h
+++ b/bsd/machine/cons.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_CONS_H_
 #define _BSD_MACHINE_CONS_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include <dev/ppc/cons.h>
 #elif defined (__i386__) || defined(__x86_64__)
 #include <dev/i386/cons.h>
+#elif defined (__arm__)
+#include <dev/arm/cons.h>
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_CONS_H_ */
diff --git a/bsd/machine/dis_tables.h b/bsd/machine/dis_tables.h
new file mode 100644
index 000000000..6eaff8106
--- /dev/null
+++ b/bsd/machine/dis_tables.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _BSD_MACHINE_DIS_TABLES_H_
+#define _BSD_MACHINE_DIS_TABLES_H_
+
+#if defined (__ppc__) || defined (__ppc64__)
+#include "ppc/dis_tables.h"
+#elif defined (__i386__) || defined(__x86_64__)
+#include "i386/dis_tables.h"
+#else
+#error architecture not supported
+#endif
+
+#endif /* _BSD_MACHINE_DIS_TABLES_H_ */
diff --git a/bsd/machine/disklabel.h b/bsd/machine/disklabel.h
index 632e5e92e..1a2e2b230 100644
--- a/bsd/machine/disklabel.h
+++ b/bsd/machine/disklabel.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_CPU_H_
 #define _BSD_MACHINE_CPU_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/disklabel.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/disklabel.h"
+#elif defined (__arm__)
+#include "arm/disklabel.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_CPU_H_ */
diff --git a/bsd/machine/endian.h b/bsd/machine/endian.h
index 13899419a..a49d52ac9 100644
--- a/bsd/machine/endian.h
+++ b/bsd/machine/endian.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -25,14 +31,14 @@
 #ifndef _BSD_MACHINE_ENDIAN_H_
 #define _BSD_MACHINE_ENDIAN_H_
 
-
 #if defined (__ppc__) || defined(__ppc64__)
 #include "ppc/endian.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/endian.h"
+#elif defined (__arm__)
+#include "arm/endian.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_ENDIAN_H_ */
diff --git a/bsd/machine/exec.h b/bsd/machine/exec.h
index d168d5969..d866b9b42 100644
--- a/bsd/machine/exec.h
+++ b/bsd/machine/exec.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -27,6 +33,14 @@
 
 #include <sys/param.h>
 
+struct exec_info {
+	char	path[MAXPATHLEN];
+	int	ac;
+	int	ec;
+	char	**av;
+	char	**ev;
+};
+
 struct exec_archhandler {
 	char path[MAXPATHLEN];
 	uint32_t fsid;
@@ -34,16 +48,17 @@ struct exec_archhandler {
 };
 
 extern struct exec_archhandler exec_archhandler_ppc;
-extern int set_archhandler(struct proc *p, int arch);
-extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
+int set_archhandler(struct proc *, int);
+int grade_binary(cpu_type_t, cpu_subtype_t);
 
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/exec.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/exec.h"
+#elif defined (__arm__)
+#include "arm/exec.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_EXEC_H_ */
diff --git a/bsd/machine/fasttrap_isa.h b/bsd/machine/fasttrap_isa.h
new file mode 100644
index 000000000..fa881d329
--- /dev/null
+++ b/bsd/machine/fasttrap_isa.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _BSD_MACHINE_FASTTRAP_ISA_H_
+#define _BSD_MACHINE_FASTTRAP_ISA_H_
+
+#if defined (__ppc__) || defined (__ppc64__)
+#include "ppc/fasttrap_isa.h"
+#elif defined (__i386__) || defined(__x86_64__)
+#include "i386/fasttrap_isa.h"
+#elif defined (__arm__)
+#include "arm/fasttrap_isa.h"
+#else
+#error architecture not supported
+#endif
+
+#endif /* _BSD_MACHINE_FASTTRAP_ISA_H_ */
diff --git a/bsd/machine/param.h b/bsd/machine/param.h
index b8994c955..0b2a2ad30 100644
--- a/bsd/machine/param.h
+++ b/bsd/machine/param.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -25,14 +31,14 @@
 #ifndef _BSD_MACHINE_PARAM_H_
 #define _BSD_MACHINE_PARAM_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/param.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/param.h"
+#elif defined (__arm__)
+#include "arm/param.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_PARAM_H_ */
diff --git a/bsd/machine/profile.h b/bsd/machine/profile.h
index b84a10bc7..45e4de8da 100644
--- a/bsd/machine/profile.h
+++ b/bsd/machine/profile.h
@@ -1,27 +1,31 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- * Copyright 1997 Apple Computer, Inc. All rights reserved.
- *
  * History :
  *	29-Sep-1997  Umesh Vaishampayan
  *		Created.
@@ -29,14 +33,14 @@
 #ifndef _BSD_MACHINE_PROFILE_H_
 #define _BSD_MACHINE_PROFILE_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/profile.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/profile.h"
+#elif defined (__arm__)
+#include "arm/profile.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_PROFILE_H_ */
diff --git a/bsd/machine/psl.h b/bsd/machine/psl.h
index aa74ef173..5abfaef6f 100644
--- a/bsd/machine/psl.h
+++ b/bsd/machine/psl.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_PSL_H_
 #define _BSD_MACHINE_PSL_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/psl.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/psl.h"
+#elif defined (__arm__)
+#include "arm/psl.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_PSL_H_ */
diff --git a/bsd/machine/ptrace.h b/bsd/machine/ptrace.h
index 1c0cde0be..964abe15c 100644
--- a/bsd/machine/ptrace.h
+++ b/bsd/machine/ptrace.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -25,14 +31,14 @@
 #ifndef _BSD_MACHINE_PTRACE_H_
 #define _BSD_MACHINE_PTRACE_H_
 
-
 #if defined (__ppc__) || defined(__ppc64__)
 #include "ppc/ptrace.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/ptrace.h"
+#elif defined (__arm__)
+#include "arm/ptrace.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_PTRACE_H_ */
diff --git a/bsd/machine/reboot.h b/bsd/machine/reboot.h
index 368be9e2a..50c61c021 100644
--- a/bsd/machine/reboot.h
+++ b/bsd/machine/reboot.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_REBOOT_H_
 #define _BSD_MACHINE_REBOOT_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/reboot.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/reboot.h"
+#elif defined (__arm__)
+#include "arm/reboot.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_REBOOT_H_ */
diff --git a/bsd/machine/reg.h b/bsd/machine/reg.h
index e8166db67..f5eb3b9e1 100644
--- a/bsd/machine/reg.h
+++ b/bsd/machine/reg.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_REG_H_
 #define _BSD_MACHINE_REG_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/reg.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/reg.h"
+#elif defined (__arm__)
+#include "arm/reg.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_REG_H_ */
diff --git a/bsd/machine/setjmp.h b/bsd/machine/setjmp.h
index 79b088b82..9aa71b5af 100644
--- a/bsd/machine/setjmp.h
+++ b/bsd/machine/setjmp.h
@@ -1,37 +1,44 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved
  */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-
 #ifndef	_MACHINE_SETJMP_H_
 #define	_MACHINE_SETJMP_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/setjmp.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/setjmp.h"
+#elif defined (__arm__)
+#include "arm/setjmp.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif	/* _MACHINE_SETJMP_H_ */
diff --git a/bsd/machine/signal.h b/bsd/machine/signal.h
index 34a5e06cd..7457d1ca2 100644
--- a/bsd/machine/signal.h
+++ b/bsd/machine/signal.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_SIGNAL_H_
 #define _BSD_MACHINE_SIGNAL_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/signal.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/signal.h"
+#elif defined (__arm__)
+#include "arm/signal.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_SIGNAL_H_ */
diff --git a/bsd/machine/spl.h b/bsd/machine/spl.h
index 36ab465e9..1667c5099 100644
--- a/bsd/machine/spl.h
+++ b/bsd/machine/spl.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_SPL_H_
 #define _BSD_MACHINE_SPL_H_
@@ -43,6 +49,7 @@ extern unsigned	int splnet(void);
 extern unsigned	int splsoftclock(void);
 
 extern void	spllo(void);
+extern void	spl0(void);
 extern void	splon(unsigned int level);
 extern void	splx(unsigned int level);
 extern void	spln(unsigned int level);
diff --git a/bsd/machine/types.h b/bsd/machine/types.h
index b5c580f24..2bff80981 100644
--- a/bsd/machine/types.h
+++ b/bsd/machine/types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -25,14 +31,14 @@
 #ifndef _BSD_MACHINE_TYPES_H_
 #define _BSD_MACHINE_TYPES_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/types.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/types.h"
+#elif defined (__arm__)
+#include "arm/types.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_TYPES_H_ */
diff --git a/bsd/machine/ucontext.h b/bsd/machine/ucontext.h
index b5fdd1ad5..6ecf3e26f 100644
--- a/bsd/machine/ucontext.h
+++ b/bsd/machine/ucontext.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_MACHINE_UCONTEXT_H_
 #define	_MACHINE_UCONTEXT_H_
@@ -26,6 +32,8 @@
 #include "ppc/ucontext.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/ucontext.h"
+#elif defined (__arm__)
+#include "arm/ucontext.h"
 #else
 #error architecture not supported
 #endif
diff --git a/bsd/machine/vmparam.h b/bsd/machine/vmparam.h
index f084c03bd..d9d0d74ed 100644
--- a/bsd/machine/vmparam.h
+++ b/bsd/machine/vmparam.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _BSD_MACHINE_VMPARAM_H_
 #define _BSD_MACHINE_VMPARAM_H_
 
-
 #if defined (__ppc__) || defined (__ppc64__)
 #include "ppc/vmparam.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "i386/vmparam.h"
+#elif defined (__arm__)
+#include "arm/vmparam.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _BSD_MACHINE_VMPARAM_H_ */
diff --git a/bsd/man/Makefile b/bsd/man/Makefile
index a287125bd..dfdb1b572 100644
--- a/bsd/man/Makefile
+++ b/bsd/man/Makefile
@@ -9,8 +9,10 @@ include $(MakeInc_def)
 
 INSTMAN_SUBDIRS = \
 	man2 \
+	man3 \
 	man4 \
 	man5 \
+	man8 \
 	man9
 
 include $(MakeInc_rule)
diff --git a/bsd/man/man2/EV_SET.2 b/bsd/man/man2/EV_SET.2
new file mode 100644
index 000000000..9f491e699
--- /dev/null
+++ b/bsd/man/man2/EV_SET.2
@@ -0,0 +1 @@
+.so man2/kqueue.2
diff --git a/bsd/man/man2/FD_CLR.2 b/bsd/man/man2/FD_CLR.2
new file mode 100644
index 000000000..e17784318
--- /dev/null
+++ b/bsd/man/man2/FD_CLR.2
@@ -0,0 +1 @@
+.so man2/select.2
diff --git a/bsd/man/man2/FD_COPY.2 b/bsd/man/man2/FD_COPY.2
new file mode 100644
index 000000000..e17784318
--- /dev/null
+++ b/bsd/man/man2/FD_COPY.2
@@ -0,0 +1 @@
+.so man2/select.2
diff --git a/bsd/man/man2/FD_ISSET.2 b/bsd/man/man2/FD_ISSET.2
new file mode 100644
index 000000000..e17784318
--- /dev/null
+++ b/bsd/man/man2/FD_ISSET.2
@@ -0,0 +1 @@
+.so man2/select.2
diff --git a/bsd/man/man2/FD_SET.2 b/bsd/man/man2/FD_SET.2
new file mode 100644
index 000000000..e17784318
--- /dev/null
+++ b/bsd/man/man2/FD_SET.2
@@ -0,0 +1 @@
+.so man2/select.2
diff --git a/bsd/man/man2/FD_ZERO.2 b/bsd/man/man2/FD_ZERO.2
new file mode 100644
index 000000000..e17784318
--- /dev/null
+++ b/bsd/man/man2/FD_ZERO.2
@@ -0,0 +1 @@
+.so man2/select.2
diff --git a/bsd/man/man2/Makefile b/bsd/man/man2/Makefile
index 7312eba32..5a44dee4f 100644
--- a/bsd/man/man2/Makefile
+++ b/bsd/man/man2/Makefile
@@ -7,178 +7,204 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 DATAFILES = \
-	__syscall.2	\
-	_exit.2	\
-	aio_cancel.2	\
-	aio_error.2	\
-	aio_read.2	\
-	aio_return.2	\
-	aio_suspend.2	\
-	aio_write.2	\
-	accept.2	\
-	access.2	\
-	acct.2	\
-	adjtime.2	\
-	bind.2	\
-	chdir.2	\
-	chflags.2	\
-	chmod.2	\
-	chown.2	\
-	chroot.2	\
-	close.2	\
-	connect.2	\
-	dup.2	\
-	dup2.2	\
-	execve.2	\
-	exchangedata.2	\
-	fchdir.2	\
-	fchflags.2	\
-	fchmod.2	\
-	fchown.2	\
-	fcntl.2	\
-	flock.2	\
-	fork.2	\
-	fpathconf.2	\
-	fstat.2	\
-	fstatfs.2	\
-	fsync.2	\
-	ftruncate.2	\
-	futimes.2	\
-	getattrlist.2	\
-	getdirentries.2	\
+	__syscall.2		\
+	_exit.2			\
+	EV_SET.2		\
+	FD_CLR.2		\
+	FD_COPY.2		\
+	FD_ISSET.2		\
+	FD_SET.2		\
+	FD_ZERO.2		\
+	aio_cancel.2		\
+	aio_error.2		\
+	aio_read.2		\
+	aio_return.2		\
+	aio_suspend.2		\
+	aio_write.2		\
+	accept.2		\
+	access.2		\
+	acct.2			\
+	adjtime.2		\
+	audit.2			\
+	auditctl.2		\
+	auditon.2		\
+	bind.2			\
+	chdir.2			\
+	chflags.2		\
+	chmod.2			\
+	chown.2			\
+	chroot.2		\
+	close.2			\
+	connect.2		\
+	dup.2			\
+	dup2.2			\
+	execve.2		\
+	exchangedata.2		\
+	fchdir.2		\
+	fchflags.2		\
+	fchmod.2		\
+	fchown.2		\
+	fcntl.2			\
+	fgetxattr.2		\
+	fhopen.2		\
+	flistxattr.2		\
+	flock.2			\
+	fork.2			\
+	fpathconf.2		\
+	fremovexattr.2		\
+	fsetxattr.2		\
+	fstat.2			\
+	fstat64.2		\
+	fstatfs.2		\
+	fstatfs64.2		\
+	fsync.2			\
+	ftruncate.2		\
+	futimes.2		\
+	getattrlist.2		\
+	getaudit.2		\
+	getauid.2		\
+	getdirentries.2		\
 	getdirentriesattr.2	\
-	getegid.2	\
-	geteuid.2	\
-	getfh.2	\
-	getfsstat.2	\
-	getgid.2	\
-	getgroups.2	\
-	getitimer.2	\
-	getlogin.2	\
-	getpeername.2	\
-	getpgrp.2	\
-	getpgid.2	\
-	getpid.2	\
-	getppid.2	\
-	getpriority.2	\
-	getrlimit.2	\
-	getrusage.2	\
-	getsid.2	\
-	getsockname.2	\
-	getsockopt.2	\
-	gettimeofday.2	\
-	getuid.2	\
-	getxattr.2	\
-	i386_get_ldt.2	\
-	intro.2	\
-	ioctl.2	\
-	issetugid.2	\
-	kill.2	\
-	kqueue.2	\
-	ktrace.2	\
-	lchown.2	\
-	link.2	\
-	listen.2	\
-	listxattr.2	\
-	lseek.2	\
-	lstat.2	\
-	madvise.2	\
-	mincore.2	\
-	minherit.2	\
-	mkdir.2	\
-	mkfifo.2	\
-	mknod.2	\
-	mlock.2	\
-	mmap.2	\
-	mount.2	\
-	mprotect.2	\
-	msync.2	\
-	munlock.2	\
-	munmap.2	\
-	nfssvc.2	\
-	open.2	\
-	pathconf.2	\
-	pipe.2	\
-	poll.2 \
-	posix_madvise.2	\
-	pread.2	\
-	profil.2	\
-	ptrace.2	\
-	pwrite.2	\
-	quotactl.2	\
-	read.2	\
-	readlink.2	\
-	readv.2	\
-	reboot.2	\
-	recv.2	\
-	recvfrom.2	\
-	recvmsg.2	\
-	rename.2	\
-	removexattr.2	\
-	revoke.2	\
-	rmdir.2	\
-	searchfs.2	\
-	select.2	\
-	semctl.2	\
-	semget.2	\
-	semop.2	\
-	send.2	\
-	sendmsg.2	\
-	sendto.2	\
-	setattrlist.2	\
-	setegid.2	\
-	seteuid.2	\
-	setgid.2	\
-	setgroups.2	\
-	setitimer.2	\
-	setlogin.2	\
-	setpgid.2	\
-	setpgrp.2	\
-	setpriority.2	\
-	setrlimit.2	\
-	setsid.2	\
-	setsockopt.2	\
-	settimeofday.2	\
-	setuid.2	\
-	setxattr.2	\
-	shmat.2	\
-	shmctl.2	\
-	shmdt.2	\
-	shmget.2	\
-	shutdown.2	\
-	sigaction.2	\
-	sigaltstack.2	\
-	sigpending.2	\
-	sigprocmask.2	\
-	sigreturn.2	\
-	sigstack.2	\
-	sigsuspend.2	\
-	socket.2	\
-	socketpair.2	\
-	stat.2	\
-	statfs.2	\
-	symlink.2	\
-	sync.2	\
-	syscall.2	\
-	truncate.2	\
-	umask.2	\
-	unlink.2	\
-	unmount.2	\
-	utimes.2	\
-	vfork.2	\
-	wait.2	\
-	wait3.2	\
-	wait4.2	\
-	waitpid.2	\
-	write.2	\
-	writev.2	\
+	getegid.2		\
+	geteuid.2		\
+	getfh.2			\
+	getfsstat.2		\
+	getgid.2		\
+	getgroups.2		\
+	getitimer.2		\
+	getlcid.2		\
+	getlogin.2		\
+	getpeername.2		\
+	getpgrp.2		\
+	getpgid.2		\
+	getpid.2		\
+	getppid.2		\
+	getpriority.2		\
+	getrlimit.2		\
+	getrusage.2		\
+	getsid.2		\
+	getsockname.2		\
+	getsockopt.2		\
+	gettimeofday.2		\
+	getuid.2		\
+	getxattr.2		\
+	i386_get_ldt.2		\
+	i386_set_ldt.2		\
+	intro.2			\
+	ioctl.2			\
+	issetugid.2		\
+	kill.2			\
+	kevent.2		\
+	kqueue.2		\
+	lchown.2		\
+	link.2			\
+	listen.2		\
+	listxattr.2		\
+	lseek.2			\
+	lstat.2			\
+	lstat64.2		\
+	madvise.2		\
+	mincore.2		\
+	minherit.2		\
+	mkdir.2			\
+	mkfifo.2		\
+	mknod.2			\
+	mlock.2			\
+	mmap.2			\
+	mount.2			\
+	mprotect.2		\
+	msync.2			\
+	munlock.2		\
+	munmap.2		\
+	nfsclnt.2		\
+	nfssvc.2		\
+	open.2			\
+	pathconf.2		\
+	pipe.2			\
+	poll.2 			\
+	posix_madvise.2		\
+	pread.2			\
+	profil.2		\
+	ptrace.2		\
+	pwrite.2		\
+	quotactl.2		\
+	read.2			\
+	readlink.2		\
+	readv.2			\
+	reboot.2		\
+	recv.2			\
+	recvfrom.2		\
+	recvmsg.2		\
+	rename.2		\
+	removexattr.2		\
+	revoke.2		\
+	rmdir.2			\
+	searchfs.2		\
+	select.2		\
+	semctl.2		\
+	semget.2		\
+	semop.2			\
+	send.2			\
+	sendfile.2		\
+	sendmsg.2		\
+	sendto.2		\
+	setattrlist.2		\
+	setaudit.2		\
+	setauid.2		\
+	setegid.2		\
+	seteuid.2		\
+	setgid.2		\
+	setgroups.2		\
+	setitimer.2		\
+	setlcid.2		\
+	setlogin.2		\
+	setpgid.2		\
+	setpgrp.2		\
+	setpriority.2		\
+	setrlimit.2		\
+	setsid.2		\
+	setsockopt.2		\
+	settimeofday.2		\
+	setuid.2		\
+	setxattr.2		\
+	shmat.2			\
+	shmctl.2		\
+	shmdt.2			\
+	shmget.2		\
+	shutdown.2		\
+	sigaction.2		\
+	sigaltstack.2		\
+	sigpending.2		\
+	sigprocmask.2		\
+	sigstack.2		\
+	sigsuspend.2		\
+	socket.2		\
+	socketpair.2		\
+	stat.2			\
+	stat64.2		\
+	statfs.2		\
+	statfs64.2		\
+	symlink.2		\
+	sync.2			\
+	syscall.2		\
+	truncate.2		\
+	umask.2			\
+	unlink.2		\
+	unmount.2		\
+	utimes.2		\
+	vfork.2			\
+	wait.2			\
+	wait3.2			\
+	wait4.2			\
+	waitpid.2		\
+	write.2			\
+	writev.2		\
+	posix_spawn.2
 
-# List of source/target hard link pairs for installed manual pages; source
-# names may be repeated
-MLINKS= kqueue.2 kevent.2 i386_get_ldt.2 i386_set_ldt.2
+INSTALL_MAN_LINKS = \
+	posix_spawn.2 posix_spawnp.2
 
 INSTALL_MAN_LIST = ${DATAFILES}
-INSTALL_MAN_LINKS = ${MLINKS}
 
 INSTALL_MAN_DIR = man2
 
diff --git a/bsd/man/man2/_exit.2 b/bsd/man/man2/_exit.2
index 402de8fb9..9c520f76f 100644
--- a/bsd/man/man2/_exit.2
+++ b/bsd/man/man2/_exit.2
@@ -42,17 +42,19 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft void
-.Fn _exit "int status"
+.Fo _exit
+.Fa "int status"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn _exit
 function
-terminates a process with the following consequences:
+terminates a process, with the following consequences:
 .Bl -bullet
 .It
-All of the descriptors open in the calling process are closed.
-This may entail delays, for example, waiting for output to drain;
-a process in this state may not be killed, as it is already dying.
+All of the descriptors that were open in the calling process are closed.
+This may entail delays; for example, waiting for output to drain.
+A process in this state may not be killed, as it is already dying.
 .It
 If the parent process of the calling process has an outstanding
 .Xr wait
@@ -60,7 +62,7 @@ call
 or catches the
 .Dv SIGCHLD
 signal,
-it is notified of the calling process's termination and
+it is notified of the calling process's termination;
 the
 .Em status
 is set as defined by
@@ -89,8 +91,8 @@ If the process is a controlling process (see
 .Xr intro 2 ) ,
 the
 .Dv SIGHUP
-signal is sent to the foreground process group of the controlling terminal,
-and all current access to the controlling terminal is revoked.
+signal is sent to the foreground process group of the controlling terminal.
+All current access to the controlling terminal is revoked.
 .El
 .Pp
 Most C programs call the library routine
diff --git a/bsd/man/man2/accept.2 b/bsd/man/man2/accept.2
index 3b5ec370f..cbe526799 100644
--- a/bsd/man/man2/accept.2
+++ b/bsd/man/man2/accept.2
@@ -40,26 +40,27 @@
 .Nm accept
 .Nd accept a connection on a socket
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn accept "int s" "struct sockaddr *addr" "socklen_t *addrlen"
+.Fo accept
+.Fa "int socket"
+.Fa "struct sockaddr *restrict address"
+.Fa "socklen_t *restrict address_len"
+.Fc
 .Sh DESCRIPTION
 The argument
-.Fa s
+.Fa socket
 is a socket that has been created with
 .Xr socket 2 ,
 bound to an address with
 .Xr bind 2 ,
 and is listening for connections after a
 .Xr listen 2 .
-The
 .Fn accept
-argument
 extracts the first connection request
 on the queue of pending connections, creates
 a new socket with the same properties of 
-.Fa s
+.Fa socket ,
 and allocates a new file descriptor
 for the socket.  If no pending connections are
 present on the queue, and the socket is not marked
@@ -73,23 +74,23 @@ returns an error as described below.
 The accepted socket
 may not be used
 to accept more connections.  The original socket
-.Fa s
+.Fa socket,
 remains open.
 .Pp
 The argument
-.Fa addr
+.Fa address
 is a result parameter that is filled in with
 the address of the connecting entity,
 as known to the communications layer.
 The exact format of the
-.Fa addr
+.Fa address
 parameter is determined by the domain in which the communication
 is occurring.
 The 
-.Fa addrlen
+.Fa address_len
 is a value-result parameter; it should initially contain the
 amount of space pointed to by
-.Fa addr ;
+.Fa address ;
 on return it will contain the actual length (in bytes) of the
 address returned.
 This call
@@ -132,39 +133,78 @@ call with providing only the control information,
 or by calling
 .Xr setsockopt 2 .
 .Sh RETURN VALUES
-The call returns \-1 on error.  If it succeeds, it returns a non-negative
-integer that is a descriptor for the accepted socket.
+The call returns \-1 on error and the global variable
+.Va errno
+is set to indicate the error.
+If it succeeds, it returns a non-negative integer
+that is a descriptor for the accepted socket.
 .Sh ERRORS
 The
 .Fn accept
-will fail if:
+system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-The descriptor is invalid.
-.It Bq Er ENOTSOCK
-The descriptor references a file, not a socket.
-.It Bq Er EOPNOTSUPP
-The referenced socket is not of type
-.Dv SOCK_STREAM . 
+.Fa socket
+is not a valid file descriptor.
+.\" ==========
+.It Bq Er ECONNABORTED
+The connection to
+.Fa socket
+has been aborted.
+.\" ==========
 .It Bq Er EFAULT
 The
-.Fa addr
+.Fa address
 parameter is not in a writable part of the
 user address space.
-.It Bq Er EWOULDBLOCK
-The socket is marked non-blocking and no connections
-are present to be accepted.
+.\" ==========
+.It Bq Er EINTR
+The
+.Fn accept
+system call was terminated by a signal.
+.\" ==========
+.It Bq Er EINVAL
+.Fa socket
+is unwilling to accept connections.
+.\" ==========
 .It Bq Er EMFILE
 The per-process descriptor table is full.
+.\" ==========
 .It Bq Er ENFILE
 The system file table is full.
+.\" ==========
+.It Bq Er ENOMEM
+Insufficient memory was available to complete the operation.
+.\" ==========
+.It Bq Er ENOTSOCK
+.Fa socket
+references a file type other than a socket.
+.\" ==========
+.It Bq Er EOPNOTSUPP
+.Fa socket
+is not of type
+.Dv SOCK_STREAM
+and thus does not accept connections. 
+.\" ==========
+.It Bq Er EWOULDBLOCK
+.Fa socket
+is marked as non-blocking and no connections are present to be accepted.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr bind 2 ,
 .Xr connect 2 ,
 .Xr listen 2 ,
 .Xr select 2 ,
-.Xr socket 2
+.Xr socket 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn accept
diff --git a/bsd/man/man2/access.2 b/bsd/man/man2/access.2
index 0d168d3c8..d1114f5e9 100644
--- a/bsd/man/man2/access.2
+++ b/bsd/man/man2/access.2
@@ -42,7 +42,10 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn access "const char *path" "int mode"
+.Fo access
+.Fa "const char *path"
+.Fa "int amode"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn access
@@ -50,9 +53,9 @@ function checks the accessibility of the
 file named by
 .Fa path
 for the access permissions indicated by
-.Fa mode .
+.Fa amode .
 The value of
-.Fa mode
+.Fa amode
 is the bitwise inclusive OR of the access permissions to be
 checked
 .Pf ( Dv R_OK
@@ -82,46 +85,60 @@ and
 .Sh RETURN VALUES
 If
 .Fa path
-cannot be found or if any of the desired access modes would
-not be granted, then a -1 value is returned; otherwise
-a 0 value is returned.
+cannot be found
+or if any of the desired access modes would not be granted,
+then a -1 value is returned and the global integer variable
+.Va errno
+is set to indicate the error.
+Otherwise, a 0 value is returned.
 .Sh ERRORS
 Access to the file is denied if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EACCES
+Permission bits of the file mode do not permit the requested access,
+or search permission is denied on a component of the path prefix.
+.Pp
+The owner of a file has permission checked
+with respect to the ``owner'' read, write, and execute mode bits,
+members of the file's group other than the owner have permission checked
+with respect to the ``group'' mode bits,
+and all others have permissions checked
+with respect to the ``other'' mode bits.
+.\" 
+.\" ==========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.It Bq Er EINVAL
+An invalid value was specified for
+.Ar amode .
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+.\" ==========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ==========
 .It Bq Er ENOENT
 The named file does not exist.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
 .It Bq Er EROFS
 Write access is requested for a file on a read-only file system.
+.\" ==========
 .It Bq Er ETXTBSY
 Write access is requested for a pure procedure (shared text)
-file presently being executed.
-.It Bq Er EACCES
-Permission bits of the file mode do not permit the requested
-access, or search permission is denied on a component of the
-path prefix.  The owner of a file has permission checked with
-respect to the ``owner'' read, write, and execute mode bits,
-members of the file's group other than the owner have permission
-checked with respect to the ``group'' mode bits, and all
-others have permissions checked with respect to the ``other''
-mode bits.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
-.It Bq Er EINVAL
-An invalid value was specified for
-.Ar mode .
+file that is presently being executed.
 .El
 .Sh SEE ALSO
 .Xr chmod 2 ,
diff --git a/bsd/man/man2/acct.2 b/bsd/man/man2/acct.2
index 5c041ec7e..582aa8940 100644
--- a/bsd/man/man2/acct.2
+++ b/bsd/man/man2/acct.2
@@ -78,10 +78,10 @@ Accounting is automatically disabled when the file system the
 accounting file resides on runs out of space; it is enabled when
 space once again becomes available.
 .Sh RETURN VALUES
-On error -1 is returned.
+.Rv -std acct
 The file must exist and the call may be exercised only by the super-user.
 .Sh ERRORS
-.Fn Acct
+.Fn acct
 will fail if one of the following is true:
 .Bl -tag -width Er
 .It Bq Er EPERM
diff --git a/bsd/man/man2/adjtime.2 b/bsd/man/man2/adjtime.2
index 30bf72b78..845d88254 100644
--- a/bsd/man/man2/adjtime.2
+++ b/bsd/man/man2/adjtime.2
@@ -100,7 +100,7 @@ The process's effective user ID is not that of the super-user.
 .Xr date 1 ,
 .Xr gettimeofday 2 ,
 .Xr timed 8 ,
-.Xr timedc 8 ,
+.Xr timedc 8
 .Rs
 .%T "TSP: The Time Synchronization Protocol for UNIX 4.3BSD"
 .%A R. Gusella
diff --git a/bsd/man/man2/aio_cancel.2 b/bsd/man/man2/aio_cancel.2
index a5f1392c6..00029b15f 100644
--- a/bsd/man/man2/aio_cancel.2
+++ b/bsd/man/man2/aio_cancel.2
@@ -35,7 +35,10 @@
 .Sh SYNOPSIS
 .In aio.h
 .Ft int
-.Fn aio_cancel "int fildes" "struct aiocb * iocb"
+.Fo aio_cancel
+.Fa "int fildes"
+.Fa "struct aiocb *aiocbp"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_cancel
@@ -43,7 +46,7 @@ system call cancels the outstanding asynchronous
 I/O request for the file descriptor specified in
 .Fa fildes .
 If
-.Fa iocb
+.Fa aiocbp
 is specified, only that specific asynchronous I/O request is cancelled.
 .Pp
 Normal asynchronous notification occurs for cancelled requests.
@@ -63,25 +66,26 @@ The
 .Fn aio_cancel
 system call returns -1 to indicate an error, or one of the following:
 .Bl -tag -width Dv
+.It Bq Dv AIO_ALLDONE
+All of the requests meeting the criteria have finished.
 .It Bq Dv AIO_CANCELED
 All outstanding requests meeting the criteria specified were cancelled.
 .It Bq Dv AIO_NOTCANCELED
 Some requests were not cancelled, status for the requests should be
 checked with
 .Xr aio_error 2 .
-.It Bq Dv AIO_ALLDONE
-All of the requests meeting the criteria have finished.
 .El
 .Sh ERRORS
 An error return from
 .Fn aio_cancel
 indicates:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
 .Fa fildes
 argument
-is an invalid file descriptor.
+is not a valid file descriptor.
 .El
 .Sh SEE ALSO
 .Xr aio_error 2 ,
diff --git a/bsd/man/man2/aio_error.2 b/bsd/man/man2/aio_error.2
index 8c13ca3f5..492b3f466 100644
--- a/bsd/man/man2/aio_error.2
+++ b/bsd/man/man2/aio_error.2
@@ -35,20 +35,23 @@
 .Sh SYNOPSIS
 .In aio.h
 .Ft int
-.Fn aio_error "const struct aiocb *iocb"
+.Fo aio_error
+.Fa "const struct aiocb *aiocbp"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_error
 system call returns the error status of the asynchronous I/O request
 associated with the structure pointed to by
-.Fa iocb .
+.Fa aiocbp .
 .Sh RETURN VALUES
 If the asynchronous I/O request has completed successfully,
 .Fn aio_error
 returns 0.  If the request has not yet completed,
 .Er EINPROGRESS
-is returned.  If the request has completed unsuccessfully the error
-status is returned as described in
+is returned.
+If the request has completed unsuccessfully,
+the error status is returned as described in
 .Xr read 2 ,
 .Xr write 2 ,
 or
@@ -66,9 +69,10 @@ The
 .Fn aio_error
 system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EINVAL
 The
-.Fa iocb
+.Fa aiocbp
 argument
 does not reference an outstanding asynchronous I/O request.
 .El
diff --git a/bsd/man/man2/aio_read.2 b/bsd/man/man2/aio_read.2
index e0ef5a537..fd3a53c31 100644
--- a/bsd/man/man2/aio_read.2
+++ b/bsd/man/man2/aio_read.2
@@ -35,36 +35,38 @@
 .Sh SYNOPSIS
 .In aio.h
 .Ft int
-.Fn aio_read "struct aiocb *iocb"
+.Fo aio_read
+.Fa "struct aiocb *aiocbp"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_read
 system call allows the calling process to read
-.Fa iocb->aio_nbytes
+.Fa aiocbp->aio_nbytes
 from the descriptor
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes ,
 beginning at the offset
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset ,
 into the buffer pointed to by
-.Fa iocb->aio_buf .
+.Fa aiocbp->aio_buf .
 The call returns immediately after the read request has
-been enqueued to the descriptor; the read may or may not have
-completed at the time the call returns.
+been enqueued to the descriptor;
+the read may or may not have completed at the time the call returns.
 .Pp
 If _POSIX_PRIORITIZED_IO is defined, and the descriptor supports it,
 then the enqueued operation is submitted at a priority equal to that
 of the calling process minus
-.Fa iocb->aio_reqprio .
+.Fa aiocbp->aio_reqprio .
 .Pp
 The
-.Fa iocb->aio_lio_opcode
+.Fa aiocbp->aio_lio_opcode
 argument
 is ignored by the
 .Fn aio_read
 system call.
 .Pp
 The
-.Fa iocb
+.Fa aiocbp
 pointer may be subsequently used as an argument to
 .Fn aio_return
 and
@@ -76,20 +78,20 @@ If the request could not be enqueued (generally due to invalid arguments),
 then the call returns without having enqueued the request.
 .Pp
 If the request is successfully enqueued, the value of
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 can be modified during the request as context, so this value must
 not be referenced after the request is enqueued.
 .Sh RESTRICTIONS
 The Asynchronous I/O Control Block structure pointed to by
-.Fa iocb
+.Fa aiocbp
 and the buffer that the
-.Fa iocb->aio_buf
+.Fa aiocbp->aio_buf
 member of that structure references must remain valid until the
 operation has completed.  For this reason, use of auto (stack) variables
 for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
-.Fa iocb
+.Fa aiocbp
 should be zeroed before the
 .Fn aio_read
 call to avoid passing bogus context information to the kernel.
@@ -99,9 +101,9 @@ buffer contents after the request has been enqueued, but before the
 request has completed, are not allowed.
 .Pp
 If the file offset in
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 is past the offset maximum  for
-.Fa iocb->aio_fildes ,
+.Fa aiocbp->aio_fildes ,
 no I/O will occur.
 .Sh RETURN VALUES
 .Rv -std aio_read
@@ -112,8 +114,11 @@ The
 .Fn aio_read
 system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EAGAIN
-The request was not queued because of system resource limitations.
+Because of system resource limitations,
+the request was not queued.
+.\" ==========
 .It Bq Er ENOSYS
 The
 .Fn aio_read
@@ -122,40 +127,45 @@ system call is not supported.
 .Pp
 The following conditions may be synchronously detected when the
 .Fn aio_read
-system call is made, or asynchronously, at any time thereafter.  If they
-are detected at call time,
+system call is made, or asynchronously, at any time thereafter.
+If they are detected at call time,
 .Fn aio_read
 returns -1 and sets
 .Va errno
-appropriately; otherwise the
+appropriately.
+Otherwise, the
 .Fn aio_return
-system call must be called, and will return -1, and
+system call must be called.
+It will return -1;
 .Fn aio_error
-must be called to determine the actual value that would have been
-returned in
+must then be called to determine the actual value
+that would have been returned in
 .Va errno .
 .Pp
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes
 argument
 is invalid.
+.\" ==========
 .It Bq Er EINVAL
 The offset
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 is not valid, the priority specified by
-.Fa iocb->aio_reqprio
+.Fa aiocbp->aio_reqprio
 is not a valid priority, or the number of bytes specified by
-.Fa iocb->aio_nbytes
+.Fa aiocbp->aio_nbytes
 is not valid.
+.\" ==========
 .It Bq Er EOVERFLOW
 The file is a regular file,
-.Fa iocb->aio_nbytes
+.Fa aiocbp->aio_nbytes
 is greater than zero, the starting offset in
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 is before the end of the file, but is at or beyond the
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes
 offset maximum.
 .El
 .Pp
@@ -170,17 +180,20 @@ system call is either one of the error returns from the
 .Xr read 2
 system call, or one of:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes
 argument
 is invalid for reading.
+.\" ==========
 .It Bq Er ECANCELED
 The request was explicitly cancelled via a call to
 .Fn aio_cancel .
+.\" ==========
 .It Bq Er EINVAL
 The offset
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 would be invalid.
 .El
 .Sh SEE ALSO
@@ -207,5 +220,5 @@ manual page was written by
 .An Terry Lambert Aq terry@whistle.com .
 .Sh BUGS
 Invalid information in
-.Fa iocb->_aiocb_private
+.Fa aiocbp->_aiocb_private
 may confuse the kernel.
diff --git a/bsd/man/man2/aio_return.2 b/bsd/man/man2/aio_return.2
index 8c4e28ff2..2bf094cea 100644
--- a/bsd/man/man2/aio_return.2
+++ b/bsd/man/man2/aio_return.2
@@ -34,14 +34,16 @@
 .Lb libc
 .Sh SYNOPSIS
 .In aio.h
-.Ft int
-.Fn aio_return "struct aiocb *iocb"
+.Ft ssize_t
+.Fo aio_return
+.Fa "struct aiocb *aiocbp"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_return
 system call returns the final status of the asynchronous I/O request
 associated with the structure pointed to by
-.Fa iocb .
+.Fa aiocbp .
 .Pp
 The
 .Fn aio_return
@@ -52,8 +54,8 @@ I/O operation once
 returns something other than
 .Er EINPROGRESS .
 .Sh RETURN VALUES
-If the asynchronous I/O request has completed, the status is returned
-as described in
+If the asynchronous I/O request has completed,
+the status is returned as described in
 .Xr read 2 ,
 .Xr write 2 ,
 or
@@ -70,12 +72,23 @@ The
 .Fn aio_return
 system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EINVAL
 The
-.Fa iocb
+.Fa aiocbp
 argument
 does not reference an outstanding asynchronous I/O request.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <aio.h>
+.Pp
+.Ft int
+.br
+.Fo aio_return
+.Fa "struct aiocb *aiocbp"
+.Fc ;
+.Pp
+The type of the return value has changed.
 .Sh SEE ALSO
 .Xr aio_cancel 2 ,
 .Xr aio_error 2 ,
@@ -84,7 +97,8 @@ does not reference an outstanding asynchronous I/O request.
 .Xr fsync 2 ,
 .Xr read 2 ,
 .Xr write 2 ,
-.Xr aio 4
+.Xr aio 4 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn aio_return
diff --git a/bsd/man/man2/aio_suspend.2 b/bsd/man/man2/aio_suspend.2
index c0b85ce10..3b576c834 100644
--- a/bsd/man/man2/aio_suspend.2
+++ b/bsd/man/man2/aio_suspend.2
@@ -35,7 +35,11 @@
 .Sh SYNOPSIS
 .In aio.h
 .Ft int
-.Fn aio_suspend "const struct aiocb * const iocbs[]" "int niocb" "const struct timespec * timeout"
+.Fo aio_suspend
+.Fa "const struct aiocb *const list[]"
+.Fa "int nent"
+.Fa "const struct timespec *timeout"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_suspend
@@ -46,10 +50,10 @@ delivered, or the
 has passed.
 .Pp
 The
-.Fa iocbs
+.Fa list
 argument
 is an array of
-.Fa niocb
+.Fa nent
 pointers to asynchronous I/O requests.  Array members containing
 NULL will be silently ignored.
 .Pp
@@ -63,10 +67,10 @@ poll, the
 .Fa timeout
 should point to a zero-value timespec structure.
 .Sh RETURN VALUES
-If one or more of the specified asynchronous I/O requests have
-completed,
+If one or more of the specified asynchronous I/O requests have completed,
 .Fn aio_suspend
-returns 0.  Otherwise it returns -1 and sets
+returns 0.
+Otherwise, it returns -1 and sets
 .Va errno
 to indicate the error, as enumerated below.
 .Sh ERRORS
@@ -75,19 +79,21 @@ The
 system call will fail if:
 .Bl -tag -width Er
 .It Bq Er EAGAIN
-the
+The
 .Fa timeout
-expired before any I/O requests completed.
+expired before any of the listed I/O requests completed.
+.\" ==========
+.It Bq Er EINTR
+The suspend was interrupted by a signal.
+.\" ==========
 .It Bq Er EINVAL
 The
-.Fa iocbs
+.Fa list
 argument
 contains more than
 .Dv AIO_LISTIO_MAX
 asynchronous I/O requests, or at least one of the requests is not
 valid.
-.It Bq Er EINTR
-the suspend was interrupted by a signal.
 .El
 .Sh SEE ALSO
 .Xr aio_cancel 2 ,
diff --git a/bsd/man/man2/aio_write.2 b/bsd/man/man2/aio_write.2
index 097daaf4a..a69d74ae0 100644
--- a/bsd/man/man2/aio_write.2
+++ b/bsd/man/man2/aio_write.2
@@ -35,16 +35,18 @@
 .Sh SYNOPSIS
 .In aio.h
 .Ft int
-.Fn aio_write "struct aiocb *iocb"
+.Fo aio_write
+.Fa "struct aiocb *aiocbp"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn aio_write
 system call allows the calling process to write
-.Fa iocb->aio_nbytes
+.Fa aiocbp->aio_nbytes
 from the buffer pointed to by
-.Fa iocb->aio_buf
+.Fa aiocbp->aio_buf
 to the descriptor
-.Fa iocb->aio_fildes .
+.Fa aiocbp->aio_fildes .
 The call returns immediately after the write request has been enqueued
 to the descriptor; the write may or may not have completed at the time
 the call returns.  If the request could not be enqueued, generally due
@@ -54,24 +56,24 @@ request.
 If
 .Dv O_APPEND
 is set for
-.Fa iocb->aio_fildes ,
+.Fa aiocbp->aio_fildes ,
 .Fn aio_write
 operations append to the file in the same order as the calls were
 made.  If
 .Dv O_APPEND
 is not set for the file descriptor, the write operation will occur at
 the absolute position from the beginning of the file plus
-.Fa iocb->aio_offset .
+.Fa aiocbp->aio_offset .
 .Pp
 If
 .Dv _POSIX_PRIORITIZED_IO
 is defined, and the descriptor supports it, then the enqueued
 operation is submitted at a priority equal to that of the calling
 process minus
-.Fa iocb->aio_reqprio .
+.Fa aiocbp->aio_reqprio .
 .Pp
 The
-.Fa iocb
+.Fa aiocbp
 pointer may be subsequently used as an argument to
 .Fn aio_return
 and
@@ -80,20 +82,20 @@ in order to determine return or error status for the enqueued operation
 while it is in progress.
 .Pp
 If the request is successfully enqueued, the value of
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 can be modified during the request as context, so this value must not
 be referenced after the request is enqueued.
 .Sh RESTRICTIONS
 The Asynchronous I/O Control Block structure pointed to by
-.Fa iocb
+.Fa aiocbp
 and the buffer that the
-.Fa iocb->aio_buf
+.Fa aiocbp->aio_buf
 member of that structure references must remain valid until the
 operation has completed.  For this reason, use of auto (stack) variables
 for these objects is discouraged.
 .Pp
 The asynchronous I/O control buffer
-.Fa iocb
+.Fa aiocbp
 should be zeroed before the
 .Fn aio_write
 system call to avoid passing bogus context information to the kernel.
@@ -103,9 +105,9 @@ buffer contents after the request has been enqueued, but before the
 request has completed, are not allowed.
 .Pp
 If the file offset in
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 is past the offset maximum  for
-.Fa iocb->aio_fildes ,
+.Fa aiocbp->aio_fildes ,
 no I/O will occur.
 .Sh RETURN VALUES
 .Rv -std aio_write
@@ -114,8 +116,11 @@ The
 .Fn aio_write
 system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EAGAIN
-The request was not queued because of system resource limitations.
+Due to system resource limitations,
+the request was not queued.
+.\" ==========
 .It Bq Er ENOSYS
 The
 .Fn aio_write
@@ -138,18 +143,20 @@ returned in
 .Va errno .
 .Pp
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes
 argument
 is invalid, or is not opened for writing.
+.\" ==========
 .It Bq Er EINVAL
 The offset
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 is not valid, the priority specified by
-.Fa iocb->aio_reqprio
+.Fa aiocbp->aio_reqprio
 is not a valid priority, or the number of bytes specified by
-.Fa iocb->aio_nbytes
+.Fa aiocbp->aio_nbytes
 is not valid.
 .El
 .Pp
@@ -164,17 +171,20 @@ system call is either one of the error returns from the
 .Xr write 2
 system call, or one of:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
-.Fa iocb->aio_fildes
+.Fa aiocbp->aio_fildes
 argument
 is invalid for writing.
+.\" ==========
 .It Bq Er ECANCELED
 The request was explicitly canceled via a call to
 .Fn aio_cancel .
+.\" ==========
 .It Bq Er EINVAL
 The offset
-.Fa iocb->aio_offset
+.Fa aiocbp->aio_offset
 would be invalid.
 .El
 .Sh SEE ALSO
@@ -200,5 +210,5 @@ This manual page was written by
 .An Wes Peters Aq wes@softweyr.com .
 .Sh BUGS
 Invalid information in
-.Fa iocb->_aiocb_private
+.Fa aiocbp->_aiocb_private
 may confuse the kernel.
diff --git a/bsd/man/man2/audit.2 b/bsd/man/man2/audit.2
new file mode 100644
index 000000000..ba7208392
--- /dev/null
+++ b/bsd/man/man2/audit.2
@@ -0,0 +1,78 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt AUDIT 2
+.Os Darwin
+.Sh NAME
+.Nm audit
+.Nd submit a record to the kernel for auditing 
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn audit "const void * record" "int length"
+.Sh DESCRIPTION
+The
+.Fn audit
+function submits a record to the kernel for inclusion in the global audit 
+trail. The record must already be in BSM format. To protect the integrity
+of the audit trail, this system call must be made with sufficient privileges.
+.Fa Libbsm
+can be used to create and manipulate BSM data.
+.Fa Length
+is the length in bytes of the BSM record and
+.Fa record
+points to the data.
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh ERRORS
+.Bl -tag -width Er
+The
+.Fn audit
+system call will fail if:
+.\" ===========
+.It Bq Er EINVAL
+.Fa Length
+is greater than MAX_AUDIT_RECORD_SIZE, less than zero, greater than the
+internal buffer size, or the record fails verification.
+.\" ===========
+.It Bq Er ENOTSUP
+The security auditing service is not available.
+.\" ===========
+.It Bq Er EPERM
+The call was made with insufficient privileges to complete.
+.\" ===========
+.El
+.Sh SEE ALSO
+.Xr auditon 2 ,
+.Xr auditctl 2 ,
+.Xr getauid 2 ,
+.Xr setauid 2 ,
+.Xr getaudit 2 ,
+.Xr setaudit 2
+.Sh HISTORY
+The
+.Fn audit
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/auditctl.2 b/bsd/man/man2/auditctl.2
new file mode 100644
index 000000000..c5982d5f4
--- /dev/null
+++ b/bsd/man/man2/auditctl.2
@@ -0,0 +1,59 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt AUDITCTL 2
+.Os Darwin
+.Sh NAME
+.Nm auditctl
+.Nd specify a new file for the audit trail to log to  
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn auditctl "const char * path"
+.Sh DESCRIPTION
+The
+.Fn auditctl
+function directs the kernel to begin writing the global audit trail to the 
+specified to the file specified by
+.Fa path .
+Specifying a 
+.Fa path
+of NULL will tell the system to stop writing to the audit trail, without
+disabling audit. To begin writing to a file again, simply pass another 
+path.
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditon 2 ,
+.Xr getauid 2 ,
+.Xr setauid 2 ,
+.Xr getaudit 2 ,
+.Xr setaudit 2
+.Sh HISTORY
+The
+.Fn auditctl
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/auditon.2 b/bsd/man/man2/auditon.2
new file mode 100644
index 000000000..bfd9bad2e
--- /dev/null
+++ b/bsd/man/man2/auditon.2
@@ -0,0 +1,195 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt AUDITON 2
+.Os Darwin
+.Sh NAME
+.Nm auditon
+.Nd configure the current audit parameters on the system
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn auditon "int cmd" "void * data" "int length"
+.Sh DESCRIPTION
+The
+.Fn auditon
+function manipulates various audit parameters. The 
+.Fa data 
+argument points to the appropriate structure from the header file.
+.Fa Length
+is the size of the 
+.Fa data 
+parameter in bytes. It will typically be the sizeof the the structure.
+.Sh PARAMETERS
+.Bl -tag -width Er
+.It A_GETPOLICY
+Get the current audit policy. 
+.Fa Data
+should point to a long. The policy is the bitwise OR'ing of the 
+appropriate flags from 
+.Fa bsm/audit.h .
+If AUDIT_AHLT is set, the system will kernel panic if it cannot write to the
+global audit trail.  If AUDIT_CNT is not set and the system becomes low on
+space, audited events will block until the low space condition is remedied.
+Unaudited events are unaffected. The other policy flags are not implemented.
+.It A_SETPOLICY
+Set the current audit policy.
+.Fa Data 
+should point to a long specifying the desired audit policy, as described in
+A_GETPOLICY.
+.It A_GETKMASK
+Get the current value of the audit preselection mask for non-attributable events.
+.Fa Data
+should point to an
+.Fa au_mask_t .
+The field
+.Fa am_success
+specifies which classes of successful audit events are to be logged to the
+audit trail. The field
+.Fa am_failure
+specifies which classes of failed audit events are to be logged. The value of
+both fields is the bitwise OR'ing of the event classes specified in
+.Fa bsm/audit.h .
+The various audit classes are described more fully in
+.Xr audit_class 5 .
+.It A_SETKMASK
+Set the current value of the audit preselection mask for non-attributable events.
+.Fa Data
+should point to an
+.Fa au_mask_t .
+The masks are defined as described in A_GETKMASK.
+.It A_GETQCTRL
+Get the current settings for the audit queue (specifying in kernel buffer size, 
+percentage of free filesystem blocks, and limits to the number of audit records
+allowed).
+.Fa Data
+should point to an
+.Fa au_qctrl_t .
+.It A_SETQCTRL
+Set the current settings for the audit queue.
+.Fa Data
+should point to an
+.Fa au_qctrl_t .
+.\" The following are not yet implemented, but as mentioned in the header file.
+.\" .It A_GETCWD
+.\" .It A_GETCAR
+.\" .It A_GETSTAT
+.\" .It A_SETSTAT
+.\" .It A_SETUMASK
+.\" .It A_SETSMASK
+.It A_GETCOND
+Gets the current condition of the auditing subsystem. If the value is
+AUC_AUDITING, then the audit implementation is currently running. If the 
+value is AUC_NOAUDIT then the audit implementation is currently turned off.
+.Fa Data
+should point to a long.
+.It A_SETCOND
+Sets the condition of the auditing subsystem. If AUC_NOAUDIT is set, then
+auditing is temporarily suspended. If AUC_AUDITING is set, auditing is resumed.
+If AUC_DISABLED is set, the auditing system will shutdown, draining all audit
+records and closing out the audit trail file. 
+To re-enable auditing, a call to
+.Fa auditctl
+is required in addition to setting the condition to AUC_AUDITING. 
+.Fa Data
+should point to a long.
+.It A_GETCLASS
+Returns the audit class for the specified audit event. 
+.Fa Data
+should point to a 
+.Fa au_evclassmap_t .
+.It A_SETCLASS
+Sets the audit class for the specified audit event. 
+.Fa Data
+should point to a 
+.Fa au_evclassmap_t .
+.It A_GETPINFO
+Returns the audit information stored in the credential for the current process.
+.Fa Data
+should point to a 
+.Fa auditpinfo_t .
+.It A_SETPMASK
+Sets the audit settings for a process. The audit user ID, preselection masks 
+for both success and failure, and terminal IDs must be set.
+.Fa Data
+should point to a 
+.Fa auditpinfo_t 
+struct.
+.It A_SETFSIZE
+Set the limit on audit trail file size. File size is in bytes. The file size 
+specified is treated as an advisory limit. The system will make a best effort
+attempt to rotate log files before they exceed the requested maximum size, but
+makes no guarantees on log file size
+.Fa Data
+should point to a 
+.Fa au_fstat_t 
+struct. The
+.Fa af_filesz
+field is used to specify the new file size, which must be greater than
+MIN_AUDIT_FILE_SIZE. A value of 0 indicates no limit on the audit trail's size. The 
+.Fa af_currsz
+field is ignored. A errno value of EINVAL indicates a maximum file size that is
+too small.
+.It A_GETFSIZE
+Return the maximum allowable size of the audit trail, and the current size of 
+the audit trail.
+.Fa Data
+should point to a
+.Fa au_fstat_t
+struct. 
+.It A_GETPINFO_ADDR
+Not implemented, returns ENOSYS.
+.It A_GETKAUDIT
+Not implemented, returns ENOSYS.
+.It A_SETKAUDIT
+Not implemented, returns ENOSYS.
+.El
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh ERRORS
+.Bl -tag -width Er
+Errors are specific to the operation requested. In addition, rhe
+.Fn auditon
+system call will fail if:
+.\" ===========
+.It Bq Er EINVAL
+.Fa Length
+is less than or equal to zero, or if it is greater than any of the expected structures.
+.El
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditctl 2 ,
+.Xr getauid 2 ,
+.Xr setauid 2 ,
+.Xr getaudit 2 ,
+.Xr setaudit 2 ,
+.Xr getaudit_addr 2 ,
+.Xr setaudit_addr 2 ,
+.Xr audit_class 5
+.Sh HISTORY
+The
+.Fn auditon
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/bind.2 b/bsd/man/man2/bind.2
index 742a58858..694ad472e 100644
--- a/bsd/man/man2/bind.2
+++ b/bsd/man/man2/bind.2
@@ -40,10 +40,13 @@
 .Nm bind
 .Nd bind a name to a socket
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn bind "int s" "const struct sockaddr *name" "socklen_t namelen"
+.Fo bind
+.Fa "int socket"
+.Fa "const struct sockaddr *address"
+.Fa "socklen_t address_len"
+.Fc
 .Sh DESCRIPTION
 .Fn Bind
 assigns a name to an unnamed socket.
@@ -54,7 +57,7 @@ it exists in a name space (address family)
 but has no name assigned.
 .Fn Bind
 requests that
-.Fa name
+.Fa address
 be assigned to the socket.
 .Sh NOTES
 Binding a name in the UNIX domain creates a socket in the file
@@ -65,63 +68,109 @@ needed (using
 The rules used in name binding vary between communication domains.
 Consult the manual entries in section 4 for detailed information.
 .Sh RETURN VALUES
-If the bind is successful, a 0 value is returned.
-A return value of -1 indicates an error, which is
-further specified in the global
-.Va errno .
+Upon successful completion, a value of 0 is returned.
+Otherwise, a value of -1 is returned and the global integer variable
+.Va errno
+is set to indicate the error.
 .Sh ERRORS
 The
 .Fn bind
-call will fail if:
+system call will fail if:
 .Bl -tag -width Er
-.It Bq Er EBADF
-.Fa S
-is not a valid descriptor.
-.It Bq Er ENOTSOCK
-.Fa S
-is not a socket.
-.It Bq Er EADDRNOTAVAIL
-The specified address is not available from the local machine.
-.It Bq Er EADDRINUSE
-The specified address is already in use.
-.It Bq Er EINVAL
-The socket is already bound to an address.
+.\" ==========
 .It Bq Er EACCES
 The requested address is protected, and the current user
 has inadequate permission to access it.
+.\" ==========
+.It Bq Er EADDRINUSE
+The specified address is already in use.
+.\" ==========
+.It Bq Er EADDRNOTAVAIL
+The specified address is not available from the local machine.
+.\" ==========
+.It Bq Er EAFNOSUPPORT
+.Fa address
+is not valid for the address family of
+.Fa socket .
+.\" ==========
+.It Bq Er EBADF
+.Fa socket
+is not a valid file descriptor.
+.\" ==========
+.It Bq Er EDESTADDRREQ
+.Fa socket
+is a null pointer.
+.\" ==========
 .It Bq Er EFAULT
 The
-.Fa name
+.Fa address
 parameter is not in a valid part of the user
 address space.
+.\" ==========
+.It Bq Er EINVAL
+.Fa socket
+is already bound to an address
+and the protocol does not support binding to a new address.
+Alternatively,
+.Fa socket
+may have been shut down.
+.\" ==========
+.It Bq Er ENOTSOCK
+.Fa socket
+does not refer to a socket.
+.\" ==========
+.It Bq Er EOPNOTSUPP
+.Fa socket
+is not of a type that can be bound to an address.
 .El
 .Pp
 The following errors are specific to binding names in the UNIX domain.
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EACCES
+A component of the path prefix does not allow searching
+or the node's parent directory denies write permission.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while making the directory entry
+or allocating the inode.
+.\" ==========
+.It Bq Er EISDIR
+An empty pathname was specified.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ==========
 .It Bq Er ENOENT
-A prefix component of the path name does not exist.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EIO
-An I/O error occurred while making the directory entry or allocating the inode.
+A component of the path name does not refer to an existing file.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
 .It Bq Er EROFS
 The name would reside on a read-only file system.
-.It Bq Er EISDIR
-An empty pathname was specified.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr connect 2 ,
+.Xr getsockname 2 ,
 .Xr listen 2 ,
 .Xr socket 2 ,
-.Xr getsockname 2
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn bind
diff --git a/bsd/man/man2/chdir.2 b/bsd/man/man2/chdir.2
index 407f709e2..d8476b8e6 100644
--- a/bsd/man/man2/chdir.2
+++ b/bsd/man/man2/chdir.2
@@ -43,9 +43,13 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn chdir "const char *path"
+.Fo chdir
+.Fa "const char *path"
+.Fc
 .Ft int
-.Fn fchdir "int fd"
+.Fo fchdir
+.Fa "int fildes"
+.Fc
 .Sh DESCRIPTION
 The
 .Fa path
@@ -63,7 +67,7 @@ The
 .Fn fchdir
 function
 causes the directory referenced by
-.Fa fd
+.Fa fildes
 to become the current working directory,
 the starting point for path searches of pathnames not beginning with
 a slash,
@@ -78,45 +82,64 @@ Otherwise, a value of -1 is returned and
 is set to indicate
 the error.
 .Sh ERRORS
+The
 .Fn Chdir
-will fail and the current working directory will be unchanged if
-one or more of the following are true:
+system call will fail
+and the current working directory will be unchanged
+if one or more of the following are true:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The named directory does not exist.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for any component of
 the path name.
+.\" ==========
 .It Bq Er EFAULT
 .Fa Path
 points outside the process's allocated address space.
+.\" ==========
 .It Bq Er EIO
 An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded 
+.Dv {NAME_MAX}
+characters, or an entire path name exceeded 
+.Dv {PATH_MAX}
+characters.
+.\" ==========
+.It Bq Er ENOENT
+The named directory does not exist.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
 .Pp
 .Fn Fchdir
 will fail and the current working directory will be unchanged if
 one or more of the following are true:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for the directory referenced by the
 file descriptor.
-.It Bq Er ENOTDIR
-The file descriptor does not reference a directory.
+.\" ==========
 .It Bq Er EBADF
 The argument
-.Fa fd
+.Fa fildes
 is not a valid file descriptor.
+.\" ==========
+.It Bq Er EINTR
+.Fn Fchdir was interrupted by a signal.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ENOTDIR
+The file descriptor does not reference a directory.
 .El
 .Sh SEE ALSO
 .Xr chroot 2
diff --git a/bsd/man/man2/chflags.2 b/bsd/man/man2/chflags.2
index 66a036bbb..ac7301709 100644
--- a/bsd/man/man2/chflags.2
+++ b/bsd/man/man2/chflags.2
@@ -60,15 +60,19 @@ The flags specified are formed by
 .Em or Ns 'ing
 the following values
 .Pp
-.Bl -tag -width "SF_IMMUTABLE" -compact -offset indent
+.Bl -tag -width "SF_IMMUTABLEX" -compact -offset indent
 .It UF_NODUMP
 Do not dump the file.
 .It UF_IMMUTABLE
 The file may not be changed.
 .It UF_APPEND
 The file may only be appended to.
-.\".It ARCHIVED
-.\"File is archived.
+.It UF_OPAQUE
+The directory is opaque when viewed through a union stack.
+.It UF_HIDDEN
+The file or directory is not intended to be displayed to the user.
+.It SF_ARCHIVED
+The file has been archived.
 .It SF_IMMUTABLE
 The file may not be changed.
 .It SF_APPEND
@@ -76,12 +80,15 @@ The file may only be appended to.
 .El
 .Pp
 The
-.Dq UF_IMMUTABLE
+.Dq UF_IMMUTABLE ,
+.Dq UF_APPEND ,
+.Dq UF_OPAQUE ,
 and
-.Dq UF_APPEND
+.Dq UF_HIDDEN
 flags may be set or unset by either the owner of a file or the super-user.
 .Pp
 The
+.Dq SF_ARCHIVED ,
 .Dq SF_IMMUTABLE
 and
 .Dq SF_APPEND
@@ -148,6 +155,8 @@ error occurred while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
 .Xr chflags 1 ,
+.Xr fflagstostr 3 ,
+.Xr strtofflags 3 ,
 .Xr init 8
 .Sh HISTORY
 The
diff --git a/bsd/man/man2/chmod.2 b/bsd/man/man2/chmod.2
index bc34d2417..69002be40 100644
--- a/bsd/man/man2/chmod.2
+++ b/bsd/man/man2/chmod.2
@@ -44,9 +44,15 @@
 .Fd #include <sys/types.h>
 .Fd #include <sys/stat.h>
 .Ft int
-.Fn chmod "const char *path" "mode_t mode"
+.Fo chmod
+.Fa "const char *path"
+.Fa "mode_t mode"
+.Fc
 .Ft int
-.Fn fchmod "int fd" "mode_t mode"
+.Fo fchmod
+.Fa "int fildes"
+.Fa "mode_t mode"
+.Fc
 .Sh DESCRIPTION
 The function
 .Fn chmod
@@ -59,13 +65,13 @@ to
 .Fn Fchmod
 sets the permission bits of the specified
 file descriptor
-.Fa fd .
+.Fa fildes .
 .Fn Chmod
 verifies that the process owner (user) either owns
 the file specified by
 .Fa path
 (or
-.Fa fd ) ,
+.Fa fildes ) ,
 or
 is the super-user.
 A mode is created from
@@ -126,53 +132,91 @@ Otherwise, a value of -1 is returned and
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Chmod
-will fail and the file mode will be unchanged if:
+The
+.Fn chmod
+system call will fail and the file mode will be unchanged if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ==========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.\" ==========
+.It Bq Er EINTR
+Its execution was interrupted by a signal.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ==========
 .It Bq Er ENOENT
 The named file does not exist.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
 .It Bq Er EPERM
 The effective user ID does not match the owner of the file and
 the effective user ID is not the super-user.
+.\" ==========
 .It Bq Er EROFS
 The named file resides on a read-only file system.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
 .El
 .Pp
-.Fn Fchmod
+.Fn fchmod
 will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-The descriptor is not valid.
+.Fa fildes
+is not a valid file descriptor.
+.\" ==========
 .It Bq Er EINVAL
-.Fa fd
+.Fa fildes
 refers to a socket, not to a file.
-.It Bq Er EROFS
-The file resides on a read-only file system.
+.\" ==========
+.It Bq Er EINVAL
+.Fa mode
+is not a valid file mode.
+.\" ==========
+.It Bq Er EINTR
+Its execution was interrupted by a signal.
+.\" ==========
 .It Bq Er EIO
 An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er EPERM
+The effective user ID does not match the owner of the file and
+the effective user ID is not the super-user.
+.\" ==========
+.It Bq Er EROFS
+The file resides on a read-only file system.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/stat.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr chmod 1 ,
-.Xr open 2 ,
 .Xr chown 2 ,
+.Xr open 2 ,
 .Xr stat 2 ,
+.Xr compat 5 ,
 .Xr sticky 8
 .Sh STANDARDS
 The
diff --git a/bsd/man/man2/chown.2 b/bsd/man/man2/chown.2
index 7ba416f38..7179b362e 100644
--- a/bsd/man/man2/chown.2
+++ b/bsd/man/man2/chown.2
@@ -42,17 +42,29 @@
 .Sh SYNOPSIS
 .In unistd.h
 .Ft int
-.Fn chown "const char *path" "uid_t owner" "gid_t group"
+.Fo chown
+.Fa "const char *path"
+.Fa "uid_t owner"
+.Fa "gid_t group"
+.Fc
 .Ft int
-.Fn fchown "int fd" "uid_t owner" "gid_t group"
+.Fo fchown
+.Fa "int fildes"
+.Fa "uid_t owner"
+.Fa "gid_t group"
+.Fc
 .Ft int
-.Fn lchown "const char *path" "uid_t owner" "gid_t group"
+.Fo lchown
+.Fa "const char *path"
+.Fa "uid_t owner"
+.Fa "gid_t group"
+.Fc
 .Sh DESCRIPTION
 The owner ID and group ID of the file
 named by
 .Fa path
 or referenced by
-.Fa fd
+.Fa fildes
 is changed as specified by the arguments
 .Fa owner
 and
@@ -97,56 +109,72 @@ may be left unchanged by specifying it as -1.
 .Sh RETURN VALUES
 .Rv -std
 .Sh ERRORS
+.Pp
 The
 .Fn chown
 and
 .Fn lchown
-will fail and the file will be unchanged if:
+system calls will fail if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 255 characters,
-or an entire path name exceeded 1023 characters.
-.It Bq Er ENOENT
-The named file does not exist.
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EPERM
-The effective user ID is not the super-user.
-.It Bq Er EROFS
-The named file resides on a read-only file system.
 .It Bq Er EFAULT
 The
 .Fa path
 argument
 points outside the process's allocated address space.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded 255 characters,
+or an entire path name exceeded 1023 characters.
+.\" ==========
+.It Bq Er ENOENT
+A component of
+.Fa path
+does not exist.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
 .Pp
 The
 .Fn fchown
 system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The
-.Fa fd
+.Fa fildes
 argument
 does not refer to a valid descriptor.
+.\" ==========
 .It Bq Er EINVAL
 The
-.Fa fd
+.Fa fildes
 argument
 refers to a socket, not a file.
+.El
+.Pp
+Any of these calls will fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINTR
+Its execution is interrupted by a signal.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurs while reading from or writing to the file system.
+.\" ==========
 .It Bq Er EPERM
-The effective user ID is not the super-user.
+The effective user ID does not match the owner of the file
+and the calling process does not have appropriate (i.e., root) privileges.
+.\" ==========
 .It Bq Er EROFS
 The named file resides on a read-only file system.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
 .Xr chgrp 1 ,
diff --git a/bsd/man/man2/close.2 b/bsd/man/man2/close.2
index 5cd4d18ec..0aa08b400 100644
--- a/bsd/man/man2/close.2
+++ b/bsd/man/man2/close.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn close "int d"
+.Fo close
+.Fa "int fildes"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn close
@@ -98,24 +100,32 @@ Otherwise, a value of -1 is returned and the global integer variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Close
-will fail if:
+The
+.Fn close
+system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-.Fa D
-is not an active descriptor.
+.Fa fildes
+is not a valid, active file descriptor.
+.\" ==========
 .It Bq Er EINTR
-An interrupt was received.
+Its execution was interrupted by a signal.
+.\" ==========
+.It Bq Er EIO
+A previously-uncommitted
+.Xr write 2
+encountered an input/output error.
 .El
 .Sh SEE ALSO
 .Xr accept 2 ,
+.Xr execve 2 ,
+.Xr fcntl 2 ,
 .Xr flock 2 ,
 .Xr open 2 ,
 .Xr pipe 2 ,
 .Xr socket 2 ,
-.Xr socketpair 2 ,
-.Xr execve 2 ,
-.Xr fcntl 2
+.Xr socketpair 2
 .Sh STANDARDS
 .Fn Close
 conforms to 
diff --git a/bsd/man/man2/connect.2 b/bsd/man/man2/connect.2
index c778b8d3a..8b32e6c2c 100644
--- a/bsd/man/man2/connect.2
+++ b/bsd/man/man2/connect.2
@@ -43,10 +43,14 @@
 .Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn connect "int s" "const struct sockaddr *name" "socklen_t namelen"
+.Fo connect
+.Fa "int socket"
+.Fa "const struct sockaddr *address"
+.Fa "socklen_t address_len"
+.Fc
 .Sh DESCRIPTION
 The parameter
-.Fa s
+.Fa socket
 is a socket.
 If it is of type
 .Dv SOCK_DGRAM ,
@@ -58,10 +62,11 @@ If the socket is of type
 this call attempts to make a connection to
 another socket.
 The other socket is specified by
-.Fa name ,
+.Fa address ,
 which is an address in the communications space of the socket.
+.Pp
 Each communications space interprets the
-.Fa name
+.Fa address
 parameter in its own way.
 Generally, stream sockets may successfully
 .Fn connect
@@ -77,40 +82,54 @@ the address family set to
 .Dv EAFNOSUPPORT 
 will be harmlessly returned).
 .Sh RETURN VALUES
-If the connection or binding succeeds, 0 is returned.
-Otherwise a -1 is returned, and a more specific error
-code is stored in
-.Va errno .
+Upon successful completion, a value of 0 is returned.
+Otherwise, a value of -1 is returned and the global integer variable
+.Va errno
+is set to indicate the error.
 .Sh ERRORS
 The
 .Fn connect
-call fails if:
+system call will fail if:
 .Bl -tag -width Er
-.It Bq Er EBADF
-.Fa S
-is not a valid descriptor.
-.It Bq Er ENOTSOCK
-.Fa S
-is a descriptor for a file, not a socket.
+.\" ==========
+.It Bq Er EACCES
+The destination address is a broadcast address and the 
+socket option 
+.Dv SO_BROADCAST 
+is not set.
+.\" ==========
+.It Bq Er EADDRINUSE
+The address is already in use.
+.\" ==========
 .It Bq Er EADDRNOTAVAIL
 The specified address is not available on this machine.
+.\" ==========
 .It Bq Er EAFNOSUPPORT
 Addresses in the specified address family cannot be used with this socket.
-.It Bq Er EISCONN
-The socket is already connected.
-.It Bq Er ETIMEDOUT
-Connection establishment timed out without establishing a connection.
+.\" ==========
+.It Bq Er EALREADY
+The socket is non-blocking
+and a previous connection attempt
+has not yet been completed.
+.\" ==========
+.It Bq Er EBADF
+.Fa socket
+is not a valid descriptor.
+.\" ==========
 .It Bq Er ECONNREFUSED
-The attempt to connect was forcefully rejected.
-.It Bq Er ENETUNREACH
-The network isn't reachable from this host.
-.It Bq Er EADDRINUSE
-The address is already in use.
+The attempt to connect was ignored
+(because the target is not listening for connections)
+or explicitly rejected.
+.\" ==========
 .It Bq Er EFAULT
 The
-.Fa name
+.Fa address
 parameter specifies an area outside
 the process address space.
+.\" ==========
+.It Bq Er EHOSTUNREACH
+The target host cannot be reached (e.g., down, disconnected).
+.\" ==========
 .It Bq Er EINPROGRESS
 The socket is non-blocking 
 and the connection cannot
@@ -118,42 +137,90 @@ be completed immediately.
 It is possible to
 .Xr select 2
 for completion by selecting the socket for writing.
-.It Bq Er EALREADY
-The socket is non-blocking
-and a previous connection attempt
-has not yet been completed.
-.It Bq Er EACCES
-The destination address is a broadcast address and the 
-socket option 
-.Dv SO_BROADCAST 
-is not set.
+.\" ==========
+.It Bq Er EINTR
+Its execution was interrupted by a signal.
+.\" ==========
+.It Bq Er EINVAL
+An invalid argument was detected
+(e.g.,
+.Fa address_len
+is not valid for the address family,
+the specified address family is invalid).
+.\" ==========
+.It Bq Er EISCONN
+The socket is already connected.
+.\" ==========
+.It Bq Er ENETDOWN
+The local network interface is not functioning.
+.\" ==========
+.It Bq Er ENETUNREACH
+The network isn't reachable from this host.
+.\" ==========
+.It Bq Er ENOBUFS
+The system call was unable to allocate a needed memory buffer.
+.\" ==========
+.It Bq Er ENOTSOCK
+.Fa socket
+is not a file descriptor for a socket.
+.\" ==========
+.It Bq Er EOPNOTSUPP
+Because
+.Fa socket
+is listening, no connection is allowed.
+.\" ==========
+.It Bq Er EPROTOTYPE
+.Fa address
+has a different type than the socket
+that is bound to the specified peer address.
+.\" ==========
+.It Bq Er ETIMEDOUT
+Connection establishment timed out without establishing a connection.
 .El
 .Pp
 The following errors are specific to connecting names in the UNIX domain.
 These errors may not apply in future versions of the UNIX IPC domain.
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ==========
+.It Bq Er EACCES
+Write access to the named socket is denied.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ==========
 .It Bq Er ENOENT
 The named socket does not exist.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er EACCES
-Write access to the named socket is denied.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr accept 2 ,
+.Xr getsockname 2 ,
 .Xr select 2 ,
 .Xr socket 2 ,
-.Xr getsockname 2
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn connect
diff --git a/bsd/man/man2/dup.2 b/bsd/man/man2/dup.2
index b9c0b1980..019b13fbf 100644
--- a/bsd/man/man2/dup.2
+++ b/bsd/man/man2/dup.2
@@ -43,21 +43,27 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn dup "int oldd"
+.Fo dup
+.Fa "int fildes"
+.Fc
 .Ft int
-.Fn dup2 "int oldd" "int newd"
+.Fo dup2
+.Fa "int fildes"
+.Fa "int fildes2"
+.Fc
 .Sh DESCRIPTION
 .Fn Dup
-duplicates an existing object descriptor and returns its value to
-the calling process
-.Fa ( newd
+duplicates an existing object descriptor
+and returns its value to the calling process
+.Fa ( fildes2
 =
-.Fn dup oldd ) .
+.Fn dup fildes ) .
 The argument
-.Fa oldd
-is a small non-negative integer index in
-the per-process descriptor table.  The value must be less
-than the size of the table, which is returned by
+.Fa fildes
+is a small non-negative integer index
+in the per-process descriptor table.
+The value must be less than the size of the table,
+which is returned by
 .Xr getdtablesize 2 .
 The new descriptor returned by the call
 is the lowest numbered descriptor
@@ -65,14 +71,14 @@ currently not in use by the process.
 .Pp
 The object referenced by the descriptor does not distinguish
 between
-.Fa oldd
+.Fa fildes
 and
-.Fa newd
+.Fa fildes2
 in any way.
 Thus if
-.Fa newd
+.Fa fildes2
 and
-.Fa oldd
+.Fa fildes
 are duplicate references to an open
 file,
 .Xr read 2 ,
@@ -82,9 +88,9 @@ and
 calls all move a single pointer into the file,
 and append mode, non-blocking I/O and asynchronous I/O options
 are shared between the references.
-If a separate pointer into the file is desired, a different
-object reference to the file must be obtained by issuing an
-additional
+If a separate pointer into the file is desired,
+a different object reference to the file must be obtained
+by issuing an additional
 .Xr open 2
 call.
 The close-on-exec flag on the new file descriptor is unset.
@@ -92,39 +98,46 @@ The close-on-exec flag on the new file descriptor is unset.
 In 
 .Fn dup2 ,
 the value of the new descriptor
-.Fa newd
-is specified.  If this descriptor is already
-in use, the descriptor is first deallocated as if a
+.Fa fildes2
+is specified.
+If this descriptor is already in use,
+the descriptor is first deallocated as if a
 .Xr close 2
 call had been done first.
 .Sh RETURN VALUES
-The value -1 is returned if an error occurs in either call.
-The external variable
+Upon successful completion, a value of 0 is returned.
+Otherwise, a value of -1 is returned and the global integer variable
 .Va errno
-indicates the cause of the error.
+is set to indicate the error.
 .Sh ERRORS
-.Fn Dup
+The
+.Fn dup
 and
 .Fn dup2
-fail if:
+system calls will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-.Fa Oldd
+.Fa fildes
 or
-.Fa newd
-is not a valid active descriptor
+.Fa fildes2
+is not an active, valid file descriptor.
+.\" ==========
+.It Bq Er EINTR
+Execution is interrupted by a signal.
+.\" ==========
 .It Bq Er EMFILE
-Too many descriptors are active.
+Too many file descriptors are active.
 .El
 .Sh SEE ALSO
 .Xr accept 2 ,
-.Xr open 2 ,
 .Xr close 2 ,
 .Xr fcntl 2 ,
+.Xr getdtablesize 2 ,
+.Xr open 2 ,
 .Xr pipe 2 ,
 .Xr socket 2 ,
-.Xr socketpair 2 ,
-.Xr getdtablesize 2
+.Xr socketpair 2
 .Sh STANDARDS
 .Fn Dup
 and
diff --git a/bsd/man/man2/execve.2 b/bsd/man/man2/execve.2
index f2bc9dd8c..6f7e4f336 100644
--- a/bsd/man/man2/execve.2
+++ b/bsd/man/man2/execve.2
@@ -42,7 +42,11 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn execve "const char *path" "char *const argv[]" "char *const envp[]"
+.Fo execve
+.Fa "const char *path"
+.Fa "char *const argv[]"
+.Fa "char *const envp[]"
+.Fc
 .Sh DESCRIPTION
 .Fn Execve
 transforms the calling process into a new process.
@@ -64,26 +68,26 @@ An interpreter file begins with a line of the form:
 .Bd -filled -offset indent -compact
 .Sy \&#!
 .Em interpreter
-.Bq Em arg
+.Bq Em arg ...
 .Ed
 .Pp
 When an interpreter file is
 .Fn execve Ap d ,
-the system
-.Fn execve Ap s
-runs the specified
+the system runs the specified
 .Em interpreter .
-If the optional
-.Em arg
-is specified, it becomes the first argument to the
-.Em interpreter ,
-and the name of the originally
+If any optional
+.Em args
+are specified, they become the first (second, ...) argument to the
+.Em interpreter.
+The name of the originally
 .Fn execve Ap d
-file becomes the second argument;
+file becomes the subsequent argument;
 otherwise, the name of the originally
 .Fn execve Ap d
-file becomes the first argument.  The original arguments are shifted over to
-become the subsequent arguments.  The zeroth argument, normally the name of the
+file is the first argument.
+The original arguments to the invocation of the interpreter
+are shifted over to become the final arguments.
+The zeroth argument, normally the name of the
 .Fn execve Ap d
 file, is left unchanged.
 .Pp
@@ -183,63 +187,47 @@ to the arguments themselves.
 .Sh RETURN VALUES
 As the
 .Fn execve
-function overlays the current process image 
-with a new process image the successful call
-has no process to return to.
+function overlays the current process image  with a new process image,
+the successful call has no process to return to.
 If
 .Fn execve
-does return to the calling process an error has occurred; the
-return value will be -1 and the global variable
+does return to the calling process, an error has occurred;
+the return value will be -1 and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
 .Fn Execve
 will fail and return to the calling process if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The new process file does not exist.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ==========
+.It Bq Er E2BIG
+The number of bytes in the new process's argument list
+is larger than the system-imposed limit.
+This limit is specified by the
+.Xr sysctl 3
+MIB variable
+.Dv KERN_ARGMAX .
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
+.\" ==========
 .It Bq Er EACCES
 The new process file is not an ordinary file.
+.\" ==========
 .It Bq Er EACCES
 The new process file mode denies execute permission.
+.\" ==========
 .It Bq Er EACCES
-The new process file is on a filesystem mounted with execution
-disabled
+The new process file is on a filesystem mounted
+with execution disabled
 .Pf ( Dv MNT_NOEXEC
 in
 .Ao Pa sys/mount.h Ac ) .
-.It Bq Er ENOEXEC
-The new process file has the appropriate access
-permission, but has an invalid magic number in its header.
-.It Bq Er ETXTBSY
-The new process file is a pure procedure (shared text)
-file that is currently open for writing or reading by some process.
-.It Bq Er ENOMEM
-The new process requires more virtual memory than
-is allowed by the imposed maximum
-.Pq Xr getrlimit 2 .
-.It Bq Er E2BIG
-The number of bytes in the new process's argument list
-is larger than the system-imposed limit.
-This limit is specified by the
-.Xr sysctl 3
-MIB variable
-.Dv KERN_ARGMAX .
+.\" ==========
 .It Bq Er EFAULT
 The new process file is not as long as indicated by
 the size values in its header.
+.\" ==========
 .It Bq Er EFAULT
 .Fa Path ,
 .Fa argv ,
@@ -247,8 +235,40 @@ or
 .Fa envp
 point
 to an illegal address.
+.\" ==========
 .It Bq Er EIO
 An I/O error occurred while reading from the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded 
+.Dv {NAME_MAX}
+characters, or an entire path name exceeded 
+.Dv {PATH_MAX}
+characters.
+.\" ==========
+.It Bq Er ENOENT
+The new process file does not exist.
+.\" ==========
+.It Bq Er ENOEXEC
+The new process file has the appropriate access
+permission, but has an unrecognized format
+(e.g., an invalid magic number in its header).
+.\" ==========
+.It Bq Er ENOMEM
+The new process requires more virtual memory than
+is allowed by the imposed maximum
+.Pq Xr getrlimit 2 .
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er ETXTBSY
+The new process file is a pure procedure (shared text)
+file that is currently open for writing or reading by some process.
 .El
 .Sh CAVEAT
 If a program is
@@ -262,6 +282,7 @@ of a super-user as well.
 .Xr exit 2 ,
 .Xr fork 2 ,
 .Xr execl 3 ,
+.Xr sysctl 3 ,
 .Xr environ 7
 .Sh HISTORY
 The
diff --git a/bsd/man/man2/fcntl.2 b/bsd/man/man2/fcntl.2
index 12fd284f4..d228a17c1 100644
--- a/bsd/man/man2/fcntl.2
+++ b/bsd/man/man2/fcntl.2
@@ -42,12 +42,16 @@
 .Sh SYNOPSIS
 .Fd #include <fcntl.h>
 .Ft int
-.Fn fcntl "int fd" "int cmd" "int arg"
+.Fo fcntl
+.Fa "int fildes"
+.Fa "int cmd"
+.Fa "..."
+.Fc
 .Sh DESCRIPTION
 .Fn Fcntl
 provides for control over descriptors.
 The argument
-.Fa fd
+.Fa fildes
 is a descriptor to be operated on by
 .Fa cmd
 as follows:
@@ -77,7 +81,7 @@ system calls.
 .El
 .It Dv F_GETFD
 Get the close-on-exec flag associated with the file descriptor
-.Fa fd .
+.Fa fildes .
 If the low-order bit of the returned value is 0,
 the file will remain open across
 .Fn exec ,
@@ -87,7 +91,7 @@ otherwise the file will be closed upon execution of
 is ignored).
 .It Dv F_SETFD
 Set the close-on-exec flag associated with
-.Fa fd
+.Fa fildes
 to the low order bit of
 .Fa arg
 (0 or 1 as above).
@@ -120,6 +124,12 @@ process groups are specified by supplying
 as negative, otherwise 
 .Fa arg
 is interpreted as a process ID.
+.It Dv F_GETPATH
+Get the path of the file descriptor 
+.Fa Fildes .  
+The argument must be a buffer of size
+.Sy MAXPATHLEN
+or greater.
 .It Dv F_PREALLOCATE
 Preallocate file storage space.
 .It Dv F_SETSIZE
@@ -160,11 +170,11 @@ flush all buffered data to
 the permanent storage device
 .Fa ( arg
 is ignored).
-This is currently
-only implemented on HFS filesystems and
-the operation may take quite a while to
-complete.  Certain FireWire drives have
-also been known to ignore this request.
+This is currently implemented on HFS, MS-DOS (FAT),
+and Universal Disk Format (UDF) file systems.
+The operation may take quite a while to complete.
+Certain FireWire drives have also been known
+to ignore the request to flush their buffered data.
 .El
 .Pp
 The flags for the
@@ -471,9 +481,11 @@ Otherwise, a value of -1 is returned and
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Fcntl
-will fail if:
+The
+.Fn fcntl
+system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EACCES
 The argument
 .Fa cmd
@@ -498,6 +510,7 @@ is either
 or
 .Dv F_WRITEBOOTSTRAP
 and the calling process does not have root privileges.
+.\" ==========
 .It Bq Er EBADF
 .Fa Fildes
 is not a valid open file descriptor.
@@ -544,24 +557,21 @@ is
 and
 .Fa fildes
 is not a valid file descriptor open for reading.
-.It Bq Er EMFILE
-.Fa Cmd
-is
-.Dv F_DUPFD
-and the maximum allowed number of file descriptors are currently
-open.
+.\" ==========
 .It Bq Er EDEADLK
 The argument
 .Fa cmd
 is
 .Dv F_SETLKW ,
 and a deadlock condition was detected.
+.\" ==========
 .It Bq Er EINTR
 The argument
 .Fa cmd
 is
 .Dv F_SETLKW ,
 and the function was interrupted by a signal.
+.\" ==========
 .It Bq Er EINVAL
 .Fa Cmd
 is
@@ -610,6 +620,14 @@ is either
 or
 .Dv F_WRITEBOOTSTRAP
 and the operation was attempted on a non-HFS disk type.
+.\" ==========
+.It Bq Er EMFILE
+.Fa Cmd
+is
+.Dv F_DUPFD
+and the maximum allowed number of file descriptors are currently
+open.
+.\" ==========
 .It Bq Er EMFILE
 The argument
 .Fa cmd
@@ -620,6 +638,7 @@ process are already in use,
 or no file descriptors greater than or equal to
 .Fa arg
 are available.
+.\" ==========
 .It Bq Er ENOLCK
 The argument
 .Fa cmd
@@ -629,6 +648,16 @@ or
 .Dv F_SETLKW ,
 and satisfying the lock or unlock request would result in the
 number of locked regions in the system exceeding a system-imposed limit.
+.\" ==========
+.It Bq Er EOVERFLOW
+A return value would overflow its representation.
+For example,
+.Fa cmd
+is F_GETLK, F_SETLK, or F_SETLKW
+and the smallest (or, if l_len is non-zero, the largest) offset
+of a byte in the requested segment
+will not fit in an object of type off_t.
+.\" ==========
 .It Bq Er ESRCH
 .Fa Cmd
 is
diff --git a/bsd/man/man2/fgetxattr.2 b/bsd/man/man2/fgetxattr.2
new file mode 100644
index 000000000..d9e5d9037
--- /dev/null
+++ b/bsd/man/man2/fgetxattr.2
@@ -0,0 +1 @@
+.so man2/getxattr.2
diff --git a/bsd/man/man2/fhopen.2 b/bsd/man/man2/fhopen.2
new file mode 100644
index 000000000..ddade3eef
--- /dev/null
+++ b/bsd/man/man2/fhopen.2
@@ -0,0 +1,64 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc.  All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd January 9, 2007
+.Dt FHOPEN 2
+.Os
+.Sh NAME
+.Nm fhopen
+.Nd open a file by file handle
+.Sh SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mount.h>
+.Ft int
+.Fn fhopen "const struct fhandle *fhp" "int flags"
+.Sh DESCRIPTION
+The file specified by the given file handle is opened for reading and/or
+writing, as specified by the argument
+.Fa flags ;
+the file descriptor is returned to the calling process.
+.Pp
+This system call is restricted to the super-user and is used by an
+NFS server's
+.Xr rpc.lockd 8
+daemon to open a file in order to perform locking operations on it.
+.Sh RETURN VALUES
+Upon successful completion, a value of 0 is returned.
+Otherwise, -1 is returned and the global variable
+.Va errno
+is set to indicate the error.
+.Sh ERRORS
+.Fn fhopen
+fails if one or more of the following are true:
+.Bl -tag -width Er
+.It Bq ESTALE
+The file handle no longer refers to a valid, NFS-exported object.
+.It Bq EPERM
+The caller is not the super-user.
+.It Bq EINVAL
+There was a problem with the value of one of the arguments.
+.It Bq EFAULT
+.Fa fhp
+points to an invalid address.
+.El
+.Sh SEE ALSO
+.Xr rpc.lockd 8
diff --git a/bsd/man/man2/flistxattr.2 b/bsd/man/man2/flistxattr.2
new file mode 100644
index 000000000..117bd2b53
--- /dev/null
+++ b/bsd/man/man2/flistxattr.2
@@ -0,0 +1 @@
+.so man2/listxattr.2
diff --git a/bsd/man/man2/flock.2 b/bsd/man/man2/flock.2
index c74ac5b69..e72823d8d 100644
--- a/bsd/man/man2/flock.2
+++ b/bsd/man/man2/flock.2
@@ -137,11 +137,11 @@ refers to an object other than a file.
 The referenced descriptor is not of the correct type.
 .El
 .Sh SEE ALSO
-.Xr open 2 ,
 .Xr close 2 ,
 .Xr dup 2 ,
 .Xr execve 2 ,
-.Xr fork 2
+.Xr fork 2 ,
+.Xr open 2
 .Sh HISTORY
 The
 .Fn flock
diff --git a/bsd/man/man2/fork.2 b/bsd/man/man2/fork.2
index 22fbd9354..52aea6389 100644
--- a/bsd/man/man2/fork.2
+++ b/bsd/man/man2/fork.2
@@ -40,10 +40,11 @@
 .Nm fork
 .Nd create a new process
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft pid_t
-.Fn fork void
+.Fo fork
+.Fa void
+.Fc
 .Sh DESCRIPTION
 .Fn Fork
 causes creation of a new process.
@@ -77,35 +78,59 @@ are set to 0; see
 .Sh RETURN VALUES
 Upon successful completion,
 .Fn fork
-returns a value
-of 0 to the child process and returns the process ID of the child
-process to the parent process.  Otherwise, a value of -1 is returned
-to the parent process, no child process is created, and the global
-variable
+returns a value of 0 to the child process
+and returns the process ID of the child process to the parent process.
+Otherwise, a value of -1 is returned to the parent process,
+no child process is created, and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
 .Fn Fork
 will fail and no child process will be created if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EAGAIN
-The system-imposed limit on the total
-number of processes under execution would be exceeded.
+The system-imposed limit
+on the total number of processes under execution
+would be exceeded.
 This limit is configuration-dependent.
+.\" ==========
 .It Bq Er EAGAIN
 The system-imposed limit
 .Dv MAXUPRC
 .Pq Aq Pa sys/param.h
 on the total number of
 processes under execution by a single user would be exceeded.
+.\" ==========
 .It Bq Er ENOMEM
 There is insufficient swap space for the new process.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr execve 2 ,
-.Xr wait 2
+.Xr sigaction 2 ,
+.Xr wait 2 ,
+.Xr compat 5
 .Sh HISTORY
 A
 .Fn fork
 function call appeared in
 .At v6 .
+.Sh CAVEATS
+There are limits to what you can do in the child process.
+To be totally safe you should restrict yourself to only
+executing async-signal safe operations until such time
+as one of the exec functions is called.  All APIs, including
+global data symbols, in any framework or library should be 
+assumed to be unsafe after a 
+.Fn fork
+unless explicitly documented to be safe or async-signal 
+safe.  If you need to use these frameworks in the child 
+process, you must exec.  In this situation it is reasonable 
+to exec yourself.
\ No newline at end of file
diff --git a/bsd/man/man2/fremovexattr.2 b/bsd/man/man2/fremovexattr.2
new file mode 100644
index 000000000..38d01ccde
--- /dev/null
+++ b/bsd/man/man2/fremovexattr.2
@@ -0,0 +1 @@
+.so man2/removexattr.2
diff --git a/bsd/man/man2/fsctl.2 b/bsd/man/man2/fsctl.2
index d5fc742b3..1874f2ff3 100644
--- a/bsd/man/man2/fsctl.2
+++ b/bsd/man/man2/fsctl.2
@@ -126,8 +126,8 @@ or
 is not valid.
 .El
 .Sh SEE ALSO
-.Xr ioctl 2 ,
 .Xr getattrlist 2 ,
+.Xr ioctl 2 ,
 .Xr setattrlist 2
 .Sh HISTORY
 The
diff --git a/bsd/man/man2/fsetxattr.2 b/bsd/man/man2/fsetxattr.2
new file mode 100644
index 000000000..dc0780751
--- /dev/null
+++ b/bsd/man/man2/fsetxattr.2
@@ -0,0 +1 @@
+.so man2/setxattr.2
diff --git a/bsd/man/man2/fstat64.2 b/bsd/man/man2/fstat64.2
new file mode 100644
index 000000000..b1a86c195
--- /dev/null
+++ b/bsd/man/man2/fstat64.2
@@ -0,0 +1 @@
+.so man2/stat.2
diff --git a/bsd/man/man2/fstatfs64.2 b/bsd/man/man2/fstatfs64.2
new file mode 100644
index 000000000..923d3c0cc
--- /dev/null
+++ b/bsd/man/man2/fstatfs64.2
@@ -0,0 +1 @@
+.so man2/statfs.2
diff --git a/bsd/man/man2/fsync.2 b/bsd/man/man2/fsync.2
index b75a4229d..ad72f78de 100644
--- a/bsd/man/man2/fsync.2
+++ b/bsd/man/man2/fsync.2
@@ -42,72 +42,85 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn fsync "int fd"
+.Fo fsync
+.Fa "int fildes"
+.Fc
 .Sh DESCRIPTION
 .Fn Fsync
 causes all modified data and attributes of
-.Fa fd
+.Fa fildes
 to be moved to a permanent storage device.
 This normally results in all in-core modified copies
 of buffers for the associated file to be written to a disk.
 .Pp
 Note that while
 .Fn fsync
-will flush all data from the host
-to the drive (i.e. the "permanent storage
-device"), the
-drive itself may not physically
-write the data to the
-platters for quite some time
-and it may be written in an
-out-of-order sequence.
+will flush all data from the host to the drive
+(i.e. the "permanent storage device"),
+the drive itself may not physically write the data
+to the platters for quite some time
+and it may be written in an out-of-order sequence.
 .Pp
 Specifically, if the drive loses power
 or the OS crashes,
-the application
-may find that only some or none of their data was
-written.  The disk drive may also re-order
-the data so that later writes
-may be present while earlier writes are not.
+the application may find that only some or none of their data was written.
+The disk drive may also re-order the data
+so that later writes may be present, while earlier writes are not.
 .Pp
-This is not a theoretical
-edge case.  This scenario is easily reproduced
-with real world workloads and drive
-power failures.
+This is not a theoretical edge case.
+This scenario is easily reproduced with real world workloads
+and drive power failures.
 .Pp
-For applications that require tighter guarantess about
-the integrity of their data, MacOS X provides the
-F_FULLFSYNC fcntl.  The F_FULLFSYNC fcntl asks the
-drive to flush all buffered data to permanent
-storage.  Applications such as databases that require
-a strict ordering of writes should use F_FULLFSYNC to
-ensure their data is written in the order they expect.
+For applications that require tighter guarantees
+about the integrity of their data,
+Mac OS X provides the F_FULLFSYNC fcntl.
+The F_FULLFSYNC fcntl asks the drive to flush all buffered data
+to permanent storage.
+Applications, such as databases,
+that require a strict ordering of writes
+should use F_FULLFSYNC to ensure that their data
+is written in the order they expect.
 Please see
 .Xr fcntl 2
 for more detail.
 .Pp
 .Sh RETURN VALUES
-A 0 value is returned on success.  A -1 value indicates
-an error.
+.Rv -std fsync
 .Sh ERRORS
 The
 .Fn fsync
-fails if:
+system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-.Fa fd
+.Fa fildes
 is not a valid descriptor.
+.\" ==========
+.It Bq Er EINTR
+Its execution is interrupted by a signal.
+.\" ==========
 .It Bq Er EINVAL
-.Fa fd
-refers to a socket, not to a file.
+.Fa fildes
+refers to a file type (e.g., a socket)
+that does not support this operation.
+.\" ==========
 .It Bq Er EIO
 An I/O error occurred while reading from or writing to the file system.
 .El
+.Pp
+If a queued I/O operation fails,
+.Fn fsync
+may fail with any of the errors defined for
+.Xr read 2
+or
+.Xr write 2 .
 .Sh SEE ALSO
+.Xr fcntl 2 ,
+.Xr read 2 ,
 .Xr sync 2 ,
+.Xr write 2 ,
 .Xr sync 8 ,
-.Xr update 8 ,
-.Xr fcntl 2
+.Xr update 8
 .Sh HISTORY
 The
 .Fn fsync
diff --git a/bsd/man/man2/getattrlist.2 b/bsd/man/man2/getattrlist.2
index e9bb5c33f..d0f718d0d 100644
--- a/bsd/man/man2/getattrlist.2
+++ b/bsd/man/man2/getattrlist.2
@@ -1,11 +1,11 @@
 .\" Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
-.\" 
+.\"
 .\" The contents of this file constitute Original Code as defined in and
 .\" are subject to the Apple Public Source License Version 1.1 (the
 .\" "License").  You may not use this file except in compliance with the
 .\" License.  Please obtain a copy of the License at
 .\" http://www.apple.com/publicsource and read it before using this file.
-.\" 
+.\"
 .\" This Original Code and all software distributed under the License are
 .\" distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 .\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -13,7 +13,7 @@
 .\" FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 .\" License for the specific language governing rights and limitations
 .\" under the License.
-.\" 
+.\"
 .\"     @(#)getattrlist.2
 .
 .Dd October 14, 2004
@@ -32,44 +32,44 @@
 The
 .Fn getattrlist
 function returns attributes (that is, metadata) of file system objects.
-You can think of 
+You can think of
 .Fn getattrlist
 as a seriously enhanced version of
 .Xr stat 2 .
-The function returns attributes about the file system object specified by 
+The function returns attributes about the file system object specified by
 .Fa path
-in the buffer specified by 
+in the buffer specified by
 .Fa attrBuf
 and
 .Fa attrBufSize .
-The 
-.Fa attrList 
-parameter determines what attributes are returned. 
-The 
-.Fa options 
-parameter lets you control specific aspects of the function's behaviour.
-.Pp
-.
-The 
-.Fn getattrlist 
-function is only supported by certain volume format implementations. 
-For maximum compatibility, client programs should use high-level APIs 
+The
+.Fa attrList
+parameter determines what attributes are returned.
+The
+.Fa options
+parameter lets you control specific aspects of the function's behavior.
+.Pp
+.
+The
+.Fn getattrlist
+function is only supported by certain volume format implementations.
+For maximum compatibility, client programs should use high-level APIs
 (such as the Carbon File Manager) to access file system attributes.
-These high-level APIs include logic to emulate file system attributes 
-on volumes that don't support 
+These high-level APIs include logic to emulate file system attributes
+on volumes that don't support
 .Fn getattrlist .
 .Pp
 .
 Not all volumes support all attributes.
-See the discussion of 
-.Dv ATTR_VOL_ATTRIBUTES 
-for a discussion of how to determine whether a particular volume supports a 
+See the discussion of
+.Dv ATTR_VOL_ATTRIBUTES
+for a discussion of how to determine whether a particular volume supports a
 particular attribute.
 .Pp
-Furthermore, you should only request the attributes that you need. 
-Some attributes are expensive to calculate on some volume formats. 
-For example, 
-.Dv ATTR_DIR_ENTRYCOUNT 
+Furthermore, you should only request the attributes that you need.
+Some attributes are expensive to calculate on some volume formats.
+For example,
+.Dv ATTR_DIR_ENTRYCOUNT
 is usually expensive to calculate on non-HFS [Plus] volumes.
 If you don't need a particular attribute, you should not ask for it.
 .Pp
@@ -79,8 +79,8 @@ If you don't need a particular attribute, you should not ask for it.
 The
 .Fa path
 parameter must reference a valid file system object.
-Read, write or execute permission of the object itself is not required, but 
-all directories listed in the path name leading to the object must be 
+Read, write or execute permission of the object itself is not required, but
+all directories listed in the path name leading to the object must be
 searchable.
 .Pp
 .
@@ -88,13 +88,13 @@ searchable.
 .
 The
 .Fa attrList
-parameter is a pointer to an 
-.Vt attrlist 
+parameter is a pointer to an
+.Vt attrlist
 structure, as defined by
 .Aq Pa sys/attr.h
 (shown below).
 It determines what attributes are returned by the function.
-You are responsible for filling out all fields of this structure before calling the function. 
+You are responsible for filling out all fields of this structure before calling the function.
 .Bd -literal
 typedef u_int32_t attrgroup_t;
 .Pp
@@ -113,14 +113,14 @@ struct attrlist {
 .
 .\" attrlist elements
 .
-The fields of the 
+The fields of the
 .Vt attrlist
 structure are defined as follows.
 .Bl -tag -width XXXbitmapcount
 .
 .It bitmapcount
 Number of attribute bit sets in the structure.
-In current systems you must set this to 
+In current systems you must set this to
 .Dv ATTR_BIT_MAP_COUNT .
 .
 .It reserved
@@ -129,17 +129,17 @@ You must set this to 0.
 .
 .It commonattr
 A bit set that specifies the common attributes that you require.
-Common attributes relate to all types of file system objects. 
+Common attributes relate to all types of file system objects.
 See below for a description of these attributes.
 .
 .It volattr
 A bit set that specifies the volume attributes that you require.
 Volume attributes relate to volumes (that is, mounted file systems).
 See below for a description of these attributes.
-If you request volume attributes, 
+If you request volume attributes,
 .Fa path
 must reference the root of a volume.
-In addition, you can't request volume attributes if you also request 
+In addition, you can't request volume attributes if you also request
 file or directory attributes.
 .
 .It dirattr
@@ -152,15 +152,15 @@ See below for a description of these attributes.
 .
 .It forkattr
 A bit set that specifies the fork attributes that you require.
-Fork attributes relate to the actual data in the file, 
+Fork attributes relate to the actual data in the file,
 which can be held in multiple named contiguous ranges, or forks.
 See below for a description of these attributes.
 .
 .El
 .Pp
 .
-Unless otherwise noted in the lists below, attributes are read-only. 
-Attributes labelled as read/write can be set using 
+Unless otherwise noted in the lists below, attributes are read-only.
+Attributes labelled as read/write can be set using
 .Xr setattrlist 2 .
 .Pp
 .
@@ -168,10 +168,10 @@ Attributes labelled as read/write can be set using
 .
 The
 .Fa attrBuf
-and 
+and
 .Fa attrBufSize
-parameters specify a buffer into which the function places attribute values. 
-The format of this buffer is sufficiently complex that its description 
+parameters specify a buffer into which the function places attribute values.
+The format of this buffer is sufficiently complex that its description
 requires a separate section (see below).
 The initial contents of this buffer are ignored.
 .Pp
@@ -187,9 +187,9 @@ The following option bits are defined.
 .Bl -tag -width XXXbitmapcount
 .
 .It FSOPT_NOFOLLOW
-If this bit is set, 
-.Fn getattrlist 
-will not follow a symlink if it occurs as 
+If this bit is set,
+.Fn getattrlist
+will not follow a symlink if it occurs as
 the last component of
 .Fa path .
 .
@@ -197,37 +197,37 @@ the last component of
 .
 .Sh ATTRIBUTE BUFFER
 .
-The data returned in the buffer described by 
-.Fa attrBuf 
-and 
-.Fa attrBufSize 
+The data returned in the buffer described by
+.Fa attrBuf
+and
+.Fa attrBufSize
 is formatted as follows.
 .Pp
 .
 .Bl -enum
 .
 .It
-The first element of the buffer is a 
-.Vt unsigned long 
-that contains the overall length, in bytes, of the attributes returned. 
-This size includes the length field itself. 
+The first element of the buffer is a
+.Vt u_int32_t
+that contains the overall length, in bytes, of the attributes returned.
+This size includes the length field itself.
 .
 .It
-Following the length field is a list of attributes. 
-Each attribute is represented by a field of its type, 
-where the type is given as part of the attribute description (below). 
+Following the length field is a list of attributes.
+Each attribute is represented by a field of its type,
+where the type is given as part of the attribute description (below).
 .
 .It
-The attributes are placed into the attribute buffer in the order 
+The attributes are placed into the attribute buffer in the order
 that they are described below.
 .
 .El
 .Pp
 .
-If the attribute is of variable length, it is represented 
-in the list by an 
-.Vt attrreference 
-structure, as defined by 
+If the attribute is of variable length, it is represented
+in the list by an
+.Vt attrreference
+structure, as defined by
 .Aq Pa sys/attr.h
 (shown below).
 .
@@ -240,65 +240,65 @@ typedef struct attrreference {
 .Pp
 .
 This structure contains a 'pointer' to the variable length attribute data.
-The 
-.Fa attr_length 
-field is the length of the attribute data (in bytes). 
-The 
-.Fa attr_dataoffset 
-field is the offset in bytes from the 
-.Vt attrreference 
-structure 
-to the attribute data. 
-This offset will always be a multiple of sizeof(unsigned long) bytes, 
-so you can safely access common data types without fear of alignment 
+The
+.Fa attr_length
+field is the length of the attribute data (in bytes).
+The
+.Fa attr_dataoffset
+field is the offset in bytes from the
+.Vt attrreference
+structure
+to the attribute data.
+This offset will always be a multiple of sizeof(unsigned long) bytes,
+so you can safely access common data types without fear of alignment
 exceptions.
 .Pp
 .
-The 
-.Fn getattrlist 
-function will silently truncate attribute data if 
-.Fa attrBufSize 
-is too small. 
-The length field at the front of the attribute list always represents 
-the length of the data actually copied into the attribute buffer. 
-If the data is truncated, there is no easy way to determine the 
-buffer size that's required to get all of the requested attributes. 
-You should always pass an 
-.Fa attrBufSize 
-that is large enough to accommodate the known size of the attributes 
+The
+.Fn getattrlist
+function will silently truncate attribute data if
+.Fa attrBufSize
+is too small.
+The length field at the front of the attribute list always represents
+the length of the data actually copied into the attribute buffer.
+If the data is truncated, there is no easy way to determine the
+buffer size that's required to get all of the requested attributes.
+You should always pass an
+.Fa attrBufSize
+that is large enough to accommodate the known size of the attributes
 in the attribute list (including the leading length field).
 .Pp
 .
-Because the returned attributes are simply truncated if the buffer is 
-too small, it's possible for a variable length attribute to reference 
-data beyond the end of the attribute buffer.  That is, it's possible 
-for the attribute data to start beyond the end of the attribute buffer 
-(that is, if 
-.Fa attrRef 
-is a pointer to the 
+Because the returned attributes are simply truncated if the buffer is
+too small, it's possible for a variable length attribute to reference
+data beyond the end of the attribute buffer.  That is, it's possible
+for the attribute data to start beyond the end of the attribute buffer
+(that is, if
+.Fa attrRef
+is a pointer to the
 .Vt attrreference_t ,
-( ( (char *) 
-.Fa attrRef 
-) + 
-.Fa attr_dataoffset 
-) > ( ( (char *) 
-.Fa attrBuf 
-) + 
-.Fa attrSize 
-) ) or, indeed, for the attribute data to extend beyond the end of the attribute buffer (that is, 
-( ( (char *) 
-.Fa attrRef 
-) + 
-.Fa attr_dataoffset 
-+ 
-.Fa attr_datalength 
-) > ( ( (char *) 
-.Fa attrBuf 
-) + 
-.Fa attrSize 
+( ( (char *)
+.Fa attrRef
+) +
+.Fa attr_dataoffset
+) > ( ( (char *)
+.Fa attrBuf
+) +
+.Fa attrSize
+) ) or, indeed, for the attribute data to extend beyond the end of the attribute buffer (that is,
+( ( (char *)
+.Fa attrRef
+) +
+.Fa attr_dataoffset
++
+.Fa attr_datalength
+) > ( ( (char *)
+.Fa attrBuf
+) +
+.Fa attrSize
 ) ).
-If this happens you must increase the size of the buffer and call 
-.Fn getattrlist 
+If this happens you must increase the size of the buffer and call
+.Fn getattrlist
 to get an accurate copy of the attribute.
 .
 .Sh COMMON ATTRIBUTES
@@ -309,281 +309,291 @@ The following common attributes are defined.
 .Bl -tag -width ATTR_VOL_ALLOCATIONCLUMP
 .
 .It ATTR_CMN_NAME
-An 
-.Vt attrreference 
-structure containing the name of the file system object as 
+An
+.Vt attrreference
+structure containing the name of the file system object as
 UTF-8 encoded, null terminated C string.
-The attribute data length will not be greater than 
-.Dv NAME_MAX + 
+The attribute data length will not be greater than
+.Dv NAME_MAX +
 1.
 .Pp
 .
 .It ATTR_CMN_DEVID
-A 
-.Vt dev_t 
-containing the device number of the device on which this 
+A
+.Vt dev_t
+containing the device number of the device on which this
 file system object's volume is mounted.
-Equivalent to the 
-.Fa st_dev 
-field of the 
-.Vt stat 
-structure returned by 
+Equivalent to the
+.Fa st_dev
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_FSID
-An 
-.Vt fsid_t 
-structure containing the file system identifier for the volume on which 
+An
+.Vt fsid_t
+structure containing the file system identifier for the volume on which
 the file system object resides.
-Equivalent to the 
-.Fa f_fsid 
-field of the 
-.Vt statfs 
-structure returned by 
+Equivalent to the
+.Fa f_fsid
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
 .
 .Pp
-This value is not related to the file system ID from traditional Mac OS (for example, 
-the 
-.Fa filesystemID 
-field of the 
+This value is not related to the file system ID from traditional Mac OS (for example,
+the
+.Fa filesystemID
+field of the
 .Vt FSVolumeInfo
 structure returned by Carbon's FSGetVolumeInfo() function).
 On current versions of Mac OS X that value is synthesised by the Carbon File Manager.
 .
 .It ATTR_CMN_OBJTYPE
-An 
-.Vt fsobj_type_t 
-that identifies the type of file system object. 
-The values are taken from 
-.Vt enum vtype 
-in 
+An
+.Vt fsobj_type_t
+that identifies the type of file system object.
+The values are taken from
+.Vt enum vtype
+in
 .Aq Pa sys/vnode.h .
 .
 .It ATTR_CMN_OBJTAG
-An 
-.Vt fsobj_tag_t 
+An
+.Vt fsobj_tag_t
 that identifies the type of file system containing the object.
-The values are taken from 
-.Vt enum vtagtype 
+The values are taken from
+.Vt enum vtagtype
 in
 .Aq Pa sys/vnode.h .
 .
 .It ATTR_CMN_OBJID
-An 
-.Vt fsobj_id_t 
-structure that uniquely identifies the file system object 
+An
+.Vt fsobj_id_t
+structure that uniquely identifies the file system object
 within its volume.
 The fid_generation field of this structure will be zero for all non-root callers
 (effective UID not 0).
 This identifier need not be persistent across an unmount/mount sequence.
 .Pp
 .
-Some volume formats use well known values for the 
-.Fa fid_objno 
-field for the root directory (2) and the parent of root directory (1). 
+Some volume formats use well known values for the
+.Fa fid_objno
+field for the root directory (2) and the parent of root directory (1).
 This is not a required behaviour of this attribute.
 .
 .It ATTR_CMN_OBJPERMANENTID
-An 
-.Vt fsobj_id_t 
-structure that uniquely identifies the file system object 
+An
+.Vt fsobj_id_t
+structure that uniquely identifies the file system object
 within its volume.
 The fid_generation field of this structure will be zero for all non-root callers
 (effective UID not 0).
 This identifier should be persistent across an unmount/mount sequence.
 .Pp
-Some file systems (for example, original HFS) may need to modify the on-disk 
-structure to return a persistent identifier. 
-If such a file system is mounted read-only, an attempt to get this attribute 
-will fail with the error 
+Some file systems (for example, original HFS) may need to modify the on-disk
+structure to return a persistent identifier.
+If such a file system is mounted read-only, an attempt to get this attribute
+will fail with the error
 .Dv EROFS .
 .
 .It ATTR_CMN_PAROBJID
-An 
-.Vt fsobj_id_t 
-structure that identifies the parent directory of the file system object. 
-The fid_generation field of this structure will be zero for all non-root callers 
+An
+.Vt fsobj_id_t
+structure that identifies the parent directory of the file system object.
+The fid_generation field of this structure will be zero for all non-root callers
 (effective UID not 0).
 Equivalent to the ATTR_CMN_OBJID attribute of the parent directory.
 This identifier need not be persistent across an unmount/mount sequence.
 .Pp
 .
-On a volume that supports hard links, a multiply linked file has no unique parent. 
+On a volume that supports hard links, a multiply linked file has no unique parent.
 This attribute will return an unspecified parent.
 .Pp
 .
-For some volume formats this attribute is very expensive to calculate. 
+For some volume formats this attribute is very expensive to calculate.
 .
 .It ATTR_CMN_SCRIPT
-(read/write) A 
-.Vt text_encoding_t 
-containing a text encoding hint for 
-the file system object's name. 
-It is included to facilitate the lossless round trip conversion of names between 
+(read/write) A
+.Vt text_encoding_t
+containing a text encoding hint for
+the file system object's name.
+It is included to facilitate the lossless round trip conversion of names between
 Unicode and traditional Mac OS script encodings.
-The values are defined in 
+The values are defined in
 .Aq Pa CarbonCore/TextCommon.h .
-File systems that do not have an appropriate text encoding value should return 
-kTextEncodingMacUnicode. 
+File systems that do not have an appropriate text encoding value should return
+kTextEncodingMacUnicode.
 See DTS Q&A 1173 "File Manager Text Encoding Hints".
 .
 .It ATTR_CMN_CRTIME
-(read/write) A 
-.Vt timespec 
-structure containing the time that the file system object 
-was created. 
+(read/write) A
+.Vt timespec
+structure containing the time that the file system object
+was created.
 .
 .It ATTR_CMN_MODTIME
-(read/write) A 
-.Vt timespec 
-structure containing the time that the file system object 
-was last modified. 
-Equivalent to the 
-.Fa st_mtimespec 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt timespec
+structure containing the time that the file system object
+was last modified.
+Equivalent to the
+.Fa st_mtimespec
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_CHGTIME
-(read/write) A 
-.Vt timespec 
-structure containing the time that the file system object's 
-attributes were last modified. 
-Equivalent to the 
-.Fa st_ctimespec 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt timespec
+structure containing the time that the file system object's
+attributes were last modified.
+Equivalent to the
+.Fa st_ctimespec
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_ACCTIME
-(read/write) A 
-.Vt timespec 
-structure containing the time that the file system object 
-was last accessed. 
-Equivalent to the 
-.Fa st_atimespec 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt timespec
+structure containing the time that the file system object
+was last accessed.
+Equivalent to the
+.Fa st_atimespec
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_BKUPTIME
-(read/write) A 
-.Vt timespec 
-structure containing the time that the file system object was 
-last backed up. 
-This value is for use by backup utilities. 
+(read/write) A
+.Vt timespec
+structure containing the time that the file system object was
+last backed up.
+This value is for use by backup utilities.
 The file system stores but does not interpret the value.
 .
 .It ATTR_CMN_FNDRINFO
 (read/write) 32 bytes of data for use by the Finder.
-Equivalent to the concatenation of a 
-.Vt FileInfo 
-structure and an 
-.Vt ExtendedFileInfo 
-structure 
-(or, for directories, a 
-.Vt FolderInfo 
-structure and an 
-.Vt ExtendedFolderInfo 
+Equivalent to the concatenation of a
+.Vt FileInfo
+structure and an
+.Vt ExtendedFileInfo
+structure
+(or, for directories, a
+.Vt FolderInfo
+structure and an
+.Vt ExtendedFolderInfo
 structure).
-These structures are defined in 
+These structures are defined in
 .Aq Pa CarbonCore/Finder.h .
 .Pp
 This attribute is not byte swapped by the file system.
-The value of multibyte fields on disk is always big endian. 
-When running on a little endian system (such as Darwin on x86), 
+The value of multibyte fields on disk is always big endian.
+When running on a little endian system (such as Darwin on x86),
 you must byte swap any multibyte fields.
 .
 .It ATTR_CMN_OWNERID
-(read/write) A 
-.Vt uid_t 
-containing the owner of the file system object. 
-Equivalent to the 
-.Fa st_uid 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt uid_t
+containing the owner of the file system object.
+Equivalent to the
+.Fa st_uid
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_GRPID
-(read/write) A 
-.Vt gid_t 
-containing the group of the file system object. 
-Equivalent to the 
-.Fa st_gid 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt gid_t
+containing the group of the file system object.
+Equivalent to the
+.Fa st_gid
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_ACCESSMASK
-(read/write) A 
-.Vt mode_t 
-containing the access permissions of the file system object. 
-Equivalent to the 
-.Fa st_mode 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt u_int32_t
+containing the access permissions of the file system object.
+Equivalent to the
+.Fa st_mode
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_CMN_NAMEDATTRCOUNT
-A 
-.Vt unsigned long
-containing the number of named attributes of the file system object. 
-No built-in file systems on Mac OS X currently support named attributes.
+A
+.Vt u_int32_t
+containing the number of named attributes of the file system object.
 .
 .It ATTR_CMN_NAMEDATTRLIST
-An 
-.Vt attrreference 
+An
+.Vt attrreference
 structure containing a list of named attributes of the file system object.
 No built-in file systems on Mac OS X currently support named attributes.
 Because of this, the structure of this attribute's value is not yet defined.
 .
 .It ATTR_CMN_FLAGS
-(read/write) A 
-.Vt unsigned long 
+(read/write) A
+.Vt u_int32_t
 containing file flags.
-Equivalent to the 
-.Fa st_flags 
-field of the 
-.Vt stat 
-structure returned by 
+Equivalent to the
+.Fa st_flags
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
-For more information about these flags, see 
+For more information about these flags, see
 .Xr chflags 2 .
 .Pp
 .
-The order that attributes are placed into the attribute buffer 
-almost invariably matches the order of the attribute mask bit values. 
-The exception is 
-.Dv ATTR_CMN_FLAGS . 
-If its order was based on its bit position, it would be before 
-the 
-.Dv ATTR_CMN_NAMEDATTRCOUNT 
-/ 
-.Dv ATTR_CMN_NAMEDATTRLIST 
-pair, however, 
+The order that attributes are placed into the attribute buffer
+almost invariably matches the order of the attribute mask bit values.
+The exception is
+.Dv ATTR_CMN_FLAGS .
+If its order was based on its bit position, it would be before
+the
+.Dv ATTR_CMN_NAMEDATTRCOUNT
+/
+.Dv ATTR_CMN_NAMEDATTRLIST
+pair, however,
 it is placed in the buffer after them.
 .
 .It ATTR_CMN_USERACCESS
-A 
-.Vt unsigned long
-containing the effective permissions of the current user 
-(the calling process's effective UID) for this file system object. 
-You can test for read, write, and execute permission using 
+A
+.Vt u_int32_t
+containing the effective permissions of the current user
+(the calling process's effective UID) for this file system object.
+You can test for read, write, and execute permission using
 .Dv R_OK ,
 .Dv W_OK ,
 and
-.Dv X_OK , 
-respectively.  See 
-.Xr access 2 
+.Dv X_OK ,
+respectively.
+See
+.Xr access 2
 for more details.
 .
+.It ATTR_CMN_FILEID
+A
+.Vt u_int64_t
+that uniquely identifies the file system object within its volume. 
+.
+.It ATTR_CMN_PARENTID
+A
+.Vt u_int64_t
+that identifies the parent directory of the file system object.
+.
 .El
 .
 .Sh VOLUME ATTRIBUTES
@@ -594,187 +604,187 @@ The following volume attributes are defined.
 .Bl -tag -width ATTR_VOL_ALLOCATIONCLUMP
 .
 .It ATTR_VOL_INFO
-For reasons that are not at all obvious, you must set 
-.Dv ATTR_VOL_INFO 
-in the 
+For reasons that are not at all obvious, you must set
+.Dv ATTR_VOL_INFO
+in the
 .Fa volattr
 field if you request any other volume attributes.
 This does not result in any attribute data being added to the attribute buffer.
 .
 .It ATTR_VOL_FSTYPE
-A 
-.Vt unsigned long 
+A
+.Vt u_int32_t
 containing the file system type.
-Equivalent to the 
-.Fa f_type 
-field of the 
-.Vt statfs 
-structure returned by 
+Equivalent to the
+.Fa f_type
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
 Generally not a useful value.
 .
 .It ATTR_VOL_SIGNATURE
-A 
-.Vt unsigned long 
-containing the volume signature word. 
-This value is unique within a given file system type and lets you 
+A
+.Vt u_int32_t
+containing the volume signature word.
+This value is unique within a given file system type and lets you
 distinguish between different volume formats handled by the same file system.
-See 
-.Aq Pa CarbonCore/Files.h 
+See
+.Aq Pa CarbonCore/Files.h
 for more details.
 .
 .It ATTR_VOL_SIZE
-An 
+An
 .Vt off_t
 containing the total size of the volume in bytes.
 .
 .It ATTR_VOL_SPACEFREE
-An 
+An
 .Vt off_t
 containing the free space on the volume in bytes.
 .
 .It ATTR_VOL_SPACEAVAIL
-An 
+An
 .Vt off_t
-containing the space, in bytes, on the volume available to non-privileged processes. 
-This is the free space minus the amount of space reserved by the system to prevent critical 
-disk exhaustion errors. 
-Non-privileged programs, like a disk management tool, should use this value to display the 
+containing the space, in bytes, on the volume available to non-privileged processes.
+This is the free space minus the amount of space reserved by the system to prevent critical
+disk exhaustion errors.
+Non-privileged programs, like a disk management tool, should use this value to display the
 space available to the user.
 .Pp
-.Dv ATTR_VOL_SPACEAVAIL 
-is to 
-.Dv ATTR_VOL_SPACEFREE 
-as 
-.Fa f_bavail 
-is to 
-.Fa f_bfree 
-in 
+.Dv ATTR_VOL_SPACEAVAIL
+is to
+.Dv ATTR_VOL_SPACEFREE
+as
+.Fa f_bavail
+is to
+.Fa f_bfree
+in
 .Xr statfs 2 .
 .
 .It ATTR_VOL_MINALLOCATION
-An 
+An
 .Vt off_t
-containing the minimum allocation size on the volume in bytes. 
+containing the minimum allocation size on the volume in bytes.
 If you create a file containing one byte, it will consume this much space.
 .
 .It ATTR_VOL_ALLOCATIONCLUMP
-An 
+An
 .Vt off_t
-containing the allocation clump size on the volume, in bytes. 
-As a file is extended, the file system will attempt to allocate 
+containing the allocation clump size on the volume, in bytes.
+As a file is extended, the file system will attempt to allocate
 this much space each time in order to reduce fragmentation.
 .
 .It ATTR_VOL_IOBLOCKSIZE
-A 
-.Vt unsigned long 
-containing the optimal block size when reading or writing data. 
-Equivalent to the 
-.Fa f_iosize 
-field of the 
-.Vt statfs 
-structure returned by 
+A
+.Vt u_int32_t
+containing the optimal block size when reading or writing data.
+Equivalent to the
+.Fa f_iosize
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
 .
 .It ATTR_VOL_OBJCOUNT
-A 
-.Vt unsigned long 
+A
+.Vt u_int32_t
 containing the number of file system objects on the volume.
 .
 .It ATTR_VOL_FILECOUNT
-A 
-.Vt unsigned long 
+A
+.Vt u_int32_t
 containing the number of files on the volume.
 .
 .It ATTR_VOL_DIRCOUNT
-A 
-.Vt unsigned long 
+A
+.Vt u_int32_t
 containing the number of directories on the volume.
 .
 .It ATTR_VOL_MAXOBJCOUNT
-A 
-.Vt unsigned long 
+A
+.Vt u_int32_t
 containing the maximum number of file system objects that can be stored on the volume.
 .
 .It ATTR_VOL_MOUNTPOINT
-An 
+An
 .Vt attrreference
-structure containing the path to the volume's mount point as a 
+structure containing the path to the volume's mount point as a
 UTF-8 encoded, null terminated C string.
-The attribute data length will not be greater than 
+The attribute data length will not be greater than
 .Dv MAXPATHLEN .
-Equivalent to the 
-.Fa f_mntonname 
-field of the 
-.Vt statfs 
-structure returned by 
+Equivalent to the
+.Fa f_mntonname
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
 .
 .It ATTR_VOL_NAME
-(read/write) An 
+(read/write) An
 .Vt attrreference
-structure containing the name of the volume as a 
+structure containing the name of the volume as a
 UTF-8 encoded, null terminated C string.
-The attribute data length will not be greater than 
+The attribute data length will not be greater than
 .Dv NAME_MAX +
 1.
 .Pp
 .
-This attribute is only read/write if the 
+This attribute is only read/write if the
 .Dv VOL_CAP_INT_VOL_RENAME
 bit is set in the volume capabilities (see below).
 .Pp
 .
 .It ATTR_VOL_MOUNTFLAGS
-A 
-.Vt unsigned long 
-containing the volume mount flags. 
-This is a copy of the value passed to the 
+A
+.Vt u_int32_t
+containing the volume mount flags.
+This is a copy of the value passed to the
 .Fa flags
-parameter of 
+parameter of
 .Xr mount 2
 when the volume was mounted.
-Equivalent to the 
-.Fa f_flags 
-field of the 
-.Vt statfs 
-structure returned by 
+Equivalent to the
+.Fa f_flags
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
 .
 .It ATTR_VOL_MOUNTEDDEVICE
-An 
+An
 .Vt attrreference
-structure that returns the same value as the 
-.Fa f_mntfromname 
-field of the 
-.Vt statfs 
-structure returned by 
+structure that returns the same value as the
+.Fa f_mntfromname
+field of the
+.Vt statfs
+structure returned by
 .Xr statfs 2 .
-For local volumes this is the path to the device on which the volume is mounted as a 
+For local volumes this is the path to the device on which the volume is mounted as a
 UTF-8 encoded, null terminated C string.
 For network volumes, this is a unique string that identifies the mount.
-The attribute data length will not be greater than 
+The attribute data length will not be greater than
 .Dv MAXPATHLEN .
 .Pp
 .
 .It ATTR_VOL_ENCODINGSUSED
-An 
+An
 .Vt unsigned long long
-containing a bitmap of the text encodings used on this volume. 
-For more information about this, see the discussion of 
-.Fa encodingsBitmap 
+containing a bitmap of the text encodings used on this volume.
+For more information about this, see the discussion of
+.Fa encodingsBitmap
 in DTS Technote 1150 "HFS Plus Volume Format".
 .
 .It ATTR_VOL_CAPABILITIES
 A
 .Vt vol_capabilities_attr_t
-structure describing the optional features supported by this volume. 
+structure describing the optional features supported by this volume.
 See below for a discussion of volume capabilities.
 .
 .It ATTR_VOL_ATTRIBUTES
 A
 .Vt vol_attributes_attr_t
-structure describing the attributes supported by this volume. 
+structure describing the attributes supported by this volume.
 This structure is discussed below, along with volume capabilities.
 .
 .El
@@ -786,35 +796,25 @@ The following directory attributes are defined.
 .Bl -tag -width ATTR_VOL_ALLOCATIONCLUMP
 .
 .It ATTR_DIR_LINKCOUNT
-A 
-.Vt unsigned long 
-containing the number of file system objects in the directory, including 
-synthetic items such as "." and "..". 
-For historical reasons, you should not always rely on this value being accurate.
-.Pp
-If you're implementing a volume format on which this is hard to calculate, 
-you should not support this attribute. 
-While it's traditional to return a constant value of 1 in the 
-.Fa st_nlink 
-field of the 
-.Vt stat
-structure as returned by 
-.Xr stat 2 ,
-it's not necessary to do this here because there is a 
-defined way to indicate that you do not support the attribute.
+A
+.Vt u_int32_t
+containing the number of hard links to the directory;
+this does not include the historical "." and ".." entries.
+For filesystems that do not support hard links to directories,
+this value will be 1.
 .
 .It ATTR_DIR_ENTRYCOUNT
-A 
-.Vt unsigned long 
-containing the number of file system objects in the directory, not including 
+A
+.Vt u_int32_t
+containing the number of file system objects in the directory, not including
 any synthetic items.
 .
 .It ATTR_DIR_MOUNTSTATUS
-A 
-.Vt unsigned long 
-containing flags describing what's mounted on the directory. 
-Currently the only flag defined is 
-.Dv DIR_MNTSTATUS_MNTPOINT, 
+A
+.Vt u_int32_t
+containing flags describing what's mounted on the directory.
+Currently the only flag defined is
+.Dv DIR_MNTSTATUS_MNTPOINT,
 which indicates that there is a file system mounted on this directory.
 Due to a bug (r. 3502822), this flag is never set on current system.
 .
@@ -827,69 +827,69 @@ The following file attributes are defined.
 .Bl -tag -width ATTR_VOL_ALLOCATIONCLUMP
 .
 .It ATTR_FILE_LINKCOUNT
-A 
-.Vt unsigned long
-containing the number of hard links to this file. 
-Equivalent to the 
-.Fa st_nlink 
-field of the 
-.Vt stat 
-structure returned by 
+A
+.Vt u_int32_t
+containing the number of hard links to this file.
+Equivalent to the
+.Fa st_nlink
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_FILE_TOTALSIZE
-An 
+An
 .Vt off_t
 containing the total number of bytes in all forks of the file (the logical size).
 .
 .It ATTR_FILE_ALLOCSIZE
-An 
+An
 .Vt off_t
 containing a count of the bytes on disk used by all of the file's forks (the physical size).
 .
 .It ATTR_FILE_IOBLOCKSIZE
-A 
-.Vt unsigned long
-containing the optimal block size when reading or writing this file's data. 
+A
+.Vt u_int32_t
+containing the optimal block size when reading or writing this file's data.
 .
 .It ATTR_FILE_CLUMPSIZE
-A 
-.Vt unsigned long
-containing the allocation clump size for this file, in bytes. 
-As the file is extended, the file system will attempt to allocate 
-this much space each time in order to reduce fragmentation. 
+A
+.Vt u_int32_t
+containing the allocation clump size for this file, in bytes.
+As the file is extended, the file system will attempt to allocate
+this much space each time in order to reduce fragmentation.
 This value applies to the data fork.
 .
 .It ATTR_FILE_DEVTYPE
-(read/write) A 
-.Vt unsigned long
-containing the device type for a special device file. 
-Equivalent to the 
-.Fa st_rdev 
-field of the 
-.Vt stat 
-structure returned by 
+(read/write) A
+.Vt u_int32_t
+containing the device type for a special device file.
+Equivalent to the
+.Fa st_rdev
+field of the
+.Vt stat
+structure returned by
 .Xr stat 2 .
 .
 .It ATTR_FILE_FILETYPE
 A
-.Vt unsigned long
+.Vt u_int32_t
 that whose value is reserved.
 Clients should ignore its value.
 New volume format implementations should not support this attribute.
 .
 .It ATTR_FILE_FORKCOUNT
 A
-.Vt unsigned long
+.Vt u_int32_t
 containing the number of forks in the file.
-No built-in file systems on Mac OS X currently support forks other 
+No built-in file systems on Mac OS X currently support forks other
 than the data and resource fork.
 .
 .It ATTR_FILE_FORKLIST
-An 
+An
 .Vt attrreference
 structure containing a list of named forks of the file.
-No built-in file systems on Mac OS X currently support forks 
+No built-in file systems on Mac OS X currently support forks
 other than the data and resource fork.
 Because of this, the structure of this attribute's value is not yet defined.
 .
@@ -906,21 +906,21 @@ containing a count of the bytes on disk used by the data fork (the physical size
 .It ATTR_FILE_DATAEXTENTS
 An
 .Vt extentrecord
-array for the data fork. 
-The array contains eight 
-.Vt diskextent 
-structures which represent the first 
+array for the data fork.
+The array contains eight
+.Vt diskextent
+structures which represent the first
 eight extents of the fork.
 .Pp
-This attributes exists for compatibility reasons. 
+This attributes exists for compatibility reasons.
 New clients should not use this attribute.
-Rather, they should use the 
-.Dv F_LOG2PHYS 
-command in 
+Rather, they should use the
+.Dv F_LOG2PHYS
+command in
 .Xr fcntl 2 .
 .Pp
 .
-In current implementations the value may not be entirely accurate for 
+In current implementations the value may not be entirely accurate for
 a variety of reasons.
 .
 .It ATTR_FILE_RSRCLENGTH
@@ -936,20 +936,20 @@ containing a count of the bytes on disk used by the resource fork (the physical
 .It ATTR_FILE_RSRCEXTENTS
 An
 .Vt extentrecord
-array for the resource fork. 
-The array contains eight 
-.Vt diskextent 
-structures which represent the first 
+array for the resource fork.
+The array contains eight
+.Vt diskextent
+structures which represent the first
 eight extents of the fork.
 .Pp
-See also 
+See also
 .Dv ATTR_FILE_DATAEXTENTS .
 .
 .El
 .
 .Sh FORK ATTRIBUTES
 .
-Fork attributes relate to the actual data in the file, 
+Fork attributes relate to the actual data in the file,
 which can be held in multiple named contiguous ranges, or forks.
 The following fork attributes are defined.
 .
@@ -968,19 +968,20 @@ containing a count of the bytes on disk used by the fork (the physical size).
 .El
 .Pp
 .
-Fork attributes are not properly implemented by any current Mac OS X 
-volume format implementation. 
-We strongly recommend that client programs do not request fork attributes. 
-If you are implementing a volume format, you should not support these attributes. 
+Fork attributes are not properly implemented by any current Mac OS X
+volume format implementation.
+We strongly recommend that client programs do not request fork attributes.
+If you are implementing a volume format, you should not support these attributes.
 .
 .Sh VOLUME CAPABILITIES
 .
 .\" vol_capabilities_attr_t
 .
-Not all volumes support all features.  The 
-.Dv ATTR_VOL_CAPABILITIES 
-attribute returns a 
-.Vt vol_capabilities_attr_t 
+Not all volumes support all features.
+The
+.Dv ATTR_VOL_CAPABILITIES
+attribute returns a
+.Vt vol_capabilities_attr_t
 structure (shown below) that indicates which features are supported by the volume.
 .
 .Bd -literal
@@ -1000,101 +1001,101 @@ typedef struct vol_capabilities_attr {
 .Ed
 .Pp
 .
-The structure contains two fields, 
-.Fa capabilities 
-and 
+The structure contains two fields,
+.Fa capabilities
+and
 .Fa valid .
-Each consists of an array of four elements. 
+Each consists of an array of four elements.
 The arrays are indexed by the following values.
 .
 .Bl -tag -width VOL_CAP_FMT_PERSISTENTOBJECTIDS
 .
 .It VOL_CAPABILITIES_FORMAT
-This element contains information about the volume format. 
-See 
-.Dv VOL_CAP_FMT_PERSISTENTOBJECTIDS 
+This element contains information about the volume format.
+See
+.Dv VOL_CAP_FMT_PERSISTENTOBJECTIDS
 and so on, below.
 .
 .It VOL_CAPABILITIES_INTERFACES
-This element contains information about which optional functions are 
+This element contains information about which optional functions are
 supported by the volume format implementation.
-See 
-.Dv VOL_CAP_INT_SEARCHFS 
+See
+.Dv VOL_CAP_INT_SEARCHFS
 and so on, below.
 .
 .It VOL_CAPABILITIES_RESERVED1
-Reserved. 
-A file system implementation should set this element to zero. 
+Reserved.
+A file system implementation should set this element to zero.
 A client program should ignore this element.
 .
 .It VOL_CAPABILITIES_RESERVED2
-Reserved. 
-A file system implementation should set this element to zero. 
+Reserved.
+A file system implementation should set this element to zero.
 A client program should ignore this element.
 .
 .El
 .Pp
 .
-The 
-.Fa valid 
-field contains bit sets that indicate which flags are known to the volume format 
-implementation. 
-Each bit indicates whether the contents of the corresponding bit in the 
-.Fa capabilities 
+The
+.Fa valid
+field contains bit sets that indicate which flags are known to the volume format
+implementation.
+Each bit indicates whether the contents of the corresponding bit in the
+.Fa capabilities
 field is valid.
 .Pp
 .
-The 
-.Fa capabilities 
-field contains bit sets that indicate whether a particular feature is implemented 
-by this volume format. 
+The
+.Fa capabilities
+field contains bit sets that indicate whether a particular feature is implemented
+by this volume format.
 .Pp
 .
-The following bits are defined in the first element (indexed by 
-.Dv VOL_CAPABILITIES_FORMAT ) 
-of the 
-.Fa capabilities 
-and 
-.Fa valid 
-fields of the 
-.Vt vol_capabilities_attr_t 
+The following bits are defined in the first element (indexed by
+.Dv VOL_CAPABILITIES_FORMAT )
+of the
+.Fa capabilities
+and
+.Fa valid
+fields of the
+.Vt vol_capabilities_attr_t
 structure.
 .
 .Bl -tag -width VOL_CAP_FMT_PERSISTENTOBJECTIDS
 .
 .It VOL_CAP_FMT_PERSISTENTOBJECTIDS
-If this bit is set the volume format supports persistent object identifiers 
+If this bit is set the volume format supports persistent object identifiers
 and can look up file system objects by their IDs.
-See 
-.Dv ATTR_CMN_OBJPERMANENTID 
+See
+.Dv ATTR_CMN_OBJPERMANENTID
 for details about how to obtain these identifiers.
 .
 .It VOL_CAP_FMT_SYMBOLICLINKS
-If this bit is set the volume format supports symbolic links. 
+If this bit is set the volume format supports symbolic links.
 .
 .It VOL_CAP_FMT_HARDLINKS
-If this bit is set the volume format supports hard links. 
+If this bit is set the volume format supports hard links.
 .
 .It VOL_CAP_FMT_JOURNAL
-If this bit is set the volume format supports a journal used to 
-speed recovery in case of unplanned restart (such as a power outage 
+If this bit is set the volume format supports a journal used to
+speed recovery in case of unplanned restart (such as a power outage
 or crash).
 This does not necessarily mean the volume is actively using a journal.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_JOURNAL_ACTIVE
-If this bit is set the volume is currently using a journal for 
+If this bit is set the volume is currently using a journal for
 speedy recovery after an unplanned restart.
-This bit can be set only if 
+This bit can be set only if
 .Dv VOL_CAP_FMT_JOURNAL
 is also set.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_NO_ROOT_TIMES
-If this bit is set the volume format does not store reliable times for 
-the root directory, so you should not depend on them to detect changes, 
+If this bit is set the volume format does not store reliable times for
+the root directory, so you should not depend on them to detect changes,
 identify volumes across unmount/mount, and so on.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
@@ -1102,9 +1103,9 @@ Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .It VOL_CAP_FMT_SPARSE_FILES
 If this bit is set the volume format supports sparse files,
 that is, files which can have 'holes' that have never been written
-to, and thus do not consume space on disk. 
+to, and thus do not consume space on disk.
 A sparse file may have an allocated size on disk that is less than its logical length (that is,
-.Dv ATTR_FILE_ALLOCSIZE 
+.Dv ATTR_FILE_ALLOCSIZE
 <
 .Dv ATTR_FILE_TOTALSIZE ).
 .
@@ -1112,93 +1113,93 @@ A sparse file may have an allocated size on disk that is less than its logical l
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_ZERO_RUNS
-For security reasons, parts of a file (runs) that have never been 
-written to must appear to contain zeroes. 
+For security reasons, parts of a file (runs) that have never been
+written to must appear to contain zeroes.
 When this bit is set, the volume keeps track of allocated but unwritten
 runs of a file so that it can substitute zeroes without actually
-writing zeroes to the media. 
+writing zeroes to the media.
 This provides performance similar to sparse files, but not the space savings.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_CASE_SENSITIVE
-If this bit is set the volume format treats upper and lower case 
-characters in file and directory names as different. 
-Otherwise an upper case character is equivalent to a lower case character, 
+If this bit is set the volume format treats upper and lower case
+characters in file and directory names as different.
+Otherwise an upper case character is equivalent to a lower case character,
 and you can't have two names that differ solely in the case of
 the characters.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_CASE_PRESERVING
-If this bit is set the volume format preserves the case of 
-file and directory names. 
-Otherwise the volume may change the case of some characters 
+If this bit is set the volume format preserves the case of
+file and directory names.
+Otherwise the volume may change the case of some characters
 (typically making them all upper or all lower case).
-A volume that sets 
+A volume that sets
 .Dv VOL_CAP_FMT_CASE_SENSITIVE
-must also set 
+must also set
 .Dv VOL_CAP_FMT_CASE_PRESERVING .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_FAST_STATFS
-This bit is used as a hint to upper layers (specifically the Carbon File Manager) to 
-indicate that 
+This bit is used as a hint to upper layers (specifically the Carbon File Manager) to
+indicate that
+.Xr statfs 2
+is fast enough that its results need not be cached by the caller.
+A volume format implementation that caches the
 .Xr statfs 2
-is fast enough that its results need not be cached by the caller. 
-A volume format implementation that caches the 
-.Xr statfs 2 
 information in memory should set this bit.
-An implementation that must always read from disk or always perform a network 
-transaction to satisfy 
+An implementation that must always read from disk or always perform a network
+transaction to satisfy
 .Xr statfs 2
 should not set this bit.
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_FMT_2TB_FILESIZE
-If this bit is set the volume format supports file 
-sizes upto 2TB.  This bit does not necessarily mean that the file 
-system does not support file size more than 2TB.   This bit does 
-not mean that the currently available space on the volume is 2TB. 
+If this bit is set the volume format supports file
+sizes upto 2TB.  This bit does not necessarily mean that the file
+system does not support file size more than 2TB.
+This bit does not mean that the currently available space on the volume is 2TB.
 .Pp
 Introduced with Darwin 8.0 (Mac OS X version 10.4).
 .
 .El
 .Pp
 .
-The following bits are defined in the second element (indexed by 
-.Dv VOL_CAPABILITIES_INTERFACES ) 
-of the 
-.Fa capabilities 
-and 
-.Fa valid 
-fields of the 
-.Vt vol_capabilities_attr_t 
+The following bits are defined in the second element (indexed by
+.Dv VOL_CAPABILITIES_INTERFACES )
+of the
+.Fa capabilities
+and
+.Fa valid
+fields of the
+.Vt vol_capabilities_attr_t
 structure.
 .
 .Bl -tag -width VOL_CAP_FMT_PERSISTENTOBJECTIDS
 .
 .It VOL_CAP_INT_SEARCHFS
-If this bit is set the volume format implementation supports 
+If this bit is set the volume format implementation supports
 .Xr searchfs 2 .
 .
 .It VOL_CAP_INT_ATTRLIST
-If this bit is set the volume format implementation supports 
-.Fn getattrlist 
-and 
+If this bit is set the volume format implementation supports
+.Fn getattrlist
+and
 .Xr setattrlist 2 .
 .
 .It VOL_CAP_INT_NFSEXPORT
-If this bit is set the volume format implementation allows this volume to be exported via NFS. 
+If this bit is set the volume format implementation allows this volume to be exported via NFS.
 .
 .It VOL_CAP_INT_READDIRATTR
-If this bit is set the volume format implementation supports 
+If this bit is set the volume format implementation supports
 .Xr getdirentriesattr 2 .
 .
 .It VOL_CAP_INT_EXCHANGEDATA
-If this bit is set the volume format implementation supports 
+If this bit is set the volume format implementation supports
 .Xr exchangedata 2 .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
@@ -1210,56 +1211,68 @@ copyfile() function.
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_INT_ALLOCATE
-If this bit is set the volume format implementation supports the 
-.Dv F_PREALLOCATE 
-selector of 
+If this bit is set the volume format implementation supports the
+.Dv F_PREALLOCATE
+selector of
 .Xr fcntl 2 .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_INT_VOL_RENAME
-If this bit is set the volume format implementation allows you to 
-modify the volume name using 
+If this bit is set the volume format implementation allows you to
+modify the volume name using
 .Xr setattrlist 2 .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_INT_ADVLOCK
-If this bit is set the volume format implementation supports 
-advisory locking, that is, the 
+If this bit is set the volume format implementation supports
+advisory locking, that is, the
 .Dv F_GETLK ,
-.Dv F_SETLK , 
-and 
-.Dv F_SETLKW 
-selectors to 
+.Dv F_SETLK ,
+and
+.Dv F_SETLKW
+selectors to
 .Xr fcntl 2 .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
 .It VOL_CAP_INT_FLOCK
-If this bit is set the volume format implementation supports 
-whole file locks. 
-This includes 
-.Xr flock 2 
-and the 
-.Dv O_EXLOCK 
-and 
-.Dv O_SHLOCK 
-flags to 
+If this bit is set the volume format implementation supports
+whole file locks.
+This includes
+.Xr flock 2
+and the
+.Dv O_EXLOCK
+and
+.Dv O_SHLOCK
+flags to
 .Xr open 2 .
 .Pp
 Introduced with Darwin 7.0 (Mac OS X version 10.3).
 .
+.It VOL_CAP_INT_EXTENDED_SECURITY
+If this bit is set the volume format implementation supports
+extended security controls (ACLs).
+.Pp
+Introduced with Darwin 8.0 (Mac OS X version 10.4).
+.
+.It VOL_CAP_INT_USERACCESS
+If this bit is set the volume format implementation supports the
+ATTR_CMN_USERACCESS attribute.
+.Pp
+Introduced with Darwin 8.0 (Mac OS X version 10.4).
+.
 .El
 .Pp
 .
 .\" vol_attributes_attr_t
 .
-A volume can also report which attributes it supports. 
+A volume can also report which attributes it supports.
 This information is returned by the
-.Dv ATTR_VOL_ATTRIBUTES 
-attribute, which returns a 
-.Vt vol_attributes_attr_t 
+.Dv ATTR_VOL_ATTRIBUTES
+attribute, which returns a
+.Vt vol_attributes_attr_t
 structure (shown below).
 .
 .Bd -literal
@@ -1279,29 +1292,29 @@ typedef struct vol_attributes_attr {
 .Ed
 .Pp
 .
-The 
-.Fa validattr 
-field consists of a number of bit sets that indicate whether an attribute is 
-supported by the volume format implementation. 
-The 
-.Fa nativeattr 
-is similar except that the bit sets indicate whether an attribute is supported 
-natively by the volume format. 
-An attribute is supported natively if the volume format implementation does not have to do 
-any complex conversions to access the attribute. 
-For example, a volume format might support persistent object identifiers, but 
-doing so requires a complex table lookup that is not part of the core volume 
-format. 
-In that case, the 
-.Dv ATTR_VOL_ATTRIBUTES 
-attribute would return 
-.Dv ATTR_CMN_OBJPERMANENTID 
-set in the 
-.Fa validattr 
-field of the 
+The
+.Fa validattr
+field consists of a number of bit sets that indicate whether an attribute is
+supported by the volume format implementation.
+The
+.Fa nativeattr
+is similar except that the bit sets indicate whether an attribute is supported
+natively by the volume format.
+An attribute is supported natively if the volume format implementation does not have to do
+any complex conversions to access the attribute.
+For example, a volume format might support persistent object identifiers, but
+doing so requires a complex table lookup that is not part of the core volume
+format.
+In that case, the
+.Dv ATTR_VOL_ATTRIBUTES
+attribute would return
+.Dv ATTR_CMN_OBJPERMANENTID
+set in the
+.Fa validattr
+field of the
 .Vt vol_attributes_attr_t ,
-but not in the 
-.Fa nativeattr 
+but not in the
+.Fa nativeattr
 field.
 .
 .Sh RETURN VALUES
@@ -1311,27 +1324,27 @@ Otherwise, a value of -1 is returned and
 is set to indicate the error.
 .
 .Sh COMPATIBILITY
-Not all volumes support 
+Not all volumes support
 .Fn getattrlist .
-The best way to test whether a volume supports this function is to 
+The best way to test whether a volume supports this function is to
 simply call it and check the error result.
-.Fn getattrlist 
-will return  
+.Fn getattrlist
+will return
 .Dv ENOTSUP
 if it is not supported on a particular volume.
 .Pp
 .
-The 
-.Fn getattrlist 
-function has been undocumented for more than two years. 
-In that time a number of volume format implementations have been created without 
-a proper specification for the behaviour of this routine. 
-You may encounter volume format implementations with slightly different 
-behaviour than what is described here. 
+The
+.Fn getattrlist
+function has been undocumented for more than two years.
+In that time a number of volume format implementations have been created without
+a proper specification for the behaviour of this routine.
+You may encounter volume format implementations with slightly different
+behaviour than what is described here.
 Your program is expected to be tolerant of this variant behaviour.
 .Pp
 .
-If you're implementing a volume format that supports 
+If you're implementing a volume format that supports
 .Fn getattrlist ,
 you should be careful to support the behaviour specified by this document.
 .
@@ -1348,9 +1361,9 @@ The volume does not support
 A component of the path prefix is not a directory.
 .
 .It Bq Er ENAMETOOLONG
-A component of a path name exceeded 
+A component of a path name exceeded
 .Dv NAME_MAX
-characters, or an entire path name exceeded 
+characters, or an entire path name exceeded
 .Dv PATH_MAX
 characters.
 .
@@ -1371,11 +1384,11 @@ or
 points to an invalid address.
 .
 .It Bq Er EINVAL
-The 
-.Fa bitmapcount 
-field of 
-.Fa attrList 
-is not 
+The
+.Fa bitmapcount
+field of
+.Fa attrList
+is not
 .Dv ATTR_BIT_MAP_COUNT .
 .
 .It Bq Er EINVAL
@@ -1388,8 +1401,8 @@ You requested an attribute that is not supported for this file system object.
 You requested volume attributes and directory or file attributes.
 .
 .It Bq Er EINVAL
-You requested volume attributes but 
-.Fa path 
+You requested volume attributes but
+.Fa path
 does not reference the root of the volume.
 .
 .It Bq Er EROFS
@@ -1402,30 +1415,30 @@ An I/O error occurred while reading from or writing to the file system.
 .
 .Sh CAVEATS
 .
-If you request any volume attributes, you must set 
-.Dv ATTR_VOL_INFO 
-in the 
+If you request any volume attributes, you must set
+.Dv ATTR_VOL_INFO
+in the
 .Fa volattr
 field, even though it generates no result in the attribute buffer.
 .Pp
 .
-The order that attributes are stored in the attribute buffer almost 
+The order that attributes are stored in the attribute buffer almost
 invariably matches the order of attribute mask bit values.
-For example, 
-.Dv ATTR_CMN_NAME 
-(0x00000001) comes before 
-.Dv ATTR_CMN_DEVID 
+For example,
+.Dv ATTR_CMN_NAME
+(0x00000001) comes before
+.Dv ATTR_CMN_DEVID
 (0x00000002) because its value is smaller.
 However, you can not rely on this ordering because there is one key exception:
-.Dv ATTR_CMN_FLAGS 
-is placed after the 
+.Dv ATTR_CMN_FLAGS
+is placed after the
 .Dv ATTR_CMN_NAMEDATTRCOUNT
 /
-.Dv ATTR_CMN_NAMEDATTRLIST 
-pair, even though its bit position indicates that it should come before. 
-This is due to a bug in an early version of Mac OS X that can't be fixed for 
-binary compatibility reasons. 
-When ordering attributes, you should always use the order in which they 
+.Dv ATTR_CMN_NAMEDATTRLIST
+pair, even though its bit position indicates that it should come before.
+This is due to a bug in an early version of Mac OS X that can't be fixed for
+binary compatibility reasons.
+When ordering attributes, you should always use the order in which they
 are described above.
 .Pp
 .
@@ -1433,7 +1446,7 @@ For more caveats, see also the compatibility notes above.
 .
 .Sh EXAMPLES
 .
-The following code prints the file type and creator of a file, 
+The following code prints the file type and creator of a file,
 assuming that the volume supports the required attributes.
 .
 .Bd -literal
@@ -1450,7 +1463,7 @@ typedef struct attrlist attrlist_t;
 .Pp
 .
 struct FInfoAttrBuf {
-    unsigned long   length;
+    u_int32_t       length;
     fsobj_type_t    objType;
     char            finderInfo[32];
 };
@@ -1468,17 +1481,17 @@ static int FInfoDemo(const char *path)
     attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
     attrList.commonattr  = ATTR_CMN_OBJTYPE | ATTR_CMN_FNDRINFO;
 .Pp
-    
+
     err = getattrlist(path, &attrList, &attrBuf, sizeof(attrBuf), 0);
     if (err != 0) {
         err = errno;
     }
 .Pp
-    
+
     if (err == 0) {
         assert(attrBuf.length == sizeof(attrBuf));
 .Pp
-        
+
         printf("Finder information for %s:\en", path);
         switch (attrBuf.objType) {
             case VREG:
@@ -1500,7 +1513,7 @@ static int FInfoDemo(const char *path)
 .Ed
 .Pp
 .
-The following code is an alternative implementation that uses nested structures 
+The following code is an alternative implementation that uses nested structures
 to group the related attributes.
 .
 .Bd -literal
@@ -1525,7 +1538,7 @@ typedef struct FInfo2CommonAttrBuf FInfo2CommonAttrBuf;
 .Pp
 .
 struct FInfo2AttrBuf {
-    unsigned long       length;
+    u_int32_t           length;
     FInfo2CommonAttrBuf common;
 };
 typedef struct FInfo2AttrBuf FInfo2AttrBuf;
@@ -1557,11 +1570,11 @@ static int FInfo2Demo(const char *path)
         switch (attrBuf.common.objType) {
             case VREG:
                 printf(
-                    "file type    = '%.4s'\en", 
+                    "file type    = '%.4s'\en",
                     &attrBuf.common.finderInfo[0]
                 );
                 printf(
-                    "file creator = '%.4s'\en", 
+                    "file creator = '%.4s'\en",
                     &attrBuf.common.finderInfo[4]
                 );
                 break;
@@ -1570,7 +1583,7 @@ static int FInfo2Demo(const char *path)
                 break;
             default:
                 printf(
-                    "other object type, %d\en", 
+                    "other object type, %d\en",
                     attrBuf.common.objType
                 );
                 break;
@@ -1583,9 +1596,9 @@ static int FInfo2Demo(const char *path)
 .Ed
 .Pp
 .
-The following example shows how to deal with variable length attributes. 
-It assumes that the volume specified by 
-.Fa path 
+The following example shows how to deal with variable length attributes.
+It assumes that the volume specified by
+.Fa path
 supports the necessary attributes.
 .
 .Bd -literal
@@ -1603,9 +1616,9 @@ typedef struct attrlist attrlist_t;
 .Pp
 .
 struct VolAttrBuf {
-    unsigned long   length;
-    unsigned long   fileCount;
-    unsigned long   dirCount;
+    u_int32_t       length;
+    u_int32_t       fileCount;
+    u_int32_t       dirCount;
     attrreference_t mountPointRef;
     attrreference_t volNameRef;
     char            mountPointSpace[MAXPATHLEN];
@@ -1623,37 +1636,37 @@ static int VolDemo(const char *path)
 .
     memset(&attrList, 0, sizeof(attrList));
     attrList.bitmapcount = ATTR_BIT_MAP_COUNT;
-    attrList.volattr     =   ATTR_VOL_INFO 
-                           | ATTR_VOL_FILECOUNT 
-                           | ATTR_VOL_DIRCOUNT 
-                           | ATTR_VOL_MOUNTPOINT 
+    attrList.volattr     =   ATTR_VOL_INFO
+                           | ATTR_VOL_FILECOUNT
+                           | ATTR_VOL_DIRCOUNT
+                           | ATTR_VOL_MOUNTPOINT
                            | ATTR_VOL_NAME;
 .Pp
-    
+
     err = getattrlist(path, &attrList, &attrBuf, sizeof(attrBuf), 0);
     if (err != 0) {
         err = errno;
     }
 .Pp
-    
+
     if (err == 0) {
         assert(attrBuf.length >  offsetof(VolAttrBuf, mountPointSpace));
         assert(attrBuf.length <= sizeof(attrBuf));
 .Pp
-        
+
         printf("Volume information for %s:\en", path);
         printf("ATTR_VOL_FILECOUNT:  %lu\en", attrBuf.fileCount);
         printf("ATTR_VOL_DIRCOUNT:   %lu\en", attrBuf.dirCount);
         printf(
-            "ATTR_VOL_MOUNTPOINT: %.*s\en", 
+            "ATTR_VOL_MOUNTPOINT: %.*s\en",
             (int) attrBuf.mountPointRef.attr_length,
-            ( ((char *) &attrBuf.mountPointRef) 
+            ( ((char *) &attrBuf.mountPointRef)
               + attrBuf.mountPointRef.attr_dataoffset )
         );
         printf(
-            "ATTR_VOL_NAME:       %.*s\en", 
+            "ATTR_VOL_NAME:       %.*s\en",
             (int) attrBuf.volNameRef.attr_length,
-            ( ((char *) &attrBuf.volNameRef) 
+            ( ((char *) &attrBuf.volNameRef)
               + attrBuf.volNameRef.attr_dataoffset )
         );
     }
@@ -1678,7 +1691,7 @@ static int VolDemo(const char *path)
 .Xr statfs 2
 .
 .Sh HISTORY
-A 
+A
 .Fn getattrlist
 function call appeared in Darwin 1.3.1 (Mac OS X version 10.0).
 .
diff --git a/bsd/man/man2/getaudit.2 b/bsd/man/man2/getaudit.2
new file mode 100644
index 000000000..4b40f6d32
--- /dev/null
+++ b/bsd/man/man2/getaudit.2
@@ -0,0 +1,61 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt GETAUDIT 2
+.Os Darwin
+.Sh NAME
+.Nm getaudit
+.Nd get the audit information for the current process  
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn getaudit "struct auditinfo * auditinfo"
+.Sh DESCRIPTION
+The
+.Fn getaudit
+function returns a 
+.Fa struct auditinfo 
+describing the current user audit settings for the calling process. The
+information is returned in the struct pointed at by
+.Fa auditinfo .
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditon 2 ,
+.Xr auditctl 2 ,
+.Xr getauid 2 ,
+.Xr setauid 2 ,
+.Xr setaudit 2
+.Sh HISTORY
+The
+.Fn getaudit
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/getauid.2 b/bsd/man/man2/getauid.2
new file mode 100644
index 000000000..eb61c0f21
--- /dev/null
+++ b/bsd/man/man2/getauid.2
@@ -0,0 +1,55 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt GETAUID 2
+.Os Darwin
+.Sh NAME
+.Nm getauid
+.Nd get the audit user ID of the current process  
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn getauid "au_id_t * auid"
+.Sh DESCRIPTION
+The
+.Fn getauid
+function returns the audit user ID for the calling process. The variable pointed
+at by 
+.Fa auid
+holds the ID returned by the system.
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditon 2 ,
+.Xr auditctl 2 ,
+.Xr setauid 2 ,
+.Xr getaudit 2 ,
+.Xr setaudit 2
+.Sh HISTORY
+The
+.Fn getauid
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/getdirentries.2 b/bsd/man/man2/getdirentries.2
index 798847160..b0fedddaa 100644
--- a/bsd/man/man2/getdirentries.2
+++ b/bsd/man/man2/getdirentries.2
@@ -168,8 +168,8 @@ An
 error occurred while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
-.Xr open 2 ,
-.Xr lseek 2
+.Xr lseek 2 ,
+.Xr open 2
 .Sh HISTORY
 The
 .Fn getdirentries
diff --git a/bsd/man/man2/getfh.2 b/bsd/man/man2/getfh.2
index ff12795ad..c5152fbdd 100644
--- a/bsd/man/man2/getfh.2
+++ b/bsd/man/man2/getfh.2
@@ -1,3 +1,25 @@
+.\"
+.\" Copyright (c) 2002-2007 Apple Inc.  All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
 .\"	$NetBSD: getfh.2,v 1.7 1995/10/12 15:40:53 jtc Exp $
 .\"
 .\" Copyright (c) 1989, 1991, 1993
@@ -33,7 +55,7 @@
 .\"
 .\"	@(#)getfh.2	8.1 (Berkeley) 6/9/93
 .\"
-.Dd June 9, 1993
+.Dd January 9, 2007
 .Dt GETFH 2
 .Os
 .Sh NAME
@@ -49,10 +71,13 @@
 returns a file handle for the specified file or directory
 in the file handle pointed to by
 .Fa fhp .
-This system call is restricted to the superuser.
+This system call is restricted to the super-user and is used by an
+NFS server's
+.Xr nfsd 8
+daemon to obtain file handles used in NFS MOUNT service replies.
 .Sh RETURN VALUES
-Upon successful completion, a value of 0 is returned.
-Otherwise, -1 is returned and the global variable
+Upon successful completion, a value of 0 is returned.  Otherwise, -1
+is returned and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
@@ -66,17 +91,19 @@ is not a directory.
 .It Bq ENAMETOOLONG
 The length of a component of
 .Fa path
-exceeds 
+exceeds
 .Dv {NAME_MAX}
 characters, or the length of
 .Fa path
-exceeds 
-.Dv {PATH_MAX} 
+exceeds
+.Dv {PATH_MAX}
 characters.
 .It Bq ENOENT
 The file referred to by
 .Fa path
 does not exist.
+.It Bq EPERM
+The caller is not the super-user.
 .It Bq EACCES
 Search permission is denied for a component of the path prefix of
 .Fa path .
@@ -91,8 +118,9 @@ An
 .Tn I/O
 error occurred while reading from or writing to the file system.
 .El
+.Sh SEE ALSO
+.Xr nfsd 8
 .Sh HISTORY
 The
 .Fn getfh
-function
-first appeared in 4.4BSD.
+function first appeared in 4.4BSD.
diff --git a/bsd/man/man2/getgid.2 b/bsd/man/man2/getgid.2
index ee395cd65..9b8a31834 100644
--- a/bsd/man/man2/getgid.2
+++ b/bsd/man/man2/getgid.2
@@ -37,16 +37,19 @@
 .Dt GETGID 2
 .Os BSD 4.2
 .Sh NAME
-.Nm getgid ,
-.Nm getegid
+.Nm getegid ,
+.Nm getgid
 .Nd get group process identification
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft gid_t
-.Fn getgid void
+.Fo getegid
+.Fa void
+.Fc
 .Ft gid_t
-.Fn getegid void
+.Fo getgid
+.Fa void
+.Fc
 .Sh DESCRIPTION
 The
 .Fn getgid
@@ -68,8 +71,15 @@ The
 .Fn getgid
 and
 .Fn getegid
-functions are always successful, and no return value is reserved to
-indicate an error.
+functions are always successful;
+no return value is reserved to indicate an error.
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr getuid 2 ,
 .Xr setregid 2 ,
diff --git a/bsd/man/man2/getgroups.2 b/bsd/man/man2/getgroups.2
index ecc9c82ee..160022ca3 100644
--- a/bsd/man/man2/getgroups.2
+++ b/bsd/man/man2/getgroups.2
@@ -40,57 +40,71 @@
 .Nm getgroups
 .Nd get group access list
 .Sh SYNOPSIS
-.Fd #include <sys/param.h>
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft int
-.Fn getgroups "int gidsetlen" "gid_t *gidset"
+.Fo getgroups
+.Fa "int gidsetsize"
+.Fa "gid_t grouplist[]"
+.Fc
 .Sh DESCRIPTION
 .Fn Getgroups
 gets the current group access list of the current user process
 and stores it in the array 
-.Fa gidset .
+.Fa grouplist[] .
 The parameter
-.Fa gidsetlen
+.Fa gidsetsize
 indicates the number of entries that may be placed in 
-.Fa gidset .
+.Fa grouplist[] .
 .Fn Getgroups
 returns the actual number of groups returned in
-.Fa gidset .
+.Fa grouplist[] .
 No more than
 .Dv {NGROUPS_MAX}
 will ever
 be returned.
 If
-.Fa gidsetlen
+.Fa gidsetsize
 is 0, 
 .Fn getgroups
 returns the number of groups without modifying the
-.Fa gidset
+.Fa grouplist[]
 array.
 .Sh RETURN VALUES
 A successful call returns the number of groups in the group set.
-A value of -1 indicates that an error occurred, and the error
-code is stored in the global variable
-.Va errno .
+Otherwise, a value of -1 is returned and the global integer variable
+.Va errno
+is set to indicate the error.
 .Sh ERRORS
 The possible errors for
 .Fn getgroups
 are:
 .Bl -tag -width Er
+.\" ==========
+.It Bq Er EFAULT
+The argument
+.Fa grouplist
+specifies an invalid address.
+.\" ==========
 .It Bq Er EINVAL
 The argument
-.Fa gidsetlen
+.Fa gidsetsize ,
+although non-zero,
 is smaller than the number of groups in the group set.
-.It Bq Er EFAULT
-The argument
-.Fa gidset
-specifies
-an invalid address.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/param.h>
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include files
+.In sys/param.h
+and
+.In sys/types.h
+are necessary.
 .Sh SEE ALSO
 .Xr setgroups 2 ,
-.Xr initgroups 3
+.Xr initgroups 3 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn getgroups
diff --git a/bsd/man/man2/getitimer.2 b/bsd/man/man2/getitimer.2
index 2f22c203f..fb11f6cc3 100644
--- a/bsd/man/man2/getitimer.2
+++ b/bsd/man/man2/getitimer.2
@@ -42,13 +42,21 @@
 .Nd get/set value of interval timer
 .Sh SYNOPSIS
 .Fd #include <sys/time.h>
+.Pp
 .Fd #define ITIMER_REAL		0
 .Fd #define ITIMER_VIRTUAL	1
 .Fd #define ITIMER_PROF		2
 .Ft int
-.Fn getitimer "int which" "struct itimerval *value"
+.Fo getitimer
+.Fa "int which"
+.Fa "struct itimerval *value"
+.Fc
 .Ft int
-.Fn setitimer "int which" "const struct itimerval *value" "struct itimerval *ovalue"
+.Fo setitimer
+.Fa "int which"
+.Fa "const struct itimerval *restrict value"
+.Fa "struct itimerval *restrict ovalue"
+.Fc
 .Sh DESCRIPTION
 The system provides each process with three interval timers,
 defined in
@@ -137,30 +145,37 @@ tests if a time value is non-zero, and
 compares two time values (beware that >= and <= do not
 work with this macro).
 .Sh RETURN VALUES
-If the calls succeed, a value of 0 is returned.  If an error occurs,
-the value -1 is returned, and a more precise error code is placed
-in the global variable
-.Va errno .
+Upon successful completion, a value of 0 is returned.
+Otherwise, a value of -1 is returned and the global integer variable
+.Va errno
+is set to indicate the error.
 .Sh ERRORS
 .Fn Getitimer
 and
 .Fn setitimer
 will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EFAULT
 The
 .Fa value
 parameter specified a bad address.
+.\" ==========
 .It Bq Er EINVAL
-A
+The
 .Fa value
-parameter specified a time that was too large
-to be handled.
+parameter specified a time that was too large to be handled
+or not in the canonical form.
+.\" ==========
+.It Bq Er EINVAL
+The
+.Fa which
+parameter was invalid.
 .El
 .Sh SEE ALSO
+.Xr gettimeofday 2 ,
 .Xr select 2 ,
-.Xr sigaction 2 ,
-.Xr gettimeofday 2
+.Xr sigaction 2
 .Sh HISTORY
 The
 .Fn getitimer
diff --git a/bsd/man/man2/getlcid.2 b/bsd/man/man2/getlcid.2
new file mode 100644
index 000000000..6625da578
--- /dev/null
+++ b/bsd/man/man2/getlcid.2
@@ -0,0 +1,87 @@
+.\" Copyright (c) 2005 SPARTA, Inc.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.\" Note: The date here should be updated whenever a non-trivial
+.\" change is made to the manual page.
+.Dd May 5, 2005
+.Dt GETLCID 2
+.Os
+.Sh NAME
+.Nm getlcid
+.Nd "get login context"
+.Sh SYNOPSIS
+.In sys/lctx.h
+.Ft pid_t
+.Fn getlcid "pid_t pid"
+.Sh DESCRIPTION
+The login context of the process identified by
+.Fa pid
+is returned by
+.Fn getlcid .
+If
+.Fa pid
+is zero,
+.Fn getlcid
+returns the login context of the current process.
+.Pp
+Login contexts are used for tracking processes that originated from a users
+login session.  A login context is inherited via
+.Fn fork
+or by explicit creation using the
+.Fn setlcid
+call.
+.Sh RETURN VALUES
+If successful,
+.Fn getlcid
+returns a non-negative integer, the Login Context ID; otherwise the
+value \-1 is returned and the global variable
+.Fa errno
+is set to indicate
+the error.
+.Sh ERRORS
+The
+.Fn getlcid
+function will fail if:
+.Bl -tag -width Er
+.It Bq Er EPERM
+Operation not permitted.
+.It Bq Er ESRCH
+No such process.
+.It Bq Er ENOATTR
+Attribute not found.
+.El
+.Sh SEE ALSO
+.Xr setlcid 2
+.Sh HISTORY
+The
+.Nm
+manual page
+first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+This
+manual page was written by
+.An Matthew N. Dodd Aq mdodd@FreeBSD.org .
diff --git a/bsd/man/man2/getlogin.2 b/bsd/man/man2/getlogin.2
index 33f127fad..cf7562b74 100644
--- a/bsd/man/man2/getlogin.2
+++ b/bsd/man/man2/getlogin.2
@@ -43,9 +43,13 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft char *
-.Fn getlogin void
+.Fo getlogin
+.Fa void
+.Fc
 .Ft int
-.Fn setlogin "const char *name"
+.Fo setlogin
+.Fa "const char *name"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn getlogin
diff --git a/bsd/man/man2/getpeername.2 b/bsd/man/man2/getpeername.2
index 3fe1c98d4..88edc5d55 100644
--- a/bsd/man/man2/getpeername.2
+++ b/bsd/man/man2/getpeername.2
@@ -42,49 +42,68 @@
 .Sh SYNOPSIS
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn getpeername "int s" "struct sockaddr *name" "socklen_t *namelen"
+.Fo getpeername
+.Fa "int socket"
+.Fa "struct sockaddr *restrict address"
+.Fa "socklen_t *restrict address_len"
+.Fc
 .Sh DESCRIPTION
 .Fn Getpeername
 returns the name of the peer connected to
 socket
-.Fa s .
+.Fa socket .
 The
-.Fa namelen
+.Fa address_len
 parameter should be initialized to indicate
 the amount of space pointed to by
 .Fa name .
 On return it contains the actual size of the name
 returned (in bytes).
 The name is truncated if the buffer provided is too small.
-.Sh DIAGNOSTICS
-A 0 is returned if the call succeeds, -1 if it fails.
+.Sh RETURN VALUES
+.Rv -std getpeername
 .Sh ERRORS
 The call succeeds unless:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The argument
-.Fa s
+.Fa socket
 is not a valid descriptor.
-.It Bq Er ENOTSOCK
-The argument
-.Fa s
-is a file, not a socket.
-.It Bq Er ENOTCONN
-The socket is not connected.
-.It Bq Er ENOBUFS
-Insufficient resources were available in the system
-to perform the operation.
+.\" ==========
 .It Bq Er EFAULT
 The 
-.Fa name
+.Fa address
 parameter points to memory not in a valid part of the
 process address space.
+.\" ==========
+.It Bq Er EINVAL
+.Fa socket
+has been shut down.
+.\" ==========
+.It Bq Er ENOBUFS
+Insufficient resources were available in the system
+to perform the operation.
+.\" ==========
+.It Bq Er ENOTCONN
+Either the socket is not connected
+or it has not had the peer pre-specified.
+.\" ==========
+.It Bq Er ENOTSOCK
+The argument
+.Fa socket
+refers to something other than a socket (e.g., a file).
+.\" ==========
+.It Bq Er EOPNOTSUPP
+.Fn getpeername
+is not supported for the protocol in use by
+.Fa socket .
 .El
 .Sh SEE ALSO
 .Xr accept 2 ,
 .Xr bind 2 ,
-.Xr socket 2 ,
-.Xr getsockname 2
+.Xr getsockname 2 ,
+.Xr socket 2
 .Sh HISTORY
 The
 .Fn getpeername
diff --git a/bsd/man/man2/getpgrp.2 b/bsd/man/man2/getpgrp.2
index 7b5a0f51d..7a27f1552 100644
--- a/bsd/man/man2/getpgrp.2
+++ b/bsd/man/man2/getpgrp.2
@@ -36,6 +36,7 @@
 .Dt GETPGRP 2
 .Os
 .Sh NAME
+.Nm getpgid ,
 .Nm getpgrp
 .Nd get process group
 .Sh LIBRARY
@@ -43,9 +44,13 @@
 .Sh SYNOPSIS
 .In unistd.h
 .Ft pid_t
-.Fn getpgrp void
+.Fo getpgid
+.Fa "pid_t pid"
+.Fc
 .Ft pid_t
-.Fn getpgid "pid_t pid"
+.Fo getpgrp
+.Fa void
+.Fc
 .Sh DESCRIPTION
 The process group of the current process is returned by
 .Fn getpgrp .
@@ -90,9 +95,10 @@ to indicate the error.
 .Fn getpgid
 will succeed unless:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er ESRCH
-there is no process whose process ID equals
-.Fa pid
+There is no process whose process ID equals
+.Fa pid .
 .El
 .Sh SEE ALSO
 .Xr getsid 2 ,
diff --git a/bsd/man/man2/getpid.2 b/bsd/man/man2/getpid.2
index 0c7f5e365..102889356 100644
--- a/bsd/man/man2/getpid.2
+++ b/bsd/man/man2/getpid.2
@@ -41,12 +41,15 @@
 .Nm getppid
 .Nd get parent or calling process identification
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft pid_t
-.Fn getpid void
+.Fo getpid
+.Fa void
+.Fc
 .Ft pid_t
-.Fn getppid void
+.Fo getppid
+.Fa void
+.Fc
 .Sh DESCRIPTION
 .Fn Getpid
 returns
@@ -65,8 +68,16 @@ and
 .Fn getppid
 functions are always successful, and no return value is reserved to
 indicate an error.
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
-.Xr gethostid 2
+.Xr gethostid 2 ,
+.Xr compat 5
 .Sh STANDARDS
 .Fn Getpid
 and
diff --git a/bsd/man/man2/getpriority.2 b/bsd/man/man2/getpriority.2
index 1c5713afd..07ed5b6d8 100644
--- a/bsd/man/man2/getpriority.2
+++ b/bsd/man/man2/getpriority.2
@@ -41,15 +41,21 @@
 .Nm setpriority
 .Nd get/set program scheduling priority
 .Sh SYNOPSIS
-.Fd #include <sys/time.h>
 .Fd #include <sys/resource.h>
 .Ft int
-.Fn getpriority "int which" "int who"
+.Fo getpriority
+.Fa "int which"
+.Fa "id_t who"
+.Fc
 .Ft int
-.Fn setpriority "int which" "int who" "int prio"
+.Fo setpriority
+.Fa "int which"
+.Fa "id_t who"
+.Fa "int prio"
+.Fc
 .Sh DESCRIPTION
 The scheduling
-priority of the process, process group, or user, as indicated by
+priority of the process, process group, or user as indicated by
 .Fa which
 and
 .Fa who
@@ -57,13 +63,15 @@ is obtained with the
 .Fn getpriority
 call and set with the
 .Fn setpriority
-call.
+call.  Additionally, the current thread can be set to background
+state.
 .Fa Which
 is one of
 .Dv PRIO_PROCESS ,
 .Dv PRIO_PGRP ,
-or
 .Dv PRIO_USER ,
+or
+.Dv PRIO_DARWIN_THREAD ,
 and 
 .Fa who
 is interpreted relative to 
@@ -77,18 +85,31 @@ and a user ID for
 .Dv PRIO_USER ) .
 A zero value of
 .Fa who
-denotes the current process, process group, or user.
-.Fa Prio
+denotes the current process, process group, user, thread (for PRIO_DARWIN_THREAD).
+.Fa prio
 is a value in the range -20 to 20.  The default priority is 0;
 lower priorities cause more favorable scheduling.
+For PRIO_DARWIN_THREAD,
+.Fa prio
+is either 0 (to remove current thread from background status) or PRIO_DARWIN_BG
+(to set current thread into background state).
 .Pp
 The
 .Fn getpriority
 call returns the highest priority (lowest numerical value)
-enjoyed by any of the specified processes.  The
+enjoyed by any of the specified processes or for PRIO_DARWIN_THREAD
+returns 0 when current thread is not in background state or 1
+when the current thread is in background state.
+The
 .Fn setpriority
 call sets the priorities of all of the specified processes
-to the specified value.  Only the super-user may lower priorities.
+to the specified value.  When setting a thread into background
+state the scheduling priority is set to lowest value, disk and
+network IO are throttled.  Network IO will be throttled for any
+sockets the thread opens after going into background state.  Any
+previously opened sockets are not affected.
+Only the super-user may lower priorities, but any thread can set
+itself into background state.
 .Sh RETURN VALUES
 Since
 .Fn getpriority
@@ -108,34 +129,77 @@ and
 .Fn setpriority
 will fail if:
 .Bl -tag -width Er
-.It Bq Er ESRCH
-No process was located using the 
-.Fa which
-and
-.Fa who
-values specified.
+.\" ==========
 .It Bq Er EINVAL
 .Fa Which
-was not one of
+is not one of
 .Dv PRIO_PROCESS ,
 .Dv PRIO_PGRP ,
+.Dv PRIO_USER ,
 or
-.Dv PRIO_USER .
+.Dv PRIO_DARWIN_THREAD .
+.\" ==========
+.It Bq Er EINVAL
+.Fa Who
+is not a valid process, process group, or user ID.
+.\" ==========
+.It Bq Er EINVAL
+.Fa Who
+is not 0 when
+.Fa which
+is PRIO_DARWIN_THREAD.
+.\" ==========
+.It Bq Er ESRCH
+No process can be located using the 
+.Fa which
+and
+.Fa who
+values specified.
 .El
 .Pp
 .Bl -tag -width Er
 In addition to the errors indicated above,
 .Fn setpriority
 will fail if:
-.It Bq Er EPERM
-A process was located, but neither its effective nor real user
-ID matched the effective user ID of the caller.
+.\" ==========
 .It Bq Er EACCES
-A non super-user attempted to lower a process priority.
+A non super-user attempts to lower a process priority.
+.\" ==========
+.It Bq Er EPERM
+A process is located,
+but neither its effective nor real user ID
+matches the effective user ID of the caller.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/resource.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Pp
+.Ft int
+.br
+.Fo getpriority
+.Fa "int which"
+.Fa "int who"
+.Fc ;
+.Pp
+.Ft int
+.br
+.Fo setpriority
+.Fa "int which"
+.Fa "int who"
+.Fa "int value"
+.Fc ;
+.Pp
+The type of
+.Fa who
+has changed.
 .Sh SEE ALSO
 .Xr nice 1 ,
 .Xr fork 2 ,
+.Xr compat 5 ,
 .Xr renice 8
 .Sh HISTORY
 The
diff --git a/bsd/man/man2/getrlimit.2 b/bsd/man/man2/getrlimit.2
index c81379974..98229e32b 100644
--- a/bsd/man/man2/getrlimit.2
+++ b/bsd/man/man2/getrlimit.2
@@ -41,13 +41,17 @@
 .Nm setrlimit
 .Nd control maximum system resource consumption
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
-.Fd #include <sys/time.h>
 .Fd #include <sys/resource.h>
 .Ft int
-.Fn getrlimit "int resource" "struct rlimit *rlp"
+.Fo getrlimit
+.Fa "int resource"
+.Fa "struct rlimit *rlp"
+.Fc
 .Ft int
-.Fn setrlimit "int resource" "const struct rlimit *rlp"
+.Fo setrlimit
+.Fa "int resource"
+.Fa "const struct rlimit *rlp"
+.Fc
 .Sh DESCRIPTION
 Limits on the consumption of system resources by the current process
 and each process it creates may be obtained with the
@@ -60,35 +64,45 @@ The
 .Fa resource
 parameter is one of the following:
 .Bl -tag -width RLIMIT_FSIZEAA
+.\" ========
 .It Li RLIMIT_CORE
 The largest size (in bytes)
 .Xr core
 file that may be created.
+.\" ========
 .It Li RLIMIT_CPU
 The maximum amount of cpu time (in seconds) to be used by
 each process.
+.\" ========
 .It Li RLIMIT_DATA
 The maximum size (in bytes) of the data segment for a process;
 this defines how far a program may extend its break with the
 .Xr sbrk 2
 system call.
+.\" ========
 .It Li RLIMIT_FSIZE
 The largest size (in bytes) file that may be created.
+.\" ========
 .It Li RLIMIT_MEMLOCK
 The maximum size (in bytes) which a process may lock into memory
 using the
 .Xr mlock 2
 function.
+.\" ========
 .It Li RLIMIT_NOFILE
 The maximum number of open files for this process.
+.\" ========
 .It Li RLIMIT_NPROC
 The maximum number of simultaneous processes for this user id.
+.\" ========
 .It Li RLIMIT_RSS
-The maximum size (in bytes) to which a process's resident set size may
-grow.
-This imposes a limit on the amount of physical memory to be given to
-a process; if memory is tight, the system will prefer to take memory
+The maximum size (in bytes)
+to which a process's resident set size may grow.
+This imposes a limit on the amount of physical memory
+to be given to a process;
+if memory is tight, the system will prefer to take memory
 from processes that are exceeding their declared resident set size.
+.\" ========
 .It Li RLIMIT_STACK
 The maximum size (in bytes) of the stack segment for a process;
 this defines how far a program's stack segment may be extended.
@@ -117,11 +131,6 @@ within the range from 0 to
 or (irreversibly) lower
 .Fa rlim_max .
 .Pp
-An
-.Dq infinite
-value for a limit is defined as
-.Dv RLIM_INFINITY .
-.Pp
 Because this information is stored in the per-process information,
 this system call must be executed directly by the shell if it
 is to affect all future processes created by the shell;
@@ -161,28 +170,61 @@ that an error occurred, and an error code is stored in the global
 location
 .Va errno .
 .Sh ERRORS
-.Fn Getrlimit
+The
+.Fn getrlimit
 and
 .Fn setrlimit
-will fail if:
+system calls will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EFAULT
 The address specified for
 .Fa rlp
 is invalid.
-.It Bq Er EPERM
-The limit specified to
+.\" ==========
+.It Bq Er EINVAL
+.Fa resource
+is invalid.
+.El
+.Pp
+The
 .Fn setrlimit
-would have
-raised the maximum limit value, and the caller is not the super-user.
+call will fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The specified limit is invalid
+(e.g., RLIM_INFINITY or lower than rlim_cur).
+.\" ==========
+.It Bq Er EPERM
+The limit specified would have raised the maximum limit value
+and the caller is not the super-user.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/time.h>
+.Fd #include <sys/resource.h>
+.Pp
+The include files
+.In sys/types.h
+and
+.In sys/time.h
+are necessary.
+.Sh COMPATIBILITY
+.Fn setrlimit
+now returns with
+.Va errno
+set to EINVAL in places that historically succeeded.
+It no longer accepts "rlim_cur = RLIM_INFINITY" for RLIM_NOFILE.
+Use "rlim_cur = min(OPEN_MAX, rlim_max)".
 .Sh SEE ALSO
 .Xr csh 1 ,
 .Xr sh 1 ,
 .Xr quota 2 ,
 .Xr sigaction 2 ,
 .Xr sigaltstack 2 ,
-.Xr sysctl 3
+.Xr sysctl 3 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn getrlimit
diff --git a/bsd/man/man2/getrusage.2 b/bsd/man/man2/getrusage.2
index a361d1d36..d06572017 100644
--- a/bsd/man/man2/getrusage.2
+++ b/bsd/man/man2/getrusage.2
@@ -40,12 +40,15 @@
 .Nm getrusage
 .Nd get information about resource utilization
 .Sh SYNOPSIS
-.Fd #include <sys/time.h>
 .Fd #include <sys/resource.h>
+.Pp
 .Fd #define	RUSAGE_SELF	 0
 .Fd #define	RUSAGE_CHILDREN	-1
 .Ft int
-.Fn getrusage "int who" "struct rusage *rusage"
+.Fo getrusage
+.Fa "int who"
+.Fa "struct rusage *r_usage"
+.Fc
 .Sh DESCRIPTION
 .Fn Getrusage
 returns information describing the resources utilized by the current
@@ -57,7 +60,7 @@ parameter is either
 or
 .Dv RUSAGE_CHILDREN .
 The buffer to which
-.Fa rusage
+.Fa r_usage
 points will be filled in with
 the following structure:
 .Bd -literal
@@ -139,23 +142,33 @@ and
 account only for real
 I/O; data supplied by the caching mechanism is charged only
 to the first process to read or write the data.
+.Sh RETURN VALUES
+.Rv -std getrusage
 .Sh ERRORS
-.Fn Getrusage
-returns -1 on error.
-The possible errors are:
+The
+.Fn getrusage
+system call will fail if:
 .Bl -tag -width Er
+.It Bq Er EFAULT
+The address specified by the
+.Fa r_usage
+parameter is not in a valid part of the process address space.
 .It Bq Er EINVAL
 The
 .Fa who
 parameter is not a valid value.
-.It Bq Er EFAULT
-The address specified by the
-.Fa rusage
-parameter is not in a valid part of the process address space.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/time.h>
+.Fd #include <sys/resource.h>
+.Pp
+The include file
+.In sys/time.h
+is necessary.
 .Sh SEE ALSO
 .Xr gettimeofday 2 ,
-.Xr wait 2
+.Xr wait 2 ,
+.Xr compat 5
 .Sh BUGS
 There is no way to obtain information about a child process
 that has not yet terminated.
diff --git a/bsd/man/man2/getsid.2 b/bsd/man/man2/getsid.2
index 2bd3c272e..811c2fcd5 100644
--- a/bsd/man/man2/getsid.2
+++ b/bsd/man/man2/getsid.2
@@ -34,7 +34,9 @@
 .Sh SYNOPSIS
 .In unistd.h
 .Ft pid_t
-.Fn getsid "pid_t pid"
+.Fo getsid
+.Fa "pid_t pid"
+.Fc
 .Sh DESCRIPTION
 The session ID of the process identified by
 .Fa pid
@@ -52,11 +54,13 @@ returns the session ID of
 the specified process; otherwise, it returns a value of -1 and
 sets errno to indicate an error.
 .Sh ERRORS
+The
 .Fn getsid
-will succeed unless:
+system call will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er ESRCH
-if there is no process with a process ID equal to
+There is no process with a process ID equal to
 .Fa pid .
 .El
 .Pp
diff --git a/bsd/man/man2/getsockname.2 b/bsd/man/man2/getsockname.2
index 2d63acb38..45241e307 100644
--- a/bsd/man/man2/getsockname.2
+++ b/bsd/man/man2/getsockname.2
@@ -42,49 +42,68 @@
 .Sh SYNOPSIS
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn getsockname "int s" "struct sockaddr *name" "socklen_t *namelen"
+.Fo getsockname
+.Fa "int socket"
+.Fa "struct sockaddr *restrict address"
+.Fa "socklen_t *restrict address_len"
+.Fc
 .Sh DESCRIPTION
 .Fn Getsockname
 returns the current 
-.Fa name
+.Fa address
 for the specified socket.  The
-.Fa namelen
+.Fa address_len
 parameter should be initialized to indicate
 the amount of space pointed to by
-.Fa name .
+.Fa address .
 On return it contains the actual size of the name
 returned (in bytes).
-.Sh DIAGNOSTICS
-A 0 is returned if the call succeeds, -1 if it fails.
+.Sh RETURN VALUES
+.Rv -std getsockname
 .Sh ERRORS
-The call succeeds unless:
+The
+.Fn getsockname
+system call will succeed unless:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The argument
-.Fa s
-is not a valid descriptor.
-.It Bq Er ENOTSOCK
-The argument
-.Fa s
-is a file, not a socket.
-.It Bq Er ENOBUFS
-Insufficient resources were available in the system
-to perform the operation.
+.Fa socket
+is not a valid file descriptor.
+.\" ==========
 .It Bq Er EFAULT
 The 
-.Fa name
+.Fa address
 parameter points to memory not in a valid part of the
 process address space.
+.\" ==========
+.It Bq Er EINVAL
+.Fa socket
+has been shut down.
+.\" ==========
+.It Bq Er ENOBUFS
+Insufficient resources were available in the system
+to perform the operation.
+.\" ==========
+.It Bq Er ENOTSOCK
+The argument
+.Fa socket
+is not a socket (e.g., a plain file).
+.\" ==========
+.It Bq Er EOPNOTSUPP
+.Fn getsockname
+is not supported for the protocol in use by
+.Fa socket .
 .El
 .Sh SEE ALSO
 .Xr bind 2 ,
 .Xr socket 2
 .Sh BUGS
 Names bound to sockets in the UNIX domain are inaccessible;
-.Xr getsockname
-returns a zero length name.
+.Fn getsockname
+returns a zero-length name.
 .Sh HISTORY
 The
 .Fn getsockname
-function call appeared in
+call appeared in
 .Bx 4.2 .
diff --git a/bsd/man/man2/getsockopt.2 b/bsd/man/man2/getsockopt.2
index 1f22a55da..9e3bef41f 100644
--- a/bsd/man/man2/getsockopt.2
+++ b/bsd/man/man2/getsockopt.2
@@ -41,12 +41,23 @@
 .Nm setsockopt
 .Nd get and set options on sockets
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn getsockopt "int s" "int level" "int optname" "void *optval" "socklen_t *optlen"
+.Fo getsockopt
+.Fa "int socket"
+.Fa "int level"
+.Fa "int option_name"
+.Fa "void *restrict option_value"
+.Fa "socklen_t *restrict option_len"
+.Fc
 .Ft int
-.Fn setsockopt "int s" "int level" "int optname" "const void *optval" "socklen_t optlen"
+.Fo setsockopt
+.Fa "int socket"
+.Fa "int level"
+.Fa "int option_name"
+.Fa "const void *option_value"
+.Fa "socklen_t option_len"
+.Fc
 .Sh DESCRIPTION
 .Fn Getsockopt
 and
@@ -77,9 +88,9 @@ see
 .Xr getprotoent 3 .
 .Pp
 The parameters
-.Fa optval
+.Fa option_value
 and
-.Fa optlen
+.Fa option_len
 are used to access option values for
 .Fn setsockopt .
 For
@@ -87,17 +98,17 @@ For
 they identify a buffer in which the value for the
 requested option(s) are to be returned.  For
 .Fn getsockopt ,
-.Fa optlen
+.Fa option_len
 is a value-result parameter, initially containing the
 size of the buffer pointed to by
-.Fa optval ,
+.Fa option_value ,
 and modified on return to indicate the actual size of
 the value returned.  If no option value is
 to be supplied or returned,
-.Fa optval
+.Fa option_value
 may be NULL.
 .Pp
-.Fa Optname
+.Fa option_name
 and any specified options are passed uninterpreted to the appropriate
 protocol module for interpretation.
 The include file
@@ -112,7 +123,7 @@ section
 Most socket-level options utilize an
 .Fa int
 parameter for
-.Fa optval .
+.Fa option_value .
 For
 .Fn setsockopt ,
 the parameter should be non-zero to enable a boolean option,
@@ -235,35 +246,38 @@ The system places an absolute limit on these values.
 .Pp
 .Dv SO_SNDLOWAT
 is an option to set the minimum count for output operations.
-Most output operations process all of the data supplied
-by the call, delivering data to the protocol for transmission
+Most output operations process all of the data supplied by the call,
+delivering data to the protocol for transmission
 and blocking as necessary for flow control.
 Nonblocking output operations will process as much data as permitted
-subject to flow control without blocking, but will process no data
-if flow control does not allow the smaller of the low water mark value
+(subject to flow control) without blocking,
+but will process no data if flow control
+does not allow the smaller of the low-water mark value
 or the entire request to be processed.
 A
 .Xr select 2
 operation testing the ability to write to a socket will return true
-only if the low water mark amount could be processed.
+only if the low-water mark amount could be processed.
 The default value for
 .Dv SO_SNDLOWAT
 is set to a convenient size for network efficiency, often 1024.
+.Pp
 .Dv SO_RCVLOWAT
 is an option to set the minimum count for input operations.
 In general, receive calls will block until any (non-zero) amount of data
-is received, then return with the smaller of the amount available or the amount
-requested.
+is received, then return with the smaller of the amount available
+or the amount requested.
 The default value for
 .Dv SO_RCVLOWAT
 is 1.
 If 
 .Dv SO_RCVLOWAT
-is set to a larger value, blocking receive calls normally
-wait until they have received the smaller of the low water mark value
-or the requested amount.
-Receive calls may still return less than the low water mark if an error
-occurs, a signal is caught, or the type of data next in the receive queue
+is set to a larger value, blocking receive calls
+normally wait until they have received the smaller
+of the low-water mark value or the requested amount.
+Receive calls may still return less than the low-water mark
+if an error occurs, a signal is caught,
+or the type of data next in the receive queue
 is different than that returned.
 .Pp
 .Dv SO_SNDTIMEO
@@ -280,7 +294,8 @@ if no data were sent.
 In the current implementation, this timer is restarted each time additional
 data are delivered to the protocol,
 implying that the limit applies to output portions ranging in size
-from the low water mark to the high water mark for output.
+from the low-water mark to the high-water mark for output.
+.Pp
 .Dv SO_RCVTIMEO
 is an option to set a timeout value for input operations.
 It accepts a
@@ -294,15 +309,16 @@ If a receive operation has been blocked for this much time without
 receiving additional data, it returns with a short count
 or with the error
 .Er EWOULDBLOCK
-if no data were received. The struct timeval parameter must represent a 
-positive time interval otherwise 
+if no data were received.
+The struct timeval parameter must represent a positive time interval;
+otherwise,
 .Fn setsockopt 
 returns with the error
 .Er EDOM .
 .Pp
 .Dv SO_NOSIGPIPE is an option that prevents SIGPIPE from being raised 
 when a write fails on a socket to which there is no reader; 
-instead the write to the socket returns with the error 
+instead, the write to the socket returns with the error 
 .Er EPIPE
 when there is no reader.
 .Pp
@@ -322,36 +338,72 @@ the error status.
 It may be used to check for asynchronous errors on connected
 datagram sockets or for other asynchronous errors.
 .Sh RETURN VALUES
-A 0 is returned if the call succeeds, -1 if it fails.
+.Rv -std
 .Sh ERRORS
-The call succeeds unless:
+The
+.Fn getsockopt
+and
+.Fn setsockopt
+system calls will succeed unless:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 The argument
-.Fa s
-is not a valid descriptor.
-.It Bq Er ENOTSOCK
-The argument
-.Fa s
-is a file, not a socket.
-.It Bq Er ENOPROTOOPT
-The option is unknown at the level indicated.
+.Fa socket
+is not a valid file descriptor.
+.\" ==========
 .It Bq Er EFAULT
 The address pointed to by 
-.Fa optval
+.Fa option_value
 is not in a valid part of the process address space.
 For
 .Fn getsockopt ,
 this error may also be returned if
-.Fa optlen
+.Fa option_len
 is not in a valid part of the process address space.
+.\" ==========
+.It Bq Er EINVAL
+The option is invalid at the level indicated.
+.\" ==========
+.It Bq Er ENOBUFS
+Insufficient memory buffers are available.
+.\" ==========
+.It Bq Er ENOPROTOOPT
+The option is unknown at the level indicated.
+.\" ==========
+.It Bq Er ENOTSOCK
+The argument
+.Fa socket
+is not a socket (e.g., a plain file).
+.El
+.Pp
+The
+.Fn setsockopt
+system call will succeed unless:
+.Bl -tag -width Er
+.\" ==========
 .It Bq Er EDOM
-The argument value is out of bounds. 
+The argument
+.Fa option_value
+is out of bounds. 
+.\" ==========
+.It Bq Er EISCONN
+.Fa socket
+is already connected
+and a specified option cannot be set
+while this is the case.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr ioctl 2 ,
 .Xr socket 2 ,
-.Xr getprotoent 3
+.Xr getprotoent 3 ,
 .Xr protocols 5
 .Sh BUGS
 Several of the socket options should be handled at lower levels of the system.
diff --git a/bsd/man/man2/gettimeofday.2 b/bsd/man/man2/gettimeofday.2
index febc61dcd..67f2760c6 100644
--- a/bsd/man/man2/gettimeofday.2
+++ b/bsd/man/man2/gettimeofday.2
@@ -43,9 +43,15 @@
 .Sh SYNOPSIS
 .Fd #include <sys/time.h>
 .Ft int
-.Fn gettimeofday "struct timeval *tp" "struct timezone *tzp"
+.Fo gettimeofday
+.Fa "struct timeval *restrict tp"
+.Fa "void *restrict tzp"
+.Fc
 .Ft int
-.Fn settimeofday "const struct timeval *tp" "const struct timezone *tzp"
+.Fo settimeofday
+.Fa "const struct timeval *tp"
+.Fa "const struct timezone *tzp"
+.Fc
 .Sh DESCRIPTION
 .Bf -symbolic
 Note: timezone is no longer used; this information is kept outside
@@ -77,8 +83,8 @@ as:
 .Pp
 .Bd -literal
 struct timeval {
-	long	tv_sec;		/* seconds since Jan. 1, 1970 */
-	long	tv_usec;	/* and microseconds */
+	time_t       tv_sec;   /* seconds since Jan. 1, 1970 */
+	suseconds_t  tv_usec;  /* and microseconds */
 };
 
 struct timezone {
@@ -88,6 +94,14 @@ struct timezone {
 .Ed
 .Pp
 The 
+.Fa timeval
+structure specifies a time value in seconds and microseconds.
+The values in
+.Fa timeval
+are opaque types whose length may vary on different machines;
+depending on them to have any given length may lead to errors.
+.Pp
+The 
 .Fa timezone
 structure indicates the local time zone
 (measured in minutes of time westward from Greenwich),
@@ -118,10 +132,24 @@ An argument address referenced invalid memory.
 .It Bq Er EPERM
 A user other than the super-user attempted to set the time.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/time.h>
+.Pp
+.Ft int
+.br
+.Fo gettimeofday
+.Fa "struct timeval *tp"
+.Fa "struct timezone *tzp"
+.Fc ;
+.Pp
+The type of
+.Fa tzp
+has changed.
 .Sh SEE ALSO
 .Xr date 1 ,
 .Xr adjtime 2 ,
 .Xr ctime 3 ,
+.Xr compat 5 ,
 .Xr timed 8
 .Sh HISTORY
 The
diff --git a/bsd/man/man2/getuid.2 b/bsd/man/man2/getuid.2
index 271a48dab..e8be23ce2 100644
--- a/bsd/man/man2/getuid.2
+++ b/bsd/man/man2/getuid.2
@@ -37,16 +37,19 @@
 .Dt GETUID 2
 .Os BSD 4
 .Sh NAME
-.Nm getuid ,
-.Nm geteuid
+.Nm geteuid ,
+.Nm getuid
 .Nd get user identification
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft uid_t
-.Fn getuid void
+.Fo geteuid
+.Fa void
+.Fc
 .Ft uid_t
-.Fn geteuid void
+.Fo getuid
+.Fa void
+.Fc
 .Sh DESCRIPTION
 The
 .Fn getuid
@@ -71,9 +74,17 @@ and
 .Fn geteuid
 functions are always successful, and no return value is reserved to
 indicate an error.
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary for both functions.
 .Sh SEE ALSO
 .Xr getgid 2 ,
-.Xr setreuid 2
+.Xr setreuid 2 ,
+.Xr compat 5
 .Sh STANDARDS
 .Fn Geteuid
 and
diff --git a/bsd/man/man2/getxattr.2 b/bsd/man/man2/getxattr.2
index 8f04a9194..581d746e3 100644
--- a/bsd/man/man2/getxattr.2
+++ b/bsd/man/man2/getxattr.2
@@ -155,9 +155,9 @@ points to an invalid address.
 An I/O error occurred while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
-.Xr setxattr 2 ,
+.Xr listxattr 2 ,
 .Xr removexattr 2 ,
-.Xr listxattr 2
+.Xr setxattr 2
 .Sh HISTORY
 .Fn getxattr
 and
diff --git a/bsd/man/man2/i386_set_ldt.2 b/bsd/man/man2/i386_set_ldt.2
new file mode 100644
index 000000000..ef05c363a
--- /dev/null
+++ b/bsd/man/man2/i386_set_ldt.2
@@ -0,0 +1 @@
+.so man2/i386_get_ldt.2
diff --git a/bsd/man/man2/intro.2 b/bsd/man/man2/intro.2
index addbe6aa3..1f6ba1495 100644
--- a/bsd/man/man2/intro.2
+++ b/bsd/man/man2/intro.2
@@ -198,7 +198,10 @@ call requested write access.
 The size of a file exceeded the maximum (about
 .if t 2\u\s-231\s+2\d
 .if n 2.1E9
-bytes).
+bytes on some filesystems including UFS,
+.if t 2\u\s-231\s+2\d
+.if n 1.8E19
+bytes on HFS+ and others).
 .It Er 28 ENOSPC Em "Device out of space" .
 A
 .Xr write
diff --git a/bsd/man/man2/ioctl.2 b/bsd/man/man2/ioctl.2
index 1c3d6d6ba..9d6343bf2 100644
--- a/bsd/man/man2/ioctl.2
+++ b/bsd/man/man2/ioctl.2
@@ -42,7 +42,11 @@
 .Sh SYNOPSIS
 .Fd #include <sys/ioctl.h>
 .Ft int
-.Fn ioctl "int d" "unsigned long request" "char *argp"
+.Fo ioctl
+.Fa "int fildes"
+.Fa "unsigned long request"
+.Fa "..."
+.Fc
 .Sh DESCRIPTION
 The
 .Fn ioctl
@@ -53,7 +57,7 @@ may be controlled with
 .Fn ioctl
 requests.
 The argument
-.Fa d
+.Fa fildes
 must be an open file descriptor.
 .Pp
 An  ioctl
@@ -78,32 +82,36 @@ is set to indicate the error.
 .Fn Ioctl
 will fail if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
-.Fa d
+.Fa fildes
 is not a valid descriptor.
+.\" ==========
+.It Bq Er EINVAL
+.Fa Request
+or
+.Fa argp
+is not valid.
+.\" ==========
 .It Bq Er ENOTTY
-.Fa d
+.Fa fildes
 is not associated with a character
 special device.
+.\" ==========
 .It Bq Er ENOTTY
 The specified request does not apply to the kind
 of object that the descriptor
-.Fa d
+.Fa fildes
 references.
-.It Bq Er EINVAL
-.Fa Request
-or
-.Fa argp
-is not valid.
 .El
 .Sh SEE ALSO
-.Xr mt 1 ,
 .Xr cdio 1 ,
 .Xr chio 1 ,
+.Xr mt 1 ,
 .Xr execve 2 ,
 .Xr fcntl 2 ,
-.Xr tty 4 ,
-.Xr intro 4
+.Xr intro 4 ,
+.Xr tty 4
 .Sh HISTORY
 An
 .Fn ioctl
diff --git a/bsd/man/man2/issetugid.2 b/bsd/man/man2/issetugid.2
index aca73d10e..6b202e55a 100644
--- a/bsd/man/man2/issetugid.2
+++ b/bsd/man/man2/issetugid.2
@@ -31,57 +31,72 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd August, 25 1996
+.\" $FreeBSD: src/lib/libc/sys/issetugid.2,v 1.13 2004/07/02 23:52:13 ru Exp $
+.\"
+.Dd August 25, 1996
 .Dt ISSETUGID 2
-.Os OpenBSD 2.0
+.Os
 .Sh NAME
 .Nm issetugid
-.Nd is current executable running setuid or setgid
+.Nd is current process tainted by uid or gid changes
+.Sh LIBRARY
+.Lb libc
 .Sh SYNOPSIS
-.Fd #include <unistd.h>
+.In unistd.h
 .Ft int
 .Fn issetugid void
 .Sh DESCRIPTION
 The
 .Fn issetugid
-function returns 1 if the process was made setuid or setgid as
-the result of the last
-.Fn execve
-system call.
-Otherwise it returns 0.
+system call returns 1 if the process environment or memory address space
+is considered
+.Dq tainted ,
+and returns 0 otherwise.
 .Pp
-This system call exists so that library routines (inside libc, libtermlib,
-or other libraries) can gaurantee safe behavior when used inside
-setuid or setgid programs.
-Some library routines may not be passed sufficient information to know
-if the current program was started setuid or setgid because higher level
-calling code may have made changes to the uid or the euid.
-In particular, it is wise to use this call to determine if a
-pathname returned from a
-.Fn getenv
-call may safely be used to
-.Fn open
-the specified file.
+A process is tainted if it was created as a result of an
+.Xr execve 2
+system call which had either of the setuid or setgid bits set (and extra
+privileges were given as a result) or if it has changed any of its real,
+effective or saved user or group ID's since it began execution.
 .Pp
-.Fn issetugid
-is unaffected by calls to
-.Fn setuid ,
-.Fn fork ,
-and other such calls.  It is only controlled by
-.Fn execve .
+This system call exists so that library routines (eg: libc, libtermcap)
+can reliably determine if it is safe to use information
+that was obtained from the user, in particular the results from
+.Xr getenv 3
+should be viewed with suspicion if it is used to control operation.
+.Pp
+A
+.Dq tainted
+status is inherited by child processes as a result of the
+.Xr fork 2
+system call (or other library code that calls fork, such as
+.Xr popen 3 ) .
+.Pp
+It is assumed that a program that clears all privileges as it prepares
+to execute another will also reset the environment, hence the
+.Dq tainted
+status will not be passed on.
+This is important for programs such as
+.Xr su 1
+which begin setuid but need to be able to create an untainted process.
 .Sh ERRORS
 The
 .Fn issetugid
-function is always successful, and no return value is reserved to
+system call is always successful, and no return value is reserved to
 indicate an error.
 .Sh SEE ALSO
 .Xr execve 2 ,
-.Xr setuid 2 ,
-.Xr seteuid 2,
+.Xr fork 2 ,
+.Xr setegid 2 ,
+.Xr seteuid 2 ,
 .Xr setgid 2 ,
-.Xr setegid 2
+.Xr setregid 2 ,
+.Xr setreuid 2 ,
+.Xr setuid 2
 .Sh HISTORY
-A
-.Fn lstat
-function call appeared in
-OpenBSD 2.0
+The
+.Fn issetugid
+system call first appeared in
+.Ox 2.0
+and was also implemented in
+.Fx 3.0 .
diff --git a/bsd/man/man2/kevent.2 b/bsd/man/man2/kevent.2
new file mode 100644
index 000000000..9f491e699
--- /dev/null
+++ b/bsd/man/man2/kevent.2
@@ -0,0 +1 @@
+.so man2/kqueue.2
diff --git a/bsd/man/man2/kill.2 b/bsd/man/man2/kill.2
index 6b7157f35..92b09e4c2 100644
--- a/bsd/man/man2/kill.2
+++ b/bsd/man/man2/kill.2
@@ -42,22 +42,25 @@
 .Sh SYNOPSIS
 .Fd #include <signal.h>
 .Ft int
-.Fn kill "pid_t pid" "int sig"
+.Fo kill
+.Fa "pid_t pid"
+.Fa "int sig"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn kill
-function sends the signal given by
+function sends the signal specified by
 .Fa sig
 to
 .Fa pid ,
 a
 process or a group of processes.
+Typically,
 .Fa Sig
-may be one of the signals specified in
-.Xr sigaction 2
-or it may be 0, in which case
-error checking is performed but no
-signal is actually sent. 
+will be one of the signals specified in
+.Xr sigaction 2 .
+A value of 0, however,
+will cause error checking to be performed (with no signal being sent). 
 This can be used to check the validity of
 .Fa pid .
 .Pp
@@ -67,8 +70,8 @@ by
 the real or effective user ID of the receiving process must match
 that of the sending process or the user must have appropriate privileges
 (such as given by a set-user-ID program or the user is the super-user).
-A single exception is the signal SIGCONT, which may always be sent
-to any descendant of the current process.
+A single exception is the signal SIGCONT,
+which may always be sent to any descendant of the current process.
 .Bl -tag -width Ds
 .It \&If Fa pid No \&is greater than zero :
 .Fa Sig
@@ -77,16 +80,17 @@ is sent to the process whose ID is equal to
 .It \&If Fa pid No \&is zero :
 .Fa Sig
 is sent to all processes whose group ID is equal
-to the process group ID of the sender, and for which the
-process has permission;
+to the process group ID of the sender,
+and for which the process has permission;
 this is a variant of
 .Xr killpg 2 .
 .It \&If Fa pid No \&is -1 :
 If the user has super-user privileges,
 the signal is sent to all processes excluding
 system processes and the process sending the signal.
-If the user is not the super user, the signal is sent to all processes
-with the same uid as the user excluding the process sending the signal.
+If the user is not the super user,
+the signal is sent to all processes with the same uid as the user,
+excluding the process sending the signal.
 No error is returned if any process could be signaled.
 .El
 .Pp
@@ -105,24 +109,29 @@ is set to indicate the error.
 .Fn Kill
 will fail and no signal will be sent if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EINVAL
 .Fa Sig
-is not a valid signal number.
-.It Bq Er ESRCH
-No process can be found corresponding to that specified by
-.Fa pid .
-.It Bq Er ESRCH
-The process id was given as 0
-but the sending process does not have a process group.
+is not a valid, supported signal number.
+.\" ==========
 .It Bq Er EPERM
 The sending process is not the super-user and its effective
 user id does not match the effective user-id of the receiving process.
 When signaling a process group, this error is returned if any members
 of the group could not be signaled.
+.\" ==========
+.It Bq Er ESRCH
+No process or process group can be found
+corresponding to that specified by
+.Fa pid .
+.\" ==========
+.It Bq Er ESRCH
+The process id was given as 0,
+but the sending process does not have a process group.
 .El
 .Sh SEE ALSO
-.Xr getpid 2 ,
 .Xr getpgrp 2 ,
+.Xr getpid 2 ,
 .Xr killpg 2 ,
 .Xr sigaction 2
 .Sh STANDARDS
diff --git a/bsd/man/man2/kqueue.2 b/bsd/man/man2/kqueue.2
index 7006995f1..684ee806c 100644
--- a/bsd/man/man2/kqueue.2
+++ b/bsd/man/man2/kqueue.2
@@ -189,6 +189,12 @@ will not return it.  The filter itself is not disabled.
 Removes the event from the kqueue.  Events which are attached to
 file descriptors are automatically deleted on the last close of
 the descriptor.
+.It EV_RECEIPT
+This flag is useful for making bulk changes to a kqueue without draining any
+pending events. When passed as input, it forces EV_ERROR to always be returned.
+When a filter is successfully added. The
+.Va data
+field will be zero.
 .It EV_ONESHOT
 Causes the event to return only the first occurrence of the filter
 being triggered.  After the user retrieves the event from the kqueue,
@@ -343,28 +349,25 @@ in
 and returns when the process performs one or more of the requested events.
 If a process can normally see another process, it can attach an event to it.
 The events to monitor are:
-.Bl -tag -width XXNOTE_TRACKERR
+.Bl -tag -width 
 .It NOTE_EXIT
 The process has exited.
 .It NOTE_FORK
-The process has called
-.Fn fork .
+The process created a child process via
+.Xr fork 2
+or similar call.
 .It NOTE_EXEC
-The process has executed a new process via
+The process executed a new process via
 .Xr execve 2
 or similar call.
-.It NOTE_TRACK
-Follow a process across
-.Fn fork
-calls.  The parent process will return with NOTE_TRACK set in the
-.Va fflags
-field, while the child process will return with NOTE_CHILD set in
-.Va fflags
-and the parent PID in
-.Va data .
-.It NOTE_TRACKERR
-This flag is returned if the system was unable to attach an event to
-the child process, usually due to resource limitations.
+.It NOTE_SIGNAL
+The process was sent a signal. Status can be checked via
+.Xr waitpid 2
+or similar call.
+.It NOTE_REAP
+The process was reaped by the parent via
+.Xr wait 2
+or similar call.
 .El
 .Pp
 On return,
diff --git a/bsd/man/man2/ktrace.2 b/bsd/man/man2/ktrace.2
deleted file mode 100644
index 3fdc9851a..000000000
--- a/bsd/man/man2/ktrace.2
+++ /dev/null
@@ -1,168 +0,0 @@
-.\" Copyright (c) 1993
-.\"	The Regents of the University of California.  All rights reserved.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\" 3. All advertising materials mentioning features or use of this software
-.\"    must display the following acknowledgement:
-.\"	This product includes software developed by the University of
-.\"	California, Berkeley and its contributors.
-.\" 4. Neither the name of the University nor the names of its contributors
-.\"    may be used to endorse or promote products derived from this software
-.\"    without specific prior written permission.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
-.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
-.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-.\" SUCH DAMAGE.
-.\"
-.\"     @(#)ktrace.2	8.1 (Berkeley) 6/4/93
-.\" $FreeBSD: src/lib/libc/sys/ktrace.2,v 1.9.2.7 2001/12/14 18:34:01 ru Exp $
-.\"
-.Dd June 4, 1993
-.Dt KTRACE 2
-.Os
-.Sh NAME
-.Nm ktrace
-.Nd process tracing
-.Sh LIBRARY
-.Lb libc
-.Sh SYNOPSIS
-.In sys/param.h
-.In sys/time.h
-.In sys/uio.h
-.In sys/ktrace.h
-.Ft int
-.Fn ktrace "const char *tracefile" "int ops" "int trpoints" "int pid"
-.Sh DESCRIPTION
-The
-.Fn ktrace
-function enables or disables tracing of one or more processes.
-Users may only trace their own processes.
-Only the super-user can trace setuid or setgid programs.
-.Pp
-The
-.Fa tracefile
-gives the pathname of the file to be used for tracing.
-The file must exist and be a regular file writable by the calling process.
-All trace records are always appended to the file,
-so the file must be truncated to zero length to discard
-previous trace data.
-If tracing points are being disabled (see KTROP_CLEAR below),
-.Fa tracefile
-may be NULL.
-.Pp
-The
-.Fa ops
-parameter specifies the requested ktrace operation.
-The defined operations are:
-.Bl -column KTRFLAG_DESCENDXXX -offset indent
-.It "KTROP_SET	Enable trace points specified in"
-.Fa trpoints .
-.It "KTROP_CLEAR	Disable trace points specified in
-.Fa trpoints .
-.It "KTROP_CLEARFILE	Stop all tracing."
-.It "KTRFLAG_DESCEND	The tracing change should apply to the"
-specified process and all its current children.
-.El
-.Pp
-The
-.Fa trpoints
-parameter specifies the trace points of interest.
-The defined trace points are:
-.Bl -column KTRFAC_SYSCALLXXX -offset indent
-.It "KTRFAC_SYSCALL	Trace system calls."
-.It "KTRFAC_SYSRET	Trace return values from system calls."
-.It "KTRFAC_NAMEI	Trace name lookup operations."
-.It "KTRFAC_GENIO	Trace all I/O (note that this option can"
-generate much output).
-.It "KTRFAC_PSIG	Trace posted signals."
-.It "KTRFAC_CSW	Trace context switch points."
-.It "KTRFAC_INHERIT	Inherit tracing to future children."
-.El
-.Pp
-Each tracing event outputs a record composed of a generic header
-followed by a trace point specific structure.
-The generic header is:
-.Bd -literal
-struct ktr_header {
-	int	ktr_len;		/* length of buf */
-	short	ktr_type;		/* trace record type */
-	pid_t	ktr_pid;		/* process id */
-	char	ktr_comm[MAXCOMLEN+1];	/* command name */
-	struct	timeval ktr_time;	/* timestamp */
-	caddr_t	ktr_buf;
-};
-.Ed
-.Pp
-The
-.Va ktr_len
-field specifies the length of the
-.Va ktr_type
-data that follows this header.
-The
-.Va ktr_pid
-and
-.Va ktr_comm
-fields specify the process and command generating the record.
-The
-.Va ktr_time
-field gives the time (with microsecond resolution)
-that the record was generated.
-The
-.Va ktr_buf
-is an internal kernel pointer and is not useful.
-.Pp
-The generic header is followed by
-.Va ktr_len
-bytes of a
-.Va ktr_type
-record.
-The type specific records are defined in the
-.Aq Pa sys/ktrace.h
-include file.
-.Sh RETURN VALUES
-.Rv -std ktrace
-.Sh ERRORS
-.Fn Ktrace
-will fail if:
-.Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 255 characters,
-or an entire path name exceeded 1023 characters.
-.It Bq Er ENOENT
-The named tracefile does not exist.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
-.It Bq Er ENOSYS
-The kernel was not compiled with
-.Nm
-support.
-.El
-.Sh SEE ALSO
-.Xr kdump 1 ,
-.Xr ktrace 1
-.Sh HISTORY
-A
-.Fn ktrace
-function call first appeared in
-.Bx 4.4 .
diff --git a/bsd/man/man2/link.2 b/bsd/man/man2/link.2
index 2a6f8b66c..6392e7cea 100644
--- a/bsd/man/man2/link.2
+++ b/bsd/man/man2/link.2
@@ -42,42 +42,44 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn link "const char *name1" "const char *name2"
+.Fo link
+.Fa "const char *path1"
+.Fa "const char *path2"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn link
 function call
 atomically creates the specified directory entry (hard link)
-.Fa name2
+.Fa path2
 with the attributes of the underlying object pointed at by
-.Fa name1
-If the link is successful: the link count of the underlying object
-is incremented;
-.Fa name1
+.Fa path1 .
+If the link is successful,
+the link count of the underlying object is incremented;
+.Fa path1
 and
-.Fa name2
+.Fa path2
 share equal access and rights
 to the
 underlying object.
 .Pp
 If
-.Fa name1
+.Fa path1
 is removed, the file
-.Fa name2
+.Fa path2
 is not deleted and the link count of the
 underlying object is
 decremented.
 .Pp
-.Fa Name1
-must exist for the hard link to
-succeed and
-both
-.Fa name1
+In order for the system call to succeed,
+.Fa path1
+must exist and both
+.Fa path1
 and
-.Fa name2
+.Fa path2
 must be in the same file system.
-As mandated by POSIX.1
-.Fa name1
+As mandated by POSIX.1,
+.Fa path1
 may not be a directory.
 .Sh RETURN VALUES
 Upon successful completion, a value of 0 is returned.  Otherwise,
@@ -88,60 +90,81 @@ is set to indicate the error.
 .Fn Link
 will fail and no link will be created if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of either path prefix is not a directory.
+.\" ==========
+.It Bq Er EACCES
+A component of either path prefix denies search permission.
+.\" ==========
+.It Bq Er EACCES
+The requested link requires writing in a directory with a mode
+that denies write permission.
+.\" ==========
+.It Bq Er EACCES
+The current process cannot access the existing file.
+.\" ==========
+.It Bq Er EDQUOT
+The directory in which the entry for the new link
+is being placed cannot be extended because the
+user's quota of disk blocks on the file system
+containing the directory has been exhausted.
+.\" ==========
+.It Bq Er EEXIST
+The link named by
+.Fa path2
+already exists.
+.\" ==========
+.It Bq Er EFAULT
+One of the pathnames specified
+is outside the process's allocated address space.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurs while reading from or writing to 
+the file system to make the directory entry.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating one of the pathnames.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er EMLINK
+The file already has {LINK_MAX} links.
+.\" ==========
 .It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
+A component of a pathname exceeds 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ==========
 .It Bq Er ENOENT
 A component of either path prefix does not exist.
-.It Bq Er EACCES
-A component of either path prefix denies search permission.
-.It Bq Er EACCES
-The requested link requires writing in a directory with a mode
-that denies write permission.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating one of the pathnames.
+.\" ==========
 .It Bq Er ENOENT
 The file named by
-.Fa name1
+.Fa path1
 does not exist.
-.It Bq Er EEXIST
-The link named by
-.Fa name2
-does exist.
+.\" ==========
+.It Bq Er ENOSPC
+The directory in which the entry for the new link is being placed
+cannot be extended because there is no space left on the file
+system containing the directory.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of either path prefix is not a directory.
+.\" ==========
 .It Bq Er EPERM
 The file named by
-.Fa name1
+.Fa path1
 is a directory.
+.\" ==========
+.It Bq Er EROFS
+The requested link requires writing in a directory
+on a read-only file system.
+.\" ==========
 .It Bq Er EXDEV
 The link named by
-.Fa name2
+.Fa path2
 and the file named by
-.Fa name1
+.Fa path1
 are on different file systems.
-.It Bq Er ENOSPC
-The directory in which the entry for the new link is being placed
-cannot be extended because there is no space left on the file
-system containing the directory.
-.ne 3v
-.It Bq Er EDQUOT
-The directory in which the entry for the new link
-is being placed cannot be extended because the
-user's quota of disk blocks on the file system
-containing the directory has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to 
-the file system to make the directory entry.
-.It Bq Er EROFS
-The requested link requires writing in a directory on a read-only file
-system.
-.It Bq Er EFAULT
-One of the pathnames specified
-is outside the process's allocated address space.
 .El
 .Sh SEE ALSO
 .Xr symlink 2 ,
diff --git a/bsd/man/man2/listen.2 b/bsd/man/man2/listen.2
index a44a951bd..f62a62c1e 100644
--- a/bsd/man/man2/listen.2
+++ b/bsd/man/man2/listen.2
@@ -43,15 +43,18 @@
 .Sh SYNOPSIS
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn listen "int s" "int backlog"
+.Fo listen
+.Fa "int socket"
+.Fa "int backlog"
+.Fc
 .Sh DESCRIPTION
-To accept connections, a socket
-is first created with
-.Xr socket 2 ,
-a willingness to accept incoming connections and
+Creation of socket-based connections requires several operations.
+First, a socket is created with
+.Xr socket 2 .
+Next, a willingness to accept incoming connections and
 a queue limit for incoming connections are specified with
-.Fn listen ,
-and then the connections are
+.Fn listen .
+Finally, the connections are
 accepted with
 .Xr accept 2 .
 The
@@ -63,28 +66,42 @@ or
 .Pp
 The
 .Fa backlog
-parameter defines the maximum length the queue of
-pending connections may grow to.
+parameter defines the maximum length
+for the queue of pending connections.
 If a connection
-request arrives with the queue full the client may
-receive an error with an indication of
-.Er ECONNREFUSED ,
-or, if the underlying protocol supports retransmission,
+request arrives with the queue full,
+the client may receive an error with an indication of
+.Er ECONNREFUSED .
+Alternatively, if the underlying protocol supports retransmission,
 the request may be ignored so that retries may succeed.
 .Sh RETURN VALUES
-A 0 return value indicates success; -1 indicates an error.
+.Rv -std listen
 .Sh ERRORS
 .Fn Listen
 will fail if:
 .Bl -tag -width Er
+.\" ==========
+.It Bq Er EACCES
+The current process has insufficient privileges.
+.\" ==========
 .It Bq Er EBADF
 The argument
-.Fa s
-is not a valid descriptor.
+.Fa socket
+is not a valid file descriptor.
+.\" ==========
+.It Bq Er EDESTADDRREQ
+The socket is not bound to a local address
+and the protocol does not support listening on an unbound socket.
+.\" ==========
+.It Bq Er EINVAL
+.Fa socket
+is already connected.
+.\" ==========
 .It Bq Er ENOTSOCK
 The argument
-.Fa s
-is not a socket.
+.Fa socket
+does not reference a socket.
+.\" ==========
 .It Bq Er EOPNOTSUPP
 The socket is not of a type that supports the operation
 .Fn listen .
diff --git a/bsd/man/man2/listxattr.2 b/bsd/man/man2/listxattr.2
index b466439be..499def160 100644
--- a/bsd/man/man2/listxattr.2
+++ b/bsd/man/man2/listxattr.2
@@ -143,9 +143,9 @@ An I/O error occurred.
 does not make sense.
 .El
 .Sh SEE ALSO
-.Xr setxattr 2 ,
 .Xr getxattr 2 ,
-.Xr removexattr 2
+.Xr removexattr 2 ,
+.Xr setxattr 2
 .Sh HISTORY
 .Fn listxattr
 and
diff --git a/bsd/man/man2/lseek.2 b/bsd/man/man2/lseek.2
index 09374f764..0487277c6 100644
--- a/bsd/man/man2/lseek.2
+++ b/bsd/man/man2/lseek.2
@@ -42,7 +42,11 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft off_t
-.Fn lseek "int fildes" "off_t offset" "int whence"
+.Fo lseek
+.Fa "int fildes"
+.Fa "off_t offset"
+.Fa "int whence"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn lseek
@@ -50,9 +54,9 @@ function repositions the offset of the file descriptor
 .Fa fildes
 to the
 argument
-.Fa offset
+.Fa offset ,
 according to the directive
-.Fa whence.
+.Fa whence .
 The argument
 .Fa fildes
 must be an open
@@ -112,15 +116,29 @@ the error.
 .Fn Lseek
 will fail and the file pointer will remain unchanged if:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EBADF
 .Em Fildes
 is not an open file descriptor.
-.It Bq Er ESPIPE
-.Em Fildes
-is associated with a pipe, socket, or FIFO.
+.\" ==========
 .It Bq Er EINVAL
 .Fa Whence
 is not a proper value.
+.\" ==========
+.It Bq Er EINVAL
+The seek location (calculated from
+.Fa offset
+and
+.Fa whence )
+is negative.
+.\" ==========
+.It Bq Er EOVERFLOW
+The seek location is too large to be stored
+in an object of type off_t.
+.\" ==========
+.It Bq Er ESPIPE
+.Em Fildes
+is associated with a pipe, socket, or FIFO.
 .El
 .Sh SEE ALSO
 .Xr dup 2 ,
diff --git a/bsd/man/man2/lstat64.2 b/bsd/man/man2/lstat64.2
new file mode 100644
index 000000000..4fe4fb441
--- /dev/null
+++ b/bsd/man/man2/lstat64.2
@@ -0,0 +1,2 @@
+.so man2/stat.2
+
diff --git a/bsd/man/man2/madvise.2 b/bsd/man/man2/madvise.2
index 1c85fce72..b285f4f2d 100644
--- a/bsd/man/man2/madvise.2
+++ b/bsd/man/man2/madvise.2
@@ -37,25 +37,41 @@
 .Dt MADVISE 2
 .Os
 .Sh NAME
-.Nm madvise
+.Nm madvise ,
+.Nm posix_madvise
 .Nd give advice about use of memory
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn madvise "caddr_t addr" "size_t len" "int behav"
+.Fo madvise
+.Fa "void *addr"
+.Fa "size_t len"
+.Fa "int advice"
+.Fc
 .Ft int
-.Fn posix_madvise "caddr_t addr" "size_t len" "int behav"
+.Fo posix_madvise
+.Fa "void *addr"
+.Fa "size_t len"
+.Fa "int advice"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn madvise
-system call allows a process that has knowledge of its memory behavior to describe it to the system. The advice passed in may be used by the system to alter its virtual memory paging strategy. This advice may improve application and system performance. The behavior specified in
-.Fa behav
+system call allows a process that has knowledge of its memory behavior
+to describe it to the system.
+The advice passed in may be used by the system
+to alter its virtual memory paging strategy.
+This advice may improve application and system performance.
+The behavior specified in
+.Fa advice
 can only be one of the following values:
 .Pp
 .Bl -tag -width MADV_SEQUENTIAL
 .It Dv  MADV_NORMAL
-Indicates that the application has no advice to give on its behavior in the specifed address range. This is the system default behavior. This is used with 
+Indicates that the application has no advice to give on its behavior
+in the specified address range.
+This is the system default behavior.
+This is used with 
 .Fn madvise
 system call.
 .It Dv POSIX_MADV_NORMAL
@@ -63,7 +79,9 @@ Same as MADV_NORMAL but used with
 .Fn posix_madvise
 system call.
 .It Dv MADV_SEQUENTIAL
-Indicates that the application expects to access this address range in a sequential manner. This is used with
+Indicates that the application expects to access this address range
+in a sequential manner.
+This is used with
 .Fn madvise
 system call.
 .It Dv POSIX_MADV_SEQUENTIAL
@@ -71,7 +89,9 @@ Same as MADV_SEQUENTIAL but used with
 .Fn posix_madvise
 system call.
 .It Dv MADV_RANDOM
-Indicates that the application expects to access this address range in a random manner. This is used with
+Indicates that the application expects to access this address range
+in a random manner.
+This is used with
 .Fn madvise
 system call.
 .It Dv POSIX_MADV_RANDOM
@@ -79,7 +99,8 @@ Same as MADV_RANDOM but used with
 .Fn posix_madvise
 system call.
 .It Dv MADV_WILLNEED
-Indicates that the application expects to access this address range soon. This is used with
+Indicates that the application expects to access this address range soon.
+This is used with
 .Fn madvise
 system call.
 .It Dv POSIX_MADV_WILLNEED
@@ -87,7 +108,9 @@ Same as MADV_WILLNEED but used with
 .Fn posix_madvise
 system call.
 .It Dv MADV_DONTNEED
-Indicates that the application is not expecting to access this address range soon. This is used with
+Indicates that the application is not expecting
+to access this address range soon.
+This is used with
 .Fn madvise
 system call.
 .It Dv POSIX_MADV_DONTNEED
@@ -95,7 +118,10 @@ Same as MADV_DONTNEED but used with
 .Fn posix_madvise
 system call.
 .It Dv MADV_FREE
-Indicates that the application will not need the information contained in this address range so the pages may be reused right away. The address range will remain valid. This is used with
+Indicates that the application will not need the information contained
+in this address range, so the pages may be reused right away.
+The address range will remain valid.
+This is used with
 .Fn madvise
 system call.
 .El
@@ -105,7 +131,7 @@ The
 behaves same as 
 .Fn madvise
 except that it uses values with POSIX_ prefix for the 
-.Fa behav
+.Fa advice
 system call argument.
 .Sh RETURN VALUES
 Upon successful completion,
@@ -117,29 +143,60 @@ is set to indicate the error.
 .Fn madvise
 fails if one or more of the following are true:
 .Bl -tag -width Er
+.\" ==========
 .It Bq Er EINVAL
 The value of 
-.Fa behav 
+.Fa advice 
 is incorrect.
+.\" ==========
+.It Bq Er EINVAL
+The address range includes unallocated regions.
+.\" ==========
 .It Bq Er ENOMEM
 The virtual address range specified by the 
 .Fa addr
 and
 .Fa len
 are outside the range allowed for the address space.
-.It Bq Er EINVAL
-The address range includes unallocated regions.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+.Ft int
+.br
+.Fo madvise
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fa "int advice"
+.Fc ;
+.Pp
+.Ft int
+.br
+.Fo posix_madvise
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fa "int advice"
+.Fc ;
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+The type of
+.Fa addr
+has changed.
 .Sh SEE ALSO
 .Xr mincore 2 ,
 .Xr minherit 2 ,
 .Xr mprotect 2 , 
 .Xr msync 2 ,
-.Xr munmap 2
+.Xr munmap 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Nm madvise
 function first appeared in 4.4BSD.
 The
 .Nm posix_madvise
-function is part of IEEE 1003.1-2001 and was first implemented in Mac OS X 10.2.
+function is part of IEEE 1003.1-2001
+and was first implemented in Mac OS X 10.2.
diff --git a/bsd/man/man2/minherit.2 b/bsd/man/man2/minherit.2
index 01e4ffbbd..19961fa75 100644
--- a/bsd/man/man2/minherit.2
+++ b/bsd/man/man2/minherit.2
@@ -1,4 +1,4 @@
-.\"	$Id: minherit.2,v 1.2.2684.1 2005/06/24 01:47:10 lindak Exp $
+.\"	$Id: minherit.2,v 1.3 2005/07/29 21:49:48 lindak Exp $
 .\"
 .\" Copyright (c) 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
@@ -43,7 +43,11 @@
 .Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn minherit "caddr_t addr" "size_t len" "int inherit"
+.Fo minherit
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fa "int inherit"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn minherit
@@ -51,9 +55,33 @@ system call
 changes the specified pages to have the inheritance characteristic
 .Fa inherit ,
 which can be set to VM_INHERIT_NONE, VM_INHERIT_COPY, or VM_INHERIT_SHARE.
-Not all implementations will guarantee that the inheritance characteristic
-can be set on a page basis;
+Not all implementations will guarantee
+that the inheritance characteristic can be set on a page basis;
 the granularity of changes may be as large as an entire region.
+.Sh RETURN VALUES
+.Rv -std minherit
+.Sh ERRORS
+The
+.Fn minherit
+system call will fail if:
+.Bl -tag -width Er
+.\" ========
+.It Bq Er EINVAL
+The virtual address range specified by the
+.Fa addr
+and
+.Fa len
+arguments is not valid.
+.\" ========
+.It Bq Er EACCES
+The flags specified by the
+.Fa inherit
+argument were not valid for the pages specified by the
+.Fa addr
+and
+.Fa len
+arguments.
+.El
 .Sh SEE ALSO
 .Xr madvise 2 , 
 .Xr mincore 2 , 
diff --git a/bsd/man/man2/mkdir.2 b/bsd/man/man2/mkdir.2
index dd305fb0d..4df68194d 100644
--- a/bsd/man/man2/mkdir.2
+++ b/bsd/man/man2/mkdir.2
@@ -40,10 +40,12 @@
 .Nm mkdir
 .Nd make a directory file
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/stat.h>
 .Ft int
-.Fn mkdir "const char *path" "mode_t mode"
+.Fo mkdir
+.Fa "const char *path"
+.Fa "mode_t mode"
+.Fc
 .Sh DESCRIPTION
 The directory
 .Fa path
@@ -64,49 +66,79 @@ indicates an error, and an error code is stored in
 .Fn Mkdir
 will fail and no directory will be created if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-A component of the path prefix does not exist.
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EROFS
-The named file resides on a read-only file system.
-.It Bq Er EEXIST
-The named file exists.
-.It Bq Er ENOSPC
-The new directory cannot be created because there is no space left
-on the file system that will contain the directory.
-.It Bq Er ENOSPC
-There are no free inodes on the file system on which the
-directory is being created.
+.\" ==========
+.It Bq Er EACCES
+Write permission is denied for the parent directory.
+.\" ==========
 .It Bq Er EDQUOT
 The new directory cannot be created because the user's
 quota of disk blocks on the file system that will
 contain the directory has been exhausted.
+.\" ==========
 .It Bq Er EDQUOT
 The user's quota of inodes on the file system on
 which the directory is being created has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while making the directory entry or allocating the inode.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er EEXIST
+The named file exists.
+.\" ==========
 .It Bq Er EFAULT
 .Fa Path
 points outside the process's allocated address space.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while making the directory entry
+or allocating the inode.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er EMLINK
+The parent directory already has {LINK_MAX} links.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded 
+.Dv {NAME_MAX}
+characters, or an entire path name exceeded 
+.Dv {PATH_MAX}
+characters.
+.\" ==========
+.It Bq Er ENOENT
+A component of the path prefix does not exist
+or path is an empty string.
+.It Bq Er ENOSPC
+The new directory cannot be created because there is no space left
+on the file system that would contain it.
+.\" ==========
+.It Bq Er ENOSPC
+There are no free inodes on the file system on which the
+directory is being created.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EROFS
+The parent directory resides on a read-only file system.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/stat.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr chmod 2 ,
 .Xr stat 2 ,
-.Xr umask 2
+.Xr umask 2 ,
+.Xr compat 5
 .Sh STANDARDS
 The 
 .Fn mkdir
diff --git a/bsd/man/man2/mknod.2 b/bsd/man/man2/mknod.2
index e92dd449f..017bb2508 100644
--- a/bsd/man/man2/mknod.2
+++ b/bsd/man/man2/mknod.2
@@ -40,18 +40,22 @@
 .Nm mknod
 .Nd make a special file node
 .Sh SYNOPSIS
-.Fd #include <unistd.h>
+.Fd #include <sys/stat.h>
 .Ft int
-.Fn mknod "const char *path" "mode_t mode" "dev_t dev"
+.Fo mknod
+.Fa "const char *path"
+.Fa "mode_t mode"
+.Fa "dev_t dev"
+.Fc
 .Sh DESCRIPTION
 The device special file
 .Fa path
 is created with the major and minor
 device numbers extracted from
-.Fa mode.
+.Fa mode .
 The access permissions of
 .Fa path
-are descendant from the
+are constrained by the
 .Xr umask 2
 of the parent process.
 .Pp
@@ -59,7 +63,7 @@ If
 .Fa mode
 indicates a block or character special file,
 .Fa dev
-is a configuration dependent specification of a character or block
+is a configuration-dependent specification of a character or block
 I/O device and the superblock of the device.  If
 .Fa mode
 does not indicate a block special or character special device,
@@ -69,7 +73,7 @@ is ignored.
 .Fn Mknod
 requires super-user privileges.
 .Sh RETURN VALUES
-Upon successful completion a value of 0 is returned.
+Upon successful completion, a value of 0 is returned.
 Otherwise, a value of -1 is returned and
 .Va errno
 is set to indicate the error.
@@ -77,51 +81,79 @@ is set to indicate the error.
 .Fn Mknod
 will fail and the file will be not created if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-A component of the path prefix does not exist.
+.\" ==========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EPERM
-The process's effective user ID is not super-user.
-.It Bq Er EIO
-An I/O error occurred while making the directory entry or allocating the inode.
-.It Bq Er ENOSPC
-The directory in which the entry for the new node is being placed
-cannot be extended because there is no space left on the file
-system containing the directory.
-.It Bq Er ENOSPC
-There are no free inodes on the file system on which the
-node is being created.
+.\" ==========
+.It Bq Er EACCES
+Write permission is denied for the parent directory.
+.\" ==========
 .It Bq Er EDQUOT
 The directory in which the entry for the new node
-is being placed cannot be extended because the
-user's quota of disk blocks on the file system
+is being placed cannot be extended;
+the user's quota of disk blocks on the file system
 containing the directory has been exhausted.
+.\" ==========
 .It Bq Er EDQUOT
-The user's quota of inodes on the file system on
-which the node is being created has been exhausted.
-.It Bq Er EROFS
-The named file resides on a read-only file system.
+The user's quota of inodes for the file system
+on which the node is being created has been exhausted.
+.\" ==========
 .It Bq Er EEXIST
 The named file exists.
+.\" ==========
 .It Bq Er EFAULT
 .Fa Path
 points outside the process's allocated address space.
+.\" ==========
+.It Bq Er EINVAL
+One or more of the arguments is invalid.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurs while making the directory entry
+or allocating the inode.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds 
+.Dv {NAME_MAX}
+characters or an entire path name exceeds 
+.Dv {PATH_MAX}
+characters.
+.\" ==========
+.It Bq Er ENOENT
+A component of the path prefix does not exist
+or path is an empty string.
+.\" ==========
+.It Bq Er ENOSPC
+The directory in which the entry for the new node is being placed
+cannot be extended,
+because there is no space left on the file system containing the directory.
+.\" ==========
+.It Bq Er ENOSPC
+There are no free inodes on the file system
+on which the node is being created.
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er EPERM
+The process's effective user ID is not super-user.
+.\" ==========
+.It Bq Er EROFS
+The created node would reside on a read-only file system.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <unistd.h>
+.Pp
+The include file has changed.
 .Sh SEE ALSO
 .Xr chmod 2 ,
 .Xr stat 2 ,
-.Xr umask 2
+.Xr umask 2 ,
+.Xr compat 5
 .Sh HISTORY
 A
 .Fn mknod
diff --git a/bsd/man/man2/mlock.2 b/bsd/man/man2/mlock.2
index be33648c5..94584a7b0 100644
--- a/bsd/man/man2/mlock.2
+++ b/bsd/man/man2/mlock.2
@@ -41,60 +41,67 @@
 .Nm munlock
 .Nd lock (unlock) physical pages in memory
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn mlock "caddr_t addr" "size_t len"
+.Fo mlock
+.Fa "const void *addr"
+.Fa "size_t len"
+.Fc
 .Ft int
-.Fn munlock "caddr_t addr" "size_t len"
+.Fo munlock
+.Fa "const void *addr"
+.Fa "size_t len"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm mlock
 system call
-locks into memory the physical pages associated with the virtual address
-range starting at
+locks a set of physical pages into memory.
+The pages are associated with a virtual address range
+that starts at
 .Fa addr
-for
+and extends for
 .Fa len
 bytes.
 The
 .Nm munlock
-call unlocks pages previously locked by one or more
+call unlocks pages that were previously locked by one or more
 .Nm mlock
 calls.
-For both, the
+For both calls, the
 .Fa addr
 parameter should be aligned to a multiple of the page size.
 If the
 .Fa len
-parameter is not a multiple of the page size, it will be rounded up
-to be so.
+parameter is not a multiple of the page size,
+it will be rounded up to be so.
 The entire range must be allocated.
 .Pp
 After an
 .Nm mlock
 call, the indicated pages will cause neither a non-resident page
 nor address-translation fault until they are unlocked.
-They may still cause protection-violation faults or TLB-miss faults on
-architectures with software-managed TLBs.
-The physical pages remain in memory until all locked mappings for the pages
-are removed.
-Multiple processes may have the same physical pages locked via their own
-virtual address mappings.
-A single process may likewise have pages multiply-locked via different virtual
-mappings of the same pages or via nested
+They may still cause protection-violation faults or TLB-miss faults
+on architectures with software-managed TLBs.
+The physical pages remain in memory until all locked mappings
+for the pages are removed.
+.Pp
+Multiple processes may have the same physical pages locked
+via their own virtual address mappings.
+Similarly, a single process may have pages multiply-locked
+via different virtual mappings of the same pages or via nested
 .Nm mlock
 calls on the same address range.
 Unlocking is performed explicitly by
 .Nm munlock
 or implicitly by a call to
-.Nm munmap
+.Nm munmap ,
 which deallocates the unmapped address range.
 Locked mappings are not inherited by the child process after a
 .Xr fork 2 .
 .Pp
-Since physical memory is a potentially scarce resource, processes are
-limited in how much they can lock down.
+Because physical memory is a potentially scarce resource,
+processes are limited in how much memory they can lock down.
 A single process can
 .Nm mlock
 the minimum of
@@ -103,35 +110,75 @@ the per-process
 .Li RLIMIT_MEMLOCK
 resource limit.
 .Sh RETURN VALUES
-A return value of 0 indicates that the call
-succeeded and all pages in the range have either been locked or unlocked.
-A return value of -1 indicates an error occurred and the locked
-status of all pages in the range remains unchanged.
+A return value of 0 indicates that the call succeeded
+and all pages in the range have either been locked or unlocked,
+as requested.
+A return value of -1 indicates an error occurred
+and the locked status of all pages in the range remains unchanged.
 In this case, the global location
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
 .Fn Mlock
+and 
+.Fn munlock
 will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EINVAL
-The address given is not page aligned or the length is negative.
+The address given is not page-aligned or the length is negative.
+.\" ===========
+.It Bq Er ENOMEM
+Part or all of the specified address range
+is not mapped to the process.
+.El
+.Pp
+.Fn Mlock
+will fail if:
+.Bl -tag -width Er
+.\" ===========
 .It Bq Er EAGAIN
 Locking the indicated range would exceed either the system or per-process
 limit for locked memory.
+.\" ===========
 .It Bq Er ENOMEM
 Some portion of the indicated address range is not allocated.
 There was an error faulting/mapping a page.
 .El
+.Pp
 .Fn Munlock
 will fail if:
 .Bl -tag -width Er
-.It Bq Er EINVAL
-The address given is not page aligned or the length is negative.
+.\" ===========
 .It Bq Er ENOMEM
 Some portion of the indicated address range is not allocated.
 Some portion of the indicated address range is not locked.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Pp
+.Ft int
+.br
+.Fo mlock
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fc ;
+.Pp
+.Ft int
+.br
+.Fo munlock
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fc ;
+.Pp
+The variable type of
+.Fa addr
+has changed.
 .Sh "SEE ALSO"
 .Xr fork 2 ,
 .Xr mincore 2 ,
@@ -139,7 +186,8 @@ Some portion of the indicated address range is not locked.
 .Xr mmap 2 ,
 .Xr munmap 2 ,
 .Xr setrlimit 2 ,
-.Xr getpagesize 3
+.Xr getpagesize 3 ,
+.Xr compat 5
 .Sh BUGS
 Unlike The Sun implementation, multiple
 .Nm mlock
diff --git a/bsd/man/man2/mmap.2 b/bsd/man/man2/mmap.2
index ba4541c78..648b09c7a 100644
--- a/bsd/man/man2/mmap.2
+++ b/bsd/man/man2/mmap.2
@@ -40,10 +40,16 @@
 .Nm mmap
 .Nd map files or devices into memory
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft void *
-.Fn mmap "void *addr" "size_t len" "int prot" "int flags" "int fd" "off_t offset"
+.Fo mmap
+.Fa "void *addr"
+.Fa "size_t len"
+.Fa "int prot"
+.Fa "int flags"
+.Fa "int fildes"
+.Fa "off_t offset"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm mmap
@@ -52,7 +58,7 @@ function causes the pages starting at
 and continuing for at most
 .Fa len
 bytes to be mapped from the object described by
-.Fa fd ,
+.Fa fildes ,
 starting at byte offset
 .Fa offset .
 If
@@ -92,23 +98,35 @@ Pages may be written.
 .Pp
 The
 .Fa flags
-parameter specifies the type of the mapped object, mapping options and
-whether modifications made to the mapped copy of the page are private
-to the process or are to be shared with other references.
-Sharing, mapping type and options are specified in the
+parameter specifies the type of the mapped object, mapping options,
+and whether modifications made to the mapped copy of the page
+are private to the process (copy-on-write)
+or are to be shared with other references.
+Sharing, mapping type, and options are specified in the
 .Fa flags
 argument by
 .Em or Ns 'ing
 the following values:
 .Pp
-.Bl -tag -width MAP_FIXEDX
+.Bl -tag -width MAP_HASSEMAPHOREX
 .It Dv MAP_ANON
 Map anonymous memory not associated with any specific file.
-The file descriptor used for creating
+Mac OS X specific: the file descriptor used for creating
 .Dv MAP_ANON
-regions is used only for
-naming, and may be specified as \-1 if no name is associated with the
-region.
+regions can be used to pass some Mach VM flags, and can 
+be specified as \-1 if no such flags are associated with 
+the region.  Mach VM flags are defined in 
+<mach/vm_statistics.h> and the ones that currently apply 
+to 
+.Nm mmap
+are:
+.Pp
+VM_FLAGS_PURGABLE	to create Mach purgable (i.e. volatile) memory
+VM_MAKE_TAG(tag)	to associate an 8-bit tag with the region
+.Pp
+<mach/vm_statistics.h> defines some preset tags (with a VM_MEMORY_ prefix).
+Users are encouraged to use tags between 240 and 255.
+Tags are used by tools such as vmmap(1) to help identify specific memory regions.
 .It Dv MAP_FILE
 Mapped from a regular file or character-special device memory.  (This is
 the default mapping type, and need not be specified.)
@@ -126,11 +144,20 @@ Use of this option is discouraged.
 Notify the kernel that the region may contain semaphores and that special
 handling may be necessary.
 .It Dv MAP_PRIVATE
-Modifications are private.
+Modifications are private (copy-on-write).
 .It Dv MAP_SHARED
 Modifications are shared.
+.It Dv MAP_NOCACHE
+Pages in this mapping are not retained in the kernel's memory cache.
+If the system runs low on memory, pages in MAP_NOCACHE mappings will be among
+the first to be reclaimed.
+This flag is intended for mappings that have little locality and 
+provides a hint to the kernel that pages in this mapping are unlikely to be needed
+again in the near future.
 .El
 .Pp
+Conforming applications must specify either MAP_PRIVATE or MAP_SHARED.
+.Pp
 The 
 .Xr close 2
 function does not unmap pages, see
@@ -155,49 +182,117 @@ is set to indicate the error.
 .Fn Mmap
 will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+.Fa Fildes
+is not open for reading.
+.\" ===========
 .It Bq Er EACCES
-The flag
-.Dv PROT_READ
-was specified as part of the
-.Fa prot
-parameter and
-.Fa fd
-was not open for reading.
 The flags
 .Dv PROT_WRITE
 and
 .Dv MAP_SHARED
-were specified as part
-of the
+are specified as part of the
 .Fa flags
 and
 .Fa prot
 parameters and
-.Fa fd
-was not open for writing.
+.Fa fildes
+is not open for writing.
+.\" ===========
 .It Bq Er EBADF
-.Fa fd
-is not a valid open file descriptor.
+.Fa fildes
+is not a valid file descriptor for an open file.
 .It Bq Er EINVAL
 .Dv MAP_FIXED
-was specified and the
+is specified and the
 .I addr
-parameter was not page aligned.
-.Fa fd
-did not reference a regular or character special file.
+parameter is not page aligned.
+.\" ===========
+.It Bq Er EINVAL
+.Fa fildes
+does not reference a regular or character special file.
+.\" ===========
+.It Bq Er EINVAL
+.Fa flags
+does not include either MAP_PRIVATE or MAP_SHARED.
+.\" ===========
+.It Bq Er EINVAL
+.Fa len
+is not greater than zero.
+.\" ===========
+.It Bq Er EINVAL
+.Fa offset
+is not a multiple of the page size,
+as returned by
+.Xr sysconf 3 .
+.\" ===========
+.It Bq Er EMFILE
+The limit on mapped regions (per process or system) is exceeded.
+.\" ===========
+.It Bq Er ENODEV
+The file type for
+.Fa fildes
+is not supported for mapping.
+.\" ===========
+.It Bq Er ENOMEM
+.Dv MAP_FIXED
+is specified and the address range specified
+exceeds the address space limit for the process.
+.\" ===========
 .It Bq Er ENOMEM
 .Dv MAP_FIXED
-was specified and the
+is specified and the address specified by the
 .Fa addr
-parameter wasn't available.
+parameter isn't available.
+.\" ===========
+.It Bq Er ENOMEM
 .Dv MAP_ANON
-was specified and insufficient memory was available.
+is specified and insufficient memory is available.
+.\" ===========
+.It Bq Er ENXIO
+Addresses in the specified range are invalid for fildes.
+.\" ===========
+.It Bq Er EOVERFLOW
+Addresses in the specified range exceed the maximum offset
+set for fildes.
+.El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Sh COMPATIBILITY
+.Fn mmap
+now returns with
+.Va errno
+set to EINVAL in places that historically succeeded.
+The rules have changed as follows:
+.Bl -bullet
+.It
+The
+.Fa flags
+parameter must specify either MAP_PRIVATE or MAP_SHARED.
+.It
+The
+.Fa size
+parameter must not be 0.
+.It
+The
+.Fa off
+parameter must be a multiple of pagesize,
+as returned by
+.Fn sysconf .
 .El
 .Sh "SEE ALSO"
 .Xr getpagesize 2 ,
-.Xr msync 2 ,
-.Xr munmap 2 ,
-.Xr mprotect 2 ,
 .Xr madvise 2 ,
 .Xr mincore 2 ,
-.Xr mlock 2
+.Xr mlock 2 ,
+.Xr mprotect 2 ,
+.Xr msync 2 ,
+.Xr munmap 2 ,
+.Xr sysconf 3 ,
+.Xr compat 5
diff --git a/bsd/man/man2/mount.2 b/bsd/man/man2/mount.2
index aae88f178..dc70c857a 100644
--- a/bsd/man/man2/mount.2
+++ b/bsd/man/man2/mount.2
@@ -215,7 +215,7 @@ points outside the process's allocated address space.
 .El
 .Sh SEE ALSO
 .Xr mount 8 ,
-.Xr umount 8 ,
+.Xr umount 8
 .Sh BUGS
 Some of the error codes need translation to more obvious messages.
 .Sh HISTORY
diff --git a/bsd/man/man2/mprotect.2 b/bsd/man/man2/mprotect.2
index 4870751bb..e92b2a69b 100644
--- a/bsd/man/man2/mprotect.2
+++ b/bsd/man/man2/mprotect.2
@@ -40,10 +40,13 @@
 .Nm mprotect
 .Nd control the protection of pages
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn mprotect "caddr_t addr" "size_t len" "int prot"
+.Fo mprotect
+.Fa "void *addr"
+.Fa "size_t len"
+.Fa "int prot"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn mprotect
@@ -58,11 +61,58 @@ a value of 0 is returned.
 Otherwise, a value of -1 is returned and
 .Va errno
 is set to indicate the error.
+.Sh ERRORS
+.Fn mprotect
+will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+The requested protection conflicts with
+the access permissions of the process
+on the specified address range.
+.\" ===========
+.\" .It Bq Er EAGAIN
+.\" Insufficient memory resources exist to allow locking a private page
+.\" under PROT_WRITE.
+.\" ===========
+.It Bq Er EINVAL
+.Fa addr
+is not a multiple of the page size.
+.\" ===========
+.\" .It Bq Er ENOMEM
+.\" The specified address range is outside of the address range
+.\" of the process or includes an unmapped page.
+.\" ===========
+.It Bq Er ENOTSUP
+The combination of accesses requested in
+.Fa prot
+is not supported.
+.El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Pp
+.Ft int
+.br
+.Fo mprotect
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fa "int prot"
+.Fc ;
+.Pp
+The type of
+.Fa addr
+has changed.
 .Sh SEE ALSO
 .Xr madvise 2 , 
 .Xr mincore 2 , 
 .Xr msync 2 ,
-.Xr munmap 2
+.Xr munmap 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn mprotect
diff --git a/bsd/man/man2/msync.2 b/bsd/man/man2/msync.2
index 32ec238f4..f61a3b813 100644
--- a/bsd/man/man2/msync.2
+++ b/bsd/man/man2/msync.2
@@ -41,10 +41,13 @@
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn msync "void *addr" "size_t len" "int flags"
+.Fo msync
+.Fa "void *addr"
+.Fa "size_t len"
+.Fa "int flags"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn msync
@@ -55,6 +58,7 @@ Only those pages containing
 and
 .Fa len-1
 succeeding locations will be examined.
+.Pp
 The
 .Fa flags
 argument may be specified as follows:
@@ -75,23 +79,45 @@ Otherwise, a 0 value is returned.
 .Fn msync
 will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EBUSY
+Some of the specified addresses are locked
+and MS_INVALIDATE is specified.
+.\" ===========
 .It Bq Er EINVAL
 .Fa addr
 is not a multiple of the hardware page size.
+.\" ===========
 .It Bq Er EINVAL
 .Fa len
 is too large, or less than 1.
+.\" ===========
 .It Bq Er EINVAL
 .Fa flags
-combined MS_ASYNC with another flag, which is not permitted.
+is invalid
+(e.g., it combines MS_ASYNC with another flag,
+which is not permitted).
+.\" ===========
 .It Bq Er EIO
-An I/O error occurred while writing to the file system.
+An I/O error occurs while writing to the file system.
+.\" ===========
+.It Bq Er ENOMEM
+The specified address range is outside of the address range
+of the process or includes an unmapped page.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr madvise 2 ,
 .Xr mincore 2 ,
 .Xr mprotect 2 ,
-.Xr munmap 2
+.Xr munmap 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn msync
diff --git a/bsd/man/man2/munmap.2 b/bsd/man/man2/munmap.2
index 7dd9cdbcf..b32a122f2 100644
--- a/bsd/man/man2/munmap.2
+++ b/bsd/man/man2/munmap.2
@@ -40,16 +40,18 @@
 .Nm munmap
 .Nd remove a mapping
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/mman.h>
 .Ft int
-.Fn munmap "caddr_t addr" "size_t len"
+.Fo munmap
+.Fa "void *addr"
+.Fa "size_t len"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn munmap
 system call
 deletes the mappings for the specified address range,
-and causes further references to addresses within the range
+causing further references to addresses within the range
 to generate invalid memory references.
 .Sh RETURN VALUES
 Upon successful completion,
@@ -62,22 +64,47 @@ is set to indicate the error.
 .Fn Munmap
 will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EINVAL
 The
 .Fa addr
-parameter was not page aligned, the
+parameter was not page aligned (i.e., a multiple of the page size).
+.\" ===========
+.It Bq Er EINVAL
+The
 .Fa len
-parameter was negative, or
-some part of the region being unmapped is not part of the currently
+parameter was negative or zero.
+.\" ===========
+.It Bq Er EINVAL
+Some part of the region being unmapped is not part of the currently
 valid address space.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/mman.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Pp
+.Ft int
+.br
+.Fo munmap
+.Fa "caddr_t addr"
+.Fa "size_t len"
+.Fc ;
+.Pp
+The type of
+.Fa addr
+has changed.
 .Sh "SEE ALSO"
 .Xr getpagesize 3 ,
 .Xr msync 2 ,
 .Xr munmap 2 ,
 .Xr mprotect 2 ,
 .Xr madvise 2 ,
-.Xr mincore 2
+.Xr mincore 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn munmap
diff --git a/bsd/man/man2/nfsclnt.2 b/bsd/man/man2/nfsclnt.2
new file mode 100644
index 000000000..0350f89a7
--- /dev/null
+++ b/bsd/man/man2/nfsclnt.2
@@ -0,0 +1,97 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc.  All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd January 9, 2007
+.Dt NFSCLNT 2
+.Os
+.Sh NAME
+.Nm nfsclnt
+.Nd NFS client services
+.Sh SYNOPSIS
+.Fd #include <unistd.h>
+.Fd #include <nfs/nfs.h>
+.Ft int
+.Fn nfsclnt "int flags" "void *argstructp"
+.Sh DESCRIPTION
+The
+.Fn nfsclnt
+function is used by the NFS client daemons to pass information into and out
+of the kernel.
+The
+.Fa flags
+argument consists of several bits that show what action is to be taken
+once in the kernel and the
+.Fa argstructp
+points to any corresponding data that the action may require.
+.Pp
+.Xr rpc.lockd 8
+calls
+.Fn nfsclnt
+with the flag
+.Dv NFSCLNT_LOCKDANS
+and a pointer to a
+.Bd -literal
+struct lockd_ans {
+	int		la_version;		/* lockd_ans version */
+	int		la_errno;		/* return status */
+	u_int64_t	la_xid;			/* unique message transaction ID */
+	int		la_flags;		/* answer flags */
+	pid_t		la_pid;			/* pid of lock requester/owner */
+	off_t		la_start;		/* lock starting offset */
+	off_t		la_len;			/* lock length */
+	int 		la_fh_len;		/* The file handle length. */
+	u_int8_t	la_fh[NFSV3_MAX_FH_SIZE];/* The file handle. */
+};
+.Ed
+.sp
+to pass the response to an NFS file locking request back
+into the kernel.
+.Sh RETURN VALUES
+Upon successful completion, a value of 0 is returned.  Otherwise, -1
+is returned and the global variable
+.Va errno
+is set to specify the error.
+.Sh ERRORS
+.Bl -tag -width Er
+.It Bq Er EPERM
+The caller is not the super-user.
+.It Bq Er EINVAL
+The action specified by the
+.Fa flags
+argument was not valid.
+.It Bq EFAULT
+.Fa argstructp
+points to an invalid address.
+.It Bq EPIPE
+The response doesn't match any currently-outstanding lock request.
+.El
+.Sh SEE ALSO
+.Xr rpc.lockd 8
+.Sh BUGS
+The
+.Nm nfsclnt
+system call is designed specifically for the
+.Tn NFS
+client daemons and as such is specific to their requirements.  Several
+fields of the argument structures are assumed to be valid, such that
+.Nm nfsclnt
+must be used with extreme care.
diff --git a/bsd/man/man2/nfssvc.2 b/bsd/man/man2/nfssvc.2
index 297472162..3670146e0 100644
--- a/bsd/man/man2/nfssvc.2
+++ b/bsd/man/man2/nfssvc.2
@@ -1,3 +1,25 @@
+.\"
+.\" Copyright (c) 1999-2007 Apple Inc.  All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
 .\"	$NetBSD: nfssvc.2,v 1.6 1995/02/27 12:35:08 cgd Exp $
 .\"
 .\" Copyright (c) 1989, 1991, 1993
@@ -33,7 +55,7 @@
 .\"
 .\"	@(#)nfssvc.2	8.1 (Berkeley) 6/9/93
 .\"
-.Dd June 9, 1993
+.Dd January 9, 2007
 .Dt NFSSVC 2
 .Os
 .Sh NAME
@@ -47,139 +69,27 @@
 .Sh DESCRIPTION
 The
 .Fn nfssvc
-function is used by the NFS daemons to pass information into and out
+function is used by the NFS server daemon to pass information into and out
 of the kernel and also to enter the kernel as a server daemon.
 The
 .Fa flags
 argument consists of several bits that show what action is to be taken
 once in the kernel and the
 .Fa argstructp
-points to one of three structures depending on which bits are set in
-flags.
-.Pp
-On the client side,
-.Xr nfsiod 8
-calls
-.Fn nfssvc
-with the
-.Fa flags
-argument set to
-.Dv NFSSVC_BIOD
-and
-.Fa argstructp
-set to
-.Dv NULL
-to enter the kernel as a block I/O server daemon.
-For
-.Nm NQNFS ,
-.Xr mount_nfs 8
-calls
-.Fn nfssvc
-with the
-.Dv NFSSVC_MNTD
-flag, optionally or'd with the flags
-.Dv NFSSVC_GOTAUTH
-and
-.Dv NFSSVC_AUTHINFAIL
-along with a pointer to a
-.Bd -literal
-struct nfsd_cargs {
-	char		*ncd_dirp;	/* Mount dir path */
-	uid_t		ncd_authuid;	/* Effective uid */
-	int		ncd_authtype;	/* Type of authenticator */
-	int		ncd_authlen;	/* Length of authenticator string */
-	char		*ncd_authstr;	/* Authenticator string */
-};
-.Ed
-.sp
-structure.
-The initial call has only the
-.Dv NFSSVC_MNTD
-flag set to specify service for the mount point.
-If the mount point is using Kerberos, then the
-.Xr mount_nfs 8
-daemon will return from
-.Fn nfssvc
-with errno == ENEEDAUTH whenever the client side requires an ``rcmd''
-authentication ticket for the user.
-.Xr Mount_nfs 8
-will attempt to get the Kerberos ticket, and if successful will call
-.Fn nfssvc
-with the flags
-.Dv NFSSVC_MNTD
-and
-.Dv NFSSVC_GOTAUTH
-after filling the ticket into the 
-ncd_authstr field
-and
-setting the ncd_authlen and ncd_authtype
-fields of the nfsd_cargs structure.
-If
-.Xr mount_nfs 8
-failed to get the ticket,
-.Fn nfssvc
-will be called with the flags
-.Dv NFSSVC_MNTD ,
-.Dv NFSSVC_GOTAUTH
-and
-.Dv NFSSVC_AUTHINFAIL
-to denote a failed authentication attempt.
+points to any corresponding data that the action may require.
 .Pp
-On the server side,
 .Fn nfssvc
 is called with the flag
 .Dv NFSSVC_NFSD
-and a pointer to a
-.Bd -literal
-struct nfsd_srvargs {
-	struct nfsd	*nsd_nfsd;	/* Pointer to in kernel nfsd struct */
-	uid_t		nsd_uid;	/* Effective uid mapped to cred */
-	u_long		nsd_haddr;	/* Ip address of client */
-	struct ucred	nsd_cr;		/* Cred. uid maps to */
-	int		nsd_authlen;	/* Length of auth string (ret) */
-	char		*nsd_authstr;	/* Auth string (ret) */
-};
-.Ed
-.sp
+and a NULL
+.Fa argstructp
 to enter the kernel as an
 .Xr nfsd 8
-daemon.
-Whenever an
-.Xr nfsd 8
-daemon receives a Kerberos authentication ticket, it will return from
-.Fn nfssvc
-with errno == ENEEDAUTH.
-The
-.Xr nfsd 8
-will attempt to authenticate the ticket and generate a set of credentials
-on the server for the ``user id'' specified in the field nsd_uid.
-This is done by first authenticating the Kerberos ticket and then mapping
-the Kerberos principal to a local name and getting a set of credentials for
-that user via.
-.Xr getpwnam 3
-and
-.Xr getgrouplist 3 .
-If successful, the
-.Xr nfsd 8
-will call
-.Fn nfssvc
-with the
-.Dv NFSSVC_NFSD
-and
-.Dv NFSSVC_AUTHIN
-flags set to pass the credential mapping in nsd_cr into the
-kernel to be cached on the server socket for that client.
-If the authentication failed,
-.Xr nfsd 8
-calls
-.Fn nfssvc
-with the flags
+daemon.  The
 .Dv NFSSVC_NFSD
-and
-.Dv NFSSVC_AUTHINFAIL
-to denote an authentication failure.
+action normally does not return until the NFS server is stopped.
 .Pp
-The master
+The
 .Xr nfsd 8
 server daemon calls
 .Fn nfssvc
@@ -199,27 +109,53 @@ to pass a server side
 socket into the kernel for servicing by the
 .Xr nfsd 8
 daemons.
+.Pp
+The
+.Xr nfsd 8
+server daemon calls
+.Fn nfssvc
+with the flag
+.Dv NFSSVC_EXPORT
+and a pointer to a
+.Bd -literal
+struct nfs_export_args {
+	uint32_t		nxa_fsid;	/* export FS ID */
+	uint32_t		nxa_expid;	/* export ID */
+	char			*nxa_fspath;	/* export FS path */
+	char			*nxa_exppath;	/* export sub-path */
+	uint32_t		nxa_flags;	/* export arg flags */
+	uint32_t		nxa_netcount;	/* #entries in ex_nets array */
+	struct nfs_export_net_args *nxa_nets;	/* array of net args */
+};
+.Ed
+.sp
+to pass exported file system information into the kernel.
 .Sh RETURN VALUES
-Normally
-.Nm nfssvc
-does not return unless the server
-is terminated by a signal when a value of 0 is returned.
-Otherwise, -1 is returned and the global variable
+Upon successful completion, a value of 0 is returned.  Otherwise, -1 is
+returned the global variable
 .Va errno
 is set to specify the error.
+.Pp
 .Sh ERRORS
 .Bl -tag -width Er
-.It Bq Er ENEEDAUTH
-This special error value
-is really used for authentication support, particularly Kerberos,
-as explained above.
 .It Bq Er EPERM
 The caller is not the super-user.
+.It Bq Er EINVAL
+The action specified by the
+.Fa flags
+argument was not valid.
+.It Bq EFAULT
+.Fa argstructp
+points to an invalid address.
+.It Bq ENOMEM
+A memory allocation failure prevented the action from completing.
+.It Bq EEXIST
+An attempt was made to add a UDP socket via the
+.Dv NFSSVC_ADDSOCK
+action, but the UDP socket has already been added.
 .El
 .Sh SEE ALSO
-.Xr nfsd 8 ,
-.Xr mount_nfs 8 ,
-.Xr nfsiod 8
+.Xr nfsd 8
 .Sh HISTORY
 The
 .Nm nfssvc
@@ -229,12 +165,7 @@ The
 .Nm nfssvc
 system call is designed specifically for the
 .Tn NFS
-support daemons and as such is specific to their requirements.
-It should really return values to indicate the need for authentication
-support, since
-.Dv ENEEDAUTH
-is not really an error.
-Several fields of the argument structures are assumed to be valid and
-sometimes to be unchanged from a previous call, such that
+server daemons and as such is specific to their requirements.  Several
+fields of the argument structures are assumed to be valid, such that
 .Nm nfssvc
 must be used with extreme care.
diff --git a/bsd/man/man2/open.2 b/bsd/man/man2/open.2
index cf215d87d..e12aa2e15 100644
--- a/bsd/man/man2/open.2
+++ b/bsd/man/man2/open.2
@@ -40,23 +40,33 @@
 .Nm open
 .Nd open or create a file for reading or writing
 .Sh SYNOPSIS
+.\" OH??? .Fd #include <sys/stat.h>
 .Fd #include <fcntl.h>
 .Ft int
-.Fn open "const char *path" "int flags" "mode_t mode"
+.Fo open
+.Fa "const char *path"
+.Fa "int oflag"
+.Fa "..."
+.Fc
 .Sh DESCRIPTION
 The file name specified by
 .Fa path
 is opened
-for reading and/or writing as specified by the
-argument
-.Fa flags
-and the file descriptor returned to the calling process.
+for reading and/or writing,
+as specified by the argument
+.Fa oflag ;
+the file descriptor is returned to the calling process.
+.Pp
 The
-.Fa flags
-argument may indicate the file is to be
+.Fa oflag
+argument may indicate that the file is to be
 created if it does not exist (by specifying the
 .Dv O_CREAT
-flag), in which case the file is created with mode
+flag).  In this case,
+.Nm
+requires a third argument
+.Fa "mode_t mode" ;
+the file is created with mode
 .Fa mode
 as described in
 .Xr chmod 2
@@ -65,7 +75,7 @@ and modified by the process' umask value (see
 .Pp
 The flags specified are formed by
 .Em or Ns 'ing
-the following values
+the following values:
 .Pp
 .Bd -literal -offset indent -compact
 O_RDONLY	open for reading only
@@ -75,15 +85,16 @@ O_NONBLOCK	do not block on open or for data to become available
 O_APPEND	append on each write
 O_CREAT		create file if it does not exist
 O_TRUNC		truncate size to 0
-O_EXCL		error if create and file exists
+O_EXCL		error if O_CREAT and the file exists
 O_SHLOCK	atomically obtain a shared lock
 O_EXLOCK	atomically obtain an exclusive lock
+O_NOFOLLOW	do not follow symlinks
+O_SYMLINK	allow open of symlinks
 .Ed
 .Pp
 Opening a file with
 .Dv O_APPEND
-set causes each write on the file
-to be appended to the end.  If
+set causes each write on the file to be appended to the end.  If
 .Dv O_TRUNC
 is specified and the
 file exists, the file is truncated to zero length.
@@ -94,26 +105,26 @@ is set with
 and the file already
 exists,
 .Fn open
-returns an error.  This may be used to
-implement a simple exclusive access locking mechanism.
+returns an error.
+This may be used to implement a simple exclusive-access locking mechanism.
 If
 .Dv O_EXCL
-is set and the last component of the pathname is
-a symbolic link,
+is set and the last component of the pathname is a symbolic link,
 .Fn open
-will fail even if the symbolic
-link points to a non-existent name.
+will fail even if the symbolic link points to a non-existent name.
+.Pp
 If the
 .Dv O_NONBLOCK
-flag is specified, do not wait for the device or file to be ready or
-available.  If the
+flag is specified, do not wait for the device or file
+to be ready or available.  If the
 .Fn open
 call would result
-in the process being blocked for some reason (e.g., waiting for
-carrier on a dialup line),
+in the process being blocked for some reason
+(e.g., waiting for carrier on a dialup line),
 .Fn open
 returns immediately.
-This flag also has the effect of making all subsequent I/O on the open file non-blocking.
+This flag also has the effect of making all subsequent I/O
+on the open file non-blocking.
 .Pp
 When opening a file, a lock with
 .Xr flock 2
@@ -127,15 +138,31 @@ If creating a file with
 the request for the lock will never fail
 (provided that the underlying filesystem supports locking).
 .Pp
+If
+.Dv O_NOFOLLOW
+is used in the mask and the target file passed to
+.Fn open
+is a symbolic link then the
+.Fn open
+will fail.
+.Pp
+If
+.Dv O_SYMLINK
+is used in the mask and the target file passed to
+.Fn open
+is a symbolic link then the
+.Fn open
+will be for the symbolic link itself, not what it links to.
+.Pp
 If successful,
 .Fn open
 returns a non-negative integer, termed a file descriptor.
 It returns -1 on failure.
-The file pointer used to mark the current position within the
-file is set to the beginning of the file.
+The file pointer (used to mark the current position within the file)
+is set to the beginning of the file.
 .Pp
-When a new file is created it is given the group of the directory
-which contains it.
+When a new file is created,
+it is given the group of the directory which contains it.
 .Pp
 The new descriptor is set to remain open across
 .Xr execve
@@ -145,61 +172,115 @@ and
 .Xr fcntl 2 .
 .Pp
 The system imposes a limit on the number of file descriptors
-open simultaneously by one process.
+that can be held open simultaneously by one process.
 .Xr Getdtablesize 2
 returns the current system limit.
+.Sh RETURN VALUES
+If successful,
+.Fn open
+returns a non-negative integer, termed a file descriptor.
+It returns -1 on failure, and sets
+.Va errno
+to indicate the error.
 .Sh ERRORS
 The named file is opened unless:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-.Dv O_CREAT
-is not set and the named file does not exist.
-.It Bq Er ENOENT
-A component of the path name that must exist does not exist.
+.\" ===========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
+.\" ===========
 .It Bq Er EACCES
 The required permissions (for reading and/or writing)
 are denied for the given flags.
+.\" ===========
 .It Bq Er EACCES
 .Dv O_CREAT
 is specified,
 the file does not exist,
 and the directory in which it is to be created
 does not permit writing.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ===========
+.It Bq Er EACCES
+.Dv O_TRUNC
+is specified and write permission is denied.
+.\" ===========
+.It Bq Er EAGAIN
+.Fa path
+specifies the slave side of a locked pseudo-terminal device.
+.\" ===========
+.It Bq Er EDQUOT
+.Dv O_CREAT
+is specified,
+the file does not exist,
+and the directory in which the entry for the new file
+is being placed cannot be extended because the
+user's quota of disk blocks on the file system
+containing the directory has been exhausted.
+.\" ===========
+.It Bq Er EDQUOT
+.Dv O_CREAT
+is specified,
+the file does not exist,
+and the user's quota of inodes on the file system
+on which the file is being created has been exhausted.
+.\" ===========
+.It Bq Er EEXIST
+.Dv O_CREAT
+and
+.Dv O_EXCL
+are specified and the file exists.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.\" ===========
+.It Bq Er EINTR
+The
+.Fn open
+operation is interrupted by a signal.
+.\" ===========
+.It Bq Er EINVAL
+The value of
+.Fa oflag
+is not valid.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while making the directory entry or
+allocating the inode for
+.Dv O_CREAT .
+.\" ===========
 .It Bq Er EISDIR
 The named file is a directory, and the arguments specify
-it is to be opened for writing.
-.It Bq Er EROFS
-The named file resides on a read-only file system,
-and the file is to be modified.
+that it is to be opened for writing.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er EMFILE
 The process has already reached its limit for open file descriptors.
+.\" ===========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds
+.Dv {NAME_MAX}
+characters, or an entire path name exceeded 
+.Dv {PATH_MAX}
+characters.
+.\" ===========
 .It Bq Er ENFILE
 The system file table is full.
-.It Bq Er ENXIO
-The named file is a character special or block
-special file, and the device associated with this special file
-does not exist.
-.It Bq Er EINTR
-The
-.Fn open
-operation was interrupted by a signal.
-.It Bq Er EOPNOTSUPP
-.Dv O_SHLOCK
-or
-.Dv O_EXLOCK
-is specified but the underlying filesystem does not support locking.
+.\" ===========
+.It Bq Er ELOOP
+.Dv O_NOFOLLOW
+was specified and the target is a symbolic link.
+.\" ===========
+.It Bq Er ENOENT
+.Dv O_CREAT
+is not set and the named file does not exist.
+.\" ===========
+.It Bq Er ENOENT
+A component of the path name that must exist does not exist.
+.\" ===========
 .It Bq Er ENOSPC
 .Dv O_CREAT
 is specified,
@@ -207,46 +288,54 @@ the file does not exist,
 and the directory in which the entry for the new file is being placed
 cannot be extended because there is no space left on the file
 system containing the directory.
+.\" ===========
 .It Bq Er ENOSPC
 .Dv O_CREAT
 is specified,
 the file does not exist,
 and there are no free inodes on the file system on which the
 file is being created.
-.It Bq Er EDQUOT
-.Dv O_CREAT
-is specified,
-the file does not exist,
-and the directory in which the entry for the new file
-is being placed cannot be extended because the
-user's quota of disk blocks on the file system
-containing the directory has been exhausted.
-.It Bq Er EDQUOT
-.Dv O_CREAT
-is specified,
-the file does not exist,
-and the user's quota of inodes on the file system on
-which the file is being created has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while making the directory entry or
-allocating the inode for
-.Dv O_CREAT .
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ===========
+.It Bq Er ENXIO
+The named file is a character-special or block-special file
+and the device associated with this special file does not exist.
+.\" ===========
+.It Bq Er ENXIO
+O_NONBLOCK and O_WRONLY are set, the file is a FIFO,
+and no process has it open for reading.
+.\" ===========
+.It Bq Er EOPNOTSUPP
+.Dv O_SHLOCK
+or
+.Dv O_EXLOCK
+is specified, but the underlying filesystem does not support locking.
+.\" ===========
+.It Bq Er EOPNOTSUPP
+An attempt is made to open a socket (not currently implemented).
+.\" ===========
+.It Bq Er EOVERFLOW
+The named file is a regular file
+and its size does not fit in an object of type off_t.
+.\" ===========
+.It Bq Er EROFS
+The named file resides on a read-only file system,
+and the file is to be modified.
+.\" ===========
 .It Bq Er ETXTBSY
 The file is a pure procedure (shared text) file that is being
 executed and the
 .Fn open
 call requests write access.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
-.It Bq Er EEXIST
-.Dv O_CREAT
-and
-.Dv O_EXCL
-were specified and the file exists.
-.It Bq Er EOPNOTSUPP
-An attempt was made to open a socket (not currently implemented).
 .El
+.Sh COMPATIBILITY
+.Fn open
+on a terminal device (i.e., /dev/console)
+will now make that device a controlling terminal for the process.
+Use the O_NOCTTY flag to open a terminal device
+without changing your controlling terminal.
 .Sh SEE ALSO
 .Xr chmod 2 ,
 .Xr close 2 ,
@@ -254,8 +343,8 @@ An attempt was made to open a socket (not currently implemented).
 .Xr getdtablesize 2 ,
 .Xr lseek 2 ,
 .Xr read 2 ,
-.Xr write 2 ,
-.Xr umask 2
+.Xr umask 2 ,
+.Xr write 2
 .Sh HISTORY
 An
 .Fn open
diff --git a/bsd/man/man2/pathconf.2 b/bsd/man/man2/pathconf.2
index a947bd1e3..afe640327 100644
--- a/bsd/man/man2/pathconf.2
+++ b/bsd/man/man2/pathconf.2
@@ -37,15 +37,21 @@
 .Dt PATHCONF 2
 .Os BSD 4
 .Sh NAME
-.Nm pathconf ,
-.Nm fpathconf
+.Nm fpathconf ,
+.Nm pathconf
 .Nd get configurable pathname variables
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft long
-.Fn pathconf "const char *path" "int name"
+.Fo fpathconf
+.Fa "int fildes"
+.Fa "int name"
+.Fc
 .Ft long
-.Fn fpathconf "int fd" "int name"
+.Fo pathconf
+.Fa "const char *path"
+.Fa "int name"
+.Fc
 .Sh DESCRIPTION
 .Pp
 The
@@ -64,7 +70,7 @@ argument is the name of a file or directory.
 For
 .Nm fpathconf ,
 the
-.Fa fd
+.Fa fildes
 argument is an open file descriptor.
 The
 .Fa name
@@ -119,41 +125,53 @@ and
 functions shall return -1 and set
 .Va errno
 to the corresponding value.
-.Bl -tag -width Er
+.Bl -tag -width ENAMETOOLONGAA
+.\" ===========
 .It Bq Er EINVAL
 The value of the
 .Fa name
 argument is invalid.
+.\" ===========
 .It Bq Er EINVAL
 The implementation does not support an association of the variable
 name with the associated file.
 .El
+.Pp
 .Fn Pathconf
 will fail if:
 .Bl -tag -width ENAMETOOLONGAA
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ===========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while reading from or writing to the file system.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 255 characters,
 or an entire path name exceeded 1023 characters.
+.\" ===========
 .It Bq Er ENOENT
 The named file does not exist.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
 .Pp
-.Bl -tag -width [EFAULT]
+.Bl -tag -width ENAMETOOLONGAA
 .Fn Fpathconf
 will fail if:
+.\" ===========
 .It Bq Er EBADF
-.Fa fd
+.Fa fildes
 is not a valid open file descriptor.
+.\" ===========
 .It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+An I/O error occurs while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
 .Xr sysctl 3
diff --git a/bsd/man/man2/pipe.2 b/bsd/man/man2/pipe.2
index 759b9b9ef..03f12c196 100644
--- a/bsd/man/man2/pipe.2
+++ b/bsd/man/man2/pipe.2
@@ -42,34 +42,36 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn pipe "int *fildes"
+.Fo pipe
+.Fa "int fildes[2]"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn pipe
 function
 creates a
-.Em pipe ,
-which is an object allowing
-unidirectional data flow,
+.Em pipe
+(an object that allows unidirectional data flow)
 and allocates a pair of file descriptors.
 The first descriptor connects to the
 .Em read end
-of the pipe,
-and the second connects to the
-.Em write end  ,
-so that data written to
+of the pipe;
+the second connects to the
+.Em write end .
+.Pp
+Data written to
 .Fa fildes[1]
 appears on (i.e., can be read from)
 .Fa fildes[0] .
 This allows the output of one program to be
 sent
 to another program:
-the source's standard output is set up to be
-the write end of the pipe,
-and the sink's standard input is set up to be
-the read end of the pipe.
-The pipe itself persists until all its associated descriptors are
-closed.
+the source's standard output is set up
+to be the write end of the pipe;
+the sink's standard input is set up
+to be the read end of the pipe.
+The pipe itself persists
+until all of its associated descriptors are closed.
 .Pp
 A pipe whose read or write end has been closed is considered
 .Em widowed .
@@ -91,22 +93,25 @@ The
 .Fn pipe
 call will fail if:
 .Bl -tag -width Er
-.It Bq Er EMFILE
-Too many descriptors are active.
-.It Bq Er ENFILE
-The system file table is full.
+.\" ===========
 .It Bq Er EFAULT
 The
 .Fa fildes
 buffer is in an invalid area of the process's address
 space.
+.\" ===========
+.It Bq Er EMFILE
+Too many descriptors are active.
+.\" ===========
+.It Bq Er ENFILE
+The system file table is full.
 .El
 .Sh SEE ALSO
 .Xr sh 1 ,
-.Xr read 2 ,
-.Xr write 2 ,
 .Xr fork 2 ,
-.Xr socketpair 2
+.Xr read 2 ,
+.Xr socketpair 2 ,
+.Xr write 2
 .Sh HISTORY
 A
 .Fn pipe
diff --git a/bsd/man/man2/poll.2 b/bsd/man/man2/poll.2
index a91b73094..c0da72110 100644
--- a/bsd/man/man2/poll.2
+++ b/bsd/man/man2/poll.2
@@ -57,18 +57,24 @@
 .Sh SYNOPSIS
 .In poll.h
 .Ft int
-.Fn poll "struct pollfd *fds" "nfds_t nfds" "int timeout"
+.Fo poll
+.Fa "struct pollfd fds[]"
+.Fa "nfds_t nfds"
+.Fa "int timeout"
+.Fc
 .Sh DESCRIPTION
 .Fn Poll
-examines a set of file descriptors to see if some of them are ready for
-I/O or if certain events have occurred on them.
+examines a set of file descriptors
+to see if some of them are ready for I/O
+or if certain events have occurred on them.
 The
 .Fa fds
-argument is a pointer to an array of pollfd structures as defined in
+argument is a pointer to an array of pollfd structures,
+as defined in
 .Aq Pa poll.h
 (shown below).  The
 .Fa nfds
-argument determines the size of the
+argument specifies the size of the
 .Fa fds
 array.
 .Bd -literal
@@ -82,7 +88,7 @@ struct pollfd {
 The fields of
 .Fa struct pollfd
 are as follows:
-.Bl -tag -width XXXrevents
+.Bl -tag -width XXXPOLLWRNORM
 .It fd
 File descriptor to poll.
 .It events
@@ -97,49 +103,63 @@ and
 .Fa revents
 have the following bits:
 .Bl -tag -width XXXPOLLWRNORM
-.It POLLIN
-Data other than high priority data may be read without blocking.
-This is equivalent to ( POLLRDNORM | POLLRDBAND ).
-.It POLLRDNORM
-Normal data may be read without blocking.
-.It POLLRDBAND
-Priority data may be read without blocking.
-.It POLLPRI
-High priority data may be read without blocking.
-.It POLLOUT
-.It POLLWRNORM
-Normal data may be written without blocking.
-.It POLLWRBAND
-Priority data may be written without blocking.
+.\" ===========
 .It POLLERR
-An exceptional condition has occurred on the device or socket.  This
-flag is output only, and ignored if present in the input
+An exceptional condition has occurred on the device or socket.
+This flag is output only, and ignored if present in the input
 .Fa events
 bitmask.
+.\" ===========
 .It POLLHUP
-The device or socket has been disconnected.  This flag is output only,
+The device or socket has been disconnected.
+This flag is output only,
 and ignored if present in the input
 .Fa events
-bitmask.  Note that
-POLLHUP
-and
-POLLOUT
+bitmask.
+Note that POLLHUP and POLLOUT
 are mutually exclusive and should never be present in the
 .Fa revents
 bitmask at the same time.
+.\" ===========
+.It POLLIN
+Data other than high priority data may be read without blocking.
+This is equivalent to ( POLLRDNORM | POLLRDBAND ).
+.\" ===========
 .It POLLNVAL
-The file descriptor is not open.  This flag is output only, and ignored if present in the input
+The file descriptor is not open.
+This flag is output only, and ignored if present in the input
 .Fa events
 bitmask.
+.\" ===========
+.It POLLOUT
+Normal data may be written without blocking.
+This is equivalent to POLLWRNORM.
+.\" ===========
+.It POLLPRI
+High priority data may be read without blocking.
+.\" ===========
+.It POLLRDBAND
+Priority data may be read without blocking.
+.\" ===========
+.It POLLRDNORM
+Normal data may be read without blocking.
+.\" ===========
+.It POLLWRBAND
+Priority data may be written without blocking.
+.\" ===========
+.It POLLWRNORM
+Normal data may be written without blocking.
 .El
 .Pp
-The distinction between normal, priority, and high-priority data is file type
-or device specific.
+The distinction between normal, priority, and high-priority data
+is specific to particular file types or devices.
 .Pp
 If
 .Fa timeout
-is greater than zero, it specifies a maximum interval to
-wait for any file descriptor to become ready, in milliseconds.  If
+is greater than zero,
+it specifies a maximum interval (in milliseconds)
+to wait for any file descriptor to become ready.
+If
 .Fa timeout
 is zero, then
 .Fn poll
@@ -148,8 +168,9 @@ will return without blocking. If the value of
 is -1, the poll blocks indefinitely.
 .Sh RETURN VALUES
 .Fn Poll
-returns the number of descriptors that are ready for I/O, or -1 if an
-error occured.  If the time limit expires,
+returns the number of descriptors that are ready for I/O,
+or -1 if an error occurred.
+If the time limit expires,
 .Fn poll
 returns 0.
 If
@@ -158,22 +179,30 @@ returns with an error,
 including one due to an interrupted call,
 the
 .Fa fds
-array will be unmodified.
+array will be unmodified and the global variable
+.Va errno
+will be set to indicate the error.
 .Sh ERRORS
-An error return from
-.Fn poll
-indicates:
+.Fn Poll
+will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EAGAIN
+Allocation of internal data structures fails.
+A subsequent request may succeed.
+.\" ===========
 .It Bq Er EFAULT
 .Fa Fds
 points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EINTR
-A signal was delivered before the time limit expired and
-before any of the selected events occurred.
+A signal is delivered before the time limit expires
+and before any of the selected events occurs.
+.\" ===========
 .It Bq Er EINVAL
 The
 .Fa nfds
-argument is greater than OPEN_MAX, or the
+argument is greater than OPEN_MAX or the
 .Fa timeout
 argument is less than -1.
 .El
@@ -195,4 +224,3 @@ The
 .Fn poll
 function call appeared in
 .At V .
-
diff --git a/bsd/man/man2/posix_madvise.2 b/bsd/man/man2/posix_madvise.2
index c83f56ab9..d0f9ea997 100644
--- a/bsd/man/man2/posix_madvise.2
+++ b/bsd/man/man2/posix_madvise.2
@@ -1 +1 @@
-.so man2/madvise.2
\ No newline at end of file
+.so man2/madvise.2
diff --git a/bsd/man/man2/posix_spawn.2 b/bsd/man/man2/posix_spawn.2
new file mode 100644
index 000000000..70cac7bfc
--- /dev/null
+++ b/bsd/man/man2/posix_spawn.2
@@ -0,0 +1,349 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawn.2
+.
+.Dd August 9, 2007
+.Dt POSIX_SPAWN 2
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawn
+.Nm posix_spawnp
+.Nd spawn a process
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawn
+.Fa "pid_t *restrict pid"
+.Fa "const char *restrict path"
+.Fa "const posix_spawn_file_actions_t *file_actions"
+.Fa "const posix_spawnattr_t *restrict attrp"
+.Fa "char *const argv[restrict]"
+.Fa "char *const envp[restrict]"
+.Fc
+.Ft int
+.Fo posix_spawnp
+.Fa "pid_t *restrict pid"
+.Fa "const char *restrict file"
+.Fa "const posix_spawn_file_actions_t *file_actions"
+.Fa "const posix_spawnattr_t *restrict attrp"
+.Fa "char *const argv[restrict]"
+.Fa "char *const envp[restrict]"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawn
+function creates a new process from the executable file, called the
+.Em new process file ,
+specified by
+.Em path ,
+which is an absolute or relative path to the file.
+The
+.Fn posix_spawnp
+function is identical to the
+.Fn posix_spawn
+function if the
+.Em file
+specified contains a slash character; otherwise, the
+.Em file
+parameter is used to construct a pathname, with its path prefix being
+obtained by a search of the path specified in the environment by the
+.Dq Ev PATH variable .
+If this variable isn't specified, the default path is set according
+to the
+.Dv _PATH_DEFPATH
+definition in
+.In paths.h ,
+which is set to
+.Dq Ev /usr/bin:/bin .
+This pathname either refers to an executable object file,
+or a file of data for an interpreter;
+.Xr execve 2
+for more details.
+.Pp
+The argument
+.Fa pid
+is a pointer to a pid_t variable to receive the pid of the spawned
+process; if this is NULL, then the pid of the spawned process is
+not returned.  If this pointer is non-NULL, then on successful
+completion, the variable will be modified to contain the pid of the
+spawned process.  The value is undefined in the case of a failure.
+.Pp
+The argument
+.Fa file_actions
+is either NULL, or it is a a pointer to a file actions object that was
+initialized by a call to
+.Xr posix_spawn_file_actions_init 3
+and represents zero or more file actions.
+.Pp
+File descriptors open in the calling process image remain open in
+the new process image, except for those for which the close-on-exec
+flag is set (see
+.Xr close 2
+and
+.Xr fcntl 2 ) .
+Descriptors that remain open are unaffected by
+.Fn posix_spawn
+unless their behaviour is modified by a file action; see
+.Xr posix_spawn_file_actions_init 3
+for more information.
+.Pp
+The argument
+.Fa attrp
+is either NULL, or it is a pointer to an attributes object that was
+initialized by a call to
+.Xr posix_spawnattr_init 3
+and represents a set of spawn attributes to apply.  If NULL, then the
+default attributes are applied; otherwise, these attributes can control
+various aspects of the spawned process, and are applied prior to the
+spawned process beginning execution; see
+.Xr posix_spawnattr_init 3
+for more information.
+.Pp
+The argument
+.Fa argv
+is a pointer to a null-terminated array of
+character pointers to null-terminated character strings.
+These strings construct the argument list to be made available to the new
+process.  At least
+.Fa argv[0]
+must be present in the array, and should contain the file name of the
+program being spawned, e.g. the last component of the
+.Em path
+or
+.Em file
+argument.
+.Pp
+The argument
+.Fa envp
+is a pointer to a null-terminated array of character pointers to
+null-terminated strings.  A pointer to this array is normally stored
+in the global variable
+.Va environ.
+These strings pass information to the
+new process that is not directly an argument to the command (see
+.Xr environ 7 ) .
+.Pp
+Signals set to be ignored in the calling process are set to be ignored in
+the new process, unless the behaviour is modified by user specified
+.Em spawn attributes .
+Signals which are set to be caught in the calling process image are set to
+default action in the new process image.
+Blocked signals remain blocked regardless of changes to the signal action,
+unless the mask is overridden by user specified
+.Em spawn attributes .
+The signal stack is reset to be undefined (see
+.Xr sigaction 2
+for more information).
+.Pp
+By default, the effective user ID and group ID will be the same as those of
+the calling process image; however, this may be overridden to force them to
+be the real user ID and group ID of the parent process by user specified
+.Em spawn attributes
+(see
+.Xr posix_spawnattr_init 3
+for more information).
+.Pp
+If the set-user-ID mode bit of the new process image file is set
+(see
+.Xr chmod 2 ) ,
+the effective user ID of the new process image is set to the owner ID
+of the new process image file.
+If the set-group-ID mode bit of the new process image file is set,
+the effective group ID of the new process image is set to the group ID
+of the new process image file.
+(The effective group ID is the first element of the group list.)
+The real user ID, real group ID and supplementary group IDs of the new
+process image remain the same as the calling process image.
+After any set-user-ID and set-group-ID processing,
+the effective user ID is recorded as the saved set-user-ID,
+and the effective group ID is recorded as the saved set-group-ID.
+These values may be used in changing the effective IDs later (see
+.Xr setuid 2 ) .
+.Pp
+The new process also inherits the following attributes from
+the calling process:
+.Pp
+.Bl -column parent_process_ID -offset indent -compact
+.It parent process ID Ta see Xr getppid 2
+.It process group ID Ta see Xr getpgrp 2 , Xr posix_spawnattr_init 3
+.It access groups Ta see Xr getgroups 2
+.It working directory Ta see Xr chdir 2
+.It root directory Ta see Xr chroot 2
+.It control terminal Ta see Xr termios 4
+.It resource usages Ta see Xr getrusage 2
+.It interval timers Ta see Xr getitimer 2
+.It resource limits Ta see Xr getrlimit 2
+.It file mode mask Ta see Xr umask 2
+.It signal mask Ta see Xr sigaction 2 , Xr sigsetmask 2 ,
+.Xr posix_spawnattr_init 3
+.El
+.Pp
+When a program is executed as a result of a
+.Fn posix_spawn
+or
+.Fn posix_spawnp
+call, it is entered as follows:
+.Bd -literal -offset indent
+main(argc, argv, envp)
+int argc;
+char **argv, **envp;
+.Ed
+.Pp
+where
+.Fa argc
+is the number of elements in
+.Fa argv
+(the ``arg count'')
+and
+.Fa argv
+points to the array of character pointers
+to the arguments themselves.
+.Sh RETURN VALUES
+If the
+.Em pid
+argument is NULL, no pid is returned to the calling process; if it is
+non-NULL, then
+.Fn posix_spawn
+and
+.Fn posix_spawnp
+functions return the process ID of the child process into the pid_t
+variable pointed to by the
+.Em pid
+argument and return a 0 on success.  If an error occurs, they return
+a non-zero error code as the function return value, and no child process
+is created.
+.Sh ERRORS
+The
+.Fn posix_spawn
+and
+.Fn posix_spawnp
+functions will fail and return to the calling process if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa file_actions
+or
+.Fa attrp
+is invalid.
+.\" ==========
+.It Bq Er E2BIG
+The number of bytes in the new process's argument list
+is larger than the system-imposed limit.
+This limit is specified by the
+.Xr sysctl 3
+MIB variable
+.Dv KERN_ARGMAX .
+.\" ==========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ==========
+.It Bq Er EACCES
+The new process file is not an ordinary file.
+.\" ==========
+.It Bq Er EACCES
+The new process file mode denies execute permission.
+.\" ==========
+.It Bq Er EACCES
+The new process file is on a filesystem mounted
+with execution disabled
+.Pf ( Dv MNT_NOEXEC
+in
+.Ao Pa sys/mount.h Ac ) .
+.\" ==========
+.It Bq Er EFAULT
+The new process file is not as long as indicated by
+the size values in its header.
+.\" ==========
+.It Bq Er EFAULT
+.Fa Path ,
+.Fa argv ,
+or
+.Fa envp
+point
+to an illegal address.
+.\" ==========
+.It Bq Er EIO
+An I/O error occurred while reading from the file system.
+.\" ==========
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ==========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded 
+.Dv {NAME_MAX}
+characters, or an entire path name exceeded 
+.Dv {PATH_MAX}
+characters.
+.\" ==========
+.It Bq Er ENOENT
+The new process file does not exist.
+.\" ==========
+.It Bq Er ENOEXEC
+The new process file has the appropriate access
+permission, but has an unrecognized format
+(e.g., an invalid magic number in its header).
+.\" ==========
+.It Bq Er ENOMEM
+The new process requires more virtual memory than
+is allowed by the imposed maximum
+.Pq Xr getrlimit 2 .
+.\" ==========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ==========
+.It Bq Er ETXTBSY
+The new process file is a pure procedure (shared text)
+file that is currently open for writing or reading by some process.
+.El
+.Sh CAVEAT
+If a program is
+.Em setuid
+to a non-super-user, but is executed when
+the real
+.Em uid
+is ``root'', then the program has some of the powers
+of a super-user as well.
+.Sh SEE ALSO
+.Xr exit 2 ,
+.Xr fork 2 ,
+.Xr execl 3 ,
+.Xr sysctl 3 ,
+.Xr environ 7 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_file_actions_init 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawn
+and
+.Fn posix_spawnp
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man2/profil.2 b/bsd/man/man2/profil.2
index ee6d78602..dd31dfff1 100644
--- a/bsd/man/man2/profil.2
+++ b/bsd/man/man2/profil.2
@@ -42,7 +42,10 @@
 .Sh NAME
 .Nm profil
 .Nd control process profiling
+.Sh LIBRARY
+.Lb libc
 .Sh SYNOPSIS
+.In unistd.h
 .Ft int
 .Fn profil "char *samples" "size_t size" "u_long offset" "u_int scale"
 .Sh DESCRIPTION
@@ -117,8 +120,11 @@ contains an invalid address.
 .El
 .Sh SEE ALSO
 .Xr gprof 1
-.\" .Sh HISTORY
-.\" wish I knew...  probably v7.
+.Sh HISTORY
+The
+.Fn profil
+function appeared in
+.At v7 .
 .Sh BUGS
 This routine should be named
 .Fn profile .
diff --git a/bsd/man/man2/quotactl.2 b/bsd/man/man2/quotactl.2
index 68b2e3c24..c60519313 100644
--- a/bsd/man/man2/quotactl.2
+++ b/bsd/man/man2/quotactl.2
@@ -43,6 +43,7 @@
 .Nm quotactl
 .Nd manipulate filesystem quotas
 .Sh SYNOPSIS
+.Fd #include <sys/types.h>	/* types needed by quota.h */
 .Fd #include <sys/quota.h>	/* for disk quotas */
 .Ft int
 .Fn quotactl "const char *path" "int cmd" "int id" "char *addr"
diff --git a/bsd/man/man2/read.2 b/bsd/man/man2/read.2
index 097b9a360..65886dc03 100644
--- a/bsd/man/man2/read.2
+++ b/bsd/man/man2/read.2
@@ -36,9 +36,9 @@
 .Dt READ 2
 .Os
 .Sh NAME
+.Nm pread ,
 .Nm read ,
-.Nm readv ,
-.Nm pread
+.Nm readv
 .Nd read input
 .Sh LIBRARY
 .Lb libc
@@ -47,29 +47,43 @@
 .In sys/uio.h
 .In unistd.h
 .Ft ssize_t
-.Fn read "int d" "void *buf" "size_t nbytes"
+.Fo pread
+.Fa "int d"
+.Fa "void *buf"
+.Fa "size_t nbyte"
+.Fa "off_t offset"
+.Fc
 .Ft ssize_t
-.Fn readv "int d" "const struct iovec *iov" "int iovcnt"
+.Fo read
+.Fa "int fildes"
+.Fa "void *buf"
+.Fa "size_t nbyte"
+.Fc
 .Ft ssize_t
-.Fn pread "int d" "void *buf" "size_t nbytes" "off_t offset"
+.Fo readv
+.Fa "int d"
+.Fa "const struct iovec *iov"
+.Fa "int iovcnt"
+.Fc
 .Sh DESCRIPTION
 .Fn Read
 attempts to read
-.Fa nbytes
-of data from the object referenced by the descriptor
-.Fa d
+.Fa nbyte
+bytes of data from the object referenced by the descriptor
+.Fa fildes
 into the buffer pointed to by
 .Fa buf .
 .Fn Readv
-performs the same action, but scatters the input data
-into the
+performs the same action,
+but scatters the input data into the
 .Fa iovcnt
 buffers specified by the members of the
 .Fa iov
 array: iov[0], iov[1], ..., iov[iovcnt\|\-\|1].
 .Fn Pread
-performs the same function, but reads from the specified position in
-the file without modifying the file pointer.
+performs the same function,
+but reads from the specified position in the file
+without modifying the file pointer.
 .Pp
 For
 .Fn readv ,
@@ -96,7 +110,7 @@ On objects capable of seeking, the
 .Fn read
 starts at a position
 given by the pointer associated with
-.Fa d
+.Fa fildes
 (see
 .Xr lseek 2 ) .
 Upon return from
@@ -125,66 +139,141 @@ Otherwise, a -1 is returned and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Read ,
-.Fn readv ,
+The
+.Fn pread ,
+.Fn read ,
 and
-.Fn pread
+.Fn readv
+calls
 will succeed unless:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EAGAIN
+The file was marked for non-blocking I/O,
+and no data were ready to be read.
+.\" ===========
 .It Bq Er EBADF
-.Fa D
+.Fa fildes
 is not a valid file or socket descriptor open for reading.
+.\" ===========
 .It Bq Er EFAULT
 .Fa Buf
 points outside the allocated address space.
-.It Bq Er EIO
-An I/O error occurred while reading from the file system.
+.\" ===========
 .It Bq Er EINTR
 A read from a slow device was interrupted before
 any data arrived by the delivery of a signal.
 .It Bq Er EINVAL
 The pointer associated with
-.Fa d
+.Fa fildes
 was negative.
-.It Bq Er EAGAIN
-The file was marked for non-blocking I/O,
-and no data were ready to be read.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurred while reading from the file system.
+.\" ===========
+.\" .It Bq Er EIO
+.\" The process is a member of a background process
+.\" attempting to read from its controlling terminal.
+.\" ===========
+.\" .It Bq Er EIO
+.\" The process is ignoring or blocking the SIGTTIN signal.
+.\" ===========
+.It Bq Er EIO
+The process group is orphaned.
+.\" ===========
+.It Bq Er EIO
+The file is a regular file,
+.Fa nbyte
+is greater than 0,
+the starting position is before the end-of-file,
+and the starting position is greater than or equal
+to the offset maximum established
+for the open file descriptor associated with
+.Fa fildes .
+.\" ===========
+.It Bq Er EISDIR
+An attempt is made to read a directory.
+.\" ===========
+.It Bq Er ENOBUFS
+An attempt to allocate a memory buffer fails.
+.\" ===========
+.It Bq Er ENOMEM
+Insufficient memory is available.
+.\" ===========
+.It Bq Er ENXIO
+An action is requested of a device that does not exist..
+.\" ===========
+.It Bq Er ENXIO
+A requested action cannot be performed by the device.
+.El
+.Pp
+The
+.Fn pread
+call may also return the following errors:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+The specified file offset is invalid.
+.\" ===========
+.It Bq Er ESPIPE
+The file descriptor is associated with a pipe, socket, or FIFO.
 .El
 .Pp
-In addition,
+The
+.Fn read
+call may also return the following errors:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er ECONNRESET
+The connection is closed by the peer
+during a read attempt on a socket.
+.\" ===========
+.It Bq Er ENOTCONN
+A read is attempted on an unconnected socket.
+.\" ===========
+.It Bq Er ETIMEDOUT
+A transmission timeout occurs
+during a read attempt on a socket.
+.El
+.Pp
+The
 .Fn readv
-may return one of the following errors:
+call may also return one of the following errors:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EFAULT
+Part of the
+.Fa iov
+points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EINVAL
 .Fa Iovcnt
 was less than or equal to 0, or greater than 16.
+.\" ===========
 .It Bq Er EINVAL
 One of the
 .Fa iov_len
 values in the
 .Fa iov
 array was negative.
+.\" ===========
 .It Bq Er EINVAL
 The sum of the
 .Fa iov_len
 values in the
 .Fa iov
 array overflowed a 32-bit integer.
-.It Bq Er EFAULT
-Part of the
-.Fa iov
-points outside the process's allocated address space.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/uio.h>
+.Fd #include <unistd.h>
 .Pp
-The
-.Fn pread
-call may also return the following errors:
-.Bl -tag -width Er
-.It Bq Er EINVAL
-The specified file offset is invalid.
-.It Bq Er ESPIPE
-The file descriptor is associated with a pipe, socket, or FIFO.
-.El
+The include files
+.In sys/types.h
+and
+.In sys/uio.h
+are necessary for all functions.
 .Sh SEE ALSO
 .Xr dup 2 ,
 .Xr fcntl 2 ,
@@ -192,7 +281,8 @@ The file descriptor is associated with a pipe, socket, or FIFO.
 .Xr pipe 2 ,
 .Xr select 2 ,
 .Xr socket 2 ,
-.Xr socketpair 2
+.Xr socketpair 2 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn read
diff --git a/bsd/man/man2/readlink.2 b/bsd/man/man2/readlink.2
index 0a604f256..acd80233f 100644
--- a/bsd/man/man2/readlink.2
+++ b/bsd/man/man2/readlink.2
@@ -41,8 +41,12 @@
 .Nd read value of a symbolic link
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
-.Ft int
-.Fn readlink "const char *path" "char *buf" "int bufsiz"
+.Ft  ssize_t
+.Fo readlink
+.Fa "const char *restrict path"
+.Fa "char *restrict buf"
+.Fa "size_t bufsize"
+.Fc
 .Sh DESCRIPTION
 .Fn Readlink
 places the contents of the symbolic link
@@ -50,7 +54,7 @@ places the contents of the symbolic link
 in the buffer
 .Fa buf ,
 which has size
-.Fa bufsiz .
+.Fa bufsize .
 .Nm Readlink
 does not append a
 .Dv NUL
@@ -65,33 +69,58 @@ code in the global variable
 .Fn Readlink
 will fail if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
+.\" ===========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Buf
+extends outside the process's allocated address space.
+.\" ===========
+.It Bq Er EINVAL
+The named file is not a symbolic link.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurred while reading from the file system.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er ENAMETOOLONG
 A component of a pathname exceeded 
 .Dv {NAME_MAX}
-characters, or an entire path name exceeded 
+characters, or an entire path name
+(possibly expanded by a symbolic link) exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ===========
 .It Bq Er ENOENT
 The named file does not exist.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
-.It Bq Er EINVAL
-The named file is not a symbolic link.
-.It Bq Er EIO
-An I/O error occurred while reading from the file system.
-.It Bq Er EFAULT
-.Fa Buf
-extends outside the process's allocated address space.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <unistd.h>
+.Pp
+.Ft int
+.br
+.Fo readlink
+.Fa "const char *path"
+.Fa "char *buf"
+.Fa "int bufsize"
+.Fc ;
+.Pp
+The function type and the type of
+.Fa bufsize
+have changed.
 .Sh SEE ALSO
-.Xr stat 2 ,
 .Xr lstat 2 ,
-.Xr symlink 2
-.Xr symlink 7 ,
+.Xr stat 2 ,
+.Xr symlink 2 ,
+.Xr compat 5 ,
+.Xr symlink 7
 .Sh HISTORY
 The
 .Fn readlink
diff --git a/bsd/man/man2/reboot.2 b/bsd/man/man2/reboot.2
index ee2829c9e..4edf12252 100644
--- a/bsd/man/man2/reboot.2
+++ b/bsd/man/man2/reboot.2
@@ -47,10 +47,14 @@
 .Fn reboot "int howto"
 .Sh DESCRIPTION
 .Fn Reboot
-reboots the system.
-Only the super-user may reboot a machine on demand.
-However, a reboot is invoked
-automatically in the event of unrecoverable system failures.
+reboots the system.  Only the super-user may reboot a machine on demand.
+However, a reboot may be invoked automatically in the event of
+unrecoverable system failures.  Programs other than
+.Xr reboot 8
+should not call
+.Fn reboot .
+.Xr Shutdown 8
+or a higher-level API will shut the system down cleanly.
 .Pp
 .Fa Howto
 is a mask of options; the system call interface allows the following
@@ -143,12 +147,10 @@ variable
 The caller is not the super-user.
 .El
 .Sh SEE ALSO
-.Xr kadb 4 ,
-.Xr crash 8 ,
+.Xr shutdown 8 ,
 .Xr halt 8 ,
-.Xr init 8 ,
-.Xr reboot 8 ,
-.Xr savecore 8
+.Xr launchd 8 ,
+.Xr reboot 8
 .Sh BUGS
 The HP300 implementation supports neither
 .Dv RB_DFLTROOT
diff --git a/bsd/man/man2/recv.2 b/bsd/man/man2/recv.2
index 5ceee5989..433bca52a 100644
--- a/bsd/man/man2/recv.2
+++ b/bsd/man/man2/recv.2
@@ -1,5 +1,3 @@
-.\"	$NetBSD: recv.2,v 1.6 1995/02/27 12:36:08 cgd Exp $
-.\"
 .\" Copyright (c) 1983, 1990, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
 .\"
@@ -33,58 +31,82 @@
 .\"
 .\"     @(#)recv.2	8.3 (Berkeley) 2/21/94
 .\"
-.Dd February 21, 1994
+.Dd May 15, 2006
 .Dt RECV 2
-.Os BSD 4.3r
+.Os
 .Sh NAME
 .Nm recv ,
 .Nm recvfrom ,
 .Nm recvmsg
 .Nd receive a message from a socket
+.Sh LIBRARY
+.Lb libc
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
-.Fd #include <sys/socket.h>
+.In sys/socket.h
 .Ft ssize_t
-.Fn recv "int s" "void *buf" "size_t len" "int flags"
+.Fo recv
+.Fa "int socket"
+.Fa "void *buffer"
+.Fa "size_t length"
+.Fa "int flags"
+.Fc
 .Ft ssize_t
-.Fn recvfrom "int s" "void *buf" "size_t len" "int flags" "struct sockaddr *from" "socklen_t *fromlen"
+.Fo recvfrom
+.Fa "int socket"
+.Fa "void *restrict buffer"
+.Fa "size_t length"
+.Fa "int flags"
+.Fa "struct sockaddr *restrict address"
+.Fa "socklen_t *restrict address_len"
+.Fc
 .Ft ssize_t
-.Fn recvmsg "int s" "struct msghdr *msg" "int flags"
+.Fo recvmsg
+.Fa "int socket"
+.Fa "struct msghdr *message"
+.Fa "int flags"
+.Fc
 .Sh DESCRIPTION
-.Fn Recvfrom
+The
+.Fn recvfrom
 and
 .Fn recvmsg
+system calls
 are used to receive messages from a socket,
 and may be used to receive data on a socket whether or not
 it is connection-oriented.
 .Pp
 If
-.Fa from
-is non-nil, and the socket is not connection-oriented,
+.Fa address
+is not a null pointer
+and the socket is not connection-oriented,
 the source address of the message is filled in.
-.Fa Fromlen
-is a value-result parameter, initialized to the size of
+The
+.Fa address_len
+argument
+is a value-result argument, initialized to the size of
 the buffer associated with
-.Fa from ,
+.Fa address ,
 and modified on return to indicate the actual size of the
 address stored there.
 .Pp
-The 
+The
 .Fn recv
-call is normally used only on a 
+function is normally used only on a
 .Em connected
 socket (see
 .Xr connect 2 )
 and is identical to
 .Fn recvfrom
-with a nil
-.Fa from
-parameter.
+with a
+null pointer passed as its
+.Fa address
+argument.
 As it is redundant, it may not be supported in future releases.
 .Pp
-On successful completion, all three routines return the number of
-message bytes read.  If a message is too long to fit in the supplied
-buffer, excess bytes may be discarded depending on the type of socket
+All three routines return the length of the message on successful
+completion.
+If a message is too long to fit in the supplied buffer,
+excess bytes may be discarded depending on the type of socket
 the message is received from (see
 .Xr socket 2 ) .
 .Pp
@@ -109,11 +131,16 @@ described in
 .Pp
 The
 .Xr select 2
-call may be used to determine when more data arrive.
+system call may be used to determine when more data arrive.
+.Pp
+If no messages are available to be received and the peer has
+performed an orderly shutdown, the value 0 is returned.
 .Pp
 The
 .Fa flags
-argument to a recv call is formed by 
+argument to a
+.Fn recv
+function is formed by
 .Em or Ap ing
 one or more of the values:
 .Bl -column MSG_WAITALL -offset indent
@@ -121,17 +148,22 @@ one or more of the values:
 .It Dv MSG_PEEK Ta peek at incoming message
 .It Dv MSG_WAITALL Ta wait for full request or error
 .El
+.Pp
 The
 .Dv MSG_OOB
 flag requests receipt of out-of-band data
 that would not be received in the normal data stream.
 Some protocols place expedited data at the head of the normal
 data queue, and thus this flag cannot be used with such protocols.
-The MSG_PEEK flag causes the receive operation to return data
+The
+.Dv MSG_PEEK
+flag causes the receive operation to return data
 from the beginning of the receive queue without removing that
 data from the queue.
 Thus, a subsequent receive call will return the same data.
-The MSG_WAITALL flag requests that the operation block until
+The
+.Dv MSG_WAITALL
+flag requests that the operation block until
 the full request is satisfied.
 However, the call may still return less data than requested
 if a signal is caught, an error or disconnect occurs,
@@ -139,21 +171,21 @@ or the next data to be received is of a different type than that returned.
 .Pp
 The
 .Fn recvmsg
-call uses a 
+system call uses a
 .Fa msghdr
-structure to minimize the number of directly supplied parameters.
+structure to minimize the number of directly supplied arguments.
 This structure has the following form, as defined in
-.Ao Pa sys/socket.h Ac :
+.In sys/socket.h :
 .Pp
 .Bd -literal
 struct msghdr {
-	caddr_t		msg_name;	/* optional address */
-	socklen_t	msg_namelen;	/* size of address */
-	struct		iovec *msg_iov;	/* scatter/gather array */
-	u_int		msg_iovlen;	/* # elements in msg_iov */
-	caddr_t		msg_control;	/* ancillary data, see below */
+	void		*msg_name;      /* optional address */
+	socklen_t	msg_namelen;    /* size of address */
+	struct		iovec *msg_iov; /* scatter/gather array */
+	int		msg_iovlen;     /* # elements in msg_iov */
+	void		*msg_control;   /* ancillary data, see below */
 	socklen_t	msg_controllen; /* ancillary data buffer len */
-	int			msg_flags;	/* flags on received message */
+	int		msg_flags;      /* flags on received message */
 };
 .Ed
 .Pp
@@ -161,15 +193,19 @@ Here
 .Fa msg_name
 and
 .Fa msg_namelen
-specify the source address if the socket is unconnected;
+specify the destination address if the socket is unconnected;
 .Fa msg_name
 may be given as a null pointer if no names are desired or required.
-.Fa Msg_iov
+The
+.Fa msg_iov
 and
 .Fa msg_iovlen
+arguments
 describe scatter gather locations, as discussed in
 .Xr read 2 .
-.Fa Msg_control ,
+The
+.Fa msg_control
+argument,
 which has length
 .Fa msg_controllen ,
 points to a buffer for other protocol control related messages
@@ -184,11 +220,14 @@ struct cmsghdr {
 	u_char	cmsg_data[]; */
 };
 .Ed
-As an example, one could use this to learn of changes in the data-stream
-in XNS/SPP, or in ISO, to obtain user-connection-request data by requesting
-a recvmsg with no data buffer provided immediately after an
+.Pp
+As an example, one could use this to learn of changes
+in the data-stream in XNS/SPP,
+or in ISO, to obtain user-connection-request data by requesting a
+.Fn recvmsg
+with no data buffer provided immediately after an
 .Fn accept
-call.
+system call.
 .Pp
 Open file descriptors are now passed as ancillary data for
 .Dv AF_UNIX
@@ -210,55 +249,102 @@ the data returned completed a record (generally used with sockets of type
 .Dv SOCK_SEQPACKET ) .
 .Dv MSG_TRUNC
 indicates that
-the trailing portion of a datagram was discarded because the datagram
-was larger than the buffer supplied.
+the trailing portion of a datagram was discarded
+because the datagram was larger than the buffer supplied.
 .Dv MSG_CTRUNC
-indicates that some
-control data were discarded due to lack of space in the buffer
-for ancillary data.
+indicates that some control data were discarded
+due to lack of space in the buffer for ancillary data.
 .Dv MSG_OOB
 is returned to indicate that expedited or out-of-band data were received.
-.Pp
 .Sh RETURN VALUES
 These calls return the number of bytes received, or -1
 if an error occurred.
+.Pp
+For TCP sockets, the return value 0 means the peer has closed its 
+half side of the connection.
 .Sh ERRORS
 The calls fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EAGAIN
+The socket is marked non-blocking, and the receive operation
+would block, or
+a receive timeout had been set,
+and the timeout expired before data were received.
+.\" ===========
 .It Bq Er EBADF
 The argument
-.Fa s
+.Fa socket
 is an invalid descriptor.
+.\" ===========
+.It Bq Er ECONNRESET
+The connection is closed by the peer
+during a receive attempt on a socket.
+.\" ===========
+.It Bq Er EFAULT
+The receive buffer pointer(s) point outside the process's
+address space.
+.\" ===========
+.It Bq Er EINTR
+The receive was interrupted by delivery of a signal before
+any data were available.
+.\" ===========
+.It Bq Er EINVAL
+MSG_OOB is set, but no out-of-band data is available.
+.\" ===========
+.It Bq Er ENOBUFS
+An attempt to allocate a memory buffer fails.
+.\" ===========
 .It Bq Er ENOTCONN
 The socket is associated with a connection-oriented protocol
 and has not been connected (see
 .Xr connect 2
 and
-.Xr accept 2 ).
+.Xr accept 2 ) .
+.\" ===========
 .It Bq Er ENOTSOCK
 The argument
-.Fa s
+.Fa socket
 does not refer to a socket.
-.It Bq Er EAGAIN
-The socket is marked non-blocking, and the receive operation
-would block, or
-a receive timeout had been set,
-and the timeout expired before data were received.
-.It Bq Er EINTR
-The receive was interrupted by delivery of a signal before
-any data were available.
-.It Bq Er EFAULT
-The receive buffer pointer(s) point outside the process's
-address space.
+.\" ===========
+.It Bq Er EOPNOTSUPP
+The type and/or protocol of
+.Fa socket
+do not support the option(s) specified in
+.Fa flags . 
+.\" ===========
+.It Bq Er ETIMEDOUT
+The connection timed out.
+.El
+.Pp
+The
+.Fn recvfrom
+call may also fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+The total of the iov_len values overflows a ssize_t.
+.El
+.Pp
+The
+.Fn recvmsg
+call may also fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EMSGSIZE
+The requested message size is invalid.
+.\" ===========
+.It Bq Er ENOMEM
+Insufficient memory is available.
 .El
 .Sh SEE ALSO
 .Xr fcntl 2 ,
+.Xr getsockopt 2 ,
 .Xr read 2 ,
 .Xr select 2 ,
-.Xr getsockopt 2 ,
 .Xr socket 2
 .Sh HISTORY
 The
 .Fn recv
-function call appeared in
+function appeared in
 .Bx 4.2 .
diff --git a/bsd/man/man2/removexattr.2 b/bsd/man/man2/removexattr.2
index acfa319d5..0e0366b1a 100644
--- a/bsd/man/man2/removexattr.2
+++ b/bsd/man/man2/removexattr.2
@@ -126,8 +126,8 @@ An I/O error occurred while reading from or writing to the file system.
 .El
 .Sh SEE ALSO
 .Xr getxattr 2 ,
-.Xr setxattr 2 ,
-.Xr listxattr 2
+.Xr listxattr 2 ,
+.Xr setxattr 2
 .Sh HISTORY
 .Fn removexattr
 and
diff --git a/bsd/man/man2/rename.2 b/bsd/man/man2/rename.2
index 4167485d3..39487e6c2 100644
--- a/bsd/man/man2/rename.2
+++ b/bsd/man/man2/rename.2
@@ -42,36 +42,42 @@
 .Sh SYNOPSIS
 .Fd #include <stdio.h>
 .Ft int
-.Fn rename "const char *from" "const char *to"
+.Fo rename
+.Fa "const char *old"
+.Fa "const char *new"
+.Fc
 .Sh DESCRIPTION
-.Fn Rename
-causes the link named
-.Fa from
+The
+.Fn rename
+system call causes the link named
+.Fa old
 to be renamed as
-.Fa to .
+.Fa new .
 If 
-.Fa to
+.Fa new
 exists, it is first removed.
 Both 
-.Fa from
+.Fa old
 and
-.Fa to
-must be of the same type (that is, both directories or both
-non-directories), and must reside on the same file system.
+.Fa new
+must be of the same type
+(that is, both must be either directories or non-directories)
+and must reside on the same file system.
 .Pp
-.Fn Rename
-guarantees that an instance of
-.Fa to
-will always exist, even if the system should crash in
-the middle of the operation.
+The
+.Fn rename
+system call guarantees that an instance of
+.Fa new
+will always exist,
+even if the system should crash in the middle of the operation.
 .Pp
 If the final component of
-.Fa from
+.Fa old
 is a symbolic link,
 the symbolic link is renamed,
 not the file or directory to which it points.
 .Sh CAVEAT
-The system can deadlock if a loop in the file system graph is present.
+The system can deadlock if a loop is present in the file system graph.
 This loop takes the form of an entry in directory
 .Ql Pa a ,
 say
@@ -94,8 +100,13 @@ and
 respectively, 
 the system may deadlock attempting to lock
 both directories for modification.
-Hard links to directories should be
-replaced by symbolic links by the system administrator.
+.Pp
+Whether or not hard links to directories are supported is specific to
+the underlying filesystem implementation.
+.Pp
+It is recommended that any hard links to directories in an underlying
+filesystem should be replaced by symbolic links by the system administrator
+to avoid the possibility of deadlocks.
 .Sh RETURN VALUES
 A 0 value is returned if the operation succeeds, otherwise
 .Fn rename
@@ -103,98 +114,130 @@ returns -1 and the global variable
 .Va errno
 indicates the reason for the failure.
 .Sh ERRORS
-.Fn Rename
-will fail and neither of the argument files will be
+The
+.Fn rename
+system call will fail and neither of the argument files will be
 affected if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+A component of either path prefix denies search permission.
+.\" ===========
+.It Bq Er EACCES
+The requested operation requires writing in a directory
+(e.g.,
+.Fa new ,
+new/.., or old/..) whose modes disallow this.
+.\" ===========
+.It Bq Er EDQUOT
+The directory in which the entry for the new name
+is being placed cannot be extended because the
+user's quota of disk blocks on the file system
+containing the directory has been exhausted.
+.\" ===========
+.It Bq Er EFAULT
+.Em Path
+points outside the process's allocated address space.
+.\" ===========
+.It Bq Er EINVAL
+.Fa Old
+is a parent directory of
+.Fa new ,
+or an attempt is made to rename
+.Ql \&.
+or
+.Ql \&.. .
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while making or updating a directory entry.
+.\" ===========
+.It Bq Er EISDIR
+.Fa new
+is a directory, but
+.Fa old
+is not a directory.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating either pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
+A component of a pathname exceeds 
 .Dv {NAME_MAX}
-characters, or an entire path name exceeded 
+characters, or an entire path name exceeds
 .Dv {PATH_MAX}
 characters.
+.\" ===========
 .It Bq Er ENOENT
 A component of the
-.Fa from
+.Fa old
 path does not exist,
 or a path prefix of
-.Fa to
+.Fa new
 does not exist.
-.It Bq Er EACCES
-A component of either path prefix denies search permission.
-.It Bq Er EACCES
-The requested link requires writing in a directory with a mode
-that denies write permission.
+.\" ===========
+.It Bq Er ENOSPC
+The directory in which the entry for the new name is being placed
+cannot be extended because there is no space left on the file
+system containing the directory.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of either path prefix is not a directory.
+.\" ===========
+.It Bq Er ENOTDIR
+.Fa old
+is a directory, but
+.Fa new
+is not a directory.
+.\" ===========
+.It Bq Er ENOTEMPTY
+.Fa New
+is a directory and is not empty.
+.\" ===========
 .It Bq Er EPERM
 The directory containing
-.Fa from
+.Fa old
 is marked sticky,
 and neither the containing directory nor
-.Fa from
+.Fa old
 are owned by the effective user ID.
+.\" ===========
 .It Bq Er EPERM
 The
-.Fa to
+.Fa new
 file exists,
 the directory containing
-.Fa to
+.Fa new
 is marked sticky,
 and neither the containing directory nor
-.Fa to
+.Fa new
 are owned by the effective user ID.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating either pathname.
-.It Bq Er ENOTDIR
-A component of either path prefix is not a directory.
-.It Bq Er ENOTDIR
-.Fa from
-is a directory, but
-.Fa to
-is not a directory.
-.It Bq Er EISDIR
-.Fa to
-is a directory, but
-.Fa from
-is not a directory.
-.It Bq Er EXDEV
-The link named by
-.Fa to
-and the file named by
-.Fa from
-are on different logical devices (file systems).  Note that this error
-code will not be returned if the implementation permits cross-device
-links.
-.It Bq Er ENOSPC
-The directory in which the entry for the new name is being placed
-cannot be extended because there is no space left on the file
-system containing the directory.
-.It Bq Er EDQUOT
-The directory in which the entry for the new name
-is being placed cannot be extended because the
-user's quota of disk blocks on the file system
-containing the directory has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while making or updating a directory entry.
+.\" ===========
 .It Bq Er EROFS
 The requested link requires writing in a directory on a read-only file
 system.
-.It Bq Er EFAULT
-.Em Path
-points outside the process's allocated address space.
-.It Bq Er EINVAL
-.Fa From
-is a parent directory of
-.Fa to ,
-or an attempt is made to rename
-.Ql \&.
-or
-.Ql \&.. .
-.It Bq Er ENOTEMPTY
-.Fa To
-is a directory and is not empty.
+.\" ===========
+.It Bq Er EXDEV
+The link named by
+.Fa new
+and the file named by
+.Fa old
+are on different logical devices (file systems).
+Note that this error code will not be returned
+if the implementation permits cross-device links.
 .El
+.Sh CONFORMANCE
+The restriction on renaming a directory whose permissions disallow writing
+is based on the fact that UFS directories contain a ".." entry.
+If renaming a directory would move it to another parent directory,
+this entry needs to be changed.
+.Pp
+This restriction has been generalized to disallow renaming
+of any write-disabled directory,
+even when this would not require a change to the ".." entry.
+For consistency, HFS+ directories emulate this behavior.
 .Sh SEE ALSO
-.Xr open 2
+.Xr open 2 ,
 .Xr symlink 7
 .Sh STANDARDS
 The
diff --git a/bsd/man/man2/rmdir.2 b/bsd/man/man2/rmdir.2
index 8a8d595b2..5a47ed7b7 100644
--- a/bsd/man/man2/rmdir.2
+++ b/bsd/man/man2/rmdir.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn rmdir "const char *path"
+.Fo rmdir
+.Fa "const char *path"
+.Fc
 .Sh DESCRIPTION
 .Fn Rmdir
 removes a directory file
@@ -60,44 +62,57 @@ returned and an error code is stored in the global location
 .Sh ERRORS
 The named file is removed unless:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path is not a directory.
+.\" ===========
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.\" ===========
+.It Bq Er EACCES
+Write permission is denied on the directory containing the link
+to be removed.
+.\" ===========
+.It Bq Er EBUSY
+The directory to be removed is the mount point
+for a mounted file system.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while deleting the directory entry
+or deallocating the inode.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
+A component of a pathname (possibly expanded by a symbolic link) exceeds 
 .Dv {NAME_MAX}
 characters, or an entire path name exceeded 
 .Dv {PATH_MAX}
 characters.
+.\" ===========
 .It Bq Er ENOENT
 The named directory does not exist.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path is not a directory.
+.\" ===========
 .It Bq Er ENOTEMPTY
 The named directory contains files other than
 .Ql \&.
 and
 .Ql \&..
 in it.
-.It Bq Er EACCES
-Search permission is denied for a component of the path prefix.
-.It Bq Er EACCES
-Write permission is denied on the directory containing the link
-to be removed.
+.\" ===========
 .It Bq Er EPERM
 The directory containing the directory to be removed is marked sticky,
 and neither the containing directory nor the directory to be removed
 are owned by the effective user ID.
-.It Bq Er EBUSY
-The directory to be removed is the mount point
-for a mounted file system.
-.It Bq Er EIO
-An I/O error occurred while deleting the directory entry
-or deallocating the inode.
+.\" ===========
 .It Bq Er EROFS
 The directory entry to be removed resides on a read-only file system.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
 .El
 .Sh SEE ALSO
 .Xr mkdir 2 ,
diff --git a/bsd/man/man2/select.2 b/bsd/man/man2/select.2
index f4c446606..eeb8aceb8 100644
--- a/bsd/man/man2/select.2
+++ b/bsd/man/man2/select.2
@@ -37,28 +37,55 @@
 .Dt SELECT 2
 .Os BSD 4.2
 .Sh NAME
+.Nm FD_CLR ,
+.Nm FD_COPY ,
+.Nm FD_ISSET ,
+.Nm FD_SET ,
+.Nm FD_ZERO ,
 .Nm select
 .Nd synchronous I/O multiplexing
 .Sh SYNOPSIS
 .Fd #include <sys/select.h>
-.D1 "- or -"
-.Fd #include <sys/types.h>
-.Fd #include <sys/time.h>
-.Fd #include <unistd.h>
+.\"
+.Ft void
+.Fo FD_CLR
+.Fa fd
+.Fa "fd_set *fdset"
+.Fc
+.Ft void
+.Fo FD_COPY
+.Fa "fd_set *fdset_orig"
+.Fa "fd_set *fdset_copy"
+.Fc
 .Ft int
-.Fn select "int nfds" "fd_set *readfds" "fd_set *writefds" "fd_set *exceptfds" "struct timeval *timeout"
-.Fn FD_SET fd &fdset
-.Fn FD_CLR fd &fdset
-.Fn FD_ISSET fd &fdset
-.Fn FD_COPY &fdset_orig &fdset_copy
-.Fn FD_ZERO &fdset
+.Fo FD_ISSET
+.Fa fd
+.Fa "fd_set *fdset"
+.Fc
+.Ft void
+.Fo FD_SET
+.Fa fd
+.Fa "fd_set *fdset"
+.Fc
+.Ft void
+.Fo FD_ZERO
+.Fa "fd_set *fdset"
+.Fc
+.Ft int
+.Fo select
+.Fa "int nfds"
+.Fa "fd_set *restrict readfds"
+.Fa "fd_set *restrict writefds"
+.Fa "fd_set *restrict errorfds"
+.Fa "struct timeval *restrict timeout"
+.Fc
 .Sh DESCRIPTION
 .Fn Select
 examines the I/O descriptor sets whose addresses are passed in
 .Fa readfds ,
 .Fa writefds ,
 and
-.Fa exceptfds
+.Fa errorfds
 to see if some of their descriptors
 are ready for reading, are ready for writing, or have an exceptional
 condition pending, respectively.
@@ -130,7 +157,7 @@ Any of
 .Fa readfds ,
 .Fa writefds ,
 and
-.Fa exceptfds
+.Fa errorfds
 may be given as nil pointers if no descriptors are of interest.
 .Sh RETURN VALUES
 .Fn Select
@@ -144,21 +171,74 @@ If
 .Fn select
 returns with an error,
 including one due to an interrupted call,
-the descriptor sets will be unmodified.
+the descriptor sets will be unmodified and the global variable
+.Va errno
+will be set to indicate the error.
 .Sh ERRORS
 An error return from
 .Fn select
 indicates:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EAGAIN
+The kernel was (perhaps temporarily) unable
+to allocate the requested number of file descriptors.
+.\" ===========
 .It Bq Er EBADF
 One of the descriptor sets specified an invalid descriptor.
+.\" ===========
 .It Bq Er EINTR
 A signal was delivered before the time limit expired and
 before any of the selected events occurred.
+.\" ===========
 .It Bq Er EINVAL
 The specified time limit is invalid.  One of its components is
 negative or too large.
+.\" ===========
+.It Bq Er EINVAL
+.Fa ndfs
+is greater than FD_SETSIZE and _DARWIN_UNLIMITED_SELECT is not defined.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/select.h>
+.D1 "- or -"
+.Fd #include <sys/types.h>
+.Fd #include <sys/time.h>
+.Fd #include <unistd.h>
+.Pp
+.Fo FD_SET
+.Fa fd
+.Fa &fdset
+.Fc ;
+.Pp
+.Fo FD_CLR
+.Fa fd
+.Fa &fdset
+.Fc ;
+.Pp
+.Fo FD_ISSET
+.Fa fd
+.Fa &fdset
+.Fc ;
+.Pp
+.Fo FD_COPY
+.Fa &fdset_orig
+.Fa &fdset_copy
+.Fc ;
+.Pp
+.Fo FD_ZERO
+.Fa &fdset
+.Fc ;
+.Sh COMPATIBILITY
+.Fn select
+now returns with
+.Va errno
+set to EINVAL when 
+.Fa nfds  
+is greater than FD_SETSIZE.
+Use a smaller value for
+.Fa nfds  
+or compile with -D_DARWIN_UNLIMITED_SELECT.
 .Sh SEE ALSO
 .Xr accept 2 ,
 .Xr connect 2 ,
@@ -167,7 +247,8 @@ negative or too large.
 .Xr read 2 ,
 .Xr recv 2 ,
 .Xr send 2 ,
-.Xr write 2
+.Xr write 2 ,
+.Xr compat 5
 .Sh BUGS
 Although the provision of
 .Xr getdtablesize 2
diff --git a/bsd/man/man2/semctl.2 b/bsd/man/man2/semctl.2
index f5d6e8c32..b76b40d34 100644
--- a/bsd/man/man2/semctl.2
+++ b/bsd/man/man2/semctl.2
@@ -32,11 +32,14 @@
 .Nm semctl
 .Nd control operations on a semaphore set
 .Sh SYNOPSIS
-.In sys/types.h
-.In sys/ipc.h
 .In sys/sem.h
 .Ft int
-.Fn semctl "int semid" "int semnum" "int cmd" ...
+.Fo semctl
+.Fa "int semid"
+.Fa "int semnum"
+.Fa "int cmd"
+.Fa ...
+.Fc
 .Sh DESCRIPTION
 The
 .Fn semctl
@@ -171,20 +174,25 @@ The
 system call
 will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+Permission denied due to mismatch between operation and mode of
+semaphore set.
+.\" ===========
 .It Bq Er EINVAL
 No semaphore set corresponds to
 .Fa semid .
+.\" ===========
 .It Bq Er EINVAL
 The
 .Fa semnum
 argument
 is not in the range of valid semaphores for given semaphore set.
+.\" ===========
 .It Bq Er EPERM
 The calling process's effective uid does not match the uid of
 the semaphore set's owner or creator.
-.It Bq Er EACCES
-Permission denied due to mismatch between operation and mode of
-semaphore set.
+.\" ===========
 .It Bq Er ERANGE
 .Dv SETVAL
 or
@@ -192,9 +200,20 @@ or
 attempted to set a semaphore outside the allowable range
 .Bq 0 .. Dv SEMVMX .
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/sem.h>
+.Pp
+The include files
+.In sys/types.h
+and
+.In sys/ipc.h
+are necessary.
 .Sh SEE ALSO
 .Xr semget 2 ,
-.Xr semop 2
+.Xr semop 2 ,
+.Xr compat 5
 .Sh BUGS
 .Dv SETALL
 may update some semaphore elements before returning an error.
diff --git a/bsd/man/man2/semget.2 b/bsd/man/man2/semget.2
index 8705b29e1..03f30464f 100644
--- a/bsd/man/man2/semget.2
+++ b/bsd/man/man2/semget.2
@@ -32,16 +32,18 @@
 .Nm semget
 .Nd obtain a semaphore id
 .Sh SYNOPSIS
-.In sys/types.h
-.In sys/ipc.h
 .In sys/sem.h
 .Ft int
-.Fn semget "key_t key" "int nsems" "int flag"
+.Fo semget
+.Fa "key_t key"
+.Fa "int nsems"
+.Fa "int semflg"
+.Fc
 .Sh DESCRIPTION
 Based on the values of
 .Fa key
 and
-.Fa flag ,
+.Fa semflg ,
 .Fn semget
 returns the identifier of a newly created or previously existing
 set of semaphores.
@@ -61,7 +63,7 @@ An integer constant may be specified.  If no IPC object corresponding
 to
 .Fa key
 is specified and the IPC_CREAT bit is set in
-.Fa flag ,
+.Fa semflg ,
 a new one will be created.
 .It
 The
@@ -76,7 +78,7 @@ may be used to generate a key from a pathname.
 The mode of a newly created IPC object is determined by
 .Em OR Ns 'ing
 the following constants into the
-.Fa flag
+.Fa semflg
 argument:
 .Bl -tag -width XSEM_WXX6XXX
 .It Dv SEM_R
@@ -113,32 +115,58 @@ The
 system call
 will fail if:
 .Bl -tag -width Er
+.\" ===========
 .\" ipcperm could fail (we're opening to read and write, as it were)
 .It Bq Er EACCES
 Access permission failure.
+.\" ===========
 .\"
-.\" sysv_sem.c is quite explicit about these, so I'm pretty sure
-.\" this is accurate
+.\" sysv_sem.c is quite explicit about these,
+.\" so I'm pretty sure this is accurate
 .\"
 .It Bq Er EEXIST
-IPC_CREAT and IPC_EXCL were specified, and a semaphore set
-corresponding to
+IPC_CREAT and IPC_EXCL were specified,
+and a semaphore set corresponding to
 .Fa key
 already exists.
+.\" ===========
+.It Bq Er EINVAL
+The number of semaphores requested
+is either less than 1 or greater than the system imposed maximum per set.
+.\" ===========
 .It Bq Er EINVAL
-The number of semaphores requested exceeds the system imposed maximum
-per set.
+A semaphore identifier exists for the argument key,
+but the number of semaphores in the set associated with it
+is less than
+.Fa nsems ,
+and
+.Fa nsems
+is non-zero.
+.\" ===========
+.It Bq Er ENOENT
+No semaphore set was found corresponding to
+.Fa key ,
+and IPC_CREAT was not specified.
+.\" ===========
 .It Bq Er ENOSPC
 Insufficiently many semaphores are available.
+.\" ===========
 .It Bq Er ENOSPC
 The kernel could not allocate a
 .Fa "struct semid_ds" .
-.It Bq Er ENOENT
-No semaphore set was found corresponding to
-.Fa key ,
-and IPC_CREAT was not specified.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/sem.h>
+.Pp
+The include files
+.In sys/types.h
+and
+.In sys/ipc.h
+are necessary.
 .Sh SEE ALSO
 .Xr semctl 2 ,
 .Xr semop 2 ,
-.Xr ftok 3
+.Xr ftok 3 ,
+.Xr compat 5
diff --git a/bsd/man/man2/semop.2 b/bsd/man/man2/semop.2
index 82701cd74..8009d1659 100644
--- a/bsd/man/man2/semop.2
+++ b/bsd/man/man2/semop.2
@@ -32,23 +32,25 @@
 .Nm semop
 .Nd atomic array of operations on a semaphore set
 .Sh SYNOPSIS
-.In sys/types.h
-.In sys/ipc.h
 .In sys/sem.h
 .Ft int
-.Fn semop "int semid" "struct sembuf *array" "size_t nops"
+.Fo semop
+.Fa "int semid"
+.Fa "struct sembuf *sops"
+.Fa "size_t nsops"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn semop
 system call
 atomically performs the array of operations indicated by
-.Fa array
+.Fa sops
 on the semaphore set indicated by
 .Fa semid .
 The length of
-.Fa array
+.Fa sops
 is indicated by
-.Fa nops .
+.Fa nsops .
 Each operation is encoded in a
 .Vt "struct sembuf" ,
 which is defined as follows:
@@ -64,7 +66,7 @@ struct sembuf {
 .Ed
 .Pp
 For each element in
-.Fa array ,
+.Fa sops ,
 .Va sem_op
 and
 .Va sem_flg
@@ -229,55 +231,75 @@ The
 .Fn semop
 system call will fail if:
 .Bl -tag -width Er
-.It Bq Er EINVAL
-No semaphore set corresponds to
-.Fa semid ,
-or the process would exceed the system-defined limit for the number of
-per-process
-.Dv SEM_UNDO
-structures.
+.\" ===========
+.It Bq Er E2BIG
+Too many operations are specified.
+.Bq Dv SEMOPM
+.\" ===========
 .It Bq Er EACCES
-Permission denied due to mismatch between operation and mode of
-semaphore set.
+Permission is denied, due to a mismatch between the operation
+and the mode of the semaphore set.
+.\" ===========
 .It Bq Er EAGAIN
-The semaphore's value would have resulted in the process being put to sleep
-and
+The semaphore's value would result
+in the process being put to sleep and
 .Dv IPC_NOWAIT
-was specified.
-.It Bq Er E2BIG
-Too many operations were specified.
-.Bq Dv SEMOPM
+is specified.
+.\" ===========
 .It Bq Er EFBIG
 .\"
-.\" I'd have thought this would be EINVAL, but the source says
-.\" EFBIG.
+.\" I'd have thought this would be EINVAL,
+.\" but the source says EFBIG.
 .\"
 .Va sem_num
-was not in the range of valid semaphores for the set.
+is not in the range of valid semaphores for the set.
+.\" ===========
 .It Bq Er EIDRM
-The semaphore set was removed from the system.
+The semaphore set is removed from the system.
+.\" ===========
 .It Bq Er EINTR
 The
 .Fn semop
-system call was interrupted by a signal.
+system call is interrupted by a signal.
+.\" ===========
+.It Bq Er EINVAL
+No semaphore set corresponds to
+.Fa semid ,
+or the process would exceed the system-defined limit
+for the number of per-process
+.Dv SEM_UNDO
+structures.
+.\" ===========
 .It Bq Er ENOSPC
 The system
 .Dv SEM_UNDO
 pool
 .Bq Dv SEMMNU
 is full.
+.\" ===========
 .It Bq Er ERANGE
 The requested operation would cause either
 the semaphore's current value
 .Bq Dv SEMVMX
-or its adjust on exit value
+or its adjust-on-exit value
 .Bq Dv SEMAEM
 to exceed the system-imposed limits.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/sem.h>
+.Pp
+The include files
+.In sys/types.h
+and
+.In sys/ipc.h
+are necessary.
 .Sh SEE ALSO
 .Xr semctl 2 ,
 .Xr semget 2 ,
-.Xr sigaction 2
+.Xr sigaction 2 ,
+.Xr compat 5
 .Sh BUGS
 The
 .Fn semop
diff --git a/bsd/man/man2/send.2 b/bsd/man/man2/send.2
index 36bf6c6c8..9b6346372 100644
--- a/bsd/man/man2/send.2
+++ b/bsd/man/man2/send.2
@@ -38,18 +38,33 @@
 .Os BSD 4.2
 .Sh NAME
 .Nm send ,
-.Nm sendto ,
-.Nm sendmsg
+.Nm sendmsg ,
+.Nm sendto
 .Nd send a message from a socket
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft ssize_t
-.Fn send "int s" "const void *msg" "size_t len" "int flags"
+.Fo send
+.Fa "int socket"
+.Fa "const void *buffer"
+.Fa "size_t length"
+.Fa "int flags"
+.Fc
 .Ft ssize_t
-.Fn sendto "int s" "const void *msg" "size_t len" "int flags" "const struct sockaddr *to" "socklen_t tolen"
+.Fo sendmsg
+.Fa "int socket"
+.Fa "const struct msghdr *buffer"
+.Fa "int flags"
+.Fc
 .Ft ssize_t
-.Fn sendmsg "int s" "const struct msghdr *msg" "int flags"
+.Fo sendto
+.Fa "int socket"
+.Fa "const void *buffer"
+.Fa "size_t length"
+.Fa "int flags"
+.Fa "const struct sockaddr *dest_addr"
+.Fa "socklen_t dest_len"
+.Fc
 .Sh DESCRIPTION
 .Fn Send ,
 .Fn sendto ,
@@ -66,12 +81,12 @@ and
 may be used at any time.
 .Pp
 The address of the target is given by
-.Fa to
+.Fa dest_addr
 with 
-.Fa tolen
+.Fa dest_len
 specifying its size.
 The length of the message is given by
-.Fa len .
+.Fa length .
 If the message is too long to pass atomically through the
 underlying protocol, the error
 .Er EMSGSIZE
@@ -118,49 +133,147 @@ for a description of the
 .Fa msghdr
 structure.
 .Sh RETURN VALUES
-The call returns the number of characters sent, or -1
-if an error occurred.
+Upon successful completion,
+the number of bytes which were sent is returned.
+Otherwise, -1 is returned and the global variable
+.Va errno
+is set to indicate the error.
 .Sh ERRORS
-.Fn Send ,
-.Fn sendto ,
+The
+.Fn send ,
+.Fn sendmsg ,
 and
-.Fn sendmsg
-fail if:
+.Fn sendto
+system calls will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+The SO_BROADCAST option is not set on the socket
+and a broadcast address is given as the destination.
+.\" ===========
+.It Bq Er EAGAIN
+The socket is marked non-blocking
+and the requested operation would block.
+.\" ===========
 .It Bq Er EBADF
-An invalid descriptor was specified.
-.It Bq Er ENOTSOCK
-The argument
-.Fa s
-is not a socket.
+An invalid descriptor is specified.
+.\" ===========
+.It Bq Er ECONNRESET
+A connection is forcibly closed by a peer.
+.\" ===========
 .It Bq Er EFAULT
-An invalid user space address was specified for a parameter.
+An invalid user space address is specified for a parameter.
+.\" ===========
+.It Bq Er EHOSTUNREACH
+The destination address specifies an unreachable host.
+.\" ===========
+.It Bq Er EINTR
+A signal interrupts the system call
+before any data is transmitted.
+.\" ===========
 .It Bq Er EMSGSIZE
 The socket requires that message be sent atomically,
-and the size of the message to be sent made this impossible.
-.It Bq Er EAGAIN
-The socket is marked non-blocking and the requested operation
-would block.
+and the size of the message to be sent makes this impossible.
+.\" ===========
+.It Bq Er ENETDOWN
+The local network interface used to reach the destination is down.
+.\" ===========
+.It Bq Er ENETUNREACH
+No route to the network is present.
+.\" ===========
 .It Bq Er ENOBUFS
-The system was unable to allocate an internal buffer.
+The system is unable to allocate an internal buffer.
 The operation may succeed when buffers become available.
+.\" ===========
 .It Bq Er ENOBUFS
-The output queue for a network interface was full.
+The output queue for a network interface is full.
 This generally indicates that the interface has stopped sending,
 but may be caused by transient congestion.
-.It Bq Er EACCES
-The SO_BROADCAST option is not set on the socket, and a broadcast address
-was given as the destination.
-.It Bq Er EHOSTUNREACH
-The destination address specified an unreachable host.
+.\" ===========
+.It Bq Er ENOTSOCK
+The argument
+.Fa socket
+is not a socket.
+.\" ===========
+.It Bq Er EOPNOTSUPP
+.Fa socket
+does not support (some of) the option(s) specified in
+.Fa flags . 
+.\" ===========
+.It Bq Er EPIPE
+The socket is shut down for writing
+or the socket is connection-mode and is no longer connected.
+In the latter case, and if the socket is of type SOCK_STREAM,
+the SIGPIPE signal is generated to the calling thread.
+.El
+.Pp
+The
+.Fn sendmsg
+and
+.Fn sendto
+system calls will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EAFNOSUPPORT
+Addresses in the specified address family cannot be used
+with this socket.
+.\" ===========
+.It Bq Er EDESTADDRREQ
+The socket is not connection-mode and does not have its peer address set,
+and no destination address is specified.
+.\" ===========
+.It Bq Er EISCONN
+A destination address was specified and the socket is already connected.
+.\" ===========
+.It Bq Er ENOENT
+A component of the pathname does not name an existing file
+or the path name is an empty string.
+.\" ===========
+.It Bq Er ENOMEM
+Insufficient memory is available to fulfill the request.
+.\" ===========
+.It Bq Er ENOTCONN
+The socket is connection-mode, but is not connected.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix of the pathname in the socket address
+is not a directory.
+.El
+.Pp
+The
+.Fn send
+system call will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EDESTADDRREQ
+The socket is not connection-mode and no peer address is set.
+.\" ===========
+.It Bq Er ENOTCONN
+The socket is not connected or otherwise has not had the peer pre-specified.
 .El
+.Pp
+The
+.Fn sendmsg
+system call will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+The sum of the iov_len values overflows an ssize_t.
+.El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h is necessary.
 .Sh SEE ALSO
 .Xr fcntl 2 ,
+.Xr getsockopt 2 ,
 .Xr recv 2 ,
 .Xr select 2 ,
-.Xr getsockopt 2 ,
 .Xr socket 2 ,
-.Xr write 2
+.Xr write 2 ,
+.Xr compat 5
 .Sh HISTORY
 The
 .Fn send
diff --git a/bsd/man/man2/sendfile.2 b/bsd/man/man2/sendfile.2
new file mode 100644
index 000000000..8e31581e2
--- /dev/null
+++ b/bsd/man/man2/sendfile.2
@@ -0,0 +1,236 @@
+.\" Copyright (c) 2003, David G. Lawrence
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice unmodified, this list of conditions, and the following
+.\"    disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"
+.Dd March 31, 2006
+.Dt SENDFILE 2
+.Os "Mac OS X"
+.Sh NAME
+.Nm sendfile
+.Nd send a file to a socket
+.Sh SYNOPSIS
+.In sys/types.h
+.In sys/socket.h
+.In sys/uio.h
+.Ft int
+.Fo sendfile
+.Fa "int fd" "int s" "off_t offset" "off_t *len" 
+.Fa "struct sf_hdtr *hdtr" "int flags"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn sendfile
+system call
+sends a regular file specified by descriptor
+.Fa fd
+out a stream socket specified by descriptor
+.Fa s .
+.Pp
+The
+.Fa offset
+argument specifies where to begin in the file.
+Should
+.Fa offset
+fall beyond the end of file, the system will return
+success and report 0 bytes sent as described below.
+.Pp
+The
+.Fa len
+argument is a value-result parameter, that specifies how many bytes 
+of the file should be sent and/or how many bytes have been sent.
+Initially the value pointed to by the 
+.Fa len
+argument specifies how many bytes should be sent with 0 having the special
+meaning to send until the end of file has been reached.
+On return the value pointed to by the 
+.Fa len
+argument indicates how many bytes have been sent.
+The 
+.Fa len
+pointer may not be NULL.
+.Pp
+An optional header and/or trailer can be sent before and after the file data by 
+specifying a pointer to a
+.Vt "struct sf_hdtr" ,
+which has the following structure:
+.Pp
+.Bd -literal -offset indent -compact
+struct sf_hdtr {
+	struct iovec *headers;	/* pointer to header iovecs */
+	int hdr_cnt;		/* number of header iovecs */
+	struct iovec *trailers;	/* pointer to trailer iovecs */
+	int trl_cnt;		/* number of trailer iovecs */
+};
+.Ed
+.Pp
+The
+.Fa headers
+and
+.Fa trailers
+pointers, if
+.Pf non- Dv NULL ,
+point to arrays of
+.Vt "struct iovec"
+structures.
+See the
+.Fn writev
+system call for information on the iovec structure.
+The number of iovecs in these
+arrays is specified by
+.Fa hdr_cnt
+and
+.Fa trl_cnt .
+.Pp
+The
+.Fa flags
+parameter is reserved for future expansion and must be set to 0. Any other value 
+will cause
+.Fn sendfile
+to return
+.Er EINVAL .
+.Pp
+When using a socket marked for non-blocking I/O,
+.Fn sendfile
+may send fewer bytes than requested.
+In this case, the number of bytes successfully
+sent is returned in the via the 
+.Fa len
+parameters and the error
+.Er EAGAIN
+is returned.
+.Pp
+When a signal causes 
+.Fn sendfile
+to return the error 
+.Er EINTR ,
+the 
+.Fa len 
+argument may return 0 without necessarily meaning the end of file has been reached
+as the signal may have been caught before any data was sent.
+.Sh IMPLEMENTATION NOTES
+The
+Mac OS X 
+implementation of 
+.Fn sendfile
+uses 64 bits types for size and offset parameters so there is no need for 
+a 64 bits version 
+.Fn sendfile64
+as found on some other operating systems.
+.Sh RETURN VALUES
+.Rv -std sendfile
+.Pp
+The number of bytes sent is returned via the parameter
+.Fa len .
+A value of 0 means the end of the file specified by descriptor 
+.Fa fd
+has been reached or that the value passed in 
+.Fa offset 
+falls beyond the end of file.
+.Sh ERRORS
+.Bl -tag -width Er
+.It Bq Er EAGAIN
+The socket is marked for non-blocking I/O and not all data was sent due to
+the socket buffer being full.
+If specified, the number of bytes successfully sent will be returned in
+.Fa *len .
+.It Bq Er EBADF
+The
+.Fa fd
+argument
+is not a valid file descriptor.
+.It Bq Er ENOTSUP
+The
+.Fa fd
+argument
+does not refer to a regular file.
+.It Bq Er EBADF
+The
+.Fa s
+argument
+is not a valid socket descriptor.
+.It Bq Er ENOTSOCK
+The
+.Fa s
+argument
+does not refer stream oriented socket.
+.It Bq Er EFAULT
+An invalid address was specified for an argument.
+.It Bq Er EINTR
+A signal interrupted
+.Fn sendfile
+before it could be completed.
+If specified, the number
+of bytes successfully sent will be returned in
+.Fa *len .
+.It Bq Er EINVAL
+The
+.Fa offset
+argument
+is negative.
+.It Bq Er EINVAL
+The
+.Fa len
+argument
+is NULL.
+.It Bq Er EINVAL
+The
+.Fa flags
+argument
+is not set to 0.
+.It Bq Er EIO
+An error occurred while reading from
+.Fa fd .
+.It Bq Er ENOTCONN
+The
+.Fa s
+argument
+points to an unconnected socket.
+.It Bq Er ENOTSOCK
+The
+.Fa s
+argument
+is not a socket.
+.It Bq Er EOPNOTSUPP
+The file system for descriptor
+.Fa fd
+does not support
+.Fn sendfile .
+.It Bq Er EPIPE
+The socket peer has closed the connection.
+.El
+.Sh SEE ALSO
+.Xr open 2 ,
+.Xr send 2 ,
+.Xr socket 2 ,
+.Xr writev 2
+.Sh HISTORY
+The
+.Fn sendfile
+system call
+first appeared in
+Darwin 9.0 (Mac OS X version 10.5) .
+.Sh AUTHORS
+This manual page is based on the FreeBSD version written by
+.An David G. Lawrence Aq dg@dglawrence.com 
diff --git a/bsd/man/man2/setaudit.2 b/bsd/man/man2/setaudit.2
new file mode 100644
index 000000000..c0e739cc7
--- /dev/null
+++ b/bsd/man/man2/setaudit.2
@@ -0,0 +1,56 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt SETAUDIT 2
+.Os Darwin
+.Sh NAME
+.Nm setaudit
+.Nd set the audit information for the current process  
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn setaudit "const struct auditinfo * auditinfo"
+.Sh DESCRIPTION
+The
+.Fn setaudit
+function sets the audit information for the current process. 
+.Fa auditinfo
+should point at a 
+.Fa struct auditinfo 
+describing the requested user audit settings.
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditon 2 ,
+.Xr auditctl 2 ,
+.Xr getauid 2 ,
+.Xr setauid 2 ,
+.Xr getaudit 2
+.Sh HISTORY
+The
+.Fn setaudit
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/setauid.2 b/bsd/man/man2/setauid.2
new file mode 100644
index 000000000..0fe8e9df4
--- /dev/null
+++ b/bsd/man/man2/setauid.2
@@ -0,0 +1,54 @@
+.\"
+.\" Copyright (c) 2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this
+.\" file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_LICENSE_HEADER_END@
+.\"
+.Dd July 30, 2007
+.Dt SETAUID 2
+.Os Darwin
+.Sh NAME
+.Nm setauid
+.Nd set the audit user ID for the calling process
+.Sh SYNOPSIS
+.Fd #include <bsm/audit.h>
+.Ft int
+.Fn setauid "const au_id_t * auid"
+.Sh DESCRIPTION
+The
+.Fn setauid
+function sets the audit user ID for the calling process. The ID is set to the 
+value pointed at by 
+.Fa auid .
+.Sh RETURN VALUES
+Upon successful completion a value of 0 is returned.
+Otherwise, a value of -1 is returned and
+.Va errno
+is set to indicate the error.
+.Sh SEE ALSO
+.Xr audit 2 ,
+.Xr auditon 2 ,
+.Xr auditctl 2 ,
+.Xr getauid 2 ,
+.Xr getaudit 2 ,
+.Xr setaudit 2
+.Sh HISTORY
+The
+.Fn setauid
+function call first appeared in Mac OS X 10.3 (Panther).
diff --git a/bsd/man/man2/setlcid.2 b/bsd/man/man2/setlcid.2
new file mode 100644
index 000000000..a564bfd3e
--- /dev/null
+++ b/bsd/man/man2/setlcid.2
@@ -0,0 +1,90 @@
+.\" Copyright (c) 2005 SPARTA, Inc.
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.\" Note: The date here should be updated whenever a non-trivial
+.\" change is made to the manual page.
+.Dd May 5, 2005
+.Dt SETLCID 3
+.Os
+.Sh NAME
+.Nm setlcid
+.Nd "set login context"
+.Sh SYNOPSIS
+.In sys/lctx.h 
+.Ft int
+.Fn setlcid "pid_t pid" "pid_t lcid"
+.Sh DESCRIPTION
+The
+.Fn setlcid
+system call sets the login context of the specified process
+.Fa pid
+to the specified
+.Fa lcid .
+If the
+.Fa pid
+is zero, then the call applies to the current process.
+If the
+.Fa lcid
+is zero a new login context will be created.
+If the
+.Fa lcid
+is \-1 the process will be removed from the login context
+it is currently a member of, if any.
+.Pp
+Creation of a new login context is only valid for the current process.
+A process may not create a new login context if it is currently a member
+of one.
+.Pp
+Superuser privilege is required to add or remove a process from
+a login context.
+.Sh RETURN VALUES
+.Rv -std setlcid
+.Sh ERRORS
+The
+.Fn setlcid
+function will fail if:
+.Bl -tag -width Er
+.It Bq Er EPERM
+Operation not permitted.
+.It Bq Er ESRCH
+No such process.
+.It Bq Er ENOMEM
+Cannot allocate memory.
+.It Bq Er ENOATTR
+Attribute not found.
+.El
+.Sh SEE ALSO
+.Xr getlcid 2
+.Sh HISTORY
+The
+.Nm
+manual page
+first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+This
+manual page was written by
+.An Matthew N. Dodd Aq mdodd@FreeBSD.org .
diff --git a/bsd/man/man2/setpgid.2 b/bsd/man/man2/setpgid.2
index 7767b23fc..36d82683a 100644
--- a/bsd/man/man2/setpgid.2
+++ b/bsd/man/man2/setpgid.2
@@ -43,15 +43,20 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn setpgid "pid_t pid" "pid_t pgrp"
-.Ft int
-.Fn setpgrp "pid_t pid" "pid_t pgrp"
+.Fo setpgid
+.Fa "pid_t pid"
+.Fa "pid_t pgid"
+.Fc
+.Ft pid_t
+.Fo setpgrp
+.Fa void
+.Fc
 .Sh DESCRIPTION
 .Fn Setpgid
 sets the process group of the specified process
 .Ar pid
 to the specified
-.Ar pgrp .
+.Ar pgid .
 If
 .Ar pid
 is zero, then the call applies to the current process.
@@ -59,6 +64,12 @@ is zero, then the call applies to the current process.
 If the invoker is not the super-user, then the affected process
 must have the same effective user-id as the invoker or be a descendant
 of the invoking process.
+.Pp
+If the calling process is not already a session leader,
+.Fn setpgrp
+sets the process group ID of the calling process
+to that of the calling process.
+Any new session that this creates will have no controlling terminal.
 .Sh RETURN VALUES
 .Fn Setpgid
 returns 0 when the operation was successful.
@@ -69,32 +80,81 @@ indicates the reason.
 .Fn Setpgid
 will fail and the process group will not be altered if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EACCES
 The value of the
 .Fa pid
 argument matches the process ID of a child process of the calling process, 
 and the child process has successfully executed one of the exec functions.
+.\" ===========
+.It Bq Er EINVAL
+The value of the
+.Fa pgid
+argument is less than 0
+or is not a value supported by the implementation.
+.\" ===========
+.It Bq Er EPERM
+The process indicated by the
+.Fa pid
+argument is a session leader.
+.\" ===========
 .It Bq Er EPERM
 The effective user ID of the requested process is different
 from that of the caller and the process is not a descendant
 of the calling process.
+.\" ===========
+.It Bq Er EPERM
+The value of the
+.Fa pgid argument
+is valid, but does not match the process ID
+of the process indicated by the
+.Fa pid
+argument and there is no process with a process group ID
+that matches the value of the
+.Fa pgid
+argument in the same session as the calling process.
+.\" ===========
 .It Bq Er ESRCH
 The value of the 
 .Fa pid 
 argument does not match the process ID of the calling process or of a 
 child process of the calling process.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <unistd.h>
+.Pp
+.Ft int
+.br
+.Fo setpgrp
+.Fa "pid_t pid"
+.Fa "pid_t pgid"
+.Fc ;
+.Pp
+The legacy
+.Fn setpgrp
+function is a clone of the
+.Fn setpgid
+function, retained for calling convention compatibility
+with historical versions of
+.Bx .
+.Sh COMPATIBILITY
+Use of the legacy version of the
+.Fn setpgrp
+call will cause compiler diagnostics.
+Use
+.Fn setpgid
+instead.
+.Pp
+Use of private (and conflicting) prototypes for
+.Fn setpgrp
+will cause compiler diagnostics.
+Delete the private prototypes and include
+.In unistd.h .
 .Sh SEE ALSO
-.Xr getpgrp 2
+.Xr getpgrp 2 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn setpgid
 function conforms to 
 .St -p1003.1-88 .
-.Sh COMPATIBILITY
-.Fn Setpgrp
-is identical to
-.Fn setpgid ,
-and is retained for calling convention compatibility with historical
-versions of
-.Bx .
diff --git a/bsd/man/man2/setsid.2 b/bsd/man/man2/setsid.2
index 8f9018b57..a7fc304b8 100644
--- a/bsd/man/man2/setsid.2
+++ b/bsd/man/man2/setsid.2
@@ -40,10 +40,11 @@
 .Nm setsid
 .Nd create session and set process group ID
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft pid_t
-.Fn setsid "void"
+.Fo setsid
+.Fa void
+.Fc
 .Sh DESCRIPTION
 The
 .Nm setsid
@@ -70,10 +71,18 @@ The calling process is already a process group leader, or the process
 group ID of a process other than the calling process matches the process
 ID of the calling process.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr setpgid 3 ,
 .Xr tcgetpgrp 3 ,
-.Xr tcsetpgrp 3
+.Xr tcsetpgrp 3 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Nm setsid
diff --git a/bsd/man/man2/setuid.2 b/bsd/man/man2/setuid.2
index b271097c8..a49ca1068 100644
--- a/bsd/man/man2/setuid.2
+++ b/bsd/man/man2/setuid.2
@@ -37,22 +37,29 @@
 .Dt SETUID 2
 .Os BSD 4.2
 .Sh NAME
-.Nm setuid ,
+.Nm setegid ,
 .Nm seteuid ,
 .Nm setgid ,
-.Nm setegid
+.Nm setuid
 .Nd set user and group ID
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <unistd.h>
 .Ft int
-.Fn setuid "uid_t uid"
+.Fo setegid
+.Fa "gid_t egid"
+.Fc
 .Ft int
-.Fn seteuid "uid_t euid"
+.Fo seteuid
+.Fa "uid_t euid"
+.Fc
 .Ft int
-.Fn setgid "gid_t gid"
+.Fo setgid
+.Fa "gid_t gid"
+.Fc
 .Ft int
-.Fn setegid "gid_t egid"
+.Fo setuid
+.Fa "uid_t uid"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn setuid
@@ -106,9 +113,38 @@ otherwise \-1 is returned.
 If the user is not the super user, or the uid
 specified is not the real, effective ID, or saved ID,
 these functions return \-1.
+.Pp
+.Sh ERRORS
+The
+.Fn setegid ,
+.Fn seteuid ,
+.Fn setgid ,
+and
+.Fn setuid
+system calls will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+The value of the {group,user} ID argument is invalid
+and is not supported by the implementation.
+.\" ===========
+.It Bq Er EPERM
+The process does not have appropriate privileges and
+the ID argument
+does not match the real ID
+or the saved set-{group,user}-ID.
+.El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <unistd.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary for all functions.
 .Sh SEE ALSO
+.Xr getgid 2 ,
 .Xr getuid 2 ,
-.Xr getgid 2
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn setuid
diff --git a/bsd/man/man2/setxattr.2 b/bsd/man/man2/setxattr.2
index 01b444355..6fe4f86b8 100644
--- a/bsd/man/man2/setxattr.2
+++ b/bsd/man/man2/setxattr.2
@@ -166,8 +166,8 @@ Not enough space left on the file system.
 .El
 .Sh SEE ALSO
 .Xr getxattr 2 ,
-.Xr removexattr 2 ,
-.Xr listxattr 2
+.Xr listxattr 2 ,
+.Xr removexattr 2
 .Sh HISTORY
 .Fn setxattr
 and
diff --git a/bsd/man/man2/shmat.2 b/bsd/man/man2/shmat.2
index be77bdce0..69cb5980f 100644
--- a/bsd/man/man2/shmat.2
+++ b/bsd/man/man2/shmat.2
@@ -38,13 +38,17 @@
 .Nm shmdt
 .Nd map/unmap shared memory
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
-.Fd #include <sys/ipc.h>
 .Fd #include <sys/shm.h>
 .Ft void *
-.Fn shmat "int shmid" "void *shmaddr" "int shmflg"
+.Fo shmat
+.Fa "int shmid"
+.Fa "const void *shmaddr"
+.Fa "int shmflg"
+.Fc
 .Ft int
-.Fn shmdt "void *shmaddr"
+.Fo shmdt
+.Fa "const void *shmaddr"
+.Fc
 .Sh DESCRIPTION
 .Fn shmat
 maps the shared memory segment associated with the shared memory identifier
@@ -86,30 +90,48 @@ and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
+The
 .Fn shmat
-will fail if:
+system call will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EACCES
 The calling process has no permission to access this shared memory segment.
-.It Bq Er ENOMEM
-There is not enough available data space for the calling process to
-map the shared memory segment.
+.\" ===========
 .It Bq Er EINVAL
 .Fa shmid
 is not a valid shared memory identifier.
 .Fa shmaddr
 specifies an illegal address.
+.\" ===========
 .It Bq Er EMFILE
 The number of shared memory segments has reached the system-wide limit.
+.\" ===========
+.It Bq Er ENOMEM
+There is not enough available data space for the calling process to
+map the shared memory segment.
+.Pp
 .El
+The
 .Fn shmdt
-will fail if:
+system call will fail if:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 .Fa shmaddr
 is not the start address of a mapped shared memory segment.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/shm.h>
+.Pp
+The include files
+.In sys/types.h
+and
+.In sys/ipc.h
+are necessary for both functions.
 .Sh SEE ALSO
+.Xr mmap 2 ,
 .Xr shmctl 2 ,
 .Xr shmget 2 ,
-.Xr mmap 2
+.Xr compat 5
diff --git a/bsd/man/man2/shmctl.2 b/bsd/man/man2/shmctl.2
index 6efae862d..0af2016ed 100644
--- a/bsd/man/man2/shmctl.2
+++ b/bsd/man/man2/shmctl.2
@@ -37,11 +37,13 @@
 .Nm shmctl
 .Nd shared memory control operations
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
-.Fd #include <sys/ipc.h>
-.Fd #include <sys/msg.h>
+.Fd #include <sys/shm.h>
 .Ft int
-.Fn shmctl "int shmid" "int cmd" "struct shmid_ds *buf"
+.Fo shmctl
+.Fa "int shmid"
+.Fa "int cmd"
+.Fa "struct shmid_ds *buf"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn shmctl
@@ -57,17 +59,18 @@ This structure is defined as follows in
 .Aq Pa sys/shm.h :
 .Bd -literal
 struct shmid_ds {
-    struct ipc_perm shm_perm;     /* operation permissions */
-    int             shm_segsz;    /* size of segment in bytes */
-    pid_t           shm_lpid;     /* pid of last shm op */
-    pid_t           shm_cpid;     /* pid of creator */
-    short           shm_nattch;   /* # of current attaches */
-    time_t          shm_atime;    /* last shmat() time*/
-    time_t          shm_dtime;    /* last shmdt() time */
-    time_t          shm_ctime;    /* last change by shmctl() */
-    void           *shm_internal; /* sysv stupidity */
+    struct ipc_perm  shm_perm;     /* operation permissions */
+    int              shm_segsz;    /* size of segment in bytes */
+    pid_t            shm_lpid;     /* pid of last shm op */
+    pid_t            shm_cpid;     /* pid of creator */
+    short            shm_nattch;   /* # of current attaches */
+    time_t           shm_atime;    /* last shmat() time*/
+    time_t           shm_dtime;    /* last shmdt() time */
+    time_t           shm_ctime;    /* last change by shmctl() */
+    void            *shm_internal; /* sysv stupidity */
 };
 .Ed
+.Pp
 The
 .Bf -literal
 ipc_perm
@@ -81,15 +84,16 @@ structure is defined in
 and looks like this:
 .Bd -literal
 struct ipc_perm {
-    ushort cuid; /* creator user id */
-    ushort cgid; /* creator group id */
-    ushort uid;	 /* user id */
-    ushort gid;	 /* group id */
-    ushort mode; /* r/w permission (see chmod(2)) */
-    ushort seq;	 /* sequence # (to generate unique msg/sem/shm id) */
-    key_t key;	 /* user specified msg/sem/shm key */
+  uid_t           uid;   /* Owner's user ID */
+  gid_t           gid;   /* Owner's group ID */
+  uid_t           cuid;  /* Creator's user ID */
+  gid_t           cgid;  /* Creator's group ID */
+  mode_t          mode;  /* r/w permission (see chmod(2)) */
+  unsigned short  _seq;  /* Reserved for internal use */
+  key_t           _key;  /* Reserved for internal use */
 };
 .Ed
+.Pp
 The operation to be performed by
 .Fn shmctl
 is specified in
@@ -127,6 +131,7 @@ or
 .Va shm_perm.uid
 values in the data structure associated with the queue can do this.
 .El
+.Pp
 The read and write permissions on a shared memory identifier
 are determined by the
 .Va shm_perm.mode
@@ -143,18 +148,33 @@ effective gid can match either
 or
 .Va shm_perm.gid .
 .Sh RETURN VALUES
-Upon successful completion, a value of 0 is returned. Otherwise, -1 is
-returned and the global variable
+Upon successful completion, a value of 0 is returned.
+Otherwise, -1 is returned and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
 .Fn shmctl
 will fail if:
 .Bl -tag -width Er
+.\" ===========
+.It Bq Er EACCES
+The command is IPC_STAT
+and the caller has no read permission for this shared memory segment.
+.\" ===========
+.It Bq Er EFAULT
+.Fa buf
+specifies an invalid address.
+.\" ===========
+.It Bq Er EINVAL
+.Fa shmid
+is not a valid shared memory segment identifier.
+.Va cmd
+is not a valid command.
+.\" ===========
 .It Bq Er EPERM
 .Fa cmd
-is equal to IPC_SET or IPC_RMID and the caller is not the super-user, nor does
-the effective uid match either the
+is equal to IPC_SET or IPC_RMID and the caller is not the super-user,\
+nor does the effective uid match either the
 .Va shm_perm.uid
 or
 .Va shm_perm.cuid
@@ -163,19 +183,48 @@ An attempt is made to increase the value of
 .Va shm_qbytes
 through IPC_SET
 but the caller is not the super-user.
-.It Bq Er EACCES
-The command is IPC_STAT
-and the caller has no read permission for this shared memory segment.
-.It Bq Er EINVAL
-.Fa shmid
-is not a valid shared memory segment identifier.
-.Va cmd
-is not a valid command.
-.It Bq Er EFAULT
-.Fa buf
-specifies an invalid address.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/shm.h>
+.Pp
+All of these include files are necessary.
+.Sh LEGACY DESCRIPTION
+The
+.Bf -literal
+ipc_perm
+.Ef
+structure used inside the
+.Bf -literal
+shmid_ds
+.Ef
+structure, as defined in
+.Aq Pa sys/ipc.h ,
+looks like this:
+.Bd -literal
+struct ipc_perm {
+    __uint16_t cuid;  /* Creator's user id */
+    __uint16_t cgid;  /* Creator's group id */
+    __uint16_t uid;   /* Owner's user id */
+    __uint16_t gid;   /* Owner's group id */
+    mode_t     mode;  /* r/w permission (see chmod(2)) */
+    __uint16_t seq;   /* Reserved for internal use */
+    key_t      key;   /* Reserved for internal use */
+};
+.Ed
+.Pp
+This structure is maintained for binary backward compatibility
+with previous versions of the interface.
+New code should not use this interface,
+because ID values may be truncated.
+.Pp
+Specifically,
+LEGACY mode limits the allowable uid/gid ranges to 0-32767.
+If the user has a UID that is out of this range (e.g., "nobody"),
+software using the LEGACY API will not behave as expected.
 .Sh SEE ALSO
 .Xr shmat 2 ,
 .Xr shmdt 2 ,
-.Xr shmget 2
+.Xr shmget 2 ,
+.Xr compat 5
diff --git a/bsd/man/man2/shmget.2 b/bsd/man/man2/shmget.2
index 1d13c4960..d6a135dce 100644
--- a/bsd/man/man2/shmget.2
+++ b/bsd/man/man2/shmget.2
@@ -37,11 +37,13 @@
 .Nm shmget
 .Nd get shared memory area identifier
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
-.Fd #include <sys/ipc.h>
 .Fd #include <sys/shm.h>
 .Ft int
-.Fn shmget "key_t key" "int size" "int shmflg"
+.Fo shmget
+.Fa "key_t key"
+.Fa "size_t size"
+.Fa "int shmflg"
+.Fc
 .Sh DESCRIPTION
 .Fn shmget
 returns the shared memory identifier associated with the key
@@ -96,30 +98,65 @@ Otherwise, -1 is returned and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
+The
+.Fn shmget
+system call will fail if:
 .Bl -tag -width Er
-.It Bq Er EACESS
+.\" ===========
+.It Bq Er EACCES
 A shared memory segment is already associated with
 .Fa key
 and the caller has no permission to access it.
+.\" ===========
 .It Bq Er EEXIST
 Both IPC_CREAT and IPC_EXCL are set in
 .Fa shmflg ,
 and a shared memory segment is already associated with
 .Fa key .
-.It Bq Er ENOSPC
-A new shared memory indentifier could not be created because the system limit
-for the number of shared memory identifiers has been reached.
+.\" ===========
+.It Bq Er EINVAL
+No shared memory segment is to be created,
+and a shared memory segment exists for
+.Fa key ,
+but the size of the segment associated with it
+is less than
+.Fa size ,
+which is non-zero.
+.\" ===========
 .It Bq Er ENOENT
 IPC_CREAT was not set in
 .Fa shmflg
 and no shared memory segment associated with
 .Fa key
 was found.
+.\" ===========
 .It Bq Er ENOMEM
 There is not enough memory left to created a shared memory segment of the
 requested size.
+.\" ===========
+.It Bq Er ENOSPC
+A new shared memory identifier could not be created because the system limit
+for the number of shared memory identifiers has been reached.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/ipc.h>
+.Fd #include <sys/shm.h>
+.Pp
+.Ft int
+.br
+.Fo shmget
+.Fa "key_t key"
+.Fa "int size"
+.Fa "int shmflg"
+.Fc ;
+.Pp
+All of these include files are necessary.
+The type of
+.Fa size
+has changed.
 .Sh SEE ALSO
-.Xr shmctl 2 ,
 .Xr shmat 2 ,
-.Xr shmdt 2
+.Xr shmctl 2 ,
+.Xr shmdt 2 ,
+.Xr compat 5
diff --git a/bsd/man/man2/shutdown.2 b/bsd/man/man2/shutdown.2
index e6799e727..4ac610045 100644
--- a/bsd/man/man2/shutdown.2
+++ b/bsd/man/man2/shutdown.2
@@ -42,13 +42,16 @@
 .Sh SYNOPSIS
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn shutdown "int s" "int how"
+.Fo shutdown
+.Fa "int socket"
+.Fa "int how"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn shutdown
 call causes all or part of a full-duplex connection on
 the socket associated with
-.Fa s
+.Fa socket
 to be shut down.
 If
 .Fa how
@@ -65,19 +68,27 @@ If
 is 
 .Dv SHUT_RDWR ,
 further sends and receives will be disallowed.
-.Sh DIAGNOSTICS
-A 0 is returned if the call succeeds, -1 if it fails.
+.Sh RETURN VALUES
+.Rv -std shutdown
 .Sh ERRORS
 The call succeeds unless:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EBADF
-.Fa S
+.Fa Socket
 is not a valid descriptor.
-.It Bq Er ENOTSOCK
-.Fa S
-is a file, not a socket.
+.\" ===========
+.It Bq Er EINVAL
+The
+.Fa how
+argument is invalid.
+.\" ===========
 .It Bq Er ENOTCONN
 The specified socket is not connected.
+.\" ===========
+.It Bq Er ENOTSOCK
+.Fa Socket
+is a file, not a socket.
 .El
 .Sh SEE ALSO
 .Xr connect 2 ,
diff --git a/bsd/man/man2/sigaction.2 b/bsd/man/man2/sigaction.2
index 42e779002..01feeb7e3 100644
--- a/bsd/man/man2/sigaction.2
+++ b/bsd/man/man2/sigaction.2
@@ -58,8 +58,8 @@ struct  sigaction {
 .Ft int
 .Fo sigaction
 .Fa "int sig"
-.Fa "const struct sigaction * restrict act"
-.Fa "struct sigaction * restrict oact"
+.Fa "const struct sigaction *restrict act"
+.Fa "struct sigaction *restrict oact"
 .Fc
 .Sh DESCRIPTION
 The system defines a set of signals that may be delivered to a process.
@@ -534,6 +534,7 @@ system call
 will fail and no new signal handler will be installed if one
 of the following occurs:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EFAULT
 Either
 .Fa act
@@ -541,16 +542,22 @@ or
 .Fa oact
 points to memory that is not a valid part of the process
 address space.
+.\" ===========
 .It Bq Er EINVAL
 The
 .Fa sig
 argument
 is not a valid signal number.
+.\" ===========
 .It Bq Er EINVAL
 An attempt is made to ignore or supply a handler for
 .Dv SIGKILL
 or
 .Dv SIGSTOP .
+.\" ===========
+.It Bq Er EINVAL
+An attempt was made to set the action to SIG_DFL
+for a signal that cannot be caught or ignored (or both).
 .El
 .Sh STANDARDS
 The
diff --git a/bsd/man/man2/sigaltstack.2 b/bsd/man/man2/sigaltstack.2
index dba3f28db..c84d99eaf 100644
--- a/bsd/man/man2/sigaltstack.2
+++ b/bsd/man/man2/sigaltstack.2
@@ -40,17 +40,12 @@
 .Nm sigaltstack
 .Nd set and/or get signal stack context
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <signal.h>
-.Bd -literal
-struct sigaltstack {
-	char   *ss_sp;
-	int	ss_size;
-	int     ss_flags;
-};
-.Ed
 .Ft int
-.Fn sigaltstack "const struct sigaltstack *ss" "struct sigaltstack *oss"
+.Fo sigaltstack
+.Fa "const stack_t *restrict ss"
+.Fa "stack_t *restrict oss"
+.Fc
 .Sh DESCRIPTION
 .Fn Sigaltstack
 allows users to define an alternate stack on which signals
@@ -138,10 +133,11 @@ Otherwise, a value of -1 is returned and
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Sigstack
+.Fn Sigaltstack
 will fail and the signal stack context will remain unchanged
 if one of the following occurs.
 .Bl -tag -width [ENOMEM]
+.\" ===========
 .It Bq Er EFAULT
 Either
 .Fa ss
@@ -149,15 +145,64 @@ or
 .Fa oss
 points to memory that is not a valid part of the process
 address space.
+.\" ===========
 .It Bq Er EINVAL
-An attempt was made to disable an active stack.
+An attempt is made to disable an active stack.
+.\" ===========
+.It Bq Er EINVAL
+The
+.Fa ss
+argument is not a null pointer, and the ss_flags member
+pointed to by
+.Fa ss
+contains flags other than SS_DISABLE.
+.\" ===========
 .It Bq Er ENOMEM
-Size of alternate stack area is less than or equal to
+The size of the alternate stack area is less than or equal to
 .Dv MINSIGSTKSZ .
+.\" ===========
+.It Bq Er EPERM
+An attempt was made to modify an active stack.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <signal.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
+.Pp
+.Bd -literal
+struct sigaltstack {
+	char   *ss_sp;
+	int	ss_size;
+	int     ss_flags;
+};
+.Ed
+.Pp
+.Ft int
+.br
+.Fo sigaltstack
+.Fa "const struct sigaltstack *ss"
+.Fa "struct sigaltstack *oss"
+.Fc ;
+.Pp
+The variable types have changed.
+Specifically, the
+.Vt sigaltstack
+struct is no longer used.
+.Sh COMPATIBILITY
+Use of the (obsolete)
+.Vt sigaltstack
+struct will cause compiler diagnostics.
+Use
+.Vt stack_t ,
+defined in
+.In signal.h .
 .Sh SEE ALSO
 .Xr sigaction 2 ,
-.Xr setjmp 3
+.Xr setjmp 3 ,
+.Xr compat 5
 .Sh HISTORY
 The predecessor to
 .Nm sigaltstack ,
diff --git a/bsd/man/man2/sigpending.2 b/bsd/man/man2/sigpending.2
index 3a0b95e9e..88fba758c 100644
--- a/bsd/man/man2/sigpending.2
+++ b/bsd/man/man2/sigpending.2
@@ -45,7 +45,9 @@
 .Sh SYNOPSIS
 .Fd #include <signal.h>
 .Ft int
-.Fn sigpending "sigset_t *set"
+.Fo sigpending
+.Fa "sigset_t *set"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm sigpending
diff --git a/bsd/man/man2/sigprocmask.2 b/bsd/man/man2/sigprocmask.2
index cd33571d5..2d36f838c 100644
--- a/bsd/man/man2/sigprocmask.2
+++ b/bsd/man/man2/sigprocmask.2
@@ -42,12 +42,16 @@
 .Sh SYNOPSIS
 .Fd #include <signal.h>
 .Ft int
-.Fn sigprocmask "int how" "const sigset_t *set" "sigset_t *oset"
+.Fo sigprocmask
+.Fa "int how"
+.Fa "const sigset_t *restrict set"
+.Fa "sigset_t *restrict oset"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn sigprocmask
-function examines and/or changes the current signal mask (those signals
-that are blocked from delivery).
+function examines and/or changes the current signal mask
+(those signals that are blocked from delivery).
 Signals are blocked if they are members of the current signal mask set.
 .Pp
 If
@@ -105,6 +109,7 @@ The
 call will fail and the signal mask will be unchanged if one
 of the following occurs:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EINVAL
 .Fa how
 has a value other than those listed here.
@@ -112,8 +117,8 @@ has a value other than those listed here.
 .Sh SEE ALSO
 .Xr kill 2 ,
 .Xr sigaction 2 ,
-.Xr sigsetops 3 ,
-.Xr sigsuspend 2
+.Xr sigsuspend 2 ,
+.Xr sigsetops 3
 .Sh STANDARDS
 The
 .Fn sigprocmask
diff --git a/bsd/man/man2/sigsuspend.2 b/bsd/man/man2/sigsuspend.2
index 93f7ff56a..29bb42891 100644
--- a/bsd/man/man2/sigsuspend.2
+++ b/bsd/man/man2/sigsuspend.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <signal.h>
 .Ft int
-.Fn sigsuspend "const sigset_t *sigmask"
+.Fo sigsuspend
+.Fa "const sigset_t *sigmask"
+.Fc
 .Sh DESCRIPTION
 .Fn Sigsuspend
 temporarily changes the blocked signal mask to the set to which
@@ -71,12 +73,12 @@ always terminates by being interrupted, returning -1 with
 set to
 .Dv EINTR .
 .Sh SEE ALSO
-.Xr sigprocmask 2 ,
 .Xr sigaction 2 ,
+.Xr sigprocmask 2 ,
 .Xr sigsetops 3
 .Sh STANDARDS
 The
-.Nm sigsupend
+.Nm sigsuspend
 function call
 conforms to
 .St -p1003.1-88 .
diff --git a/bsd/man/man2/socket.2 b/bsd/man/man2/socket.2
index 3083d3983..c8d59a4a2 100644
--- a/bsd/man/man2/socket.2
+++ b/bsd/man/man2/socket.2
@@ -40,10 +40,13 @@
 .Nm socket
 .Nd create an endpoint for communication
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn socket "int domain" "int type" "int protocol"
+.Fo socket
+.Fa "int domain"
+.Fa "int type"
+.Fa "int protocol"
+.Fc
 .Sh DESCRIPTION
 .Fn Socket
 creates an endpoint for communication and returns a descriptor.
@@ -212,27 +215,57 @@ value is a descriptor referencing the socket.
 .Sh ERRORS
 The
 .Fn socket
-call fails if:
+system call fails if:
 .Bl -tag -width Er
-.It Bq Er EPROTONOSUPPORT
-The protocol type or the specified protocol is not supported
-within this domain.
+.\" ===========
+.It Bq Er EACCES
+Permission to create a socket of the specified type and/or protocol
+is denied.
+.\" ===========
+.It Bq Er EAFNOSUPPORT
+The specified address family is not supported.
+.\" ===========
+.It Bq Er EISCONN
+The per-process descriptor table is full.
+.\" ===========
 .It Bq Er EMFILE
 The per-process descriptor table is full.
+.\" ===========
 .It Bq Er ENFILE
 The system file table is full.
-.It Bq Er EACCES
-Permission to create a socket of the specified type and/or protocol
-is denied.
+.\" ===========
 .It Bq Er ENOBUFS
 Insufficient buffer space is available.
 The socket cannot be created until sufficient resources are freed.
+.\" ===========
+.It Bq Er ENOMEM
+Insufficient memory was available to fulfill the request.
+.\" ===========
+.It Bq Er EPROTONOSUPPORT
+The protocol type or the specified protocol is not supported
+within this domain.
+.\" ===========
+.It Bq Er EPROTOTYPE
+The socket type is not supported by the protocol.
 .El
+.Pp
+If a new protocol family is defined,
+the socreate process is free to return any desired error code.
+The
+.Fn socket
+system call will pass this error code along
+(even if it is undefined).
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr accept 2 ,
 .Xr bind 2 ,
 .Xr connect 2 ,
-.Xr getprotoent 3 ,
 .Xr getsockname 2 ,
 .Xr getsockopt 2 ,
 .Xr ioctl 2 ,
@@ -243,7 +276,12 @@ The socket cannot be created until sufficient resources are freed.
 .Xr send 2 ,
 .Xr shutdown 2 ,
 .Xr socketpair 2 ,
-.Xr write 2
+.Xr write 2 ,
+.Xr getprotoent 3 ,
+.Xr inet 4 ,
+.Xr inet6 4 ,
+.Xr unix 4 ,
+.Xr compat 5
 .Rs
 .%T "An Introductory 4.3 BSD Interprocess Communication Tutorial"
 .%O "reprinted in UNIX Programmer's Supplementary Documents Volume 1"
diff --git a/bsd/man/man2/socketpair.2 b/bsd/man/man2/socketpair.2
index 32952769e..62a33306e 100644
--- a/bsd/man/man2/socketpair.2
+++ b/bsd/man/man2/socketpair.2
@@ -40,49 +40,78 @@
 .Nm socketpair
 .Nd create a pair of connected sockets
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/socket.h>
 .Ft int
-.Fn socketpair "int d" "int type" "int protocol" "int *sv"
+.Fo socketpair
+.Fa "int domain"
+.Fa "int type"
+.Fa "int protocol"
+.Fa "int socket_vector[2]"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn socketpair
 call creates an unnamed pair of connected sockets in
 the specified domain
-.Fa d ,
+.Fa domain ,
 of the specified
 .Fa type ,
 and using the optionally specified
 .Fa protocol .
 The descriptors used in referencing the new sockets
 are returned in
-.Fa sv Ns [0]
+.Fa socket_vector Ns [0]
 and
-.Fa sv Ns [1] .
+.Fa socket_vector Ns [1] .
 The two sockets are indistinguishable.
-.Sh DIAGNOSTICS
-A 0 is returned if the call succeeds, -1 if it fails.
+.Sh RETURN VALUES
+.Rv -std socketpair
 .Sh ERRORS
 The call succeeds unless:
 .Bl -tag -width Er
-.It Bq Er EMFILE
-Too many descriptors are in use by this process.
+.\" ===========
 .It Bq Er EAFNOSUPPORT
 The specified address family is not supported on this machine.
-.It Bq Er EPROTONOSUPPORT
-The specified protocol is not supported on this machine.
-.It Bq Er EOPNOSUPPORT
-The specified protocol does not support creation of socket pairs.
+.\" ===========
 .It Bq Er EFAULT
 The address
-.Fa sv
-does not specify a valid part of the
-process address space.
+.Fa socket_vector
+does not specify a valid part of the process address space.
+.\" ===========
+.It Bq Er EMFILE
+No more file descriptors are available for this process.
+.\" ===========
+.It Bq Er ENFILE
+No more file descriptors are available for the system.
+.\" ===========
+.It Bq Er ENOBUFS
+Insufficient resources were available in the system
+to perform the operation.
+.\" ===========
+.It Bq Er ENOMEM
+Insufficient memory was available to fulfill the request.
+.\" ===========
+.It Bq Er EOPNOTSUPP
+The specified protocol does not support creation of socket pairs.
+.\" ===========
+.It Bq Er EPROTONOSUPPORT
+The specified protocol is not supported on this machine.
+.\" ===========
+.It Bq Er EPROTOTYPE
+The socket type is not supported by the protocol.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/socket.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
+.Xr pipe 2 ,
 .Xr read 2 ,
 .Xr write 2 ,
-.Xr pipe 2
+.Xr compat 5
 .Sh BUGS
 This call is currently implemented only for the
 .Tn UNIX
diff --git a/bsd/man/man2/stat.2 b/bsd/man/man2/stat.2
index 04f63c7ba..0c9f062e0 100644
--- a/bsd/man/man2/stat.2
+++ b/bsd/man/man2/stat.2
@@ -37,22 +37,50 @@
 .Dt STAT 2
 .Os BSD 4
 .Sh NAME
-.Nm stat ,
+.Nm fstat ,
+.Nm fstat64 ,
 .Nm lstat ,
-.Nm fstat
+.Nm lstat64 ,
+.Nm stat ,
+.Nm stat64
 .Nd get file status
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/stat.h>
 .Ft int
-.Fn stat "const char *path" "struct stat *sb"
+.Fo fstat
+.Fa "int fildes"
+.Fa "struct stat *buf"
+.Fc
+.Ft int
+.Fo fstat64
+.Fa "int fildes"
+.Fa "struct stat64 *buf"
+.Fc
 .Ft int
-.Fn lstat "const char *path" "struct stat *sb"
+.Fo lstat
+.Fa "const char *restrict path"
+.Fa "struct stat *restrict buf"
+.Fc
 .Ft int
-.Fn fstat "int fd" "struct stat *sb"
+.Fo lstat64
+.Fa "const char *restrict path"
+.Fa "struct stat64 *restrict buf"
+.Fc
+.Ft int
+.Fo stat
+.Fa "const char *restrict path"
+.Fa "struct stat *restrict buf"
+.Fc
+.Ft int
+.Fo stat64
+.Fa "const char *restrict path"
+.Fa "struct stat64 *restrict buf"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn stat
+family of functions and their 64 bit variants obtain information about a file. The 
+.Fn stat 
 function obtains information about the file pointed to by
 .Fa path .
 Read, write or execute
@@ -82,16 +110,18 @@ The
 .Fn fstat
 obtains the same information about an open file
 known by the file descriptor
-.Fa fd .
+.Fa fildes .
 .Pp
 The
-.Fa sb
+.Fa buf
 argument is a pointer to a
-.Fn stat
-structure
+.Fa stat
+or
+.Fa stat64
+structure 
 as defined by
 .Aq Pa sys/stat.h
-(shown below)
+(both shown below)
 and into which information is placed concerning the file.
 .Bd -literal
 struct stat {
@@ -111,12 +141,38 @@ struct stat {
     u_long   st_flags;  /* user defined flags for file */
     u_long   st_gen;    /* file generation number */
 };
+
+
+struct stat64 {
+    dev_t           st_dev;           /* ID of device containing file */
+    mode_t          st_mode;          /* Mode of file (see below) */
+    nlink_t         st_nlink;         /* Number of hard links */
+    ino64_t 	    st_ino;          /* File serial number */
+    uid_t           st_uid;           /* User ID of the file */
+    gid_t           st_gid;           /* Group ID of the file */
+    dev_t           st_rdev;          /* Device ID */
+    struct timespec st_atimespec;     /* time of last access */
+    struct timespec st_mtimespec;     /* time of last data modification */
+    struct timespec st_ctimespec;     /* time of last status change */
+    struct timespec st_birthtimespec; /* time of file creation(birth) */
+    off_t           st_size;          /* file size, in bytes */
+    blkcnt_t        st_blocks;        /* blocks allocated for file */
+    blksize_t       st_blksize;       /* optimal blocksize for I/O */
+    uint32_t        st_flags;         /* user defined flags for file */
+    uint32_t        st_gen;           /* file generation number */
+    int32_t         st_lspare;        /* RESERVED: DO NOT USE! */
+    int64_t         st_qspare[2];     /* RESERVED: DO NOT USE! */
+};
+
+
 .Ed
 .Pp
 The time-related fields of
 .Fa struct stat
+and
+.Fa struct stat64
 are as follows:
-.Bl -tag -width XXXst_mtime
+.Bl -tag -width XXXst_birthtime
 .It st_atime
 Time when file data last accessed.
 Changed by the
@@ -146,11 +202,15 @@ Changed by the
 and
 .Xr write 2
 system calls.
+.It st_birthtime
+Time of file creation. Only set once when the file is created. This field is 
+only available in the 64 bit variants. On filesystems where birthtime is 
+not available, this field holds the
+.Fa ctime
+instead.
 .El
 .Pp
-The size-related fields of the
-.Fa struct stat
-are as follows:
+The size-related fields of the structures are as follows:
 .Bl -tag -width XXXst_blksize
 .It st_blksize
 The optimal I/O block size for the file.
@@ -186,6 +246,13 @@ For a list of access modes, see
 .Xr access 2
 and
 .Xr chmod 2 .
+.Pp
+For a list of the file flags in the
+.Fa st_flags
+field, see
+.Aq Pa sys/stat.h
+and
+.Xr chflags 2 .
 .Sh RETURN VALUES
 Upon successful completion a value of 0 is returned.
 Otherwise, a value of -1 is returned and
@@ -203,45 +270,74 @@ and
 .Li st_blocks
 fields.
 .Sh ERRORS
-.Fn Stat
-and
+.Bl -tag -width Er
+The
+.Fn fstat
+system call will fail if:
+.\" ===========
+.It Bq Er EBADF
+.Fa fildes
+is not a valid open file descriptor.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Sb
+points to an invalid address.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while reading from or writing to the file system.
+.El
+.Pp
+The
 .Fn lstat
-will fail if:
+and
+.Fn stat
+system calls will fail if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The named file does not exist.
+.\" ===========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ===========
 .It Bq Er EFAULT
 .Fa Sb
 or
 .Em name
 points to an invalid address.
+.\" ===========
 .It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+An I/O error occurs while reading from or writing to the file system.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds
+.Dv {NAME_MAX}
+characters, or an entire path name exceeds
+.Dv {PATH_MAX}
+characters.
+.\" ===========
+.It Bq Er ENOENT
+The named file does not exist.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
 .El
 .Pp
+The
+.Fn fstat ,
+.Fn lstat ,
+and
+.Fn stat
+system calls will fail if:
 .Bl -tag -width Er
-.Fn Fstat
-will fail if:
-.It Bq Er EBADF
-.Fa fd
-is not a valid open file descriptor.
-.It Bq Er EFAULT
-.Fa Sb
-points to an invalid address.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
+.\" ===========
+.It Bq Er EOVERFLOW
+The file size in bytes 
+or the number of blocks allocated to the file
+or the file serial number cannot be represented correctly
+in the structure pointed to by
+.Fa buf .
 .El
 .Sh CAVEATS
 The file generation number,
@@ -257,10 +353,19 @@ are present in preparation for inode time stamps expanding
 to 64 bits.  This, however, can break certain programs that
 depend on the time stamps being contiguous (in calls to
 .Xr utimes 2 ) .
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/stat.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
+.Xr chflags 2 ,
 .Xr chmod 2 ,
 .Xr chown 2 ,
-.Xr utimes 2
+.Xr utimes 2 ,
+.Xr compat 5 ,
 .Xr symlink 7
 .Sh BUGS
 Applying
@@ -281,3 +386,9 @@ An
 .Fn lstat
 function call appeared in
 .Bx 4.2 .
+The 
+.Fn stat64 ,
+.Fn fstat64 ,
+and
+.Fn lstat64
+system calls first appeared in Mac OS X 10.5 (Leopard).
diff --git a/bsd/man/man2/stat64.2 b/bsd/man/man2/stat64.2
new file mode 100644
index 000000000..b1a86c195
--- /dev/null
+++ b/bsd/man/man2/stat64.2
@@ -0,0 +1 @@
+.so man2/stat.2
diff --git a/bsd/man/man2/statfs.2 b/bsd/man/man2/statfs.2
index 1a534dd9b..1ababf3f8 100644
--- a/bsd/man/man2/statfs.2
+++ b/bsd/man/man2/statfs.2
@@ -37,7 +37,10 @@
 .Dt STATFS 2
 .Os
 .Sh NAME
-.Nm statfs
+.Nm statfs,
+.Nm statfs64,
+.Nm fstatfs,
+.Nm fstatfs64
 .Nd get file system statistics
 .Sh SYNOPSIS
 .Fd #include <sys/param.h>
@@ -45,19 +48,29 @@
 .Ft int
 .Fn statfs "const char *path" "struct statfs *buf"
 .Ft int
+.Fn statfs64 "const char *path" "struct statfs64 *buf"
+.Ft int
 .Fn fstatfs "int fd" "struct statfs *buf"
+.Ft int
+.Fn fstatfs64 "int fd" "struct statfs64 *buf"
 .Sh DESCRIPTION
 .Fn Statfs
 returns information about a mounted file system.
 .Fa Path
 is the path name of any file within the mounted file system.
 .Fa Buf
-is a pointer to a statfs structure defined as follows:
+is a pointer to a 
+.Fa statfs
+or
+.Fa statfs64
+structure defined as follows:
 .Bd -literal
 typedef struct { int32_t val[2]; } fsid_t;
 
-#define MFSNAMELEN   15 /* length of fs type name, not inc. nul */
-#define MNAMELEN     90	/* length of buffer for returned name */
+#define MFSNAMELEN      15 /* length of fs type name, not inc. nul */
+#define MNAMELEN        90 /* length of buffer for returned name */
+#define MFSTYPENAMELEN  16 /* length of fs type name including null */
+#define MAXPATHLEN      1024	
 
 struct statfs {
     short   f_otype;	/* type of file system (reserved: zero) */
@@ -69,7 +82,7 @@ struct statfs {
     long    f_bavail;	/* free blocks avail to non-superuser */
     long    f_files;	/* total file nodes in file system */
     long    f_ffree;	/* free file nodes in fs */
-    fsid_t  f_fsid;	/* file system id (super-user only) */
+    fsid_t  f_fsid;	/* file system id */
     uid_t   f_owner;	/* user that mounted the file system */
     short   f_reserved1;	/* reserved for future use */
     short   f_type;	/* type of file system (reserved) */
@@ -81,12 +94,74 @@ struct statfs {
     char    f_reserved3;	/* reserved for future use */
     long    f_reserved4[4];	/* reserved for future use */
 };
+
+struct statfs64 {
+    uint32_t        f_bsize;        /* fundamental file system block size */ 
+    int32_t         f_iosize;       /* optimal transfer block size */ 
+    uint64_t        f_blocks;       /* total data blocks in file system */ 
+    uint64_t        f_bfree;        /* free blocks in fs */ 
+    uint64_t        f_bavail;       /* free blocks avail to non-superuser */ 
+    uint64_t        f_files;        /* total file nodes in file system */ 
+    uint64_t        f_ffree;        /* free file nodes in fs */ 
+    fsid_t          f_fsid;         /* file system id */ 
+    uid_t           f_owner;        /* user that mounted the filesystem */ 
+    uint32_t        f_type;         /* type of filesystem */ 
+    uint32_t        f_flags;        /* copy of mount exported flags */ 
+    uint32_t        f_fssubtype;    /* fs sub-type (flavor) */ 
+    char            f_fstypename[MFSTYPENAMELEN];   /* fs type name */ 
+    char            f_mntonname[MAXPATHLEN];        /* directory on which mounted */ 
+    char            f_mntfromname[MAXPATHLEN];      /* mounted filesystem */ 
+    uint32_t        f_reserved[8];  /* For future use */ 
+};
 .Ed
 .Pp
 Fields that are undefined for a particular file system are set to -1.
 .Fn Fstatfs
 returns the same information about an open file referenced by descriptor
 .Fa fd .
+.Sh FLAGS
+.Bl -tag -width MNT_UNKOWNPERMISSIONS
+These are some of the flags that may be present in the f_flags field. 
+.It Dv MNT_RDONLY
+A read-only filesystem
+.It Dv MNT_SYNCHRONOUS
+File system is written to synchronously
+.It Dv MNT_NOEXEC
+Can't exec from filesystem
+.It Dv MNT_NOSUID
+Setuid bits are not honored on this filesystem
+.It Dv MNT_NODEV
+Don't interpret special files
+.It Dv MNT_UNION
+Union with underlying filesysten
+.It Dv MNT_ASYNC
+File system written to asynchronously
+.It Dv MNT_EXPORTED
+File system is exported
+.it Dv MNT_LOCAL
+File system is stored locally
+.It Dv MNT_QUOTA
+Quotas are enabled on this file system
+.It Dv MNT_ROOTFS
+This file system is the root of the file system
+.It Dv MNT_DOVOLFS
+File system supports volfs
+.It Dv MNT_DONTBROWSE
+File system is not appropriate path to user data
+.It Dv MNT_UNKNOWNPERMISSIONS
+VFS will ignore ownership information on filesystem objects
+.It Dv MNT_AUTOMOUNTED
+File system was mounted by automounter
+.It Dv MNT_JOURNALED
+File system is journaled
+.It Dv MNT_DEFWRITE
+File system should defer writes
+.It Dv MNT_MULTILABEL
+MAC support for individual labels
+.El
+.Sh CAVEATS
+In Mac OS X versions before 10.4, f_iosize is 4096. On these older
+systems, use MAXBSIZE instead.
 .Sh RETURN VALUES
 Upon successful completion, a value of 0 is returned.
 Otherwise, -1 is returned and the global variable
@@ -148,4 +223,8 @@ error occurred while reading from or writing to the file system.
 .Sh HISTORY
 The
 .Fn statfs
-function first appeared in 4.4BSD.
+function first appeared in 4.4BSD. The
+.Fn statfs64
+and 
+.Fn fstatfs64
+first appeared in Max OS X 10.5 (Leopard).
diff --git a/bsd/man/man2/statfs64.2 b/bsd/man/man2/statfs64.2
new file mode 100644
index 000000000..3a64852e4
--- /dev/null
+++ b/bsd/man/man2/statfs64.2
@@ -0,0 +1,3 @@
+.so man2/statfs.2
+
+
diff --git a/bsd/man/man2/symlink.2 b/bsd/man/man2/symlink.2
index cee3f9ff6..ff2d106de 100644
--- a/bsd/man/man2/symlink.2
+++ b/bsd/man/man2/symlink.2
@@ -42,16 +42,19 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn symlink "const char *name1" "const char *name2"
+.Fo symlink
+.Fa "const char *path1"
+.Fa "const char *path2"
+.Fc
 .Sh DESCRIPTION
 A symbolic link
-.Fa name2
+.Fa path2
 is created to
-.Fa name1
-.Pf ( Fa name2
+.Fa path1
+.Pf ( Fa path2
 is the name of the
 file created,
-.Fa name1
+.Fa path1
 is the string
 used in creating the symbolic link).
 Either name may be an arbitrary path name; the files need not
@@ -64,74 +67,100 @@ and a -1 value is returned.
 .Sh ERRORS
 The symbolic link succeeds unless:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the
-.Fa name2
-prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The named file does not exist.
+.\" ===========
+.It Bq Er EACCES
+Write permission is denied in the directory
+where the symbolic link is being created.
+.\" ===========
 .It Bq Er EACCES
 A component of the
-.Fa name2
+.Fa path2
 path prefix denies search permission.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ===========
+.It Bq Er EDQUOT
+The directory in which the entry for the new symbolic link
+is being placed cannot be extended because the
+user's quota of disk blocks on the file system
+containing the directory has been exhausted.
+.\" ===========
+.It Bq Er EDQUOT
+The new symbolic link cannot be created because the user's
+quota of disk blocks on the file system that will
+contain the symbolic link has been exhausted.
+.\" ===========
+.It Bq Er EDQUOT
+The user's quota of inodes on the file system on
+which the symbolic link is being created has been exhausted.
+.\" ===========
 .It Bq Er EEXIST
-.Fa Name2
+.Fa Path2
 already exists.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Path1
+or
+.Fa path2
+points outside the process's allocated address space.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while making the directory entry
+or allocating the inode.
+.\" ===========
 .It Bq Er EIO
-An I/O error occurred while making the directory entry for
-.Fa name2 ,
+An I/O error occurs while making the directory entry for
+.Fa path2 ,
 or allocating the inode for
-.Fa name2 ,
+.Fa path2 ,
 or writing out the link contents of
-.Fa name2 .
-.It Bq Er EROFS
-The file
-.Fa name2
-would reside on a read-only file system.
+.Fa path2 .
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds 
+.Dv {NAME_MAX}
+characters, or an entire path name exceeds
+.Dv {PATH_MAX}
+characters.
+.\" ===========
+.It Bq Er ENOENT
+A component of
+.Fa path2
+does not name an existing file or
+.Fa path2
+is an empty string.
+.\" ===========
 .It Bq Er ENOSPC
 The directory in which the entry for the new symbolic link is being placed
 cannot be extended because there is no space left on the file
 system containing the directory.
+.\" ===========
 .It Bq Er ENOSPC
 The new symbolic link cannot be created because there
 there is no space left on the file
 system that will contain the symbolic link.
+.\" ===========
 .It Bq Er ENOSPC
 There are no free inodes on the file system on which the
 symbolic link is being created.
-.It Bq Er EDQUOT
-The directory in which the entry for the new symbolic link
-is being placed cannot be extended because the
-user's quota of disk blocks on the file system
-containing the directory has been exhausted.
-.It Bq Er EDQUOT
-The new symbolic link cannot be created because the user's
-quota of disk blocks on the file system that will
-contain the symbolic link has been exhausted.
-.It Bq Er EDQUOT
-The user's quota of inodes on the file system on
-which the symbolic link is being created has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while making the directory entry or allocating the inode.
-.It Bq Er EFAULT
-.Fa Name1
-or
-.Fa name2
-points outside the process's allocated address space.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the
+.Fa path2
+prefix is not a directory.
+.\" ===========
+.It Bq Er EROFS
+The file
+.Fa path2
+would reside on a read-only file system.
 .El
 .Sh SEE ALSO
 .Xr ln 1 ,
 .Xr link 2 ,
-.Xr unlink 2
-.Xr symlink 7 ,
+.Xr unlink 2 ,
+.Xr symlink 7
 .Sh HISTORY
 The
 .Fn symlink
diff --git a/bsd/man/man2/sync.2 b/bsd/man/man2/sync.2
index a94475b4e..7440b25ee 100644
--- a/bsd/man/man2/sync.2
+++ b/bsd/man/man2/sync.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft void
-.Fn sync void
+.Fo sync
+.Fa void
+.Fc
 .Sh DESCRIPTION
 The 
 .Fn sync
diff --git a/bsd/man/man2/truncate.2 b/bsd/man/man2/truncate.2
index d54ad0897..660045be5 100644
--- a/bsd/man/man2/truncate.2
+++ b/bsd/man/man2/truncate.2
@@ -37,21 +37,27 @@
 .Dt TRUNCATE 2
 .Os BSD 4.2
 .Sh NAME
-.Nm truncate ,
-.Nm ftruncate
+.Nm ftruncate ,
+.Nm truncate
 .Nd truncate or extend a file to a specified length
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn truncate "const char *path" "off_t length"
+.Fo ftruncate
+.Fa "int fildes"
+.Fa "off_t length"
+.Fc
 .Ft int
-.Fn ftruncate "int fd" "off_t length"
+.Fo truncate
+.Fa "const char *path"
+.Fa "off_t length"
+.Fc
 .Sh DESCRIPTION
 .Fn Truncate
 causes the file named by
 .Fa path
 or referenced by
-.Fa fd
+.Fa fildes
 to be truncated or extended to
 .Fa length
 bytes in size.  If the file previously
@@ -67,53 +73,97 @@ fails a -1 is returned, and the global variable
 .Va errno
 specifies the error.
 .Sh ERRORS
-.Fn Truncate
-succeeds unless:
+.Pp
+The
+.Fn ftruncate
+system call will fail if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The named file does not exist.
+.\" ===========
+.It Bq Er EBADF
+.Fa fildes
+is not a valid descriptor open for writing.
+.\" ===========
+.It Bq Er EFBIG
+The file is a regular file and
+.Fa length
+is greater than the offset maximum established
+in the open file description associated with
+.Fa fildes .
+.\" ===========
+.It Bq Er EINVAL
+.Fa fildes
+references a socket, not a file.
+.\" ===========
+.It Bq Er EINVAL
+.Fa fildes
+is not open for writing.
+.\" ===========
+.It Bq Er EROFS
+The named file resides on a read-only file system.
+.El
+.Pp
+The
+.Fn truncate
+system call will fail if:
+.Bl -tag -width Er
+.\" ===========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
+.\" ===========
 .It Bq Er EACCES
 The named file is not writable by the user.
-.It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EISDIR
 The named file is a directory.
+.\" ===========
+.It Bq Er ELOOP
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds
+.Dv {NAME_MAX}
+characters, or an entire path name exceeds 
+.Dv {PATH_MAX}
+characters.
+.\" ===========
+.It Bq Er ENOENT
+The named file does not exist.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ===========
 .It Bq Er EROFS
 The named file resides on a read-only file system.
+.\" ===========
 .It Bq Er ETXTBSY
 The file is a pure procedure (shared text) file that is being executed.
-.It Bq Er EIO
-An I/O error occurred updating the inode.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
 .El
 .Pp
-.Fn Ftruncate
-succeeds unless:
-.Bl -tag -width Er
-.It Bq Er EBADF
 The
-.Fa fd
-is not a valid descriptor.
-.It Bq Er EINVAL
-The
-.Fa fd
-references a socket, not a file.
+.Fn ftruncate
+and
+.Fn truncate
+system calls will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EFBIG
+The length argument was greater than the maximum file size.
+.\" ===========
+.It Bq Er EINTR
+A signal is caught during execution.
+.\" ===========
 .It Bq Er EINVAL
 The
-.Fa fd
-is not open for writing.
+.Fa length
+argument is less than 0.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurred while reading from or writing to a file system.
 .El
 .Sh SEE ALSO
 .Xr open 2
diff --git a/bsd/man/man2/umask.2 b/bsd/man/man2/umask.2
index 92eff34f7..faad5f1b5 100644
--- a/bsd/man/man2/umask.2
+++ b/bsd/man/man2/umask.2
@@ -40,23 +40,23 @@
 .Nm umask
 .Nd set file creation mode mask
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/stat.h>
 .Ft mode_t
-.Fn umask "mode_t numask"
+.Fo umask
+.Fa "mode_t cmask"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn umask
 routine sets the process's file mode creation mask to
-.Fa numask
-and returns the previous value of the mask.  The 9 low-order
-access permission
-bits of
-.Fa numask
+.Fa cmask
+and returns the previous value of the mask.
+The 9 low-order access permission bits of
+.Fa cmask
 are used by system calls, including
 .Xr open 2 ,
 .Xr mkdir 2 ,
-.Xr mkfifo 2
+.Xr mkfifo 2 ,
 and
 .Xr mknod 2
 to turn off corresponding bits
@@ -75,12 +75,20 @@ The previous value of the file mode mask is returned by the call.
 The
 .Fn umask
 function is always successful.
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/stat.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
 .Xr chmod 2 ,
 .Xr mkdir 2 ,
 .Xr mkfifo 2 ,
 .Xr mknod 2 ,
-.Xr open 2
+.Xr open 2 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn umask
diff --git a/bsd/man/man2/unlink.2 b/bsd/man/man2/unlink.2
index 6dfd46035..3418f53fa 100644
--- a/bsd/man/man2/unlink.2
+++ b/bsd/man/man2/unlink.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft int
-.Fn unlink "const char *path"
+.Fo unlink
+.Fa "const char *path"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn unlink
@@ -66,48 +68,67 @@ is set to indicate the error.
 .Sh ERRORS
 The
 .Fn unlink
-succeeds unless:
+system call will fail if:
 .Bl -tag -width Er
-.It Bq Er ENOTDIR
-A component of the path prefix is not a directory.
-.It Bq Er ENAMETOOLONG
-A component of a pathname exceeded 
-.Dv {NAME_MAX}
-characters, or an entire path name exceeded 
-.Dv {PATH_MAX}
-characters.
-.It Bq Er ENOENT
-The named file does not exist.
+.\" ===========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix.
+.\" ===========
 .It Bq Er EACCES
 Write permission is denied on the directory containing the link
 to be removed.
+.\" ===========
+.It Bq Er EBUSY
+The entry to be unlinked is the mount point for a
+mounted file system.
+.\" ===========
+.It Bq Er EBUSY
+The file named by the
+.Fa path
+argument cannot be unlinked
+because it is being used by the system or by another process.
+.\" ===========
+.It Bq Er EFAULT
+.Fa Path
+points outside the process's allocated address space.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while deleting the directory entry
+or deallocating the inode.
+.\" ===========
 .It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeds
+.Dv {NAME_MAX}
+characters, or an entire path name exceeds
+.Dv {PATH_MAX}
+characters (possibly as a result of expanding a symlink).
+.\" ===========
+.It Bq Er ENOENT
+The named file does not exist.
+.\" ===========
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.\" ===========
 .It Bq Er EPERM
 The named file is a directory and the effective user ID
 of the process is not the super-user.
+.\" ===========
 .It Bq Er EPERM
 The directory containing the file is marked sticky,
 and neither the containing directory nor the file to be removed
 are owned by the effective user ID.
-.It Bq Er EBUSY
-The entry to be unlinked is the mount point for a
-mounted file system.
-.It Bq Er EIO
-An I/O error occurred while deleting the directory entry
-or deallocating the inode.
+.\" ===========
 .It Bq Er EROFS
 The named file resides on a read-only file system.
-.It Bq Er EFAULT
-.Fa Path
-points outside the process's allocated address space.
 .El
 .Sh SEE ALSO
 .Xr close 2 ,
 .Xr link 2 ,
-.Xr rmdir 2
+.Xr rmdir 2 ,
 .Xr symlink 7
 .Sh HISTORY
 An
diff --git a/bsd/man/man2/utimes.2 b/bsd/man/man2/utimes.2
index 94dabd258..cb650f186 100644
--- a/bsd/man/man2/utimes.2
+++ b/bsd/man/man2/utimes.2
@@ -38,22 +38,28 @@
 .Dt UTIMES 2
 .Os
 .Sh NAME
-.Nm utimes ,
-.Nm futimes
+.Nm futimes ,
+.Nm utimes
 .Nd set file access and modification times
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
 .In sys/time.h
 .Ft int
-.Fn utimes "const char *path" "const struct timeval *times"
+.Fo futimes
+.Fa "int fildes"
+.Fa "const struct timeval times[2]"
+.Fc
 .Ft int
-.Fn futimes "int fd" "const struct timeval *times"
+.Fo utimes
+.Fa "const char *path"
+.Fa "const struct timeval times[2]"
+.Fc
 .Sh DESCRIPTION
 The access and modification times of the file named by
 .Fa path
 or referenced by
-.Fa fd
+.Fa fildes
 are changed as specified by the argument
 .Fa times .
 .Pp
@@ -79,9 +85,11 @@ time.
 .Sh RETURN VALUES
 .Rv -std
 .Sh ERRORS
+The
 .Fn utimes
-will fail if:
+system call will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EACCES
 Search permission is denied for a component of the path prefix;
 or the
@@ -91,25 +99,33 @@ argument is
 and the effective user ID of the process does not
 match the owner of the file, and is not the super-user, and write
 access is denied.
+.\" ===========
 .It Bq Er EFAULT
 .Fa path
 or
 .Fa times
 points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EIO
-An I/O error occurred while reading or writing the affected inode.
+An I/O error occurs while reading or writing the affected inode.
+.\" ===========
 .It Bq Er ELOOP
-Too many symbolic links were encountered in translating the pathname.
+Too many symbolic links are encountered in translating the pathname.
+This is taken to be indicative of a looping symbolic link.
+.\" ===========
 .It Bq Er ENAMETOOLONG
-A component of a pathname exceeded
+A component of a pathname exceeds
 .Dv NAME_MAX
 characters, or an entire path name exceeded
 .Dv PATH_MAX
 characters.
+.\" ===========
 .It Bq Er ENOENT
 The named file does not exist.
+.\" ===========
 .It Bq Er ENOTDIR
 A component of the path prefix is not a directory.
+.\" ===========
 .It Bq Er EPERM
 The
 .Fa times
@@ -117,20 +133,24 @@ argument is not
 .Dv NULL
 and the calling process's effective user ID
 does not match the owner of the file and is not the super-user.
+.\" ===========
 .It Bq Er EROFS
 The file system containing the file is mounted read-only.
 .El
 .Pp
+The
 .Fn futimes
-will fail if:
+system call will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EBADF
-.Fa fd
+.Fa fildes
 does not refer to a valid descriptor.
 .El
 .Pp
 All of the functions will fail if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EACCES
 The
 .Fa times
@@ -139,11 +159,14 @@ argument is
 and the effective user ID of the process does not
 match the owner of the file, and is not the super-user, and write
 access is denied.
+.\" ===========
 .It Bq Er EFAULT
 .Fa times
 points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EIO
 An I/O error occurred while reading or writing the affected inode.
+.\" ===========
 .It Bq Er EPERM
 The
 .Fa times
@@ -151,6 +174,7 @@ argument is not
 .Dv NULL
 and the calling process's effective user ID
 does not match the owner of the file and is not the super-user.
+.\" ===========
 .It Bq Er EROFS
 The file system containing the file is mounted read-only.
 .El
diff --git a/bsd/man/man2/vfork.2 b/bsd/man/man2/vfork.2
index c8b1d3bbd..c512eae01 100644
--- a/bsd/man/man2/vfork.2
+++ b/bsd/man/man2/vfork.2
@@ -42,7 +42,9 @@
 .Sh SYNOPSIS
 .Fd #include <unistd.h>
 .Ft pid_t
-.Fn vfork void
+.Fo vfork
+.Fa void
+.Fc
 .Sh DESCRIPTION
 .Fn Vfork
 can be used to create new processes without fully copying the address
@@ -91,13 +93,30 @@ it is wrong to call
 .Xr exit
 since buffered data would then be flushed twice.)
 .Sh SEE ALSO
-.Xr fork 2 ,
 .Xr execve 2 ,
+.Xr fork 2 ,
 .Xr sigaction 2 ,
 .Xr wait 2 
-.Sh DIAGNOSTICS
-Same as for
-.Xr fork .
+.Sh ERRORS
+The
+.Fn vfork
+system call will fail for any of the reasons described
+in the
+.Xr fork
+man page.
+In addition, it will fail if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+A system call other than
+.Xr _exit()
+or
+.Xr execve()
+(or libc functions that make no system calls other than those)
+is called following calling a
+.Fn vfork
+call.
+.El
 .Sh BUGS
 This system call will be eliminated when proper system sharing
 mechanisms are implemented. 
diff --git a/bsd/man/man2/wait.2 b/bsd/man/man2/wait.2
index 054a0d0ac..6be0a7546 100644
--- a/bsd/man/man2/wait.2
+++ b/bsd/man/man2/wait.2
@@ -38,35 +38,47 @@
 .Os BSD 4
 .Sh NAME
 .Nm wait ,
-.Nm waitpid ,
+.Nm wait3 ,
 .Nm wait4 ,
-.Nm wait3
+.Nm waitpid
 .Nd wait for process termination
 .Sh SYNOPSIS
-.Fd #include <sys/types.h>
 .Fd #include <sys/wait.h>
 .Ft pid_t
-.Fn wait "int *status"
+.Fo wait
+.Fa "int *stat_loc"
+.Fc
 .Ft pid_t
-.Fn waitpid "pid_t wpid" "int *status" "int options"
-.Fd #include <sys/time.h>
-.Fd #include <sys/resource.h>
+.Fo wait3
+.Fa "int *stat_loc"
+.Fa "int options"
+.Fa "struct rusage *rusage"
+.Fc
 .Ft pid_t
-.Fn wait3 "int *status" "int options" "struct rusage *rusage"
+.Fo wait4
+.Fa "pid_t pid"
+.Fa "int *stat_loc"
+.Fa "int options"
+.Fa "struct rusage *rusage"
+.Fc
 .Ft pid_t
-.Fn wait4 "pid_t wpid" "int *status" "int options" "struct rusage *rusage"
+.Fo waitpid
+.Fa "pid_t pid"
+.Fa "int *stat_loc"
+.Fa "int options"
+.Fc
 .Sh DESCRIPTION
 The
 .Fn wait
 function suspends execution of its calling process until
-.Fa status
+.Fa stat_loc
 information is available for a terminated child process,
 or a signal is received.
 On return from a successful 
 .Fn wait
 call, 
 the
-.Fa status
+.Fa stat_loc
 area contains termination information about the process that exited
 as defined below.
 .Pp
@@ -80,27 +92,27 @@ The other wait functions are implemented using
 .Fn wait4 .
 .Pp
 The
-.Fa wpid
+.Fa pid
 parameter specifies the set of child processes for which to wait.
 If
-.Fa wpid
+.Fa pid
 is -1, the call waits for any child process.
 If
-.Fa wpid
+.Fa pid
 is 0,
 the call waits for any child process in the process group of the caller.
 If
-.Fa wpid
+.Fa pid
 is greater than zero, the call waits for the process with process id
-.Fa wpid .
+.Fa pid .
 If
-.Fa wpid
+.Fa pid
 is less than -1, the call waits for any process whose process group id
 equals the absolute value of
-.Fa wpid .
+.Fa pid .
 .Pp
 The
-.Fa status
+.Fa stat_loc
 parameter is defined below.  The
 .Fa options
 parameter contains the bitwise OR of any of the following options.
@@ -148,7 +160,7 @@ The older
 call is the same as
 .Fn wait4
 with a
-.Fa wpid
+.Fa pid
 value of -1.
 .Pp
 The following macros may be used to test the manner of exit of the process.
@@ -230,8 +242,8 @@ is returned and
 is set to indicate the error.
 .Pp
 If
+.Fn wait3 ,
 .Fn wait4 ,
-.Fn wait3
 or
 .Fn waitpid
 returns due to a stopped
@@ -253,38 +265,72 @@ is returned and
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Wait
-will fail and return immediately if:
+The
+.Fn wait
+system call will fail and return immediately if:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er ECHILD
-The calling process has no existing unwaited-for
-child processes.
+The calling process has no existing unwaited-for child processes.
+.\" ===========
 .It Bq Er EFAULT
 The
 .Fa status
 or
 .Fa rusage
-arguments point to an illegal address.
-(May not be detected before exit of a child process.)
-.It Bq Er EINTR
-The call was interrupted by a caught signal,
-or the signal did not have the
-.Dv SA_RESTART
-flag set.
+argument points to an illegal address
+(may not be detected before the exit of a child process).
+.\" ===========
 .It Bq Er EINVAL
-Invalid or underfined flags were passed in the
+Invalid or undefined flags are passed in the
 .Fa options
 argument.
 .El
+.Pp
+The
+.Fn wait3
+and
+.Fn waitpid
+calls will fail and return immediately if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er ECHILD
+The process specified by
+.Fa pid
+does not exist or is not a child of the calling process,
+or the process group specified by
+.Fa pid
+does not exist or does not have any member process
+that is a child of the calling process.
+.El
+.Pp
+The
+.Fn waitpid
+call will fail and return immediately if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINVAL
+The options argument is not valid.
+.El
+.Pp
+Any of these calls will fail and return immediately if:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EINTR
+The call is interrupted by a caught signal
+or the signal does not have the
+.Dv SA_RESTART
+flag set.
+.El
 .Sh STANDARDS
 The
 .Fn wait
 and
 .Fn waitpid
 functions are defined by POSIX;
-.Fn wait4
-and
 .Fn wait3
+and
+.Fn wait4
 are not specified by POSIX.
 The
 .Fn WCOREDUMP
@@ -292,9 +338,17 @@ macro
 and the ability to restart a pending
 .Fn wait
 call are extensions to the POSIX interface.
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/wait.h>
+.Pp
+The include file
+.In sys/types.h
+is necessary.
 .Sh SEE ALSO
+.Xr sigaction 2 ,
 .Xr exit 3 ,
-.Xr sigaction 2
+.Xr compat 5
 .Sh HISTORY
 A
 .Fn wait
diff --git a/bsd/man/man2/write.2 b/bsd/man/man2/write.2
index 941b122c3..fe41a2e53 100644
--- a/bsd/man/man2/write.2
+++ b/bsd/man/man2/write.2
@@ -36,28 +36,40 @@
 .Dt WRITE 2
 .Os
 .Sh NAME
+.Nm pwrite ,
 .Nm write ,
-.Nm writev ,
-.Nm pwrite
+.Nm writev
 .Nd write output
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
-.In sys/types.h
-.In sys/uio.h
 .In unistd.h
 .Ft ssize_t
-.Fn write "int d" "const void *buf" "size_t nbytes"
+.Fo pwrite
+.Fa "int fildes"
+.Fa "const void *buf"
+.Fa "size_t nbyte"
+.Fa "off_t offset"
+.Fc
 .Ft ssize_t
-.Fn writev "int d" "const struct iovec *iov" "int iovcnt"
+.Fo write
+.Fa "int fildes"
+.Fa "const void *buf"
+.Fa "size_t nbyte"
+.Fc
+.In sys/uio.h
 .Ft ssize_t
-.Fn pwrite "int d" "const void *buf" "size_t nbytes" "off_t offset"
+.Fo writev
+.Fa "int fildes"
+.Fa "const struct iovec *iov"
+.Fa "int iovcnt"
+.Fc
 .Sh DESCRIPTION
 .Fn Write
 attempts to write
-.Fa nbytes
+.Fa nbyte
 of data to the object referenced by the descriptor
-.Fa d
+.Fa fildes
 from the buffer pointed to by
 .Fa buf .
 .Fn Writev
@@ -96,7 +108,7 @@ On objects capable of seeking, the
 .Fn write
 starts at a position
 given by the pointer associated with
-.Fa d ,
+.Fa fildes ,
 see
 .Xr lseek 2 .
 Upon return from
@@ -116,8 +128,8 @@ by a user who
 a writable set-user-id file
 owned by the super-user.
 .Pp
-When using non-blocking I/O on objects such as sockets that are subject
-to flow control,
+When using non-blocking I/O on objects, such as sockets,
+that are subject to flow control,
 .Fn write
 and
 .Fn writev
@@ -125,80 +137,145 @@ may write fewer bytes than requested;
 the return value must be noted,
 and the remainder of the operation should be retried when possible.
 .Sh RETURN VALUES
-Upon successful completion the number of bytes which were written
-is returned.  Otherwise a -1 is returned and the global variable
+Upon successful completion the number of bytes
+which were written is returned.
+Otherwise, a -1 is returned and the global variable
 .Va errno
 is set to indicate the error.
 .Sh ERRORS
-.Fn Write ,
+The
+.Fn write ,
 .Fn writev ,
 and
 .Fn pwrite
-will fail and the file pointer will remain unchanged if:
+system calls will fail and the file pointer will remain unchanged if:
 .Bl -tag -width Er
-.It Bq Er EBADF
-.Fa D
-is not a valid descriptor open for writing.
-.It Bq Er EPIPE
-An attempt is made to write to a pipe that is not open
-for reading by any process.
-.It Bq Er EPIPE
-An attempt is made to write to a socket of type
-.Dv SOCK_STREAM
-that is not connected to a peer socket.
-.It Bq Er EFBIG
-An attempt was made to write a file that exceeds the process's
-file size limit or the maximum file size.
+.\" ===========
+.It Bq Er EDQUOT
+The user's quota of disk blocks on the file system
+containing the file is exhausted.
+.\" ===========
 .It Bq Er EFAULT
 Part of
 .Fa iov
 or data to be written to the file
 points outside the process's allocated address space.
+.\" ===========
 .It Bq Er EINVAL
 The pointer associated with
-.Fa d
-was negative.
-.It Bq Er ENOSPC
-There is no free space remaining on the file system
-containing the file.
-.It Bq Er EDQUOT
-The user's quota of disk blocks on the file system
-containing the file has been exhausted.
-.It Bq Er EIO
-An I/O error occurred while reading from or writing to the file system.
-.It Bq Er EINTR
-A signal interrupted the write before it could be completed.
+.Fa fildes
+is negative.
+.El
+.Pp
+The
+.Fn write
+and
+.Fn pwrite
+system calls will fail and the file pointer will remain unchanged if:
+.Bl -tag -width Er
+.\" ===========
 .It Bq Er EAGAIN
-The file was marked for non-blocking I/O,
+The file is marked for non-blocking I/O,
 and no data could be written immediately.
+.\" ===========
+.It Bq Er EBADF
+.Fa fildes
+is not a valid file descriptor open for writing.
+.\" ===========
+.It Bq Er ECONNRESET
+A write is attempted on a socket that is not connected.
+.\" ===========
+.It Bq Er EFBIG
+An attempt is made to write a file that exceeds the process's
+file size limit or the maximum file size.
+.\" ===========
+.It Bq Er EFBIG
+The file is a regular file,
+.Fa nbyte
+is greater than 0,
+and the starting position is greater than or equal
+to the offset maximum established in the open file description
+associated with
+.Fa fildes .
+.\" ===========
+.It Bq Er EINTR
+A signal interrupts the write before it could be completed.
+.\" ===========
+.It Bq Er EIO
+An I/O error occurs while reading from or writing to the file system.
+.\" ===========
+.It Bq Er ENETDOWN
+A write is attempted on a socket
+and the local network interface used to reach the destination is down.
+.\" ===========
+.It Bq Er ENETUNREACH
+A write is attempted on a socket and no route to the network is present.
+.\" ===========
+.It Bq Er ENOSPC
+There is no free space remaining on the file system containing the file.
+.\" ===========
+.It Bq Er ENXIO
+A request is made of a nonexistent device,
+or the request is outside the capabilities of the device.
+.\" ===========
+.It Bq Er EPIPE
+An attempt is made to write to a pipe that is not open
+for reading by any process.
+.\" ===========
+.It Bq Er EPIPE
+An attempt is made to write to a socket of type
+.Dv SOCK_STREAM
+that is not connected to a peer socket.
+.El
+.Pp
+The
+.Fn write
+and
+.Fn writev
+calls may also return the following errors:
+.Bl -tag -width Er
+.\" ===========
+.It Bq Er EAGAIN
+See EWOULDBLOCK, below.
+.\" ===========
+.It Bq Er EWOULDBLOCK
+The file descriptor is for a socket, is marked O_NONBLOCK,
+and write would block.
+The exact error code depends on the protocol,
+but EWOULDBLOCK is more common.
 .El
 .Pp
 In addition,
 .Fn writev
 may return one of the following errors:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EDESTADDRREQ
 The destination is no longer available when writing to a
 .Ux
 domain datagram socket on which
 .Xr connect 2
 had been used to set a destination address.
+.\" ===========
 .It Bq Er EINVAL
 .Fa Iovcnt
-was less than or equal to 0, or greater than
+is less than or equal to 0, or greater than
 .Dv UIO_MAXIOV .
+.\" ===========
 .It Bq Er EINVAL
 One of the
 .Fa iov_len
 values in the
 .Fa iov
-array was negative.
+array is negative.
+.\" ===========
 .It Bq Er EINVAL
 The sum of the
 .Fa iov_len
 values in the
 .Fa iov
-array overflowed a 32-bit integer.
+array overflows a 32-bit integer.
+.\" ===========
 .It Bq Er ENOBUFS
 The mbuf pool has been completely exhausted when writing to a socket.
 .El
@@ -207,17 +284,26 @@ The
 .Fn pwrite
 call may also return the following errors:
 .Bl -tag -width Er
+.\" ===========
 .It Bq Er EINVAL
 The specified file offset is invalid.
+.\" ===========
 .It Bq Er ESPIPE
 The file descriptor is associated with a pipe, socket, or FIFO.
 .El
+.Sh LEGACY SYNOPSIS
+.Fd #include <sys/types.h>
+.Fd #include <sys/uio.h>
+.Fd #include <unistd.h>
+.Pp
+These include files are needed for all three functions.
 .Sh SEE ALSO
 .Xr fcntl 2 ,
 .Xr lseek 2 ,
 .Xr open 2 ,
 .Xr pipe 2 ,
-.Xr select 2
+.Xr select 2 ,
+.Xr compat 5
 .Sh STANDARDS
 The
 .Fn write
diff --git a/bsd/man/man3/Makefile b/bsd/man/man3/Makefile
new file mode 100644
index 000000000..605cee606
--- /dev/null
+++ b/bsd/man/man3/Makefile
@@ -0,0 +1,62 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+DATAFILES = \
+	queue.3 \
+	posix_spawn_file_actions_addclose.3  \
+	posix_spawn_file_actions_init.3  \
+	posix_spawnattr_init.3  \
+	posix_spawnattr_setbinpref_np.3  \
+	posix_spawnattr_setflags.3  \
+	posix_spawnattr_setpgroup.3  \
+	posix_spawnattr_setsigdefault.3  \
+	posix_spawnattr_setsigmask.3  \
+	posix_spawnattr_setspecialport_np.3 
+
+INSTALL_MAN_LIST = ${DATAFILES}
+
+INSTALL_MAN_LINKS = \
+	queue.3 CIRCLEQ_ENTRY.3 \
+	queue.3 CIRCLEQ_HEAD.3 \
+	queue.3 CIRCLEQ_INIT.3 \
+	queue.3 CIRCLEQ_INSERT_AFTER.3 \
+	queue.3 CIRCLEQ_INSERT_BEFORE.3 \
+	queue.3 CIRCLEQ_INSERT_HEAD.3 \
+	queue.3 CIRCLEQ_INSERT_TAIL.3 \
+	queue.3 CIRCLEQ_REMOVE.3 \
+	queue.3 LIST_ENTRY.3 \
+	queue.3 LIST_HEAD.3 \
+	queue.3 LIST_INIT.3 \
+	queue.3 LIST_INSERT_AFTER.3 \
+	queue.3 LIST_INSERT_BEFORE.3 \
+	queue.3 LIST_INSERT_HEAD.3 \
+	queue.3 LIST_REMOVE.3 \
+	queue.3 TAILQ_ENTRY.3 \
+	queue.3 TAILQ_HEAD.3 \
+	queue.3 TAILQ_INIT.3 \
+	queue.3 TAILQ_INSERT_AFTER.3 \
+	queue.3 TAILQ_INSERT_BEFORE.3 \
+	queue.3 TAILQ_INSERT_HEAD.3 \
+	queue.3 TAILQ_INSERT_TAIL.3 \
+	queue.3 TAILQ_REMOVE.3 \
+	posix_spawn_file_actions_addclose.3 posix_spawn_file_actions_addopen.3 \
+	posix_spawn_file_actions_addclose.3 posix_spawn_file_actions_adddup2.3 \
+	posix_spawn_file_actions_init.3 posix_spawn_file_actions_destroy.3 \
+	posix_spawnattr_init.3 posix_spawnattr_destroy.3 \
+	posix_spawnattr_setbinpref_np.3 posix_spawnattr_getbinpref_np.3 \
+	posix_spawnattr_setflags.3 posix_spawnattr_getflags.3 \
+	posix_spawnattr_setpgroup.3 posix_spawnattr_getpgroup.3 \
+	posix_spawnattr_setsigdefault.3 posix_spawnattr_getsigdefault.3 \
+	posix_spawnattr_setsigmask.3 posix_spawnattr_getsigmask.3 \
+	posix_spawnattr_setspecialport_np.3 posix_spawnattr_setexceptionports_np.3
+
+INSTALL_MAN_DIR = man3
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
+
diff --git a/bsd/man/man3/posix_spawn_file_actions_addclose.3 b/bsd/man/man3/posix_spawn_file_actions_addclose.3
new file mode 100644
index 000000000..6cd2033c3
--- /dev/null
+++ b/bsd/man/man3/posix_spawn_file_actions_addclose.3
@@ -0,0 +1,154 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawn_file_actions_addclose.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWN_FILE_ACTIONS_ADDCLOSE 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawn_file_actions_addclose
+.Nm posix_spawn_file_actions_addopen
+.Nd add open or close actions to a
+.Em posix_spawn_file_actions_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawn_file_actions_addclose
+.Fa "posix_spawn_file_actions_t *file_actions"
+.Fa "int filedes"
+.Fc
+.Ft int
+.Fo posix_spawn_file_actions_addopen
+.Fa "posix_spawn_file_actions_t *restrict file_actions"
+.Fa "int filedes"
+.Fa "const char *restrict path"
+.Fa "int oflag"
+.Fa "mode_t mode"
+.Fc
+.Ft int
+.Fo posix_spawn_file_actions_adddup2
+.Fa "posix_spawn_file_actions_t *file_actions"
+.Fa "int filedes"
+.Fa "int newfiledes"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawn_file_actions_addclose
+function adds a close operation to the list of operations associated with
+the object referenced by
+.Em file_actions ,
+for subsequent use in a call to
+.Xr posix_spawn 2
+or 
+.Xr posix_spawnp 2 .
+The descriptor referred to by
+.Em filedes
+is closed as if
+.Fn close
+had been called on it prior to the new child process
+starting execution.
+.Pp
+The
+.Fn posix_spawn_file_actions_addopen
+function adds an open operation to the list of operations associated with
+the object referenced by
+.Em file_actions ,
+for subsequent use in a call to
+.Xr posix_spawn 2
+or 
+.Xr posix_spawnp 2 .
+The descriptor referred to by
+.Em filedes
+is opened using the
+.Em path ,
+.Em oflag ,
+and
+.Em mode
+arguments as if
+.Fn open
+had been called on it prior to the new child process
+starting execution.  The string
+.Em path
+is copied by the
+.Fn posix_spawn_file_actions_addopen
+function during this process, so storage need not be persistent in the
+caller.
+.Pp
+The
+.Fn posix_spawn_file_actions_adddup2
+function adds a dup2 operation to the list of operations associated with
+the object referenced by
+.Em file_actions ,
+for subsequent use in a call to
+.Xr posix_spawn 2
+or 
+.Xr posix_spawnp 2 .
+The descriptor referred to by
+.Em newfiledes
+is created as if
+.Fn dup2
+had been called on
+.Em filedes
+prior to the new child process starting execution.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EBADF
+The value specified by
+.Fa filedes
+is negative or would cause the process to exceed the maximum number of
+open files it is allowed..
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa file_actions
+is invalid.
+.\" ==========
+.It Bq Er ENOMEM
+Insufficient memory was available eo add to the
+.Fa file_actions .
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawn_file_actions_init 3 ,
+.Xr posix_spawn_file_actions_destroy 3 ,
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawn_file_actions_init
+and
+.Fn posix_spawn_file_actions_destroy
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawn_file_actions_init.3 b/bsd/man/man3/posix_spawn_file_actions_init.3
new file mode 100644
index 000000000..6be07a4ae
--- /dev/null
+++ b/bsd/man/man3/posix_spawn_file_actions_init.3
@@ -0,0 +1,115 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawn_file_actions_init.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWN_FILE_ACTIONS_INIT 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawn_file_actions_init
+.Nm posix_spawn_file_actions_destroy
+.Nd initialize or destroy spawn file actions objects
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawn_file_actions_init
+.Fa "posix_spawn_file_actions_t *file_actions"
+.Fc
+.Ft int
+.Fo posix_spawn_file_actions_destroy
+.Fa "posix_spawn_file_actions_t *file_actions"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawn_file_actions_init
+function initializes the object referenced by
+.Em file_actions ,
+to an empty set of file actions for subsequent use in a call to
+.Xr posix_spawn 2
+or 
+.Xr posix_spawnp 2 .
+A
+.Em posix_spawn_file_actions_t
+object is an opaque type that is used to collect set of file actions
+specified by calls to
+.Xr posix_spawn_file_actions_addclose 3
+and
+.Xr posix_spawn_file_actions_addopen 3
+such that once the spawn operation is complete, the child process will
+have a different set of file descriptors than the parent, in the order
+that the operations were added to the
+.Em posix_spawn_file_actions_t .
+.Pp
+Additionally, any descriptor in the parent not modified by a file action,
+and which has the FD_CLOEXEC flag set, will be closed in the child.
+.Pp
+The
+.Fn posix_spawn_file_actions_destroy
+function destroys the object referenced by
+.Em file_actions
+which was previously intialized by
+.Fn posix_spawn_file_actions_init ,
+returning any resources obtained at the time of initialization to the
+system for subsequent reuse.  A
+.Em posix_spawn_file_actions_t
+may be reinitialized after having been destroyed, but
+.Em must not
+be reused after destruction, unless it has been reinitialized.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+.Sh ERRORS
+These functions will fail and return an error if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er ENOMEM
+Insufficient resources to initialize the
+.Fa posix_spawn_file_actions_t
+object.
+.\" ==========
+.It Bq Er EINVAL
+The
+.Fa posix_spawn_file_actions_t
+being destroyed is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawn_file_actions_addclose 3 ,
+.Xr posix_spawn_file_actions_addopen 3 ,
+.Xr posix_spawn_file_actions_adddup2 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawn_file_actions_init
+and
+.Fn posix_spawn_file_actions_destroy
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_init.3 b/bsd/man/man3/posix_spawnattr_init.3
new file mode 100644
index 000000000..27759a591
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_init.3
@@ -0,0 +1,126 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_init.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_INIT 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_init
+.Nm posix_spawnattr_destroy
+.Nd initialize or destroy a spawn attributes object
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_init
+.Fa "posix_spawnattr_t *attr"
+.Fc
+.Ft int
+.Fo posix_spawnattr_destroy
+.Fa "posix_spawnattr_t *attr"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_init
+function initializes the object referenced by
+.Em attr ,
+to an empty set of spawn attributes for subsequent use in a call to
+.Xr posix_spawn 2
+or
+.Xr posix_spawnp 2 .
+.Pp
+A
+.Em posix_spawnattr_t
+object is an opaque type that is used to collect set of spawn attributes
+specified by calls to
+.Xr posix_spawnattr_setflags 3 ,
+.Xr posix_spawnattr_setpgroup 3 ,
+.Xr posix_spawnattr_setsigdefault 3 ,
+.Xr posix_spawnattr_setsigmask 3 ,
+.Xr posix_spawnattr_setbinpref_np 3 ,
+.Xr posix_spawnattr_setspecialport_np 3 ,
+or
+.Xr posix_spawnattr_setexceptionports_np 3
+such that once the spawn operation is complete, the child process will
+have a different set of attributes than the parent.  The order of these
+operations is unspecified and may change; programs should not depend on
+their ordering.
+.Pp
+The
+.Fn posix_spawnattr_destroy
+function destroys the object referenced by
+.Em attr 
+which was previously intialized by
+.Fn posix_spawnattr_init ,
+returning any resources obtained at the time of initialization to the
+system for subsequent reuse.  A
+.Em posix_spawnattr_t
+may be reinitialized after having been destroyed, but
+.Em must not
+be reused after destruction, unless it has been reinitialized.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+.Sh ERRORS
+The
+.Fn posix_spawnattr_init
+and
+.Fn posix_spawnattr_destroy
+functions will fail and return an error if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er ENOMEM
+Insufficient resources to initialize the
+.Fa posix_spawnattr_t
+object.
+.\" ==========
+.It Bq Er EINVAL
+The 
+.Fa posix_spawnattr_t
+being destroyed is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_setflags 3 ,
+.Xr posix_spawnattr_setpgroup 3 ,
+.Xr posix_spawnattr_setsigdefault 3 ,
+.Xr posix_spawnattr_setsigmask 3 ,
+.Xr posix_spawnattr_setbinpref_np 3 ,
+.Xr posix_spawnattr_setspecialport_np 3 ,
+.Xr posix_spawnattr_setexceptionports_np 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawn
+and
+.Fn posix_spawnp
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_setbinpref_np.3 b/bsd/man/man3/posix_spawnattr_setbinpref_np.3
new file mode 100644
index 000000000..5ddbd6582
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setbinpref_np.3
@@ -0,0 +1,152 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setbinpref_np.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETBINPREF_NP 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setbinpref_np
+.Nm posix_spawnattr_getbinpref_np
+.Nd set or get the
+.Em spawn-binpref
+attribute on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setbinpref_np
+.Fa "posix_spawnattr_t *restrict attr"
+.Fa "size_t count"
+.Fa "cpu_type_t *pref"
+.Fa "size_t *restrict ocount"
+.Fc
+.Ft int
+.Fo posix_spawnattr_getbinpref_np
+.Fa "const posix_spawnattr_t *restrict attr"
+.Fa "size_t count"
+.Fa "cpu_type_t *pref"
+.Fa "size_t * restrict ocount"
+.Fc
+.Sh IMPORTANT
+These functions represent an Apple extension to
+.Xr posix_spawn 2
+and
+.Xr posix_spawnp 2 ,
+and as such should not be used by programs intending their code to be
+portable to other platforms.
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setbinpref_np
+function sets the universal binary preferences for the spawn attribute
+value referenced by
+.Fa attr
+from the memory containing the
+.Em cpu_type_t
+referenced by 
+.Fa pref
+with a size of
+.Fa count
+elements; the actual number of elements that are set in the attribute
+is returned in
+.Fa ocount .
+.Pp
+When
+.Xr spawn 2
+or
+.Xr spawnp 2
+is subsequently invoked on a Universal binary with the
+.Em posix_spawnattr_t ,
+the elements which were set will be used, in the order they were set,
+to select the first element in the list which matches any
+.Em cpu_type_t
+of those available in the Universal binary.  If there is no match, then
+the attempt to create the child process will fail with the error
+EBADARCH.
+If the
+.Em cpu_type_t
+.Em CPU_TYPE_ANY
+is the last element in the list, then rather than returning
+EBADARCH
+on no match, the system will instead fall back to the standard Universal
+binary grading preference order.
+.Pp
+The
+.Fn posix_spawnattr_getbinpref_np
+function gets the universal binary preferences for the spawn attribute
+value referenced by
+.Fa attr
+into the memory containing the
+.Em cpu_type_t
+referenced by 
+.Fa pref
+with a prereserved size of
+.Fa count
+elements; the actual number of elements that are copied from the attribute
+is returned in
+.Fa ocount .
+.Pp
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h 
+and modify the value of
+.Fa ocount .
+Additionally, if successful,
+.Fn posix_spawnattr_getbinpref_np
+will modify the contents of the
+.Fa pref
+array with the current attribute values.
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3 ,
+.Xr posix_spawnattr_setflags 3
+.Sh STANDARDS
+Nonstandard
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setbinpref_np
+and
+.Fn posix_spawnattr_getbinpref_np
+function calls appeared in Mac OS X.
diff --git a/bsd/man/man3/posix_spawnattr_setflags.3 b/bsd/man/man3/posix_spawnattr_setflags.3
new file mode 100644
index 000000000..78cd67398
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setflags.3
@@ -0,0 +1,169 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setflags.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETFLAGS 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setflags
+.Nm posix_spawnattr_getflags
+.Nd get or set flags on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setflags
+.Fa "posix_spawnattr_t *attr"
+.Fa "short flags"
+.Fc
+.Ft int
+.Fo posix_spawnattr_getflags
+.Fa "const posix_spawnattr_t *restrict attr"
+.Fa "short *restrict flags"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setflags
+function sets the
+.Fa flags
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The
+.Fn posix_spawnattr_getflags
+function retrieves the
+.Fa flags
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The argument
+.Fa flags
+is either 0 or a logical OR of one or more of the following flags:
+.Bl -tag -width POSIX_SPAWN_START_SUSPENDED
+.It Dv POSIX_SPAWN_RESETIDS
+If the set group bit is set on the process image being spawned, this
+bit has no effect; otherwise, if not set, the child process will
+inherit the effective group ID of the parent process, and if set, the
+child process will inherit the real group ID of the parent process.
+.It Dv POSIX_SPAWN_SETPGROUP
+If this bit is not set, then the child process inherits the parent
+process group; if set, then the child process shall behave as if the
+.Xr setpgid 2
+function had been called with a
+.Fa pid
+parameter of 0 and a
+.Fa pgid
+parameter equal to the value of the spawn-pgroup value of the
+.Em posix_spawnattr_t ,
+as set by
+.Xr posix_spawnattr_setpgroup 3
+.It Dv POSIX_SPAWN_SETSIGDEF
+Signals set to to either be caught or to the default action in the
+parent process will also be set to the default action in the child
+process.  Signals set to be ignored in the parent process will be
+ignored in the child.  However, if this bit is set, then signals in
+the
+.Em spawn-sigdefault
+value of the
+.Em posix_spawnattr_t ,
+as set by
+.Xr posix_spawnattr_setsigdefault 3
+which are caught or ignored in the parent will instead be reset to
+their default actions in the child.
+.It Dv POSIX_SPAWN_SETSIGMASK
+If this bit is set, then the initial signal mask of the child process will
+be set to the
+.Em spawn-sigmask
+value of the
+.Em posix_spawnattr_t ,
+as set by
+.Xr posix_spawnattr_setsigmask 3 .
+.It Dv POSIX_SPAWN_SETEXEC
+.Em Apple Extension :
+If this bit is set, rather than returning to the caller,
+.Xr posix_spawn 2
+and
+.Xr posix_spawnp 2
+will behave as a more featureful
+.Xr execve 2 .
+.It Dv POSIX_SPAWN_START_SUSPENDED
+.Em Apple Extension :
+If this bit is set, then the child process will be created with its task
+suspended, permitting debuggers, profilers, and other programs to
+manipulate the process before it begins execution in user space.  This
+permits, for example, obtaining exact instruction counts, or debugging
+very early in
+.Xr dyld 1 .
+.El
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+The
+.Fn posix_spawnattr_getflags
+additionally, upon successful completion, modifies the value pointed to be the
+.Fa attr
+argument by making it equal to the
+.Em spawn-flags
+attribute of the
+.Em posix_spawnattr_t .
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3 ,
+.Xr posix_spawnattr_setpgroup 3 ,
+.Xr posix_spawnattr_setsigdefault 3 ,
+.Xr posix_spawnattr_setsigmask 3 ,
+.Xr setpgid 2 ,
+.Xr execve 2 ,
+.Xr dyld 1
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setflags
+and
+.Fn posix_spawnattr_getflags
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_setpgroup.3 b/bsd/man/man3/posix_spawnattr_setpgroup.3
new file mode 100644
index 000000000..1b2bcacde
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setpgroup.3
@@ -0,0 +1,114 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setpgroup.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETPGROUP 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setpgroup
+.Nm posix_spawnattr_getpgroup
+.Nd set or get the
+.Em spawn-pgroup
+attribute on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setpgroup
+.Fa "posix_spawnattr_t *attr"
+.Fa "pid_t pgroup"
+.Fc
+.Ft int
+.Fo posix_spawnattr_getpgroup
+.Fa "const posix_spawnattr_t *restrict attr"
+.Fa "pid_t *restrict pgroup"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setpgroup
+function sets the
+.Fa pgroup
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The
+.Fn posix_spawnattr_getpgroup
+function retrieves the
+.Fa pgroup
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The argument
+.Fa pgroup
+is the process group to be joined or created when the new process is
+created if the
+.Em POSIX_SPAWN_SETPGROUP
+flag is set in the
+.Em posix_spawnattr_t
+via a call to
+.Xr posix_spawnattr_setflags 3 ;
+its default value is 0, which means a new process group will be created.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+Additionally, if successful,
+.Fn posix_spawnattr_getpgroup
+will modify the contents of the
+.Fa pgroup
+parameter with the current attribute value.
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3 ,
+.Xr posix_spawnattr_setflags 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setpgroup
+and
+.Fn posix_spawnattr_getpgroup
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_setsigdefault.3 b/bsd/man/man3/posix_spawnattr_setsigdefault.3
new file mode 100644
index 000000000..25db194ad
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setsigdefault.3
@@ -0,0 +1,114 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setsigdefault.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETSIGDEFAULT 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setsigdefault
+.Nm posix_spawnattr_getsigdefault
+.Nd set or get the
+.Em spawn-sigdefault
+attribute on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setsigdefault
+.Fa "posix_spawnattr_t *restrict attr"
+.Fa "const sigset_t *restrict sigdefault"
+.Fc
+.Ft int
+.Fo posix_spawnattr_getsigdefault
+.Fa "const posix_spawnattr_t *restrict attr"
+.Fa "sigset_t *restrict sigdefault"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setsigdefault
+function sets the
+.Fa sigdefault
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The
+.Fn posix_spawnattr_getsigdefault
+function retrieves the
+.Fa sigdefault
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The argument
+.Fa sigdefault
+is the set of signals to be returned to default behaviour in the new
+process on creation if the
+.Em POSIX_SPAWN_SETSIGDEF
+flag is set in the
+.Em posix_spawnattr_t .
+It's default value must be set via a call to
+.Xr posix_spawnattr_setflags 3
+or the behaviour is undefined.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+Additionally, if successful,
+.Fn posix_spawnattr_getsigdefault
+will modify the contents of the
+.Fa sigdefault
+parameter with the current attribute value.
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3 ,
+.Xr posix_spawnattr_setflags 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setsigdefault
+and
+.Fn posix_spawnattr_getsigdefault
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_setsigmask.3 b/bsd/man/man3/posix_spawnattr_setsigmask.3
new file mode 100644
index 000000000..86914e94c
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setsigmask.3
@@ -0,0 +1,113 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setsigmask.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETSIGMASK 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setsigmask
+.Nm posix_spawnattr_getsigmask
+.Nd set or get the
+.Em spawn-sigmask
+attribute on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setsigmask
+.Fa "posix_spawnattr_t *restrict attr"
+.Fa "const sigset_t *restrict sigmask"
+.Fc
+.Ft int
+.Fo posix_spawnattr_getsigmask
+.Fa "const posix_spawnattr_t *restrict attr"
+.Fa "sigset_t *restrict sigmask"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setsigmask
+function sets the
+.Fa sigmask
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The
+.Fn posix_spawnattr_getsigmask
+function retrieves the
+.Fa sigmask
+on the attributes object referenced by
+.Fa attr .
+.Pp
+The argument
+.Fa sigmask
+is the initial signal mask to be set for the new process on creation if the
+.Em POSIX_SPAWN_SETSIGMASK
+flag is set in the
+.Em posix_spawnattr_t .
+It's default value must be set via a call to
+.Xr posix_spawnattr_setflags 3
+or the behaviour is undefined.
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+Additionally, if successful,
+.Fn posix_spawnattr_getsigmask
+will modify the contents of the
+.Fa sigmask
+parameter with the current attribute value.
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3 ,
+.Xr posix_spawnattr_setflags 3
+.Sh STANDARDS
+.St -susv3 [SPN]
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setsigmask
+and
+.Fn posix_spawnattr_getsigmask
+function calls appeared in
+.St -susv3 [SPN] .
diff --git a/bsd/man/man3/posix_spawnattr_setspecialport_np.3 b/bsd/man/man3/posix_spawnattr_setspecialport_np.3
new file mode 100644
index 000000000..8110125e2
--- /dev/null
+++ b/bsd/man/man3/posix_spawnattr_setspecialport_np.3
@@ -0,0 +1,125 @@
+.\"
+.\" Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+.\"
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+.\" 
+.\" This file contains Original Code and/or Modifications of Original Code
+.\" as defined in and that are subject to the Apple Public Source License
+.\" Version 2.0 (the 'License'). You may not use this file except in
+.\" compliance with the License. The rights granted to you under the License
+.\" may not be used to create, or enable the creation or redistribution of,
+.\" unlawful or unlicensed copies of an Apple operating system, or to
+.\" circumvent, violate, or enable the circumvention or violation of, any
+.\" terms of an Apple operating system software license agreement.
+.\" 
+.\" Please obtain a copy of the License at
+.\" http://www.opensource.apple.com/apsl/ and read it before using this file.
+.\" 
+.\" The Original Code and all software distributed under the License are
+.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+.\" Please see the License for the specific language governing rights and
+.\" limitations under the License.
+.\" 
+.\" @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+.\"
+.\"     @(#)posix_spawnattr_setspecialport_np.3
+.
+.Dd August 22, 2007
+.Dt POSIX_SPAWNATTR_SETSPECIALPORT_NP 3
+.Os "Mac OS X"
+.Sh NAME
+.Nm posix_spawnattr_setspecialport_np
+.Nm posix_spawnattr_setexceptionports_np
+.Nd set or get the
+.Em spawn-binpref
+attribute on a
+.Em posix_spawnattr_t
+.Sh SYNOPSIS
+.Fd #include <spawn.h>
+.Ft int
+.Fo posix_spawnattr_setspecialport_np
+.Fa "posix_spawnattr_t *attr"
+.Fa "mach_port_t new_port"
+.Fa "int which"
+.Fc
+.Ft int
+.Fo posix_spawnattr_setexceptionports_np
+.Fa "posix_spawnattr_t *attr"
+.Fa "exception_mask_t mask"
+.Fa "mach_port_t new_port"
+.Fa "exception_behavior_t behavior"
+.Fa "thread_state_flavor_t flavor"
+.Fc
+.Sh IMPORTANT
+These functions represent an Apple extension to
+.Xr posix_spawn 2
+and
+.Xr posix_spawnp 2 ,
+and as such should not be used by programs intending their code to be
+portable to other platforms.
+.Sh DESCRIPTION
+The
+.Fn posix_spawnattr_setspecialport_np
+function sets a new value for a
+.Em Mach
+special port into the value referenced by
+.Fa attr
+to the port referenced by
+.Fa new_port
+for a port value of
+.Fa which ,
+as if the new process had called
+.Fn task_set_special_port .
+.Pp
+The
+.Fn posix_spawnattr_setexceptionports_np
+function sets the
+.Em Mach
+port to
+.Fa new_port
+for a number of exception ports represented by the bitfield
+.Fa mask ,
+and sets the new default
+.Fa behavior
+and default
+.Fa flavor ,
+as if the new process had called
+.Fn task_set_exception_ports .
+.Pp
+.Sh RETURN VALUES
+On success, these functions return 0; on failure they return an error
+number from
+.In errno.h .
+.Sh ERRORS
+These functions may fail if:
+.Bl -tag -width Er
+.\" ==========
+.It Bq Er EINVAL
+The value specified by
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er EINVAL
+The value of
+.Fa attr
+is invalid.
+.\" ==========
+.It Bq Er ENOMEM
+Insufficient resources exist to set the ports.
+.El
+.Sh SEE ALSO
+.Xr posix_spawn 2 ,
+.Xr posix_spawnp 2 ,
+.Xr posix_spawnattr_init 3 ,
+.Xr posix_spawnattr_destroy 3
+.Sh STANDARDS
+Nonstandard
+.Sh HISTORY
+The
+.Fn posix_spawnattr_setspecialport_np
+and
+.Fn posix_spawnattr_setexceptionports_np
+function calls appeared in Mac OS X.
diff --git a/bsd/man/man3/queue.3 b/bsd/man/man3/queue.3
new file mode 100644
index 000000000..598f2fd22
--- /dev/null
+++ b/bsd/man/man3/queue.3
@@ -0,0 +1,977 @@
+.\" Copyright (c) 1993
+.\"	The Regents of the University of California.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 3. All advertising materials mentioning features or use of this software
+.\"    must display the following acknowledgement:
+.\"	This product includes software developed by the University of
+.\"	California, Berkeley and its contributors.
+.\" 4. Neither the name of the University nor the names of its contributors
+.\"    may be used to endorse or promote products derived from this software
+.\"    without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"	@(#)queue.3	8.2 (Berkeley) 1/24/94
+.\" $FreeBSD: /repoman/r/ncvs/src/share/man/man3/queue.3,v 1.37 2006/03/22 02:40:38 mckusick Exp $
+.\"
+.Dd January 24, 1994
+.Dt QUEUE 3
+.Os
+.Sh NAME
+.Nm SLIST_EMPTY ,
+.Nm SLIST_ENTRY ,
+.Nm SLIST_FIRST ,
+.Nm SLIST_FOREACH ,
+.Nm SLIST_FOREACH_SAFE ,
+.Nm SLIST_HEAD ,
+.Nm SLIST_HEAD_INITIALIZER ,
+.Nm SLIST_INIT ,
+.Nm SLIST_INSERT_AFTER ,
+.Nm SLIST_INSERT_HEAD ,
+.Nm SLIST_NEXT ,
+.Nm SLIST_REMOVE_HEAD ,
+.Nm SLIST_REMOVE ,
+.Nm STAILQ_CONCAT ,
+.Nm STAILQ_EMPTY ,
+.Nm STAILQ_ENTRY ,
+.Nm STAILQ_FIRST ,
+.Nm STAILQ_FOREACH ,
+.Nm STAILQ_FOREACH_SAFE ,
+.Nm STAILQ_HEAD ,
+.Nm STAILQ_HEAD_INITIALIZER ,
+.Nm STAILQ_INIT ,
+.Nm STAILQ_INSERT_AFTER ,
+.Nm STAILQ_INSERT_HEAD ,
+.Nm STAILQ_INSERT_TAIL ,
+.Nm STAILQ_LAST ,
+.Nm STAILQ_NEXT ,
+.Nm STAILQ_REMOVE_HEAD ,
+.Nm STAILQ_REMOVE ,
+.Nm LIST_EMPTY ,
+.Nm LIST_ENTRY ,
+.Nm LIST_FIRST ,
+.Nm LIST_FOREACH ,
+.Nm LIST_FOREACH_SAFE ,
+.Nm LIST_HEAD ,
+.Nm LIST_HEAD_INITIALIZER ,
+.Nm LIST_INIT ,
+.Nm LIST_INSERT_AFTER ,
+.Nm LIST_INSERT_BEFORE ,
+.Nm LIST_INSERT_HEAD ,
+.Nm LIST_NEXT ,
+.Nm LIST_REMOVE ,
+.Nm TAILQ_CONCAT ,
+.Nm TAILQ_EMPTY ,
+.Nm TAILQ_ENTRY ,
+.Nm TAILQ_FIRST ,
+.Nm TAILQ_FOREACH ,
+.Nm TAILQ_FOREACH_SAFE ,
+.Nm TAILQ_FOREACH_REVERSE ,
+.Nm TAILQ_FOREACH_REVERSE_SAFE ,
+.Nm TAILQ_HEAD ,
+.Nm TAILQ_HEAD_INITIALIZER ,
+.Nm TAILQ_INIT ,
+.Nm TAILQ_INSERT_AFTER ,
+.Nm TAILQ_INSERT_BEFORE ,
+.Nm TAILQ_INSERT_HEAD ,
+.Nm TAILQ_INSERT_TAIL ,
+.Nm TAILQ_LAST ,
+.Nm TAILQ_NEXT ,
+.Nm TAILQ_PREV ,
+.Nm TAILQ_REMOVE
+.Nd implementations of singly-linked lists, singly-linked tail queues,
+lists and tail queues
+.Sh SYNOPSIS
+.In sys/queue.h
+.\"
+.Fn SLIST_EMPTY "SLIST_HEAD *head"
+.Fn SLIST_ENTRY "TYPE"
+.Fn SLIST_FIRST "SLIST_HEAD *head"
+.Fn SLIST_FOREACH "TYPE *var" "SLIST_HEAD *head" "SLIST_ENTRY NAME"
+.Fn SLIST_FOREACH_SAFE "TYPE *var" "SLIST_HEAD *head" "SLIST_ENTRY NAME" "TYPE *temp_var"
+.Fn SLIST_HEAD "HEADNAME" "TYPE"
+.Fn SLIST_HEAD_INITIALIZER "SLIST_HEAD head"
+.Fn SLIST_INIT "SLIST_HEAD *head"
+.Fn SLIST_INSERT_AFTER "TYPE *listelm" "TYPE *elm" "SLIST_ENTRY NAME"
+.Fn SLIST_INSERT_HEAD "SLIST_HEAD *head" "TYPE *elm" "SLIST_ENTRY NAME"
+.Fn SLIST_NEXT "TYPE *elm" "SLIST_ENTRY NAME"
+.Fn SLIST_REMOVE_HEAD "SLIST_HEAD *head" "SLIST_ENTRY NAME"
+.Fn SLIST_REMOVE "SLIST_HEAD *head" "TYPE *elm" "TYPE" "SLIST_ENTRY NAME"
+.\"
+.Fn STAILQ_CONCAT "STAILQ_HEAD *head1" "STAILQ_HEAD *head2"
+.Fn STAILQ_EMPTY "STAILQ_HEAD *head"
+.Fn STAILQ_ENTRY "TYPE"
+.Fn STAILQ_FIRST "STAILQ_HEAD *head"
+.Fn STAILQ_FOREACH "TYPE *var" "STAILQ_HEAD *head" "STAILQ_ENTRY NAME"
+.Fn STAILQ_FOREACH_SAFE "TYPE *var" "STAILQ_HEAD *head" "STAILQ_ENTRY NAME" "TYPE *temp_var"
+.Fn STAILQ_HEAD "HEADNAME" "TYPE"
+.Fn STAILQ_HEAD_INITIALIZER "STAILQ_HEAD head"
+.Fn STAILQ_INIT "STAILQ_HEAD *head"
+.Fn STAILQ_INSERT_AFTER "STAILQ_HEAD *head" "TYPE *listelm" "TYPE *elm" "STAILQ_ENTRY NAME"
+.Fn STAILQ_INSERT_HEAD "STAILQ_HEAD *head" "TYPE *elm" "STAILQ_ENTRY NAME"
+.Fn STAILQ_INSERT_TAIL "STAILQ_HEAD *head" "TYPE *elm" "STAILQ_ENTRY NAME"
+.Fn STAILQ_LAST "STAILQ_HEAD *head" "TYPE" "STAILQ_ENTRY NAME"
+.Fn STAILQ_NEXT "TYPE *elm" "STAILQ_ENTRY NAME"
+.Fn STAILQ_REMOVE_HEAD "STAILQ_HEAD *head" "STAILQ_ENTRY NAME"
+.Fn STAILQ_REMOVE "STAILQ_HEAD *head" "TYPE *elm" "TYPE" "STAILQ_ENTRY NAME"
+.\"
+.Fn LIST_EMPTY "LIST_HEAD *head"
+.Fn LIST_ENTRY "TYPE"
+.Fn LIST_FIRST "LIST_HEAD *head"
+.Fn LIST_FOREACH "TYPE *var" "LIST_HEAD *head" "LIST_ENTRY NAME"
+.Fn LIST_FOREACH_SAFE "TYPE *var" "LIST_HEAD *head" "LIST_ENTRY NAME" "TYPE *temp_var"
+.Fn LIST_HEAD "HEADNAME" "TYPE"
+.Fn LIST_HEAD_INITIALIZER "LIST_HEAD head"
+.Fn LIST_INIT "LIST_HEAD *head"
+.Fn LIST_INSERT_AFTER "TYPE *listelm" "TYPE *elm" "LIST_ENTRY NAME"
+.Fn LIST_INSERT_BEFORE "TYPE *listelm" "TYPE *elm" "LIST_ENTRY NAME"
+.Fn LIST_INSERT_HEAD "LIST_HEAD *head" "TYPE *elm" "LIST_ENTRY NAME"
+.Fn LIST_NEXT "TYPE *elm" "LIST_ENTRY NAME"
+.Fn LIST_REMOVE "TYPE *elm" "LIST_ENTRY NAME"
+.\"
+.Fn TAILQ_CONCAT "TAILQ_HEAD *head1" "TAILQ_HEAD *head2" "TAILQ_ENTRY NAME"
+.Fn TAILQ_EMPTY "TAILQ_HEAD *head"
+.Fn TAILQ_ENTRY "TYPE"
+.Fn TAILQ_FIRST "TAILQ_HEAD *head"
+.Fn TAILQ_FOREACH "TYPE *var" "TAILQ_HEAD *head" "TAILQ_ENTRY NAME"
+.Fn TAILQ_FOREACH_SAFE "TYPE *var" "TAILQ_HEAD *head" "TAILQ_ENTRY NAME" "TYPE *temp_var"
+.Fn TAILQ_FOREACH_REVERSE "TYPE *var" "TAILQ_HEAD *head" "HEADNAME" "TAILQ_ENTRY NAME"
+.Fn TAILQ_FOREACH_REVERSE_SAFE "TYPE *var" "TAILQ_HEAD *head" "HEADNAME" "TAILQ_ENTRY NAME" "TYPE *temp_var"
+.Fn TAILQ_HEAD "HEADNAME" "TYPE"
+.Fn TAILQ_HEAD_INITIALIZER "TAILQ_HEAD head"
+.Fn TAILQ_INIT "TAILQ_HEAD *head"
+.Fn TAILQ_INSERT_AFTER "TAILQ_HEAD *head" "TYPE *listelm" "TYPE *elm" "TAILQ_ENTRY NAME"
+.Fn TAILQ_INSERT_BEFORE "TYPE *listelm" "TYPE *elm" "TAILQ_ENTRY NAME"
+.Fn TAILQ_INSERT_HEAD "TAILQ_HEAD *head" "TYPE *elm" "TAILQ_ENTRY NAME"
+.Fn TAILQ_INSERT_TAIL "TAILQ_HEAD *head" "TYPE *elm" "TAILQ_ENTRY NAME"
+.Fn TAILQ_LAST "TAILQ_HEAD *head" "HEADNAME"
+.Fn TAILQ_NEXT "TYPE *elm" "TAILQ_ENTRY NAME"
+.Fn TAILQ_PREV "TYPE *elm" "HEADNAME" "TAILQ_ENTRY NAME"
+.Fn TAILQ_REMOVE "TAILQ_HEAD *head" "TYPE *elm" "TAILQ_ENTRY NAME"
+.\"
+.Sh DESCRIPTION
+These macros define and operate on four types of data structures:
+singly-linked lists, singly-linked tail queues, lists, and tail queues.
+All four structures support the following functionality:
+.Bl -enum -compact -offset indent
+.It
+Insertion of a new entry at the head of the list.
+.It
+Insertion of a new entry after any element in the list.
+.It
+O(1) removal of an entry from the head of the list.
+.It
+O(n) removal of any entry in the list.
+.It
+Forward traversal through the list.
+.El
+.Pp
+Singly-linked lists are the simplest of the five data structures
+and support only the above functionality.
+Singly-linked lists are ideal for applications with large datasets
+and few or no removals,
+or for implementing a LIFO queue.
+.Pp
+Singly-linked tail queues add the following functionality:
+.Bl -enum -compact -offset indent
+.It
+Entries can be added at the end of a list.
+.It
+They may be concatenated.
+.El
+However:
+.Bl -enum -compact -offset indent
+.It
+All list insertions must specify the head of the list.
+.It
+Each head entry requires two pointers rather than one.
+.It
+Code size is about 15% greater and operations run about 20% slower
+than singly-linked lists.
+.El
+.Pp
+Singly-linked tailqs are ideal for applications with large datasets and
+few or no removals,
+or for implementing a FIFO queue.
+.Pp
+All doubly linked types of data structures (lists and tail queues)
+additionally allow:
+.Bl -enum -compact -offset indent
+.It
+Insertion of a new entry before any element in the list.
+.It
+O(1) removal of any entry in the list.
+.El
+However:
+.Bl -enum -compact -offset indent
+.It
+Each elements requires two pointers rather than one.
+.It
+Code size and execution time of operations (except for removal) is about
+twice that of the singly-linked data-structures.
+.El
+.Pp
+Linked lists are the simplest of the doubly linked data structures and support
+only the above functionality over singly-linked lists.
+.Pp
+Tail queues add the following functionality:
+.Bl -enum -compact -offset indent
+.It
+Entries can be added at the end of a list.
+.It
+They may be traversed backwards, from tail to head.
+.It
+They may be concatenated.
+.El
+However:
+.Bl -enum -compact -offset indent
+.It
+All list insertions and removals must specify the head of the list.
+.It
+Each head entry requires two pointers rather than one.
+.It
+Code size is about 15% greater and operations run about 20% slower
+than singly-linked lists.
+.El
+.Pp
+In the macro definitions,
+.Fa TYPE
+is the name of a user defined structure,
+that must contain a field of type
+.Li SLIST_ENTRY ,
+.Li STAILQ_ENTRY ,
+.Li LIST_ENTRY ,
+or
+.Li TAILQ_ENTRY ,
+named
+.Fa NAME .
+The argument
+.Fa HEADNAME
+is the name of a user defined structure that must be declared
+using the macros
+.Li SLIST_HEAD ,
+.Li STAILQ_HEAD ,
+.Li LIST_HEAD ,
+or
+.Li TAILQ_HEAD .
+See the examples below for further explanation of how these
+macros are used.
+.Sh SINGLY-LINKED LISTS
+A singly-linked list is headed by a structure defined by the
+.Nm SLIST_HEAD
+macro.
+This structure contains a single pointer to the first element
+on the list.
+The elements are singly linked for minimum space and pointer manipulation
+overhead at the expense of O(n) removal for arbitrary elements.
+New elements can be added to the list after an existing element or
+at the head of the list.
+An
+.Fa SLIST_HEAD
+structure is declared as follows:
+.Bd -literal -offset indent
+SLIST_HEAD(HEADNAME, TYPE) head;
+.Ed
+.Pp
+where
+.Fa HEADNAME
+is the name of the structure to be defined, and
+.Fa TYPE
+is the type of the elements to be linked into the list.
+A pointer to the head of the list can later be declared as:
+.Bd -literal -offset indent
+struct HEADNAME *headp;
+.Ed
+.Pp
+(The names
+.Li head
+and
+.Li headp
+are user selectable.)
+.Pp
+The macro
+.Nm SLIST_HEAD_INITIALIZER
+evaluates to an initializer for the list
+.Fa head .
+.Pp
+The macro
+.Nm SLIST_EMPTY
+evaluates to true if there are no elements in the list.
+.Pp
+The macro
+.Nm SLIST_ENTRY
+declares a structure that connects the elements in
+the list.
+.Pp
+The macro
+.Nm SLIST_FIRST
+returns the first element in the list or NULL if the list is empty.
+.Pp
+The macro
+.Nm SLIST_FOREACH
+traverses the list referenced by
+.Fa head
+in the forward direction, assigning each element in
+turn to
+.Fa var .
+.Pp
+The macro
+.Nm SLIST_FOREACH_SAFE
+traverses the list referenced by
+.Fa head
+in the forward direction, assigning each element in
+turn to
+.Fa var .
+However, unlike
+.Fn SLIST_FOREACH
+here it is permitted to both remove
+.Fa var
+as well as free it from within the loop safely without interfering with the
+traversal.
+.Pp
+The macro
+.Nm SLIST_INIT
+initializes the list referenced by
+.Fa head .
+.Pp
+The macro
+.Nm SLIST_INSERT_HEAD
+inserts the new element
+.Fa elm
+at the head of the list.
+.Pp
+The macro
+.Nm SLIST_INSERT_AFTER
+inserts the new element
+.Fa elm
+after the element
+.Fa listelm .
+.Pp
+The macro
+.Nm SLIST_NEXT
+returns the next element in the list.
+.Pp
+The macro
+.Nm SLIST_REMOVE_HEAD
+removes the element
+.Fa elm
+from the head of the list.
+For optimum efficiency,
+elements being removed from the head of the list should explicitly use
+this macro instead of the generic
+.Fa SLIST_REMOVE
+macro.
+.Pp
+The macro
+.Nm SLIST_REMOVE
+removes the element
+.Fa elm
+from the list.
+.Sh SINGLY-LINKED LIST EXAMPLE
+.Bd -literal
+SLIST_HEAD(slisthead, entry) head =
+    SLIST_HEAD_INITIALIZER(head);
+struct slisthead *headp;		/* Singly-linked List head. */
+struct entry {
+	...
+	SLIST_ENTRY(entry) entries;	/* Singly-linked List. */
+	...
+} *n1, *n2, *n3, *np;
+
+SLIST_INIT(&head);			/* Initialize the list. */
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the head. */
+SLIST_INSERT_HEAD(&head, n1, entries);
+
+n2 = malloc(sizeof(struct entry));	/* Insert after. */
+SLIST_INSERT_AFTER(n1, n2, entries);
+
+SLIST_REMOVE(&head, n2, entry, entries);/* Deletion. */
+free(n2);
+
+n3 = SLIST_FIRST(&head);
+SLIST_REMOVE_HEAD(&head, entries);	/* Deletion from the head. */
+free(n3);
+					/* Forward traversal. */
+SLIST_FOREACH(np, &head, entries)
+	np-> ...
+					/* Safe forward traversal. */
+SLIST_FOREACH_SAFE(np, &head, entries, np_temp) {
+	np->do_stuff();
+	...
+	SLIST_REMOVE(&head, np, entry, entries);
+	free(np);
+}
+
+while (!SLIST_EMPTY(&head)) {		/* List Deletion. */
+	n1 = SLIST_FIRST(&head);
+	SLIST_REMOVE_HEAD(&head, entries);
+	free(n1);
+}
+.Ed
+.Sh SINGLY-LINKED TAIL QUEUES
+A singly-linked tail queue is headed by a structure defined by the
+.Nm STAILQ_HEAD
+macro.
+This structure contains a pair of pointers,
+one to the first element in the tail queue and the other to
+the last element in the tail queue.
+The elements are singly linked for minimum space and pointer
+manipulation overhead at the expense of O(n) removal for arbitrary
+elements.
+New elements can be added to the tail queue after an existing element,
+at the head of the tail queue, or at the end of the tail queue.
+A
+.Fa STAILQ_HEAD
+structure is declared as follows:
+.Bd -literal -offset indent
+STAILQ_HEAD(HEADNAME, TYPE) head;
+.Ed
+.Pp
+where
+.Li HEADNAME
+is the name of the structure to be defined, and
+.Li TYPE
+is the type of the elements to be linked into the tail queue.
+A pointer to the head of the tail queue can later be declared as:
+.Bd -literal -offset indent
+struct HEADNAME *headp;
+.Ed
+.Pp
+(The names
+.Li head
+and
+.Li headp
+are user selectable.)
+.Pp
+The macro
+.Nm STAILQ_HEAD_INITIALIZER
+evaluates to an initializer for the tail queue
+.Fa head .
+.Pp
+The macro
+.Nm STAILQ_CONCAT
+concatenates the tail queue headed by
+.Fa head2
+onto the end of the one headed by
+.Fa head1
+removing all entries from the former.
+.Pp
+The macro
+.Nm STAILQ_EMPTY
+evaluates to true if there are no items on the tail queue.
+.Pp
+The macro
+.Nm STAILQ_ENTRY
+declares a structure that connects the elements in
+the tail queue.
+.Pp
+The macro
+.Nm STAILQ_FIRST
+returns the first item on the tail queue or NULL if the tail queue
+is empty.
+.Pp
+The macro
+.Nm STAILQ_FOREACH
+traverses the tail queue referenced by
+.Fa head
+in the forward direction, assigning each element
+in turn to
+.Fa var .
+.Pp
+The macro
+.Nm STAILQ_FOREACH_SAFE
+traverses the tail queue referenced by
+.Fa head
+in the forward direction, assigning each element
+in turn to
+.Fa var .
+However, unlike
+.Fn STAILQ_FOREACH
+here it is permitted to both remove
+.Fa var
+as well as free it from within the loop safely without interfering with the
+traversal.
+.Pp
+The macro
+.Nm STAILQ_INIT
+initializes the tail queue referenced by
+.Fa head .
+.Pp
+The macro
+.Nm STAILQ_INSERT_HEAD
+inserts the new element
+.Fa elm
+at the head of the tail queue.
+.Pp
+The macro
+.Nm STAILQ_INSERT_TAIL
+inserts the new element
+.Fa elm
+at the end of the tail queue.
+.Pp
+The macro
+.Nm STAILQ_INSERT_AFTER
+inserts the new element
+.Fa elm
+after the element
+.Fa listelm .
+.Pp
+The macro
+.Nm STAILQ_LAST
+returns the last item on the tail queue.
+If the tail queue is empty the return value is NULL.
+.Pp
+The macro
+.Nm STAILQ_NEXT
+returns the next item on the tail queue, or NULL this item is the last.
+.Pp
+The macro
+.Nm STAILQ_REMOVE_HEAD
+removes the element at the head of the tail queue.
+For optimum efficiency,
+elements being removed from the head of the tail queue should
+use this macro explicitly rather than the generic
+.Fa STAILQ_REMOVE
+macro.
+.Pp
+The macro
+.Nm STAILQ_REMOVE
+removes the element
+.Fa elm
+from the tail queue.
+.Sh SINGLY-LINKED TAIL QUEUE EXAMPLE
+.Bd -literal
+STAILQ_HEAD(stailhead, entry) head =
+    STAILQ_HEAD_INITIALIZER(head);
+struct stailhead *headp;		/* Singly-linked tail queue head. */
+struct entry {
+	...
+	STAILQ_ENTRY(entry) entries;	/* Tail queue. */
+	...
+} *n1, *n2, *n3, *np;
+
+STAILQ_INIT(&head);			/* Initialize the queue. */
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the head. */
+STAILQ_INSERT_HEAD(&head, n1, entries);
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the tail. */
+STAILQ_INSERT_TAIL(&head, n1, entries);
+
+n2 = malloc(sizeof(struct entry));	/* Insert after. */
+STAILQ_INSERT_AFTER(&head, n1, n2, entries);
+					/* Deletion. */
+STAILQ_REMOVE(&head, n2, entry, entries);
+free(n2);
+					/* Deletion from the head. */
+n3 = STAILQ_FIRST(&head);
+STAILQ_REMOVE_HEAD(&head, entries);
+free(n3);
+					/* Forward traversal. */
+STAILQ_FOREACH(np, &head, entries)
+	np-> ...
+					/* Safe forward traversal. */
+STAILQ_FOREACH_SAFE(np, &head, entries, np_temp) {
+	np->do_stuff();
+	...
+	STAILQ_REMOVE(&head, np, entry, entries);
+	free(np);
+}
+					/* TailQ Deletion. */
+while (!STAILQ_EMPTY(&head)) {
+	n1 = STAILQ_FIRST(&head);
+	STAILQ_REMOVE_HEAD(&head, entries);
+	free(n1);
+}
+					/* Faster TailQ Deletion. */
+n1 = STAILQ_FIRST(&head);
+while (n1 != NULL) {
+	n2 = STAILQ_NEXT(n1, entries);
+	free(n1);
+	n1 = n2;
+}
+STAILQ_INIT(&head);
+.Ed
+.Sh LISTS
+A list is headed by a structure defined by the
+.Nm LIST_HEAD
+macro.
+This structure contains a single pointer to the first element
+on the list.
+The elements are doubly linked so that an arbitrary element can be
+removed without traversing the list.
+New elements can be added to the list after an existing element,
+before an existing element, or at the head of the list.
+A
+.Fa LIST_HEAD
+structure is declared as follows:
+.Bd -literal -offset indent
+LIST_HEAD(HEADNAME, TYPE) head;
+.Ed
+.Pp
+where
+.Fa HEADNAME
+is the name of the structure to be defined, and
+.Fa TYPE
+is the type of the elements to be linked into the list.
+A pointer to the head of the list can later be declared as:
+.Bd -literal -offset indent
+struct HEADNAME *headp;
+.Ed
+.Pp
+(The names
+.Li head
+and
+.Li headp
+are user selectable.)
+.Pp
+The macro
+.Nm LIST_HEAD_INITIALIZER
+evaluates to an initializer for the list
+.Fa head .
+.Pp
+The macro
+.Nm LIST_EMPTY
+evaluates to true if there are no elements in the list.
+.Pp
+The macro
+.Nm LIST_ENTRY
+declares a structure that connects the elements in
+the list.
+.Pp
+The macro
+.Nm LIST_FIRST
+returns the first element in the list or NULL if the list
+is empty.
+.Pp
+The macro
+.Nm LIST_FOREACH
+traverses the list referenced by
+.Fa head
+in the forward direction, assigning each element in turn to
+.Fa var .
+.Pp
+The macro
+.Nm LIST_FOREACH_SAFE
+traverses the list referenced by
+.Fa head
+in the forward direction, assigning each element in turn to
+.Fa var .
+However, unlike
+.Fn LIST_FOREACH
+here it is permitted to both remove
+.Fa var
+as well as free it from within the loop safely without interfering with the
+traversal.
+.Pp
+The macro
+.Nm LIST_INIT
+initializes the list referenced by
+.Fa head .
+.Pp
+The macro
+.Nm LIST_INSERT_HEAD
+inserts the new element
+.Fa elm
+at the head of the list.
+.Pp
+The macro
+.Nm LIST_INSERT_AFTER
+inserts the new element
+.Fa elm
+after the element
+.Fa listelm .
+.Pp
+The macro
+.Nm LIST_INSERT_BEFORE
+inserts the new element
+.Fa elm
+before the element
+.Fa listelm .
+.Pp
+The macro
+.Nm LIST_NEXT
+returns the next element in the list, or NULL if this is the last.
+.Pp
+The macro
+.Nm LIST_REMOVE
+removes the element
+.Fa elm
+from the list.
+.Sh LIST EXAMPLE
+.Bd -literal
+LIST_HEAD(listhead, entry) head =
+    LIST_HEAD_INITIALIZER(head);
+struct listhead *headp;			/* List head. */
+struct entry {
+	...
+	LIST_ENTRY(entry) entries;	/* List. */
+	...
+} *n1, *n2, *n3, *np, *np_temp;
+
+LIST_INIT(&head);			/* Initialize the list. */
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the head. */
+LIST_INSERT_HEAD(&head, n1, entries);
+
+n2 = malloc(sizeof(struct entry));	/* Insert after. */
+LIST_INSERT_AFTER(n1, n2, entries);
+
+n3 = malloc(sizeof(struct entry));	/* Insert before. */
+LIST_INSERT_BEFORE(n2, n3, entries);
+
+LIST_REMOVE(n2, entries);		/* Deletion. */
+free(n2);
+					/* Forward traversal. */
+LIST_FOREACH(np, &head, entries)
+	np-> ...
+
+					/* Safe forward traversal. */
+LIST_FOREACH_SAFE(np, &head, entries, np_temp) {
+	np->do_stuff();
+	...
+	LIST_REMOVE(np, entries);
+	free(np);
+}
+
+while (!LIST_EMPTY(&head)) {		/* List Deletion. */
+	n1 = LIST_FIRST(&head);
+	LIST_REMOVE(n1, entries);
+	free(n1);
+}
+
+n1 = LIST_FIRST(&head);			/* Faster List Deletion. */
+while (n1 != NULL) {
+	n2 = LIST_NEXT(n1, entries);
+	free(n1);
+	n1 = n2;
+}
+LIST_INIT(&head);
+.Ed
+.Sh TAIL QUEUES
+A tail queue is headed by a structure defined by the
+.Nm TAILQ_HEAD
+macro.
+This structure contains a pair of pointers,
+one to the first element in the tail queue and the other to
+the last element in the tail queue.
+The elements are doubly linked so that an arbitrary element can be
+removed without traversing the tail queue.
+New elements can be added to the tail queue after an existing element,
+before an existing element, at the head of the tail queue,
+or at the end of the tail queue.
+A
+.Fa TAILQ_HEAD
+structure is declared as follows:
+.Bd -literal -offset indent
+TAILQ_HEAD(HEADNAME, TYPE) head;
+.Ed
+.Pp
+where
+.Li HEADNAME
+is the name of the structure to be defined, and
+.Li TYPE
+is the type of the elements to be linked into the tail queue.
+A pointer to the head of the tail queue can later be declared as:
+.Bd -literal -offset indent
+struct HEADNAME *headp;
+.Ed
+.Pp
+(The names
+.Li head
+and
+.Li headp
+are user selectable.)
+.Pp
+The macro
+.Nm TAILQ_HEAD_INITIALIZER
+evaluates to an initializer for the tail queue
+.Fa head .
+.Pp
+The macro
+.Nm TAILQ_CONCAT
+concatenates the tail queue headed by
+.Fa head2
+onto the end of the one headed by
+.Fa head1
+removing all entries from the former.
+.Pp
+The macro
+.Nm TAILQ_EMPTY
+evaluates to true if there are no items on the tail queue.
+.Pp
+The macro
+.Nm TAILQ_ENTRY
+declares a structure that connects the elements in
+the tail queue.
+.Pp
+The macro
+.Nm TAILQ_FIRST
+returns the first item on the tail queue or NULL if the tail queue
+is empty.
+.Pp
+The macro
+.Nm TAILQ_FOREACH
+traverses the tail queue referenced by
+.Fa head
+in the forward direction, assigning each element in turn to
+.Fa var .
+.Fa var
+is set to
+.Dv NULL
+if the loop completes normally, or if there were no elements.
+.Pp
+The macro
+.Nm TAILQ_FOREACH_REVERSE
+traverses the tail queue referenced by
+.Fa head
+in the reverse direction, assigning each element in turn to
+.Fa var .
+.Pp
+The macros
+.Nm TAILQ_FOREACH_SAFE
+and
+.Nm TAILQ_FOREACH_REVERSE_SAFE
+traverse the list referenced by
+.Fa head
+in the forward or reverse direction respectively,
+assigning each element in turn to
+.Fa var .
+However, unlike their unsafe counterparts,
+.Nm TAILQ_FOREACH
+and
+.Nm TAILQ_FOREACH_REVERSE
+permit to both remove
+.Fa var
+as well as free it from within the loop safely without interfering with the
+traversal.
+.Pp
+The macro
+.Nm TAILQ_INIT
+initializes the tail queue referenced by
+.Fa head .
+.Pp
+The macro
+.Nm TAILQ_INSERT_HEAD
+inserts the new element
+.Fa elm
+at the head of the tail queue.
+.Pp
+The macro
+.Nm TAILQ_INSERT_TAIL
+inserts the new element
+.Fa elm
+at the end of the tail queue.
+.Pp
+The macro
+.Nm TAILQ_INSERT_AFTER
+inserts the new element
+.Fa elm
+after the element
+.Fa listelm .
+.Pp
+The macro
+.Nm TAILQ_INSERT_BEFORE
+inserts the new element
+.Fa elm
+before the element
+.Fa listelm .
+.Pp
+The macro
+.Nm TAILQ_LAST
+returns the last item on the tail queue.
+If the tail queue is empty the return value is NULL.
+.Pp
+The macro
+.Nm TAILQ_NEXT
+returns the next item on the tail queue, or NULL if this item is the last.
+.Pp
+The macro
+.Nm TAILQ_PREV
+returns the previous item on the tail queue, or NULL if this item
+is the first.
+.Pp
+The macro
+.Nm TAILQ_REMOVE
+removes the element
+.Fa elm
+from the tail queue.
+.Sh TAIL QUEUE EXAMPLE
+.Bd -literal
+TAILQ_HEAD(tailhead, entry) head =
+    TAILQ_HEAD_INITIALIZER(head);
+struct tailhead *headp;			/* Tail queue head. */
+struct entry {
+	...
+	TAILQ_ENTRY(entry) entries;	/* Tail queue. */
+	...
+} *n1, *n2, *n3, *np;
+
+TAILQ_INIT(&head);			/* Initialize the queue. */
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the head. */
+TAILQ_INSERT_HEAD(&head, n1, entries);
+
+n1 = malloc(sizeof(struct entry));	/* Insert at the tail. */
+TAILQ_INSERT_TAIL(&head, n1, entries);
+
+n2 = malloc(sizeof(struct entry));	/* Insert after. */
+TAILQ_INSERT_AFTER(&head, n1, n2, entries);
+
+n3 = malloc(sizeof(struct entry));	/* Insert before. */
+TAILQ_INSERT_BEFORE(n2, n3, entries);
+
+TAILQ_REMOVE(&head, n2, entries);	/* Deletion. */
+free(n2);
+					/* Forward traversal. */
+TAILQ_FOREACH(np, &head, entries)
+	np-> ...
+					/* Safe forward traversal. */
+TAILQ_FOREACH_SAFE(np, &head, entries, np_temp) {
+	np->do_stuff();
+	...
+	TAILQ_REMOVE(&head, np, entries);
+	free(np);
+}
+					/* Reverse traversal. */
+TAILQ_FOREACH_REVERSE(np, &head, tailhead, entries)
+	np-> ...
+					/* TailQ Deletion. */
+while (!TAILQ_EMPTY(&head)) {
+	n1 = TAILQ_FIRST(&head);
+	TAILQ_REMOVE(&head, n1, entries);
+	free(n1);
+}
+					/* Faster TailQ Deletion. */
+n1 = TAILQ_FIRST(&head);
+while (n1 != NULL) {
+	n2 = TAILQ_NEXT(n1, entries);
+	free(n1);
+	n1 = n2;
+}
+TAILQ_INIT(&head);
+.Ed
+.Sh HISTORY
+The
+.Nm queue
+functions first appeared in
+.Bx 4.4 .
diff --git a/bsd/man/man4/Makefile b/bsd/man/man4/Makefile
index 3cc1cc953..ab182ec48 100644
--- a/bsd/man/man4/Makefile
+++ b/bsd/man/man4/Makefile
@@ -7,6 +7,7 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 DATAFILES = \
+	aio.4	\
 	arp.4	\
 	bpf.4	\
 	divert.4	\
@@ -38,7 +39,6 @@ DATAFILES = \
 	tcp.4	\
 	termios.4	\
 	tty.4	\
-	tun.4	\
 	udp.4	\
 	unix.4	\
 	urandom.4	\
diff --git a/bsd/man/man4/aio.4 b/bsd/man/man4/aio.4
new file mode 100644
index 000000000..255a6df1e
--- /dev/null
+++ b/bsd/man/man4/aio.4
@@ -0,0 +1,52 @@
+.\"-
+.\" Copyright (c) 2002 Dag-Erling Coïdan Smørgrav
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 3. The name of the author may not be used to endorse or promote products
+.\"    derived from this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD: src/share/man/man4/aio.4,v 1.4 2003/01/14 03:42:16 tjr Exp $
+.\"
+.Dd February 16, 2006
+.Dt AIO 4
+.Os
+.Sh NAME
+.Nm aio
+.Nd asynchronous I/O
+.Sh DESCRIPTION
+The
+.Nm
+facility provides system calls for asynchronous I/O.
+.Sh SEE ALSO
+.Xr aio_cancel 2 ,
+.Xr aio_error 2 ,
+.Xr aio_read 2 ,
+.Xr aio_return 2 ,
+.Xr aio_suspend 2 ,
+.Xr aio_write 2 ,
+.Xr lio_listio 2
+.Sh HISTORY
+The
+.Nm
+facility appeared in
+.Fx 3.0 .
diff --git a/bsd/man/man4/icmp.4 b/bsd/man/man4/icmp.4
index 65015098d..4013ef692 100644
--- a/bsd/man/man4/icmp.4
+++ b/bsd/man/man4/icmp.4
@@ -87,6 +87,9 @@ them (based on the destination address).
 Incoming packets are received with the
 .Tn IP
 header and options intact.
+Note that the ip_off and ip_len fields are in host byte order.
+For more information about the IP header structure, see 
+.Xr ip 4 .
 .Ss "Non-privileged ICMP"
 .Pp
 .Tn ICMP
@@ -96,13 +99,6 @@ socket type without requiring root privileges. The synopsis is the following:
 .Pp
 .Fn socket AF_INET SOCK_DGRAM IPPROTO_ICMP
 .Pp
-This can be used by non root privileged processes to send 
-.Tn ICMP
-echo requests to gauge the quality of the connectivity to a host, to receive 
-.Tn ICMP
-destination unreachable message for path MTU discovery, or to receveive 
-time exceeded message for traceroute.
-.Pp
 Datagram oriented
 .Tn ICMP
 sockets offer a subset of the functionality available to raw 
@@ -116,6 +112,16 @@ The minimal length of an
 .Tn ICMP 
 message request is eight (8) octets.
 .Pp
+The advantage of using datagram oriented
+.Tn ICMP
+sockets is that even a non-privileged process can use 
+.Tn ICMP
+echo requests to gauge the quality of the connectivity to a host, 
+or to receive 
+.Tn ICMP
+destination unreachable message for path MTU discovery, or to receive 
+time exceeded messages for traceroute. 
+.Pp
 The following 
 .Tn IP
 level option can be used with datagram oriented 
@@ -195,10 +201,10 @@ or
 header.
 .El
 .Sh SEE ALSO
-.Xr send 2 ,
 .Xr recv 2 ,
-.Xr intro 4 ,
+.Xr send 2 ,
 .Xr inet 4 ,
+.Xr intro 4 ,
 .Xr ip 4
 .Sh HISTORY
 The
diff --git a/bsd/man/man4/icmp6.4 b/bsd/man/man4/icmp6.4
index 28b1325d1..40a30a31f 100644
--- a/bsd/man/man4/icmp6.4
+++ b/bsd/man/man4/icmp6.4
@@ -228,6 +228,26 @@ and name
 with a pointer to the
 .Vt icmp6_filter
 structure as the option value.
+.Ss "Non-privileged ICMPv6"
+.Pp
+.Tn ICMPv6
+sockets can be opened with the 
+.Dv SOCK_DGRAM
+socket type without requiring root privileges. The synopsis is the following:
+.Pp
+.Fn socket AF_INET6 SOCK_DGRAM IPPROTO_ICMP6
+.Pp
+This can only be used to send 
+.Tn ICMPv6
+echo requests to gauge the quality of the connectivity to a host, to receive 
+destination unreachable message for path MTU discovery, or to receveive 
+time exceeded message for traceroute. 
+.Pp
+A socket opened with superuser privileges can send any kind of 
+.Tn ICMPv6 
+message and can also use the 
+.Xr ip6 4
+options that require superuser privileges.
 .Sh SEE ALSO
 .Xr getsockopt 2 ,
 .Xr recv 2 ,
diff --git a/bsd/man/man4/inet.4 b/bsd/man/man4/inet.4
index 5d70e315d..52ba9239b 100644
--- a/bsd/man/man4/inet.4
+++ b/bsd/man/man4/inet.4
@@ -157,11 +157,11 @@ Get interface network mask.
 .Sh SEE ALSO
 .Xr ioctl 2 ,
 .Xr socket 2 ,
+.Xr icmp 4 ,
 .Xr intro 4 ,
-.Xr tcp 4 ,
-.Xr udp 4 ,
 .Xr ip 4 ,
-.Xr icmp 4
+.Xr tcp 4 ,
+.Xr udp 4
 .Rs
 .%T "An Introductory 4.3 BSD Interprocess Communication Tutorial"
 .%B PS1
diff --git a/bsd/man/man4/ip.4 b/bsd/man/man4/ip.4
index c351178e1..cf0337391 100644
--- a/bsd/man/man4/ip.4
+++ b/bsd/man/man4/ip.4
@@ -324,13 +324,14 @@ the fields of the IP header, including the following:
 ip->ip_v = IPVERSION;
 ip->ip_hl = hlen >> 2;
 ip->ip_id = 0;  /* 0 means kernel set appropriate value */
-ip->ip_off = htons(offset);
-ip->ip_len = htons(len);
+ip->ip_off = offset;
+ip->ip_len = len;
 .Ed
 .sp .5
-Additionally note that starting with
-.Tn OpenBSD 2.1
-the ip_off and ip_len fields are in network byte order.
+.Pp
+Note that 
+the ip_off and ip_len fields are in host byte order.
+.Pp
 If the header source address is set to 
 .Dv INADDR_ANY,
 the kernel will choose an appropriate address.
@@ -372,11 +373,11 @@ or longer than the option buffer provided.
 .El
 .Sh SEE ALSO
 .Xr getsockopt 2 ,
-.Xr send 2 ,
 .Xr recv 2 ,
-.Xr intro 4 ,
+.Xr send 2 ,
 .Xr icmp 4 ,
-.Xr inet 4
+.Xr inet 4 ,
+.Xr intro 4
 .Sh HISTORY
 The
 .Nm
diff --git a/bsd/man/man4/lo.4 b/bsd/man/man4/lo.4
index 0491936b4..0c3f08015 100644
--- a/bsd/man/man4/lo.4
+++ b/bsd/man/man4/lo.4
@@ -68,8 +68,8 @@ a message with addresses formatted in an unsuitable address
 family; the packet was dropped.
 .El
 .Sh SEE ALSO
-.Xr intro 4 ,
 .Xr inet 4 ,
+.Xr intro 4 ,
 .Xr ns 4
 .Sh HISTORY
 The
diff --git a/bsd/man/man4/netintro.4 b/bsd/man/man4/netintro.4
index 1dd5f5c9e..725797146 100644
--- a/bsd/man/man4/netintro.4
+++ b/bsd/man/man4/netintro.4
@@ -317,10 +317,10 @@ struct ifconf {
 };
 .Ed
 .Sh SEE ALSO
-.Xr socket 2 ,
 .Xr ioctl 2 ,
+.Xr socket 2 ,
 .Xr intro 4 ,
-.Xr config 8 ,
+.Xr config 5 ,
 .Xr routed 8
 .Sh HISTORY
 The
diff --git a/bsd/man/man4/random.4 b/bsd/man/man4/random.4
index 491bf1d64..bc0dbc76c 100644
--- a/bsd/man/man4/random.4
+++ b/bsd/man/man4/random.4
@@ -36,7 +36,7 @@ The
 device implements the
 .Nm Yarrow
 pseudo random number generator algorithm and maintains its entropy pool.
-Addditional entropy is fed to the generator regularly by the
+Additional entropy is fed to the generator regularly by the
 .Nm SecurityServer
 daemon from random jitter measurements of the kernel.
 .Nm SecurityServer
@@ -59,7 +59,7 @@ over time without any explicit indication from the
 .Nm
 device itself.
 .Pp
-Paranoid programmers can counter-act this risk somewhat by collecting
+Paranoid programmers can counteract this risk somewhat by collecting
 entropy of their choice (e.g. from keystroke or mouse timings)
 and seeding it into
 .Nm
diff --git a/bsd/man/man4/route.4 b/bsd/man/man4/route.4
index fd9ef63f1..06e142b1a 100644
--- a/bsd/man/man4/route.4
+++ b/bsd/man/man4/route.4
@@ -177,12 +177,7 @@ be reclaimed until all references to it are released.
 User processes can obtain information about the routing
 entry to a specific destination by using a
 .Dv RTM_GET
-message,
-or by reading the
-.Pa /dev/kmem
-device, or by issuing a
-.Xr getkerninfo 2
-system call.
+message.
 .Pp
 Messages include:
 .Bd -literal
diff --git a/bsd/man/man4/tcp.4 b/bsd/man/man4/tcp.4
index 62bb958d5..18a5f98bb 100644
--- a/bsd/man/man4/tcp.4
+++ b/bsd/man/man4/tcp.4
@@ -170,8 +170,8 @@ exists.
 .Sh SEE ALSO
 .Xr getsockopt 2 ,
 .Xr socket 2 ,
-.Xr intro 4 ,
 .Xr inet 4 ,
+.Xr intro 4 ,
 .Xr ip 4
 .Sh HISTORY
 The
diff --git a/bsd/man/man4/tun.4 b/bsd/man/man4/tun.4
index 60ecbb06a..47608b2cd 100644
--- a/bsd/man/man4/tun.4
+++ b/bsd/man/man4/tun.4
@@ -337,8 +337,8 @@ or MTU, for
 This interface is useful for command-line reconfiguration, such as
 setting the interface type at boot time, with 
 .Sh SEE ALSO
-.Xr intro 4 ,
-.Xr inet 4
+.Xr inet 4 ,
+.Xr intro 4
 .Sh BUGS
 The
 .Dv SUONLY
diff --git a/bsd/man/man4/udp.4 b/bsd/man/man4/udp.4
index 5e8f7eae3..c104618fe 100644
--- a/bsd/man/man4/udp.4
+++ b/bsd/man/man4/udp.4
@@ -129,8 +129,8 @@ exists.
 .Xr recv 2 ,
 .Xr send 2 ,
 .Xr socket 2 ,
-.Xr intro 4 ,
 .Xr inet 4 ,
+.Xr intro 4 ,
 .Xr ip 4
 .Sh HISTORY
 The
diff --git a/bsd/man/man5/acct.5 b/bsd/man/man5/acct.5
index 64745330b..43b928f8b 100644
--- a/bsd/man/man5/acct.5
+++ b/bsd/man/man5/acct.5
@@ -102,10 +102,10 @@ and its status is saved by setting one of more of the following flags in
 and
 .Dv ASIG .
 .Sh SEE ALSO
+.Xr lastcomm 1 ,
 .Xr acct 2 ,
-.Xr accton 8 ,
 .Xr execve 2 ,
-.Xr lastcomm 1 ,
+.Xr accton 8 ,
 .Xr sa 8
 .Sh HISTORY
 A
diff --git a/bsd/man/man5/core.5 b/bsd/man/man5/core.5
index 6e6ea7a2d..dae3c4c51 100644
--- a/bsd/man/man5/core.5
+++ b/bsd/man/man5/core.5
@@ -27,10 +27,13 @@ to disk for later examination by one of the available debuggers.
 (See
 .Xr sigaction 2 . )
 This memory image is written to a file named by default
-.Nm core.pid
+.Pa core.pid ,
+where
+.Va pid
+is the process ID of the process,
 in the
-.Nm /cores
-directory;
+.Pa /cores
+directory,
 provided the terminated process had write permission in the directory,
 and the directory existed.
 .Pp
@@ -39,7 +42,7 @@ The maximum size of a core file is limited by
 Files which would be larger than the limit are not created.
 .Pp
 The core file consists of the
-.Pa Xr Mach-O 5
+.Xr Mach-O 5
 header as  described in the
 .Aq Pa mach-o/loader.h
 file.
@@ -48,8 +51,8 @@ file consists of various sections described in the
 .Xr Mach-O 5
 header.
 .Sh NOTE
-Core dumps are disabled by default under Darwin/Mac OS X.  To re-enable core dumps, a 
-privlaged user must edit
+Core dumps are disabled by default under Darwin/Mac OS X.  To re-enable 
+core dumps, a privileged user must edit
 .Pa /etc/hostconfig
 to contain the line:
 .Bd -literal
diff --git a/bsd/man/man5/types.5 b/bsd/man/man5/types.5
index 48b2372ef..8226db199 100644
--- a/bsd/man/man5/types.5
+++ b/bsd/man/man5/types.5
@@ -138,10 +138,10 @@ typedef	struct fd_set {
 #endif /* !_TYPES_H_ */
 .Ed
 .Sh SEE ALSO
-.Xr fs 5 ,
-.Xr time 3 ,
+.Xr adb 1 ,
 .Xr lseek 2 ,
-.Xr adb 1
+.Xr time 3 ,
+.Xr fs 5
 .Sh HISTORY
 A
 .Nm
diff --git a/bsd/man/man8/Makefile b/bsd/man/man8/Makefile
new file mode 100644
index 000000000..4c985959c
--- /dev/null
+++ b/bsd/man/man8/Makefile
@@ -0,0 +1,18 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+DATAFILES = \
+	sticky.8
+
+INSTALL_MAN_LIST = ${DATAFILES}
+
+INSTALL_MAN_DIR = man8
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
+
diff --git a/bsd/man/man2/sigreturn.2 b/bsd/man/man8/sticky.8
similarity index 53%
rename from bsd/man/man2/sigreturn.2
rename to bsd/man/man8/sticky.8
index d92b3cf55..a595cb059 100644
--- a/bsd/man/man2/sigreturn.2
+++ b/bsd/man/man8/sticky.8
@@ -1,6 +1,6 @@
-.\"	$NetBSD: sigreturn.2,v 1.6 1995/02/27 12:37:40 cgd Exp $
+.\"	$NetBSD: sticky.8,v 1.3 1994/11/30 19:36:27 jtc Exp $
 .\"
-.\" Copyright (c) 1985, 1991, 1993
+.\" Copyright (c) 1980, 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -31,69 +31,55 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.\"     @(#)sigreturn.2	8.1 (Berkeley) 6/4/93
+.\"     @(#)sticky.8	8.1 (Berkeley) 6/5/93
 .\"
-.Dd June 4, 1993
-.Dt SIGRETURN 2
-.Os BSD 4.3
+.Dd June 5, 1993
+.Dt STICKY 8
+.Os BSD 4
 .Sh NAME
-.Nm sigreturn
-.Nd return from signal
-.Sh SYNOPSIS
-.Fd #include <signal.h>
-.Bd -literal
-struct sigcontext {
-	int sc_onstack;
-	int sc_mask;
-	int sc_sp;
-	int sc_fp;
-	int sc_ap;
-	int sc_pc;
-	int sc_ps;
-};
-.Ed
-.Ft int
-.Fn sigreturn "struct sigcontext *scp"
+.Nm sticky
+.Nd sticky text and append-only directories
 .Sh DESCRIPTION
-.Fn Sigreturn
-allows users to atomically unmask, switch stacks,
-and return from a signal context.
-The processes signal mask and stack status are
-restored from the context.
-The system call does not return;
-the users stack pointer, frame pointer, argument pointer,
-and processor status longword are restored from the context.
-Execution resumes at the specified pc.
-This system call is used by the trampoline code and
-.Xr longjmp 3
-when returning from a signal to the previously executing program.
-.Sh NOTES
-This system call is not available in 4.2
-.Tn BSD
-hence it should not be used if backward compatibility is needed.
-.Sh RETURN VALUES
-If successful, the system call does not return.
-Otherwise, a value of -1 is returned and 
-.Va errno
-is set to indicate the error.
-.Sh ERRORS
-.Fn Sigreturn
-will fail and the process context will remain unchanged
-if one of the following occurs.
-.Bl -tag -width Er
-.It Bq Er EFAULT
-.Fa Scp
-points to memory that is not a valid part of the process
-address space.
-.It Bq Er EINVAL
-The process status longword is invalid or would improperly
-raise the privilege level of the process.
-.El
-.Sh SEE ALSO
-.Xr sigaction 2 ,
-.Xr setjmp 3
+A special file mode, called the
+.Em sticky bit
+(mode S_ISVTX),
+is used to indicate special treatment
+for shareable executable files and directories.
+See
+.Xr chmod 2
+or
+the file
+.Pa /usr/include/sys/stat.h
+for an explanation of file modes.
+.Sh STICKY TEXT EXECUTABLE FILES
+The sticky bit has no effect on executable files. All optimization on
+whether text images remain resident in memory is handled by the
+kernel's virtual memory system.
+.Sh STICKY DIRECTORIES
+A directory whose `sticky bit' is set
+becomes an append-only directory, or, more accurately,
+a directory in which the deletion of files is restricted.
+A file in a sticky directory may only be removed or renamed
+by a user if the user has write permission for the directory and
+the user is the owner of the file, the owner of the directory,
+or the super-user.
+This feature is usefully applied to directories such as
+.Pa /tmp
+which must be publicly writable but
+should deny users the license to arbitrarily
+delete or rename each others' files.
+.Pp
+Any user may create a sticky directory.
+See
+.Xr chmod 1
+for details about modifying file modes.
+.Sh BUGS
+Neither
+.Xr open 2
+nor
+.Xr mkdir 2
+will create a file with the sticky bit set.
 .Sh HISTORY
-The
-.Fn sigreturn
-function call appeared in
-.Bx 4.3 .
+A
+.Nm
+command appeared in Version 32V AT&T UNIX.
diff --git a/bsd/man/man9/Makefile b/bsd/man/man9/Makefile
index 407b7d235..769918fec 100644
--- a/bsd/man/man9/Makefile
+++ b/bsd/man/man9/Makefile
@@ -7,10 +7,26 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 DATAFILES = \
-	copy.9	\
-	fetch.9	\
-	store.9	\
-	style.9	\
+	copy.9		\
+	copyin.9	\
+	copyinstr.9	\
+	copyout.9	\
+	copystr.9	\
+	fetch.9		\
+	fubyte.9	\
+	fuibyte.9	\
+	fuiword.9	\
+	fulong.9	\
+	fuulong.9	\
+	fuword.9	\
+	store.9		\
+	style.9		\
+	subyte.9	\
+	suibyte.9	\
+	suiword.9	\
+	sulong.9	\
+	suulong.9	\
+	suword.9	\
 	intro.9
 
 INSTALL_MAN_LIST = ${DATAFILES}
diff --git a/bsd/man/man9/copy.9 b/bsd/man/man9/copy.9
index 5d82b9da4..13c2fe044 100644
--- a/bsd/man/man9/copy.9
+++ b/bsd/man/man9/copy.9
@@ -40,23 +40,46 @@
 .Sh NAME
 .Nm copy ,
 .Nm copyin ,
+.Nm copyinstr ,
 .Nm copyout ,
-.Nm copystr ,
-.Nm copyinstr
+.Nm copystr
 .Nd kernel copy functions
 .Sh SYNOPSIS
 .In sys/types.h
 .In sys/systm.h
 .Ft int
-.Fn copyin "const void *uaddr" "void *kaddr" "size_t len"
+.Fo copyin
+.Fa "const void *uaddr"
+.Fa "void *kaddr"
+.Fa "size_t len"
+.Fc
 .Ft int
-.Fn copyout "const void *kaddr" "void *uaddr" "size_t len"
+.Fo copyinstr
+.Fa "const void *uaddr"
+.Fa "void *kaddr"
+.Fa "size_t len"
+.Fa "size_t *done"
+.Fc
 .Ft int
-.Fn copystr "const void *kfaddr" "void *kdaddr" "size_t len" "size_t *done"
-.Ft int
-.Fn copyinstr "const void *uaddr" "void *kaddr" "size_t len" "size_t *done"
+.Fo copyout
+.Fa "const void *kaddr"
+.Fa "void *uaddr"
+.Fa "size_t len"
+.Fc
 .\" .Ft int
-.\" .Fn copyoutstr "const void *kaddr" "void *uaddr" "size_t len" "size_t *done"
+.\" .Fn copyoutstr
+.\" .Fa "const void *kaddr"
+.\" .Fa "void *uaddr"
+.\" .Fa "size_t len"
+.\" .Fa "size_t *done"
+.\" .Fc
+.Ft int
+.Fo copystr
+.Fa "const void *kfaddr"
+.Fa "void *kdaddr"
+.Fa "size_t len"
+.Fa "size_t *done"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm
@@ -69,6 +92,7 @@ The
 .Nm
 routines provide the following functionality:
 .Bl -tag -width "copyoutstr()"
+.\" ========
 .It Fn copyin
 Copies
 .Pa len
@@ -76,23 +100,7 @@ bytes of data from the user-space address
 .Pa uaddr
 to the kernel-space address
 .Pa kaddr .
-.It Fn copyout
-Copies
-.Pa len
-bytes of data from the kernel-space address
-.Pa kaddr
-to the user-space address
-.Pa uaddr .
-.It Fn copystr
-Copies a NUL-terminated string, at most
-.Pa len
-bytes long, from kernel-space address
-.Pa kfaddr
-to kernel-space address
-.Pa kdaddr .
-The number of bytes actually copied, including the terminating
-NUL, is returned in
-.Pa *done .
+.\" ========
 .It Fn copyinstr
 Copies a NUL-terminated string, at most
 .Pa len
@@ -103,6 +111,15 @@ to kernel-space address
 The number of bytes actually copied, including the terminating
 NUL, is returned in
 .Pa *done .
+.\" ========
+.It Fn copyout
+Copies
+.Pa len
+bytes of data from the kernel-space address
+.Pa kaddr
+to the user-space address
+.Pa uaddr .
+.\" ========
 .\" .It Fn copyoutstr
 .\" Copies a NUL-terminated string, at most
 .\" bytes long, from kernel-space address
@@ -112,6 +129,17 @@ NUL, is returned in
 .\" The number of bytes actually copied, including the terminating
 .\" NUL, is returned in
 .\" .Pa *done .
+.\" ========
+.It Fn copystr
+Copies a NUL-terminated string, at most
+.Pa len
+bytes long, from kernel-space address
+.Pa kfaddr
+to kernel-space address
+.Pa kdaddr .
+The number of bytes actually copied, including the terminating
+NUL, is returned in
+.Pa *done .
 .El
 .Sh RETURN VALUES
 The
diff --git a/bsd/man/man9/copyin.9 b/bsd/man/man9/copyin.9
new file mode 100644
index 000000000..df9640d8e
--- /dev/null
+++ b/bsd/man/man9/copyin.9
@@ -0,0 +1 @@
+.so man9/copy.9
\ No newline at end of file
diff --git a/bsd/man/man9/copyinstr.9 b/bsd/man/man9/copyinstr.9
new file mode 100644
index 000000000..df9640d8e
--- /dev/null
+++ b/bsd/man/man9/copyinstr.9
@@ -0,0 +1 @@
+.so man9/copy.9
\ No newline at end of file
diff --git a/bsd/man/man9/copyout.9 b/bsd/man/man9/copyout.9
new file mode 100644
index 000000000..df9640d8e
--- /dev/null
+++ b/bsd/man/man9/copyout.9
@@ -0,0 +1 @@
+.so man9/copy.9
\ No newline at end of file
diff --git a/bsd/man/man9/copystr.9 b/bsd/man/man9/copystr.9
new file mode 100644
index 000000000..df9640d8e
--- /dev/null
+++ b/bsd/man/man9/copystr.9
@@ -0,0 +1 @@
+.so man9/copy.9
\ No newline at end of file
diff --git a/bsd/man/man9/fetch.9 b/bsd/man/man9/fetch.9
index 24d3182e0..7e6531c78 100644
--- a/bsd/man/man9/fetch.9
+++ b/bsd/man/man9/fetch.9
@@ -41,28 +41,41 @@
 .Nm fetch ,
 .Nm fubyte ,
 .Nm fuibyte ,
-.Nm fuword ,
 .Nm fuiword ,
 .Nm fulong ,
-.Nm fuulong
+.Nm fuulong ,
+.Nm fuword
 .Nd fetch data from user-space
 .Sh SYNOPSIS
 .In sys/types.h
 .In sys/time.h
 .In sys/systm.h
 .In sys/resourcevar.h
+.\"
 .Ft int
-.Fn fubyte "const user_addr_t addr"
-.Ft int
-.Fn fuibyte "const user_addr_t addr"
+.Fo fubyte
+.Fa "const user_addr_t addr"
+.Fc
 .Ft int
-.Fn fuword "user_addr_t addr"
+.Fo fuibyte
+.Fa "const user_addr_t addr"
+.Fc
 .Ft int
-.Fn fuiword "user_addr_t addr"
+.Fo fuiword
+.Fa "user_addr_t addr"
+.Fc
 .Ft int64_t
-.Fn fulong "user_addr_t addr"
+.Fo fulong
+.Fa "user_addr_t addr"
+.Fc
 .Ft uint64_t
-.Fn fuulong "user_addr_t addr"
+.Fo fuulong
+.Fa "user_addr_t addr"
+.Fc
+.Ft int
+.Fo fuword
+.Fa "user_addr_t addr"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm
@@ -72,26 +85,32 @@ The
 .Nm
 routines provide the following functionality:
 .Bl -tag -width "fuiword()"
+.\" ==========
 .It Fn fubyte
 Fetches a byte of data from the user-space address
 .Pa addr .
+.\" ==========
 .It Fn fuibyte
 Fetches a byte of data from the user-space address
 .Pa addr .
 This function is safe to call during an interrupt context.
-.It Fn fuword
-Fetches a word of data from the user-space address
-.Pa addr .
+.\" ==========
 .It Fn fuiword
 Fetches a word of data from the user-space address
 .Pa addr .
 This function is safe to call during an interrupt context.
+.\" ==========
 .It Fn fulong
 Fetches a long word of data from the user-space address
 .Pa addr .
+.\" ==========
 .It Fn fuulong
 Fetches a unsigned long word of data from the user-space address
 .Pa addr .
+.\" ==========
+.It Fn fuword
+Fetches a word of data from the user-space address
+.Pa addr .
 .El
 .Sh RETURN VALUES
 The
diff --git a/bsd/man/man9/fubyte.9 b/bsd/man/man9/fubyte.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fubyte.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/fuibyte.9 b/bsd/man/man9/fuibyte.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fuibyte.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/fuiword.9 b/bsd/man/man9/fuiword.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fuiword.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/fulong.9 b/bsd/man/man9/fulong.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fulong.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/fuulong.9 b/bsd/man/man9/fuulong.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fuulong.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/fuword.9 b/bsd/man/man9/fuword.9
new file mode 100644
index 000000000..114b2594d
--- /dev/null
+++ b/bsd/man/man9/fuword.9
@@ -0,0 +1 @@
+.so man9/fetch.9
\ No newline at end of file
diff --git a/bsd/man/man9/store.9 b/bsd/man/man9/store.9
index 1092c2fce..fd6378c9a 100644
--- a/bsd/man/man9/store.9
+++ b/bsd/man/man9/store.9
@@ -41,10 +41,10 @@
 .Nm store ,
 .Nm subyte ,
 .Nm suibyte ,
-.Nm suword ,
 .Nm suiword ,
 .Nm sulong ,
-.Nm suulong
+.Nm suulong ,
+.Nm suword
 .Nd store data to user-space
 .Sh SYNOPSIS
 .In sys/types.h
@@ -52,17 +52,35 @@
 .In sys/systm.h
 .In sys/resourcevar.h
 .Ft int
-.Fn subyte "user_addr_t addr" "int byte"
+.Fo subyte
+.Fa "user_addr_t addr"
+.Fa "int byte"
+.Fc
 .Ft int
-.Fn suibyte "user_addr_t addr" "int byte"
+.Fo suibyte
+.Fa "user_addr_t addr"
+.Fa "int byte"
+.Fc
 .Ft int
-.Fn suword "user_addr_t addr" "int word"
+.Fo suiword
+.Fa "user_addr_t addr"
+.Fa "int word"
+.Fc
 .Ft int
-.Fn suiword "user_addr_t addr" "int word"
+.Fo sulong
+.Fa "user_addr_t addr"
+.Fa "int64_t longword"
+.Fc
 .Ft int
-.Fn sulong "user_addr_t addr" "int64_t longword"
+.Fo suulong
+.Fa "user_addr_t addr"
+.Fa "uint64_t longword"
+.Fc
 .Ft int
-.Fn suulong "user_addr_t addr" "uint64_t longword"
+.Fo suword
+.Fa "user_addr_t addr"
+.Fa "int word"
+.Fc
 .Sh DESCRIPTION
 The
 .Nm
@@ -72,26 +90,32 @@ The
 .Nm
 routines provide the following functionality:
 .Bl -tag -width "suibyte()"
+.\" ========
 .It Fn subyte
 Stores a byte of data to the user-space address
 .Pa addr .
+.\" ========
 .It Fn suibyte
 Stores a byte of data to the user-space address
 .Pa addr .
 This function is safe to call during an interrupt context.
-.It Fn suword
-Stores a word of data to the user-space address
-.Pa addr .
+.\" ========
 .It Fn suiword
 Stores a word of data to the user-space address
 .Pa addr .
 This function is safe to call during an interrupt context.
+.\" ========
 .It Fn sulong
 Stores a long word of data to the user-space address
 .Pa addr .
+.\" ========
 .It Fn suulong
 Stores a unsigned long word of data to the user-space address
 .Pa addr .
+.\" ========
+.It Fn suword
+Stores a word of data to the user-space address
+.Pa addr .
 .El
 .Sh RETURN VALUES
 The
diff --git a/bsd/man/man9/style.9 b/bsd/man/man9/style.9
index 115fe669a..82281373e 100644
--- a/bsd/man/man9/style.9
+++ b/bsd/man/man9/style.9
@@ -220,21 +220,24 @@ struct foo {
 LIST_HEAD(, foo) foohead;		/* Head of global foo list. */
 .Ed
 .Pp
-Avoid using typedefs for structure types.
-This makes it impossible
-for applications to use pointers to such a structure opaquely, which
-is both possible and beneficial when using an ordinary struct tag.
+Avoid using typedefs for structure types.  Typedefs are problematic
+because they do not properly hide their underlying type; for example you
+need to know if the typedef is the structure itself or a pointer to the
+structure.  In addition they must be declared exactly once, whereas an
+incomplete structure type can be mentioned as many times as necessary.
+Typedefs are difficult to use in stand-alone header files: the header
+that defines the typedef must be included before the header that uses it,
+or by the header that uses it (which causes namespace pollution), or
+there must be a back-door mechanism for obtaining the typedef.
+.Pp
 When convention requires a
 .Ic typedef ,
 make its name match the struct tag.
-Avoid typedefs ending in
-.Dq Li _t ,
-except as specified in Standard C or by \*[Px].
 .Bd -literal
 /* Make the structure name match the typedef. */
 typedef struct bar {
 	int	level;
-} BAR;
+} bar_t;
 .Ed
 .Pp
 All functions are prototyped somewhere.
@@ -244,6 +247,9 @@ elsewhere) go at the top of the first source module.
 Functions
 local to one source module should be declared
 .Ic static .
+Functions that are not exported outside of the kernel should
+be declared
+.Ic __private_extern__ .
 .Pp
 Functions used from other parts of the kernel are prototyped in the
 relevant include file.
@@ -252,16 +258,7 @@ Functions that are used locally in more than one module go into a
 separate header file, e.g.\&
 .Qq Pa extern.h .
 .Pp
-Only use the
-.Dv __P
-macro from the include file
-.Aq Pa sys/cdefs.h
-if the source
-file in general is (to be) compilable with a K&R Old Testament compiler.
-Use of the
-.Dv __P
-macro in new code is discouraged, although modifications
-to existing files should be consistent with that file's conventions.
+Do not use the __P macro.
 .Pp
 In general code can be considered
 .Dq "new code"
@@ -400,13 +397,15 @@ inside blocks unless the routine is unusually complicated.
 	}
 .Ed
 .Pp
-Indentation is an 8 character tab.
-Second level indents are four spaces.
+Variable names should contain underscores to separate words.  DO NOT use
+StudlyCaps.
+.Pp
+Indentation is an 8 character tab.  All code should fit in 80 columns.
 If you have to wrap a long statement, put the operator at the end of the
 line.
 .Bd -literal
-	while (cnt < 20 && this_variable_name_is_too_long_for_its_own_good &&
-	    ep != NULL)
+	while (cnt < 20 && this_variable_name_is_too_long &&
+	       ep != NULL)
 		z = a + really + long + statement + that + needs +
 		    two lines + gets + indented + four + spaces +
 		    on + the + second + and + subsequent + lines;
@@ -508,6 +507,8 @@ are not followed by a space.
 Note that
 .Xr indent 1
 does not understand this rule.
+.Ic sizeof Ns 's
+are written with parentheses always.
 .Pp
 .Dv NULL
 is the preferred null pointer constant.
@@ -556,6 +557,10 @@ Routines returning
 should not have their return values cast
 to any pointer type.
 .Pp
+Values in
+.Ic return
+statements should be enclosed in parentheses.
+.Pp
 Use
 .Xr err 3
 or
@@ -570,18 +575,7 @@ do not roll your own.
 }
 .Ed
 .Pp
-Old-style function declarations look like this:
-.Bd -literal
-static char *
-function(a1, a2, fl, a4)
-	int a1, a2;	/* Declare ints, too, don't default them. */
-	float fl;	/* Beware double vs. float prototype differences. */
-	int a4;		/* List in order declared. */
-{
-.Ed
-.Pp
-Use ANSI function declarations unless you explicitly need K&R compatibility.
-Long parameter lists are wrapped with a normal four space indent.
+Use ANSI function declarations.
 .Pp
 Variable numbers of arguments should look like this.
 .Bd -literal
@@ -597,11 +591,6 @@ vaf(const char *fmt, ...)
 	va_end(ap);
 	/* No return needed for void functions. */
 }
-
-static void
-usage()
-{
-	/* Insert an empty line if the function has no local variables. */
 .Ed
 .Pp
 Use
@@ -663,11 +652,9 @@ That is, without regard to whether an option takes arguments or not.
 The alphabetical ordering should take into account the case ordering
 shown above.
 .Pp
-New core kernel code should be reasonably compliant with the
+New core kernel code should be compliant with the
 .Nm
 guides.
-The guidelines for third-party maintained modules and device drivers are more
-relaxed but at a minimum should be internally consistent with their style.
 .Pp
 Stylistic changes (including whitespace changes) are hard on the source
 repository and are to be avoided without good reason.
@@ -677,12 +664,9 @@ KNF
 .Nm
 compliant in the repository must not diverge from compliance.
 .Pp
-Whenever possible, code should be run through a code checker
-(e.g.,
-.Xr lint 1
-or
-.Nm gcc Fl Wall )
-and produce minimal warnings.
+Code should be run through a code checker (e.g., sparse or
+.Nm gcc Fl Wall Fl Werror
+).
 .Sh SEE ALSO
 .Xr indent 1 ,
 .Xr lint 1 ,
diff --git a/bsd/man/man9/subyte.9 b/bsd/man/man9/subyte.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/subyte.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/man/man9/suibyte.9 b/bsd/man/man9/suibyte.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/suibyte.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/man/man9/suiword.9 b/bsd/man/man9/suiword.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/suiword.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/man/man9/sulong.9 b/bsd/man/man9/sulong.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/sulong.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/man/man9/suulong.9 b/bsd/man/man9/suulong.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/suulong.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/man/man9/suword.9 b/bsd/man/man9/suword.9
new file mode 100644
index 000000000..996135b81
--- /dev/null
+++ b/bsd/man/man9/suword.9
@@ -0,0 +1 @@
+.so man9/store.9
\ No newline at end of file
diff --git a/bsd/miscfs/deadfs/dead_vnops.c b/bsd/miscfs/deadfs/dead_vnops.c
index 2e58fddac..4af10dd02 100644
--- a/bsd/miscfs/deadfs/dead_vnops.c
+++ b/bsd/miscfs/deadfs/dead_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -64,6 +70,8 @@
 #include <sys/buf.h>
 #include <vfs/vfs_support.h>
 
+int	chkvnlock(vnode_t vp);
+
 /*
  * Prototypes for dead operations on vnodes.
  */
@@ -151,13 +159,7 @@ struct vnodeopv_desc dead_vnodeop_opv_desc =
  */
 /* ARGSUSED */
 int
-dead_lookup(ap)
-	struct vnop_lookup_args /* {
-		struct vnode * a_dvp;
-		struct vnode ** a_vpp;
-		struct componentname * a_cnp;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_lookup(struct vnop_lookup_args *ap)
 {
 
 	*ap->a_vpp = NULL;
@@ -169,14 +171,8 @@ dead_lookup(ap)
  */
 /* ARGSUSED */
 int
-dead_open(ap)
-	struct vnop_open_args /* {
-		struct vnode *a_vp;
-		int  a_mode;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_open(__unused struct vnop_open_args *ap)
 {
-
 	return (ENXIO);
 }
 
@@ -185,13 +181,7 @@ dead_open(ap)
  */
 /* ARGSUSED */
 int
-dead_read(ap)
-	struct vnop_read_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_read(struct vnop_read_args *ap)
 {
 
 	if (chkvnlock(ap->a_vp))
@@ -209,13 +199,7 @@ dead_read(ap)
  */
 /* ARGSUSED */
 int
-dead_write(ap)
-	struct vnop_write_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_write(struct vnop_write_args *ap)
 {
 
 	if (chkvnlock(ap->a_vp))
@@ -228,14 +212,7 @@ dead_write(ap)
  */
 /* ARGSUSED */
 int
-dead_ioctl(ap)
-	struct vnop_ioctl_args /* {
-		struct vnode *a_vp;
-		u_long a_command;
-		caddr_t  a_data;
-		int  a_fflag;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_ioctl(struct vnop_ioctl_args *ap)
 {
 
 	if (!chkvnlock(ap->a_vp))
@@ -245,15 +222,7 @@ dead_ioctl(ap)
 
 /* ARGSUSED */
 int
-dead_select(ap)
-	struct vnop_select_args /* {
-		struct vnode *a_vp;
-		int  a_which;
-		int  a_fflags;
-		kauth_cred_t a_cred;
-		void *a_wql;
-		struct proc *a_p;
-	} */ *ap;
+dead_select(__unused struct vnop_select_args *ap)
 {
 
 	/*
@@ -266,10 +235,7 @@ dead_select(ap)
  * Just call the device strategy routine
  */
 int
-dead_strategy(ap)
-	struct vnop_strategy_args /* {
-		struct buf *a_bp;
-	} */ *ap;
+dead_strategy(struct vnop_strategy_args *ap)
 {
 
 	if (buf_vnode(ap->a_bp) == NULL || !chkvnlock(buf_vnode(ap->a_bp))) {
@@ -284,17 +250,7 @@ dead_strategy(ap)
  * Wait until the vnode has finished changing state.
  */
 int
-dead_blockmap(ap)
-	struct vnop_blockmap_args /* {
-		struct vnode *a_vp;
-		off_t a_foffset;
-		size_t a_size;
-		daddr64_t *a_bpn;
-		size_t *a_run;
-		void *a_poff;
-		int flags;
-		vfs_context_t a_context;
-	} */ *ap;
+dead_blockmap(struct vnop_blockmap_args *ap)
 {
 
 	if (!chkvnlock(ap->a_vp))
@@ -308,7 +264,7 @@ dead_blockmap(ap)
  */
 /* ARGSUSED */
 int
-dead_ebadf(void *dummy)
+dead_ebadf(__unused void *dummy)
 {
 
 	return (EBADF);
@@ -319,22 +275,12 @@ dead_ebadf(void *dummy)
  */
 /* ARGSUSED */
 int
-dead_badop(void *dummy)
+dead_badop(__unused void *dummy)
 {
 
 	panic("dead_badop called");
 	/* NOTREACHED */
-}
-
-/*
- * Empty vnode null operation
- */
-/* ARGSUSED */
-int
-dead_nullop(void *dummy)
-{
-
-	return (0);
+	return (-1);
 }
 
 /*
@@ -350,12 +296,7 @@ chkvnlock(__unused vnode_t vp)
 
 /* Blktooff */
 int
-dead_blktooff(ap)
-	struct vnop_blktooff_args /* {
-		struct vnode *a_vp;
-		daddr64_t a_lblkno;
-		off_t *a_offset;    
-	} */ *ap;
+dead_blktooff(struct vnop_blktooff_args *ap)
 {
     if (!chkvnlock(ap->a_vp))
 		return (EIO);
@@ -365,12 +306,7 @@ dead_blktooff(ap)
 }
 /* Blktooff */
 int
-dead_offtoblk(ap)
-struct vnop_offtoblk_args /* {
-	struct vnode *a_vp;
-	off_t a_offset;    
-	daddr64_t *a_lblkno;
-	} */ *ap;
+dead_offtoblk(struct vnop_offtoblk_args *ap)
 {
     if (!chkvnlock(ap->a_vp))
 		return (EIO);
diff --git a/bsd/miscfs/devfs/Makefile b/bsd/miscfs/devfs/Makefile
index b94c89528..2996cf090 100644
--- a/bsd/miscfs/devfs/Makefile
+++ b/bsd/miscfs/devfs/Makefile
@@ -20,10 +20,9 @@ EXPINC_SUBDIRS_PPC = \
 EXPINC_SUBDIRS_I386 = \
 
 DATAFILES = \
-	devfs.h
+	devfs.h	devfs_proto.h devfsdefs.h
 
-PRIVATE_DATAFILES = \
-	devfs_proto.h devfsdefs.h
+PRIVATE_DATAFILES = 
 
 INSTALL_MI_LIST	= ${DATAFILES}
 
diff --git a/bsd/miscfs/devfs/devfs.h b/bsd/miscfs/devfs/devfs.h
index 647e94785..88e63b4f5 100644
--- a/bsd/miscfs/devfs/devfs.h
+++ b/bsd/miscfs/devfs/devfs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1997,1998 Julian Elischer.  All rights reserved.
@@ -55,8 +61,37 @@
 #define DEVFS_CHAR 	0
 #define DEVFS_BLOCK 	1
 
+/*
+ * Argument to clone callback after dev
+ */
+#define	DEVFS_CLONE_ALLOC	1	/* Allocate minor number slot */
+#define	DEVFS_CLONE_FREE	0	/* Free minor number slot */
+
 __BEGIN_DECLS
 
+/*
+ * Function: devfs_make_node_clone
+ *
+ * Purpose
+ *   Create a device node with the given pathname in the devfs namespace;
+ *   before returning a dev_t value for an open instance, the dev_t has
+ *   it's minor number updated by calling the supplied clone function on
+ *   the supplied dev..
+ *
+ * Parameters:
+ *   dev 	- the dev_t value to associate
+ *   chrblk	- block or character device (DEVFS_CHAR or DEVFS_BLOCK)
+ *   uid, gid	- ownership
+ *   perms	- permissions
+ *   clone	- minor number cloning function
+ *   fmt, ...	- print format string and args to format the path name
+ * Returns:
+ *   A handle to a device node if successful, NULL otherwise.
+ */
+void * 	devfs_make_node_clone(dev_t dev, int chrblk, uid_t uid, gid_t gid, 
+			     int perms, int (*clone)(dev_t dev, int action),
+			     const char *fmt, ...);
+
 /*
  * Function: devfs_make_node
  *
@@ -85,7 +120,7 @@ void * 	devfs_make_node(dev_t dev, int chrblk, uid_t uid, gid_t gid,
  * Returns:
  *   0 if successful, -1 if failed
  */
-int	devfs_link(void * handle, char *fmt, ...);
+int	devfs_make_link(void * handle, char *fmt, ...);
 #endif /* BSD_KERNEL_PRIVATE */
 
 /*
@@ -108,6 +143,7 @@ __END_DECLS
 /* XXX */
 #define	GID_WHEEL	0
 #define	GID_KMEM	2
+#define	GID_TTY		4
 #define	GID_OPERATOR	5
 #define	GID_BIN		7
 #define	GID_GAMES	13
diff --git a/bsd/miscfs/devfs/devfs_proto.h b/bsd/miscfs/devfs/devfs_proto.h
index 77e9b1c2f..dc6cab891 100644
--- a/bsd/miscfs/devfs/devfs_proto.h
+++ b/bsd/miscfs/devfs/devfs_proto.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* THIS FILE HAS BEEN PRODUCED AUTOMATICALLY */
 #ifndef __DEVFS_DEVFS_PROTO_H__
@@ -27,8 +33,8 @@
 
 #ifdef __APPLE_API_PRIVATE
 int	devfs_sinit(void);
-devdirent_t *	dev_findname(devnode_t * dir,char *name);
-int	dev_add_name(char * name, devnode_t * dirnode, devdirent_t * back, 
+devdirent_t *	dev_findname(devnode_t * dir, const char *name);
+int	dev_add_name(const char * name, devnode_t * dirnode, devdirent_t * back, 
     devnode_t * dnp, devdirent_t * *dirent_pp);
 int	dev_add_node(int entrytype, devnode_type_t * typeinfo, devnode_t * proto,
 	     devnode_t * *dn_pp, struct devfsmount *dvm);
@@ -37,10 +43,11 @@ int	dev_dup_plane(struct devfsmount *devfs_mp_p);
 void	devfs_free_plane(struct devfsmount *devfs_mp_p);
 int	dev_free_name(devdirent_t * dirent_p);
 int	devfs_dntovn(devnode_t * dnp, struct vnode **vn_pp, struct proc * p);
-int	dev_add_entry(char *name, devnode_t * parent, int type, devnode_type_t * typeinfo,
+int	dev_add_entry(const char *name, devnode_t * parent, int type, devnode_type_t * typeinfo,
 	      devnode_t * proto, struct devfsmount *dvm, devdirent_t * *nm_pp);
 int	devfs_mount(struct mount *mp, vnode_t devvp, user_addr_t data,
 	    vfs_context_t context);
+int	devfs_kernel_mount(char * mntname);
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* __DEVFS_DEVFS_PROTO_H__ */
diff --git a/bsd/miscfs/devfs/devfs_tree.c b/bsd/miscfs/devfs/devfs_tree.c
index ffecc22d2..28ee35c2d 100644
--- a/bsd/miscfs/devfs/devfs_tree.c
+++ b/bsd/miscfs/devfs/devfs_tree.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -47,6 +53,12 @@
  * 
  * devfs_tree.c
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * HISTORY
@@ -84,16 +96,21 @@
 #include <sys/malloc.h>
 #include <sys/mount_internal.h>
 #include <sys/proc.h>
-#include <sys/vnode.h>
+#include <sys/vnode_internal.h>
 #include <stdarg.h>
-
+#include <libkern/OSAtomic.h>
+#define BSD_KERNEL_PRIVATE	1	/* devfs_make_link() prototype */
 #include "devfs.h"
 #include "devfsdefs.h"
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 static void	devfs_release_busy(devnode_t *);
 static void	dev_free_hier(devdirent_t *);
 static int	devfs_propogate(devdirent_t *, devdirent_t *);
-static int	dev_finddir(char *, devnode_t *, int, devnode_t **);
+static int	dev_finddir(const char *, devnode_t *, int, devnode_t **);
 static int	dev_dup_entry(devnode_t *, devdirent_t *, devdirent_t **, struct devfsmount *);
 
 
@@ -111,8 +128,8 @@ static struct mount *devfs_hidden_mount;
 
 static int devfs_ready = 0;
 
-#define NOCREATE	FALSE
-#define CREATE		TRUE
+#define DEVFS_NOCREATE	FALSE
+#define DEVFS_CREATE	TRUE
 
 /*
  * Set up the root directory node in the backing plane
@@ -152,15 +169,26 @@ devfs_sinit(void)
 	TAILQ_INIT(&devfs_hidden_mount->mnt_vnodelist);
 	TAILQ_INIT(&devfs_hidden_mount->mnt_workerqueue);
 	TAILQ_INIT(&devfs_hidden_mount->mnt_newvnodes);
+#if CONFIG_MACF
+	mac_mount_label_init(devfs_hidden_mount);
+	mac_mount_label_associate(vfs_context_kernel(), devfs_hidden_mount);
+#endif
 
 	/* Initialize the default IO constraints */
 	mp->mnt_maxreadcnt = mp->mnt_maxwritecnt = MAXPHYS;
 	mp->mnt_segreadcnt = mp->mnt_segwritecnt = 32;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
 
 	devfs_mount(devfs_hidden_mount,"dummy",NULL,NULL,NULL);
 	dev_root->de_dnp->dn_dvm 
 	    = (struct devfsmount *)devfs_hidden_mount->mnt_data;
 #endif /* HIDDEN_MOUNTPOINT */
+#if CONFIG_MACF
+	mac_devfs_label_associate_directory("/", strlen("/"),
+	    dev_root->de_dnp, "/");
+#endif
 	devfs_ready = 1;
 	return (0);
 }
@@ -180,7 +208,7 @@ devfs_sinit(void)
  * called with DEVFS_LOCK held
  ***************************************************************/
 devdirent_t *
-dev_findname(devnode_t * dir, char *name)
+dev_findname(devnode_t * dir, const char *name)
 {
 	devdirent_t * newfp;
 	if (dir->dn_type != DEV_DIR) return 0;/*XXX*/ /* printf?*/
@@ -201,7 +229,7 @@ dev_findname(devnode_t * dir, char *name)
 
 	while(newfp)
 	{
-		if(!(strcmp(name,newfp->de_name)))
+		if(!(strncmp(name, newfp->de_name, sizeof(newfp->de_name))))
 			return newfp;
 		newfp = newfp->de_next;
 	}
@@ -210,7 +238,7 @@ dev_findname(devnode_t * dir, char *name)
 
 /***********************************************************************
  * Given a starting node (0 for root) and a pathname, return the node	
- * for the end item on the path. It MUST BE A DIRECTORY. If the 'CREATE'
+ * for the end item on the path. It MUST BE A DIRECTORY. If the 'DEVFS_CREATE'
  * option is true, then create any missing nodes in the path and create
  * and return the final node as well.					
  * This is used to set up a directory, before making nodes in it..
@@ -218,14 +246,17 @@ dev_findname(devnode_t * dir, char *name)
  * called with DEVFS_LOCK held
  ***********************************************************************/
 static int
-dev_finddir(char * path, 
+dev_finddir(const char * path, 
 	    devnode_t * dirnode,
 	    int create, 
 	    devnode_t * * dn_pp)
 {
 	devnode_t *	dnp = NULL;
 	int		error = 0;
-	char *		scan;
+	const char *		scan;
+#if CONFIG_MACF
+	char            fullpath[DEVMAXPATHSIZE];
+#endif
 
 
 	if (!dirnode) /* dirnode == NULL means start at root */
@@ -237,6 +268,9 @@ dev_finddir(char * path,
 	if (strlen(path) > (DEVMAXPATHSIZE - 1)) 
 	    return ENAMETOOLONG;
 
+#if CONFIG_MACF
+	strlcpy (fullpath, path, DEVMAXPATHSIZE);
+#endif
 	scan = path;
 
 	while (*scan == '/') 
@@ -247,7 +281,7 @@ dev_finddir(char * path,
 	while (1) {
 	    char		component[DEVMAXPATHSIZE];
 	    devdirent_t *	dirent_p;
-	    char * 		start;
+	    const char * 	start;
 
 	    if (*scan == 0) { 
 		/* we hit the end of the string, we're done */
@@ -258,8 +292,7 @@ dev_finddir(char * path,
 	    while (*scan != '/' && *scan)
 		scan++;
 
-	    strncpy(component, start, scan - start);
-		component[ scan - start ] = '\0';
+	    strlcpy(component, start, scan - start);
 	    if (*scan == '/')
 		scan++;
 
@@ -281,6 +314,12 @@ dev_finddir(char * path,
 		if (error)
 		    break;
 		dnp = dirent_p->de_dnp;
+#if CONFIG_MACF
+		mac_devfs_label_associate_directory(
+		    dirnode->dn_typeinfo.Dir.myname->de_name, 
+		    strlen(dirnode->dn_typeinfo.Dir.myname->de_name),
+		    dnp, fullpath);
+#endif
 		devfs_propogate(dirnode->dn_typeinfo.Dir.myname, dirent_p);
 	    }
 	    dirnode = dnp; /* continue relative to this directory */
@@ -299,7 +338,7 @@ dev_finddir(char * path,
  * called with DEVFS_LOCK held
  ***********************************************************************/
 int
-dev_add_name(char * name, devnode_t * dirnode, __unused devdirent_t * back, 
+dev_add_name(const char * name, devnode_t * dirnode, __unused devdirent_t * back, 
     devnode_t * dnp, devdirent_t * *dirent_pp)
 {
 	devdirent_t * 	dirent_p = NULL;
@@ -384,7 +423,7 @@ dev_add_name(char * name, devnode_t * dirnode, __unused devdirent_t * back,
 	/*
 	 * put the name into the directory entry.
 	 */
-	strcpy(dirent_p->de_name, name);
+	strlcpy(dirent_p->de_name, name, DEVMAXNAMESIZE);
 
 
 	/*
@@ -418,7 +457,7 @@ dev_add_name(char * name, devnode_t * dirnode, __unused devdirent_t * back,
  * reused. (Is a DIR, or we select SPLIT_DEVS at compile time)
  * typeinfo gives us info to make our node if we don't have a prototype.
  * If typeinfo is null and proto exists, then the typeinfo field of
- * the proto is used intead in the CREATE case.
+ * the proto is used intead in the DEVFS_CREATE case.
  * note the 'links' count is 0 (except if a dir)
  * but it is only cleared on a transition
  * so this is ok till we link it to something
@@ -480,6 +519,10 @@ dev_add_node(int entrytype, devnode_type_t * typeinfo, devnode_t * proto,
 		*(dnp->dn_prevsiblingp) = dnp;
 		dnp->dn_nextsibling = proto;
 		proto->dn_prevsiblingp = &(dnp->dn_nextsibling);
+#if CONFIG_MACF
+		mac_devfs_label_init(dnp);
+		mac_devfs_label_copy(proto->dn_label, dnp->dn_label);
+#endif
 	} else {
 	        struct timeval tv;
 
@@ -494,6 +537,9 @@ dev_add_node(int entrytype, devnode_type_t * typeinfo, devnode_t * proto,
 		dnp->dn_atime.tv_sec = tv.tv_sec;
 		dnp->dn_mtime.tv_sec = tv.tv_sec;
 		dnp->dn_ctime.tv_sec = tv.tv_sec;
+#if CONFIG_MACF
+		mac_devfs_label_init(dnp);
+#endif
 	}
 	dnp->dn_dvm = dvm;
 
@@ -536,9 +582,8 @@ dev_add_node(int entrytype, devnode_type_t * typeinfo, devnode_t * proto,
 		    	FREE(dnp,M_DEVFSNODE);
 			return ENOMEM;
 		}
-		strncpy(dnp->dn_typeinfo.Slnk.name, typeinfo->Slnk.name,
-			typeinfo->Slnk.namelen);
-		dnp->dn_typeinfo.Slnk.name[typeinfo->Slnk.namelen] = '\0';
+		strlcpy(dnp->dn_typeinfo.Slnk.name, typeinfo->Slnk.name,
+			typeinfo->Slnk.namelen + 1);
 		dnp->dn_typeinfo.Slnk.namelen = typeinfo->Slnk.namelen;
 		DEVFS_INCR_STRINGSPACE(dnp->dn_typeinfo.Slnk.namelen + 1);
 		dnp->dn_ops = &devfs_vnodeop_p;
@@ -573,6 +618,9 @@ devnode_free(devnode_t * dnp)
             dnp->dn_lflags |= DN_DELETE;
 	    return;
     }
+#if CONFIG_MACF
+	mac_devfs_label_destroy(dnp);
+#endif
     if (dnp->dn_type == DEV_SLNK) {
         DEVFS_DECR_STRINGSPACE(dnp->dn_typeinfo.Slnk.namelen + 1);
 	FREE(dnp->dn_typeinfo.Slnk.name, M_DEVFSNODE);
@@ -927,10 +975,12 @@ int
 devfs_dntovn(devnode_t * dnp, struct vnode **vn_pp, __unused struct proc * p)
 {
 	struct vnode *vn_p;
+	struct vnode ** vnptr;
 	int error = 0;
 	struct vnode_fsparam vfsp;
 	enum vtype vtype = 0;
 	int markroot = 0;
+	int n_minor = DEVFS_CLONE_ALLOC; /* new minor number for clone device */
 
 retry:
 	*vn_pp = NULL;
@@ -1008,10 +1058,28 @@ retry:
 	vfsp.vnfs_cnp = 0;
 	vfsp.vnfs_vops = *(dnp->dn_ops);
 		
-	if (vtype == VBLK || vtype == VCHR)
+	if (vtype == VBLK || vtype == VCHR) {
+		/*
+		 * Ask the clone minor number function for a new minor number
+		 * to use for the next device instance.  If an administative
+		 * limit has been reached, this function will return -1.
+		 */
+		if (dnp->dn_clone != NULL) {
+			int	n_major = major(dnp->dn_typeinfo.dev);
+
+			n_minor = (*dnp->dn_clone)(dnp->dn_typeinfo.dev, DEVFS_CLONE_ALLOC);
+			if (n_minor == -1) {
+				devfs_release_busy(dnp);
+				return ENOMEM;
+			}
+
+			vfsp.vnfs_rdev = makedev(n_major, n_minor);;
+		} else {
 		vfsp.vnfs_rdev = dnp->dn_typeinfo.dev;
-	else
+		}
+	} else {
 		vfsp.vnfs_rdev = 0;
+	}
 	vfsp.vnfs_filesize = 0;
 	vfsp.vnfs_flags = VNFS_NOCACHE | VNFS_CANTCACHE;
 	/* Tag system files */
@@ -1020,22 +1088,42 @@ retry:
 
 	DEVFS_UNLOCK();
 
-	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &vn_p);
+	if (dnp->dn_clone == NULL)
+		vnptr = &dnp->dn_vn;
+	else
+		vnptr = &vn_p;	
+	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, vnptr);
 
 	DEVFS_LOCK();
 
 	if (error == 0) {
-		if ((dnp->dn_vn)) {
-			panic("devnode already has a vnode?");
-		} else {
-			dnp->dn_vn = vn_p;
-			*vn_pp = vn_p;
-			vnode_settag(vn_p, VT_DEVFS);
-		}
+			if ((dnp->dn_clone != NULL) && (dnp->dn_vn != NULLVP) )
+				panic("devnode already has a vnode?");
+			/*
+			 * Don't cache the vnode for the next open, if the
+			 * device is a cloning device (each open gets it's
+			 * own per-device instance vnode).
+			 */
+			if (dnp->dn_clone == NULL) {
+				*vn_pp = dnp->dn_vn;
+			 } else {
+				*vn_pp = vn_p;
+			}
+
+	} else if (n_minor != DEVFS_CLONE_ALLOC) {
+		/*
+		 * If we failed the create, we need to release the cloned minor
+		 * back to the free list.  In general, this is only useful if
+		 * the clone function results in a state change in the cloned
+		 * device for which the minor number was obtained.  If we get
+		 * past this point withouth falling into this case, it's
+		 * assumed that any state to be released will be released when
+		 * the vnode is dropped, instead.
+		 */
+		 (void)(*dnp->dn_clone)(dnp->dn_typeinfo.dev, DEVFS_CLONE_FREE);
 	}
 
 	dnp->dn_lflags &= ~DN_CREATE;
-
 	if (dnp->dn_lflags & DN_CREATEWAIT) {
 		dnp->dn_lflags &= ~DN_CREATEWAIT;
 		wakeup(&dnp->dn_lflags);
@@ -1066,7 +1154,7 @@ devfs_release_busy(devnode_t *dnp) {
  * called with DEVFS_LOCK held
  ***********************************************************************/
 int
-dev_add_entry(char *name, devnode_t * parent, int type, devnode_type_t * typeinfo,
+dev_add_entry(const char *name, devnode_t * parent, int type, devnode_type_t * typeinfo,
 	      devnode_t * proto, struct devfsmount *dvm, devdirent_t * *nm_pp)
 {
 	devnode_t *	dnp;
@@ -1090,6 +1178,90 @@ dev_add_entry(char *name, devnode_t * parent, int type, devnode_type_t * typeinf
 }
 
 
+/*
+ * Function: devfs_make_node
+ *
+ * Purpose
+ *   Create a device node with the given pathname in the devfs namespace.
+ *
+ * Parameters:
+ *   dev 	- the dev_t value to associate
+ *   chrblk	- block or character device (DEVFS_CHAR or DEVFS_BLOCK)
+ *   uid, gid	- ownership
+ *   perms	- permissions
+ *   clone	- minor number cloning function
+ *   fmt, ...	- path format string with printf args to format the path name
+ * Returns:
+ *   A handle to a device node if successful, NULL otherwise.
+ */
+void *
+devfs_make_node_clone(dev_t dev, int chrblk, uid_t uid,
+		gid_t gid, int perms, int (*clone)(dev_t dev, int action),
+		const char *fmt, ...)
+{
+	devdirent_t *	new_dev = NULL;
+	devnode_t *	dnp;	/* devnode for parent directory */
+	devnode_type_t	typeinfo;
+
+	char *name, buf[256]; /* XXX */
+	const char *path;
+	int i;
+	va_list ap;
+
+
+	DEVFS_LOCK();
+
+	if (!devfs_ready) {
+		printf("devfs_make_node: not ready for devices!\n");
+		goto out;
+	}
+	if (chrblk != DEVFS_CHAR && chrblk != DEVFS_BLOCK)
+		goto out;
+
+	DEVFS_UNLOCK();
+
+	va_start(ap, fmt);
+	vsnprintf(buf, sizeof(buf), fmt, ap);
+	va_end(ap);
+
+	name = NULL;
+
+	for(i=strlen(buf); i>0; i--)
+		if(buf[i] == '/') {
+			name=&buf[i];
+			buf[i]=0;
+			break;
+		}
+
+	if (name) {
+		*name++ = '\0';
+		path = buf;
+	} else {
+		name = buf;
+		path = "/";
+	}
+	DEVFS_LOCK();
+
+	/* find/create directory path ie. mkdir -p */
+	if (dev_finddir(path, NULL, DEVFS_CREATE, &dnp) == 0) {
+	    typeinfo.dev = dev;
+	    if (dev_add_entry(name, dnp, 
+			      (chrblk == DEVFS_CHAR) ? DEV_CDEV : DEV_BDEV, 
+			      &typeinfo, NULL, NULL, &new_dev) == 0) {
+		new_dev->de_dnp->dn_gid = gid;
+		new_dev->de_dnp->dn_uid = uid;
+		new_dev->de_dnp->dn_mode |= perms;
+		new_dev->de_dnp->dn_clone = clone;
+		devfs_propogate(dnp->dn_typeinfo.Dir.myname, new_dev);
+	    }
+	}
+out:
+	DEVFS_UNLOCK();
+
+	return new_dev;
+}
+
+
 /*
  * Function: devfs_make_node
  *
@@ -1113,7 +1285,11 @@ devfs_make_node(dev_t dev, int chrblk, uid_t uid,
 	devnode_t *	dnp;	/* devnode for parent directory */
 	devnode_type_t	typeinfo;
 
-	char *name, *path, buf[256]; /* XXX */
+	char *name, buf[256]; /* XXX */
+	const char *path;
+#if CONFIG_MACF
+	char buff[sizeof(buf)];
+#endif
 	int i;
 	va_list ap;
 
@@ -1133,6 +1309,10 @@ devfs_make_node(dev_t dev, int chrblk, uid_t uid,
 	vsnprintf(buf, sizeof(buf), fmt, ap);
 	va_end(ap);
 
+#if CONFIG_MACF
+	bcopy(buf, buff, sizeof(buff));
+	buff[sizeof(buff)-1] = 0;
+#endif
 	name = NULL;
 
 	for(i=strlen(buf); i>0; i--)
@@ -1152,7 +1332,7 @@ devfs_make_node(dev_t dev, int chrblk, uid_t uid,
 	DEVFS_LOCK();
 
 	/* find/create directory path ie. mkdir -p */
-	if (dev_finddir(path, NULL, CREATE, &dnp) == 0) {
+	if (dev_finddir(path, NULL, DEVFS_CREATE, &dnp) == 0) {
 	    typeinfo.dev = dev;
 	    if (dev_add_entry(name, dnp, 
 			      (chrblk == DEVFS_CHAR) ? DEV_CDEV : DEV_BDEV, 
@@ -1160,6 +1340,11 @@ devfs_make_node(dev_t dev, int chrblk, uid_t uid,
 		new_dev->de_dnp->dn_gid = gid;
 		new_dev->de_dnp->dn_uid = uid;
 		new_dev->de_dnp->dn_mode |= perms;
+		new_dev->de_dnp->dn_clone = NULL;
+
+#if CONFIG_MACF
+		mac_devfs_label_associate_device(dev, new_dev->de_dnp, buff);
+#endif
 		devfs_propogate(dnp->dn_typeinfo.Dir.myname, new_dev);
 	    }
 	}
@@ -1215,11 +1400,11 @@ devfs_make_link(void *original, char *fmt, ...)
 	if (p) {
 	        *p++ = '\0';
 
-		if (dev_finddir(buf, NULL, CREATE, &dirnode)
+		if (dev_finddir(buf, NULL, DEVFS_CREATE, &dirnode)
 		    || dev_add_name(p, dirnode, NULL, orig->de_dnp, &new_dev))
 		        goto fail;
 	} else {
-	        if (dev_finddir("", NULL, CREATE, &dirnode)
+	        if (dev_finddir("", NULL, DEVFS_CREATE, &dirnode)
 		    || dev_add_name(buf, dirnode, NULL, orig->de_dnp, &new_dev))
 		        goto fail;
 	}
@@ -1230,4 +1415,3 @@ out:
 
 	return ((new_dev != NULL) ? 0 : -1);
 }
-
diff --git a/bsd/miscfs/devfs/devfs_vfsops.c b/bsd/miscfs/devfs/devfs_vfsops.c
index a1ed0909d..fc9b55f24 100644
--- a/bsd/miscfs/devfs/devfs_vfsops.c
+++ b/bsd/miscfs/devfs/devfs_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright 1997,1998 Julian Elischer.  All rights reserved.
@@ -47,6 +53,12 @@
  * devfs_vfsops.c
  *
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  * HISTORY
  *  Dieter Siegmund (dieter@apple.com) Wed Jul 14 13:37:59 PDT 1999
@@ -64,14 +76,21 @@
 #include <sys/mount_internal.h>
 #include <sys/malloc.h>
 
+#include <libkern/OSAtomic.h>
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 #include "devfs.h"
 #include "devfsdefs.h"
 
-static int devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, vfs_context_t context);
-static int devfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context);
+static int devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, vfs_context_t ctx);
+static int devfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx);
 
 static struct vfstable * devfs_vfsp = 0;
 extern int setup_kmem;
+__private_extern__ void devfs_setup_kmem(void);
 
 
 /*-
@@ -130,7 +149,7 @@ devfs_setup_kmem(void)
  */
 /*proto*/
 int
-devfs_mount(struct mount *mp, __unused vnode_t devvp, __unused user_addr_t data, vfs_context_t context)
+devfs_mount(struct mount *mp, __unused vnode_t devvp, __unused user_addr_t data, vfs_context_t ctx)
 {
 	struct devfsmount *devfs_mp_p;	/* devfs specific mount info */
 	int error;
@@ -187,14 +206,14 @@ devfs_mount(struct mount *mp, __unused vnode_t devvp, __unused user_addr_t data,
 	
 	bzero(mp->mnt_vfsstat.f_mntfromname, MAXPATHLEN);
 	bcopy("devfs",mp->mnt_vfsstat.f_mntfromname, 5);
-	(void)devfs_statfs(mp, &mp->mnt_vfsstat, context);
+	(void)devfs_statfs(mp, &mp->mnt_vfsstat, ctx);
 
 	return 0;
 }
 
 
 static int
-devfs_start(__unused struct mount *mp, __unused int flags, __unused vfs_context_t context)
+devfs_start(__unused struct mount *mp, __unused int flags, __unused vfs_context_t ctx)
 {
 	return 0;
 }
@@ -203,7 +222,7 @@ devfs_start(__unused struct mount *mp, __unused int flags, __unused vfs_context_
  *  Unmount the filesystem described by mp.
  */
 static int
-devfs_unmount( struct mount *mp, int mntflags, __unused vfs_context_t context)
+devfs_unmount( struct mount *mp, int mntflags, __unused vfs_context_t ctx)
 {
 	struct devfsmount *devfs_mp_p = (struct devfsmount *)mp->mnt_data;
 	int flags = 0;
@@ -233,20 +252,21 @@ devfs_unmount( struct mount *mp, int mntflags, __unused vfs_context_t context)
 
 /* return the address of the root vnode  in *vpp */
 static int
-devfs_root(struct mount *mp, struct vnode **vpp, vfs_context_t context)
+devfs_root(struct mount *mp, struct vnode **vpp, __unused vfs_context_t ctx)
 {
 	struct devfsmount *devfs_mp_p = (struct devfsmount *)(mp->mnt_data);
 	int error;
 
 	DEVFS_LOCK();
-	error = devfs_dntovn(devfs_mp_p->plane_root->de_dnp, vpp, context->vc_proc);
+	/* last parameter to devfs_dntovn() is ignored */
+	error = devfs_dntovn(devfs_mp_p->plane_root->de_dnp, vpp, NULL);
 	DEVFS_UNLOCK();
 
 	return error;
 }
 
 static int
-devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, __unused vfs_context_t context)
+devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, __unused vfs_context_t ctx)
 {
 	struct devfsmount *devfs_mp_p = (struct devfsmount *)mp->mnt_data;
 
@@ -273,7 +293,7 @@ devfs_statfs( struct mount *mp, struct vfsstatfs *sbp, __unused vfs_context_t co
 }
 
 static int
-devfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
+devfs_vfs_getattr(__unused mount_t mp, struct vfs_attr *fsap, __unused vfs_context_t ctx)
 {
 	VFSATTR_RETURN(fsap, f_objcount, devfs_stats.nodes);
 	VFSATTR_RETURN(fsap, f_maxobjcount, devfs_stats.nodes);
@@ -299,14 +319,14 @@ devfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 }
 
 static int
-devfs_sync(__unused struct mount *mp, __unused int waitfor, __unused vfs_context_t context)
+devfs_sync(__unused struct mount *mp, __unused int waitfor, __unused vfs_context_t ctx)
 {
     return (0);
 }
 
 
 static int
-devfs_vget(__unused struct mount *mp, __unused ino64_t ino, __unused struct vnode **vpp, __unused vfs_context_t context)
+devfs_vget(__unused struct mount *mp, __unused ino64_t ino, __unused struct vnode **vpp, __unused vfs_context_t ctx)
 {
 	return ENOTSUP;
 }
@@ -317,14 +337,14 @@ devfs_vget(__unused struct mount *mp, __unused ino64_t ino, __unused struct vnod
  */
 
 static int
-devfs_fhtovp (__unused struct mount *mp, __unused int fhlen, __unused unsigned char *fhp, __unused struct vnode **vpp, __unused vfs_context_t context)
+devfs_fhtovp (__unused struct mount *mp, __unused int fhlen, __unused unsigned char *fhp, __unused struct vnode **vpp, __unused vfs_context_t ctx)
 {
 	return (EINVAL);
 }
 
 
 static int
-devfs_vptofh (__unused struct vnode *vp, __unused int *fhlenp, __unused unsigned char *fhp, __unused vfs_context_t context)
+devfs_vptofh (__unused struct vnode *vp, __unused int *fhlenp, __unused unsigned char *fhp, __unused vfs_context_t ctx)
 {
 	return (EINVAL);
 }
@@ -332,7 +352,7 @@ devfs_vptofh (__unused struct vnode *vp, __unused int *fhlenp, __unused unsigned
 static int
 devfs_sysctl(__unused int *name, __unused u_int namelen, __unused user_addr_t oldp, 
              __unused size_t *oldlenp, __unused user_addr_t newp, 
-             __unused size_t newlen, __unused vfs_context_t context)
+             __unused size_t newlen, __unused vfs_context_t ctx)
 {
     return (ENOTSUP);
 }
@@ -351,20 +371,18 @@ devfs_kernel_mount(char * mntname)
 	int error;
 	struct nameidata nd;
 	struct vnode  * vp;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_kernel();
 
 	if (devfs_vfsp == NULL) {
 	    printf("devfs_kernel_mount: devfs_vfsp is NULL\n");
 	    return (EINVAL);
 	}
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
 
 	/*
 	 * Get vnode to be covered
 	 */
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
-	    CAST_USER_ADDR_T(mntname), &context);
+	    CAST_USER_ADDR_T(mntname), ctx);
 	if ((error = namei(&nd))) {
 	    printf("devfs_kernel_mount: failed to find directory '%s', %d", 
 		   mntname, error);
@@ -373,7 +391,7 @@ devfs_kernel_mount(char * mntname)
 	nameidone(&nd);
 	vp = nd.ni_vp;
 
-	if ((error = VNOP_FSYNC(vp, MNT_WAIT, &context))) {
+	if ((error = VNOP_FSYNC(vp, MNT_WAIT, ctx))) {
 	    printf("devfs_kernel_mount: vnop_fsync failed: %d\n", error);
 	    vnode_put(vp);
 	    return (error);
@@ -403,6 +421,9 @@ devfs_kernel_mount(char * mntname)
 	/* Initialize the default IO constraints */
 	mp->mnt_maxreadcnt = mp->mnt_maxwritecnt = MAXPHYS;
 	mp->mnt_segreadcnt = mp->mnt_segwritecnt = 32;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
 
 	mount_lock_init(mp);
 	TAILQ_INIT(&mp->mnt_vnodelist);
@@ -417,13 +438,17 @@ devfs_kernel_mount(char * mntname)
 	devfs_vfsp->vfc_64bitready = TRUE;
 	mp->mnt_flag = 0;
 	mp->mnt_flag |= devfs_vfsp->vfc_flags & MNT_VISFLAGMASK;
-	strncpy(mp->mnt_vfsstat.f_fstypename, devfs_vfsp->vfc_name, MFSTYPENAMELEN);
+	strlcpy(mp->mnt_vfsstat.f_fstypename, devfs_vfsp->vfc_name, MFSTYPENAMELEN);
 	vp->v_mountedhere = mp;
 	mp->mnt_vnodecovered = vp;
 	mp->mnt_vfsstat.f_owner = kauth_cred_getuid(kauth_cred_get());
 	(void) copystr(mntname, mp->mnt_vfsstat.f_mntonname, MAXPATHLEN - 1, 0);
+#if CONFIG_MACF
+	mac_mount_label_init(mp);
+	mac_mount_label_associate(ctx, mp);
+#endif
 
-	error = devfs_mount(mp, NULL, NULL, &context);
+	error = devfs_mount(mp, NULL, USER_ADDR_NULL, ctx);
 
 	if (error) {
 	    printf("devfs_kernel_mount: mount %s failed: %d", mntname, error);
@@ -432,6 +457,9 @@ devfs_kernel_mount(char * mntname)
 	    vfs_unbusy(mp);
 
 	    mount_lock_destroy(mp);
+#if CONFIG_MACF
+	    mac_mount_label_destroy(mp);
+#endif
 	    FREE_ZONE(mp, sizeof (struct mount), M_MOUNT);
 	    vnode_put(vp);
 	    return (error);
@@ -455,5 +483,7 @@ struct vfsops devfs_vfsops = {
 	devfs_fhtovp,
 	devfs_vptofh,
 	devfs_init,
-	devfs_sysctl
+	devfs_sysctl,
+	NULL,
+	{NULL}
 };
diff --git a/bsd/miscfs/devfs/devfs_vnops.c b/bsd/miscfs/devfs/devfs_vnops.c
index 40f719892..d9aedf798 100644
--- a/bsd/miscfs/devfs/devfs_vnops.c
+++ b/bsd/miscfs/devfs/devfs_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1997,1998 Julian Elischer.  All rights reserved.
@@ -66,6 +72,12 @@
  *  Dieter Siegmund (dieter@apple.com) Fri Sep 17 09:58:38 PDT 1999
  *  - update the mod/access times
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -87,6 +99,10 @@
 #include <sys/vm.h>
 #include <sys/uio_internal.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 #include "devfsdefs.h"
 
 static int devfs_update(struct vnode *vp, struct timeval *access,
@@ -157,8 +173,7 @@ retry:
 
 	*result_vnode = NULL; /* safe not sorry */ /*XXX*/
 
-	//if (dir_vnode->v_usecount == 0)
-	    //printf("devfs_lookup: dir had no refs ");
+	/*  okay to look at directory vnodes ourside devfs lock as they are not aliased */
 	dir_node = VTODN(dir_vnode);
 
 	/*
@@ -328,15 +343,20 @@ devfs_getattr(struct vnop_getattr_args *ap)
 	devnode_t *	file_node;
 	struct timeval now;
 
-	file_node = VTODN(vp);
 
 	DEVFS_LOCK();
+	file_node = VTODN(vp);
 
 	microtime(&now);
 	dn_times(file_node, &now, &now, &now);
 
 	VATTR_RETURN(vap, va_mode, file_node->dn_mode);
 
+	/*
+	 * Note: for DEV_CDEV and DEV_BDEV, we return the device from
+	 * the vp, not the file_node; if we getting information on a
+	 * cloning device, we want the cloned information, not the template.
+	 */
 	switch (file_node->dn_type)
 	{
 	case 	DEV_DIR:
@@ -344,11 +364,11 @@ devfs_getattr(struct vnop_getattr_args *ap)
 		vap->va_mode |= (S_IFDIR);
 		break;
 	case	DEV_CDEV:
-		VATTR_RETURN(vap, va_rdev, file_node->dn_typeinfo.dev);
+		VATTR_RETURN(vap, va_rdev, vp->v_rdev);
 		vap->va_mode |= (S_IFCHR);
 		break;
 	case	DEV_BDEV:
-		VATTR_RETURN(vap, va_rdev, file_node->dn_typeinfo.dev);
+		VATTR_RETURN(vap, va_rdev, vp->v_rdev);
 		vap->va_mode |= (S_IFBLK);
 		break;
 	case	DEV_SLNK:
@@ -405,15 +425,13 @@ devfs_setattr(struct vnop_setattr_args *ap)
 {
   	struct vnode *vp = ap->a_vp;
  	struct vnode_attr *vap = ap->a_vap;
-  	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
-  	struct proc *p = vfs_context_proc(ap->a_context);
   	int error = 0;
   	devnode_t *	file_node;
   	struct timeval atimeval, mtimeval;
   
-  	file_node = VTODN(vp);
-  
  	DEVFS_LOCK();
+
+  	file_node = VTODN(vp);
   	/*
   	 * Go through the fields and update if set.
   	 */
@@ -465,6 +483,29 @@ devfs_setattr(struct vnop_setattr_args *ap)
 	return error;
 }
 
+#if CONFIG_MACF
+static int
+devfs_setlabel(struct vnop_setlabel_args *ap)
+			/* struct vnop_setlabel_args {
+					struct vnodeop_desc *a_desc;
+					struct vnode *a_vp;
+					struct label *a_vl;
+				vfs_context_t a_context;
+				} */
+{
+	struct vnode *vp;
+	struct devnode *de;
+
+	vp = ap->a_vp;
+	de = VTODN(vp);
+
+	mac_vnode_label_update(ap->a_context, vp, ap->a_vl);
+	mac_devfs_label_update(vp->v_mount, de, vp);
+
+	return (0);
+}
+#endif
+
 static int
 devfs_read(struct vnop_read_args *ap)
         /* struct vnop_read_args {
@@ -500,11 +541,12 @@ devfs_close(struct vnop_close_args *ap)
 	} */
 {
     	struct vnode *	    	vp = ap->a_vp;
-	register devnode_t * 	dnp = VTODN(vp);
+	register devnode_t * 	dnp;
 	struct timeval now;
 
 	if (vnode_isinuse(vp, 1)) {
 	    DEVFS_LOCK();
+	    dnp = VTODN(vp);
 	    microtime(&now);
 	    dn_times(dnp, &now, &now, &now);
 	    DEVFS_UNLOCK();
@@ -521,12 +563,13 @@ devfsspec_close(struct vnop_close_args *ap)
 	} */
 {
     	struct vnode *	    	vp = ap->a_vp;
-	register devnode_t * 	dnp = VTODN(vp);
+	register devnode_t * 	dnp;
 	struct timeval now;
 
 	if (vnode_isinuse(vp, 1)) {
 	    DEVFS_LOCK();
 	    microtime(&now);
+	    dnp = VTODN(vp);
 	    dn_times(dnp, &now, &now, &now);
 	    DEVFS_UNLOCK();
 	}
@@ -599,23 +642,22 @@ devfs_remove(struct vnop_remove_args *ap)
 	struct vnode *vp = ap->a_vp;
 	struct vnode *dvp = ap->a_dvp;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
 	devnode_t *  tp;
 	devnode_t *  tdp;
 	devdirent_t * tnp;
 	int doingdirectory = 0;
 	int error = 0;
-	uid_t ouruid = kauth_cred_getuid(vfs_context_ucred(ctx));
 
 	/*
 	 * assume that the name is null terminated as they
 	 * are the end of the path. Get pointers to all our
 	 * devfs structures.
 	 */
+	DEVFS_LOCK();
+
 	tp = VTODN(vp);
 	tdp = VTODN(dvp);
 
-	DEVFS_LOCK();
 
 	tnp = dev_findname(tdp, cnp->cn_nameptr);
 
@@ -695,14 +737,15 @@ devfs_link(struct vnop_link_args *ap)
 	 * are the end of the path. Get pointers to all our
 	 * devfs structures.
 	 */
+	/* can lookup dnode safely for tdvp outside of devfs lock as it is not aliased */
 	tdp = VTODN(tdvp);
-	fp = VTODN(vp);
 	
 	if (tdvp->v_mount != vp->v_mount) {
 		return (EXDEV);
 	}
 	DEVFS_LOCK();
 
+	fp = VTODN(vp);
 
 	/***********************************
 	 * Start actually doing things.... *
@@ -882,7 +925,6 @@ devfs_rename(struct vnop_rename_args *ap)
 	 * We could do that as well but won't
  	 */
 	if (tp) {
-		int ouruid = kauth_cred_getuid(vfs_context_ucred(tcnp->cn_context));
 		/*
 		 * Target must be empty if a directory and have no links
 		 * to it. Also, ensure source and target are compatible
@@ -907,6 +949,73 @@ out:
 	return (error);
 }
 
+static int
+devfs_mkdir(struct vnop_mkdir_args *ap)
+        /*struct vnop_mkdir_args {
+                struct vnode *a_dvp;
+                struct vnode **a_vpp;
+                struct componentname *a_cnp;
+                struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+        } */
+{
+	struct componentname * cnp = ap->a_cnp;
+	vfs_context_t ctx = cnp->cn_context;
+	struct proc *p = vfs_context_proc(ctx);
+	int error = 0;
+	devnode_t * dir_p;
+	devdirent_t * nm_p;
+	devnode_t * dev_p;
+	struct vnode_attr *	vap = ap->a_vap;
+	struct vnode * * vpp = ap->a_vpp;
+
+	DEVFS_LOCK();
+
+	dir_p = VTODN(ap->a_dvp);
+	error = dev_add_entry(cnp->cn_nameptr, dir_p, DEV_DIR, 
+			      NULL, NULL, NULL, &nm_p);
+	if (error) {
+	    goto failure;
+	}
+	dev_p = nm_p->de_dnp;
+	dev_p->dn_uid = dir_p->dn_uid;
+	dev_p->dn_gid = dir_p->dn_gid;
+	dev_p->dn_mode = vap->va_mode;
+	dn_copy_times(dev_p, dir_p);
+
+	error = devfs_dntovn(dev_p, vpp, p);
+failure:
+	DEVFS_UNLOCK();
+
+	return error;
+}
+
+/*
+ * An rmdir is a special type of remove, which we already support; we wrap
+ * and reexpress the arguments to call devfs_remove directly.  The only
+ * different argument is flags, which we do not set, since it's ignored.
+ */
+static int
+devfs_rmdir(struct vnop_rmdir_args *ap)
+	/* struct vnop_rmdir_args {
+		struct vnode *a_dvp;
+		struct vnode *a_vp;
+		struct componentname *a_cnp;
+		vfs_context_t a_context;
+	} */
+{
+	struct vnop_remove_args ra;
+
+	ra.a_dvp = ap->a_dvp;
+	ra.a_vp = ap->a_vp;
+	ra.a_cnp = ap->a_cnp;
+	ra.a_flags = 0;		/* XXX */
+	ra.a_context = ap->a_context;
+
+	return devfs_remove(&ra);
+}
+
+
 static int
 devfs_symlink(struct vnop_symlink_args *ap)
         /*struct vnop_symlink_args {
@@ -929,11 +1038,13 @@ devfs_symlink(struct vnop_symlink_args *ap)
 	struct vnode_attr *	vap = ap->a_vap;
 	struct vnode * * vpp = ap->a_vpp;
 
-	dir_p = VTODN(ap->a_dvp);
 	typeinfo.Slnk.name = ap->a_target;
 	typeinfo.Slnk.namelen = strlen(ap->a_target);
 
 	DEVFS_LOCK();
+
+	dir_p = VTODN(ap->a_dvp);
+
 	error = dev_add_entry(cnp->cn_nameptr, dir_p, DEV_SLNK, 
 			      &typeinfo, NULL, NULL, &nm_p);
 	if (error) {
@@ -981,10 +1092,12 @@ devfs_mknod(struct vnop_mknod_args *ap)
 	if (!(vap->va_type == VBLK) && !(vap->va_type == VCHR)) {
 	        return (EINVAL); /* only support mknod of special files */
 	}
-	dir_p = VTODN(dvp);
 	typeinfo.dev = vap->va_rdev;
 
 	DEVFS_LOCK();
+
+	dir_p = VTODN(dvp);
+
 	error = dev_add_entry(cnp->cn_nameptr, dir_p, 
 			      (vap->va_type == VBLK) ? DEV_BDEV : DEV_CDEV,
 			      &typeinfo, NULL, NULL, &devent);
@@ -1026,7 +1139,7 @@ devfs_readdir(struct vnop_readdir_args *ap)
 	struct dirent dirent;
 	devnode_t * dir_node;
 	devdirent_t *	name_node;
-	char	*name;
+	const char *name;
 	int error = 0;
 	int reclen;
 	int nodenumber;
@@ -1099,7 +1212,7 @@ devfs_readdir(struct vnop_readdir_args *ap)
 		{
 			if (uio_resid(uio) < reclen) /* will it fit? */
 				break;
-			strcpy( dirent.d_name,name);
+			strlcpy(dirent.d_name, name, DEVMAXNAMESIZE);
 			if ((error = uiomove ((caddr_t)&dirent,
 					dirent.d_reclen, uio)) != 0)
 				break;
@@ -1151,10 +1264,12 @@ devfs_reclaim(struct vnop_reclaim_args *ap)
         } */
 {
     struct vnode *	vp = ap->a_vp;
-    devnode_t * 	dnp = VTODN(vp);
+    devnode_t * 	dnp;
     
     DEVFS_LOCK();
 
+    dnp = VTODN(vp);
+
     if (dnp) {
 	/* 
 	 * do the same as devfs_inactive in case it is not called
@@ -1197,7 +1312,7 @@ devs_vnop_pathconf(
 		*ap->a_retval = DEVMAXPATHSIZE - 1;	/* XXX nonconformant */
 		break;
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		break;
 	case _PC_NO_TRUNC:
 		*ap->a_retval = 0;
@@ -1282,8 +1397,8 @@ static struct vnodeopv_entry_desc devfs_vnodeop_entries[] = {
 	{ &vnop_remove_desc, (VOPFUNC)devfs_remove },		/* remove */
 	{ &vnop_link_desc, (VOPFUNC)devfs_link },		/* link */
 	{ &vnop_rename_desc, (VOPFUNC)devfs_rename },		/* rename */
-	{ &vnop_mkdir_desc, (VOPFUNC)err_mkdir },		/* mkdir */
-	{ &vnop_rmdir_desc, (VOPFUNC)err_rmdir },		/* rmdir */
+	{ &vnop_mkdir_desc, (VOPFUNC)devfs_mkdir },		/* mkdir */
+	{ &vnop_rmdir_desc, (VOPFUNC)devfs_rmdir },		/* rmdir */
 	{ &vnop_symlink_desc, (VOPFUNC)devfs_symlink },		/* symlink */
 	{ &vnop_readdir_desc, (VOPFUNC)devfs_readdir },		/* readdir */
 	{ &vnop_readlink_desc, (VOPFUNC)devfs_readlink },	/* readlink */
@@ -1299,6 +1414,9 @@ static struct vnodeopv_entry_desc devfs_vnodeop_entries[] = {
 	{ &vnop_blktooff_desc, (VOPFUNC)err_blktooff },		/* blktooff */
 	{ &vnop_offtoblk_desc, (VOPFUNC)err_offtoblk },		/* offtoblk */
 	{ &vnop_blockmap_desc, (VOPFUNC)err_blockmap },		/* blockmap */
+#if CONFIG_MACF
+	{ &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel },       /* setlabel */
+#endif
 	{ (struct vnodeop_desc*)NULL, (int(*)())NULL }
 };
 struct vnodeopv_desc devfs_vnodeop_opv_desc =
@@ -1342,8 +1460,10 @@ static struct vnodeopv_entry_desc devfs_spec_vnodeop_entries[] = {
 	{ &vnop_blktooff_desc, (VOPFUNC)spec_blktooff },	/* blktooff */
 	{ &vnop_blktooff_desc, (VOPFUNC)spec_offtoblk  },	/* blkofftoblk */
 	{ &vnop_blockmap_desc, (VOPFUNC)spec_blockmap },	/* blockmap */
+#if CONFIG_MACF
+	{ &vnop_setlabel_desc, (VOPFUNC)devfs_setlabel },	/* setlabel */
+#endif
 	{ (struct vnodeop_desc*)NULL, (int(*)())NULL }
 };
 struct vnodeopv_desc devfs_spec_vnodeop_opv_desc =
 	{ &devfs_spec_vnodeop_p, devfs_spec_vnodeop_entries };
-
diff --git a/bsd/miscfs/devfs/devfsdefs.h b/bsd/miscfs/devfs/devfsdefs.h
index b104d6927..701b8f43d 100644
--- a/bsd/miscfs/devfs/devfsdefs.h
+++ b/bsd/miscfs/devfs/devfsdefs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1997,1998 Julian Elischer.  All rights reserved.
@@ -46,6 +52,12 @@
  * 
  * devfsdefs.h
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * HISTORY
@@ -60,6 +72,8 @@
 
 #include  <sys/appleapiopts.h>
 
+#include <security/mac.h>
+
 #ifdef __APPLE_API_PRIVATE
 #define DEVMAXNAMESIZE 	32 		/* XXX */
 #define DEVMAXPATHSIZE 	128		/* XXX */
@@ -125,6 +139,8 @@ struct devnode
     int			dn_update;
     int			dn_access;
   int			dn_lflags;
+    int			(*dn_clone)(dev_t dev, int action); /* get minor # */
+    struct label *	dn_label;	/* security label */
 };
 
 #define	DN_BUSY			0x01
@@ -172,6 +188,7 @@ struct devfsmount
  */
 #include <sys/lock.h>
 #include <miscfs/devfs/devfs_proto.h>
+#include <libkern/OSAtomic.h>		/* required for OSAddAtomic() */
 
 //#define HIDDEN_MOUNTPOINT	1
 
@@ -187,53 +204,55 @@ struct devfsmount
 #define DEVFS_UNLOCK()	lck_mtx_unlock(&devfs_mutex)
 
 
-
+/*
+ * XXX all the (SInt32 *) casts below assume sizeof(int) == sizeof(long)
+ */
 static __inline__ void
-DEVFS_INCR_ENTRIES()
+DEVFS_INCR_ENTRIES(void)
 {
-    OSAddAtomic(1, &devfs_stats.entries);
+    OSAddAtomic(1, (SInt32 *)&devfs_stats.entries);
 }
 
 static __inline__ void
-DEVFS_DECR_ENTRIES()
+DEVFS_DECR_ENTRIES(void)
 {
-    OSAddAtomic(-1, &devfs_stats.entries);
+    OSAddAtomic(-1, (SInt32 *)&devfs_stats.entries);
 }
 
 static __inline__ void
-DEVFS_INCR_NODES()
+DEVFS_INCR_NODES(void)
 {
-    OSAddAtomic(1, &devfs_stats.nodes);
+    OSAddAtomic(1, (SInt32 *)&devfs_stats.nodes);
 }
 
 static __inline__ void
-DEVFS_DECR_NODES()
+DEVFS_DECR_NODES(void)
 {
-    OSAddAtomic(-1, &devfs_stats.nodes);
+    OSAddAtomic(-1, (SInt32 *)&devfs_stats.nodes);
 }
 
 static __inline__ void
-DEVFS_INCR_MOUNTS()
+DEVFS_INCR_MOUNTS(void)
 {
-    OSAddAtomic(1, &devfs_stats.mounts);
+    OSAddAtomic(1, (SInt32 *)&devfs_stats.mounts);
 }
 
 static __inline__ void
-DEVFS_DECR_MOUNTS()
+DEVFS_DECR_MOUNTS(void)
 {
-    OSAddAtomic(-1, &devfs_stats.mounts);
+    OSAddAtomic(-1, (SInt32 *)&devfs_stats.mounts);
 }
 
 static __inline__ void
 DEVFS_INCR_STRINGSPACE(int space)
 {
-    OSAddAtomic(space, &devfs_stats.stringspace);
+    OSAddAtomic(space, (SInt32 *)&devfs_stats.stringspace);
 }
 
 static __inline__ void
 DEVFS_DECR_STRINGSPACE(int space)
 {
-    OSAddAtomic(-space, &devfs_stats.stringspace);
+    OSAddAtomic(-space, (SInt32 *)&devfs_stats.stringspace);
 }
 
 static __inline__ void
diff --git a/bsd/miscfs/devfs/reproto.sh b/bsd/miscfs/devfs/reproto.sh
index 4012a1b36..97f9010c3 100644
--- a/bsd/miscfs/devfs/reproto.sh
+++ b/bsd/miscfs/devfs/reproto.sh
@@ -11,7 +11,7 @@ print PROTO "" .
 "/*\n" .
 " * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.\n" .
 " *\n" .
-" * \@APPLE_LICENSE_HEADER_START\@\n" .
+" * \@APPLE_OSREFERENCE_LICENSE_HEADER_START\@\n" .
 " *\n" .
 " * The contents of this file constitute Original Code as defined in and\n" .
 " * are subject to the Apple Public Source License Version 1.1 (the\n" .
@@ -27,7 +27,7 @@ print PROTO "" .
 " * License for the specific language governing rights and limitations\n" .
 " * under the License.\n" .
 " *\n" .
-" * \@APPLE_LICENSE_HEADER_END\@\n" .
+" * \@APPLE_OSREFERENCE_LICENSE_HEADER_END\@\n" .
 " */\n"; 
 
 print PROTO "/* THIS FILE HAS BEEN PRODUCED AUTOMATICALLY */\n";
diff --git a/bsd/miscfs/fdesc/Makefile b/bsd/miscfs/fdesc/Makefile
index 80e84ca5a..49b06dbdb 100644
--- a/bsd/miscfs/fdesc/Makefile
+++ b/bsd/miscfs/fdesc/Makefile
@@ -28,12 +28,10 @@ INSTALL_MI_LIST	= ${DATAFILES}
 
 INSTALL_MI_DIR = miscfs/fdesc
 
-EXPORT_MI_LIST	= ${DATAFILES}
+EXPORT_MI_LIST	= ${DATAFILES} ${PRIVATE_DATAFILES}
 
 EXPORT_MI_DIR =	miscfs/fdesc
 
-INSTALL_MI_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
-
 include $(MakeInc_rule)
 include $(MakeInc_dir)
 
diff --git a/bsd/miscfs/fdesc/fdesc.h b/bsd/miscfs/fdesc/fdesc.h
index b4141e2c1..8cb66f68b 100644
--- a/bsd/miscfs/fdesc/fdesc.h
+++ b/bsd/miscfs/fdesc/fdesc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -86,12 +92,12 @@ typedef enum {
 } fdntype;
 
 struct fdescnode {
-	LIST_ENTRY(fdescnode) fd_hash;	/* Hash list */
-	struct vnode	*fd_vnode;	/* Back ptr to vnode */
+	LIST_ENTRY(fdescnode)	fd_hash;	/* Hash list */
+	struct vnode		*fd_vnode;	/* Back ptr to vnode */
 	fdntype			fd_type;	/* Type of this node */
 	unsigned		fd_fd;		/* Fd to be dup'ed */
-	char			*fd_link;	/* Link to fd/n */
-	int				fd_ix;		/* filesystem index */
+	const char		*fd_link;	/* Link to fd/n */
+	int			fd_ix;		/* filesystem index */
 };
 
 #define VFSTOFDESC(mp)	((struct fdescmount *)((mp)->mnt_data))
diff --git a/bsd/miscfs/fdesc/fdesc_vfsops.c b/bsd/miscfs/fdesc/fdesc_vfsops.c
index b0173ec44..9d419783c 100644
--- a/bsd/miscfs/fdesc/fdesc_vfsops.c
+++ b/bsd/miscfs/fdesc/fdesc_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,9 +64,8 @@
  *	@(#)fdesc_vfsops.c	8.10 (Berkeley) 5/14/95
  *
  */
-
 /*
- * /dev/fd Filesystem
+* /dev/fd Filesystem
  */
 
 #include <sys/param.h>
@@ -79,8 +84,8 @@
 /*
  * Mount the per-process file descriptors (/dev/fd)
  */
-int
-fdesc_mount(struct mount *mp, vnode_t devvp, __unused user_addr_t data, vfs_context_t context)
+static int
+fdesc_mount(struct mount *mp, __unused vnode_t devvp, __unused user_addr_t data, __unused vfs_context_t context)
 {
 	int error = 0;
 	struct fdescmount *fmp;
@@ -114,20 +119,14 @@ fdesc_mount(struct mount *mp, vnode_t devvp, __unused user_addr_t data, vfs_cont
 	return (0);
 }
 
-int
-fdesc_start(mp, flags, context)
-	struct mount *mp;
-	int flags;
-	vfs_context_t context;
+static int
+fdesc_start(__unused struct mount *mp, __unused int flags, __unused vfs_context_t context)
 {
 	return (0);
 }
 
-int
-fdesc_unmount(mp, mntflags, context)
-	struct mount *mp;
-	int mntflags;
-	vfs_context_t context;
+static int
+fdesc_unmount(struct mount *mp, int mntflags, __unused vfs_context_t context)
 {
 	int error;
 	int flags = 0;
@@ -156,16 +155,13 @@ fdesc_unmount(mp, mntflags, context)
 	 * Finally, throw away the fdescmount structure
 	 */
 	_FREE(mp->mnt_data, M_UFSMNT);	/* XXX */
-	mp->mnt_data = 0;
+	mp->mnt_data = NULL;
 
 	return (0);
 }
 
 int
-fdesc_root(mp, vpp, context)
-	struct mount *mp;
-	struct vnode **vpp;
-	vfs_context_t context;
+fdesc_root(struct mount *mp, struct vnode **vpp, __unused vfs_context_t context)
 {
 	struct vnode *vp;
 
@@ -178,13 +174,14 @@ fdesc_root(mp, vpp, context)
 	return (0);
 }
 
+#if 0
+/*
+ * XXX commented out in mount.h
+ */
 int
-fdesc_statfs(mp, sbp, context)
-	struct mount *mp;
-	struct vfsstatfs *sbp;
-	vfs_context_t context;
+fdesc_statfs(__unused struct mount *mp, struct vfsstatfs *sbp, vfs_context_t context)
 {
-	struct proc *p = vfs_context_proc(context);
+	proc_t p = vfs_context_proc(context);
 	struct filedesc *fdp;
 	int lim;
 	int i;
@@ -224,9 +221,10 @@ fdesc_statfs(mp, sbp, context)
 
 	return (0);
 }
+#endif	/* 0 */
 
 static int
-fdesc_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
+fdesc_vfs_getattr(__unused mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 {
 	VFSATTR_RETURN(fsap, f_bsize, DEV_BSIZE);
 	VFSATTR_RETURN(fsap, f_iosize, DEV_BSIZE);
@@ -240,7 +238,7 @@ fdesc_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 	    VFSATTR_IS_ACTIVE(fsap, f_files) ||
 	    VFSATTR_IS_ACTIVE(fsap, f_ffree))
 	{
-		struct proc *p = vfs_context_proc(context);
+		proc_t p = vfs_context_proc(context);
 		struct filedesc *fdp;
 		int lim;
 		int i;
@@ -278,11 +276,8 @@ fdesc_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 	return 0;
 }
 
-int
-fdesc_sync(mp, waitfor, context)
-	struct mount *mp;
-	int waitfor;
-	vfs_context_t context;
+static int
+fdesc_sync(__unused struct mount *mp, __unused int waitfor, __unused vfs_context_t context)
 {
 
 	return (0);
@@ -300,10 +295,13 @@ struct vfsops fdesc_vfsops = {
 	fdesc_root,
 	NULL, 			/* quotactl */
 	fdesc_vfs_getattr,
+/*	fdesc_statfs,	XXX commented out in mount.h */
 	fdesc_sync,
 	fdesc_vget,
 	fdesc_fhtovp,
 	fdesc_vptofh,
 	fdesc_init,
-	fdesc_sysctl
+	fdesc_sysctl,
+	NULL,
+	{NULL}
 };
diff --git a/bsd/miscfs/fdesc/fdesc_vnops.c b/bsd/miscfs/fdesc/fdesc_vnops.c
index 185a74c61..68ff35dfa 100644
--- a/bsd/miscfs/fdesc/fdesc_vnops.c
+++ b/bsd/miscfs/fdesc/fdesc_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -87,6 +93,9 @@
 #include <vfs/vfs_support.h>
 #include <pexpert/pexpert.h>
 
+/* XXX should be prototyped in header for here, kern_descrip.c */
+extern int soo_stat(struct socket *so, void *ub, int isstat64);
+
 #define FDL_WANT	0x01
 #define FDL_LOCKED	0x02
 static int fdcache_lock;
@@ -119,12 +128,8 @@ fdesc_init(__unused struct vfsconf *vfsp)
 }
 
 int
-fdesc_allocvp(ftype, ix, mp, vpp, vtype)
-	fdntype ftype;
-	int ix;
-	struct mount *mp;
-	struct vnode **vpp;
-	enum vtype vtype;
+fdesc_allocvp(fdntype ftype, int ix, struct mount *mp, struct vnode **vpp,
+	      enum vtype vtype)
 {
 	struct fdhashhead *fc;
 	struct fdescnode *fd;
@@ -171,9 +176,9 @@ loop:
 	vfsp.vnfs_mp = mp;
 	vfsp.vnfs_vtype = vtype;
 	vfsp.vnfs_str = "fdesc";
-	vfsp.vnfs_dvp = 0;
+	vfsp.vnfs_dvp = NULL;
 	vfsp.vnfs_fsnode = fd;
-	vfsp.vnfs_cnp = 0;
+	vfsp.vnfs_cnp = NULL;
 	vfsp.vnfs_vops = fdesc_vnodeop_p;
 	vfsp.vnfs_rdev = 0;
 	vfsp.vnfs_filesize = 0;
@@ -193,7 +198,7 @@ loop:
 	fd->fd_vnode = *vpp;
 	fd->fd_type = ftype;
 	fd->fd_fd = -1;
-	fd->fd_link = 0;
+	fd->fd_link = NULL;
 	fd->fd_ix = ix;
 	LIST_INSERT_HEAD(fc, fd, fd_hash);
 
@@ -213,13 +218,7 @@ out:
  * ndp is the name to locate in that directory...
  */
 int
-fdesc_lookup(ap)
-	struct vnop_lookup_args /* {
-		struct vnode * a_dvp;
-		struct vnode ** a_vpp;
-		struct componentname * a_cnp;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_lookup(struct vnop_lookup_args *ap)
 {
 	struct vnode **vpp = ap->a_vpp;
 	struct vnode *dvp = ap->a_dvp;
@@ -230,7 +229,7 @@ fdesc_lookup(ap)
 	int fd;
 	int error;
 	struct vnode *fvp;
-	char *ln;
+	const char *ln;
 
 	if (cnp->cn_namelen == 1 && *pname == '.') {
 		*vpp = dvp;
@@ -332,31 +331,33 @@ bad:;
 }
 
 int
-fdesc_open(ap)
-	struct vnop_open_args /* {
-		struct vnode *a_vp;
-		int  a_mode;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_open(struct vnop_open_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
-	struct proc *p = vfs_context_proc(ap->a_context);
+	thread_t thr = vfs_context_thread(ap->a_context);
+	uthread_t uu;
 	int error = 0;
 
+	if (thr == NULL)
+		return (EINVAL);
+
+	uu = get_bsdthread_info(thr);
+
 	switch (VTOFDESC(vp)->fd_type) {
 	case Fdesc:
 		/*
-		 * XXX Kludge: set p->p_dupfd to contain the value of the
+		 * XXX Kludge: set uu->uu_dupfd to contain the value of the
 		 * the file descriptor being sought for duplication. The error 
 		 * return ensures that the vnode for this device will be
 		 * released by vn_open. Open will detect this special error and
 		 * take the actions in dupfdopen.  Other callers of vn_open or
 		 * vnop_open will simply report the error.
 		 */
-		p->p_dupfd = VTOFDESC(vp)->fd_fd;	/* XXX */
+		uu->uu_dupfd = VTOFDESC(vp)->fd_fd;	/* XXX */
 		error = ENODEV;
 		break;
-
+	default:	/* Froot / Fdevfd / Flink */
+		break;
 	}
 
 	return (error);
@@ -374,7 +375,7 @@ fdesc_attr(int fd, struct vnode_attr *vap, vfs_context_t a_context)
 		return (error);
 	switch (fp->f_fglob->fg_type) {
 	case DTYPE_VNODE:
-		if(error = vnode_getwithref((struct vnode *) fp->f_fglob->fg_data)) {
+		if((error = vnode_getwithref((struct vnode *) fp->f_fglob->fg_data)) != 0) {
 			break;
 		}
 		if ((error = vnode_authorize((struct vnode *)fp->f_fglob->fg_data,
@@ -396,10 +397,12 @@ fdesc_attr(int fd, struct vnode_attr *vap, vfs_context_t a_context)
 
 	case DTYPE_SOCKET:
 	case DTYPE_PIPE:
-	        if (fp->f_fglob->fg_type == DTYPE_SOCKET)
-		        error = soo_stat((struct socket *)fp->f_fglob->fg_data, &stb);
+#if SOCKETS
+		if (fp->f_fglob->fg_type == DTYPE_SOCKET)
+			error = soo_stat((struct socket *)fp->f_fglob->fg_data, (void *)&stb, 0);
 		else
-		        error = pipe_stat((struct socket *)fp->f_fglob->fg_data, &stb);
+#endif /* SOCKETS */
+		error = pipe_stat((struct pipe *)fp->f_fglob->fg_data, (void *)&stb, 0);
 
 		if (error == 0) {
 			if (fp->f_fglob->fg_type == DTYPE_SOCKET)
@@ -434,12 +437,7 @@ fdesc_attr(int fd, struct vnode_attr *vap, vfs_context_t a_context)
 }
 
 int
-fdesc_getattr(ap)
-	struct vnop_getattr_args /* {
-		struct vnode *a_vp;
-		struct vnode_attr *a_vap;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_getattr(struct vnop_getattr_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 	struct vnode_attr *vap = ap->a_vap;
@@ -501,12 +499,7 @@ fdesc_getattr(ap)
 }
 
 int
-fdesc_setattr(ap)
-	struct vnop_setattr_args /* {
-		struct vnode *a_vp;
-		struct vnode_attr *a_vap;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_setattr(struct vnop_setattr_args *ap)
 {
 	struct fileproc *fp;
 	unsigned fd;
@@ -572,15 +565,7 @@ static struct dirtmp {
 };
 
 int
-fdesc_readdir(ap)
-	struct vnop_readdir_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int a_flags;
-		int *a_eofflag;
-		int *a_numdirent;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_readdir(struct vnop_readdir_args *ap)
 {
 	struct uio *uio = ap->a_uio;
 	struct proc *p = current_proc();
@@ -657,7 +642,8 @@ fdesc_readdir(ap)
 
 			bzero((caddr_t) dp, UIO_MX);
 
-			dp->d_namlen = sprintf(dp->d_name, "%d", i);
+			dp->d_namlen = snprintf(dp->d_name, sizeof(dp->d_name),
+						"%d", i);
 			dp->d_reclen = UIO_MX;
 			dp->d_type = DT_UNKNOWN;
 			dp->d_fileno = i + FD_STDIN;
@@ -676,12 +662,7 @@ fdesc_readdir(ap)
 }
 
 int
-fdesc_readlink(ap)
-	struct vnop_readlink_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_readlink(struct vnop_readlink_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 	int error;
@@ -690,7 +671,7 @@ fdesc_readlink(ap)
 		return (EPERM);
 
 	if (VTOFDESC(vp)->fd_type == Flink) {
-		char *ln = VTOFDESC(vp)->fd_link;
+		const char *ln = VTOFDESC(vp)->fd_link;
 		error = uiomove(ln, strlen(ln), ap->a_uio);
 	} else {
 		error = ENOTSUP;
@@ -725,11 +706,7 @@ fdesc_select(__unused struct vnop_select_args *ap)
 }
 
 int
-fdesc_inactive(ap)
-	struct vnop_inactive_args /* {
-		struct vnode *a_vp;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_inactive(struct vnop_inactive_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 
@@ -742,18 +719,14 @@ fdesc_inactive(ap)
 }
 
 int
-fdesc_reclaim(ap)
-	struct vnop_reclaim_args /* {
-		struct vnode *a_vp;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_reclaim(struct vnop_reclaim_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 	struct fdescnode *fd = VTOFDESC(vp);
 
 	LIST_REMOVE(fd, fd_hash);
 	FREE(vp->v_data, M_TEMP);
-	vp->v_data = 0;
+	vp->v_data = NULL;
 
 	return (0);
 }
@@ -762,13 +735,7 @@ fdesc_reclaim(ap)
  * Return POSIX pathconf information applicable to special devices.
  */
 int
-fdesc_pathconf(ap)
-	struct vnop_pathconf_args /* {
-		struct vnode *a_vp;
-		int a_name;
-		int *a_retval;
-		vfs_context_t a_context;
-	} */ *ap;
+fdesc_pathconf(struct vnop_pathconf_args *ap)
 {
 
 	switch (ap->a_name) {
@@ -785,7 +752,7 @@ fdesc_pathconf(ap)
 		*ap->a_retval = PIPE_BUF;
 		return (0);
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		return (0);
 	case _PC_VDISABLE:
 		*ap->a_retval = _POSIX_VDISABLE;
@@ -796,15 +763,6 @@ fdesc_pathconf(ap)
 	/* NOTREACHED */
 }
 
-
-/*void*/
-int
-fdesc_vfree(__unused struct vnop_vfree_args *ap)
-{
-
-	return (0);
-}
-
 /*
  * /dev/fd "should never get here" operation
  */
diff --git a/bsd/miscfs/fifofs/fifo.h b/bsd/miscfs/fifofs/fifo.h
index a083b5948..97cb8eb34 100644
--- a/bsd/miscfs/fifofs/fifo.h
+++ b/bsd/miscfs/fifofs/fifo.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/miscfs/fifofs/fifo_vnops.c b/bsd/miscfs/fifofs/fifo_vnops.c
index 72358ae91..40fb3727a 100644
--- a/bsd/miscfs/fifofs/fifo_vnops.c
+++ b/bsd/miscfs/fifofs/fifo_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -73,8 +79,8 @@
 
 #define VOPFUNC int (*)(void *)
 
-extern int	soo_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p);
-extern int	soo_select(struct fileproc *fp, int which, void * wql, struct proc *p);
+extern int	soo_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, vfs_context_t ctx);
+extern int	soo_select(struct fileproc *fp, int which, void * wql, vfs_context_t ctx);
 
 int (**fifo_vnodeop_p)(void *);
 struct vnodeopv_entry_desc fifo_vnodeop_entries[] = {
@@ -124,13 +130,7 @@ struct vnodeopv_desc fifo_vnodeop_opv_desc =
  */
 /* ARGSUSED */
 int
-fifo_lookup(ap)
-	struct vnop_lookup_args /* {
-		struct vnode * a_dvp;
-		struct vnode ** a_vpp;
-		struct componentname * a_cnp;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_lookup(struct vnop_lookup_args *ap)
 {
 	
 	*ap->a_vpp = NULL;
@@ -143,12 +143,7 @@ fifo_lookup(ap)
  */
 /* ARGSUSED */
 int
-fifo_open(ap)
-	struct vnop_open_args /* {
-		struct vnode *a_vp;
-		int  a_mode;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_open(struct vnop_open_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
 	struct fifoinfo *fip;
@@ -167,7 +162,7 @@ retry:
 	if ((fip->fi_flags & FIFO_CREATED) == 0) {
 		if (fip->fi_flags & FIFO_INCREATE) {
 			fip->fi_flags |= FIFO_CREATEWAIT;	
-			error = msleep(&fip->fi_flags, &vp->v_lock, PRIBIO | PCATCH, "fifocreatewait", 0);
+			error = msleep(&fip->fi_flags, &vp->v_lock, PRIBIO | PCATCH, "fifocreatewait", NULL);
 			if (error) {
 				vnode_unlock(vp);
 				return(error);
@@ -248,7 +243,7 @@ retry:
 	if ((ap->a_mode & FREAD) && (ap->a_mode & O_NONBLOCK) == 0) {
 		if (fip->fi_writers == 0) {
 			error = msleep((caddr_t)&fip->fi_readers, &vp->v_lock,
-					PCATCH | PSOCK, "fifoor", 0);
+					PCATCH | PSOCK, "fifoor", NULL);
 			if (error)
 				goto bad;
 			if (fip->fi_readers == 1) {
@@ -266,7 +261,7 @@ retry:
 		} else {
 			if (fip->fi_readers == 0) {
 				error = msleep((caddr_t)&fip->fi_writers,&vp->v_lock,
-						PCATCH | PSOCK, "fifoow", 0);
+						PCATCH | PSOCK, "fifoow", NULL);
 				if (error)
 					goto bad;
 				if (fip->fi_writers == 1) {
@@ -302,13 +297,7 @@ bad1:
  * Vnode op for read
  */
 int
-fifo_read(ap)
-	struct vnop_read_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_read(struct vnop_read_args *ap)
 {
 	struct uio *uio = ap->a_uio;
 	struct socket *rso = ap->a_vp->v_fifoinfo->fi_readsock;
@@ -363,13 +352,7 @@ fifo_read(ap)
  * Vnode op for write
  */
 int
-fifo_write(ap)
-	struct vnop_write_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_write(struct vnop_write_args *ap)
 {
 	struct socket *wso = ap->a_vp->v_fifoinfo->fi_writesock;
 	int error;
@@ -378,7 +361,7 @@ fifo_write(ap)
 	if (ap->a_uio->uio_rw != UIO_WRITE)
 		panic("fifo_write mode");
 #endif
-	error = sosend(wso, (struct sockaddr *)0, ap->a_uio, 0,
+	error = sosend(wso, (struct sockaddr *)0, ap->a_uio, NULL,
 		       (struct mbuf *)0, (ap->a_ioflag & IO_NDELAY) ? MSG_NBIO : 0);
 
 	return (error);
@@ -388,16 +371,8 @@ fifo_write(ap)
  * Device ioctl operation.
  */
 int
-fifo_ioctl(ap)
-	struct vnop_ioctl_args /* {
-		struct vnode *a_vp;
-		int  a_command;
-		caddr_t  a_data;
-		int  a_fflag;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_ioctl(struct vnop_ioctl_args *ap)
 {
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct fileproc filetmp;
 	struct fileglob filefg;
 	int error;
@@ -408,13 +383,13 @@ fifo_ioctl(ap)
 	filetmp.f_fglob = &filefg;
 	if (ap->a_fflag & FREAD) {
 		filetmp.f_fglob->fg_data = (caddr_t)ap->a_vp->v_fifoinfo->fi_readsock;
-		error = soo_ioctl(&filetmp, ap->a_command, ap->a_data, p);
+		error = soo_ioctl(&filetmp, ap->a_command, ap->a_data, ap->a_context);
 		if (error)
 			return (error);
 	}
 	if (ap->a_fflag & FWRITE) {
 		filetmp.f_fglob->fg_data = (caddr_t)ap->a_vp->v_fifoinfo->fi_writesock;
-		error = soo_ioctl(&filetmp, ap->a_command, ap->a_data, p);
+		error = soo_ioctl(&filetmp, ap->a_command, ap->a_data, ap->a_context);
 		if (error)
 			return (error);
 	}
@@ -422,16 +397,8 @@ fifo_ioctl(ap)
 }
 
 int
-fifo_select(ap)
-	struct vnop_select_args /* {
-		struct vnode *a_vp;
-		int  a_which;
-		int  a_fflags;
-		void * a_wql;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_select(struct vnop_select_args *ap)
 {
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct fileproc filetmp;
 	struct fileglob filefg;
 	int ready;
@@ -440,13 +407,13 @@ fifo_select(ap)
 	filetmp.f_fglob = &filefg;
 	if (ap->a_which & FREAD) {
 		filetmp.f_fglob->fg_data = (caddr_t)ap->a_vp->v_fifoinfo->fi_readsock;
-		ready = soo_select(&filetmp, ap->a_which, ap->a_wql, p);
+		ready = soo_select(&filetmp, ap->a_which, ap->a_wql, ap->a_context);
 		if (ready)
 			return (ready);
 	}
 	if (ap->a_which & FWRITE) {
 		filetmp.f_fglob->fg_data = (caddr_t)ap->a_vp->v_fifoinfo->fi_writesock;
-		ready = soo_select(&filetmp, ap->a_which, ap->a_wql, p);
+		ready = soo_select(&filetmp, ap->a_which, ap->a_wql, ap->a_context);
 		if (ready)
 			return (ready);
 	}
@@ -464,12 +431,7 @@ fifo_inactive(__unused struct vnop_inactive_args *ap)
  * Device close routine
  */
 int
-fifo_close(ap)
-	struct vnop_close_args /* {
-		struct vnode *a_vp;
-		int  a_fflag;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_close(struct vnop_close_args *ap)
 {
 	return fifo_close_internal(ap->a_vp, ap->a_fflag, ap->a_context, 0);
 }
@@ -477,7 +439,7 @@ fifo_close(ap)
 int
 fifo_close_internal(vnode_t vp, int fflag, __unused vfs_context_t context, int locked)
 {
-	register struct fifoinfo *fip = vp->v_fifoinfo;
+	struct fifoinfo *fip = vp->v_fifoinfo;
 	int error1, error2;
 	struct socket *rso;
 	struct socket *wso;
@@ -525,8 +487,8 @@ fifo_close_internal(vnode_t vp, int fflag, __unused vfs_context_t context, int l
 
 	wso = fip->fi_writesock;
 	rso = fip->fi_readsock;
-	fip->fi_readsock = 0;
-	fip->fi_writesock = 0;
+	fip->fi_readsock = NULL;
+	fip->fi_writesock = NULL;
 	fip->fi_flags &= ~FIFO_CREATED;
 	if (!locked)
 		vnode_unlock(vp);
@@ -538,33 +500,26 @@ fifo_close_internal(vnode_t vp, int fflag, __unused vfs_context_t context, int l
 	return (error2);
 }
 
-
+#if !CONFIG_NO_PRINTF_STRINGS
 /*
  * Print out internal contents of a fifo vnode.
  */
 void
-fifo_printinfo(vp)
-	struct vnode *vp;
+fifo_printinfo(struct vnode *vp)
 {
-	register struct fifoinfo *fip = vp->v_fifoinfo;
+	struct fifoinfo *fip = vp->v_fifoinfo;
 
-	printf(", fifo with %d readers and %d writers",
+	printf(", fifo with %ld readers and %ld writers",
 		fip->fi_readers, fip->fi_writers);
 }
+#endif /* !CONFIG_NO_PRINTF_STRINGS */
 
 /*
  * Return POSIX pathconf information applicable to fifo's.
  */
 int
-fifo_pathconf(ap)
-	struct vnop_pathconf_args /* {
-		struct vnode *a_vp;
-		int a_name;
-		int *a_retval;
-		vfs_context_t a_context;
-	} */ *ap;
+fifo_pathconf(struct vnop_pathconf_args *ap)
 {
-
 	switch (ap->a_name) {
 	case _PC_LINK_MAX:
 		*ap->a_retval = LINK_MAX;
@@ -573,7 +528,7 @@ fifo_pathconf(ap)
 		*ap->a_retval = PIPE_BUF;
 		return (0);
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		return (0);
 	default:
 		return (EINVAL);
diff --git a/bsd/miscfs/nullfs/null.h b/bsd/miscfs/nullfs/null.h
index 734b1d772..3209be3d9 100644
--- a/bsd/miscfs/nullfs/null.h
+++ b/bsd/miscfs/nullfs/null.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -79,7 +85,6 @@ struct null_mount {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with null_args
  */
-/* LP64todo - should this move? */
 struct user_null_args {
 	user_addr_t		target;	/* Target of loopback  */
 };
diff --git a/bsd/miscfs/nullfs/null_subr.c b/bsd/miscfs/nullfs/null_subr.c
index cd7f6618e..c97977a48 100644
--- a/bsd/miscfs/nullfs/null_subr.c
+++ b/bsd/miscfs/nullfs/null_subr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/miscfs/nullfs/null_vfsops.c b/bsd/miscfs/nullfs/null_vfsops.c
index d916c8cd8..e81c64059 100644
--- a/bsd/miscfs/nullfs/null_vfsops.c
+++ b/bsd/miscfs/nullfs/null_vfsops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/miscfs/nullfs/null_vnops.c b/bsd/miscfs/nullfs/null_vnops.c
index ae4e8db1b..4b2fb2bbf 100644
--- a/bsd/miscfs/nullfs/null_vnops.c
+++ b/bsd/miscfs/nullfs/null_vnops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/miscfs/specfs/spec_vnops.c b/bsd/miscfs/specfs/spec_vnops.c
index cf66f74c0..752afe8df 100644
--- a/bsd/miscfs/specfs/spec_vnops.c
+++ b/bsd/miscfs/specfs/spec_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -73,11 +79,17 @@
 #include <sys/malloc.h>
 #include <sys/disk.h>
 #include <sys/uio_internal.h>
+#include <sys/resource.h>
 #include <miscfs/specfs/specdev.h>
 #include <vfs/vfs_support.h>
 
 #include <sys/kdebug.h>
 
+/* XXX following three prototypes should be in a header file somewhere */
+extern int	isdisk(dev_t dev, int type);
+extern dev_t	chrtoblk(dev_t dev);
+extern int	iskmemdev(dev_t dev);
+
 struct vnode *speclisth[SPECHSZ];
 
 /* symbolic sleep message strings for devices */
@@ -142,13 +154,7 @@ static void set_blocksize(vnode_t, dev_t);
  * Trivial lookup routine that always fails.
  */
 int
-spec_lookup(ap)
-	struct vnop_lookup_args /* {
-		struct vnode *a_dvp;
-		struct vnode **a_vpp;
-		struct componentname *a_cnp;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_lookup(struct vnop_lookup_args *ap)
 {
 
 	*ap->a_vpp = NULL;
@@ -195,12 +201,7 @@ set_fsblocksize(struct vnode *vp)
  * Open a special file.
  */
 int
-spec_open(ap)
-	struct vnop_open_args /* {
-		struct vnode *a_vp;
-		int  a_mode;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_open(struct vnop_open_args *ap)
 {
 	struct proc *p = vfs_context_proc(ap->a_context);
 	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
@@ -313,16 +314,10 @@ spec_open(ap)
  * Vnode op for read
  */
 int
-spec_read(ap)
-	struct vnop_read_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_read(struct vnop_read_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
-	register struct uio *uio = ap->a_uio;
+	struct vnode *vp = ap->a_vp;
+	struct uio *uio = ap->a_uio;
 	struct buf *bp;
 	daddr64_t bn, nextbn;
 	long bsize, bscale;
@@ -387,7 +382,7 @@ spec_read(ap)
 			// LP64todo - fix this!
 			n = min((unsigned)(n  - on), uio_resid(uio));
 
-			error = uiomove((char *)buf_dataptr(bp) + on, n, uio);
+			error = uiomove((char *)0 + buf_dataptr(bp) + on, n, uio);
 			if (n + on == bsize)
 				buf_markaged(bp);
 			buf_brelse(bp);
@@ -406,23 +401,17 @@ spec_read(ap)
  * Vnode op for write
  */
 int
-spec_write(ap)
-	struct vnop_write_args /* {
-		struct vnode *a_vp;
-		struct uio *a_uio;
-		int  a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_write(struct vnop_write_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
-	register struct uio *uio = ap->a_uio;
+	struct vnode *vp = ap->a_vp;
+	struct uio *uio = ap->a_uio;
 	struct buf *bp;
 	daddr64_t bn;
 	int bsize, blkmask, bscale;
-	register int io_sync;
-	register int io_size;
+	int io_sync;
+	int io_size;
 	int devBlockSize=0;
-	register int n, on;
+	int n, on;
 	int error = 0;
 	dev_t dev;
 
@@ -499,7 +488,7 @@ spec_write(ap)
 			}
 			n = min(n, bsize - buf_resid(bp));
 
-			error = uiomove((char *)buf_dataptr(bp) + on, n, uio);
+			error = uiomove((char *)0 + buf_dataptr(bp) + on, n, uio);
 			if (error) {
 				buf_brelse(bp);
 				return (error);
@@ -529,14 +518,7 @@ spec_write(ap)
  * Device ioctl operation.
  */
 int
-spec_ioctl(ap)
-	struct vnop_ioctl_args /* {
-		struct vnode *a_vp;
-		int  a_command;
-		caddr_t  a_data;
-		int  a_fflag;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_ioctl(struct vnop_ioctl_args *ap)
 {
 	proc_t p = vfs_context_proc(ap->a_context);
 	dev_t dev = ap->a_vp->v_rdev;
@@ -548,7 +530,7 @@ spec_ioctl(ap)
 		    ap->a_fflag, p));
 
 	case VBLK:
-	        if (ap->a_command == 0 && (int)ap->a_data == B_TAPE) {
+	        if (ap->a_command == 0 && (unsigned int)ap->a_data == B_TAPE) {
 			if (bdevsw[major(dev)].d_type == D_TAPE)
 				return (0);
 			else
@@ -565,17 +547,10 @@ spec_ioctl(ap)
 }
 
 int
-spec_select(ap)
-	struct vnop_select_args /* {
-		struct vnode *a_vp;
-		int  a_which;
-		int  a_fflags;
-		void * a_wql;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_select(struct vnop_select_args *ap)
 {
 	proc_t p = vfs_context_proc(ap->a_context);
-	register dev_t dev;
+	dev_t dev;
 
 	switch (ap->a_vp->v_type) {
 
@@ -599,18 +574,13 @@ spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
 	/*
 	 * Flush all dirty buffers associated with a block device.
 	 */
-	buf_flushdirtyblks(vp, waitfor == MNT_WAIT, 0, (char *)"spec_fsync");
+	buf_flushdirtyblks(vp, waitfor == MNT_WAIT, 0, "spec_fsync");
 
 	return (0);
 }
 
 int
-spec_fsync(ap)
-	struct vnop_fsync_args /* {
-		struct vnode *a_vp;
-		int  a_waitfor;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_fsync(struct vnop_fsync_args *ap)
 {
 	return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
 }
@@ -619,29 +589,68 @@ spec_fsync(ap)
  * Just call the device strategy routine
  */
 extern int hard_throttle_on_root;
+void IOSleep(int);
+extern void throttle_lowpri_io(int *lowpri_window,mount_t v_mount);
+
+// the low priority process may wait for at most LOWPRI_MAX_DELAY millisecond
+#define LOWPRI_INITIAL_WINDOW_MSECS 100
+#define LOWPRI_WINDOW_MSECS_INC	50
+#define LOWPRI_MAX_WINDOW_MSECS 200
+#define LOWPRI_MAX_WAITING_MSECS 200
+#define LOWPRI_SLEEP_INTERVAL 5
+
+int 	lowpri_IO_initial_window_msecs  = LOWPRI_INITIAL_WINDOW_MSECS;
+int 	lowpri_IO_window_msecs_inc  = LOWPRI_WINDOW_MSECS_INC;
+int 	lowpri_max_window_msecs  = LOWPRI_MAX_WINDOW_MSECS;
+int     lowpri_max_waiting_msecs = LOWPRI_MAX_WAITING_MSECS;
+
+SYSCTL_INT(_debug, OID_AUTO, lowpri_IO_initial_window_msecs, CTLFLAG_RW, &lowpri_IO_initial_window_msecs, LOWPRI_INITIAL_WINDOW_MSECS, "");
+SYSCTL_INT(_debug, OID_AUTO, lowpri_IO_window_inc, CTLFLAG_RW, &lowpri_IO_window_msecs_inc, LOWPRI_INITIAL_WINDOW_MSECS, "");
+SYSCTL_INT(_debug, OID_AUTO, lowpri_max_window_msecs, CTLFLAG_RW, &lowpri_max_window_msecs, LOWPRI_INITIAL_WINDOW_MSECS, "");
+SYSCTL_INT(_debug, OID_AUTO, lowpri_max_waiting_msecs, CTLFLAG_RW, &lowpri_max_waiting_msecs, LOWPRI_INITIAL_WINDOW_MSECS, "");
+
+void throttle_lowpri_io(int *lowpri_window,mount_t v_mount)
+{
+	int i;
+	struct timeval last_lowpri_IO_timestamp,last_normal_IO_timestamp;
+	struct timeval elapsed;
+	int lowpri_IO_window_msecs;
+	struct timeval lowpri_IO_window;
+	int max_try_num = lowpri_max_waiting_msecs / LOWPRI_SLEEP_INTERVAL;
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
+		     *lowpri_window, 0, 0, 0, 0);
+
+        last_normal_IO_timestamp = v_mount->last_normal_IO_timestamp;
+                        
+	for (i=0; i<max_try_num; i++) {
+		microuptime(&last_lowpri_IO_timestamp);
+
+		elapsed = last_lowpri_IO_timestamp;
+		timevalsub(&elapsed, &last_normal_IO_timestamp);
+
+		lowpri_IO_window_msecs = *lowpri_window;
+		lowpri_IO_window.tv_sec  = lowpri_IO_window_msecs / 1000;
+		lowpri_IO_window.tv_usec = (lowpri_IO_window_msecs % 1000) * 1000;
+
+		if (timevalcmp(&elapsed, &lowpri_IO_window, <)) {
+			IOSleep(LOWPRI_SLEEP_INTERVAL);
+		} else {
+			break;
+		}
+	}
 
-
-#define LOWPRI_DELAY_MSECS	200
-#define LOWPRI_WINDOW_MSECS	200
-
-int 	lowpri_IO_window_msecs = LOWPRI_WINDOW_MSECS;
-int 	lowpri_IO_delay_msecs  = LOWPRI_DELAY_MSECS;
-
-struct timeval last_normal_IO_timestamp;
-struct timeval last_lowpri_IO_timestamp;
-struct timeval lowpri_IO_window = { 0, LOWPRI_WINDOW_MSECS * 1000 };
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
+		     *lowpri_window, i*5, 0, 0, 0);
+	*lowpri_window = 0;
+}
 
 int
-spec_strategy(ap)
-	struct vnop_strategy_args /* {
-		struct buf *a_bp;
-	} */ *ap;
+spec_strategy(struct vnop_strategy_args *ap)
 {
         buf_t	bp;
 	int	bflags;
 	dev_t	bdev;
-	proc_t	p;
-        struct timeval elapsed;
 
         bp = ap->a_bp;
 	bdev = buf_device(bp);
@@ -667,36 +676,66 @@ spec_strategy(ap)
 	    (buf_vnode(bp)->v_mount->mnt_kern_flag & MNTK_ROOTDEV))
 	        hard_throttle_on_root = 1;
 
-	if ( lowpri_IO_delay_msecs && lowpri_IO_window_msecs ) {
-	        p = current_proc();
+	if (lowpri_IO_initial_window_msecs) {
+		proc_t	p;
+		struct uthread	*ut;
+		int policy = IOPOL_DEFAULT;
+		int is_throttleable_io = 0;
+		int is_passive_io = 0;
+		p = current_proc();
+		ut = get_bsdthread_info(current_thread());
+		
+		if (p != NULL)
+			policy = p->p_iopol_disk;
+
+		if (ut != NULL) {
+			// the I/O policy of the thread overrides that of the process
+			// unless the I/O policy of the thread is default
+			if (ut->uu_iopol_disk != IOPOL_DEFAULT)
+				policy = ut->uu_iopol_disk;
+		}
+
+		switch (policy) {
+		case IOPOL_DEFAULT:
+		case IOPOL_NORMAL:
+			break;
+		case IOPOL_THROTTLE:
+			is_throttleable_io = 1;
+			break;
+		case IOPOL_PASSIVE:
+			is_passive_io = 1;
+			break;
+		default:
+			printf("unknown I/O policy %d", policy);
+			break;
+		}
 
-	        if ( (p == NULL) || !(p->p_lflag & P_LLOW_PRI_IO)) {
-		        if (!(p->p_lflag & P_LBACKGROUND_IO))
-			        microuptime(&last_normal_IO_timestamp);
+		if (!is_throttleable_io && ISSET(bflags, B_PASSIVE))
+		    is_passive_io |= 1;
+
+		if (!is_throttleable_io) {
+			if (!is_passive_io && buf_vnode(bp)->v_mount != NULL){
+				microuptime(&(buf_vnode(bp)->v_mount->last_normal_IO_timestamp));
+			}
 		} else {
-		        microuptime(&last_lowpri_IO_timestamp);
-
-			elapsed = last_lowpri_IO_timestamp;
-			timevalsub(&elapsed, &last_normal_IO_timestamp);
-
-			lowpri_IO_window.tv_sec  = lowpri_IO_window_msecs / 1000;
-			lowpri_IO_window.tv_usec = (lowpri_IO_window_msecs % 1000) * 1000;
-
-			if (timevalcmp(&elapsed, &lowpri_IO_window, <)) {
-			        struct uthread	*ut;
-
-				/*
-				 * I'd really like to do the IOSleep here, but
-				 * we may be holding all kinds of filesystem related locks
-				 * and the pages for this I/O marked 'busy'...
-				 * we don't want to cause a normal task to block on
-				 * one of these locks while we're throttling a task marked
-				 * for low priority I/O... we'll mark the uthread and
-				 * do the delay just before we return from the system
-				 * call that triggered this I/O or from vnode_pagein
-				 */
-				ut = get_bsdthread_info(current_thread());
-				ut->uu_lowpri_delay = lowpri_IO_delay_msecs;
+			/*
+			 * I'd really like to do the IOSleep here, but
+			 * we may be holding all kinds of filesystem related locks
+			 * and the pages for this I/O marked 'busy'...
+			 * we don't want to cause a normal task to block on
+			 * one of these locks while we're throttling a task marked
+			 * for low priority I/O... we'll mark the uthread and
+			 * do the delay just before we return from the system
+			 * call that triggered this I/O or from vnode_pagein
+			 */
+			if(buf_vnode(bp)->v_mount != NULL)
+                                ut->v_mount = buf_vnode(bp)->v_mount;
+			if (ut->uu_lowpri_window == 0) {
+				ut->uu_lowpri_window = lowpri_IO_initial_window_msecs;
+			} else {
+				ut->uu_lowpri_window += lowpri_IO_window_msecs_inc;
+				if (ut->uu_lowpri_window > lowpri_max_window_msecs)
+					ut->uu_lowpri_window = lowpri_max_window_msecs;
 			}
 		}
 	}
@@ -720,18 +759,15 @@ spec_blockmap(__unused struct vnop_blockmap_args *ap)
  * Device close routine
  */
 int
-spec_close(ap)
-	struct vnop_close_args /* {
-		struct vnode *a_vp;
-		int  a_fflag;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_close(struct vnop_close_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
+	struct vnode *vp = ap->a_vp;
 	dev_t dev = vp->v_rdev;
 	int (*devclose)(dev_t, int, int, struct proc *);
 	int mode, error;
+	int flags = ap->a_fflag;
 	struct proc *p = vfs_context_proc(ap->a_context);
+	struct session *sessp;
 
 	switch (vp->v_type) {
 
@@ -745,18 +781,30 @@ spec_close(ap)
 		 * if the reference count is 2 (this last descriptor
 		 * plus the session), release the reference from the session.
 		 */
-		if (vcount(vp) == 2 && p &&
-		    vp == p->p_session->s_ttyvp) {
-			p->p_session->s_ttyvp = NULL;
-			vnode_rele(vp);
+		sessp = proc_session(p);
+		if (sessp != SESSION_NULL) {
+			if ((vcount(vp) == 2) && 
+		    		(vp == sessp->s_ttyvp)) {
+				session_lock(sessp);
+				sessp->s_ttyvp = NULL;
+				sessp->s_ttyvid = 0;
+				sessp->s_ttyp = NULL;
+				sessp->s_ttypgrpid = NO_PID;
+				session_unlock(sessp);
+				vnode_rele(vp);
+			}
+			session_rele(sessp);
 		}
+
+		devclose = cdevsw[major(dev)].d_close;
+		mode = S_IFCHR;
 		/*
-		 * close on last reference.
+		 * close on last reference or on vnode revoke call
 		 */
+		if ((flags & IO_REVOKE) != 0)
+			break;
 		if (vcount(vp) > 1)
 			return (0);
-		devclose = cdevsw[major(dev)].d_close;
-		mode = S_IFCHR;
 		break;
 
 	case VBLK:
@@ -810,22 +858,17 @@ spec_close(ap)
 
 	default:
 		panic("spec_close: not special");
+		return(EBADF);
 	}
 
-	return ((*devclose)(dev, ap->a_fflag, mode, p));
+	return ((*devclose)(dev, flags, mode, p));
 }
 
 /*
  * Return POSIX pathconf information applicable to special devices.
  */
 int
-spec_pathconf(ap)
-	struct vnop_pathconf_args /* {
-		struct vnode *a_vp;
-		int a_name;
-		int *a_retval;
-		vfs_context_t a_context;
-	} */ *ap;
+spec_pathconf(struct vnop_pathconf_args *ap)
 {
 
 	switch (ap->a_name) {
@@ -842,7 +885,7 @@ spec_pathconf(ap)
 		*ap->a_retval = PIPE_BUF;
 		return (0);
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		return (0);
 	case _PC_VDISABLE:
 		*ap->a_retval = _POSIX_VDISABLE;
@@ -863,27 +906,11 @@ spec_ebadf(__unused void *dummy)
 	return (EBADF);
 }
 
-/*
- * Special device bad operation
- */
-int
-spec_badop()
-{
-
-	panic("spec_badop called");
-	/* NOTREACHED */
-}
-
 /* Blktooff derives file offset from logical block number */
 int
-spec_blktooff(ap)
-	struct vnop_blktooff_args /* {
-		struct vnode *a_vp;
-		daddr64_t a_lblkno;
-		off_t *a_offset;    
-	} */ *ap;
+spec_blktooff(struct vnop_blktooff_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
+	struct vnode *vp = ap->a_vp;
 
 	switch (vp->v_type) {
 	case VCHR:
@@ -905,14 +932,9 @@ spec_blktooff(ap)
 
 /* Offtoblk derives logical block number from file offset */
 int
-spec_offtoblk(ap)
-	struct vnop_offtoblk_args /* {
-		struct vnode *a_vp;
-		off_t a_offset;    
-		daddr64_t *a_lblkno;
-	} */ *ap;
+spec_offtoblk(struct vnop_offtoblk_args *ap)
 {
-	register struct vnode *vp = ap->a_vp;
+	struct vnode *vp = ap->a_vp;
 
 	switch (vp->v_type) {
 	case VCHR:
diff --git a/bsd/miscfs/specfs/specdev.h b/bsd/miscfs/specfs/specdev.h
index f40247957..052085f17 100644
--- a/bsd/miscfs/specfs/specdev.h
+++ b/bsd/miscfs/specfs/specdev.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995, 1998 Apple Computer, Inc. All Rights Reserved.
@@ -111,7 +117,6 @@ extern struct vnode *speclisth[SPECHSZ];
 extern	int (**spec_vnodeop_p)(void *);
 struct	nameidata;
 struct	componentname;
-struct	ucred;
 struct	flock;
 struct	buf;
 struct	uio;
diff --git a/bsd/miscfs/synthfs/synthfs.h b/bsd/miscfs/synthfs/synthfs.h
index cb152a45e..b76e2d802 100644
--- a/bsd/miscfs/synthfs/synthfs.h
+++ b/bsd/miscfs/synthfs/synthfs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, Apple Computer, Inc. All rights reserved. */
 /*
diff --git a/bsd/miscfs/synthfs/synthfs_util.c b/bsd/miscfs/synthfs/synthfs_util.c
index d28e6ec5d..3f93e3f0e 100644
--- a/bsd/miscfs/synthfs/synthfs_util.c
+++ b/bsd/miscfs/synthfs/synthfs_util.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, Apple Computer, Inc. All rights reserved. */
 /*
@@ -81,37 +87,39 @@ static int synthfs_insertnode(struct synthfsnode *newnode_sp, struct synthfsnode
 
 
 
-static int synthfs_newnode(mount_t mp, vnode_t dp, const char *name, unsigned long nodeid,
-			   mode_t mode, __unused proc_t p, enum vtype vtype, vnode_t *vpp) {
+static int
+synthfs_newnode(mount_t mp, vnode_t dp, const char *name, unsigned long nodeid,
+		 mode_t mode, __unused proc_t p, enum vtype vtype, vnode_t *vpp)
+{
 	int result;
-    struct synthfsnode *sp;
+	struct synthfsnode *sp;
 	struct vnode *vp;
-    struct timeval now;
-    char *nodename;
+	struct timeval now;
+	char *nodename;
 	struct vnode_fsparam vfsp;
 
-     MALLOC(sp, struct synthfsnode *, sizeof(struct synthfsnode), M_SYNTHFS, M_WAITOK);
-    
-    if (name == NULL) {
-        MALLOC(nodename, char *, 1, M_TEMP, M_WAITOK);
-        nodename[0] = 0;
-    } else {
-        MALLOC(nodename, char *, strlen(name) + 1, M_TEMP, M_WAITOK);
-        strcpy(nodename, name);
-    };
+	MALLOC(sp, struct synthfsnode *, sizeof(struct synthfsnode), M_SYNTHFS, M_WAITOK);
 
-    /* Initialize the relevant synthfsnode fields: */
-    bzero(sp, sizeof(*sp));
-    sp->s_nodeid = nodeid;
-    
-    /* Initialize all times from a consistent snapshot of the clock: */
+	if (name == NULL) {
+		MALLOC(nodename, char *, 1, M_TEMP, M_WAITOK);
+		nodename[0] = 0;
+	} else {
+		MALLOC(nodename, char *, strlen(name) + 1, M_TEMP, M_WAITOK);
+		strlcpy(nodename, name, strlen(name) + 1);
+	};
+
+	/* Initialize the relevant synthfsnode fields: */
+	bzero(sp, sizeof(*sp));
+	sp->s_nodeid = nodeid;
+
+	/* Initialize all times from a consistent snapshot of the clock: */
 	microtime(&now);
-    sp->s_createtime = now;
-    sp->s_accesstime = now;
-    sp->s_modificationtime = now;
-    sp->s_changetime = now;
-    sp->s_name = nodename;
-    sp->s_mode = mode;
+	sp->s_createtime = now;
+	sp->s_accesstime = now;
+	sp->s_modificationtime = now;
+	sp->s_changetime = now;
+	sp->s_name = nodename;
+	sp->s_mode = mode;
 
 
 	//bzero(&vfsp, sizeof(struct vnode_fsparam));
@@ -130,23 +138,23 @@ static int synthfs_newnode(mount_t mp, vnode_t dp, const char *name, unsigned lo
 
 	result = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &vp); 
 	if (result != 0) {
-	    DBG_VOP(("getnewvnode failed with error code %d\n", result));
-	    FREE(nodename, M_TEMP);
-	    FREE(sp, M_TEMP);
-	    return result;
+		DBG_VOP(("getnewvnode failed with error code %d\n", result));
+		FREE(nodename, M_TEMP);
+		FREE(sp, M_TEMP);
+		return result;
 	}
 	vnode_ref(vp);
 
-    sp->s_vp = vp;
+	sp->s_vp = vp;
 
-    /* If there's a parent directory, update its subnode structures to insert this new node: */
-    if (dp) {
-    	result = synthfs_insertnode(sp, VTOS(dp));
-    };
+	/* If there's a parent directory, update its subnode structures to insert this new node: */
+	if (dp) {
+		result = synthfs_insertnode(sp, VTOS(dp));
+	};
 
-    *vpp = vp;
+	*vpp = vp;
 
-    return result;
+	return result;
 }
 
 
@@ -184,7 +192,7 @@ int synthfs_move_rename_entry(struct vnode *source_vp, struct vnode *newparent_v
 	if (new_name) {
 		FREE(source_sp->s_name, M_TEMP);
 		MALLOC(new_name_ptr, char *, strlen(new_name) + 1, M_TEMP, M_WAITOK);
-		strcpy(new_name_ptr, new_name);
+		strlcpy(new_name_ptr, new_name, strlen(new_name) + 1);
 		source_sp->s_name = new_name_ptr;
 	};
 	
@@ -240,35 +248,37 @@ int synthfs_remove_directory(struct vnode *vp) {
 
 
 
-int synthfs_new_symlink(
+int
+synthfs_new_symlink(
 		struct mount *mp,
 		struct vnode *dp,
 		const char *name,
 		unsigned long nodeid,
 		char *targetstring,
 		struct proc *p,
-		struct vnode **vpp) {
-	
+		struct vnode **vpp)
+{
 	int result;
 	struct vnode *vp;
 	struct synthfsnode *sp;
-	
+
 	result = synthfs_newnode(mp, dp, name, nodeid, 0, p, VLNK,  &vp);
-	if (result) {
+	if (result)
 		return result;
-	};
-    sp = VTOS(vp);
-    sp->s_linkcount = 1;
-	
-    /* Set up the symlink-specific fields: */
-    sp->s_type = SYNTHFS_SYMLINK;
-    sp->s_u.s.s_length = strlen(targetstring);
-    MALLOC(sp->s_u.s.s_symlinktarget, char *, sp->s_u.s.s_length + 1, M_TEMP, M_WAITOK);
-    strcpy(sp->s_u.s.s_symlinktarget, targetstring);
-    
-    *vpp = vp;
-    
-    return 0;
+	sp = VTOS(vp);
+	sp->s_linkcount = 1;
+
+	/* Set up the symlink-specific fields: */
+	sp->s_type = SYNTHFS_SYMLINK;
+	sp->s_u.s.s_length = strlen(targetstring);
+	MALLOC(sp->s_u.s.s_symlinktarget, char *, sp->s_u.s.s_length + 1,
+	       M_TEMP, M_WAITOK);
+	strlcpy(sp->s_u.s.s_symlinktarget, targetstring,
+		sp->s_u.s.s_lenghth + 1);
+
+	*vpp = vp;
+
+	return 0;
 }
 
 
diff --git a/bsd/miscfs/synthfs/synthfs_vfsops.c b/bsd/miscfs/synthfs/synthfs_vfsops.c
index 742de36d1..2db55aa2c 100644
--- a/bsd/miscfs/synthfs/synthfs_vfsops.c
+++ b/bsd/miscfs/synthfs/synthfs_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -121,8 +127,8 @@ synthfs_mount_fs(struct mount *mp, vnode_t devvp, __unused user_addr_t data,  st
 	MALLOC(priv_mnt_data, struct synthfs_mntdata *, sizeof(struct synthfs_mntdata), M_SYNTHFS, M_WAITOK);
 	DBG_VOP(("MALLOC succeeded...\n"));
 
-	strncpy(mp->mnt_vfsstat.f_fstypename, synthfs_fs_name, sizeof(mp->mnt_vfsstat.f_fstypename));
-	strncpy(mp->mnt_vfsstat.f_mntfromname, synthfs_fake_mntfromname, sizeof(mp->mnt_vfsstat.f_mntfromname));
+	strlcpy(mp->mnt_vfsstat.f_fstypename, synthfs_fs_name, sizeof(mp->mnt_vfsstat.f_fstypename));
+	strlcpy(mp->mnt_vfsstat.f_mntfromname, synthfs_fake_mntfromname, sizeof(mp->mnt_vfsstat.f_mntfromname));
     priv_mnt_data->synthfs_mounteddev = (dev_t)0;
     priv_mnt_data->synthfs_nextid = FIRST_SYNTHFS_ID;
     priv_mnt_data->synthfs_filecount = 0;
@@ -396,17 +402,14 @@ vn_mkdir(struct proc *p, char *path, int mode)
 	struct nameidata nd;
 	struct vnode *vp;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_kernel();
 	int error;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
 
-	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(path), &context);
+	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(path), ctx);
 	error = namei(&nd);
 	if (error) {
 		DBG_VOP(("vn_mkdir: error from namei, error = %d.\n", error));
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	};
 	vp = nd.ni_vp;
@@ -416,7 +419,7 @@ vn_mkdir(struct proc *p, char *path, int mode)
 		VATTR_SET(&va, va_type, VDIR);
 		VATTR_SET(&va, va_mode, (mode & ACCESSPERMS) &~ p->p_fd->fd_cmask);
 
-		error = vn_create(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &va, 0, &context);
+		error = vn_create(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &va, 0, ctx);
 		if (error)
 		        DBG_VOP(("vn_mkdir: error from vnop_mkdir (%d).\n", error));
 	} else {
@@ -428,7 +431,6 @@ vn_mkdir(struct proc *p, char *path, int mode)
 	        vnode_put(nd.ni_vp);
 	nameidone(&nd);
 
-	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
@@ -438,15 +440,11 @@ int
 vn_symlink(struct proc *p, char *path, char *link) {
 	struct nameidata nd;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_kernel();
 	int error;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? proxy */
-
-	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(link), &context);
+	NDINIT(&nd, CREATE, LOCKPARENT, UIO_SYSSPACE32, CAST_USER_ADDR_T(link), ctx);
 	if ((error = namei(&nd))) {
-		kauth_cred_unref(&context.vc_ucred);
 		return error;
 	}
 
@@ -455,7 +453,7 @@ vn_symlink(struct proc *p, char *path, char *link) {
 		VATTR_SET(&va, va_type, VLNK);
 		VATTR_SET(&va, va_mode, ACCESSPERMS &~ p->p_fd->fd_cmask);
 
-		error = VNOP_SYMLINK(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &va, path, &context);
+		error = VNOP_SYMLINK(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &va, path, ctx);
 	} else
 	        error = EEXIST;
 
@@ -464,7 +462,6 @@ vn_symlink(struct proc *p, char *path, char *link) {
 		vnode_put(nd.ni_vp);
 	nameidone(&nd);
 
-	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
diff --git a/bsd/miscfs/synthfs/synthfs_vnops.c b/bsd/miscfs/synthfs/synthfs_vnops.c
index c6367ec4d..d409494e8 100644
--- a/bsd/miscfs/synthfs/synthfs_vnops.c
+++ b/bsd/miscfs/synthfs/synthfs_vnops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-1999 Apple Computer, Inc. All Rights Reserved.
@@ -1190,10 +1196,10 @@ struct vnop_pathconf_args /* {
             *ap->a_retval = PIPE_BUF;
             return (0);
         case _PC_CHOWN_RESTRICTED:
-            *ap->a_retval = 1;
+	    *ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
             return (0);
         case _PC_NO_TRUNC:
-            *ap->a_retval = 1;
+            *ap->a_retval = 200112;		/* _POSIX_NO_TRUNC */
             return (0);
         default:
             return (EINVAL);
diff --git a/bsd/miscfs/union/union.h b/bsd/miscfs/union/union.h
index 9c64c4c7d..5411f538d 100644
--- a/bsd/miscfs/union/union.h
+++ b/bsd/miscfs/union/union.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -70,25 +76,44 @@ struct union_args {
 	int		mntflags;	/* Options on the mount */
 };
 
-#define UNMNT_ABOVE	0x0001		/* Target appears below mount point */
+#define UNMNT_ABOVE	0x0001		/* Target appears above mount point */
 #define UNMNT_BELOW	0x0002		/* Target appears below mount point */
 #define UNMNT_REPLACE	0x0003		/* Target replaces mount point */
+#ifdef FAULTFS
+#define UNMNT_FAULTIN	0x0004		/* get the files to TOT on lookup */
+#define UNMNT_OPMASK	0x0007
+#else
 #define UNMNT_OPMASK	0x0003
+#endif
+
+#ifdef BSD_KERNEL_PRIVATE
 
 struct union_mount {
-	struct vnode	*um_uppervp;
-	struct vnode	*um_lowervp;
-	struct ucred	*um_cred;	/* Credentials of user calling mount */
+	struct vnode	*um_uppervp;	/* */
+	int		um_uppervid;	/* vid of upper vnode */
+	struct vnode	*um_lowervp;	/* Left unlocked */
+	int		um_lowervid;	/* vid of lower vnode */
+	kauth_cred_t	um_cred;	/* Credentials of user calling mount */
 	int		um_cmode;	/* cmask from mount process */
 	int		um_op;		/* Operation mode */
+	dev_t		um_upperdev;	/* Upper root node fsid[0]*/
 };
 
-#ifdef KERNEL
+
+#define  UNION_ABOVE(x) (x->um_op == UNMNT_ABOVE)
+#define  UNION_LOWER(x) (x->um_op == UNMNT_BELOW)
+#define  UNION_REPLACE(x) (x->um_op == UNMNT_REPLACE)
+#ifdef FAULTFS
+#define  UNION_FAULTIN(x) (x->um_op == UNMNT_FAULTIN)
+#else
+#define  UNION_FAULTIN(x) (0)
+
+#endif
+
 /* LP64 version of union_args.  all pointers 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with union_args
  */
-/* LP64todo - should this move? */
 
 struct user_union_args {
 	user_addr_t	target;		/* Target of loopback  */
@@ -110,59 +135,88 @@ struct user_union_args {
 struct union_node {
 	LIST_ENTRY(union_node)	un_cache;	/* Hash chain */
 	struct vnode		*un_vnode;	/* Back pointer */
+
 	struct vnode	        *un_uppervp;	/* overlaying object */
+	int			un_uppervid;	/* vid of upper vnode */
+	off_t			un_uppersz;	/* size of upper object */
+
 	struct vnode	        *un_lowervp;	/* underlying object */
+	int			un_lowervid;	/* vid of upper vnode */
+	off_t			un_lowersz;	/* size of lower object */
+
 	struct vnode		*un_dirvp;	/* Parent dir of uppervp */
 	struct vnode		*un_pvp;	/* Parent vnode */
+
 	char			*un_path;	/* saved component name */
 	int			un_hash;	/* saved un_path hash value */
 	int			un_openl;	/* # of opens on lowervp */
+	int			un_exclcnt;	/* exclusive count */
 	unsigned int		un_flags;
+	mount_t			un_mount;
 	struct vnode		**un_dircache;	/* cached union stack */
-	off_t			un_uppersz;	/* size of upper object */
-	off_t			un_lowersz;	/* size of lower object */
-#if DIAGNOSTIC
-	pid_t			un_pid;
-#endif
 };
 
-#define UN_WANT		0x01
-#define UN_LOCKED	0x02
-#define UN_ULOCK	0x04		/* Upper node is locked */
-#define UN_KLOCK	0x08		/* Keep upper node locked on vput */
-#define UN_CACHED	0x10		/* In union cache */
+#define UN_WANT		0x01		/* union node is needed */
+#define UN_LOCKED	0x02		/* union node is locked */
+#define UN_CACHED	0x04		/* In union cache */
+#define UN_TRANSIT	0x08		/* The union node is in creation */
+#define UN_DELETED	0x10		/* The union node is deleted  */
+#ifdef FAULTFS
+#define UN_FAULTFS	0x80		/* The union node is for faultfs */
+#endif
+#define UN_DIRENVN	0x100		/* The union node is created for dir enumeration */
+
+
+#ifdef FAULTFS
+#define  UNNODE_FAULTIN(x) ((x->un_flags & UN_FAULTFS)== UN_FAULTFS)
+#else
+#define  UNNODE_FAULTIN(x) (0)
+#endif
+/*
+ * Hash table locking flags
+ */
+
+#define UNVP_WANT	0x01
+#define UNVP_LOCKED	0x02
+
+#define	MOUNTTOUNIONMOUNT(mp) ((struct union_mount *)((mp)->mnt_data))
+#define	VTOUNION(vp) ((struct union_node *)(vp)->v_data)
+#define	UNIONTOV(un) ((un)->un_vnode)
+#define	LOWERVP(vp) (VTOUNION(vp)->un_lowervp)
+#define	UPPERVP(vp) (VTOUNION(vp)->un_uppervp)
+#define OTHERVP(vp) (UPPERVP(vp) ? UPPERVP(vp) : LOWERVP(vp))
+
 
 extern int union_allocvp(struct vnode **, struct mount *,
 				struct vnode *, struct vnode *,
 				struct componentname *, struct vnode *,
 				struct vnode *, int);
-extern int union_copyfile(struct vnode *, struct vnode *,
-					struct ucred *, struct proc *);
-extern int union_copyup(struct union_node *, int, struct ucred *,
-				struct proc *);
+extern int union_freevp(struct vnode *);
+extern struct vnode * union_dircache(struct vnode *, vfs_context_t);
+extern int union_copyfile(struct vnode *, struct vnode *,vfs_context_t );
+extern int union_copyup(struct union_node *, int, vfs_context_t );
 extern int union_dowhiteout(struct union_node *, vfs_context_t);
 extern int union_mkshadow(struct union_mount *, struct vnode *,
 				struct componentname *, struct vnode **);
 extern int union_mkwhiteout(struct union_mount *, struct vnode *,
 				struct componentname *, char *);
-extern int union_vn_create(struct vnode **, struct union_node *, struct proc *);
-extern int union_cn_close(struct vnode *, int, struct ucred *, struct proc *);
+extern int union_vn_create(struct vnode **, struct union_node *, mode_t  mode, vfs_context_t context);
+extern int union_cn_close(struct vnode *, int, vfs_context_t context);
 extern void union_removed_upper(struct union_node *un);
 extern struct vnode *union_lowervp(struct vnode *);
-extern void union_newlower(struct union_node *, struct vnode *);
-extern void union_newupper(struct union_node *, struct vnode *);
 extern void union_newsize(struct vnode *, off_t, off_t);
-
-#define	MOUNTTOUNIONMOUNT(mp) ((struct union_mount *)((mp)->mnt_data))
-#define	VTOUNION(vp) ((struct union_node *)(vp)->v_data)
-#define	UNIONTOV(un) ((un)->un_vnode)
-#define	LOWERVP(vp) (VTOUNION(vp)->un_lowervp)
-#define	UPPERVP(vp) (VTOUNION(vp)->un_uppervp)
-#define OTHERVP(vp) (UPPERVP(vp) ? UPPERVP(vp) : LOWERVP(vp))
-
+extern int union_init(struct vfsconf *);
+extern void union_updatevp(struct union_node *, struct vnode *, struct vnode *);
+extern void union_dircache_free(struct union_node *);
+extern int (*union_dircheckp)(struct vnode **, struct fileproc *, vfs_context_t);
+extern int union_faultin_copyup(struct vnode ** uvpp, vnode_t udvp, vnode_t lvp, struct componentname * cnp, vfs_context_t context);
 extern int (**union_vnodeop_p)(void *);
 extern struct vfsops union_vfsops;
-#endif /* KERNEL */
+void union_lock(void);
+void union_unlock(void);
+
+#endif /* BSD_KERNEL_PRIVATE */
 
 #endif /* __APPLE_API_PRIVATE */
+
 #endif /* __UNION_UNION_H__ */
diff --git a/bsd/miscfs/union/union_subr.c b/bsd/miscfs/union/union_subr.c
index 7ff6930f3..b5b7b43a7 100644
--- a/bsd/miscfs/union/union_subr.c
+++ b/bsd/miscfs/union/union_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,6 +64,12 @@
  *
  *	@(#)union_subr.c	8.20 (Berkeley) 5/20/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -68,7 +80,7 @@
 #include <sys/vnode_internal.h>
 #include <sys/namei.h>
 #include <sys/malloc.h>
-#include <sys/file.h>
+#include <sys/file_internal.h>
 #include <sys/filedesc.h>
 #include <sys/queue.h>
 #include <sys/mount_internal.h>
@@ -76,11 +88,15 @@
 #include <sys/ubc.h>
 #include <sys/uio_internal.h>
 #include <miscfs/union/union.h>
-
-#if DIAGNOSTIC
-#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/kdebug.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
 #endif
 
+
+static int union_vn_close(struct vnode *vp, int fmode, vfs_context_t ctx);
+
 /* must be power of two, otherwise change UNION_HASH() */
 #define NHASH 32
 
@@ -91,55 +107,106 @@
 static LIST_HEAD(unhead, union_node) unhead[NHASH];
 static int unvplock[NHASH];
 
+static lck_grp_t * union_lck_grp;
+static lck_grp_attr_t * union_lck_grp_attr;
+static lck_attr_t * union_lck_attr;
+static lck_mtx_t *  union_mtxp;
+
+static int union_dircheck(struct vnode **, struct fileproc *, vfs_context_t ctx);
+static void union_newlower(struct union_node *, struct vnode *);
+static void union_newupper(struct union_node *, struct vnode *);
+
+
 int
-union_init()
+union_init(__unused struct vfsconf *vfsp)
 {
 	int i;
 
+	union_lck_grp_attr= lck_grp_attr_alloc_init();
+#if DIAGNOSTIC
+	lck_grp_attr_setstat(union_lck_grp_attr);
+#endif
+	union_lck_grp = lck_grp_alloc_init("union",  union_lck_grp_attr);
+	union_lck_attr = lck_attr_alloc_init();
+#if DIAGNOSTIC
+	lck_attr_setdebug(union_lck_attr);
+#endif
+	union_mtxp = lck_mtx_alloc_init(union_lck_grp, union_lck_attr);
+
 	for (i = 0; i < NHASH; i++)
 		LIST_INIT(&unhead[i]);
 	bzero((caddr_t) unvplock, sizeof(unvplock));
+	/* add the hook for getdirentries */
+	union_dircheckp = union_dircheck;
+	
+	return (0);
 }
 
+void
+union_lock()
+{
+	lck_mtx_lock(union_mtxp);
+}
+
+void
+union_unlock()
+{
+	lck_mtx_unlock(union_mtxp);
+}
+
+
 static int
-union_list_lock(ix)
-	int ix;
+union_list_lock(int ix)
 {
 
-	if (unvplock[ix] & UN_LOCKED) {
-		unvplock[ix] |= UN_WANT;
-		sleep((caddr_t) &unvplock[ix], PINOD);
+	if (unvplock[ix] & UNVP_LOCKED) {
+		unvplock[ix] |= UNVP_WANT;
+		msleep((caddr_t) &unvplock[ix], union_mtxp, PINOD, "union_list_lock", NULL);
 		return (1);
 	}
 
-	unvplock[ix] |= UN_LOCKED;
+	unvplock[ix] |= UNVP_LOCKED;
 
 	return (0);
 }
 
 static void
-union_list_unlock(ix)
-	int ix;
+union_list_unlock(int ix)
 {
 
-	unvplock[ix] &= ~UN_LOCKED;
+	unvplock[ix] &= ~UNVP_LOCKED;
 
-	if (unvplock[ix] & UN_WANT) {
-		unvplock[ix] &= ~UN_WANT;
+	if (unvplock[ix] & UNVP_WANT) {
+		unvplock[ix] &= ~UNVP_WANT;
 		wakeup((caddr_t) &unvplock[ix]);
 	}
 }
 
+/*
+ *	union_updatevp:
+ *
+ *	The uppervp, if not NULL, must be referenced and not locked by us
+ *	The lowervp, if not NULL, must be referenced.
+ *
+ *	If uppervp and lowervp match pointers already installed, then
+ *	nothing happens. The passed vp's (when matching) are not adjusted.
+ *
+ *	This routine may only be called by union_newupper() and
+ *	union_newlower().
+ */
+
+/* always called with union lock held */
 void
-union_updatevp(un, uppervp, lowervp)
-	struct union_node *un;
-	struct vnode *uppervp;
-	struct vnode *lowervp;
+union_updatevp(struct union_node *un, struct vnode *uppervp,
+		struct vnode *lowervp)
 {
 	int ohash = UNION_HASH(un->un_uppervp, un->un_lowervp);
 	int nhash = UNION_HASH(uppervp, lowervp);
 	int docache = (lowervp != NULLVP || uppervp != NULLVP);
 	int lhash, uhash;
+	vnode_t freevp;
+	vnode_t freedirvp;
+	caddr_t freepath;
 
 	/*
 	 * Ensure locking is ordered from lower to higher
@@ -153,9 +220,10 @@ union_updatevp(un, uppervp, lowervp)
 		uhash = nhash;
 	}
 
-	if (lhash != uhash)
+	if (lhash != uhash) {
 		while (union_list_lock(lhash))
 			continue;
+	}
 
 	while (union_list_lock(uhash))
 		continue;
@@ -171,27 +239,47 @@ union_updatevp(un, uppervp, lowervp)
 		union_list_unlock(ohash);
 
 	if (un->un_lowervp != lowervp) {
+		freevp = freedirvp = NULLVP;
+		freepath = (caddr_t)0;
 		if (un->un_lowervp) {
-			vnode_put(un->un_lowervp);
+			freevp = un->un_lowervp;
+			un->un_lowervp = lowervp;
 			if (un->un_path) {
-				_FREE(un->un_path, M_TEMP);
+				freepath = un->un_path; 
 				un->un_path = 0;
 			}
 			if (un->un_dirvp) {
-				vnode_put(un->un_dirvp);
+				freedirvp = un->un_dirvp;
 				un->un_dirvp = NULLVP;
 			}
-		}
-		un->un_lowervp = lowervp;
+			union_unlock();
+			if (freevp)
+				vnode_put(freevp);
+			if (freedirvp)
+				vnode_put(freedirvp);
+			if (freepath)
+				_FREE(un->un_path, M_TEMP);
+			union_lock();
+		} else 
+			un->un_lowervp = lowervp;
+		if (lowervp != NULLVP)
+			un->un_lowervid = vnode_vid(lowervp);
 		un->un_lowersz = VNOVAL;
 	}
 
 	if (un->un_uppervp != uppervp) {
-		if (un->un_uppervp)
-			vnode_put(un->un_uppervp);
-
+		freevp = NULLVP;
+		if (un->un_uppervp) {
+			freevp = un->un_uppervp;
+		}
 		un->un_uppervp = uppervp;
+		if (uppervp != NULLVP)
+			un->un_uppervid = vnode_vid(uppervp);
 		un->un_uppersz = VNOVAL;
+		union_unlock();
+		if (freevp)
+			vnode_put(freevp);
+		union_lock();
 	}
 
 	if (docache && (ohash != nhash)) {
@@ -202,21 +290,32 @@ union_updatevp(un, uppervp, lowervp)
 	union_list_unlock(nhash);
 }
 
-void
+/*
+ * Set a new lowervp.  The passed lowervp must be referenced and will be
+ * stored in the vp in a referenced state. 
+ */
+/* always called with union lock held */
+
+static void
 union_newlower(un, lowervp)
 	struct union_node *un;
 	struct vnode *lowervp;
 {
-
 	union_updatevp(un, un->un_uppervp, lowervp);
 }
 
-void
+/*
+ * Set a new uppervp.  The passed uppervp must be locked and will be 
+ * stored in the vp in a locked state.  The caller should not unlock
+ * uppervp.
+ */
+
+/* always called with union lock held */
+static void
 union_newupper(un, uppervp)
 	struct union_node *un;
 	struct vnode *uppervp;
 {
-
 	union_updatevp(un, uppervp, un->un_lowervp);
 }
 
@@ -225,6 +324,7 @@ union_newupper(un, uppervp)
  * If the size changes, then callback to the vm layer
  * giving priority to the upper layer size.
  */
+/* always called with union lock held */
 void
 union_newsize(vp, uppersz, lowersz)
 	struct vnode *vp;
@@ -257,67 +357,100 @@ union_newsize(vp, uppersz, lowersz)
 		printf("union: %s size now %ld\n",
 			uppersz != VNOVAL ? "upper" : "lower", (long) sz);
 #endif
+		union_unlock();
 		ubc_setsize(vp, sz);
+		union_lock();
 	}
 }
 
 /*
- * allocate a union_node/vnode pair.  the vnode is
- * referenced and locked.  the new vnode is returned
- * via (vpp).  (mp) is the mountpoint of the union filesystem,
- * (dvp) is the parent directory where the upper layer object
- * should exist (but doesn't) and (cnp) is the componentname
- * information which is partially copied to allow the upper
- * layer object to be created at a later time.  (uppervp)
- * and (lowervp) reference the upper and lower layer objects
- * being mapped.  either, but not both, can be nil.
- * if supplied, (uppervp) is locked.
- * the reference is either maintained in the new union_node
- * object which is allocated, or they are vnode_put'd.
+ *	union_allocvp:	allocate a union_node and associate it with a
+ *			parent union_node and one or two vnodes.
+ *
+ *	vpp	Holds the returned vnode locked and referenced if no 
+ *		error occurs.
+ *
+ *	mp	Holds the mount point.  mp may or may not be busied. 
+ *		allocvp() makes no changes to mp.
+ *
+ *	dvp	Holds the parent union_node to the one we wish to create.
+ *		XXX may only be used to traverse an uncopied lowervp-based
+ *		tree?  XXX
  *
- * all union_nodes are maintained on a singly-linked
- * list.  new nodes are only allocated when they cannot
- * be found on this list.  entries on the list are
+ *		dvp may or may not be locked.  allocvp() makes no changes
+ *		to dvp.
+ *
+ *	upperdvp Holds the parent vnode to uppervp, generally used along
+ *		with path component information to create a shadow of
+ *		lowervp when uppervp does not exist.
+ *
+ *		upperdvp is referenced but unlocked on entry, and will be
+ *		dereferenced on return.
+ *
+ *	uppervp	Holds the new uppervp vnode to be stored in the 
+ *		union_node we are allocating.  uppervp is referenced but
+ *		not locked, and will be dereferenced on return.
+ *
+ *	lowervp	Holds the new lowervp vnode to be stored in the
+ *		union_node we are allocating.  lowervp is referenced but
+ *		not locked, and will be dereferenced on return.
+ * 
+ *	cnp	Holds path component information to be coupled with
+ *		lowervp and upperdvp to allow unionfs to create an uppervp
+ *		later on.  Only used if lowervp is valid.  The contents
+ *		of cnp is only valid for the duration of the call.
+ *
+ *	docache	Determine whether this node should be entered in the
+ *		cache or whether it should be destroyed as soon as possible.
+ *
+ * All union_nodes are maintained on a singly-linked
+ * list.  New nodes are only allocated when they cannot
+ * be found on this list.  Entries on the list are
  * removed when the vfs reclaim entry is called.
  *
- * a single lock is kept for the entire list.  this is
+ * A single lock is kept for the entire list.  This is
  * needed because the getnewvnode() function can block
  * waiting for a vnode to become free, in which case there
  * may be more than one process trying to get the same
- * vnode.  this lock is only taken if we are going to
- * call getnewvnode, since the kernel itself is single-threaded.
+ * vnode.  This lock is only taken if we are going to
+ * call getnewvnode(), since the kernel itself is single-threaded.
  *
- * if an entry is found on the list, then call vnode_get() to
- * take a reference.  this is done because there may be
+ * If an entry is found on the list, then call vget() to
+ * take a reference.  This is done because there may be
  * zero references to it and so it needs to removed from
  * the vnode free list.
  */
+
+/* always called with union lock held */
+
 int
-union_allocvp(vpp, mp, undvp, dvp, cnp, uppervp, lowervp, docache)
-	struct vnode **vpp;
-	struct mount *mp;
-	struct vnode *undvp;		/* parent union vnode */
-	struct vnode *dvp;		/* may be null */
-	struct componentname *cnp;	/* may be null */
-	struct vnode *uppervp;		/* may be null */
-	struct vnode *lowervp;		/* may be null */
-	int docache;
+union_allocvp(struct vnode **vpp,
+	struct mount *mp,
+	struct vnode *undvp,
+	struct vnode *dvp,
+	struct componentname *cnp,
+	struct vnode *uppervp,
+	struct vnode *lowervp,
+	int docache)
 {
 	int error;
-	struct union_node *un;
-	struct union_node **pp;
+	struct union_node *un = NULL;
+	struct union_node *unp;
 	struct vnode *xlowervp = NULLVP;
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
-	int hash;
+	int hash = 0;		/* protected by docache */
 	int markroot;
 	int try;
-	struct union_node *unp;
 	struct vnode_fsparam vfsp;
 	enum vtype vtype;
 
 	if (uppervp == NULLVP && lowervp == NULLVP)
 		panic("union: unidentifiable allocation");
 
+	/*
+	 * if both upper and lower vp are provided and are off different type
+	 * consider lowervp as NULL
+	 */
 	if (uppervp && lowervp && (uppervp->v_type != lowervp->v_type)) {
 		xlowervp = lowervp;
 		lowervp = NULLVP;
@@ -329,15 +462,18 @@ union_allocvp(vpp, mp, undvp, dvp, cnp, uppervp, lowervp, docache)
 	    ((lowervp == NULLVP) || lowervp == um->um_lowervp)) {
 		if (lowervp == NULLVP) {
 			lowervp = um->um_lowervp;
-			if (lowervp != NULLVP)
+			if (lowervp != NULLVP) {
+				union_unlock();
 				vnode_get(lowervp);
+				union_lock();
+			}
 		}
 		markroot = VROOT;
 	}
 
 loop:
 	if (!docache) {
-		un = 0;
+		un = NULL;
 	} else for (try = 0; try < 3; try++) {
 		switch (try) {
 		case 0:
@@ -355,6 +491,7 @@ loop:
 		case 2:
 			if (lowervp == NULLVP)
 				continue;
+			/* Not sure how this path gets exercised ? */
 			hash = UNION_HASH(NULLVP, lowervp);
 			break;
 		}
@@ -368,11 +505,7 @@ loop:
 			     un->un_lowervp == NULLVP) &&
 			    (un->un_uppervp == uppervp ||
 			     un->un_uppervp == NULLVP) &&
-			    (UNIONTOV(un)->v_mount == mp)) {
-				if (vnode_get(UNIONTOV(un))) {
-					union_list_unlock(hash);
-					goto loop;
-				}
+			    (un->un_mount == mp)) {
 				break;
 			}
 		}
@@ -393,36 +526,26 @@ loop:
 		 * with uppervp.
 		 */
 
-		if ((dvp != NULLVP) && (uppervp == dvp)) {
-			/*
-			 * Access ``.'', so (un) will already
-			 * be locked.  Since this process has
-			 * the lock on (uppervp) no other
-			 * process can hold the lock on (un).
-			 */
-#if DIAGNOSTIC
-			if ((un->un_flags & UN_LOCKED) == 0)
-				panic("union: . not locked");
-			else if (current_proc() && un->un_pid != current_proc()->p_pid &&
-				    un->un_pid > -1 && current_proc()->p_pid > -1)
-				panic("union: allocvp not lock owner");
-#endif
-		} else {
-			if (un->un_flags & UN_LOCKED) {
-				vnode_put(UNIONTOV(un));
-				un->un_flags |= UN_WANT;
-				sleep((caddr_t) &un->un_flags, PINOD);
-				goto loop;
+		if (un->un_flags & UN_LOCKED) {
+			un->un_flags |= UN_WANT;
+			msleep((caddr_t) &un->un_flags, union_mtxp, PINOD, "union node locked", 0);
+			goto loop;
+		}       
+		un->un_flags |= UN_LOCKED;
+                        
+		union_unlock();
+		if (UNIONTOV(un) == NULLVP)
+			panic("null vnode in union node\n");
+		if (vnode_get(UNIONTOV(un))) {
+			union_lock();
+			un->un_flags &= ~UN_LOCKED;
+			if ((un->un_flags & UN_WANT) == UN_WANT) {
+				un->un_flags &=  ~UN_LOCKED;
+				wakeup(&un->un_flags);
 			}
-			un->un_flags |= UN_LOCKED;
-
-#if DIAGNOSTIC
-			if (current_proc())
-				un->un_pid = current_proc()->p_pid;
-			else
-				un->un_pid = -1;
-#endif
+			goto loop;
 		}
+		union_lock();
 
 		/*
 		 * At this point, the union_node is locked,
@@ -436,12 +559,9 @@ loop:
 		if (uppervp != un->un_uppervp) {
 			union_newupper(un, uppervp);
 		} else if (uppervp) {
+			union_unlock();
 			vnode_put(uppervp);
-		}
-
-		if (un->un_uppervp) {
-			un->un_flags |= UN_ULOCK;
-			un->un_flags &= ~UN_KLOCK;
+			union_lock();
 		}
 
 		/*
@@ -454,18 +574,27 @@ loop:
 			union_newlower(un, lowervp);
 			if (cnp && (lowervp != NULLVP)) {
 				un->un_hash = cnp->cn_hash;
+				union_unlock();
 				MALLOC(un->un_path, caddr_t, cnp->cn_namelen+1,
 						M_TEMP, M_WAITOK);
 				bcopy(cnp->cn_nameptr, un->un_path,
 						cnp->cn_namelen);
-				un->un_path[cnp->cn_namelen] = '\0';
 				vnode_get(dvp);
+				union_lock();
+				un->un_path[cnp->cn_namelen] = '\0';
 				un->un_dirvp = dvp;
 			}
 		} else if (lowervp) {
+			union_unlock();
 			vnode_put(lowervp);
+			union_lock();
 		}
 		*vpp = UNIONTOV(un);
+		un->un_flags &= ~UN_LOCKED;
+		if ((un->un_flags & UN_WANT) == UN_WANT) {
+			un->un_flags &= ~UN_WANT;
+			wakeup(&un->un_flags);
+		}
 		return (0);
 	}
 
@@ -480,17 +609,51 @@ loop:
 			goto loop;
 	}
 
+	union_unlock();
 	MALLOC(unp, void *, sizeof(struct union_node), M_TEMP, M_WAITOK);
+	union_lock();
+
+	bzero(unp, sizeof(struct union_node));
+	un = unp;
+	un->un_uppervp = uppervp;
+	if (uppervp != NULLVP)
+		un->un_uppervid = vnode_vid(uppervp);
+	un->un_uppersz = VNOVAL;
+	un->un_lowervp = lowervp;
+	if (lowervp != NULLVP)
+		un->un_lowervid = vnode_vid(lowervp);
+	un->un_lowersz = VNOVAL;
+	un->un_pvp = undvp;
+	if (undvp != NULLVP)
+		vnode_get(undvp);
+	un->un_dircache = 0;
+	un->un_openl = 0;
+	un->un_mount = mp;
+	un->un_flags = UN_LOCKED;
+#ifdef FAULTFS
+	if (UNION_FAULTIN(um))
+		un->un_flags |= UN_FAULTFS;
+#endif
+
+	if (docache) {
+		/* Insert with lock held */
+		LIST_INSERT_HEAD(&unhead[hash], un, un_cache);
+		un->un_flags |= UN_CACHED;
+		union_list_unlock(hash);
+	}
+
+	union_unlock();
 
 	if (uppervp)
 		vtype = uppervp->v_type;
 	else
 		vtype = lowervp->v_type;
-	//bzero(&vfsp, sizeof(struct vnode_fsparam));
+
+	bzero(&vfsp, sizeof(struct vnode_fsparam));
 	vfsp.vnfs_mp = mp;
 	vfsp.vnfs_vtype = vtype;
 	vfsp.vnfs_str = "unionfs";
-	vfsp.vnfs_dvp = dvp;
+	vfsp.vnfs_dvp = undvp;
 	vfsp.vnfs_fsnode = unp;
 	vfsp.vnfs_cnp = cnp;
 	vfsp.vnfs_vops = union_vnodeop_p;
@@ -502,37 +665,26 @@ loop:
 
 	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, vpp);
 	if (error) {
-		FREE(unp, M_TEMP);
+		/*  XXXXX Is this right ????  XXXXXXX */
 		if (uppervp) {
 		        vnode_put(uppervp);
 		}
 		if (lowervp)
 			vnode_put(lowervp);
 
-		goto out;
+		union_lock();
+		if (un->un_flags & UN_CACHED) {
+			un->un_flags &= ~UN_CACHED;
+			LIST_REMOVE(un, un_cache);
+		}
+		if (docache)
+			union_list_unlock(hash);
+
+		FREE(unp, M_TEMP);
+
+		return (error);
 	}
 
-	(*vpp)->v_tag = VT_UNION;
-	un = VTOUNION(*vpp);
-	un->un_vnode = *vpp;
-	un->un_uppervp = uppervp;
-	un->un_uppersz = VNOVAL;
-	un->un_lowervp = lowervp;
-	un->un_lowersz = VNOVAL;
-	un->un_pvp = undvp;
-	if (undvp != NULLVP)
-		vnode_get(undvp);
-	un->un_dircache = 0;
-	un->un_openl = 0;
-	un->un_flags = UN_LOCKED;
-	if (un->un_uppervp)
-		un->un_flags |= UN_ULOCK;
-#if DIAGNOSTIC
-	if (current_proc())
-		un->un_pid = current_proc()->p_pid;
-	else
-		un->un_pid = -1;
-#endif
 	if (cnp && (lowervp != NULLVP)) {
 		un->un_hash = cnp->cn_hash;
 		un->un_path = _MALLOC(cnp->cn_namelen+1, M_TEMP, M_WAITOK);
@@ -546,24 +698,34 @@ loop:
 		un->un_dirvp = 0;
 	}
 
-	if (docache) {
-		LIST_INSERT_HEAD(&unhead[hash], un, un_cache);
-		un->un_flags |= UN_CACHED;
-	}
-
 	if (xlowervp)
 		vnode_put(xlowervp);
 
-out:
-	if (docache)
-		union_list_unlock(hash);
+	union_lock();
+
+	vnode_settag(*vpp, VT_UNION);
+	un->un_vnode = *vpp;
+	if (un->un_vnode->v_type == VDIR) {
+		if (un->un_uppervp == NULLVP) {
+			panic("faulting fs and no upper vp for dir?");
+		}
+
+	}
+
+
+	un->un_flags &= ~UN_LOCKED;
+	if ((un->un_flags & UN_WANT) == UN_WANT) {
+		un->un_flags &=  ~UN_WANT;
+		wakeup(&un->un_flags);
+	} 
+
+	return(error);
 
-	return (error);
 }
 
+/* always called with union lock held */
 int
-union_freevp(vp)
-	struct vnode *vp;
+union_freevp(struct vnode *vp)
 {
 	struct union_node *un = VTOUNION(vp);
 
@@ -572,6 +734,7 @@ union_freevp(vp)
 		LIST_REMOVE(un, un_cache);
 	}
 
+	union_unlock();
 	if (un->un_pvp != NULLVP)
 		vnode_put(un->un_pvp);
 	if (un->un_uppervp != NULLVP)
@@ -585,6 +748,7 @@ union_freevp(vp)
 
 	FREE(vp->v_data, M_TEMP);
 	vp->v_data = 0;
+	union_lock();
 
 	return (0);
 }
@@ -594,14 +758,13 @@ union_freevp(vp)
  * using a sequence of reads and writes.  both (fvp)
  * and (tvp) are locked on entry and exit.
  */
+/* called with no union lock held */
 int
-union_copyfile(struct vnode *fvp, struct vnode *tvp, kauth_cred_t cred,
-	struct proc *p)
+union_copyfile(struct vnode *fvp, struct vnode *tvp, vfs_context_t context)
 {
 	char *bufp;
 	struct uio uio;
 	struct iovec_32 iov;
-	struct vfs_context context;
 	int error = 0;
 
 	/*
@@ -612,8 +775,6 @@ union_copyfile(struct vnode *fvp, struct vnode *tvp, kauth_cred_t cred,
 	 * give up at the first sign of trouble.
 	 */
 
-	context.vc_proc = p;
-	context.vc_ucred = cred;
 
 #if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
 	uio.uio_segflg = UIO_SYSSPACE;
@@ -634,7 +795,7 @@ union_copyfile(struct vnode *fvp, struct vnode *tvp, kauth_cred_t cred,
 		iov.iov_len = MAXPHYSIO;
 		uio_setresid(&uio, iov.iov_len);
 		uio.uio_rw = UIO_READ;
-		error = VNOP_READ(fvp, &uio, 0, &context);
+		error = VNOP_READ(fvp, &uio, 0, context);
 
 		if (error == 0) {
 			uio.uio_iovs.iov32p = &iov;
@@ -649,7 +810,7 @@ union_copyfile(struct vnode *fvp, struct vnode *tvp, kauth_cred_t cred,
 				break;
 
 			do {
-				error = VNOP_WRITE(tvp, &uio, 0, &context);
+				error = VNOP_WRITE(tvp, &uio, 0, context);
 			} while ((uio_resid(&uio) > 0) && (error == 0));
 		}
 
@@ -663,36 +824,55 @@ union_copyfile(struct vnode *fvp, struct vnode *tvp, kauth_cred_t cred,
  * (un) is assumed to be locked on entry and remains
  * locked on exit.
  */
+/* always called with union lock held */
 int
-union_copyup(struct union_node *un, int docopy, kauth_cred_t cred,
-	struct proc *p)
+union_copyup(struct union_node *un, int docopy, vfs_context_t context)
 {
 	int error;
 	struct vnode *lvp, *uvp;
-	struct vfs_context context;
+	struct vnode_attr vattr;
+	mode_t  cmode = 0;
 
-	error = union_vn_create(&uvp, un, p);
-	if (error)
-		return (error);
+	
+	lvp = un->un_lowervp;
 
-	context.vc_proc = p;
-	context.vc_ucred = cred;
+	union_unlock();
 
+	if (UNNODE_FAULTIN(un)) {
+		/* Need to inherit exec mode in faulting fs */
+		VATTR_INIT(&vattr);
+		VATTR_WANTED(&vattr, va_flags);
+		if (vnode_getattr(lvp, &vattr, context) == 0 )
+			cmode = vattr.va_mode;
+		
+	}
+	error = union_vn_create(&uvp, un, cmode, context);
+	if (error) {
+		union_lock();
+		if (error == EEXIST) {
+			if (uvp != NULLVP) {
+				union_newupper(un, uvp);
+				error = 0;	
+			} 
+		}
+		return (error);
+	}
+
+	union_lock();
 	/* at this point, uppervp is locked */
 	union_newupper(un, uvp);
-	un->un_flags |= UN_ULOCK;
+	union_unlock();
 
-	lvp = un->un_lowervp;
 
 	if (docopy) {
 		/*
 		 * XX - should not ignore errors
 		 * from vnop_close
 		 */
-		error = VNOP_OPEN(lvp, FREAD, &context);
+		error = VNOP_OPEN(lvp, FREAD, context);
 		if (error == 0) {
-			error = union_copyfile(lvp, uvp, cred, p);
-			(void) VNOP_CLOSE(lvp, FREAD, &context);
+			error = union_copyfile(lvp, uvp, context);
+			(void) VNOP_CLOSE(lvp, FREAD, context);
 		}
 #ifdef UNION_DIAGNOSTIC
 		if (error == 0)
@@ -700,9 +880,7 @@ union_copyup(struct union_node *un, int docopy, kauth_cred_t cred,
 #endif
 
 	}
-	un->un_flags &= ~UN_ULOCK;
-	union_vn_close(uvp, FWRITE, cred, p);
-	un->un_flags |= UN_ULOCK;
+	union_vn_close(uvp, FWRITE, context);
 
 	/*
 	 * Subsequent IOs will go to the top layer, so
@@ -712,29 +890,184 @@ union_copyup(struct union_node *un, int docopy, kauth_cred_t cred,
 	 * the right thing with (cred) and (FREAD) though.
 	 * Ignoring error returns is not right, either.
 	 */
+
+	/* No need to hold the lock as the union node should be locked for this(it is in faultin mode) */
 	if (error == 0) {
 		int i;
 
 		for (i = 0; i < un->un_openl; i++) {
-			(void) VNOP_CLOSE(lvp, FREAD, &context);
-			(void) VNOP_OPEN(uvp, FREAD, &context);
+			(void) VNOP_CLOSE(lvp, FREAD, context);
+			(void) VNOP_OPEN(uvp, FREAD, context);
 		}
 		un->un_openl = 0;
 	}
 
+	union_lock();
+
 	return (error);
 
 }
 
+
+int 
+union_faultin_copyup(struct vnode **vpp, vnode_t udvp, vnode_t lvp, struct componentname * cnp, vfs_context_t context)
+{
+	int error;
+	struct vnode *uvp;
+	struct vnode_attr vattr;
+	struct vnode_attr *vap;
+	mode_t  cmode = 0;
+	int fmode = FFLAGS(O_WRONLY|O_CREAT|O_TRUNC|O_EXCL);
+	struct proc * p = vfs_context_proc(context);
+	struct componentname cn;
+	
+
+	vap = &vattr;
+	VATTR_INIT(vap);
+	VATTR_WANTED(vap, va_flags);
+	if (vnode_getattr(lvp, vap, context) == 0 )
+		cmode = vattr.va_mode;
+		
+	*vpp = NULLVP;
+
+
+	if (cmode == (mode_t)0)
+		cmode = UN_FILEMODE & ~p->p_fd->fd_cmask;
+	else
+		cmode = cmode & ~p->p_fd->fd_cmask;
+
+
+	/*
+	 * Build a new componentname structure (for the same
+	 * reasons outlines in union_mkshadow()).
+	 * The difference here is that the file is owned by
+	 * the current user, rather than by the person who
+	 * did the mount, since the current user needs to be
+	 * able to write the file (that's why it is being
+	 * copied in the first place).
+	 */
+	bzero(&cn, sizeof(struct componentname));
+
+	cn.cn_namelen = cnp->cn_namelen;
+	cn.cn_pnbuf = (caddr_t) _MALLOC_ZONE(cn.cn_namelen+1,
+						M_NAMEI, M_WAITOK);
+	cn.cn_pnlen = cn.cn_namelen+1;
+	bcopy(cnp->cn_nameptr, cn.cn_pnbuf, cn.cn_namelen+1);
+	cn.cn_nameiop = CREATE;
+	cn.cn_flags = (HASBUF|SAVENAME|SAVESTART|ISLASTCN|UNIONCREATED);
+	cn.cn_context = context;
+	cn.cn_nameptr = cn.cn_pnbuf;
+	cn.cn_hash = 0;
+	cn.cn_consume = 0;
+
+	/*
+	 * Pass dvp unlocked and referenced on call to relookup().
+	 *
+	 * If an error occurs, dvp will be returned unlocked and dereferenced.
+	 */
+	if ((error = relookup(udvp, &uvp, &cn)) != 0) {
+		goto out;
+	}
+
+	/*
+	 * If no error occurs, dvp will be returned locked with the reference
+	 * left as before, and vpp will be returned referenced and locked.
+	 */
+	if (uvp) {
+		*vpp = uvp;
+		error = EEXIST;
+		goto out;
+	}
+
+	/*
+	 * Good - there was no race to create the file
+	 * so go ahead and create it.  The permissions
+	 * on the file will be 0666 modified by the
+	 * current user's umask.  Access to the file, while
+	 * it is unioned, will require access to the top *and*
+	 * bottom files.  Access when not unioned will simply
+	 * require access to the top-level file.
+	 *
+	 * TODO: confirm choice of access permissions.
+	 *       decide on authorisation behaviour
+	 */
+	
+	VATTR_INIT(vap);
+	VATTR_SET(vap, va_type, VREG);
+	VATTR_SET(vap, va_mode, cmode);
+
+	cn.cn_flags |= (UNIONCREATED);
+	if ((error = vn_create(udvp, &uvp, &cn, vap, 0, context)) != 0) {
+		goto out;
+	}
+
+	
+	if ((error = VNOP_OPEN(uvp, fmode, context)) != 0) {
+		vn_clearunionwait(uvp, 0);
+		vnode_recycle(uvp);
+		vnode_put(uvp);
+		goto out;
+	}
+
+	error = vnode_ref_ext(uvp, fmode);
+	if (error ) {
+		vn_clearunionwait(uvp, 0);
+		VNOP_CLOSE(uvp, fmode, context);
+		vnode_recycle(uvp);
+		vnode_put(uvp);
+		goto out;
+	}
+
+
+	/*
+	 * XX - should not ignore errors
+	 * from vnop_close
+	 */
+	error = VNOP_OPEN(lvp, FREAD, context);
+	if (error == 0) {
+		error = union_copyfile(lvp, uvp, context);
+		(void) VNOP_CLOSE(lvp, FREAD, context);
+	}
+
+	VNOP_CLOSE(uvp, fmode, context);
+	vnode_rele_ext(uvp, fmode, 0);
+	vn_clearunionwait(uvp, 0);
+
+	*vpp = uvp;
+out:
+	if ((cn.cn_flags & HASBUF) == HASBUF) {
+		FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+		cn.cn_flags &= ~HASBUF;
+	}
+	return (error);
+}
+
+
+/*
+ *	union_relookup:
+ *
+ *	dvp should be locked on entry and will be locked on return.  No
+ *	net change in the ref count will occur.
+ *
+ *	If an error is returned, *vpp will be invalid, otherwise it
+ *	will hold a locked, referenced vnode.  If *vpp == dvp then
+ *	remember that only one exclusive lock is held.
+ */
+
+/* No union lock held for this call */
 static int
-union_relookup(um, dvp, vpp, cnp, cn, path, pathlen)
-	struct union_mount *um;
-	struct vnode *dvp;
-	struct vnode **vpp;
-	struct componentname *cnp;
-	struct componentname *cn;
-	char *path;
-	int pathlen;
+union_relookup(
+#ifdef XXX_HELP_ME
+	struct union_mount *um,
+#else	/* !XXX_HELP_ME */
+	__unused struct union_mount *um,
+#endif	/* !XXX_HELP_ME */
+	struct vnode *dvp,
+	struct vnode **vpp,
+	struct componentname *cnp,
+	struct componentname *cn,
+	char *path,
+	int pathlen)
 {
 	int error;
 
@@ -754,7 +1087,7 @@ union_relookup(um, dvp, vpp, cnp, cn, path, pathlen)
 	cn->cn_pnbuf[cn->cn_namelen] = '\0';
 
 	cn->cn_nameiop = CREATE;
-	cn->cn_flags = (LOCKPARENT|HASBUF|SAVENAME|SAVESTART|ISLASTCN);
+	cn->cn_flags = (HASBUF|SAVENAME|SAVESTART|ISLASTCN );
 #ifdef XXX_HELP_ME
 	cn->cn_proc = cnp->cn_proc;
 	if (um->um_op == UNMNT_ABOVE)
@@ -764,13 +1097,12 @@ union_relookup(um, dvp, vpp, cnp, cn, path, pathlen)
 #endif
 	cn->cn_context = cnp->cn_context;	/* XXX !UNMNT_ABOVE  case ??? */
 	cn->cn_nameptr = cn->cn_pnbuf;
-	cn->cn_hash = cnp->cn_hash;
+	cn->cn_hash = 0;
 	cn->cn_consume = cnp->cn_consume;
 
 	vnode_get(dvp);
 	error = relookup(dvp, vpp, cn);
-	if (!error)
-		vnode_put(dvp);
+	vnode_put(dvp);
 
 	return (error);
 }
@@ -781,12 +1113,13 @@ union_relookup(um, dvp, vpp, cnp, cn, path, pathlen)
  *
  * (um) points to the union mount structure for access to the
  * the mounting process's credentials.
- * (dvp) is the directory in which to create the shadow directory.
- * it is unlocked on entry and exit.
- * (cnp) is the componentname to be created.
+ * (dvp) is the directory in which to create the shadow directory,
+ * It is locked (but not ref'd) on entry and return.
+ * (cnp) is the component name to be created.
  * (vpp) is the returned newly created shadow directory, which
- * is returned locked.
+ * is returned locked and ref'd
  */
+/* No union lock held for this call */
 int
 union_mkshadow(um, dvp, cnp, vpp)
 	struct union_mount *um;
@@ -798,29 +1131,37 @@ union_mkshadow(um, dvp, cnp, vpp)
 	struct vnode_attr va;
 	struct componentname cn;
 
+	bzero(&cn, sizeof(struct componentname));
+
+
 	error = union_relookup(um, dvp, vpp, cnp, &cn,
 			cnp->cn_nameptr, cnp->cn_namelen);
-	if (error)
-		return (error);
+	if (error) 
+		goto out;
 
 	if (*vpp) {
-		vnode_put(*vpp);
-		*vpp = NULLVP;
-		return (EEXIST);
+		error = EEXIST;
+		goto out;
 	}
 
 	/*
-	 * policy: when creating the shadow directory in the
+	 * Policy: when creating the shadow directory in the
 	 * upper layer, create it owned by the user who did
 	 * the mount, group from parent directory, and mode
 	 * 777 modified by umask (ie mostly identical to the
 	 * mkdir syscall).  (jsp, kb)
 	 */
+
 	VATTR_INIT(&va);
 	VATTR_SET(&va, va_type, VDIR);
 	VATTR_SET(&va, va_mode, um->um_cmode);
 
 	error = vn_create(dvp, vpp, &cn, &va, 0, cnp->cn_context);
+out:
+	if ((cn.cn_flags & HASBUF) == HASBUF) {
+		FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+		cn.cn_flags &= ~HASBUF;
+	}
 	return (error);
 }
 
@@ -833,6 +1174,7 @@ union_mkshadow(um, dvp, cnp, vpp)
  * it is locked on entry and exit.
  * (cnp) is the componentname to be created.
  */
+/* No union lock held for this call */
 int
 union_mkwhiteout(um, dvp, cnp, path)
 	struct union_mount *um;
@@ -844,55 +1186,66 @@ union_mkwhiteout(um, dvp, cnp, path)
 	struct vnode *wvp;
 	struct componentname cn;
 
+	bzero(&cn, sizeof(struct componentname));
+
 	error = union_relookup(um, dvp, &wvp, cnp, &cn, path, strlen(path));
 	if (error) {
-		return (error);
+		goto out;
 	}
 	if (wvp) {
-		vnode_put(dvp);
-		vnode_put(wvp);
-		return (EEXIST);
+		error = EEXIST;
+		goto out;
 	}
 
 	error = VNOP_WHITEOUT(dvp, &cn, CREATE, cnp->cn_context);
 
-	vnode_put(dvp);
-
+out:
+	if ((cn.cn_flags & HASBUF) == HASBUF) {
+		FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+		cn.cn_flags &= ~HASBUF;
+	}
 	return (error);
 }
 
+
 /*
  * union_vn_create: creates and opens a new shadow file
- * on the upper union layer.  this function is similar
- * in spirit to calling vn_open but it avoids calling namei().
- * the problem with calling namei is that a) it locks too many
+ * on the upper union layer.  This function is similar
+ * in spirit to calling vn_open() but it avoids calling namei().
+ * The problem with calling namei() is that a) it locks too many
  * things, and b) it doesn't start at the "right" directory,
- * whereas relookup is told where to start.
+ * whereas relookup() is told where to start.
+ *
+ * On entry, the vnode associated with un is locked.  It remains locked
+ * on return.
+ *
+ * If no error occurs, *vpp contains a locked referenced vnode for your
+ * use.  If an error occurs *vpp iis undefined.
  */
+/* called with no union lock held */
 int
-union_vn_create(vpp, un, p)
-	struct vnode **vpp;
-	struct union_node *un;
-	struct proc *p;
+union_vn_create(struct vnode **vpp, struct union_node *un, mode_t cmode, vfs_context_t  context)
 {
 	struct vnode *vp;
 	struct vnode_attr vat;
 	struct vnode_attr *vap = &vat;
-	struct vfs_context context;
 	int fmode = FFLAGS(O_WRONLY|O_CREAT|O_TRUNC|O_EXCL);
 	int error;
-	int cmode = UN_FILEMODE & ~p->p_fd->fd_cmask;
-	char *cp;
+	struct proc * p = vfs_context_proc(context);
 	struct componentname cn;
 
+	bzero(&cn, sizeof(struct componentname));
 	*vpp = NULLVP;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);
+	if (cmode == (mode_t)0)
+		cmode = UN_FILEMODE & ~p->p_fd->fd_cmask;
+	else
+		cmode = cmode & ~p->p_fd->fd_cmask;
+
 
 	/*
 	 * Build a new componentname structure (for the same
-	 * reasons outlines in union_mkshadow).
+	 * reasons outlines in union_mkshadow()).
 	 * The difference here is that the file is owned by
 	 * the current user, rather than by the person who
 	 * did the mount, since the current user needs to be
@@ -905,24 +1258,35 @@ union_vn_create(vpp, un, p)
 	cn.cn_pnlen = cn.cn_namelen+1;
 	bcopy(un->un_path, cn.cn_pnbuf, cn.cn_namelen+1);
 	cn.cn_nameiop = CREATE;
-	cn.cn_flags = (LOCKPARENT|HASBUF|SAVENAME|SAVESTART|ISLASTCN);
-	cn.cn_context = &context;
+	if (UNNODE_FAULTIN(un))
+		cn.cn_flags = (HASBUF|SAVENAME|SAVESTART|ISLASTCN|UNIONCREATED);
+	else
+		cn.cn_flags = (HASBUF|SAVENAME|SAVESTART|ISLASTCN);
+	cn.cn_context = context;
 	cn.cn_nameptr = cn.cn_pnbuf;
 	cn.cn_hash = un->un_hash;
 	cn.cn_consume = 0;
 
+	/*
+	 * Pass dvp unlocked and referenced on call to relookup().
+	 *
+	 * If an error occurs, dvp will be returned unlocked and dereferenced.
+	 */
 	vnode_get(un->un_dirvp);
-	if (error = relookup(un->un_dirvp, &vp, &cn)) {
-		kauth_cred_unref(&context.vc_ucred);
-		return (error);
+	if ((error = relookup(un->un_dirvp, &vp, &cn)) != 0) {
+		vnode_put(un->un_dirvp);
+		goto out;
 	}
 	vnode_put(un->un_dirvp);
 
+	/*
+	 * If no error occurs, dvp will be returned locked with the reference
+	 * left as before, and vpp will be returned referenced and locked.
+	 */
 	if (vp) {
-	        vnode_put(un->un_dirvp);
-		kauth_cred_unref(&context.vc_ucred);
-		vnode_put(vp);
-		return (EEXIST);
+		*vpp = vp;
+		error = EEXIST;
+		goto out;
 	}
 
 	/*
@@ -942,15 +1306,13 @@ union_vn_create(vpp, un, p)
 	VATTR_SET(vap, va_type, VREG);
 	VATTR_SET(vap, va_mode, cmode);
 
-	if (error = vn_create(un->un_dirvp, &vp, &cn, vap, 0, &context)) {
-		kauth_cred_unref(&context.vc_ucred);
-		return (error);
+	if ((error = vn_create(un->un_dirvp, &vp, &cn, vap, 0, context)) != 0) {
+		goto out;
 	}
 
-	if (error = VNOP_OPEN(vp, fmode, &context)) {
+	if ((error = VNOP_OPEN(vp, fmode, context)) != 0) {
 		vnode_put(vp);
-		kauth_cred_unref(&context.vc_ucred);
-		return (error);
+		goto out;
 	}
 
 	vnode_lock(vp);
@@ -958,42 +1320,49 @@ union_vn_create(vpp, un, p)
 		panic("union: v_writecount");
 	vnode_unlock(vp);
 	*vpp = vp;
-	kauth_cred_unref(&context.vc_ucred);
-	return (0);
+	error = 0;
+
+out:
+	if ((cn.cn_flags & HASBUF) == HASBUF) {
+		FREE_ZONE(cn.cn_pnbuf, cn.cn_pnlen, M_NAMEI);
+		cn.cn_flags &= ~HASBUF;
+	}
+	return(error);
 }
 
-int
-union_vn_close(struct vnode *vp, int fmode, kauth_cred_t cred,
-	struct proc *p)
+/* called with no union lock held */
+static int
+union_vn_close(struct vnode *vp, int fmode, vfs_context_t context)
 {
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = cred;
 
 	if (fmode & FWRITE) {
 		vnode_lock(vp);
 		--vp->v_writecount;
 		vnode_unlock(vp);
 	}
-	return (VNOP_CLOSE(vp, fmode, &context));
+	return (VNOP_CLOSE(vp, fmode, context));
 }
 
+/*
+ *	union_removed_upper:
+ *
+ *	An upper-only file/directory has been removed; un-cache it so
+ *	that unionfs vnode gets reclaimed and the last uppervp reference
+ *	disappears.
+ *
+ *	Called with union_node unlocked.
+ */
+/* always called with union lock held */
 void
 union_removed_upper(un)
 	struct union_node *un;
 {
-	struct proc *p = current_proc();	/* XXX */
-
 	union_newupper(un, NULLVP);
 	if (un->un_flags & UN_CACHED) {
 		un->un_flags &= ~UN_CACHED;
 		LIST_REMOVE(un, un_cache);
 	}
 
-	if (un->un_flags & UN_ULOCK) {
-		un->un_flags &= ~UN_ULOCK;
-	}
 }
 
 #if 0
@@ -1014,15 +1383,19 @@ union_lowervp(vp)
 #endif
 
 /*
- * determine whether a whiteout is needed
+ * Determine whether a whiteout is needed
  * during a remove/rmdir operation.
  */
+/* called with no union lock held */
 int
 union_dowhiteout(struct union_node *un, vfs_context_t ctx)
 {
 	struct vnode_attr va;
 
-	if (un->un_lowervp != NULLVP)
+	if (UNNODE_FAULTIN(un))
+		return(0);
+
+	if ((un->un_lowervp != NULLVP) )
 		return (1);
 
 	VATTR_INIT(&va);
@@ -1034,11 +1407,9 @@ union_dowhiteout(struct union_node *un, vfs_context_t ctx)
 	return (0);
 }
 
+/* called with no union lock held */
 static void
-union_dircache_r(vp, vppp, cntp)
-	struct vnode *vp;
-	struct vnode ***vppp;
-	int *cntp;
+union_dircache_r(struct vnode *vp, struct vnode ***vppp, int *cntp)
 {
 	struct union_node *un;
 
@@ -1062,53 +1433,178 @@ union_dircache_r(vp, vppp, cntp)
 		union_dircache_r(un->un_lowervp, vppp, cntp);
 }
 
+/* called with no union lock held */
 struct vnode *
-union_dircache(vp, p)
-	struct vnode *vp;
-	struct proc *p;
+union_dircache(struct vnode *vp, __unused vfs_context_t context)
 {
 	int count;
-	struct vnode *nvp;
+	struct vnode *nvp, *lvp;
 	struct vnode **vpp;
-	struct vnode **dircache;
+	struct vnode **dircache, **newdircache;
 	struct union_node *un;
 	int error;
+	int alloced = 0;
 
-	dircache = VTOUNION(vp)->un_dircache;
+	union_lock();
+	newdircache = NULL;
 
 	nvp = NULLVP;
+	un = VTOUNION(vp);
 
+	dircache = un->un_dircache;
 	if (dircache == 0) {
+		union_unlock();
 		count = 0;
 		union_dircache_r(vp, 0, &count);
 		count++;
+#if 0
+		/* too bad; we need Union now! */
+#if MAC_XXX
+                panic("MAC Framework doesn't support unionfs (yet)\n");
+#endif /* MAC */
+#endif
+
 		dircache = (struct vnode **)
 				_MALLOC(count * sizeof(struct vnode *),
 					M_TEMP, M_WAITOK);
+		newdircache = dircache;
+		alloced = 1;
 		vpp = dircache;
 		union_dircache_r(vp, &vpp, &count);
 		*vpp = NULLVP;
 		vpp = dircache + 1;
+		union_lock();
 	} else {
 		vpp = dircache;
 		do {
-			if (*vpp++ == VTOUNION(vp)->un_uppervp)
+			if (*vpp++ == un->un_uppervp)
 				break;
 		} while (*vpp != NULLVP);
 	}
 
-	if (*vpp == NULLVP)
+	lvp = *vpp;
+	union_unlock();
+	if (lvp == NULLVP) {
 		goto out;
+	}
 
-	vnode_get(*vpp);
-	error = union_allocvp(&nvp, vp->v_mount, NULLVP, NULLVP, 0, *vpp, NULLVP, 0);
-	if (error)
+	vnode_get(lvp);
+	union_lock();
+
+	error = union_allocvp(&nvp, vp->v_mount, NULLVP, NULLVP, 0, lvp, NULLVP, 0);
+	if (error) {
+		union_unlock();
+		vnode_put(lvp);
 		goto out;
+	}
 
-	VTOUNION(vp)->un_dircache = 0;
+	un->un_dircache = 0;
 	un = VTOUNION(nvp);
+#if 0
+	if ((alloced != 0) && (un->un_dircache != 0)) {
+		union_unlock();
+		for (vpp = newdircache; *vpp != NULLVP; vpp++)
+			vnode_put(*vpp);
+		_FREE(newdircache, M_TEMP);
+		newdircache = NULL;
+		union_lock();
+		if (nvp != NULLVP)
+			union_freevp(nvp);
+		goto loop;
+	}
+#endif
 	un->un_dircache = dircache;
+	un->un_flags |= UN_DIRENVN;
+	
+	newdircache = NULL;
+	union_unlock();
+	return (nvp);
 
 out:
-	return (nvp);
+	/*
+	 * If we allocated a new dircache and couldn't attach
+	 * it to a new vp, free the resources we allocated.
+	 */
+	if (newdircache) {
+		for (vpp = newdircache; *vpp != NULLVP; vpp++)
+			vnode_put(*vpp);
+		_FREE(newdircache, M_TEMP);
+	}
+	return (NULLVP);
+}
+
+/*
+ * Module glue to remove #ifdef UNION from vfs_syscalls.c
+ */
+/* Called with no union lock, the union_dircache takes locks when necessary */
+static int
+union_dircheck(struct vnode **vpp, struct fileproc *fp, vfs_context_t ctx)
+{
+	int error = 0;
+	vnode_t vp = *vpp;
+	
+	if (vp->v_op == union_vnodeop_p) {
+		struct vnode *lvp;
+
+		lvp = union_dircache(vp, ctx);
+		if (lvp != NULLVP) {
+			struct vnode_attr va;
+			/*
+			 * If the directory is opaque,
+			 * then don't show lower entries
+			 */
+			VATTR_INIT(&va);
+			VATTR_WANTED(&va, va_flags);
+			error = vnode_getattr(vp, &va, ctx);
+			if (va.va_flags & OPAQUE) {
+				vnode_put(lvp);
+				lvp = NULL;
+			}
+		}
+
+		if (lvp != NULLVP) {
+#if CONFIG_MACF
+			error = mac_vnode_check_open(ctx, lvp, FREAD);
+			if (error) {
+				vnode_put(lvp);
+				return(error);
+			}
+#endif /* MAC */
+			error = VNOP_OPEN(lvp, FREAD, ctx);
+			if (error) {
+				vnode_put(lvp);
+				return(error);
+			}
+			vnode_ref(lvp);
+			fp->f_fglob->fg_data = (caddr_t) lvp;
+			fp->f_fglob->fg_offset = 0;
+
+			error = VNOP_CLOSE(vp, FREAD, ctx);
+			vnode_rele(vp);
+			vnode_put(vp);
+			if (error)
+				return(error);
+
+			*vpp = lvp;
+			return -1;	/* goto unionread */
+		}
+	}
+	return error;
 }
+
+/*  called from inactive with union lock held */
+void
+union_dircache_free(struct union_node *un)
+{
+	struct vnode **vpp;
+
+	vpp = un->un_dircache;
+	un->un_dircache = NULL;
+	union_unlock();	
+	
+	for (; *vpp != NULLVP; vpp++)
+		vnode_put(*vpp);
+	_FREE(un->un_dircache, M_TEMP);
+	union_lock();
+}
+
diff --git a/bsd/miscfs/union/union_vfsops.c b/bsd/miscfs/union/union_vfsops.c
index 6e0dfce28..05b4dfa9e 100644
--- a/bsd/miscfs/union/union_vfsops.c
+++ b/bsd/miscfs/union/union_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -77,12 +83,13 @@
 #include <sys/queue.h>
 #include <miscfs/union/union.h>
 
-static	int union_itercallback(__unused vnode_t, void *);
+static	int union_itercallback(vnode_t, void *);
+static int union_root(mount_t, vnode_t *, vfs_context_t);
 
 /*
  * Mount union filesystem
  */
-int
+static int
 union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t context)
 {
 	proc_t p = vfs_context_proc(context);
@@ -90,9 +97,10 @@ union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t
 	struct user_union_args args;
 	struct vnode *lowerrootvp = NULLVP;
 	struct vnode *upperrootvp = NULLVP;
-	struct union_mount *um = 0;
+	struct union_mount *um = NULL;
 	kauth_cred_t cred = NOCRED;
-	char *cp;
+	const char *cp = NULL;
+	char *vcp;
 	int len;
 	u_int size;
 	struct nameidata nd;
@@ -152,8 +160,6 @@ union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t
 		goto bad;
 	}
 	
-//	um = (struct union_mount *) malloc(sizeof(struct union_mount),
-//				M_UFSMNT, M_WAITOK);	/* XXX */
 	MALLOC(um, struct union_mount *, sizeof(struct union_mount),
 				M_UFSMNT, M_WAITOK);
 
@@ -185,16 +191,32 @@ union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t
 		um->um_lowervp = lowerrootvp;
 		break;
 
+#ifdef FAULTFS
+	case UNMNT_FAULTIN:
+		um->um_lowervp = upperrootvp;
+		um->um_uppervp = lowerrootvp;
+		break;
+#endif
+
 	default:
 		error = EINVAL;
 		goto bad;
 	}
 
+	if (um->um_lowervp != NULLVP)
+		um->um_lowervid = vnode_vid(um->um_lowervp);
+	if (um->um_uppervp != NULLVP)
+		um->um_uppervid = vnode_vid(um->um_uppervp);
 	/*
 	 * Unless the mount is readonly, ensure that the top layer
 	 * supports whiteout operations
 	 */
-	if ((mp->mnt_flag & MNT_RDONLY) == 0) {
+#ifdef FAULTFS
+	if ((um->um_op != UNMNT_FAULTIN) && (mp->mnt_flag & MNT_RDONLY) == 0)
+#else
+	if ((mp->mnt_flag & MNT_RDONLY) == 0)
+#endif
+	{
 		error = VNOP_WHITEOUT(um->um_uppervp, (struct componentname *) 0,
 		                      LOOKUP, context);
 		if (error)
@@ -243,15 +265,23 @@ union_mount(mount_t mp, __unused vnode_t devvp, user_addr_t data, vfs_context_t
 	case UNMNT_REPLACE:
 		cp = "";
 		break;
+#ifdef FAULTFS
+	case UNMNT_FAULTIN:
+		cp = "/FaultingFS/";
+		break;
+#endif
 	}
 	len = strlen(cp);
 	bcopy(cp, mp->mnt_vfsstat.f_mntfromname, len);
 
-	cp = mp->mnt_vfsstat.f_mntfromname + len;
+	vcp = mp->mnt_vfsstat.f_mntfromname + len;
 	len = MNAMELEN - len;
 
-	(void) copyinstr(args.target, cp, len - 1, (size_t *)&size);
-	bzero(cp + size, len - size);
+	(void) copyinstr(args.target, vcp, len - 1, (size_t *)&size);
+	bzero(vcp + size, len - size);
+
+	/* mark the filesystem thred safe */
+	 mp->mnt_vtable->vfc_threadsafe = TRUE;
 
 #ifdef UNION_DIAGNOSTIC
 	printf("union_mount: from %s, on %s\n",
@@ -276,7 +306,7 @@ bad:
  * on the underlying filesystem(s) will have been called
  * when that filesystem was mounted.
  */
-int
+static int
 union_start(__unused struct mount *mp, __unused int flags, __unused vfs_context_t context)
 {
 
@@ -297,15 +327,14 @@ union_itercallback(__unused vnode_t vp, void *args)
 /*
  * Free reference to union layer
  */
-int
-union_unmount(mount_t mp, int mntflags, __unused vfs_context_t context)
+static int
+union_unmount(mount_t mp, int mntflags, vfs_context_t context)
 {
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
 	struct vnode *um_rootvp;
 	int error;
 	int freeing;
 	int flags = 0;
-	kauth_cred_t cred;
 
 #ifdef UNION_DIAGNOSTIC
 	printf("union_unmount(mp = %x)\n", mp);
@@ -314,7 +343,7 @@ union_unmount(mount_t mp, int mntflags, __unused vfs_context_t context)
 	if (mntflags & MNT_FORCE)
 		flags |= FORCECLOSE;
 
-	if ((error = union_root(mp, &um_rootvp)))
+	if ((error = union_root(mp, &um_rootvp, context)))
 		return (error);
 
 	/*
@@ -369,11 +398,11 @@ union_unmount(mount_t mp, int mntflags, __unused vfs_context_t context)
 	 * Finally, throw away the union_mount structure
 	 */
 	_FREE(mp->mnt_data, M_UFSMNT);	/* XXX */
-	mp->mnt_data = 0;
+	mp->mnt_data = NULL;
 	return (0);
 }
 
-int
+static int
 union_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
 {
 	struct union_mount *um = MOUNTTOUNIONMOUNT(mp);
@@ -385,6 +414,8 @@ union_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
 	vnode_get(um->um_uppervp);
 	if (um->um_lowervp)
 		vnode_get(um->um_lowervp);
+
+	union_lock();
 	error = union_allocvp(vpp, mp,
 			      (struct vnode *) 0,
 			      (struct vnode *) 0,
@@ -392,6 +423,7 @@ union_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
 			      um->um_uppervp,
 			      um->um_lowervp,
 			      1);
+	union_unlock();
 
 	if (error) {
 	        vnode_put(um->um_uppervp);
@@ -507,10 +539,9 @@ union_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 /*
  * XXX - Assumes no data cached at union layer.
  */
-#define union_sync (int (*) (mount_t, int, ucred_t, vfs_context_t))nullop
+#define union_sync (int (*) (mount_t, int, vfs_context_t))nullop
 
 #define union_fhtovp (int (*) (mount_t, int, unsigned char *, vnode_t *, vfs_context_t))eopnotsupp
-int union_init (struct vfsconf *);
 #define union_sysctl (int (*) (int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t))eopnotsupp
 #define union_vget (int (*) (mount_t, ino64_t, vnode_t *, vfs_context_t))eopnotsupp
 #define union_vptofh (int (*) (vnode_t, int *, unsigned char *, vfs_context_t))eopnotsupp
@@ -527,5 +558,9 @@ struct vfsops union_vfsops = {
 	union_fhtovp,
 	union_vptofh,
 	union_init,
-	union_sysctl
+	union_sysctl,
+	NULL,
+	{NULL}
 };
+
+
diff --git a/bsd/miscfs/union/union_vnops.c b/bsd/miscfs/union/union_vnops.c
index 4b1ca8ef1..240ab9f40 100644
--- a/bsd/miscfs/union/union_vnops.c
+++ b/bsd/miscfs/union/union_vnops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -77,23 +83,10 @@
 #include <miscfs/union/union.h>
 #include <vfs/vfs_support.h>
 #include <sys/ubc.h>
+#include <sys/kdebug.h>
 #include <sys/uio_internal.h>
 
-#define FIXUP(un, p) { \
-	if (((un)->un_flags & UN_ULOCK) == 0) { \
-		union_fixup(un, p); \
-	} \
-}
-
-static void
-union_fixup(un, p)
-	struct union_node *un;
-	struct proc *p;
-{
-
-	un->un_flags |= UN_ULOCK;
-}
-
+/* called with no union lock held */
 static int
 union_lookup1(struct vnode *udvp, struct vnode **dvpp, struct vnode **vpp,
 	struct componentname *cnp)
@@ -158,30 +151,35 @@ union_lookup1(struct vnode *udvp, struct vnode **dvpp, struct vnode **vpp,
 	return (0);
 }
 
-int
-union_lookup(
-	struct vnop_lookup_args /* {
+static int
+union_lookup(struct vnop_lookup_args *ap)
+/*
+	struct vnop_lookup_args {
 		struct vnodeop_desc *a_desc;
 		struct vnode *a_dvp;
 		struct vnode **a_vpp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap)
+*/
 {
 	int error;
-	int uerror, lerror;
+	int uerror = 0;
+	int  lerror = 0;
 	struct vnode *uppervp, *lowervp;
 	struct vnode *upperdvp, *lowerdvp;
 	struct vnode *dvp = ap->a_dvp;
-	struct union_node *dun = VTOUNION(dvp);
+	struct union_node *dun;
 	struct componentname *cnp = ap->a_cnp;
 	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 	int lockparent = cnp->cn_flags & LOCKPARENT;
-	struct union_mount *um = MOUNTTOUNIONMOUNT(dvp->v_mount);
+	struct union_mount *um;
 	kauth_cred_t saved_cred;
 	int iswhiteout;
 	struct vnode_attr va;
+	int isfaultfs = 0;
+	int upperlookup = 0;
+	int retry_count = 0;
 
 #ifdef notyet
 	if (cnp->cn_namelen == 3 &&
@@ -197,14 +195,26 @@ union_lookup(
 	}
 #endif
 
-	cnp->cn_flags |= LOCKPARENT;
+	
 
+retry:
+	union_lock();
+	um = MOUNTTOUNIONMOUNT(dvp->v_mount);
+	dun = VTOUNION(dvp);
 	upperdvp = dun->un_uppervp;
 	lowerdvp = dun->un_lowervp;
 	uppervp = NULLVP;
 	lowervp = NULLVP;
 	iswhiteout = 0;
 
+	union_unlock();
+
+	if(UNION_FAULTIN(um))
+		isfaultfs = 1;
+
+	if (isfaultfs == 0)
+		cnp->cn_flags |= LOCKPARENT;
+
 	/*
 	 * do the lookup in the upper level.
 	 * if that level comsumes additional pathnames,
@@ -212,16 +222,18 @@ union_lookup(
 	 * on and just return that vnode.
 	 */
 	if (upperdvp != NULLVP) {
-		FIXUP(dun, p);
+		if (lockparent != 0) 
+			cnp->cn_flags &= ~LOCKPARENT;
 		uerror = union_lookup1(um->um_uppervp, &upperdvp,
 					&uppervp, cnp);
-		/*if (uppervp == upperdvp)
-			dun->un_flags |= UN_KLOCK;*/
+		upperlookup = 1;
 
 		if (cnp->cn_consume != 0) {
 			*ap->a_vpp = uppervp;
 			if (!lockparent)
 				cnp->cn_flags &= ~LOCKPARENT;
+			else
+				cnp->cn_flags |= LOCKPARENT;	
 			return (uerror);
 		}
 		if (uerror == ENOENT || uerror == EJUSTRETURN) {
@@ -239,6 +251,15 @@ union_lookup(
 		uerror = ENOENT;
 	}
 
+	/*
+	 * faultingfs: If upper layer lookup is succesful
+	 * we will return that vp if it is regular file.
+	 * So so skip lower level lookup
+	 */
+
+	if ((isfaultfs == 1) && (upperlookup == 1) && (uerror == 0) && ((vnode_isreg(uppervp) != 0)))
+		goto donelowerlookup;
+
 	/*
 	 * in a similar way to the upper layer, do the lookup
 	 * in the lower layer.   this time, if there is some
@@ -259,10 +280,14 @@ union_lookup(
 			/* XXX BOGUS */
 			saved_cred = cnp->cn_context->vc_ucred;
 			cnp->cn_context->vc_ucred = um->um_cred;
+			if (lockparent != 0) 
+				cnp->cn_flags &= ~LOCKPARENT;
 			lerror = union_lookup1(um->um_lowervp, &lowerdvp,
 					&lowervp, cnp);
 			cnp->cn_context->vc_ucred = saved_cred;
 		} else {
+			if (lockparent != 0) 
+				cnp->cn_flags &= ~LOCKPARENT;
 			lerror = union_lookup1(um->um_lowervp, &lowerdvp,
 					&lowervp, cnp);
 		}
@@ -276,6 +301,8 @@ union_lookup(
 			*ap->a_vpp = lowervp;
 			if (!lockparent)
 				cnp->cn_flags &= ~LOCKPARENT;
+			else
+				cnp->cn_flags |= LOCKPARENT;	
 			return (lerror);
 		}
 	} else {
@@ -283,12 +310,13 @@ union_lookup(
 		if ((cnp->cn_flags & ISDOTDOT) && dun->un_pvp != NULLVP) {
 			lowervp = LOWERVP(dun->un_pvp);
 			if (lowervp != NULLVP) {
-				vnode_get(lowervp);
 				lerror = 0;
 			}
 		}
 	}
 
+donelowerlookup:
+
 	if (!lockparent)
 		cnp->cn_flags &= ~LOCKPARENT;
 
@@ -320,27 +348,72 @@ union_lookup(
 
 	/* case 1. */
 	if ((uerror != 0) && (lerror != 0)) {
+			if (!lockparent)
+				cnp->cn_flags &= ~LOCKPARENT;
+			else
+				cnp->cn_flags |= LOCKPARENT;	
 		return (uerror);
 	}
 
 	/* case 2. */
 	if (uerror != 0 /* && (lerror == 0) */ ) {
 		if (lowervp->v_type == VDIR) { /* case 2b. */
-			dun->un_flags &= ~UN_ULOCK;
+			/* No need to lock the union here */
+			/* if the vnode exists it returns it even if it marks error */
+
+			uppervp = NULLVP;
+
 			uerror = union_mkshadow(um, upperdvp, cnp, &uppervp);
-			dun->un_flags |= UN_ULOCK;
 
+			if ((uerror == EEXIST)){
+				if (uppervp == NULLVP) {
+					retry_count++;
+					if (retry_count <= 2) {
+						if (lowervp != NULLVP) 
+							vnode_put(lowervp);
+						goto retry;
+					}
+				}
+				uerror = 0;
+			}
+			
 			if (uerror) {
+				if (uppervp != NULLVP) {
+					vnode_put(uppervp);
+				}
 				if (lowervp != NULLVP) {
 					vnode_put(lowervp);
-					lowervp = NULLVP;
 				}
+				if (!lockparent)
+					cnp->cn_flags &= ~LOCKPARENT;
+				else
+					cnp->cn_flags |= LOCKPARENT;	
 				return (uerror);
 			}
+		} else if ((lowervp->v_type == VREG) && (isfaultfs == 1)) {
+			error = union_faultin_copyup(&uppervp, upperdvp, lowervp, cnp, ctx);	
+			uerror = 0;
 		}
 	}
+
+
+	/* if this is faulting filesystem and upper vp exisits skip allocation of union node */
+	if ((isfaultfs == 1) && (uerror == 0) && (uppervp != NULLVP) &&  ((vnode_isreg(uppervp) != 0)|| (vnode_islnk(uppervp) != 0))) {
+		vn_checkunionwait(uppervp);
+		*ap->a_vpp = uppervp;
+		if (lowervp != NULLVP)
+			vnode_put(lowervp);
+		if (!lockparent)
+			cnp->cn_flags &= ~LOCKPARENT;
+		else
+			cnp->cn_flags |= LOCKPARENT;	
+		return(0);
+	}
+
+	union_lock();
 	error = union_allocvp(ap->a_vpp, dvp->v_mount, dvp, upperdvp, cnp,
 			      uppervp, lowervp, 1);
+	union_unlock();
 
 	if (error) {
 		if (uppervp != NULLVP)
@@ -349,33 +422,35 @@ union_lookup(
 			vnode_put(lowervp);
 	}
 
+	if (!lockparent)
+		cnp->cn_flags &= ~LOCKPARENT;
+	else
+		cnp->cn_flags |= LOCKPARENT;	
 	return (error);
 }
 
-int
-union_create(
-	struct vnop_create_args /* {
+static int
+union_create(struct vnop_create_args *ap)
+/*
+	struct vnop_create_args {
 		struct vnode *a_dvp;
 		struct vnode **a_vpp;
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_dvp);
 	struct vnode *dvp = un->un_uppervp;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 
 	if (dvp != NULLVP) {
 		int error;
 		struct vnode *vp;
 		struct mount *mp;
 
-		FIXUP(un, p);
 
-		un->un_flags |= UN_KLOCK;
 		mp = ap->a_dvp->v_mount;
 
 		/* note that this is a direct passthrough to the filesystem */
@@ -383,60 +458,71 @@ union_create(
 		if (error)
 			return (error);
 
+		/* if this is faulting filesystem and is a reg file, skip allocation of union node */
+		if (UNNODE_FAULTIN(un) && (vp != NULLVP) && ((vnode_isreg(vp) != 0)|| (vnode_islnk(vp) != 0))) {
+			*ap->a_vpp = vp;
+			return(0);
+		}
+
+
+		union_lock();
 		error = union_allocvp(ap->a_vpp, mp, NULLVP, NULLVP, cnp, vp,
 				NULLVP, 1);
+		union_unlock();
 		if (error)
 			vnode_put(vp);
 		return (error);
 	}
+
 	return (EROFS);
 }
 
-int
-union_whiteout(
-	struct vnop_whiteout_args /* {
+static int
+union_whiteout(struct vnop_whiteout_args *ap)
+/*
+	struct vnop_whiteout_args {
 		struct vnode *a_dvp;
 		struct componentname *a_cnp;
 		int a_flags;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_dvp);
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
+	int error;
 
-	if (un->un_uppervp == NULLVP)
+	if (un->un_uppervp == NULLVP) {
 		return (ENOTSUP);
+	}
 
-	FIXUP(un, p);
-	return (VNOP_WHITEOUT(un->un_uppervp, cnp, ap->a_flags, ap->a_context));
+	error =  (VNOP_WHITEOUT(un->un_uppervp, cnp, ap->a_flags, ap->a_context));
+	return(error);
 }
 
-int
-union_mknod(
-	struct vnop_mknod_args /* {
+/* mknod can do  fifos, chr, blk or whiteout entries */
+static int
+union_mknod(struct vnop_mknod_args *ap)
+/*
+	struct vnop_mknod_args {
 		struct vnode *a_dvp;
 		struct vnode **a_vpp;
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_dvp);
 	struct vnode *dvp = un->un_uppervp;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 
 	if (dvp != NULLVP) {
 		int error;
 		struct vnode *vp;
 		struct mount *mp;
 
-		FIXUP(un, p);
 
-		un->un_flags |= UN_KLOCK;
 		mp = ap->a_dvp->v_mount;
 
 		/* note that this is a direct passthrough to the filesystem */
@@ -445,8 +531,10 @@ union_mknod(
 			return (error);
 
 		if (vp != NULLVP) {
+			union_lock();
 			error = union_allocvp(ap->a_vpp, mp, NULLVP, NULLVP,
 					cnp, vp, NULLVP, 1);
+			union_unlock();
 			if (error)
 				vnode_put(vp);
 		}
@@ -455,27 +543,29 @@ union_mknod(
 	return (EROFS);
 }
 
-int
-union_open(
-	struct vnop_open_args /* {
+static int
+union_open(struct vnop_open_args *ap)
+/*
+	struct vnop_open_args {
 		struct vnodeop_desc *a_desc;
 		struct vnode *a_vp;
 		int a_mode;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct vnode *tvp;
 	int mode = ap->a_mode;
-	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
-	struct proc *p = vfs_context_proc(ap->a_context);
 	int error;
 
 	/*
 	 * If there is an existing upper vp then simply open that.
 	 */
+
 	tvp = un->un_uppervp;
 	if (tvp == NULLVP) {
+
 		/*
 		 * If the lower vnode is being opened for writing, then
 		 * copy the file contents to the upper vnode and open that,
@@ -483,7 +573,19 @@ union_open(
 		 */
 		tvp = un->un_lowervp;
 		if ((ap->a_mode & FWRITE) && (tvp->v_type == VREG)) {
-			error = union_copyup(un, (mode&O_TRUNC) == 0, cred, p);
+			/* For above below mounts we need draining.. */
+			/* This path is not taken for faultin mode */
+			/*  LOCK the union node as well **/
+			union_lock();
+			un->un_flags |= UN_LOCKED;
+
+			error = union_copyup(un, (mode&O_TRUNC) == 0, ap->a_context);
+			un->un_flags &= ~UN_LOCKED;
+			if ((un->un_flags & UN_WANT) == UN_WANT) {
+				un->un_flags &=  ~UN_WANT;
+				wakeup(&un->un_flags);
+			}
+			union_unlock();
 			if (error == 0)
 				error = VNOP_OPEN(un->un_uppervp, mode, ap->a_context);
 			return (error);
@@ -499,23 +601,24 @@ union_open(
 		return (error);
 	}
 
-	FIXUP(un, p);
-
 	error = VNOP_OPEN(tvp, mode, ap->a_context);
 
 	return (error);
 }
 
-int
-union_close(ap)
-	struct vnop_close_args /* {
+static int
+union_close(struct vnop_close_args *ap)
+/*
+	struct vnop_close_args {
 		struct vnode *a_vp;
 		int  a_fflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct vnode *vp;
+	int error = 0;
 
 	if ((vp = un->un_uppervp) == NULLVP) {
 #ifdef UNION_DIAGNOSTIC
@@ -527,7 +630,8 @@ union_close(ap)
 	}
 
 	ap->a_vp = vp;
-	return (VCALL(vp, VOFFSET(vnop_close), ap));
+	error =  (VCALL(vp, VOFFSET(vnop_close), ap));
+	return(error);
 }
 
 /*
@@ -538,22 +642,22 @@ union_close(ap)
  * file permissions are given away simply because
  * the user caused an implicit file copy.
  */
-int
-union_access(
-	struct vnop_access_args /* {
+static int
+union_access(struct vnop_access_args *ap)
+/*
+	struct vnop_access_args {
 		struct vnodeop_desc *a_desc;
 		struct vnode *a_vp;
 		int a_action;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_vp);
-	struct proc *p = vfs_context_proc(ap->a_context);
 	int error = EACCES;
 	struct vnode *vp;
 
 	if ((vp = un->un_uppervp) != NULLVP) {
-		FIXUP(un, p);
 		ap->a_vp = vp;
 		return (VCALL(vp, VOFFSET(vnop_access), ap));
 	}
@@ -565,8 +669,6 @@ union_access(
 			struct union_mount *um = MOUNTTOUNIONMOUNT(vp->v_mount);
 
 			if (um->um_op == UNMNT_BELOW) {
-				/* XXX fix me */
-			//	ap->a_cred = um->um_cred;
 				error = VCALL(vp, VOFFSET(vnop_access), ap);
 			}
 		}
@@ -581,18 +683,19 @@ union_access(
  * We handle getattr only to change the fsid and
  * track object sizes
  */
-int
-union_getattr(ap)
-	struct vnop_getattr_args /* {
+static int
+union_getattr(struct vnop_getattr_args *ap)
+/*
+	struct vnop_getattr_args {
 		struct vnode *a_vp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
-	int error;
+	int error=0;
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct vnode *vp = un->un_uppervp;
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct vnode_attr *vap;
 	struct vnode_attr va;
 
@@ -618,13 +721,14 @@ union_getattr(ap)
 		 * In the mean time, compensate here by checking
 		 * the union_node's lock flag.
 		 */
-		if (un->un_flags & UN_LOCKED)
-			FIXUP(un, p);
 
 		error = vnode_getattr(vp, vap, ap->a_context);
-		if (error)
+		if (error) {
 			return (error);
+		}
+		union_lock();
 		union_newsize(ap->a_vp, vap->va_data_size, VNOVAL);
+		union_unlock();
 	}
 
 	if (vp == NULLVP) {
@@ -641,9 +745,12 @@ union_getattr(ap)
 
 	if (vp != NULLVP) {
 		error = vnode_getattr(vp, vap, ap->a_context);
-		if (error)
+		if (error) {
 			return (error);
+		}
+		union_lock();
 		union_newsize(ap->a_vp, VNOVAL, vap->va_data_size);
+		union_unlock();
 	}
 
 	if ((vap != ap->a_vap) && (vap->va_type == VDIR))
@@ -653,17 +760,17 @@ union_getattr(ap)
 	return (0);
 }
 
-int
-union_setattr(ap)
-	struct vnop_setattr_args /* {
+static int
+union_setattr(struct vnop_setattr_args *ap)
+/*
+	struct vnop_setattr_args {
 		struct vnode *a_vp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_vp);
-	struct proc *p = vfs_context_proc(ap->a_context);
-	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
 	int error;
 
 	/*
@@ -675,9 +782,12 @@ union_setattr(ap)
 	    (un->un_uppervp == NULLVP) &&
 	    /* assert(un->un_lowervp != NULLVP) */
 	    (un->un_lowervp->v_type == VREG)) {
-		error = union_copyup(un, (ap->a_vap->va_data_size != 0), cred, p);
-		if (error)
+		union_lock();
+		error = union_copyup(un, (ap->a_vap->va_data_size != 0), ap->a_context);
+		union_unlock();
+		if (error) {
 			return (error);
+		}
 	}
 
 	/*
@@ -685,10 +795,12 @@ union_setattr(ap)
 	 * otherwise return read-only filesystem error.
 	 */
 	if (un->un_uppervp != NULLVP) {
-		FIXUP(un, p);
 		error = vnode_setattr(un->un_uppervp, ap->a_vap, ap->a_context);
-		if ((error == 0) && VATTR_IS_ACTIVE(ap->a_vap, va_data_size))
+		if ((error == 0) && VATTR_IS_ACTIVE(ap->a_vap, va_data_size)) {
+			union_lock();
 			union_newsize(ap->a_vp, ap->a_vap->va_data_size, VNOVAL);
+			union_unlock();
+		}
 	} else {
 		error = EROFS;
 	}
@@ -696,22 +808,20 @@ union_setattr(ap)
 	return (error);
 }
 
-int
-union_read(ap)
-	struct vnop_read_args /* {
+static int
+union_read(struct vnop_read_args *ap)
+/*
+	struct vnop_read_args {
 		struct vnode *a_vp;
 		struct uio *a_uio;
 		int  a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct vnode *vp = OTHERVP(ap->a_vp);
-	int dolock = (vp == LOWERVP(ap->a_vp));
 
-	if (!dolock)
-		FIXUP(VTOUNION(ap->a_vp), p);
 	error = VNOP_READ(vp, ap->a_uio, ap->a_ioflag, ap->a_context);
 
 	/*
@@ -725,36 +835,42 @@ union_read(ap)
 		off_t cur = ap->a_uio->uio_offset;
 
 		if (vp == un->un_uppervp) {
-			if (cur > un->un_uppersz)
+			if (cur > un->un_uppersz) {
+				union_lock();
 				union_newsize(ap->a_vp, cur, VNOVAL);
+				union_unlock();
+			}
 		} else {
-			if (cur > un->un_lowersz)
+			if (cur > un->un_lowersz) {
+				union_lock();
 				union_newsize(ap->a_vp, VNOVAL, cur);
+				union_unlock();
+			}
 		}
 	}
 
 	return (error);
 }
 
-int
-union_write(ap)
-	struct vnop_read_args /* {
+static int
+union_write(struct vnop_write_args *ap)
+/*
+	struct vnop_write_args {
 		struct vnode *a_vp;
 		struct uio *a_uio;
 		int  a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 	struct vnode *vp;
 	struct union_node *un = VTOUNION(ap->a_vp);
-	struct proc *p = vfs_context_proc(ap->a_context);
 
 	vp = UPPERVP(ap->a_vp);
 	if (vp == NULLVP)
 		panic("union: missing upper layer in write");
 
-	FIXUP(un, p);
 	error = VNOP_WRITE(vp, ap->a_uio, ap->a_ioflag, ap->a_context);
 
 	/*
@@ -764,23 +880,28 @@ union_write(ap)
 	if (error == 0) {
 		off_t cur = ap->a_uio->uio_offset;
 
-		if (cur > un->un_uppersz)
+		if (cur > un->un_uppersz) {
+			union_lock();
 			union_newsize(ap->a_vp, cur, VNOVAL);
+			union_unlock();
+		}
 	}
 
 	return (error);
 }
 
 
-int
-union_ioctl(ap)
-	struct vnop_ioctl_args /* {
+static int
+union_ioctl(struct vnop_ioctl_args *ap)
+/*
+	struct vnop_ioctl_args {
 		struct vnode *a_vp;
 		int  a_command;
 		caddr_t  a_data;
 		int  a_fflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	register struct vnode *ovp = OTHERVP(ap->a_vp);
 
@@ -788,15 +909,17 @@ union_ioctl(ap)
 	return (VCALL(ovp, VOFFSET(vnop_ioctl), ap));
 }
 
-int
-union_select(ap)
-	struct vnop_select_args /* {
+static int
+union_select(struct vnop_select_args *ap)
+/*
+	struct vnop_select_args {
 		struct vnode *a_vp;
 		int  a_which;
 		int  a_fflags;
 		void * a_wql;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	register struct vnode *ovp = OTHERVP(ap->a_vp);
 
@@ -804,13 +927,15 @@ union_select(ap)
 	return (VCALL(ovp, VOFFSET(vnop_select), ap));
 }
 
-int
-union_revoke(ap)
-	struct vnop_revoke_args /* {
+static int
+union_revoke(struct vnop_revoke_args *ap)
+/*
+	struct vnop_revoke_args {
 		struct vnode *a_vp;
 		int a_flags;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	struct vnode *vp = ap->a_vp;
 
@@ -819,16 +944,20 @@ union_revoke(ap)
 	if (LOWERVP(vp))
 		VNOP_REVOKE(LOWERVP(vp), ap->a_flags, ap->a_context);
 	vnode_reclaim(vp);
+
+	return (0);
 }
 
-int
-union_mmap(ap)
-	struct vnop_mmap_args /* {
+static int
+union_mmap(struct vnop_mmap_args *ap)
+/*
+	struct vnop_mmap_args {
 		struct vnode *a_vp;
 		int  a_fflags;
 		kauth_cred_t a_cred;
 		struct proc *a_p;
-	} */ *ap;
+	} *ap;
+*/
 {
 	register struct vnode *ovp = OTHERVP(ap->a_vp);
 
@@ -836,64 +965,102 @@ union_mmap(ap)
 	return (VCALL(ovp, VOFFSET(vnop_mmap), ap));
 }
 
-int
-union_fsync(
-	struct vnop_fsync_args /* {
+static int
+union_mnomap(struct vnop_mnomap_args *ap)
+/*
+	struct vnop_mnomap_args {
+		struct vnode *a_vp;
+		int  a_fflags;
+		kauth_cred_t a_cred;
+		struct proc *a_p;
+	} *ap;
+*/
+{
+	register struct vnode *ovp = OTHERVP(ap->a_vp);
+
+	ap->a_vp = ovp;
+	return (VCALL(ovp, VOFFSET(vnop_mnomap), ap));
+}
+
+static int
+union_fsync(struct vnop_fsync_args *ap)
+/*
+	struct vnop_fsync_args {
 		struct vnode *a_vp;
 		int  a_waitfor;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	int error = 0;
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct vnode *targetvp = OTHERVP(ap->a_vp);
 
 	if (targetvp != NULLVP) {
-		int dolock = (targetvp == LOWERVP(ap->a_vp));
 
-		if (!dolock)
-			FIXUP(VTOUNION(ap->a_vp), p);
 		error = VNOP_FSYNC(targetvp, ap->a_waitfor, ap->a_context);
 	}
 
 	return (error);
 }
 
-int
-union_remove(
-	struct vnop_remove_args /* {
+static int
+union_remove(struct vnop_remove_args *ap)
+/*
+	struct vnop_remove_args {
 		struct vnode *a_dvp;
 		struct vnode *a_vp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
-	int error;
+	int error, flags;
 	struct union_node *dun = VTOUNION(ap->a_dvp);
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
+	int busydel = 0;
 
 	if (dun->un_uppervp == NULLVP)
 		panic("union remove: null upper vnode");
 
+	if (UNNODE_FAULTIN(dun) && ((ap->a_vp != NULLVP) &&
+		((vnode_isreg(ap->a_vp) != 0) || (vnode_islnk(ap->a_vp) != 0)))) {
+			return(VNOP_REMOVE(dun->un_uppervp, ap->a_vp, ap->a_cnp, ap->a_flags, ap->a_context));
+	}
+
 	if (un->un_uppervp != NULLVP) {
 		struct vnode *dvp = dun->un_uppervp;
 		struct vnode *vp = un->un_uppervp;
 
-		FIXUP(dun, p);
-		dun->un_flags |= UN_KLOCK;
-		FIXUP(un, p);
-		un->un_flags |= UN_KLOCK;
-
+		flags = ap->a_flags;
+		if (vnode_isinuse(ap->a_vp, 0))
+			busydel = 1;
+		if ((flags & VNODE_REMOVE_NODELETEBUSY) && (busydel != 0)) {
+				return(EBUSY);
+		}
 		if (union_dowhiteout(un, cnp->cn_context))
 			cnp->cn_flags |= DOWHITEOUT;
+	
+		if (busydel != 0)  {
+			union_lock();
+			un->un_flags |= UN_DELETED;
+			if (un->un_flags & UN_CACHED) {
+				un->un_flags &= ~UN_CACHED;
+				LIST_REMOVE(un, un_cache);
+			}
+			union_unlock();
+			vnode_ref(vp);
+		}
 		error = VNOP_REMOVE(dvp, vp, cnp, 0, ap->a_context);
-		if (!error)
-			union_removed_upper(un);
+		if (!error) {
+			union_lock();
+			if (busydel == 0)
+				union_removed_upper(un);
+			union_unlock();
+		}
 	} else {
-		FIXUP(dun, p);
+		if (UNNODE_FAULTIN(un))
+			panic("faultfs: No uppervp");
 		error = union_mkwhiteout(
 			MOUNTTOUNIONMOUNT(UNIONTOV(dun)->v_mount),
 			dun->un_uppervp, ap->a_cnp, un->un_path);
@@ -902,19 +1069,19 @@ union_remove(
 	return (error);
 }
 
-int
-union_link(
-	struct vnop_link_args /* {
+static int
+union_link(struct vnop_link_args *ap)
+/*
+	struct vnop_link_args {
 		struct vnode *a_vp;
 		struct vnode *a_tdvp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	int error = 0;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 	struct union_node *un;
 	struct vnode *vp;
 	struct vnode *tdvp;
@@ -926,13 +1093,20 @@ union_link(
 	} else {
 		struct union_node *tun = VTOUNION(ap->a_vp);
 		if (tun->un_uppervp == NULLVP) {
+			if (UNNODE_FAULTIN(tun))
+				panic("faultfs: No uppervp");
 			if (un->un_uppervp == tun->un_dirvp) {
-				un->un_flags &= ~UN_ULOCK;
 			}
-			error = union_copyup(tun, 1, vfs_context_ucred(ctx), p);
-			if (un->un_uppervp == tun->un_dirvp) {
-				un->un_flags |= UN_ULOCK;
+			union_lock();
+			/* Would need to drain for above,below mount and faulin does not enter this path */
+			un->un_flags |= UN_LOCKED;
+			error = union_copyup(tun, 1, ap->a_context);
+			un->un_flags &= ~UN_LOCKED;
+			if ((un->un_flags & UN_WANT) == UN_WANT) {
+				un->un_flags &=  ~UN_WANT;
+				wakeup(&un->un_flags);
 			}
+			union_unlock();
 		}
 		vp = tun->un_uppervp;
 	}
@@ -944,16 +1118,15 @@ union_link(
 		return (error);
 	}
 
-	FIXUP(un, p);
-	vnode_get(tdvp);
-	un->un_flags |= UN_KLOCK;
 
-	return (VNOP_LINK(vp, tdvp, cnp, ap->a_context));
+	error =  (VNOP_LINK(vp, tdvp, cnp, ap->a_context));
+	return(error);
 }
 
-int
-union_rename(ap)
-	struct vnop_rename_args  /* {
+static int
+union_rename(struct vnop_rename_args *ap)
+/*
+	struct vnop_rename_args {
 		struct vnode *a_fdvp;
 		struct vnode *a_fvp;
 		struct componentname *a_fcnp;
@@ -961,7 +1134,8 @@ union_rename(ap)
 		struct vnode *a_tvp;
 		struct componentname *a_tcnp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 
@@ -970,9 +1144,12 @@ union_rename(ap)
 	struct vnode *tdvp = ap->a_tdvp;
 	struct vnode *tvp = ap->a_tvp;
 
+
 	if (fdvp->v_op == union_vnodeop_p) {	/* always true */
 		struct union_node *un = VTOUNION(fdvp);
 		if (un->un_uppervp == NULLVP) {
+			if (UNNODE_FAULTIN(un))
+				panic("faultfs rename: No uppervp");
 			/*
 			 * this should never happen in normal
 			 * operation but might if there was
@@ -984,12 +1161,13 @@ union_rename(ap)
 		}
 
 		fdvp = un->un_uppervp;
-		vnode_get(fdvp);
 	}
 
 	if (fvp->v_op == union_vnodeop_p) {	/* always true */
 		struct union_node *un = VTOUNION(fvp);
 		if (un->un_uppervp == NULLVP) {
+			if (UNNODE_FAULTIN(un))
+				panic("faultfs rename: No uppervp");
 			/* XXX: should do a copyup */
 			error = EXDEV;
 			goto bad;
@@ -999,7 +1177,6 @@ union_rename(ap)
 			ap->a_fcnp->cn_flags |= DOWHITEOUT;
 
 		fvp = un->un_uppervp;
-		vnode_get(fvp);
 	}
 
 	if (tdvp->v_op == union_vnodeop_p) {
@@ -1011,23 +1188,19 @@ union_rename(ap)
 			 * a problem creating the top-level shadow
 			 * directory.
 			 */
+			if (UNNODE_FAULTIN(un))
+				panic("faultfs rename: No uppervp");
 			error = EXDEV;
 			goto bad;
 		}
 
 		tdvp = un->un_uppervp;
-		vnode_get(tdvp);
-		un->un_flags |= UN_KLOCK;
 	}
 
 	if (tvp != NULLVP && tvp->v_op == union_vnodeop_p) {
 		struct union_node *un = VTOUNION(tvp);
 
 		tvp = un->un_uppervp;
-		if (tvp != NULLVP) {
-			vnode_get(tvp);
-			un->un_flags |= UN_KLOCK;
-		}
 	}
 
 	return (VNOP_RENAME(fdvp, fvp, ap->a_fcnp, tdvp, tvp, ap->a_tcnp, ap->a_context));
@@ -1036,36 +1209,36 @@ bad:
 	return (error);
 }
 
-int
-union_mkdir(
-	struct vnop_mkdir_args /* {
+static int
+union_mkdir(struct vnop_mkdir_args *ap)
+/*
+	struct vnop_mkdir_args {
 		struct vnode *a_dvp;
 		struct vnode **a_vpp;
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_dvp);
 	struct vnode *dvp = un->un_uppervp;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 
 	if (dvp != NULLVP) {
 		int error;
 		struct vnode *vp;
 
-		FIXUP(un, p);
-		un->un_flags |= UN_KLOCK;
 
 		/* note that this is a direct fallthrough to the filesystem */
 		error = VNOP_MKDIR(dvp, &vp, cnp, ap->a_vap, ap->a_context);
 		if (error)
 			return (error);
 
+		union_lock();
 		error = union_allocvp(ap->a_vpp, ap->a_dvp->v_mount, ap->a_dvp,
 				NULLVP, cnp, vp, NULLVP, 1);
+		union_unlock();
 		if (error)
 			vnode_put(vp);
 		return (error);
@@ -1073,22 +1246,24 @@ union_mkdir(
 	return (EROFS);
 }
 
-int
-union_rmdir(
-	struct vnop_rmdir_args /* {
+static int
+union_rmdir(struct vnop_rmdir_args *ap)
+/*
+	struct vnop_rmdir_args {
 		struct vnode *a_dvp;
 		struct vnode *a_vp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	int error;
 	struct union_node *dun = VTOUNION(ap->a_dvp);
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
+	int busydel = 0;
 
+	/******* NODE HAS TO BE LOCKED ******/
 	if (dun->un_uppervp == NULLVP)
 		panic("union rmdir: null upper vnode");
 
@@ -1096,20 +1271,31 @@ union_rmdir(
 		struct vnode *dvp = dun->un_uppervp;
 		struct vnode *vp = un->un_uppervp;
 
-		FIXUP(dun, p);
-		vnode_get(dvp);
-		dun->un_flags |= UN_KLOCK;
-		FIXUP(un, p);
-		vnode_get(vp);
-		un->un_flags |= UN_KLOCK;
+		if (vnode_isinuse(ap->a_vp, 0)) {
+			busydel = 1;
+			union_lock();
+			un->un_flags |= UN_DELETED;
+			if (un->un_flags & UN_CACHED) {
+				un->un_flags &= ~UN_CACHED;
+				LIST_REMOVE(un, un_cache);
+			}
+			union_unlock();
+			vnode_ref(vp);
+		}
+
 
 		if (union_dowhiteout(un, cnp->cn_context))
 			cnp->cn_flags |= DOWHITEOUT;
 		error = VNOP_RMDIR(dvp, vp, ap->a_cnp, ap->a_context);
-		if (!error)
-			union_removed_upper(un);
+		if (!error) {
+			union_lock();
+			if (busydel == 0)
+				union_removed_upper(un);
+			union_unlock();
+		}
 	} else {
-		FIXUP(dun, p);
+		if (UNNODE_FAULTIN(un))
+			panic("faultfs: No uppervp");
 		error = union_mkwhiteout(
 			MOUNTTOUNIONMOUNT(UNIONTOV(dun)->v_mount),
 			dun->un_uppervp, ap->a_cnp, un->un_path);
@@ -1117,32 +1303,29 @@ union_rmdir(
 	return (error);
 }
 
-int
-union_symlink(
-	struct vnop_symlink_args /* {
+static int
+union_symlink(struct vnop_symlink_args *ap)
+/*
+	struct vnop_symlink_args {
 		struct vnode *a_dvp;
 		struct vnode **a_vpp;
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		char *a_target;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_dvp);
 	struct vnode *dvp = un->un_uppervp;
 	struct componentname *cnp = ap->a_cnp;
-	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
 
 	if (dvp != NULLVP) {
 		int error;
 		struct vnode *vp;
 
-		FIXUP(un, p);
-		un->un_flags |= UN_KLOCK;
-
 		error = VNOP_SYMLINK(dvp, &vp, cnp, ap->a_vap, ap->a_target, ap->a_context);
-		*ap->a_vpp = NULLVP;
+		*ap->a_vpp = vp;
 		return (error);
 	}
 	return (EROFS);
@@ -1155,9 +1338,10 @@ union_symlink(
  * down the union stack.  readdir(3) is responsible for
  * eliminating duplicate names from the returned data stream.
  */
-int
-union_readdir(ap)
-	struct vnop_readdir_args /* {
+static int
+union_readdir(struct vnop_readdir_args *ap)
+/*
+	struct vnop_readdir_args {
 		struct vnodeop_desc *a_desc;
 		struct vnode *a_vp;
 		struct uio *a_uio;
@@ -1165,11 +1349,11 @@ union_readdir(ap)
 		int *a_eofflag;
 		int *a_numdirent;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	struct union_node *un = VTOUNION(ap->a_vp);
 	struct vnode *uvp = un->un_uppervp;
-	struct proc *p = vfs_context_proc(ap->a_context);
 
 	if (ap->a_flags & (VNODE_READDIR_EXTENDED | VNODE_READDIR_REQSEEKOFF))
 		return (EINVAL);
@@ -1177,43 +1361,40 @@ union_readdir(ap)
 	if (uvp == NULLVP)
 		return (0);
 
-	FIXUP(un, p);
 	ap->a_vp = uvp;
 	return (VCALL(uvp, VOFFSET(vnop_readdir), ap));
 }
 
-int
-union_readlink(ap)
-	struct vnop_readlink_args /* {
+static int
+union_readlink(struct vnop_readlink_args *ap)
+/*
+	struct vnop_readlink_args {
 		struct vnode *a_vp;
 		struct uio *a_uio;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
-	struct uio *uio = ap->a_uio;
-	struct proc *p = vfs_context_proc(ap->a_context);
 	struct vnode *vp = OTHERVP(ap->a_vp);
-	int dolock = (vp == LOWERVP(ap->a_vp));
 
-	if (!dolock)
-	        FIXUP(VTOUNION(ap->a_vp), p);
 	ap->a_vp = vp;
 	error = VCALL(vp, VOFFSET(vnop_readlink), ap);
 
 	return (error);
 }
 
-int
-union_inactive(
-	struct vnop_inactive_args /* {
+static int
+union_inactive(struct vnop_inactive_args *ap)
+/*
+	struct vnop_inactive_args {
 		struct vnode *a_vp;
 		vfs_context_t a_context;
-	} */ *ap)
+	} *ap;
+*/
 {
 	struct vnode *vp = ap->a_vp;
 	struct union_node *un = VTOUNION(vp);
-	struct vnode **vpp;
 
 	/*
 	 * Do nothing (and _don't_ bypass).
@@ -1228,35 +1409,47 @@ union_inactive(
 	 * That's too much work for now.
 	 */
 
-	if (un->un_dircache != 0) {
-		for (vpp = un->un_dircache; *vpp != NULLVP; vpp++)
-			vnode_put(*vpp);
-		_FREE(un->un_dircache, M_TEMP);
-		un->un_dircache = 0;
+	union_lock();
+	if (un->un_flags & UN_DELETED) {
+		if(un->un_uppervp != NULLVP) {
+			vnode_rele(un->un_uppervp);
+		}
+		union_removed_upper(un);
 	}
 
-	if ((un->un_flags & UN_CACHED) == 0)
+	if (un->un_dircache != 0)  {
+			union_dircache_free(un);
+	}
+	if (un->un_flags & UN_DIRENVN) {
 		vnode_recycle(vp);
+	}
+
+	union_unlock();
 
 	return (0);
 }
 
-int
-union_reclaim(ap)
-	struct vnop_reclaim_args /* {
+static int
+union_reclaim(struct vnop_reclaim_args *ap)
+/*
+	struct vnop_reclaim_args {
 		struct vnode *a_vp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 
+	union_lock();
 	union_freevp(ap->a_vp);
+	union_unlock();
 
 	return (0);
 }
 
-int
-union_blockmap(ap)
-	struct vnop_blockmap_args /* {
+static int
+union_blockmap(struct vnop_blockmap_args *ap)
+/*
+	struct vnop_blockmap_args {
 		struct vnode *a_vp;
 		off_t a_offset;    
 		size_t a_size;
@@ -1264,53 +1457,50 @@ union_blockmap(ap)
 		size_t *a_run;
 		void *a_poff;
 		int a_flags;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
-	struct proc *p = current_proc();		/* XXX */
 	struct vnode *vp = OTHERVP(ap->a_vp);
-	int dolock = (vp == LOWERVP(ap->a_vp));
 
-	if (!dolock)
-	        FIXUP(VTOUNION(ap->a_vp), p);
 	ap->a_vp = vp;
 	error = VCALL(vp, VOFFSET(vnop_blockmap), ap);
 
 	return (error);
 }
 
-int
-union_pathconf(ap)
-	struct vnop_pathconf_args /* {
+static int
+union_pathconf(struct vnop_pathconf_args *ap)
+/*
+	struct vnop_pathconf_args {
 		struct vnode *a_vp;
 		int a_name;
 		int *a_retval;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
-	struct proc *p = current_proc();		/* XXX */
 	struct vnode *vp = OTHERVP(ap->a_vp);
-	int dolock = (vp == LOWERVP(ap->a_vp));
 
-	if (!dolock)
-	        FIXUP(VTOUNION(ap->a_vp), p);
 	ap->a_vp = vp;
 	error = VCALL(vp, VOFFSET(vnop_pathconf), ap);
 
 	return (error);
 }
 
-int
-union_advlock(ap)
-	struct vnop_advlock_args /* {
+static int
+union_advlock(struct vnop_advlock_args *ap)
+/*
+	struct vnop_advlock_args {
 		struct vnode *a_vp;
 		caddr_t  a_id;
 		int  a_op;
 		struct flock *a_fl;
 		int  a_flags;
 		vfs_context_t a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	register struct vnode *ovp = OTHERVP(ap->a_vp);
 
@@ -1324,11 +1514,13 @@ union_advlock(ap)
  * vnode in its arguments.
  * This goes away with a merged VM/buffer cache.
  */
-int
-union_strategy(ap)
-	struct vnop_strategy_args /* {
+static int
+union_strategy(struct vnop_strategy_args *ap)
+/*
+	struct vnop_strategy_args {
 		struct buf *a_bp;
-	} */ *ap;
+	} *ap;
+*/
 {
 	struct buf *bp = ap->a_bp;
 	int error;
@@ -1352,9 +1544,10 @@ union_strategy(ap)
 }
 
 /* Pagein */
-int
-union_pagein(ap)
-	struct vnop_pagein_args /* {
+static int
+union_pagein(struct vnop_pagein_args *ap)
+/*
+	struct vnop_pagein_args {
 	   	struct vnode 	*a_vp,
 	   	upl_t		a_pl,
 		vm_offset_t	a_pl_offset,
@@ -1362,7 +1555,8 @@ union_pagein(ap)
 		size_t		a_size,
 		int		a_flags
 		vfs_context_t	a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 	struct vnode *vp = OTHERVP(ap->a_vp);
@@ -1381,11 +1575,17 @@ union_pagein(ap)
 		off_t cur = ap->a_f_offset + (off_t)ap->a_pl_offset;
 
 		if (vp == un->un_uppervp) {
-			if (cur > un->un_uppersz)
+			if (cur > un->un_uppersz) {
+				union_lock();
 				union_newsize(ap->a_vp, cur, VNOVAL);
+				union_unlock();
+			}
 		} else {
-			if (cur > un->un_lowersz)
+			if (cur > un->un_lowersz) {
+				union_lock();
 				union_newsize(ap->a_vp, VNOVAL, cur);
+				union_unlock();
+			}
 		}
 	}
 
@@ -1393,9 +1593,10 @@ union_pagein(ap)
 }
 
 /* Pageout  */
-int
-union_pageout(ap)
-	struct vnop_pageout_args /* {
+static int
+union_pageout(struct vnop_pageout_args *ap)
+/*
+	struct vnop_pageout_args {
 	   	struct vnode 	*a_vp,
 	   	upl_t		a_pl,
 		vm_offset_t	a_pl_offset,
@@ -1403,7 +1604,8 @@ union_pageout(ap)
 		size_t		a_size,
 		int		a_flags
 		vfs_context_t	a_context;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 	struct vnode *vp;
@@ -1423,21 +1625,26 @@ union_pageout(ap)
 	if (error == 0) {
 		off_t cur = ap->a_f_offset + (off_t)ap->a_pl_offset;
 
-		if (cur > un->un_uppersz)
+		if (cur > un->un_uppersz) {
+			union_lock();
 			union_newsize(ap->a_vp, cur, VNOVAL);
+			union_unlock();
+		}
 	}
 
 	return (error);
 }
 
 /* Blktooff derives file offset for the given logical block number */
-int
-union_blktooff(ap)
-	struct vnop_blktooff_args /* {
+static int
+union_blktooff(struct vnop_blktooff_args *ap)
+/*
+	struct vnop_blktooff_args {
 		struct vnode *a_vp;
 		daddr64_t a_lblkno;
 		off_t *a_offset;    
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 	struct vnode *vp = OTHERVP(ap->a_vp);
@@ -1448,13 +1655,15 @@ union_blktooff(ap)
 }
 
 /* offtoblk derives file offset for the given logical block number */
-int
-union_offtoblk(ap)
-	struct vnop_offtoblk_args /* {
+static int
+union_offtoblk(struct vnop_offtoblk_args *ap)
+/*
+	struct vnop_offtoblk_args  {
 		struct vnode *a_vp;
 		off_t a_offset;    
 		daddr64_t *a_lblkno;
-	} */ *ap;
+	} *ap;
+*/
 {
 	int error;
 	struct vnode *vp = OTHERVP(ap->a_vp);
@@ -1487,6 +1696,7 @@ struct vnodeopv_entry_desc union_vnodeop_entries[] = {
 	{ &vnop_select_desc, (VOPFUNC)union_select },		/* select */
 	{ &vnop_revoke_desc, (VOPFUNC)union_revoke },		/* revoke */
 	{ &vnop_mmap_desc, (VOPFUNC)union_mmap },		/* mmap */
+	{ &vnop_mnomap_desc, (VOPFUNC)union_mnomap },		/* mnomap */
 	{ &vnop_fsync_desc, (VOPFUNC)union_fsync },		/* fsync */
 	{ &vnop_remove_desc, (VOPFUNC)union_remove },		/* remove */
 	{ &vnop_link_desc, (VOPFUNC)union_link },		/* link */
diff --git a/bsd/miscfs/volfs/volfs.h b/bsd/miscfs/volfs/volfs.h
deleted file mode 100644
index 0b083ee7d..000000000
--- a/bsd/miscfs/volfs/volfs.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#ifndef __VOLFS_VOLFS_H__
-#define __VOLFS_VOLFS_H__
-
-#include  <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_PRIVATE
-struct volfs_mntdata
-{
-	struct vnode *volfs_rootvp;
-};
-
-/*
- * Volfs vnodes exist only for the root, which allows for the enumeration
- * of all volfs accessible filesystems, and for the filesystems which
- * volfs handles.
- */
-#define VOLFS_ROOT	1	/* This volfs vnode represents root of volfs */
-#define	VOLFS_FSNODE	2	/* This volfs vnode represents a file system */
-
-struct volfs_vndata
-{
-	int		vnode_type;
-	unsigned int	nodeID;	/* the dev entry of a file system */
-	struct mount *	fs_mount;
-	fsid_t	fs_fsid;
-};
-
-#define MAXVLFSNAMLEN	24	/* max length is really 10, pad to 24 since
-				 * some of the math depends on VLFSDIRENTLEN
-				 * being a power of 2 */
-#define VLFSDIRENTLEN	(MAXVLFSNAMLEN + sizeof(u_int32_t) + sizeof(u_int16_t) + sizeof(u_int8_t) + sizeof(u_int8_t))
-
-#define ROOT_DIRID	2
-
-#define MAXPLCENTRIES 250
-#define PLCHASHSIZE 128
-
-
-#define VTOVL(VP) ((struct volfs_vndata *)((VP)->v_data))
-
-#define PRINTIT kprintf
-
-
-#endif /* __APPLE_API_PRIVATE */
-#endif /* __VOLFS_VOLFS_H__ */
diff --git a/bsd/miscfs/volfs/volfs_vfsops.c b/bsd/miscfs/volfs/volfs_vfsops.c
deleted file mode 100644
index 6cdd7f2ed..000000000
--- a/bsd/miscfs/volfs/volfs_vfsops.c
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/namei.h>
-#include <sys/proc.h>
-#include <sys/kernel.h>
-#include <mach/machine/vm_types.h>
-#include <sys/vnode.h>
-#include <sys/socket.h>
-#include <sys/mount_internal.h>
-#include <sys/mbuf.h>
-#include <sys/file.h>
-#include <sys/disk.h>
-#include <sys/ioctl.h>
-#include <sys/errno.h>
-#include <sys/malloc.h>
-#include <dev/ldd.h>
-
-#include <miscfs/specfs/specdev.h>
-#include "volfs.h"
-
-static int  volfs_mount(struct mount *, vnode_t , user_addr_t, vfs_context_t);
-static int  volfs_start(struct mount *, int, vfs_context_t);
-static int  volfs_unmount(struct mount *, int, vfs_context_t);
-static int  volfs_root(struct mount *, struct vnode **, vfs_context_t);
-static int  volfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context);
-static int  volfs_sync(struct mount *, int, vfs_context_t);
-static int  volfs_vget(struct mount *, ino64_t, struct vnode **, vfs_context_t);
-static int  volfs_fhtovp(struct mount *, int, unsigned char *, struct vnode **, vfs_context_t);
-static int  volfs_vptofh(struct vnode *, int *, unsigned char *, vfs_context_t);
-static int  volfs_init(struct vfsconf *);
-static int  volfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
-void volfs_load(int loadArgument);
-
-
-struct vfsops volfs_vfsops = {
-	volfs_mount,
-	volfs_start,
-	volfs_unmount,
-	volfs_root,
-	NULL,		/* quotactl */
-	volfs_vfs_getattr,
-	volfs_sync,
-	volfs_vget,
-	volfs_fhtovp,
-	volfs_vptofh,
-	volfs_init,
-	volfs_sysctl
-};
-
-// static char volfs_fs_name[MFSNAMELEN] = "volfs";
-extern struct vnodeopv_desc volfs_vnodeop_opv_desc;
-
-extern int (**volfs_vnodeop_p)(void *);
-
-/* The following refer to kernel global variables used in the loading/initialization: */
-extern int                      vfs_opv_numops; /* The total number of defined vnode operations */
-extern int kdp_flag;
-
-void
-volfs_load(__unused int loadArgument) 
-{
-#if 0
-    struct vfsconf *vfsconflistentry;
-    int entriesRemaining;
-    struct vfsconf *newvfsconf = NULL;
-    struct vfsconf *lastentry = NULL;
-        int j;
-        int (***opv_desc_vector_p)();
-        int (**opv_desc_vector)();
-        struct vnodeopv_entry_desc *opve_descp;
-        
-        /*
-         * This routine is responsible for all the initialization that would
-         * ordinarily be done as part of the system startup; it calls volfs_init
-         * to do the initialization that is strictly volfs-specific.
-         */
-
-        /*
-           prevvfsconf is supposed to be the entry preceding the new entry.
-           To make sure we can always get hooked in SOMEWHERE in the list,
-           start it out at the first entry of the list.  This assumes the
-           first entry in the list will be non-empty and not volfs.
-
-           This becomes irrelevant when volfs is compiled into the list.
-         */
-    vfsconflistentry = vfsconf;
-    for (entriesRemaining = maxvfsslots; entriesRemaining > 0; --entriesRemaining) {
-        if (vfsconflistentry->vfc_vfsops != NULL) {
-            /*
-             * Check to see if we're reloading a new version of volfs during debugging
-             * and overwrite the previously assigned entry if we find one:
-             */
-            if (strcmp(vfsconflistentry->vfc_name, volfs_fs_name) == 0) {
-                newvfsconf = vfsconflistentry;
-                break;
-            } else {
-                lastentry = vfsconflistentry;
-            };
-        } else {
-            /*
-             * This is at least a POSSIBLE place to insert the new entry...
-             */
-            newvfsconf = vfsconflistentry;
-        };
-        ++vfsconflistentry;
-    };
-
-    if (newvfsconf) {
-         newvfsconf->vfc_vfsops = &volfs_vfsops;
-        strncpy(&newvfsconf->vfc_name[0], "volfs", MFSNAMELEN);
-        newvfsconf->vfc_typenum = maxvfsconf++;
-        newvfsconf->vfc_refcount = 0;
-        newvfsconf->vfc_flags = 0;
-        newvfsconf->vfc_mountroot = NULL;       /* Can't mount root of file system [yet] */
-
-                /* Hook into the list: */
-        newvfsconf->vfc_next = NULL;
-        if (lastentry) {
-            newvfsconf->vfc_next = lastentry->vfc_next;
-            lastentry->vfc_next = newvfsconf;
-        };
-
-        /* Based on vfs_op_init and ... */
-        opv_desc_vector_p = volfs_vnodeop_opv_desc.opv_desc_vector_p;
-
-        /*
-         * Allocate and init the vector.
-         * Also handle backwards compatibility.
-         */
-        MALLOC(*opv_desc_vector_p, PFI *, vfs_opv_numops*sizeof(PFI), M_TEMP, M_WAITOK);
-
-        bzero (*opv_desc_vector_p, vfs_opv_numops*sizeof(PFI));
-
-        opv_desc_vector = *opv_desc_vector_p;
-        for (j=0; volfs_vnodeop_opv_desc.opv_desc_ops[j].opve_op; j++) {
-            opve_descp = &(volfs_vnodeop_opv_desc.opv_desc_ops[j]);
-
-            /*
-             * Sanity check:  is this operation listed
-             * in the list of operations?  We check this
-             * by seeing if its offest is zero.  Since
-             * the default routine should always be listed
-             * first, it should be the only one with a zero
-             * offset.  Any other operation with a zero
-             * offset is probably not listed in
-             * vfs_op_descs, and so is probably an error.
-             *
-             * A panic here means the layer programmer
-             * has committed the all-too common bug
-             * of adding a new operation to the layer's
-             * list of vnode operations but
-             * not adding the operation to the system-wide
-             * list of supported operations.
-             */
-            if (opve_descp->opve_op->vdesc_offset == 0 &&
-                opve_descp->opve_op->vdesc_offset != VOFFSET(vnop_default)) {
-                panic ("load_volfs: bad operation");
-                }
-            /*
-             * Fill in this entry.
-             */
-            opv_desc_vector[opve_descp->opve_op->vdesc_offset] =
-                opve_descp->opve_impl;
-            }
-
-        /*
-         * Finally, go back and replace unfilled routines
-         * with their default.  (Sigh, an O(n^3) algorithm.  I
-                                 * could make it better, but that'd be work, and n is small.)
-         */
-        opv_desc_vector_p = volfs_vnodeop_opv_desc.opv_desc_vector_p;
-
-        /*
-         * Force every operations vector to have a default routine.
-         */
-        opv_desc_vector = *opv_desc_vector_p;
-        if (opv_desc_vector[VOFFSET(vnop_default)]==NULL) {
-            panic("load_vp;fs: operation vector without default routine.");
-            }
-        for (j = 0;j<vfs_opv_numops; j++)
-            if (opv_desc_vector[j] == NULL)
-                opv_desc_vector[j] =
-                    opv_desc_vector[VOFFSET(vnop_default)];
-
-         volfs_init(newvfsconf);
-        };
-#else
-	panic("volfs load not ported");
-#endif
-}
-
-/*
- * VFS Operations.
- *
- * mount system call
- */
-static int
-volfs_mount(struct mount *mp, __unused vnode_t devvp, __unused user_addr_t data,  __unused vfs_context_t context)
-{
-	struct volfs_mntdata *priv_mnt_data;
-	struct vnode *root_vp;
-	struct volfs_vndata	*priv_vn_data;
-	int	error;
-	struct vnode_fsparam vfsp;
-
-	MALLOC(priv_mnt_data, struct volfs_mntdata *, sizeof(struct volfs_mntdata),
-		M_VOLFSMNT, M_WAITOK);
-
-	mp->mnt_data = (void *)priv_mnt_data;
-	strcpy(mp->mnt_vfsstat.f_fstypename, "volfs");
-	strcpy(mp->mnt_vfsstat.f_mntfromname, "<volfs>");
-	
-	/* Set up the root vnode for fast reference in the future.
-	   Note that the root is maintained unlocked but with a pos. ref count until unmount. */
-	
-	MALLOC(priv_vn_data, struct volfs_vndata *, sizeof(struct volfs_vndata), M_VOLFSNODE, M_WAITOK);
-
-	priv_vn_data->vnode_type = VOLFS_ROOT;
-	priv_vn_data->nodeID = ROOT_DIRID;
-	priv_vn_data->fs_mount = mp;
-	priv_vn_data->fs_fsid = mp->mnt_vfsstat.f_fsid;
-
-	vfsp.vnfs_mp = mp;
-	vfsp.vnfs_vtype = VDIR;
-	vfsp.vnfs_str = "volfs";
-	vfsp.vnfs_dvp = 0;
-	vfsp.vnfs_fsnode = priv_vn_data;
-	vfsp.vnfs_cnp = 0;
-	vfsp.vnfs_vops = volfs_vnodeop_p;
-	vfsp.vnfs_rdev = 0;
-	vfsp.vnfs_filesize = 0;
-	vfsp.vnfs_flags = VNFS_NOCACHE | VNFS_CANTCACHE;
-	vfsp.vnfs_marksystem = 0;
-	vfsp.vnfs_markroot = 1;
-
-	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &root_vp);
-	if (error != 0) {
-		FREE(priv_mnt_data, M_VOLFSMNT);
-		FREE(priv_vn_data, M_VOLFSNODE);
-		return(error);
-	}
-	vnode_ref(root_vp);
-	vnode_put(root_vp);
-
-	/* obtain a new fsid for the mount point */
-	vfs_getnewfsid(mp);
-
-	vnode_settag(root_vp, VT_VOLFS);
-    
-	priv_mnt_data->volfs_rootvp = root_vp;
-	mp->mnt_flag &= ~MNT_RDONLY;
-
-	mp->mnt_vtable->vfc_threadsafe = TRUE;
-	
-	return (0);
-}
-
-static int
-volfs_start(__unused struct mount * mp, __unused int flags, __unused vfs_context_t context)
-{
-	return (0);
-}
-
-/*
- * Return the root of a filesystem.  For volfs the root vnode is a directory
- * containing the list of all filesystems volfs can work with.
- */
-static int
-volfs_root(struct mount *mp, struct vnode **vpp, __unused vfs_context_t context)
-{
-	struct volfs_mntdata *priv_data;
-
-	priv_data = (struct volfs_mntdata *)mp->mnt_data;
-
-	if (priv_data->volfs_rootvp) {
-		vnode_get(priv_data->volfs_rootvp);
-		*vpp = priv_data->volfs_rootvp;
-	} else {
-		panic("volfs: root vnode missing!");
-	};
-
-	return(0);
-}
-
-/*
- * unmount system call
- */
-static int
-volfs_unmount(struct mount *mp, __unused int mntflags, __unused vfs_context_t context)
-{
-    struct	volfs_mntdata	*priv_data;
-    struct vnode *root_vp;
-    int		retval;
-
-    priv_data = (struct volfs_mntdata *)mp->mnt_data;
-
-    root_vp = priv_data->volfs_rootvp;
-    retval = vflush(mp, root_vp, 0);
-    if (retval) goto Err_Exit;
-
-    /* Free the root vnode.
-         Note that there's no need to vget() or vref() it before locking it here:
-         the ref. count has been maintained at +1 ever since mount time. */
-    if (root_vp) {
-        if (vnode_isinuse(root_vp, 1)) {
-             retval = EBUSY;
-            goto Err_Exit;
-        };
-
-        priv_data->volfs_rootvp = NULL;
-        vnode_rele(root_vp);				/* This drops volfs's own refcount */
-        vnode_reclaim(root_vp);
-    };
-
-	/* All vnodes should be gone, and no errors, clean up the last */
-
-    mp->mnt_data = NULL;
-    FREE(priv_data, M_VOLFSMNT);
-
-Err_Exit:
-
-    return(retval);
-}
-
-/*
- * Get file system statistics.
- */
-static int
-volfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
-{
-	VFSATTR_RETURN(fsap, f_bsize, 512);
-	VFSATTR_RETURN(fsap, f_iosize, 512);
-	VFSATTR_RETURN(fsap, f_blocks, 1024);
-	VFSATTR_RETURN(fsap, f_bfree, 0);
-	VFSATTR_RETURN(fsap, f_bavail, 0);
-	VFSATTR_RETURN(fsap, f_bused, 1024);
-	VFSATTR_RETURN(fsap, f_files, 0);
-	VFSATTR_RETURN(fsap, f_ffree, 0);
-	VFSATTR_RETURN(fsap, f_fssubtype, 0);
-	return 0;
-}
-
-/*
- * volfs doesn't have any data and you can't write into any of the volfs 
- * structures, so don't do anything
- */
-static int
-volfs_sync(__unused struct mount *mp, __unused int waitfor, __unused vfs_context_t context)
-{
-	return 0;
-}
-
-/*
- *
- */
-static int
-volfs_vget(__unused struct mount *mp, __unused ino64_t ino, 
-		   __unused struct vnode **vpp, __unused vfs_context_t context)
-{
-	return(ENOTSUP);
-}
-
-/*
- * File handle to vnode
- */
-static int
-volfs_fhtovp(__unused struct mount *mp, __unused int fhlen,  
-			 __unused unsigned char *fhp, __unused struct vnode **vpp,
-			 __unused vfs_context_t context)
-{
-	return(ENOTSUP);
-}
-
-/*
- * Vnode pointer to File handle
- */
-static int
-volfs_vptofh(__unused struct vnode *vp, __unused int *fhlenp, __unused unsigned char *fhp, __unused vfs_context_t context)
-{
-	return(ENOTSUP);
-}
-
-/*
- * Initialize the filesystem
- */
-static int
-volfs_init(__unused struct vfsconf *vfsp)
-{	
-	return (0);
-}
-
-/*
- * fast filesystem related variables.
- */
-static int
-volfs_sysctl(__unused int *name, __unused u_int namelen, __unused user_addr_t oldp, 
-			 __unused size_t *oldlenp, __unused user_addr_t newp, __unused size_t newlen, 
-			 __unused vfs_context_t context)
-{
-	return (ENOTSUP);
-}
-
diff --git a/bsd/miscfs/volfs/volfs_vnops.c b/bsd/miscfs/volfs/volfs_vnops.c
deleted file mode 100644
index 82a1bd6ad..000000000
--- a/bsd/miscfs/volfs/volfs_vnops.c
+++ /dev/null
@@ -1,978 +0,0 @@
-/*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#include <mach/mach_types.h>
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/resourcevar.h>
-#include <sys/kernel.h>
-#include <sys/file.h>
-#include <sys/filedesc.h>
-#include <sys/stat.h>
-#include <sys/proc_internal.h>	/* for p_fd */
-#include <sys/kauth.h>
-#include <sys/conf.h>
-#include <sys/mount_internal.h>
-#include <sys/vnode_internal.h>
-#include <sys/malloc.h>
-#include <sys/dirent.h>
-#include <sys/namei.h>
-#include <sys/attr.h>
-#include <sys/kdebug.h>
-#include <sys/queue.h>
-#include <sys/uio_internal.h>
-
-#include <sys/vm.h>
-#include <sys/errno.h>
-#include <vfs/vfs_support.h>
-
-#include <kern/locks.h>
-
-#include "volfs.h"
-
-/*
- * volfs acts as a bridge between the requirements of the MacOS API and the Unix API.
- * MacOS applications describe files by a <Volume ID><Directory ID><File Name> triple.
- * The Unix API describes files by pathname.  Volfs is a virtual file system that sits over
- * the HFS VFS interface and allows files to be described by a <Volume ID>/<Directory ID>/<File Name>
- * pathname.
- *
- * The root of the volfs filesystem consists of directories named the volume ID's of all the
- * currently mounted filesystems which support the VFS vget() routine.  Each of those directories
- * supports the lookup by file ID of all files and directories within the filesystem.  When a
- * file or directory is resolved its vnode from that filesystem rather than a volfs vnode is returned
- * allowing immediate access to the target file or directory.
- *
- * Readdir on the root of the volfs filesystem returns the list of available file systems.  Readdir
- * on a filesystem node, however, returns only . and .. since it is not practical to list all
- * of the file ID's in a timely fashion and furthermore VFS does not provide a mechanism for
- * enumerating all of the file id's.
- *
- * Volume ID's are taken from the low 32 bits of the f_fsid field, formatted as a base 10 ASCII
- * string with no leading zeros (volume ID 1 is represented as "1").
- *
- * File ID's are created in same manner, with their 32 bits formatted as a base 10 ASCII
- * string with no leading zeros.
- *
- * Volfs does create a security hole since it is possible to bypass directory permissions higher
- * in the namespace tree.  This security hole is about the same as the one created by NFS which uses
- * a similar mechanism.
- */
-
-static int  volfs_reclaim  (struct vnop_reclaim_args*);
-static int  volfs_getattr  (struct vnop_getattr_args *);
-static int  volfs_select   (struct vnop_select_args *);
-static int  volfs_rmdir    (struct vnop_rmdir_args *);
-static int  volfs_readdir  (struct vnop_readdir_args *);
-static int  volfs_pathconf (struct vnop_pathconf_args *);
-static int  volfs_lookup   (struct vnop_lookup_args *);
-
-static int volfs_readdir_callback(mount_t, void *);
-static int get_filevnode(struct mount *parent_fs, u_int id, vnode_t *ret_vnode, vfs_context_t context);
-static int get_fsvnode(struct mount *our_mount, int id, vnode_t *ret_vnode);
-
-/* for the call back function in volfs_readdir */
-struct volfs_rdstruct {
-	int 		validindex;
-	vnode_t 	vp;
-	int 		rec_offset;
-	struct uio * uio;
-};
-
-#define VOPFUNC int (*)(void *)
-
-/* Global vfs data structures for volfs. */
-int                 (**volfs_vnodeop_p) (void *);
-struct vnodeopv_entry_desc volfs_vnodeop_entries[] = {
-    {&vnop_default_desc, (VOPFUNC)vn_default_error},
-    {&vnop_strategy_desc, (VOPFUNC)err_strategy},	/* strategy */
-    {&vnop_bwrite_desc, (VOPFUNC)err_bwrite},		/* bwrite */
-    {&vnop_lookup_desc, (VOPFUNC)volfs_lookup},		/* lookup */
-    {&vnop_create_desc, (VOPFUNC)err_create},		/* create */
-    {&vnop_whiteout_desc, (VOPFUNC)err_whiteout},	/* whiteout */
-    {&vnop_mknod_desc, (VOPFUNC)err_mknod},		/* mknod */
-    {&vnop_open_desc, (VOPFUNC)nop_open},		/* open */
-    {&vnop_close_desc, (VOPFUNC)nop_close},		/* close */
-    {&vnop_getattr_desc, (VOPFUNC)volfs_getattr},	/* getattr */
-    {&vnop_setattr_desc, (VOPFUNC)err_setattr},		/* setattr */
-    {&vnop_getattrlist_desc, (VOPFUNC)err_getattrlist},	/* getattrlist */
-    {&vnop_setattrlist_desc, (VOPFUNC)err_setattrlist},	/* setattrlist */
-    {&vnop_read_desc, (VOPFUNC)err_read},		/* read */
-    {&vnop_write_desc, (VOPFUNC)err_write},		/* write */
-    {&vnop_ioctl_desc, (VOPFUNC)err_ioctl},		/* ioctl */
-    {&vnop_select_desc, (VOPFUNC)volfs_select},		/* select */
-    {&vnop_exchange_desc, (VOPFUNC)err_exchange},	/* exchange */
-    {&vnop_revoke_desc, (VOPFUNC)nop_revoke},		/* revoke */
-    {&vnop_mmap_desc, (VOPFUNC)err_mmap},		/* mmap */
-    {&vnop_fsync_desc, (VOPFUNC)err_fsync},		/* fsync */
-    {&vnop_remove_desc, (VOPFUNC)err_remove},		/* remove */
-    {&vnop_link_desc, (VOPFUNC)err_link},		/* link */
-    {&vnop_rename_desc, (VOPFUNC)err_rename},		/* rename */
-    {&vnop_mkdir_desc, (VOPFUNC)err_mkdir},		/* mkdir */
-    {&vnop_rmdir_desc, (VOPFUNC)volfs_rmdir},		/* rmdir */
-    {&vnop_symlink_desc, (VOPFUNC)err_symlink},		/* symlink */
-    {&vnop_readdir_desc, (VOPFUNC)volfs_readdir},	/* readdir */
-    {&vnop_readdirattr_desc, (VOPFUNC)err_readdirattr},	/* readdirattr */
-    {&vnop_readlink_desc, (VOPFUNC)err_readlink},	/* readlink */
-    {&vnop_inactive_desc, (VOPFUNC)err_inactive},	/* inactive */
-    {&vnop_reclaim_desc, (VOPFUNC)volfs_reclaim},	/* reclaim */
-    {&vnop_pathconf_desc, (VOPFUNC)volfs_pathconf},	/* pathconf */
-    {&vnop_advlock_desc, (VOPFUNC)err_advlock},		/* advlock */
-    {&vnop_allocate_desc, (VOPFUNC)err_allocate},	/* allocate */
-	{&vnop_pagein_desc, (VOPFUNC)err_pagein},	/* pagein */
-	{&vnop_pageout_desc, (VOPFUNC)err_pageout},	/* pageout */
-	{&vnop_searchfs_desc, (VOPFUNC)err_searchfs},	/* searchfs */
-	{&vnop_copyfile_desc, (VOPFUNC)err_copyfile },	/* Copyfile */
-	{&vnop_blktooff_desc, (VOPFUNC)err_blktooff},	/* blktooff */
-	{&vnop_offtoblk_desc, (VOPFUNC)err_offtoblk },	/* offtoblk */
- 	{&vnop_blockmap_desc, (VOPFUNC)err_blockmap },		/* blockmap */
-   {(struct vnodeop_desc *) NULL, (int (*) ()) NULL}
-};
-
-/*
- * Oh what a tangled web we weave.  This structure will be used by
- * bsd/vfs/vfs_conf.c to actually do the initialization of volfs_vnodeop_p
- */
-struct vnodeopv_desc volfs_vnodeop_opv_desc =
-{&volfs_vnodeop_p, volfs_vnodeop_entries};
-
-static char gDotDot[] = "..";
-
-struct finfo {
-    fsobj_id_t parID;
-};
-
-struct finfoattrbuf {
-    unsigned long length;
-    struct finfo fi;
-};
-
-
-static int volfs_getattr_callback(mount_t, void *);
-
-
-/*
- * volfs_reclaim - Reclaim a vnode so that it can be used for other purposes.
- */
-static int
-volfs_reclaim(ap)
-    struct vnop_reclaim_args /* { struct vnode *a_vp; vfs_context_t a_context; } */ *ap;
-{
-	struct vnode *vp = ap->a_vp;
-	void *data = vp->v_data;
-
-	vp->v_data = NULL;
-	FREE(data, M_VOLFSNODE);
-
-	return (0);
-}
-
-struct volfsgetattr_struct{
-	int 	numMounts;
-	vnode_t	a_vp;
-};
-
-static int
-volfs_getattr_callback(mount_t mp, void * arg)
-{
-	struct volfsgetattr_struct *vstrp = (struct volfsgetattr_struct *)arg;
-
-	if (mp != vnode_mount(vstrp->a_vp) && validfsnode(mp))
-		vstrp->numMounts++;
-	return(VFS_RETURNED);
-}
-
-/*
- * volfs_getattr - fill in the attributes for this vnode
- */
-static int
-volfs_getattr(ap)
-    struct vnop_getattr_args	/* { struct vnode *a_vp; struct vnode_attr *a_vap;
-        vfs_context_t a_context; } */ *ap;
-{
-    struct volfs_vndata *priv_data;
-    struct vnode		*a_vp;
-    struct vnode_attr	*a_vap;
-    int                 numMounts = 0;
-    struct volfsgetattr_struct vstr;
-    struct timespec ts;
-
-    a_vp = ap->a_vp;
-    a_vap = ap->a_vap;
-
-    priv_data = a_vp->v_data;
-
-    VATTR_RETURN(a_vap, va_type, VDIR);
-    VATTR_RETURN(a_vap, va_mode, 0555);
-    VATTR_RETURN(a_vap, va_nlink, 2);
-    VATTR_RETURN(a_vap, va_uid, 0);
-    VATTR_RETURN(a_vap, va_gid, 0);
-    VATTR_RETURN(a_vap, va_fsid, (int) a_vp->v_mount->mnt_vfsstat.f_fsid.val[0]);
-    VATTR_RETURN(a_vap, va_fileid, (uint64_t)((u_long)priv_data->nodeID));
-    VATTR_RETURN(a_vap, va_acl, NULL);
-
-    /*
-     * If it's the root vnode calculate its size based on the number of eligible
-     * file systems
-     */
-    if (priv_data->vnode_type == VOLFS_ROOT) {
-	    vstr.numMounts = 0;
-	    vstr.a_vp = a_vp;
-
-	    vfs_iterate(LK_NOWAIT, volfs_getattr_callback, (void *)&vstr);
-
-	    numMounts = vstr.numMounts;
-
-	    VATTR_RETURN(a_vap, va_data_size, (numMounts + 2) * VLFSDIRENTLEN);
-    } else {
-	    VATTR_RETURN(a_vap, va_data_size, 2 * VLFSDIRENTLEN);
-    }
-
-    VATTR_RETURN(a_vap, va_iosize, 512);
-    ts.tv_sec = boottime_sec();
-    ts.tv_nsec = 0;
-    VATTR_RETURN(a_vap, va_access_time, ts);
-    VATTR_RETURN(a_vap, va_modify_time, ts);
-    VATTR_RETURN(a_vap, va_change_time, ts);
-
-    VATTR_RETURN(a_vap, va_gen, 0);
-    VATTR_RETURN(a_vap, va_flags, 0);
-    VATTR_RETURN(a_vap, va_rdev, 0);
-    VATTR_RETURN(a_vap, va_filerev, 0);
-
-    return (0);
-}
-
-/*
- * volfs_select - just say OK.  Only possible op is readdir
- */
-static int
-volfs_select(__unused struct vnop_select_args *ap)
-{
-	return (1);
-}
-
-/*
- * vofls_rmdir - not possible to remove directories in volfs
- */
-static int
-volfs_rmdir(ap)
-    struct vnop_rmdir_args	/* { struct vnode *a_dvp; struct vnode *a_vp;
-        struct componentname *a_cnp; vfs_context_t a_context; } */ *ap;
-{
-    if (ap->a_dvp == ap->a_vp) {
-		(void) nop_rmdir(ap);
-		return (EINVAL);
-    } else
-		return (err_rmdir(ap));
-}
-
-
-
-static int
-volfs_readdir_callback(mount_t mp, void * v)
-{
-	struct volfs_rdstruct * vcsp = (struct volfs_rdstruct *)v;
-	struct dirent       local_dir;
-	int error;
-
-            if ((mp != vnode_mount(vcsp->vp)) && validfsnode(mp))
-				vcsp->validindex++;
-
-            if (vcsp->rec_offset == vcsp->validindex)
-              {
-                local_dir.d_fileno = mp->mnt_vfsstat.f_fsid.val[0];
-                local_dir.d_type = DT_DIR;
-                local_dir.d_reclen = VLFSDIRENTLEN;
-                local_dir.d_namlen = sprintf(&local_dir.d_name[0], "%d", mp->mnt_vfsstat.f_fsid.val[0]);
-                error = uiomove((char *) &local_dir, VLFSDIRENTLEN, vcsp->uio);
-                vcsp->rec_offset++;
-              }
-
-		return(VFS_RETURNED);
-}
-
-/*
- * volfs_readdir - Get directory entries
- *
- * Directory listings are only produced for the root volfs node.  Filesystems
- * just return . & ..
- * Filesystems contained within the volfs root are named by the decimal
- * equivalent of the f_fsid.val[0] from their mount structure (typically
- * the device id of the volume).  The maximum length for a name, then is
- * 10 characters.
- */
-static int
-volfs_readdir(ap)
-    struct vnop_readdir_args	/* { struct vnode *a_vp; struct uio *a_uio;
-				 * int *a_eofflag; int
-        *ncookies; u_long **a_cookies; vfs_context_t a_context; } */ *ap;
-{
-    struct volfs_vndata *priv_data;
-    register struct uio *uio = ap->a_uio;
-    int                 error = 0;
-    size_t              count, lost;
-    int                 rec_offset;
-    struct dirent       local_dir;
-    int                 i;
-    int					starting_resid;
-    off_t               off;
-	struct volfs_rdstruct vcs;
-    
-	off = uio->uio_offset;
-    priv_data = ap->a_vp->v_data;
-	// LP64todo - fix this!
-    starting_resid = count = uio_resid(uio);
- 
-    /* Make sure we don't return partial entries. */
-    count -= (uio->uio_offset + count) & (VLFSDIRENTLEN - 1);
-	if (count <= 0) {
-		return (EINVAL);
-	}
-    /*
-     * Make sure we're starting on a directory boundary
-     */
-	if (off & (VLFSDIRENTLEN - 1)) {
-		return (EINVAL);
-	}
-    rec_offset = off / VLFSDIRENTLEN;
-	// LP64todo - fix this!
-    lost = uio_resid(uio) - count;
-    uio_setresid(uio, count);
-    uio_iov_len_set(uio, count); 
-#if LP64_DEBUG
-	if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
-		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); 
-	}
-#endif /* LP64_DEBUG */
-
-    local_dir.d_reclen = VLFSDIRENTLEN;
-    /*
-     * We must synthesize . and ..
-     */
-
-    if (rec_offset == 0)
-      {
-        /*
-         * Synthesize .
-         */
-        local_dir.d_fileno = priv_data->nodeID;
-        local_dir.d_type = DT_DIR;
-        local_dir.d_namlen = 1;
-        local_dir.d_name[0] = '.';
-        for (i = 1; i < MAXVLFSNAMLEN; i++)
-            local_dir.d_name[i] = 0;
-        error = uiomove((char *) &local_dir, VLFSDIRENTLEN, uio);
-        rec_offset++;
-      }
-    if (rec_offset == 1)
-      {
-        /*
-         * Synthesize ..
-         * We only have two levels in the volfs hierarchy.  Root's
-         * .. points to itself and the second level points to root,
-         * hence we've hardcoded d_fileno for .. here
-         */
-        local_dir.d_fileno = ROOT_DIRID;
-        local_dir.d_type = DT_DIR;
-        local_dir.d_namlen = 2;
-        local_dir.d_name[0] = '.';
-        local_dir.d_name[1] = '.';
-        for (i = 2; i < MAXVLFSNAMLEN; i++)
-            local_dir.d_name[i] = 0;
-        error = uiomove((char *) &local_dir, VLFSDIRENTLEN, uio);
-        rec_offset++;
-      }
-
-    /*
-     * OK, we've given them the . & .. entries.  If this is a
-     * filesystem node then we've gone as far as we're going
-     * to go
-     */
-    if (priv_data->vnode_type == VOLFS_FSNODE)
-        {
-        *ap->a_eofflag = 1;	/* we got all the way to the end */
-        return (error);
-        }
-
-    if (rec_offset > 1) {
-		vcs.validindex = 1;		/* we always have "." and ".." */
-		vcs.rec_offset = rec_offset;
-		vcs.vp = ap->a_vp;
-		vcs.uio = uio;
-		
-	
-		vfs_iterate(0, volfs_readdir_callback, &vcs);
-
-        //if (mp == (void *) &mountlist)
-            *ap->a_eofflag = 1;	/* we got all the way to the end */
-    }
-    uio_setresid(uio, (uio_resid(uio) + lost));
-
-    if (starting_resid == uio_resid(uio))
-        uio->uio_offset = 0;
-
-    return (error);
-}
-
-
-/*
- * validfsnode - test to see if a file system supports VGET
- *
- * This can cause context switching, so caller should be lock safe
- */
-int
-validfsnode(struct mount *fsnode)
-{
-
-	/*
-	 * Just check to see if the the mount flag is set, if it is we assume the
-	 * file system supports all of volfs symantecs
-	 */
-
-    if ((! (fsnode->mnt_kern_flag & MNTK_UNMOUNT)) && (fsnode->mnt_flag & MNT_DOVOLFS))
-		return 1;
-	else 
-		return 0;
-}
-
-/*
- * volfs_pathconf - Return POSIX pathconf information applicable to ufs filesystems.
- */
-static int
-volfs_pathconf(ap)
-    struct vnop_pathconf_args	/* { struct vnode *a_vp; int a_name; int
-        *a_retval; vfs_context_t a_context; } */ *ap;
-{
-    switch (ap->a_name)
-      {
-        case _PC_LINK_MAX:
-            *ap->a_retval = LINK_MAX;
-            return (0);
-        case _PC_NAME_MAX:
-            *ap->a_retval = NAME_MAX;
-            return (0);
-        case _PC_PATH_MAX:
-            *ap->a_retval = PATH_MAX;
-            return (0);
-        case _PC_PIPE_BUF:
-            *ap->a_retval = PIPE_BUF;
-            return (0);
-        case _PC_CHOWN_RESTRICTED:
-            *ap->a_retval = 1;
-            return (0);
-        case _PC_NO_TRUNC:
-            *ap->a_retval = 1;
-            return (0);
-        default:
-            return (EINVAL);
-      }
-    /* NOTREACHED */
-}
-
-/* 
- * get_parentvp() - internal routine that tries to lookup the parent of vpp.
- * On success, *vpp is the parent vp and is returned with a reference. 
- */
-static int
-get_parentvp(struct vnode **vpp, struct mount *mp, vfs_context_t context)
-{
-	int result;
-	struct vnode_attr va;
-	struct vnode *child_vp = *vpp;
-	
-	VATTR_INIT(&va);
-	VATTR_WANTED(&va, va_parentid);
-	result = vnode_getattr(child_vp, &va, context);
-	if (result) {
-		return result;
-	}
-
-	/* Shift attention to the parent directory vnode: */
-	result = VFS_VGET(mp, (ino64_t)va.va_parentid, vpp, context);
-
-	if (result == 0 && child_vp->v_parent != *vpp) {
-	        vnode_update_identity(child_vp, *vpp, NULL, 0, 0, VNODE_UPDATE_PARENT);
-	}
-
-	return result;
-}	
-
-
-/*
- * Look up the parent directory of a given vnode.
- */
-static int
-lookup_parent(vnode_t child_vp, vnode_t *parent_vpp, int is_authorized, vfs_context_t context)
-{
-	struct componentname cn;
-	vnode_t new_vp;
-	int error;
-
-	*parent_vpp = NULLVP;
-
-	if (is_authorized == 0) {
-		error = vnode_authorize(child_vp, NULL, KAUTH_VNODE_SEARCH, context);
-		if (error != 0) {
-			return (error);
-		}
-	}
-	new_vp = child_vp->v_parent;
-
-	if (new_vp != NULLVP) {
-	        if ( (error = vnode_getwithref(new_vp)) == 0 )
-		        *parent_vpp = new_vp;
-		return (error);
-	}
-	bzero(&cn, sizeof(cn));
-	cn.cn_nameiop = LOOKUP;
-	cn.cn_context = context;
-	cn.cn_pnbuf = CAST_DOWN(caddr_t, &gDotDot);
-	cn.cn_pnlen = strlen(cn.cn_pnbuf);
-	cn.cn_nameptr = cn.cn_pnbuf;
-	cn.cn_namelen = cn.cn_pnlen;
-	cn.cn_flags = (FOLLOW | LOCKLEAF | ISLASTCN | ISDOTDOT);
-
-	error = VNOP_LOOKUP(child_vp, &new_vp, &cn, context);
-	if (error != 0) {
-		return(error);
-	}
-	if (new_vp == child_vp) {
-		vnode_put(new_vp);
-		return ELOOP;
-	}
-	if (child_vp->v_parent == NULLVP) {
-	        vnode_update_identity(child_vp, new_vp, NULL, 0, 0, VNODE_UPDATE_PARENT);
-	}
-	*parent_vpp = new_vp;
-	return 0;
-}
-
-
-/*
- * 	verify_fullpathaccess(ret_vnode);
- */
-
-static int
-verify_fullpathaccess(struct vnode *targetvp, vfs_context_t context)
-{
-	struct vnode *vp, *parent_vp;
-	struct mount *mp = targetvp->v_mount;
-	struct proc *p = vfs_context_proc(context);
-	int result;
-	int dp_authorized;
-	struct filedesc *fdp = p->p_fd;	/* pointer to file descriptor state */
-	
-	vp = targetvp;
-	dp_authorized = 0;
-	
-	/* get the parent directory. */
-	if ((vp->v_flag & VROOT) == 0 && vp != fdp->fd_cdir && vp != fdp->fd_rdir) {
-		if (vp->v_parent == NULLVP || (vp->v_flag & VISHARDLINK) || (vnode_getwithref(vp->v_parent) != 0)) {
-			if (vp->v_type == VDIR) {
-				result = lookup_parent(vp, &parent_vp, dp_authorized, context);
-	
-				/*
-				 * If the lookup fails with EACCES and the vp is a directory,
-				 * we should try again but bypass authorization check. Without this
-				 * workaround directories that you can navigate to but not traverse will 
-				 * disappear when clicked in the Finder.
-				 */
-				if (result == EACCES && (vp->v_flag & VROOT) == 0) {
-					dp_authorized = 1;  /* bypass auth check */
-					if (lookup_parent(vp, &parent_vp, dp_authorized, context) == 0) {
-						result = 0;
-					}
-					dp_authorized = 0; /* force us to authorize */
-				}
-				vp = parent_vp;
-			}
-			else {
-				/*
-				 * this is not a directory so we must get parent object ID
-				 */
-				result = get_parentvp(&vp, mp, context);
-				parent_vp = vp;
-			}
-			if (result != 0) 
-				goto err_exit;
-		}
-		else {
-			/*
-			 * we where able to get a reference on v_parent
-			 */
-			parent_vp = vp = vp->v_parent;
-		}
-	} 
-
-	/*
-	 * Keep going up until either the process's root or the process's working 
-	 * directory is hit, either one of which are potential valid starting points 
-	 * for a full pathname
-	 */
-	while (vp != NULLVP) {
-
-		result = reverse_lookup(vp, &parent_vp, fdp, context, &dp_authorized);
-		if (result == 0) {
-			/*
-			 * we're done and we have access
-			 */
-			break;
-		}
-		if (vp != parent_vp) {
-		        /*
-			 * we where able to walk up the parent chain so now we don't need
-			 * vp any longer
-			 */
-			vnode_put(vp);	
-			vp = parent_vp;
-		}
-		/*
-		 * we have a referenced vp at this point... if dp_authorized == 1, than
-		 * it's been authorized for search, but v_parent was NULL...
-		 * if dp_authorized == 0, than we need to do the authorization check
-		 * before looking up the parent
-		 */
-		if ((vp->v_flag & VROOT) != 0 ||
-		    vp == fdp->fd_cdir || vp == fdp->fd_rdir) {
-		        /*
-			 * we're already at the termination point, which implies that
-			 * the authorization check in the cache failed (otherwise we
-			 * would have returned 'done' from "reverse_lookup"... so,
-			 * do the authorization and bail
-			 */
-			result = vnode_authorize(vp, NULL, KAUTH_VNODE_SEARCH, context);
-			goto lookup_exit;
-		}
-		result = lookup_parent(vp, &parent_vp, dp_authorized, context);
-		if (result != 0) {
-			goto lookup_exit;
-		}
-		if (vp != parent_vp) {
-		        /*
-			 * got the parent so now we don't need vp any longer
-			 */
-			vnode_put(vp);	
-			vp = parent_vp;
-		}
-	} /* while loop */
-
-	/*
-	 * Success: the caller has complete access to the initial vnode
-	 */
-	result = 0; 
-
-lookup_exit:
-	if (vp != NULLVP && vp != targetvp) {
-		vnode_put(vp);
-	}
-		
-err_exit:
-	return result;
-};
-
-
-/*
- * get_fsvnode - internal routine to create a vnode for a file system.  Called with mount pointer,
- *   id of filesystem to lookup and pointer to vnode pointer to fill in
- */
-static int
-get_fsvnode(struct mount *our_mount, int id, vnode_t *ret_vnode)
-{
-    struct mount       *cur_mount;
-	fsid_t				cur_fsid;
-    struct vnode       *cur_vnode;
-    struct volfs_vndata *cur_privdata;
-	int					retval;
-	struct vnode_fsparam vfsp;
-	int	vid = 0;
-
-    /*
-     * OK, first look up the matching mount on the list of mounted file systems
-     */
-	/* the following will return the mount point with vfs_busy held */
-	cur_mount = mount_lookupby_volfsid(id, 1);
-
-    if (cur_mount == NULL) {
-        /*
-         * No mounted file system by the specified ID currently exists in the system.
-         *
-         * XXX We could deal with a vnode that is still hanging about for an FS that
-         * does not exists or has been unmounted now, or count on the update below
-         * to happen later...
-         */
-        *ret_vnode = NULL;
-        return ENOENT;
-    };
-
-	cur_fsid = cur_mount->mnt_vfsstat.f_fsid;
-
-    /*
-     * Now search the list attached to the mount structure to
-     * see if this vnode is already floating around
-     */
-search_vnodelist:
-	mount_lock(our_mount);
-	TAILQ_FOREACH(cur_vnode, &our_mount->mnt_vnodelist, v_mntvnodes)  {
-        cur_privdata = (struct volfs_vndata *) cur_vnode->v_data;
-        if (cur_privdata->nodeID == (unsigned int)id)
-            {
-            if (cur_privdata->fs_mount != cur_mount) {
-                cur_privdata->fs_mount = cur_mount;
-                cur_privdata->fs_fsid = cur_fsid;
-            };
-            break;
-            }
-	}
-	mount_unlock(our_mount);
-
-    if (cur_vnode) {
-        vid = vnode_vid(cur_vnode);
-
-        /*
-	 * use vnode_getwithvid since it will wait for a vnode currently being
-	 * terminated... if it returns an error, cur_vnode will not be what we
-	 * think it is, try again
-	 */
-        if (vnode_getwithvid(cur_vnode, vid) != 0) {
-            goto search_vnodelist;
-        };
-        }
-    else
-      {
-        MALLOC(cur_privdata, struct volfs_vndata *,
-               sizeof(struct volfs_vndata), M_VOLFSNODE, M_WAITOK);
-
-        cur_privdata->vnode_type = VOLFS_FSNODE;
-        cur_privdata->nodeID = id;
-
-        cur_privdata->fs_mount = cur_mount;
-        cur_privdata->fs_fsid = cur_fsid;
-
-		vfsp.vnfs_mp = our_mount;
-		vfsp.vnfs_vtype = VDIR;
-		vfsp.vnfs_str = "volfs";
-		vfsp.vnfs_dvp = 0;
-		vfsp.vnfs_fsnode = cur_privdata;
-		vfsp.vnfs_cnp = 0;
-		vfsp.vnfs_vops = volfs_vnodeop_p;
-		vfsp.vnfs_rdev = 0;
-		vfsp.vnfs_filesize = 0;
-		vfsp.vnfs_flags = VNFS_NOCACHE | VNFS_CANTCACHE;
-		vfsp.vnfs_marksystem = 0;
-		vfsp.vnfs_markroot = 0;
-
-		retval = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &cur_vnode);
-        if (retval != 0) {
-            FREE(cur_privdata, M_VOLFSNODE);
-			goto out;
-        };
-		cur_vnode->v_tag = VT_VOLFS;
-			
-      }
-
-    *ret_vnode = cur_vnode;
-	retval = 0;
-out:
-	vfs_unbusy(cur_mount);
-    return (retval);
-}
-
-
-
-/*
- * get_filevnode - returns the vnode for the given id within a filesystem.  The parent vnode
- * 	   is a filesystem, id is the 32-bit id of the file/directory and ret_vnode is a pointer
- *		to a vnode pointer
- */
-static int
-get_filevnode(struct mount *parent_fs, u_int id, vnode_t *ret_vnode, vfs_context_t context)
-{
-    int                 retval;
-
-again:
-	/*
-	 * Special case 2 to mean the root of a file system
-	 */
-	if (id == 2)
-		retval = VFS_ROOT(parent_fs, ret_vnode, context);
-	else
-		retval = VFS_VGET(parent_fs, (ino64_t)id, ret_vnode, context);
-	if (retval) goto error;
-
-	retval = verify_fullpathaccess(*ret_vnode, context);
-	if (retval) {
-		/* An error was encountered verifying that the caller has,
-		   in fact, got access all the way from "/" or their working
-		   directory to the specified item...
-		 */
-		vnode_put(*ret_vnode);
-		*ret_vnode = NULL;
-		/* vnode was recycled during access verification. */
-		if (retval == EAGAIN) {
-			goto again;
-		}
-	};
-
-error:
-    return (retval);
-}
-
-
-static int
-volfs_lookup(struct vnop_lookup_args *ap)
-{
-	struct volfs_vndata *priv_data;
-	char  *nameptr;
-	long  namelen;
-	struct mount  *parent_fs;
-	vnode_t	vp;
-	int  isdot_or_dotdot = 0;
-	int  ret_err = ENOENT;
-	char firstchar;
-	int ret_val;
-
-#if 0
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSVN, 8)) | DBG_FUNC_START,
-		     (unsigned int)ap->a_dvp, (unsigned int)ap->a_cnp, (unsigned int)p, 0, 0);
-#endif
-	priv_data = ap->a_dvp->v_data;
-	nameptr = ap->a_cnp->cn_nameptr;
-	namelen = ap->a_cnp->cn_namelen;
-	firstchar = nameptr[0];
-
-	/* First check for "." and ".." */
-	if (firstchar == '.') {
-		if (namelen == 1) {
-			/* "." requested */
-			isdot_or_dotdot = 1;
-			*ap->a_vpp = ap->a_dvp;
-			vnode_get(*ap->a_vpp);
-			ret_err = 0;
-		} else if (nameptr[1] == '.' && namelen == 2) {
-			/* ".." requested */
-			isdot_or_dotdot = 1;
-			ret_err = VFS_ROOT(ap->a_dvp->v_mount, ap->a_vpp, ap->a_context);
-		}
-	} else if (firstchar == '@') { /* '@' is alias for system root */
-		if ((namelen == 1) && (priv_data->vnode_type != VOLFS_ROOT)) {
-			/* the following returns with iteration count on mount point */
-			parent_fs = mount_list_lookupby_fsid(&priv_data->fs_fsid, 0, 1);
-			if (parent_fs) {
-				ret_val = vfs_busy(parent_fs, LK_NOWAIT);
-				mount_iterdrop(parent_fs);
-				if (ret_val !=0) {
-					*ap->a_vpp = NULL;
-					ret_err = ENOENT;
-				} else {
-					ret_err = VFS_ROOT(parent_fs, ap->a_vpp, ap->a_context);
-					vfs_unbusy(parent_fs);
-				}
-			} else {
-				*ap->a_vpp = NULL;
-				ret_err = ENOENT;
-			}
-		} else {
-			*ap->a_vpp = NULL;
-			ret_err = ENOENT;
-		}
-	} else if (namelen <= 10 && firstchar > '0' && firstchar <= '9') {
-		char	*check_ptr;
-		u_long	id;
-
-		id = strtoul(nameptr, &check_ptr, 10);
-
-		/*
-		 * strtol will leave us at the first non-numeric character.
-		 * we've checked to make sure the component name does
-		 * begin with a numeric so check_ptr must wind up on
-		 * the terminating null or there was other junk following the
-		 * number
-		 */
-		if ((check_ptr - nameptr) == namelen) {
-			if (priv_data->vnode_type == VOLFS_ROOT) {
-				/*
-				 * OPTIMIZATION
-				 *
-				 * Obtain the mountpoint and call VFS_VGET in
-				 * one step (ie without creating a vnode for
-				 * the mountpoint).
-				 */
-				if (check_ptr[0] == '/' &&
-				    check_ptr[1] > '0' && check_ptr[1] <= '9') {
-					struct mount *mp;
-					struct vnode *vp;
-					u_long	id2;
-					char *endptr;
-		
-					/* this call will return mount point with vfs_busy held */
-					mp = mount_lookupby_volfsid(id, 1);
-					if (mp == NULL) {
-						*ap->a_vpp = NULL;
-						return ENOENT;
-					}
-					id2 = strtoul(&check_ptr[1], &endptr, 10);
-					if ((endptr[0] == '/' || endptr[0] == '\0') &&
-					    get_filevnode(mp, id2, &vp, ap->a_context) == 0) {
-						ap->a_cnp->cn_consume = endptr - check_ptr;
-						*ap->a_vpp = vp;
-						vfs_unbusy(mp);
-						return (0);
-					}
-					vfs_unbusy(mp);
-				}
-				/* Fall through to default behavior... */
-
-				ret_err = get_fsvnode(ap->a_dvp->v_mount, id, ap->a_vpp);
-
-			} else {
-				parent_fs = mount_list_lookupby_fsid(&priv_data->fs_fsid, 0, 1);
-				if (parent_fs) {
-					ret_val = vfs_busy(parent_fs, LK_NOWAIT);
-					mount_iterdrop(parent_fs);
-					if (ret_val !=0) {
-						*ap->a_vpp = NULL;
-						ret_err = ENOENT;
-					} else {
-						ret_err = get_filevnode(parent_fs, id, ap->a_vpp, ap->a_context);
-						vfs_unbusy(parent_fs);
-					}
-				} else {
-						*ap->a_vpp = NULL;
-						ret_err = ENOENT;
-				}
-			}
-		}
-	}
-	vp = *ap->a_vpp;
-
-	if ( ret_err == 0 && !isdot_or_dotdot && (vp != NULLVP) && (vp->v_parent == NULLVP))
-	        vnode_update_identity(vp, ap->a_dvp, NULL, 0, 0, VNODE_UPDATE_PARENT);
-
-#if 0
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSVN, 8)) | DBG_FUNC_START,
-		     (unsigned int)ap->a_dvp, (unsigned int)ap->a_cnp, (unsigned int)p, ret_err, 0);
-#endif
-	return (ret_err);
-}
-
diff --git a/bsd/net/Makefile b/bsd/net/Makefile
index e29e3a849..1c5ed5df9 100644
--- a/bsd/net/Makefile
+++ b/bsd/net/Makefile
@@ -29,7 +29,7 @@ DATAFILES= \
 
 KERNELFILES= \
 	kpi_interface.h kpi_interfacefilter.h kpi_protocol.h \
-	if_ether.h init.h
+	if_ether.h init.h radix.h
 
 PRIVATE_DATAFILES = \
 	if_atm.h if_vlan_var.h if_ppp.h firewire.h \
@@ -37,7 +37,7 @@ PRIVATE_DATAFILES = \
 	raw_cb.h etherdefs.h iso88025.h
 
 PRIVATE_KERNELFILES = ${KERNELFILES} \
-	bpfdesc.h dlil_pvt.h if_faith.h ppp_comp.h \
+	bpfdesc.h dlil_pvt.h ppp_comp.h \
 	zlib.h bpf_compat.h net_osdep.h
 
 INSTALL_MI_LIST	= ${DATAFILES}
diff --git a/bsd/net/bpf.c b/bsd/net/bpf.c
index 5581b6ea2..5880506f1 100644
--- a/bsd/net/bpf.c
+++ b/bsd/net/bpf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1991, 1993
@@ -60,6 +66,12 @@
  *
  * $FreeBSD: src/sys/net/bpf.c,v 1.59.2.5 2001/01/05 04:49:09 jdp Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include "bpf.h"
 
@@ -101,15 +113,16 @@
 #include <sys/sysctl.h>
 #include <net/firewire.h>
 
-#include <machine/spl.h>
 #include <miscfs/devfs/devfs.h>
 #include <net/dlil.h>
 
 #include <kern/locks.h>
 
-extern int tvtohz(struct timeval *);
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
 
-#if NBPFILTER > 0
+extern int tvtohz(struct timeval *);
 
 /*
  * Older BSDs don't have kernel malloc.
@@ -174,25 +187,27 @@ static lck_attr_t		*bpf_mlock_attr;
 #endif /* __APPLE__ */
 
 static int	bpf_allocbufs(struct bpf_d *);
-static void	bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
+static errno_t	bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
 static void	bpf_detachd(struct bpf_d *d);
 static void	bpf_freed(struct bpf_d *);
 static void	bpf_mcopy(const void *, void *, size_t);
 static int	bpf_movein(struct uio *, int,
 		    struct mbuf **, struct sockaddr *, int *);
-static int	bpf_setif(struct bpf_d *, struct ifreq *);
+static int	bpf_setif(struct bpf_d *, ifnet_t ifp, u_int32_t dlt);
 static void bpf_wakeup(struct bpf_d *);
 static void	catchpacket(struct bpf_d *, u_char *, u_int,
 		    u_int, void (*)(const void *, void *, size_t));
 static void	reset_d(struct bpf_d *);
-static int bpf_setf(struct bpf_d *, struct user_bpf_program *);
+static int bpf_setf(struct bpf_d *, u_int bf_len, user_addr_t bf_insns);
+static int	bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *);
+static int	bpf_setdlt(struct bpf_d *, u_int);
 
 /*static  void *bpf_devfs_token[MAXBPFILTER];*/
 
 static  int bpf_devsw_installed;
 
 void bpf_init(void *unused);
-int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m);
+static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m);
 
 /*
  * Darwin differs from BSD here, the following are static
@@ -233,86 +248,99 @@ bpf_movein(struct uio *uio, int linktype, struct mbuf **mp, struct sockaddr *soc
 	struct mbuf *m;
 	int error;
 	int len;
+	uint8_t sa_family;
 	int hlen;
 
-	if (sockp) {
-		/*
-		 * Build a sockaddr based on the data link layer type.
-		 * We do this at this level because the ethernet header
-		 * is copied directly into the data field of the sockaddr.
-		 * In the case of SLIP, there is no header and the packet
-		 * is forwarded as is.
-		 * Also, we are careful to leave room at the front of the mbuf
-		 * for the link level header.
-		 */
-		switch (linktype) {
+	switch (linktype) {
 	
-		case DLT_SLIP:
-			sockp->sa_family = AF_INET;
-			hlen = 0;
-			break;
+#if SLIP
+	case DLT_SLIP:
+		sa_family = AF_INET;
+		hlen = 0;
+		break;
+#endif /* SLIP */
 	
-		case DLT_EN10MB:
-			sockp->sa_family = AF_UNSPEC;
-			/* XXX Would MAXLINKHDR be better? */
-			hlen = sizeof(struct ether_header);
-			break;
+	case DLT_EN10MB:
+		sa_family = AF_UNSPEC;
+		/* XXX Would MAXLINKHDR be better? */
+		hlen = sizeof(struct ether_header);
+		break;
 	
-		case DLT_FDDI:
+#if FDDI
+	case DLT_FDDI:
 	#if defined(__FreeBSD__) || defined(__bsdi__)
-			sockp->sa_family = AF_IMPLINK;
-			hlen = 0;
+		sa_family = AF_IMPLINK;
+		hlen = 0;
 	#else
-			sockp->sa_family = AF_UNSPEC;
-			/* XXX 4(FORMAC)+6(dst)+6(src)+3(LLC)+5(SNAP) */
-			hlen = 24;
+		sa_family = AF_UNSPEC;
+		/* XXX 4(FORMAC)+6(dst)+6(src)+3(LLC)+5(SNAP) */
+		hlen = 24;
 	#endif
-			break;
+		break;
+#endif /* FDDI */
 	
-		case DLT_RAW:
-		case DLT_NULL:
-			sockp->sa_family = AF_UNSPEC;
-			hlen = 0;
-			break;
+	case DLT_RAW:
+	case DLT_NULL:
+		sa_family = AF_UNSPEC;
+		hlen = 0;
+		break;
 	
 	#ifdef __FreeBSD__
-		case DLT_ATM_RFC1483:
-			/*
-			 * en atm driver requires 4-byte atm pseudo header.
-			 * though it isn't standard, vpi:vci needs to be
-			 * specified anyway.
-			 */
-			sockp->sa_family = AF_UNSPEC;
-			hlen = 12; 	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
-			break;
+	case DLT_ATM_RFC1483:
+		/*
+		 * en atm driver requires 4-byte atm pseudo header.
+		 * though it isn't standard, vpi:vci needs to be
+		 * specified anyway.
+		 */
+		sa_family = AF_UNSPEC;
+		hlen = 12; 	/* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
+		break;
 	#endif
-		case DLT_PPP:
-			sockp->sa_family = AF_UNSPEC;
-			hlen = 4;	/* This should match PPP_HDRLEN */
-			break;
+
+	case DLT_PPP:
+		sa_family = AF_UNSPEC;
+		hlen = 4;	/* This should match PPP_HDRLEN */
+		break;
 	
-		case DLT_APPLE_IP_OVER_IEEE1394:
-			sockp->sa_family = AF_UNSPEC;
-			hlen = sizeof(struct firewire_header);
-			break;
+	case DLT_APPLE_IP_OVER_IEEE1394:
+		sa_family = AF_UNSPEC;
+		hlen = sizeof(struct firewire_header);
+		break;
 	
-		default:
-			return (EIO);
-		}
-		if ((hlen + SOCKADDR_HDR_LEN) > sockp->sa_len) {
-			return (EIO);
-		}
-	}
-	else {
-		hlen = 0;
+	default:
+		return (EIO);
 	}
-	
+
 	// LP64todo - fix this!
 	len = uio_resid(uio);
 	*datlen = len - hlen;
 	if ((unsigned)len > MCLBYTES)
 		return (EIO);
 
+	if (sockp) {
+		/*
+		 * Build a sockaddr based on the data link layer type.
+		 * We do this at this level because the ethernet header
+		 * is copied directly into the data field of the sockaddr.
+		 * In the case of SLIP, there is no header and the packet
+		 * is forwarded as is.
+		 * Also, we are careful to leave room at the front of the mbuf
+		 * for the link level header.
+		 */
+		if ((hlen + SOCKADDR_HDR_LEN) > sockp->sa_len) {
+			return (EIO);
+		}
+		sockp->sa_family = sa_family;
+	} else {
+		/*
+		 * We're directly sending the packet data supplied by
+		 * the user; we don't need to make room for the link
+		 * header, and don't need the header length value any
+		 * more, so set it to 0.
+		 */
+		hlen = 0;
+	}
+	
 	MGETHDR(m, M_WAIT, MT_DATA);
 	if (m == 0)
 		return (ENOBUFS);
@@ -355,19 +383,6 @@ bpf_movein(struct uio *uio, int linktype, struct mbuf **mp, struct sockaddr *soc
 }
 
 #ifdef __APPLE__
-/* Callback registered with Ethernet driver. */
-int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m)
-{
-    /*
-     * Do nothing if the BPF tap has been turned off.
-     * This is to protect from a potential race where this
-     * call blocks on the lock. And in the meantime
-     * BPF is turned off, which will clear if_bpf.
-     */
-    if (ifp->if_bpf)
-        bpf_mtap(ifp, m);
-    return 0;
-}
 
 /*
  * The dynamic addition of a new device node must block all processes that are opening 
@@ -429,11 +444,13 @@ done:
 
 /*
  * Attach file to the bpf interface, i.e. make d listen on bp.
- * Must be called at splimp.
  */
-static void
+static errno_t
 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
 {
+	int first = bp->bif_dlist == NULL;
+	int	error = 0;
+	
 	/*
 	 * Point d at bp, and add d to the interface's list of listeners.
 	 * Finally, point the driver's bpf cookie at the interface so
@@ -442,12 +459,28 @@ bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
 	d->bd_bif = bp;
 	d->bd_next = bp->bif_dlist;
 	bp->bif_dlist = d;
+	
+	if (first) {
+		/* Find the default bpf entry for this ifp */
+		if (bp->bif_ifp->if_bpf == NULL) {
+			struct bpf_if	*primary;
+			
+			for (primary = bpf_iflist; primary && primary->bif_ifp != bp->bif_ifp;
+				 primary = primary->bif_next)
+				;
+		
+			bp->bif_ifp->if_bpf = primary;
+		}
+		
+		/* Only call dlil_set_bpf_tap for primary dlt */
+		if (bp->bif_ifp->if_bpf == bp)
+			dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback);
+		
+		if (bp->bif_tap)
+			error = bp->bif_tap(bp->bif_ifp, bp->bif_dlt, BPF_TAP_INPUT_OUTPUT);
+	}
 
-	bp->bif_ifp->if_bpf = bp;
-
-#ifdef __APPLE__
-	dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT, bpf_tap_callback);
-#endif
+	return error;
 }
 
 /*
@@ -458,21 +491,44 @@ bpf_detachd(struct bpf_d *d)
 {
 	struct bpf_d **p;
 	struct bpf_if *bp;
-#ifdef __APPLE__
 	struct ifnet  *ifp;
 
 	ifp = d->bd_bif->bif_ifp;
-
-#endif
-
 	bp = d->bd_bif;
+	
+	/* Remove d from the interface's descriptor list. */
+	p = &bp->bif_dlist;
+	while (*p != d) {
+		p = &(*p)->bd_next;
+		if (*p == 0)
+			panic("bpf_detachd: descriptor not in list");
+	}
+	*p = (*p)->bd_next;
+	if (bp->bif_dlist == 0) {
+		/*
+		 * Let the driver know that there are no more listeners.
+		 */
+		/* Only call dlil_set_bpf_tap for primary dlt */
+		if (bp->bif_ifp->if_bpf == bp)
+			dlil_set_bpf_tap(ifp, BPF_TAP_DISABLE, NULL);
+		if (bp->bif_tap)
+			bp->bif_tap(ifp, bp->bif_dlt, BPF_TAP_DISABLE);
+		
+		for (bp = bpf_iflist; bp; bp = bp->bif_next)
+			if (bp->bif_ifp == ifp && bp->bif_dlist != 0)
+				break;
+		if (bp == NULL)
+			ifp->if_bpf = NULL;
+	}
+	d->bd_bif = NULL;
 	/*
 	 * Check if this descriptor had requested promiscuous mode.
 	 * If so, turn it off.
 	 */
 	if (d->bd_promisc) {
 		d->bd_promisc = 0;
-		if (ifnet_set_promiscuous(bp->bif_ifp, 0))
+		lck_mtx_unlock(bpf_mlock);
+		if (ifnet_set_promiscuous(ifp, 0)) {
 			/*
 			 * Something is really wrong if we were able to put
 			 * the driver into promiscuous mode, but can't
@@ -480,24 +536,9 @@ bpf_detachd(struct bpf_d *d)
 			 * Most likely the network interface is gone.
 			 */
 			printf("bpf: ifnet_set_promiscuous failed");
+		}
+		lck_mtx_lock(bpf_mlock);
 	}
-	/* Remove d from the interface's descriptor list. */
-	p = &bp->bif_dlist;
-	while (*p != d) {
-		p = &(*p)->bd_next;
-		if (*p == 0)
-			panic("bpf_detachd: descriptor not in list");
-	}
-	*p = (*p)->bd_next;
-	if (bp->bif_dlist == 0) {
-		/*
-		 * Let the driver know that there are no more listeners.
-		 */
-		if (ifp->if_set_bpf_tap)
-			(*ifp->if_set_bpf_tap)(ifp, BPF_TAP_DISABLE, 0);
-		d->bd_bif->bif_ifp->if_bpf = 0;
-	}
-	d->bd_bif = 0;
 }
 
 
@@ -506,14 +547,17 @@ bpf_detachd(struct bpf_d *d)
  * EBUSY if file is open by another process.
  */
 /* ARGSUSED */
-	int
-bpfopen(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *p)
+int
+bpfopen(dev_t dev, __unused int flags, __unused int fmt,
+	__unused struct proc *p)
 {
-	register struct bpf_d *d;
+	struct bpf_d *d;
 
-	if ((unsigned int) minor(dev) >= nbpfilter)
+	lck_mtx_lock(bpf_mlock);
+	if ((unsigned int) minor(dev) >= nbpfilter) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-		
+	}
 	/* 
 	 * New device nodes are created on demand when opening the last one. 
 	 * The programming model is for processes to loop on the minor starting at 0 
@@ -534,16 +578,18 @@ bpfopen(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *p
 	 * in the same lockin context otherwise the device may be leaked because the vnode use count 
 	 * will be unpextectly greater than 1 when close() is called.
 	 */
-	if (bpf_dtab[minor(dev)] == 0)
+	if (bpf_dtab[minor(dev)] == 0) {
 		bpf_dtab[minor(dev)] = (void *)1;	/* Mark opening */
-	else
+	} else {
+		lck_mtx_unlock(bpf_mlock);
 		return (EBUSY);
-
+	}
 	d = (struct bpf_d *)_MALLOC(sizeof(struct bpf_d), M_DEVBUF, M_WAIT);
 	if (d == NULL) {
 		/* this really is a catastrophic failure */
 		printf("bpfopen: malloc bpf_d failed\n");
-		bpf_dtab[minor(dev)] = 0;
+		bpf_dtab[minor(dev)] = NULL;
+		lck_mtx_unlock(bpf_mlock);
 		return ENOMEM;
 	}
 	bzero(d, sizeof(struct bpf_d));
@@ -557,7 +603,12 @@ bpfopen(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *p
 	d->bd_bufsize = bpf_bufsize;
 	d->bd_sig = SIGIO;
 	d->bd_seesent = 1;
+#if CONFIG_MACF_NET
+	mac_bpfdesc_label_init(d);
+	mac_bpfdesc_label_associate(kauth_cred_get(), d);
+#endif
 	bpf_dtab[minor(dev)] = d; 				/* Mark opened */
+	lck_mtx_unlock(bpf_mlock);
 
 	return (0);
 }
@@ -567,29 +618,34 @@ bpfopen(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *p
  * deallocating its buffers, and marking it free.
  */
 /* ARGSUSED */
-	int
-bpfclose(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *p)
+int
+bpfclose(dev_t dev, __unused int flags, __unused int fmt,
+	 __unused struct proc *p)
 {
-	register struct bpf_d *d;
+	struct bpf_d *d;
+
+	/* Take BPF lock to ensure no other thread is using the device */
+	lck_mtx_lock(bpf_mlock);
 
 	d = bpf_dtab[minor(dev)];
-	if (d == 0 || d == (void *)1)
+	if (d == 0 || d == (void *)1) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-	
+	}	
 	bpf_dtab[minor(dev)] = (void *)1;		/* Mark closing */
 
-	/* Take BPF lock to ensure no other thread is using the device */
-	lck_mtx_lock(bpf_mlock);
-
 	if (d->bd_bif)
 		bpf_detachd(d);
 	selthreadclear(&d->bd_sel);
+#if CONFIG_MACF_NET
+	mac_bpfdesc_label_destroy(d);
+#endif
 	bpf_freed(d);
 
+	/* Mark free in same context as bpfopen comes to check */
+	bpf_dtab[minor(dev)] = NULL;			/* Mark closed */
 	lck_mtx_unlock(bpf_mlock);
 	
-	/* Mark free in same context as bpfopen comes to check */
-	bpf_dtab[minor(dev)] = 0;				/* Mark closed */
 	_FREE(d, M_DEVBUF);
 	
 	return (0);
@@ -601,7 +657,7 @@ bpfclose(dev_t dev, __unused int flags, __unused int fmt, __unused struct proc *
 static int
 bpf_sleep(struct bpf_d *d, int pri, const char *wmesg, int timo)
 {
-	register int st;
+	int st;
 
 	lck_mtx_unlock(bpf_mlock);
 	
@@ -622,22 +678,23 @@ bpf_sleep(struct bpf_d *d, int pri, const char *wmesg, int timo)
 	(d)->bd_hlen = (d)->bd_slen; \
 	(d)->bd_sbuf = (d)->bd_fbuf; \
 	(d)->bd_slen = 0; \
-	(d)->bd_fbuf = 0;
+	(d)->bd_fbuf = NULL;
 /*
  *  bpfread - read next chunk of packets from buffers
  */
-	int
+int
 bpfread(dev_t dev, struct uio *uio, int ioflag)
 {
-	register struct bpf_d *d;
+	struct bpf_d *d;
 	int error;
-	int s;
+
+	lck_mtx_lock(bpf_mlock);
 
 	d = bpf_dtab[minor(dev)];
-	if (d == 0 || d == (void *)1)
+	if (d == 0 || d == (void *)1) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-
-	lck_mtx_lock(bpf_mlock);
+	}
 
 
 	/*
@@ -650,7 +707,6 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 		return (EINVAL);
 	}
 
-	s = splimp();
 	/*
 	 * If the hold buffer is empty, then do a timed sleep, which
 	 * ends when the timeout expires or when enough packets
@@ -674,7 +730,6 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 		 * it before using it again.
 		 */
 		if (d->bd_bif == NULL) {
-			splx(s);
 			lck_mtx_unlock(bpf_mlock);
 			return (ENXIO);
 		}
@@ -684,8 +739,15 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 		else
 			error = BPF_SLEEP(d, PRINET|PCATCH, "bpf",
 					  d->bd_rtout);
+		/*
+		 * Make sure device is still opened
+		 */
+		d = bpf_dtab[minor(dev)];
+		if (d == 0 || d == (void *)1) {
+			lck_mtx_unlock(bpf_mlock);
+			return (ENXIO);
+		}
 		if (error == EINTR || error == ERESTART) {
-			splx(s);
 			lck_mtx_unlock(bpf_mlock);
 			return (error);
 		}
@@ -704,7 +766,6 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 				break;
 
 			if (d->bd_slen == 0) {
-				splx(s);
 				lck_mtx_unlock(bpf_mlock);
 				return (0);
 			}
@@ -715,7 +776,6 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 	/*
 	 * At this point, we know we have something in the hold slot.
 	 */
-	splx(s);
 
 	/*
 	 * Move data from hold buffer into user space.
@@ -724,11 +784,9 @@ bpfread(dev_t dev, struct uio *uio, int ioflag)
 	 */
 	error = UIOMOVE(d->bd_hbuf, d->bd_hlen, UIO_READ, uio);
 
-	s = splimp();
 	d->bd_fbuf = d->bd_hbuf;
-	d->bd_hbuf = 0;
+	d->bd_hbuf = NULL;
 	d->bd_hlen = 0;
-	splx(s);
 	lck_mtx_unlock(bpf_mlock);
 	return (error);
 }
@@ -742,7 +800,7 @@ bpf_wakeup(struct bpf_d *d)
 {
 	wakeup((caddr_t)d);
 	if (d->bd_async && d->bd_sig && d->bd_sigio)
-		pgsigio(d->bd_sigio, d->bd_sig, 0);
+		pgsigio(d->bd_sigio, d->bd_sig);
 
 #if BSD >= 199103
 	selwakeup(&d->bd_sel);
@@ -762,55 +820,73 @@ bpf_wakeup(struct bpf_d *d)
 /* keep in sync with bpf_movein above: */
 #define MAX_DATALINK_HDR_LEN	(sizeof(struct firewire_header))
 
-	int
+int
 bpfwrite(dev_t dev, struct uio *uio, __unused int ioflag)
 {
-	register struct bpf_d *d;
+	struct bpf_d *d;
 	struct ifnet *ifp;
-	struct mbuf *m;
+	struct mbuf *m = NULL;
 	int error;
 	char 		  dst_buf[SOCKADDR_HDR_LEN + MAX_DATALINK_HDR_LEN];
 	int datlen;
 
+	lck_mtx_lock(bpf_mlock);
+
 	d = bpf_dtab[minor(dev)];
-	if (d == 0 || d == (void *)1)
+	if (d == 0 || d == (void *)1) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-	
-	lck_mtx_lock(bpf_mlock);
-	
+	}
 	if (d->bd_bif == 0) {
 		lck_mtx_unlock(bpf_mlock);
-	     return (ENXIO);
+		return (ENXIO);
 	}
 
 	ifp = d->bd_bif->bif_ifp;
 
 	if (uio->uio_resid == 0) {
 		lck_mtx_unlock(bpf_mlock);
-	     return (0);
+		return (0);
 	}
 	((struct sockaddr *)dst_buf)->sa_len = sizeof(dst_buf);
 	error = bpf_movein(uio, (int)d->bd_bif->bif_dlt, &m, 
-			   d->bd_hdrcmplt ? 0 : (struct sockaddr *)dst_buf, &datlen);
+			   d->bd_hdrcmplt ? NULL : (struct sockaddr *)dst_buf,
+			   &datlen);
 	if (error) {
 		lck_mtx_unlock(bpf_mlock);
-	     return (error);
+		return (error);
 	}
 
 	if ((unsigned)datlen > ifp->if_mtu) {
 		lck_mtx_unlock(bpf_mlock);
-	     return (EMSGSIZE);
+		m_freem(m);
+		return (EMSGSIZE);
+	}
+	
+	if ((error = ifp_use(ifp, kIfNetUseCount_MustNotBeZero)) != 0) {
+		lck_mtx_unlock(bpf_mlock);
+		m_freem(m);
+		return (error);
 	}
 
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_bpfdesc(d, m);
+#endif
 	lck_mtx_unlock(bpf_mlock);
 
 	if (d->bd_hdrcmplt) {
-		error = dlil_output(ifp, 0, m, NULL, NULL, 1);
+		if (d->bd_bif->bif_send)
+			error = d->bd_bif->bif_send(ifp, d->bd_bif->bif_dlt, m);
+		else
+			error = dlil_output(ifp, 0, m, NULL, NULL, 1);
 	}
 	else {
 		error = dlil_output(ifp, PF_INET, m, NULL, (struct sockaddr *)dst_buf, 0);
 	}
 	
+	if (ifp_unuse(ifp) != 0)
+		ifp_use_reached_zero(ifp);
+	
 	/*
 	 * The driver frees the mbuf.
 	 */
@@ -819,7 +895,7 @@ bpfwrite(dev_t dev, struct uio *uio, __unused int ioflag)
 
 /*
  * Reset a descriptor by flushing its packet buffer and clearing the
- * receive and drop counts.  Should be called at splimp.
+ * receive and drop counts.
  */
 static void
 reset_d(struct bpf_d *d)
@@ -827,7 +903,7 @@ reset_d(struct bpf_d *d)
 	if (d->bd_hbuf) {
 		/* Free the hold buffer. */
 		d->bd_fbuf = d->bd_hbuf;
-		d->bd_hbuf = 0;
+		d->bd_hbuf = NULL;
 	}
 	d->bd_slen = 0;
 	d->bd_hlen = 0;
@@ -857,16 +933,19 @@ reset_d(struct bpf_d *d)
  */
 /* ARGSUSED */
 int
-bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p)
+bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags,
+		 __unused struct proc *p)
 {
-	register struct bpf_d *d;
-	int s, error = 0;
+	struct bpf_d *d;
+	int error = 0;
+
+	lck_mtx_lock(bpf_mlock);
 
 	d = bpf_dtab[minor(dev)];
-	if (d == 0 || d == (void *)1)
+	if (d == 0 || d == (void *)1) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-
-	lck_mtx_lock(bpf_mlock);
+	}
 
 	switch (cmd) {
 
@@ -881,11 +960,9 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 		{
 			int n;
 
-			s = splimp();
 			n = d->bd_slen;
 			if (d->bd_hbuf)
 				n += d->bd_hlen;
-			splx(s);
 
 			*(int *)addr = n;
 			break;
@@ -899,7 +976,7 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 				error = EINVAL;
 			else {
 				ifp = d->bd_bif->bif_ifp;
-				error = dlil_ioctl(0, ifp, cmd, addr);
+				error = ifnet_ioctl(ifp, 0, cmd, addr);
 			}
 			break;
 		}
@@ -921,7 +998,7 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 		if (d->bd_bif != 0)
 			error = EINVAL;
 		else {
-			register u_int size = *(u_int *)addr;
+			u_int size = *(u_int *)addr;
 
 			if (size > bpf_maxbufsize)
 				*(u_int *)addr = size = bpf_maxbufsize;
@@ -935,28 +1012,29 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 	/*
 	 * Set link layer read filter.
 	 */
-	case BIOCSETF:
-		if (proc_is64bit(p)) {
-			error = bpf_setf(d, (struct user_bpf_program *)addr);
+	case BIOCSETF64:
+	case BIOCSETF: {
+		if (proc_is64bit(current_proc())) {
+		    struct bpf_program64 * 	prg64;
+
+		    prg64 = (struct bpf_program64 *)addr;
+		    error = bpf_setf(d, prg64->bf_len,
+				     prg64->bf_insns);
 		}
 		else {
-			struct bpf_program *	tmpp;
-			struct user_bpf_program	tmp;
-
-			tmpp = (struct bpf_program *)addr;
-			tmp.bf_len = tmpp->bf_len;
-			tmp.bf_insns = CAST_USER_ADDR_T(tmpp->bf_insns);
-			error = bpf_setf(d, &tmp);
+		    struct bpf_program *	prg;
+		    
+		    prg = (struct bpf_program *)addr;
+		    error = bpf_setf(d, prg->bf_len,
+				     CAST_USER_ADDR_T(prg->bf_insns));
 		}
 		break;
-
+	}
 	/*
 	 * Flush read packet buffer.
 	 */
 	case BIOCFLUSH:
-		s = splimp();
 		reset_d(d);
-		splx(s);
 		break;
 
 	/*
@@ -970,13 +1048,13 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 			error = EINVAL;
 			break;
 		}
-		s = splimp();
 		if (d->bd_promisc == 0) {
+			lck_mtx_unlock(bpf_mlock);
 			error = ifnet_set_promiscuous(d->bd_bif->bif_ifp, 1);
+			lck_mtx_lock(bpf_mlock);
 			if (error == 0)
 				d->bd_promisc = 1;
 		}
-		splx(s);
 		break;
 
 	/*
@@ -989,6 +1067,26 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 			*(u_int *)addr = d->bd_bif->bif_dlt;
 		break;
 
+	/*
+	 * Get a list of supported data link types.
+	 */
+	case BIOCGDLTLIST:
+			if (d->bd_bif == NULL)
+					error = EINVAL;
+			else
+					error = bpf_getdltlist(d, (struct bpf_dltlist *)addr);
+			break;
+
+	/*
+	 * Set data link type.
+	 */
+	case BIOCSDLT:
+			if (d->bd_bif == NULL)
+					error = EINVAL;
+			else
+					error = bpf_setdlt(d, *(u_int *)addr);
+			break;
+
 	/*
 	 * Get interface name.
 	 */
@@ -1007,9 +1105,15 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
 	/*
 	 * Set interface.
 	 */
-	case BIOCSETIF:
-		error = bpf_setif(d, (struct ifreq *)addr);
+	case BIOCSETIF: {
+		ifnet_t	ifp;
+		ifp = ifunit(((struct ifreq *)addr)->ifr_name);
+		if (ifp == NULL)
+			error = ENXIO;
+		else
+			error = bpf_setif(d, ifp, 0);
 		break;
+	}
 
 	/*
 	 * Set read timeout.
@@ -1147,25 +1251,22 @@ bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags, struct proc *p
  * free it and replace it.  Returns EINVAL for bogus requests.
  */
 static int
-bpf_setf(struct bpf_d *d, struct user_bpf_program *fp)
+bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns)
 {
 	struct bpf_insn *fcode, *old;
 	u_int flen, size;
-	int s;
 
 	old = d->bd_filter;
-	if (fp->bf_insns == USER_ADDR_NULL) {
-		if (fp->bf_len != 0)
+	if (bf_insns == USER_ADDR_NULL) {
+		if (bf_len != 0)
 			return (EINVAL);
-		s = splimp();
-		d->bd_filter = 0;
+		d->bd_filter = NULL;
 		reset_d(d);
-		splx(s);
 		if (old != 0)
 			FREE((caddr_t)old, M_DEVBUF);
 		return (0);
 	}
-	flen = fp->bf_len;
+	flen = bf_len;
 	if (flen > BPF_MAXINSNS)
 		return (EINVAL);
 
@@ -1175,12 +1276,10 @@ bpf_setf(struct bpf_d *d, struct user_bpf_program *fp)
 	if (fcode == NULL)
 		return (ENOBUFS);
 #endif
-	if (copyin(fp->bf_insns, (caddr_t)fcode, size) == 0 &&
+	if (copyin(bf_insns, (caddr_t)fcode, size) == 0 &&
 	    bpf_validate(fcode, (int)flen)) {
-		s = splimp();
 		d->bd_filter = fcode;
 		reset_d(d);
-		splx(s);
 		if (old != 0)
 			FREE((caddr_t)old, M_DEVBUF);
 
@@ -1196,23 +1295,18 @@ bpf_setf(struct bpf_d *d, struct user_bpf_program *fp)
  * Return an errno or 0.
  */
 static int
-bpf_setif(struct bpf_d *d, struct ifreq *ifr)
+bpf_setif(struct bpf_d *d, ifnet_t theywant, u_int32_t dlt)
 {
 	struct bpf_if *bp;
-	int s, error;
-	struct ifnet *theywant;
-
-	theywant = ifunit(ifr->ifr_name);
-	if (theywant == 0)
-		return ENXIO;
-
+	int error;
+	
 	/*
 	 * Look through attached interfaces for the named one.
 	 */
 	for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
 		struct ifnet *ifp = bp->bif_ifp;
 
-		if (ifp == 0 || ifp != theywant)
+		if (ifp == 0 || ifp != theywant || (dlt != 0 && dlt != bp->bif_dlt))
 			continue;
 		/*
 		 * We found the requested interface.
@@ -1229,7 +1323,6 @@ bpf_setif(struct bpf_d *d, struct ifreq *ifr)
 			if (error != 0)
 				return (error);
 		}
-		s = splimp();
 		if (bp != d->bd_bif) {
 			if (d->bd_bif)
 				/*
@@ -1237,16 +1330,102 @@ bpf_setif(struct bpf_d *d, struct ifreq *ifr)
 				 */
 				bpf_detachd(d);
 
-			bpf_attachd(d, bp);
+			if (bpf_attachd(d, bp) != 0) {
+				return ENXIO;
+			}
 		}
 		reset_d(d);
-		splx(s);
 		return (0);
 	}
 	/* Not found. */
 	return (ENXIO);
 }
 
+
+
+/*
+ * Get a list of available data link type of the interface.
+ */
+static int
+bpf_getdltlist(
+        struct bpf_d *d,
+        struct bpf_dltlist *bfl)
+{
+	u_int			n;
+	int				error;
+	struct ifnet	*ifp;
+	struct bpf_if	*bp;
+	user_addr_t		dlist;
+	
+	if (IS_64BIT_PROCESS(current_proc())) {
+		dlist = CAST_USER_ADDR_T(bfl->bfl_u.bflu_pad);
+	}
+	else {
+		dlist = CAST_USER_ADDR_T(bfl->bfl_u.bflu_list);
+	}
+	
+	ifp = d->bd_bif->bif_ifp;
+	n = 0;
+	error = 0;
+	for (bp = bpf_iflist; bp; bp = bp->bif_next) {
+		if (bp->bif_ifp != ifp)
+			continue;
+		if (dlist != 0) {
+			if (n >= bfl->bfl_len) {
+				return (ENOMEM);
+			}
+			error = copyout(&bp->bif_dlt, dlist, sizeof(bp->bif_dlt));
+			dlist += sizeof(bp->bif_dlt);
+		}
+		n++;
+	}
+	bfl->bfl_len = n;
+	return (error);
+}
+
+/*
+ * Set the data link type of a BPF instance.
+ */
+static int
+bpf_setdlt(struct bpf_d *d, uint32_t dlt)
+	
+
+{
+	int error, opromisc;
+	struct ifnet *ifp;
+	struct bpf_if *bp;
+	
+	if (d->bd_bif->bif_dlt == dlt)
+		return (0);
+	ifp = d->bd_bif->bif_ifp;
+	for (bp = bpf_iflist; bp; bp = bp->bif_next) {
+		if (bp->bif_ifp == ifp && bp->bif_dlt == dlt)
+			break;
+	}
+	if (bp != NULL) {
+		opromisc = d->bd_promisc;
+		bpf_detachd(d);
+		error = bpf_attachd(d, bp);
+		if (error) {
+			printf("bpf_setdlt: bpf_attachd %s%d failed (%d)\n",
+				ifnet_name(bp->bif_ifp), ifnet_unit(bp->bif_ifp), error);
+			return error;
+		}
+		reset_d(d);
+		if (opromisc) {
+			lck_mtx_unlock(bpf_mlock);
+			error = ifnet_set_promiscuous(bp->bif_ifp, 1);
+			lck_mtx_lock(bpf_mlock);
+			if (error)
+				printf("bpf_setdlt: ifpromisc %s%d failed (%d)\n",
+					   ifnet_name(bp->bif_ifp), ifnet_unit(bp->bif_ifp), error);
+			else
+				d->bd_promisc = 1;
+		}
+	}
+	return (bp == NULL ? EINVAL : 0);
+}
+
 /*
  * Support for select() and poll() system calls
  *
@@ -1256,15 +1435,16 @@ bpf_setif(struct bpf_d *d, struct ifreq *ifr)
 int
 bpfpoll(dev_t dev, int events, void * wql, struct proc *p)
 {
-	register struct bpf_d *d;
-	register int s;
+	struct bpf_d *d;
 	int revents = 0;
 
+	lck_mtx_lock(bpf_mlock);
+
 	d = bpf_dtab[minor(dev)];
-	if (d == 0 || d == (void *)1)
+	if (d == 0 || d == (void *)1) {
+		lck_mtx_unlock(bpf_mlock);
 		return (ENXIO);
-
-	lck_mtx_lock(bpf_mlock);
+	}
 
 	/*
 	 * An imitation of the FIONREAD ioctl code.
@@ -1274,52 +1454,26 @@ bpfpoll(dev_t dev, int events, void * wql, struct proc *p)
 		return (ENXIO);
 	}
 
-	s = splimp();
 	if (events & (POLLIN | POLLRDNORM)) {
 		if (d->bd_hlen != 0 || (d->bd_immediate && d->bd_slen != 0))
 			revents |= events & (POLLIN | POLLRDNORM);
 		else
 			selrecord(p, &d->bd_sel, wql);
 	}
-	splx(s);
 
 	lck_mtx_unlock(bpf_mlock);
 	return (revents);
 }
 
-/*
- * Incoming linkage from device drivers.  Process the packet pkt, of length
- * pktlen, which is stored in a contiguous buffer.  The packet is parsed
- * by each process' filter, and if accepted, stashed into the corresponding
- * buffer.
- */
-void
-bpf_tap(struct ifnet *ifp, u_char *pkt, u_int pktlen)
-{
-	struct bpf_if *bp;
-	register struct bpf_d *d;
-	register u_int slen;
-	/*
-	 * Note that the ipl does not have to be raised at this point.
-	 * The only problem that could arise here is that if two different
-	 * interfaces shared any data.  This is not the case.
-	 */
-	lck_mtx_lock(bpf_mlock);
+static inline void*
+_cast_non_const(const void * ptr) {
+	union {
+		const void*		cval;
+		void*			val;
+	} ret;
 	
-	bp = ifp->if_bpf;
-#ifdef __APPLE__
-	if (bp) {
-#endif
-		for (d = bp->bif_dlist; d != 0; d = d->bd_next) {
-			++d->bd_rcount;
-			slen = bpf_filter(d->bd_filter, pkt, pktlen, pktlen);
-			if (slen != 0)
-				catchpacket(d, pkt, pktlen, slen, bcopy);
-		}
-#ifdef __APPLE__
-    }
-	lck_mtx_unlock(bpf_mlock);
-#endif
+	ret.cval = ptr;
+	return (ret.val);
 }
 
 /*
@@ -1329,55 +1483,135 @@ bpf_tap(struct ifnet *ifp, u_char *pkt, u_int pktlen)
 static void
 bpf_mcopy(const void *src_arg, void *dst_arg, size_t len)
 {
-	const struct mbuf *m;
+	struct mbuf *m = _cast_non_const(src_arg);
 	u_int count;
 	u_char *dst;
 
-	m = src_arg;
 	dst = dst_arg;
 	while (len > 0) {
 		if (m == 0)
 			panic("bpf_mcopy");
 		count = min(m->m_len, len);
-		bcopy(mtod(m, const void *), dst, count);
+		bcopy(mbuf_data(m), dst, count);
 		m = m->m_next;
 		dst += count;
 		len -= count;
 	}
 }
 
-/*
- * Incoming linkage from device drivers, when packet is in an mbuf chain.
- */
-void
-bpf_mtap(struct ifnet *ifp, struct mbuf *m)
+static inline void
+bpf_tap_imp(
+	ifnet_t		ifp,
+	u_int32_t	dlt,
+	mbuf_t		m,
+	void*		hdr,
+	size_t		hlen,
+	int			outbound)
 {
 	struct bpf_if *bp;
-	struct bpf_d *d;
-	u_int pktlen, slen;
-	struct mbuf *m0;
 
+	/*
+	 * It's possible that we get here after the bpf descriptor has been
+	 * detached from the interface; in such a case we simply return.
+	 * Lock ordering is important since we can be called asynchronously
+	 * (from the IOKit) to process an inbound packet; when that happens
+	 * we would have been holding its "gateLock" and will be acquiring
+	 * "bpf_mlock" upon entering this routine.  Due to that, we release
+	 * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will
+	 * acquire "gateLock" in the IOKit), in order to avoid a deadlock
+	 * when a ifnet_set_promiscuous request simultaneously collides with
+	 * an inbound packet being passed into the tap callback.
+	 */
 	lck_mtx_lock(bpf_mlock);
-	
+	if (ifp->if_bpf == NULL) {
+		lck_mtx_unlock(bpf_mlock);
+		return;
+	}
 	bp = ifp->if_bpf;
-	if (bp) {
-	pktlen = 0;
-	for (m0 = m; m0 != 0; m0 = m0->m_next)
-		pktlen += m0->m_len;
-	
-		for (d = bp->bif_dlist; d != 0; d = d->bd_next) {
-			if (!d->bd_seesent && (m->m_pkthdr.rcvif == NULL))
+	for (bp = ifp->if_bpf; bp && bp->bif_ifp == ifp &&
+		 (dlt != 0 && bp->bif_dlt != dlt); bp = bp->bif_next)
+		;
+	if (bp && bp->bif_ifp == ifp && bp->bif_dlist != NULL) {
+		struct bpf_d	*d;
+		struct m_hdr	hack_hdr;
+		u_int	pktlen = 0;
+		u_int	slen = 0;
+		struct mbuf *m0;
+		
+		if (hdr) {
+			/*
+			 * This is gross. We mock up an mbuf that points to the
+			 * header buffer. This means we don't have to copy the
+			 * header. A number of interfaces prepended headers just
+			 * for bpf by allocating an mbuf on the stack. We want to
+			 * give developers an easy way to prepend a header for bpf.
+			 * Since a developer allocating an mbuf on the stack is bad,
+			 * we do even worse here, allocating only a header to point
+			 * to a buffer the developer supplied. This makes assumptions
+			 * that bpf_filter and catchpacket will not look at anything
+			 * in the mbuf other than the header. This was true at the
+			 * time this code was written.
+			 */
+			hack_hdr.mh_next = m;
+			hack_hdr.mh_nextpkt = NULL;
+			hack_hdr.mh_len = hlen;
+			hack_hdr.mh_data = hdr;
+			hack_hdr.mh_type = m->m_type;
+			hack_hdr.mh_flags = 0;
+			
+			m = (mbuf_t)&hack_hdr;
+		}
+
+		for (m0 = m; m0 != 0; m0 = m0->m_next)
+			pktlen += m0->m_len;
+		
+		for (d = bp->bif_dlist; d; d = d->bd_next) {
+			if (outbound && !d->bd_seesent)
 				continue;
 			++d->bd_rcount;
 			slen = bpf_filter(d->bd_filter, (u_char *)m, pktlen, 0);
-			if (slen != 0)
+			if (slen != 0) {
+#if CONFIG_MACF_NET
+				if (mac_bpfdesc_check_receive(d, bp->bif_ifp) != 0)
+					continue;
+#endif
 				catchpacket(d, (u_char *)m, pktlen, slen, bpf_mcopy);
+			}
 		}
 	}
-	
 	lck_mtx_unlock(bpf_mlock);
 }
 
+void
+bpf_tap_out(
+	ifnet_t		ifp,
+	u_int32_t	dlt,
+	mbuf_t		m,
+	void*		hdr,
+	size_t		hlen)
+{
+	bpf_tap_imp(ifp, dlt, m, hdr, hlen, 1);
+}
+
+void
+bpf_tap_in(
+	ifnet_t		ifp,
+	u_int32_t	dlt,
+	mbuf_t		m,
+	void*		hdr,
+	size_t		hlen)
+{
+	bpf_tap_imp(ifp, dlt, m, hdr, hlen, 0);
+}
+
+/* Callback registered with Ethernet driver. */
+static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m)
+{
+	bpf_tap_imp(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL);
+	
+	return 0;
+}
+
 /*
  * Move the packet data from interface memory (pkt) into the
  * store buffer.  Return 1 if it's time to wakeup a listener (buffer full),
@@ -1390,9 +1624,9 @@ static void
 catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, 
 	void (*cpfn)(const void *, void *, size_t))
 {
-	register struct bpf_hdr *hp;
-	register int totlen, curlen;
-	register int hdrlen = d->bd_bif->bif_hdrlen;
+	struct bpf_hdr *hp;
+	int totlen, curlen;
+	int hdrlen = d->bd_bif->bif_hdrlen;
 	/*
 	 * Figure out how many bytes to move.  If the packet is
 	 * greater or equal to the snapshot length, transfer that
@@ -1436,13 +1670,7 @@ catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen,
 	 * Append the bpf header.
 	 */
 	hp = (struct bpf_hdr *)(d->bd_sbuf + curlen);
-#if BSD >= 199103
 	microtime(&hp->bh_tstamp);
-#elif defined(sun)
-	uniqtime(&hp->bh_tstamp);
-#else
-	hp->bh_tstamp = time;
-#endif
 	hp->bh_datalen = pktlen;
 	hp->bh_hdrlen = hdrlen;
 	/*
@@ -1503,32 +1731,72 @@ bpf_freed(struct bpf_d *d)
 void
 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
 {
-	struct bpf_if *bp;
-	bp = (struct bpf_if *) _MALLOC(sizeof(*bp), M_DEVBUF, M_WAIT);
-	if (bp == 0)
+	bpf_attach(ifp, dlt, hdrlen, NULL, NULL);
+}
+
+errno_t
+bpf_attach(
+	ifnet_t			ifp,
+	u_int32_t		dlt,
+	u_int32_t		hdrlen,
+	bpf_send_func	send,
+	bpf_tap_func	tap)
+{
+	struct bpf_if *bp_new;
+	struct bpf_if *bp_temp;
+	struct bpf_if *bp_first = NULL;
+	
+	bp_new = (struct bpf_if *) _MALLOC(sizeof(*bp_new), M_DEVBUF, M_WAIT);
+	if (bp_new == 0)
 		panic("bpfattach");
 
 	lck_mtx_lock(bpf_mlock);
 
-	bp->bif_dlist = 0;
-	bp->bif_ifp = ifp;
-	bp->bif_dlt = dlt;
-
-	bp->bif_next = bpf_iflist;
-	bpf_iflist = bp;
-
-	bp->bif_ifp->if_bpf = 0;
-
+	/*
+	 * Check if this interface/dlt is already attached, record first
+	 * attachment for this interface.
+	 */
+	for (bp_temp = bpf_iflist; bp_temp && (bp_temp->bif_ifp != ifp ||
+		 bp_temp->bif_dlt != dlt); bp_temp = bp_temp->bif_next) {
+		 if (bp_temp->bif_ifp == ifp && bp_first == NULL)
+		 	bp_first = bp_temp;
+	}
+	
+	if (bp_temp != NULL) {
+		printf("bpfattach - %s%d with dlt %d is already attached\n",
+			ifp->if_name, ifp->if_unit, dlt);
+		FREE(bp_new, M_DEVBUF);
+		lck_mtx_unlock(bpf_mlock);
+		return EEXIST;
+	}
+	
+	bzero(bp_new, sizeof(*bp_new));
+	bp_new->bif_ifp = ifp;
+	bp_new->bif_dlt = dlt;
+	bp_new->bif_send = send;
+	bp_new->bif_tap = tap;
+	
+	if (bp_first == NULL) {
+		/* No other entries for this ifp */
+		bp_new->bif_next = bpf_iflist;
+		bpf_iflist = bp_new;
+	}
+	else {
+		/* Add this after the first entry for this interface */
+		bp_new->bif_next = bp_first->bif_next;
+		bp_first->bif_next = bp_new;
+	}
+	
 	/*
 	 * Compute the length of the bpf header.  This is not necessarily
 	 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
 	 * that the network layer header begins on a longword boundary (for
 	 * performance reasons and to alleviate alignment restrictions).
 	 */
-	bp->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
+	bp_new->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
 	
 	/* Take a reference on the interface */
-	ifp_reference(ifp);
+	ifnet_reference(ifp);
 
 	lck_mtx_unlock(bpf_mlock);
 
@@ -1536,6 +1804,8 @@ bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
 	if (bootverbose)
 		printf("bpf: %s%d attached\n", ifp->if_name, ifp->if_unit);
 #endif
+
+	return 0;
 }
 
 /*
@@ -1547,57 +1817,43 @@ bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
 void
 bpfdetach(struct ifnet *ifp)
 {
-	struct bpf_if	*bp, *bp_prev;
+	struct bpf_if	*bp, *bp_prev, *bp_next;
+	struct bpf_if	*bp_free = NULL;
 	struct bpf_d	*d;
-	int	s;
 
-	s = splimp();
 	
 	lck_mtx_lock(bpf_mlock);
 
 	/* Locate BPF interface information */
 	bp_prev = NULL;
-	for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
-		if (ifp == bp->bif_ifp)
-			break;
-		bp_prev = bp;
-	}
-
-#ifdef __APPLE__
-	/* Check for no BPF interface information */
-	if (bp == NULL) {
-		return;
-	}
-#endif
-
-	/* Interface wasn't attached */
-	if (bp->bif_ifp == NULL) {
-		splx(s);
-#ifndef __APPLE__
-		printf("bpfdetach: %s%d was not attached\n", ifp->if_name,
-		    ifp->if_unit);
-#endif
-		return;
-	}
-
-	while ((d = bp->bif_dlist) != NULL) {
-		bpf_detachd(d);
-		bpf_wakeup(d);
+	for (bp = bpf_iflist; bp != NULL; bp = bp_next) {
+		bp_next = bp->bif_next;
+		if (ifp != bp->bif_ifp) {
+			bp_prev = bp;
+			continue;
+		}
+		
+		while ((d = bp->bif_dlist) != NULL) {
+			bpf_detachd(d);
+			bpf_wakeup(d);
+		}
+	
+		if (bp_prev) {
+			bp_prev->bif_next = bp->bif_next;
+		} else {
+			bpf_iflist = bp->bif_next;
+		}
+		
+		bp->bif_next = bp_free;
+		bp_free = bp;
+		
+		ifnet_release(ifp);
 	}
 
-	if (bp_prev) {
-		bp_prev->bif_next = bp->bif_next;
-	} else {
-		bpf_iflist = bp->bif_next;
-	}
-	
-	ifp_release(ifp);
-	
 	lck_mtx_unlock(bpf_mlock);
 
 	FREE(bp, M_DEVBUF);
 
-	splx(s);
 }
 
 void
@@ -1635,10 +1891,10 @@ bpf_init(__unused void *unused)
 			if (bpf_mlock_grp_attr)
 				lck_grp_attr_free(bpf_mlock_grp_attr);
 			
-			bpf_mlock = 0;
-			bpf_mlock_attr = 0;
-			bpf_mlock_grp = 0;
-			bpf_mlock_grp_attr = 0;
+			bpf_mlock = NULL;
+			bpf_mlock_attr = NULL;
+			bpf_mlock_grp = NULL;
+			bpf_mlock_grp_attr = NULL;
 			bpf_devsw_installed = 0;
 			printf("bpf_init: failed to allocate a major number!\n");
 			return;
@@ -1656,51 +1912,18 @@ bpf_init(__unused void *unused)
 SYSINIT(bpfdev,SI_SUB_DRIVERS,SI_ORDER_MIDDLE+CDEV_MAJOR,bpf_drvinit,NULL)
 #endif
 
-#else /* !BPF */
-#ifndef __APPLE__
-/*
- * NOP stubs to allow bpf-using drivers to load and function.
- *
- * A 'better' implementation would allow the core bpf functionality
- * to be loaded at runtime.
- */
-
-void
-bpf_tap(ifp, pkt, pktlen)
-	struct ifnet *ifp;
-	register u_char *pkt;
-	register u_int pktlen;
+#if CONFIG_MACF_NET
+struct label *
+mac_bpfdesc_label_get(struct bpf_d *d)
 {
-}
 
-void
-bpf_mtap(ifp, m)
-	struct ifnet *ifp;
-	struct mbuf *m;
-{
+	return (d->bd_label);
 }
 
 void
-bpfattach(ifp, dlt, hdrlen)
-	struct ifnet *ifp;
-	u_int dlt, hdrlen;
+mac_bpfdesc_label_set(struct bpf_d *d, struct label *label)
 {
-}
 
-void
-bpfdetach(ifp)
-	struct ifnet *ifp;
-{
+	d->bd_label = label;
 }
-
-u_int
-bpf_filter(pc, p, wirelen, buflen)
-	register const struct bpf_insn *pc;
-	register u_char *p;
-	u_int wirelen;
-	register u_int buflen;
-{
-	return -1;	/* "no filter" behaviour */
-}
-#endif /* !defined(__APPLE__) */
-#endif /* NBPFILTER > 0 */
+#endif
diff --git a/bsd/net/bpf.h b/bsd/net/bpf.h
index e20ed1a91..321284b23 100644
--- a/bsd/net/bpf.h
+++ b/bsd/net/bpf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1991, 1993
@@ -61,6 +67,12 @@
  *
  * $FreeBSD: src/sys/net/bpf.h,v 1.21.2.3 2001/08/01 00:23:13 fenner Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _NET_BPF_H_
 #define _NET_BPF_H_
@@ -83,7 +95,7 @@ typedef	u_int32_t bpf_u_int32;
  * Alignment macros.  BPF_WORDALIGN rounds up to the next
  * even multiple of BPF_ALIGNMENT.
  */
-#define BPF_ALIGNMENT sizeof(long)
+#define BPF_ALIGNMENT sizeof(int32_t)
 #define BPF_WORDALIGN(x) (((x)+(BPF_ALIGNMENT-1))&~(BPF_ALIGNMENT-1))
 
 #define BPF_MAXINSNS 512
@@ -98,17 +110,17 @@ struct bpf_program {
 	struct bpf_insn *bf_insns;
 };
 
-#ifdef KERNEL
+#ifdef KERNEL_PRIVATE
 /* LP64 version of bpf_program.  all pointers 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with bpf_program
  */
-struct user_bpf_program {
+struct bpf_program64 {
 	u_int		bf_len;
 	user_addr_t	bf_insns __attribute__((aligned(8)));
 };
 
-#endif // KERNEL
+#endif // KERNEL_PRIVATE
 
 /*
  * Struct returned by BIOCGSTATS.
@@ -133,6 +145,13 @@ struct bpf_version {
 	u_short bv_major;
 	u_short bv_minor;
 };
+#if defined(__LP64__)
+#define __need_struct_timeval32
+#include <sys/_structs.h>
+#define BPF_TIMEVAL timeval32
+#else
+#define BPF_TIMEVAL timeval
+#endif
 /* Current version number of filter architecture. */
 #define BPF_MAJOR_VERSION 1
 #define BPF_MINOR_VERSION 1
@@ -140,13 +159,16 @@ struct bpf_version {
 #define	BIOCGBLEN	_IOR('B',102, u_int)
 #define	BIOCSBLEN	_IOWR('B',102, u_int)
 #define	BIOCSETF	_IOW('B',103, struct bpf_program)
+#ifdef KERNEL_PRIVATE
+#define	BIOCSETF64	_IOW('B',103, struct bpf_program64)
+#endif // KERNEL_PRIVATE
 #define	BIOCFLUSH	_IO('B',104)
 #define BIOCPROMISC	_IO('B',105)
 #define	BIOCGDLT	_IOR('B',106, u_int)
 #define BIOCGETIF	_IOR('B',107, struct ifreq)
 #define BIOCSETIF	_IOW('B',108, struct ifreq)
-#define BIOCSRTIMEOUT	_IOW('B',109, struct timeval)
-#define BIOCGRTIMEOUT	_IOR('B',110, struct timeval)
+#define BIOCSRTIMEOUT	_IOW('B',109, struct BPF_TIMEVAL)
+#define BIOCGRTIMEOUT	_IOR('B',110, struct BPF_TIMEVAL)
 #define BIOCGSTATS	_IOR('B',111, struct bpf_stat)
 #define BIOCIMMEDIATE	_IOW('B',112, u_int)
 #define BIOCVERSION	_IOR('B',113, struct bpf_version)
@@ -156,12 +178,14 @@ struct bpf_version {
 #define BIOCSHDRCMPLT	_IOW('B',117, u_int)
 #define BIOCGSEESENT	_IOR('B',118, u_int)
 #define BIOCSSEESENT	_IOW('B',119, u_int)
+#define BIOCSDLT        _IOW('B',120, u_int)
+#define BIOCGDLTLIST    _IOWR('B',121, struct bpf_dltlist)
 
 /*
  * Structure prepended to each packet.
  */
 struct bpf_hdr {
-	struct timeval	bh_tstamp;	/* time stamp */
+	struct BPF_TIMEVAL bh_tstamp;	/* time stamp */
 	bpf_u_int32	bh_caplen;	/* length of captured portion */
 	bpf_u_int32	bh_datalen;	/* original length of packet */
 	u_short		bh_hdrlen;	/* length of bpf header (this struct
@@ -349,28 +373,90 @@ struct bpf_insn {
 #define BPF_STMT(code, k) { (u_short)(code), 0, 0, k }
 #define BPF_JUMP(code, k, jt, jf) { (u_short)(code), jt, jf, k }
 
+#pragma pack(4)
+
+/*
+ * Structure to retrieve available DLTs for the interface.
+ */
+struct bpf_dltlist {
+	u_int32_t	bfl_len;        /* number of bfd_list array */
+	union {
+		u_int32_t   *bflu_list;      /* array of DLTs */
+		u_int64_t	bflu_pad;
+	} bfl_u;
+};
+#define bfl_list bfl_u.bflu_list
+
+#pragma pack()
+
 #ifdef KERNEL_PRIVATE
 /* Forward declerations */
 struct ifnet;
 struct mbuf;
 
-int	 bpf_validate(const struct bpf_insn *, int);
-void	 bpf_tap(struct ifnet *, u_char *, u_int);
-void	 bpf_mtap(struct ifnet *, struct mbuf *);
+int		bpf_validate(const struct bpf_insn *, int);
+void	bpfdetach(struct ifnet *);
+void	bpfilterattach(int);
+u_int	bpf_filter(const struct bpf_insn *, u_char *, u_int, u_int);
 
-void	 bpfdetach(struct ifnet *);
+#endif /* KERNEL_PRIVATE */
 
-void	 bpfilterattach(int);
-u_int	 bpf_filter(const struct bpf_insn *, u_char *, u_int, u_int);
+#ifdef KERNEL
+#ifndef BPF_TAP_MODE_T
+#define BPF_TAP_MODE_T
+/*!
+	@enum BPF tap mode
+	@abstract Constants defining interface families.
+	@constant BPF_MODE_DISABLED Disable bpf.
+	@constant BPF_MODE_INPUT Enable input only.
+	@constant BPF_MODE_OUTPUT Enable output only.
+	@constant BPF_MODE_INPUT_OUTPUT Enable input and output.
+*/
+
+enum {
+		BPF_MODE_DISABLED		= 0,
+		BPF_MODE_INPUT			= 1,
+		BPF_MODE_OUTPUT			= 2,
+		BPF_MODE_INPUT_OUTPUT	= 3
+};
+/*!
+	@typedef bpf_tap_mode
+	@abstract Mode for tapping. BPF_MODE_DISABLED/BPF_MODE_INPUT_OUTPUT etc.
+*/
+typedef u_int32_t bpf_tap_mode;
+#endif /* !BPF_TAP_MODE_T */
 
-#ifdef __APPLE__
-#define BPF_TAP(x, y, z) bpf_tap(x,y,z) 
-#define BPF_MTAP(x, y) bpf_mtap(x, y)
-#endif /* __APPLE__ */
+/*!
+	@typedef bpf_send_func
+	@discussion bpf_send_func is called when a bpf file descriptor is
+		used to send a raw packet on the interface. The mbuf and data
+		link type are specified. The callback is responsible for
+		releasing the mbuf whether or not it returns an error.
+	@param interface The interface the packet is being sent on.
+	@param dlt The data link type the bpf device is attached to.
+	@param packet The packet to be sent.
+ */
+typedef errno_t (*bpf_send_func)(ifnet_t interface, u_int32_t data_link_type,
+								 mbuf_t packet);
 
-#endif /* KERNEL_PRIVATE */
+/*!
+	@typedef bpf_tap_func
+	@discussion bpf_tap_func is called when the tap state of the
+		interface changes. This happens when a bpf device attaches to an
+		interface or detaches from an interface. The tap mode will join
+		together (bit or) the modes of all bpf devices using that
+		interface for that dlt. If you return an error from this
+		function, the bpf device attach attempt that triggered the tap
+		will fail. If this function was called bacuse the tap state was
+		decreasing (tap in or out is stopping), the error will be
+		ignored.
+	@param interface The interface being tapped.
+	@param dlt The data link type being tapped.
+	@param direction The direction of the tap.
+ */
+typedef errno_t (*bpf_tap_func)(ifnet_t interface, u_int32_t data_link_type,
+								bpf_tap_mode direction);
 
-#ifdef KERNEL
 /*!
 	@function bpfattach
 	@discussion Registers an interface with BPF. This allows bpf devices
@@ -383,6 +469,55 @@ u_int	 bpf_filter(const struct bpf_insn *, u_char *, u_int, u_int);
 	@param header_length The length, in bytes, of the data link header.
  */
 void	 bpfattach(ifnet_t interface, u_int data_link_type, u_int header_length);
+
+/*!
+	@function bpf_attach
+	@discussion Registers an interface with BPF. This allows bpf devices
+		to attach to your interface to capture and transmit packets.
+		Your interface will be unregistered automatically when your
+		interface is detached. You may register multiple times with
+		different data link types. An 802.11 interface would use this to
+		allow clients to pick whether they want just an ethernet style
+		frame or the 802.11 wireless headers as well. The first dlt you
+		register will be considered the default. Any bpf device attaches
+		that do not specify a data link type will use the default.
+	@param interface The interface to register with BPF.
+	@param data_link_type The data link type of the interface. See the
+		DLT_* defines in bpf.h.
+	@param header_length The length, in bytes, of the data link header.
+ */
+errno_t	 bpf_attach(ifnet_t interface, u_int32_t data_link_type,
+					u_int32_t header_length, bpf_send_func send,
+					bpf_tap_func tap);
+
+/*!
+	@function bpf_tap_in
+	@discussion Call this function when your interface receives a
+		packet. This function will check if any bpf devices need a
+		a copy of the packet.
+	@param interface The interface the packet was received on.
+	@param dlt The data link type of the packet.
+	@param packet The packet received.
+	@param header An optional pointer to a header that will be prepended.
+	@param headerlen If the header was specified, the length of the header.
+ */
+void	bpf_tap_in(ifnet_t interface, u_int32_t dlt, mbuf_t packet,
+				   void* header, size_t header_len);
+
+/*!
+	@function bpf_tap_out
+	@discussion Call this function when your interface trasmits a
+		packet. This function will check if any bpf devices need a
+		a copy of the packet.
+	@param interface The interface the packet was or will be transmitted on.
+	@param dlt The data link type of the packet.
+	@param packet The packet received.
+	@param header An optional pointer to a header that will be prepended.
+	@param headerlen If the header was specified, the length of the header.
+ */
+void	bpf_tap_out(ifnet_t interface, u_int32_t dlt, mbuf_t packet,
+					void* header, size_t header_len);
+
 #endif /* KERNEL */
 
 /*
diff --git a/bsd/net/bpf_compat.h b/bsd/net/bpf_compat.h
index ddec919d9..560571434 100644
--- a/bsd/net/bpf_compat.h
+++ b/bsd/net/bpf_compat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1992, 1993
diff --git a/bsd/net/bpf_filter.c b/bsd/net/bpf_filter.c
index d697c1363..31ce77023 100644
--- a/bsd/net/bpf_filter.c
+++ b/bsd/net/bpf_filter.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1991, 1993
@@ -349,7 +355,7 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
 					return 0;
 				m = (struct mbuf *)p;
 				MINDEX(m, k);
-				A = mtod(m, char *)[k];
+				A = mtod(m, u_char *)[k];
 				continue;
 #else
 				return 0;
@@ -368,7 +374,7 @@ bpf_filter(const struct bpf_insn *pc, u_char *p, u_int wirelen, u_int buflen)
 					return 0;
 				m = (struct mbuf *)p;
 				MINDEX(m, k);
-				X = (mtod(m, char *)[k] & 0xf) << 2;
+				X = (mtod(m, u_char *)[k] & 0xf) << 2;
 				continue;
 #else
 				return 0;
diff --git a/bsd/net/bpfdesc.h b/bsd/net/bpfdesc.h
index 1958e635b..9f514f89c 100644
--- a/bsd/net/bpfdesc.h
+++ b/bsd/net/bpfdesc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1991, 1993
@@ -120,7 +126,9 @@ struct bpf_d {
 #endif
 	int		bd_hdrcmplt;	/* false to fill in src lladdr automatically */
 	int		bd_seesent;	/* true if bpf should see sent packets */
-
+#if CONFIG_MACF_NET
+	struct label *	bd_label;	/* MAC label for descriptor */
+#endif
 };
 
 /*
@@ -132,6 +140,8 @@ struct bpf_if {
 	u_int bif_dlt;			/* link layer type */
 	u_int bif_hdrlen;		/* length of header (with padding) */
 	struct ifnet *bif_ifp;		/* corresponding interface */
+	bpf_send_func	bif_send;
+	bpf_tap_func	bif_tap;
 };
 
 #endif /* KERNEL_PRIVATE */
diff --git a/bsd/net/bridge.c b/bsd/net/bridge.c
index 2770e4a76..74d6a18d2 100644
--- a/bsd/net/bridge.c
+++ b/bsd/net/bridge.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Luigi Rizzo
@@ -304,7 +310,7 @@ sysctl_bdg_cfg(SYSCTL_HANDLER_ARGS)
     int error = 0 ;
     char oldval[256] ;
 
-    strcpy(oldval, bridge_cfg) ;
+    strlcpy(oldval, bridge_cfg, sizeof (oldval));
 
     error = sysctl_handle_string(oidp,
 	    bridge_cfg, oidp->oid_arg2, req);
diff --git a/bsd/net/bridge.h b/bsd/net/bridge.h
index 3f49914b4..967a87df2 100644
--- a/bsd/net/bridge.h
+++ b/bsd/net/bridge.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Luigi Rizzo
diff --git a/bsd/net/bsd_comp.c b/bsd/net/bsd_comp.c
index 1bfc725ea..14fb57111 100644
--- a/bsd/net/bsd_comp.c
+++ b/bsd/net/bsd_comp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Because this code is derived from the 4.3BSD compress source:
  *
@@ -57,6 +63,12 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * This version is for use with mbufs on BSD-derived systems.
@@ -72,6 +84,10 @@
 #define PACKETPTR	struct mbuf *
 #include <net/ppp_comp.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
+
 #if DO_BSD_COMPRESS
 /*
  * PPP "BSD compress" compression
@@ -899,6 +915,9 @@ bsd_decompress(state, cmp, dmpp)
     dmp->m_data += db->hdrlen;
     wptr = mtod(dmp, u_char *);
     space = M_TRAILINGSPACE(dmp) - PPP_HDRLEN + 1;
+#if CONFIG_MACF_NET
+    mac_mbuf_label_copy(cmp, dmp);
+#endif
 
     /*
      * Fill in the ppp header, but not the last byte of the protocol
diff --git a/bsd/net/devtimer.c b/bsd/net/devtimer.c
index 4344f3a9f..a7cd37bd0 100644
--- a/bsd/net/devtimer.c
+++ b/bsd/net/devtimer.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -96,7 +102,7 @@ devtimer_valid(devtimer_ref timer)
 __private_extern__ void
 devtimer_retain(devtimer_ref timer)
 {
-    OSIncrementAtomic(&timer->dt_retain_count);
+    OSIncrementAtomic((SInt32 *)&timer->dt_retain_count);
     return;
 }
 
@@ -117,7 +123,7 @@ devtimer_release(devtimer_ref timer)
 {
     UInt32	old_retain_count;
 
-    old_retain_count = OSDecrementAtomic(&timer->dt_retain_count);
+    old_retain_count = OSDecrementAtomic((SInt32 *)&timer->dt_retain_count);
     switch (old_retain_count) {
     case 0:
 	panic("devtimer_release: retain count is 0\n");
diff --git a/bsd/net/devtimer.h b/bsd/net/devtimer.h
index 9e8aeca91..f9c48a30a 100644
--- a/bsd/net/devtimer.h
+++ b/bsd/net/devtimer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/net/dlil.c b/bsd/net/dlil.c
index 6e00eab21..47690269a 100644
--- a/bsd/net/dlil.c
+++ b/bsd/net/dlil.c
@@ -1,30 +1,40 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- *	Copyright (c) 1999 Apple Computer, Inc. 
- *
  *	Data Link Inteface Layer
  *	Author: Ted Walker
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -34,6 +44,7 @@
 #include <sys/socket.h>
 #include <sys/domain.h>
 #include <sys/user.h>
+#include <sys/random.h>
 #include <net/if_dl.h>
 #include <net/if.h>
 #include <net/route.h>
@@ -48,6 +59,7 @@
 #include <kern/thread.h>
 #include <kern/sched_prim.h>
 #include <kern/locks.h>
+#include <net/kpi_protocol.h>
 
 #include <net/if_types.h>
 #include <net/kpi_interfacefilter.h>
@@ -56,31 +68,33 @@
 
 #include <machine/machine_routines.h>
 
-#define DBG_LAYER_BEG		DLILDBG_CODE(DBG_DLIL_STATIC, 0)
-#define DBG_LAYER_END		DLILDBG_CODE(DBG_DLIL_STATIC, 2)
+#include <mach/thread_act.h>
+
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
+
+#define DBG_LAYER_BEG			DLILDBG_CODE(DBG_DLIL_STATIC, 0)
+#define DBG_LAYER_END			DLILDBG_CODE(DBG_DLIL_STATIC, 2)
 #define DBG_FNC_DLIL_INPUT      DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
 #define DBG_FNC_DLIL_OUTPUT     DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
 #define DBG_FNC_DLIL_IFOUT      DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
 
 
-#define MAX_DL_TAGS 		16
-#define MAX_DLIL_FILTERS 	16
 #define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
 #define MAX_LINKADDR	    4 /* LONGWORDS */
 #define M_NKE M_IFADDR
 
-#define PFILT(x) ((struct dlil_filterq_entry *) (x))->variants.pr_filter
-#define IFILT(x) ((struct dlil_filterq_entry *) (x))->variants.if_filter
-
-#if 0
+#if 1
 #define DLIL_PRINTF	printf
 #else
 #define DLIL_PRINTF	kprintf
 #endif
 
+
 enum {
-	kProtoKPI_DLIL	= 0,
-	kProtoKPI_v1	= 1
+	kProtoKPI_v1	= 1,
+	kProtoKPI_v2	= 2
 };
 
 struct if_proto {
@@ -92,14 +106,6 @@ struct if_proto {
     protocol_family_t		protocol_family;
     int						proto_kpi;
     union {
-    	struct {
-			dl_input_func			 dl_input;
-			dl_pre_output_func		 dl_pre_output;
-			dl_event_func			 dl_event;
-			dl_offer_func			 dl_offer;
-			dl_ioctl_func			 dl_ioctl;
-			dl_detached_func		 dl_detached;
-		} dlil;
 		struct {
 			proto_media_input			input;
 			proto_media_preout			pre_output;
@@ -109,6 +115,15 @@ struct if_proto {
 			proto_media_resolve_multi	resolve_multi;
 			proto_media_send_arp		send_arp;
 		} v1;
+		struct {
+			proto_media_input_v2		input;
+			proto_media_preout			pre_output;
+			proto_media_event			event;
+			proto_media_ioctl			ioctl;
+			proto_media_detached		detached;
+			proto_media_resolve_multi	resolve_multi;
+			proto_media_send_arp		send_arp;
+		} v2;
 	} kpi;
 };
 
@@ -143,46 +158,13 @@ struct ifnet_filter {
     iff_detached_func			filt_detached;
 };
 
-struct if_family_str {
-    TAILQ_ENTRY(if_family_str) if_fam_next;
-    u_long	if_family;
-    int		refcnt;
-    int		flags;
-
-#define DLIL_SHUTDOWN 1
-
-    int (*add_if)(struct ifnet *ifp);
-    int (*del_if)(struct ifnet *ifp);
-    int (*init_if)(struct ifnet *ifp);
-    int (*add_proto)(struct ifnet *ifp, u_long protocol_family, struct ddesc_head_str *demux_desc_head);
-	ifnet_del_proto_func	del_proto;
-    ifnet_ioctl_func		ifmod_ioctl;
-    int (*shutdown)(void);
-};
-
-struct proto_family_str {
-	TAILQ_ENTRY(proto_family_str) proto_fam_next;
-	u_long	proto_family;
-	u_long	if_family;
-	int		usecnt;
-
-	int (*attach_proto)(struct ifnet *ifp, u_long protocol_family);
-	int (*detach_proto)(struct ifnet *ifp, u_long protocol_family);
-};
-
-enum {
-	kIfNetUseCount_MayBeZero = 0,
-	kIfNetUseCount_MustNotBeZero = 1
-};
+struct proto_input_entry;
 
 static TAILQ_HEAD(, dlil_ifnet) dlil_ifnet_head;
-static TAILQ_HEAD(, if_family_str) if_family_head;
-static TAILQ_HEAD(, proto_family_str) proto_family_head;
 static lck_grp_t *dlil_lock_group;
 static lck_grp_t *ifnet_lock_group;
 static lck_grp_t *ifnet_head_lock_group;
 static lck_attr_t *ifnet_lock_attr;
-static lck_mtx_t *proto_family_mutex;
 static lck_rw_t *ifnet_head_mutex;
 static lck_mtx_t *dlil_ifnet_mutex;
 static lck_mtx_t *dlil_mutex;
@@ -190,39 +172,40 @@ static unsigned long dlil_read_count = 0;
 static unsigned long dlil_detach_waiting = 0;
 extern u_int32_t	ipv4_ll_arp_aware;
 
-int dlil_initialized = 0;
-lck_spin_t *dlil_input_lock;
-__private_extern__ thread_t	dlil_input_thread_ptr = 0;
-int dlil_input_thread_wakeup = 0;
-__private_extern__ int dlil_output_thread_wakeup = 0;
-static struct mbuf *dlil_input_mbuf_head = NULL;
-static struct mbuf *dlil_input_mbuf_tail = NULL;
-#if NLOOP > 1
-#error dlil_input() needs to be revised to support more than on loopback interface
+static struct dlil_threading_info dlil_lo_thread;
+__private_extern__  struct dlil_threading_info *dlil_lo_thread_ptr = &dlil_lo_thread;
+
+static struct mbuf *dlil_lo_input_mbuf_head = NULL;
+static struct mbuf *dlil_lo_input_mbuf_tail = NULL;
+
+#if IFNET_INPUT_SANITY_CHK
+static int dlil_lo_input_mbuf_count = 0;
+int dlil_input_sanity_check = 0;	/* sanity checking of input packet lists received */
 #endif
-static struct mbuf *dlil_input_loop_head = NULL;
-static struct mbuf *dlil_input_loop_tail = NULL;
+int dlil_multithreaded_input = 1;
+static int cur_dlil_input_threads = 0; 
 
-static void dlil_input_thread(void);
 static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg);
-struct ifnet *ifbyfamily(u_long family, short unit);
 static int dlil_detach_filter_internal(interface_filter_t filter, int detached);
 static void dlil_call_delayed_detach_thread(void);
 
 static void	dlil_read_begin(void);
-static void	dlil_read_end(void);
+static __inline__ void	dlil_read_end(void);
 static int	dlil_write_begin(void);
 static void	dlil_write_end(void);
 
-static int ifp_use(struct ifnet *ifp, int handle_zero);
-static int ifp_unuse(struct ifnet *ifp);
-static void ifp_use_reached_zero(struct ifnet *ifp);
+unsigned int net_affinity = 1;
+static kern_return_t dlil_affinity_set(struct thread *, u_int32_t);
 
 extern void bpfdetach(struct ifnet*);
 extern void proto_input_run(void); // new run_netisr
 
-
-int dlil_input_packet(struct ifnet  *ifp, struct mbuf *m, char *frame_header);
+void dlil_input_packet_list(struct ifnet  *ifp, struct mbuf *m);
+static void dlil_input_thread_func(struct dlil_threading_info *inpthread); 
+__private_extern__ int dlil_create_input_thread(
+		ifnet_t, struct dlil_threading_info *);
+__private_extern__ void dlil_terminate_input_thread(
+		struct dlil_threading_info *);
 
 __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
 
@@ -231,8 +214,11 @@ int dlil_expand_mcl;
 extern u_int32_t	inject_buckets;
 
 static const u_int32_t dlil_writer_waiting = 0x80000000;
+static	lck_grp_attr_t	*dlil_grp_attributes = NULL;
+static	lck_attr_t	*dlil_lck_attributes = NULL;
+static	lck_grp_t	*dlil_input_lock_grp = NULL;
 
-static __inline__ void*
+static inline void*
 _cast_non_const(const void * ptr) {
 	union {
 		const void*		cval;
@@ -275,7 +261,7 @@ dlil_read_end(void)
 {
 	struct uthread *uth = get_bsdthread_info(current_thread());
 	
-	OSDecrementAtomic((UInt32*)&dlil_read_count);
+	OSDecrementAtomic((SInt32*)&dlil_read_count);
 	uth->dlil_incremented_read--;
 	if (dlil_read_count == dlil_writer_waiting)
 		wakeup(_cast_non_const(&dlil_writer_waiting));
@@ -338,34 +324,6 @@ proto_hash_value(u_long protocol_family)
 	}
 }
 
-static 
-struct if_family_str *find_family_module(u_long if_family)
-{
-    struct if_family_str  *mod = NULL;
-
-    TAILQ_FOREACH(mod, &if_family_head, if_fam_next) {
-	if (mod->if_family == (if_family & 0xffff)) 
-	    break;
-    }
-
-    return mod;
-}
-
-static 
-struct proto_family_str*
-find_proto_module(u_long proto_family, u_long if_family)
-{
-	struct proto_family_str  *mod = NULL;
-
-	TAILQ_FOREACH(mod, &proto_family_head, proto_fam_next) {
-		if ((mod->proto_family == (proto_family & 0xffff)) 
-			&& (mod->if_family == (if_family & 0xffff))) 
-			break;
-		}
-
-	return mod;
-}
-
 static struct if_proto*
 find_attached_proto(struct ifnet *ifp, u_long protocol_family)
 {
@@ -385,13 +343,13 @@ find_attached_proto(struct ifnet *ifp, u_long protocol_family)
 static void
 if_proto_ref(struct if_proto *proto)
 {
-	OSAddAtomic(1, (UInt32*)&proto->refcount);
+	OSAddAtomic(1, (SInt32*)&proto->refcount);
 }
 
 static void
 if_proto_free(struct if_proto *proto)
 {
-	int oldval = OSAddAtomic(-1, (UInt32*)&proto->refcount);
+	int oldval = OSAddAtomic(-1, (SInt32*)&proto->refcount);
 	
 	if (oldval == 1) { /* This was the last reference */
 		FREE(proto, M_IFADDR);
@@ -452,39 +410,23 @@ ifnet_lock_done(
 }
 
 __private_extern__ void
-ifnet_head_lock_shared()
+ifnet_head_lock_shared(void)
 {
 	lck_rw_lock_shared(ifnet_head_mutex);
 }
 
 __private_extern__ void
-ifnet_head_lock_exclusive()
+ifnet_head_lock_exclusive(void)
 {
 	lck_rw_lock_exclusive(ifnet_head_mutex);
 }
 
 __private_extern__ void
-ifnet_head_done()
+ifnet_head_done(void)
 {
 	lck_rw_done(ifnet_head_mutex);
 }
 
-/*
- * Public functions.
- */
-struct ifnet *ifbyfamily(u_long family, short unit)
-{
-	struct ifnet *ifp;
-
-	ifnet_head_lock_shared();
-	TAILQ_FOREACH(ifp, &ifnet_head, if_link)
-		if ((family == ifp->if_family) && (ifp->if_unit == unit))
-			break;
-	ifnet_head_done();
-
-	return ifp;
-}
-
 static int dlil_ifp_proto_count(struct ifnet * ifp) 
 {
 	int				count = 0;
@@ -510,7 +452,7 @@ dlil_post_msg(struct ifnet *ifp, u_long event_subclass, u_long event_code,
 	struct kev_msg  		ev_msg;
 	
 	/* 
-	 * a net event always start with a net_event_data structure
+	 * a net event always starts with a net_event_data structure
 	 * but the caller can generate a simple net event or
 	 * provide a longer event structure to post
 	 */
@@ -536,57 +478,122 @@ dlil_post_msg(struct ifnet *ifp, u_long event_subclass, u_long event_code,
 	dlil_event_internal(ifp, &ev_msg);
 }
 
-void dlil_init(void);
+__private_extern__ int
+dlil_create_input_thread(
+	ifnet_t ifp, struct dlil_threading_info *inputthread)
+{
+	int error;
+
+	bzero(inputthread, sizeof(*inputthread));
+	// loopback ifp may not be configured at dlil_init time.
+	if (ifp == lo_ifp)
+		strlcat(inputthread->input_name, "dlil_input_main_thread_mtx", 32);
+	else
+		snprintf(inputthread->input_name, 32, "dlil_input_%s%d_mtx", ifp->if_name, ifp->if_unit);	
+
+	inputthread->lck_grp = lck_grp_alloc_init(inputthread->input_name, dlil_grp_attributes);
+	inputthread->input_lck  = lck_mtx_alloc_init(inputthread->lck_grp, dlil_lck_attributes);
+
+	error= kernel_thread_start((thread_continue_t)dlil_input_thread_func, inputthread, &inputthread->input_thread);
+	if (error == 0) {
+       		ml_thread_policy(inputthread->input_thread, MACHINE_GROUP,
+				 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_NETISR));
+		/*
+		 * Except for the loopback dlil input thread, we create
+		 * an affinity set so that the matching workloop thread
+		 * can be scheduled on the same processor set.
+		 */
+		if (net_affinity && inputthread != dlil_lo_thread_ptr) {
+			struct thread *tp = inputthread->input_thread;
+			u_int32_t tag;
+			/*
+			 * Randomize to reduce the probability
+			 * of affinity tag namespace collision.
+			 */
+			read_random(&tag, sizeof (tag));
+			if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) {
+				thread_reference(tp);
+				inputthread->tag = tag;
+				inputthread->net_affinity = TRUE;
+			}
+		}
+	} else {
+		panic("dlil_create_input_thread: couldn't create thread\n");
+	}
+	OSAddAtomic(1, (SInt32*)&cur_dlil_input_threads);
+#if DLIL_DEBUG
+	printf("dlil_create_input_thread: threadinfo: %p input_thread=%p threads: cur=%d max=%d\n", 
+		inputthread, inputthread->input_thread, dlil_multithreaded_input, cur_dlil_input_threads);
+#endif
+	return error;
+}
+__private_extern__ void
+dlil_terminate_input_thread(
+	struct dlil_threading_info *inputthread)
+{
+	OSAddAtomic(-1, (SInt32*)&cur_dlil_input_threads);
+
+	lck_mtx_unlock(inputthread->input_lck);
+	lck_mtx_free(inputthread->input_lck, inputthread->lck_grp);
+	lck_grp_free(inputthread->lck_grp);
+
+	FREE(inputthread, M_NKE);
+
+	/* For the extra reference count from kernel_thread_start() */
+	thread_deallocate(current_thread());
+
+	thread_terminate(current_thread());
+}
+
+static kern_return_t
+dlil_affinity_set(struct thread *tp, u_int32_t tag)
+{
+	thread_affinity_policy_data_t policy;
+
+	bzero(&policy, sizeof (policy));
+	policy.affinity_tag = tag;
+	return (thread_policy_set(tp, THREAD_AFFINITY_POLICY,
+	    (thread_policy_t)&policy, THREAD_AFFINITY_POLICY_COUNT));
+}
+
 void
 dlil_init(void)
 {
-	lck_grp_attr_t	*grp_attributes = 0;
-	lck_attr_t		*lck_attributes = 0;
-	lck_grp_t		*input_lock_grp = 0;
+	PE_parse_boot_arg("net_affinity", &net_affinity);
 	
 	TAILQ_INIT(&dlil_ifnet_head);
-	TAILQ_INIT(&if_family_head);
-	TAILQ_INIT(&proto_family_head);
 	TAILQ_INIT(&ifnet_head);
 	
 	/* Setup the lock groups we will use */
-	grp_attributes = lck_grp_attr_alloc_init();
+	dlil_grp_attributes = lck_grp_attr_alloc_init();
 
-	dlil_lock_group = lck_grp_alloc_init("dlil internal locks", grp_attributes);
-#if IFNET_RW_LOCK
-	ifnet_lock_group = lck_grp_alloc_init("ifnet locks", grp_attributes);
-#else
-	ifnet_lock_group = lck_grp_alloc_init("ifnet locks", grp_attributes);
-#endif
-	ifnet_head_lock_group = lck_grp_alloc_init("ifnet head lock", grp_attributes);
-	input_lock_grp = lck_grp_alloc_init("dlil input lock", grp_attributes);
-	lck_grp_attr_free(grp_attributes);
-	grp_attributes = 0;
+	dlil_lock_group = lck_grp_alloc_init("dlil internal locks", dlil_grp_attributes);
+	ifnet_lock_group = lck_grp_alloc_init("ifnet locks", dlil_grp_attributes);
+	ifnet_head_lock_group = lck_grp_alloc_init("ifnet head lock", dlil_grp_attributes);
+	dlil_input_lock_grp = lck_grp_alloc_init("dlil input lock", dlil_grp_attributes);
 	
 	/* Setup the lock attributes we will use */
-	lck_attributes = lck_attr_alloc_init();
+	dlil_lck_attributes = lck_attr_alloc_init();
 	
 	ifnet_lock_attr = lck_attr_alloc_init();
 	
-	dlil_input_lock = lck_spin_alloc_init(input_lock_grp, lck_attributes);
-	input_lock_grp = 0;
 	
-	ifnet_head_mutex = lck_rw_alloc_init(ifnet_head_lock_group, lck_attributes);
-	proto_family_mutex = lck_mtx_alloc_init(dlil_lock_group, lck_attributes);
-	dlil_ifnet_mutex = lck_mtx_alloc_init(dlil_lock_group, lck_attributes);
-	dlil_mutex = lck_mtx_alloc_init(dlil_lock_group, lck_attributes);
+	ifnet_head_mutex = lck_rw_alloc_init(ifnet_head_lock_group, dlil_lck_attributes);
+	dlil_ifnet_mutex = lck_mtx_alloc_init(dlil_lock_group, dlil_lck_attributes);
+	dlil_mutex = lck_mtx_alloc_init(dlil_lock_group, dlil_lck_attributes);
 	
-	lck_attr_free(lck_attributes);
-	lck_attributes = 0;
+	lck_attr_free(dlil_lck_attributes);
+	dlil_lck_attributes = NULL;
 	
 	/*
-	 * Start up the dlil input thread once everything is initialized
+	 * Create and start up the first dlil input thread once everything is initialized
 	 */
-	(void) kernel_thread(kernel_task, dlil_input_thread);
+	dlil_create_input_thread(0, dlil_lo_thread_ptr);
+
 	(void) kernel_thread(kernel_task, dlil_call_delayed_detach_thread);
 }
 
-int
+__private_extern__ int
 dlil_attach_filter(
 	struct ifnet			*ifp,
 	const struct iff_filter	*if_filter,
@@ -623,7 +630,9 @@ dlil_attach_filter(
 }
 
 static int
-dlil_detach_filter_internal(interface_filter_t filter, int detached)
+dlil_detach_filter_internal(
+	interface_filter_t	filter,
+	int					detached)
 {
 	int retval = 0;
 	
@@ -685,7 +694,7 @@ dlil_detach_filter_internal(interface_filter_t filter, int detached)
 	return retval;
 }
 
-void
+__private_extern__ void
 dlil_detach_filter(interface_filter_t filter)
 {
 	if (filter == NULL)
@@ -694,97 +703,226 @@ dlil_detach_filter(interface_filter_t filter)
 }
 
 static void
-dlil_input_thread_continue(
-	__unused void*			foo,
-	__unused wait_result_t	wait)
+dlil_input_thread_func(
+	struct dlil_threading_info *inputthread)
 {
 	while (1) {
-		struct mbuf *m, *m_loop;
+		struct mbuf *m = NULL, *m_loop = NULL;
+#if IFNET_INPUT_SANITY_CHK
+		int		loop_cnt = 0, mbuf_cnt;
+		int		count;
+		struct mbuf *m1;
+#endif /* IFNET_INPUT_SANITY_CHK */
+		
+		lck_mtx_lock(inputthread->input_lck);
+		
+		/* Wait until there is work to be done */
+		while ((inputthread->input_waiting & ~DLIL_INPUT_RUNNING) == 0) {
+			inputthread->input_waiting &= ~DLIL_INPUT_RUNNING;
+			msleep(&inputthread->input_waiting, inputthread->input_lck, 0, inputthread->input_name, 0);
+		}
+
+	
+		lck_mtx_assert(inputthread->input_lck, LCK_MTX_ASSERT_OWNED);
+
+		m = inputthread->mbuf_head;
+		inputthread->mbuf_head = NULL;
+		inputthread->mbuf_tail = NULL;
+
+		if (inputthread->input_waiting & DLIL_INPUT_TERMINATE) {
+				if (m)
+					mbuf_freem_list(m);
+				/* this is the end */
+				dlil_terminate_input_thread(inputthread);
+				return;
+		}
+
+		inputthread->input_waiting |= DLIL_INPUT_RUNNING;
+		inputthread->input_waiting &= ~DLIL_INPUT_WAITING;
+
+		if (inputthread == dlil_lo_thread_ptr) {
+			m_loop = dlil_lo_input_mbuf_head;
+			dlil_lo_input_mbuf_head = NULL;
+			dlil_lo_input_mbuf_tail = NULL;
+		}
+
+#if IFNET_INPUT_SANITY_CHK
+		if (dlil_input_sanity_check != 0) {
+			mbuf_cnt = inputthread->mbuf_count;
+			inputthread->mbuf_count = 0;
+			if (inputthread == dlil_lo_thread_ptr) {
+				loop_cnt = dlil_lo_input_mbuf_count;
+				dlil_lo_input_mbuf_count = 0;
+			}
 		
-		lck_spin_lock(dlil_input_lock);
-		m = dlil_input_mbuf_head;
-		dlil_input_mbuf_head = NULL;
-		dlil_input_mbuf_tail = NULL;
-		m_loop = dlil_input_loop_head;
-		dlil_input_loop_head = NULL;
-		dlil_input_loop_tail = NULL;
-		lck_spin_unlock(dlil_input_lock);
+			lck_mtx_unlock(inputthread->input_lck);
 		
+			for (m1 = m, count = 0; m1; m1 = mbuf_nextpkt(m1)) {
+				count++;
+			}
+			if (count != mbuf_cnt) {
+				panic("dlil_input_func - thread=%p reg. loop queue has %d packets, should have %d\n",
+					  inputthread, count, mbuf_cnt);
+			}
+	
+			if (inputthread == dlil_lo_thread_ptr) {
+				for (m1 = m_loop, count = 0; m1; m1 = mbuf_nextpkt(m1)) {
+					count++;
+				}
+				if (count != loop_cnt) {
+					panic("dlil_input_func - thread=%p loop queue has %d packets, should have %d\n",
+					  inputthread, count, loop_cnt);
+				}
+			}
+		} else 
+#endif /* IFNET_INPUT_SANITY_CHK */
+		{
+			lck_mtx_unlock(inputthread->input_lck);
+		}
+
+
 		/*
 		* NOTE warning %%% attention !!!!
 		* We should think about putting some thread starvation safeguards if 
 		* we deal with long chains of packets.
 		*/
-		while (m) {
-			struct mbuf *m0 = m->m_nextpkt;
-			void *header = m->m_pkthdr.header;
-			
-			m->m_nextpkt = NULL;
-			m->m_pkthdr.header = NULL;
-			(void) dlil_input_packet(m->m_pkthdr.rcvif, m, header);
-			m = m0;
-		}
-		m = m_loop;
-		while (m) {
-			struct mbuf *m0 = m->m_nextpkt;
-			void *header = m->m_pkthdr.header;
-			struct ifnet *ifp = &loif[0];
-			
-			m->m_nextpkt = NULL;
-			m->m_pkthdr.header = NULL;
-			(void) dlil_input_packet(ifp, m, header);
-			m = m0;
+		if (m_loop) {
+			if (inputthread == dlil_lo_thread_ptr)
+				dlil_input_packet_list(lo_ifp, m_loop);
+#if IFNET_INPUT_SANITY_CHK
+			else
+				panic("dlil_input_func - thread=%p loop queue has %d packets, should have none!\n",
+				  inputthread, loop_cnt);
+#endif /* IFNET_INPUT_SANITY_CHK */
 		}
+
+
+		if (m)
+			dlil_input_packet_list(0, m);
+
+
+		lck_mtx_lock(inputthread->input_lck);
+
+		if ((inputthread->input_waiting & (DLIL_PROTO_WAITING | DLIL_PROTO_REGISTER)) != 0)  {
+			lck_mtx_unlock(inputthread->input_lck);
+			proto_input_run();
+		}	
+		else	
+			lck_mtx_unlock(inputthread->input_lck);
+	}
+}
+
+errno_t
+ifnet_input(
+	ifnet_t									ifp,
+	mbuf_t									m_head,
+	const struct ifnet_stat_increment_param	*stats)
+{
+	struct thread *tp = current_thread();
+	mbuf_t		m_tail;
+	struct dlil_threading_info *inp;
+#if IFNET_INPUT_SANITY_CHK
+	u_int32_t	pkt_count = 0;
+#endif /* IFNET_INPUT_SANITY_CHK */
+
+	if (ifp == NULL || m_head == NULL) {
+		if (m_head)
+			mbuf_freem_list(m_head);
+		return EINVAL;
+	}
+
+	m_tail = m_head;
+	while (1) {
+#if IFNET_INPUT_SANITY_CHK
+		if (dlil_input_sanity_check != 0) {
+			ifnet_t	rcvif;
 		
-		proto_input_run();
+			rcvif = mbuf_pkthdr_rcvif(m_tail);
+			pkt_count++;
 		
-		if (dlil_input_mbuf_head == NULL && 
-			dlil_input_loop_head == NULL && inject_buckets == 0) {
-			assert_wait(&dlil_input_thread_wakeup, THREAD_UNINT);
-			(void) thread_block(dlil_input_thread_continue);
-			/* NOTREACHED */
+			if (rcvif == NULL ||
+				(ifp->if_type != IFT_LOOP && rcvif != ifp) ||
+				(mbuf_flags(m_head) & MBUF_PKTHDR) == 0) {
+				panic("ifnet_input - invalid mbuf %p\n", m_tail);
+			}
 		}
+#endif /* IFNET_INPUT_SANITY_CHK */
+		if (mbuf_nextpkt(m_tail) == NULL)
+			break;
+		m_tail = mbuf_nextpkt(m_tail);
 	}
-}
 
-void dlil_input_thread(void)
-{
-	register thread_t self = current_thread();
-	
-	ml_thread_policy(self, MACHINE_GROUP,
-					 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_NETISR));
-	
-	dlil_initialized = 1;
-	dlil_input_thread_ptr = current_thread();
-	dlil_input_thread_continue(NULL, THREAD_RESTART);
-}
+	inp = ifp->if_input_thread;
 
-int
-dlil_input_with_stats(
-	struct ifnet *ifp,
-	struct mbuf *m_head,
-	struct mbuf *m_tail,
-	const struct ifnet_stat_increment_param *stats)
-{
-	/* WARNING
+	if (dlil_multithreaded_input == 0 || inp == NULL) 
+		inp = dlil_lo_thread_ptr;
+
+	/*
+	 * If there is a matching dlil input thread associated with an
+	 * affinity set, associate this workloop thread with the same set.
+	 * We will only do this once.
+	 */
+	lck_mtx_lock(inp->input_lck);
+	if (inp->net_affinity && inp->workloop_thread == NULL) {
+		u_int32_t tag = inp->tag;
+		inp->workloop_thread = tp;
+		lck_mtx_unlock(inp->input_lck);
+
+		/* Associated the current thread with the new affinity tag */
+		(void) dlil_affinity_set(tp, tag);
+
+		/*
+		 * Take a reference on the workloop (current) thread; during
+		 * detach, we will need to refer to it in order ot tear down
+		 * its affinity.
+		 */
+		thread_reference(tp);
+		lck_mtx_lock(inp->input_lck);
+	}
+
+        /* WARNING
 	 * Because of loopbacked multicast we cannot stuff the ifp in
 	 * the rcvif of the packet header: loopback has its own dlil
 	 * input queue
 	 */
-	
-	lck_spin_lock(dlil_input_lock);
-	if (ifp->if_type != IFT_LOOP) {
-		if (dlil_input_mbuf_head == NULL)
-			dlil_input_mbuf_head = m_head;
-		else if (dlil_input_mbuf_tail != NULL)
-			dlil_input_mbuf_tail->m_nextpkt = m_head;
-		dlil_input_mbuf_tail = m_tail ? m_tail : m_head;
-	} else {
-		if (dlil_input_loop_head == NULL)
-			dlil_input_loop_head = m_head;
-		else if (dlil_input_loop_tail != NULL)
-			dlil_input_loop_tail->m_nextpkt = m_head;
-		dlil_input_loop_tail = m_tail ? m_tail : m_head;
+
+	if (inp == dlil_lo_thread_ptr && ifp->if_type == IFT_LOOP) {
+		if (dlil_lo_input_mbuf_head == NULL)
+			dlil_lo_input_mbuf_head = m_head;
+		else if (dlil_lo_input_mbuf_tail != NULL)
+			dlil_lo_input_mbuf_tail->m_nextpkt = m_head;
+		dlil_lo_input_mbuf_tail = m_tail;
+#if IFNET_INPUT_SANITY_CHK
+		if (dlil_input_sanity_check != 0) {
+			dlil_lo_input_mbuf_count += pkt_count;
+			inp->input_mbuf_cnt += pkt_count;
+			inp->input_wake_cnt++;
+
+			lck_mtx_assert(inp->input_lck, LCK_MTX_ASSERT_OWNED);
+		}
+#endif
+	}
+	else {
+		if (inp->mbuf_head == NULL)
+			inp->mbuf_head = m_head;
+		else if (inp->mbuf_tail != NULL)
+			inp->mbuf_tail->m_nextpkt = m_head;
+		inp->mbuf_tail = m_tail;
+#if IFNET_INPUT_SANITY_CHK
+		if (dlil_input_sanity_check != 0) {
+			inp->mbuf_count += pkt_count;
+			inp->input_mbuf_cnt += pkt_count;
+			inp->input_wake_cnt++;
+
+			lck_mtx_assert(inp->input_lck, LCK_MTX_ASSERT_OWNED);
+		}
+#endif
+	}
+
+
+	inp->input_waiting |= DLIL_INPUT_WAITING;
+	if ((inp->input_waiting & DLIL_INPUT_RUNNING) == 0) {
+		wakeup((caddr_t)&inp->input_waiting);
 	}
 	if (stats) {
 		ifp->if_data.ifi_ipackets += stats->packets_in;
@@ -798,131 +936,191 @@ dlil_input_with_stats(
 		ifp->if_data.ifi_collisions += stats->collisions;
 		ifp->if_data.ifi_iqdrops += stats->dropped;
 	}
-	lck_spin_unlock(dlil_input_lock);
-	
-	wakeup((caddr_t)&dlil_input_thread_wakeup);
+
+	lck_mtx_unlock(inp->input_lck);
 	
 	return 0; 
 }
 
-int
-dlil_input(struct ifnet  *ifp, struct mbuf *m_head, struct mbuf *m_tail)
+static int
+dlil_interface_filters_input(struct ifnet * ifp, struct mbuf * * m_p,
+			     char * * frame_header_p,
+			     protocol_family_t protocol_family)
 {
-	return dlil_input_with_stats(ifp, m_head, m_tail, NULL);
+	struct ifnet_filter * 		filter;
+
+	TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
+		int result;
+
+		if (filter->filt_input 
+		    && (filter->filt_protocol == 0
+			|| filter->filt_protocol == protocol_family)) {
+			result = (*filter->filt_input)(filter->filt_cookie,
+						       ifp, protocol_family,
+						       m_p, frame_header_p);
+			if (result != 0) {
+				return (result);
+			}
+		}
+	}
+	return (0);
 }
 
-int
-dlil_input_packet(struct ifnet  *ifp, struct mbuf *m,
-	   char *frame_header)
+static void
+dlil_ifproto_input(struct if_proto * ifproto, mbuf_t m)
 {
-    int				 retval;
-    struct if_proto		 *ifproto = 0;
-    protocol_family_t	protocol_family;
-    struct ifnet_filter	*filter;
-
-
-    KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START,0,0,0,0,0);
-
-	/*
-	 * Lock the interface while we run through
-	 * the filters and the demux. This lock
-	 * protects the filter list and the demux list.
-	 */
-	dlil_read_begin();
-
-	/*
-	 * Call family demux module. If the demux module finds a match
-	 * for the frame it will fill-in the ifproto pointer.
-	 */
+	int error;
 
-	retval = ifp->if_demux(ifp, m, frame_header, &protocol_family);
-	if (retval != 0)
-		protocol_family = 0;
-	if (retval == EJUSTRETURN) {
-		dlil_read_end();
-		return 0;
+	if (ifproto->proto_kpi == kProtoKPI_v1) {
+		/* Version 1 protocols get one packet at a time */
+		while (m != NULL) {
+			char *	frame_header;
+			mbuf_t	next_packet;
+			
+			next_packet = m->m_nextpkt;
+			m->m_nextpkt = NULL;
+			frame_header = m->m_pkthdr.header;
+			m->m_pkthdr.header = NULL;
+			error = (*ifproto->kpi.v1.input)(ifproto->ifp, 
+							 ifproto->protocol_family,
+							 m, frame_header);
+			if (error != 0 && error != EJUSTRETURN)
+				m_freem(m);
+			m = next_packet;
+		}
+	}
+	else if (ifproto->proto_kpi == kProtoKPI_v2) {
+		/* Version 2 protocols support packet lists */
+		error = (*ifproto->kpi.v2.input)(ifproto->ifp,
+						 ifproto->protocol_family,
+						 m);
+		if (error != 0 && error != EJUSTRETURN)
+			m_freem_list(m);
 	}
+	return;
+}
 
-	/* DANGER!!! */
-	if (m->m_flags & (M_BCAST|M_MCAST))
-		ifp->if_imcasts++;
+__private_extern__ void
+dlil_input_packet_list(struct ifnet * ifp_param, struct mbuf *m)
+{
+	int				error = 0;
+	int				locked = 0;
+	protocol_family_t		protocol_family;
+	mbuf_t				next_packet;
+	ifnet_t				ifp = ifp_param;
+	char *				frame_header;
+	struct if_proto	*		last_ifproto = NULL;
+	mbuf_t				pkt_first = NULL;
+	mbuf_t *			pkt_next = NULL;
+
+	KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START,0,0,0,0,0);
+
+	while (m != NULL) {
+		struct if_proto *	ifproto = NULL;
+
+		next_packet = m->m_nextpkt;
+		m->m_nextpkt = NULL;
+		if (ifp_param == NULL)
+			ifp = m->m_pkthdr.rcvif;
+		frame_header = m->m_pkthdr.header;
+		m->m_pkthdr.header = NULL;
+
+		if (locked == 0) {
+			/* dlil lock protects the demux and interface filters */
+			locked = 1;
+			dlil_read_begin();
+		}
+		/* find which protocol family this packet is for */
+		error = (*ifp->if_demux)(ifp, m, frame_header,
+					 &protocol_family);
+		if (error != 0) {
+			if (error == EJUSTRETURN) {
+				goto next;
+			}
+			protocol_family = 0;
+		}
+		
+		/* DANGER!!! */
+		if (m->m_flags & (M_BCAST|M_MCAST))
+			ifp->if_imcasts++;
 
-	/*
-	 * Run interface filters
-	 */
-	
-	/* Do not pass VLAN tagged packets to filters PR-3586856 */
-	if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
-		TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
+		/* run interface filters, exclude VLAN packets PR-3586856 */
+		if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
 			int	filter_result;
-			if (filter->filt_input && (filter->filt_protocol == 0 ||
-				 filter->filt_protocol == protocol_family)) {
-				filter_result = filter->filt_input(filter->filt_cookie, ifp, protocol_family, &m, &frame_header);
-				
-				if (filter_result) {
-					dlil_read_end();
-					if (filter_result == EJUSTRETURN) {
-						filter_result = 0;
-					}
-					else {
-						m_freem(m);
-					}
-					
-					return filter_result;
+
+			filter_result = dlil_interface_filters_input(ifp, &m, 
+							  &frame_header,
+							  protocol_family);
+			if (filter_result != 0) {
+				if (filter_result != EJUSTRETURN) {
+					m_freem(m);
 				}
+				goto next;
 			}
 		}
-	}
-
-	/* Demux is done, interface filters have been processed, unlock the mutex */
-	if (retval || ((m->m_flags & M_PROMISC) != 0) ) {
-		dlil_read_end();
-		if (retval != EJUSTRETURN) {
+		if (error != 0 || ((m->m_flags & M_PROMISC) != 0) ) {
 			m_freem(m);
-			return retval;
+			goto next;
 		}
-		else
-			return 0;
-	}
-	
-	ifproto = find_attached_proto(ifp, protocol_family);
-	
-	if (ifproto == 0) {
-		dlil_read_end();
-		DLIL_PRINTF("ERROR - dlil_input - if_demux didn't return an if_proto pointer\n");
-		m_freem(m);
-		return 0;
-	}
-	
-	/*
-	 * Hand the packet off to the protocol.
-	 */
+		
+		/* Lookup the protocol attachment to this interface */
+		if (protocol_family == 0) {
+			ifproto = NULL;
+		}
+		else if (last_ifproto != NULL
+			 && last_ifproto->ifp == ifp
+			 && (last_ifproto->protocol_family
+			     == protocol_family)) {
+			ifproto = last_ifproto;
+		}
+		else {
+			ifproto	= find_attached_proto(ifp, protocol_family);
+		}
+		if (ifproto == NULL) {
+			/* no protocol for this packet, discard */
+			m_freem(m);
+			goto next;
+		}
+		if (ifproto != last_ifproto) {
+			/* make sure ifproto can't go away during input */
+			if_proto_ref(ifproto);
+			if (last_ifproto != NULL) {
+				/* pass up the list for the previous protocol */
+				dlil_read_end();
+				
+				dlil_ifproto_input(last_ifproto, pkt_first);
+				pkt_first = NULL;
+				if_proto_free(last_ifproto);
+				dlil_read_begin();
+			}
+			last_ifproto = ifproto;
+		}
+		/* extend the list */
+		m->m_pkthdr.header = frame_header;
+		if (pkt_first == NULL) {
+			pkt_first = m;
+		} else {
+			*pkt_next = m;
+		}
+		pkt_next = &m->m_nextpkt;
 
-	if (ifproto->dl_domain && (ifproto->dl_domain->dom_flags & DOM_REENTRANT) == 0) {
-		lck_mtx_lock(ifproto->dl_domain->dom_mtx);
-	}
+	next:
+		if (next_packet == NULL && last_ifproto != NULL) {
+			/* pass up the last list of packets */
+			dlil_read_end();
 
-	if (ifproto->proto_kpi == kProtoKPI_DLIL)
-		retval = (*ifproto->kpi.dlil.dl_input)(m, frame_header, 
-					  ifp, ifproto->protocol_family, 
-					  TRUE);
-	else
-		retval = ifproto->kpi.v1.input(ifp, ifproto->protocol_family, m, frame_header);
+			dlil_ifproto_input(last_ifproto, pkt_first);
+			if_proto_free(last_ifproto);
+			locked = 0;
+		}
+		m = next_packet;
 
-	if (ifproto->dl_domain && (ifproto->dl_domain->dom_flags & DOM_REENTRANT) == 0) {
-		lck_mtx_unlock(ifproto->dl_domain->dom_mtx);
 	}
-
-	dlil_read_end();
-
-	if (retval == EJUSTRETURN)
-		retval = 0;
-	else 
-		if (retval)
-			m_freem(m);
-
+	if (locked != 0) {
+		dlil_read_end();
+	}
 	KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_END,0,0,0,0,0);
-	return retval;
+	return;
 }
 
 static int
@@ -946,15 +1144,11 @@ dlil_event_internal(struct ifnet *ifp, struct kev_msg *event)
 				struct if_proto *proto;
 				
 				SLIST_FOREACH(proto, &ifp->if_proto_hash[i], next_hash) {
-					/* Pass the event to the protocol */
-					if (proto->proto_kpi == kProtoKPI_DLIL) {
-						if (proto->kpi.dlil.dl_event)
-							proto->kpi.dlil.dl_event(ifp, event);
-					}
-					else {
-						if (proto->kpi.v1.event)
-							proto->kpi.v1.event(ifp, proto->protocol_family, event);
-					}
+					proto_media_event eventp = proto->proto_kpi == kProtoKPI_v1
+						 ? proto->kpi.v1.event : proto->kpi.v2.event;
+					
+					if (eventp)
+						eventp(ifp, proto->protocol_family, event);
 				}
 			}
 		}
@@ -972,12 +1166,15 @@ dlil_event_internal(struct ifnet *ifp, struct kev_msg *event)
 	return kev_post_msg(event);
 }
 
-int
-dlil_event(struct ifnet *ifp, struct kern_event_msg *event)
+errno_t
+ifnet_event(
+	ifnet_t					ifp,
+	struct kern_event_msg	*event)
 {
-    int result = 0;
-
 	struct kev_msg               kev_msg;
+	int result = 0;
+
+	if (ifp == NULL || event == NULL) return EINVAL;
 
 	kev_msg.vendor_code    = event->vendor_code;
 	kev_msg.kev_class      = event->kev_class;
@@ -987,13 +1184,53 @@ dlil_event(struct ifnet *ifp, struct kern_event_msg *event)
 	kev_msg.dv[0].data_length = event->total_size - KEV_MSG_HEADER_SIZE;
 	kev_msg.dv[1].data_length = 0;
 	
-
 	result = dlil_event_internal(ifp, &kev_msg);
 
-
 	return result;
 }
 
+#if CONFIG_MACF_NET
+#include <netinet/ip6.h>
+#include <netinet/ip.h>
+static int dlil_get_socket_type(struct mbuf **mp, int family, int raw)
+{
+	struct mbuf *m;
+	struct ip *ip;
+	struct ip6_hdr *ip6;
+	int type = SOCK_RAW;
+
+	if (!raw) {
+		switch (family) {
+		case PF_INET:
+			m = m_pullup(*mp, sizeof(struct ip));
+			if (m == NULL)
+				break;
+			*mp = m;
+			ip = mtod(m, struct ip *);
+			if (ip->ip_p == IPPROTO_TCP)
+				type = SOCK_STREAM;
+			else if (ip->ip_p == IPPROTO_UDP)
+				type = SOCK_DGRAM;
+			break;
+		case PF_INET6:
+			m = m_pullup(*mp, sizeof(struct ip6_hdr));
+			if (m == NULL)
+				break;
+			*mp = m;
+			ip6 = mtod(m, struct ip6_hdr *);
+			if (ip6->ip6_nxt == IPPROTO_TCP)
+				type = SOCK_STREAM;
+			else if (ip6->ip6_nxt == IPPROTO_UDP)
+				type = SOCK_DGRAM;
+			break;
+		}
+	}
+
+	return (type);
+}
+#endif
+
+#if 0
 int
 dlil_output_list(
 	struct ifnet* ifp,
@@ -1001,81 +1238,100 @@ dlil_output_list(
 	struct mbuf		*packetlist,
 	caddr_t		route,
 	const struct sockaddr	*dest,
-	int			raw)
+	int						raw)
 {
-	char			*frame_type = 0;
-	char			*dst_linkaddr = 0;
-	int			error, retval = 0;
+	char			*frame_type = NULL;
+	char			*dst_linkaddr = NULL;
+	int				retval = 0;
 	char			frame_type_buffer[MAX_FRAME_TYPE_SIZE * 4];
 	char			dst_linkaddr_buffer[MAX_LINKADDR * 4];
 	struct ifnet_filter *filter;
 	struct if_proto	*proto = 0;
-	struct mbuf *m;
+	mbuf_t	m;
+	mbuf_t	send_head = NULL;
+	mbuf_t	*send_tail = &send_head;
 	
 	KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START,0,0,0,0,0);
-#if BRIDGE
-	if ((raw != 0) || proto_family != PF_INET || do_brige) {
-#else
-	if ((raw != 0) || proto_family != PF_INET) {
-#endif
-		while (packetlist) {
-			m = packetlist;
-			packetlist = packetlist->m_nextpkt;
-			m->m_nextpkt = NULL;
-			error = dlil_output(ifp, proto_family, m, route, dest, raw);
-			if (error) {
-				if (packetlist)
-					m_freem_list(packetlist);
-				return (error);
-			}
-		}
-		return (0);
-	}
 	
 	dlil_read_begin();
 	
 	frame_type	   = frame_type_buffer;
 	dst_linkaddr   = dst_linkaddr_buffer;
+	
+	if (raw == 0) {
+		proto = find_attached_proto(ifp, proto_family);
+		if (proto == NULL) {
+			retval = ENXIO;
+			goto cleanup;
+		}
+	}
+	
+preout_again:
+	if (packetlist == NULL)
+		goto cleanup;
 	m = packetlist;
 	packetlist = packetlist->m_nextpkt;
 	m->m_nextpkt = NULL;
 	
-	proto = find_attached_proto(ifp, proto_family);
-	if (proto == NULL) {
-		retval = ENXIO;
-		goto cleanup;
-	}
-
-	retval = 0;
-	if (proto->proto_kpi == kProtoKPI_DLIL) {
-		if (proto->kpi.dlil.dl_pre_output)
-		retval = proto->kpi.dlil.dl_pre_output(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
-	}
-	else {
-		if (proto->kpi.v1.pre_output)
-		retval = proto->kpi.v1.pre_output(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
-	}
-
-	if (retval) {
-		if (retval != EJUSTRETURN)  {
+	if (raw == 0) {
+		proto_media_preout preoutp = proto->proto_kpi == kProtoKPI_v1
+			 ? proto->kpi.v1.pre_output : proto->kpi.v2.pre_output;
+		retval = 0;
+		if (preoutp)
+			retval = preoutp(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
+	
+		if (retval) {
+			if (retval == EJUSTRETURN) {
+				goto preout_again;
+			}
+			
 			m_freem(m);
+			goto cleanup;
 		}
-		goto cleanup;
 	}
 
 	do {
-		
+#if CONFIG_MACF_NET
+		retval = mac_ifnet_check_transmit(ifp, m, proto_family,
+		    dlil_get_socket_type(&m, proto_family, raw));
+		if (retval) {
+			m_freem(m);
+			goto cleanup;
+		}
+#endif
 	
-		if (ifp->if_framer) {
+		if (raw == 0 && ifp->if_framer) {
 			retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr, frame_type); 
 			if (retval) {
 				if (retval != EJUSTRETURN) {
 					m_freem(m);
 				}
-				goto cleanup;
+				goto next;
 			}
 		}
 	
+#if BRIDGE
+		/* !!!LOCKING!!!
+		 *
+		 * Need to consider how to handle this.
+		 * Also note that return should be a goto cleanup
+		 */
+		broken-locking
+		if (do_bridge) {
+			struct mbuf *m0 = m;
+			struct ether_header *eh = mtod(m, struct ether_header *);
+			
+			if (m->m_pkthdr.rcvif)
+				m->m_pkthdr.rcvif = NULL;
+			ifp = bridge_dst_lookup(eh);
+			bdg_forward(&m0, ifp);
+			if (m0)
+				m_freem(m0);
+			
+			return 0 - should be goto cleanup?
+		}
+#endif
+
 		/* 
 		 * Let interface filters (if any) do their thing ...
 		 */
@@ -1093,17 +1349,25 @@ dlil_output_list(
 				}
 			}
 		}
+		
 		/*
-		* Finally, call the driver.
-		*/
+		 * Finally, call the driver.
+		 */
 	
-		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
-		retval = ifp->if_output(ifp, m);
-		if (retval) {
-			printf("dlil_output_list: output error retval = %x\n", retval);
-			goto cleanup;
+		if ((ifp->if_eflags & IFEF_SENDLIST) != 0) {
+			*send_tail = m;
+			send_tail = &m->m_nextpkt;
+		}
+		else {
+			KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
+			retval = ifp->if_output(ifp, m);
+			if (retval) {
+				printf("dlil_output: output error retval = %x\n", retval);
+			}
+			KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
 		}
 		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
+
 next:
 		m = packetlist;
 		if (m) {
@@ -1112,17 +1376,26 @@ next:
 		}
 	} while (m);
 
+	if (send_head) {
+		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
+		retval = ifp->if_output(ifp, send_head);
+		if (retval) {
+			printf("dlil_output: output error retval = %x\n", retval);
+		}
+		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
+	}
 	
 	KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
 
 cleanup:
 	dlil_read_end();
 	if (packetlist) /* if any packet left, clean up */
-		m_freem_list(packetlist);
+		mbuf_freem_list(packetlist);
 	if (retval == EJUSTRETURN)
 		retval = 0;
 	return retval;
 }
+#endif
 
 /*
  * dlil_output
@@ -1137,21 +1410,25 @@ cleanup:
  * because a protocol is likely to interact with an ifp while it
  * is under the protocol lock.
  */
-int
+__private_extern__ errno_t
 dlil_output(
-	struct ifnet* ifp,
-	u_long proto_family,
-	struct mbuf		*m,
-	caddr_t		route,
+	ifnet_t					ifp,
+	protocol_family_t		proto_family,
+	mbuf_t					packetlist,
+	void					*route,
 	const struct sockaddr	*dest,
-	int			raw)
+	int						raw)
 {
-	char			*frame_type = 0;
-	char			*dst_linkaddr = 0;
+	char			*frame_type = NULL;
+	char			*dst_linkaddr = NULL;
 	int				retval = 0;
 	char			frame_type_buffer[MAX_FRAME_TYPE_SIZE * 4];
 	char			dst_linkaddr_buffer[MAX_LINKADDR * 4];
 	struct ifnet_filter *filter;
+	struct if_proto	*proto = 0;
+	mbuf_t	m;
+	mbuf_t	send_head = NULL;
+	mbuf_t	*send_tail = &send_head;
 	
 	KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START,0,0,0,0,0);
 	
@@ -1161,116 +1438,171 @@ dlil_output(
 	dst_linkaddr   = dst_linkaddr_buffer;
 	
 	if (raw == 0) {
-		struct if_proto	*proto = 0;
-		
 		proto = find_attached_proto(ifp, proto_family);
 		if (proto == NULL) {
-			m_freem(m);
 			retval = ENXIO;
 			goto cleanup;
 		}
-		
+	}
+	
+preout_again:
+	if (packetlist == NULL)
+		goto cleanup;
+	m = packetlist;
+	packetlist = packetlist->m_nextpkt;
+	m->m_nextpkt = NULL;
+	
+	if (raw == 0) {
+		proto_media_preout preoutp = proto->proto_kpi == kProtoKPI_v1
+			 ? proto->kpi.v1.pre_output : proto->kpi.v2.pre_output;
 		retval = 0;
-		if (proto->proto_kpi == kProtoKPI_DLIL) {
-			if (proto->kpi.dlil.dl_pre_output)
-				retval = proto->kpi.dlil.dl_pre_output(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
-		}
-		else {
-			if (proto->kpi.v1.pre_output)
-				retval = proto->kpi.v1.pre_output(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
-		}
-		
+		if (preoutp)
+			retval = preoutp(ifp, proto_family, &m, dest, route, frame_type, dst_linkaddr);
+	
 		if (retval) {
-			if (retval != EJUSTRETURN) {
-				m_freem(m);
+			if (retval == EJUSTRETURN) {
+				goto preout_again;
 			}
+			
+			m_freem(m);
 			goto cleanup;
 		}
 	}
-	
-	/*
-	 * Call framing module 
-	 */
-	if ((raw == 0) && (ifp->if_framer)) {
-		retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr, frame_type); 
-		if (retval) {
-			if (retval != EJUSTRETURN) {
-				m_freem(m);
+
+#if CONFIG_MACF_NET
+	retval = mac_ifnet_check_transmit(ifp, m, proto_family,
+	    dlil_get_socket_type(&m, proto_family, raw));
+	if (retval) {
+		m_freem(m);
+		goto cleanup;
+	}
+#endif
+
+	do {
+		if (raw == 0 && ifp->if_framer) {
+			retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr, frame_type); 
+			if (retval) {
+				if (retval != EJUSTRETURN) {
+					m_freem(m);
+				}
+				goto next;
 			}
-			goto cleanup;
 		}
-	}
 	
 #if BRIDGE
-	/* !!!LOCKING!!!
-	 *
-	 * Need to consider how to handle this.
-	 */
-	broken-locking
-	if (do_bridge) {
-		struct mbuf *m0 = m;
-		struct ether_header *eh = mtod(m, struct ether_header *);
-		
-		if (m->m_pkthdr.rcvif)
-			m->m_pkthdr.rcvif = NULL;
-		ifp = bridge_dst_lookup(eh);
-		bdg_forward(&m0, ifp);
-		if (m0)
-			m_freem(m0);
-		
-		return 0;
-	}
+		/* !!!LOCKING!!!
+		 *
+		 * Need to consider how to handle this.
+		 * Also note that return should be a goto cleanup
+		 */
+		broken-locking
+		if (do_bridge) {
+			struct mbuf *m0 = m;
+			struct ether_header *eh = mtod(m, struct ether_header *);
+			
+			if (m->m_pkthdr.rcvif)
+				m->m_pkthdr.rcvif = NULL;
+			ifp = bridge_dst_lookup(eh);
+			bdg_forward(&m0, ifp);
+			if (m0)
+				m_freem(m0);
+			
+			return 0 - should be goto cleanup?
+		}
 #endif
-	
-	
-	/* 
-	 * Let interface filters (if any) do their thing ...
-	 */
-	
-	/* Do not pass VLAN tagged packets to filters PR-3586856 */
-	if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
-		TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
-			if ((filter->filt_protocol == 0 || (filter->filt_protocol == proto_family)) &&
-				filter->filt_output) {
-				retval = filter->filt_output(filter->filt_cookie, ifp, proto_family, &m);
-				if (retval) {
-					if (retval != EJUSTRETURN)
-						m_freem(m);
-					goto cleanup;
+
+		/* 
+		 * Let interface filters (if any) do their thing ...
+		 */
+		/* Do not pass VLAN tagged packets to filters PR-3586856 */
+		if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
+			TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
+				if ((filter->filt_protocol == 0 || (filter->filt_protocol == proto_family)) &&
+					filter->filt_output) {
+					retval = filter->filt_output(filter->filt_cookie, ifp, proto_family, &m);
+					if (retval) {
+						if (retval != EJUSTRETURN)
+							m_freem(m);
+						goto next;
+					}
 				}
 			}
 		}
+
+		/*
+		 * If the underlying interface is not capable of handling a
+		 * packet whose data portion spans across physically disjoint
+		 * pages, we need to "normalize" the packet so that we pass
+		 * down a chain of mbufs where each mbuf points to a span that
+		 * resides in the system page boundary.  If the packet does
+		 * not cross page(s), the following is a no-op.
+		 */
+		if (!(ifp->if_hwassist & IFNET_MULTIPAGES)) {
+			if ((m = m_normalize(m)) == NULL)
+				goto next;
+		}
+
+		/*
+		 * Finally, call the driver.
+		 */
+	
+		if ((ifp->if_eflags & IFEF_SENDLIST) != 0) {
+			*send_tail = m;
+			send_tail = &m->m_nextpkt;
+		}
+		else {
+			KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
+			retval = ifp->if_output(ifp, m);
+			if (retval) {
+				printf("dlil_output: output error retval = %x\n", retval);
+			}
+			KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
+		}
+		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
+
+next:
+		m = packetlist;
+		if (m) {
+			packetlist = packetlist->m_nextpkt;
+			m->m_nextpkt = NULL;
+		}
+	} while (m);
+
+	if (send_head) {
+		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
+		retval = ifp->if_output(ifp, send_head);
+		if (retval) {
+			printf("dlil_output: output error retval = %x\n", retval);
+		}
+		KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
 	}
 	
-	/*
-	* Finally, call the driver.
-	*/
-	
-	KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START, 0,0,0,0,0);
-	retval = ifp->if_output(ifp, m);
-	KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0,0,0,0,0);
-	
 	KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_END,0,0,0,0,0);
 
 cleanup:
 	dlil_read_end();
+	if (packetlist) /* if any packet left, clean up */
+		mbuf_freem_list(packetlist);
 	if (retval == EJUSTRETURN)
 		retval = 0;
 	return retval;
 }
 
-int
-dlil_ioctl(u_long	proto_fam,
-	   struct ifnet *ifp,
-	   u_long	ioctl_code,
-	   caddr_t	ioctl_arg)
+errno_t
+ifnet_ioctl(
+	ifnet_t				ifp,
+	protocol_family_t	proto_fam,
+	u_int32_t			ioctl_code,
+	void				*ioctl_arg)
 {
 	struct ifnet_filter		*filter;
 	int						retval = EOPNOTSUPP;
 	int						result = 0;
-	struct if_family_str	*if_family;
 	int						holding_read = 0;
 	
+	if (ifp == NULL || ioctl_code == 0)
+		return EINVAL;
+	
 	/* Attempt to increment the use count. If it's zero, bail out, the ifp is invalid */
 	result = ifp_use(ifp, kIfNetUseCount_MustNotBeZero);
 	if (result != 0)
@@ -1304,15 +1636,11 @@ dlil_ioctl(u_long	proto_fam,
 		struct if_proto	*proto = find_attached_proto(ifp, proto_fam);
 		
 		if (proto != 0) {
+			proto_media_ioctl ioctlp = proto->proto_kpi == kProtoKPI_v1
+				 ? proto->kpi.v1.ioctl : proto->kpi.v2.ioctl;
 			result = EOPNOTSUPP;
-			if (proto->proto_kpi == kProtoKPI_DLIL) {
-				if (proto->kpi.dlil.dl_ioctl)
-					result = proto->kpi.dlil.dl_ioctl(proto_fam, ifp, ioctl_code, ioctl_arg);
-			}
-			else {
-				if (proto->kpi.v1.ioctl)
-					result = proto->kpi.v1.ioctl(ifp, proto_fam, ioctl_code, ioctl_arg);
-			}
+			if (ioctlp)
+				result = ioctlp(ifp, proto_fam, ioctl_code, ioctl_arg);
 			
 			/* Only update retval if no one has handled the ioctl */
 			if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
@@ -1338,26 +1666,6 @@ dlil_ioctl(u_long	proto_fam,
 	
 	/* retval is either 0 or EOPNOTSUPP */
 	
-	/*
-	 * Let the family handle this ioctl.
-	 * If it returns something non-zero and not EOPNOTSUPP, we're done.
-	 * If it returns zero, the ioctl was handled, so set retval to zero.
-	 */
-	if_family = find_family_module(ifp->if_family);
-	if ((if_family) && (if_family->ifmod_ioctl)) {
-		result = (*if_family->ifmod_ioctl)(ifp, ioctl_code, ioctl_arg);
-		
-		/* Only update retval if no one has handled the ioctl */
-		if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
-			if (result == ENOTSUP)
-				result = EOPNOTSUPP;
-			retval = result;
-			if (retval && retval != EOPNOTSUPP) {
-				goto cleanup;
-			}
-		}
-	}
-	
 	/*
 	 * Let the interface handle this ioctl.
 	 * If it returns EOPNOTSUPP, ignore that, we may have
@@ -1403,7 +1711,7 @@ dlil_set_bpf_tap(
 	return error;
 }
 
-__private_extern__ errno_t
+errno_t
 dlil_resolve_multi(
 	struct ifnet *ifp,
 	const struct sockaddr *proto_addr,
@@ -1413,6 +1721,7 @@ dlil_resolve_multi(
 	errno_t	result = EOPNOTSUPP;
 	struct if_proto *proto;
 	const struct sockaddr *verify;
+	proto_media_resolve_multi resolvep;
 	
 	dlil_read_begin();
 	
@@ -1420,10 +1729,12 @@ dlil_resolve_multi(
 	
 	/* Call the protocol first */
 	proto = find_attached_proto(ifp, proto_addr->sa_family);
-	if (proto != NULL && proto->proto_kpi != kProtoKPI_DLIL &&
-		proto->kpi.v1.resolve_multi != NULL) {
-		result = proto->kpi.v1.resolve_multi(ifp, proto_addr,
-										(struct sockaddr_dl*)ll_addr, ll_len);
+	if (proto != NULL) {
+		resolvep = proto->proto_kpi == kProtoKPI_v1
+			 ? proto->kpi.v1.resolve_multi : proto->kpi.v2.resolve_multi;
+		if (resolvep != NULL)
+			result = resolvep(ifp, proto_addr,(struct sockaddr_dl*)ll_addr,
+							  ll_len);
 	}
 	
 	/* Let the interface verify the multicast address */
@@ -1455,13 +1766,18 @@ dlil_send_arp_internal(
 	dlil_read_begin();
 	
 	proto = find_attached_proto(ifp, target_proto->sa_family);
-	if (proto == NULL || proto->proto_kpi == kProtoKPI_DLIL ||
-		proto->kpi.v1.send_arp == NULL) {
+	if (proto == NULL) {
 		result = ENOTSUP;
 	}
 	else {
-		result = proto->kpi.v1.send_arp(ifp, arpop, sender_hw, sender_proto,
-										target_hw, target_proto);
+		proto_media_send_arp	arpp;
+		arpp = proto->proto_kpi == kProtoKPI_v1
+			 ? proto->kpi.v1.send_arp : proto->kpi.v2.send_arp;
+		if (arpp == NULL)
+			result = ENOTSUP;
+		else
+			result = arpp(ifp, arpop, sender_hw, sender_proto, target_hw,
+						  target_proto);
 	}
 	
 	dlil_read_end();
@@ -1469,6 +1785,16 @@ dlil_send_arp_internal(
 	return result;
 }
 
+static __inline__ int
+_is_announcement(const struct sockaddr_in * sender_sin,
+		     const struct sockaddr_in * target_sin)
+{
+	if (sender_sin == NULL) {
+		return FALSE;
+	}
+	return (sender_sin->sin_addr.s_addr == target_sin->sin_addr.s_addr);
+}
+
 __private_extern__ errno_t
 dlil_send_arp(
 	ifnet_t	ifp,
@@ -1479,6 +1805,8 @@ dlil_send_arp(
 	const struct sockaddr* target_proto)
 {
 	errno_t	result = 0;
+	const struct sockaddr_in * sender_sin;
+	const struct sockaddr_in * target_sin;
 	
 	if (target_proto == NULL || (sender_proto &&
 		sender_proto->sa_family != target_proto->sa_family))
@@ -1486,11 +1814,17 @@ dlil_send_arp(
 	
 	/*
 	 * If this is an ARP request and the target IP is IPv4LL,
-	 * send the request on all interfaces.
+	 * send the request on all interfaces.  The exception is
+	 * an announcement, which must only appear on the specific
+	 * interface.
 	 */
-	if (IN_LINKLOCAL(((const struct sockaddr_in*)target_proto)->sin_addr.s_addr)
-		 && ipv4_ll_arp_aware != 0 && target_proto->sa_family == AF_INET &&
-		arpop == ARPOP_REQUEST) {
+	sender_sin = (const struct sockaddr_in *)sender_proto;
+	target_sin = (const struct sockaddr_in *)target_proto;
+	if (target_proto->sa_family == AF_INET
+	    && IN_LINKLOCAL(ntohl(target_sin->sin_addr.s_addr))
+	    && ipv4_ll_arp_aware != 0
+	    && arpop == ARPOP_REQUEST
+	    && !_is_announcement(target_sin, sender_sin)) {
 		ifnet_t		*ifp_list;
 		u_int32_t	count;
 		u_int32_t	ifp_on;
@@ -1558,7 +1892,7 @@ dlil_send_arp(
 	return result;
 }
 
-static int
+__private_extern__ int
 ifp_use(
 	struct ifnet *ifp,
 	int	handle_zero)
@@ -1586,11 +1920,10 @@ ifp_use(
  * returns a non-zero value. The caller must call ifp_use_reached_zero
  * after the caller has called dlil_write_end.
  */
-static void
+__private_extern__ void
 ifp_use_reached_zero(
 	struct ifnet *ifp)
 {
-	struct if_family_str *if_family;
 	ifnet_detached_func	free_func;
 	
 	dlil_read_begin();
@@ -1598,15 +1931,12 @@ ifp_use_reached_zero(
 	if (ifp->if_usecnt != 0)
 		panic("ifp_use_reached_zero: ifp->if_usecnt != 0");
 	
-	/* Let BPF know we're detaching */
-	bpfdetach(ifp);
-	
 	ifnet_head_lock_exclusive();
 	ifnet_lock_exclusive(ifp);
 	
 	/* Remove ourselves from the list */
 	TAILQ_REMOVE(&ifnet_head, ifp, if_link);
-	ifnet_addrs[ifp->if_index - 1] = 0;
+	ifnet_addrs[ifp->if_index - 1] = NULL;
 	
 	/* ifp should be removed from the interface list */
 	while (ifp->if_multiaddrs.lh_first) {
@@ -1627,35 +1957,22 @@ ifp_use_reached_zero(
 	ifp->if_eflags &= ~IFEF_DETACHING; // clear the detaching flag
 	ifnet_lock_done(ifp);
 
-	if_family = find_family_module(ifp->if_family);
-	if (if_family && if_family->del_if)
-		if_family->del_if(ifp);
-#if 0
-	if (--if_family->if_usecnt == 0) {
-		if (if_family->shutdown)
-			(*if_family->shutdown)();
-		
-		TAILQ_REMOVE(&if_family_head, if_family, if_fam_next);
-		FREE(if_family, M_IFADDR);
-	}
-#endif
-
-	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHED, 0, 0);
 	free_func = ifp->if_free;
 	dlil_read_end();
+	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHED, NULL, 0);
 	
 	if (free_func)
 		free_func(ifp);
 }
 
-static int
+__private_extern__ int
 ifp_unuse(
 	struct ifnet *ifp)
 {
 	int	oldval;
-	oldval = OSDecrementAtomic((UInt32*)&ifp->if_usecnt);
+	oldval = OSDecrementAtomic((SInt32*)&ifp->if_usecnt);
 	if (oldval == 0)
-		panic("ifp_unuse: ifp(%s%n)->if_usecnt was zero\n", ifp->if_name, ifp->if_unit);
+		panic("ifp_unuse: ifp(%s%d)->if_usecnt was zero\n", ifp->if_name, ifp->if_unit);
  	
 	if (oldval > 1)
 		return 0;
@@ -1666,131 +1983,49 @@ ifp_unuse(
  	return 1; /* caller must call ifp_use_reached_zero */
 }
 
-void
-ifp_reference(
-	struct ifnet *ifp)
-{
-	int	oldval;
-	oldval = OSIncrementAtomic(&ifp->if_refcnt);
-}
-
-void
-ifp_release(
-	struct ifnet *ifp)
-{
-	int	oldval;
-	oldval = OSDecrementAtomic((UInt32*)&ifp->if_refcnt);
-	if (oldval == 0)
-		panic("dlil_if_reference - refcount decremented past zero!");
-}
-
 extern lck_mtx_t 	*domain_proto_mtx;
 
-static int
+static errno_t
 dlil_attach_protocol_internal(
 	struct if_proto	*proto,
-	const struct ddesc_head_str *demux,
 	const struct ifnet_demux_desc *demux_list,
 	u_int32_t	demux_count)
 {
-	struct ddesc_head_str temp_head;
 	struct kev_dl_proto_data	ev_pr_data;
 	struct ifnet *ifp = proto->ifp;
 	int retval = 0;
 	u_long hash_value = proto_hash_value(proto->protocol_family);
-	int	if_using_kpi = (ifp->if_eflags & IFEF_USEKPI) != 0;
-	void* free_me = NULL;
     
     /* setup some of the common values */
-	
 	{
+		struct domain *dp;
 		lck_mtx_lock(domain_proto_mtx);
-		struct domain *dp = domains;
+		dp = domains;
 		while (dp && (protocol_family_t)dp->dom_family != proto->protocol_family)
 			dp = dp->dom_next;
 		proto->dl_domain = dp;
 		lck_mtx_unlock(domain_proto_mtx);
 	}
 	
-	/*
-	 * Convert the demux descriptors to a type the interface
-	 * will understand. Checking e_flags should be safe, this
-	 * flag won't change.
-	 */
-	if (if_using_kpi && demux) {
-		/* Convert the demux linked list to a demux_list */
-		struct dlil_demux_desc	*demux_entry;
-		struct ifnet_demux_desc *temp_list = NULL;
-		u_int32_t i = 0;
-		
-		TAILQ_FOREACH(demux_entry, demux, next) {
-			i++;
-		}
-		
-		temp_list = _MALLOC(sizeof(struct ifnet_demux_desc) * i, M_TEMP, M_WAITOK);
-		free_me = temp_list;
-		
-		if (temp_list == NULL)
-			return ENOMEM;
-		
-		i = 0;
-		TAILQ_FOREACH(demux_entry, demux, next) {
-			/* dlil_demux_desc types 1, 2, and 3 are obsolete and can not be translated */
-			if (demux_entry->type == 1 ||
-				demux_entry->type == 2 ||
-				demux_entry->type == 3) {
-				FREE(free_me, M_TEMP);
-				return ENOTSUP;
-			}
-			
-			temp_list[i].type = demux_entry->type;
-			temp_list[i].data = demux_entry->native_type;
-			temp_list[i].datalen = demux_entry->variants.native_type_length;
-			i++;
-		}
-		demux_count = i;
-		demux_list = temp_list;
-	}
-	else if (!if_using_kpi && demux_list != NULL) {
-		struct dlil_demux_desc *demux_entry;
-		u_int32_t i = 0;
-		
-		demux_entry = _MALLOC(sizeof(struct dlil_demux_desc) * demux_count, M_TEMP, M_WAITOK);
-		free_me = demux_entry;
-		if (demux_entry == NULL)
-			return ENOMEM;
-		
-		TAILQ_INIT(&temp_head);
-		
-		for (i = 0; i < demux_count; i++) {
-			demux_entry[i].type = demux_list[i].type;
-			demux_entry[i].native_type = demux_list[i].data;
-			demux_entry[i].variants.native_type_length = demux_list[i].datalen;
-			TAILQ_INSERT_TAIL(&temp_head, &demux_entry[i], next);
-		}
-		demux = &temp_head;
-	}
-	
 	/*
 	 * Take the write lock to protect readers and exclude other writers.
 	 */
-	dlil_write_begin();
+	if ((retval = dlil_write_begin()) != 0) {
+		printf("dlil_attach_protocol_internal - dlil_write_begin returned %d\n", retval);
+		return retval;
+	}
 	
 	/* Check that the interface isn't currently detaching */
 	ifnet_lock_shared(ifp);
 	if ((ifp->if_eflags & IFEF_DETACHING) != 0) {
 		ifnet_lock_done(ifp);
 		dlil_write_end();
-		if (free_me)
-			FREE(free_me, M_TEMP);
 		return ENXIO;
 	}
 	ifnet_lock_done(ifp);
 	
 	if (find_attached_proto(ifp, proto->protocol_family) != NULL) {
 		dlil_write_end();
-		if (free_me)
-			FREE(free_me, M_TEMP);
 		return EEXIST;
 	}
 	
@@ -1798,16 +2033,9 @@ dlil_attach_protocol_internal(
 	 * Call family module add_proto routine so it can refine the
 	 * demux descriptors as it wishes.
 	 */
-	if (if_using_kpi)
-		retval = ifp->if_add_proto_u.kpi(ifp, proto->protocol_family, demux_list, demux_count);
-	else {
-		retval = ifp->if_add_proto_u.original(ifp, proto->protocol_family,
-											  _cast_non_const(demux));
-	}
+	retval = ifp->if_add_proto(ifp, proto->protocol_family, demux_list, demux_count);
 	if (retval) {
 		dlil_write_end();
-		if (free_me)
-			FREE(free_me, M_TEMP);
 		return retval;
 	}
 	
@@ -1834,8 +2062,6 @@ dlil_attach_protocol_internal(
 	 * Add to if_proto list for this interface
 	 */
 	if_proto_ref(proto);
-	if (proto->proto_kpi == kProtoKPI_DLIL && proto->kpi.dlil.dl_offer)
-		ifp->offercnt++;
 	dlil_write_end();
 	
 	/* the reserved field carries the number of protocol still attached (subject to change) */
@@ -1844,24 +2070,26 @@ dlil_attach_protocol_internal(
 	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED, 
 				  (struct net_event_data *)&ev_pr_data, 
 				  sizeof(struct kev_dl_proto_data));
-	
-	DLIL_PRINTF("Attached protocol %d to %s%d - %d\n", proto->protocol_family,
+#if 0	
+	DLIL_PRINTF("dlil. Attached protocol %d to %s%d - %d\n", proto->protocol_family,
 			 ifp->if_name, ifp->if_unit, retval);
-	if (free_me)
-		FREE(free_me, M_TEMP);
+#endif
 	return retval;
 }
 
-__private_extern__ int
-dlil_attach_protocol_kpi(ifnet_t ifp, protocol_family_t protocol,
+errno_t
+ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol,
 	const struct ifnet_attach_proto_param *proto_details)
 {
 	int retval = 0;
 	struct if_proto  *ifproto = NULL;
 	
+	if (ifp == NULL || protocol == 0 || proto_details == NULL)
+		return EINVAL;
+	
 	ifproto = _MALLOC(sizeof(struct if_proto), M_IFADDR, M_WAITOK);
 	if (ifproto == 0) {
-		DLIL_PRINTF("ERROR - DLIL failed if_proto allocation\n");
+		DLIL_PRINTF("ERROR - dlil failed if_proto allocation\n");
 		retval = ENOMEM;
 		goto end;
 	}
@@ -1878,7 +2106,7 @@ dlil_attach_protocol_kpi(ifnet_t ifp, protocol_family_t protocol,
 	ifproto->kpi.v1.resolve_multi = proto_details->resolve;
 	ifproto->kpi.v1.send_arp = proto_details->send_arp;
 	
-	retval = dlil_attach_protocol_internal(ifproto, NULL,
+	retval = dlil_attach_protocol_internal(ifproto,
 				proto_details->demux_list, proto_details->demux_count);
 	
 end:
@@ -1887,53 +2115,37 @@ end:
 	return retval;
 }
 
-int
-dlil_attach_protocol(struct dlil_proto_reg_str	 *proto)
+errno_t
+ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol,
+	const struct ifnet_attach_proto_param_v2 *proto_details)
 {
-	struct ifnet     *ifp = NULL;
+	int retval = 0;
 	struct if_proto  *ifproto = NULL;
-	int	retval = 0;
-
-	/*
-	 * Do everything we can before taking the write lock
-	 */
 	
-	if ((proto->protocol_family == 0) || (proto->interface_family == 0))
+	if (ifp == NULL || protocol == 0 || proto_details == NULL)
 		return EINVAL;
-
-	/*
-	 * Allocate and init a new if_proto structure
-	 */
+	
 	ifproto = _MALLOC(sizeof(struct if_proto), M_IFADDR, M_WAITOK);
-	if (!ifproto) {
-		DLIL_PRINTF("ERROR - DLIL failed if_proto allocation\n");
+	if (ifproto == 0) {
+		DLIL_PRINTF("ERROR - dlil failed if_proto allocation\n");
 		retval = ENOMEM;
 		goto end;
 	}
+	bzero(ifproto, sizeof(*ifproto));
 	
-
-	/* ifbyfamily returns us an ifp with an incremented if_usecnt */
-	ifp = ifbyfamily(proto->interface_family, proto->unit_number);
-	if (!ifp) {
-		DLIL_PRINTF("dlil_attach_protocol -- no such interface %d unit %d\n", 
-				proto->interface_family, proto->unit_number);
-		retval = ENXIO;
-		goto end;
-	}
-
-    bzero(ifproto, sizeof(struct if_proto));
-	
-	ifproto->ifp			= ifp;
-	ifproto->protocol_family = proto->protocol_family;
-	ifproto->proto_kpi = kProtoKPI_DLIL;
-	ifproto->kpi.dlil.dl_input		= proto->input;
-	ifproto->kpi.dlil.dl_pre_output	= proto->pre_output;
-	ifproto->kpi.dlil.dl_event		= proto->event;
-	ifproto->kpi.dlil.dl_offer		= proto->offer;
-	ifproto->kpi.dlil.dl_ioctl		= proto->ioctl;
-	ifproto->kpi.dlil.dl_detached 	= proto->detached;
-	
-	retval = dlil_attach_protocol_internal(ifproto, &proto->demux_desc_head, NULL, 0);
+	ifproto->ifp = ifp;
+	ifproto->protocol_family = protocol;
+	ifproto->proto_kpi = kProtoKPI_v2;
+	ifproto->kpi.v2.input = proto_details->input;
+	ifproto->kpi.v2.pre_output = proto_details->pre_output;
+	ifproto->kpi.v2.event = proto_details->event;
+	ifproto->kpi.v2.ioctl = proto_details->ioctl;
+	ifproto->kpi.v2.detached = proto_details->detached;
+	ifproto->kpi.v2.resolve_multi = proto_details->resolve;
+	ifproto->kpi.v2.send_arp = proto_details->send_arp;
+	
+	retval = dlil_attach_protocol_internal(ifproto,
+				proto_details->demux_list, proto_details->demux_count);
 	
 end:
 	if (retval && ifproto)
@@ -1951,14 +2163,14 @@ dlil_detach_protocol_internal(
 	u_long proto_family = proto->protocol_family;
 	struct kev_dl_proto_data	ev_pr_data;
 	
-	if (proto->proto_kpi == kProtoKPI_DLIL) {
-		if (proto->kpi.dlil.dl_detached)
-			proto->kpi.dlil.dl_detached(proto->protocol_family, ifp);
-	}
-	else {
+	if (proto->proto_kpi == kProtoKPI_v1) {
 		if (proto->kpi.v1.detached)
 			proto->kpi.v1.detached(ifp, proto->protocol_family);
 	}
+	if (proto->proto_kpi == kProtoKPI_v2) {
+		if (proto->kpi.v2.detached)
+			proto->kpi.v2.detached(ifp, proto->protocol_family);
+	}
 	if_proto_free(proto);
     
 	/*
@@ -1976,13 +2188,14 @@ dlil_detach_protocol_internal(
 	return 0;
 }
 
-int
-dlil_detach_protocol(struct ifnet *ifp, u_long proto_family)
+errno_t
+ifnet_detach_protocol(ifnet_t ifp, protocol_family_t proto_family)
 {
 	struct if_proto *proto = NULL;
 	int	retval = 0;
 	int use_reached_zero = 0;
 	
+	if (ifp == NULL || proto_family == 0) return EINVAL;
 
 	if ((retval = dlil_write_begin()) != 0) {
 		if (retval == EDEADLK) {
@@ -2017,9 +2230,6 @@ dlil_detach_protocol(struct ifnet *ifp, u_long proto_family)
 	if (ifp->if_del_proto)
 		ifp->if_del_proto(ifp, proto->protocol_family);
 
-	if (proto->proto_kpi == kProtoKPI_DLIL && proto->kpi.dlil.dl_offer)
-		ifp->offercnt--;
-
 	SLIST_REMOVE(&ifp->if_proto_hash[proto_hash_value(proto_family)], proto, if_proto, next_hash);
 	
 	/*
@@ -2104,8 +2314,6 @@ dlil_delayed_detach_thread(__unused void* foo, __unused wait_result_t wait)
 						if (proto->detaching) {
 							if (ifp->if_del_proto)
 								ifp->if_del_proto(ifp, proto->protocol_family);
-							if (proto->proto_kpi == kProtoKPI_DLIL && proto->kpi.dlil.dl_offer)
-								ifp->offercnt--;
 							*prev_nextptr = SLIST_NEXT(proto, next_hash);
 							SLIST_INSERT_HEAD(&detached_protos, proto, next_hash);
 							reached_zero = ifp_unuse(ifp);
@@ -2183,18 +2391,25 @@ dlil_call_delayed_detach_thread(void) {
 
 extern int if_next_index(void);
 
-__private_extern__ int
-dlil_if_attach_with_address(
-	struct ifnet		*ifp,
+errno_t
+ifnet_attach(
+	ifnet_t						ifp,
 	const struct sockaddr_dl	*ll_addr)
 {
-	u_long		    interface_family = ifp->if_family;
-	struct if_family_str    *if_family = NULL;
-	int			    stat;
+	u_long		    interface_family;
 	struct ifnet *tmp_if;
 	struct proto_hash_entry *new_proto_list = NULL;
 	int locked = 0;
 	
+	if (ifp == NULL) return EINVAL;
+	if (ll_addr && ifp->if_addrlen == 0) {
+		ifp->if_addrlen = ll_addr->sdl_alen;
+	}
+	else if (ll_addr && ll_addr->sdl_alen != ifp->if_addrlen) {
+		return EINVAL;
+	}
+	
+	interface_family = ifp->if_family;
 	
 	ifnet_head_lock_shared();
 
@@ -2219,19 +2434,13 @@ dlil_if_attach_with_address(
 		return ENOMEM;
 	}
 
-	// Only use family if this is not a KPI interface
-	if ((ifp->if_eflags & IFEF_USEKPI) == 0) {
-		if_family = find_family_module(interface_family);
-	}
-
 	/*
 	 * Allow interfaces withouth protocol families to attach
 	 * only if they have the necessary fields filled out.
 	 */
 	
-	if ((if_family == 0) &&
-		(ifp->if_add_proto == 0 || ifp->if_del_proto == 0)) {
-		DLIL_PRINTF("Attempt to attach interface without family module - %d\n", 
+	if (ifp->if_add_proto == 0 || ifp->if_del_proto == 0) {
+		DLIL_PRINTF("dlil Attempt to attach interface without family module - %ld\n", 
 				interface_family);
 		return ENODEV;
 	}
@@ -2248,36 +2457,20 @@ dlil_if_attach_with_address(
 	dlil_write_begin();
 	locked = 1;
 
-	/*
-	 * Call the family module to fill in the appropriate fields in the
-	 * ifnet structure.
-	 */
-	
-	if (if_family) {
-		stat = if_family->add_if(ifp);
-		if (stat) {
-			DLIL_PRINTF("dlil_if_attach -- add_if failed with %d\n", stat);
-			dlil_write_end();
-			return stat;
-		}
-		ifp->if_add_proto_u.original = if_family->add_proto;
-		ifp->if_del_proto = if_family->del_proto;
-		if_family->refcnt++;
-	}
-	
-	ifp->offercnt = 0;
 	TAILQ_INIT(&ifp->if_flt_head);
 	
 		
 	if (new_proto_list) {
 		bzero(new_proto_list, (PROTO_HASH_SLOTS * sizeof(struct proto_hash_entry)));
 		ifp->if_proto_hash = new_proto_list;
-		new_proto_list = 0;
+		new_proto_list = NULL;
 	}
 	
 	/* old_if_attach */
 	{
-		struct ifaddr *ifa = 0;
+		char workbuf[64];
+		int namelen, masklen, socksize, ifasize;
+		struct ifaddr *ifa = NULL;
 		
 		if (ifp->if_snd.ifq_maxlen == 0)
 			ifp->if_snd.ifq_maxlen = ifqmaxlen;
@@ -2292,53 +2485,56 @@ dlil_if_attach_with_address(
 		ifnet_head_lock_exclusive();
 		ifnet_lock_exclusive(ifp);
 		
-		if ((ifp->if_eflags & IFEF_REUSE) == 0 || ifp->if_index == 0) {
-			char workbuf[64];
-			int namelen, masklen, socksize, ifasize;
-			
+		if ((ifp->if_eflags & IFEF_REUSE) == 0 || ifp->if_index == 0)
 			ifp->if_index = if_next_index();
-			
-			namelen = snprintf(workbuf, sizeof(workbuf), "%s%d", ifp->if_name, ifp->if_unit);
+		else
+			ifa = TAILQ_FIRST(&ifp->if_addrhead);
+		
+		namelen = snprintf(workbuf, sizeof(workbuf), "%s%d", ifp->if_name, ifp->if_unit);
 #define _offsetof(t, m) ((int)((caddr_t)&((t *)0)->m))
-			masklen = _offsetof(struct sockaddr_dl, sdl_data[0]) + namelen;
-			socksize = masklen + ifp->if_addrlen;
+		masklen = _offsetof(struct sockaddr_dl, sdl_data[0]) + namelen;
+		socksize = masklen + ifp->if_addrlen;
 #define ROUNDUP(a) (1 + (((a) - 1) | (sizeof(long) - 1)))
-			if ((u_long)socksize < sizeof(struct sockaddr_dl))
-				socksize = sizeof(struct sockaddr_dl);
-			socksize = ROUNDUP(socksize);
-			ifasize = sizeof(struct ifaddr) + 2 * socksize;
+		if ((u_long)socksize < sizeof(struct sockaddr_dl))
+			socksize = sizeof(struct sockaddr_dl);
+		socksize = ROUNDUP(socksize);
+		ifasize = sizeof(struct ifaddr) + 2 * socksize;
+		
+		/*
+		 * Allocate a new ifa if we don't have one
+		 * or the old one is too small.
+		 */
+		if (ifa == NULL || socksize > ifa->ifa_addr->sa_len) {
+			if (ifa)
+				if_detach_ifa(ifp, ifa);
 			ifa = (struct ifaddr*)_MALLOC(ifasize, M_IFADDR, M_WAITOK);
-			if (ifa) {
-				struct sockaddr_dl *sdl = (struct sockaddr_dl *)(ifa + 1);
-				ifnet_addrs[ifp->if_index - 1] = ifa;
-				bzero(ifa, ifasize);
-				sdl->sdl_len = socksize;
-				sdl->sdl_family = AF_LINK;
-				bcopy(workbuf, sdl->sdl_data, namelen);
-				sdl->sdl_nlen = namelen;
-				sdl->sdl_index = ifp->if_index;
-				sdl->sdl_type = ifp->if_type;
-				if (ll_addr) {
-					sdl->sdl_alen = ll_addr->sdl_alen;
-					if (ll_addr->sdl_alen != ifp->if_addrlen)
-						panic("dlil_if_attach - ll_addr->sdl_alen != ifp->if_addrlen");
-					bcopy(CONST_LLADDR(ll_addr), LLADDR(sdl), sdl->sdl_alen);
-				}
-				ifa->ifa_ifp = ifp;
-				ifa->ifa_rtrequest = link_rtrequest;
-				ifa->ifa_addr = (struct sockaddr*)sdl;
-				sdl = (struct sockaddr_dl*)(socksize + (caddr_t)sdl);
-				ifa->ifa_netmask = (struct sockaddr*)sdl;
-				sdl->sdl_len = masklen;
-				while (namelen != 0)
-					sdl->sdl_data[--namelen] = 0xff;
-			}
-		}
-		else {
-			/* preserve the first ifaddr */
-			ifnet_addrs[ifp->if_index - 1] = TAILQ_FIRST(&ifp->if_addrhead);
 		}
 		
+		if (ifa) {
+			struct sockaddr_dl *sdl = (struct sockaddr_dl *)(ifa + 1);
+			ifnet_addrs[ifp->if_index - 1] = ifa;
+			bzero(ifa, ifasize);
+			sdl->sdl_len = socksize;
+			sdl->sdl_family = AF_LINK;
+			bcopy(workbuf, sdl->sdl_data, namelen);
+			sdl->sdl_nlen = namelen;
+			sdl->sdl_index = ifp->if_index;
+			sdl->sdl_type = ifp->if_type;
+			if (ll_addr) {
+				sdl->sdl_alen = ll_addr->sdl_alen;
+				if (ll_addr->sdl_alen != ifp->if_addrlen)
+					panic("ifnet_attach - ll_addr->sdl_alen != ifp->if_addrlen");
+				bcopy(CONST_LLADDR(ll_addr), LLADDR(sdl), sdl->sdl_alen);
+			}
+			ifa->ifa_ifp = ifp;
+			ifa->ifa_rtrequest = link_rtrequest;
+			ifa->ifa_addr = (struct sockaddr*)sdl;
+			sdl = (struct sockaddr_dl*)(socksize + (caddr_t)sdl);
+			ifa->ifa_netmask = (struct sockaddr*)sdl;
+			sdl->sdl_len = masklen;
+			while (namelen != 0)
+				sdl->sdl_data[--namelen] = 0xff;
+		}
 
 		TAILQ_INIT(&ifp->if_addrhead);
 		ifa = ifnet_addrs[ifp->if_index - 1];
@@ -2352,43 +2548,58 @@ dlil_if_attach_with_address(
 			ifa->ifa_debug |= IFA_ATTACHED;
 			TAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link);
 		}
+#if CONFIG_MACF_NET
+		mac_ifnet_label_associate(ifp);
+#endif
 		
 		TAILQ_INSERT_TAIL(&ifnet_head, ifp, if_link);
 		ifindex2ifnet[ifp->if_index] = ifp;
 		
 		ifnet_head_done();
 	}
-    dlil_write_end();
-		
-	if (if_family && if_family->init_if) {
-		stat = if_family->init_if(ifp);
-		if (stat) {
-			DLIL_PRINTF("dlil_if_attach -- init_if failed with %d\n", stat);
+
+	/* 
+	 * A specific dlil input thread is created per Ethernet interface.
+	 * pseudo interfaces or other types of interfaces use the main ("loopback") thread.
+	 * If the sysctl "net.link.generic.system.multi_threaded_input" is set to zero, all packets will
+	 * be handled by the main loopback thread, reverting to 10.4.x behaviour.
+	 * 
+	 */
+
+	if (ifp->if_type == IFT_ETHER) {
+		int err;
+
+		if (dlil_multithreaded_input > 0) {
+			ifp->if_input_thread = _MALLOC(sizeof(struct dlil_threading_info), M_NKE, M_WAITOK);
+			if (ifp->if_input_thread == NULL)
+				panic("ifnet_attach ifp=%p couldn't alloc threading\n", ifp);
+			if ((err = dlil_create_input_thread(ifp, ifp->if_input_thread)) != 0)
+				panic("ifnet_attach ifp=%p couldn't get a thread. err=%x\n", ifp, err);
+#ifdef DLIL_DEBUG
+			printf("ifnet_attach: dlil thread for ifp=%p if_index=%x\n", ifp, ifp->if_index);
+#endif
 		}
 	}
-    
+    	dlil_write_end();
 	ifnet_lock_done(ifp);
-    dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_ATTACHED, 0, 0);
+	
+	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_ATTACHED, NULL, 0);
 
     return 0;
 }
 
-int
-dlil_if_attach(struct ifnet	*ifp)
-{
-	dlil_if_attach_with_address(ifp, NULL);
-}
-
-
-int
-dlil_if_detach(struct ifnet *ifp)
+errno_t
+ifnet_detach(
+	ifnet_t	ifp)
 {
 	struct ifnet_filter *filter;
 	struct ifnet_filter	*filter_next;
 	int zeroed = 0;
 	int retval = 0;
 	struct ifnet_filter_head fhead;
+	struct dlil_threading_info *inputthread;
 	
+	if (ifp == NULL) return EINVAL;
 	
 	ifnet_lock_exclusive(ifp);
 	
@@ -2408,11 +2619,14 @@ dlil_if_detach(struct ifnet *ifp)
 	ifp->if_eflags |= IFEF_DETACHING;
 	ifnet_lock_done(ifp);
 	
-	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHING, 0, 0);
+	dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHING, NULL, 0);
+	
+	/* Let BPF know we're detaching */
+	bpfdetach(ifp);
 	
 	if ((retval = dlil_write_begin()) != 0) {
 		if (retval == EDEADLK) {
-			retval = DLIL_WAIT_FOR_FREE;
+			retval = 0;
 			
 			/* We need to perform a delayed detach */
 			ifp->if_delayed_detach = 1;
@@ -2425,207 +2639,74 @@ dlil_if_detach(struct ifnet *ifp)
 	/* Steal the list of interface filters */
 	fhead = ifp->if_flt_head;
 	TAILQ_INIT(&ifp->if_flt_head);
-	
+
 	/* unuse the interface */
 	zeroed = ifp_unuse(ifp);
-	
-	dlil_write_end();
-	
-	for (filter = TAILQ_FIRST(&fhead); filter; filter = filter_next) {
-		filter_next = TAILQ_NEXT(filter, filt_next);
-		dlil_detach_filter_internal(filter, 1);
-	}
-	
-	if (zeroed == 0) {
-		retval = DLIL_WAIT_FOR_FREE;
-	}
-	else
-	{
-		ifp_use_reached_zero(ifp);
-	}
-	
-	return retval;
-}
 
+	/*
+	 * If thread affinity was set for the workloop thread, we will need
+	 * to tear down the affinity and release the extra reference count
+	 * taken at attach time;
+	 */
+	if ((inputthread = ifp->if_input_thread) != NULL) {
+		if (inputthread->net_affinity) {
+			struct thread *tp;
+
+			if (inputthread == dlil_lo_thread_ptr)
+				panic("Thread affinity should not be enabled "
+				    "on the loopback dlil input thread\n");
+
+			lck_mtx_lock(inputthread->input_lck);
+			tp = inputthread->workloop_thread;
+			inputthread->workloop_thread = NULL;
+			inputthread->tag = 0;
+			inputthread->net_affinity = FALSE;
+			lck_mtx_unlock(inputthread->input_lck);
+
+			/* Tear down workloop thread affinity */
+			if (tp != NULL) {
+				(void) dlil_affinity_set(tp,
+				    THREAD_AFFINITY_TAG_NULL);
+				thread_deallocate(tp);
+			}
 
-int
-dlil_reg_if_modules(u_long  interface_family, 
-		    struct dlil_ifmod_reg_str  *ifmod)
-{
-    struct if_family_str *if_family;
-
-
-    if (find_family_module(interface_family))  {
-	DLIL_PRINTF("Attempt to register dlil family module more than once - %d\n", 
-	       interface_family);
-	return EEXIST;
-    }
-
-    if ((!ifmod->add_if) || (!ifmod->del_if) ||
-	(!ifmod->add_proto) || (!ifmod->del_proto)) {
-	DLIL_PRINTF("dlil_reg_if_modules passed at least one null pointer\n");
-	return EINVAL;
-    }
-    
-    /*
-     * The following is a gross hack to keep from breaking
-     * Vicomsoft's internet gateway on Jaguar. Vicomsoft
-     * does not zero the reserved fields in dlil_ifmod_reg_str.
-     * As a result, we have to zero any function that used to
-     * be reserved fields at the time Vicomsoft built their
-     * kext. Radar #2974305
-     */
-    if (ifmod->reserved[0] != 0 || ifmod->reserved[1] != 0 || ifmod->reserved[2]) {
-    	if (interface_family == 123) {	/* Vicom */
-			ifmod->init_if = 0;
-		} else {
-			return EINVAL;
+			/* Tear down dlil input thread affinity */
+			tp = inputthread->input_thread;
+			(void) dlil_affinity_set(tp, THREAD_AFFINITY_TAG_NULL);
+			thread_deallocate(tp);
 		}
-    }
 
-    if_family = (struct if_family_str *) _MALLOC(sizeof(struct if_family_str), M_IFADDR, M_WAITOK);
-    if (!if_family) {
-	DLIL_PRINTF("dlil_reg_if_modules failed allocation\n");
-	return ENOMEM;
-    }
-    
-    bzero(if_family, sizeof(struct if_family_str));
-
-    if_family->if_family	= interface_family & 0xffff;
-    if_family->shutdown		= ifmod->shutdown;
-    if_family->add_if		= ifmod->add_if;
-    if_family->del_if		= ifmod->del_if;
-    if_family->init_if		= ifmod->init_if;
-    if_family->add_proto	= ifmod->add_proto;
-    if_family->del_proto	= ifmod->del_proto;
-    if_family->ifmod_ioctl	= ifmod->ifmod_ioctl;
-    if_family->refcnt		= 1;
-    if_family->flags		= 0;
-
-    TAILQ_INSERT_TAIL(&if_family_head, if_family, if_fam_next);
-    return 0;
-}
-
-int dlil_dereg_if_modules(u_long interface_family)
-{
-    struct if_family_str  *if_family;
-    int ret = 0;
-
-
-    if_family = find_family_module(interface_family);
-    if (if_family == 0) {
-	return ENXIO;
-    }
-
-    if (--if_family->refcnt == 0) {
-	if (if_family->shutdown)
-	    (*if_family->shutdown)();
-	
-	TAILQ_REMOVE(&if_family_head, if_family, if_fam_next);
-	FREE(if_family, M_IFADDR);
-    }	
-    else {
-	if_family->flags |= DLIL_SHUTDOWN;
-        ret = DLIL_WAIT_FOR_FREE;
-    }
-
-    return ret;
-}
-					    
-	    
-
-int
-dlil_reg_proto_module(
-	u_long protocol_family,
-	u_long  interface_family, 
-	int (*attach)(struct ifnet *ifp, u_long protocol_family),
-	int (*detach)(struct ifnet *ifp, u_long protocol_family))
-{
-	struct proto_family_str *proto_family;
+		/* cleanup ifp dlil input thread, if any */
+		ifp->if_input_thread = NULL;
 
-	if (attach == NULL) return EINVAL;
+		if (inputthread != dlil_lo_thread_ptr) {
+#ifdef DLIL_DEBUG
+			printf("ifnet_detach: wakeup thread threadinfo: %p "
+			    "input_thread=%p threads: cur=%d max=%d\n",
+			    inputthread, inputthread->input_thread,
+			    dlil_multithreaded_input, cur_dlil_input_threads);
+#endif
+			lck_mtx_lock(inputthread->input_lck);
 
-	lck_mtx_lock(proto_family_mutex);
-	
-	TAILQ_FOREACH(proto_family, &proto_family_head, proto_fam_next) {
-		if (proto_family->proto_family == protocol_family &&
-			proto_family->if_family == interface_family) {
-			lck_mtx_unlock(proto_family_mutex);
-			return EEXIST;
+			inputthread->input_waiting |= DLIL_INPUT_TERMINATE;
+			if ((inputthread->input_waiting & DLIL_INPUT_RUNNING) == 0) {
+				wakeup((caddr_t)&inputthread->input_waiting);
+			}
+			lck_mtx_unlock(inputthread->input_lck);
 		}
 	}
-
-	proto_family = (struct proto_family_str *) _MALLOC(sizeof(struct proto_family_str), M_IFADDR, M_WAITOK);
-	if (!proto_family) {
-		lck_mtx_unlock(proto_family_mutex);
-		return ENOMEM;
-	}
-
-	bzero(proto_family, sizeof(struct proto_family_str));
-	proto_family->proto_family	= protocol_family;
-	proto_family->if_family		= interface_family & 0xffff;
-	proto_family->attach_proto	= attach;
-	proto_family->detach_proto	= detach;
-
-	TAILQ_INSERT_TAIL(&proto_family_head, proto_family, proto_fam_next);
-	lck_mtx_unlock(proto_family_mutex);
-	return 0;
-}
-
-int dlil_dereg_proto_module(u_long protocol_family, u_long interface_family)
-{
-	struct proto_family_str  *proto_family;
-	int ret = 0;
-
-	lck_mtx_lock(proto_family_mutex);
-
-	proto_family = find_proto_module(protocol_family, interface_family);
-	if (proto_family == 0) {
-		lck_mtx_unlock(proto_family_mutex);
-		return ENXIO;
+	dlil_write_end();
+	
+	for (filter = TAILQ_FIRST(&fhead); filter; filter = filter_next) {
+		filter_next = TAILQ_NEXT(filter, filt_next);
+		dlil_detach_filter_internal(filter, 1);
 	}
-
-	TAILQ_REMOVE(&proto_family_head, proto_family, proto_fam_next);
-	FREE(proto_family, M_IFADDR);
 	
-	lck_mtx_unlock(proto_family_mutex);
-	return ret;
-}
-
-int dlil_plumb_protocol(u_long protocol_family, struct ifnet *ifp)
-{
-	struct proto_family_str  *proto_family;
-	int ret = 0;
-
-	lck_mtx_lock(proto_family_mutex);
-	proto_family = find_proto_module(protocol_family, ifp->if_family);
-	if (proto_family == 0) {
-		lck_mtx_unlock(proto_family_mutex);
-		return ENXIO;
+	if (zeroed != 0) {
+		ifp_use_reached_zero(ifp);
 	}
-
-	ret = proto_family->attach_proto(ifp, protocol_family);
-
-	lck_mtx_unlock(proto_family_mutex);
-   	return ret;
-}
-
-
-int dlil_unplumb_protocol(u_long protocol_family, struct ifnet *ifp)
-{
-	struct proto_family_str  *proto_family;
-	int ret = 0;
-
-	lck_mtx_lock(proto_family_mutex);
-
-	proto_family = find_proto_module(protocol_family, ifp->if_family);
-	if (proto_family && proto_family->detach_proto)
-		ret = proto_family->detach_proto(ifp, protocol_family);
-	else
-		ret = dlil_detach_protocol(ifp, protocol_family);
-    
-	lck_mtx_unlock(proto_family_mutex);
-	return ret;
+	
+	return retval;
 }
 
 static errno_t
@@ -2662,6 +2743,7 @@ dlil_recycle_set_bpf_tap(
     return 0;
 }
 
+__private_extern__
 int dlil_if_acquire(
 	u_long family,
 	const void *uniqueid,
@@ -2726,6 +2808,9 @@ int dlil_if_acquire(
     ifp1 = (struct ifnet *)dlifp1;
     ifp1->if_eflags |= IFEF_INUSE;
     ifp1->if_name = dlifp1->if_namestorage;
+#if CONFIG_MACF_NET
+    mac_ifnet_label_init(ifp1);
+#endif
 
     TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link);
      
@@ -2737,10 +2822,11 @@ end:
     return ret;
 }
 
-void dlil_if_release(struct ifnet *ifp)
+__private_extern__ void
+dlil_if_release(
+	ifnet_t	ifp)
 {
     struct dlil_ifnet	*dlifp = (struct dlil_ifnet *)ifp;
-
     
     /* Interface does not have a lock until it is attached - radar 3713951 */
     if (ifp->if_lock)
@@ -2753,6 +2839,15 @@ void dlil_if_release(struct ifnet *ifp)
 
     strncpy(dlifp->if_namestorage, ifp->if_name, IFNAMSIZ);
     ifp->if_name = dlifp->if_namestorage;
+#if CONFIG_MACF_NET
+    /*
+     * We can either recycle the MAC label here or in dlil_if_acquire().
+     * It seems logical to do it here but this means that anything that
+     * still has a handle on ifp will now see it as unlabeled.
+     * Since the interface is "dead" that may be OK.  Revisit later.
+     */
+    mac_ifnet_label_recycle(ifp);
+#endif
     if (ifp->if_lock)
 		ifnet_lock_done(ifp);
     
diff --git a/bsd/net/dlil.h b/bsd/net/dlil.h
index 6824572cb..3f19f7108 100644
--- a/bsd/net/dlil.h
+++ b/bsd/net/dlil.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1999 Apple Computer, Inc. 
@@ -28,35 +34,10 @@
 #ifndef DLIL_H
 #define DLIL_H
 #ifdef KERNEL
+
 #include <sys/kernel_types.h>
 #include <net/kpi_interface.h>
 
-#if __STDC__
-
-struct ifnet;
-struct mbuf;
-struct ether_header;
-struct sockaddr_dl;
-
-#endif
-
-
-#ifdef KERNEL_PRIVATE
-#define DLIL_LAST_FILTER   -1
-#define DLIL_NULL_FILTER   -2
-
-#define DLIL_WAIT_FOR_FREE -2
-
-#define DLIL_BLUEBOX 1
-
-
-
-#include <net/if.h>
-#include <net/if_var.h>
-#include <sys/kern_event.h>
-
-#endif KERNEL_PRIVATE
-
 enum {
 	BPF_TAP_DISABLE,
 	BPF_TAP_INPUT,
@@ -64,76 +45,6 @@ enum {
 	BPF_TAP_INPUT_OUTPUT
 };
 
-#ifdef KERNEL_PRIVATE
-struct kev_msg;
-struct iff_filter;
-
-struct dlil_if_flt_str {
-    caddr_t				   cookie;
-    int	(*filter_if_input)(caddr_t         cookie,
-			   struct ifnet    **ifp,
-			   struct mbuf     **mbuf_ptr,
-			   char		   **frame_ptr);
-
-    int	(*filter_if_event)(caddr_t          cookie,
-			   struct ifnet		*ifp,
-			   struct kev_msg	*event_msg_ptr);
-
-    int	(*filter_if_output)(caddr_t      cookie,
-			    struct ifnet **ifp,
-			    struct mbuf  **mbuf_ptr);
-
-
-    int	(*filter_if_ioctl)(caddr_t       cookie,
-			   struct ifnet  *ifp,
-			   u_long	 ioctl_code_ptr,
-			   caddr_t       ioctl_arg_ptr);
-
-    int	(*filter_if_free)(caddr_t      cookie,
-			  struct ifnet *ifp);
-
-    int	(*filter_detach)(caddr_t  cookie);
-    u_long	reserved[2];
-};
-
-#define DLIL_PR_FILTER  1
-#define DLIL_IF_FILTER  2
-
-
-
-typedef int (*dl_input_func)(struct mbuf *m, char *frame_header,
-			     struct ifnet *ifp, u_long protocol_family, int sync_ok);
-typedef int (*dl_pre_output_func)(struct ifnet		*ifp,
-					u_long protocol_family,
-					struct mbuf		**m,
-					const struct sockaddr	*dest,
-					caddr_t		route_entry,
-					char			*frame_type,
-					char			*dst_addr);
-
-typedef void (*dl_event_func)(struct ifnet *ifp, struct kev_msg *event);
-
-typedef int (*dl_offer_func)(struct mbuf *m, char *frame_header);
-typedef int (*dl_ioctl_func)(u_long	protocol_family,
-			     struct ifnet *ifp,
-			     u_long	ioctl_cmd,
-			     caddr_t	ioctl_arg);
-typedef int (*dl_detached_func)(u_long	protocol_family, struct ifnet *ifp);
-
-/* Obsolete types */
-#define DLIL_DESC_RAW		1
-#define DLIL_DESC_802_2		2
-#define DLIL_DESC_802_2_SNAP	3
-/*
- * DLIL_DESC_RAW - obsolete type, data in variants.bitmask or native_type
- *				   if variants.bitmask.proto_id_length, native_type in host
- *				   byte order.
- * DLIL_DESC_802_2 - obsolete, data in variants.desc_802_2
- * DLIL_DESC_802_2_SNAP - obsolete, data in variants.desc_802_2_SNAP
- *						  protocol field in host byte order
- */
-#endif KERNEL_PRIVATE
-
 /* Ethernet specific types */
 #define DLIL_DESC_ETYPE2	4
 #define DLIL_DESC_SAP		5
@@ -155,102 +66,58 @@ typedef int (*dl_detached_func)(u_long	protocol_family, struct ifnet *ifp);
  */
 
 #ifdef KERNEL_PRIVATE
-struct dlil_demux_desc {
-    TAILQ_ENTRY(dlil_demux_desc) next;
-    
-    int		type;
-    u_char	*native_type;
-    
-    union {
-        /* Structs in this union are obsolete. They exist for binary compatability only */
-        /* Only the native_type_length is used */
-        struct {
-            u_long   proto_id_length; /* IN LONGWORDS!!! */
-            u_char   *proto_id;		  /* No longer supported by Ethernet family */
-            u_char   *proto_id_mask;
-        } bitmask;
-        
-        struct {
-            u_char   dsap;
-            u_char   ssap;
-            u_char   control_code;
-            u_char   pad;
-        } desc_802_2;
-        
-        struct {
-            u_char   dsap;			/* Ignored, assumed to be 0xAA */
-            u_char   ssap;			/* Ignored, assumed to be 0xAA */
-            u_char   control_code; 	/* Ignored, assumed to be 0x03 */
-            u_char   org[3];
-            u_short  protocol_type; /* In host byte order */
-        } desc_802_2_SNAP;
-        
-        /* Length of data pointed to by native_type, must be set correctly */
-        u_int32_t	native_type_length;
-    } variants;
-};
 
-TAILQ_HEAD(ddesc_head_str, dlil_demux_desc);
+#include <net/if.h>
+#include <net/if_var.h>
+#include <sys/kern_event.h>
+#include <kern/thread.h>
+#include <kern/locks.h>
 
-struct dlil_proto_reg_str {
-    struct ddesc_head_str	demux_desc_head;
-    u_long			interface_family;
-    u_long			protocol_family;
-    short			unit_number;
-    int				default_proto; /* 0 or 1 */
-    dl_input_func		input;
-    dl_pre_output_func	pre_output;
-    dl_event_func		event;
-    dl_offer_func		offer;
-    dl_ioctl_func		ioctl;
-    dl_detached_func	detached;
-    u_long			reserved[3];
-};
+#if __STDC__
 
+struct ifnet;
+struct mbuf;
+struct ether_header;
+struct sockaddr_dl;
 
-int dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter,
-					   interface_filter_t *filter_ref);
+#endif
 
+#ifdef BSD_KERNEL_PRIVATE
 struct ifnet_stat_increment_param;
+struct iff_filter;
 
-int
-dlil_input_with_stats(struct ifnet  *ifp, struct mbuf *m_head, struct mbuf *m_tail,
-					  const struct ifnet_stat_increment_param *stats);
-
-int
-dlil_input(struct ifnet  *ifp, struct mbuf *m_head, struct mbuf *m_tail);
-
-int
-dlil_output_list(
-		struct ifnet *ifp,
-		u_long protocol_family,
-		struct mbuf	*packetlist,
-		caddr_t		route,
-		const struct sockaddr *dest,
-		int		raw);
-
-int
-dlil_output(
-		struct ifnet *ifp,
-		u_long protocol_family,
-	    struct mbuf		*m,
-	    caddr_t		route,
-	    const struct sockaddr     *dest,
-	    int			raw);
+struct dlil_threading_info {
+	mbuf_t 		mbuf_head;	/* start of mbuf list from if */
+	mbuf_t 		mbuf_tail;
+	u_int32_t 	mbuf_count;
+	boolean_t	net_affinity;	/* affinity set is available */
+	u_int32_t 	input_waiting;	/* DLIL condition of thread */
+	struct thread	*input_thread;	/* thread data for this input */
+	struct thread	*workloop_thread; /* current workloop thread */
+	u_int32_t	tag;		/* current affinity tag */
+	lck_mtx_t	*input_lck;	
+	lck_grp_t	*lck_grp;	/* lock group (for lock stats) */
+	char 		input_name[32];		 
+#if IFNET_INPUT_SANITY_CHK
+	u_int32_t	input_wake_cnt;	/* number of times the thread was awaken with packets to process */
+	u_long		input_mbuf_cnt;	/* total number of mbuf packets processed by this thread */
+#endif
+};
 
+/*
+	The following are shared with kpi_protocol.c so that it may wakeup
+	the input thread to run through packets queued for protocol input.
+*/
+#define	DLIL_INPUT_RUNNING	0x80000000
+#define	DLIL_INPUT_WAITING	0x40000000
+#define	DLIL_PROTO_REGISTER	0x20000000
+#define	DLIL_PROTO_WAITING	0x10000000
+#define	DLIL_INPUT_TERMINATE	0x08000000
 
-int
-dlil_ioctl(u_long	proto_family,
-	   struct ifnet	*ifp,
-	   u_long	ioctl_code,
-	   caddr_t	ioctl_arg);
+void dlil_init(void);
 
-errno_t
-dlil_resolve_multi(
-	struct ifnet *ifp,
-	const struct sockaddr *proto_addr,
-	struct sockaddr *ll_addr,
-	size_t ll_len);
+errno_t dlil_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode,
+						 bpf_packet_func callback);
 
 /*
  * Send arp internal bypasses the check for
@@ -265,6 +132,22 @@ dlil_send_arp_internal(
 	const struct sockaddr_dl* target_hw,
 	const struct sockaddr* target_proto);
 
+int
+dlil_output(
+	ifnet_t					ifp,
+	protocol_family_t		proto_family,
+	mbuf_t					packetlist,
+	void					*route,
+	const struct sockaddr	*dest,
+	int						raw);
+
+errno_t
+dlil_resolve_multi(
+	struct ifnet *ifp,
+	const struct sockaddr *proto_addr,
+	struct sockaddr *ll_addr,
+	size_t ll_len);
+
 errno_t
 dlil_send_arp(
 	ifnet_t	ifp,
@@ -274,334 +157,29 @@ dlil_send_arp(
 	const struct sockaddr_dl* target_hw,
 	const struct sockaddr* target_proto);
 
-int
-dlil_ioctl_locked(u_long	proto_family,
-	   struct ifnet	*ifp,
-	   u_long	ioctl_code,
-	   caddr_t	ioctl_arg);
-
-int
-dlil_attach_protocol(struct dlil_proto_reg_str   *proto);
-
-int
-dlil_detach_protocol(struct ifnet *ifp, u_long protocol_family);
-
-int
-dlil_if_attach(struct ifnet	*ifp);
-
-#ifdef BSD_KERNEL_PRIVATE
-
-int
-dlil_if_attach_with_address(
-	struct ifnet		*ifp,
-	const struct sockaddr_dl	*ll_addr);
-
-int
-dlil_attach_protocol_kpi(ifnet_t ifp, protocol_family_t protocol,
-	const struct ifnet_attach_proto_param *proto_details);
-
-errno_t dlil_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode,
-						 bpf_packet_func callback);
-
-#endif
-
-void
-dlil_detach_filter(interface_filter_t filter);
-
-struct dlil_ifmod_reg_str {
-    int (*add_if)(struct ifnet *ifp);
-    int (*del_if)(struct ifnet *ifp);
-	int	(*add_proto)(struct ifnet *ifp, u_long protocol_family,
-		struct ddesc_head_str *demux_desc_head);
-#ifdef __KPI_INTERFACE__
-    ifnet_del_proto_func	del_proto;
-    ifnet_ioctl_func		ifmod_ioctl;
-#else
-	void*					del_proto;
-	void*					ifmod_ioctl;
-#endif
-    int	(*shutdown)(void);
-    int (*init_if)(struct ifnet *ifp);
-    u_long	reserved[3];
-};
-
-
-int dlil_reg_if_modules(u_long  interface_family,
-			struct dlil_ifmod_reg_str  *ifmod_reg);
-
-/* 
-
-Function : dlil_reg_proto_module
-
-    A DLIL protocol module is a piece of code that know how to handle a certain type
-    of protocol (PF_INET, PF_INET6, ...) for a certain family of interface (APPLE_IF_FAM_ETHERNET, 
-    APPLE_IF_FAM_PPP, ...).
-    
-    dlil_reg_proto_module() allows the registration of such a protocol/interface handler before any 
-    interface is attached.
-    Typically, the attach and detach function of the protocol handler will call 
-    dlil_{attach/detach}_protocol with the parameter specific to the protocol.
-    
-    The goal of this modules is to insulate the actual protocol (IP, IPv6) from the DLIL details.
-
-Parameters :
-    'protocol_family' is PF_INET, PF_INET6, ...
-    'interface_family' is APPLE_IF_FAM_ETHERNET, APPLE_IF_FAM_PPP, ...
-    'protomod_reg' is the protocol registration structure.
-            'attach_proto' funtion is mandatory.
-            'detach_proto' funtion is optional (DLIL will manage it).
-
-Return code :    
-
-0 :
-
-    No error.
-
-ENOMEM:
-
-    No memory can be allocated for internal data structure.
-
-EEXIST:
-
-    The protocol family has already been registered for this interface family.
-
-EINVAL:
-
-    The dlil_protomod_reg_str structure contains incorrect values.
-
-*/
-
-int dlil_reg_proto_module(u_long protocol_family, u_long interface_family,
-			int (*attach)(struct ifnet *ifp, u_long protocol_family),
-			int (*detach)(struct ifnet *ifp, u_long protocol_family));
-
-/* 
-
-Function : dlil_dereg_proto_module
-
-    dlil_dereg_proto_module() will unregister the protocol module previously 
-    registered with dlil_dereg_proto_module().
-    
-    There is no restriction when to call it. 
-    Interfaces or protoco can be attached, it will not prevent the deregistration of the module.
-    
-Parameters :
-    'protocol_family' is PF_INET, PF_INET6, ...
-    'interface_family' is APPLE_IF_FAM_ETHERNET, APPLE_IF_FAM_PPP, ...
-
-Return code :    
-
-0 :
-
-    No error.
-
-ENOENT:
-
-    No module was registered..
-
-*/
-
-int dlil_dereg_proto_module(u_long protocol_family, u_long interface_family);
-
-/* 
-
-Function : dlil_plumb_protocol
-
-    dlil_plumb_protocol() will plumb a protocol to an actual interface.
-    This will find a registered protocol module and call its attach function.
-    The module will typically call dlil_attach_protocol with the appropriate parameters.
-            
-Parameters :
-    'protocol_family' is PF_INET, PF_INET6, ...
-    'ifp' is the interface to plumb the protocol to.
-    
-Return code :    
-
-0 :
-
-    No error.
-
-ENOENT:
-
-    No module was registered.
-
-other: 
-    
-    Error returned by the attach_proto function
-
-*/
-int dlil_plumb_protocol(u_long protocol_family, struct ifnet *ifp);
-
-/* 
-
-Function : dlil_unplumb_protocol
-
-    dlil_unplumb_protocol() will unplumb a protocol from an interface.
-    This will find a registered protocol module and call its detach function.
-    The module will typically call dlil_detach_protocol with the appropriate parameters.
-    If no module is found, this function will call dlil_detach_protocol directly. 
-    
-Parameters :
-    'protocol_family' is PF_INET, PF_INET6, ...
-    'ifp' is APPLE_IF_FAM_ETHERNET, APPLE_IF_FAM_PPP, ...
-    
-Return code :    
-
-0 :
-
-    No error.
-
-ENOENT:
-
-    No module was registered.
-
-other: 
-    
-    Error returned by the attach_proto function
-
-*/
-int dlil_unplumb_protocol(u_long protocol_family, struct ifnet *ifp);
-
-int 
-dlil_inject_if_input(struct mbuf *m, char *frame_header, u_long from_id);
-
-int
-dlil_inject_pr_input(struct mbuf *m, char *frame_header, u_long from_id);
-
-int
-dlil_inject_pr_output(struct mbuf		*m,
-		      struct sockaddr		*dest,
-		      int			raw, 
-		      char			*frame_type,
-		      char			*dst_linkaddr,
-		      u_long			from_id);
+int dlil_attach_filter(ifnet_t ifp, const struct iff_filter *if_filter,
+					   interface_filter_t *filter_ref);
+void dlil_detach_filter(interface_filter_t filter);
+int dlil_detach_protocol(ifnet_t ifp, u_long protocol);
 
-int
-dlil_inject_if_output(struct mbuf *m, u_long from_id);
+#endif /* BSD_KERNEL_PRIVATE */
 
-#ifdef KERNEL_PRIVATE
 void
 dlil_post_msg(struct ifnet *ifp,u_long event_subclass, u_long event_code, 
 		   struct net_event_data *event_data, u_long event_data_len);
-#endif
-
-int
-dlil_event(struct ifnet *ifp, struct kern_event_msg *event);
-
-int dlil_dereg_if_modules(u_long interface_family);
-
-int
-dlil_if_detach(struct ifnet *ifp);
-
-void
-ifp_reference(struct ifnet *ifp);
-
-void
-ifp_release(struct ifnet *ifp);
-
 
 /* 
-
-Function : dlil_if_acquire
-
-    DLIL manages the list of ifnet interfaces allocated using the dlil_if_acquire
-    function. This list if not the same as the list of attached interfaces, 
-    visible with ifconfig.
-    This list contains attached as well as detached interfaces.
-	Detached interfaces are kept in the list to prevent the kernel from crashing
-	by using an old ifp.
-
-    if it succeeds, dlil_if_acquire returns an ifnet data structure.
-    This ifnet can either be a new allocated block of memory, or an ifnet
-    that already existed and that DLIL has found in its list of unused
-    interface and that matches the family/uniqueid tuple.
-
-    dlil_if_acquire can fail if the requested interface is already in use, 
-    or if no memory is available to create a new interface.
-
-    The typical sequence of call for a driver will be :
-    dlil_if_acquire(... &ifp)
-    ... Fill in the ifnet ...
-    dlil_if_attach(ifp)
-    ... Driver work ...
-    dlil_if_detach(ifp)
-    dlil_if_release(ifp)
-
-    Important : ifnet allocated by DLIL are managed by DLIL. DLIL takes care
-    of them, and keeps them until a driver wants to reuse them, but DLIL may
-    also decide to free them when not in use by a driver.
-
-    Note : the structure returned will actually be large enough to contain
-    an arpcom structure (ifnet + ethernet) structure.
-    Drivers cannot extend the structure and must to store their private 
-    information in if_sofc and if_private.
-
-Parameters :
-    'family' uniquely identifies DLIL interface family.
-    'uniqueid' is a unique identifier for that interface, managed by the
-        driver (for example MAC address for ethernet).
-    'uniqueid_len' is the length of the unique id.
-    'ifp' contains on output the allocated ifnet.
-
-Return code :    
-
-0 :
-
-    If an ifnet matching the uniqueid is found, the matching ifnet is returned
-    in ifp and the flags IFEF_REUSE and IF_INUSE are set in the if_eflags.
-    The fields in the ifnet are NOT zeroed and may contain old values that
-    the driver can reuse. [They are not necessarily the values that were
-    there when the driver released the ifnet, as protocol might have
-    continued to update them].
-
-    If no matching ifnet is found, a new structure is allocated and returned
-    in ifp with all fields initialized to 0.
-    The flag IF_INUSE is set in the if_eflags. IFEF_REUSE is NOT set.
-    dlil_if_acquire will copy the uniqueid and keep it for matching purpose.
-
-    If 'uniqueid' is NULL, then dlil_if_acquire will return the first
-    ifnet that contains a null uniqueid for that family, with the flags
-    IFEF_REUSE and IF_INUSE set.
-    If no ifnet is available, a new one will be created.
-
-ENOMEM:
-
-    If no matching interface is found, and no memory can be allocated,
-    dlil_if_acquire will return ENOMEM.
-
-
-EBUSY:
-
-    If the unique id matches the id of an interface currently in use,
-    dlil_if_acquire will return EBUSY.
-    An interface 'in use' is an allocated interface, not necessarily attached.
-
-*/
+ * dlil_if_acquire is obsolete. Use ifnet_allocate.
+ */
 
 int dlil_if_acquire(u_long family, const void *uniqueid, size_t uniqueid_len, 
 			struct ifnet **ifp);
 			
 
 /* 
-
-Function : dlil_if_release
-
-	dlil_if_release will transfer control of the ifnet to DLIL.
-	DLIL will keep the interface in its list, marking it unused.
-	The fields will be left in their current state, so the driver can reuse
-	the ifnet later, by calling dlil_if_acquire.
-	The if_eflags IF_INUSE will be cleared.
-	The fields if_output, if_ioctl, if_free and if_set_bpf_tap will be changed 
-	to point to DLIL private functions.
-	After calling dlil_if_release, the driver can safely terminate and
-	unload if necessary.
-	Note: your driver should only call dlil_if_release once your if_free
-	function has been called.
-
-Parameters :
-	ifp is the pointer to the ifnet to release.
-
-*/
+ * dlil_if_release is obsolete. The equivalent is called automatically when
+ * an interface is detached.
+ */
 
 void dlil_if_release(struct ifnet *ifp);
 
diff --git a/bsd/net/dlil_pvt.h b/bsd/net/dlil_pvt.h
index af688c107..05cae784f 100644
--- a/bsd/net/dlil_pvt.h
+++ b/bsd/net/dlil_pvt.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef DLIL_PVT_H
 #define DLIL_PVT_H
diff --git a/bsd/net/ether_at_pr_module.c b/bsd/net/ether_at_pr_module.c
index 1bf55ad72..9ae109b85 100644
--- a/bsd/net/ether_at_pr_module.c
+++ b/bsd/net/ether_at_pr_module.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -70,6 +76,8 @@
 #include <net/if_dl.h>
 #include <net/if_types.h>
 #include <netinet/if_ether.h>
+#include <net/kpi_interface.h>
+#include <net/kpi_protocol.h>
 
 #include <sys/socketvar.h>
 
@@ -89,129 +97,100 @@ extern struct ifqueue atalkintrq;
 #include <net/if_vlan_var.h>
 #endif /* NVLAN > 0 */
 
-struct dl_es_at_entry 
-{
-	struct ifnet *ifp;
-	int    ref_count;
-};
-
-/* Local fuction declerations */
-int at_ether_input(struct mbuf *m, char *frame_header, struct ifnet *ifp,
-				   u_long protocol_family, int sync_ok);
-int ether_pre_output(struct ifnet *ifp, u_long protocol_family, struct mbuf **m0,
-					 const struct sockaddr *dst_netaddr, caddr_t route, char *type, char *edst);
-int ether_prmod_ioctl(u_long protocol_family, struct ifnet *ifp, u_long command,
-					  caddr_t data);
-int ether_attach_at(struct ifnet *ifp);
-void ether_detach_at(struct ifnet *ifp);
-
-
-/*
- * Temp static for protocol registration XXX
- */
-
-#define MAX_EN_COUNT 30
-
-static struct dl_es_at_entry en_at_array[MAX_EN_COUNT];
+#include <net/ether_if_module.h>
 
 /*
  * Process a received Ethernet packet;
  * the packet is in the mbuf chain m without
  * the ether header, which is provided separately.
  */
-int
-at_ether_input(
-	struct mbuf				*m,
-	__unused char			*frame_header,
-	__unused struct ifnet	*ifp,
-	__unused u_long			protocol_family,
-	__unused int			sync_ok)
-
+static errno_t
+ether_at_input(
+	__unused ifnet_t			ifp,
+	__unused protocol_family_t	protocol_family,
+	mbuf_t				m,
+	__unused char			*frame_header)
 {
-   /*
-	* note: for AppleTalk we need to pass the enet header of the
-	* packet up stack. To do so, we made sure in that the FULL packet
-	* is copied in the mbuf by the mace driver, and only the m_data and
-	* length have been shifted to make IP and the other guys happy.
-	*/
+	errno_t error;
+	/*
+	 * note: for AppleTalk we need to pass the enet header of the
+	 * packet up stack. To do so, we made sure in that the FULL packet
+	 * is copied in the mbuf by the driver, and only the m_data and
+	 * length have been shifted to make IP and the other guys happy.
+	 */
 
 	m->m_data -= sizeof(struct ether_header);
 	m->m_len += sizeof(struct ether_header);
 	m->m_pkthdr.len += sizeof(struct ether_header);
-	proto_input(PF_APPLETALK, m);
 
-	return 0;
+	error = proto_input(PF_APPLETALK, m);
+	
+	if (error)
+		m_freem(m);
+
+	return error;
 }
 
 
 
-int
-ether_pre_output(
-	struct ifnet			*ifp,
-	__unused u_long			protocol_family,
-	struct mbuf				**m0,
-	const struct sockaddr	*dst_netaddr,
-	__unused caddr_t		route,
-	char					*type,
-	char					*edst)
+static errno_t
+ether_at_pre_output(
+	ifnet_t						ifp,
+	__unused protocol_family_t	protocol_family,
+	mbuf_t						*m0,
+	const struct sockaddr		*dst_netaddr,
+	__unused void				*route,
+	char						*type,
+	char						*edst)
 {
-    register struct mbuf *m = *m0;
-    register struct ether_header *eh;
-    int hlen;	/* link layer header lenght */
+    struct mbuf *m = *m0;
+    const struct ether_header *eh;
+    int hlen;	/* link layer header length */
 
+	if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) 
+		return ENETDOWN;
 
+	hlen = ETHER_HDR_LEN;
 
-    if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING)) 
-	return ENETDOWN;
-
-    hlen = ETHER_HDR_LEN;
-
-    /*
-     * Tell ether_frameout it's ok to loop packet unless negated below.
-     */
-    m->m_flags |= M_LOOP;
-
-    switch (dst_netaddr->sa_family) {
-    case AF_UNSPEC:
-	m->m_flags &= ~M_LOOP;
-	eh = (struct ether_header *)dst_netaddr->sa_data;
-	(void)memcpy(edst, eh->ether_dhost, 6);
-	*(u_short *)type = eh->ether_type;
-	break;
+	/*
+	 * Tell ether_frameout it's ok to loop packet unless negated below.
+	 */
+	m->m_flags |= M_LOOP;
+
+	switch (dst_netaddr->sa_family) {
+		case AF_UNSPEC:
+			m->m_flags &= ~M_LOOP;
+			eh = (const struct ether_header *)dst_netaddr->sa_data;
+			(void)memcpy(edst, eh->ether_dhost, 6);
+			*(u_short *)type = eh->ether_type;
+			break;
 	
-
-    case AF_APPLETALK:
-    {
-	eh = (struct ether_header *)dst_netaddr->sa_data;
-	bcopy((caddr_t)eh->ether_dhost, (caddr_t)edst, 6);
+		case AF_APPLETALK:
+			eh = (const struct ether_header *)dst_netaddr->sa_data;
+			(void)memcpy(edst, eh->ether_dhost, 6);
+			*(u_short *)type = htons(m->m_pkthdr.len);
+			break;
 		
-	*(u_short *)type = htons(m->m_pkthdr.len);
-    }
-    break;
-
-
-    default:
-	kprintf("%s%d: can't handle af%d\n", ifp->if_name, ifp->if_unit,
-	       dst_netaddr->sa_family);
-
-        return EAFNOSUPPORT;
-    }
-
-    return (0);
+		default:
+			printf("%s%d: can't handle af%d\n", ifp->if_name, ifp->if_unit,
+				   dst_netaddr->sa_family);
+			return EAFNOSUPPORT;
+	}
+	
+	return (0);
 }
 
 
 
 
-
-int
-ether_prmod_ioctl(
-    __unused u_long	protocol_family,
-    struct ifnet	*ifp,
-    u_long			command,
-    caddr_t			data)
+static errno_t
+ether_at_prmod_ioctl(
+    ifnet_t						ifp,
+    __unused protocol_family_t	protocol_family,
+    u_int32_t					command,
+    void						*data)
 {
-    struct ifreq *ifr = (struct ifreq *) data;
+    struct ifreq *ifr = data;
     int error = 0;
 
     switch (command) {
@@ -219,116 +198,62 @@ ether_prmod_ioctl(
     case SIOCSIFADDR:
 	 if ((ifp->if_flags & IFF_RUNNING) == 0) {
 	      ifnet_set_flags(ifp, IFF_UP, IFF_UP);
-	      dlil_ioctl(0, ifp, SIOCSIFFLAGS, (caddr_t) 0);
+	      ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
 	 }
 
 	break;
 
     case SIOCGIFADDR:
 	ifnet_lladdr_copy_bytes(ifp, ifr->ifr_addr.sa_data, ETHER_ADDR_LEN);
-    break;
-
-    case SIOCSIFMTU:
-	/*
-	 * Set the interface MTU.
-	 */
-	if (ifr->ifr_mtu > ETHERMTU) {
-	    error = EINVAL;
-	} else {
-	    ifp->if_mtu = ifr->ifr_mtu;
-	}
 	break;
 
     default:
-	 return EOPNOTSUPP;
+	error = EOPNOTSUPP;
+	break;
     }
-
-
     return (error);
 }
 
 
 
-int
+__private_extern__ errno_t
 ether_attach_at(
-	struct ifnet *ifp)
+	ifnet_t	ifp,
+	__unused protocol_family_t proto_family)
 {
-    struct dlil_proto_reg_str   reg;
-    struct dlil_demux_desc      desc;
-    struct dlil_demux_desc      desc2;
-    int   stat;
-    int   first_empty;
-    int   i;
+	struct ifnet_attach_proto_param proto;
+	struct ifnet_demux_desc demux[2];
     u_int8_t	atalk_snap[5] = {0x08, 0x00, 0x07, 0x80, 0x9b};
     u_int8_t	aarp_snap[5] = {0x00, 0x00, 0x00, 0x80, 0xf3};
-
-    first_empty = MAX_EN_COUNT;
-	for (i=0; i < MAX_EN_COUNT; i++) {
-		if (en_at_array[i].ifp == 0)
-			first_empty = i;
-		
-		if (en_at_array[i].ifp == ifp) {
-			en_at_array[i].ref_count++;
-			return 0;
-		}
-    }
-    
-	if (first_empty == MAX_EN_COUNT)
-		return ENOMEM;
-	
-	bzero(&reg, sizeof(reg));
-	bzero(&desc, sizeof(desc));
-	bzero(&desc2, sizeof(desc2));
-	
-	TAILQ_INIT(&reg.demux_desc_head);
-	reg.interface_family = ifp->if_family;
-	reg.unit_number      = ifp->if_unit;
-	reg.input            = at_ether_input;
-	reg.pre_output       = ether_pre_output;
-	reg.ioctl            = ether_prmod_ioctl;
-	reg.protocol_family  = PF_APPLETALK;
-
-	desc.type = DLIL_DESC_SNAP;
-	desc.native_type = atalk_snap;
-	desc.variants.native_type_length = sizeof(atalk_snap);
-	TAILQ_INSERT_TAIL(&reg.demux_desc_head, &desc, next);
-	
-	desc2.type = DLIL_DESC_SNAP;
-	desc2.native_type = aarp_snap;
-	desc2.variants.native_type_length = sizeof(aarp_snap);
-	TAILQ_INSERT_TAIL(&reg.demux_desc_head, &desc2, next);
+	int 		error;
+
+	bzero(demux, sizeof(demux));
+	demux[0].type = DLIL_DESC_SNAP;
+	demux[0].data = atalk_snap;
+	demux[0].datalen = sizeof(atalk_snap);
+	demux[1].type = DLIL_DESC_SNAP;
+	demux[1].data = aarp_snap;
+	demux[1].datalen = sizeof(aarp_snap);
+
+	bzero(&proto, sizeof(proto));
+	proto.demux_list	= demux;
+	proto.demux_count	= sizeof(demux) / sizeof(demux[0]);
+	proto.input			= ether_at_input;
+	proto.pre_output	= ether_at_pre_output;
+	proto.ioctl			= ether_at_prmod_ioctl;
 	
-    stat = dlil_attach_protocol(&reg);
-	if (stat) {
-		printf("WARNING: ether_attach_at can't attach at to interface\n");
-		return stat;
+	error = ifnet_attach_protocol(ifp, PF_APPLETALK, &proto);
+	if (error && error != EEXIST) {
+		printf("WARNING: ether_attach_at failed to attach"
+		       " AppleTalk to %s%d\n", ifp->if_name, ifp->if_unit);
 	}
+	return (error);
+}
 
-	en_at_array[first_empty].ifp = ifp;
-	en_at_array[first_empty].ref_count = 1;
-	
-	return 0;
-} /* ether_attach_at */
-
-
-void
-ether_detach_at(struct ifnet *ifp)
+__private_extern__ void
+ether_detach_at(
+	ifnet_t ifp,
+	__unused protocol_family_t proto_family)
 {
-	int i;
-	
-	for (i=0; i < MAX_EN_COUNT; i++) {
-		if (en_at_array[i].ifp == ifp)
-			break;
-	}
-	
-	if (i < MAX_EN_COUNT) {
-		if (en_at_array[i].ref_count > 1) 
-			en_at_array[i].ref_count--;
-		else {
-			if (en_at_array[i].ref_count == 1) {
-				dlil_detach_protocol(ifp, PF_APPLETALK);
-				en_at_array[i].ifp = 0;
-			}
-		}
-	}
+	(void)ifnet_detach_protocol(ifp, PF_APPLETALK);
 }
diff --git a/bsd/net/ether_if_module.c b/bsd/net/ether_if_module.c
index 0e50b415b..4673b53a3 100644
--- a/bsd/net/ether_if_module.c
+++ b/bsd/net/ether_if_module.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -64,6 +70,8 @@
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 
+#include <pexpert/pexpert.h>
+
 #include <net/if.h>
 #include <net/route.h>
 #include <net/if_llc.h>
@@ -72,6 +80,8 @@
 #include <net/if_ether.h>
 #include <netinet/if_ether.h>
 #include <netinet/in.h>	/* For M_LOOP */
+#include <net/kpi_interface.h>
+#include <net/kpi_protocol.h>
 
 /*
 #if INET
@@ -82,7 +92,7 @@
 #include <netinet/ip.h>
 #endif
 */
-
+#include <net/ether_if_module.h>
 #include <sys/socketvar.h>
 #include <net/if_vlan_var.h>
 #include <net/if_bond_var.h>
@@ -109,15 +119,20 @@ extern struct ifqueue atalkintrq;
 
 
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_ETHER, ether, CTLFLAG_RW, 0, "Ethernet");
+SYSCTL_NODE(_net_link, IFT_ETHER, ether, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Ethernet");
 
 struct en_desc {
 	u_int16_t	type;			/* Type of protocol stored in data */
 	u_long 		protocol_family;	/* Protocol family */
 	u_long		data[2];		/* Protocol data */
 };
+
 /* descriptors are allocated in blocks of ETHER_DESC_BLK_SIZE */
-#define ETHER_DESC_BLK_SIZE (10) 
+#if CONFIG_EMBEDDED
+#define ETHER_DESC_BLK_SIZE (2) /* IP, ARP */
+#else
+#define ETHER_DESC_BLK_SIZE (10)
+#endif
 
 /*
  * Header for the demux list, hangs off of IFP at family_cookie
@@ -130,15 +145,32 @@ struct ether_desc_blk_str {
 	struct en_desc  block_ptr[1];
 };
 /* Size of the above struct before the array of struct en_desc */
-#define ETHER_DESC_HEADER_SIZE	((size_t)&(((struct ether_desc_blk_str*)0)->block_ptr[0]))
+#define ETHER_DESC_HEADER_SIZE	((size_t)offsetof(struct ether_desc_blk_str, block_ptr))
 __private_extern__ u_char	etherbroadcastaddr[ETHER_ADDR_LEN] =
 								{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
-int ether_add_proto_old(struct ifnet *ifp, u_long protocol_family, struct ddesc_head_str *desc_head);
-int ether_add_if(struct ifnet *ifp);
-int ether_del_if(struct ifnet *ifp);
-int ether_init_if(struct ifnet *ifp);
-int ether_family_init(void);
+#if defined (__arm__)
+static __inline__ int
+_ether_cmp(const void * a, const void * b)
+{
+	return (memcmp(a, b, ETHER_ADDR_LEN));
+}
+
+#else
+static __inline__ int
+_ether_cmp(const void * a, const void * b)
+{
+	const u_int16_t * a_s = (const u_int16_t *)a;
+	const u_int16_t * b_s = (const u_int16_t *)b;
+	
+	if (a_s[0] != b_s[0]
+	    || a_s[1] != b_s[1]
+	    || a_s[2] != b_s[2]) {
+		return (1);
+	}
+	return (0);
+}
+#endif
 
 /*
  * Release all descriptor entries owned by this protocol (there may be several).
@@ -263,8 +295,8 @@ ether_add_proto_internal(
 			 */
 			return ENOMEM;
 		}
-		
-		bzero(tmp + old_size, new_size - old_size);
+
+		bzero(((char *)tmp) + old_size, new_size - old_size);
 		if (desc_blk) {
 			bcopy(desc_blk, tmp, old_size);
 			FREE(desc_blk, M_IFADDR);
@@ -339,55 +371,6 @@ ether_add_proto(
 	return error;
 }
 
-__private_extern__ int
-ether_add_proto_old(
-	struct ifnet *ifp,
-	u_long protocol_family,
-	struct ddesc_head_str *desc_head)
-{
-	struct dlil_demux_desc  *desc;
-	int error = 0;
-	
-	TAILQ_FOREACH(desc, desc_head, next) {
-		struct ifnet_demux_desc dmx;
-		int swapped = 0;
-		
-		// Convert dlil_demux_desc to ifnet_demux_desc
-		dmx.type = desc->type;
-		dmx.datalen = desc->variants.native_type_length;
-		dmx.data = desc->native_type;
-		
-#ifdef DLIL_DESC_RAW
-		if (dmx.type == DLIL_DESC_RAW) {
-			swapped = 1;
-			dmx.type = DLIL_DESC_ETYPE2;
-			dmx.datalen = 2;
-			*(u_int16_t*)dmx.data = htons(*(u_int16_t*)dmx.data);
-		}
-#endif
-		
-		error = ether_add_proto_internal(ifp, protocol_family, &dmx);
-		if (swapped) {
-			*(u_int16_t*)dmx.data = ntohs(*(u_int16_t*)dmx.data);
-			swapped = 0;
-		}
-		if (error) {
-			ether_del_proto(ifp, protocol_family);
-			break;
-		}
-	}
-	
-	return error;
-} 
-
-
-static int
-ether_shutdown(void)
-{
-    return 0;
-}
-
-
 int
 ether_demux(
 	ifnet_t				ifp,
@@ -408,8 +391,7 @@ ether_demux(
 
 	if (eh->ether_dhost[0] & 1) {
 		/* Check for broadcast */
-		if (*(u_int32_t*)eh->ether_dhost == 0xFFFFFFFF &&
-			*(u_int16_t*)(eh->ether_dhost + sizeof(u_int32_t)) == 0xFFFF)
+		if (_ether_cmp(etherbroadcastaddr, eh->ether_dhost) == 0)
 			m->m_flags |= M_BCAST;
 		else
 			m->m_flags |= M_MCAST;
@@ -423,16 +405,12 @@ ether_demux(
 
 	if ((eh->ether_dhost[0] & 1) == 0) {
 		/*
- 		* When the driver is put into promiscuous mode we may receive unicast
-		* frames that are not intended for our interfaces.  They are marked here
-		* as being promiscuous so the caller may dispose of them after passing
-		* the packets to any interface filters.
-		*/
-		#define ETHER_CMP(x, y) ( ((u_int16_t *) x)[0] != ((u_int16_t *) y)[0] || \
-								  ((u_int16_t *) x)[1] != ((u_int16_t *) y)[1] || \
-								  ((u_int16_t *) x)[2] != ((u_int16_t *) y)[2] )
-		
-		if (ETHER_CMP(eh->ether_dhost, ifnet_lladdr(ifp))) {
+		 * When the driver is put into promiscuous mode we may receive unicast
+		 * frames that are not intended for our interfaces.  They are marked here
+		 * as being promiscuous so the caller may dispose of them after passing
+		 * the packets to any interface filters.
+		 */
+		if (_ether_cmp(eh->ether_dhost, ifnet_lladdr(ifp))) {
 			m->m_flags |= M_PROMISC;
 		}
 	}
@@ -539,11 +517,11 @@ ether_frameout(
             if ((*m)->m_flags & M_BCAST) {
                 struct mbuf *n = m_copy(*m, 0, (int)M_COPYALL);
                 if (n != NULL)
-                    dlil_output(lo_ifp, ndest->sa_family, n, 0, ndest, 0);
+                    dlil_output(lo_ifp, ndest->sa_family, n, NULL, ndest, 0);
             }
             else {
-                if (bcmp(edst, ifnet_lladdr(ifp), ETHER_ADDR_LEN) == 0) {
-                    dlil_output(lo_ifp, ndest->sa_family, *m, 0, ndest, 0);
+					if (_ether_cmp(edst, ifnet_lladdr(ifp)) == 0) {
+                    dlil_output(lo_ifp, ndest->sa_family, *m, NULL, ndest, 0);
                     return EJUSTRETURN;
                 }
             }
@@ -563,53 +541,15 @@ ether_frameout(
 	eh = mtod(*m, struct ether_header *);
 	(void)memcpy(&eh->ether_type, ether_type,
 		sizeof(eh->ether_type));
- 	(void)memcpy(eh->ether_dhost, edst, 6);
+ 	(void)memcpy(eh->ether_dhost, edst, ETHER_ADDR_LEN);
  	ifnet_lladdr_copy_bytes(ifp, eh->ether_shost, ETHER_ADDR_LEN);
 
 	return 0;
 }
 
-
-__private_extern__ int
-ether_add_if(struct ifnet *ifp)
-{
-	ifp->if_framer = ether_frameout;
-	ifp->if_demux = ether_demux;
-    
-    return 0;
-}
-
-__private_extern__ int
-ether_del_if(struct ifnet *ifp)
-{
-	if (ifp->family_cookie) {
-		FREE(ifp->family_cookie, M_IFADDR);
-		return 0;
-	}
-	else
-		return ENOENT;
-}
-
-__private_extern__ int
-ether_init_if(struct ifnet *ifp)
-{
-	/*
-	 * Copy ethernet address out of old style arpcom. New
-	 * interfaces created using the KPIs will not have an
-	 * interface family. Those interfaces will have the
-	 * lladdr passed in when the interface is created.
-	 */
-	u_char *enaddr = ((u_char*)ifp) + sizeof(struct ifnet);
-	ifnet_set_lladdr(ifp, enaddr, 6);
-	bzero(enaddr, 6);
-	
-    return 0;
-}
-
-
 errno_t
 ether_check_multi(
-	ifnet_t					ifp,
+	__unused ifnet_t		ifp,
 	const struct sockaddr	*proto_addr)
 {
 	errno_t	result = EAFNOSUPPORT;
@@ -649,52 +589,36 @@ ether_ioctl(
 	return EOPNOTSUPP;
 }
 
-
-extern int ether_attach_inet(struct ifnet *ifp, u_long proto_family);
-extern int ether_detach_inet(struct ifnet *ifp, u_long proto_family);
-extern int ether_attach_inet6(struct ifnet *ifp, u_long proto_family);
-extern int ether_detach_inet6(struct ifnet *ifp, u_long proto_family);
-
-extern void kprintf(const char *, ...);
-
-int ether_family_init(void)
+__private_extern__ int ether_family_init(void)
 {
-    int  error=0;
-    struct dlil_ifmod_reg_str  ifmod_reg;
-
-    /* ethernet family is built-in, called from bsd_init */
-
-    bzero(&ifmod_reg, sizeof(ifmod_reg));
-    ifmod_reg.add_if = ether_add_if;
-    ifmod_reg.del_if = ether_del_if;
-    ifmod_reg.init_if = ether_init_if;
-    ifmod_reg.add_proto = ether_add_proto_old;
-    ifmod_reg.del_proto = ether_del_proto;
-    ifmod_reg.ifmod_ioctl = ether_ioctl;
-    ifmod_reg.shutdown    = ether_shutdown;
-
-    if (dlil_reg_if_modules(APPLE_IF_FAM_ETHERNET, &ifmod_reg)) {
-        printf("WARNING: ether_family_init -- Can't register if family modules\n");
-        error = EIO;
-	goto done;
-    }
-
+	errno_t	error = 0;
+	
 	/* Register protocol registration functions */
-
-	if ((error = dlil_reg_proto_module(PF_INET, APPLE_IF_FAM_ETHERNET,
+	if ((error = proto_register_plumber(PF_INET, APPLE_IF_FAM_ETHERNET,
 									  ether_attach_inet, ether_detach_inet)) != 0) {
-		kprintf("dlil_reg_proto_module failed for AF_INET6 error=%d\n", error);
+		printf("proto_register_plumber failed for PF_INET error=%d\n", error);
 		goto done;
 	}
-
-
-	if ((error = dlil_reg_proto_module(PF_INET6, APPLE_IF_FAM_ETHERNET,
+#if INET6
+	if ((error = proto_register_plumber(PF_INET6, APPLE_IF_FAM_ETHERNET,
 									  ether_attach_inet6, ether_detach_inet6)) != 0) {
-		kprintf("dlil_reg_proto_module failed for AF_INET6 error=%d\n", error);
+		printf("proto_register_plumber failed for PF_INET6 error=%d\n", error);
+		goto done;
+	}
+#endif /* INET6 */
+#if NETAT
+	if ((error = proto_register_plumber(PF_APPLETALK, APPLE_IF_FAM_ETHERNET,
+									  ether_attach_at, ether_detach_at)) != 0) {
+		printf("proto_register_plumber failed PF_APPLETALK error=%d\n", error);
 		goto done;
 	}
+#endif /* NETAT */
+#if VLAN
 	vlan_family_init();
+#endif /* VLAN */
+#if BOND
 	bond_family_init();
+#endif /* BOND */
 
  done:
 
diff --git a/bsd/net/ether_if_module.h b/bsd/net/ether_if_module.h
new file mode 100644
index 000000000..dd20bcba9
--- /dev/null
+++ b/bsd/net/ether_if_module.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _NET_ETHER_IF_MODULE_H
+#define _NET_ETHER_IF_MODULE_H
+
+#include <net/dlil.h>
+
+extern errno_t ether_attach_inet(ifnet_t ifp, protocol_family_t protocol_family);
+extern void ether_detach_inet(ifnet_t ifp, protocol_family_t protocol_family);
+extern errno_t ether_attach_inet6(ifnet_t ifp, protocol_family_t protocol_family);
+extern void ether_detach_inet6(ifnet_t ifp, protocol_family_t protocol_family);
+extern errno_t ether_attach_at(struct ifnet *ifp, protocol_family_t proto_family);
+extern void ether_detach_at(struct ifnet *ifp, protocol_family_t proto_family);
+
+#endif _NET_ETHER_IF_MODULE_H
diff --git a/bsd/net/ether_inet6_pr_module.c b/bsd/net/ether_inet6_pr_module.c
index 1c2175efe..979239c66 100644
--- a/bsd/net/ether_inet6_pr_module.c
+++ b/bsd/net/ether_inet6_pr_module.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -105,9 +111,7 @@ extern struct ifqueue pkintrq;
 #include <net/if_vlan_var.h>
 #endif /* NVLAN > 0 */
 
-/* Local function declerations */
-int ether_attach_inet6(struct ifnet *ifp, u_long protocol_family);
-int ether_detach_inet6(struct ifnet *ifp, u_long protocol_family);
+#include <net/ether_if_module.h>
 
 /*
  * Process a received Ethernet packet;
@@ -115,18 +119,21 @@ int ether_detach_inet6(struct ifnet *ifp, u_long protocol_family);
  * the ether header, which is provided separately.
  */
 static errno_t
-inet6_ether_input(
+ether_inet6_input(
 	__unused ifnet_t	ifp,
 	protocol_family_t	protocol,
 	mbuf_t				packet,
 	__unused char		*header)
 {
-	proto_input(protocol, packet);
-	return 0;
+	errno_t error;
+
+	if ((error = proto_input(protocol, packet)))
+		m_freem(packet);
+	return error;
 }
 
 static errno_t
-inet6_ether_pre_output(
+ether_inet6_pre_output(
     ifnet_t		    			ifp,
     __unused protocol_family_t	protocol_family,
     mbuf_t			     		*m0,
@@ -200,37 +207,29 @@ ether_inet6_prmod_ioctl(
     struct ifreq *ifr = (struct ifreq *) data;
     int error = 0;
 
-    switch (command) {
-    case SIOCSIFADDR:
-	 if ((ifp->if_flags & IFF_RUNNING) == 0) {
-	      ifnet_set_flags(ifp, IFF_UP, IFF_UP);
-	      dlil_ioctl(0, ifp, SIOCSIFFLAGS, (caddr_t) 0);
-	 }
+	switch (command) {
+	case SIOCSIFADDR:
+		if ((ifp->if_flags & IFF_RUNNING) == 0) {
+			ifnet_set_flags(ifp, IFF_UP, IFF_UP);
+			ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
+		}
+		break;
 
-	break;
-
-    case SIOCGIFADDR:
+	case SIOCGIFADDR:
 	ifnet_lladdr_copy_bytes(ifp, ifr->ifr_addr.sa_data, ETHER_ADDR_LEN);
-    break;
-
-    case SIOCSIFMTU:
-	/*
-	 * IOKit IONetworkFamily will set the right MTU according to the driver
-	 */
-
-	 return (0);
+	break;
 
     default:
-	 return EOPNOTSUPP;
+	error = EOPNOTSUPP;
+	break;
     }
-
     return (error);
 }
 
-int
+errno_t
 ether_attach_inet6(
 	struct ifnet	*ifp,
-	__unused u_long	protocol_family)
+	__unused protocol_family_t protocol_family)
 {
 	struct ifnet_attach_proto_param	proto;
 	struct ifnet_demux_desc demux[1];
@@ -243,8 +242,8 @@ ether_attach_inet6(
 	demux[0].datalen = sizeof(en_6native);
 	proto.demux_list = demux;
 	proto.demux_count = 1;
-	proto.input = inet6_ether_input;
-	proto.pre_output = inet6_ether_pre_output;
+	proto.input = ether_inet6_input;
+	proto.pre_output = ether_inet6_pre_output;
 	proto.ioctl = ether_inet6_prmod_ioctl;
 	proto.resolve = ether_inet6_resolve_multi;
 	error = ifnet_attach_protocol(ifp, protocol_family, &proto);
@@ -256,19 +255,17 @@ ether_attach_inet6(
 	return error;
 }
 
-int
+void
 ether_detach_inet6(
 	struct ifnet	*ifp,
-	u_long			protocol_family)
+	protocol_family_t protocol_family)
 {
-    errno_t         error;
+	errno_t         error;
 
 	error = ifnet_detach_protocol(ifp, protocol_family);
 	if (error && error != ENOENT) {
 		printf("WARNING: ether_detach_inet6 can't detach ipv6 from %s%d\n",
 			ifp->if_name, ifp->if_unit);
 	}
-
-    return error;
 }
 
diff --git a/bsd/net/ether_inet_pr_module.c b/bsd/net/ether_inet_pr_module.c
index c3a518e28..062109012 100644
--- a/bsd/net/ether_inet_pr_module.c
+++ b/bsd/net/ether_inet_pr_module.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -52,7 +58,12 @@
  * SUCH DAMAGE.
  *
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 
 #include <sys/param.h>
@@ -83,10 +94,6 @@
 
 #include <net/dlil.h>
 
-#if LLC && CCITT
-extern struct ifqueue pkintrq;
-#endif
-
 #if BRIDGE
 #include <net/bridge.h>
 #endif
@@ -95,17 +102,35 @@ extern struct ifqueue pkintrq;
 #if NVLAN > 0
 #include <net/if_vlan_var.h>
 #endif /* NVLAN > 0 */
+#include <net/ether_if_module.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 /* Local function declerations */
-int ether_attach_inet(struct ifnet *ifp, u_long proto_family);
-int ether_detach_inet(struct ifnet *ifp, u_long proto_family);
+extern void *kdp_get_interface(void);
+extern void kdp_set_ip_and_mac_addresses(struct in_addr *ipaddr,
+										 struct ether_addr *macaddr);
 
-extern void * kdp_get_interface(void);
-extern void ipintr(void);
-extern void arp_input(struct mbuf* m);
+#if defined (__arm__)
+static __inline__ void
+_ip_copy(struct in_addr * dst, const struct in_addr * src)
+{
+	memcpy(dst, src, sizeof(*dst));
+	return;
+}
+
+#else
+static __inline__ void
+_ip_copy(struct in_addr * dst, const struct in_addr * src)
+{
+	*dst = *src;
+	return;
+}
+#endif
 
 static void
-inet_ether_arp_input(
+ether_inet_arp_input(
 	struct mbuf *m)
 {
 	struct ether_arp *ea;
@@ -137,9 +162,9 @@ inet_ether_arp_input(
 	bzero(&sender_ip, sizeof(sender_ip));
 	sender_ip.sin_len = sizeof(sender_ip);
 	sender_ip.sin_family = AF_INET;
-	sender_ip.sin_addr = *(struct in_addr*)ea->arp_spa;
+	_ip_copy(&sender_ip.sin_addr, (const struct in_addr *)ea->arp_spa);
 	target_ip = sender_ip;
-	target_ip.sin_addr = *(struct in_addr*)ea->arp_tpa;
+	_ip_copy(&target_ip.sin_addr, (const struct in_addr *)ea->arp_tpa);
 	
 	bzero(&sender_hw, sizeof(sender_hw));
 	sender_hw.sdl_len = sizeof(sender_hw);
@@ -158,38 +183,49 @@ inet_ether_arp_input(
  * the ether header, which is provided separately.
  */
 static errno_t
-inet_ether_input(
+ether_inet_input(
 	__unused ifnet_t			ifp,
 	__unused protocol_family_t	protocol_family,
-	mbuf_t						m,
-	char     					*frame_header)
+	mbuf_t						m_list)
 {
-    register struct ether_header *eh = (struct ether_header *) frame_header;
-    u_short ether_type;
-
-    ether_type = ntohs(eh->ether_type);
-
-    switch (ether_type) {
-
-		case ETHERTYPE_IP:
-			proto_input(PF_INET, m);
-			break;
-		
-		case ETHERTYPE_ARP: {
-			inet_ether_arp_input(m);
-		}
-		break;
+	mbuf_t	m;
+	mbuf_t	*tailptr = &m_list;
+	mbuf_t	nextpkt;
+	
+	/* Strip ARP and non-IP packets out of the list */
+	for (m = m_list; m; m = nextpkt) {
+    	struct ether_header *eh = mbuf_pkthdr_header(m);
+    	
+    	nextpkt = m->m_nextpkt;
 		
-		default: {
-			return ENOENT;
-		}
-    }
+    	if (eh->ether_type == htons(ETHERTYPE_IP)) {
+    		/* put this packet in the list */
+    		*tailptr = m;
+    		tailptr = &m->m_nextpkt;
+    	}
+    	else {
+    		/* Pass ARP packets to arp input */
+			m->m_nextpkt = NULL;
+    		if (eh->ether_type == htons(ETHERTYPE_ARP))
+    			ether_inet_arp_input(m);
+    		else
+    			mbuf_freem(m);
+    	}
+	}
+	
+	*tailptr = NULL;
+	
+	/* Pass IP list to ip input */
+	if (m_list != NULL && proto_input(PF_INET, m_list) != 0)
+	{
+		mbuf_freem_list(m_list);
+	}
 	
     return 0;
 }
 
 static errno_t
-inet_ether_pre_output(
+ether_inet_pre_output(
 	ifnet_t						ifp,
 	__unused protocol_family_t	protocol_family,
 	mbuf_t						*m0,
@@ -199,7 +235,7 @@ inet_ether_pre_output(
 	char						*edst)
 {
     register struct mbuf *m = *m0;
-    register struct ether_header *eh;
+    const struct ether_header *eh;
     errno_t	result = 0;
 
 
@@ -227,7 +263,7 @@ inet_ether_pre_output(
 		case pseudo_AF_HDRCMPLT:	
 		case AF_UNSPEC:
 			m->m_flags &= ~M_LOOP;
-			eh = (struct ether_header *)dst_netaddr->sa_data;
+			eh = (const struct ether_header *)dst_netaddr->sa_data;
 			(void)memcpy(edst, eh->ether_dhost, 6);
 			*(u_short *)type = eh->ether_type;
 			break;
@@ -274,7 +310,6 @@ ether_inet_resolve_multi(
 	return 0;
 }
 
-
 static errno_t
 ether_inet_prmod_ioctl(
     ifnet_t						ifp,
@@ -289,6 +324,7 @@ ether_inet_prmod_ioctl(
 
     switch (command) {
     case SIOCSIFADDR:
+    case SIOCAIFADDR:
 	if ((ifnet_flags(ifp) & IFF_RUNNING) == 0) {
 		ifnet_set_flags(ifp, IFF_UP, IFF_UP);
 		ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
@@ -300,13 +336,19 @@ ether_inet_prmod_ioctl(
 
 	    inet_arp_init_ifaddr(ifp, ifa);
 	    /*
-	     * Register new IP and MAC addresses with the kernel debugger
-	     * if the interface is the same as was registered by IOKernelDebugger. If
-		 * no interface was registered, fall back and just match against en0 interface.
+	     * Register new IP and MAC addresses with the kernel
+	     * debugger if the interface is the same as was registered
+	     * by IOKernelDebugger. If no interface was registered,
+	     * fall back and just match against en0 interface.
+	     * Do this only for the first address of the interface
+	     * and not for aliases.
 	     */
-	    if ((kdp_get_interface() != 0 && kdp_get_interface() == ifp->if_softc)
-		 || (kdp_get_interface() == 0 && ifp->if_unit == 0))
-			kdp_set_ip_and_mac_addresses(&(IA_SIN(ifa)->sin_addr), ifnet_lladdr(ifp));
+	    if (command == SIOCSIFADDR &&
+	        ((kdp_get_interface() != 0 &&
+	        kdp_get_interface() == ifp->if_softc) ||
+		(kdp_get_interface() == 0 && ifp->if_unit == 0)))
+			kdp_set_ip_and_mac_addresses(&(IA_SIN(ifa)->sin_addr),
+			    ifnet_lladdr(ifp));
 
 	    break;
 
@@ -317,18 +359,12 @@ ether_inet_prmod_ioctl(
 	break;
 
     case SIOCGIFADDR:
-	ifnet_lladdr_copy_bytes(ifp, ifr->ifr_addr.sa_data, ETHER_ADDR_LEN);
-    break;
-
-    case SIOCSIFMTU:
-	/*
-	 * IOKit IONetworkFamily will set the right MTU according to the driver
-	 */
-
-	 return (0);
+		ifnet_lladdr_copy_bytes(ifp, ifr->ifr_addr.sa_data, ETHER_ADDR_LEN);
+		break;
 
     default:
-	 return EOPNOTSUPP;
+		error = EOPNOTSUPP;
+		break;
     }
 
     return (error);
@@ -381,10 +417,10 @@ ether_inet_arp(
 		return EINVAL;
 	
 	if ((sender_ip && sender_ip->sin_family != AF_INET) ||
-		(target_ip && target_ip->sin_family != AF_INET))
+	    target_ip->sin_family != AF_INET)
 		return EAFNOSUPPORT;
 	
-	result = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &m);
+	result = mbuf_gethdr(MBUF_DONTWAIT, MBUF_TYPE_DATA, &m);
 	if (result != 0)
 		return result;
 	
@@ -398,10 +434,20 @@ ether_inet_arp(
 	mbuf_setdata(m, datap, sizeof(*ea));
 	ea = mbuf_data(m);
 	
-	/* Prepend the ethernet header, we will send the raw frame */
-	mbuf_prepend(&m, sizeof(*eh), MBUF_WAITOK);
+	/*
+	 * Prepend the ethernet header, we will send the raw frame;
+	 * callee frees the original mbuf when allocation fails.
+	 */
+	result = mbuf_prepend(&m, sizeof(*eh), MBUF_DONTWAIT);
+	if (result != 0)
+		return result;
+
 	eh = mbuf_data(m);
 	eh->ether_type = htons(ETHERTYPE_ARP);
+
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_linklayer(ifp, m);
+#endif
 	
 	/* Fill out the arp header */
 	ea->arp_pro = htons(ETHERTYPE_IP);
@@ -462,12 +508,12 @@ ether_inet_arp(
 	return 0;
 }
 
-int
+errno_t
 ether_attach_inet(
 	struct ifnet	*ifp,
-	__unused u_long	proto_family)
+	__unused protocol_family_t proto_family)
 {
-	struct ifnet_attach_proto_param proto;
+	struct ifnet_attach_proto_param_v2 proto;
 	struct ifnet_demux_desc demux[2];
 	u_short en_native=htons(ETHERTYPE_IP);
 	u_short arp_native=htons(ETHERTYPE_ARP);
@@ -484,14 +530,14 @@ ether_attach_inet(
 	bzero(&proto, sizeof(proto));
 	proto.demux_list = demux;
 	proto.demux_count = sizeof(demux) / sizeof(demux[0]);
-	proto.input = inet_ether_input;
-	proto.pre_output = inet_ether_pre_output;
+	proto.input = ether_inet_input;
+	proto.pre_output = ether_inet_pre_output;
 	proto.ioctl = ether_inet_prmod_ioctl;
 	proto.event = ether_inet_event;
 	proto.resolve = ether_inet_resolve_multi;
 	proto.send_arp = ether_inet_arp;
 	
-	error = ifnet_attach_protocol(ifp, proto_family, &proto);
+	error = ifnet_attach_protocol_v2(ifp, proto_family, &proto);
 	if (error && error != EEXIST) {
 		printf("WARNING: ether_attach_inet can't attach ip to %s%d\n",
 			   ifp->if_name, ifp->if_unit);
@@ -499,15 +545,11 @@ ether_attach_inet(
 	return error;
 }
 
-int
+void
 ether_detach_inet(
 	struct ifnet *ifp,
-	u_long proto_family)
+	protocol_family_t proto_family)
 {
-    int         stat;
-
-	stat = dlil_detach_protocol(ifp, proto_family);
-
-    return stat;
+	(void)ifnet_detach_protocol(ifp, proto_family);
 }
 
diff --git a/bsd/net/etherdefs.h b/bsd/net/etherdefs.h
index 6e577eb14..82ac3713f 100644
--- a/bsd/net/etherdefs.h
+++ b/bsd/net/etherdefs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
diff --git a/bsd/net/ethernet.h b/bsd/net/ethernet.h
index b52bd815f..efbf23c0f 100644
--- a/bsd/net/ethernet.h
+++ b/bsd/net/ethernet.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Fundamental constants relating to ethernet.
@@ -27,6 +33,7 @@
 #ifndef _NET_ETHERNET_H_
 #define _NET_ETHERNET_H_
 #include <sys/appleapiopts.h>
+#include <sys/types.h>		/* u_ types */
 
 /*
  * The number of bytes in an ethernet (MAC) address.
diff --git a/bsd/net/firewire.h b/bsd/net/firewire.h
index d5520fe10..da8450b9d 100644
--- a/bsd/net/firewire.h
+++ b/bsd/net/firewire.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Fundamental constants relating to FireWire network device.
diff --git a/bsd/net/ieee8023ad.h b/bsd/net/ieee8023ad.h
index 6de8d3ea9..944e48875 100644
--- a/bsd/net/ieee8023ad.h
+++ b/bsd/net/ieee8023ad.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/net/if.c b/bsd/net/if.c
index d5e1a5eaf..04b3cadf6 100644
--- a/bsd/net/if.c
+++ b/bsd/net/if.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -54,6 +60,12 @@
  *	@(#)if.c	8.3 (Berkeley) 1/4/94
  * $FreeBSD: src/sys/net/if.c,v 1.85.2.9 2001/07/24 19:10:17 brooks Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <kern/locks.h>
 
@@ -76,6 +88,7 @@
 #include <net/if_types.h>
 #include <net/if_var.h>
 #include <net/net_osdep.h>
+#include <net/ethernet.h>
 
 #include <net/radix.h>
 #include <net/route.h>
@@ -86,7 +99,7 @@
 #include <libkern/OSAtomic.h>
 #endif
 
-#if defined(INET) || defined(INET6)
+#if INET || INET6
 /*XXX*/
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -96,6 +109,15 @@
 #endif
 #endif
 
+extern u_long route_generation;
+extern int use_routegenid;
+extern int dlil_multithreaded_input;
+extern struct dlil_threading_info *dlil_lo_thread_ptr;
+
+#if CONFIG_MACF_NET 
+#include <security/mac_framework.h>
+#endif
+
 /*
  * System initialization
  */
@@ -105,6 +127,9 @@ static void if_qflush(struct ifqueue *);
 __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
 void if_rtproto_del(struct ifnet *ifp, int protocol);
 
+static int if_rtmtu(struct radix_node *, void *);
+static void if_rtmtu_update(struct ifnet *);
+
 static struct	if_clone *if_clone_lookup(const char *, int *);
 #ifdef IF_CLONE_LIST
 static int	if_clone_list(int count, int * total, user_addr_t dst);
@@ -163,7 +188,7 @@ if_detach_ifa(
 #if 1
 	/* Debugging code */
 	if ((ifa->ifa_debug & IFA_ATTACHED) == 0) {
-		printf("if_detach_ifa: ifa is not attached to any interface! flags=%\n", ifa->ifa_debug);
+		printf("if_detach_ifa: ifa is not attached to any interface! flags=%lu\n", ifa->ifa_debug);
 		return;
 	}
 	else {
@@ -478,7 +503,7 @@ if_clone_list(int count, int * total, user_addr_t dst)
 
 	for (ifc = LIST_FIRST(&if_cloners); ifc != NULL && count != 0;
 	     ifc = LIST_NEXT(ifc, ifc_list), count--, dst += IFNAMSIZ) {
-		strncpy(outbuf, ifc->ifc_name, IFNAMSIZ - 1);
+		strlcpy(outbuf, ifc->ifc_name, IFNAMSIZ);
 		error = copyout(outbuf, dst, IFNAMSIZ);
 		if (error)
 			break;
@@ -488,7 +513,6 @@ if_clone_list(int count, int * total, user_addr_t dst)
 }
 #endif IF_CLONE_LIST
 
-int ifa_foraddr(unsigned int addr);
 __private_extern__ int
 ifa_foraddr(
 	unsigned int addr)
@@ -519,6 +543,28 @@ ifa_foraddr(
 	return result;
 }
 
+/*
+ * Return the first (primary) address of a given family on an interface.
+ */
+__private_extern__ struct ifaddr *
+ifa_ifpgetprimary(struct ifnet *ifp, int family)
+{
+	struct ifaddr *ifa0 = NULL, *ifa;
+
+	ifnet_lock_shared(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr->sa_family == family && ifa0 == NULL) {
+			ifa0 = ifa;
+			break;
+		}
+	}
+	if (ifa0 != NULL)
+		ifaref(ifa0);
+	ifnet_lock_done(ifp);
+
+	return (ifa0);
+}
+
 /*
  * Locate an interface based on a complete address.
  */
@@ -529,7 +575,7 @@ ifa_ifwithaddr(
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
-	struct ifaddr *result = 0;
+	struct ifaddr *result = NULL;
 
 #define	equal(a1, a2) \
   (bcmp((const void*)(a1), (const void*)(a2), ((const struct sockaddr *)(a1))->sa_len) == 0)
@@ -571,7 +617,7 @@ ifa_ifwithdstaddr(
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
-	struct ifaddr *result = 0;
+	struct ifaddr *result = NULL;
 
 	ifnet_head_lock_shared();
 	for (ifp = ifnet_head.tqh_first; ifp && !result; ifp = ifp->if_link.tqe_next) {
@@ -746,14 +792,15 @@ ifaof_ifpforaddr(
 	const struct sockaddr *addr,
 	struct ifnet *ifp)
 {
-	struct ifaddr *ifa = 0;
+	struct ifaddr *ifa = NULL;
 	const char *cp, *cp2, *cp3;
 	char *cplim;
-	struct ifaddr *ifa_maybe = 0;
+	struct ifaddr *ifa_maybe = NULL;
+	struct ifaddr *better_ifa_maybe = NULL;
 	u_int af = addr->sa_family;
 
 	if (af >= AF_MAX)
-		return (0);
+		return (NULL);
 	
 	ifnet_lock_shared(ifp);
 	for (ifa = ifp->if_addrhead.tqh_first; ifa;
@@ -772,6 +819,10 @@ ifaof_ifpforaddr(
 			if (equal(addr, ifa->ifa_dstaddr))
 				break;
 		} else {
+		    	if (equal(addr, ifa->ifa_addr)) {
+				/* exact match */
+				break;
+			}
 			cp = addr->sa_data;
 			cp2 = ifa->ifa_addr->sa_data;
 			cp3 = ifa->ifa_netmask->sa_data;
@@ -779,12 +830,22 @@ ifaof_ifpforaddr(
 			for (; cp3 < cplim; cp3++)
 				if ((*cp++ ^ *cp2++) & *cp3)
 					break;
-			if (cp3 == cplim)
-				break;
+			if (cp3 == cplim) {
+				/* subnet match */
+				if (better_ifa_maybe == NULL) {
+					better_ifa_maybe = ifa;
+				}
+			}
 		}
 	}
 	
-	if (!ifa) ifa = ifa_maybe;
+	if (ifa == NULL) {
+		if (better_ifa_maybe != NULL) {
+			ifa = better_ifa_maybe;
+		} else {
+			ifa = ifa_maybe;
+		}
+	}
 	if (ifa) ifaref(ifa);
 	
 	ifnet_lock_done(ifp);
@@ -799,10 +860,7 @@ ifaof_ifpforaddr(
  * This should be moved to /sys/net/link.c eventually.
  */
 void
-link_rtrequest(cmd, rt, sa)
-	int cmd;
-	struct rtentry *rt;
-	struct sockaddr *sa;
+link_rtrequest(int cmd, struct rtentry *rt, struct sockaddr *sa)
 {
 	struct ifaddr *ifa;
 	struct sockaddr *dst;
@@ -918,8 +976,7 @@ if_up(
  * Flush an interface queue.
  */
 static void
-if_qflush(ifq)
-	struct ifqueue *ifq;
+if_qflush(struct ifqueue *ifq)
 {
 	struct mbuf *m, *n;
 
@@ -928,8 +985,8 @@ if_qflush(ifq)
 		n = m->m_act;
 		m_freem(m);
 	}
-	ifq->ifq_head = 0;
-	ifq->ifq_tail = 0;
+	ifq->ifq_head = NULL;
+	ifq->ifq_tail = NULL;
 	ifq->ifq_len = 0;
 }
 
@@ -973,7 +1030,7 @@ ifunit(const char *name)
 	 */
 	ifnet_head_lock_shared();
 	TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
-		if (strcmp(ifp->if_name, namebuf))
+		if (strncmp(ifp->if_name, namebuf, len))
 			continue;
 		if (unit == ifp->if_unit)
 			break;
@@ -988,8 +1045,7 @@ ifunit(const char *name)
  * interface structure pointer.
  */
 struct ifnet *
-if_withname(sa)
-	struct sockaddr *sa;
+if_withname(struct sockaddr *sa)
 {
 	char ifname[IFNAMSIZ+1];
 	struct sockaddr_dl *sdl = (struct sockaddr_dl *)sa;
@@ -1015,11 +1071,7 @@ if_withname(sa)
  * Interface ioctls.
  */
 int
-ifioctl(so, cmd, data, p)
-	struct socket *so;
-	u_long cmd;
-	caddr_t data;
-	struct proc *p;
+ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 {
 	struct ifnet *ifp;
 	struct ifreq *ifr;
@@ -1078,6 +1130,13 @@ ifioctl(so, cmd, data, p)
 		ifnet_lock_done(ifp);
 		break;
 
+#if CONFIG_MACF_NET
+	case SIOCGIFMAC:
+		error = mac_ifnet_label_get(kauth_cred_get(), ifr, ifp);
+		if (error)
+			return (error);
+		break;
+#endif
 	case SIOCGIFMETRIC:
 		ifnet_lock_shared(ifp);
 		ifr->ifr_metric = ifp->if_metric;
@@ -1100,11 +1159,11 @@ ifioctl(so, cmd, data, p)
 		error = proc_suser(p);
 		if (error)
 			return (error);
-		
-		ifnet_set_flags(ifp, ifr->ifr_flags, ~IFF_CANTCHANGE);
 
-		error = dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-				   ifp, cmd, (caddr_t) data);
+		ifnet_set_flags(ifp, ifr->ifr_flags, (u_int16_t)~IFF_CANTCHANGE);
+
+		error = ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+				   			cmd, data);
 
 		if (error == 0) {
 			 ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -1112,7 +1171,7 @@ ifioctl(so, cmd, data, p)
 			 ev_msg.kev_subclass   = KEV_DL_SUBCLASS;
 
 			 ev_msg.event_code = KEV_DL_SIFFLAGS;
-			 strncpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+			 strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
 			 ev_data.if_family = ifp->if_family;
 			 ev_data.if_unit   = (unsigned long) ifp->if_unit;
 			 ev_msg.dv[0].data_length = sizeof(struct net_event_data);
@@ -1123,6 +1182,13 @@ ifioctl(so, cmd, data, p)
 		ifnet_touch_lastchange(ifp);
 		break;
 
+#if CONFIG_MACF_NET
+	case SIOCSIFMAC:
+		error = mac_ifnet_label_set(kauth_cred_get(), ifr, ifp);
+		if (error)
+			return (error);
+		break;
+#endif
 	case SIOCSIFMETRIC:
 		error = proc_suser(p);
 		if (error)
@@ -1135,7 +1201,7 @@ ifioctl(so, cmd, data, p)
 		ev_msg.kev_subclass   = KEV_DL_SUBCLASS;
 	
 		ev_msg.event_code = KEV_DL_SIFMETRICS;
-		strncpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+		strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
 		ev_data.if_family = ifp->if_family;
 		ev_data.if_unit   = (unsigned long) ifp->if_unit;
 		ev_msg.dv[0].data_length = sizeof(struct net_event_data);
@@ -1152,8 +1218,8 @@ ifioctl(so, cmd, data, p)
 		if (error)
 			return error;
 
-		error = dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-					ifp, cmd, (caddr_t) data);
+		error = ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+							cmd, data);
 
 		if (error == 0) {
 			ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -1161,7 +1227,7 @@ ifioctl(so, cmd, data, p)
 			ev_msg.kev_subclass   = KEV_DL_SUBCLASS;
 
 			ev_msg.event_code = KEV_DL_SIFPHYS;
-			strncpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+			strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
 			ev_data.if_family = ifp->if_family;
 			ev_data.if_unit   = (unsigned long) ifp->if_unit;
 			ev_msg.dv[0].data_length = sizeof(struct net_event_data);
@@ -1185,8 +1251,8 @@ ifioctl(so, cmd, data, p)
 		if (ifr->ifr_mtu < IF_MINMTU || ifr->ifr_mtu > IF_MAXMTU)
 			return (EINVAL);
 
-		error = dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-				   ifp, cmd, (caddr_t) data);
+		error = ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+				   			cmd, data);
 
 		if (error == 0) {
 		     ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -1194,7 +1260,7 @@ ifioctl(so, cmd, data, p)
 		     ev_msg.kev_subclass   = KEV_DL_SUBCLASS;
 	
 		     ev_msg.event_code = KEV_DL_SIFMTU;
-		     strncpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+		     strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
 		     ev_data.if_family = ifp->if_family;
 		     ev_data.if_unit   = (unsigned long) ifp->if_unit;
 		     ev_msg.dv[0].data_length = sizeof(struct net_event_data);
@@ -1206,9 +1272,12 @@ ifioctl(so, cmd, data, p)
 			rt_ifmsg(ifp);
 		}
 		/*
-		 * If the link MTU changed, do network layer specific procedure.
+		 * If the link MTU changed, do network layer specific procedure
+		 * and update all route entries associated with the interface,
+		 * so that their MTU metric gets updated.
 		 */
-		if (ifp->if_mtu != oldmtu) {
+		if (error == 0 && ifp->if_mtu != oldmtu) {
+			if_rtmtu_update(ifp);
 #if INET6
 			nd6_setmtu(ifp);
 #endif
@@ -1243,7 +1312,7 @@ ifioctl(so, cmd, data, p)
 		     ev_msg.vendor_code    = KEV_VENDOR_APPLE;
 		     ev_msg.kev_class      = KEV_NETWORK_CLASS;
 		     ev_msg.kev_subclass   = KEV_DL_SUBCLASS;
-		     strncpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
+		     strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
 	
 		     ev_data.if_family = ifp->if_family;
 		     ev_data.if_unit   = (unsigned long) ifp->if_unit;
@@ -1258,7 +1327,7 @@ ifioctl(so, cmd, data, p)
 
 	case SIOCSIFPHYADDR:
 	case SIOCDIFPHYADDR:
-#ifdef INET6
+#if INET6
 	case SIOCSIFPHYADDR_IN6:
 #endif
 	case SIOCSLIFPHYADDR:
@@ -1272,8 +1341,8 @@ ifioctl(so, cmd, data, p)
 		if (error)
 			return (error);
 
-		error = dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-				   ifp, cmd, (caddr_t) data);
+		error = ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+				   			cmd, data);
 
 		if (error == 0)
 			ifnet_touch_lastchange(ifp);
@@ -1289,23 +1358,17 @@ ifioctl(so, cmd, data, p)
 	case SIOCGIFMEDIA:
 	case SIOCGIFGENERIC:
 	case SIOCGIFDEVMTU:
-		return dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-				   ifp, cmd, (caddr_t) data);
+		return ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+				   		   cmd, data);
 	case SIOCGIFVLAN:
 	case SIOCGIFBOND:
-		return dlil_ioctl(so->so_proto->pr_domain->dom_family, 
-				   ifp, cmd, (caddr_t) data);
+		return ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family, 
+				   		   cmd, data);
 
 	default:
 		oif_flags = ifp->if_flags;
 		if (so->so_proto == 0)
 			return (EOPNOTSUPP);
-#if !COMPAT_43_SOCKET
-		socket_lock(so, 1);
-		error =(*so->so_proto->pr_usrreqs->pru_control)(so, cmd, data, ifp, p));
-		socket_unlock(so, 1);
-		return (error);
-#else
 	    {
 		int ocmd = cmd;
 
@@ -1356,11 +1419,15 @@ ifioctl(so, cmd, data, p)
 
 		}
 	    }
-#endif /* COMPAT_43_SOCKET */
+		if (cmd == SIOCSIFKPI) {
+			int temperr = proc_suser(p);
+			if (temperr != 0)
+				error = temperr;
+		}
 
 		if (error == EOPNOTSUPP || error == ENOTSUP)
-			error = dlil_ioctl(so->so_proto->pr_domain->dom_family,
-								ifp, cmd, (caddr_t) data);
+			error = ifnet_ioctl(ifp, so->so_proto->pr_domain->dom_family,
+								cmd, data);
 
 		return (error);
 	}
@@ -1368,11 +1435,7 @@ ifioctl(so, cmd, data, p)
 }
 
 int
-ifioctllocked(so, cmd, data, p)
-	struct socket *so;
-	u_long cmd;
-	caddr_t data;
-	struct proc *p;
+ifioctllocked(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 {
 	int error;
 
@@ -1423,7 +1486,7 @@ ifnet_set_promiscuous(
 	ifr.ifr_flags = ifp->if_flags;
 	locked = 0;
 	ifnet_lock_done(ifp);
-	error = dlil_ioctl(0, ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
+	error = ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, &ifr);
 	if (error == 0)
 		rt_ifmsg(ifp);
 	else
@@ -1472,7 +1535,7 @@ ifconf(u_long cmd, user_addr_t ifrp, int * ret_space)
 			error = ENAMETOOLONG;
 			break;
 		} else {
-			strcpy(ifr.ifr_name, workbuf);
+			strlcpy(ifr.ifr_name, workbuf, IFNAMSIZ);
 		}
 		
 		ifnet_lock_shared(ifp);
@@ -1487,7 +1550,6 @@ ifconf(u_long cmd, user_addr_t ifrp, int * ret_space)
 				continue;
 #endif
 			addrs++;
-#if COMPAT_43_SOCKET
 			if (cmd == OSIOCGIFCONF) {
 				struct osockaddr *osa =
 					 (struct osockaddr *)&ifr.ifr_addr;
@@ -1495,9 +1557,7 @@ ifconf(u_long cmd, user_addr_t ifrp, int * ret_space)
 				osa->sa_family = sa->sa_family;
 				error = copyout((caddr_t)&ifr, ifrp, sizeof(ifr));
 				ifrp += sizeof(struct ifreq);
-			} else
-#endif
-			if (sa->sa_len <= sizeof(*sa)) {
+			} else if (sa->sa_len <= sizeof(*sa)) {
 				ifr.ifr_addr = *sa;
 				error = copyout((caddr_t)&ifr, ifrp, sizeof(ifr));
 				ifrp += sizeof(struct ifreq);
@@ -1539,9 +1599,7 @@ ifconf(u_long cmd, user_addr_t ifrp, int * ret_space)
  * Just like if_promisc(), but for all-multicast-reception mode.
  */
 int
-if_allmulti(ifp, onswitch)
-	struct ifnet *ifp;
-	int onswitch;
+if_allmulti(struct ifnet *ifp, int onswitch)
 {
 	int error = 0;
 	int	modified = 0;
@@ -1565,7 +1623,7 @@ if_allmulti(ifp, onswitch)
 	ifnet_lock_done(ifp);
 	
 	if (modified)
-		error = dlil_ioctl(0, ifp, SIOCSIFFLAGS, (caddr_t) 0);
+		error = ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
 
 	if (error == 0)
 		rt_ifmsg(ifp);
@@ -1632,6 +1690,71 @@ if_addmulti_doesexist(
 	return ENOENT;
 }
 
+/*
+ * Radar 3642395, make sure all multicasts are in a standard format.
+ */
+static struct sockaddr*
+copy_and_normalize(
+	const struct sockaddr	*original)
+{
+	int					alen = 0;
+	const u_char		*aptr = NULL;
+	struct sockaddr		*copy = NULL;
+	struct sockaddr_dl	*sdl_new = NULL;
+	int					len = 0;
+	
+	if (original->sa_family != AF_LINK &&
+		original->sa_family != AF_UNSPEC) {
+		/* Just make a copy */
+		MALLOC(copy, struct sockaddr*, original->sa_len, M_IFADDR, M_WAITOK);
+		if (copy != NULL)
+			bcopy(original, copy, original->sa_len);
+		return copy;
+	}
+	
+	switch (original->sa_family) {
+		case AF_LINK: {
+			const struct sockaddr_dl	*sdl_original =
+											(const struct sockaddr_dl*)original;
+			
+			if (sdl_original->sdl_nlen + sdl_original->sdl_alen + sdl_original->sdl_slen +
+				offsetof(struct sockaddr_dl, sdl_data) > sdl_original->sdl_len)
+				return NULL;
+			
+			alen = sdl_original->sdl_alen;
+			aptr = CONST_LLADDR(sdl_original);
+		}
+		break;
+		
+		case AF_UNSPEC: {
+			if (original->sa_len < ETHER_ADDR_LEN +
+				offsetof(struct sockaddr, sa_data)) {
+				return NULL;
+			}
+			
+			alen = ETHER_ADDR_LEN;
+			aptr = (const u_char*)original->sa_data;
+		}
+		break;
+	}
+	
+	if (alen == 0 || aptr == NULL)
+		return NULL;
+	
+	len = alen + offsetof(struct sockaddr_dl, sdl_data);
+	MALLOC(sdl_new, struct sockaddr_dl*, len, M_IFADDR, M_WAITOK);
+	
+	if (sdl_new != NULL) {
+		bzero(sdl_new, len);
+		sdl_new->sdl_len = len;
+		sdl_new->sdl_family = AF_LINK;
+		sdl_new->sdl_alen = alen;
+		bcopy(aptr, LLADDR(sdl_new), alen);
+	}
+	
+	return (struct sockaddr*)sdl_new;
+}
+
 /*
  * Add a multicast listenership to the interface in question.
  * The link layer provides a routine which converts
@@ -1644,42 +1767,70 @@ if_addmulti(
 {
 	struct sockaddr_storage storage;
 	struct sockaddr *llsa = NULL;
-	struct sockaddr *dupsa;
-	int error;
-	struct ifmultiaddr *ifma;
+	struct sockaddr *dupsa = NULL;
+	int error = 0;
+	struct ifmultiaddr *ifma = NULL;
 	struct ifmultiaddr *llifma = NULL;
 	
+	/* If sa is a AF_LINK or AF_UNSPEC, duplicate and normalize it */
+	if (sa->sa_family == AF_LINK || sa->sa_family == AF_UNSPEC) {
+		dupsa = copy_and_normalize(sa);
+		if (dupsa == NULL) {
+			return ENOMEM;
+		}
+		sa = dupsa;
+	}
+	
 	ifnet_lock_exclusive(ifp);
 	error = if_addmulti_doesexist(ifp, sa, retifma);
 	ifnet_lock_done(ifp);
 	
-	if (error == 0)
-		return 0;
+	if (error == 0) {
+		goto cleanup;
+	}
 
 	/*
 	 * Give the link layer a chance to accept/reject it, and also
 	 * find out which AF_LINK address this maps to, if it isn't one
 	 * already.
 	 */
-	error = dlil_resolve_multi(ifp, sa, (struct sockaddr*)&storage, sizeof(storage));
+	error = dlil_resolve_multi(ifp, sa, (struct sockaddr*)&storage,
+							   sizeof(storage));
 	if (error == 0 && storage.ss_len != 0) {
-		MALLOC(llsa, struct sockaddr*, storage.ss_len, M_IFMADDR, M_WAITOK);
+		llsa = copy_and_normalize((struct sockaddr*)&storage);
+		if (llsa == NULL) {
+			error = ENOMEM;
+			goto cleanup;
+		}
+		
 		MALLOC(llifma, struct ifmultiaddr *, sizeof *llifma, M_IFMADDR, M_WAITOK);
-		bcopy(&storage, llsa, storage.ss_len);
+		if (llifma == NULL) {
+			error = ENOMEM;
+			goto cleanup;
+		}
 	}
 	
 	/* to be similar to FreeBSD */
-	if (error == EOPNOTSUPP)
+	if (error == EOPNOTSUPP) {
 		error = 0;
-
-	if (error) {
-		return error;
+	}
+	else if (error) {
+		goto cleanup;
 	}
 
 	/* Allocate while we aren't holding any locks */
+	if (dupsa == NULL) {
+		dupsa = copy_and_normalize(sa);
+		if (dupsa == NULL) {
+			error = ENOMEM;
+			goto cleanup;
+		}
+	}
 	MALLOC(ifma, struct ifmultiaddr *, sizeof *ifma, M_IFMADDR, M_WAITOK);
-	MALLOC(dupsa, struct sockaddr *, sa->sa_len, M_IFMADDR, M_WAITOK);
-	bcopy(sa, dupsa, sa->sa_len);
+	if (ifma == NULL) {
+		error = ENOMEM;
+		goto cleanup;
+	}
 	
 	ifnet_lock_exclusive(ifp);
 	/*
@@ -1687,11 +1838,7 @@ if_addmulti(
 	 */
 	if ((error = if_addmulti_doesexist(ifp, sa, retifma)) == 0) {
 		ifnet_lock_done(ifp);
-		FREE(ifma, M_IFMADDR);
-		FREE(dupsa, M_IFMADDR);
-		if (llsa)
-			FREE(llsa, M_IFMADDR);
-		return 0;
+		goto cleanup;
 	}
 
 	bzero(ifma, sizeof(*ifma));
@@ -1733,9 +1880,21 @@ if_addmulti(
 	 * We are certain we have added something, so call down to the
 	 * interface to let them know about it.
 	 */
-	dlil_ioctl(0, ifp, SIOCADDMULTI, (caddr_t) 0);
+	ifnet_ioctl(ifp, 0, SIOCADDMULTI, NULL);
 	
 	return 0;
+	
+cleanup:
+	if (ifma)
+		FREE(ifma, M_IFADDR);
+	if (dupsa)
+		FREE(dupsa, M_IFADDR);
+	if (llifma)
+		FREE(llifma, M_IFADDR);
+	if (llsa)
+		FREE(llsa, M_IFADDR);
+	
+	return error;
 }
 
 int
@@ -1797,7 +1956,7 @@ if_delmultiaddr(
 	if (do_del_multi) {
 		if (locked)
 			ifnet_lock_done(ifp);
-		dlil_ioctl(0, ifp, SIOCDELMULTI, 0);
+		ifnet_ioctl(ifp, 0, SIOCDELMULTI, NULL);
 		if (locked)
 			ifnet_lock_exclusive(ifp);
 	}
@@ -1814,9 +1973,18 @@ if_delmulti(
 	struct ifnet *ifp,
 	const struct sockaddr *sa)
 {
-	struct ifmultiaddr *ifma;
+	struct ifmultiaddr	*ifma;
+	struct sockaddr		*dupsa = NULL;
 	int retval = 0;
 
+	if (sa->sa_family == AF_LINK || sa->sa_family == AF_UNSPEC) {
+		dupsa = copy_and_normalize(sa);
+		if (dupsa == NULL) {
+			return ENOMEM;
+		}
+		sa = dupsa;
+	}
+	
 	ifnet_lock_exclusive(ifp);
 	for (ifma = ifp->if_multiaddrs.lh_first; ifma;
 	     ifma = ifma->ifma_link.le_next)
@@ -1824,11 +1992,15 @@ if_delmulti(
 			break;
 	if (ifma == 0) {
 		ifnet_lock_done(ifp);
+		if (dupsa)
+			FREE(dupsa, M_IFADDR);
 		return ENOENT;
 	}
 	
 	retval = if_delmultiaddr(ifma, 1);
 	ifnet_lock_done(ifp);
+	if (dupsa)
+		FREE(dupsa, M_IFADDR);
 	
 	return retval;
 }
@@ -1847,9 +2019,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
 #endif
 
 struct ifmultiaddr *
-ifmaof_ifpforaddr(sa, ifp)
-	const struct sockaddr *sa;
-	struct ifnet *ifp;
+ifmaof_ifpforaddr(const struct sockaddr *sa, struct ifnet *ifp)
 {
 	struct ifmultiaddr *ifma;
 	
@@ -1863,16 +2033,16 @@ ifmaof_ifpforaddr(sa, ifp)
 	return ifma;
 }
 
-SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers");
-SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management");
+SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Link layers");
+SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Generic link-management");
 
 
 /*
  * Shutdown all network activity.  Used boot() when halting
  * system.
  */
-int if_down_all(void);
-int if_down_all(void)
+int
+if_down_all(void)
 {
 	struct ifnet **ifp;
 	u_int32_t	count;
@@ -1939,7 +2109,8 @@ if_rtdel(
 void if_rtproto_del(struct ifnet *ifp, int protocol)
 {
 	struct radix_node_head  *rnh;
-
+	if (use_routegenid) 
+		route_generation++;
 	if ((protocol <= AF_MAX) && (protocol >= 0) &&
 		((rnh = rt_tables[protocol]) != NULL) && (ifp != NULL)) {
 		lck_mtx_lock(rt_mtx);
@@ -1948,13 +2119,59 @@ void if_rtproto_del(struct ifnet *ifp, int protocol)
 	}
 }
 
-extern lck_spin_t *dlil_input_lock;
+static int
+if_rtmtu(struct radix_node *rn, void *arg)
+{
+	struct rtentry *rt = (struct rtentry *)rn;
+	struct ifnet *ifp = arg;
+
+	if (rt->rt_ifp == ifp) {
+		/*
+		 * Update the MTU of this entry only if the MTU
+		 * has not been locked (RTV_MTU is not set) and
+		 * if it was non-zero to begin with.
+		 */
+		if (!(rt->rt_rmx.rmx_locks & RTV_MTU) && rt->rt_rmx.rmx_mtu)
+			rt->rt_rmx.rmx_mtu = ifp->if_mtu;
+	}
+
+	return (0);
+}
+
+/*
+ * Update the MTU metric of all route entries in all protocol tables
+ * associated with a particular interface; this is called when the
+ * MTU of that interface has changed.
+ */
+static
+void if_rtmtu_update(struct ifnet *ifp)
+{
+	struct radix_node_head *rnh;
+	int p;
+
+	for (p = 0; p < AF_MAX + 1; p++) {
+		if ((rnh = rt_tables[p]) == NULL)
+			continue;
+
+		lck_mtx_lock(rt_mtx);
+		(void) rnh->rnh_walktree(rnh, if_rtmtu, ifp);
+		lck_mtx_unlock(rt_mtx);
+	}
+
+	if (use_routegenid)
+		route_generation++;
+}
 
 __private_extern__ void
 if_data_internal_to_if_data(
+	struct ifnet *ifp,
 	const struct if_data_internal	*if_data_int,
 	struct if_data					*if_data)
 {
+	struct dlil_threading_info *thread;
+       	if ((thread = ifp->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
 #define COPYFIELD(fld)	if_data->fld = if_data_int->fld
 #define COPYFIELD32(fld)	if_data->fld = (u_int32_t)(if_data_int->fld)
 	COPYFIELD(ifi_type);
@@ -1974,7 +2191,7 @@ if_data_internal_to_if_data(
 		COPYFIELD32(ifi_baudrate);
 	}
 	
-	lck_spin_lock(dlil_input_lock);
+	lck_mtx_lock(thread->input_lck);
 	COPYFIELD32(ifi_ipackets);
 	COPYFIELD32(ifi_ierrors);
 	COPYFIELD32(ifi_opackets);
@@ -1989,7 +2206,7 @@ if_data_internal_to_if_data(
 	COPYFIELD32(ifi_recvtiming);
 	COPYFIELD32(ifi_xmittiming);
 	COPYFIELD(ifi_lastchange);
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 #if IF_LASTCHANGEUPTIME
 	if_data->ifi_lastchange.tv_sec += boottime_sec();
@@ -2005,9 +2222,14 @@ if_data_internal_to_if_data(
 
 __private_extern__ void
 if_data_internal_to_if_data64(
+	struct ifnet *ifp,
 	const struct if_data_internal	*if_data_int,
 	struct if_data64				*if_data64)
 {
+	struct dlil_threading_info *thread;
+       	if ((thread = ifp->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
 #define COPYFIELD(fld)	if_data64->fld = if_data_int->fld
 	COPYFIELD(ifi_type);
 	COPYFIELD(ifi_typelen);
@@ -2021,7 +2243,7 @@ if_data_internal_to_if_data64(
 	COPYFIELD(ifi_metric);
 	COPYFIELD(ifi_baudrate);
 
-	lck_spin_lock(dlil_input_lock);
+	lck_mtx_lock(thread->input_lck);
 	COPYFIELD(ifi_ipackets);
 	COPYFIELD(ifi_ierrors);
 	COPYFIELD(ifi_opackets);
@@ -2036,7 +2258,7 @@ if_data_internal_to_if_data64(
 	COPYFIELD(ifi_recvtiming);
 	COPYFIELD(ifi_xmittiming);
 	COPYFIELD(ifi_lastchange);
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 #if IF_LASTCHANGEUPTIME
 	if_data64->ifi_lastchange.tv_sec += boottime_sec();
diff --git a/bsd/net/if.h b/bsd/net/if.h
index 1763d9383..a883f4715 100644
--- a/bsd/net/if.h
+++ b/bsd/net/if.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -58,9 +64,11 @@
 #ifndef _NET_IF_H_
 #define	_NET_IF_H_
 
+#include <sys/cdefs.h>
+
 #define	IF_NAMESIZE	16
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #include <sys/appleapiopts.h>
 #ifdef __APPLE__
 /*
@@ -140,6 +148,7 @@ struct if_clonereq64 {
 #define IFEF_VLAN		0x200	/* interface has one or more vlans */
 #define IFEF_BOND		0x400	/* interface is part of bond */
 #define	IFEF_ARPLL		0x800	/* ARP for IPv4LL addresses on this port */
+#define	IFEF_SENDLIST	0x10000000 /* Interface supports sending a list of packets */
 #define IFEF_REUSE	0x20000000 /* DLIL ifnet recycler, ifnet is not new */
 #define IFEF_INUSE	0x40000000 /* DLIL ifnet recycler, ifnet in use */
 #define IFEF_UPDOWNCHANGE	0x80000000 /* Interface's up/down state is changing */
@@ -238,6 +247,47 @@ struct ifdevmtu {
 	int	ifdm_max;
 };
 
+#pragma pack(4)
+
+/*
+ ifkpi: interface kpi ioctl
+ Used with SIOCSIFKPI and SIOCGIFKPI.
+
+ ifk_module_id - From in the kernel, a value from kev_vendor_code_find. From
+ 	user space, a value from SIOCGKEVVENDOR ioctl on a kernel event socket.
+ ifk_type - The type. Types are specific to each module id.
+ ifk_data - The data. ifk_ptr may be a 64bit pointer for 64 bit processes.
+ 
+ Copying data between user space and kernel space is done using copyin
+ and copyout. A process may be running in 64bit mode. In such a case,
+ the pointer will be a 64bit pointer, not a 32bit pointer. The following
+ sample is a safe way to copy the data in to the kernel from either a
+ 32bit or 64bit process:
+ 
+ user_addr_t tmp_ptr;
+ if (IS_64BIT_PROCESS(current_proc())) {
+ 	tmp_ptr = CAST_USER_ADDR_T(ifkpi.ifk_data.ifk_ptr64);
+ }
+ else {
+ 	tmp_ptr = CAST_USER_ADDR_T(ifkpi.ifk_data.ifk_ptr);
+ }
+ error = copyin(tmp_ptr, allocated_dst_buffer, size of allocated_dst_buffer);
+ */
+
+struct ifkpi {
+	unsigned int	ifk_module_id;
+	unsigned int	ifk_type;
+	union {
+		void		*ifk_ptr;
+		int		ifk_value;
+#ifdef KERNEL
+		u_int64_t	ifk_ptr64;
+#endif /* KERNEL */
+	} ifk_data;
+};
+
+#pragma pack()
+
 /*
  * Interface request structure used for socket
  * ioctl's.  All interface ioctl's must have parameter
@@ -264,6 +314,7 @@ struct	ifreq {
 		u_int64_t ifru_data64;	/* 64-bit ifru_data */
 #endif KERNEL_PRIVATE
 		struct	ifdevmtu ifru_devmtu;
+		struct	ifkpi	ifru_kpi;
 	} ifr_ifru;
 #define	ifr_addr	ifr_ifru.ifru_addr	/* address */
 #define	ifr_dstaddr	ifr_ifru.ifru_dstaddr	/* other end of p-to-p link */
@@ -284,6 +335,7 @@ struct	ifreq {
 #ifdef KERNEL_PRIVATE
 #define ifr_data64	ifr_ifru.ifru_data64	/* 64-bit pointer */
 #endif KERNEL_PRIVATE
+#define ifr_kpi		ifr_ifru.ifru_kpi
 };
 
 #define	_SIZEOF_ADDR_IFREQ(ifr) \
@@ -303,6 +355,8 @@ struct rslvmulti_req {
      struct sockaddr **llsa;
 };
 
+#pragma pack(4)
+
 struct ifmediareq {
 	char	ifm_name[IFNAMSIZ];	/* if name, e.g. "en0" */
 	int	ifm_current;		/* current media options */
@@ -313,6 +367,8 @@ struct ifmediareq {
 	int	*ifm_ulist;		/* media words */
 };
 
+#pragma pack()
+
 #ifdef KERNEL_PRIVATE
 /* LP64 version of ifmediareq.  all pointers 
  * grow when we're dealing with a 64-bit process.
@@ -345,6 +401,8 @@ struct ifstat {
 	char	ascii[IFSTATMAX + 1];
 };
 
+#pragma pack(4)
+
 /*
  * Structure used in SIOCGIFCONF request.
  * Used to retrieve interface configuration
@@ -357,9 +415,11 @@ struct	ifconf {
 		caddr_t	ifcu_buf;
 		struct	ifreq *ifcu_req;
 	} ifc_ifcu;
+};
 #define	ifc_buf	ifc_ifcu.ifcu_buf	/* buffer address */
 #define	ifc_req	ifc_ifcu.ifcu_req	/* array of structures returned */
-};
+
+#pragma pack()
 
 #ifdef KERNEL_PRIVATE
 /* LP64 version of ifconf.  all pointers 
@@ -381,8 +441,8 @@ struct ifconf64 {
  */
 struct kev_dl_proto_data {
      struct net_event_data   	link_data;
-     unsigned long		proto_family;
-     unsigned long		proto_remaining_count;
+     u_int32_t			proto_family;
+     u_int32_t			proto_remaining_count;
 };
 
 /*
@@ -403,7 +463,7 @@ MALLOC_DECLARE(M_IFADDR);
 MALLOC_DECLARE(M_IFMADDR);
 #endif
 #endif
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifndef KERNEL
 struct if_nameindex {
diff --git a/bsd/net/if_arp.h b/bsd/net/if_arp.h
index 24caabe63..0a6c2e874 100644
--- a/bsd/net/if_arp.h
+++ b/bsd/net/if_arp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1986, 1993
diff --git a/bsd/net/if_atm.h b/bsd/net/if_atm.h
index e8fe0eceb..dc4689b56 100644
--- a/bsd/net/if_atm.h
+++ b/bsd/net/if_atm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*      $NetBSD: if_atm.h,v 1.7 1996/11/09 23:02:27 chuck Exp $       */
 /* $FreeBSD: src/sys/net/if_atm.h,v 1.4 1999/12/29 04:38:34 peter Exp $ */
diff --git a/bsd/net/if_bond.c b/bsd/net/if_bond.c
index f530e874f..fd632b8a4 100644
--- a/bsd/net/if_bond.c
+++ b/bsd/net/if_bond.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -59,6 +65,7 @@
 #include <sys/time.h>
 #include <net/devtimer.h>
 #include <net/if_vlan_var.h>
+#include <net/kpi_protocol.h>
 
 #include <kern/locks.h>
 #include <libkern/OSAtomic.h>
@@ -72,7 +79,7 @@
 #include <net/if_media.h>
 #include <net/multicast_list.h>
 
-extern int dlil_input_packet(struct ifnet *, struct mbuf *, char *);
+extern void dlil_input_packet_list(struct ifnet *, struct mbuf *);
 
 static struct ether_addr slow_proto_multicast = {
     IEEE8023AD_SLOW_PROTO_MULTICAST
@@ -195,7 +202,7 @@ typedef struct partner_state_s {
 struct ifbond_s {
     TAILQ_ENTRY(ifbond_s) 	ifb_bond_list;
     int				ifb_flags;
-    UInt32			ifb_retain_count;
+    SInt32			ifb_retain_count;
     char 			ifb_name[IFNAMSIZ];
     struct ifnet *		ifb_ifp;
     bpf_packet_func		ifb_bpf_input;
@@ -210,6 +217,8 @@ struct ifbond_s {
     struct ifmultiaddr *	ifb_ifma_slow_proto;
     bondport_ref *		ifb_distributing_array;
     int				ifb_distributing_count;
+    int				ifb_last_link_event;
+    int				ifb_mode; /* LACP, STATIC */
 };
 
 struct media_info {
@@ -506,7 +515,7 @@ packet_buffer_allocate(int length)
     size = length + sizeof(struct ether_header);
     if (size > (int)MHLEN) {
 	/* XXX doesn't handle large payloads */
-	printf("bond: packet_buffer_allocate size %d > max %d\n", size, MHLEN);
+	printf("bond: packet_buffer_allocate size %d > max %lu\n", size, MHLEN);
 	return (NULL);
     }
     m = m_gethdr(M_WAITOK, MT_DATA);
@@ -550,6 +559,8 @@ bondport_periodic_transmit_machine(bondport_ref p, LAEvent event,
 /**
  ** Transmit machine
  **/
+#define TRANSMIT_MACHINE_TX_IMMEDIATE	((void *)1)
+
 static void
 bondport_transmit_machine(bondport_ref p, LAEvent event,
 			  void * event_data);
@@ -626,7 +637,10 @@ bondport_disable_distributing(bondport_ref p);
 static __inline__ int
 bondport_collecting(bondport_ref p)
 {
-    return (lacp_actor_partner_state_collecting(p->po_actor_state));
+    if (p->po_bond->ifb_mode == IF_BOND_MODE_LACP) {
+	return (lacp_actor_partner_state_collecting(p->po_actor_state));
+    }
+    return (TRUE);
 }
 
 /**
@@ -634,8 +648,8 @@ bondport_collecting(bondport_ref p)
  **/
 static int bond_clone_create(struct if_clone *, int);
 static void bond_clone_destroy(struct ifnet *);
-static int bond_input(struct mbuf *m, char *frame_header, struct ifnet *ifp,
-		      u_long protocol_family, int sync_ok);
+static int bond_input(ifnet_t ifp, protocol_family_t protocol, mbuf_t m,
+					  char *frame_header);
 static int bond_output(struct ifnet *ifp, struct mbuf *m);
 static int bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * addr);
 static int bond_set_bpf_tap(struct ifnet * ifp, bpf_tap_mode mode,
@@ -661,7 +675,7 @@ siocsifmtu(struct ifnet * ifp, int mtu)
 
     bzero(&ifr, sizeof(ifr));
     ifr.ifr_mtu = mtu;
-    return (dlil_ioctl(0, ifp, SIOCSIFMTU, (caddr_t)&ifr));
+    return (ifnet_ioctl(ifp, 0, SIOCSIFMTU, &ifr));
 }
 
 static int
@@ -671,7 +685,7 @@ siocgifdevmtu(struct ifnet * ifp, struct ifdevmtu * ifdm_p)
     int 		error;
 
     bzero(&ifr, sizeof(ifr));
-    error = dlil_ioctl(0, ifp, SIOCGIFDEVMTU, (caddr_t)&ifr);
+    error = ifnet_ioctl(ifp, 0, SIOCGIFDEVMTU, &ifr);
     if (error == 0) {
 	*ifdm_p = ifr.ifr_devmtu;
     }
@@ -850,7 +864,7 @@ interface_media_info(struct ifnet * ifp)
 
     bzero(&mi, sizeof(mi));
     bzero(&ifmr, sizeof(ifmr));
-    if (dlil_ioctl(0, ifp, SIOCGIFMEDIA, (caddr_t)&ifmr) == 0) {
+    if (ifnet_ioctl(ifp, 0, SIOCGIFMEDIA, &ifmr) == 0) {
 	if (ifmr.ifm_count != 0) {
 	    mi.mi_status = ifmr.ifm_status;
 	    mi.mi_active = ifmr.ifm_active;
@@ -874,7 +888,7 @@ ifindex_get_ifaddr(int i)
 static __inline__ struct ifaddr *
 ifp_get_ifaddr(struct ifnet * ifp)
 {
-    return (ifindex_get_ifaddr(ifp->if_index));
+    return (ifindex_get_ifaddr(ifnet_index(ifp)));
 }
 
 static __inline__ struct sockaddr_dl *
@@ -899,10 +913,10 @@ if_siflladdr(struct ifnet * ifp, const struct ether_addr * ea_p)
     ifr.ifr_addr.sa_len = ETHER_ADDR_LEN;
     ether_addr_copy(ifr.ifr_addr.sa_data, ea_p);
 #if 0
-    snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", ifp->if_name,
-	     ifp->if_unit);
+    snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s%d", ifnet_name(ifp),
+	     ifnet_unit(ifp));
 #endif 0
-    return (dlil_ioctl(0, ifp, SIOCSIFLLADDR, (caddr_t)&ifr));
+    return (ifnet_ioctl(ifp, 0, SIOCSIFLLADDR, &ifr));
 }
 
 /**
@@ -957,8 +971,7 @@ bond_globals_init(void)
     }
     b = NULL;
     if (ifp != NULL) {
-	b = bond_globals_create(0x8000, 
-				(lacp_system_ref)LLADDR(ifp_get_sdl(ifp)));
+	b = bond_globals_create(0x8000, (lacp_system_ref)ifnet_lladdr(ifp));
     }
     bond_lock();
     if (g_bond != NULL) {
@@ -1057,7 +1070,7 @@ bond_setmulti(struct ifnet * ifp)
     bondport_ref	p;
 
     bond_lock();
-    ifb = ifp->if_private;
+    ifb = ifnet_softc(ifp);
     if (ifb == NULL || ifbond_flags_if_detaching(ifb) 
 	|| TAILQ_EMPTY(&ifb->ifb_port_list)) {
 	bond_unlock();
@@ -1082,16 +1095,16 @@ bond_setmulti(struct ifnet * ifp)
 	if (error != 0) {
 	    printf("bond_setmulti(%s): "
 		   "multicast_list_program(%s%d) failed, %d\n",
-		   ifb->ifb_name, port_ifp->if_name,
-		   port_ifp->if_unit, error);
+		   ifb->ifb_name, ifnet_name(port_ifp),
+		   ifnet_unit(port_ifp), error);
 	    result = error;
 	}
     }
     bond_lock();
  signal_done:
-    ifbond_release(ifb);
     ifbond_signal(ifb, "bond_setmulti");
     bond_unlock();
+    ifbond_release(ifb);
     return (result);
 }
 
@@ -1130,82 +1143,88 @@ ifbond_add_slow_proto_multicast(ifbond_ref ifb)
 static int
 bond_clone_create(struct if_clone * ifc, int unit)
 {
-    int 		error;
-    ifbond_ref		ifb;
-    struct ifnet *	ifp;
-
-    error = bond_globals_init();
-    if (error != 0) {
-	return (error);
-    }
-
-    ifb = _MALLOC(sizeof(ifbond), M_BOND, M_WAITOK);
-    if (ifb == NULL) {
-	return (ENOMEM);
-    }
-    bzero(ifb, sizeof(*ifb));
-
-    ifbond_retain(ifb);
-    TAILQ_INIT(&ifb->ifb_port_list);
-    TAILQ_INIT(&ifb->ifb_lag_list);
-    ifb->ifb_key = unit + 1;
-
-    /* use the interface name as the unique id for ifp recycle */
-    if ((u_long)snprintf(ifb->ifb_name, sizeof(ifb->ifb_name), "%s%d",
-			 ifc->ifc_name, unit) >= sizeof(ifb->ifb_name)) {
-	ifbond_release(ifb);
-	return (EINVAL);
-    }
-    error = dlil_if_acquire(APPLE_IF_FAM_BOND,
-			    ifb->ifb_name,
-			    strlen(ifb->ifb_name),
-			    &ifp);
-    if (error) {
-	ifbond_release(ifb);
-	return (error);
-    }
-    ifb->ifb_ifp = ifp;
-    ifp->if_name = ifc->ifc_name;
-    ifp->if_unit = unit;
-    ifp->if_family = APPLE_IF_FAM_BOND;
-    ifp->if_private = NULL;
-    ifp->if_ioctl = bond_ioctl;
-    ifp->if_set_bpf_tap = bond_set_bpf_tap;
-    ifp->if_free = bond_if_free;
-    ifp->if_output = bond_output;
-    ifp->if_hwassist = 0;
-    ifp->if_addrlen = ETHER_ADDR_LEN;
-    ifp->if_baudrate = 0;
-    ifp->if_type = IFT_IEEE8023ADLAG;
-    ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
-    ifp->if_mtu = 0;
-
-    /* XXX ethernet specific */
-    ifp->if_broadcast.length = ETHER_ADDR_LEN;
-    bcopy(etherbroadcastaddr, ifp->if_broadcast.u.buffer, ETHER_ADDR_LEN);
-    
-    error = dlil_if_attach(ifp);
-    if (error != 0) {
-	dlil_if_release(ifp);
-	ifbond_release(ifb);
-	return (error);
-    }
-    error = ifbond_add_slow_proto_multicast(ifb);
-    if (error != 0) {
-	printf("bond_clone_create(%s): "
-	       "failed to add slow_proto multicast, %d\n",
-	       ifb->ifb_name, error);
-    }
-
-    /* attach as ethernet */
-    bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
-
-    bond_lock();
-    ifp->if_private = ifb;
-    TAILQ_INSERT_HEAD(&g_bond->ifbond_list, ifb, ifb_bond_list);
-    bond_unlock();
-
-    return (0);
+	int 						error;
+	ifbond_ref					ifb;
+	ifnet_t						ifp;
+	struct ifnet_init_params	bond_init;
+	
+	error = bond_globals_init();
+	if (error != 0) {
+		return (error);
+	}
+	
+	ifb = _MALLOC(sizeof(ifbond), M_BOND, M_WAITOK);
+	if (ifb == NULL) {
+		return (ENOMEM);
+	}
+	bzero(ifb, sizeof(*ifb));
+	
+	ifbond_retain(ifb);
+	TAILQ_INIT(&ifb->ifb_port_list);
+	TAILQ_INIT(&ifb->ifb_lag_list);
+	ifb->ifb_key = unit + 1;
+	
+	/* use the interface name as the unique id for ifp recycle */
+	if ((u_long)snprintf(ifb->ifb_name, sizeof(ifb->ifb_name), "%s%d",
+						 ifc->ifc_name, unit) >= sizeof(ifb->ifb_name)) {
+		ifbond_release(ifb);
+		return (EINVAL);
+	}
+	
+	bzero(&bond_init, sizeof(bond_init));
+	bond_init.uniqueid = ifb->ifb_name;
+	bond_init.uniqueid_len = strlen(ifb->ifb_name);
+	bond_init.name = ifc->ifc_name;
+	bond_init.unit = unit;
+	bond_init.family = IFNET_FAMILY_BOND;
+	bond_init.type = IFT_IEEE8023ADLAG;
+	bond_init.output = bond_output;
+	bond_init.demux = ether_demux;
+	bond_init.add_proto = ether_add_proto;
+	bond_init.del_proto = ether_del_proto;
+	bond_init.check_multi = ether_check_multi;
+	bond_init.framer = ether_frameout;
+	bond_init.ioctl = bond_ioctl;
+	bond_init.set_bpf_tap = bond_set_bpf_tap;
+	bond_init.detach = bond_if_free;
+	bond_init.broadcast_addr = etherbroadcastaddr;
+	bond_init.broadcast_len = ETHER_ADDR_LEN;
+	bond_init.softc = ifb;
+	error = ifnet_allocate(&bond_init, &ifp);
+	
+	if (error) {
+		ifbond_release(ifb);
+		return (error);
+	}
+	
+	ifb->ifb_ifp = ifp;
+	ifnet_set_offload(ifp, 0);
+	ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); /* XXX ethernet specific */
+	ifnet_set_flags(ifp, IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX, 0xffff);
+	ifnet_set_baudrate(ifp, 0);
+	ifnet_set_mtu(ifp, 0);
+	
+	error = ifnet_attach(ifp, NULL);
+	if (error != 0) {
+		ifnet_release(ifp);
+		ifbond_release(ifb);
+		return (error);
+	}
+	error = ifbond_add_slow_proto_multicast(ifb);
+	if (error != 0) {
+		printf("bond_clone_create(%s): "
+			   "failed to add slow_proto multicast, %d\n",
+			   ifb->ifb_name, error);
+	}
+	
+	/* attach as ethernet */
+	bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
+	
+	bond_lock();
+	TAILQ_INSERT_HEAD(&g_bond->ifbond_list, ifb, ifb_bond_list);
+	bond_unlock();
+	
+	return (0);
 }
 
 static void
@@ -1240,14 +1259,12 @@ bond_if_detach(struct ifnet * ifp)
 {
     int		error;
 
-    error = dlil_if_detach(ifp);
-    if (error != DLIL_WAIT_FOR_FREE) {
+    error = ifnet_detach(ifp);
 	if (error) {
-	    printf("bond_if_detach %s%d: dlil_if_detach failed, %d\n",
-		   ifp->if_name, ifp->if_unit, error);
+	    printf("bond_if_detach %s%d: ifnet_detach failed, %d\n",
+		   ifnet_name(ifp), ifnet_unit(ifp), error);
 	}
-	bond_if_free(ifp);
-    }
+	
     return;
 }
 
@@ -1257,8 +1274,8 @@ bond_clone_destroy(struct ifnet * ifp)
     ifbond_ref ifb;
 
     bond_lock();
-    ifb = ifp->if_private;
-    if (ifb == NULL || ifp->if_type != IFT_IEEE8023ADLAG) {
+    ifb = ifnet_softc(ifp);
+    if (ifb == NULL || ifnet_type(ifp) != IFT_IEEE8023ADLAG) {
 	bond_unlock();
 	return;
     }
@@ -1278,7 +1295,7 @@ bond_set_bpf_tap(struct ifnet * ifp, bpf_tap_mode mode, bpf_packet_func func)
     ifbond_ref	ifb;
 
     bond_lock();
-    ifb = ifp->if_private;
+    ifb = ifnet_softc(ifp);
     if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	bond_unlock();
 	return (ENODEV);
@@ -1515,7 +1532,7 @@ bond_output(struct ifnet * ifp, struct mbuf * m)
 	}
     }
     bond_lock();
-    ifb = ifp->if_private;
+    ifb = ifnet_softc(ifp);
     if (ifb == NULL || ifbond_flags_if_detaching(ifb)
 	|| ifb->ifb_distributing_count == 0) {
 	goto done;
@@ -1534,7 +1551,7 @@ bond_output(struct ifnet * ifp, struct mbuf * m)
     }
     bond_bpf_output(ifp, m, bpf_func);
 
-    return (dlil_output(port_ifp, 0, m, NULL, NULL, 1));
+    return (ifnet_output_raw(port_ifp, PF_BOND, m));
 
  done:
     bond_unlock();
@@ -1573,11 +1590,12 @@ static void
 bond_receive_lacpdu(struct mbuf * m, struct ifnet * port_ifp)
 {
     struct ifnet *		bond_ifp = NULL;
+    ifbond_ref			ifb;
     int				event_code = 0;
     bondport_ref		p;
 
     bond_lock();
-    if ((port_ifp->if_eflags & IFEF_BOND) == 0) {
+    if ((ifnet_eflags(port_ifp) & IFEF_BOND) == 0) {
 	goto done;
     }
     p = bond_lookup_port(port_ifp);
@@ -1587,13 +1605,31 @@ bond_receive_lacpdu(struct mbuf * m, struct ifnet * port_ifp)
     if (p->po_enabled == 0) {
 	goto done;
     }
+    ifb = p->po_bond;
+    if (ifb->ifb_mode != IF_BOND_MODE_LACP) {
+	goto done;
+    }
     bondport_receive_lacpdu(p, (lacpdu_ref)m->m_data);
-    if (ifbond_selection(p->po_bond)) {
-	event_code = (p->po_bond->ifb_active_lag == NULL) 
+    if (ifbond_selection(ifb)) {
+	event_code = (ifb->ifb_active_lag == NULL) 
 	    ? KEV_DL_LINK_OFF 
 	    : KEV_DL_LINK_ON;
 	/* XXX need to take a reference on bond_ifp */
-	bond_ifp = p->po_bond->ifb_ifp;
+	bond_ifp = ifb->ifb_ifp;
+	ifb->ifb_last_link_event = event_code;
+    }
+    else {
+	event_code = (ifb->ifb_active_lag == NULL) 
+	    ? KEV_DL_LINK_OFF 
+	    : KEV_DL_LINK_ON;
+	if (event_code != ifb->ifb_last_link_event) {
+	    if (g_bond->verbose) {
+		timestamp_printf("%s: (receive) generating LINK event\n",
+				 ifb->ifb_name);
+	    }
+	    bond_ifp = ifb->ifb_ifp;
+	    ifb->ifb_last_link_event = event_code;
+	}
     }
 
  done:
@@ -1616,12 +1652,13 @@ bond_receive_la_marker_pdu(struct mbuf * m, struct ifnet * port_ifp)
 	goto failed;
     }
     bond_lock();
-    if ((port_ifp->if_eflags & IFEF_BOND) == 0) {
+    if ((ifnet_eflags(port_ifp) & IFEF_BOND) == 0) {
 	bond_unlock();
 	goto failed;
     }
     p = bond_lookup_port(port_ifp);
-    if (p == NULL || p->po_enabled == 0) {
+    if (p == NULL || p->po_enabled == 0
+	|| p->po_bond->ifb_mode != IF_BOND_MODE_LACP) {
 	bond_unlock();
 	goto failed;
     }
@@ -1637,8 +1674,8 @@ bond_receive_la_marker_pdu(struct mbuf * m, struct ifnet * port_ifp)
 }
 
 static int
-bond_input(struct mbuf * m, char * frame_header, struct ifnet * port_ifp, 
-	   __unused u_long protocol_family, __unused int sync_ok)
+bond_input(ifnet_t port_ifp, __unused protocol_family_t protocol, mbuf_t m,
+		   char * frame_header)
 {
     bpf_packet_func		bpf_func;
     const struct ether_header *	eh_p;
@@ -1699,7 +1736,7 @@ bond_input(struct mbuf * m, char * frame_header, struct ifnet * port_ifp,
 	}
     }
     bond_lock();
-    if ((port_ifp->if_eflags & IFEF_BOND) == 0) {
+    if ((ifnet_eflags(port_ifp) & IFEF_BOND) == 0) {
 	goto done;
     }
     p = bond_lookup_port(port_ifp);
@@ -1724,7 +1761,8 @@ bond_input(struct mbuf * m, char * frame_header, struct ifnet * port_ifp,
     }
     m->m_pkthdr.rcvif = ifp;
     bond_bpf_input(ifp, m, eh_p, bpf_func);
-    dlil_input_packet(ifp, m, frame_header);
+    m->m_pkthdr.header = frame_header;
+    dlil_input_packet_list(ifp, m);
     return 0;
 
  done:
@@ -1742,7 +1780,7 @@ bondport_get_name(bondport_ref p)
 static __inline__ int
 bondport_get_index(bondport_ref p)
 {
-    return (p->po_ifp->if_index);
+    return (ifnet_index(p->po_ifp));
 }
 
 static void
@@ -1756,7 +1794,7 @@ bondport_slow_proto_transmit(bondport_ref p, packet_buffer_ref buf)
     bcopy(&slow_proto_multicast, &eh_p->ether_dhost, sizeof(eh_p->ether_dhost));
     bcopy(&p->po_saved_addr, eh_p->ether_shost, sizeof(eh_p->ether_shost));
     eh_p->ether_type = htons(IEEE8023AD_SLOW_PROTO_ETHERTYPE);
-    error = dlil_output(p->po_ifp, 0, buf, NULL, NULL, 1);
+    error = ifnet_output_raw(p->po_ifp, PF_BOND, buf);
     if (error != 0) {
 	printf("bondport_slow_proto_transmit(%s) failed %d\n",
 	       bondport_get_name(p), error);
@@ -1788,6 +1826,20 @@ bondport_timer_process_func(devtimer_ref timer,
 		    : KEV_DL_LINK_ON;
 		/* XXX need to take a reference on bond_ifp */
 		bond_ifp = p->po_bond->ifb_ifp;
+		p->po_bond->ifb_last_link_event = event_code;
+	    }
+	    else {
+		event_code = (p->po_bond->ifb_active_lag == NULL) 
+		    ? KEV_DL_LINK_OFF 
+		    : KEV_DL_LINK_ON;
+		if (event_code != p->po_bond->ifb_last_link_event) {
+		    if (g_bond->verbose) {
+			timestamp_printf("%s: (timer) generating LINK event\n",
+					 p->po_bond->ifb_name);
+		    }
+		    bond_ifp = p->po_bond->ifb_ifp;
+		    p->po_bond->ifb_last_link_event = event_code;
+		}
 	    }
 	    devtimer_release(timer);
 	    bond_unlock();
@@ -1823,7 +1875,7 @@ bondport_create(struct ifnet * port_ifp, lacp_port_priority priority,
     bzero(p, sizeof(*p));
     multicast_list_init(&p->po_multicast);
     if ((u_long)snprintf(p->po_name, sizeof(p->po_name), "%s%d",
-			 port_ifp->if_name, port_ifp->if_unit) 
+			 ifnet_name(port_ifp), ifnet_unit(port_ifp)) 
 	>= sizeof(p->po_name)) {
 	printf("if_bond: name too large\n");
 	*ret_error = EINVAL;
@@ -1836,7 +1888,7 @@ bondport_create(struct ifnet * port_ifp, lacp_port_priority priority,
 	goto failed;
     }
     /* remember the current interface MTU so it can be restored */
-    p->po_devmtu.ifdm_current = port_ifp->if_mtu;
+    p->po_devmtu.ifdm_current = ifnet_mtu(port_ifp);
     p->po_ifp = port_ifp;
     p->po_media_info = interface_media_info(port_ifp);
     p->po_current_while_timer = devtimer_create(bondport_timer_process_func, p);
@@ -1902,6 +1954,20 @@ bondport_invalidate_timers(bondport_ref p)
     devtimer_invalidate(p->po_transmit_timer);
 }
 
+/*
+ * Function: bondport_cancel_timers
+ * Purpose:
+ *   Cancel all of the timers for the bondport.
+ */
+static void
+bondport_cancel_timers(bondport_ref p)
+{
+    devtimer_cancel(p->po_current_while_timer);
+    devtimer_cancel(p->po_periodic_timer);
+    devtimer_cancel(p->po_wait_while_timer);
+    devtimer_cancel(p->po_transmit_timer);
+}
+
 static void
 bondport_free(bondport_ref p)
 {
@@ -1922,8 +1988,8 @@ bondport_free(bondport_ref p)
 static __inline__ int
 bond_device_mtu(struct ifnet * ifp, ifbond_ref ifb)
 {
-    return (((int)ifp->if_mtu > ifb->ifb_altmtu) 
-	    ? (int)ifp->if_mtu : ifb->ifb_altmtu);
+    return (((int)ifnet_mtu(ifp) > ifb->ifb_altmtu) 
+	    ? (int)ifnet_mtu(ifp) : ifb->ifb_altmtu);
 }
 
 static int
@@ -1932,12 +1998,11 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
     int				devmtu;
     int				error = 0;
     int				event_code = 0;
+    int				first = FALSE;
     ifbond_ref			ifb;
-    struct sockaddr_dl *	ifb_sdl;
     bondport_ref *		new_array = NULL;
     bondport_ref *		old_array = NULL;
     bondport_ref 		p;
-    struct sockaddr_dl *	port_sdl;
     int				progress = 0;
 
     /* pre-allocate space for new port */
@@ -1946,7 +2011,7 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
 	return (error);
     }
     bond_lock();
-    ifb = (ifbond_ref)ifp->if_private;
+    ifb = (ifbond_ref)ifnet_softc(ifp);
     if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	bond_unlock();
 	bondport_free(p);
@@ -1981,7 +2046,7 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
 	goto signal_done;
     }
     ifnet_lock_exclusive(port_ifp);
-    if ((port_ifp->if_eflags & (IFEF_VLAN | IFEF_BOND)) != 0) {
+    if ((ifnet_eflags(port_ifp) & (IFEF_VLAN | IFEF_BOND)) != 0) {
 	/* interface already has VLAN's, or is part of bond */
 	ifnet_lock_done(port_ifp);
 	error = EBUSY;
@@ -1989,49 +2054,60 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
     }
 
     /* mark the interface busy */
+    /* can't use ifnet_set_eflags because that takes the lock */
     port_ifp->if_eflags |= IFEF_BOND;
     ifnet_lock_done(port_ifp);
 
-    port_sdl = ifp_get_sdl(port_ifp);
-    ifb_sdl = ifp_get_sdl(ifp);
-
     if (TAILQ_EMPTY(&ifb->ifb_port_list)) {
-	ifp->if_hwassist = port_ifp->if_hwassist;
-	ifp->if_flags |= IFF_RUNNING;
+	ifnet_set_offload(ifp, ifnet_offload(port_ifp));
+	ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING);
 	if (ifbond_flags_lladdr(ifb) == FALSE) {
-	    /* first port added to bond determines bond's ethernet address */
-	    ether_addr_copy(LLADDR(ifb_sdl), LLADDR(port_sdl));
-	    ifb_sdl->sdl_type = IFT_ETHER;
-	    ifb_sdl->sdl_alen = ETHER_ADDR_LEN;
+	    first = TRUE;
 	}
     } else {
-	if (ifp->if_hwassist != port_ifp->if_hwassist) {
+	ifnet_offload_t		ifp_offload;
+	ifnet_offload_t		port_ifp_offload;
+
+	ifp_offload = ifnet_offload(ifp);
+	port_ifp_offload = ifnet_offload(port_ifp);
+	if (ifp_offload != port_ifp_offload) {
+	    ifnet_offload_t	offload;
+
+	    offload = ifp_offload & port_ifp_offload;
 	    printf("bond_add_interface(%s, %s)  "
-		   "hwassist values don't match 0x%x != 0x%x\n", 
+		   "hwassist values don't match 0x%x != 0x%x, using 0x%x instead\n",
 		   ifb->ifb_name, bondport_get_name(p),
-		   ifp->if_hwassist, port_ifp->if_hwassist);
+		   ifp_offload, port_ifp_offload, offload);
 	    /*
 	     * XXX
 	     * if the bond has VLAN's, we can't simply change the hwassist
 	     * field behind its back: this needs work
 	     */
-	    ifp->if_hwassist = 0;
+	    ifnet_set_offload(ifp, offload);
 	}
     }
     p->po_bond = ifb;
 
     /* remember the port's ethernet address so it can be restored */
-    ether_addr_copy(&p->po_saved_addr, LLADDR(port_sdl));
+    ether_addr_copy(&p->po_saved_addr, ifnet_lladdr(port_ifp));
 
     /* add it to the list of ports */
     TAILQ_INSERT_TAIL(&ifb->ifb_port_list, p, po_port_list);
     ifb->ifb_port_count++;
 
     /* set the default MTU */
-    if (ifp->if_mtu == 0) {
-	ifp->if_mtu = ETHERMTU;
+    if (ifnet_mtu(ifp) == 0) {
+	ifnet_set_mtu(ifp, ETHERMTU);
     }
     bond_unlock();
+
+
+    /* first port added to bond determines bond's ethernet address */
+    if (first) {
+	ifnet_set_lladdr_and_type(ifp, ifnet_lladdr(port_ifp), ETHER_ADDR_LEN,
+				  IFT_ETHER);
+    }
+
     progress |= BOND_ADD_PROGRESS_IN_LIST;
 
     /* allocate a larger distributing array */
@@ -2070,9 +2146,9 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
     }
 
     /* mark the interface up */
-	ifnet_set_flags(port_ifp, IFF_UP, IFF_UP);
+    ifnet_set_flags(port_ifp, IFF_UP, IFF_UP);
 
-    error = dlil_ioctl(0, port_ifp, SIOCSIFFLAGS, (caddr_t)NULL);
+    error = ifnet_ioctl(port_ifp, 0, SIOCSIFFLAGS, NULL);
     if (error != 0) {
 	printf("bond_add_interface(%s, %s): SIOCSIFFLAGS failed %d\n", 
 	       ifb->ifb_name, bondport_get_name(p), error);
@@ -2081,7 +2157,7 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
 
     /* re-program the port's ethernet address */
     error = if_siflladdr(port_ifp, 
-			 (const struct ether_addr *)LLADDR(ifb_sdl));
+			 (const struct ether_addr *)ifnet_lladdr(ifp));
     if (error != 0) {
 	/* port doesn't support setting the link address */
 	printf("bond_add_interface(%s, %s): if_siflladdr failed %d\n", 
@@ -2103,16 +2179,31 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
     old_array = ifb->ifb_distributing_array;
     ifb->ifb_distributing_array = new_array;
 
-    /* clear the busy state, and wakeup anyone waiting */
-    ifbond_signal(ifb, "bond_add_interface");
-    bondport_start(p);
+    if (ifb->ifb_mode == IF_BOND_MODE_LACP) {
+	bondport_start(p);
 
-    /* check if we need to generate a link status event */
-    if (ifbond_selection(ifb)) {
-	event_code = (ifb->ifb_active_lag == NULL) 
-	    ? KEV_DL_LINK_OFF 
-	    : KEV_DL_LINK_ON;
+	/* check if we need to generate a link status event */
+	if (ifbond_selection(ifb)) {
+	    event_code = (ifb->ifb_active_lag == NULL) 
+		? KEV_DL_LINK_OFF 
+		: KEV_DL_LINK_ON;
+	    ifb->ifb_last_link_event = event_code;
+	}
+    }
+    else {
+	/* are we adding the first distributing interface? */
+	if (media_active(&p->po_media_info)) {
+	    if (ifb->ifb_distributing_count == 0) {
+		ifb->ifb_last_link_event = event_code = KEV_DL_LINK_ON;
+	    }
+	    bondport_enable_distributing(p);
+	}
+	else {
+	    bondport_disable_distributing(p);
+	}
     }
+    /* clear the busy state, and wakeup anyone waiting */
+    ifbond_signal(ifb, "bond_add_interface");
     bond_unlock();
     if (event_code != 0) {
 	interface_link_event(ifp, event_code);
@@ -2125,6 +2216,11 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
  failed:
     bond_assert_lock_not_held();
 
+    /* if this was the first port to be added, clear our address */
+    if (first) {
+	ifnet_set_lladdr_and_type(ifp, NULL, 0, IFT_IEEE8023ADLAG);
+    }
+
     if (new_array != NULL) {
 	FREE(new_array, M_BOND);
     }
@@ -2146,8 +2242,8 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
 	error1 = siocsifmtu(port_ifp, p->po_devmtu.ifdm_current);
 	if (error1 != 0) {
 	    printf("bond_add_interface(%s, %s): SIOCSIFMTU %d failed %d\n", 
-		   ifb->ifb_name, bondport_get_name(p), p->po_devmtu.ifdm_current,
-		   error1);
+		   ifb->ifb_name, bondport_get_name(p),
+		   p->po_devmtu.ifdm_current, error1);
 	}
     }
     bond_lock();
@@ -2155,24 +2251,17 @@ bond_add_interface(struct ifnet * ifp, struct ifnet * port_ifp)
 	TAILQ_REMOVE(&ifb->ifb_port_list, p, po_port_list);
 	ifb->ifb_port_count--;
     }
-    ifnet_lock_exclusive(port_ifp);
-    port_ifp->if_eflags &= ~IFEF_BOND;
-    ifnet_lock_done(port_ifp);
+    ifnet_set_eflags(ifp, 0, IFEF_BOND);
     if (TAILQ_EMPTY(&ifb->ifb_port_list)) {
 	ifb->ifb_altmtu = 0;
-	ifp->if_mtu = 0;
-	ifp->if_hwassist = 0;
-	if (ifbond_flags_lladdr(ifb) == FALSE) {
-	    bzero(LLADDR(ifb_sdl), ETHER_ADDR_LEN);
-	    ifb_sdl->sdl_type = IFT_IEEE8023ADLAG;
-	    ifb_sdl->sdl_alen = 0;
-	}
+	ifnet_set_mtu(ifp, 0);
+	ifnet_set_offload(ifp, 0);
     }
 
  signal_done:
-    ifbond_release(ifb);
     ifbond_signal(ifb, "bond_add_interface");
     bond_unlock();
+    ifbond_release(ifb);
     bondport_free(p);
     return (error);
 }
@@ -2184,11 +2273,12 @@ bond_remove_interface(ifbond_ref ifb, struct ifnet * port_ifp)
     int 			error = 0;
     int				event_code = 0;
     bondport_ref		head_port;
-    struct sockaddr_dl *	ifb_sdl;
     struct ifnet *		ifp;
-    int				new_link_address = 0;
+    int				last = FALSE;
+    int				new_link_address = FALSE;
     bondport_ref 		p;
     lacp_actor_partner_state	s;
+    int				was_distributing;
 
     bond_assert_lock_held();
 
@@ -2203,68 +2293,82 @@ bond_remove_interface(ifbond_ref ifb, struct ifnet * port_ifp)
     }
 
     /* de-select it and remove it from the lists */
+    was_distributing = bondport_flags_distributing(p);
     bondport_disable_distributing(p);
-    bondport_set_selected(p, SelectedState_UNSELECTED);
-    active_lag = bondport_remove_from_LAG(p);
-    TAILQ_REMOVE(&ifb->ifb_port_list, p, po_port_list);
-    ifb->ifb_port_count--;
+    if (ifb->ifb_mode == IF_BOND_MODE_LACP) {
+	bondport_set_selected(p, SelectedState_UNSELECTED);
+	active_lag = bondport_remove_from_LAG(p);
+	/* invalidate timers here while holding the bond_lock */
+	bondport_invalidate_timers(p);
 
-    /* invalidate timers here while holding the bond_lock */
-    bondport_invalidate_timers(p);
+	/* announce that we're Individual now */
+	s = p->po_actor_state;
+	s = lacp_actor_partner_state_set_individual(s);
+	s = lacp_actor_partner_state_set_not_collecting(s);
+	s = lacp_actor_partner_state_set_not_distributing(s);
+	s = lacp_actor_partner_state_set_out_of_sync(s);
+	p->po_actor_state = s;
+	bondport_flags_set_ntt(p);
+    }
 
-    /* announce that we're Individual now */
-    s = p->po_actor_state;
-    s = lacp_actor_partner_state_set_individual(s);
-    s = lacp_actor_partner_state_set_not_collecting(s);
-    s = lacp_actor_partner_state_set_not_distributing(s);
-    s = lacp_actor_partner_state_set_out_of_sync(s);
-    p->po_actor_state = s;
-    bondport_flags_set_ntt(p);
+    TAILQ_REMOVE(&ifb->ifb_port_list, p, po_port_list);
+    ifb->ifb_port_count--;
 
     ifp = ifb->ifb_ifp;
-    ifb_sdl = ifp_get_sdl(ifp);
     head_port = TAILQ_FIRST(&ifb->ifb_port_list);
     if (head_port == NULL) {
-	ifp->if_flags &= ~IFF_RUNNING;
+	ifnet_set_flags(ifp, 0, IFF_RUNNING);
 	if (ifbond_flags_lladdr(ifb) == FALSE) {
-	    ifb_sdl->sdl_type = IFT_IEEE8023ADLAG;
-	    ifb_sdl->sdl_alen = 0;
-	    bzero(LLADDR(ifb_sdl), ETHER_ADDR_LEN);
+	    last = TRUE;
 	}
-	ifp->if_hwassist = 0;
-	ifp->if_mtu = 0;
+	ifnet_set_offload(ifp, 0);
+	ifnet_set_mtu(ifp, 0);
 	ifb->ifb_altmtu = 0;
     } else if (ifbond_flags_lladdr(ifb) == FALSE
-	       && bcmp(&p->po_saved_addr, LLADDR(ifb_sdl), 
+	       && bcmp(&p->po_saved_addr, ifnet_lladdr(ifp), 
 		       ETHER_ADDR_LEN) == 0) {
-	/* this port gave the bond its ethernet address, switch to new one */
-	ether_addr_copy(LLADDR(ifb_sdl), &head_port->po_saved_addr);
-	ifb_sdl->sdl_type = IFT_ETHER;
-	ifb_sdl->sdl_alen = ETHER_ADDR_LEN;
-	new_link_address = 1;
+	new_link_address = TRUE;
     }
     /* check if we need to generate a link status event */
-    if (ifbond_selection(ifb) || active_lag) {
-	event_code = (ifb->ifb_active_lag == NULL) 
-	    ? KEV_DL_LINK_OFF 
-	    : KEV_DL_LINK_ON;
+    if (ifb->ifb_mode == IF_BOND_MODE_LACP ) {
+	if (ifbond_selection(ifb) || active_lag) {
+	    event_code = (ifb->ifb_active_lag == NULL) 
+		? KEV_DL_LINK_OFF 
+		: KEV_DL_LINK_ON;
+	    ifb->ifb_last_link_event = event_code;
+	}
+	bondport_transmit_machine(p, LAEventStart,
+				  TRANSMIT_MACHINE_TX_IMMEDIATE);
+    }
+    else {
+	/* are we removing the last distributing interface? */
+	if (was_distributing && ifb->ifb_distributing_count == 0) {
+	    ifb->ifb_last_link_event = event_code = KEV_DL_LINK_OFF;
+	}
     }
-    bond_unlock();
 
-    bondport_transmit_machine(p, LAEventStart, (void *)1);
+    bond_unlock();
 
-    if (new_link_address) {
+    if (last) {
+	ifnet_set_lladdr_and_type(ifp, NULL, 0, IFT_IEEE8023ADLAG);
+    }
+    else if (new_link_address) {
 	struct ifnet *	scan_ifp;
 	bondport_ref	scan_port;
 
 	/* ifbond_wait() allows port list traversal without holding the lock */
 
+	/* this port gave the bond its ethernet address, switch to new one */
+	ifnet_set_lladdr_and_type(ifp,
+				  &head_port->po_saved_addr, ETHER_ADDR_LEN,
+				  IFT_ETHER);
+
 	/* re-program each port with the new link address */
 	TAILQ_FOREACH(scan_port, &ifb->ifb_port_list, po_port_list) {
 	    scan_ifp = scan_port->po_ifp;
 
 	    error = if_siflladdr(scan_ifp,
-				 (const struct ether_addr *) LLADDR(ifb_sdl));
+				 (const struct ether_addr *) ifnet_lladdr(ifp));
 	    if (error != 0) {
 		printf("bond_remove_interface(%s, %s): "
 		       "if_siflladdr (%s) failed %d\n", 
@@ -2298,15 +2402,118 @@ bond_remove_interface(ifbond_ref ifb, struct ifnet * port_ifp)
     }
 
     bond_lock();
-    ifbond_release(ifb);
     bondport_free(p);
-    ifnet_lock_exclusive(port_ifp);
-    port_ifp->if_eflags &= ~IFEF_BOND;
-    ifnet_lock_done(port_ifp);
+    ifnet_set_eflags(port_ifp, 0, IFEF_BOND);
+    /* release this bondport's reference to the ifbond */
+    ifbond_release(ifb);
 
  signal_done:
     ifbond_signal(ifb, "bond_remove_interface");
-    ifbond_release(ifb); /* a second release for the second reference */
+    ifbond_release(ifb);
+    return (error);
+}
+
+static void
+bond_set_lacp_mode(ifbond_ref ifb)
+{
+    bondport_ref		p;
+
+    TAILQ_FOREACH(p, &ifb->ifb_port_list, po_port_list) {
+	bondport_disable_distributing(p);
+	bondport_start(p);
+    }
+    return;
+}
+
+static void
+bond_set_static_mode(ifbond_ref ifb)
+{
+    bondport_ref		p;
+    lacp_actor_partner_state	s;
+
+    TAILQ_FOREACH(p, &ifb->ifb_port_list, po_port_list) {
+	bondport_disable_distributing(p);
+	bondport_set_selected(p, SelectedState_UNSELECTED);
+	(void)bondport_remove_from_LAG(p);
+	bondport_cancel_timers(p);
+
+	/* announce that we're Individual now */
+	s = p->po_actor_state;
+	s = lacp_actor_partner_state_set_individual(s);
+	s = lacp_actor_partner_state_set_not_collecting(s);
+	s = lacp_actor_partner_state_set_not_distributing(s);
+	s = lacp_actor_partner_state_set_out_of_sync(s);
+	p->po_actor_state = s;
+	bondport_flags_set_ntt(p);
+	bondport_transmit_machine(p, LAEventStart,
+				  TRANSMIT_MACHINE_TX_IMMEDIATE);
+	/* clear state */
+	p->po_actor_state = 0;
+	bzero(&p->po_partner_state, sizeof(p->po_partner_state));
+
+	if (media_active(&p->po_media_info)) {
+	    bondport_enable_distributing(p);
+	}
+	else {
+	    bondport_disable_distributing(p);
+	}
+    }
+    return;
+}
+
+static int
+bond_set_mode(struct ifnet * ifp, int mode)
+{
+    int				error = 0;
+    int				event_code = 0;
+    ifbond_ref			ifb;
+
+    bond_lock();
+    ifb = (ifbond_ref)ifnet_softc(ifp);
+    if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
+	bond_unlock();
+	return ((ifb == NULL) ? EOPNOTSUPP : EBUSY);
+    }
+    if (ifb->ifb_mode == mode) {
+	bond_unlock();
+	return (0);
+    }
+
+    ifbond_retain(ifb);
+    ifbond_wait(ifb, "bond_set_mode");
+
+    /* verify (again) that the mode is actually different */
+    if (ifb->ifb_mode == mode) {
+	/* nothing to do */
+	goto signal_done;
+    }
+
+    ifb->ifb_mode = mode;
+    if (mode == IF_BOND_MODE_LACP) {
+	bond_set_lacp_mode(ifb);
+	
+	/* check if we need to generate a link status event */
+	if (ifbond_selection(ifb)) {
+	    event_code = (ifb->ifb_active_lag == NULL) 
+		? KEV_DL_LINK_OFF 
+		: KEV_DL_LINK_ON;
+	}
+    } else {
+	bond_set_static_mode(ifb);
+	event_code = (ifb->ifb_distributing_count == 0) 
+	    ? KEV_DL_LINK_OFF 
+	    : KEV_DL_LINK_ON;
+    }
+    ifb->ifb_last_link_event = event_code;
+
+ signal_done:
+    ifbond_signal(ifb, "bond_set_mode");
+    bond_unlock();
+    ifbond_release(ifb);
+
+    if (event_code != 0) {
+	interface_link_event(ifp, event_code);
+    }
     return (error);
 }
 
@@ -2325,10 +2532,11 @@ bond_get_status(ifbond_ref ifb, struct if_bond_req * ibr_p, user_addr_t datap)
 	return (EINVAL);
     }
     ibsr->ibsr_key = ifb->ifb_key;
+    ibsr->ibsr_mode = ifb->ifb_mode;
     ibsr->ibsr_total = ifb->ifb_port_count;
     dst = proc_is64bit(current_proc())
 	? ibsr->ibsr_ibsru.ibsru_buffer64
-	: CAST_USER_ADDR_T(ibsr->ibsr_ibsru.ibsru_buffer32);
+	: CAST_USER_ADDR_T(ibsr->ibsr_ibsru.ibsru_buffer);
     if (dst == USER_ADDR_NULL) {
 	/* just want to know how many there are */
 	goto done;
@@ -2348,16 +2556,23 @@ bond_get_status(ifbond_ref ifb, struct if_bond_req * ibr_p, user_addr_t datap)
 	bzero(&ibs, sizeof(ibs));
 	strncpy(ibs.ibs_if_name, port->po_name, sizeof(ibs.ibs_if_name));
 	ibs.ibs_port_priority = port->po_priority;
-	ibs.ibs_state = port->po_actor_state;
-	ibs.ibs_selected_state = port->po_selected;
-	ps = &port->po_partner_state;
-	ibps_p = &ibs.ibs_partner_state;
-	ibps_p->ibps_system = ps->ps_lag_info.li_system;
-	ibps_p->ibps_system_priority = ps->ps_lag_info.li_system_priority;
-	ibps_p->ibps_key = ps->ps_lag_info.li_key;
-	ibps_p->ibps_port = ps->ps_port;
-	ibps_p->ibps_port_priority = ps->ps_port_priority;
-	ibps_p->ibps_state = ps->ps_state;
+	if (ifb->ifb_mode == IF_BOND_MODE_LACP) {
+	    ibs.ibs_state = port->po_actor_state;
+	    ibs.ibs_selected_state = port->po_selected;
+	    ps = &port->po_partner_state;
+	    ibps_p = &ibs.ibs_partner_state;
+	    ibps_p->ibps_system = ps->ps_lag_info.li_system;
+	    ibps_p->ibps_system_priority = ps->ps_lag_info.li_system_priority;
+	    ibps_p->ibps_key = ps->ps_lag_info.li_key;
+	    ibps_p->ibps_port = ps->ps_port;
+	    ibps_p->ibps_port_priority = ps->ps_port_priority;
+	    ibps_p->ibps_state = ps->ps_state;
+	}
+	else {
+	    /* fake the selected information */
+	    ibs.ibs_selected_state = bondport_flags_distributing(port)
+		? SelectedState_SELECTED : SelectedState_UNSELECTED;
+	}
 	error = copyout(&ibs, dst, sizeof(ibs));
 	if (error != 0) {
 	    break;
@@ -2381,10 +2596,10 @@ bond_set_promisc(__unused struct ifnet *ifp)
 {
     int 		error = 0;
 #if 0
-    ifbond_ref	ifb = ifp->if_private;
+    ifbond_ref	ifb = ifnet_softc(ifp);
 
 
-    if ((ifp->if_flags & IFF_PROMISC) != 0) {
+    if ((ifnet_flags(ifp) & IFF_PROMISC) != 0) {
 	if ((ifb->ifb_flags & IFBF_PROMISC) == 0) {
 	    error = ifnet_set_promiscuous(ifb->ifb_p, 1);
 	    if (error == 0)
@@ -2454,7 +2669,7 @@ bond_set_mtu(struct ifnet * ifp, int mtu, int isdevmtu)
     int			old_max;
 
     bond_lock();
-    ifb = (ifbond_ref)ifp->if_private;
+    ifb = (ifbond_ref)ifnet_softc(ifp);
     if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	error = (ifb == NULL) ? EOPNOTSUPP : EBUSY;
 	goto done;
@@ -2463,7 +2678,7 @@ bond_set_mtu(struct ifnet * ifp, int mtu, int isdevmtu)
     ifbond_wait(ifb, "bond_set_mtu");
 
     /* check again */
-    if (ifp->if_private == NULL || ifbond_flags_if_detaching(ifb)) {
+    if (ifnet_softc(ifp) == NULL || ifbond_flags_if_detaching(ifb)) {
 	error = EBUSY;
 	goto signal_done;
     }
@@ -2478,13 +2693,13 @@ bond_set_mtu(struct ifnet * ifp, int mtu, int isdevmtu)
 	goto signal_done;
     }
     if (isdevmtu) {
-	new_max = (mtu > (int)ifp->if_mtu) ? mtu : (int)ifp->if_mtu;
+	new_max = (mtu > (int)ifnet_mtu(ifp)) ? mtu : (int)ifnet_mtu(ifp);
     }
     else {
 	new_max = (mtu > ifb->ifb_altmtu) ? mtu : ifb->ifb_altmtu;
     }
-    old_max = ((int)ifp->if_mtu > ifb->ifb_altmtu) 
-	? (int)ifp->if_mtu : ifb->ifb_altmtu;
+    old_max = ((int)ifnet_mtu(ifp) > ifb->ifb_altmtu) 
+	? (int)ifnet_mtu(ifp) : ifb->ifb_altmtu;
     if (new_max != old_max) {
 	/* we can safely walk the list of port without the lock held */
 	bond_unlock();
@@ -2500,7 +2715,7 @@ bond_set_mtu(struct ifnet * ifp, int mtu, int isdevmtu)
 	    ifb->ifb_altmtu = mtu;
 	}
 	else {
-	    ifp->if_mtu = mtu;
+		ifnet_set_mtu(ifp, mtu);
 	}
     }
 
@@ -2525,7 +2740,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
     struct ifnet *	port_ifp = NULL;
     user_addr_t		user_addr;
 
-    if (ifp->if_type != IFT_IEEE8023ADLAG) {
+    if (ifnet_type(ifp) != IFT_IEEE8023ADLAG) {
 	return (EOPNOTSUPP);
     }
     ifr = (struct ifreq *)data;
@@ -2539,7 +2754,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
     case SIOCGIFMEDIA64:
     case SIOCGIFMEDIA:
 	bond_lock();
-	ifb = (ifbond_ref)ifp->if_private;
+	ifb = (ifbond_ref)ifnet_softc(ifp);
 	if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	    bond_unlock();
 	    return (ifb == NULL ? EOPNOTSUPP : EBUSY);
@@ -2550,12 +2765,19 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 	ifmr->ifm_status = IFM_AVALID;
 	ifmr->ifm_active = IFM_ETHER;
 	ifmr->ifm_count = 1;
-	if (ifb->ifb_active_lag != NULL) {
-	    ifmr->ifm_active = ifb->ifb_active_lag->lag_active_media;
+	if (ifb->ifb_mode == IF_BOND_MODE_LACP) {
+	    if (ifb->ifb_active_lag != NULL) {
+		ifmr->ifm_active = ifb->ifb_active_lag->lag_active_media;
+		ifmr->ifm_status |= IFM_ACTIVE;
+	    }
+	}
+	else if (ifb->ifb_distributing_count > 0) {
+	    ifmr->ifm_active
+		= ifb->ifb_distributing_array[0]->po_media_info.mi_active;
 	    ifmr->ifm_status |= IFM_ACTIVE;
 	}
 	bond_unlock();
-	user_addr = (cmd == SIOCGIFMEDIA64)
+	user_addr = proc_is64bit(current_proc())
 	    ? ifmr->ifm_ifmu.ifmu_ulist64
 	    : CAST_USER_ADDR_T(ifmr->ifm_ifmu.ifmu_ulist32);
 	if (user_addr != USER_ADDR_NULL) {
@@ -2572,7 +2794,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 
     case SIOCGIFDEVMTU:
 	bond_lock();
-	ifb = (ifbond_ref)ifp->if_private;
+	ifb = (ifbond_ref)ifnet_softc(ifp);
 	if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	    bond_unlock();
 	    error = (ifb == NULL) ? EOPNOTSUPP : EBUSY;
@@ -2586,7 +2808,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 
     case SIOCGIFALTMTU:
 	bond_lock();
-	ifb = (ifbond_ref)ifp->if_private;
+	ifb = (ifbond_ref)ifnet_softc(ifp);
 	if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	    bond_unlock();
 	    error = (ifb == NULL) ? EOPNOTSUPP : EBUSY;
@@ -2620,12 +2842,13 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 		error = ENXIO;
 		break;
 	    }
-	    if (port_ifp->if_type != IFT_ETHER) {
+	    if (ifnet_type(port_ifp) != IFT_ETHER) {
 		error = EPROTONOSUPPORT;
 		break;
 	    }
 	    break;
 	case IF_BOND_OP_SET_VERBOSE:
+	case IF_BOND_OP_SET_MODE:
 	    break;
 	default:
 	    error = EOPNOTSUPP;
@@ -2640,7 +2863,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 	    break;
 	case IF_BOND_OP_REMOVE_INTERFACE:
 	    bond_lock();
-	    ifb = (ifbond_ref)ifp->if_private;
+	    ifb = (ifbond_ref)ifnet_softc(ifp);
 	    if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 		bond_unlock();
 		return (ifb == NULL ? EOPNOTSUPP : EBUSY);
@@ -2658,8 +2881,22 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 	    g_bond->verbose = ibr.ibr_ibru.ibru_int_val;
 	    bond_unlock();
 	    break;
+	case IF_BOND_OP_SET_MODE:
+	    switch (ibr.ibr_ibru.ibru_int_val) {
+	    case IF_BOND_MODE_LACP:
+	    case IF_BOND_MODE_STATIC:
+		break;
+	    default:
+		error = EINVAL;
+		break;
+	    }
+	    if (error != 0) {
+		break;
+	    }
+	    error = bond_set_mode(ifp, ibr.ibr_ibru.ibru_int_val);
+	    break;
 	}
-	break;
+	break; /* SIOCSIFBOND */
 		
     case SIOCGIFBOND:
 	user_addr = proc_is64bit(current_proc())
@@ -2679,7 +2916,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 	    break;
 	}
 	bond_lock();
-	ifb = (ifbond_ref)ifp->if_private;
+	ifb = (ifbond_ref)ifnet_softc(ifp);
 	if (ifb == NULL || ifbond_flags_if_detaching(ifb)) {
 	    bond_unlock();
 	    return (ifb == NULL ? EOPNOTSUPP : EBUSY);
@@ -2690,7 +2927,7 @@ bond_ioctl(struct ifnet *ifp, u_int32_t cmd, void * data)
 	    break;
 	}
 	bond_unlock();
-	break;
+	break; /* SIOCGIFBOND */
 		
     case SIOCSIFLLADDR:
 	error = EOPNOTSUPP;
@@ -2722,25 +2959,27 @@ bond_if_free(struct ifnet * ifp)
 	return;
     }
     bond_lock();
-    ifb = (ifbond_ref)ifp->if_private;
+    ifb = (ifbond_ref)ifnet_softc(ifp);
     if (ifb == NULL) {
 	bond_unlock();
 	return;
     }
-    ifp->if_private = NULL;
     ifbond_release(ifb);
     bond_unlock();
-    dlil_if_release(ifp);
+    ifnet_release(ifp);
     return;
 }
 
 static void
-bond_event(struct ifnet * port_ifp, struct kev_msg * event)
+bond_event(struct ifnet * port_ifp, __unused protocol_family_t protocol,
+		   const struct kev_msg * event)
 {
     struct ifnet *	bond_ifp = NULL;
     int			event_code = 0;
+    ifbond_ref		ifb;
+    int			old_distributing_count;
     bondport_ref	p;
-    struct media_info	media_info;
+    struct media_info	media_info = { 0, 0};
 
     if (event->vendor_code != KEV_VENDOR_APPLE 
 	|| event->kev_class != KEV_NETWORK_CLASS 
@@ -2763,9 +3002,11 @@ bond_event(struct ifnet * port_ifp, struct kev_msg * event)
 	bond_unlock();
 	return;
     }
+    ifb = p->po_bond;
+    old_distributing_count = ifb->ifb_distributing_count;
     switch (event->event_code) {
     case KEV_DL_IF_DETACHING:
-	bond_remove_interface(p->po_bond, p->po_ifp);
+	bond_remove_interface(ifb, p->po_ifp);
 	break;
     case KEV_DL_LINK_OFF:
     case KEV_DL_LINK_ON:
@@ -2776,13 +3017,48 @@ bond_event(struct ifnet * port_ifp, struct kev_msg * event)
 	break;
     }
     /* generate a link-event */
-    if (ifbond_selection(p->po_bond)) {
-	event_code = (p->po_bond->ifb_active_lag == NULL) 
-	    ? KEV_DL_LINK_OFF 
-	    : KEV_DL_LINK_ON;
-	/* XXX need to take a reference on bond_ifp */
-	bond_ifp = p->po_bond->ifb_ifp;
+    if (ifb->ifb_mode == IF_BOND_MODE_LACP) {
+	if (ifbond_selection(ifb)) {
+	    event_code = (ifb->ifb_active_lag == NULL) 
+		? KEV_DL_LINK_OFF 
+		: KEV_DL_LINK_ON;
+	    /* XXX need to take a reference on bond_ifp */
+	    bond_ifp = ifb->ifb_ifp;
+	    ifb->ifb_last_link_event = event_code;
+	}
+	else {
+	    event_code = (ifb->ifb_active_lag == NULL) 
+		? KEV_DL_LINK_OFF 
+		: KEV_DL_LINK_ON;
+	    if (event_code != ifb->ifb_last_link_event) {
+		if (g_bond->verbose) {
+		    timestamp_printf("%s: (event) generating LINK event\n",
+				     ifb->ifb_name);
+		}
+		bond_ifp = ifb->ifb_ifp;
+		ifb->ifb_last_link_event = event_code;
+	    }
+	}
     }
+    else {
+	/*
+	 * if the distributing array membership changed from 0 <-> !0
+	 * generate a link event
+	 */
+	if (old_distributing_count == 0 
+	    && ifb->ifb_distributing_count != 0) {
+	    event_code = KEV_DL_LINK_ON;
+	}
+	else if (old_distributing_count != 0 
+		 && ifb->ifb_distributing_count == 0) {
+	    event_code = KEV_DL_LINK_OFF;
+	}
+	if (event_code != 0 && event_code != ifb->ifb_last_link_event) {
+	    bond_ifp = ifb->ifb_ifp;
+	    ifb->ifb_last_link_event = event_code;
+	}
+    }
+
     bond_unlock();
     if (bond_ifp != NULL) {
 	interface_link_event(bond_ifp, event_code);
@@ -2804,10 +3080,10 @@ interface_link_event(struct ifnet * ifp, u_long event_code)
     event.header.kev_class     = KEV_NETWORK_CLASS;
     event.header.kev_subclass  = KEV_DL_SUBCLASS;
     event.header.event_code    = event_code;
-    event.header.event_data[0] = ifp->if_family;
-    event.unit                 = (u_long) ifp->if_unit;
-    strncpy(event.if_name, ifp->if_name, IFNAMSIZ);
-    dlil_event(ifp, &event.header);
+    event.header.event_data[0] = ifnet_family(ifp);
+    event.unit                 = (u_long) ifnet_unit(ifp);
+    strncpy(event.if_name, ifnet_name(ifp), IFNAMSIZ);
+    ifnet_event(ifp, &event.header);
     return;
 }
 
@@ -2824,21 +3100,17 @@ interface_link_event(struct ifnet * ifp, u_long event_code)
 static int
 bond_attach_protocol(struct ifnet *ifp)
 {
-    int  			error;
-    struct dlil_proto_reg_str   reg;
+    int								error;
+    struct ifnet_attach_proto_param	reg;
 	
     bzero(&reg, sizeof(reg));
-    TAILQ_INIT(&reg.demux_desc_head);
-    reg.interface_family = ifp->if_family;
-    reg.unit_number = ifp->if_unit;
     reg.input = bond_input;
     reg.event = bond_event;
-    reg.protocol_family = PF_BOND;
 	
-    error = dlil_attach_protocol(&reg);
+    error = ifnet_attach_protocol(ifp, PF_BOND, &reg);
     if (error) {
-	printf("bond over %s%d: dlil_attach_protocol failed, %d\n",
-	       ifp->if_name, ifp->if_unit, error);
+	printf("bond over %s%d: ifnet_attach_protocol failed, %d\n",
+	       ifnet_name(ifp), ifnet_unit(ifp), error);
     }
     return (error);
 }
@@ -2853,10 +3125,10 @@ bond_detach_protocol(struct ifnet *ifp)
 {
     int         error;
 
-    error = dlil_detach_protocol(ifp, PF_BOND);
+    error = ifnet_detach_protocol(ifp, PF_BOND);
     if (error) {
-	printf("bond over %s%d: dlil_detach_protocol failed, %d\n",
-	       ifp->if_name, ifp->if_unit, error);
+	printf("bond over %s%d: ifnet_detach_protocol failed, %d\n",
+	       ifnet_name(ifp), ifnet_unit(ifp), error);
     }
     return (error);
 }
@@ -2864,53 +3136,41 @@ bond_detach_protocol(struct ifnet *ifp)
 /*
  * DLIL interface family functions
  */
-extern int ether_add_if(struct ifnet *ifp);
-extern int ether_del_if(struct ifnet *ifp);
-extern int ether_init_if(struct ifnet *ifp);
-extern int ether_add_proto_old(struct ifnet *ifp, u_long protocol_family,
-			   struct ddesc_head_str *desc_head);
-
-extern int ether_attach_inet(struct ifnet *ifp, u_long protocol_family);
-extern int ether_detach_inet(struct ifnet *ifp, u_long protocol_family);
-extern int ether_attach_inet6(struct ifnet *ifp, u_long protocol_family);
-extern int ether_detach_inet6(struct ifnet *ifp, u_long protocol_family);
+extern int ether_attach_inet(ifnet_t ifp, protocol_family_t protocol_family);
+extern void ether_detach_inet(ifnet_t ifp, protocol_family_t protocol_family);
+extern int ether_attach_inet6(ifnet_t ifp, protocol_family_t protocol_family);
+extern void ether_detach_inet6(ifnet_t ifp, protocol_family_t protocol_family);
+extern int ether_attach_at(ifnet_t ifp, protocol_family_t protocol_family);
+extern void ether_detach_at(ifnet_t ifp, protocol_family_t protocol_family);
 
 __private_extern__ int
 bond_family_init(void)
 {
     int error=0;
-    struct dlil_ifmod_reg_str  ifmod_reg;
-    
-    bzero(&ifmod_reg, sizeof(ifmod_reg));
-    ifmod_reg.add_if = ether_add_if;
-    ifmod_reg.del_if = ether_del_if;
-    ifmod_reg.init_if = NULL;
-    ifmod_reg.add_proto = ether_add_proto_old;
-    ifmod_reg.del_proto = ether_del_proto;
-    ifmod_reg.ifmod_ioctl = ether_ioctl;
-    ifmod_reg.shutdown = NULL;
-
-    if (dlil_reg_if_modules(APPLE_IF_FAM_BOND, &ifmod_reg)) {
-	printf("WARNING: bond_family_init -- "
-	       "Can't register if family modules\n");
-	error = EIO;
-	goto done;
-    }
 
-    error = dlil_reg_proto_module(PF_INET, APPLE_IF_FAM_BOND, 
+    error = proto_register_plumber(PF_INET, APPLE_IF_FAM_BOND, 
 				  ether_attach_inet, 
 				  ether_detach_inet);
     if (error != 0) {
-	printf("bond: dlil_reg_proto_module failed for AF_INET6 error=%d\n",
+	printf("bond: proto_register_plumber failed for AF_INET error=%d\n",
 	       error);
 	goto done;
     }
-
-    error = dlil_reg_proto_module(PF_INET6, APPLE_IF_FAM_BOND, 
+#if INET6
+    error = proto_register_plumber(PF_INET6, APPLE_IF_FAM_BOND, 
 				  ether_attach_inet6, 
 				  ether_detach_inet6);
     if (error != 0) {
-	printf("bond: dlil_reg_proto_module failed for AF_INET6 error=%d\n",
+	printf("bond: proto_register_plumber failed for AF_INET6 error=%d\n",
+	       error);
+	goto done;
+    }
+#endif
+    error = proto_register_plumber(PF_APPLETALK, APPLE_IF_FAM_BOND, 
+				  ether_attach_at, 
+				  ether_detach_at);
+    if (error != 0) {
+	printf("bond: proto_register_plumber failed for AppleTalk error=%d\n",
 	       error);
 	goto done;
     }
@@ -3283,24 +3543,33 @@ bondport_link_status_changed(bondport_ref p)
 	    timestamp_printf("[%s] Link DOWN\n", bondport_get_name(p));
 	}
     }
-    if (media_active(&p->po_media_info)
-	&& bond->ifb_active_lag != NULL
-	&& p->po_lag == bond->ifb_active_lag
-	&& p->po_selected != SelectedState_UNSELECTED) {
-	if (media_speed(&p->po_media_info) != p->po_lag->lag_active_media) {
-	    if (g_bond->verbose) {
-		timestamp_printf("[%s] Port speed %d differs from LAG %d\n",
-				 bondport_get_name(p),
-				 media_speed(&p->po_media_info),
-				 link_speed(p->po_lag->lag_active_media));
+    if (bond->ifb_mode == IF_BOND_MODE_LACP) {
+	if (media_active(&p->po_media_info)
+	    && bond->ifb_active_lag != NULL
+	    && p->po_lag == bond->ifb_active_lag
+	    && p->po_selected != SelectedState_UNSELECTED) {
+	    if (media_speed(&p->po_media_info) != p->po_lag->lag_active_media) {
+		if (g_bond->verbose) {
+		    timestamp_printf("[%s] Port speed %d differs from LAG %d\n",
+				     bondport_get_name(p),
+				     media_speed(&p->po_media_info),
+				     link_speed(p->po_lag->lag_active_media));
+		}
+		bondport_set_selected(p, SelectedState_UNSELECTED);
 	    }
-	    bondport_set_selected(p, SelectedState_UNSELECTED);
+	}
+	bondport_receive_machine(p, LAEventMediaChange, NULL);
+	bondport_mux_machine(p, LAEventMediaChange, NULL);
+	bondport_periodic_transmit_machine(p, LAEventMediaChange, NULL);
+    }
+    else {
+	if (media_active(&p->po_media_info)) {
+	    bondport_enable_distributing(p);
+	}
+	else {
+	    bondport_disable_distributing(p);
 	}
     }
-    bondport_receive_machine(p, LAEventMediaChange, NULL);
-    bondport_mux_machine(p, LAEventMediaChange, NULL);
-    bondport_periodic_transmit_machine(p, LAEventMediaChange, NULL);
-
     return;
 }
 
@@ -4147,7 +4416,7 @@ bondport_transmit_machine(bondport_ref p, LAEvent event,
 	if (p->po_periodic_interval == 0 || bondport_flags_ntt(p) == 0) {
 	    break;
 	}
-	if (event_data != NULL) {
+	if (event_data == TRANSMIT_MACHINE_TX_IMMEDIATE) {
 	    /* we're going away, transmit the packet no matter what */
 	}
 	else if (bondport_can_transmit(p, devtimer_current_secs(),
@@ -4166,7 +4435,7 @@ bondport_transmit_machine(bondport_ref p, LAEvent event,
 		if (g_bond->verbose > 0) {
 		    timestamp_printf("[%s] Transmit Timer Deadline %d secs\n",
 				     bondport_get_name(p),
-				     next_tick_time.tv_sec);
+				     (int)next_tick_time.tv_sec);
 		}
 	    }
 	    break;
diff --git a/bsd/net/if_bond_var.h b/bsd/net/if_bond_var.h
index f07728fbf..87f9d5bd5 100644
--- a/bsd/net/if_bond_var.h
+++ b/bsd/net/if_bond_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _NET_IF_BOND_VAR_H_
@@ -27,10 +33,16 @@
 
 #include <net/lacp.h>
 
+#pragma pack(4)
+
 #define IF_BOND_OP_ADD_INTERFACE		1
 #define IF_BOND_OP_REMOVE_INTERFACE		2
 #define IF_BOND_OP_GET_STATUS			3
 #define IF_BOND_OP_SET_VERBOSE			4
+#define IF_BOND_OP_SET_MODE			5
+
+#define IF_BOND_MODE_LACP			0
+#define IF_BOND_MODE_STATIC			1
 
 struct if_bond_partner_state {
     lacp_system 		ibps_system;
@@ -62,19 +74,15 @@ struct if_bond_status_req {
     int		ibsr_total;	/* returned number of struct if_bond_status's */
     int		ibsr_count;	/* number that will fit in ibsr_buffer */
     union {			/* buffer to hold if_bond_status's */
-	char *		ibsru_buffer32;
+	void *		ibsru_buffer;
 	u_int64_t	ibsru_buffer64;
     } ibsr_ibsru;
     lacp_key	ibsr_key;	/* returned */
-    u_int16_t	ibsr_reserved0;	/* for future use */
+    u_int8_t	ibsr_mode;	/* returned (IF_BOND_MODE_{LACP, STATIC}) */
+    u_int8_t	ibsr_reserved0;	/* for future use */
     u_int32_t	ibsr_reserved[3];/* for future use */
 };
-
-#if defined(__LP64__)
-#define ibsr_buffer	ibsr_ibsru.ibsru_buffer64
-#else
-#define ibsr_buffer	ibsr_ibsru.ibsru_buffer32
-#endif
+#define ibsr_buffer	ibsr_ibsru.ibsru_buffer
 
 struct if_bond_req {
     u_int32_t	ibr_op;				/* operation */
@@ -85,8 +93,10 @@ struct if_bond_req {
     } ibr_ibru;
 };
 
+#pragma pack()
+
 #ifdef KERNEL_PRIVATE
-int bond_family_init(void);
+int bond_family_init(void) __attribute__((section("__TEXT, initcode")));
 #endif KERNEL_PRIVATE
 
 #endif /* _NET_IF_BOND_VAR_H_ */
diff --git a/bsd/net/if_disc.c b/bsd/net/if_disc.c
index b5e751f41..00531fdff 100644
--- a/bsd/net/if_disc.c
+++ b/bsd/net/if_disc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -147,21 +153,10 @@ discoutput(ifp, m, dst, rt)
 	}
 
 	if (discif.if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
+		/* We need to prepend the address family as a four byte field. */
 		u_int af = dst->sa_family;
 
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&af;
-
-		bpf_mtap(&discif, &m0);
+		bpf_tap_out(ifp, 0, m, &af, sizeof(af));
 	}
 	m->m_pkthdr.rcvif = ifp;
 
diff --git a/bsd/net/if_dl.h b/bsd/net/if_dl.h
index 51e9262e0..1dd0d8c07 100644
--- a/bsd/net/if_dl.h
+++ b/bsd/net/if_dl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1993
diff --git a/bsd/net/if_dummy.c b/bsd/net/if_dummy.c
index c9b330fa3..50dac08da 100644
--- a/bsd/net/if_dummy.c
+++ b/bsd/net/if_dummy.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -87,13 +93,8 @@
 #include <netinet/ip.h>
 #endif
 
-#if IPX
-#include <netipx/ipx.h>
-#include <netipx/ipx_if.h>
-#endif
-
 #if INET6
-#ifndef INET
+#if !INET
 #include <netinet/in.h>
 #endif
 #include <netinet6/in6_var.h>
@@ -183,25 +184,10 @@ dummyoutput(ifp, m, dst, rt)
 	}
 
 	if (ifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
+		/* We need to prepend the address family as a four byte field. */
 		u_int af = dst->sa_family;
 
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&af;
-
-#ifdef HAVE_OLD_BPF
-		bpf_mtap(ifp, &m0);
-#else
-		bpf_mtap(ifp->if_bpf, &m0);
-#endif
+		bpf_tap_out(ifp, 0, m, &af, sizeof(af));
 	}
 #endif
 	m->m_pkthdr.rcvif = ifp;
diff --git a/bsd/net/if_ether.h b/bsd/net/if_ether.h
index a8e1bce0d..44e73fa71 100644
--- a/bsd/net/if_ether.h
+++ b/bsd/net/if_ether.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _NET_IF_ETHER_H
@@ -27,6 +33,8 @@
 #include <net/kpi_interface.h>
 
 __BEGIN_DECLS
+/* Not exported */
+extern int ether_family_init(void) __attribute__((section("__TEXT, initcode")));
 
 /*
  * These functions may be used for an interface emulating an ethernet
diff --git a/bsd/net/if_ethersubr.c b/bsd/net/if_ethersubr.c
index 205ec4392..df6d5c284 100644
--- a/bsd/net/if_ethersubr.c
+++ b/bsd/net/if_ethersubr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -184,8 +190,10 @@ ether_resolvemulti(
                 sdl->sdl_slen = 0;
                 e_addr = LLADDR(sdl);
                 ETHER_MAP_IPV6_MULTICAST(&sin6->sin6_addr, e_addr);
-                kprintf("ether_resolvemulti Adding %x:%x:%x:%x:%x:%x\n",
+#if 0
+                printf("ether_resolvemulti Adding %x:%x:%x:%x:%x:%x\n",
                                 e_addr[0], e_addr[1], e_addr[2], e_addr[3], e_addr[4], e_addr[5]);
+#endif
                 *llsa = (struct sockaddr *)sdl;
                 return 0;
 #endif
diff --git a/bsd/net/if_faith.c b/bsd/net/if_faith.c
deleted file mode 100644
index 5c4c8e487..000000000
--- a/bsd/net/if_faith.c
+++ /dev/null
@@ -1,452 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*	$KAME: if_faith.c,v 1.21 2001/02/20 07:59:26 itojun Exp $	*/
-
-/*
- * Copyright (c) 1982, 1986, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD: src/sys/net/if_faith.c,v 1.3.2.2 2001/07/05 14:46:25 ume Exp $
- */
-/*
- * derived from
- *	@(#)if_loop.c	8.1 (Berkeley) 6/10/93
- * Id: if_loop.c,v 1.22 1996/06/19 16:24:10 wollman Exp
- */
-
-/*
- * Loopback interface driver for protocol testing and timing.
- */
-
-#include "faith.h"
-#if NFAITH > 0
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/kernel.h>
-#include <sys/mbuf.h>
-#include <sys/socket.h>
-#include <sys/errno.h>
-#include <sys/sockio.h>
-#include <sys/time.h>
-#include <sys/queue.h>
-
-#include <net/if.h>
-#include <net/if_types.h>
-#include <net/route.h>
-#include <net/bpf.h>
-#include <net/if_faith.h>
-
-#if	INET
-#include <netinet/in.h>
-#include <netinet/in_systm.h>
-#include <netinet/in_var.h>
-#include <netinet/ip.h>
-#endif
-
-#if INET6
-#ifndef INET
-#include <netinet/in.h>
-#endif
-#include <netinet6/in6_var.h>
-#include <netinet/ip6.h>
-#include <netinet6/ip6_var.h>
-#endif
-
-#include <net/dlil.h>
-#include "bpf.h"
-
-#include <net/net_osdep.h>
-
-static int faithioctl(struct ifnet *, u_long, void*);
-int faith_pre_output(struct ifnet *, register struct mbuf **,
-	const struct sockaddr *, caddr_t, char *, char *, u_long);
-static void faithrtrequest(int, struct rtentry *, struct sockaddr *);
-
-void faithattach(void);
-#ifndef __APPLE__
-PSEUDO_SET(faithattach, if_faith);
-#endif
-
-static struct ifnet faithif[NFAITH];
-static struct if_proto *faith_array[NFAITH];
-static int faith_count = 0;
-
-#define	FAITHMTU	1500
-
-static
-int  faith_add_if(struct ifnet *ifp)
-{
-    ifp->if_demux  = 0;
-    ifp->if_framer = 0;
-    return 0;
-}
-
-static 
-int  faith_del_if(struct ifnet *ifp)
-{
-    return 0;
-}
-
-static
-int  faith_add_proto(struct ddesc_head_str *desc_head, struct if_proto *proto, u_long dl_tag)
-{       
-    int i;
-    
-    for (i=0; i < faith_count; i++)
-        if (faith_array[i] == 0) {
-            faith_array[faith_count] = proto;
-            return 0;
-        }
-    
-    if ((i == faith_count) && (faith_count == NFAITH))
-       panic("faith_add_proto -- Too many attachments\n");
-
-    faith_array[faith_count++] = proto;
-
-    return (0);
-}
-
-static
-int  faith_del_proto(struct if_proto *proto, u_long dl_tag)
-{   
-    int i;
-
-
-    for (i=0; i < faith_count; i++)
-        if (faith_array[i] == proto) {
-            faith_array[i] = 0;
-            return 0;
-        }
-
-    return ENOENT;
-}
-
-int faith_shutdown()
-{
-    return 0;
-}
-
-int  faith_attach_inet(struct ifnet *ifp, u_long *dl_tag)
-{       
-    struct dlil_proto_reg_str   reg;
-    struct dlil_demux_desc      desc;
-    short native=0;
-    int   stat;
-    int i;
-
-    for (i=0; i < faith_count; i++) {
-        if (faith_array[i] && (faith_array[i]->ifp == ifp) &&
-            (faith_array[i]->protocol_family == PF_INET)) {
-#if 0
-	        kprintf("faith_array for %s%d found dl_tag=%d\n",
-                       ifp->if_name, ifp->if_unit, faith_array[i]->dl_tag);
-#endif
-                *dl_tag = faith_array[i]->dl_tag;
-		return 0;
-       
-        }
-    }
-
-	bzero(&reg, sizeof(reg));
-	bzero(&desc, sizeof(desc));
-    TAILQ_INIT(&reg.demux_desc_head); 
-    desc.type = DLIL_DESC_RAW;
-    desc.native_type = (char *) &native;
-    TAILQ_INSERT_TAIL(&reg.demux_desc_head, &desc, next);
-    reg.interface_family = ifp->if_family;
-    reg.unit_number      = ifp->if_unit;
-    reg.pre_output       = faith_pre_output;
-    reg.protocol_family  = PF_INET;
-
-    stat = dlil_attach_protocol(&reg, dl_tag);
-    if (stat) {
-        panic("faith_attach_inet can't attach interface\n");
-    }
-
-    return stat;
-}
-
-void faith_reg_if_mods()
-{   
-     struct dlil_ifmod_reg_str  faith_ifmod;
-     struct dlil_protomod_reg_str faith_protoreg;
-     int error;
-
-     bzero(&faith_ifmod, sizeof(faith_ifmod));
-     faith_ifmod.add_if = faith_add_if;
-     faith_ifmod.del_if = faith_del_if;
-     faith_ifmod.add_proto = faith_add_proto;
-     faith_ifmod.del_proto = faith_del_proto;
-     faith_ifmod.ifmod_ioctl = 0;
-     faith_ifmod.shutdown    = faith_shutdown;
-
-    
-    if (dlil_reg_if_modules(APPLE_IF_FAM_FAITH, &faith_ifmod))
-        panic("Couldn't register faith modules\n");
-
-	/* Register protocol registration functions */
-
-	bzero(&faith_protoreg, sizeof(faith_protoreg));
-	faith_protoreg.attach_proto = faith_attach_inet;
-	faith_protoreg.detach_proto = 0;
-	
-	if ( error = dlil_reg_proto_module(PF_INET, APPLE_IF_FAM_FAITH, &faith_protoreg) != 0)
-		kprintf("dlil_reg_proto_module failed for AF_INET error=%d\n", error);
-
-    
-}   
-    
-void
-faithattach(void)
-{
-	struct ifnet *ifp;
-	int i;
-
-	faith_reg_if_mods(); /* DLIL modules */
-
-	for (i = 0; i < NFAITH; i++) {
-		ifp = &faithif[i];
-		bzero(ifp, sizeof(faithif[i]));
-		ifp->if_name = "faith";
-		ifp->if_unit = i;
-		ifp->if_family = APPLE_IF_FAM_FAITH;
-		ifp->if_mtu = FAITHMTU;
-		/* LOOPBACK commented out to announce IPv6 routes to faith */
-		ifp->if_flags = /* IFF_LOOPBACK | */ IFF_MULTICAST;
-		ifp->if_ioctl = faithioctl;
-		ifp->if_output = NULL;
-		ifp->if_type = IFT_FAITH;
-		ifp->if_hdrlen = 0;
-		ifp->if_addrlen = 0;
-		dlil_if_attach(ifp);
-#if NBPFILTER > 0
-#ifdef HAVE_OLD_BPF
-		bpfattach(ifp, DLT_NULL, sizeof(u_int));
-#else
-		bpfattach(&ifp->if_bpf, ifp, DLT_NULL, sizeof(u_int));
-#endif
-#endif
-	}
-}
-
-int
-faith_pre_output(ifp, m0, dst, route_entry, frame_type, dst_addr, dl_tag)
-	struct ifnet *ifp;
-	register struct mbuf **m0;
-	const struct sockaddr *dst;
-	caddr_t			 route_entry;
-	char		     *frame_type;
-	char		     *dst_addr;
-	u_long		     dl_tag;
-{
-	register struct mbuf *m = *m0;
-	struct rtentry *rt = (struct rtentry*)route_entry;
-
-	if ((m->m_flags & M_PKTHDR) == 0)
-		panic("faithoutput no HDR");
-#if NBPFILTER > 0
-	/* BPF write needs to be handled specially */
-	if (dst && dst->sa_family == AF_UNSPEC) {
-		dst->sa_family = *(mtod(m, int *));
-		m->m_len -= sizeof(int);
-		m->m_pkthdr.len -= sizeof(int);
-		m->m_data += sizeof(int);
-	}
-
-	if (ifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a faith header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
-		u_int32_t af = dst->sa_family;
-
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&af;
-
-#ifdef HAVE_OLD_BPF
-		bpf_mtap(ifp, &m0);
-#else
-		bpf_mtap(ifp->if_bpf, &m0);
-#endif
-	}
-#endif
-
-	if (rt && rt->rt_flags & (RTF_REJECT|RTF_BLACKHOLE)) {
-		return (rt->rt_flags & RTF_BLACKHOLE ? 0 :
-		        rt->rt_flags & RTF_HOST ? EHOSTUNREACH : ENETUNREACH);
-	}
-	ifp->if_opackets++;
-	ifp->if_obytes += m->m_pkthdr.len;
-	m->m_pkthdr.rcvif = ifp;
-	proto_inject(dst->sa_family, m);
-	ifp->if_ipackets++;
-	ifp->if_ibytes += m->m_pkthdr.len;
-	return (EJUSTRETURN);
-}
-
-/* ARGSUSED */
-static void
-faithrtrequest(cmd, rt, sa)
-	int cmd;
-	struct rtentry *rt;
-	struct sockaddr *sa;
-{
-	if (rt) {
-		rt->rt_rmx.rmx_mtu = rt->rt_ifp->if_mtu; /* for ISO */
-		/*
-		 * For optimal performance, the send and receive buffers
-		 * should be at least twice the MTU plus a little more for
-		 * overhead.
-		 */
-		rt->rt_rmx.rmx_recvpipe =
-			rt->rt_rmx.rmx_sendpipe = 3 * FAITHMTU;
-	}
-}
-
-/*
- * Process an ioctl request.
- */
-/* ARGSUSED */
-static int
-faithioctl(ifp, cmd, data)
-	struct ifnet *ifp;
-	u_long cmd;
-	void* data;
-{
-	struct ifaddr *ifa;
-	struct ifreq *ifr = (struct ifreq *)data;
-	int error = 0;
-
-	switch (cmd) {
-
-	case SIOCSIFADDR:
-		ifnet_set_flags(ifp, IFF_UP | IFF_RUNNING, IFF_UP | IFF_RUNNING);
-		ifa = (struct ifaddr *)data;
-		ifa->ifa_rtrequest = faithrtrequest;
-		/*
-		 * Everything else is done at a higher level.
-		 */
-		break;
-
-	case SIOCADDMULTI:
-	case SIOCDELMULTI:
-		if (ifr == 0) {
-			error = EAFNOSUPPORT;		/* XXX */
-			break;
-		}
-		switch (ifr->ifr_addr.sa_family) {
-#if INET
-		case AF_INET:
-			break;
-#endif
-#if INET6
-		case AF_INET6:
-			break;
-#endif
-
-		default:
-			error = EAFNOSUPPORT;
-			break;
-		}
-		break;
-
-#ifdef SIOCSIFMTU
-	case SIOCSIFMTU:
-		ifp->if_mtu = ifr->ifr_mtu;
-		break;
-#endif
-
-	case SIOCSIFFLAGS:
-		break;
-
-	default:
-		error = EINVAL;
-	}
-	return (error);
-}
-
-#if INET6
-/*
- * XXX could be slow
- * XXX could be layer violation to call sys/net from sys/netinet6
- */
-int
-faithprefix(in6)
-	struct in6_addr *in6;
-{
-	struct rtentry *rt;
-	struct sockaddr_in6 sin6;
-	int ret;
-
-	if (ip6_keepfaith == 0)
-		return 0;
-
-	bzero(&sin6, sizeof(sin6));
-	sin6.sin6_family = AF_INET6;
-	sin6.sin6_len = sizeof(struct sockaddr_in6);
-	sin6.sin6_addr = *in6;
-	rt = rtalloc1((struct sockaddr *)&sin6, 0, 0UL);
-	if (rt && rt->rt_ifp && rt->rt_ifp->if_type == IFT_FAITH &&
-	    (rt->rt_ifp->if_flags & IFF_UP) != 0)
-		ret = 1;
-	else
-		ret = 0;
-	if (rt)
-		rtfree(rt);
-	return ret;
-}
-#endif
-#endif /* NFAITH > 0 */
diff --git a/bsd/net/if_fddisubr.c b/bsd/net/if_fddisubr.c
index dd1ef3a32..1de331796 100644
--- a/bsd/net/if_fddisubr.c
+++ b/bsd/net/if_fddisubr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995, 1996
diff --git a/bsd/net/if_gif.c b/bsd/net/if_gif.c
index 6db06ef62..1381fc6fd 100644
--- a/bsd/net/if_gif.c
+++ b/bsd/net/if_gif.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$FreeBSD: src/sys/net/if_gif.c,v 1.4.2.6 2001/07/24 19:10:18 brooks Exp $	*/
 /*	$KAME: if_gif.c,v 1.47 2001/05/01 05:28:42 itojun Exp $	*/
@@ -50,6 +56,12 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -68,6 +80,8 @@
 #include <net/if_types.h>
 #include <net/route.h>
 #include <net/bpf.h>
+#include <net/kpi_protocol.h>
+#include <net/kpi_interface.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
@@ -92,6 +106,10 @@
 
 #include <net/net_osdep.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #define GIFNAME		"gif"
 #define GIFDEV		"if_gif"
 #define GIF_MAXUNIT	0x7fff	/* ifp->if_unit is only 15 bits */
@@ -104,11 +122,12 @@ TAILQ_HEAD(gifhead, gif_softc) gifs = TAILQ_HEAD_INITIALIZER(gifs);
 
 #ifdef __APPLE__
 void gifattach(void);
-int gif_pre_output(struct ifnet *ifp, u_long protocol_family, struct mbuf **m0,
-				   const struct sockaddr *dst, caddr_t rt, char *frame, char *address);
 static void gif_create_dev(void);
 static int gif_encapcheck(const struct mbuf*, int, int, void*);
-
+static errno_t gif_output(ifnet_t ifp, mbuf_t m);
+static errno_t gif_input(ifnet_t ifp, protocol_family_t protocol_family,
+						 mbuf_t m, char *frame_header);
+static errno_t gif_ioctl(ifnet_t ifp, u_int32_t cmd, void *data);
 
 int ngif = 0;		/* number of interfaces */
 #endif
@@ -121,7 +140,7 @@ struct protosw in_gif_protosw =
   0,		0,	0,	0,	
   0,
   &rip_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0, {0, 0}, 0, {0}
 };
 #endif
 #if INET6
@@ -132,26 +151,11 @@ struct ip6protosw in6_gif_protosw =
   0,		0,		0,		0,
   0,  
   &rip6_usrreqs,
-  0,		rip_unlock,		0,
+  0,		rip_unlock,		0, {0, 0}, 0, {0}
 
 };
 #endif
 
-#ifndef MAX_GIF_NEST
-/*
- * This macro controls the upper limitation on nesting of gif tunnels.
- * Since, setting a large value to this macro with a careless configuration
- * may introduce system crash, we don't allow any nestings by default.
- * If you need to configure nested gif tunnels, you can define this macro
- * in your kernel configuration file. However, if you do so, please be
- * careful to configure the tunnels so that it won't make a loop.
- */
-#define MAX_GIF_NEST 1
-#endif
-static int max_gif_nesting = MAX_GIF_NEST;
-
-
-
 #ifdef __APPLE__
 /*
  * Theory of operation: initially, one gif interface is created.
@@ -164,65 +168,63 @@ static int max_gif_nesting = MAX_GIF_NEST;
  */
 
 /* GIF interface module support */
-int gif_demux(
-    struct ifnet *ifp,
-    struct mbuf  *m,
-    char         *frame_header,
-    u_long *protocol_family)
+static int gif_demux(
+    ifnet_t					ifp,
+    __unused mbuf_t			m,
+    __unused char			*frame_header,
+    protocol_family_t		*protocol_family)
 {
-	struct gif_softc* gif = (struct gif_softc*)ifp->if_softc;
-	
 	/* Only one protocol may be attached to a gif interface. */
-	*protocol_family = gif->gif_proto;
+	*protocol_family = ((struct gif_softc*)ifnet_softc(ifp))->gif_proto;
 	
 	return 0;
 }
 
-static
-int  gif_add_proto(struct ifnet *ifp, u_long protocol_family, struct ddesc_head_str *desc_head)
+static errno_t
+gif_add_proto(
+	ifnet_t									ifp,
+	protocol_family_t						protocol_family,
+	__unused const struct ifnet_demux_desc	*demux_array,
+	__unused u_int32_t						demux_count)
 {
 	/* Only one protocol may be attached at a time */
-	struct gif_softc* gif = (struct gif_softc*)ifp->if_softc;
+	struct gif_softc* gif = ifnet_softc(ifp);
 
 	if (gif->gif_proto != 0)
-		printf("gif_add_proto: request add_proto for gif%d\n", gif->gif_if.if_unit);
+		printf("gif_add_proto: request add_proto for gif%d\n", ifnet_unit(ifp));
 
 	gif->gif_proto = protocol_family;
 
 	return 0;
 }
 
-static
-int  gif_del_proto(struct ifnet *ifp, u_long protocol_family)
+static errno_t
+gif_del_proto(
+	ifnet_t				ifp,
+	protocol_family_t	protocol_family)
 {
-	if (((struct gif_softc*)ifp)->gif_proto == protocol_family)
-		((struct gif_softc*)ifp)->gif_proto = 0;
-	else
-		return ENOENT;
-
+	if (((struct gif_softc*)ifnet_softc(ifp))->gif_proto == protocol_family)
+		((struct gif_softc*)ifnet_softc(ifp))->gif_proto = 0;
+	
 	return 0;
 }
 
 /* Glue code to attach inet to a gif interface through DLIL */
-int
+static errno_t
 gif_attach_proto_family(
-	struct ifnet *ifp,
-	u_long protocol_family)
+	ifnet_t				ifp,
+	protocol_family_t	protocol_family)
 {
-    struct dlil_proto_reg_str   reg;
-    int   stat;
+    struct ifnet_attach_proto_param	reg;
+    errno_t							stat;
 
 	bzero(&reg, sizeof(reg));
-    TAILQ_INIT(&reg.demux_desc_head);
-    reg.interface_family = ifp->if_family;
-    reg.unit_number      = ifp->if_unit;
     reg.input            = gif_input;
-    reg.pre_output       = gif_pre_output;
-    reg.protocol_family  = protocol_family;
 
-    stat = dlil_attach_protocol(&reg);
+    stat = ifnet_attach_protocol(ifp, protocol_family, &reg);
     if (stat && stat != EEXIST) {
-        panic("gif_attach_proto_family can't attach interface fam=%d\n", protocol_family);
+        printf("gif_attach_proto_family can't attach interface fam=%d\n",
+        	   protocol_family);
     }
 
     return stat;
@@ -231,30 +233,50 @@ gif_attach_proto_family(
 #endif
 
 /* Function to setup the first gif interface */
-void
+__private_extern__ void
 gifattach(void)
 {
-	int error;
+	errno_t result;
 
 	/* Init the list of interfaces */
 	TAILQ_INIT(&gifs);
 
 	/* Register protocol registration functions */
-	if ( error = dlil_reg_proto_module(AF_INET, APPLE_IF_FAM_GIF, gif_attach_proto_family, NULL) != 0)
-		printf("dlil_reg_proto_module failed for AF_INET error=%d\n", error);
+	result = proto_register_plumber(PF_INET, APPLE_IF_FAM_GIF,
+									gif_attach_proto_family, NULL);
+	if (result != 0)
+		printf("proto_register_plumber failed for AF_INET error=%d\n", result);
 	
-	if ( error = dlil_reg_proto_module(AF_INET6, APPLE_IF_FAM_GIF, gif_attach_proto_family, NULL) != 0)
-		printf("dlil_reg_proto_module failed for AF_INET6 error=%d\n", error);
+	result = proto_register_plumber(PF_INET6, APPLE_IF_FAM_GIF,
+									gif_attach_proto_family, NULL);
+	if (result != 0)
+		printf("proto_register_plumber failed for AF_INET6 error=%d\n", result);
 
 	/* Create first device */
 	gif_create_dev();
 }
 
+static errno_t
+gif_set_bpf_tap(
+	ifnet_t			ifp,
+	bpf_tap_mode	mode,
+	bpf_packet_func	callback)
+{
+	struct gif_softc	*sc = ifnet_softc(ifp);
+	
+	sc->tap_mode = mode;
+	sc->tap_callback = callback;
+	
+	return 0;
+}
+
 /* Creates another gif device if there are none free */
 static void
 gif_create_dev(void)
 {
-	struct gif_softc *sc;
+	struct gif_softc			*sc;
+	struct ifnet_init_params	gif_init;
+	errno_t						result = 0;
 	
 	
 	/* Can't create more than GIF_MAXUNIT */
@@ -264,7 +286,7 @@ gif_create_dev(void)
 	/* Check for unused gif interface */
 	TAILQ_FOREACH(sc, &gifs, gif_link) {
 		/* If unused, return, no need to create a new interface */
-		if ((sc->gif_if.if_flags & IFF_RUNNING) == 0)
+		if ((ifnet_flags(sc->gif_if) & IFF_RUNNING) == 0)
 			return;
 	}
 
@@ -273,23 +295,39 @@ gif_create_dev(void)
 		log(LOG_ERR, "gifattach: failed to allocate gif%d\n", ngif);
 		return;
 	}
+	
+	bzero(&gif_init, sizeof(gif_init));
+	gif_init.name = GIFNAME;
+	gif_init.unit = ngif;
+	gif_init.type = IFT_GIF;
+	gif_init.family = IFNET_FAMILY_GIF;
+	gif_init.output = gif_output;
+	gif_init.demux = gif_demux;
+	gif_init.add_proto = gif_add_proto;
+	gif_init.del_proto = gif_del_proto;
+	gif_init.softc = sc;
+	gif_init.ioctl = gif_ioctl;
+	gif_init.set_bpf_tap = gif_set_bpf_tap;
 
 	bzero(sc, sizeof(struct gif_softc));
-	sc->gif_if.if_softc	= sc;
-	sc->gif_if.if_name	= GIFNAME;
-	sc->gif_if.if_unit	= ngif;
-	
+	result = ifnet_allocate(&gif_init, &sc->gif_if);
+	if (result != 0) {
+		printf("gif_create_dev, ifnet_allocate failed - %d\n", result);
+		_FREE(sc, M_DEVBUF);
+		return;
+	}
 	sc->encap_cookie4 = sc->encap_cookie6 = NULL;
-#ifdef INET
+#if INET
 	sc->encap_cookie4 = encap_attach_func(AF_INET, -1,
 	    gif_encapcheck, &in_gif_protosw, sc);
 	if (sc->encap_cookie4 == NULL) {
-		printf("%s: unable to attach encap4\n", if_name(&sc->gif_if));
+		printf("%s: unable to attach encap4\n", if_name(sc->gif_if));
+		ifnet_release(sc->gif_if);
 		FREE(sc, M_DEVBUF);
 		return;
 	}
 #endif
-#ifdef INET6
+#if INET6
 	sc->encap_cookie6 = encap_attach_func(AF_INET6, -1,
 	    gif_encapcheck, (struct protosw*)&in6_gif_protosw, sc);
 	if (sc->encap_cookie6 == NULL) {
@@ -297,38 +335,40 @@ gif_create_dev(void)
 			encap_detach(sc->encap_cookie4);
 			sc->encap_cookie4 = NULL;
 		}
-		printf("%s: unable to attach encap6\n", if_name(&sc->gif_if));
+		printf("%s: unable to attach encap6\n", if_name(sc->gif_if));
+		ifnet_release(sc->gif_if);
 		FREE(sc, M_DEVBUF);
 		return;
 	}
 #endif
-	
 	sc->gif_called = 0;
-	sc->gif_if.if_family= APPLE_IF_FAM_GIF;
-	sc->gif_if.if_mtu	= GIF_MTU;
-	sc->gif_if.if_flags  = IFF_POINTOPOINT | IFF_MULTICAST;
+	ifnet_set_mtu(sc->gif_if, GIF_MTU);
+	ifnet_set_flags(sc->gif_if, IFF_POINTOPOINT | IFF_MULTICAST, 0xffff);
 #if 0
 	/* turn off ingress filter */
 	sc->gif_if.if_flags  |= IFF_LINK2;
 #endif
-	sc->gif_if.if_demux = gif_demux;
-	sc->gif_if.if_ioctl	= gif_ioctl;
-	sc->gif_if.if_output = NULL;	/* pre_output returns error or EJUSTRETURN */
-	sc->gif_if.if_type   = IFT_GIF;
-	sc->gif_if.if_add_proto = gif_add_proto;
-	sc->gif_if.if_del_proto = gif_del_proto;
-	dlil_if_attach(&sc->gif_if);
-	bpfattach(&sc->gif_if, DLT_NULL, sizeof(u_int));
+	result = ifnet_attach(sc->gif_if, NULL);
+	if (result != 0) {
+		printf("gif_create_dev - ifnet_attach failed - %d\n", result);
+		ifnet_release(sc->gif_if);
+		FREE(sc, M_DEVBUF);
+		return;
+	}
+#if CONFIG_MACF_NET
+	mac_ifnet_label_init(&sc->gif_if);
+#endif
+	bpfattach(sc->gif_if, DLT_NULL, sizeof(u_int));
 	TAILQ_INSERT_TAIL(&gifs, sc, gif_link);
 	ngif++;
 }
 
 static int
-gif_encapcheck(m, off, proto, arg)
-	const struct mbuf *m;
-	int off;
-	int proto;
-	void *arg;
+gif_encapcheck(
+	const struct mbuf *m,
+	int off,
+	int proto,
+	void *arg)
 {
 	struct ip ip;
 	struct gif_softc *sc;
@@ -337,7 +377,7 @@ gif_encapcheck(m, off, proto, arg)
 	if (sc == NULL)
 		return 0;
 
-	if ((sc->gif_if.if_flags & IFF_UP) == 0)
+	if ((ifnet_flags(sc->gif_if) & IFF_UP) == 0)
 		return 0;
 
 	/* no physical address */
@@ -357,8 +397,7 @@ gif_encapcheck(m, off, proto, arg)
 		return 0;
 	}
 
-	/* LINTED const cast */
-	m_copydata((struct mbuf *)m, 0, sizeof(ip), (caddr_t)&ip);
+	mbuf_copydata(m, 0, sizeof(ip), &ip);
 
 	switch (ip.ip_v) {
 #if INET
@@ -380,65 +419,31 @@ gif_encapcheck(m, off, proto, arg)
 	}
 }
 
-int
-gif_pre_output(
-	struct ifnet *ifp,
-	u_long protocol_family,
-	struct mbuf **m0,
-	const struct sockaddr *dst,
-	caddr_t rt,
-	char *frame,
-	char *address)
+static errno_t
+gif_output(
+	ifnet_t		ifp,
+	mbuf_t		m)
 {
-	struct gif_softc *sc = (struct gif_softc*)ifp;
-	register struct mbuf * m = *m0;
+	struct gif_softc *sc = ifnet_softc(ifp);
 	int error = 0;
 	
 	/*
-	 * gif may cause infinite recursion calls when misconfigured.
-	 * We'll prevent this by introducing upper limit.
-	 * XXX: this mechanism may introduce another problem about
-	 *      mutual exclusion of the variable CALLED, especially if we
-	 *      use kernel thread.
+	   max_gif_nesting check used to live here. It doesn't anymore
+	   because there is no guaruntee that we won't be called
+	   concurrently from more than one thread.
 	 */
-	if (++sc->gif_called > max_gif_nesting) {
-		log(LOG_NOTICE,
-		    "gif_output: recursively called too many times(%d)\n",
-		    sc->gif_called);
-		m_freem(m);	/* free it here not in dlil_output*/
-		error = EIO;	/* is there better errno? */
-		goto end;
-	}
-
-	ifnet_touch_lastchange(ifp);
+	
 	m->m_flags &= ~(M_BCAST|M_MCAST);
-	if (!(ifp->if_flags & IFF_UP) ||
+	if (!(ifnet_flags(ifp) & IFF_UP) ||
 	    sc->gif_psrc == NULL || sc->gif_pdst == NULL) {
+		ifnet_touch_lastchange(ifp);
 		m_freem(m);	/* free it here not in dlil_output */
 		error = ENETDOWN;
 		goto end;
 	}
 
-	if (ifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
-		u_int32_t protocol_family = dst->sa_family;
-
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&protocol_family;
-		
-		bpf_mtap(ifp, &m0);
-	}
-	ifp->if_opackets++;	
-	ifp->if_obytes += m->m_pkthdr.len;
-
+	bpf_tap_out(ifp, 0, m, &sc->gif_proto, sizeof(sc->gif_proto));
+	
 	/* inner AF-specific encapsulation */
 
 	/* XXX should we check if our outer source is legal? */
@@ -447,12 +452,12 @@ gif_pre_output(
 	switch (sc->gif_psrc->sa_family) {
 #if INET
 	case AF_INET:
-		error = in_gif_output(ifp, dst->sa_family, m, (struct rtentry*)rt);
+		error = in_gif_output(ifp, sc->gif_proto, m, NULL);
 		break;
 #endif
 #if INET6
 	case AF_INET6:
-		error = in6_gif_output(ifp, dst->sa_family, m, (struct rtentry*)rt);
+		error = in6_gif_output(ifp, sc->gif_proto, m, NULL);
 		break;
 #endif
 	default:
@@ -461,52 +466,32 @@ gif_pre_output(
 	}
 
   end:
-	sc->gif_called = 0;		/* reset recursion counter */
 	if (error) {
 		/* the mbuf was freed either by in_gif_output or in here */
-		*m0 = NULL; /* avoid getting dlil_output freeing it */
-		ifp->if_oerrors++;
+		ifnet_stat_increment_out(ifp, 0, 0, 1);
+	}
+	else {
+		ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0);
 	}
 	if (error == 0) 
 		error = EJUSTRETURN; /* if no error, packet got sent already */
 	return error;
 }
 
-int
+/*
+ * gif_input is the input handler for IP and IPv6 attached to gif
+ */
+static errno_t
 gif_input(
-	struct mbuf *m,
-	char* frame_header,
-	struct ifnet* gifp,
-	u_long protocol_family,
-	int sync_ok)
+	ifnet_t				ifp,
+	protocol_family_t	protocol_family,
+	mbuf_t				m,
+	__unused char		*frame_header)
 {
-
-	if (gifp == NULL) {
-		/* just in case */
-		m_freem(m);
-		return;
-	}
-
-	if (m->m_pkthdr.rcvif)
-		m->m_pkthdr.rcvif = gifp;
+	errno_t error;
+	struct gif_softc *sc = ifnet_softc(ifp);
 	
-	if (gifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
-		u_int32_t protocol_family1 = protocol_family;
-		
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&protocol_family1;
-		
-		bpf_mtap(gifp, &m0);
-	}
+	bpf_tap_in(ifp, 0, m, &sc->gif_proto, sizeof(sc->gif_proto));
 
 	/*
 	 * Put the packet to the network layer input queue according to the
@@ -520,26 +505,24 @@ gif_input(
 	 * it occurs more times than we thought, we may change the policy
 	 * again.
 	 */
-	proto_input(protocol_family, m);
-	gifp->if_ipackets++;
-	gifp->if_ibytes += m->m_pkthdr.len;
+	error = proto_input(protocol_family, m);
+	ifnet_stat_increment_in(ifp, 1, m->m_pkthdr.len, 0);
 
 	return (0);
 }
 
 /* XXX how should we handle IPv6 scope on SIOC[GS]IFPHYADDR? */
-int
-gif_ioctl(ifp, cmd, data)
-	struct ifnet *ifp;
-	u_long cmd;
-	void* data;
+static errno_t
+gif_ioctl(
+	ifnet_t			ifp,
+	u_int32_t		cmd,
+	void			*data)
 {
-	struct gif_softc *sc  = (struct gif_softc*)ifp;
+	struct gif_softc *sc  = ifnet_softc(ifp);
 	struct ifreq     *ifr = (struct ifreq*)data;
 	int error = 0, size;
-	struct sockaddr *dst, *src;
+	struct sockaddr *dst = NULL, *src = NULL;
 	struct sockaddr *sa;
-	int s;
 	struct ifnet *ifp2;
 	struct gif_softc *sc2;
 		
@@ -565,7 +548,7 @@ gif_ioctl(ifp, cmd, data)
 			if (mtu < GIF_MTU_MIN || mtu > GIF_MTU_MAX) {
 				return (EINVAL);
 			}
-			ifp->if_mtu = mtu;
+			ifnet_set_mtu(ifp, mtu);
 		}
 		break;
 #endif /* SIOCSIFMTU */
@@ -656,9 +639,9 @@ gif_ioctl(ifp, cmd, data)
 
 		ifnet_head_lock_shared();
 		TAILQ_FOREACH(ifp2, &ifnet_head, if_link) {
-			if (strcmp(ifp2->if_name, GIFNAME) != 0)
+			if (strcmp(ifnet_name(ifp2), GIFNAME) != 0)
 				continue;
-			sc2 = ifp2->if_softc;
+			sc2 = ifnet_softc(ifp2);
 			if (sc2 == sc)
 				continue;
 			if (!sc2->gif_pdst || !sc2->gif_psrc)
@@ -714,15 +697,12 @@ gif_ioctl(ifp, cmd, data)
 		bcopy((caddr_t)dst, (caddr_t)sa, dst->sa_len);
 		sc->gif_pdst = sa;
 
-		ifp->if_flags |= IFF_RUNNING;
-
-		s = splimp();
-		if_up(ifp);	/* mark interface UP and send up RTM_IFINFO */
+		ifnet_set_flags(ifp, IFF_RUNNING | IFF_UP, IFF_RUNNING | IFF_UP);
+		
 #ifdef __APPLE__
 		/* Make sure at least one unused device is still available */
 		gif_create_dev();
 #endif
-		splx(s);
 
 		error = 0;
 		break;
diff --git a/bsd/net/if_gif.h b/bsd/net/if_gif.h
index bfd647244..dc193f74c 100644
--- a/bsd/net/if_gif.h
+++ b/bsd/net/if_gif.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: if_gif.h,v 1.7 2000/02/22 14:01:46 itojun Exp $	*/
 
@@ -65,11 +71,11 @@
 struct encaptab;
 
 struct gif_softc {
-	struct ifnet	gif_if;	   /* common area - must be at the top */
+	ifnet_t			gif_if;	   /* pointer back to the interface */
 	struct sockaddr	*gif_psrc; /* Physical src addr */
 	struct sockaddr	*gif_pdst; /* Physical dst addr */
 #ifdef __APPLE__
-	u_long	gif_proto; /* dlil protocol attached */
+	protocol_family_t gif_proto; /* dlil protocol attached */
 #endif
 	union {
 		struct route  gifscr_ro;    /* xxx */
@@ -82,6 +88,8 @@ struct gif_softc {
 	const struct encaptab *encap_cookie4;
 	const struct encaptab *encap_cookie6;
 	TAILQ_ENTRY(gif_softc) gif_link; /* all gif's are linked */
+	bpf_tap_mode	tap_mode;
+	bpf_packet_func tap_callback;
 };
 
 #define gif_ro gifsc_gifscr.gifscr_ro
@@ -95,13 +103,4 @@ struct gif_softc {
 #define	GIF_MTU_MIN	(1280)	/* Minimum MTU */
 #define	GIF_MTU_MAX	(8192)	/* Maximum MTU */
 
-#ifdef KERNEL_PRIVATE
-
-/* Prototypes */
-int gif_input(struct mbuf *, char*, struct ifnet *, u_long, int);
-int gif_output(struct ifnet *, struct mbuf *,
-		    struct sockaddr *, struct rtentry *);
-int gif_ioctl(struct ifnet *, u_long, void*);
-
-#endif /* KERNEL_PRIVATE */
 #endif /* _NET_IF_GIF_H_ */
diff --git a/bsd/net/if_llc.h b/bsd/net/if_llc.h
index 536b68e8b..7b0d446e2 100644
--- a/bsd/net/if_llc.h
+++ b/bsd/net/if_llc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1993
diff --git a/bsd/net/if_loop.c b/bsd/net/if_loop.c
index 1ec4f8e50..67e5f1f81 100644
--- a/bsd/net/if_loop.c
+++ b/bsd/net/if_loop.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -54,6 +60,12 @@
  *	@(#)if_loop.c	8.1 (Berkeley) 6/10/93
  * $FreeBSD: src/sys/net/if_loop.c,v 1.47.2.5 2001/07/03 11:01:41 ume Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * Loopback interface driver for protocol testing and timing.
@@ -79,13 +91,8 @@
 #include <netinet/in_var.h>
 #endif
 
-#if IPX
-#include <netipx/ipx.h>
-#include <netipx/ipx_if.h>
-#endif
-
 #if INET6
-#ifndef INET
+#if !INET
 #include <netinet/in.h>
 #endif
 #include <netinet6/in6_var.h>
@@ -104,6 +111,10 @@ extern struct ifqueue atalkintrq;
 #include <net/bpfdesc.h>
 #endif
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #define NLOOP_ATTACHMENTS (NLOOP * 12)
 
 struct lo_statics_str {
@@ -111,7 +122,7 @@ struct lo_statics_str {
 	bpf_packet_func	bpf_callback;
 };
 
-void loopattach(void *dummy);
+void loopattach(void);
 
 static struct lo_statics_str lo_statics[NLOOP];
 int loopattach_done = 0; /* used to sync ip6_init2 loopback configuration */
@@ -122,14 +133,13 @@ int loopattach_done = 0; /* used to sync ip6_init2 loopback configuration */
 #define LOMTU	16384
 #endif
 
-struct	ifnet loif[NLOOP];
-struct ifnet *lo_ifp = &loif[0];
+ifnet_t	lo_ifp = NULL;
 
 struct	loopback_header {
-	u_long		protocol;
+	protocol_family_t	protocol;
 };
 
-void lo_reg_if_mods(void);
+static void lo_reg_if_mods(void);
 
 /* Local forward declerations */
 
@@ -166,9 +176,10 @@ lo_framer(
 
 static errno_t
 lo_add_proto(
-	__unused struct ifnet			*ifp,
-	__unused u_long					protocol_family,
-	__unused struct ddesc_head_str	*demux_desc_head)
+    __unused ifnet_t						interface,
+	__unused protocol_family_t				protocol_family,
+	__unused const struct ifnet_demux_desc	*demux_array,
+	__unused u_int32_t						demux_count)
 {
     return 0;
 }
@@ -184,59 +195,48 @@ lo_del_proto(
 
 static int
 lo_output(
-	struct ifnet *ifp,
-	struct mbuf *m)
+	ifnet_t	ifp,
+	mbuf_t	m_list)
 {
+	mbuf_t	m;
+	
+	for (m = m_list; m; m = m->m_nextpkt) {
+		if ((m->m_flags & M_PKTHDR) == 0)
+			panic("lo_output: no HDR");
 
-	if ((m->m_flags & M_PKTHDR) == 0)
-		panic("lo_output: no HDR");
-
-	/*
-	 * Don't overwrite the rcvif field if it is in use.
-	 *  This is used to match multicast packets, sent looping
-	 *  back, with the appropriate group record on input.
-	 */
-	if (m->m_pkthdr.rcvif == NULL)
-		m->m_pkthdr.rcvif = ifp;
-
-	ifp->if_ibytes += m->m_pkthdr.len;
-	ifp->if_obytes += m->m_pkthdr.len;
+		/*
+		 * Don't overwrite the rcvif field if it is in use.
+		 *  This is used to match multicast packets, sent looping
+		 *  back, with the appropriate group record on input.
+		 */
+		if (m->m_pkthdr.rcvif == NULL)
+			m->m_pkthdr.rcvif = ifp;
 
-	ifp->if_opackets++;
-	ifp->if_ipackets++;
+		ifp->if_ibytes += m->m_pkthdr.len;
+		ifp->if_obytes += m->m_pkthdr.len;
 
-	m->m_pkthdr.header = mtod(m, char *);
-	m->m_pkthdr.csum_data = 0xffff; /* loopback checksums are always OK */
-	m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR | 
-							 CSUM_IP_CHECKED | CSUM_IP_VALID;
-	m_adj(m, sizeof(struct loopback_header));
+		ifp->if_opackets++;
+		ifp->if_ipackets++;
 
-#if NBPFILTER > 0
-	if (lo_statics[ifp->if_unit].bpf_mode != BPF_TAP_DISABLE) {
-		struct mbuf m0, *n;
-
-		n = m;
-		if (ifp->if_bpf->bif_dlt == DLT_NULL) {
-			struct loopback_header  *header;
-			/*
-			 * We need to prepend the address family as
-			 * a four byte field.  Cons up a dummy header
-			 * to pacify bpf.  This is safe because bpf
-			 * will only read from the mbuf (i.e., it won't
-			 * try to free it or keep a pointer a to it).
-			 */
-			header = (struct loopback_header*)m->m_pkthdr.header;
-			m0.m_next = m;
-			m0.m_len = 4;
-			m0.m_data = (char *)&header->protocol;
-			n = &m0;
+		m->m_pkthdr.header = mtod(m, char *);
+		if (apple_hwcksum_tx != 0) {
+			/* loopback checksums are always OK */
+			m->m_pkthdr.csum_data = 0xffff;
+			m->m_pkthdr.csum_flags = CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
+				CSUM_IP_CHECKED | CSUM_IP_VALID;
+		}
+		m_adj(m, sizeof(struct loopback_header));
+
+		{
+			/* We need to prepend the address family as a four byte field. */
+			u_int32_t protocol_family =
+				((struct loopback_header*)m->m_pkthdr.header)->protocol;
+		
+			bpf_tap_out(ifp, DLT_NULL, m, &protocol_family, sizeof(protocol_family));
 		}
-
-		lo_statics[ifp->if_unit].bpf_callback(ifp, n);
 	}
-#endif
 
-	return dlil_input(ifp, m, m);
+	return ifnet_input(ifp, m_list, NULL);
 }
 
 
@@ -245,18 +245,20 @@ lo_output(
  * (should?) be split into separate pre-output routines for each protocol.
  */
 
-static int
+static errno_t
 lo_pre_output(
-	__unused struct ifnet	*ifp,
-	u_long			protocol_family,
-	struct mbuf		**m,
+	__unused ifnet_t	ifp,
+	protocol_family_t	protocol_family,
+	mbuf_t				*m,
 	__unused const struct sockaddr	*dst,
-	caddr_t			route,
-	char			*frame_type,
-	__unused char	*dst_addr)
+	void				*route,
+	char				*frame_type,
+	__unused char		*dst_addr)
 
 {
-	register struct rtentry *rt = (struct rtentry *) route;
+	register struct rtentry *rt = route;
+
+	(*m)->m_flags |= M_LOOP;
 
 	if (((*m)->m_flags & M_PKTHDR) == 0)
 		panic("looutput no HDR");
@@ -270,7 +272,7 @@ lo_pre_output(
 			return ((rt->rt_flags & RTF_HOST) ? EHOSTUNREACH : ENETUNREACH);
 	}
 	
-	*(u_long *)frame_type = protocol_family;
+	*(protocol_family_t*)frame_type = protocol_family;
 
 	return 0;
 }
@@ -279,13 +281,11 @@ lo_pre_output(
  *  lo_input - This should work for all attached protocols that use the
  *             ifq/schednetisr input mechanism.
  */
-static int
+static errno_t
 lo_input(
-	struct mbuf				*m,
-	__unused char			*fh,
-	__unused struct ifnet	*ifp,
-	__unused u_long			protocol_family,
-	__unused int			sync_ok)
+	__unused ifnet_t			ifp,
+	__unused protocol_family_t	protocol_family,
+	mbuf_t						m)
 {
 	if (proto_input(protocol_family, m) != 0)
 		m_freem(m);
@@ -377,39 +377,35 @@ loioctl(
 #endif /* NLOOP > 0 */
 
 
-static int  lo_attach_proto(struct ifnet *ifp, u_long protocol_family)
+static errno_t  lo_attach_proto(ifnet_t ifp, protocol_family_t protocol_family)
 {
-	struct dlil_proto_reg_str   reg;
-	int   stat =0 ;
+	struct ifnet_attach_proto_param_v2	proto;
+	errno_t							result = 0;
 	
-	bzero(&reg, sizeof(reg));
-	TAILQ_INIT(&reg.demux_desc_head);
-	reg.interface_family = ifp->if_family;
-	reg.unit_number      = ifp->if_unit;
-	reg.input			 = lo_input;
-	reg.pre_output       = lo_pre_output;
-	reg.protocol_family  = protocol_family;
+	bzero(&proto, sizeof(proto));
+	proto.input = lo_input;
+	proto.pre_output = lo_pre_output;
 	
-	stat = dlil_attach_protocol(&reg);
+	result = ifnet_attach_protocol_v2(ifp, protocol_family, &proto);
 
-	if (stat && stat != EEXIST) {
-		printf("lo_attach_proto: dlil_attach_protocol for %d returned=%d\n",
-			   protocol_family, stat);
+	if (result && result != EEXIST) {
+		printf("lo_attach_proto: ifnet_attach_protocol for %u returned=%d\n",
+			   protocol_family, result);
 	}
 	
-	return stat;
+	return result;
 }
 
-void lo_reg_if_mods()
+static void lo_reg_if_mods(void)
 {
      int error;
 
 	/* Register protocol registration functions */
-	if ((error = dlil_reg_proto_module(PF_INET, APPLE_IF_FAM_LOOPBACK, lo_attach_proto, NULL)) != 0)
-		printf("dlil_reg_proto_module failed for AF_INET error=%d\n", error);
+	if ((error = proto_register_plumber(PF_INET, APPLE_IF_FAM_LOOPBACK, lo_attach_proto, NULL)) != 0)
+		printf("proto_register_plumber failed for AF_INET error=%d\n", error);
 
-	if ((error = dlil_reg_proto_module(PF_INET6, APPLE_IF_FAM_LOOPBACK, lo_attach_proto, NULL)) != 0)
-		printf("dlil_reg_proto_module failed for AF_INET6 error=%d\n", error);
+	if ((error = proto_register_plumber(PF_INET6, APPLE_IF_FAM_LOOPBACK, lo_attach_proto, NULL)) != 0)
+		printf("proto_register_plumber failed for AF_INET6 error=%d\n", error);
 }
 
 static errno_t
@@ -437,40 +433,55 @@ lo_set_bpf_tap(
 
 /* ARGSUSED */
 void
-loopattach(
-	__unused void *dummy)
+loopattach(void)
 {
-	struct ifnet *ifp;
-	int i = 0;
+	struct ifnet_init_params	lo_init;
+	errno_t	result = 0;
 
-	lo_reg_if_mods();
+#if NLOOP != 1
+More than one loopback interface is not supported.
+#endif
 
-	for (ifp = loif; i < NLOOP; ifp++) {
-		lo_statics[i].bpf_callback = 0;
-		lo_statics[i].bpf_mode      = BPF_TAP_DISABLE;
-		bzero(ifp, sizeof(struct ifnet));
-		ifp->if_name = "lo";
-		ifp->if_family = APPLE_IF_FAM_LOOPBACK;
-		ifp->if_unit = i++;
-		ifp->if_mtu = LOMTU;
-		ifp->if_flags = IFF_LOOPBACK | IFF_MULTICAST;
-		ifp->if_ioctl = loioctl;
-		ifp->if_demux = lo_demux;
-		ifp->if_framer = lo_framer;
-		ifp->if_add_proto = lo_add_proto;
-		ifp->if_del_proto = lo_del_proto;
-		ifp->if_set_bpf_tap = lo_set_bpf_tap;
-		ifp->if_output = lo_output;
-		ifp->if_type = IFT_LOOP;
-		ifp->if_hwassist = IF_HWASSIST_CSUM_IP | IF_HWASSIST_CSUM_TCP |
-		    IF_HWASSIST_CSUM_UDP | IF_HWASSIST_CSUM_IP_FRAGS |
-		    IF_HWASSIST_CSUM_FRAGMENT;
-		ifp->if_hdrlen = sizeof(struct loopback_header);
-		lo_ifp = ifp;
-		dlil_if_attach(ifp);
-#if NBPFILTER > 0
-		bpfattach(ifp, DLT_NULL, sizeof(u_int));
+	lo_reg_if_mods();
+	
+	lo_statics[0].bpf_callback = 0;
+	lo_statics[0].bpf_mode      = BPF_TAP_DISABLE;
+	
+	bzero(&lo_init, sizeof(lo_init));
+	lo_init.name = "lo";
+	lo_init.unit = 0;
+	lo_init.family = IFNET_FAMILY_LOOPBACK;
+	lo_init.type = IFT_LOOP;
+	lo_init.output = lo_output;
+	lo_init.demux = lo_demux;
+	lo_init.add_proto = lo_add_proto;
+	lo_init.del_proto = lo_del_proto;
+	lo_init.framer = lo_framer;
+	lo_init.softc = &lo_statics[0];
+	lo_init.ioctl = loioctl;
+	lo_init.set_bpf_tap = lo_set_bpf_tap;
+	result = ifnet_allocate(&lo_init, &lo_ifp);
+	if (result != 0) {
+		printf("ifnet_allocate for lo0 failed - %d\n", result);
+		return;
+	}
+	
+	ifnet_set_mtu(lo_ifp, LOMTU);
+	ifnet_set_flags(lo_ifp, IFF_LOOPBACK | IFF_MULTICAST, IFF_LOOPBACK | IFF_MULTICAST);
+	ifnet_set_offload(lo_ifp, IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP | IFNET_CSUM_FRAGMENT | IFNET_IP_FRAGMENT | IFNET_MULTIPAGES);
+	ifnet_set_hdrlen(lo_ifp, sizeof(struct loopback_header));
+	ifnet_set_eflags(lo_ifp, IFEF_SENDLIST, IFEF_SENDLIST);
+
+#if CONFIG_MACF_NET
+		mac_ifnet_label_init(ifp);
 #endif
+
+	result = ifnet_attach(lo_ifp, NULL);
+	if (result != 0) {
+		printf("ifnet_attach lo0 failed - %d\n", result);
+		return;
 	}
+	bpfattach(lo_ifp, DLT_NULL, sizeof(u_int));
+	
 	loopattach_done = 1;
 }
diff --git a/bsd/net/if_media.c b/bsd/net/if_media.c
index f7c4e96dc..d96442ada 100644
--- a/bsd/net/if_media.c
+++ b/bsd/net/if_media.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: if_media.c,v 1.1 1997/03/17 02:55:15 thorpej Exp $	*/
 /* $FreeBSD: src/sys/net/if_media.c,v 1.9.2.4 2001/07/04 00:12:38 brooks Exp $ */
diff --git a/bsd/net/if_media.h b/bsd/net/if_media.h
index 077fc3e69..6c8ac892f 100644
--- a/bsd/net/if_media.h
+++ b/bsd/net/if_media.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: if_media.h,v 1.3 1997/03/26 01:19:27 thorpej Exp $	*/
 /* $FreeBSD: src/sys/net/if_media.h,v 1.9.2.1 2001/07/04 00:12:38 brooks Exp $ */
diff --git a/bsd/net/if_mib.c b/bsd/net/if_mib.c
index d78dabb5b..17e1433bc 100644
--- a/bsd/net/if_mib.c
+++ b/bsd/net/if_mib.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1996 Massachusetts Institute of Technology
@@ -87,7 +93,7 @@
 
 SYSCTL_DECL(_net_link_generic);
 
-SYSCTL_NODE(_net_link_generic, IFMIB_SYSTEM, system, CTLFLAG_RD, 0,
+SYSCTL_NODE(_net_link_generic, IFMIB_SYSTEM, system, CTLFLAG_RD|CTLFLAG_LOCKED, 0,
 	    "Variables global to all interfaces");
 
 SYSCTL_INT(_net_link_generic_system, IFMIB_IFCOUNT, ifcount, CTLFLAG_RD,
@@ -101,6 +107,15 @@ static int sysctl_ifalldata SYSCTL_HANDLER_ARGS;
 SYSCTL_NODE(_net_link_generic, IFMIB_IFALLDATA, ifalldata, CTLFLAG_RD,
             sysctl_ifalldata, "Interface table");
 
+extern int dlil_multithreaded_input;
+SYSCTL_INT(_net_link_generic_system, OID_AUTO, multi_threaded_input, CTLFLAG_RW,
+		    &dlil_multithreaded_input , 0, "Uses multiple input thread for DLIL input");
+#ifdef IFNET_INPUT_SANITY_CHK
+extern int dlil_input_sanity_check;
+SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_input_sanity_check, CTLFLAG_RW,
+		    &dlil_input_sanity_check , 0, "Turn on sanity checking in DLIL input");
+#endif
+
 static int make_ifmibdata(struct ifnet *, int *, struct sysctl_req *);
 
 
@@ -124,7 +139,7 @@ make_ifmibdata(struct ifnet *ifp, int *name, struct sysctl_req *req)
 #define COPY(fld) ifmd.ifmd_##fld = ifp->if_##fld
 		COPY(pcount);
 		COPY(flags);
-		if_data_internal_to_if_data64(&ifp->if_data, &ifmd.ifmd_data);
+		if_data_internal_to_if_data64(ifp, &ifp->if_data, &ifmd.ifmd_data);
 #undef COPY
 		ifmd.ifmd_snd_len = ifp->if_snd.ifq_len;
 		ifmd.ifmd_snd_maxlen = ifp->if_snd.ifq_maxlen;
diff --git a/bsd/net/if_mib.h b/bsd/net/if_mib.h
index 0175e68a9..5b0d38a14 100644
--- a/bsd/net/if_mib.h
+++ b/bsd/net/if_mib.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1996 Massachusetts Institute of Technology
diff --git a/bsd/net/if_ppp.h b/bsd/net/if_ppp.h
index 9d5f7cd36..ae89246e2 100644
--- a/bsd/net/if_ppp.h
+++ b/bsd/net/if_ppp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * if_ppp.h - Point-to-Point Protocol definitions.
diff --git a/bsd/net/if_pppvar.h b/bsd/net/if_pppvar.h
index 35628358c..c717c0d88 100644
--- a/bsd/net/if_pppvar.h
+++ b/bsd/net/if_pppvar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * if_pppvar.h - private structures and declarations for PPP.
diff --git a/bsd/net/if_stf.c b/bsd/net/if_stf.c
index c9d646ea1..5f86780c0 100644
--- a/bsd/net/if_stf.c
+++ b/bsd/net/if_stf.c
@@ -29,6 +29,12 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * 6to4 interface, based on RFC3056.
@@ -102,70 +108,88 @@
 #include <netinet/ip_ecn.h>
 
 #include <netinet/ip_encap.h>
-#include <net/dlil.h>
+#include <net/kpi_interface.h>
+#include <net/kpi_protocol.h>
 
 
 #include <net/net_osdep.h>
 
 #include <net/bpf.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #define IN6_IS_ADDR_6TO4(x)	(ntohs((x)->s6_addr16[0]) == 0x2002)
-#define GET_V4(x)	((struct in_addr *)(&(x)->s6_addr16[1]))
+#define GET_V4(x)	((const struct in_addr *)(&(x)->s6_addr16[1]))
 
 struct stf_softc {
-	struct ifnet	sc_if;	   /* common area */
-#ifdef __APPLE__
-	u_long sc_protocol_family; /* dlil protocol attached */
-#endif
+	ifnet_t				sc_if;	   /* common area */
+	u_long				sc_protocol_family; /* dlil protocol attached */
 	union {
 		struct route  __sc_ro4;
 		struct route_in6 __sc_ro6; /* just for safety */
 	} __sc_ro46;
 #define sc_ro	__sc_ro46.__sc_ro4
 	const struct encaptab *encap_cookie;
+	bpf_tap_mode		tap_mode;
+	bpf_packet_func		tap_callback;
 };
 
-static struct stf_softc *stf;
-
-#ifdef __APPLE__
 void stfattach (void);
-#endif
 
-#ifndef __APPLE__
-static MALLOC_DEFINE(M_STF, "stf", "6to4 Tunnel Interface");
-#endif
 static int ip_stf_ttl = 40;
 
 static void in_stf_input(struct mbuf *, int);
 extern  struct domain inetdomain;
 struct protosw in_stf_protosw =
 { SOCK_RAW,	&inetdomain,	IPPROTO_IPV6,	PR_ATOMIC|PR_ADDR,
-  in_stf_input, 0,	0,		rip_ctloutput,
-  0,
-  0,
+  in_stf_input, NULL,	NULL,		rip_ctloutput,
+  NULL,
+  NULL,		NULL,	NULL,	NULL,
+  NULL,
   &rip_usrreqs,
-  0,            rip_unlock,	0
+  NULL,		rip_unlock,	NULL, {NULL, NULL}, NULL, {0}
 };
 
 static int stf_encapcheck(const struct mbuf *, int, int, void *);
 static struct in6_ifaddr *stf_getsrcifa6(struct ifnet *);
-int stf_pre_output(struct ifnet *, u_long, register struct mbuf **,
-	const struct sockaddr *, caddr_t, char *, char *);
-static int stf_checkaddr4(struct stf_softc *, struct in_addr *,
+int stf_pre_output(struct ifnet *, protocol_family_t, struct mbuf **,
+	const struct sockaddr *, void *, char *, char *);
+static int stf_checkaddr4(struct stf_softc *, const struct in_addr *,
 	struct ifnet *);
 static int stf_checkaddr6(struct stf_softc *, struct in6_addr *,
 	struct ifnet *);
 static void stf_rtrequest(int, struct rtentry *, struct sockaddr *);
-int stf_ioctl(struct ifnet *, u_long, void *);
-
-static
-int  stf_add_proto(
-	struct ifnet *ifp,
-	u_long protocol_family,
-	struct ddesc_head_str *desc_head)
-{       
+static errno_t stf_ioctl(ifnet_t ifp, u_int32_t cmd, void *data);
+static errno_t stf_output(ifnet_t ifp, mbuf_t m);
+
+/*
+ * gif_input is the input handler for IP and IPv6 attached to gif
+ */
+static errno_t
+stf_media_input(
+	__unused ifnet_t	ifp,
+	protocol_family_t	protocol_family,
+	mbuf_t				m,
+	__unused char		*frame_header)
+{
+	proto_input(protocol_family, m);
+
+	return (0);
+}
+
+
+
+static errno_t
+stf_add_proto(
+	ifnet_t									ifp,
+	protocol_family_t						protocol_family,
+	__unused const struct ifnet_demux_desc	*demux_array,
+	__unused u_int32_t						demux_count)
+{
 	/* Only one protocol may be attached at a time */
-	struct stf_softc* stf = (struct stf_softc*)ifp;
+	struct stf_softc* stf = ifnet_softc(ifp);
 	if (stf->sc_protocol_family == 0)
 		stf->sc_protocol_family = protocol_family;
 	else {
@@ -176,115 +200,148 @@ int  stf_add_proto(
 	return 0;
 }
 
-static
-int  stf_del_proto(
-	struct ifnet *ifp,
-	u_long protocol_family)
-{   
-	if (((struct stf_softc*)ifp)->sc_protocol_family == protocol_family)
-		((struct stf_softc*)ifp)->sc_protocol_family = 0;
-	else
-		return ENOENT;
-
+static errno_t
+stf_del_proto(
+	ifnet_t				ifp,
+	protocol_family_t	protocol_family)
+{
+	if (((struct stf_softc*)ifnet_softc(ifp))->sc_protocol_family == protocol_family)
+		((struct stf_softc*)ifnet_softc(ifp))->sc_protocol_family = 0;
+	
 	return 0;
 }
 
-static int
-stf_attach_inet6(struct ifnet *ifp, u_long protocol_family)
-{       
-    struct dlil_proto_reg_str   reg;
-    int   stat, i;
+static errno_t
+stf_attach_inet6(
+	ifnet_t				ifp,
+	protocol_family_t	protocol_family)
+{
+    struct ifnet_attach_proto_param	reg;
+    errno_t							stat;
+    
+    if (protocol_family != PF_INET6)
+    	return EPROTONOSUPPORT;
 
 	bzero(&reg, sizeof(reg));
-    TAILQ_INIT(&reg.demux_desc_head); 
-    reg.interface_family = ifp->if_family;
-    reg.unit_number      = ifp->if_unit;
-    reg.pre_output       = stf_pre_output;
-    reg.protocol_family  = PF_INET6;
+    reg.input = stf_media_input;
+    reg.pre_output = stf_pre_output;
 
-    stat = dlil_attach_protocol(&reg);
+    stat = ifnet_attach_protocol(ifp, protocol_family, &reg);
+    if (stat && stat != EEXIST) {
+        printf("stf_attach_proto_family can't attach interface fam=%d\n",
+        	   protocol_family);
+    }
 
     return stat;
 }
 
-static int
+static errno_t
 stf_demux(
-	struct ifnet *ifp,
-	struct mbuf *m,
-	char *frame_ptr,
-	u_long *protocol_family)
+	ifnet_t					ifp,
+	__unused mbuf_t			m,
+	__unused char			*frame_ptr,
+	protocol_family_t		*protocol_family)
 {
-	*protocol_family = PF_INET6;
+	struct stf_softc* stf = ifnet_softc(ifp);
+	*protocol_family = stf->sc_protocol_family;
 	return 0;
 }
 
-void stf_reg_if_mods()
-{   
-	int error;
-
-	/* Register protocol registration functions */
-	if ( error = dlil_reg_proto_module(AF_INET6, APPLE_IF_FAM_STF, stf_attach_inet6, NULL) != 0)
-		kprintf("dlil_reg_proto_module failed for AF_INET6 error=%d\n", error);
+static errno_t
+stf_set_bpf_tap(
+	ifnet_t			ifp,
+	bpf_tap_mode	mode,
+	bpf_packet_func	callback)
+{
+	struct stf_softc	*sc = ifnet_softc(ifp);
+	
+	sc->tap_mode = mode;
+	sc->tap_callback = callback;
+	
+	return 0;
 }
 
 void
 stfattach(void)
 {
-	struct ifnet *ifp;
 	struct stf_softc *sc;
-	int i, error;
-	int err;
+	int error;
 	const struct encaptab *p;
+	struct ifnet_init_params	stf_init;
 
-	stf_reg_if_mods(); /* DLIL modules */
+	error = proto_register_plumber(PF_INET6, APPLE_IF_FAM_STF,
+								   stf_attach_inet6, NULL);
+	if (error != 0)
+		printf("proto_register_plumber failed for AF_INET6 error=%d\n", error);
 
 	sc = _MALLOC(sizeof(struct stf_softc), M_DEVBUF, M_WAITOK);
 	if (sc == 0) {
 		printf("stf softc attach failed\n" );
 		return;
 	}
-		
+	
 	bzero(sc, sizeof(*sc));
-	sc->sc_if.if_name = "stf";
-	sc->sc_if.if_unit = 0;
-
+	
 	p = encap_attach_func(AF_INET, IPPROTO_IPV6, stf_encapcheck,
 	    &in_stf_protosw, sc);
 	if (p == NULL) {
-		printf("%s: attach failed\n", if_name(&sc->sc_if));
+		printf("sftattach encap_attach_func failed\n");
 		FREE(sc, M_DEVBUF);
 		return;
 	}
 	sc->encap_cookie = p;
-	sc->sc_if.if_mtu    = IPV6_MMTU;
-	sc->sc_if.if_flags  = 0;
-	sc->sc_if.if_ioctl  = stf_ioctl;
-	sc->sc_if.if_output = NULL; /* processing done in pre_output */
-	sc->sc_if.if_type   = IFT_STF;
-	sc->sc_if.if_family= APPLE_IF_FAM_STF;
-	sc->sc_if.if_add_proto = stf_add_proto;
-	sc->sc_if.if_del_proto = stf_del_proto;
-	sc->sc_if.if_demux = stf_demux;
+	
+	bzero(&stf_init, sizeof(stf_init));
+	stf_init.name = "stf";
+	stf_init.unit = 0;
+	stf_init.type = IFT_STF;
+	stf_init.family = IFNET_FAMILY_STF;
+	stf_init.output = stf_output;
+	stf_init.demux = stf_demux;
+	stf_init.add_proto = stf_add_proto;
+	stf_init.del_proto = stf_del_proto;
+	stf_init.softc = sc;
+	stf_init.ioctl = stf_ioctl;
+	stf_init.set_bpf_tap = stf_set_bpf_tap;
+	
+	error = ifnet_allocate(&stf_init, &sc->sc_if);
+	if (error != 0) {
+		printf("stfattach, ifnet_allocate failed - %d\n", error);
+		encap_detach(sc->encap_cookie);
+		FREE(sc, M_DEVBUF);
+		return;
+	}
+	ifnet_set_mtu(sc->sc_if, IPV6_MMTU);
+	ifnet_set_flags(sc->sc_if, 0, 0xffff); /* clear all flags */
 #if 0
 	/* turn off ingress filter */
-	sc->sc_if.if_flags  |= IFF_LINK2;
+	ifnet_set_flags(sc->sc_if, IFF_LINK2, IFF_LINK2);
 #endif
-	sc->sc_if.if_snd.ifq_maxlen = IFQ_MAXLEN;
 
-	if (error = dlil_if_attach(&sc->sc_if))
-		printf("stfattach: can't dlil_if_attach error=%d\n");
-	else 
-		bpfattach(&sc->sc_if, DLT_NULL, sizeof(u_int));
+#if CONFIG_MACF_NET
+	mac_ifnet_label_init(&sc->sc_if);
+#endif
+	
+	error = ifnet_attach(sc->sc_if, NULL);
+	if (error != 0) {
+		printf("stfattach: ifnet_attach returned error=%d\n", error);
+		encap_detach(sc->encap_cookie);
+		ifnet_release(sc->sc_if);
+		FREE(sc, M_DEVBUF);
+		return;
+	}
 	
-	return ;
+	bpfattach(sc->sc_if, DLT_NULL, sizeof(u_int));
+	
+	return;
 }
 
 static int
-stf_encapcheck(m, off, proto, arg)
-	const struct mbuf *m;
-	int off;
-	int proto;
-	void *arg;
+stf_encapcheck(
+	const struct mbuf *m,
+	__unused int off,
+	int proto,
+	void *arg)
 {
 	struct ip ip;
 	struct in6_ifaddr *ia6;
@@ -295,23 +352,23 @@ stf_encapcheck(m, off, proto, arg)
 	if (sc == NULL)
 		return 0;
 
-	if ((sc->sc_if.if_flags & IFF_UP) == 0)
+	if ((ifnet_flags(sc->sc_if) & IFF_UP) == 0)
 		return 0;
 
 	/* IFF_LINK0 means "no decapsulation" */
-	if ((sc->sc_if.if_flags & IFF_LINK0) != 0)
+	if ((ifnet_flags(sc->sc_if) & IFF_LINK0) != 0)
 		return 0;
 
 	if (proto != IPPROTO_IPV6)
 		return 0;
 
 	/* LINTED const cast */
-	m_copydata((struct mbuf *)m, 0, sizeof(ip), (caddr_t)&ip);
+	mbuf_copydata(m, 0, sizeof(ip), &ip);
 
 	if (ip.ip_v != 4)
 		return 0;
 
-	ia6 = stf_getsrcifa6(&sc->sc_if);
+	ia6 = stf_getsrcifa6(sc->sc_if);
 	if (ia6 == NULL)
 		return 0;
 
@@ -343,8 +400,7 @@ stf_encapcheck(m, off, proto, arg)
 }
 
 static struct in6_ifaddr *
-stf_getsrcifa6(ifp)
-	struct ifnet *ifp;
+stf_getsrcifa6(struct ifnet *ifp)
 {
 	struct ifaddr *ia;
 	struct in_ifaddr *ia4;
@@ -387,30 +443,30 @@ stf_getsrcifa6(ifp)
 
 int
 stf_pre_output(
-	struct ifnet *ifp,
-	u_long protocol_family,
-	register struct mbuf **m0,
-	const struct sockaddr *dst,
-	caddr_t			rt,
-	char		     *frame_type,
-	char		     *address)
+	struct ifnet	*ifp,
+	__unused protocol_family_t  protocol_family,
+	struct mbuf	**m0,
+	const struct sockaddr	*dst,
+	__unused void *route,
+	__unused char *desk_linkaddr,
+	__unused char *frame_type)
 {
-	register struct mbuf *m = *m0;
+	struct mbuf *m = *m0;
 	struct stf_softc *sc;
-	struct sockaddr_in6 *dst6;
-	struct in_addr *in4;
-	struct sockaddr_in *dst4;
+	const struct sockaddr_in6 *dst6;
+	const struct in_addr *in4;
 	u_int8_t tos;
 	struct ip *ip;
 	struct ip6_hdr *ip6;
 	struct in6_ifaddr *ia6;
-	int error = 0 ;
+	struct sockaddr_in 	*dst4;
+	errno_t				result = 0;
 
-	sc = (struct stf_softc*)ifp;
-	dst6 = (struct sockaddr_in6 *)dst;
+	sc = ifnet_softc(ifp);
+	dst6 = (const struct sockaddr_in6 *)dst;
 
 	/* just in case */
-	if ((ifp->if_flags & IFF_UP) == 0) {
+	if ((ifnet_flags(ifp) & IFF_UP) == 0) {
 		printf("stf: IFF_DOWN\n");
 		return ENETDOWN;
 	}
@@ -425,7 +481,7 @@ stf_pre_output(
 		return ENETDOWN;
 	}
 
-	if (m->m_len < sizeof(*ip6)) {
+	if (mbuf_len(m) < sizeof(*ip6)) {
 		m = m_pullup(m, sizeof(*ip6));
 		if (!m) {
 			*m0 = NULL; /* makes sure this won't be double freed */
@@ -448,25 +504,14 @@ stf_pre_output(
 	}
 
 	if (ifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m1;
+		/* We need to prepend the address family as a four byte field. */
 		u_int32_t af = AF_INET6;
 		
-		m1.m_next = m;
-		m1.m_len = 4;
-		m1.m_data = (char *)&af;
-		
-		bpf_mtap(ifp, &m1);
+		bpf_tap_out(ifp, 0, m, &af, sizeof(af));
 	}
 
 	M_PREPEND(m, sizeof(struct ip), M_DONTWAIT);
-	if (m && m->m_len < sizeof(struct ip))
+	if (m && mbuf_len(m) < sizeof(struct ip))
 		m = m_pullup(m, sizeof(struct ip));
 	if (m == NULL) {
 		*m0 = NULL; 
@@ -491,6 +536,7 @@ stf_pre_output(
 	if (dst4->sin_family != AF_INET ||
 	    bcmp(&dst4->sin_addr, &ip->ip_dst, sizeof(ip->ip_dst)) != 0) {
 		/* cache route doesn't match */
+		printf("stf_output: cached route doesn't match \n");
 		dst4->sin_family = AF_INET;
 		dst4->sin_len = sizeof(struct sockaddr_in);
 		bcopy(&ip->ip_dst, &dst4->sin_addr, sizeof(dst4->sin_addr));
@@ -507,19 +553,30 @@ stf_pre_output(
 		}
 	}
 
-	error = ip_output(m, NULL, &sc->sc_ro, 0, NULL);
-	if (error == 0)
-		return EJUSTRETURN;
-
-	*m0 = NULL; 
-	return error;
+	result = ip_output_list(m, 0, NULL, &sc->sc_ro, 0, NULL, NULL);
+	/* Assumption: ip_output will free mbuf on errors */
+	/* All the output processing is done here, don't let stf_output be called */
+	if (result == 0)
+		result = EJUSTRETURN;
+	*m0 = NULL;
+	return result;
 }
+static errno_t
+stf_output(
+	__unused ifnet_t	ifp,
+	__unused mbuf_t	m)
+{
+	/* All processing is done in stf_pre_output
+	 * this shouldn't be called as the pre_output returns "EJUSTRETURN"
+	 */
+	return 0;
+}	
 
 static int
-stf_checkaddr4(sc, in, inifp)
-	struct stf_softc *sc;
-	struct in_addr *in;
-	struct ifnet *inifp;	/* incoming interface */
+stf_checkaddr4(
+	struct stf_softc *sc,
+	const struct in_addr *in,
+	struct ifnet *inifp)	/* incoming interface */
 {
 	struct in_ifaddr *ia4;
 
@@ -554,7 +611,7 @@ stf_checkaddr4(sc, in, inifp)
 	/*
 	 * perform ingress filter
 	 */
-	if (sc && (sc->sc_if.if_flags & IFF_LINK2) == 0 && inifp) {
+	if (sc && (ifnet_flags(sc->sc_if) & IFF_LINK2) == 0 && inifp) {
 		struct sockaddr_in sin;
 		struct rtentry *rt;
 
@@ -566,7 +623,7 @@ stf_checkaddr4(sc, in, inifp)
 		if (!rt || rt->rt_ifp != inifp) {
 #if 1
 			log(LOG_WARNING, "%s: packet from 0x%x dropped "
-			    "due to ingress filter\n", if_name(&sc->sc_if),
+			    "due to ingress filter\n", if_name(sc->sc_if),
 			    (u_int32_t)ntohl(sin.sin_addr.s_addr));
 #endif
 			if (rt)
@@ -580,10 +637,10 @@ stf_checkaddr4(sc, in, inifp)
 }
 
 static int
-stf_checkaddr6(sc, in6, inifp)
-	struct stf_softc *sc;
-	struct in6_addr *in6;
-	struct ifnet *inifp;	/* incoming interface */
+stf_checkaddr6(
+	struct stf_softc *sc,
+	struct in6_addr *in6,
+	struct ifnet *inifp)	/* incoming interface */
 {
 	/*
 	 * check 6to4 addresses
@@ -604,21 +661,21 @@ stf_checkaddr6(sc, in6, inifp)
 }
 
 static void
-in_stf_input(m, off)
-	struct mbuf *m;
-	int off;
+in_stf_input(
+	struct mbuf *m,
+	int off)
 {
 	struct stf_softc *sc;
 	struct ip *ip;
-	struct ip6_hdr *ip6;
+	struct ip6_hdr ip6;
 	u_int8_t otos, itos;
 	int proto;
 	struct ifnet *ifp;
+	struct ifnet_stat_increment_param	stats;
 
 	ip = mtod(m, struct ip *);
 	proto = ip->ip_p;
 
-
 	if (proto != IPPROTO_IPV6) {
 		m_freem(m);
 		return;
@@ -628,12 +685,16 @@ in_stf_input(m, off)
 
 	sc = (struct stf_softc *)encap_getarg(m);
 
-	if (sc == NULL || (sc->sc_if.if_flags & IFF_UP) == 0) {
+	if (sc == NULL || (ifnet_flags(sc->sc_if) & IFF_UP) == 0) {
 		m_freem(m);
 		return;
 	}
 
-	ifp = &sc->sc_if;
+	ifp = sc->sc_if;
+
+#if MAC_LABEL
+	mac_mbuf_label_associate_ifnet(ifp, m);
+#endif
 
 	/*
 	 * perform sanity check against outer src/dst.
@@ -646,55 +707,34 @@ in_stf_input(m, off)
 	}
 
 	otos = ip->ip_tos;
-	m_adj(m, off);
-
-	if (m->m_len < sizeof(*ip6)) {
-		m = m_pullup(m, sizeof(*ip6));
-		if (!m)
-			return;
-	}
-	ip6 = mtod(m, struct ip6_hdr *);
+	mbuf_copydata(m, off, sizeof(ip6), &ip6);
 
 	/*
 	 * perform sanity check against inner src/dst.
 	 * for source, perform ingress filter as well.
 	 */
-	if (stf_checkaddr6(sc, &ip6->ip6_dst, NULL) < 0 ||
-	    stf_checkaddr6(sc, &ip6->ip6_src, m->m_pkthdr.rcvif) < 0) {
+	if (stf_checkaddr6(sc, &ip6.ip6_dst, NULL) < 0 ||
+	    stf_checkaddr6(sc, &ip6.ip6_src, m->m_pkthdr.rcvif) < 0) {
 		m_freem(m);
 		return;
 	}
 
-	itos = (ntohl(ip6->ip6_flow) >> 20) & 0xff;
-	if ((ifp->if_flags & IFF_LINK1) != 0)
+	itos = (ntohl(ip6.ip6_flow) >> 20) & 0xff;
+	if ((ifnet_flags(ifp) & IFF_LINK1) != 0)
 		ip_ecn_egress(ECN_ALLOWED, &otos, &itos);
 	else
 		ip_ecn_egress(ECN_NOCARE, &otos, &itos);
-	ip6->ip6_flow &= ~htonl(0xff << 20);
-	ip6->ip6_flow |= htonl((u_int32_t)itos << 20);
+	ip6.ip6_flow &= ~htonl(0xff << 20);
+	ip6.ip6_flow |= htonl((u_int32_t)itos << 20);
 
 	m->m_pkthdr.rcvif = ifp;
+	mbuf_pkthdr_setheader(m, mbuf_data(m));
+	mbuf_adj(m, off);
 	
 	if (ifp->if_bpf) {
-		/*
-		 * We need to prepend the address family as
-		 * a four byte field.  Cons up a dummy header
-		 * to pacify bpf.  This is safe because bpf
-		 * will only read from the mbuf (i.e., it won't
-		 * try to free it or keep a pointer a to it).
-		 */
-		struct mbuf m0;
+		/* We need to prepend the address family as a four byte field. */
 		u_int32_t af = AF_INET6;
-		
-		m0.m_next = m;
-		m0.m_len = 4;
-		m0.m_data = (char *)&af;
-		
-#ifdef HAVE_OLD_BPF
-		bpf_mtap(ifp, &m0);
-#else
-		bpf_mtap(ifp->if_bpf, &m0);
-#endif
+		bpf_tap_in(ifp, 0, m, &af, sizeof(af));
 	}
 
 	/*
@@ -703,30 +743,31 @@ in_stf_input(m, off)
 	 * See net/if_gif.c for possible issues with packet processing
 	 * reorder due to extra queueing.
 	 */
-	proto_input(PF_INET6, m);
-	ifp->if_ipackets++;
-	ifp->if_ibytes += m->m_pkthdr.len;
-
+	bzero(&stats, sizeof(stats));
+	stats.packets_in = 1;
+	stats.bytes_in = mbuf_pkthdr_len(m);
+	mbuf_pkthdr_setrcvif(m, ifp);
+	ifnet_input(ifp, m, &stats);
+	
 	return;
 }
 
-/* ARGSUSED */
 static void
-stf_rtrequest(cmd, rt, sa)
-	int cmd;
-	struct rtentry *rt;
-	struct sockaddr *sa;
+stf_rtrequest(
+	__unused int cmd,
+	struct rtentry *rt,
+	__unused struct sockaddr *sa)
 {
 
 	if (rt)
 		rt->rt_rmx.rmx_mtu = IPV6_MMTU;
 }
 
-int
-stf_ioctl(ifp, cmd, data)
-	struct ifnet *ifp;
-	u_long cmd;
-	void *data;
+static errno_t
+stf_ioctl(
+	ifnet_t		ifp,
+	u_int32_t	cmd,
+	void		*data)
 {
 	struct ifaddr *ifa;
 	struct ifreq *ifr;
diff --git a/bsd/net/if_types.h b/bsd/net/if_types.h
index 282c0f811..41b50c4f0 100644
--- a/bsd/net/if_types.h
+++ b/bsd/net/if_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1989, 1993, 1994
diff --git a/bsd/net/if_var.h b/bsd/net/if_var.h
index 6feecc2b9..02735e846 100644
--- a/bsd/net/if_var.h
+++ b/bsd/net/if_var.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -126,49 +132,57 @@
  */
 
 struct net_event_data {
-     unsigned long	if_family;
-     unsigned long	if_unit;
-     char		if_name[IFNAMSIZ];
+	u_int32_t	if_family;
+	u_int32_t	if_unit;
+	char		if_name[IFNAMSIZ];
 };
 
+#if defined(__LP64__)
+#define __need_struct_timeval32
+#include <sys/_structs.h>
+#define IF_DATA_TIMEVAL timeval32
+#else
+#define IF_DATA_TIMEVAL timeval
+#endif
+
+#pragma pack(4)
+
 /*
  * Structure describing information about an interface
  * which may be of interest to management entities.
  */
 struct if_data {
 	/* generic interface information */
-	unsigned char	ifi_type;	/* ethernet, tokenring, etc */
-#ifdef __APPLE__
-	unsigned char	ifi_typelen;	/* Length of frame type id */
-#endif
-	unsigned char	ifi_physical;	/* e.g., AUI, Thinnet, 10base-T, etc */
-	unsigned char	ifi_addrlen;	/* media address length */
-	unsigned char	ifi_hdrlen;	/* media header length */
-	unsigned char	ifi_recvquota;	/* polling quota for receive intrs */
-	unsigned char	ifi_xmitquota;	/* polling quota for xmit intrs */
-    	unsigned char	ifi_unused1;	/* for future use */
-	unsigned long	ifi_mtu;	/* maximum transmission unit */
-	unsigned long	ifi_metric;	/* routing metric (external only) */
-	unsigned long	ifi_baudrate;	/* linespeed */
+	u_char		ifi_type;	/* ethernet, tokenring, etc */
+	u_char		ifi_typelen;	/* Length of frame type id */
+	u_char		ifi_physical;	/* e.g., AUI, Thinnet, 10base-T, etc */
+	u_char		ifi_addrlen;	/* media address length */
+	u_char		ifi_hdrlen;	/* media header length */
+	u_char		ifi_recvquota;	/* polling quota for receive intrs */
+	u_char		ifi_xmitquota;	/* polling quota for xmit intrs */
+	u_char		ifi_unused1;	/* for future use */
+	u_int32_t	ifi_mtu;	/* maximum transmission unit */
+	u_int32_t	ifi_metric;	/* routing metric (external only) */
+	u_int32_t	ifi_baudrate;	/* linespeed */
 	/* volatile statistics */
-	unsigned long	ifi_ipackets;	/* packets received on interface */
-	unsigned long	ifi_ierrors;	/* input errors on interface */
-	unsigned long	ifi_opackets;	/* packets sent on interface */
-	unsigned long	ifi_oerrors;	/* output errors on interface */
-	unsigned long	ifi_collisions;	/* collisions on csma interfaces */
-	unsigned long	ifi_ibytes;	/* total number of octets received */
-	unsigned long	ifi_obytes;	/* total number of octets sent */
-	unsigned long	ifi_imcasts;	/* packets received via multicast */
-	unsigned long	ifi_omcasts;	/* packets sent via multicast */
-	unsigned long	ifi_iqdrops;	/* dropped on input, this interface */
-	unsigned long	ifi_noproto;	/* destined for unsupported protocol */
-	unsigned long	ifi_recvtiming;	/* usec spent receiving when timing */
-	unsigned long	ifi_xmittiming;	/* usec spent xmitting when timing */
-	struct	timeval ifi_lastchange;	/* time of last administrative change */
-	unsigned long	ifi_unused2;	/* used to be the default_proto */
-	unsigned long	ifi_hwassist;	/* HW offload capabilities */
-	unsigned long	ifi_reserved1;	/* for future use */
-	unsigned long	ifi_reserved2;	/* for future use */
+	u_int32_t	ifi_ipackets;	/* packets received on interface */
+	u_int32_t	ifi_ierrors;	/* input errors on interface */
+	u_int32_t	ifi_opackets;	/* packets sent on interface */
+	u_int32_t	ifi_oerrors;	/* output errors on interface */
+	u_int32_t	ifi_collisions;	/* collisions on csma interfaces */
+	u_int32_t	ifi_ibytes;	/* total number of octets received */
+	u_int32_t	ifi_obytes;	/* total number of octets sent */
+	u_int32_t	ifi_imcasts;	/* packets received via multicast */
+	u_int32_t	ifi_omcasts;	/* packets sent via multicast */
+	u_int32_t	ifi_iqdrops;	/* dropped on input, this interface */
+	u_int32_t	ifi_noproto;	/* destined for unsupported protocol */
+	u_int32_t	ifi_recvtiming;	/* usec spent receiving when timing */
+	u_int32_t	ifi_xmittiming;	/* usec spent xmitting when timing */
+	struct IF_DATA_TIMEVAL ifi_lastchange;	/* time of last administrative change */
+	u_int32_t	ifi_unused2;	/* used to be the default_proto */
+	u_int32_t	ifi_hwassist;	/* HW offload capabilities */
+	u_int32_t	ifi_reserved1;	/* for future use */
+	u_int32_t	ifi_reserved2;	/* for future use */
 };
 
 /*
@@ -177,18 +191,16 @@ struct if_data {
  */
 struct if_data64 {
 	/* generic interface information */
-	u_char	ifi_type;		/* ethernet, tokenring, etc */
-#ifdef __APPLE__
-	u_char	ifi_typelen;		/* Length of frame type id */
-#endif
+	u_char		ifi_type;		/* ethernet, tokenring, etc */
+	u_char		ifi_typelen;		/* Length of frame type id */
 	u_char		ifi_physical;		/* e.g., AUI, Thinnet, 10base-T, etc */
 	u_char		ifi_addrlen;		/* media address length */
 	u_char		ifi_hdrlen;		/* media header length */
 	u_char		ifi_recvquota;		/* polling quota for receive intrs */
 	u_char		ifi_xmitquota;		/* polling quota for xmit intrs */
 	u_char		ifi_unused1;		/* for future use */
-	u_long		ifi_mtu;		/* maximum transmission unit */
-	u_long		ifi_metric;		/* routing metric (external only) */
+	u_int32_t	ifi_mtu;		/* maximum transmission unit */
+	u_int32_t	ifi_metric;		/* routing metric (external only) */
 	u_int64_t	ifi_baudrate;		/* linespeed */
 	/* volatile statistics */
 	u_int64_t	ifi_ipackets;		/* packets received on interface */
@@ -202,11 +214,13 @@ struct if_data64 {
 	u_int64_t	ifi_omcasts;		/* packets sent via multicast */
 	u_int64_t	ifi_iqdrops;		/* dropped on input, this interface */
 	u_int64_t	ifi_noproto;		/* destined for unsupported protocol */
-	u_long		ifi_recvtiming;		/* usec spent receiving when timing */
-	u_long		ifi_xmittiming;		/* usec spent xmitting when timing */
-	struct	timeval ifi_lastchange;	/* time of last administrative change */
+	u_int32_t	ifi_recvtiming;		/* usec spent receiving when timing */
+	u_int32_t	ifi_xmittiming;		/* usec spent xmitting when timing */
+	struct IF_DATA_TIMEVAL ifi_lastchange;	/* time of last administrative change */
 };
 
+#pragma pack()
+
 #ifdef PRIVATE
 /*
  * Internal storage of if_data. This is bound to change. Various places in the
@@ -215,34 +229,34 @@ struct if_data64 {
  */
 struct if_data_internal {
 	/* generic interface information */
-	u_char	ifi_type;		/* ethernet, tokenring, etc */
-	u_char	ifi_typelen;	/* Length of frame type id */
-	u_char	ifi_physical;	/* e.g., AUI, Thinnet, 10base-T, etc */
-	u_char	ifi_addrlen;	/* media address length */
-	u_char	ifi_hdrlen;		/* media header length */
-	u_char	ifi_recvquota;	/* polling quota for receive intrs */
-	u_char	ifi_xmitquota;	/* polling quota for xmit intrs */
-    u_char	ifi_unused1;	/* for future use */
-	u_long	ifi_mtu;		/* maximum transmission unit */
-	u_long	ifi_metric;		/* routing metric (external only) */
-	u_long	ifi_baudrate;	/* linespeed */
+	u_char		ifi_type;	/* ethernet, tokenring, etc */
+	u_char		ifi_typelen;	/* Length of frame type id */
+	u_char		ifi_physical;	/* e.g., AUI, Thinnet, 10base-T, etc */
+	u_char		ifi_addrlen;	/* media address length */
+	u_char		ifi_hdrlen;	/* media header length */
+	u_char		ifi_recvquota;	/* polling quota for receive intrs */
+	u_char		ifi_xmitquota;	/* polling quota for xmit intrs */
+	u_char		ifi_unused1;	/* for future use */
+	u_int32_t	ifi_mtu;	/* maximum transmission unit */
+	u_int32_t	ifi_metric;	/* routing metric (external only) */
+	u_int32_t	ifi_baudrate;	/* linespeed */
 	/* volatile statistics */
 	u_int64_t	ifi_ipackets;	/* packets received on interface */
 	u_int64_t	ifi_ierrors;	/* input errors on interface */
 	u_int64_t	ifi_opackets;	/* packets sent on interface */
 	u_int64_t	ifi_oerrors;	/* output errors on interface */
 	u_int64_t	ifi_collisions;	/* collisions on csma interfaces */
-	u_int64_t	ifi_ibytes;		/* total number of octets received */
-	u_int64_t	ifi_obytes;		/* total number of octets sent */
+	u_int64_t	ifi_ibytes;	/* total number of octets received */
+	u_int64_t	ifi_obytes;	/* total number of octets sent */
 	u_int64_t	ifi_imcasts;	/* packets received via multicast */
 	u_int64_t	ifi_omcasts;	/* packets sent via multicast */
 	u_int64_t	ifi_iqdrops;	/* dropped on input, this interface */
 	u_int64_t	ifi_noproto;	/* destined for unsupported protocol */
-	u_long	ifi_recvtiming;		/* usec spent receiving when timing */
-	u_long	ifi_xmittiming;		/* usec spent xmitting when timing */
+	u_int32_t	ifi_recvtiming;	/* usec spent receiving when timing */
+	u_int32_t	ifi_xmittiming;	/* usec spent xmitting when timing */
 #define IF_LASTCHANGEUPTIME	1	/* lastchange: 1-uptime 0-calendar time */
 	struct	timeval ifi_lastchange;	/* time of last administrative change */
-	u_long	ifi_hwassist;		/* HW offload capabilities */
+	u_int32_t	ifi_hwassist;	/* HW offload capabilities */
 };
 
 #define	if_mtu		if_data.ifi_mtu
@@ -308,6 +322,7 @@ TAILQ_HEAD(ddesc_head_name, dlil_demux_desc);
 
 #define IFNET_RW_LOCK 1
 
+#endif /* PRIVATE */
 /*
  * Structure defining a queue for a network interface.
  */
@@ -319,9 +334,12 @@ struct	ifqueue {
 	int	ifq_drops;
 };
 
+#ifdef PRIVATE
+
 struct ddesc_head_str;
 struct proto_hash_entry;
 struct kev_msg;
+struct dlil_threading_info;
 
 /*
  * Structure defining a network interface.
@@ -356,7 +374,6 @@ struct ifnet {
 #else
 	int	refcnt;
 #endif
-	int	offercnt;
 #ifdef __KPI_INTERFACE__
 	ifnet_output_func	if_output;
 	ifnet_ioctl_func	if_ioctl;
@@ -389,11 +406,7 @@ struct ifnet {
 	int	if_amcount;		/* number of all-multicast requests */
 /* procedure handles */
 #ifdef __KPI_INTERFACE__
-	union {
-		int	(*original)(struct ifnet *ifp, u_long protocol_family,
-			struct ddesc_head_str *demux_desc_head);
-		ifnet_add_proto_func	kpi;
-	} if_add_proto_u;
+	ifnet_add_proto_func	if_add_proto;
 	ifnet_del_proto_func	if_del_proto;
 #else __KPI_INTERFACE__
 	void*	if_add_proto;
@@ -401,8 +414,11 @@ struct ifnet {
 #endif __KPI_INTERFACE__
 	struct proto_hash_entry	*if_proto_hash;
 	void					*if_kpi_storage;
-	
+#if 0	
 	void	*unused_was_init;
+#else
+	struct dlil_threading_info *if_input_thread;
+#endif
 	void	*unused_was_resolvemulti;
 	
 	struct ifqueue	if_snd;
@@ -431,10 +447,11 @@ struct ifnet {
 			u_char	*ptr;
 		} u;
 	} if_broadcast;
+#if CONFIG_MACF_NET
+	struct  label *if_label;	/* interface MAC label */
+#endif
 };
 
-#define if_add_proto	if_add_proto_u.original
-
 #ifndef __APPLE__
 /* for compatibility with other BSDs */
 #define	if_addrlist	if_addrhead
@@ -462,7 +479,7 @@ struct if_clone {
 };
 
 #define IF_CLONE_INITIALIZER(name, create, destroy, minifs, maxunit)	\
-    { { 0, 0 }, name, sizeof(name) - 1, minifs, maxunit, NULL, 0, create, destroy }
+    { { NULL, NULL }, name, sizeof(name) - 1, minifs, maxunit, NULL, 0, create, destroy }
 
 /*
  * Bit values in if_ipending
@@ -615,10 +632,9 @@ struct ifmultiaddr {
 extern	struct ifnethead ifnet_head;
 extern struct	ifnet	**ifindex2ifnet;
 extern	int ifqmaxlen;
-extern	struct ifnet loif[];
+extern	ifnet_t  lo_ifp;
 extern	int if_index;
 extern	struct ifaddr **ifnet_addrs;
-extern struct ifnet *lo_ifp;
 
 int	if_addmulti(struct ifnet *, const struct sockaddr *, struct ifmultiaddr **);
 int	if_allmulti(struct ifnet *, int);
@@ -626,6 +642,7 @@ void	if_attach(struct ifnet *);
 int	if_delmultiaddr(struct ifmultiaddr *ifma, int locked);
 int	if_delmulti(struct ifnet *, const struct sockaddr *);
 void	if_down(struct ifnet *);
+int 	if_down_all(void);
 void	if_route(struct ifnet *, int flag, int fam);
 void	if_unroute(struct ifnet *, int flag, int fam);
 void	if_up(struct ifnet *);
@@ -658,16 +675,29 @@ struct	ifaddr *ifa_ifwithaddr(const struct sockaddr *);
 struct	ifaddr *ifa_ifwithdstaddr(const struct sockaddr *);
 struct	ifaddr *ifa_ifwithnet(const struct sockaddr *);
 struct	ifaddr *ifa_ifwithroute(int, const struct sockaddr *, const struct sockaddr *);
+struct	ifaddr *ifa_ifwithroute_locked(int, const struct sockaddr *, const struct sockaddr *);
 struct	ifaddr *ifaof_ifpforaddr(const struct sockaddr *, struct ifnet *);
+struct	ifaddr *ifa_ifpgetprimary(struct ifnet *, int);
 void	ifafree(struct ifaddr *);
 void	ifaref(struct ifaddr *);
 
 struct	ifmultiaddr *ifmaof_ifpforaddr(const struct sockaddr *, struct ifnet *);
 
+int	ifa_foraddr(unsigned int addr);
+
 #ifdef BSD_KERNEL_PRIVATE
-void	if_data_internal_to_if_data(const struct if_data_internal *if_data_int,
-							   struct if_data *if_data);
-void	if_data_internal_to_if_data64(const struct if_data_internal *if_data_int,
+enum {
+	kIfNetUseCount_MayBeZero = 0,
+	kIfNetUseCount_MustNotBeZero = 1
+};
+
+int ifp_use(struct ifnet *ifp, int handle_zero);
+int ifp_unuse(struct ifnet *ifp);
+void ifp_use_reached_zero(struct ifnet *ifp);
+
+void	if_data_internal_to_if_data(struct ifnet *ifp, const struct if_data_internal *if_data_int,
+			   struct if_data *if_data);
+void	if_data_internal_to_if_data64(struct ifnet *ifp, const struct if_data_internal *if_data_int,
 							   struct if_data64 *if_data64);
 #endif /* BSD_KERNEL_PRIVATE */
 #endif /* KERNEL_PRIVATE */
diff --git a/bsd/net/if_vlan.c b/bsd/net/if_vlan.c
index a9d91c055..3f5c2c14c 100644
--- a/bsd/net/if_vlan.c
+++ b/bsd/net/if_vlan.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1998 Massachusetts Institute of Technology
@@ -86,6 +92,9 @@
 
 #include <net/dlil.h>
 
+#include <net/kpi_interface.h>
+#include <net/kpi_protocol.h>
+
 #include <kern/locks.h>
 
 #ifdef INET
@@ -95,6 +104,7 @@
 
 #include <net/if_media.h>
 #include <net/multicast_list.h>
+#include <net/ether_if_module.h>
 
 #define	IF_MAXUNIT		0x7fff	/* historical value */
 
@@ -186,7 +196,7 @@ typedef struct vlan_parent {
 #define VLPF_DETACHING		0x4
     u_int32_t			vlp_flags;
     struct ifdevmtu		vlp_devmtu;
-    UInt32			vlp_retain_count;
+    SInt32			vlp_retain_count;
 } vlan_parent, * vlan_parent_ref;
 
 struct ifvlan {
@@ -338,16 +348,16 @@ ifvlan_flags_set_detaching(ifvlan_ref ifv)
 
 #if 0
 SYSCTL_DECL(_net_link);
-SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW, 0, "IEEE 802.1Q VLAN");
-SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW, 0, "for consistency");
+SYSCTL_NODE(_net_link, IFT_L2VLAN, vlan, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "IEEE 802.1Q VLAN");
+SYSCTL_NODE(_net_link_vlan, PF_LINK, link, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "for consistency");
 #endif 0
 
 #define M_VLAN 		M_DEVBUF
 
 static	int vlan_clone_create(struct if_clone *, int);
 static	void vlan_clone_destroy(struct ifnet *);
-static	int vlan_input(struct mbuf *m, char *frame_header, struct ifnet *ifp,
-					   u_long protocol_family, int sync_ok);
+static	int vlan_input(ifnet_t ifp, protocol_family_t protocol,
+					   mbuf_t m, char *frame_header);
 static	int vlan_output(struct ifnet *ifp, struct mbuf *m);
 static	int vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * addr);
 static  int vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode,
@@ -370,7 +380,7 @@ static struct if_clone vlan_cloner = IF_CLONE_INITIALIZER(VLANNAME,
 static	void interface_link_event(struct ifnet * ifp, u_long event_code);
 static	void vlan_parent_link_event(vlan_parent_ref vlp, 
 				    u_long event_code);
-extern int dlil_input_packet(struct ifnet  *ifp, struct mbuf *m, char *frame_header);
+extern void dlil_input_packet_list(struct ifnet  *ifp, struct mbuf *m);
 
 static int
 vlan_globals_init(void)
@@ -410,7 +420,7 @@ siocgifdevmtu(struct ifnet * ifp, struct ifdevmtu * ifdm_p)
     int 		error;
 
     bzero(&ifr, sizeof(ifr));
-    error = dlil_ioctl(0, ifp, SIOCGIFDEVMTU, (caddr_t)&ifr);
+    error = ifnet_ioctl(ifp, 0,SIOCGIFDEVMTU, &ifr);
     if (error == 0) {
 	*ifdm_p = ifr.ifr_devmtu;
     }
@@ -424,7 +434,7 @@ siocsifaltmtu(struct ifnet * ifp, int mtu)
 
     bzero(&ifr, sizeof(ifr));
     ifr.ifr_mtu = mtu;
-    return (dlil_ioctl(0, ifp, SIOCSIFALTMTU, (caddr_t)&ifr));
+    return (ifnet_ioctl(ifp, 0, SIOCSIFALTMTU, &ifr));
 }
 
 static __inline__ void 
@@ -460,15 +470,6 @@ vlan_bpf_input(struct ifnet * ifp, struct mbuf * m,
     return;
 }
 
-static struct ifaddr * 
-ifaddr_byindex(int i)
-{
-    if (i > if_index || i == 0) {
-	return (NULL);
-    }
-    return (ifnet_addrs[i - 1]);
-}
-
 /**
  ** vlan_parent synchronization routines
  **/
@@ -491,8 +492,8 @@ vlan_parent_release(vlan_parent_ref vlp)
     case 1:
 	if (g_vlan->verbose) {
 	    struct ifnet * ifp = vlp->vlp_ifp;
-	    printf("vlan_parent_release(%s%d)\n", ifp->if_name,
-		   ifp->if_unit);
+	    printf("vlan_parent_release(%s%d)\n", ifnet_name(ifp),
+		   ifnet_unit(ifp));
 	}
 	FREE(vlp, M_VLAN);
 	break;
@@ -524,7 +525,7 @@ vlan_parent_wait(vlan_parent_ref vlp, const char * msg)
 	if (g_vlan->verbose) {
 	    struct ifnet * ifp = vlp->vlp_ifp;
 
-	    printf("%s%d: %s msleep\n", ifp->if_name, ifp->if_unit, msg);
+	    printf("%s%d: %s msleep\n", ifnet_name(ifp), ifnet_unit(ifp), msg);
 	}
 	waited = 1;
 	(void)msleep(vlp, vlan_lck_mtx, PZERO, msg, 0);
@@ -534,7 +535,7 @@ vlan_parent_wait(vlan_parent_ref vlp, const char * msg)
     if (g_vlan->verbose && waited) {
 	struct ifnet * ifp = vlp->vlp_ifp;
 
-	printf("%s: %s woke up\n", ifp->if_name, ifp->if_unit, msg);
+	printf("%s%d: %s woke up\n", ifnet_name(ifp), ifnet_unit(ifp), msg);
     }
     return;
 }
@@ -557,7 +558,7 @@ vlan_parent_signal(vlan_parent_ref vlp, const char * msg)
     if (g_vlan->verbose) {
 	struct ifnet * ifp = vlp->vlp_ifp;
 
-	printf("%s%d: %s wakeup\n", ifp->if_name, ifp->if_unit, msg);
+	printf("%s%d: %s wakeup\n", ifnet_name(ifp), ifnet_unit(ifp), msg);
     }
     return;
 }
@@ -580,7 +581,7 @@ vlan_setmulti(struct ifnet * ifp)
     vlan_parent_ref	vlp;
 
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	goto unlock_done;
     }
@@ -596,7 +597,7 @@ vlan_setmulti(struct ifnet * ifp)
     vlan_parent_wait(vlp, "vlan_setmulti");
 
     /* check again, things could have changed */
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	goto signal_done;
     }
@@ -677,7 +678,7 @@ vlan_parent_find_max_mtu(vlan_parent_ref vlp, ifvlan_ref exclude_ifv)
 	if (exclude_ifv == ifv) {
 	    continue;
 	}
-	req_mtu = ifv->ifv_ifp->if_mtu + ifv->ifv_mtufudge;
+	req_mtu = ifnet_mtu(ifv->ifv_ifp) + ifv->ifv_mtufudge;
 	if (req_mtu > max_mtu) {
 	    max_mtu = req_mtu;
 	}
@@ -706,14 +707,14 @@ vlan_parent_create(struct ifnet * p, vlan_parent_ref * ret_vlp)
     error = siocgifdevmtu(p, &vlp->vlp_devmtu);
     if (error != 0) {
 	printf("vlan_parent_create (%s%d): siocgifdevmtu failed, %d\n",
-	       p->if_name, p->if_unit, error);
+	       ifnet_name(p), ifnet_unit(p), error);
 	FREE(vlp, M_VLAN);
 	return (error);
     }
     LIST_INIT(&vlp->vlp_vlan_list);
     vlp->vlp_ifp = p;
     vlan_parent_retain(vlp);
-    if (p->if_hwassist 
+    if (ifnet_offload(p)
 	& (IF_HWASSIST_VLAN_MTU | IF_HWASSIST_VLAN_TAGGING)) {
 	vlan_parent_flags_set_supports_vlan_mtu(vlp);
     }
@@ -738,9 +739,7 @@ vlan_parent_remove_all_vlans(vlan_parent_ref vlp)
 
     /* the vlan parent has no more VLAN's */
     p = vlp->vlp_ifp;
-    ifnet_lock_exclusive(p);
-    p->if_eflags &= ~IFEF_VLAN;
-    ifnet_lock_done(p);
+    ifnet_set_eflags(p, 0, IFEF_VLAN); /* clear IFEF_VLAN */
     LIST_REMOVE(vlp, vlp_parent_list);
     vlan_unlock();
     vlan_parent_release(vlp);
@@ -783,69 +782,74 @@ vlan_clone_attach(void)
 static int
 vlan_clone_create(struct if_clone *ifc, int unit)
 {
-    int 		error;
-    ifvlan_ref		ifv;
-    struct ifnet *	ifp;
-
-    error = vlan_globals_init();
-    if (error != 0) {
-	return (error);
-    }
-    ifv = _MALLOC(sizeof(struct ifvlan), M_VLAN, M_WAITOK);
-    bzero(ifv, sizeof(struct ifvlan));
-    multicast_list_init(&ifv->ifv_multicast);
-
-    /* use the interface name as the unique id for ifp recycle */
-    if ((unsigned int)snprintf(ifv->ifv_name, sizeof(ifv->ifv_name), "%s%d",
-			       ifc->ifc_name, unit) >= sizeof(ifv->ifv_name)) {
-	FREE(ifv, M_VLAN);
-	return (EINVAL);
-    }
-    error = dlil_if_acquire(APPLE_IF_FAM_VLAN,
-			    ifv->ifv_name,
-			    strlen(ifv->ifv_name),
-			    &ifp);
-    if (error) {
-	FREE(ifv, M_VLAN);
-	return (error);
-    }
-    ifp->if_name = ifc->ifc_name;
-    ifp->if_unit = unit;
-    ifp->if_family = APPLE_IF_FAM_VLAN;
-
+	int							error;
+	ifvlan_ref					ifv;
+	ifnet_t						ifp;
+	struct ifnet_init_params	vlan_init;
+	
+	error = vlan_globals_init();
+	if (error != 0) {
+		return (error);
+	}
+	ifv = _MALLOC(sizeof(struct ifvlan), M_VLAN, M_WAITOK);
+	bzero(ifv, sizeof(struct ifvlan));
+	multicast_list_init(&ifv->ifv_multicast);
+	
+	/* use the interface name as the unique id for ifp recycle */
+	if ((unsigned int)snprintf(ifv->ifv_name, sizeof(ifv->ifv_name), "%s%d",
+				   ifc->ifc_name, unit) >= sizeof(ifv->ifv_name)) {
+		FREE(ifv, M_VLAN);
+		return (EINVAL);
+	}
+	
+	bzero(&vlan_init, sizeof(vlan_init));
+	vlan_init.uniqueid = ifv->ifv_name;
+	vlan_init.uniqueid_len = strlen(ifv->ifv_name);
+	vlan_init.name = ifc->ifc_name;
+	vlan_init.unit = unit;
+	vlan_init.family = IFNET_FAMILY_VLAN;
+	vlan_init.type = IFT_L2VLAN;
+	vlan_init.output = vlan_output;
+	vlan_init.demux = ether_demux;
+	vlan_init.add_proto = ether_add_proto;
+	vlan_init.del_proto = ether_del_proto;
+	vlan_init.check_multi = ether_check_multi;
+	vlan_init.framer = ether_frameout;
+	vlan_init.softc = ifv;
+	vlan_init.ioctl = vlan_ioctl;
+	vlan_init.set_bpf_tap = vlan_set_bpf_tap;
+	vlan_init.detach = vlan_if_free;
+	vlan_init.broadcast_addr = etherbroadcastaddr;
+	vlan_init.broadcast_len = ETHER_ADDR_LEN;
+	error = ifnet_allocate(&vlan_init, &ifp);
+	
+	if (error) {
+		FREE(ifv, M_VLAN);
+		return (error);
+	}
+	
 #if 0
-    /* NB: flags are not set here */
-    ifp->if_linkmib = &ifv->ifv_mib;
-    ifp->if_linkmiblen = sizeof ifv->ifv_mib;
-    /* NB: mtu is not set here */
+	/* NB: flags are not set here */
+	ifnet_set_link_mib_data(ifp, &ifv->ifv_mib, sizeof ifv->ifv_mib);
+	/* NB: mtu is not set here */
 #endif 0
-
-    ifp->if_ioctl = vlan_ioctl;
-    ifp->if_set_bpf_tap = vlan_set_bpf_tap;
-    ifp->if_free = vlan_if_free;
-    ifp->if_output = vlan_output;
-    ifp->if_hwassist = 0;
-    ifp->if_addrlen = ETHER_ADDR_LEN; /* XXX ethernet specific */
-    ifp->if_baudrate = 0;
-    ifp->if_type = IFT_L2VLAN;
-    ifp->if_hdrlen = ETHER_VLAN_ENCAP_LEN;
-    
-    /* XXX ethernet specific */
-    ifp->if_broadcast.length = ETHER_ADDR_LEN;
-    bcopy(etherbroadcastaddr, ifp->if_broadcast.u.buffer, ETHER_ADDR_LEN);
-    
-    error = dlil_if_attach(ifp);
-    if (error) {
-	dlil_if_release(ifp);
-	FREE(ifv, M_VLAN);
-	return (error);
-    }
-    ifp->if_private = ifv;
-    ifv->ifv_ifp = ifp;
-
-    /* attach as ethernet */
-    bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
-    return (0);
+	
+	ifnet_set_offload(ifp, 0);
+	ifnet_set_addrlen(ifp, ETHER_ADDR_LEN); /* XXX ethernet specific */
+	ifnet_set_baudrate(ifp, 0);
+	ifnet_set_hdrlen(ifp, ETHER_VLAN_ENCAP_LEN);
+	
+	error = ifnet_attach(ifp, NULL);
+	if (error) {
+		ifnet_release(ifp);
+		FREE(ifv, M_VLAN);
+		return (error);
+	}
+	ifv->ifv_ifp = ifp;
+	
+	/* attach as ethernet */
+	bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
+	return (0);
 }
 
 static void
@@ -860,9 +864,7 @@ vlan_remove(ifvlan_ref ifv)
 static void
 vlan_if_detach(struct ifnet * ifp)
 {
-    if (dlil_if_detach(ifp) != DLIL_WAIT_FOR_FREE) {
-	vlan_if_free(ifp);
-    }
+	ifnet_detach(ifp);
     return;
 }
 
@@ -872,8 +874,8 @@ vlan_clone_destroy(struct ifnet *ifp)
     ifvlan_ref ifv;
 
     vlan_lock();
-    ifv = ifp->if_private;
-    if (ifv == NULL || ifp->if_type != IFT_L2VLAN) {
+    ifv = ifnet_softc(ifp);
+    if (ifv == NULL || ifnet_type(ifp) != IFT_L2VLAN) {
 	vlan_unlock();
 	return;
     }
@@ -893,7 +895,7 @@ vlan_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func func)
     ifvlan_ref	ifv;
 
     vlan_lock();
-    ifv = ifp->if_private;
+    ifv = ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	vlan_unlock();
 	return (ENODEV);
@@ -941,7 +943,7 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
 	return (0);
     }
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)
 	|| ifvlan_flags_ready(ifv) == 0) {
 	vlan_unlock();
@@ -956,7 +958,7 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
     }
     p = vlp->vlp_ifp;
     (void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0);
-    soft_vlan = (p->if_hwassist & IF_HWASSIST_VLAN_TAGGING) == 0;
+    soft_vlan = (ifnet_offload(p) & IF_HWASSIST_VLAN_TAGGING) == 0;
     bpf_func = ifv->ifv_bpf_output;
     tag = ifv->ifv_tag;
     encaplen = ifv->ifv_encaplen;
@@ -964,7 +966,7 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
     vlan_bpf_output(ifp, m, bpf_func);
 	
     /* do not run parent's if_output() if the parent is not up */
-    if ((p->if_flags & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) {
+    if ((ifnet_flags(p) & (IFF_UP | IFF_RUNNING)) != (IFF_UP | IFF_RUNNING)) {
 	m_freem(m);
 	ifp->if_collisions++;
 	return (0);
@@ -983,8 +985,8 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
     } else {
 	M_PREPEND(m, encaplen, M_DONTWAIT);
 	if (m == NULL) {
-	    printf("%s%d: unable to prepend VLAN header\n", ifp->if_name,
-		   ifp->if_unit);
+	    printf("%s%d: unable to prepend VLAN header\n", ifnet_name(ifp),
+		   ifnet_unit(ifp));
 	    ifp->if_oerrors++;
 	    return (0);
 	}
@@ -992,8 +994,8 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
 	if (m->m_len < (int)sizeof(*evl)) {
 	    m = m_pullup(m, sizeof(*evl));
 	    if (m == NULL) {
-		printf("%s%d: unable to pullup VLAN header\n", ifp->if_name,
-		       ifp->if_unit);
+		printf("%s%d: unable to pullup VLAN header\n", ifnet_name(ifp),
+		       ifnet_unit(ifp));
 		ifp->if_oerrors++;
 		return (0);
 	    }
@@ -1010,12 +1012,12 @@ vlan_output(struct ifnet * ifp, struct mbuf * m)
 	evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
 	evl->evl_tag = htons(tag);
     }
-    return dlil_output(p, 0, m, NULL, NULL, 1);
+    return ifnet_output_raw(p, PF_VLAN, m);
 }
 
 static int
-vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
-	   __unused u_long protocol_family, __unused int sync_ok)
+vlan_input(ifnet_t p, __unused protocol_family_t protocol,
+					   mbuf_t m, char *frame_header)
 {
     bpf_packet_func 		bpf_func = NULL;
     struct ether_vlan_header *	evl;
@@ -1033,7 +1035,7 @@ vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
 	m->m_pkthdr.vlan_tag = 0;
     } else {
 	soft_vlan = 1;
-	switch (p->if_type) {
+	switch (ifnet_type(p)) {
 	case IFT_ETHER:
 	    if (m->m_len < ETHER_VLAN_ENCAP_LEN) {
 		m_freem(m);
@@ -1056,7 +1058,7 @@ vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
 	    break;
 	default:
 	    printf("vlan_demux: unsupported if type %u", 
-		   p->if_type);
+		   ifnet_type(p));
 	    m_freem(m);
 	    return 0;
 	    break;
@@ -1065,7 +1067,7 @@ vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
     if (tag != 0) {
 	ifvlan_ref		ifv;
 
-	if ((p->if_eflags & IFEF_VLAN) == 0) {
+	if ((ifnet_eflags(p) & IFEF_VLAN) == 0) {
 	    /* don't bother looking through the VLAN list */
 	    m_freem(m);
 	    return 0;
@@ -1077,7 +1079,7 @@ vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
 	}
 	if (ifv == NULL 
 	    || ifvlan_flags_ready(ifv) == 0
-	    || (ifp->if_flags & IFF_UP) == 0) {
+	    || (ifnet_flags(ifp) & IFF_UP) == 0) {
 	    vlan_unlock();
 	    m_freem(m);
 	    return 0;
@@ -1098,15 +1100,16 @@ vlan_input(struct mbuf * m, char * frame_header, struct ifnet * p,
     }
     if (tag != 0) {
 	m->m_pkthdr.rcvif = ifp;
+	m->m_pkthdr.header = frame_header;
 	(void)ifnet_stat_increment_in(ifp, 1, 
 				      m->m_pkthdr.len + ETHER_HDR_LEN, 0);
 	vlan_bpf_input(ifp, m, bpf_func, frame_header, ETHER_HDR_LEN, 
 		       soft_vlan ? ETHER_VLAN_ENCAP_LEN : 0);
 	/* We found a vlan interface, inject on that interface. */
-	dlil_input_packet(ifp, m, frame_header);
+	dlil_input_packet_list(ifp, m);
     } else {
 	/* Send priority-tagged packet up through the parent */
-	dlil_input_packet(p, m, frame_header);
+	dlil_input_packet_list(p, m);
     }
     return 0;
 }
@@ -1120,13 +1123,10 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
     int			error;
     int			first_vlan = 0;
     ifvlan_ref 		ifv = NULL;
-    struct ifaddr *	ifa1;
-    struct ifaddr *	ifa2;
     vlan_parent_ref	new_vlp = NULL;
     int			need_vlp_release = 0;
+    u_int16_t		parent_flags;
     u_int32_t		progress = 0;
-    struct sockaddr_dl *sdl1;
-    struct sockaddr_dl *sdl2;
     vlan_parent_ref	vlp = NULL;
 
     /* pre-allocate space for vlan_parent, in case we're first */
@@ -1136,7 +1136,7 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
     }
 
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv != NULL && ifv->ifv_vlp != NULL) {
 	vlan_unlock();
 	vlan_parent_release(new_vlp);
@@ -1161,7 +1161,7 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
     progress |= VLAN_CONFIG_PROGRESS_VLP_RETAINED;
     vlan_parent_wait(vlp, "vlan_config");
 
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL) {
 	error = EOPNOTSUPP;
 	goto signal_done;
@@ -1187,13 +1187,14 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
 
     /* check whether bond interface is using parent interface */
     ifnet_lock_exclusive(p);
-    if ((p->if_eflags & IFEF_BOND) != 0) {
+    if ((ifnet_eflags(p) & IFEF_BOND) != 0) {
 	ifnet_lock_done(p);
 	/* don't allow VLAN over interface that's already part of a bond */
 	error = EBUSY;
 	goto signal_done;
     }
     /* prevent BOND interface from using it */
+    /* Can't use ifnet_set_eflags because that would take the lock */
     p->if_eflags |= IFEF_VLAN;
     ifnet_lock_done(p);
     vlan_unlock();
@@ -1206,10 +1207,8 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
 	    goto signal_done;
 	}
 	/* mark the parent interface up */
-	ifnet_lock_exclusive(p);
-	p->if_flags |= IFF_UP;
-	ifnet_lock_done(p);
-	(void)dlil_ioctl(0, p, SIOCSIFFLAGS, (caddr_t)NULL);
+	ifnet_set_flags(p, IFF_UP, IFF_UP);
+	(void)ifnet_ioctl(p, 0, SIOCSIFFLAGS, (caddr_t)NULL);
     }
 
     /* configure parent to receive our multicast addresses */
@@ -1222,6 +1221,9 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
 	goto signal_done;
     }
 
+    /* set our ethernet address to that of the parent */
+    ifnet_set_lladdr_and_type(ifp, ifnet_lladdr(p), ETHER_ADDR_LEN, IFT_ETHER);
+
     /* no failures past this point */
     vlan_lock();
 
@@ -1239,33 +1241,27 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
 	 */
 	ifv->ifv_mtufudge = ifv->ifv_encaplen;
     }
-    ifp->if_mtu = ETHERMTU - ifv->ifv_mtufudge;
+    ifnet_set_mtu(ifp, ETHERMTU - ifv->ifv_mtufudge);
 
     /*
      * Copy only a selected subset of flags from the parent.
      * Other flags are none of our business.
      */
-    ifp->if_flags |= (p->if_flags &
-		      (IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX));
+    parent_flags = ifnet_flags(p) 
+	& (IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX);
+    ifnet_set_flags(ifp, parent_flags,
+		    IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX);
+
     /*
      * If the parent interface can do hardware-assisted
      * VLAN encapsulation, then propagate its hardware-
      * assisted checksumming flags.
      */
-    if (p->if_hwassist & IF_HWASSIST_VLAN_TAGGING) {
-	ifp->if_hwassist |= IF_HWASSIST_CSUM_FLAGS(p->if_hwassist);
+    if (ifnet_offload(p) & IF_HWASSIST_VLAN_TAGGING) {
+	ifnet_set_offload(ifp, IF_HWASSIST_CSUM_FLAGS(ifnet_offload(p)));
     }
 
-    /* set our ethernet address to that of the parent */
-    ifa1 = ifaddr_byindex(ifp->if_index);
-    ifa2 = ifaddr_byindex(p->if_index);
-    sdl1 = (struct sockaddr_dl *)ifa1->ifa_addr;
-    sdl2 = (struct sockaddr_dl *)ifa2->ifa_addr;
-    sdl1->sdl_type = IFT_ETHER;
-    sdl1->sdl_alen = ETHER_ADDR_LEN;
-    bcopy(LLADDR(sdl2), LLADDR(sdl1), ETHER_ADDR_LEN);
-
-    ifp->if_flags |= IFF_RUNNING;
+    ifnet_set_flags(ifp, IFF_RUNNING, IFF_RUNNING);
     ifvlan_flags_set_ready(ifv);
     vlan_parent_signal(vlp, "vlan_config");
     vlan_unlock();
@@ -1285,9 +1281,7 @@ vlan_config(struct ifnet * ifp, struct ifnet * p, int tag)
     }
     if (!vlan_parent_flags_detaching(vlp) && vlan_parent_no_vlans(vlp)) {
 	/* the vlan parent has no more VLAN's */
-	ifnet_lock_exclusive(p);
-	p->if_eflags &= ~IFEF_VLAN;
-	ifnet_lock_done(p);
+	ifnet_set_eflags(p, 0, IFEF_VLAN);
 	LIST_REMOVE(vlp, vlp_parent_list);
 	/* release outside of the lock below */
 	need_vlp_release = 1;
@@ -1314,8 +1308,8 @@ vlan_link_event(struct ifnet * ifp, struct ifnet * p)
     /* generate a link event based on the state of the underlying interface */
     bzero(&ifmr, sizeof(ifmr));
     snprintf(ifmr.ifm_name, sizeof(ifmr.ifm_name),
-	     "%s%d", p->if_name, p->if_unit);
-    if ((*p->if_ioctl)(p, SIOCGIFMEDIA, (caddr_t)&ifmr) == 0
+	     "%s%d", ifnet_name(p), ifnet_unit(p));
+	if (ifnet_ioctl(p, 0, SIOCGIFMEDIA, &ifmr) == 0
 	&& ifmr.ifm_count > 0 && ifmr.ifm_status & IFM_AVALID) {
 	u_long	event;
 	
@@ -1330,16 +1324,14 @@ static int
 vlan_unconfig(struct ifnet * ifp)
 {
     int			error = 0;
-    struct ifaddr *	ifa;
     ifvlan_ref		ifv;
     int			last_vlan = 0;
     int			need_vlp_release = 0;
     struct ifnet *	p;
-    struct sockaddr_dl *sdl;
     vlan_parent_ref	vlp;
 
     vlan_assert_lock_held();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL) {
 	return (0);
     }
@@ -1351,7 +1343,7 @@ vlan_unconfig(struct ifnet * ifp)
     vlan_parent_wait(vlp, "vlan_unconfig");
 
     /* check again because another thread could be in vlan_unconfig */
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL) {
 	goto signal_done;
     }
@@ -1366,7 +1358,7 @@ vlan_unconfig(struct ifnet * ifp)
     if (LIST_NEXT(LIST_FIRST(&vlp->vlp_vlan_list), ifv_vlan_list) == NULL) {
 	if (g_vlan->verbose) {
 	    printf("vlan_unconfig: last vlan on %s%d\n",
-		   p->if_name, p->if_unit);
+		   ifnet_name(p), ifnet_unit(p));
 	}
 	last_vlan = 1;
     }
@@ -1384,31 +1376,25 @@ vlan_unconfig(struct ifnet * ifp)
     /* un-join multicast on parent interface */
     (void)multicast_list_remove(&ifv->ifv_multicast);
 
+    /* Clear our MAC address. */
+    ifnet_set_lladdr_and_type(ifp, NULL, 0, IFT_L2VLAN);
+
     vlan_lock();
 
     /* Disconnect from parent. */
     vlan_parent_remove_vlan(vlp, ifv);
 
     /* return to the state we were in before SIFVLAN */
-    ifp->if_mtu = 0;
-    ifp->if_flags &= ~(IFF_BROADCAST | IFF_MULTICAST 
-		       | IFF_SIMPLEX | IFF_RUNNING);
-    ifp->if_hwassist = 0;
+    ifnet_set_mtu(ifp, 0);
+    ifnet_set_flags(ifp, 0, 
+		    IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX | IFF_RUNNING);
+    ifnet_set_offload(ifp, 0);
     ifv->ifv_flags = 0;
     ifv->ifv_mtufudge = 0;
 
-    /* Clear our MAC address. */
-    ifa = ifaddr_byindex(ifp->if_index);
-    sdl = (struct sockaddr_dl *)(ifa->ifa_addr);
-    sdl->sdl_type = IFT_L2VLAN;
-    sdl->sdl_alen = 0;
-    bzero(LLADDR(sdl), ETHER_ADDR_LEN);
-
     if (!vlan_parent_flags_detaching(vlp) && vlan_parent_no_vlans(vlp)) {
 	/* the vlan parent has no more VLAN's */
-	ifnet_lock_exclusive(p);
-	p->if_eflags &= ~IFEF_VLAN;
-	ifnet_lock_done(p);
+	ifnet_set_eflags(p, 0, IFEF_VLAN);
 	LIST_REMOVE(vlp, vlp_parent_list);
 	/* release outside of the lock below */
 	need_vlp_release++;
@@ -1434,7 +1420,7 @@ vlan_set_promisc(struct ifnet * ifp)
     vlan_parent_ref		vlp;
 
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	error = (ifv == NULL) ? EOPNOTSUPP : EBUSY;
 	goto done;
@@ -1444,7 +1430,7 @@ vlan_set_promisc(struct ifnet * ifp)
     if (vlp == NULL) {
 	goto done;
     }
-    if ((ifp->if_flags & IFF_PROMISC) != 0) {
+    if ((ifnet_flags(ifp) & IFF_PROMISC) != 0) {
 	if (!ifvlan_flags_promisc(ifv)) {
 	    error = ifnet_set_promiscuous(vlp->vlp_ifp, 1);
 	    if (error == 0) {
@@ -1476,7 +1462,7 @@ vlan_new_mtu(struct ifnet * ifp, int mtu)
     vlan_parent_ref	vlp;
 
     vlan_assert_lock_held();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     vlp = ifv->ifv_vlp;
     devmtu_p = &vlp->vlp_devmtu;
     req_mtu = mtu + ifv->ifv_mtufudge;
@@ -1500,7 +1486,7 @@ vlan_new_mtu(struct ifnet * ifp, int mtu)
 	if (new_mtu != 0) {
 	    devmtu_p->ifdm_current = new_mtu;
 	}
-	ifp->if_mtu = mtu;
+	ifnet_set_mtu(ifp, mtu);
     }
     return (error);
 }
@@ -1516,7 +1502,7 @@ vlan_set_mtu(struct ifnet * ifp, int mtu)
 	return (EINVAL);
     }
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	vlan_unlock();
 	return ((ifv == NULL) ? EOPNOTSUPP : EBUSY);
@@ -1533,7 +1519,7 @@ vlan_set_mtu(struct ifnet * ifp, int mtu)
     vlan_parent_wait(vlp, "vlan_set_mtu");
 
     /* check again, something might have changed */
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	error = (ifv == NULL) ? EOPNOTSUPP : EBUSY;
 	goto signal_done;
@@ -1573,7 +1559,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
     vlan_parent_ref	vlp;
     struct vlanreq 	vlr;
 
-    if (ifp->if_type != IFT_L2VLAN) {
+    if (ifnet_type(ifp) != IFT_L2VLAN) {
 	return (EOPNOTSUPP);
     }
     ifr = (struct ifreq *)data;
@@ -1587,7 +1573,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
     case SIOCGIFMEDIA64:
     case SIOCGIFMEDIA:
 	vlan_lock();
-	ifv = (ifvlan_ref)ifp->if_private;
+	ifv = (ifvlan_ref)ifnet_softc(ifp);
 	if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	    vlan_unlock();
 	    return (ifv == NULL ? EOPNOTSUPP : EBUSY);
@@ -1595,14 +1581,14 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 	p = (ifv->ifv_vlp == NULL) ? NULL : ifv->ifv_vlp->vlp_ifp;
 	vlan_unlock();
 	ifmr = (struct ifmediareq64 *)data;
-	user_addr = (cmd == SIOCGIFMEDIA64)
+	user_addr =  proc_is64bit(current_proc())
 	    ? ifmr->ifm_ifmu.ifmu_ulist64
 	    : CAST_USER_ADDR_T(ifmr->ifm_ifmu.ifmu_ulist32);
 	if (p != NULL) {
 	    struct ifmediareq64		p_ifmr;
 
 	    bzero(&p_ifmr, sizeof(p_ifmr));
-	    error = dlil_ioctl(0, p, SIOCGIFMEDIA, (caddr_t)&p_ifmr);
+	    error = ifnet_ioctl(p, 0, SIOCGIFMEDIA, &p_ifmr);
 	    if (error == 0) {
 		ifmr->ifm_active = p_ifmr.ifm_active;
 		ifmr->ifm_current = p_ifmr.ifm_current;
@@ -1633,7 +1619,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 
     case SIOCGIFDEVMTU:
 	vlan_lock();
-	ifv = (ifvlan_ref)ifp->if_private;
+	ifv = (ifvlan_ref)ifnet_softc(ifp);
 	if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	    vlan_unlock();
 	    return (ifv == NULL ? EOPNOTSUPP : EBUSY);
@@ -1642,7 +1628,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 	if (vlp != NULL) {
 	    int		min_mtu = vlp->vlp_devmtu.ifdm_min - ifv->ifv_mtufudge;
 	    devmtu_p = &ifr->ifr_devmtu;
-	    devmtu_p->ifdm_current = ifp->if_mtu;
+	    devmtu_p->ifdm_current = ifnet_mtu(ifp);
 	    devmtu_p->ifdm_min = max(min_mtu, IF_MINMTU);
 	    devmtu_p->ifdm_max = vlp->vlp_devmtu.ifdm_max - ifv->ifv_mtufudge;
 	}
@@ -1682,7 +1668,8 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 		break;
 	    }
 	    /* can't do VLAN over anything but ethernet or ethernet aggregate */
-	    if (p->if_type != IFT_ETHER	&& p->if_type != IFT_IEEE8023ADLAG) {
+	    if (ifnet_type(p) != IFT_ETHER 
+		&& ifnet_type(p) != IFT_IEEE8023ADLAG) {
 		error = EPROTONOSUPPORT;
 		break;
 	    }
@@ -1698,7 +1685,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 	    vlan_link_event(ifp, p);
 	} else {
 	    vlan_lock();
-	    ifv = (ifvlan_ref)ifp->if_private;
+	    ifv = (ifvlan_ref)ifnet_softc(ifp);
 	    if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 		vlan_unlock();
 		error = (ifv == NULL ? EOPNOTSUPP : EBUSY);
@@ -1715,7 +1702,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
     case SIOCGIFVLAN:
 	bzero(&vlr, sizeof vlr);
 	vlan_lock();
-	ifv = (ifvlan_ref)ifp->if_private;
+	ifv = (ifvlan_ref)ifnet_softc(ifp);
 	if (ifv == NULL || ifvlan_flags_detaching(ifv)) {
 	    vlan_unlock();
 	    return (ifv == NULL ? EOPNOTSUPP : EBUSY);
@@ -1725,7 +1712,7 @@ vlan_ioctl(ifnet_t ifp, u_int32_t cmd, void * data)
 	vlan_unlock();
 	if (p != NULL) {
 	    snprintf(vlr.vlr_parent, sizeof(vlr.vlr_parent),
-		     "%s%d", p->if_name, p->if_unit);
+		     "%s%d", ifnet_name(p), ifnet_unit(p));
 	    vlr.vlr_tag = tag;
 	}
 	user_addr = proc_is64bit(current_proc()) 
@@ -1760,19 +1747,19 @@ vlan_if_free(struct ifnet * ifp)
 	return;
     }
     vlan_lock();
-    ifv = (ifvlan_ref)ifp->if_private;
+    ifv = (ifvlan_ref)ifnet_softc(ifp);
     if (ifv == NULL) {
 	vlan_unlock();
 	return;
     }
-    ifp->if_private = NULL;
     vlan_unlock();
-    dlil_if_release(ifp);
+    ifnet_release(ifp);
     FREE(ifv, M_VLAN);
 }
 
 static void
-vlan_event(struct ifnet	* p, struct kev_msg * event)
+vlan_event(struct ifnet	* p, __unused protocol_family_t protocol,
+		   const struct kev_msg * event)
 {
     vlan_parent_ref	vlp;
 
@@ -1791,7 +1778,7 @@ vlan_event(struct ifnet	* p, struct kev_msg * event)
 	return;
     }
     vlan_lock();
-    if ((p->if_eflags & IFEF_VLAN) == 0) {
+    if ((ifnet_eflags(p) & IFEF_VLAN) == 0) {
 	vlan_unlock();
 	/* no VLAN's */
 	return;
@@ -1833,10 +1820,10 @@ interface_link_event(struct ifnet * ifp, u_long event_code)
     event.header.kev_class     = KEV_NETWORK_CLASS;
     event.header.kev_subclass  = KEV_DL_SUBCLASS;
     event.header.event_code    = event_code;
-    event.header.event_data[0] = ifp->if_family;
-    event.unit                 = (u_long) ifp->if_unit;
-    strncpy(event.if_name, ifp->if_name, IFNAMSIZ);
-    dlil_event(ifp, &event.header);
+    event.header.event_data[0] = ifnet_family(ifp);
+    event.unit                 = (u_long) ifnet_unit(ifp);
+    strncpy(event.if_name, ifnet_name(ifp), IFNAMSIZ);
+    ifnet_event(ifp, &event.header);
     return;
 }
 
@@ -1865,20 +1852,16 @@ vlan_parent_link_event(vlan_parent_ref vlp, u_long event_code)
 static int
 vlan_attach_protocol(struct ifnet *ifp)
 {
-    int  			error;
-    struct dlil_proto_reg_str   reg;
+    int								error;
+    struct ifnet_attach_proto_param	reg;
 	
     bzero(&reg, sizeof(reg));
-    TAILQ_INIT(&reg.demux_desc_head);
-    reg.interface_family = ifp->if_family;
-    reg.unit_number      = ifp->if_unit;
     reg.input            = vlan_input;
     reg.event            = vlan_event;
-    reg.protocol_family  = PF_VLAN;
-    error = dlil_attach_protocol(&reg);
+    error = ifnet_attach_protocol(ifp, PF_VLAN, &reg);
     if (error) {
-	printf("vlan_proto_attach(%s%d) dlil_attach_protocol failed, %d\n",
-	       ifp->if_name, ifp->if_unit, error);
+	printf("vlan_proto_attach(%s%d) ifnet_attach_protocol failed, %d\n",
+	       ifnet_name(ifp), ifnet_unit(ifp), error);
     }
     return (error);
 }
@@ -1893,10 +1876,10 @@ vlan_detach_protocol(struct ifnet *ifp)
 {
     int         error;
 
-    error = dlil_detach_protocol(ifp, PF_VLAN);
+    error = ifnet_detach_protocol(ifp, PF_VLAN);
     if (error) {
-	printf("vlan_proto_detach(%s%d) dlil_detach_protocol failed, %d\n",
-	       ifp->if_name, ifp->if_unit, error);
+	printf("vlan_proto_detach(%s%d) ifnet_detach_protocol failed, %d\n",
+	       ifnet_name(ifp), ifnet_unit(ifp), error);
     }
 	
     return (error);
@@ -1904,92 +1887,74 @@ vlan_detach_protocol(struct ifnet *ifp)
 
 /*
  * DLIL interface family functions
- *   We use the ethernet dlil functions, since that's all we support.
+ *   We use the ethernet plumb functions, since that's all we support.
  *   If we wanted to handle multiple LAN types (tokenring, etc.), we'd
  *   call the appropriate routines for that LAN type instead of hard-coding
  *   ethernet.
  */
-extern int ether_add_if(struct ifnet *ifp);
-extern int ether_del_if(struct ifnet *ifp);
-extern int ether_init_if(struct ifnet *ifp);
-extern int ether_add_proto_old(struct ifnet *ifp, u_long protocol_family,
-				struct ddesc_head_str *desc_head);
-
-extern int ether_attach_inet(struct ifnet *ifp, u_long protocol_family);
-extern int ether_detach_inet(struct ifnet *ifp, u_long protocol_family);
-extern int ether_attach_inet6(struct ifnet *ifp, u_long protocol_family);
-extern int ether_detach_inet6(struct ifnet *ifp, u_long protocol_family);
-
-static int
-vlan_attach_inet(struct ifnet *ifp, u_long protocol_family)
+static errno_t
+vlan_attach_inet(struct ifnet *ifp, protocol_family_t protocol_family)
 {
     return (ether_attach_inet(ifp, protocol_family));
 }
 
-static int
-vlan_detach_inet(struct ifnet *ifp, u_long protocol_family)
+static void
+vlan_detach_inet(struct ifnet *ifp, protocol_family_t protocol_family)
 {
-    return (ether_detach_inet(ifp, protocol_family));
+    ether_detach_inet(ifp, protocol_family);
 }
 
-static int
-vlan_attach_inet6(struct ifnet *ifp, u_long protocol_family)
+#if INET6
+static errno_t
+vlan_attach_inet6(struct ifnet *ifp, protocol_family_t protocol_family)
 {
     return (ether_attach_inet6(ifp, protocol_family));
 }
 
-static int
-vlan_detach_inet6(struct ifnet *ifp, u_long protocol_family)
+static void
+vlan_detach_inet6(struct ifnet *ifp, protocol_family_t protocol_family)
 {
-    return (ether_detach_inet6(ifp, protocol_family));
+    ether_detach_inet6(ifp, protocol_family);
 }
+#endif /* INET6 */
 
-static int
-vlan_add_if(struct ifnet *ifp)
+static errno_t
+vlan_attach_at(struct ifnet *ifp, protocol_family_t protocol_family)
 {
-    return (ether_add_if(ifp));
+    return (ether_attach_at(ifp, protocol_family));
 }
 
-static int
-vlan_del_if(struct ifnet *ifp)
+static void
+vlan_detach_at(struct ifnet *ifp, protocol_family_t protocol_family)
 {
-    return (ether_del_if(ifp));
+    ether_detach_at(ifp, protocol_family);
 }
 
-
 __private_extern__ int
 vlan_family_init(void)
 {
     int error=0;
-    struct dlil_ifmod_reg_str  ifmod_reg;
-    
-    bzero(&ifmod_reg, sizeof(ifmod_reg));
-    ifmod_reg.add_if = vlan_add_if;
-    ifmod_reg.del_if = vlan_del_if;
-    ifmod_reg.init_if = NULL;
-    ifmod_reg.add_proto = ether_add_proto_old;
-    ifmod_reg.del_proto = ether_del_proto;
-    ifmod_reg.ifmod_ioctl = ether_ioctl;
-    ifmod_reg.shutdown = NULL;
-
-    if (dlil_reg_if_modules(APPLE_IF_FAM_VLAN, &ifmod_reg)) {
-	printf("WARNING: vlan_family_init -- "
-	       "Can't register if family modules\n");
-	error = EIO;
+
+    error = proto_register_plumber(PF_INET, IFNET_FAMILY_VLAN, 
+				   vlan_attach_inet, vlan_detach_inet);
+    if (error != 0) {
+	printf("proto_register_plumber failed for AF_INET error=%d\n",
+	       error);
 	goto done;
     }
-
-    error = dlil_reg_proto_module(PF_INET, APPLE_IF_FAM_VLAN, 
-				  vlan_attach_inet, vlan_detach_inet);
+#if INET6
+    error = proto_register_plumber(PF_INET6, IFNET_FAMILY_VLAN, 
+				   vlan_attach_inet6, vlan_detach_inet6);
     if (error != 0) {
-	printf("dlil_reg_proto_module failed for AF_INET error=%d\n",
+	printf("proto_register_plumber failed for AF_INET6 error=%d\n",
 	       error);
 	goto done;
     }
-    error = dlil_reg_proto_module(PF_INET6, APPLE_IF_FAM_VLAN, 
-				  vlan_attach_inet6, vlan_detach_inet6);
+#endif
+    error = proto_register_plumber(PF_APPLETALK, IFNET_FAMILY_VLAN, 
+				  vlan_attach_at, vlan_detach_at);
     if (error != 0) {
-	printf("dlil_reg_proto_module failed for AF_INET6 error=%d\n",
+	printf("proto_register_plumber failed for AF_APPLETALK error=%d\n",
 	       error);
 	goto done;
     }
diff --git a/bsd/net/if_vlan_var.h b/bsd/net/if_vlan_var.h
index 6772ddac8..df7c97a5a 100644
--- a/bsd/net/if_vlan_var.h
+++ b/bsd/net/if_vlan_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1998 Massachusetts Institute of Technology
@@ -81,6 +87,6 @@ struct	vlanreq {
 };
 
 #ifdef KERNEL_PRIVATE
-int vlan_family_init(void);
+int vlan_family_init(void) __attribute__((section("__TEXT, initcode")));
 #endif KERNEL_PRIVATE
 #endif /* _NET_IF_VLAN_VAR_H_ */
diff --git a/bsd/net/init.c b/bsd/net/init.c
index 82c2882df..0c2286f82 100644
--- a/bsd/net/init.c
+++ b/bsd/net/init.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/kalloc.h>
diff --git a/bsd/net/init.h b/bsd/net/init.h
index 57a55107b..570fa12dc 100644
--- a/bsd/net/init.h
+++ b/bsd/net/init.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header init.h
@@ -53,6 +59,7 @@ errno_t	net_init_add(net_init_func_ptr	init_func);
 
 #ifdef BSD_KERNEL_PRIVATE
 /* net_init_run is called from bsd_init */
-extern void net_init_run(void);
+extern void net_init_run(void) __attribute__((section("__TEXT, initcode")));
 #endif /* BSD_KERNEL_PRIVATE */
 
+#endif /* _NET_INIT_H_ */
diff --git a/bsd/net/kext_net.c b/bsd/net/kext_net.c
index 3acfce5de..c9ba8eb83 100644
--- a/bsd/net/kext_net.c
+++ b/bsd/net/kext_net.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (C) 1999 Apple Computer, Inc.  */
 
diff --git a/bsd/net/kext_net.h b/bsd/net/kext_net.h
index 38e3eb120..b7b98dd00 100644
--- a/bsd/net/kext_net.h
+++ b/bsd/net/kext_net.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -72,7 +78,7 @@ struct socket_filter {
 TAILQ_HEAD(socket_filter_list, socket_filter);
 
 /* Private, internal implementation functions */
-void	sflt_init(void);
+void	sflt_init(void) __attribute__((section("__TEXT, initcode")));
 void	sflt_initsock(struct socket *so);
 void	sflt_termsock(struct socket *so);
 void	sflt_use(struct socket *so);
@@ -104,7 +110,7 @@ struct so_nke
 {	unsigned int nke_handle;
 	unsigned int nke_where;
 	int nke_flags; /* NFF_BEFORE, NFF_AFTER: net/kext_net.h */
-	unsigned long reserved[4];	/* for future use */
+	u_int32_t reserved[4];	/* for future use */
 };
 
 #pragma pack()
diff --git a/bsd/net/kpi_interface.c b/bsd/net/kpi_interface.c
index 74143ae02..1878cde46 100644
--- a/bsd/net/kpi_interface.c
+++ b/bsd/net/kpi_interface.c
@@ -1,29 +1,36 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include "kpi_interface.h"
 
 #include <sys/queue.h>
 #include <sys/param.h>	/* for definition of NULL */
+#include <kern/debug.h> /* for panic */
 #include <sys/errno.h>
 #include <sys/socket.h>
 #include <sys/kern_event.h>
@@ -37,6 +44,7 @@
 #include <net/if_dl.h>
 #include <net/if_arp.h>
 #include <libkern/libkern.h>
+#include <libkern/OSAtomic.h>
 #include <kern/locks.h>
 
 #if IF_LASTCHANGEUPTIME
@@ -45,7 +53,8 @@
 #define TOUCHLASTCHANGE(__if_lastchange) microtime(__if_lastchange)
 #endif
 
-extern lck_spin_t *dlil_input_lock;
+extern struct dlil_threading_info *dlil_lo_thread_ptr;
+extern int dlil_multithreaded_input;
 
 /*
 	Temporary work around until we have real reference counting
@@ -72,6 +81,17 @@ ifnet_kpi_free(
 	dlil_if_release(ifp);
 }
 
+static __inline__ void*
+_cast_non_const(const void * ptr) {
+	union {
+		const void*		cval;
+		void*			val;
+	} ret;
+	
+	ret.cval = ptr;
+	return (ret.val);
+}
+
 errno_t
 ifnet_allocate(
 	const struct ifnet_init_params *init,
@@ -92,14 +112,19 @@ ifnet_allocate(
 	
 	error = dlil_if_acquire(init->family, init->uniqueid, init->uniqueid_len, &ifp);
 	if (error == 0)
-	{		
-		strncpy(ifp->if_name, init->name, IFNAMSIZ);
+	{
+		/*
+		 * Cast ifp->if_name as non const. dlil_if_acquire sets it up
+		 * to point to storage of at least IFNAMSIZ bytes. It is safe
+		 * to write to this.
+		 */
+		strncpy(_cast_non_const(ifp->if_name), init->name, IFNAMSIZ);
 		ifp->if_type = init->type;
 		ifp->if_family = init->family;
 		ifp->if_unit = init->unit;
 		ifp->if_output = init->output;
 		ifp->if_demux = init->demux;
-		ifp->if_add_proto_u.kpi = init->add_proto;
+		ifp->if_add_proto = init->add_proto;
 		ifp->if_del_proto = init->del_proto;
 		ifp->if_check_multi = init->check_multi;
 		ifp->if_framer = init->framer;
@@ -155,49 +180,30 @@ ifnet_allocate(
 
 errno_t
 ifnet_reference(
-	ifnet_t interface)
+	ifnet_t	ifp)
 {
-	if (interface == NULL) return EINVAL;
-	ifp_reference(interface);
+	int	oldval;
+	
+	if (ifp == NULL) return EINVAL;
+	
+	oldval = OSIncrementAtomic((SInt32 *)&ifp->if_refcnt);
+	
 	return 0;
 }
 
 errno_t
 ifnet_release(
-	ifnet_t interface)
+	ifnet_t	ifp)
 {
-	if (interface == NULL) return EINVAL;
-	ifp_release(interface);
-	return 0;
-}
-
-errno_t
-ifnet_attach(
-	ifnet_t interface,
-	const struct sockaddr_dl *ll_addr)
-{
-	if (interface == NULL) return EINVAL;
-	if (ll_addr && interface->if_addrlen == 0) {
-		interface->if_addrlen = ll_addr->sdl_alen;
-	}
-	else if (ll_addr && ll_addr->sdl_alen != interface->if_addrlen) {
-		return EINVAL;
-	}
-	return dlil_if_attach_with_address(interface, ll_addr);
-}
-
-errno_t
-ifnet_detach(
-	ifnet_t interface)
-{
-	errno_t	error;
+	int	oldval;
 	
-	if (interface == NULL) return EINVAL;
+	if (ifp == NULL) return EINVAL;
 	
-	error = dlil_if_detach(interface);
-	if (error == DLIL_WAIT_FOR_FREE) error = 0; /* Client should always wait for detach */
+	oldval = OSDecrementAtomic((SInt32*)&ifp->if_refcnt);
+	if (oldval == 0)
+		panic("ifnet_release - refcount decremented past zero!");
 	
-	return error;
+	return 0;
 }
 
 void*
@@ -293,7 +299,8 @@ ifnet_eflags(
 
 static const ifnet_offload_t offload_mask = IFNET_CSUM_IP | IFNET_CSUM_TCP |
 			IFNET_CSUM_UDP | IFNET_CSUM_FRAGMENT | IFNET_IP_FRAGMENT |
-			IFNET_CSUM_SUM16 | IFNET_VLAN_TAGGING | IFNET_VLAN_MTU;
+			IFNET_CSUM_SUM16 | IFNET_VLAN_TAGGING | IFNET_VLAN_MTU |
+			IFNET_MULTIPAGES;
 
 errno_t
 ifnet_set_offload(
@@ -374,26 +381,6 @@ ifnet_get_link_mib_data_length(
 	return interface == NULL ? 0 : interface->if_linkmiblen;
 }
 
-errno_t
-ifnet_attach_protocol(
-	ifnet_t interface,
-	protocol_family_t protocol,
-	const struct ifnet_attach_proto_param *proto_details)
-{
-	if (interface == NULL || protocol == 0 || proto_details == NULL)
-		return EINVAL;
-	return dlil_attach_protocol_kpi(interface, protocol, proto_details);
-}
-
-errno_t
-ifnet_detach_protocol(
-	ifnet_t interface,
-	protocol_family_t protocol)
-{
-	if (interface == NULL || protocol == 0) return EINVAL;
-	return dlil_detach_protocol(interface, protocol);
-}
-
 errno_t
 ifnet_output(
 	ifnet_t interface,
@@ -416,7 +403,7 @@ ifnet_output_raw(
 	protocol_family_t protocol_family,
 	mbuf_t m)
 {
-	if (interface == NULL || protocol_family == 0 || m == NULL) {
+	if (interface == NULL || m == NULL) {
 		if (m)
 			mbuf_freem_list(m);
 		return EINVAL;
@@ -424,47 +411,6 @@ ifnet_output_raw(
 	return dlil_output(interface, protocol_family, m, NULL, NULL, 1);
 }
 
-errno_t
-ifnet_input(
-	ifnet_t interface,
-	mbuf_t first_packet,
-	const struct ifnet_stat_increment_param *stats)
-{
-	mbuf_t	last_packet = first_packet;
-	
-	if (interface == NULL || first_packet == NULL) {
-		if (first_packet)
-			mbuf_freem_list(first_packet);
-		return EINVAL;
-	}
-	
-	while (mbuf_nextpkt(last_packet) != NULL)
-		last_packet = mbuf_nextpkt(last_packet);
-	return dlil_input_with_stats(interface, first_packet, last_packet, stats);
-}
-
-errno_t
-ifnet_ioctl(
-	ifnet_t interface,
-	protocol_family_t	protocol_family,
-	u_int32_t ioctl_code,
-	void *ioctl_arg)
-{
-	if (interface == NULL || protocol_family == 0 || ioctl_code == 0)
-		return EINVAL;
-	return dlil_ioctl(protocol_family, interface,
-					  ioctl_code, ioctl_arg);
-}
-
-errno_t
-ifnet_event(
-	ifnet_t interface,
-	struct kern_event_msg* event_ptr)
-{
-	if (interface == NULL || event_ptr == NULL) return EINVAL;
-	return dlil_event(interface, event_ptr);
-}
-
 errno_t
 ifnet_set_mtu(
 	ifnet_t interface,
@@ -599,9 +545,13 @@ ifnet_stat_increment(
 	ifnet_t interface,
 	const struct ifnet_stat_increment_param *counts)
 {
+	struct dlil_threading_info *thread;
 	if (interface == NULL) return EINVAL;
-	
-	lck_spin_lock(dlil_input_lock);
+
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
 
 	interface->if_data.ifi_ipackets += counts->packets_in;
 	interface->if_data.ifi_ibytes += counts->bytes_in;
@@ -617,7 +567,7 @@ ifnet_stat_increment(
 	/* Touch the last change time. */
 	TOUCHLASTCHANGE(&interface->if_lastchange);
 
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -629,9 +579,14 @@ ifnet_stat_increment_in(
 	u_int32_t bytes_in,
 	u_int32_t errors_in)
 {
+	struct dlil_threading_info *thread;
+
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
 
 	interface->if_data.ifi_ipackets += packets_in;
 	interface->if_data.ifi_ibytes += bytes_in;
@@ -639,7 +594,7 @@ ifnet_stat_increment_in(
 
 	TOUCHLASTCHANGE(&interface->if_lastchange);
 
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -651,9 +606,13 @@ ifnet_stat_increment_out(
 	u_int32_t bytes_out,
 	u_int32_t errors_out)
 {
+	struct dlil_threading_info *thread;
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
 
 	interface->if_data.ifi_opackets += packets_out;
 	interface->if_data.ifi_obytes += bytes_out;
@@ -661,7 +620,7 @@ ifnet_stat_increment_out(
 
 	TOUCHLASTCHANGE(&interface->if_lastchange);
 
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -671,9 +630,14 @@ ifnet_set_stat(
 	ifnet_t interface,
 	const struct ifnet_stats_param *stats)
 {
+	struct dlil_threading_info *thread;
+
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
 
 	interface->if_data.ifi_ipackets = stats->packets_in;
 	interface->if_data.ifi_ibytes = stats->bytes_in;
@@ -692,7 +656,7 @@ ifnet_set_stat(
 	/* Touch the last change time. */
 	TOUCHLASTCHANGE(&interface->if_lastchange);
 
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -702,9 +666,14 @@ ifnet_stat(
 	ifnet_t interface,
 	struct ifnet_stats_param *stats)
 {
+	struct dlil_threading_info *thread;
+
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
 
 	stats->packets_in = interface->if_data.ifi_ipackets;
 	stats->bytes_in = interface->if_data.ifi_ibytes;
@@ -720,7 +689,7 @@ ifnet_stat(
 	stats->dropped = interface->if_data.ifi_iqdrops;
 	stats->no_protocol = interface->if_data.ifi_noproto;
 
-	lck_spin_unlock(dlil_input_lock);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -729,11 +698,18 @@ errno_t
 ifnet_touch_lastchange(
 	ifnet_t interface)
 {
+	struct dlil_threading_info *thread;
+
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
+
 	TOUCHLASTCHANGE(&interface->if_lastchange);
-	lck_spin_unlock(dlil_input_lock);
+
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -743,11 +719,18 @@ ifnet_lastchange(
 	ifnet_t interface,
 	struct timeval *last_change)
 {
+	struct dlil_threading_info *thread;
+
 	if (interface == NULL) return EINVAL;
 	
-	lck_spin_lock(dlil_input_lock);
+       	if ((thread = interface->if_input_thread) == NULL || (dlil_multithreaded_input == 0))
+		thread = dlil_lo_thread_ptr;
+
+	lck_mtx_lock(thread->input_lck);
+
 	*last_change = interface->if_data.ifi_lastchange;
-	lck_spin_unlock(dlil_input_lock);
+	
+	lck_mtx_unlock(thread->input_lck);
 	
 #if IF_LASTCHANGEUPTIME
 	/* Crude conversion from uptime to calendar time */
@@ -762,7 +745,7 @@ ifnet_get_address_list(
 	ifnet_t interface,
 	ifaddr_t **addresses)
 {
-	if (interface == NULL || addresses == NULL) return EINVAL;
+	if (addresses == NULL) return EINVAL;
 	return ifnet_get_address_list_family(interface, addresses, 0);
 }
 
@@ -776,7 +759,7 @@ ifnet_get_address_list_family(
 	int count = 0;
 	int cmax = 0;
 	
-	if (interface == NULL || addresses == NULL) return EINVAL;
+	if (addresses == NULL) return EINVAL;
 	*addresses = NULL;
 	
 	ifnet_head_lock_shared();
@@ -1078,12 +1061,12 @@ ifnet_find_by_name(
 	{
 		struct ifaddr *ifa = ifnet_addrs[ifp->if_index - 1];
 		struct sockaddr_dl *ll_addr;
-		
+
 		if (!ifa || !ifa->ifa_addr)
 			continue;
-		
+
 		ll_addr = (struct sockaddr_dl *)ifa->ifa_addr;
-		
+
 		if ((ifp->if_eflags & IFEF_DETACHING) == 0 &&
 			namelen == ll_addr->sdl_nlen &&
 			(strncmp(ll_addr->sdl_data, ifname, ll_addr->sdl_nlen) == 0))
diff --git a/bsd/net/kpi_interface.h b/bsd/net/kpi_interface.h
index 8f09d0985..8a0cd2b7c 100644
--- a/bsd/net/kpi_interface.h
+++ b/bsd/net/kpi_interface.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_interface.h
@@ -85,6 +91,8 @@ enum {
 */
 typedef u_int32_t ifnet_family_t;
 
+#ifndef BPF_TAP_MODE_T
+#define BPF_TAP_MODE_T
 /*!
 	@enum BPF tap mode
 	@abstract Constants defining interface families.
@@ -105,6 +113,7 @@ enum {
 	@abstract Mode for tapping. BPF_MODE_DISABLED/BPF_MODE_INPUT_OUTPUT etc.
 */
 typedef u_int32_t bpf_tap_mode;
+#endif /* !BPF_TAP_MODE_T */
 
 /*!
 	@typedef protocol_family_t
@@ -122,6 +131,18 @@ typedef u_int32_t protocol_family_t;
 	@constant IFNET_IP_FRAGMENT Hardware will fragment IP packets.
 	@constant IFNET_VLAN_TAGGING Hardware will generate VLAN headers.
 	@constant IFNET_VLAN_MTU Hardware supports VLAN MTU.
+	@constant IFNET_MULTIPAGES Driver is capable of handling packets
+		coming down from the network stack that reside in virtually,
+		but not in physically contiguous span of the external mbuf
+		clusters.  In this case, the data area of a packet in the
+		external mbuf cluster might cross one or more physical
+		pages that are disjoint, depending on the interface MTU
+		and the packet size.  Such a use of larger than system page
+		size clusters by the network stack is done for better system
+		efficiency.  Drivers that utilize the IOMbufNaturalMemoryCursor
+		with the getPhysicalSegmentsWithCoalesce interfaces and
+		enumerate the list of vectors should set this flag for
+		possible gain in performance during bulk data transfer.
 */
 
 enum {
@@ -135,6 +156,7 @@ enum {
 #endif
 		IFNET_VLAN_TAGGING	= 0x00010000,
 		IFNET_VLAN_MTU		= 0x00020000,
+		IFNET_MULTIPAGES	= 0x00100000,
 };
 /*!
 	@typedef ifnet_offload_t
@@ -178,6 +200,10 @@ typedef errno_t (*ifnet_output_func)(ifnet_t interface, mbuf_t data);
 	@typedef ifnet_ioctl_func
 	@discussion ifnet_ioctl_func is used to communicate ioctls from the
 		stack to the driver.
+		
+		All undefined ioctls are reserved for future use by Apple. If
+		you need to communicate with your kext using an ioctl, please
+		use SIOCSIFKPI and SIOCGIFKPI.
 	@param interface The interface the ioctl is being sent to.
 	@param proto_family The protocol family to handle the ioctl, may be
 		zero for no protocol_family.
@@ -188,13 +214,8 @@ typedef errno_t (*ifnet_ioctl_func)(ifnet_t interface, u_int32_t cmd, void *data
 
 /*!
 	@typedef ifnet_set_bpf_tap
-	@discussion ifnet_set_bpf_tap is used to set the bpf tap function to
-		be called when packets are sent and/or received.
-	@param interface The interface the bpf tap function is being set on.
-	@param mode Sets the mode of the tap to either disabled, input,
-		output, or input/output.
-	@param callback A function pointer to be called when a packet is
-		sent or received.
+	@discussion Deprecated. Specify NULL. Call bpf_tap_in/bpf_tap_out
+		for all packets.
  */
 typedef errno_t (*ifnet_set_bpf_tap)(ifnet_t interface, bpf_tap_mode mode,
 				bpf_packet_func callback);
@@ -347,6 +368,26 @@ typedef errno_t (*ifnet_check_multi)(ifnet_t interface,
 typedef errno_t (*proto_media_input)(ifnet_t ifp, protocol_family_t protocol,
 									 mbuf_t packet, char* header);
 
+/*!
+	@typedef proto_media_input_v2
+	@discussion proto_media_input_v2 is called for all inbound packets for
+		a specific protocol on a specific interface. This function is
+		registered on an interface using ifnet_attach_protocolv2.
+		proto_media_input_v2 differs from proto_media_input in that it will
+		be called for a list of packets instead of once for each individual
+		packet. The frame header can be retrieved using mbuf_pkthdr_header.
+	@param ifp The interface the packet was received on.
+	@param protocol_family The protocol of the packet received.
+	@param packet The packet being input.
+	@result
+		If the result is zero, the caller will assume the packets were passed
+		to the protocol.
+		If the result is non-zero and not EJUSTRETURN, the caller will free
+		the packets.
+ */
+typedef errno_t (*proto_media_input_v2)(ifnet_t ifp, protocol_family_t protocol,
+										mbuf_t packet);
+
 /*!
 	@typedef proto_media_preout
 	@discussion proto_media_preout is called just before the packet
@@ -392,6 +433,10 @@ typedef void (*proto_media_event)(ifnet_t ifp, protocol_family_t protocol,
 		EOPNOTSUPP, other parts of the stack may not get an opportunity
 		to process the ioctl. If you return EJUSTRETURN, processing will
 		stop and a result of zero will be returned to the caller.
+		
+		All undefined ioctls are reserved for future use by Apple. If
+		you need to communicate with your kext using an ioctl, please
+		use SIOCSIFKPI and SIOCGIFKPI.
 	@param ifp The interface.
 	@param protocol_family The protocol family.
 	@param command The ioctl command.
@@ -539,7 +584,7 @@ struct ifnet_init_params {
 	ifnet_framer_func		framer;			/* optional */
 	void*					softc;			/* optional */
 	ifnet_ioctl_func		ioctl;			/* optional */
-	ifnet_set_bpf_tap		set_bpf_tap;	/* optional */
+	ifnet_set_bpf_tap		set_bpf_tap;	/* deprecated */
 	ifnet_detached_func		detach;			/* optional */
 	ifnet_event_func		event;			/* optional */
 	const void				*broadcast_addr;/* required for non point-to-point interfaces */
@@ -630,6 +675,19 @@ struct ifnet_attach_proto_param {
 	proto_media_send_arp		send_arp;	/* optional */
 };
 
+struct ifnet_attach_proto_param_v2 {
+	struct ifnet_demux_desc	*demux_array;	/* interface may/may not require */
+	u_int32_t				demux_count;	/* interface may/may not require */
+	
+	proto_media_input_v2		input;		/* required */
+	proto_media_preout			pre_output;	/* required */
+	proto_media_event			event;		/* optional */
+	proto_media_ioctl			ioctl;		/* optional */
+	proto_media_detached		detached;	/* optional */
+	proto_media_resolve_multi	resolve;	/* optional */
+	proto_media_send_arp		send_arp;	/* optional */
+};
+
 __BEGIN_DECLS
 
 /*
@@ -900,6 +958,20 @@ u_int32_t ifnet_get_link_mib_data_length(ifnet_t interface);
 errno_t ifnet_attach_protocol(ifnet_t interface, protocol_family_t protocol_family,
 			const struct ifnet_attach_proto_param *proto_details);
 
+/*!
+	@function ifnet_attach_protocol_v2
+	@discussion Attaches a protocol to an interface using the newer version 2
+		style interface. So far the only difference is support for packet
+		chains which improve performance.
+	@param interface The interface.
+	@param protocol_family The protocol family being attached
+		(PF_INET/PF_APPLETALK/etc...).
+	@param proto_details Details of the protocol being attached.
+	@result 0 on success otherwise the errno error.
+ */
+errno_t ifnet_attach_protocol_v2(ifnet_t interface, protocol_family_t protocol_family,
+			const struct ifnet_attach_proto_param_v2 *proto_details);
+
 /*!
 	@function ifnet_detach_protocol
 	@discussion Detaches a protocol from an interface.
@@ -919,8 +991,7 @@ errno_t ifnet_detach_protocol(ifnet_t interface, protocol_family_t protocol_fami
 		determine which preoutput function to call. The route and dest
 		parameters will be passed to the preoutput function defined for
 		the attachment of the specified protocol to the specified
-		interface. ifnet_output will free the mbuf chain in the event of
-		an error.
+		interface. ifnet_output will always free the mbuf chain.
 	@param interface The interface.
 	@param protocol_family The family of the protocol generating this
 		packet (i.e. AF_INET).
@@ -946,8 +1017,7 @@ errno_t ifnet_output(ifnet_t interface, protocol_family_t protocol_family, mbuf_
 		packet. Processing, such as calling the protocol preoutput and
 		interface framer functions will be bypassed. The packet will
 		pass through the filters and be sent on the interface as is.
-		ifnet_output_raw will free the packet chain in the event of an
-		error.
+		ifnet_output_raw will always free the packet chain.
 	@param interface The interface.
 	@param protocol_family The family of the protocol generating this
 		packet (i.e. AF_INET).
@@ -977,6 +1047,10 @@ errno_t ifnet_input(ifnet_t interface, mbuf_t first_packet,
 	@function ifnet_ioctl
 	@discussion Calls the interface's ioctl function with the parameters
 		passed.
+		
+		All undefined ioctls are reserved for future use by Apple. If
+		you need to communicate with your kext using an ioctl, please
+		use SIOCSIFKPI and SIOCGIFKPI.
 	@param interface The interface.
 	@param protocol The protocol family of the protocol to send the
 		ioctl to (may be zero). Some ioctls apply to a protocol while
@@ -1317,6 +1391,23 @@ errno_t ifnet_llbroadcast_copy_bytes(ifnet_t interface, void* addr,
 errno_t ifnet_set_lladdr_and_type(ifnet_t interface, const void* lladdr, size_t length, u_char type);
 #endif KERNEL_PRIVATE
 
+/*!
+	@function ifnet_resolve_multicast
+	@discussion Resolves a multicast address for an attached protocol to
+		a link-layer address. If a link-layer address is passed in, the
+		interface will verify that it is a valid multicast address.
+	@param interface The interface.
+	@param proto_addr A protocol address to be converted to a link-layer
+		address.
+	@param ll_addr Storage for the resulting link-layer address.
+	@param ll_len Length of the storage for the link-layer address.
+	@result 0 on success. EOPNOTSUPP indicates the multicast address was
+		not supported or could not be translated. Other errors may
+		indicate other failures.
+ */
+errno_t ifnet_resolve_multicast(ifnet_t ifp, const struct sockaddr *proto_addr,
+								struct sockaddr *ll_addr, size_t ll_len);
+
 /*!
 	@function ifnet_add_multicast
 	@discussion Joins a multicast and returns an ifmultiaddr_t with the
diff --git a/bsd/net/kpi_interfacefilter.c b/bsd/net/kpi_interfacefilter.c
index ee9b28174..7bd9ea69f 100644
--- a/bsd/net/kpi_interfacefilter.c
+++ b/bsd/net/kpi_interfacefilter.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include "kpi_interfacefilter.h"
diff --git a/bsd/net/kpi_interfacefilter.h b/bsd/net/kpi_interfacefilter.h
index d52cbb1f3..e5c17d896 100644
--- a/bsd/net/kpi_interfacefilter.h
+++ b/bsd/net/kpi_interfacefilter.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_interfacefilter.h
@@ -35,6 +41,8 @@
 
 struct kev_msg;
 
+__BEGIN_DECLS
+
 /*!
 	@typedef iff_input_func
 	
@@ -114,14 +122,19 @@ typedef	void (*iff_event_func)(void* cookie, ifnet_t interface, protocol_family_
 		interface. The interface is only valid for the duration of the
 		filter call. If you need to keep a reference to the interface,
 		be sure to call ifnet_reference and ifnet_release.
+		
+		All undefined ioctls are reserved for future use by Apple. If
+		you need to communicate with your kext using an ioctl, please
+		use SIOCSIFKPI and SIOCGIFKPI.
 	@param cookie The cookie specified when this filter was attached.
 	@param interface The interface the packet is being transmitted on.
 	@param ioctl_cmd The ioctl command.
 	@param ioctl_arg A pointer to the ioctl argument.
 	@result Return:
-		0 - The caller will continue with normal processing of the packet.
-		EJUSTRETURN - The caller will stop processing the packet, the packet will not be freed.
-		Anything Else - The caller will free the packet and stop processing.
+		0 - This filter function handled the ioctl.
+		EOPNOTSUPP - This filter function does not understand/did not handle this ioctl.
+		EJUSTRETURN - This filter function handled the ioctl, processing should stop.
+		Anything Else - Processing will stop, the error will be returned.
 */
 typedef	errno_t (*iff_ioctl_func)(void* cookie, ifnet_t interface, protocol_family_t protocol,
 								  u_long ioctl_cmd, void* ioctl_arg);
@@ -192,3 +205,5 @@ errno_t iflt_attach(ifnet_t interface, const struct iff_filter* filter,
  */
 void iflt_detach(interface_filter_t filter_ref);
 
+__END_DECLS
+#endif
diff --git a/bsd/net/kpi_protocol.c b/bsd/net/kpi_protocol.c
index e33558dcc..8b3614a49 100644
--- a/bsd/net/kpi_protocol.c
+++ b/bsd/net/kpi_protocol.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include "kpi_protocol.h"
@@ -32,36 +38,50 @@
 #include <net/dlil.h>
 #include <libkern/OSAtomic.h>
 
-void proto_kpi_init(void);
 void proto_input_run(void);
 
 typedef int (*attach_t)(struct ifnet *ifp, u_long protocol_family);
 typedef int (*detach_t)(struct ifnet *ifp, u_long protocol_family);
 
-/****************************************************************************/
-/* WARNING: Big assumption made here - there can be only one input thread	*/
 struct proto_input_entry {
 	struct proto_input_entry		*next;
 	int								detach;
 	struct domain					*domain;
+	int								hash;
+	int								chain;
 	
 	protocol_family_t				protocol;
 	proto_input_handler				input;
 	proto_input_detached_handler	detached;
 	
-	mbuf_t							first_packet;
-	mbuf_t							last_packet;
+	mbuf_t							inject_first;
+	mbuf_t							inject_last;
+	
+	struct proto_input_entry		*input_next;
+	mbuf_t							input_first;
+	mbuf_t							input_last;
+};
+
+
+struct proto_family_str {
+	TAILQ_ENTRY(proto_family_str)	proto_fam_next;
+	protocol_family_t				proto_family;
+	ifnet_family_t					if_family;
+	proto_plumb_handler				attach_proto;
+	proto_unplumb_handler			detach_proto;
 };
 
 #define PROTO_HASH_SLOTS	5
 
-static struct proto_input_entry	*proto_hash[PROTO_HASH_SLOTS];
-static struct proto_input_entry *proto_input_add_list;
-static lck_mtx_t				*proto_input_lock = 0;
-__private_extern__ u_int32_t	inject_buckets = 0;
+static struct proto_input_entry			*proto_hash[PROTO_HASH_SLOTS];
+static int								proto_total_waiting = 0;
+static struct proto_input_entry 		*proto_input_add_list = NULL;
+static lck_mtx_t						*proto_family_mutex = 0;
+static TAILQ_HEAD(, proto_family_str)	proto_family_head =
+			TAILQ_HEAD_INITIALIZER(proto_family_head);
 
-extern thread_t	dlil_input_thread_ptr;
-extern int		dlil_input_thread_wakeup;
+extern lck_mtx_t	*domain_proto_mtx;
+extern struct dlil_threading_info *dlil_lo_thread_ptr;
 
 static int
 proto_hash_value(
@@ -92,19 +112,23 @@ proto_kpi_init(void)
 	lck_group = lck_grp_alloc_init("protocol kpi", grp_attrib);
 	lck_grp_attr_free(grp_attrib);
 	lck_attrib = lck_attr_alloc_init();
-	proto_input_lock = lck_mtx_alloc_init(lck_group, lck_attrib);
+	proto_family_mutex = lck_mtx_alloc_init(lck_group, lck_attrib);
 	lck_grp_free(lck_group);
 	lck_attr_free(lck_attrib);
+	
+	bzero(proto_hash, sizeof(proto_hash));
 }
 
 __private_extern__ errno_t
 proto_register_input(
 	protocol_family_t protocol,
 	proto_input_handler input,
-	proto_input_detached_handler detached)
+	proto_input_detached_handler detached,
+	int	chains)
 {
 	
 	struct proto_input_entry *entry;
+	struct dlil_threading_info *thread = dlil_lo_thread_ptr;
 	
 	entry = _MALLOC(sizeof(*entry), M_IFADDR, M_WAITOK);
 	
@@ -115,25 +139,29 @@ proto_register_input(
 	entry->protocol = protocol;
 	entry->input = input;
 	entry->detached = detached;
+	entry->hash = proto_hash_value(protocol);
+	entry->chain = chains;
 	
 	{
 		struct domain *dp = domains;
-		extern lck_mtx_t *domain_proto_mtx;
 
 		lck_mtx_assert(domain_proto_mtx, LCK_MTX_ASSERT_NOTOWNED);
          	lck_mtx_lock(domain_proto_mtx);
-		while (dp && dp->dom_family != protocol)
+		while (dp && (protocol_family_t)dp->dom_family != protocol)
 			dp = dp->dom_next;
 		entry->domain = dp;
          	lck_mtx_unlock(domain_proto_mtx);	
 	}
 
 	
-	do {
-		entry->next = proto_input_add_list;
-	} while(!OSCompareAndSwap((UInt32)entry->next, (UInt32)entry, (UInt32*)&proto_input_add_list));
+	lck_mtx_lock(thread->input_lck);
+	entry->next = proto_input_add_list;
+	proto_input_add_list = entry;
 	
-	wakeup((caddr_t)&dlil_input_thread_wakeup);
+	thread->input_waiting |= DLIL_PROTO_REGISTER;
+	if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0)
+		wakeup((caddr_t)&thread->input_waiting);
+	lck_mtx_unlock(thread->input_lck);
 	
 	return 0;
 }
@@ -183,71 +211,67 @@ proto_delayed_attach(
 	}
 }
 
-static void
-proto_delayed_inject(
-	struct proto_input_entry *entry)
-{
-	mbuf_t	packet_list;
-	mbuf_t	packet;
-	int		locked = 0;
-	
-	lck_mtx_lock(proto_input_lock);
-	packet_list = entry->first_packet;
-	entry->first_packet = entry->last_packet = 0;
-	lck_mtx_unlock(proto_input_lock);
-	
-	if (packet_list == NULL)
-		return;
-	
-	if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
-		lck_mtx_lock(entry->domain->dom_mtx);
-		locked = 1;
-	}
-	
-	for (packet = packet_list; packet; packet = packet_list) {
-		packet_list = mbuf_nextpkt(packet);
-		mbuf_setnextpkt(packet, NULL);
-		entry->input(entry->protocol, packet);
-	}
-	
-	if (locked) {
-		lck_mtx_unlock(entry->domain->dom_mtx);
-	}
-}
-
-/* This function must be called from a single dlil input thread */
 __private_extern__ void
 proto_input_run(void)
 {
 	struct proto_input_entry	*entry;
-	u_int32_t					inject;
-	int							i;
-	
-	if (current_thread() != dlil_input_thread_ptr)
-		panic("proto_input_run called from a thread other than dlil_input_thread!\n");
+	struct dlil_threading_info *thread = dlil_lo_thread_ptr;
+	mbuf_t packet_list;
+	int i, locked = 0;
+
+	lck_mtx_assert(thread->input_lck,  LCK_MTX_ASSERT_NOTOWNED);
 
-	do {
+	if ((thread->input_waiting & DLIL_PROTO_REGISTER) != 0) {
+		lck_mtx_lock(thread->input_lck);
 		entry = proto_input_add_list;
-	} while (entry && !OSCompareAndSwap((UInt32)entry, 0, (UInt32*)&proto_input_add_list));
-	
-	if (entry)
+		proto_input_add_list = NULL;
+		thread->input_waiting &= ~DLIL_PROTO_REGISTER;
+		lck_mtx_unlock(thread->input_lck);
 		proto_delayed_attach(entry);
-	
-	do {
-		inject = inject_buckets;
-	} while (inject && !OSCompareAndSwap(inject, 0, (UInt32*)&inject_buckets));
-	
-	if (inject) {
-		for (i = 0; i < PROTO_HASH_SLOTS; i++) {
-			if ((inject & (1L << i)) != 0) {
-				for (entry = proto_hash[i]; entry; entry = entry->next) {
-					if (entry->first_packet) {
-						proto_delayed_inject(entry);
+	}
+	/*
+	  Move everything from the lock protected list to the thread
+	  specific list.
+	 */
+	for (i = 0; proto_total_waiting != 0 && i < PROTO_HASH_SLOTS; i++) {
+		for (entry = proto_hash[i]; entry && proto_total_waiting;
+			 entry = entry->next) {
+			if (entry->inject_first) {
+				lck_mtx_lock(thread->input_lck);
+				thread->input_waiting &= ~DLIL_PROTO_WAITING;
+
+				packet_list = entry->inject_first;
+
+				entry->inject_first = NULL;
+				entry->inject_last = NULL;
+				proto_total_waiting--;
+
+				lck_mtx_unlock(thread->input_lck);
+
+				if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
+					lck_mtx_lock(entry->domain->dom_mtx);
+					locked = 1;
+				}
+		
+				if (entry->chain) {
+					entry->input(entry->protocol, packet_list);
+				}
+				else {
+					mbuf_t	packet;
+				
+					for (packet = packet_list; packet; packet = packet_list) {
+						packet_list = mbuf_nextpkt(packet);
+						mbuf_setnextpkt(packet, NULL);
+						entry->input(entry->protocol, packet);
 					}
 				}
-			}
+				if (locked) {
+					lck_mtx_unlock(entry->domain->dom_mtx);
+				}	
 		}
 	}
+	}
+
 }
 
 errno_t
@@ -255,53 +279,37 @@ proto_input(
 	protocol_family_t	protocol,
 	mbuf_t				packet_list)
 {
-	struct proto_input_entry *entry;
-	
-	if (current_thread() != dlil_input_thread_ptr)
-		panic("proto_input called from a thread other than dlil_input_thread!\n");
-	
-	for (entry = proto_hash[proto_hash_value(protocol)]; entry; entry = entry->next) {
+	struct proto_input_entry	*entry;
+	errno_t				locked =0, result = 0;
+
+	for (entry = proto_hash[proto_hash_value(protocol)]; entry;
+		 entry = entry->next) {
 		if (entry->protocol == protocol)
 			break;
 	}
 
-	if (entry) {
+	if (entry->domain && (entry->domain->dom_flags & DOM_REENTRANT) == 0) {
+		lck_mtx_lock(entry->domain->dom_mtx);
+		locked = 1;
+	}
+	
+	if (entry->chain) {
+		entry->input(entry->protocol, packet_list);
+	}
+	else {
 		mbuf_t	packet;
-#if DIRECT_PROTO_INPUT
-		// See <rdar://problem/3687868> for why this is disabled
-		// We need to release the dlil lock before taking the protocol lock
+		
 		for (packet = packet_list; packet; packet = packet_list) {
 			packet_list = mbuf_nextpkt(packet);
 			mbuf_setnextpkt(packet, NULL);
 			entry->input(entry->protocol, packet);
 		}
-#else
-		mbuf_t	last_packet;
-		int		hash_slot = proto_hash_value(protocol);
-		
-		for (last_packet = packet_list; mbuf_nextpkt(last_packet);
-			 last_packet = mbuf_nextpkt(last_packet))
-			/* find the last packet */;
-		
-		lck_mtx_lock(proto_input_lock);
-		if (entry->first_packet == NULL) {
-			entry->first_packet = packet_list;
-		}
-		else {
-			mbuf_setnextpkt(entry->last_packet, packet_list);
-		}
-		entry->last_packet = last_packet;
-		lck_mtx_unlock(proto_input_lock);
-	
-		OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets);
-#endif
-	}
-	else
-	{
-		return ENOENT;
 	}
 	
-	return 0;
+	if (locked) {
+		lck_mtx_unlock(entry->domain->dom_mtx);
+	}	
+	return result;
 }
 
 errno_t
@@ -310,8 +318,9 @@ proto_inject(
 	mbuf_t				packet_list)
 {
 	struct proto_input_entry	*entry;
-	mbuf_t						last_packet;
-	int							hash_slot = proto_hash_value(protocol);
+	mbuf_t				last_packet;
+	int				hash_slot = proto_hash_value(protocol);
+	struct dlil_threading_info 	*thread = dlil_lo_thread_ptr;
 	
 	for (last_packet = packet_list; mbuf_nextpkt(last_packet);
 		 last_packet = mbuf_nextpkt(last_packet))
@@ -323,19 +332,20 @@ proto_inject(
 	}
 	
 	if (entry) {
-		lck_mtx_lock(proto_input_lock);
-		if (entry->first_packet == NULL) {
-			entry->first_packet = packet_list;
+		lck_mtx_lock(thread->input_lck);
+		if (entry->inject_first == NULL) {
+			proto_total_waiting++;
+			thread->input_waiting |= DLIL_PROTO_WAITING;
+			entry->inject_first = packet_list;
 		}
 		else {
-			mbuf_setnextpkt(entry->last_packet, packet_list);
+			mbuf_setnextpkt(entry->inject_last, packet_list);
 		}
-		entry->last_packet = last_packet;
-		lck_mtx_unlock(proto_input_lock);
-	
-		OSBitOrAtomic((1L << hash_slot), (UInt32*)&inject_buckets);
-		
-		wakeup((caddr_t)&dlil_input_thread_wakeup);
+		entry->inject_last = last_packet;
+		if ((thread->input_waiting & DLIL_INPUT_RUNNING) == 0) {
+			wakeup((caddr_t)&thread->input_waiting);
+		}
+		lck_mtx_unlock(thread->input_lck);
 	}
 	else
 	{
@@ -345,20 +355,120 @@ proto_inject(
 	return 0;
 }
 
+static struct proto_family_str*
+proto_plumber_find(
+	protocol_family_t	proto_family,
+	ifnet_family_t		if_family)
+{
+	struct proto_family_str  *mod = NULL;
+
+	TAILQ_FOREACH(mod, &proto_family_head, proto_fam_next) {
+		if ((mod->proto_family == (proto_family & 0xffff)) 
+			&& (mod->if_family == (if_family & 0xffff))) 
+			break;
+		}
+
+	return mod;
+}
+
 errno_t
 proto_register_plumber(
-	protocol_family_t proto_fam,
-	ifnet_family_t if_fam,
-	proto_plumb_handler plumb,
-	proto_unplumb_handler unplumb)
+	protocol_family_t		protocol_family,
+	ifnet_family_t 			interface_family, 
+	proto_plumb_handler		attach,
+	proto_unplumb_handler	detach)
 {
-	return dlil_reg_proto_module(proto_fam, if_fam, (attach_t)plumb, (detach_t)unplumb);
+	struct proto_family_str *proto_family;
+
+	if (attach == NULL) return EINVAL;
+
+	lck_mtx_lock(proto_family_mutex);
+	
+	TAILQ_FOREACH(proto_family, &proto_family_head, proto_fam_next) {
+		if (proto_family->proto_family == protocol_family &&
+			proto_family->if_family == interface_family) {
+			lck_mtx_unlock(proto_family_mutex);
+			return EEXIST;
+		}
+	}
+
+	proto_family = (struct proto_family_str *) _MALLOC(sizeof(struct proto_family_str), M_IFADDR, M_WAITOK);
+	if (!proto_family) {
+		lck_mtx_unlock(proto_family_mutex);
+		return ENOMEM;
+	}
+
+	bzero(proto_family, sizeof(struct proto_family_str));
+	proto_family->proto_family	= protocol_family;
+	proto_family->if_family		= interface_family & 0xffff;
+	proto_family->attach_proto	= attach;
+	proto_family->detach_proto	= detach;
+
+	TAILQ_INSERT_TAIL(&proto_family_head, proto_family, proto_fam_next);
+	lck_mtx_unlock(proto_family_mutex);
+	return 0;
 }
 
 void
 proto_unregister_plumber(
-	protocol_family_t proto_fam,
-	ifnet_family_t if_fam)
+	protocol_family_t	protocol_family,
+	ifnet_family_t		interface_family)
 {
-	(void)dlil_dereg_proto_module(proto_fam, if_fam);
+	struct proto_family_str  *proto_family;
+
+	lck_mtx_lock(proto_family_mutex);
+
+	proto_family = proto_plumber_find(protocol_family, interface_family);
+	if (proto_family == 0) {
+		lck_mtx_unlock(proto_family_mutex);
+		return;
+	}
+
+	TAILQ_REMOVE(&proto_family_head, proto_family, proto_fam_next);
+	FREE(proto_family, M_IFADDR);
+	
+	lck_mtx_unlock(proto_family_mutex);
+	return;
+}
+
+__private_extern__ errno_t
+proto_plumb(
+	protocol_family_t	protocol_family,
+	ifnet_t				ifp)
+{
+	struct proto_family_str  *proto_family;
+	int ret = 0;
+
+	lck_mtx_lock(proto_family_mutex);
+	proto_family = proto_plumber_find(protocol_family, ifp->if_family);
+	if (proto_family == 0) {
+		lck_mtx_unlock(proto_family_mutex);
+		return ENXIO;
+	}
+
+	ret = proto_family->attach_proto(ifp, protocol_family);
+
+	lck_mtx_unlock(proto_family_mutex);
+   	return ret;
+}
+
+
+__private_extern__ errno_t
+proto_unplumb(
+	protocol_family_t	protocol_family,
+	ifnet_t				ifp)
+{
+	struct proto_family_str  *proto_family;
+	int ret = 0;
+
+	lck_mtx_lock(proto_family_mutex);
+
+	proto_family = proto_plumber_find(protocol_family, ifp->if_family);
+	if (proto_family && proto_family->detach_proto)
+		proto_family->detach_proto(ifp, protocol_family);
+	else
+		ret = ifnet_detach_protocol(ifp, protocol_family);
+    
+	lck_mtx_unlock(proto_family_mutex);
+	return ret;
 }
diff --git a/bsd/net/kpi_protocol.h b/bsd/net/kpi_protocol.h
index eefc5ddc0..60801e824 100644
--- a/bsd/net/kpi_protocol.h
+++ b/bsd/net/kpi_protocol.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_protocol.h
@@ -68,10 +74,11 @@ typedef void (*proto_input_detached_handler)(protocol_family_t protocol);
 	@param protocol The protocol family these functions will receive
 		packets for.
 	@param input The function called when a packet is input.
+	@param chains Input function supports packet chains.
 	@result A errno error on failure.
  */
 errno_t	proto_register_input(protocol_family_t protocol, proto_input_handler input,
-							 proto_input_detached_handler detached);
+							 proto_input_detached_handler detached, int chains);
 
 /*!
 	@function proto_unregister_input
@@ -171,5 +178,70 @@ errno_t	proto_register_plumber(protocol_family_t proto_fam, ifnet_family_t if_fa
  */
 void	proto_unregister_plumber(protocol_family_t proto_fam, ifnet_family_t if_fam);
 
+#ifdef KERNEL_PRIVATE
+
+/* 
+
+Function : proto_plumb
+
+    proto_plumb() will plumb a protocol to an actual interface.
+    This will find a registered protocol module and call its attach function.
+    The module will typically call dlil_attach_protocol with the appropriate parameters.
+            
+Parameters :
+    'protocol_family' is PF_INET, PF_INET6, ...
+    'ifp' is the interface to plumb the protocol to.
+    
+Return code :    
+
+0 :
+
+    No error.
+
+ENOENT:
+
+    No module was registered.
+
+other: 
+    
+    Error returned by the attach_proto function
+
+*/
+errno_t proto_plumb(protocol_family_t protocol_family, ifnet_t ifp);
+
+/* 
+
+Function : proto_unplumb
+
+    proto_unplumb() will unplumb a protocol from an interface.
+    This will find a registered protocol module and call its detach function.
+    The module will typically call dlil_detach_protocol with the appropriate parameters.
+    If no module is found, this function will call dlil_detach_protocol directly. 
+    
+Parameters :
+    'protocol_family' is PF_INET, PF_INET6, ...
+    'ifp' is APPLE_IF_FAM_ETHERNET, APPLE_IF_FAM_PPP, ...
+    
+Return code :    
+
+0 :
+
+    No error.
+
+ENOENT:
+
+    No module was registered.
+
+other: 
+    
+    Error returned by the attach_proto function
+
+*/
+errno_t proto_unplumb(protocol_family_t protocol_family, ifnet_t ifp);
+
+__private_extern__ void proto_kpi_init(void) __attribute__((section("__TEXT, initcode")));
+
+#endif KERNEL_PRIVATE
 __END_DECLS
 
+#endif
diff --git a/bsd/net/lacp.h b/bsd/net/lacp.h
index 0aad344c0..73fb8a1ab 100644
--- a/bsd/net/lacp.h
+++ b/bsd/net/lacp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/net/multicast_list.c b/bsd/net/multicast_list.c
index 6fbc66f25..0f9604b42 100644
--- a/bsd/net/multicast_list.c
+++ b/bsd/net/multicast_list.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/net/multicast_list.h b/bsd/net/multicast_list.h
index 6a6f64b65..c63c4abeb 100644
--- a/bsd/net/multicast_list.h
+++ b/bsd/net/multicast_list.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _NET_MULTICAST_LIST_H
diff --git a/bsd/net/ndrv.c b/bsd/net/ndrv.c
index d5e7e80b3..adc642bc3 100644
--- a/bsd/net/ndrv.c
+++ b/bsd/net/ndrv.c
@@ -1,25 +1,30 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1997-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/* Copyright (c) 1997, 1998 Apple Computer, Inc. All Rights Reserved */
 /*
  *	@(#)ndrv.c	1.1 (MacOSX) 6/10/43
  * Justin Walker, 970604
@@ -31,7 +36,7 @@
 /*
  * PF_NDRV allows raw access to a specified network device, directly
  *  with a socket.  Expected use involves a socket option to request
- *  protocol packets.  This lets ndrv_output() call dlil_output(), and
+ *  protocol packets.  This lets ndrv_output() call ifnet_output(), and
  *  lets DLIL find the proper recipient for incoming packets.
  *  The purpose here is for user-mode protocol implementation.
  * Note that "pure raw access" will still be accomplished with BPF.
@@ -65,6 +70,7 @@
 #include <net/if_dl.h>
 #include <net/if_types.h>
 #include <net/ndrv_var.h>
+#include <net/dlil.h>
 
 #if INET
 #include <netinet/in.h>
@@ -79,12 +85,13 @@ static int ndrv_do_disconnect(struct ndrv_cb *);
 static struct ndrv_cb *ndrv_find_inbound(struct ifnet *ifp, u_long protocol_family);
 static int ndrv_setspec(struct ndrv_cb *np, struct sockopt *sopt);
 static int ndrv_delspec(struct ndrv_cb *);
-static int ndrv_to_dlil_demux(struct ndrv_demux_desc* ndrv, struct dlil_demux_desc* dlil);
+static int ndrv_to_ifnet_demux(struct ndrv_demux_desc* ndrv, struct ifnet_demux_desc* ifdemux);
 static void ndrv_handle_ifp_detach(u_long family, short unit);
 static int ndrv_do_add_multicast(struct ndrv_cb *np, struct sockopt *sopt);
 static int ndrv_do_remove_multicast(struct ndrv_cb *np, struct sockopt *sopt);
 static struct ndrv_multiaddr* ndrv_have_multicast(struct ndrv_cb *np, struct sockaddr* addr);
 static void ndrv_remove_all_multicast(struct ndrv_cb *np);
+static void ndrv_dominit(void) __attribute__((section("__TEXT, initcode")));
 
 unsigned long  ndrv_sendspace = NDRVSNDQ;
 unsigned long  ndrv_recvspace = NDRVRCVQ;
@@ -94,8 +101,6 @@ extern struct domain ndrvdomain;
 extern struct protosw ndrvsw;
 extern lck_mtx_t *domain_proto_mtx;
 
-extern void kprintf(const char *, ...);
-
 /*
  * Verify these values match.
  * To keep clients from including dlil.h, we define
@@ -133,15 +138,14 @@ ndrv_output(struct mbuf *m, struct socket *so)
 	if ((m->m_flags&M_PKTHDR) == 0)
 		return(EINVAL);
 
-	/* Unlock before calling dlil_output */
+	/* Unlock before calling ifnet_output */
 	socket_unlock(so, 0);
 	
 	/*
      * Call DLIL if we can. DLIL is much safer than calling the
      * ifp directly.
      */
-	result = dlil_output(ifp, np->nd_proto_family, m, (caddr_t)NULL,
-						(struct sockaddr*)NULL, 1);
+	result = ifnet_output_raw(ifp, np->nd_proto_family, m);
 	
 	socket_lock(so, 0);
 	
@@ -149,18 +153,21 @@ ndrv_output(struct mbuf *m, struct socket *so)
 }
 
 /* Our input routine called from DLIL */
-static int
-ndrv_input(struct mbuf *m,
-	   char *frame_header,
-	   struct ifnet *ifp,
-	   u_long  proto_family,
-	   __unused int sync_ok)
+static errno_t
+ndrv_input(
+	ifnet_t				ifp,
+	protocol_family_t	proto_family,
+	mbuf_t				m,
+	char				*frame_header)
 {
 	struct socket *so;
-	struct sockaddr_dl ndrvsrc = {sizeof (struct sockaddr_dl), AF_NDRV};
+	struct sockaddr_dl ndrvsrc;
 	struct ndrv_cb *np;
 	int error = 0;
 
+    ndrvsrc.sdl_len = sizeof (struct sockaddr_dl);
+    ndrvsrc.sdl_family = AF_NDRV;
+    ndrvsrc.sdl_index = 0;
 
     /* move packet from if queue to socket */
 	/* Should be media-independent */
@@ -177,10 +184,10 @@ ndrv_input(struct mbuf *m,
 	}
 	so = np->nd_socket;
     /* prepend the frame header */
-    m = m_prepend(m, ifp->if_hdrlen, M_NOWAIT);
+    m = m_prepend(m, ifnet_hdrlen(ifp), M_NOWAIT);
     if (m == NULL)
         return EJUSTRETURN;
-    bcopy(frame_header, m->m_data, ifp->if_hdrlen);
+    bcopy(frame_header, m->m_data, ifnet_hdrlen(ifp));
 
 	lck_mtx_assert(so->so_proto->pr_domain->dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
@@ -283,7 +290,8 @@ ndrv_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 }
 
 static void
-ndrv_event(struct ifnet *ifp, struct kev_msg *event)
+ndrv_event(struct ifnet *ifp, __unused protocol_family_t protocol,
+		   const struct kev_msg *event)
 {
 	if (event->vendor_code == KEV_VENDOR_APPLE &&
 		event->kev_class == KEV_NETWORK_CLASS &&
@@ -291,7 +299,7 @@ ndrv_event(struct ifnet *ifp, struct kev_msg *event)
 		event->event_code == KEV_DL_IF_DETACHING) {
 		lck_mtx_assert(ndrvdomain.dom_mtx, LCK_MTX_ASSERT_NOTOWNED);
 		lck_mtx_lock(ndrvdomain.dom_mtx);
-		ndrv_handle_ifp_detach(ifp->if_family, ifp->if_unit);
+		ndrv_handle_ifp_detach(ifnet_family(ifp), ifnet_unit(ifp));
 		lck_mtx_unlock(ndrvdomain.dom_mtx);
 	}
 }
@@ -326,7 +334,7 @@ ndrv_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 	if (np->nd_laddr == NULL)
 		return(ENOMEM);
 	bcopy((caddr_t) sa, (caddr_t) np->nd_laddr, sizeof(struct sockaddr_ndrv));
-	dname = sa->snd_name;
+	dname = (char *) sa->snd_name;
 	if (*dname == '\0')
 		return(EINVAL);
 #if NDRV_DEBUG
@@ -347,22 +355,17 @@ ndrv_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 		return(EADDRNOTAVAIL);
 	
 	// PPP doesn't support PF_NDRV.
-	if (ifp->if_family != APPLE_IF_FAM_PPP)
+	if (ifnet_family(ifp) != APPLE_IF_FAM_PPP)
 	{
 		/* NDRV on this interface */
-		struct dlil_proto_reg_str	ndrv_proto;
+		struct ifnet_attach_proto_param	ndrv_proto;
 		result = 0;
 		bzero(&ndrv_proto, sizeof(ndrv_proto));
-		TAILQ_INIT(&ndrv_proto.demux_desc_head);
-		
-		ndrv_proto.interface_family = ifp->if_family;
-		ndrv_proto.protocol_family = PF_NDRV;
-		ndrv_proto.unit_number = ifp->if_unit;
 		ndrv_proto.event = ndrv_event;
 		
 		/* We aren't worried about double attaching, that should just return an error */
 		socket_unlock(so, 0);
-		result = dlil_attach_protocol(&ndrv_proto);
+		result = ifnet_attach_protocol(ifp, PF_NDRV, &ndrv_proto);
 		socket_lock(so, 0);
 		if (result && result != EEXIST) {
 			return result;
@@ -374,8 +377,8 @@ ndrv_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 	}
     
 	np->nd_if = ifp;
-    np->nd_family = ifp->if_family;
-    np->nd_unit = ifp->if_unit;
+    np->nd_family = ifnet_family(ifp);
+    np->nd_unit = ifnet_unit(ifp);
     
 	return(0);
 }
@@ -495,7 +498,7 @@ ndrv_ctloutput(struct socket *so, struct sockopt *sopt)
     {
         case NDRV_DELDMXSPEC: /* Delete current spec */
             /* Verify no parameter was passed */
-            if (sopt->sopt_val != NULL || sopt->sopt_valsize != 0) {
+            if (sopt->sopt_val != 0 || sopt->sopt_valsize != 0) {
                 /*
                  * We don't support deleting a specific demux, it's
                  * all or nothing.
@@ -535,6 +538,7 @@ ndrv_do_detach(struct ndrv_cb *np)
 	kprintf("NDRV detach: %x, %x\n", so, np);
 #endif
     ndrv_remove_all_multicast(np);
+
     ifp = np->nd_if;
     /* Remove from the linked list of control blocks */
     TAILQ_REMOVE(&ndrvl, np, nd_next);
@@ -543,7 +547,7 @@ ndrv_do_detach(struct ndrv_cb *np)
 
 		if (proto_family != PF_NDRV && proto_family != 0) {
 			socket_unlock(so, 0);
-			dlil_detach_protocol(ifp, proto_family);
+			ifnet_detach_protocol(ifp, proto_family);
 			socket_lock(so, 0);
 		}
 		
@@ -558,11 +562,10 @@ ndrv_do_detach(struct ndrv_cb *np)
 		/* If there are no other interfaces, detach PF_NDRV from the interface */
 		if (cur_np == NULL) {
 			socket_unlock(so, 0);
-			dlil_detach_protocol(ifp, PF_NDRV);
+			ifnet_detach_protocol(ifp, PF_NDRV);
 			socket_lock(so, 0);
 		}
 	}
-    
     	if (np->nd_laddr != NULL) {
 		FREE((caddr_t)np->nd_laddr, M_IFADDR);
 		np->nd_laddr = NULL;
@@ -618,10 +621,10 @@ static int name_cmp(struct ifnet *ifp, char *q)
 	char buf[IFNAMSIZ];
 
 	r = buf;
-	len = strlen(ifp->if_name);
-	strncpy(r, ifp->if_name, IFNAMSIZ);
+	len = strlen(ifnet_name(ifp));
+	strncpy(r, ifnet_name(ifp), IFNAMSIZ);
 	r += len;
-	(void)sprint_d(ifp->if_unit, r, IFNAMSIZ-(r-buf));
+	(void)sprint_d(ifnet_unit(ifp), r, IFNAMSIZ-(r-buf));
 #if NDRV_DEBUG
 	kprintf("Comparing %s, %s\n", buf, q);
 #endif
@@ -652,102 +655,121 @@ ndrv_flushq(struct ifqueue *q)
 int
 ndrv_setspec(struct ndrv_cb *np, struct sockopt *sopt)
 {
-    struct dlil_proto_reg_str	dlilSpec;
-    struct ndrv_protocol_desc	ndrvSpec;
-    struct dlil_demux_desc*		dlilDemux = NULL;
-    struct ndrv_demux_desc*		ndrvDemux = NULL;
-    int							error = 0;
-    struct socket *so = np->nd_socket; 
-    
-    /* Sanity checking */
-    if (np->nd_proto_family != PF_NDRV)
-        return EBUSY;
-    if (np->nd_if == NULL)
-        return EINVAL;
-    if (sopt->sopt_valsize != sizeof(struct ndrv_protocol_desc))
-        return EINVAL;
-    
-    /* Copy the ndrvSpec */
-    error = sooptcopyin(sopt, &ndrvSpec, sizeof(struct ndrv_protocol_desc),
-                        sizeof(struct ndrv_protocol_desc));
-    if (error != 0)
-        return error;
-    
-    /* Verify the parameter */
-    if (ndrvSpec.version > NDRV_PROTOCOL_DESC_VERS)
-        return ENOTSUP; // version is too new!
-    else if (ndrvSpec.version < 1)
-        return EINVAL; // version is not valid
-    
-    /* Allocate storage for demux array */
-    MALLOC(ndrvDemux, struct ndrv_demux_desc*,
-            ndrvSpec.demux_count * sizeof(struct ndrv_demux_desc), M_TEMP, M_WAITOK);
-    if (ndrvDemux == NULL)
-        return ENOMEM;
-    
-    /* Allocate enough dlil_demux_descs */
-    MALLOC(dlilDemux, struct dlil_demux_desc*,
-            sizeof(*dlilDemux) * ndrvSpec.demux_count, M_TEMP, M_WAITOK);
-    if (dlilDemux == NULL)
-        error = ENOMEM;
-    
-    if (error == 0)
-    {
-        /* Copy the ndrv demux array from userland */
-        error = copyin(CAST_USER_ADDR_T(ndrvSpec.demux_list), ndrvDemux,
-                    ndrvSpec.demux_count * sizeof(struct ndrv_demux_desc));
-        ndrvSpec.demux_list = ndrvDemux;
-    }
-    
-    if (error == 0)
-    {
-        /* At this point, we've at least got enough bytes to start looking around */
-        u_long	demuxOn = 0;
-        
-        bzero(&dlilSpec, sizeof(dlilSpec));
-        TAILQ_INIT(&dlilSpec.demux_desc_head);
-        dlilSpec.interface_family = np->nd_family;
-        dlilSpec.unit_number = np->nd_unit;
-        dlilSpec.input = ndrv_input;
-		dlilSpec.event = ndrv_event;
-        dlilSpec.protocol_family = ndrvSpec.protocol_family;
-        
-        for (demuxOn = 0; demuxOn < ndrvSpec.demux_count; demuxOn++)
-        {
-            /* Convert an ndrv_demux_desc to a dlil_demux_desc */
-            error = ndrv_to_dlil_demux(&ndrvSpec.demux_list[demuxOn], &dlilDemux[demuxOn]);
-            if (error)
-                break;
-            
-            /* Add the dlil_demux_desc to the list */
-            TAILQ_INSERT_TAIL(&dlilSpec.demux_desc_head, &dlilDemux[demuxOn], next);
-        }
-    }
-    
-    if (error == 0)
-    {
-        /* We've got all our ducks lined up...lets attach! */
-	socket_unlock(so, 0);
-        error = dlil_attach_protocol(&dlilSpec);
-	socket_lock(so, 0);
-        if (error == 0)
-        	np->nd_proto_family = dlilSpec.protocol_family;
-    }
-    
-    /* Free any memory we've allocated */
-    if (dlilDemux)
-        FREE(dlilDemux, M_TEMP);
-    if (ndrvDemux)
-        FREE(ndrvDemux, M_TEMP);
-    
-    return error;
+	struct ifnet_attach_proto_param	proto_param;
+	struct ndrv_protocol_desc	ndrvSpec;
+	struct ndrv_demux_desc*		ndrvDemux = NULL;
+	int							error = 0;
+	struct socket *				so = np->nd_socket; 
+	user_addr_t					user_addr;
+	
+	/* Sanity checking */
+	if (np->nd_proto_family != PF_NDRV)
+		return EBUSY;
+	if (np->nd_if == NULL)
+		return EINVAL;
+
+	/* Copy the ndrvSpec */
+	if (proc_is64bit(current_proc())) {
+		struct ndrv_protocol_desc64	ndrvSpec64;
+
+		if (sopt->sopt_valsize != sizeof(ndrvSpec64))
+			return EINVAL;
+	
+		error = sooptcopyin(sopt, &ndrvSpec64, sizeof(ndrvSpec64), sizeof(ndrvSpec64));
+		if (error != 0)
+			return error;
+
+		ndrvSpec.version         = ndrvSpec64.version;
+		ndrvSpec.protocol_family = ndrvSpec64.protocol_family;
+		ndrvSpec.demux_count     = ndrvSpec64.demux_count;
+
+		user_addr = ndrvSpec64.demux_list;
+	}
+	else {
+		if (sopt->sopt_valsize != sizeof(ndrvSpec))
+			return EINVAL;
+	
+		error = sooptcopyin(sopt, &ndrvSpec, sizeof(ndrvSpec), sizeof(ndrvSpec));
+		if (error != 0)
+			return error;
+
+		user_addr = CAST_USER_ADDR_T(ndrvSpec.demux_list);
+	}
+	
+	/* Verify the parameter */
+	if (ndrvSpec.version > NDRV_PROTOCOL_DESC_VERS)
+		return ENOTSUP; // version is too new!
+	else if (ndrvSpec.version < 1)
+		return EINVAL; // version is not valid
+	
+	bzero(&proto_param, sizeof(proto_param));
+	proto_param.demux_count = ndrvSpec.demux_count;
+	
+	/* Allocate storage for demux array */
+	MALLOC(ndrvDemux, struct ndrv_demux_desc*, proto_param.demux_count *
+		   sizeof(struct ndrv_demux_desc), M_TEMP, M_WAITOK);
+	if (ndrvDemux == NULL)
+		return ENOMEM;
+	
+	/* Allocate enough ifnet_demux_descs */
+	MALLOC(proto_param.demux_array, struct ifnet_demux_desc*,
+		   sizeof(*proto_param.demux_array) * ndrvSpec.demux_count,
+		   M_TEMP, M_WAITOK);
+	if (proto_param.demux_array == NULL)
+		error = ENOMEM;
+	
+	if (error == 0)
+	{
+		/* Copy the ndrv demux array from userland */
+		error = copyin(user_addr, ndrvDemux,
+					   ndrvSpec.demux_count * sizeof(struct ndrv_demux_desc));
+		ndrvSpec.demux_list = ndrvDemux;
+	}
+	
+	if (error == 0)
+	{
+		/* At this point, we've at least got enough bytes to start looking around */
+		u_long	demuxOn = 0;
+		
+		proto_param.demux_count = ndrvSpec.demux_count;
+		proto_param.input = ndrv_input;
+		proto_param.event = ndrv_event;
+		
+		for (demuxOn = 0; demuxOn < ndrvSpec.demux_count; demuxOn++)
+		{
+			/* Convert an ndrv_demux_desc to a ifnet_demux_desc */
+			error = ndrv_to_ifnet_demux(&ndrvSpec.demux_list[demuxOn],
+										&proto_param.demux_array[demuxOn]);
+			if (error)
+				break;
+		}
+	}
+	
+	if (error == 0)
+	{
+		/* We've got all our ducks lined up...lets attach! */
+		socket_unlock(so, 0);
+		error = ifnet_attach_protocol(np->nd_if, ndrvSpec.protocol_family,
+									  &proto_param);
+		socket_lock(so, 0);
+		if (error == 0)
+			np->nd_proto_family = ndrvSpec.protocol_family;
+	}
+	
+	/* Free any memory we've allocated */
+	if (proto_param.demux_array)
+		FREE(proto_param.demux_array, M_TEMP);
+	if (ndrvDemux)
+		FREE(ndrvDemux, M_TEMP);
+	
+	return error;
 }
 
 
 int
-ndrv_to_dlil_demux(struct ndrv_demux_desc* ndrv, struct dlil_demux_desc* dlil)
+ndrv_to_ifnet_demux(struct ndrv_demux_desc* ndrv, struct ifnet_demux_desc* ifdemux)
 {
-    bzero(dlil, sizeof(*dlil));
+    bzero(ifdemux, sizeof(*ifdemux));
     
     if (ndrv->type < DLIL_DESC_ETYPE2)
     {
@@ -760,9 +782,9 @@ ndrv_to_dlil_demux(struct ndrv_demux_desc* ndrv, struct dlil_demux_desc* dlil)
         return EINVAL;
     }
     
-    dlil->type = ndrv->type;
-    dlil->native_type = ndrv->data.other;
-    dlil->variants.native_type_length = ndrv->length;
+    ifdemux->type = ndrv->type;
+    ifdemux->data = ndrv->data.other;
+    ifdemux->datalen = ndrv->length;
     
     return 0;
 }
@@ -777,7 +799,7 @@ ndrv_delspec(struct ndrv_cb *np)
         return EINVAL;
     
     /* Detach the protocol */
-    result = dlil_detach_protocol(np->nd_if, np->nd_proto_family);
+    result = ifnet_detach_protocol(np->nd_if, np->nd_proto_family);
     np->nd_proto_family = PF_NDRV;
     
 	return result;
@@ -846,7 +868,7 @@ ndrv_handle_ifp_detach(u_long family, short unit)
     
     /* Unregister our protocol */
     if (ifp) {
-        dlil_detach_protocol(ifp, PF_NDRV);
+        ifnet_detach_protocol(ifp, PF_NDRV);
     }
 }
 
@@ -856,7 +878,7 @@ ndrv_do_add_multicast(struct ndrv_cb *np, struct sockopt *sopt)
     struct ndrv_multiaddr*	ndrv_multi;
     int						result;
     
-    if (sopt->sopt_val == NULL || sopt->sopt_valsize < 2 ||
+    if (sopt->sopt_val == 0 || sopt->sopt_valsize < 2 ||
         sopt->sopt_level != SOL_NDRVPROTO)
         return EINVAL;
     if (np->nd_if == NULL)
@@ -881,7 +903,8 @@ ndrv_do_add_multicast(struct ndrv_cb *np, struct sockopt *sopt)
     if (result == 0)
     {
         // Try adding the multicast
-        result = if_addmulti(np->nd_if, &ndrv_multi->addr, &ndrv_multi->ifma);
+        result = ifnet_add_multicast(np->nd_if, &ndrv_multi->addr,
+        							 &ndrv_multi->ifma);
     }
     
     if (result == 0)
@@ -906,7 +929,7 @@ ndrv_do_remove_multicast(struct ndrv_cb *np, struct sockopt *sopt)
     struct ndrv_multiaddr*	ndrv_entry = NULL;
     int					result;
     
-    if (sopt->sopt_val == NULL || sopt->sopt_valsize < 2 ||
+    if (sopt->sopt_val == 0 || sopt->sopt_valsize < 2 ||
         sopt->sopt_level != SOL_NDRVPROTO)
         return EINVAL;
     if (np->nd_if == NULL)
@@ -937,7 +960,7 @@ ndrv_do_remove_multicast(struct ndrv_cb *np, struct sockopt *sopt)
     if (result == 0)
     {
         // Try deleting the multicast
-        result = if_delmultiaddr(ndrv_entry->ifma, 0);
+        result = ifnet_remove_multicast(ndrv_entry->ifma);
     }
     
     if (result == 0)
@@ -945,7 +968,7 @@ ndrv_do_remove_multicast(struct ndrv_cb *np, struct sockopt *sopt)
         // Remove from our linked list
         struct ndrv_multiaddr*	cur = np->nd_multiaddrs;
         
-        ifma_release(ndrv_entry->ifma);
+        ifmaddr_release(ndrv_entry->ifma);
         
         if (cur == ndrv_entry)
         {
@@ -1001,8 +1024,8 @@ ndrv_remove_all_multicast(struct ndrv_cb* np)
             cur = np->nd_multiaddrs;
             np->nd_multiaddrs = cur->next;
             
-            if_delmultiaddr(cur->ifma, 0);
-            ifma_release(cur->ifma);
+            ifnet_remove_multicast(cur->ifma);
+            ifmaddr_release(cur->ifma);
             FREE(cur, M_IFADDR);
         }
     }
@@ -1018,15 +1041,29 @@ struct pr_usrreqs ndrv_usrreqs = {
 
 struct protosw ndrvsw =
 {	SOCK_RAW, &ndrvdomain, NDRVPROTO_NDRV, PR_ATOMIC|PR_ADDR,
-	0, ndrv_output, 0, ndrv_ctloutput,
-	0, 0, 0, 0,
-	0, 0,
+	NULL, ndrv_output, NULL, ndrv_ctloutput,
+	NULL,
+	NULL, NULL, NULL, NULL, NULL, 
 	&ndrv_usrreqs,
-	0, 0, 0
+	NULL, NULL, NULL,
+	{ NULL, NULL}, NULL,
+	{ 0 }
 };
 
 struct domain ndrvdomain =
-{	AF_NDRV, "NetDriver", ndrv_dominit, NULL, NULL,
+{	AF_NDRV, 
+	"NetDriver", 
+	ndrv_dominit, 
+	NULL, 
+	NULL,
+	NULL, 
+	NULL, 
+	NULL, 
+	0, 
+	0, 
+	0, 
+	0,
 	NULL,
-	NULL, NULL, 0, 0, 0, 0
+	0, 
+	{0, 0}
 };
diff --git a/bsd/net/ndrv.h b/bsd/net/ndrv.h
index 6aa8e14bd..5a1feb1f5 100644
--- a/bsd/net/ndrv.h
+++ b/bsd/net/ndrv.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997, 1998 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -114,6 +120,20 @@ struct ndrv_protocol_desc
     struct ndrv_demux_desc*	demux_list;
 };
 
+#ifdef KERNEL_PRIVATE
+/* LP64 version of ndrv_protocol_desc.  all pointers 
+ * grow when we're dealing with a 64-bit process.
+ * WARNING - keep in sync with ndrv_protocol_desc
+ */
+struct ndrv_protocol_desc64 {
+    u_int32_t				version;
+    u_int32_t				protocol_family;
+    u_int32_t				demux_count;
+    user_addr_t				demux_list __attribute__((aligned(8)));
+};
+
+#endif // KERNEL_PRIVATE
+
 #define SOL_NDRVPROTO		NDRVPROTO_NDRV	/* Use this socket level */
 #define NDRV_DELDMXSPEC		0x02			/* Delete the registered protocol */
 #define NDRV_SETDMXSPEC		0x04			/* Set the protocol spec */
diff --git a/bsd/net/ndrv_var.h b/bsd/net/ndrv_var.h
index d39c2a7ad..ba6bf781c 100644
--- a/bsd/net/ndrv_var.h
+++ b/bsd/net/ndrv_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997, 1998 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -37,7 +43,7 @@
 struct ndrv_multiaddr
 {
     struct ndrv_multiaddr*	next;
-    struct if_multiaddr*	ifma;
+    ifmultiaddr_t			ifma;
     struct sockaddr			addr;
 };
 
diff --git a/bsd/net/net_osdep.c b/bsd/net/net_osdep.c
index 51bea3c12..dbe6f6692 100644
--- a/bsd/net/net_osdep.c
+++ b/bsd/net/net_osdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -67,8 +73,7 @@
 #include <net/net_osdep.h>
 
 const char *
-if_name(ifp)
-	struct ifnet *ifp;
+if_name(struct ifnet *ifp)
 {
 #define MAXNUMBUF	8
 	static char nam[MAXNUMBUF][IFNAMSIZ + 10];	/*enough?*/
diff --git a/bsd/net/net_osdep.h b/bsd/net/net_osdep.h
index 15775cb29..4208380cf 100644
--- a/bsd/net/net_osdep.h
+++ b/bsd/net/net_osdep.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
diff --git a/bsd/net/pfkeyv2.h b/bsd/net/pfkeyv2.h
index eeff8a049..ccb5bbaa8 100644
--- a/bsd/net/pfkeyv2.h
+++ b/bsd/net/pfkeyv2.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: pfkeyv2.h,v 1.10 2000/03/22 07:04:20 sakane Exp $	*/
 
@@ -352,6 +358,7 @@ struct sadb_x_ipsecrequest {
 #define SADB_X_EALG_CAST128CBC	5	/*6*/
 #define SADB_X_EALG_BLOWFISHCBC	4	/*7*/
 #define SADB_X_EALG_RIJNDAELCBC	12
+#define SADB_X_EALG_AESCBC      12
 #define SADB_X_EALG_AES		12
 /* private allocations should use 249-255 (RFC2407) */
 
@@ -377,7 +384,9 @@ struct sadb_x_ipsecrequest {
 #define SADB_X_EXT_NATT				0x0002	/* Use UDP encapsulation to traverse NAT */
 #define SADB_X_EXT_NATT_KEEPALIVE	0x0004	/* Local node is behind NAT, send keepalives */
 											/* Should only be set for outbound SAs */
-#endif /* PRIVATE */
+#define SADB_X_EXT_NATT_MULTIPLEUSERS 0x0008	/* For use on VPN server - support multiple users 	*/
+
+#endif /* PRIVATE */	
 
 #define SADB_X_EXT_IV4B		0x0010	/* IV length of 4 bytes in use */
 #define SADB_X_EXT_DERIV	0x0020	/* DES derived */
diff --git a/bsd/net/ppp_comp.h b/bsd/net/ppp_comp.h
index 69a7f8b89..67af45e90 100644
--- a/bsd/net/ppp_comp.h
+++ b/bsd/net/ppp_comp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * ppp_comp.h - Definitions for doing PPP packet compression.
diff --git a/bsd/net/ppp_deflate.c b/bsd/net/ppp_deflate.c
index 3d2b95df3..5968578b0 100644
--- a/bsd/net/ppp_deflate.c
+++ b/bsd/net/ppp_deflate.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * ppp_deflate.c - interface the zlib procedures for Deflate compression
@@ -47,17 +53,27 @@
  * OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
  * OR MODIFICATIONS.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <net/ppp_defs.h>
-#include <net/zlib.h>
+#include <libkern/zlib.h>
 
 #define PACKETPTR	struct mbuf *
 #include <net/ppp_comp.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #if DO_DEFLATE
 
 #define DEFLATE_DEBUG	1
@@ -537,6 +553,9 @@ z_decompress(arg, mi, mop)
 	mo->m_data += state->hdrlen;
 	ospace -= state->hdrlen;
     }
+#if CONFIG_MACF_NET
+   mac_mbuf_label_copy(mi, mo);
+#endif
 
     /*
      * Fill in the first part of the PPP header.  The protocol field
diff --git a/bsd/net/ppp_defs.h b/bsd/net/ppp_defs.h
index b902632a9..2cfd6ac01 100644
--- a/bsd/net/ppp_defs.h
+++ b/bsd/net/ppp_defs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * ppp_defs.h - PPP definitions.
diff --git a/bsd/net/radix.c b/bsd/net/radix.c
index 21825d1dc..36aa3bc0d 100644
--- a/bsd/net/radix.c
+++ b/bsd/net/radix.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1989, 1993
@@ -144,12 +150,10 @@ static int	rn_satsifies_leaf(char *trial, struct radix_node *leaf,
  */
 
 static struct radix_node *
-rn_search(v_arg, head)
-	void *v_arg;
-	struct radix_node *head;
+rn_search(void *v_arg, struct radix_node *head)
 {
-	register struct radix_node *x;
-	register caddr_t v;
+	struct radix_node *x;
+	caddr_t v;
 
 	for (x = head, v = v_arg; x->rn_bit >= 0;) {
 		if (x->rn_bmask & v[x->rn_offset])
@@ -161,12 +165,10 @@ rn_search(v_arg, head)
 }
 
 static struct radix_node *
-rn_search_m(v_arg, head, m_arg)
-	struct radix_node *head;
-	void *v_arg, *m_arg;
+rn_search_m(void *v_arg, struct radix_node *head, void *m_arg)
 {
-	register struct radix_node *x;
-	register caddr_t v = v_arg, m = m_arg;
+	struct radix_node *x;
+	caddr_t v = v_arg, m = m_arg;
 
 	for (x = head; x->rn_bit >= 0;) {
 		if ((x->rn_bmask & m[x->rn_offset]) &&
@@ -179,11 +181,10 @@ rn_search_m(v_arg, head, m_arg)
 }
 
 int
-rn_refines(m_arg, n_arg)
-	void *m_arg, *n_arg;
+rn_refines(void *m_arg, void *n_arg)
 {
-	register caddr_t m = m_arg, n = n_arg;
-	register caddr_t lim, lim2 = lim = n + *(u_char *)n;
+	caddr_t m = m_arg, n = n_arg;
+	caddr_t lim, lim2 = lim = n + *(u_char *)n;
 	int longer = (*(u_char *)n++) - (int)(*(u_char *)m++);
 	int masks_are_equal = 1;
 
@@ -206,17 +207,15 @@ rn_refines(m_arg, n_arg)
 }
 
 struct radix_node *
-rn_lookup(v_arg, m_arg, head)
-	void *v_arg, *m_arg;
-	struct radix_node_head *head;
+rn_lookup(void *v_arg, void *m_arg, struct radix_node_head *head)
 {
-	register struct radix_node *x;
-	caddr_t netmask = 0;
+	struct radix_node *x;
+	caddr_t netmask = NULL;
 
 	if (m_arg) {
 		x = rn_addmask(m_arg, 1, head->rnh_treetop->rn_offset);
 		if (x == 0)
-			return (0);
+			return (NULL);
 		netmask = x->rn_key;
 	}
 	x = rn_match(v_arg, head);
@@ -228,12 +227,9 @@ rn_lookup(v_arg, m_arg, head)
 }
 
 static int
-rn_satsifies_leaf(trial, leaf, skip)
-	char *trial;
-	register struct radix_node *leaf;
-	int skip;
+rn_satsifies_leaf(char *trial, struct radix_node *leaf, int skip)
 {
-	register char *cp = trial, *cp2 = leaf->rn_key, *cp3 = leaf->rn_mask;
+	char *cp = trial, *cp2 = leaf->rn_key, *cp3 = leaf->rn_mask;
 	char *cplim;
 	int length = min(*(u_char *)cp, *(u_char *)cp2);
 
@@ -249,17 +245,15 @@ rn_satsifies_leaf(trial, leaf, skip)
 }
 
 struct radix_node *
-rn_match(v_arg, head)
-	void *v_arg;
-	struct radix_node_head *head;
+rn_match(void *v_arg, struct radix_node_head *head)
 {
 	caddr_t v = v_arg;
-	register struct radix_node *t = head->rnh_treetop, *x;
-	register caddr_t cp = v, cp2;
+	struct radix_node *t = head->rnh_treetop, *x;
+	caddr_t cp = v, cp2;
 	caddr_t cplim;
 	struct radix_node *saved_t, *top = t;
 	int off = t->rn_offset, vlen = *(u_char *)cp, matched_off;
-	register int test, b, rn_bit;
+	int test, b, rn_bit;
 
 	/*
 	 * Open code rn_search(v, top) to avoid overhead of extra
@@ -324,7 +318,7 @@ on1:
 	t = saved_t;
 	/* start searching up the tree */
 	do {
-		register struct radix_mask *m;
+		struct radix_mask *m;
 		t = t->rn_parent;
 		m = t->rn_mklist;
 		/*
@@ -348,7 +342,7 @@ on1:
 			m = m->rm_mklist;
 		}
 	} while (t != top);
-	return 0;
+	return NULL;
 }
 
 #ifdef RN_DEBUG
@@ -359,12 +353,9 @@ int	rn_debug =  1;
 #endif
 
 static struct radix_node *
-rn_newpair(v, b, nodes)
-	void *v;
-	int b;
-	struct radix_node nodes[2];
+rn_newpair(void *v, int b, struct radix_node nodes[2])
 {
-	register struct radix_node *tt = nodes, *t = tt + 1;
+	struct radix_node *tt = nodes, *t = tt + 1;
 	t->rn_bit = b;
 	t->rn_bmask = 0x80 >> (b & 7);
 	t->rn_left = tt;
@@ -373,7 +364,7 @@ rn_newpair(v, b, nodes)
 	tt->rn_key = (caddr_t)v;
 	tt->rn_parent = t;
 	tt->rn_flags = t->rn_flags = RNF_ACTIVE;
-	tt->rn_mklist = t->rn_mklist = 0;
+	tt->rn_mklist = t->rn_mklist = NULL;
 #ifdef RN_DEBUG
 	tt->rn_info = rn_nodenum++; t->rn_info = rn_nodenum++;
 	tt->rn_twin = t;
@@ -384,25 +375,22 @@ rn_newpair(v, b, nodes)
 }
 
 static struct radix_node *
-rn_insert(v_arg, head, dupentry, nodes)
-	void *v_arg;
-	struct radix_node_head *head;
-	int *dupentry;
-	struct radix_node nodes[2];
+rn_insert(void *v_arg, struct radix_node_head *head, int *dupentry,
+	  struct radix_node nodes[2])
 {
 	caddr_t v = v_arg;
 	struct radix_node *top = head->rnh_treetop;
 	int head_off = top->rn_offset, vlen = (int)*((u_char *)v);
-	register struct radix_node *t = rn_search(v_arg, top);
-	register caddr_t cp = v + head_off;
-	register int b;
+	struct radix_node *t = rn_search(v_arg, top);
+	caddr_t cp = v + head_off;
+	int b;
 	struct radix_node *tt;
     	/*
 	 * Find first bit at which v and t->rn_key differ
 	 */
     {
-	register caddr_t cp2 = t->rn_key + head_off;
-	register int cmp_res;
+	caddr_t cp2 = t->rn_key + head_off;
+	int cmp_res;
 	caddr_t cplim = v + vlen;
 
 	while (cp < cplim)
@@ -417,7 +405,7 @@ on1:
 		cmp_res >>= 1;
     }
     {
-	register struct radix_node *p, *x = top;
+	struct radix_node *p, *x = top;
 	cp = v;
 	do {
 		p = x;
@@ -454,14 +442,12 @@ on1:
 }
 
 struct radix_node *
-rn_addmask(n_arg, search, skip)
-	int search, skip;
-	void *n_arg;
+rn_addmask(void *n_arg, int search, int skip)
 {
 	caddr_t netmask = (caddr_t)n_arg;
-	register struct radix_node *x;
-	register caddr_t cp, cplim;
-	register int b = 0, mlen, j;
+	struct radix_node *x;
+	caddr_t cp, cplim;
+	int b = 0, mlen, j;
 	int maskduplicated, m0, isnormal;
 	struct radix_node *saved_x;
 	static int last_zeroed = 0;
@@ -492,12 +478,12 @@ rn_addmask(n_arg, search, skip)
 	*addmask_key = last_zeroed = mlen;
 	x = rn_search(addmask_key, rn_masktop);
 	if (Bcmp(addmask_key, x->rn_key, mlen) != 0)
-		x = 0;
+		x = NULL;
 	if (x || search)
 		return (x);
 	R_Malloc(x, struct radix_node *, max_keylen + 2 * sizeof (*x));
 	if ((saved_x = x) == 0)
-		return (0);
+		return (NULL);
 	Bzero(x, max_keylen + 2 * sizeof (*x));
 	netmask = cp = (caddr_t)(x + 2);
 	Bcopy(addmask_key, cp, mlen);
@@ -528,10 +514,9 @@ rn_addmask(n_arg, search, skip)
 }
 
 static int	/* XXX: arbitrary ordering for non-contiguous masks */
-rn_lexobetter(m_arg, n_arg)
-	void *m_arg, *n_arg;
+rn_lexobetter(void *m_arg, void *n_arg)
 {
-	register u_char *mp = m_arg, *np = n_arg, *lim;
+	u_char *mp = m_arg, *np = n_arg, *lim;
 
 	if (*mp > *np)
 		return 1;  /* not really, but need to check longer one first */
@@ -543,16 +528,14 @@ rn_lexobetter(m_arg, n_arg)
 }
 
 static struct radix_mask *
-rn_new_radix_mask(tt, next)
-	register struct radix_node *tt;
-	register struct radix_mask *next;
+rn_new_radix_mask(struct radix_node *tt, struct radix_mask *next)
 {
-	register struct radix_mask *m;
+	struct radix_mask *m;
 
 	MKGet(m);
 	if (m == 0) {
 		log(LOG_ERR, "Mask for route not entered\n");
-		return (0);
+		return (NULL);
 	}
 	Bzero(m, sizeof *m);
 	m->rm_bit = tt->rn_bit;
@@ -567,13 +550,11 @@ rn_new_radix_mask(tt, next)
 }
 
 struct radix_node *
-rn_addroute(v_arg, n_arg, head, treenodes)
-	void *v_arg, *n_arg;
-	struct radix_node_head *head;
-	struct radix_node treenodes[2];
+rn_addroute(void *v_arg, void *n_arg, struct radix_node_head *head,
+	    struct radix_node treenodes[2])
 {
 	caddr_t v = (caddr_t)v_arg, netmask = (caddr_t)n_arg;
-	register struct radix_node *t, *x = 0, *tt;
+	struct radix_node *t, *x = NULL, *tt;
 	struct radix_node *saved_tt, *top = head->rnh_treetop;
 	short b = 0, b_leaf = 0;
 	int keyduplicated;
@@ -589,7 +570,7 @@ rn_addroute(v_arg, n_arg, head, treenodes)
 	 */
 	if (netmask)  {
 		if ((x = rn_addmask(netmask, 0, top->rn_offset)) == 0)
-			return (0);
+			return (NULL);
 		b_leaf = x->rn_bit;
 		b = -1 - x->rn_bit;
 		netmask = x->rn_key;
@@ -601,7 +582,7 @@ rn_addroute(v_arg, n_arg, head, treenodes)
 	if (keyduplicated) {
 		for (t = tt; tt; t = tt, tt = tt->rn_dupedkey) {
 			if (tt->rn_mask == netmask)
-				return (0);
+				return (NULL);
 			if (netmask == 0 ||
 			    (tt->rn_mask &&
 			     ((b_leaf < tt->rn_bit) /* index(netmask) > node */
@@ -667,7 +648,7 @@ rn_addroute(v_arg, n_arg, head, treenodes)
 	if (x->rn_bit < 0) {
 	    for (mp = &t->rn_mklist; x; x = x->rn_dupedkey)
 		if (x->rn_mask && (x->rn_bit >= b_leaf) && x->rn_mklist == 0) {
-			*mp = m = rn_new_radix_mask(x, 0);
+			*mp = m = rn_new_radix_mask(x, NULL);
 			if (m)
 				mp = &m->rm_mklist;
 		}
@@ -678,7 +659,7 @@ rn_addroute(v_arg, n_arg, head, treenodes)
 		for (mp = &x->rn_mklist; (m = *mp); mp = &m->rm_mklist)
 			if (m->rm_bit >= b_leaf)
 				break;
-		t->rn_mklist = m; *mp = 0;
+		t->rn_mklist = m; *mp = NULL;
 	}
 on2:
 	/* Add new route to highest possible ancestor's list */
@@ -723,11 +704,9 @@ on2:
 }
 
 struct radix_node *
-rn_delete(v_arg, netmask_arg, head)
-	void *v_arg, *netmask_arg;
-	struct radix_node_head *head;
+rn_delete(void *v_arg, void *netmask_arg, struct radix_node_head *head)
 {
-	register struct radix_node *t, *p, *x, *tt;
+	struct radix_node *t, *p, *x, *tt;
 	struct radix_mask *m, *saved_m, **mp;
 	struct radix_node *dupedkey, *saved_tt, *top;
 	caddr_t v, netmask;
@@ -743,24 +722,24 @@ rn_delete(v_arg, netmask_arg, head)
 	top = x;
 	if (tt == 0 ||
 	    Bcmp(v + head_off, tt->rn_key + head_off, vlen - head_off))
-		return (0);
+		return (NULL);
 	/*
 	 * Delete our route from mask lists.
 	 */
 	if (netmask) {
 		if ((x = rn_addmask(netmask, 1, head_off)) == 0)
-			return (0);
+			return (NULL);
 		netmask = x->rn_key;
 		while (tt->rn_mask != netmask)
 			if ((tt = tt->rn_dupedkey) == 0)
-				return (0);
+				return (NULL);
 	}
 	if (tt->rn_mask == 0 || (saved_m = m = tt->rn_mklist) == 0)
 		goto on1;
 	if (tt->rn_flags & RNF_NORMAL) {
 		if (m->rm_leaf != tt || m->rm_refs > 0) {
 			log(LOG_ERR, "rn_delete: inconsistent annotation\n");
-			return 0;  /* dangling ref could cause disaster */
+			return NULL;  /* dangling ref could cause disaster */
 		}
 	} else {
 		if (m->rm_mask != tt->rn_mask) {
@@ -787,14 +766,14 @@ rn_delete(v_arg, netmask_arg, head)
 	if (m == 0) {
 		log(LOG_ERR, "rn_delete: couldn't find our annotation\n");
 		if (tt->rn_flags & RNF_NORMAL)
-			return (0); /* Dangling ref to us */
+			return (NULL); /* Dangling ref to us */
 	}
 on1:
 	/*
 	 * Eliminate us from tree
 	 */
 	if (tt->rn_flags & RNF_ROOT)
-		return (0);
+		return (NULL);
 	head->rnh_cnt--;
 #ifdef RN_DEBUG
 	/* Get us out of the creation list */
@@ -871,7 +850,7 @@ on1:
 			for (m = t->rn_mklist; m && x; x = x->rn_dupedkey)
 				if (m == x->rn_mklist) {
 					struct radix_mask *mm = m->rm_mklist;
-					x->rn_mklist = 0;
+					x->rn_mklist = NULL;
 					if (--(m->rm_refs) < 0)
 						MKFree(m);
 					m = mm;
@@ -913,11 +892,8 @@ out:
  * exit.
  */
 static int
-rn_walktree_from(h, a, m, f, w)
-	struct radix_node_head *h;
-	void *a, *m;
-	walktree_f_t *f;
-	void *w;
+rn_walktree_from(struct radix_node_head *h, void *a, void *m, walktree_f_t *f,
+    void *w)
 {
 	int error;
 	struct radix_node *base, *next;
@@ -991,6 +967,34 @@ restart:
 			}
 		}
 
+		/*
+		 * The following code (bug fix) inherited from FreeBSD is
+		 * currently disabled, because our implementation uses the
+		 * RTF_PRCLONING scheme that has been abandoned in current
+		 * FreeBSD release.  The scheme involves setting such a flag
+		 * for the default route entry, and therefore all off-link
+		 * destinations would become clones of that entry.  Enabling
+		 * the following code would be problematic at this point,
+		 * because the removal of default route would cause only
+		 * the left-half of the tree to be traversed, leaving the
+		 * right-half untouched.  If there are clones of the entry
+		 * that reside in that right-half, they would not be deleted
+		 * and would linger around until they expire or explicitly
+		 * deleted, which is a very bad thing.
+		 *
+		 * This code should be uncommented only after we get rid
+		 * of the RTF_PRCLONING scheme.
+		 */
+#if 0
+		/*
+		 * At the top of the tree, no need to traverse the right
+		 * half, prevent the traversal of the entire tree in the
+		 * case of default route.
+		 */
+		if (rn->rn_parent->rn_flags & RNF_ROOT)
+			stopping = 1;
+#endif
+
 		/* Find the next *leaf* to start from */
 		for (rn = rn->rn_parent->rn_right; rn->rn_bit >= 0;)
 			rn = rn->rn_left;
@@ -1013,10 +1017,7 @@ restart:
 }
 
 static int
-rn_walktree(h, f, w)
-	struct radix_node_head *h;
-	walktree_f_t *f;
-	void *w;
+rn_walktree(struct radix_node_head *h, walktree_f_t *f, void *w)
 {
 	int error;
 	struct radix_node *base, *next;
@@ -1069,12 +1070,10 @@ restart:
 }
 
 int
-rn_inithead(head, off)
-	void **head;
-	int off;
+rn_inithead(void **head, int off)
 {
-	register struct radix_node_head *rnh;
-	register struct radix_node *t, *tt, *ttt;
+	struct radix_node_head *rnh;
+	struct radix_node *t, *tt, *ttt;
 	if (*head)
 		return (1);
 	R_Malloc(rnh, struct radix_node_head *, sizeof (*rnh));
@@ -1103,7 +1102,7 @@ rn_inithead(head, off)
 }
 
 void
-rn_init()
+rn_init(void)
 {
 	char *cp, *cplim;
 #ifdef KERNEL
@@ -1132,38 +1131,3 @@ rn_init()
 
 	rn_mutex = lck_mtx_alloc_init(domain_proto_mtx_grp, domain_proto_mtx_attr);
 }
-int
-rn_lock(so, refcount, lr)
-	struct socket *so;
-	int refcount;
-	int lr;
-{
-//	printf("rn_lock: (global) so=%x ref=%d lr=%x\n", so, so->so_usecount, lr);
-	lck_mtx_assert(rn_mutex, LCK_MTX_ASSERT_NOTOWNED);
-	lck_mtx_lock(rn_mutex);
-	if (refcount)
-		so->so_usecount++;
-	return (0);
-}
-
-int
-rn_unlock(so, refcount, lr)
-	struct socket *so;
-	int refcount;
-	int lr;
-{
-//	printf("rn_unlock: (global) so=%x ref=%d lr=%x\n", so, so->so_usecount, lr);
-	if (refcount)
-		so->so_usecount--;
-	lck_mtx_assert(rn_mutex, LCK_MTX_ASSERT_OWNED);
-	lck_mtx_unlock(rn_mutex);
-	return (0);
-}
-lck_mtx_t *
-rn_getlock(so, locktype)
-	struct socket *so;
-	int locktype;
-{
-//	printf("rn_getlock: (global) so=%x\n", so);
-	return (rn_mutex);
-}
diff --git a/bsd/net/radix.h b/bsd/net/radix.h
index 088fd38d2..3431a2d25 100644
--- a/bsd/net/radix.h
+++ b/bsd/net/radix.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1989, 1993
@@ -180,7 +186,7 @@ struct radix_node_head {
 #define R_Free(p) FREE((caddr_t)p, M_RTABLE);
 #endif /*KERNEL*/
 
-void	 rn_init(void);
+void	 rn_init(void) __attribute__((section("__TEXT, initcode")));
 int	 rn_inithead(void **, int);
 int	 rn_refines(void *, void *);
 struct radix_node
diff --git a/bsd/net/raw_cb.c b/bsd/net/raw_cb.c
index cba291fe0..dee6205f0 100644
--- a/bsd/net/raw_cb.c
+++ b/bsd/net/raw_cb.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -84,11 +90,9 @@ extern lck_mtx_t 	*raw_mtx;	/*### global raw cb mutex for now */
  * of buffer space for the socket.
  */
 int
-raw_attach(so, proto)
-	register struct socket *so;
-	int proto;
+raw_attach(struct socket *so, int proto)
 {
-	register struct rawcb *rp = sotorawcb(so);
+	struct rawcb *rp = sotorawcb(so);
 	int error;
 
 	/*
@@ -115,8 +119,7 @@ raw_attach(so, proto)
  * socket resources.
  */
 void
-raw_detach(rp)
-	register struct rawcb *rp;
+raw_detach(struct rawcb *rp)
 {
 	struct socket *so = rp->rcb_socket;
 
@@ -143,10 +146,8 @@ raw_detach(rp)
  * Disconnect and possibly release resources.
  */
 void
-raw_disconnect(rp)
-	struct rawcb *rp;
+raw_disconnect(struct rawcb *rp)
 {
-
 #ifdef notdef
 	if (rp->rcb_faddr)
 		m_freem(dtom(rp->rcb_faddr));
@@ -160,12 +161,10 @@ raw_disconnect(rp)
 #include <sys/mbuf.h>
 
 int
-raw_bind(so, nam)
-	register struct socket *so;
-	struct mbuf *nam;
+raw_bind(struct socket *so, struct mbuf *nam)
 {
 	struct sockaddr *addr = mtod(nam, struct sockaddr *);
-	register struct rawcb *rp;
+	struct rawcb *rp;
 
 	if (ifnet == 0)
 		return (EADDRNOTAVAIL);
diff --git a/bsd/net/raw_cb.h b/bsd/net/raw_cb.h
index 478cdd571..2ad95f527 100644
--- a/bsd/net/raw_cb.h
+++ b/bsd/net/raw_cb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1993
diff --git a/bsd/net/raw_usrreq.c b/bsd/net/raw_usrreq.c
index 20d017a26..4128af9f5 100644
--- a/bsd/net/raw_usrreq.c
+++ b/bsd/net/raw_usrreq.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -75,7 +81,7 @@ lck_grp_attr_t 	*raw_mtx_grp_attr;
  * Initialize raw connection block q.
  */
 void
-raw_init()
+raw_init(void)
 {
 	raw_mtx_grp_attr = lck_grp_attr_alloc_init();
 
@@ -100,14 +106,12 @@ raw_init()
  * Raw protocol interface.
  */
 void
-raw_input(m0, proto, src, dst)
-	struct mbuf *m0;
-	register struct sockproto *proto;
-	struct sockaddr *src, *dst;
+raw_input(struct mbuf *m0, struct sockproto *proto, struct sockaddr *src,
+	  struct sockaddr *dst)
 {
-	register struct rawcb *rp;
-	register struct mbuf *m = m0;
-	register int sockets = 0;
+	struct rawcb *rp;
+	struct mbuf *m = m0;
+	int sockets = 0;
 	struct socket *last;
 	int error;
 
@@ -115,7 +119,7 @@ raw_input(m0, proto, src, dst)
 //####LD socket we'll find and need to append to is unlocked.
 //####LD calls from the output (locked) path need to make sure the socket is not locked when
 //####LD we call in raw_input
-	last = 0;
+	last = NULL;
 	lck_mtx_lock(raw_mtx);
 	LIST_FOREACH(rp, &rawcb_list, list) {
 		if (rp->rcb_proto.sp_family != proto->sp_family)
@@ -167,10 +171,7 @@ raw_input(m0, proto, src, dst)
 
 /*ARGSUSED*/
 void
-raw_ctlinput(cmd, arg, dummy)
-	int cmd;
-	struct sockaddr *arg;
-	void *dummy;
+raw_ctlinput(int cmd, __unused struct sockaddr *arg, __unused void *dummy)
 {
 
 	if (cmd < 0 || cmd > PRC_NCMDS)
@@ -201,33 +202,25 @@ raw_uabort(struct socket *so)
 /* pru_accept is EOPNOTSUPP */
 
 static int
-raw_uattach(struct socket *so, int proto, struct proc *p)
+raw_uattach(struct socket *so, int proto, __unused struct proc *p)
 {
 	struct rawcb *rp = sotorawcb(so);
-#ifndef __APPLE__
-	int error;
-#endif
 
 	if (rp == 0)
 		return EINVAL;
-#ifdef __APPLE__
 		if ((so->so_state & SS_PRIV) == 0)
 			return (EPERM);
-#else
-	if (p && (error = suser(p)) != 0)
-		return error;
-#endif
 	return raw_attach(so, proto);
 }
 
 static int
-raw_ubind(struct socket *so, struct sockaddr *nam, struct proc *p)
+raw_ubind(__unused struct socket *so, __unused struct sockaddr *nam, __unused struct proc *p)
 {
 	return EINVAL;
 }
 
 static int
-raw_uconnect(struct socket *so, struct sockaddr *nam, struct proc *p)
+raw_uconnect(__unused struct socket *so, __unused struct sockaddr *nam, __unused struct proc *p)
 {
 	return EINVAL;
 }
@@ -289,7 +282,7 @@ raw_upeeraddr(struct socket *so, struct sockaddr **nam)
 
 static int
 raw_usend(struct socket *so, int flags, struct mbuf *m,
-	  struct sockaddr *nam, struct mbuf *control, struct proc *p)
+	  struct sockaddr *nam, struct mbuf *control, __unused struct proc *p)
 {
 	int error;
 	struct rawcb *rp = sotorawcb(so);
@@ -328,7 +321,7 @@ raw_usend(struct socket *so, int flags, struct mbuf *m,
 	error = (*so->so_proto->pr_output)(m, so);
 	m = NULL;
 	if (nam)
-		rp->rcb_faddr = 0;
+		rp->rcb_faddr = NULL;
 release:
 	if (m != NULL)
 		m_freem(m);
diff --git a/bsd/net/route.c b/bsd/net/route.c
index c05c0c069..ffd62033f 100644
--- a/bsd/net/route.c
+++ b/bsd/net/route.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1991, 1993
@@ -62,6 +68,7 @@
 #include <sys/socket.h>
 #include <sys/domain.h>
 #include <sys/syslog.h>
+#include <sys/queue.h>
 #include <kern/lock.h>
 #include <kern/zalloc.h>
 
@@ -73,8 +80,16 @@
 
 #include <net/if_dl.h>
 
+#include <libkern/OSAtomic.h>
+#include <libkern/OSDebug.h>
+
+#include <pexpert/pexpert.h>
+
+#define	equal(a1, a2) (bcmp((caddr_t)(a1), (caddr_t)(a2), (a1)->sa_len) == 0)
 #define	SA(p) ((struct sockaddr *)(p))
 
+extern void kdp_set_gateway_mac (void *gatewaymac);
+
 extern struct domain routedomain;
 struct route_cb route_cb;
 __private_extern__ struct rtstat rtstat  = { 0, 0, 0, 0, 0 };
@@ -86,25 +101,78 @@ lck_grp_t 	*rt_mtx_grp;
 lck_grp_attr_t 	*rt_mtx_grp_attr;
 
 lck_mtx_t 	*route_domain_mtx;	/*### global routing tables mutex for now */
-__private_extern__ int	rttrash = 0;		/* routes not in table but not freed */
+int rttrash = 0;		/* routes not in table but not freed */
+
+static unsigned int rte_debug;
+
+/* Possible flags for rte_debug */
+#define	RTD_DEBUG	0x1	/* enable or disable rtentry debug facility */
+#define	RTD_TRACE	0x2	/* trace alloc, free and refcnt */
+#define	RTD_NO_FREE	0x4	/* don't free (good to catch corruptions) */
 
 static struct zone *rte_zone;			/* special zone for rtentry */
 #define	RTE_ZONE_MAX		65536		/* maximum elements in zone */
 #define	RTE_ZONE_NAME		"rtentry"	/* name of rtentry zone */
 
+#define	RTD_INUSE		0xFEEDFACE	/* entry is in use */
+#define	RTD_FREED		0xDEADBEEF	/* entry is freed */
+
+#define	RTD_TRSTACK_SIZE	8		/* depth of stack trace */
+#define	RTD_REFHIST_SIZE	4		/* refcnt history size */
+
+/*
+ * Debug variant of rtentry structure.
+ */
+struct rtentry_dbg {
+	struct rtentry	rtd_entry;			/* rtentry */
+	struct rtentry	rtd_entry_saved;		/* saved rtentry */
+	u_int32_t	rtd_inuse;			/* in use pattern */
+	u_int16_t	rtd_refhold_cnt;		/* # of rtref */
+	u_int16_t	rtd_refrele_cnt;		/* # of rtunref */
+	/*
+	 * Thread and PC stack trace up to RTD_TRSTACK_SIZE
+	 * deep during alloc and free.
+	 */
+	struct thread	*rtd_alloc_thread;
+	void		*rtd_alloc_stk_pc[RTD_TRSTACK_SIZE];
+	struct thread	*rtd_free_thread;
+	void		*rtd_free_stk_pc[RTD_TRSTACK_SIZE];
+	/*
+	 * Circular lists of rtref and rtunref callers.
+	 */
+	u_int16_t	rtd_refhold_next;
+	u_int16_t	rtd_refrele_next;
+	struct {
+		struct thread *th;
+		void *pc[RTD_TRSTACK_SIZE];
+	} rtd_refhold[RTD_REFHIST_SIZE];
+	struct {
+		struct thread *th;
+		void *pc[RTD_TRSTACK_SIZE];
+	} rtd_refrele[RTD_REFHIST_SIZE];
+	/*
+	 * Trash list linkage
+	 */
+	TAILQ_ENTRY(rtentry_dbg) rtd_trash_link;
+};
+
+/* List of trash route entries protected by rt_mtx */
+static TAILQ_HEAD(, rtentry_dbg) rttrash_head;
+
+static inline struct rtentry *rte_alloc_debug(void);
+static inline void rte_free_debug(struct rtentry *);
 static void rt_maskedcopy(struct sockaddr *,
 	    struct sockaddr *, struct sockaddr *);
 static void rtable_init(void **);
-static struct rtentry *rte_alloc(void);
-static void rte_free(struct rtentry *);
+static inline void rtref_audit(struct rtentry_dbg *);
+static inline void rtunref_audit(struct rtentry_dbg *);
 
 __private_extern__ u_long route_generation = 0;
 extern int use_routegenid;
 
 
 static void
-rtable_init(table)
-	void **table;
+rtable_init(void **table)
 {
 	struct domain *dom;
 	for (dom = domains; dom; dom = dom->dom_next)
@@ -114,8 +182,14 @@ rtable_init(table)
 }
 
 void
-route_init()
+route_init(void)
 {
+	int size;
+
+	PE_parse_boot_arg("rte_debug", &rte_debug);
+	if (rte_debug != 0)
+		rte_debug |= RTD_DEBUG;
+
 	rt_mtx_grp_attr = lck_grp_attr_alloc_init();
 
 	rt_mtx_grp = lck_grp_alloc_init("route", rt_mtx_grp_attr);
@@ -133,28 +207,31 @@ route_init()
 	rtable_init((void **)rt_tables);
 	route_domain_mtx = routedomain.dom_mtx;
 
-	rte_zone = zinit(sizeof (struct rtentry),
-	    RTE_ZONE_MAX * sizeof (struct rtentry), 0, RTE_ZONE_NAME);
+	if (rte_debug & RTD_DEBUG)
+		size = sizeof (struct rtentry_dbg);
+	else
+		size = sizeof (struct rtentry);
+
+	rte_zone = zinit(size, RTE_ZONE_MAX * size, 0, RTE_ZONE_NAME);
 	if (rte_zone == NULL)
 		panic("route_init: failed allocating rte_zone");
 
 	zone_change(rte_zone, Z_EXPAND, TRUE);
+
+	TAILQ_INIT(&rttrash_head);
 }
 
 /*
  * Packet routing routines.
  */
 void
-rtalloc(ro)
-	register struct route *ro;
+rtalloc(struct route *ro)
 {
 	rtalloc_ign(ro, 0UL);
 }
 
 void
-rtalloc_ign_locked(ro, ignore)
-	register struct route *ro;
-	u_long ignore;
+rtalloc_ign_locked(struct route *ro, u_long ignore)
 {
 	struct rtentry *rt;
 
@@ -170,9 +247,7 @@ rtalloc_ign_locked(ro, ignore)
 		ro->ro_rt->generation_id = route_generation;
 }
 void
-rtalloc_ign(ro, ignore)
-	register struct route *ro;
-	u_long ignore;
+rtalloc_ign(struct route *ro, u_long ignore)
 {
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(rt_mtx);
@@ -185,14 +260,11 @@ rtalloc_ign(ro, ignore)
  * Or, at least try.. Create a cloned route if needed.
  */
 struct rtentry *
-rtalloc1_locked(dst, report, ignflags)
-	const struct sockaddr *dst;
-	int report;
-	u_long ignflags;
+rtalloc1_locked(struct sockaddr *dst, int report, u_long ignflags)
 {
-	register struct radix_node_head *rnh = rt_tables[dst->sa_family];
-	register struct rtentry *rt;
-	register struct radix_node *rn;
+	struct radix_node_head *rnh = rt_tables[dst->sa_family];
+	struct rtentry *rt;
+	struct radix_node *rn;
 	struct rtentry *newrt = 0;
 	struct rt_addrinfo info;
 	u_long nflags;
@@ -257,10 +329,7 @@ rtalloc1_locked(dst, report, ignflags)
 }
 
 struct rtentry *
-rtalloc1(dst, report, ignflags)
-	register struct sockaddr *dst;
-	int report;
-	u_long ignflags;
+rtalloc1(struct sockaddr *dst, int report, u_long ignflags)
 {
 	struct rtentry * entry;
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_NOTOWNED);
@@ -275,14 +344,13 @@ rtalloc1(dst, report, ignflags)
  * If the count gets low enough, take it out of the routing table
  */
 void
-rtfree_locked(rt)
-	register struct rtentry *rt;
+rtfree_locked(struct rtentry *rt)
 {
 	/*
 	 * find the tree for that address family
 	 * Note: in the case of igmp packets, there might not be an rnh
 	 */
-	register struct radix_node_head *rnh;
+	struct radix_node_head *rnh;
 
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
@@ -298,28 +366,47 @@ rtfree_locked(rt)
 	 * decrement the reference count by one and if it reaches 0,
 	 * and there is a close function defined, call the close function
 	 */
-	rt->rt_refcnt--;
-	if(rnh && rnh->rnh_close && rt->rt_refcnt == 0) {
+	rtunref(rt);
+	if (rt->rt_refcnt > 0)
+		return;
+
+	if ((rt->rt_flags & RTF_TRACKREFS) != 0)
+		printf("%s rt(%p)->rt_refcnt(%d), caller=%p\n", __FUNCTION__,
+			rt, rt->rt_refcnt, __builtin_return_address(0));
+	
+	/*
+	 * On last reference give the "close method" a chance to cleanup
+	 * private state.  This also permits (for IPv4 and IPv6) a chance
+	 * to decide if the routing table entry should be purged immediately
+	 * or at a later time.  When an immediate purge is to happen the
+	 * close routine typically issues RTM_DELETE which clears the RTF_UP
+	 * flag on the entry so that the code below reclaims the storage.
+	 */
+	if (rnh->rnh_close && rt->rt_refcnt == 0)
 		rnh->rnh_close((struct radix_node *)rt, rnh);
-	}
 
 	/*
 	 * If we are no longer "up" (and ref == 0)
 	 * then we can free the resources associated
 	 * with the route.
 	 */
-	if (rt->rt_refcnt <= 0 && (rt->rt_flags & RTF_UP) == 0) {
+	if (!(rt->rt_flags & RTF_UP)) {
 		if (rt->rt_nodes->rn_flags & (RNF_ACTIVE | RNF_ROOT))
 			panic ("rtfree 2");
 		/*
 		 * the rtentry must have been removed from the routing table
 		 * so it is represented in rttrash.. remove that now.
 		 */
-		rttrash--;
+		(void) OSDecrementAtomic((SInt32 *)&rttrash);
+		if (rte_debug & RTD_DEBUG) {
+			TAILQ_REMOVE(&rttrash_head, (struct rtentry_dbg *)rt,
+			    rtd_trash_link);
+		}
 
 #ifdef	DIAGNOSTIC
 		if (rt->rt_refcnt < 0) {
-			printf("rtfree: %p not freed (neg refs) cnt=%d\n", rt, rt->rt_refcnt);
+			printf("rtfree: %p not freed (neg refs) cnt=%d\n",
+			    rt, rt->rt_refcnt);
 			return;
 		}
 #endif
@@ -351,8 +438,7 @@ rtfree_locked(rt)
 }
 
 void
-rtfree(rt)
-	register struct rtentry *rt;
+rtfree(struct rtentry *rt)
 {
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_NOTOWNED);
 	lck_mtx_lock(rt_mtx);	
@@ -366,31 +452,78 @@ rtfree(rt)
  * use rtfree not rtunref.
  */
 void
-rtunref(struct rtentry* rt)
+rtunref(struct rtentry *p)
 {
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
-	if (rt == NULL)
-		panic("rtunref");
-	rt->rt_refcnt--;
-#if DEBUG
-	if (rt->rt_refcnt <= 0 && (rt->rt_flags & RTF_UP) == 0)
-		printf("rtunref - if rtfree were called, we would have freed route\n");
-#endif
+	if (p->rt_refcnt <= 0)
+		panic("rtunref: bad refcnt %d for rt=%p\n", p->rt_refcnt, p);
+
+	if (rte_debug & RTD_DEBUG)
+		rtunref_audit((struct rtentry_dbg *)p);
+
+	p->rt_refcnt--;
+}
+
+static inline void
+rtunref_audit(struct rtentry_dbg *rte)
+{
+	if (rte->rtd_inuse != RTD_INUSE)
+		panic("rtunref: on freed rte=%p\n", rte);
+
+	rte->rtd_refrele_cnt++;
+
+	if (rte_debug & RTD_TRACE) {
+		rte->rtd_refrele[rte->rtd_refrele_next].th = current_thread();
+		bzero(rte->rtd_refrele[rte->rtd_refrele_next].pc,
+		    sizeof (rte->rtd_refrele[rte->rtd_refrele_next].pc));
+		(void) OSBacktrace(rte->rtd_refrele[rte->rtd_refrele_next].pc,
+		    RTD_TRSTACK_SIZE);
+
+		rte->rtd_refrele_next =
+		    (rte->rtd_refrele_next + 1) % RTD_REFHIST_SIZE;
+	}
 }
 
 /*
  * Add a reference count from an rtentry.
  */
 void
-rtref(struct rtentry* rt)
+rtref(struct rtentry *p)
 {
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
-	if (rt == NULL)
-		panic("rtref");
+	if (p->rt_refcnt < 0)
+		panic("rtref: bad refcnt %d for rt=%p\n", p->rt_refcnt, p);
+
+	if (rte_debug & RTD_DEBUG)
+		rtref_audit((struct rtentry_dbg *)p);
+
+	p->rt_refcnt++;
+	
+	if ((p->rt_flags & RTF_TRACKREFS) != 0)
+		printf("%s rt(%p)->rt_refcnt(%d), caller=%p\n", __FUNCTION__,
+			p, p->rt_refcnt, __builtin_return_address(0));
+}
+
+static inline void
+rtref_audit(struct rtentry_dbg *rte)
+{
+	if (rte->rtd_inuse != RTD_INUSE)
+		panic("rtref_audit: on freed rte=%p\n", rte);
 
-	rt->rt_refcnt++;
+	rte->rtd_refhold_cnt++;
+
+	if (rte_debug & RTD_TRACE) {
+		rte->rtd_refhold[rte->rtd_refhold_next].th = current_thread();
+		bzero(rte->rtd_refhold[rte->rtd_refhold_next].pc,
+		    sizeof (rte->rtd_refhold[rte->rtd_refhold_next].pc));
+		(void) OSBacktrace(rte->rtd_refhold[rte->rtd_refhold_next].pc,
+		    RTD_TRSTACK_SIZE);
+
+		rte->rtd_refhold_next =
+		    (rte->rtd_refhold_next + 1) % RTD_REFHIST_SIZE;
+	}
 }
 
 void
@@ -415,19 +548,17 @@ rtsetifa(struct rtentry *rt, struct ifaddr* ifa)
 }
 
 void
-ifafree(ifa)
-	register struct ifaddr *ifa;
+ifafree(struct ifaddr *ifa)
 {
-	int i, oldval;
-	u_char *ptr = (u_char*)ifa;
-	
+	int oldval;
+
 	if (ifa == NULL)
 		panic("ifafree");
-	
-	oldval = OSAddAtomic(-1, &ifa->ifa_refcnt);
+
+	oldval = OSAddAtomic(-1, (SInt32 *)&ifa->ifa_refcnt);
 
 	if (oldval == 0) {
-		if  ((ifa->ifa_flags & IFA_ATTACHED) != 0) {
+		if  ((ifa->ifa_debug & IFA_ATTACHED) != 0) {
 			panic("ifa attached to ifp is being freed\n");
 		}
 		FREE(ifa, M_IFADDR);
@@ -440,7 +571,7 @@ ifaref(struct ifaddr *ifa)
 	if (ifa == NULL)
 		panic("ifaref");
 
-	if (OSAddAtomic(1, &ifa->ifa_refcnt) == 0xffffffff)
+	if (OSAddAtomic(1, (SInt32 *)&ifa->ifa_refcnt) == 0xffffffff)
 		panic("ifaref - reference count rolled over!");
 }
 
@@ -454,12 +585,11 @@ ifaref(struct ifaddr *ifa)
  *
  */
 void
-rtredirect(dst, gateway, netmask, flags, src, rtp)
-	struct sockaddr *dst, *gateway, *netmask, *src;
-	int flags;
-	struct rtentry **rtp;
+rtredirect(struct sockaddr *dst, struct sockaddr *gateway,
+	   struct sockaddr *netmask, int flags, struct sockaddr *src,
+	   struct rtentry **rtp)
 {
-	register struct rtentry *rt;
+	struct rtentry *rt;
 	int error = 0;
 	short *stat = 0;
 	struct rt_addrinfo info;
@@ -474,18 +604,18 @@ rtredirect(dst, gateway, netmask, flags, src, rtp)
 		goto out;
 	}
 
-	rt = rtalloc1_locked(dst, 0, 0UL);
+	rt = rtalloc1_locked(dst, 0, RTF_CLONING | RTF_PRCLONING);
 	/*
 	 * If the redirect isn't from our current router for this dst,
 	 * it's either old or wrong.  If it redirects us to ourselves,
 	 * we have a routing loop, perhaps as a result of an interface
 	 * going down recently.
 	 */
-#define	equal(a1, a2) (bcmp((caddr_t)(a1), (caddr_t)(a2), (a1)->sa_len) == 0)
 	if (!(flags & RTF_DONE) && rt &&
-	     (!equal(src, rt->rt_gateway) || rt->rt_ifa != ifa))
+	     (!equal(src, rt->rt_gateway) || !equal(rt->rt_ifa->ifa_addr,
+	     ifa->ifa_addr))) {
 		error = EINVAL;
-	else {
+	} else {
 		ifafree(ifa);
 		if ((ifa = ifa_ifwithaddr(gateway))) {
 			ifafree(ifa);
@@ -565,21 +695,14 @@ out:
 * Routing table ioctl interface.
 */
 int
-rtioctl(req, data, p)
-	int req;
-	caddr_t data;
-	struct proc *p;
+rtioctl(int req, caddr_t data, struct proc *p)
 {
-#if INET
-	/* Multicast goop, grrr... */
-#if MROUTING
+#pragma unused(p)
+#if INET && MROUTING
 	return mrt_ioctl(req, data);
 #else
-	return mrt_ioctl(req, data, p);
-#endif
-#else /* INET */
 	return ENXIO;
-#endif /* INET */
+#endif
 }
 
 struct ifaddr *
@@ -588,11 +711,27 @@ ifa_ifwithroute(
 	const struct sockaddr	*dst,
 	const struct sockaddr *gateway)
 {
+	struct ifaddr *ifa;
+
+	lck_mtx_lock(rt_mtx);
+	ifa = ifa_ifwithroute_locked(flags, dst, gateway);
+	lck_mtx_unlock(rt_mtx);
+
+	return (ifa);
+}
+
+struct ifaddr *
+ifa_ifwithroute_locked(
+	int flags,
+	const struct sockaddr *dst,
+	const struct sockaddr *gateway)
+{
+	struct ifaddr *ifa = NULL;
+	struct rtentry *rt = NULL;
 
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
-	struct ifaddr *ifa = 0;
-	if ((flags & RTF_GATEWAY) == 0) {
+	if (!(flags & RTF_GATEWAY)) {
 		/*
 		 * If we are adding a route to an interface,
 		 * and the interface is a pt to pt link
@@ -603,7 +742,7 @@ ifa_ifwithroute(
 		if (flags & RTF_HOST) {
 			ifa = ifa_ifwithdstaddr(dst);
 		}
-		if (ifa == 0)
+		if (ifa == NULL)
 			ifa = ifa_ifwithaddr(gateway);
 	} else {
 		/*
@@ -613,27 +752,47 @@ ifa_ifwithroute(
 		 */
 		ifa = ifa_ifwithdstaddr(gateway);
 	}
-	if (ifa == 0)
+	if (ifa == NULL)
 		ifa = ifa_ifwithnet(gateway);
-	if (ifa == 0) {
-		struct rtentry *rt = rtalloc1_locked(dst, 0, 0UL);
-		if (rt == 0)
-			return (0);
-		ifa = rt->rt_ifa;
-		if (ifa)
-			ifaref(ifa);
-		rtunref(rt);
-		if (ifa == 0)
-			return 0;
+	if (ifa == NULL) {
+		/* Workaround to avoid gcc warning regarding const variable */
+		rt = rtalloc1_locked((struct sockaddr *)(size_t)dst, 0, 0UL);
+		if (rt != NULL) {
+			ifa = rt->rt_ifa;
+			if (ifa != NULL)
+				ifaref(ifa);
+			rtunref(rt);
+			rt = NULL;
+		}
 	}
-	if (ifa->ifa_addr->sa_family != dst->sa_family) {
+	if (ifa != NULL && ifa->ifa_addr->sa_family != dst->sa_family) {
 		struct ifaddr *newifa;
+		/* Callee adds reference to newifa upon success */
 		newifa = ifaof_ifpforaddr(dst, ifa->ifa_ifp);
-		if (newifa != 0) {
+		if (newifa != NULL) {
 			ifafree(ifa);
 			ifa = newifa;
 		}
 	}
+	/*
+	 * If we are adding a gateway, it is quite possible that the
+	 * routing table has a static entry in place for the gateway,
+	 * that may not agree with info garnered from the interfaces.
+	 * The routing table should carry more precedence than the
+	 * interfaces in this matter.  Must be careful not to stomp
+	 * on new entries from rtinit, hence (ifa->ifa_addr != gateway).
+	 */
+	if ((ifa == NULL ||
+	    !equal(ifa->ifa_addr, (struct sockaddr *)(size_t)gateway)) &&
+	    (rt = rtalloc1_locked((struct sockaddr *)(size_t)gateway,
+	    0, 0UL)) != NULL) {
+		if (ifa != NULL)
+			ifafree(ifa);
+		ifa = rt->rt_ifa;
+		if (ifa != NULL)
+			ifaref(ifa);
+		rtunref(rt);
+	}
 	return (ifa);
 }
 
@@ -651,6 +810,7 @@ struct rtfc_arg {
  * Do appropriate manipulations of a routing tree given
  * all the bits of info needed
  */
+int
 rtrequest_locked(
 	int req,
 	struct sockaddr *dst,
@@ -660,9 +820,9 @@ rtrequest_locked(
 	struct rtentry **ret_nrt)
 {
 	int error = 0;
-	register struct rtentry *rt;
-	register struct radix_node *rn;
-	register struct radix_node_head *rnh;
+	struct rtentry *rt;
+	struct radix_node *rn;
+	struct radix_node_head *rnh;
 	struct ifaddr *ifa = NULL;
 	struct sockaddr *ndst;
 #define senderr(x) { error = x ; goto bad; }
@@ -691,6 +851,18 @@ rtrequest_locked(
 			panic ("rtrequest delete");
 		rt = (struct rtentry *)rn;
 
+		/*
+		 * Take an extra reference to handle the deletion of a route
+		 * entry whose reference count is already 0; e.g. an expiring
+		 * cloned route entry or an entry that was added to the table
+		 * with 0 reference. If the caller is interested in this route,
+		 * we will return it with the reference intact. Otherwise we
+		 * will decrement the reference via rtfree_locked() and then
+		 * possibly deallocate it.
+		 */
+		rtref(rt);
+		rt->rt_flags &= ~RTF_UP;
+
 		/*
 		 * Now search what's left of the subtree for any cloned
 		 * routes which might have been formed from this node.
@@ -712,15 +884,6 @@ rtrequest_locked(
 			(rt = (struct rtentry *)rn)->rt_gwroute = 0;
 		}
 
-		/*
-		 * NB: RTF_UP must be set during the search above,
-		 * because we might delete the last ref, causing
-		 * rt to get freed prematurely.
-		 *  eh? then why not just add a reference?
-		 * I'm not sure how RTF_UP helps matters. (JRE)
-		 */
-		rt->rt_flags &= ~RTF_UP;
-
 		/*
 		 * give the protocol a chance to keep things in sync.
 		 */
@@ -732,17 +895,22 @@ rtrequest_locked(
 		 * one more rtentry floating around that is not
 		 * linked to the routing table.
 		 */
-		rttrash++;
+		(void) OSIncrementAtomic((SInt32 *)&rttrash);
+		if (rte_debug & RTD_DEBUG) {
+			TAILQ_INSERT_TAIL(&rttrash_head,
+			    (struct rtentry_dbg *)rt, rtd_trash_link);
+		}
 
 		/*
 		 * If the caller wants it, then it can have it,
 		 * but it's up to it to free the rtentry as we won't be
 		 * doing it.
 		 */
-		if (ret_nrt)
+		if (ret_nrt != NULL) {
+			/* Return the route to caller with reference intact */
 			*ret_nrt = rt;
-		else if (rt->rt_refcnt <= 0) {
-			rt->rt_refcnt++; /* make a 1->0 transition */
+		} else {
+			/* Dereference or deallocate the route */
 			rtfree_locked(rt);
 		}
 		break;
@@ -764,7 +932,7 @@ rtrequest_locked(
 		if ((flags & RTF_GATEWAY) && !gateway)
 			panic("rtrequest: GATEWAY but no gateway");
 
-		if ((ifa = ifa_ifwithroute(flags, dst, gateway)) == 0)
+		if ((ifa = ifa_ifwithroute_locked(flags, dst, gateway)) == 0)
 			senderr(ENETUNREACH);
 
 	makeroute:
@@ -815,7 +983,8 @@ rtrequest_locked(
 			 * mechanism, then we just blow it away and retry
 			 * the insertion of the new one.
 			 */
-			rt2 = rtalloc1_locked(dst, 0, RTF_PRCLONING);
+			rt2 = rtalloc1_locked(dst, 0,
+			    RTF_CLONING | RTF_PRCLONING);
 			if (rt2 && rt2->rt_parent) {
 				rtrequest_locked(RTM_DELETE,
 					  (struct sockaddr *)rt_key(rt2),
@@ -923,16 +1092,15 @@ rtrequest(
  * the late parent (passed in as VP here) are themselves deleted.
  */
 static int
-rt_fixdelete(rn, vp)
-	struct radix_node *rn;
-	void *vp;
+rt_fixdelete(struct radix_node *rn, void *vp)
 {
 	struct rtentry *rt = (struct rtentry *)rn;
 	struct rtentry *rt0 = vp;
 
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
-	if (rt->rt_parent == rt0 && !(rt->rt_flags & RTF_PINNED)) {
+	if (rt->rt_parent == rt0 &&
+	    !(rt->rt_flags & (RTF_PINNED | RTF_CLONING | RTF_PRCLONING))) {
 		return rtrequest_locked(RTM_DELETE, rt_key(rt),
 				 (struct sockaddr *)0, rt_mask(rt),
 				 rt->rt_flags, (struct rtentry **)0);
@@ -958,9 +1126,7 @@ static int rtfcdebug = 0;
 #endif
 
 static int
-rt_fixchange(rn, vp)
-	struct radix_node *rn;
-	void *vp;
+rt_fixchange(struct radix_node *rn, void *vp)
 {
 	struct rtentry *rt = (struct rtentry *)rn;
 	struct rtfc_arg *ap = vp;
@@ -976,7 +1142,8 @@ rt_fixchange(rn, vp)
 
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
-	if (!rt->rt_parent || (rt->rt_flags & RTF_PINNED)) {
+	if (!rt->rt_parent ||
+	    (rt->rt_flags & (RTF_PINNED | RTF_CLONING | RTF_PRCLONING))) {
 #ifdef DEBUG
 		if(rtfcdebug) printf("no parent or pinned\n");
 #endif
@@ -1046,15 +1213,12 @@ rt_fixchange(rn, vp)
 }
 
 int
-rt_setgate(rt0, dst, gate)
-	struct rtentry *rt0;
-	struct sockaddr *dst, *gate;
+rt_setgate(struct rtentry *rt0, struct sockaddr *dst, struct sockaddr *gate)
 {
 	caddr_t new, old;
 	int dlen = ROUNDUP(dst->sa_len), glen = ROUNDUP(gate->sa_len);
-	register struct rtentry *rt = rt0;
+	struct rtentry *rt = rt0;
 	struct radix_node_head *rnh = rt_tables[dst->sa_family];
-	extern void kdp_set_gateway_mac (void *gatewaymac);
 	/*
 	 * A host route with the destination equal to the gateway
 	 * will interfere with keeping LLINFO in the routing
@@ -1161,12 +1325,12 @@ rt_setgate(rt0, dst, gate)
 }
 
 static void
-rt_maskedcopy(src, dst, netmask)
-	struct sockaddr *src, *dst, *netmask;
+rt_maskedcopy(struct sockaddr *src, struct sockaddr *dst,
+	      struct sockaddr *netmask)
 {
-	register u_char *cp1 = (u_char *)src;
-	register u_char *cp2 = (u_char *)dst;
-	register u_char *cp3 = (u_char *)netmask;
+	u_char *cp1 = (u_char *)src;
+	u_char *cp2 = (u_char *)dst;
+	u_char *cp3 = (u_char *)netmask;
 	u_char *cplim = cp2 + *cp3;
 	u_char *cplim2 = cp2 + *cp1;
 
@@ -1185,9 +1349,7 @@ rt_maskedcopy(src, dst, netmask)
  * for an interface.
  */
 int
-rtinit(ifa, cmd, flags)
-	register struct ifaddr *ifa;
-	int cmd, flags;
+rtinit(struct ifaddr *ifa, int cmd, int flags)
 {
 	int error;
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_NOTOWNED);
@@ -1198,13 +1360,11 @@ rtinit(ifa, cmd, flags)
 }
 
 int
-rtinit_locked(ifa, cmd, flags)
-	register struct ifaddr *ifa;
-	int cmd, flags;
+rtinit_locked(struct ifaddr *ifa, int cmd, int flags)
 {
-	register struct rtentry *rt;
-	register struct sockaddr *dst;
-	register struct sockaddr *deldst;
+	struct rtentry *rt;
+	struct sockaddr *dst;
+	struct sockaddr *deldst;
 	struct mbuf *m = 0;
 	struct rtentry *nrt = 0;
 	int error;
@@ -1290,10 +1450,7 @@ rtinit_locked(ifa, cmd, flags)
 		rt_newaddrmsg(cmd, ifa, error, nrt);
 		if (use_routegenid)
 			route_generation++;
-		if (rt->rt_refcnt <= 0) {
-			rt->rt_refcnt++; /* need a 1->0 transition to free */
-			rtfree_locked(rt);
-		}
+		rtfree_locked(rt);
 	}
 
 	/*
@@ -1301,10 +1458,6 @@ rtinit_locked(ifa, cmd, flags)
 	 * We need to sanity check the result.
 	 */
 	if (cmd == RTM_ADD && error == 0 && (rt = nrt)) {
-		/*
-		 * We just wanted to add it.. we don't actually need a reference
-		 */
-		rtunref(rt);
 		/*
 		 * If it came back with an unexpected interface, then it must
 		 * have already existed or something. (XXX)
@@ -1344,23 +1497,82 @@ rtinit_locked(ifa, cmd, flags)
 		rt_newaddrmsg(cmd, ifa, error, nrt);
 		if (use_routegenid)
 			route_generation++;
-	}	
+		/*
+		 * We just wanted to add it; we don't actually need a
+		 * reference.  This will result in a route that's added
+		 * to the routing table without a reference count.  The
+		 * RTM_DELETE code will do the necessary step to adjust
+		 * the reference count at deletion time.
+		 */
+		rtunref(rt);
+	}
 	return (error);
 }
 
-static struct rtentry *
+struct rtentry *
 rte_alloc(void)
 {
+	if (rte_debug & RTD_DEBUG)
+		return (rte_alloc_debug());
+
 	return ((struct rtentry *)zalloc(rte_zone));
 }
 
-static void
+void
 rte_free(struct rtentry *p)
 {
+	if (rte_debug & RTD_DEBUG) {
+		rte_free_debug(p);
+		return;
+	}
+
 	if (p->rt_refcnt != 0)
 		panic("rte_free: rte=%p refcnt=%d non-zero\n", p, p->rt_refcnt);
 
-	bzero((caddr_t)p, sizeof (*p));
 	zfree(rte_zone, p);
 }
 
+static inline struct rtentry *
+rte_alloc_debug(void)
+{
+	struct rtentry_dbg *rte;
+
+	rte = ((struct rtentry_dbg *)zalloc(rte_zone));
+	if (rte != NULL) {
+		bzero(rte, sizeof (*rte));
+		if (rte_debug & RTD_TRACE) {
+			rte->rtd_alloc_thread = current_thread();
+			(void) OSBacktrace(rte->rtd_alloc_stk_pc,
+			    RTD_TRSTACK_SIZE);
+		}
+		rte->rtd_inuse = RTD_INUSE;
+	}
+	return ((struct rtentry *)rte);
+}
+
+static inline void
+rte_free_debug(struct rtentry *p)
+{
+	struct rtentry_dbg *rte = (struct rtentry_dbg *)p;
+
+	if (p->rt_refcnt != 0)
+		panic("rte_free: rte=%p refcnt=%d\n", p, p->rt_refcnt);
+
+	if (rte->rtd_inuse == RTD_FREED)
+		panic("rte_free: double free rte=%p\n", rte);
+	else if (rte->rtd_inuse != RTD_INUSE)
+		panic("rte_free: corrupted rte=%p\n", rte);
+
+	bcopy((caddr_t)p, (caddr_t)&rte->rtd_entry_saved, sizeof (*p));
+	bzero((caddr_t)p, sizeof (*p));
+
+	rte->rtd_inuse = RTD_FREED;
+
+	if (rte_debug & RTD_TRACE) {
+		rte->rtd_free_thread = current_thread();
+		(void) OSBacktrace(rte->rtd_free_stk_pc, RTD_TRSTACK_SIZE);
+	}
+
+	if (!(rte_debug & RTD_NO_FREE))
+		zfree(rte_zone, p);
+}
diff --git a/bsd/net/route.h b/bsd/net/route.h
index 5d1ab8ab2..9d26a8bbd 100644
--- a/bsd/net/route.h
+++ b/bsd/net/route.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1980, 1986, 1993
@@ -89,26 +95,23 @@ struct route;
  * retransmission behavior and are included in the routing structure.
  */
 struct rt_metrics {
-	u_long	rmx_locks;	/* Kernel must leave these values alone */
-	u_long	rmx_mtu;	/* MTU for this path */
-	u_long	rmx_hopcount;	/* max hops expected */
-	int32_t	rmx_expire;	/* lifetime for route, e.g. redirect */
-	u_long	rmx_recvpipe;	/* inbound delay-bandwidth product */
-	u_long	rmx_sendpipe;	/* outbound delay-bandwidth product */
-	u_long	rmx_ssthresh;	/* outbound gateway buffer limit */
-	u_long	rmx_rtt;	/* estimated round trip time */
-	u_long	rmx_rttvar;	/* estimated rtt variance */
-	u_long	rmx_pksent;	/* packets sent using this route */
-	u_long	rmx_filler[4];	/* will be used for T/TCP later */
+	u_int32_t	rmx_locks;	/* Kernel must leave these values alone */
+	u_int32_t	rmx_mtu;	/* MTU for this path */
+	u_int32_t	rmx_hopcount;	/* max hops expected */
+	int32_t		rmx_expire;	/* lifetime for route, e.g. redirect */
+	u_int32_t	rmx_recvpipe;	/* inbound delay-bandwidth product */
+	u_int32_t	rmx_sendpipe;	/* outbound delay-bandwidth product */
+	u_int32_t	rmx_ssthresh;	/* outbound gateway buffer limit */
+	u_int32_t	rmx_rtt;	/* estimated round trip time */
+	u_int32_t	rmx_rttvar;	/* estimated rtt variance */
+	u_int32_t	rmx_pksent;	/* packets sent using this route */
+	u_int32_t	rmx_filler[4];	/* will be used for T/TCP later */
 };
 
 /*
  * rmx_rtt and rmx_rttvar are stored as microseconds;
- * RTTTOPRHZ(rtt) converts to a value suitable for use
- * by a protocol slowtimo counter.
  */
 #define	RTM_RTTUNIT	1000000	/* units for rtt, rttvar, as units per sec */
-#define	RTTTOPRHZ(r)	((r) / (RTM_RTTUNIT / PR_SLOWHZ))
 
 /*
  * XXX kernel function pointer `rt_output' is visible to applications.
@@ -141,9 +144,9 @@ struct rtentry {
 	struct	rt_metrics rt_rmx;	/* metrics used by rx'ing protocols */
 	struct	rtentry *rt_gwroute;	/* implied entry for gatewayed routes */
 	int	(*rt_output)(struct ifnet *, struct mbuf *,
-				  struct sockaddr *, struct rtentry *);
+			     struct sockaddr *, struct rtentry *);
 					/* output routine for this (rt,if) */
-	struct	rtentry *rt_parent; 	/* cloning parent of this route */
+	struct	rtentry *rt_parent;	/* cloning parent of this route */
 	u_long	generation_id;		/* route generation id */
 };
 #endif /* PRIVATE */
@@ -189,9 +192,10 @@ struct ortentry {
 #define RTF_PROTO3	0x40000		/* protocol specific routing flag */
 					/* 0x80000 unused */
 #define RTF_PINNED	0x100000	/* future use */
-#define	RTF_LOCAL	0x200000 	/* route represents a local address */
+#define	RTF_LOCAL	0x200000	/* route represents a local address */
 #define	RTF_BROADCAST	0x400000	/* route represents a bcast address */
 #define	RTF_MULTICAST	0x800000	/* route represents a mcast address */
+#define RTF_TRACKREFS	0x1000000	/* Debug references and releases */
 					/* 0x1000000 and up unassigned */
 
 /*
@@ -209,33 +213,33 @@ struct	rtstat {
  * Structures for routing messages.
  */
 struct rt_msghdr {
-	u_short	rtm_msglen;	/* to skip over non-understood messages */
-	u_char	rtm_version;	/* future binary compatibility */
-	u_char	rtm_type;	/* message type */
-	u_short	rtm_index;	/* index for associated ifp */
-	int	rtm_flags;	/* flags, incl. kern & message, e.g. DONE */
-	int	rtm_addrs;	/* bitmask identifying sockaddrs in msg */
-	pid_t   rtm_pid;	/* identify sender */
-	int     rtm_seq;	/* for sender to identify action */
-	int     rtm_errno;	/* why failed */
-	int	rtm_use;	/* from rtentry */
-	u_long	rtm_inits;	/* which metrics we are initializing */
-	struct	rt_metrics rtm_rmx; /* metrics themselves */
+	u_short	rtm_msglen;		/* to skip over non-understood messages */
+	u_char	rtm_version;		/* future binary compatibility */
+	u_char	rtm_type;		/* message type */
+	u_short	rtm_index;		/* index for associated ifp */
+	int	rtm_flags;		/* flags, incl. kern & message, e.g. DONE */
+	int	rtm_addrs;		/* bitmask identifying sockaddrs in msg */
+	pid_t	rtm_pid;		/* identify sender */
+	int	rtm_seq;		/* for sender to identify action */
+	int	rtm_errno;		/* why failed */
+	int	rtm_use;		/* from rtentry */
+	u_int32_t rtm_inits;		/* which metrics we are initializing */
+	struct rt_metrics rtm_rmx;	/* metrics themselves */
 };
 
 struct rt_msghdr2 {
-        u_short rtm_msglen;     /* to skip over non-understood messages */
-        u_char  rtm_version;    /* future binary compatibility */
-        u_char  rtm_type;       /* message type */
-        u_short rtm_index;      /* index for associated ifp */
-        int     rtm_flags;      /* flags, incl. kern & message, e.g. DONE */
-        int     rtm_addrs;      /* bitmask identifying sockaddrs in msg */
-	int32_t rtm_refcnt;         /* reference count */
-	int     rtm_parentflags;      /* flags of the parent route */
-        int     rtm_reserved;      /* reserved field set to 0 */
-        int     rtm_use;        /* from rtentry */
-        u_long  rtm_inits;      /* which metrics we are initializing */
-        struct  rt_metrics rtm_rmx; /* metrics themselves */
+	u_short	rtm_msglen;		/* to skip over non-understood messages */
+	u_char	rtm_version;		/* future binary compatibility */
+	u_char	rtm_type;		/* message type */
+	u_short	rtm_index;		/* index for associated ifp */
+	int	rtm_flags;		/* flags, incl. kern & message, e.g. DONE */
+	int	rtm_addrs;		/* bitmask identifying sockaddrs in msg */
+	int32_t	rtm_refcnt;		/* reference count */
+	int	rtm_parentflags;	/* flags of the parent route */
+	int	rtm_reserved;		/* reserved field set to 0 */
+	int	rtm_use;		/* from rtentry */
+	u_int32_t rtm_inits;		/* which metrics we are initializing */
+	struct rt_metrics rtm_rmx;	/* metrics themselves */
 };
 
 
@@ -326,37 +330,40 @@ extern struct radix_node_head *rt_tables[AF_MAX+1];
 struct ifmultiaddr;
 struct proc;
 
-void	 route_init(void);
-void	 rt_ifmsg(struct ifnet *);
-void	 rt_missmsg(int, struct rt_addrinfo *, int, int);
-void	 rt_newaddrmsg(int, struct ifaddr *, int, struct rtentry *);
-void	 rt_newmaddrmsg(int, struct ifmultiaddr *);
-int	 rt_setgate(struct rtentry *, struct sockaddr *, struct sockaddr *);
-void	 rtalloc(struct route *);
-void	 rtalloc_ign(struct route *, u_long);
-struct rtentry *
-	 rtalloc1(struct sockaddr *, int, u_long);
-struct rtentry *
-	 rtalloc1_locked(const struct sockaddr *, int, u_long);
-void	rtfree(struct rtentry *);
-void	rtfree_locked(struct rtentry *);
-void	rtref(struct rtentry *);
+__private_extern__ int rttrash;
+
+extern void route_init(void) __attribute__((section("__TEXT, initcode")));
+extern void rt_ifmsg(struct ifnet *);
+extern void rt_missmsg(int, struct rt_addrinfo *, int, int);
+extern void rt_newaddrmsg(int, struct ifaddr *, int, struct rtentry *);
+extern void rt_newmaddrmsg(int, struct ifmultiaddr *);
+extern int rt_setgate(struct rtentry *, struct sockaddr *, struct sockaddr *);
+extern void rtalloc(struct route *);
+extern void rtalloc_ign(struct route *, u_long);
+extern void rtalloc_ign_locked(struct route *, u_long );
+extern struct rtentry *rtalloc1(struct sockaddr *, int, u_long);
+extern struct rtentry *rtalloc1_locked(struct sockaddr *, int, u_long);
+extern void rtfree(struct rtentry *);
+extern void rtfree_locked(struct rtentry *);
+extern void rtref(struct rtentry *);
 /*
  * rtunref will decrement the refcount, rtfree will decrement and free if
  * the refcount has reached zero and the route is not up.
  * Unless you have good reason to do otherwise, use rtfree.
  */
-void	rtunref(struct rtentry *);
-void	rtsetifa(struct rtentry *, struct ifaddr *);
-int	 rtinit(struct ifaddr *, int, int);
-int	 rtinit_locked(struct ifaddr *, int, int);
-int	 rtioctl(int, caddr_t, struct proc *);
-void	 rtredirect(struct sockaddr *, struct sockaddr *,
-	    struct sockaddr *, int, struct sockaddr *, struct rtentry **);
-int	 rtrequest(int, struct sockaddr *,
-	    struct sockaddr *, struct sockaddr *, int, struct rtentry **);
-int	 rtrequest_locked(int, struct sockaddr *,
-	    struct sockaddr *, struct sockaddr *, int, struct rtentry **);
+extern void rtunref(struct rtentry *);
+extern void rtsetifa(struct rtentry *, struct ifaddr *);
+extern int rtinit(struct ifaddr *, int, int);
+extern int rtinit_locked(struct ifaddr *, int, int);
+extern int rtioctl(int, caddr_t, struct proc *);
+extern void rtredirect(struct sockaddr *, struct sockaddr *,
+    struct sockaddr *, int, struct sockaddr *, struct rtentry **);
+extern int rtrequest(int, struct sockaddr *,
+    struct sockaddr *, struct sockaddr *, int, struct rtentry **);
+extern int rtrequest_locked(int, struct sockaddr *,
+    struct sockaddr *, struct sockaddr *, int, struct rtentry **);
+extern struct rtentry *rte_alloc(void);
+extern void rte_free(struct rtentry *);
 #endif KERNEL_PRIVATE
 
 #endif
diff --git a/bsd/net/rtsock.c b/bsd/net/rtsock.c
index 82cef1b0a..b4aee361b 100644
--- a/bsd/net/rtsock.c
+++ b/bsd/net/rtsock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1991, 1993
@@ -76,11 +82,11 @@
 
 #include <machine/spl.h>
 
-extern void m_copydata(struct mbuf *, int, int, caddr_t);
-extern void m_copyback(struct mbuf *, int, int, caddr_t);
-
 extern struct rtstat rtstat;
 extern int rttrash;
+extern u_long route_generation;
+extern int use_routegenid;
+extern int check_routeselfref;
 
 MALLOC_DEFINE(M_RTABLE, "routetbl", "routing tables");
 
@@ -158,7 +164,7 @@ rts_attach(struct socket *so, int proto, __unused struct proc *p)
 	rp = sotorawcb(so);
 	if (error) {
 		FREE(rp, M_PCB);
-		so->so_pcb = 0;
+		so->so_pcb = NULL;
 		so->so_flags |= SOF_PCBCLEARING;
 		return error;
 	}
@@ -189,20 +195,16 @@ rts_attach(struct socket *so, int proto, __unused struct proc *p)
 static int
 rts_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_bind(so, nam, p); /* xxx just EINVAL */
-	splx(s);
 	return error;
 }
 
 static int
 rts_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_connect(so, nam, p); /* XXX just EINVAL */
-	splx(s);
 	return error;
 }
 
@@ -213,9 +215,8 @@ static int
 rts_detach(struct socket *so)
 {
 	struct rawcb *rp = sotorawcb(so);
-	int s, error;
+	int error;
 
-	s = splnet();
 	if (rp != 0) {
 		switch(rp->rcb_proto.sp_protocol) {
 		case AF_INET:
@@ -234,17 +235,14 @@ rts_detach(struct socket *so)
 		route_cb.any_count--;
 	}
 	error = raw_usrreqs.pru_detach(so);
-	splx(s);
 	return error;
 }
 
 static int
 rts_disconnect(struct socket *so)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_disconnect(so);
-	splx(s);
 	return error;
 }
 
@@ -253,10 +251,8 @@ rts_disconnect(struct socket *so)
 static int
 rts_peeraddr(struct socket *so, struct sockaddr **nam)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_peeraddr(so, nam);
-	splx(s);
 	return error;
 }
 
@@ -267,10 +263,8 @@ static int
 rts_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	 struct mbuf *control, struct proc *p)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_send(so, flags, m, nam, control, p);
-	splx(s);
 	return error;
 }
 
@@ -279,20 +273,16 @@ rts_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 static int
 rts_shutdown(struct socket *so)
 {
-	int s, error;
-	s = splnet();
+	int  error;
 	error = raw_usrreqs.pru_shutdown(so);
-	splx(s);
 	return error;
 }
 
 static int
 rts_sockaddr(struct socket *so, struct sockaddr **nam)
 {
-	int s, error;
-	s = splnet();
+	int error;
 	error = raw_usrreqs.pru_sockaddr(so, nam);
-	splx(s);
 	return error;
 }
 
@@ -306,17 +296,15 @@ static struct pr_usrreqs route_usrreqs = {
 
 /*ARGSUSED*/
 static int
-route_output(m, so)
-	struct mbuf *m;
-	struct socket *so;
+route_output(struct mbuf *m, struct socket *so)
 {
-	struct rt_msghdr *rtm = 0;
-	struct rtentry *rt = 0;
-	struct rtentry *saved_nrt = 0;
+	struct rt_msghdr *rtm = NULL;
+	struct rtentry *rt = NULL;
+	struct rtentry *saved_nrt = NULL;
 	struct radix_node_head *rnh;
 	struct rt_addrinfo info;
 	int len, error = 0;
-	struct ifnet *ifp = 0;
+	struct ifnet *ifp = NULL;
 #ifndef __APPLE__
 	struct proc  *curproc = current_proc();
 #endif
@@ -406,7 +394,6 @@ route_output(m, so)
  * confusing the routing table with a wrong route to the previous default gateway
  */
 {
-			extern int check_routeselfref;
 #define satosinaddr(sa) (((struct sockaddr_in *)sa)->sin_addr.s_addr)
 	
 			if (check_routeselfref && (dst && dst->sa_family == AF_INET) && 
@@ -470,8 +457,7 @@ route_output(m, so)
 			error = rtrequest_locked(RTM_DELETE, dst, gate, netmask,
 					rtm->rtm_flags, &saved_nrt);
 			if (error == 0) {
-				if ((rt = saved_nrt))
-					rtref(rt);
+				rt = saved_nrt;
 				goto report;
 			}
 			break;
@@ -634,9 +620,7 @@ flush:
 }
 
 static void
-rt_setmetrics(which, in, out)
-	u_long which;
-	struct rt_metrics *in, *out;
+rt_setmetrics(u_long which, struct rt_metrics *in, struct rt_metrics *out)
 {
 #define metric(f, e) if (which & (f)) out->e = in->e;
 	metric(RTV_RPIPE, rmx_recvpipe);
@@ -665,6 +649,10 @@ rt_setif(
 
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
+	/* trigger route cache reevaluation */
+	if (use_routegenid)
+		route_generation++;
+
 	/* new gateway could require new ifaddr, ifp;
 	   flags may also be different; ifp may be specified
 	   by ll sockaddr when protocol address is ambiguous */
@@ -694,7 +682,7 @@ rt_setif(
 		else if (Ifaaddr && (ifa = ifa_ifwithaddr(Ifaaddr))) {
 			ifp = ifa->ifa_ifp;
 		}
-		else if (Gate && (ifa = ifa_ifwithroute(rt->rt_flags,
+		else if (Gate && (ifa = ifa_ifwithroute_locked(rt->rt_flags,
 						rt_key(rt), Gate))) {
 			ifp = ifa->ifa_ifp;
 		}
@@ -735,9 +723,7 @@ rt_setif(
  * This data is derived straight from userland.
  */
 static int
-rt_xaddrs(cp, cplim, rtinfo)
-	caddr_t cp, cplim;
-	struct rt_addrinfo *rtinfo;
+rt_xaddrs(caddr_t cp, caddr_t cplim, struct rt_addrinfo *rtinfo)
 {
 	struct sockaddr *sa;
 	int i;
@@ -838,11 +824,7 @@ rt_msg1(
 }
 
 static int
-rt_msg2(type, rtinfo, cp, w)
-	int type;
-	struct rt_addrinfo *rtinfo;
-	caddr_t cp;
-	struct walkarg *w;
+rt_msg2(int type, struct rt_addrinfo *rtinfo, caddr_t cp, struct walkarg *w)
 {
 	int i;
 	int len, dlen, second_time = 0;
@@ -933,9 +915,7 @@ again:
  * destination.
  */
 void
-rt_missmsg(type, rtinfo, flags, error)
-	int type, flags, error;
-	struct rt_addrinfo *rtinfo;
+rt_missmsg(int type, struct rt_addrinfo *rtinfo, int flags, int error)
 {
 	struct rt_msghdr *rtm;
 	struct mbuf *m;
@@ -977,7 +957,7 @@ rt_ifmsg(
 	ifm = mtod(m, struct if_msghdr *);
 	ifm->ifm_index = ifp->if_index;
 	ifm->ifm_flags = (u_short)ifp->if_flags;
-	if_data_internal_to_if_data(&ifp->if_data, &ifm->ifm_data);
+	if_data_internal_to_if_data(ifp, &ifp->if_data, &ifm->ifm_data);
 	ifm->ifm_addrs = 0;
 	route_proto.sp_protocol = 0;
 	raw_input(m, &route_proto, &route_src, &route_dst);
@@ -995,10 +975,7 @@ rt_ifmsg(
  * interface will be locked.
  */
 void
-rt_newaddrmsg(cmd, ifa, error, rt)
-	int cmd, error;
-	struct ifaddr *ifa;
-	struct rtentry *rt;
+rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
 {
 	struct rt_addrinfo info;
 	struct sockaddr *sa = 0;
@@ -1055,9 +1032,7 @@ rt_newaddrmsg(cmd, ifa, error, rt)
  * there is no route state to worry about.
  */
 void
-rt_newmaddrmsg(cmd, ifma)
-	int cmd;
-	struct ifmultiaddr *ifma;
+rt_newmaddrmsg(int cmd, struct ifmultiaddr *ifma)
 {
 	struct rt_addrinfo info;
 	struct mbuf *m = 0;
@@ -1091,9 +1066,7 @@ rt_newmaddrmsg(cmd, ifma)
  * This is used in dumping the kernel table via sysctl().
  */
 int
-sysctl_dumpentry(rn, vw)
-	struct radix_node *rn;
-	void *vw;
+sysctl_dumpentry(struct radix_node *rn, void *vw)
 {
 	struct walkarg *w = vw;
 	struct rtentry *rt = (struct rtentry *)rn;
@@ -1175,7 +1148,7 @@ sysctl_iflist(
 			ifm = (struct if_msghdr *)w->w_tmem;
 			ifm->ifm_index = ifp->if_index;
 			ifm->ifm_flags = (u_short)ifp->if_flags;
-			if_data_internal_to_if_data(&ifp->if_data, &ifm->ifm_data);
+			if_data_internal_to_if_data(ifp, &ifp->if_data, &ifm->ifm_data);
 			ifm->ifm_addrs = info.rti_addrs;
 			error = SYSCTL_OUT(w->w_req,(caddr_t)ifm, len);
 			if (error) {
@@ -1186,10 +1159,6 @@ sysctl_iflist(
 		while ((ifa = ifa->ifa_link.tqe_next) != 0) {
 			if (af && af != ifa->ifa_addr->sa_family)
 				continue;
-#ifndef __APPLE__
-			if (curproc->p_prison && prison_if(curproc, ifa->ifa_addr))
-				continue;
-#endif
 			ifaaddr = ifa->ifa_addr;
 			netmask = ifa->ifa_netmask;
 			brdaddr = ifa->ifa_dstaddr;
@@ -1247,7 +1216,7 @@ sysctl_iflist2(
 			ifm->ifm_snd_maxlen = ifp->if_snd.ifq_maxlen;
 			ifm->ifm_snd_drops = ifp->if_snd.ifq_drops;
 			ifm->ifm_timer = ifp->if_timer;
-			if_data_internal_to_if_data64(&ifp->if_data, &ifm->ifm_data);
+			if_data_internal_to_if_data64(ifp, &ifp->if_data, &ifm->ifm_data);
 			error = SYSCTL_OUT(w->w_req, w->w_tmem, len);
 			if (error) {
 				ifnet_lock_done(ifp);
@@ -1415,7 +1384,7 @@ static struct protosw routesw[] = {
 struct domain routedomain =
     { PF_ROUTE, "route", route_init, 0, 0,
       routesw, 
-      0, 0, 0, 0, 0, 0, 0, 0, 
+      NULL, NULL, 0, 0, 0, 0, NULL, 0, 
       { 0, 0 } };
 
 DOMAIN_SET(route);
diff --git a/bsd/net/rtsock_mip.c b/bsd/net/rtsock_mip.c
index 6a6662206..2acd2585d 100644
--- a/bsd/net/rtsock_mip.c
+++ b/bsd/net/rtsock_mip.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME$	*/
 
diff --git a/bsd/net/slip.h b/bsd/net/slip.h
index 7a8c06c66..c933d4402 100644
--- a/bsd/net/slip.h
+++ b/bsd/net/slip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1994
diff --git a/bsd/net/zlib.h b/bsd/net/zlib.h
index 5f90b2049..1d8223213 100644
--- a/bsd/net/zlib.h
+++ b/bsd/net/zlib.h
@@ -1,1190 +1,33 @@
-/* $FreeBSD: src/sys/net/zlib.h,v 1.7 1999/12/29 04:38:38 peter Exp $	*/
-
-/*
- * This file is derived from zlib.h and zconf.h from the zlib-1.1.4
- * distribution by Jean-loup Gailly and Mark Adler.
- */
-
-/* +++ zlib.h */
-/* zlib.h -- interface of the 'zlib' general purpose compression library
-  version 1.1.4, March 11th, 2002
-
-  Copyright (C) 1995-2002 Jean-loup Gailly and Mark Adler
-
-  This software is provided 'as-is', without any express or implied
-  warranty.  In no event will the authors be held liable for any damages
-  arising from the use of this software.
-
-  Permission is granted to anyone to use this software for any purpose,
-  including commercial applications, and to alter it and redistribute it
-  freely, subject to the following restrictions:
-
-  1. The origin of this software must not be misrepresented; you must not
-     claim that you wrote the original software. If you use this software
-     in a product, an acknowledgment in the product documentation would be
-     appreciated but is not required.
-  2. Altered source versions must be plainly marked as such, and must not be
-     misrepresented as being the original software.
-  3. This notice may not be removed or altered from any source distribution.
-
-  Jean-loup Gailly        Mark Adler
-  jloup@gzip.org          madler@alumni.caltech.edu
-
-
-  The data format used by the zlib library is described by RFCs (Request for
-  Comments) 1950 to 1952 in the files ftp://ds.internic.net/rfc/rfc1950.txt
-  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
-*/
-
-#ifndef _ZLIB_H
-#define _ZLIB_H
-#include <sys/appleapiopts.h>
-
-#ifdef KERNEL_PRIVATE
-
-#if __cplusplus
-extern "C" {
-#endif
-
-
-/* +++ zconf.h */
-/* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2002 Jean-loup Gailly.
- * For conditions of distribution and use, see copyright notice in zlib.h 
- */
-
-#ifndef _ZCONF_H
-#define _ZCONF_H
-
 /*
- * If you *really* need a unique prefix for all types and library functions,
- * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it.
- */
-#ifdef Z_PREFIX
-#  define deflateInit_	z_deflateInit_
-#  define deflate	z_deflate
-#  define deflateEnd	z_deflateEnd
-#  define inflateInit_ 	z_inflateInit_
-#  define inflate	z_inflate
-#  define inflateEnd	z_inflateEnd
-#  define deflateInit2_	z_deflateInit2_
-#  define deflateSetDictionary z_deflateSetDictionary
-#  define deflateCopy	z_deflateCopy
-#  define deflateReset	z_deflateReset
-#  define deflateParams	z_deflateParams
-#  define inflateInit2_	z_inflateInit2_
-#  define inflateSetDictionary z_inflateSetDictionary
-#  define inflateSync	z_inflateSync
-#  define inflateSyncPoint z_inflateSyncPoint
-#  define inflateReset	z_inflateReset
-#  define compress	z_compress
-#  define compress2	z_compress2
-#  define uncompress	z_uncompress
-#  define adler32	z_adler32
-#if 0
-#  define crc32		z_crc32
-#  define get_crc_table z_get_crc_table
-#endif
-
-#  define Byte		z_Byte
-#  define uInt		z_uInt
-#  define uLong		z_uLong
-#  define Bytef	        z_Bytef
-#  define charf		z_charf
-#  define intf		z_intf
-#  define uIntf		z_uIntf
-#  define uLongf	z_uLongf
-#  define voidpf	z_voidpf
-#  define voidp		z_voidp
-#endif
-
-#if (defined(_WIN32) || defined(__WIN32__)) && !defined(WIN32)
-#  define WIN32
-#endif
-#if defined(__GNUC__) || defined(WIN32) || defined(__386__) || defined(i386)
-#  ifndef __32BIT__
-#    define __32BIT__
-#  endif
-#endif
-#if defined(__MSDOS__) && !defined(MSDOS)
-#  define MSDOS
-#endif
-
-/*
- * Compile with -DMAXSEG_64K if the alloc function cannot allocate more
- * than 64k bytes at a time (needed on systems with 16-bit int).
- */
-#if defined(MSDOS) && !defined(__32BIT__)
-#  define MAXSEG_64K
-#endif
-#ifdef MSDOS
-#  define UNALIGNED_OK
-#endif
-
-#if (defined(MSDOS) || defined(_WINDOWS) || defined(WIN32))  && !defined(STDC)
-#  define STDC
-#endif
-#if defined(__STDC__) || defined(__cplusplus) || defined(__OS2__)
-#  ifndef STDC
-#    define STDC
-#  endif
-#endif
-
-#ifndef STDC
-#  ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */
-#    define const
-#  endif
-#endif
-
-/* Some Mac compilers merge all .h files incorrectly: */
-#if defined(__MWERKS__) || defined(applec) ||defined(THINK_C) ||defined(__SC__)
-#  define NO_DUMMY_DECL
-#endif
-
-/* Old Borland C incorrectly complains about missing returns: */
-#if defined(__BORLANDC__) && (__BORLANDC__ < 0x500)
-#  define NEED_DUMMY_RETURN
-#endif
-
-
-/* Maximum value for memLevel in deflateInit2 */
-#ifndef MAX_MEM_LEVEL
-#  ifdef MAXSEG_64K
-#    define MAX_MEM_LEVEL 8
-#  else
-#    define MAX_MEM_LEVEL 9
-#  endif
-#endif
-
-/* Maximum value for windowBits in deflateInit2 and inflateInit2.
- * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
- * created by gzip. (Files created by minigzip can still be extracted by
- * gzip.)
- */
-#ifndef MAX_WBITS
-#  define MAX_WBITS   15 /* 32K LZ77 window */
-#endif
-
-/* The memory requirements for deflate are (in bytes):
-            (1 << (windowBits+2)) +  (1 << (memLevel+9))
- that is: 128K for windowBits=15  +  128K for memLevel = 8  (default values)
- plus a few kilobytes for small objects. For example, if you want to reduce
- the default memory requirements from 256K to 128K, compile with
-     make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
- Of course this will generally degrade compression (there's no free lunch).
-
-   The memory requirements for inflate are (in bytes) 1 << windowBits
- that is, 32K for windowBits=15 (default value) plus a few kilobytes
- for small objects.
-*/
-
-                        /* Type declarations */
-
-#ifndef OF /* function prototypes */
-#  ifdef STDC
-#    define OF(args)  args
-#  else
-#    define OF(args)  ()
-#  endif
-#endif
-
-/* The following definitions for FAR are needed only for MSDOS mixed
- * model programming (small or medium model with some far allocations).
- * This was tested only with MSC; for other MSDOS compilers you may have
- * to define NO_MEMCPY in zutil.h.  If you don't need the mixed model,
- * just define FAR to be empty.
- */
-#if (defined(M_I86SM) || defined(M_I86MM)) && !defined(__32BIT__)
-   /* MSC small or medium model */
-#  define SMALL_MEDIUM
-#  ifdef _MSC_VER
-#    define FAR _far
-#  else
-#    define FAR far
-#  endif
-#endif
-#if defined(__BORLANDC__) && (defined(__SMALL__) || defined(__MEDIUM__))
-#  ifndef __32BIT__
-#    define SMALL_MEDIUM
-#    define FAR _far
-#  endif
-#endif
-
-/* Compile with -DZLIB_DLL for Windows DLL support */
-#if defined(ZLIB_DLL)
-#  if defined(_WINDOWS) || defined(WINDOWS)
-#    ifdef FAR
-#      undef FAR
-#    endif
-#    include <windows.h>
-#    define ZEXPORT  WINAPI
-#    ifdef WIN32
-#      define ZEXPORTVA  WINAPIV
-#    else
-#      define ZEXPORTVA  FAR _cdecl _export
-#    endif
-#  endif
-#  if defined (__BORLANDC__)
-#    if (__BORLANDC__ >= 0x0500) && defined (WIN32)
-#      include <windows.h>
-#      define ZEXPORT __declspec(dllexport) WINAPI
-#      define ZEXPORTRVA __declspec(dllexport) WINAPIV
-#    else
-#      if defined (_Windows) && defined (__DLL__)
-#        define ZEXPORT _export
-#        define ZEXPORTVA _export
-#      endif
-#    endif
-#  endif
-#endif
-
-#if defined (__BEOS__)
-#  if defined (ZLIB_DLL)
-#    define ZEXTERN extern __declspec(dllexport)
-#  else
-#    define ZEXTERN extern __declspec(dllimport)
-#  endif
-#endif
-
-#ifndef ZEXPORT
-#  define ZEXPORT
-#endif
-#ifndef ZEXPORTVA
-#  define ZEXPORTVA
-#endif
-#ifndef ZEXTERN
-#  define ZEXTERN extern
-#endif
-
-#ifndef FAR
-#   define FAR
-#endif
-
-#if !defined(MACOS) && !defined(TARGET_OS_MAC)
-typedef unsigned char  Byte;  /* 8 bits */
-#endif
-typedef unsigned int   uInt;  /* 16 bits or more */
-typedef unsigned long  uLong; /* 32 bits or more */
-
-#ifdef SMALL_MEDIUM
-   /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */
-#  define Bytef Byte FAR
-#else
-   typedef Byte  FAR Bytef;
-#endif
-typedef char  FAR charf;
-typedef int   FAR intf;
-typedef uInt  FAR uIntf;
-typedef uLong FAR uLongf;
-
-#ifdef STDC
-   typedef void FAR *voidpf;
-   typedef void     *voidp;
-#else
-   typedef Byte FAR *voidpf;
-   typedef Byte     *voidp;
-#endif
-
-#ifdef HAVE_UNISTD_H
-#  include <sys/types.h> /* for off_t */
-#  include <unistd.h>    /* for SEEK_* and off_t */
-#  define z_off_t  off_t
-#endif
-#ifndef SEEK_SET
-#  define SEEK_SET        0       /* Seek from beginning of file.  */
-#  define SEEK_CUR        1       /* Seek from current position.  */
-#  define SEEK_END        2       /* Set file pointer to EOF plus "offset" */
-#endif
-#ifndef z_off_t
-#  define  z_off_t long
-#endif
-
-/* MVS linker does not support external names larger than 8 bytes */
-#if defined(__MVS__)
-#   pragma map(deflateInit_,"DEIN")
-#   pragma map(deflateInit2_,"DEIN2")
-#   pragma map(deflateEnd,"DEEND")
-#   pragma map(inflateInit_,"ININ")
-#   pragma map(inflateInit2_,"ININ2")
-#   pragma map(inflateEnd,"INEND")
-#   pragma map(inflateSync,"INSY")
-#   pragma map(inflateSetDictionary,"INSEDI")
-#   pragma map(inflate_blocks,"INBL")
-#   pragma map(inflate_blocks_new,"INBLNE")
-#   pragma map(inflate_blocks_free,"INBLFR")
-#   pragma map(inflate_blocks_reset,"INBLRE")
-#   pragma map(inflate_codes_free,"INCOFR")
-#   pragma map(inflate_codes,"INCO")
-#   pragma map(inflate_fast,"INFA")
-#   pragma map(inflate_flush,"INFLU")
-#   pragma map(inflate_mask,"INMA")
-#   pragma map(inflate_set_dictionary,"INSEDI2")
-#   pragma map(inflate_copyright,"INCOPY")
-#   pragma map(inflate_trees_bits,"INTRBI")
-#   pragma map(inflate_trees_dynamic,"INTRDY")
-#   pragma map(inflate_trees_fixed,"INTRFI")
-#   pragma map(inflate_trees_free,"INTRFR")
-#endif
-
-#endif /* _ZCONF_H */
-/* --- zconf.h */
-
-#define ZLIB_VERSION "1.1.4"
-
-/* 
-     The 'zlib' compression library provides in-memory compression and
-  decompression functions, including integrity checks of the uncompressed
-  data.  This version of the library supports only one compression method
-  (deflation) but other algorithms will be added later and will have the same
-  stream interface.
-
-     Compression can be done in a single step if the buffers are large
-  enough (for example if an input file is mmap'ed), or can be done by
-  repeated calls of the compression function.  In the latter case, the
-  application must provide more input and/or consume the output
-  (providing more output space) before each call.
-
-     The library also supports reading and writing files in gzip (.gz) format
-  with an interface similar to that of stdio.
-
-     The library does not install any signal handler. The decoder checks
-  the consistency of the compressed data, so the library should never
-  crash even in case of corrupted input.
-*/
-
-typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
-typedef void   (*free_func)  OF((voidpf opaque, voidpf address));
-
-struct internal_state;
-
-typedef struct z_stream_s {
-    Bytef    *next_in;  /* next input byte */
-    uInt     avail_in;  /* number of bytes available at next_in */
-    uLong    total_in;  /* total nb of input bytes read so far */
-
-    Bytef    *next_out; /* next output byte should be put there */
-    uInt     avail_out; /* remaining free space at next_out */
-    uLong    total_out; /* total nb of bytes output so far */
-
-    char     *msg;      /* last error message, NULL if no error */
-    struct internal_state FAR *state; /* not visible by applications */
-
-    alloc_func zalloc;  /* used to allocate the internal state */
-    free_func  zfree;   /* used to free the internal state */
-    voidpf     opaque;  /* private data object passed to zalloc and zfree */
-
-    int     data_type;  /* best guess about the data type: ascii or binary */
-    uLong   adler;      /* adler32 value of the uncompressed data */
-    uLong   reserved;   /* reserved for future use */
-} z_stream;
-
-typedef z_stream FAR *z_streamp;
-
-/*
-   The application must update next_in and avail_in when avail_in has
-   dropped to zero. It must update next_out and avail_out when avail_out
-   has dropped to zero. The application must initialize zalloc, zfree and
-   opaque before calling the init function. All other fields are set by the
-   compression library and must not be updated by the application.
-
-   The opaque value provided by the application will be passed as the first
-   parameter for calls of zalloc and zfree. This can be useful for custom
-   memory management. The compression library attaches no meaning to the
-   opaque value.
-
-   zalloc must return Z_NULL if there is not enough memory for the object.
-   If zlib is used in a multi-threaded application, zalloc and zfree must be
-   thread safe.
-
-   On 16-bit systems, the functions zalloc and zfree must be able to allocate
-   exactly 65536 bytes, but will not be required to allocate more than this
-   if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS,
-   pointers returned by zalloc for objects of exactly 65536 bytes *must*
-   have their offset normalized to zero. The default allocation function
-   provided by this library ensures this (see zutil.c). To reduce memory
-   requirements and avoid any allocation of 64K objects, at the expense of
-   compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h).
-
-   The fields total_in and total_out can be used for statistics or
-   progress reports. After compression, total_in holds the total size of
-   the uncompressed data and may be saved for use in the decompressor
-   (particularly if the decompressor wants to decompress everything in
-   a single step).
-*/
-
-                        /* constants */
-
-#define Z_NO_FLUSH      0
-#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */
-#define Z_PACKET_FLUSH	2
-#define Z_SYNC_FLUSH    3
-#define Z_FULL_FLUSH    4
-#define Z_FINISH        5
-/* Allowed flush values; see deflate() below for details */
-
-#define Z_OK            0
-#define Z_STREAM_END    1
-#define Z_NEED_DICT     2
-#define Z_ERRNO        (-1)
-#define Z_STREAM_ERROR (-2)
-#define Z_DATA_ERROR   (-3)
-#define Z_MEM_ERROR    (-4)
-#define Z_BUF_ERROR    (-5)
-#define Z_VERSION_ERROR (-6)
-/* Return codes for the compression/decompression functions. Negative
- * values are errors, positive values are used for special but normal events.
- */
-
-#define Z_NO_COMPRESSION         0
-#define Z_BEST_SPEED             1
-#define Z_BEST_COMPRESSION       9
-#define Z_DEFAULT_COMPRESSION  (-1)
-/* compression levels */
-
-#define Z_FILTERED            1
-#define Z_HUFFMAN_ONLY        2
-#define Z_DEFAULT_STRATEGY    0
-/* compression strategy; see deflateInit2() below for details */
-
-#define Z_BINARY   0
-#define Z_ASCII    1
-#define Z_UNKNOWN  2
-/* Possible values of the data_type field */
-
-#define Z_DEFLATED   8
-/* The deflate compression method (the only one supported in this version) */
-
-#define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
-
-#define zlib_version zlibVersion()
-/* for compatibility with versions < 1.0.2 */
-
-                        /* basic functions */
-
-ZEXTERN const char * ZEXPORT zlibVersion OF((void));
-/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
-   If the first character differs, the library code actually used is
-   not compatible with the zlib.h header file used by the application.
-   This check is automatically made by deflateInit and inflateInit.
- */
-
-/* 
-ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
-
-     Initializes the internal stream state for compression. The fields
-   zalloc, zfree and opaque must be initialized before by the caller.
-   If zalloc and zfree are set to Z_NULL, deflateInit updates them to
-   use default allocation functions.
-
-     The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
-   1 gives best speed, 9 gives best compression, 0 gives no compression at
-   all (the input data is simply copied a block at a time).
-   Z_DEFAULT_COMPRESSION requests a default compromise between speed and
-   compression (currently equivalent to level 6).
-
-     deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_STREAM_ERROR if level is not a valid compression level,
-   Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
-   with the version assumed by the caller (ZLIB_VERSION).
-   msg is set to null if there is no error message.  deflateInit does not
-   perform any compression: this will be done by deflate().
-*/
-
-
-ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
-/*
-    deflate compresses as much data as possible, and stops when the input
-  buffer becomes empty or the output buffer becomes full. It may introduce some
-  output latency (reading input without producing any output) except when
-  forced to flush.
-
-    The detailed semantics are as follows. deflate performs one or both of the
-  following actions:
-
-  - Compress more input starting at next_in and update next_in and avail_in
-    accordingly. If not all input can be processed (because there is not
-    enough room in the output buffer), next_in and avail_in are updated and
-    processing will resume at this point for the next call of deflate().
-
-  - Provide more output starting at next_out and update next_out and avail_out
-    accordingly. This action is forced if the parameter flush is non zero.
-    Forcing flush frequently degrades the compression ratio, so this parameter
-    should be set only when necessary (in interactive applications).
-    Some output may be provided even if flush is not set.
-
-  Before the call of deflate(), the application should ensure that at least
-  one of the actions is possible, by providing more input and/or consuming
-  more output, and updating avail_in or avail_out accordingly; avail_out
-  should never be zero before the call. The application can consume the
-  compressed output when it wants, for example when the output buffer is full
-  (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK
-  and with zero avail_out, it must be called again after making room in the
-  output buffer because there might be more output pending.
-
-    If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
-  flushed to the output buffer and the output is aligned on a byte boundary, so
-  that the decompressor can get all input data available so far. (In particular
-  avail_in is zero after the call if enough output space has been provided
-  before the call.)  Flushing may degrade compression for some compression
-  algorithms and so it should be used only when necessary.
-
-    If flush is set to Z_FULL_FLUSH, all output is flushed as with
-  Z_SYNC_FLUSH, and the compression state is reset so that decompression can
-  restart from this point if previous compressed data has been damaged or if
-  random access is desired. Using Z_FULL_FLUSH too often can seriously degrade
-  the compression.
-
-    If deflate returns with avail_out == 0, this function must be called again
-  with the same value of the flush parameter and more output space (updated
-  avail_out), until the flush is complete (deflate returns with non-zero
-  avail_out).
-
-    If the parameter flush is set to Z_FINISH, pending input is processed,
-  pending output is flushed and deflate returns with Z_STREAM_END if there
-  was enough output space; if deflate returns with Z_OK, this function must be
-  called again with Z_FINISH and more output space (updated avail_out) but no
-  more input data, until it returns with Z_STREAM_END or an error. After
-  deflate has returned Z_STREAM_END, the only possible operations on the
-  stream are deflateReset or deflateEnd.
-  
-    Z_FINISH can be used immediately after deflateInit if all the compression
-  is to be done in a single step. In this case, avail_out must be at least
-  0.1% larger than avail_in plus 12 bytes.  If deflate does not return
-  Z_STREAM_END, then it must be called again as described above.
-
-    deflate() sets strm->adler to the adler32 checksum of all input read
-  so far (that is, total_in bytes).
-
-    deflate() may update data_type if it can make a good guess about
-  the input data type (Z_ASCII or Z_BINARY). In doubt, the data is considered
-  binary. This field is only for information purposes and does not affect
-  the compression algorithm in any manner.
-
-    deflate() returns Z_OK if some progress has been made (more input
-  processed or more output produced), Z_STREAM_END if all input has been
-  consumed and all output has been produced (only when flush is set to
-  Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
-  if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible
-  (for example avail_in or avail_out was zero).
-*/
-
-
-ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
-/*
-     All dynamically allocated data structures for this stream are freed.
-   This function discards any unprocessed input and does not flush any
-   pending output.
-
-     deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
-   stream state was inconsistent, Z_DATA_ERROR if the stream was freed
-   prematurely (some input or output was discarded). In the error case,
-   msg may be set but then points to a static string (which must not be
-   deallocated).
-*/
-
-
-/* 
-ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
-
-     Initializes the internal stream state for decompression. The fields
-   next_in, avail_in, zalloc, zfree and opaque must be initialized before by
-   the caller. If next_in is not Z_NULL and avail_in is large enough (the exact
-   value depends on the compression method), inflateInit determines the
-   compression method from the zlib header and allocates all data structures
-   accordingly; otherwise the allocation will be deferred to the first call of
-   inflate.  If zalloc and zfree are set to Z_NULL, inflateInit updates them to
-   use default allocation functions.
-
-     inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
-   version assumed by the caller.  msg is set to null if there is no error
-   message. inflateInit does not perform any decompression apart from reading
-   the zlib header if present: this will be done by inflate().  (So next_in and
-   avail_in may be modified, but next_out and avail_out are unchanged.)
-*/
-
-
-ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
-/*
-    inflate decompresses as much data as possible, and stops when the input
-  buffer becomes empty or the output buffer becomes full. It may some
-  introduce some output latency (reading input without producing any output)
-  except when forced to flush.
-
-  The detailed semantics are as follows. inflate performs one or both of the
-  following actions:
-
-  - Decompress more input starting at next_in and update next_in and avail_in
-    accordingly. If not all input can be processed (because there is not
-    enough room in the output buffer), next_in is updated and processing
-    will resume at this point for the next call of inflate().
-
-  - Provide more output starting at next_out and update next_out and avail_out
-    accordingly.  inflate() provides as much output as possible, until there
-    is no more input data or no more space in the output buffer (see below
-    about the flush parameter).
-
-  Before the call of inflate(), the application should ensure that at least
-  one of the actions is possible, by providing more input and/or consuming
-  more output, and updating the next_* and avail_* values accordingly.
-  The application can consume the uncompressed output when it wants, for
-  example when the output buffer is full (avail_out == 0), or after each
-  call of inflate(). If inflate returns Z_OK and with zero avail_out, it
-  must be called again after making room in the output buffer because there
-  might be more output pending.
-
-    If the parameter flush is set to Z_SYNC_FLUSH, inflate flushes as much
-  output as possible to the output buffer. The flushing behavior of inflate is
-  not specified for values of the flush parameter other than Z_SYNC_FLUSH
-  and Z_FINISH, but the current implementation actually flushes as much output
-  as possible anyway.
-
-    inflate() should normally be called until it returns Z_STREAM_END or an
-  error. However if all decompression is to be performed in a single step
-  (a single call of inflate), the parameter flush should be set to
-  Z_FINISH. In this case all pending input is processed and all pending
-  output is flushed; avail_out must be large enough to hold all the
-  uncompressed data. (The size of the uncompressed data may have been saved
-  by the compressor for this purpose.) The next operation on this stream must
-  be inflateEnd to deallocate the decompression state. The use of Z_FINISH
-  is never required, but can be used to inform inflate that a faster routine
-  may be used for the single inflate() call.
-
-     If a preset dictionary is needed at this point (see inflateSetDictionary
-  below), inflate sets strm-adler to the adler32 checksum of the
-  dictionary chosen by the compressor and returns Z_NEED_DICT; otherwise 
-  it sets strm->adler to the adler32 checksum of all output produced
-  so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or
-  an error code as described below. At the end of the stream, inflate()
-  checks that its computed adler32 checksum is equal to that saved by the
-  compressor and returns Z_STREAM_END only if the checksum is correct.
-
-    inflate() returns Z_OK if some progress has been made (more input processed
-  or more output produced), Z_STREAM_END if the end of the compressed data has
-  been reached and all uncompressed output has been produced, Z_NEED_DICT if a
-  preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
-  corrupted (input stream not conforming to the zlib format or incorrect
-  adler32 checksum), Z_STREAM_ERROR if the stream structure was inconsistent
-  (for example if next_in or next_out was NULL), Z_MEM_ERROR if there was not
-  enough memory, Z_BUF_ERROR if no progress is possible or if there was not
-  enough room in the output buffer when Z_FINISH is used. In the Z_DATA_ERROR
-  case, the application may then call inflateSync to look for a good
-  compression block.
-*/
-
-
-ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
-/*
-     All dynamically allocated data structures for this stream are freed.
-   This function discards any unprocessed input and does not flush any
-   pending output.
-
-     inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
-   was inconsistent. In the error case, msg may be set but then points to a
-   static string (which must not be deallocated).
-*/
-
-                        /* Advanced functions */
-
-/*
-    The following functions are needed only in some special applications.
-*/
-
-/*   
-ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
-                                     int  level,
-                                     int  method,
-                                     int  windowBits,
-                                     int  memLevel,
-                                     int  strategy));
-
-     This is another version of deflateInit with more compression options. The
-   fields next_in, zalloc, zfree and opaque must be initialized before by
-   the caller.
-
-     The method parameter is the compression method. It must be Z_DEFLATED in
-   this version of the library.
-
-     The windowBits parameter is the base two logarithm of the window size
-   (the size of the history buffer).  It should be in the range 8..15 for this
-   version of the library. Larger values of this parameter result in better
-   compression at the expense of memory usage. The default value is 15 if
-   deflateInit is used instead.
-
-     The memLevel parameter specifies how much memory should be allocated
-   for the internal compression state. memLevel=1 uses minimum memory but
-   is slow and reduces compression ratio; memLevel=9 uses maximum memory
-   for optimal speed. The default value is 8. See zconf.h for total memory
-   usage as a function of windowBits and memLevel.
-
-     The strategy parameter is used to tune the compression algorithm. Use the
-   value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
-   filter (or predictor), or Z_HUFFMAN_ONLY to force Huffman encoding only (no
-   string match).  Filtered data consists mostly of small values with a
-   somewhat random distribution. In this case, the compression algorithm is
-   tuned to compress them better. The effect of Z_FILTERED is to force more
-   Huffman coding and less string matching; it is somewhat intermediate
-   between Z_DEFAULT and Z_HUFFMAN_ONLY. The strategy parameter only affects
-   the compression ratio but not the correctness of the compressed output even
-   if it is not set appropriately.
-
-      deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid
-   method). msg is set to null if there is no error message.  deflateInit2 does
-   not perform any compression: this will be done by deflate().
-*/
-                            
-ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
-/*
-     Initializes the compression dictionary from the given byte sequence
-   without producing any compressed output. This function must be called
-   immediately after deflateInit, deflateInit2 or deflateReset, before any
-   call of deflate. The compressor and decompressor must use exactly the same
-   dictionary (see inflateSetDictionary).
-
-     The dictionary should consist of strings (byte sequences) that are likely
-   to be encountered later in the data to be compressed, with the most commonly
-   used strings preferably put towards the end of the dictionary. Using a
-   dictionary is most useful when the data to be compressed is short and can be
-   predicted with good accuracy; the data can then be compressed better than
-   with the default empty dictionary.
-
-     Depending on the size of the compression data structures selected by
-   deflateInit or deflateInit2, a part of the dictionary may in effect be
-   discarded, for example if the dictionary is larger than the window size in
-   deflate or deflate2. Thus the strings most likely to be useful should be
-   put at the end of the dictionary, not at the front.
-
-     Upon return of this function, strm->adler is set to the Adler32 value
-   of the dictionary; the decompressor may later use this value to determine
-   which dictionary has been used by the compressor. (The Adler32 value
-   applies to the whole dictionary even if only a subset of the dictionary is
-   actually used by the compressor.)
-
-     deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
-   parameter is invalid (such as NULL dictionary) or the stream state is
-   inconsistent (for example if deflate has already been called for this stream
-   or if the compression method is bsort). deflateSetDictionary does not
-   perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
-                                    z_streamp source));
-/*
-     Sets the destination stream as a complete copy of the source stream.
-
-     This function can be useful when several compression strategies will be
-   tried, for example when there are several ways of pre-processing the input
-   data with a filter. The streams that will be discarded should then be freed
-   by calling deflateEnd.  Note that deflateCopy duplicates the internal
-   compression state which can be quite large, so this strategy is slow and
-   can consume lots of memory.
-
-     deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
-   (such as zalloc being NULL). msg is left unchanged in both source and
-   destination.
-*/
-
-ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
-/*
-     This function is equivalent to deflateEnd followed by deflateInit,
-   but does not free and reallocate all the internal compression state.
-   The stream will keep the same compression level and any other attributes
-   that may have been set by deflateInit2.
-
-      deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
-				      int level,
-				      int strategy));
-/*
-     Dynamically update the compression level and compression strategy.  The
-   interpretation of level and strategy is as in deflateInit2.  This can be
-   used to switch between compression and straight copy of the input data, or
-   to switch to a different kind of input data requiring a different
-   strategy. If the compression level is changed, the input available so far
-   is compressed with the old level (and may be flushed); the new level will
-   take effect only at the next call of deflate().
-
-     Before the call of deflateParams, the stream state must be set as for
-   a call of deflate(), since the currently available input may have to
-   be compressed and flushed. In particular, strm->avail_out must be non-zero.
-
-     deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
-   stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR
-   if strm->avail_out was zero.
-*/
-
-/*   
-ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
-                                     int  windowBits));
-
-     This is another version of inflateInit with an extra parameter. The
-   fields next_in, avail_in, zalloc, zfree and opaque must be initialized
-   before by the caller.
-
-     The windowBits parameter is the base two logarithm of the maximum window
-   size (the size of the history buffer).  It should be in the range 8..15 for
-   this version of the library. The default value is 15 if inflateInit is used
-   instead. If a compressed stream with a larger window size is given as
-   input, inflate() will return with the error code Z_DATA_ERROR instead of
-   trying to allocate a larger window.
-
-      inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_STREAM_ERROR if a parameter is invalid (such as a negative
-   memLevel). msg is set to null if there is no error message.  inflateInit2
-   does not perform any decompression apart from reading the zlib header if
-   present: this will be done by inflate(). (So next_in and avail_in may be
-   modified, but next_out and avail_out are unchanged.)
-*/
-
-ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
-/*
-     Initializes the decompression dictionary from the given uncompressed byte
-   sequence. This function must be called immediately after a call of inflate
-   if this call returned Z_NEED_DICT. The dictionary chosen by the compressor
-   can be determined from the Adler32 value returned by this call of
-   inflate. The compressor and decompressor must use exactly the same
-   dictionary (see deflateSetDictionary).
-
-     inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
-   parameter is invalid (such as NULL dictionary) or the stream state is
-   inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
-   expected one (incorrect Adler32 value). inflateSetDictionary does not
-   perform any decompression: this will be done by subsequent calls of
-   inflate().
-*/
-
-ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
-/* 
-    Skips invalid compressed data until a full flush point (see above the
-  description of deflate with Z_FULL_FLUSH) can be found, or until all
-  available input is skipped. No output is provided.
-
-    inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
-  if no more input was provided, Z_DATA_ERROR if no flush point has been found,
-  or Z_STREAM_ERROR if the stream structure was inconsistent. In the success
-  case, the application may save the current current value of total_in which
-  indicates where valid compressed data was found. In the error case, the
-  application may repeatedly call inflateSync, providing more input each time,
-  until success or end of the input data.
-*/
-
-ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
-/*
-     This function is equivalent to inflateEnd followed by inflateInit,
-   but does not free and reallocate all the internal decompression state.
-   The stream will keep attributes that may have been set by inflateInit2.
-
-      inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent (such as zalloc or state being NULL).
-*/
-
-
-                        /* utility functions */
-
-/*
-     The following utility functions are implemented on top of the
-   basic stream-oriented functions. To simplify the interface, some
-   default options are assumed (compression level and memory usage,
-   standard memory allocation functions). The source code of these
-   utility functions can easily be modified if you need special options.
-*/
-
-ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
-                                 const Bytef *source, uLong sourceLen));
-/*
-     Compresses the source buffer into the destination buffer.  sourceLen is
-   the byte length of the source buffer. Upon entry, destLen is the total
-   size of the destination buffer, which must be at least 0.1% larger than
-   sourceLen plus 12 bytes. Upon exit, destLen is the actual size of the
-   compressed buffer.
-     This function can be used to compress a whole file at once if the
-   input file is mmap'ed.
-     compress returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_BUF_ERROR if there was not enough room in the output
-   buffer.
-*/
-
-ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
-                                  const Bytef *source, uLong sourceLen,
-                                  int level));
-/*
-     Compresses the source buffer into the destination buffer. The level
-   parameter has the same meaning as in deflateInit.  sourceLen is the byte
-   length of the source buffer. Upon entry, destLen is the total size of the
-   destination buffer, which must be at least 0.1% larger than sourceLen plus
-   12 bytes. Upon exit, destLen is the actual size of the compressed buffer.
-
-     compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_BUF_ERROR if there was not enough room in the output buffer,
-   Z_STREAM_ERROR if the level parameter is invalid.
-*/
-
-ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
-                                   const Bytef *source, uLong sourceLen));
-/*
-     Decompresses the source buffer into the destination buffer.  sourceLen is
-   the byte length of the source buffer. Upon entry, destLen is the total
-   size of the destination buffer, which must be large enough to hold the
-   entire uncompressed data. (The size of the uncompressed data must have
-   been saved previously by the compressor and transmitted to the decompressor
-   by some mechanism outside the scope of this compression library.)
-   Upon exit, destLen is the actual size of the compressed buffer.
-     This function can be used to decompress a whole file at once if the
-   input file is mmap'ed.
-
-     uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_BUF_ERROR if there was not enough room in the output
-   buffer, or Z_DATA_ERROR if the input data was corrupted.
-*/
-
-
-typedef voidp gzFile;
-
-ZEXTERN gzFile ZEXPORT gzopen  OF((const char *path, const char *mode));
-/*
-     Opens a gzip (.gz) file for reading or writing. The mode parameter
-   is as in fopen ("rb" or "wb") but can also include a compression level
-   ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for
-   Huffman only compression as in "wb1h". (See the description
-   of deflateInit2 for more information about the strategy parameter.)
-
-     gzopen can be used to read a file which is not in gzip format; in this
-   case gzread will directly read from the file without decompression.
-
-     gzopen returns NULL if the file could not be opened or if there was
-   insufficient memory to allocate the (de)compression state; errno
-   can be checked to distinguish the two cases (if errno is zero, the
-   zlib error is Z_MEM_ERROR).  */
-
-ZEXTERN gzFile ZEXPORT gzdopen  OF((int fd, const char *mode));
-/*
-     gzdopen() associates a gzFile with the file descriptor fd.  File
-   descriptors are obtained from calls like open, dup, creat, pipe or
-   fileno (in the file has been previously opened with fopen).
-   The mode parameter is as in gzopen.
-     The next call of gzclose on the returned gzFile will also close the
-   file descriptor fd, just like fclose(fdopen(fd), mode) closes the file
-   descriptor fd. If you want to keep fd open, use gzdopen(dup(fd), mode).
-     gzdopen returns NULL if there was insufficient memory to allocate
-   the (de)compression state.
-*/
-
-ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
-/*
-     Dynamically update the compression level or strategy. See the description
-   of deflateInit2 for the meaning of these parameters.
-     gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
-   opened for writing.
-*/
-
-ZEXTERN int ZEXPORT    gzread  OF((gzFile file, voidp buf, unsigned len));
-/*
-     Reads the given number of uncompressed bytes from the compressed file.
-   If the input file was not in gzip format, gzread copies the given number
-   of bytes into the buffer.
-     gzread returns the number of uncompressed bytes actually read (0 for
-   end of file, -1 for error). */
-
-ZEXTERN int ZEXPORT    gzwrite OF((gzFile file, 
-				   const voidp buf, unsigned len));
-/*
-     Writes the given number of uncompressed bytes into the compressed file.
-   gzwrite returns the number of uncompressed bytes actually written
-   (0 in case of error).
-*/
-
-ZEXTERN int ZEXPORTVA   gzprintf OF((gzFile file, const char *format, ...));
-/*
-     Converts, formats, and writes the args to the compressed file under
-   control of the format string, as in fprintf. gzprintf returns the number of
-   uncompressed bytes actually written (0 in case of error).
-*/
-
-ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
-/*
-      Writes the given null-terminated string to the compressed file, excluding
-   the terminating null character.
-      gzputs returns the number of characters written, or -1 in case of error.
-*/
-
-ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
-/*
-      Reads bytes from the compressed file until len-1 characters are read, or
-   a newline character is read and transferred to buf, or an end-of-file
-   condition is encountered.  The string is then terminated with a null
-   character.
-      gzgets returns buf, or Z_NULL in case of error.
-*/
-
-ZEXTERN int ZEXPORT    gzputc OF((gzFile file, int c));
-/*
-      Writes c, converted to an unsigned char, into the compressed file.
-   gzputc returns the value that was written, or -1 in case of error.
-*/
-
-ZEXTERN int ZEXPORT    gzgetc OF((gzFile file));
-/*
-      Reads one byte from the compressed file. gzgetc returns this byte
-   or -1 in case of end of file or error.
-*/
-
-ZEXTERN int ZEXPORT    gzflush OF((gzFile file, int flush));
-/*
-     Flushes all pending output into the compressed file. The parameter
-   flush is as in the deflate() function. The return value is the zlib
-   error number (see function gzerror below). gzflush returns Z_OK if
-   the flush parameter is Z_FINISH and all output could be flushed.
-     gzflush should be called only when strictly necessary because it can
-   degrade compression.
-*/
-
-ZEXTERN z_off_t ZEXPORT    gzseek OF((gzFile file,
-				      z_off_t offset, int whence));
-/* 
-      Sets the starting position for the next gzread or gzwrite on the
-   given compressed file. The offset represents a number of bytes in the
-   uncompressed data stream. The whence parameter is defined as in lseek(2);
-   the value SEEK_END is not supported.
-     If the file is opened for reading, this function is emulated but can be
-   extremely slow. If the file is opened for writing, only forward seeks are
-   supported; gzseek then compresses a sequence of zeroes up to the new
-   starting position.
-
-      gzseek returns the resulting offset location as measured in bytes from
-   the beginning of the uncompressed stream, or -1 in case of error, in
-   particular if the file is opened for writing and the new starting position
-   would be before the current position.
-*/
-
-ZEXTERN int ZEXPORT    gzrewind OF((gzFile file));
-/*
-     Rewinds the given file. This function is supported only for reading.
-
-   gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
-*/
-
-ZEXTERN z_off_t ZEXPORT    gztell OF((gzFile file));
-/*
-     Returns the starting position for the next gzread or gzwrite on the
-   given compressed file. This position represents a number of bytes in the
-   uncompressed data stream.
-
-   gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
-*/
-
-ZEXTERN int ZEXPORT gzeof OF((gzFile file));
-/*
-     Returns 1 when EOF has previously been detected reading the given
-   input stream, otherwise zero.
-*/
-
-ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
-/*
-     Flushes all pending output if necessary, closes the compressed file
-   and deallocates all the (de)compression state. The return value is the zlib
-   error number (see function gzerror below).
-*/
-
-ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
-/*
-     Returns the error message for the last error which occurred on the
-   given compressed file. errnum is set to zlib error number. If an
-   error occurred in the file system and not in the compression library,
-   errnum is set to Z_ERRNO and the application may consult errno
-   to get the exact error code.
-*/
-
-                        /* checksum functions */
-
-/*
-     These functions are not related to compression but are exported
-   anyway because they might be useful in applications using the
-   compression library.
-*/
-
-ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
-
-/*
-     Update a running Adler-32 checksum with the bytes buf[0..len-1] and
-   return the updated checksum. If buf is NULL, this function returns
-   the required initial value for the checksum.
-   An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
-   much faster. Usage example:
-
-     uLong adler = adler32(0L, Z_NULL, 0);
-
-     while (read_buffer(buffer, length) != EOF) {
-       adler = adler32(adler, buffer, length);
-     }
-     if (adler != original_adler) error();
-*/
-
-#if 0
-ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
-/*
-     Update a running crc with the bytes buf[0..len-1] and return the updated
-   crc. If buf is NULL, this function returns the required initial value
-   for the crc. Pre- and post-conditioning (one's complement) is performed
-   within this function so it shouldn't be done by the application.
-   Usage example:
-
-     uLong crc = crc32(0L, Z_NULL, 0);
-
-     while (read_buffer(buffer, length) != EOF) {
-       crc = crc32(crc, buffer, length);
-     }
-     if (crc != original_crc) error();
-*/
-#endif
-
-
-                        /* various hacks, don't look :) */
-
-/* deflateInit and inflateInit are macros to allow checking the zlib version
- * and the compiler's view of z_stream:
- */
-ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
-                                      int windowBits, int memLevel,
-                                      int strategy, const char *version,
-                                      int stream_size));
-ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
-                                      const char *version, int stream_size));
-#define deflateInit(strm, level) \
-        deflateInit_((strm), (level),       ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit(strm) \
-        inflateInit_((strm),                ZLIB_VERSION, sizeof(z_stream))
-#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
-        deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
-                      (strategy),           ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit2(strm, windowBits) \
-        inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
-
-
-#if !defined(_Z_UTIL_H) && !defined(NO_DUMMY_DECL)
-    struct internal_state {int dummy;}; /* hack for buggy compilers */
-#endif
-
-ZEXTERN const char   * ZEXPORT zError           OF((int err));
-ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp z));
-ZEXTERN const uLongf * ZEXPORT get_crc_table    OF((void));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif KERNEL_PRIVATE
-#endif /* _ZLIB_H */
-/* --- zlib.h */
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * This header file is kept for legacy reasons and may be removed in
+ * future; the supported interface resides in <libkern/zlib.h>.
+ */
+#include <libkern/zlib.h>
diff --git a/bsd/netat/Makefile b/bsd/netat/Makefile
index 9a37bee86..501f46d93 100644
--- a/bsd/netat/Makefile
+++ b/bsd/netat/Makefile
@@ -3,6 +3,7 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+CWARNFLAGS += -Wno-sign-compare
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
diff --git a/bsd/netat/adsp.c b/bsd/netat/adsp.c
index 1b88d6750..65fe77206 100644
--- a/bsd/netat/adsp.c
+++ b/bsd/netat/adsp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Change log:
@@ -50,7 +56,7 @@ struct adsp_debug adsp_dtable[1025];
 int ad_entry = 0;
 #endif
 
-
+int
 adspAllocateCCB(gref)
     register gref_t *gref;	/* READ queue */
 {
@@ -71,6 +77,7 @@ adspAllocateCCB(gref)
     return 1;
 }
 
+int
 adspRelease(gref)
     register gref_t *gref;	/* READ queue */
 {
@@ -104,11 +111,12 @@ adspRelease(gref)
 	    sp->state = sClosed;
 	    DoClose(sp, errAborted, 0);  /* to closed and remove CCB */
     } 
+	return 0;
 }
 
 
 
-
+int
 adspWriteHandler(gref, mp)
     gref_t *gref;			/* WRITE queue */
     gbuf_t *mp;
@@ -139,7 +147,7 @@ adspWriteHandler(gref, mp)
 				gbuf_freem(mp);
 				return(STR_IGNORE);
 			case dspAttention:
-				if ((error = adspAttention(sp, ap)))
+				if ((error = adspAttention(sp, (CCBPtr)ap)))
 				gbuf_freem(mp);
 				return(STR_IGNORE);
 		}
@@ -218,7 +226,7 @@ adspWriteHandler(gref, mp)
 				return(STR_IGNORE);
 			case ADSPCLDENY:
 				ap->csCode = dspCLDeny;
-				if ((error = adspCLDeny(sp, ap))) {
+				if ((error = adspCLDeny(sp, (CCBPtr)ap))) {
 				adspioc_ack(error, mp, gref);
 				}
 				return(STR_IGNORE);
@@ -236,7 +244,7 @@ adspWriteHandler(gref, mp)
 				return(STR_IGNORE);
 			case ADSPATTENTION:
 				ap->csCode = dspAttention;
-				if ((error = adspReadAttention(sp, ap))) {
+				if ((error = adspReadAttention((CCBPtr)sp, ap))) {
 				adspioc_ack(error, mp, gref);
 				}
 				return(STR_IGNORE);
@@ -266,9 +274,10 @@ adspWriteHandler(gref, mp)
 		default:
 		gbuf_freem(mp);
     }
+	return(STR_IGNORE);
 }
 
-
+int
 adspReadHandler(gref, mp)
     gref_t *gref;
     gbuf_t *mp;
@@ -320,7 +329,7 @@ adspReadHandler(gref, mp)
  * Side Effects:
  *      NONE
  */
-
+int
 adsp_sendddp(sp, mp, length, dstnetaddr, ddptype)
    CCBPtr sp;
    gbuf_t *mp;
@@ -368,8 +377,8 @@ adsp_sendddp(sp, mp, length, dstnetaddr, ddptype)
    return 0;
 }
 
-void NotifyUser(sp)
-    register CCBPtr sp;
+void NotifyUser(
+	__unused CCBPtr sp)
 
 {
 /*
@@ -377,8 +386,8 @@ void NotifyUser(sp)
 */
 }
 
-void UrgentUser(sp)
-    register CCBPtr sp;
+void UrgentUser(
+    __unused CCBPtr sp)
 {
 /*
     pidsig(sp->pid, SIGURG);
diff --git a/bsd/netat/adsp.h b/bsd/netat/adsp.h
index 315a7decc..7641f8d22 100644
--- a/bsd/netat/adsp.h
+++ b/bsd/netat/adsp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -273,7 +279,7 @@ struct tpb {
 #endif
 */
 
-typedef long (*ProcPtr)();	/* XXX */
+typedef long (*ProcPtr)(void *, void *);	/* XXX */
 typedef ProcPtr *ProcHandle;
 typedef char *Ptr;
 typedef Ptr *Handle;
@@ -692,6 +698,10 @@ extern int adspOptions(CCBPtr sp, struct adspcmd *pb);
 extern int adspReset(CCBPtr sp, struct adspcmd *pb);
 extern int adspNewCID(CCBPtr sp, struct adspcmd *pb);
 extern int adspPacket(gref_t *gref, gbuf_t *mp);
+
+int adsp_open(gref_t *);
+void adsp_input(gbuf_t *);
+
 #undef CCBPtr
 
 
diff --git a/bsd/netat/adsp_CLDeny.c b/bsd/netat/adsp_CLDeny.c
index 62b14b93b..3830fda0b 100644
--- a/bsd/netat/adsp_CLDeny.c
+++ b/bsd/netat/adsp_CLDeny.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * CLDeny.c
@@ -86,7 +92,7 @@ int adspCLDeny(struct adspcmd *pb, CCBPtr sp)
     UAS_ASSIGN_HTON(adspop->version, 0x100);
     adsp_sendddp(sp, mp, 
 		 DDPL_FRAME_LEN + ADSP_FRAME_LEN + ADSP_OPEN_FRAME_LEN, 
-		 &pb->u.openParams.remoteAddress, DDP_ADSP);
-    adspioc_ack(0, pb->ioc, pb->gref);
+		 (AddrUnion *)&pb->u.openParams.remoteAddress, DDP_ADSP);
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     return 0;
 }
diff --git a/bsd/netat/adsp_CLListen.c b/bsd/netat/adsp_CLListen.c
index be7cf9f3d..a7215f094 100644
--- a/bsd/netat/adsp_CLListen.c
+++ b/bsd/netat/adsp_CLListen.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* dspCLListen.c 
  * 
@@ -80,13 +86,13 @@ int adspCLListen(sp, pb)	/* (DSPPBPtr pb) */
 	return EALREADY;
     }
 
-    if (mp = gbuf_copym(pb->mp)) {	/* keep a copy of the parameter block */
+    if ((mp = gbuf_copym(pb->mp))) {	/* keep a copy of the parameter block */
 	    pb->ioResult = 1;	/* not done */
-	    adspioc_ack(0, pb->ioc, pb->gref); /* release user ioctl block */
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); /* release user ioctl block */
 	    clpb = (struct adspcmd *)gbuf_rptr(mp);
 	    clpb->ioc = 0;
 	    clpb->mp = mp;
-	    if (qAddToEnd(&sp->opb, clpb))	/* Add to list of listeners */
+	    if (qAddToEnd((struct qlink **)&sp->opb, (struct qlink *)clpb))	/* Add to list of listeners */
 			return EFAULT; /* bogus, but discriminate from other errors */
     } else {
 	    pb->ioResult = errDSPQueueSize;
diff --git a/bsd/netat/adsp_Close.c b/bsd/netat/adsp_Close.c
index 381e22200..d769015cd 100644
--- a/bsd/netat/adsp_Close.c
+++ b/bsd/netat/adsp_Close.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1995-1998 Apple Computer, Inc.
@@ -119,7 +125,7 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
     register struct adspcmd *p;
     register struct adspcmd *n;
     register gref_t *gref;
-    register int    total = 0;
+    register int    _total = 0;
     CCBPtr sp = 0;
 
     n = *qhead;			/* Get first item */
@@ -132,16 +138,16 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
 	    }
     }
 
-    while (p = n) {		/* while items left */
+    while ((p = n)) {		/* while items left */
 	n = (struct adspcmd *)(p->qLink); /* Save next guy */
 	p->ioResult = code;
 	if (sp) {
 	    completepb(sp, p); 	/* complete the copy of the request */
-	    total++;
+	    _total++;
 	} else
 	    gbuf_freem(p->mp);
     }				/* while */
-    return(total);
+    return(_total);
 }
 
 /*
@@ -156,6 +162,7 @@ int  CompleteQueue(qhead, code)	/* (DSPPBPtr FPTR qhead, OSErr code) */
  * OUTPUTS:
  * 		none
  */
+void RemoveCCB(CCBPtr, struct adspcmd *);
 
 void RemoveCCB(sp, pb)		/* (CCBPtr sp, DSPPBPtr pb) */
     CCBPtr sp;
@@ -173,7 +180,7 @@ void RemoveCCB(sp, pb)		/* (CCBPtr sp, DSPPBPtr pb) */
     if (pb) {
 	pb->ioResult = 0;
 	if (pb->ioc)		/* is this a current or queued request */
-	    adspioc_ack(0, pb->ioc, pb->gref);	/* current */
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);	/* current */
 	else {
 	    completepb(sp, pb);	/* queued */
 	}
@@ -206,18 +213,20 @@ void RemoveCCB(sp, pb)		/* (CCBPtr sp, DSPPBPtr pb) */
 				 * to release our resources too */
 }
 
+int  AbortIO(CCBPtr, short);
+
 int  AbortIO(sp, err)
     CCBPtr sp;
     short err;
 {
-    register int    total;
+    register int    _total;
 
 	if (sp->gref == 0)
 		return 0;
     /*
      * Complete all outstanding transactions.  
      */
-    total = CompleteQueue(&sp->sapb, err); /* Abort outstanding send attentions */
+    _total = CompleteQueue(&sp->sapb, err); /* Abort outstanding send attentions */
     CompleteQueue(&sp->frpb, err); /* Abort outstanding forward resets */
 
     if (sp->sbuf_mb) { /* clear the send queue */
@@ -231,7 +240,7 @@ int  AbortIO(sp, err)
     }
     sp->sData = 0;
     
-    return(total);
+    return(_total);
 }
 
 /*
@@ -251,6 +260,7 @@ int  AbortIO(sp, err)
 void DoClose(sp, err, force_abort)	/* (CCBPtr sp, OSErr err) */
     register CCBPtr sp;
     int err;
+	int force_abort;
 {
     register struct adspcmd *pb, *np;
     register gbuf_t *mp;
@@ -275,7 +285,7 @@ void DoClose(sp, err, force_abort)	/* (CCBPtr sp, OSErr err) */
     np = sp->opb;		/* Get list of close/removes to complete */
     sp->opb = 0;		/* set this list null */
 	
-    while (pb = np) {		/* Handle all of the close/remove param blks */
+    while ((pb = np)) {		/* Handle all of the close/remove param blks */
 	np = (struct adspcmd *)pb->qLink; /* Get next guy (if any) */
 	pb->qLink = 0;
 	pb->ioResult = err;
@@ -308,7 +318,7 @@ void DoClose(sp, err, force_abort)	/* (CCBPtr sp, OSErr err) */
 	/* then fake a read completion to force the notification */
 
 	if (force_abort && aborted_count == 0) {
-	    if (mp = gbuf_alloc(sizeof(struct adspcmd), PRI_HI)) {
+	    if ((mp = gbuf_alloc(sizeof(struct adspcmd), PRI_HI))) {
 	        pb = (struct adspcmd *)gbuf_rptr(mp);
 		gbuf_wset(mp,sizeof(struct adspcmd));
 
@@ -402,13 +412,13 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
 	    sp->state = sClosed;
 	    DoClose(sp, errAborted, 0);
 	    pb->ioResult = 0;
-	    adspioc_ack(0, pb->ioc, pb->gref);
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 	    return 0;
 	}
 		
 	if (sp->state == (word)sClosed)	{ /* Ok to close a closed connection */
 	    pb->ioResult = 0;
-	    adspioc_ack(0, pb->ioc, pb->gref);
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 	    return 0;
 	}
 	if ((sp->state != (word)sOpen) && (sp->state != (word)sClosing)) {
@@ -442,14 +452,14 @@ int adspClose(sp, pb)		/* (DSPPBPtr pb) */
 
     pb->ioResult = 1;
     if ( (mp = gbuf_copym(pb->mp)) ) {	/* duplicate user request */
-	    adspioc_ack(0, pb->ioc, pb->gref); /* release user */
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); /* release user */
 	    pb = (struct adspcmd *)gbuf_rptr(mp); /* get new parameter block */
 	    pb->ioc = 0;
 	    pb->mp = mp;
-	    qAddToEnd(&sp->opb, pb);	/* and save it */
+	    qAddToEnd((struct qlink **)&sp->opb, (struct qlink *)pb);	/* and save it */
     } else {
 	    pb->ioResult = 0;
-	    adspioc_ack(0, pb->ioc, pb->gref); /* release user, and keep no copy
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); /* release user, and keep no copy
 					     * for kernel bookkeeping, yetch!
 					     */
     }
diff --git a/bsd/netat/adsp_Control.c b/bsd/netat/adsp_Control.c
index 44e521158..040683e0b 100644
--- a/bsd/netat/adsp_Control.c
+++ b/bsd/netat/adsp_Control.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1995-1998 Apple Computer, Inc.
@@ -59,6 +65,8 @@
 
 int adsp_window = 1;
 
+int attachData(CCBPtr, gbuf_t *mp);
+
 /*
  * CalcRecvWdw
  *
@@ -75,7 +83,7 @@ int CalcRecvWdw(sp)		/* (CCBPtr sp) */
     bytes = calcRecvQ(sp);
     bytes = sp->rbuflen - bytes; /* get what is left */
 
-    if (bytes <= 16) {		/* %%% this should be zero */
+    if ((bytes <= 16)) {		/* %%% this should be zero */
 	sp->rbufFull = 1;	/* Save flag that our recv buf is full */
 	return 0;
     }
@@ -83,6 +91,7 @@ int CalcRecvWdw(sp)		/* (CCBPtr sp) */
 	return ((bytes+bytes+bytes) >> 2) + 1; /* %%% */
 }
 
+int
 calcRecvQ(sp)
     CCBPtr sp;
 {
@@ -98,13 +107,13 @@ calcRecvQ(sp)
     register gbuf_t *mb;
 
     if (sp->rData) {		/* There is data in buffer */
-	if (mb = sp->rbuf_mb) {
+	if ((mb = sp->rbuf_mb)) {
 	    do {
 		bytes += gbuf_msgsize(mb);
 		mb = gbuf_next(mb);
 	    } while (mb);
 	}
-	if (mb = sp->crbuf_mb)
+	if ((mb = sp->crbuf_mb))
 	    bytes += gbuf_msgsize(mb);
     }
 #endif
@@ -138,7 +147,7 @@ void CheckSend(sp)		/* (CCBPtr sp) */
     int use_attention_code;
     int len;			/* length used in allocd mblk */
     int datalen;		/* amount of data attached to mblk */
-    gbuf_t *mprev, *mlist = 0;
+    gbuf_t *mprev = 0, *mlist = 0;
 
 top:
 
@@ -291,7 +300,7 @@ top:
 					    * in a row. */
     {
 	attnMsg = 0;
-	if (datalen = attachData(sp, mp)) /* attach data to mp */
+	if ((datalen = attachData(sp, mp))) /* attach data to mp */
 	    goto sendit;	/* if successful, sendit */
     }
 
@@ -369,6 +378,8 @@ void completepb(sp, pb)
 	gbuf_freem(pb->mp);
 }
 
+
+int
 attachData(sp, mp)
     register CCBPtr sp;
     register gbuf_t *mp;
@@ -398,9 +409,9 @@ attachData(sp, mp)
      */
     UAL_ASSIGN_HTON(sp->f.pktFirstByteSeq, sp->sendSeq);
 		
-    if (smp = sp->sbuf_mb) /* Get oldest header */
+    if ((smp = sp->sbuf_mb)) /* Get oldest header */
 	eom = 1;
-    else if (smp = sp->csbuf_mb)
+    else if ((smp = sp->csbuf_mb))
 	eom = 0;
 
     if (smp == 0) {		/* this shouldn't happen... */
diff --git a/bsd/netat/adsp_Init.c b/bsd/netat/adsp_Init.c
index 8400cd061..eb79cdf2b 100644
--- a/bsd/netat/adsp_Init.c
+++ b/bsd/netat/adsp_Init.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
@@ -50,6 +56,7 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
+static void InitContinue(CCBPtr, struct adspcmd *);
 
 /*
  * InitContinue
@@ -76,7 +83,7 @@ static void InitContinue(sp, pb) /* (CCBPtr sp, DSPPBPtr pb, int soc) */
     /*
      * Link the new ccb onto queue.  Must be done with interrupts off.
      */
-    qAddToEnd(AT_ADSP_STREAMS, sp); /* Put on linked list of connections */
+    qAddToEnd((struct qlink **)AT_ADSP_STREAMS, (struct qlink *)sp); /* Put on linked list of connections */
     return;
 }
 
@@ -152,6 +159,7 @@ int adspInit(sp, ap)		/* (DSPPBPtr pb) */
 }
 
 
+#if 0
 /*
  * AdspBad
  *
@@ -168,3 +176,5 @@ int AdspBad(ap)			/* (DSPPBPtr pb) */
 	ap->ioResult = controlErr; /* Unknown csCode in the param block */
 	return EINVAL;
 }
+
+#endif
diff --git a/bsd/netat/adsp_InitGlobals.c b/bsd/netat/adsp_InitGlobals.c
index e15b0b74b..8aeeac0c9 100644
--- a/bsd/netat/adsp_InitGlobals.c
+++ b/bsd/netat/adsp_InitGlobals.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* InitGlobals.c 
  *
diff --git a/bsd/netat/adsp_NewCID.c b/bsd/netat/adsp_NewCID.c
index 0e15ee0a8..422f9718a 100644
--- a/bsd/netat/adsp_NewCID.c
+++ b/bsd/netat/adsp_NewCID.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * dspNewCID.c 
@@ -78,6 +84,6 @@ int adspNewCID(sp, pb)		/* (DSPPBPtr pb) */
     sp->locCID = pb->u.newCIDParams.newcid = NextCID();
 
     pb->ioResult = 0;
-    adspioc_ack(0, pb->ioc, pb->gref);
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     return 0;
 }
diff --git a/bsd/netat/adsp_Open.c b/bsd/netat/adsp_Open.c
index f68b624e3..b41ee1198 100644
--- a/bsd/netat/adsp_Open.c
+++ b/bsd/netat/adsp_Open.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* adspOpen.c v01.20
  *
@@ -47,6 +53,8 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
+extern int *adsp_pidM;
+
 
 /*
  * NextCID
@@ -58,7 +66,8 @@
  * OUTPUTS:
  * 		unique connection ID
  */
-unsigned short NextCID()
+
+unsigned short NextCID(void)
 {
 	unsigned short num;
 	register CCB *queue;
@@ -129,8 +138,6 @@ int adspOpen(sp, pb)		/* (DSPPBPtr pb) */
     register CCBPtr sp;
     register struct adspcmd *pb;
 {
-    extern int adsp_pidM[];
-
     int ocMode;
     register gbuf_t *mp;
 
@@ -257,7 +264,7 @@ int adspOpen(sp, pb)		/* (DSPPBPtr pb) */
 	
     if (ocMode == ocEstablish) { /* For establish call, we're done */
 	pb->ioResult = 0;
-	adspioc_ack(0, pb->ioc, pb->gref);
+	adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 	return 0;
     }
     
@@ -269,7 +276,7 @@ int adspOpen(sp, pb)		/* (DSPPBPtr pb) */
 	    return ENOBUFS;
     }
     pb->ioResult = 1;	/* not open -> not done */
-    adspioc_ack(0, pb->ioc, pb->gref); /* release user */
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); /* release user */
     sp->opb = (struct adspcmd *)gbuf_rptr(mp);
     sp->opb->ioc = 0;		/* unlink saved pb from ioctl block */
     sp->opb->mp = mp;
diff --git a/bsd/netat/adsp_Options.c b/bsd/netat/adsp_Options.c
index 5e302bcaa..f40ce433b 100644
--- a/bsd/netat/adsp_Options.c
+++ b/bsd/netat/adsp_Options.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * dspOptions.c 
@@ -66,8 +72,6 @@ int adspOptions(sp, pb)	/* (DSPPBPtr pb) */
     CCBPtr		sp;
     struct adspcmd *pb;
 {
-    short err;
-	
     if (sp == 0) {
 	pb->ioResult = errRefNum; 
 	return EINVAL;
@@ -93,7 +97,7 @@ int adspOptions(sp, pb)	/* (DSPPBPtr pb) */
     if (pb->u.optionParams.newPID)
     	sp->pid = pb->u.optionParams.newPID;
     pb->ioResult = 0;
-    adspioc_ack(0, pb->ioc, pb->gref);
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     return 0;
 
 }
diff --git a/bsd/netat/adsp_Packet.c b/bsd/netat/adsp_Packet.c
index 3293496f0..c73d82e58 100644
--- a/bsd/netat/adsp_Packet.c
+++ b/bsd/netat/adsp_Packet.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Packet.c 
@@ -67,6 +73,8 @@ extern at_ifaddr_t *ifID_home;
  * OUTPUTS:
  *    none
  */
+static void GleanSession(CCBPtr);
+
 static void GleanSession(sp)		/* (CCBPtr sp) */
     CCBPtr sp;
 {
@@ -392,6 +400,7 @@ typedef struct {
  * request/ack/req+ack/deny packet.
  *
  */
+static boolean MatchStream(CCBPtr, MATCHPtr);
 
 static boolean
 MatchStream(sp, m)		/* (CCBPtr sp, MATCHPtr m) */
@@ -496,6 +505,8 @@ MatchStream(sp, m)		/* (CCBPtr sp, MATCHPtr m) */
  *
  */
 
+static boolean MatchListener(CCBPtr, MATCHPtr);
+
 static boolean MatchListener(sp, m) /* (CCBPtr sp, MATCHPtr m) */
     CCBPtr sp;
     MATCHPtr m;
@@ -525,14 +536,13 @@ static boolean MatchListener(sp, m) /* (CCBPtr sp, MATCHPtr m) */
  * OUTPUTS:
  *    Returns 1 if packet was ignored
  */
-static int RXConnection(gref, spPtr, f, len, addr, dsoc) 
-    /* (CCBPtr *spPtr, ADSP_FRAMEPtr f, word len, AddrUnion addr, byte dsoc) */
-    gref_t *gref;			/* READ queue */
-    CCBPtr *spPtr;
-    ADSP_FRAMEPtr f;
-    int len;
-    AddrUnion addr;
-    unsigned char dsoc;
+static int RXConnection(
+    __unused gref_t *gref,			/* READ queue */
+    CCBPtr *spPtr,
+    ADSP_FRAMEPtr f,
+    int len,
+    AddrUnion addr,
+    unsigned char dsoc)
 {
     CCBPtr sp;
     ADSP_OPEN_DATAPtr op;
@@ -582,7 +592,7 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
     /*
      * See if we can find a stream that knows what to do with this packet
      */
-    if ((sp = (CCBPtr)qfind_m(AT_ADSP_STREAMS, &m, (ProcPtr)MatchStream)) == 0)
+    if ((sp = (CCBPtr)qfind_m((CCB *)AT_ADSP_STREAMS, &m, (ProcPtr)MatchStream)) == 0)
     {
 	struct adspcmd *p;
 	struct adspcmd *n;
@@ -593,12 +603,12 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
 	if ((f->descriptor & ADSP_CONTROL_MASK) != (byte)ADSP_CTL_OREQ)
 	    return 1;
 
-	if ((sp = (CCBPtr)qfind_m(AT_ADSP_STREAMS, &m, 
+	if ((sp = (CCBPtr)qfind_m((CCB *)AT_ADSP_STREAMS, &m, 
 				  (ProcPtr)MatchListener)) == 0)
 	    return 1;
 
 	p = (struct adspcmd *)&sp->opb;
-	while (n = (struct adspcmd *)p->qLink) /* Hunt down list of listens */
+	while ((n = (struct adspcmd *)p->qLink)) /* Hunt down list of listens */
 	{
 	    /* Check address filter */
 	    if (((n->u.openParams.filterAddress.net == 0) ||
@@ -669,7 +679,7 @@ static int RXConnection(gref, spPtr, f, len, addr, dsoc)
     if (m.t->action & A_COMPLETE) { /* Need to complete open param blk */
         RemoveTimerElem(&adspGlobal.slowTimers, &sp->ProbeTimer);
 			
-	if (pb = sp->opb) {
+	if ((pb = sp->opb)) {
 	    sp->opb = 0;
 	    pb->u.openParams.localCID = sp->locCID;
 	    pb->u.openParams.remoteCID = sp->remCID;
@@ -741,7 +751,7 @@ int adspPacket(gref, mp)
 
     dsoc = ddp->ddpx_dest;
 
-    if (sp = (CCBPtr)FindSender(f, a))
+    if ((sp = (CCBPtr)FindSender(f, a)))
 	GleanSession(sp);
 
     if (f->descriptor & ADSP_ATTENTION_BIT) { /* ATTN packet */
@@ -815,7 +825,7 @@ int adspPacket(gref, mp)
     if (mp)
 	gbuf_freem(mp);
 
-checksend:			/* incoming data was not ignored */
+	/* incoming data was not ignored */
     if (sp && sp->callSend)	/* If we have a stream & we need to send */
 	CheckSend(sp);
     
diff --git a/bsd/netat/adsp_Read.c b/bsd/netat/adsp_Read.c
index 6c6c2390c..64f097b26 100644
--- a/bsd/netat/adsp_Read.c
+++ b/bsd/netat/adsp_Read.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -74,7 +80,7 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 
     while (sp->rData && (pb = sp->rpb)) {		/* have data */
         dPrintf(D_M_ADSP, D_L_TRACE, 
-		 (" pb=0x%x, gref=0x%x, ioc=0x%x, reqCount=%d (have data)\n", 
+		 (" pb=0x%p, gref=0x%p, ioc=0x%p, reqCount=%d (have data)\n", 
 		  pb, pb->gref, pb->ioc, pb->u.ioParams.reqCount));
     	KERNEL_DEBUG(DBG_ADSP_READ, 1, pb, pb->gref, pb->ioc, pb->u.ioParams.reqCount);
 	if (pb->u.ioParams.reqCount == 0) {
@@ -82,7 +88,7 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 	    sp->rpb = pb->qLink;
 	    if (pb->ioc) {
     		KERNEL_DEBUG(DBG_ADSP_READ, 2, pb, pb->gref, pb->ioc, 0); 
-		adspioc_ack(0, pb->ioc, pb->gref);
+		adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 	    } else {
     		KERNEL_DEBUG(DBG_ADSP_READ, 3, pb, pb->gref, 0, 0);
 		completepb(sp, pb);
@@ -91,12 +97,12 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 	}
 	    
 	/* take the first packet off of sp->rbuf_mb or sp->crbuf_mb */
-	if (mp = sp->rbuf_mb) {	/* Get header for oldest data */
+	if ((mp = sp->rbuf_mb)) {	/* Get header for oldest data */
     	    KERNEL_DEBUG(DBG_ADSP_READ, 4, pb, mp, gbuf_msgsize(mp), gbuf_next(mp));
 	    sp->rbuf_mb = gbuf_next(mp);
 	    gbuf_next(mp) = 0;
 	    eom = 1;
-	} else if (mp = sp->crbuf_mb) {
+	} else if ((mp = sp->crbuf_mb)) {
     	    KERNEL_DEBUG(DBG_ADSP_READ, 5, pb, mp, gbuf_msgsize(mp), gbuf_next(mp));
 	    sp->crbuf_mb = 0;
 	    eom = 0;
@@ -150,8 +156,8 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 		mp = gbuf_cont(pb->mp); /* ioctl call */
 		gbuf_cont(pb->mp) = 0;
 		gref = (gref_t *)pb->gref;
-		adspioc_ack(0, pb->ioc, pb->gref);
-		dPrintf(D_M_ADSP, D_L_TRACE, ("    (pb->ioc) mp=%x\n", mp));
+		adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
+		dPrintf(D_M_ADSP, D_L_TRACE, ("    (pb->ioc) mp=%p\n", mp));
     		KERNEL_DEBUG(DBG_ADSP_READ, 0x0A, pb,  mp, 
 			     gbuf_next(mp), gbuf_cont(mp));
 		SndMsgUp(gref, mp);
@@ -167,9 +173,9 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 	}
     }	/* while */
 
-    if (pb = sp->rpb) {		/* if there is an outstanding request */
+    if ((pb = sp->rpb)) {		/* if there is an outstanding request */
         dPrintf(D_M_ADSP, D_L_TRACE, 
-		 (" pb=0x%x, ioc=0x%x, reqCount=%d (no more data)\n", 
+		 (" pb=0x%p, ioc=0x%p, reqCount=%d (no more data)\n", 
 		  pb, pb->ioc, pb->u.ioParams.reqCount));
     	KERNEL_DEBUG(DBG_ADSP_READ, 0x0D, pb, pb->ioc, 
 		     pb->u.ioParams.reqCount, pb->u.ioParams.actCount);
@@ -182,7 +188,7 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 		    pb->u.ioParams.eom = 0;
 		    sp->rpb = pb->qLink;
 		    if (pb->ioc) {
-			    adspioc_ack(0, pb->ioc, pb->gref);
+			    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 		    } else {
 			    completepb(sp, pb);
 		    }
@@ -194,9 +200,9 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 	    pb->ioResult = 1;
 	    tmp = gbuf_cont(pb->mp); /* detatch perhaps delayed data */
 	    gbuf_cont(pb->mp) = 0;
-	    if (mp = gbuf_copym(pb->mp)) { /* otherwise, duplicate user request */
+	    if ((mp = gbuf_copym(pb->mp))) { /* otherwise, duplicate user request */
     		    KERNEL_DEBUG(DBG_ADSP_READ, 0x0F, pb, sp, pb->mp, 0);
-		    adspioc_ack(0, pb->ioc, pb->gref); 	/* release user */
+		    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); 	/* release user */
 		    pb = (struct adspcmd *)gbuf_rptr(mp); /* get new parameter block */
 		    pb->ioc = 0;
 		    pb->mp = mp;
@@ -215,7 +221,7 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
 			    sp->rData = 1;
 		    }
 		    pb->ioResult = errDSPQueueSize;
-		    adspioc_ack(ENOBUFS, pb->ioc, pb->gref);
+		    adspioc_ack(ENOBUFS, (gbuf_t *)pb->ioc, pb->gref);
 	    }
 	} 
     }
@@ -250,17 +256,19 @@ int CheckReadQueue(sp)		/* (CCBPtr sp) */
  * OUTPUTS:
  * 	
  */
+int CheckAttn(CCBPtr, struct adspcmd *); 
+
 int CheckAttn(sp, pb)		/* (CCBPtr sp) */
     register CCBPtr sp;
     register struct adspcmd *pb;
 {
     gbuf_t *mp;
-    gref_t *gref;
+    gref_t *gref = 0;
 	
     dPrintf(D_M_ADSP, D_L_TRACE, 
 	    ("CheckAttn: sp=0x%x, pb=0x%x\n", (unsigned)sp, (unsigned)pb));
 
-    if (mp = sp->attn_mb) {
+    if ((mp = sp->attn_mb)) {
 
 	/*
 	 * Deliver the attention data to the user. 
@@ -287,7 +295,7 @@ int CheckAttn(sp, pb)		/* (CCBPtr sp) */
 	pb->u.attnParams.attnCode = 0;
 	pb->ioResult = 1;	/* not done */
     }
-    adspioc_ack(0, pb->ioc, pb->gref);
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     if (mp) {
 	SndMsgUp(gref, mp);
 	}
@@ -335,21 +343,21 @@ int adspRead(sp, pb)		/* (DSPPBPtr pb) */
 	return EINVAL;
     }
     if (sp->rData && (sp->rpb == 0)) { /* if data, and no queue of pbs */
-	qAddToEnd(&sp->rpb, pb); /* deliver data to user directly */
+	qAddToEnd((struct qlink **)&sp->rpb, (struct qlink *)pb); /* deliver data to user directly */
 	CheckReadQueue(sp);
     } else if ((pb->u.ioParams.reqCount == 0) && (sp->rpb == 0)) {
 	    /* empty read */
 	    pb->ioResult = 0;
-	    adspioc_ack(0, pb->ioc, pb->gref);
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
 	    return 0;
     } else {
 	pb->ioResult = 1;
-	if (mp = gbuf_copym(pb->mp)) { /* otherwise, duplicate user request */
-		adspioc_ack(0, pb->ioc, pb->gref); 	/* release user */
+	if ((mp = gbuf_copym(pb->mp))) { /* otherwise, duplicate user request */
+		adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); 	/* release user */
 		pb = (struct adspcmd *)gbuf_rptr(mp); 	/* get new parameter block */
 		pb->ioc = 0;
 		pb->mp = mp;
-		qAddToEnd(&sp->rpb, pb); /* and queue it for later */
+		qAddToEnd((struct qlink **)&sp->rpb, (struct qlink *)pb); /* and queue it for later */
 	} else {
 		pb->ioResult = errDSPQueueSize;
 		return ENOBUFS;
diff --git a/bsd/netat/adsp_RxAttn.c b/bsd/netat/adsp_RxAttn.c
index c4792ddb6..d912492f3 100644
--- a/bsd/netat/adsp_RxAttn.c
+++ b/bsd/netat/adsp_RxAttn.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * RxAttn.c 
@@ -60,6 +66,7 @@ typedef struct
  * MatchSender
  *
  */
+static boolean MatchSender(CCBPtr, MATCH_SENDERPtr);
 
 static boolean MatchSender(sp, m) /* (CCBPtr sp, MATCH_SENDERPtr m) */
     CCBPtr sp;
@@ -105,7 +112,7 @@ CCBPtr FindSender(f, a)		/* (ADSP_FRAMEPtr f, AddrUnion a) */
 
     m.addr = a;
     m.srcCID = UAS_VALUE_NTOH(f->CID);
-    return (CCBPtr)qfind_m(AT_ADSP_STREAMS, &m, (ProcPtr)MatchSender);
+    return (CCBPtr)qfind_m((CCB *)AT_ADSP_STREAMS, &m, (ProcPtr)MatchSender);
 }
 
 /*
diff --git a/bsd/netat/adsp_RxData.c b/bsd/netat/adsp_RxData.c
index 02d732044..267e8c313 100644
--- a/bsd/netat/adsp_RxData.c
+++ b/bsd/netat/adsp_RxData.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * RxData.c 
@@ -52,6 +58,8 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
+gbuf_t *releaseData(gbuf_t *, int);
+
 gbuf_t *releaseData(mp, len)
     gbuf_t *mp;
     int len;
@@ -176,7 +184,7 @@ void CheckRecvSeq(sp, f)	/* (CCBPtr sp, ADSP_FRAMEPtr f) */
 
 
     do {			/* This acks bytes in our buffer */
-	if (mp = sp->sbuf_mb) {	/* Get ptr to oldest data header */
+	if ((mp = sp->sbuf_mb)) {	/* Get ptr to oldest data header */
 	    sp->sbuf_mb = gbuf_next(mp); /* unlink it from send queue */
 	    eom = 1;
 	} else {
@@ -354,7 +362,7 @@ int RXData(sp, mp, f, len)	/* (CCBPtr sp, ADSP_FRAMEPtr f, word len) */
 		mp = sp->crbuf_mb;
 		sp->crbuf_mb = 0;
 	    }
-	    if (rmp = sp->rbuf_mb) {
+	    if ((rmp = sp->rbuf_mb)) {
 		/*
 		 * Add it to the end
 		 */
diff --git a/bsd/netat/adsp_Status.c b/bsd/netat/adsp_Status.c
index 4ec89730a..4643504b9 100644
--- a/bsd/netat/adsp_Status.c
+++ b/bsd/netat/adsp_Status.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * dspStatus.c 
@@ -45,6 +51,8 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
+int calcSendQ(CCBPtr);
+
 /*
  * calcSendFree
  *
@@ -66,6 +74,7 @@ int CalcSendQFree(sp)		/* (CCBPtr sp) */
     return bytes;
 }
 
+int
 calcSendQ(sp)
     CCBPtr sp;
 {
@@ -73,13 +82,13 @@ calcSendQ(sp)
     int bytes = 0;
 
     if (sp->sData) {		/* There is data in buffer */
-	if (mp = sp->sbuf_mb) {
+	if ((mp = sp->sbuf_mb)) {
 	    do {
 		bytes += gbuf_msgsize(mp);
 		mp = gbuf_next(mp);
 	    } while (mp);
 	}
-	if (mp = sp->csbuf_mb)
+	if ((mp = sp->csbuf_mb))
 	    bytes += gbuf_msgsize(mp);
     }
     return bytes;
@@ -106,7 +115,6 @@ int adspStatus(sp, pb)	/* (DSPPBPtr pb) */
     CCBPtr sp;
     register struct adspcmd *pb;
 {
-    short err;
     short bytes;
 
     if (sp == 0) {
@@ -141,7 +149,7 @@ int adspStatus(sp, pb)	/* (DSPPBPtr pb) */
     pb->u.statusParams.recvQFree = CalcRecvWdw(sp);
 
     pb->ioResult = 0;
-    adspioc_ack(0, pb->ioc, pb->gref);
+    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     return 0;
 
 }
diff --git a/bsd/netat/adsp_Timer.c b/bsd/netat/adsp_Timer.c
index db7f045ac..455ab5e33 100644
--- a/bsd/netat/adsp_Timer.c
+++ b/bsd/netat/adsp_Timer.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
@@ -45,14 +51,16 @@
 #include <sys/socket.h>
 #include <sys/time.h>
 
+#include <net/if.h>
+
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
 #include <netat/at_pcb.h>
+#include <netat/at_var.h>
 #include <netat/debug.h>
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-void TimerTick();
 
 /*
  * TrashSession
@@ -67,6 +75,8 @@ void TimerTick();
  * OUTPUTS:
  * 		none
  */
+void TrashSession(CCBPtr);
+
 void TrashSession(sp)		/* (CCBPtr sp) */
     CCBPtr sp;
 {
@@ -87,6 +97,8 @@ void TrashSession(sp)		/* (CCBPtr sp) */
  * OUTPUTS:
  * 	
  */
+void DoTimerElem(TimerElemPtr);
+
 void DoTimerElem(t) /* (TimerElemPtr t) */
     TimerElemPtr t;
 {
@@ -168,14 +180,16 @@ send:
     CheckSend(sp);
 }
 
-void TimerTick_funnel(void *arg)
+void TimerTick_funnel(void *arg);
+
+void TimerTick_funnel(__unused void *arg)
 {
 	atalk_lock();
 	TimerTick();
 	atalk_unlock();
 }
 
-static StopTimer;
+static int StopTimer;
 
 /*
  * TimerTick
diff --git a/bsd/netat/adsp_TimerElem.c b/bsd/netat/adsp_TimerElem.c
index 2ed056d3b..dd04ee244 100644
--- a/bsd/netat/adsp_TimerElem.c
+++ b/bsd/netat/adsp_TimerElem.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * TimerElem.c 
@@ -47,7 +53,7 @@
 #include <netat/adsp_internal.h>
 
 
-extern	void DoTimerElem();	/* (TimerElemPtr t);  
+extern	void DoTimerElem(TimerElemPtr);	/* (TimerElemPtr t);  
 				 * External routine called to 
 				 * process each one. */
 
@@ -78,7 +84,7 @@ void InsertTimerElem(qhead, t, val)
     }
     p = (TimerElemPtr)qhead;
 
-    while (n = p->link) {
+    while ((n = p->link)) {
 	if (val <= n->timer)	/* Do we go in front of this? */
 	{
 	    n->timer -= val;	/* Yes, adjust his delta */
@@ -123,11 +129,11 @@ void RemoveTimerElem(qhead, t)	/* (TimerElemPtr *qhead, TimerElemPtr t) */
     }
     p = (TimerElemPtr)qhead;
 
-    while (n = p->link)	/* Get next item in queue */
+    while ((n = p->link))	/* Get next item in queue */
     {
 	if (n == t)		/* Is it us? */
 	{
-	    if (p->link = n->link) /* Link our parent to our child */
+	    if ((p->link = n->link)) /* Link our parent to our child */
 	    {
 		n->link->timer += t->timer; /* and update child's timer */
 	    }
diff --git a/bsd/netat/adsp_Write.c b/bsd/netat/adsp_Write.c
index 4d2b1a632..acc5ad4e9 100644
--- a/bsd/netat/adsp_Write.c
+++ b/bsd/netat/adsp_Write.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* dspWrite.c 
  * From Mike Shoemaker v01.13 06/21/90 mbs for MacOS
@@ -49,7 +55,8 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-void completepb();
+
+int FillSendQueue(CCBPtr, struct adspcmd *);
 
 /*
  * FillSendQueue
@@ -59,9 +66,9 @@ void completepb();
  * OUTPUTS:
  * 		none
  */
-int FillSendQueue(sp, pb)		/* (CCBPtr sp) */
-    register CCBPtr sp;
-    register struct adspcmd *pb; /* The write PB we're playing with */
+int FillSendQueue(		/* (CCBPtr sp) */
+    register CCBPtr sp,
+    register struct adspcmd *pb) /* The write PB we're playing with */
 {
 	gbuf_t *mb, *nmb;
 	int eom;		/* True if should set eom in header */
diff --git a/bsd/netat/adsp_attention.c b/bsd/netat/adsp_attention.c
index 726692d45..095780c1e 100644
--- a/bsd/netat/adsp_attention.c
+++ b/bsd/netat/adsp_attention.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * dspAttention.c 
@@ -115,7 +121,7 @@ l_err:
     }
     pb->ioDirection = 1;	/* outgoing attention data */
     if (sp->sapb) {		/* Pending attentions already? */
-		qAddToEnd(&sp->sapb, pb); /* Just add to end of queue */
+		qAddToEnd((struct qlink **)&sp->sapb, (struct qlink *)pb); /* Just add to end of queue */
     } else {
 		sp->sendAttnData = 1;	/* Start off this attention */
 		pb->qLink = 0;
diff --git a/bsd/netat/adsp_internal.h b/bsd/netat/adsp_internal.h
index 129a6197d..a26ff49af 100644
--- a/bsd/netat/adsp_internal.h
+++ b/bsd/netat/adsp_internal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _NETAT_ADSP_INTERNAL_H_
 #define _NETAT_ADSP_INTERNAL_H_
@@ -349,6 +355,56 @@ extern GLOBAL adspGlobal;
 /* Address of ptr to list of ccb's */
 #define AT_ADSP_STREAMS ((CCB **)&(adspGlobal.ccbList))
 
+void CheckSend(CCBPtr);
+
+struct qlink {
+	struct qlink *qlinkp;
+};
+
+int qAddToEnd(struct qlink **, struct qlink *);
+
+void adspioc_ack(int, gbuf_t *, gref_t *);
+int CalcRecvWdw(CCBPtr);
+int calcRecvQ(CCBPtr);
+int CalcSendQFree(CCBPtr);
+int adsp_sendddp(CCBPtr, gbuf_t *, int, AddrUnion *, int);
+int CheckReadQueue(CCBPtr);
+int CheckOkToClose(CCBPtr);
+
+int RXData(CCBPtr, gbuf_t *, ADSP_FRAMEPtr, int);
+int RXFResetAck(CCBPtr, ADSP_FRAMEPtr);
+int RxClose(CCBPtr);
+void CheckRecvSeq(CCBPtr, ADSP_FRAMEPtr);
+int RXFReset(CCBPtr, ADSP_FRAMEPtr);
+int RXAttention(CCBPtr, gbuf_t *, ADSP_FRAMEPtr, int);
+CCBPtr FindSender(ADSP_FRAMEPtr, AddrUnion);
+void DoClose(CCBPtr, int, int);
+void completepb(CCBPtr, struct adspcmd *);
+int adspReadAttention(CCBPtr, struct adspcmd *);
+int adspMode(struct adspcmd *);
+int CompleteQueue(struct adspcmd **, int);
+
+void CleanupGlobals(void);
+void InitGlobals(void);
+void TimerStop(void);
+void TimerTick(void);
+
+void SndMsgUp(gref_t *, gbuf_t *);
+int adspDeassignSocket(CCBPtr);
+unsigned char adspAssignSocket(gref_t *gref, int);
+int adspWriteHandler(gref_t *, gbuf_t *);
+int adspReadHandler(gref_t *, gbuf_t *);
+
+int adsp_wput(gref_t *gref, gbuf_t *m);
+int adspRelease(gref_t *);
+int adsp_close(gref_t *);
+int adspAllocateCCB(gref_t *);
+
+void NotifyUser(CCBPtr);
+void UrgentUser(CCBPtr);
+
+unsigned short NextCID(void);
+
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
 
diff --git a/bsd/netat/adsp_misc.c b/bsd/netat/adsp_misc.c
index 2b4efb778..f546b3160 100644
--- a/bsd/netat/adsp_misc.c
+++ b/bsd/netat/adsp_misc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <sys/errno.h>
 #include <sys/types.h>
@@ -46,10 +52,6 @@
  */
 
 
-struct qlink {
-	struct qlink *qlinkp;
-};
-
 /* ----------------------------------------------------------------------
  * void qAddToEnd(void *qhead, void *qelem)
  * 
diff --git a/bsd/netat/adsp_reset.c b/bsd/netat/adsp_reset.c
index 6f19c238e..f52f42ab5 100644
--- a/bsd/netat/adsp_reset.c
+++ b/bsd/netat/adsp_reset.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Reset.c
@@ -83,7 +89,7 @@ int RXFReset(sp, f)		/* (CCBPtr sp, ADSP_FRAMEPtr f) */
     if (BETWEEN(sp->recvSeq, pktFirstByteSeq, hi)) /* Is this acceptable? */
     {
 	sp->recvSeq = pktFirstByteSeq;
-	while (mp = sp->rbuf_mb) { /* clear the receive queue */
+	while ((mp = sp->rbuf_mb)) { /* clear the receive queue */
 	    sp->rbuf_mb = gbuf_next(mp);
 	    gbuf_freem(mp);
 	}
@@ -201,7 +207,7 @@ int adspReset(sp, pb)		/* (DSPPBPtr pb) */
     }
 	
 
-    while (mp = sp->sbuf_mb) { /* clear the send queue */
+    while ((mp = sp->sbuf_mb)) { /* clear the send queue */
 	sp->sbuf_mb = gbuf_next(mp);
 	gbuf_freem(mp);
     }
@@ -214,18 +220,18 @@ int adspReset(sp, pb)		/* (DSPPBPtr pb) */
     sp->sendCtl |= B_CTL_FRESET;
 
     sp->firstRtmtSeq = sp->sendSeq; /* Reset sequence #'s */
-    if (mp = gbuf_copym(pb->mp)) {	/* copy the parameter block */
-	    adspioc_ack(0, pb->ioc, pb->gref); /* release user */
+    if ((mp = gbuf_copym(pb->mp))) {	/* copy the parameter block */
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref); /* release user */
 	    rpb = (struct adspcmd *)gbuf_rptr(mp);
 	    rpb->ioc = 0;		/* unlink copy */
 	    rpb->mp = mp;
 
-	    qAddToEnd(&sp->frpb, rpb); 
+	    qAddToEnd((struct qlink **)&sp->frpb, (struct qlink *)rpb); 
 				/* Hold on to pb (will be completed when */
 				/* forward reset ack is received). */
     } else {			/* assume it will work... but keep no
 				 * bookkeeping for it.  yetch! */
-	    adspioc_ack(0, pb->ioc, pb->gref);
+	    adspioc_ack(0, (gbuf_t *)pb->ioc, pb->gref);
     }
 
     CheckSend(sp);
diff --git a/bsd/netat/adsp_stream.c b/bsd/netat/adsp_stream.c
index 424eb65b1..26ad46569 100644
--- a/bsd/netat/adsp_stream.c
+++ b/bsd/netat/adsp_stream.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1995-1998 Apple Computer, Inc.
@@ -57,13 +63,10 @@
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
 
-void SndMsgUp();
-void adsp_rput();
-static void adsp_iocack();
-static void adsp_iocnak();
-void adsp_dequeue_ccb();
-unsigned char adspAssignSocket();
-int adspallocate(), adsprelease();
+void adsp_rput(gref_t *, gbuf_t *);
+static void adsp_iocack(gref_t *, gbuf_t *);
+static void adsp_iocnak(gref_t *, gbuf_t *, int err);
+void adsp_dequeue_ccb(CCB *);
 int adspInited = 0;
 
 GLOBAL adspGlobal;
@@ -148,6 +151,8 @@ void adsp_input(mp)
 }
 
 /**********/
+int adsp_readable(gref_t *);
+
 int adsp_readable(gref)
 	gref_t *gref;
 {
@@ -172,6 +177,7 @@ int adsp_readable(gref)
 	return rc;
 }
 
+int adsp_writeable(gref_t *);
 int adsp_writeable(gref)
 	gref_t *gref;
 {
@@ -196,7 +202,9 @@ int adsp_writeable(gref)
 	return rc;
 }
 
-static void adsp_init()
+static void adsp_init(void);
+
+static void adsp_init(void)
 {
 	adspInited++;
 	InitGlobals();
@@ -249,7 +257,7 @@ int adsp_close(gref)
 		adsp_dequeue_ccb(sp);
 		gbuf_freeb((gbuf_t *)gref->info);
 	}
-  } else
+  }
     return 0;
 }
 
@@ -284,8 +292,8 @@ void adsp_rput(gref, mp)
 #endif
 	/* fall through */
   default:
-	CheckReadQueue(gbuf_rptr(((gbuf_t *)gref->info)));
-	CheckSend(gbuf_rptr(((gbuf_t *)gref->info)));
+	CheckReadQueue((CCBPtr)gbuf_rptr(((gbuf_t *)gref->info)));
+	CheckSend((CCBPtr)gbuf_rptr(((gbuf_t *)gref->info)));
 
     	switch (gbuf_type(mp)) {
 	case MSG_IOCTL:
@@ -507,7 +515,7 @@ adspAssignSocket(gref, flag)
 	gref_t *gref;
 	int flag;
 {
-	unsigned char sVal, sMax, sMin, sSav, inputC;
+	unsigned char sVal, sMax, sMin, sSav = 0, inputC;
 	CCBPtr sp;
 
 	sMax = flag ? DDP_SOCKET_LAST-46 : DDP_SOCKET_LAST-6;
diff --git a/bsd/netat/appletalk.h b/bsd/netat/appletalk.h
index 496f8af55..db08c693d 100644
--- a/bsd/netat/appletalk.h
+++ b/bsd/netat/appletalk.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
diff --git a/bsd/netat/asp.h b/bsd/netat/asp.h
index fd7d85c24..ab0c32173 100644
--- a/bsd/netat/asp.h
+++ b/bsd/netat/asp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -162,6 +168,9 @@ typedef struct {
 	unsigned short param2;
 } asp_word_t;
 
+struct asp_scb;
+typedef void (*asp_tmo_func)(struct asp_scb *);
+
 /*
  * ASP session control block
  */
@@ -196,7 +205,7 @@ typedef struct asp_scb {
 	at_inet_t svc_addr;
 	gbuf_t *sess_ioc;
 	gbuf_t *stat_msg;
-	void (*tmo_func)();
+	asp_tmo_func tmo_func;
 	struct asp_scb *next_tmo;
 	struct asp_scb *prev_tmo;
 	struct asp_scb *sess_scb;
@@ -215,6 +224,19 @@ typedef struct asp_scb {
 	atevent_t delay_event;
 } asp_scb_t;
 
+
+int ASPgetmsg(gref_t *, strbuf_t *, strbuf_t *, gbuf_t **,  int *, int *);
+int ASPputmsg(gref_t *, strbuf_t *, strbuf_t *, gbuf_t *, int , int *);
+void asp_init(void);
+void asp_ack_reply(gref_t *, gbuf_t *);
+void asp_nak_reply(gref_t *, gbuf_t *);
+int asp_wput(gref_t *, gbuf_t *);
+void asp_clock(void *);
+void asp_clock_locked(void *);
+int asp_open(gref_t *);
+int asp_close(gref_t *);
+	
+
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_ASP_H_ */
diff --git a/bsd/netat/asp_proto.c b/bsd/netat/asp_proto.c
index 7c43f147f..7f12b749a 100644
--- a/bsd/netat/asp_proto.c
+++ b/bsd/netat/asp_proto.c
@@ -1,27 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- *	Copyright (c) 1995 Apple Computer, Inc. 
- *
  *  Change Log:
  *    Created February 20, 1995 by Tuyen Nguyen
  *    Modified for MP, 1996 by Tuyen Nguyen
@@ -46,8 +50,8 @@
 
 #include <net/if.h>
 
-#include <netat/appletalk.h>
 #include <netat/sysglue.h>
+#include <netat/appletalk.h>
 #include <netat/at_pcb.h>
 #include <netat/atp.h>
 #include <netat/ddp.h>
@@ -55,13 +59,6 @@
 #include <netat/at_var.h>
 #include <netat/debug.h>
 
-static int loop_cnt;
-#define CHK_LOOP(str) { \
-  if (loop_cnt++ > 100) { \
-     kprintf("%s", str); \
-     break; \
-  } \
-}
 
 #define atpBDSsize    (sizeof(struct atpBDS)*ATP_TRESP_MAX)
 #define aspCMDsize    (atpBDSsize+sizeof(struct atp_set_default)+TOTAL_ATP_HDR_SIZE)
@@ -76,31 +73,22 @@ static int loop_cnt;
 	&& ((scb->rem_addr.net != addr.net) \
 		|| (scb->rem_addr.node != addr.node)) )
 
-int  ASPputmsg();
-int  ASPgetmsg();
-void asp_init();
-void asp_ack_reply();
-void asp_nak_reply();
-void asp_clock();
-void asp_clock_locked(void *);
-int  asp_open();
-int  asp_close();
-int  asp_wput();
-StaticProc asp_scb_t *asp_find_scb();
-StaticProc asp_scb_t *asp_scb_alloc();
-
-StaticProc void asp_putnext();
-StaticProc void asp_iocack();
-StaticProc void asp_iocnak();
-StaticProc void asp_dequeue_scb();
-StaticProc void asp_scb_free();
-StaticProc void asp_timout();
-StaticProc void asp_untimout();
-StaticProc void asp_hangup();
-StaticProc void asp_send_tickle();
+StaticProc asp_scb_t *asp_find_scb(unsigned char, at_inet_t *);
+StaticProc asp_scb_t *asp_scb_alloc(void);
+
+StaticProc void asp_putnext(gref_t *, gbuf_t *);
+StaticProc void asp_iocack(gref_t *, gbuf_t *);
+StaticProc void asp_iocnak(gref_t *, gbuf_t *, int);
+StaticProc void asp_dequeue_scb(asp_scb_t *);
+StaticProc void asp_scb_free(asp_scb_t *);
+StaticProc void asp_timout(asp_tmo_func,  asp_scb_t *, int);
+StaticProc void asp_untimout(asp_tmo_func,  asp_scb_t *);
+StaticProc void asp_hangup(asp_scb_t *);
+StaticProc void asp_send_tickle(asp_scb_t *);
 StaticProc void asp_send_tickle_locked(void *);
-StaticProc void asp_accept();
-StaticProc int  asp_send_req();
+StaticProc void asp_accept(asp_scb_t *scb, asp_scb_t *sess_scb, gbuf_t *m);
+StaticProc int  asp_send_req(gref_t *, gbuf_t *, at_inet_t *, at_retry_t *, asp_word_t *, 
+							unsigned char , unsigned char, unsigned char);
 
 extern at_ifaddr_t *ifID_home;
 extern int atp_pidM[];
@@ -116,6 +104,8 @@ asp_scb_t *scb_used_list;
 static asp_scb_t *scb_tmo_list;
 asp_scb_t *scb_free_list;
 
+int asp_readable(gref_t *);
+
 int
 asp_readable(gref)
 	gref_t *gref;
@@ -265,7 +255,11 @@ asp_close(gref)
 	return 0;
 } /* asp_close */
 
-static char *aspStateStr(state)
+#ifdef DEBUG
+
+static const char *aspStateStr(int);
+
+static const char *aspStateStr(state)
      int state;
 {
   return ((state==ASPSTATE_Close)? "Close":
@@ -280,7 +274,9 @@ static char *aspStateStr(state)
 	  "unknown");
 }
 
-static char *aspCmdStr(aspCmd)
+static const char *aspCmdStr(int);
+
+static const char *aspCmdStr(aspCmd)
      int aspCmd;
 { 
 return ((aspCmd==ASPFUNC_CloseSess)? "CloseSess":
@@ -294,7 +290,9 @@ return ((aspCmd==ASPFUNC_CloseSess)? "CloseSess":
 	(aspCmd==ASPFUNC_CmdReply)? "CmdReply": "unknown");
 }
 
-static char *aspIOCStr(aspIOC)
+static const char *aspIOCStr(int);
+
+static const char *aspIOCStr(aspIOC)
      int aspIOC;
 {
 return (
@@ -314,6 +312,7 @@ return (
 	"unknown"
 	);
 }
+#endif /* DEBUG */
 
 #ifdef AT_MBUF_TRACE
 
@@ -893,8 +892,8 @@ void asp_clock(arg)
 	void *arg;
 {
 	asp_scb_t *scb;
-	void (*tmo_func)();
-
+	asp_tmo_func tmo_func;
+	
 	if (scb_tmo_list)
 		scb_tmo_list->tmo_delta--;
 	while (((scb = scb_tmo_list) != 0) && (scb_tmo_list->tmo_delta == 0)) {
@@ -1098,8 +1097,8 @@ asp_ack_reply(gref, mioc)
 			     || (scb->rcv_seq_num != ntohs(awp->param2))
 			     || BAD_REMADDR(rem_addr) ) {
 				char era[8], ra[8];
-				sprintf(era,"%d.%d", scb->rem_addr.node,scb->rem_addr.socket);
-				sprintf(ra,"%d.%d", rem_addr.node,rem_addr.socket);
+				snprintf(era, sizeof(era), "%d.%d", scb->rem_addr.node,scb->rem_addr.socket);
+				snprintf(ra, sizeof(ra), "%d.%d", rem_addr.node,rem_addr.socket);
 				dPrintf(D_M_ASP, D_L_WARNING,
 					("             : DROP, id=%d,esn=%d,sn=%d,erem=%s,rem=%s\n",
 					scb->sess_id,scb->rcv_seq_num,awp->param2,era,ra));
@@ -1604,7 +1603,7 @@ asp_find_scb(sock_num, rem_addr)
  */
 StaticProc void
 asp_timout(func, scb, seconds)
-	void (*func)();
+	asp_tmo_func func;
 	register asp_scb_t *scb;
 	int seconds;
 {
@@ -1660,9 +1659,9 @@ asp_timout(func, scb, seconds)
  * untimout routine
  */
 StaticProc void
-asp_untimout(func, scb)
-	void (*func)();
-	register asp_scb_t *scb;
+asp_untimout(
+	__unused asp_tmo_func tmo_func,
+	register asp_scb_t *scb)
 {
 
 	if ((scb->tmo_cnt == scb_tmo_cnt) || (scb->tmo_func == 0))
@@ -1690,8 +1689,6 @@ StaticProc void
 asp_hangup(scb)
 	asp_scb_t *scb;
 {
-	int s;
-
 	/*
 	 * set the state to Close
 	 */
@@ -1844,10 +1841,10 @@ asp_putnext(gref, mproto)
     2)  datptr->len = size of data
 */
 
-int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, int flags, int *errp)
+int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, __unused int flags, int *errp)
 {
     int i, err, len, offset, remain, size, copy_len;
-    gbuf_t *mioc, *mdata, *mx, *m0;
+    gbuf_t *mioc, *mdata, *mx;
     ioc_t *iocbp;
     strbuf_t ctlbuf;
     strbuf_t datbuf;
@@ -1894,7 +1891,7 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
             goto l_err;
      }
 
-     /* Radar 5423396: check for bogus length
+     /* Radar 5398072: check for bogus length
       * 	       Max ASP data is 8 ATP packets
       */
 
@@ -2015,7 +2012,7 @@ int ASPputmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t *mreq, in
         /*
          * build the command/write/write_continue request
          */
-        wptr = gbuf_rptr(mdata);
+        wptr = (unsigned char *)gbuf_rptr(mdata);
         atpBDS = (struct atpBDS *)wptr;
         wptr += atpBDSsize;
         for (i=0; i < ATP_TRESP_MAX; i++) {
@@ -2154,7 +2151,7 @@ l_err:
 
 
 /* bms:  make this callable from kernel.  reply date is passed back as a mbuf chain in *mreply  */
-int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply, int *flags, int *errp)
+int ASPgetmsg(gref_t *gref, strbuf_t *ctlptr, strbuf_t *datptr, gbuf_t **mreply, __unused int *flags, int *errp)
 {
     int err, len, sum, rval;
     gbuf_t *mproto, *mdata;
diff --git a/bsd/netat/at.c b/bsd/netat/at.c
index 0fe26ac45..572b7f58b 100644
--- a/bsd/netat/at.c
+++ b/bsd/netat/at.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1998 Apple Computer, Inc. 
@@ -41,8 +47,8 @@
 #include <net/if_types.h>
 #include <net/dlil.h>
 
-#include <netat/appletalk.h>
 #include <netat/sysglue.h>
+#include <netat/appletalk.h>
 #include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
@@ -51,13 +57,13 @@
 #include <netat/debug.h>
 
 #include <sys/kern_event.h>
+#include <net/kpi_protocol.h>
+
+int lap_online( at_ifaddr_t *, at_if_cfg_t *cfgp);
 
-extern int at_ioctl(struct atpcb *, u_long, caddr_t, int fromKernel);
 extern int routerStart(at_kern_err_t *);
 extern void elap_offline(at_ifaddr_t *);
 extern at_ifaddr_t *find_ifID(char *);
-extern at_nvestr_t *getRTRLocalZone(zone_usage_t *);
-extern int setLocalZones(at_nvestr_t *, int);
 
 extern int xpatcnt;
 extern at_ifaddr_t at_interfaces[];
@@ -136,7 +142,6 @@ at_control(so, cmd, data, ifp)
 {
 	struct ifreq *ifr = (struct ifreq *)data;
 	int pat_id = 0, error = 0;
-	struct proc *p = current_proc();       
 	at_ifaddr_t *ifID = 0;
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
@@ -222,7 +227,7 @@ at_control(so, cmd, data, ifp)
 	  	at_def_zone_t *defzonep = (at_def_zone_t *)data;
 
 		/* check for root access */
-		if (error = suser(kauth_cred_get(), 0))
+		if ((error = suser(kauth_cred_get(), 0)))
 			return(EACCES);
 
 		ifID = 0;
@@ -460,7 +465,7 @@ at_control(so, cmd, data, ifp)
 	  	at_router_params_t *rt = (at_router_params_t *)data;
 
 		/* check for root access */
-		if (error = suser(kauth_cred_get(), 0))
+		if ((error = suser(kauth_cred_get(), 0)))
 			return(EACCES);
 
 		/* when in routing/multihome mode the AIOCSETROUTER IOCTL 
@@ -531,7 +536,7 @@ at_control(so, cmd, data, ifp)
 		    ret;
 
 		/* check for root access */
-		if (error = suser(kauth_cred_get(), 0))
+		if ((error = suser(kauth_cred_get(), 0)))
 			return(EACCES);
 
 		ret = ddp_shutdown(*count_only);
@@ -558,12 +563,11 @@ at_control(so, cmd, data, ifp)
 
 	case SIOCSIFADDR:
 		/* check for root access */
-		if (error = suser(kauth_cred_get(), 0))
+		if ((error = suser(kauth_cred_get(), 0)))
 			error = EACCES;
 		else if (ifID)
 			error = EEXIST;
 		else {
-			int s;
 			if (xpatcnt == 0) {
 				at_state.flags |= AT_ST_STARTING;
 				ddp_brt_init();
@@ -576,6 +580,17 @@ at_control(so, cmd, data, ifp)
 
 			ifID->aa_ifp = ifp;
 			ifa = &ifID->aa_ifa;
+			error = proto_plumb(PF_APPLETALK, ifp);
+			if (error == EEXIST) {
+			    ifID->at_was_attached = 1;
+			    error = 0;
+			}
+			if (error != 0) {
+				break;
+			}
+			/* XXX ethernet-specific */
+			ifID->cable_multicast_addr = etalk_multicast_addr;
+			xpatcnt++;
 			ifnet_lock_exclusive(ifp);
 			TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) 
 				if ((sdl = (struct sockaddr_dl *)ifa->ifa_addr) &&
@@ -599,31 +614,6 @@ at_control(so, cmd, data, ifp)
 			   is set */
 			if_attach_ifa(ifp, ifa);
 			ifnet_lock_done(ifp);
-
-			switch (ifp->if_type) {
-			case IFT_ETHER:
-                        case IFT_L2VLAN:
-                        case IFT_IEEE8023ADLAG: /* bonded ethernet */
-				ether_attach_at(ifp);
-				error = 0;
-				ifID->cable_multicast_addr = etalk_multicast_addr;
-
-				xpatcnt++;
-				break;
-			case IFT_FDDI:
-				ifID->cable_multicast_addr = etalk_multicast_addr;
-				ddp_bit_reverse(&ifID->cable_multicast_addr);
-				xpatcnt++;
-				break;
-			case IFT_ISO88025: /* token ring */	
-				ifID->cable_multicast_addr = ttalk_multicast_addr;
-				ddp_bit_reverse(&ifID->cable_multicast_addr);
-
-				xpatcnt++;
-				break;
-			default:
-				error = EINVAL;
-			}
 		}
 	  break;
 
@@ -682,10 +672,36 @@ at_control(so, cmd, data, ifp)
         break;
     }
         
+	case SIOCPROTOATTACH:
+		/* check for root access */
+		if (suser(kauth_cred_get(), 0) != 0) {
+			error = EACCES;
+			break;
+		}
+		error = proto_plumb(PF_APPLETALK, ifp);
+		if (ifID != NULL
+		    && (error == 0 || error == EEXIST)) {
+			ifID->at_was_attached = 1;
+		}
+		break;
+
+	case SIOCPROTODETACH:
+		/* check for root access */
+		if (suser(kauth_cred_get(), 0) != 0) {
+			error = EACCES;
+			break;
+		}
+		if (ifID != NULL) {
+			error = EBUSY;
+			break;
+		}
+		error = proto_unplumb(PF_APPLETALK, ifp);
+		break;
+
 	default:
 		if (ifp == 0 || ifp->if_ioctl == 0)
 			return (EOPNOTSUPP);
-		return dlil_ioctl(0, ifp, cmd, (caddr_t) data);
+		return ifnet_ioctl(ifp, 0, cmd, data);
 	}
 
 	return(error);
diff --git a/bsd/netat/at_aarp.h b/bsd/netat/at_aarp.h
index 7abb25815..28b03bcbe 100644
--- a/bsd/netat/at_aarp.h
+++ b/bsd/netat/at_aarp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _NETAT_AT_AARP_H_
 #define _NETAT_AT_AARP_H_
@@ -130,7 +136,7 @@ typedef struct {
 
 /* at_addr - net # in network byte order */
 #define	AMT_LOOK(at, at_addr, elapp) {							\
-	register n; 								\
+	int n; 								\
 	at = &aarp_table[elapp->ifPort]->et_aarp_amt[AMT_HASH(at_addr) * AMT_BSIZ];	 		\
 	for (n = 0 ; ; at++) {					                \
 	    if (at->dest_at_addr.atalk_node == (at_addr).atalk_node &&									\
@@ -145,8 +151,8 @@ typedef struct {
 
 /* at_addr - net # in network byte order */
 #define	NEW_AMT(at, at_addr, elapp) {							\
-	register n; 								\
-	register aarp_amt_t *myat;                                              \
+	int n; 								\
+	aarp_amt_t *myat;                                              \
 	myat = at = &aarp_table[elapp->ifPort]->et_aarp_amt[AMT_HASH(at_addr) * AMT_BSIZ];			\
 	for (n = 0 ; ; at++) {					                \
 	    if (at->last_time == 0)             				\
@@ -187,6 +193,8 @@ typedef struct {
 
 int aarp_chk_addr(at_ddp_t  *, at_ifaddr_t *);
 int aarp_rcv_pkt(aarp_pkt_t *, at_ifaddr_t *);
+void AARPwakeup(aarp_amt_t *);
+int	aarp_send_data(gbuf_t *, at_ifaddr_t *, struct  atalk_addr *, int);
 
 #endif /* KERNEL_PRIVATE */
 
diff --git a/bsd/netat/at_config.h b/bsd/netat/at_config.h
index 0955e7457..5601d3538 100644
--- a/bsd/netat/at_config.h
+++ b/bsd/netat/at_config.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *        Copyright (c) 1988-1993 Apple Computer, Inc.
diff --git a/bsd/netat/at_ddp_brt.h b/bsd/netat/at_ddp_brt.h
index aab59d573..781311a8c 100644
--- a/bsd/netat/at_ddp_brt.h
+++ b/bsd/netat/at_ddp_brt.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -53,7 +59,7 @@ typedef struct {
 #define	BRT_HASH(a)   ((a) % BRT_NB)
 
 #define	BRT_LOOK(brt, dst_net) {				\
-	register n; 						\
+	int n; 						\
 	brt = &at_ddp_brt[BRT_HASH(dst_net) * BRT_BSIZ];	\
 	for (n = 0 ; ; brt++) {		                        \
 		if (brt->net == dst_net) 			\
@@ -66,7 +72,7 @@ typedef struct {
 	}
 
 #define	NEW_BRT(brt, net) {					\
-	register n; 						\
+	int n; 						\
 	brt = &at_ddp_brt[BRT_HASH(net) * BRT_BSIZ]; 		\
 	for (n = 0 ; ; brt++) { 		                \
 		if (brt->age_flag == BRT_EMPTY)			\
diff --git a/bsd/netat/at_pat.h b/bsd/netat/at_pat.h
index 99dc7ee29..a34bf693e 100644
--- a/bsd/netat/at_pat.h
+++ b/bsd/netat/at_pat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -65,4 +71,5 @@ typedef struct {
 	 (a1[4] == a2[4])				                \
 	)
 #endif /* __APPLE_API_OBSOLETE */
+
 #endif /* _NETAT_AT_PAT_H_ */
diff --git a/bsd/netat/at_pcb.c b/bsd/netat/at_pcb.c
index 4594fccc5..a1e542bdf 100644
--- a/bsd/netat/at_pcb.c
+++ b/bsd/netat/at_pcb.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -91,7 +97,7 @@ extern struct atpcb *atp_inputQ[];
 extern CCB *adsp_inputQ[];
 extern at_ifaddr_t *ifID_home;
 extern struct {
-  void (*func)();
+  ddp_handler_func func;
   } ddp_handler[];
 
 int DDP_chksum_on = FALSE;
@@ -99,7 +105,7 @@ int DDP_slfsnd_on = FALSE;
 
 zone_t atpcb_zone;
 
-void at_memzone_init()
+void at_memzone_init(void)
 {
 	vm_size_t str_size;
 
diff --git a/bsd/netat/at_pcb.h b/bsd/netat/at_pcb.h
index 6d9dfb28b..b936f6751 100644
--- a/bsd/netat/at_pcb.h
+++ b/bsd/netat/at_pcb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1997-1999 Apple Computer, Inc.
@@ -121,6 +127,20 @@ int	at_pcbbind(struct atpcb *, struct sockaddr *);
 int atalk_getref(struct fileproc *, int , gref_t ** , struct proc *, int);
 int atalk_getref_locked(struct fileproc *, int , gref_t ** , struct proc *, int);
 
+void atalk_notify(gref_t *, int);
+void atalk_notify_sel(gref_t *);
+
+int atalk_peek(gref_t *, unsigned char *);
+
+void ddp_putmsg(gref_t *gref, gbuf_t *mp);
+int ddp_socket_inuse(u_char, u_char);
+void ddp_notify_nbp(unsigned char, int, unsigned char);
+
+void atalk_putnext(gref_t *gref, gbuf_t *m);
+void atalk_enablew(gref_t *gref);
+void atalk_flush(gref_t *gref);
+
+void at_memzone_init(void);
 
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
diff --git a/bsd/netat/at_proto.c b/bsd/netat/at_proto.c
index 74e1808e6..e6acc4660 100644
--- a/bsd/netat/at_proto.c
+++ b/bsd/netat/at_proto.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1998 Apple Computer, Inc. 
@@ -42,36 +48,13 @@
 
 #include <net/if.h>
 
+#include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 
 
-extern int	ddp_pru_abort(struct socket *so);
-
-extern int	ddp_pru_attach(struct socket *so, int proto,
-			       struct proc *p);
-extern int	ddp_pru_bind(struct socket *so, struct sockaddr *nam,
-			     struct proc *p);
-extern int	ddp_pru_connect(struct socket *so, struct sockaddr *nam,
-				struct proc *p);
-
-extern int	ddp_pru_control(struct socket *so, u_long cmd, caddr_t data,
-				struct ifnet *ifp, struct proc *p);
-extern int	ddp_pru_detach(struct socket *so);
-extern int	ddp_pru_disconnect(struct socket *so);
-
-extern int	ddp_pru_peeraddr(struct socket *so, 
-				 struct sockaddr **nam);
-
-extern int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m, 
-				 struct sockaddr *addr, struct mbuf *control,
-				 struct proc *p);
-
-extern int	ddp_pru_shutdown(struct socket *so);
-extern int	ddp_pru_sockaddr(struct socket *so, 
-				 struct sockaddr **nam);
-
 /*
  * Dummy usrreqs struct created by Ted for FreeBSD 3.x integration. 
  * Fill in supported functions as appropriate.
@@ -117,14 +100,25 @@ struct domain atalkdomain =
 struct domain * atalkdom = &atalkdomain;
 lck_mtx_t  *atalk_mutex = NULL;
 
+lck_mtx_t *atalk_cluster_lock = NULL;
+static lck_attr_t *atalk_lock_attr;
+static lck_grp_t *atalk_lock_grp;
+static lck_grp_attr_t *atalk_lock_grp_attr;
+
 static int at_saved_lock, at_saved_unlock;
 
-SYSCTL_NODE(_net, PF_APPLETALK, appletalk, CTLFLAG_RW, 0, "AppleTalk Family");
+SYSCTL_NODE(_net, PF_APPLETALK, appletalk, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "AppleTalk Family");
 
 void
 atalk_dominit(void)
 {
 	atalk_mutex = atalkdom->dom_mtx;
+
+	atalk_lock_grp_attr = lck_grp_attr_alloc_init();
+	atalk_lock_grp = lck_grp_alloc_init("appletalk", atalk_lock_grp_attr);
+	atalk_lock_attr = lck_attr_alloc_init();
+	atalk_cluster_lock = lck_mtx_alloc_init(atalk_lock_grp,
+	    atalk_lock_attr);
 }
 
 void
diff --git a/bsd/netat/at_snmp.h b/bsd/netat/at_snmp.h
index 3ec477cac..da6249dfc 100644
--- a/bsd/netat/at_snmp.h
+++ b/bsd/netat/at_snmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _NETAT_AT_SNMP_H_
 #define _NETAT_AT_SNMP_H_
@@ -214,5 +220,11 @@ typedef struct snmpStats {
 
 #define SNMP_TYPE(var,type)	 ((var & SNMP_OBJ_TYPE_MASK) == type)	
 
+#ifdef KERNEL_PRIVATE
+int getNbpTable(snmpNbpEntry_t *, int, int);
+int getNbpTableSize(void);
+snmpAarpEnt_t *getAarp(int *);
+#endif
+
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_AT_SNMP_H_ */
diff --git a/bsd/netat/at_var.h b/bsd/netat/at_var.h
index fdb445d3b..64025eb6f 100644
--- a/bsd/netat/at_var.h
+++ b/bsd/netat/at_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1998 Apple Computer, Inc. 
@@ -159,9 +165,7 @@ typedef struct at_ifaddr {
 #define			aa_flags	aa_ifa.ifa_flags
 
 	TAILQ_ENTRY(at_ifaddr) aa_link; /* tailq macro glue */
-
-	u_long at_dl_tag;		/* DLIL tag to be used in packet output */
-	u_long aarp_dl_tag;		/* DLIL tag for Appletalk ARP */
+	int			at_was_attached; /* 1=attached, 0=detached */
 
 	/* from pat_unit_t */
 	unsigned char 	mcast[MAX_MCASTS];
@@ -280,7 +284,7 @@ int at_control(struct socket *, u_long, caddr_t, struct ifnet *);
 int ddp_usrreq(struct socket *, int, struct mbuf *, struct mbuf *, 
 		    struct mbuf *);
 int ddp_ctloutput(struct socket *, struct sockopt *);
-void ddp_init(void);;
+void ddp_init(void);
 void ddp_slowtimo(void);
 #endif /* KERNEL_PRIVATE */
 
@@ -312,6 +316,42 @@ void atalk_post_msg(struct ifnet *ifp, u_long event_code, struct at_addr *addres
 void aarp_sched_probe(void *);
 void atalk_lock(void);
 void atalk_unlock(void);
+void appletalk_hack_start(void);
+void ddp_input(gbuf_t *, at_ifaddr_t *);
+struct etalk_addr;
+void ddp_glean(gbuf_t *, at_ifaddr_t  *, struct etalk_addr  *);
+
+int pat_output(at_ifaddr_t *, struct mbuf *, unsigned char *, int);
+
+void ep_input(gbuf_t *, at_ifaddr_t *);
+void zip_router_input(gbuf_t *, at_ifaddr_t *);
+void nbp_input(gbuf_t *, at_ifaddr_t *);
+void sip_input(gbuf_t *, at_ifaddr_t *);
+
+void ioc_ack(int, gbuf_t *, gref_t *);
+int ddp_adjmsg(gbuf_t *, int );
+gbuf_t *ddp_growmsg(gbuf_t  *, int );
+
+struct proc;
+int atalk_openref(gref_t *, int *, struct proc *);
+
+struct fileglob;
+int atalk_closeref(struct fileglob *, gref_t **);
+
+int _ATputmsg(int, strbuf_t *, strbuf_t *, int , int *, void *);
+int _ATgetmsg(int, strbuf_t *, strbuf_t *, int *, int *, void *);
+int _ATsocket(int, int *, void *);
+
+void ddp_start(void);
+
+typedef void (*ddp_handler_func)(gbuf_t *, at_ifaddr_t *);
+void add_ddp_handler(u_char, ddp_handler_func);
+void init_ddp_handler(void);
+
+int  elap_wput(gref_t *gref, gbuf_t *m);
+int at_ioctl(struct atpcb *, u_long, caddr_t, int );
+
+
 
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
diff --git a/bsd/netat/atp.h b/bsd/netat/atp.h
index 87a474a89..dda3669a9 100644
--- a/bsd/netat/atp.h
+++ b/bsd/netat/atp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -299,6 +305,9 @@ struct atp_socket {
  *	transaction control block (local context at requester end)
  */
 
+struct atp_trans;
+typedef void (*atp_tmo_func)(struct atp_trans *);
+
 struct atp_trans {
 	struct atp_trans_q	tr_list;		/* trans list */
 	struct atp_state	*tr_queue;		/* state data structure */
@@ -320,7 +329,7 @@ struct atp_trans {
 	at_net			tr_local_net;
 	gbuf_t                  *tr_bdsp;               /* bds structure pointer */
 	unsigned int		tr_tmo_delta;
-	void 				(*tr_tmo_func)();
+	atp_tmo_func		tr_tmo_func;
 	struct atp_trans	*tr_tmo_next;
 	struct atp_trans	*tr_tmo_prev;
 	atlock_t tr_lock;
@@ -428,14 +437,13 @@ extern struct atp_rcb* atp_rcb_data;
 extern struct atp_state* atp_state_data;
 extern struct atp_trans_qhead atp_trans_abort;		/* aborting trans list */
 
-extern void atp_req_timeout();
-extern void atp_rcb_timer();
-extern void atp_x_done();
-extern struct atp_rcb *atp_rcb_alloc();
-extern struct atp_trans *atp_trans_alloc();
 #endif /* ATP_DECLARE */
 
 /* prototypes */
+void atp_rcb_timer(struct atp_trans *);
+struct atp_rcb *atp_rcb_alloc(struct  atp_state *);
+struct atp_trans *atp_trans_alloc(struct  atp_state *);
+
 void atp_send_req(gref_t *, gbuf_t *);
 void atp_drop_req(gref_t *, gbuf_t *);
 void atp_send_rsp(gref_t *, gbuf_t *, int);
@@ -458,13 +466,18 @@ void atp_reply(struct atp_rcb *);
 void atp_rcb_free(struct atp_rcb *);
 void atp_send_replies(struct atp_state *, struct atp_rcb *);
 void atp_dequeue_atp(struct atp_state *);
-int atp_iocack(struct atp_state *, gbuf_t *);
+void atp_iocack(struct atp_state *, gbuf_t *);
 void atp_req_ind(struct atp_state *, gbuf_t *);
-int atp_iocnak(struct atp_state *, gbuf_t *, int);
+void atp_iocnak(struct atp_state *, gbuf_t *, int);
 void atp_trp_timer(void *, int);
-void atp_timout(void (*func)(), struct atp_trans *, int);
-void atp_untimout(void (*func)(), struct atp_trans *);
+void atp_timout(atp_tmo_func, struct atp_trans *, int);
+void atp_untimout(atp_tmo_func, struct atp_trans *);
 int atp_tid(struct atp_state *);
+void atp_init(void);
+void atp_link(void);
+void atp_unlink(void);
+int atp_input(gbuf_t *);
+void atp_delete_free_clusters(void *);
 
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
diff --git a/bsd/netat/atp_alloc.c b/bsd/netat/atp_alloc.c
index 783a64ab9..cf9a4375c 100644
--- a/bsd/netat/atp_alloc.c
+++ b/bsd/netat/atp_alloc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*    Modified for MP, 1996 by Tuyen Nguyen */
 /*
@@ -52,7 +58,8 @@
 
 gbuf_t *atp_resource_m = 0;
 extern caddr_t atp_free_cluster_list;
-extern void atp_delete_free_clusters();
+extern struct atp_rcb_qhead atp_need_rel;
+
 
 struct atp_trans *atp_trans_alloc(atp)
 struct  atp_state *atp;
@@ -62,7 +69,8 @@ struct  atp_state *atp;
 	register struct atp_trans *trp, *trp_array;
 
 	if (atp_trans_free_list == 0) {
-		if ((m = gbuf_alloc(TRPS_PER_BLK*sizeof(struct atp_trans),PRI_HI)) == 0)
+		if ((m = gbuf_alloc_wait(TRPS_PER_BLK*sizeof(struct atp_trans),
+		    atp == NULL)) == 0)
 			return (struct atp_trans *)0;
 		bzero(gbuf_rptr(m), TRPS_PER_BLK*sizeof(struct atp_trans));
 		trp_array = (struct atp_trans *)gbuf_rptr(m);
@@ -150,8 +158,6 @@ register struct atp_rcb *rcbp;
 	rcbp->rc_queue = 0;
 
 	if (rcbp->rc_timestamp) {
-	        extern struct atp_rcb_qhead atp_need_rel;
-
 		rcbp->rc_timestamp = 0;
 		ATP_Q_REMOVE(atp_need_rel, rcbp, rc_tlist);
 		rcbp->rc_tlist.prev = NULL;
@@ -165,7 +171,7 @@ register struct atp_rcb *rcbp;
 			rcbp->rc_snd[i] = 0;
 	}
 	if (atp_free_cluster_list)
-		atp_delete_free_clusters();
+		atp_delete_free_clusters(NULL);
 	if (rc_state != RCB_UNQUEUED) {
 		if (rc_state == RCB_PENDING) {
 			ATP_Q_REMOVE(atp->atp_attached, rcbp, rc_list);
diff --git a/bsd/netat/atp_misc.c b/bsd/netat/atp_misc.c
index 5c70dd94d..730b55e5f 100644
--- a/bsd/netat/atp_misc.c
+++ b/bsd/netat/atp_misc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996-1998 Apple Computer, Inc.
@@ -46,10 +52,11 @@
 #include <netat/ddp.h>
 #include <netat/at_pcb.h>
 #include <netat/atp.h>
+#include <netat/asp.h>
 #include <netat/debug.h>
 
-void atp_free();
-void atp_send(struct atp_trans *);
+extern   struct atp_rcb_qhead atp_need_rel;
+extern struct atp_trans *trp_tmo_rcb;
 
 /*
  *	The request timer retries a request, if all retries are used up
@@ -252,12 +259,10 @@ register struct atp_rcb *rcbp;
  *	The rcb timer just frees the rcb, this happens when we missed a release for XO
  */
 
-void atp_rcb_timer()
+void atp_rcb_timer(__unused struct atp_trans *junk)
 {  
     register struct atp_rcb *rcbp;
 	register struct atp_rcb *next_rcbp;
-	extern   struct atp_rcb_qhead atp_need_rel;
-	extern struct atp_trans *trp_tmo_rcb;
 	struct timeval timenow;
 
 l_again:
@@ -273,7 +278,7 @@ l_again:
 	atp_timout(atp_rcb_timer, trp_tmo_rcb, 10 * HZ);
 }
 
-atp_iocack(atp, m)
+void atp_iocack(atp, m)
 struct   atp_state *atp;
 register gbuf_t *m;
 {
@@ -290,7 +295,7 @@ register gbuf_t *m;
 		atalk_putnext(atp->atp_gref, m);
 }
 
-atp_iocnak(atp, m, err)
+void atp_iocnak(atp, m, err)
 struct   atp_state *atp;
 register gbuf_t *m;
 register int err;
@@ -315,7 +320,7 @@ register int err;
  *	Generate a transaction id for a socket
  */
 static int lasttid;
-atp_tid(atp)
+int atp_tid(atp)
 register struct atp_state *atp;
 {
 	register int i;
diff --git a/bsd/netat/atp_open.c b/bsd/netat/atp_open.c
index 192071a85..28847f502 100644
--- a/bsd/netat/atp_open.c
+++ b/bsd/netat/atp_open.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996-1998 Apple Computer, Inc.
@@ -49,6 +55,7 @@
 #include <netat/ddp.h>
 #include <netat/at_pcb.h>
 #include <netat/atp.h>
+#include <netat/asp.h>
 #include <netat/debug.h>
 
 /*
@@ -98,10 +105,8 @@ int atp_input(mp)
 }
 
 /**********/
-void atp_init()
+void atp_init(void)
 {
-  int i;
-
   if (!atp_inited) {
 	atp_inited = 1;
 	atp_used_list = 0;
@@ -218,7 +223,6 @@ int atp_close(gref, flag)
 	gref_t *gref;
 	int flag;
 {
-	extern void atp_req_timeout();
 	register struct atp_state *atp;
 	register struct atp_trans *trp;
 	register struct atp_rcb *rcbp;
diff --git a/bsd/netat/atp_read.c b/bsd/netat/atp_read.c
index aacef82b2..a3853eae9 100644
--- a/bsd/netat/atp_read.c
+++ b/bsd/netat/atp_read.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996-1998 Apple Computer, Inc.
@@ -41,17 +47,20 @@
 #include <sys/malloc.h>
 #include <sys/socket.h>
 
+#include <net/if.h>
+
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
 #include <netat/ddp.h>
 #include <netat/at_pcb.h>
+#include <netat/at_var.h>
 #include <netat/atp.h>
+#include <netat/asp.h>
 #include <netat/debug.h>
 
-static void atp_trans_complete();
-void atp_x_done();
+static void atp_trans_complete(struct atp_trans *);
 void atp_x_done_locked(void *);
-extern void atp_req_timeout();
+void atp_treq_event(void *);
 
 /*
  *	Decide what to do about received messages
@@ -94,7 +103,6 @@ gbuf_t   *m;
 {
 	register at_atp_t *athp;
 	register struct atp_state *atp;
-	register int s_gen;
 	gbuf_t *m_asp = NULL;
 	struct timeval timenow;
 	u_short		temp_net;
diff --git a/bsd/netat/atp_write.c b/bsd/netat/atp_write.c
index 3fcbecea4..a46fa471f 100644
--- a/bsd/netat/atp_write.c
+++ b/bsd/netat/atp_write.c
@@ -1,27 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1996-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *	Copyright (c) 1996-1998 Apple Computer, Inc.
- *	All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*    Modified for MP, 1996 by Tuyen Nguyen
@@ -44,6 +46,7 @@
 #include <sys/malloc.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
+#include <sys/file.h>
 
 #include <net/if.h>
 
@@ -57,18 +60,13 @@
 #include <netat/at_pat.h>
 #include <netat/debug.h>
 
-static int loop_cnt; /* for debugging loops */
-#define CHK_LOOP(str) { \
-  if (loop_cnt++ > 100) { \
-     kprintf("%s", str); \
-     break; \
-  } \
-}
+
+int asp_pack_bdsp(struct atp_trans *, gbuf_t **);
 
 static int atp_pack_bdsp(struct atp_trans *, struct atpBDS *);
 static int atp_unpack_bdsp(struct atp_state *, gbuf_t *, struct atp_rcb *, 
 			   int, int);
-void atp_trp_clock(), asp_clock(), asp_clock_locked(), atp_trp_clock_locked();;
+void atp_trp_clock(void *arg), atp_trp_clock_locked(void *arg);
 
 extern struct atp_rcb_qhead atp_need_rel;
 extern int atp_inited;
@@ -92,7 +90,7 @@ struct atp_trans *trp_tmo_rcb;
 
 #define atpBDSsize (sizeof(struct atpBDS)*ATP_TRESP_MAX)
 
-void atp_link()
+void atp_link(void)
 {
 	trp_tmo_list = 0;
 	trp_tmo_rcb = atp_trans_alloc(0);
@@ -197,7 +195,6 @@ atp_wput(gref, m)
 		case AT_ATP_ISSUE_REQUEST_DEF:
 		case AT_ATP_ISSUE_REQUEST_DEF_NOTE: {
 			gbuf_t *bds, *tmp, *m2;
-			struct atp_rcb *rcbp;
 			at_ddp_t *ddp;
 			at_atp_t *athp;
 
@@ -441,13 +438,14 @@ void atp_send_replies(atp, rcbp)
      register struct atp_rcb   *rcbp;
 {       register gbuf_t *m;
 	register int     i, len;
-	int              s_gen, cnt, err, offset, space;
-	unsigned char *m0_rptr = NULL, *m0_wptr = NULL;
+	int              cnt, offset, space;
 	register at_atp_t *athp;
 	register struct atpBDS *bdsp;
-	register gbuf_t *m2, *m1, *m0, *mhdr;
-	caddr_t lastPage;
-	gbuf_t *mprev, *mlist = 0;
+	register gbuf_t *m1, *m0, *mhdr;
+#if DEBUG
+	gbuf_t *m2 = NULL;
+#endif /* DEBUG */
+	gbuf_t *mprev = 0, *mlist = 0;
 	at_socket src_socket = (at_socket)atp->atp_socket_no;
 	gbuf_t *rc_xmt[ATP_TRESP_MAX];
 	struct   ddp_atp {
@@ -663,7 +661,6 @@ atp_unpack_bdsp(atp, m, rcbp, cnt, wait)
 {
 	register struct atpBDS *bdsp;
 	register	 gbuf_t        *m2, *m1, *m0, *mhdr;
-	caddr_t 	lastPage;
     at_atp_t    *athp;
 	int  		i, len;
 	at_socket 	src_socket;
@@ -671,10 +668,9 @@ atp_unpack_bdsp(atp, m, rcbp, cnt, wait)
 	struct ddp_atp {
 	         char    ddp_atp_hdr[TOTAL_ATP_HDR_SIZE];
 	};
-	gbuf_t 			*mprev, *mlist = 0;
+	gbuf_t 			*mprev = 0, *mlist = 0;
 	gbuf_t 			*rc_xmt[ATP_TRESP_MAX];
-	unsigned char 	*m0_rptr, *m0_wptr;
-	int				err, offset, space;
+	int				offset, space = 0;
 	struct timeval timenow;
 
 	/*
@@ -826,16 +822,16 @@ l_send:
 #define ATP_SOCKET_LAST  (DDP_SOCKET_LAST-6)
 #define ATP_SOCKET_FIRST (DDP_SOCKET_1st_DYNAMIC)
 static unsigned int sNext = 0;
+extern unsigned char asp_inpC[];
+extern asp_scb_t *asp_scbQ[];
 
 int atp_bind(gref, sVal, flag)
 	gref_t *gref;
 	unsigned int sVal;
 	unsigned char *flag;
 {
-	extern unsigned char asp_inpC[];
-	extern asp_scb_t *asp_scbQ[];
 	unsigned char inpC, sNextUsed = 0;
-	unsigned int sMin, sMax, sSav;
+	unsigned int sMin, sMax, sSav = 0;
 	struct atp_state *atp;
 
 	atp = (struct atp_state *)gref->info;
@@ -1006,7 +1002,7 @@ atp_dequeue_atp(atp)
 
 void
 atp_timout(func, trp, ticks)
-	void (*func)();
+	atp_tmo_func func;
 	struct atp_trans *trp;
 	int ticks;
 {
@@ -1058,9 +1054,9 @@ atp_timout(func, trp, ticks)
 }
 
 void
-atp_untimout(func, trp)
-	void (*func)();
-	struct atp_trans *trp;
+atp_untimout(
+	__unused atp_tmo_func func,
+	struct atp_trans *trp)
 {
 
 	if (trp->tr_tmo_func == 0)
@@ -1094,7 +1090,7 @@ atp_trp_clock(arg)
 	void *arg;
 {
 	struct atp_trans *trp;
-	void (*tr_tmo_func)();
+	atp_tmo_func tr_tmo_func;
 
 	if (trp_tmo_list)
 		trp_tmo_list->tr_tmo_delta--;
@@ -1303,7 +1299,7 @@ int asp_pack_bdsp(trp, xm)
 	register struct atpBDS *bdsp;
 	register gbuf_t        *m, *m2;
 	register int           i;
-	gbuf_t        *m_prev, *m_head = 0;
+	gbuf_t        *m_prev = 0, *m_head = 0;
 
 	dPrintf(D_M_ATP, D_L_INFO, ("asp_pack_bdsp: socket=%d\n",
 		trp->tr_queue->atp_socket_no));
@@ -1617,15 +1613,12 @@ _ATPsndrsp(fd, respbuff, resplen, datalen, err, proc)
 	void *proc;
 {
 	gref_t 		*gref;
-	int 		s, rc;
 	long 		bufaddr;
 	gbuf_t 		*m, *mdata;
 	short 		space;
 	int 		size;
 	struct atp_state *atp;
 	struct atpBDS *bdsp;
-	u_int16_t	*bufsz;
-	char 		*buf;
 	int			bds_cnt, count, len;
 	caddr_t		dataptr;
 
@@ -1720,7 +1713,7 @@ _ATPsndrsp(fd, respbuff, resplen, datalen, err, proc)
 				if (!(mdata->m_flags & M_EXT)) {
 					m_freem(m);
 					file_drop(fd);
-					return(NULL);
+					return(0);
 				}
 				dataptr = mtod(mdata, caddr_t);
 				space = MCLBYTES;
diff --git a/bsd/netat/aurp.h b/bsd/netat/aurp.h
index cd063cefe..9df123b1b 100644
--- a/bsd/netat/aurp.h
+++ b/bsd/netat/aurp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -287,3 +293,4 @@ extern struct aurp_global_t aurp_global;
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_AURP_H_ */
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_aurpd.c b/bsd/netat/aurp_aurpd.c
index d7351bfd5..674ea01c7 100644
--- a/bsd/netat/aurp_aurpd.c
+++ b/bsd/netat/aurp_aurpd.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -57,6 +63,7 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
 #include <netat/at_pcb.h>
@@ -405,7 +412,7 @@ atalk_to_ip(register gbuf_t *m)
 	int flags = MSG_DONTWAIT;
 	struct sockaddr_in rem_addr;
 
-	m->m_type = MT_HEADER;
+	m_mchtype(m, MT_HEADER);
 	m->m_pkthdr.len = gbuf_msgsize(m);
 	m->m_pkthdr.rcvif = 0;
 
@@ -436,3 +443,4 @@ atalk_to_ip(register gbuf_t *m)
 	return;
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_cfg.c b/bsd/netat/aurp_cfg.c
index 7f3dada02..c8c0e003b 100644
--- a/bsd/netat/aurp_cfg.c
+++ b/bsd/netat/aurp_cfg.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -47,6 +53,7 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
 #include <netat/at_pcb.h>
@@ -99,3 +106,4 @@ int aurp_close(gref)
 	return 0;
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_gdata.c b/bsd/netat/aurp_gdata.c
index 36c9d338e..8765cfa58 100644
--- a/bsd/netat/aurp_gdata.c
+++ b/bsd/netat/aurp_gdata.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,6 +52,7 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/rtmp.h>
 #include <netat/routing_tables.h>
@@ -62,3 +69,4 @@ int update_tmo;
 aurp_state_t aurp_state[256];
 unsigned short net_access[AURP_MaxNetAccess];
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_misc.c b/bsd/netat/aurp_misc.c
index 0bcdd5cd3..e9fdf41e0 100644
--- a/bsd/netat/aurp_misc.c
+++ b/bsd/netat/aurp_misc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,10 +52,10 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/rtmp.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -212,3 +218,4 @@ void AURPaccess()
 	}
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_open.c b/bsd/netat/aurp_open.c
index df8553ef0..d3bec0441 100644
--- a/bsd/netat/aurp_open.c
+++ b/bsd/netat/aurp_open.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,9 +52,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -250,3 +256,4 @@ void AURPrcvOpenRsp(state, m)
 	AURPsndRIReq(state);
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_rd.c b/bsd/netat/aurp_rd.c
index f43b3dece..30b15236b 100644
--- a/bsd/netat/aurp_rd.c
+++ b/bsd/netat/aurp_rd.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,9 +52,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 
 /* */
@@ -119,3 +125,4 @@ void AURPrcvRDReq(state, m)
 	AURPsndRIAck(state, m, 0);
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_ri.c b/bsd/netat/aurp_ri.c
index dd002da66..c4f36f4a0 100644
--- a/bsd/netat/aurp_ri.c
+++ b/bsd/netat/aurp_ri.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -841,3 +847,4 @@ void AURPrtupdate(entry, ev)
 	}
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_rx.c b/bsd/netat/aurp_rx.c
index ff8e40414..416345fac 100644
--- a/bsd/netat/aurp_rx.c
+++ b/bsd/netat/aurp_rx.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,9 +52,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -210,3 +216,4 @@ at_insert(m, type, node)
 	return 0;
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_tickle.c b/bsd/netat/aurp_tickle.c
index c207a0072..9b68f9447 100644
--- a/bsd/netat/aurp_tickle.c
+++ b/bsd/netat/aurp_tickle.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,9 +52,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -160,3 +166,4 @@ void AURPrcvTickleAck(state, m)
 	state->tickle_retry = 0;
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_tx.c b/bsd/netat/aurp_tx.c
index 71de54906..dcdc11c1f 100644
--- a/bsd/netat/aurp_tx.c
+++ b/bsd/netat/aurp_tx.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -46,9 +52,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -143,3 +149,4 @@ void AURPcmdx(code, mdata, param)
 	}
 }
 
+#endif  /* AURP_SUPPORT */
diff --git a/bsd/netat/aurp_zi.c b/bsd/netat/aurp_zi.c
index 93ec3f043..b70b3ecf6 100644
--- a/bsd/netat/aurp_zi.c
+++ b/bsd/netat/aurp_zi.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1996 Apple Computer, Inc. 
@@ -47,9 +53,9 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
-#include <netat/at_pcb.h>
 #include <netat/aurp.h>
 #include <netat/debug.h>
 
@@ -615,3 +621,4 @@ AURPsetzi(node, m, sub_code, tuples_cnt)
 	}
 }
 
+#endif /* AURP_SUPPORT */
diff --git a/bsd/netat/ddp.c b/bsd/netat/ddp.c
index dcd25cf5d..31467fa6b 100644
--- a/bsd/netat/ddp.c
+++ b/bsd/netat/ddp.c
@@ -1,28 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- *	Copyright (c) 1987, 1988, 1989 Apple Computer, Inc. 
- *
- *
  *    Modified for MP, 1996 by Tuyen Nguyen
  *    Added AURP support, April 8, 1996 by Tuyen Nguyen
  *   Modified, March 17, 1997 by Tuyen Nguyen for MacOSX.
@@ -51,13 +54,13 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/ep.h>
 #include <netat/nbp.h>
 #include <netat/rtmp.h>
 #include <netat/zip.h>
-#include <netat/at_pcb.h>
 #include <netat/routing_tables.h>
 #include <netat/at_snmp.h>
 #include <netat/aurp.h>
@@ -66,13 +69,16 @@
 #include <netat/at_aarp.h>
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
+#include <netat/at_pat.h>
+#include <netat/atp.h>
+
+#include <net/kpi_protocol.h>
 
 /* globals */
 
 /* Queue of LAP interfaces which have registered themselves with DDP */
 struct at_ifQueueHd at_ifQueueHd;
 
-extern at_state_t at_state;
 extern TAILQ_HEAD(name_registry, _nve_) name_registry;
 
 snmpStats_t snmpStats;		/* snmp ddp & echo stats */
@@ -84,35 +90,35 @@ extern aarp_amt_array *aarp_table[];
 extern at_ifaddr_t at_interfaces[];
 
 /* routing mode special */
-void (*ddp_AURPsendx)();
+void (*ddp_AURPsendx)(void) = NULL;
 at_ifaddr_t *aurp_ifID = 0;
-extern pktsIn,pktsOut;
-int pktsDropped,pktsHome;
+
+int pktsIn = 0;
+int pktsOut = 0;
+int pktsDropped = 0;
+int pktsHome = 0;
 
 extern int *atp_pidM;
 extern int *adsp_pidM;
 extern struct atpcb *atp_inputQ[];
 extern CCB *adsp_inputQ[];
 
-at_ifaddr_t *forUs(at_ddp_t *);
+static void fillin_pkt_chain(gbuf_t *);
+static int ot_ddp_check_socket(unsigned char ,int pid);
 
-void ddp_input(), ddp_notify_nbp();
-
-extern void routing_needed();
-extern void ddp_brt_sweep();
 
 struct {
-	void (*func)();
+	ddp_handler_func func;
 } ddp_handler[256];
 
-void init_ddp_handler()
+void init_ddp_handler(void)
 {
 	bzero(ddp_handler, sizeof(ddp_handler));
 }
 
 void add_ddp_handler(ddp_socket, input_func)
      u_char ddp_socket;
-     void (*input_func)();
+     ddp_handler_func input_func;
 {
 	ddp_handler[ddp_socket].func = input_func;
 }
@@ -384,10 +390,8 @@ void  ddp_rem_if(ifID)
 		ifa->ifa_addr = NULL;
 		ifnet_lock_done(ifID->aa_ifp);
 	}
-	if (ifID->at_dl_tag) {
-/*		dlil_detach_protocol(ifID->at_dl_tag); */
-		ether_detach_at(ifID->aa_ifp);
-		ifID->at_dl_tag = 0;
+	if (ifID->at_was_attached == 0 && ifID->aa_ifp != NULL) {
+		(void)proto_unplumb(PF_APPLETALK, ifID->aa_ifp);
 	}
 
 	/* un-do processing done in ddp_add_if() */
@@ -420,7 +424,7 @@ void  ddp_rem_if(ifID)
  */
 
 /* *** Do we still need to do this? *** */
-int ot_ddp_check_socket(socket, pid)
+static int ot_ddp_check_socket(socket, pid)
      unsigned char socket;
      int pid;
 {
@@ -440,12 +444,11 @@ int ot_ddp_check_socket(socket, pid)
 	return(cnt);
 }
 
-void ddp_notify_nbp(socket, pid, ddptype)
-     unsigned char socket;
-     int pid;
-     unsigned char ddptype; /* not used */
+void ddp_notify_nbp(
+     unsigned char socket,
+     int pid,
+     __unused unsigned char ddptype)
 {
-	extern int nve_lock;
 	nve_entry_t *nve_entry, *nve_next;
 
 	if (at_state.flags & AT_ST_STARTED) {
@@ -519,7 +522,7 @@ int ddp_output(mp, src_socket, src_addr_included)
 {
 	register at_ifaddr_t	*ifID = ifID_home, *ifIDTmp = NULL;
 	register at_ddp_t	*ddp;
-	register ddp_brt_t	*brt;
+	register ddp_brt_t	*brt = NULL;
 	register at_net_al	dst_net;
 	register int 		len;
 	struct	 atalk_addr	at_dest;
@@ -527,7 +530,7 @@ int ddp_output(mp, src_socket, src_addr_included)
 	int loop = 0;
 	int error = 0;
 	int addr_type;
-	u_char	addr_flag;
+	u_char	addr_flag = 0;
 	char	*addr = NULL;
 	register gbuf_t	*m;
 
@@ -779,7 +782,8 @@ int ddp_output(mp, src_socket, src_addr_included)
 	{ /* begin block */
 	struct	etalk_addr	dest_addr;
 	struct	atalk_addr	dest_at_addr;
-	int		loop = TRUE;		/* flag to aarp to loopback (default) */
+	
+	loop = TRUE;		/* flag to aarp to loopback (default) */
 
 	m = *mp;
 
@@ -852,7 +856,7 @@ int ddp_output(mp, src_socket, src_addr_included)
 	    aarp_send_data(m,ifID, &dest_at_addr, loop);
 	    break;
 	case ET_ADDR :
-	    pat_output(ifID, m, &dest_addr, 0);
+	    pat_output(ifID, m, (unsigned char *)&dest_addr, 0);
 	    break;
         }
 	} /* end block */
@@ -935,7 +939,7 @@ void ddp_input(mp, ifID)
 	len = DDPLEN_VALUE(ddp);
 
 	if (msgsize != len) {
-	        if ((unsigned) msgsize > len) {
+	        if (msgsize > len) {
 		        if (len < DDP_X_HDR_SIZE) {
 			        dPrintf(D_M_DDP, D_L_ERROR,
 				       ("Length problems, ddp length %d, buffer length %d",
@@ -1005,7 +1009,7 @@ void ddp_input(mp, ifID)
 	   for packets of this type on a raw DDP socket *** */
 	if (ddp_handler[socket].func) {
 		dPrintf(D_M_DDP,D_L_INPUT,
-			("ddp_input: skt %d hdnlr:0x%x\n",
+			("ddp_input: skt %u hdnlr:0x%p\n",
 			 (u_int) socket, ddp_handler[socket].func));
 		pktsHome++;
 		snmpStats.dd_inLocal++;
@@ -1132,7 +1136,7 @@ int ddp_router_output(mp, ifID, addr_type, router_net, router_node, enet_addr)
 {
 	register at_ddp_t	*ddp;
 	struct	 atalk_addr	at_dest;
-	int		addr_flag;
+	int		addr_flag = 0;
 	char	*addr = NULL;
 	register gbuf_t	*m;
 
@@ -1292,7 +1296,7 @@ int ddp_router_output(mp, ifID, addr_type, router_net, router_node, enet_addr)
 	      aarp_send_data(m,ifID,&dest_at_addr, loop);
 	      break;
 	    case ET_ADDR :
-	      pat_output(ifID, m, &dest_addr, 0);
+	      pat_output(ifID, m, (unsigned char *)&dest_addr, 0);
 	      break;
 	    }
 	} /* end block */
@@ -1302,12 +1306,13 @@ int ddp_router_output(mp, ifID, addr_type, router_net, router_node, enet_addr)
 
 /*****************************************/
 
+#ifdef AURP_SUPPORT
+
 void rt_delete(NetStop, NetStart)
 	unsigned short NetStop;
 	unsigned short NetStart;
 {
 	RT_entry *found;
-	int s;
 
 	if ((found = rt_bdelete(NetStop, NetStart)) != 0) {
 		bzero(found, sizeof(RT_entry));
@@ -1316,14 +1321,11 @@ void rt_delete(NetStop, NetStart)
 	}
 }
 
-#ifdef AURP_SUPPORT
 int ddp_AURPfuncx(code, param, node)
 	int code;
 	void *param;
 	unsigned char node;
 {
-	extern void rtmp_timeout();
-	extern void rtmp_send_port();
 	at_ifaddr_t *ifID;
 	int k;
 
diff --git a/bsd/netat/ddp.h b/bsd/netat/ddp.h
index 234249649..c270b6908 100644
--- a/bsd/netat/ddp.h
+++ b/bsd/netat/ddp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -165,38 +171,97 @@ typedef	struct {
 
 #ifdef NOT_YET
 /* from sys_glue.c */
-int ddp_adjmsg(gbuf_t *m, int len);
-gbuf_t *ddp_growmsg(gbuf_t  *mp, int len);
 	     
 /* from ddp.c */
-int ddp_add_if(at_ifaddr_t *ifID);
-int ddp_rem_if(at_ifaddr_t *ifID);
 int ddp_bind_socket(ddp_socket_t *socketp);
 int ddp_close_socket(ddp_socket_t *socketp);
-int ddp_output(gbuf_t **mp, at_socket src_socket, int src_addr_included);
-void ddp_input(gbuf_t   *mp, at_ifaddr_t *ifID);
-int ddp_router_output(
-     gbuf_t  *mp,
-     at_ifaddr_t *ifID,
-     int addr_type,
-     at_net_al router_net,
-     at_node router_node,
-     etalk_addr_t *enet_addr);
 
 /* from ddp_proto.c */
 int ddp_close(gref_t *gref);
 void ddp_putmsg(gref_t *gref, gbuf_t *mp);
-gbuf_t *ddp_compress_msg(gbuf_t *mp);
 void ddp_stop(gbuf_t *mioc, gref_t *gref);
 	     
 /* in ddp_lap.c */
-void ddp_bit_reverse(unsigned char *);
 
 #endif /* NOT_YET */
 
-/* in ddp_lap.c */
+void ddp_bit_reverse(unsigned char *);
+
+int	ddp_pru_abort(struct socket *so);
+
+int	ddp_pru_attach(struct socket *so, int proto,
+			       struct proc *p);
+int	ddp_pru_bind(struct socket *so, struct sockaddr *nam,
+			     struct proc *p);
+int	ddp_pru_connect(struct socket *so, struct sockaddr *nam,
+				struct proc *p);
+
+int	ddp_pru_control(struct socket *so, u_long cmd, caddr_t data,
+				struct ifnet *ifp, struct proc *p);
+int	ddp_pru_detach(struct socket *so);
+int	ddp_pru_disconnect(struct socket *so);
+
+int	ddp_pru_peeraddr(struct socket *so, 
+				 struct sockaddr **nam);
+
+int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m, 
+				 struct sockaddr *addr, struct mbuf *control,
+				 struct proc *p);
+
+int	ddp_pru_shutdown(struct socket *so);
+int	ddp_pru_sockaddr(struct socket *so, 
+				 struct sockaddr **nam);
+
+int ddp_output(gbuf_t **, at_socket , int );
+u_short ddp_checksum(gbuf_t	*, int);
+gbuf_t *ddp_compress_msg(gbuf_t *);
+
+struct at_ifaddr;
+struct etalk_addr;
+
+int ddp_router_output(
+     gbuf_t  *mp,
+     struct at_ifaddr *ifID,
+     int addr_type,
+     at_net_al router_net,
+     at_node router_node,
+     struct etalk_addr *enet_addr);
+
+struct at_ifaddr *forUs(at_ddp_t *);
+
+void zip_send_queries(struct at_ifaddr *, at_net_al, at_node);
+int zip_handle_getmyzone(struct at_ifaddr *, gbuf_t *);
+int zip_type_packet(gbuf_t *);
+void zip_sched_getnetinfo (void *);
+
+int at_unreg_mcast(struct at_ifaddr *, caddr_t);
+int at_reg_mcast(struct at_ifaddr *, caddr_t);
+
 int ddp_shutdown(int);
 
+void routing_needed(gbuf_t *, struct at_ifaddr *, char);
+
+int getPhysAddrSize(int);
+int getAarpTableSize(int);
+
+int aarp_init1(struct at_ifaddr *);
+int aarp_init2(struct at_ifaddr *);
+
+int getRtmpTableSize(void);
+
+void sethzonehash(struct at_ifaddr *);
+
+int ddp_add_if(struct at_ifaddr *);
+void ddp_rem_if(struct at_ifaddr *);
+
+void ddp_brt_init(void);
+void ddp_brt_shutdown(void);
+
+int setLocalZones(at_nvestr_t *, int);
+
+void ddp_brt_sweep(void);
+
+
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_DDP_H_ */
diff --git a/bsd/netat/ddp_aarp.c b/bsd/netat/ddp_aarp.c
index 08c50531a..09e1fc460 100644
--- a/bsd/netat/ddp_aarp.c
+++ b/bsd/netat/ddp_aarp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	Copyright (c) 1988, 1989, 1997, 1998 Apple Computer, Inc. 
  *
@@ -56,6 +62,7 @@
 #include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/at_aarp.h>
+#include <netat/at_pat.h>
 #include <netat/debug.h>
 
 #include <sys/kern_event.h>
@@ -76,26 +83,21 @@ struct	etalk_addr	et_zeroaddr = {
 aarp_amt_t		probe_cb;
 aarp_amt_array *aarp_table[IF_TOTAL_MAX];
 
-int aarp_init1(), aarp_init2();
-int aarp_send_data();
-
-StaticProc int aarp_req_cmd_in();
-StaticProc int aarp_resp_cmd_in();
-StaticProc int aarp_probe_cmd_in();
-StaticProc int aarp_send_resp();
-StaticProc int aarp_send_req();
-StaticProc int aarp_send_probe();
-StaticProc aarp_amt_t *aarp_lru_entry();
-StaticProc int aarp_glean_info();
-StaticProc int aarp_delete_amt_info();
-StaticProc void aarp_build_pkt();
-StaticProc void aarp_sched_req(void *);
-StaticProc int aarp_get_rand_node();
-StaticProc int aarp_get_next_node();
-StaticProc int aarp_get_rand_net();
 
-extern void AARPwakeup(aarp_amt_t *);
-extern int pat_output(at_ifaddr_t *, gbuf_t *, unsigned char *, int);
+StaticProc int aarp_req_cmd_in(aarp_pkt_t *, at_ifaddr_t*);
+StaticProc int aarp_resp_cmd_in(aarp_pkt_t *, at_ifaddr_t*);
+StaticProc int aarp_probe_cmd_in(aarp_pkt_t *, at_ifaddr_t*);
+StaticProc int aarp_send_resp(at_ifaddr_t *, aarp_pkt_t *);
+StaticProc int aarp_send_req(aarp_amt_t *);
+StaticProc int aarp_send_probe(void);
+StaticProc aarp_amt_t *aarp_lru_entry(aarp_amt_t *);
+StaticProc int aarp_glean_info(aarp_pkt_t *, at_ifaddr_t*);
+StaticProc int aarp_delete_amt_info(aarp_amt_t	*);
+StaticProc void aarp_build_pkt(aarp_pkt_t *, at_ifaddr_t*);
+StaticProc void aarp_sched_req(void *);
+StaticProc int aarp_get_rand_node(at_ifaddr_t	*);
+StaticProc int aarp_get_next_node(at_ifaddr_t	*);
+StaticProc int aarp_get_rand_net(at_ifaddr_t *);
 
 /****************************************************************************
  * aarp_init()
@@ -299,7 +301,7 @@ StaticProc int aarp_resp_cmd_in (pkt, elapp)
 		}
 		amt_ptr->dest_addr = pkt->src_addr;
 		if (FDDI_OR_TOKENRING(elapp->aa_ifp->if_type))
-			ddp_bit_reverse(&amt_ptr->dest_addr);
+			ddp_bit_reverse((unsigned char *)&amt_ptr->dest_addr);
 		m = amt_ptr->m;
 		amt_ptr->m = NULL;
 		pat_output(amt_ptr->elapp, m,
@@ -561,7 +563,7 @@ StaticProc   int	aarp_send_resp(elapp, pkt)
 
 	gbuf_winc(m,sizeof(aarp_pkt_t));
 	if (FDDI_OR_TOKENRING(elapp->aa_ifp->if_type))
-		ddp_bit_reverse(&new_pkt->dest_addr);
+		ddp_bit_reverse((unsigned char *)&new_pkt->dest_addr);
 
 	if (pat_output(elapp, m, (unsigned char *)&new_pkt->dest_addr,
 		       AARP_AT_TYPE))
@@ -621,7 +623,7 @@ register aarp_amt_t 	*amt_ptr;
  * aarp_send_probe()
  *
  ****************************************************************************/
-StaticProc  int	aarp_send_probe()
+StaticProc  int	aarp_send_probe(void)
 {
 	register aarp_pkt_t  *pkt;
 	register gbuf_t	     *m;
@@ -715,7 +717,7 @@ at_ifaddr_t	*elapp;
 	 */
 	amt_ptr->dest_addr = pkt->src_addr;
 	if (FDDI_OR_TOKENRING(elapp->aa_ifp->if_type))
-		ddp_bit_reverse(&amt_ptr->dest_addr);
+		ddp_bit_reverse((unsigned char *)&amt_ptr->dest_addr);
 	return(1);
 }
 
@@ -748,7 +750,7 @@ register aarp_amt_t	*amt_ptr;
  *
  ****************************************************************************/
 
-void  aarp_sched_probe(void *arg)
+void  aarp_sched_probe(__unused void *arg)
 {
 
 	atalk_lock();
@@ -921,8 +923,8 @@ register at_ifaddr_t	*elapp;
 }
 
 
-int getAarpTableSize(elapId)
-     int	elapId;			/* elap_specifics array index (should be
+int getAarpTableSize(__unused int elapId)
+      			/* elap_specifics array index (should be
 					 * changed when we add a non-ethernet type
 					 * of I/F to the mix. Unused for now.
 					 */
@@ -930,8 +932,8 @@ int getAarpTableSize(elapId)
 	return(AMTSIZE);
 }
 
-int getPhysAddrSize(elapId)
-     int	elapId;			/* elap_specifics array index (should be
+int getPhysAddrSize(__unused int elapId)
+     			/* elap_specifics array index (should be
 					 * changed when we add a non-ethernet type
 					 * of I/F to the mix. Unused for now.
 					 */
diff --git a/bsd/netat/ddp_aep.c b/bsd/netat/ddp_aep.c
index 77e78451e..11beb6b96 100644
--- a/bsd/netat/ddp_aep.c
+++ b/bsd/netat/ddp_aep.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1997-1998 Apple Computer, Inc.
@@ -82,8 +88,8 @@ void ep_input (mp, ifID)
 	 * Check if this AEP message is for us or need to be forwarded
 	 */
 	if (!ROUTING_MODE || 
-	    (ifID->ifThisNode.s_net == NET_VALUE(ddp->dst_net))
-	    && (ifID->ifThisNode.s_node == ddp->dst_node)) {
+	    ((ifID->ifThisNode.s_net == NET_VALUE(ddp->dst_net))
+	    && (ifID->ifThisNode.s_node == ddp->dst_node))) {
 
 		dPrintf(D_M_AEP, D_L_INFO, ("aep_input: received for this port from %d:%d\n",
 			NET_VALUE(ddp->src_net), ddp->src_node));
diff --git a/bsd/netat/ddp_brt.c b/bsd/netat/ddp_brt.c
index 6f125f76e..ce35ae98a 100644
--- a/bsd/netat/ddp_brt.c
+++ b/bsd/netat/ddp_brt.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1988, 1989 Apple Computer, Inc. 
@@ -68,7 +74,6 @@
 ddp_brt_t at_ddp_brt[BRTSIZE];
 int ddp_brt_sweep_timer;
 
-void ddp_brt_sweep();
 
 void ddp_glean(mp, ifID, src_addr)
      register gbuf_t	  *mp;
@@ -145,14 +150,16 @@ void ddp_brt_shutdown()
 }
 
 /* locked version */
+#ifdef NOT_USED
 void ddp_brt_sweep_locked()
 {
 	atalk_lock();
 	ddp_brt_sweep();
 	atalk_unlock();
 }
+#endif
 
-void ddp_brt_sweep()
+void ddp_brt_sweep(void)
 {
         register ddp_brt_t      *brt;
 	register int		i;
diff --git a/bsd/netat/ddp_lap.c b/bsd/netat/ddp_lap.c
index a5361340e..c7e075c5a 100644
--- a/bsd/netat/ddp_lap.c
+++ b/bsd/netat/ddp_lap.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1988, 1989, 1993-1998 Apple Computer, Inc. 
@@ -66,6 +72,7 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/lap.h>
@@ -73,13 +80,14 @@
 #include <netat/zip.h>
 #include <netat/nbp.h>
 #include <netat/at_snmp.h>
-#include <netat/at_pcb.h>
 #include <netat/at_aarp.h>
 #include <netat/asp.h>
 #include <netat/atp.h>
 #include <netat/debug.h>
 #include <netat/adsp.h>
 #include <netat/adsp_internal.h>
+#include <netat/at_pat.h>
+#include <netat/rtmp.h>
 
 #include <sys/kern_event.h>
 
@@ -128,19 +136,21 @@ extern at_ddp_stats_t at_ddp_stats;
 extern lck_mtx_t * atalk_mutex;
 
 /* protos */
-extern snmpAarpEnt_t * getAarp(int *);
-extern void nbp_shutdown(), routershutdown(), ddp_brt_shutdown();
-extern void ddp_brt_init(), rtmp_init(), rtmp_input();
-extern rtmp_router_start(at_kern_err_t *);
-static void getIfNames(at_ifnames_t *);
-static void add_route();
-static int set_zones();
-void elap_offline();
-static int elap_online1(), re_aarp();
-int at_reg_mcast(), at_unreg_mcast();
-void  AARPwakeup(), ZIPwakeup();
-static void elap_hangup();
-static getSnmpCfg();
+int rtmp_router_start(at_kern_err_t *);
+static void add_route(RT_entry 	*);
+void elap_offline(at_ifaddr_t *);
+static int elap_online1(at_ifaddr_t *);
+static void elap_online2(at_ifaddr_t *);
+ int elap_online3(at_ifaddr_t *);
+static int re_aarp(at_ifaddr_t *);
+static int getSnmpCfg(snmpCfg_t *);
+
+int routerStart(at_kern_err_t *);
+
+static int validate_msg_size(gbuf_t *, gref_t *, at_ifaddr_t **);
+at_ifaddr_t *find_ifID(char *);
+int lap_online( at_ifaddr_t *, at_if_cfg_t *cfgp);
+
 
 at_ifaddr_t *find_ifID(if_name)
 	char	*if_name;
@@ -166,7 +176,6 @@ static int validate_msg_size(m, gref, elapp)
 */
 {
 	register ioc_t *iocbp;
-	register at_if_cfg_t *cfgp;
 	int i = 0, size = 1;
 	
 	*elapp = NULL;		
@@ -249,8 +258,9 @@ int lap_online(elapp, cfgp)
 			elapp->flags |= ELAP_CFG_SEED;
 	}
 
-	if (!DEFAULT_ZONE(&cfgp->zonename) &&
-	    (elapp->flags & ELAP_CFG_HOME) || MULTIHOME_MODE) {
+	/* (VL) !? */
+	if ((!DEFAULT_ZONE(&cfgp->zonename) &&
+	    (elapp->flags & ELAP_CFG_HOME)) || MULTIHOME_MODE) {
 		elapp->startup_zone = cfgp->zonename;
 	}
 
@@ -306,10 +316,12 @@ int elap_wput(gref, m)
 	register ioc_t		*iocbp;
 	register at_if_cfg_t	*cfgp;
 	at_elap_stats_t		*statsp;
-	int 		i;
-	int			(*func)();
-	gbuf_t		*tmpm;
-	at_ifaddr_t *patp;
+	int i,j;
+	int size, totalsize = 0, tabsize;
+	gbuf_t	*mn;		/* new gbuf */
+	gbuf_t	*mo;		/* old gbuf */
+	gbuf_t	*mt = NULL;		/* temp */
+	snmpNbpTable_t		*nbp;
 
 
 	switch (gbuf_type(m)) {
@@ -458,7 +470,6 @@ int elap_wput(gref, m)
 				kprintf("LAP_IOC_SNMP_GET_CFG\n");
 #endif
 			{
-				int i,size;
 				snmpCfg_t 	snmp;
 
 				i =  *(int *)gbuf_rptr(gbuf_cont(m));
@@ -542,12 +553,6 @@ int elap_wput(gref, m)
 				kprintf("LAP_IOC_SNMP_GET_ZIP\n");
 #endif
 			{ /* matching brace NOT in this case */
-				register int i,j;
-				register int size, total, tabsize;
-				gbuf_t	*mn;		/* new gbuf */
-				gbuf_t	*mo;		/* old gbuf */
-				gbuf_t	*mt;		/* temp */
-				snmpNbpTable_t		*nbp;
 
 				i =  *(int *)gbuf_rptr(gbuf_cont(m));
 				gbuf_freem(gbuf_cont(m));
@@ -578,11 +583,11 @@ int elap_wput(gref, m)
 					}
 					if (!mo)	{ 		/* if first new one */
 						mt = mn;
-						total = size;
+						totalsize = size;
 					}
 					else {
 						gbuf_cont(mo) = mn;
-						total += size;
+						totalsize += size;
 					}
 					mo = mn;
 					getZipTable((ZT_entry*)gbuf_rptr(mn),i,j); 
@@ -598,9 +603,9 @@ int elap_wput(gref, m)
 				if (!tabsize) {
 					dPrintf(D_M_ELAP,D_L_WARNING,
 						("elap_wput:snmp: empty zip table\n"));
-					total = 0;
+					totalsize = 0;
 				}
-				*(int*)gbuf_rptr(gbuf_cont(m)) = total; 	/* return table size */
+				*(int*)gbuf_rptr(gbuf_cont(m)) = totalsize; 	/* return table size */
 				gbuf_wset(gbuf_cont(m),sizeof(int));
 				iocbp->ioc_count = sizeof(int);
 				ioc_ack(0, m, gref);
@@ -643,11 +648,11 @@ int elap_wput(gref, m)
 					}
 					if (!mo)	{ 		/* if first new one */
 						mt = mn;
-						total = size;
+						totalsize = size;
 					}
 					else {
 						gbuf_cont(mo) = mn;
-						total += size;
+						totalsize += size;
 					}
 					mo = mn;
 					getRtmpTable((RT_entry*)gbuf_rptr(mn),i,j); 
@@ -661,8 +666,8 @@ int elap_wput(gref, m)
 					break;
 				}
 				if (!tabsize)
-					total = 0;
-				*(int*)gbuf_rptr(gbuf_cont(m)) = total;	/* return table size */
+					totalsize = 0;
+				*(int*)gbuf_rptr(gbuf_cont(m)) = totalsize;	/* return table size */
 				gbuf_wset(gbuf_cont(m),sizeof(int));
 				iocbp->ioc_count = sizeof(int);
 				ioc_ack(0, m, gref);
@@ -709,7 +714,7 @@ int elap_wput(gref, m)
 					}
 					if (!mo)	{ 		/* if first new one */
 						mt = mn;
-						total = size;
+						totalsize = size;
 						nbp = (snmpNbpTable_t*)gbuf_rptr(mn);
 						nbp->nbpt_entries = tabsize;
 						nbp->nbpt_zone = ifID_home->ifZoneName;
@@ -717,7 +722,7 @@ int elap_wput(gref, m)
 					}
 					else {
 						gbuf_cont(mo) = mn;
-						total += size;
+						totalsize += size;
 						getNbpTable((snmpNbpEntry_t *)gbuf_rptr(mn),i,j); 
 					}
 					mo = mn;
@@ -731,8 +736,8 @@ int elap_wput(gref, m)
 					break;
 				}
 				if (!tabsize)
-					total = 0;
-				*(int*)gbuf_rptr(gbuf_cont(m)) = total;	/* return table size */
+					totalsize = 0;
+				*(int*)gbuf_rptr(gbuf_cont(m)) = totalsize;	/* return table size */
 				gbuf_wset(gbuf_cont(m),sizeof(int));
 				iocbp->ioc_count = sizeof(int);
 				ioc_ack(0, m, gref);
@@ -768,6 +773,7 @@ int elap_wput(gref, m)
 
 /* Called directly by ddp/zip.
  */
+int
 elap_dataput(m, elapp, addr_flag, addr)
      register	gbuf_t	*m;
      register at_ifaddr_t *elapp;
@@ -776,10 +782,8 @@ elap_dataput(m, elapp, addr_flag, addr)
 {
 	register int		size;
 	int			error = 0;
-	extern	int		zip_type_packet();
 	struct	etalk_addr	dest_addr;
 	struct	atalk_addr	dest_at_addr;
-	extern	gbuf_t		*growmsg();
 	int			loop = TRUE;
 				/* flag to aarp to loopback (default) */
 
@@ -856,7 +860,7 @@ elap_dataput(m, elapp, addr_flag, addr)
 	    error = aarp_send_data(m, elapp, &dest_at_addr, loop);
 	    break;
 	case ET_ADDR :
-	    error = pat_output(elapp, m, &dest_addr, 0);
+	    error = pat_output(elapp, m, (unsigned char *)&dest_addr, 0);
 	    break;
         }
 	return (error);
@@ -955,9 +959,11 @@ static void elap_online2(elapp)
 			/* LD 081694: set the RTR_SEED_PORT flag for seed ports */
 			elapp->ifFlags |= RTR_SEED_PORT;
 		}
+#if DEBUG
 		else 
 			dPrintf(D_M_ELAP,D_L_STARTUP_INFO,
 				("elap_online: it's a router, but non seed\n"));
+#endif
 	}
 
 	if (elapp->flags & ELAP_CFG_ZONELESS) {
@@ -1022,9 +1028,6 @@ void elap_offline(elapp)
      register at_ifaddr_t *elapp;
 
 {
-	void	zip_sched_getnetinfo(); /* forward reference */
-	int	errno;
-
 	dPrintf(D_M_ELAP, D_L_SHUTDN_INFO, ("elap_offline:%s\n", elapp->ifName));
 	if (elapp->ifState != LAP_OFFLINE) {
 
@@ -1091,7 +1094,6 @@ int ddp_shutdown(count_only)
 	struct atp_state *atp, *atp_next;
 	CCB *sp, *sp_next;
 	gref_t *gref;
-	vm_offset_t temp_rcb_data, temp_state_data;
 	int i, active_skts = 0;	/* count of active pids for non-socketized
 				   AppleTalk protocols */
 
@@ -1416,6 +1418,8 @@ static unsigned char reverse_data[] = {
 		addr[k] = reverse_data[addr[k]];
 }
 
+static int elap_trackMcast(at_ifaddr_t *, int, caddr_t);
+
 static int elap_trackMcast(patp, func, addr)
 	at_ifaddr_t    *patp;
 	int func;
@@ -1495,7 +1499,7 @@ static int elap_trackMcast(patp, func, addr)
 }
 
 
-static getSnmpCfg(snmp)
+static int getSnmpCfg(snmp)
 	snmpCfg_t *snmp;
 {
 	int i;
@@ -1598,7 +1602,7 @@ int at_reg_mcast(ifID, data)
 			 *(unsigned*)data, (*(unsigned *)(data+2))&0x0000ffff, 
 			 (unsigned)ifID));
 
-		if (if_addmulti(nddp, &sdl, 0))
+		if (if_addmulti(nddp, (struct sockaddr *)&sdl, 0))
 			return -1;
 	}
 	return 0;
@@ -1637,7 +1641,7 @@ int at_unreg_mcast(ifID, data)
 			 (unsigned)ifID));
 		bzero(data, sizeof(struct etalk_addr));
 
-		if (if_delmulti(nddp, &sdl))
+		if (if_delmulti(nddp, (struct sockaddr *)&sdl))
 			return -1;
 	}
 	return 0;
diff --git a/bsd/netat/ddp_nbp.c b/bsd/netat/ddp_nbp.c
index 8b51c6128..698b1083e 100644
--- a/bsd/netat/ddp_nbp.c
+++ b/bsd/netat/ddp_nbp.c
@@ -1,27 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- *	Copyright (c) 1988, 1989, 1997, 1998 Apple Computer, Inc. 
- *
  *   Modified, March 17, 1997 by Tuyen Nguyen for MacOSX.
  */
 
@@ -47,6 +51,7 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/nbp.h>
@@ -54,7 +59,6 @@
 #include <netat/rtmp.h>
 #include <netat/routing_tables.h>	/* router */
 #include <netat/at_snmp.h>
-#include <netat/at_pcb.h>
 #include <netat/debug.h>
 
 /* reaching for DDP and NBP headers in the datagram */
@@ -82,60 +86,25 @@ static	int		errno;
 static  gbuf_t  *lzones=0;	/* head of local zones list */
 static int	lzonecnt=0;		/* # zones stored in lzones	*/
 static u_int  hzonehash=0;	/* hash val of home zone */
-static int	nve_lock_pri;
 
 static int	nbp_lkup_reply(nbp_req_t *, nve_entry_t *);
 static int	nbp_strcmp(at_nvestr_t *, at_nvestr_t *, u_char);
 static int	nbp_setup_resp(nbp_req_t *, int);
 static int	nbp_send_resp(nbp_req_t *);
 static int	nbp_validate_n_hash(nbp_req_t *, int, int);
-static	nve_entry_t	*nbp_search_nve();
+static	nve_entry_t	*nbp_search_nve(nbp_req_t *, at_ifaddr_t *);
 static int isZoneLocal(at_nvestr_t *);
+static int nbp_enum_gen (nve_entry_t *);
+static	void	nbp_setup_hdr (nbp_req_t *);
+static	void	nbp_upshift (u_char *, int);
+static u_char *nbp2zone(at_nbp_t *, u_char *);
 
 /* macros */
 #define NVE_LOCK nve_lock
 
-/* prototypes */
-void nbp_delete_entry();
-extern int at_reg_mcast();
-extern at_nvestr_t *getRTRLocalZone(zone_usage_t *);
-extern void	nbp_add_multicast( at_nvestr_t *, at_ifaddr_t *);
 
 static long nbp_id_count = 0;
 
-/*
- * Copy src to string dst of size siz.  At most siz-1 characters
- * will be copied.  Always NUL terminates (unless siz == 0).
- * Returns strlen(src); if retval >= siz, truncation occurred.
- */
-__private_extern__ size_t
-strlcpy(char *dst, const char *src, size_t siz)
-{
-        char *d = dst;
-        const char *s = src;
-        size_t n = siz;
-
-        /* Copy as many bytes as will fit */
-        if (n != 0 && --n != 0) {
-                do {
-                        if ((*d++ = *s++) == 0)
-                                break;
-                } while (--n != 0);
-        }
-
-        /* Not enough room in dst, add NUL and traverse rest of src */
-        if (n == 0) {
-                if (siz != 0)
-                        *d = '\0';              /* NUL-terminate dst */
-                while (*s++)
-                        ;
-        }
-
-        return(s - src - 1);    /* count does not include NUL */
-}
-
-
-
 void sethzonehash(elapp)
      at_ifaddr_t *elapp;
 {
@@ -144,7 +113,7 @@ void sethzonehash(elapp)
 	}
 }
 
-void nbp_shutdown()
+void nbp_shutdown(void)
 {
 	/* delete all NVE's and release buffers */
 	register nve_entry_t	*nve_entry, *nve_next;
@@ -188,7 +157,6 @@ void nbp_input(m, ifID)
 {
 	register at_ddp_t 	*ddp = DATA_DDP(m);
 	register at_nbp_t	*nbp = DATA_NBP(m);
-	register nve_entry_t	*nve_entry, *next_nve;
 	register RT_entry	*rt;
 	register int ddpSent = FALSE; 	/* true if we re-sent this pkt (don't free) */
 	struct etalk_addr mcastAddr;
@@ -313,15 +281,15 @@ void nbp_input(m, ifID)
 
  /*### LD 01/18/94 Check if the dest is also the home zone. */
  
- 		p = nbp2zone(nbp, gbuf_wptr(m));
- 		if ((p == NULL) || !(zno = zt_find_zname(p))) {
+ 		p = nbp2zone(nbp, (u_char *)gbuf_wptr(m));
+ 		if ((p == NULL) || !(zno = zt_find_zname((at_nvestr_t *)p))) {
  			dPrintf(D_M_NBP,D_L_WARNING,
  				("nbp_input: FWDRQ:zone not found\n"));
 			break;
  		}
 		if (isZoneLocal((at_nvestr_t*)p)) 
 			zhome = TRUE;				/* one of our  ports is in destination zone */
-		if (!zt_get_zmcast(ifID, p, &mcastAddr)) {
+		if (!zt_get_zmcast(ifID, (at_nvestr_t*)p, (char *)&mcastAddr)) {
 			dPrintf(D_M_NBP,D_L_ERROR,
 				("nbp_input: FDWREQ:zt_get_zmcast error\n"));
 			break;
@@ -347,8 +315,8 @@ void nbp_input(m, ifID)
  		}
  
 		if (FDDI_OR_TOKENRING(ifID->aa_ifp->if_type))
-			ddp_bit_reverse(&mcastAddr);
-		ddp_router_output(m, ifID, ET_ADDR,NULL,NULL, &mcastAddr);
+			ddp_bit_reverse((unsigned char *)&mcastAddr);
+		ddp_router_output(m, ifID, ET_ADDR, 0, 0, &mcastAddr);
 		ddpSent = TRUE;
 		}
 		break;
@@ -362,7 +330,7 @@ void nbp_input(m, ifID)
 		register int	i;
 		register  gbuf_t	*m2, *m3;
 		register int fromUs = FALSE;
-		register at_socket ourSkt;	/* originating skt */
+		register at_socket ourSkt = 0;	/* originating skt */
 
 		/* for router & MH local only */
 		if ((!(MULTIHOME_MODE && FROM_US(ddp))) && !ROUTING_MODE) {
@@ -371,8 +339,8 @@ void nbp_input(m, ifID)
 
 			break;
 		}
- 		p = nbp2zone(nbp, gbuf_wptr(m));
-		if ((p == NULL) || !(zno = zt_find_zname(p))) {
+ 		p = nbp2zone(nbp, (u_char *)gbuf_wptr(m));
+		if ((p == NULL) || !(zno = zt_find_zname((at_nvestr_t *)p))) {
 			break;
 		}
 		if (MULTIHOME_MODE && ifID->ifRouterState == NO_ROUTER) {
@@ -435,24 +403,25 @@ void nbp_input(m, ifID)
 				nbp->tuple[0].enu_addr.socket = ourSkt; 
 				ddp->src_socket = NBP_SOCKET;
 			}
+#if DEBUG
 			else
 				dPrintf(D_M_NBP, D_L_USR3, 
 					("nbp_input: BRREQ: not from us\n"));
-
+#endif /* DEBUG */
 			dPrintf(D_M_NBP, D_L_USR3,
 				("nbp_input dist:%d\n", rt->NetDist));
 			if (rt->NetDist == 0) {			/* if direct connect, *we* do the LKUP */
 				nbp->control  = NBP_LKUP;
 	 			NET_ASSIGN(ddp->dst_net, 0);
 	 			ddp->dst_node = 255;
-				if (!zt_get_zmcast(ifID, p, &mcastAddr)) {
+				if (!zt_get_zmcast(ifID, (at_nvestr_t*)p, (char *)&mcastAddr)) {
 					dPrintf(D_M_NBP,D_L_ERROR, 
 						("nbp_input: BRRQ:zt_get_zmcast error\n"));
 					break;
 				}
 				if (FDDI_OR_TOKENRING(ifID->aa_ifp->if_type))
-					ddp_bit_reverse(&mcastAddr);
-				ddp_router_output(m2, ifID, ET_ADDR, NULL, NULL, &mcastAddr); 
+					ddp_bit_reverse((unsigned char *)&mcastAddr);
+				ddp_router_output(m2, ifID, ET_ADDR, 0, 0, &mcastAddr); 
 			}
 			else {							/* else fwd to router */
 				ddp->dst_node = 0;
@@ -565,7 +534,7 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 
 	if (checkLocal && !isZoneLocal(zone)) {
 		char str[35];
-		strlcpy(str,zone->str,sizeof(str));
+		strlcpy((char *)str,(char *)zone->str,sizeof(str));
 		dPrintf(D_M_NBP_LOW,D_L_WARNING,
 			("nbp_val_n_hash bad zone: %s\n", str));
 		errno = EINVAL;
@@ -591,8 +560,8 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 		}
 	} else{
 		for (i = part_wild = 0; (unsigned) i<object->len; i++) {
-			if (object->str[i] == NBP_SPL_WILDCARD)
-				if (wild_ok)
+			if (object->str[i] == NBP_SPL_WILDCARD) {
+				if (wild_ok) {
 					if (part_wild) {
 					  dPrintf(D_M_NBP_LOW, D_L_WARNING, 
 						  ("nbp_val_n_hash: too many parts wild\n"));
@@ -600,12 +569,13 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 					  return (-1);
 					} else
 					  part_wild++;
-				else {
+				} else {
 				  dPrintf(D_M_NBP_LOW, D_L_WARNING, 
 					  ("nbp_val_n_hash: wild not okay2\n"));
 				  errno = EINVAL;
 				  return (-1);
 				}
+			}
 			nbp_req->nve.object.str[i] = object->str[i];
 		}
 		if (!part_wild)
@@ -627,8 +597,8 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 		}
 	} else {
 		for (i = part_wild = 0; (unsigned) i<type->len; i++) {
-			if (type->str[i] == NBP_SPL_WILDCARD)
-				if (wild_ok)
+			if (type->str[i] == NBP_SPL_WILDCARD) {
+				if (wild_ok) {
 					if (part_wild) {
 					  dPrintf(D_M_NBP_LOW, D_L_WARNING, 
 						  ("nbp_val_n_hash: too many parts wild2\n"));
@@ -636,10 +606,11 @@ static	int	nbp_validate_n_hash (nbp_req, wild_ok, checkLocal)
 					  return (-1);
 					} else
 					  part_wild++;
-				else {
+				} else {
 					errno = EINVAL;
 					return (-1);
 				}
+			}
 			nbp_req->nve.type.str[i] = type->str[i];
 		}
 		if (!part_wild)
@@ -699,7 +670,7 @@ u_int nbp_strhash (nvestr)
 	} un;
 
 	for (i=0; (unsigned) i < nvestr->len; i+=sizeof(int)) {
-		len = MIN((nvestr->len-i), sizeof(int));
+		len = MIN((size_t)(nvestr->len-i), sizeof(int));
 		if (len == sizeof(int))
 			bcopy(&(nvestr->str[i]), &un, sizeof(un));
 		else {
@@ -763,11 +734,13 @@ static	nve_entry_t *nbp_search_nve (nbp_req, ifID)
 					 ifID->ifThisNode.s_net));
 				continue;
 			    }
+#if DEBUG
 			    if (ifID)
 				dPrintf(D_M_NBP, D_L_USR4, 
 					("nbp search ifID (%d) & req net (%d)  equal\n",
 					 nve_entry->address.net,
 					 ifID->ifThisNode.s_net));
+#endif /* DEBUG */
 			}
 		
 		}
@@ -1026,31 +999,30 @@ void nbp_add_multicast(zone, ifID)
      at_ifaddr_t *ifID;
 {
 	char data[ETHERNET_ADDR_LEN];
-	int i;
 
 	if (zone->str[0] == '*')
 		return;
 
 	{
 	  char str[35];
-	  strlcpy(str,zone->str,sizeof(str));
+	  strlcpy((char *)str,(char *)zone->str,sizeof(str));
 	  dPrintf(D_M_NBP_LOW, D_L_USR3,
 		  ("nbp_add_multi getting mc for %s\n", str));
 	}
 	zt_get_zmcast(ifID, zone, data); 
 	if (FDDI_OR_TOKENRING(ifID->aa_ifp->if_type))
-	  ddp_bit_reverse(data);
+	  ddp_bit_reverse((unsigned char *)data);
 	dPrintf(D_M_NBP_LOW,D_L_USR3,
 		("nbp_add_multi adding  0x%x%x port:%d ifID:0x%x if:%s\n",
 		 *(unsigned*)data, (*(unsigned *)(data+2))&0x0000ffff,
-		 i, (u_int) ifID, ifID->ifName));
+		 /*i*/0, (u_int) ifID, ifID->ifName));
 
 	bcopy((caddr_t)data, (caddr_t)&ifID->ZoneMcastAddr, ETHERNET_ADDR_LEN);
 	(void)at_reg_mcast(ifID, (caddr_t)&ifID->ZoneMcastAddr);
 }
 
-
-getNbpTableSize()
+int
+getNbpTableSize(void)
 
 /* for SNMP, returns size in # of entries */
 {
@@ -1062,6 +1034,7 @@ getNbpTableSize()
 	return(i);
 }
 
+int
 getNbpTable(p, s, c)
      snmpNbpEntry_t	*p;
      int 		s;		/* starting entry */
@@ -1095,6 +1068,8 @@ getNbpTable(p, s, c)
 		next = (nve_entry_t*)NULL;
 		nextNo = 0;
 	}
+	
+	return 0;
 }
 
 
@@ -1121,25 +1096,26 @@ int setLocalZones(newzones, size)
 	while (bytesread < size) {		/* for each new zone */
 		{
 			char str[35];
-			strlcpy(str,pnew->str,sizeof(str));
+			strlcpy((char *)str,(char *)pnew->str,sizeof(str));
 		}
 		m = lzones;				
 		pnve = (at_nvestr_t*)gbuf_rptr(m);
 		dupe = 0;
 		for (i=0; i<lzonecnt && !dupe; i++,pnve++)  {
-			if (i && !(i%ZONES_PER_BLK))
+			if (i && !(i%ZONES_PER_BLK)) {
 				if (gbuf_cont(m)) {
 					m = gbuf_cont(m);
 					pnve = (at_nvestr_t*)gbuf_rptr(m);
 				}
 				else
 					break;
+			}
 			if (pnew->len != pnve->len)
 				continue;
 			if (pnew->len > NBP_NVE_STR_SIZE) {
 				return(0);
 			}
-			if (!strncmp(pnew->str, pnve->str, pnew->len)) {
+			if (!strncmp((char *)pnew->str, (char *)pnve->str, pnew->len)) {
 				dupe=1;
 				continue;
 			}
@@ -1152,7 +1128,7 @@ int setLocalZones(newzones, size)
 				gbuf_wset(gbuf_cont(m),0);
 				pnve = (at_nvestr_t*)gbuf_rptr(gbuf_cont(m));
 			}
-			strlcpy(pnve->str,pnew->str,sizeof(pnve->str));
+			strlcpy((char *)pnve->str,(char *)pnew->str,sizeof(pnve->str));
 			pnve->len = pnew->len;
 			lzonecnt++;
 		}
@@ -1181,6 +1157,7 @@ showLocalZones1()
 
 *********/
 
+int
 isZoneLocal(zone)
 at_nvestr_t *zone;
 {
diff --git a/bsd/netat/ddp_proto.c b/bsd/netat/ddp_proto.c
index 6a940157f..b55d1968c 100644
--- a/bsd/netat/ddp_proto.c
+++ b/bsd/netat/ddp_proto.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1988, 1989, 1997, 1998 Apple Computer, Inc. 
@@ -47,10 +53,10 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/zip.h>
-#include <netat/at_pcb.h>
 
 extern at_ifaddr_t *ifID_home;
 
@@ -58,7 +64,6 @@ void ddp_putmsg(gref, mp)
 	gref_t *gref;
 	gbuf_t *mp;
 {
-	u_char socket;
 	register ioc_t	*iocbp;
 	register int	error;
 	at_ddp_t *ddp;
diff --git a/bsd/netat/ddp_r_rtmp.c b/bsd/netat/ddp_r_rtmp.c
index 932861d27..584508e63 100644
--- a/bsd/netat/ddp_r_rtmp.c
+++ b/bsd/netat/ddp_r_rtmp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1994, 1996-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*--------------------------------------------------------------------------
  * Router RTMP protocol functions: 
@@ -57,10 +63,10 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/rtmp.h>
-#include <netat/at_pcb.h>
 #include <netat/zip.h>
 #include <netat/routing_tables.h>
 #include <netat/aurp.h>
@@ -68,32 +74,34 @@
 
 #include <sys/kern_event.h>
 
-extern void (*ddp_AURPsendx)();
+extern void (*ddp_AURPsendx)(void);
 extern at_ifaddr_t *aurp_ifID;
 extern at_ifaddr_t *ifID_table[];
 extern at_ifaddr_t *ifID_home;
 
-/*DEBUG ONLY */
-static int dump_counter =0;
-/*DEBUG ONLY */
+
+int rtmp_router_start(at_kern_err_t *);
+void rtmp_router_start_tmo(void *);
+
+
 
 static at_kern_err_t ke; 
 			/* Used to record error discovered in rtmp_update() */
-gbuf_t *rtmp_prep_new_packet();
 
-void rtmp_timeout();
-void rtmp_send_port();
-void rtmp_send_port_locked();
+void rtmp_timeout(void *arg);
+void rtmp_send_port(at_ifaddr_t *);
+void rtmp_send_port_locked(void *);
 void rtmp_dropper(void *);
-void rtmp_shutdown();
-static void rtmp_update();
-static void rtmp_request();
-extern int elap_online3();
+static void rtmp_update(at_ifaddr_t *, at_rtmp *, short);
+static void rtmp_request(at_ifaddr_t *, at_ddp_t *);
+int elap_online3(at_ifaddr_t	*);
 
-extern pktsIn, pktsOut, pktsDropped, pktsHome;
 extern short ErrorRTMPoverflow, ErrorZIPoverflow;
 extern lck_mtx_t * atalk_mutex;
 
+extern int pktsIn, pktsOut, pktsDropped, pktsHome;
+
+
 /*
  * rtmp_router_input: function called by DDP (in router mode) to handle
  *                    all incoming RTMP packets. Listen to the RTMP socket
@@ -251,11 +259,11 @@ static void rtmp_update(ifID, rtmp, tuple_nb)
 	register at_rtmp_tuple *FirstTuple =  (at_rtmp_tuple *)&rtmp->at_rtmp_id[1];
 	register at_rtmp_tuple *SecondTuple = (at_rtmp_tuple *)&rtmp->at_rtmp_id[4];
 	RT_entry NewRoute, *CurrentRoute;
-
 	register u_char SenderNodeID = rtmp->at_rtmp_id[0];
 	char *TuplePtr;
 	short state;
 
+	bzero(&NewRoute, sizeof(RT_entry));
 
 	/* Make sure this an AppleTalk node sending us the RTMP packet */
 
@@ -466,14 +474,14 @@ static void rtmp_update(ifID, rtmp, tuple_nb)
 				state = CurrentRoute->EntryState & 0x0F;
 				/* if entry has been updated recently, just clear the UPDATED 
 				   bit. if bit not set, then we can age the entry */
-				if (state)
+				if (state) {
 					if (CurrentRoute->EntryState & RTE_STATE_UPDATED) {
 						CurrentRoute->EntryState &= ~RTE_STATE_UPDATED; 
 					}
 					else {
 						state  = state >> 1 ;	/* decrement state */
 					}
-
+				}
 				CurrentRoute->EntryState = (CurrentRoute->EntryState & 0xF0) | state;
 			}
 		}
@@ -736,9 +744,9 @@ static void rtmp_update(ifID, rtmp, tuple_nb)
  *      RTE_STATE_UNUSED :  Unused or removed entry in the table
  */
 
-void rtmp_timeout(ifID)
-register at_ifaddr_t        *ifID;
+void rtmp_timeout(void *arg)
 {
+		at_ifaddr_t *ifID = (at_ifaddr_t *)arg;
 		register u_char state;
 		short i;
 		RT_entry *en = &RT_table[0];
@@ -818,6 +826,8 @@ register at_ifaddr_t        *ifID;
  *
  */				
 		
+gbuf_t *rtmp_prep_new_packet (at_ifaddr_t *, at_net, u_char, char);
+
 gbuf_t *rtmp_prep_new_packet (ifID, DstNet, DstNode, socket)
 register at_ifaddr_t        *ifID;
 register at_net DstNet;
@@ -878,6 +888,8 @@ register char socket;
 
 
 }
+int rtmp_r_find_bridge(at_ifaddr_t *, at_ddp_t *);
+
 int rtmp_r_find_bridge(ifID, orig_ddp)
 register at_ifaddr_t    *ifID;
 register at_ddp_t 	*orig_ddp;
@@ -951,8 +963,8 @@ register at_ddp_t 	*orig_ddp;
 
 	dPrintf(D_M_RTMP, D_L_INFO, ("rtmp_r_find_bridge: for net %d send back router %d.%d\n",
 				NET_VALUE(orig_ddp->dst_net), Entry->NextIRNet, Entry->NextIRNode));
-	if (status = ddp_router_output(m, ifID, AT_ADDR, NET_VALUE(orig_ddp->src_net),
-			orig_ddp->src_node, 0)){
+	if ((status = ddp_router_output(m, ifID, AT_ADDR, NET_VALUE(orig_ddp->src_net),
+			orig_ddp->src_node, 0))){
 		dPrintf(D_M_RTMP, D_L_WARNING,
 			("rtmp_r_find_bridge: ddp_router_output failed status=%d\n", status));
 				return (status);
@@ -969,6 +981,8 @@ register at_ddp_t 	*orig_ddp;
  *	the table...
  *
  */
+static int rtmp_send_table(at_ifaddr_t *, at_net, u_char, short, char, short);
+
 static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket, 
 			   n_neighbors)
      register at_ifaddr_t *ifID;	/* interface/port params */
@@ -1040,10 +1054,10 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
 		}
 
 		if (size > (DDP_DATA_SIZE-20)) {
-			DDPLEN_ASSIGN(ddp, size + DDP_X_HDR_SIZE + 10);
+			DDPLEN_ASSIGN(ddp, (size + DDP_X_HDR_SIZE + 10));
 			gbuf_winc(m,size);
-			if (status = ddp_router_output(m, ifID, AT_ADDR,
-				NET_VALUE(DestNet),DestNode, 0)){
+			if ((status = ddp_router_output(m, ifID, AT_ADDR,
+				NET_VALUE(DestNet),DestNode, 0))){
 			  dPrintf(D_M_RTMP, D_L_WARNING,
 				  ("rtmp_send_table: ddp_router_output failed status=%d\n",
 				   status));
@@ -1074,7 +1088,7 @@ static int rtmp_send_table(ifID, DestNet, DestNode, split_hz, socket,
          * otherwise, the last packet we sent was full, we need to send an empty one
          */
 
-	DDPLEN_ASSIGN(ddp, size + DDP_X_HDR_SIZE + 10);
+	DDPLEN_ASSIGN(ddp, (size + DDP_X_HDR_SIZE + 10));
 	gbuf_winc(m,size);
 	if ((status = 
 	     ddp_router_output(m, ifID, AT_ADDR, NET_VALUE(DestNet),DestNode, 0))){
@@ -1165,9 +1179,9 @@ static void rtmp_request(ifID, ddp)
 }
 
 /* locked version of rtmp_send_port */
-void rtmp_send_port_locked(ifID)
-     register at_ifaddr_t *ifID;
+void rtmp_send_port_locked(void *arg)
 {
+	at_ifaddr_t *ifID = (at_ifaddr_t *)arg;
 	atalk_lock();
 	rtmp_send_port(ifID);
 	atalk_unlock();
@@ -1200,6 +1214,7 @@ void rtmp_send_port(ifID)
 		rtmp_send_table(ifID, DestNet, 0xFF, 1, RTMP_SOCKET, 0);
 	}
 
+#ifdef DEBUG
 	if (ifID == ifID_home)
 		dPrintf(D_M_RTMP_LOW, D_L_VERBOSE,
 			("I:%5d O:%5d H:%5d dropped:%d\n",
@@ -1208,6 +1223,8 @@ void rtmp_send_port(ifID)
 	dPrintf(D_M_RTMP_LOW, D_L_TRACE,
 		("rtmp_send_port: func=0x%x, ifID=0x%x\n", 
 		 (u_int) rtmp_send_port, (u_int) ifID));
+#endif
+
 	timeout (rtmp_send_port_locked, (caddr_t)ifID, 10 * SYS_HZ);
 
 }
@@ -1216,7 +1233,7 @@ void rtmp_send_port(ifID)
  *               the actual packet dropping is done in ddp_input
  */
 
-void rtmp_dropper(void *arg)
+void rtmp_dropper(__unused void *arg)
 {
 
 	atalk_lock();
@@ -1235,8 +1252,7 @@ void rtmp_dropper(void *arg)
  *   ### LD 01/09/95 Changed to correct Zone problem on non seed ports.
  */
      
-int rtmp_router_start(keP)
-     at_kern_err_t *keP; /* used to report errors (if any) */
+int rtmp_router_start(at_kern_err_t *keP)
 {
 	int err = 0;
 	register at_ifaddr_t *ifID, *ifID2;
@@ -1574,11 +1590,14 @@ error:
 	return((keP->error)? 0: err);
 } /* rtmp_router_start */
 
+void rtmp_router_start_tmo(void *arg)
+{
+	(void)rtmp_router_start_tmo((at_kern_err_t*)arg);
+}
 
-void rtmp_shutdown()
+void rtmp_shutdown(void)
 {
 	register at_ifaddr_t *ifID;
-	register short i;
 	at_net DestNet;	
 
 	NET_ASSIGN(DestNet, 0);
@@ -1586,9 +1605,9 @@ void rtmp_shutdown()
 	dPrintf(D_M_RTMP, D_L_SHUTDN,
 		("rtmp_shutdown:stop sending to all ports\n"));
 
-	untimeout(rtmp_dropper, (caddr_t)0);
-	untimeout(rtmp_router_start, 1); /* added for 2225395 */
-	untimeout(rtmp_router_start, 3); /* added for 2225395 */
+	untimeout(rtmp_dropper, (void *)0);
+	untimeout(rtmp_router_start_tmo, (void *)1); /* added for 2225395 */
+	untimeout(rtmp_router_start_tmo, (void *)3); /* added for 2225395 */
 	
 	TAILQ_FOREACH(ifID, &at_ifQueueHd, aa_link) {
 		if (ifID->ifRoutingState > PORT_OFFLINE ) {
diff --git a/bsd/netat/ddp_r_zip.c b/bsd/netat/ddp_r_zip.c
index 890c8a374..3c377d737 100644
--- a/bsd/netat/ddp_r_zip.c
+++ b/bsd/netat/ddp_r_zip.c
@@ -1,26 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1988-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *	Copyright (c) 1988-1998 Apple Computer, Inc. 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *   0.01 05/12/94	Laurent Dumont		Creation
@@ -59,20 +62,18 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/nbp.h>
 #include <netat/zip.h>
-#include <netat/at_pcb.h>
 #include <netat/atp.h>
 #include <netat/routing_tables.h>
+#include <netat/rtmp.h>
 #include <netat/debug.h>
 
 #include <sys/kern_event.h>
 
-static void zip_reply_to_getmyzone();
-extern int at_reg_mcast(), at_unreg_mcast();
-
 /* globals */
 extern at_ifaddr_t *ifID_table[], *ifID_home;
 extern short	ErrorZIPoverflow;
@@ -92,6 +93,17 @@ static	void	zip_getnetinfo(at_ifaddr_t *);
 static	void	zip_getnetinfo_locked(void *);
 static	void	send_phony_reply(void *);
 
+int zip_reply_received(gbuf_t *, at_ifaddr_t *, int);
+int zip_reply_to_getlocalzones(at_ifaddr_t *, gbuf_t *);
+int zip_reply_to_getzonelist(at_ifaddr_t *, gbuf_t *);
+static void zip_reply_to_getmyzone(at_ifaddr_t *, gbuf_t *);
+gbuf_t *zip_prep_query_packet(at_ifaddr_t *, at_net_al, at_node);
+
+static void zip_send_reply_to_query(gbuf_t *, at_ifaddr_t *);
+static void zip_send_ext_reply_to_query(gbuf_t *, at_ifaddr_t *, RT_entry *, u_short);
+static gbuf_t *prep_ZIP_reply_packet(gbuf_t *, at_ifaddr_t *);
+static void zip_send_getnetinfo_reply(gbuf_t *, at_ifaddr_t *);
+
 /*
  * zip_send_getnetinfo_reply: we received a GetNetInfo packet, we need to reply
  *		   with the right information for the port.
@@ -105,7 +117,7 @@ static void zip_send_getnetinfo_reply(m, ifID)
 	at_ddp_t	*ddp, *ddp_sent;
 	short ZoneNameProvided = FALSE;
 	short RequestIsBroadcasted = FALSE;
-	u_short znumber, len, packet_length, size, status;
+	u_short znumber, len, packet_length = 0, size, status;
 	RT_entry *Entry;
 	char GNIReply[128];
 
@@ -150,13 +162,13 @@ static void zip_send_getnetinfo_reply(m, ifID)
 		bcopy(&zname->str, &GNIReply[7], zname->len);
 
 
-		if (znumber = zt_find_zname(zname)) {
+		if ((znumber = zt_find_zname(zname))) {
 
 			if (ZT_ISIN_ZMAP((znumber), Entry->ZoneBitMap)) {
 
 			  GNIReply[1] = 0; /* Zone Valid */
 
-			  if (len = zt_get_zmcast(ifID, zname, &GNIReply[8+zname->len]))
+			  if ((len = zt_get_zmcast(ifID, zname, &GNIReply[8+zname->len])))
 				GNIReply[7+zname->len] = len;
 			  else {
 				GNIReply[1] |= ZIP_USE_BROADCAST;
@@ -190,7 +202,7 @@ static void zip_send_getnetinfo_reply(m, ifID)
 
 		Index--;
 
-		if (len = zt_get_zmcast(ifID, &ZT_table[Index].Zone, &GNIReply[8+zname->len]))
+		if ((len = zt_get_zmcast(ifID, &ZT_table[Index].Zone, &GNIReply[8+zname->len])))
 			GNIReply[7+zname->len] = len;
 		else {
 			GNIReply[1] |= ZIP_USE_BROADCAST;
@@ -363,7 +375,7 @@ newPacket:
 
 						zone_count -= *ZonesInPacket;
 
-						DDPLEN_ASSIGN(ddp, reply_length + DDP_X_HDR_SIZE);
+						DDPLEN_ASSIGN(ddp, (reply_length + DDP_X_HDR_SIZE));
 						gbuf_winc(m,reply_length);
 						if ((status = 
 						     ddp_router_output(m, ifID, AT_ADDR,
@@ -404,7 +416,7 @@ newPacket:
 	 */
 
 	if (zone_count) {
-			DDPLEN_ASSIGN(ddp, reply_length + DDP_X_HDR_SIZE);
+			DDPLEN_ASSIGN(ddp, (reply_length + DDP_X_HDR_SIZE));
 			gbuf_winc(m,reply_length);
 			if ((status = 
 			     ddp_router_output(m, ifID, AT_ADDR,
@@ -429,7 +441,7 @@ static void zip_send_reply_to_query(mreceived, ifID)
      register at_ifaddr_t	*ifID;
 {
 	register gbuf_t	*m;
-	register at_ddp_t	*ddp, *ddp_received;
+	register at_ddp_t	*ddp = NULL, *ddp_received;
 	RT_entry *Entry;
 	short i, reply_length, Index, status;
 	u_char	network_count;
@@ -494,7 +506,7 @@ newPacket:
 
 			/* we need to send the packet before, this won't fit... */
 		
-			DDPLEN_ASSIGN(ddp, reply_length + DDP_X_HDR_SIZE);
+			DDPLEN_ASSIGN(ddp, (reply_length + DDP_X_HDR_SIZE));
 			gbuf_winc(m,reply_length);
 
 			if ((status = 
@@ -552,7 +564,7 @@ newPacket:
 	 */
 
 	if ( reply_length > 2)  {
-		DDPLEN_ASSIGN(ddp, reply_length + DDP_X_HDR_SIZE);
+		DDPLEN_ASSIGN(ddp, (reply_length + DDP_X_HDR_SIZE));
 		gbuf_winc(m,reply_length);
 		if ((status = 
 		     ddp_router_output(m, ifID, AT_ADDR,
@@ -587,7 +599,6 @@ void zip_router_input (m, ifID)
 	register char	new_zone_len;
 	register char	*old_zone;
 	char		*new_zone;
-	void		zip_sched_getnetinfo(); /* forward reference */
 
 	if (gbuf_type(m) != MSG_DATA) {
 		/* If this is a M_ERROR message, DDP is shutting down, 
@@ -684,7 +695,6 @@ void zip_router_input (m, ifID)
 			    ((NET_VALUE(ddp->src_net) != ifID->ifThisNode.s_net) ||
 			     (ddp->src_node != ifID->ifThisNode.s_node)) && netinfo_reply_pending)
 			{
-				extern void trackrouter();
 				dPrintf(D_M_ZIP, D_L_INPUT,
 					("zip_input: Received a GetNetInfo Reply from %d.%d\n",
 					NET_VALUE(ddp->src_net), ddp->src_node));
@@ -806,7 +816,6 @@ int zonename_equal (zone1, zone2)
      register at_nvestr_t	*zone1, *zone2;
 {
 	register char c1, c2;
-	char	upshift8();
 	register int	i;
 
 	if (zone1->len != zone2->len)
@@ -861,7 +870,6 @@ static void zip_netinfo_reply (netinfo, ifID)
      register at_ifaddr_t	*ifID;
 {
 	u_char	mcast_len;
-	void	zip_sched_getnetinfo(); /* forward reference */
 	register at_net_al	this_net;
 	char	*default_zone;
 	register u_char	zone_name_len;
@@ -993,13 +1001,13 @@ int zip_control (ifID, control)
 }
 
 /* locked version of zip_getnetinfo */
-static void zip_getnetinfo_locked(arg)
-     void       *arg;
+static void
+zip_getnetinfo_locked(void *arg)
 {
 	at_ifaddr_t       *ifID;
 	
 	atalk_lock();
-	if (ifID != NULL) {		// make sure it hasn't been closed
+	if (arg != NULL) {		// make sure it hasn't been closed
 		ifID = (at_ifaddr_t *)arg;
 		ifID->ifGNIScheduled = 0;
 		zip_getnetinfo(ifID);
@@ -1018,7 +1026,6 @@ static void zip_getnetinfo (ifID)
 	register at_x_zip_t	*zip;
 	gbuf_t			*m;
 	register at_ddp_t	*ddp;
-	void			zip_sched_getnetinfo();
 	register struct	atalk_addr	*at_dest;
 	register int		size;
 	
@@ -1058,7 +1065,7 @@ static void zip_getnetinfo (ifID)
 	/* let the lap fields be uninitialized, 'cause it doesn't 
 	 * matter.
 	 */
-	DDPLEN_ASSIGN(ddp, size - (sizeof(struct atalk_addr) + 1));
+	DDPLEN_ASSIGN(ddp, (size - (sizeof(struct atalk_addr) + 1)));
 	UAS_ASSIGN(ddp->checksum, 0);
 	ddp->hopcount = ddp->unused = 0;
 	NET_ASSIGN(ddp->dst_net, 0); /* cable-wide broadcast */
@@ -1096,9 +1103,9 @@ static void zip_getnetinfo (ifID)
  *
  **********************************************************************/
 
-void	zip_sched_getnetinfo (ifID)
-     register at_ifaddr_t	     *ifID;
+void	zip_sched_getnetinfo(void *arg)
 {
+     register at_ifaddr_t *ifID = (at_ifaddr_t *)arg;
 
 	atalk_lock();
 	
@@ -1383,7 +1390,7 @@ newPacket:
 			zip_sent = TRUE;
 
 			gbuf_winc(m,DDP_X_HDR_SIZE + Query_index);
-			DDPLEN_ASSIGN(ddp, DDP_X_HDR_SIZE + Query_index);
+			DDPLEN_ASSIGN(ddp, (DDP_X_HDR_SIZE + Query_index));
 
 			if ((status = 
 			     ddp_router_output(m, ifID, AT_ADDR,
@@ -1431,7 +1438,7 @@ newPacket:
 	if (*ZoneCount) { 	/* non-full Query needs to be sent */
 		zip_sent = TRUE;
 		gbuf_winc(m,DDP_X_HDR_SIZE + Query_index);
-		DDPLEN_ASSIGN(ddp, DDP_X_HDR_SIZE + Query_index);
+		DDPLEN_ASSIGN(ddp, (DDP_X_HDR_SIZE + Query_index));
 
 		if ((status = 
 		     ddp_router_output(m, ifID, AT_ADDR,
@@ -1454,7 +1461,7 @@ newPacket:
  *		we receive two types of replies: non extended and extended.
  *	    For extended replies, the network count is the Total of zones for that net.
  */
-
+int
 zip_reply_received(m, ifID, reply_type)
      register gbuf_t	*m;
      register at_ifaddr_t	*ifID;
@@ -1539,19 +1546,25 @@ zip_reply_received(m, ifID, reply_type)
 	}
 
 	if ((reply_type == ZIP_REPLY) && network_count > 0) {
+#if DEBUG
 		if (Entry)
 			dPrintf(D_M_ZIP, D_L_WARNING,
 			("zip_reply_received: Problem decoding zone (after net:%d-%d)\n",
 			Entry->NetStart, Entry->NetStop));
+#endif
 		ifID->ifZipNeedQueries = 1;
 	}
 	else {
 		ifID->ifZipNeedQueries = 0;
+#if DEBUG
 		if (Entry)
 			dPrintf(D_M_ZIP_LOW, D_L_INFO,
 			("zip_reply_received: entry %d-%d all zones known\n",
 			Entry->NetStart, Entry->NetStop));
+#endif
 	}
+	
+	return 0;
 }
 
 /*
@@ -1653,6 +1666,7 @@ static void zip_reply_to_getmyzone (ifID, m)
  * zip_reply_to_getzonelist: replies to ZIP GetZoneList requested from the Net
  */
 
+int
 zip_reply_to_getzonelist (ifID, m)
      register at_ifaddr_t   *ifID;
      register gbuf_t      *m;
@@ -1765,8 +1779,8 @@ zip_reply_to_getzonelist (ifID, m)
 			NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, ifID->ifPort, PacketLen));
 
 
-		if (status= ddp_router_output(rm, ifID, AT_ADDR,
-				 NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, 0)) {
+		if ((status= ddp_router_output(rm, ifID, AT_ADDR,
+				 NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, 0))) {
 			dPrintf(D_M_ZIP, D_L_ERROR, ("zip_reply_to_GZL: ddp_router_output returns=%d\n",
 				 status));
 			return (status);
@@ -1792,7 +1806,7 @@ int zip_reply_to_getlocalzones (ifID, m)
 		short	Index, Index_wanted, ZLength;
 		short i,j, packet_len;
 		short  zCount, ZoneCount, ZonesInPacket;
-		char *zmap, last_flag = 0;
+		unsigned char *zmap, last_flag = 0;
 		RT_entry *Entry;
 		char *Reply;
 
@@ -1962,8 +1976,8 @@ FullPacket:
 		("zip_r_GLZ: send packet to %d:%d port %d atp_len =%d\n",
 		 NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, ifID->ifPort, packet_len));
 
-	if (status= ddp_router_output(rm, ifID, AT_ADDR,
-				      NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, 0)) {
+	if ((status= ddp_router_output(rm, ifID, AT_ADDR,
+				      NET_VALUE(r_ddp->dst_net), r_ddp->dst_node, 0))) {
 		dPrintf(D_M_ZIP, D_L_ERROR, 
 			("zip_reply_to_GLZ: ddp_router_output returns =%d\n",
 			 status));
@@ -1975,7 +1989,6 @@ FullPacket:
 int regDefaultZone(ifID)
      at_ifaddr_t *ifID;
 {
-	int i;
 	char data[ETHERNET_ADDR_LEN];
 
 	if (!ifID)
@@ -1983,7 +1996,7 @@ int regDefaultZone(ifID)
 
 	zt_get_zmcast(ifID, &ifID->ifZoneName, data); 
 	if (FDDI_OR_TOKENRING(ifID->aa_ifp->if_type))
-		ddp_bit_reverse(data);
+		ddp_bit_reverse((unsigned char *)data);
 	bcopy((caddr_t)data, (caddr_t)&ifID->ZoneMcastAddr, ETHERNET_ADDR_LEN);
 	(void)at_reg_mcast(ifID, (caddr_t)&ifID->ZoneMcastAddr);
 	return(0);
diff --git a/bsd/netat/ddp_rtmp.c b/bsd/netat/ddp_rtmp.c
index 023418dfc..cbbc254d9 100644
--- a/bsd/netat/ddp_rtmp.c
+++ b/bsd/netat/ddp_rtmp.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1993-1998 Apple Computer, Inc.
@@ -47,17 +53,16 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/rtmp.h>
 #include <netat/zip.h>
 #include <netat/routing_tables.h>
 #include <netat/debug.h>
-#include <netat/at_pcb.h>
 
 #include <sys/kern_event.h>
 
-extern void rtmp_router_input();
 
 /****************************************************************/
 /*								*/
@@ -77,7 +82,7 @@ extern void rtmp_router_input();
 #define NET(r)		((r)->ifARouter.s_net)
 #define	INUSE(r)	(NODE(r))
 
-void ddp_age_router();
+void ddp_age_router(void *arg);
 
 static struct routerinfo {
 	struct at_addr ifARouter;
@@ -88,7 +93,7 @@ static struct routerinfo {
 void trackrouter_rem_if(ifID)
      register at_ifaddr_t *ifID;
 {
-	register i;
+	int i;
 	register struct routerinfo *router;
 
 	for (i = NROUTERS2TRAK; --i >= 0;) {
@@ -101,9 +106,9 @@ void trackrouter_rem_if(ifID)
 }
 
 
-void routershutdown()
+void routershutdown(void)
 {
-	register i;
+	int i;
 
 	for (i = NROUTERS2TRAK; --i >= 0;) {
 		register struct routerinfo *router;
@@ -127,7 +132,7 @@ void trackrouter(ifID, net, node)
      register unsigned char	node;
 {
 	register struct routerinfo *unused = NULL;
-	register i;
+	int i;
 
 	for (i = NROUTERS2TRAK; --i >= 0;) {
 		register struct routerinfo *router;
@@ -171,9 +176,9 @@ void trackrouter(ifID, net, node)
  * frames to something that is not there. Untimeout is called if 
  * an RTMP packet comes in so this routine will not be called.
  */
-void ddp_age_router(deadrouter)
-     register struct routerinfo *deadrouter;
+void ddp_age_router(void *arg)
 {
+	struct routerinfo *deadrouter = (struct routerinfo*)arg;
 	register at_ifaddr_t *ourrouter;
 
 	atalk_lock();
@@ -192,8 +197,8 @@ void ddp_age_router(deadrouter)
 	if (NODE(ourrouter) == NODE(deadrouter) && 
 	    NET(ourrouter) == NET(deadrouter)) {
 		register unsigned long	atrandom = random();
-		register struct routerinfo *newrouter;
-		register i;
+		register struct routerinfo *newrouter = NULL;
+		int i;
 
 		bzero((caddr_t) deadrouter, sizeof(struct routerinfo));
 		for (i = NROUTERS2TRAK; --i >= 0;) {
diff --git a/bsd/netat/ddp_rtmptable.c b/bsd/netat/ddp_rtmptable.c
index d6a683564..95acab081 100644
--- a/bsd/netat/ddp_rtmptable.c
+++ b/bsd/netat/ddp_rtmptable.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1994-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*----------------------------------------------------------------------------
  *
@@ -42,7 +48,6 @@
  *
  *----------------------------------------------------------------------------
  *
- *      Copyright (c) 1994, 1996, 1997, 1998 Apple Computer, Inc.
  */
 
 #include <sys/errno.h>
@@ -65,10 +70,10 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/ddp.h>
 #include <netat/rtmp.h>
-#include <netat/at_pcb.h>
 #include <netat/zip.h>
 #include <netat/routing_tables.h>
 #include <netat/at_snmp.h>
@@ -90,16 +95,12 @@ extern snmpStats_t	snmpStats;
 short ErrorRTMPoverflow = 0;	/* flag if RTMP table is too small for this net */
 short ErrorZIPoverflow  = 0;	/* flag if ZIP table is too small for this net */
 
-	/* prototypes */
-void getIfUsage( int, at_ifnames_t *);
 
 /*
  * This a temporary function : just to display the router error 
  */
 
-void RouterError(port, err_number)
-short port, err_number;
-
+void RouterError(__unused short port, short err_number)
 {
 	switch (err_number) {
 
@@ -209,7 +210,9 @@ RT_entry *NewEntry;
 {
 	RT_entry *ptree = &RT_table_start;
 
+#if DEBUG
 	register at_net_al NetStart = NewEntry->NetStart;
+#endif
 	register at_net_al NetStop  = NewEntry->NetStop;
 
 	dPrintf(D_M_RTMP_LOW, D_L_ROUTING, ("rt_binsert: for Net %d-%d state=x%x NextIR %d:%d\n",
@@ -280,11 +283,10 @@ RT_entry *rt_insert(NStop, NStart, NxNet, NxNode, NtDist, NtPort, EntS)
  *
  */
 
-RT_entry *rt_bdelete (NetStop, NetStart)
-     at_net_al NetStop, NetStart;
+RT_entry *rt_bdelete (at_net_al NetStop, __unused at_net_al NetStart)
 {
 
-	RT_entry *rt_found, *pprevious, *pnext, *pnextl, *psub;
+	RT_entry *rt_found, *pprevious = NULL, *pnext, *pnextl, *psub;
 	at_net_al LowEnd;
 
 	rt_found = &RT_table_start;
@@ -385,8 +387,9 @@ RT_entry *rt_bdelete (NetStop, NetStart)
 }
 					
 
-RT_entry *rt_sortedshow(parent)
-RT_entry *parent;
+#if DEBUG
+RT_entry *rt_sortedshow(RT_entry *parent);
+RT_entry *rt_sortedshow(RT_entry *parent)
 {
 	RT_entry *me;
 	
@@ -408,7 +411,8 @@ RT_entry *parent;
  * debug only: display the contents of the routing table
  */
 
-void rt_show ()
+void rt_show(void);
+void rt_show(void)
 {
 	RT_entry *ptree;
 	int i=0;
@@ -429,11 +433,13 @@ void rt_show ()
 	i++;
 	}
 }
+#endif /* DEBUG */
 
 /*
  * prepare the indexing of the free entries in the RTMP table
  */
 
+int
 rt_table_init()
 {
 	short i;
@@ -473,6 +479,7 @@ rt_table_init()
  * zt_add_zone: add a zone name in the zone table.
  */
 
+int
 zt_add_zone(name, length)
 char *name;
 short length;
@@ -492,7 +499,7 @@ at_nvestr_t *zname;
 {
 	register short res,i;
 
-	if (res = zt_find_zname(zname))
+	if ((res = zt_find_zname(zname)))
 		return(res);
 
 	for (i = 0; i < ZT_maxentry ; i++) {
@@ -551,8 +558,9 @@ u_char *zmap;
 /*
  * zt_compute_hash: compute hash index from the zone name string
  */
+static short zt_compute_hash(at_nvestr_t *);
 
-short zt_compute_hash(zname)
+static short zt_compute_hash(zname)
 at_nvestr_t *zname;
 {
 	register u_short checksum=0, i;
@@ -591,6 +599,7 @@ at_nvestr_t *zname;
  * zt_upper_zname: translate the name string into uppercase
  */
 
+#if 0
 void zt_upper_zname(zname)
 at_nvestr_t *zname;
 {
@@ -608,6 +617,7 @@ at_nvestr_t *zname;
 			zname->str[i] = c1;
 	}
 }
+#endif
 
 /*
  * zt_get_zmcast: calcularte the zone multicast address for a
@@ -615,6 +625,7 @@ at_nvestr_t *zname;
  *                    Returns the result in "buffer"
  */
 
+int
 zt_get_zmcast(ifID, zname, buffer)
      at_ifaddr_t *ifID;		/* we want to know the media type */
      at_nvestr_t *zname;	/* source name for multicast address */
@@ -691,7 +702,7 @@ u_char *zmap;
 /*
  * zt_ent_zcount: count the number of actives zone for a routing entry
  */
-
+int
 zt_ent_zcount(ent)
 RT_entry *ent;
 {
@@ -720,6 +731,7 @@ RT_entry *ent;
 /*
  * zt_find_zname: match a zone name in the zone table and return the entry if found
  */
+int
 zt_find_zname(zname)
 at_nvestr_t *zname;
 {
@@ -766,7 +778,7 @@ at_nvestr_t *zname;
  */
 void zt_set_zmap(znum, zmap)
      u_short znum;
-     char *zmap;
+     unsigned char *zmap;
 {
 	register u_short num = znum -1;
 
@@ -780,6 +792,7 @@ void zt_set_zmap(znum, zmap)
 /*
  * zt_clr_zmap: clear a bit for the corresponding zone map in an entry bitmap
  */
+#if 0
 void zt_clr_zmap(znum, zmap)
      u_short znum;
      char *zmap;
@@ -791,7 +804,7 @@ void zt_clr_zmap(znum, zmap)
 		ZT_table[num].ZoneCount--;
 	}
 }
-
+#endif
 
 /*
  * routing_needed : 
@@ -976,7 +989,7 @@ RT_entry *rt_getNextRoute(first)
 		return(NULL);
 }
 
-
+int
 getRtmpTableSize()
 {
 	register int i;
@@ -994,7 +1007,8 @@ getRtmpTableSize()
 	return(0);
 }
 
-getZipTableSize()
+int
+getZipTableSize(void)
 {
 	register int i;
 	register ZT_entry *zt;
@@ -1011,6 +1025,7 @@ getZipTableSize()
 	return(0);
 }
 
+void
 getRtmpTable(d,s,c)
      RT_entry	*d;		/* destination */
      int 	s;		/* starting entry */
@@ -1026,6 +1041,7 @@ getRtmpTable(d,s,c)
 		}
 }
 
+void
 getZipTable(d,s,c)
      ZT_entry	*d;		/* destination */
      int 	s;		/* starting entry */
@@ -1038,7 +1054,7 @@ getZipTable(d,s,c)
 at_nvestr_t *getRTRLocalZone(ifz)
      zone_usage_t *ifz;
 {
-	char *zmap;
+	unsigned char *zmap = NULL;
 	RT_entry	*route;
 	int i, j, index;
 	int  zcnt=0;		/* zone we're pointing to in the list */
diff --git a/bsd/netat/ddp_sip.c b/bsd/netat/ddp_sip.c
index b4cef944b..cc4b47495 100644
--- a/bsd/netat/ddp_sip.c
+++ b/bsd/netat/ddp_sip.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -66,12 +72,12 @@
 
 #include <net/if.h>
 
+#include <netat/sysglue.h>  /* nbp.h needs the gbuf definiton */
 #include <netat/appletalk.h>
 #include <netat/ddp.h>
-#include <netat/sysglue.h>  /* nbp.h needs the gbuf definiton */
-#include <netat/nbp.h>
 #include <netat/at_pcb.h>
 #include <netat/at_var.h>
+#include <netat/nbp.h>
 #include <netat/atp.h>
 
 #define	SIP_SYSINFO_CMD		1
@@ -89,9 +95,9 @@ typedef	struct {
 	u_short	responder_version;
 } sip_userbytes_t;
 
-void sip_input(mp, ifID)
-     gbuf_t	*mp;
-     int	*ifID; /* not used */
+void sip_input(
+     gbuf_t	*mp,
+     __unused at_ifaddr_t	*ifID)
 {
 	/* Packets arriving here are actually ATP packets, but since
 	 * A/UX only send dummy responses, we're implementing responder as
@@ -178,4 +184,3 @@ void sip_input(mp, ifID)
 	(void)ddp_output(&mp, DDP_SOCKET_1st_DYNAMIC, FALSE);
 	return;
 } /* sip_input */
-
diff --git a/bsd/netat/ddp_usrreq.c b/bsd/netat/ddp_usrreq.c
index 49f9ae768..fd6b8ab92 100644
--- a/bsd/netat/ddp_usrreq.c
+++ b/bsd/netat/ddp_usrreq.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1998 Apple Computer, Inc. 
@@ -44,24 +50,17 @@
 
 #include <net/if.h>
 
+#include <netat/sysglue.h>
 #include <netat/appletalk.h>
+#include <netat/at_pcb.h>
 #include <netat/at_var.h>
-#include <netat/sysglue.h>
 #include <netat/ddp.h>
 #include <netat/ep.h>
 #include <netat/rtmp.h>
 #include <netat/zip.h>
-#include <netat/at_pcb.h>
 #include <netat/routing_tables.h>
 #include <netat/nbp.h>
 
-extern int at_control(), at_memzone_init();
-extern void  nbp_input(), ep_input(), zip_router_input(), 
-  sip_input(), add_ddp_handler(), init_ddp_handler(), 
-  ddp_start(), ddp_input(), appletalk_hack_start();
-extern u_short ddp_checksum();
-extern at_ifaddr_t *forUs();
-extern struct mbuf *m_dup(struct mbuf *, int);
 
 extern at_ifaddr_t *ifID_home;
 extern int xpatcnt;
@@ -71,17 +70,16 @@ u_long ddp_sendspace = 600,	/* *** what should this value be? *** */
   ddp_recvspace = 50 * (600 + sizeof(struct sockaddr_at));
 
 int ddp_pru_control(struct socket *so, u_long cmd, caddr_t data,
-		struct ifnet *ifp, struct proc *p)
+		struct ifnet *ifp, __unused struct proc *p)
 {
 	return(at_control(so, cmd, data, ifp));
 }
 
 
 int	ddp_pru_attach(struct socket *so, int proto,
-		   struct proc *p)
+		   __unused struct proc *p)
 {
 	int error = 0;
-	at_ddp_t *ddp = NULL;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
         error = soreserve(so, ddp_sendspace, ddp_recvspace);
@@ -104,7 +102,6 @@ int  ddp_pru_disconnect(struct socket *so)
 {
 
 	int error = 0;
-	at_ddp_t *ddp = NULL;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
 	if (pcb == NULL) 
@@ -157,7 +154,7 @@ int	ddp_pru_shutdown(struct socket *so)
 					  
 
 int    ddp_pru_bind(struct socket *so, struct sockaddr *nam,
-		struct proc *p)
+		__unused struct proc *p)
 {
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 
@@ -168,9 +165,9 @@ int    ddp_pru_bind(struct socket *so, struct sockaddr *nam,
 }
 
 
-int	ddp_pru_send(struct socket *so, int flags, struct mbuf *m, 
-		     struct sockaddr *addr, struct mbuf *control,
-		     struct proc *p)
+int	ddp_pru_send(struct socket *so, __unused int flags, struct mbuf *m, 
+		     struct sockaddr *addr, __unused struct mbuf *control,
+		     __unused struct proc *p)
 {
 	at_ddp_t *ddp = NULL;
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
@@ -303,8 +300,8 @@ int  ddp_pru_peeraddr(struct socket *so,
 }
 
 
-int    ddp_pru_connect(struct socket *so, struct sockaddr *nam,
-		   struct proc *p)
+int ddp_pru_connect(struct socket *so, struct sockaddr *nam,
+		   __unused struct proc *p)
 {
 	struct atpcb *pcb = (struct atpcb *)((so)->so_pcb);
 	struct sockaddr_at *faddr = (struct sockaddr_at *) nam;
@@ -324,7 +321,6 @@ int    ddp_pru_connect(struct socket *so, struct sockaddr *nam,
 }
 
 
-
 /*
  * One-time AppleTalk initialization
  */
@@ -344,4 +340,3 @@ void ddp_init()
 
 	appletalk_hack_start();
 } /* ddp_init */
-
diff --git a/bsd/netat/debug.h b/bsd/netat/debug.h
index d59f52235..7228eb26d 100644
--- a/bsd/netat/debug.h
+++ b/bsd/netat/debug.h
@@ -1,26 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1988-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *      Copyright (c) 1988, 1989, 1997 Apple Computer, Inc.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* netat/debug.h */
@@ -99,7 +102,7 @@ extern dbgBits_t 	dbgBits;
 #ifdef DEBUG
 #define dPrintf(mod, lev, p) \
 	if (((mod) & dbgBits.dbgMod) && ((lev) & dbgBits.dbgLev)) {\
-		 kprintf p;  \
+		 printf p;  \
 	}
 #else
 #define dPrintf(mod, lev, p)
@@ -147,13 +150,13 @@ extern dbgBits_t 	dbgBits;
 
 #define trace_mbufs(pri, str, start)\
 { if (start)\
-{   int i; gbuf_t *tmp;\
-    for (tmp=start, i=0; tmp && i < 10; tmp = gbuf_cont(tmp), i++) {\
-	dPrintf(pri, D_L_TRACE, ("%s=0x%x, len=%d %s\n",\
-                                 str, tmp, gbuf_len(tmp),\
-                                 (((struct mbuf *)tmp)->m_flags & M_EXT)?"CL":""));\
-	KERNEL_DEBUG(DBG_ADSP_MBUF,  0, tmp, gbuf_len(tmp), gbuf_next(tmp), \
-		((struct mbuf *)tmp)->m_flags & M_EXT);\
+{   int _i; gbuf_t *_tmp;\
+    for (_tmp=start, _i=0; _tmp && _i < 10; _tmp = gbuf_cont(_tmp), _i++) {\
+	dPrintf(pri, D_L_TRACE, ("%s=0x%p, len=%ld %s\n",\
+                                 str, _tmp, gbuf_len(_tmp),\
+                                 (((struct mbuf *)_tmp)->m_flags & M_EXT)?"CL":""));\
+	KERNEL_DEBUG(DBG_ADSP_MBUF,  0, _tmp, gbuf_len(_tmp), gbuf_next(_tmp), \
+		((struct mbuf *)_tmp)->m_flags & M_EXT);\
 }}}
 
 /* from h/atlog.h */
diff --git a/bsd/netat/drv_dep.c b/bsd/netat/drv_dep.c
index 61ed827eb..01ba010ca 100644
--- a/bsd/netat/drv_dep.c
+++ b/bsd/netat/drv_dep.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1994 Apple Computer, Inc.
@@ -46,7 +52,9 @@
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/if_dl.h>
+#include <net/dlil.h>
 #include <net/ethernet.h>
+#include <net/kpi_protocol.h>
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
@@ -55,12 +63,11 @@
 #include <netat/ddp.h>
 #include <netat/at_aarp.h>
 #include <netat/at_pat.h>
+#include <netat/atp.h>
 #include <netat/debug.h>
 
 #define DSAP_SNAP 0xaa
 
-extern void gref_init(), atp_init(), atp_link(), atp_unlink();
-
 extern int adspInited;
 
 static llc_header_t	snap_hdr_at = SNAP_HDR_AT;
@@ -70,19 +77,20 @@ static unsigned char snap_proto_aarp[5] = SNAP_PROTO_AARP;
 
 static void at_input_packet(protocol_family_t protocol, mbuf_t m);
 
-int pktsIn, pktsOut;
-
 struct ifqueue atalkintrq; 	/* appletalk and aarp packet input queue */
 
 short appletalk_inited = 0;
 
+void atalk_load(void);
+void atalk_unload(void);
 
+extern lck_mtx_t *domain_proto_mtx;
 
-void atalk_load()
-{
-	extern lck_mtx_t *domain_proto_mtx;
+extern int pktsIn, pktsOut;
 
 
+void atalk_load()
+{
 	atp_init();
 	atp_link();
 	adspInited = 0;
@@ -92,36 +100,37 @@ void atalk_load()
 		this happens in adsp_open and is undone on ADSP_UNLINK 
 */
 	lck_mtx_unlock(domain_proto_mtx);
-	proto_register_input(PF_APPLETALK, at_input_packet, NULL);
+	proto_register_input(PF_APPLETALK, at_input_packet, NULL, 0);
 	lck_mtx_lock(domain_proto_mtx);
 } /* atalk_load */
 
 /* Undo everything atalk_load() did. */
 void atalk_unload()  /* not currently used */
 {
-	extern gbuf_t *scb_resource_m;
-	extern gbuf_t *atp_resource_m;
-
 	atp_unlink();
 
 #ifdef NOT_YET
-	if (scb_resource_m) { 
-		gbuf_freem(scb_resource_m);
-		scb_resource_m = 0;
-		scb_free_list = 0;
-	}
-	/* allocated in atp_trans_alloc() */
-	if (atp_resource_m) {
-		gbuf_freem(atp_resource_m);
-		atp_resource_m = 0;
-		atp_trans_free_list = 0;
+	{
+		extern gbuf_t *scb_resource_m;
+		extern gbuf_t *atp_resource_m;
+		if (scb_resource_m) { 
+			gbuf_freem(scb_resource_m);
+			scb_resource_m = 0;
+			scb_free_list = 0;
+		}
+		/* allocated in atp_trans_alloc() */
+		if (atp_resource_m) {
+			gbuf_freem(atp_resource_m);
+			atp_resource_m = 0;
+			atp_trans_free_list = 0;
+		}
 	}
 #endif
 
 	appletalk_inited = 0;
 } /* atalk_unload */
 
-void appletalk_hack_start()
+void appletalk_hack_start(void)
 {
 	if (!appletalk_inited) {
 		atalk_load();
@@ -293,7 +302,7 @@ at_input_packet(
 				ifID->stats.rcv_bytes += m1->m_len;
 
 			if (!MULTIPORT_MODE)
-				ddp_glean(m, ifID, src);
+				ddp_glean(m, ifID, (struct etalk_addr *)src);
 
 			ddp_input(m, ifID);
 		} else {
diff --git a/bsd/netat/ep.h b/bsd/netat/ep.h
index fd917a57a..5ab512f3c 100644
--- a/bsd/netat/ep.h
+++ b/bsd/netat/ep.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * ORIGINS: 82
diff --git a/bsd/netat/lap.h b/bsd/netat/lap.h
index 85542c5d9..bcedbfbcc 100644
--- a/bsd/netat/lap.h
+++ b/bsd/netat/lap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1988, 1989 Apple Computer, Inc. 
diff --git a/bsd/netat/nbp.h b/bsd/netat/nbp.h
index 648d373a1..39c49daef 100644
--- a/bsd/netat/nbp.h
+++ b/bsd/netat/nbp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -155,6 +161,16 @@ extern int	nbp_fillin_nve(at_entity_t *, nve_entry_t *);
 extern at_nvestr_t *getSPLocalZone(int);
 extern at_nvestr_t *getLocalZone(int);
 
+struct at_ifaddr;
+void	nbp_add_multicast( at_nvestr_t *, struct at_ifaddr *);
+void nbp_shutdown(void );
+
+int nbp_mh_reg(at_nbp_reg_t *);
+int nbp_new_nve_entry(nve_entry_t *, struct at_ifaddr *);
+void nbp_delete_entry(nve_entry_t *);
+
+
+
 #endif /* KERNEL_PRIVATE */
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_NBP_H_ */
diff --git a/bsd/netat/pap.h b/bsd/netat/pap.h
index 51388c274..a9ce98aff 100644
--- a/bsd/netat/pap.h
+++ b/bsd/netat/pap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
diff --git a/bsd/netat/routing_tables.h b/bsd/netat/routing_tables.h
index f3b46283d..9d2985c07 100644
--- a/bsd/netat/routing_tables.h
+++ b/bsd/netat/routing_tables.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *    This include file defines the RTMP table and ZIP table
@@ -202,15 +208,28 @@ extern ZT_entry *ZT_table;
 extern short	RT_maxentry;
 extern short	ZT_maxentry;
 
-extern volatile int RouterMix;
+extern int RouterMix;
 
 extern int zt_add_zone(char *, short);
 extern int zt_add_zonename(at_nvestr_t *);
 extern int zt_ent_zindex(u_char *);
 extern ZT_entryno *zt_getNextZone(int); 
 extern void zt_remove_zones(u_char *);
-extern void zt_set_zmap(u_short, char *);
+extern void zt_set_zmap(u_short, unsigned char *);
 extern void rtmp_router_input(gbuf_t *, at_ifaddr_t *);
+void trackrouter(at_ifaddr_t *, unsigned short, unsigned char);
+int zt_find_zname(at_nvestr_t *);
+struct at_nvestr *getRTRLocalZone(struct zone_usage *);
+int zt_ent_zcount(RT_entry *);
+int zt_get_zmcast(at_ifaddr_t *, at_nvestr_t *, char *);
+
+void getRtmpTable(RT_entry *, int, int	c);
+void getZipTable(ZT_entry *, int, int	c);
+int getZipTableSize(void);
+
+int rt_table_init(void );
+void getIfUsage( int, at_ifnames_t *);
+
 
 #endif /* KERNEL_PRIVATE */
 
diff --git a/bsd/netat/rtmp.h b/bsd/netat/rtmp.h
index 31b1a1b68..1404657da 100644
--- a/bsd/netat/rtmp.h
+++ b/bsd/netat/rtmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -64,5 +70,22 @@ typedef struct {
 	unsigned char	at_rtmp_data;
 } at_rtmp_tuple;
 
+#ifdef KERNEL_PRIVATE
+
+void rtmp_purge(at_ifaddr_t *);
+void rtmp_shutdown(void);
+void rtmp_input (gbuf_t *, at_ifaddr_t *);
+void RouterError(short, short);
+void rtmp_init(void);
+int zip_control (at_ifaddr_t *, int);
+void routershutdown(void);
+void trackrouter_rem_if(at_ifaddr_t *);
+char	upshift8(char);
+void ZIPwakeup(at_ifaddr_t *, int);
+
+int elap_dataput(gbuf_t *, at_ifaddr_t *, u_char, char *);
+
+#endif /* KERNEL_PRIVATE */
+
 #endif /* __APPLE_API_OBSOLETE */
 #endif /* _NETAT_RTMP_H_ */
diff --git a/bsd/netat/sys_dep.c b/bsd/netat/sys_dep.c
index 3ea33dbd9..47a18d89c 100644
--- a/bsd/netat/sys_dep.c
+++ b/bsd/netat/sys_dep.c
@@ -1,27 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- *	Copyright (c) 1995-1998 Apple Computer, Inc. 
- *
  *  Change Log:
  *    Created February 20, 1995 by Tuyen Nguyen
  *    Modified for MP, 1996 by Tuyen Nguyen
@@ -47,12 +51,12 @@
 
 #include <netat/sysglue.h>
 #include <netat/appletalk.h>
-#include <netat/at_var.h>
 #include <netat/at_pcb.h>
+#include <netat/at_var.h>
 #include <netat/debug.h>
 
+int falloc_locked(proc_t, struct fileproc **, int *, vfs_context_t, int);
 
-extern at_state_t at_state;	/* global state of AT network */
 extern at_ifaddr_t *ifID_home;	/* default interface */
 extern lck_mtx_t * atalk_mutex;
 
@@ -64,11 +68,13 @@ extern lck_mtx_t * atalk_mutex;
 #define f_offset f_fglob->fg_offset
 #define f_data f_fglob->fg_data
 
-extern int _ATsocket(int, int *, void *);
-extern int _ATgetmsg(int, strbuf_t *, strbuf_t *, int *, int *, void *);
-extern int _ATputmsg();
-extern int _ATPsndreq(), _ATPsndrsp(), _ATPgetreq(), _ATPgetrsp();
 
+int _ATkqfilter(struct fileproc *, struct knote *, vfs_context_t);
+int _ATselect(struct fileproc *, int, void *, vfs_context_t);
+int _ATioctl(struct fileproc *, u_long, caddr_t, vfs_context_t);
+int _ATwrite(struct fileproc *, struct uio *, int, vfs_context_t);
+int _ATread(struct fileproc *, struct uio *, int, vfs_context_t);
+int _ATclose(struct fileglob *, vfs_context_t);
 
 int ATsocket(proc, uap, retval)
 	struct proc *proc;
@@ -77,7 +83,7 @@ int ATsocket(proc, uap, retval)
 {
 	int err;
 	atalk_lock();
-	if (_ATsocket) {
+	if (1 /* _ATsocket*/) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -101,7 +107,7 @@ int ATgetmsg(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (_ATgetmsg) {
+	if (1 /* _ATgetmsg */) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -127,7 +133,7 @@ int ATputmsg(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (_ATputmsg) {
+	if (1 /* _ATputmsg */) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -153,7 +159,7 @@ int ATPsndreq(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (_ATPsndreq) {
+	if (1 /* _ATPsndreq */) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -179,7 +185,7 @@ int ATPsndrsp(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (_ATPsndrsp) {
+	if (1 /*_ATPsndrsp*/) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -205,7 +211,7 @@ int ATPgetreq(proc, uap, retval)
 	int err;
 
 	atalk_lock();
-	if (_ATPgetreq) {
+	if (1 /* _ATPgetreq */) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
@@ -231,14 +237,14 @@ int ATPgetrsp(proc, uap, retval)
 	int err = 0;
 
 	atalk_lock();
-	if (_ATPgetrsp) {
+	if (1 /*_ATPgetrsp*/) {
 		/* required check for all AppleTalk system calls */
 		if (!(at_state.flags & AT_ST_STARTED) || !ifID_home) {
 			*retval = -1;
 			err = ENOTREADY;
 		} else {
 			*retval = 
-			  _ATPgetrsp(uap->fd, uap->bdsp, &err, proc);
+			  _ATPgetrsp(uap->fd, (struct atpBDS *)uap->bdsp, &err, proc);
 		}
 	} else {
 		*retval = -1;
@@ -264,7 +270,6 @@ int atalk_openref(gref, retfd, proc)
 	int *retfd;
 	struct proc *proc;
 {
-	extern int _ATread(), _ATwrite(),_ATioctl(), _ATselect(), _ATclose(), _ATkqfilter();
 	static struct fileops fileops = 
 		{_ATread, _ATwrite, _ATioctl, _ATselect, _ATclose, _ATkqfilter, 0};
 	int err, fd;
@@ -273,7 +278,7 @@ int atalk_openref(gref, retfd, proc)
 	lck_mtx_assert(atalk_mutex, LCK_MTX_ASSERT_OWNED);
 	
 	proc_fdlock(proc);
-	if ((err = falloc_locked(proc, &fp, &fd, 1)) != 0) {
+	if ((err = falloc_locked(proc, &fp, &fd, vfs_context_current(), 1)) != 0) {
 		proc_fdunlock(proc);
 		return err;
 	}
@@ -339,7 +344,7 @@ int droponerr;
 	if (fp->f_type != (DTYPE_ATALK+1) || *grefp == 0 || *grefp == (gref_t *)(-1)) {
 		if (droponerr)
 			fp_drop(proc, fd, fp, 1);
-		printf("atalk_getref_locked EBADF f_data: %x\n", fp->f_data);
+		printf("atalk_getref_locked EBADF f_data: %p\n", fp->f_data);
 		return EBADF;
 	}
 	
diff --git a/bsd/netat/sys_glue.c b/bsd/netat/sys_glue.c
index 6c6793e76..a1d2a402c 100644
--- a/bsd/netat/sys_glue.c
+++ b/bsd/netat/sys_glue.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (c) 1995 Apple Computer, Inc. 
@@ -44,6 +50,8 @@
 #include <sys/socketvar.h>
 #include <sys/ioccom.h>
 #include <sys/uio_internal.h>
+#include <sys/file.h>
+#include <sys/vnode.h>
 
 #include <sys/sysctl.h>
 
@@ -55,26 +63,29 @@
 #include <netat/at_pcb.h>
 #include <netat/at_var.h>
 #include <netat/routing_tables.h>
+#include <netat/adsp.h>
+#include <netat/adsp_internal.h>
+#include <netat/asp.h>
 #include <netat/atp.h>
 #include <netat/debug.h>
 
+int _ATkqfilter(struct fileproc *, struct knote *, vfs_context_t);
+int _ATselect(struct fileproc *, int, void *, vfs_context_t);
+int _ATioctl(struct fileproc *, u_long, caddr_t, vfs_context_t);
+int _ATwrite(struct fileproc *, struct uio *, int, vfs_context_t);
+int _ATread(struct fileproc *, struct uio *, int, vfs_context_t);
+int _ATclose(struct fileglob *, vfs_context_t);
+
+int _ATrw(struct fileproc *, enum uio_rw, struct uio *, vfs_context_t);
+
 extern struct atpcb ddp_head;
 extern lck_mtx_t * atalk_mutex;
-
-extern void 
-  ddp_putmsg(gref_t *gref, gbuf_t *m),
-  elap_wput(gref_t *gref, gbuf_t *m),
-  atp_wput(gref_t *gref, gbuf_t *m),
-  asp_wput(gref_t *gref, gbuf_t *m),
-#ifdef AURP_SUPPORT
-  aurp_wput(gref_t *gref, gbuf_t *m),
-#endif
-  adsp_wput(gref_t *gref, gbuf_t *m);
   
 int atp_free_cluster_timeout_set = 0;
 
+int gref_alloc(gref_t **);
+
 
-void atalk_putnext(gref_t *gref, gbuf_t *m);
 /* bms:  make gref_close non static so its callable from kernel */
 int gref_close(gref_t *gref);
 
@@ -82,9 +93,9 @@ SYSCTL_DECL(_net_appletalk);
 dbgBits_t dbgBits;
 SYSCTL_STRUCT(_net_appletalk, OID_AUTO, debug, CTLFLAG_WR, 
 	      &dbgBits, dbgBits, "AppleTalk Debug Flags");
-volatile int RouterMix = RT_MIX_DEFAULT; /* default for nbr of ppsec */
+int RouterMix = RT_MIX_DEFAULT; /* default for nbr of ppsec */
 SYSCTL_INT(_net_appletalk, OID_AUTO, routermix, CTLFLAG_WR, 
-	   (int *)&RouterMix, 0, "Appletalk RouterMix");
+			&RouterMix, 0, "Appletalk RouterMix");
 at_ddp_stats_t at_ddp_stats;		/* DDP statistics */
 SYSCTL_STRUCT(_net_appletalk, OID_AUTO, ddpstats, CTLFLAG_RD,
 	      &at_ddp_stats, at_ddp_stats, "AppleTalk DDP Stats");
@@ -92,8 +103,10 @@ SYSCTL_STRUCT(_net_appletalk, OID_AUTO, ddpstats, CTLFLAG_RD,
 static void ioccmd_t_32_to_64( ioccmd_t *from_p, user_ioccmd_t *to_p );
 static void ioccmd_t_64_to_32( user_ioccmd_t *from_p, ioccmd_t *to_p );
 
+extern lck_mtx_t *atalk_cluster_lock;
+caddr_t	atp_free_cluster_list = NULL;
 
-caddr_t	atp_free_cluster_list = 0;
+void gref_wput(gref_t *, gbuf_t *m);
 
 void gref_wput(gref, m)
 	gref_t *gref;
@@ -266,9 +279,9 @@ int _ATputmsg(fd, ctlptr, datptr, flags, err, proc)
 	return rc;
 }
 
-int _ATclose(fg, proc)
-	struct fileglob *fg;
-	struct proc *proc;
+int _ATclose(
+	struct fileglob *fg,
+	__unused vfs_context_t ctx)
 {
 	int err;
 	gref_t *gref;
@@ -282,15 +295,16 @@ int _ATclose(fg, proc)
 	return err;
 }
 
-int _ATrw(fp, rw, uio, p)
-     void *fp;
+int _ATrw(fp, rw, uio, ctx)
+     struct fileproc *fp;
      enum uio_rw rw;
      struct uio *uio;
-     struct proc *p;
+	 vfs_context_t ctx;
 {
     int err, len, clen = 0, res;
     gref_t *gref;
     gbuf_t *m, *mhead, *mprev;
+	proc_t p = vfs_context_proc(ctx);
 
 	/* no need to get/drop iocount as the fp already has one */
     if ((err = atalk_getref_locked(fp, 0, &gref, p, 1)) != 0)
@@ -395,32 +409,30 @@ int _ATrw(fp, rw, uio, p)
   return 0;
 } /* _ATrw */
 
-int _ATread(fp, uio, cred, flags, p)
-	struct fileproc *fp;
-	struct uio *uio;
-	void *cred;
-	int flags;
-	struct proc *p;
+int _ATread(
+	struct fileproc *fp,
+	struct uio *uio,
+	__unused int flags,
+	vfs_context_t ctx)
 {
      int stat;
 
 	atalk_lock();
-	stat = _ATrw(fp, UIO_READ, uio, p);
+	stat = _ATrw(fp, UIO_READ, uio, ctx);
 	atalk_unlock();
 	return stat;
 }
 
-int _ATwrite(fp, uio, cred, flags, p)
-	struct fileproc *fp;
-	struct uio *uio;
-	void *cred;
-	int flags;
-	struct proc *p;
+int _ATwrite(
+	struct fileproc *fp,
+	struct uio *uio,
+	__unused int flags,
+	vfs_context_t ctx)
 {
-     int stat;
+	int stat;
 
 	atalk_lock();
-	stat = _ATrw(fp, UIO_WRITE, uio, p);
+	stat = _ATrw(fp, UIO_WRITE, uio, ctx);
 	atalk_unlock();
 
 	return stat;
@@ -588,11 +600,11 @@ l_done:
 	return err;
 } /* at_ioctl */
 
-int _ATioctl(fp, cmd, arg, proc)
-	void *fp;
-	u_long cmd;
-	register caddr_t arg;
-	void *proc;
+int _ATioctl(
+	struct fileproc *fp,
+	u_long cmd,
+	register caddr_t arg,
+	__unused vfs_context_t ctx)
 {
 	int err;
 	gref_t *gref;
@@ -612,14 +624,15 @@ int _ATioctl(fp, cmd, arg, proc)
 	return err;
 }
 
-int _ATselect(fp, which, wql, proc)
+int _ATselect(fp, which, wql, ctx)
 	struct fileproc *fp;
 	int which;
 	void * wql;
-	struct proc *proc;
+	vfs_context_t ctx;
 {
 	int err, rc = 0;
 	gref_t *gref;
+	proc_t proc = vfs_context_proc(ctx);
 
 	/* Radar 4128949: Drop the proc_fd lock here to avoid lock inversion issues with the other AT calls
       * select() is already holding a reference on the fd, so it won't go away during the time it is unlocked.
@@ -660,10 +673,10 @@ int _ATselect(fp, which, wql, proc)
 	return rc;
 }
 
-int _ATkqfilter(fp, kn, p)
-	struct fileproc *fp;
-	struct knote *kn;
-	struct proc *p;
+int _ATkqfilter(
+	__unused struct fileproc *fp,
+	__unused struct knote *kn,
+	__unused vfs_context_t ctx)
 {
 	return (EOPNOTSUPP);
 }
@@ -805,10 +818,10 @@ int atalk_peek(gref, event)
 	return rc;
 }
 
+#if 0
 static gbuf_t *trace_msg;
 
-void atalk_settrace(str, p1, p2, p3, p4, p5)
-	char *str;
+void atalk_settrace(char * str, p1, p2, p3, p4, p5)
 {
 	int len;
 	gbuf_t *m, *nextm;
@@ -838,14 +851,15 @@ void atalk_gettrace(m)
 		trace_msg = 0;
 	}
 }
+#endif /* 0 */
 
 #define GREF_PER_BLK 32
 static gref_t *gref_free_list = 0;
+extern gbuf_t *atp_resource_m;
 
 int gref_alloc(grefp)
 	gref_t **grefp;
 {
-	extern gbuf_t *atp_resource_m;
 	int i;
 	gbuf_t *m;
 	gref_t *gref, *gref_array;
@@ -919,101 +933,65 @@ int gref_close(gref_t *gref)
 	return rc;
 }
 
-/* 
-   Buffer Routines
-
-   *** Some to be replaced with mbuf routines, some to be re-written
-       as mbuf routines (and moved to kern/uicp_mbuf.c or sys/mbuf.h?).
-   ***
-
-*/
-
-/*
- * LD 5/12/97 Added for MacOSX, defines a m_clattach function that:
- * "Allocates an mbuf structure and attaches an external cluster."
- */
-
-struct mbuf *m_clattach(extbuf, extfree, extsize, extarg, wait)
-	caddr_t extbuf;	
-	void (*extfree)(caddr_t , u_int, caddr_t);
-	u_int extsize;
-	caddr_t extarg;
-	int wait;
-{
-        struct mbuf *m;
-
-        if ((m = m_gethdr(wait, MSG_DATA)) == NULL)
-                return (NULL);
-
-        m->m_ext.ext_buf = extbuf;
-        m->m_ext.ext_free = extfree;
-        m->m_ext.ext_size = extsize;
-        m->m_ext.ext_arg = extarg;
-        m->m_ext.ext_refs.forward = 
-	  	m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
-        m->m_data = extbuf;
-        m->m_flags |= M_EXT;
-
-        return (m);
-}
-
-
-
 /*
 	temp fix for bug 2731148  - until this code is re-written to use standard clusters
 	Deletes any free clusters on the free list.
 */
-void atp_delete_free_clusters()
+void atp_delete_free_clusters(__unused void *junk)
 {
 	caddr_t cluster;
 	caddr_t cluster_list;
-	
-	
+
 	/* check for free clusters on the free_cluster_list to be deleted */
-	MBUF_LOCK();	/* lock used by mbuf routines */
 
-		untimeout(&atp_delete_free_clusters, NULL);
-		atp_free_cluster_timeout_set = 0;
+	untimeout(&atp_delete_free_clusters, NULL);
 
-		cluster_list = atp_free_cluster_list;
-		atp_free_cluster_list = 0;
-		
-	MBUF_UNLOCK();
-	
-	while (cluster = cluster_list)
+	lck_mtx_lock(atalk_cluster_lock);
+
+	atp_free_cluster_timeout_set = 0;
+
+	cluster_list = atp_free_cluster_list;
+	atp_free_cluster_list = NULL;
+
+	lck_mtx_unlock(atalk_cluster_lock);
+
+	while ((cluster = cluster_list))
 	{
 		cluster_list = *((caddr_t*)cluster);
 		FREE(cluster, M_MCLUST);
 	}
-	
 }
 
 
 /* 
    Used as the "free" routine for over-size clusters allocated using
-   m_lgbuf_alloc(). Called by m_free while under MBUF_LOCK.
+   m_lgbuf_alloc().
 */
 
-void m_lgbuf_free(buf, size, arg)
-     caddr_t buf;
-     u_int size;
-     caddr_t arg; /* not needed, but they're in m_free() */
+void m_lgbuf_free(caddr_t, u_int, caddr_t);
+
+void m_lgbuf_free(
+     caddr_t buf,
+     __unused u_int size,
+     __unused caddr_t arg) /* not needed, but they're in m_free() */
 {
-	/* FREE(buf, M_MCLUST); - can't free here - called from m_free while under lock */
-	
+	int t;
+
 	/* move to free_cluster_list to be deleted later */
 	caddr_t cluster = (caddr_t)buf;
-	
-	/* don't need a lock because this is only called called from m_free which */
-	/* is under MBUF_LOCK */
+
+	lck_mtx_lock(atalk_cluster_lock);
+
 	*((caddr_t*)cluster) = atp_free_cluster_list;
 	atp_free_cluster_list = cluster;
-	
-	if (atp_free_cluster_timeout_set == 0)
-	{
+
+	if ((t = atp_free_cluster_timeout_set) == 0)
 		atp_free_cluster_timeout_set = 1;
+
+	lck_mtx_unlock(atalk_cluster_lock);
+
+	if (t == 0)
 		timeout(&atp_delete_free_clusters, NULL, (1 * HZ));
-	}
 }
 
 /*
@@ -1029,7 +1007,7 @@ struct mbuf *m_lgbuf_alloc(size, wait)
 	if (atp_free_cluster_list)
 		atp_delete_free_clusters(NULL);	/* delete any free clusters on the free list */
 
-	/* Radar 5423402 
+	/* Radar 5398094 
 	 * check that the passed size is within admissible boundaries
 	 * The max data size being ASP of 4576 (8 * ATP_DATA_SIZE),
 	 * allow for extra space for control data
@@ -1048,14 +1026,14 @@ struct mbuf *m_lgbuf_alloc(size, wait)
 			return(NULL);
 		}
 		if (NULL == 
-		    (m = m_clattach(buf, m_lgbuf_free, size, 0, 
+		    (m = m_clattach(NULL, MSG_DATA, buf, m_lgbuf_free, size, 0, 
 				    (wait)? M_WAIT: M_DONTWAIT))) {
 			m_lgbuf_free(buf, 0, 0);
 			return(NULL);
 		}
 	} else {
 		m = m_gethdr(((wait)? M_WAIT: M_DONTWAIT), MSG_DATA);
-		if (m && (size > MHLEN)) {
+		if (m && ((size_t)size > MHLEN)) {
 			MCLGET(m, ((wait)? M_WAIT: M_DONTWAIT));
 			if (!(m->m_flags & M_EXT)) {
 				(void)m_free(m);
@@ -1083,8 +1061,8 @@ gbuf_t *gbuf_alloc_wait(size, wait)
 	/* Standard mbuf allocation routines assume that the caller
 	   will set the size. */
 	if (m) {
-		(struct mbuf *)m->m_pkthdr.len = size;
-		(struct mbuf *)m->m_len = size;
+		m->m_pkthdr.len = size;
+		m->m_len = size;
 	}
 
 	return(m);
@@ -1243,8 +1221,9 @@ l_cont:	prev_m = 0;
 		len -= buf_len;
 		goto l_cont;
 
-	} else
-		return 1;
+	}
+
+	return 1;
 }
 
 /*
diff --git a/bsd/netat/sysglue.h b/bsd/netat/sysglue.h
index 45ceec8fd..10d74f988 100644
--- a/bsd/netat/sysglue.h
+++ b/bsd/netat/sysglue.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Title:	sysglue.h - AppleTalk protocol to Unix System V/streams interface
@@ -170,6 +176,8 @@ int gbuf_freel(gbuf_t *m);
 void gbuf_linkb(gbuf_t *m1, gbuf_t *m2);
 void gbuf_linkpkt(gbuf_t *m1, gbuf_t *m2);
 int gbuf_msgsize(gbuf_t *m);
+struct mbuf *copy_pkt(struct mbuf *, int);
+int append_copy(struct mbuf *, struct mbuf *, int);
 
 #define gbuf_cont(m)	m->m_next
 #define gbuf_next(m)	m->m_nextpkt
@@ -202,6 +210,12 @@ int gbuf_msgsize(gbuf_t *m);
 #undef timeout
 #undef untimeout
 
+struct atpBDS;
+int _ATPgetrsp(int, struct atpBDS *, int *, void *);
+int _ATPgetreq(int , unsigned char *, int , int *, void *);
+int _ATPsndrsp(int , unsigned char *, int , int, int *, void *);
+int _ATPsndreq(int , unsigned char *, int , int, int *, void *);
+
 #endif /* KERNEL_PRIVATE */
 #endif /* KERNEL */
 #endif /* __APPLE_API_OBSOLETE */
diff --git a/bsd/netat/zip.h b/bsd/netat/zip.h
index fd0b38a5f..175280c07 100644
--- a/bsd/netat/zip.h
+++ b/bsd/netat/zip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
diff --git a/bsd/netinet/bootp.h b/bsd/netinet/bootp.h
index 8cbbfa7c9..27f2581de 100644
--- a/bsd/netinet/bootp.h
+++ b/bsd/netinet/bootp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Bootstrap Protocol (BOOTP).  RFC 951.
@@ -44,7 +50,7 @@ struct bootp {
 	u_char	bp_htype;	/* hardware addr type */
 	u_char	bp_hlen;	/* hardware addr length */
 	u_char	bp_hops;	/* gateway hops */
-	u_long	bp_xid;		/* transaction ID */
+	u_int32_t bp_xid;	/* transaction ID */
 	u_short	bp_secs;	/* seconds since boot began */	
 	u_short	bp_unused;
 	iaddr_t	bp_ciaddr;	/* client IP address */
@@ -68,7 +74,7 @@ struct bootp {
  */
 struct vend {
 	u_char	v_magic[4];	/* magic number */
-	u_long	v_flags;	/* flags/opcodes, etc. */
+	u_int32_t v_flags;	/* flags/opcodes, etc. */
 	u_char	v_unused[56];	/* currently unused */
 };
 #define	VM_STANFORD	"STAN"	/* v_magic for Stanford */
diff --git a/bsd/netinet/dhcp.h b/bsd/netinet/dhcp.h
index 76b310f4e..e2868508f 100644
--- a/bsd/netinet/dhcp.h
+++ b/bsd/netinet/dhcp.h
@@ -4,25 +4,31 @@
 #include <sys/appleapiopts.h>
 
 /*
- * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * dhcp.h
@@ -58,8 +64,9 @@ struct dhcp_packet {
     struct dhcp 	dhcp;
 };
 
-#define DHCP_PACKET_OPTIONS_MIN	312
-#define DHCP_PACKET_MIN		(sizeof(struct dhcp) + DHCP_PACKET_OPTIONS_MIN)
+#define DHCP_OPTIONS_MIN	312
+#define DHCP_PACKET_MIN		(sizeof(struct dhcp_packet) + DHCP_OPTIONS_MIN)
+#define DHCP_PAYLOAD_MIN	(sizeof(struct dhcp) + DHCP_OPTIONS_MIN)
 
 /* dhcp message types */
 #define DHCPDISCOVER	1
@@ -71,6 +78,20 @@ struct dhcp_packet {
 #define DHCPRELEASE	7
 #define DHCPINFORM	8
 
+enum {
+    dhcp_msgtype_none_e		= 0,
+    dhcp_msgtype_discover_e 	= DHCPDISCOVER,
+    dhcp_msgtype_offer_e	= DHCPOFFER,
+    dhcp_msgtype_request_e	= DHCPREQUEST,
+    dhcp_msgtype_decline_e	= DHCPDECLINE,
+    dhcp_msgtype_ack_e		= DHCPACK,
+    dhcp_msgtype_nak_e		= DHCPNAK,
+    dhcp_msgtype_release_e	= DHCPRELEASE,
+    dhcp_msgtype_inform_e	= DHCPINFORM,
+};
+
+typedef uint8_t dhcp_msgtype_t;
+
 typedef int32_t			dhcp_time_secs_t; /* absolute time */
 typedef int32_t			dhcp_lease_t;     /* relative time */
 #define dhcp_time_hton		htonl
diff --git a/bsd/netinet/dhcp_options.c b/bsd/netinet/dhcp_options.c
index 8dccba9fd..1992e3af3 100644
--- a/bsd/netinet/dhcp_options.c
+++ b/bsd/netinet/dhcp_options.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2002-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*
  * dhcp_options.c
  * - routines to parse and access dhcp options
@@ -13,14 +40,22 @@
  * - imported from bootp project
  */
 
+#include <string.h>
 #include <sys/types.h>
 #include <sys/param.h>
 #include <netinet/in.h>
 #include <sys/malloc.h>
+#include <libkern/libkern.h>
 
 #include <netinet/dhcp.h>
 #include <netinet/dhcp_options.h>
 
+#ifdef	DHCP_DEBUG
+#define	dprintf(x) printf x;
+#else	/* !DHCP_DEBUG */
+#define	dprintf(x)
+#endif	/* DHCP_DEBUG */
+
 static __inline__ void
 my_free(void * ptr)
 {
@@ -79,7 +114,7 @@ ptrlist_count(ptrlist_t * list)
     return (list->count);
 }
 
-static void *
+static const void *
 ptrlist_element(ptrlist_t * list, int i)
 {
     if (list->array == NULL)
@@ -100,9 +135,7 @@ ptrlist_grow(ptrlist_t * list)
 	list->array = my_malloc(sizeof(*list->array) * list->size);
     }
     else if (list->size == list->count) {
-#ifdef DEBUG
-	printf("doubling %d to %d\n", list->size, list->size * 2);
-#endif /* DEBUG */
+	dprintf(("doubling %d to %d\n", list->size, list->size * 2));
 	list->array = my_realloc(list->array, 
 				 sizeof(*list->array) * list->size,
 				 sizeof(*list->array) * list->size * 2);
@@ -114,7 +147,7 @@ ptrlist_grow(ptrlist_t * list)
 }
 
 static boolean_t
-ptrlist_add(ptrlist_t * list, void * element)
+ptrlist_add(ptrlist_t * list, const void * element)
 {
     if (ptrlist_grow(list) == FALSE)
 	return (FALSE);
@@ -156,7 +189,7 @@ ptrlist_concat(ptrlist_t * list, ptrlist_t * extra)
  *   Routines to parse/access existing options buffers.
  */
 boolean_t
-dhcpol_add(dhcpol_t * list, void * element)
+dhcpol_add(dhcpol_t * list, const void * element)
 {
     return (ptrlist_add((ptrlist_t *)list, element));
 }
@@ -167,7 +200,7 @@ dhcpol_count(dhcpol_t * list)
     return (ptrlist_count((ptrlist_t *)list));
 }
 
-void *
+const void *
 dhcpol_element(dhcpol_t * list, int i)
 {
     return (ptrlist_element((ptrlist_t *)list, i));
@@ -201,27 +234,24 @@ dhcpol_concat(dhcpol_t * list, dhcpol_t * extra)
  *   the end tag.
  */
 boolean_t
-dhcpol_parse_buffer(dhcpol_t * list, void * buffer, int length,
-		    unsigned char * err)
+dhcpol_parse_buffer(dhcpol_t * list, const void * buffer, int length)
 {
     int			len;
-    unsigned char *	scan;
-    unsigned char	tag;
-
-    if (err)
-	err[0] = '\0';
+    const uint8_t *	scan;
+    uint8_t		tag;
 
     dhcpol_init(list);
 
     len = length;
     tag = dhcptag_pad_e;
-    for (scan = (unsigned char *)buffer; tag != dhcptag_end_e && len > 0; ) {
+    for (scan = (const uint8_t *)buffer; tag != dhcptag_end_e && len > 0; ) {
 
 	tag = scan[DHCP_TAG_OFFSET];
 
 	switch (tag) {
 	  case dhcptag_end_e:
-	      dhcpol_add(list, scan); /* remember that it was terminated */
+	      /* remember that it was terminated */
+	      dhcpol_add(list, scan);
 	      scan++;
 	      len--;
 	      break;
@@ -230,7 +260,7 @@ dhcpol_parse_buffer(dhcpol_t * list, void * buffer, int length,
 	      len--;
 	      break;
 	  default: {
-	      unsigned char	option_len = scan[DHCP_LEN_OFFSET];
+	      uint8_t	option_len = scan[DHCP_LEN_OFFSET];
 	    
 	      dhcpol_add(list, scan);
 	      len -= (option_len + 2);
@@ -241,8 +271,7 @@ dhcpol_parse_buffer(dhcpol_t * list, void * buffer, int length,
     }
     if (len < 0) {
 	/* ran off the end */
-	if (err)
-	    sprintf(err, "parse failed near tag %d", tag);
+	dprintf(("dhcp_options: parse failed near tag %d", tag));
 	dhcpol_free(list);
 	return (FALSE);
     }
@@ -261,7 +290,7 @@ dhcpol_parse_buffer(dhcpol_t * list, void * buffer, int length,
  *   calls will retrieve the next occurence of the option.
  *   Before the first call, *start should be set to 0.
  */
-void *
+const void *
 dhcpol_find(dhcpol_t * list, int tag, int * len_p, int * start)
 {
     int 	i = 0;
@@ -273,7 +302,7 @@ dhcpol_find(dhcpol_t * list, int tag, int * len_p, int * start)
 	i = *start;
 
     for (; i < dhcpol_count(list); i++) {
-	unsigned char * option = dhcpol_element(list, i);
+	const uint8_t * 	option = dhcpol_element(list, i);
 	
 	if (option[DHCP_TAG_OFFSET] == tag) {
 	    if (len_p)
@@ -286,6 +315,7 @@ dhcpol_find(dhcpol_t * list, int tag, int * len_p, int * start)
     return (NULL);
 }
 
+#if 0
 /*
  * Function: dhcpol_get
  * 
@@ -308,7 +338,7 @@ dhcpol_get(dhcpol_t * list, int tag, int * len_p)
 	return (NULL);
 
     for (i = 0; i < dhcpol_count(list); i++) {
-	unsigned char * option = dhcpol_element(list, i);
+	const uint8_t * 	option = dhcpol_element(list, i);
 	
 	if (option[DHCP_TAG_OFFSET] == tag) {
 	    int len = option[DHCP_LEN_OFFSET];
@@ -326,6 +356,7 @@ dhcpol_get(dhcpol_t * list, int tag, int * len_p)
     *len_p = data_len;
     return (data);
 }
+#endif 0
 
 /*
  * Function: dhcpol_parse_packet
@@ -339,46 +370,38 @@ dhcpol_get(dhcpol_t * list, int tag, int * len_p)
  *    then parses pkt->dp_sname if specified.
  */
 boolean_t
-dhcpol_parse_packet(dhcpol_t * options, struct dhcp * pkt, int len,
-		    unsigned char * err)
+dhcpol_parse_packet(dhcpol_t * options, const struct dhcp * pkt, int len)
 {
     char		rfc_magic[4] = RFC_OPTIONS_MAGIC;
 
     dhcpol_init(options);	/* make sure it's empty */
 
-    if (err)
-	err[0] = '\0';
-
     if (len < (sizeof(*pkt) + RFC_MAGIC_SIZE)) {
-	if (err) {
-	    sprintf(err, "packet is too short: %d < %d",
-		    len, (int)sizeof(*pkt) + RFC_MAGIC_SIZE);
-	}
+	dprintf(("dhcp_options: packet is too short: %d < %d\n",
+		 len, (int)sizeof(*pkt) + RFC_MAGIC_SIZE));
 	return (FALSE);
     }
     if (bcmp(pkt->dp_options, rfc_magic, RFC_MAGIC_SIZE)) {
-	if (err)
-	    sprintf(err, "missing magic number");
+	dprintf(("dhcp_options: missing magic number\n"));
 	return (FALSE);
     }
     if (dhcpol_parse_buffer(options, pkt->dp_options + RFC_MAGIC_SIZE,
-			    len - sizeof(*pkt) - RFC_MAGIC_SIZE, err) == FALSE)
+			    len - sizeof(*pkt) - RFC_MAGIC_SIZE) == FALSE)
 	return (FALSE);
     { /* get overloaded options */
-	unsigned char *	overload;
+	const uint8_t *	overload;
 	int		overload_len;
 
-	overload = (unsigned char *)
-	    dhcpol_find(options, dhcptag_option_overload_e, 
-				&overload_len, NULL);
+	overload = dhcpol_find(options, dhcptag_option_overload_e, 
+			       &overload_len, NULL);
 	if (overload && overload_len == 1) { /* has overloaded options */
 	    dhcpol_t	extra;
-
+	    
 	    dhcpol_init(&extra);
 	    if (*overload == DHCP_OVERLOAD_FILE
 		|| *overload == DHCP_OVERLOAD_BOTH) {
 		if (dhcpol_parse_buffer(&extra, pkt->dp_file, 
-					 sizeof(pkt->dp_file), NULL)) {
+					sizeof(pkt->dp_file))) {
 		    dhcpol_concat(options, &extra);
 		    dhcpol_free(&extra);
 		}
@@ -386,7 +409,7 @@ dhcpol_parse_packet(dhcpol_t * options, struct dhcp * pkt, int len,
 	    if (*overload == DHCP_OVERLOAD_SNAME
 		|| *overload == DHCP_OVERLOAD_BOTH) {
 		if (dhcpol_parse_buffer(&extra, pkt->dp_sname, 
-					 sizeof(pkt->dp_sname), NULL)) {
+					sizeof(pkt->dp_sname))) {
 		    dhcpol_concat(options, &extra);
 		    dhcpol_free(&extra);
 		}
@@ -397,63 +420,156 @@ dhcpol_parse_packet(dhcpol_t * options, struct dhcp * pkt, int len,
 }
 
 /*
- * Function: dhcpol_parse_vendor
+ * Module: dhcpoa
  *
  * Purpose:
- *   Given a set of options, find the vendor specific option(s)
- *   and parse all of them into a single option list.
- *  
- * Return value:
- *   TRUE if vendor specific options existed and were parsed succesfully,
- *   FALSE otherwise.
+ *   Types and functions to create new dhcp option areas.
  */
-boolean_t
-dhcpol_parse_vendor(dhcpol_t * vendor, dhcpol_t * options,
-		    unsigned char * err)
+
+/*
+ * Function: dhcpoa_{init_common, init_no_end, init}
+ *
+ * Purpose:
+ *   Initialize an option area structure so that it can be used
+ *   in calling the dhcpoa_* routines.
+ */
+static void
+dhcpoa_init_common(dhcpoa_t * oa_p, void * buffer, int size, int reserve)
+{
+    bzero(oa_p, sizeof(*oa_p));
+    oa_p->oa_buffer = buffer;
+    oa_p->oa_size = size;
+    oa_p->oa_reserve = reserve;
+}
+
+void
+dhcpoa_init_no_end(dhcpoa_t * oa_p, void * buffer, int size)
 {
-    dhcpol_t		extra;
-    boolean_t		ret = FALSE;
-    int 		start = 0;
+    dhcpoa_init_common(oa_p, buffer, size, 0);
+    return;
+}
 
-    if (err)
-	err[0] = '\0';
+int
+dhcpoa_size(dhcpoa_t * oa_p)
+{
+    return (oa_p->oa_size);
+}
 
-    dhcpol_init(vendor);
-    dhcpol_init(&extra);
+void
+dhcpoa_init(dhcpoa_t * oa_p, void * buffer, int size)
+{
+    /* initialize the area, reserve space for the end tag */
+    dhcpoa_init_common(oa_p, buffer, size, 1);
+    return;
+}
+/*
+ * Function: dhcpoa_add
+ *
+ * Purpose:
+ *   Add an option to the option area.
+ */
+dhcpoa_ret_t
+dhcpoa_add(dhcpoa_t * oa_p, dhcptag_t tag, int len, const void * option)
+{
+    if (len > DHCP_OPTION_SIZE_MAX) {
+	dprintf(("tag %d option %d > %d\n", tag, len, DHCP_OPTION_SIZE_MAX));
+	return (dhcpoa_failed_e);
+    }
 
-    for (;;) {
-	void *		data;
-	int		len;
+    if (oa_p->oa_end_tag) {
+	dprintf(("attempt to add data after end tag\n"));
+	return (dhcpoa_failed_e);
+    }
 
-	data = dhcpol_find(options, dhcptag_vendor_specific_e, &len, &start);
-	if (data == NULL) {
-	    break; /* out of for */
+    switch (tag) {
+      case dhcptag_end_e:
+	if ((oa_p->oa_offset + 1) > oa_p->oa_size) {
+	    /* this can't happen since we're careful to leave space */
+	    dprintf(("can't add end tag %d > %d\n",
+		     oa_p->oa_offset + oa_p->oa_reserve, oa_p->oa_size));
+	    return (dhcpoa_failed_e);
 	}
-
-	if (dhcpol_parse_buffer(&extra, data, len, err) == FALSE) {
-	    goto failed;
+	((uint8_t *)oa_p->oa_buffer)[oa_p->oa_offset + DHCP_TAG_OFFSET] = tag;
+	oa_p->oa_offset++;
+	oa_p->oa_end_tag = 1;
+	break;
+
+      case dhcptag_pad_e:
+	/* 1 for pad tag */
+	if ((oa_p->oa_offset + oa_p->oa_reserve + 1) > oa_p->oa_size) {
+	    dprintf(("can't add pad tag %d > %d\n",
+		     oa_p->oa_offset + oa_p->oa_reserve + 1, oa_p->oa_size));
+	    return (dhcpoa_full_e);
 	}
-
-	if (dhcpol_concat(vendor, &extra) == FALSE) {
-	    if (err)
-		sprintf(err, "dhcpol_concat() failed at %d\n", start);
-	    goto failed;
+	((uint8_t *)oa_p->oa_buffer)[oa_p->oa_offset + DHCP_TAG_OFFSET] = tag;
+	oa_p->oa_offset++;
+	break;
+
+      default:
+	/* 2 for tag/len */
+	if ((oa_p->oa_offset + len + 2 + oa_p->oa_reserve) > oa_p->oa_size) {
+	    dprintf(("can't add tag %d (%d > %d)\n", tag,
+		     oa_p->oa_offset + len + 2 + oa_p->oa_reserve, 
+		     oa_p->oa_size));
+	    return (dhcpoa_full_e);
 	}
-	dhcpol_free(&extra);
-	ret = TRUE;
+	((uint8_t *)oa_p->oa_buffer)[oa_p->oa_offset + DHCP_TAG_OFFSET] = tag;
+	((uint8_t *)oa_p->oa_buffer)[oa_p->oa_offset + DHCP_LEN_OFFSET] = (uint8_t)len;
+	if (len) {
+	    memcpy(oa_p->oa_buffer + (DHCP_OPTION_OFFSET + oa_p->oa_offset),
+		   option, len);
+	}
+	oa_p->oa_offset += len + DHCP_OPTION_OFFSET;
+	break;
     }
-    if (ret == FALSE) {
-	if (err)
-	    strcpy(err, "missing vendor specific options");
+    oa_p->oa_option_count++;
+    return (dhcpoa_success_e);
+}
+
+/*
+ * Function: dhcpoa_add_dhcpmsg
+ *
+ * Purpose:
+ *   Add a dhcp message option to the option area.
+ */
+dhcpoa_ret_t
+dhcpoa_add_dhcpmsg(dhcpoa_t * oa_p, dhcp_msgtype_t msgtype)
+{
+    return (dhcpoa_add(oa_p, dhcptag_dhcp_message_type_e,
+		       sizeof(msgtype), &msgtype));
+}
+
+int
+dhcpoa_used(dhcpoa_t * oa_p)
+{
+    return (oa_p->oa_offset);
+}
+
+int
+dhcpoa_freespace(dhcpoa_t * oa_p)
+{
+    int	freespace;
+
+    freespace = oa_p->oa_size - oa_p->oa_offset - oa_p->oa_reserve;
+    if (freespace < 0) {
+	freespace = 0;
     }
-    return (ret);
+    return (freespace);
+}
 
- failed:
-    dhcpol_free(vendor);
-    dhcpol_free(&extra);
-    return (FALSE);
+int
+dhcpoa_count(dhcpoa_t * oa_p)
+{
+    return (oa_p->oa_option_count);
 }
 
+void *
+dhcpoa_buffer(dhcpoa_t * oa_p) 
+{
+    return (oa_p->oa_buffer);
+}
+
+
 #ifdef TEST_DHCP_OPTIONS
 char test_empty[] = {
     99, 130, 83, 99,
@@ -514,7 +630,6 @@ main()
 {
     int 	i;
     dhcpol_t 	options;
-    char	error[256];
     struct dhcp * pkt = (struct dhcp *)buf;
 
     dhcpol_init(&options);
@@ -523,18 +638,12 @@ main()
 	printf("\nTest %d: ", i);
 	bcopy(tests[i].data, pkt->dp_options, tests[i].len);
 	if (dhcpol_parse_packet(&options, pkt, 
-				sizeof(*pkt) + tests[i].len,
-				error) != tests[i].result) {
+				sizeof(*pkt) + tests[i].len)
+	    != tests[i].result) {
 	    printf("test '%s' FAILED\n", tests[i].name);
-	    if (tests[i].result == TRUE) {
-		printf("error message returned was %s\n", error);
-	    }
 	}
 	else {
 	    printf("test '%s' PASSED\n", tests[i].name);
-	    if (tests[i].result == FALSE) {
-		printf("error message returned was %s\n", error);
-	    }
 	}
 	dhcpol_free(&options);
     }
diff --git a/bsd/netinet/dhcp_options.h b/bsd/netinet/dhcp_options.h
index 8c5daabde..ca6197a92 100644
--- a/bsd/netinet/dhcp_options.h
+++ b/bsd/netinet/dhcp_options.h
@@ -3,25 +3,31 @@
 #define _NETINET_DHCP_OPTIONS_H
 #include <sys/appleapiopts.h>
 /*
- * Copyright (c) 1999-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -59,7 +65,7 @@
 #define RFC_OPTIONS_MAGIC		{ 99, 130, 83, 99 }
 #define RFC_MAGIC_SIZE			4 /* bytes */
 
-typedef enum {
+enum {
     /* rfc 1497 vendor extensions: 0..18, 255 */
     dhcptag_pad_e                      	= 0,
     dhcptag_end_e                      	= 255,
@@ -157,7 +163,8 @@ typedef enum {
 
     /* ad-hoc network disable option */
     dhcptag_auto_configure_e           	= 116,
-} dhcptag_t;
+};
+typedef uint8_t dhcptag_t; 
 
 /*
  * Module: dhcpol (dhcp options list)
@@ -167,9 +174,9 @@ typedef enum {
  */
 
 typedef struct {
-    void * *	array;	/* malloc'd array of pointers */
-    int		size;	/* number of elements in array */
-    int		count;	/* number of occupied elements */
+    const void * *	array;	/* malloc'd array of pointers */
+    int			size;	/* number of elements in array */
+    int			count;	/* number of occupied elements */
 } ptrlist_t;
 
 typedef ptrlist_t dhcpol_t;
@@ -179,20 +186,93 @@ typedef ptrlist_t dhcpol_t;
 void			dhcpol_init(dhcpol_t * list);
 void			dhcpol_free(dhcpol_t * list);
 int			dhcpol_count(dhcpol_t * list);
-boolean_t		dhcpol_add(dhcpol_t * list, void * element);
-void *			dhcpol_element(dhcpol_t * list, int i);
+boolean_t		dhcpol_add(dhcpol_t * list, const void * element);
+const void *		dhcpol_element(dhcpol_t * list, int i);
 boolean_t		dhcpol_concat(dhcpol_t * list, dhcpol_t * extra);
-boolean_t		dhcpol_parse_buffer(dhcpol_t * list, void * buffer, 
-					    int length, unsigned char * err);
-void *			dhcpol_find(dhcpol_t * list, int tag, int * len_p, 
+boolean_t		dhcpol_parse_buffer(dhcpol_t * list,
+					    const void * buffer, 
+					    int length);
+const void *		dhcpol_find(dhcpol_t * list, int tag, int * len_p, 
 				    int * start);
+#if 0
 void *			dhcpol_get(dhcpol_t * list, int tag, int * len_p);
+#endif 0
 boolean_t		dhcpol_parse_packet(dhcpol_t * options, 
-					    struct dhcp * pkt, int len,
-					    unsigned char * err);
-boolean_t		dhcpol_parse_vendor(dhcpol_t * vendor, 
-					    dhcpol_t * options,
-					    unsigned char * err);
+					    const struct dhcp * pkt, int len);
 void			dhcpol_print(dhcpol_t * list);
+/*
+ * Module: dhcpoa (dhcp options area)
+ *
+ * Purpose:
+ *   Types and functions to create new dhcp option areas.
+ */
+
+/*
+ * Struct: dhcpoa_s
+ * Purpose:
+ *   To record information about a dhcp option data area.
+ */
+struct dhcpoa_s {
+    uint8_t *	oa_buffer;	/* data area to hold options */
+    int		oa_size;	/* size of buffer */
+    int		oa_offset;	/* offset of next option to write */
+    int		oa_end_tag;	/* to mark when options are terminated */
+    int		oa_option_count;/* number of options present */
+    int		oa_reserve; 	/* space to reserve, either 0 or 1 */
+};
+
+/*
+ * Type: dhcpoa_t
+ *
+ * Purpose:
+ *   To record information about a dhcp option data area.
+ */
+typedef struct dhcpoa_s dhcpoa_t;
+
+/* 
+ * Type:dhcpoa_ret_t
+ *
+ * Purpose:
+ *  outine return codes 
+ */
+typedef enum {
+    dhcpoa_success_e = 0,
+    dhcpoa_failed_e,
+    dhcpoa_full_e,
+} dhcpoa_ret_t;
+
+void
+dhcpoa_init(dhcpoa_t * opt, void * buffer, int size);
+
+void
+dhcpoa_init_no_end(dhcpoa_t * opt, void * buffer, int size);
+
+dhcpoa_ret_t
+dhcpoa_add(dhcpoa_t * oa_p, dhcptag_t tag, int len, const void * data);
+
+dhcpoa_ret_t
+dhcpoa_add_dhcpmsg(dhcpoa_t * oa_p, dhcp_msgtype_t msgtype);
+
+#if 0
+dhcpoa_ret_t
+dhcpoa_vendor_add(dhcpoa_t * oa_p, dhcpoa_t * vendor_oa_p,
+		  dhcptag_t tag, int len, void * option);
+#endif 0
+
+int
+dhcpoa_used(dhcpoa_t * oa_p);
+
+int
+dhcpoa_count(dhcpoa_t * oa_p);
+
+void *
+dhcpoa_buffer(dhcpoa_t * oa_p);
+
+int
+dhcpoa_freespace(dhcpoa_t * oa_p);
+
+int
+dhcpoa_size(dhcpoa_t * oa_p);
+
 #endif KERNEL_PRIVATE
 #endif /* _NETINET_DHCP_OPTIONS_H */
diff --git a/bsd/netinet/icmp6.h b/bsd/netinet/icmp6.h
index 87380d493..5c1dd0239 100644
--- a/bsd/netinet/icmp6.h
+++ b/bsd/netinet/icmp6.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: icmp6.h,v 1.46 2001/04/27 15:09:48 itojun Exp $	*/
 
@@ -508,10 +514,10 @@ struct icmp6_filter {
 #ifdef KERNEL
 #define	ICMP6_FILTER_SETPASSALL(filterp) \
 do {								\
-	int i; u_char *p;					\
-	p = (u_char *)filterp;					\
+	int i; u_char *ptr;					\
+	ptr = (u_char *)filterp;					\
 	for (i = 0; i < sizeof(struct icmp6_filter); i++)	\
-		p[i] = 0xff;					\
+		ptr[i] = 0xff;					\
 } while (0)
 #define	ICMP6_FILTER_SETBLOCKALL(filterp) \
 	bzero(filterp, sizeof(struct icmp6_filter))
diff --git a/bsd/netinet/icmp_var.h b/bsd/netinet/icmp_var.h
index 948997e69..4b767b901 100644
--- a/bsd/netinet/icmp_var.h
+++ b/bsd/netinet/icmp_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -65,19 +71,19 @@
  */
 struct	icmpstat {
 /* statistics related to icmp packets generated */
-	u_long	icps_error;		/* # of calls to icmp_error */
-	u_long	icps_oldshort;		/* no error 'cuz old ip too short */
-	u_long	icps_oldicmp;		/* no error 'cuz old was icmp */
-	u_long	icps_outhist[ICMP_MAXTYPE + 1];
+	u_int32_t	icps_error;	/* # of calls to icmp_error */
+	u_int32_t	icps_oldshort;	/* no error 'cuz old ip too short */
+	u_int32_t	icps_oldicmp;	/* no error 'cuz old was icmp */
+	u_int32_t	icps_outhist[ICMP_MAXTYPE + 1];
 /* statistics related to input messages processed */
- 	u_long	icps_badcode;		/* icmp_code out of range */
-	u_long	icps_tooshort;		/* packet < ICMP_MINLEN */
-	u_long	icps_checksum;		/* bad checksum */
-	u_long	icps_badlen;		/* calculated bound mismatch */
-	u_long	icps_reflect;		/* number of responses */
-	u_long	icps_inhist[ICMP_MAXTYPE + 1];
-	u_long	icps_bmcastecho; 	/* b/mcast echo requests dropped */
-	u_long	icps_bmcasttstamp; 	/* b/mcast tstamp requests dropped */
+	u_int32_t	icps_badcode;	/* icmp_code out of range */
+	u_int32_t	icps_tooshort;	/* packet < ICMP_MINLEN */
+	u_int32_t	icps_checksum;	/* bad checksum */
+	u_int32_t	icps_badlen;	/* calculated bound mismatch */
+	u_int32_t	icps_reflect;	/* number of responses */
+	u_int32_t	icps_inhist[ICMP_MAXTYPE + 1];
+	u_int32_t	icps_bmcastecho;/* b/mcast echo requests dropped */
+	u_int32_t	icps_bmcasttstamp; /* b/mcast tstamp requests dropped */
 };
 
 /*
diff --git a/bsd/netinet/if_atm.c b/bsd/netinet/if_atm.c
index f4639f632..0fa54144f 100644
--- a/bsd/netinet/if_atm.c
+++ b/bsd/netinet/if_atm.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*      $NetBSD: if_atm.c,v 1.6 1996/10/13 02:03:01 christos Exp $       */
 
@@ -172,8 +178,7 @@ atm_rtrequest(req, rt, sa)
 		 */
 		bcopy(LLADDR(SDL(gate)), &api.aph, sizeof(api.aph));
 		api.rxhand = NULL;
-		if (dlil_ioctl(0, rt->rt_ifp, SIOCATMENA, 
-							(caddr_t)&api) != 0) {
+		if (ifnet_ioctl(rt->rt_ifpm 0, SIOCATMENA, &api) != 0) {
 			printf("atm: couldn't add VC\n");
 			goto failed;
 		}
@@ -215,8 +220,7 @@ failed:
 
 		bcopy(LLADDR(SDL(gate)), &api.aph, sizeof(api.aph));
 		api.rxhand = NULL;
-		dlil_ioctl(0, rt->rt_ifp, SIOCATMDIS, 
-							(caddr_t)&api);
+		ifnet_ioctl(rt->rt_ifp, 0, SIOCATMDIS, &api);
 
 		break;
 	}
diff --git a/bsd/netinet/if_atm.h b/bsd/netinet/if_atm.h
index e1b147905..989fa974d 100644
--- a/bsd/netinet/if_atm.h
+++ b/bsd/netinet/if_atm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* $FreeBSD: src/sys/netinet/if_atm.h,v 1.2.6.1 2000/08/03 01:07:02 peter Exp $ */
 /*      $NetBSD: if_atm.h,v 1.2 1996/07/03 17:17:17 chuck Exp $       */
diff --git a/bsd/netinet/if_ether.h b/bsd/netinet/if_ether.h
index 79440febd..d0bb1057d 100644
--- a/bsd/netinet/if_ether.h
+++ b/bsd/netinet/if_ether.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/if_fddi.h b/bsd/netinet/if_fddi.h
index ed9071d4c..5aee8f1bf 100644
--- a/bsd/netinet/if_fddi.h
+++ b/bsd/netinet/if_fddi.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/if_tun.h b/bsd/netinet/if_tun.h
index 9f748c7a3..e4d3e7a67 100644
--- a/bsd/netinet/if_tun.h
+++ b/bsd/netinet/if_tun.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/netinet/igmp.c b/bsd/netinet/igmp.c
index 0f427e7e3..961549d60 100644
--- a/bsd/netinet/igmp.c
+++ b/bsd/netinet/igmp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 Stephen Deering.
@@ -57,6 +63,12 @@
  *
  *	@(#)igmp.c	8.1 (Berkeley) 7/19/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * Internet Group Management Protocol (IGMP) routines.
@@ -89,6 +101,10 @@
 #include <netinet/igmp.h>
 #include <netinet/igmp_var.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #ifndef __APPLE__
 static MALLOC_DEFINE(M_IGMP, "igmp", "igmp state");
 #endif
@@ -110,7 +126,7 @@ static struct router_info *Head;
 static void igmp_sendpkt(struct in_multi *, int, unsigned long);
 
 void
-igmp_init()
+igmp_init(void)
 {
 	struct ipoption *ra;
 
@@ -374,8 +390,7 @@ igmp_input(
 }
 
 int
-igmp_joingroup(inm)
-	struct in_multi *inm;
+igmp_joingroup(struct in_multi *inm)
 {
 
 	if (inm->inm_addr.s_addr == igmp_all_hosts_group
@@ -395,8 +410,7 @@ igmp_joingroup(inm)
 }
 
 void
-igmp_leavegroup(inm)
-	struct in_multi *inm;
+igmp_leavegroup(struct in_multi *inm)
 {
 	if (inm->inm_state == IGMP_IREPORTEDLAST &&
 	    inm->inm_addr.s_addr != igmp_all_hosts_group &&
@@ -406,7 +420,7 @@ igmp_leavegroup(inm)
 }
 
 void
-igmp_fasttimo()
+igmp_fasttimo(void)
 {
 	struct in_multi *inm;
 	struct in_multistep step;
@@ -435,7 +449,7 @@ igmp_fasttimo()
 }
 
 void
-igmp_slowtimo()
+igmp_slowtimo(void)
 {
 	struct router_info *rti =  Head;
 
@@ -459,21 +473,21 @@ igmp_slowtimo()
 static struct route igmprt;
 
 static void
-igmp_sendpkt(inm, type, addr)
-	struct in_multi *inm;
-	int type;
-	unsigned long addr;
+igmp_sendpkt(struct in_multi *inm, int type, unsigned long addr)
 {
         struct mbuf *m;
         struct igmp *igmp;
         struct ip *ip;
         struct ip_moptions imo;
 
-        MGETHDR(m, M_DONTWAIT, MT_HEADER);
+        MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
         if (m == NULL)
                 return;
 
-	m->m_pkthdr.rcvif = loif;
+	m->m_pkthdr.rcvif = lo_ifp;
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_linklayer(inm->inm_ifp, m);
+#endif
 	m->m_pkthdr.len = sizeof(struct ip) + IGMP_MINLEN;
 	MH_ALIGN(m, IGMP_MINLEN + sizeof(struct ip));
 	m->m_data += sizeof(struct ip);
@@ -499,18 +513,23 @@ igmp_sendpkt(inm, type, addr)
 
         imo.imo_multicast_ifp  = inm->inm_ifp;
         imo.imo_multicast_ttl  = 1;
-	imo.imo_multicast_vif  = -1;
+		imo.imo_multicast_vif  = -1;
+#if MROUTING
         /*
          * Request loopback of the report if we are acting as a multicast
          * router, so that the process-level routing demon can hear it.
          */
         imo.imo_multicast_loop = (ip_mrouter != NULL);
+#else
+        imo.imo_multicast_loop = 0;
+#endif
 
 	/*
 	 * XXX
 	 * Do we have to worry about reentrancy here?  Don't think so.
 	 */
-        ip_output(m, router_alert, &igmprt, 0, &imo);
+        ip_output(m, router_alert, &igmprt, 0, &imo, NULL);
 
         ++igmpstat.igps_snd_reports;
 }
+
diff --git a/bsd/netinet/igmp.h b/bsd/netinet/igmp.h
index dc62c7da5..3774cd860 100644
--- a/bsd/netinet/igmp.h
+++ b/bsd/netinet/igmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 Stephen Deering.
diff --git a/bsd/netinet/igmp_var.h b/bsd/netinet/igmp_var.h
index 435197b7c..5e9f7e983 100644
--- a/bsd/netinet/igmp_var.h
+++ b/bsd/netinet/igmp_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 Stephen Deering.
@@ -108,7 +114,7 @@ struct igmpstat {
  */
 #define IGMP_AGE_THRESHOLD			540
 
-void	igmp_init(void);
+void	igmp_init(void) __attribute__((section("__TEXT, initcode")));
 void	igmp_input(struct mbuf *, int);
 int		igmp_joingroup(struct in_multi *);
 void	igmp_leavegroup(struct in_multi *);
diff --git a/bsd/netinet/in.c b/bsd/netinet/in.c
index 9db5f54b9..2663dd4c3 100644
--- a/bsd/netinet/in.c
+++ b/bsd/netinet/in.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1991, 1993
@@ -67,9 +73,12 @@
 #include <sys/kern_event.h>
 #include <sys/syslog.h>
 
+#include <pexpert/pexpert.h>
+
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/route.h>
+#include <net/kpi_protocol.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -102,11 +111,32 @@ SYSCTL_INT(_net_inet_ip, OID_AUTO, subnets_are_local, CTLFLAG_RW,
 
 struct in_multihead in_multihead; /* XXX BSS initialization */
 
-extern lck_mtx_t *rt_mtx;
-
 /* Track whether or not the SIOCARPIPLL ioctl has been called */
 __private_extern__	u_int32_t	ipv4_ll_arp_aware = 0;
 
+int
+inaddr_local(struct in_addr in)
+{
+	struct rtentry *rt;
+	struct sockaddr_in sin;
+	int local = 0;
+
+	sin.sin_family = AF_INET;
+	sin.sin_len = sizeof (sin);
+	sin.sin_addr = in;
+	rt = rtalloc1((struct sockaddr *)&sin, 0, 0UL);
+
+	if (rt != NULL) {
+		if (rt->rt_gateway->sa_family == AF_LINK ||
+		    (rt->rt_ifp->if_flags & IFF_LOOPBACK))
+			local = 1;
+		rtfree(rt);
+	} else {
+		local = in_localaddr(in);
+	}
+	return (local);
+}
+
 /*
  * Return 1 if an internet address is for a ``local'' host
  * (one to which we have a connection).  If subnetsarelocal
@@ -114,8 +144,7 @@ __private_extern__	u_int32_t	ipv4_ll_arp_aware = 0;
  * Otherwise, it includes only the directly-connected (sub)nets.
  */
 int
-in_localaddr(in)
-	struct in_addr in;
+in_localaddr(struct in_addr in)
 {
 	u_long i = ntohl(in.s_addr);
 	struct in_ifaddr *ia;
@@ -148,8 +177,7 @@ in_localaddr(in)
  * may be forwarded.
  */
 int
-in_canforward(in)
-	struct in_addr in;
+in_canforward(struct in_addr in)
 {
 	u_long i = ntohl(in.s_addr);
 	u_long net;
@@ -168,8 +196,7 @@ in_canforward(in)
  * Trim a mask in a sockaddr
  */
 static void
-in_socktrim(ap)
-struct sockaddr_in *ap;
+in_socktrim(struct sockaddr_in *ap)
 {
     char *cplim = (char *) &ap->sin_addr;
     char *cp = (char *) (&ap->sin_addr + 1);
@@ -183,10 +210,9 @@ struct sockaddr_in *ap;
 }
 
 static int
-in_mask2len(mask)
-	struct in_addr *mask;
+in_mask2len(struct in_addr *mask)
 {
-	int x, y;
+	size_t x, y;
 	u_char *p;
 
 	p = (u_char *)mask;
@@ -205,9 +231,7 @@ in_mask2len(mask)
 }
 
 static void
-in_len2mask(mask, len)
-	struct in_addr *mask;
-	int len;
+in_len2mask(struct in_addr *mask, int len)
 {
 	int i;
 	u_char *p;
@@ -225,6 +249,22 @@ static int in_interfaces;	/* number of external internet interfaces */
 /*
  * Generic internet control operations (ioctl's).
  * Ifp is 0 if not an interface-specific ioctl.
+ *
+ * Returns:	0			Success
+ *		EINVAL
+ *		EADDRNOTAVAIL
+ *		EDESTADDRREQ
+ *		EPERM
+ *		ENOBUFS
+ *		EBUSY
+ *		EOPNOTSUPP
+ *	proc_suser:EPERM
+ *	suser:EPERM
+ *	in_lifaddr_ioctl:???
+ *	dlil_ioctl:???
+ *	in_ifinit:???
+ *	dlil_plumb_protocol:???
+ *	dlil_unplumb_protocol:???
  */
 /* ARGSUSED */
 int
@@ -236,13 +276,12 @@ in_control(
 	struct proc *p)
 {
 	struct ifreq *ifr = (struct ifreq *)data;
-	struct in_ifaddr *ia = 0, *iap;
+	struct in_ifaddr *ia = NULL, *iap;
 	struct ifaddr *ifa;
-	struct in_ifaddr *oia;
 	struct in_aliasreq *ifra = (struct in_aliasreq *)data;
 	struct sockaddr_in oldaddr;
-	int error, hostIsNew, maskIsNew;
-	u_long i;
+	int error = 0;
+	int hostIsNew, maskIsNew;
 	struct kev_msg        ev_msg;
 	struct kev_in_data    in_event_data;
 
@@ -280,20 +319,33 @@ in_control(
 						break;
 				}
 			}
+		/* take a reference on ia before releasing mutex */
+		if (ia != NULL) {
+			ifaref(&ia->ia_ifa);
+		}
 		lck_mtx_unlock(rt_mtx);
 	}
 	switch (cmd) {
 	case SIOCAUTOADDR:
 	case SIOCARPIPLL:
-		if (p && (error = proc_suser(p)) != 0)
-			return error;
+		if (p && (error = proc_suser(p)) != 0) {
+			goto done;
+		}
+		if (ifp == 0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
 		break;
 
 	case SIOCAIFADDR:
 	case SIOCDIFADDR:
-		if (ifp == 0)
-			return (EADDRNOTAVAIL);
+		if (ifp == 0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
 		if (ifra->ifra_addr.sin_family == AF_INET) {
+			struct in_ifaddr *oia;
+
 			lck_mtx_lock(rt_mtx);
 			for (oia = ia; ia; ia = ia->ia_link.tqe_next) {
 				if (ia->ia_ifp == ifp  &&
@@ -301,34 +353,54 @@ in_control(
 				    ifra->ifra_addr.sin_addr.s_addr)
 					break;
 			}
+			/* take a reference on ia before releasing mutex */
+			if (ia != NULL && ia != oia) {
+				ifaref(&ia->ia_ifa);
+			}
 			lck_mtx_unlock(rt_mtx);
+			if (oia != NULL && oia != ia) {
+				ifafree(&oia->ia_ifa);
+			}
 			if ((ifp->if_flags & IFF_POINTOPOINT)
 			    && (cmd == SIOCAIFADDR)
 			    && (ifra->ifra_dstaddr.sin_addr.s_addr
 				== INADDR_ANY)) {
-				return EDESTADDRREQ;
+				error = EDESTADDRREQ;
+				goto done;
 			}
 		}
-        else if (cmd == SIOCAIFADDR)
-            return (EINVAL);
-		if (cmd == SIOCDIFADDR && ia == 0)
-			return (EADDRNOTAVAIL);
+		else if (cmd == SIOCAIFADDR) {
+			error = EINVAL;
+			goto done;
+		}
+		if (cmd == SIOCDIFADDR && ia == 0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
 		/* FALLTHROUGH */
 	case SIOCSIFADDR:
 	case SIOCSIFNETMASK:
 	case SIOCSIFDSTADDR:
-		if ((so->so_state & SS_PRIV) == 0)
-			return (EPERM);
-
-		if (ifp == 0)
-			return (EADDRNOTAVAIL);
-        if (ifra->ifra_addr.sin_family != AF_INET && cmd == SIOCSIFADDR)
-            return (EINVAL);
+		if ((so->so_state & SS_PRIV) == 0) {
+			error = EPERM;
+			goto done;
+		}
+		if (ifp == 0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
+		if (ifra->ifra_addr.sin_family != AF_INET 
+		    && cmd == SIOCSIFADDR) {
+			error = EINVAL;
+			goto done;
+		}
 		if (ia == (struct in_ifaddr *)0) {
 			ia = (struct in_ifaddr *)
 				_MALLOC(sizeof *ia, M_IFADDR, M_WAITOK);
-			if (ia == (struct in_ifaddr *)NULL)
-				return (ENOBUFS);
+			if (ia == (struct in_ifaddr *)NULL) {
+				error = ENOBUFS;
+				goto done;
+			}
 			bzero((caddr_t)ia, sizeof *ia);
 			/*
 			 * Protect from ipintr() traversing address list
@@ -337,6 +409,8 @@ in_control(
 			
 			ifa = &ia->ia_ifa;
 
+			ia->ia_addr.sin_family = AF_INET;
+			ia->ia_addr.sin_len = sizeof (ia->ia_addr);
 			ifa->ifa_addr = (struct sockaddr *)&ia->ia_addr;
 			ifa->ifa_dstaddr = (struct sockaddr *)&ia->ia_dstaddr;
 			ifa->ifa_netmask = (struct sockaddr *)&ia->ia_sockmask;
@@ -353,14 +427,17 @@ in_control(
 			ifnet_lock_done(ifp);
 
 			lck_mtx_lock(rt_mtx);
+			ifaref(&ia->ia_ifa);
 			TAILQ_INSERT_TAIL(&in_ifaddrhead, ia, ia_link);
 			lck_mtx_unlock(rt_mtx);
 
 			/* Generic protocol plumbing */
 
-			if (error = dlil_plumb_protocol(PF_INET, ifp)) {
-				kprintf("in.c: warning can't plumb proto if=%s%n type %d error=%d\n",
-					ifp->if_name, ifp->if_unit, ifp->if_type, error);
+			if ((error = proto_plumb(PF_INET, ifp))) {
+				if (error != EEXIST) {
+					kprintf("in.c: warning can't plumb proto if=%s%d type %d error=%d\n",
+						ifp->if_name, ifp->if_unit, ifp->if_type, error);
+				}
 				error = 0; /*discard error, can be cold with unsupported interfaces */
 			}
 
@@ -369,19 +446,25 @@ in_control(
 
 	case SIOCPROTOATTACH:
 	case SIOCPROTODETACH:
-		if (p && (error = proc_suser(p)) != 0)
-			return error;
-		if (ifp == 0)
-			return (EADDRNOTAVAIL);
+		if (p && (error = proc_suser(p)) != 0) {
+			goto done;
+		}
+		if (ifp == 0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
 		break;
                 
 	case SIOCSIFBRDADDR:
 #ifdef __APPLE__
-		if ((so->so_state & SS_PRIV) == 0)
-			return (EPERM);
+		if ((so->so_state & SS_PRIV) == 0) {
+			error = EPERM;
+			goto done;
+		}
 #else
-		if (p && (error = suser(p)) != 0)
-			return error;
+		if (p && (error = suser(p)) != 0) {
+			goto done;
+		}
 #endif
 		/* FALLTHROUGH */
 
@@ -389,14 +472,14 @@ in_control(
 	case SIOCGIFNETMASK:
 	case SIOCGIFDSTADDR:
 	case SIOCGIFBRDADDR:
-		if (ia == (struct in_ifaddr *)0)
-			return (EADDRNOTAVAIL);
+		if (ia == (struct in_ifaddr *)0) {
+			error = EADDRNOTAVAIL;
+			goto done;
+		}
 		break;
 	}
 	switch (cmd) {
 	case SIOCAUTOADDR:
-		if (ifp == 0)
-			return (EADDRNOTAVAIL);
 		ifnet_lock_exclusive(ifp);
 		if (ifr->ifr_intval)
 			ifp->if_eflags |= IFEF_AUTOCONFIGURING;
@@ -406,8 +489,6 @@ in_control(
 		break;
 	
 	case SIOCARPIPLL:
-		if (ifp == 0)
-			return (EADDRNOTAVAIL);
 		ipv4_ll_arp_aware = 1;
 		ifnet_lock_exclusive(ifp);
 		if (ifr->ifr_data)
@@ -422,14 +503,18 @@ in_control(
 		break;
 
 	case SIOCGIFBRDADDR:
-		if ((ifp->if_flags & IFF_BROADCAST) == 0)
-			return (EINVAL);
+		if ((ifp->if_flags & IFF_BROADCAST) == 0) {
+			error = EINVAL;
+			break;
+		}
 		*((struct sockaddr_in *)&ifr->ifr_dstaddr) = ia->ia_broadaddr;
 		break;
 
 	case SIOCGIFDSTADDR:
-		if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
-			return (EINVAL);
+		if ((ifp->if_flags & IFF_POINTOPOINT) == 0) {
+			error = EINVAL;
+			break;
+		}
 		*((struct sockaddr_in *)&ifr->ifr_dstaddr) = ia->ia_dstaddr;
 		break;
 
@@ -438,17 +523,19 @@ in_control(
 		break;
 
 	case SIOCSIFDSTADDR:
-		if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
-			return (EINVAL);
+		if ((ifp->if_flags & IFF_POINTOPOINT) == 0) {
+			error = EINVAL;
+			break;
+		}
 		oldaddr = ia->ia_dstaddr;
 		ia->ia_dstaddr = *(struct sockaddr_in *)&ifr->ifr_dstaddr;
-		error = dlil_ioctl(PF_INET, ifp, SIOCSIFDSTADDR, (caddr_t)ia);
-		if (error == EOPNOTSUPP)
-		     error = 0;
-
+		error = ifnet_ioctl(ifp, PF_INET, SIOCSIFDSTADDR, ia);
+		if (error == EOPNOTSUPP) {
+			error = 0;
+		}
 		if (error) {
-		     ia->ia_dstaddr = oldaddr;
-		     return error;
+			ia->ia_dstaddr = oldaddr;
+			break;
 		}
 
 		ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -490,8 +577,10 @@ in_control(
 		break;
 
 	case SIOCSIFBRDADDR:
-		if ((ifp->if_flags & IFF_BROADCAST) == 0)
-			return (EINVAL);
+		if ((ifp->if_flags & IFF_BROADCAST) == 0) {
+			error = EINVAL;
+			break;
+		}
 		ia->ia_broadaddr = *(struct sockaddr_in *)&ifr->ifr_broadaddr;
 
 		ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -525,14 +614,12 @@ in_control(
 		break;
 
 	case SIOCSIFADDR:
-		return (in_ifinit(ifp, ia,
-		    (struct sockaddr_in *) &ifr->ifr_addr, 1));
+		error = in_ifinit(ifp, ia, (struct sockaddr_in *) &ifr->ifr_addr, 1);
+		break;
 
 	case SIOCPROTOATTACH:
-		error = dlil_plumb_protocol(PF_INET, ifp);
-		if (error)
-			return(error);
-                break;
+		error = proto_plumb(PF_INET, ifp);
+		break;
                 
 	case SIOCPROTODETACH:
                 // if an ip address is still present, refuse to detach
@@ -541,16 +628,18 @@ in_control(
 			if (ifa->ifa_addr->sa_family == AF_INET)
 				break;
 		ifnet_lock_done(ifp);
-		if (ifa != 0)
-			return EBUSY;
+		if (ifa != 0) {
+			error =  EBUSY;
+			break;
+		}
 
-		error = dlil_unplumb_protocol(PF_INET, ifp);
-		if (error)
-			return(error);
+		error = proto_unplumb(PF_INET, ifp);
 		break;
 		
 
-	case SIOCSIFNETMASK:
+	case SIOCSIFNETMASK: {
+		u_long i;
+		
 		i = ifra->ifra_addr.sin_addr.s_addr;
 		ia->ia_subnetmask = ntohl(ia->ia_sockmask.sin_addr.s_addr = i);
 		ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -582,7 +671,7 @@ in_control(
 		kev_post_msg(&ev_msg);
 
 		break;
-
+	}
 	case SIOCAIFADDR:
 		maskIsNew = 0;
 		hostIsNew = 1;
@@ -652,15 +741,15 @@ in_control(
 
 		     kev_post_msg(&ev_msg);
 		}
-
-		return (error);
+		break;
 
 	case SIOCDIFADDR:
-		error = dlil_ioctl(PF_INET, ifp, SIOCDIFADDR, (caddr_t)ia);
+		error = ifnet_ioctl(ifp, PF_INET, SIOCDIFADDR, ia);
 		if (error == EOPNOTSUPP)
 			error = 0;
-		if (error)
-		    return error;
+		if (error != 0) {
+			break;
+		}
 
 		/* Fill out the kernel event information */
 		ev_msg.vendor_code    = KEV_VENDOR_APPLE;
@@ -696,10 +785,18 @@ in_control(
 		 */
 		in_ifscrub(ifp, ia, 1);
 		ifa = &ia->ia_ifa;
+#if CONFIG_FORCE_OUT_IFP	
+		// Cleanup any pdp hack related route
+		if (ia->ia_route)
+		{
+			ia->ia_route->rt_flags &= ~RTF_UP;
+			rtfree_locked(ia->ia_route);
+			ia->ia_route = NULL;
+		}
+#endif
 		lck_mtx_unlock(rt_mtx);
 		ifnet_lock_exclusive(ifp);
 		if_detach_ifa(ifp, ifa);
-		ifafree(&ia->ia_ifa);
         
 #ifdef __APPLE__
        /*
@@ -727,6 +824,20 @@ in_control(
 
 		/* Post the kernel event */
 		kev_post_msg(&ev_msg);
+
+		/*
+		 * See if there is any IPV4 address left and if so,
+		 * reconfigure KDP to use current primary address.
+		 */
+		ifa = ifa_ifpgetprimary(ifp, AF_INET);
+		if (ifa != NULL) {
+			error = ifnet_ioctl(ifp, PF_INET, SIOCSIFADDR, ifa);
+			if (error == EOPNOTSUPP)
+				error = 0;
+
+			/* Release reference from ifa_ifpgetprimary() */
+			ifafree(ifa);
+		}
 		break;
 
 #ifdef __APPLE__
@@ -819,9 +930,13 @@ in_control(
 #endif /* __APPLE__ */
 
 	default:
-		return EOPNOTSUPP;
+		error = EOPNOTSUPP;
 	}
-	return (0);
+ done:
+	if (ia != NULL) {
+		ifafree(&ia->ia_ifa);
+	}
+	return (error);
 }
 
 /*
@@ -914,7 +1029,8 @@ in_lifaddr_ioctl(
 	case SIOCDLIFADDR:
 	    {
 		struct in_ifaddr *ia;
-		struct in_addr mask, candidate, match;
+		struct in_addr mask, candidate;
+		struct in_addr match = { 0 };
 		struct sockaddr_in *sin;
 		int cmp;
 
@@ -1041,18 +1157,40 @@ in_ifinit(
 	u_long i = ntohl(sin->sin_addr.s_addr);
 	struct sockaddr_in oldaddr;
 	int flags = RTF_UP, error;
+	struct ifaddr *ifa0;
+	unsigned int cmd;
 
 	oldaddr = ia->ia_addr;
 	ia->ia_addr = *sin;
 
 	/*
-	 * Give the interface a chance to initialize
-	 * if this is its first address,
-	 * and to validate the address if necessary.
+	 * Give the interface a chance to initialize if this is its first
+	 * address, and to validate the address if necessary.  Send down
+	 * SIOCSIFADDR for first address, and SIOCAIFADDR for alias(es).
+	 * We find the first IPV4 address assigned to it and check if this
+	 * is the same as the one passed into this routine.
 	 */
-	error = dlil_ioctl(PF_INET, ifp, SIOCSIFADDR, (caddr_t)ia);
+	ifa0 = ifa_ifpgetprimary(ifp, AF_INET);
+	cmd = (&ia->ia_ifa == ifa0) ? SIOCSIFADDR : SIOCAIFADDR;
+	error = ifnet_ioctl(ifp, PF_INET, cmd, ia);
 	if (error == EOPNOTSUPP)
-	     error = 0;
+		error = 0;
+	/*
+	 * If we've just sent down SIOCAIFADDR, send another ioctl down
+	 * for SIOCSIFADDR for the first IPV4 address of the interface,
+	 * because an address change on one of the addresses will result
+	 * in the removal of the previous first IPV4 address.  KDP needs
+	 * be reconfigured with the current primary IPV4 address.
+	 */
+	if (error == 0 && cmd == SIOCAIFADDR) {
+		error = ifnet_ioctl(ifp, PF_INET, SIOCSIFADDR, ifa0);
+		if (error == EOPNOTSUPP)
+			error = 0;
+	}
+
+	/* Release reference from ifa_ifpgetprimary() */
+	ifafree(ifa0);
+
 	if (error) {
 		ia->ia_addr = oldaddr;
 		return (error);
@@ -1284,7 +1422,7 @@ in_delmulti(
 	}
 	if (inm2 != *inm) {
 		lck_mtx_unlock(rt_mtx);
-		printf("in_delmulti - ignorning invalid inm (0x%x)\n", *inm);
+		printf("in_delmulti - ignoring invalid inm (%p)\n", *inm);
 		return;
 	}
 	lck_mtx_unlock(rt_mtx);
@@ -1298,10 +1436,11 @@ in_delmulti(
 }
 
 #if !NFSCLIENT
+int inet_aton(char *cp, struct in_addr *pin);
 int
 inet_aton(char * cp, struct in_addr * pin)
 {
-    u_char * b = (char *)pin;
+    u_char * b = (unsigned char *)pin;
     int	   i;
     char * p;
 
diff --git a/bsd/netinet/in.h b/bsd/netinet/in.h
index 868bcb95a..8ed004c82 100644
--- a/bsd/netinet/in.h
+++ b/bsd/netinet/in.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993
@@ -59,6 +65,7 @@
 #define _NETINET_IN_H_
 #include <sys/appleapiopts.h>
 #include <sys/_types.h>
+#include <stdint.h>		/* uint(8|16|32)_t */
 
 #ifndef _IN_ADDR_T
 #define _IN_ADDR_T
@@ -92,18 +99,18 @@ typedef	__uint16_t	in_port_t;
  * Protocols (RFC 1700)
  */
 #define	IPPROTO_IP		0		/* dummy for IP */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_HOPOPTS	0		/* IP6 hop-by-hop options */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	IPPROTO_ICMP		1		/* control message protocol */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_IGMP		2		/* group mgmt protocol */
 #define	IPPROTO_GGP		3		/* gateway^2 (deprecated) */
 #define IPPROTO_IPV4		4 		/* IPv4 encapsulation */
 #define IPPROTO_IPIP		IPPROTO_IPV4	/* for compatibility */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	IPPROTO_TCP		6		/* tcp */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_ST		7		/* Stream protocol II */
 #define	IPPROTO_EGP		8		/* exterior gateway protocol */
 #define	IPPROTO_PIGP		9		/* private interior gateway */
@@ -114,9 +121,9 @@ typedef	__uint16_t	in_port_t;
 #define	IPPROTO_EMCON		14		/* EMCON */
 #define	IPPROTO_XNET		15		/* Cross Net Debugger */
 #define	IPPROTO_CHAOS		16		/* Chaos*/
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	IPPROTO_UDP		17		/* user datagram protocol */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_MUX		18		/* Multiplexing */
 #define	IPPROTO_MEAS		19		/* DCN Measurement Subsystems */
 #define	IPPROTO_HMP		20		/* Host Monitoring */
@@ -140,9 +147,9 @@ typedef	__uint16_t	in_port_t;
 #define	IPPROTO_CMTP		38		/* Control Message Transport */
 #define	IPPROTO_TPXX		39		/* TP++ Transport */
 #define	IPPROTO_IL		40		/* IL transport protocol */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define 	IPPROTO_IPV6		41		/* IP6 header */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_SDRP		42		/* Source Demand Routing */
 #define 	IPPROTO_ROUTING	43		/* IP6 routing header */
 #define 	IPPROTO_FRAGMENT	44		/* IP6 fragmentation header */
@@ -207,15 +214,15 @@ typedef	__uint16_t	in_port_t;
 /* 255: Reserved */
 /* BSD Private, local use, namespace incursion */
 #define	IPPROTO_DIVERT		254		/* divert pseudo-protocol */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	IPPROTO_RAW		255		/* raw IP packet */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IPPROTO_MAX		256
 
 /* last return value of *_input(), meaning "all job for this pkt is done".  */
 #define	IPPROTO_DONE		257
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Local port number conventions:
@@ -264,7 +271,7 @@ typedef	__uint16_t	in_port_t;
 
 #define	__DARWIN_IPPORT_RESERVED	1024
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * Ports < IPPORT_RESERVED are reserved for
  * privileged processes (e.g. root).         (IP_PORTRANGE_LOW)
@@ -289,7 +296,7 @@ typedef	__uint16_t	in_port_t;
  * have a fit if we use.
  */
 #define IPPORT_RESERVEDSTART	600
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Internet address (a structure for historical reasons)
@@ -306,7 +313,7 @@ struct in_addr {
 #define	INADDR_ANY		(u_int32_t)0x00000000
 #define	INADDR_BROADCAST	(u_int32_t)0xffffffff	/* must be masked */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	IN_CLASSA(i)		(((u_int32_t)(i) & 0x80000000) == 0)
 #define	IN_CLASSA_NET		0xff000000
 #define	IN_CLASSA_NSHIFT	24
@@ -349,7 +356,7 @@ struct in_addr {
 #endif
 
 #define	IN_LOOPBACKNET		127			/* official! */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Socket address, internet style.
@@ -364,7 +371,7 @@ struct sockaddr_in {
 
 #define INET_ADDRSTRLEN                 16
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * Structure used to describe IP options.
  * Used to store options internally, to pass them to a process,
@@ -431,6 +438,19 @@ struct ip_opts {
 #define	IP_DUMMYNET_FLUSH	62   /* flush dummynet */
 #define	IP_DUMMYNET_GET		64   /* get entire dummynet pipes */
 
+#define	IP_TRAFFIC_MGT_BACKGROUND	65   /* int*; get background IO flags; set background IO */
+
+#if CONFIG_FORCE_OUT_IFP
+/* This is a hack, this is only a hack. */
+#define	IP_FORCE_OUT_IFP	69	/* char ifname[] - send traffic on this interface */
+#endif
+
+/* Background socket configuration flags */
+#ifdef __APPLE_API_UNSTABLE
+#define TRAFFIC_MGT_SO_BACKGROUND	0x0001
+#define TRAFFIC_MGT_SO_BG_SUPPRESSED	0x0002
+#endif /* __APPLE_API_UNSTABLE */
+
 /*
  * Defaults and limits for options
  */
@@ -568,7 +588,7 @@ struct ip_mreq {
 }
 #endif /* KERNEL_PRIVATE */
 
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 /* INET6 stuff */
@@ -580,19 +600,26 @@ struct ip_mreq {
 #ifdef KERNEL_PRIVATE
 struct ifnet; struct mbuf;	/* forward declarations for Standard C */
 
-int	 in_broadcast(struct in_addr, struct ifnet *);
-int	 in_canforward(struct in_addr);
-int	 in_cksum(struct mbuf *, int);
-int      in_cksum_skip(struct mbuf *, u_short, u_short);
-u_short	 in_addword(u_short, u_short);
-u_short  in_pseudo(u_int, u_int, u_int);
-int	 in_localaddr(struct in_addr);
-u_long	in_netof(struct in_addr);
+extern int in_broadcast(struct in_addr, struct ifnet *);
+extern int in_canforward(struct in_addr);
+
+#define	in_cksum(m, l)		inet_cksum(m, 0, 0, l)
+#define	in_cksum_skip(m, l, o)	inet_cksum(m, 0, o, (l) - (o))
+
+extern u_int16_t inet_cksum(struct mbuf *m, unsigned int proto,
+    unsigned int offset, unsigned int transport_len);
+extern u_short in_addword(u_short, u_short);
+extern u_short in_pseudo(u_int, u_int, u_int);
+
+extern int in_localaddr(struct in_addr);
+extern u_long in_netof(struct in_addr);
+
+extern int inaddr_local(struct in_addr);
 #endif /* KERNEL_PRIVATE */
 #define MAX_IPv4_STR_LEN	16
 #define MAX_IPv6_STR_LEN	64
 
-const char	*inet_ntop(int, const void *, char *, size_t); /* in libkern */
+extern const char *inet_ntop(int, const void *, char *, socklen_t); /* in libkern */
 #endif /* KERNEL */
 
 #endif /* _NETINET_IN_H_ */
diff --git a/bsd/netinet/in_arp.c b/bsd/netinet/in_arp.c
index dbefbcc80..d764b3718 100644
--- a/bsd/netinet/in_arp.c
+++ b/bsd/netinet/in_arp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1989, 1993
@@ -79,7 +85,7 @@
 static const size_t MAX_HW_LEN = 10;
 
 SYSCTL_DECL(_net_link_ether);
-SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
+SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "");
 
 /* timer values */
 static int arpt_prune = (5*60*1); /* walk list every 5 minutes */
@@ -115,6 +121,7 @@ static int	arp_inuse, arp_allocated;
 static int	arp_maxtries = 5;
 static int	useloopback = 1; /* use loopback interface for local traffic */
 static int	arp_proxyall = 0;
+static int	arp_sendllconflict = 0;
 
 SYSCTL_INT(_net_link_ether_inet, OID_AUTO, maxtries, CTLFLAG_RW,
 	   &arp_maxtries, 0, "");
@@ -122,6 +129,8 @@ SYSCTL_INT(_net_link_ether_inet, OID_AUTO, useloopback, CTLFLAG_RW,
 	   &useloopback, 0, "");
 SYSCTL_INT(_net_link_ether_inet, OID_AUTO, proxyall, CTLFLAG_RW,
 	   &arp_proxyall, 0, "");
+SYSCTL_INT(_net_link_ether_inet, OID_AUTO, sendllconflict, CTLFLAG_RW,
+	   &arp_sendllconflict, 0, "");
 
 static int log_arp_warnings = 0;
 
@@ -129,6 +138,16 @@ SYSCTL_INT(_net_link_ether_inet, OID_AUTO, log_arp_warnings, CTLFLAG_RW,
 	&log_arp_warnings, 0,
 	"log arp warning messages");
 
+static int keep_announcements = 1;
+SYSCTL_INT(_net_link_ether_inet, OID_AUTO, keep_announcements, CTLFLAG_RW,
+	&keep_announcements, 0,
+	"keep arp announcements");
+
+static int send_conflicting_probes = 1;
+SYSCTL_INT(_net_link_ether_inet, OID_AUTO, send_conflicting_probes, CTLFLAG_RW,
+	&send_conflicting_probes, 0,
+	"send conflicting link-local arp probes");
+
 extern u_int32_t	ipv4_ll_arp_aware;
 
 /*
@@ -159,9 +178,9 @@ arptfree(
  */
 /* ARGSUSED */
 static void
-arptimer(
-	__unused void *ignored_arg)
+arptimer(void *ignored_arg)
 {
+#pragma unused (ignored_arg)
 	struct llinfo_arp *la, *ola;
 	struct timeval timenow;
 
@@ -296,7 +315,7 @@ arp_rtrequest(
 			rt->rt_expire = 0;
 			ifnet_lladdr_copy_bytes(rt->rt_ifp, LLADDR(SDL(gate)), SDL(gate)->sdl_alen = 6);
 			if (useloopback)
-				rt->rt_ifp = loif;
+				rt->rt_ifp = lo_ifp;
 
 		}
 		break;
@@ -306,7 +325,7 @@ arp_rtrequest(
 			break;
 		arp_inuse--;
 		LIST_REMOVE(la, la_le);
-		rt->rt_llinfo = 0;
+		rt->rt_llinfo = NULL;
 		rt->rt_flags &= ~RTF_LLINFO;
 		if (la->la_hold) {
 			m_freem(la->la_hold);
@@ -324,7 +343,7 @@ sdl_addr_to_hex(const struct sockaddr_dl *sdl, char * orig_buf, int buflen)
 {
 	char *		buf = orig_buf;
 	int 		i;
-	const u_char *	lladdr = sdl->sdl_data;
+	const u_char *	lladdr = (u_char *)sdl->sdl_data;
 	int			maxbytes = buflen / 3;
 	
 	if (maxbytes > sdl->sdl_alen) {
@@ -354,7 +373,7 @@ arp_lookup_route(
 	route_t *route)
 {
 	struct sockaddr_inarp sin = {sizeof(sin), AF_INET, 0, {0}, {0}, 0, 0};
-	const char *why = 0;
+	const char *why = NULL;
 	errno_t	error = 0;
 	
 	// Caller is responsible for taking the routing lock
@@ -363,7 +382,7 @@ arp_lookup_route(
 	sin.sin_addr.s_addr = addr->s_addr;
 	sin.sin_other = proxy ? SIN_PROXY : 0;
 	
-	*route = rtalloc1_locked((const struct sockaddr*)&sin, create, 0);
+	*route = rtalloc1_locked((struct sockaddr*)&sin, create, 0);
 	if (*route == NULL)
 		return ENETUNREACH;
 	
@@ -422,9 +441,9 @@ arp_route_to_gateway_route(
 	route_t	hint,
 	route_t *out_route)
 {
+	struct timeval timenow;
 	route_t route = hint;
 	*out_route = NULL;
-	struct timeval timenow;
 	
 	/* If we got a hint from the higher layers, check it out */
 	if (route) {
@@ -592,7 +611,7 @@ arp_lookup_ip(
 				route->rt_flags |= RTF_REJECT;
 				route->rt_rmx.rmx_expire += arpt_down;
 				llinfo->la_asked = 0;
-				llinfo->la_hold = 0;
+				llinfo->la_hold = NULL;
 				lck_mtx_unlock(rt_mtx);
 				return EHOSTUNREACH;
 			}
@@ -618,8 +637,8 @@ arp_ip_handle_input(
 	route_t	route = NULL;
 	char buf[3 * MAX_HW_LEN]; // enough for MAX_HW_LEN byte hw address
 	struct llinfo_arp *llinfo;
-	struct timeval timenow;
 	errno_t	error;
+	int created_announcement = 0;
 	
 	/* Do not respond to requests for 0.0.0.0 */
 	if (target_ip->sin_addr.s_addr == 0 && arpop == ARPOP_REQUEST) {
@@ -689,10 +708,15 @@ arp_ip_handle_input(
 	 */
 	error = arp_lookup_route(&sender_ip->sin_addr, (target_ip->sin_addr.s_addr ==
 				best_ia->ia_addr.sin_addr.s_addr), 0, &route);
-	
 	if (error || route == 0 || route->rt_gateway == 0) {
-		if (ipv4_ll_arp_aware != 0 && IN_LINKLOCAL(target_ip->sin_addr.s_addr)
-			&& arpop == ARPOP_REQUEST && sender_ip->sin_addr.s_addr == 0) {
+		if (arpop != ARPOP_REQUEST) {
+			goto respond;
+		}
+		if (arp_sendllconflict
+		    && send_conflicting_probes != 0
+		    && (ifp->if_eflags & IFEF_ARPLL) != 0 
+		    && IN_LINKLOCAL(ntohl(target_ip->sin_addr.s_addr))
+		    && sender_ip->sin_addr.s_addr == 0) {
 			/*
 			 * Verify this ARP probe doesn't conflict with an IPv4LL we know of
 			 * on another interface.
@@ -700,28 +724,29 @@ arp_ip_handle_input(
 			error = arp_lookup_route(&target_ip->sin_addr, 0, 0, &route);
 			if (error == 0 && route && route->rt_gateway) {
 				gateway = SDL(route->rt_gateway);
-				if (route->rt_ifp != ifp &&
-					(gateway->sdl_alen != sender_hw->sdl_alen ||
-			 		 bcmp(CONST_LLADDR(gateway), CONST_LLADDR(sender_hw),
-			 		 gateway->sdl_alen) != 0)) {
+				if (route->rt_ifp != ifp && gateway->sdl_alen != 0 
+				    && (gateway->sdl_alen != sender_hw->sdl_alen 
+					|| bcmp(CONST_LLADDR(gateway), CONST_LLADDR(sender_hw),
+						gateway->sdl_alen) != 0)) {
 					/*
 					 * A node is probing for an IPv4LL we know exists on a
 					 * different interface. We respond with a conflicting probe
 					 * to force the new device to pick a different IPv4LL
 					 * address.
 					 */
-					log(LOG_INFO,
+					if (log_arp_warnings) {
+					    log(LOG_INFO,
 						"arp: %s on %s%d sent probe for %s, already on %s%d\n",
 						sdl_addr_to_hex(sender_hw, buf, sizeof(buf)),
 						ifp->if_name, ifp->if_unit,
 						inet_ntop(AF_INET, &target_ip->sin_addr, ipv4str,
 								  sizeof(ipv4str)),
 						route->rt_ifp->if_name, route->rt_ifp->if_unit);
-					log(LOG_INFO,
+					    log(LOG_INFO,
 						"arp: sending conflicting probe to %s on %s%d\n",
 						sdl_addr_to_hex(sender_hw, buf, sizeof(buf)),
 						ifp->if_name, ifp->if_unit);
-					
+					}
 					/*
 					 * Send a conservative unicast "ARP probe".
 					 * This should force the other device to pick a new number.
@@ -735,14 +760,28 @@ arp_ip_handle_input(
 						(const struct sockaddr*)target_ip);
 			 	}
 			}
+			goto respond;
+		} else if (keep_announcements != 0
+			   && target_ip->sin_addr.s_addr == sender_ip->sin_addr.s_addr) {
+			/* don't create entry if link-local address and link-local is disabled */
+			if (!IN_LINKLOCAL(ntohl(sender_ip->sin_addr.s_addr)) 
+			    || (ifp->if_eflags & IFEF_ARPLL) != 0) {
+				error = arp_lookup_route(&sender_ip->sin_addr, 1, 0, &route);
+				if (error == 0 && route != NULL && route->rt_gateway != NULL) {
+					created_announcement = 1;
+				}
+			}
+			if (created_announcement == 0) {
+				goto respond;
+			}
+		} else {
+			goto respond;
 		}
-		
-		goto respond;
 	}
 	
 	gateway = SDL(route->rt_gateway);
 	if (route->rt_ifp != ifp) {
-		if (!IN_LINKLOCAL(sender_ip->sin_addr.s_addr) || (ifp->if_eflags & IFEF_ARPLL) == 0) {
+		if (!IN_LINKLOCAL(ntohl(sender_ip->sin_addr.s_addr)) || (ifp->if_eflags & IFEF_ARPLL) == 0) {
 			if (log_arp_warnings)
 				log(LOG_ERR, "arp: %s is on %s%d but got reply from %s on %s%d\n",
 					inet_ntop(AF_INET, &sender_ip->sin_addr, ipv4str,
@@ -776,22 +815,25 @@ arp_ip_handle_input(
 	}
 	
 	if (gateway->sdl_alen && bcmp(LLADDR(gateway), CONST_LLADDR(sender_hw), gateway->sdl_alen)) {
-		if (route->rt_rmx.rmx_expire) {
+		if (route->rt_rmx.rmx_expire && log_arp_warnings) {
 			char buf2[3 * MAX_HW_LEN];
 			log(LOG_INFO, "arp: %s moved from %s to %s on %s%d\n",
-				inet_ntop(AF_INET, &sender_ip->sin_addr, ipv4str,
-						  sizeof(ipv4str)),
-				sdl_addr_to_hex(gateway, buf, sizeof(buf)),
-				sdl_addr_to_hex(sender_hw, buf2, sizeof(buf2)), ifp->if_name,
-				ifp->if_unit);
+			    inet_ntop(AF_INET, &sender_ip->sin_addr, ipv4str,
+			    sizeof(ipv4str)),
+			    sdl_addr_to_hex(gateway, buf, sizeof(buf)),
+			    sdl_addr_to_hex(sender_hw, buf2, sizeof(buf2)),
+			    ifp->if_name, ifp->if_unit);
 		}
-		else {
-			log(LOG_ERR,
-				"arp: %s attempts to modify permanent entry for %s on %s%d\n",
-				sdl_addr_to_hex(sender_hw, buf, sizeof(buf)),
-				inet_ntop(AF_INET, &sender_ip->sin_addr, ipv4str,
-						  sizeof(ipv4str)),
-				ifp->if_name, ifp->if_unit);
+		else if (route->rt_rmx.rmx_expire == 0) {
+			if (log_arp_warnings) {
+				log(LOG_ERR, "arp: %s attempts to modify "
+				    "permanent entry for %s on %s%d\n",
+				    sdl_addr_to_hex(sender_hw, buf,
+				    sizeof(buf)),
+				    inet_ntop(AF_INET, &sender_ip->sin_addr,
+				    ipv4str, sizeof(ipv4str)),
+				    ifp->if_name, ifp->if_unit);
+			}
 			goto respond;
 		}
 	}
@@ -801,9 +843,12 @@ arp_ip_handle_input(
 	bcopy(CONST_LLADDR(sender_hw), LLADDR(gateway), gateway->sdl_alen);
 	
 	/* Update the expire time for the route and clear the reject flag */
-	getmicrotime(&timenow);
-	if (route->rt_rmx.rmx_expire)
+	if (route->rt_rmx.rmx_expire) {
+		struct timeval timenow;
+
+		getmicrotime(&timenow);
 		route->rt_rmx.rmx_expire = timenow.tv_sec + arpt_keep;
+	}
 	route->rt_flags &= ~RTF_REJECT;
 	
 	/* update the llinfo, send a queued packet if there is one */
diff --git a/bsd/netinet/in_arp.h b/bsd/netinet/in_arp.h
index e32f597a7..08b656637 100644
--- a/bsd/netinet/in_arp.h
+++ b/bsd/netinet/in_arp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _NETINET_IN_ARP_H_
diff --git a/bsd/netinet/in_bootp.c b/bsd/netinet/in_bootp.c
deleted file mode 100644
index 903262d4c..000000000
--- a/bsd/netinet/in_bootp.c
+++ /dev/null
@@ -1,612 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *  Copyright (c) 1988-1999 Apple Computer, Inc. All Rights Reserved 
- */
-
-/*
- * bootp.c
- * - be a BOOTP client over a particular interface to retrieve
- *   the IP address, netmask, and router
- */
-
-/* 
- * Modification History
- *
- * February 19, 1999	Dieter Siegmund	(dieter@apple.com)
- * - completely rewritten
- */
-
-#include <sys/param.h>
-#include <sys/types.h>
-#include <mach/boolean.h>
-#include <sys/kernel.h>
-#include <sys/errno.h>
-#include <sys/file.h>
-#include <sys/uio.h>
-#include <sys/ioctl.h>
-#include <sys/time.h>
-#include <sys/mbuf.h>
-#include <sys/vnode.h>
-#include <sys/socket.h>
-#include <sys/socketvar.h>
-#include <sys/uio_internal.h>
-#include <net/if.h>
-#include <net/if_dl.h>
-#include <net/if_types.h>
-#include <net/route.h>
-#include <netinet/in.h>
-#include <netinet/in_systm.h>
-#include <netinet/if_ether.h>
-#include <netinet/ip.h>
-#include <netinet/ip_var.h>
-#include <netinet/udp.h>
-#include <netinet/udp_var.h>
-#include <netinet/ip_icmp.h>
-#include <netinet/bootp.h>
-#include <sys/systm.h>
-#include <sys/malloc.h>
-#include <netinet/dhcp_options.h>
-
-#include <kern/kern_types.h>
-#include <kern/kalloc.h>
-
-#ifdef	BOOTP_DEBUG
-#define	dprintf(x) printf x;
-#else	/* !BOOTP_DEBUG */
-#define	dprintf(x)
-#endif	/* BOOTP_DEBUG */
-
-int bootp(struct ifnet * ifp, struct in_addr * iaddr_p, int max_try,
-	  struct in_addr * netmask_p, struct in_addr * router_p,
-	  struct proc * procp);
-struct mbuf * ip_pkt_to_mbuf(caddr_t pkt, int pktsize);
-int receive_packet(struct socket * so, caddr_t pp, int psize, int * actual_size);
-
-
-/* ip address formatting macros */
-#define IP_FORMAT	"%d.%d.%d.%d"
-#define IP_CH(ip)	((u_char *)ip)
-#define IP_LIST(ip)	IP_CH(ip)[0],IP_CH(ip)[1],IP_CH(ip)[2],IP_CH(ip)[3]
-
-static __inline__ struct sockaddr_in
-blank_sin()
-{
-    struct sockaddr_in	blank = { sizeof(struct sockaddr_in), AF_INET };
-    return (blank);
-}
-
-static __inline__ void
-print_reply(struct bootp *bp, __unused int bp_len)
-{
-	int i, j, len;
-
-	printf("bp_op = ");
-	if (bp->bp_op == BOOTREQUEST) printf("BOOTREQUEST\n");
-	else if (bp->bp_op == BOOTREPLY) printf("BOOTREPLY\n");
-	else
-	{
-		i = bp->bp_op;
-		printf("%d\n", i);
-	}
-
-	i = bp->bp_htype;
-	printf("bp_htype = %d\n", i);
-
-	len = bp->bp_hlen;
-	printf("bp_hlen = %d\n", len);
-
-	i = bp->bp_hops;
-	printf("bp_hops = %d\n", i);
-
-	printf("bp_xid = %lu\n", bp->bp_xid);
-
-	printf("bp_secs = %u\n", bp->bp_secs);
-
-	printf("bp_ciaddr = " IP_FORMAT "\n", IP_LIST(&bp->bp_ciaddr));
-	printf("bp_yiaddr = " IP_FORMAT "\n", IP_LIST(&bp->bp_yiaddr));
-	printf("bp_siaddr = " IP_FORMAT "\n", IP_LIST(&bp->bp_siaddr));
-	printf("bp_giaddr = " IP_FORMAT "\n", IP_LIST(&bp->bp_giaddr));
-
-	printf("bp_chaddr = ");
-	for (j = 0; j < len; j++)
-	{
-		i = bp->bp_chaddr[j];
-		printf("%0x", i);
-		if (j < (len - 1)) printf(":");
-	}
-	printf("\n");
-
-	printf("bp_sname = %s\n", bp->bp_sname);
-	printf("bp_file = %s\n", bp->bp_file);
-}
-
-static __inline__ void
-print_reply_short(struct bootp *bp, __unused int bp_len)
-{
-	printf("bp_yiaddr = " IP_FORMAT "\n", IP_LIST(&bp->bp_yiaddr));
-	printf("bp_sname = %s\n", bp->bp_sname);
-}
-
-
-static __inline__ long
-random_range(long bottom, long top)
-{
-    long number = top - bottom + 1;
-    long range_size = LONG_MAX / number;
-    return (((long)random()) / range_size + bottom);
-}
-
-/*
- * Function: make_bootp_request
- * Purpose:
- *   Create a "blank" bootp packet.
- */
-static void
-make_bootp_request(struct bootp_packet * pkt, 
-		   u_char * hwaddr, u_char hwtype, u_char hwlen)
-{
-    char		rfc_magic[4] = RFC_OPTIONS_MAGIC;
-
-    bzero(pkt, sizeof (*pkt));
-    pkt->bp_ip.ip_v = IPVERSION;
-    pkt->bp_ip.ip_hl = sizeof (struct ip) >> 2;
-#ifdef RANDOM_IP_ID
-    pkt->bp_ip.ip_id = ip_randomid();
-#else
-    pkt->bp_ip.ip_id = htons(ip_id++);
-#endif
-    pkt->bp_ip.ip_ttl = MAXTTL;
-    pkt->bp_ip.ip_p = IPPROTO_UDP;
-    pkt->bp_ip.ip_src.s_addr = 0;
-    pkt->bp_ip.ip_dst.s_addr = htonl(INADDR_BROADCAST);
-    pkt->bp_udp.uh_sport = htons(IPPORT_BOOTPC);
-    pkt->bp_udp.uh_dport = htons(IPPORT_BOOTPS);
-    pkt->bp_udp.uh_sum = 0;
-    pkt->bp_bootp.bp_op = BOOTREQUEST;
-    pkt->bp_bootp.bp_htype = hwtype;
-    pkt->bp_bootp.bp_hlen = hwlen;
-    pkt->bp_bootp.bp_ciaddr.s_addr = 0;
-    bcopy(hwaddr, pkt->bp_bootp.bp_chaddr, hwlen);
-    bcopy(rfc_magic, pkt->bp_bootp.bp_vend, sizeof(rfc_magic));
-    pkt->bp_bootp.bp_vend[4] = dhcptag_end_e;
-    pkt->bp_udp.uh_ulen = htons(sizeof(pkt->bp_udp) + sizeof(pkt->bp_bootp));
-    pkt->bp_ip.ip_len = htons(sizeof(struct ip) + ntohs(pkt->bp_udp.uh_ulen));
-    pkt->bp_ip.ip_sum = 0;
-    return;
-}
-
-/*
- * Function: ip_pkt_to_mbuf
- * Purpose:
- *   Put the given IP packet into an mbuf, calculate the
- *   IP checksum.
- */
-struct mbuf *
-ip_pkt_to_mbuf(caddr_t pkt, int pktsize)
-{
-    struct ip *		ip;
-    struct mbuf	*	m;
-    
-    m = (struct mbuf *)m_devget(pkt, pktsize, 0, 0, 0);
-    if (m == 0) {
-	printf("bootp: ip_pkt_to_mbuf: m_devget failed\n");
-	return 0;
-    }
-    m->m_flags |= M_BCAST;
-    /* Compute the checksum */
-    ip = mtod(m, struct ip *);
-    ip->ip_sum = 0;
-    ip->ip_sum = in_cksum(m, sizeof (struct ip));
-    return (m);
-}
-
-static __inline__ u_char *
-link_address(struct sockaddr_dl * dl_p)
-{
-    return (dl_p->sdl_data + dl_p->sdl_nlen);
-}
-
-static __inline__ void
-link_print(struct sockaddr_dl * dl_p)
-{
-    int i;
-
-#if 0
-    printf("len %d index %d family %d type 0x%x nlen %d alen %d"
-	   " slen %d addr ", dl_p->sdl_len, 
-	   dl_p->sdl_index,  dl_p->sdl_family, dl_p->sdl_type,
-	   dl_p->sdl_nlen, dl_p->sdl_alen, dl_p->sdl_slen);
-#endif
-    for (i = 0; i < dl_p->sdl_alen; i++) 
-	printf("%s%x", i ? ":" : "", 
-	       (link_address(dl_p))[i]);
-    printf("\n");
-    return;
-}
-
-static struct sockaddr_dl *
-link_from_ifnet(struct ifnet * ifp)
-{
-    struct ifaddr * addr;
-
-/*    for (addr = ifp->if_addrlist; addr; addr = addr->ifa_next) */
-
-	ifnet_lock_shared(ifp);
-    TAILQ_FOREACH(addr, &ifp->if_addrhead, ifa_link) {
-	if (addr->ifa_addr->sa_family == AF_LINK) {
-	    struct sockaddr_dl * dl_p = (struct sockaddr_dl *)(addr->ifa_addr);
-	    
-	    ifnet_lock_done(ifp);
-	    return (dl_p);
-	}
-    }
-    ifnet_lock_done(ifp);
-    return (NULL);
-}
-
-/*
- * Function: send_bootp_request
- * Purpose:
- *     Send the request by calling the interface's output routine
- *     bypassing routing code.
- */
-static int
-send_bootp_request(struct ifnet * ifp, __unused struct socket * so,
-		   struct bootp_packet * pkt)
-{
-    struct mbuf	*	m;
-    struct sockaddr_in	sin;
-    
-    /* Address to send to */
-    sin = blank_sin();
-    sin.sin_port = htons(IPPORT_BOOTPS);
-    sin.sin_addr.s_addr = INADDR_BROADCAST;
-    
-    m = ip_pkt_to_mbuf((caddr_t)pkt, sizeof(*pkt));
-    return dlil_output(ifp, PF_INET, m, 0, (struct sockaddr *)&sin, 0);
-}
-
-/*
- * Function: receive_packet
- * Purpose:
- *   Return a received packet or an error if none available.
- */
-int
-receive_packet(struct socket * so, caddr_t pp, int psize, int * actual_size)
-{
-    uio_t		auio;
-    int			rcvflg;
-    int			error;
-	char		uio_buf[ UIO_SIZEOF(1) ];
-
- 	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
-								  &uio_buf[0], sizeof(uio_buf));
-	uio_addiov(auio, CAST_USER_ADDR_T(pp), psize);
-    rcvflg = MSG_WAITALL;
-    
-    error = soreceive(so, (struct sockaddr **) 0, auio, 0, 0, &rcvflg);
-    *actual_size = psize - uio_resid(auio);
-    return (error);
-}
-
-/*
- * Function: bootp_timeout
- * Purpose:
- *   Wakeup the process waiting for something on a socket.
- */
-static void
-bootp_timeout(void * arg)
-{
-    struct socket * * socketflag = (struct socket * *)arg;
-    struct socket * so = *socketflag;
-    
-    dprintf(("bootp: timeout\n"));
-
-    *socketflag = NULL;
-    socket_lock(so, 1);
-    sowakeup(so, &so->so_rcv);
-    socket_unlock(so, 1);
-    return;
-}
-
-/*
- * Function: rate_packet
- * Purpose:
- *   Return an integer point rating value for the given bootp packet.
- *   If yiaddr non-zero, the packet gets a rating of 1.
- *   Another point is given if the packet contains the subnet mask,
- *   and another if the router is present.
- */
-#define GOOD_RATING	3
-static __inline__ int 
-rate_packet(__unused struct bootp * pkt, __unused int pkt_size, dhcpol_t * options_p)
-{
-    int		len;
-    int 	rating = 1;
-
-    if (dhcpol_find(options_p, dhcptag_subnet_mask_e, &len, NULL) != NULL) {
-	rating++;
-    }
-    if (dhcpol_find(options_p, dhcptag_router_e, &len, NULL) != NULL) {
-	rating++;
-    }
-    return (rating);
-}
-
-#define INITIAL_WAIT_SECS		4
-#define MAX_WAIT_SECS			64
-#define GATHER_TIME_SECS		2
-#define RAND_TICKS			(hz)	/* one second */
-
-/*
- * Function: bootp_loop
- * Purpose:
- *   Do the actual BOOTP protocol.
- *   The algorithm sends out a packet, waits for a response.
- *   We try max_try times, waiting in an exponentially increasing
- *   amount of time.  Once we receive a good response, we start
- *   a new time period called the "gather time", during which we
- *   either find the perfect packet (one that has ip, mask and router)
- *   or we continue to gather responses.  At the end of the gather period,
- *   we use the best response gathered.
- */
-static int
-bootp_loop(struct socket * so, struct ifnet * ifp, int max_try,
-	   struct in_addr * iaddr_p, struct in_addr * netmask_p,
-	   struct in_addr * router_p)
-{
-    struct timeval		current_time;
-    struct sockaddr_dl *	dl_p;
-    int				error = 0;
-    char *			hwaddr;
-    int				hwlen;
-    char			hwtype = 0;
-    struct bootp_packet * 	request = NULL;
-    struct bootp *		reply = NULL;
-    int				reply_size = DHCP_PACKET_MIN;
-    struct timeval		start_time;
-    u_long			xid;
-    int				retry;
-    struct socket *		timeflag;
-    int				wait_ticks = INITIAL_WAIT_SECS * hz;
-
-    /* get the hardware address from the interface */
-    dl_p = link_from_ifnet(ifp);
-    if (dl_p == NULL) {
-	printf("bootp: can't get link address\n");
-	return (ENXIO);
-    }
-
-    printf("bootp: h/w addr ");
-    link_print(dl_p);
-
-    hwaddr = link_address(dl_p);
-    hwlen = dl_p->sdl_alen;
-    switch (dl_p->sdl_type) {
-        case IFT_ETHER:
-	    hwtype = ARPHRD_ETHER;
-	    break;
-        default:
-	    printf("bootp: hardware type %d not supported\n",
-		   dl_p->sdl_type);
-	    panic("bootp: hardware type not supported");
-	    break;
-    }
-
-    /* set transaction id and remember the start time */
-    microtime(&start_time);
-    current_time = start_time;
-    xid = random();
-    
-    /* make a request/reply packet */
-    request = (struct bootp_packet *)kalloc(sizeof(*request));
-    make_bootp_request(request, hwaddr, hwtype, hwlen);
-    reply = (struct bootp *)kalloc(reply_size);
-    iaddr_p->s_addr = 0;
-    printf("bootp: sending request");
-    for (retry = 0; retry < max_try; retry++) {
-	int	 	gather_count = 0;
-	int		last_rating = 0;
-
-	/* Send the request */
-	printf(".");
-	request->bp_bootp.bp_secs = htons((u_short)(current_time.tv_sec 
-						    - start_time.tv_sec));
-	request->bp_bootp.bp_xid = htonl(xid);
-	error = send_bootp_request(ifp, so, request);
-	if (error)
-	    goto cleanup;
-
-	timeflag = so;
-	wait_ticks += random_range(-RAND_TICKS, RAND_TICKS);
-	dprintf(("bootp: waiting %d ticks\n", wait_ticks));
-	timeout((timeout_fcn_t)bootp_timeout, &timeflag, wait_ticks);
-
-	while (TRUE) {
-	    int 	n = 0;
-
-	    error = receive_packet(so, (caddr_t)reply, reply_size, &n);
-	    if (error == 0) {
-		dprintf(("\nbootp: received packet\n"));
-		if (ntohl(reply->bp_xid) == xid
-		    && reply->bp_yiaddr.s_addr
-		    && bcmp(reply->bp_chaddr, hwaddr, hwlen) == 0) {
-		    int 		rating;
-		    dhcpol_t		options;
-
-#ifdef	BOOTP_DEBUG
-		    print_reply_short(reply, n);
-#endif /* BOOTP_DEBUG */
-		    (void)dhcpol_parse_packet(&options, (struct dhcp *)reply, 
-					      n, NULL);
-		    rating = rate_packet(reply, n, &options);
-		    if (rating > last_rating) {
-			struct in_addr * 	ip;
-			int			len;
-
-			*iaddr_p = reply->bp_yiaddr;
-			ip = (struct in_addr *)
-			    dhcpol_find(&options, 
-					dhcptag_subnet_mask_e, &len, NULL);
-			if (ip) {
-			    *netmask_p = *ip;
-			}
-			ip = (struct in_addr *)
-			    dhcpol_find(&options, dhcptag_router_e, &len, NULL);
-			if (ip) {
-			    *router_p = *ip;
-			}
-			printf("%sbootp: got "
-			       "response from %s (" IP_FORMAT ")\n",
-			       last_rating == 0 ? "\n" : "",
-			       reply->bp_sname, 
-			       IP_LIST(&reply->bp_siaddr));
-		    }
-		    dhcpol_free(&options);
-		    if (rating >= GOOD_RATING) {
-			untimeout((timeout_fcn_t)bootp_timeout, &timeflag);
-			goto done;
-		    }
-		    if (gather_count == 0) {
-			untimeout((timeout_fcn_t)bootp_timeout, &timeflag);
-			timeflag = so;
-			timeout((timeout_fcn_t)bootp_timeout, &timeflag, 
-				hz * GATHER_TIME_SECS);
-		    }
-		    gather_count++;
-		}
-		else {
-		    dprintf(("bootp: packet ignored\n"));
-		}
-	    }
-	    else if ((error != EWOULDBLOCK)) {
-		break;
-	    }
-	    else if (timeflag == NULL) { /* timed out */
-		if (gather_count) {
-		    dprintf(("bootp: gathering time has expired"));
-		    goto done; /* we have a packet */
-		}
-		break; /* retry */
-	    }
-	    else {
-		socket_lock(so, 1);
-		error = sbwait(&so->so_rcv);
-		socket_unlock(so, 1);
-	    }
-	}
-	if (error && (error != EWOULDBLOCK)) {
-	    dprintf(("bootp: failed to receive packets: %d\n", error));
-	    untimeout((timeout_fcn_t)bootp_timeout, &timeflag);
-	    goto cleanup;
-	}
-	wait_ticks *= 2;
-	if (wait_ticks > (MAX_WAIT_SECS * hz))
-	    wait_ticks = MAX_WAIT_SECS * hz;
-	xid++;
-	microtime(&current_time);
-    }
-    error = ETIMEDOUT;
-    goto cleanup;
-
- done:
-    error = 0;
-
- cleanup:
-    if (request)
-	kfree(request, sizeof (*request));
-    if (reply)
-	kfree(reply, reply_size);
-    return (error);
-}
-
-/*
- * Routine: bootp
- * Function:
- *	Use the BOOTP protocol to resolve what our IP address should be
- *	on a particular interface.
- */
-int bootp(struct ifnet * ifp, struct in_addr * iaddr_p, int max_try,
-	  struct in_addr * netmask_p, struct in_addr * router_p,
-	  struct proc * procp)
-{
-    boolean_t			addr_set = FALSE;
-    struct ifreq 		ifr;
-    int				error;
-    struct socket *		so = NULL;
-
-    /* get a socket */
-    error = socreate(AF_INET, &so, SOCK_DGRAM, 0);
-    if (error) {
-	dprintf(("bootp: socreate failed %d\n", error));
-	return (error);
-    }
-
-    /* assign the all-zeroes address */
-    bzero(&ifr, sizeof(ifr));
-    sprintf(ifr.ifr_name, "%s%d", ifp->if_name, ifp->if_unit);
-    *((struct sockaddr_in *)&ifr.ifr_addr) = blank_sin();
-    error = ifioctl(so, SIOCSIFADDR, (caddr_t)&ifr, procp);
-    if (error) {
-	dprintf(("bootp: SIOCSIFADDR all-zeroes IP failed: %d\n",
-		 error));
-	goto cleanup;
-    }
-    dprintf(("bootp: all-zeroes IP address assigned\n"));
-    addr_set = TRUE;
-    
-    { /* bind the socket */
-	struct sockaddr_in * sin;
-
-	sin = _MALLOC(sizeof(struct sockaddr_in), M_IFADDR, M_WAIT);
-	if (sin == NULL) {
-	  	error = ENOMEM;
-		goto cleanup;
-	}
-	sin->sin_len = sizeof(struct sockaddr_in);
-	sin->sin_family = AF_INET;
-	sin->sin_port = htons(IPPORT_BOOTPC);
-	sin->sin_addr.s_addr = INADDR_ANY;
-	error = sobind(so, (struct sockaddr *) sin);
-
-	FREE(sin, M_IFADDR);
-	if (error) {
-	    dprintf(("bootp: sobind failed, %d\n", error));
-	    goto cleanup;
-	}
-	socket_lock(so, 1);
-	so->so_state |= SS_NBIO;
-	socket_unlock(so, 1);
-    }
-    /* do the protocol */
-    error = bootp_loop(so, ifp, max_try, iaddr_p, netmask_p, router_p);
-    
- cleanup:
-    if (so) {
-	if (addr_set) {
-	    (void) ifioctl(so, SIOCDIFADDR, (caddr_t) &ifr, procp);
-	}
-	soclose(so);
-    }
-    return (error);
-}
diff --git a/bsd/netinet/in_cksum.c b/bsd/netinet/in_cksum.c
index 9d349d7b5..28601114c 100644
--- a/bsd/netinet/in_cksum.c
+++ b/bsd/netinet/in_cksum.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1992, 1993
@@ -57,6 +63,9 @@
 #include <sys/param.h>
 #include <sys/mbuf.h>
 #include <sys/kdebug.h>
+#include <kern/debug.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
 
 #define DBG_FNC_IN_CKSUM	NETDBG_CODE(DBG_NETIP, (3 << 8))
 
@@ -74,14 +83,14 @@ union s_util {
 
 union l_util {
         u_int16_t s[2];
-        u_int32_t l;   
+        u_int32_t l;
 };
 
 union q_util {
         u_int16_t s[4];
         u_int32_t l[2];
         u_int64_t q;
-};    
+};
 
 #define ADDCARRY(x)  (x > 65535 ? x -= 65535 : x)
 
@@ -100,24 +109,34 @@ union q_util {
 
 #define REDUCE {l_util.l = sum; sum = l_util.s[0] + l_util.s[1]; ADDCARRY(sum);}
 
-                
-#if defined(ppc)
+u_int16_t inet_cksum_simple(struct mbuf *, int);
+
+u_int16_t
+inet_cksum_simple(struct mbuf *m, int len)
+{
+	return (inet_cksum(m, 0, 0, len));
+}
 
-__inline unsigned short
+#if defined(__ppc__)
+
+extern u_short xsum_assym(u_short *p, int len, u_short xsum, int odd);
+
+inline u_short
 in_addword(u_short a, u_short b)
 {
-        union l_util l_util;   
+        union l_util l_util;
 	u_int32_t sum = a + b;
+
 	REDUCE;
 	return (sum);
 }
 
-__inline unsigned short
+inline u_short
 in_pseudo(u_int a, u_int b, u_int c)
 {
         u_int64_t sum;
         union q_util q_util;
-        union l_util l_util;   
+        union l_util l_util;
 
         sum = (u_int64_t) a + b + c;
         REDUCE16;
@@ -125,64 +144,47 @@ in_pseudo(u_int a, u_int b, u_int c)
 
 }
 
-int
-in_cksum(m, len)
-	register struct mbuf *m;
-	register int len;
+u_int16_t
+inet_cksum(struct mbuf *m, unsigned int nxt, unsigned int skip,
+    unsigned int len)
 {
-	register u_short *w;
-	register int sum = 0;
-	register int mlen = 0;
+	u_short *w;
+	u_int32_t sum = 0;
+	int mlen = 0;
 	int starting_on_odd  = 0;
 
-
 	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_START, len,0,0,0,0);
 
-	for (;m && len; m = m->m_next) {
-		if (m->m_len == 0)
-			continue;
-		mlen = m->m_len;
-		w = mtod(m, u_short *);
+	/* sanity check */
+	if (m->m_pkthdr.len < skip + len) {
+		panic("inet_cksum: mbuf len (%d) < off+len (%d+%d)\n",
+		    m->m_pkthdr.len, skip, len);
+	}
 
-		if (len < mlen)
-			mlen = len;
+	/* include pseudo header checksum? */
+	if (nxt != 0) {
+		struct ip *iph;
 
-		sum = xsum_assym(w, mlen, sum, starting_on_odd);
-		len -= mlen;
-		if (mlen & 0x1)
-		{
-		    if (starting_on_odd)
-			starting_on_odd = 0;
-		    else
-			starting_on_odd = 1;
-		}
-	}
+		if (m->m_len < sizeof (struct ip))
+			panic("inet_cksum: bad mbuf chain");
 
-	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_END, 0,0,0,0,0);
-	return (~sum & 0xffff);
-}
+		iph = mtod(m, struct ip *);
+		sum = in_pseudo(iph->ip_src.s_addr, iph->ip_dst.s_addr,
+		    htonl(len + nxt));
+	}
 
-u_short
-in_cksum_skip(m, len, skip)
-        register struct mbuf *m;
-        register int len;
-        register int skip;
-{
-	register u_short *w;
-	register int sum = 0;
-	register int mlen = 0;
-	int starting_on_odd  = 0;
+	if (skip != 0) {
+		for (; skip && m; m = m->m_next) {
+			if (m->m_len > skip) {
+				mlen = m->m_len - skip;
+				w = (u_short *)(m->m_data+skip);
+				goto skip_start;
+			} else {
+				skip -= m->m_len;
+			}
+		}
+	}
 
-	len -= skip;
-        for (; skip && m; m = m->m_next) {
-                if (m->m_len > skip) {
-                        mlen = m->m_len - skip;
-			w = (u_short *)(m->m_data+skip);
-                        goto skip_start;
-                } else {    
-                        skip -= m->m_len;
-                }
-        }
 	for (;m && len; m = m->m_next) {
 		if (m->m_len == 0)
 			continue;
@@ -203,155 +205,77 @@ skip_start:
 		}
 	}
 
+	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_END, 0,0,0,0,0);
+
 	return (~sum & 0xffff);
 }
+
 #else
 
-u_short 
+inline u_short
 in_addword(u_short a, u_short b)
-{       
-        union l_util l_util;   
+{
+        union l_util l_util;
         u_int32_t sum = a + b;
+
         REDUCE(sum);
         return (sum);
-}       
+}
 
-u_short
+inline u_short
 in_pseudo(u_int a, u_int b, u_int c)
 {
-        u_int64_t sum;  
+        u_int64_t sum;
         union q_util q_util;
-        union l_util l_util;   
+        union l_util l_util;
 
         sum = (u_int64_t) a + b + c;
         REDUCE16;
         return (sum);
 }
 
-
-int
-in_cksum(m, len)
-	register struct mbuf *m;
-	register int len;
+u_int16_t
+inet_cksum(struct mbuf *m, unsigned int nxt, unsigned int skip,
+    unsigned int len)
 {
-	register u_short *w;
-	register int sum = 0;
-	register int mlen = 0;
+	u_short *w;
+	u_int32_t sum = 0;
+	int mlen = 0;
 	int byte_swapped = 0;
 	union s_util s_util;
-	union l_util l_util;   
+	union l_util l_util;
 
 	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_START, len,0,0,0,0);
 
-	for (;m && len; m = m->m_next) {
-		if (m->m_len == 0)
-			continue;
-		w = mtod(m, u_short *);
-		if (mlen == -1) {
-			/*
-			 * The first byte of this mbuf is the continuation
-			 * of a word spanning between this mbuf and the
-			 * last mbuf.
-			 *
-			 * s_util.c[0] is already saved when scanning previous
-			 * mbuf.
-			 */
-			s_util.c[1] = *(char *)w;
-			sum += s_util.s;
-			w = (u_short *)((char *)w + 1);
-			mlen = m->m_len - 1;
-			len--;
-		} else
-			mlen = m->m_len;
-		if (len < mlen)
-			mlen = len;
-		len -= mlen;
-		/*
-		 * Force to even boundary.
-		 */
-		if ((1 & (int) w) && (mlen > 0)) {
-			REDUCE;
-			sum <<= 8;
-			s_util.c[0] = *(u_char *)w;
-			w = (u_short *)((char *)w + 1);
-			mlen--;
-			byte_swapped = 1;
-		}
-		/*
-		 * Unroll the loop to make overhead from
-		 * branches &c small.
-		 */
-		while ((mlen -= 32) >= 0) {
-			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
-			sum += w[4]; sum += w[5]; sum += w[6]; sum += w[7];
-			sum += w[8]; sum += w[9]; sum += w[10]; sum += w[11];
-			sum += w[12]; sum += w[13]; sum += w[14]; sum += w[15];
-			w += 16;
-		}
-		mlen += 32;
-		while ((mlen -= 8) >= 0) {
-			sum += w[0]; sum += w[1]; sum += w[2]; sum += w[3];
-			w += 4;
-		}
-		mlen += 8;
-		if (mlen == 0 && byte_swapped == 0)
-			continue;
-		REDUCE;
-		while ((mlen -= 2) >= 0) {
-			sum += *w++;
-		}
-		if (byte_swapped) {
-			REDUCE;
-			sum <<= 8;
-			byte_swapped = 0;
-			if (mlen == -1) {
-				s_util.c[1] = *(char *)w;
-				sum += s_util.s;
-				mlen = 0;
-			} else
-				mlen = -1;
-		} else if (mlen == -1)
-			s_util.c[0] = *(char *)w;
+	/* sanity check */
+	if (m->m_pkthdr.len < skip + len) {
+		panic("inet_cksum: mbuf len (%d) < off+len (%d+%d)\n",
+		    m->m_pkthdr.len, skip, len);
 	}
-	if (len)
-		printf("cksum: out of data\n");
-	if (mlen == -1) {
-		/* The last mbuf has odd # of bytes. Follow the
-		   standard (the odd byte may be shifted left by 8 bits
-		   or not as determined by endian-ness of the machine) */
-		s_util.c[1] = 0;
-		sum += s_util.s;
-	}
-	REDUCE;
-	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_END, 0,0,0,0,0);
-	return (~sum & 0xffff);
-}
 
-int
-in_cksum_skip(m, len, skip)
-	register struct mbuf *m;
-	register u_short len;
-	register u_short skip;
-{
-	register u_short *w;
-	register int sum = 0;
-	register int mlen = 0;
-	int byte_swapped = 0;
-	union s_util s_util;
-	union l_util l_util;   
+	/* include pseudo header checksum? */
+	if (nxt != 0) {
+		struct ip *iph;
 
-	KERNEL_DEBUG(DBG_FNC_IN_CKSUM | DBG_FUNC_START, len,0,0,0,0);
+		if (m->m_len < sizeof (struct ip))
+			panic("inet_cksum: bad mbuf chain");
+
+		iph = mtod(m, struct ip *);
+		sum = in_pseudo(iph->ip_src.s_addr, iph->ip_dst.s_addr,
+		    htonl(len + nxt));
+	}
 
-	len -= skip;
-        for (; skip && m; m = m->m_next) {
-                if (m->m_len > skip) {
-                        mlen = m->m_len - skip;
-			w = (u_short *)(m->m_data+skip);
-                        goto skip_start;
-                } else {    
-                        skip -= m->m_len;
-                }
-        }
+	if (skip != 0) {
+		for (; skip && m; m = m->m_next) {
+			if (m->m_len > skip) {
+				mlen = m->m_len - skip;
+				w = (u_short *)(m->m_data+skip);
+				goto skip_start;
+			} else {
+				skip -= m->m_len;
+			}
+		}
+	}
 	for (;m && len; m = m->m_next) {
 		if (m->m_len == 0)
 			continue;
@@ -372,11 +296,11 @@ in_cksum_skip(m, len, skip)
 			mlen = m->m_len - 1;
 			len--;
 		} else {
-		  mlen = m->m_len;
+			mlen = m->m_len;
 		}
 skip_start:
 		if (len < mlen)
-		    mlen = len;
+			mlen = len;
 
 		len -= mlen;
 		/*
@@ -427,7 +351,7 @@ skip_start:
 			s_util.c[0] = *(char *)w;
 	}
 	if (len)
-		printf("cksum: out of data\n");
+		printf("cksum: out of data by %d\n", len);
 	if (mlen == -1) {
 		/* The last mbuf has odd # of bytes. Follow the
 		   standard (the odd byte may be shifted left by 8 bits
diff --git a/bsd/netinet/in_dhcp.c b/bsd/netinet/in_dhcp.c
new file mode 100644
index 000000000..c6fdffdd8
--- /dev/null
+++ b/bsd/netinet/in_dhcp.c
@@ -0,0 +1,962 @@
+/*
+ * Copyright (c) 1988-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * in_dhcp.c
+ * - use DHCP to allocate an IP address and get the subnet mask and router
+ */
+
+/* 
+ * Modification History
+ *
+ * April 17, 2007	Dieter Siegmund	(dieter@apple.com)
+ * - created based on in_bootp.c
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <mach/boolean.h>
+#include <sys/kernel.h>
+#include <sys/errno.h>
+#include <sys/file.h>
+#include <sys/uio.h>
+#include <sys/ioctl.h>
+#include <sys/time.h>
+#include <sys/mbuf.h>
+#include <sys/vnode.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/uio_internal.h>
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <net/dlil.h>
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip_var.h>
+#include <netinet/udp.h>
+#include <netinet/udp_var.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/bootp.h>
+#include <netinet/dhcp.h>
+#include <netinet/in_dhcp.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <netinet/dhcp_options.h>
+
+#include <kern/kern_types.h>
+#include <kern/kalloc.h>
+
+#ifdef	DHCP_DEBUG
+#define	dprintf(x) printf x;
+#else	/* !DHCP_DEBUG */
+#define	dprintf(x)
+#endif	/* DHCP_DEBUG */
+
+#define INITIAL_WAIT_SECS		2
+#define MAX_WAIT_SECS			64
+#define GATHER_TIME_SECS		4
+#define RAND_TICKS			(hz)	/* one second */
+
+const struct sockaddr_in blank_sin = {
+    sizeof(struct sockaddr_in), 
+    AF_INET, 
+    0, 
+    { 0 },
+    { 0, 0, 0, 0, 0, 0, 0, 0 }
+};
+
+__private_extern__ int
+inet_aifaddr(struct socket * so, const char * name,
+	     const struct in_addr * addr, 
+	     const struct in_addr * mask,
+	     const struct in_addr * broadcast)
+{
+    struct ifaliasreq	ifra;
+
+    bzero(&ifra, sizeof(ifra));
+    strlcpy(ifra.ifra_name, name, sizeof(ifra.ifra_name));
+    if (addr) {
+	*((struct sockaddr_in *)&ifra.ifra_addr) = blank_sin;
+	((struct sockaddr_in *)&ifra.ifra_addr)->sin_addr = *addr;
+    }
+    if (mask) {
+	*((struct sockaddr_in *)&ifra.ifra_mask) = blank_sin;
+	((struct sockaddr_in *)&ifra.ifra_mask)->sin_addr = *mask;
+    }
+    if (broadcast) {
+	*((struct sockaddr_in *)&ifra.ifra_broadaddr) = blank_sin;
+	((struct sockaddr_in *)&ifra.ifra_broadaddr)->sin_addr = *broadcast;
+    }
+    return (ifioctl(so, SIOCAIFADDR, (caddr_t)&ifra, current_proc()));
+}
+
+
+struct dhcp_context {
+    struct ifnet *		ifp;
+    struct sockaddr_dl *	dl_p;
+    struct ifreq 		ifr;
+    struct socket *		so;
+    uint8_t			request[DHCP_PACKET_MIN];
+    dhcpoa_t			request_options;
+    uint8_t			reply[DHCP_PAYLOAD_MIN];
+    struct timeval		start_time;
+    uint32_t			xid;
+    int				max_try;
+    struct in_addr  		iaddr;
+    struct in_addr  		netmask;
+    struct in_addr 		router;
+    struct in_addr		server_id;
+};
+
+static __inline__ struct dhcp_packet *
+dhcp_context_request(struct dhcp_context * context)
+{
+    return ((struct dhcp_packet *)context->request);
+}
+
+static __inline__ struct dhcp *
+dhcp_context_reply(struct dhcp_context * context)
+{
+    return ((struct dhcp *)context->reply);
+}
+
+struct mbuf * ip_pkt_to_mbuf(caddr_t pkt, int pktsize);
+
+static int
+receive_packet(struct socket * so, void * pp, int psize,
+	       int * actual_size);
+
+/* ip address formatting macros */
+#define IP_FORMAT	"%d.%d.%d.%d"
+#define IP_CH(ip)	((const uint8_t *)ip)
+#define IP_LIST(ip)	IP_CH(ip)[0],IP_CH(ip)[1],IP_CH(ip)[2],IP_CH(ip)[3]
+
+#define SUGGESTED_LEASE_LENGTH		(60 * 60 * 24 * 30 * 3) /* 3 months */
+
+static const uint8_t dhcp_params[] = {
+    dhcptag_subnet_mask_e, 
+    dhcptag_router_e,
+};
+
+#define	N_DHCP_PARAMS 	(sizeof(dhcp_params) / sizeof(dhcp_params[0]))
+
+static __inline__ long
+random_range(long bottom, long top)
+{
+    long number = top - bottom + 1;
+    long range_size = LONG_MAX / number;
+    return (((long)random()) / range_size + bottom);
+}
+
+static void
+init_dhcp_packet_header(struct dhcp_packet * pkt, int pkt_size)
+{
+    bzero(&pkt->ip, sizeof(pkt->ip));
+    bzero(&pkt->udp, sizeof(pkt->udp));
+    pkt->ip.ip_v = IPVERSION;
+    pkt->ip.ip_hl = sizeof(struct ip) >> 2;
+    pkt->ip.ip_ttl = MAXTTL;
+    pkt->ip.ip_p = IPPROTO_UDP;
+    pkt->ip.ip_src.s_addr = 0;
+    pkt->ip.ip_dst.s_addr = htonl(INADDR_BROADCAST);
+    pkt->ip.ip_len = htons(pkt_size);
+    pkt->ip.ip_sum = 0;
+    pkt->udp.uh_sport = htons(IPPORT_BOOTPC);
+    pkt->udp.uh_dport = htons(IPPORT_BOOTPS);
+    pkt->udp.uh_sum = 0;
+    pkt->udp.uh_ulen = htons(pkt_size - sizeof(pkt->ip));
+    return;
+}
+
+/*
+ * Function: make_dhcp_request
+ * Purpose:
+ *   Initialize the DHCP-specific parts of the message.
+ */
+static void
+make_dhcp_request(struct dhcp * request, int request_size,
+		  dhcp_msgtype_t msg, 
+		  const uint8_t * hwaddr, uint8_t hwtype, int hwlen,
+		  dhcpoa_t * options_p)
+{
+    uint8_t		cid[ETHER_ADDR_LEN + 1];
+    uint8_t		rfc_magic[RFC_MAGIC_SIZE] = RFC_OPTIONS_MAGIC;
+
+    if (hwlen > (int)sizeof(cid)) {
+	printf("dhcp: hwlen is %d (> %d), truncating\n", hwlen,
+	       (int)sizeof(cid));
+	hwlen = sizeof(cid);
+    }
+    bzero(request, request_size);
+    request->dp_op = BOOTREQUEST;
+    request->dp_htype = hwtype;
+    request->dp_hlen = hwlen;
+    bcopy(hwaddr, request->dp_chaddr, hwlen);
+    bcopy(rfc_magic, request->dp_options, RFC_MAGIC_SIZE);
+    dhcpoa_init(options_p, request->dp_options + RFC_MAGIC_SIZE,
+		request_size - sizeof(struct dhcp) - RFC_MAGIC_SIZE);
+    /* make the request a dhcp packet */
+    dhcpoa_add_dhcpmsg(options_p, msg);
+
+    /* add the list of required parameters */
+    dhcpoa_add(options_p, dhcptag_parameter_request_list_e,
+	       N_DHCP_PARAMS, dhcp_params);
+
+    /* add the DHCP client identifier */
+    cid[0] = hwtype;
+    bcopy(hwaddr, cid + 1, hwlen);
+    dhcpoa_add(options_p, dhcptag_client_identifier_e, hwlen + 1, cid);
+
+    return;
+}
+
+/*
+ * Function: ip_pkt_to_mbuf
+ * Purpose:
+ *   Put the given IP packet into an mbuf, calculate the
+ *   IP checksum.
+ */
+struct mbuf *
+ip_pkt_to_mbuf(caddr_t pkt, int pktsize)
+{
+    struct ip *		ip;
+    struct mbuf	*	m;
+    
+    m = (struct mbuf *)m_devget(pkt, pktsize, 0, NULL, NULL);
+    if (m == 0) {
+	printf("dhcp: ip_pkt_to_mbuf: m_devget failed\n");
+	return NULL;
+    }
+    m->m_flags |= M_BCAST;
+    /* Compute the checksum */
+    ip = mtod(m, struct ip *);
+    ip->ip_sum = 0;
+    ip->ip_sum = in_cksum(m, sizeof(struct ip));
+    return (m);
+}
+
+static __inline__ u_char *
+link_address(struct sockaddr_dl * dl_p)
+{
+    return (u_char *)(dl_p->sdl_data + dl_p->sdl_nlen);
+}
+
+static __inline__ int
+link_address_length(struct sockaddr_dl * dl_p)
+{
+    return (dl_p->sdl_alen);
+}
+
+static __inline__ void
+link_print(struct sockaddr_dl * dl_p)
+{
+    int i;
+
+#if 0
+    printf("len %d index %d family %d type 0x%x nlen %d alen %d"
+	   " slen %d addr ", dl_p->sdl_len, 
+	   dl_p->sdl_index,  dl_p->sdl_family, dl_p->sdl_type,
+	   dl_p->sdl_nlen, dl_p->sdl_alen, dl_p->sdl_slen);
+#endif
+    for (i = 0; i < dl_p->sdl_alen; i++) 
+	printf("%s%x", i ? ":" : "", 
+	       (link_address(dl_p))[i]);
+    printf("\n");
+    return;
+}
+
+static struct sockaddr_dl *
+link_from_ifnet(struct ifnet * ifp)
+{
+    struct ifaddr * addr;
+
+    ifnet_lock_shared(ifp);
+    TAILQ_FOREACH(addr, &ifp->if_addrhead, ifa_link) {
+	if (addr->ifa_addr->sa_family == AF_LINK) {
+	    struct sockaddr_dl * dl_p = (struct sockaddr_dl *)(addr->ifa_addr);
+	    
+	    ifnet_lock_done(ifp);
+	    return (dl_p);
+	}
+    }
+    ifnet_lock_done(ifp);
+    return (NULL);
+}
+
+/*
+ * Function: send_packet
+ * Purpose:
+ *     Send the request directly on the interface, bypassing the routing code.
+ */
+static int
+send_packet(struct ifnet * ifp, struct dhcp_packet * pkt, int pkt_size)
+{
+    struct mbuf	*	m;
+    struct sockaddr_in	dest;
+    
+    dest = blank_sin;
+    dest.sin_port = htons(IPPORT_BOOTPS);
+    dest.sin_addr.s_addr = INADDR_BROADCAST;
+    m = ip_pkt_to_mbuf((caddr_t)pkt, pkt_size);
+    return dlil_output(ifp, PF_INET, m, 0, (struct sockaddr *)&dest, 0);
+}
+
+/*
+ * Function: receive_packet
+ * Purpose:
+ *   Return a received packet or an error if none available.
+ */
+static int
+receive_packet(struct socket * so, void * pp, int psize, int * actual_size)
+{
+    uio_t	auio;
+    int		error;
+    int		rcvflg;
+    char	uio_buf[ UIO_SIZEOF(1) ];
+
+    auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
+				&uio_buf[0], sizeof(uio_buf));
+    uio_addiov(auio, CAST_USER_ADDR_T(pp), psize);
+    rcvflg = MSG_WAITALL;
+    
+    error = soreceive(so, (struct sockaddr **) 0, auio, 0, 0, &rcvflg);
+    *actual_size = psize - uio_resid(auio);
+    return (error);
+}
+
+/*
+ * Function: dhcp_timeout
+ * Purpose:
+ *   Wakeup the process waiting for something on a socket.
+ */
+static void
+dhcp_timeout(void * arg)
+{
+    struct socket * * timer_arg = (struct socket * *)arg;
+    struct socket * so = *timer_arg;
+    
+    dprintf(("dhcp: timeout\n"));
+
+    *timer_arg = NULL;
+    socket_lock(so, 1);
+    sowakeup(so, &so->so_rcv);
+    socket_unlock(so, 1);
+    return;
+}
+
+/*
+ * Function: rate_packet
+ * Purpose:
+ *   Return an integer point rating value for the given dhcp packet.
+ *   If yiaddr non-zero, the packet gets a rating of 1.
+ *   Another point is given if the packet contains the subnet mask,
+ *   and another if the router is present.
+ */
+#define GOOD_RATING	3
+static __inline__ int 
+rate_packet(dhcpol_t * options_p)
+{
+    int		len;
+    int 	rating = 1;
+
+    if (dhcpol_find(options_p, dhcptag_subnet_mask_e, &len, NULL) != NULL) {
+	rating++;
+    }
+    if (dhcpol_find(options_p, dhcptag_router_e, &len, NULL) != NULL) {
+	rating++;
+    }
+    return (rating);
+}
+
+static dhcp_msgtype_t
+get_dhcp_msgtype(dhcpol_t * options_p)
+{
+    int				len;
+    const uint8_t * 		opt;
+
+    opt = dhcpol_find(options_p, dhcptag_dhcp_message_type_e, &len, NULL);
+    if (opt != NULL && len == 1) {
+	return (*opt);
+    }
+    return (dhcp_msgtype_none_e);
+}
+
+static int
+dhcp_get_ack(struct dhcp_context * context, int wait_ticks)
+{
+    int				error = 0;
+    const struct in_addr * 	ip;
+    int				len;
+    int				n;
+    struct dhcp *		reply;
+    struct in_addr		server_id;
+    struct socket * 		timer_arg;
+
+    timer_arg = context->so;
+    reply = dhcp_context_reply(context);
+    timeout((timeout_fcn_t)dhcp_timeout, &timer_arg, wait_ticks);
+    while (1) {
+	error = receive_packet(context->so, context->reply,
+			       sizeof(context->reply), &n);
+	if (error == 0) {
+	    dhcp_msgtype_t	msg;
+	    dhcpol_t		options;
+
+	    dprintf(("\ndhcp: received packet length %d\n", n));
+	    if (n < (int)sizeof(struct dhcp)) {
+		dprintf(("dhcp: packet is too short %d < %d\n",
+			 n, (int)sizeof(struct dhcp)));
+		continue;
+	    }
+	    if (ntohl(reply->dp_xid) != context->xid
+		|| bcmp(reply->dp_chaddr, link_address(context->dl_p), 
+			link_address_length(context->dl_p)) != 0) {
+		/* not for us */
+		continue;
+	    }
+	    (void)dhcpol_parse_packet(&options, reply, n);
+	    server_id.s_addr = 0;
+	    ip = (const struct in_addr *)
+		dhcpol_find(&options, 
+			    dhcptag_server_identifier_e, &len, NULL);
+	    if (ip != NULL && len >= (int)sizeof(*ip)) {
+		server_id = *ip;
+	    }
+	    msg = get_dhcp_msgtype(&options);
+	    if (msg == dhcp_msgtype_nak_e
+		&& server_id.s_addr == context->server_id.s_addr) {
+		/* server NAK'd us, start over */
+		dhcpol_free(&options);
+		error = EPROTO;
+		untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+		break;
+	    }
+	    if (msg != dhcp_msgtype_ack_e
+		|| reply->dp_yiaddr.s_addr == 0
+		|| reply->dp_yiaddr.s_addr == INADDR_BROADCAST) {
+		/* ignore the packet */
+		goto next_packet;
+	    }
+	    printf("dhcp: received ACK: server " IP_FORMAT
+		   " IP address "  IP_FORMAT "\n",
+		   IP_LIST(&server_id), IP_LIST(&reply->dp_yiaddr));
+	    context->iaddr = reply->dp_yiaddr;
+	    ip = (const struct in_addr *)
+		dhcpol_find(&options, 
+			    dhcptag_subnet_mask_e, &len, NULL);
+	    if (ip != NULL && len >= (int)sizeof(*ip)) {
+		context->netmask = *ip;
+	    }
+	    ip = (const struct in_addr *)
+		dhcpol_find(&options, dhcptag_router_e, &len, NULL);
+	    if (ip != NULL && len >= (int)sizeof(*ip)) {
+		context->router = *ip;
+	    }
+	    dhcpol_free(&options);
+	    untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+	    break;
+
+	next_packet:
+	    dhcpol_free(&options);
+	}
+	else if ((error != EWOULDBLOCK)) {
+	    /* if some other error occurred, we're done */
+	    untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+	    break;
+	}
+	else if (timer_arg == NULL) { 
+	    /* timed out */
+	    break;
+	}
+	else {
+	    /* wait for a wait to arrive, or a timeout to occur */
+	    socket_lock(context->so, 1);
+	    error = sbwait(&context->so->so_rcv);
+	    socket_unlock(context->so, 1);
+	}
+    }
+    return (error);
+}
+
+static int
+dhcp_select(struct dhcp_context * context)
+{
+    struct timeval		current_time;
+    int				error = 0;
+    dhcpoa_t *			options_p;
+    struct dhcp_packet *	request;
+    int				request_size;
+    int				retry;
+    int				wait_ticks;
+
+    /* format a DHCP Request packet */
+    request = dhcp_context_request(context);
+    options_p = &context->request_options;
+
+    make_dhcp_request(&request->dhcp, DHCP_PAYLOAD_MIN,
+		      dhcp_msgtype_request_e,
+		      link_address(context->dl_p), ARPHRD_ETHER,
+		      link_address_length(context->dl_p),
+		      options_p);
+    /* insert server identifier and requested ip address */
+    dhcpoa_add(options_p, dhcptag_requested_ip_address_e,
+	       sizeof(context->iaddr), &context->iaddr);
+    dhcpoa_add(options_p, dhcptag_server_identifier_e,
+	       sizeof(context->server_id), &context->server_id);
+    dhcpoa_add(options_p, dhcptag_end_e, 0, 0);
+    request_size = sizeof(*request) + RFC_MAGIC_SIZE 
+	+ dhcpoa_used(options_p);
+    if (request_size < (int)sizeof(struct bootp_packet)) {
+	/* pad out to BOOTP-sized packet */
+	request_size = sizeof(struct bootp_packet);
+    }
+    init_dhcp_packet_header(request, request_size);
+
+    wait_ticks = INITIAL_WAIT_SECS * hz;
+#define SELECT_RETRY_COUNT	3
+    for (retry = 0; retry < SELECT_RETRY_COUNT; retry++) {
+	/* Send the request */
+	printf("dhcp: sending REQUEST: server " IP_FORMAT 
+	       " IP address " IP_FORMAT "\n",
+	       IP_LIST(&context->server_id),
+	       IP_LIST(&context->iaddr));
+	microtime(&current_time);
+	request->dhcp.dp_secs 
+	    = htons((u_short)
+		    (current_time.tv_sec - context->start_time.tv_sec));
+	request->dhcp.dp_xid = htonl(context->xid);
+#ifdef RANDOM_IP_ID
+	request->ip.ip_id = ip_randomid();
+#else
+	request->ip.ip_id = htons(ip_id++);
+#endif
+	error = send_packet(context->ifp, request, request_size);
+	if (error != 0) {
+	    printf("dhcp: send_packet failed with %d\n", error);
+	    goto failed;
+	}
+
+	wait_ticks += random_range(-RAND_TICKS, RAND_TICKS);
+	dprintf(("dhcp: waiting %d ticks\n", wait_ticks));
+	error = dhcp_get_ack(context, wait_ticks);
+	switch (error) {
+	case 0:
+	    /* we're done */
+	    goto done;
+	case EPROTO:
+	    printf("dhcp: server " IP_FORMAT " send us a NAK\n",
+		   IP_LIST(&context->server_id));
+	    goto failed;
+	case EWOULDBLOCK:
+	    break;
+	default:
+	    dprintf(("dhcp: failed to receive packets: %d\n", error));
+	    goto failed;
+	}
+	wait_ticks *= 2;
+	if (wait_ticks > (MAX_WAIT_SECS * hz))
+	    wait_ticks = MAX_WAIT_SECS * hz;
+	microtime(&current_time);
+    }
+    error = ETIMEDOUT;
+    goto failed;
+    
+ done:
+    error = 0;
+
+ failed:
+    return (error);
+}
+
+static int
+dhcp_get_offer(struct dhcp_context * context, int wait_ticks)
+{
+    int				error = 0;
+    int	 			gather_count = 0;
+    const struct in_addr * 	ip;
+    int				last_rating = 0;
+    int				len;
+    int				n;
+    int 			rating;
+    struct dhcp *		reply;
+    struct in_addr		server_id;
+    struct socket * 		timer_arg;
+
+    timer_arg = context->so;
+    reply = dhcp_context_reply(context);
+    timeout((timeout_fcn_t)dhcp_timeout, &timer_arg, wait_ticks);
+    while (1) {
+	error = receive_packet(context->so, context->reply,
+			       sizeof(context->reply), &n);
+	if (error == 0) {
+	    dhcpol_t		options;
+
+	    dprintf(("\ndhcp: received packet length %d\n", n));
+	    if (n < (int)sizeof(struct dhcp)) {
+		dprintf(("dhcp: packet is too short %d < %d\n",
+			 n, (int)sizeof(struct dhcp)));
+		continue;
+	    }
+	    if (ntohl(reply->dp_xid) != context->xid
+		|| reply->dp_yiaddr.s_addr == 0
+		|| reply->dp_yiaddr.s_addr == INADDR_BROADCAST
+		|| bcmp(reply->dp_chaddr,
+			link_address(context->dl_p), 
+			link_address_length(context->dl_p)) != 0) {
+		/* not for us */
+		continue;
+	    }
+	    (void)dhcpol_parse_packet(&options, reply, n);
+	    if (get_dhcp_msgtype(&options) != dhcp_msgtype_offer_e) {
+		/* not an offer */
+		goto next_packet;
+	    }
+	    ip = (const struct in_addr *)
+		dhcpol_find(&options, 
+			    dhcptag_server_identifier_e, &len, NULL);
+	    if (ip == NULL || len < (int)sizeof(*ip)) {
+		/* missing/invalid server identifier */
+		goto next_packet;
+	    }
+	    printf("dhcp: received OFFER: server " IP_FORMAT
+		   " IP address "  IP_FORMAT "\n",
+		   IP_LIST(ip), IP_LIST(&reply->dp_yiaddr));
+	    server_id = *ip;
+	    rating = rate_packet(&options);
+	    if (rating > last_rating) {
+		context->iaddr = reply->dp_yiaddr;
+		ip = (const struct in_addr *)
+		    dhcpol_find(&options, 
+				dhcptag_subnet_mask_e, &len, NULL);
+		if (ip != NULL && len >= (int)sizeof(*ip)) {
+		    context->netmask = *ip;
+		}
+		ip = (const struct in_addr *)
+		    dhcpol_find(&options, dhcptag_router_e, &len, NULL);
+		if (ip != NULL && len >= (int)sizeof(*ip)) {
+		    context->router = *ip;
+		}
+		context->server_id = server_id;
+	    }
+	    if (rating >= GOOD_RATING) {
+		dhcpol_free(&options);
+		/* packet is good enough */
+		untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+		break;
+	    }
+	    if (gather_count == 0) {
+		untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+		timer_arg = context->so;
+		timeout((timeout_fcn_t)dhcp_timeout, &timer_arg, 
+			hz * GATHER_TIME_SECS);
+	    }
+	    gather_count = 1;
+	next_packet:
+	    dhcpol_free(&options);
+	}
+	else if ((error != EWOULDBLOCK)) {
+	    untimeout((timeout_fcn_t)dhcp_timeout, &timer_arg);
+	    break;
+	}
+	else if (timer_arg == NULL) { /* timed out */
+	    if (gather_count != 0) {
+		dprintf(("dhcp: gathering time has expired\n"));
+		error = 0;
+	    }
+	    break;
+	}
+	else {
+	    socket_lock(context->so, 1);
+	    error = sbwait(&context->so->so_rcv);
+	    socket_unlock(context->so, 1);
+	}
+    }
+    return (error);
+}
+
+/*
+ * Function: dhcp_init
+ * Purpose:
+ *   Start in the DHCP INIT state sending DISCOVER's.  When we get OFFER's,
+ *   try to select one of them by sending a REQUEST and waiting for an ACK.
+ */
+static int
+dhcp_init(struct dhcp_context * context)
+{
+    struct timeval		current_time;
+    int				error = 0;
+    uint32_t			lease_option = htonl(SUGGESTED_LEASE_LENGTH);
+    dhcpoa_t *			options_p;
+    struct dhcp_packet *	request;
+    int				request_size;
+    int				retry;
+    int				wait_ticks;
+
+    /* remember the time we started */
+    microtime(&context->start_time);
+    current_time = context->start_time;
+    
+    request = dhcp_context_request(context);
+    options_p = &context->request_options;
+
+ retry:
+    /* format a DHCP DISCOVER packet */
+    make_dhcp_request(&request->dhcp, DHCP_PAYLOAD_MIN,
+		      dhcp_msgtype_discover_e,
+		      link_address(context->dl_p), ARPHRD_ETHER,
+		      link_address_length(context->dl_p), 
+		      options_p);
+    /* add the requested lease time */
+    dhcpoa_add(options_p, dhcptag_lease_time_e,
+	       sizeof(lease_option), &lease_option);
+    dhcpoa_add(options_p, dhcptag_end_e, 0, 0);
+    request_size = sizeof(*request) + RFC_MAGIC_SIZE 
+	+ dhcpoa_used(options_p);
+    if (request_size < (int)sizeof(struct bootp_packet)) {
+	/* pad out to BOOTP-sized packet */
+	request_size = sizeof(struct bootp_packet);
+    }
+    init_dhcp_packet_header(request, request_size);
+
+    wait_ticks = INITIAL_WAIT_SECS * hz;
+    for (retry = 0; retry < context->max_try; retry++) {
+	/* Send the request */
+	printf("dhcp: sending DISCOVER\n");
+	request->dhcp.dp_secs 
+	    = htons((u_short)(current_time.tv_sec 
+			      - context->start_time.tv_sec));
+	request->dhcp.dp_xid = htonl(context->xid);
+#ifdef RANDOM_IP_ID
+	request->ip.ip_id = ip_randomid();
+#else
+	request->ip.ip_id = htons(ip_id++);
+#endif
+	error = send_packet(context->ifp, request, request_size);
+	if (error != 0) {
+	    printf("dhcp: send_packet failed with %d\n", error);
+	    goto failed;
+	}
+	wait_ticks += random_range(-RAND_TICKS, RAND_TICKS);
+	dprintf(("dhcp: waiting %d ticks\n", wait_ticks));
+	error = dhcp_get_offer(context, wait_ticks);
+	if (error == 0) {
+	    /* send a REQUEST */
+	    error = dhcp_select(context);
+	    if (error == 0) {
+		/* we're done !*/
+		goto done;
+	    }
+	    if (error != EPROTO && error != ETIMEDOUT) {
+		/* fatal error */ 
+		dprintf(("dhcp: dhcp_select failed %d\n", error));
+		goto failed;
+	    }
+	    /* wait 10 seconds, and try again */
+	    printf("dhcp: trying again in 10 seconds\n");
+	    tsleep(&error, PRIBIO, "dhcp_init", 10 * hz);
+	    context->xid++;
+	    goto retry;
+	}
+	else if (error != EWOULDBLOCK) {
+	    dprintf(("dhcp: failed to receive packets: %d\n", error));
+	    goto failed;
+	}
+	wait_ticks *= 2;
+	if (wait_ticks > (MAX_WAIT_SECS * hz))
+	    wait_ticks = MAX_WAIT_SECS * hz;
+	microtime(&current_time);
+    }
+    error = ETIMEDOUT;
+    goto failed;
+    
+ done:
+    error = 0;
+
+ failed:
+    return (error);
+}
+
+static void
+dhcp_context_free(struct dhcp_context * context, struct proc * procp)
+{
+    if (context == NULL) {
+	return;
+    }
+    if (context->so != NULL) {
+	int		error;
+
+	/* disable reception of DHCP packets before address assignment */
+	context->ifr.ifr_intval = 0;
+	error = ifioctl(context->so, SIOCAUTOADDR,
+			(caddr_t)&context->ifr, procp);
+	if (error) {
+	    printf("dhcp: SIOCAUTOADDR failed: %d\n", error);
+	}
+	soclose(context->so);
+    }
+    kfree(context, sizeof(*context));
+    return;
+}
+
+static struct dhcp_context *
+dhcp_context_create(struct ifnet * ifp, int max_try,
+		    struct proc * procp, int * error_p)
+{
+    struct dhcp_context	*	context = NULL;
+    struct sockaddr_dl *	dl_p;
+    struct in_addr		lo_addr;
+    struct in_addr		lo_mask;
+    int				error;
+    struct sockaddr_in		sin;
+
+    /* get the hardware address from the interface */
+    dl_p = link_from_ifnet(ifp);
+    if (dl_p == NULL) {
+	printf("dhcp: can't get link address\n");
+	error = ENXIO;
+	goto failed;
+    }
+
+    printf("dhcp: h/w addr ");
+    link_print(dl_p);
+    if (dl_p->sdl_type != IFT_ETHER) {
+	printf("dhcp: hardware type %d not supported\n",
+	       dl_p->sdl_type);
+	error = ENXIO;
+	goto failed;
+    }
+
+    context = (struct dhcp_context *)kalloc(sizeof(*context));
+    if (context == NULL) {
+	printf("dhcp: failed to allocate context\n");
+	error = ENOMEM;
+	goto failed;
+    }
+    bzero(context, sizeof(*context));
+
+    /* get a socket */
+    error = socreate(AF_INET, &context->so, SOCK_DGRAM, 0);
+    if (error != 0) {
+	printf("dhcp: socreate failed %d\n", error);
+	goto failed;
+    }
+
+    /* assign 127.0.0.1 to lo0 so that the bind will succeed */
+    lo_addr.s_addr = htonl(INADDR_LOOPBACK);
+    lo_mask.s_addr = htonl(IN_CLASSA_NET);
+    error = inet_aifaddr(context->so, "lo0", &lo_addr, &lo_mask, NULL);
+    if (error != 0) {
+	printf("dhcp: assigning loopback address failed %d\n", error);
+    }
+
+    /* enable reception of DHCP packets before an address is assigned */
+    snprintf(context->ifr.ifr_name, 
+	     sizeof(context->ifr.ifr_name), "%s%d", ifp->if_name,
+	     ifp->if_unit);
+    context->ifr.ifr_intval = 1;
+
+    error = ifioctl(context->so, SIOCAUTOADDR, (caddr_t)&context->ifr, procp);
+    if (error) {
+	printf("dhcp: SIOCAUTOADDR failed: %d\n", error);
+	goto failed;
+    }
+    dprintf(("dhcp: SIOCAUTOADDR done\n"));
+
+    error = ifioctl(context->so, SIOCPROTOATTACH, (caddr_t)&context->ifr, 
+		    procp);
+    if (error) {
+	printf("dhcp: SIOCPROTOATTACH failed: %d\n", error);
+	goto failed;
+    }
+    dprintf(("dhcp: SIOCPROTOATTACH done\n"));
+    
+    /* bind the socket */
+    sin.sin_len = sizeof(sin);
+    sin.sin_family = AF_INET;
+    sin.sin_port = htons(IPPORT_BOOTPC);
+    sin.sin_addr.s_addr = INADDR_ANY;
+    error = sobind(context->so, (struct sockaddr *)&sin);
+    if (error) {
+	printf("dhcp: sobind failed, %d\n", error);
+	goto failed;
+    }
+
+    /* make it non-blocking I/O */
+    socket_lock(context->so, 1);
+    context->so->so_state |= SS_NBIO;
+    socket_unlock(context->so, 1);
+
+    /* save passed-in information */
+    context->max_try = max_try;
+    context->dl_p = dl_p;
+    context->ifp = ifp;
+
+    /* get a random transaction id */
+    context->xid = random();
+
+    return (context);
+
+ failed:
+    dhcp_context_free(context, procp);
+    *error_p = error;
+    return (NULL);
+}
+
+/*
+ * Routine: dhcp
+ * Function:
+ *   Do DHCP over the specified interface to retrieve the IP address,
+ *   subnet mask, and router.	
+ */
+int 
+dhcp(struct ifnet * ifp, struct in_addr * iaddr_p, int max_try,
+     struct in_addr * netmask_p, struct in_addr * router_p,
+     struct proc * procp)
+{
+    int				error = 0;
+    struct dhcp_context	*	context;
+
+    context = dhcp_context_create(ifp, max_try, procp, &error);
+    if (context == NULL) {
+	return (error);
+    }
+ 
+    /* start DHCP in the INIT state */
+    error = dhcp_init(context);
+    if (error == 0) {
+	*iaddr_p = context->iaddr;
+	*netmask_p = context->netmask;
+	*router_p = context->router;
+    }
+    dhcp_context_free(context, procp);
+    return (error);
+}
diff --git a/bsd/netinet/in_dhcp.h b/bsd/netinet/in_dhcp.h
new file mode 100644
index 000000000..3a898af38
--- /dev/null
+++ b/bsd/netinet/in_dhcp.h
@@ -0,0 +1,49 @@
+#ifndef _NETINET_IN_DHCP_H
+#define _NETINET_IN_DHCP_H
+#include <sys/appleapiopts.h>
+
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * in_dhcp.h
+ * - definitions for in_dhcp.c
+ */
+
+extern int
+inet_aifaddr(struct socket * so, const char * name,
+	     const struct in_addr * addr, 
+	     const struct in_addr * mask,
+	     const struct in_addr * broadcast);
+
+extern int
+dhcp(struct ifnet * ifp, struct in_addr * iaddr_p, int max_try,
+     struct in_addr * netmask_p, struct in_addr * router_p,
+     struct proc * procp);
+
+#endif /* _NETINET_IN_DHCP_H */
diff --git a/bsd/netinet/in_gif.c b/bsd/netinet/in_gif.c
index 6b5b9efff..2076a730d 100644
--- a/bsd/netinet/in_gif.c
+++ b/bsd/netinet/in_gif.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: in_gif.c,v 1.54 2001/05/14 14:02:16 itojun Exp $	*/
 
@@ -61,6 +67,7 @@
 #include <sys/sysctl.h>
 
 #include <sys/malloc.h>
+#include <libkern/OSAtomic.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -95,9 +102,9 @@ in_gif_output(
 	struct ifnet	*ifp,
 	int		family,
 	struct mbuf	*m,
-	struct rtentry *rt)
+	__unused struct rtentry *rt)
 {
-	struct gif_softc *sc = (struct gif_softc*)ifp;
+	struct gif_softc *sc = ifnet_softc(ifp);
 	struct sockaddr_in *dst = (struct sockaddr_in *)&sc->gif_ro.ro_dst;
 	struct sockaddr_in *sin_src = (struct sockaddr_in *)sc->gif_psrc;
 	struct sockaddr_in *sin_dst = (struct sockaddr_in *)sc->gif_pdst;
@@ -119,7 +126,7 @@ in_gif_output(
 		struct ip *ip;
 
 		proto = IPPROTO_IPV4;
-		if (m->m_len < sizeof(*ip)) {
+		if (mbuf_len(m) < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (!m)
 				return ENOBUFS;
@@ -134,7 +141,7 @@ in_gif_output(
 	    {
 		struct ip6_hdr *ip6;
 		proto = IPPROTO_IPV6;
-		if (m->m_len < sizeof(*ip6)) {
+		if (mbuf_len(m) < sizeof(*ip6)) {
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m)
 				return ENOBUFS;
@@ -173,7 +180,7 @@ in_gif_output(
 
 	/* prepend new IP header */
 	M_PREPEND(m, sizeof(struct ip), M_DONTWAIT);
-	if (m && m->m_len < sizeof(struct ip))
+	if (m && mbuf_len(m) < sizeof(struct ip))
 		m = m_pullup(m, sizeof(struct ip));
 	if (m == NULL) {
 		printf("ENOBUFS in in_gif_output %d\n", __LINE__);
@@ -214,7 +221,7 @@ in_gif_output(
 #endif
 	}
 
-	error = ip_output(m, NULL, &sc->gif_ro, 0, NULL);
+	error = ip_output(m, NULL, &sc->gif_ro, 0, NULL, NULL);
 	return(error);
 }
 
@@ -225,18 +232,18 @@ in_gif_input(m, off)
 {
 	struct ifnet *gifp = NULL;
 	struct ip *ip;
-	int i, af, proto;
+	int af, proto;
 	u_int8_t otos;
 
 	ip = mtod(m, struct ip *);
 	proto = ip->ip_p;
 
 
-	gifp = (struct ifnet *)encap_getarg(m);
+	gifp = ((struct gif_softc*)encap_getarg(m))->gif_if;
 
 	if (gifp == NULL || (gifp->if_flags & IFF_UP) == 0) {
 		m_freem(m);
-		ipstat.ips_nogif++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_nogif);
 		return;
 	}
 
@@ -247,9 +254,8 @@ in_gif_input(m, off)
 #if INET
 	case IPPROTO_IPV4:
 	    {
-		struct ip *ip;
 		af = AF_INET;
-		if (m->m_len < sizeof(*ip)) {
+		if (mbuf_len(m) < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (!m)
 				return;
@@ -268,7 +274,7 @@ in_gif_input(m, off)
 		struct ip6_hdr *ip6;
 		u_int8_t itos;
 		af = AF_INET6;
-		if (m->m_len < sizeof(*ip6)) {
+		if (mbuf_len(m) < sizeof(*ip6)) {
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m)
 				return;
@@ -285,7 +291,7 @@ in_gif_input(m, off)
 	    }
 #endif /* INET6 */
 	default:
-		ipstat.ips_nogif++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_nogif);
 		m_freem(m);
 		return;
 	}
@@ -293,23 +299,34 @@ in_gif_input(m, off)
 	/* Should we free m if dlil_input returns an error? */
 	if (m->m_pkthdr.rcvif)	/* replace the rcvif by gifp for dlil to route it correctly */
 		m->m_pkthdr.rcvif = gifp;
-	dlil_input_packet(gifp, m, NULL);
+	ifnet_input(gifp, m, NULL);
 #else
 	gif_input(m, af, gifp);
 #endif
 	return;
 }
 
+static __inline__ void*
+_cast_non_const(const void * ptr) {
+	union {
+		const void*		cval;
+		void*			val;
+	} ret;
+	
+	ret.cval = ptr;
+	return (ret.val);
+}
+
 /*
  * we know that we are in IFF_UP, outer address available, and outer family
  * matched the physical addr family.  see gif_encapcheck().
  */
 int
-gif_encapcheck4(m, off, proto, arg)
-	const struct mbuf *m;
-	int off;
-	int proto;
-	void *arg;
+gif_encapcheck4(
+	const struct mbuf *m,
+	__unused int off,
+	__unused int proto,
+	void *arg)
 {
 	struct ip ip;
 	struct gif_softc *sc;
@@ -322,8 +339,7 @@ gif_encapcheck4(m, off, proto, arg)
 	src = (struct sockaddr_in *)sc->gif_psrc;
 	dst = (struct sockaddr_in *)sc->gif_pdst;
 
-	/* LINTED const cast */
-	m_copydata((struct mbuf *)m, 0, sizeof(ip), (caddr_t)&ip);
+	mbuf_copydata(m, 0, sizeof(ip), &ip);
 
 	/* check for address match */
 	addrmatch = 0;
@@ -346,7 +362,7 @@ gif_encapcheck4(m, off, proto, arg)
 	for (ia4 = TAILQ_FIRST(&in_ifaddrhead); ia4;
 	     ia4 = TAILQ_NEXT(ia4, ia_link))
 	{
-		if ((ia4->ia_ifa.ifa_ifp->if_flags & IFF_BROADCAST) == 0)
+		if ((ifnet_flags(ia4->ia_ifa.ifa_ifp) & IFF_BROADCAST) == 0)
 			continue;
 		if (ip.ip_src.s_addr == ia4->ia_broadaddr.sin_addr.s_addr) {
 			lck_mtx_unlock(rt_mtx);
@@ -356,7 +372,7 @@ gif_encapcheck4(m, off, proto, arg)
 	lck_mtx_unlock(rt_mtx);
 
 	/* ingress filters on outer source */
-	if ((sc->gif_if.if_flags & IFF_LINK2) == 0 &&
+	if ((ifnet_flags(sc->gif_if) & IFF_LINK2) == 0 &&
 	    (m->m_flags & M_PKTHDR) != 0 && m->m_pkthdr.rcvif) {
 		struct sockaddr_in sin;
 		struct rtentry *rt;
diff --git a/bsd/netinet/in_gif.h b/bsd/netinet/in_gif.h
index 4321eba46..0fb9240b1 100644
--- a/bsd/netinet/in_gif.h
+++ b/bsd/netinet/in_gif.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: in_gif.h,v 1.3 2000/02/22 14:01:59 itojun Exp $	*/
 
diff --git a/bsd/netinet/in_pcb.c b/bsd/netinet/in_pcb.c
index c3b56f784..a17a1fb7e 100644
--- a/bsd/netinet/in_pcb.c
+++ b/bsd/netinet/in_pcb.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1991, 1993, 1995
@@ -101,7 +107,6 @@
 
 #if IPSEC
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif
 
 extern u_long  route_generation;
@@ -134,6 +139,7 @@ int	ipport_hilastauto  = IPPORT_HILASTAUTO;		/* 65535 */
 static int
 sysctl_net_ipport_check SYSCTL_HANDLER_ARGS
 {
+#pragma unused(arg1, arg2)
 	int error = sysctl_handle_int(oidp,
 		oidp->oid_arg1, oidp->oid_arg2, req);
 	if (!error) {
@@ -149,7 +155,7 @@ sysctl_net_ipport_check SYSCTL_HANDLER_ARGS
 
 #undef RANGECHK
 
-SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
+SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "IP Ports");
 
 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
 	   &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
@@ -174,19 +180,24 @@ SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
 
 /*
  * Allocate a PCB and associate it with the socket.
+ *
+ * Returns:	0			Success
+ *		ENOBUFS
+ *		ENOMEM
+ *	ipsec_init_policy:???		[IPSEC]
  */
 int
-in_pcballoc(so, pcbinfo, p)
-	struct socket *so;
-	struct inpcbinfo *pcbinfo;
-	struct proc *p;
+in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo, __unused struct proc *p)
 {
-	register struct inpcb *inp;
+	struct inpcb *inp;
 	caddr_t		      temp;
 #if IPSEC
 #ifndef __APPLE__
 	int error;
 #endif
+#endif
+#if CONFIG_MACF_NET
+	int mac_error;
 #endif
 
 	if (so->cached_in_sock_layer == 0) {
@@ -211,12 +222,21 @@ in_pcballoc(so, pcbinfo, p)
 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 	inp->inp_pcbinfo = pcbinfo;
 	inp->inp_socket = so;
+#if CONFIG_MACF_NET
+	mac_error = mac_inpcb_label_init(inp, M_WAITOK);
+	if (mac_error != 0) {
+		if (so->cached_in_sock_layer == 0)
+			zfree(pcbinfo->ipi_zone, inp);
+		return (mac_error);
+	}
+	mac_inpcb_label_associate(so, inp);
+#endif
 	so->so_pcb = (caddr_t)inp;
 
 	if (so->so_proto->pr_flags & PR_PCBLOCK) {
 		inp->inpcb_mtx = lck_mtx_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr);
 		if (inp->inpcb_mtx == NULL) {
-			printf("in_pcballoc: can't alloc mutex! so=%x\n", so);
+			printf("in_pcballoc: can't alloc mutex! so=%p\n", so);
 			return(ENOMEM);
 		}
 	}
@@ -224,9 +244,7 @@ in_pcballoc(so, pcbinfo, p)
 #if IPSEC
 #ifndef __APPLE__
 	if (ipsec_bypass == 0) {
-		lck_mtx_lock(sadb_mutex);
 		error = ipsec_init_policy(so, &inp->inp_sp);
-		lck_mtx_unlock(sadb_mutex);
 		if (error != 0) {
 			zfree(pcbinfo->ipi_zone, inp);
 			return error;
@@ -234,7 +252,7 @@ in_pcballoc(so, pcbinfo, p)
 	}
 #endif
 #endif /*IPSEC*/
-#if defined(INET6)
+#if INET6
 	if (INP_SOCKAF(so) == AF_INET6 && !ip6_mapped_addr_on)
 		inp->inp_flags |= IN6P_IPV6_V6ONLY;
 #endif
@@ -251,19 +269,88 @@ in_pcballoc(so, pcbinfo, p)
 	return (0);
 }
 
+
+/*
+  in_pcblookup_local_and_cleanup does everything
+  in_pcblookup_local does but it checks for a socket
+  that's going away. Since we know that the lock is
+  held read+write when this funciton is called, we
+  can safely dispose of this socket like the slow
+  timer would usually do and return NULL. This is
+  great for bind.
+*/
+struct inpcb*
+in_pcblookup_local_and_cleanup(
+	struct inpcbinfo *pcbinfo,
+	struct in_addr laddr,
+	u_int lport_arg,
+	int wild_okay)
+{
+	struct inpcb *inp;
+	
+	/* Perform normal lookup */
+	inp = in_pcblookup_local(pcbinfo, laddr, lport_arg, wild_okay);
+	
+	/* Check if we found a match but it's waiting to be disposed */
+	if (inp && inp->inp_wantcnt == WNT_STOPUSING) {
+		struct socket *so = inp->inp_socket;
+		
+		lck_mtx_lock(inp->inpcb_mtx);
+		
+		if (so->so_usecount == 0) {
+			in_pcbdispose(inp);
+			inp = NULL;
+		}
+		else {
+			lck_mtx_unlock(inp->inpcb_mtx);
+		}
+	}
+	
+	return inp;
+}
+
+#ifdef __APPLE_API_PRIVATE
+in_pcb_conflict_post_msg(u_int16_t port)
+{
+	/* 
+	 * Radar 5523020 send a kernel event notification if a non-participating socket tries to bind
+	 * 		 the port a socket who has set SOF_NOTIFYCONFLICT owns.
+	 */
+	struct kev_msg        ev_msg;
+	struct kev_in_portinuse	in_portinuse;
+
+	in_portinuse.port = ntohs(port);	/* port in host order */
+	in_portinuse.req_pid = proc_selfpid();
+	ev_msg.vendor_code = KEV_VENDOR_APPLE;
+	ev_msg.kev_class = KEV_NETWORK_CLASS;
+	ev_msg.kev_subclass = KEV_INET_SUBCLASS;
+	ev_msg.event_code = KEV_INET_PORTINUSE;
+	ev_msg.dv[0].data_ptr = &in_portinuse;
+	ev_msg.dv[0].data_length      = sizeof(struct kev_in_portinuse);
+	ev_msg.dv[1].data_length = 0;
+	kev_post_msg(&ev_msg);
+}
+#endif
+/*
+ * Returns:	0			Success
+ *		EADDRNOTAVAIL		Address not available.
+ *		EINVAL			Invalid argument
+ *		EAFNOSUPPORT		Address family not supported [notdef]
+ *		EACCES			Permission denied
+ *		EADDRINUSE		Address in use
+ *		EAGAIN			Resource unavailable, try again
+ *		proc_suser:EPERM	Operation not permitted
+ */
 int
-in_pcbbind(inp, nam, p)
-	register struct inpcb *inp;
-	struct sockaddr *nam;
-	struct proc *p;
+in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct proc *p)
 {
-	register struct socket *so = inp->inp_socket;
+	struct socket *so = inp->inp_socket;
 	unsigned short *lastport;
 	struct sockaddr_in *sin;
 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 	u_short lport = 0;
 	int wild = 0, reuseport = (so->so_options & SO_REUSEPORT);
-	int error;
+	int error, conflict = 0;
 
 	if (TAILQ_EMPTY(&in_ifaddrhead)) /* XXX broken! */
 		return (EADDRNOTAVAIL);
@@ -326,29 +413,43 @@ in_pcbbind(inp, nam, p)
 			}
 			if (so->so_uid &&
 			    !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
-				t = in_pcblookup_local(inp->inp_pcbinfo,
+				t = in_pcblookup_local_and_cleanup(inp->inp_pcbinfo,
 				    sin->sin_addr, lport, INPLOOKUP_WILDCARD);
 				if (t &&
 				    (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 				     ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 				     (t->inp_socket->so_options &
 					 SO_REUSEPORT) == 0) &&
-				     (so->so_uid != t->inp_socket->so_uid)) {
+				     (so->so_uid != t->inp_socket->so_uid) &&
+				      ((t->inp_socket->so_flags & SOF_REUSESHAREUID) == 0)) {
 #if INET6
 					if (ntohl(sin->sin_addr.s_addr) !=
 					    INADDR_ANY ||
 					    ntohl(t->inp_laddr.s_addr) !=
 					    INADDR_ANY ||
 					    INP_SOCKAF(so) ==
-					    INP_SOCKAF(t->inp_socket)) {
-#endif /* defined(INET6) */
+					    INP_SOCKAF(t->inp_socket))
+#endif /* INET6 */
+					{
+#ifdef __APPLE_API_PRIVATE
+
+						if ((t->inp_socket->so_flags & SOF_NOTIFYCONFLICT) && ((so->so_flags & SOF_NOTIFYCONFLICT) == 0)) 
+							conflict = 1;
+
+						lck_rw_done(pcbinfo->mtx);
+
+						if (conflict)
+							in_pcb_conflict_post_msg(lport);
+#else
 						lck_rw_done(pcbinfo->mtx);
+#endif /* __APPLE_API_PRIVATE */
+
 						socket_lock(so, 0);
 						return (EADDRINUSE);
 					}
 				}
 			}
-			t = in_pcblookup_local(pcbinfo, sin->sin_addr,
+			t = in_pcblookup_local_and_cleanup(pcbinfo, sin->sin_addr,
 			    lport, wild);
 			if (t &&
 			    (reuseport & t->inp_socket->so_options) == 0) {
@@ -359,9 +460,21 @@ in_pcbbind(inp, nam, p)
 				    ntohl(t->inp_laddr.s_addr) !=
 				    INADDR_ANY ||
 				    INP_SOCKAF(so) ==
-				    INP_SOCKAF(t->inp_socket)) {
-#endif /* defined(INET6) */
+				    INP_SOCKAF(t->inp_socket))
+#endif /* INET6 */
+				{
+#ifdef __APPLE_API_PRIVATE
+
+					if ((t->inp_socket->so_flags & SOF_NOTIFYCONFLICT) && ((so->so_flags & SOF_NOTIFYCONFLICT) == 0)) 
+						conflict = 1;
+
 					lck_rw_done(pcbinfo->mtx);
+
+					if (conflict)
+						in_pcb_conflict_post_msg(lport);
+#else
+					lck_rw_done(pcbinfo->mtx);
+#endif /* __APPLE_API_PRIVATE */
 					socket_lock(so, 0);
 					return (EADDRINUSE);
 				}
@@ -417,7 +530,7 @@ in_pcbbind(inp, nam, p)
 				if (*lastport > first || *lastport < last)
 					*lastport = first;
 				lport = htons(*lastport);
-			} while (in_pcblookup_local(pcbinfo,
+			} while (in_pcblookup_local_and_cleanup(pcbinfo,
 				 inp->inp_laddr, lport, wild));
 		} else {
 			/*
@@ -436,7 +549,7 @@ in_pcbbind(inp, nam, p)
 				if (*lastport < first || *lastport > last)
 					*lastport = first;
 				lport = htons(*lastport);
-			} while (in_pcblookup_local(pcbinfo,
+			} while (in_pcblookup_local_and_cleanup(pcbinfo,
 				 inp->inp_laddr, lport, wild));
 		}
 	}
@@ -449,9 +562,81 @@ in_pcbbind(inp, nam, p)
 		return (EAGAIN);
 	}
 	lck_rw_done(pcbinfo->mtx);
+	sflt_notify(so, sock_evt_bound, NULL);
 	return (0);
 }
 
+#if CONFIG_FORCE_OUT_IFP
+/*
+ * pdp_context_route_locked is losely based on rtalloc_ign_locked with
+ * the hope that it can be used anywhere rtalloc_ign_locked is.
+ */
+__private_extern__ void
+pdp_context_route_locked(ifnet_t ifp, struct route *ro)
+{
+	struct in_ifaddr	*ia;
+	struct rtentry		*rt;
+
+	if ((rt = ro->ro_rt) != NULL) {
+		if (rt->rt_ifp == ifp && rt->rt_flags & RTF_UP)
+			return;
+
+		rtfree_locked(rt);
+		ro->ro_rt = NULL;
+	}
+
+	if (ifp == NULL)
+		return;
+
+	/* Find the first IP address, we will use a fake route off of that */
+	TAILQ_FOREACH(ia, &in_ifaddrhead, ia_link) {
+		if (ia->ia_ifp == ifp)
+			break;
+	}
+
+	/* Hrmm no IP addresses here :( */
+	if (ia == NULL)
+		return;
+
+	rt = ia->ia_route;
+	if (rt == NULL) {
+		struct sockaddr *ifa = ia->ia_ifa.ifa_addr;
+
+		/* Allocate and set up a fake route */
+		if ((rt = rte_alloc()) == NULL)
+			return;
+
+		bzero(rt, sizeof(*rt));
+		rt->rt_flags = RTF_UP | RTF_STATIC;
+		if (rt_setgate(rt, ifa, ifa) != 0) {
+			rte_free(rt);
+			return;
+		}
+		/*
+		 * Explicitly zero the key so that:
+		 *   rt_tables[rt_key(rt)->sa_family] == rt_tables[0] == NULL
+		 */
+		bzero(rt_key(rt), ifa->sa_len);
+
+		rtsetifa(rt, &ia->ia_ifa);
+		rt->rt_ifp = rt->rt_ifa->ifa_ifp;
+
+		/* Take a reference for the ia pointer to this */
+		ia->ia_route = rt;
+		rtref(rt);
+
+		/*
+		 * One more rtentry floating around that is not
+		 * linked to the routing table.
+		 */
+		(void) OSIncrementAtomic((SInt32 *)&rttrash);
+	}
+	rt->generation_id = route_generation;
+	rtref(rt); /* increment the reference count */
+	ro->ro_rt = rt;
+}
+#endif
+
 /*
  *   Transform old in_pcbconnect() into an inner subroutine for new
  *   in_pcbconnect(): Do some validity-checking on the remote
@@ -462,16 +647,18 @@ in_pcbbind(inp, nam, p)
  *   slightly different version for T/TCP.  (This is more than
  *   a bit of a kludge, but cleaning up the internal interfaces would
  *   have forced minor changes in every protocol).
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument
+ *		EAFNOSUPPORT		Address family not supported
+ *		EADDRNOTAVAIL		Address not available
  */
-
 int
-in_pcbladdr(inp, nam, plocal_sin)
-	register struct inpcb *inp;
-	struct sockaddr *nam;
-	struct sockaddr_in **plocal_sin;
+in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
+	    struct sockaddr_in **plocal_sin)
 {
 	struct in_ifaddr *ia;
-	register struct sockaddr_in *sin = (struct sockaddr_in *)nam;
+	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 
 	if (nam->sa_len != sizeof (*sin))
 		return (EINVAL);
@@ -479,7 +666,9 @@ in_pcbladdr(inp, nam, plocal_sin)
 		return (EAFNOSUPPORT);
 	if (sin->sin_port == 0)
 		return (EADDRNOTAVAIL);
+	
 	lck_mtx_lock(rt_mtx);
+	
 	if (!TAILQ_EMPTY(&in_ifaddrhead)) {
 		/*
 		 * If the destination address is INADDR_ANY,
@@ -498,7 +687,7 @@ in_pcbladdr(inp, nam, plocal_sin)
 		    sin->sin_addr = satosin(&TAILQ_FIRST(&in_ifaddrhead)->ia_broadaddr)->sin_addr;
 	}
 	if (inp->inp_laddr.s_addr == INADDR_ANY) {
-		register struct route *ro;
+		struct route *ro;
 
 		ia = (struct in_ifaddr *)0;
 		/*
@@ -526,7 +715,14 @@ in_pcbladdr(inp, nam, plocal_sin)
 			ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 			((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 				sin->sin_addr;
-			rtalloc_ign_locked(ro, 0UL);
+#if CONFIG_FORCE_OUT_IFP			
+			/* If the socket has requested a specific interface, use that address */
+			if (inp->pdp_ifp != NULL) {
+				pdp_context_route_locked(inp->pdp_ifp, ro);
+			}
+			else 
+#endif /* CONFIG_FORCE_OUT_IFP */
+				rtalloc_ign_locked(ro, 0UL);
 		}
 		/*
 		 * If we found a route, use the address
@@ -581,7 +777,7 @@ in_pcbladdr(inp, nam, plocal_sin)
 					lck_mtx_unlock(rt_mtx);
 					return (EADDRNOTAVAIL);
 				}
-				ifaref(ia);
+				ifaref(&ia->ia_ifa);
 			}
 		}
 		/*
@@ -603,10 +799,7 @@ in_pcbladdr(inp, nam, plocal_sin)
  * then pick one.
  */
 int
-in_pcbconnect(inp, nam, p)
-	register struct inpcb *inp;
-	struct sockaddr *nam;
-	struct proc *p;
+in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct proc *p)
 {
 	struct sockaddr_in *ifaddr;
 	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
@@ -659,8 +852,7 @@ in_pcbconnect(inp, nam, p)
 }
 
 void
-in_pcbdisconnect(inp)
-	struct inpcb *inp;
+in_pcbdisconnect(struct inpcb *inp)
 {
 
 	inp->inp_faddr.s_addr = INADDR_ANY;
@@ -681,28 +873,24 @@ in_pcbdisconnect(inp)
 }
 
 void
-in_pcbdetach(inp)
-	struct inpcb *inp;
+in_pcbdetach(struct inpcb *inp)
 {
 	struct socket *so = inp->inp_socket;
-	struct rtentry *rt  = inp->inp_route.ro_rt;
 
 	if (so->so_pcb == 0) { /* we've been called twice */
-		panic("in_pcbdetach: inp=%x so=%x proto=%x so_pcb is null!\n",
+		panic("in_pcbdetach: inp=%p so=%p proto=%d so_pcb is null!\n",
 			inp, so, so->so_proto->pr_protocol);
 	}
 
 #if IPSEC
 	if (ipsec_bypass == 0) {
-		lck_mtx_lock(sadb_mutex);
 		ipsec4_delete_pcbpolicy(inp);
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /*IPSEC*/
 
 	/* mark socket state as dead */
 	if (in_pcb_checkstate(inp, WNT_STOPUSING, 1) != WNT_STOPUSING)
-		panic("in_pcbdetach so=%x prot=%x couldn't set to STOPUSING\n", so, so->so_proto->pr_protocol);
+		panic("in_pcbdetach so=%p prot=%x couldn't set to STOPUSING\n", so, so->so_proto->pr_protocol);
 
 #if TEMPDEBUG
 	if (so->cached_in_sock_layer)
@@ -711,29 +899,17 @@ in_pcbdetach(inp)
 	    printf("in_pcbdetach for allocated socket %x flags=%x\n", so, so->so_flags);
 #endif
 	if ((so->so_flags & SOF_PCBCLEARING) == 0) {
+		struct rtentry *rt;
+
 		inp->inp_vflag = 0;
 		if (inp->inp_options) 
 			(void)m_free(inp->inp_options);
-		if (rt) {
-			/* 
-	 	 	* route deletion requires reference count to be <= zero 
-	 		*/
-			lck_mtx_lock(rt_mtx);
-			if ((rt->rt_flags & RTF_DELCLONE) &&
-	    	  	   (rt->rt_flags & RTF_WASCLONED) &&
-	    		   (rt->rt_refcnt <= 1)) {
-				rtunref(rt);
-				rt->rt_flags &= ~RTF_UP;
-				rtrequest_locked(RTM_DELETE, rt_key(rt),
-					  rt->rt_gateway, rt_mask(rt),
-					  rt->rt_flags, (struct rtentry **)0);
-			}
-			else {
-				rtfree_locked(rt);
-				inp->inp_route.ro_rt = 0;
-			}
-			lck_mtx_unlock(rt_mtx);
+		lck_mtx_lock(rt_mtx);
+		if ((rt = inp->inp_route.ro_rt) != NULL) {
+			inp->inp_route.ro_rt = NULL;
+			rtfree_locked(rt);
 		}
+		lck_mtx_unlock(rt_mtx);
 		ip_freemoptions(inp->inp_moptions);
 		inp->inp_moptions = NULL;
 		sofreelastref(so, 0);
@@ -744,21 +920,21 @@ in_pcbdetach(inp)
 
 
 void 
-in_pcbdispose(inp) 
-	struct inpcb *inp;
+in_pcbdispose(struct inpcb *inp) 
 {
 	struct socket *so = inp->inp_socket;
 	struct inpcbinfo *ipi = inp->inp_pcbinfo;
 
 #if TEMPDEBUG
 	if (inp->inp_state != INPCB_STATE_DEAD) {
-		printf("in_pcbdispose: not dead yet? so=%x\n", so);
+		printf("in_pcbdispose: not dead yet? so=%p\n", so);
 	}
 #endif
 
 	if (so && so->so_usecount != 0)
-		panic("in_pcbdispose: use count=%x so=%x\n", so->so_usecount, so);
+		panic("in_pcbdispose: use count=%x so=%p\n", so->so_usecount, so);
 
+	lck_rw_assert(ipi->mtx, LCK_RW_ASSERT_EXCLUSIVE);
 
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	/*### access ipi in in_pcbremlists */
@@ -769,22 +945,25 @@ in_pcbdispose(inp)
 			sofreelastref(so, 0);
 			if (so->so_rcv.sb_cc || so->so_snd.sb_cc) {
 #if TEMPDEBUG
-				printf("in_pcbdispose sb not cleaned up so=%x rc_cci=%x snd_cc=%x\n",
+				printf("in_pcbdispose sb not cleaned up so=%p rc_cci=%x snd_cc=%x\n",
 				       	so, so->so_rcv.sb_cc, so->so_snd.sb_cc);	
 #endif
 				sbrelease(&so->so_rcv);
 				sbrelease(&so->so_snd);
 			}
 			if (so->so_head != NULL)
-				panic("in_pcbdispose, so=%x head still exist\n", so);
+				panic("in_pcbdispose, so=%p head still exist\n", so);
   			lck_mtx_unlock(inp->inpcb_mtx);	
   			lck_mtx_free(inp->inpcb_mtx, ipi->mtx_grp);	
 		}
 		so->so_flags |= SOF_PCBCLEARING; /* makes sure we're not called twice from so_close */
-	     	so->so_saved_pcb = (caddr_t) inp;
+		so->so_saved_pcb = (caddr_t) inp;
 		so->so_pcb = 0; 
 		inp->inp_socket = 0;
-		inp->reserved[0] = so;
+		inp->reserved[0] = (u_int32_t)so;
+#if CONFIG_MACF_NET
+		mac_inpcb_label_destroy(inp);
+#endif
 		if (so->cached_in_sock_layer == 0) {
 			zfree(ipi->ipi_zone, inp);
 		}
@@ -792,7 +971,7 @@ in_pcbdispose(inp)
 	}
 #if TEMPDEBUG
 	else
-		printf("in_pcbdispose: no socket for inp=%x\n", inp);
+		printf("in_pcbdispose: no socket for inp=%p\n", inp);
 #endif
 }
 
@@ -805,14 +984,16 @@ in_pcbdispose(inp)
  * except through a kernel programming error, so it is acceptable to panic
  * (or in this case trap) if the PCB is invalid.  (Actually, we don't trap
  * because there actually /is/ a programming error somewhere... XXX)
+ *
+ * Returns:	0			Success
+ *		ENOBUFS			No buffer space available
+ *		ECONNRESET		Connection reset
  */
 int
-in_setsockaddr(so, nam)
-	struct socket *so;
-	struct sockaddr **nam;
+in_setsockaddr(struct socket *so, struct sockaddr **nam)
 {
-	register struct inpcb *inp;
-	register struct sockaddr_in *sin;
+	struct inpcb *inp;
+	struct sockaddr_in *sin;
 
 	/*
 	 * Do the malloc first in case it blocks.
@@ -837,12 +1018,10 @@ in_setsockaddr(so, nam)
 }
 
 int
-in_setpeeraddr(so, nam)
-	struct socket *so;
-	struct sockaddr **nam;
+in_setpeeraddr(struct socket *so, struct sockaddr **nam)
 {
 	struct inpcb *inp;
-	register struct sockaddr_in *sin;
+	struct sockaddr_in *sin;
 
 	/*
 	 * Do the malloc first in case it blocks.
@@ -867,10 +1046,8 @@ in_setpeeraddr(so, nam)
 }
 
 void
-in_pcbnotifyall(pcbinfo, faddr, errno, notify)
-	struct inpcbinfo *pcbinfo;
-	struct in_addr faddr;
-	void (*notify) (struct inpcb *, int);
+in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr,
+		int errno, void (*notify)(struct inpcb *, int))
 {
 	struct inpcb *inp;
 
@@ -939,14 +1116,18 @@ in_pcbpurgeif0(
  * (by a redirect), time to try a default gateway again.
  */
 void
-in_losing(inp)
-	struct inpcb *inp;
+in_losing(struct inpcb *inp)
 {
-	register struct rtentry *rt;
+	struct rtentry *rt;
 	struct rt_addrinfo info;
 
 	if ((rt = inp->inp_route.ro_rt)) {
 		lck_mtx_lock(rt_mtx);
+		/* Check again, this time while holding the lock */
+		if ((rt = inp->inp_route.ro_rt) == NULL) {
+			lck_mtx_unlock(rt_mtx);
+			return;
+		}
 		bzero((caddr_t)&info, sizeof(info));
 		info.rti_info[RTAX_DST] =
 			(struct sockaddr *)&inp->inp_route.ro_dst;
@@ -957,8 +1138,11 @@ in_losing(inp)
 			(void) rtrequest_locked(RTM_DELETE, rt_key(rt),
 				rt->rt_gateway, rt_mask(rt), rt->rt_flags,
 				(struct rtentry **)0);
-		inp->inp_route.ro_rt = 0;
-		rtfree_locked(rt);
+		/* if the address is gone keep the old route in the pcb */
+		if ((ifa_foraddr(inp->inp_laddr.s_addr)) != 0) {
+			inp->inp_route.ro_rt = 0;
+			rtfree_locked(rt);
+		}
 		lck_mtx_unlock(rt_mtx);
 		/*
 		 * A new route can be allocated
@@ -972,15 +1156,22 @@ in_losing(inp)
  * and allocate a (hopefully) better one.
  */
 void
-in_rtchange(inp, errno)
-	register struct inpcb *inp;
-	int errno;
+in_rtchange(struct inpcb *inp, __unused int errno)
 {
-	if (inp->inp_route.ro_rt) {
+	struct rtentry *rt;
+
+	if ((rt = inp->inp_route.ro_rt) != NULL) {
 		if ((ifa_foraddr(inp->inp_laddr.s_addr)) == 0)
 			return; /* we can't remove the route now. not sure if still ok to use src */
-		rtfree(inp->inp_route.ro_rt);
-		inp->inp_route.ro_rt = 0;
+		lck_mtx_lock(rt_mtx);
+		/* Check again, this time while holding the lock */
+		if ((rt = inp->inp_route.ro_rt) == NULL) {
+			lck_mtx_unlock(rt_mtx);
+			return;
+		}
+		rtfree_locked(rt);
+		inp->inp_route.ro_rt = NULL;
+		lck_mtx_unlock(rt_mtx);
 		/*
 		 * A new route can be allocated the next time
 		 * output is attempted.
@@ -992,13 +1183,10 @@ in_rtchange(inp, errno)
  * Lookup a PCB based on the local address and port.
  */
 struct inpcb *
-in_pcblookup_local(pcbinfo, laddr, lport_arg, wild_okay)
-	struct inpcbinfo *pcbinfo;
-	struct in_addr laddr;
-	u_int lport_arg;
-	int wild_okay;
+in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
+		   unsigned int lport_arg, int wild_okay)
 {
-	register struct inpcb *inp;
+	struct inpcb *inp;
 	int matchwild = 3, wildcard;
 	u_short lport = lport_arg;
 
@@ -1093,10 +1281,10 @@ in_pcblookup_hash(
 	struct in_addr laddr,
 	u_int lport_arg,
 	int wildcard,
-	struct ifnet *ifp)
+	__unused struct ifnet *ifp)
 {
 	struct inpcbhead *head;
-	register struct inpcb *inp;
+	struct inpcb *inp;
 	u_short fport = fport_arg, lport = lport_arg;
 
 	/*
@@ -1161,18 +1349,18 @@ in_pcblookup_hash(
 					}
 				}
 				else if (inp->inp_laddr.s_addr == INADDR_ANY) {
-#if defined(INET6)
+#if INET6
 					if (INP_CHECK_SOCKAF(inp->inp_socket,
 							     AF_INET6))
 						local_wild_mapped = inp;
 					else
-#endif /* defined(INET6) */
+#endif /* INET6 */
 					local_wild = inp;
 				}
 			}
 		}
-#if defined(INET6)
 		if (local_wild == NULL) {
+#if INET6
 			if (local_wild_mapped != NULL) {
 				if (in_pcb_checkstate(local_wild_mapped, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
 					lck_rw_done(pcbinfo->mtx);
@@ -1183,10 +1371,10 @@ in_pcblookup_hash(
 	    				return(NULL);
 				}
 			}
+#endif /* INET6 */
 			lck_rw_done(pcbinfo->mtx);
 			return (NULL);
 		}
-#endif /* defined(INET6) */
 		if (in_pcb_checkstate(local_wild, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
 			lck_rw_done(pcbinfo->mtx);
 			return (local_wild);
@@ -1208,9 +1396,7 @@ in_pcblookup_hash(
  * Insert PCB onto various hash lists.
  */
 int
-in_pcbinshash(inp, locked)
-	struct inpcb *inp;
-	int locked;	/* list already locked exclusive */
+in_pcbinshash(struct inpcb *inp, int locked)
 {
 	struct inpcbhead *pcbhash;
 	struct inpcbporthead *pcbporthash;
@@ -1277,8 +1463,7 @@ in_pcbinshash(inp, locked)
  * not change after in_pcbinshash() has been called.
  */
 void
-in_pcbrehash(inp)
-	struct inpcb *inp;
+in_pcbrehash(struct inpcb *inp)
 {
 	struct inpcbhead *head;
 	u_int32_t hashkey_faddr;
@@ -1302,8 +1487,7 @@ in_pcbrehash(inp)
  */
 //###LOCK must be called with list lock held
 void
-in_pcbremlists(inp)
-	struct inpcb *inp;
+in_pcbremlists(struct inpcb *inp)
 {
 	inp->inp_gencnt = ++inp->inp_pcbinfo->ipi_gencnt;
 
@@ -1506,13 +1690,13 @@ in_pcb_detach_port(
 	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 	
 	if (so != &pcbinfo->nat_dummy_socket)
-		panic("in_pcb_detach_port: not a dummy_sock: so=%x, inp=%x\n", so, inp);
+		panic("in_pcb_detach_port: not a dummy_sock: so=%p, inp=%p\n", so, inp);
 	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 	/*### access ipi in in_pcbremlists */
 	in_pcbremlists(inp);
 	
 	inp->inp_socket = 0;
-	inp->reserved[0] = so;
+	inp->reserved[0] = (u_int32_t) so;
 	zfree(pcbinfo->ipi_zone, inp);
     	pcbinfo->nat_dummy_socket.so_pcb = (caddr_t)pcbinfo->nat_dummy_pcb; /* restores dummypcb */
 }
@@ -1522,8 +1706,7 @@ in_pcb_letgo_port(struct inpcbinfo *pcbinfo, struct in_addr laddr, u_short lport
 		       struct in_addr faddr, u_short fport, u_char owner_id)
 {
     struct inpcbhead *head;
-    register struct inpcb *inp;
-
+    struct inpcb *inp;
 
     /*
      * First look for an exact match.
@@ -1678,12 +1861,15 @@ void  in_pcb_nat_init(struct inpcbinfo *pcbinfo, int afamily,
     struct proc *p = current_proc();
 
     bzero(&pcbinfo->nat_dummy_socket, sizeof(struct socket));
+#if CONFIG_MACF_NET
+    mac_socket_label_init(&pcbinfo->nat_dummy_socket, M_WAITOK);
+#endif
     pcbinfo->nat_dummy_socket.so_proto = pffindproto_locked(afamily, pfamily, protocol);
     pcbinfo->all_owners = 0;
     stat = in_pcballoc(&pcbinfo->nat_dummy_socket, pcbinfo, p);
     if (stat)
-        panic("in_pcb_nat_init: can't alloc fakepcb err=%\n", stat);
-    pcbinfo->nat_dummy_pcb = pcbinfo->nat_dummy_socket.so_pcb;
+        panic("in_pcb_nat_init: can't alloc fakepcb err=%d\n", stat);
+    pcbinfo->nat_dummy_pcb = (struct  inpcb *)pcbinfo->nat_dummy_socket.so_pcb;
 }
 
 /* Mechanism used to defer the memory release of PCBs
@@ -1694,12 +1880,11 @@ void  in_pcb_nat_init(struct inpcbinfo *pcbinfo, int afamily,
 */
 int
 in_pcb_checkstate(struct inpcb *pcb, int mode, int locked)
-
 {
 
 	volatile UInt32 *wantcnt	= (volatile UInt32 *)&pcb->inp_wantcnt;
-    	UInt32 origwant;
-    	UInt32 newwant;
+	UInt32 origwant;
+	UInt32 newwant;
 
 	switch (mode) {
 
@@ -1712,7 +1897,7 @@ in_pcb_checkstate(struct inpcb *pcb, int mode, int locked)
 			pcb->inp_state = INPCB_STATE_DEAD;
 stopusing:
 			if (pcb->inp_socket->so_usecount < 0)
-				panic("in_pcb_checkstate STOP pcb=%x so=%x usecount is negative\n", pcb, pcb->inp_socket);
+				panic("in_pcb_checkstate STOP pcb=%p so=%p usecount is negative\n", pcb, pcb->inp_socket);
 			if (locked == 0)
 				socket_unlock(pcb->inp_socket, 1);
 
@@ -1721,7 +1906,7 @@ stopusing:
 				return (WNT_STOPUSING);
 			newwant = 0xffff;			
 			if ((UInt16) origwant == 0) {/* try to mark it as unsuable now */
-    				OSCompareAndSwap(origwant, newwant, (UInt32 *) wantcnt) ;
+    				OSCompareAndSwap(origwant, newwant, wantcnt) ;
 			}
 			return (WNT_STOPUSING);
 			break;
@@ -1735,11 +1920,11 @@ stopusing:
 			do {
 				origwant = *wantcnt;
         			if ((UInt16) origwant == 0xffff ) {/* should stop using */
-//					printf("in_pcb_checkstate: ACQ PCB was STOPUSING while release. odd pcb=%x\n", pcb);
+//					printf("in_pcb_checkstate: ACQ PCB was STOPUSING while release. odd pcb=%p\n", pcb);
 					return (WNT_STOPUSING);
 				}
 				newwant = origwant + 1;		
-			} while (!OSCompareAndSwap(origwant, newwant, (UInt32 *) wantcnt));
+			} while (!OSCompareAndSwap(origwant, newwant, wantcnt));
 			return (WNT_ACQUIRE);
 			break;
 
@@ -1753,22 +1938,22 @@ stopusing:
 			do {
 				origwant = *wantcnt;
         			if ((UInt16) origwant == 0x0 ) 
-					panic("in_pcb_checkstate pcb=%x release with zero count", pcb);
+					panic("in_pcb_checkstate pcb=%p release with zero count", pcb);
         			if ((UInt16) origwant == 0xffff ) {/* should stop using */
 #if TEMPDEBUG
-					printf("in_pcb_checkstate: REL PCB was STOPUSING while release. odd pcb=%x\n", pcb);
+					printf("in_pcb_checkstate: REL PCB was STOPUSING while release. odd pcb=%p\n", pcb);
 #endif
 					if (locked == 0)
 						socket_unlock(pcb->inp_socket, 1);
 					return (WNT_STOPUSING);
 				}
 				newwant = origwant - 1;		
-			} while (!OSCompareAndSwap(origwant, newwant, (UInt32 *) wantcnt));
+			} while (!OSCompareAndSwap(origwant, newwant, wantcnt));
 
 			if (pcb->inp_state == INPCB_STATE_DEAD) 
 				goto stopusing;
 			if (pcb->inp_socket->so_usecount < 0)
-				panic("in_pcb_checkstate RELEASE pcb=%x so=%x usecount is negative\n", pcb, pcb->inp_socket);
+				panic("in_pcb_checkstate RELEASE pcb=%p so=%p usecount is negative\n", pcb, pcb->inp_socket);
 				
 			if (locked == 0)
 				socket_unlock(pcb->inp_socket, 1);
@@ -1777,7 +1962,7 @@ stopusing:
 
 		default:
 
-			panic("in_pcb_checkstate: so=%x not a valid state =%x\n", pcb->inp_socket, mode);
+			panic("in_pcb_checkstate: so=%p not a valid state =%x\n", pcb->inp_socket, mode);
 	}
 
 	/* NOTREACHED */
@@ -1816,13 +2001,3 @@ inpcb_to_compat(
 	inp_compat->inp_depend6.inp6_hops = inp->inp_depend6.inp6_hops;
 }
 
-#ifndef __APPLE__
-prison_xinpcb(struct proc *p, struct inpcb *inp)
-{
-	if (!p->p_prison)
-		return (0);
-	if (ntohl(inp->inp_laddr.s_addr) == p->p_prison->pr_ip)
-		return (0);
-	return (1);
-}
-#endif
diff --git a/bsd/netinet/in_pcb.h b/bsd/netinet/in_pcb.h
index 9f8b77058..3fd86be88 100644
--- a/bsd/netinet/in_pcb.h
+++ b/bsd/netinet/in_pcb.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993
@@ -54,6 +60,12 @@
  *	@(#)in_pcb.h	8.1 (Berkeley) 6/10/93
  * $FreeBSD: src/sys/netinet/in_pcb.h,v 1.32.2.4 2001/08/13 16:26:17 ume Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2007 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _NETINET_IN_PCB_H_
 #define _NETINET_IN_PCB_H_
@@ -104,6 +116,9 @@ struct in_addr_4in6 {
  * stable.
  */
 struct	icmp6_filter;
+#if CONFIG_MACF_NET
+struct	label;
+#endif
 
 struct inpcb {
 	LIST_ENTRY(inpcb) inp_hash;	/* hash list */
@@ -116,7 +131,7 @@ struct inpcb {
 	struct	inpcbinfo *inp_pcbinfo;	/* PCB list info */
 	struct	socket *inp_socket;	/* back pointer to socket */
 	u_char	nat_owner;		/* Used to NAT TCP/UDP traffic */
-	u_long  nat_cookie;		/* Cookie stored and returned to NAT */
+	u_int32_t nat_cookie;		/* Cookie stored and returned to NAT */
 	LIST_ENTRY(inpcb) inp_portlist;	/* list for this PCB's local port */
 	struct	inpcbport *inp_phd;	/* head of this list */
 	inp_gen_t inp_gencnt;		/* generation count of this instance */
@@ -177,7 +192,17 @@ struct inpcb {
 #else
 	void	  *inpcb_mtx;
 #endif
-	u_long	reserved[2];		/* For future use */
+	u_int32_t reserved[4];		/* future use (some already used) */
+#if CONFIG_MACF_NET
+	struct label *inp_label;	/* MAC label */
+#endif
+#if CONFIG_FORCE_OUT_IFP
+#ifdef _KERN_SYS_KERNELTYPES_H_
+	ifnet_t	pdp_ifp;
+#else
+	void	*pdp_ifp;
+#endif /* _KERN_SYS_KERNELTYPES_H_ */
+#endif /* CONFIG_EMBEDDED */
 };
 
 #endif /* KERNEL_PRIVATE */
@@ -202,32 +227,47 @@ struct inpcb {
  * CAUTION: Many fields may not be filled out. Fewer may be filled out
  * in the future. Code defensively.
  */
+
+#pragma pack(4)
+
+#if defined(__LP64__)
+struct _inpcb_list_entry {
+    u_int32_t	le_next;
+    u_int32_t	le_prev;
+};
+#define _INPCB_PTR(x)		u_int32_t
+#define _INPCB_LIST_ENTRY(x)	struct _inpcb_list_entry
+#else
+#define _INPCB_PTR(x)		x
+#define _INPCB_LIST_ENTRY(x)	LIST_ENTRY(x)	
+#endif
+
 #ifdef KERNEL_PRIVATE
 struct inpcb_compat {
 #else
 struct inpcbinfo;
 struct inpcbport;
 struct mbuf;
-struct	ip6_pktopts;
-struct	ip6_moptions;
-struct	icmp6_filter;
+struct ip6_pktopts;
+struct ip6_moptions;
+struct icmp6_filter;
 struct inpcbpolicy;
 
 struct inpcb {
 #endif /* KERNEL_PRIVATE */
-	LIST_ENTRY(inpcb) inp_hash;	/* hash list */
+	_INPCB_LIST_ENTRY(inpcb) inp_hash;	/* hash list */
 	struct	in_addr reserved1;	/* APPLE reserved: inp_faddr defined in protcol indep. part */
 	struct	in_addr reserved2; /* APPLE reserved */
 	u_short	inp_fport;		/* foreign port */
 	u_short	inp_lport;		/* local port */
-	LIST_ENTRY(inpcb) inp_list;	/* list for all PCBs of this proto */
-	caddr_t	inp_ppcb;		/* pointer to per-protocol pcb */
-	struct	inpcbinfo *inp_pcbinfo;	/* PCB list info */
-	void*	inp_socket;	/* back pointer to socket */
+	_INPCB_LIST_ENTRY(inpcb) inp_list;	/* list for all PCBs of this proto */
+	_INPCB_PTR(caddr_t)	inp_ppcb;	/* pointer to per-protocol pcb */
+	_INPCB_PTR(struct inpcbinfo *)	inp_pcbinfo;	/* PCB list info */
+	_INPCB_PTR(void *)	inp_socket;	/* back pointer to socket */
 	u_char	nat_owner;		/* Used to NAT TCP/UDP traffic */
-	u_long  nat_cookie;		/* Cookie stored and returned to NAT */
-	LIST_ENTRY(inpcb) inp_portlist;	/* list for this PCB's local port */
-	struct	inpcbport *inp_phd;	/* head of this list */
+	u_int32_t nat_cookie;		/* Cookie stored and returned to NAT */
+	_INPCB_LIST_ENTRY(inpcb) inp_portlist;	/* list for this PCB's local port */
+	_INPCB_PTR(struct inpcbport *)	inp_phd;		/* head of this list */
 	inp_gen_t inp_gencnt;		/* generation count of this instance */
 	int	inp_flags;		/* generic IP/datagram flags */
 	u_int32_t inp_flow;
@@ -256,23 +296,23 @@ struct inpcb {
 		/* type of service proto */
 		u_char inp4_ip_tos;
 		/* IP options */
-		struct mbuf *inp4_options;
+		_INPCB_PTR(struct mbuf *) inp4_options;
 		/* IP multicast options */
-		struct ip_moptions *inp4_moptions;
+		_INPCB_PTR(struct ip_moptions *) inp4_moptions;
 	} inp_depend4;
 
 	struct {
 		/* IP options */
-		struct mbuf *inp6_options;
+		_INPCB_PTR(struct mbuf *)	inp6_options;
 		u_int8_t	inp6_hlim;
 		u_int8_t	unused_uint8_1;
 		ushort	unused_uint16_1;
 		/* IP6 options for outgoing packets */
-		struct	ip6_pktopts *inp6_outputopts;
+		_INPCB_PTR(struct ip6_pktopts *)	inp6_outputopts;
 		/* IP multicast options */
-		struct	ip6_moptions *inp6_moptions;
+		_INPCB_PTR(struct ip6_moptions *)	inp6_moptions;
 		/* ICMPv6 code type filter */
-		struct	icmp6_filter *inp6_icmp6filt;
+		_INPCB_PTR(struct icmp6_filter *)	inp6_icmp6filt;
 		/* IPV6_CHECKSUM setsockopt */
 		int	inp6_cksum;
 		u_short	inp6_ifindex;
@@ -280,13 +320,13 @@ struct inpcb {
 	} inp_depend6;
 
 	int	hash_element;           /* Array index of pcb's hash list    */
-	caddr_t inp_saved_ppcb;		/* place to save pointer while cached */
-	struct inpcbpolicy *inp_sp;
-	u_long	reserved[3];		/* For future use */
+	_INPCB_PTR(caddr_t)	inp_saved_ppcb;	/* place to save pointer while cached */
+	_INPCB_PTR(struct inpcbpolicy *)	inp_sp;
+	u_int32_t	reserved[3];	/* For future use */
 };
 
 struct	xinpcb {
-	size_t	xi_len;		/* length of this structure */
+	u_int32_t	xi_len;		/* length of this structure */
 #ifdef KERNEL_PRIVATE
 	struct	inpcb_compat xi_inp;
 #else
@@ -297,12 +337,14 @@ struct	xinpcb {
 };
 
 struct	xinpgen {
-	size_t	xig_len;	/* length of this structure */
+	u_int32_t xig_len;	/* length of this structure */
 	u_int	xig_count;	/* number of PCBs at this time */
 	inp_gen_t xig_gen;	/* generation count at this time */
 	so_gen_t xig_sogen;	/* socket generation count at this time */
 };
 
+#pragma pack()
+
 /*
  * These defines are for use with the inpcb.
  */
@@ -401,6 +443,7 @@ struct inpcbinfo {		/* XXX documentation, prefixes */
 #define  INP_INADDR_ANY 	0x800   /* local address wasn't specified */
 
 #define INP_RECVTTL		0x1000
+#define	INP_UDP_NOCKSUM		0x2000	/* Turn off outbound UDP checksum */
 
 #define IN6P_IPV6_V6ONLY	0x008000 /* restrict AF_INET6 socket for v6 */
 
@@ -480,6 +523,8 @@ int	in_pcbinshash(struct inpcb *, int);
 int	in_pcbladdr(struct inpcb *, struct sockaddr *, struct sockaddr_in **);
 struct inpcb *
 	in_pcblookup_local(struct inpcbinfo *, struct in_addr, u_int, int);
+struct inpcb *
+	in_pcblookup_local_and_cleanup(struct inpcbinfo *, struct in_addr, u_int, int);
 struct inpcb *
 	in_pcblookup_hash(struct inpcbinfo *,
 			       struct in_addr, u_int, struct in_addr, u_int,
@@ -528,6 +573,9 @@ in_pcb_rem_share_client(struct inpcbinfo *pcbinfo, u_char owner_id);
 void	in_pcbremlists(struct inpcb *inp);
 int 	in_pcb_ckeckstate(struct inpcb *, int, int);
 void	inpcb_to_compat(struct inpcb *inp, struct inpcb_compat *inp_compat);
+#if CONFIG_FORCE_OUT_IFP
+void	pdp_context_route_locked(ifnet_t ifp, struct route *ro);
+#endif
 
 #endif /* KERNEL */
 #endif /* KERNEL_PRIVATE */
diff --git a/bsd/netinet/in_proto.c b/bsd/netinet/in_proto.c
index d910aa342..f08af184f 100644
--- a/bsd/netinet/in_proto.c
+++ b/bsd/netinet/in_proto.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -115,7 +121,7 @@ struct protosw inetsw[] = {
   0,		0,		0,	{ 0, 0 },	0,	{ 0 }
 },
 { SOCK_DGRAM,	&inetdomain,	IPPROTO_UDP,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK|PR_PCBLOCK,
-  udp_input,	0,		udp_ctlinput,	ip_ctloutput,
+  udp_input,	0,		udp_ctlinput,	udp_ctloutput,
   0,
   udp_init,	0,		udp_slowtimo,		0,
   0,
@@ -123,10 +129,10 @@ struct protosw inetsw[] = {
   udp_lock,	udp_unlock,	udp_getlock,	{ 0, 0 },	0,	{ 0 }
 },
 { SOCK_STREAM,	&inetdomain,	IPPROTO_TCP, 
-	PR_CONNREQUIRED|PR_IMPLOPCL|PR_WANTRCVD|PR_PCBLOCK|PR_PROTOLOCK|PR_DISPOSE,
+	PR_CONNREQUIRED|PR_WANTRCVD|PR_PCBLOCK|PR_PROTOLOCK|PR_DISPOSE,
   tcp_input,	0,		tcp_ctlinput,	tcp_ctloutput,
   0,
-  tcp_init,	tcp_fasttimo,	tcp_slowtimo,	tcp_drain,
+  tcp_init,	0,	tcp_slowtimo,	tcp_drain,
   0,
   &tcp_usrreqs,
   tcp_lock,	tcp_unlock,	tcp_getlock,	{ 0, 0 },	0,	{ 0 }
@@ -163,6 +169,7 @@ struct protosw inetsw[] = {
   &rip_usrreqs,
   0,		rip_unlock,	0,	{ 0, 0 },	0,	{ 0 }
 },
+#if MROUTING
 { SOCK_RAW,	&inetdomain,	IPPROTO_RSVP,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
   rsvp_input,	0,		0,		rip_ctloutput,
   0,
@@ -171,6 +178,7 @@ struct protosw inetsw[] = {
   &rip_usrreqs,
   0,		rip_unlock,		0,	{ 0, 0 },	0,	{ 0 }
 },
+#endif /* MROUTING */
 #if IPSEC
 { SOCK_RAW,	&inetdomain,	IPPROTO_AH,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK,
   ah4_input,	0,	 	0,		0,
@@ -262,7 +270,7 @@ extern int in_inithead(void **, int);
 
 int in_proto_count = (sizeof (inetsw) / sizeof (struct protosw));
 
-extern void in_dinit(void);
+extern void in_dinit(void) __attribute__((section("__TEXT, initcode")));
 /* A routing init function, and a header size */
 struct domain inetdomain =
     { AF_INET, 
@@ -284,19 +292,19 @@ struct domain inetdomain =
 
 DOMAIN_SET(inet);
 
-SYSCTL_NODE(_net,      PF_INET,		inet,	CTLFLAG_RW, 0,
+SYSCTL_NODE(_net,      PF_INET,		inet,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,
 	"Internet Family");
 
-SYSCTL_NODE(_net_inet, IPPROTO_IP,	ip,	CTLFLAG_RW, 0,	"IP");
-SYSCTL_NODE(_net_inet, IPPROTO_ICMP,	icmp,	CTLFLAG_RW, 0,	"ICMP");
-SYSCTL_NODE(_net_inet, IPPROTO_UDP,	udp,	CTLFLAG_RW, 0,	"UDP");
-SYSCTL_NODE(_net_inet, IPPROTO_TCP,	tcp,	CTLFLAG_RW, 0,	"TCP");
-SYSCTL_NODE(_net_inet, IPPROTO_IGMP,	igmp,	CTLFLAG_RW, 0,	"IGMP");
+SYSCTL_NODE(_net_inet, IPPROTO_IP,	ip,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"IP");
+SYSCTL_NODE(_net_inet, IPPROTO_ICMP,	icmp,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"ICMP");
+SYSCTL_NODE(_net_inet, IPPROTO_UDP,	udp,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"UDP");
+SYSCTL_NODE(_net_inet, IPPROTO_TCP,	tcp,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"TCP");
+SYSCTL_NODE(_net_inet, IPPROTO_IGMP,	igmp,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"IGMP");
 #if IPSEC
-SYSCTL_NODE(_net_inet, IPPROTO_AH,	ipsec,	CTLFLAG_RW, 0,	"IPSEC");
+SYSCTL_NODE(_net_inet, IPPROTO_AH,	ipsec,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"IPSEC");
 #endif /* IPSEC */
-SYSCTL_NODE(_net_inet, IPPROTO_RAW,	raw,	CTLFLAG_RW, 0,	"RAW");
+SYSCTL_NODE(_net_inet, IPPROTO_RAW,	raw,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"RAW");
 #if IPDIVERT
-SYSCTL_NODE(_net_inet, IPPROTO_DIVERT,	div,	CTLFLAG_RW, 0,	"DIVERT");
+SYSCTL_NODE(_net_inet, IPPROTO_DIVERT,	div,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"DIVERT");
 #endif
 
diff --git a/bsd/netinet/in_rmx.c b/bsd/netinet/in_rmx.c
index 3a7afc3cd..371a6db64 100644
--- a/bsd/netinet/in_rmx.c
+++ b/bsd/netinet/in_rmx.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1994, 1995 Massachusetts Institute of Technology
@@ -77,14 +83,15 @@
 #include <netinet/in.h>
 #include <netinet/in_var.h>
 
+extern int tvtohz(struct timeval *);
 extern int	in_inithead(void **head, int off);
+extern u_long route_generation;
 
 #ifdef __APPLE__
 static void in_rtqtimo(void *rock);
 #endif
 
 #define RTPRF_OURS		RTF_PROTO3	/* set on routes we manage */
-extern lck_mtx_t *rt_mtx;
 
 /*
  * Do what we need to do when inserting a route.
@@ -231,35 +238,42 @@ SYSCTL_INT(_net_inet_ip, OID_AUTO, use_route_genid, CTLFLAG_RW,
  * timed out.
  */
 static void
-in_clsroute(struct radix_node *rn, struct radix_node_head *head)
+in_clsroute(struct radix_node *rn, __unused struct radix_node_head *head)
 {
 	struct rtentry *rt = (struct rtentry *)rn;
-	struct timeval timenow;
 
-	if(!(rt->rt_flags & RTF_UP))
+	if (!(rt->rt_flags & RTF_UP))
 		return;		/* prophylactic measures */
 
-	if((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST)
+	if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST)
 		return;
 
-	if((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS))
-	   != RTF_WASCLONED)
+	if ((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS)) != RTF_WASCLONED)
 		return;
 
 	/*
-	 * As requested by David Greenman:
-	 * If rtq_reallyold is 0, just delete the route without
-	 * waiting for a timeout cycle to kill it.
+	 * Delete the route immediately if RTF_DELCLONE is set or
+	 * if route caching is disabled (rtq_reallyold set to 0).
+	 * Otherwise, let it expire and be deleted by in_rtqkill().
 	 */
-	if(rtq_reallyold != 0) {
+	if ((rt->rt_flags & RTF_DELCLONE) || rtq_reallyold == 0) {
+		/*
+		 * Delete the route from the radix tree but since we are
+		 * called when the route's reference count is 0, don't
+		 * deallocate it until we return from this routine by
+		 * telling rtrequest that we're interested in it.
+		 */
+		if (rtrequest_locked(RTM_DELETE, (struct sockaddr *)rt_key(rt),
+		    rt->rt_gateway, rt_mask(rt), rt->rt_flags, &rt) == 0) {
+			/* Now let the caller free it */
+			rtunref(rt);
+		}
+	} else {
+		struct timeval timenow;
+
 		getmicrotime(&timenow);
 		rt->rt_flags |= RTPRF_OURS;
 		rt->rt_rmx.rmx_expire = timenow.tv_sec + rtq_reallyold;
-	} else {
-		rtrequest_locked(RTM_DELETE,
-			  (struct sockaddr *)rt_key(rt),
-			  rt->rt_gateway, rt_mask(rt),
-			  rt->rt_flags, 0);
 	}
 }
 
@@ -287,24 +301,25 @@ in_rtqkill(struct radix_node *rn, void *rock)
 
 	getmicrotime(&timenow);
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
-	if(rt->rt_flags & RTPRF_OURS) {
+
+	if (rt->rt_flags & RTPRF_OURS) {
 		ap->found++;
 
-		if(ap->draining || rt->rt_rmx.rmx_expire <= timenow.tv_sec) {
-			if(rt->rt_refcnt > 0)
+		if (ap->draining || rt->rt_rmx.rmx_expire <= timenow.tv_sec) {
+			if (rt->rt_refcnt > 0)
 				panic("rtqkill route really not free");
 
 			err = rtrequest_locked(RTM_DELETE,
 					(struct sockaddr *)rt_key(rt),
 					rt->rt_gateway, rt_mask(rt),
 					rt->rt_flags, 0);
-			if(err) {
+			if (err) {
 				log(LOG_WARNING, "in_rtqkill: error %d\n", err);
 			} else {
 				ap->killed++;
 			}
 		} else {
-			if(ap->updating
+			if (ap->updating
 			   && (rt->rt_rmx.rmx_expire - timenow.tv_sec
 			       > rtq_reallyold)) {
 				rt->rt_rmx.rmx_expire = timenow.tv_sec
@@ -336,12 +351,14 @@ in_rtqtimo(void *rock)
 	static time_t last_adjusted_timeout = 0;
 	struct timeval timenow;
 
+	lck_mtx_lock(rt_mtx);
+	/* Get the timestamp after we acquire the lock for better accuracy */
 	getmicrotime(&timenow);
+
 	arg.found = arg.killed = 0;
 	arg.rnh = rnh;
 	arg.nextstop = timenow.tv_sec + rtq_timeout;
 	arg.draining = arg.updating = 0;
-	lck_mtx_lock(rt_mtx);
 	rnh->rnh_walktree(rnh, in_rtqkill, &arg);
 
 	/*
@@ -472,6 +489,10 @@ in_ifadown(struct ifaddr *ifa, int delete)
 	if (ifa->ifa_addr->sa_family != AF_INET)
 		return 1;
 
+	/* trigger route cache reevaluation */
+	if (use_routegenid) 
+		route_generation++;
+
 	arg.rnh = rnh = rt_tables[AF_INET];
 	arg.ifa = ifa;
 	arg.del = delete;
diff --git a/bsd/netinet/in_systm.h b/bsd/netinet/in_systm.h
index ecfcc405d..bbed62112 100644
--- a/bsd/netinet/in_systm.h
+++ b/bsd/netinet/in_systm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/in_var.h b/bsd/netinet/in_var.h
index 1ec7420be..35826b528 100644
--- a/bsd/netinet/in_var.h
+++ b/bsd/netinet/in_var.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1985, 1986, 1993
@@ -65,6 +71,7 @@
 #endif
 
 #ifdef PRIVATE
+#include <net/route.h>
 
 /*
  * Interface address, Internet version.  One of these structures
@@ -73,32 +80,33 @@
  * of the structure and is assumed to be first.
  */
 struct in_ifaddr {
-	struct	ifaddr ia_ifa;		/* protocol-independent info */
-#define	ia_ifp		ia_ifa.ifa_ifp
-#define ia_flags	ia_ifa.ifa_flags
-					/* ia_{,sub}net{,mask} in host order */
-	u_long	ia_net;			/* network number of interface */
-	u_long	ia_netmask;		/* mask of net part */
-	u_long	ia_subnet;		/* subnet number, including net */
-	u_long	ia_subnetmask;		/* mask of subnet part */
-	struct	in_addr ia_netbroadcast; /* to recognize net broadcasts */
-	TAILQ_ENTRY(in_ifaddr) ia_link;	/* tailq macro glue */
-	struct	sockaddr_in ia_addr;	/* reserve space for interface name */
-	struct	sockaddr_in ia_dstaddr; /* reserve space for broadcast addr */
-#define	ia_broadaddr	ia_dstaddr
-	struct	sockaddr_in ia_sockmask; /* reserve space for general netmask */
+	struct ifaddr		ia_ifa;		/* protocol-independent info */
+#define	ia_ifp			ia_ifa.ifa_ifp
+#define	ia_flags		ia_ifa.ifa_flags
+						/* ia_{,sub}net{,mask} in host order */
+	u_long			ia_net;		/* network number of interface */
+	u_long			ia_netmask;	/* mask of net part */
+	u_long			ia_subnet;	/* subnet number, including net */
+	u_long			ia_subnetmask;	/* mask of subnet part */
+	struct in_addr		ia_netbroadcast; /* to recognize net broadcasts */
+	TAILQ_ENTRY(in_ifaddr)	ia_link;	/* tailq macro glue */
+	struct sockaddr_in	ia_addr;	/* reserve space for interface name */
+	struct sockaddr_in	ia_dstaddr;	/* reserve space for broadcast addr */
+#define	ia_broadaddr		ia_dstaddr
+	struct sockaddr_in	ia_sockmask;	/* reserve space for general netmask */
+#if CONFIG_FORCE_OUT_IFP
+	struct	rtentry		*ia_route; /* PDP context hack - a faux route we can use */
+#endif
 };
 #endif /* PRIVATE */
 
 struct	in_aliasreq {
-	char	ifra_name[IFNAMSIZ];		/* if name, e.g. "en0" */
-	struct	sockaddr_in ifra_addr;
-	struct	sockaddr_in ifra_broadaddr;
-#define ifra_dstaddr ifra_broadaddr
-	struct	sockaddr_in ifra_mask;
-#ifdef __APPLE__
-        u_long              dlt;
-#endif
+	char			ifra_name[IFNAMSIZ];	/* if name, e.g. "en0" */
+	struct sockaddr_in	ifra_addr;
+	struct sockaddr_in	ifra_broadaddr;
+#define ifra_dstaddr		ifra_broadaddr
+	struct sockaddr_in	ifra_mask;
+	u_int32_t		ifra_unused;	/* not used: used to be 'dlt' */
 };
 
 /*
@@ -106,21 +114,29 @@ struct	in_aliasreq {
  */
 struct kev_in_data {
 	struct net_event_data   link_data;
-	struct in_addr  ia_addr;
-	u_long	ia_net;			/* network number of interface */
-	u_long	ia_netmask;		/* mask of net part */
-	u_long	ia_subnet;		/* subnet number, including net */
-	u_long	ia_subnetmask;		/* mask of subnet part */
-	struct	in_addr ia_netbroadcast; /* to recognize net broadcasts */
-	struct  in_addr ia_dstaddr;
+	struct in_addr  	ia_addr;
+	u_int32_t		ia_net;		/* network number of interface */
+	u_int32_t		ia_netmask;	/* mask of net part */
+	u_int32_t		ia_subnet;	/* subnet number, including net */
+	u_int32_t		ia_subnetmask;	/* mask of subnet part */
+	struct in_addr		ia_netbroadcast;/* to recognize net broadcasts */
+	struct in_addr		ia_dstaddr;
 };
 
 struct kev_in_collision {
-	struct	net_event_data	link_data;	/* link colliding arp was received on */
-	struct	in_addr	ia_ipaddr;	/* IP address we and another node are using */
-	u_char	hw_len;	/* length of hardware address */
-	u_char	hw_addr[0];	/* variable length hardware address */
+	struct net_event_data	link_data;	/* link colliding arp was received on */
+	struct in_addr		ia_ipaddr;	/* IP address we and another node are using */
+	u_char			hw_len;		/* length of hardware address */
+	u_char			hw_addr[0];	/* variable length hardware address */
+};
+
+#ifdef __APPLE_API_PRIVATE
+struct kev_in_portinuse {
+	u_int16_t		port;		/* conflicting port number in host order */
+	u_int32_t		req_pid;	/* PID port requestor */
+	u_int32_t		reserved[2];	
 };
+#endif
 
 
 /*
@@ -137,6 +153,10 @@ struct kev_in_collision {
 #define KEV_INET_SIFNETMASK   6
 #define KEV_INET_ARPCOLLISION 7	/* use kev_in_collision */
 
+#ifdef __APPLE_API_PRIVATE
+#define KEV_INET_PORTINUSE    8	/* use ken_in_portinuse */
+#endif
+
 #ifdef KERNEL_PRIVATE
 /*
  * Given a pointer to an in_ifaddr (ifaddr),
@@ -154,6 +174,9 @@ extern	struct	in_addr zeroin_addr;
 extern	u_char	inetctlerrmap[];
 extern	lck_mtx_t *rt_mtx;
 
+extern int apple_hwcksum_tx;
+extern int apple_hwcksum_rx;
+
 /*
  * Macro for finding the interface (ifnet structure) corresponding to one
  * of our IP addresses.
@@ -252,7 +275,7 @@ do { \
 		    (addr).s_addr) \
 			break; \
 	} \
-	(inm) = ifma ? ifma->ifma_protospec : 0; \
+	(inm) = ifma ? ifma->ifma_protospec : NULL; \
 } while(0)
 
 /*
diff --git a/bsd/netinet/ip.h b/bsd/netinet/ip.h
index e134cb929..c21698f61 100644
--- a/bsd/netinet/ip.h
+++ b/bsd/netinet/ip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -115,10 +121,21 @@ struct ip {
 #define	IPTOS_THROUGHPUT	0x08
 #define	IPTOS_RELIABILITY	0x04
 #define	IPTOS_MINCOST		0x02
-/* ECN bits proposed by Sally Floyd */
-#define	IPTOS_CE		0x01	/* congestion experienced */
-#define	IPTOS_ECT		0x02	/* ECN-capable transport */
+#if 1
+/* ECN RFC3168 obsoletes RFC2481, and these will be deprecated soon. */
+#define	IPTOS_CE		0x01
+#define	IPTOS_ECT		0x02
+#endif
 
+/*
+ * ECN (Explicit Congestion Notification) codepoints in RFC3168
+ * mapped to the lower 2 bits of the TOS field.
+ */
+#define	IPTOS_ECN_NOTECT	0x00	/* not-ECT */
+#define	IPTOS_ECN_ECT1		0x01	/* ECN-capable transport (1) */
+#define	IPTOS_ECN_ECT0		0x02	/* ECN-capable transport (0) */
+#define	IPTOS_ECN_CE		0x03	/* congestion experienced */
+#define	IPTOS_ECN_MASK		0x03	/* ECN field mask */
 
 /*
  * Definitions for IP precedence (also in ip_tos) (hopefully unused)
diff --git a/bsd/netinet/ip6.h b/bsd/netinet/ip6.h
index c677f2b6a..fcdbd2d08 100644
--- a/bsd/netinet/ip6.h
+++ b/bsd/netinet/ip6.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: ip6.h,v 1.18 2001/03/29 05:34:30 itojun Exp $*/
 
diff --git a/bsd/netinet/ip_compat.h b/bsd/netinet/ip_compat.h
index 8fc0d1ea5..a04817ebe 100644
--- a/bsd/netinet/ip_compat.h
+++ b/bsd/netinet/ip_compat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1993-1997 by Darren Reed.
diff --git a/bsd/netinet/ip_divert.c b/bsd/netinet/ip_divert.c
index e3da4e2f9..12d193d69 100644
--- a/bsd/netinet/ip_divert.c
+++ b/bsd/netinet/ip_divert.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1993
@@ -54,7 +60,7 @@
  * $FreeBSD: src/sys/netinet/ip_divert.c,v 1.98 2004/08/17 22:05:54 andre Exp $
  */
 
-#ifndef INET
+#if !INET
 #error "IPDIVERT requires INET."
 #endif
 
@@ -73,6 +79,7 @@
 
 #include <net/if.h>
 #include <net/route.h>
+#include <net/kpi_protocol.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
@@ -84,6 +91,7 @@
 #include <netinet/ip_divert.h>
 
 #include <kern/zalloc.h>
+#include <libkern/OSAtomic.h>
 
 /*
  * Divert sockets
@@ -127,7 +135,7 @@ static u_long	div_sendspace = DIVSNDQ;	/* XXX sysctl ? */
 static u_long	div_recvspace = DIVRCVQ;	/* XXX sysctl ? */
 
 /* Optimization: have this preinitialized */
-static struct sockaddr_in divsrc = { sizeof(divsrc), AF_INET,  };
+static struct sockaddr_in divsrc = { sizeof(divsrc), AF_INET, 0, { 0 }, { 0,0,0,0,0,0,0,0 } };
 
 /* Internal functions */
 static int div_output(struct socket *so,
@@ -168,9 +176,11 @@ div_init(void)
 	if ((pcbinfo->mtx = lck_rw_alloc_init(pcbinfo->mtx_grp, pcbinfo->mtx_attr)) == NULL)
 		return;	/* pretty much dead if this fails... */
 
+#if IPFIREWALL
 	if (!IPFW_LOADED) {
 		load_ipfw();
 	}
+#endif
 }
 
 /*
@@ -180,7 +190,7 @@ div_init(void)
 void
 div_input(struct mbuf *m, __unused int off)
 {
-	ipstat.ips_noproto++;
+	OSAddAtomic(1, (SInt32*)&ipstat.ips_noproto);
 	m_freem(m);
 }
 
@@ -281,8 +291,8 @@ divert_packet(struct mbuf *m, int incoming, int port, int rule)
 		socket_unlock(sa, 1);
 	} else {
 		m_freem(m);
-		ipstat.ips_noproto++;
-		ipstat.ips_delivered--;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_noproto);
+		OSAddAtomic(-1, (SInt32*)&ipstat.ips_delivered);
         }
 	lck_rw_done(divcbinfo.mtx); 	
 }
@@ -296,14 +306,11 @@ divert_packet(struct mbuf *m, int incoming, int port, int rule)
  * ###LOCK  called in inet_proto mutex when from div_send. 
  */
 static int
-div_output(so, m, addr, control)
-	struct socket *so;
-	register struct mbuf *m;
-	struct sockaddr *addr;
-	struct mbuf *control;
+div_output(struct socket *so, struct mbuf *m, struct sockaddr *addr,
+	   struct mbuf *control)
 {
-	register struct inpcb *const inp = sotoinpcb(so);
-	register struct ip *const ip = mtod(m, struct ip *);
+	struct inpcb *const inp = sotoinpcb(so);
+	struct ip *const ip = mtod(m, struct ip *);
 	struct sockaddr_in *sin = (struct sockaddr_in *)addr;
 	int error = 0;
 
@@ -359,13 +366,16 @@ div_output(so, m, addr, control)
 		NTOHS(ip->ip_off);
 
 		/* Send packet to output processing */
-		ipstat.ips_rawout++;			/* XXX */
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_rawout);
 		socket_unlock(so, 0);
+#if CONFIG_MACF_NET
+		mac_mbuf_label_associate_inpcb(inp, m);
+#endif
 		error = ip_output(m,
 			    inp->inp_options, &inp->inp_route,
 			(so->so_options & SO_DONTROUTE) |
 			IP_ALLOWBROADCAST | IP_RAWOUTPUT,
-			inp->inp_moptions);
+			inp->inp_moptions, NULL);
 		socket_lock(so, 0);
 	} else {
 		struct	ifaddr *ifa;
@@ -411,6 +421,9 @@ div_output(so, m, addr, control)
 			ip->ip_sum = in_cksum(m, hlen);
 		}
 
+#if CONFIG_MACF_NET
+		mac_mbuf_label_associate_socket(so, m);
+#endif
 		/* Send packet to input processing */
 		proto_inject(PF_INET, m);
 	}
@@ -450,7 +463,7 @@ div_attach(struct socket *so, int proto, struct proc *p)
 	so->so_state |= SS_ISCONNECTED;
 
 #ifdef MORE_DICVLOCK_DEBUG
-	printf("div_attach: so=%x sopcb=%x lock=%x ref=%x\n",
+	printf("div_attach: so=%p sopcb=%p lock=%x ref=%x\n",
 			so, so->so_pcb, ((struct inpcb *)so->so_pcb)->inpcb_mtx, so->so_usecount);
 #endif
 	return 0;
@@ -462,12 +475,12 @@ div_detach(struct socket *so)
 	struct inpcb *inp;
 
 #ifdef MORE_DICVLOCK_DEBUG
-	printf("div_detach: so=%x sopcb=%x lock=%x ref=%x\n",
+	printf("div_detach: so=%p sopcb=%p lock=%x ref=%x\n",
 			so, so->so_pcb, ((struct inpcb *)so->so_pcb)->inpcb_mtx, so->so_usecount);
 #endif
 	inp = sotoinpcb(so);
 	if (inp == 0)
-		panic("div_detach: so=%x null inp\n", so);
+		panic("div_detach: so=%p null inp\n", so);
 	in_pcbdetach(inp);
 	inp->inp_state = INPCB_STATE_DEAD;
 	return 0;
@@ -525,7 +538,7 @@ div_send(struct socket *so, __unused int flags, struct mbuf *m, struct sockaddr
 	/* Packet must have a header (but that's about it) */
 	if (m->m_len < sizeof (struct ip) &&
 	    (m = m_pullup(m, sizeof (struct ip))) == 0) {
-		ipstat.ips_toosmall++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_toosmall);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -638,7 +651,7 @@ div_lock(struct socket *so, int refcount, int lr)
 	else lr_saved = lr;
 	
 #ifdef MORE_DICVLOCK_DEBUG
-	printf("div_lock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n",
+	printf("div_lock: so=%p sopcb=%p lock=%x ref=%x lr=%x\n",
 			so, 
 			so->so_pcb, 
 			so->so_pcb ? ((struct inpcb *)so->so_pcb)->inpcb_mtx : 0, 
@@ -648,17 +661,17 @@ div_lock(struct socket *so, int refcount, int lr)
 	if (so->so_pcb) {
 		lck_mtx_lock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	} else  {
-		panic("div_lock: so=%x NO PCB! lr=%x\n", so, lr_saved);
+		panic("div_lock: so=%p NO PCB! lr=%x\n", so, lr_saved);
 		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
 	}
 	
 	if (so->so_usecount < 0)
-		panic("div_lock: so=%x so_pcb=%x lr=%x ref=%x\n",
+		panic("div_lock: so=%p so_pcb=%p lr=%x ref=%x\n",
 		so, so->so_pcb, lr_saved, so->so_usecount);
 	
 	if (refcount)
 		so->so_usecount++;
-	so->lock_lr[so->next_lock_lr] = (u_int32_t *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = (u_int32_t)lr_saved;
 	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 
 	return (0);
@@ -677,7 +690,7 @@ div_unlock(struct socket *so, int refcount, int lr)
 
 	
 #ifdef MORE_DICVLOCK_DEBUG
-	printf("div_unlock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n",
+	printf("div_unlock: so=%p sopcb=%p lock=%x ref=%x lr=%x\n",
 			so, 
 			so->so_pcb, 
 			so->so_pcb ? ((struct inpcb *)so->so_pcb)->inpcb_mtx : 0, 
@@ -688,9 +701,9 @@ div_unlock(struct socket *so, int refcount, int lr)
 		so->so_usecount--;
 	
 	if (so->so_usecount < 0)
-		panic("div_unlock: so=%x usecount=%x\n", so, so->so_usecount);
+		panic("div_unlock: so=%p usecount=%x\n", so, so->so_usecount);
 	if (so->so_pcb == NULL) {
-		panic("div_unlock: so=%x NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
+		panic("div_unlock: so=%p NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
 		mutex_held = so->so_proto->pr_domain->dom_mtx;
 	} else {
 		mutex_held = ((struct inpcb *)so->so_pcb)->inpcb_mtx;
@@ -703,7 +716,7 @@ div_unlock(struct socket *so, int refcount, int lr)
 		return (0);
 	}
 	lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
-	so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+	so->unlock_lr[so->next_unlock_lr] = (u_int32_t) lr_saved;
 	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(mutex_held);
 	return (0);
@@ -716,10 +729,10 @@ div_getlock(struct socket *so, __unused int locktype)
 	
 	if (so->so_pcb)  {
 		if (so->so_usecount < 0)
-			panic("div_getlock: so=%x usecount=%x\n", so, so->so_usecount);
+			panic("div_getlock: so=%p usecount=%x\n", so, so->so_usecount);
 		return(inpcb->inpcb_mtx);
 	} else {
-		panic("div_getlock: so=%x NULL so_pcb\n", so);
+		panic("div_getlock: so=%p NULL so_pcb\n", so);
 		return (so->so_proto->pr_domain->dom_mtx);
 	}
 }
diff --git a/bsd/netinet/ip_divert.h b/bsd/netinet/ip_divert.h
index b2f1a2939..3199acbd9 100644
--- a/bsd/netinet/ip_divert.h
+++ b/bsd/netinet/ip_divert.h
@@ -79,7 +79,7 @@ divert_find_info(struct mbuf *m)
 	return mtag ? divert_info(mtag) : 0;
 }
 
-extern	void div_init(void);
+extern	void div_init(void) __attribute__((section("__TEXT, initcode")));
 extern	void div_input(struct mbuf *, int);
 lck_mtx_t * 
         div_getlock(struct socket *,  int );
diff --git a/bsd/netinet/ip_dummynet.c b/bsd/netinet/ip_dummynet.c
index 6b599e42c..426b22d5d 100644
--- a/bsd/netinet/ip_dummynet.c
+++ b/bsd/netinet/ip_dummynet.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-2002 Luigi Rizzo, Universita` di Pisa
@@ -181,6 +187,13 @@ SYSCTL_INT(_net_inet_ip_dummynet, OID_AUTO, debug, CTLFLAG_RW, &dummynet_debug,
 #define	DPRINTF(X)
 #endif
 
+/* contrary to the comment above random(), it does not actually
+ * return a value [0, 2^31 - 1], which breaks plr amongst other
+ * things. Masking it should work even if the behavior of
+ * the function is fixed.
+ */
+#define MY_RANDOM (random() & 0x7FFFFFFF)
+
 /* dummynet lock */
 lck_grp_t         *dn_mutex_grp;
 lck_grp_attr_t    *dn_mutex_grp_attr;
@@ -198,8 +211,6 @@ static void dn_rule_delete(void *);
 
 int if_tx_rdy(struct ifnet *ifp);
 
-extern lck_mtx_t *rt_mtx; /* route global lock */
-
 /*
  * Heap management functions.
  *
@@ -300,9 +311,9 @@ heap_insert(struct dn_heap *h, dn_key key1, void *p)
 static void
 heap_extract(struct dn_heap *h, void *obj)
 {
-    int child, father, max = h->elements - 1 ;
+    int child, father, maxelt = h->elements - 1 ;
 
-    if (max < 0) {
+    if (maxelt < 0) {
 	printf("dummynet: warning, extract from empty heap 0x%p\n", h);
 	return ;
     }
@@ -319,8 +330,8 @@ heap_extract(struct dn_heap *h, void *obj)
     }
     RESET_OFFSET(h, father);
     child = HEAP_LEFT(father) ;		/* left child */
-    while (child <= max) {		/* valid entry */
-	if (child != max && DN_KEY_LT(h->p[child+1].key, h->p[child].key) )
+    while (child <= maxelt) {		/* valid entry */
+	if (child != maxelt && DN_KEY_LT(h->p[child+1].key, h->p[child].key) )
 	    child = child+1 ;		/* take right child, otherwise left */
 	h->p[father] = h->p[child] ;
 	SET_OFFSET(h, father);
@@ -328,11 +339,11 @@ heap_extract(struct dn_heap *h, void *obj)
 	child = HEAP_LEFT(child) ;   /* left child for next loop */
     }
     h->elements-- ;
-    if (father != max) {
+    if (father != maxelt) {
 	/*
 	 * Fill hole with last entry and bubble up, reusing the insert code
 	 */
-	h->p[father] = h->p[max] ;
+	h->p[father] = h->p[maxelt] ;
 	heap_insert(h, father, NULL); /* this one cannot fail */
     }
 }
@@ -347,7 +358,7 @@ heap_move(struct dn_heap *h, dn_key new_key, void *object)
 {
     int temp;
     int i ;
-    int max = h->elements-1 ;
+    int maxelt = h->elements-1 ;
     struct dn_heap_entry buf ;
 
     if (h->offset <= 0)
@@ -363,8 +374,8 @@ heap_move(struct dn_heap *h, dn_key new_key, void *object)
 	}
     } else {		/* must move down */
 	h->p[i].key = new_key ;
-	while ( (temp = HEAP_LEFT(i)) <= max ) { /* found left child */
-	    if ((temp != max) && DN_KEY_GT(h->p[temp].key, h->p[temp+1].key))
+	while ( (temp = HEAP_LEFT(i)) <= maxelt ) { /* found left child */
+	    if ((temp != maxelt) && DN_KEY_GT(h->p[temp].key, h->p[temp+1].key))
 		temp++ ; /* select child with min key */
 	    if (DN_KEY_GT(new_key, h->p[temp].key)) { /* go down */
 		HEAP_SWAP(h->p[i], h->p[temp], buf) ;
@@ -399,7 +410,7 @@ heap_free(struct dn_heap *h)
 {
     if (h->size >0 )
 	FREE(h->p, M_DUMMYNET);
-    bzero(h, sizeof(*h) );
+    bzero(h, sizeof(*h));
 }
 
 /*
@@ -446,7 +457,6 @@ transmit_event(struct dn_pipe *pipe)
 {
     struct mbuf *m ;
     struct dn_pkt_tag *pkt ;
-    struct ip *ip;
 	
 	lck_mtx_assert(dn_mutex, LCK_MTX_ASSERT_OWNED);
 	
@@ -466,7 +476,7 @@ transmit_event(struct dn_pipe *pipe)
 		switch (pkt->dn_dir) {
 			case DN_TO_IP_OUT: {
 				struct route tmp_rt = pkt->ro;
-				(void)ip_output(m, NULL, NULL, pkt->flags, NULL);
+				(void)ip_output(m, NULL, NULL, pkt->flags, NULL, NULL);
 				if (tmp_rt.ro_rt) {
 					rtfree(tmp_rt.ro_rt);
 				}
@@ -749,7 +759,7 @@ ready_event_wfq(struct dn_pipe *p)
  * increment the current tick counter and schedule expired events.
  */
 static void
-dummynet(void * __unused unused)
+dummynet(__unused void * unused)
 {
     void *p ; /* generic parameter to handler */
     struct dn_heap *h ;
@@ -836,7 +846,7 @@ if_tx_rdy(struct ifnet *ifp)
 	    break ;
     if (p == NULL) {
 	char buf[32];
-	sprintf(buf, "%s%d",ifp->if_name, ifp->if_unit);
+	snprintf(buf, sizeof(buf), "%s%d",ifp->if_name, ifp->if_unit);
 	for (p = all_pipes; p ; p = p->next )
 	    if (!strcmp(p->if_name, buf) ) {
 		p->ifp = ifp ;
@@ -1074,7 +1084,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len)
     if (fs->flags_fs & DN_QSIZE_IS_BYTES)
 	p_b = (p_b * len) / fs->max_pkt_size;
     if (++q->count == 0)
-	q->random = random() & 0xffff;
+	q->random = MY_RANDOM & 0xffff;
     else {
 	/*
 	 * q->count counts packets arrived since last drop, so a greater
@@ -1084,7 +1094,7 @@ red_drops(struct dn_flow_set *fs, struct dn_flow_queue *q, int len)
 	    q->count = 0;
 	    DPRINTF(("dummynet: - red drop"));
 	    /* after a drop we calculate a new random value */
-	    q->random = random() & 0xffff;
+	    q->random = MY_RANDOM & 0xffff;
 	    return 1;    /* drop */
 	}
     }
@@ -1201,7 +1211,7 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
      */
     q->tot_bytes += len ;
     q->tot_pkts++ ;
-    if ( fs->plr && random() < fs->plr )
+    if ( fs->plr && (MY_RANDOM < fs->plr) )
 	goto dropit ;		/* random pkt drop			*/
     if ( fs->flags_fs & DN_QSIZE_IS_BYTES) {
     	if (q->len_bytes > fs->qsize)
@@ -1237,7 +1247,7 @@ dummynet_io(struct mbuf *m, int pipe_nr, int dir, struct ip_fw_args *fwa)
 	lck_mtx_lock(rt_mtx);
 	pkt->ro = *(fwa->ro);
 	if (fwa->ro->ro_rt)
-	    fwa->ro->ro_rt->rt_refcnt++ ;
+	    rtref(fwa->ro->ro_rt);
 	if (fwa->dst == (struct sockaddr_in *)&fwa->ro->ro_dst) /* dst points into ro */
 	    fwa->dst = (struct sockaddr_in *)&(pkt->ro.ro_dst) ;
 	lck_mtx_unlock(rt_mtx);
@@ -1428,7 +1438,7 @@ purge_pipe(struct dn_pipe *pipe)
  * remove references from all ipfw rules to all pipes.
  */
 static void
-dummynet_flush()
+dummynet_flush(void)
 {
     struct dn_pipe *curr_p, *p ;
     struct dn_flow_set *fs, *curr_fs;
@@ -1785,7 +1795,7 @@ pipe_remove_from_heap(struct dn_heap *h, struct dn_pipe *p)
  * drain all queues. Called in case of severe mbuf shortage.
  */
 void
-dummynet_drain()
+dummynet_drain(void)
 {
     struct dn_flow_set *fs;
     struct dn_pipe *p;
@@ -1913,7 +1923,7 @@ dn_copy_set(struct dn_flow_set *set, char *bp)
 		printf("dummynet: ++ at %d: wrong fs ptr (have %p, should be %p)\n",
 			i, q->fs, set);
 	    copied++ ;
-	    bcopy(q, qp, sizeof( *q ) );
+	    bcopy(q, qp, sizeof(*q));
 	    /* cleanup pointers */
 	    qp->next = NULL ;
 	    qp->head = qp->tail = NULL ;
@@ -1938,10 +1948,10 @@ dn_calc_size(void)
      * compute size of data structures: list of pipes and flow_sets.
      */
     for (p = all_pipes, size = 0 ; p ; p = p->next )
-	size += sizeof( *p ) +
+	size += sizeof(*p) +
 	    p->fs.rq_elements * sizeof(struct dn_flow_queue);
     for (set = all_flow_sets ; set ; set = set->next )
-	size += sizeof ( *set ) +
+	size += sizeof(*set) +
 	    set->rq_elements * sizeof(struct dn_flow_queue);
     return size ;
 }
@@ -1983,7 +1993,7 @@ dummynet_get(struct sockopt *sopt)
 	 * then copy the flow_set descriptor(s) one at a time.
 	 * After each flow_set, copy the queue descriptor it owns.
 	 */
-	bcopy(p, bp, sizeof( *p ) );
+	bcopy(p, bp, sizeof(*p));
 	pipe_bp->delay = (pipe_bp->delay * 1000) / (hz*10) ; 
 	/*
 	 * XXX the following is a hack based on ->next being the
@@ -1998,17 +2008,17 @@ dummynet_get(struct sockopt *sopt)
 	pipe_bp->fs.pipe = NULL ;
 	pipe_bp->fs.rq = NULL ;
 
-	bp += sizeof( *p ) ;
+	bp += sizeof(*p);
 	bp = dn_copy_set( &(p->fs), bp );
     }
     for (set = all_flow_sets ; set ; set = set->next ) {
 	struct dn_flow_set *fs_bp = (struct dn_flow_set *)bp ;
-	bcopy(set, bp, sizeof( *set ) );
+	bcopy(set, bp, sizeof(*set));
 	/* XXX same hack as above */
 	fs_bp->next = (struct dn_flow_set *)DN_IS_QUEUE ;
 	fs_bp->pipe = NULL ;
 	fs_bp->rq = NULL ;
-	bp += sizeof( *set ) ;
+	bp += sizeof(*set);
 	bp = dn_copy_set( set, bp );
     }
     lck_mtx_unlock(dn_mutex);
@@ -2046,7 +2056,7 @@ ip_dn_ctl(struct sockopt *sopt)
 
     case IP_DUMMYNET_CONFIGURE :
 	p = &tmp_pipe ;
-	error = sooptcopyin(sopt, p, sizeof *p, sizeof *p);
+	error = sooptcopyin(sopt, p, sizeof(*p), sizeof(*p));
 	if (error)
 	    break ;
 	error = config_pipe(p);
@@ -2054,7 +2064,7 @@ ip_dn_ctl(struct sockopt *sopt)
 
     case IP_DUMMYNET_DEL :	/* remove a pipe or queue */
 	p = &tmp_pipe ;
-	error = sooptcopyin(sopt, p, sizeof *p, sizeof *p);
+	error = sooptcopyin(sopt, p, sizeof(*p), sizeof(*p));
 	if (error)
 	    break ;
 
diff --git a/bsd/netinet/ip_dummynet.h b/bsd/netinet/ip_dummynet.h
index c334a1f2a..312b7e266 100644
--- a/bsd/netinet/ip_dummynet.h
+++ b/bsd/netinet/ip_dummynet.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-2002 Luigi Rizzo, Universita` di Pisa
@@ -50,6 +56,9 @@
 
 #ifndef _IP_DUMMYNET_H
 #define _IP_DUMMYNET_H
+
+#if !__LP64__
+
 #include <sys/appleapiopts.h>
 
 #ifdef PRIVATE
@@ -396,4 +405,5 @@ ip_dn_claim_rule(struct mbuf *m)
 #endif /* KERNEL */
 
 #endif /* PRIVATE */
+#endif /* !__LP64__ */
 #endif /* _IP_DUMMYNET_H */
diff --git a/bsd/netinet/ip_ecn.c b/bsd/netinet/ip_ecn.c
index 2dfc497e8..28a558961 100644
--- a/bsd/netinet/ip_ecn.c
+++ b/bsd/netinet/ip_ecn.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1999 WIDE Project.
@@ -149,6 +155,7 @@ ip6_ecn_egress(mode, outer, inner)
 		panic("NULL pointer passed to ip6_ecn_egress");
 
 	outer8 = (ntohl(*outer) >> 20) & 0xff;
+	inner8 = (ntohl(*inner) >> 20) & 0xff;
 	ip_ecn_egress(mode, &outer8, &inner8);
 	*inner &= ~htonl(0xff << 20);
 	*inner |= htonl((u_int32_t)inner8 << 20);
diff --git a/bsd/netinet/ip_ecn.h b/bsd/netinet/ip_ecn.h
index 6e452f578..9b4277ca0 100644
--- a/bsd/netinet/ip_ecn.h
+++ b/bsd/netinet/ip_ecn.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1999 WIDE Project.
diff --git a/bsd/netinet/ip_encap.c b/bsd/netinet/ip_encap.c
index 9517bbb05..6dd02fe9e 100644
--- a/bsd/netinet/ip_encap.c
+++ b/bsd/netinet/ip_encap.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$FreeBSD: src/sys/netinet/ip_encap.c,v 1.1.2.2 2001/07/03 11:01:46 ume Exp $	*/
 /*	$KAME: ip_encap.c,v 1.41 2001/03/15 08:35:08 itojun Exp $	*/
@@ -345,9 +351,7 @@ encap_attach(af, proto, sp, sm, dp, dm, psw, arg)
 {
 	struct encaptab *ep;
 	int error;
-	int s;
 
-	s = splnet();
 	/* sanity check on args */
 	if (sp->sa_len > sizeof(ep->src) || dp->sa_len > sizeof(ep->dst)) {
 		error = EINVAL;
@@ -400,11 +404,9 @@ encap_attach(af, proto, sp, sm, dp, dm, psw, arg)
 	encap_add(ep);
 
 	error = 0;
-	splx(s);
 	return ep;
 
 fail:
-	splx(s);
 	return NULL;
 }
 
@@ -418,9 +420,7 @@ encap_attach_func(af, proto, func, psw, arg)
 {
 	struct encaptab *ep;
 	int error;
-	int s;
 
-	s = splnet();
 	/* sanity check on args */
 	if (!func) {
 		error = EINVAL;
@@ -443,11 +443,9 @@ encap_attach_func(af, proto, func, psw, arg)
 	encap_add(ep);
 
 	error = 0;
-	splx(s);
 	return ep;
 
 fail:
-	splx(s);
 	return NULL;
 }
 
@@ -522,43 +520,42 @@ mask_match(ep, sp, dp)
 		return 0;
 }
 
+struct encaptabtag {
+	void*			*arg;
+};
+
 static void
-encap_fillarg(m, ep)
-	struct mbuf *m;
-	const struct encaptab *ep;
+encap_fillarg(
+	struct mbuf *m,
+	const struct encaptab *ep)
 {
-#if 0
-	m->m_pkthdr.aux = ep->arg;
-#else
-	struct mbuf *n;
-
-	n = m_aux_add(m, AF_INET, IPPROTO_IPV4);
-	if (n) {
-		*mtod(n, void **) = ep->arg;
-		n->m_len = sizeof(void *);
+	struct m_tag	*tag;
+	struct encaptabtag *et;
+	
+	tag = m_tag_alloc(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_ENCAP,
+					  sizeof(struct encaptabtag), M_WAITOK);
+	
+	if (tag != NULL) {
+		et = (struct encaptabtag*)(tag + 1);
+		et->arg = ep->arg;
+		m_tag_prepend(m, tag);
 	}
-#endif
 }
 
 void *
 encap_getarg(m)
 	struct mbuf *m;
 {
-	void *p;
-#if 0
-	p = m->m_pkthdr.aux;
-	m->m_pkthdr.aux = NULL;
-	return p;
-#else
-	struct mbuf *n;
-
-	p = NULL;
-	n = m_aux_find(m, AF_INET, IPPROTO_IPV4);
-	if (n) {
-		if (n->m_len == sizeof(void *))
-			p = *mtod(n, void **);
-		m_aux_delete(m, n);
+	struct m_tag	*tag;
+	struct encaptabtag *et;
+	void *p = NULL;
+	
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_ENCAP, NULL);
+	if (tag) {
+		et = (struct encaptabtag*)(tag + 1);
+		p = et->arg;
+		m_tag_delete(m, tag);
 	}
+	
 	return p;
-#endif
 }
diff --git a/bsd/netinet/ip_encap.h b/bsd/netinet/ip_encap.h
index 0a3aba152..93ba512c2 100644
--- a/bsd/netinet/ip_encap.h
+++ b/bsd/netinet/ip_encap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$KAME: ip_encap.h,v 1.6 2000/03/06 04:34:21 itojun Exp $	*/
 
@@ -69,7 +75,7 @@ struct encaptab {
 	void *arg;			/* passed via m->m_pkthdr.aux */
 };
 
-void	encap_init(void);
+void	encap_init(void) __attribute__((section("__TEXT, initcode")));
 void	encap4_input(struct mbuf *, int);
 int	encap6_input(struct mbuf **, int *);
 const struct encaptab *encap_attach(int, int, const struct sockaddr *,
diff --git a/bsd/netinet/ip_flow.c b/bsd/netinet/ip_flow.c
index ba3ca9816..d6c1f128f 100644
--- a/bsd/netinet/ip_flow.c
+++ b/bsd/netinet/ip_flow.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -66,6 +72,7 @@
 #include <sys/kernel.h>
 
 #include <sys/sysctl.h>
+#include <libkern/OSAtomic.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -212,24 +219,21 @@ ipflow_addstats(
 	struct ipflow *ipf)
 {
 	ipf->ipf_ro.ro_rt->rt_use += ipf->ipf_uses;
-	ipstat.ips_cantforward += ipf->ipf_errors + ipf->ipf_dropped;
-	ipstat.ips_forward += ipf->ipf_uses;
-	ipstat.ips_fastforward += ipf->ipf_uses;
+	OSAddAtomic(ipf->ipf_errors + ipf->ipf_dropped, (SInt32*)&ipstat.ips_cantforward);
+	OSAddAtomic(ipf->ipf_uses, (SInt32*)&ipstat.ips_forward);
+	OSAddAtomic(ipf->ipf_uses, (SInt32*)&ipstat.ips_fastforward);
 }
 
 static void
 ipflow_free(
 	struct ipflow *ipf)
 {
-	int s;
 	/*
 	 * Remove the flow from the hash table (at elevated IPL).
 	 * Once it's off the list, we can deal with it at normal
 	 * network IPL.
 	 */
-	s = splimp();
 	LIST_REMOVE(ipf, ipf_next);
-	splx(s);
 	ipflow_addstats(ipf);
 	rtfree(ipf->ipf_ro.ro_rt);
 	ipflow_inuse--;
@@ -242,7 +246,6 @@ ipflow_reap(
 {
 	struct ipflow *ipf, *maybe_ipf = NULL;
 	int idx;
-	int s;
 
 	for (idx = 0; idx < IPFLOW_HASHSIZE; idx++) {
 		ipf = LIST_FIRST(&ipflows[idx]);
@@ -273,14 +276,12 @@ ipflow_reap(
 	/*
 	 * Remove the entry from the flow table.
 	 */
-	s = splimp();
 	LIST_REMOVE(ipf, ipf_next);
-	splx(s);
 	ipflow_addstats(ipf);
 	rtfree(ipf->ipf_ro.ro_rt);
 	return ipf;
 }
-
+/* note: called under the ip_mutex lock */
 void
 ipflow_slowtimo(
 	void)
@@ -297,6 +298,8 @@ ipflow_slowtimo(
 			} else {
 				ipf->ipf_last_uses = ipf->ipf_uses;
 				ipf->ipf_ro.ro_rt->rt_use += ipf->ipf_uses;
+				OSAddAtomic(ipf->ipf_uses, (SInt32*)&ipstat.ips_forward);
+				OSAddAtomic(ipf->ipf_uses, (SInt32*)&ipstat.ips_fastforward);
 				ipstat.ips_forward += ipf->ipf_uses;
 				ipstat.ips_fastforward += ipf->ipf_uses;
 				ipf->ipf_uses = 0;
@@ -314,7 +317,6 @@ ipflow_create(
 	const struct ip *const ip = mtod(m, struct ip *);
 	struct ipflow *ipf;
 	unsigned hash;
-	int s;
 
 	/*
 	 * Don't create cache entries for ICMP messages.
@@ -339,9 +341,7 @@ ipflow_create(
 		}
 		bzero((caddr_t) ipf, sizeof(*ipf));
 	} else {
-		s = splimp();
 		LIST_REMOVE(ipf, ipf_next);
-		splx(s);
 		ipflow_addstats(ipf);
 		rtfree(ipf->ipf_ro.ro_rt);
 		ipf->ipf_uses = ipf->ipf_last_uses = 0;
@@ -351,8 +351,10 @@ ipflow_create(
 	/*
 	 * Fill in the updated information.
 	 */
+	lck_mtx_lock(rt_mtx);
 	ipf->ipf_ro = *ro;
-	rtref(ro->ro_rt);		//### LD 5/25/04 needs rt_mtx lock
+	rtref(ro->ro_rt);
+	lck_mtx_unlock(rt_mtx);
 	ipf->ipf_dst = ip->ip_dst;
 	ipf->ipf_src = ip->ip_src;
 	ipf->ipf_tos = ip->ip_tos;
@@ -361,7 +363,5 @@ ipflow_create(
 	 * Insert into the approriate bucket of the flow table.
 	 */
 	hash = ipflow_hash(ip->ip_dst, ip->ip_src, ip->ip_tos);
-	s = splimp();
 	LIST_INSERT_HEAD(&ipflows[hash], ipf, ipf_next);
-	splx(s);
 }
diff --git a/bsd/netinet/ip_flow.h b/bsd/netinet/ip_flow.h
index 0fed616e0..37dcbddee 100644
--- a/bsd/netinet/ip_flow.h
+++ b/bsd/netinet/ip_flow.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
diff --git a/bsd/netinet/ip_fw.h b/bsd/netinet/ip_fw.h
index 606c62844..174bf8ff1 100644
--- a/bsd/netinet/ip_fw.h
+++ b/bsd/netinet/ip_fw.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1993 Daniel Boulet
@@ -36,6 +42,9 @@
 
 #ifndef _IP_FW_H
 #define _IP_FW_H
+
+#if !__LP64__
+
 #include <sys/appleapiopts.h>
 
 #ifdef IPFW2
@@ -43,7 +52,7 @@
 #else /* !IPFW2, good old ipfw */
 
 #include <sys/queue.h>
-
+#include <sys/types.h>		/* u_ types */
 
 #define IP_FW_CURRENT_API_VERSION 20	/* Version of this API */
 
@@ -312,8 +321,10 @@ extern	ip_fw_chk_t *ip_fw_chk_ptr;
 extern	ip_fw_ctl_t *ip_fw_ctl_ptr;
 extern int fw_one_pass;
 extern int fw_enable;
+#define IPFW_LOADED     (ip_fw_chk_ptr != NULL)
 extern struct ipfw_flow_id last_pkt ;
 #endif KERNEL_PRIVATE
 
+#endif /* !__LP64__ */
 #endif !IPFW2
 #endif /* _IP_FW_H */
diff --git a/bsd/netinet/ip_fw2.c b/bsd/netinet/ip_fw2.c
index 84f8f0c1a..ea482f0c6 100644
--- a/bsd/netinet/ip_fw2.c
+++ b/bsd/netinet/ip_fw2.c
@@ -36,7 +36,7 @@
 #error IPFIREWALL requires INET.
 #endif /* INET */
 
-#ifdef IPFW2
+#if IPFW2
 #include <machine/spl.h>
 
 #include <sys/param.h>
@@ -50,6 +50,8 @@
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/ucred.h>
+#include <sys/kern_event.h>
+
 #include <net/if.h>
 #include <net/route.h>
 #include <netinet/in.h>
@@ -108,6 +110,7 @@ static u_int32_t set_disable;
 
 int fw_verbose;
 static int verbose_limit;
+extern int fw_bypass;
 
 #define	IPFW_DEFAULT_RULE	65535
 
@@ -120,14 +123,19 @@ static struct ip_fw *layer3_chain;
 
 MALLOC_DEFINE(M_IPFW, "IpFw/IpAcct", "IpFw/IpAcct chain's");
 
-static int fw_debug = 1;
+static int fw_debug = 0;
 static int autoinc_step = 100; /* bounded to 1..1000 in add_rule() */
 
+static void ipfw_kev_post_msg(u_int32_t );
+
 #ifdef SYSCTL_NODE
-SYSCTL_NODE(_net_inet_ip, OID_AUTO, fw, CTLFLAG_RW, 0, "Firewall");
-SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, enable,
-    CTLFLAG_RW,
-    &fw_enable, 0, "Enable ipfw");
+
+static int ipfw_sysctl SYSCTL_HANDLER_ARGS;
+
+SYSCTL_NODE(_net_inet_ip, OID_AUTO, fw, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Firewall");
+SYSCTL_PROC(_net_inet_ip_fw, OID_AUTO, enable,
+    CTLTYPE_INT | CTLFLAG_RW,
+    &fw_enable, 0, ipfw_sysctl, "I", "Enable ipfw");
 SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, autoinc_step, CTLFLAG_RW,
     &autoinc_step, 0, "Rule number autincrement step");
 SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, one_pass,
@@ -235,6 +243,21 @@ SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_short_lifetime, CTLFLAG_RW,
 SYSCTL_INT(_net_inet_ip_fw, OID_AUTO, dyn_keepalive, CTLFLAG_RW,
     &dyn_keepalive, 0, "Enable keepalives for dyn. rules");
 
+static int
+ipfw_sysctl SYSCTL_HANDLER_ARGS
+{
+#pragma unused(arg1, arg2)
+	int error;
+	
+	error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+	if (error || !req->newptr)
+		return (error);
+	
+	ipfw_kev_post_msg(KEV_IPFW_ENABLE);
+	
+	return error;
+}
+
 #endif /* SYSCTL_NODE */
 
 
@@ -246,7 +269,7 @@ lck_grp_attr_t    *ipfw_mutex_grp_attr;
 lck_attr_t        *ipfw_mutex_attr;
 lck_mtx_t         *ipfw_mutex;
 
-extern  void    ipfwsyslog( int level, char *format,...);
+extern  void    ipfwsyslog( int level, const char *format,...);
 
 #if DUMMYNET
 ip_dn_ruledel_t *ip_dn_ruledel_ptr = NULL;	/* hook into dummynet */
@@ -260,11 +283,11 @@ static          size_t		ipfwstringlen;
 
 #define dolog( a ) {		\
 	if ( fw_verbose == 2 )  	/* Apple logging, log to ipfw.log */ \
-		ipfwsyslog a ; 	\ 
+		ipfwsyslog a ; 	\
 	else log a ;		\
 }
 
-void    ipfwsyslog( int level, char *format,...)
+void    ipfwsyslog( int level, const char *format,...)
 {
 #define		msgsize		100
 
@@ -537,7 +560,7 @@ static void
 ipfw_log(struct ip_fw *f, u_int hlen, struct ether_header *eh,
 	struct mbuf *m, struct ifnet *oif)
 {
-	char *action;
+	const char *action;
 	int limit_reached = 0;
 	char ipv4str[MAX_IPv4_STR_LEN];
 	char action2[40], proto[48], fragment[28];
@@ -1218,7 +1241,7 @@ send_pkt(struct ipfw_flow_id *id, u_int32_t seq, u_int32_t ack, int flags)
 	struct tcphdr *tcp;
 	struct route sro;	/* fake route */
 
-	MGETHDR(m, M_DONTWAIT, MT_HEADER);
+	MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 	if (m == 0)
 		return;
 	m->m_pkthdr.rcvif = (struct ifnet *)0;
@@ -1285,7 +1308,7 @@ send_pkt(struct ipfw_flow_id *id, u_int32_t seq, u_int32_t ack, int flags)
 	bzero (&sro, sizeof (sro));
 	ip_rtaddr(ip->ip_dst, &sro);
 	m->m_flags |= M_SKIP_FIREWALL;
-	ip_output_list(m, 0, NULL, &sro, 0, NULL);
+	ip_output_list(m, 0, NULL, &sro, 0, NULL, NULL);
 	if (sro.ro_rt)
 		RTFREE(sro.ro_rt);
 }
@@ -1294,7 +1317,7 @@ send_pkt(struct ipfw_flow_id *id, u_int32_t seq, u_int32_t ack, int flags)
  * sends a reject message, consuming the mbuf passed as an argument.
  */
 static void
-send_reject(struct ip_fw_args *args, int code, int offset, int ip_len)
+send_reject(struct ip_fw_args *args, int code, int offset, __unused int ip_len)
 {
 
 	if (code != ICMP_REJECT_RST) { /* Send an ICMP unreach */
@@ -1304,7 +1327,7 @@ send_reject(struct ip_fw_args *args, int code, int offset, int ip_len)
 			ip->ip_len = ntohs(ip->ip_len);
 			ip->ip_off = ntohs(ip->ip_off);
 		}
-                args->m->m_flags |= M_SKIP_FIREWALL;
+		args->m->m_flags |= M_SKIP_FIREWALL;
 		icmp_error(args->m, ICMP_UNREACH, code, 0L, 0);
 	} else if (offset == 0 && args->f_id.proto == IPPROTO_TCP) {
 		struct tcphdr *const tcp =
@@ -1459,17 +1482,26 @@ ipfw_chk(struct ip_fw_args *args)
 	 */
 	u_int8_t proto;
 	u_int16_t src_port = 0, dst_port = 0;	/* NOTE: host format	*/
-	struct in_addr src_ip, dst_ip;		/* NOTE: network format	*/
+	struct in_addr src_ip = { 0 } , dst_ip = { 0 };		/* NOTE: network format	*/
 	u_int16_t ip_len=0;
 	int pktlen;
 	int dyn_dir = MATCH_UNKNOWN;
 	ipfw_dyn_rule *q = NULL;
 	struct timeval timenow;
 
-	if (m->m_flags & M_SKIP_FIREWALL) {
+	if (m->m_flags & M_SKIP_FIREWALL || fw_bypass) {
 		return 0;	/* accept */
 	}
 
+	/* 
+	 * Clear packet chain if we find one here.
+	 */
+	
+	if (m->m_nextpkt != NULL) {
+		m_freem_list(m->m_nextpkt);
+		m->m_nextpkt = NULL;
+	}
+	
 	lck_mtx_lock(ipfw_mutex);
 
 	getmicrotime(&timenow);
@@ -2104,7 +2136,7 @@ check_body:
 				 * if the packet is not ICMP (or is an ICMP
 				 * query), and it is not multicast/broadcast.
 				 */
-				if (hlen > 0 &&
+				if (hlen > 0 && offset == 0 &&
 				    (proto != IPPROTO_ICMP ||
 				     is_icmp_query(ip)) &&
 				    !(m->m_flags & (M_BCAST|M_MCAST)) &&
@@ -2216,7 +2248,6 @@ static int
 add_rule(struct ip_fw **head, struct ip_fw *input_rule)
 {
 	struct ip_fw *rule, *f, *prev;
-	int s;
 	int l = RULESIZE(input_rule);
 
 	if (*head == NULL && input_rule->rulenum != IPFW_DEFAULT_RULE)
@@ -2477,7 +2508,6 @@ static int
 del_entry(struct ip_fw **chain, u_int32_t arg)
 {
 	struct ip_fw *prev = NULL, *rule = *chain;
-	int s;
 	u_int16_t rulenum;	/* rule or old_set */
 	u_int8_t cmd, new_set;
 
@@ -2513,12 +2543,12 @@ del_entry(struct ip_fw **chain, u_int32_t arg)
 		 */
 		flush_rule_ptrs();
 		while (rule->rulenum == rulenum) {
-			ipfw_insn	*cmd = ACTION_PTR(rule);
+			ipfw_insn	*insn = ACTION_PTR(rule);
 			
 			/* keep forwarding rules around so struct isn't 
 			 * deleted while pointer is still in use elsewhere
 			 */
-			if (cmd->opcode == O_FORWARD_IP) {
+			if (insn->opcode == O_FORWARD_IP) {
 				mark_inactive(&prev, &rule);
 			}
 			else {
@@ -2531,12 +2561,12 @@ del_entry(struct ip_fw **chain, u_int32_t arg)
 		flush_rule_ptrs();
 		while (rule->rulenum < IPFW_DEFAULT_RULE) {
 			if (rule->set == rulenum) {
-				ipfw_insn	*cmd = ACTION_PTR(rule);
+				ipfw_insn	*insn = ACTION_PTR(rule);
 				
 				/* keep forwarding rules around so struct isn't 
 				 * deleted while pointer is still in use elsewhere
 				 */
-				if (cmd->opcode == O_FORWARD_IP) {
+				if (insn->opcode == O_FORWARD_IP) {
 					mark_inactive(&prev, &rule);
 				}
 				else {
@@ -2599,8 +2629,7 @@ static int
 zero_entry(int rulenum, int log_only)
 {
 	struct ip_fw *rule;
-	int s;
-	char *msg;
+	const char *msg;
 
 	if (rulenum == 0) {
 		norule_counter = 0;
@@ -2827,6 +2856,22 @@ bad_size:
 }
 
 
+static void
+ipfw_kev_post_msg(u_int32_t event_code)
+{
+	struct kev_msg		ev_msg;
+
+	bzero(&ev_msg, sizeof(struct kev_msg));
+	
+	ev_msg.vendor_code = KEV_VENDOR_APPLE;
+	ev_msg.kev_class = KEV_FIREWALL_CLASS;
+	ev_msg.kev_subclass = KEV_IPFW_SUBCLASS;
+	ev_msg.event_code = event_code;
+
+	kev_post_msg(&ev_msg);
+
+}
+
 /**
  * {set|get}sockopt parser.
  */
@@ -2836,7 +2881,7 @@ ipfw_ctl(struct sockopt *sopt)
 #define	RULE_MAXSIZE	(256*sizeof(u_int32_t))
 	u_int32_t api_version;
 	int command;
-	int error, s;
+	int error;
 	size_t size;
 	struct ip_fw *bp , *buf, *rule;
 	
@@ -3000,7 +3045,7 @@ ipfw_ctl(struct sockopt *sopt)
 				if (ipfw_dyn_v) {
 					for (i = 0; i < curr_dyn_buckets; i++) {
 						for ( p = ipfw_dyn_v[i] ; p != NULL ; p = p->next) {
-							(int) dyn_rule_vers1->chain = p->rule->rulenum;
+							dyn_rule_vers1->chain = p->rule->rulenum;
 							dyn_rule_vers1->id = p->id;
 							dyn_rule_vers1->mask = p->id;
 							dyn_rule_vers1->type = p->dyn_type;
@@ -3010,7 +3055,7 @@ ipfw_ctl(struct sockopt *sopt)
 							dyn_rule_vers1->bucket = p->bucket;
 							dyn_rule_vers1->state = p->state;
 							
-							dyn_rule_vers1->next = dyn_rule_vers1;
+							dyn_rule_vers1->next = (struct ipfw_dyn_rule *) dyn_rule_vers1;
 							dyn_last = dyn_rule_vers1;
 							
 							len += sizeof(*dyn_rule_vers1);
@@ -3050,6 +3095,7 @@ ipfw_ctl(struct sockopt *sopt)
 
 		lck_mtx_lock(ipfw_mutex);
 		free_chain(&layer3_chain, 0 /* keep default rule */);
+		fw_bypass = 1;
 #if DEBUG_INACTIVE_RULES
 			print_chain(&layer3_chain);
 #endif
@@ -3084,6 +3130,8 @@ ipfw_ctl(struct sockopt *sopt)
 			if (!error) {
 				lck_mtx_lock(ipfw_mutex);
 				error = add_rule(&layer3_chain, rule);
+				if (!error && fw_bypass)
+					fw_bypass = 0;
 				lck_mtx_unlock(ipfw_mutex);
 				
 				size = RULESIZE(rule);
@@ -3171,7 +3219,9 @@ ipfw_ctl(struct sockopt *sopt)
 					(set_disable | temp_rule.set_masks[0]) & ~temp_rule.set_masks[1] &
 					~(1<<RESVD_SET); /* set RESVD_SET always enabled */
 			}
-						
+			
+			if (!layer3_chain->next)
+				fw_bypass = 1;
 			lck_mtx_unlock(ipfw_mutex);
 		}
 		break;
@@ -3182,7 +3232,9 @@ ipfw_ctl(struct sockopt *sopt)
 		/* there is only a simple rule passed in
 		 * (no cmds), so use a temp struct to copy
 		 */
-		struct ip_fw	temp_rule = { 0 };
+		struct ip_fw	temp_rule;
+		
+		bzero(&temp_rule, sizeof(struct ip_fw));
 		
 		if (api_version != IP_FW_CURRENT_API_VERSION) {
 			error = ipfw_convert_to_latest(sopt, &temp_rule, api_version);
@@ -3206,6 +3258,26 @@ ipfw_ctl(struct sockopt *sopt)
 		error = EINVAL;
 	}
 
+	if (error != EINVAL) {
+		switch (command) {
+			case IP_FW_ADD:
+			case IP_OLD_FW_ADD:
+				ipfw_kev_post_msg(KEV_IPFW_ADD);
+				break;
+			case IP_OLD_FW_DEL:
+			case IP_FW_DEL:
+				ipfw_kev_post_msg(KEV_IPFW_DEL);
+				break;
+			case IP_FW_FLUSH:
+			case IP_OLD_FW_FLUSH:
+				ipfw_kev_post_msg(KEV_IPFW_FLUSH);
+				break;
+
+			default:
+				break;
+		}
+	}
+
 	return (error);
 }
 
@@ -3223,10 +3295,9 @@ struct ip_fw *ip_fw_default_rule;
  * every dyn_keepalive_period
  */
 static void
-ipfw_tick(void * __unused unused)
+ipfw_tick(__unused void * unused)
 {
 	int i;
-	int s;
 	ipfw_dyn_rule *q;
 	struct timeval timenow;
 
@@ -3298,17 +3369,6 @@ ipfw_init(void)
 	}
 	else {
 		ip_fw_default_rule = layer3_chain;
-#if 0
-		/* Radar 3920649, don't print unncessary messages to the log */
-		printf("ipfw2 initialized, divert %s, "
-			"rule-based forwarding enabled, default to %s, logging ",
-	#ifdef IPDIVERT
-			"enabled",
-	#else
-			"disabled",
-	#endif
-			default_rule.cmd[0].opcode == O_ACCEPT ? "accept" : "deny");
-#endif
 	
 	#ifdef IPFIREWALL_VERBOSE
 		fw_verbose = 1;
@@ -3316,13 +3376,13 @@ ipfw_init(void)
 	#ifdef IPFIREWALL_VERBOSE_LIMIT
 		verbose_limit = IPFIREWALL_VERBOSE_LIMIT;
 	#endif
-		if (fw_verbose == 0)
-			printf("disabled\n");
-		else if (verbose_limit == 0)
-			printf("unlimited\n");
-		else
-			printf("limited to %d packets/entry by default\n",
-				verbose_limit);
+		if (fw_verbose) {
+			if (!verbose_limit)
+				printf("ipfw2 verbose logging enabled: unlimited logging by default\n");
+			else
+				printf("ipfw2 verbose logging enabled: limited to %d packets/entry by default\n",
+					verbose_limit);
+		}
 	}
 
 	ip_fw_chk_ptr = ipfw_chk;
@@ -3334,3 +3394,4 @@ ipfw_init(void)
 }
 
 #endif /* IPFW2 */
+
diff --git a/bsd/netinet/ip_fw2.h b/bsd/netinet/ip_fw2.h
index 43dcf98ce..cd1514ffd 100644
--- a/bsd/netinet/ip_fw2.h
+++ b/bsd/netinet/ip_fw2.h
@@ -28,6 +28,44 @@
 #ifndef _IPFW2_H
 #define _IPFW2_H
 
+/*
+ * Define IP Firewall event subclass, and associated events.
+ */
+
+/*!
+	@defined KEV_IPFW_SUBCLASS
+	@discussion The kernel event subclass for IP Firewall.
+*/
+#define KEV_IPFW_SUBCLASS	1
+
+/*!
+	@defined KEV_IPFW_ADD
+	@discussion The event code indicating a rule has been added.
+*/
+#define KEV_IPFW_ADD		1
+
+/*!
+	@defined KEV_IPFW_DEL
+	@discussion The event code indicating a rule has been removed.
+*/
+#define KEV_IPFW_DEL		2
+
+/*!
+	@defined KEV_IPFW_FLUSH
+	@discussion The event code indicating the rule set has been flushed.
+*/
+#define KEV_IPFW_FLUSH		3
+
+/*!
+	@defined KEV_IPFW_ENABLE
+	@discussion The event code indicating the enable flag has been changed 
+*/
+#define KEV_IPFW_ENABLE		4
+
+
+
+#if !__LP64__
+
 /*
  * The kernel representation of ipfw rules is made of a list of
  * 'instructions' (for all practical purposes equivalent to BPF
@@ -440,4 +478,5 @@ extern int fw_enable;
 #define	IPFW_LOADED	(ip_fw_chk_ptr != NULL)
 #endif /* KERNEL */
 
+#endif /* !__LP64__ */
 #endif /* _IPFW2_H */
diff --git a/bsd/netinet/ip_fw2_compat.c b/bsd/netinet/ip_fw2_compat.c
index 1e0ee62a6..eb7eb43fe 100644
--- a/bsd/netinet/ip_fw2_compat.c
+++ b/bsd/netinet/ip_fw2_compat.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /* IPFW2 Backward Compatibility */
 
 /* Convert to and from IPFW2 structures. */
@@ -239,8 +266,8 @@ ipfw_print_vers1_struct(struct ip_fw_compat *vers1_rule)
 	
 	ipfw_print_fw_flags(vers1_rule->fw_flg);
 	
-	printf("fw_pcnt: %d\n", vers1_rule->fw_pcnt);
-	printf("fw_bcnt: %d\n", vers1_rule->fw_bcnt);
+	printf("fw_pcnt: %llu\n", vers1_rule->fw_pcnt);
+	printf("fw_bcnt: %llu\n", vers1_rule->fw_bcnt);
 	printf("fw_src: %s\n",
 		inet_ntop(AF_INET, &vers1_rule->fw_src, ipv4str, sizeof(ipv4str)));
 	printf("fw_dst: %s\n",
@@ -302,7 +329,7 @@ ipfw_print_vers1_struct(struct ip_fw_compat *vers1_rule)
 	printf("fw_tcpnopt: %d\n", vers1_rule->fw_tcpnopt);
 	printf("fw_tcpf: %d\n", vers1_rule->fw_tcpf);
 	printf("fw_tcpnf: %d\n", vers1_rule->fw_tcpnf);
-	printf("timestamp: %d\n", vers1_rule->timestamp);
+	printf("timestamp: %ld\n", vers1_rule->timestamp);
 
 	if ((vers1_rule->fw_flg & IF_FW_F_VIAHACK_COMPAT) == IF_FW_F_VIAHACK_COMPAT) {
 		printf("fw_in_if: ");
@@ -332,11 +359,11 @@ ipfw_print_vers1_struct(struct ip_fw_compat *vers1_rule)
 	
 	printf("fw_prot: %d\n", vers1_rule->fw_prot);
 	printf("fw_nports: %d\n", vers1_rule->fw_nports);
-	printf("pipe_ptr: %x\n", vers1_rule->pipe_ptr);
-	printf("next_rule_ptr: %x\n", vers1_rule->next_rule_ptr);
+	printf("pipe_ptr: %p\n", vers1_rule->pipe_ptr);
+	printf("next_rule_ptr: %p\n", vers1_rule->next_rule_ptr);
 	printf("fw_uid: %d\n", vers1_rule->fw_uid);
 	printf("fw_logamount: %d\n", vers1_rule->fw_logamount);
-	printf("fw_loghighest: %d\n", vers1_rule->fw_loghighest);
+	printf("fw_loghighest: %llu\n", vers1_rule->fw_loghighest);
 }
 
 static void
@@ -476,8 +503,8 @@ ipfw_print_vers2_struct(struct ip_fw *vers2_rule)
 	printf("cmd_len: %d\n", vers2_rule->cmd_len);
 	printf("rulenum: %d\n", vers2_rule->rulenum);
 	printf("set: %d\n", vers2_rule->set);
-	printf("pcnt: %d\n", vers2_rule->pcnt);
-	printf("bcnt: %d\n", vers2_rule->bcnt);
+	printf("pcnt: %llu\n", vers2_rule->pcnt);
+	printf("bcnt: %llu\n", vers2_rule->bcnt);
 	printf("timestamp: %d\n", vers2_rule->timestamp);
 	
 	/*
@@ -777,11 +804,11 @@ ipfw_print_vers2_struct(struct ip_fw *vers2_rule)
 						break;
 		
 					case O_TCPACK:
-						printf(" tcpack %ld", ntohl(cmd32->d[0]));
+						printf(" tcpack %u", ntohl(cmd32->d[0]));
 						break;
 		
 					case O_TCPSEQ:
-						printf(" tcpseq %ld", ntohl(cmd32->d[0]));
+						printf(" tcpseq %u", ntohl(cmd32->d[0]));
 						break;
 		
 					case O_UID:
@@ -1486,9 +1513,7 @@ ipfw_convert_to_cmds(struct ip_fw *curr_rule, struct ip_fw_compat *compat_rule)
 	int			k;	
 	uint32_t	actbuf[255], cmdbuf[255];
 	ipfw_insn	*action, *cmd, *src, *dst;
-	ipfw_insn	*have_state = NULL,	/* track check-state or keep-state */
-				*end_action = NULL, 
-				*end_cmd = NULL;
+	ipfw_insn	*have_state = NULL;	/* track check-state or keep-state */
 	
 	if (!compat_rule || !curr_rule || !(curr_rule->cmd)) {
 		return;
@@ -2126,7 +2151,7 @@ ipfw_version_zero_to_latest(struct sockopt *sopt, struct ip_fw *curr_rule)
 
 	if (sopt->sopt_name == IP_OLD_FW_GET || 
 		sopt->sopt_name == IP_OLD_FW_FLUSH || 
-		sopt->sopt_val == NULL) {
+		sopt->sopt_val == USER_ADDR_NULL) {
 		/* In the old-style API, it was legal to not pass in a rule 
 		 * structure for certain firewall operations (e.g. flush, 
 		 * reset log).  If that's the situation, we pretend we received 
diff --git a/bsd/netinet/ip_fw2_compat.h b/bsd/netinet/ip_fw2_compat.h
index 76324197f..a38a7d3f1 100644
--- a/bsd/netinet/ip_fw2_compat.h
+++ b/bsd/netinet/ip_fw2_compat.h
@@ -3,6 +3,7 @@
 #ifndef _IP_FW_COMPAT_H_
 #define _IP_FW_COMPAT_H_
 
+#if !__LP64__
 
 /* prototypes */
 void	ipfw_convert_from_latest(struct ip_fw *curr_rule, void *old_rule, u_int32_t api_version);
@@ -371,4 +372,5 @@ struct ip_old_fw {
  */
 #define IP_OLD_FW_TCPF_ESTAB	0x40
 
-
+#endif /* !__LP64__ */
+#endif /* _IP_FW_COMPAT_H_ */
diff --git a/bsd/netinet/ip_icmp.c b/bsd/netinet/ip_icmp.c
index 030ef0680..3ed8a2d45 100644
--- a/bsd/netinet/ip_icmp.c
+++ b/bsd/netinet/ip_icmp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1993
@@ -53,6 +59,12 @@
  *
  *	@(#)ip_icmp.c	8.2 (Berkeley) 1/4/94
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -96,7 +108,11 @@
  */
 #ifndef M_SKIP_FIREWALL
 #define M_SKIP_FIREWALL         0x4000
-#endif
+#endif 
+
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
 
 /*
  * ICMP routines: error generation, receive packet processing, and
@@ -173,10 +189,10 @@ icmp_error(
 	n_long dest,
 	struct ifnet *destifp)
 {
-	register struct ip *oip = mtod(n, struct ip *), *nip;
-	register unsigned oiplen = IP_VHL_HL(oip->ip_vhl) << 2;
-	register struct icmp *icp;
-	register struct mbuf *m;
+	struct ip *oip = mtod(n, struct ip *), *nip;
+	unsigned oiplen = IP_VHL_HL(oip->ip_vhl) << 2;
+	struct icmp *icp;
+	struct mbuf *m;
 	unsigned icmplen;
 
 #if ICMPPRINTFS
@@ -204,15 +220,18 @@ icmp_error(
 	/*
 	 * First, formulate icmp message
 	 */
-	m = m_gethdr(M_DONTWAIT, MT_HEADER);
+	m = m_gethdr(M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 	if (m == NULL)
 		goto freeit;
 
-	if (n->m_flags & M_SKIP_FIREWALL) {
+        if (n->m_flags & M_SKIP_FIREWALL) {
 		/* set M_SKIP_FIREWALL to skip firewall check, since we're called from firewall */
 		m->m_flags |= M_SKIP_FIREWALL;
 	}
 
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_netlayer(n, m);
+#endif
 	icmplen = min(oiplen + 8, oip->ip_len);
 	if (icmplen < sizeof(struct ip)) {
 		printf("icmp_error: bad length\n");
@@ -263,7 +282,6 @@ icmp_error(
 	m->m_len += sizeof(struct ip);
 	m->m_pkthdr.len = m->m_len;
 	m->m_pkthdr.rcvif = n->m_pkthdr.rcvif;
-	m->m_pkthdr.aux = NULL; /* for IPsec */
 	nip = mtod(m, struct ip *);
 	bcopy((caddr_t)oip, (caddr_t)nip, sizeof(struct ip));
 	nip->ip_len = m->m_len;
@@ -276,26 +294,26 @@ freeit:
 	m_freem(n);
 }
 
-static struct sockaddr_in icmpsrc = { sizeof (struct sockaddr_in), AF_INET };
-static struct sockaddr_in icmpdst = { sizeof (struct sockaddr_in), AF_INET };
-static struct sockaddr_in icmpgw = { sizeof (struct sockaddr_in), AF_INET };
+static struct sockaddr_in icmpsrc = { sizeof (struct sockaddr_in), AF_INET, 
+										0 , { 0 }, { 0,0,0,0,0,0,0,0 } };
+static struct sockaddr_in icmpdst = { sizeof (struct sockaddr_in), AF_INET, 
+										0 , { 0 }, { 0,0,0,0,0,0,0,0 } };
+static struct sockaddr_in icmpgw = { sizeof (struct sockaddr_in), AF_INET, 
+										0 , { 0 }, { 0,0,0,0,0,0,0,0 } };
 
 /*
  * Process a received ICMP message.
  */
 void
-icmp_input(m, hlen)
-	register struct mbuf *m;
-	int hlen;
+icmp_input(struct mbuf *m, int hlen)
 {
-	register struct icmp *icp;
-	register struct ip *ip = mtod(m, struct ip *);
+	struct icmp *icp;
+	struct ip *ip = mtod(m, struct ip *);
 	int icmplen = ip->ip_len;
-	register int i;
+	int i;
 	struct in_ifaddr *ia;
 	void (*ctlfunc)(int, struct sockaddr *, void *);
 	int code;
-	char ipv4str[MAX_IPv4_STR_LEN];
 
 	/*
 	 * Locate icmp structure in mbuf, and check
@@ -304,6 +322,7 @@ icmp_input(m, hlen)
 #if ICMPPRINTFS
 	if (icmpprintfs) {
 		char buf[MAX_IPv4_STR_LEN];
+		char ipv4str[MAX_IPv4_STR_LEN];
 
 		printf("icmp_input from %s to %s, len %d\n",
 		       inet_ntop(AF_INET, &ip->ip_src, buf, sizeof(buf)),
@@ -558,7 +577,7 @@ icmp_input(m, hlen)
 			break;
 		if (ia->ia_ifp == 0) {
 			ifafree(&ia->ia_ifa);
-			ia = 0;
+			ia = NULL;
 			break;
 		}
 		icp->icmp_type = ICMP_MASKREPLY;
@@ -658,13 +677,12 @@ freeit:
  * Reflect the ip packet back to the source
  */
 static void
-icmp_reflect(m)
-	struct mbuf *m;
+icmp_reflect(struct mbuf *m)
 {
-	register struct ip *ip = mtod(m, struct ip *);
-	register struct in_ifaddr *ia;
+	struct ip *ip = mtod(m, struct ip *);
+	struct in_ifaddr *ia;
 	struct in_addr t;
-	struct mbuf *opts = 0;
+	struct mbuf *opts = NULL;
 	int optlen = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof(struct ip);
 
 	if (!in_canforward(ip->ip_src) &&
@@ -709,6 +727,9 @@ icmp_reflect(m)
 		ifaref(&ia->ia_ifa);
 	}
 	lck_mtx_unlock(rt_mtx);
+#if CONFIG_MACF_NET
+	mac_netinet_icmp_reply(m);
+#endif
 	t = IA_SIN(ia)->sin_addr;
 	ip->ip_src = t;
 	ip->ip_ttl = ip_defttl;
@@ -716,7 +737,7 @@ icmp_reflect(m)
 	ia = NULL;
 
 	if (optlen > 0) {
-		register u_char *cp;
+		u_char *cp;
 		int opt, cnt;
 		u_int len;
 
@@ -726,7 +747,7 @@ icmp_reflect(m)
 		 */
 		cp = (u_char *) (ip + 1);
 		if ((opts = ip_srcroute()) == 0 &&
-		    (opts = m_gethdr(M_DONTWAIT, MT_HEADER))) {
+		    (opts = m_gethdr(M_DONTWAIT, MT_HEADER))) {	/* MAC-OK */
 			opts->m_len = sizeof(struct in_addr);
 			mtod(opts, struct in_addr *)->s_addr = 0;
 		}
@@ -799,15 +820,12 @@ done:
  * after supplying a checksum.
  */
 static void
-icmp_send(m, opts)
-	register struct mbuf *m;
-	struct mbuf *opts;
+icmp_send(struct mbuf *m, struct mbuf *opts)
 {
-	register struct ip *ip = mtod(m, struct ip *);
-	register int hlen;
-	register struct icmp *icp;
+	struct ip *ip = mtod(m, struct ip *);
+	int hlen;
+	struct icmp *icp;
 	struct route ro;
-	char ipv4str[MAX_IPv4_STR_LEN];
 
 	hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 	m->m_data += hlen;
@@ -817,13 +835,13 @@ icmp_send(m, opts)
 	icp->icmp_cksum = in_cksum(m, ip->ip_len - hlen);
 	m->m_data -= hlen;
 	m->m_len += hlen;
-	m->m_pkthdr.rcvif = 0;
-	m->m_pkthdr.aux = NULL;
+	m->m_pkthdr.rcvif = NULL;
 	m->m_pkthdr.csum_data = 0;
 	m->m_pkthdr.csum_flags = 0;
 #if ICMPPRINTFS
 	if (icmpprintfs) {
 		char buf[MAX_IPv4_STR_LEN];
+		char ipv4str[MAX_IPv4_STR_LEN];
 
 		printf("icmp_send dst %s src %s\n",
 		       inet_ntop(AF_INET, &ip->ip_dst, buf, sizeof(buf)),
@@ -831,13 +849,13 @@ icmp_send(m, opts)
 	}
 #endif
 	bzero(&ro, sizeof ro);
-	(void) ip_output(m, opts, &ro, 0, NULL);
+	(void) ip_output(m, opts, &ro, 0, NULL, NULL);
 	if (ro.ro_rt)
 		rtfree(ro.ro_rt);
 }
 
 n_time
-iptime()
+iptime(void)
 {
 	struct timeval atv;
 	u_long t;
@@ -854,9 +872,7 @@ iptime()
  * is returned; otherwise, a smaller value is returned.
  */
 static int
-ip_next_mtu(mtu, dir)
-	int mtu;
-	int dir;
+ip_next_mtu(int mtu, int dir)
 {
 	static int mtutab[] = {
 		65535, 32000, 17914, 8166, 4352, 2002, 1492, 1006, 508, 296,
@@ -1003,10 +1019,10 @@ __private_extern__ struct pr_usrreqs icmp_dgram_usrreqs = {
 
 /* Like rip_attach but without root privilege enforcement */
 __private_extern__ int
-icmp_dgram_attach(struct socket *so, int proto, struct proc *p)
+icmp_dgram_attach(struct socket *so, __unused int proto, struct proc *p)
 {
         struct inpcb *inp;
-        int error, s;
+        int error;
 
         inp = sotoinpcb(so);
         if (inp)
@@ -1015,9 +1031,7 @@ icmp_dgram_attach(struct socket *so, int proto, struct proc *p)
         error = soreserve(so, rip_sendspace, rip_recvspace);
         if (error)
                 return error;
-        s = splnet();
         error = in_pcballoc(so, &ripcbinfo, p);
-        splx(s);
         if (error)
                 return error;
         inp = (struct inpcb *)so->so_pcb;       
@@ -1033,8 +1047,7 @@ icmp_dgram_attach(struct socket *so, int proto, struct proc *p)
 __private_extern__ int
 icmp_dgram_ctloutput(struct socket *so, struct sockopt *sopt)
 {
-	struct	inpcb *inp = sotoinpcb(so);
-	int	error, optval;
+	int	error;
 
 	if (sopt->sopt_level != IPPROTO_IP)
 		return (EINVAL);
@@ -1164,4 +1177,3 @@ bad:
 }
 
 #endif /* __APPLE__ */
-
diff --git a/bsd/netinet/ip_icmp.h b/bsd/netinet/ip_icmp.h
index 8aff81cb0..556fb9c56 100644
--- a/bsd/netinet/ip_icmp.h
+++ b/bsd/netinet/ip_icmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/ip_id.c b/bsd/netinet/ip_id.c
index 42630415b..7f1363a79 100644
--- a/bsd/netinet/ip_id.c
+++ b/bsd/netinet/ip_id.c
@@ -57,7 +57,6 @@
  * This avoids reuse issues caused by reseeding.
  */
 
-#include "opt_random_ip_id.h"
 #include <sys/param.h>
 #include <sys/time.h>
 #include <sys/kernel.h>
@@ -72,7 +71,7 @@
 #define RU_M	31104		/* RU_M = 2^7*3^5 - don't change */
 
 #define PFAC_N 3
-const static u_int16_t pfacts[PFAC_N] = {
+static u_int16_t pfacts[PFAC_N] = {
 	2, 
 	3,
 	2729
@@ -91,6 +90,8 @@ static u_int16_t pmod(u_int16_t, u_int16_t, u_int16_t);
 static void ip_initid(void);
 u_int16_t ip_randomid(void);
 
+extern u_short ip_id;
+extern int ip_use_randomid;
 /*
  * Do a fast modular exponation, returned value will be in the range
  * of 0 - (mod-1)
@@ -133,10 +134,9 @@ ip_initid(void)
 {
 	u_int16_t j, i;
 	int noprime = 1;
-	struct timeval time;
-
-	getmicrouptime(&time);
-	read_random((void *) &tmp, sizeof(tmp));
+	struct timeval timenow;
+	
+	getmicrotime(&timenow);
 	ru_x = (tmp & 0xFFFF) % RU_M;
 
 	/* 15 bits of random seed */
@@ -176,7 +176,7 @@ ip_initid(void)
 	ru_g = pmod(RU_GEN,j,RU_N);
 	ru_counter = 0;
 
-	ru_reseed = time.tv_sec + RU_OUT;
+	ru_reseed = timenow.tv_sec + RU_OUT;
 	ru_msb = ru_msb == 0x8000 ? 0 : 0x8000; 
 }
 
@@ -184,10 +184,18 @@ u_int16_t
 ip_randomid(void)
 {
 	int i, n;
-	struct timeval time;
+	struct timeval timenow;
+
+	/* if net.inet.ip.random_id is disabled,
+	 * reverts to the incrementing ip_id
+	 */
+
+	if (ip_use_randomid == 0)
+		return htons(ip_id++);
+
 
-	getmicrouptime(&time);
-	if (ru_counter >= RU_MAX || time.tv_sec > ru_reseed)
+	getmicrotime(&timenow);
+	if (ru_counter >= RU_MAX || timenow.tv_sec > ru_reseed)
 		ip_initid();
 
 	if (!tmp)
diff --git a/bsd/netinet/ip_input.c b/bsd/netinet/ip_input.c
index b94ebd468..225164fd6 100644
--- a/bsd/netinet/ip_input.c
+++ b/bsd/netinet/ip_input.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1993
@@ -54,6 +60,12 @@
  *	@(#)ip_input.c	8.2 (Berkeley) 1/4/94
  * $FreeBSD: src/sys/netinet/ip_input.c,v 1.130.2.25 2001/08/29 21:41:37 jesper Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2007 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #define	_IP_VHL
 
@@ -72,6 +84,8 @@
 #include <kern/queue.h>
 #include <kern/locks.h>
 
+#include <pexpert/pexpert.h>
+
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_dl.h>
@@ -97,7 +111,12 @@
 #include <netinet/udp_var.h>
 #include <netinet/bootp.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #include <sys/kdebug.h>
+#include <libkern/OSAtomic.h>
 
 #define DBG_LAYER_BEG		NETDBG_CODE(DBG_NETIP, 0)
 #define DBG_LAYER_END		NETDBG_CODE(DBG_NETIP, 2)
@@ -187,10 +206,12 @@ static int	ip_checkinterface = 0;
 SYSCTL_INT(_net_inet_ip, OID_AUTO, check_interface, CTLFLAG_RW,
     &ip_checkinterface, 0, "Verify packet arrives on correct interface");
 
+
 #if DIAGNOSTIC
 static int	ipprintfs = 0;
 #endif
 
+extern int in_proto_count; 
 extern	struct domain inetdomain;
 extern	struct protosw inetsw[];
 struct protosw *ip_protox[IPPROTO_MAX];
@@ -217,12 +238,12 @@ static struct ipq ipq[IPREASS_NHASH];
 static TAILQ_HEAD(ipq_list, ipq) ipq_list =
 	TAILQ_HEAD_INITIALIZER(ipq_list);
 const  int    ipintrq_present = 1;
-lck_mtx_t			*ip_mutex;
+lck_mtx_t		*ip_mutex;
 lck_attr_t		*ip_mutex_attr;
-lck_grp_t			*ip_mutex_grp;
-lck_grp_attr_t	*ip_mutex_grp_attr;
+lck_grp_t		*ip_mutex_grp;
+lck_grp_attr_t		*ip_mutex_grp_attr;
 lck_mtx_t 		*inet_domain_mutex;
-extern lck_mtx_t 		*domain_proto_mtx;
+extern lck_mtx_t 	*domain_proto_mtx;
 
 #if IPCTL_DEFMTU
 SYSCTL_INT(_net_inet_ip, IPCTL_DEFMTU, mtu, CTLFLAG_RW,
@@ -238,8 +259,9 @@ SYSCTL_INT(_net_inet_ip, OID_AUTO, stealth, CTLFLAG_RW,
 
 /* Firewall hooks */
 ip_fw_chk_t *ip_fw_chk_ptr;
-int fw_enable = 1 ;
-int fw_one_pass = 1;
+int fw_enable = 1;
+int fw_bypass = 1;
+int fw_one_pass = 0;
 
 #if DUMMYNET
 ip_dn_io_t *ip_dn_io_ptr;
@@ -247,14 +269,14 @@ ip_dn_io_t *ip_dn_io_ptr;
 
 int (*fr_checkp)(struct ip *, int, struct ifnet *, int, struct mbuf **) = NULL;
 
-SYSCTL_NODE(_net_inet_ip, OID_AUTO, linklocal, CTLFLAG_RW, 0, "link local");
+SYSCTL_NODE(_net_inet_ip, OID_AUTO, linklocal, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "link local");
 
 struct ip_linklocal_stat ip_linklocal_stat;
 SYSCTL_STRUCT(_net_inet_ip_linklocal, OID_AUTO, stat, CTLFLAG_RD,
         &ip_linklocal_stat, ip_linklocal_stat,
         "Number of link local packets with TTL less than 255");
 
-SYSCTL_NODE(_net_inet_ip_linklocal, OID_AUTO, in, CTLFLAG_RW, 0, "link local input");
+SYSCTL_NODE(_net_inet_ip_linklocal, OID_AUTO, in, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "link local input");
 
 int ip_linklocal_in_allowbadttl = 1;
 SYSCTL_INT(_net_inet_ip_linklocal_in, OID_AUTO, allowbadttl, CTLFLAG_RW,
@@ -277,9 +299,6 @@ static	struct ip_srcrt {
 	struct	in_addr route[MAX_IPOPTLEN/sizeof(struct in_addr)];
 } ip_srcrt;
 
-#ifdef __APPLE__
-extern struct mbuf*	m_dup(register struct mbuf *m, int how);
-#endif
 
 static void	save_rte(u_char *, struct in_addr);
 static int	ip_dooptions(struct mbuf *, int, struct sockaddr_in *, struct route *ipforward_rt);
@@ -297,25 +316,28 @@ static struct	mbuf *ip_reass(struct mbuf *,
 static struct	mbuf *ip_reass(struct mbuf *, struct ipq *, struct ipq *);
 #endif
 void	ipintr(void);
+void	in_dinit(void);
 
 #if RANDOM_IP_ID
 extern u_short ip_id;
+
+int	ip_use_randomid = 1;
+SYSCTL_INT(_net_inet_ip, OID_AUTO, random_id, CTLFLAG_RW,
+    &ip_use_randomid, 0, "Randomize IP packets IDs");
 #endif
 
 extern u_long  route_generation;
-extern int apple_hwcksum_rx;
 
 /*
  * IP initialization: fill in IP protocol switch table.
  * All protocols not implemented in kernel go to raw IP protocol handler.
  */
 void
-ip_init()
+ip_init(void)
 {
-	register struct protosw *pr;
-	register int i;
-	static ip_initialized = 0;
-	struct timeval timenow;
+	struct protosw *pr;
+	int i;
+	static int ip_initialized = 0;
 
 
 	if (!ip_initialized)
@@ -340,8 +362,11 @@ ip_init()
 	maxfragsperpacket = 128; /* enough for 64k in 512 byte fragments */
 
 #if RANDOM_IP_ID
-		getmicrouptime(&timenow);
-		ip_id = timenow.tv_sec & 0xffff;
+		{
+			struct timeval timenow;
+			getmicrotime(&timenow);
+			ip_id = timenow.tv_sec & 0xffff;
+		}
 #endif
 		ipintrq.ifq_maxlen = ipqmaxlen;
 
@@ -358,30 +383,54 @@ ip_init()
 			return;
 		}
 
+#if IPSEC
+	
+		sadb_stat_mutex_grp_attr = lck_grp_attr_alloc_init();
+		sadb_stat_mutex_grp = lck_grp_alloc_init("sadb_stat", sadb_stat_mutex_grp_attr);
+		sadb_stat_mutex_attr = lck_attr_alloc_init();
+
+		if ((sadb_stat_mutex = lck_mtx_alloc_init(sadb_stat_mutex_grp, sadb_stat_mutex_attr)) == NULL) {
+			printf("ip_init: can't alloc sadb_stat_mutex\n");
+			return;
+		}
+
+#endif
 		ip_initialized = 1;
 	}
 }
 
 static void
 ip_proto_input(
-	protocol_family_t	protocol,
-	mbuf_t				packet)
+	protocol_family_t	__unused protocol,
+	mbuf_t				packet_list)
 {
-	ip_input(packet);
+	mbuf_t	packet;
+	int how_many = 0 ;
+	
+	/* ip_input should handle a list of packets but does not yet */
+	
+	for (packet = packet_list; packet; packet = packet_list) {
+		how_many++;
+		packet_list = mbuf_nextpkt(packet);
+		mbuf_setnextpkt(packet, NULL);
+		ip_input(packet);
+	}
 }
 
 /* Initialize the PF_INET domain, and add in the pre-defined protos */
 void
-in_dinit()
-{	register int i;
-	register struct protosw *pr;
-	register struct domain *dp;
-	static inetdomain_initted = 0;
-	extern int in_proto_count; 
+in_dinit(void)
+{
+	int i;
+	struct protosw *pr;
+	struct domain *dp;
+	static int inetdomain_initted = 0;
 
 	if (!inetdomain_initted)
 	{
+#if 0
 		kprintf("Initing %d protosw entries\n", in_proto_count);
+#endif
 		dp = &inetdomain;
 		dp->dom_flags = DOM_REENTRANT;
 
@@ -391,7 +440,7 @@ in_dinit()
 		inetdomain_initted = 1;
 	
 		lck_mtx_unlock(domain_proto_mtx);	
-		proto_register_input(PF_INET, ip_proto_input, NULL);
+		proto_register_input(PF_INET, ip_proto_input, NULL, 1);
 		lck_mtx_lock(domain_proto_mtx);	
 	}
 }
@@ -466,7 +515,7 @@ ip_proto_dispatch_in(
  * ipforward_rt cleared in in_addroute()
  * when a new route is successfully created.
  */
-static struct	sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET };
+static struct	sockaddr_in ipaddr = { sizeof(ipaddr), AF_INET , 0 , {0}, {0,0,0,0,0,0,0,0} };
 
 /*
  * Ip input routine.  Checksum and byte swap header.  If fragmented
@@ -478,17 +527,18 @@ ip_input(struct mbuf *m)
 	struct ip *ip;
 	struct ipq *fp;
 	struct in_ifaddr *ia = NULL;
-	int    i, hlen, mff, checkif;
+	int    i, hlen, checkif;
 	u_short sum;
 	struct in_addr pkt_dst;
 	u_int32_t div_info = 0;		/* packet divert/tee info */
 	struct ip_fw_args args;
 	ipfilter_t inject_filter_ref = 0;
 	struct m_tag	*tag;
-	struct route	ipforward_rt = { 0 };
+	struct route	ipforward_rt;
 	
-	lck_mtx_lock(ip_mutex);
-
+	bzero(&ipforward_rt, sizeof(struct route));
+	
+#if IPFIREWALL
 	args.eh = NULL;
 	args.oif = NULL;
 	args.rule = NULL;
@@ -535,6 +585,7 @@ ip_input(struct mbuf *m)
             inject_filter_ref = ipf_get_inject_filter(m);
             goto iphack ;
 	}
+#endif /* IPFIREWALL */
 	
 	/*
 	 * No need to proccess packet twice if we've 
@@ -542,7 +593,6 @@ ip_input(struct mbuf *m)
 	 */
 	inject_filter_ref = ipf_get_inject_filter(m);
 	if (inject_filter_ref != 0) {
-		lck_mtx_unlock(ip_mutex);
 		ip = mtod(m, struct ip *);
 		hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 		ip->ip_len = ntohs(ip->ip_len) - hlen;
@@ -551,15 +601,14 @@ ip_input(struct mbuf *m)
 		return;
 	}
 
-	ipstat.ips_total++;
+	OSAddAtomic(1, (SInt32*)&ipstat.ips_total);
 
 	if (m->m_pkthdr.len < sizeof(struct ip))
 		goto tooshort;
 
 	if (m->m_len < sizeof (struct ip) &&
 	    (m = m_pullup(m, sizeof (struct ip))) == 0) {
-		ipstat.ips_toosmall++;
-		lck_mtx_unlock(ip_mutex);
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_toosmall);
 		return;
 	}
 	ip = mtod(m, struct ip *);
@@ -568,19 +617,18 @@ ip_input(struct mbuf *m)
 		     ip->ip_src.s_addr, ip->ip_p, ip->ip_off, ip->ip_len);
 
 	if (IP_VHL_V(ip->ip_vhl) != IPVERSION) {
-		ipstat.ips_badvers++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_badvers);
 		goto bad;
 	}
 
 	hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 	if (hlen < sizeof(struct ip)) {	/* minimum header length */
-		ipstat.ips_badhlen++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_badhlen);
 		goto bad;
 	}
 	if (hlen > m->m_len) {
 		if ((m = m_pullup(m, hlen)) == 0) {
-			ipstat.ips_badhlen++;
-			lck_mtx_unlock(ip_mutex);
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_badhlen);
 			return;
 		}
 		ip = mtod(m, struct ip *);
@@ -590,7 +638,7 @@ ip_input(struct mbuf *m)
 	if ((ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 	    (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) {
 		if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) == 0) {
-			ipstat.ips_badaddr++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_badaddr);
 			goto bad;
 		}
 	}
@@ -600,7 +648,7 @@ ip_input(struct mbuf *m)
 	    IN_LINKLOCAL(ntohl(ip->ip_src.s_addr)))) {
 		ip_linklocal_stat.iplls_in_total++;
 		if (ip->ip_ttl != MAXTTL) {
-			ip_linklocal_stat.iplls_in_badttl++;
+			OSAddAtomic(1, (SInt32*)&ip_linklocal_stat.iplls_in_badttl);
 			/* Silently drop link local traffic with bad TTL */
 			if (!ip_linklocal_in_allowbadttl)
 				goto bad;
@@ -614,11 +662,30 @@ ip_input(struct mbuf *m)
 
 	if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) {
 		sum = !(m->m_pkthdr.csum_flags & CSUM_IP_VALID);
-	} else {
+	} else if (!(m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) ||
+	    apple_hwcksum_tx == 0) {
+		/*
+		 * Either this is not loopback packet coming from an interface
+		 * that does not support checksum offloading, or it is loopback
+		 * packet that has undergone software checksumming at the send
+		 * side because apple_hwcksum_tx was set to 0.  In this case,
+		 * calculate the checksum in software to validate the packet.
+		 */
 		sum = in_cksum(m, hlen);
+	} else {
+		/*
+		 * This is a loopback packet without any valid checksum since
+		 * the send side has bypassed it (apple_hwcksum_tx set to 1).
+		 * We get here because apple_hwcksum_rx was set to 0, and so
+		 * we pretend that all is well.
+		 */
+		sum = 0;
+		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
+		    CSUM_IP_CHECKED | CSUM_IP_VALID;
+                m->m_pkthdr.csum_data = 0xffff;
 	}
 	if (sum) {
-		ipstat.ips_badsum++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_badsum);
 		goto bad;
 	}
 
@@ -627,7 +694,7 @@ ip_input(struct mbuf *m)
 	 */
 	NTOHS(ip->ip_len);
 	if (ip->ip_len < hlen) {
-		ipstat.ips_badlen++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_badlen);
 		goto bad;
 	}
 	NTOHS(ip->ip_off);
@@ -640,7 +707,7 @@ ip_input(struct mbuf *m)
 	 */
 	if (m->m_pkthdr.len < ip->ip_len) {
 tooshort:
-		ipstat.ips_tooshort++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_tooshort);
 		goto bad;
 	}
 	if (m->m_pkthdr.len > ip->ip_len) {
@@ -672,9 +739,10 @@ tooshort:
 	 * - Encapsulate: put it in another IP and send out. <unimp.>
  	 */
 
-#if defined(IPFIREWALL) && defined(DUMMYNET)
+#if IPFIREWALL
+#if DUMMYNET
 iphack:
-#endif
+#endif /* DUMMYNET */
 	/*
 	 * Check if we want to allow this packet to be processed.
 	 * Consider it to be bad if not.
@@ -683,7 +751,6 @@ iphack:
 		struct	mbuf	*m1 = m;
 
 		if (fr_checkp(ip, hlen, m->m_pkthdr.rcvif, 0, &m1) || !m1) {
-			lck_mtx_unlock(ip_mutex);
 			return;
 		}
 		ip = mtod(m = m1, struct ip *);
@@ -699,7 +766,6 @@ iphack:
 #endif	/* IPFIREWALL_FORWARD */
 
 		args.m = m;
-		lck_mtx_unlock(ip_mutex);
 
 		i = ip_fw_chk_ptr(&args);
 		m = args.m;
@@ -710,8 +776,8 @@ iphack:
 			return;
 		}
 		ip = mtod(m, struct ip *); /* just in case m changed */
+		
 		if (i == 0 && args.next_hop == NULL) {	/* common case */
-			lck_mtx_lock(ip_mutex);
 			goto pass;
 		}
 #if DUMMYNET
@@ -724,14 +790,12 @@ iphack:
 #if IPDIVERT
 		if (i != 0 && (i & IP_FW_PORT_DYNT_FLAG) == 0) {
 			/* Divert or tee packet */
-			lck_mtx_lock(ip_mutex);
 			div_info = i;
 			goto ours;
 		}
 #endif
 #if IPFIREWALL_FORWARD
 		if (i == 0 && args.next_hop != NULL) {
-			lck_mtx_lock(ip_mutex);
 			goto pass;
 		}
 #endif
@@ -741,6 +805,7 @@ iphack:
 		m_freem(m);
 		return;
 	}
+#endif /* IPFIREWALL */
 pass:
 
 	/*
@@ -751,7 +816,6 @@ pass:
 	 */
 	ip_nhops = 0;		/* for source routed packets */
 	if (hlen > sizeof (struct ip) && ip_dooptions(m, 0, args.next_hop, &ipforward_rt)) {
-		lck_mtx_unlock(ip_mutex);
 		return;
 	}
 
@@ -841,6 +905,7 @@ pass:
 	lck_mtx_unlock(rt_mtx);
 	if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) {
 		struct in_multi *inm;
+#if MROUTING
 		if (ip_mrouter) {
 			/*
 			 * If we are acting as a multicast router, all
@@ -850,9 +915,10 @@ pass:
 			 * ip_mforward() returns a non-zero value, the packet
 			 * must be discarded, else it may be accepted below.
 			 */
+			lck_mtx_lock(ip_mutex);
 			if (ip_mforward &&
 			    ip_mforward(ip, m->m_pkthdr.rcvif, m, 0) != 0) {
-				ipstat.ips_cantforward++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_cantforward);
 				m_freem(m);
 				lck_mtx_unlock(ip_mutex);
 				return;
@@ -865,17 +931,17 @@ pass:
 			 */
 			if (ip->ip_p == IPPROTO_IGMP)
 				goto ours;
-			ipstat.ips_forward++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_forward);
 		}
+#endif /* MROUTING */
 		/*
 		 * See if we belong to the destination multicast group on the
 		 * arrival interface.
 		 */
 		IN_LOOKUP_MULTI(ip->ip_dst, m->m_pkthdr.rcvif, inm);
 		if (inm == NULL) {
-			ipstat.ips_notmember++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_notmember);
 			m_freem(m);
-			lck_mtx_unlock(ip_mutex);
 			return;
 		}
 		goto ours;
@@ -893,8 +959,7 @@ pass:
 		struct udpiphdr *ui;
 		if (m->m_len < sizeof(struct udpiphdr)
 		    && (m = m_pullup(m, sizeof(struct udpiphdr))) == 0) {
-			udpstat.udps_hdrops++;
-			lck_mtx_unlock(ip_mutex);
+			OSAddAtomic(1, (SInt32*)&udpstat.udps_hdrops);
 			return;
 		}
 		ui = mtod(m, struct udpiphdr *);
@@ -914,32 +979,25 @@ pass:
 				goto ours;
 		}
 		m_freem(m);
-		lck_mtx_unlock(ip_mutex);
 		return;
 	}
 #endif
-	lck_mtx_unlock(ip_mutex);
 	/*
 	 * Not for us; forward if possible and desirable.
 	 */
 	if (ipforwarding == 0) {
-		ipstat.ips_cantforward++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_cantforward);
 		m_freem(m);
 	} else {
 		ip_forward(m, 0, args.next_hop, &ipforward_rt);
+		if (ipforward_rt.ro_rt != NULL) {
+			rtfree(ipforward_rt.ro_rt);
+			ipforward_rt.ro_rt = NULL;
+		}
 	}
 	return;
 
 ours:
-#ifndef __APPLE__
-	/* Darwin does not have an if_data in ifaddr */
-	/* Count the packet in the ip address stats */
-	if (ia != NULL) {
-		ia->ia_ifa.if_ipackets++;
-		ia->ia_ifa.if_ibytes += m->m_pkthdr.len;
-	}
-#endif
-
 	/*
 	 * If offset or IP_MF are set, must reassemble.
 	 * Otherwise, nothing need be done.
@@ -951,8 +1009,9 @@ ours:
 
 		/* If maxnipq is 0, never accept fragments. */
 		if (maxnipq == 0) {
-			ipstat.ips_fragments++;
-			ipstat.ips_fragdropped++;
+
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_fragments);
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_fragdropped);
 			goto bad;
 		}
 		
@@ -960,9 +1019,10 @@ ours:
 		 * If we will exceed the number of fragments in queues, timeout the
 		 * oldest fragemented packet to make space.
 		 */
+		lck_mtx_lock(ip_mutex); 
 		if (currentfrags >= maxfrags) {
 			fp = TAILQ_LAST(&ipq_list, ipq_list);
-			ipstat.ips_fragtimeout += fp->ipq_nfrags;
+			OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragtimeout);
 			
 			if (ip->ip_id == fp->ipq_id &&
 				ip->ip_src.s_addr == fp->ipq_src.s_addr &&
@@ -972,8 +1032,9 @@ ours:
 				 * If we match the fragment queue we were going to
 				 * discard, drop this packet too.
 				 */
-				ipstat.ips_fragdropped++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_fragdropped);
 				ip_freef(fp);
+				lck_mtx_unlock(ip_mutex); 
 				goto bad;
 			}
 			
@@ -989,6 +1050,9 @@ ours:
 			if (ip->ip_id == fp->ipq_id &&
 			    ip->ip_src.s_addr == fp->ipq_src.s_addr &&
 			    ip->ip_dst.s_addr == fp->ipq_dst.s_addr &&
+#if CONFIG_MACF_NET
+			    mac_ipq_label_compare(m, fp) &&
+#endif
 			    ip->ip_p == fp->ipq_p)
 				goto found;
 
@@ -1002,7 +1066,7 @@ ours:
 		     * drop the oldest fragment before proceeding further
 		     */
 		    fp = TAILQ_LAST(&ipq_list, ipq_list);
-		    ipstat.ips_fragtimeout += fp->ipq_nfrags;
+		    OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragtimeout);
 		    ip_freef(fp);
 		}
 
@@ -1020,7 +1084,8 @@ found:
 				 * that's a non-zero multiple of 8 bytes.
 		         */
 			if (ip->ip_len == 0 || (ip->ip_len & 0x7) != 0) {
-				ipstat.ips_toosmall++; /* XXX */
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_toosmall);
+				lck_mtx_unlock(ip_mutex);
 				goto bad;
 			}
 			m->m_flags |= M_FRAG;
@@ -1035,7 +1100,7 @@ found:
 		 * ip_reass() will return a different mbuf, and update
 		 * the divert info in div_info and args.divert_rule.
 		 */
-			ipstat.ips_fragments++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_fragments);
 			m->m_pkthdr.header = ip;
 #if IPDIVERT
 			m = ip_reass(m,
@@ -1047,7 +1112,7 @@ found:
 				lck_mtx_unlock(ip_mutex);
 				return;
 			}
-			ipstat.ips_reassembled++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_reassembled);
 			ip = mtod(m, struct ip *);
 			/* Get the header length of the reassembled packet */
 			hlen = IP_VHL_HL(ip->ip_vhl) << 2;
@@ -1064,6 +1129,7 @@ found:
 				ip->ip_len -= hlen;
 			}
 #endif
+		lck_mtx_unlock(ip_mutex);
 		} else
 		ip->ip_len -= hlen;
 
@@ -1087,15 +1153,13 @@ found:
 		HTONS(ip->ip_off);
 
 		/* Deliver packet to divert input routine */
-		ipstat.ips_delivered++;
-		lck_mtx_unlock(ip_mutex);
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_delivered);
 		divert_packet(m, 1, div_info & 0xffff, args.divert_rule);
 
 		/* If 'tee', continue with original packet */
 		if (clone == NULL) {
 			return;
 		}
-		lck_mtx_lock(ip_mutex);
 		m = clone;
 		ip = mtod(m, struct ip *);
 	}
@@ -1108,20 +1172,17 @@ found:
 	 * code - like udp/tcp/raw ip.
 	 */
 	if (ipsec_bypass == 0 && (ip_protox[ip->ip_p]->pr_flags & PR_LASTHDR) != 0) {
-		lck_mtx_lock(sadb_mutex);
 		if (ipsec4_in_reject(m, NULL)) {
-			ipsecstat.in_polvio++;
-			lck_mtx_unlock(sadb_mutex);
-			goto bad;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
+			goto bad;	
 		}
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif
 
 	/*
 	 * Switch out to protocol's input routine.
 	 */
-	ipstat.ips_delivered++;
+	OSAddAtomic(1, (SInt32*)&ipstat.ips_delivered);
 	{
 		if (args.next_hop && ip->ip_p == IPPROTO_TCP) {
 			/* TCP needs IPFORWARD info if available */
@@ -1142,7 +1203,6 @@ found:
 			KERNEL_DEBUG(DBG_LAYER_END, ip->ip_dst.s_addr, 
 			     ip->ip_src.s_addr, ip->ip_p, ip->ip_off, ip->ip_len);
 	
-			lck_mtx_unlock(ip_mutex);
 	
 			/* TCP deals with its own locking */
 			ip_proto_dispatch_in(m, hlen, ip->ip_p, 0);
@@ -1150,7 +1210,6 @@ found:
 			KERNEL_DEBUG(DBG_LAYER_END, ip->ip_dst.s_addr, 
 			     ip->ip_src.s_addr, ip->ip_p, ip->ip_off, ip->ip_len);
 		
-			lck_mtx_unlock(ip_mutex);
 			ip_proto_dispatch_in(m, hlen, ip->ip_p, 0);
 		}
 		
@@ -1158,7 +1217,6 @@ found:
 	}
 bad:
 	KERNEL_DEBUG(DBG_LAYER_END, 0,0,0,0,0);
-	lck_mtx_unlock(ip_mutex);
 	m_freem(m);
 }
 
@@ -1173,28 +1231,25 @@ bad:
 
 static struct mbuf *
 #if IPDIVERT
-ip_reass(m, fp, where, divinfo, divcookie)
-#else
-ip_reass(m, fp, where)
-#endif
-	register struct mbuf *m;
-	register struct ipq *fp;
-	struct   ipq    *where;
-#if IPDIVERT
+ip_reass(struct mbuf *m, struct ipq *fp, struct ipq *where,
 #ifdef IPDIVERT_44
-	u_int32_t *divinfo;
-#else
-	u_int16_t *divinfo;
-#endif
-	u_int16_t *divcookie;
-#endif
+	 u_int32_t *divinfo,
+#else /* IPDIVERT_44 */
+	 u_int16_t *divinfo,
+#endif /* IPDIVERT_44 */
+	 u_int16_t *divcookie)
+#else /* IPDIVERT */
+ip_reass(struct mbuf *m, struct ipq *fp, struct ipq *where)
+#endif /* IPDIVERT */
 {
 	struct ip *ip = mtod(m, struct ip *);
-	register struct mbuf *p = 0, *q, *nq;
+	struct mbuf *p = 0, *q, *nq;
 	struct mbuf *t;
 	int hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 	int i, next;
+	u_int8_t ecn, ecn0;
 
+	lck_mtx_assert(ip_mutex, LCK_MTX_ASSERT_OWNED);
 	/*
 	 * Presence of header sizes in mbufs
 	 * would confuse code below.
@@ -1211,6 +1266,14 @@ ip_reass(m, fp, where)
 		if ((t = m_get(M_DONTWAIT, MT_FTABLE)) == NULL)
 			goto dropfrag;
 		fp = mtod(t, struct ipq *);
+#if CONFIG_MACF_NET
+		if (mac_ipq_label_init(fp, M_NOWAIT) != 0) {
+			m_free(t);
+			fp = NULL;
+			goto dropfrag;
+		}
+		mac_ipq_label_associate(m, fp);
+#endif
 		insque((void*)fp, (void*)where);
 		nipq++;
 		fp->ipq_nfrags = 1;
@@ -1233,10 +1296,29 @@ ip_reass(m, fp, where)
 		goto inserted;
 	} else {
 		fp->ipq_nfrags++;
+#if CONFIG_MACF_NET
+		mac_ipq_label_update(m, fp);
+#endif
 	}
 
 #define GETIP(m)	((struct ip*)((m)->m_pkthdr.header))
 
+	/*
+	 * Handle ECN by comparing this segment with the first one;
+	 * if CE is set, do not lose CE.
+	 * drop if CE and not-ECT are mixed for the same packet.
+	 */
+	ecn = ip->ip_tos & IPTOS_ECN_MASK;
+	ecn0 = GETIP(fp->ipq_frags)->ip_tos & IPTOS_ECN_MASK;
+	if (ecn == IPTOS_ECN_CE) {
+		if (ecn0 == IPTOS_ECN_NOTECT)
+			goto dropfrag;
+		if (ecn0 != IPTOS_ECN_CE)
+			GETIP(fp->ipq_frags)->ip_tos |= IPTOS_ECN_CE;
+	}
+	if (ecn == IPTOS_ECN_NOTECT && ecn0 != IPTOS_ECN_NOTECT)
+		goto dropfrag;
+
 	/*
 	 * Find a segment which begins after this one does.
 	 */
@@ -1287,7 +1369,7 @@ ip_reass(m, fp, where)
 		}
 		nq = q->m_nextpkt;
 		m->m_nextpkt = nq;
-		ipstat.ips_fragdropped++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_fragdropped);
 		fp->ipq_nfrags--;
 		m_freem(q);
 	}
@@ -1326,7 +1408,7 @@ inserted:
 	for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
 		if (GETIP(q)->ip_off != next) {
 			if (fp->ipq_nfrags > maxfragsperpacket) {
-				ipstat.ips_fragdropped += fp->ipq_nfrags;
+				OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragdropped);
 				ip_freef(fp);
 			}
 			return (0);
@@ -1336,7 +1418,7 @@ inserted:
 	/* Make sure the last packet didn't have the IP_MF flag */
 	if (p->m_flags & M_FRAG) {
 		if (fp->ipq_nfrags > maxfragsperpacket) {
-			ipstat.ips_fragdropped += fp->ipq_nfrags;
+			OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragdropped);
 			ip_freef(fp);
 		}
 		return (0);
@@ -1348,8 +1430,8 @@ inserted:
 	q = fp->ipq_frags;
 	ip = GETIP(q);
 	if (next + (IP_VHL_HL(ip->ip_vhl) << 2) > IP_MAXPACKET) {
-		ipstat.ips_toolong++;
-		ipstat.ips_fragdropped += fp->ipq_nfrags;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_toolong);
+		OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragdropped);
 		ip_freef(fp);
 		return (0);
 	}
@@ -1387,6 +1469,10 @@ inserted:
 	*divcookie = fp->ipq_div_cookie;
 #endif
 
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_ipq(fp, m);
+	mac_ipq_label_destroy(fp);
+#endif
 	/*
 	 * Create header for new ip packet by
 	 * modifying header of first packet;
@@ -1405,7 +1491,7 @@ inserted:
 	m->m_data -= (IP_VHL_HL(ip->ip_vhl) << 2);
 	/* some debugging cruft by sklower, below, will go away soon */
 	if (m->m_flags & M_PKTHDR) { /* XXX this should be done elsewhere */
-		register int plen = 0;
+		int plen = 0;
 		for (t = m; t; t = t->m_next)
 			plen += t->m_len;
 		m->m_pkthdr.len = plen;
@@ -1417,7 +1503,7 @@ dropfrag:
 	*divinfo = 0;
 	*divcookie = 0;
 #endif
-	ipstat.ips_fragdropped++;
+	OSAddAtomic(1, (SInt32*)&ipstat.ips_fragdropped);
 	if (fp != 0)
 		fp->ipq_nfrags--;
 	m_freem(m);
@@ -1431,9 +1517,9 @@ dropfrag:
  * associated datagrams.
  */
 static void
-ip_freef(fp)
-	struct ipq *fp;
+ip_freef(struct ipq *fp)
 {
+	lck_mtx_assert(ip_mutex, LCK_MTX_ASSERT_OWNED);
 	currentfrags -= fp->ipq_nfrags;
 	m_freem_list(fp->ipq_frags);
 	remque((void*)fp);
@@ -1448,9 +1534,9 @@ ip_freef(fp)
  * queue, discard it.
  */
 void
-ip_slowtimo()
+ip_slowtimo(void)
 {
-	register struct ipq *fp;
+	struct ipq *fp;
 	int i;
 	lck_mtx_lock(ip_mutex);
 	for (i = 0; i < IPREASS_NHASH; i++) {
@@ -1461,7 +1547,7 @@ ip_slowtimo()
 			--fp->ipq_ttl;
 			fp = fp->next;
 			if (fp->prev->ipq_ttl == 0) {
-				ipstat.ips_fragtimeout += fp->prev->ipq_nfrags;
+				OSAddAtomic(fp->ipq_nfrags, (SInt32*)&ipstat.ips_fragtimeout);
 				ip_freef(fp->prev);
 			}
 		}
@@ -1475,8 +1561,7 @@ ip_slowtimo()
 	for (i = 0; i < IPREASS_NHASH; i++) {
 			while (nipq > maxnipq &&
 				(ipq[i].next != &ipq[i])) {
-				ipstat.ips_fragdropped +=
-				    ipq[i].next->ipq_nfrags;
+				OSAddAtomic(ipq[i].next->ipq_nfrags, (SInt32*)&ipstat.ips_fragdropped);
 				ip_freef(ipq[i].next);
 			}
 		}
@@ -1489,14 +1574,14 @@ ip_slowtimo()
  * Drain off all datagram fragments.
  */
 void
-ip_drain()
+ip_drain(void)
 {
 	int     i;
 
 	lck_mtx_lock(ip_mutex);
 	for (i = 0; i < IPREASS_NHASH; i++) {
 		while (ipq[i].next != &ipq[i]) {
-			ipstat.ips_fragdropped += ipq[i].next->ipq_nfrags;
+			OSAddAtomic(ipq[i].next->ipq_nfrags, (SInt32*)&ipstat.ips_fragdropped);
 			ip_freef(ipq[i].next);
 		}
 	}
@@ -1517,12 +1602,12 @@ ip_drain()
  * 0 if the packet should be processed further.
  */
 static int
-ip_dooptions(struct mbuf *m, int pass, struct sockaddr_in *next_hop, struct route *ipforward_rt)
+ip_dooptions(struct mbuf *m, __unused int pass, struct sockaddr_in *next_hop, struct route *ipforward_rt)
 {
-	register struct ip *ip = mtod(m, struct ip *);
-	register u_char *cp;
-	register struct ip_timestamp *ipt;
-	register struct in_ifaddr *ia;
+	struct ip *ip = mtod(m, struct ip *);
+	u_char *cp;
+	struct ip_timestamp *ipt;
+	struct in_ifaddr *ia;
 	int opt, optlen, cnt, off, code, type = ICMP_PARAMPROB, forward = 0;
 	struct in_addr *sin, dst;
 	n_time ntime;
@@ -1622,7 +1707,7 @@ nosourcerouting:
 					/*
 					 * Not acting as a router, so silently drop.
 					 */
-					ipstat.ips_cantforward++;
+					OSAddAtomic(1, (SInt32*)&ipstat.ips_cantforward);
 					m_freem(m);
 					return (1);
 				}
@@ -1770,15 +1855,17 @@ nosourcerouting:
 	}
 	if (forward && ipforwarding) {
 		ip_forward(m, 1, next_hop, ipforward_rt);
+		if (ipforward_rt->ro_rt != NULL) {
+			rtfree(ipforward_rt->ro_rt);
+			ipforward_rt->ro_rt = NULL;
+		}
 		return (1);
 	}
 	return (0);
 bad:
 	ip->ip_len -= IP_VHL_HL(ip->ip_vhl) << 2;   /* XXX icmp_error adds in hdr length */
-	lck_mtx_unlock(ip_mutex);
 	icmp_error(m, type, code, 0, 0);
-	lck_mtx_lock(ip_mutex);
-	ipstat.ips_badoptions++;
+	OSAddAtomic(1, (SInt32*)&ipstat.ips_badoptions);
 	return (1);
 }
 
@@ -1787,11 +1874,9 @@ bad:
  * return internet address info of interface to be used to get there.
  */
 struct in_ifaddr *
-ip_rtaddr(dst, rt)
-	struct in_addr dst;
-	struct route *rt;
+ip_rtaddr(struct in_addr dst, struct route *rt)
 {
-	register struct sockaddr_in *sin;
+	struct sockaddr_in *sin;
 
 	sin = (struct sockaddr_in *)&rt->ro_dst;
 
@@ -1824,9 +1909,7 @@ ip_rtaddr(dst, rt)
  * to be picked up later by ip_srcroute if the receiver is interested.
  */
 void
-save_rte(option, dst)
-	u_char *option;
-	struct in_addr dst;
+save_rte(u_char *option, struct in_addr dst)
 {
 	unsigned olen;
 
@@ -1848,10 +1931,10 @@ save_rte(option, dst)
  * The first hop is placed before the options, will be removed later.
  */
 struct mbuf *
-ip_srcroute()
+ip_srcroute(void)
 {
-	register struct in_addr *p, *q;
-	register struct mbuf *m;
+	struct in_addr *p, *q;
+	struct mbuf *m;
 
 	if (ip_nhops == 0)
 		return ((struct mbuf *)0);
@@ -1919,13 +2002,11 @@ ip_srcroute()
  * XXX should be deleted; last arg currently ignored.
  */
 void
-ip_stripoptions(m, mopt)
-	register struct mbuf *m;
-	struct mbuf *mopt;
+ip_stripoptions(struct mbuf *m, __unused struct mbuf *mopt)
 {
-	register int i;
+	int i;
 	struct ip *ip = mtod(m, struct ip *);
-	register caddr_t opts;
+	caddr_t opts;
 	int olen;
 
 	olen = (IP_VHL_HL(ip->ip_vhl) << 2) - sizeof (struct ip);
@@ -1941,7 +2022,7 @@ ip_stripoptions(m, mopt)
 u_char inetctlerrmap[PRC_NCMDS] = {
 	0,		0,		0,		0,
 	0,		EMSGSIZE,	EHOSTDOWN,	EHOSTUNREACH,
-	EHOSTUNREACH,	EHOSTUNREACH,	ECONNREFUSED,	ECONNREFUSED,
+	ENETUNREACH,	EHOSTUNREACH,	ECONNREFUSED,	ECONNREFUSED,
 	EMSGSIZE,	EHOSTUNREACH,	0,		0,
 	0,		0,		0,		0,
 	ENOPROTOOPT,	ECONNREFUSED
@@ -1964,18 +2045,21 @@ u_char inetctlerrmap[PRC_NCMDS] = {
 static void
 ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route *ipforward_rt)
 {
-	register struct ip *ip = mtod(m, struct ip *);
-	register struct sockaddr_in *sin;
-	register struct rtentry *rt;
+	struct ip *ip = mtod(m, struct ip *);
+	struct sockaddr_in *sin;
+	struct rtentry *rt;
 	int error, type = 0, code = 0;
 	struct mbuf *mcopy;
 	n_long dest;
 	struct in_addr pkt_dst;
 	struct ifnet *destifp;
+	struct ifnet *rcvif = m->m_pkthdr.rcvif;
 #if IPSEC
 	struct ifnet dummyifp;
 #endif
 
+	m->m_pkthdr.rcvif = NULL;
+
 	dest = 0;
 	/*
 	 * Cache the destination address of the packet; this may be
@@ -1992,7 +2076,7 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 
 
 	if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(pkt_dst) == 0) {
-		ipstat.ips_cantforward++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_cantforward);
 		m_freem(m);
 		return;
 	}
@@ -2105,14 +2189,14 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 		m_tag_prepend(m, tag);
 	}
 	error = ip_output_list(m, 0, (struct mbuf *)0, ipforward_rt, 
-			  IP_FORWARDING, 0);
+			  IP_FORWARDING, 0, NULL);
     }
 	if (error)
-		ipstat.ips_cantforward++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_cantforward);
 	else {
-		ipstat.ips_forward++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_forward);
 		if (type)
-			ipstat.ips_redirectsent++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_redirectsent);
 		else {
 			if (mcopy) {
 				ipflow_create(ipforward_rt, mcopy);
@@ -2161,10 +2245,9 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 
 			if (ipsec_bypass) {
 				destifp = ipforward_rt->ro_rt->rt_ifp;
-				ipstat.ips_cantfrag++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_cantfrag);
 				break;
 			}
-			lck_mtx_lock(sadb_mutex);
 			sp = ipsec4_getpolicybyaddr(mcopy,
 						    IPSEC_DIR_OUTBOUND,
 			                            IP_FORWARDING,
@@ -2174,9 +2257,7 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 				destifp = ipforward_rt->ro_rt->rt_ifp;
 			else {
 				/* count IPsec header size */
-				ipsechdr = ipsec4_hdrsiz(mcopy,
-							 IPSEC_DIR_OUTBOUND,
-							 NULL);
+				ipsechdr = ipsec_hdrsiz(sp);
 
 				/*
 				 * find the correct route for outer IPv4
@@ -2188,24 +2269,53 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 				 */
 				/*XXX*/
 				destifp = NULL;
-				if (sp->req != NULL
-				 && sp->req->sav != NULL
-				 && sp->req->sav->sah != NULL) {
-					ro = &sp->req->sav->sah->sa_route;
-					if (ro->ro_rt && ro->ro_rt->rt_ifp) {
-						dummyifp.if_mtu =
-						    ro->ro_rt->rt_ifp->if_mtu;
-						dummyifp.if_mtu -= ipsechdr;
-						destifp = &dummyifp;
+
+				if (sp->req != NULL) {
+					if (sp->req->saidx.mode == IPSEC_MODE_TUNNEL) {
+						struct secasindex saidx;
+						struct ip *ipm;
+						struct secasvar *sav;
+						
+						ipm = mtod(mcopy, struct ip *);
+						bcopy(&sp->req->saidx, &saidx, sizeof(saidx));
+						saidx.mode = sp->req->saidx.mode;
+						saidx.reqid = sp->req->saidx.reqid;
+						sin = (struct sockaddr_in *)&saidx.src;
+						if (sin->sin_len == 0) {
+							sin->sin_len = sizeof(*sin);
+							sin->sin_family = AF_INET;
+							sin->sin_port = IPSEC_PORT_ANY;
+							bcopy(&ipm->ip_src, &sin->sin_addr,
+								sizeof(sin->sin_addr));
+						}
+						sin = (struct sockaddr_in *)&saidx.dst;
+						if (sin->sin_len == 0) {
+							sin->sin_len = sizeof(*sin);
+							sin->sin_family = AF_INET;
+							sin->sin_port = IPSEC_PORT_ANY;
+							bcopy(&ipm->ip_dst, &sin->sin_addr,
+								sizeof(sin->sin_addr));
+						}
+						sav = key_allocsa_policy(&saidx);
+						if (sav != NULL) {
+							if (sav->sah != NULL) {
+								ro = &sav->sah->sa_route;
+								if (ro->ro_rt && ro->ro_rt->rt_ifp) {
+									dummyifp.if_mtu =
+										ro->ro_rt->rt_ifp->if_mtu;
+									dummyifp.if_mtu -= ipsechdr;
+									destifp = &dummyifp;
+								}
+							}
+							key_freesav(sav, KEY_SADB_UNLOCKED);
+						}
 					}
 				}
-
-				key_freesp(sp);
+				key_freesp(sp, KEY_SADB_UNLOCKED);
 			}
-			lck_mtx_unlock(sadb_mutex);
 		}
 #endif /*IPSEC*/
-		ipstat.ips_cantfrag++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_cantfrag);
 		break;
 
 	case ENOBUFS:
@@ -2222,10 +2332,10 @@ ip_forward(struct mbuf *m, int srcrt, struct sockaddr_in *next_hop, struct route
 
 void
 ip_savecontrol(
-	register struct inpcb *inp,
-	register struct mbuf **mp,
-	register struct ip *ip,
-	register struct mbuf *m)
+	struct inpcb *inp,
+	struct mbuf **mp,
+	struct ip *ip,
+	struct mbuf *m)
 {
 	if (inp->inp_socket->so_options & SO_TIMESTAMP) {
 		struct timeval tv;
@@ -2275,10 +2385,10 @@ ip_savecontrol(
 		if (((ifp = m->m_pkthdr.rcvif)) 
 		&& ( ifp->if_index && (ifp->if_index <= if_index))) {
 			struct ifaddr *ifa = ifnet_addrs[ifp->if_index - 1];
-			
+
 			if (!ifa || !ifa->ifa_addr)
 				goto makedummy;
-			
+
 			sdp = (struct sockaddr_dl *)ifa->ifa_addr;
 			/*
 			 * Change our mind and don't try copy.
diff --git a/bsd/netinet/ip_mroute.c b/bsd/netinet/ip_mroute.c
index 584058aa0..939d9dbc4 100644
--- a/bsd/netinet/ip_mroute.c
+++ b/bsd/netinet/ip_mroute.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 /*
  * IP multicast forwarding procedures
@@ -47,6 +59,7 @@
 #include <sys/syslog.h>
 #include <net/if.h>
 #include <net/route.h>
+#include <net/kpi_protocol.h>
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
@@ -56,6 +69,10 @@
 #include <netinet/ip_mroute.h>
 #include <netinet/udp.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #ifndef NTOHL
 #if BYTE_ORDER != BIG_ENDIAN
 #define NTOHL(d) ((d) = ntohl((d)))
@@ -87,9 +104,8 @@ struct socket  *ip_mrouter  = NULL;
 u_int		rsvpdebug = 0;
 
 int
-_ip_mrouter_set(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+_ip_mrouter_set(__unused struct socket *so,
+		__unused struct sockopt *sopt)
 {
 	return(EOPNOTSUPP);
 }
@@ -98,9 +114,8 @@ int (*ip_mrouter_set)(struct socket *, struct sockopt *) = _ip_mrouter_set;
 
 
 int
-_ip_mrouter_get(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+_ip_mrouter_get(__unused struct socket *so,
+		__unused sockopt *sopt)
 {
 	return(EOPNOTSUPP);
 }
@@ -108,7 +123,7 @@ _ip_mrouter_get(so, sopt)
 int (*ip_mrouter_get)(struct socket *, struct sockopt *) = _ip_mrouter_get;
 
 int
-_ip_mrouter_done()
+_ip_mrouter_done(void)
 {
 	return(0);
 }
@@ -116,11 +131,8 @@ _ip_mrouter_done()
 int (*ip_mrouter_done)(void) = _ip_mrouter_done;
 
 int
-_ip_mforward(ip, ifp, m, imo)
-	struct ip *ip;
-	struct ifnet *ifp;
-	struct mbuf *m;
-	struct ip_moptions *imo;
+_ip_mforward(__unused struct ip *ip, __unused struct ifnet *ifp,
+	     __unused struct mbuf *m, __unused ip_moptions *imo)
 {
 	return(0);
 }
@@ -129,7 +141,7 @@ int (*ip_mforward)(struct ip *, struct ifnet *, struct mbuf *,
 		   struct ip_moptions *) = _ip_mforward;
 
 int
-_mrt_ioctl(int req, caddr_t data, struct proc *p)
+_mrt_ioctl(__unused int req, __unused caddr_t data, __unused struct proc *p)
 {
 	return EOPNOTSUPP;
 }
@@ -137,9 +149,7 @@ _mrt_ioctl(int req, caddr_t data, struct proc *p)
 int (*mrt_ioctl)(int, caddr_t, struct proc *) = _mrt_ioctl;
 
 void
-rsvp_input(m, iphlen)		/* XXX must fixup manually */
-	struct mbuf *m;
-	int iphlen;
+rsvp_input(struct mbuf *m, int iphlen)		/* XXX must fixup manually */
 {
     /* Can still get packets with rsvp_on = 0 if there is a local member
      * of the group to which the RSVP packet is addressed.  But in this
@@ -223,9 +233,9 @@ static int ip_mrtproto;
 #define NO_RTE_FOUND 	0x1
 #define RTE_FOUND	0x2
 
-static struct mfc	*mfctable[MFCTBLSIZ];
-static u_char		nexpire[MFCTBLSIZ];
-static struct vif	viftable[MAXVIFS];
+static struct mfc	*mfctable[CONFIG_MFCTBLSIZ];
+static u_char		nexpire[CONFIG_MFCTBLSIZ];
+static struct vif	viftable[CONFIG_MAXVIFS];
 static u_int	mrtdebug = 0;	  /* debug level 	*/
 #define		DEBUG_MFC	0x02
 #define		DEBUG_FORWARD	0x04
@@ -242,7 +252,7 @@ static u_int	rsvpdebug = 0;	  /* rsvp debug level   */
  * tbftable -> each vif has one of these for storing info 
  */
 
-static struct tbf tbftable[MAXVIFS];
+static struct tbf tbftable[CONFIG_MAXVIFS];
 #define		TBF_REPROCESS	(hz / 100)	/* 100x / second */
 
 /*
@@ -253,7 +263,7 @@ static struct tbf tbftable[MAXVIFS];
  * can't be sent this way.  They only exist as a placeholder for
  * multicast source verification.
  */
-static struct ifnet multicast_decap_if[MAXVIFS];
+static struct ifnet multicast_decap_if[CONFIG_MAXVIFS];
 
 #define ENCAP_TTL 64
 #define ENCAP_PROTO IPPROTO_IPIP	/* 4 */
@@ -271,6 +281,7 @@ static struct ip multicast_encap_iphdr = {
 	0,				/* frag offset */
 	ENCAP_TTL, ENCAP_PROTO,	
 	0,				/* checksum */
+	{ 0 }, { 0 }
 };
 
 /*
@@ -339,7 +350,7 @@ static int pim_assert;
  */
 
 #define MFCFIND(o, g, rt) { \
-	register struct mfc *_rt = mfctable[MFCHASH(o,g)]; \
+	struct mfc *_rt = mfctable[MFCHASH(o,g)]; \
 	rt = NULL; \
 	++mrtstat.mrts_mfc_lookups; \
 	while (_rt) { \
@@ -362,7 +373,7 @@ static int pim_assert;
  * Borrowed from Van Jacobson's scheduling code
  */
 #define TV_DELTA(a, b, delta) { \
-	    register int xxs; \
+	    int xxs; \
 		\
 	    delta = (a).tv_usec - (b).tv_usec; \
 	    if ((xxs = (a).tv_sec - (b).tv_sec)) { \
@@ -392,9 +403,7 @@ static void collate(struct timeval *);
  * Handle MRT setsockopt commands to modify the multicast routing tables.
  */
 static int
-X_ip_mrouter_set(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+X_ip_mrouter_set(struct socket *so, struct sockopt *sopt)
 {
 	int	error, optval;
 	vifi_t	vifi;
@@ -466,16 +475,14 @@ int (*ip_mrouter_set)(struct socket *, struct sockopt *) = X_ip_mrouter_set;
  * Handle MRT getsockopt commands
  */
 static int
-X_ip_mrouter_get(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+X_ip_mrouter_get(__unused struct socket *so, struct sockopt *sopt)
 {
 	int error;
-	static int version = 0x0305; /* !!! why is this here? XXX */
+	static int vers = 0x0305; /* !!! why is this here? XXX */
 
 	switch (sopt->sopt_name) {
 	case MRT_VERSION:
-		error = sooptcopyout(sopt, &version, sizeof version);
+		error = sooptcopyout(sopt, &vers, sizeof vers);
 		break;
 
 	case MRT_ASSERT:
@@ -496,9 +503,7 @@ int (*ip_mrouter_get)(struct socket *, struct sockopt *) = X_ip_mrouter_get;
  * Handle ioctl commands to obtain information from the cache
  */
 static int
-X_mrt_ioctl(cmd, data)
-    int cmd;
-    caddr_t data;
+X_mrt_ioctl(int cmd, caddr_t data)
 {
     int error = 0;
 
@@ -524,15 +529,11 @@ int (*mrt_ioctl)(int, caddr_t) = X_mrt_ioctl;
  * returns the packet, byte, rpf-failure count for the source group provided
  */
 static int
-get_sg_cnt(req)
-    register struct sioc_sg_req *req;
+get_sg_cnt(struct sioc_sg_req *req)
 {
-    register struct mfc *rt;
-    int s;
+    struct mfc *rt;
 
-    s = splnet();
     MFCFIND(req->src.s_addr, req->grp.s_addr, rt);
-    splx(s);
     if (rt != NULL) {
 	req->pktcnt = rt->mfc_pkt_cnt;
 	req->bytecnt = rt->mfc_byte_cnt;
@@ -547,10 +548,9 @@ get_sg_cnt(req)
  * returns the input and output packet and byte counts on the vif provided
  */
 static int
-get_vif_cnt(req)
-    register struct sioc_vif_req *req;
+get_vif_cnt(struct sioc_vif_req *req)
 {
-    register vifi_t vifi = req->vifi;
+    vifi_t vifi = req->vifi;
 
     if (vifi >= numvifs) return EINVAL;
 
@@ -566,9 +566,7 @@ get_vif_cnt(req)
  * Enable multicast routing
  */
 static int
-ip_mrouter_init(so, version)
-	struct socket *so;
-	int version;
+ip_mrouter_init(struct socket *so, int vers)
 {
     if (mrtdebug)
 	log(LOG_DEBUG,"ip_mrouter_init: so_type = %d, pr_protocol = %d\n",
@@ -577,7 +575,7 @@ ip_mrouter_init(so, version)
     if (so->so_type != SOCK_RAW ||
 	so->so_proto->pr_protocol != IPPROTO_IGMP) return EOPNOTSUPP;
 
-    if (version != 1)
+    if (vers != 1)
 	return ENOPROTOOPT;
 
     if (ip_mrouter != NULL) return EADDRINUSE;
@@ -601,7 +599,7 @@ ip_mrouter_init(so, version)
  * Disable multicast routing
  */
 static int
-X_ip_mrouter_done()
+X_ip_mrouter_done(void)
 {
     vifi_t vifi;
     int i;
@@ -609,9 +607,6 @@ X_ip_mrouter_done()
     struct ifreq ifr;
     struct mfc *rt;
     struct rtdetq *rte;
-    int s;
-
-    s = splnet();
 
     /*
      * For each phyint in use, disable promiscuous reception of all IP
@@ -637,7 +632,7 @@ X_ip_mrouter_done()
     /*
      * Free all multicast forwarding cache entries.
      */
-    for (i = 0; i < MFCTBLSIZ; i++) {
+    for (i = 0; i < CONFIG_MFCTBLSIZ; i++) {
 	for (rt = mfctable[i]; rt != NULL; ) {
 	    struct mfc *nr = rt->mfc_next;
 
@@ -664,8 +659,6 @@ X_ip_mrouter_done()
  
     ip_mrouter = NULL;
 
-    splx(s);
-
     if (mrtdebug)
 	log(LOG_DEBUG, "ip_mrouter_done\n");
 
@@ -680,8 +673,7 @@ int (*ip_mrouter_done)(void) = X_ip_mrouter_done;
  * Set PIM assert processing global
  */
 static int
-set_assert(i)
-	int i;
+set_assert(int i)
 {
     if ((i != 1) && (i != 0))
 	return EINVAL;
@@ -695,17 +687,17 @@ set_assert(i)
  * Add a vif to the vif table
  */
 static int
-add_vif(vifcp)
-    register struct vifctl *vifcp;
+add_vif(struct vifctl *vifcp)
 {
-    register struct vif *vifp = viftable + vifcp->vifc_vifi;
-    static struct sockaddr_in sin = {sizeof sin, AF_INET};
+    struct vif *vifp = viftable + vifcp->vifc_vifi;
+    static struct sockaddr_in sin = { sizeof sin, AF_INET, 
+										0 , {0}, {0,0,0,0,0,0,0,0,} };
     struct ifaddr *ifa;
     struct ifnet *ifp;
     int error, s;
     struct tbf *v_tbf = tbftable + vifcp->vifc_vifi;
 
-    if (vifcp->vifc_vifi >= MAXVIFS)  return EINVAL;
+    if (vifcp->vifc_vifi >= CONFIG_MAXVIFS)  return EINVAL;
     if (vifp->v_lcl_addr.s_addr != 0) return EADDRINUSE;
 
     /* Find the interface with an address in AF_INET family */
@@ -724,7 +716,7 @@ add_vif(vifcp)
 		 */
 		if (have_encap_tunnel == 0) {
 			have_encap_tunnel = 1;
-			for (s = 0; s < MAXVIFS; ++s) {
+			for (s = 0; s < CONFIG_MAXVIFS; ++s) {
 				multicast_decap_if[s].if_name = "mdecap";
 				multicast_decap_if[s].if_unit = s;
 				multicast_decap_if[s].if_family = APPLE_IF_FAM_MDECAP;
@@ -748,14 +740,11 @@ add_vif(vifcp)
 	    return EOPNOTSUPP;
 
 	/* Enable promiscuous reception of all IP multicasts from the if */
-	s = splnet();
 	error = if_allmulti(ifp, 1);
-	splx(s);
 	if (error)
 	    return error;
     }
 
-    s = splnet();
     /* define parameters for the tbf structure */
     vifp->v_tbf = v_tbf;
     GET_TIME(vifp->v_tbf->tbf_last_pkt_t);
@@ -778,7 +767,6 @@ add_vif(vifcp)
     vifp->v_pkt_out   = 0;
     vifp->v_bytes_in  = 0;
     vifp->v_bytes_out = 0;
-    splx(s);
 
     /* Adjust numvifs up if the vifi is higher than numvifs */
     if (numvifs <= vifcp->vifc_vifi) numvifs = vifcp->vifc_vifi + 1;
@@ -799,20 +787,16 @@ add_vif(vifcp)
  * Delete a vif from the vif table
  */
 static int
-del_vif(vifi)
-	vifi_t vifi;
+del_vif(vifi_t vifi)
 {
-    register struct vif *vifp = &viftable[vifi];
-    register struct mbuf *m;
+    struct vif *vifp = &viftable[vifi];
+    struct mbuf *m;
     struct ifnet *ifp;
     struct ifreq ifr;
-    int s;
 
     if (vifi >= numvifs) return EINVAL;
     if (vifp->v_lcl_addr.s_addr == 0) return EADDRNOTAVAIL;
 
-    s = splnet();
-
     if (!(vifp->v_flags & VIFF_TUNNEL)) {
 	((struct sockaddr_in *)&(ifr.ifr_addr))->sin_family = AF_INET;
 	((struct sockaddr_in *)&(ifr.ifr_addr))->sin_addr.s_addr = INADDR_ANY;
@@ -845,8 +829,6 @@ del_vif(vifi)
 	if (viftable[vifi-1].v_lcl_addr.s_addr != 0) break;
     numvifs = vifi;
 
-    splx(s);
-
     return 0;
 }
 
@@ -854,14 +836,12 @@ del_vif(vifi)
  * Add an mfc entry
  */
 static int
-add_mfc(mfccp)
-    struct mfcctl *mfccp;
+add_mfc(struct mfcctl *mfccp)
 {
     struct mfc *rt;
     u_long hash;
     struct rtdetq *rte;
-    register u_short nstl;
-    int s;
+    u_short nstl;
     int i;
 
     MFCFIND(mfccp->mfcc_origin.s_addr, mfccp->mfcc_mcastgrp.s_addr, rt);
@@ -874,18 +854,15 @@ add_mfc(mfccp)
 		(u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
 		mfccp->mfcc_parent);
 
-	s = splnet();
 	rt->mfc_parent = mfccp->mfcc_parent;
 	for (i = 0; i < numvifs; i++)
 	    rt->mfc_ttls[i] = mfccp->mfcc_ttls[i];
-	splx(s);
 	return 0;
     }
 
     /* 
      * Find the entry for which the upcall was made and update
      */
-    s = splnet();
     hash = MFCHASH(mfccp->mfcc_origin.s_addr, mfccp->mfcc_mcastgrp.s_addr);
     for (rt = mfctable[hash], nstl = 0; rt; rt = rt->mfc_next) {
 
@@ -970,7 +947,6 @@ add_mfc(mfccp)
 	    /* no upcall, so make a new entry */
 	    rt = (struct mfc *) _MALLOC(sizeof(*rt), M_MRTABLE, M_NOWAIT);
 	    if (rt == NULL) {
-		splx(s);
 		return ENOBUFS;
 	    }
 	    
@@ -993,7 +969,6 @@ add_mfc(mfccp)
 	    mfctable[hash] = rt;
 	}
     }
-    splx(s);
     return 0;
 }
 
@@ -1001,12 +976,12 @@ add_mfc(mfccp)
 /*
  * collect delay statistics on the upcalls 
  */
-static void collate(t)
-register struct timeval *t;
+static void
+collate(struct timeval *t)
 {
-    register u_long d;
-    register struct timeval tp;
-    register u_long delta;
+    u_long d;
+    struct timeval tp;
+    u_long delta;
     
     GET_TIME(tp);
     
@@ -1027,15 +1002,13 @@ register struct timeval *t;
  * Delete an mfc entry
  */
 static int
-del_mfc(mfccp)
-    struct mfcctl *mfccp;
+del_mfc(struct mfcctl *mfccp)
 {
     struct in_addr 	origin;
     struct in_addr 	mcastgrp;
     struct mfc 		*rt;
     struct mfc	 	**nptr;
     u_long 		hash;
-    int s;
 
     origin = mfccp->mfcc_origin;
     mcastgrp = mfccp->mfcc_mcastgrp;
@@ -1045,8 +1018,6 @@ del_mfc(mfccp)
 	log(LOG_DEBUG,"del_mfc orig %lx mcastgrp %lx\n",
 	    (u_long)ntohl(origin.s_addr), (u_long)ntohl(mcastgrp.s_addr));
 
-    s = splnet();
-
     nptr = &mfctable[hash];
     while ((rt = *nptr) != NULL) {
 	if (origin.s_addr == rt->mfc_origin.s_addr &&
@@ -1057,15 +1028,12 @@ del_mfc(mfccp)
 	nptr = &rt->mfc_next;
     }
     if (rt == NULL) {
-	splx(s);
 	return EADDRNOTAVAIL;
     }
 
     *nptr = rt->mfc_next;
     FREE(rt, M_MRTABLE);
 
-    splx(s);
-
     return 0;
 }
 
@@ -1073,10 +1041,7 @@ del_mfc(mfccp)
  * Send a message to mrouted on the multicast routing socket
  */
 static int
-socket_send(s, mm, src)
-	struct socket *s;
-	struct mbuf *mm;
-	struct sockaddr_in *src;
+socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
 {
 	socket_lock(s, 1);
 	if (s) {
@@ -1108,18 +1073,15 @@ socket_send(s, mm, src)
 #define TUNNEL_LEN  12  /* # bytes of IP option for tunnel encapsulation  */
 
 static int
-X_ip_mforward(ip, ifp, m, imo)
-    register struct ip *ip;
-    struct ifnet *ifp;
-    struct mbuf *m;
-    struct ip_moptions *imo;
+X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
+	      struct ip_moptions *imo)
 {
-    register struct mfc *rt;
-    register u_char *ipoptions;
-    static struct sockaddr_in 	k_igmpsrc	= { sizeof k_igmpsrc, AF_INET };
+    struct mfc *rt;
+    u_char *ipoptions;
+    static struct sockaddr_in 	k_igmpsrc = { sizeof k_igmpsrc, AF_INET, 
+    											0 , {0}, {0,0,0,0,0,0,0,0,} };
     static int srctun = 0;
-    register struct mbuf *mm;
-    int s;
+    struct mbuf *mm;
     vifi_t vifi;
     struct vif *vifp;
 
@@ -1152,7 +1114,7 @@ X_ip_mforward(ip, ifp, m, imo)
 		ip->ip_ttl++;	/* compensate for -1 in *_send routines */
 	if (rsvpdebug && ip->ip_p == IPPROTO_RSVP) {
 	    vifp = viftable + vifi;
-	    printf("Sending IPPROTO_RSVP from %lx to %lx on vif %d (%s%s%d)\n",
+	    printf("Sending IPPROTO_RSVP from %x to %x on vif %d (%s%s%d)\n",
 		ntohl(ip->ip_src.s_addr), ntohl(ip->ip_dst.s_addr), vifi,
 		(vifp->v_flags & VIFF_TUNNEL) ? "tunnel on " : "",
 		vifp->v_ifp->if_name, vifp->v_ifp->if_unit);
@@ -1160,7 +1122,7 @@ X_ip_mforward(ip, ifp, m, imo)
 	return (ip_mdq(m, ifp, NULL, vifi));
     }
     if (rsvpdebug && ip->ip_p == IPPROTO_RSVP) {
-	printf("Warning: IPPROTO_RSVP from %lx to %lx without vif option\n",
+	printf("Warning: IPPROTO_RSVP from %x to %x without vif option\n",
 	    ntohl(ip->ip_src.s_addr), ntohl(ip->ip_dst.s_addr));
 	if(!imo)
 		printf("In fact, no options were specified at all\n");
@@ -1177,12 +1139,10 @@ X_ip_mforward(ip, ifp, m, imo)
     /*
      * Determine forwarding vifs from the forwarding cache table
      */
-    s = splnet();
     MFCFIND(ip->ip_src.s_addr, ip->ip_dst.s_addr, rt);
 
     /* Entry exists, so forward if necessary */
     if (rt != NULL) {
-	splx(s);
 	return (ip_mdq(m, ifp, rt, -1));
     } else {
 	/*
@@ -1191,9 +1151,9 @@ X_ip_mforward(ip, ifp, m, imo)
 	 * send message to routing daemon
 	 */
 
-	register struct mbuf *mb0;
-	register struct rtdetq *rte;
-	register u_long hash;
+	struct mbuf *mb0;
+	struct rtdetq *rte;
+	u_long hash;
 	int hlen = ip->ip_hl << 2;
 #if UPCALL_TIMING
 	struct timeval tp;
@@ -1214,7 +1174,6 @@ X_ip_mforward(ip, ifp, m, imo)
 	 */
 	rte = (struct rtdetq *) _MALLOC((sizeof *rte), M_MRTABLE, M_NOWAIT);
 	if (rte == NULL) {
-	    splx(s);
 	    return ENOBUFS;
 	}
 	mb0 = m_copy(m, 0, M_COPYALL);
@@ -1222,7 +1181,6 @@ X_ip_mforward(ip, ifp, m, imo)
 	    mb0 = m_pullup(mb0, hlen);
 	if (mb0 == NULL) {
 	    FREE(rte, M_MRTABLE);
-	    splx(s);
 	    return ENOBUFS;
 	}
 
@@ -1244,7 +1202,6 @@ X_ip_mforward(ip, ifp, m, imo)
 	    if (rt == NULL) {
 		FREE(rte, M_MRTABLE);
 		m_freem(mb0);
-		splx(s);
 		return ENOBUFS;
 	    }
 	    /* Make a copy of the header to send to the user level process */
@@ -1253,7 +1210,6 @@ X_ip_mforward(ip, ifp, m, imo)
 		FREE(rte, M_MRTABLE);
 		m_freem(mb0);
 		FREE(rt, M_MRTABLE);
-		splx(s);
 		return ENOBUFS;
 	    }
 
@@ -1275,7 +1231,6 @@ X_ip_mforward(ip, ifp, m, imo)
 		FREE(rte, M_MRTABLE);
 		m_freem(mb0);
 		FREE(rt, M_MRTABLE);
-		splx(s);
 		return ENOBUFS;
 	    }
 
@@ -1305,7 +1260,6 @@ X_ip_mforward(ip, ifp, m, imo)
 		mrtstat.mrts_upq_ovflw++;
 		FREE(rte, M_MRTABLE);
 		m_freem(mb0);
-		splx(s);
 		return 0;
 	    }
 
@@ -1320,8 +1274,6 @@ X_ip_mforward(ip, ifp, m, imo)
 #endif
 	rte->next		= NULL;
 
-	splx(s);
-
 	return 0;
     }		
 }
@@ -1335,16 +1287,13 @@ int (*ip_mforward)(struct ip *, struct ifnet *, struct mbuf *,
  * Clean up the cache entry if upcall is not serviced
  */
 static void
-expire_upcalls(void *unused)
+expire_upcalls(__unused void *unused)
 {
     struct rtdetq *rte;
     struct mfc *mfc, **nptr;
     int i;
-    int s;
 
-    
-    s = splnet();
-    for (i = 0; i < MFCTBLSIZ; i++) {
+    for (i = 0; i < CONFIG_MFCTBLSIZ; i++) {
 	if (nexpire[i] == 0)
 	    continue;
 	nptr = &mfctable[i];
@@ -1382,7 +1331,6 @@ expire_upcalls(void *unused)
 	    }
 	}
     }
-    splx(s);
     timeout(expire_upcalls, (caddr_t)NULL, EXPIRE_TIMEOUT);
 }
 
@@ -1390,16 +1338,13 @@ expire_upcalls(void *unused)
  * Packet forwarding routine once entry in the cache is made
  */
 static int
-ip_mdq(m, ifp, rt, xmt_vif)
-    register struct mbuf *m;
-    register struct ifnet *ifp;
-    register struct mfc *rt;
-    register vifi_t xmt_vif;
+ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt,
+       vifi_t xmt_vif)
 {
-    register struct ip  *ip = mtod(m, struct ip *);
-    register vifi_t vifi;
-    register struct vif *vifp;
-    register int plen = ip->ip_len;
+    struct ip  *ip = mtod(m, struct ip *);
+    vifi_t vifi;
+    struct vif *vifp;
+    int plen = ip->ip_len;
 
 /*
  * Macro to send packet on vif.  Since RSVP packets don't get counted on
@@ -1447,7 +1392,7 @@ ip_mdq(m, ifp, rt, xmt_vif)
 	    struct igmpmsg *im;
 	    int hlen = ip->ip_hl << 2;
 	    struct timeval now;
-	    register u_long delta;
+	    u_long delta;
 
 	    GET_TIME(now);
 
@@ -1509,8 +1454,7 @@ ip_mdq(m, ifp, rt, xmt_vif)
  * numvifs there, 
  */
 static int
-X_legal_vif_num(vif)
-    int vif;
+X_legal_vif_num(int vif)
 {
     if (vif >= 0 && vif < numvifs)
        return(1);
@@ -1526,8 +1470,7 @@ int (*legal_vif_num)(int) = X_legal_vif_num;
  * Return the local address used by this vif
  */
 static u_long
-X_ip_mcast_src(vifi)
-    int vifi;
+X_ip_mcast_src(int vifi)
 {
     if (vifi >= 0 && vifi < numvifs)
 	return viftable[vifi].v_lcl_addr.s_addr;
@@ -1540,13 +1483,10 @@ u_long (*ip_mcast_src)(int) = X_ip_mcast_src;
 #endif
 
 static void
-phyint_send(ip, vifp, m)
-    struct ip *ip;
-    struct vif *vifp;
-    struct mbuf *m;
+phyint_send(struct ip *ip, struct vif *vifp, struct mbuf *m)
 {
-    register struct mbuf *mb_copy;
-    register int hlen = ip->ip_hl << 2;
+    struct mbuf *mb_copy;
+    int hlen = ip->ip_hl << 2;
 
     /*
      * Make a new reference to the packet; make sure that
@@ -1566,14 +1506,11 @@ phyint_send(ip, vifp, m)
 }
 
 static void
-encap_send(ip, vifp, m)
-    register struct ip *ip;
-    register struct vif *vifp;
-    register struct mbuf *m;
+encap_send(struct ip *ip, struct vif *vifp, struct mbuf *m)
 {
-    register struct mbuf *mb_copy;
-    register struct ip *ip_copy;
-    register int i, len = ip->ip_len;
+    struct mbuf *mb_copy;
+    struct ip *ip_copy;
+    int i, len = ip->ip_len;
 
     /*
      * copy the old packet & pullup its IP header into the
@@ -1583,6 +1520,9 @@ encap_send(ip, vifp, m)
     MGETHDR(mb_copy, M_DONTWAIT, MT_HEADER);
     if (mb_copy == NULL)
 	return;
+#if CONFIG_MACF_NET
+    mac_mbuf_label_associate_multicast_encap(m, vifp->v_ifp, mb_copy);
+#endif
     mb_copy->m_data += max_linkhdr;
     mb_copy->m_len = sizeof(multicast_encap_iphdr);
 
@@ -1637,17 +1577,15 @@ encap_send(ip, vifp, m)
  */
 void
 #if MROUTE_LKM
-X_ipip_input(m, iphlen)
+X_ipip_input(struct mbuf *m, int iphlen)
 #else
-ipip_input(m, iphlen)
+ipip_input(struct mbuf *m, int iphlen)
 #endif
-	register struct mbuf *m;
-	int iphlen;
 {
     struct ifnet *ifp = m->m_pkthdr.rcvif;
-    register struct ip *ip = mtod(m, struct ip *);
-    register int hlen = ip->ip_hl << 2;
-    register struct vif *vifp;
+    struct ip *ip = mtod(m, struct ip *);
+    int hlen = ip->ip_hl << 2;
+    struct vif *vifp;
 
     if (!have_encap_tunnel) {
 	    rip_input(m, iphlen);
@@ -1666,7 +1604,7 @@ ipip_input(m, iphlen)
 	return;
     }
     if (ip->ip_src.s_addr != last_encap_src) {
-	register struct vif *vife;
+	struct vif *vife;
 	
 	vifp = viftable;
 	vife = vifp + numvifs;
@@ -1706,13 +1644,10 @@ ipip_input(m, iphlen)
  */
 
 static void
-tbf_control(vifp, m, ip, p_len)
-	register struct vif *vifp;
-	register struct mbuf *m;
-	register struct ip *ip;
-	register u_long p_len;
+tbf_control(struct vif *vifp, struct mbuf *m, struct ip *ip,
+	    u_long p_len)
 {
-    register struct tbf *t = vifp->v_tbf;
+    struct tbf *t = vifp->v_tbf;
 
     if (p_len > MAX_BKT_SIZE) {
 	/* drop if packet is too large */
@@ -1760,12 +1695,9 @@ tbf_control(vifp, m, ip, p_len)
  * adds a packet to the queue at the interface
  */
 static void
-tbf_queue(vifp, m) 
-	register struct vif *vifp;
-	register struct mbuf *m;
+tbf_queue(struct vif *vifp, struct mbuf *m) 
 {
-    register int s = splnet();
-    register struct tbf *t = vifp->v_tbf;
+    struct tbf *t = vifp->v_tbf;
 
     if (t->tbf_t == NULL) {
 	/* Queue was empty */
@@ -1786,8 +1718,6 @@ tbf_queue(vifp, m)
     m->m_act = NULL;
 
     t->tbf_q_len++;
-
-    splx(s);
 }
 
 
@@ -1795,13 +1725,11 @@ tbf_queue(vifp, m)
  * processes the queue at the interface
  */
 static void
-tbf_process_q(vifp)
-    register struct vif *vifp;
+tbf_process_q(struct vif *vifp)
 {
-    register struct mbuf *m;
-    register int len;
-    register int s = splnet();
-    register struct tbf *t = vifp->v_tbf;
+    struct mbuf *m;
+    int len;
+    struct tbf *t = vifp->v_tbf;
 
     /* loop through the queue at the interface and send as many packets
      * as possible
@@ -1828,14 +1756,12 @@ tbf_process_q(vifp)
 
 	} else break;
     }
-    splx(s);
 }
 
 static void
-tbf_reprocess_q(xvifp)
-	void *xvifp;
+tbf_reprocess_q(void *xvifp)
 {
-    register struct vif *vifp = xvifp;
+    struct vif *vifp = xvifp;
 
     if (ip_mrouter == NULL)  {
 	return;
@@ -1853,15 +1779,12 @@ tbf_reprocess_q(xvifp)
  * based on the precedence value and the priority
  */
 static int
-tbf_dq_sel(vifp, ip)
-    register struct vif *vifp;
-    register struct ip *ip;
+tbf_dq_sel(struct vif *vifp, struct ip *ip)
 {
-    register int s = splnet();
-    register u_int p;
-    register struct mbuf *m, *last;
-    register struct mbuf **np;
-    register struct tbf *t = vifp->v_tbf;
+    u_int p;
+    struct mbuf *m, *last;
+    struct mbuf **np;
+    struct tbf *t = vifp->v_tbf;
 
     p = priority(vifp, ip);
 
@@ -1878,31 +1801,26 @@ tbf_dq_sel(vifp, ip)
 	     * we check anyway. */
 	    if (--t->tbf_q_len == 0)
 		t->tbf_t = NULL;
-	    splx(s);
 	    mrtstat.mrts_drop_sel++;
 	    return(1);
 	}
 	np = &m->m_act;
 	last = m;
     }
-    splx(s);
     return(0);
 }
 
 static void
-tbf_send_packet(vifp, m)
-    register struct vif *vifp;
-    register struct mbuf *m;
+tbf_send_packet(struct vif *vifp, struct mbuf *m)
 {
     struct ip_moptions imo;
     int error;
     static struct route ro;
-    int s = splnet();
 
     if (vifp->v_flags & VIFF_TUNNEL) {
 	/* If tunnel options */
 	ip_output(m, (struct mbuf *)0, &vifp->v_route,
-		  IP_FORWARDING, (struct ip_moptions *)0);
+		  IP_FORWARDING, (struct ip_moptions *)0, NULL);
     } else {
 	imo.imo_multicast_ifp  = vifp->v_ifp;
 	imo.imo_multicast_ttl  = mtod(m, struct ip *)->ip_ttl - 1;
@@ -1916,13 +1834,12 @@ tbf_send_packet(vifp, m)
 	 * the loopback interface, thus preventing looping.
 	 */
 	error = ip_output(m, (struct mbuf *)0, &ro,
-			  IP_FORWARDING, &imo);
+			  IP_FORWARDING, &imo, NULL);
 
 	if (mrtdebug & DEBUG_XMIT)
 	    log(LOG_DEBUG, "phyint_send on vif %d err %d\n", 
 		vifp - viftable, error);
     }
-    splx(s);
 }
 
 /* determine the current time and then
@@ -1930,13 +1847,11 @@ tbf_send_packet(vifp, m)
  * in milliseconds & update the no. of tokens in the bucket
  */
 static void
-tbf_update_tokens(vifp)
-    register struct vif *vifp;
+tbf_update_tokens(struct vif *vifp)
 {
     struct timeval tp;
-    register u_long tm;
-    register int s = splnet();
-    register struct tbf *t = vifp->v_tbf;
+    u_long tm;
+    struct tbf *t = vifp->v_tbf;
 
     GET_TIME(tp);
 
@@ -1956,16 +1871,12 @@ tbf_update_tokens(vifp)
 
     if (t->tbf_n_tok > MAX_BKT_SIZE)
 	t->tbf_n_tok = MAX_BKT_SIZE;
-
-    splx(s);
 }
 
 static int
-priority(vifp, ip)
-    register struct vif *vifp;
-    register struct ip *ip;
+priority(__unused struct vif *vifp, struct ip *ip)
 {
-    register int prio;
+    int prio;
 
     /* temporary hack; may add general packet classifier some day */
 
@@ -2005,11 +1916,9 @@ priority(vifp, ip)
  */
 
 int
-ip_rsvp_vif_init(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+ip_rsvp_vif_init(struct socket *so, struct sockopt *sopt)
 {
-    int error, i, s;
+    int error, i;
 
     if (rsvpdebug)
 	printf("ip_rsvp_vif_init: so_type = %d, pr_protocol = %d\n",
@@ -2026,17 +1935,13 @@ ip_rsvp_vif_init(so, sopt)
     if (rsvpdebug)
 	printf("ip_rsvp_vif_init: vif = %d rsvp_on = %d\n", i, rsvp_on);
  
-    s = splnet();
-
     /* Check vif. */
     if (!legal_vif_num(i)) {
-	splx(s);
 	return EADDRNOTAVAIL;
     }
 
     /* Check if socket is available. */
     if (viftable[i].v_rsvpd != NULL) {
-	splx(s);
 	return EADDRINUSE;
     }
 
@@ -2049,16 +1954,13 @@ ip_rsvp_vif_init(so, sopt)
 	rsvp_on++;
     }
 
-    splx(s);
     return 0;
 }
 
 int
-ip_rsvp_vif_done(so, sopt)
-	struct socket *so;
-	struct sockopt *sopt;
+ip_rsvp_vif_done(struct socket *so, struct sockopt *sopt)
 {
-	int error, i, s;
+	int error, i;
  
 	if (rsvpdebug)
 		printf("ip_rsvp_vif_done: so_type = %d, pr_protocol = %d\n",
@@ -2071,12 +1973,9 @@ ip_rsvp_vif_done(so, sopt)
 	error = sooptcopyin(sopt, &i, sizeof i, sizeof i);
 	if (error)
 		return (error);
- 
-	s = splnet();
- 
+  
 	/* Check vif. */
 	if (!legal_vif_num(i)) {
-		splx(s);
 		return EADDRNOTAVAIL;
 	}
 
@@ -2094,23 +1993,18 @@ ip_rsvp_vif_done(so, sopt)
 		rsvp_on--;
 	}
 
-	splx(s);
 	return 0;
 }
 
 void
-ip_rsvp_force_done(so)
-    struct socket *so;
+ip_rsvp_force_done(struct socket *so)
 {
     int vifi;
-    register int s;
 
     /* Don't bother if it is not the right type of socket. */
     if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_RSVP)
 	return;
 
-    s = splnet();
-
     /* The socket may be attached to more than one vif...this
      * is perfectly legal.
      */
@@ -2127,19 +2021,16 @@ ip_rsvp_force_done(so)
 	}
     }
 
-    splx(s);
     return;
 }
 
 void
-rsvp_input(m, iphlen)
-	struct mbuf *m;
-	int iphlen;
+rsvp_input(struct mbuf *m, int iphlen)
 {
     int vifi;
-    register struct ip *ip = mtod(m, struct ip *);
-    static struct sockaddr_in rsvp_src = { sizeof rsvp_src, AF_INET };
-    register int s;
+    struct ip *ip = mtod(m, struct ip *);
+    static struct sockaddr_in rsvp_src = { sizeof rsvp_src, AF_INET, 
+    										0 , {0}, {0,0,0,0,0,0,0,0,} };
     struct ifnet *ifp;
 
     if (rsvpdebug)
@@ -2154,8 +2045,6 @@ rsvp_input(m, iphlen)
 	return;
     }
 
-    s = splnet();
-
     if (rsvpdebug)
 	printf("rsvp_input: check vifs\n");
 
@@ -2187,13 +2076,12 @@ rsvp_input(m, iphlen)
 		printf("rsvp_input: No socket defined for vif %d\n",vifi);
 	    m_freem(m);
 	}
-	splx(s);
 	return;
     }
     rsvp_src.sin_addr = ip->ip_src;
 
     if (rsvpdebug && m)
-	printf("rsvp_input: m->m_len = %d, sbspace() = %ld\n",
+	printf("rsvp_input: m->m_len = %ld, sbspace() = %ld\n",
 	       m->m_len,sbspace(&(viftable[vifi].v_rsvpd->so_rcv)));
 
     if (socket_send(viftable[vifi].v_rsvpd, m, &rsvp_src) < 0) {
@@ -2204,7 +2092,6 @@ rsvp_input(m, iphlen)
 	    printf("rsvp_input: send packet up\n");
     }
 
-    splx(s);
 }
 
 #if MROUTE_LKM
diff --git a/bsd/netinet/ip_mroute.h b/bsd/netinet/ip_mroute.h
index 3fef18725..d88c9aca0 100644
--- a/bsd/netinet/ip_mroute.h
+++ b/bsd/netinet/ip_mroute.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1989 Stephen Deering.
@@ -91,10 +97,17 @@
 #define GET_TIME(t)	microtime(&t)
 #endif KERNEL_PRIVATE
 
+#ifndef CONFIG_MAXVIFS
+#define CONFIG_MAXVIFS 32  /* 4635538 temp workaround */
+#endif
+
+#ifndef CONFIG_MFCTBLSIZ
+#define CONFIG_MFCTBLSIZ 256  /* 4635538 temp workaround */
+#endif
+
 /*
  * Types and macros for handling bitmaps with one bit per virtual interface.
  */
-#define	MAXVIFS 32
 typedef u_long vifbitmap_t;
 typedef u_short vifi_t;		/* type of a vif index */
 #define ALL_VIFS (vifi_t)-1
@@ -112,12 +125,12 @@ typedef u_short vifi_t;		/* type of a vif index */
  * (MRT_DEL_VIF takes a single vifi_t argument.)
  */
 struct vifctl {
-	vifi_t	vifc_vifi;	    	/* the index of the vif to be added */
-	u_char	vifc_flags;     	/* VIFF_ flags defined below */
-	u_char	vifc_threshold; 	/* min ttl required to forward on vif */
-	u_int	vifc_rate_limit;	/* max rate */
-	struct	in_addr vifc_lcl_addr;	/* local interface address */
-	struct	in_addr vifc_rmt_addr;	/* remote address (tunnels only) */
+    vifi_t	vifc_vifi;	    	/* the index of the vif to be added */
+    u_char	vifc_flags;     	/* VIFF_ flags defined below */
+    u_char	vifc_threshold; 	/* min ttl required to forward on vif */
+    u_int	vifc_rate_limit;	/* max rate */
+    struct	in_addr vifc_lcl_addr;	/* local interface address */
+    struct	in_addr vifc_rmt_addr;	/* remote address (tunnels only) */
 };
 
 #define	VIFF_TUNNEL	0x1		/* vif represents a tunnel end-point */
@@ -128,51 +141,51 @@ struct vifctl {
  * (mfcc_tos to be added at a future point)
  */
 struct mfcctl {
-    struct in_addr  mfcc_origin;		/* ip origin of mcasts       */
-    struct in_addr  mfcc_mcastgrp; 		/* multicast group associated*/
-    vifi_t	    mfcc_parent;   		/* incoming vif              */
-    u_char	    mfcc_ttls[MAXVIFS]; 	/* forwarding ttls on vifs   */
+    struct in_addr	mfcc_origin;		/* ip origin of mcasts       */
+    struct in_addr	mfcc_mcastgrp; 		/* multicast group associated*/
+    vifi_t		mfcc_parent;		/* incoming vif              */
+    u_char		mfcc_ttls[CONFIG_MAXVIFS];	/* forwarding ttls on vifs   */
 };
 
 /*
  * The kernel's multicast routing statistics.
  */
 struct mrtstat {
-    u_long	mrts_mfc_lookups;	/* # forw. cache hash table hits   */
-    u_long	mrts_mfc_misses;	/* # forw. cache hash table misses */
-    u_long	mrts_upcalls;		/* # calls to mrouted              */
-    u_long	mrts_no_route;		/* no route for packet's origin    */
-    u_long	mrts_bad_tunnel;	/* malformed tunnel options        */
-    u_long	mrts_cant_tunnel;	/* no room for tunnel options      */
-    u_long	mrts_wrong_if;		/* arrived on wrong interface	   */
-    u_long	mrts_upq_ovflw;		/* upcall Q overflow		   */
-    u_long	mrts_cache_cleanups;	/* # entries with no upcalls 	   */
-    u_long  	mrts_drop_sel;     	/* pkts dropped selectively        */
-    u_long  	mrts_q_overflow;    	/* pkts dropped - Q overflow       */
-    u_long  	mrts_pkt2large;     	/* pkts dropped - size > BKT SIZE  */
-    u_long	mrts_upq_sockfull;	/* upcalls dropped - socket full */
+    u_int32_t	mrts_mfc_lookups;	/* # forw. cache hash table hits   */
+    u_int32_t	mrts_mfc_misses;	/* # forw. cache hash table misses */
+    u_int32_t	mrts_upcalls;		/* # calls to mrouted              */
+    u_int32_t	mrts_no_route;		/* no route for packet's origin    */
+    u_int32_t	mrts_bad_tunnel;	/* malformed tunnel options        */
+    u_int32_t	mrts_cant_tunnel;	/* no room for tunnel options      */
+    u_int32_t	mrts_wrong_if;		/* arrived on wrong interface      */
+    u_int32_t	mrts_upq_ovflw;		/* upcall Q overflow               */
+    u_int32_t	mrts_cache_cleanups;	/* # entries with no upcalls       */
+    u_int32_t	mrts_drop_sel;		/* pkts dropped selectively        */
+    u_int32_t	mrts_q_overflow;	/* pkts dropped - Q overflow       */
+    u_int32_t	mrts_pkt2large;		/* pkts dropped - size > BKT SIZE  */
+    u_int32_t	mrts_upq_sockfull;	/* upcalls dropped - socket full   */
 };
 
 /*
  * Argument structure used by mrouted to get src-grp pkt counts
  */
 struct sioc_sg_req {
-    struct in_addr src;
-    struct in_addr grp;
-    u_long pktcnt;
-    u_long bytecnt;
-    u_long wrong_if;
+    struct in_addr	src;
+    struct in_addr	grp;
+    u_int32_t		pktcnt;
+    u_int32_t		bytecnt;
+    u_int32_t		wrong_if;
 };
 
 /*
  * Argument structure used by mrouted to get vif pkt counts
  */
 struct sioc_vif_req {
-    vifi_t vifi;		/* vif number				*/
-    u_long icount;		/* Input packet count on vif		*/
-    u_long ocount;		/* Output packet count on vif		*/
-    u_long ibytes;		/* Input byte count on vif		*/
-    u_long obytes;		/* Output byte count on vif		*/
+    vifi_t		vifi;		/* vif number			*/
+    u_int32_t		icount;		/* Input packet count on vif	*/
+    u_int32_t		ocount;		/* Output packet count on vif	*/
+    u_int32_t		ibytes;		/* Input byte count on vif	*/
+    u_int32_t		obytes;		/* Output byte count on vif	*/
 };
 
 #ifdef PRIVATE
@@ -209,7 +222,7 @@ struct mfc {
     struct in_addr  mfc_origin;	 		/* IP origin of mcasts   */
     struct in_addr  mfc_mcastgrp;  		/* multicast group associated*/
     vifi_t	    mfc_parent; 		/* incoming vif              */
-    u_char	    mfc_ttls[MAXVIFS]; 		/* forwarding ttls on vifs   */
+    u_char	    mfc_ttls[CONFIG_MAXVIFS]; 		/* forwarding ttls on vifs   */
     u_long	    mfc_pkt_cnt;		/* pkt count for src-grp     */
     u_long	    mfc_byte_cnt;		/* byte count for src-grp    */
     u_long	    mfc_wrong_if;		/* wrong if for src-grp	     */
@@ -224,8 +237,8 @@ struct mfc {
  * note the convenient similarity to an IP packet
  */
 struct igmpmsg {
-    u_long	    unused1;
-    u_long	    unused2;
+    u_int32_t	    unused1;
+    u_int32_t	    unused2;
     u_char	    im_msgtype;			/* what type of message	    */
 #define IGMPMSG_NOCACHE		1
 #define IGMPMSG_WRONGVIF	2
@@ -234,7 +247,8 @@ struct igmpmsg {
     u_char	    unused3;
     struct in_addr  im_src, im_dst;
 };
-#define MFCTBLSIZ       256
+
+#define MFCTBLSIZ       CONFIG_MFCTBLSIZ
 
 #ifdef KERNEL_PRIVATE
 /*
@@ -250,10 +264,10 @@ struct rtdetq {
     struct rtdetq	*next;		/* Next in list of packets          */
 };
 
-#if (MFCTBLSIZ & (MFCTBLSIZ - 1)) == 0	  /* from sys:route.h */
-#define MFCHASHMOD(h)	((h) & (MFCTBLSIZ - 1))
+#if (CONFIG_MFCTBLSIZ & (CONFIG_MFCTBLSIZ - 1)) == 0	  /* from sys:route.h */
+#define MFCHASHMOD(h)	((h) & (CONFIG_MFCTBLSIZ - 1))
 #else
-#define MFCHASHMOD(h)	((h) % MFCTBLSIZ)
+#define MFCHASHMOD(h)	((h) % CONFIG_MFCTBLSIZ)
 #endif
 
 #define MAX_UPQ	4		/* max. no of pkts in upcall Q */
diff --git a/bsd/netinet/ip_output.c b/bsd/netinet/ip_output.c
index 5f1e3c980..c065797e7 100644
--- a/bsd/netinet/ip_output.c
+++ b/bsd/netinet/ip_output.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993
@@ -54,6 +60,12 @@
  *	@(#)ip_output.c	8.3 (Berkeley) 1/21/94
  * $FreeBSD: src/sys/netinet/ip_output.c,v 1.99.2.16 2001/07/19 06:37:26 kris Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #define _IP_VHL
 
@@ -80,10 +92,15 @@
 
 #include <netinet/kpi_ipfilter_var.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif
+
 #include "faith.h"
 
 #include <net/dlil.h>
 #include <sys/kdebug.h>
+#include <libkern/OSAtomic.h>
 
 #define DBG_LAYER_BEG		NETDBG_CODE(DBG_NETIP, 1)
 #define DBG_LAYER_END		NETDBG_CODE(DBG_NETIP, 3)
@@ -116,9 +133,6 @@
 						  (ntohl(a.s_addr))&0xFF);
 #endif
 
-#if IPSEC
-extern lck_mtx_t    *sadb_mutex;
-#endif
 
 u_short ip_id;
 
@@ -130,16 +144,17 @@ static int	ip_getmoptions(struct sockopt *, struct ip_moptions *);
 static int	ip_pcbopts(int, struct mbuf **, struct mbuf *);
 static int	ip_setmoptions(struct sockopt *, struct ip_moptions **);
 
+static void ip_out_cksum_stats(int, u_int32_t);
+
 int ip_createmoptions(struct ip_moptions **imop);
 int ip_addmembership(struct ip_moptions *imo, struct ip_mreq *mreq);
 int ip_dropmembership(struct ip_moptions *imo, struct ip_mreq *mreq);
 int	ip_optcopy(struct ip *, struct ip *);
+void in_delayed_cksum_offset(struct mbuf *, int );
+void in_cksum_offset(struct mbuf* , size_t );
+
 extern int (*fr_checkp)(struct ip *, int, struct ifnet *, int, struct mbuf **);
-#ifdef __APPLE__
-extern struct mbuf*	m_dup(register struct mbuf *m, int how);
-#endif
 
-extern int apple_hwcksum_tx;
 extern u_long  route_generation;
 
 extern	struct protosw inetsw[];
@@ -155,6 +170,11 @@ extern int ipsec_bypass;
 static int	ip_maxchainsent = 0;
 SYSCTL_INT(_net_inet_ip, OID_AUTO, maxchainsent, CTLFLAG_RW,
     &ip_maxchainsent, 0, "use dlil_output_list");
+#if DEBUG
+static int forge_ce = 0;
+SYSCTL_INT(_net_inet_ip, OID_AUTO, forge_ce, CTLFLAG_RW,
+    &forge_ce, 0, "Forge ECN CE");
+#endif /* DEBUG */
 /*
  * IP output.  The packet in mbuf chain m contains a skeletal IP
  * header (with len, off, ttl, proto, tos, src, dst).
@@ -167,13 +187,36 @@ ip_output(
 	struct mbuf *opt,
 	struct route *ro,
 	int flags,
-	struct ip_moptions *imo)
+	struct ip_moptions *imo,
+	struct ifnet *ifp)
 {
 	int error;
-	error = ip_output_list(m0, 0, opt, ro, flags, imo);
+	error = ip_output_list(m0, 0, opt, ro, flags, imo, ifp);
 	return error;
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOMEM
+ *		EADDRNOTAVAIL
+ *		ENETUNREACH
+ *		EHOSTUNREACH
+ *		EACCES
+ *		EMSGSIZE
+ *		ENOBUFS
+ *	ipsec4_getpolicybyaddr:???	[IPSEC 4th argument, contents modified]
+ *	ipsec4_getpolicybysock:???	[IPSEC 4th argument, contents modified]
+ *	key_spdacquire:???		[IPSEC]
+ *	ipsec4_output:???		[IPSEC]
+ *	<fr_checkp>:???			[firewall]
+ *	ip_dn_io_ptr:???		[dummynet]
+ *	dlil_output:???			[DLIL]
+ *	dlil_output_list:???		[DLIL]
+ *
+ * Notes:	The ipsec4_getpolicyby{addr|sock} function error returns are
+ *		only used as the error return from this function where one of
+ *		these functions fails to return a policy.
+ */
 int
 ip_output_list(
 	struct mbuf *m0,
@@ -181,13 +224,19 @@ ip_output_list(
 	struct mbuf *opt,
 	struct route *ro,
 	int flags,
-	struct ip_moptions *imo)
+	struct ip_moptions *imo,
+#if CONFIG_FORCE_OUT_IFP
+	struct ifnet *pdp_ifp
+#else
+	__unused struct ifnet *unused_ifp
+#endif
+	)
 {
 	struct ip *ip, *mhip;
 	struct ifnet *ifp = NULL;
 	struct mbuf *m = m0;
 	int hlen = sizeof (struct ip);
-	int len, off, error = 0;
+	int len = 0, off, error = 0;
 	struct sockaddr_in *dst = NULL;
 	struct in_ifaddr *ia = NULL;
 	int isbroadcast, sw_csum;
@@ -204,18 +253,18 @@ ip_output_list(
 	int didfilter = 0;
 	ipfilter_t inject_filter_ref = 0;
 	struct m_tag	*tag;
-	struct route	dn_route;
+	struct route	saved_route;
 	struct mbuf * packetlist;
 	int pktcnt = 0;
 	
-	lck_mtx_lock(ip_mutex); 
 
 	KERNEL_DEBUG(DBG_FNC_IP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
 
 	packetlist = m0;
+	args.next_hop = NULL;
+#if IPFIREWALL
 	args.eh = NULL;
 	args.rule = NULL;
-	args.next_hop = NULL;
 	args.divert_rule = 0;			/* divert cookie */
 	
 	/* Grab info from mtags prepended to the chain */
@@ -226,8 +275,8 @@ ip_output_list(
 		dn_tag = (struct dn_pkt_tag *)(tag+1);
 		args.rule = dn_tag->rule;
 		opt = NULL;
-		dn_route = dn_tag->ro;
-		ro = &dn_route;
+		saved_route = dn_tag->ro;
+		ro = &saved_route;
 			
 		imo = NULL;
 		dst = dn_tag->dn_dst;
@@ -238,6 +287,7 @@ ip_output_list(
 	}
 #endif /* DUMMYNET */
 
+#if IPDIVERT
 	if ((tag = m_tag_locate(m0, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_DIVERT, NULL)) != NULL) {
 		struct divert_tag	*div_tag;
 		
@@ -246,6 +296,9 @@ ip_output_list(
 
 		m_tag_delete(m0, tag);
 	}
+#endif /* IPDIVERT */
+#endif /* IPFIREWALL */
+
 	if ((tag = m_tag_locate(m0, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFORWARD, NULL)) != NULL) {
 		struct ip_fwd_tag	*ipfwd_tag;
 		
@@ -265,21 +318,25 @@ ip_output_list(
 		      mtod(m, struct ip *)->ip_p);
 #endif
 
+#if IPFIREWALL
 	if (args.rule != NULL) {	/* dummynet already saw us */
             ip = mtod(m, struct ip *);
             hlen = IP_VHL_HL(ip->ip_vhl) << 2 ;
-            if (ro->ro_rt != NULL)
+	    lck_mtx_lock(rt_mtx);
+            if (ro->ro_rt != NULL) 
                 ia = (struct in_ifaddr *)ro->ro_rt->rt_ifa;
-			if (ia)
-				ifaref(&ia->ia_ifa);
+   	    if (ia)
+		ifaref(&ia->ia_ifa);
+	    lck_mtx_unlock(rt_mtx);
 #if IPSEC
-			if (ipsec_bypass == 0 && (flags & IP_NOIPSEC) == 0) {
-				so = ipsec_getsocket(m);
-				(void)ipsec_setsocket(m, NULL);
-			}
+	    if (ipsec_bypass == 0 && (flags & IP_NOIPSEC) == 0) {
+		so = ipsec_getsocket(m);
+		(void)ipsec_setsocket(m, NULL);
+		}
 #endif
             goto sendit;
 	}
+#endif /* IPFIREWALL */
 
 #if IPSEC
 	if (ipsec_bypass == 0 && (flags & IP_NOIPSEC) == 0) {
@@ -312,14 +369,24 @@ loopit:
 #else
 		ip->ip_id = htons(ip_id++);
 #endif
-		ipstat.ips_localout++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_localout);
 	} else {
 		hlen = IP_VHL_HL(ip->ip_vhl) << 2;
 	}
+	
+#if DEBUG
+	/* For debugging, we let the stack forge congestion */
+	if (forge_ce != 0 &&
+		((ip->ip_tos & IPTOS_ECN_MASK) == IPTOS_ECN_ECT1 ||
+		 (ip->ip_tos & IPTOS_ECN_MASK) == IPTOS_ECN_ECT0)) {
+		ip->ip_tos = (ip->ip_tos & ~IPTOS_ECN_MASK) | IPTOS_ECN_CE;
+		forge_ce--;
+	}
+#endif /* DEBUG */
 
 	KERNEL_DEBUG(DBG_LAYER_BEG, ip->ip_dst.s_addr, 
 		     ip->ip_src.s_addr, ip->ip_p, ip->ip_off, ip->ip_len);
-
+	
 	dst = (struct sockaddr_in *)&ro->ro_dst;
 
 	/*
@@ -330,21 +397,26 @@ loopit:
 	 * cache with IPv6.
 	 */
 
-	{
-		if (ro->ro_rt && (ro->ro_rt->generation_id != route_generation) &&
-			((flags & (IP_ROUTETOIF | IP_FORWARDING)) == 0) && (ip->ip_src.s_addr != INADDR_ANY) &&
-			(ifa_foraddr(ip->ip_src.s_addr) == 0)) {
+	lck_mtx_lock(rt_mtx);
+	if (ro->ro_rt != NULL) {
+		if (ro->ro_rt->generation_id != route_generation &&
+		    ((flags & (IP_ROUTETOIF | IP_FORWARDING)) == 0) &&
+		    (ip->ip_src.s_addr != INADDR_ANY) &&
+		    (ifa_foraddr(ip->ip_src.s_addr) == 0)) {
 			error = EADDRNOTAVAIL;
+			lck_mtx_unlock(rt_mtx);
 			goto bad;
 		}
+		if ((ro->ro_rt->rt_flags & RTF_UP) == 0 ||
+		    dst->sin_family != AF_INET ||
+		    dst->sin_addr.s_addr != pkt_dst.s_addr) {
+			rtfree_locked(ro->ro_rt);
+			ro->ro_rt = NULL;
+		}
+		if (ro->ro_rt && ro->ro_rt->generation_id != route_generation)
+			ro->ro_rt->generation_id = route_generation;
 	}
-	if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 ||
-	   dst->sin_family != AF_INET ||
-	   dst->sin_addr.s_addr != pkt_dst.s_addr)) {
-		rtfree(ro->ro_rt);
-		ro->ro_rt = (struct rtentry *)0;
-	}
-	if (ro->ro_rt == 0) {
+	if (ro->ro_rt == NULL) {
 		bzero(dst, sizeof(*dst));
 		dst->sin_family = AF_INET;
 		dst->sin_len = sizeof(*dst);
@@ -361,8 +433,9 @@ loopit:
 			ifafree(&ia->ia_ifa);
 		if ((ia = ifatoia(ifa_ifwithdstaddr(sintosa(dst)))) == 0) {
 			if ((ia = ifatoia(ifa_ifwithnet(sintosa(dst)))) == 0) {
-				ipstat.ips_noroute++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
 				error = ENETUNREACH;
+				lck_mtx_unlock(rt_mtx);
 				goto bad;
 			}
 		}
@@ -370,6 +443,21 @@ loopit:
 		ip->ip_ttl = 1;
 		isbroadcast = in_broadcast(dst->sin_addr, ifp);
 	} else {
+
+#if CONFIG_FORCE_OUT_IFP
+		/* Check if this packet should be forced out a specific interface */
+		if (ro->ro_rt == 0 && pdp_ifp != NULL) {
+			pdp_context_route_locked(pdp_ifp, ro);
+			
+			if (ro->ro_rt == NULL) {
+				OSAddAtomic(1, (UInt32*)&ipstat.ips_noroute);
+				error = EHOSTUNREACH;
+				lck_mtx_unlock(rt_mtx);
+				goto bad;
+			}
+		}
+#endif
+		
 		/*
 		 * If this is the case, we probably don't want to allocate
 		 * a protocol-cloned route since we didn't get one from the
@@ -379,13 +467,37 @@ loopit:
 		 * the link layer, as this is probably required in all cases
 		 * for correct operation (as it is for ARP).
 		 */
-		if (ro->ro_rt == 0)
-			rtalloc_ign(ro, RTF_PRCLONING);
+		
+		if (ro->ro_rt == 0) {
+			unsigned long ign = RTF_PRCLONING;
+			/*
+			 * We make an exception here: if the destination
+			 * address is INADDR_BROADCAST, allocate a protocol-
+			 * cloned host route so that we end up with a route
+			 * marked with the RTF_BROADCAST flag.  Otherwise,
+			 * we would end up referring to the default route,
+			 * instead of creating a cloned host route entry.
+			 * That would introduce inconsistencies between ULPs
+			 * that allocate a route and those that don't.  The
+			 * RTF_BROADCAST route is important since we'd want
+			 * to send out undirected IP broadcast packets using
+			 * link-level broadcast address.
+			 *
+			 * This exception will no longer be necessary when
+			 * the RTF_PRCLONING scheme is no longer present.
+			 */
+			if (dst->sin_addr.s_addr == INADDR_BROADCAST)
+				ign &= ~RTF_PRCLONING;
+
+			rtalloc_ign_locked(ro, ign);
+		}
 		if (ro->ro_rt == 0) {
-			ipstat.ips_noroute++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
 			error = EHOSTUNREACH;
+			lck_mtx_unlock(rt_mtx);
 			goto bad;
 		}
+		
 		if (ia)
 			ifafree(&ia->ia_ifa);
 		ia = ifatoia(ro->ro_rt->rt_ifa);
@@ -400,6 +512,7 @@ loopit:
 		else
 			isbroadcast = in_broadcast(dst->sin_addr, ifp);
 	}
+	lck_mtx_unlock(rt_mtx);
 	if (IN_MULTICAST(ntohl(pkt_dst.s_addr))) {
 		struct in_multi *inm;
 
@@ -418,10 +531,12 @@ loopit:
 			if (imo->imo_multicast_ifp != NULL) {
 				ifp = imo->imo_multicast_ifp;
 			}
+#if MROUTING
 			if (imo->imo_multicast_vif != -1 && 
 				((flags & IP_RAWOUTPUT) == 0 || ip->ip_src.s_addr == INADDR_ANY))
 				ip->ip_src.s_addr =
 					ip_mcast_src(imo->imo_multicast_vif);
+#endif /* MROUTING */
 		} else
 			if ((flags & IP_RAWOUTPUT) == 0) ip->ip_ttl = IP_DEFAULT_MULTICAST_TTL;
 		/*
@@ -429,7 +544,7 @@ loopit:
 		 */
 		if ((imo == NULL) || (imo->imo_multicast_vif == -1)) {
 			if ((ifp->if_flags & IFF_MULTICAST) == 0) {
-				ipstat.ips_noroute++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
 				error = ENETUNREACH;
 				goto bad;
 			}
@@ -440,13 +555,14 @@ loopit:
 		 */
 		if (ip->ip_src.s_addr == INADDR_ANY) {
 			register struct in_ifaddr *ia1;
-
+			lck_mtx_lock(rt_mtx);	
 			TAILQ_FOREACH(ia1, &in_ifaddrhead, ia_link)
 				if (ia1->ia_ifp == ifp) {
 					ip->ip_src = IA_SIN(ia1)->sin_addr;
 					
 					break;
 				}
+			lck_mtx_unlock(rt_mtx);	
 			if (ip->ip_src.s_addr == INADDR_ANY) {
 				error = ENETUNREACH;
 				goto bad;
@@ -475,7 +591,6 @@ loopit:
 					ipf_pktopts.ippo_mcast_loop = imo->imo_multicast_loop;
 				}
 				
-				lck_mtx_unlock(ip_mutex);
 				ipf_ref();
 				
 				/* 4135317 - always pass network byte order to filter */
@@ -495,7 +610,6 @@ loopit:
 						}
 						if (result != 0) {
 							ipf_unref();
-							lck_mtx_lock(ip_mutex);
 							goto bad;
 						}
 					}
@@ -506,12 +620,12 @@ loopit:
 				NTOHS(ip->ip_len);
 				NTOHS(ip->ip_off);
 				
-				lck_mtx_lock(ip_mutex);
 				ipf_unref();
 				didfilter = 1;
 			}
 			ip_mloopback(ifp, m, dst, hlen);
 		}
+#if MROUTING
 		else {
 			/*
 			 * If we are acting as a multicast router, perform
@@ -536,11 +650,11 @@ loopit:
 				  imo = NULL;
 				if (ip_mforward(ip, ifp, m, imo) != 0) {
 					m_freem(m);
-					lck_mtx_unlock(ip_mutex);
 					goto done;
 				}
 			}
 		}
+#endif /* MROUTING */
 
 		/*
 		 * Multicasts with a time-to-live of zero may be looped-
@@ -552,7 +666,6 @@ loopit:
 		 */
 		if (ip->ip_ttl == 0 || ifp->if_flags & IFF_LOOPBACK) {
 			m_freem(m);
-			lck_mtx_unlock(ip_mutex);
 			goto done;
 		}
 
@@ -612,12 +725,10 @@ sendit:
 		}
         }
 
-injectit:
 	if (!didfilter && !TAILQ_EMPTY(&ipv4_filters)) {
 		struct ipfilter	*filter;
 		int seen = (inject_filter_ref == 0);
 		
-		lck_mtx_unlock(ip_mutex);
 		ipf_ref();
 		
 		/* 4135317 - always pass network byte order to filter */
@@ -637,7 +748,6 @@ injectit:
 				}
 				if (result != 0) {
 					ipf_unref();
-					lck_mtx_lock(ip_mutex);
 					goto bad;
 				}
 			}
@@ -649,7 +759,6 @@ injectit:
 		NTOHS(ip->ip_off);
 		
 		ipf_unref();
-		lck_mtx_lock(ip_mutex);
 	}
 
 #if IPSEC
@@ -660,7 +769,6 @@ injectit:
 
 	KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
 
-	lck_mtx_lock(sadb_mutex);
 
 	/* get SP for this packet */
 	if (so == NULL)
@@ -669,9 +777,8 @@ injectit:
 		sp = ipsec4_getpolicybysock(m, IPSEC_DIR_OUTBOUND, so, &error);
 
 	if (sp == NULL) {
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);		
 		KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
-		lck_mtx_unlock(sadb_mutex);
 		goto bad;
 	}
 
@@ -680,19 +787,18 @@ injectit:
 	/* check policy */
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+	case IPSEC_POLICY_GENERATE:
 		/*
 		 * This packet is just discarded.
 		 */
-		ipsecstat.out_polvio++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_polvio);
 		KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 1,0,0,0,0);
-		lck_mtx_unlock(sadb_mutex);
 		goto bad;
 
 	case IPSEC_POLICY_BYPASS:
 	case IPSEC_POLICY_NONE:
 		/* no need to do IPsec. */
 		KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 2,0,0,0,0);
-		lck_mtx_unlock(sadb_mutex);
 		goto skip_ipsec;
 	
 	case IPSEC_POLICY_IPSEC:
@@ -700,7 +806,6 @@ injectit:
 			/* acquire a policy */
 			error = key_spdacquire(sp);
 			KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 3,0,0,0,0);
-			lck_mtx_unlock(sadb_mutex);
 			goto bad;
 		}
 		break;
@@ -734,10 +839,7 @@ injectit:
 	HTONS(ip->ip_len);
 	HTONS(ip->ip_off);
 
-	lck_mtx_unlock(ip_mutex);
 	error = ipsec4_output(&state, sp, flags);
-	lck_mtx_unlock(sadb_mutex);
-	lck_mtx_lock(ip_mutex);
     
 	m0 = m = state.m;
 	
@@ -787,13 +889,15 @@ injectit:
 #endif
 	/* Check that there wasn't a route change and src is still valid */
 
+	lck_mtx_lock(rt_mtx);
 	if (ro->ro_rt && ro->ro_rt->generation_id != route_generation) {
 		if (ifa_foraddr(ip->ip_src.s_addr) == 0 && ((flags & (IP_ROUTETOIF | IP_FORWARDING)) == 0)) {
 		 	error = EADDRNOTAVAIL;
+			lck_mtx_unlock(rt_mtx);
 			KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 5,0,0,0,0);
 			goto bad;
 		}
-		rtfree(ro->ro_rt);
+		rtfree_locked(ro->ro_rt);
 		ro->ro_rt = NULL;
 	}
 
@@ -802,6 +906,7 @@ injectit:
 			printf("ip_output: "
 				"can't update route after IPsec processing\n");
 			error = EHOSTUNREACH;	/*XXX*/	
+			lck_mtx_unlock(rt_mtx);
 			KERNEL_DEBUG(DBG_FNC_IPSEC4_OUTPUT | DBG_FUNC_END, 6,0,0,0,0);
 			goto bad;
 		}
@@ -813,6 +918,7 @@ injectit:
 			ifaref(&ia->ia_ifa);
 		ifp = ro->ro_rt->rt_ifp;
 	}
+	lck_mtx_unlock(rt_mtx);
 
 	/* make it flipped, again. */
 	NTOHS(ip->ip_len);
@@ -823,7 +929,6 @@ injectit:
 	if (!TAILQ_EMPTY(&ipv4_filters)) {
 		struct ipfilter	*filter;
 		
-		lck_mtx_unlock(ip_mutex);
 		ipf_ref();
 		
 		/* 4135317 - always pass network byte order to filter */
@@ -840,7 +945,6 @@ injectit:
 				}
 				if (result != 0) {
 					ipf_unref();
-					lck_mtx_lock(ip_mutex);
 					goto bad;
 				}
 			}
@@ -852,11 +956,11 @@ injectit:
 		NTOHS(ip->ip_off);
 		
 		ipf_unref();
-		lck_mtx_lock(ip_mutex);
 	}
 skip_ipsec:
 #endif /*IPSEC*/
 
+#if IPFIREWALL
 	/*
 	 * IpHack's section.
 	 * - Xlate: translate packet's addr/port (NAT).
@@ -868,7 +972,6 @@ skip_ipsec:
 		struct  mbuf    *m1 = m;
 
 		if ((error = (*fr_checkp)(ip, hlen, ifp, 1, &m1)) || !m1) {
-			lck_mtx_unlock(ip_mutex);
 			goto done;
 		}
 		ip = mtod(m0 = m = m1, struct ip *);
@@ -884,7 +987,6 @@ skip_ipsec:
 		args.m = m;
 		args.next_hop = dst;
 		args.oif = ifp;
-		lck_mtx_unlock(ip_mutex);
 		off = ip_fw_chk_ptr(&args);
 		m = args.m;
 		dst = args.next_hop;
@@ -911,8 +1013,8 @@ skip_ipsec:
 			goto done ;
 		}
 		ip = mtod(m, struct ip *);
+		
 		if (off == 0 && dst == old) {/* common case */
-			lck_mtx_lock(ip_mutex);
 			goto pass ;
 		}
 #if DUMMYNET
@@ -935,7 +1037,6 @@ skip_ipsec:
 		    goto done;
 		}
 #endif /* DUMMYNET */
-		lck_mtx_lock(ip_mutex);
 #if IPDIVERT
 		if (off != 0 && (off & IP_FW_PORT_DYNT_FLAG) == 0) {
 			struct mbuf *clone = NULL;
@@ -966,7 +1067,6 @@ skip_ipsec:
 				ip = mtod(m, struct ip *);
 				goto pass;
 			}
-			lck_mtx_unlock(ip_mutex);
 			goto done;
 		}
 #endif
@@ -1053,7 +1153,6 @@ skip_ipsec:
 				HTONS(ip->ip_len);
 				HTONS(ip->ip_off);
 				
-				lck_mtx_unlock(ip_mutex);
 				
 				/*  we need to call dlil_output to run filters
 				 *	and resync to avoid recursion loops.
@@ -1075,11 +1174,13 @@ skip_ipsec:
 			bcopy(dst, &ro_fwd->ro_dst, sizeof(*dst));
 
 			ro_fwd->ro_rt = 0;
-			rtalloc_ign(ro_fwd, RTF_PRCLONING);
+			lck_mtx_lock(rt_mtx);
+			rtalloc_ign_locked(ro_fwd, RTF_PRCLONING);
 
 			if (ro_fwd->ro_rt == 0) {
-				ipstat.ips_noroute++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
 				error = EHOSTUNREACH;
+				lck_mtx_unlock(rt_mtx);
 				goto bad;
 			}
 
@@ -1093,9 +1194,10 @@ skip_ipsec:
 				    (ro_fwd->ro_rt->rt_flags & RTF_BROADCAST);
 			else
 				isbroadcast = in_broadcast(dst->sin_addr, ifp);
-			rtfree(ro->ro_rt);
+			rtfree_locked(ro->ro_rt);
 			ro->ro_rt = ro_fwd->ro_rt;
 			dst = (struct sockaddr_in *)&ro_fwd->ro_dst;
+			lck_mtx_unlock(rt_mtx);
 
 			/*
 			 * If we added a default src ip earlier,
@@ -1113,9 +1215,9 @@ skip_ipsec:
                  */
 		m_freem(m);
 		error = EACCES; /* not sure this is the right error msg */
-		lck_mtx_unlock(ip_mutex);
 		goto done;
 	}
+#endif /* IPFIREWALL */
 
 pass:
 #if __APPLE__
@@ -1123,7 +1225,7 @@ pass:
 	if ((ifp->if_flags & IFF_LOOPBACK) == 0 &&
 		 ((ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 		  (ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET)) {
-		ipstat.ips_badaddr++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_badaddr);
 		m_freem(m);
 		/* 
 		 * Do not simply drop the packet just like a firewall -- we want the 
@@ -1133,7 +1235,6 @@ pass:
 		 * loopback as the source address.
 		 */
 		error = ENETUNREACH;
-		lck_mtx_unlock(ip_mutex);
 		goto done;
 	}
 #endif
@@ -1161,6 +1262,9 @@ pass:
 			/* let the software handle any UDP or TCP checksums */
 			sw_csum |= (CSUM_DELAY_DATA & m->m_pkthdr.csum_flags);
 		}
+	} else if (apple_hwcksum_tx == 0) {
+		sw_csum |= (CSUM_DELAY_DATA | CSUM_DELAY_IP) &
+		    m->m_pkthdr.csum_flags;
 	}
 	
 	if (sw_csum & CSUM_DELAY_DATA) {
@@ -1168,8 +1272,13 @@ pass:
 		sw_csum &= ~CSUM_DELAY_DATA;
 		m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
 	}
-	
-	m->m_pkthdr.csum_flags &= IF_HWASSIST_CSUM_FLAGS(ifp->if_hwassist);
+
+	if (apple_hwcksum_tx != 0) {
+		m->m_pkthdr.csum_flags &=
+		    IF_HWASSIST_CSUM_FLAGS(ifp->if_hwassist);
+	} else {
+		m->m_pkthdr.csum_flags = 0;
+	}
 
 	/*
 	 * If small enough for interface, or the interface will take
@@ -1177,6 +1286,8 @@ pass:
 	 */
 	if ((u_short)ip->ip_len <= ifp->if_mtu ||
 	    ifp->if_hwassist & CSUM_FRAGMENT) {
+		struct rtentry *rte;
+
 		HTONS(ip->ip_len);
 		HTONS(ip->ip_off);
 		ip->ip_sum = 0;
@@ -1198,20 +1309,30 @@ pass:
 			ipsec_delaux(m);
 #endif
 		if (packetchain == 0) {
-			lck_mtx_unlock(ip_mutex);
-			error = dlil_output(ifp, PF_INET, m, (void *) ro->ro_rt,
-				    (struct sockaddr *)dst, 0);
-			goto done;	
+			lck_mtx_lock(rt_mtx);
+			if ((rte = ro->ro_rt) != NULL)
+				rtref(rte);
+			lck_mtx_unlock(rt_mtx);
+			error = ifnet_output(ifp, PF_INET, m, rte,
+			    (struct sockaddr *)dst);
+			if (rte != NULL)
+				rtfree(rte);
+			goto done;
 		}
 		else { /* packet chaining allows us to reuse the route for all packets */
 			m = m->m_nextpkt;
 			if (m == NULL) {
 				if (pktcnt > ip_maxchainsent)
 					ip_maxchainsent = pktcnt;
+				lck_mtx_lock(rt_mtx);
+				if ((rte = ro->ro_rt) != NULL)
+					rtref(rte);
+				lck_mtx_unlock(rt_mtx);
 				//send
-				lck_mtx_unlock(ip_mutex);
-				error = dlil_output_list(ifp, PF_INET, packetlist, (void *) ro->ro_rt,
-			    		(struct sockaddr *)dst, 0);
+				error = ifnet_output(ifp, PF_INET, packetlist,
+				    rte, (struct sockaddr *)dst);
+				if (rte != NULL)
+					rtfree(rte);
 				pktcnt = 0;
 				goto done;
 	
@@ -1234,12 +1355,15 @@ pass:
 		 * them, there is no way for one to update all its
 		 * routes when the MTU is changed.
 		 */
+		
+		lck_mtx_lock(rt_mtx);
 		if (ro->ro_rt && (ro->ro_rt->rt_flags & (RTF_UP | RTF_HOST))
 		    && !(ro->ro_rt->rt_rmx.rmx_locks & RTV_MTU)
 		    && (ro->ro_rt->rt_rmx.rmx_mtu > ifp->if_mtu)) {
 			ro->ro_rt->rt_rmx.rmx_mtu = ifp->if_mtu;
 		}
-		ipstat.ips_cantfrag++;
+		lck_mtx_unlock(rt_mtx);
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_cantfrag);
 		goto bad;
 	}
 	len = (ifp->if_mtu - hlen) &~ 7;
@@ -1255,10 +1379,6 @@ pass:
 	if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA &&
 	    (ifp->if_hwassist & CSUM_IP_FRAGS) == 0) {
 		in_delayed_cksum(m);
-		if (m == NULL) {
-			lck_mtx_unlock(ip_mutex);
-			return(ENOMEM);
-		}
 		m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA;
 	}
 
@@ -1275,10 +1395,10 @@ pass:
 	m0 = m;
 	mhlen = sizeof (struct ip);
 	for (off = hlen + len; off < (u_short)ip->ip_len; off += len) {
-		MGETHDR(m, M_DONTWAIT, MT_HEADER);
+		MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (m == 0) {
 			error = ENOBUFS;
-			ipstat.ips_odropped++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_odropped);
 			goto sendorfree;
 		}
 		m->m_flags |= (m0->m_flags & M_MCAST) | M_FRAG;
@@ -1302,13 +1422,16 @@ pass:
 		if (m->m_next == 0) {
 			(void) m_free(m);
 			error = ENOBUFS;	/* ??? */
-			ipstat.ips_odropped++;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_odropped);
 			goto sendorfree;
 		}
 		m->m_pkthdr.len = mhlen + len;
 		m->m_pkthdr.rcvif = 0;
 		m->m_pkthdr.csum_flags = m0->m_pkthdr.csum_flags;
 		m->m_pkthdr.socket_id = m0->m_pkthdr.socket_id;
+#if CONFIG_MACF_NET
+		mac_netinet_fragment(m0, m);
+#endif
 		HTONS(mhip->ip_off);
 		mhip->ip_sum = 0;
 		if (sw_csum & CSUM_DELAY_IP) {
@@ -1318,7 +1441,7 @@ pass:
 		mnext = &m->m_nextpkt;
 		nfrags++;
 	}
-	ipstat.ips_ofragments += nfrags;
+	OSAddAtomic(nfrags, (SInt32*)&ipstat.ips_ofragments);
 
 	/* set first/last markers for fragment chain */
 	m->m_flags |= M_LASTFRAG;
@@ -1344,7 +1467,6 @@ sendorfree:
 	KERNEL_DEBUG(DBG_LAYER_END, ip->ip_dst.s_addr, 
 		     ip->ip_src.s_addr, ip->ip_p, ip->ip_off, ip->ip_len);
 
-	lck_mtx_unlock(ip_mutex);
 	for (m = m0; m; m = m0) {
 		m0 = m->m_nextpkt;
 		m->m_nextpkt = 0;
@@ -1354,6 +1476,7 @@ sendorfree:
 			ipsec_delaux(m);
 #endif
 		if (error == 0) {
+			struct rtentry *rte;
 #ifndef __APPLE__
 			/* Record statistics for this interface address. */
 			if (ia != NULL) {
@@ -1361,16 +1484,22 @@ sendorfree:
 				ia->ia_ifa.if_obytes += m->m_pkthdr.len;
 			}
 #endif
-		    if ((packetchain != 0)  && (pktcnt > 0)) 	
-				panic("ip_output: mix of packet in packetlist is wrong=%x", packetlist);
-		    error = dlil_output(ifp, PF_INET, m, (void *) ro->ro_rt,
-					(struct sockaddr *)dst, 0);
+			if ((packetchain != 0)  && (pktcnt > 0))
+				panic("ip_output: mix of packet in packetlist is wrong=%p", packetlist);
+			lck_mtx_lock(rt_mtx);
+			if ((rte = ro->ro_rt) != NULL)
+				rtref(rte);
+			lck_mtx_unlock(rt_mtx);
+			error = ifnet_output(ifp, PF_INET, m, rte,
+			    (struct sockaddr *)dst);
+			if (rte != NULL)
+				rtfree(rte);
 		} else
 			m_freem(m);
 	}
 
 	if (error == 0)
-		ipstat.ips_fragmented++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_fragmented);
     }
 done:
 	if (ia) {
@@ -1386,9 +1515,7 @@ done:
 	if (sp != NULL) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP ip_output call free SP:%x\n", sp));
-		lck_mtx_lock(sadb_mutex);
-		key_freesp(sp);
-		lck_mtx_unlock(sadb_mutex);
+		key_freesp(sp, KEY_SADB_UNLOCKED);
 	}
 	}
 #endif /* IPSEC */
@@ -1397,10 +1524,25 @@ done:
 	return (error);
 bad:
 	m_freem(m0);
-	lck_mtx_unlock(ip_mutex);
 	goto done;
 }
 
+static void
+ip_out_cksum_stats(int proto, u_int32_t len)
+{
+	switch (proto) {
+	case IPPROTO_TCP:
+		tcp_out_cksum_stats(len);
+		break;
+	case IPPROTO_UDP:
+		udp_out_cksum_stats(len);
+		break;
+	default:
+		/* keep only TCP or UDP stats for now */
+		break;
+	}
+}
+
 void
 in_delayed_cksum_offset(struct mbuf *m0, int ip_offset)
 {
@@ -1413,7 +1555,7 @@ in_delayed_cksum_offset(struct mbuf *m0, int ip_offset)
 		ip_offset -= m->m_len;
 		m = m->m_next;
 		if (m == NULL) {
-			printf("in_delayed_cksum_offset failed - ip_offset wasn't in the packet\n");
+			printf("in_delayed_cksum_withoffset failed - ip_offset wasn't in the packet\n");
 			return;
 		}
 	}
@@ -1421,10 +1563,10 @@ in_delayed_cksum_offset(struct mbuf *m0, int ip_offset)
 	/* Sometimes the IP header is not contiguous, yes this can happen! */
 	if (ip_offset + sizeof(struct ip) > m->m_len) {
 #if DEBUG		
-		printf("delayed m_pullup, m->len: %d  off: %d\n",
+		printf("delayed m_pullup, m->len: %ld  off: %d\n",
 			m->m_len, ip_offset);
 #endif
-		m_copydata(m, ip_offset, sizeof(struct ip), buf);
+		m_copydata(m, ip_offset, sizeof(struct ip), (caddr_t) buf);
 		
 		ip = (struct ip *)buf;
 	} else {
@@ -1461,6 +1603,9 @@ in_delayed_cksum_offset(struct mbuf *m0, int ip_offset)
 
 	csum = in_cksum_skip(m, ip_len, offset);
 
+	/* Update stats */
+	ip_out_cksum_stats(ip->ip_p, ip_len - offset);
+
 	if (m0->m_pkthdr.csum_flags & CSUM_UDP && csum == 0)
 		csum = 0xffff;
 	offset += m0->m_pkthdr.csum_data & 0xFFFF;        /* checksum offset */
@@ -1481,7 +1626,7 @@ in_delayed_cksum_offset(struct mbuf *m0, int ip_offset)
 		char tmp[2];
 		
 #if DEBUG
-		printf("delayed m_copyback, m->len: %d  off: %d  p: %d\n",
+		printf("delayed m_copyback, m->len: %ld  off: %d  p: %d\n",
 		    m->m_len, offset + ip_offset, ip->ip_p);
 #endif
 		*(u_short *)tmp = csum;
@@ -1517,10 +1662,10 @@ in_cksum_offset(struct mbuf* m, size_t ip_offset)
 	if (ip_offset + sizeof(struct ip) > m->m_len) {
 
 #if DEBUG
-		printf("in_cksum_offset - delayed m_pullup, m->len: %d  off: %d\n",
+		printf("in_cksum_offset - delayed m_pullup, m->len: %ld  off: %lu\n",
 			m->m_len, ip_offset);
 #endif	
-		m_copydata(m, ip_offset, sizeof(struct ip), buf);
+		m_copydata(m, ip_offset, sizeof(struct ip), (caddr_t) buf);
 
 		ip = (struct ip *)buf;
 		ip->ip_sum = 0;
@@ -1556,7 +1701,7 @@ in_cksum_offset(struct mbuf* m, size_t ip_offset)
 		if (ntohs(ip->ip_len) != (m->m_pkthdr.len - ip_offset)) {
 			ip->ip_len = SWAP16(ip->ip_len);
 			printf("in_cksum_offset: ip_len %d (%d) "
-			    "doesn't match actual length %d\n",
+			    "doesn't match actual length %lu\n",
 			    ip->ip_len, SWAP16(ip->ip_len),
 			    (m->m_pkthdr.len - ip_offset));
 			return;
@@ -1581,7 +1726,7 @@ in_cksum_offset(struct mbuf* m, size_t ip_offset)
 		char tmp[2];
 
 #if DEBUG
-		printf("in_cksum_offset m_copyback, m->len: %d  off: %d  p: %d\n",
+		printf("in_cksum_offset m_copyback, m->len: %lu  off: %lu  p: %d\n",
 		    m->m_len, ip_offset + offsetof(struct ip, ip_sum), ip->ip_p);
 #endif
 		*(u_short *)tmp = ip->ip_sum;
@@ -1613,10 +1758,13 @@ ip_insertoptions(m, opt, phlen)
 	if (p->ipopt_dst.s_addr)
 		ip->ip_dst = p->ipopt_dst;
 	if (m->m_flags & M_EXT || m->m_data - optlen < m->m_pktdat) {
-		MGETHDR(n, M_DONTWAIT, MT_HEADER);
+		MGETHDR(n, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (n == 0)
 			return (m);
 		n->m_pkthdr.rcvif = 0;
+#if CONFIG_MACF_NET
+		mac_mbuf_label_copy(m, n);
+#endif
 		n->m_pkthdr.len = m->m_pkthdr.len + optlen;
 		m->m_len -= sizeof(struct ip);
 		m->m_data += sizeof(struct ip);
@@ -1787,6 +1935,61 @@ ip_ctloutput(so, sopt)
 			break;
 #undef OPTSET
 
+#if CONFIG_FORCE_OUT_IFP		
+		case IP_FORCE_OUT_IFP: {
+			char	ifname[IFNAMSIZ];
+			ifnet_t	ifp;
+			
+			/* Verify interface name parameter is sane */
+			if (sopt->sopt_valsize > sizeof(ifname)) {
+				error = EINVAL;
+				break;
+			}
+			
+			/* Copy the interface name */
+			if (sopt->sopt_valsize != 0) {
+				error = sooptcopyin(sopt, ifname, sizeof(ifname), sopt->sopt_valsize);
+				if (error)
+					break;
+			}
+			
+			if (sopt->sopt_valsize == 0 || ifname[0] == 0) {
+				// Set pdp_ifp to NULL
+				inp->pdp_ifp = NULL;
+				
+				// Flush the route
+				if (inp->inp_route.ro_rt) {
+					rtfree(inp->inp_route.ro_rt);
+					inp->inp_route.ro_rt = NULL;
+				}
+				
+				break;
+			}
+			
+			/* Verify name is NULL terminated */
+			if (ifname[sopt->sopt_valsize - 1] != 0) {
+				error = EINVAL;
+				break;
+			}
+			
+			if (ifnet_find_by_name(ifname, &ifp) != 0) {
+				error = ENXIO;
+				break;
+			}
+			
+			/* Won't actually free. Since we don't release this later, we should do it now. */
+			ifnet_release(ifp);
+			
+			/* This only works for point-to-point interfaces */
+			if ((ifp->if_flags & IFF_POINTOPOINT) == 0) {
+				error = ENOTSUP;
+				break;
+			}
+			
+			inp->pdp_ifp = ifp;
+		}
+		break;
+#endif
 		case IP_MULTICAST_IF:
 		case IP_MULTICAST_VIF:
 		case IP_MULTICAST_TTL:
@@ -1848,14 +2051,29 @@ ip_ctloutput(so, sopt)
 				len = m->m_len;
 			}
 			optname = sopt->sopt_name;
-			lck_mtx_lock(sadb_mutex);
 			error = ipsec4_set_policy(inp, optname, req, len, priv);
-			lck_mtx_unlock(sadb_mutex);
 			m_freem(m);
 			break;
 		}
 #endif /*IPSEC*/
 
+#if TRAFFIC_MGT
+		case IP_TRAFFIC_MGT_BACKGROUND:
+		{
+			unsigned	background = 0;
+			error = sooptcopyin(sopt, &background, sizeof(background), sizeof(background));
+			if (error) 
+				break;
+
+			if (background) 
+				so->so_traffic_mgt_flags |= TRAFFIC_MGT_SO_BACKGROUND;
+			else 
+				so->so_traffic_mgt_flags &= ~TRAFFIC_MGT_SO_BACKGROUND;
+
+			break;
+		}
+#endif /* TRAFFIC_MGT */
+
 		default:
 			error = ENOPROTOOPT;
 			break;
@@ -1956,9 +2174,7 @@ ip_ctloutput(so, sopt)
 				req = mtod(m, caddr_t);
 				len = m->m_len;
 			}
-			lck_mtx_lock(sadb_mutex);
 			error = ipsec4_get_policy(sotoinpcb(so), req, len, &m);
-			lck_mtx_unlock(sadb_mutex);
 			if (error == 0)
 				error = soopt_mcopyout(sopt, m); /* XXX */
 			if (error == 0)
@@ -1967,6 +2183,15 @@ ip_ctloutput(so, sopt)
 		}
 #endif /*IPSEC*/
 
+#if TRAFFIC_MGT
+		case IP_TRAFFIC_MGT_BACKGROUND:
+		{
+			unsigned	background = so->so_traffic_mgt_flags;
+			return (sooptcopyout(sopt, &background, sizeof(background)));
+			break;
+		}
+#endif /* TRAFFIC_MGT */
+
 		default:
 			error = ENOPROTOOPT;
 			break;
@@ -1982,10 +2207,10 @@ ip_ctloutput(so, sopt)
  * with destination address if source routed.
  */
 static int
-ip_pcbopts(optname, pcbopt, m)
-	int optname;
-	struct mbuf **pcbopt;
-	register struct mbuf *m;
+ip_pcbopts(
+	__unused int optname,
+	struct mbuf **pcbopt,
+	register struct mbuf *m)
 {
 	register int cnt, optlen;
 	register u_char *cp;
@@ -2148,6 +2373,7 @@ ip_setmoptions(sopt, imop)
 
 	switch (sopt->sopt_name) {
 	/* store an index number for the vif you wanna use in the send */
+#if MROUTING
 	case IP_MULTICAST_VIF:
 		if (legal_vif_num == 0) {
 			error = EOPNOTSUPP;
@@ -2162,6 +2388,7 @@ ip_setmoptions(sopt, imop)
 		}
 		imo->imo_multicast_vif = i;
 		break;
+#endif /* MROUTING */
 
 	case IP_MULTICAST_IF:
 		/*
@@ -2278,7 +2505,7 @@ ip_setmoptions(sopt, imop)
 	 * If all options have default values, no need to keep the mbuf.
 	 */
 	if (imo->imo_multicast_ifp == NULL &&
-	    imo->imo_multicast_vif == -1 &&
+	    imo->imo_multicast_vif == (u_long)-1 &&
 	    imo->imo_multicast_ttl == IP_DEFAULT_MULTICAST_TTL &&
 	    imo->imo_multicast_loop == IP_DEFAULT_MULTICAST_LOOP &&
 	    imo->imo_num_memberships == 0) {
@@ -2341,15 +2568,17 @@ ip_addmembership(
 		dst->sin_len = sizeof(*dst);
 		dst->sin_family = AF_INET;
 		dst->sin_addr = mreq->imr_multiaddr;
-		rtalloc(&ro);
+		lck_mtx_lock(rt_mtx);
+		rtalloc_ign_locked(&ro, 0UL);
 		if (ro.ro_rt != NULL) {
 			ifp = ro.ro_rt->rt_ifp;
-			rtfree(ro.ro_rt);
+			rtfree_locked(ro.ro_rt);
 		}
 		else {
 			/* If there's no default route, try using loopback */
 			mreq->imr_interface.s_addr = INADDR_LOOPBACK;
 		}
+		lck_mtx_unlock(rt_mtx);
 	}
 	
 	if (ifp == NULL) {
@@ -2470,6 +2699,7 @@ ip_getmoptions(sopt, imo)
 
 	error = 0;
 	switch (sopt->sopt_name) {
+#if MROUTING
 	case IP_MULTICAST_VIF: 
 		if (imo != NULL)
 			optval = imo->imo_multicast_vif;
@@ -2477,6 +2707,7 @@ ip_getmoptions(sopt, imo)
 			optval = -1;
 		error = sooptcopyout(sopt, &optval, sizeof optval);
 		break;
+#endif /* MROUTING */
 
 	case IP_MULTICAST_IF:
 		if (imo == NULL || imo->imo_multicast_ifp == NULL)
@@ -2553,83 +2784,83 @@ ip_mloopback(ifp, m, dst, hlen)
 {
 	register struct ip *ip;
 	struct mbuf *copym;
+	int sw_csum = (apple_hwcksum_tx == 0);
 
 	copym = m_copy(m, 0, M_COPYALL);
 	if (copym != NULL && (copym->m_flags & M_EXT || copym->m_len < hlen))
 		copym = m_pullup(copym, hlen);
-	if (copym != NULL) {
-		/*
-		 * We don't bother to fragment if the IP length is greater
-		 * than the interface's MTU.  Can this possibly matter?
-		 */
-		ip = mtod(copym, struct ip *);
-		HTONS(ip->ip_len);
-		HTONS(ip->ip_off);
-		ip->ip_sum = 0;
-		ip->ip_sum = in_cksum(copym, hlen);
-		/*
-		 * NB:
-		 * It's not clear whether there are any lingering
-		 * reentrancy problems in other areas which might
-		 * be exposed by using ip_input directly (in
-		 * particular, everything which modifies the packet
-		 * in-place).  Yet another option is using the
-		 * protosw directly to deliver the looped back
-		 * packet.  For the moment, we'll err on the side
-		 * of safety by using if_simloop().
-		 */
+
+	if (copym == NULL)
+		return;
+
+	/*
+	 * We don't bother to fragment if the IP length is greater
+	 * than the interface's MTU.  Can this possibly matter?
+	 */
+	ip = mtod(copym, struct ip *);
+	HTONS(ip->ip_len);
+	HTONS(ip->ip_off);
+	ip->ip_sum = 0;
+	ip->ip_sum = in_cksum(copym, hlen);
+	/*
+	 * NB:
+	 * It's not clear whether there are any lingering
+	 * reentrancy problems in other areas which might
+	 * be exposed by using ip_input directly (in
+	 * particular, everything which modifies the packet
+	 * in-place).  Yet another option is using the
+	 * protosw directly to deliver the looped back
+	 * packet.  For the moment, we'll err on the side
+	 * of safety by using if_simloop().
+	 */
 #if 1 /* XXX */
-		if (dst->sin_family != AF_INET) {
-			printf("ip_mloopback: bad address family %d\n",
-						dst->sin_family);
-			dst->sin_family = AF_INET;
-		}
+	if (dst->sin_family != AF_INET) {
+		printf("ip_mloopback: bad address family %d\n",
+					dst->sin_family);
+		dst->sin_family = AF_INET;
+	}
 #endif
 
-
         /*
-        * Mark checksum as valid or calculate checksum for loopback.
-        * 
-        * This is done this way because we have to embed the ifp of
-        * the interface we will send the original copy of the packet
-        * out on in the mbuf. ip_input will check if_hwassist of the
-        * embedded ifp and ignore all csum_flags if if_hwassist is 0.
-        * The UDP checksum has not been calculated yet.
-        */
-        if (copym->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
-            if (IF_HWASSIST_CSUM_FLAGS(ifp->if_hwassist)) {
-                copym->m_pkthdr.csum_flags |=
-                    CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
-                    CSUM_IP_CHECKED | CSUM_IP_VALID;
-                copym->m_pkthdr.csum_data = 0xffff;
-            } else {
-		NTOHS(ip->ip_len);
-                in_delayed_cksum(copym);
-		HTONS(ip->ip_len);
-	    }
-        }
-
-
-		/*
-		 * TedW: 
-		 * We need to send all loopback traffic down to dlil in case 
-		 * a filter has tapped-in.
-		 */
-
-		/*
-		 * Stuff the 'real' ifp into the pkthdr, to be used in matching
-		 *  in ip_input(); we need the loopback ifp/dl_tag passed as args
-		 *  to make the loopback driver compliant with the data link
-		 *  requirements.
-		 */
-		if (lo_ifp) {
-			copym->m_pkthdr.rcvif = ifp;
-		    dlil_output(lo_ifp, PF_INET, copym, 0, (struct sockaddr *) dst, 0);
+         * Mark checksum as valid or calculate checksum for loopback.
+         *
+         * This is done this way because we have to embed the ifp of
+         * the interface we will send the original copy of the packet
+         * out on in the mbuf. ip_input will check if_hwassist of the
+         * embedded ifp and ignore all csum_flags if if_hwassist is 0.
+         * The UDP checksum has not been calculated yet.
+         */
+        if (sw_csum || (copym->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) {
+		if (!sw_csum && IF_HWASSIST_CSUM_FLAGS(ifp->if_hwassist)) {
+			copym->m_pkthdr.csum_flags |=
+			    CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
+			    CSUM_IP_CHECKED | CSUM_IP_VALID;
+			copym->m_pkthdr.csum_data = 0xffff;
 		} else {
-		    printf("Warning: ip_output call to dlil_find_dltag failed!\n");
-		    m_freem(copym);
+			NTOHS(ip->ip_len);
+			in_delayed_cksum(copym);
+			HTONS(ip->ip_len);
 		}
+        }
 
-/*		if_simloop(ifp, copym, (struct sockaddr *)dst, 0);*/
+	/*
+	 * TedW: 
+	 * We need to send all loopback traffic down to dlil in case 
+	 * a filter has tapped-in.
+	 */
+
+	/*
+	 * Stuff the 'real' ifp into the pkthdr, to be used in matching
+	 *  in ip_input(); we need the loopback ifp/dl_tag passed as args
+	 *  to make the loopback driver compliant with the data link
+	 *  requirements.
+	 */
+	if (lo_ifp) {
+		copym->m_pkthdr.rcvif = ifp;
+		dlil_output(lo_ifp, PF_INET, copym, 0,
+		    (struct sockaddr *) dst, 0);
+	} else {
+		printf("Warning: ip_output call to dlil_find_dltag failed!\n");
+		m_freem(copym);
 	}
 }
diff --git a/bsd/netinet/ip_var.h b/bsd/netinet/ip_var.h
index ce5d0569f..81eb6f135 100644
--- a/bsd/netinet/ip_var.h
+++ b/bsd/netinet/ip_var.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -53,6 +59,12 @@
  *
  *	@(#)ip_var.h	8.2 (Berkeley) 1/9/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2007 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _NETINET_IP_VAR_H_
 #define	_NETINET_IP_VAR_H_
@@ -70,6 +82,9 @@ struct ipovly {
 };
 
 #ifdef KERNEL_PRIVATE
+#if CONFIG_MACF_NET
+struct label;
+#endif
 /*
  * Ip reassembly queue structure.  Each fragment
  * being reassembled is attached to one of these structures.
@@ -85,7 +100,9 @@ struct ipq {
 	struct	in_addr ipq_src,ipq_dst;
 	u_long	ipq_nfrags;
 	TAILQ_ENTRY(ipq) ipq_list;
-	u_long	reserved[1];		/* for future use */
+#if CONFIG_MACF_NET
+	struct label *ipq_label;	/* MAC label */
+#endif
 #if IPDIVERT
 #ifdef IPDIVERT_44
 	u_int32_t ipq_div_info;		/* ipfw divert port & flags */
@@ -133,49 +150,49 @@ struct ip_fwd_tag {
 #endif /* KERNEL_PRIVATE */
 
 struct	ipstat {
-	u_long	ips_total;		/* total packets received */
-	u_long	ips_badsum;		/* checksum bad */
-	u_long	ips_tooshort;		/* packet too short */
-	u_long	ips_toosmall;		/* not enough data */
-	u_long	ips_badhlen;		/* ip header length < data size */
-	u_long	ips_badlen;		/* ip length < ip header length */
-	u_long	ips_fragments;		/* fragments received */
-	u_long	ips_fragdropped;	/* frags dropped (dups, out of space) */
-	u_long	ips_fragtimeout;	/* fragments timed out */
-	u_long	ips_forward;		/* packets forwarded */
-	u_long	ips_fastforward;	/* packets fast forwarded */
-	u_long	ips_cantforward;	/* packets rcvd for unreachable dest */
-	u_long	ips_redirectsent;	/* packets forwarded on same net */
-	u_long	ips_noproto;		/* unknown or unsupported protocol */
-	u_long	ips_delivered;		/* datagrams delivered to upper level*/
-	u_long	ips_localout;		/* total ip packets generated here */
-	u_long	ips_odropped;		/* lost packets due to nobufs, etc. */
-	u_long	ips_reassembled;	/* total packets reassembled ok */
-	u_long	ips_fragmented;		/* datagrams successfully fragmented */
-	u_long	ips_ofragments;		/* output fragments created */
-	u_long	ips_cantfrag;		/* don't fragment flag was set, etc. */
-	u_long	ips_badoptions;		/* error in option processing */
-	u_long	ips_noroute;		/* packets discarded due to no route */
-	u_long	ips_badvers;		/* ip version != 4 */
-	u_long	ips_rawout;		/* total raw ip packets generated */
-	u_long	ips_toolong;		/* ip length > max ip packet size */
-	u_long	ips_notmember;		/* multicasts for unregistered grps */
-	u_long	ips_nogif;		/* no match gif found */
-	u_long	ips_badaddr;		/* invalid address on header */
+	u_int32_t	ips_total;		/* total packets received */
+	u_int32_t	ips_badsum;		/* checksum bad */
+	u_int32_t	ips_tooshort;		/* packet too short */
+	u_int32_t	ips_toosmall;		/* not enough data */
+	u_int32_t	ips_badhlen;		/* ip header length < data size */
+	u_int32_t	ips_badlen;		/* ip length < ip header length */
+	u_int32_t	ips_fragments;		/* fragments received */
+	u_int32_t	ips_fragdropped;	/* frags dropped (dups, out of space) */
+	u_int32_t	ips_fragtimeout;	/* fragments timed out */
+	u_int32_t	ips_forward;		/* packets forwarded */
+	u_int32_t	ips_fastforward;	/* packets fast forwarded */
+	u_int32_t	ips_cantforward;	/* packets rcvd for unreachable dest */
+	u_int32_t	ips_redirectsent;	/* packets forwarded on same net */
+	u_int32_t	ips_noproto;		/* unknown or unsupported protocol */
+	u_int32_t	ips_delivered;		/* datagrams delivered to upper level*/
+	u_int32_t	ips_localout;		/* total ip packets generated here */
+	u_int32_t	ips_odropped;		/* lost packets due to nobufs, etc. */
+	u_int32_t	ips_reassembled;	/* total packets reassembled ok */
+	u_int32_t	ips_fragmented;		/* datagrams successfully fragmented */
+	u_int32_t	ips_ofragments;		/* output fragments created */
+	u_int32_t	ips_cantfrag;		/* don't fragment flag was set, etc. */
+	u_int32_t	ips_badoptions;		/* error in option processing */
+	u_int32_t	ips_noroute;		/* packets discarded due to no route */
+	u_int32_t	ips_badvers;		/* ip version != 4 */
+	u_int32_t	ips_rawout;		/* total raw ip packets generated */
+	u_int32_t	ips_toolong;		/* ip length > max ip packet size */
+	u_int32_t	ips_notmember;		/* multicasts for unregistered grps */
+	u_int32_t	ips_nogif;		/* no match gif found */
+	u_int32_t	ips_badaddr;		/* invalid address on header */
 };
 
 struct ip_linklocal_stat {
-	u_long iplls_in_total;
-	u_long iplls_in_badttl;
-	u_long iplls_out_total;
-	u_long iplls_out_badttl;
+	u_int32_t	iplls_in_total;
+	u_int32_t	iplls_in_badttl;
+	u_int32_t	iplls_out_total;
+	u_int32_t	iplls_out_badttl;
 };
 
 #ifdef KERNEL_PRIVATE
 /* flags passed to ip_output as last parameter */
 #define	IP_FORWARDING		0x1		/* most of ip header exists */
 #define	IP_RAWOUTPUT		0x2		/* raw ip header exists */
-#define	IP_NOIPSEC			0x4		/* No IPSec processing */
+#define	IP_NOIPSEC		0x4		/* No IPSec processing */
 #define	IP_ROUTETOIF		SO_DONTROUTE	/* bypass routing tables (0x0010) */
 #define	IP_ALLOWBROADCAST	SO_BROADCAST	/* can send broadcast packets (0x0020) */
 
@@ -201,13 +218,13 @@ extern struct	pr_usrreqs rip_usrreqs;
 int	 ip_ctloutput(struct socket *, struct sockopt *sopt);
 void	 ip_drain(void);
 void	 ip_freemoptions(struct ip_moptions *);
-void	 ip_init(void);
+void	 ip_init(void) __attribute__((section("__TEXT, initcode")));
 extern int	 (*ip_mforward)(struct ip *, struct ifnet *, struct mbuf *,
 			  struct ip_moptions *);
 int	 ip_output(struct mbuf *,
-	    struct mbuf *, struct route *, int, struct ip_moptions *);
+	    struct mbuf *, struct route *, int, struct ip_moptions *, struct ifnet *);
 int	 ip_output_list(struct mbuf *, int,
-	    struct mbuf *, struct route *, int, struct ip_moptions *);
+	    struct mbuf *, struct route *, int, struct ip_moptions *, struct ifnet *);
 struct in_ifaddr *
 	 ip_rtaddr(struct in_addr, struct route *);
 void	 ip_savecontrol(struct inpcb *, struct mbuf **, struct ip *,
@@ -222,7 +239,7 @@ u_int16_t
 #endif
 int	rip_ctloutput(struct socket *, struct sockopt *);
 void	rip_ctlinput(int, struct sockaddr *, void *);
-void	rip_init(void);
+void	rip_init(void) __attribute__((section("__TEXT, initcode")));
 void	rip_input(struct mbuf *, int);
 int	rip_output(struct mbuf *, struct socket *, u_long);
 int	rip_unlock(struct socket *, int, int);
@@ -236,5 +253,14 @@ void	ip_rsvp_force_done(struct socket *);
 
 void	in_delayed_cksum(struct mbuf *m);
 
+extern void tcp_in_cksum_stats(u_int32_t);
+extern void tcp_out_cksum_stats(u_int32_t);
+
+extern void udp_in_cksum_stats(u_int32_t);
+extern void udp_out_cksum_stats(u_int32_t);
+
+int rip_send(struct socket *, int , struct mbuf *, struct sockaddr *, 
+			struct mbuf *, struct proc *);
+
 #endif /* KERNEL_PRIVATE */
 #endif /* !_NETINET_IP_VAR_H_ */
diff --git a/bsd/netinet/kpi_ipfilter.c b/bsd/netinet/kpi_ipfilter.c
index 5750c6151..3ade07635 100644
--- a/bsd/netinet/kpi_ipfilter.c
+++ b/bsd/netinet/kpi_ipfilter.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/param.h>	/* for definition of NULL */
@@ -307,7 +313,7 @@ ipf_injectv4_out(
 	NTOHS(ip->ip_off);
 	
 	/* Send  */
-	error = ip_output(m, NULL, &ro, IP_ALLOWBROADCAST | IP_RAWOUTPUT, imo);
+	error = ip_output(m, NULL, &ro, IP_ALLOWBROADCAST | IP_RAWOUTPUT, imo, NULL);
 	
 	/* Release the route */
 	if (ro.ro_rt)
@@ -316,6 +322,7 @@ ipf_injectv4_out(
 	return error;
 }
 
+#if INET6
 static errno_t
 ipf_injectv6_out(
 	mbuf_t data,
@@ -388,6 +395,7 @@ ipf_injectv6_out(
 	
 	return error;
 }
+#endif /* INET6 */
 
 errno_t
 ipf_inject_output(
@@ -412,9 +420,11 @@ ipf_inject_output(
 		case 4:
 			error = ipf_injectv4_out(data, filter_ref, options);
 			break;
+#if INET6
 		case 6:
 			error = ipf_injectv6_out(data, filter_ref, options);
 			break;
+#endif
 		default:
 			m_freem(m);
 			error = ENOTSUP;
diff --git a/bsd/netinet/kpi_ipfilter.h b/bsd/netinet/kpi_ipfilter.h
index bc0ae7867..a247d58dd 100644
--- a/bsd/netinet/kpi_ipfilter.h
+++ b/bsd/netinet/kpi_ipfilter.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_ipfilter.h
@@ -48,6 +54,8 @@ struct ipf_pktopts {
 
 typedef struct ipf_pktopts* ipf_pktopts_t;
 
+__BEGIN_DECLS
+
 /*!
 	@typedef ipf_input_func
 	
@@ -190,4 +198,5 @@ errno_t	ipf_inject_input(mbuf_t data, ipfilter_t filter_ref);
  */
 errno_t ipf_inject_output(mbuf_t data, ipfilter_t filter_ref, ipf_pktopts_t options);
 
+__END_DECLS
 #endif /* __KPI_IPFILTER__ */
diff --git a/bsd/netinet/kpi_ipfilter_var.h b/bsd/netinet/kpi_ipfilter_var.h
index f77f97214..f16c8908f 100644
--- a/bsd/netinet/kpi_ipfilter_var.h
+++ b/bsd/netinet/kpi_ipfilter_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __KPI_IPFILTER_VAR__
diff --git a/bsd/netinet/raw_ip.c b/bsd/netinet/raw_ip.c
index 4028d1291..df763aac5 100644
--- a/bsd/netinet/raw_ip.c
+++ b/bsd/netinet/raw_ip.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1993
@@ -53,6 +59,12 @@
  *
  *	@(#)raw_ip.c	8.7 (Berkeley) 5/15/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -65,10 +77,10 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
+#include <libkern/OSAtomic.h>
+#include <kern/zalloc.h>
 
-#if __FreeBSD__
-#include <vm/vm_zone.h>
-#endif
+#include <pexpert/pexpert.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -92,9 +104,20 @@
 #include <netinet/ip_dummynet.h>
 #endif
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
+
+int load_ipfw(void);
+int rip_detach(struct socket *);
+int rip_abort(struct socket *);
+int rip_disconnect(struct socket *);
+int rip_bind(struct socket *, struct sockaddr *, struct proc *);
+int rip_connect(struct socket *, struct sockaddr *, struct proc *);
+int rip_shutdown(struct socket *);
+ 
 #if IPSEC
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif
 
 extern u_long  route_generation;
@@ -157,7 +180,7 @@ rip_init()
 
 }
 
-static struct	sockaddr_in ripsrc = { sizeof(ripsrc), AF_INET };
+static struct	sockaddr_in ripsrc = { sizeof(ripsrc), AF_INET , 0, {0}, {0,0,0,0,0,0,0,0,} };
 /*
  * Setup generic address and protocol structures
  * for raw_input routine, then pass them along with
@@ -196,16 +219,21 @@ rip_input(m, iphlen)
 			/* check AH/ESP integrity. */
 			skipit = 0;
 			if (ipsec_bypass == 0 && n) {
-				lck_mtx_lock(sadb_mutex);
 				if (ipsec4_in_reject_so(n, last->inp_socket)) {
 					m_freem(n);
-					ipsecstat.in_polvio++;
+					IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 					/* do not inject data to pcb */
 					skipit = 1;
 				}
-				lck_mtx_unlock(sadb_mutex);
 			} 
 #endif /*IPSEC*/
+#if CONFIG_MACF_NET
+			if (n && skipit == 0) {
+				if (mac_inpcb_check_deliver(last, n, AF_INET,
+				    SOCK_RAW) != 0)
+					skipit = 1;
+			}
+#endif
 			if (n && skipit == 0) {
 				int error = 0;
 				if (last->inp_flags & INP_CONTROLOPTS ||
@@ -238,17 +266,21 @@ rip_input(m, iphlen)
 	/* check AH/ESP integrity. */
 	skipit = 0;
 	if (ipsec_bypass == 0 && last) {
-		lck_mtx_lock(sadb_mutex);
 		if (ipsec4_in_reject_so(m, last->inp_socket)) {
 			m_freem(m);
-			ipsecstat.in_polvio++;
-			ipstat.ips_delivered--;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_delivered);
 			/* do not inject data to pcb */
 			skipit = 1;
 		}
-		lck_mtx_unlock(sadb_mutex);
 	} 
 #endif /*IPSEC*/
+#if CONFIG_MACF_NET
+	if (last && skipit == 0) {
+		if (mac_inpcb_check_deliver(last, m, AF_INET, SOCK_RAW) != 0)
+			skipit = 1;
+	}
+#endif
 	if (skipit == 0) {
 		if (last) {
 			if (last->inp_flags & INP_CONTROLOPTS ||
@@ -267,8 +299,8 @@ rip_input(m, iphlen)
 			}
 		} else {
 			m_freem(m);
-			ipstat.ips_noproto++;
-			ipstat.ips_delivered--;
+			OSAddAtomic(1, (SInt32*)&ipstat.ips_noproto);
+			OSAddAtomic(-1, (SInt32*)&ipstat.ips_delivered);
 		}
 	}
 }
@@ -328,7 +360,7 @@ rip_output(m, so, dst)
 #endif
 		/* XXX prevent ip_output from overwriting header fields */
 		flags |= IP_RAWOUTPUT;
-		ipstat.ips_rawout++;
+		OSAddAtomic(1, (SInt32*)&ipstat.ips_rawout);
 	}
 
 #if IPSEC
@@ -343,12 +375,22 @@ rip_output(m, so, dst)
 		inp->inp_route.ro_rt = (struct rtentry *)0;
 	}
 
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_inpcb(inp, m);
+#endif
+
+#if CONFIG_FORCE_OUT_IFP
+	return (ip_output_list(m, 0, inp->inp_options, &inp->inp_route, flags,
+			  inp->inp_moptions, inp->pdp_ifp));
+#else
 	return (ip_output_list(m, 0, inp->inp_options, &inp->inp_route, flags,
-			  inp->inp_moptions));
+			  inp->inp_moptions, NULL));
+#endif
 }
 
-extern int
-load_ipfw()
+#if IPFIREWALL
+int
+load_ipfw(void)
 {
 	kern_return_t	err;
 	
@@ -362,6 +404,7 @@ load_ipfw()
 	
 	return err == 0 && ip_fw_ctl_ptr == NULL ? -1 : err;
 }
+#endif /* IPFIREWALL */
 
 /*
  * Raw IP socket option processing.
@@ -392,6 +435,7 @@ rip_ctloutput(so, sopt)
             error = sooptcopyout(sopt, &optval, sizeof optval);
             break;
 
+#if IPFIREWALL
 		case IP_FW_ADD:
 		case IP_FW_GET:
 		case IP_OLD_FW_ADD:
@@ -403,6 +447,7 @@ rip_ctloutput(so, sopt)
 			else
 				error = ENOPROTOOPT;
 			break;
+#endif IPFIREWALL
 
 #if DUMMYNET
 		case IP_DUMMYNET_GET:
@@ -413,6 +458,7 @@ rip_ctloutput(so, sopt)
 			break ;
 #endif /* DUMMYNET */
 
+#if MROUTING
 		case MRT_INIT:
 		case MRT_DONE:
 		case MRT_ADD_VIF:
@@ -423,6 +469,7 @@ rip_ctloutput(so, sopt)
 		case MRT_ASSERT:
 			error = ip_mrouter_get(so, sopt);
 			break;
+#endif /* MROUTING */
 
 		default:
 			error = ip_ctloutput(so, sopt);
@@ -455,6 +502,7 @@ rip_ctloutput(so, sopt)
             break;
 
 
+#if IPFIREWALL
 		case IP_FW_ADD:
 		case IP_FW_DEL:
 		case IP_FW_FLUSH:
@@ -472,6 +520,7 @@ rip_ctloutput(so, sopt)
 			else
 				error = ENOPROTOOPT;
 			break;
+#endif /* IPFIREWALL */
 
 #if DUMMYNET
 		case IP_DUMMYNET_CONFIGURE:
@@ -484,6 +533,7 @@ rip_ctloutput(so, sopt)
 			break ;
 #endif
 
+#if MROUTING
 		case IP_RSVP_ON:
 			error = ip_rsvp_init(so);
 			break;
@@ -500,7 +550,7 @@ rip_ctloutput(so, sopt)
 		case IP_RSVP_VIF_OFF:
 			error = ip_rsvp_vif_done(so, sopt);
 			break;
-
+		
 		case MRT_INIT:
 		case MRT_DONE:
 		case MRT_ADD_VIF:
@@ -511,6 +561,7 @@ rip_ctloutput(so, sopt)
 		case MRT_ASSERT:
 			error = ip_mrouter_set(so, sopt);
 			break;
+#endif /* MROUTING */
 
 		default:
 			error = ip_ctloutput(so, sopt);
@@ -530,10 +581,10 @@ rip_ctloutput(so, sopt)
  * interface routes.
  */
 void
-rip_ctlinput(cmd, sa, vip)
-	int cmd;
-	struct sockaddr *sa;
-	void *vip;
+rip_ctlinput(
+	int cmd,
+	struct sockaddr *sa,
+	__unused void *vip)
 {
 	struct in_ifaddr *ia;
 	struct ifnet *ifp;
@@ -602,25 +653,18 @@ static int
 rip_attach(struct socket *so, int proto, struct proc *p)
 {
 	struct inpcb *inp;
-	int error, s;
+	int error;
 
 	inp = sotoinpcb(so);
 	if (inp)
 		panic("rip_attach");
-#if __APPLE__
 	if ((so->so_state & SS_PRIV) == 0)
 		return (EPERM);
-#else
-	if (p && (error = suser(p)) != 0)
-		return error;
-#endif
 
 	error = soreserve(so, rip_sendspace, rip_recvspace);
 	if (error)
 		return error;
-	s = splnet();
 	error = in_pcballoc(so, &ripcbinfo, p);
-	splx(s);
 	if (error)
 		return error;
 	inp = (struct inpcb *)so->so_pcb;
@@ -638,11 +682,13 @@ rip_detach(struct socket *so)
 	inp = sotoinpcb(so);
 	if (inp == 0)
 		panic("rip_detach");
+#if MROUTING
 	if (so == ip_mrouter)
 		ip_mrouter_done();
 	ip_rsvp_force_done(so);
 	if (so == ip_rsvpd)
 		ip_rsvp_done();
+#endif /* MROUTING */
 	in_pcbdetach(inp);
 	return 0;
 }
@@ -663,7 +709,7 @@ rip_disconnect(struct socket *so)
 }
 
 __private_extern__ int
-rip_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
+rip_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in *addr = (struct sockaddr_in *)nam;
@@ -687,7 +733,7 @@ rip_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 }
 
 __private_extern__ int
-rip_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
+rip_connect(struct socket *so, struct sockaddr *nam, __unused  struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in *addr = (struct sockaddr_in *)nam;
@@ -712,8 +758,8 @@ rip_shutdown(struct socket *so)
 }
 
 __private_extern__ int
-rip_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
-	 struct mbuf *control, struct proc *p)
+rip_send(struct socket *so, __unused int flags, struct mbuf *m, struct sockaddr *nam,
+	 __unused struct mbuf *control, __unused struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	register u_long dst;
@@ -749,7 +795,7 @@ rip_unlock(struct socket *so, int refcount, int debug)
 
 	if (refcount) {
 		if (so->so_usecount <= 0)
-			panic("rip_unlock: bad refoucnt so=%x val=%x\n", so, so->so_usecount);
+			panic("rip_unlock: bad refoucnt so=%p val=%x\n", so, so->so_usecount);
 		so->so_usecount--;
 		if (so->so_usecount == 0 && (inp->inp_wantcnt == WNT_STOPUSING)) {
 			/* cleanup after last reference */
@@ -760,7 +806,7 @@ rip_unlock(struct socket *so, int refcount, int debug)
 			return(0);
 		}
 	}
-	so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+	so->unlock_lr[so->next_unlock_lr] = (u_int32_t)lr_saved;
 	so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 	lck_mtx_unlock(so->so_proto->pr_domain->dom_mtx);
 	return(0);
@@ -769,7 +815,8 @@ rip_unlock(struct socket *so, int refcount, int debug)
 static int
 rip_pcblist SYSCTL_HANDLER_ARGS
 {
-	int error, i, n, s;
+#pragma unused(oidp, arg1, arg2)
+	int error, i, n;
 	struct inpcb *inp, **inp_list;
 	inp_gen_t gencnt;
 	struct xinpgen xig;
@@ -874,3 +921,4 @@ struct pr_usrreqs rip_usrreqs = {
 	pru_rcvoob_notsupp, rip_send, pru_sense_null, rip_shutdown,
 	in_setsockaddr, sosend, soreceive, pru_sopoll_notsupp
 };
+/* DSEP Review Done pl-20051213-v02 @3253 */
diff --git a/bsd/netinet/tcp.h b/bsd/netinet/tcp.h
index 6dcd7cd91..3db7ae34e 100644
--- a/bsd/netinet/tcp.h
+++ b/bsd/netinet/tcp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -61,7 +67,7 @@
 #include <sys/_types.h>
 #include <machine/endian.h>
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 typedef	__uint32_t tcp_seq;
 typedef __uint32_t tcp_cc;		/* connection count per rfc1644 */
 
@@ -108,7 +114,7 @@ struct tcphdr {
 #define TCPOPT_WINDOW		3
 #define TCPOLEN_WINDOW		3
 #define TCPOPT_SACK_PERMITTED	4		/* Experimental */
-#define    TCPOLEN_SACK_PERMITTED	2
+#define TCPOLEN_SACK_PERMITTED	2
 #define TCPOPT_SACK		5		/* Experimental */
 #define TCPOLEN_SACK		8		/* len of sack block */
 #define TCPOPT_TIMESTAMP	8
@@ -184,17 +190,17 @@ struct tcphdr {
 #define TCP_MAXHLEN	(0xf<<2)	/* max length of header in bytes */
 #define TCP_MAXOLEN	(TCP_MAXHLEN - sizeof(struct tcphdr))
 					/* max space left for options */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * User-settable options (used with setsockopt).
  */
 #define	TCP_NODELAY	0x01	/* don't delay send to coalesce packets */
-#ifndef _POSIX_C_SOURCE
-#define TCP_MAXSEG		0x02	/* set maximum segment size */
-#define TCP_NOPUSH		0x04	/* don't push last block of write */
-#define TCP_NOOPT		0x08	/* don't use TCP options */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define	TCP_MAXSEG	0x02	/* set maximum segment size */
+#define TCP_NOPUSH	0x04	/* don't push last block of write */
+#define TCP_NOOPT	0x08	/* don't use TCP options */
 #define TCP_KEEPALIVE	0x10	/* idle time used when SO_KEEPALIVE is enabled */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #endif
diff --git a/bsd/netinet/tcp_debug.c b/bsd/netinet/tcp_debug.c
index e001cb8f0..ac173feae 100644
--- a/bsd/netinet/tcp_debug.c
+++ b/bsd/netinet/tcp_debug.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/tcp_debug.h b/bsd/netinet/tcp_debug.h
index 1fbf37aa9..0cfb8d953 100644
--- a/bsd/netinet/tcp_debug.h
+++ b/bsd/netinet/tcp_debug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/tcp_fsm.h b/bsd/netinet/tcp_fsm.h
index 6bbb41490..e9db5611f 100644
--- a/bsd/netinet/tcp_fsm.h
+++ b/bsd/netinet/tcp_fsm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/tcp_input.c b/bsd/netinet/tcp_input.c
index be1e85137..39a5fc252 100644
--- a/bsd/netinet/tcp_input.c
+++ b/bsd/netinet/tcp_input.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995
@@ -54,7 +60,12 @@
  *	@(#)tcp_input.c	8.12 (Berkeley) 5/24/95
  * $FreeBSD: src/sys/netinet/tcp_input.c,v 1.107.2.16 2001/08/22 00:59:12 silby Exp $
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -112,6 +123,10 @@ struct tcphdr tcp_savetcp;
 #include <netkey/key.h>
 #endif /*IPSEC*/
 
+#if CONFIG_MACF_NET || CONFIG_MACF_SOCKET
+#include <security/mac_framework.h>
+#endif /* CONFIG_MACF_NET || CONFIG_MACF_SOCKET */
+
 #include <sys/kdebug.h>
 
 #ifndef __APPLE__
@@ -123,13 +138,11 @@ MALLOC_DEFINE(M_TSEGQ, "tseg_qent", "TCP segment queue entry");
 #define DBG_FNC_TCP_INPUT       NETDBG_CODE(DBG_NETTCP, (3 << 8))
 #define DBG_FNC_TCP_NEWCONN     NETDBG_CODE(DBG_NETTCP, (7 << 8))
 
-static int	tcprexmtthresh = 3;
+static int	tcprexmtthresh = 2;
 tcp_cc	tcp_ccgen;
-extern int apple_hwcksum_rx;
 
 #if IPSEC
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif
 
 struct	tcpstat tcpstat;
@@ -158,7 +171,7 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, drop_synfin, CTLFLAG_RW,
     &drop_synfin, 0, "Drop TCP packets with SYN+FIN set");
 #endif
 
-SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
+SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
     "TCP Segment Reassembly Queue");
 
 __private_extern__ int tcp_reass_maxseg = 0;
@@ -181,8 +194,17 @@ __private_extern__ int slowlink_wsize = 8192;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowlink_wsize, CTLFLAG_RW,
 	&slowlink_wsize, 0, "Maximum advertised window size for slowlink");
 
+static int maxseg_unacked = 8;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, maxseg_unacked, CTLFLAG_RW,
+	&maxseg_unacked, 0, "Maximum number of outstanding segments left unacked");
+
+static int	tcp_do_rfc3465 = 1;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, rfc3465, CTLFLAG_RW,
+	&tcp_do_rfc3465, 0, "");
+extern int tcp_TCPTV_MIN;
 
 u_long tcp_now;
+
 struct inpcbhead tcb;
 #define	tcb6	tcb  /* for KAME src sync over BSD*'s */
 struct inpcbinfo tcbinfo;
@@ -194,6 +216,10 @@ static void	 tcp_pulloutofband(struct socket *,
 static int	 tcp_reass(struct tcpcb *, struct tcphdr *, int *,
 				struct mbuf *);
 static void	tcp_xmit_timer(struct tcpcb *, int);
+static inline unsigned int tcp_maxmtu(struct rtentry *);
+#if INET6
+static inline unsigned int tcp_maxmtu6(struct rtentry *);
+#endif
 
 /* Neighbor Discovery, Neighbor Unreachability Detection Upper layer hint. */
 #if INET6
@@ -210,16 +236,26 @@ do { \
 
 extern u_long	*delack_bitmask;
 
-extern  void    ipfwsyslog( int level, char *format,...);
+extern void	add_to_time_wait(struct tcpcb *);
+extern void postevent(struct socket *, struct sockbuf *, int);
+
+extern  void    ipfwsyslog( int level, const char *format,...);
 extern int ChkAddressOK( __uint32_t dstaddr, __uint32_t srcaddr );
 extern int fw_verbose;
+__private_extern__ int tcp_sockthreshold;
+__private_extern__ int tcp_win_scale;
 
+#if IPFIREWALL
 #define log_in_vain_log( a ) {            \
         if ( (log_in_vain == 3 ) && (fw_verbose == 2)) {        /* Apple logging, log to ipfw.log */ \
                 ipfwsyslog a ;  \
         }                       \
         else log a ;            \
 }
+#else
+#define log_in_vain_log( a ) { log a; }
+#endif
+
 
 /*
  * Indicate whether this ack should be delayed.  
@@ -234,8 +270,9 @@ extern int fw_verbose;
  *        need to ACK with no delay. This helps higher level protocols who won't send
  *        us more data even if the window is open because their last "segment" hasn't been ACKed
  *  - delayed acks are enabled (set to 3, "streaming detection") and
- *  	- if we receive more than 4 full packets per second on this socket, we're streaming acts as "1".
- *  	- if we don't meet that criteria, acts like "2". Allowing faster acking while browsing for example.
+ *  	- if we receive more than "maxseg_unacked"  full packets per second on this socket
+ *		- if we don't have more than "maxseg_unacked" delayed so far
+ *  	- if those criteria aren't met, acts like "2". Allowing faster acking while browsing for example.
  *
  */
 #define DELAY_ACK(tp) \
@@ -243,10 +280,11 @@ extern int fw_verbose;
 	 (((tcp_delack_enabled == 2) && (tp->t_flags & TF_RXWIN0SENT) == 0) && \
 	   ((thflags & TH_PUSH) == 0) && ((tp->t_flags & TF_DELACK) == 0)) || \
 	 (((tcp_delack_enabled == 3) && (tp->t_flags & TF_RXWIN0SENT) == 0) && \
-	   (((tp->t_rcvtime == 0) && (tp->rcv_byps > (4* tp->t_maxseg)))  || (((thflags & TH_PUSH) == 0) && ((tp->t_flags & TF_DELACK) == 0)))))
-
+	   (tp->t_rcvtime == 0) && ((thflags & TH_PUSH) == 0) && \
+	   (((tp->t_unacksegs == 0)) || \
+	   ((tp->rcv_byps > (maxseg_unacked * tp->t_maxseg))  && (tp->t_unacksegs < maxseg_unacked)))))
 
-static int tcpdropdropablreq(struct socket *head);
+static int tcp_dropdropablreq(struct socket *head);
 static void tcp_newreno_partial_ack(struct tcpcb *tp, struct tcphdr *th);
 
 
@@ -284,6 +322,7 @@ tcp_reass(tp, th, tlenp, m)
 		tcp_reass_overflows++;
 		tcpstat.tcps_rcvmemdrop++;
 		m_freem(m);
+		*tlenp = 0;
 		return (0);
 	}
 
@@ -390,7 +429,7 @@ present:
 		if (so->so_state & SS_CANTRCVMORE)
 			m_freem(q->tqe_m);
 		else {
-			if (sbappend(&so->so_rcv, q->tqe_m))
+			if (sbappendstream(&so->so_rcv, q->tqe_m))
 				dowakeup = 1;
 		}
 		FREE(q, M_TSEGQ);
@@ -423,6 +462,29 @@ present:
 
 }
 
+/*
+ * Reduce congestion window.
+ */
+static void
+tcp_reduce_congestion_window(
+	struct tcpcb	*tp)
+{
+	u_int win;
+	
+	win = min(tp->snd_wnd, tp->snd_cwnd) /
+		2 / tp->t_maxseg;
+	if (win < 2)
+		win = 2;
+	tp->snd_ssthresh = win * tp->t_maxseg;
+	ENTER_FASTRECOVERY(tp);
+	tp->snd_recover = tp->snd_max;
+	tp->t_timer[TCPT_REXMT] = 0;
+	tp->t_rtttime = 0;
+	tp->ecn_flags |= TE_SENDCWR;
+	tp->snd_cwnd = tp->snd_ssthresh +
+		 tp->t_maxseg * tcprexmtthresh;
+}
+
 
 /*
  * TCP input routine, follows pages 65-76 of the
@@ -484,13 +546,12 @@ tcp_input(m, off0)
 	int nosock = 0;
 	u_long tiwin;
 	struct tcpopt to;		/* options in this segment */
-	struct rmxp_tao *taop;		/* pointer to our TAO cache entry */
-	struct rmxp_tao	tao_noncached;	/* in case there's no cached entry */
 	struct sockaddr_in *next_hop = NULL;
 #if TCPDEBUG
 	short ostate = 0;
 #endif
 	struct m_tag *fwd_tag;
+	u_char ip_ecn = IPTOS_ECN_NOTECT;
 
 	/* Grab info from PACKET_TAG_IPFORWARD tag prepended to the chain. */
 	fwd_tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPFORWARD, NULL);
@@ -575,7 +636,7 @@ tcp_input(m, off0)
 		     th->th_seq, th->th_ack, th->th_win);
 
 	if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) {
-		if (apple_hwcksum_rx && (m->m_pkthdr.csum_flags & CSUM_TCP_SUM16)) {
+		if (m->m_pkthdr.csum_flags & CSUM_TCP_SUM16) {
 			u_short pseudo;
 			char b[9];
 			*(uint32_t*)&b[0] = *(uint32_t*)&ipov->ih_x1[0];
@@ -619,6 +680,8 @@ tcp_input(m, off0)
 		*(uint32_t*)&ipov->ih_x1[0] = *(uint32_t*)&b[0];
 		*(uint32_t*)&ipov->ih_x1[4] = *(uint32_t*)&b[4];
 		*(uint8_t*)&ipov->ih_x1[8] = *(uint8_t*)&b[8];
+
+		tcp_in_cksum_stats(len);
 	}
 	if (th->th_sum) {
 		tcpstat.tcps_rcvbadsum++;
@@ -628,6 +691,7 @@ tcp_input(m, off0)
 	/* Re-initialization for later version check */
 	ip->ip_v = IPVERSION;
 #endif
+	ip_ecn = (ip->ip_tos & IPTOS_ECN_MASK);
 	}
 
 	/*
@@ -648,17 +712,17 @@ tcp_input(m, off0)
 			th = (struct tcphdr *)((caddr_t)ip6 + off0);
 		} else
 #endif /* INET6 */
-	      {
-		if (m->m_len < sizeof(struct ip) + off) {
-			if ((m = m_pullup(m, sizeof (struct ip) + off)) == 0) {
-				tcpstat.tcps_rcvshort++;
-				return;
+		{
+			if (m->m_len < sizeof(struct ip) + off) {
+				if ((m = m_pullup(m, sizeof (struct ip) + off)) == 0) {
+					tcpstat.tcps_rcvshort++;
+					return;
+				}
+				ip = mtod(m, struct ip *);
+				ipov = (struct ipovly *)ip;
+				th = (struct tcphdr *)((caddr_t)ip + off0);
 			}
-			ip = mtod(m, struct ip *);
-			ipov = (struct ipovly *)ip;
-			th = (struct tcphdr *)((caddr_t)ip + off0);
 		}
-	      }
 		optlen = off - sizeof (struct tcphdr);
 		optp = (u_char *)(th + 1);
 		/* 
@@ -669,10 +733,10 @@ tcp_input(m, off0)
 		 * tcp_dooptions(), etc.
 		 */
 		if ((optlen == TCPOLEN_TSTAMP_APPA ||
-		     (optlen > TCPOLEN_TSTAMP_APPA &&
+			(optlen > TCPOLEN_TSTAMP_APPA &&
 			optp[TCPOLEN_TSTAMP_APPA] == TCPOPT_EOL)) &&
-		     *(u_int32_t *)optp == htonl(TCPOPT_TSTAMP_HDR) &&
-		     (th->th_flags & TH_SYN) == 0) {
+			*(u_int32_t *)optp == htonl(TCPOPT_TSTAMP_HDR) &&
+			(th->th_flags & TH_SYN) == 0) {
 			to.to_flags |= TOF_TS;
 			to.to_tsval = ntohl(*(u_int32_t *)(optp + 4));
 			to.to_tsecr = ntohl(*(u_int32_t *)(optp + 8));
@@ -718,7 +782,7 @@ findpcb:
 #if IPFIREWALL_FORWARD
 	if (next_hop != NULL
 #if INET6
-	    && isipv6 == NULL /* IPv6 support is not yet */
+	    && isipv6 == 0 /* IPv6 support is not yet */
 #endif /* INET6 */
 	    ) {
 		/*
@@ -759,22 +823,18 @@ findpcb:
 
 #if IPSEC
 	if (ipsec_bypass == 0)  {
-		lck_mtx_lock(sadb_mutex);
 #if INET6
 		if (isipv6) {
 		       	if (inp != NULL && ipsec6_in_reject_so(m, inp->inp_socket)) {
-				ipsec6stat.in_polvio++;
-				lck_mtx_unlock(sadb_mutex);
+				IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 				goto dropnosock;
 			}
 		} else
 #endif /* INET6 */
 			if (inp != NULL && ipsec4_in_reject_so(m, inp->inp_socket)) {
-				ipsecstat.in_polvio++;
-				lck_mtx_unlock(sadb_mutex);
+			  IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 				goto dropnosock;
 			}
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /*IPSEC*/
 
@@ -893,6 +953,11 @@ findpcb:
 	else
 		tiwin = th->th_win;
 
+#if CONFIG_MACF_NET
+	if (mac_inpcb_check_deliver(inp, m, AF_INET, SOCK_STREAM))
+		goto drop;
+#endif
+
 	if (so->so_options & (SO_DEBUG|SO_ACCEPTCONN)) {
 #if TCPDEBUG
 		if (so->so_options & SO_DEBUG) {
@@ -981,6 +1046,7 @@ findpcb:
 			}
 #endif
 			if (so->so_filt) {
+#if INET6
 				if (isipv6) {
 					struct sockaddr_in6	*sin6 = (struct sockaddr_in6*)&from;
 					
@@ -990,7 +1056,10 @@ findpcb:
 					sin6->sin6_flowinfo = 0;
 					sin6->sin6_addr = ip6->ip6_src;
 					sin6->sin6_scope_id = 0;
-				} else {
+ 				}
+				else
+#endif
+				{
 					struct sockaddr_in *sin = (struct sockaddr_in*)&from;
 					
 					sin->sin_len = sizeof(*sin);
@@ -1004,7 +1073,7 @@ findpcb:
 			}
 			if (so2 == 0) {
 				tcpstat.tcps_listendrop++;
-				if (tcpdropdropablreq(so)) {
+				if (tcp_dropdropablreq(so)) {
 					if (so->so_filt)
 						so2 = sonewconn(so, 0, (struct sockaddr*)&from);
 					else
@@ -1120,13 +1189,11 @@ findpcb:
 			if (sotoinpcb(oso)->inp_sp)
 			{
 				int error = 0;
-				lck_mtx_lock(sadb_mutex);
 				/* Is it a security hole here to silently fail to copy the policy? */
 				if (inp->inp_sp != NULL)
 					error = ipsec_init_policy(so, &inp->inp_sp);
 				if (error != 0 || ipsec_copy_policy(sotoinpcb(oso)->inp_sp, inp->inp_sp))
 					printf("tcp_input: could not copy policy\n");
-				lck_mtx_unlock(sadb_mutex);
 			}
 #endif
 			tcp_unlock(oso, 1, 0);	/* now drop the reference on the listener */
@@ -1135,10 +1202,16 @@ findpcb:
 			tp->t_flags |= tp0->t_flags & (TF_NOPUSH|TF_NOOPT|TF_NODELAY);
 			tp->t_inpcb->inp_ip_ttl = tp0->t_inpcb->inp_ip_ttl;
 			/* Compute proper scaling value from buffer space */
-			while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
-			   TCP_MAXWIN << tp->request_r_scale <
-			   so->so_rcv.sb_hiwat)
-				tp->request_r_scale++;
+			if (inp->inp_pcbinfo->ipi_count < tcp_sockthreshold) {
+				tp->request_r_scale = max(tcp_win_scale, tp->request_r_scale);
+				so->so_rcv.sb_hiwat = lmin(TCP_MAXWIN << tp->request_r_scale, (sb_max / (MSIZE+MCLBYTES)) * MCLBYTES);  
+			}
+			else {
+				while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
+			   	TCP_MAXWIN << tp->request_r_scale <
+			   	so->so_rcv.sb_hiwat)
+					tp->request_r_scale++;
+			}
 
 			KERNEL_DEBUG(DBG_FNC_TCP_NEWCONN | DBG_FUNC_END,0,0,0,0,0);
 		}
@@ -1169,6 +1242,8 @@ findpcb:
 	 * example interactive connections with many small packets like
 	 * telnet or SSH.
 	 *
+	 * Setting either tcp_minmssoverload or tcp_minmss to "0" disables
+	 * this check.
 	 *
 	 * Account for packet if payload packet, skip over ACK, etc.
 	 *
@@ -1180,6 +1255,8 @@ findpcb:
 		if (tp->rcv_reset > tcp_now) {
 			tp->rcv_pps++;
 			tp->rcv_byps += tlen + off;
+			if (tp->rcv_byps > tp->rcv_maxbyps)
+				tp->rcv_maxbyps = tp->rcv_byps;
 		/*
 		 * Setting either tcp_minmssoverload or tcp_minmss to "0" disables
 		 * the check. 
@@ -1190,7 +1267,7 @@ findpcb:
 					printf("too many small tcp packets from "
 					       "%s:%u, av. %lubyte/packet, "
 					       "dropping connection\n",
-#ifdef INET6
+#if INET6
 						isipv6 ?
 						inet_ntop(AF_INET6, &inp->in6p_faddr, ipstrbuf,
 								  sizeof(ipstrbuf)) :
@@ -1205,12 +1282,55 @@ findpcb:
 				}
 			}
 		} else {
-			tp->rcv_reset = tcp_now + PR_SLOWHZ;
+			tp->rcv_reset = tcp_now + TCP_RETRANSHZ;
 			tp->rcv_pps = 1;
 			tp->rcv_byps = tlen + off;
 		}
 	}
 	
+#if TRAFFIC_MGT
+	if (so->so_traffic_mgt_flags & TRAFFIC_MGT_SO_BACKGROUND) {
+		tcpstat.tcps_bg_rcvtotal++;
+
+		 /* Take snapshots of pkts recv;
+		  * tcpcb should have been initialized to 0 when allocated, 
+		  * so if 0 then this is the first time we're doing this
+		  */
+		if (!tp->tot_recv_snapshot) {
+			 tp->tot_recv_snapshot = tcpstat.tcps_rcvtotal;
+		 }
+		if (!tp->bg_recv_snapshot) {
+			 tp->bg_recv_snapshot = tcpstat.tcps_bg_rcvtotal;
+		 }
+	}
+#endif /* TRAFFIC_MGT */
+
+	/*
+	   Explicit Congestion Notification - Flag that we need to send ECT if
+		+ The IP Congestion experienced flag was set.
+		+ Socket is in established state
+		+ We negotiated ECN in the TCP setup
+		+ This isn't a pure ack (tlen > 0)
+		+ The data is in the valid window
+	
+		TE_SENDECE will be cleared when we receive a packet with TH_CWR set.
+	 */
+	if (ip_ecn == IPTOS_ECN_CE && tp->t_state == TCPS_ESTABLISHED &&
+		(tp->ecn_flags & (TE_SETUPSENT | TE_SETUPRECEIVED)) ==
+		 (TE_SETUPSENT | TE_SETUPRECEIVED) && tlen > 0 &&
+		SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
+		SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) {
+		tp->ecn_flags |= TE_SENDECE;
+	}
+	
+	/*
+	   Clear TE_SENDECE if TH_CWR is set. This is harmless, so we don't
+	   bother doing extensive checks for state and whatnot.
+	 */
+	if ((thflags & TH_CWR) == TH_CWR) {
+		tp->ecn_flags &= ~TE_SENDECE;
+	}
+	
 	/*
 	 * Segment received on connection.
 	 * Reset idle time and keep-alive timer.
@@ -1226,11 +1346,12 @@ findpcb:
 	if (tp->t_state != TCPS_LISTEN && optp)
 		tcp_dooptions(tp, optp, optlen, th, &to);
 
-
 	if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
 		if (to.to_flags & TOF_SCALE) {
 			tp->t_flags |= TF_RCVD_SCALE;
 			tp->requested_s_scale = to.to_requested_s_scale;
+			tp->snd_wnd = th->th_win << tp->snd_scale;
+			tiwin = tp->snd_wnd;
 		}
 		if (to.to_flags & TOF_TS) {
 			tp->t_flags |= TF_RCVD_TSTMP;
@@ -1265,7 +1386,7 @@ findpcb:
 	 * be TH_NEEDSYN.
 	 */
 	if (tp->t_state == TCPS_ESTABLISHED &&
-	    (thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK)) == TH_ACK &&
+	    (thflags & (TH_SYN|TH_FIN|TH_RST|TH_URG|TH_ACK|TH_ECE)) == TH_ACK &&
 	    ((tp->t_flags & (TF_NEEDSYN|TF_NEEDFIN)) == 0) &&
 	    ((to.to_flags & TOF_TS) == 0 ||
 	     TSTMP_GEQ(to.to_tsval, tp->ts_recent)) &&
@@ -1288,7 +1409,7 @@ findpcb:
 		if (tlen == 0) {
 			if (SEQ_GT(th->th_ack, tp->snd_una) &&
 			    SEQ_LEQ(th->th_ack, tp->snd_max) &&
-			    tp->snd_cwnd >= tp->snd_wnd &&
+			    tp->snd_cwnd >= tp->snd_ssthresh &&
 			    ((!tcp_do_newreno && !tp->sack_enable &&
 			      tp->t_dupacks < tcprexmtthresh) ||
 			     ((tcp_do_newreno || tp->sack_enable) &&
@@ -1302,7 +1423,8 @@ findpcb:
 				 * "bad retransmit" recovery
 				 */
 				if (tp->t_rxtshift == 1 &&
-				    tcp_now < tp->t_badrxtwin) { 
+				    tcp_now < tp->t_badrxtwin) {
+					++tcpstat.tcps_sndrexmitbad;
 					tp->snd_cwnd = tp->snd_cwnd_prev;
 					tp->snd_ssthresh =
 					    tp->snd_ssthresh_prev;
@@ -1312,15 +1434,41 @@ findpcb:
 					tp->snd_nxt = tp->snd_max;
 					tp->t_badrxtwin = 0;
 				}
-				if (((to.to_flags & TOF_TS) != 0) && (to.to_tsecr != 0)) /* Makes sure we already have a TS */
+				/*
+				 * Recalculate the transmit timer / rtt.
+				 *
+				 * Some boxes send broken timestamp replies
+				 * during the SYN+ACK phase, ignore
+				 * timestamps of 0 or we could calculate a
+				 * huge RTT and blow up the retransmit timer.
+				 */
+				if (((to.to_flags & TOF_TS) != 0) && (to.to_tsecr != 0)) { /* Makes sure we already have a TS */
+					if (!tp->t_rttlow ||
+					    tp->t_rttlow > tcp_now - to.to_tsecr)
+						tp->t_rttlow = tcp_now - to.to_tsecr;
 					tcp_xmit_timer(tp,
-					    tcp_now - to.to_tsecr + 1);
-				else if (tp->t_rtttime &&
-					    SEQ_GT(th->th_ack, tp->t_rtseq))
+					    tcp_now - to.to_tsecr);
+				} else if (tp->t_rtttime &&
+					    SEQ_GT(th->th_ack, tp->t_rtseq)) {
+					if (!tp->t_rttlow ||
+					    tp->t_rttlow > tcp_now - tp->t_rtttime)
+						tp->t_rttlow = tcp_now - tp->t_rtttime;
 					tcp_xmit_timer(tp, tp->t_rtttime);
+				}
 				acked = th->th_ack - tp->snd_una;
 				tcpstat.tcps_rcvackpack++;
 				tcpstat.tcps_rcvackbyte += acked;
+				/*
+				 * Grow the congestion window, if the
+				 * connection is cwnd bound.
+				 */
+			    	if (tp->snd_cwnd < tp->snd_wnd) {
+					tp->t_bytes_acked += acked;
+					if (tp->t_bytes_acked > tp->snd_cwnd) {
+						tp->t_bytes_acked -= tp->snd_cwnd;
+						tp->snd_cwnd += tp->t_maxseg;
+					}
+				}
 				sbdrop(&so->so_snd, acked);
 				if (SEQ_GT(tp->snd_una, tp->snd_recover) &&
 				    SEQ_LEQ(th->th_ack, tp->snd_recover))
@@ -1350,15 +1498,17 @@ findpcb:
 					tp->t_timer[TCPT_REXMT] = tp->t_rxtcur;
 
 				sowwakeup(so); /* has to be done with socket lock held */
-				if ((so->so_snd.sb_cc) || (tp->t_flags & TF_ACKNOW))
+				if ((so->so_snd.sb_cc) || (tp->t_flags & TF_ACKNOW)) {
+			    		tp->t_unacksegs = 0;
 					(void) tcp_output(tp);
+				}
 				tcp_unlock(so, 1, 0);
 				KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0);
 				return;
 			}
 		} else if (th->th_ack == tp->snd_una &&
 		    LIST_EMPTY(&tp->t_segq) &&
-		    tlen <= sbspace(&so->so_rcv)) {
+		    tlen <= tcp_sbspace(tp)) {
 			/*
 			 * this is a pure, in-sequence data packet
 			 * with nothing on the reassembly queue and
@@ -1386,7 +1536,7 @@ findpcb:
 			 * Add data to socket buffer.
 			 */
 			m_adj(m, drop_hdrlen);	/* delayed header drop */
-			if (sbappend(&so->so_rcv, m))
+			if (sbappendstream(&so->so_rcv, m))
 				sorwakeup(so);
 #if INET6
 			if (isipv6) {
@@ -1402,8 +1552,10 @@ findpcb:
 			     		th->th_seq, th->th_ack, th->th_win); 
 			}
 			if (DELAY_ACK(tp))  {
-			    tp->t_flags |= TF_DELACK;
+			    	tp->t_flags |= TF_DELACK;
+			    	tp->t_unacksegs++;
 			} else {
+			    	tp->t_unacksegs = 0;
 				tp->t_flags |= TF_ACKNOW;
 				tcp_output(tp);
 			}
@@ -1424,7 +1576,8 @@ findpcb:
 #endif
 	{ int win;
 
-	win = sbspace(&so->so_rcv);
+	win = tcp_sbspace(tp);
+
 	if (win < 0)
 		win = 0;
 	else {	/* clip rcv window to 4K for modems */
@@ -1524,7 +1677,7 @@ findpcb:
 		} else
 #endif
 	    {
-#if 1
+#if 0
 			lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
 #endif
 			MALLOC(sin, struct sockaddr_in *, sizeof *sin, M_SONAME,
@@ -1572,10 +1725,15 @@ findpcb:
 		 */
 		tp->snd_wnd = tiwin;	/* initial send-window */
 		tp->t_flags |= TF_ACKNOW;
+		tp->t_unacksegs = 0;
 		tp->t_state = TCPS_SYN_RECEIVED;
 		tp->t_timer[TCPT_KEEP] = tcp_keepinit;
 		dropsocket = 0;		/* committed to socket */
 		tcpstat.tcps_accepts++;
+		if ((thflags & (TH_ECE | TH_CWR)) == (TH_ECE | TH_CWR)) {
+			/* ECN-setup SYN */
+			tp->ecn_flags |= (TE_SETUPRECEIVED | TE_SENDIPECT);
+		}
 		goto trimthenstep6;
 		}
 
@@ -1612,10 +1770,10 @@ findpcb:
 			goto dropwithreset;
 		}
 		if (thflags & TH_RST) {
-			if (thflags & TH_ACK) {
+			if ((thflags & TH_ACK) != 0) {
 				tp = tcp_drop(tp, ECONNREFUSED);
 				postevent(so, 0, EV_RESET);
-		  }
+			}
 			goto drop;
 		}
 		if ((thflags & TH_SYN) == 0)
@@ -1626,7 +1784,22 @@ findpcb:
 		tcp_rcvseqinit(tp);
 		if (thflags & TH_ACK) {
 			tcpstat.tcps_connects++;
+			
+			if ((thflags & (TH_ECE | TH_CWR)) == (TH_ECE)) {
+				/* ECN-setup SYN-ACK */
+				tp->ecn_flags |= TE_SETUPRECEIVED;
+			}
+			else {
+				/* non-ECN-setup SYN-ACK */
+				tp->ecn_flags &= ~TE_SENDIPECT;
+			}
+			
 			soisconnected(so);
+#if CONFIG_MACF_NET && CONFIG_MACF_SOCKET
+			/* XXXMAC: recursive lock: SOCK_LOCK(so); */
+			mac_socketpeer_label_associate_mbuf(m, so);
+			/* XXXMAC: SOCK_UNLOCK(so); */
+#endif
 			/* Do window scaling on this connection? */
 			if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) ==
 				(TF_RCVD_SCALE|TF_REQ_SCALE)) {
@@ -1641,9 +1814,11 @@ findpcb:
 			 */
 			if (DELAY_ACK(tp) && tlen != 0) {
 				tp->t_flags |= TF_DELACK;
+			    	tp->t_unacksegs++;
 			}
 			else {
 				tp->t_flags |= TF_ACKNOW;
+			    	tp->t_unacksegs = 0;
 			}
 			/*
 			 * Received <SYN,ACK> in SYN_SENT[*] state.
@@ -1786,16 +1961,17 @@ trimthenstep6:
 	 *	Drop the segment - see Stevens, vol. 2, p. 964 and
 	 *      RFC 1337.
 	 *
-	 *      Radar 4776325: Allows for the case where we ACKed the FIN but
-	 *      	       there is already a RST in flight from the peer.
-	 *      	       In that case, accept the RST for non-established
-	 *      	       state if it's one off from last_ack_sent.
+	 *      Radar 4803931: Allows for the case where we ACKed the FIN but
+	 *                     there is already a RST in flight from the peer.
+	 *                     In that case, accept the RST for non-established
+	 *                     state if it's one off from last_ack_sent.
+
 	 */
 	if (thflags & TH_RST) {
 		if ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
 		    SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) ||
 		    (tp->rcv_wnd == 0 && 
-		     	(tp->last_ack_sent == th->th_seq)) || ((tp->last_ack_sent - 1) == th->th_seq)) {
+		     	((tp->last_ack_sent == th->th_seq) || ((tp->last_ack_sent -1) == th->th_seq)))) {
 			switch (tp->t_state) {
 
 			case TCPS_SYN_RECEIVED:
@@ -1804,6 +1980,7 @@ trimthenstep6:
 
 			case TCPS_ESTABLISHED:
 				if (tp->last_ack_sent != th->th_seq) {
+					tcpstat.tcps_badrst++;
 					goto drop;
 				}
 			case TCPS_FIN_WAIT_1:
@@ -1832,9 +2009,10 @@ trimthenstep6:
 		goto drop;
 	}
 
-#if 1
+#if 0
 	lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
 #endif
+
 	/*
 	 * RFC 1323 PAWS: If we have a timestamp reply on this segment
 	 * and it's less than ts_recent, drop it.
@@ -1906,6 +2084,7 @@ trimthenstep6:
 			 * But keep on processing for RST or ACK.
 			 */
 			tp->t_flags |= TF_ACKNOW;
+			tp->t_unacksegs = 0;
 			todrop = tlen;
 			tcpstat.tcps_rcvduppack++;
 			tcpstat.tcps_rcvdupbyte += todrop;
@@ -1968,6 +2147,7 @@ trimthenstep6:
 			 */
 			if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) {
 				tp->t_flags |= TF_ACKNOW;
+			    	tp->t_unacksegs = 0;
 				tcpstat.tcps_rcvwinprobe++;
 			} else
 				goto dropafterack;
@@ -2024,6 +2204,8 @@ trimthenstep6:
 		if (tp->t_state == TCPS_SYN_RECEIVED ||
 		    (tp->t_flags & TF_NEEDSYN))
 			goto step6;
+		else if (tp->t_flags & TF_ACKNOW)
+			goto dropafterack;
 		else
 			goto drop;
 	}
@@ -2067,7 +2249,7 @@ trimthenstep6:
 		 * later; if not, do so now to pass queued data to user.
 		 */
 		if (tlen == 0 && (thflags & TH_FIN) == 0)
-			(void) tcp_reass(tp, (struct tcphdr *)0, 0,
+			(void) tcp_reass(tp, (struct tcphdr *)0, &tlen,
 			    (struct mbuf *)0);
 		tp->snd_wl1 = th->th_seq - 1;
 		/* FALLTHROUGH */
@@ -2127,7 +2309,7 @@ trimthenstep6:
 				else if (++tp->t_dupacks > tcprexmtthresh ||
 					 ((tcp_do_newreno || tp->sack_enable) &&
 					  IN_FASTRECOVERY(tp))) {
-                                        if (tp->sack_enable && IN_FASTRECOVERY(tp)) {
+					if (tp->sack_enable && IN_FASTRECOVERY(tp)) {
 						int awnd;
 						
 						/*
@@ -2145,6 +2327,7 @@ trimthenstep6:
 						}
 					} else
 						tp->snd_cwnd += tp->t_maxseg;
+					tp->t_unacksegs = 0;
 					(void) tcp_output(tp);
 					goto drop;
 				} else if (tp->t_dupacks == tcprexmtthresh) {
@@ -2179,15 +2362,18 @@ trimthenstep6:
 					tp->snd_recover = tp->snd_max;
 					tp->t_timer[TCPT_REXMT] = 0;
 					tp->t_rtttime = 0;
+					tp->ecn_flags |= TE_SENDCWR;
 					if (tp->sack_enable) {
 						tcpstat.tcps_sack_recovery_episode++;
 						tp->sack_newdata = tp->snd_nxt;
 						tp->snd_cwnd = tp->t_maxseg;
+						tp->t_unacksegs = 0;
 						(void) tcp_output(tp);
 						goto drop;
 					}
 					tp->snd_nxt = th->th_ack;
 					tp->snd_cwnd = tp->t_maxseg;
+					tp->t_unacksegs = 0;
 					(void) tcp_output(tp);
 					tp->snd_cwnd = tp->snd_ssthresh +
 					     tp->t_maxseg * tp->t_dupacks;
@@ -2236,6 +2422,7 @@ trimthenstep6:
 				tp->snd_cwnd = tp->snd_ssthresh;
 		}
 		tp->t_dupacks = 0;
+		tp->t_bytes_acked = 0;
 		/*
 		 * If we reach this point, ACK is not a duplicate,
 		 *     i.e., it ACKs something we sent.
@@ -2271,6 +2458,7 @@ process_ACK:
 		 * we left off.
 		 */
 		if (tp->t_rxtshift == 1 && tcp_now < tp->t_badrxtwin) {
+			++tcpstat.tcps_sndrexmitbad;
 			tp->snd_cwnd = tp->snd_cwnd_prev;
 			tp->snd_ssthresh = tp->snd_ssthresh_prev;
 			tp->snd_recover = tp->snd_recover_prev;
@@ -2289,11 +2477,21 @@ process_ACK:
 		 * timer backoff (cf., Phil Karn's retransmit alg.).
 		 * Recompute the initial retransmit timer.
 		 * Also makes sure we have a valid time stamp in hand
+		 *
+		 * Some boxes send broken timestamp replies
+		 * during the SYN+ACK phase, ignore
+		 * timestamps of 0 or we could calculate a
+		 * huge RTT and blow up the retransmit timer.
 		 */
-		if (((to.to_flags & TOF_TS) != 0) && (to.to_tsecr != 0))
-			tcp_xmit_timer(tp, tcp_now - to.to_tsecr + 1);
-		else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq))
+		if (((to.to_flags & TOF_TS) != 0) && (to.to_tsecr != 0)) {
+			if (!tp->t_rttlow || tp->t_rttlow > tcp_now - to.to_tsecr)
+				tp->t_rttlow = tcp_now - to.to_tsecr;
+			tcp_xmit_timer(tp, tcp_now - to.to_tsecr);
+		} else if (tp->t_rtttime && SEQ_GT(th->th_ack, tp->t_rtseq)) {
+			if (!tp->t_rttlow || tp->t_rttlow > tcp_now - tp->t_rtttime)
+				tp->t_rttlow = tcp_now - tp->t_rtttime;
 			tcp_xmit_timer(tp, tp->t_rtttime);
+		}
 
 		/*
 		 * If all outstanding data is acked, stop retransmit
@@ -2316,17 +2514,58 @@ process_ACK:
 
 		/*
 		 * When new data is acked, open the congestion window.
-		 * If the window gives us less than ssthresh packets
-		 * in flight, open exponentially (maxseg per packet).
-		 * Otherwise open linearly: maxseg per window
-		 * (maxseg^2 / cwnd per packet).
 		 */
-		if ((!tcp_do_newreno && !tp->sack_enable) ||
+		if ((thflags & TH_ECE) != 0 &&
+			(tp->ecn_flags & TE_SETUPSENT) != 0) {
+			/*
+			 * Reduce the congestion window if we haven't done so.
+			 */
+			if (!(tp->sack_enable && IN_FASTRECOVERY(tp)) &&
+				!(tcp_do_newreno && SEQ_LEQ(th->th_ack, tp->snd_recover))) {
+				tcp_reduce_congestion_window(tp);
+			}
+		} else if ((!tcp_do_newreno && !tp->sack_enable) ||
 		    !IN_FASTRECOVERY(tp)) {
+			/*
+			 * RFC 3465 - Appropriate Byte Counting.
+			 *
+			 * If the window is currently less than ssthresh,
+			 * open the window by the number of bytes ACKed by
+			 * the last ACK, however clamp the window increase
+			 * to an upper limit "L".
+			 *
+			 * In congestion avoidance phase, open the window by
+			 * one segment each time "bytes_acked" grows to be
+			 * greater than or equal to the congestion window.
+			 */
+
 			register u_int cw = tp->snd_cwnd;
 			register u_int incr = tp->t_maxseg;
-			if (cw > tp->snd_ssthresh)
-				incr = incr * incr / cw;
+
+			if (cw >= tp->snd_ssthresh) {
+				tp->t_bytes_acked += acked;
+				if (tp->t_bytes_acked >= cw) {
+					/* Time to increase the window. */
+					tp->t_bytes_acked -= cw;
+				} else {
+					/* No need to increase yet. */
+					incr = 0;
+				}
+			} else {
+				/*
+				 * If the user explicitly enables RFC3465
+				 * use 2*SMSS for the "L" param.  Otherwise
+				 * use the more conservative 1*SMSS.
+				 *
+				 * (See RFC 3465 2.3 Choosing the Limit)
+				 */
+				u_int abc_lim;
+
+				abc_lim = (tcp_do_rfc3465 == 0) ?
+				    incr : incr * 2;
+				incr = min(acked, abc_lim);
+			}
+
 			tp->snd_cwnd = min(cw+incr, TCP_MAXWIN<<tp->snd_scale);
 		}
 		if (acked > so->so_snd.sb_cc) {
@@ -2376,8 +2615,8 @@ process_ACK:
 				if (so->so_state & SS_CANTRCVMORE) {
 					soisdisconnected(so);
 					tp->t_timer[TCPT_2MSL] = tcp_maxidle;
+					add_to_time_wait(tp);
 				}
-				add_to_time_wait(tp);
 				tp->t_state = TCPS_FIN_WAIT_2;
 				goto drop;
 			}
@@ -2395,7 +2634,7 @@ process_ACK:
 				tcp_canceltimers(tp);
 				/* Shorten TIME_WAIT [RFC-1644, p.28] */
 				if (tp->cc_recv != 0 &&
-				    tp->t_starttime < tcp_msl)
+				    tp->t_starttime < (u_long)tcp_msl)
 					tp->t_timer[TCPT_2MSL] =
 					    tp->t_rxtcur * TCPTV_TWTRUNC;
 				else
@@ -2545,8 +2784,10 @@ dodata:							/* XXX */
 		    TCPS_HAVEESTABLISHED(tp->t_state)) {
 			if (DELAY_ACK(tp) && ((tp->t_flags & TF_ACKNOW) == 0)) {
 				tp->t_flags |= TF_DELACK;
+			    	tp->t_unacksegs++;
 			}         
 			else {
+			    	tp->t_unacksegs = 0;
 				tp->t_flags |= TF_ACKNOW;
 			}
 			tp->rcv_nxt += tlen;
@@ -2554,11 +2795,12 @@ dodata:							/* XXX */
 			tcpstat.tcps_rcvpack++;
 			tcpstat.tcps_rcvbyte += tlen;
 			ND6_HINT(tp);
-			if (sbappend(&so->so_rcv, m))
+			if (sbappendstream(&so->so_rcv, m))
 				sorwakeup(so);
 		} else {
 			thflags = tcp_reass(tp, th, &tlen, m);
 			tp->t_flags |= TF_ACKNOW;
+			tp->t_unacksegs = 0;
 		}
 
 		if (tlen > 0 && tp->sack_enable)
@@ -2586,7 +2828,9 @@ dodata:							/* XXX */
 		 * our window, in order to estimate the sender's
 		 * buffer size.
 		 */
-		len = so->so_rcv.sb_hiwat - (tp->rcv_adv - tp->rcv_nxt);
+		len = (u_int)(so->so_rcv.sb_hiwat - (tp->rcv_adv - tp->rcv_nxt));
+		if (len > so->so_rcv.sb_maxused)
+			so->so_rcv.sb_maxused = len;
 	} else {
 		m_freem(m);
 		thflags &= ~TH_FIN;
@@ -2611,9 +2855,11 @@ dodata:							/* XXX */
 			 */
 			if (DELAY_ACK(tp) && (tp->t_flags & TF_NEEDSYN)) {
 				tp->t_flags |= TF_DELACK;
+			    	tp->t_unacksegs++;
 			}
 			else {
 				tp->t_flags |= TF_ACKNOW;
+			    	tp->t_unacksegs = 0;
 			}
 			tp->rcv_nxt++;
 		}
@@ -2647,11 +2893,12 @@ dodata:							/* XXX */
 			tcp_canceltimers(tp);
 			/* Shorten TIME_WAIT [RFC-1644, p.28] */
 			if (tp->cc_recv != 0 &&
-			    tp->t_starttime < tcp_msl) {
+			    tp->t_starttime < (u_long)tcp_msl) {
 				tp->t_timer[TCPT_2MSL] =
 				    tp->t_rxtcur * TCPTV_TWTRUNC;
 				/* For transaction client, force ACK now. */
 				tp->t_flags |= TF_ACKNOW;
+			    	tp->t_unacksegs = 0;
 			}
 			else
 				tp->t_timer[TCPT_2MSL] = 2 * tcp_msl;
@@ -2678,8 +2925,10 @@ dodata:							/* XXX */
 	/*
 	 * Return any desired output.
 	 */
-	if (needoutput || (tp->t_flags & TF_ACKNOW))
+	if (needoutput || (tp->t_flags & TF_ACKNOW)) {
+		tp->t_unacksegs = 0;
 		(void) tcp_output(tp);
+	}
 	tcp_unlock(so, 1, 0);
 	KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0);
 	return;
@@ -2713,6 +2962,7 @@ dropafterack:
 #endif
 	m_freem(m);
 	tp->t_flags |= TF_ACKNOW;
+    	tp->t_unacksegs = 0;
 	(void) tcp_output(tp);
 	tcp_unlock(so, 1, 0);
 	KERNEL_DEBUG(DBG_FNC_TCP_INPUT | DBG_FUNC_END,0,0,0,0,0);
@@ -2757,13 +3007,13 @@ dropwithreset:
 	if (thflags & TH_ACK)
 		/* mtod() below is safe as long as hdr dropping is delayed */
 		tcp_respond(tp, mtod(m, void *), th, m, (tcp_seq)0, th->th_ack,
-			    TH_RST);
+			    TH_RST, m->m_pkthdr.rcvif);
 	else {
 		if (thflags & TH_SYN)
 			tlen++;
 		/* mtod() below is safe as long as hdr dropping is delayed */
 		tcp_respond(tp, mtod(m, void *), th, m, th->th_seq+tlen,
-			    (tcp_seq)0, TH_RST|TH_ACK);
+			    (tcp_seq)0, TH_RST|TH_ACK, m->m_pkthdr.rcvif);
 	}
 	/* destroy temporarily created socket */
 	if (dropsocket) {
@@ -2969,6 +3219,8 @@ tcp_xmit_timer(tp, rtt)
 		delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
 		if ((tp->t_rttvar += delta) <= 0)
 			tp->t_rttvar = 1;
+		if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
+		    tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
 	} else {
 		/*
 		 * No rtt measurement yet - use the unsmoothed rtt.
@@ -2977,6 +3229,7 @@ tcp_xmit_timer(tp, rtt)
 		 */
 		tp->t_srtt = rtt << TCP_RTT_SHIFT;
 		tp->t_rttvar = rtt << (TCP_RTTVAR_SHIFT - 1);
+		tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
 	}
 	tp->t_rtttime = 0;
 	tp->t_rxtshift = 0;
@@ -3005,6 +3258,34 @@ tcp_xmit_timer(tp, rtt)
 	tp->t_softerror = 0;
 }
 
+static inline unsigned int
+tcp_maxmtu(struct rtentry *rt)
+{
+	unsigned int maxmtu;
+
+	if (rt->rt_rmx.rmx_mtu == 0)
+		maxmtu = rt->rt_ifp->if_mtu;
+	else
+		maxmtu = MIN(rt->rt_rmx.rmx_mtu, rt->rt_ifp->if_mtu);
+
+	return (maxmtu);
+}
+
+#if INET6
+static inline unsigned int
+tcp_maxmtu6(struct rtentry *rt)
+{
+	unsigned int maxmtu;
+
+	if (rt->rt_rmx.rmx_mtu == 0)
+		maxmtu = IN6_LINKMTU(rt->rt_ifp);
+	else
+		maxmtu = MIN(rt->rt_rmx.rmx_mtu, IN6_LINKMTU(rt->rt_ifp));
+
+	return (maxmtu);
+}
+#endif
+
 /*
  * Determine a reasonable value for maxseg size.
  * If the route is known, check route for mtu.
@@ -3029,9 +3310,6 @@ tcp_xmit_timer(tp, rtt)
  * NOTE that this routine is only called when we process an incoming
  * segment, for outgoing segments only tcp_mssopt is called.
  *
- * In case of T/TCP, we call this routine during implicit connection
- * setup as well (offer = -1), to initialize maxseg from the cached
- * MSS of our peer.
  */
 void
 tcp_mss(tp, offer)
@@ -3046,6 +3324,8 @@ tcp_mss(tp, offer)
 	struct socket *so;
 	struct rmxp_tao *taop;
 	int origoffer = offer;
+	u_long sb_max_corrected;
+	int isnetlocal = 0;
 #if INET6
 	int isipv6;
 	int min_protoh;
@@ -3059,18 +3339,28 @@ tcp_mss(tp, offer)
 #else
 #define min_protoh  (sizeof (struct tcpiphdr))
 #endif
+	lck_mtx_lock(rt_mtx);
 #if INET6
-	if (isipv6)
+	if (isipv6) {
 		rt = tcp_rtlookup6(inp);
+		if (rt && (IN6_IS_ADDR_LOOPBACK(&inp->in6p_faddr) || IN6_IS_ADDR_LINKLOCAL(&inp->in6p_faddr) || rt->rt_gateway->sa_family == AF_LINK))
+			         isnetlocal = TRUE;
+	}
 	else
 #endif /* INET6 */
-	rt = tcp_rtlookup(inp);
+	{
+		rt = tcp_rtlookup(inp);
+		if (rt && (rt->rt_gateway->sa_family == AF_LINK ||
+			rt->rt_ifp->if_flags & IFF_LOOPBACK)) 
+		         isnetlocal = TRUE;
+	}
 	if (rt == NULL) {
 		tp->t_maxopd = tp->t_maxseg =
 #if INET6
 		isipv6 ? tcp_v6mssdflt :
 #endif /* INET6 */
 		tcp_mssdflt;
+		lck_mtx_unlock(rt_mtx);
 		return;
 	}
 	ifp = rt->rt_ifp;
@@ -3130,12 +3420,14 @@ tcp_mss(tp, offer)
 		 * is also a minimum value; this is subject to time.
 		 */
 		if (rt->rt_rmx.rmx_locks & RTV_RTT)
-			tp->t_rttmin = rtt / (RTM_RTTUNIT / PR_SLOWHZ);
-		tp->t_srtt = rtt / (RTM_RTTUNIT / (PR_SLOWHZ * TCP_RTT_SCALE));
+			tp->t_rttmin = rtt / (RTM_RTTUNIT / TCP_RETRANSHZ);
+		else
+			tp->t_rttmin = isnetlocal ? tcp_TCPTV_MIN : TCP_RETRANSHZ;
+		tp->t_srtt = rtt / (RTM_RTTUNIT / (TCP_RETRANSHZ * TCP_RTT_SCALE));
 		tcpstat.tcps_usedrtt++;
 		if (rt->rt_rmx.rmx_rttvar) {
 			tp->t_rttvar = rt->rt_rmx.rmx_rttvar /
-			    (RTM_RTTUNIT / (PR_SLOWHZ * TCP_RTTVAR_SCALE));
+			    (RTM_RTTUNIT / (TCP_RETRANSHZ * TCP_RTTVAR_SCALE));
 			tcpstat.tcps_usedrttvar++;
 		} else {
 			/* default variation is +- 1 rtt */
@@ -3146,32 +3438,27 @@ tcp_mss(tp, offer)
 			      ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,
 			      tp->t_rttmin, TCPTV_REXMTMAX);
 	}
-	/*
-	 * if there's an mtu associated with the route, use it
-	 * else, use the link mtu.
-	 */
-	if (rt->rt_rmx.rmx_mtu)
-		mss = rt->rt_rmx.rmx_mtu - min_protoh;
 	else
-	{
-		mss =
-#if INET6
-			(isipv6 ? nd_ifinfo[rt->rt_ifp->if_index].linkmtu :
-#endif
-			 ifp->if_mtu
+		tp->t_rttmin = isnetlocal ? tcp_TCPTV_MIN : TCP_RETRANSHZ;
+
 #if INET6
-			 )
+	mss = (isipv6 ? tcp_maxmtu6(rt) : tcp_maxmtu(rt));
+#else
+	mss = tcp_maxmtu(rt);
 #endif
-			- min_protoh;
+	mss -= min_protoh;
+
+	if (rt->rt_rmx.rmx_mtu == 0) {
 #if INET6
 		if (isipv6) {
-			if (!in6_localaddr(&inp->in6p_faddr))
+			if (!isnetlocal)
 				mss = min(mss, tcp_v6mssdflt);
 		} else
 #endif /* INET6 */
-		if (!in_localaddr(inp->inp_faddr))
+		if (!isnetlocal)
 			mss = min(mss, tcp_mssdflt);
 	}
+
 	mss = min(mss, offer);
 	/*
 	 * maxopd stores the maximum length of data AND options
@@ -3192,6 +3479,12 @@ tcp_mss(tp, offer)
 		mss -= TCPOLEN_TSTAMP_APPA;
 	tp->t_maxseg = mss;
 
+	/*
+	 * Calculate corrected value for sb_max; ensure to upgrade the
+	 * numerator for large sb_max values else it will overflow.
+	 */
+	sb_max_corrected = (sb_max * (u_int64_t)MCLBYTES) / (MSIZE + MCLBYTES);
+
 	/*
 	 * If there's a pipesize (ie loopback), change the socket
 	 * buffer to that size only if it's bigger than the current
@@ -3207,9 +3500,9 @@ tcp_mss(tp, offer)
 	if (bufsize < mss)
 		mss = bufsize;
 	else {
-		bufsize = roundup(bufsize, mss);
-		if (bufsize > sb_max)
-			bufsize = sb_max;
+		bufsize = (((bufsize + (u_int64_t)mss - 1) / (u_int64_t)mss) * (u_int64_t)mss);
+		if (bufsize > sb_max_corrected)
+			bufsize = sb_max_corrected;
 		(void)sbreserve(&so->so_snd, bufsize);
 	}
 	tp->t_maxseg = mss;
@@ -3220,9 +3513,9 @@ tcp_mss(tp, offer)
 #endif
 		bufsize = so->so_rcv.sb_hiwat;
 	if (bufsize > mss) {
-		bufsize = roundup(bufsize, mss);
-		if (bufsize > sb_max)
-			bufsize = sb_max;
+		bufsize = (((bufsize + (u_int64_t)mss - 1) / (u_int64_t)mss) * (u_int64_t)mss);
+		if (bufsize > sb_max_corrected)
+			bufsize = sb_max_corrected;
 		(void)sbreserve(&so->so_rcv, bufsize);
 	}
 
@@ -3230,16 +3523,7 @@ tcp_mss(tp, offer)
 	 * Set the slow-start flight size depending on whether this
 	 * is a local network or not.
 	 */
-	if (
-#if INET6
-	    (isipv6 && in6_localaddr(&inp->in6p_faddr)) ||
-	    (!isipv6 &&
-#endif
-	     in_localaddr(inp->inp_faddr)
-#if INET6
-	     )
-#endif
-	    )
+	if (isnetlocal)
 		tp->snd_cwnd = mss * ss_fltsz_local;
 	else 
 		tp->snd_cwnd = mss * ss_fltsz;
@@ -3254,6 +3538,7 @@ tcp_mss(tp, offer)
 		tp->snd_ssthresh = max(2 * mss, rt->rt_rmx.rmx_ssthresh);
 		tcpstat.tcps_usedssthresh++;
 	}
+	lck_mtx_unlock(rt_mtx);
 }
 
 /*
@@ -3277,18 +3562,21 @@ tcp_mssopt(tp)
 #else
 #define min_protoh  (sizeof (struct tcpiphdr))
 #endif
+	lck_mtx_lock(rt_mtx);
 #if INET6
 	if (isipv6)
 		rt = tcp_rtlookup6(tp->t_inpcb);
 	else
 #endif /* INET6 */
 	rt = tcp_rtlookup(tp->t_inpcb);
-	if (rt == NULL)
-		return
+	if (rt == NULL) {
+		lck_mtx_unlock(rt_mtx);
+		return (
 #if INET6
 			isipv6 ? tcp_v6mssdflt :
 #endif /* INET6 */
-			tcp_mssdflt;
+			tcp_mssdflt);
+	}
 	/*
 	 * Slower link window correction:
 	 * If a value is specificied for slowlink_wsize use it for PPP links
@@ -3300,18 +3588,12 @@ tcp_mssopt(tp)
 		tp->t_flags |= TF_SLOWLINK;
 	}
 
-	if (rt->rt_rmx.rmx_mtu)
-		mss =  rt->rt_rmx.rmx_mtu;
-	else {
-		mss =
-#if INET6
-			(isipv6 ? nd_ifinfo[rt->rt_ifp->if_index].linkmtu :
-#endif
-			 rt->rt_ifp->if_mtu
 #if INET6
-			 );
+	mss = (isipv6 ? tcp_maxmtu6(rt) : tcp_maxmtu(rt));
+#else
+	mss = tcp_maxmtu(rt);
 #endif
-	}
+	lck_mtx_unlock(rt_mtx);
 	return (mss - min_protoh);
 }
 
@@ -3338,6 +3620,7 @@ tcp_newreno_partial_ack(tp, th)
 		 */
 		tp->snd_cwnd = tp->t_maxseg + (th->th_ack - tp->snd_una);
 		tp->t_flags |= TF_ACKNOW;
+	    	tp->t_unacksegs = 0;
 		(void) tcp_output(tp);
 		tp->snd_cwnd = ocwnd;
 		if (SEQ_GT(onxt, tp->snd_nxt))
@@ -3346,7 +3629,12 @@ tcp_newreno_partial_ack(tp, th)
 		 * Partial window deflation.  Relies on fact that tp->snd_una
 		 * not updated yet.
 		 */
-		tp->snd_cwnd -= (th->th_ack - tp->snd_una - tp->t_maxseg);
+		if (tp->snd_cwnd > th->th_ack - tp->snd_una)
+			tp->snd_cwnd -= th->th_ack - tp->snd_una;
+		else
+			tp->snd_cwnd = 0;
+		tp->snd_cwnd += tp->t_maxseg;
+
 }
 
 /*
@@ -3359,9 +3647,9 @@ tcp_newreno_partial_ack(tp, th)
  * The listening TCP socket "head" must be locked
  */
 static int
-tcpdropdropablreq(struct socket *head)
+tcp_dropdropablreq(struct socket *head)
 {
-	struct socket *so;
+	struct socket *so, *sonext;
 	unsigned int i, j, qlen;
 	static int rnd;
 	static struct timeval old_runtime;
@@ -3369,7 +3657,14 @@ tcpdropdropablreq(struct socket *head)
 	struct timeval tv;
 	struct inpcb *inp = NULL;
 	struct tcpcb *tp;
-	
+
+	if ((head->so_options & SO_ACCEPTCONN) == 0)
+		return 0;
+
+	so = TAILQ_FIRST(&head->so_incomp);
+	if (!so)
+		return 0;
+
 	microtime(&tv);
 	if ((i = (tv.tv_sec - old_runtime.tv_sec)) != 0) {
 		old_runtime = tv;
@@ -3377,9 +3672,6 @@ tcpdropdropablreq(struct socket *head)
 		cur_cnt = 0;
 	}
 	
-	so = TAILQ_FIRST(&head->so_incomp);
-	if (!so)
-		return 0;
 	
 	qlen = head->so_incqlen;
 	if (++cur_cnt > qlen || old_cnt > qlen) {
@@ -3389,57 +3681,77 @@ tcpdropdropablreq(struct socket *head)
 		while (j-- && so)
 			so = TAILQ_NEXT(so, so_list);
 	}
-	/* Find a connection that is not already closing */
+	/* Find a connection that is not already closing (or being served) */
 	while (so) {
 		inp = (struct inpcb *)so->so_pcb;
 		
-		if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING)
-			break;
+		sonext = TAILQ_NEXT(so, so_list);
+
+		if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) != WNT_STOPUSING) {
+			/* Avoid the issue of a socket being accepted by one input thread
+			 * and being dropped by another input thread.
+			 * If we can't get a hold on this mutex, then grab the next socket in line.
+			 */
+			if (lck_mtx_try_lock(inp->inpcb_mtx)) {
+				so->so_usecount++;
+				if ((so->so_usecount == 2) && so->so_state & SS_INCOMP) 
+					break;
+				else {/* don't use if beeing accepted or used in any other way */
+					in_pcb_checkstate(inp, WNT_RELEASE, 1);
+					tcp_unlock(so, 1, 0);
+				}
+			}
+		}
+		so = sonext;
 		
-		so = TAILQ_NEXT(so, so_list);
 	}
 	if (!so)
 		return 0;
-	
-	head->so_incqlen--;
-	head->so_qlen--;
+
 	TAILQ_REMOVE(&head->so_incomp, so, so_list);
 	tcp_unlock(head, 0, 0);
 
-	/* Let's remove this connection from the incomplete list */
-	tcp_lock(so, 1, 0);
-	
+	/* Makes sure socket is still in the right state to be discarded */
+
 	if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
 		tcp_unlock(so, 1, 0);
+		tcp_lock(head, 0, 0);
 		return 0;
 	}
-	
+
+	if (so->so_usecount != 2 || !(so->so_state & SS_INCOMP)) {
+		/* do not discard: that socket is beeing accepted */
+		tcp_unlock(so, 1, 0);
+		tcp_lock(head, 0, 0);
+		return 0;
+	}
+
 	so->so_head = NULL;
-	so->so_usecount--;	/* No more held by so_head */
 
 	/* 
 	 * We do not want to lose track of the PCB right away in case we receive 
 	 * more segments from the peer
 	 */
 	tp = sototcpcb(so);
-	tp->t_flags |= TF_LQ_OVERFLOW;
-	tp->t_state = TCPS_CLOSED;
-	(void) tcp_output(tp);
+	so->so_flags |= SOF_OVERFLOW;
+	tp->t_state = TCPS_TIME_WAIT;
+	(void) tcp_close(tp);
+	tp->t_unacksegs = 0;
 	tcpstat.tcps_drops++;
-	soisdisconnected(so);
 	tcp_canceltimers(tp);
 	add_to_time_wait(tp);
 	
 	tcp_unlock(so, 1, 0);
 	tcp_lock(head, 0, 0);
-	
+	head->so_incqlen--;
+	head->so_qlen--;
 	return 1;
-	
 }
 
 static int
 tcp_getstat SYSCTL_HANDLER_ARGS
 {
+#pragma unused(oidp, arg1, arg2)
 
 	int error;
 
@@ -3447,7 +3759,7 @@ tcp_getstat SYSCTL_HANDLER_ARGS
 		req->oldlen= (size_t)sizeof(struct tcpstat);
 	}
 
-	error = SYSCTL_OUT(req, &tcpstat, (size_t) req->oldlen);
+	error = SYSCTL_OUT(req, &tcpstat, MIN(sizeof (tcpstat), req->oldlen));
 
         return (error);
 
@@ -3456,4 +3768,30 @@ tcp_getstat SYSCTL_HANDLER_ARGS
 SYSCTL_PROC(_net_inet_tcp, TCPCTL_STATS, stats, CTLFLAG_RD, 0, 0,
     tcp_getstat, "S,tcpstat", "TCP statistics (struct tcpstat, netinet/tcp_var.h)");
 
+static int
+sysctl_rexmtthresh SYSCTL_HANDLER_ARGS
+{
+#pragma unused(arg1, arg2)
+
+	int error, val = tcprexmtthresh;
+
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error || !req->newptr)
+                return (error);
+
+	/*
+	 * Constrain the number of duplicate ACKs
+	 * to consider for TCP fast retransmit 
+	 * to either 2 or 3
+	 */
+
+        if (val < 2 || val > 3)
+		return (EINVAL);
+
+	 tcprexmtthresh = val;
+
+	return (0);
+}
 
+SYSCTL_PROC(_net_inet_tcp, OID_AUTO, rexmt_thresh, CTLTYPE_INT|CTLFLAG_RW,
+	&tcprexmtthresh, 0, &sysctl_rexmtthresh, "I", "Duplicate ACK Threshold for Fast Retransmit");
diff --git a/bsd/netinet/tcp_output.c b/bsd/netinet/tcp_output.c
index b336fa6d8..250d4a2d6 100644
--- a/bsd/netinet/tcp_output.c
+++ b/bsd/netinet/tcp_output.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
@@ -54,6 +60,12 @@
  *	@(#)tcp_output.c	8.4 (Berkeley) 5/24/95
  * $FreeBSD: src/sys/netinet/tcp_output.c,v 1.39.2.10 2001/07/07 04:30:38 silby Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #define	_IP_VHL
 
@@ -69,9 +81,11 @@
 #include <sys/socketvar.h>
 
 #include <net/route.h>
+#include <net/if_var.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
+#include <netinet/in_var.h>
 #include <netinet/ip.h>
 #include <netinet/in_pcb.h>
 #include <netinet/ip_var.h>
@@ -96,6 +110,10 @@
 #include <netinet6/ipsec.h>
 #endif /*IPSEC*/
 
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_SOCKET */
+
 #define DBG_LAYER_BEG		NETDBG_CODE(DBG_NETTCP, 1)
 #define DBG_LAYER_END		NETDBG_CODE(DBG_NETTCP, 3)
 #define DBG_FNC_TCP_OUTPUT	NETDBG_CODE(DBG_NETTCP, (4 << 8) | 1)
@@ -105,7 +123,7 @@
 extern struct mbuf *m_copypack();
 #endif
 
-static int path_mtu_discovery = 1;
+int path_mtu_discovery = 1;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW,
 	&path_mtu_discovery, 1, "Enable Path MTU Discovery");
 
@@ -113,7 +131,7 @@ int ss_fltsz = 1;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW,
 	&ss_fltsz, 1, "Slow start flight size");
 
-int ss_fltsz_local = 4; /* starts with four segments max */
+int ss_fltsz_local = 8; /* starts with eight segments max */
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize, CTLFLAG_RW,
 	&ss_fltsz_local, 1, "Slow start flight size for local networks");
 
@@ -121,11 +139,22 @@ int     tcp_do_newreno = 0;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, newreno, CTLFLAG_RW, &tcp_do_newreno,
         0, "Enable NewReno Algorithms");
 
+int     tcp_ecn_outbound = 0;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_initiate_out, CTLFLAG_RW, &tcp_ecn_outbound,
+        0, "Initiate ECN for outbound connections");
+
+int     tcp_ecn_inbound = 0;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, ecn_negotiate_in, CTLFLAG_RW, &tcp_ecn_inbound,
+        0, "Allow ECN negotiation for inbound connections");
+
 int	tcp_packet_chaining = 50;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, packetchain, CTLFLAG_RW, &tcp_packet_chaining,
         0, "Enable TCP output packet chaining");
 
-struct	mbuf *m_copym_with_hdrs(struct mbuf*, int, int, int, struct mbuf**, int*);
+int	tcp_output_unlocked = 1;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, socket_unlocked_on_output, CTLFLAG_RW, &tcp_output_unlocked,
+        0, "Unlock TCP when sending packets down to IP");
+
 static long packchain_newlist = 0;
 static long packchain_looped = 0;
 static long packchain_sent = 0;
@@ -138,11 +167,14 @@ extern int ipsec_bypass;
 
 extern int slowlink_wsize;	/* window correction for slow links */
 extern u_long  route_generation;
-extern int fw_enable; 		/* firewall is on: disable packet chaining */
-extern int ipsec_bypass;
+extern int fw_enable; 		/* firewall check for packet chaining */
+extern int fw_bypass; 		/* firewall check: disable packet chaining if there is rules */
 
 extern vm_size_t	so_cache_zone_element_size;
 
+static int tcp_ip_output(struct socket *, struct tcpcb *, struct mbuf *, int,
+    struct mbuf *, int);
+
 static __inline__ u_int16_t
 get_socket_id(struct socket * s)
 {
@@ -160,6 +192,22 @@ get_socket_id(struct socket * s)
 
 /*
  * Tcp output routine: figure out what should be sent and send it.
+ *
+ * Returns:	0			Success
+ *		EADDRNOTAVAIL
+ *		ENOBUFS
+ *		EMSGSIZE
+ *		EHOSTUNREACH
+ *		ENETDOWN
+ *	ip_output_list:ENOMEM
+ *	ip_output_list:EADDRNOTAVAIL
+ *	ip_output_list:ENETUNREACH
+ *	ip_output_list:EHOSTUNREACH
+ *	ip_output_list:EACCES
+ *	ip_output_list:EMSGSIZE
+ *	ip_output_list:ENOBUFS
+ *	ip_output_list:???		[ignorable: mostly IPSEC/firewall/DLIL]
+ *	ip6_output:???			[IPV6 only]
  */
 int
 tcp_output(struct tcpcb *tp)
@@ -176,26 +224,26 @@ tcp_output(struct tcpcb *tp)
 	register struct tcphdr *th;
 	u_char opt[TCP_MAXOLEN];
 	unsigned ipoptlen, optlen, hdrlen;
-	int idle, sendalot, howmuchsent = 0;
+	int idle, sendalot, lost = 0;
 	int i, sack_rxmit;
 	int sack_bytes_rxmt;
 	struct sackhole *p;
 
 	int maxburst = TCP_MAXBURST;
-	struct rmxp_tao *taop;
-	struct rmxp_tao tao_noncached;
 	int    last_off = 0;
 	int    m_off;
-	struct mbuf *m_last = 0;
-	struct mbuf *m_head = 0;
-	struct mbuf *packetlist = 0;
-	struct mbuf *lastpacket = 0;
+	struct mbuf *m_last = NULL;
+	struct mbuf *m_head = NULL;
+	struct mbuf *packetlist = NULL;
+	struct mbuf *tp_inp_options = tp->t_inpcb->inp_depend4.inp4_options;
 #if INET6
 	int isipv6 = tp->t_inpcb->inp_vflag & INP_IPV6 ;
+	struct ip6_pktopts *inp6_pktopts = tp->t_inpcb->inp_depend6.inp6_outputopts;
 #endif
 	short packchain_listadd = 0;
 	u_int16_t	socket_id = get_socket_id(so);
-
+	int so_options = so->so_options;
+	struct rtentry *rt;
 
 	/*
 	 * Determine length of data that should be transmitted,
@@ -261,13 +309,18 @@ again:
 	 * return error or silently do nothing (assuming the address will
 	 * come back before the TCP connection times out).
 	 */
+	rt = tp->t_inpcb->inp_route.ro_rt;
+	if (rt != NULL && rt->generation_id != route_generation) {
+		struct ifnet *ifp;
+
+		/* disable multipages at the socket */
+		somultipages(so, FALSE);
 
-      if ((tp->t_inpcb->inp_route.ro_rt != NULL &&
-           (tp->t_inpcb->inp_route.ro_rt->generation_id != route_generation)) || (tp->t_inpcb->inp_route.ro_rt == NULL)) {
 		/* check that the source address is still valid */
 		if (ifa_foraddr(tp->t_inpcb->inp_laddr.s_addr) == 0) {
+
 			if (tp->t_state >= TCPS_CLOSE_WAIT) {
-				tcp_close(tp);
+				tcp_drop(tp, EADDRNOTAVAIL);
 				return(EADDRNOTAVAIL);
 			}
 
@@ -284,16 +337,44 @@ again:
 				}
 			}
 
-			if (packetlist) {
-				error = ip_output_list(packetlist, packchain_listadd, tp->t_inpcb->inp_options, &tp->t_inpcb->inp_route,
-    					(so->so_options & SO_DONTROUTE), 0);
-				tp->t_lastchain = 0;
-			}
-			if (so->so_flags & SOF_NOADDRAVAIL)
+			if (tp->t_pktlist_head != NULL)
+				m_freem_list(tp->t_pktlist_head);
+			TCP_PKTLIST_CLEAR(tp);
+
+			/* drop connection if source address isn't available */
+			if (so->so_flags & SOF_NOADDRAVAIL) { 
+				tcp_drop(tp, EADDRNOTAVAIL);
 				return(EADDRNOTAVAIL);
+			}
 			else
-				return(0); /* silently ignore and keep data in socket */
+				return(0); /* silently ignore, keep data in socket: address may be back */
 		}
+
+		/*
+		 * Address is still valid; check for multipages capability
+		 * again in case the outgoing interface has changed.
+		 */
+		lck_mtx_lock(rt_mtx);
+		rt = tp->t_inpcb->inp_route.ro_rt;
+		if (rt != NULL && (ifp = rt->rt_ifp) != NULL)
+			somultipages(so, (ifp->if_hwassist & IFNET_MULTIPAGES));
+		if (rt != NULL && rt->generation_id != route_generation)
+			rt->generation_id = route_generation;
+		/*
+		 * See if we should do MTU discovery. Don't do it if:
+		 *	1) it is disabled via the sysctl
+		 *	2) the route isn't up
+		 *	3) the MTU is locked (if it is, then discovery has been
+		 *	   disabled)
+		 */
+
+	    	if (!path_mtu_discovery || ((rt != NULL) && 
+		    (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU)))) 
+			tp->t_flags &= ~TF_PMTUD;
+		else
+			tp->t_flags |= TF_PMTUD;
+
+		lck_mtx_unlock(rt_mtx);
         }
 	}
 
@@ -463,13 +544,29 @@ after_sack_rexmit:
 		flags &= ~TH_SYN;
 		off--, len++;
 		if (len > 0 && tp->t_state == TCPS_SYN_SENT) {
-			if (packetlist) {
-				error = ip_output_list(packetlist, packchain_listadd, tp->t_inpcb->inp_options, &tp->t_inpcb->inp_route,
-    				(so->so_options & SO_DONTROUTE), 0);
-				tp->t_lastchain = 0;
+			while (!(tp->t_flags & TF_SENDINPROG) &&
+			    tp->t_pktlist_head != NULL) {
+				packetlist = tp->t_pktlist_head;
+				packchain_listadd = tp->t_lastchain;
+				packchain_sent++;
+				TCP_PKTLIST_CLEAR(tp);
+				tp->t_flags |= TF_SENDINPROG;
+
+				error = tcp_ip_output(so, tp, packetlist,
+				    packchain_listadd, tp_inp_options,
+				    (so_options & SO_DONTROUTE));
+
+				tp->t_flags &= ~TF_SENDINPROG;
+			}
+			/* tcp was closed while we were in ip; resume close */
+			if ((tp->t_flags &
+			    (TF_CLOSING|TF_SENDINPROG)) == TF_CLOSING) {
+				tp->t_flags &= ~TF_CLOSING;
+				(void) tcp_close(tp);
 			}
-		  KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
-		  return 0;
+			KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END,
+			    0,0,0,0,0);
+			return 0;
 		}
 	}
 
@@ -511,7 +608,6 @@ after_sack_rexmit:
 	 */
 	if (len > tp->t_maxseg) {
 		len = tp->t_maxseg;
-		howmuchsent += len;
 		sendalot = 1;
 	}
 	if (sack_rxmit) {
@@ -522,10 +618,7 @@ after_sack_rexmit:
 			flags &= ~TH_FIN;
 	}
 
-	if (tp->t_flags & TF_SLOWLINK && slowlink_wsize > 0 )	/* Clips window size for slow links */
-		recwin = min(sbspace(&so->so_rcv), slowlink_wsize);
-	else
-		recwin = sbspace(&so->so_rcv);
+	recwin = tcp_sbspace(tp);
 
 	/*
 	 * Sender silly window avoidance.   We transmit under the following
@@ -540,19 +633,29 @@ after_sack_rexmit:
 	 *	- we need to retransmit
 	 */
 	if (len) {
-		if (len == tp->t_maxseg)
+		if (len == tp->t_maxseg) {
+			tp->t_flags |= TF_MAXSEGSNT;
 			goto send;
+		}
 		if (!(tp->t_flags & TF_MORETOCOME) &&
-		    (idle || tp->t_flags & TF_NODELAY) &&
+		    (idle || tp->t_flags & TF_NODELAY || tp->t_flags & TF_MAXSEGSNT) &&
 		    (tp->t_flags & TF_NOPUSH) == 0 &&
-		    len + off >= so->so_snd.sb_cc)
+		    len + off >= so->so_snd.sb_cc) {
+			tp->t_flags &= ~TF_MAXSEGSNT;
 			goto send;
-		if (tp->t_force)
+		}
+		if (tp->t_force) {
+			tp->t_flags &= ~TF_MAXSEGSNT;
 			goto send;
-		if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
+		}
+		if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
+			tp->t_flags &= ~TF_MAXSEGSNT;
 			goto send;
-		if (SEQ_LT(tp->snd_nxt, tp->snd_max))	/* retransmit case */
+		}
+		if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
+			tp->t_flags &= ~TF_MAXSEGSNT;
 			goto send;
+		}
 		if (sack_rxmit)
 			goto send;
 	}
@@ -571,7 +674,7 @@ after_sack_rexmit:
 		 * taking into account that we are limited by
 		 * TCP_MAXWIN << tp->rcv_scale.
 		 */
-		long adv = min(recwin, (long)TCP_MAXWIN << tp->rcv_scale) -
+		long adv = lmin(recwin, (long)TCP_MAXWIN << tp->rcv_scale) -
 			(tp->rcv_adv - tp->rcv_nxt);
 
 		if (adv >= (long) (2 * tp->t_maxseg))
@@ -641,9 +744,22 @@ just_return:
 	 * If there is no reason to send a segment, just return.
 	 * but if there is some packets left in the packet list, send them now.
 	 */
-	if (packetlist) {
-		error = ip_output_list(packetlist, packchain_listadd, tp->t_inpcb->inp_options, &tp->t_inpcb->inp_route,
-    			(so->so_options & SO_DONTROUTE), 0);
+	while (!(tp->t_flags & TF_SENDINPROG) && tp->t_pktlist_head != NULL) {
+		packetlist = tp->t_pktlist_head;
+		packchain_listadd = tp->t_lastchain;
+		packchain_sent++;
+		TCP_PKTLIST_CLEAR(tp);
+		tp->t_flags |= TF_SENDINPROG;
+
+		error = tcp_ip_output(so, tp, packetlist, packchain_listadd,
+		    tp_inp_options, (so_options & SO_DONTROUTE));
+
+		tp->t_flags &= ~TF_SENDINPROG;
+	}
+	/* tcp was closed while we were in ip; resume close */
+	if ((tp->t_flags & (TF_CLOSING|TF_SENDINPROG)) == TF_CLOSING) {
+		tp->t_flags &= ~TF_CLOSING;
+		(void) tcp_close(tp);
 	}
 	KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
 	return (0);
@@ -686,7 +802,90 @@ send:
 				optlen += 4;
 			}
 		}
+		
  	}
+ 	
+ 	/*
+ 	  RFC 3168 states that:
+ 	   - If you ever sent an ECN-setup SYN/SYN-ACK you must be prepared
+ 	   to handle the TCP ECE flag, even if you also later send a
+ 	   non-ECN-setup SYN/SYN-ACK.
+ 	   - If you ever send a non-ECN-setup SYN/SYN-ACK, you must not set
+ 	   the ip ECT flag.
+ 	   
+ 	   It is not clear how the ECE flag would ever be set if you never
+ 	   set the IP ECT flag on outbound packets. All the same, we use
+ 	   the TE_SETUPSENT to indicate that we have committed to handling
+ 	   the TCP ECE flag correctly. We use the TE_SENDIPECT to indicate
+ 	   whether or not we should set the IP ECT flag on outbound packets.
+ 	 */
+	/*
+	 * For a SYN-ACK, send an ECN setup SYN-ACK
+	 */
+	if (tcp_ecn_inbound && (flags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
+		if ((tp->ecn_flags & TE_SETUPRECEIVED) != 0) {
+			if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
+				/* Setting TH_ECE makes this an ECN-setup SYN-ACK */
+				flags |= TH_ECE;
+				
+				/*
+				 * Record that we sent the ECN-setup and default to
+				 * setting IP ECT.
+				 */
+				tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
+			}
+			else {
+				/*
+				 * We sent an ECN-setup SYN-ACK but it was dropped.
+				 * Fallback to non-ECN-setup SYN-ACK and clear flag
+				 * that to indicate we should not send data with IP ECT set.
+				 *
+				 * Pretend we didn't receive an ECN-setup SYN.
+				 */
+				tp->ecn_flags &= ~TE_SETUPRECEIVED;
+			}
+		}
+	}
+	else if (tcp_ecn_outbound && (flags & (TH_SYN | TH_ACK)) == TH_SYN) {
+		if ((tp->ecn_flags & TE_SETUPSENT) == 0) {
+			/* Setting TH_ECE and TH_CWR makes this an ECN-setup SYN */
+			flags |= (TH_ECE | TH_CWR);
+			
+			/*
+			 * Record that we sent the ECN-setup and default to
+			 * setting IP ECT.
+			 */
+			tp->ecn_flags |= (TE_SETUPSENT | TE_SENDIPECT);
+		}
+		else {
+			/*
+			 * We sent an ECN-setup SYN but it was dropped.
+			 * Fall back to no ECN and clear flag indicating
+			 * we should send data with IP ECT set.
+			 */
+			tp->ecn_flags &= ~TE_SENDIPECT;
+		}
+	}
+	
+	/*
+	 * Check if we should set the TCP CWR flag.
+	 * CWR flag is sent when we reduced the congestion window because
+	 * we received a TCP ECE or we performed a fast retransmit. We
+	 * never set the CWR flag on retransmitted packets. We only set
+	 * the CWR flag on data packets. Pure acks don't have this set.
+	 */
+	if ((tp->ecn_flags & TE_SENDCWR) != 0 && len != 0 &&
+		!SEQ_LT(tp->snd_nxt, tp->snd_max)) {
+		flags |= TH_CWR;
+		tp->ecn_flags &= ~TE_SENDCWR;
+	}
+	
+	/*
+	 * Check if we should set the TCP ECE flag.
+	 */
+	if ((tp->ecn_flags & TE_SENDECE) != 0 && len == 0) {
+		flags |= TH_ECE;
+	}
 
  	/*
 	 * Send a timestamp and echo-reply if this is a SYN and our side
@@ -798,8 +997,8 @@ send:
 	else
 #endif
 	{
-		if (tp->t_inpcb->inp_options) {
-			ipoptlen = tp->t_inpcb->inp_options->m_len -
+		if (tp_inp_options) {
+			ipoptlen = tp_inp_options->m_len -
 				offsetof(struct ipoption, ipopt_list);
 		} else
 			ipoptlen = 0;
@@ -821,7 +1020,6 @@ send:
 		 */
 		flags &= ~TH_FIN;
 		len = tp->t_maxopd - optlen - ipoptlen;
-		howmuchsent += len;
 		sendalot = 1;
 	}
 
@@ -877,7 +1075,7 @@ send:
 		m = NULL;
 #if INET6
  		if (MHLEN < hdrlen + max_linkhdr) {
-		        MGETHDR(m, M_DONTWAIT, MT_HEADER);
+		        MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 			if (m == NULL) {
 			        error = ENOBUFS;
 				goto out;
@@ -894,7 +1092,7 @@ send:
 #endif
 		if (len <= MHLEN - hdrlen - max_linkhdr) {
 		        if (m == NULL) {
-			        MGETHDR(m, M_DONTWAIT, MT_HEADER);
+			        MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 				if (m == NULL) {
 				        error = ENOBUFS;
 					goto out;
@@ -972,7 +1170,7 @@ send:
 		else
 			tcpstat.tcps_sndwinup++;
 
-		MGETHDR(m, M_DONTWAIT, MT_HEADER);
+		MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (m == NULL) {
 			error = ENOBUFS;
 			goto out;
@@ -987,6 +1185,9 @@ send:
 		m->m_len = hdrlen;
 	}
 	m->m_pkthdr.rcvif = 0;
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_inpcb(tp->t_inpcb, m);
+#endif
 #if INET6
 	if (isipv6) {
 		ip6 = mtod(m, struct ip6_hdr *);
@@ -1000,6 +1201,10 @@ send:
 		th = (struct tcphdr *)(ip + 1);
 		/* this picks up the pseudo header (w/o the length) */
 		tcp_fillheaders(tp, ip, th);
+		if ((tp->ecn_flags & TE_SENDIPECT) != 0 && len &&
+			!SEQ_LT(tp->snd_nxt, tp->snd_max)) {
+			ip->ip_tos = IPTOS_ECN_ECT0;
+		}
 	}
 
 	/*
@@ -1053,8 +1258,8 @@ send:
 			th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
 	}
 	else {
-		if (recwin > (long)TCP_MAXWIN << tp->rcv_scale)
-			recwin = (long)TCP_MAXWIN << tp->rcv_scale;
+		if (recwin > (long)(TCP_MAXWIN << tp->rcv_scale))
+			recwin = (long)(TCP_MAXWIN << tp->rcv_scale);
 		th->th_win = htons((u_short) (recwin>>tp->rcv_scale));
 	}
 
@@ -1103,7 +1308,7 @@ send:
 		if (len + optlen)
 			th->th_sum = in_addword(th->th_sum, 
 				htons((u_short)(optlen + len)));
-      }
+	}
 
 	/*
 	 * In transmit state, time the transmission and arrange for
@@ -1177,7 +1382,7 @@ timer:
 	/*
 	 * Trace.
 	 */
-	if (so->so_options & SO_DEBUG)
+	if (so_options & SO_DEBUG)
 		tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
 #endif
 
@@ -1214,13 +1419,12 @@ timer:
 #endif /*IPSEC*/
 		m->m_pkthdr.socket_id = socket_id;
 		error = ip6_output(m,
-			    tp->t_inpcb->in6p_outputopts,
+			    inp6_pktopts,
 			    &tp->t_inpcb->in6p_route,
-			    (so->so_options & SO_DONTROUTE), NULL, NULL, 0);
+			    (so_options & SO_DONTROUTE), NULL, NULL, 0);
 	} else
 #endif /* INET6 */
     {
-	struct rtentry *rt;
 	ip->ip_len = m->m_pkthdr.len;
 #if INET6
  	if (isipv6)
@@ -1231,7 +1435,7 @@ timer:
  	else
 #endif /* INET6 */
 	ip->ip_ttl = tp->t_inpcb->inp_ip_ttl;	/* XXX */
-	ip->ip_tos = tp->t_inpcb->inp_ip_tos;	/* XXX */
+	ip->ip_tos |= (tp->t_inpcb->inp_ip_tos & ~IPTOS_ECN_MASK);	/* XXX */
 
 
 #if INET6
@@ -1253,18 +1457,18 @@ timer:
 	}
 
 	/*
-	 * See if we should do MTU discovery.  We do it only if the following
-	 * are true:
-	 *	1) we have a valid route to the destination
-	 *	2) the MTU is not locked (if it is, then discovery has been
-	 *	   disabled)
+	 * See if we should do MTU discovery.
+	 * Look at the flag updated on the following criterias:
+	 *	1) Path MTU discovery is authorized by the sysctl
+	 *	2) The route isn't set yet (unlikely but could happen)
+	 *	3) The route is up
+	 *	4) the MTU is not locked (if it is, then discovery has been
+	 *	   disabled for that route)
 	 */
-	if (path_mtu_discovery
-	    && (rt = tp->t_inpcb->inp_route.ro_rt)
-	    && rt->rt_flags & RTF_UP
-	    && !(rt->rt_rmx.rmx_locks & RTV_MTU)) {
+
+	if (path_mtu_discovery && (tp->t_flags & TF_PMTUD))
 		ip->ip_off |= IP_DF;
-	}
+
 #if IPSEC
 	if (ipsec_bypass == 0)
  		ipsec_setsocket(m, so);
@@ -1273,38 +1477,63 @@ timer:
 	/*
 	 * The socket is kept locked while sending out packets in ip_output, even if packet chaining is not active.
 	 */
-
+	lost = 0;
 	m->m_pkthdr.socket_id = socket_id;
-	if (packetlist) {
-		m->m_nextpkt = NULL;
-		lastpacket->m_nextpkt = m;
-		lastpacket = m;
-		packchain_listadd++;
-	}
-	else {
-		m->m_nextpkt = NULL;
+	m->m_nextpkt = NULL;
+	tp->t_pktlist_sentlen += len;
+	tp->t_lastchain++;
+	if (tp->t_pktlist_head != NULL) {
+		tp->t_pktlist_tail->m_nextpkt = m;
+		tp->t_pktlist_tail = m;
+	} else {
 		packchain_newlist++;
-		packetlist = lastpacket = m;
-		packchain_listadd=0;
+		tp->t_pktlist_head = tp->t_pktlist_tail = m;
 	}
 
-       if ((ipsec_bypass == 0) || fw_enable || sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) || 
-		      (tp->snd_cwnd <= (tp->snd_wnd / 4)) || 
-		      (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || tp->t_force != 0 ||
-		      packchain_listadd >= tcp_packet_chaining) {
-	       	lastpacket->m_nextpkt = 0;
-		error = ip_output_list(packetlist, packchain_listadd, tp->t_inpcb->inp_options, &tp->t_inpcb->inp_route,
-    			(so->so_options & SO_DONTROUTE), 0);
-		tp->t_lastchain = packchain_listadd;
-		packchain_sent++;
-		packetlist = NULL;
-		if (error == 0)
-			howmuchsent = 0;
+	if (sendalot == 0 || (tp->t_state != TCPS_ESTABLISHED) ||
+	      (tp->snd_cwnd <= (tp->snd_wnd / 8)) ||
+	      (tp->t_flags & (TH_PUSH | TF_ACKNOW)) || tp->t_force != 0 ||
+	      tp->t_lastchain >= tcp_packet_chaining) {
+		error = 0;
+		while (!(tp->t_flags & TF_SENDINPROG) &&
+		    tp->t_pktlist_head != NULL) {
+			packetlist = tp->t_pktlist_head;
+			packchain_listadd = tp->t_lastchain;
+			packchain_sent++;
+			lost = tp->t_pktlist_sentlen;
+			TCP_PKTLIST_CLEAR(tp);
+			tp->t_flags |= TF_SENDINPROG;
+
+			error = tcp_ip_output(so, tp, packetlist,
+			    packchain_listadd, tp_inp_options,
+			    (so_options & SO_DONTROUTE));
+
+			tp->t_flags &= ~TF_SENDINPROG;
+			if (error) {
+				/*
+				 * Take into account the rest of unsent
+				 * packets in the packet list for this tcp
+				 * into "lost", since we're about to free
+				 * the whole list below.
+				 */
+				lost += tp->t_pktlist_sentlen;
+				break;
+			} else {
+				lost = 0;
+			}
+		}
+		/* tcp was closed while we were in ip; resume close */
+		if ((tp->t_flags & (TF_CLOSING|TF_SENDINPROG)) == TF_CLOSING) {
+			tp->t_flags &= ~TF_CLOSING;
+			(void) tcp_close(tp);
+			return (0);
+		}
 	}
 	else {
 		error = 0;
 		packchain_looped++;
 		tcpstat.tcps_sndtotal++;
+
 		if (recwin > 0 && SEQ_GT(tp->rcv_nxt+recwin, tp->rcv_adv))
 			tp->rcv_adv = tp->rcv_nxt + recwin;
 		tp->last_ack_sent = tp->rcv_nxt;
@@ -1313,10 +1542,12 @@ timer:
 	}
    }
 	if (error) {
-
 		/*
-		 * We know that the packet was lost, so back out the
-		 * sequence number advance, if any.
+		 * Assume that the packets were lost, so back out the
+		 * sequence number advance, if any.  Note that the "lost"
+		 * variable represents the amount of user data sent during
+		 * the recent call to ip_output_list() plus the amount of
+		 * user data in the packet list for this tcp at the moment.
 		 */
 		if (tp->t_force == 0 || tp->t_timer[TCPT_PERSIST] == 0) {
 			/*
@@ -1325,22 +1556,22 @@ timer:
 			 */
 			if ((flags & TH_SYN) == 0) {
 				if (sack_rxmit) {
-					p->rxmit -= howmuchsent;
-					tp->sackhint.sack_bytes_rexmit -= howmuchsent;
+					p->rxmit -= lost;
+					tp->sackhint.sack_bytes_rexmit -= lost;
 				} else
-					tp->snd_nxt -= howmuchsent;
+					tp->snd_nxt -= lost;
 			}
 		}
-		howmuchsent = 0;
 out:
+		if (tp->t_pktlist_head != NULL)
+			m_freem_list(tp->t_pktlist_head);
+		TCP_PKTLIST_CLEAR(tp);
+
 		if (error == ENOBUFS) {
                         if (!tp->t_timer[TCPT_REXMT] &&
                              !tp->t_timer[TCPT_PERSIST])
                                 tp->t_timer[TCPT_REXMT] = tp->t_rxtcur;
 			tcp_quench(tp->t_inpcb, 0);
-			if (packetlist)
-				m_freem_list(packetlist);
-			tp->t_lastchain = 0;
 			KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
 			return (0);
 		}
@@ -1352,28 +1583,19 @@ out:
 			 * not do so here.
 			 */
 			tcp_mtudisc(tp->t_inpcb, 0);
-			if (packetlist)
-				m_freem_list(packetlist);
-			tp->t_lastchain = 0;
 			KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
 			return 0;
 		}
 		if ((error == EHOSTUNREACH || error == ENETDOWN)
 		    && TCPS_HAVERCVDSYN(tp->t_state)) {
 			tp->t_softerror = error;
-			if (packetlist)
-				m_freem_list(packetlist);
-			tp->t_lastchain = 0;
 			KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
 			return (0);
 		}
-		if (packetlist)
-			m_freem_list(packetlist);
-		tp->t_lastchain = 0;
 		KERNEL_DEBUG(DBG_FNC_TCP_OUTPUT | DBG_FUNC_END, 0,0,0,0,0);
 		return (error);
 	}
-sentit:
+
 	tcpstat.tcps_sndtotal++;
 
 	/*
@@ -1393,6 +1615,83 @@ sentit:
 	return (0);
 }
 
+static int
+tcp_ip_output(struct socket *so, struct tcpcb *tp, struct mbuf *pkt,
+    int cnt, struct mbuf *opt, int flags)
+{
+	int error = 0;
+	boolean_t chain;
+	boolean_t unlocked = FALSE;
+
+	/*
+	 * If allowed, unlock TCP socket while in IP 
+	 * but only if the connection is established and
+	 * if we're not sending from an upcall.
+	 */ 
+
+	if (tcp_output_unlocked && ((so->so_flags & SOF_UPCALLINUSE) == 0) &&
+	    (tp->t_state == TCPS_ESTABLISHED)) {
+			unlocked = TRUE;
+			socket_unlock(so, 0);
+	}
+
+	/*
+	 * Don't send down a chain of packets when:
+	 * - TCP chaining is disabled
+	 * - there is an IPsec rule set
+	 * - there is a non default rule set for the firewall
+	 */
+
+	chain = tcp_packet_chaining > 1 &&
+#if IPSEC
+		ipsec_bypass &&
+#endif
+		(fw_enable == 0 || fw_bypass);
+
+	while (pkt != NULL) {
+		struct mbuf *npkt = pkt->m_nextpkt;
+
+		if (!chain) {
+			pkt->m_nextpkt = NULL;
+			/*
+			 * If we are not chaining, make sure to set the packet
+			 * list count to 0 so that IP takes the right path;
+			 * this is important for cases such as IPSec where a
+			 * single mbuf might result in multiple mbufs as part
+			 * of the encapsulation.  If a non-zero count is passed
+			 * down to IP, the head of the chain might change and
+			 * we could end up skipping it (thus generating bogus
+			 * packets).  Fixing it in IP would be desirable, but
+			 * for now this would do it.
+			 */
+			cnt = 0;
+		}
+#if CONFIG_FORCE_OUT_IFP
+		error = ip_output_list(pkt, cnt, opt, &tp->t_inpcb->inp_route,
+		    flags, 0, tp->t_inpcb->pdp_ifp);
+#else
+		error = ip_output_list(pkt, cnt, opt, &tp->t_inpcb->inp_route,
+		    flags, 0, NULL);
+#endif
+		if (chain || error) {
+			/*
+			 * If we sent down a chain then we are done since
+			 * the callee had taken care of everything; else
+			 * we need to free the rest of the chain ourselves.
+			 */
+			if (!chain)
+				m_freem_list(npkt);
+			break;
+		}
+		pkt = npkt;
+	}
+
+	if (unlocked)
+		socket_lock(so, 0);
+
+	return (error);
+}
+
 void
 tcp_setpersist(tp)
 	register struct tcpcb *tp;
diff --git a/bsd/netinet/tcp_sack.c b/bsd/netinet/tcp_sack.c
index 0d032db5b..2426bee5d 100644
--- a/bsd/netinet/tcp_sack.c
+++ b/bsd/netinet/tcp_sack.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004,2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
@@ -66,6 +72,8 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 
+#include <kern/zalloc.h>
+
 #include <net/route.h>
 
 #include <netinet/in.h>
diff --git a/bsd/netinet/tcp_seq.h b/bsd/netinet/tcp_seq.h
index 42c015017..ac032da62 100644
--- a/bsd/netinet/tcp_seq.h
+++ b/bsd/netinet/tcp_seq.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993, 1995
@@ -101,7 +107,7 @@
 	(tp)->snd_una = (tp)->snd_nxt = (tp)->snd_max = (tp)->snd_up = \
 	(tp)->snd_recover = (tp)->snd_high = (tp)->iss  
 
-#define TCP_PAWS_IDLE	(24 * 24 * 60 * 60 * PR_SLOWHZ)
+#define TCP_PAWS_IDLE	(24 * 24 * 60 * 60 * TCP_RETRANSHZ)
 					/* timestamp wrap-around time */
 
 extern tcp_cc	tcp_ccgen;		/* global connection count */
diff --git a/bsd/netinet/tcp_subr.c b/bsd/netinet/tcp_subr.c
index dc401fdd8..f120008cd 100644
--- a/bsd/netinet/tcp_subr.c
+++ b/bsd/netinet/tcp_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
@@ -54,7 +60,12 @@
  *	@(#)tcp_subr.c	8.2 (Berkeley) 5/24/95
  * $FreeBSD: src/sys/netinet/tcp_subr.c,v 1.73.2.22 2001/08/22 00:59:12 silby Exp $
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -63,9 +74,7 @@
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
-#if INET6
 #include <sys/domain.h>
-#endif
 #include <sys/proc.h>
 #include <sys/kauth.h>
 #include <sys/socket.h>
@@ -74,8 +83,7 @@
 #include <sys/random.h>
 #include <sys/syslog.h>
 #include <kern/locks.h>
-
-
+#include <kern/zalloc.h>
 
 #include <net/route.h>
 #include <net/if.h>
@@ -117,7 +125,11 @@
 #endif
 #endif /*IPSEC*/
 
-#include <sys/md5.h>
+#if CONFIG_MACF_NET
+#include <security/mac_framework.h>
+#endif /* MAC_NET */
+
+#include <libkern/crypto/md5.h>
 #include <sys/kdebug.h>
 
 #define DBG_FNC_TCP_CLOSE	NETDBG_CODE(DBG_NETTCP, ((5 << 8) | 2))
@@ -127,7 +139,6 @@ extern int tcp_lq_overflow;
 /* temporary: for testing */
 #if IPSEC
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif
 
 int 	tcp_mssdflt = TCP_MSS;
@@ -200,11 +211,26 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, strict_rfc1948, CTLFLAG_RW,
 static int	tcp_isn_reseed_interval = 0;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_RW,
     &tcp_isn_reseed_interval, 0, "Seconds between reseeding of ISN secret");
+static int 	tcp_background_io_enabled = 1;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, background_io_enabled, CTLFLAG_RW,
+    &tcp_background_io_enabled, 0, "Background IO Enabled");
+
+int 	tcp_TCPTV_MIN = 1;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, rtt_min, CTLFLAG_RW,
+    &tcp_TCPTV_MIN, 0, "min rtt value allowed");
 
 static void	tcp_cleartaocache(void);
 static void	tcp_notify(struct inpcb *, int);
 struct zone	*sack_hole_zone;
 
+extern unsigned int total_mb_cnt;
+extern unsigned int total_cl_cnt;
+extern int sbspace_factor;
+extern int tcp_sockthreshold;
+extern int slowlink_wsize;	/* window correction for slow links */
+extern int path_mtu_discovery;
+
+
 /*
  * Target size of TCP PCB hash tables. Must be a power of two.
  *
@@ -212,7 +238,7 @@ struct zone	*sack_hole_zone;
  * variable net.inet.tcp.tcbhashsize
  */
 #ifndef TCBHASHSIZE
-#define TCBHASHSIZE	4096
+#define TCBHASHSIZE	CONFIG_TCBHASHSIZE
 #endif
 
 /*
@@ -243,14 +269,17 @@ extern int		cur_tw_slot;
 extern u_long		*delack_bitmask;
 extern u_long  route_generation;
 
+int  get_inpcb_str_size(void);
+int  get_tcp_str_size(void);
 
-int  get_inpcb_str_size()
+
+int  get_inpcb_str_size(void)
 {
 	return sizeof(struct inpcb);
 }
 
 
-int  get_tcp_str_size()
+int  get_tcp_str_size(void)
 {
 	return sizeof(struct tcpcb);
 }
@@ -272,14 +301,13 @@ tcp_init()
 	tcp_ccgen = 1;
 	tcp_cleartaocache();
 
-	tcp_delacktime = TCPTV_DELACK;
 	tcp_keepinit = TCPTV_KEEP_INIT;
 	tcp_keepidle = TCPTV_KEEP_IDLE;
 	tcp_keepintvl = TCPTV_KEEPINTVL;
 	tcp_maxpersistidle = TCPTV_KEEP_IDLE;
 	tcp_msl = TCPTV_MSL;
 	read_random(&tcp_now, sizeof(tcp_now));
-	tcp_now  = tcp_now & 0x7fffffff; /* Starts tcp internal 500ms clock at a random value */
+	tcp_now = tcp_now & 0x3fffffff; /* Starts tcp internal 100ms clock at a random value */
 
 
 	LIST_INIT(&tcb);
@@ -342,6 +370,8 @@ tcp_init()
 	for (i=0; i < N_TIME_WAIT_SLOTS; i++) {
 	     LIST_INIT(&time_wait_slots[i]);
 	}
+
+	timeout(tcp_fasttimo, NULL, hz/TCP_RETRANSHZ);
 }
 
 /*
@@ -439,13 +469,20 @@ tcp_maketemplate(tp)
  * NOTE: If m != NULL, then ti must point to *inside* the mbuf.
  */
 void
-tcp_respond(tp, ipgen, th, m, ack, seq, flags)
-	struct tcpcb *tp;
-	void *ipgen;
-	register struct tcphdr *th;
-	register struct mbuf *m;
-	tcp_seq ack, seq;
-	int flags;
+tcp_respond(
+	struct tcpcb *tp,
+	void *ipgen,
+	register struct tcphdr *th,
+	register struct mbuf *m,
+	tcp_seq ack,
+	tcp_seq seq,
+	int flags,
+#if CONFIG_FORCE_OUT_IFP
+	ifnet_t	ifp
+#else
+	__unused ifnet_t ifp
+#endif
+	)
 {
 	register int tlen;
 	int win = 0;
@@ -469,7 +506,7 @@ tcp_respond(tp, ipgen, th, m, ack, seq, flags)
 
 	if (tp) {
 		if (!(flags & TH_RST)) {
-			win = sbspace(&tp->t_inpcb->inp_socket->so_rcv);
+			win = tcp_sbspace(tp);
 			if (win > (long)TCP_MAXWIN << tp->rcv_scale)
 				win = (long)TCP_MAXWIN << tp->rcv_scale;
 		}
@@ -492,7 +529,7 @@ tcp_respond(tp, ipgen, th, m, ack, seq, flags)
 		}
 	}
 	if (m == 0) {
-		m = m_gethdr(M_DONTWAIT, MT_HEADER);
+		m = m_gethdr(M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (m == NULL)
 			return;
 		tlen = 0;
@@ -556,6 +593,21 @@ tcp_respond(tp, ipgen, th, m, ack, seq, flags)
 	m->m_len = tlen;
 	m->m_pkthdr.len = tlen;
 	m->m_pkthdr.rcvif = 0;
+#if CONFIG_MACF_NET
+	if (tp != NULL && tp->t_inpcb != NULL) {
+		/*
+		 * Packet is associated with a socket, so allow the
+		 * label of the response to reflect the socket label.
+		 */
+		mac_mbuf_label_associate_inpcb(tp->t_inpcb, m);
+	} else {
+		/*
+		 * Packet is not associated with a socket, so possibly
+		 * update the label in place.
+		 */
+		mac_netinet_tcp_reply(m);
+	}
+#endif
 	nth->th_seq = htonl(seq);
 	nth->th_ack = htonl(ack);
 	nth->th_x2 = 0;
@@ -604,7 +656,11 @@ tcp_respond(tp, ipgen, th, m, ack, seq, flags)
 	} else
 #endif /* INET6 */
 	{
-		(void) ip_output_list(m, 0, NULL, ro, ipflags, NULL);
+#if CONFIG_FORCE_OUT_IFP
+		ifp = (tp && tp->t_inpcb) ? tp->t_inpcb->pdp_ifp :
+			  (ifp && (ifp->if_flags & IFF_POINTOPOINT) != 0) ? ifp : NULL;
+#endif
+		(void) ip_output_list(m, 0, NULL, ro, ipflags, NULL, ifp);
 		if (ro == &sro && ro->ro_rt) {
 			rtfree(ro->ro_rt);
 			ro->ro_rt = NULL;
@@ -656,11 +712,13 @@ tcp_newtcpcb(inp)
 	 */
 	tp->t_srtt = TCPTV_SRTTBASE;
 	tp->t_rttvar = ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
-	tp->t_rttmin = TCPTV_MIN;
+	tp->t_rttmin = tcp_TCPTV_MIN;
 	tp->t_rxtcur = TCPTV_RTOBASE;
 	tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
+	tp->snd_bwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
 	tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
 	tp->t_rcvtime = 0;
+	tp->t_bw_rtttime = 0;
 	/*
 	 * IPv4 TTL initialization is necessary for an IPv6 socket as well,
 	 * because the socket may be bound to an IPv6 wildcard address,
@@ -735,7 +793,24 @@ tcp_close(tp)
 	     break;
 	}
 
+	/*
+	 * If another thread for this tcp is currently in ip (indicated by
+	 * the TF_SENDINPROG flag), defer the cleanup until after it returns
+	 * back to tcp.  This is done to serialize the close until after all
+	 * pending output is finished, in order to avoid having the PCB be
+	 * detached and the cached route cleaned, only for ip to cache the
+	 * route back into the PCB again.  Note that we've cleared all the
+	 * timers at this point.  Set TF_CLOSING to indicate to tcp_output()
+	 * that is should call us again once it returns from ip; at that
+	 * point both flags should be cleared and we can proceed further
+	 * with the cleanup.
+	 */
+	if (tp->t_flags & (TF_CLOSING|TF_SENDINPROG)) {
+		tp->t_flags |= TF_CLOSING;
+		return (NULL);
+	}
 
+	lck_mtx_lock(rt_mtx);
 	/*
 	 * If we got enough samples through the srtt filter,
 	 * save the rtt and rttvar in the routing entry.
@@ -749,6 +824,7 @@ tcp_close(tp)
 	 */
 	if (tp->t_rttupdated >= 16) {
 		register u_long i = 0;
+
 #if INET6
 		if (isipv6) {
 			struct sockaddr_in6 *sin6;
@@ -773,7 +849,7 @@ tcp_close(tp)
 
 		if ((rt->rt_rmx.rmx_locks & RTV_RTT) == 0) {
 			i = tp->t_srtt *
-			    (RTM_RTTUNIT / (PR_SLOWHZ * TCP_RTT_SCALE));
+			    (RTM_RTTUNIT / (TCP_RETRANSHZ * TCP_RTT_SCALE));
 			if (rt->rt_rmx.rmx_rtt && i)
 				/*
 				 * filter this update to half the old & half
@@ -789,7 +865,7 @@ tcp_close(tp)
 		}
 		if ((rt->rt_rmx.rmx_locks & RTV_RTTVAR) == 0) {
 			i = tp->t_rttvar *
-			    (RTM_RTTUNIT / (PR_SLOWHZ * TCP_RTTVAR_SCALE));
+			    (RTM_RTTUNIT / (TCP_RETRANSHZ * TCP_RTTVAR_SCALE));
 			if (rt->rt_rmx.rmx_rttvar && i)
 				rt->rt_rmx.rmx_rttvar =
 				    (rt->rt_rmx.rmx_rttvar + i) / 2;
@@ -850,7 +926,7 @@ tcp_close(tp)
 		 * mark route for deletion if no information is
 		 * cached.
 		 */
-		if ((tp->t_flags & TF_LQ_OVERFLOW) && tcp_lq_overflow && 
+		if ((so->so_flags & SOF_OVERFLOW) && tcp_lq_overflow && 
 		    ((rt->rt_rmx.rmx_locks & RTV_RTT) == 0)){
 			if (rt->rt_rmx.rmx_rtt == 0)
 				rt->rt_flags |= RTF_DELCLONE;
@@ -858,10 +934,17 @@ tcp_close(tp)
 	}
     no_valid_rt:
 	/* free the reassembly queue, if any */
+	lck_mtx_unlock(rt_mtx);
+
 	(void) tcp_freeq(tp);
 
 	tcp_free_sackholes(tp);
 
+	/* Free the packet list */
+	if (tp->t_pktlist_head != NULL)
+		m_freem_list(tp->t_pktlist_head);
+	TCP_PKTLIST_CLEAR(tp);
+
 #ifdef __APPLE__
 	if (so->cached_in_sock_layer)
 	    inp->inp_saved_ppcb = (caddr_t) tp;
@@ -900,9 +983,6 @@ tcp_freeq(tp)
 void
 tcp_drain()
 {
-/*
- * ###LD 05/19/04 locking issue, tcpdrain is disabled, deadlock situation with tcbinfo.mtx
- */
 	if (do_tcpdrain)
 	{
 		struct inpcb *inpb;
@@ -917,7 +997,9 @@ tcp_drain()
 	 * 	where we're really low on mbufs, this is potentially
 	 *  	usefull.	
 	 */
-		lck_rw_lock_exclusive(tcbinfo.mtx);
+		if (!lck_rw_try_lock_exclusive(tcbinfo.mtx)) /* do it next time if the lock is in use */
+			return;
+
 		for (inpb = LIST_FIRST(tcbinfo.listhead); inpb;
 	    		inpb = LIST_NEXT(inpb, inp_list)) {
 				if ((tcpb = intotcpcb(inpb))) {
@@ -981,10 +1063,12 @@ tcp_notify(inp, error)
 static int
 tcp_pcblist SYSCTL_HANDLER_ARGS
 {
+#pragma unused(oidp, arg1, arg2)
 	int error, i, n;
 	struct inpcb *inp, **inp_list;
 	inp_gen_t gencnt;
 	struct xinpgen xig;
+	int slot;
 
 	/*
 	 * The process of preparing the TCB list is too time-consuming and
@@ -1020,12 +1104,12 @@ tcp_pcblist SYSCTL_HANDLER_ARGS
 		lck_rw_done(tcbinfo.mtx);
 		return error;
 	}
-        /*
-         * We are done if there is no pcb
-         */
-        if (n == 0) {
-	    lck_rw_done(tcbinfo.mtx);
-            return 0; 
+	/*
+	 * We are done if there is no pcb
+	 */
+	if (n == 0) {
+		lck_rw_done(tcbinfo.mtx);
+		return 0; 
 	}
 
 	inp_list = _MALLOC(n * sizeof *inp_list, M_TEMP, M_WAITOK);
@@ -1043,6 +1127,17 @@ tcp_pcblist SYSCTL_HANDLER_ARGS
 #endif
 			inp_list[i++] = inp;
 	}
+
+	for (slot = 0; slot < N_TIME_WAIT_SLOTS; slot++) {
+		struct inpcb *inpnxt;
+
+		for (inp = time_wait_slots[slot].lh_first; inp && i < n; inp = inpnxt) {
+			inpnxt = inp->inp_list.le_next;
+			if (inp->inp_gencnt <= gencnt && inp->inp_state != INPCB_STATE_DEAD)
+				inp_list[i++] = inp;
+		}
+	}
+
 	n = i;
 
 	error = 0;
@@ -1337,9 +1432,10 @@ tcp6_ctlinput(cmd, sa, d)
 
 #define ISN_BYTES_PER_SECOND 1048576
 
-u_char isn_secret[32];
-int isn_last_reseed;
-MD5_CTX isn_ctx;
+//PWC - md5 routines cause alignment exceptions.  Need to figure out why.  For now use lame incremental
+// isn.  how's that for not easily guessable!?
+
+int pwc_bogus;
 
 tcp_seq
 tcp_new_isn(tp)
@@ -1348,6 +1444,9 @@ tcp_new_isn(tp)
 	u_int32_t md5_buffer[4];
 	tcp_seq new_isn;
 	struct timeval timenow;
+	u_char isn_secret[32];
+	int isn_last_reseed = 0;
+	MD5_CTX isn_ctx;
 
 	/* Use arc4random for SYN-ACKs when not in exact RFC1948 mode. */
 	if (((tp->t_state == TCPS_LISTEN) || (tp->t_state == TCPS_TIME_WAIT))
@@ -1409,8 +1508,10 @@ tcp_quench(
 {
 	struct tcpcb *tp = intotcpcb(inp);
 
-	if (tp)
+	if (tp) {
 		tp->snd_cwnd = tp->t_maxseg;
+		tp->t_bytes_acked = 0;
+	}
 }
 
 /*
@@ -1452,6 +1553,7 @@ tcp_mtudisc(
 #endif /* INET6 */
 
 	if (tp) {
+		lck_mtx_lock(rt_mtx);
 #if INET6
 		if (isipv6)
 			rt = tcp_rtlookup6(inp);
@@ -1464,6 +1566,7 @@ tcp_mtudisc(
 				isipv6 ? tcp_v6mssdflt :
 #endif /* INET6 */
 				tcp_mssdflt;
+			lck_mtx_unlock(rt_mtx);
 			return;
 		}
 		taop = rmx_taop(rt->rt_rmx);
@@ -1479,6 +1582,7 @@ tcp_mtudisc(
 #endif /* INET6 */
 			;
 
+		lck_mtx_unlock(rt_mtx);
 		if (offered)
 			mss = min(mss, offered);
 		/*
@@ -1530,11 +1634,15 @@ tcp_rtlookup(inp)
 {
 	struct route *ro;
 	struct rtentry *rt;
+	struct tcpcb *tp;
 
 	ro = &inp->inp_route;
 	if (ro == NULL)
 		return (NULL);
 	rt = ro->ro_rt;
+
+	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
+
 	if (rt == NULL || !(rt->rt_flags & RTF_UP) || rt->generation_id != route_generation) {
 		/* No route yet, so try to acquire one */
 		if (inp->inp_faddr.s_addr != INADDR_ANY) {
@@ -1542,10 +1650,30 @@ tcp_rtlookup(inp)
 			ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
 			((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
 				inp->inp_faddr;
-			rtalloc(ro);
+			rtalloc_ign_locked(ro, 0UL);
 			rt = ro->ro_rt;
 		}
 	}
+	if (rt != NULL && rt->rt_ifp != NULL)
+		somultipages(inp->inp_socket,
+		    (rt->rt_ifp->if_hwassist & IFNET_MULTIPAGES));
+
+	/*
+	 * Update MTU discovery determination. Don't do it if:
+	 *	1) it is disabled via the sysctl
+	 *	2) the route isn't up
+	 *	3) the MTU is locked (if it is, then discovery has been
+	 *	   disabled)
+	 */
+
+	 tp = intotcpcb(inp);
+
+	if (!path_mtu_discovery || ((rt != NULL) && 
+	    (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU)))) 
+		tp->t_flags &= ~TF_PMTUD;
+	else
+		tp->t_flags |= TF_PMTUD;
+
 	return rt;
 }
 
@@ -1556,6 +1684,9 @@ tcp_rtlookup6(inp)
 {
 	struct route_in6 *ro6;
 	struct rtentry *rt;
+	struct tcpcb *tp;
+
+	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
 
 	ro6 = &inp->in6p_route;
 	rt = ro6->ro_rt;
@@ -1568,10 +1699,38 @@ tcp_rtlookup6(inp)
 			dst6->sin6_family = AF_INET6;
 			dst6->sin6_len = sizeof(*dst6);
 			dst6->sin6_addr = inp->in6p_faddr;
-			rtalloc((struct route *)ro6);
+			rtalloc_ign_locked((struct route *)ro6, 0UL);
 			rt = ro6->ro_rt;
 		}
 	}
+	if (rt != NULL && rt->rt_ifp != NULL)
+		somultipages(inp->inp_socket,
+		    (rt->rt_ifp->if_hwassist & IFNET_MULTIPAGES));
+	/*
+	 * Update path MTU Discovery determination
+	 * while looking up the route:
+	 *  1) we have a valid route to the destination
+	 *  2) the MTU is not locked (if it is, then discovery has been
+	 *    disabled)
+	 */
+
+
+	 tp = intotcpcb(inp);
+
+	/*
+	 * Update MTU discovery determination. Don't do it if:
+	 *	1) it is disabled via the sysctl
+	 *	2) the route isn't up
+	 *	3) the MTU is locked (if it is, then discovery has been
+	 *	   disabled)
+	 */
+
+	if (!path_mtu_discovery || ((rt != NULL) && 
+	    (!(rt->rt_flags & RTF_UP) || (rt->rt_rmx.rmx_locks & RTV_MTU)))) 
+		tp->t_flags &= ~TF_PMTUD;
+	else
+		tp->t_flags |= TF_PMTUD;
+
 	return rt;
 }
 #endif /* INET6 */
@@ -1593,11 +1752,10 @@ ipsec_hdrsiz_tcp(tp)
 
 	if ((tp == NULL) || ((inp = tp->t_inpcb) == NULL))
 		return 0;
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
+	MGETHDR(m, M_DONTWAIT, MT_DATA);	/* MAC-OK */
 	if (!m)
 		return 0;
 
-	lck_mtx_lock(sadb_mutex);
 #if INET6
 	if ((inp->inp_vflag & INP_IPV6) != 0) {
 		ip6 = mtod(m, struct ip6_hdr *);
@@ -1615,7 +1773,6 @@ ipsec_hdrsiz_tcp(tp)
 	tcp_fillheaders(tp, ip, th);
 	hdrsiz = ipsec4_hdrsiz(m, IPSEC_DIR_OUTBOUND, inp);
       }
-	lck_mtx_unlock(sadb_mutex);
 	m_free(m);
 	return hdrsiz;
 }
@@ -1631,7 +1788,9 @@ tcp_gettaocache(inp)
 	struct inpcb *inp;
 {
 	struct rtentry *rt;
+	struct rmxp_tao *taop;
 
+	lck_mtx_lock(rt_mtx);
 #if INET6
 	if ((inp->inp_vflag & INP_IPV6) != 0)
 		rt = tcp_rtlookup6(inp);
@@ -1641,10 +1800,14 @@ tcp_gettaocache(inp)
 
 	/* Make sure this is a host route and is up. */
 	if (rt == NULL ||
-	    (rt->rt_flags & (RTF_UP|RTF_HOST)) != (RTF_UP|RTF_HOST))
+	    (rt->rt_flags & (RTF_UP|RTF_HOST)) != (RTF_UP|RTF_HOST)) {
+		lck_mtx_unlock(rt_mtx);
 		return NULL;
-
-	return rmx_taop(rt->rt_rmx);
+	}
+	
+	taop = rmx_taop(rt->rt_rmx);
+	lck_mtx_unlock(rt_mtx);
+	return (taop);
 }
 
 /*
@@ -1675,17 +1838,17 @@ tcp_lock(so, refcount, lr)
 		lck_mtx_lock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
 	else  {
-		panic("tcp_lock: so=%x NO PCB! lr=%x\n", so, lr_saved);
+		panic("tcp_lock: so=%p NO PCB! lr=%x\n", so, lr_saved);
 		lck_mtx_lock(so->so_proto->pr_domain->dom_mtx);
 	}
 
 	if (so->so_usecount < 0)
-		panic("tcp_lock: so=%x so_pcb=%x lr=%x ref=%x\n",
+		panic("tcp_lock: so=%p so_pcb=%p lr=%x ref=%x\n",
 	 	so, so->so_pcb, lr_saved, so->so_usecount);
 
 	if (refcount)
 		so->so_usecount++;
-	so->lock_lr[so->next_lock_lr] = (u_int32_t *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = (u_int32_t)lr_saved;
 	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
@@ -1702,19 +1865,19 @@ tcp_unlock(so, refcount, lr)
 	else lr_saved = lr;
 
 #ifdef MORE_TCPLOCK_DEBUG
-	printf("tcp_unlock: so=%x sopcb=%x lock=%x ref=%x lr=%x\n", 
+	printf("tcp_unlock: so=%p sopcb=%x lock=%x ref=%x lr=%x\n", 
 		so, so->so_pcb, ((struct inpcb *)so->so_pcb)->inpcb_mtx, so->so_usecount, lr_saved);
 #endif
 	if (refcount)
 		so->so_usecount--;
 
 	if (so->so_usecount < 0)
-		panic("tcp_unlock: so=%x usecount=%x\n", so, so->so_usecount);	
+		panic("tcp_unlock: so=%p usecount=%x\n", so, so->so_usecount);	
 	if (so->so_pcb == NULL) 
-		panic("tcp_unlock: so=%x NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
+		panic("tcp_unlock: so=%p NO PCB usecount=%x lr=%x\n", so, so->so_usecount, lr_saved);
 	else {
 		lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
-		so->unlock_lr[so->next_unlock_lr] = (u_int *)lr_saved;
+		so->unlock_lr[so->next_unlock_lr] = (u_int32_t)lr_saved;
 		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
@@ -1722,19 +1885,69 @@ tcp_unlock(so, refcount, lr)
 }
 
 lck_mtx_t *
-tcp_getlock(so, locktype)
-	struct socket *so;
-	int locktype;
+tcp_getlock(
+	struct socket *so,
+	__unused int locktype)
 {
 	struct inpcb *inp = sotoinpcb(so);
 
 	if (so->so_pcb)  {
 		if (so->so_usecount < 0)
-			panic("tcp_getlock: so=%x usecount=%x\n", so, so->so_usecount);	
+			panic("tcp_getlock: so=%p usecount=%x\n", so, so->so_usecount);	
 		return(inp->inpcb_mtx);
 	}
 	else {
-		panic("tcp_getlock: so=%x NULL so_pcb\n", so);
+		panic("tcp_getlock: so=%p NULL so_pcb\n", so);
 		return (so->so_proto->pr_domain->dom_mtx);
 	}
 }
+long
+tcp_sbspace(struct tcpcb *tp)
+{
+	struct sockbuf *sb = &tp->t_inpcb->inp_socket->so_rcv;
+	long space, newspace;
+
+	space =  ((long) lmin((sb->sb_hiwat - sb->sb_cc), 
+		(sb->sb_mbmax - sb->sb_mbcnt)));
+
+#if TRAFFIC_MGT
+	if (tp->t_inpcb->inp_socket->so_traffic_mgt_flags & TRAFFIC_MGT_SO_BACKGROUND) {
+		if (tcp_background_io_enabled &&
+			tp->t_inpcb->inp_socket->so_traffic_mgt_flags & TRAFFIC_MGT_SO_BG_SUPPRESSED) {
+			tp->t_flags |= TF_RXWIN0SENT;
+			return 0; /* Triggers TCP window closing by responding there is no space */
+		}
+	}
+#endif /* TRAFFIC_MGT */
+
+	/* Avoid inscreasing window size if the current window
+	 * is already very low, we could be in "persist" mode and
+	 * we could break some apps (see rdar://5409343)
+	 */
+
+	if (space < tp->t_maxseg) 
+		return space;
+
+	/* Clip window size for slower link */ 
+
+	if (((tp->t_flags & TF_SLOWLINK) != 0) && slowlink_wsize > 0 )	
+		return lmin(space, slowlink_wsize);
+
+	/*
+	 * Check for ressources constraints before over-ajusting the amount of space we can
+	 * advertise in the TCP window size updates.
+	 */
+
+	if (sbspace_factor && (tp->t_inpcb->inp_pcbinfo->ipi_count < tcp_sockthreshold) &&
+      		    (total_mb_cnt / 8) < (mbstat.m_clusters / sbspace_factor)) {
+		if (space < (long)(sb->sb_maxused - sb->sb_cc)) {/* make sure we don't constrain the window if we have enough ressources */
+			space = (long) lmax((sb->sb_maxused - sb->sb_cc), tp->rcv_maxbyps);
+		}
+		newspace = (long) lmax(((long)sb->sb_maxused - sb->sb_cc), (long)tp->rcv_maxbyps);
+
+		if (newspace > space)
+			space = newspace;
+	}
+	return space;
+}
+/* DSEP Review Done pl-20051213-v02 @3253,@3391,@3400 */
diff --git a/bsd/netinet/tcp_timer.c b/bsd/netinet/tcp_timer.c
index effeffde5..739e2816b 100644
--- a/bsd/netinet/tcp_timer.c
+++ b/bsd/netinet/tcp_timer.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
@@ -88,9 +94,15 @@
 #endif
 #include <sys/kdebug.h>
 
+extern void postevent(struct socket *, struct sockbuf *,
+                                               int);
 #define DBG_FNC_TCP_FAST	NETDBG_CODE(DBG_NETTCP, (5 << 8))
 #define DBG_FNC_TCP_SLOW	NETDBG_CODE(DBG_NETTCP, (5 << 8) | 1)
 
+static int 	background_io_trigger = 5;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, background_io_trigger, CTLFLAG_RW,
+    &background_io_trigger, 0, "Background IO Trigger Setting");
+
 /*
  * NOTE - WARNING
  *
@@ -101,16 +113,17 @@
 static int
 sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
 {
+#pragma unused(arg1, arg2)
 	int error, s, tt;
 
 	tt = *(int *)oidp->oid_arg1;
-	s = tt * 1000 / hz;
+	s = tt * 1000 / TCP_RETRANSHZ;;
 
 	error = sysctl_handle_int(oidp, &s, 0, req);
 	if (error || !req->newptr)
 		return (error);
 
-	tt = s * hz / 1000;
+	tt = s * TCP_RETRANSHZ / 1000;
 	if (tt < 1)
 		return (EINVAL);
 
@@ -130,11 +143,6 @@ int	tcp_keepintvl;
 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl, CTLTYPE_INT|CTLFLAG_RW,
     &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
 
-int	tcp_delacktime;
-SYSCTL_PROC(_net_inet_tcp, TCPCTL_DELACKTIME, delacktime,
-    CTLTYPE_INT|CTLFLAG_RW, &tcp_delacktime, 0, sysctl_msec_to_ticks, "I",
-    "Time before a delayed ACK is sent");
- 
 int	tcp_msl;
 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl, CTLTYPE_INT|CTLFLAG_RW,
     &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
@@ -144,6 +152,7 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, always_keepalive, CTLFLAG_RW,
     &always_keepalive , 0, "Assume SO_KEEPALIVE on all TCP connections");
 
 static int	tcp_keepcnt = TCPTV_KEEPCNT;
+static int	tcp_gc_done = FALSE;	/* perfromed garbage collection of "used" sockets */
 	/* max idle probes */
 int	tcp_maxpersistidle;
 	/* max idle time in persist */
@@ -154,23 +163,34 @@ int		cur_tw_slot = 0;
 
 u_long		*delack_bitmask;
 
+void	add_to_time_wait_locked(struct tcpcb *tp);
+void	add_to_time_wait(struct tcpcb *tp) ;
+
 
-void	add_to_time_wait_locked(tp) 
-	struct tcpcb	*tp;
+void	add_to_time_wait_locked(struct tcpcb *tp) 
 {
 	int		tw_slot;
+    struct inpcbinfo *pcbinfo	= &tcbinfo;
 
 	/* pcb list should be locked when we get here */	
-#if 0
-	lck_mtx_assert(tp->t_inpcb->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
-#endif
+	lck_rw_assert(pcbinfo->mtx, LCK_RW_ASSERT_EXCLUSIVE);
 
 	LIST_REMOVE(tp->t_inpcb, inp_list);
 
-	if (tp->t_timer[TCPT_2MSL] == 0) 
+	if (tp->t_timer[TCPT_2MSL] <= 0) 
 	    tp->t_timer[TCPT_2MSL] = 1;
 
-	tp->t_rcvtime += tp->t_timer[TCPT_2MSL] & (N_TIME_WAIT_SLOTS - 1);
+	/*
+	 * Because we're pulling this pcb out of the main TCP pcb list,
+	 * we need to recalculate the TCPT_2MSL timer value for tcp_slowtimo
+	 * higher timer granularity.
+	 */
+
+	tp->t_timer[TCPT_2MSL] = (tp->t_timer[TCPT_2MSL] / TCP_RETRANSHZ) * PR_SLOWHZ;
+	tp->t_rcvtime = (tp->t_rcvtime / TCP_RETRANSHZ) * PR_SLOWHZ;
+
+	tp->t_rcvtime += tp->t_timer[TCPT_2MSL] & (N_TIME_WAIT_SLOTS - 1); 
+
 	tw_slot = (tp->t_timer[TCPT_2MSL] & (N_TIME_WAIT_SLOTS - 1)) + cur_tw_slot; 
 	if (tw_slot >= N_TIME_WAIT_SLOTS)
 	    tw_slot -= N_TIME_WAIT_SLOTS;
@@ -178,8 +198,7 @@ void	add_to_time_wait_locked(tp)
 	LIST_INSERT_HEAD(&time_wait_slots[tw_slot], tp->t_inpcb, inp_list);
 }
 
-void	add_to_time_wait(tp) 
-	struct tcpcb	*tp;
+void	add_to_time_wait(struct tcpcb *tp) 
 {
     	struct inpcbinfo *pcbinfo		= &tcbinfo;
 	
@@ -201,66 +220,154 @@ void	add_to_time_wait(tp)
 void
 tcp_fasttimo()
 {
-    struct inpcb *inp, *inpnxt;
+    struct inpcb *inp;
     register struct tcpcb *tp;
+    struct socket *so;
+#if TCPDEBUG
+    int ostate;
+#endif
 
 
     struct inpcbinfo *pcbinfo	= &tcbinfo;
 
-    int delack_checked = 0, delack_done = 0;
+    int delack_done = 0;
 
     KERNEL_DEBUG(DBG_FNC_TCP_FAST | DBG_FUNC_START, 0,0,0,0,0);
 
-    if (tcp_delack_enabled == 0) 
-	return;
 
     lck_rw_lock_shared(pcbinfo->mtx);
 
     /* Walk the list of valid tcpcbs and send ACKS on the ones with DELACK bit set */
 
-    for (inp = tcb.lh_first; inp != NULL; inp = inpnxt) {
-	inpnxt = inp->inp_list.le_next;
-	/* NOTE: it's OK to check the tp because the pcb can't be removed while we hold pcbinfo->mtx) */
-	if ((tp = (struct tcpcb *)inp->inp_ppcb) && (tp->t_flags & TF_DELACK)) {
-		if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) 
-			continue;
-		tcp_lock(inp->inp_socket, 1, 0);
-		if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
-			tcp_unlock(inp->inp_socket, 1, 0);
-			continue;
-		}
-		if (tp->t_flags & TF_DELACK) {
-			delack_done++;
-			tp->t_flags &= ~TF_DELACK;
-			tp->t_flags |= TF_ACKNOW;
-			tcpstat.tcps_delack++;
-			(void) tcp_output(tp);
-		}
-		tcp_unlock(inp->inp_socket, 1, 0);
-    	}
+    LIST_FOREACH(inp, &tcb, inp_list) {
+
+	so = inp->inp_socket;
+
+	if (so == &tcbinfo.nat_dummy_socket)
+		continue;
+
+	if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) 
+		continue;
+
+	tcp_lock(so, 1, 0);
+
+	if ((in_pcb_checkstate(inp, WNT_RELEASE,1) == WNT_STOPUSING)  && so->so_usecount == 1) {
+		tcp_unlock(so, 1, 0);
+		continue;
+	}
+
+	tp = intotcpcb(inp);
+
+	if (tp == 0 || tp->t_state == TCPS_LISTEN) {
+		tcp_unlock(so, 1, 0);
+		continue; 
+	}
+
+
+	/* Only run the retransmit timer in that case */
+	if (tp->t_timer[0] && --tp->t_timer[0] == 0) {
+		tp = tcp_timers(tp, 0);
+		if (tp == NULL)
+			goto tpgone;
+	}
+
+	/* TCP pcb  timers following the tcp_now clock rate */
+
+	tp->t_rcvtime++;
+	tp->t_starttime++;
+	if (tp->t_rtttime)
+		tp->t_rtttime++;	
+
+	/*
+	 * Process delayed acks (if enabled) according to PR_FASTHZ, not the retrans timer
+	 */
+
+	if (tcp_delack_enabled && (tcp_now % (TCP_RETRANSHZ/PR_FASTHZ)) && tp->t_flags & TF_DELACK) {
+		delack_done++;
+		tp->t_flags &= ~TF_DELACK;
+		tp->t_flags |= TF_ACKNOW;
+		tcpstat.tcps_delack++;
+		tp->t_unacksegs = 0;
+		(void) tcp_output(tp);
+	}
+tpgone:
+	tcp_unlock(so, 1, 0);
     }
-    KERNEL_DEBUG(DBG_FNC_TCP_FAST | DBG_FUNC_END, delack_checked, delack_done, tcpstat.tcps_delack,0,0);
+    KERNEL_DEBUG(DBG_FNC_TCP_FAST | DBG_FUNC_END, delack_done, 0, tcpstat.tcps_delack,0,0);
     lck_rw_done(pcbinfo->mtx);
+
+    tcp_now++;
+    timeout(tcp_fasttimo, 0, hz/TCP_RETRANSHZ);
 }
 
-/*
- * Tcp protocol timeout routine called every 500 ms.
- * Updates the timers in all active tcb's and
- * causes finite state machine actions if timers expire.
- */
+void 
+tcp_garbage_collect(inp, istimewait)
+	struct inpcb *inp;
+	int istimewait;
+{
+	struct socket *so;
+	struct tcpcb *tp;
+
+
+		if (inp->inp_socket == &tcbinfo.nat_dummy_socket)
+				return;
+
+
+		if (!lck_mtx_try_lock(inp->inpcb_mtx)) /* skip if still in use */
+			return;
+
+		so = inp->inp_socket;
+		tp = intotcpcb(inp);
+
+		if ((so->so_usecount == 1) && 
+		     	(so->so_flags & SOF_OVERFLOW)) {
+				in_pcbdetach(inp);
+				so->so_usecount--; 
+				lck_mtx_unlock(inp->inpcb_mtx);
+				return;
+		}
+		else {
+			if (inp->inp_wantcnt != WNT_STOPUSING) {
+				lck_mtx_unlock(inp->inpcb_mtx);
+				return;
+			}
+		}
+
+
+		if (so->so_usecount == 0) 
+			in_pcbdispose(inp);
+		else {
+			/* Special case:
+			 * - Check for embryonic socket stuck on listener queue (4023660)
+			 * - overflowed socket dropped from the listening queue
+			 * and dispose of remaining reference
+			 */
+			if ((so->so_usecount == 1) && 
+			  (((tp->t_state == TCPS_CLOSED) && (so->so_head != NULL) && (so->so_state & SS_INCOMP)) || 
+			     	(istimewait && (so->so_flags & SOF_OVERFLOW)))) {
+					so->so_usecount--; 
+					in_pcbdispose(inp);
+			} else
+				lck_mtx_unlock(inp->inpcb_mtx);
+		}
+}
+
+static int bg_cnt = 0;
+#define BG_COUNTER_MAX 3
+
 void
 tcp_slowtimo()
 {
-	struct inpcb *inp, *inpnxt;
+	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct socket *so;
 	int i;
 #if TCPDEBUG
 	int ostate;
 #endif
-#if KDEBUG
-	static int tws_checked;
-#endif
+
+	static int tws_checked = 0;
+
 	struct inpcbinfo *pcbinfo		= &tcbinfo;
 
 	KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0,0,0,0,0);
@@ -269,18 +376,16 @@ tcp_slowtimo()
 
 	lck_rw_lock_shared(pcbinfo->mtx);
 
-	/*
-	 * Search through tcb's and update active timers.
-	 */
-	for (inp = tcb.lh_first; inp != NULL; inp = inpnxt) {
-		inpnxt = inp->inp_list.le_next;
+	bg_cnt++;
 
-		so = inp->inp_socket;
+    	LIST_FOREACH(inp, &tcb, inp_list) {
 
+		so = inp->inp_socket;
+	
 		if (so == &tcbinfo.nat_dummy_socket)
-				continue;
+			continue;
 
-		if (in_pcb_checkstate(inp, WNT_ACQUIRE,0) == WNT_STOPUSING) 
+		if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) 
 			continue;
 
 		tcp_lock(so, 1, 0);
@@ -295,44 +400,109 @@ tcp_slowtimo()
 			continue; 
 		}
 
-		for (i = 0; i < TCPT_NTIMERS; i++) {
-			if (tp->t_timer[i] && --tp->t_timer[i] == 0) {
+		tp = intotcpcb(inp);
+
+		if (tp == 0 || tp->t_state == TCPS_LISTEN) 
+			goto tpgone;
+
+#if TRAFFIC_MGT
+	        if (so->so_traffic_mgt_flags & TRAFFIC_MGT_SO_BACKGROUND && 
+	        	bg_cnt > BG_COUNTER_MAX) {
+			u_int32_t	curr_recvtotal = tcpstat.tcps_rcvtotal;
+			u_int32_t	curr_bg_recvtotal = tcpstat.tcps_bg_rcvtotal;
+			u_int32_t	bg_recvdiff = curr_bg_recvtotal - tp->bg_recv_snapshot;
+			u_int32_t	tot_recvdiff = curr_recvtotal - tp->tot_recv_snapshot;
+			u_int32_t	fg_recv_change = tot_recvdiff - bg_recvdiff;
+			u_int32_t	recv_change;
+			
+			if (!(so->so_traffic_mgt_flags & TRAFFIC_MGT_SO_BG_SUPPRESSED)) {
+				if (tot_recvdiff) 
+					recv_change = (fg_recv_change * 100) / tot_recvdiff;
+				else 
+					recv_change = 0;
+
+				if (recv_change > background_io_trigger) {
+					so->so_traffic_mgt_flags |= TRAFFIC_MGT_SO_BG_SUPPRESSED;
+				}
+				
+				tp->tot_recv_snapshot = curr_recvtotal;
+				tp->bg_recv_snapshot = curr_bg_recvtotal;
+			}
+			else {	// SUPPRESSED
+				// this allows for bg traffic to subside before we start measuring total traffic change
+				if (tot_recvdiff)
+					recv_change = (bg_recvdiff * 100) / tot_recvdiff;
+				else
+					recv_change = 0;
+					
+				if (recv_change < background_io_trigger) {
+					// Draconian for now: if there is any change at all, keep suppressed
+					if (!tot_recvdiff) {
+						so->so_traffic_mgt_flags &= ~TRAFFIC_MGT_SO_BG_SUPPRESSED;
+						tp->t_unacksegs = 0;
+						(void) tcp_output(tp);	// open window
+					}
+				}
+
+				tp->tot_recv_snapshot = curr_recvtotal;
+				tp->bg_recv_snapshot = curr_bg_recvtotal;
+			}
+		}
+#endif /* TRAFFIC_MGT */
+
+		for (i = 1; i < TCPT_NTIMERS; i++) {
+			if (tp->t_timer[i] != 0) {
+				tp->t_timer[i] -= TCP_RETRANSHZ/PR_SLOWHZ;
+			       	if (tp->t_timer[i] <=  0) {
 #if TCPDEBUG
-				ostate = tp->t_state;
+					ostate = tp->t_state;
 #endif
-				tp = tcp_timers(tp, i);
-				if (tp == NULL)
-					goto tpgone;
+
+					tp->t_timer[i] = 0; /* account for granularity change between tcp_now and slowtimo */
+					tp = tcp_timers(tp, i);
+					if (tp == NULL)
+						goto tpgone;
 #if TCPDEBUG
-				if (tp->t_inpcb->inp_socket->so_options
-				    & SO_DEBUG)
-					tcp_trace(TA_USER, ostate, tp,
-						  (void *)0,
-						  (struct tcphdr *)0,
-						  PRU_SLOWTIMO);
+					if (tp->t_inpcb->inp_socket->so_options
+					    & SO_DEBUG)
+						tcp_trace(TA_USER, ostate, tp,
+							  (void *)0,
+							  (struct tcphdr *)0,
+							  PRU_SLOWTIMO);
 #endif
+				}
 			}
 		}
-		tp->t_rcvtime++;
-		tp->t_starttime++;
-		if (tp->t_rtttime)
-			tp->t_rtttime++;	
 tpgone:
 		tcp_unlock(so, 1, 0);
 	}
+	
+	if (bg_cnt > 3) 
+		bg_cnt = 0;
 
-#if KDEBUG
-	tws_checked = 0;
-#endif
-	KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_NONE, tws_checked,0,0,0,0);
+	/* Second part of tcp_slowtimo: garbage collect socket/tcpcb
+	 * We need to acquire the list lock exclusively to do this
+	 */
+
+	if (lck_rw_lock_shared_to_exclusive(pcbinfo->mtx) == FALSE) {
+		if (tcp_gc_done == TRUE) {	/* don't sweat it this time. cleanup was done last time */
+			tcp_gc_done = FALSE;
+			KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked, cur_tw_slot,0,0,0);
+			return; /* Upgrade failed and lost lock - give up this time. */
+		}
+		lck_rw_lock_exclusive(pcbinfo->mtx);	/* Upgrade failed, lost lock now take it again exclusive */
+	}
+	tcp_gc_done = TRUE;
 
 	/*
 	 * Process the items in the current time-wait slot
 	 */
+#if  KDEBUG
+	tws_checked = 0;
+#endif
+	KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_NONE, tws_checked,0,0,0,0);
 
-	for (inp = time_wait_slots[cur_tw_slot].lh_first; inp; inp = inpnxt)
-	{
-		inpnxt = inp->inp_list.le_next;
+    	LIST_FOREACH(inp, &time_wait_slots[cur_tw_slot], inp_list) {
 #if KDEBUG
 	        tws_checked++;
 #endif
@@ -346,12 +516,9 @@ tpgone:
 			goto twunlock;
 
 		tp = intotcpcb(inp);
-		if (tp == NULL) { /* tp already closed, remove from list */
-#if TEMPDEBUG
-			printf("tcp_slowtimo: tp is null in time-wait slot!\n");
-#endif
+		if (tp == NULL)  /* tp already closed, remove from list */
 			goto twunlock;
-		}
+
 		if (tp->t_timer[TCPT_2MSL] >= N_TIME_WAIT_SLOTS) {
 		    tp->t_timer[TCPT_2MSL] -= N_TIME_WAIT_SLOTS;
 		    tp->t_rcvtime += N_TIME_WAIT_SLOTS;
@@ -359,73 +526,26 @@ tpgone:
 		else
 		    tp->t_timer[TCPT_2MSL] = 0;
 
-		if (tp->t_timer[TCPT_2MSL] == 0)  
-		    tp = tcp_timers(tp, TCPT_2MSL);	/* tp can be returned null if tcp_close is called */
+		if (tp->t_timer[TCPT_2MSL] == 0)  {
+
+			/* That pcb is ready for a close */	
+			tcp_free_sackholes(tp);
+			tp = tcp_close(tp);
+		}
 twunlock:
 		tcp_unlock(inp->inp_socket, 1, 0);
 	}
 
-	if (lck_rw_lock_shared_to_exclusive(pcbinfo->mtx) != 0)
-		lck_rw_lock_exclusive(pcbinfo->mtx);	/* Upgrade failed, lost lock no take it again exclusive */
-
-
-	for (inp = tcb.lh_first; inp != NULL; inp = inpnxt) {
-		inpnxt = inp->inp_list.le_next;
-		/* Ignore nat/SharedIP dummy pcbs */
-		if (inp->inp_socket == &tcbinfo.nat_dummy_socket)
-				continue;
-
-		if (inp->inp_wantcnt != WNT_STOPUSING) 
-			continue;
 
-		so = inp->inp_socket;
-		if (!lck_mtx_try_lock(inp->inpcb_mtx)) {/* skip if in use */
-#if TEMPDEBUG
-			printf("tcp_slowtimo so=%x STOPUSING but locked...\n", so);
-#endif
-			continue;
-		}
-
-		if (so->so_usecount == 0) 
-			in_pcbdispose(inp);
-		else {
-			tp = intotcpcb(inp);
-			/* Check for embryonic socket stuck on listener queue (4023660) */
-			if ((so->so_usecount == 1) && (tp->t_state == TCPS_CLOSED) &&
-		       	    (so->so_head != NULL) && (so->so_state & SS_INCOMP)) {
-				so->so_usecount--; 
-				in_pcbdispose(inp);
-			} else
-				lck_mtx_unlock(inp->inpcb_mtx);
-		}
+    	LIST_FOREACH(inp, &tcb, inp_list) {
+		tcp_garbage_collect(inp, 0);
 	}
 
 	/* Now cleanup the time wait ones */
-	for (inp = time_wait_slots[cur_tw_slot].lh_first; inp; inp = inpnxt)
-	{
-		inpnxt = inp->inp_list.le_next;
-
-		if (inp->inp_wantcnt != WNT_STOPUSING) 
-			continue;
-
-		so = inp->inp_socket;
-		if (!lck_mtx_try_lock(inp->inpcb_mtx)) /* skip if in use */
-			continue;
-		if (so->so_usecount == 0)  
-			in_pcbdispose(inp);
-		else  {
-			tp = intotcpcb(inp);
-			/* Check for embryonic socket stuck on listener queue (4023660) */
-			if ((so->so_usecount == 1) && (tp->t_state == TCPS_CLOSED) &&
-		       	    (so->so_head != NULL) && (so->so_state & SS_INCOMP)) {
-				so->so_usecount--; 
-				in_pcbdispose(inp);
-			} else
-				lck_mtx_unlock(inp->inpcb_mtx);
-		}
+    	LIST_FOREACH(inp, &time_wait_slots[cur_tw_slot], inp_list) {
+		tcp_garbage_collect(inp, 1);
 	}
 
-	tcp_now++;
 	if (++cur_tw_slot >= N_TIME_WAIT_SLOTS)
 		cur_tw_slot = 0;
 	
@@ -464,6 +584,7 @@ tcp_timers(tp, timer)
 {
 	register int rexmt;
 	struct socket *so_tmp;
+	struct inpcbinfo *pcbinfo		= &tcbinfo;
 	struct tcptemp *t_template;
 
 #if TCPDEBUG
@@ -481,15 +602,17 @@ tcp_timers(tp, timer)
 	/*
 	 * 2 MSL timeout in shutdown went off.  If we're closed but
 	 * still waiting for peer to close and connection has been idle
-	 * too long, or if 2MSL time is up from TIME_WAIT, delete connection
-	 * control block.  Otherwise, check again in a bit.
+	 * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2,
+	 * delete connection control block.
+	 * Otherwise, (this case shouldn't happen) check again in a bit
+	 * we keep the socket in the main list in that case.
 	 */
 	case TCPT_2MSL:
 		tcp_free_sackholes(tp);
 		if (tp->t_state != TCPS_TIME_WAIT &&
-		    tp->t_rcvtime <= tcp_maxidle) {
+		    tp->t_state != TCPS_FIN_WAIT_2 &&
+		    tp->t_rcvtime < tcp_maxidle) {
 			tp->t_timer[TCPT_2MSL] = (unsigned long)tcp_keepintvl;
-			add_to_time_wait_locked(tp);
 		}
 		else {
 			tp = tcp_close(tp);
@@ -530,7 +653,7 @@ tcp_timers(tp, timer)
 				  tp->t_flags |= TF_WASFRECOVERY;
 			else
 				  tp->t_flags &= ~TF_WASFRECOVERY;
-			tp->t_badrxtwin = tcp_now + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
+			tp->t_badrxtwin = tcp_now  + (tp->t_srtt >> (TCP_RTT_SHIFT)); 
 		}
 		tcpstat.tcps_rexmttimeo++;
 		if (tp->t_state == TCPS_SYN_SENT)
@@ -612,7 +735,9 @@ tcp_timers(tp, timer)
 			win = 2;
 		tp->snd_cwnd = tp->t_maxseg;
 		tp->snd_ssthresh = win * tp->t_maxseg;
+		tp->t_bytes_acked = 0;
 		tp->t_dupacks = 0;
+		tp->t_unacksegs = 0;
 		}
 		EXIT_FASTRECOVERY(tp);
 		(void) tcp_output(tp);
@@ -642,6 +767,7 @@ tcp_timers(tp, timer)
 		}
 		tcp_setpersist(tp);
 		tp->t_force = 1;
+		tp->t_unacksegs = 0;
 		(void) tcp_output(tp);
 		tp->t_force = 0;
 		break;
@@ -656,7 +782,7 @@ tcp_timers(tp, timer)
 			goto dropit;
 		if ((always_keepalive ||
 		    tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) &&
-		    tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2) {
+		    (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
 		    	if (tp->t_rcvtime >= TCP_KEEPIDLE(tp) + (unsigned long)tcp_maxidle)
 				goto dropit;
 			/*
@@ -676,7 +802,7 @@ tcp_timers(tp, timer)
 			if (t_template) {
 				tcp_respond(tp, t_template->tt_ipgen,
 				    &t_template->tt_t, (struct mbuf *)NULL,
-				    tp->rcv_nxt, tp->snd_una - 1, 0);
+				    tp->rcv_nxt, tp->snd_una - 1, 0, NULL);
 				(void) m_free(dtom(t_template));
 			}
 			tp->t_timer[TCPT_KEEP] = tcp_keepintvl;
diff --git a/bsd/netinet/tcp_timer.h b/bsd/netinet/tcp_timer.h
index e4979a078..ef29f41a3 100644
--- a/bsd/netinet/tcp_timer.h
+++ b/bsd/netinet/tcp_timer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -111,22 +117,23 @@
 /*
  * Time constants.
  */
-#define	TCPTV_MSL	( 30*PR_SLOWHZ)		/* max seg lifetime (hah!) */
+#define	TCPTV_MSL	( 15*TCP_RETRANSHZ)		/* max seg lifetime (hah!) */
 #define	TCPTV_SRTTBASE	0			/* base roundtrip time;
 						   if 0, no idea yet */
-#define	TCPTV_RTOBASE	(  3*PR_SLOWHZ)		/* assumed RTO if no info */
-#define	TCPTV_SRTTDFLT	(  3*PR_SLOWHZ)		/* assumed RTT if no info */
+#define	TCPTV_RTOBASE	(  1*TCP_RETRANSHZ)		/* assumed RTO if no info */
+#define	TCPTV_SRTTDFLT	(  1*TCP_RETRANSHZ)		/* assumed RTT if no info */
 
-#define	TCPTV_PERSMIN	(  5*PR_SLOWHZ)		/* retransmit persistence */
-#define	TCPTV_PERSMAX	( 60*PR_SLOWHZ)		/* maximum persist interval */
+#define	TCPTV_PERSMIN	(  5*TCP_RETRANSHZ)		/* retransmit persistence */
+#define	TCPTV_PERSMAX	( 60*TCP_RETRANSHZ)		/* maximum persist interval */
 
-#define	TCPTV_KEEP_INIT	( 75*PR_SLOWHZ)		/* initial connect keep alive */
-#define	TCPTV_KEEP_IDLE	(120*60*PR_SLOWHZ)	/* dflt time before probing */
-#define	TCPTV_KEEPINTVL	( 75*PR_SLOWHZ)		/* default probe interval */
+#define	TCPTV_KEEP_INIT	( 75*TCP_RETRANSHZ)		/* initial connect keep alive */
+#define	TCPTV_KEEP_IDLE	(120*60*TCP_RETRANSHZ)	/* dflt time before probing */
+#define	TCPTV_KEEPINTVL	( 75*TCP_RETRANSHZ)		/* default probe interval */
 #define	TCPTV_KEEPCNT	8			/* max probes before drop */
 
-#define	TCPTV_MIN	(  1*PR_SLOWHZ)		/* minimum allowable value */
-#define	TCPTV_REXMTMAX	( 64*PR_SLOWHZ)		/* max allowable REXMT value */
+//#define	TCPTV_MIN	(  3*TCP_RETRANSHZ)		/* minimum allowable value */
+#define	TCPTV_MIN	(1) 	/* minimum allowable value */
+#define	TCPTV_REXMTMAX	( 64*TCP_RETRANSHZ)		/* max allowable REXMT value */
 
 #define TCPTV_TWTRUNC	8			/* RTO factor to truncate TW */
 
@@ -134,8 +141,6 @@
 
 #define	TCP_MAXRXTSHIFT	12			/* maximum retransmits */
 
-#define	TCPTV_DELACK	PR_FASTHZ 		/* 125ms timeout */
-
 #ifdef	TCPTIMERS
 static char *tcptimers[] =
     { "REXMT", "PERSIST", "KEEP", "2MSL" };
@@ -161,7 +166,6 @@ extern int tcp_keepinit;		/* time to establish connection */
 extern int tcp_keepidle;		/* time before keepalive probes begin */
 extern int tcp_keepintvl;		/* time between keepalive probes */
 extern int tcp_maxidle;			/* time to drop after starting probes */
-extern int tcp_delacktime;		/* time before sending a delayed ACK */
 extern int tcp_maxpersistidle;
 extern int tcp_msl;
 extern int tcp_ttl;			/* time to live for TCP segs */
diff --git a/bsd/netinet/tcp_usrreq.c b/bsd/netinet/tcp_usrreq.c
index 2a5ab6988..ed796e474 100644
--- a/bsd/netinet/tcp_usrreq.c
+++ b/bsd/netinet/tcp_usrreq.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1993
@@ -114,6 +120,30 @@ static struct tcpcb *
 static struct tcpcb *
 		tcp_usrclosed(struct tcpcb *);
 
+__private_extern__ int	tcp_win_scale = 3;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, win_scale_factor, CTLFLAG_RW,
+    &tcp_win_scale, 0, "Window scaling factor");
+
+static u_int32_t tcps_in_sw_cksum;
+SYSCTL_UINT(_net_inet_tcp, OID_AUTO, in_sw_cksum, CTLFLAG_RD,
+    &tcps_in_sw_cksum, 0,
+    "Number of received packets checksummed in software");
+
+static u_int64_t tcps_in_sw_cksum_bytes;
+SYSCTL_QUAD(_net_inet_tcp, OID_AUTO, in_sw_cksum_bytes, CTLFLAG_RD,
+    &tcps_in_sw_cksum_bytes,
+    "Amount of received data checksummed in software");
+
+static u_int32_t tcps_out_sw_cksum;
+SYSCTL_UINT(_net_inet_tcp, OID_AUTO, out_sw_cksum, CTLFLAG_RD,
+    &tcps_out_sw_cksum, 0,
+    "Number of transmitted packets checksummed in software");
+
+static u_int64_t tcps_out_sw_cksum_bytes;
+SYSCTL_QUAD(_net_inet_tcp, OID_AUTO, out_sw_cksum_bytes, CTLFLAG_RD,
+    &tcps_out_sw_cksum_bytes,
+    "Amount of transmitted data checksummed in software");
+
 #if TCPDEBUG
 #define	TCPDEBUG0	int ostate = 0
 #define	TCPDEBUG1()	ostate = tp ? tp->t_state : 0
@@ -125,12 +155,22 @@ static struct tcpcb *
 #define	TCPDEBUG2(req)
 #endif
 
+__private_extern__ unsigned int	tcp_sockthreshold = 64;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, sockthreshold, CTLFLAG_RW, 
+    &tcp_sockthreshold , 0, "TCP Socket size increased if less than threshold");
+
 /*
  * TCP attaches to socket via pru_attach(), reserving space,
  * and an internet control block.
+ *
+ * Returns:	0			Success
+ *		EISCONN
+ *	tcp_attach:ENOBUFS
+ *	tcp_attach:ENOMEM
+ *	tcp_attach:???			[IPSEC specific]
  */
 static int
-tcp_usr_attach(struct socket *so, int proto, struct proc *p)
+tcp_usr_attach(struct socket *so, __unused int proto, struct proc *p)
 {
 	int error;
 	struct inpcb *inp = sotoinpcb(so);
@@ -201,6 +241,17 @@ out:
 
 /*
  * Give the socket an address.
+ *
+ * Returns:	0			Success
+ *		EINVAL			Invalid argument [COMMON_START]
+ *		EAFNOSUPPORT		Address family not supported
+ *	in_pcbbind:EADDRNOTAVAIL	Address not available.
+ *	in_pcbbind:EINVAL		Invalid argument
+ *	in_pcbbind:EAFNOSUPPORT		Address family not supported [notdef]
+ *	in_pcbbind:EACCES		Permission denied
+ *	in_pcbbind:EADDRINUSE		Address in use
+ *	in_pcbbind:EAGAIN		Resource unavailable, try again
+ *	in_pcbbind:EPERM		Operation not permitted
  */
 static int
 tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
@@ -212,6 +263,11 @@ tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 
 	COMMON_START();
 
+	if (nam->sa_family != 0 && nam->sa_family != AF_INET) {
+		error = EAFNOSUPPORT;
+		goto out;
+	}
+
 	/*
 	 * Must check for multicast addresses and disallow binding
 	 * to them.
@@ -240,6 +296,11 @@ tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 
 	COMMON_START();
 
+	if (nam->sa_family != 0 && nam->sa_family != AF_INET6) {
+		error = EAFNOSUPPORT;
+		goto out;
+	}
+
 	/*
 	 * Must check for multicast addresses and disallow binding
 	 * to them.
@@ -274,6 +335,16 @@ tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 
 /*
  * Prepare to accept connections.
+ *
+ * Returns:	0			Success
+ *		EINVAL [COMMON_START]
+ *	in_pcbbind:EADDRNOTAVAIL	Address not available.
+ *	in_pcbbind:EINVAL		Invalid argument
+ *	in_pcbbind:EAFNOSUPPORT		Address family not supported [notdef]
+ *	in_pcbbind:EACCES		Permission denied
+ *	in_pcbbind:EADDRINUSE		Address in use
+ *	in_pcbbind:EAGAIN		Resource unavailable, try again
+ *	in_pcbbind:EPERM		Operation not permitted
  */
 static int
 tcp_usr_listen(struct socket *so, struct proc *p)
@@ -326,8 +397,24 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
 	struct tcpcb *tp;
 	struct sockaddr_in *sinp;
 
-	COMMON_START();
+	TCPDEBUG0;
+	if (inp == 0)
+		return EINVAL;
+	else if (inp->inp_state == INPCB_STATE_DEAD) {
+		if (so->so_error) {
+			error = so->so_error;
+			so->so_error = 0;
+			return error;
+		} else
+			return EINVAL;
+	}
+	tp = intotcpcb(inp);
+	TCPDEBUG1();
 
+	if (nam->sa_family != 0 && nam->sa_family != AF_INET) {
+		error = EAFNOSUPPORT;
+		goto out;
+	}
 	/*
 	 * Must disallow TCP ``connections'' to multicast addresses.
 	 */
@@ -338,9 +425,6 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
 		goto out;
 	}
 
-#ifndef __APPLE__
-	prison_remote_ip(p, 0, &sinp->sin_addr.s_addr);
-#endif
 
 	if ((error = tcp_connect(tp, nam, p)) != 0)
 		goto out;
@@ -359,6 +443,11 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
 
 	COMMON_START();
 
+	if (nam->sa_family != 0 && nam->sa_family != AF_INET6) {
+		error = EAFNOSUPPORT;
+		goto out;
+	}
+
 	/*
 	 * Must disallow TCP ``connections'' to multicast addresses.
 	 */
@@ -471,8 +560,23 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam)
 	COMMON_END(PRU_ACCEPT);
 }
 #endif /* INET6 */
+
 /*
  * Mark the connection as being incapable of further output.
+ *
+ * Returns:	0			Success
+ *		EINVAL [COMMON_START]
+ *	tcp_output:EADDRNOTAVAIL
+ *	tcp_output:ENOBUFS
+ *	tcp_output:EMSGSIZE
+ *	tcp_output:EHOSTUNREACH
+ *	tcp_output:ENETUNREACH
+ *	tcp_output:ENETDOWN
+ *	tcp_output:ENOMEM
+ *	tcp_output:EACCES
+ *	tcp_output:EMSGSIZE
+ *	tcp_output:ENOBUFS
+ *	tcp_output:???			[ignorable: mostly IPSEC/firewall/DLIL]
  */
 static int
 tcp_usr_shutdown(struct socket *so)
@@ -496,7 +600,7 @@ tcp_usr_shutdown(struct socket *so)
  * After a receive, possibly send window update to peer.
  */
 static int
-tcp_usr_rcvd(struct socket *so, int flags)
+tcp_usr_rcvd(struct socket *so, __unused int flags)
 {
 	int error = 0;
 	struct inpcb *inp = sotoinpcb(so);
@@ -516,6 +620,30 @@ tcp_usr_rcvd(struct socket *so, int flags)
  * pru_*() routines, the mbuf chains are our responsibility.  We
  * must either enqueue them or free them.  The other pru_* routines
  * generally are caller-frees.
+ *
+ * Returns:	0			Success
+ *		ECONNRESET
+ *		EINVAL
+ *		ENOBUFS
+ *	tcp_connect:EADDRINUSE		Address in use
+ *	tcp_connect:EADDRNOTAVAIL	Address not available.
+ *	tcp_connect:EINVAL		Invalid argument
+ *	tcp_connect:EAFNOSUPPORT	Address family not supported [notdef]
+ *	tcp_connect:EACCES		Permission denied
+ *	tcp_connect:EAGAIN		Resource unavailable, try again
+ *	tcp_connect:EPERM		Operation not permitted
+ *	tcp_output:EADDRNOTAVAIL
+ *	tcp_output:ENOBUFS
+ *	tcp_output:EMSGSIZE
+ *	tcp_output:EHOSTUNREACH
+ *	tcp_output:ENETUNREACH
+ *	tcp_output:ENETDOWN
+ *	tcp_output:ENOMEM
+ *	tcp_output:EACCES
+ *	tcp_output:EMSGSIZE
+ *	tcp_output:ENOBUFS
+ *	tcp_output:???			[ignorable: mostly IPSEC/firewall/DLIL]
+ *	tcp6_connect:???		[IPV6 only]
  */
 static int
 tcp_usr_send(struct socket *so, int flags, struct mbuf *m, 
@@ -561,7 +689,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 		m_freem(control);	/* empty control, just free it */
 	}
 	if(!(flags & PRUS_OOB)) {
-		sbappend(&so->so_snd, m);
+		sbappendstream(&so->so_snd, m);
 		if (nam && tp->t_state < TCPS_SYN_SENT) {
 			/*
 			 * Do implied connect if not yet connected,
@@ -610,7 +738,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 		 * of data past the urgent section.
 		 * Otherwise, snd_up should be one lower.
 		 */
-		sbappend(&so->so_snd, m);
+		sbappendstream(&so->so_snd, m);
 		if (nam && tp->t_state < TCPS_SYN_SENT) {
 			/*
 			 * Do implied connect if not yet connected,
@@ -659,6 +787,11 @@ tcp_usr_abort(struct socket *so)
 
 /*
  * Receive out-of-band data.
+ *
+ * Returns:	0			Success
+ *		EINVAL [COMMON_START]
+ *		EINVAL
+ *		EWOULDBLOCK
  */
 static int
 tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
@@ -714,6 +847,20 @@ struct pr_usrreqs tcp6_usrreqs = {
  * sending CC options and if the connection duration was < MSL, then
  * truncate the previous TIME-WAIT state and proceed.
  * Initialize connection parameters and enter SYN-SENT state.
+ *
+ * Returns:	0			Success
+ *		EADDRINUSE
+ *		EINVAL
+ *	in_pcbbind:EADDRNOTAVAIL	Address not available.
+ *	in_pcbbind:EINVAL		Invalid argument
+ *	in_pcbbind:EAFNOSUPPORT		Address family not supported [notdef]
+ *	in_pcbbind:EACCES		Permission denied
+ *	in_pcbbind:EADDRINUSE		Address in use
+ *	in_pcbbind:EAGAIN		Resource unavailable, try again
+ *	in_pcbbind:EPERM		Operation not permitted
+ *	in_pcbladdr:EINVAL		Invalid argument
+ *	in_pcbladdr:EAFNOSUPPORT	Address family not supported
+ *	in_pcbladdr:EADDRNOTAVAIL	Address not available
  */
 static int
 tcp_connect(tp, nam, p)
@@ -764,11 +911,11 @@ tcp_connect(tp, nam, p)
 
 		if (oinp != inp && (otp = intotcpcb(oinp)) != NULL &&
 		otp->t_state == TCPS_TIME_WAIT &&
-		    otp->t_starttime < tcp_msl &&
+		    otp->t_starttime < (u_long)tcp_msl &&
 		    (otp->t_flags & TF_RCVD_CC))
 			otp = tcp_close(otp);
 		else {
-			printf("tcp_connect: inp=%x err=EADDRINUSE\n", inp);
+			printf("tcp_connect: inp=%p err=EADDRINUSE\n", inp);
 			if (oinp != inp)
 				tcp_unlock(oinp->inp_socket, 1, 0);
 			return EADDRINUSE;
@@ -794,11 +941,24 @@ skip_oinp:
 	in_pcbrehash(inp);
 	lck_rw_done(inp->inp_pcbinfo->mtx);
 
-	/* Compute window scaling to request.  */
+	/* Compute window scaling to requesti according to sb_hiwat
+	 * or leave us some room to increase potentially increase the window size depending
+	 * on the default win scale
+	 */
 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
-	    (TCP_MAXWIN << tp->request_r_scale) < so->so_rcv.sb_hiwat)
+   	 (TCP_MAXWIN << tp->request_r_scale) < so->so_rcv.sb_hiwat)
 		tp->request_r_scale++;
 
+	/*
+	 * Inflate window size only if no setsockopt was performed on the recv sockbuf and
+	 * if we're not over our number of active pcbs.
+	 */
+
+	if (((so->so_rcv.sb_flags & SB_USRSIZE) == 0) && (inp->inp_pcbinfo->ipi_count < tcp_sockthreshold)) {
+		tp->request_r_scale = max(tcp_win_scale, tp->request_r_scale);
+		so->so_rcv.sb_hiwat = min(TCP_MAXWIN << tp->request_r_scale, (sb_max / (MSIZE+MCLBYTES)) * MCLBYTES);  
+	}
+
 	soisconnecting(so);
 	tcpstat.tcps_connattempt++;
 	tp->t_state = TCPS_SYN_SENT;
@@ -868,7 +1028,7 @@ tcp6_connect(tp, nam, p)
 	if (oinp) {
 		if (oinp != inp && (otp = intotcpcb(oinp)) != NULL &&
 		    otp->t_state == TCPS_TIME_WAIT &&
-		    otp->t_starttime < tcp_msl &&
+		    otp->t_starttime < (u_long)tcp_msl &&
 		    (otp->t_flags & TF_RCVD_CC))
 			otp = tcp_close(otp);
 		else
@@ -884,7 +1044,7 @@ tcp6_connect(tp, nam, p)
 		inp->in6p_laddr = addr6;
 	inp->in6p_faddr = sin6->sin6_addr;
 	inp->inp_fport = sin6->sin6_port;
-	if ((sin6->sin6_flowinfo & IPV6_FLOWINFO_MASK) != NULL)
+	if ((sin6->sin6_flowinfo & IPV6_FLOWINFO_MASK) != 0)
 		inp->in6p_flowinfo = sin6->sin6_flowinfo;
 	in_pcbrehash(inp);
 	lck_rw_done(inp->inp_pcbinfo->mtx);
@@ -1003,15 +1163,17 @@ tcp_ctloutput(so, sopt)
 				error = EINVAL;
 			break;
 
-                case TCP_KEEPALIVE:
-                        error = sooptcopyin(sopt, &optval, sizeof optval,
-                                            sizeof optval);
-                        if (error)
-                                break;
-                        if (optval < 0)
-                                error = EINVAL;
-			else
-				tp->t_keepidle = optval * PR_SLOWHZ;
+		case TCP_KEEPALIVE:
+			error = sooptcopyin(sopt, &optval, sizeof optval,
+						sizeof optval);
+			if (error)
+				break;
+			if (optval < 0)
+				error = EINVAL;
+			else {
+				tp->t_keepidle = optval * TCP_RETRANSHZ;
+				tp->t_timer[TCPT_KEEP] = TCP_KEEPIDLE(tp); /* reset the timer to new value */
+			}
                         break;
 		
 		default:
@@ -1029,7 +1191,7 @@ tcp_ctloutput(so, sopt)
 			optval = tp->t_maxseg;
 			break;
 		case TCP_KEEPALIVE:
-			optval = tp->t_keepidle / PR_SLOWHZ;
+			optval = tp->t_keepidle / TCP_RETRANSHZ;
 			break;
 		case TCP_NOOPT:
 			optval = tp->t_flags & TF_NOOPT;
@@ -1053,22 +1215,24 @@ tcp_ctloutput(so, sopt)
  * sizes, respectively.  These are obsolescent (this information should
  * be set by the route).
  */
-u_long	tcp_sendspace = 1024*16;
+u_long	tcp_sendspace = 1448*256;
 SYSCTL_INT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_RW, 
     &tcp_sendspace , 0, "Maximum outgoing TCP datagram size");
-u_long	tcp_recvspace = 1024*16;
+u_long	tcp_recvspace = 1448*384;
 SYSCTL_INT(_net_inet_tcp, TCPCTL_RECVSPACE, recvspace, CTLFLAG_RW, 
     &tcp_recvspace , 0, "Maximum incoming TCP datagram size");
 
-__private_extern__ int	tcp_sockthreshold = 256;
-SYSCTL_INT(_net_inet_tcp, OID_AUTO, sockthreshold, CTLFLAG_RW, 
-    &tcp_sockthreshold , 0, "TCP Socket size increased if less than threshold");
 
-#define TCP_INCREASED_SPACE	65535	/* Automatically increase tcp send/rcv space to this value */
 /*
  * Attach TCP protocol to socket, allocating
  * internet protocol control block, tcp control block,
  * bufer space, and entering LISTEN state if to accept connections.
+ *
+ * Returns:	0			Success
+ *	in_pcballoc:ENOBUFS
+ *	in_pcballoc:ENOMEM
+ *	in_pcballoc:???			[IPSEC specific]
+ *	soreserve:ENOBUFS
  */
 static int
 tcp_attach(so, p)
@@ -1078,8 +1242,9 @@ tcp_attach(so, p)
 	register struct tcpcb *tp;
 	struct inpcb *inp;
 	int error;
+	u_long sb_effective_max;
 #if INET6
-	int isipv6 = INP_CHECK_SOCKAF(so, AF_INET6) != NULL;
+	int isipv6 = INP_CHECK_SOCKAF(so, AF_INET6) != 0;
 #endif
 
 	error = in_pcballoc(so, &tcbinfo, p);
@@ -1090,14 +1255,21 @@ tcp_attach(so, p)
 
 	if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) {
 		/*
-		 * The goal is to let clients have large send/rcv default windows (TCP_INCREASED_SPACE)
-		 * while not hogging mbuf space for servers. This is done by watching a threshold
-		 * of tcpcbs in use and bumping the default send and rcvspace only if under that threshold.
-		 * The theory being that busy servers have a lot more active tcpcbs and don't want the potential
-		 * memory penalty of having much larger sockbuffs. The sysctl allows to fine tune that threshold value.		 */
-
-		if (inp->inp_pcbinfo->ipi_count < tcp_sockthreshold)
-			error = soreserve(so, MAX(TCP_INCREASED_SPACE, tcp_sendspace), MAX(TCP_INCREASED_SPACE,tcp_recvspace));
+		 * The goal is to let clients machines use large send/rcv default windows to compensate for link
+		 * latency and make sure the receiver is not constraining the sender window.
+		 * But we doon't want to have a few connections use all our mbuf space for servers.
+		 * This is done by watching a threshold of tcpcbs in use and bumping the default send and rcvspace
+		 * only if that threshold isn't reached.
+		 * We're also advertising a much bigger window size (tuneable by sysctl) in correlation with				 * the max socket buffer size if 
+		 * we consider that we have enough ressources for it. This window will be adjusted depending on the
+		 * global socket layer buffer use with the use of tcp_sbpace
+		 */
+
+		if (inp->inp_pcbinfo->ipi_count < tcp_sockthreshold) {
+			sb_effective_max = (sb_max / (MSIZE+MCLBYTES)) * MCLBYTES;  
+			error = soreserve(so, max(min((TCP_MAXWIN << tcp_win_scale)/4, sb_effective_max), tcp_sendspace),
+				       	max(min((TCP_MAXWIN << tcp_win_scale)/2, sb_effective_max), tcp_recvspace));
+		}
 		else	
 			error = soreserve(so, tcp_sendspace, tcp_recvspace);
 		if (error)
@@ -1203,3 +1375,16 @@ tcp_usrclosed(tp)
 	return (tp);
 }
 
+void
+tcp_in_cksum_stats(u_int32_t len)
+{
+	tcps_in_sw_cksum++;
+	tcps_in_sw_cksum_bytes += len;
+}
+
+void
+tcp_out_cksum_stats(u_int32_t len)
+{
+	tcps_out_sw_cksum++;
+	tcps_out_sw_cksum_bytes += len;
+}
diff --git a/bsd/netinet/tcp_var.h b/bsd/netinet/tcp_var.h
index d8c69adee..281e250ff 100644
--- a/bsd/netinet/tcp_var.h
+++ b/bsd/netinet/tcp_var.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993, 1994, 1995
@@ -62,9 +68,21 @@
 #include <netinet/in_pcb.h>
 #include <netinet/tcp_timer.h>
 
-#ifdef KERNEL_PRIVATE
+#if defined(__LP64__)
+#define _TCPCB_PTR(x)			u_int32_t
+#define _TCPCB_LIST_HEAD(name, type)	\
+struct name {				\
+	u_int32_t	lh_first;	\
+};
+#else
+#define _TCPCB_PTR(x)			x
+#define _TCPCB_LIST_HEAD(name, type)	LIST_HEAD(name, type)
+#endif
+
+#define TCP_RETRANSHZ	    10		/* tcp retrans timer (100ms) per hz */		
 
-#define N_TIME_WAIT_SLOTS   128                /* must be power of 2 */
+#ifdef KERNEL_PRIVATE
+#define N_TIME_WAIT_SLOTS   128     	/* must be power of 2 */
 
 /*
  * Kernel variables for tcp.
@@ -139,16 +157,19 @@ struct tcpcb {
 #define	TF_RCVD_CC	0x04000		/* a CC was received in SYN */
 #define	TF_SENDCCNEW	0x08000		/* send CCnew instead of CC in SYN */
 #define	TF_MORETOCOME	0x10000		/* More data to be appended to sock */
-#define	TF_LQ_OVERFLOW	0x20000		/* listen queue overflow */
+#define	TF_LQ_OVERFLOW	0x20000		/* UNUSED listen queue overflow */
 #define	TF_RXWIN0SENT	0x40000		/* sent a receiver win 0 in response */
 #define	TF_SLOWLINK	0x80000		/* route is a on a modem speed link */
 
 
-#define	TF_LASTIDLE	0x100000		/* connection was previously idle */
+#define	TF_LASTIDLE	0x100000	/* connection was previously idle */
 #define	TF_FASTRECOVERY	0x200000	/* in NewReno Fast Recovery */
 #define	TF_WASFRECOVERY	0x400000	/* was in NewReno Fast Recovery */
 #define	TF_SIGNATURE	0x800000	/* require MD5 digests (RFC2385) */
 #define	TF_MAXSEGSNT	0x1000000	/* last segment sent was a full segment */
+#define	TF_SENDINPROG	0x2000000	/* send is in progress */
+#define TF_PMTUD	0x4000000	/* Perform Path MTU Discovery for this connection */
+#define	TF_CLOSING	0x8000000	/* pending tcp close */
 
 	int	t_force;		/* 1 if forcing out a byte */
 
@@ -171,10 +192,14 @@ struct tcpcb {
 
 	u_long	snd_wnd;		/* send window */
 	u_long	snd_cwnd;		/* congestion-controlled window */
+	u_long	snd_bwnd;		/* bandwidth-controlled window */
 	u_long	snd_ssthresh;		/* snd_cwnd size threshold for
 					 * for slow start exponential to
 					 * linear switch
 					 */
+	u_long	snd_bandwidth;		/* calculated bandwidth or 0 */
+	tcp_seq	snd_recover;		/* for use in NewReno Fast Recovery */
+
 	u_int	t_maxopd;		/* mss plus options */
 
 	u_long	t_rcvtime;		/* inactivity time */
@@ -182,6 +207,9 @@ struct tcpcb {
 	int	t_rtttime;		/* round trip time */
 	tcp_seq	t_rtseq;		/* sequence number being timed */
 
+	int	t_bw_rtttime;		/* used for bandwidth calculation */
+	tcp_seq	t_bw_rtseq;		/* used for bandwidth calculation */
+
 	int	t_rxtcur;		/* current retransmit value (ticks) */
 	u_int	t_maxseg;		/* maximum segment size */
 	int	t_srtt;			/* smoothed round-trip time */
@@ -189,6 +217,7 @@ struct tcpcb {
 
 	int	t_rxtshift;		/* log(2) of rexmt exp. backoff */
 	u_int	t_rttmin;		/* minimum rtt allowed */
+	u_int	t_rttbest;		/* best rtt we've seen */
 	u_long	t_rttupdated;		/* number of times rtt sampled */
 	u_long	max_sndwnd;		/* largest window peer has offered */
 
@@ -210,7 +239,8 @@ struct tcpcb {
 /* RFC 1644 variables */
 	tcp_cc	cc_send;		/* send connection count */
 	tcp_cc	cc_recv;		/* receive connection count */
-	tcp_seq	snd_recover;	/* for use in NewReno Fast Recovery */
+/* RFC 3465 variables */
+	u_long	t_bytes_acked;		/* ABC "bytes_acked" parameter */
 /* experimental */
 	u_long	snd_cwnd_prev;		/* cwnd prior to retransmit */
 	u_long	snd_ssthresh_prev;	/* ssthresh prior to retransmit */
@@ -218,11 +248,14 @@ struct tcpcb {
 
 	int     t_keepidle;		/* keepalive idle timer (override global if > 0) */
 	int	t_lastchain;		/* amount of packets chained last time around */
+	int	t_unacksegs;		/* received but unacked segments: used for delaying acks */
+
 
 /* 3529618 MSS overload prevention */
 	u_long	rcv_reset;
 	u_long	rcv_pps;
 	u_long	rcv_byps;
+	u_long  rcv_maxbyps;
 	tcp_seq snd_high;		/* for use in NewReno Fast Recovery */
 	tcp_seq snd_high_prev;	/* snd_high prior to retransmit */
 
@@ -243,6 +276,20 @@ struct tcpcb {
    					   episode starts at this seq number */
 	struct sackhint	sackhint;	/* SACK scoreboard hint */
 	int	t_rttlow;		/* smallest observerved RTT */
+	u_long		ecn_flags;
+#define TE_SETUPSENT		0x01	/* Indicate we have sent ECN-SETUP SYN or SYN-ACK */
+#define TE_SETUPRECEIVED	0x02	/* Indicate we have received ECN-SETUP SYN or SYN-ACK */
+#define TE_SENDIPECT		0x04	/* Indicate we haven't sent or received non-ECN-setup SYN or SYN-ACK */
+#define TE_SENDCWR			0x08	/* Indicate that the next non-retransmit should have the TCP CWR flag set */
+#define TE_SENDECE			0x10	/* Indicate that the next packet should have the TCP ECE flag set */
+	
+#if TRAFFIC_MGT
+	u_int32_t		tot_recv_snapshot;	/* snapshot of global total pkts received */
+	u_int32_t		bg_recv_snapshot;	/* snapshot of global background pkts received */
+#endif /* TRAFFIC_MGT */
+	u_int32_t	t_pktlist_sentlen; /* total bytes in transmit chain */
+	struct mbuf	*t_pktlist_head; /* First packet in transmit chain */
+	struct mbuf	*t_pktlist_tail; /* Last packet in transmit chain */
 };
 
 #define IN_FASTRECOVERY(tp)	(tp->t_flags & TF_FASTRECOVERY)
@@ -334,17 +381,17 @@ struct rmxp_tao {
 struct otcpcb {
 #else
 struct tseg_qent;
-LIST_HEAD(tsegqe_head, tseg_qent);
+_TCPCB_LIST_HEAD(tsegqe_head, tseg_qent);
 
 struct tcpcb {
 #endif /* KERNEL_PRIVATE */
 	struct	tsegqe_head t_segq;
 	int	t_dupacks;		/* consecutive dup acks recd */
-	struct	tcptemp	*unused;	/* unused now: was t_template */
+	u_int32_t unused;		/* unused now: was t_template */
 
 	int	t_timer[TCPT_NTIMERS];	/* tcp timers */
 
-	struct	inpcb *t_inpcb;		/* back pointer to internet pcb */
+	_TCPCB_PTR(struct inpcb *) t_inpcb;	/* back pointer to internet pcb */
 	int	t_state;		/* state of this connection */
 	u_int	t_flags;
 #define	TF_ACKNOW	0x00001		/* ack peer immediately */
@@ -384,19 +431,19 @@ struct tcpcb {
 
 	tcp_seq	rcv_nxt;		/* receive next */
 	tcp_seq	rcv_adv;		/* advertised window */
-	u_long	rcv_wnd;		/* receive window */
+	u_int32_t rcv_wnd;		/* receive window */
 	tcp_seq	rcv_up;			/* receive urgent pointer */
 
-	u_long	snd_wnd;		/* send window */
-	u_long	snd_cwnd;		/* congestion-controlled window */
-	u_long	snd_ssthresh;		/* snd_cwnd size threshold for
+	u_int32_t snd_wnd;		/* send window */
+	u_int32_t snd_cwnd;		/* congestion-controlled window */
+	u_int32_t snd_ssthresh;		/* snd_cwnd size threshold for
 					 * for slow start exponential to
 					 * linear switch
 					 */
 	u_int	t_maxopd;		/* mss plus options */
 
-	u_long	t_rcvtime;		/* inactivity time */
-	u_long	t_starttime;		/* time connection was established */
+	u_int32_t t_rcvtime;		/* inactivity time */
+	u_int32_t t_starttime;		/* time connection was established */
 	int	t_rtttime;		/* round trip time */
 	tcp_seq	t_rtseq;		/* sequence number being timed */
 
@@ -407,8 +454,8 @@ struct tcpcb {
 
 	int	t_rxtshift;		/* log(2) of rexmt exp. backoff */
 	u_int	t_rttmin;		/* minimum rtt allowed */
-	u_long	t_rttupdated;		/* number of times rtt sampled */
-	u_long	max_sndwnd;		/* largest window peer has offered */
+	u_int32_t t_rttupdated;		/* number of times rtt sampled */
+	u_int32_t max_sndwnd;		/* largest window peer has offered */
 
 	int	t_softerror;		/* possible error not yet reported */
 /* out-of-band data */
@@ -421,18 +468,18 @@ struct tcpcb {
 	u_char	rcv_scale;		/* window scaling for recv window */
 	u_char	request_r_scale;	/* pending window scaling */
 	u_char	requested_s_scale;
-	u_long	ts_recent;		/* timestamp echo data */
+	u_int32_t ts_recent;		/* timestamp echo data */
 
-	u_long	ts_recent_age;		/* when last updated */
+	u_int32_t ts_recent_age;	/* when last updated */
 	tcp_seq	last_ack_sent;
 /* RFC 1644 variables */
 	tcp_cc	cc_send;		/* send connection count */
 	tcp_cc	cc_recv;		/* receive connection count */
-	tcp_seq	snd_recover;	/* for use in fast recovery */
+	tcp_seq	snd_recover;		/* for use in fast recovery */
 /* experimental */
-	u_long	snd_cwnd_prev;		/* cwnd prior to retransmit */
-	u_long	snd_ssthresh_prev;	/* ssthresh prior to retransmit */
-	u_long	t_badrxtwin;		/* window for retransmit recovery */
+	u_int32_t snd_cwnd_prev;	/* cwnd prior to retransmit */
+	u_int32_t snd_ssthresh_prev;	/* ssthresh prior to retransmit */
+	u_int32_t t_badrxtwin;		/* window for retransmit recovery */
 };
 
 /*
@@ -441,87 +488,116 @@ struct tcpcb {
  * but that's inconvenient at the moment.
  */
 struct	tcpstat {
-	u_long	tcps_connattempt;	/* connections initiated */
-	u_long	tcps_accepts;		/* connections accepted */
-	u_long	tcps_connects;		/* connections established */
-	u_long	tcps_drops;		/* connections dropped */
-	u_long	tcps_conndrops;		/* embryonic connections dropped */
-	u_long	tcps_closed;		/* conn. closed (includes drops) */
-	u_long	tcps_segstimed;		/* segs where we tried to get rtt */
-	u_long	tcps_rttupdated;	/* times we succeeded */
-	u_long	tcps_delack;		/* delayed acks sent */
-	u_long	tcps_timeoutdrop;	/* conn. dropped in rxmt timeout */
-	u_long	tcps_rexmttimeo;	/* retransmit timeouts */
-	u_long	tcps_persisttimeo;	/* persist timeouts */
-	u_long	tcps_keeptimeo;		/* keepalive timeouts */
-	u_long	tcps_keepprobe;		/* keepalive probes sent */
-	u_long	tcps_keepdrops;		/* connections dropped in keepalive */
-
-	u_long	tcps_sndtotal;		/* total packets sent */
-	u_long	tcps_sndpack;		/* data packets sent */
-	u_long	tcps_sndbyte;		/* data bytes sent */
-	u_long	tcps_sndrexmitpack;	/* data packets retransmitted */
-	u_long	tcps_sndrexmitbyte;	/* data bytes retransmitted */
-	u_long	tcps_sndacks;		/* ack-only packets sent */
-	u_long	tcps_sndprobe;		/* window probes sent */
-	u_long	tcps_sndurg;		/* packets sent with URG only */
-	u_long	tcps_sndwinup;		/* window update-only packets sent */
-	u_long	tcps_sndctrl;		/* control (SYN|FIN|RST) packets sent */
-
-	u_long	tcps_rcvtotal;		/* total packets received */
-	u_long	tcps_rcvpack;		/* packets received in sequence */
-	u_long	tcps_rcvbyte;		/* bytes received in sequence */
-	u_long	tcps_rcvbadsum;		/* packets received with ccksum errs */
-	u_long	tcps_rcvbadoff;		/* packets received with bad offset */
-	u_long	tcps_rcvmemdrop;	/* packets dropped for lack of memory */
-	u_long	tcps_rcvshort;		/* packets received too short */
-	u_long	tcps_rcvduppack;	/* duplicate-only packets received */
-	u_long	tcps_rcvdupbyte;	/* duplicate-only bytes received */
-	u_long	tcps_rcvpartduppack;	/* packets with some duplicate data */
-	u_long	tcps_rcvpartdupbyte;	/* dup. bytes in part-dup. packets */
-	u_long	tcps_rcvoopack;		/* out-of-order packets received */
-	u_long	tcps_rcvoobyte;		/* out-of-order bytes received */
-	u_long	tcps_rcvpackafterwin;	/* packets with data after window */
-	u_long	tcps_rcvbyteafterwin;	/* bytes rcvd after window */
-	u_long	tcps_rcvafterclose;	/* packets rcvd after "close" */
-	u_long	tcps_rcvwinprobe;	/* rcvd window probe packets */
-	u_long	tcps_rcvdupack;		/* rcvd duplicate acks */
-	u_long	tcps_rcvacktoomuch;	/* rcvd acks for unsent data */
-	u_long	tcps_rcvackpack;	/* rcvd ack packets */
-	u_long	tcps_rcvackbyte;	/* bytes acked by rcvd acks */
-	u_long	tcps_rcvwinupd;		/* rcvd window update packets */
-	u_long	tcps_pawsdrop;		/* segments dropped due to PAWS */
-	u_long	tcps_predack;		/* times hdr predict ok for acks */
-	u_long	tcps_preddat;		/* times hdr predict ok for data pkts */
-	u_long	tcps_pcbcachemiss;
-	u_long	tcps_cachedrtt;		/* times cached RTT in route updated */
-	u_long	tcps_cachedrttvar;	/* times cached rttvar updated */
-	u_long	tcps_cachedssthresh;	/* times cached ssthresh updated */
-	u_long	tcps_usedrtt;		/* times RTT initialized from route */
-	u_long	tcps_usedrttvar;	/* times RTTVAR initialized from rt */
-	u_long	tcps_usedssthresh;	/* times ssthresh initialized from rt*/
-	u_long	tcps_persistdrop;	/* timeout in persist state */
-	u_long	tcps_badsyn;		/* bogus SYN, e.g. premature ACK */
-	u_long	tcps_mturesent;		/* resends due to MTU discovery */
-	u_long	tcps_listendrop;	/* listen queue overflows */
+	u_int32_t	tcps_connattempt;	/* connections initiated */
+	u_int32_t	tcps_accepts;		/* connections accepted */
+	u_int32_t	tcps_connects;		/* connections established */
+	u_int32_t	tcps_drops;		/* connections dropped */
+	u_int32_t	tcps_conndrops;		/* embryonic connections dropped */
+	u_int32_t	tcps_closed;		/* conn. closed (includes drops) */
+	u_int32_t	tcps_segstimed;		/* segs where we tried to get rtt */
+	u_int32_t	tcps_rttupdated;	/* times we succeeded */
+	u_int32_t	tcps_delack;		/* delayed acks sent */
+	u_int32_t	tcps_timeoutdrop;	/* conn. dropped in rxmt timeout */
+	u_int32_t	tcps_rexmttimeo;	/* retransmit timeouts */
+	u_int32_t	tcps_persisttimeo;	/* persist timeouts */
+	u_int32_t	tcps_keeptimeo;		/* keepalive timeouts */
+	u_int32_t	tcps_keepprobe;		/* keepalive probes sent */
+	u_int32_t	tcps_keepdrops;		/* connections dropped in keepalive */
+
+	u_int32_t	tcps_sndtotal;		/* total packets sent */
+	u_int32_t	tcps_sndpack;		/* data packets sent */
+	u_int32_t	tcps_sndbyte;		/* data bytes sent */
+	u_int32_t	tcps_sndrexmitpack;	/* data packets retransmitted */
+	u_int32_t	tcps_sndrexmitbyte;	/* data bytes retransmitted */
+	u_int32_t	tcps_sndacks;		/* ack-only packets sent */
+	u_int32_t	tcps_sndprobe;		/* window probes sent */
+	u_int32_t	tcps_sndurg;		/* packets sent with URG only */
+	u_int32_t	tcps_sndwinup;		/* window update-only packets sent */
+	u_int32_t	tcps_sndctrl;		/* control (SYN|FIN|RST) packets sent */
+
+	u_int32_t	tcps_rcvtotal;		/* total packets received */
+	u_int32_t	tcps_rcvpack;		/* packets received in sequence */
+	u_int32_t	tcps_rcvbyte;		/* bytes received in sequence */
+	u_int32_t	tcps_rcvbadsum;		/* packets received with ccksum errs */
+	u_int32_t	tcps_rcvbadoff;		/* packets received with bad offset */
+	u_int32_t	tcps_rcvmemdrop;	/* packets dropped for lack of memory */
+	u_int32_t	tcps_rcvshort;		/* packets received too short */
+	u_int32_t	tcps_rcvduppack;	/* duplicate-only packets received */
+	u_int32_t	tcps_rcvdupbyte;	/* duplicate-only bytes received */
+	u_int32_t	tcps_rcvpartduppack;	/* packets with some duplicate data */
+	u_int32_t	tcps_rcvpartdupbyte;	/* dup. bytes in part-dup. packets */
+	u_int32_t	tcps_rcvoopack;		/* out-of-order packets received */
+	u_int32_t	tcps_rcvoobyte;		/* out-of-order bytes received */
+	u_int32_t	tcps_rcvpackafterwin;	/* packets with data after window */
+	u_int32_t	tcps_rcvbyteafterwin;	/* bytes rcvd after window */
+	u_int32_t	tcps_rcvafterclose;	/* packets rcvd after "close" */
+	u_int32_t	tcps_rcvwinprobe;	/* rcvd window probe packets */
+	u_int32_t	tcps_rcvdupack;		/* rcvd duplicate acks */
+	u_int32_t	tcps_rcvacktoomuch;	/* rcvd acks for unsent data */
+	u_int32_t	tcps_rcvackpack;	/* rcvd ack packets */
+	u_int32_t	tcps_rcvackbyte;	/* bytes acked by rcvd acks */
+	u_int32_t	tcps_rcvwinupd;		/* rcvd window update packets */
+	u_int32_t	tcps_pawsdrop;		/* segments dropped due to PAWS */
+	u_int32_t	tcps_predack;		/* times hdr predict ok for acks */
+	u_int32_t	tcps_preddat;		/* times hdr predict ok for data pkts */
+	u_int32_t	tcps_pcbcachemiss;
+	u_int32_t	tcps_cachedrtt;		/* times cached RTT in route updated */
+	u_int32_t	tcps_cachedrttvar;	/* times cached rttvar updated */
+	u_int32_t	tcps_cachedssthresh;	/* times cached ssthresh updated */
+	u_int32_t	tcps_usedrtt;		/* times RTT initialized from route */
+	u_int32_t	tcps_usedrttvar;	/* times RTTVAR initialized from rt */
+	u_int32_t	tcps_usedssthresh;	/* times ssthresh initialized from rt*/
+	u_int32_t	tcps_persistdrop;	/* timeout in persist state */
+	u_int32_t	tcps_badsyn;		/* bogus SYN, e.g. premature ACK */
+	u_int32_t	tcps_mturesent;		/* resends due to MTU discovery */
+	u_int32_t	tcps_listendrop;	/* listen queue overflows */
+
+	/* new stats from FreeBSD 5.4 sync up */
+	u_int32_t	tcps_minmssdrops;	/* average minmss too low drops */
+	u_int32_t	tcps_sndrexmitbad;	/* unnecessary packet retransmissions */
+	u_int32_t	tcps_badrst;		/* ignored RSTs in the window */
+
+	u_int32_t	tcps_sc_added;		/* entry added to syncache */
+	u_int32_t	tcps_sc_retransmitted;	/* syncache entry was retransmitted */
+	u_int32_t	tcps_sc_dupsyn;		/* duplicate SYN packet */
+	u_int32_t	tcps_sc_dropped;	/* could not reply to packet */
+	u_int32_t	tcps_sc_completed;	/* successful extraction of entry */
+	u_int32_t	tcps_sc_bucketoverflow;	/* syncache per-bucket limit hit */
+	u_int32_t	tcps_sc_cacheoverflow;	/* syncache cache limit hit */
+	u_int32_t	tcps_sc_reset;		/* RST removed entry from syncache */
+	u_int32_t	tcps_sc_stale;		/* timed out or listen socket gone */
+	u_int32_t	tcps_sc_aborted;	/* syncache entry aborted */
+	u_int32_t	tcps_sc_badack;		/* removed due to bad ACK */
+	u_int32_t	tcps_sc_unreach;	/* ICMP unreachable received */
+	u_int32_t	tcps_sc_zonefail;	/* zalloc() failed */
+	u_int32_t	tcps_sc_sendcookie;	/* SYN cookie sent */
+	u_int32_t	tcps_sc_recvcookie;	/* SYN cookie received */
+
+	u_int32_t	tcps_hc_added;		/* entry added to hostcache */
+	u_int32_t	tcps_hc_bucketoverflow;	/* hostcache per bucket limit hit */
 
 	/* SACK related stats */
-	u_long	tcps_sack_recovery_episode; /* SACK recovery episodes */
-	u_long  tcps_sack_rexmits;	    /* SACK rexmit segments   */
-	u_long  tcps_sack_rexmit_bytes;	    /* SACK rexmit bytes      */
-	u_long  tcps_sack_rcv_blocks;	    /* SACK blocks (options) received */
-	u_long  tcps_sack_send_blocks;	    /* SACK blocks (options) sent     */
-	u_long  tcps_sack_sboverflow;	    /* SACK sendblock overflow   */
-
+	u_int32_t	tcps_sack_recovery_episode; /* SACK recovery episodes */
+	u_int32_t  tcps_sack_rexmits;	    /* SACK rexmit segments   */
+	u_int32_t  tcps_sack_rexmit_bytes;	    /* SACK rexmit bytes      */
+	u_int32_t  tcps_sack_rcv_blocks;	    /* SACK blocks (options) received */
+	u_int32_t  tcps_sack_send_blocks;	    /* SACK blocks (options) sent     */
+	u_int32_t  tcps_sack_sboverflow;	    /* SACK sendblock overflow   */
+
+#if TRAFFIC_MGT
+	u_int32_t	tcps_bg_rcvtotal;	/* total background packets received */
+#endif /* TRAFFIC_MGT */
 };
 
+#pragma pack(4)
+
 /*
  * TCB structure exported to user-land via sysctl(3).
  * Evil hack: declare only if in_pcb.h and sys/socketvar.h have been
  * included.  Not all of our clients do.
  */
 struct	xtcpcb {
-	size_t	xt_len;
+	u_int32_t	xt_len;
 #ifdef KERNEL_PRIVATE
 	struct	inpcb_compat	xt_inp;
 #else
@@ -536,6 +612,8 @@ struct	xtcpcb {
 	u_quad_t	xt_alignment_hack;
 };
 
+#pragma pack()
+
 /*
  * Names for TCP sysctl objects
  */
@@ -555,6 +633,11 @@ struct	xtcpcb {
 #define	TCPCTL_MAXID		14
 
 #ifdef KERNEL_PRIVATE
+#define	TCP_PKTLIST_CLEAR(tp) {						\
+	(tp)->t_pktlist_head = (tp)->t_pktlist_tail = NULL;		\
+	(tp)->t_lastchain = (tp)->t_pktlist_sentlen = 0;		\
+}
+
 #define TCPCTL_NAMES { \
 	{ 0, 0 }, \
 	{ "rfc1323", CTLTYPE_INT }, \
@@ -603,7 +686,7 @@ void	 tcp_drain(void);
 void	 tcp_fasttimo(void);
 struct rmxp_tao *
 	 tcp_gettaocache(struct inpcb *);
-void	 tcp_init(void);
+void	 tcp_init(void) __attribute__((section("__TEXT, initcode")));
 void	 tcp_input(struct mbuf *, int);
 void	 tcp_mss(struct tcpcb *, int);
 int	 tcp_mssopt(struct tcpcb *);
@@ -614,7 +697,7 @@ struct tcpcb *
 int	 tcp_output(struct tcpcb *);
 void	 tcp_quench(struct inpcb *, int);
 void	 tcp_respond(struct tcpcb *, void *,
-	    struct tcphdr *, struct mbuf *, tcp_seq, tcp_seq, int);
+	    struct tcphdr *, struct mbuf *, tcp_seq, tcp_seq, int, ifnet_t);
 struct rtentry *
 	 tcp_rtlookup(struct inpcb *);
 void	 tcp_setpersist(struct tcpcb *);
@@ -626,12 +709,14 @@ struct tcpcb *
 	 tcp_timers(struct tcpcb *, int);
 void	 tcp_trace(int, int, struct tcpcb *, void *, struct tcphdr *, int);
 
+void tcp_sack_doack(struct tcpcb *, struct tcpopt *, tcp_seq);
 void	 tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_laststart, tcp_seq rcv_lastend);
 void	 tcp_clean_sackreport(struct tcpcb *tp);
 void	 tcp_sack_adjust(struct tcpcb *tp);
 struct sackhole *tcp_sack_output(struct tcpcb *tp, int *sack_bytes_rexmt);
 void	 tcp_sack_partialack(struct tcpcb *, struct tcphdr *);
 void	 tcp_free_sackholes(struct tcpcb *tp);
+long	 tcp_sbspace(struct tcpcb *tp);
 
 
 int	 tcp_lock (struct socket *, int, int);
diff --git a/bsd/netinet/tcpip.h b/bsd/netinet/tcpip.h
index b9830e140..4868d6e0d 100644
--- a/bsd/netinet/tcpip.h
+++ b/bsd/netinet/tcpip.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/bsd/netinet/udp.h b/bsd/netinet/udp.h
index 9fa2bb55e..ef82a6f15 100644
--- a/bsd/netinet/udp.h
+++ b/bsd/netinet/udp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -57,6 +63,7 @@
 #ifndef _NETINET_UDP_H_
 #define _NETINET_UDP_H_
 #include <sys/appleapiopts.h>
+#include <sys/types.h>		/* u_short */
 
 /*
  * Udp protocol header.
@@ -69,4 +76,8 @@ struct udphdr {
 	u_short	uh_sum;			/* udp checksum */
 };
 
+/*
+ * User-settable options (used with setsockopt).
+ */
+#define	UDP_NOCKSUM	0x01	/* don't checksum outbound payloads */
 #endif
diff --git a/bsd/netinet/udp_usrreq.c b/bsd/netinet/udp_usrreq.c
index af06d9264..7e97ffd9a 100644
--- a/bsd/netinet/udp_usrreq.c
+++ b/bsd/netinet/udp_usrreq.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
@@ -67,6 +73,8 @@
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 
+#include <kern/zalloc.h>
+
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/route.h>
@@ -91,8 +99,8 @@
 
 #if IPSEC
 #include <netinet6/ipsec.h>
+#include <netinet6/esp.h>
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif /*IPSEC*/
 
 
@@ -115,6 +123,26 @@ static int	udpcksum = 0;		/* XXX */
 SYSCTL_INT(_net_inet_udp, UDPCTL_CHECKSUM, checksum, CTLFLAG_RW,
 		&udpcksum, 0, "");
 
+static u_int32_t udps_in_sw_cksum;
+SYSCTL_UINT(_net_inet_udp, OID_AUTO, in_sw_cksum, CTLFLAG_RD,
+    &udps_in_sw_cksum, 0,
+    "Number of received packets checksummed in software");
+
+static u_int64_t udps_in_sw_cksum_bytes;
+SYSCTL_QUAD(_net_inet_udp, OID_AUTO, in_sw_cksum_bytes, CTLFLAG_RD,
+    &udps_in_sw_cksum_bytes,
+    "Amount of received data checksummed in software");
+
+static u_int32_t udps_out_sw_cksum;
+SYSCTL_UINT(_net_inet_udp, OID_AUTO, out_sw_cksum, CTLFLAG_RD,
+    &udps_out_sw_cksum, 0,
+    "Number of transmitted packets checksummed in software");
+
+static u_int64_t udps_out_sw_cksum_bytes;
+SYSCTL_QUAD(_net_inet_udp, OID_AUTO, out_sw_cksum_bytes, CTLFLAG_RD,
+    &udps_out_sw_cksum_bytes,
+    "Amount of transmitted data checksummed in software");
+
 int	log_in_vain = 0;
 SYSCTL_INT(_net_inet_udp, OID_AUTO, log_in_vain, CTLFLAG_RW, 
     &log_in_vain, 0, "Log all incoming UDP packets");
@@ -131,20 +159,24 @@ struct	inpcbinfo udbinfo;
 #define UDBHASHSIZE 16
 #endif
 
-extern  int apple_hwcksum_rx;
 extern	int	esp_udp_encap_port;
 extern	u_long  route_generation;
 
-extern  void    ipfwsyslog( int level, char *format,...);
+extern  void    ipfwsyslog( int level, const char *format,...);
  
 extern int fw_verbose;
+static int udp_gc_done = FALSE; /* Garbage collection performed last slowtimo */
 
+#if IPFIREWALL
 #define log_in_vain_log( a ) {            \
         if ( (log_in_vain == 3 ) && (fw_verbose == 2)) {        /* Apple logging, log to ipfw.log */ \
                 ipfwsyslog a ;  \
         }                       \
         else log a ;            \
 }
+#else
+#define log_in_vain_log( a ) { log a; }
+#endif
 
 struct	udpstat udpstat;	/* from udp_var.h */
 SYSCTL_STRUCT(_net_inet_udp, UDPCTL_STATS, stats, CTLFLAG_RD,
@@ -152,25 +184,22 @@ SYSCTL_STRUCT(_net_inet_udp, UDPCTL_STATS, stats, CTLFLAG_RD,
 SYSCTL_INT(_net_inet_udp, OID_AUTO, pcbcount, CTLFLAG_RD, 
     &udbinfo.ipi_count, 0, "Number of active PCBs");
 
-static struct	sockaddr_in udp_in = { sizeof(udp_in), AF_INET };
 #if INET6
 struct udp_in6 {
 	struct sockaddr_in6	uin6_sin;
 	u_char			uin6_init_done : 1;
-} udp_in6 = {
-	{ sizeof(udp_in6.uin6_sin), AF_INET6 },
-	0
 };
 struct udp_ip6 {
 	struct ip6_hdr		uip6_ip6;
 	u_char			uip6_init_done : 1;
-} udp_ip6;
-#endif /* INET6 */
-
-static void udp_append(struct inpcb *last, struct ip *ip,
-			    struct mbuf *n, int off);
-#if INET6
+};
 static void ip_2_ip6_hdr(struct ip6_hdr *ip6, struct ip *ip);
+static void udp_append(struct inpcb *last, struct ip *ip,
+    struct mbuf *n, int off, struct sockaddr_in *pudp_in,
+    struct udp_in6 *pudp_in6, struct udp_ip6 *pudp_ip6);
+#else
+static void udp_append(struct inpcb *last, struct ip *ip,
+    struct mbuf *n, int off, struct sockaddr_in *pudp_in);
 #endif
 
 static int udp_detach(struct socket *so);
@@ -253,6 +282,17 @@ udp_input(m, iphlen)
 	struct ip save_ip;
 	struct sockaddr *append_sa;
 	struct inpcbinfo *pcbinfo = &udbinfo;
+	struct sockaddr_in udp_in = {
+		sizeof (udp_in), AF_INET, 0, { 0 }, { 0, 0, 0, 0, 0, 0, 0, 0 }
+	};
+#if INET6
+	struct udp_in6 udp_in6 = {
+		{ sizeof (udp_in6.uin6_sin), AF_INET6, 0, 0,
+		    IN6ADDR_ANY_INIT, 0 },
+		0
+	};
+	struct udp_ip6 udp_ip6;
+#endif /* INET6 */
 
 	udpstat.udps_ipackets++;
 
@@ -335,6 +375,7 @@ doudpcksum:
 			*(uint32_t*)&((struct ipovly *)ip)->ih_x1[0] = *(uint32_t*)&b[0];
 			*(uint32_t*)&((struct ipovly *)ip)->ih_x1[4] = *(uint32_t*)&b[4];
 			*(uint8_t*)&((struct ipovly *)ip)->ih_x1[8] = *(uint8_t*)&b[8];
+			udp_in_cksum_stats(len);
 		}
 		if (uh->uh_sum) {
 			udpstat.udps_badsum++;
@@ -350,7 +391,10 @@ doudpcksum:
 
 	if (IN_MULTICAST(ntohl(ip->ip_dst.s_addr)) ||
 	    in_broadcast(ip->ip_dst, m->m_pkthdr.rcvif)) {
-		struct inpcb *last;
+
+		int reuse_sock = 0, mcast_delivered = 0;
+		struct mbuf *n = NULL;
+
 		lck_rw_lock_shared(pcbinfo->mtx);
 		/*
 		 * Deliver a multicast or broadcast datagram to *all* sockets
@@ -378,7 +422,6 @@ doudpcksum:
 		 * Locate pcb(s) for datagram.
 		 * (Algorithm copied from raw_intr().)
 		 */
-		last = NULL;
 #if INET6
 		udp_in6.uin6_init_done = udp_ip6.uip6_init_done = 0;
 #endif
@@ -391,11 +434,12 @@ doudpcksum:
 			if (inp->inp_socket == NULL) 
 				continue;
 			if (inp != sotoinpcb(inp->inp_socket))
-				panic("udp_input: bad so back ptr inp=%x\n", inp);
+				panic("udp_input: bad so back ptr inp=%p\n", inp);
 #if INET6
                         if ((inp->inp_vflag & INP_IPV4) == 0)
                                 continue;
 #endif
+
 			if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) {
 				continue;
 			}
@@ -427,44 +471,50 @@ doudpcksum:
 				}
 			}
 
-			if (last != NULL) {
-				struct mbuf *n;
+			reuse_sock = inp->inp_socket->so_options& (SO_REUSEPORT|SO_REUSEADDR);
+			{
 #if IPSEC
 				int skipit = 0;
 				/* check AH/ESP integrity. */
 				if (ipsec_bypass == 0) {
-					lck_mtx_lock(sadb_mutex);
-					if (ipsec4_in_reject_so(m, last->inp_socket)) {
-						ipsecstat.in_polvio++;
+					if (ipsec4_in_reject_so(m, inp->inp_socket)) {
+						IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 						/* do not inject data to pcb */
 						skipit = 1;
 					}
-					lck_mtx_unlock(sadb_mutex);
 				}
-				if (skipit == 0)
+				if (skipit == 0) 
 #endif /*IPSEC*/
-				if ((n = m_copy(m, 0, M_COPYALL)) != NULL) {
-					udp_append(last, ip, n,
-						   iphlen +
-						   sizeof(struct udphdr));
+				{
+					if (reuse_sock) 
+						n = m_copy(m, 0, M_COPYALL);
+#if INET6
+					udp_append(inp, ip, m,
+					    iphlen + sizeof(struct udphdr),
+					    &udp_in, &udp_in6, &udp_ip6);
+#else
+					udp_append(inp, ip, m,
+					    iphlen + sizeof(struct udphdr),
+					    &udp_in);
+#endif /* INET6 */
+					mcast_delivered++;
 				}
-				udp_unlock(last->inp_socket, 1, 0);
+				udp_unlock(inp->inp_socket, 1, 0);
 			}
-			last = inp;
 			/*
 			 * Don't look for additional matches if this one does
 			 * not have either the SO_REUSEPORT or SO_REUSEADDR
 			 * socket options set.  This heuristic avoids searching
 			 * through all pcbs in the common case of a non-shared
-			 * port.  It * assumes that an application will never
+			 * port.  It assumes that an application will never
 			 * clear these options after setting them.
 			 */
-			if ((last->inp_socket->so_options&(SO_REUSEPORT|SO_REUSEADDR)) == 0)
+			if (reuse_sock == 0 || ((m = n) == NULL))
 				break;
 		}
 		lck_rw_done(pcbinfo->mtx);
 
-		if (last == NULL) {
+		if (mcast_delivered == 0) {
 			/*
 			 * No matching pcb found; discard datagram.
 			 * (No need to send an ICMP Port Unreachable
@@ -473,21 +523,9 @@ doudpcksum:
 			udpstat.udps_noportbcast++;
 			goto bad;
 		}
-#if IPSEC
-		/* check AH/ESP integrity. */
-		if (ipsec_bypass == 0 && m) {
-			lck_mtx_lock(sadb_mutex);
-			if (ipsec4_in_reject_so(m, last->inp_socket)) {
-				ipsecstat.in_polvio++;
-				lck_mtx_unlock(sadb_mutex);
-				udp_unlock(last->inp_socket, 1, 0);
-				goto bad;
-			}
-			lck_mtx_unlock(sadb_mutex);
-		}
-#endif /*IPSEC*/
-		udp_append(last, ip, m, iphlen + sizeof(struct udphdr));
-		udp_unlock(last->inp_socket, 1, 0);
+
+		if (reuse_sock != 0)	/* free the extra copy of mbuf */
+			m_freem(m);
 		return;
 	}
 
@@ -592,14 +630,11 @@ doudpcksum:
 	}
 #if IPSEC
 	if (ipsec_bypass == 0 && inp != NULL) {
-		lck_mtx_lock(sadb_mutex);
 		if (ipsec4_in_reject_so(m, inp->inp_socket)) {
-			ipsecstat.in_polvio++;
-			lck_mtx_unlock(sadb_mutex);
+			IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 			udp_unlock(inp->inp_socket, 1, 0);
 			goto bad;
 	        }
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /*IPSEC*/
 
@@ -632,7 +667,7 @@ doudpcksum:
 #if INET6
 	if (inp->inp_vflag & INP_IPV6) {
 		in6_sin_2_v4mapsin6(&udp_in, &udp_in6.uin6_sin);
-		append_sa = (struct sockaddr *)&udp_in6;
+		append_sa = (struct sockaddr *)&udp_in6.uin6_sin;
 	} else
 #endif
 	append_sa = (struct sockaddr *)&udp_in;
@@ -674,31 +709,39 @@ ip_2_ip6_hdr(ip6, ip)
 
 /*
  * subroutine of udp_input(), mainly for source code readability.
- * caller must properly init udp_ip6 and udp_in6 beforehand.
  */
 static void
-udp_append(last, ip, n, off)
-	struct inpcb *last;
-	struct ip *ip;
-	struct mbuf *n;
-	int off;
+#if INET6
+udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off,
+    struct sockaddr_in *pudp_in, struct udp_in6 *pudp_in6,
+    struct udp_ip6 *pudp_ip6)
+#else
+udp_append(struct inpcb *last, struct ip *ip, struct mbuf *n, int off,
+    struct sockaddr_in *pudp_in)
+#endif
 {
 	struct sockaddr *append_sa;
 	struct mbuf *opts = 0;
 
+#if CONFIG_MACF_NET
+	if (mac_inpcb_check_deliver(last, n, AF_INET, SOCK_DGRAM) != 0) {
+		m_freem(n);
+		return;
+	}
+#endif
 	if (last->inp_flags & INP_CONTROLOPTS ||
 	    last->inp_socket->so_options & SO_TIMESTAMP) {
 #if INET6
 		if (last->inp_vflag & INP_IPV6) {
 			int savedflags;
 
-			if (udp_ip6.uip6_init_done == 0) {
-				ip_2_ip6_hdr(&udp_ip6.uip6_ip6, ip);
-				udp_ip6.uip6_init_done = 1;
+			if (pudp_ip6->uip6_init_done == 0) {
+				ip_2_ip6_hdr(&pudp_ip6->uip6_ip6, ip);
+				pudp_ip6->uip6_init_done = 1;
 			}
 			savedflags = last->inp_flags;
 			last->inp_flags &= ~INP_UNMAPPABLEOPTS;
-			ip6_savecontrol(last, &opts, &udp_ip6.uip6_ip6, n);
+			ip6_savecontrol(last, &opts, &pudp_ip6->uip6_ip6, n);
 			last->inp_flags = savedflags;
 		} else
 #endif
@@ -706,14 +749,14 @@ udp_append(last, ip, n, off)
 	}
 #if INET6
 	if (last->inp_vflag & INP_IPV6) {
-		if (udp_in6.uin6_init_done == 0) {
-			in6_sin_2_v4mapsin6(&udp_in, &udp_in6.uin6_sin);
-			udp_in6.uin6_init_done = 1;
+		if (pudp_in6->uin6_init_done == 0) {
+			in6_sin_2_v4mapsin6(pudp_in, &pudp_in6->uin6_sin);
+			pudp_in6->uin6_init_done = 1;
 		}
-		append_sa = (struct sockaddr *)&udp_in6.uin6_sin;
+		append_sa = (struct sockaddr *)&pudp_in6->uin6_sin;
 	} else
 #endif
-	append_sa = (struct sockaddr *)&udp_in;
+	append_sa = (struct sockaddr *)pudp_in;
 	m_adj(n, off);
 	if (sbappendaddr(&last->inp_socket->so_rcv, append_sa, n, opts, NULL) == 0) {
 		udpstat.udps_fullsock++;
@@ -775,9 +818,65 @@ udp_ctlinput(cmd, sa, vip)
 		in_pcbnotifyall(&udbinfo, faddr, inetctlerrmap[cmd], notify);
 }
 
+int
+udp_ctloutput(struct socket *so, struct sockopt *sopt)
+{
+	int	error, optval;
+	struct	inpcb *inp;
+
+	if (sopt->sopt_level != IPPROTO_UDP)
+		return (ip_ctloutput(so, sopt));
+
+	error = 0;
+	inp = sotoinpcb(so);
+
+	switch (sopt->sopt_dir) {
+	case SOPT_SET:
+		switch (sopt->sopt_name) {
+		case UDP_NOCKSUM:
+			/* This option is settable only for UDP over IPv4 */
+			if (!(inp->inp_vflag & INP_IPV4)) {
+				error = EINVAL;
+				break;
+			}
+
+			if ((error = sooptcopyin(sopt, &optval, sizeof (optval),
+			    sizeof (optval))) != 0)
+				break;
+
+			if (optval != 0)
+				inp->inp_flags |= INP_UDP_NOCKSUM;
+			else
+				inp->inp_flags &= ~INP_UDP_NOCKSUM;
+			break;
+
+		default:
+			error = ENOPROTOOPT;
+			break;
+		}
+		break;
+
+	case SOPT_GET:
+		switch (sopt->sopt_name) {
+		case UDP_NOCKSUM:
+			optval = inp->inp_flags & INP_UDP_NOCKSUM;
+			break;
+
+		default:
+			error = ENOPROTOOPT;
+			break;
+		}
+		if (error == 0)
+			error = sooptcopyout(sopt, &optval, sizeof (optval));
+		break;
+	}
+	return (error);
+}
+
 static int
 udp_pcblist SYSCTL_HANDLER_ARGS
 {
+#pragma unused(oidp, arg1, arg2)
 	int error, i, n;
 	struct inpcb *inp, **inp_list;
 	inp_gen_t gencnt;
@@ -903,12 +1002,16 @@ udp_output(inp, m, addr, control, p)
 {
 	register struct udpiphdr *ui;
 	register int len = m->m_pkthdr.len;
-	struct sockaddr_in *sin, src;
+	struct sockaddr_in *sin;
 	struct in_addr origladdr, laddr, faddr;
 	u_short lport, fport;
 	struct sockaddr_in *ifaddr;
 	int error = 0, udp_dodisconnect = 0;
-
+	struct socket *so = inp->inp_socket;
+	int soopts;
+	struct mbuf *inpopts;
+	struct ip_moptions *mopts;
+	struct route ro;
 
 	KERNEL_DEBUG(DBG_FNC_UDP_OUTPUT | DBG_FUNC_START, 0,0,0,0,0);
 
@@ -996,6 +1099,10 @@ udp_output(inp, m, addr, control, p)
 		}
 	}
 
+#if CONFIG_MACF_NET
+	mac_mbuf_label_associate_inpcb(inp, m);
+#endif
+
 
 	/*
 	 * Calculate data length and get a mbuf
@@ -1023,7 +1130,7 @@ udp_output(inp, m, addr, control, p)
 	/*
 	 * Set up checksum and output datagram.
 	 */
-	if (udpcksum) {
+	if (udpcksum && !(inp->inp_flags & INP_UDP_NOCKSUM)) {
         	ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr,
 		    htons((u_short)len + sizeof(struct udphdr) + IPPROTO_UDP));
 		m->m_pkthdr.csum_flags = CSUM_UDP;
@@ -1046,11 +1153,27 @@ udp_output(inp, m, addr, control, p)
 	}
 #endif /*IPSEC*/
 	m->m_pkthdr.socket_id = get_socket_id(inp->inp_socket);
-	error = ip_output_list(m, 0,  inp->inp_options, &inp->inp_route,
-	    (inp->inp_socket->so_options & (SO_DONTROUTE | SO_BROADCAST)),
-	    inp->inp_moptions);
+
+	inpopts = inp->inp_options;
+	soopts = (inp->inp_socket->so_options & (SO_DONTROUTE | SO_BROADCAST));
+	mopts = inp->inp_moptions;
+
+	/* We don't want to cache the route for non-connected UDP */
+	if (udp_dodisconnect) {
+		bcopy(&inp->inp_route, &ro, sizeof (ro));
+		ro.ro_rt = NULL;
+	}
+
+	socket_unlock(so, 0);
+	/* XXX jgraessley please look at XXX */
+	error = ip_output_list(m, 0, inpopts,
+	    udp_dodisconnect ? &ro : &inp->inp_route, soopts, mopts, NULL);
+	socket_lock(so, 0);
 
 	if (udp_dodisconnect) {
+		/* Discard the cached route, if there is one */
+		if (ro.ro_rt != NULL)
+			rtfree(ro.ro_rt);
 		in_pcbdisconnect(inp);
 		inp->inp_laddr = origladdr;	/* XXX rehash? */
 	}
@@ -1091,21 +1214,21 @@ udp_abort(struct socket *so)
 
 	inp = sotoinpcb(so);
 	if (inp == 0)
-		panic("udp_abort: so=%x null inp\n", so);	/* ??? possible? panic instead? */
+		panic("udp_abort: so=%p null inp\n", so);	/* ??? possible? panic instead? */
 	soisdisconnected(so);
 	in_pcbdetach(inp);
 	return 0;
 }
 
 static int
-udp_attach(struct socket *so, int proto, struct proc *p)
+udp_attach(struct socket *so, __unused int proto, struct proc *p)
 {
 	struct inpcb *inp;
 	int error;
 
 	inp = sotoinpcb(so);
 	if (inp != 0)
-		panic ("udp_attach so=%x inp=%x\n", so, inp);
+		panic ("udp_attach so=%p inp=%p\n", so, inp);
 
 	error = in_pcballoc(so, &udbinfo, p);
 	if (error)
@@ -1125,6 +1248,10 @@ udp_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 	struct inpcb *inp;
 	int error;
 
+	if (nam->sa_family != 0 && nam->sa_family != AF_INET
+	    && nam->sa_family != AF_INET6) {
+		return EAFNOSUPPORT;
+	}
 	inp = sotoinpcb(so);
 	if (inp == 0)
 		return EINVAL;
@@ -1156,7 +1283,7 @@ udp_detach(struct socket *so)
 
 	inp = sotoinpcb(so);
 	if (inp == 0)
-		panic("udp_detach: so=%x null inp\n", so);	/* ??? possible? panic instead? */
+		panic("udp_detach: so=%p null inp\n", so);	/* ??? possible? panic instead? */
 	in_pcbdetach(inp);
 	inp->inp_state = INPCB_STATE_DEAD;
 	return 0;
@@ -1180,7 +1307,7 @@ udp_disconnect(struct socket *so)
 }
 
 static int
-udp_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
+udp_send(struct socket *so, __unused int flags, struct mbuf *m, struct sockaddr *addr,
 	    struct mbuf *control, struct proc *p)
 {
 	struct inpcb *inp;
@@ -1215,9 +1342,7 @@ struct pr_usrreqs udp_usrreqs = {
 
 
 int
-udp_lock(so, refcount, debug)
-	struct socket *so;
-	int refcount, debug;
+udp_lock(struct socket *so, int refcount, int debug)
 {
 	int lr_saved;
 	if (debug == 0) 
@@ -1229,25 +1354,20 @@ udp_lock(so, refcount, debug)
 		lck_mtx_lock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
 	else 
-		panic("udp_lock: so=%x NO PCB! lr=%x\n", so, lr_saved);
+		panic("udp_lock: so=%p NO PCB! lr=%x\n", so, lr_saved);
 
 	if (refcount) 
 		so->so_usecount++;
 
-	so->lock_lr[so->next_lock_lr] = (void *)lr_saved;
+	so->lock_lr[so->next_lock_lr] = (u_int32_t)lr_saved;
 	so->next_lock_lr = (so->next_lock_lr+1) % SO_LCKDBG_MAX;
 	return (0);
 }
 
 int
-udp_unlock(so, refcount, debug)
-	struct socket *so;
-	int refcount;
-	int debug;
+udp_unlock(struct socket *so, int refcount, int debug)
 {
 	int lr_saved;
-	struct inpcb *inp = sotoinpcb(so);
-    	struct inpcbinfo *pcbinfo	= &udbinfo;
 
 	if (debug == 0) 
 		lr_saved = (unsigned int) __builtin_return_address(0);
@@ -1256,20 +1376,26 @@ udp_unlock(so, refcount, debug)
 	if (refcount) {
 		so->so_usecount--;
 #if 0
-		if (so->so_usecount == 0 && (inp->inp_wantcnt == WNT_STOPUSING)) {
-			if (lck_rw_try_lock_exclusive(pcbinfo->mtx)) {
-				in_pcbdispose(inp);
-				lck_rw_done(pcbinfo->mtx);
-				return(0);
+		{
+			struct inpcb *inp = sotoinpcb(so);
+		    struct inpcbinfo *pcbinfo	= &udbinfo;
+		    
+			if (so->so_usecount == 0 && (inp->inp_wantcnt == WNT_STOPUSING)) {
+				
+				if (lck_rw_try_lock_exclusive(pcbinfo->mtx)) {
+					in_pcbdispose(inp);
+					lck_rw_done(pcbinfo->mtx);
+					return(0);
+				}
 			}
 		}
 #endif
 	}
 	if (so->so_pcb == NULL) 
-		panic("udp_unlock: so=%x NO PCB! lr=%x\n", so, lr_saved);
+		panic("udp_unlock: so=%p NO PCB! lr=%x\n", so, lr_saved);
 	else {
 		lck_mtx_assert(((struct inpcb *)so->so_pcb)->inpcb_mtx, LCK_MTX_ASSERT_OWNED);
-		so->unlock_lr[so->next_unlock_lr] = (void *)lr_saved;
+		so->unlock_lr[so->next_unlock_lr] = (u_int32_t)lr_saved;
 		so->next_unlock_lr = (so->next_unlock_lr+1) % SO_LCKDBG_MAX;
 		lck_mtx_unlock(((struct inpcb *)so->so_pcb)->inpcb_mtx);
 	}
@@ -1279,9 +1405,7 @@ udp_unlock(so, refcount, debug)
 }
 
 lck_mtx_t *
-udp_getlock(so, locktype)
-	struct socket *so;
-	int locktype;
+udp_getlock(struct socket *so, __unused int locktype)
 {
 	struct inpcb *inp = sotoinpcb(so);
 
@@ -1289,7 +1413,7 @@ udp_getlock(so, locktype)
 	if (so->so_pcb)
 		return(inp->inpcb_mtx);
 	else {
-		panic("udp_getlock: so=%x NULL so_pcb\n", so);
+		panic("udp_getlock: so=%p NULL so_pcb\n", so);
 		return (so->so_proto->pr_domain->dom_mtx);
 	}
 }
@@ -1301,7 +1425,15 @@ udp_slowtimo()
 	struct socket *so;
     	struct inpcbinfo *pcbinfo	= &udbinfo;
 
-	lck_rw_lock_exclusive(pcbinfo->mtx);
+	if (lck_rw_try_lock_exclusive(pcbinfo->mtx) == FALSE) {
+		if (udp_gc_done == TRUE) {
+			udp_gc_done = FALSE;
+			return; /* couldn't get the lock, better lock next time */
+		}
+		lck_rw_lock_exclusive(pcbinfo->mtx);
+	}
+
+	udp_gc_done = TRUE;
 
 	for (inp = udb.lh_first; inp != NULL; inp = inpnxt) {
 		inpnxt = inp->inp_list.le_next;
@@ -1334,3 +1466,16 @@ ChkAddressOK( __uint32_t dstaddr, __uint32_t srcaddr )
         return 1;
 }
 
+void
+udp_in_cksum_stats(u_int32_t len)
+{
+	udps_in_sw_cksum++;
+	udps_in_sw_cksum_bytes += len;
+}
+
+void
+udp_out_cksum_stats(u_int32_t len)
+{
+	udps_out_sw_cksum++;
+	udps_out_sw_cksum_bytes += len;
+}
diff --git a/bsd/netinet/udp_var.h b/bsd/netinet/udp_var.h
index 3de8e3add..19d4aad60 100644
--- a/bsd/netinet/udp_var.h
+++ b/bsd/netinet/udp_var.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -81,22 +87,22 @@ struct	udpiphdr {
 
 struct	udpstat {
 				/* input statistics: */
-	u_long	udps_ipackets;		/* total input packets */
-	u_long	udps_hdrops;		/* packet shorter than header */
-	u_long	udps_badsum;		/* checksum error */
-	u_long	udps_badlen;		/* data length larger than packet */
-	u_long	udps_noport;		/* no socket on port */
-	u_long	udps_noportbcast;	/* of above, arrived as broadcast */
-	u_long	udps_fullsock;		/* not delivered, input socket full */
-	u_long	udpps_pcbcachemiss;	/* input packets missing pcb cache */
-	u_long	udpps_pcbhashmiss;	/* input packets not for hashed pcb */
+	u_int32_t	udps_ipackets;		/* total input packets */
+	u_int32_t	udps_hdrops;		/* packet shorter than header */
+	u_int32_t	udps_badsum;		/* checksum error */
+	u_int32_t	udps_badlen;		/* data length larger than packet */
+	u_int32_t	udps_noport;		/* no socket on port */
+	u_int32_t	udps_noportbcast;	/* of above, arrived as broadcast */
+	u_int32_t	udps_fullsock;		/* not delivered, input socket full */
+	u_int32_t	udpps_pcbcachemiss;	/* input packets missing pcb cache */
+	u_int32_t	udpps_pcbhashmiss;	/* input packets not for hashed pcb */
 				/* output statistics: */
-	u_long	udps_opackets;		/* total output packets */
-	u_long	udps_fastout;		/* output packets on fast path */
+	u_int32_t	udps_opackets;		/* total output packets */
+	u_int32_t	udps_fastout;		/* output packets on fast path */
 #ifndef __APPLE__
-	u_long	udps_nosum;		/* no checksum */
-	/* of no socket on port, arrived as multicast */
-	u_long	udps_noportmcast;
+	u_int32_t	udps_nosum;		/* no checksum */
+				/* of no socket on port, arrived as multicast */
+	u_int32_t	udps_noportmcast;
 #endif
 };
 
@@ -131,7 +137,9 @@ extern struct	udpstat udpstat;
 extern int	log_in_vain;
 
 void	udp_ctlinput(int, struct sockaddr *, void *);
-void	udp_init(void);
+int	udp_ctloutput(struct socket *, struct sockopt *);
+
+void	udp_init(void) __attribute__((section("__TEXT, initcode")));
 void	udp_input(struct mbuf *, int);
 
 void	udp_notify(struct inpcb *inp, int errno);
diff --git a/bsd/netinet6/ah.h b/bsd/netinet6/ah.h
index e2a75ebd8..4b01eb27f 100644
--- a/bsd/netinet6/ah.h
+++ b/bsd/netinet6/ah.h
@@ -83,7 +83,7 @@ extern int ah_hdrlen(struct secasvar *);
 
 extern size_t ah_hdrsiz(struct ipsecrequest *);
 extern void ah4_input(struct mbuf *, int);
-extern int ah4_output(struct mbuf *, struct ipsecrequest *);
+extern int ah4_output(struct mbuf *, struct secasvar *);
 extern int ah4_calccksum(struct mbuf *, caddr_t, size_t,
 	const struct ah_algorithm *, struct secasvar *);
 #endif KERNEL_PRIVATE
diff --git a/bsd/netinet6/ah6.h b/bsd/netinet6/ah6.h
index 1525e3fc5..22cda6f12 100644
--- a/bsd/netinet6/ah6.h
+++ b/bsd/netinet6/ah6.h
@@ -43,7 +43,7 @@ struct secasvar;
 
 extern int ah6_input(struct mbuf **, int *);
 extern int ah6_output(struct mbuf *, u_char *, struct mbuf *,
-	struct ipsecrequest *);
+	struct secasvar *);
 extern int ah6_calccksum(struct mbuf *, caddr_t, size_t,
 	const struct ah_algorithm *, struct secasvar *);
 
diff --git a/bsd/netinet6/ah_core.c b/bsd/netinet6/ah_core.c
index c30b9c759..97527725c 100644
--- a/bsd/netinet6/ah_core.c
+++ b/bsd/netinet6/ah_core.c
@@ -78,8 +78,8 @@
 #endif
 #include <net/pfkeyv2.h>
 #include <netkey/keydb.h>
-#include <sys/md5.h>
-#include <crypto/sha1.h>
+#include <libkern/crypto/md5.h>
+#include <libkern/crypto/sha1.h>
 #include <crypto/sha2/sha2.h>
 
 #include <net/net_osdep.h>
@@ -108,6 +108,7 @@ static int ah_hmac_sha1_mature(struct secasvar *);
 static int ah_hmac_sha1_init(struct ah_algorithm_state *, struct secasvar *);
 static void ah_hmac_sha1_loop(struct ah_algorithm_state *, caddr_t, size_t);
 static void ah_hmac_sha1_result(struct ah_algorithm_state *, caddr_t);
+#if ALLCRYPTO
 static int ah_hmac_sha2_256_mature(struct secasvar *);
 static int ah_hmac_sha2_256_init(struct ah_algorithm_state *,
 	struct secasvar *);
@@ -123,6 +124,7 @@ static int ah_hmac_sha2_512_init(struct ah_algorithm_state *,
 	struct secasvar *);
 static void ah_hmac_sha2_512_loop(struct ah_algorithm_state *, caddr_t, size_t);
 static void ah_hmac_sha2_512_result(struct ah_algorithm_state *, caddr_t);
+#endif /* ALLCRYPTO */
 
 static void ah_update_mbuf(struct mbuf *, int, int,
 	const struct ah_algorithm *, struct ah_algorithm_state *);
@@ -132,52 +134,62 @@ ah_algorithm_lookup(idx)
 	int idx;
 {
 	/* checksum algorithms */
-	static struct ah_algorithm ah_algorithms[] = {
+	static struct ah_algorithm hmac_md5 =
 		{ ah_sumsiz_1216, ah_hmac_md5_mature, 128, 128, "hmac-md5",
 			ah_hmac_md5_init, ah_hmac_md5_loop,
-			ah_hmac_md5_result, },
-		{ ah_sumsiz_1216, ah_hmac_sha1_mature, 160, 160, "hmac-sha1",
-			ah_hmac_sha1_init, ah_hmac_sha1_loop,
-			ah_hmac_sha1_result, },
+			ah_hmac_md5_result, };
+	static struct ah_algorithm keyed_md5 =
 		{ ah_sumsiz_1216, ah_keyed_md5_mature, 128, 128, "keyed-md5",
 			ah_keyed_md5_init, ah_keyed_md5_loop,
-			ah_keyed_md5_result, },
+			ah_keyed_md5_result, };
+	static struct ah_algorithm hmac_sha1 =
+		{ ah_sumsiz_1216, ah_hmac_sha1_mature, 160, 160, "hmac-sha1",
+			ah_hmac_sha1_init, ah_hmac_sha1_loop,
+			ah_hmac_sha1_result, };
+	static struct ah_algorithm keyed_sha1 =
 		{ ah_sumsiz_1216, ah_keyed_sha1_mature, 160, 160, "keyed-sha1",
 			ah_keyed_sha1_init, ah_keyed_sha1_loop,
-			ah_keyed_sha1_result, },
+			ah_keyed_sha1_result, };
+	static struct ah_algorithm ah_none =
 		{ ah_sumsiz_zero, ah_none_mature, 0, 2048, "none",
-			ah_none_init, ah_none_loop, ah_none_result, },
+			ah_none_init, ah_none_loop, ah_none_result, };
+#if ALLCRYPTO
+	static struct ah_algorithm hmac_sha2_256 =
 		{ ah_sumsiz_1216, ah_hmac_sha2_256_mature, 256, 256,
 			"hmac-sha2-256",
 			ah_hmac_sha2_256_init, ah_hmac_sha2_256_loop,
-			ah_hmac_sha2_256_result, },
+			ah_hmac_sha2_256_result, };
+	static struct ah_algorithm hmac_sha2_384 =
 		{ ah_sumsiz_1216, ah_hmac_sha2_384_mature, 384, 384,
 			"hmac-sha2-384",
 			ah_hmac_sha2_384_init, ah_hmac_sha2_384_loop,
-			ah_hmac_sha2_384_result, },
+			ah_hmac_sha2_384_result, };
+	static struct ah_algorithm hmac_sha2_512 =
 		{ ah_sumsiz_1216, ah_hmac_sha2_512_mature, 512, 512,
 			"hmac-sha2-512",
 			ah_hmac_sha2_512_init, ah_hmac_sha2_512_loop,
-			ah_hmac_sha2_512_result, },
-	};
+			ah_hmac_sha2_512_result, };
+#endif /* ALLCRYPTO */
 
 	switch (idx) {
 	case SADB_AALG_MD5HMAC:
-		return &ah_algorithms[0];
+		return &hmac_md5;
 	case SADB_AALG_SHA1HMAC:
-		return &ah_algorithms[1];
+		return &hmac_sha1;
 	case SADB_X_AALG_MD5:
-		return &ah_algorithms[2];
+		return &keyed_md5;
 	case SADB_X_AALG_SHA:
-		return &ah_algorithms[3];
+		return &keyed_sha1;
 	case SADB_X_AALG_NULL:
-		return &ah_algorithms[4];
+		return &ah_none;
+#if ALLCRYPTO
 	case SADB_X_AALG_SHA2_256:
-		return &ah_algorithms[5];
+		return &hmac_sha2_256;
 	case SADB_X_AALG_SHA2_384:
-		return &ah_algorithms[6];
+		return &hmac_sha2_384;
 	case SADB_X_AALG_SHA2_512:
-		return &ah_algorithms[7];
+		return &hmac_sha2_512;
+#endif /* ALLCRYPTO */
 	default:
 		return NULL;
 	}
@@ -218,32 +230,32 @@ ah_none_mature(sav)
 }
 
 static int
-ah_none_init(state, sav)
-	struct ah_algorithm_state *state;
-	struct secasvar *sav;
+ah_none_init(
+	struct ah_algorithm_state *state,
+	__unused struct secasvar *sav)
 {
 	state->foo = NULL;
 	return 0;
 }
 
 static void
-ah_none_loop(state, addr, len)
-	struct ah_algorithm_state *state;
-	caddr_t addr;
-	size_t len;
+ah_none_loop(
+	__unused struct ah_algorithm_state *state,
+	__unused caddr_t addr,
+	__unused size_t len)
 {
 }
 
 static void
-ah_none_result(state, addr)
-	struct ah_algorithm_state *state;
-	caddr_t addr;
+ah_none_result(
+	__unused struct ah_algorithm_state *state,
+	__unused caddr_t addr)
 {
 }
 
 static int
-ah_keyed_md5_mature(sav)
-	struct secasvar *sav;
+ah_keyed_md5_mature(
+	__unused struct secasvar *sav)
 {
 	/* anything is okay */
 	return 0;
@@ -262,7 +274,7 @@ ah_keyed_md5_init(state, sav)
 		panic("ah_keyed_md5_init: what?");
 
 	state->sav = sav;
-	state->foo = (void *)_MALLOC(sizeof(MD5_CTX), M_TEMP, M_WAITOK);
+	state->foo = (void *)_MALLOC(sizeof(MD5_CTX), M_TEMP, M_NOWAIT);
 	if (state->foo == NULL)
 		return ENOBUFS;
 
@@ -381,7 +393,7 @@ ah_keyed_sha1_init(state, sav)
 		panic("ah_keyed_sha1_init: what?");
 
 	state->sav = sav;
-	state->foo = (void *)_MALLOC(sizeof(SHA1_CTX), M_TEMP, M_WAITOK);
+	state->foo = (void *)_MALLOC(sizeof(SHA1_CTX), M_TEMP, M_NOWAIT);
 	if (!state->foo)
 		return ENOBUFS;
 
@@ -507,7 +519,7 @@ ah_hmac_md5_init(state, sav)
 		panic("ah_hmac_md5_init: what?");
 
 	state->sav = sav;
-	state->foo = (void *)_MALLOC(64 + 64 + sizeof(MD5_CTX), M_TEMP, M_WAITOK);
+	state->foo = (void *)_MALLOC(64 + 64 + sizeof(MD5_CTX), M_TEMP, M_NOWAIT);
 	if (!state->foo)
 		return ENOBUFS;
 
@@ -632,7 +644,7 @@ ah_hmac_sha1_init(state, sav)
 
 	state->sav = sav;
 	state->foo = (void *)_MALLOC(64 + 64 + sizeof(SHA1_CTX),
-			M_TEMP, M_WAITOK);
+			M_TEMP, M_NOWAIT);
 	if (!state->foo)
 		return ENOBUFS;
 
@@ -712,6 +724,7 @@ ah_hmac_sha1_result(state, addr)
 	FREE(state->foo, M_TEMP);
 }
 
+#if ALLCRYPTO
 static int
 ah_hmac_sha2_256_mature(sav)
 	struct secasvar *sav;
@@ -1109,6 +1122,7 @@ ah_hmac_sha2_512_result(state, addr)
 
 	FREE(state->foo, M_TEMP);
 }
+#endif /* ALLCRYPTO */
 
 /*------------------------------------------------------------*/
 
diff --git a/bsd/netinet6/ah_input.c b/bsd/netinet6/ah_input.c
index 06340aa1e..b79bf0895 100644
--- a/bsd/netinet6/ah_input.c
+++ b/bsd/netinet6/ah_input.c
@@ -86,12 +86,13 @@
 #define	KEYDEBUG(lev,arg)
 #endif
 
+#include <net/kpi_protocol.h>
+#include <netinet/kpi_ipfilter_var.h>
 
 #include <net/net_osdep.h>
 
 #define IPLEN_FLIPPED
 
-extern lck_mtx_t *sadb_mutex;
 #if INET
 extern struct protosw inetsw[];
 
@@ -108,10 +109,8 @@ ah4_input(struct mbuf *m, int off)
 	struct secasvar *sav = NULL;
 	u_int16_t nxt;
 	size_t hlen;
-	int s;
 	size_t stripsiz = 0;
-
-	lck_mtx_lock(sadb_mutex);
+	sa_family_t ifamily;
 
 #ifndef PULLDOWN_TEST
 	if (m->m_len < off + sizeof(struct newah)) {
@@ -119,7 +118,7 @@ ah4_input(struct mbuf *m, int off)
 		if (!m) {
 			ipseclog((LOG_DEBUG, "IPv4 AH input: can't pullup;"
 				"dropping the packet for simplicity\n"));
-			ipsecstat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 			goto fail;
 		}
 	}
@@ -132,7 +131,7 @@ ah4_input(struct mbuf *m, int off)
 	if (ah == NULL) {
 		ipseclog((LOG_DEBUG, "IPv4 AH input: can't pullup;"
 			"dropping the packet for simplicity\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 #endif
@@ -152,7 +151,7 @@ ah4_input(struct mbuf *m, int off)
 		ipseclog((LOG_WARNING,
 		    "IPv4 AH input: no key association found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_nosa);
 		goto fail;
 	}
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -162,7 +161,7 @@ ah4_input(struct mbuf *m, int off)
 		ipseclog((LOG_DEBUG,
 		    "IPv4 AH input: non-mature/dying SA found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
 		goto fail;
 	}
 
@@ -171,7 +170,7 @@ ah4_input(struct mbuf *m, int off)
 		ipseclog((LOG_DEBUG, "IPv4 AH input: "
 		    "unsupported authentication algorithm for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
 		goto fail;
 	}
 
@@ -212,7 +211,7 @@ ah4_input(struct mbuf *m, int off)
 		    "(%lu, should be at least %lu): %s\n",
 		    (u_long)siz1, (u_long)siz,
 		    ipsec4_logpacketstr(ip, spi)));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 	if ((ah->ah_len << 2) - sizoff != siz1) {
@@ -220,7 +219,7 @@ ah4_input(struct mbuf *m, int off)
 		    "(%d should be %lu): %s\n",
 		    (ah->ah_len << 2) - sizoff, (u_long)siz1,
 		    ipsec4_logpacketstr(ip, spi)));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 
@@ -229,7 +228,7 @@ ah4_input(struct mbuf *m, int off)
 		m = m_pullup(m, off + sizeof(struct ah) + sizoff + siz1);
 		if (!m) {
 			ipseclog((LOG_DEBUG, "IPv4 AH input: can't pullup\n"));
-			ipsecstat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 			goto fail;
 		}
 
@@ -241,7 +240,7 @@ ah4_input(struct mbuf *m, int off)
 		sizeof(struct ah) + sizoff + siz1);
 	if (ah == NULL) {
 		ipseclog((LOG_DEBUG, "IPv4 AH input: can't pullup\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 #endif
@@ -254,7 +253,7 @@ ah4_input(struct mbuf *m, int off)
 		if (ipsec_chkreplay(ntohl(((struct newah *)ah)->ah_seq), sav))
 			; /*okey*/
 		else {
-			ipsecstat.in_ahreplay++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_ahreplay);
 			ipseclog((LOG_WARNING,
 			    "replay packet in IPv4 AH input: %s %s\n",
 			    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
@@ -270,7 +269,7 @@ ah4_input(struct mbuf *m, int off)
 	if (!cksum) {
 		ipseclog((LOG_DEBUG, "IPv4 AH input: "
 		    "couldn't alloc temporary region for cksum\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 	
@@ -282,10 +281,10 @@ ah4_input(struct mbuf *m, int off)
 	ip->ip_off = htons(ip->ip_off);
 	if (ah4_calccksum(m, (caddr_t)cksum, siz1, algo, sav)) {
 		FREE(cksum, M_TEMP);
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
-	ipsecstat.in_ahhist[sav->alg_auth]++;
+	IPSEC_STAT_INCREMENT(ipsecstat.in_ahhist[sav->alg_auth]);
 	/*
 	 * flip them back.
 	 */
@@ -308,7 +307,7 @@ ah4_input(struct mbuf *m, int off)
 		    "checksum mismatch in IPv4 AH input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
 		FREE(cksum, M_TEMP);
-		ipsecstat.in_ahauthfail++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_ahauthfail);
 		goto fail;
 	}
     }
@@ -335,7 +334,7 @@ ah4_input(struct mbuf *m, int off)
 			if (!m) {
 				ipseclog((LOG_DEBUG,
 				    "IPv4 AH input: can't pullup\n"));
-				ipsecstat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 				goto fail;
 			}
 		}
@@ -361,12 +360,12 @@ ah4_input(struct mbuf *m, int off)
 		ipseclog((LOG_DEBUG,
 		    "IPv4 AH input: authentication succeess\n"));
 #endif
-		ipsecstat.in_ahauthsucc++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_ahauthsucc);
 	} else {
 		ipseclog((LOG_WARNING,
 		    "authentication failed in IPv4 AH input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-		ipsecstat.in_ahauthfail++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_ahauthfail);
 		goto fail;
 	}
 
@@ -375,7 +374,7 @@ ah4_input(struct mbuf *m, int off)
 	 */
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
 		if (ipsec_updatereplay(ntohl(((struct newah *)ah)->ah_seq), sav)) {
-			ipsecstat.in_ahreplay++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_ahreplay);
 			goto fail;
 		}
 	}
@@ -388,7 +387,7 @@ ah4_input(struct mbuf *m, int off)
 		/* RFC 2402 */
 		stripsiz = sizeof(struct newah) + siz1;
 	}
-	if (ipsec4_tunnel_validate(m, off + stripsiz, nxt, sav)) {
+	if (ipsec4_tunnel_validate(m, off + stripsiz, nxt, sav, &ifamily)) {
 		/*
 		 * strip off all the headers that precedes AH.
 		 *	IP xx AH IP' payload -> IP' payload
@@ -397,13 +396,18 @@ ah4_input(struct mbuf *m, int off)
 		 * XXX relationship with gif?
 		 */
 		u_int8_t tos;
-
+		
+		if (ifamily == AF_INET6) {
+			ipseclog((LOG_NOTICE, "ipsec tunnel protocol mismatch "
+			    "in IPv4 AH input: %s\n", ipsec_logsastr(sav)));
+			goto fail;
+		}
 		tos = ip->ip_tos;
 		m_adj(m, off + stripsiz);
 		if (m->m_len < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (!m) {
-				ipsecstat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 				goto fail;
 			}
 		}
@@ -415,7 +419,7 @@ ah4_input(struct mbuf *m, int off)
 			ipseclog((LOG_NOTICE, "ipsec tunnel address mismatch "
 			    "in IPv4 AH input: %s %s\n",
 			    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-			ipsecstat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 			goto fail;
 		}
 
@@ -446,12 +450,10 @@ ah4_input(struct mbuf *m, int off)
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_AH, spi) != 0 ||
 		    ipsec_addhist(m, IPPROTO_IPV4, 0) != 0) {
-			ipsecstat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 			goto fail;
 		}
-		lck_mtx_unlock(sadb_mutex);
 		proto_input(PF_INET, m);
-		lck_mtx_lock(sadb_mutex);
 		nxt = IPPROTO_DONE;
 	} else {
 		/*
@@ -500,7 +502,7 @@ ah4_input(struct mbuf *m, int off)
 		if (m->m_len < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (m == NULL) {
-				ipsecstat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 				goto fail;
 			}
 		}
@@ -515,19 +517,17 @@ ah4_input(struct mbuf *m, int off)
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_AH, spi) != 0) {
-			ipsecstat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 			goto fail;
 		}
 
 		if (nxt != IPPROTO_DONE) {
 			if ((ip_protox[nxt]->pr_flags & PR_LASTHDR) != 0 &&
 			    ipsec4_in_reject(m, NULL)) {
-				ipsecstat.in_polvio++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 				goto fail;
 			}
-			lck_mtx_unlock(sadb_mutex);
 			ip_proto_dispatch_in(m, off, nxt, 0);
-			lck_mtx_lock(sadb_mutex);
 		} else
 			m_freem(m);
 		m = NULL;
@@ -536,19 +536,17 @@ ah4_input(struct mbuf *m, int off)
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP ah4_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	ipsecstat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsecstat.in_success);
 	return;
 
 fail:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP ah4_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	lck_mtx_unlock(sadb_mutex);
 	if (m)
 		m_freem(m);
 	return;
@@ -572,20 +570,17 @@ ah6_input(mp, offp)
 	u_char *cksum;
 	struct secasvar *sav = NULL;
 	u_int16_t nxt;
-	int s;
 	size_t stripsiz = 0;
 
-	lck_mtx_lock(sadb_mutex);
 
 #ifndef PULLDOWN_TEST
-	IP6_EXTHDR_CHECK(m, off, sizeof(struct ah), {lck_mtx_unlock(sadb_mutex);return IPPROTO_DONE;});
+	IP6_EXTHDR_CHECK(m, off, sizeof(struct ah), {return IPPROTO_DONE;});
 	ah = (struct ah *)(mtod(m, caddr_t) + off);
 #else
 	IP6_EXTHDR_GET(ah, struct ah *, m, off, sizeof(struct newah));
 	if (ah == NULL) {
 		ipseclog((LOG_DEBUG, "IPv6 AH input: can't pullup\n"));
 		ipsec6stat.in_inval++;
-		lck_mtx_unlock(sadb_mutex);
 		return IPPROTO_DONE;
 	}
 #endif
@@ -598,7 +593,7 @@ ah6_input(mp, offp)
 	if (ntohs(ip6->ip6_plen) == 0) {
 		ipseclog((LOG_ERR, "IPv6 AH input: "
 		    "AH with IPv6 jumbogram is not supported.\n"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
 
@@ -608,7 +603,7 @@ ah6_input(mp, offp)
 		ipseclog((LOG_WARNING,
 		    "IPv6 AH input: no key association found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_nosa);
 		goto fail;
 	}
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -618,7 +613,7 @@ ah6_input(mp, offp)
 		ipseclog((LOG_DEBUG,
 		    "IPv6 AH input: non-mature/dying SA found for spi %u; ",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
 		goto fail;
 	}
 
@@ -627,7 +622,7 @@ ah6_input(mp, offp)
 		ipseclog((LOG_DEBUG, "IPv6 AH input: "
 		    "unsupported authentication algorithm for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
 		goto fail;
 	}
 
@@ -651,7 +646,7 @@ ah6_input(mp, offp)
 		    "(%lu, should be at least %lu): %s\n",
 		    (u_long)siz1, (u_long)siz,
 		    ipsec6_logpacketstr(ip6, spi)));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
 	if ((ah->ah_len << 2) - sizoff != siz1) {
@@ -659,18 +654,18 @@ ah6_input(mp, offp)
 		    "(%d should be %lu): %s\n",
 		    (ah->ah_len << 2) - sizoff, (u_long)siz1,
 		    ipsec6_logpacketstr(ip6, spi)));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
 #ifndef PULLDOWN_TEST
 	IP6_EXTHDR_CHECK(m, off, sizeof(struct ah) + sizoff + siz1, 
-		{lck_mtx_unlock(sadb_mutex);return IPPROTO_DONE;});
+		{return IPPROTO_DONE;});
 #else
 	IP6_EXTHDR_GET(ah, struct ah *, m, off,
 		sizeof(struct ah) + sizoff + siz1);
 	if (ah == NULL) {
 		ipseclog((LOG_NOTICE, "couldn't pullup gather IPv6 AH checksum part"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		m = NULL;
 		goto fail;
 	}
@@ -684,7 +679,7 @@ ah6_input(mp, offp)
 		if (ipsec_chkreplay(ntohl(((struct newah *)ah)->ah_seq), sav))
 			; /*okey*/
 		else {
-			ipsec6stat.in_ahreplay++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_ahreplay);
 			ipseclog((LOG_WARNING,
 			    "replay packet in IPv6 AH input: %s %s\n",
 			    ipsec6_logpacketstr(ip6, spi),
@@ -701,16 +696,16 @@ ah6_input(mp, offp)
 	if (!cksum) {
 		ipseclog((LOG_DEBUG, "IPv6 AH input: "
 		    "couldn't alloc temporary region for cksum\n"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
 	
 	if (ah6_calccksum(m, (caddr_t)cksum, siz1, algo, sav)) {
 		FREE(cksum, M_TEMP);
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
-	ipsec6stat.in_ahhist[sav->alg_auth]++;
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_ahhist[sav->alg_auth]);
 
     {
 	caddr_t sumpos = NULL;
@@ -728,7 +723,7 @@ ah6_input(mp, offp)
 		    "checksum mismatch in IPv6 AH input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
 		FREE(cksum, M_TEMP);
-		ipsec6stat.in_ahauthfail++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_ahauthfail);
 		goto fail;
 	}
     }
@@ -751,7 +746,7 @@ ah6_input(mp, offp)
 
 		IP6_EXTHDR_CHECK(m, off, sizeof(struct ah) + sizoff + siz1
 				+ sizeof(struct ip6_hdr), 
-				{lck_mtx_unlock(sadb_mutex);return IPPROTO_DONE;});
+				{return IPPROTO_DONE;});
 
 		nip6 = (struct ip6_hdr *)((u_char *)(ah + 1) + sizoff + siz1);
 		if (!IN6_ARE_ADDR_EQUAL(&nip6->ip6_src, &ip6->ip6_src)
@@ -774,12 +769,12 @@ ah6_input(mp, offp)
 		ipseclog((LOG_DEBUG,
 		    "IPv6 AH input: authentication succeess\n"));
 #endif
-		ipsec6stat.in_ahauthsucc++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_ahauthsucc);
 	} else {
 		ipseclog((LOG_WARNING,
 		    "authentication failed in IPv6 AH input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-		ipsec6stat.in_ahauthfail++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_ahauthfail);
 		goto fail;
 	}
 
@@ -788,7 +783,7 @@ ah6_input(mp, offp)
 	 */
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
 		if (ipsec_updatereplay(ntohl(((struct newah *)ah)->ah_seq), sav)) {
-			ipsec6stat.in_ahreplay++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_ahreplay);
 			goto fail;
 		}
 	}
@@ -820,7 +815,7 @@ ah6_input(mp, offp)
 			 */
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m) {
-				ipsec6stat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 				goto fail;
 			}
 		}
@@ -833,7 +828,7 @@ ah6_input(mp, offp)
 			    "in IPv6 AH input: %s %s\n",
 			    ipsec6_logpacketstr(ip6, spi),
 			    ipsec_logsastr(sav)));
-			ipsec6stat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 			goto fail;
 		}
 
@@ -849,12 +844,10 @@ ah6_input(mp, offp)
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_AH, spi) != 0 ||
 		    ipsec_addhist(m, IPPROTO_IPV6, 0) != 0) {
-			ipsec6stat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 			goto fail;
 		}
-		lck_mtx_unlock(sadb_mutex);
 		proto_input(PF_INET6, m);
-		lck_mtx_lock(sadb_mutex);
 		nxt = IPPROTO_DONE;
 	} else {
 		/*
@@ -914,7 +907,7 @@ ah6_input(mp, offp)
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_AH, spi) != 0) {
-			ipsec6stat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 			goto fail;
 		}
 	}
@@ -925,19 +918,17 @@ ah6_input(mp, offp)
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP ah6_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	ipsec6stat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_success);
 	return nxt;
 
 fail:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP ah6_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	lck_mtx_unlock(sadb_mutex);
 	if (m)
 		m_freem(m);
 	return IPPROTO_DONE;
@@ -1004,7 +995,6 @@ ah6_ctlinput(cmd, sa, d)
 			 */
 			sa6_src = ip6cp->ip6c_src;
 			sa6_dst = (struct sockaddr_in6 *)sa;
-			lck_mtx_lock(sadb_mutex);
 			sav = key_allocsa(AF_INET6,
 					  (caddr_t)&sa6_src->sin6_addr,
 					  (caddr_t)&sa6_dst->sin6_addr,
@@ -1013,9 +1003,8 @@ ah6_ctlinput(cmd, sa, d)
 				if (sav->state == SADB_SASTATE_MATURE ||
 				    sav->state == SADB_SASTATE_DYING)
 					valid++;
-				key_freesav(sav);
+				key_freesav(sav, KEY_SADB_UNLOCKED);
 			}
-			lck_mtx_unlock(sadb_mutex);
 
 			/* XXX Further validation? */
 
diff --git a/bsd/netinet6/ah_output.c b/bsd/netinet6/ah_output.c
index 4a7940c46..13c4157b0 100644
--- a/bsd/netinet6/ah_output.c
+++ b/bsd/netinet6/ah_output.c
@@ -81,6 +81,8 @@
 static struct in_addr *ah4_finaldst(struct mbuf *);
 #endif
 
+extern lck_mtx_t *sadb_mutex;
+
 /*
  * compute AH header size.
  * transport mode only.  for tunnel mode, we should implement
@@ -90,8 +92,6 @@ size_t
 ah_hdrsiz(isr)
 	struct ipsecrequest *isr;
 {
-	const struct ah_algorithm *algo;
-	size_t hdrsiz;
 
 	/* sanity check */
 	if (isr == NULL)
@@ -100,33 +100,46 @@ ah_hdrsiz(isr)
 	if (isr->saidx.proto != IPPROTO_AH)
 		panic("unsupported mode passed to ah_hdrsiz");
 
-	if (isr->sav == NULL)
-		goto estimate;
-	if (isr->sav->state != SADB_SASTATE_MATURE
-	 && isr->sav->state != SADB_SASTATE_DYING)
-		goto estimate;
+#if 0
+	{
 
-	/* we need transport mode AH. */
-	algo = ah_algorithm_lookup(isr->sav->alg_auth);
-	if (!algo)
-		goto estimate;
+		lck_mtx_lock(sadb_mutex);
+		const struct ah_algorithm *algo;
+		size_t hdrsiz;
 
-	/*
-	 * XXX
-	 * right now we don't calcurate the padding size.  simply
-	 * treat the padding size as constant, for simplicity.
-	 *
-	 * XXX variable size padding support
-	 */
-	hdrsiz = (((*algo->sumsiz)(isr->sav) + 3) & ~(4 - 1));
-	if (isr->sav->flags & SADB_X_EXT_OLD)
-		hdrsiz += sizeof(struct ah);
-	else
-		hdrsiz += sizeof(struct newah);
+		/*%%%%% this needs to change - no sav in ipsecrequest any more */
+		if (isr->sav == NULL)
+			goto estimate;
+		if (isr->sav->state != SADB_SASTATE_MATURE
+		 && isr->sav->state != SADB_SASTATE_DYING)
+			goto estimate;
+	
+		/* we need transport mode AH. */
+		algo = ah_algorithm_lookup(isr->sav->alg_auth);
+		if (!algo)
+			goto estimate;
+	
+		/*
+		 * XXX
+		 * right now we don't calcurate the padding size.  simply
+		 * treat the padding size as constant, for simplicity.
+		 *
+		 * XXX variable size padding support
+		 */
+		hdrsiz = (((*algo->sumsiz)(isr->sav) + 3) & ~(4 - 1));
+		if (isr->sav->flags & SADB_X_EXT_OLD)
+			hdrsiz += sizeof(struct ah);
+		else
+			hdrsiz += sizeof(struct newah);
+	
+		lck_mtx_unlock(sadb_mutex);
+		return hdrsiz;
+	}
 
-	return hdrsiz;
+estimate:
+#endif
 
-    estimate:
+    //lck_mtx_unlock(sadb_mutex);
 	/* ASSUMING:
 	 *	sizeof(struct newah) > sizeof(struct ah).
 	 *	16 = (16 + 3) & ~(4 - 1).
@@ -143,11 +156,10 @@ ah_hdrsiz(isr)
  * the function does not modify m.
  */
 int
-ah4_output(m, isr)
+ah4_output(m, sav)
 	struct mbuf *m;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
-	struct secasvar *sav = isr->sav;
 	const struct ah_algorithm *algo;
 	u_int32_t spi;
 	u_char *ahdrpos;
@@ -156,21 +168,19 @@ ah4_output(m, isr)
 	size_t plen = 0;	/*AH payload size in bytes*/
 	size_t ahlen = 0;	/*plen + sizeof(ah)*/
 	struct ip *ip;
-	struct in_addr dst;
+	struct in_addr dst = { 0 };
 	struct in_addr *finaldst;
 	int error;
 
 	/* sanity checks */
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && !sav->replay) {
-		struct ip *ip;
-
 		ip = mtod(m, struct ip *);
 		ipseclog((LOG_DEBUG, "ah4_output: internal error: "
 			"sav->replay is null: %x->%x, SPI=%u\n",
 			(u_int32_t)ntohl(ip->ip_src.s_addr),
 			(u_int32_t)ntohl(ip->ip_dst.s_addr),
 			(u_int32_t)ntohl(sav->spi)));
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -179,7 +189,7 @@ ah4_output(m, isr)
 	if (!algo) {
 		ipseclog((LOG_ERR, "ah4_output: unsupported algorithm: "
 		    "SPI=%u\n", (u_int32_t)ntohl(sav->spi)));
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -261,12 +271,14 @@ ah4_output(m, isr)
 				ipseclog((LOG_WARNING,
 				    "replay counter overflowed. %s\n",
 				    ipsec_logsastr(sav)));
-				ipsecstat.out_inval++;
+				IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 				m_freem(m);
 				return EINVAL;
 			}
 		}
+		lck_mtx_lock(sadb_mutex);
 		sav->replay->count++;
+		lck_mtx_unlock(sadb_mutex);
 		/*
 		 * XXX sequence number must not be cycled, if the SA is
 		 * installed by IKE daemon.
@@ -283,7 +295,7 @@ ah4_output(m, isr)
 		ip->ip_len = htons(ntohs(ip->ip_len) + ahlen);
 	else {
 		ipseclog((LOG_ERR, "IPv4 AH output: size exceeds limit\n"));
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 		m_freem(m);
 		return EMSGSIZE;
 	}
@@ -311,7 +323,7 @@ ah4_output(m, isr)
 		    "error after ah4_calccksum, called from ah4_output"));
 		m_freem(m);
 		m = NULL;
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 		return error;
 	}
 
@@ -319,8 +331,10 @@ ah4_output(m, isr)
 		ip = mtod(m, struct ip *);	/*just to make sure*/
 		ip->ip_dst.s_addr = dst.s_addr;
 	}
+	lck_mtx_lock(sadb_stat_mutex);
 	ipsecstat.out_success++;
 	ipsecstat.out_ahhist[sav->alg_auth]++;
+	lck_mtx_unlock(sadb_stat_mutex);
 	key_sa_recordxfer(sav, m);
 
 	return 0;
@@ -356,15 +370,14 @@ ah_hdrlen(sav)
  * Fill in the Authentication Header and calculate checksum.
  */
 int
-ah6_output(m, nexthdrp, md, isr)
+ah6_output(m, nexthdrp, md, sav)
 	struct mbuf *m;
 	u_char *nexthdrp;
 	struct mbuf *md;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
 	struct mbuf *mprev;
 	struct mbuf *mah;
-	struct secasvar *sav = isr->sav;
 	const struct ah_algorithm *algo;
 	u_int32_t spi;
 	u_char *ahsumpos = NULL;
@@ -421,9 +434,9 @@ ah6_output(m, nexthdrp, md, isr)
 
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && !sav->replay) {
 		ipseclog((LOG_DEBUG, "ah6_output: internal error: "
-			"sav->replay is null: SPI=%u\n",
-			(u_int32_t)ntohl(sav->spi)));
-		ipsec6stat.out_inval++;
+			  "sav->replay is null: SPI=%u\n",
+			  (u_int32_t)ntohl(sav->spi)));
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -432,7 +445,7 @@ ah6_output(m, nexthdrp, md, isr)
 	if (!algo) {
 		ipseclog((LOG_ERR, "ah6_output: unsupported algorithm: "
 		    "SPI=%u\n", (u_int32_t)ntohl(sav->spi)));
-		ipsec6stat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -466,14 +479,16 @@ ah6_output(m, nexthdrp, md, isr)
 			if ((sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
 				/* XXX Is it noisy ? */
 				ipseclog((LOG_WARNING,
-				    "replay counter overflowed. %s\n",
+				     "replay counter overflowed. %s\n",
 				    ipsec_logsastr(sav)));
-				ipsec6stat.out_inval++;
+				IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 				m_freem(m);
 				return EINVAL;
 			}
 		}
+		lck_mtx_lock(sadb_mutex);
 		sav->replay->count++;
+		lck_mtx_unlock(sadb_mutex);
 		/*
 		 * XXX sequence number must not be cycled, if the SA is
 		 * installed by IKE daemon.
@@ -488,13 +503,13 @@ ah6_output(m, nexthdrp, md, isr)
 	 */
 	error = ah6_calccksum(m, (caddr_t)ahsumpos, plen, algo, sav);
 	if (error) {
-		ipsec6stat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		m_freem(m);
 	} else {
-		ipsec6stat.out_success++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_success);
 		key_sa_recordxfer(sav, m);
 	}
-	ipsec6stat.out_ahhist[sav->alg_auth]++;
+	IPSEC_STAT_INCREMENT(ipsec6stat.out_ahhist[sav->alg_auth]);
 
 	return(error);
 }
diff --git a/bsd/netinet6/esp.h b/bsd/netinet6/esp.h
index 6802e81a3..e2c59fc81 100644
--- a/bsd/netinet6/esp.h
+++ b/bsd/netinet6/esp.h
@@ -95,7 +95,7 @@ extern const struct esp_algorithm *esp_algorithm_lookup(int);
 extern int esp_max_ivlen(void);
 
 /* crypt routines */
-extern int esp4_output(struct mbuf *, struct ipsecrequest *);
+extern int esp4_output(struct mbuf *, struct secasvar *);
 extern void esp4_input(struct mbuf *, int off);
 extern size_t esp_hdrsiz(struct ipsecrequest *);
 
diff --git a/bsd/netinet6/esp6.h b/bsd/netinet6/esp6.h
index bb2c20529..55c0d8043 100644
--- a/bsd/netinet6/esp6.h
+++ b/bsd/netinet6/esp6.h
@@ -40,7 +40,7 @@
 
 #ifdef KERNEL_PRIVATE
 extern int esp6_output(struct mbuf *, u_char *, struct mbuf *,
-	struct ipsecrequest *);
+	struct secasvar *);
 extern int esp6_input(struct mbuf **, int *);
 
 extern void esp6_ctlinput(int, struct sockaddr *, void *);
diff --git a/bsd/netinet6/esp_core.c b/bsd/netinet6/esp_core.c
index 21daa3f41..bd51351c9 100644
--- a/bsd/netinet6/esp_core.c
+++ b/bsd/netinet6/esp_core.c
@@ -102,6 +102,7 @@ static int esp_des_blockdecrypt(const struct esp_algorithm *,
 static int esp_des_blockencrypt(const struct esp_algorithm *,
 	struct secasvar *, u_int8_t *, u_int8_t *);
 static int esp_cbc_mature(struct secasvar *);
+#if ALLCRYPTO
 static int esp_blowfish_schedule(const struct esp_algorithm *,
 	struct secasvar *);
 static int esp_blowfish_schedlen(const struct esp_algorithm *);
@@ -116,6 +117,7 @@ static int esp_cast128_blockdecrypt(const struct esp_algorithm *,
 	struct secasvar *, u_int8_t *, u_int8_t *);
 static int esp_cast128_blockencrypt(const struct esp_algorithm *,
 	struct secasvar *, u_int8_t *, u_int8_t *);
+#endif /* ALLCRYPTO */
 static int esp_3des_schedule(const struct esp_algorithm *,
 	struct secasvar *);
 static int esp_3des_schedlen(const struct esp_algorithm *);
@@ -132,34 +134,51 @@ static int esp_cbc_encrypt(struct mbuf *, size_t, size_t,
 
 #define MAXIVLEN	16
 
-static const struct esp_algorithm esp_algorithms[] = {
+static const struct esp_algorithm des_cbc =
 	{ 8, -1, esp_descbc_mature, 64, 64, esp_des_schedlen,
 		"des-cbc",
 		esp_descbc_ivlen, esp_cbc_decrypt,
 		esp_cbc_encrypt, esp_des_schedule,
-		esp_des_blockdecrypt, esp_des_blockencrypt, },
+		esp_des_blockdecrypt, esp_des_blockencrypt, };
+static const struct esp_algorithm des3_cbc =
 	{ 8, 8, esp_cbc_mature, 192, 192, esp_3des_schedlen,
 		"3des-cbc",
 		esp_common_ivlen, esp_cbc_decrypt,
 		esp_cbc_encrypt, esp_3des_schedule,
-		esp_3des_blockdecrypt, esp_3des_blockencrypt, },
+		esp_3des_blockdecrypt, esp_3des_blockencrypt, };
+static const struct esp_algorithm null_esp =
 	{ 1, 0, esp_null_mature, 0, 2048, 0, "null",
 		esp_common_ivlen, esp_null_decrypt,
-		esp_null_encrypt, NULL, },
+		esp_null_encrypt, NULL, NULL, NULL };
+#if ALLCRYPTO
+static const struct esp_algorithm blowfish_cbc =
 	{ 8, 8, esp_cbc_mature, 40, 448, esp_blowfish_schedlen, "blowfish-cbc",
 		esp_common_ivlen, esp_cbc_decrypt,
 		esp_cbc_encrypt, esp_blowfish_schedule,
-		esp_blowfish_blockdecrypt, esp_blowfish_blockencrypt, },
+		esp_blowfish_blockdecrypt, esp_blowfish_blockencrypt, };
+static const struct esp_algorithm cast128_cbc =
 	{ 8, 8, esp_cbc_mature, 40, 128, esp_cast128_schedlen,
 		"cast128-cbc",
 		esp_common_ivlen, esp_cbc_decrypt,
 		esp_cbc_encrypt, esp_cast128_schedule,
-		esp_cast128_blockdecrypt, esp_cast128_blockencrypt, },
+		esp_cast128_blockdecrypt, esp_cast128_blockencrypt, };
+#endif /* ALLCRYPTO */
+static const struct esp_algorithm aes_cbc =
 	{ 16, 16, esp_cbc_mature, 128, 256, esp_aes_schedlen,
 		"aes-cbc",
 		esp_common_ivlen, esp_cbc_decrypt_aes,
 		esp_cbc_encrypt_aes, esp_aes_schedule,
-		0, 0 },
+		0, 0 };
+
+static const struct esp_algorithm *esp_algorithms[] = {
+	&des_cbc,
+	&des3_cbc,
+	&null_esp,
+#if ALLCRYPTO
+	&blowfish_cbc,
+	&cast128_cbc,
+#endif /* ALLCRYPTO */
+	&aes_cbc
 };
 
 const struct esp_algorithm *
@@ -169,17 +188,19 @@ esp_algorithm_lookup(idx)
 
 	switch (idx) {
 	case SADB_EALG_DESCBC:
-		return &esp_algorithms[0];
+		return &des_cbc;
 	case SADB_EALG_3DESCBC:
-		return &esp_algorithms[1];
+		return &des3_cbc;
 	case SADB_EALG_NULL:
-		return &esp_algorithms[2];
+		return &null_esp;
+#if ALLCRYPTO
 	case SADB_X_EALG_BLOWFISHCBC:
-		return &esp_algorithms[3];
+		return &blowfish_cbc;
 	case SADB_X_EALG_CAST128CBC:
-		return &esp_algorithms[4];
+		return &cast128_cbc;
+#endif /* ALLCRYPTO */
 	case SADB_X_EALG_RIJNDAELCBC:
-		return &esp_algorithms[5];
+		return &aes_cbc;
 	default:
 		return NULL;
 	}
@@ -194,8 +215,8 @@ esp_max_ivlen()
 	ivlen = 0;
 	for (idx = 0; idx < sizeof(esp_algorithms)/sizeof(esp_algorithms[0]);
 	     idx++) {
-		if (esp_algorithms[idx].ivlenval > ivlen)
-			ivlen = esp_algorithms[idx].ivlenval;
+		if (esp_algorithms[idx]->ivlenval > ivlen)
+			ivlen = esp_algorithms[idx]->ivlenval;
 	}
 
 	return ivlen;
@@ -218,21 +239,29 @@ esp_schedule(algo, sav)
 		return EINVAL;
 	}
 
+	lck_mtx_lock(sadb_mutex);
 	/* already allocated */
-	if (sav->sched && sav->schedlen != 0)
+	if (sav->sched && sav->schedlen != 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return 0;
+	}
 	/* no schedule necessary */
-	if (!algo->schedule || !algo->schedlen)
+	if (!algo->schedule || !algo->schedlen) {
+		lck_mtx_unlock(sadb_mutex);
 		return 0;
+	}
 
 	sav->schedlen = (*algo->schedlen)(algo);
-	if (sav->schedlen < 0)
+	if (sav->schedlen < 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return EINVAL;
+	}
 
 //#### that malloc should be replaced by a saved buffer...
 	sav->sched = _MALLOC(sav->schedlen, M_SECA, M_DONTWAIT);
 	if (!sav->sched) {
 		sav->schedlen = 0;
+		lck_mtx_unlock(sadb_mutex);
 		return ENOBUFS;
 	}
 
@@ -245,12 +274,13 @@ esp_schedule(algo, sav)
 		sav->sched = NULL;
 		sav->schedlen = 0;
 	}
+	lck_mtx_unlock(sadb_mutex);
 	return error;
 }
 
 static int
-esp_null_mature(sav)
-	struct secasvar *sav;
+esp_null_mature(
+	__unused struct secasvar *sav)
 {
 
 	/* anything is okay */
@@ -258,25 +288,25 @@ esp_null_mature(sav)
 }
 
 static int
-esp_null_decrypt(m, off, sav, algo, ivlen)
-	struct mbuf *m;
-	size_t off;		/* offset to ESP header */
-	struct secasvar *sav;
-	const struct esp_algorithm *algo;
-	int ivlen;
+esp_null_decrypt(
+	__unused struct mbuf *m,
+	__unused size_t off,		/* offset to ESP header */
+	__unused struct secasvar *sav,
+	__unused const struct esp_algorithm *algo,
+	__unused int ivlen)
 {
 
 	return 0; /* do nothing */
 }
 
 static int
-esp_null_encrypt(m, off, plen, sav, algo, ivlen)
-	struct mbuf *m;
-	size_t off;	/* offset to ESP header */
-	size_t plen;	/* payload length (to be encrypted) */
-	struct secasvar *sav;
-	const struct esp_algorithm *algo;
-	int ivlen;
+esp_null_encrypt(
+	__unused struct mbuf *m,
+	__unused size_t off,	/* offset to ESP header */
+	__unused size_t plen,	/* payload length (to be encrypted) */
+	__unused struct secasvar *sav,
+	__unused const struct esp_algorithm *algo,
+	__unused int ivlen)
 {
 
 	return 0; /* do nothing */
@@ -325,9 +355,9 @@ esp_descbc_mature(sav)
 }
 
 static int
-esp_descbc_ivlen(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_descbc_ivlen(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
 
 	if (!sav)
@@ -340,19 +370,20 @@ esp_descbc_ivlen(algo, sav)
 }
 
 static int
-esp_des_schedlen(algo)
-	const struct esp_algorithm *algo;
+esp_des_schedlen(
+	__unused const struct esp_algorithm *algo)
 {
 
 	return sizeof(des_key_schedule);
 }
 
 static int
-esp_des_schedule(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_des_schedule(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	if (des_key_sched((des_cblock *)_KEYBUF(sav->key_enc),
 	    *(des_key_schedule *)sav->sched))
 		return EINVAL;
@@ -361,11 +392,11 @@ esp_des_schedule(algo, sav)
 }
 
 static int
-esp_des_blockdecrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_des_blockdecrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 
 	/* assumption: d has a good alignment */
@@ -376,11 +407,11 @@ esp_des_blockdecrypt(algo, sav, s, d)
 }
 
 static int
-esp_des_blockencrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_des_blockencrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 
 	/* assumption: d has a good alignment */
@@ -456,31 +487,33 @@ esp_cbc_mature(sav)
 	return 0;
 }
 
+#if ALLCRYPTO
 static int
-esp_blowfish_schedlen(algo)
-	const struct esp_algorithm *algo;
+esp_blowfish_schedlen(
+	__unused const struct esp_algorithm *algo)
 {
 
 	return sizeof(BF_KEY);
 }
 
 static int
-esp_blowfish_schedule(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_blowfish_schedule(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	BF_set_key((BF_KEY *)sav->sched, _KEYLEN(sav->key_enc),
 	    _KEYBUF(sav->key_enc));
 	return 0;
 }
 
 static int
-esp_blowfish_blockdecrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_blowfish_blockdecrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 	/* HOLY COW!  BF_decrypt() takes values in host byteorder */
 	BF_LONG t[2];
@@ -496,11 +529,11 @@ esp_blowfish_blockdecrypt(algo, sav, s, d)
 }
 
 static int
-esp_blowfish_blockencrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_blowfish_blockencrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 	/* HOLY COW!  BF_encrypt() takes values in host byteorder */
 	BF_LONG t[2];
@@ -516,30 +549,30 @@ esp_blowfish_blockencrypt(algo, sav, s, d)
 }
 
 static int
-esp_cast128_schedlen(algo)
-	const struct esp_algorithm *algo;
+esp_cast128_schedlen(
+	__unused const struct esp_algorithm *algo)
 {
 
 	return sizeof(u_int32_t) * 32;
 }
 
 static int
-esp_cast128_schedule(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_cast128_schedule(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	set_cast128_subkey((u_int32_t *)sav->sched, _KEYBUF(sav->key_enc),
 		_KEYLEN(sav->key_enc));
 	return 0;
 }
 
 static int
-esp_cast128_blockdecrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_cast128_blockdecrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 
 	if (_KEYLEN(sav->key_enc) <= 80 / 8)
@@ -550,11 +583,11 @@ esp_cast128_blockdecrypt(algo, sav, s, d)
 }
 
 static int
-esp_cast128_blockencrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_cast128_blockencrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 
 	if (_KEYLEN(sav->key_enc) <= 80 / 8)
@@ -563,25 +596,27 @@ esp_cast128_blockencrypt(algo, sav, s, d)
 		cast128_encrypt_round16(d, s, (u_int32_t *)sav->sched);
 	return 0;
 }
+#endif /* ALLCRYPTO */
 
 static int
-esp_3des_schedlen(algo)
-	const struct esp_algorithm *algo;
+esp_3des_schedlen(
+	__unused const struct esp_algorithm *algo)
 {
 
 	return sizeof(des_key_schedule) * 3;
 }
 
 static int
-esp_3des_schedule(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_3des_schedule(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
 	int error;
 	des_key_schedule *p;
 	int i;
 	char *k;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	p = (des_key_schedule *)sav->sched;
 	k = _KEYBUF(sav->key_enc);
 	for (i = 0; i < 3; i++) {
@@ -593,11 +628,11 @@ esp_3des_schedule(algo, sav)
 }
 
 static int
-esp_3des_blockdecrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_3des_blockdecrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 	des_key_schedule *p;
 
@@ -610,11 +645,11 @@ esp_3des_blockdecrypt(algo, sav, s, d)
 }
 
 static int
-esp_3des_blockencrypt(algo, sav, s, d)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
-	u_int8_t *s;
-	u_int8_t *d;
+esp_3des_blockencrypt(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav,
+	u_int8_t *s,
+	u_int8_t *d)
 {
 	des_key_schedule *p;
 
@@ -627,9 +662,9 @@ esp_3des_blockencrypt(algo, sav, s, d)
 }
 
 static int
-esp_common_ivlen(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_common_ivlen(
+	const struct esp_algorithm *algo,
+	__unused struct secasvar *sav)
 {
 
 	if (!algo)
@@ -837,13 +872,13 @@ esp_cbc_decrypt(m, off, sav, algo, ivlen)
 }
 
 static int
-esp_cbc_encrypt(m, off, plen, sav, algo, ivlen)
-	struct mbuf *m;
-	size_t off;
-	size_t plen;
-	struct secasvar *sav;
-	const struct esp_algorithm *algo;
-	int ivlen;
+esp_cbc_encrypt(
+	struct mbuf *m,
+	size_t off,
+	__unused size_t plen,
+	struct secasvar *sav,
+	const struct esp_algorithm *algo,
+	int ivlen)
 {
 	struct mbuf *s;
 	struct mbuf *d, *d0, *dp;
@@ -1125,7 +1160,6 @@ esp_auth(m0, skip, length, sav, sum)
 		KERNEL_DEBUG(DBG_FNC_ESPAUTH | DBG_FUNC_END, 5,0,0,0,0);
 		return error;
 	}
-	lck_mtx_unlock(sadb_mutex);
 	while (0 < length) {
 		if (!m)
 			panic("mbuf chain?");
@@ -1143,7 +1177,6 @@ esp_auth(m0, skip, length, sav, sum)
 	}
 	(*algo->result)(&s, sumbuf);
 	bcopy(sumbuf, sum, siz);	/*XXX*/
-	lck_mtx_lock(sadb_mutex);
 	KERNEL_DEBUG(DBG_FNC_ESPAUTH | DBG_FUNC_END, 6,0,0,0,0);
 	return 0;
 }
diff --git a/bsd/netinet6/esp_input.c b/bsd/netinet6/esp_input.c
index 4b080c6b4..1286b439f 100644
--- a/bsd/netinet6/esp_input.c
+++ b/bsd/netinet6/esp_input.c
@@ -58,6 +58,7 @@
 #include <netinet/in_var.h>
 #include <netinet/ip_ecn.h>
 #include <netinet/in_pcb.h>
+#include <netinet/udp.h>
 #if INET6
 #include <netinet6/ip6_ecn.h>
 #endif
@@ -86,6 +87,8 @@
 #include <netkey/keydb.h>
 #include <netkey/key_debug.h>
 
+#include <net/kpi_protocol.h>
+#include <netinet/kpi_ipfilter_var.h>
 
 #include <net/net_osdep.h>
 
@@ -96,7 +99,8 @@
 #define DBG_FNC_DECRYPT		NETDBG_CODE(DBG_NETIPSEC, (7 << 8))
 #define IPLEN_FLIPPED
 
-extern lck_mtx_t *sadb_mutex;
+extern lck_mtx_t  *sadb_mutex;
+
 #if INET
 extern struct protosw inetsw[];
 
@@ -110,6 +114,7 @@ esp4_input(m, off)
 	int off;
 {
 	struct ip *ip;
+	struct ip6_hdr *ip6;
 	struct esp *esp;
 	struct esptail esptail;
 	u_int32_t spi;
@@ -120,16 +125,14 @@ esp4_input(m, off)
 	int ivlen;
 	size_t hlen;
 	size_t esplen;
-	int s;
-
-	lck_mtx_lock(sadb_mutex);
+	sa_family_t	ifamily;
 
 	KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_START, 0,0,0,0,0);
 	/* sanity check for alignment. */
 	if (off % 4 != 0 || m->m_pkthdr.len % 4 != 0) {
 		ipseclog((LOG_ERR, "IPv4 ESP input: packet alignment problem "
 			"(off=%d, pktlen=%d)\n", off, m->m_pkthdr.len));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -138,7 +141,7 @@ esp4_input(m, off)
 		if (!m) {
 			ipseclog((LOG_DEBUG,
 			    "IPv4 ESP input: can't pullup in esp4_input\n"));
-			ipsecstat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 			goto bad;
 		}
 	}
@@ -160,7 +163,7 @@ esp4_input(m, off)
 		ipseclog((LOG_WARNING,
 		    "IPv4 ESP input: no key association found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_nosa);
 		goto bad;
 	}
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -170,7 +173,7 @@ esp4_input(m, off)
 		ipseclog((LOG_DEBUG,
 		    "IPv4 ESP input: non-mature/dying SA found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
 		goto bad;
 	}
 	algo = esp_algorithm_lookup(sav->alg_enc);
@@ -178,7 +181,7 @@ esp4_input(m, off)
 		ipseclog((LOG_DEBUG, "IPv4 ESP input: "
 		    "unsupported encryption algorithm for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsecstat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_badspi);
 		goto bad;
 	}
 
@@ -187,7 +190,7 @@ esp4_input(m, off)
 	if (ivlen < 0) {
 		ipseclog((LOG_ERR, "inproper ivlen in IPv4 ESP input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -205,7 +208,7 @@ esp4_input(m, off)
 	if (ipsec_chkreplay(ntohl(((struct newesp *)esp)->esp_seq), sav))
 		; /*okey*/
 	else {
-		ipsecstat.in_espreplay++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_espreplay);
 		ipseclog((LOG_WARNING,
 		    "replay packet in IPv4 ESP input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
@@ -224,14 +227,14 @@ esp4_input(m, off)
 		goto noreplaycheck;
 	siz = (((*sumalgo->sumsiz)(sav) + 3) & ~(4 - 1));
 	if (m->m_pkthdr.len < off + ESPMAXLEN + siz) {
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 	if (AH_MAXSUMSIZE < siz) {
 		ipseclog((LOG_DEBUG,
 		    "internal error: AH_MAXSUMSIZE must be larger than %lu\n",
 		    (u_long)siz));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -240,14 +243,14 @@ esp4_input(m, off)
 	if (esp_auth(m, off, m->m_pkthdr.len - off - siz, sav, sum)) {
 		ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-		ipsecstat.in_espauthfail++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail);
 		goto bad;
 	}
 
 	if (bcmp(sum0, sum, siz) != 0) {
 		ipseclog((LOG_WARNING, "auth fail in IPv4 ESP input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-		ipsecstat.in_espauthfail++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_espauthfail);
 		goto bad;
 	}
 
@@ -260,7 +263,7 @@ esp4_input(m, off)
 	ip->ip_len = htons(ntohs(ip->ip_len) - siz);
 #endif
 	m->m_flags |= M_AUTHIPDGM;
-	ipsecstat.in_espauthsucc++;
+	IPSEC_STAT_INCREMENT(ipsecstat.in_espauthsucc);
     }
 
 	/*
@@ -268,7 +271,7 @@ esp4_input(m, off)
 	 */
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
 		if (ipsec_updatereplay(ntohl(((struct newesp *)esp)->esp_seq), sav)) {
-			ipsecstat.in_espreplay++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_espreplay);
 			goto bad;
 		}
 	}
@@ -290,7 +293,7 @@ noreplaycheck:
 	if (m->m_pkthdr.len < off + esplen + ivlen + sizeof(esptail)) {
 		ipseclog((LOG_WARNING,
 		    "IPv4 ESP input: packet too short\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -299,7 +302,7 @@ noreplaycheck:
 		if (!m) {
 			ipseclog((LOG_DEBUG,
 			    "IPv4 ESP input: can't pullup in esp4_input\n"));
-			ipsecstat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 			goto bad;
 		}
 	}
@@ -308,7 +311,7 @@ noreplaycheck:
 	 * pre-compute and cache intermediate key
 	 */
 	if (esp_schedule(algo, sav) != 0) {
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -323,12 +326,12 @@ noreplaycheck:
 		m = NULL;
 		ipseclog((LOG_ERR, "decrypt fail in IPv4 ESP input: %s\n",
 		    ipsec_logsastr(sav)));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		KERNEL_DEBUG(DBG_FNC_DECRYPT | DBG_FUNC_END, 1,0,0,0,0);
 		goto bad;
 	}
 	KERNEL_DEBUG(DBG_FNC_DECRYPT | DBG_FUNC_END, 2,0,0,0,0);
-	ipsecstat.in_esphist[sav->alg_enc]++;
+	IPSEC_STAT_INCREMENT(ipsecstat.in_esphist[sav->alg_enc]);
 
 	m->m_flags |= M_DECRYPTED;
 
@@ -345,7 +348,7 @@ noreplaycheck:
 		ipseclog((LOG_WARNING,
 		    "bad pad length in IPv4 ESP input: %s %s\n",
 		    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 
@@ -359,7 +362,7 @@ noreplaycheck:
 #endif
 
 	/* was it transmitted over the IPsec tunnel SA? */
-	if (ipsec4_tunnel_validate(m, off + esplen + ivlen, nxt, sav)) {
+	if (ipsec4_tunnel_validate(m, off + esplen + ivlen, nxt, sav, &ifamily)) {
 		/*
 		 * strip off all the headers that precedes ESP header.
 		 *	IP4 xx ESP IP4' payload -> IP4' payload
@@ -371,38 +374,74 @@ noreplaycheck:
 
 		tos = ip->ip_tos;
 		m_adj(m, off + esplen + ivlen);
-		if (m->m_len < sizeof(*ip)) {
-			m = m_pullup(m, sizeof(*ip));
-			if (!m) {
-				ipsecstat.in_inval++;
-				goto bad;
+		if (ifamily == AF_INET) {
+			if (m->m_len < sizeof(*ip)) {
+				m = m_pullup(m, sizeof(*ip));
+				if (!m) {
+					IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+					goto bad;
+				}
 			}
-		}
-		ip = mtod(m, struct ip *);
-		/* ECN consideration. */
-		ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
-		if (!key_checktunnelsanity(sav, AF_INET,
+			ip = mtod(m, struct ip *);
+			/* ECN consideration. */
+			ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
+			if (!key_checktunnelsanity(sav, AF_INET,
 			    (caddr_t)&ip->ip_src, (caddr_t)&ip->ip_dst)) {
-			ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
-			    "in IPv4 ESP input: %s %s\n",
+				ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
+			    "in ESP input: %s %s\n",
 			    ipsec4_logpacketstr(ip, spi), ipsec_logsastr(sav)));
-			ipsecstat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+				goto bad;
+			}
+#if INET6
+		} else if (ifamily == AF_INET6) {
+#ifndef PULLDOWN_TEST
+			/*
+			 * m_pullup is prohibited in KAME IPv6 input processing
+			 * but there's no other way!
+			 */
+#else
+			/* okay to pullup in m_pulldown style */
+#endif
+			if (m->m_len < sizeof(*ip6)) {
+				m = m_pullup(m, sizeof(*ip6));
+				if (!m) {
+					IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+					goto bad;
+				}
+			}
+
+			ip6 = mtod(m, struct ip6_hdr *);
+
+			/* ECN consideration. */
+			/* XXX To be fixed later if needed */
+			//  ip_ecn_egress(ip4_ipsec_ecn, &tos, &ip->ip_tos);
+
+			if (!key_checktunnelsanity(sav, AF_INET6,
+			    (caddr_t)&ip6->ip6_src, (caddr_t)&ip6->ip6_dst)) {
+				ipseclog((LOG_ERR, "ipsec tunnel address mismatch "
+			    "in ESP input: %s %s\n",
+			    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
+				IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+				goto bad;
+			}		
+#endif /* INET6 */
+		} else {
+			ipseclog((LOG_ERR, "ipsec tunnel unsupported address family "
+				  "in ESP input\n"));
 			goto bad;
 		}
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0 ||
 		    ipsec_addhist(m, IPPROTO_IPV4, 0) != 0) {
-			ipsecstat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 			goto bad;
 		}
 		
 		/* Clear the csum flags, they can't be valid for the inner headers */
 		m->m_pkthdr.csum_flags = 0;
-
-		lck_mtx_unlock(sadb_mutex);
-		proto_input(PF_INET, m);
-		lck_mtx_lock(sadb_mutex);
+		proto_input(ifamily == AF_INET ? PF_INET : PF_INET6, m);
 		nxt = IPPROTO_DONE;
 		KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 2,0,0,0,0);
 	} else {
@@ -431,7 +470,7 @@ noreplaycheck:
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0) {
-			ipsecstat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 			goto bad;
 		}
 		
@@ -449,13 +488,45 @@ noreplaycheck:
 		if (nxt != IPPROTO_DONE) {
 			if ((ip_protox[nxt]->pr_flags & PR_LASTHDR) != 0 &&
 			    ipsec4_in_reject(m, NULL)) {
-				ipsecstat.in_polvio++;
+				IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 				goto bad;
 			}
 			KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 3,0,0,0,0);
-			lck_mtx_unlock(sadb_mutex);
+			
+			/* translate encapsulated UDP port ? */
+			if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0)  {
+				struct udphdr	*udp;
+				
+				if (nxt != IPPROTO_UDP)	{	/* not UPD packet - drop it */
+					IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+					goto bad;
+				}
+					
+				if (m->m_len < off + sizeof(struct udphdr)) {
+					m = m_pullup(m, off + sizeof(struct udphdr));
+					if (!m) {
+						ipseclog((LOG_DEBUG,
+							"IPv4 ESP input: can't pullup UDP header in esp4_input\n"));
+						IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+						goto bad;
+					}
+					ip = mtod(m, struct ip *);
+				}
+				udp = (struct udphdr *)(((u_int8_t *)ip) + off);
+			
+				lck_mtx_lock(sadb_mutex);
+				if (sav->natt_encapsulated_src_port == 0) {	
+					sav->natt_encapsulated_src_port = udp->uh_sport;
+				} else if (sav->natt_encapsulated_src_port != udp->uh_sport) {	/* something wrong */
+					IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+					lck_mtx_unlock(sadb_mutex);
+					goto bad;
+				}
+				lck_mtx_unlock(sadb_mutex);
+				udp->uh_sport = htons(sav->remote_ike_port);
+				udp->uh_sum = 0;
+			}
 			ip_proto_dispatch_in(m, off, nxt, 0);
-			lck_mtx_lock(sadb_mutex);
 		} else
 			m_freem(m);
 		m = NULL;
@@ -464,19 +535,17 @@ noreplaycheck:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP esp4_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	ipsecstat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsecstat.in_success);
 	return;
 
 bad:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP esp4_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	lck_mtx_unlock(sadb_mutex);
 	if (m)
 		m_freem(m);
 	KERNEL_DEBUG(DBG_FNC_ESPIN | DBG_FUNC_END, 4,0,0,0,0);
@@ -502,26 +571,22 @@ esp6_input(mp, offp)
 	const struct esp_algorithm *algo;
 	int ivlen;
 	size_t esplen;
-	int s;
-
-	lck_mtx_lock(sadb_mutex);
 
 	/* sanity check for alignment. */
 	if (off % 4 != 0 || m->m_pkthdr.len % 4 != 0) {
 		ipseclog((LOG_ERR, "IPv6 ESP input: packet alignment problem "
 			"(off=%d, pktlen=%d)\n", off, m->m_pkthdr.len));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
 #ifndef PULLDOWN_TEST
-	IP6_EXTHDR_CHECK(m, off, ESPMAXLEN, {lck_mtx_unlock(sadb_mutex); return IPPROTO_DONE;});
+	IP6_EXTHDR_CHECK(m, off, ESPMAXLEN, {return IPPROTO_DONE;});
 	esp = (struct esp *)(mtod(m, caddr_t) + off);
 #else
 	IP6_EXTHDR_GET(esp, struct esp *, m, off, ESPMAXLEN);
 	if (esp == NULL) {
-		ipsec6stat.in_inval++;
-		lck_mtx_unlock(sadb_mutex);
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		return IPPROTO_DONE;
 	}
 #endif
@@ -530,7 +595,7 @@ esp6_input(mp, offp)
 	if (ntohs(ip6->ip6_plen) == 0) {
 		ipseclog((LOG_ERR, "IPv6 ESP input: "
 		    "ESP with IPv6 jumbogram is not supported.\n"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
@@ -543,7 +608,7 @@ esp6_input(mp, offp)
 		ipseclog((LOG_WARNING,
 		    "IPv6 ESP input: no key association found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_nosa);
 		goto bad;
 	}
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -553,7 +618,7 @@ esp6_input(mp, offp)
 		ipseclog((LOG_DEBUG,
 		    "IPv6 ESP input: non-mature/dying SA found for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
 		goto bad;
 	}
 	algo = esp_algorithm_lookup(sav->alg_enc);
@@ -561,7 +626,7 @@ esp6_input(mp, offp)
 		ipseclog((LOG_DEBUG, "IPv6 ESP input: "
 		    "unsupported encryption algorithm for spi %u\n",
 		    (u_int32_t)ntohl(spi)));
-		ipsec6stat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
 		goto bad;
 	}
 
@@ -570,7 +635,7 @@ esp6_input(mp, offp)
 	if (ivlen < 0) {
 		ipseclog((LOG_ERR, "inproper ivlen in IPv6 ESP input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-		ipsec6stat.in_badspi++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_badspi);
 		goto bad;
 	}
 
@@ -588,7 +653,7 @@ esp6_input(mp, offp)
 	if (ipsec_chkreplay(ntohl(((struct newesp *)esp)->esp_seq), sav))
 		; /*okey*/
 	else {
-		ipsec6stat.in_espreplay++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_espreplay);
 		ipseclog((LOG_WARNING,
 		    "replay packet in IPv6 ESP input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
@@ -607,14 +672,14 @@ esp6_input(mp, offp)
 		goto noreplaycheck;
 	siz = (((*sumalgo->sumsiz)(sav) + 3) & ~(4 - 1));
 	if (m->m_pkthdr.len < off + ESPMAXLEN + siz) {
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto bad;
 	}
 	if (AH_MAXSUMSIZE < siz) {
 		ipseclog((LOG_DEBUG,
 		    "internal error: AH_MAXSUMSIZE must be larger than %lu\n",
 		    (u_long)siz));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
@@ -623,14 +688,14 @@ esp6_input(mp, offp)
 	if (esp_auth(m, off, m->m_pkthdr.len - off - siz, sav, sum)) {
 		ipseclog((LOG_WARNING, "auth fail in IPv6 ESP input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-		ipsec6stat.in_espauthfail++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthfail);
 		goto bad;
 	}
 
 	if (bcmp(sum0, sum, siz) != 0) {
 		ipseclog((LOG_WARNING, "auth fail in IPv6 ESP input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-		ipsec6stat.in_espauthfail++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthfail);
 		goto bad;
 	}
 
@@ -640,7 +705,7 @@ esp6_input(mp, offp)
 	ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - siz);
 
 	m->m_flags |= M_AUTHIPDGM;
-	ipsec6stat.in_espauthsucc++;
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_espauthsucc);
     }
 
 	/*
@@ -648,7 +713,7 @@ esp6_input(mp, offp)
 	 */
 	if ((sav->flags & SADB_X_EXT_OLD) == 0 && sav->replay) {
 		if (ipsec_updatereplay(ntohl(((struct newesp *)esp)->esp_seq), sav)) {
-			ipsec6stat.in_espreplay++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_espreplay);
 			goto bad;
 		}
 	}
@@ -670,7 +735,7 @@ noreplaycheck:
 	if (m->m_pkthdr.len < off + esplen + ivlen + sizeof(esptail)) {
 		ipseclog((LOG_WARNING,
 		    "IPv6 ESP input: packet too short\n"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
@@ -679,7 +744,7 @@ noreplaycheck:
 #else
 	IP6_EXTHDR_GET(esp, struct esp *, m, off, esplen + ivlen);
 	if (esp == NULL) {
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		m = NULL;
 		goto bad;
 	}
@@ -690,7 +755,7 @@ noreplaycheck:
 	 * pre-compute and cache intermediate key
 	 */
 	if (esp_schedule(algo, sav) != 0) {
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
@@ -704,10 +769,10 @@ noreplaycheck:
 		m = NULL;
 		ipseclog((LOG_ERR, "decrypt fail in IPv6 ESP input: %s\n",
 		    ipsec_logsastr(sav)));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
-	ipsec6stat.in_esphist[sav->alg_enc]++;
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_esphist[sav->alg_enc]);
 
 	m->m_flags |= M_DECRYPTED;
 
@@ -724,7 +789,7 @@ noreplaycheck:
 		ipseclog((LOG_WARNING,
 		    "bad pad length in IPv6 ESP input: %s %s\n",
 		    ipsec6_logpacketstr(ip6, spi), ipsec_logsastr(sav)));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto bad;
 	}
 
@@ -756,7 +821,7 @@ noreplaycheck:
 #endif
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m) {
-				ipsec6stat.in_inval++;
+				IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 				goto bad;
 			}
 		}
@@ -769,19 +834,17 @@ noreplaycheck:
 			    "in IPv6 ESP input: %s %s\n",
 			    ipsec6_logpacketstr(ip6, spi),
 			    ipsec_logsastr(sav)));
-			ipsec6stat.in_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 			goto bad;
 		}
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0 || 
 		    ipsec_addhist(m, IPPROTO_IPV6, 0) != 0) {
-			ipsec6stat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 			goto bad;
 		}
-		lck_mtx_unlock(sadb_mutex);
 		proto_input(PF_INET6, m);
-		lck_mtx_lock(sadb_mutex);
 		nxt = IPPROTO_DONE;
 	} else {
 		/*
@@ -835,7 +898,7 @@ noreplaycheck:
 			struct mbuf *n = NULL;
 			int maxlen;
 
-			MGETHDR(n, M_DONTWAIT, MT_HEADER);
+			MGETHDR(n, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 			maxlen = MHLEN;
 			if (n)
 				M_COPY_PKTHDR(n, m);
@@ -875,7 +938,7 @@ noreplaycheck:
 
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_ESP, spi) != 0) {
-			ipsec6stat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 			goto bad;
 		}
 	}
@@ -886,19 +949,17 @@ noreplaycheck:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP esp6_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	ipsec6stat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_success);
 	return nxt;
 
 bad:
 	if (sav) {
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP esp6_input call free SA:%p\n", sav));
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	}
-	lck_mtx_unlock(sadb_mutex);
 	if (m)
 		m_freem(m);
 	return IPPROTO_DONE;
@@ -983,7 +1044,6 @@ esp6_ctlinput(cmd, sa, d)
 			 */
 			sa6_src = ip6cp->ip6c_src;
 			sa6_dst = (struct sockaddr_in6 *)sa;
-			lck_mtx_lock(sadb_mutex);
 			sav = key_allocsa(AF_INET6,
 					  (caddr_t)&sa6_src->sin6_addr,
 					  (caddr_t)&sa6_dst->sin6_addr,
@@ -992,9 +1052,8 @@ esp6_ctlinput(cmd, sa, d)
 				if (sav->state == SADB_SASTATE_MATURE ||
 				    sav->state == SADB_SASTATE_DYING)
 					valid++;
-				key_freesav(sav);
+				key_freesav(sav, KEY_SADB_LOCKED);
 			}
-			lck_mtx_unlock(sadb_mutex);
 
 			/* XXX Further validation? */
 
diff --git a/bsd/netinet6/esp_output.c b/bsd/netinet6/esp_output.c
index b1b00cda2..21e449b7d 100644
--- a/bsd/netinet6/esp_output.c
+++ b/bsd/netinet6/esp_output.c
@@ -88,11 +88,13 @@
 #define DBG_FNC_ENCRYPT		NETDBG_CODE(DBG_NETIPSEC, (5 << 8))
 
 static int esp_output(struct mbuf *, u_char *, struct mbuf *,
-	struct ipsecrequest *, int);
+	int, struct secasvar *sav);
 
 extern int	esp_udp_encap_port;
 extern u_int32_t natt_now;
 
+extern lck_mtx_t *sadb_mutex;
+
 /*
  * compute ESP header size.
  */
@@ -100,65 +102,74 @@ size_t
 esp_hdrsiz(isr)
 	struct ipsecrequest *isr;
 {
-	struct secasvar *sav;
-	const struct esp_algorithm *algo;
-	const struct ah_algorithm *aalgo;
-	size_t ivlen;
-	size_t authlen;
-	size_t hdrsiz;
-	size_t maxpad;
 
 	/* sanity check */
 	if (isr == NULL)
 		panic("esp_hdrsiz: NULL was passed.\n");
 
-	sav = isr->sav;
-
-	if (isr->saidx.proto != IPPROTO_ESP)
-		panic("unsupported mode passed to esp_hdrsiz");
-
-	if (sav == NULL)
-		goto estimate;
-	if (sav->state != SADB_SASTATE_MATURE
-	 && sav->state != SADB_SASTATE_DYING)
-		goto estimate;
-
-	/* we need transport mode ESP. */
-	algo = esp_algorithm_lookup(sav->alg_enc);
-	if (!algo)
-		goto estimate;
-	ivlen = sav->ivlen;
-	if (ivlen < 0)
-		goto estimate;
 
-	if (algo->padbound)
-		maxpad = algo->padbound;
-	else
-		maxpad = 4;
-	maxpad += 1; /* maximum 'extendsiz' is padbound + 1, see esp_output */
+#if 0
+	lck_mtx_lock(sadb_mutex);
+	{
+		struct secasvar *sav;
+		const struct esp_algorithm *algo;
+		const struct ah_algorithm *aalgo;
+		size_t ivlen;
+		size_t authlen;
+		size_t hdrsiz;
+		size_t maxpad;
 	
-	if (sav->flags & SADB_X_EXT_OLD) {
-		/* RFC 1827 */
-		hdrsiz = sizeof(struct esp) + ivlen + maxpad;
-	} else {
-		/* RFC 2406 */
-		aalgo = ah_algorithm_lookup(sav->alg_auth);
-		if (aalgo && sav->replay && sav->key_auth)
-			authlen = (aalgo->sumsiz)(sav);
+		/*%%%% this needs to change - no sav in ipsecrequest any more */
+		sav = isr->sav;
+	
+		if (isr->saidx.proto != IPPROTO_ESP)
+			panic("unsupported mode passed to esp_hdrsiz");
+	
+		if (sav == NULL)
+			goto estimate;
+		if (sav->state != SADB_SASTATE_MATURE
+		 && sav->state != SADB_SASTATE_DYING)
+			goto estimate;
+	
+		/* we need transport mode ESP. */
+		algo = esp_algorithm_lookup(sav->alg_enc);
+		if (!algo)
+			goto estimate;
+		ivlen = sav->ivlen;
+		if (ivlen < 0)
+			goto estimate;
+	
+		if (algo->padbound)
+			maxpad = algo->padbound;
 		else
-			authlen = 0;
-		hdrsiz = sizeof(struct newesp) + ivlen + maxpad + authlen;
-	}
+			maxpad = 4;
+		maxpad += 1; /* maximum 'extendsiz' is padbound + 1, see esp_output */
+		
+		if (sav->flags & SADB_X_EXT_OLD) {
+			/* RFC 1827 */
+			hdrsiz = sizeof(struct esp) + ivlen + maxpad;
+		} else {
+			/* RFC 2406 */
+			aalgo = ah_algorithm_lookup(sav->alg_auth);
+			if (aalgo && sav->replay && sav->key_auth)
+				authlen = (aalgo->sumsiz)(sav);
+			else
+				authlen = 0;
+			hdrsiz = sizeof(struct newesp) + ivlen + maxpad + authlen;
+		}
+		
+		/*
+		 * If the security association indicates that NATT is required,
+		 * add the size of the NATT encapsulation header:
+		 */
+		if ((sav->flags & SADB_X_EXT_NATT) != 0) hdrsiz += sizeof(struct udphdr) + 4;
 	
-	/*
-	 * If the security association indicates that NATT is required,
-	 * add the size of the NATT encapsulation header:
-	 */
-	if ((sav->flags & SADB_X_EXT_NATT) != 0) hdrsiz += sizeof(struct udphdr) + 4;
-
-	return hdrsiz;
-
-   estimate:
+		lck_mtx_unlock(sadb_mutex);
+		return hdrsiz;
+	}
+estimate:
+   lck_mtx_unlock(sadb_mutex);
+#endif
 	/*
 	 * ASSUMING:
 	 *	sizeof(struct newesp) > sizeof(struct esp). (8)
@@ -192,18 +203,17 @@ esp_hdrsiz(isr)
  *	<-----------------> espoff
  */
 static int
-esp_output(m, nexthdrp, md, isr, af)
+esp_output(m, nexthdrp, md, af, sav)
 	struct mbuf *m;
 	u_char *nexthdrp;
 	struct mbuf *md;
-	struct ipsecrequest *isr;
 	int af;
+	struct secasvar *sav;
 {
 	struct mbuf *n;
 	struct mbuf *mprev;
 	struct esp *esp;
 	struct esptail *esptail;
-	struct secasvar *sav = isr->sav;
 	const struct esp_algorithm *algo;
 	u_int32_t spi;
 	u_int8_t nxt = 0;
@@ -252,7 +262,7 @@ esp_output(m, nexthdrp, md, isr, af)
 				(u_int32_t)ntohl(ip->ip_src.s_addr),
 				(u_int32_t)ntohl(ip->ip_dst.s_addr),
 				(u_int32_t)ntohl(sav->spi)));
-			ipsecstat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 			break;
 		    }
 #endif /*INET*/
@@ -261,7 +271,7 @@ esp_output(m, nexthdrp, md, isr, af)
 			ipseclog((LOG_DEBUG, "esp6_output: internal error: "
 				"sav->replay is null: SPI=%u\n",
 				(u_int32_t)ntohl(sav->spi)));
-			ipsec6stat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 			break;
 #endif /*INET6*/
 		default:
@@ -357,6 +367,42 @@ esp_output(m, nexthdrp, md, isr, af)
 		goto fail;
 	}
 	mprev->m_next = md;
+	
+	/* 
+	 * Translate UDP source port back to its original value.
+	 * SADB_X_EXT_NATT_MULTIPLEUSERS is only set for transort mode.
+	 */
+	if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0) {
+		/* if not UDP - drop it */
+		if (ip->ip_p != IPPROTO_UDP)	{
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
+			m_freem(m);
+			error = EINVAL;
+			goto fail;
+		}			
+		
+		udp = mtod(md, struct udphdr *);
+
+		/* if src port not set in sav - find it */
+		if (sav->natt_encapsulated_src_port == 0)
+			if (key_natt_get_translated_port(sav) == 0) {
+				m_freem(m);
+				error = EINVAL;
+				goto fail;
+			}
+		if (sav->remote_ike_port == htons(udp->uh_dport)) {
+			/* translate UDP port */
+			udp->uh_dport = sav->natt_encapsulated_src_port;
+			udp->uh_sum = 0;	/* don't need checksum with ESP auth */
+		} else {
+			/* drop the packet - can't translate the port */
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
+			m_freem(m);
+			error = EINVAL;
+			goto fail;
+		}
+	}
+		
 
 	espoff = m->m_pkthdr.len - plen;
 	
@@ -408,7 +454,7 @@ esp_output(m, nexthdrp, md, isr, af)
 		else {
 			ipseclog((LOG_ERR,
 			    "IPv4 ESP output: size exceeds limit\n"));
-			ipsecstat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 			m_freem(m);
 			error = EMSGSIZE;
 			goto fail;
@@ -434,13 +480,15 @@ esp_output(m, nexthdrp, md, isr, af)
 				ipseclog((LOG_WARNING,
 				    "replay counter overflowed. %s\n",
 				    ipsec_logsastr(sav)));
-				stat->out_inval++;
+				IPSEC_STAT_INCREMENT(stat->out_inval);
 				m_freem(m);
 				KERNEL_DEBUG(DBG_FNC_ESPOUT | DBG_FUNC_END, 5,0,0,0,0);
 				return EINVAL;
 			}
 		}
+		lck_mtx_lock(sadb_mutex);
 		sav->replay->count++;
+		lck_mtx_unlock(sadb_mutex);
 		/*
 		 * XXX sequence number must not be cycled, if the SA is
 		 * installed by IKE daemon.
@@ -491,16 +539,16 @@ esp_output(m, nexthdrp, md, isr, af)
 	if (randpadmax < 0 || plen + extendsiz >= randpadmax)
 		;
 	else {
-		int n;
+		int pad;
 
 		/* round */
 		randpadmax = (randpadmax / padbound) * padbound;
-		n = (randpadmax - plen + extendsiz) / padbound;
+		pad = (randpadmax - plen + extendsiz) / padbound;
 
-		if (n > 0)
-			n = (random() % n) * padbound;
+		if (pad > 0)
+			pad = (random() % pad) * padbound;
 		else
-			n = 0;
+			pad = 0;
 
 		/*
 		 * make sure we do not pad too much.
@@ -511,8 +559,8 @@ esp_output(m, nexthdrp, md, isr, af)
 		 * limitation (but is less strict than sequential padding
 		 * as length field do not count the last 2 octets).
 		 */
-		if (extendsiz + n <= MLEN && extendsiz + n < 256)
-			extendsiz += n;
+		if (extendsiz + pad <= MLEN && extendsiz + pad < 256)
+			extendsiz += pad;
 	}
 
 #if DIAGNOSTIC
@@ -596,7 +644,7 @@ esp_output(m, nexthdrp, md, isr, af)
 		else {
 			ipseclog((LOG_ERR,
 			    "IPv4 ESP output: size exceeds limit\n"));
-			ipsecstat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 			m_freem(m);
 			error = EMSGSIZE;
 			goto fail;
@@ -617,7 +665,7 @@ esp_output(m, nexthdrp, md, isr, af)
 	error = esp_schedule(algo, sav);
 	if (error) {
 		m_freem(m);
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 		goto fail;
 	}
 
@@ -631,7 +679,7 @@ esp_output(m, nexthdrp, md, isr, af)
 	if ((*algo->encrypt)(m, espoff, plen + extendsiz, sav, algo, ivlen)) {
 		/* m is already freed */
 		ipseclog((LOG_ERR, "packet encryption failure\n"));
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 		error = EINVAL;
 		KERNEL_DEBUG(DBG_FNC_ENCRYPT | DBG_FUNC_END, 1,error,0,0,0);
 		goto fail;
@@ -649,81 +697,80 @@ esp_output(m, nexthdrp, md, isr, af)
 		goto noantireplay;
 
     {
-	const struct ah_algorithm *aalgo;
-	u_char authbuf[AH_MAXSUMSIZE];
-	struct mbuf *n;
-	u_char *p;
-	size_t siz;
-#if INET
-	struct ip *ip;
-#endif
-
-	aalgo = ah_algorithm_lookup(sav->alg_auth);
-	if (!aalgo)
-		goto noantireplay;
-	siz = ((aalgo->sumsiz)(sav) + 3) & ~(4 - 1);
-	if (AH_MAXSUMSIZE < siz)
-		panic("assertion failed for AH_MAXSUMSIZE");
-
-	if (esp_auth(m, espoff, m->m_pkthdr.len - espoff, sav, authbuf)) {
-		ipseclog((LOG_ERR, "ESP checksum generation failure\n"));
-		m_freem(m);
-		error = EINVAL;
-		stat->out_inval++;
-		goto fail;
-	}
-
-	n = m;
-	while (n->m_next)
-		n = n->m_next;
-
-	if (!(n->m_flags & M_EXT) && siz < M_TRAILINGSPACE(n)) { /* XXX */
-		n->m_len += siz;
-		m->m_pkthdr.len += siz;
-		p = mtod(n, u_char *) + n->m_len - siz;
-	} else {
-		struct mbuf *nn;
-
-		MGET(nn, M_DONTWAIT, MT_DATA);
-		if (!nn) {
-			ipseclog((LOG_DEBUG, "can't alloc mbuf in esp%d_output",
-			    afnumber));
+		const struct ah_algorithm *aalgo;
+		u_char authbuf[AH_MAXSUMSIZE];
+		u_char *p;
+		size_t siz;
+	#if INET
+		struct ip *ip;
+	#endif
+	
+		aalgo = ah_algorithm_lookup(sav->alg_auth);
+		if (!aalgo)
+			goto noantireplay;
+		siz = ((aalgo->sumsiz)(sav) + 3) & ~(4 - 1);
+		if (AH_MAXSUMSIZE < siz)
+			panic("assertion failed for AH_MAXSUMSIZE");
+	
+		if (esp_auth(m, espoff, m->m_pkthdr.len - espoff, sav, authbuf)) {
+			ipseclog((LOG_ERR, "ESP checksum generation failure\n"));
 			m_freem(m);
-			error = ENOBUFS;
+			error = EINVAL;
+			IPSEC_STAT_INCREMENT(stat->out_inval);
 			goto fail;
 		}
-		nn->m_len = siz;
-		nn->m_next = NULL;
-		n->m_next = nn;
-		n = nn;
-		m->m_pkthdr.len += siz;
-		p = mtod(nn, u_char *);
-	}
-	bcopy(authbuf, p, siz);
-
-	/* modify IP header (for ESP header part only) */
-	switch (af) {
-#if INET
-	case AF_INET:
-		ip = mtod(m, struct ip *);
-		if (siz < (IP_MAXPACKET - ntohs(ip->ip_len)))
-			ip->ip_len = htons(ntohs(ip->ip_len) + siz);
-		else {
-			ipseclog((LOG_ERR,
-			    "IPv4 ESP output: size exceeds limit\n"));
-			ipsecstat.out_inval++;
-			m_freem(m);
-			error = EMSGSIZE;
-			goto fail;
+	
+		n = m;
+		while (n->m_next)
+			n = n->m_next;
+	
+		if (!(n->m_flags & M_EXT) && siz < M_TRAILINGSPACE(n)) { /* XXX */
+			n->m_len += siz;
+			m->m_pkthdr.len += siz;
+			p = mtod(n, u_char *) + n->m_len - siz;
+		} else {
+			struct mbuf *nn;
+	
+			MGET(nn, M_DONTWAIT, MT_DATA);
+			if (!nn) {
+				ipseclog((LOG_DEBUG, "can't alloc mbuf in esp%d_output",
+					afnumber));
+				m_freem(m);
+				error = ENOBUFS;
+				goto fail;
+			}
+			nn->m_len = siz;
+			nn->m_next = NULL;
+			n->m_next = nn;
+			n = nn;
+			m->m_pkthdr.len += siz;
+			p = mtod(nn, u_char *);
+		}
+		bcopy(authbuf, p, siz);
+	
+		/* modify IP header (for ESP header part only) */
+		switch (af) {
+	#if INET
+		case AF_INET:
+			ip = mtod(m, struct ip *);
+			if (siz < (IP_MAXPACKET - ntohs(ip->ip_len)))
+				ip->ip_len = htons(ntohs(ip->ip_len) + siz);
+			else {
+				ipseclog((LOG_ERR,
+					"IPv4 ESP output: size exceeds limit\n"));
+				IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
+				m_freem(m);
+				error = EMSGSIZE;
+				goto fail;
+			}
+			break;
+	#endif
+	#if INET6
+		case AF_INET6:
+			/* total packet length will be computed in ip6_output() */
+			break;
+	#endif
 		}
-		break;
-#endif
-#if INET6
-	case AF_INET6:
-		/* total packet length will be computed in ip6_output() */
-		break;
-#endif
-	}
     }
     
 	if (udp_encapsulate) {
@@ -734,12 +781,14 @@ esp_output(m, nexthdrp, md, isr, af)
 
 
 noantireplay:
+	lck_mtx_lock(sadb_mutex);
 	if (!m) {
 		ipseclog((LOG_ERR,
 		    "NULL mbuf after encryption in esp%d_output", afnumber));
 	} else
 		stat->out_success++;
 	stat->out_esphist[sav->alg_enc]++;
+	lck_mtx_unlock(sadb_mutex);
 	key_sa_recordxfer(sav, m);
 	KERNEL_DEBUG(DBG_FNC_ESPOUT | DBG_FUNC_END, 6,0,0,0,0);
 	return 0;
@@ -755,35 +804,35 @@ fail:
 
 #if INET
 int
-esp4_output(m, isr)
+esp4_output(m, sav)
 	struct mbuf *m;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
 	struct ip *ip;
 	if (m->m_len < sizeof(struct ip)) {
 		ipseclog((LOG_DEBUG, "esp4_output: first mbuf too short\n"));
 		m_freem(m);
-		return 0;
+		return EINVAL;
 	}
 	ip = mtod(m, struct ip *);
 	/* XXX assumes that m->m_next points to payload */
-	return esp_output(m, &ip->ip_p, m->m_next, isr, AF_INET);
+	return esp_output(m, &ip->ip_p, m->m_next, AF_INET, sav);
 }
 #endif /*INET*/
 
 #if INET6
 int
-esp6_output(m, nexthdrp, md, isr)
+esp6_output(m, nexthdrp, md, sav)
 	struct mbuf *m;
 	u_char *nexthdrp;
 	struct mbuf *md;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
 	if (m->m_len < sizeof(struct ip6_hdr)) {
 		ipseclog((LOG_DEBUG, "esp6_output: first mbuf too short\n"));
 		m_freem(m);
-		return 0;
+		return EINVAL;
 	}
-	return esp_output(m, nexthdrp, md, isr, AF_INET6);
+	return esp_output(m, nexthdrp, md, AF_INET6, sav);
 }
 #endif /*INET6*/
diff --git a/bsd/netinet6/esp_rijndael.c b/bsd/netinet6/esp_rijndael.c
index f2ebe936d..0b6df997a 100644
--- a/bsd/netinet6/esp_rijndael.c
+++ b/bsd/netinet6/esp_rijndael.c
@@ -48,6 +48,8 @@
 
 #include <crypto/aes/aes.h>
 
+#include <netkey/key.h>
+
 #include <net/net_osdep.h>
 
 #define AES_BLOCKLEN 16
@@ -55,21 +57,22 @@
 extern lck_mtx_t *sadb_mutex;
 
 int
-esp_aes_schedlen(algo)
-	const struct esp_algorithm *algo;
+esp_aes_schedlen(
+	__unused const struct esp_algorithm *algo)
 {
 
 	return sizeof(aes_ctx);
 }
 
 int
-esp_aes_schedule(algo, sav)
-	const struct esp_algorithm *algo;
-	struct secasvar *sav;
+esp_aes_schedule(
+	__unused const struct esp_algorithm *algo,
+	struct secasvar *sav)
 {
+
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	aes_ctx *ctx = (aes_ctx*)sav->sched;
 	
-	gen_tabs();
 	aes_decrypt_key(_KEYBUF(sav->key_enc), _KEYLEN(sav->key_enc), &ctx->decrypt);
 	aes_encrypt_key(_KEYBUF(sav->key_enc), _KEYLEN(sav->key_enc), &ctx->encrypt);
 	
@@ -158,7 +161,6 @@ esp_cbc_decrypt_aes(m, off, sav, algo, ivlen)
 	/* grab iv */
 	m_copydata(m, ivoff, ivlen, iv);
 
-	lck_mtx_unlock(sadb_mutex);
 	s = m;
 	soff = sn = dn = 0;
 	d = d0 = dp = NULL;
@@ -204,15 +206,17 @@ esp_cbc_decrypt_aes(m, off, sav, algo, ivlen)
 			if (d && i > MLEN) {
 				MCLGET(d, M_DONTWAIT);
 				if ((d->m_flags & M_EXT) == 0) {
-					m_free(d);
-					d = NULL;
+					d = m_mbigget(d, M_DONTWAIT);
+					if ((d->m_flags & M_EXT) == 0) {
+						m_free(d);
+						d = NULL;
+					}
 				}
 			}
 			if (!d) {
 				m_freem(m);
 				if (d0)
 					m_freem(d0);
-				lck_mtx_lock(sadb_mutex);
 				return ENOBUFS;
 			}
 			if (!d0)
@@ -259,23 +263,22 @@ esp_cbc_decrypt_aes(m, off, sav, algo, ivlen)
 	/* just in case */
 	bzero(iv, sizeof(iv));
 	bzero(sbuf, sizeof(sbuf));
-	lck_mtx_lock(sadb_mutex);
 
 	return 0;
 }
 
 int
-esp_cbc_encrypt_aes(m, off, plen, sav, algo, ivlen)
-	struct mbuf *m;
-	size_t off;
-	size_t plen;
-	struct secasvar *sav;
-	const struct esp_algorithm *algo;
-	int ivlen;
+esp_cbc_encrypt_aes(
+	struct mbuf *m,
+	size_t off,
+	__unused size_t plen,
+	struct secasvar *sav,
+	const struct esp_algorithm *algo,
+	int ivlen)
 {
 	struct mbuf *s;
 	struct mbuf *d, *d0, *dp;
-	int soff, doff;	/* offset from the head of chain, to head of this mbuf */
+	int soff;	/* offset from the head of chain, to head of this mbuf */
 	int sn, dn;	/* offset from the head of the mbuf, to meat */
 	size_t ivoff, bodyoff;
 	u_int8_t *ivp, *dptr;
@@ -317,7 +320,6 @@ esp_cbc_encrypt_aes(m, off, plen, sav, algo, ivlen)
 		m_freem(m);
 		return EINVAL;
 	}
-	lck_mtx_unlock(sadb_mutex);
 
 	s = m;
 	soff = sn = dn = 0;
@@ -364,15 +366,17 @@ esp_cbc_encrypt_aes(m, off, plen, sav, algo, ivlen)
 			if (d && i > MLEN) {
 				MCLGET(d, M_DONTWAIT);
 				if ((d->m_flags & M_EXT) == 0) {
-					m_free(d);
-					d = NULL;
+					d = m_mbigget(d, M_DONTWAIT);
+					if ((d->m_flags & M_EXT) == 0) {
+						m_free(d);
+						d = NULL;
+					}
 				}
 			}
 			if (!d) {
 				m_freem(m);
 				if (d0)
 					m_freem(d0);
-				lck_mtx_lock(sadb_mutex);
 				return ENOBUFS;
 			}
 			if (!d0)
@@ -419,7 +423,6 @@ esp_cbc_encrypt_aes(m, off, plen, sav, algo, ivlen)
 
 	/* just in case */
 	bzero(sbuf, sizeof(sbuf));
-	lck_mtx_lock(sadb_mutex);
 	key_sa_stir_iv(sav);
 
 	return 0;
diff --git a/bsd/netinet6/frag6.c b/bsd/netinet6/frag6.c
index a495a0464..aaac2e2b9 100644
--- a/bsd/netinet6/frag6.c
+++ b/bsd/netinet6/frag6.c
@@ -684,23 +684,6 @@ frag6_slowtimo()
 		frag6_freef(ip6q.ip6q_prev);
 	}
 	frag6_doing_reass = 0;
-
-#if 0
-	/*
-	 * Routing changes might produce a better route than we last used;
-	 * make sure we notice eventually, even if forwarding only for one
-	 * destination and the cache is never replaced.
-	 */
-	if (ip6_forward_rt.ro_rt) {
-		rtfree(ip6_forward_rt.ro_rt);
-		ip6_forward_rt.ro_rt = 0;
-	}
-	if (ipsrcchk_rt.ro_rt) {
-		rtfree(ipsrcchk_rt.ro_rt);
-		ipsrcchk_rt.ro_rt = 0;
-	}
-#endif
-
 	lck_mtx_unlock(inet6_domain_mutex);
 }
 
diff --git a/bsd/netinet6/icmp6.c b/bsd/netinet6/icmp6.c
index c747a78f6..75abd5156 100644
--- a/bsd/netinet6/icmp6.c
+++ b/bsd/netinet6/icmp6.c
@@ -103,17 +103,15 @@
 extern int ipsec_bypass;
 #endif
 
-#include "faith.h"
-#if defined(NFAITH) && 0 < NFAITH
-#include <net/if_faith.h>
-#endif
-
 #include <net/net_osdep.h>
 
 extern struct domain inet6domain;
 extern struct ip6protosw inet6sw[];
 extern struct ip6protosw *ip6_protox[];
 
+extern u_long rip_sendspace;
+extern u_long rip_recvspace;
+
 struct icmp6stat icmp6stat;
 
 extern struct inpcbhead ripcb;
@@ -535,7 +533,7 @@ icmp6_input(mp, offp)
 				m_freem(n0);
 				break;
 			}
-			MGETHDR(n, M_DONTWAIT, n0->m_type);
+			MGETHDR(n, M_DONTWAIT, n0->m_type);	/* MAC-OK */
 			if (n && maxlen >= MHLEN) {
 				MCLGET(n, M_DONTWAIT);
 				if ((n->m_flags & M_EXT) == 0) {
@@ -569,7 +567,8 @@ icmp6_input(mp, offp)
 			n0->m_flags &= ~M_PKTHDR;
 		} else {
 			nip6 = mtod(n, struct ip6_hdr *);
-			nicmp6 = (struct icmp6_hdr *)((caddr_t)nip6 + off);
+			IP6_EXTHDR_GET(nicmp6, struct icmp6_hdr *, n, off,
+				sizeof(*nicmp6));
 			noff = off;
 		}
 		nicmp6->icmp6_type = ICMP6_ECHO_REPLY;
@@ -617,7 +616,7 @@ icmp6_input(mp, offp)
 		/* XXX: per-interface statistics? */
 		break;		/* just pass it to applications */
 
-	case ICMP6_FQDN_QUERY:
+	case ICMP6_NI_QUERY:
 		if (!icmp6_nodeinfo)
 			break;
 
@@ -1051,8 +1050,7 @@ icmp6_mtudisc_update(ip6cp, validated)
 		    htons(m->m_pkthdr.rcvif->if_index);
 	}
 	/* sin6.sin6_scope_id = XXX: should be set if DST is a scoped addr */
-	rt = rtalloc1((struct sockaddr *)&sin6, 0,
-		      RTF_CLONING | RTF_PRCLONING);
+	rt = rtalloc1((struct sockaddr *)&sin6, 0, RTF_CLONING | RTF_PRCLONING);
 
 	if (rt && (rt->rt_flags & RTF_HOST)
 	    && !(rt->rt_rmx.rmx_locks & RTV_MTU)) {
@@ -1303,7 +1301,7 @@ ni6_input(m, off)
 	}
 
 	/* allocate an mbuf to reply. */
-	MGETHDR(n, M_DONTWAIT, m->m_type);
+	MGETHDR(n, M_DONTWAIT, m->m_type);	/* MAC-OK */
 	if (n == NULL) {
 		m_freem(m);
 		return(NULL);
@@ -2394,7 +2392,7 @@ icmp6_redirect_output(m0, rt)
 #if IPV6_MMTU >= MCLBYTES
 # error assumption failed about IPV6_MMTU and MCLBYTES
 #endif
-	MGETHDR(m, M_DONTWAIT, MT_HEADER);
+	MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 	if (m && IPV6_MMTU >= MHLEN)
 		MCLGET(m, M_DONTWAIT);
 	if (!m)
@@ -2710,6 +2708,157 @@ icmp6_ctloutput(so, sopt)
 #undef in6p_icmp6filt
 #endif
 
+/*
+ * ICMPv6 socket datagram option processing.
+ */
+int
+icmp6_dgram_ctloutput(struct socket *so, struct sockopt *sopt)
+{
+	if (so->so_uid == 0)
+		return icmp6_ctloutput(so, sopt);
+
+	if (sopt->sopt_level == IPPROTO_ICMPV6) {
+		switch (sopt->sopt_name) {
+			case ICMP6_FILTER:
+				return icmp6_ctloutput(so, sopt);
+			default:
+				return EPERM;
+		}
+	}
+	
+	if (sopt->sopt_level != IPPROTO_IPV6)
+		return EINVAL;
+		
+	switch (sopt->sopt_name) {
+		case IPV6_PKTOPTIONS:
+		case IPV6_UNICAST_HOPS:
+		case IPV6_CHECKSUM:
+		case IPV6_FAITH:
+		case IPV6_V6ONLY:
+		case IPV6_PKTINFO:
+		case IPV6_HOPLIMIT:
+		case IPV6_HOPOPTS:
+		case IPV6_DSTOPTS:
+		case IPV6_RTHDR:
+		case IPV6_MULTICAST_IF:
+		case IPV6_MULTICAST_HOPS:
+		case IPV6_MULTICAST_LOOP:
+		case IPV6_JOIN_GROUP:
+		case IPV6_LEAVE_GROUP:
+		case IPV6_PORTRANGE:
+		case IPV6_IPSEC_POLICY:
+			return ip6_ctloutput(so, sopt);
+		
+		default:
+			return EPERM;
+			
+		
+	}
+}
+
+__private_extern__ int
+icmp6_dgram_send(struct socket *so, __unused int flags, struct mbuf *m, struct sockaddr *nam,
+         struct mbuf *control, __unused struct proc *p)
+{
+	int error = 0;
+	struct inpcb *inp = sotoinpcb(so);
+	struct sockaddr_in6 tmp;
+	struct sockaddr_in6 *dst;
+	struct icmp6_hdr *icmp6;
+
+	if (so->so_uid == 0)
+		return rip6_output(m, so, (struct sockaddr_in6 *) nam, control);
+
+	/* always copy sockaddr to avoid overwrites */
+	if (so->so_state & SS_ISCONNECTED) {
+		if (nam) {
+				m_freem(m);
+				return EISCONN;
+		}
+		/* XXX */
+		bzero(&tmp, sizeof(tmp));
+		tmp.sin6_family = AF_INET6;
+		tmp.sin6_len = sizeof(struct sockaddr_in6);
+		bcopy(&inp->in6p_faddr, &tmp.sin6_addr,
+			  sizeof(struct in6_addr));
+		dst = &tmp;
+	} else {
+		if (nam == NULL) {
+				m_freem(m);
+				return ENOTCONN;
+		}
+		tmp = *(struct sockaddr_in6 *)nam;
+		dst = &tmp;
+	}
+
+	/*
+	 * For an ICMPv6 packet, we should know its type and code
+	 */
+	if (so->so_proto->pr_protocol == IPPROTO_ICMPV6) {
+		if (m->m_len < sizeof(struct icmp6_hdr) &&
+			(m = m_pullup(m, sizeof(struct icmp6_hdr))) == NULL) {
+				error = ENOBUFS;
+				goto bad;
+		}
+		icmp6 = mtod(m, struct icmp6_hdr *);
+	
+		/*
+		 * Allow only to send echo request type 128 with code 0
+		 * See RFC 2463 for Echo Request Message format
+		 */
+		if (icmp6->icmp6_type != 128 || icmp6->icmp6_code != 0) {
+			error = EPERM;
+			goto bad;
+		}
+	}
+
+#if ENABLE_DEFAULT_SCOPE
+	if (dst->sin6_scope_id == 0) {  /* not change if specified  */
+			dst->sin6_scope_id = scope6_addr2default(&dst->sin6_addr);
+	}
+#endif
+
+	return rip6_output(m, so, (struct sockaddr_in6 *) nam, control);
+bad:
+	m_freem(m);
+	return error;
+}
+
+/* Like rip6_attach but without root privilege enforcement */
+__private_extern__ int
+icmp6_dgram_attach(struct socket *so, int proto, struct proc *p)
+{
+        struct inpcb *inp;
+        int error;
+
+        inp = sotoinpcb(so);
+        if (inp)
+                panic("icmp6_dgram_attach");
+		
+		if (proto != IPPROTO_ICMPV6)
+			return EINVAL;
+			
+        error = soreserve(so, rip_sendspace, rip_recvspace);
+        if (error)
+                return error;
+        error = in_pcballoc(so, &ripcbinfo, p);
+        if (error)
+                return error;
+        inp = (struct inpcb *)so->so_pcb;
+        inp->inp_vflag |= INP_IPV6;
+        inp->in6p_ip6_nxt = IPPROTO_ICMPV6;
+        inp->in6p_hops = -1;    /* use kernel default */
+        inp->in6p_cksum = -1;
+        MALLOC(inp->in6p_icmp6filt, struct icmp6_filter *,
+               sizeof(struct icmp6_filter), M_PCB, M_WAITOK);
+        if (inp->in6p_icmp6filt == NULL)
+                return (ENOMEM);
+        ICMP6_FILTER_SETPASSALL(inp->in6p_icmp6filt);
+        return 0;
+}
+
+
+
 #ifndef HAVE_PPSRATECHECK
 #ifndef timersub
 #define	timersub(tvp, uvp, vvp)						\
@@ -2733,11 +2882,9 @@ ppsratecheck(lasttime, curpps, maxpps)
 	int maxpps;	/* maximum pps allowed */
 {
 	struct timeval tv, delta;
-	int s, rv;
+	int rv;
 
-	s = splclock(); 
 	microtime(&tv);
-	splx(s);
 
 	timersub(&tv, lasttime, &delta);
 
@@ -2789,10 +2936,10 @@ ppsratecheck(lasttime, curpps, maxpps)
  * XXX per-destination/type check necessary?
  */
 static int
-icmp6_ratelimit(dst, type, code)
-	const struct in6_addr *dst;	/* not used at this moment */
-	const int type;			/* not used at this moment */
-	const int code;			/* not used at this moment */
+icmp6_ratelimit(
+	__unused const struct in6_addr *dst,	/* not used at this moment */
+	__unused const int type,			/* not used at this moment */
+	__unused const int code)			/* not used at this moment */
 {
 	int ret;
 
@@ -2807,3 +2954,4 @@ icmp6_ratelimit(dst, type, code)
 
 	return ret;
 }
+
diff --git a/bsd/netinet6/in6.c b/bsd/netinet6/in6.c
index 8887a091d..c5da17932 100644
--- a/bsd/netinet6/in6.c
+++ b/bsd/netinet6/in6.c
@@ -1,3 +1,31 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
 /*	$FreeBSD: src/sys/netinet6/in6.c,v 1.7.2.7 2001/08/06 20:26:22 ume Exp $	*/
 /*	$KAME: in6.c,v 1.187 2001/05/24 07:43:59 itojun Exp $	*/
 
@@ -78,12 +106,14 @@
 #include <sys/kernel.h>
 #include <sys/syslog.h>
 #include <sys/kern_event.h>
-#include <kern/lock.h>
+#include <kern/locks.h>
 
 #include <net/if.h>
 #include <net/if_types.h>
+#include <net/if_var.h>
 #include <net/route.h>
 #include <net/if_dl.h>
+#include <net/kpi_protocol.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -138,7 +168,8 @@ static int in6_ifinit(struct ifnet *, struct in6_ifaddr *,
 static void in6_unlink_ifa(struct in6_ifaddr *, struct ifnet *, int);
 
 struct in6_multihead in6_multihead;	/* XXX BSS initialization */
-extern struct lck_mtx_t *nd6_mutex;
+extern lck_mtx_t *nd6_mutex;
+extern int in6_init2done;
 
 /*
  * Subroutine for in6_ifaddloop() and in6_ifremloop().
@@ -195,11 +226,7 @@ in6_ifloop_request(int cmd, struct ifaddr *ifa)
 	if (nrt) {
 		rt_newaddrmsg(cmd, ifa, e, nrt);
 		if (cmd == RTM_DELETE) {
-			if (nrt->rt_refcnt <= 0) {
-				/* XXX: we should free the entry ourselves. */
-				rtref(nrt);
-				rtfree_locked(nrt);
-			}
+			rtfree_locked(nrt);
 		} else {
 			/* the cmd must be RTM_ADD here */
 			rtunref(nrt);
@@ -221,12 +248,12 @@ in6_ifaddloop(struct ifaddr *ifa)
 
 	lck_mtx_lock(rt_mtx);
 	/* If there is no loopback entry, allocate one. */
-	rt = rtalloc1_locked(ifa->ifa_addr, 0, 0);
+	rt = rtalloc1_locked(ifa->ifa_addr, 0, 0UL);
 	if (rt == NULL || (rt->rt_flags & RTF_HOST) == 0 ||
 	    (rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0)
 		in6_ifloop_request(RTM_ADD, ifa);
 	if (rt)
-		rt->rt_refcnt--;
+		rtunref(rt);
 	lck_mtx_unlock(rt_mtx);
 }
 
@@ -279,10 +306,10 @@ in6_ifremloop(struct ifaddr *ifa, int locked)
 		 * to a shared medium.
 		 */
 		lck_mtx_lock(rt_mtx);
-		rt = rtalloc1_locked(ifa->ifa_addr, 0, 0);
+		rt = rtalloc1_locked(ifa->ifa_addr, 0, 0UL);
 		if (rt != NULL && (rt->rt_flags & RTF_HOST) != 0 &&
 		    (rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
-			rt->rt_refcnt--;
+			rtunref(rt);
 			in6_ifloop_request(RTM_DELETE, ifa);
 		}
 		lck_mtx_unlock(rt_mtx);
@@ -485,7 +512,7 @@ in6_control(so, cmd, data, ifp, p)
 
 	case SIOCAUTOCONF_STOP: 
 		{
-			struct in6_ifaddr *ia, *nia = NULL;
+			struct in6_ifaddr *nia = NULL;
 			
 			ifnet_lock_exclusive(ifp);
 			ifp->if_eflags &= ~IFEF_ACCEPT_RTADVD;
@@ -524,7 +551,7 @@ in6_control(so, cmd, data, ifp, p)
 
 	case SIOCLL_STOP:
 		{
-			struct in6_ifaddr *ia, *nia = NULL;
+			struct in6_ifaddr *nia = NULL;
 			
 			/* removed link local addresses from interface */
 
@@ -551,7 +578,7 @@ in6_control(so, cmd, data, ifp, p)
 #endif
 			default:
 
-				if (error = dlil_plumb_protocol(PF_INET6, ifp))
+				if ((error = proto_plumb(PF_INET6, ifp)))
 					printf("SIOCPROTOATTACH_IN6: %s error=%d\n", 
 						if_name(ifp), error);
 				break;
@@ -564,7 +591,7 @@ in6_control(so, cmd, data, ifp, p)
 
 		in6_purgeif(ifp);	/* Cleanup interface routes and addresses */
 
-		if (error = dlil_unplumb_protocol(PF_INET6, ifp))
+		if ((error = proto_unplumb(PF_INET6, ifp)))
 			 printf("SIOCPROTODETACH_IN6: %s error=%d\n",
 			 	if_name(ifp), error);
 		return(error);
@@ -764,7 +791,7 @@ in6_control(so, cmd, data, ifp, p)
 		struct nd_prefix pr0, *pr;
 		
 		/* Attempt to attache the protocol, in case it isn't attached */
-		error = dlil_plumb_protocol(PF_INET6, ifp);
+		error = proto_plumb(PF_INET6, ifp);
 		if (error) {
 			if (error != EEXIST) {
 				printf("SIOCAIFADDR_IN6: %s can't plumb protocol error=%d\n", 
@@ -847,8 +874,10 @@ in6_control(so, cmd, data, ifp, p)
 		} else {
 			if ((ia->ia6_flags & IN6_IFF_AUTOCONF) != 0 &&
 			    ia->ia6_ndpr == NULL) { /* new autoconfed addr */
-				ia->ia6_ndpr = pr;
+				lck_mtx_lock(nd6_mutex);
 				pr->ndpr_refcnt++;
+				lck_mtx_unlock(nd6_mutex);
+				ia->ia6_ndpr = pr;
 
 				/*
 				 * If this is the first autoconf address from
@@ -876,6 +905,9 @@ in6_control(so, cmd, data, ifp, p)
 			pfxlist_onlink_check(0);
 		}
 
+		/* Drop use count held above during lookup/add */
+		ndpr_rele(pr, FALSE);
+
 		break;
 	}
 
@@ -921,20 +953,18 @@ in6_control(so, cmd, data, ifp, p)
 			pr->ndpr_expire = 1; /* XXX: just for expiration */
 		}
 
+		/* Drop use count held above during lookup */
+		if (pr != NULL)
+			ndpr_rele(pr, FALSE);
+
 	  purgeaddr:
 		in6_purgeaddr(&ia->ia_ifa, 0);
 		break;
 	}
 
 	default:
-#ifdef __APPLE__
-		error = dlil_ioctl(PF_INET6, ifp, cmd, (caddr_t)data);
+		error = ifnet_ioctl(ifp, PF_INET6, cmd, data);
 		goto ioctl_cleanup;
-#else
-		if (ifp == NULL || ifp->if_ioctl == 0)
-			return(EOPNOTSUPP);
-		return((*ifp->if_ioctl)(ifp, cmd, data));
-#endif
 	}
 ioctl_cleanup:
 	return error;
@@ -1089,10 +1119,12 @@ in6_update_ifa(ifp, ifra, ia)
 		/*
 		 * When in6_update_ifa() is called in a process of a received
 		 * RA, it is called under splnet().  So, we should call malloc
-		 * with M_NOWAIT.
+		 * with M_NOWAIT.  The exception to this is during init time
+		 * when we know it's okay to do M_WAITOK, hence the check
+		 * against in6_init2done flag to see if it's not yet set.
 		 */
-		ia = (struct in6_ifaddr *)
-			_MALLOC(sizeof(*ia), M_IFADDR, M_NOWAIT);
+		ia = (struct in6_ifaddr *) _MALLOC(sizeof(*ia), M_IFADDR,
+		    in6_init2done ? M_NOWAIT : M_WAITOK);
 		if (ia == NULL)
 			return ENOBUFS;
 		bzero((caddr_t)ia, sizeof(*ia));
@@ -1514,7 +1546,7 @@ in6_purgeif(ifp)
  *		get first address that matches the specified prefix.
  *	SIOCALIFADDR: add the specified address.
  *	SIOCALIFADDR with IFLR_PREFIX:
- *		add the specified prefix, filling hostid part from
+ *		add the specified prefix, filling hostaddr part from
  *		the first link-local address.  prefixlen must be <= 64.
  *	SIOCDLIFADDR: delete the specified address.
  *	SIOCDLIFADDR with IFLR_PREFIX:
@@ -1584,7 +1616,7 @@ in6_lifaddr_ioctl(so, cmd, data, ifp, p)
 	case SIOCALIFADDR:
 	    {
 		struct in6_aliasreq ifra;
-		struct in6_addr hostid;
+		struct in6_addr hostaddr;
 		int prefixlen;
 		int hostid_found = 0;
 
@@ -1592,14 +1624,14 @@ in6_lifaddr_ioctl(so, cmd, data, ifp, p)
 			struct sockaddr_in6 *sin6;
 
 			/*
-			 * hostid is to fill in the hostid part of the
-			 * address.  hostid points to the first link-local
+			 * hostaddr is to fill in the hostaddr part of the
+			 * address.  hostaddr points to the first link-local
 			 * address attached to the interface.
 			 */
 			ifa = (struct ifaddr *)in6ifa_ifpforlinklocal(ifp, 0);
 			if (!ifa)
 				return EADDRNOTAVAIL;
-			hostid = *IFA_IN6(ifa);
+			hostaddr = *IFA_IN6(ifa);
 			hostid_found = 1;
 
 		 	/* prefixlen must be <= 64. */
@@ -1607,7 +1639,7 @@ in6_lifaddr_ioctl(so, cmd, data, ifp, p)
 				return EINVAL;
 			prefixlen = iflr->prefixlen;
 
-			/* hostid part must be zero. */
+			/* hostaddr part must be zero. */
 			sin6 = (struct sockaddr_in6 *)&iflr->addr;
 			if (sin6->sin6_addr.s6_addr32[2] != 0
 			 || sin6->sin6_addr.s6_addr32[3] != 0) {
@@ -1624,11 +1656,11 @@ in6_lifaddr_ioctl(so, cmd, data, ifp, p)
 		bcopy(&iflr->addr, &ifra.ifra_addr,
 			((struct sockaddr *)&iflr->addr)->sa_len);
 		if (hostid_found) {
-			/* fill in hostid part */
+			/* fill in hostaddr part */
 			ifra.ifra_addr.sin6_addr.s6_addr32[2] =
-				hostid.s6_addr32[2];
+				hostaddr.s6_addr32[2];
 			ifra.ifra_addr.sin6_addr.s6_addr32[3] =
-				hostid.s6_addr32[3];
+				hostaddr.s6_addr32[3];
 		}
 
 		if (((struct sockaddr *)&iflr->dstaddr)->sa_family) {	/*XXX*/
@@ -1636,9 +1668,9 @@ in6_lifaddr_ioctl(so, cmd, data, ifp, p)
 				((struct sockaddr *)&iflr->dstaddr)->sa_len);
 			if (hostid_found) {
 				ifra.ifra_dstaddr.sin6_addr.s6_addr32[2] =
-					hostid.s6_addr32[2];
+					hostaddr.s6_addr32[2];
 				ifra.ifra_dstaddr.sin6_addr.s6_addr32[3] =
-					hostid.s6_addr32[3];
+					hostaddr.s6_addr32[3];
 			}
 		}
 
@@ -1818,7 +1850,7 @@ in6_ifinit(ifp, ia, sin6, newhost)
 
 
 	if (ifacount <= 1 && 
-	    (error = dlil_ioctl(PF_INET6, ifp, SIOCSIFADDR, (caddr_t)ia))) {
+	    (error = ifnet_ioctl(ifp, PF_INET6, SIOCSIFADDR, ia))) {
 		if (error) {
 			return(error);
 		}
@@ -2060,6 +2092,31 @@ ip6_sprintf(addr)
 	return(ip6buf[ip6round]);
 }
 
+int
+in6addr_local(struct in6_addr *in6)
+{
+	struct rtentry *rt;
+	struct sockaddr_in6 sin6;
+	int local = 0;
+
+	if (IN6_IS_ADDR_LOOPBACK(in6) || IN6_IS_ADDR_LINKLOCAL(in6))
+		return (1);
+
+	sin6.sin6_family = AF_INET6;
+	sin6.sin6_len = sizeof (sin6);
+	bcopy(in6, &sin6.sin6_addr, sizeof (*in6));
+	rt = rtalloc1((struct sockaddr *)&sin6, 0, 0UL);
+
+	if (rt != NULL) {
+		if (rt->rt_gateway->sa_family == AF_LINK)
+			local = 1;
+		rtfree(rt);
+	} else {
+		local = in6_localaddr(in6);
+	}
+	return (local);
+}
+
 int
 in6_localaddr(in6)
 	struct in6_addr *in6;
@@ -2557,8 +2614,6 @@ in6_ifawithifp(
 	return NULL;
 }
 
-extern int in6_init2done;
-
 /*
  * perform DAD when interface becomes IFF_UP.
  */
@@ -2642,8 +2697,8 @@ in6_setmaxmtu()
 	ifnet_head_lock_shared();
 	TAILQ_FOREACH(ifp, &ifnet_head, if_list) {
 		if ((ifp->if_flags & IFF_LOOPBACK) == 0 &&
-		    nd_ifinfo[ifp->if_index].linkmtu > maxmtu)
-			maxmtu =  nd_ifinfo[ifp->if_index].linkmtu;
+		    IN6_LINKMTU(ifp) > maxmtu)
+			maxmtu = IN6_LINKMTU(ifp);
 	}
 	ifnet_head_done();
 	if (maxmtu)	/* update only when maxmtu is positive */
diff --git a/bsd/netinet6/in6.h b/bsd/netinet6/in6.h
index 989ea25a4..29188674f 100644
--- a/bsd/netinet6/in6.h
+++ b/bsd/netinet6/in6.h
@@ -88,7 +88,7 @@ typedef __uint8_t		sa_family_t;
 #define __KAME__
 #define __KAME_VERSION		"20010528/apple-darwin"
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * Local port number conventions:
  *
@@ -122,7 +122,7 @@ typedef __uint8_t		sa_family_t;
 #define	IPV6PORT_ANONMAX	65535
 #define	IPV6PORT_RESERVEDMIN	600
 #define	IPV6PORT_RESERVEDMAX	(IPV6PORT_RESERVED-1)
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * IPv6 address
@@ -147,9 +147,9 @@ struct in6_addr {
 /*
  * Socket address for IPv6
  */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define SIN6_LEN
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 struct sockaddr_in6 {
 	__uint8_t	sin6_len;	/* length of this struct(sa_family_t)*/
 	sa_family_t	sin6_family;	/* AF_INET6 (sa_family_t) */
@@ -219,7 +219,7 @@ extern const struct in6_addr in6mask128;
 #define IN6ADDR_LOOPBACK_INIT \
 	{{{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
 	    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }}}
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define IN6ADDR_NODELOCAL_ALLNODES_INIT \
 	{{{ 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
 	    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 }}}
@@ -229,15 +229,15 @@ extern const struct in6_addr in6mask128;
 #define IN6ADDR_LINKLOCAL_ALLROUTERS_INIT \
 	{{{ 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
 	    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }}}
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 extern const struct in6_addr in6addr_any;
 extern const struct in6_addr in6addr_loopback;
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 extern const struct in6_addr in6addr_nodelocal_allnodes;
 extern const struct in6_addr in6addr_linklocal_allnodes;
 extern const struct in6_addr in6addr_linklocal_allrouters;
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Equality
@@ -249,10 +249,10 @@ extern const struct in6_addr in6addr_linklocal_allrouters;
 #define IN6_ARE_ADDR_EQUAL(a, b)			\
     (bcmp(&(a)->s6_addr[0], &(b)->s6_addr[0], sizeof(struct in6_addr)) == 0)
 #else
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define IN6_ARE_ADDR_EQUAL(a, b)			\
     (memcmp(&(a)->s6_addr[0], &(b)->s6_addr[0], sizeof(struct in6_addr)) == 0)
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #endif
 
 #ifdef KERNEL			/* non standard */
@@ -393,21 +393,21 @@ extern const struct in6_addr in6addr_linklocal_allrouters;
 /*
  * IP6 route structure
  */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifdef PRIVATE
 struct route_in6 {
 	struct	rtentry *ro_rt;
 	struct	sockaddr_in6 ro_dst;
 };
 #endif /* PRIVATE */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Options for use with [gs]etsockopt at the IPV6 level.
  * First word of comment is data type; bool is stored in int.
  */
 /* no hdrincl */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #if 0 /* the followings are relic in IPv4 and hence are disabled */
 #define IPV6_OPTIONS		1  /* buf/ip6_opts; set/get IP6 options */
 #define IPV6_RECVOPTS		5  /* bool; receive all IP6 opts w/dgram */
@@ -416,7 +416,7 @@ struct route_in6 {
 #define IPV6_RETOPTS		8  /* ip6_opts; set/get IP6 options */
 #endif /* 0 */
 #define IPV6_SOCKOPT_RESERVED1	3  /* reserved for future use */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define IPV6_UNICAST_HOPS	4  /* int; IP6 hops */
 #define IPV6_MULTICAST_IF	9  /* __uint8_t; set/get IP6 multicast i/f  */
 #define IPV6_MULTICAST_HOPS	10 /* __uint8_t; set/get IP6 multicast hops */
@@ -424,7 +424,7 @@ struct route_in6 {
 #define IPV6_JOIN_GROUP		12 /* ip6_mreq; join a group membership */
 #define IPV6_LEAVE_GROUP	13 /* ip6_mreq; leave a group membership */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define IPV6_PORTRANGE		14 /* int; range to choose for unspec port */
 #define ICMP6_FILTER		18 /* icmp6_filter; icmp6 filter */
 /* RFC2292 options */
@@ -437,9 +437,9 @@ struct route_in6 {
 #define IPV6_PKTOPTIONS		25 /* buf/cmsghdr; set/get IPv6 options */
 
 #define IPV6_CHECKSUM		26 /* int; checksum offset for raw socket */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define IPV6_V6ONLY		27 /* bool; only bind INET6 at wildcard bind */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifndef KERNEL
 #define IPV6_BINDV6ONLY		IPV6_V6ONLY
 #endif /* KERNEL */
@@ -609,24 +609,29 @@ struct mbuf;
 struct ifnet;
 struct in6_aliasreq;
 
-int	in6_cksum(struct mbuf *, __uint8_t, __uint32_t, __uint32_t);
-int	in6_localaddr(struct in6_addr *);
-int	in6_addrscope(struct in6_addr *);
-struct	in6_ifaddr *in6_ifawithscope(struct ifnet *, struct in6_addr *);
-struct	in6_ifaddr *in6_ifawithifp(struct ifnet *, struct in6_addr *);
+#define	in6_cksum(m, n, o, l)	inet6_cksum(m, n, o, l)
+
+extern u_int16_t inet6_cksum(struct mbuf *m, unsigned int proto,
+    unsigned int offset, unsigned int transport_len);
+
+extern int in6_localaddr(struct in6_addr *);
+extern int in6_addrscope(struct in6_addr *);
+extern struct in6_ifaddr *in6_ifawithscope(struct ifnet *, struct in6_addr *);
+extern struct in6_ifaddr *in6_ifawithifp(struct ifnet *, struct in6_addr *);
 extern void in6_if_up(struct ifnet *, struct in6_aliasreq *);
 struct sockaddr;
 
-void	in6_sin6_2_sin(struct sockaddr_in *sin,
-			    struct sockaddr_in6 *sin6);
-void	in6_sin_2_v4mapsin6(struct sockaddr_in *sin,
-				 struct sockaddr_in6 *sin6);
-void	in6_sin6_2_sin_in_sock(struct sockaddr *nam);
-void	in6_sin_2_v4mapsin6_in_sock(struct sockaddr **nam);
+extern void in6_sin6_2_sin(struct sockaddr_in *sin, struct sockaddr_in6 *sin6);
+extern void in6_sin_2_v4mapsin6(struct sockaddr_in *sin,
+    struct sockaddr_in6 *sin6);
+extern void in6_sin6_2_sin_in_sock(struct sockaddr *nam);
+extern void in6_sin_2_v4mapsin6_in_sock(struct sockaddr **nam);
 
 #define	satosin6(sa)	((struct sockaddr_in6 *)(sa))
 #define	sin6tosa(sin6)	((struct sockaddr *)(sin6))
 #define	ifatoia6(ifa)	((struct in6_ifaddr *)(ifa))
+
+extern int in6addr_local(struct in6_addr *);
 #endif /* KERNEL_PRIVATE */
 
 #ifndef KERNEL
@@ -672,5 +677,5 @@ extern int inet6_rth_segments(const void *);
 extern struct in6_addr *inet6_rth_getaddr(const void *, int);
 __END_DECLS
 #endif /* !KERNEL */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #endif /* !_NETINET6_IN6_H_ */
diff --git a/bsd/netinet6/in6_cksum.c b/bsd/netinet6/in6_cksum.c
index 60cdf2e5b..fd3e143dc 100644
--- a/bsd/netinet6/in6_cksum.c
+++ b/bsd/netinet6/in6_cksum.c
@@ -68,6 +68,7 @@
 #include <sys/param.h>
 #include <sys/mbuf.h>
 #include <sys/systm.h>
+#include <kern/debug.h>
 #include <netinet/in.h>
 #include <netinet/ip6.h>
 
@@ -90,20 +91,15 @@
  * (e.g. TCP header + TCP payload)
  */
 
-int
-in6_cksum(m, nxt, off, len)
-	struct mbuf *m;
-	u_int8_t nxt;
-	u_int32_t off, len;
+u_int16_t
+inet6_cksum(struct mbuf *m, unsigned int nxt, unsigned int off,
+    unsigned int len)
 {
 	u_int16_t *w;
 	int sum = 0;
 	int mlen = 0;
 	int byte_swapped = 0;
-#if 0
-	int srcifid = 0, dstifid = 0;
-#endif
-	struct ip6_hdr *ip6;	
+	struct ip6_hdr *ip6;
 	union {
 		u_int16_t phs[4];
 		struct {
@@ -123,52 +119,38 @@ in6_cksum(m, nxt, off, len)
 
 	/* sanity check */
 	if (m->m_pkthdr.len < off + len) {
-		panic("in6_cksum: mbuf len (%d) < off+len (%d+%d)\n",
-			m->m_pkthdr.len, off, len);
+		panic("inet6_cksum: mbuf len (%d) < off+len (%d+%d)\n",
+		    m->m_pkthdr.len, off, len);
 	}
 
-	bzero(&uph, sizeof(uph));
+	if (nxt != 0) {
+		bzero(&uph, sizeof (uph));
 
-	/*
-	 * First create IP6 pseudo header and calculate a summary.
-	 */
-	ip6 = mtod(m, struct ip6_hdr *);
-#if 0
-	if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src)) {
-		srcifid = ip6->ip6_src.s6_addr16[1];
-		ip6->ip6_src.s6_addr16[1] = 0;
-	}
-	if (IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst)) {
-		dstifid = ip6->ip6_dst.s6_addr16[1];
-		ip6->ip6_dst.s6_addr16[1] = 0;
-	}
-#endif
-	w = (u_int16_t *)&ip6->ip6_src;
-	uph.ph.ph_len = htonl(len);
-	uph.ph.ph_nxt = nxt;
+		/*
+		 * First create IP6 pseudo header and calculate a summary.
+		 */
+		ip6 = mtod(m, struct ip6_hdr *);
+		w = (u_int16_t *)&ip6->ip6_src;
+		uph.ph.ph_len = htonl(len);
+		uph.ph.ph_nxt = nxt;
 
-	/* IPv6 source address */
-	sum += w[0];
-	if (!IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src))
-		sum += w[1];
-	sum += w[2]; sum += w[3]; sum += w[4]; sum += w[5];
-	sum += w[6]; sum += w[7];
-	/* IPv6 destination address */
-	sum += w[8];
-	if (!IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst))
-		sum += w[9];
-	sum += w[10]; sum += w[11]; sum += w[12]; sum += w[13];
-	sum += w[14]; sum += w[15];
-	/* Payload length and upper layer identifier */
-	sum += uph.phs[0];  sum += uph.phs[1];
-	sum += uph.phs[2];  sum += uph.phs[3];
+		/* IPv6 source address */
+		sum += w[0];
+		if (!IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_src))
+			sum += w[1];
+		sum += w[2]; sum += w[3]; sum += w[4]; sum += w[5];
+		sum += w[6]; sum += w[7];
+		/* IPv6 destination address */
+		sum += w[8];
+		if (!IN6_IS_SCOPE_LINKLOCAL(&ip6->ip6_dst))
+			sum += w[9];
+		sum += w[10]; sum += w[11]; sum += w[12]; sum += w[13];
+		sum += w[14]; sum += w[15];
+		/* Payload length and upper layer identifier */
+		sum += uph.phs[0];  sum += uph.phs[1];
+		sum += uph.phs[2];  sum += uph.phs[3];
+	}
 
-#if 0
-	if (srcifid)
-		ip6->ip6_src.s6_addr16[1] = srcifid;
-	if (dstifid)
-		ip6->ip6_dst.s6_addr16[1] = dstifid;
-#endif
 	/*
 	 * Secondly calculate a summary of the first mbuf excluding offset.
 	 */
@@ -236,7 +218,7 @@ in6_cksum(m, nxt, off, len)
 	/*
 	 * Lastly calculate a summary of the rest of mbufs.
 	 */
-	
+
 	for (;m && len; m = m->m_next) {
 		if (m->m_len == 0)
 			continue;
@@ -308,7 +290,7 @@ in6_cksum(m, nxt, off, len)
 			s_util.c[0] = *(char *)w;
 	}
 	if (len)
-		panic("in6_cksum: out of data\n");
+		printf("inet6_cksum: out of data by %d\n", len);
 	if (mlen == -1) {
 		/* The last mbuf has odd # of bytes. Follow the
 		   standard (the odd byte may be shifted left by 8 bits
diff --git a/bsd/netinet6/in6_gif.c b/bsd/netinet6/in6_gif.c
index 5c1138d0a..090d0ad31 100644
--- a/bsd/netinet6/in6_gif.c
+++ b/bsd/netinet6/in6_gif.c
@@ -67,14 +67,25 @@
 
 #include <net/net_osdep.h>
 
+static __inline__ void*
+_cast_non_const(const void * ptr) {
+	union {
+		const void*		cval;
+		void*			val;
+	} ret;
+	
+	ret.cval = ptr;
+	return (ret.val);
+}
+
 int
 in6_gif_output(
 	struct ifnet *ifp,
 	int family, /* family of the packet to be encapsulate. */
 	struct mbuf *m,
-	struct rtentry *rt)
+	__unused struct rtentry *rt)
 {
-	struct gif_softc *sc = (struct gif_softc*)ifp;
+	struct gif_softc *sc = ifnet_softc(ifp);
 	struct sockaddr_in6 *dst = (struct sockaddr_in6 *)&sc->gif_ro6.ro_dst;
 	struct sockaddr_in6 *sin6_src = (struct sockaddr_in6 *)sc->gif_psrc;
 	struct sockaddr_in6 *sin6_dst = (struct sockaddr_in6 *)sc->gif_pdst;
@@ -96,7 +107,7 @@ in6_gif_output(
 		struct ip *ip;
 
 		proto = IPPROTO_IPV4;
-		if (m->m_len < sizeof(*ip)) {
+		if (mbuf_len(m) < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (!m)
 				return ENOBUFS;
@@ -109,9 +120,8 @@ in6_gif_output(
 #if INET6
 	case AF_INET6:
 	    {
-		struct ip6_hdr *ip6;
 		proto = IPPROTO_IPV6;
-		if (m->m_len < sizeof(*ip6)) {
+		if (mbuf_len(m) < sizeof(*ip6)) {
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m)
 				return ENOBUFS;
@@ -132,7 +142,7 @@ in6_gif_output(
 	
 	/* prepend new IP header */
 	M_PREPEND(m, sizeof(struct ip6_hdr), M_DONTWAIT);
-	if (m && m->m_len < sizeof(struct ip6_hdr))
+	if (m && mbuf_len(m) < sizeof(struct ip6_hdr))
 		m = m_pullup(m, sizeof(struct ip6_hdr));
 	if (m == NULL) {
 		printf("ENOBUFS in in6_gif_output %d\n", __LINE__);
@@ -220,7 +230,7 @@ int in6_gif_input(mp, offp)
 
 	ip6 = mtod(m, struct ip6_hdr *);
 
-	gifp = (struct ifnet *)encap_getarg(m);
+	gifp = ((struct gif_softc*)encap_getarg(m))->gif_if;
 
 	if (gifp == NULL || (gifp->if_flags & IFF_UP) == 0) {
 		m_freem(m);
@@ -240,7 +250,7 @@ int in6_gif_input(mp, offp)
 		u_int8_t otos8;
 		af = AF_INET;
 		otos8 = (ntohl(otos) >> 20) & 0xff;
-		if (m->m_len < sizeof(*ip)) {
+		if (mbuf_len(m) < sizeof(*ip)) {
 			m = m_pullup(m, sizeof(*ip));
 			if (!m)
 				return IPPROTO_DONE;
@@ -256,9 +266,8 @@ int in6_gif_input(mp, offp)
 #if INET6
 	case IPPROTO_IPV6:
 	    {
-		struct ip6_hdr *ip6;
 		af = AF_INET6;
-		if (m->m_len < sizeof(*ip6)) {
+		if (mbuf_len(m) < sizeof(*ip6)) {
 			m = m_pullup(m, sizeof(*ip6));
 			if (!m)
 				return IPPROTO_DONE;
@@ -276,8 +285,11 @@ int in6_gif_input(mp, offp)
 		m_freem(m);
 		return IPPROTO_DONE;
 	}
-		
-	dlil_input(gifp, m, m);
+
+	if (m->m_pkthdr.rcvif)  /* replace the rcvif by gifp for ifnet_input to route it correctly */
+		m->m_pkthdr.rcvif = gifp;
+
+	ifnet_input(gifp, m, NULL);
 	return IPPROTO_DONE;
 }
 
@@ -285,10 +297,10 @@ int in6_gif_input(mp, offp)
  * validate outer address.
  */
 static int
-gif_validate6(ip6, sc, ifp)
-	const struct ip6_hdr *ip6;
-	struct gif_softc *sc;
-	struct ifnet *ifp;
+gif_validate6(
+	const struct ip6_hdr *ip6,
+	struct gif_softc *sc,
+	struct ifnet *ifp)
 {
 	struct sockaddr_in6 *src, *dst;
 
@@ -307,7 +319,7 @@ gif_validate6(ip6, sc, ifp)
 	/* martian filters on outer source - done in ip6_input */
 
 	/* ingress filters on outer source */
-	if ((sc->gif_if.if_flags & IFF_LINK2) == 0 && ifp) {
+	if ((ifnet_flags(sc->gif_if) & IFF_LINK2) == 0 && ifp) {
 		struct sockaddr_in6 sin6;
 		struct rtentry *rt;
 
@@ -342,11 +354,11 @@ gif_validate6(ip6, sc, ifp)
  * sanity check for arg should have been done in the caller.
  */
 int
-gif_encapcheck6(m, off, proto, arg)
-	const struct mbuf *m;
-	int off;
-	int proto;
-	void *arg;
+gif_encapcheck6(
+	const struct mbuf *m,
+	__unused int off,
+	__unused int proto,
+	void *arg)
 {
 	struct ip6_hdr ip6;
 	struct gif_softc *sc;
@@ -355,8 +367,7 @@ gif_encapcheck6(m, off, proto, arg)
 	/* sanity check done in caller */
 	sc = (struct gif_softc *)arg;
 
-	/* LINTED const cast */
-	m_copydata(m, 0, sizeof(ip6), (caddr_t)&ip6);
+	mbuf_copydata(m, 0, sizeof(ip6), &ip6);
 	ifp = ((m->m_flags & M_PKTHDR) != 0) ? m->m_pkthdr.rcvif : NULL;
 
 	return gif_validate6(&ip6, sc, ifp);
diff --git a/bsd/netinet6/in6_ifattach.c b/bsd/netinet6/in6_ifattach.c
index 2ccb29cd0..b711892ab 100644
--- a/bsd/netinet6/in6_ifattach.c
+++ b/bsd/netinet6/in6_ifattach.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*	$FreeBSD: src/sys/netinet6/in6_ifattach.c,v 1.8 2002/04/19 04:46:22 suz Exp $	*/
 /*	$KAME: in6_ifattach.c,v 1.118 2001/05/24 07:44:00 itojun Exp $	*/
 
@@ -38,13 +65,15 @@
 #include <sys/sockio.h>
 #include <sys/kernel.h>
 #include <sys/syslog.h>
-#include <sys/md5.h>
+#include <libkern/crypto/md5.h>
+#include <libkern/OSAtomic.h>
 #include <kern/lock.h>
 
 #include <net/if.h>
 #include <net/if_dl.h>
 #include <net/if_types.h>
 #include <net/route.h>
+#include <net/kpi_protocol.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -75,10 +104,10 @@ int ip6_auto_linklocal = IP6_AUTO_LINKLOCAL;
 int ip6_auto_linklocal = 1;	/* enable by default */
 #endif
 
+int loopattach6_done = 0;
 
 extern struct inpcbinfo udbinfo;
 extern struct inpcbinfo ripcbinfo;
-extern lck_mtx_t *rt_mtx;
 
 static int get_rand_ifid(struct ifnet *, struct in6_addr *);
 static int generate_tmp_ifid(u_int8_t *, const u_int8_t *, u_int8_t *);
@@ -107,23 +136,23 @@ static int in6_ifattach_loopback(struct ifnet *);
  */
 static int
 get_rand_ifid(
-	struct ifnet *ifp,
+	__unused struct ifnet *ifp,
 	struct in6_addr *in6)	/* upper 64bits are preserved */
 {
 	MD5_CTX ctxt;
 	u_int8_t digest[16];
-	int hostnamelen	= strlen(hostname);
+	int len	= strlen(hostname);
 
 #if 0
 	/* we need at least several letters as seed for ifid */
-	if (hostnamelen < 3)
+	if (len < 3)
 		return -1;
 #endif
 
 	/* generate 8 bytes of pseudo-random value. */
 	bzero(&ctxt, sizeof(ctxt));
 	MD5Init(&ctxt);
-	MD5Update(&ctxt, hostname, hostnamelen);
+	MD5Update(&ctxt, hostname, len);
 	MD5Final(digest, &ctxt);
 
 	/* assumes sizeof(digest) > sizeof(ifid) */
@@ -264,7 +293,7 @@ get_hw_ifid(
 
 found:
 	ifnet_lock_done(ifp);
-	addr = LLADDR(sdl);
+	addr = (u_int8_t *) LLADDR(sdl);
 	addrlen = sdl->sdl_alen;
 
 	/* get EUI64 */
@@ -442,7 +471,7 @@ in6_ifattach_linklocal(
 {
 	struct in6_ifaddr *ia;
 	struct in6_aliasreq ifra;
-	struct nd_prefix pr0;
+	struct nd_prefix pr0, *pr;
 	int i, error;
 
 	/*
@@ -450,7 +479,7 @@ in6_ifattach_linklocal(
 	 */
 	bzero(&ifra, sizeof(ifra));
 
-	dlil_plumb_protocol(PF_INET6, ifp);
+	proto_plumb(PF_INET6, ifp);
 
 	/*
 	 * in6_update_ifa() does not use ifra_name, but we accurately set it
@@ -574,12 +603,17 @@ in6_ifattach_linklocal(
 	 * address, and then reconfigure another one, the prefix is still
 	 * valid with referring to the old link-local address.
 	 */
-	if (nd6_prefix_lookup(&pr0) == NULL) {
-		if ((error = nd6_prelist_add(&pr0, NULL, NULL)) != 0)
+	if ((pr = nd6_prefix_lookup(&pr0)) == NULL) {
+		if ((error = nd6_prelist_add(&pr0, NULL, &pr)) != 0)
 			return(error);
 	}
 
 	in6_post_msg(ifp, KEV_INET6_NEW_LL_ADDR, ia);
+
+	/* Drop use count held above during lookup/add */
+	if (pr != NULL)
+		ndpr_rele(pr, FALSE);
+
 	return 0;
 }
 
@@ -669,7 +703,7 @@ in6_nigroup(
 	l = p - name;
 	strncpy(n, name, l);
 	n[(int)l] = '\0';
-	for (q = n; *q; q++) {
+	for (q = (u_char *) n; *q; q++) {
 		if ('A' <= *q && *q <= 'Z')
 			*q = *q - 'A' + 'a';
 	}
@@ -848,13 +882,17 @@ in6_ifattach(
 	 */
 	if ((ifp->if_flags & IFF_LOOPBACK) != 0) {
 		struct in6_ifaddr *ia6 = NULL;
-		in6 = in6addr_loopback;
-		if ((ia6 = in6ifa_ifpwithaddr(ifp, &in6)) == NULL) {
-			if (in6_ifattach_loopback(ifp) != 0)
-				return;
-		}
-		else {
-			ifafree(&ia6->ia_ifa);
+		if (!OSCompareAndSwap(0, 1, (UInt32 *)&loopattach6_done)) {
+			in6 = in6addr_loopback;
+			if ((ia6 = in6ifa_ifpwithaddr(ifp, &in6)) == NULL) {
+				if (in6_ifattach_loopback(ifp) != 0) {
+					OSCompareAndSwap(1, 0, (UInt32 *)&loopattach6_done);
+					return;
+				}
+			}
+			else {
+				ifafree(&ia6->ia_ifa);
+			}
 		}
 	}
 
@@ -909,8 +947,6 @@ in6_ifdetach(
 	struct rtentry *rt;
 	short rtflags;
 	struct sockaddr_in6 sin6;
-	struct in6_multi *in6m;
-	struct in6_multi *in6m_next;
 
 	/* nuke prefix list.  this may try to remove some of ifaddrs as well */
 	in6_purgeprefix(ifp);
@@ -946,8 +982,8 @@ in6_ifdetach(
 
 		/* remove from the routing table */
 		lck_mtx_lock(rt_mtx);
-		if ((ia->ia_flags & IFA_ROUTE)
-		 && (rt = rtalloc1_locked((struct sockaddr *)&ia->ia_addr, 0, 0UL))) {
+		if ((ia->ia_flags & IFA_ROUTE) &&
+		    (rt = rtalloc1_locked((struct sockaddr *)&ia->ia_addr, 0, 0UL))) {
 			rtflags = rt->rt_flags;
 			rtfree_locked(rt);
 			rtrequest_locked(RTM_DELETE,
@@ -1002,9 +1038,11 @@ in6_ifdetach(
 	sin6.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
 	lck_mtx_lock(rt_mtx);
 	rt = rtalloc1_locked((struct sockaddr *)&sin6, 0, 0UL);
-	if (rt && rt->rt_ifp == ifp) {
-		rtrequest_locked(RTM_DELETE, (struct sockaddr *)rt_key(rt),
-			rt->rt_gateway, rt_mask(rt), rt->rt_flags, 0);
+	if (rt != NULL) {
+	        if (rt->rt_ifp == ifp) {
+			rtrequest_locked(RTM_DELETE, (struct sockaddr *)rt_key(rt),
+				rt->rt_gateway, rt_mask(rt), rt->rt_flags, 0);
+		}
 		rtfree_locked(rt);
 	}
 	lck_mtx_unlock(rt_mtx);
@@ -1037,15 +1075,13 @@ in6_get_tmpifid(
 }
 
 extern size_t nd_ifinfo_indexlim;
-extern int ip6_use_tempaddr;
 void
 in6_tmpaddrtimer(
-	void *ignored_arg)
+	__unused void *ignored_arg)
 {
 	int i;
 	struct nd_ifinfo *ndi;
 	u_int8_t nullbuf[8];
-	int s = splnet();
 
 	timeout(in6_tmpaddrtimer, (caddr_t)0,
 		      (ip6_temp_preferred_lifetime - ip6_desync_factor -
@@ -1069,6 +1105,4 @@ in6_tmpaddrtimer(
 			}
 		}
 	}
-
-	splx(s);
 }
diff --git a/bsd/netinet6/in6_pcb.c b/bsd/netinet6/in6_pcb.c
index 2e82a0376..f98b3d35f 100644
--- a/bsd/netinet6/in6_pcb.c
+++ b/bsd/netinet6/in6_pcb.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -115,11 +121,6 @@
 #include <kern/kern_types.h>
 #include <kern/zalloc.h>
 
-#include "faith.h"
-#if defined(NFAITH) && NFAITH > 0
-#include <net/if_faith.h>
-#endif
-
 #if IPSEC
 #include <netinet6/ipsec.h>
 #if INET6
@@ -130,11 +131,48 @@
 #include <netinet6/ah6.h>
 #endif
 #include <netkey/key.h>
-extern lck_mtx_t *sadb_mutex;
 #endif /* IPSEC */
 
 struct	in6_addr zeroin6_addr;
 
+/*
+  in6_pcblookup_local_and_cleanup does everything
+  in6_pcblookup_local does but it checks for a socket
+  that's going away. Since we know that the lock is
+  held read+write when this function is called, we
+  can safely dispose of this socket like the slow
+  timer would usually do and return NULL. This is
+  great for bind.
+*/
+static struct inpcb*
+in6_pcblookup_local_and_cleanup(
+	struct inpcbinfo *pcbinfo,
+	struct in6_addr *laddr,
+	u_int lport_arg,
+	int wild_okay)
+{
+	struct inpcb *inp;
+	
+	/* Perform normal lookup */
+	inp = in6_pcblookup_local(pcbinfo, laddr, lport_arg, wild_okay);
+	
+	/* Check if we found a match but it's waiting to be disposed */
+	if (inp && inp->inp_wantcnt == WNT_STOPUSING) {
+		struct socket *so = inp->inp_socket;
+		
+		lck_mtx_lock(inp->inpcb_mtx);
+		
+		if (so->so_usecount == 0) {
+			in_pcbdispose(inp);
+			inp = NULL;
+		}
+		else {
+			lck_mtx_unlock(inp->inpcb_mtx);
+		}
+	}
+	
+	return inp;
+}
 int
 in6_pcbbind(
 	struct inpcb *inp,
@@ -231,7 +269,7 @@ in6_pcbbind(
 
 			if (so->so_uid &&
 			    !IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
-				t = in6_pcblookup_local(pcbinfo,
+				t = in6_pcblookup_local_and_cleanup(pcbinfo,
 				    &sin6->sin6_addr, lport,
 				    INPLOOKUP_WILDCARD);
 				if (t &&
@@ -239,7 +277,8 @@ in6_pcbbind(
 				     !IN6_IS_ADDR_UNSPECIFIED(&t->in6p_laddr) ||
 				     (t->inp_socket->so_options &
 				      SO_REUSEPORT) == 0) &&
-				    so->so_uid != t->inp_socket->so_uid) {
+				     (so->so_uid != t->inp_socket->so_uid) &&
+				     ((t->inp_socket->so_flags & SOF_REUSESHAREUID) == 0)) {
 					lck_rw_done(pcbinfo->mtx);
 					socket_lock(so, 0);
 					return (EADDRINUSE);
@@ -249,10 +288,10 @@ in6_pcbbind(
 					struct sockaddr_in sin;
 
 					in6_sin6_2_sin(&sin, sin6);
-					t = in_pcblookup_local(pcbinfo,
+					t = in_pcblookup_local_and_cleanup(pcbinfo,
 						sin.sin_addr, lport,
 						INPLOOKUP_WILDCARD);
-					if (t &&
+					if (t && (t->inp_socket->so_options & SO_REUSEPORT) == 0 &&
 					    (so->so_uid !=
 					     t->inp_socket->so_uid) &&
 					    (ntohl(t->inp_laddr.s_addr) !=
@@ -266,7 +305,7 @@ in6_pcbbind(
 					}
 				}
 			}
-			t = in6_pcblookup_local(pcbinfo, &sin6->sin6_addr,
+			t = in6_pcblookup_local_and_cleanup(pcbinfo, &sin6->sin6_addr,
 						lport, wild);
 			if (t && (reuseport & t->inp_socket->so_options) == 0) {
 				lck_rw_done(pcbinfo->mtx);
@@ -278,7 +317,7 @@ in6_pcbbind(
 				struct sockaddr_in sin;
 
 				in6_sin6_2_sin(&sin, sin6);
-				t = in_pcblookup_local(pcbinfo, sin.sin_addr,
+				t = in_pcblookup_local_and_cleanup(pcbinfo, sin.sin_addr,
 						       lport, wild);
 				if (t &&
 				    (reuseport & t->inp_socket->so_options)
@@ -313,6 +352,7 @@ in6_pcbbind(
 		}
 	}	
 	lck_rw_done(pcbinfo->mtx);
+	sflt_notify(so, sock_evt_bound, NULL);
 	return(0);
 }
 
@@ -552,7 +592,7 @@ in6_selectsrc(
 		struct ifnet *ifp = mopts ? mopts->im6o_multicast_ifp : NULL;
 
 		if (ifp == NULL && IN6_IS_ADDR_MC_NODELOCAL(dst)) {
-			ifp = &loif[0];
+			ifp = lo_ifp;
 		}
 
 		if (ifp) {
@@ -617,8 +657,8 @@ in6_selectsrc(
 			dst6->sin6_len = sizeof(struct sockaddr_in6);
 			dst6->sin6_addr = *dst;
 			if (IN6_IS_ADDR_MULTICAST(dst)) {
-				ro->ro_rt = rtalloc1(&((struct route *)ro)
-						     ->ro_dst, 0, 0UL);
+				ro->ro_rt =
+				    rtalloc1(&((struct route *)ro)->ro_dst, 0, 0UL);
 			} else {
 				rtalloc((struct route *)ro);
 			}
@@ -701,14 +741,12 @@ in6_pcbdetach(inp)
 
 #if IPSEC
 	if (inp->in6p_sp != NULL) {
-		lck_mtx_lock(sadb_mutex);
 		ipsec6_delete_pcbpolicy(inp);
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /* IPSEC */
 
 	if (in_pcb_checkstate(inp, WNT_STOPUSING, 1) != WNT_STOPUSING)
-		printf("in6_pcbdetach so=%x can't be marked dead ok\n", so);
+		printf("in6_pcbdetach so=%p can't be marked dead ok\n", so);
 
 	inp->inp_state = INPCB_STATE_DEAD;
 
@@ -1138,7 +1176,7 @@ in6_losing(in6p)
 void
 in6_rtchange(
 	struct inpcb *inp,
-	int errno)
+	__unused int errno)
 {
 	if (inp->in6p_route.ro_rt) {
 		rtfree(inp->in6p_route.ro_rt);
@@ -1161,7 +1199,7 @@ in6_pcblookup_hash(
 	struct in6_addr *laddr,
 	u_int lport_arg,
 	int wildcard,
-	struct ifnet *ifp)
+	__unused struct ifnet *ifp)
 {
 	struct inpcbhead *head;
 	struct inpcb *inp;
diff --git a/bsd/netinet6/in6_prefix.c b/bsd/netinet6/in6_prefix.c
index 88a945059..2ff2db707 100644
--- a/bsd/netinet6/in6_prefix.c
+++ b/bsd/netinet6/in6_prefix.c
@@ -300,7 +300,7 @@ mark_matched_prefixes(u_long cmd, struct ifnet *ifp, struct in6_rrenumreq *irr)
  * Mark global prefixes as to be deleted.
  */
 static void
-delmark_global_prefixes(struct ifnet *ifp, struct in6_rrenumreq *irr)
+delmark_global_prefixes(struct ifnet *ifp, __unused struct in6_rrenumreq *irr)
 {
 	struct ifprefix *ifpr;
 
@@ -412,7 +412,6 @@ assign_ra_entry(struct rr_prefix *rpp, int iilen, struct in6_ifaddr *ia)
 {
 	int error = 0;
 	struct rp_addr *rap;
-	int s;
 
 	if ((error = create_ra_entry(&rap)) != 0)
 		return error;
@@ -439,7 +438,7 @@ assign_ra_entry(struct rr_prefix *rpp, int iilen, struct in6_ifaddr *ia)
  * (prefix database + new interface id).
  */
 static int
-in6_prefix_add_llifid(int iilen, struct in6_ifaddr *ia)
+in6_prefix_add_llifid(__unused int iilen, struct in6_ifaddr *ia)
 {
 	struct rr_prefix *rpp;
 	struct rp_addr *rap;
@@ -558,7 +557,7 @@ in6_prefix_add_ifid(int iilen, struct in6_ifaddr *ia)
 }
 
 void
-in6_prefix_remove_ifid(int iilen, struct in6_ifaddr *ia)
+in6_prefix_remove_ifid(__unused int iilen, struct in6_ifaddr *ia)
 {
 	struct rp_addr *rap;
 
@@ -694,7 +693,6 @@ rrpr_update(struct socket *so, struct rr_prefix *new)
 	struct rr_prefix *rpp;
 	struct ifprefix *ifpr;
 	struct rp_addr *rap;
-	int s;
 
 	/* search existing prefix */
 	ifnet_lock_exclusive(new->rp_ifp);
@@ -783,7 +781,6 @@ rrpr_update(struct socket *so, struct rr_prefix *new)
 		/* link rr_prefix entry to if_prefixlist */
 		{
 			struct ifnet *ifp = rpp->rp_ifp;
-			struct ifprefix *ifpr;
 
 			if ((ifpr = TAILQ_FIRST(&ifp->if_prefixhead))
 			    != NULL) {
@@ -1012,7 +1009,6 @@ delete_each_prefix(struct rr_prefix *rpp, u_char origin)
 	lck_mtx_lock(prefix6_mutex);
 	while (rpp->rp_addrhead.lh_first != NULL) {
 		struct rp_addr *rap;
-		int s;
 
 		rap = LIST_FIRST(&rpp->rp_addrhead);
 		if (rap == NULL) {
@@ -1228,7 +1224,7 @@ in6_prefix_ioctl(struct socket *so, u_long cmd, caddr_t data,
 }
 
 void
-in6_rr_timer(void *ignored_arg)
+in6_rr_timer(__unused void *ignored_arg)
 {
 	struct rr_prefix *rpp;
 	struct timeval timenow;
diff --git a/bsd/netinet6/in6_proto.c b/bsd/netinet6/in6_proto.c
index 0f5281bfb..c27a77892 100644
--- a/bsd/netinet6/in6_proto.c
+++ b/bsd/netinet6/in6_proto.c
@@ -140,9 +140,8 @@ lck_mtx_t *inet6_domain_mutex;
 #define PR_LISTEN	0
 #define PR_ABRTACPTDIS	0
 
-extern struct domain inet6domain;
 extern int in6_inithead(void **, int);
-void in6_dinit(void);
+void in6_dinit(void) __attribute__((section("__TEXT, initcode")));
 
 static int rip6_pr_output(struct mbuf *m, struct socket *so, struct sockaddr_in6 *, struct mbuf *);
 
@@ -153,8 +152,8 @@ struct ip6protosw inet6sw[] = {
   ip6_init,	0,		frag6_slowtimo,	frag6_drain,
   0,
   &nousrreqs,
-  0,		0,		0
-
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_DGRAM,	&inet6domain,	IPPROTO_UDP,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK|PR_PCBLOCK,
   udp6_input,	0,		udp6_ctlinput,	ip6_ctloutput,
@@ -162,8 +161,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0, 
   &udp6_usrreqs,
-  udp_lock,	udp_unlock,	udp_getlock
-
+  udp_lock,	udp_unlock,	udp_getlock,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_STREAM,	&inet6domain,	IPPROTO_TCP,	PR_CONNREQUIRED|PR_WANTRCVD|PR_LISTEN|PR_PROTOLOCK|PR_PCBLOCK|PR_DISPOSE,
   tcp6_input,	0,		tcp6_ctlinput,	tcp_ctloutput,
@@ -171,12 +170,12 @@ struct ip6protosw inet6sw[] = {
 #if INET	/* don't call initialization and timeout routines twice */
   0,		0,		0,		tcp_drain,
 #else
-  tcp_init,	tcp_fasttimo,	tcp_slowtimo,	tcp_drain,
+  tcp_init,	0,	tcp_slowtimo,	tcp_drain,
 #endif
   0,
   &tcp6_usrreqs,
-  tcp_lock,	tcp_unlock,	tcp_getlock
-
+  tcp_lock,	tcp_unlock,	tcp_getlock,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,	&inet6domain,	IPPROTO_RAW,	PR_ATOMIC|PR_ADDR,
   rip6_input,	rip6_pr_output,	rip6_ctlinput,	rip6_ctloutput,
@@ -184,7 +183,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &rip6_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,	&inet6domain,	IPPROTO_ICMPV6,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
   icmp6_input,	rip6_pr_output,	rip6_ctlinput,	rip6_ctloutput,
@@ -192,7 +192,17 @@ struct ip6protosw inet6sw[] = {
   icmp6_init,	icmp6_fasttimo,	0,		0,
   0,
   &rip6_usrreqs,
-  0,		rip_unlock,		0
+  0,		rip_unlock,		0,
+  { 0, 0 }, NULL, { 0 }
+},
+{ SOCK_DGRAM,     &inet6domain,   IPPROTO_ICMPV6, PR_ATOMIC|PR_ADDR|PR_LASTHDR,
+  icmp6_input,  rip6_pr_output, rip6_ctlinput,  icmp6_dgram_ctloutput,
+  0,
+  icmp6_init,   icmp6_fasttimo, 0,              0,
+  0,
+  &icmp6_dgram_usrreqs,
+  0,            rip_unlock,             0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,	&inet6domain,	IPPROTO_DSTOPTS,PR_ATOMIC|PR_ADDR,
   dest6_input,	0,	 	0,		0,
@@ -200,7 +210,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,	&inet6domain,	IPPROTO_ROUTING,PR_ATOMIC|PR_ADDR,
   route6_input,	0,	 	0,		0,
@@ -208,7 +219,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,	&inet6domain,	IPPROTO_FRAGMENT,PR_ATOMIC|PR_ADDR,
   frag6_input,	0,	 	0,		0,
@@ -216,7 +228,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 #if IPSEC
 { SOCK_RAW,	&inet6domain,	IPPROTO_AH,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK,
@@ -225,7 +238,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 #if IPSEC_ESP
 { SOCK_RAW,	&inet6domain,	IPPROTO_ESP,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK,
@@ -236,7 +250,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 #endif
 { SOCK_RAW,	&inet6domain,	IPPROTO_IPCOMP,	PR_ATOMIC|PR_ADDR|PR_PROTOLOCK,
@@ -245,7 +260,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &nousrreqs,
-  0,		0,		0
+  0,		0,		0,
+  { 0, 0 }, NULL, { 0 }
 },
 #endif /* IPSEC */
 #if INET
@@ -255,7 +271,8 @@ struct ip6protosw inet6sw[] = {
   encap_init,	0,		0,		0,
   0,
   &rip6_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0,
+  { 0, 0 }, NULL, { 0 }
 },
 #endif /*INET*/
 { SOCK_RAW,	&inet6domain,	IPPROTO_IPV6,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
@@ -264,7 +281,8 @@ struct ip6protosw inet6sw[] = {
   encap_init,	0,		0,		0,
   0,
   &rip6_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0,
+  { 0, 0 }, NULL, { 0 }
 },
 { SOCK_RAW,     &inet6domain,	IPPROTO_PIM,	PR_ATOMIC|PR_ADDR|PR_LASTHDR,
   pim6_input,	rip6_pr_output,	0,              rip6_ctloutput,
@@ -272,7 +290,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,	
   &rip6_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0,
+  { 0, 0 }, NULL, { 0 }
 },
 /* raw wildcard */
 { SOCK_RAW,	&inet6domain,	0,		PR_ATOMIC|PR_ADDR|PR_LASTHDR,
@@ -281,7 +300,8 @@ struct ip6protosw inet6sw[] = {
   0,		0,		0,		0,
   0,
   &rip6_usrreqs,
-  0,		rip_unlock,	0
+  0,		rip_unlock,	0,
+  { 0, 0 }, NULL, { 0 }
 },
 };
 
@@ -292,19 +312,20 @@ struct domain inet6domain =
     { AF_INET6, "internet6", in6_dinit, 0, 0, 
       (struct protosw *)inet6sw, 0,
       in6_inithead, offsetof(struct sockaddr_in6, sin6_addr) << 3, sizeof(struct sockaddr_in6) ,
-      sizeof(struct sockaddr_in6), 0
+      sizeof(struct sockaddr_in6), 0, 
+      NULL, 0, {0,0}
     };
 
 DOMAIN_SET(inet6);
 
 /* Initialize the PF_INET6 domain, and add in the pre-defined protos */
 void
-in6_dinit()
+in6_dinit(void)
 {
 	register int i; 
 	register struct ip6protosw *pr;
 	register struct domain *dp;
-	static inet6domain_initted = 0;
+	static int inet6domain_initted = 0;
 
 	if (!inet6domain_initted) {
 		dp = &inet6domain; 
@@ -317,7 +338,8 @@ in6_dinit()
 	}
 }
 
-int rip6_pr_output(struct mbuf *m, struct socket *so, struct sockaddr_in6 *sin6, struct mbuf *m1)
+int rip6_pr_output(__unused struct mbuf *m, __unused struct socket *so, 
+					__unused struct sockaddr_in6 *sin6, __unused struct mbuf *m1)
 {
 	panic("rip6_pr_output\n");
 	return 0;
@@ -397,12 +419,12 @@ SYSCTL_NODE(_net,	PF_INET6,	inet6,	CTLFLAG_RW,	0,
 	"Internet6 Family");
 
 /* net.inet6 */
-SYSCTL_NODE(_net_inet6,	IPPROTO_IPV6,	ip6,	CTLFLAG_RW, 0,	"IP6");
-SYSCTL_NODE(_net_inet6,	IPPROTO_ICMPV6,	icmp6,	CTLFLAG_RW, 0,	"ICMP6");
-SYSCTL_NODE(_net_inet6,	IPPROTO_UDP,	udp6,	CTLFLAG_RW, 0,	"UDP6");
-SYSCTL_NODE(_net_inet6,	IPPROTO_TCP,	tcp6,	CTLFLAG_RW, 0,	"TCP6");
+SYSCTL_NODE(_net_inet6,	IPPROTO_IPV6,	ip6,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"IP6");
+SYSCTL_NODE(_net_inet6,	IPPROTO_ICMPV6,	icmp6,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"ICMP6");
+SYSCTL_NODE(_net_inet6,	IPPROTO_UDP,	udp6,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"UDP6");
+SYSCTL_NODE(_net_inet6,	IPPROTO_TCP,	tcp6,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"TCP6");
 #if IPSEC
-SYSCTL_NODE(_net_inet6,	IPPROTO_ESP,	ipsec6,	CTLFLAG_RW, 0,	"IPSEC6");
+SYSCTL_NODE(_net_inet6,	IPPROTO_ESP,	ipsec6,	CTLFLAG_RW|CTLFLAG_LOCKED, 0,	"IPSEC6");
 #endif /* IPSEC */
 
 /* net.inet6.ip6 */
diff --git a/bsd/netinet6/in6_rmx.c b/bsd/netinet6/in6_rmx.c
index 1aa407220..fdaf9143f 100644
--- a/bsd/netinet6/in6_rmx.c
+++ b/bsd/netinet6/in6_rmx.c
@@ -1,3 +1,31 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
 /*	$FreeBSD: src/sys/netinet6/in6_rmx.c,v 1.1.2.2 2001/07/03 11:01:52 ume Exp $	*/
 /*	$KAME: in6_rmx.c,v 1.10 2001/05/24 05:44:58 itojun Exp $	*/
 
@@ -103,7 +131,7 @@
 extern int	in6_inithead(void **head, int off);
 static void	in6_rtqtimo(void *rock);
 static void in6_mtutimo(void *rock);
-extern lck_mtx_t *rt_mtx;
+extern int tvtohz(struct timeval *);
 
 #define RTPRF_OURS		RTF_PROTO3	/* set on routes we manage */
 
@@ -251,11 +279,9 @@ SYSCTL_INT(_net_inet6_ip6, IPV6CTL_RTMAXCACHE, rtmaxcache,
  * timed out.
  */
 static void
-in6_clsroute(struct radix_node *rn, struct radix_node_head *head)
+in6_clsroute(struct radix_node *rn, __unused struct radix_node_head *head)
 {
 	struct rtentry *rt = (struct rtentry *)rn;
-	struct timeval timenow;
-
 
 	if (!(rt->rt_flags & RTF_UP))
 		return;		/* prophylactic measures */
@@ -263,24 +289,32 @@ in6_clsroute(struct radix_node *rn, struct radix_node_head *head)
 	if ((rt->rt_flags & (RTF_LLINFO | RTF_HOST)) != RTF_HOST)
 		return;
 
-	if ((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS))
-	   != RTF_WASCLONED)
+	if ((rt->rt_flags & (RTF_WASCLONED | RTPRF_OURS)) != RTF_WASCLONED)
 		return;
 
 	/*
-	 * As requested by David Greenman:
-	 * If rtq_reallyold is 0, just delete the route without
-	 * waiting for a timeout cycle to kill it.
+	 * Delete the route immediately if RTF_DELCLONE is set or
+	 * if route caching is disabled (rtq_reallyold set to 0).
+	 * Otherwise, let it expire and be deleted by in6_rtqkill().
 	 */
-	getmicrotime(&timenow);
-	if (rtq_reallyold != 0) {
+	if ((rt->rt_flags & RTF_DELCLONE) || rtq_reallyold == 0) {
+		/*
+		 * Delete the route from the radix tree but since we are
+		 * called when the route's reference count is 0, don't
+		 * deallocate it until we return from this routine by
+		 * telling rtrequest that we're interested in it.
+		 */
+		if (rtrequest_locked(RTM_DELETE, (struct sockaddr *)rt_key(rt),
+		    rt->rt_gateway, rt_mask(rt), rt->rt_flags, &rt) == 0) {
+			/* Now let the caller free it */
+			rtunref(rt);
+		}
+	} else {
+		struct timeval timenow;
+
+		getmicrotime(&timenow);
 		rt->rt_flags |= RTPRF_OURS;
 		rt->rt_rmx.rmx_expire = timenow.tv_sec + rtq_reallyold;
-	} else {
-		rtrequest_locked(RTM_DELETE,
-			  (struct sockaddr *)rt_key(rt),
-			  rt->rt_gateway, rt_mask(rt),
-			  rt->rt_flags, 0);
 	}
 }
 
@@ -308,8 +342,8 @@ in6_rtqkill(struct radix_node *rn, void *rock)
 	struct timeval timenow;
 
 	getmicrotime(&timenow);
-
 	lck_mtx_assert(rt_mtx, LCK_MTX_ASSERT_OWNED);
+
 	if (rt->rt_flags & RTPRF_OURS) {
 		ap->found++;
 
@@ -353,13 +387,14 @@ in6_rtqtimo(void *rock)
 	static time_t last_adjusted_timeout = 0;
 	struct timeval timenow;
 
+	lck_mtx_lock(rt_mtx);
+	/* Get the timestamp after we acquire the lock for better accuracy */
 	getmicrotime(&timenow);
 
 	arg.found = arg.killed = 0;
 	arg.rnh = rnh;
 	arg.nextstop = timenow.tv_sec + rtq_timeout;
 	arg.draining = arg.updating = 0;
-	lck_mtx_lock(rt_mtx);
 	rnh->rnh_walktree(rnh, in6_rtqkill, &arg);
 
 	/*
diff --git a/bsd/netinet6/in6_src.c b/bsd/netinet6/in6_src.c
index dd0b5c0b2..6a6782cc5 100644
--- a/bsd/netinet6/in6_src.c
+++ b/bsd/netinet6/in6_src.c
@@ -98,7 +98,6 @@
 #include <net/net_osdep.h>
 
 #include "loop.h"
-extern lck_mtx_t *rt_mtx;
 
 /*
  * Return an IPv6 address, which is the most appropriate for a given
@@ -198,7 +197,7 @@ in6_selectsrc(
 		struct ifnet *ifp = mopts ? mopts->im6o_multicast_ifp : NULL;
 
 		if (ifp == NULL && IN6_IS_ADDR_MC_NODELOCAL(dst)) {
-			ifp = &loif[0];
+			ifp = lo_ifp;
 		}
 
 		if (ifp) {
@@ -262,10 +261,12 @@ in6_selectsrc(
 			sa6->sin6_family = AF_INET6;
 			sa6->sin6_len = sizeof(struct sockaddr_in6);
 			sa6->sin6_addr = *dst;
+#if SCOPEDROUTING
 			sa6->sin6_scope_id = dstsock->sin6_scope_id;
+#endif
 			if (IN6_IS_ADDR_MULTICAST(dst)) {
-				ro->ro_rt = rtalloc1_locked(&((struct route *)ro)
-						     ->ro_dst, 0, 0UL);
+				ro->ro_rt = rtalloc1_locked(
+				    &((struct route *)ro)->ro_dst, 0, 0UL);
 			} else {
 				rtalloc_ign_locked((struct route *)ro, 0UL);
 			}
@@ -348,11 +349,11 @@ in6_selecthlim(
  * share this function by all *bsd*...
  */
 int
-in6_pcbsetport(laddr, inp, p, locked)
-	struct in6_addr *laddr;
-	struct inpcb *inp;
-	struct proc *p;
-	int locked;
+in6_pcbsetport(
+	__unused struct in6_addr *laddr,
+	struct inpcb *inp,
+	struct proc *p,
+	int locked)
 {
 	struct socket *so = inp->inp_socket;
 	u_int16_t lport = 0, first, last, *lastport;
diff --git a/bsd/netinet6/in6_var.h b/bsd/netinet6/in6_var.h
index 0ef5ef924..b6b232f68 100644
--- a/bsd/netinet6/in6_var.h
+++ b/bsd/netinet6/in6_var.h
@@ -456,6 +456,7 @@ void in6_post_msg(struct ifnet *, u_long, struct in6_ifaddr *);
 #define SIOCGETMIFCNT_IN6	_IOWR('u', 107, \
 				      struct sioc_mif_req6) /* get pkt cnt per if */
 
+#ifdef PRIVATE
 /*
  * temporary control calls to attach/detach IP to/from an ethernet interface 
  */
@@ -466,6 +467,7 @@ void in6_post_msg(struct ifnet *, u_long, struct in6_ifaddr *);
 #define SIOCLL_STOP _IOWR('i', 131, struct in6_ifreq)    /* deconfigure linklocal from interface */
 #define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq)    /* accept rtadvd on this interface */
 #define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq)    /* stop accepting rtadv for this interface */
+#endif /* PRIVATE */
 
 #define IN6_IFF_ANYCAST		0x01	/* anycast address */
 #define IN6_IFF_TENTATIVE	0x02	/* tentative address */
@@ -575,15 +577,15 @@ struct	in6_multistep {
 /* struct ifnet *ifp; */					\
 /* struct in6_multi *in6m; */					\
 do { \
-	struct ifmultiaddr *ifma; \
-	for (ifma = (ifp)->if_multiaddrs.lh_first; ifma; \
-	     ifma = ifma->ifma_link.le_next) { \
-		if (ifma->ifma_addr->sa_family == AF_INET6 \
-		    && IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6 *)ifma->ifma_addr)->sin6_addr, \
+	struct ifmultiaddr *_ifma; \
+	for (_ifma = (ifp)->if_multiaddrs.lh_first; _ifma; \
+	     _ifma = _ifma->ifma_link.le_next) { \
+		if (_ifma->ifma_addr->sa_family == AF_INET6 \
+		    && IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6 *)_ifma->ifma_addr)->sin6_addr, \
 					  &(addr))) \
 			break; \
 	} \
-	(in6m) = (struct in6_multi *)(ifma ? ifma->ifma_protospec : 0); \
+	(in6m) = (struct in6_multi *)(_ifma ? _ifma->ifma_protospec : 0); \
 } while(0)
 
 /*
diff --git a/bsd/netinet6/ip6_forward.c b/bsd/netinet6/ip6_forward.c
index 4cb3871ad..9f6f70b5f 100644
--- a/bsd/netinet6/ip6_forward.c
+++ b/bsd/netinet6/ip6_forward.c
@@ -66,16 +66,13 @@
 #endif
 #include <netkey/key.h>
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
-extern lck_mtx_t *ip6_mutex;
 #endif /* IPSEC */
+extern lck_mtx_t *ip6_mutex;
 
 #include <netinet6/ip6_fw.h>
 
 #include <net/net_osdep.h>
 
-struct	route_in6 ip6_forward_rt;
-
 /*
  * Forward a packet.  If some error occurs return the sender
  * an icmp packet.  Note we can't always generate a meaningful
@@ -90,10 +87,8 @@ struct	route_in6 ip6_forward_rt;
  */
 
 void
-ip6_forward(m, srcrt, locked)
-	struct mbuf *m;
-	int srcrt;
-	int locked;
+ip6_forward(struct mbuf *m, struct route_in6 *ip6forward_rt,
+    int srcrt, int locked)
 {
 	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
 	struct sockaddr_in6 *dst;
@@ -105,6 +100,7 @@ ip6_forward(m, srcrt, locked)
 	struct secpolicy *sp = NULL;
 #endif
 	struct timeval timenow;
+	int	tunneledv4 = 0;
 
 	getmicrotime(&timenow);
 
@@ -118,14 +114,11 @@ ip6_forward(m, srcrt, locked)
 	 * before forwarding packet actually.
 	 */
 	if (ipsec_bypass == 0) {
-		lck_mtx_lock(sadb_mutex);
 		if (ipsec6_in_reject(m, NULL)) {
-			ipsec6stat.in_polvio++;
-			lck_mtx_unlock(sadb_mutex);
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 			m_freem(m);
 			return;
 		}
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /*IPSEC*/
 
@@ -180,12 +173,11 @@ ip6_forward(m, srcrt, locked)
 #if IPSEC
 	if (ipsec_bypass != 0)
 		goto skip_ipsec;
-	lck_mtx_lock(sadb_mutex);
 	/* get a security policy for this packet */
 	sp = ipsec6_getpolicybyaddr(m, IPSEC_DIR_OUTBOUND, IP_FORWARDING,
 	    &error);
 	if (sp == NULL) {
-		ipsec6stat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		ip6stat.ip6s_cantforward++;
 		if (mcopy) {
 #if 0
@@ -194,7 +186,6 @@ ip6_forward(m, srcrt, locked)
 			m_freem(mcopy);
 #endif
 		}
-		lck_mtx_unlock(sadb_mutex);
 		m_freem(m);
 		return;
 	}
@@ -204,12 +195,13 @@ ip6_forward(m, srcrt, locked)
 	/* check policy */
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+        case IPSEC_POLICY_GENERATE:
 		/*
 		 * This packet is just discarded.
 		 */
-		ipsec6stat.out_polvio++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_polvio);
 		ip6stat.ip6s_cantforward++;
-		key_freesp(sp);
+		key_freesp(sp, KEY_SADB_UNLOCKED);
 		if (mcopy) {
 #if 0
 			/* XXX: what icmp ? */
@@ -217,15 +209,13 @@ ip6_forward(m, srcrt, locked)
 			m_freem(mcopy);
 #endif
 		}
-		lck_mtx_unlock(sadb_mutex);
 		m_freem(m);
 		return;
 
 	case IPSEC_POLICY_BYPASS:
 	case IPSEC_POLICY_NONE:
 		/* no need to do IPsec. */
-		key_freesp(sp);
-		lck_mtx_unlock(sadb_mutex);
+		key_freesp(sp, KEY_SADB_UNLOCKED);
 		goto skip_ipsec;
 
 	case IPSEC_POLICY_IPSEC:
@@ -233,7 +223,7 @@ ip6_forward(m, srcrt, locked)
 			/* XXX should be panic ? */
 			printf("ip6_forward: No IPsec request specified.\n");
 			ip6stat.ip6s_cantforward++;
-			key_freesp(sp);
+			key_freesp(sp, KEY_SADB_UNLOCKED);
 			if (mcopy) {
 #if 0
 				/* XXX: what icmp ? */
@@ -241,7 +231,6 @@ ip6_forward(m, srcrt, locked)
 				m_freem(mcopy);
 #endif
 			}
-			lck_mtx_unlock(sadb_mutex);
 			m_freem(m);
 			return;
 		}
@@ -252,8 +241,7 @@ ip6_forward(m, srcrt, locked)
 	default:
 		/* should be panic ?? */
 		printf("ip6_forward: Invalid policy found. %d\n", sp->policy);
-		key_freesp(sp);
-		lck_mtx_unlock(sadb_mutex);
+		key_freesp(sp, KEY_SADB_UNLOCKED);
 		goto skip_ipsec;
 	}
 
@@ -274,17 +262,15 @@ ip6_forward(m, srcrt, locked)
 	state.dst = NULL;	/* update at ipsec6_output_tunnel() */
 
 	if (locked)
-			lck_mtx_unlock(ip6_mutex);
-	error = ipsec6_output_tunnel(&state, sp, 0);
-	if (locked) {
-			lck_mtx_unlock(sadb_mutex);
-			lck_mtx_lock(ip6_mutex);
-			lck_mtx_lock(sadb_mutex);
-	}
-
+		lck_mtx_unlock(ip6_mutex);
+	error = ipsec6_output_tunnel(&state, sp, 0, &tunneledv4);
+	if (locked)
+		lck_mtx_lock(ip6_mutex);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
+	if (tunneledv4)
+		return;  /* packet is gone - sent over IPv4 */
+		
 	m = state.m;
-	key_freesp(sp);
-
 	if (error) {
 		/* mbuf is already reclaimed in ipsec6_output_tunnel. */
 		switch (error) {
@@ -309,32 +295,30 @@ ip6_forward(m, srcrt, locked)
 			m_freem(mcopy);
 #endif
 		}
-		lck_mtx_unlock(sadb_mutex);
 		m_freem(m);
 		return;
 	}
     }
-	lck_mtx_unlock(sadb_mutex);
     skip_ipsec:
 #endif /* IPSEC */
 
-	dst = (struct sockaddr_in6 *)&ip6_forward_rt.ro_dst;
+	dst = (struct sockaddr_in6 *)&ip6forward_rt->ro_dst;
 	if (!srcrt) {
 		/*
-		 * ip6_forward_rt.ro_dst.sin6_addr is equal to ip6->ip6_dst
+		 * ip6forward_rt->ro_dst.sin6_addr is equal to ip6->ip6_dst
 		 */
-		if (ip6_forward_rt.ro_rt == 0 ||
-		    (ip6_forward_rt.ro_rt->rt_flags & RTF_UP) == 0) {
-			if (ip6_forward_rt.ro_rt) {
-				rtfree(ip6_forward_rt.ro_rt);
-				ip6_forward_rt.ro_rt = 0;
+		if (ip6forward_rt->ro_rt == 0 ||
+		    (ip6forward_rt->ro_rt->rt_flags & RTF_UP) == 0) {
+			if (ip6forward_rt->ro_rt) {
+				rtfree(ip6forward_rt->ro_rt);
+				ip6forward_rt->ro_rt = 0;
 			}
 			/* this probably fails but give it a try again */
-			rtalloc_ign((struct route *)&ip6_forward_rt,
+			rtalloc_ign((struct route *)ip6forward_rt,
 				    RTF_PRCLONING);
 		}
 
-		if (ip6_forward_rt.ro_rt == 0) {
+		if (ip6forward_rt->ro_rt == 0) {
 			ip6stat.ip6s_noroute++;
 			in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_noroute);
 			if (mcopy) {
@@ -348,19 +332,19 @@ ip6_forward(m, srcrt, locked)
 			m_freem(m);
 			return;
 		}
-	} else if ((rt = ip6_forward_rt.ro_rt) == 0 ||
+	} else if ((rt = ip6forward_rt->ro_rt) == 0 ||
 		 !IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &dst->sin6_addr)) {
-		if (ip6_forward_rt.ro_rt) {
-			rtfree(ip6_forward_rt.ro_rt);
-			ip6_forward_rt.ro_rt = 0;
+		if (ip6forward_rt->ro_rt) {
+			rtfree(ip6forward_rt->ro_rt);
+			ip6forward_rt->ro_rt = 0;
 		}
 		bzero(dst, sizeof(*dst));
 		dst->sin6_len = sizeof(struct sockaddr_in6);
 		dst->sin6_family = AF_INET6;
 		dst->sin6_addr = ip6->ip6_dst;
 
-  		rtalloc_ign((struct route *)&ip6_forward_rt, RTF_PRCLONING);
-		if (ip6_forward_rt.ro_rt == 0) {
+  		rtalloc_ign((struct route *)ip6forward_rt, RTF_PRCLONING);
+		if (ip6forward_rt->ro_rt == 0) {
 			ip6stat.ip6s_noroute++;
 			in6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_noroute);
 			if (mcopy) {
@@ -375,7 +359,7 @@ ip6_forward(m, srcrt, locked)
 			return;
 		}
 	}
-	rt = ip6_forward_rt.ro_rt;
+	rt = ip6forward_rt->ro_rt;
 
 	/*
 	 * Scope check: if a packet can't be delivered to its destination
@@ -417,7 +401,7 @@ ip6_forward(m, srcrt, locked)
 		if (mcopy) {
 			u_long mtu;
 #if IPSEC
-			struct secpolicy *sp;
+			struct secpolicy *sp2;
 			int ipsecerror;
 			size_t ipsechdrsiz;
 #endif
@@ -431,16 +415,15 @@ ip6_forward(m, srcrt, locked)
 			 * case, as we have the outgoing interface for
 			 * encapsulated packet as "rt->rt_ifp".
 			 */
-			lck_mtx_lock(sadb_mutex);
-			sp = ipsec6_getpolicybyaddr(mcopy, IPSEC_DIR_OUTBOUND,
+			sp2 = ipsec6_getpolicybyaddr(mcopy, IPSEC_DIR_OUTBOUND,
 				IP_FORWARDING, &ipsecerror);
-			if (sp) {
+			if (sp2) {
 				ipsechdrsiz = ipsec6_hdrsiz(mcopy,
 					IPSEC_DIR_OUTBOUND, NULL);
 				if (ipsechdrsiz < mtu)
 					mtu -= ipsechdrsiz;
+				key_freesp(sp2, KEY_SADB_UNLOCKED);
 			}
-			lck_mtx_unlock(sadb_mutex);
 			/*
 			 * if mtu becomes less than minimum MTU,
 			 * tell minimum MTU (and I'll need to fragment it).
diff --git a/bsd/netinet6/ip6_fw.c b/bsd/netinet6/ip6_fw.c
index db7926f14..7d5aa9b72 100644
--- a/bsd/netinet6/ip6_fw.c
+++ b/bsd/netinet6/ip6_fw.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*	$FreeBSD: src/sys/netinet6/ip6_fw.c,v 1.2.2.9 2002/04/28 05:40:27 suz Exp $	*/
@@ -93,6 +99,8 @@
 #include <sys/syslog.h>
 #include <sys/lock.h>
 #include <sys/time.h>
+#include <sys/kern_event.h>
+
 #include <net/if.h>
 #include <net/route.h>
 #include <netinet/in_systm.h>
@@ -120,7 +128,7 @@
 
 MALLOC_DEFINE(M_IP6FW, "Ip6Fw/Ip6Acct", "Ip6Fw/Ip6Acct chain's");
 
-static int fw6_debug = 1;
+static int fw6_debug = 0;
 #ifdef IPV6FIREWALL_VERBOSE
 static int fw6_verbose = 1;
 #else
@@ -134,14 +142,35 @@ static int fw6_verbose_limit = 0;
 
 LIST_HEAD (ip6_fw_head, ip6_fw_chain) ip6_fw_chain;
 
+static void ip6fw_kev_post_msg(u_int32_t );
+
 #ifdef SYSCTL_NODE
+static int ip6fw_sysctl SYSCTL_HANDLER_ARGS;
+
 SYSCTL_DECL(_net_inet6_ip6);
-SYSCTL_NODE(_net_inet6_ip6, OID_AUTO, fw, CTLFLAG_RW, 0, "Firewall");
-SYSCTL_INT(_net_inet6_ip6_fw, OID_AUTO, enable, CTLFLAG_RW,
-	&ip6_fw_enable, 0, "Enable ip6fw");
+SYSCTL_NODE(_net_inet6_ip6, OID_AUTO, fw, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Firewall");
+SYSCTL_PROC(_net_inet6_ip6_fw, OID_AUTO, enable, 
+	CTLTYPE_INT | CTLFLAG_RW,
+	&ip6_fw_enable, 0, ip6fw_sysctl, "I", "Enable ip6fw");
 SYSCTL_INT(_net_inet6_ip6_fw, OID_AUTO, debug, CTLFLAG_RW, &fw6_debug, 0, "");
 SYSCTL_INT(_net_inet6_ip6_fw, OID_AUTO, verbose, CTLFLAG_RW, &fw6_verbose, 0, "");
 SYSCTL_INT(_net_inet6_ip6_fw, OID_AUTO, verbose_limit, CTLFLAG_RW, &fw6_verbose_limit, 0, "");
+
+static int
+ip6fw_sysctl SYSCTL_HANDLER_ARGS
+{
+#pragma unused(arg1, arg2)
+	int error;
+	
+	error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
+	if (error || !req->newptr)
+		return (error);
+	
+	ip6fw_kev_post_msg(KEV_IP6FW_ENABLE);
+	
+	return error;
+}
+
 #endif
 
 #define dprintf(a)	do {						\
@@ -386,7 +415,7 @@ ip6fw_report(struct ip6_fw *f, struct ip6_hdr *ip6,
 	struct udphdr *const udp = (struct udphdr *) ((caddr_t) ip6+ off);
 	struct icmp6_hdr *const icmp6 = (struct icmp6_hdr *) ((caddr_t) ip6+ off);
 	int count;
-	char *action;
+	const char *action;
 	char action2[32], proto[102], name[18];
 	int len;
 
@@ -837,7 +866,7 @@ got_match:
 			}
 			bcopy(&ti, ip6, sizeof(ti));
 			tcp_respond(NULL, ip6, (struct tcphdr *)(ip6 + 1),
-				*m, ack, seq, flags);
+				*m, ack, seq, flags, NULL);
 			*m = NULL;
 			break;
 		  }
@@ -1226,6 +1255,23 @@ ip6_fw_ctl(int stage, struct mbuf **mm)
 }
 #endif
 
+static void
+ip6fw_kev_post_msg(u_int32_t event_code)
+{
+	struct kev_msg		ev_msg;
+
+	bzero(&ev_msg, sizeof(struct kev_msg));
+	
+	ev_msg.vendor_code = KEV_VENDOR_APPLE;
+	ev_msg.kev_class = KEV_FIREWALL_CLASS;
+	ev_msg.kev_subclass = KEV_IP6FW_SUBCLASS;
+	ev_msg.event_code = event_code;
+
+	kev_post_msg(&ev_msg);
+
+}
+
+
 static int
 ip6_fw_ctl(struct sockopt *sopt)
 {
@@ -1246,7 +1292,7 @@ ip6_fw_ctl(struct sockopt *sopt)
 
 	/* save sopt->sopt_valsize */
 	valsize = sopt->sopt_valsize;
-	if (error = sooptcopyin(sopt, &rule, sizeof(rule), sizeof(rule)))
+	if ((error = sooptcopyin(sopt, &rule, sizeof(rule), sizeof(rule))))
 		return error;
 
 	if (rule.version != IPV6_FW_CURRENT_API_VERSION) return EINVAL;
@@ -1299,6 +1345,7 @@ ip6_fw_ctl(struct sockopt *sopt)
 				FREE(fcp, M_IP6FW);
 			}
 			splx(spl);
+			ip6fw_kev_post_msg(KEV_IP6FW_FLUSH);
 			break;
 
 		case IPV6_FW_ZERO:
@@ -1306,9 +1353,11 @@ ip6_fw_ctl(struct sockopt *sopt)
 			break;
 
 		case IPV6_FW_ADD:
-			if (check_ip6fw_struct(&rule))
+			if (check_ip6fw_struct(&rule)) {
 				error = add_entry6(&ip6_fw_chain, &rule);
-			else
+
+				ip6fw_kev_post_msg(KEV_IP6FW_ADD);
+			} else
 				error = EINVAL;
 			break;
 
@@ -1318,8 +1367,11 @@ ip6_fw_ctl(struct sockopt *sopt)
 				dprintf(("%s can't delete rule 65535\n", err_prefix));
 				error = EINVAL;
 			}
-			else
+			else {
 				error = del_entry6(&ip6_fw_chain, rule.fw_number);
+
+				ip6fw_kev_post_msg(KEV_IP6FW_DEL);
+			}
 			break;
 
 		default:
@@ -1350,7 +1402,7 @@ ip6_fw_init(void)
 	default_rule.fw_flg |= IPV6_FW_F_IN | IPV6_FW_F_OUT;
 	if (check_ip6fw_struct(&default_rule) == NULL ||
 		add_entry6(&ip6_fw_chain, &default_rule))
-		panic(__FUNCTION__);
+		panic("%s", __FUNCTION__);
 
 	printf("IPv6 packet filtering initialized, ");
 #ifdef IPV6FIREWALL_DEFAULT_TO_ACCEPT
diff --git a/bsd/netinet6/ip6_fw.h b/bsd/netinet6/ip6_fw.h
index cfb2c4ab3..1d996fef1 100644
--- a/bsd/netinet6/ip6_fw.h
+++ b/bsd/netinet6/ip6_fw.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1993 Daniel Boulet
@@ -36,6 +42,45 @@
 
 #ifndef _IP6_FW_H
 #define _IP6_FW_H
+
+/*
+ * Define IPv6 Firewall event subclass, and associated events.
+ */
+
+/*!
+	@defined KEV_IP6FW_SUBCLASS
+	@discussion The kernel event subclass for IPv6 Firewall.
+*/
+#define KEV_IP6FW_SUBCLASS	2
+
+/*!
+	@defined KEV_IP6FW_ADD
+	@discussion The event code indicating a rule has been added.
+*/
+#define KEV_IP6FW_ADD		1
+
+/*!
+	@defined KEV_IP6FW_DEL
+	@discussion The event code indicating a rule has been removed.
+*/
+#define KEV_IP6FW_DEL		2
+
+/*!
+	@defined KEV_IP6FW_FLUSH
+	@discussion The event code indicating the rule set has been flushed.
+*/
+#define KEV_IP6FW_FLUSH		3
+
+/*!
+	@defined KEV_IP6FW_FLUSH
+	@discussion The event code indicating the enable flag has been changed 
+*/
+#define KEV_IP6FW_ENABLE	4
+
+
+
+#if !__LP64__
+
 #include <sys/appleapiopts.h>
 
 #include <net/if.h>
@@ -229,4 +274,5 @@ extern	int ip6_fw_enable;
 
 #endif /* KERNEL_PRIVATE */
 
+#endif /* !__LP64__ */
 #endif /* _IP6_FW_H */
diff --git a/bsd/netinet6/ip6_input.c b/bsd/netinet6/ip6_input.c
index 64221cf42..1c08cc09f 100644
--- a/bsd/netinet6/ip6_input.c
+++ b/bsd/netinet6/ip6_input.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*	$FreeBSD: src/sys/netinet6/ip6_input.c,v 1.11.2.10 2001/07/24 19:10:18 brooks Exp $	*/
 /*	$KAME: ip6_input.c,v 1.194 2001/05/27 13:28:35 itojun Exp $	*/
 
@@ -86,6 +113,7 @@
 #include <net/if_types.h>
 #include <net/if_dl.h>
 #include <net/route.h>
+#include <net/kpi_protocol.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
@@ -108,7 +136,6 @@
 #include <netinet6/ipsec6.h>
 #endif
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif
 
 #include <netinet6/ip6_fw.h>
@@ -160,7 +187,7 @@ extern lck_mtx_t	*inet6_domain_mutex;
 extern int loopattach_done;
 
 static void ip6_init2(void *);
-static struct mbuf *ip6_setdstifaddr(struct mbuf *, struct in6_ifaddr *);
+static struct ip6aux *ip6_setdstifaddr(struct mbuf *, struct in6_ifaddr *);
 
 static int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *);
 #if PULLDOWN_TEST
@@ -173,9 +200,12 @@ void faithattach(void);
 void stfattach(void);
 #endif
 
+extern lck_mtx_t *domain_proto_mtx;
+
+
 static void
 ip6_proto_input(
-	protocol_family_t	protocol,
+	__unused protocol_family_t	protocol,
 	mbuf_t				packet)
 {
 	ip6_input(packet);
@@ -191,7 +221,6 @@ ip6_init()
 	struct ip6protosw *pr;
 	int i;
 	struct timeval tv;
-	extern lck_mtx_t *domain_proto_mtx;
 
 #if DIAGNOSTIC
 	if (sizeof(struct protosw) != sizeof(struct ip6protosw))
@@ -250,13 +279,13 @@ ip6_init()
 	timeout(ip6_init2, (caddr_t)0, 1 * hz);
 
 	lck_mtx_unlock(domain_proto_mtx);	
-	proto_register_input(PF_INET6, ip6_proto_input, NULL);
+	proto_register_input(PF_INET6, ip6_proto_input, NULL, 0);
 	lck_mtx_lock(domain_proto_mtx);	
 }
 
 static void
-ip6_init2(dummy)
-	void *dummy;
+ip6_init2(
+	__unused void *dummy)
 {
 	/*
 	 * to route local address of p2p link to loopback,
@@ -266,7 +295,7 @@ ip6_init2(dummy)
 		timeout(ip6_init2, (caddr_t)0, 1 * hz);
 		return;
 	}
-	in6_ifattach(&loif[0], NULL, NULL);
+	in6_ifattach(lo_ifp, NULL, NULL);
 
 #ifdef __APPLE__
 	/* nd6_timer_init */
@@ -315,7 +344,12 @@ ip6_init2(dummy)
 SYSINIT(netinet6init2, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ip6_init2, NULL);
 #endif
 
-extern struct	route_in6 ip6_forward_rt;
+/*
+ * ip6_forward_rt contains the route entry that was recently used during
+ * the forwarding of an IPv6 packet and thus acts as a route cache.  Access
+ * to this variable is protected by the global lock ip6_mutex.
+ */
+static struct route_in6 ip6_forward_rt;
 
 void
 ip6_input(m)
@@ -372,7 +406,7 @@ ip6_input(m)
 #define M2MMAX	(sizeof(ip6stat.ip6s_m2m)/sizeof(ip6stat.ip6s_m2m[0]))
 		if (m->m_next) {
 			if (m->m_flags & M_LOOP) {
-				ip6stat.ip6s_m2m[loif[0].if_index]++;	/* XXX */
+				ip6stat.ip6s_m2m[ifnet_index(lo_ifp)]++;	/* XXX */
 			} else if (m->m_pkthdr.rcvif->if_index < M2MMAX)
 				ip6stat.ip6s_m2m[m->m_pkthdr.rcvif->if_index]++;
 			else
@@ -394,7 +428,7 @@ ip6_input(m)
 	if (m && m->m_next != NULL && m->m_pkthdr.len < MCLBYTES) {
 		struct mbuf *n;
 
-		MGETHDR(n, M_DONTWAIT, MT_HEADER);
+		MGETHDR(n, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (n)
 			M_COPY_PKTHDR(n, m);
 		if (n && m->m_pkthdr.len > MHLEN) {
@@ -587,17 +621,10 @@ ip6_input(m)
 	/*
 	 *  Unicast check
 	 */
-	switch (ip6_ours_check_algorithm) {
-	default:
-		/*
-		 * XXX: I intentionally broke our indentation rule here,
-		 *      since this switch-case is just for measurement and
-		 *      therefore should soon be removed.
-		 */
 	if (ip6_forward_rt.ro_rt != NULL &&
 	    (ip6_forward_rt.ro_rt->rt_flags & RTF_UP) != 0 && 
 	    IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst,
-			       &((struct sockaddr_in6 *)(&ip6_forward_rt.ro_dst))->sin6_addr))
+	    &((struct sockaddr_in6 *)(&ip6_forward_rt.ro_dst))->sin6_addr))
 		ip6stat.ip6s_forward_cachehit++;
 	else {
 		struct sockaddr_in6 *dst6;
@@ -695,7 +722,6 @@ ip6_input(m)
 			goto bad;
 		}
 	}
-	} /* XXX indentation (see above) */
 
 	/*
 	 * FAITH(Firewall Aided Internet Translator)
@@ -850,7 +876,7 @@ ip6_input(m)
 			return;
 		}
 	} else if (!ours) {
-		ip6_forward(m, 0, 1);
+		ip6_forward(m, &ip6_forward_rt, 0, 1);
 		lck_mtx_unlock(ip6_mutex);
 		return;
 	}	
@@ -924,20 +950,17 @@ injectit:
 		 * code - like udp/tcp/raw ip.
 		 */
 		if ((ipsec_bypass == 0) && (ip6_protox[nxt]->pr_flags & PR_LASTHDR) != 0) {
-			lck_mtx_lock(sadb_mutex);
 			if (ipsec6_in_reject(m, NULL)) {
-				ipsec6stat.in_polvio++;
-				lck_mtx_unlock(sadb_mutex);
+				IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 				goto badunlocked;
-		        }
-			lck_mtx_unlock(sadb_mutex);
+		    }
 		}
 #endif
 
 		/*
-		 * Call IP filter on last header only
+		 * Call IP filter
 		 */
-		if ((ip6_protox[nxt]->pr_flags & PR_LASTHDR) != 0 && !TAILQ_EMPTY(&ipv6_filters)) {
+		if (!TAILQ_EMPTY(&ipv6_filters)) {
 			ipf_ref();
 			TAILQ_FOREACH(filter, &ipv6_filters, ipf_link) {
 				if (seen == 0) {
@@ -981,28 +1004,26 @@ injectit:
  * set/grab in6_ifaddr correspond to IPv6 destination address.
  * XXX backward compatibility wrapper
  */
-static struct mbuf *
-ip6_setdstifaddr(m, ia6)
-	struct mbuf *m;
-	struct in6_ifaddr *ia6;
+static struct ip6aux *
+ip6_setdstifaddr(struct mbuf *m, struct in6_ifaddr *ia6)
 {
-	struct mbuf *n;
+	struct ip6aux *n;
 
 	n = ip6_addaux(m);
 	if (n)
-		mtod(n, struct ip6aux *)->ip6a_dstia6 = ia6;
-	return n;	/* NULL if failed to set */
+		n->ip6a_dstia6 = ia6;
+	return (struct ip6aux *)n;	/* NULL if failed to set */
 }
 
 struct in6_ifaddr *
 ip6_getdstifaddr(m)
 	struct mbuf *m;
 {
-	struct mbuf *n;
+	struct ip6aux *n;
 
 	n = ip6_findaux(m);
 	if (n)
-		return mtod(n, struct ip6aux *)->ip6a_dstia6;
+		return n->ip6a_dstia6;
 	else
 		return NULL;
 }
@@ -1294,6 +1315,10 @@ ip6_savecontrol(in6p, mp, ip6, m)
 	}
 #endif
 
+	/* some OSes call this logic with IPv4 packet, for SO_TIMESTAMP */
+	if ((ip6->ip6_vfc & IPV6_VERSION_MASK) != IPV6_VERSION)
+		return;
+
 	/* RFC 2292 sec. 5 */
 	if ((in6p->in6p_flags & IN6P_PKTINFO) != 0) {
 		struct in6_pktinfo pi6;
@@ -1598,7 +1623,7 @@ ip6_get_prevhdr(m, off)
 	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
 
 	if (off == sizeof(struct ip6_hdr))
-		return(&ip6->ip6_nxt);
+		return((char *) &ip6->ip6_nxt);
 	else {
 		int len, nxt;
 		struct ip6_ext *ip6e = NULL;
@@ -1622,7 +1647,7 @@ ip6_get_prevhdr(m, off)
 			nxt = ip6e->ip6e_nxt;
 		}
 		if (ip6e)
-			return(&ip6e->ip6e_nxt);
+			return((char *) &ip6e->ip6e_nxt);
 		else
 			return NULL;
 	}
@@ -1738,55 +1763,49 @@ ip6_lasthdr(m, off, proto, nxtp)
 	}
 }
 
-struct mbuf *
-ip6_addaux(m)
-	struct mbuf *m;
+struct ip6aux *
+ip6_addaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-#if DIAGNOSTIC
-	if (sizeof(struct ip6aux) > MHLEN)
-		panic("assumption failed on sizeof(ip6aux)");
-#endif
-	n = m_aux_find(m, AF_INET6, -1);
-	if (n) {
-		if (n->m_len < sizeof(struct ip6aux)) {
-			printf("conflicting use of ip6aux");
-			return NULL;
-		}
-	} else {
-		n = m_aux_add(m, AF_INET6, -1);
-		if (n) {
-			n->m_len = sizeof(struct ip6aux);
-			bzero(mtod(n, caddr_t), n->m_len);
+	struct m_tag		*tag;
+	
+	/* Check if one is already allocated */
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_INET6, NULL);
+	if (tag == NULL) {
+		/* Allocate a tag */
+		tag = m_tag_alloc(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_INET6,
+						  sizeof(*tag), M_DONTWAIT);
+		
+		/* Attach it to the mbuf */
+		if (tag) {
+			m_tag_prepend(m, tag);
 		}
 	}
-	return n;
+	
+	return tag ? (struct ip6aux*)(tag + 1) : NULL;
 }
 
-struct mbuf *
-ip6_findaux(m)
-	struct mbuf *m;
+struct ip6aux *
+ip6_findaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = m_aux_find(m, AF_INET6, -1);
-	if (n && n->m_len < sizeof(struct ip6aux)) {
-		printf("conflicting use of ip6aux");
-		n = NULL;
-	}
-	return n;
+	struct m_tag	*tag;
+	
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_ENCAP, NULL);
+	
+	return tag ? (struct ip6aux*)(tag + 1) : NULL;
 }
 
 void
-ip6_delaux(m)
-	struct mbuf *m;
+ip6_delaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
+	struct m_tag	*tag;
 
-	n = m_aux_find(m, AF_INET6, -1);
-	if (n)
-		m_aux_delete(m, n);
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_ENCAP, NULL);
+	if (tag) {
+		m_tag_delete(m, tag);
+	}
 }
 
 /*
diff --git a/bsd/netinet6/ip6_mroute.c b/bsd/netinet6/ip6_mroute.c
index 84e1a08ad..e3e214ed6 100644
--- a/bsd/netinet6/ip6_mroute.c
+++ b/bsd/netinet6/ip6_mroute.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*	$FreeBSD: src/sys/netinet6/ip6_mroute.c,v 1.16.2.1 2002/12/18 21:39:40 suz Exp $	*/
 /*	$KAME: ip6_mroute.c,v 1.58 2001/12/18 02:36:31 itojun Exp $	*/
 
@@ -29,6 +56,12 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*	BSDI ip_mroute.c,v 2.10 1996/11/14 00:29:52 jch Exp	*/
 
@@ -63,6 +96,8 @@
 #include <net/if.h>
 #include <net/route.h>
 #include <net/raw_cb.h>
+#include <net/dlil.h>
+#include <net/net_osdep.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -73,6 +108,10 @@
 #include <netinet6/pim6.h>
 #include <netinet6/pim6_var.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac.h>
+#endif /* MAC_NET */
+
 #ifndef __APPLE__
 static MALLOC_DEFINE(M_MRTABLE, "mf6c", "multicast forwarding cache entry");
 #endif
@@ -444,8 +483,6 @@ ip6_mrouter_done()
 {
 	mifi_t mifi;
 	int i;
-	struct ifnet *ifp;
-	struct in6_ifreq ifr;
 	struct mf6c *rt;
 	struct rtdetq *rte;
 
@@ -469,15 +506,18 @@ ip6_mrouter_done()
 		for (mifi = 0; mifi < nummifs; mifi++) {
 			if (mif6table[mifi].m6_ifp &&
 			    !(mif6table[mifi].m6_flags & MIFF_REGISTER)) {
-				ifr.ifr_addr.sin6_family = AF_INET6;
-				ifr.ifr_addr.sin6_addr= in6addr_any;
-				ifp = mif6table[mifi].m6_ifp;
 #ifdef __APPLE__
-				dlil_ioctl(0, ifp, SIOCDELMULTI,
-						 (caddr_t)&ifr);
+				if_allmulti(mif6table[mifi].m6_ifp, 0);
 #else
-				(*ifp->if_ioctl)(ifp, SIOCDELMULTI,
-						 (caddr_t)&ifr);
+				{
+					struct ifnet *ifp;
+					struct in6_ifreq ifr;
+			
+					ifr.ifr_addr.sin6_family = AF_INET6;
+					ifr.ifr_addr.sin6_addr= in6addr_any;
+					ifp = mif6table[mifi].m6_ifp;
+					ifnet_ioctl(ifp, 0, SIOCDELMULTI, &ifr);
+				}
 #endif
 			}
 		}
@@ -538,7 +578,8 @@ ip6_mrouter_done()
 	return 0;
 }
 
-static struct sockaddr_in6 sin6 = { sizeof(sin6), AF_INET6 };
+static struct sockaddr_in6 sin6 = { sizeof(sin6), AF_INET6 ,
+                                     0, 0, IN6ADDR_ANY_INIT, 0};
 
 /*
  * Add a mif to the mif table
@@ -670,7 +711,6 @@ add_m6fc(mfccp)
 	u_long hash;
 	struct rtdetq *rte;
 	u_short nstl;
-	int s;
 
 	MF6CFIND(mfccp->mf6cc_origin.sin6_addr,
 		 mfccp->mf6cc_mcastgrp.sin6_addr, rt);
@@ -786,7 +826,6 @@ add_m6fc(mfccp)
 			rt = (struct mf6c *)_MALLOC(sizeof(*rt), M_MRTABLE,
 						  M_NOWAIT);
 			if (rt == NULL) {
-				splx(s);
 				return ENOBUFS;
 			}
 	
@@ -923,7 +962,6 @@ ip6_mforward(ip6, ifp, m)
 	struct mf6c *rt;
 	struct mif6 *mifp;
 	struct mbuf *mm;
-	int s;
 	mifi_t mifi;
 	struct timeval timenow;
 
@@ -969,12 +1007,10 @@ ip6_mforward(ip6, ifp, m)
 	/*
 	 * Determine forwarding mifs from the forwarding cache table
 	 */
-	s = splnet();
 	MF6CFIND(ip6->ip6_src, ip6->ip6_dst, rt);
 
 	/* Entry exists, so forward if necessary */
 	if (rt) {
-		splx(s);
 		return (ip6_mdq(m, ifp, rt));
 	} else {
 		/*
@@ -1008,7 +1044,6 @@ ip6_mforward(ip6, ifp, m)
 		rte = (struct rtdetq *)_MALLOC(sizeof(*rte), M_MRTABLE,
 					      M_NOWAIT);
 		if (rte == NULL) {
-			splx(s);
 			return ENOBUFS;
 		}
 		mb0 = m_copy(m, 0, M_COPYALL);
@@ -1021,7 +1056,6 @@ ip6_mforward(ip6, ifp, m)
 			mb0 = m_pullup(mb0, sizeof(struct ip6_hdr));
 		if (mb0 == NULL) {
 			FREE(rte, M_MRTABLE);
-			splx(s);
 			return ENOBUFS;
 		}
 	
@@ -1048,7 +1082,6 @@ ip6_mforward(ip6, ifp, m)
 			if (rt == NULL) {
 				FREE(rte, M_MRTABLE);
 				m_freem(mb0);
-				splx(s);
 				return ENOBUFS;
 			}
 			/*
@@ -1061,7 +1094,6 @@ ip6_mforward(ip6, ifp, m)
 				FREE(rte, M_MRTABLE);
 				m_freem(mb0);
 				FREE(rt, M_MRTABLE);
-				splx(s);
 				return ENOBUFS;
 			}
 
@@ -1091,7 +1123,6 @@ ip6_mforward(ip6, ifp, m)
 				FREE(rte, M_MRTABLE);
 				m_freem(mb0);
 				FREE(rt, M_MRTABLE);
-				splx(s);
 				return EINVAL;
 			}
 
@@ -1124,7 +1155,6 @@ ip6_mforward(ip6, ifp, m)
 				FREE(rte, M_MRTABLE);
 				m_freem(mb0);
 				FREE(rt, M_MRTABLE);
-				splx(s);
 				return ENOBUFS;
 			}
 
@@ -1157,7 +1187,6 @@ ip6_mforward(ip6, ifp, m)
 					mrt6stat.mrt6s_upq_ovflw++;
 					FREE(rte, M_MRTABLE);
 					m_freem(mb0);
-					splx(s);
 					return 0;
 				}
 
@@ -1172,7 +1201,6 @@ ip6_mforward(ip6, ifp, m)
 		rte->t = tp;
 #endif /* UPCALL_TIMING */
 
-		splx(s);
 
 		return 0;
 	}
@@ -1183,15 +1211,13 @@ ip6_mforward(ip6, ifp, m)
  * Call from the Slow Timeout mechanism, every half second.
  */
 static void
-expire_upcalls(unused)
-	void *unused;
+expire_upcalls(
+	__unused void *unused)
 {
 	struct rtdetq *rte;
 	struct mf6c *mfc, **nptr;
 	int i;
-	int s;
 
-	s = splnet();
 	for (i = 0; i < MF6CTBLSIZ; i++) {
 		if (n6expire[i] == 0)
 			continue;
@@ -1232,7 +1258,6 @@ expire_upcalls(unused)
 			}
 		}
 	}
-	splx(s);
 
 #ifndef __APPLE__
 	callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT,
@@ -1298,8 +1323,8 @@ ip6_mdq(m, ifp, rt)
 				 * unnecessary PIM assert.
 				 * XXX: M_LOOP is an ad-hoc hack...
 				 */
-				static struct sockaddr_in6 sin6 =
-				{ sizeof(sin6), AF_INET6 };
+				static struct sockaddr_in6 addr =
+				{ sizeof(addr), AF_INET6 , 0, 0, IN6ADDR_ANY_INIT, 0};
 
 				struct mbuf *mm;
 				struct mrt6msg *im;
@@ -1347,18 +1372,18 @@ ip6_mdq(m, ifp, rt)
 #if MRT6_OINIT
 				case MRT6_OINIT:
 					oim->im6_mif = iif;
-					sin6.sin6_addr = oim->im6_src;
+					addr.sin6_addr = oim->im6_src;
 					break;
 #endif
 				case MRT6_INIT:
 					im->im6_mif = iif;
-					sin6.sin6_addr = im->im6_src;
+					addr.sin6_addr = im->im6_src;
 					break;
 				}
 
 				mrt6stat.mrt6s_upcalls++;
 
-				if (socket_send(ip6_mrouter, mm, &sin6) < 0) {
+				if (socket_send(ip6_mrouter, mm, &addr) < 0) {
 #if MRT6DEBUG
 					if (mrt6debug)
 						log(LOG_WARNING, "mdq, ip6_mrouter socket queue full\n");
@@ -1423,7 +1448,6 @@ phyint_send(ip6, mifp, m)
 	struct mbuf *mb_copy;
 	struct ifnet *ifp = mifp->m6_ifp;
 	int error = 0;
-	int s = splnet();	/* needs to protect static "ro" below. */
 	static struct route_in6 ro;
 	struct	in6_multi *in6m;
 	struct sockaddr_in6 *dst6;
@@ -1438,7 +1462,6 @@ phyint_send(ip6, mifp, m)
 	    (M_HASCL(mb_copy) || mb_copy->m_len < sizeof(struct ip6_hdr)))
 		mb_copy = m_pullup(mb_copy, sizeof(struct ip6_hdr));
 	if (mb_copy == NULL) {
-		splx(s);
 		return;
 	}
 	/* set MCAST flag to the outgoing packet */
@@ -1467,7 +1490,6 @@ phyint_send(ip6, mifp, m)
 			log(LOG_DEBUG, "phyint_send on mif %d err %d\n",
 			    mifp - mif6table, error);
 #endif
-		splx(s);
 		return;
 	}
 
@@ -1533,8 +1555,6 @@ phyint_send(ip6, mifp, m)
 		m_freem(mb_copy); /* simply discard the packet */
 #endif
 	}
-
-	splx(s);
 }
 
 static int
@@ -1545,7 +1565,8 @@ register_send(ip6, mif, m)
 {
 	struct mbuf *mm;
 	int i, len = m->m_pkthdr.len;
-	static struct sockaddr_in6 sin6 = { sizeof(sin6), AF_INET6 };
+	static struct sockaddr_in6 addr = { sizeof(addr), AF_INET6 ,
+                                     0, 0, IN6ADDR_ANY_INIT, 0};
 	struct mrt6msg *im6;
 
 #if MRT6DEBUG
@@ -1559,6 +1580,11 @@ register_send(ip6, mif, m)
 	MGETHDR(mm, M_DONTWAIT, MT_HEADER);
 	if (mm == NULL)
 		return ENOBUFS;
+#ifdef __darwin8_notyet
+#if CONFIG_MACF_NET
+	mac_create_mbuf_multicast_encap(m, mif->m6_ifp, mm);
+#endif
+#endif
 	mm->m_pkthdr.rcvif = NULL;
 	mm->m_data += max_linkhdr;
 	mm->m_len = sizeof(struct ip6_hdr);
@@ -1581,7 +1607,7 @@ register_send(ip6, mif, m)
 	/*
 	 * Send message to routing daemon
 	 */
-	sin6.sin6_addr = ip6->ip6_src;
+	addr.sin6_addr = ip6->ip6_src;
 
 	im6 = mtod(mm, struct mrt6msg *);
 	im6->im6_msgtype      = MRT6MSG_WHOLEPKT;
@@ -1592,7 +1618,7 @@ register_send(ip6, mif, m)
 	/* iif info is not given for reg. encap.n */
 	mrt6stat.mrt6s_upcalls++;
 
-	if (socket_send(ip6_mrouter, mm, &sin6) < 0) {
+	if (socket_send(ip6_mrouter, mm, &addr) < 0) {
 #if MRT6DEBUG
 		if (mrt6debug)
 			log(LOG_WARNING,
@@ -1717,12 +1743,12 @@ pim6_input(mp, offp)
 		 * headers ip6+pim+u_int32_t+encap_ip6, to be passed up to the
 		 * routing daemon.
 		 */
-		static struct sockaddr_in6 dst = { sizeof(dst), AF_INET6 };
+		static struct sockaddr_in6 dst = { sizeof(dst), AF_INET6 , 
+											0, 0, IN6ADDR_ANY_INIT, 0 };
 
 		struct mbuf *mcp;
 		struct ip6_hdr *eip6;
 		u_int32_t *reghdr;
-		int rc;
 	
 		++pim6stat.pim6s_rcv_registers;
 
@@ -1836,7 +1862,7 @@ pim6_input(mp, offp)
                     m_freem(m);
                 }
 #else
- 		rc = if_simloop(mif6table[reg_mif_num].m6_ifp, m,
+ 		(void) if_simloop(mif6table[reg_mif_num].m6_ifp, m,
 				dst.sin6_family, NULL);
 #endif
 	
diff --git a/bsd/netinet6/ip6_output.c b/bsd/netinet6/ip6_output.c
index 5693fbc17..9e41d205f 100644
--- a/bsd/netinet6/ip6_output.c
+++ b/bsd/netinet6/ip6_output.c
@@ -64,7 +64,12 @@
  *
  *	@(#)ip_output.c	8.3 (Berkeley) 1/21/94
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/malloc.h>
@@ -80,6 +85,7 @@
 
 #include <net/if.h>
 #include <net/route.h>
+#include <net/dlil.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -98,9 +104,12 @@
 #endif
 #include <netkey/key.h>
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
-extern lck_mtx_t *nd6_mutex;
 #endif /* IPSEC */
+extern lck_mtx_t *nd6_mutex;
+
+#if CONFIG_MACF_NET
+#include <security/mac.h>
+#endif /* MAC_NET */
 
 #include <netinet6/ip6_fw.h>
 
@@ -222,7 +231,6 @@ ip6_output(
 	if (ipsec_bypass != 0)
 		goto skip_ipsec;
 	
-	lck_mtx_lock(sadb_mutex);
 	/* get a security policy for this packet */
 	if (so == NULL)
 		sp = ipsec6_getpolicybyaddr(m, IPSEC_DIR_OUTBOUND, 0, &error);
@@ -230,8 +238,7 @@ ip6_output(
 		sp = ipsec6_getpolicybysock(m, IPSEC_DIR_OUTBOUND, so, &error);
 
 	if (sp == NULL) {
-		ipsec6stat.out_inval++;
-		lck_mtx_unlock(sadb_mutex);
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		goto freehdrs;
 	}
 
@@ -240,11 +247,11 @@ ip6_output(
 	/* check policy */
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+	case IPSEC_POLICY_GENERATE:
 		/*
 		 * This packet is just discarded.
 		 */
-		ipsec6stat.out_polvio++;
-		lck_mtx_unlock(sadb_mutex);
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_polvio);
 		goto freehdrs;
 
 	case IPSEC_POLICY_BYPASS:
@@ -257,7 +264,6 @@ ip6_output(
 		if (sp->req == NULL) {
 			/* acquire a policy */
 			error = key_spdacquire(sp);
-			lck_mtx_unlock(sadb_mutex);
 			goto freehdrs;
 		}
 		needipsec = 1;
@@ -267,7 +273,6 @@ ip6_output(
 	default:
 		printf("ip6_output: Invalid policy found. %d\n", sp->policy);
 	}
-	lck_mtx_unlock(sadb_mutex);
 	skip_ipsec:
 #endif /* IPSEC */
 
@@ -458,10 +463,8 @@ ip6_output(
 		bzero(&state, sizeof(state));
 		state.m = m;
 		lck_mtx_unlock(ip6_mutex);
-		lck_mtx_lock(sadb_mutex);
 		error = ipsec6_output_trans(&state, nexthdrp, mprev, sp, flags,
 			&needipsectun);
-		lck_mtx_unlock(sadb_mutex);
 		lck_mtx_lock(ip6_mutex);
 		m = state.m;
 		if (error) {
@@ -572,6 +575,7 @@ skip_ipsec2:;
 #if IPSEC
 	if (needipsec && needipsectun) {
 		struct ipsec_output_state state;
+		int tunneledv4 = 0;
 
 		/*
 		 * All the extension headers will become inaccessible
@@ -589,10 +593,10 @@ skip_ipsec2:;
 		state.ro = (struct route *)ro;
 		state.dst = (struct sockaddr *)dst;
 		lck_mtx_unlock(ip6_mutex);
-		lck_mtx_lock(sadb_mutex);
-		error = ipsec6_output_tunnel(&state, sp, flags);
-		lck_mtx_unlock(sadb_mutex);
+		error = ipsec6_output_tunnel(&state, sp, flags, &tunneledv4);
 		lck_mtx_lock(ip6_mutex);
+		if (tunneledv4)	/* tunneled in IPv4 - packet is gone */
+			goto done;
 		m = state.m;
 		ro = (struct route_in6 *)state.ro;
 		dst = (struct sockaddr_in6 *)state.dst;
@@ -615,7 +619,6 @@ skip_ipsec2:;
 				error = 0;
 				break;
 			}
-			lck_mtx_unlock(sadb_mutex);
 			goto bad;
 		}
 
@@ -719,7 +722,7 @@ skip_ipsec2:;
 				in6_ifstat_inc(ifp, ifs6_out_discard);
 				goto bad;
 			} else {
-				ifp = &loif[0];
+				ifp = lo_ifp;
 			}
 		}
 
@@ -736,8 +739,8 @@ skip_ipsec2:;
 		if (ifp == NULL) {
 			lck_mtx_lock(rt_mtx);
 			if (ro->ro_rt == 0) {
-				ro->ro_rt = rtalloc1_locked((struct sockaddr *)
-						&ro->ro_dst, 0, 0UL);
+				ro->ro_rt = rtalloc1_locked(
+				    (struct sockaddr *)&ro->ro_dst, 0, 0UL);
 			}
 			if (ro->ro_rt == 0) {
 				ip6stat.ip6s_noroute++;
@@ -790,7 +793,7 @@ skip_ipsec2:;
 			 * if necessary.
 			 */
 			if (ip6_mrouter && (flags & IPV6_FORWARDING) == 0) {
-				if (ip6_mforward(ip6, ifp, m) != NULL) {
+				if (ip6_mforward(ip6, ifp, m) != 0) {
 					m_freem(m);
 					goto done;
 				}
@@ -840,7 +843,7 @@ skip_ipsec2:;
 		}
 	}
 	if (ro_pmtu->ro_rt != NULL) {
-		u_int32_t ifmtu = nd_ifinfo[ifp->if_index].linkmtu;
+		u_int32_t ifmtu = IN6_LINKMTU(ifp);
 
 		mtu = ro_pmtu->ro_rt->rt_rmx.rmx_mtu;
 		if (mtu > ifmtu || mtu == 0) {
@@ -860,7 +863,7 @@ skip_ipsec2:;
 				 ro_pmtu->ro_rt->rt_rmx.rmx_mtu = mtu; /* XXX */
 		}
 	} else {
-		mtu = nd_ifinfo[ifp->if_index].linkmtu;
+		mtu = IN6_LINKMTU(ifp);
 	}
 
 	/*
@@ -1017,7 +1020,7 @@ skip_ipsec2:;
 		goto bad;
 	} else {
 		struct mbuf **mnext, *m_frgpart;
-		struct ip6_frag *ip6f;
+		struct ip6_frag *ip6f = NULL;
 		u_int32_t id = htonl(ip6_id++);
 		u_char nextproto;
 
@@ -1064,7 +1067,7 @@ skip_ipsec2:;
 		 */
 		m0 = m;
 		for (off = hlen; off < tlen; off += len) {
-			MGETHDR(m, M_DONTWAIT, MT_HEADER);
+			MGETHDR(m, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 			if (!m) {
 				error = ENOBUFS;
 				ip6stat.ip6s_odropped++;
@@ -1100,6 +1103,11 @@ skip_ipsec2:;
 			m->m_pkthdr.len = len + hlen + sizeof(*ip6f);
 			m->m_pkthdr.rcvif = 0;
 			m->m_pkthdr.socket_id = m0->m_pkthdr.socket_id;
+#ifdef __darwin8_notyet
+#if CONFIG_MACF_NET
+			mac_create_fragment(m0, m);
+#endif
+#endif
 			ip6f->ip6f_reserved = 0;
 			ip6f->ip6f_ident = id;
 			ip6f->ip6f_nxt = nextproto;
@@ -1151,11 +1159,8 @@ done:
 	}
 
 #if IPSEC
-	if (sp != NULL) {
-		lck_mtx_lock(sadb_mutex);
-		key_freesp(sp);
-		lck_mtx_unlock(sadb_mutex);
-	}
+	if (sp != NULL)
+		key_freesp(sp, KEY_SADB_UNLOCKED);
 #endif /* IPSEC */
 
 	return(error);
@@ -1267,7 +1272,7 @@ ip6_insert_jumboopt(exthdrs, plen)
 			n->m_len = oldoptlen + JUMBOOPTLEN;
 			bcopy(mtod(mopt, caddr_t), mtod(n, caddr_t),
 			      oldoptlen);
-			optbuf = mtod(n, caddr_t) + oldoptlen;
+			optbuf = (u_char *) (mtod(n, caddr_t) + oldoptlen);
 			m_freem(mopt);
 			mopt = exthdrs->ip6e_hbh = n;
 		} else {
@@ -1344,7 +1349,7 @@ ip6_insertfraghdr(m0, m, hlen, frghdrp)
 	return(0);
 }
 
-extern int load_ipfw();
+extern int load_ipfw(void);
 
 /*
  * IP6 socket option processing.
@@ -1357,9 +1362,9 @@ ip6_ctloutput(so, sopt)
 	int privileged;
 	struct inpcb *in6p = sotoinpcb(so);
 	int error, optval;
-	int level, op, optname;
-	int optlen;
-	struct proc *p;
+	int level, op = -1, optname = 0;
+	int optlen = 0;
+	struct proc *p = NULL;
 
 	level = error = optval = 0;
 	if (sopt == NULL) 
@@ -1388,10 +1393,10 @@ ip6_ctloutput(so, sopt)
 					break;
 				}
 				error = soopt_getm(sopt, &m); /* XXX */
-				if (error != NULL)
+				if (error != 0)
 					break;
 				error = soopt_mcopyin(sopt, m); /* XXX */
-				if (error != NULL)
+				if (error != 0)
 					break;
 				error = ip6_pcbopts(&in6p->in6p_outputopts,
 						    m, so, sopt);
@@ -1591,15 +1596,14 @@ do { \
 					req = mtod(m, caddr_t);
 					len = m->m_len;
 				}
-				lck_mtx_lock(sadb_mutex);
 				error = ipsec6_set_policy(in6p, optname, req,
 				                          len, privileged);
-				lck_mtx_unlock(sadb_mutex);
 				m_freem(m);
 			    }
 				break;
 #endif /* KAME IPSEC */
 
+#if IPFIREWALL
 			case IPV6_FW_ADD:
 			case IPV6_FW_DEL:
 			case IPV6_FW_FLUSH:
@@ -1611,6 +1615,7 @@ do { \
 				error = (*ip6_fw_ctl_ptr)(sopt);
 				}
 				break;
+#endif /* IPFIREWALL */
 
 			default:
 				error = ENOPROTOOPT;
@@ -1737,18 +1742,16 @@ do { \
                                         break;
                                 }
 				error = soopt_getm(sopt, &m); /* XXX */
-				if (error != NULL)
+				if (error != 0)
 					break;
 				error = soopt_mcopyin(sopt, m); /* XXX */
-				if (error != NULL)
+				if (error != 0)
 					break;
 				if (m) {
 					req = mtod(m, caddr_t);
 					len = m->m_len;
 				}
-				lck_mtx_lock(sadb_mutex);
 				error = ipsec6_get_policy(in6p, req, len, mp);
-				lck_mtx_unlock(sadb_mutex);
 				if (error == 0)
 					error = soopt_mcopyout(sopt, m); /*XXX*/
 				if (error == 0 && m)
@@ -1757,6 +1760,7 @@ do { \
 			  }
 #endif /* KAME IPSEC */
 
+#if IPFIREWALL
 			case IPV6_FW_GET:
 				{
 				if (ip6_fw_ctl_ptr == NULL && load_ipfw() != 0)
@@ -1765,6 +1769,7 @@ do { \
 				error = (*ip6_fw_ctl_ptr)(sopt);
 				}
 				break;
+#endif /* IPFIREWALL */
 
 			default:
 				error = ENOPROTOOPT;
@@ -1783,11 +1788,11 @@ do { \
  * specifying behavior of outgoing packets.
  */
 static int
-ip6_pcbopts(pktopt, m, so, sopt)
-	struct ip6_pktopts **pktopt;
-	struct mbuf *m;
-	struct socket *so;
-	struct sockopt *sopt;
+ip6_pcbopts(
+	struct ip6_pktopts **pktopt,
+	struct mbuf *m,
+	__unused struct socket *so,
+	struct sockopt *sopt)
 {
 	struct ip6_pktopts *opt = *pktopt;
 	int error = 0;
@@ -1982,7 +1987,6 @@ ip6_setmoptions(
 	struct route_in6 ro;
 	struct sockaddr_in6 *dst;
 	struct in6_multi_mship *imm;
-	struct proc *p = current_proc();	/* XXX */
 
 	if (im6o == NULL) {
 		/*
@@ -2155,7 +2159,7 @@ ip6_setmoptions(
 			 *   XXX: is it a good approach?
 			 */
 			if (IN6_IS_ADDR_MC_NODELOCAL(&mreq->ipv6mr_multiaddr)) {
-				ifp = &loif[0];
+				ifp = lo_ifp;
 			} else {
 				ro.ro_rt = NULL;
 				dst = (struct sockaddr_in6 *)&ro.ro_dst;
@@ -2711,7 +2715,7 @@ ip6_splithdr(m, exthdrs)
 
 	ip6 = mtod(m, struct ip6_hdr *);
 	if (m->m_len > sizeof(*ip6)) {
-		MGETHDR(mh, M_DONTWAIT, MT_HEADER);
+		MGETHDR(mh, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (mh == 0) {
 			m_freem(m);
 			return ENOBUFS;
@@ -2755,3 +2759,4 @@ ip6_optlen(in6p)
 	return len;
 #undef elen
 }
+
diff --git a/bsd/netinet6/ip6_var.h b/bsd/netinet6/ip6_var.h
index 96ca49088..c4a2fb286 100644
--- a/bsd/netinet6/ip6_var.h
+++ b/bsd/netinet6/ip6_var.h
@@ -291,11 +291,16 @@ extern int   ip6_lowportmax;		/* maximum reserved port */
 extern int	ip6_use_tempaddr; /* whether to use temporary addresses. */
 
 extern struct	pr_usrreqs rip6_usrreqs;
+extern struct   pr_usrreqs icmp6_dgram_usrreqs;
 struct sockopt;
 
 struct inpcb;
 
-int	icmp6_ctloutput(struct socket *, struct sockopt *sopt);
+int	icmp6_ctloutput(struct socket *, struct sockopt *);
+int	icmp6_dgram_ctloutput(struct socket *, struct sockopt *);
+int 	icmp6_dgram_send(struct socket *, int , struct mbuf *, struct sockaddr *, struct mbuf *, struct proc *);
+int 	icmp6_dgram_attach(struct socket *, int , struct proc *);
+
 
 struct in6_ifaddr;
 void	ip6_init(void);
@@ -309,8 +314,8 @@ char *	ip6_get_prevhdr(struct mbuf *, int);
 int	ip6_nexthdr(struct mbuf *, int, int, int *);
 int	ip6_lasthdr(struct mbuf *, int, int, int *);
 
-struct mbuf *ip6_addaux(struct mbuf *);
-struct mbuf *ip6_findaux(struct mbuf *);
+struct ip6aux *ip6_addaux(struct mbuf *);
+struct ip6aux *ip6_findaux(struct mbuf *);
 void	ip6_delaux(struct mbuf *);
 
 int	ip6_mforward(struct ip6_hdr *, struct ifnet *, struct mbuf *);
@@ -322,7 +327,7 @@ void	ip6_notify_pmtu(struct inpcb *, struct sockaddr_in6 *,
 			     u_int32_t *);
 int	ip6_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 
-void	ip6_forward(struct mbuf *, int, int);
+void	ip6_forward(struct mbuf *, struct route_in6 *, int, int);
 
 void	ip6_mloopback(struct ifnet *, struct mbuf *, struct sockaddr_in6 *);
 int	ip6_output(struct mbuf *, struct ip6_pktopts *,
diff --git a/bsd/netinet6/ip6protosw.h b/bsd/netinet6/ip6protosw.h
index f0386fa39..cc4b14191 100644
--- a/bsd/netinet6/ip6protosw.h
+++ b/bsd/netinet6/ip6protosw.h
@@ -150,7 +150,7 @@ struct ip6protosw {
 					/* flush any excess space possible */
 #ifdef __APPLE__
 	/* for compat. with IPv4 protosw */
-	int	(*pr_sysctl)();		/* sysctl for protocol */
+	int	(*pr_sysctl)(void);		/* sysctl for protocol */
 #endif
 
 	struct	pr_usrreqs *pr_usrreqs;	/* supersedes pr_usrreq() */
diff --git a/bsd/netinet6/ipcomp.h b/bsd/netinet6/ipcomp.h
index 8d17dc7de..d584de236 100644
--- a/bsd/netinet6/ipcomp.h
+++ b/bsd/netinet6/ipcomp.h
@@ -37,6 +37,7 @@
 #ifndef _NETINET6_IPCOMP_H_
 #define _NETINET6_IPCOMP_H_
 #include <sys/appleapiopts.h>
+#include <netinet6/ipsec.h>
 
 struct ipcomp {
 	u_int8_t comp_nxt;	/* Next Header */
@@ -63,7 +64,7 @@ struct ipcomp_algorithm {
 struct ipsecrequest;
 extern const struct ipcomp_algorithm *ipcomp_algorithm_lookup(int);
 extern void ipcomp4_input(struct mbuf *, int);
-extern int ipcomp4_output(struct mbuf *, struct ipsecrequest *);
+extern int ipcomp4_output(struct mbuf *, struct secasvar *);
 #endif /* KERNEL_PRIVATE */
 #endif /* KERNEL */
 
diff --git a/bsd/netinet6/ipcomp6.h b/bsd/netinet6/ipcomp6.h
index 3091dc6b5..fa72e314e 100644
--- a/bsd/netinet6/ipcomp6.h
+++ b/bsd/netinet6/ipcomp6.h
@@ -37,11 +37,12 @@
 #ifndef _NETINET6_IPCOMP6_H_
 #define _NETINET6_IPCOMP6_H_
 #include <sys/appleapiopts.h>
+#include <netinet6/ipsec.h>
 
 #ifdef KERNEL_PRIVATE
 extern int ipcomp6_input(struct mbuf **, int *);
 extern int ipcomp6_output(struct mbuf *, u_char *, struct mbuf *,
-	struct ipsecrequest *);
+	struct secasvar *);
 #endif KERNEL_PRIVATE
 
 #endif /*_NETINET6_IPCOMP6_H_*/
diff --git a/bsd/netinet6/ipcomp_core.c b/bsd/netinet6/ipcomp_core.c
index 38f70861c..55dcbf506 100644
--- a/bsd/netinet6/ipcomp_core.c
+++ b/bsd/netinet6/ipcomp_core.c
@@ -50,7 +50,9 @@
 
 #include <net/if.h>
 #include <net/route.h>
-#include <net/zlib.h>
+#if ZLIB
+#include <libkern/zlib.h>
+#endif
 #include <kern/cpu_number.h>
 
 #include <netinet6/ipcomp.h>
@@ -64,6 +66,7 @@
 
 #include <net/net_osdep.h>
 
+#if ZLIB
 static void *deflate_alloc(void *, u_int, u_int);
 static void deflate_free(void *, void *);
 static int deflate_common(struct mbuf *, struct mbuf *, size_t *, int);
@@ -83,16 +86,24 @@ static int deflate_memlevel = MAX_MEM_LEVEL;
 
 static z_stream	deflate_stream;
 static z_stream	inflate_stream;
+#endif /* ZLIB */
 
 static const struct ipcomp_algorithm ipcomp_algorithms[] = {
+#if ZLIB
 	{ deflate_compress, deflate_decompress, 90 },
+#endif /* ZLIB */
 };
 
 const struct ipcomp_algorithm *
-ipcomp_algorithm_lookup(idx)
-	int idx;
+ipcomp_algorithm_lookup(
+#if ZLIB
+		int idx
+#else
+		__unused int idx
+#endif
+		)
 {
-
+#if ZLIB
  	if (idx == SADB_X_CALG_DEFLATE) {
 		/*
 		 * Avert your gaze, ugly hack follows!
@@ -129,24 +140,26 @@ ipcomp_algorithm_lookup(idx)
 
 		return &ipcomp_algorithms[0];
 	}
+#endif /* ZLIB */
 	return NULL;
 }
 
+#if ZLIB
 static void *
-deflate_alloc(aux, items, siz)
-	void *aux;
-	u_int items;
-	u_int siz;
+deflate_alloc(
+	__unused void *aux,
+	u_int items,
+	u_int siz)
 {
 	void *ptr;
-	ptr = _MALLOC(items * siz, M_TEMP, M_WAIT);
+	ptr = _MALLOC(items * siz, M_TEMP, M_NOWAIT);
 	return ptr;
 }
 
 static void
-deflate_free(aux, ptr)
-	void *aux;
-	void *ptr;
+deflate_free(
+	__unused void *aux,
+	void *ptr)
 {
 	FREE(ptr, M_TEMP);
 }
@@ -259,8 +272,8 @@ do { \
 			/* inflate: Z_OK can indicate the end of decode */
 			if (mode && !p && zs->avail_out != 0)
 				goto terminate;
-			else
-				; /*once more.*/
+			
+			/* else once more.*/
 		} else {
 			if (zs->msg) {
 				ipseclog((LOG_ERR, "ipcomp_%scompress: "
@@ -397,3 +410,4 @@ deflate_decompress(m, md, lenp)
 
 	return deflate_common(m, md, lenp, 1);
 }
+#endif /* ZLIB */
diff --git a/bsd/netinet6/ipcomp_input.c b/bsd/netinet6/ipcomp_input.c
index 45821aa26..3ce9ba474 100644
--- a/bsd/netinet6/ipcomp_input.c
+++ b/bsd/netinet6/ipcomp_input.c
@@ -49,7 +49,7 @@
 
 #include <net/if.h>
 #include <net/route.h>
-#include <net/zlib.h>
+#include <libkern/zlib.h>
 #include <kern/cpu_number.h>
 #include <kern/locks.h>
 
@@ -59,6 +59,7 @@
 #include <netinet/ip.h>
 #include <netinet/ip_var.h>
 #include <netinet/ip_ecn.h>
+#include <netinet/kpi_ipfilter_var.h>
 
 #if INET6
 #include <netinet/ip6.h>
@@ -80,7 +81,6 @@
 
 #define IPLEN_FLIPPED
 
-extern lck_mtx_t *sadb_mutex;
 void
 ipcomp4_input(struct mbuf *m, int off)
 {
@@ -95,12 +95,11 @@ ipcomp4_input(struct mbuf *m, int off)
 	size_t newlen, olen;
 	struct secasvar *sav = NULL;
 
-	lck_mtx_lock(sadb_mutex);
 
 	if (m->m_pkthdr.len < off + sizeof(struct ipcomp)) {
 		ipseclog((LOG_DEBUG, "IPv4 IPComp input: assumption failed "
 		    "(packet too short)\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 
@@ -109,7 +108,7 @@ ipcomp4_input(struct mbuf *m, int off)
 		m = NULL;	/*already freed*/
 		ipseclog((LOG_DEBUG, "IPv4 IPComp input: assumption failed "
 		    "(pulldown failure)\n"));
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 	ipcomp = mtod(md, struct ipcomp *);
@@ -137,7 +136,7 @@ ipcomp4_input(struct mbuf *m, int off)
 	if (!algo) {
 		ipseclog((LOG_WARNING, "IPv4 IPComp input: unknown cpi %u\n",
 			cpi));
-		ipsecstat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_nosa);
 		goto fail;
 	}
 
@@ -154,18 +153,16 @@ ipcomp4_input(struct mbuf *m, int off)
 
 	olen = m->m_pkthdr.len;
 	newlen = m->m_pkthdr.len - off;
-	lck_mtx_unlock(sadb_mutex);
 	error = (*algo->decompress)(m, m->m_next, &newlen);
-	lck_mtx_lock(sadb_mutex);
 	if (error != 0) {
-		if (error == EINVAL)
-			ipsecstat.in_inval++;
-		else if (error == ENOBUFS)
-			ipsecstat.in_nomem++;
+		if (error == EINVAL) {
+			IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+		} else if (error == ENOBUFS)
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 		m = NULL;
 		goto fail;
 	}
-	ipsecstat.in_comphist[cpi]++;
+	IPSEC_STAT_INCREMENT(ipsecstat.in_comphist[cpi]);
 
 	/*
 	 * returning decompressed packet onto icmp is meaningless.
@@ -190,7 +187,7 @@ ipcomp4_input(struct mbuf *m, int off)
 	len -= olen;
 	if (len & ~0xffff) {
 		/* packet too big after decompress */
-		ipsecstat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
 		goto fail;
 	}
 #ifdef IPLEN_FLIPPED
@@ -204,35 +201,31 @@ ipcomp4_input(struct mbuf *m, int off)
 	if (sav) {
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_IPCOMP, (u_int32_t)cpi) != 0) {
-			ipsecstat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_nomem);
 			goto fail;
 		}
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 		sav = NULL;
 	}
 
 	if (nxt != IPPROTO_DONE) {
 		if ((ip_protox[nxt]->pr_flags & PR_LASTHDR) != 0 &&
 		    ipsec4_in_reject(m, NULL)) {
-			ipsecstat.in_polvio++;
+			IPSEC_STAT_INCREMENT(ipsecstat.in_polvio);
 			goto fail;
 		}
-		lck_mtx_unlock(sadb_mutex);
 		ip_proto_dispatch_in(m, off, nxt, 0);
-		lck_mtx_lock(sadb_mutex);
 	} else
 		m_freem(m);
 	m = NULL;
 
-	ipsecstat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsecstat.in_success);
 	return;
 
 fail:
 	if (sav)
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 
-	lck_mtx_unlock(sadb_mutex);
 	if (m)
 		m_freem(m);
 	return;
@@ -259,13 +252,12 @@ ipcomp6_input(mp, offp)
 	m = *mp;
 	off = *offp;
 
-	lck_mtx_lock(sadb_mutex);
 	md = m_pulldown(m, off, sizeof(*ipcomp), NULL);
 	if (!m) {
 		m = NULL;	/*already freed*/
 		ipseclog((LOG_DEBUG, "IPv6 IPComp input: assumption failed "
 		    "(pulldown failure)\n"));
-		ipsec6stat.in_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
 		goto fail;
 	}
 	ipcomp = mtod(md, struct ipcomp *);
@@ -288,7 +280,7 @@ ipcomp6_input(mp, offp)
 	if (!algo) {
 		ipseclog((LOG_WARNING, "IPv6 IPComp input: unknown cpi %u; "
 			"dropping the packet for simplicity\n", cpi));
-		ipsec6stat.in_nosa++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.in_nosa);
 		goto fail;
 	}
 
@@ -299,18 +291,16 @@ ipcomp6_input(mp, offp)
 	m->m_pkthdr.len -= sizeof(struct ipcomp);
 
 	newlen = m->m_pkthdr.len - off;
-	lck_mtx_unlock(sadb_mutex);
 	error = (*algo->decompress)(m, md, &newlen);
-	lck_mtx_lock(sadb_mutex);
 	if (error != 0) {
-		if (error == EINVAL)
-			ipsec6stat.in_inval++;
-		else if (error == ENOBUFS)
-			ipsec6stat.in_nomem++;
+		if (error == EINVAL) {
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_inval);
+		} else if (error == ENOBUFS)
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 		m = NULL;
 		goto fail;
 	}
-	ipsec6stat.in_comphist[cpi]++;
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_comphist[cpi]);
 	m->m_pkthdr.len = off + newlen;
 
 	/*
@@ -331,24 +321,22 @@ ipcomp6_input(mp, offp)
 	if (sav) {
 		key_sa_recordxfer(sav, m);
 		if (ipsec_addhist(m, IPPROTO_IPCOMP, (u_int32_t)cpi) != 0) {
-			ipsec6stat.in_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_nomem);
 			goto fail;
 		}
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 		sav = NULL;
 	}
 	*offp = off;
 	*mp = m;
-	ipsec6stat.in_success++;
-	lck_mtx_unlock(sadb_mutex);
+	IPSEC_STAT_INCREMENT(ipsec6stat.in_success);
 	return nxt;
 
 fail:
 	if (m)
 		m_freem(m);
 	if (sav)
-		key_freesav(sav);
-	lck_mtx_unlock(sadb_mutex);
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	return IPPROTO_DONE;
 }
 #endif /* INET6 */
diff --git a/bsd/netinet6/ipcomp_output.c b/bsd/netinet6/ipcomp_output.c
index 7a8d39b1d..c9159972f 100644
--- a/bsd/netinet6/ipcomp_output.c
+++ b/bsd/netinet6/ipcomp_output.c
@@ -49,7 +49,7 @@
 
 #include <net/if.h>
 #include <net/route.h>
-#include <net/zlib.h>
+#include <libkern/zlib.h>
 #include <kern/cpu_number.h>
 #include <kern/locks.h>
 
@@ -78,10 +78,9 @@
 
 #include <net/net_osdep.h>
 
-extern lck_mtx_t  *sadb_mutex;
 
 static int ipcomp_output(struct mbuf *, u_char *, struct mbuf *,
-	struct ipsecrequest *, int);
+	int, struct secasvar *sav);
 
 /*
  * Modify the packet so that the payload is compressed.
@@ -102,19 +101,18 @@ static int ipcomp_output(struct mbuf *, u_char *, struct mbuf *,
  *	<-----------------> compoff
  */
 static int
-ipcomp_output(m, nexthdrp, md, isr, af)
+ipcomp_output(m, nexthdrp, md, af, sav)
 	struct mbuf *m;
 	u_char *nexthdrp;
 	struct mbuf *md;
-	struct ipsecrequest *isr;
 	int af;
+	struct secasvar *sav;
 {
 	struct mbuf *n;
 	struct mbuf *md0;
 	struct mbuf *mcopy;
 	struct mbuf *mprev;
 	struct ipcomp *ipcomp;
-	struct secasvar *sav = isr->sav;
 	const struct ipcomp_algorithm *algo;
 	u_int16_t cpi;		/* host order */
 	size_t plen0, plen;	/*payload length to be compressed*/
@@ -144,7 +142,7 @@ ipcomp_output(m, nexthdrp, md, isr, af)
 	/* grab parameters */
 	algo = ipcomp_algorithm_lookup(sav->alg_enc);
 	if ((ntohl(sav->spi) & ~0xffff) != 0 || !algo) {
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 		m_freem(m);
 		return EINVAL;
 	}
@@ -189,7 +187,7 @@ ipcomp_output(m, nexthdrp, md, isr, af)
 	if (mprev == NULL || mprev->m_next != md) {
 		ipseclog((LOG_DEBUG, "ipcomp%d_output: md is not in chain\n",
 		    afnumber));
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 		m_freem(m);
 		m_freem(md0);
 		m_freem(mcopy);
@@ -206,19 +204,16 @@ ipcomp_output(m, nexthdrp, md, isr, af)
 	mprev->m_next = md;
 
 	/* compress data part */
-	lck_mtx_unlock(sadb_mutex);
 	if ((*algo->compress)(m, md, &plen) || mprev->m_next == NULL) {
-		lck_mtx_lock(sadb_mutex);
 		ipseclog((LOG_ERR, "packet compression failure\n"));
 		m = NULL;
 		m_freem(md0);
 		m_freem(mcopy);
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 		error = EINVAL;
 		goto fail;
 	}
-	lck_mtx_lock(sadb_mutex);
-	stat->out_comphist[sav->alg_enc]++;
+	IPSEC_STAT_INCREMENT(stat->out_comphist[sav->alg_enc]);
 	md = mprev->m_next;
 
 	/*
@@ -311,7 +306,7 @@ ipcomp_output(m, nexthdrp, md, isr, af)
 		else {
 			ipseclog((LOG_ERR,
 			    "IPv4 ESP output: size exceeds limit\n"));
-			ipsecstat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 			m_freem(m);
 			error = EMSGSIZE;
 			goto fail;
@@ -330,9 +325,9 @@ ipcomp_output(m, nexthdrp, md, isr, af)
 		ipseclog((LOG_DEBUG,
 		    "NULL mbuf after compression in ipcomp%d_output",
 		    afnumber));
-		stat->out_inval++;
+		IPSEC_STAT_INCREMENT(stat->out_inval);
 	}
-		stat->out_success++;
+		IPSEC_STAT_INCREMENT(stat->out_success);
 
 	/* compute byte lifetime against original packet */
 	key_sa_recordxfer(sav, mcopy);
@@ -350,37 +345,37 @@ fail:
 
 #if INET
 int
-ipcomp4_output(m, isr)
+ipcomp4_output(m, sav)
 	struct mbuf *m;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
 	struct ip *ip;
 	if (m->m_len < sizeof(struct ip)) {
 		ipseclog((LOG_DEBUG, "ipcomp4_output: first mbuf too short\n"));
-		ipsecstat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsecstat.out_inval);
 		m_freem(m);
 		return 0;
 	}
 	ip = mtod(m, struct ip *);
 	/* XXX assumes that m->m_next points to payload */
-	return ipcomp_output(m, &ip->ip_p, m->m_next, isr, AF_INET);
+	return ipcomp_output(m, &ip->ip_p, m->m_next, AF_INET, sav);
 }
 #endif /*INET*/
 
-#ifdef INET6
+#if INET6
 int
-ipcomp6_output(m, nexthdrp, md, isr)
+ipcomp6_output(m, nexthdrp, md, sav)
 	struct mbuf *m;
 	u_char *nexthdrp;
 	struct mbuf *md;
-	struct ipsecrequest *isr;
+	struct secasvar *sav;
 {
 	if (m->m_len < sizeof(struct ip6_hdr)) {
 		ipseclog((LOG_DEBUG, "ipcomp6_output: first mbuf too short\n"));
-		ipsec6stat.out_inval++;
+		IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 		m_freem(m);
 		return 0;
 	}
-	return ipcomp_output(m, nexthdrp, md, isr, AF_INET6);
+	return ipcomp_output(m, nexthdrp, md, AF_INET6, sav);
 }
 #endif /*INET6*/
diff --git a/bsd/netinet6/ipsec.c b/bsd/netinet6/ipsec.c
index 9d6201d6a..97dd1761e 100644
--- a/bsd/netinet6/ipsec.c
+++ b/bsd/netinet6/ipsec.c
@@ -49,6 +49,7 @@
 #include <sys/sysctl.h>
 #include <kern/locks.h>
 #include <sys/kauth.h>
+#include <libkern/OSAtomic.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -131,6 +132,8 @@ int	esp_udp_encap_port = 0;
 static int sysctl_def_policy SYSCTL_HANDLER_ARGS;
 extern u_int32_t natt_now;
 
+struct ipsec_tag;
+
 SYSCTL_DECL(_net_inet_ipsec);
 #if INET6
 SYSCTL_DECL(_net_inet6_ipsec6);
@@ -226,7 +229,6 @@ static int ipsec_set_policy(struct secpolicy **pcb_sp,
 static int ipsec_get_policy(struct secpolicy *pcb_sp, struct mbuf **mp);
 static void vshiftl(unsigned char *, int, int);
 static int ipsec_in_reject(struct secpolicy *, struct mbuf *);
-static size_t ipsec_hdrsiz(struct secpolicy *);
 #if INET
 static struct mbuf *ipsec4_splithdr(struct mbuf *);
 #endif
@@ -238,10 +240,11 @@ static int ipsec4_encapsulate(struct mbuf *, struct secasvar *);
 #endif
 #if INET6
 static int ipsec6_encapsulate(struct mbuf *, struct secasvar *);
+static int ipsec64_encapsulate(struct mbuf *, struct secasvar *);
 #endif
-static struct mbuf *ipsec_addaux(struct mbuf *);
-static struct mbuf *ipsec_findaux(struct mbuf *);
-static void ipsec_optaux(struct mbuf *, struct mbuf *);
+static struct ipsec_tag *ipsec_addaux(struct mbuf *);
+static struct ipsec_tag *ipsec_findaux(struct mbuf *);
+static void ipsec_optaux(struct mbuf *, struct ipsec_tag *);
 void ipsec_send_natt_keepalive(struct secasvar *sav);
 
 static int
@@ -286,7 +289,7 @@ ipsec4_getpolicybysock(m, dir, so, error)
 	struct secpolicy *currsp = NULL;	/* policy on socket */
 	struct secpolicy *kernsp = NULL;	/* policy on kernel */
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (m == NULL || so == NULL || error == NULL)
 		panic("ipsec4_getpolicybysock: NULL pointer was passed.\n");
@@ -356,7 +359,9 @@ ipsec4_getpolicybysock(m, dir, so, error)
 	if (pcbsp->priv) {
 		switch (currsp->policy) {
 		case IPSEC_POLICY_BYPASS:
+			lck_mtx_lock(sadb_mutex);
 			currsp->refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			KERNEL_DEBUG(DBG_FNC_GETPOL_SOCK | DBG_FUNC_END, 2,*error,0,0,0);
 			return currsp;
@@ -376,6 +381,7 @@ ipsec4_getpolicybysock(m, dir, so, error)
 			}
 
 			/* no SP found */
+			lck_mtx_lock(sadb_mutex);
 			if (ip4_def_policy.policy != IPSEC_POLICY_DISCARD
 			 && ip4_def_policy.policy != IPSEC_POLICY_NONE) {
 				ipseclog((LOG_INFO,
@@ -384,12 +390,15 @@ ipsec4_getpolicybysock(m, dir, so, error)
 				ip4_def_policy.policy = IPSEC_POLICY_NONE;
 			}
 			ip4_def_policy.refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			KERNEL_DEBUG(DBG_FNC_GETPOL_SOCK | DBG_FUNC_END, 4,*error,0,0,0);
 			return &ip4_def_policy;
 			
 		case IPSEC_POLICY_IPSEC:
+			lck_mtx_lock(sadb_mutex);
 			currsp->refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			KERNEL_DEBUG(DBG_FNC_GETPOL_SOCK | DBG_FUNC_END, 5,*error,0,0,0);
 			return currsp;
@@ -429,6 +438,7 @@ ipsec4_getpolicybysock(m, dir, so, error)
 		return NULL;
 
 	case IPSEC_POLICY_ENTRUST:
+		lck_mtx_lock(sadb_mutex);
 		if (ip4_def_policy.policy != IPSEC_POLICY_DISCARD
 		 && ip4_def_policy.policy != IPSEC_POLICY_NONE) {
 			ipseclog((LOG_INFO,
@@ -437,12 +447,15 @@ ipsec4_getpolicybysock(m, dir, so, error)
 			ip4_def_policy.policy = IPSEC_POLICY_NONE;
 		}
 		ip4_def_policy.refcnt++;
+		lck_mtx_unlock(sadb_mutex);
 		*error = 0;
 		KERNEL_DEBUG(DBG_FNC_GETPOL_SOCK | DBG_FUNC_END, 9,*error,0,0,0);
 		return &ip4_def_policy;
 
 	case IPSEC_POLICY_IPSEC:
+		lck_mtx_lock(sadb_mutex);
 		currsp->refcnt++;
+		lck_mtx_unlock(sadb_mutex);
 		*error = 0;
 		KERNEL_DEBUG(DBG_FNC_GETPOL_SOCK | DBG_FUNC_END, 10,*error,0,0,0);
 		return currsp;
@@ -479,8 +492,8 @@ ipsec4_getpolicybyaddr(m, dir, flag, error)
 	if (ipsec_bypass != 0)
 		return 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (m == NULL || error == NULL)
 		panic("ipsec4_getpolicybyaddr: NULL pointer was passed.\n");
@@ -514,6 +527,7 @@ ipsec4_getpolicybyaddr(m, dir, flag, error)
 	}
 
 	/* no SP found */
+	lck_mtx_lock(sadb_mutex);
 	if (ip4_def_policy.policy != IPSEC_POLICY_DISCARD
 	 && ip4_def_policy.policy != IPSEC_POLICY_NONE) {
 		ipseclog((LOG_INFO, "fixed system default policy:%d->%d\n",
@@ -522,6 +536,7 @@ ipsec4_getpolicybyaddr(m, dir, flag, error)
 		ip4_def_policy.policy = IPSEC_POLICY_NONE;
 	}
 	ip4_def_policy.refcnt++;
+	lck_mtx_unlock(sadb_mutex);
 	*error = 0;
 	KERNEL_DEBUG(DBG_FNC_GETPOL_ADDR | DBG_FUNC_END, 3,*error,0,0,0);
 	return &ip4_def_policy;
@@ -549,8 +564,8 @@ ipsec6_getpolicybysock(m, dir, so, error)
 	struct secpolicy *currsp = NULL;	/* policy on socket */
 	struct secpolicy *kernsp = NULL;	/* policy on kernel */
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (m == NULL || so == NULL || error == NULL)
 		panic("ipsec6_getpolicybysock: NULL pointer was passed.\n");
@@ -592,7 +607,9 @@ ipsec6_getpolicybysock(m, dir, so, error)
 	if (pcbsp->priv) {
 		switch (currsp->policy) {
 		case IPSEC_POLICY_BYPASS:
+			lck_mtx_lock(sadb_mutex);
 			currsp->refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			return currsp;
 
@@ -610,6 +627,7 @@ ipsec6_getpolicybysock(m, dir, so, error)
 			}
 
 			/* no SP found */
+			lck_mtx_lock(sadb_mutex);
 			if (ip6_def_policy.policy != IPSEC_POLICY_DISCARD
 			 && ip6_def_policy.policy != IPSEC_POLICY_NONE) {
 				ipseclog((LOG_INFO,
@@ -618,11 +636,14 @@ ipsec6_getpolicybysock(m, dir, so, error)
 				ip6_def_policy.policy = IPSEC_POLICY_NONE;
 			}
 			ip6_def_policy.refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			return &ip6_def_policy;
 			
 		case IPSEC_POLICY_IPSEC:
+			lck_mtx_lock(sadb_mutex);
 			currsp->refcnt++;
+			lck_mtx_unlock(sadb_mutex);
 			*error = 0;
 			return currsp;
 
@@ -658,6 +679,7 @@ ipsec6_getpolicybysock(m, dir, so, error)
 		return NULL;
 
 	case IPSEC_POLICY_ENTRUST:
+		lck_mtx_lock(sadb_mutex);
 		if (ip6_def_policy.policy != IPSEC_POLICY_DISCARD
 		 && ip6_def_policy.policy != IPSEC_POLICY_NONE) {
 			ipseclog((LOG_INFO,
@@ -666,11 +688,14 @@ ipsec6_getpolicybysock(m, dir, so, error)
 			ip6_def_policy.policy = IPSEC_POLICY_NONE;
 		}
 		ip6_def_policy.refcnt++;
+		lck_mtx_unlock(sadb_mutex);
 		*error = 0;
 		return &ip6_def_policy;
 
 	case IPSEC_POLICY_IPSEC:
+		lck_mtx_lock(sadb_mutex);
 		currsp->refcnt++;
+		lck_mtx_unlock(sadb_mutex);
 		*error = 0;
 		return currsp;
 
@@ -709,8 +734,8 @@ ipsec6_getpolicybyaddr(m, dir, flag, error)
 {
 	struct secpolicy *sp = NULL;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (m == NULL || error == NULL)
 		panic("ipsec6_getpolicybyaddr: NULL pointer was passed.\n");
@@ -740,6 +765,7 @@ ipsec6_getpolicybyaddr(m, dir, flag, error)
 	}
 
 	/* no SP found */
+	lck_mtx_lock(sadb_mutex);
 	if (ip6_def_policy.policy != IPSEC_POLICY_DISCARD
 	 && ip6_def_policy.policy != IPSEC_POLICY_NONE) {
 		ipseclog((LOG_INFO, "fixed system default policy: %d->%d\n",
@@ -747,6 +773,7 @@ ipsec6_getpolicybyaddr(m, dir, flag, error)
 		ip6_def_policy.policy = IPSEC_POLICY_NONE;
 	}
 	ip6_def_policy.refcnt++;
+	lck_mtx_unlock(sadb_mutex);
 	*error = 0;
 	return &ip6_def_policy;
 }
@@ -763,11 +790,12 @@ ipsec6_getpolicybyaddr(m, dir, flag, error)
  *	other:	failure, and set errno.
  */
 int
-ipsec_setspidx_mbuf(spidx, dir, family, m, needport)
-	struct secpolicyindex *spidx;
-	u_int dir, family;
-	struct mbuf *m;
-	int needport;
+ipsec_setspidx_mbuf(
+	struct secpolicyindex *spidx,
+	u_int dir,
+	__unused u_int family,
+	struct mbuf *m,
+	int needport)
 {
 	int error;
 
@@ -937,7 +965,7 @@ ipsec_setspidx(m, spidx, needport)
 			return error;
 		ipsec4_get_ulp(m, spidx, needport);
 		return 0;
-#ifdef INET6
+#if INET6
 	case 6:
 		if (m->m_pkthdr.len < sizeof(struct ip6_hdr)) {
 			KEYDEBUG(KEYDEBUG_IPSEC_DUMP,
@@ -1194,8 +1222,6 @@ ipsec_init_policy(so, pcb_sp)
 {
 	struct inpcbpolicy *new;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (so == NULL || pcb_sp == NULL)
 		panic("ipsec_init_policy: NULL pointer was passed.\n");
@@ -1224,7 +1250,7 @@ ipsec_init_policy(so, pcb_sp)
 	new->sp_in->policy = IPSEC_POLICY_ENTRUST;
 
 	if ((new->sp_out = key_newsp()) == NULL) {
-		key_freesp(new->sp_in);
+		key_freesp(new->sp_in, KEY_SADB_UNLOCKED);
 		ipsec_delpcbpolicy(new);
 		return ENOBUFS;
 	}
@@ -1246,18 +1272,16 @@ ipsec_copy_policy(old, new)
 	if (ipsec_bypass != 0)
 		return 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	sp = ipsec_deepcopy_policy(old->sp_in);
 	if (sp) {
-		key_freesp(new->sp_in);
+		key_freesp(new->sp_in, KEY_SADB_UNLOCKED);
 		new->sp_in = sp;
 	} else
 		return ENOBUFS;
 
 	sp = ipsec_deepcopy_policy(old->sp_out);
 	if (sp) {
-		key_freesp(new->sp_out);
+		key_freesp(new->sp_out, KEY_SADB_UNLOCKED);
 		new->sp_out = sp;
 	} else
 		return ENOBUFS;
@@ -1278,8 +1302,10 @@ ipsec_deepcopy_policy(src)
 	struct ipsecrequest *r;
 	struct secpolicy *dst;
 
+	if (src == NULL)
+		return NULL;
 	dst = key_newsp();
-	if (src == NULL || dst == NULL)
+	if (dst == NULL)
 		return NULL;
 
 	/*
@@ -1303,7 +1329,6 @@ ipsec_deepcopy_policy(src)
 		bcopy(&p->saidx.src, &(*q)->saidx.src, sizeof((*q)->saidx.src));
 		bcopy(&p->saidx.dst, &(*q)->saidx.dst, sizeof((*q)->saidx.dst));
 
-		(*q)->sav = NULL;
 		(*q)->sp = dst;
 
 		q = &((*q)->next);
@@ -1322,24 +1347,23 @@ fail:
 		FREE(p, M_SECA);
 		p = NULL;
 	}
+	key_freesp(dst, KEY_SADB_UNLOCKED);
 	return NULL;
 }
 
 /* set policy and ipsec request if present. */
 static int
-ipsec_set_policy(pcb_sp, optname, request, len, priv)
-	struct secpolicy **pcb_sp;
-	int optname;
-	caddr_t request;
-	size_t len;
-	int priv;
+ipsec_set_policy(
+	struct secpolicy **pcb_sp,
+	__unused int optname,
+	caddr_t request,
+	size_t len,
+	int priv)
 {
 	struct sadb_x_policy *xpl;
 	struct secpolicy *newsp = NULL;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (pcb_sp == NULL || *pcb_sp == NULL || request == NULL)
 		return EINVAL;
@@ -1368,7 +1392,7 @@ ipsec_set_policy(pcb_sp, optname, request, len, priv)
 	newsp->state = IPSEC_SPSTATE_ALIVE;
 
 	/* clear old SP and set new SP */
-	key_freesp(*pcb_sp);
+	key_freesp(*pcb_sp, KEY_SADB_UNLOCKED);
 	*pcb_sp = newsp;
 	KEYDEBUG(KEYDEBUG_IPSEC_DUMP,
 		printf("ipsec_set_policy: new policy\n");
@@ -1383,7 +1407,6 @@ ipsec_get_policy(pcb_sp, mp)
 	struct mbuf **mp;
 {
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check. */
 	if (pcb_sp == NULL || mp == NULL)
@@ -1395,7 +1418,7 @@ ipsec_get_policy(pcb_sp, mp)
 		return ENOBUFS;
 	}
 
-	(*mp)->m_type = MT_DATA;
+	m_mchtype(*mp, MT_DATA);
 	KEYDEBUG(KEYDEBUG_IPSEC_DUMP,
 		printf("ipsec_get_policy:\n");
 		kdebug_mbuf(*mp));
@@ -1415,8 +1438,6 @@ ipsec4_set_policy(inp, optname, request, len, priv)
 	struct secpolicy **pcb_sp;
 	int	error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (inp == NULL || request == NULL)
 		return EINVAL;
@@ -1462,7 +1483,7 @@ ipsec4_get_policy(inp, request, len, mp)
 	struct secpolicy *pcb_sp;
 	int	error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 
 	/* sanity check. */
 	if (inp == NULL || request == NULL || mp == NULL)
@@ -1499,7 +1520,6 @@ int
 ipsec4_delete_pcbpolicy(inp)
 	struct inpcb *inp;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check. */
 	if (inp == NULL)
@@ -1509,12 +1529,12 @@ ipsec4_delete_pcbpolicy(inp)
 		return 0;
 
 	if (inp->inp_sp->sp_in != NULL) {
-		key_freesp(inp->inp_sp->sp_in);
+		key_freesp(inp->inp_sp->sp_in, KEY_SADB_UNLOCKED);
 		inp->inp_sp->sp_in = NULL;
 	}
 
 	if (inp->inp_sp->sp_out != NULL) {
-		key_freesp(inp->inp_sp->sp_out);
+		key_freesp(inp->inp_sp->sp_out, KEY_SADB_UNLOCKED);
 		inp->inp_sp->sp_out = NULL;
 	}
 
@@ -1537,8 +1557,6 @@ ipsec6_set_policy(in6p, optname, request, len, priv)
 	struct secpolicy **pcb_sp;
 	int error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (in6p == NULL || request == NULL)
 		return EINVAL;
@@ -1584,8 +1602,6 @@ ipsec6_get_policy(in6p, request, len, mp)
 	struct secpolicy *pcb_sp;
 	int error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (in6p == NULL || request == NULL || mp == NULL)
 		return EINVAL;
@@ -1620,7 +1636,6 @@ int
 ipsec6_delete_pcbpolicy(in6p)
 	struct in6pcb *in6p;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check. */
 	if (in6p == NULL)
@@ -1630,12 +1645,12 @@ ipsec6_delete_pcbpolicy(in6p)
 		return 0;
 
 	if (in6p->in6p_sp->sp_in != NULL) {
-		key_freesp(in6p->in6p_sp->sp_in);
+		key_freesp(in6p->in6p_sp->sp_in, KEY_SADB_UNLOCKED);
 		in6p->in6p_sp->sp_in = NULL;
 	}
 
 	if (in6p->in6p_sp->sp_out != NULL) {
-		key_freesp(in6p->in6p_sp->sp_out);
+		key_freesp(in6p->in6p_sp->sp_out, KEY_SADB_UNLOCKED);
 		in6p->in6p_sp->sp_out = NULL;
 	}
 
@@ -1655,9 +1670,7 @@ ipsec_get_reqlevel(isr)
 	struct ipsecrequest *isr;
 {
 	u_int level = 0;
-	u_int esp_trans_deflev, esp_net_deflev, ah_trans_deflev, ah_net_deflev;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	u_int esp_trans_deflev = 0, esp_net_deflev = 0, ah_trans_deflev = 0, ah_net_deflev = 0;
 
 	/* sanity check */
 	if (isr == NULL || isr->sp == NULL)
@@ -1768,11 +1781,10 @@ ipsec_in_reject(sp, m)
 		printf("ipsec_in_reject: using SP\n");
 		kdebug_secpolicy(sp));
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* check policy */
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+	case IPSEC_POLICY_GENERATE:
 		return 1;
 	case IPSEC_POLICY_BYPASS:
 	case IPSEC_POLICY_NONE:
@@ -1802,10 +1814,22 @@ ipsec_in_reject(sp, m)
 			if (level == IPSEC_LEVEL_REQUIRE) {
 				need_conf++;
 
+#if 0
+		/* this won't work with multiple input threads - isr->sav would change 
+		 * with every packet and is not necessarily related to the current packet 
+		 * being processed.  If ESP processing is required - the esp code should
+		 * make sure that the integrity check is present and correct.  I don't see
+		 * why it would be necessary to check for the presence of the integrity
+		 * check value here.  I think this is just wrong.
+		 * isr->sav has been removed.
+		 * %%%%%% this needs to be re-worked at some point but I think the code below can 
+		 * be ignored for now.
+		 */
 				if (isr->sav != NULL
 				 && isr->sav->flags == SADB_X_EXT_NONE
 				 && isr->sav->alg_auth != SADB_AALG_NONE)
 					need_icv++;
+#endif
 			}
 			break;
 		case IPPROTO_AH:
@@ -1850,8 +1874,7 @@ ipsec4_in_reject_so(m, so)
 	int error;
 	int result;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (m == NULL)
 		return 0;	/* XXX should be panic ? */
@@ -1872,7 +1895,7 @@ ipsec4_in_reject_so(m, so)
 	result = ipsec_in_reject(sp, m);
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP ipsec4_in_reject_so call free SP:%p\n", sp));
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return result;
 }
@@ -1882,8 +1905,8 @@ ipsec4_in_reject(m, inp)
 	struct mbuf *m;
 	struct inpcb *inp;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	if (inp == NULL)
 		return ipsec4_in_reject_so(m, NULL);
 	if (inp->inp_socket)
@@ -1910,8 +1933,7 @@ ipsec6_in_reject_so(m, so)
 	int error;
 	int result;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (m == NULL)
 		return 0;	/* XXX should be panic ? */
@@ -1931,7 +1953,7 @@ ipsec6_in_reject_so(m, so)
 	result = ipsec_in_reject(sp, m);
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP ipsec6_in_reject_so call free SP:%p\n", sp));
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return result;
 }
@@ -1941,8 +1963,8 @@ ipsec6_in_reject(m, in6p)
 	struct mbuf *m;
 	struct in6pcb *in6p;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	if (in6p == NULL)
 		return ipsec6_in_reject_so(m, NULL);
 	if (in6p->in6p_socket)
@@ -1960,22 +1982,22 @@ ipsec6_in_reject(m, in6p)
  * in case it is tunneled, it includes the size of outer IP header.
  * NOTE: SP passed is free in this function.
  */
-static size_t
+size_t
 ipsec_hdrsiz(sp)
 	struct secpolicy *sp;
 {
 	struct ipsecrequest *isr;
 	size_t siz, clen;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	KEYDEBUG(KEYDEBUG_IPSEC_DATA,
 		printf("ipsec_hdrsiz: using SP\n");
 		kdebug_secpolicy(sp));
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* check policy */
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+	case IPSEC_POLICY_GENERATE:
 	case IPSEC_POLICY_BYPASS:
 	case IPSEC_POLICY_NONE:
 		return 0;
@@ -2044,8 +2066,7 @@ ipsec4_hdrsiz(m, dir, inp)
 	int error;
 	size_t size;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (m == NULL)
 		return 0;	/* XXX should be panic ? */
@@ -2069,7 +2090,7 @@ ipsec4_hdrsiz(m, dir, inp)
 		printf("DP ipsec4_hdrsiz call free SP:%p\n", sp));
 	KEYDEBUG(KEYDEBUG_IPSEC_DATA,
 		printf("ipsec4_hdrsiz: size:%lu.\n", (unsigned long)size));
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return size;
 }
@@ -2088,8 +2109,7 @@ ipsec6_hdrsiz(m, dir, in6p)
 	int error;
 	size_t size;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (m == NULL)
 		return 0;	/* XXX shoud be panic ? */
@@ -2110,7 +2130,7 @@ ipsec6_hdrsiz(m, dir, in6p)
 		printf("DP ipsec6_hdrsiz call free SP:%p\n", sp));
 	KEYDEBUG(KEYDEBUG_IPSEC_DATA,
 		printf("ipsec6_hdrsiz: size:%lu.\n", (unsigned long)size));
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return size;
 }
@@ -2131,8 +2151,6 @@ ipsec4_encapsulate(m, sav)
 	size_t hlen;
 	size_t plen;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* can't tunnel between different AFs */
 	if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family
 		!= ((struct sockaddr *)&sav->sah->saidx.dst)->sa_family
@@ -2252,8 +2270,6 @@ ipsec6_encapsulate(m, sav)
 	struct ip6_hdr *ip6;
 	size_t plen;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* can't tunnel between different AFs */
 	if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family
 		!= ((struct sockaddr *)&sav->sah->saidx.dst)->sa_family
@@ -2322,6 +2338,92 @@ ipsec6_encapsulate(m, sav)
 
 	return 0;
 }
+
+static int
+ipsec64_encapsulate(m, sav)
+	struct mbuf *m;
+	struct secasvar *sav;
+{
+	struct ip6_hdr *ip6, *ip6i;
+	struct ip *ip;
+	size_t plen;
+	u_int8_t hlim;
+
+	/* tunneling over IPv4 */
+	if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family
+		!= ((struct sockaddr *)&sav->sah->saidx.dst)->sa_family
+	 || ((struct sockaddr *)&sav->sah->saidx.src)->sa_family != AF_INET) {
+		m_freem(m);
+		return EINVAL;
+	}
+#if 0
+	/* XXX if the dst is myself, perform nothing. */
+	if (key_ismyaddr((struct sockaddr *)&sav->sah->saidx.dst)) {
+		m_freem(m);
+		return EINVAL;
+	}
+#endif
+
+	plen = m->m_pkthdr.len;
+	ip6 = mtod(m, struct ip6_hdr *);
+	hlim = ip6->ip6_hlim; 
+	/*
+	 * grow the mbuf to accomodate the new IPv4 header.
+	 */
+	if (m->m_len != sizeof(struct ip6_hdr))
+		panic("ipsec6_encapsulate: assumption failed (first mbuf length)");
+	if (M_LEADINGSPACE(m->m_next) < sizeof(struct ip6_hdr)) {
+		struct mbuf *n;
+		MGET(n, M_DONTWAIT, MT_DATA);
+		if (!n) {
+			m_freem(m);
+			return ENOBUFS;
+		}
+		n->m_len = sizeof(struct ip6_hdr);
+		n->m_next = m->m_next;
+		m->m_next = n;
+		m->m_pkthdr.len += sizeof(struct ip);
+		ip6i = mtod(n, struct ip6_hdr *);
+	} else {
+		m->m_next->m_len += sizeof(struct ip6_hdr);
+		m->m_next->m_data -= sizeof(struct ip6_hdr);
+		m->m_pkthdr.len += sizeof(struct ip);
+		ip6i = mtod(m->m_next, struct ip6_hdr *);
+	}
+	/* construct new IPv4 header. see RFC 2401 5.1.2.1 */
+	/* ECN consideration. */
+	/* XXX To be fixed later if needed */
+	// ip_ecn_ingress(ip4_ipsec_ecn, &ip->ip_tos, &oip->ip_tos);	
+
+	bcopy(ip6, ip6i, sizeof(struct ip6_hdr));
+	ip = mtod(m, struct ip *);
+	m->m_len = sizeof(struct ip);
+	/* 
+	 * Fill in some of the IPv4 fields - we don't need all of them
+	 * because the rest will be filled in by ip_output
+	 */
+	ip->ip_v = IPVERSION;
+	ip->ip_hl = sizeof(struct ip) >> 2;
+	ip->ip_id = 0;
+	ip->ip_sum = 0;
+	ip->ip_tos = 0; 
+	ip->ip_off = 0;
+	ip->ip_ttl = hlim;
+	ip->ip_p = IPPROTO_IPV6;
+	if (plen + sizeof(struct ip) < IP_MAXPACKET)
+		ip->ip_len = htons(plen + sizeof(struct ip));
+	else {
+		ip->ip_len = htons(plen);
+		ipseclog((LOG_ERR, "IPv4 ipsec: size exceeds limit: "
+			"leave ip_len as is (invalid packet)\n"));
+	}
+	bcopy(&((struct sockaddr_in *)&sav->sah->saidx.src)->sin_addr,
+		&ip->ip_src, sizeof(ip->ip_src));
+	bcopy(&((struct sockaddr_in *)&sav->sah->saidx.dst)->sin_addr,
+		&ip->ip_dst, sizeof(ip->ip_dst));
+
+	return 0;
+}
 #endif /*INET6*/
 
 /*
@@ -2345,47 +2447,59 @@ ipsec_chkreplay(seq, sav)
 	u_int32_t wsizeb;	/* constant: bits of window size */
 	int frlast;		/* constant: last frame */
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (sav == NULL)
 		panic("ipsec_chkreplay: NULL pointer was passed.\n");
 
+	lck_mtx_lock(sadb_mutex);
 	replay = sav->replay;
 
-	if (replay->wsize == 0)
+	if (replay->wsize == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return 1;	/* no need to check replay. */
+	}
 
 	/* constant */
 	frlast = replay->wsize - 1;
 	wsizeb = replay->wsize << 3;
 
 	/* sequence number of 0 is invalid */
-	if (seq == 0)
+	if (seq == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return 0;
+	}
 
 	/* first time is always okay */
-	if (replay->count == 0)
+	if (replay->count == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return 1;
+	}
 
 	if (seq > replay->lastseq) {
 		/* larger sequences are okay */
+		lck_mtx_unlock(sadb_mutex);
 		return 1;
 	} else {
 		/* seq is equal or less than lastseq. */
 		diff = replay->lastseq - seq;
 
 		/* over range to check, i.e. too old or wrapped */
-		if (diff >= wsizeb)
+		if (diff >= wsizeb) {
+			lck_mtx_unlock(sadb_mutex);
 			return 0;
+		}
 
 		fr = frlast - diff / 8;
 
 		/* this packet already seen ? */
-		if ((replay->bitmap)[fr] & (1 << (diff % 8)))
+		if ((replay->bitmap)[fr] & (1 << (diff % 8))) {
+			lck_mtx_unlock(sadb_mutex);
 			return 0;
+		}
 
 		/* out of order but good */
+		lck_mtx_unlock(sadb_mutex);
 		return 1;
 	}
 }
@@ -2405,13 +2519,12 @@ ipsec_updatereplay(seq, sav)
 	int fr;
 	u_int32_t wsizeb;	/* constant: bits of window size */
 	int frlast;		/* constant: last frame */
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (sav == NULL)
 		panic("ipsec_chkreplay: NULL pointer was passed.\n");
 
+	lck_mtx_lock(sadb_mutex);
 	replay = sav->replay;
 
 	if (replay->wsize == 0)
@@ -2441,7 +2554,7 @@ ipsec_updatereplay(seq, sav)
 		if (diff < wsizeb) {
 			/* In window */
 			/* set bit for this packet */
-			vshiftl(replay->bitmap, diff, replay->wsize);
+			vshiftl((unsigned char *) replay->bitmap, diff, replay->wsize);
 			(replay->bitmap)[frlast] |= 1;
 		} else {
 			/* this packet has a "way larger" */
@@ -2456,14 +2569,18 @@ ipsec_updatereplay(seq, sav)
 		diff = replay->lastseq - seq;
 
 		/* over range to check, i.e. too old or wrapped */
-		if (diff >= wsizeb)
+		if (diff >= wsizeb) {
+			lck_mtx_unlock(sadb_mutex);
 			return 1;
+		}
 
 		fr = frlast - diff / 8;
 
 		/* this packet already seen ? */
-		if ((replay->bitmap)[fr] & (1 << (diff % 8)))
+		if ((replay->bitmap)[fr] & (1 << (diff % 8))) {
+			lck_mtx_unlock(sadb_mutex);
 			return 1;
+		}
 
 		/* mark as seen */
 		(replay->bitmap)[fr] |= (1 << (diff % 8));
@@ -2478,15 +2595,18 @@ ok:
 		replay->overflow++;
 
 		/* don't increment, no more packets accepted */
-		if ((sav->flags & SADB_X_EXT_CYCSEQ) == 0)
+		if ((sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
+			lck_mtx_unlock(sadb_mutex);
 			return 1;
+		}
 
 		ipseclog((LOG_WARNING, "replay counter made %d cycle. %s\n",
 		    replay->overflow, ipsec_logsastr(sav)));
 	}
 
 	replay->count++;
-
+	
+	lck_mtx_unlock(sadb_mutex);
 	return 0;
 }
 
@@ -2581,8 +2701,6 @@ ipsec_logsastr(sav)
 	char *p;
 	struct secasindex *saidx = &sav->sah->saidx;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* validity check */
 	if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family
 			!= ((struct sockaddr *)&sav->sah->saidx.dst)->sa_family)
@@ -2649,18 +2767,21 @@ ipsec_dumpmbuf(m)
  * IPsec output logic for IPv4.
  */
 int
-ipsec4_output(state, sp, flags)
-	struct ipsec_output_state *state;
-	struct secpolicy *sp;
-	int flags;
+ipsec4_output(
+	struct ipsec_output_state *state,
+	struct secpolicy *sp,
+	__unused int flags)
 {
 	struct ip *ip = NULL;
 	struct ipsecrequest *isr = NULL;
 	struct secasindex saidx;
-	int error;
+	struct secasvar *sav = NULL;
+	int error = 0;
 	struct sockaddr_in *dst4;
 	struct sockaddr_in *sin;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	if (!state)
 		panic("state == NULL in ipsec4_output");
 	if (!state->m)
@@ -2670,8 +2791,6 @@ ipsec4_output(state, sp, flags)
 	if (!state->dst)
 		panic("state->dst == NULL in ipsec4_output");
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	KERNEL_DEBUG(DBG_FNC_IPSEC_OUT | DBG_FUNC_START, 0,0,0,0,0);
 
 	KEYDEBUG(KEYDEBUG_IPSEC_DATA,
@@ -2709,11 +2828,42 @@ ipsec4_output(state, sp, flags)
 			sin->sin_len = sizeof(*sin);
 			sin->sin_family = AF_INET;
 			sin->sin_port = IPSEC_PORT_ANY;
+			/* 
+			* Get port from packet if upper layer is UDP and nat traversal 
+			* is enabled and transport mode.			
+			*/
+
+			if ((esp_udp_encap_port & 0xFFFF) != 0 &&
+				isr->saidx.mode == IPSEC_MODE_TRANSPORT) {	
+				
+				if (ip->ip_p == IPPROTO_UDP) {
+					struct udphdr  *udp;
+					size_t hlen;
+#ifdef _IP_VHL
+					hlen = IP_VHL_HL(ip->ip_vhl) << 2;
+#else
+					hlen = ip->ip_hl << 2;
+#endif
+					if (state->m->m_len < hlen + sizeof(struct udphdr)) {
+						state->m = m_pullup(state->m, hlen + sizeof(struct udphdr));
+						if (!state->m) {
+							ipseclog((LOG_DEBUG,
+								"IPv4 output: can't pullup UDP header\n"));
+							IPSEC_STAT_INCREMENT(ipsecstat.in_inval);
+							goto bad;
+						}
+						ip = mtod(state->m, struct ip *);
+					}
+					udp = (struct udphdr *)(((u_int8_t *)ip) + hlen);
+					sin->sin_port = udp->uh_dport;
+				}
+			}
+
 			bcopy(&ip->ip_dst, &sin->sin_addr,
 			    sizeof(sin->sin_addr));
 		}
 
-		if ((error = key_checkrequest(isr, &saidx)) != 0) {
+		if ((error = key_checkrequest(isr, &saidx, &sav)) != 0) {
 			/*
 			 * IPsec processing is required, but no SA found.
 			 * I assume that key_acquire() had been called
@@ -2721,12 +2871,12 @@ ipsec4_output(state, sp, flags)
 			 * this packet because it is responsibility for
 			 * upper layer to retransmit the packet.
 			 */
-			ipsecstat.out_nosa++;
+			IPSEC_STAT_INCREMENT(ipsecstat.out_nosa);
 			goto bad;
 		}
 
 		/* validity check */
-		if (isr->sav == NULL) {
+		if (sav == NULL) {
 			switch (ipsec_get_reqlevel(isr)) {
 			case IPSEC_LEVEL_USE:
 				continue;
@@ -2743,9 +2893,9 @@ ipsec4_output(state, sp, flags)
 		 * send to the receiver by dead SA, the receiver can
 		 * not decode a packet because SA has been dead.
 		 */
-		if (isr->sav->state != SADB_SASTATE_MATURE
-		 && isr->sav->state != SADB_SASTATE_DYING) {
-			ipsecstat.out_nosa++;
+		if (sav->state != SADB_SASTATE_MATURE
+		 && sav->state != SADB_SASTATE_DYING) {
+			IPSEC_STAT_INCREMENT(ipsecstat.out_nosa);
 			error = EINVAL;
 			goto bad;
 		}
@@ -2760,10 +2910,10 @@ ipsec4_output(state, sp, flags)
 			 * build IPsec tunnel.
 			 */
 			/* XXX should be processed with other familiy */
-			if (((struct sockaddr *)&isr->sav->sah->saidx.src)->sa_family != AF_INET) {
+			if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family != AF_INET) {
 				ipseclog((LOG_ERR, "ipsec4_output: "
 				    "family mismatched between inner and outer spi=%u\n",
-				    (u_int32_t)ntohl(isr->sav->spi)));
+				    (u_int32_t)ntohl(sav->spi)));
 				error = EAFNOSUPPORT;
 				goto bad;
 			}
@@ -2773,14 +2923,14 @@ ipsec4_output(state, sp, flags)
 				error = ENOMEM;
 				goto bad;
 			}
-			error = ipsec4_encapsulate(state->m, isr->sav);
+			error = ipsec4_encapsulate(state->m, sav);
 			if (error) {
 				state->m = NULL;
 				goto bad;
 			}
 			ip = mtod(state->m, struct ip *);
 
-			state->ro = &isr->sav->sah->sa_route;
+			state->ro = &sav->sah->sa_route;
 			state->dst = (struct sockaddr *)&state->ro->ro_dst;
 			dst4 = (struct sockaddr_in *)state->dst;
 			if (state->ro->ro_rt
@@ -2796,7 +2946,7 @@ ipsec4_output(state, sp, flags)
 				rtalloc(state->ro);
 			}
 			if (state->ro->ro_rt == 0) {
-				ipstat.ips_noroute++;
+				OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
 				error = EHOSTUNREACH;
 				goto bad;
 			}
@@ -2816,7 +2966,7 @@ ipsec4_output(state, sp, flags)
 		switch (isr->saidx.proto) {
 		case IPPROTO_ESP:
 #if IPSEC_ESP
-			if ((error = esp4_output(state->m, isr)) != 0) {
+			if ((error = esp4_output(state->m, sav)) != 0) {
 				state->m = NULL;
 				goto bad;
 			}
@@ -2828,13 +2978,13 @@ ipsec4_output(state, sp, flags)
 			goto bad;
 #endif
 		case IPPROTO_AH:
-			if ((error = ah4_output(state->m, isr)) != 0) {
+			if ((error = ah4_output(state->m, sav)) != 0) {
 				state->m = NULL;
 				goto bad;
 			}
 			break;
 		case IPPROTO_IPCOMP:
-			if ((error = ipcomp4_output(state->m, isr)) != 0) {
+			if ((error = ipcomp4_output(state->m, sav)) != 0) {
 				state->m = NULL;
 				goto bad;
 			}
@@ -2857,9 +3007,13 @@ ipsec4_output(state, sp, flags)
 	}
 
 	KERNEL_DEBUG(DBG_FNC_IPSEC_OUT | DBG_FUNC_END, 0,0,0,0,0);
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	return 0;
 
 bad:
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	m_freem(state->m);
 	state->m = NULL;
 	KERNEL_DEBUG(DBG_FNC_IPSEC_OUT | DBG_FUNC_END, error,0,0,0,0);
@@ -2872,13 +3026,13 @@ bad:
  * IPsec output logic for IPv6, transport mode.
  */
 int
-ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
-	struct ipsec_output_state *state;
-	u_char *nexthdrp;
-	struct mbuf *mprev;
-	struct secpolicy *sp;
-	int flags;
-	int *tun;
+ipsec6_output_trans(
+	struct ipsec_output_state *state,
+	u_char *nexthdrp,
+	struct mbuf *mprev,
+	struct secpolicy *sp,
+	__unused int flags,
+	int *tun)
 {
 	struct ip6_hdr *ip6;
 	struct ipsecrequest *isr = NULL;
@@ -2886,7 +3040,10 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 	int error = 0;
 	int plen;
 	struct sockaddr_in6 *sin6;
+	struct secasvar *sav = NULL;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	if (!state)
 		panic("state == NULL in ipsec6_output_trans");
 	if (!state->m)
@@ -2904,7 +3061,6 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 		printf("ipsec6_output_trans: applyed SP\n");
 		kdebug_secpolicy(sp));
 	
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	*tun = 0;
 	for (isr = sp->req; isr; isr = isr->next) {
 		if (isr->saidx.mode == IPSEC_MODE_TUNNEL) {
@@ -2944,7 +3100,7 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 			}
 		}
 
-		if (key_checkrequest(isr, &saidx) == ENOENT) {
+		if (key_checkrequest(isr, &saidx, &sav) == ENOENT) {
 			/*
 			 * IPsec processing is required, but no SA found.
 			 * I assume that key_acquire() had been called
@@ -2952,7 +3108,7 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 			 * this packet because it is responsibility for
 			 * upper layer to retransmit the packet.
 			 */
-			ipsec6stat.out_nosa++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_nosa);
 			error = ENOENT;
 
 			/*
@@ -2963,16 +3119,14 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 			 * XXX: should we directly notify sockets via
 			 *      pfctlinputs?
 			 */
-			lck_mtx_unlock(sadb_mutex);
 			icmp6_error(state->m, ICMP6_DST_UNREACH,
 				    ICMP6_DST_UNREACH_ADMIN, 0);
-			lck_mtx_lock(sadb_mutex);
 			state->m = NULL; /* icmp6_error freed the mbuf */
 			goto bad;
 		}
 
 		/* validity check */
-		if (isr->sav == NULL) {
+		if (sav == NULL) {
 			switch (ipsec_get_reqlevel(isr)) {
 			case IPSEC_LEVEL_USE:
 				continue;
@@ -2986,9 +3140,9 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 		 * If there is no valid SA, we give up to process.
 		 * see same place at ipsec4_output().
 		 */
-		if (isr->sav->state != SADB_SASTATE_MATURE
-		 && isr->sav->state != SADB_SASTATE_DYING) {
-			ipsec6stat.out_nosa++;
+		if (sav->state != SADB_SASTATE_MATURE
+		 && sav->state != SADB_SASTATE_DYING) {
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_nosa);
 			error = EINVAL;
 			goto bad;
 		}
@@ -2996,23 +3150,23 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 		switch (isr->saidx.proto) {
 		case IPPROTO_ESP:
 #if IPSEC_ESP
-			error = esp6_output(state->m, nexthdrp, mprev->m_next, isr);
+			error = esp6_output(state->m, nexthdrp, mprev->m_next, sav);
 #else
 			m_freem(state->m);
 			error = EINVAL;
 #endif
 			break;
 		case IPPROTO_AH:
-			error = ah6_output(state->m, nexthdrp, mprev->m_next, isr);
+			error = ah6_output(state->m, nexthdrp, mprev->m_next, sav);
 			break;
 		case IPPROTO_IPCOMP:
-			error = ipcomp6_output(state->m, nexthdrp, mprev->m_next, isr);
+			error = ipcomp6_output(state->m, nexthdrp, mprev->m_next, sav);
 			break;
 		default:
 			ipseclog((LOG_ERR, "ipsec6_output_trans: "
 			    "unknown ipsec protocol %d\n", isr->saidx.proto));
 			m_freem(state->m);
-			ipsec6stat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 			error = EINVAL;
 			break;
 		}
@@ -3024,7 +3178,7 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 		if (plen > IPV6_MAXPACKET) {
 			ipseclog((LOG_ERR, "ipsec6_output_trans: "
 			    "IPsec with IPv6 jumbogram is not supported\n"));
-			ipsec6stat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 			error = EINVAL;	/*XXX*/
 			goto bad;
 		}
@@ -3036,9 +3190,13 @@ ipsec6_output_trans(state, nexthdrp, mprev, sp, flags, tun)
 	if (isr != NULL)
 		*tun = 1;
 
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	return 0;
 
 bad:
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	m_freem(state->m);
 	state->m = NULL;
 	return error;
@@ -3048,18 +3206,24 @@ bad:
  * IPsec output logic for IPv6, tunnel mode.
  */
 int
-ipsec6_output_tunnel(state, sp, flags)
-	struct ipsec_output_state *state;
-	struct secpolicy *sp;
-	int flags;
+ipsec6_output_tunnel(
+	struct ipsec_output_state *state,
+	struct secpolicy *sp,
+	__unused int flags,
+	int *tunneledv4)
 {
 	struct ip6_hdr *ip6;
 	struct ipsecrequest *isr = NULL;
 	struct secasindex saidx;
+	struct secasvar *sav = NULL;
 	int error = 0;
 	int plen;
 	struct sockaddr_in6* dst6;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
+	*tunneledv4 = 0;
+	
 	if (!state)
 		panic("state == NULL in ipsec6_output_tunnel");
 	if (!state->m)
@@ -3067,8 +3231,6 @@ ipsec6_output_tunnel(state, sp, flags)
 	if (!sp)
 		panic("sp == NULL in ipsec6_output_tunnel");
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	KEYDEBUG(KEYDEBUG_IPSEC_DATA,
 		printf("ipsec6_output_tunnel: applyed SP\n");
 		kdebug_secpolicy(sp));
@@ -3124,7 +3286,7 @@ ipsec6_output_tunnel(state, sp, flags)
 			}
 		}
 
-		if (key_checkrequest(isr, &saidx) == ENOENT) {
+		if (key_checkrequest(isr, &saidx, &sav) == ENOENT) {
 			/*
 			 * IPsec processing is required, but no SA found.
 			 * I assume that key_acquire() had been called
@@ -3132,13 +3294,13 @@ ipsec6_output_tunnel(state, sp, flags)
 			 * this packet because it is responsibility for
 			 * upper layer to retransmit the packet.
 			 */
-			ipsec6stat.out_nosa++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_nosa);
 			error = ENOENT;
 			goto bad;
 		}
 
 		/* validity check */
-		if (isr->sav == NULL) {
+		if (sav == NULL) {
 			switch (ipsec_get_reqlevel(isr)) {
 			case IPSEC_LEVEL_USE:
 				continue;
@@ -3152,46 +3314,138 @@ ipsec6_output_tunnel(state, sp, flags)
 		 * If there is no valid SA, we give up to process.
 		 * see same place at ipsec4_output().
 		 */
-		if (isr->sav->state != SADB_SASTATE_MATURE
-		 && isr->sav->state != SADB_SASTATE_DYING) {
-			ipsec6stat.out_nosa++;
+		if (sav->state != SADB_SASTATE_MATURE
+		 && sav->state != SADB_SASTATE_DYING) {
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_nosa);
 			error = EINVAL;
 			goto bad;
 		}
 
-		/*
-		 * There may be the case that SA status will be changed when
-		 * we are refering to one. So calling splsoftnet().
-		 */
-
 		if (isr->saidx.mode == IPSEC_MODE_TUNNEL) {
 			/*
 			 * build IPsec tunnel.
 			 */
-			/* XXX should be processed with other familiy */
-			if (((struct sockaddr *)&isr->sav->sah->saidx.src)->sa_family != AF_INET6) {
-				ipseclog((LOG_ERR, "ipsec6_output_tunnel: "
-				    "family mismatched between inner and outer, spi=%u\n",
-				    (u_int32_t)ntohl(isr->sav->spi)));
-				ipsec6stat.out_inval++;
-				error = EAFNOSUPPORT;
-				goto bad;
-			}
-
 			state->m = ipsec6_splithdr(state->m);
 			if (!state->m) {
-				ipsec6stat.out_nomem++;
+				IPSEC_STAT_INCREMENT(ipsec6stat.out_nomem);
 				error = ENOMEM;
 				goto bad;
 			}
-			error = ipsec6_encapsulate(state->m, isr->sav);
-			if (error) {
-				state->m = 0;
+
+			if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family == AF_INET6) {			
+				error = ipsec6_encapsulate(state->m, sav);
+				if (error) {
+					state->m = 0;
+					goto bad;
+				}
+				ip6 = mtod(state->m, struct ip6_hdr *);
+			} else if (((struct sockaddr *)&sav->sah->saidx.src)->sa_family == AF_INET) {
+			
+				struct ip *ip;
+				struct sockaddr_in* dst4;
+				struct route *ro4 = NULL;
+
+				/*
+				 * must be last isr because encapsulated IPv6 packet
+				 * will be sent by calling ip_output
+				 */
+				if (isr->next) {
+					ipseclog((LOG_ERR, "ipsec6_output_tunnel: "
+				    	"IPv4 must be outer layer, spi=%u\n",
+				    	(u_int32_t)ntohl(sav->spi)));
+					error = EINVAL;
+					goto bad;
+				}
+				*tunneledv4 = 1; /* must not process any further in ip6_output */
+				error = ipsec64_encapsulate(state->m, sav);
+				if (error) {
+					state->m = 0;
+					goto bad;
+				}
+				/* Now we have an IPv4 packet */
+				ip = mtod(state->m, struct ip *);
+
+				ro4 = &sav->sah->sa_route;
+				dst4 = (struct sockaddr_in *)&ro4->ro_dst;
+				if (ro4->ro_rt
+				 && ((ro4->ro_rt->rt_flags & RTF_UP) == 0
+				  || dst4->sin_addr.s_addr != ip->ip_dst.s_addr)) {
+					rtfree(ro4->ro_rt);
+					ro4->ro_rt = NULL;
+				}
+				if (ro4->ro_rt == NULL) {
+					dst4->sin_family = AF_INET;
+					dst4->sin_len = sizeof(*dst4);
+					dst4->sin_addr = ip->ip_dst;
+					rtalloc(ro4);
+				}
+				if (ro4->ro_rt == NULL) {
+					OSAddAtomic(1, (SInt32*)&ipstat.ips_noroute);
+					error = EHOSTUNREACH;
+					goto bad;
+				}
+	
+				state->m = ipsec4_splithdr(state->m);
+				if (!state->m) {
+					error = ENOMEM;
+					goto bad;
+				}
+				switch (isr->saidx.proto) {
+				case IPPROTO_ESP:
+#if IPSEC_ESP
+					if ((error = esp4_output(state->m, sav)) != 0) {
+						state->m = NULL;
+						goto bad;
+					}
+					break;
+
+#else
+					m_freem(state->m);
+					state->m = NULL;
+					error = EINVAL;
+					goto bad;
+#endif
+				case IPPROTO_AH:
+					if ((error = ah4_output(state->m, sav)) != 0) {
+						state->m = NULL;
+						goto bad;
+					}
+					break;
+				case IPPROTO_IPCOMP:
+					if ((error = ipcomp4_output(state->m, sav)) != 0) {
+						state->m = NULL;
+						goto bad;
+					}
+					break;
+				default:
+					ipseclog((LOG_ERR,
+						"ipsec4_output: unknown ipsec protocol %d\n",
+						isr->saidx.proto));
+					m_freem(state->m);
+					state->m = NULL;
+					error = EINVAL;
+					goto bad;
+				}
+		
+				if (state->m == 0) {
+					error = ENOMEM;
+					goto bad;
+				}
+				ip = mtod(state->m, struct ip *);
+				ip->ip_len = ntohs(ip->ip_len);  /* flip len field before calling ip_output */
+				ip_output(state->m, NULL, ro4, 0, NULL, NULL);
+				state->m = NULL;
+				goto done;
+			} else {
+				ipseclog((LOG_ERR, "ipsec6_output_tunnel: "
+				    "unsupported inner family, spi=%u\n",
+				    (u_int32_t)ntohl(sav->spi)));
+				IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
+				error = EAFNOSUPPORT;
 				goto bad;
 			}
-			ip6 = mtod(state->m, struct ip6_hdr *);
-
-			state->ro = &isr->sav->sah->sa_route;
+			
+			state->ro = &sav->sah->sa_route;
 			state->dst = (struct sockaddr *)&state->ro->ro_dst;
 			dst6 = (struct sockaddr_in6 *)state->dst;
 			if (state->ro->ro_rt
@@ -3209,7 +3463,7 @@ ipsec6_output_tunnel(state, sp, flags)
 			}
 			if (state->ro->ro_rt == 0) {
 				ip6stat.ip6s_noroute++;
-				ipsec6stat.out_noroute++;
+				IPSEC_STAT_INCREMENT(ipsec6stat.out_noroute);
 				error = EHOSTUNREACH;
 				goto bad;
 			}
@@ -3223,7 +3477,7 @@ ipsec6_output_tunnel(state, sp, flags)
 
 		state->m = ipsec6_splithdr(state->m);
 		if (!state->m) {
-			ipsec6stat.out_nomem++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_nomem);
 			error = ENOMEM;
 			goto bad;
 		}
@@ -3231,14 +3485,14 @@ ipsec6_output_tunnel(state, sp, flags)
 		switch (isr->saidx.proto) {
 		case IPPROTO_ESP:
 #if IPSEC_ESP
-			error = esp6_output(state->m, &ip6->ip6_nxt, state->m->m_next, isr);
+			error = esp6_output(state->m, &ip6->ip6_nxt, state->m->m_next, sav);
 #else
 			m_freem(state->m);
 			error = EINVAL;
 #endif
 			break;
 		case IPPROTO_AH:
-			error = ah6_output(state->m, &ip6->ip6_nxt, state->m->m_next, isr);
+			error = ah6_output(state->m, &ip6->ip6_nxt, state->m->m_next, sav);
 			break;
 		case IPPROTO_IPCOMP:
 			/* XXX code should be here */
@@ -3247,7 +3501,7 @@ ipsec6_output_tunnel(state, sp, flags)
 			ipseclog((LOG_ERR, "ipsec6_output_tunnel: "
 			    "unknown ipsec protocol %d\n", isr->saidx.proto));
 			m_freem(state->m);
-			ipsec6stat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 			error = EINVAL;
 			break;
 		}
@@ -3259,18 +3513,23 @@ ipsec6_output_tunnel(state, sp, flags)
 		if (plen > IPV6_MAXPACKET) {
 			ipseclog((LOG_ERR, "ipsec6_output_tunnel: "
 			    "IPsec with IPv6 jumbogram is not supported\n"));
-			ipsec6stat.out_inval++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.out_inval);
 			error = EINVAL;	/*XXX*/
 			goto bad;
 		}
 		ip6 = mtod(state->m, struct ip6_hdr *);
 		ip6->ip6_plen = htons(plen);
 	}
-
+done:
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
 	return 0;
 
 bad:
-	m_freem(state->m);
+	if (sav)
+		key_freesav(sav, KEY_SADB_UNLOCKED);
+	if (state->m)
+		m_freem(state->m);
 	state->m = NULL;
 	return error;
 }
@@ -3297,7 +3556,7 @@ ipsec4_splithdr(m)
 	hlen = ip->ip_hl << 2;
 #endif
 	if (m->m_len > hlen) {
-		MGETHDR(mh, M_DONTWAIT, MT_HEADER);
+		MGETHDR(mh, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (!mh) {
 			m_freem(m);
 			return NULL;
@@ -3305,6 +3564,7 @@ ipsec4_splithdr(m)
 		M_COPY_PKTHDR(mh, m);
 		MH_ALIGN(mh, hlen);
 		m->m_flags &= ~M_PKTHDR;
+		m_mchtype(m, MT_DATA);
 		m->m_len -= hlen;
 		m->m_data += hlen;
 		mh->m_next = m;
@@ -3334,7 +3594,7 @@ ipsec6_splithdr(m)
 	ip6 = mtod(m, struct ip6_hdr *);
 	hlen = sizeof(struct ip6_hdr);
 	if (m->m_len > hlen) {
-		MGETHDR(mh, M_DONTWAIT, MT_HEADER);
+		MGETHDR(mh, M_DONTWAIT, MT_HEADER);	/* MAC-OK */
 		if (!mh) {
 			m_freem(m);
 			return NULL;
@@ -3342,6 +3602,7 @@ ipsec6_splithdr(m)
 		M_COPY_PKTHDR(mh, m);
 		MH_ALIGN(mh, hlen);
 		m->m_flags &= ~M_PKTHDR;
+		m_mchtype(m, MT_DATA);
 		m->m_len -= hlen;
 		m->m_data += hlen;
 		mh->m_next = m;
@@ -3359,26 +3620,28 @@ ipsec6_splithdr(m)
 
 /* validate inbound IPsec tunnel packet. */
 int
-ipsec4_tunnel_validate(m, off, nxt0, sav)
+ipsec4_tunnel_validate(m, off, nxt0, sav, ifamily)
 	struct mbuf *m;		/* no pullup permitted, m->m_len >= ip */
 	int off;
 	u_int nxt0;
 	struct secasvar *sav;
+	sa_family_t *ifamily;
 {
 	u_int8_t nxt = nxt0 & 0xff;
 	struct sockaddr_in *sin;
-	struct sockaddr_in osrc, odst, isrc, idst;
+	struct sockaddr_in osrc, odst, i4src, i4dst;
+	struct sockaddr_in6 i6src, i6dst;
 	int hlen;
 	struct secpolicy *sp;
 	struct ip *oip;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 
 #if DIAGNOSTIC
 	if (m->m_len < sizeof(struct ip))
 		panic("too short mbuf on ipsec4_tunnel_validate");
 #endif
-	if (nxt != IPPROTO_IPV4)
+	if (nxt != IPPROTO_IPV4 && nxt != IPPROTO_IPV6)
 		return 0;
 	if (m->m_pkthdr.len < off + sizeof(struct ip))
 		return 0;
@@ -3395,7 +3658,6 @@ ipsec4_tunnel_validate(m, off, nxt0, sav)
 	if (hlen != sizeof(struct ip))
 		return 0;
 
-	/* AF_INET6 should be supported, but at this moment we don't. */
 	sin = (struct sockaddr_in *)&sav->sah->saidx.dst;
 	if (sin->sin_family != AF_INET)
 		return 0;
@@ -3405,19 +3667,10 @@ ipsec4_tunnel_validate(m, off, nxt0, sav)
 	/* XXX slow */
 	bzero(&osrc, sizeof(osrc));
 	bzero(&odst, sizeof(odst));
-	bzero(&isrc, sizeof(isrc));
-	bzero(&idst, sizeof(idst));
-	osrc.sin_family = odst.sin_family = isrc.sin_family = idst.sin_family = 
-	    AF_INET;
-	osrc.sin_len = odst.sin_len = isrc.sin_len = idst.sin_len = 
-	    sizeof(struct sockaddr_in);
+	osrc.sin_family = odst.sin_family = AF_INET;
+	osrc.sin_len = odst.sin_len = sizeof(struct sockaddr_in);
 	osrc.sin_addr = oip->ip_src;
 	odst.sin_addr = oip->ip_dst;
-	m_copydata(m, off + offsetof(struct ip, ip_src), sizeof(isrc.sin_addr),
-	    (caddr_t)&isrc.sin_addr);
-	m_copydata(m, off + offsetof(struct ip, ip_dst), sizeof(idst.sin_addr),
-	    (caddr_t)&idst.sin_addr);
-
 	/*
 	 * RFC2401 5.2.1 (b): (assume that we are using tunnel mode)
 	 * - if the inner destination is multicast address, there can be
@@ -3438,13 +3691,35 @@ ipsec4_tunnel_validate(m, off, nxt0, sav)
 	 *
 	 * therefore, we do not do anything special about inner source.
 	 */
-
-	sp = key_gettunnel((struct sockaddr *)&osrc, (struct sockaddr *)&odst,
-	    (struct sockaddr *)&isrc, (struct sockaddr *)&idst);
-	if (!sp) {
+	if (nxt == IPPROTO_IPV4) {
+		bzero(&i4src, sizeof(struct sockaddr_in));
+		bzero(&i4dst, sizeof(struct sockaddr_in));
+		i4src.sin_family = i4dst.sin_family = *ifamily = AF_INET;
+		i4src.sin_len = i4dst.sin_len = sizeof(struct sockaddr_in);
+		m_copydata(m, off + offsetof(struct ip, ip_src), sizeof(i4src.sin_addr),
+			   (caddr_t)&i4src.sin_addr);
+		m_copydata(m, off + offsetof(struct ip, ip_dst), sizeof(i4dst.sin_addr),
+			   (caddr_t)&i4dst.sin_addr);
+		sp = key_gettunnel((struct sockaddr *)&osrc, (struct sockaddr *)&odst,
+				   (struct sockaddr *)&i4src, (struct sockaddr *)&i4dst);
+	} else if (nxt == IPPROTO_IPV6) {
+		bzero(&i6src, sizeof(struct sockaddr_in6));
+		bzero(&i6dst, sizeof(struct sockaddr_in6));
+		i6src.sin6_family = i6dst.sin6_family = *ifamily = AF_INET6;
+		i6src.sin6_len = i6dst.sin6_len = sizeof(struct sockaddr_in6);
+		m_copydata(m, off + offsetof(struct ip6_hdr, ip6_src), sizeof(i6src.sin6_addr),
+			   (caddr_t)&i6src.sin6_addr);
+		m_copydata(m, off + offsetof(struct ip6_hdr, ip6_dst), sizeof(i6dst.sin6_addr),
+			   (caddr_t)&i6dst.sin6_addr);
+		sp = key_gettunnel((struct sockaddr *)&osrc, (struct sockaddr *)&odst,
+				   (struct sockaddr *)&i6src, (struct sockaddr *)&i6dst);
+	} else 
+		return 0;	/* unsupported family */
+
+	if (!sp) 
 		return 0;
-	}
-	key_freesp(sp);
+
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return 1;
 }
@@ -3464,8 +3739,8 @@ ipsec6_tunnel_validate(m, off, nxt0, sav)
 	struct secpolicy *sp;
 	struct ip6_hdr *oip6;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 #if DIAGNOSTIC
 	if (m->m_len < sizeof(struct ip6_hdr))
 		panic("too short mbuf on ipsec6_tunnel_validate");
@@ -3519,7 +3794,7 @@ ipsec6_tunnel_validate(m, off, nxt0, sav)
 	 */
 	if (!sp)
 		return 0;
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_UNLOCKED);
 
 	return 1;
 }
@@ -3554,17 +3829,10 @@ ipsec_copypkt(m)
 				struct mbuf *mm;
 
 				if (n->m_flags & M_PKTHDR) {
-					MGETHDR(mnew, M_DONTWAIT, MT_HEADER);
+					MGETHDR(mnew, M_DONTWAIT, MT_HEADER); /* MAC-OK */
 					if (mnew == NULL)
 						goto fail;
 					mnew->m_pkthdr = n->m_pkthdr;
-#if 0
-					if (n->m_pkthdr.aux) {
-						mnew->m_pkthdr.aux =
-						    m_copym(n->m_pkthdr.aux,
-						    0, M_COPYALL, M_DONTWAIT);
-					}
-#endif
 					M_COPY_PKTHDR(mnew, n);
 					mnew->m_flags = n->m_flags & M_COPYFLAGS;
 				}
@@ -3613,7 +3881,7 @@ ipsec_copypkt(m)
 						break;
 
 					/* need another mbuf */
-					MGETHDR(mn, M_DONTWAIT, MT_HEADER);
+					MGETHDR(mn, M_DONTWAIT, MT_HEADER);	/* XXXMAC: tags copied next time in loop? */
 					if (mn == NULL)
 						goto fail;
 					mn->m_pkthdr.rcvif = NULL;
@@ -3640,148 +3908,172 @@ ipsec_copypkt(m)
 	return(NULL);
 }
 
-static struct mbuf *
-ipsec_addaux(m)
-	struct mbuf *m;
+/*
+ * Tags are allocated as mbufs for now, since our minimum size is MLEN, we
+ * should make use of up to that much space.
+ */
+#define	IPSEC_TAG_HEADER \
+
+struct ipsec_tag {
+	struct socket			*socket;
+	u_int32_t				history_count;
+	struct ipsec_history	history[];
+};
+
+#define	IPSEC_TAG_SIZE		(MLEN - sizeof(struct m_tag))
+#define	IPSEC_TAG_HDR_SIZE	(offsetof(struct ipsec_tag, history[0]))
+#define IPSEC_HISTORY_MAX	((IPSEC_TAG_SIZE - IPSEC_TAG_HDR_SIZE) / \
+							 sizeof(struct ipsec_history))
+
+static struct ipsec_tag *
+ipsec_addaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = m_aux_find(m, AF_INET, IPPROTO_ESP);
-	if (!n)
-		n = m_aux_add(m, AF_INET, IPPROTO_ESP);
-	if (!n)
-		return n;	/* ENOBUFS */
-	n->m_len = sizeof(struct socket *);
-	bzero(mtod(n, void *), n->m_len);
-	return n;
+	struct m_tag		*tag;
+	
+	/* Check if the tag already exists */
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPSEC, NULL);
+	
+	if (tag == NULL) {
+		struct ipsec_tag	*itag;
+		
+		/* Allocate a tag */
+		tag = m_tag_alloc(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPSEC,
+						  IPSEC_TAG_SIZE, M_DONTWAIT);
+		
+		if (tag) {
+			itag = (struct ipsec_tag*)(tag + 1);
+			itag->socket = 0;
+			itag->history_count = 0;
+			
+			m_tag_prepend(m, tag);
+		}
+	}
+	
+	return tag ? (struct ipsec_tag*)(tag + 1) : NULL;
 }
 
-static struct mbuf *
-ipsec_findaux(m)
-	struct mbuf *m;
+static struct ipsec_tag *
+ipsec_findaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = m_aux_find(m, AF_INET, IPPROTO_ESP);
-#if DIAGNOSTIC
-	if (n && n->m_len < sizeof(struct socket *))
-		panic("invalid ipsec m_aux");
-#endif
-	return n;
+	struct m_tag	*tag;
+	
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPSEC, NULL);
+	
+	return tag ? (struct ipsec_tag*)(tag + 1) : NULL;
 }
 
 void
-ipsec_delaux(m)
-	struct mbuf *m;
+ipsec_delaux(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = m_aux_find(m, AF_INET, IPPROTO_ESP);
-	if (n)
-		m_aux_delete(m, n);
+	struct m_tag	*tag;
+	
+	tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_IPSEC, NULL);
+	
+	if (tag) {
+		m_tag_delete(m, tag);
+	}
 }
 
 /* if the aux buffer is unnecessary, nuke it. */
 static void
-ipsec_optaux(m, n)
-	struct mbuf *m;
-	struct mbuf *n;
+ipsec_optaux(
+	struct mbuf			*m,
+	struct ipsec_tag	*itag)
 {
-
-	if (!n)
-		return;
-	if (n->m_len == sizeof(struct socket *) && !*mtod(n, struct socket **))
-		ipsec_delaux(m);
+	if (itag && itag->socket == NULL && itag->history_count == 0) {
+		m_tag_delete(m, ((struct m_tag*)itag) - 1);
+	}
 }
 
 int
-ipsec_setsocket(m, so)
-	struct mbuf *m;
-	struct socket *so;
+ipsec_setsocket(
+	struct mbuf *m,
+	struct socket *so)
 {
-	struct mbuf *n;
+	struct ipsec_tag	*tag;
 
 	/* if so == NULL, don't insist on getting the aux mbuf */
 	if (so) {
-		n = ipsec_addaux(m);
-		if (!n)
+		tag = ipsec_addaux(m);
+		if (!tag)
 			return ENOBUFS;
 	} else
-		n = ipsec_findaux(m);
-	if (n && n->m_len >= sizeof(struct socket *))
-		*mtod(n, struct socket **) = so;
-	ipsec_optaux(m, n);
+		tag = ipsec_findaux(m);
+	if (tag) {
+		tag->socket = so;
+		ipsec_optaux(m, tag);
+	}
 	return 0;
 }
 
 struct socket *
-ipsec_getsocket(m)
-	struct mbuf *m;
+ipsec_getsocket(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = ipsec_findaux(m);
-	if (n && n->m_len >= sizeof(struct socket *))
-		return *mtod(n, struct socket **);
+	struct ipsec_tag	*itag;
+	
+	itag = ipsec_findaux(m);
+	if (itag)
+		return itag->socket;
 	else
 		return NULL;
 }
 
 int
-ipsec_addhist(m, proto, spi)
-	struct mbuf *m;
-	int proto;
-	u_int32_t spi;
+ipsec_addhist(
+	struct mbuf *m,
+	int proto,
+	u_int32_t spi)
 {
-	struct mbuf *n;
-	struct ipsec_history *p;
-
-	n = ipsec_addaux(m);
-	if (!n)
+	struct ipsec_tag		*itag;
+	struct ipsec_history	*p;
+	itag = ipsec_addaux(m);
+	if (!itag)
 		return ENOBUFS;
-	if (M_TRAILINGSPACE(n) < sizeof(*p))
+	if (itag->history_count == IPSEC_HISTORY_MAX)
 		return ENOSPC;	/* XXX */
-	p = (struct ipsec_history *)(mtod(n, caddr_t) + n->m_len);
-	n->m_len += sizeof(*p);
+	
+	p = &itag->history[itag->history_count];
+	itag->history_count++;
+	
 	bzero(p, sizeof(*p));
 	p->ih_proto = proto;
 	p->ih_spi = spi;
+	
 	return 0;
 }
 
 struct ipsec_history *
-ipsec_gethist(m, lenp)
-	struct mbuf *m;
-	int *lenp;
+ipsec_gethist(
+	struct mbuf *m,
+	int *lenp)
 {
-	struct mbuf *n;
-	int l;
-
-	n = ipsec_findaux(m);
-	if (!n)
-		return NULL;
-	l = n->m_len;
-	if (sizeof(struct socket *) > l)
+	struct ipsec_tag	*itag;
+	
+	itag = ipsec_findaux(m);
+	if (!itag)
 		return NULL;
-	if ((l - sizeof(struct socket *)) % sizeof(struct ipsec_history))
+	if (itag->history_count == 0)
 		return NULL;
-	/* XXX does it make more sense to divide by sizeof(ipsec_history)? */
 	if (lenp)
-		*lenp = l - sizeof(struct socket *);
-	return (struct ipsec_history *)
-	    (mtod(n, caddr_t) + sizeof(struct socket *));
+		*lenp = (int)(itag->history_count * sizeof(struct ipsec_history));
+	return itag->history;
 }
 
 void
-ipsec_clearhist(m)
-	struct mbuf *m;
+ipsec_clearhist(
+	struct mbuf *m)
 {
-	struct mbuf *n;
-
-	n = ipsec_findaux(m);
-	if ((n) && n->m_len > sizeof(struct socket *))
-		n->m_len = sizeof(struct socket *);
-	ipsec_optaux(m, n);
+	struct ipsec_tag	*itag;
+	
+	itag = ipsec_findaux(m);
+	if (itag) {
+		itag->history_count = 0;
+	}
+	ipsec_optaux(m, itag);
 }
 
 __private_extern__ void
@@ -3793,10 +4085,10 @@ ipsec_send_natt_keepalive(
 	struct ip *ip;
 	int error;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	if ((esp_udp_encap_port & 0xFFFF) == 0 || sav->remote_ike_port == 0) return;
 	
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	m = m_gethdr(M_NOWAIT, MT_DATA);
 	if (m == NULL) return;
 	
@@ -3824,9 +4116,7 @@ ipsec_send_natt_keepalive(
 	uh->uh_sum = 0;
 	*(u_int8_t*)((char*)m_mtod(m) + sizeof(struct ip) + sizeof(struct udphdr)) = 0xFF;
 	
-	lck_mtx_unlock(sadb_mutex);
-	error = ip_output(m, NULL, &sav->sah->sa_route, IP_NOIPSEC, NULL);
-	lck_mtx_lock(sadb_mutex);
+	error = ip_output(m, NULL, &sav->sah->sa_route, IP_NOIPSEC, NULL, NULL);
 	if (error == 0)
 		sav->natt_last_activity = natt_now;
 
diff --git a/bsd/netinet6/ipsec.h b/bsd/netinet6/ipsec.h
index 154ff39a1..0fba00274 100644
--- a/bsd/netinet6/ipsec.h
+++ b/bsd/netinet6/ipsec.h
@@ -42,6 +42,17 @@
 #ifdef KERNEL_PRIVATE
 #include <netkey/keydb.h>
 
+/* lock for IPSec stats */
+lck_grp_t         *sadb_stat_mutex_grp;
+lck_grp_attr_t    *sadb_stat_mutex_grp_attr;
+lck_attr_t        *sadb_stat_mutex_attr;
+lck_mtx_t         *sadb_stat_mutex;
+
+
+#define IPSEC_STAT_INCREMENT(x)	\
+	{lck_mtx_lock(sadb_stat_mutex); (x)++; lck_mtx_unlock(sadb_stat_mutex);}
+
+
 /*
  * Security Policy Index
  * Ensure that both address families in the "src" and "dst" are same.
@@ -101,7 +112,6 @@ struct ipsecrequest {
 				/* if __ss_len == 0 then no address specified.*/
 	u_int level;		/* IPsec level defined below. */
 
-	struct secasvar *sav;	/* place holder of SA for use */
 	struct secpolicy *sp;	/* back pointer to SP */
 };
 
@@ -157,6 +167,7 @@ struct secspacq {
 #define IPSEC_POLICY_IPSEC	2	/* do IPsec */
 #define IPSEC_POLICY_ENTRUST	3	/* consulting SPD if present. */
 #define IPSEC_POLICY_BYPASS	4	/* only for privileged socket. */
+#define IPSEC_POLICY_GENERATE   5       /* same as discard - IKE daemon can override with generated policy */
 
 /* Security protocol level */
 #define	IPSEC_LEVEL_DEFAULT	0	/* reference to system default */
@@ -318,6 +329,7 @@ extern int ipsec_updatereplay(u_int32_t, struct secasvar *);
 
 extern size_t ipsec4_hdrsiz(struct mbuf *, u_int, struct inpcb *);
 extern size_t ipsec_hdrsiz_tcp(struct tcpcb *);
+extern size_t ipsec_hdrsiz(struct secpolicy *);
 
 struct ip;
 extern const char *ipsec4_logpacketstr(struct ip *, u_int32_t);
@@ -326,7 +338,7 @@ extern const char *ipsec_logsastr(struct secasvar *);
 extern void ipsec_dumpmbuf(struct mbuf *);
 
 extern int ipsec4_output(struct ipsec_output_state *, struct secpolicy *, int);
-extern int ipsec4_tunnel_validate(struct mbuf *, int, u_int, struct secasvar *);
+extern int ipsec4_tunnel_validate(struct mbuf *, int, u_int, struct secasvar *, sa_family_t *);
 extern struct mbuf *ipsec_copypkt(struct mbuf *);
 extern void ipsec_delaux(struct mbuf *);
 extern int ipsec_setsocket(struct mbuf *, struct socket *);
diff --git a/bsd/netinet6/ipsec6.h b/bsd/netinet6/ipsec6.h
index 33bcfe125..319670313 100644
--- a/bsd/netinet6/ipsec6.h
+++ b/bsd/netinet6/ipsec6.h
@@ -76,7 +76,7 @@ extern const char *ipsec6_logpacketstr(struct ip6_hdr *, u_int32_t);
 extern int ipsec6_output_trans(struct ipsec_output_state *, u_char *,
 	struct mbuf *, struct secpolicy *, int, int *);
 extern int ipsec6_output_tunnel(struct ipsec_output_state *,
-	struct secpolicy *, int);
+				struct secpolicy *, int, int*);
 extern int ipsec6_tunnel_validate(struct mbuf *, int, u_int,
 	struct secasvar *);
 #endif KERNEL_PRIVATE
diff --git a/bsd/netinet6/mld6.c b/bsd/netinet6/mld6.c
index df44cb746..afc28c6fd 100644
--- a/bsd/netinet6/mld6.c
+++ b/bsd/netinet6/mld6.c
@@ -68,7 +68,12 @@
  *
  *	@(#)igmp.c	8.1 (Berkeley) 7/19/93
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -88,6 +93,10 @@
 
 #include <net/net_osdep.h>
 
+#if CONFIG_MACF_NET
+#include <security/mac.h>
+#endif /* MAC_NET */
+
 /*
  * Protocol constants
  */
@@ -141,8 +150,6 @@ void
 mld6_start_listening(
 	struct in6_multi *in6m)
 {
-	int s = splnet();
-
 	/*
 	 * RFC2710 page 10:
 	 * The node never sends a Report or Done for the link-scope all-nodes
@@ -163,7 +170,6 @@ mld6_start_listening(
 		in6m->in6m_state = MLD6_IREPORTEDLAST;
 		mld6_timers_are_running = 1;
 	}
-	splx(s);
 }
 
 void
@@ -418,6 +424,11 @@ mld6_sendpkt(
 	mh->m_next = md;
 
 	mh->m_pkthdr.rcvif = NULL;
+#ifdef __darwin8_notyet
+#if CONFIG_MACF_NET
+	mac_create_mbuf_linklayer(in6m->in6m_ifp, m);
+#endif
+#endif
 	mh->m_pkthdr.len = sizeof(struct ip6_hdr) + sizeof(struct mld6_hdr);
 	mh->m_len = sizeof(struct ip6_hdr);
 	MH_ALIGN(mh, sizeof(struct ip6_hdr));
@@ -478,3 +489,4 @@ mld6_sendpkt(
 		}
 	}
 }
+
diff --git a/bsd/netinet6/nd6.c b/bsd/netinet6/nd6.c
index 228c6fa2e..1bee938d4 100644
--- a/bsd/netinet6/nd6.c
+++ b/bsd/netinet6/nd6.c
@@ -46,6 +46,7 @@
 #include <sys/sockio.h>
 #include <sys/time.h>
 #include <sys/kernel.h>
+#include <sys/sysctl.h>
 #include <sys/errno.h>
 #include <sys/syslog.h>
 #include <sys/protosw.h>
@@ -102,7 +103,7 @@ int nd6_debug = 0;
 /* for debugging? */
 static int nd6_inuse, nd6_allocated;
 
-struct llinfo_nd6 llinfo_nd6 = {&llinfo_nd6, &llinfo_nd6};
+struct llinfo_nd6 llinfo_nd6 = {&llinfo_nd6, &llinfo_nd6, NULL, NULL, 0, 0, 0, 0, 0 };
 size_t nd_ifinfo_indexlim = 8;
 struct nd_ifinfo *nd_ifinfo = NULL;
 struct nd_drhead nd_defrouter;
@@ -198,15 +199,10 @@ nd6_ifattach(
  * changes, which means we might have to adjust the ND level MTU.
  */
 void
-nd6_setmtu(
-	struct ifnet *ifp)
+nd6_setmtu(struct ifnet *ifp)
 {
-#ifndef MIN
-#define MIN(a,b) ((a) < (b) ? (a) : (b))
-#endif
- 
 	struct nd_ifinfo *ndi;
-	u_long oldmaxmtu, oldlinkmtu;
+	u_long oldmaxmtu;
 
 	/*
 	 * Make sure IPv6 is enabled for the interface first, 
@@ -219,60 +215,35 @@ nd6_setmtu(
 
 	ndi = &nd_ifinfo[ifp->if_index];
 	oldmaxmtu = ndi->maxmtu;
-	oldlinkmtu = ndi->linkmtu;
 
-	switch (ifp->if_type) {
-	case IFT_ARCNET:	/* XXX MTU handling needs more work */
-		ndi->maxmtu = MIN(60480, ifp->if_mtu);
-		break;
-	case IFT_L2VLAN:	/* XXX what if the VLAN isn't over ethernet? */
-	case IFT_IEEE8023ADLAG:
-	case IFT_ETHER:
-		ndi->maxmtu = MIN(ETHERMTU, ifp->if_mtu);
-		break;
-	case IFT_FDDI:
-		ndi->maxmtu = MIN(FDDIIPMTU, ifp->if_mtu);
-		break;
-	case IFT_ATM:
-		ndi->maxmtu = MIN(ATMMTU, ifp->if_mtu);
-		break;
-	case IFT_IEEE1394:	/* XXX should be IEEE1394MTU(1500) */
-		ndi->maxmtu = MIN(ETHERMTU, ifp->if_mtu);
-		break;
-#if IFT_IEEE80211
-	case IFT_IEEE80211:	/* XXX should be IEEE80211MTU(1500) */
-		ndi->maxmtu = MIN(ETHERMTU, ifp->if_mtu);
-		break;
-#endif
-	default:
-		ndi->maxmtu = ifp->if_mtu;
-		break;
-	}
+	/*
+	 * The ND level maxmtu is somewhat redundant to the interface MTU
+	 * and is an implementation artifact of KAME.  Instead of hard-
+	 * limiting the maxmtu based on the interface type here, we simply
+	 * take the if_mtu value since SIOCSIFMTU would have taken care of
+	 * the sanity checks related to the maximum MTU allowed for the
+	 * interface (a value that is known only by the interface layer),
+	 * by sending the request down via ifnet_ioctl().  The use of the
+	 * ND level maxmtu and linkmtu (the latter obtained via RA) are done
+	 * via IN6_LINKMTU() which does further checking against if_mtu.
+	 */
+	ndi->maxmtu = ifp->if_mtu;
 
-	if (oldmaxmtu != ndi->maxmtu) {
-		/*
-		 * If the ND level MTU is not set yet, or if the maxmtu
-		 * is reset to a smaller value than the ND level MTU,
-		 * also reset the ND level MTU.
-		 */
-		if (ndi->linkmtu == 0 ||
-		    ndi->maxmtu < ndi->linkmtu) {
-			ndi->linkmtu = ndi->maxmtu;
-			/* also adjust in6_maxmtu if necessary. */
-			if (oldlinkmtu == 0) {
-				/*
-				 * XXX: the case analysis is grotty, but
-				 * it is not efficient to call in6_setmaxmtu()
-				 * here when we are during the initialization
-				 * procedure.
-				 */
-				if (in6_maxmtu < ndi->linkmtu)
-					in6_maxmtu = ndi->linkmtu;
-			} else
-				in6_setmaxmtu();
-		}
+	/*
+	* Decreasing the interface MTU under IPV6 minimum MTU may cause
+	* undesirable situation.  We thus notify the operator of the change
+	* explicitly.  The check for oldmaxmtu is necessary to restrict the
+	* log to the case of changing the MTU, not initializing it.
+	*/
+	if (oldmaxmtu >= IPV6_MMTU && ndi->maxmtu < IPV6_MMTU) {
+		log(LOG_NOTICE, "nd6_setmtu: "
+		    "new link MTU on %s%d (%lu) is too small for IPv6\n",
+		    ifp->if_name, ifp->if_unit, (unsigned long)ndi->maxmtu);
 	}
-#undef MIN
+
+	/* also adjust in6_maxmtu if necessary. */
+	if (ndi->maxmtu > in6_maxmtu)
+		in6_setmaxmtu();
 }
 
 void
@@ -429,7 +400,7 @@ skip1:
  */
 void
 nd6_timer(
-	void	*ignored_arg)
+	__unused void	*ignored_arg)
 {
 	struct llinfo_nd6 *ln;
 	struct nd_defrouter *dr;
@@ -597,8 +568,16 @@ nd6_timer(
 			 */
 			if (ip6_use_tempaddr &&
 			    (ia6->ia6_flags & IN6_IFF_TEMPORARY) != 0) {
-				if (regen_tmpaddr(ia6) == 0)
+				/* NOTE: We have to drop the lock here because 
+				 * regen_tmpaddr() eventually calls in6_update_ifa(),  
+				 * which must take the lock and would otherwise cause a 
+				 * hang. This is safe because the goto addrloop 
+				 * leads to a reevaluation of the in6_ifaddrs list
+				 */
+				lck_mtx_unlock(nd6_mutex);
+				if (regen_tmpaddr(ia6) == 0) 
 					regen = 1;
+				lck_mtx_lock(nd6_mutex);
 			}
 
 			in6_purgeaddr(&ia6->ia_ifa, 1);
@@ -619,6 +598,8 @@ nd6_timer(
 			    (ia6->ia6_flags & IN6_IFF_TEMPORARY) != 0 &&
 			    (oldflags & IN6_IFF_DEPRECATED) == 0) {
 
+				/* see NOTE above */
+				lck_mtx_unlock(nd6_mutex);
 				if (regen_tmpaddr(ia6) == 0) {
 					/*
 					 * A new temporary address is
@@ -632,8 +613,10 @@ nd6_timer(
 					 * loop just for safety.  Or does this 
 					 * significantly reduce performance??
 					 */
+					lck_mtx_lock(nd6_mutex);
 					goto addrloop;
 				}
+				lck_mtx_lock(nd6_mutex);
 			}
 		} else {
 			/*
@@ -919,9 +902,8 @@ nd6_lookup(
 	 *      use rt->rt_ifa->ifa_ifp, which would specify the REAL
 	 *      interface.
 	 */
-	if ((ifp->if_type !=IFT_PPP) && ((rt->rt_flags & RTF_GATEWAY) || (rt->rt_flags & RTF_LLINFO) == 0 ||
+	if ((ifp && ifp->if_type !=IFT_PPP) && ((rt->rt_flags & RTF_GATEWAY) || (rt->rt_flags & RTF_LLINFO) == 0 ||
 	    rt->rt_gateway->sa_family != AF_LINK ||  rt->rt_llinfo == NULL ||
-
 	    (ifp && rt->rt_ifa->ifa_ifp != ifp))) {
 		if (!rt_locked)
 			lck_mtx_unlock(rt_mtx);
@@ -1149,11 +1131,12 @@ void
 nd6_rtrequest(
 	int	req,
 	struct rtentry *rt,
-	struct sockaddr *sa) /* xxx unused */
+	__unused struct sockaddr *sa)
 {
 	struct sockaddr *gate = rt->rt_gateway;
 	struct llinfo_nd6 *ln = (struct llinfo_nd6 *)rt->rt_llinfo;
-	static struct sockaddr_dl null_sdl = {sizeof(null_sdl), AF_LINK};
+	static struct sockaddr_dl null_sdl = {sizeof(null_sdl), AF_LINK, 0, 0, 0, 0, 0, 
+											{0,0,0,0,0,0,0,0,0,0,0,0,} };
 	struct ifnet *ifp = rt->rt_ifp;
 	struct ifaddr *ifa;
 	struct timeval timenow;
@@ -1330,7 +1313,7 @@ nd6_rtrequest(
 				SDL(gate)->sdl_alen = ifp->if_addrlen;
 			}
 			if (nd6_useloopback) {
-				rt->rt_ifp = &loif[0];	/* XXX */
+				rt->rt_ifp = lo_ifp;	/* XXX */
 				/*
 				 * Make sure rt_ifa be equal to the ifaddr
 				 * corresponding to the address.
@@ -1526,7 +1509,7 @@ nd6_ioctl(
 			error = EINVAL;
 			break;
 		}
-		ndi->ndi.linkmtu = nd_ifinfo[ifp->if_index].linkmtu;
+		ndi->ndi.linkmtu = IN6_LINKMTU(ifp);
 		ndi->ndi.maxmtu = nd_ifinfo[ifp->if_index].maxmtu;
 		ndi->ndi.basereachable =
 		    nd_ifinfo[ifp->if_index].basereachable;
@@ -1568,7 +1551,7 @@ nd6_ioctl(
 	case SIOCSPFXFLUSH_IN6:
 	    {
 		/* flush all the prefix advertised by routers */
-		struct nd_prefix *pr, *next;
+		struct nd_prefix *next;
 		lck_mtx_lock(nd6_mutex);
 
 		for (pr = nd_prefix.lh_first; pr; pr = next) {
@@ -1598,7 +1581,7 @@ nd6_ioctl(
 	case SIOCSRTRFLUSH_IN6:
 	    {
 		/* flush all the default routers */
-		struct nd_defrouter *dr, *next;
+		struct nd_defrouter *next;
 
 		lck_mtx_lock(nd6_mutex);
 		if ((dr = TAILQ_FIRST(&nd_defrouter)) != NULL) {
@@ -1663,7 +1646,7 @@ nd6_cache_lladdr(
 	struct ifnet *ifp,
 	struct in6_addr *from,
 	char *lladdr,
-	int lladdrlen,
+	__unused int lladdrlen,
 	int type,	/* ICMP6 type */
 	int code)	/* type dependent information */
 {
@@ -1901,7 +1884,7 @@ fail:
 
 static void
 nd6_slowtimo(
-    void *ignored_arg)
+    __unused void *ignored_arg)
 {
 	int i;
 	struct nd_ifinfo *nd6if;
@@ -2270,7 +2253,7 @@ nd6_lookup_ipv6(
 	
 	if ((packet->m_flags & M_MCAST) != 0) {
 		return dlil_resolve_multi(ifp, (const struct sockaddr*)ip6_dest,
-								  ll_dest, ll_dest_len);
+								  (struct sockaddr *)ll_dest, ll_dest_len);
 	}
 	
 	if (route == NULL) {
@@ -2301,18 +2284,12 @@ done:
 	return result;
 }
 
-#ifndef __APPLE__
-static int nd6_sysctl_drlist SYSCTL_HANDLER_ARGS;
-static int nd6_sysctl_prlist SYSCTL_HANDLER_ARGS;
 SYSCTL_DECL(_net_inet6_icmp6);
-SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_DRLIST, nd6_drlist,
-	CTLFLAG_RD, nd6_sysctl_drlist, "");
-SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_PRLIST, nd6_prlist,
-	CTLFLAG_RD, nd6_sysctl_prlist, "");
 
 static int
 nd6_sysctl_drlist SYSCTL_HANDLER_ARGS 
 {
+#pragma unused(oidp, arg1, arg2)
 	int error;
 	char buf[1024];
 	struct in6_defrouter *d, *de;
@@ -2357,6 +2334,7 @@ nd6_sysctl_drlist SYSCTL_HANDLER_ARGS
 static int
 nd6_sysctl_prlist SYSCTL_HANDLER_ARGS 
 {
+#pragma unused(oidp, arg1, arg2)
 	int error;
 	char buf[1024];
 	struct in6_prefix *p, *pe;
@@ -2369,7 +2347,7 @@ nd6_sysctl_prlist SYSCTL_HANDLER_ARGS
 	lck_mtx_lock(nd6_mutex);
 
 	for (pr = nd_prefix.lh_first; pr; pr = pr->ndpr_next) {
-		u_short advrtrs;
+		u_short advrtrs = 0;
 		size_t advance;
 		struct sockaddr_in6 *sin6, *s6;
 		struct nd_pfxrouter *pfr;
@@ -2430,4 +2408,8 @@ nd6_sysctl_prlist SYSCTL_HANDLER_ARGS
 	lck_mtx_unlock(nd6_mutex);
 	return error;
 }
-#endif
+SYSCTL_PROC(_net_inet6_icmp6, ICMPV6CTL_ND6_DRLIST, nd6_drlist,
+	CTLFLAG_RD, 0, 0, nd6_sysctl_drlist, "S,in6_defrouter","");
+SYSCTL_PROC(_net_inet6_icmp6, ICMPV6CTL_ND6_PRLIST, nd6_prlist,
+	CTLFLAG_RD, 0, 0, nd6_sysctl_prlist, "S,in6_defrouter","");
+
diff --git a/bsd/netinet6/nd6.h b/bsd/netinet6/nd6.h
index 8f525ce33..b85be0157 100644
--- a/bsd/netinet6/nd6.h
+++ b/bsd/netinet6/nd6.h
@@ -188,6 +188,24 @@ struct	in6_ndifreq {
 #define ND6_INFINITE_LIFETIME		0xffffffff
 
 #ifdef KERNEL_PRIVATE
+#define ND_IFINFO(ifp) \
+	(&nd_ifinfo[(ifp)->if_index])
+/*
+ * In a more readable form, we derive linkmtu based on:
+ *
+ * if (ND_IFINFO(ifp)->linkmtu && ND_IFINFO(ifp)->linkmtu < ifp->if_mtu)
+ *         linkmtu = ND_IFINFO(ifp)->linkmtu;
+ * else if ((ND_IFINFO(ifp)->maxmtu && ND_IFINFO(ifp)->maxmtu < ifp->if_mtu))
+ *         linkmtu = ND_IFINFO(ifp)->maxmtu;
+ * else
+ *         linkmtu = ifp->if_mtu;
+ */
+#define IN6_LINKMTU(ifp)						      \
+	((ND_IFINFO(ifp)->linkmtu &&					      \
+	ND_IFINFO(ifp)->linkmtu < (ifp)->if_mtu) ? ND_IFINFO(ifp)->linkmtu :  \
+	((ND_IFINFO(ifp)->maxmtu && ND_IFINFO(ifp)->maxmtu < (ifp)->if_mtu) ? \
+	ND_IFINFO(ifp)->maxmtu : (ifp)->if_mtu))
+
 /* node constants */
 #define MAX_REACHABLE_TIME		3600000	/* msec */
 #define REACHABLE_TIME			30000	/* msec */
@@ -230,7 +248,8 @@ struct nd_prefix {
 	/* list of routers that advertise the prefix: */
 	LIST_HEAD(pr_rtrhead, nd_pfxrouter) ndpr_advrtrs;
 	u_char	ndpr_plen;
-	int	ndpr_refcnt;	/* reference couter from addresses */
+	int	ndpr_refcnt;	/* reference counter from addresses */
+	int	ndpr_usecnt;	/* actual use count; prevents free */
 };
 
 #define ndpr_next		ndpr_entry.le_next
@@ -398,6 +417,8 @@ int in6_init_prefix_ltimes(struct nd_prefix *ndpr);
 void rt6_flush(struct in6_addr *, struct ifnet *);
 int nd6_setdefaultiface(int);
 int in6_tmpifadd(const struct in6_ifaddr *, int);
+void ndpr_hold(struct nd_prefix *, boolean_t);
+void ndpr_rele(struct nd_prefix *, boolean_t);
 #endif /* KERNEL_PRIVATE */
 
 #ifdef KERNEL
diff --git a/bsd/netinet6/nd6_nbr.c b/bsd/netinet6/nd6_nbr.c
index 5e968a96e..c8686c533 100644
--- a/bsd/netinet6/nd6_nbr.c
+++ b/bsd/netinet6/nd6_nbr.c
@@ -216,7 +216,7 @@ nd6_ns_input(
 		tsin6.sin6_family = AF_INET6;
 		tsin6.sin6_addr = taddr6;
 
-		rt = rtalloc1((struct sockaddr *)&tsin6, 0, 0);
+		rt = rtalloc1((struct sockaddr *)&tsin6, 0, 0UL);
 		if (rt && (rt->rt_flags & RTF_ANNOUNCE) != 0 &&
 		    rt->rt_gateway->sa_family == AF_LINK) {
 			/*
@@ -367,7 +367,7 @@ nd6_ns_output(
 		return;
 	}
 
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
+	MGETHDR(m, M_DONTWAIT, MT_DATA);	/* XXXMAC: mac_create_mbuf_linklayer() probably */
 	if (m && max_linkhdr + maxlen >= MHLEN) {
 		MCLGET(m, M_DONTWAIT);
 		if ((m->m_flags & M_EXT) == 0) {
@@ -776,7 +776,6 @@ nd6_na_input(
 			 */
 			struct nd_defrouter *dr;
 			struct in6_addr *in6;
-			int s;
 
 			in6 = &((struct sockaddr_in6 *)rt_key(rt))->sin6_addr;
 
@@ -868,7 +867,7 @@ nd6_na_output(
 		return;
 	}
 
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
+	MGETHDR(m, M_DONTWAIT, MT_DATA);	/* XXXMAC: mac_create_mbuf_linklayer() probably */
 	if (m && max_linkhdr + maxlen >= MHLEN) {
 		MCLGET(m, M_DONTWAIT);
 		if ((m->m_flags & M_EXT) == 0) {
@@ -1053,7 +1052,7 @@ nd6_dad_stoptimer(
 void
 nd6_dad_start(
 	struct ifaddr *ifa,
-	int *tick)	/* minimum delay ticks for IFF_UP event */
+	int *tick_delay)	/* minimum delay ticks for IFF_UP event */
 {
 	struct in6_ifaddr *ia = (struct in6_ifaddr *)ifa;
 	struct dadq *dp;
@@ -1121,18 +1120,18 @@ nd6_dad_start(
 	dp->dad_count = ip6_dad_count;
 	dp->dad_ns_icount = dp->dad_na_icount = 0;
 	dp->dad_ns_ocount = dp->dad_ns_tcount = 0;
-	if (tick == NULL) {
+	if (tick_delay == NULL) {
 		nd6_dad_ns_output(dp, ifa);
 		timeout((void (*)(void *))nd6_dad_timer, (void *)ifa,
 			nd_ifinfo[ifa->ifa_ifp->if_index].retrans * hz / 1000);
 	} else {
 		int ntick;
 
-		if (*tick == 0)
+		if (*tick_delay == 0)
 			ntick = random() % (MAX_RTR_SOLICITATION_DELAY * hz);
 		else
-			ntick = *tick + random() % (hz / 2);
-		*tick = ntick;
+			ntick = *tick_delay + random() % (hz / 2);
+		*tick_delay = ntick;
 		timeout((void (*)(void *))nd6_dad_timer, (void *)ifa,
 			ntick);
 	}
@@ -1170,12 +1169,9 @@ static void
 nd6_dad_timer(
 	struct ifaddr *ifa)
 {
-	int s;
 	struct in6_ifaddr *ia = (struct in6_ifaddr *)ifa;
 	struct dadq *dp;
 
-	s = splnet();		/* XXX */
-
 	/* Sanity check */
 	if (ia == NULL) {
 		log(LOG_ERR, "nd6_dad_timer: called with null parameter\n");
@@ -1297,7 +1293,7 @@ nd6_dad_timer(
 	}
 
 done:
-	splx(s);
+	return;
 }
 
 void
diff --git a/bsd/netinet6/nd6_rtr.c b/bsd/netinet6/nd6_rtr.c
index 6ca948351..8ca259b74 100644
--- a/bsd/netinet6/nd6_rtr.c
+++ b/bsd/netinet6/nd6_rtr.c
@@ -1,3 +1,31 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
 /*	$FreeBSD: src/sys/netinet6/nd6_rtr.c,v 1.11 2002/04/19 04:46:23 suz Exp $	*/
 /*	$KAME: nd6_rtr.c,v 1.111 2001/04/27 01:37:15 jinmei Exp $	*/
 
@@ -590,14 +618,7 @@ defrouter_delreq(
 		  RTF_GATEWAY, &oldrt);
 	if (oldrt) {
 		nd6_rtmsg(RTM_DELETE, oldrt);
-		if (oldrt->rt_refcnt <= 0) {
-			/*
-			 * XXX: borrowed from the RTM_DELETE case of
-			 * rtrequest().
-			 */
-			rtref(oldrt);
-			rtfree_locked(oldrt);
-		}
+		rtfree_locked(oldrt);
 	}
 
 	if (dofree)		/* XXX: necessary? */
@@ -845,11 +866,45 @@ nd6_prefix_lookup(
 			break;
 		}
 	}
+	if (search != NULL)
+		ndpr_hold(search, TRUE);
 	lck_mtx_unlock(nd6_mutex);
 
 	return(search);
 }
 
+void
+ndpr_hold(struct nd_prefix *pr, boolean_t locked)
+{
+	if (!locked)
+		lck_mtx_lock(nd6_mutex);
+
+	if (pr->ndpr_usecnt < 0)
+		panic("%s: bad usecnt %d for pr %p\n", __func__,
+		    pr->ndpr_usecnt, pr);
+
+	pr->ndpr_usecnt++;
+
+	if (!locked)
+		lck_mtx_unlock(nd6_mutex);
+}
+
+void
+ndpr_rele(struct nd_prefix *pr, boolean_t locked)
+{
+	if (!locked)
+		lck_mtx_lock(nd6_mutex);
+
+	if (pr->ndpr_usecnt <= 0)
+		panic("%s: bad usecnt %d for pr %p\n", __func__,
+		    pr->ndpr_usecnt, pr);
+
+	pr->ndpr_usecnt--;
+
+	if (!locked)
+		lck_mtx_unlock(nd6_mutex);
+}
+
 int
 nd6_prelist_add(
 	struct nd_prefix *pr,
@@ -879,6 +934,9 @@ nd6_prelist_add(
 	lck_mtx_lock(nd6_mutex);
 	LIST_INSERT_HEAD(&nd_prefix, new, ndpr_entry);
 
+	new->ndpr_usecnt = 0;
+	ndpr_hold(new, TRUE);
+
 	/* ND_OPT_PI_FLAG_ONLINK processing */
 	if (new->ndpr_raf_onlink) {
 		int e;
@@ -927,11 +985,12 @@ prelist_remove(
 		/* what should we do? */
 	}
 
-	if (pr->ndpr_refcnt > 0)
-		return;		/* notice here? */
-
 	if (nd6locked == 0)
 		lck_mtx_lock(nd6_mutex);
+
+	if (pr->ndpr_usecnt > 0 || pr->ndpr_refcnt > 0)
+		goto done;	/* notice here? */
+
 	/* unlink ndpr_entry from nd_prefix list */
 	LIST_REMOVE(pr, ndpr_entry);
 
@@ -945,6 +1004,7 @@ prelist_remove(
 	FREE(pr, M_IP6NDP);
 
 	pfxlist_onlink_check(1);
+done:
 	if (nd6locked == 0)
 		lck_mtx_unlock(nd6_mutex);
 }
@@ -1192,7 +1252,9 @@ prelist_update(
 			/*
 			 * note that we should use pr (not new) for reference.
 			 */
+			lck_mtx_lock(nd6_mutex);
 			pr->ndpr_refcnt++;
+			lck_mtx_unlock(nd6_mutex);
 			ia6->ia6_ndpr = pr;
 
 #if 0
@@ -1234,9 +1296,12 @@ prelist_update(
 		}
 	}
 
-  afteraddrconf:
+afteraddrconf:
+
+end:
+	if (pr != NULL)
+		ndpr_rele(pr, FALSE);
 
- end:
 	return error;
 }
 
@@ -1644,13 +1709,9 @@ nd6_prefix_offlink(
 		    error));
 	}
 
-	if (rt != NULL) {
-		if (rt->rt_refcnt <= 0) {
-			/* XXX: we should free the entry ourselves. */
-			rtref(rt);
-			rtfree_locked(rt);
-		}
-	}
+	if (rt != NULL)
+		rtfree_locked(rt);
+
 	lck_mtx_unlock(rt_mtx);
 
 	return(error);
@@ -1895,6 +1956,7 @@ in6_tmpifadd(
 		    "no ifaddr\n"));
 		return(EINVAL); /* XXX */
 	}
+	lck_mtx_lock(nd6_mutex);
 	newia->ia6_ndpr = ia0->ia6_ndpr;
 	newia->ia6_ndpr->ndpr_refcnt++;
 
@@ -1906,7 +1968,8 @@ in6_tmpifadd(
 	 * and, in fact, we surely need the check when we create a new
 	 * temporary address due to deprecation of an old temporary address.
 	 */
-	pfxlist_onlink_check(0);
+	pfxlist_onlink_check(1);
+	lck_mtx_unlock(nd6_mutex);
 
 	return(0);
 }	    
@@ -1937,7 +2000,7 @@ in6_init_prefix_ltimes(struct nd_prefix *ndpr)
 }
 
 static void
-in6_init_address_ltimes(struct nd_prefix *new, struct in6_addrlifetime *lt6)
+in6_init_address_ltimes(__unused struct nd_prefix *new, struct in6_addrlifetime *lt6)
 {
 	struct timeval timenow;
 
diff --git a/bsd/netinet6/raw_ip6.c b/bsd/netinet6/raw_ip6.c
index 7e3094d30..e84171ae6 100644
--- a/bsd/netinet6/raw_ip6.c
+++ b/bsd/netinet6/raw_ip6.c
@@ -98,15 +98,8 @@
 #include <netinet6/ipsec.h>
 #include <netinet6/ipsec6.h>
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
 #endif /*IPSEC*/
 
-
-#include "faith.h"
-#if defined(NFAITH) && 0 < NFAITH
-#include <net/if_faith.h>
-#endif
-
 #define	satosin6(sa)	((struct sockaddr_in6 *)(sa))
 #define	ifatoia6(ifa)	((struct in6_ifaddr *)(ifa))
 
@@ -180,14 +173,10 @@ rip6_input(
 			/*
 			 * Check AH/ESP integrity.
 			 */
-			if (ipsec_bypass == 0 && n) {
-				lck_mtx_lock(sadb_mutex);
-				if (ipsec6_in_reject_so(n, last->inp_socket)) {
+			if (ipsec_bypass == 0 && n && ipsec6_in_reject_so(n, last->inp_socket)) {
 					m_freem(n);
-					ipsec6stat.in_polvio++;
+					IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 					/* do not inject data into pcb */
-			        }
-				lck_mtx_unlock(sadb_mutex);
 			} else
 #endif /*IPSEC*/
 			if (n) {
@@ -212,15 +201,11 @@ rip6_input(
 	/*
 	 * Check AH/ESP integrity.
 	 */
-	if (ipsec_bypass == 0 && last) {
-		lck_mtx_lock(sadb_mutex);
-		if  (ipsec6_in_reject_so(m, last->inp_socket)) {
+	if (ipsec_bypass == 0 && last && ipsec6_in_reject_so(m, last->inp_socket)) {
 			m_freem(m);
-			ipsec6stat.in_polvio++;
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 			ip6stat.ip6s_delivered--;
 			/* do not inject data into pcb */
-		}
-		lck_mtx_unlock(sadb_mutex);
 	} else
 #endif /*IPSEC*/
 	if (last) {
@@ -290,7 +275,7 @@ rip6_ctlinput(
 		sa6_src = &sa6_any;
 	}
 
-	(void) in6_pcbnotify(&ripcbinfo, sa, 0, (struct sockaddr *)sa6_src,
+	(void) in6_pcbnotify(&ripcbinfo, sa, 0, (const struct sockaddr *)sa6_src,
 			     0, cmd, notify);
 }
 
@@ -470,8 +455,10 @@ rip6_output(
 		m_freem(m);
 
  freectl:
-	if (optp == &opt && optp->ip6po_rthdr && optp->ip6po_route.ro_rt)
+	if (optp == &opt && optp->ip6po_rthdr && optp->ip6po_route.ro_rt) {
 		rtfree(optp->ip6po_route.ro_rt);
+		optp->ip6po_route.ro_rt = NULL;
+	}
 	if (control) {
 		if (optp == &opt)
 			ip6_clearpktopts(optp, 0, -1);
@@ -481,7 +468,7 @@ rip6_output(
 }
 
 static void
-load_ip6fw()
+load_ip6fw(void)
 {
 	ip6_fw_init();
 }
@@ -569,10 +556,10 @@ rip6_ctloutput(
 }
 
 static int
-rip6_attach(struct socket *so, int proto, struct proc *p)
+rip6_attach(struct socket *so, int proto, __unused struct proc *p)
 {
 	struct inpcb *inp;
-	int error, s;
+	int error;
 
 	inp = sotoinpcb(so);
 	if (inp)
@@ -583,9 +570,7 @@ rip6_attach(struct socket *so, int proto, struct proc *p)
 	error = soreserve(so, rip_sendspace, rip_recvspace);
 	if (error)
 		return error;
-	s = splnet();
 	error = in_pcballoc(so, &ripcbinfo, p);
-	splx(s);
 	if (error)
 		return error;
 	inp = (struct inpcb *)so->so_pcb;
@@ -639,7 +624,7 @@ rip6_disconnect(struct socket *so)
 }
 
 static int
-rip6_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
+rip6_bind(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in6 *addr = (struct sockaddr_in6 *)nam;
@@ -665,13 +650,14 @@ rip6_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 	    if (ia) ifafree(ia);
 		return(EADDRNOTAVAIL);
 	}
-	ifafree(ia);
+	if (ia != NULL)
+		ifafree(ia);
 	inp->in6p_laddr = addr->sin6_addr;
 	return 0;
 }
 
 static int
-rip6_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
+rip6_connect(struct socket *so, struct sockaddr *nam, __unused struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in6 *addr = (struct sockaddr_in6 *)nam;
@@ -716,8 +702,8 @@ rip6_shutdown(struct socket *so)
 }
 
 static int
-rip6_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
-	 struct mbuf *control, struct proc *p)
+rip6_send(struct socket *so, __unused int flags, struct mbuf *m, struct sockaddr *nam,
+	 struct mbuf *control, __unused struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
 	struct sockaddr_in6 tmp;
@@ -759,3 +745,14 @@ struct pr_usrreqs rip6_usrreqs = {
 	pru_rcvoob_notsupp, rip6_send, pru_sense_null, rip6_shutdown,
 	in6_setsockaddr, sosend, soreceive, pru_sopoll_notsupp
 };
+
+__private_extern__ struct pr_usrreqs icmp6_dgram_usrreqs = {
+        rip6_abort, pru_accept_notsupp, icmp6_dgram_attach, rip6_bind, rip6_connect,
+        pru_connect2_notsupp, in6_control, rip6_detach, rip6_disconnect,
+        pru_listen_notsupp, in6_setpeeraddr, pru_rcvd_notsupp,
+        pru_rcvoob_notsupp, icmp6_dgram_send, pru_sense_null, rip6_shutdown,
+        in6_setsockaddr, sosend, soreceive, pru_sopoll_notsupp
+};
+
+
+
diff --git a/bsd/netinet6/route6.c b/bsd/netinet6/route6.c
index a0516af41..36617f2d0 100644
--- a/bsd/netinet6/route6.c
+++ b/bsd/netinet6/route6.c
@@ -34,8 +34,10 @@
 #include <sys/mbuf.h>
 #include <sys/socket.h>
 #include <sys/queue.h>
+#include <string.h>
 
 #include <net/if.h>
+#include <net/route.h>
 
 #include <netinet/in.h>
 #include <netinet6/in6_var.h>
@@ -50,19 +52,16 @@ static int ip6_rthdr0(struct mbuf *, struct ip6_hdr *,
 #endif /* IP6_RTHDR0_ALLOWED */
 
 int
-route6_input(mp, offp)
-	struct mbuf **mp;
-	int *offp;
+route6_input(struct mbuf **mp, int *offp)
 {
 	struct ip6_hdr *ip6;
 	struct mbuf *m = *mp;
 	struct ip6_rthdr *rh;
 	int off = *offp, rhlen;
-	struct mbuf *n;
+	struct ip6aux *ip6a;
 
-	n = ip6_findaux(m);
-	if (n) {
-		struct ip6aux *ip6a = mtod(n, struct ip6aux *);
+	ip6a = ip6_findaux(m);
+	if (ip6a) {
 		/* XXX reject home-address option before rthdr */
 		if (ip6a->ip6a_flags & IP6A_SWAP) {
 			ip6stat.ip6s_badoptions++;
@@ -145,6 +144,7 @@ ip6_rthdr0(m, ip6, rh0)
 {
 	int addrs, index;
 	struct in6_addr *nextaddr, tmpaddr;
+	struct route_in6 ip6forward_rt;
 
 	if (rh0->ip6r0_segleft == 0)
 		return(0);
@@ -210,15 +210,31 @@ ip6_rthdr0(m, ip6, rh0)
 	if (IN6_IS_ADDR_LINKLOCAL(&ip6->ip6_dst))
 		ip6->ip6_dst.s6_addr16[1] = htons(m->m_pkthdr.rcvif->if_index);
 
+	/*
+	 * Don't use the globally cached route to forward packet having
+	 * Type 0 routing header(s); instead, do an explicit lookup using
+	 * a local route entry variable, in case the next address in the
+	 * packet is bogus (which would otherwise unnecessarily invalidate
+	 * the globally cached route).
+	 */
+	bzero(&ip6forward_rt, sizeof (ip6forward_rt));
+
 #if COMPAT_RFC1883
 	if (rh0->ip6r0_slmap[index / 8] & (1 << (7 - (index % 8))))
-		ip6_forward(m, IPV6_SRCRT_NEIGHBOR, 0);
+		ip6_forward(m, &ip6forward_rt, IPV6_SRCRT_NEIGHBOR, 0);
 	else
-		ip6_forward(m, IPV6_SRCRT_NOTNEIGHBOR, 0);
+		ip6_forward(m, &ip6forward_rt, IPV6_SRCRT_NOTNEIGHBOR, 0);
 #else
-	ip6_forward(m, 1, 0);
+	ip6_forward(m, &ip6forward_rt, 1, 0);
 #endif
 
+	/* Release reference to the looked up route */
+	if (ip6forward_rt.ro_rt != NULL) {
+		rtfree(ip6forward_rt.ro_rt);
+		ip6forward_rt.ro_rt = NULL;
+	}
+
 	return(-1);			/* m would be freed in ip6_forward() */
 }
 #endif /* IP6_RTHDR0_ALLOWED */
+
diff --git a/bsd/netinet6/scope6.c b/bsd/netinet6/scope6.c
index 845695f71..76640cc32 100644
--- a/bsd/netinet6/scope6.c
+++ b/bsd/netinet6/scope6.c
@@ -59,8 +59,6 @@ void
 scope6_ifattach(
 	struct ifnet *ifp)
 {
-	int s = splnet();
-
 	/*
 	 * We have some arrays that should be indexed by if_index.
 	 * since if_index will grow dynamically, they should grow too.
@@ -88,7 +86,6 @@ scope6_ifattach(
 
 	/* don't initialize if called twice */
 	if (SID.s6id_list[IPV6_ADDR_SCOPE_LINKLOCAL]) {
-		splx(s);
 		return;
 	}
 
@@ -103,8 +100,6 @@ scope6_ifattach(
 	SID.s6id_list[IPV6_ADDR_SCOPE_ORGLOCAL] = 1;
 #endif
 #undef SID
-
-	splx(s);
 }
 
 int
@@ -112,7 +107,7 @@ scope6_set(
 	struct ifnet *ifp,
 	u_int32_t *idlist)
 {
-	int i, s;
+	int i;
 	int error = 0;
 
 	if (scope6_ids == NULL)	/* paranoid? */
@@ -128,8 +123,6 @@ scope6_set(
 	 * interface addresses, routing table entries, PCB entries... 
 	 */
 
-	s = splnet();
-
 	for (i = 0; i < 16; i++) {
 		if (idlist[i] &&
 		    idlist[i] != scope6_ids[ifp->if_index].s6id_list[i]) {
@@ -141,7 +134,6 @@ scope6_set(
 				 * IDs, but we check the consistency for
 				 * safety in later use.
 				 */
-				splx(s);
 				return(EINVAL);
 			}
 
@@ -153,7 +145,6 @@ scope6_set(
 			scope6_ids[ifp->if_index].s6id_list[i] = idlist[i];
 		}
 	}
-	splx(s);
 
 	return(error);
 }
diff --git a/bsd/netinet6/udp6_usrreq.c b/bsd/netinet6/udp6_usrreq.c
index 3be50feec..b97874db6 100644
--- a/bsd/netinet6/udp6_usrreq.c
+++ b/bsd/netinet6/udp6_usrreq.c
@@ -103,14 +103,8 @@
 #include <netinet6/ipsec.h>
 #include <netinet6/ipsec6.h>
 extern int ipsec_bypass;
-extern lck_mtx_t *sadb_mutex;
-extern lck_mtx_t *nd6_mutex;
 #endif /*IPSEC*/
-
-#include "faith.h"
-#if defined(NFAITH) && NFAITH > 0
-#include <net/if_faith.h>
-#endif
+extern lck_mtx_t *nd6_mutex;
 
 /*
  * UDP protocol inplementation.
@@ -120,18 +114,22 @@ extern lck_mtx_t *nd6_mutex;
 extern	struct protosw inetsw[];
 static	int in6_mcmatch(struct inpcb *, struct in6_addr *, struct ifnet *);
 static	int udp6_detach(struct socket *so);
+static void udp6_append(struct inpcb *, struct ip6_hdr *,
+    struct sockaddr_in6 *, struct mbuf *, int);
 
-
-extern  void    ipfwsyslog( int level, char *format,...);
+extern  void    ipfwsyslog( int level, const char *format,...);
 extern int fw_verbose;
 
+#if IPFIREWALL
 #define log_in_vain_log( a ) {            \
         if ( (log_in_vain == 3 ) && (fw_verbose == 2)) {        /* Apple logging, log to ipfw.log */ \
                 ipfwsyslog a ;  \
         }                       \
         else log a ;            \
 }
-
+#else
+#define log_in_vain_log( a ) { log a; }
+#endif
 
 static int
 in6_mcmatch(
@@ -160,6 +158,33 @@ in6_mcmatch(
 	return 0;
 }
 
+/*
+ * subroutine of udp6_input(), mainly for source code readability.
+ */
+static void
+udp6_append(struct inpcb *last, struct ip6_hdr *ip6,
+    struct sockaddr_in6 *udp_in6, struct mbuf *n, int off)
+{
+	struct  mbuf *opts = NULL;
+
+#if CONFIG_MACF_NET
+	if (mac_inpcb_check_deliver(last, n, AF_INET6, SOCK_DGRAM) != 0) {
+		m_freem(n);
+		return;
+	}
+#endif
+	if (last->in6p_flags & IN6P_CONTROLOPTS ||
+	    last->in6p_socket->so_options & SO_TIMESTAMP)
+		ip6_savecontrol(last, &opts, ip6, n);
+
+	m_adj(n, off);
+	if (sbappendaddr(&last->in6p_socket->so_rcv,
+	    (struct sockaddr *)udp_in6, n, opts, NULL) == 0)
+		udpstat.udps_fullsock++;
+	else
+		sorwakeup(last->in6p_socket);
+}
+
 int
 udp6_input(
 	struct mbuf **mp,
@@ -211,7 +236,8 @@ udp6_input(
 	}
 
 	if (IN6_IS_ADDR_MULTICAST(&ip6->ip6_dst)) {
-		struct	inpcb *last;
+		int reuse_sock = 0, mcast_delivered = 0;
+		struct mbuf *n = NULL;
 
 		/*
 		 * Deliver a multicast datagram to all sockets
@@ -253,7 +279,6 @@ udp6_input(
 		 * Locate pcb(s) for datagram.
 		 * (Algorithm copied from raw_intr().)
 		 */
-		last = NULL;
 		lck_rw_lock_shared(pcbinfo->mtx);
 
 		LIST_FOREACH(in6p, &udb, inp_list) {
@@ -292,47 +317,38 @@ udp6_input(
 				}
 			}
 
-			if (last != NULL) {
-				struct	mbuf *n;
+			reuse_sock = in6p->inp_socket->so_options &
+			    (SO_REUSEPORT | SO_REUSEADDR);
 
+			{
 #if IPSEC
-				/*
-				 * Check AH/ESP integrity.
-				 */
+				int skipit = 0;
+				/* Check AH/ESP integrity. */
 				if (ipsec_bypass == 0) {
-					lck_mtx_lock(sadb_mutex);
-					if (ipsec6_in_reject_so(m, last->inp_socket))
-						ipsec6stat.in_polvio++;
-					/* do not inject data into pcb */
-					lck_mtx_unlock(sadb_mutex);
+					if (ipsec6_in_reject_so(m, in6p->inp_socket)) {
+						IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
+						/* do not inject data to pcb */
+						skipit = 1;
+					}
 				}
-				else
+				if (skipit == 0)
 #endif /*IPSEC*/
-				if ((n = m_copy(m, 0, M_COPYALL)) != NULL) {
+				{
 					/*
 					 * KAME NOTE: do not
-					 * m_copy(m, offset, ...) above.
+					 * m_copy(m, offset, ...) below.
 					 * sbappendaddr() expects M_PKTHDR,
 					 * and m_copy() will copy M_PKTHDR
 					 * only if offset is 0.
 					 */
-					if (last->in6p_flags & IN6P_CONTROLOPTS
-					    || last->in6p_socket->so_options & SO_TIMESTAMP)
-						ip6_savecontrol(last, &opts,
-								ip6, n);
-								
-					m_adj(n, off + sizeof(struct udphdr));
-					if (sbappendaddr(&last->in6p_socket->so_rcv,
-							(struct sockaddr *)&udp_in6,
-							n, opts, NULL) == 0) {
-						udpstat.udps_fullsock++;
-					} else
-						sorwakeup(last->in6p_socket);
-					opts = NULL;
+					if (reuse_sock)
+						n = m_copy(m, 0, M_COPYALL);
+					udp6_append(in6p, ip6, &udp_in6, m,
+					    off + sizeof (struct udphdr));
+					mcast_delivered++;
 				}
-				udp_unlock(last->in6p_socket, 1, 0);
+				udp_unlock(in6p->in6p_socket, 1, 0);
 			}
-			last = in6p;
 			/*
 			 * Don't look for additional matches if this one does
 			 * not have either the SO_REUSEPORT or SO_REUSEADDR
@@ -341,13 +357,12 @@ udp6_input(
 			 * port.  It assumes that an application will never
 			 * clear these options after setting them.
 			 */
-			if ((last->in6p_socket->so_options &
-			     (SO_REUSEPORT|SO_REUSEADDR)) == 0)
+			if (reuse_sock == 0 || ((m = n) == NULL))
 				break;
 		}
 		lck_rw_done(pcbinfo->mtx);
 
-		if (last == NULL) {
+		if (mcast_delivered == 0) {
 			/*
 			 * No matching pcb found; discard datagram.
 			 * (No need to send an ICMP Port Unreachable
@@ -359,37 +374,9 @@ udp6_input(
 #endif
 			goto bad;
 		}
-#if IPSEC
-		/*
-		 * Check AH/ESP integrity.
-		 */
-		if (ipsec_bypass == 0) {
-			lck_mtx_lock(sadb_mutex);
-			if (ipsec6_in_reject_so(m, last->inp_socket)) {
-				ipsec6stat.in_polvio++;
-				lck_mtx_unlock(sadb_mutex);
-				udp_unlock(last->in6p_socket, 1, 0);
-				goto bad;
-			}
-			lck_mtx_unlock(sadb_mutex);
-		}
-#endif /*IPSEC*/
-		if (last->in6p_flags & IN6P_CONTROLOPTS
-		    || last->in6p_socket->so_options & SO_TIMESTAMP)
-			ip6_savecontrol(last, &opts, ip6, m);
-
-		m_adj(m, off + sizeof(struct udphdr));
-		if (sbappendaddr(&last->in6p_socket->so_rcv,
-				(struct sockaddr *)&udp_in6,
-				m, opts, NULL) == 0) {
-			udpstat.udps_fullsock++;
-			m = NULL;
-			opts = NULL;
-			udp_unlock(last->in6p_socket, 1, 0);
-			goto bad;
-		}
-		sorwakeup(last->in6p_socket);
-		udp_unlock(last->in6p_socket, 1, 0);
+
+		if (reuse_sock != 0)	/* free the extra copy of mbuf */
+			m_freem(m);
 		return IPPROTO_DONE;
 	}
 	/*
@@ -402,7 +389,7 @@ udp6_input(
 		if (log_in_vain) {
 			char buf[INET6_ADDRSTRLEN];
 
-			strcpy(buf, ip6_sprintf(&ip6->ip6_dst));
+			strlcpy(buf, ip6_sprintf(&ip6->ip6_dst), sizeof(buf));
 			if (log_in_vain != 3)
 				log(LOG_INFO,
 				    "Connection attempt to UDP %s:%d from %s:%d\n",
@@ -431,14 +418,11 @@ udp6_input(
 	 * Check AH/ESP integrity.
 	 */
 	if (ipsec_bypass == 0) {
-		lck_mtx_lock(sadb_mutex);
 		if (ipsec6_in_reject_so(m, in6p->in6p_socket)) {
-			ipsec6stat.in_polvio++;
-			lck_mtx_unlock(sadb_mutex);
+			IPSEC_STAT_INCREMENT(ipsec6stat.in_polvio);
 			in_pcb_checkstate(in6p, WNT_RELEASE, 0);
 			goto bad;
 		}
-		lck_mtx_unlock(sadb_mutex);
 	}
 #endif /*IPSEC*/
 
@@ -552,7 +536,7 @@ udp6_getcred SYSCTL_HANDLER_ARGS
 	struct inpcb *inp;
 	int error, s;
 
-	error = suser(req->p->p_ucred, &req->p->p_acflag);
+	error = proc_suser(req->p);
 	if (error)
 		return (error);
 
@@ -599,7 +583,7 @@ udp6_abort(struct socket *so)
 }
 
 static int
-udp6_attach(struct socket *so, int proto, struct proc *p)
+udp6_attach(struct socket *so, __unused int proto, struct proc *p)
 {
 	struct inpcb *inp;
 	int error;
diff --git a/bsd/netkey/key.c b/bsd/netkey/key.c
index aee29ba54..86ea9ada0 100644
--- a/bsd/netkey/key.c
+++ b/bsd/netkey/key.c
@@ -34,6 +34,7 @@
  * This code is referd to RFC 2367
  */
 
+#include <machine/endian.h>
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -117,6 +118,13 @@ lck_grp_t         *sadb_mutex_grp;
 lck_grp_attr_t    *sadb_mutex_grp_attr;
 lck_attr_t        *sadb_mutex_attr;
 lck_mtx_t         *sadb_mutex;
+
+lck_grp_t         *pfkey_stat_mutex_grp;
+lck_grp_attr_t    *pfkey_stat_mutex_grp_attr;
+lck_attr_t        *pfkey_stat_mutex_attr;
+lck_mtx_t         *pfkey_stat_mutex;
+
+
 extern lck_mtx_t  *nd6_mutex;
 
 /*
@@ -141,6 +149,8 @@ static int key_blockacq_count = 10;	/* counter for blocking SADB_ACQUIRE.*/
 static int key_blockacq_lifetime = 20;	/* lifetime for blocking SADB_ACQUIRE.*/
 static int key_preferred_oldsa = 0;	/* preferred old sa rather than new sa.*/
 static int natt_keepalive_interval = 20;	/* interval between natt keepalives.*/
+static int ipsec_policy_count = 0;
+static int ipsec_sav_count = 0;
 
 static u_int32_t acq_seq = 0;
 static int key_tick_init_random = 0;
@@ -322,17 +332,25 @@ do { \
 } while (0)
 
 #if 1
-#define KMALLOC(p, t, n)                                                     \
+#define KMALLOC_WAIT(p, t, n)                                                     \
+	((p) = (t) _MALLOC((unsigned long)(n), M_SECA, M_WAITOK))
+#define KMALLOC_NOWAIT(p, t, n)                                              \
 	((p) = (t) _MALLOC((unsigned long)(n), M_SECA, M_NOWAIT))
 #define KFREE(p)                                                             \
 	_FREE((caddr_t)(p), M_SECA);
 #else
-#define KMALLOC(p, t, n) \
+#define KMALLOC_WAIT(p, t, n) \
 do { \
-	((p) = (t)_MALLOC((unsigned long)(n), M_SECA, M_NOWAIT));             \
-	printf("%s %d: %p <- KMALLOC(%s, %d)\n",                             \
+	((p) = (t)_MALLOC((unsigned long)(n), M_SECA, M_WAITOK));             \
+	printf("%s %d: %p <- KMALLOC_WAIT(%s, %d)\n",                             \
 		__FILE__, __LINE__, (p), #t, n);                             \
 } while (0)
+#define KMALLOC_NOWAIT(p, t, n) \
+	do { \
+		((p) = (t)_MALLOC((unsigned long)(n), M_SECA, M_NOWAIT));             \
+		printf("%s %d: %p <- KMALLOC_NOWAIT(%s, %d)\n",                             \
+		       __FILE__, __LINE__, (p), #t, n);                             \
+	} while (0)
 
 #define KFREE(p)                                                             \
 	do {                                                                 \
@@ -382,9 +400,8 @@ struct sadb_msghdr {
 	int extlen[SADB_EXT_MAX + 1];
 };
 
-static struct secasvar *key_allocsa_policy(struct secasindex *);
-static void key_freesp_so(struct secpolicy **);
-static struct secasvar *key_do_allocsa_policy(struct secashead *, u_int);
+static struct secasvar *key_do_allocsa_policy(struct secashead *, u_int, u_int16_t);
+static int key_do_get_translated_port(struct secashead *, struct secasvar *, u_int);
 static void key_delsp(struct secpolicy *);
 static struct secpolicy *key_getsp(struct secpolicyindex *);
 static struct secpolicy *key_getspbyid(u_int32_t);
@@ -440,10 +457,11 @@ static int key_ismyaddr6(struct sockaddr_in6 *);
 #endif
 
 /* flags for key_cmpsaidx() */
-#define CMP_HEAD	1	/* protocol, addresses. */
-#define CMP_MODE_REQID	2	/* additionally HEAD, reqid, mode. */
-#define CMP_REQID	3	/* additionally HEAD, reaid. */
-#define CMP_EXACTLY	4	/* all elements. */
+#define CMP_HEAD	0x1	/* protocol, addresses. */
+#define CMP_PORT	0x2	/* additionally HEAD, reqid, mode. */
+#define CMP_REQID	0x4	/* additionally HEAD, reqid. */
+#define CMP_MODE        0x8       /* additionally mode. */
+#define CMP_EXACTLY	0xF	/* all elements. */
 static int key_cmpsaidx(struct secasindex *, struct secasindex *, int);
 
 static int key_cmpspidx_exactly(struct secpolicyindex *,
@@ -501,16 +519,14 @@ static int key_promisc(struct socket *, struct mbuf *,
 static int key_senderror(struct socket *, struct mbuf *, int);
 static int key_validate_ext(const struct sadb_ext *, int);
 static int key_align(struct mbuf *, struct sadb_msghdr *);
-#if 0
-static const char *key_getfqdn(void);
-static const char *key_getuserfqdn(void);
-#endif
 static void key_sa_chgstate(struct secasvar *, u_int8_t);
 static struct mbuf *key_alloc_mbuf(int);
 
 extern int ipsec_bypass;
 void ipsec_send_natt_keepalive(struct secasvar *sav);
 
+void key_init(void);
+
 
 /*
  * PF_KEY init
@@ -531,6 +547,15 @@ key_init(void)
 		printf("key_init: can't alloc sadb_mutex\n");
 		return;
 	}
+	
+	pfkey_stat_mutex_grp_attr = lck_grp_attr_alloc_init();
+	pfkey_stat_mutex_grp = lck_grp_alloc_init("pfkey_stat", pfkey_stat_mutex_grp_attr);
+	pfkey_stat_mutex_attr = lck_attr_alloc_init();
+
+	if ((pfkey_stat_mutex = lck_mtx_alloc_init(pfkey_stat_mutex_grp, pfkey_stat_mutex_attr)) == NULL) {
+		printf("key_init: can't alloc pfkey_stat_mutex\n");
+		return;
+	}
 
 	for (i = 0; i < SPIHASHSIZE; i++)
 		LIST_INIT(&spihash[i]);
@@ -553,9 +578,8 @@ key_allocsp(spidx, dir)
 {
 	struct secpolicy *sp;
 	struct timeval tv;
-	int s;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	/* sanity check */
 	if (spidx == NULL)
 		panic("key_allocsp: NULL pointer is passed.\n");
@@ -574,6 +598,7 @@ key_allocsp(spidx, dir)
 		printf("*** objects\n");
 		kdebug_secpolicyindex(spidx));
 
+	lck_mtx_lock(sadb_mutex);
 	LIST_FOREACH(sp, &sptree[dir], chain) {
 		KEYDEBUG(KEYDEBUG_IPSEC_DATA,
 			printf("*** in SPD\n");
@@ -584,21 +609,22 @@ key_allocsp(spidx, dir)
 		if (key_cmpspidx_withmask(&sp->spidx, spidx))
 			goto found;
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	return NULL;
 
 found:
-	/* sanity check */
-	KEY_CHKSPDIR(sp->spidx.dir, dir, "key_allocsp");
 
 	/* found a SPD entry */
 	microtime(&tv);
 	sp->lastused = tv.tv_sec;
 	sp->refcnt++;
+	lck_mtx_unlock(sadb_mutex);
+	
+	/* sanity check */
+	KEY_CHKSPDIR(sp->spidx.dir, dir, "key_allocsp");
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP key_allocsp cause refcnt++:%d SP:%p\n",
 			sp->refcnt, sp));
-
 	return sp;
 }
 
@@ -613,19 +639,17 @@ key_gettunnel(osrc, odst, isrc, idst)
 	struct secpolicy *sp;
 	const int dir = IPSEC_DIR_INBOUND;
 	struct timeval tv;
-	int s;
 	struct ipsecrequest *r1, *r2, *p;
 	struct sockaddr *os, *od, *is, *id;
 	struct secpolicyindex spidx;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	if (isrc->sa_family != idst->sa_family) {
 		ipseclog((LOG_ERR, "protocol family mismatched %d != %d\n.",
 			isrc->sa_family, idst->sa_family));
 		return NULL;
 	}
 
+	lck_mtx_lock(sadb_mutex);
 	LIST_FOREACH(sp, &sptree[dir], chain) {
 		if (sp->state == IPSEC_SPSTATE_DEAD)
 			continue;
@@ -647,7 +671,7 @@ key_gettunnel(osrc, odst, isrc, idst)
 				bcopy(isrc, &spidx.src, isrc->sa_len);
 				bcopy(idst, &spidx.dst, idst->sa_len);
 				if (!key_cmpspidx_withmask(&sp->spidx, &spidx))
-					continue;
+				  continue;
 			} else {
 				is = (struct sockaddr *)&r1->saidx.src;
 				id = (struct sockaddr *)&r1->saidx.dst;
@@ -665,13 +689,14 @@ key_gettunnel(osrc, odst, isrc, idst)
 			goto found;
 		}
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	return NULL;
 
 found:
 	microtime(&tv);
 	sp->lastused = tv.tv_sec;
 	sp->refcnt++;
+	lck_mtx_unlock(sadb_mutex);
 	return sp;
 }
 
@@ -682,15 +707,19 @@ found:
  *	ENOENT: policy may be valid, but SA with REQUIRE is on acquiring.
  */
 int
-key_checkrequest(isr, saidx)
+key_checkrequest(isr, saidx, sav)
 	struct ipsecrequest *isr;
-	struct secasindex *saidx;
+	struct secasindex *saidx;	
+	struct secasvar **sav;
 {
 	u_int level;
 	int error;
+	struct sockaddr_in *sin;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
+	*sav = NULL;
+	
 	/* sanity check */
 	if (isr == NULL || saidx == NULL)
 		panic("key_checkrequest: NULL pointer is passed.\n");
@@ -708,54 +737,27 @@ key_checkrequest(isr, saidx)
 	/* get current level */
 	level = ipsec_get_reqlevel(isr);
 
-#if 0
-	/*
-	 * We do allocate new SA only if the state of SA in the holder is
-	 * SADB_SASTATE_DEAD.  The SA for outbound must be the oldest.
-	 */
-	if (isr->sav != NULL) {
-		if (isr->sav->sah == NULL)
-			panic("key_checkrequest: sah is null.\n");
-		if (isr->sav == (struct secasvar *)LIST_FIRST(
-			    &isr->sav->sah->savtree[SADB_SASTATE_DEAD])) {
-			KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
-				printf("DP checkrequest calls free SA:%p\n",
-					isr->sav));
-			key_freesav(isr->sav);
-			isr->sav = NULL;
-		}
-	}
-#else
-	/*
-	 * we free any SA stashed in the IPsec request because a different
-	 * SA may be involved each time this request is checked, either
-	 * because new SAs are being configured, or this request is
-	 * associated with an unconnected datagram socket, or this request
-	 * is associated with a system default policy.
-	 *
-	 * The operation may have negative impact to performance.  We may
-	 * want to check cached SA carefully, rather than picking new SA
-	 * every time.
-	 */
-	if (isr->sav != NULL) {
-		key_freesav(isr->sav);
-		isr->sav = NULL;
-	}
-#endif
 
 	/*
-	 * new SA allocation if no SA found.
 	 * key_allocsa_policy should allocate the oldest SA available.
 	 * See key_do_allocsa_policy(), and draft-jenkins-ipsec-rekeying-03.txt.
 	 */
-	if (isr->sav == NULL)
-		isr->sav = key_allocsa_policy(saidx);
+	if (*sav == NULL)
+		*sav = key_allocsa_policy(saidx);
 
 	/* When there is SA. */
-	if (isr->sav != NULL)
+	if (*sav != NULL)
 		return 0;
 
-	/* there is no SA */
+	/* There is no SA.
+	 *
+	 * Remove dst port - used for special natt support - don't call
+	 * key_acquire with it.
+	 */
+	if (saidx->mode == IPSEC_MODE_TRANSPORT) {
+		sin = (struct sockaddr_in *)&saidx->dst;
+		sin->sin_port = IPSEC_PORT_ANY;
+	}
 	if ((error = key_acquire(saidx, isr->sp)) != 0) {
 		/* XXX What should I do ? */
 		ipseclog((LOG_DEBUG, "key_checkrequest: error %d returned "
@@ -772,7 +774,7 @@ key_checkrequest(isr, saidx)
  * OUT:	NULL:	not found.
  *	others:	found and return the pointer.
  */
-static struct secasvar *
+struct secasvar *
 key_allocsa_policy(saidx)
 	struct secasindex *saidx;
 {
@@ -781,16 +783,17 @@ key_allocsa_policy(saidx)
 	u_int stateidx, state;
 	const u_int *saorder_state_valid;
 	int arraysize;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	struct sockaddr_in *sin;
+	u_int16_t	dstport;
+	
+	lck_mtx_lock(sadb_mutex);
 	LIST_FOREACH(sah, &sahtree, chain) {
 		if (sah->state == SADB_SASTATE_DEAD)
 			continue;
-		if (key_cmpsaidx(&sah->saidx, saidx, CMP_MODE_REQID))
+		if (key_cmpsaidx(&sah->saidx, saidx, CMP_MODE | CMP_REQID))
 			goto found;
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	return NULL;
 
     found:
@@ -807,15 +810,23 @@ key_allocsa_policy(saidx)
 		arraysize = _ARRAYLEN(saorder_state_valid_prefer_new);
 	}
 
+
+	sin = (struct sockaddr_in *)&saidx->dst;
+	dstport = sin->sin_port;
+	if (saidx->mode == IPSEC_MODE_TRANSPORT)
+		sin->sin_port = IPSEC_PORT_ANY;
+
 	for (stateidx = 0; stateidx < arraysize; stateidx++) {
 
 		state = saorder_state_valid[stateidx];
 
-		sav = key_do_allocsa_policy(sah, state);
-		if (sav != NULL)
+		sav = key_do_allocsa_policy(sah, state, dstport);
+		if (sav != NULL) {
+			lck_mtx_unlock(sadb_mutex);
 			return sav;
+		}
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	return NULL;
 }
 
@@ -827,16 +838,19 @@ key_allocsa_policy(saidx)
  *	others	: found, pointer to a SA.
  */
 static struct secasvar *
-key_do_allocsa_policy(sah, state)
+key_do_allocsa_policy(sah, state, dstport)
 	struct secashead *sah;
 	u_int state;
+	u_int16_t dstport;
 {
-	struct secasvar *sav, *nextsav, *candidate, *d;
+	struct secasvar *sav, *nextsav, *candidate, *natt_candidate, *no_natt_candidate, *d;
 
 	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
-	/* initilize */
+	/* initialize */
 	candidate = NULL;
+	natt_candidate = NULL;
+	no_natt_candidate = NULL;
 
 	for (sav = LIST_FIRST(&sah->savtree[state]);
 	     sav != NULL;
@@ -847,11 +861,32 @@ key_do_allocsa_policy(sah, state)
 		/* sanity check */
 		KEY_CHKSASTATE(sav->state, state, "key_do_allocsa_policy");
 
-		/* initialize */
-		if (candidate == NULL) {
-			candidate = sav;
+		if (sah->saidx.mode == IPSEC_MODE_TUNNEL && dstport &&
+		    ((sav->flags & SADB_X_EXT_NATT) != 0) &&
+		    ntohs(dstport) != sav->remote_ike_port)
 			continue;
-		}
+		    
+		if (sah->saidx.mode == IPSEC_MODE_TRANSPORT &&
+		    ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0) &&
+		    ntohs(dstport) != sav->remote_ike_port)
+			continue;	/* skip this one - not a match - or not UDP */
+							
+		if ((sah->saidx.mode == IPSEC_MODE_TUNNEL &&
+		     ((sav->flags & SADB_X_EXT_NATT) != 0)) ||
+		    (sah->saidx.mode == IPSEC_MODE_TRANSPORT &&
+		     ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0))) {
+			if (natt_candidate == NULL) {
+				natt_candidate = sav;
+				continue;
+			} else
+				candidate = natt_candidate;
+		} else {
+			if (no_natt_candidate == NULL) {
+				no_natt_candidate = sav;
+				continue;
+			} else
+				candidate = no_natt_candidate;
+		}			
 
 		/* Which SA is the better ? */
 
@@ -864,8 +899,11 @@ key_do_allocsa_policy(sah, state)
 		if (key_preferred_oldsa) {
 			if (candidate->lft_c->sadb_lifetime_addtime >
 					sav->lft_c->sadb_lifetime_addtime) {
-				candidate = sav;
-			}
+				if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0)
+					natt_candidate = sav;
+				else
+					no_natt_candidate = sav;
+				}
 			continue;
 			/*NOTREACHED*/
 		}
@@ -874,7 +912,10 @@ key_do_allocsa_policy(sah, state)
 		if (candidate->lft_c->sadb_lifetime_addtime <
 				sav->lft_c->sadb_lifetime_addtime) {
 			d = candidate;
-			candidate = sav;
+			if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0)
+				natt_candidate = sav;
+			else
+				no_natt_candidate = sav;
 		} else
 			d = sav;
 
@@ -935,10 +976,23 @@ key_do_allocsa_policy(sah, state)
 					KEY_SENDUP_REGISTERED))
 				goto msgfail;
 		 msgfail:
-			key_freesav(d);
+			key_freesav(d, KEY_SADB_LOCKED);
 		}
 	}
 
+	/* choose latest if both types present */
+	if (natt_candidate == NULL)
+		candidate = no_natt_candidate;
+	else if (no_natt_candidate == NULL)
+		candidate = natt_candidate;
+	else if (sah->saidx.mode == IPSEC_MODE_TUNNEL && dstport)
+		candidate = natt_candidate;
+	else if (natt_candidate->lft_c->sadb_lifetime_addtime >
+			no_natt_candidate->lft_c->sadb_lifetime_addtime)
+		candidate = natt_candidate;
+	else
+		candidate = no_natt_candidate;
+
 	if (candidate) {
 		candidate->refcnt++;
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -974,11 +1028,10 @@ key_allocsa(family, src, dst, proto, spi)
 	u_int stateidx, state, tmpidx, matchidx;
 	struct sockaddr_in sin;
 	struct sockaddr_in6 sin6;
-	int s;
 	const u_int *saorder_state_valid;
 	int arraysize;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 
 	/* sanity check */
 	if (src == NULL || dst == NULL)
@@ -1008,6 +1061,7 @@ key_allocsa(family, src, dst, proto, spi)
 	 */
 	match = NULL;
 	matchidx = arraysize;
+	lck_mtx_lock(sadb_mutex);
 	LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) {
 		if (sav->spi != spi)
 			continue;
@@ -1106,30 +1160,156 @@ key_allocsa(family, src, dst, proto, spi)
 		goto found;
 
 	/* not found */
+	lck_mtx_unlock(sadb_mutex);
 	return NULL;
 
 found:
-	match->refcnt++;	
+	match->refcnt++;
+	lck_mtx_unlock(sadb_mutex);
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP allocsa cause refcnt++:%d SA:%p\n",
 			match->refcnt, match));
 	return match;
 }
 
+u_int16_t
+key_natt_get_translated_port(outsav)
+	struct secasvar *outsav;
+{
+
+	struct secasindex saidx;
+	struct secashead *sah;
+	u_int stateidx, state;
+	const u_int *saorder_state_valid;
+	int arraysize;
+	
+	/* get sa for incoming */
+	saidx.mode = outsav->sah->saidx.mode;
+	saidx.reqid = 0;
+	saidx.proto = outsav->sah->saidx.proto;
+	bcopy(&outsav->sah->saidx.src, &saidx.dst, sizeof(struct sockaddr_in));
+	bcopy(&outsav->sah->saidx.dst, &saidx.src, sizeof(struct sockaddr_in));
+	
+	lck_mtx_lock(sadb_mutex);
+	LIST_FOREACH(sah, &sahtree, chain) {
+		if (sah->state == SADB_SASTATE_DEAD)
+			continue;
+		if (key_cmpsaidx(&sah->saidx, &saidx, CMP_MODE))
+			goto found;
+	}
+	lck_mtx_unlock(sadb_mutex);
+	return 0;
+
+found:
+	/* 
+	 * Found sah - now go thru list of SAs and find
+	 * matching remote ike port.  If found - set
+	 * sav->natt_encapsulated_src_port and return the port.
+	 */
+	/*
+	 * search a valid state list for outbound packet.
+	 * This search order is important.
+	 */
+	if (key_preferred_oldsa) {
+		saorder_state_valid = saorder_state_valid_prefer_old;
+		arraysize = _ARRAYLEN(saorder_state_valid_prefer_old);
+	} else {
+		saorder_state_valid = saorder_state_valid_prefer_new;
+		arraysize = _ARRAYLEN(saorder_state_valid_prefer_new);
+	}
+
+	for (stateidx = 0; stateidx < arraysize; stateidx++) {
+		state = saorder_state_valid[stateidx];
+		if (key_do_get_translated_port(sah, outsav, state)) {
+			lck_mtx_unlock(sadb_mutex);
+			return outsav->natt_encapsulated_src_port;
+		}
+	}
+	lck_mtx_unlock(sadb_mutex);
+	return 0;
+}
+
+static int
+key_do_get_translated_port(sah, outsav, state)
+	struct secashead *sah;
+	struct secasvar *outsav; 
+	u_int state;
+{
+	struct secasvar *currsav, *nextsav, *candidate;
+
+
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	
+	/* initilize */
+	candidate = NULL;
+
+	for (currsav = LIST_FIRST(&sah->savtree[state]);
+	     currsav != NULL;
+	     currsav = nextsav) {
+
+		nextsav = LIST_NEXT(currsav, chain);
+
+		/* sanity check */
+		KEY_CHKSASTATE(currsav->state, state, "key_do_get_translated_port");
+		
+		if ((currsav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) == 0 ||
+			currsav->remote_ike_port != outsav->remote_ike_port)
+			continue;
+
+		if (candidate == NULL) {
+			candidate = currsav;
+			continue;
+		}
+		
+		/* Which SA is the better ? */
+
+		/* sanity check 2 */
+		if (candidate->lft_c == NULL || currsav->lft_c == NULL)
+			panic("key_do_get_translated_port: "
+				"lifetime_current is NULL.\n");
+
+		/* What the best method is to compare ? */
+		if (key_preferred_oldsa) {
+			if (candidate->lft_c->sadb_lifetime_addtime >
+					currsav->lft_c->sadb_lifetime_addtime) {
+				candidate = currsav;
+			}
+			continue;
+			/*NOTREACHED*/
+		}
+
+		/* prefered new sa rather than old sa */
+		if (candidate->lft_c->sadb_lifetime_addtime <
+				currsav->lft_c->sadb_lifetime_addtime) 
+			candidate = currsav;
+	}
+
+	if (candidate) { 
+		outsav->natt_encapsulated_src_port = candidate->natt_encapsulated_src_port;
+		return 1;
+	}
+
+	return 0;
+}
+
 /*
  * Must be called after calling key_allocsp().
  * For both the packet without socket and key_freeso().
  */
 void
-key_freesp(sp)
+key_freesp(sp, locked)
 	struct secpolicy *sp;
+	int locked;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check */
 	if (sp == NULL)
 		panic("key_freesp: NULL pointer is passed.\n");
-
+	
+	if (!locked)
+		lck_mtx_lock(sadb_mutex);
+	else
+		lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	sp->refcnt--;
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP freesp cause refcnt--:%d SP:%p\n",
@@ -1137,11 +1317,14 @@ key_freesp(sp)
 
 	if (sp->refcnt == 0)
 		key_delsp(sp);
-
+	if (!locked)
+		lck_mtx_unlock(sadb_mutex);
 	return;
 }
 
 #if 0
+static void key_freesp_so(struct secpolicy **);
+
 /*
  * Must be called after calling key_allocsp().
  * For the packet with socket.
@@ -1150,12 +1333,12 @@ void
 key_freeso(so)
 	struct socket *so;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL)
 		panic("key_freeso: NULL pointer is passed.\n");
 
+	lck_mtx_lock(sadb_mutex);
 	switch (so->so_proto->pr_domain->dom_family) {
 #if INET
 	case PF_INET:
@@ -1164,7 +1347,7 @@ key_freeso(so)
 
 		/* Does it have a PCB ? */
 		if (pcb == NULL || pcb->inp_sp == NULL)
-			return;
+			goto done;
 		key_freesp_so(&pcb->inp_sp->sp_in);
 		key_freesp_so(&pcb->inp_sp->sp_out);
 	    }
@@ -1178,7 +1361,7 @@ key_freeso(so)
 
 		/* Does it have a PCB ? */
 		if (pcb == NULL || pcb->inp_sp == NULL)
-			return;
+			goto done;
 		key_freesp_so(&pcb->inp_sp->sp_in);
 		key_freesp_so(&pcb->inp_sp->sp_out);
 #else
@@ -1186,7 +1369,7 @@ key_freeso(so)
 
 		/* Does it have a PCB ? */
 		if (pcb == NULL || pcb->in6p_sp == NULL)
-			return;
+			goto done;
 		key_freesp_so(&pcb->in6p_sp->sp_in);
 		key_freesp_so(&pcb->in6p_sp->sp_out);
 #endif
@@ -1196,29 +1379,30 @@ key_freeso(so)
 	default:
 		ipseclog((LOG_DEBUG, "key_freeso: unknown address family=%d.\n",
 		    so->so_proto->pr_domain->dom_family));
-		return;
+		break;
 	}
-
+done:
+	lck_mtx_unlock(sadb_mutex);
+	
 	return;
 }
-#endif
 
 static void
 key_freesp_so(sp)
 	struct secpolicy **sp;
 {
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check */
 	if (sp == NULL || *sp == NULL)
 		panic("key_freesp_so: sp == NULL\n");
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	
 	switch ((*sp)->policy) {
 	case IPSEC_POLICY_IPSEC:
 		KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 			printf("DP freeso calls free SP:%p\n", *sp));
-		key_freesp(*sp);
+		key_freesp(*sp, KEY_SADB_LOCKED);
 		*sp = NULL;
 		break;
 	case IPSEC_POLICY_ENTRUST:
@@ -1231,21 +1415,27 @@ key_freesp_so(sp)
 	return;
 }
 
+#endif
+
 /*
  * Must be called after calling key_allocsa().
  * This function is called by key_freesp() to free some SA allocated
  * for a policy.
  */
 void
-key_freesav(sav)
+key_freesav(sav, locked)
 	struct secasvar *sav;
+	int locked;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check */
 	if (sav == NULL)
 		panic("key_freesav: NULL pointer is passed.\n");
 
+	if (!locked)
+		lck_mtx_lock(sadb_mutex);
+	else
+		lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	sav->refcnt--;
 	KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
 		printf("DP freesav cause refcnt--:%d SA:%p SPI %u\n",
@@ -1253,7 +1443,8 @@ key_freesav(sav)
 
 	if (sav->refcnt == 0)
 		key_delsav(sav);
-
+	if (!locked)
+		lck_mtx_unlock(sadb_mutex);
 	return;
 }
 
@@ -1265,46 +1456,34 @@ static void
 key_delsp(sp)
 	struct secpolicy *sp;
 {
-	int s;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check */
 	if (sp == NULL)
 		panic("key_delsp: NULL pointer is passed.\n");
-
+		
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	sp->state = IPSEC_SPSTATE_DEAD;
 
 	if (sp->refcnt > 0)
 		return; /* can't free */
 
-	s = splnet();	/*called from softclock()*/
 	/* remove from SP index */
-	if (__LIST_CHAINED(sp))
+	if (__LIST_CHAINED(sp)) {
 		LIST_REMOVE(sp, chain);
+		ipsec_policy_count--;
+	}
 
     {
-	struct ipsecrequest *isr = sp->req, *nextisr;
+		struct ipsecrequest *isr = sp->req, *nextisr;
 
-	while (isr != NULL) {
-		if (isr->sav != NULL) {
-			KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
-				printf("DP delsp calls free SA:%p\n",
-					isr->sav));
-			key_freesav(isr->sav);
-			isr->sav = NULL;
-		}
-
-		nextisr = isr->next;
-		KFREE(isr);
-		isr = nextisr;
+		while (isr != NULL) {
+			nextisr = isr->next;
+			KFREE(isr);
+			isr = nextisr;
+    	}
 	}
-    }
-
 	keydb_delsecpolicy(sp);
 
-	splx(s);
-
 	return;
 }
 
@@ -1375,7 +1554,8 @@ struct secpolicy *
 key_newsp()
 {
 	struct secpolicy *newsp = NULL;
-
+	
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
 	newsp = keydb_newsecpolicy();
 	if (!newsp)
 		return newsp;
@@ -1399,8 +1579,8 @@ key_msg2sp(xpl0, len, error)
 {
 	struct secpolicy *newsp;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (xpl0 == NULL)
 		panic("key_msg2sp: NULL pointer was passed.\n");
@@ -1423,6 +1603,7 @@ key_msg2sp(xpl0, len, error)
 	/* check policy */
 	switch (xpl0->sadb_x_policy_type) {
 	case IPSEC_POLICY_DISCARD:
+        case IPSEC_POLICY_GENERATE:
 	case IPSEC_POLICY_NONE:
 	case IPSEC_POLICY_ENTRUST:
 	case IPSEC_POLICY_BYPASS:
@@ -1439,7 +1620,7 @@ key_msg2sp(xpl0, len, error)
 		if (PFKEY_EXTLEN(xpl0) < sizeof(*xpl0)) {
 			ipseclog((LOG_DEBUG,
 			    "key_msg2sp: Invalid msg length.\n"));
-			key_freesp(newsp);
+			key_freesp(newsp, KEY_SADB_UNLOCKED);
 			*error = EINVAL;
 			return NULL;
 		}
@@ -1453,17 +1634,17 @@ key_msg2sp(xpl0, len, error)
 			if (xisr->sadb_x_ipsecrequest_len < sizeof(*xisr)) {
 				ipseclog((LOG_DEBUG, "key_msg2sp: "
 					"invalid ipsecrequest length.\n"));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = EINVAL;
 				return NULL;
 			}
 
 			/* allocate request buffer */
-			KMALLOC(*p_isr, struct ipsecrequest *, sizeof(**p_isr));
+			KMALLOC_WAIT(*p_isr, struct ipsecrequest *, sizeof(**p_isr));
 			if ((*p_isr) == NULL) {
 				ipseclog((LOG_DEBUG,
 				    "key_msg2sp: No more memory.\n"));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = ENOBUFS;
 				return NULL;
 			}
@@ -1481,7 +1662,7 @@ key_msg2sp(xpl0, len, error)
 				ipseclog((LOG_DEBUG,
 				    "key_msg2sp: invalid proto type=%u\n",
 				    xisr->sadb_x_ipsecrequest_proto));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = EPROTONOSUPPORT;
 				return NULL;
 			}
@@ -1496,7 +1677,7 @@ key_msg2sp(xpl0, len, error)
 				ipseclog((LOG_DEBUG,
 				    "key_msg2sp: invalid mode=%u\n",
 				    xisr->sadb_x_ipsecrequest_mode));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = EINVAL;
 				return NULL;
 			}
@@ -1526,7 +1707,7 @@ key_msg2sp(xpl0, len, error)
 				if (xisr->sadb_x_ipsecrequest_reqid == 0) {
 					u_int32_t reqid;
 					if ((reqid = key_newreqid()) == 0) {
-						key_freesp(newsp);
+						key_freesp(newsp, KEY_SADB_UNLOCKED);
 						*error = ENOBUFS;
 						return NULL;
 					}
@@ -1542,7 +1723,7 @@ key_msg2sp(xpl0, len, error)
 			default:
 				ipseclog((LOG_DEBUG, "key_msg2sp: invalid level=%u\n",
 					xisr->sadb_x_ipsecrequest_level));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = EINVAL;
 				return NULL;
 			}
@@ -1559,7 +1740,7 @@ key_msg2sp(xpl0, len, error)
 				    > sizeof((*p_isr)->saidx.src)) {
 					ipseclog((LOG_DEBUG, "key_msg2sp: invalid request "
 						"address length.\n"));
-					key_freesp(newsp);
+					key_freesp(newsp, KEY_SADB_UNLOCKED);
 					*error = EINVAL;
 					return NULL;
 				}
@@ -1574,7 +1755,7 @@ key_msg2sp(xpl0, len, error)
 				    > sizeof((*p_isr)->saidx.dst)) {
 					ipseclog((LOG_DEBUG, "key_msg2sp: invalid request "
 						"address length.\n"));
-					key_freesp(newsp);
+					key_freesp(newsp, KEY_SADB_UNLOCKED);
 					*error = EINVAL;
 					return NULL;
 				}
@@ -1582,7 +1763,6 @@ key_msg2sp(xpl0, len, error)
 					paddr->sa_len);
 			}
 
-			(*p_isr)->sav = NULL;
 			(*p_isr)->sp = newsp;
 
 			/* initialization for the next. */
@@ -1592,7 +1772,7 @@ key_msg2sp(xpl0, len, error)
 			/* validity check */
 			if (tlen < 0) {
 				ipseclog((LOG_DEBUG, "key_msg2sp: becoming tlen < 0.\n"));
-				key_freesp(newsp);
+				key_freesp(newsp, KEY_SADB_UNLOCKED);
 				*error = EINVAL;
 				return NULL;
 			}
@@ -1604,7 +1784,7 @@ key_msg2sp(xpl0, len, error)
 		break;
 	default:
 		ipseclog((LOG_DEBUG, "key_msg2sp: invalid policy type.\n"));
-		key_freesp(newsp);
+		key_freesp(newsp, KEY_SADB_UNLOCKED);
 		*error = EINVAL;
 		return NULL;
 	}
@@ -1616,12 +1796,12 @@ key_msg2sp(xpl0, len, error)
 static u_int32_t
 key_newreqid()
 {
+	lck_mtx_lock(sadb_mutex);
 	static u_int32_t auto_reqid = IPSEC_MANUAL_REQID_MAX + 1;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	auto_reqid = (auto_reqid == ~0
 			? IPSEC_MANUAL_REQID_MAX + 1 : auto_reqid + 1);
+	lck_mtx_unlock(sadb_mutex);
 
 	/* XXX should be unique check */
 
@@ -1640,8 +1820,6 @@ key_sp2msg(sp)
 	caddr_t p;
 	struct mbuf *m;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check. */
 	if (sp == NULL)
 		panic("key_sp2msg: NULL pointer was passed.\n");
@@ -1800,8 +1978,8 @@ key_spdadd(so, m, mhp)
 	struct timeval tv;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spdadd: NULL pointer is passed.\n");
@@ -1861,7 +2039,7 @@ key_spdadd(so, m, mhp)
 	}
 
 	/* policy requests are mandatory when action is ipsec. */
-        if (mhp->msg->sadb_msg_type != SADB_X_SPDSETIDX
+     if (mhp->msg->sadb_msg_type != SADB_X_SPDSETIDX
 	 && xpl0->sadb_x_policy_type == IPSEC_POLICY_IPSEC
 	 && mhp->extlen[SADB_X_EXT_POLICY] <= sizeof(*xpl0)) {
 		ipseclog((LOG_DEBUG, "key_spdadd: some policy requests part required.\n"));
@@ -1874,20 +2052,22 @@ key_spdadd(so, m, mhp)
 	 * If the type is either SPDADD or SPDSETIDX AND a SP is found,
 	 * then error.
 	 */
+	lck_mtx_lock(sadb_mutex);
 	newsp = key_getsp(&spidx);
 	if (mhp->msg->sadb_msg_type == SADB_X_SPDUPDATE) {
 		if (newsp) {
 			newsp->state = IPSEC_SPSTATE_DEAD;
-			key_freesp(newsp);
+			key_freesp(newsp, KEY_SADB_LOCKED);
 		}
 	} else {
 		if (newsp != NULL) {
-			key_freesp(newsp);
+			key_freesp(newsp, KEY_SADB_LOCKED);
 			ipseclog((LOG_DEBUG, "key_spdadd: a SP entry exists already.\n"));
+			lck_mtx_unlock(sadb_mutex);
 			return key_senderror(so, m, EEXIST);
 		}
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	/* allocation new SP entry */
 	if ((newsp = key_msg2sp(xpl0, PFKEY_EXTLEN(xpl0), &error)) == NULL) {
 		return key_senderror(so, m, error);
@@ -1919,20 +2099,30 @@ key_spdadd(so, m, mhp)
 		return key_senderror(so, m, EINVAL);
 	}
 #if 1
+	/* 
+	 * allow IPv6 over IPv4 tunnels using ESP - 
+	 * otherwise reject if inner and outer address families not equal 
+	 */
 	if (newsp->req && newsp->req->saidx.src.ss_family) {
 		struct sockaddr *sa;
 		sa = (struct sockaddr *)(src0 + 1);
 		if (sa->sa_family != newsp->req->saidx.src.ss_family) {
-			keydb_delsecpolicy(newsp);
-			return key_senderror(so, m, EINVAL);
+			if (newsp->req->saidx.mode != IPSEC_MODE_TUNNEL || newsp->req->saidx.proto != IPPROTO_ESP
+			    || sa->sa_family != AF_INET6 || newsp->req->saidx.src.ss_family != AF_INET) {
+				keydb_delsecpolicy(newsp);
+				return key_senderror(so, m, EINVAL);
+			}
 		}
 	}
 	if (newsp->req && newsp->req->saidx.dst.ss_family) {
 		struct sockaddr *sa;
 		sa = (struct sockaddr *)(dst0 + 1);
 		if (sa->sa_family != newsp->req->saidx.dst.ss_family) {
-			keydb_delsecpolicy(newsp);
-			return key_senderror(so, m, EINVAL);
+			if (newsp->req->saidx.mode != IPSEC_MODE_TUNNEL || newsp->req->saidx.proto != IPPROTO_ESP
+			    || sa->sa_family != AF_INET6 || newsp->req->saidx.dst.ss_family != AF_INET) {
+				keydb_delsecpolicy(newsp);
+				return key_senderror(so, m, EINVAL);
+			}
 		}
 	}
 #endif
@@ -1945,8 +2135,26 @@ key_spdadd(so, m, mhp)
 
 	newsp->refcnt = 1;	/* do not reclaim until I say I do */
 	newsp->state = IPSEC_SPSTATE_ALIVE;
-	LIST_INSERT_TAIL(&sptree[newsp->spidx.dir], newsp, secpolicy, chain);
+	lck_mtx_lock(sadb_mutex);
+	/*
+	 * policies of type generate should be at the end of the SPD
+	 * because they function as default discard policies
+       	 */
+	if (newsp->policy == IPSEC_POLICY_GENERATE)
+		LIST_INSERT_TAIL(&sptree[newsp->spidx.dir], newsp, secpolicy, chain);
+	else {  /* XXX until we have policy ordering in the kernel */
+		struct secpolicy *tmpsp;
+
+		LIST_FOREACH(tmpsp, &sptree[newsp->spidx.dir], chain)
+			if (tmpsp->policy == IPSEC_POLICY_GENERATE)
+				break;
+		if (tmpsp)
+			LIST_INSERT_BEFORE(tmpsp, newsp, chain);
+		else
+			LIST_INSERT_TAIL(&sptree[newsp->spidx.dir], newsp, secpolicy, chain);
+	}
 
+	ipsec_policy_count++;
 	/* Turn off the ipsec bypass */
 	if (ipsec_bypass != 0)
 		ipsec_bypass = 0;
@@ -1960,7 +2168,8 @@ key_spdadd(so, m, mhp)
 			spacq->created = tv.tv_sec;
 			spacq->count = 0;
 		}
-    	}
+    }
+	lck_mtx_unlock(sadb_mutex);
 
     {
 	struct mbuf *n, *mpolicy;
@@ -2021,19 +2230,18 @@ key_getnewspid()
 	u_int32_t newid = 0;
 	int count = key_spi_trycnt;	/* XXX */
 	struct secpolicy *sp;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* when requesting to allocate spi ranged */
+	lck_mtx_lock(sadb_mutex);
 	while (count--) {
 		newid = (policy_id = (policy_id == ~0 ? 1 : policy_id + 1));
 
 		if ((sp = key_getspbyid(newid)) == NULL)
 			break;
 
-		key_freesp(sp);
+		key_freesp(sp, KEY_SADB_LOCKED);
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	if (count == 0 || newid == 0) {
 		ipseclog((LOG_DEBUG, "key_getnewspid: to allocate policy id is failed.\n"));
 		return 0;
@@ -2065,8 +2273,8 @@ key_spddelete(so, m, mhp)
 	struct secpolicyindex spidx;
 	struct secpolicy *sp;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spddelete: NULL pointer is passed.\n");
@@ -2109,8 +2317,10 @@ key_spddelete(so, m, mhp)
 	}
 
 	/* Is there SP in SPD ? */
+	lck_mtx_lock(sadb_mutex);
 	if ((sp = key_getsp(&spidx)) == NULL) {
 		ipseclog((LOG_DEBUG, "key_spddelete: no SP found.\n"));
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, EINVAL);
 	}
 
@@ -2118,7 +2328,9 @@ key_spddelete(so, m, mhp)
 	xpl0->sadb_x_policy_id = sp->id;
 
 	sp->state = IPSEC_SPSTATE_DEAD;
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_LOCKED);
+	lck_mtx_unlock(sadb_mutex);
+
 
     {
 	struct mbuf *n;
@@ -2161,8 +2373,8 @@ key_spddelete2(so, m, mhp)
 	u_int32_t id;
 	struct secpolicy *sp;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spddelete2: NULL pointer is passed.\n");
@@ -2177,13 +2389,16 @@ key_spddelete2(so, m, mhp)
 	id = ((struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY])->sadb_x_policy_id;
 
 	/* Is there SP in SPD ? */
+	lck_mtx_lock(sadb_mutex);
 	if ((sp = key_getspbyid(id)) == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_spddelete2: no SP found id:%u.\n", id));
-		key_senderror(so, m, EINVAL);
+		return key_senderror(so, m, EINVAL);
 	}
 
 	sp->state = IPSEC_SPSTATE_DEAD;
-	key_freesp(sp);
+	key_freesp(sp, KEY_SADB_LOCKED);
+	lck_mtx_unlock(sadb_mutex);
 
     {
 	struct mbuf *n, *nn;
@@ -2260,8 +2475,8 @@ key_spdget(so, m, mhp)
 	struct secpolicy *sp;
 	struct mbuf *n;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spdget: NULL pointer is passed.\n");
@@ -2275,11 +2490,13 @@ key_spdget(so, m, mhp)
 	id = ((struct sadb_x_policy *)mhp->ext[SADB_X_EXT_POLICY])->sadb_x_policy_id;
 
 	/* Is there SP in SPD ? */
+	lck_mtx_lock(sadb_mutex);
 	if ((sp = key_getspbyid(id)) == NULL) {
 		ipseclog((LOG_DEBUG, "key_spdget: no SP found id:%u.\n", id));
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, ENOENT);
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	n = key_setdumpsp(sp, SADB_X_SPDGET, 0, mhp->msg->sadb_msg_pid);
 	if (n != NULL) {
 		m_freem(m);
@@ -2311,8 +2528,8 @@ key_spdacquire(sp)
 	struct secspacq *newspacq;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (sp == NULL)
 		panic("key_spdacquire: NULL pointer is passed.\n");
@@ -2322,6 +2539,7 @@ key_spdacquire(sp)
 		panic("key_spdacquire: policy mismathed. IPsec is expected.\n");
 
 	/* get a entry to check whether sent message or not. */
+	lck_mtx_lock(sadb_mutex);
 	if ((newspacq = key_getspacq(&sp->spidx)) != NULL) {
 		if (key_blockacq_count < newspacq->count) {
 			/* reset counter and do send message. */
@@ -2329,17 +2547,19 @@ key_spdacquire(sp)
 		} else {
 			/* increment counter and do nothing. */
 			newspacq->count++;
+			lck_mtx_unlock(sadb_mutex);
 			return 0;
 		}
 	} else {
 		/* make new entry for blocking to send SADB_ACQUIRE. */
-		if ((newspacq = key_newspacq(&sp->spidx)) == NULL)
+		if ((newspacq = key_newspacq(&sp->spidx)) == NULL) {
+			lck_mtx_unlock(sadb_mutex);
 			return ENOBUFS;
-
+		}
 		/* add to acqtree */
 		LIST_INSERT_HEAD(&spacqtree, newspacq, chain);
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	/* create new sadb_msg to reply. */
 	m = key_setsadbmsg(SADB_X_SPDACQUIRE, 0, 0, 0, 0, 0);
 	if (!m) {
@@ -2384,9 +2604,7 @@ key_spdflush(so, m, mhp)
 	struct sadb_msg *newmsg;
 	struct secpolicy *sp;
 	u_int dir;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spdflush: NULL pointer is passed.\n");
@@ -2394,12 +2612,14 @@ key_spdflush(so, m, mhp)
 	if (m->m_len != PFKEY_ALIGN8(sizeof(struct sadb_msg)))
 		return key_senderror(so, m, EINVAL);
 
+	lck_mtx_lock(sadb_mutex);
 	for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
 		LIST_FOREACH(sp, &sptree[dir], chain) {
 			sp->state = IPSEC_SPSTATE_DEAD;
 		}
 	}
-
+	lck_mtx_unlock(sadb_mutex);
+	
 	if (sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
 		ipseclog((LOG_DEBUG, "key_spdflush: No more memory.\n"));
 		return key_senderror(so, m, ENOBUFS);
@@ -2427,47 +2647,77 @@ key_spdflush(so, m, mhp)
  *
  * m will always be freed.
  */
+ 	
 static int
 key_spddump(so, m, mhp)
 	struct socket *so;
 	struct mbuf *m;
 	const struct sadb_msghdr *mhp;
 {
-	struct secpolicy *sp;
-	int cnt;
+	struct secpolicy *sp, **spbuf = NULL, **sp_ptr;
+	int cnt = 0, bufcount;
 	u_int dir;
 	struct mbuf *n;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	int error = 0;
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_spddump: NULL pointer is passed.\n");
 
-	/* search SPD entry and get buffer size. */
-	cnt = 0;
+	if ((bufcount = ipsec_policy_count) == 0) {
+		error = ENOENT;
+		goto end;
+	}
+	bufcount += 256;	/* extra */
+	KMALLOC_WAIT(spbuf, struct secpolicy**, bufcount * sizeof(struct secpolicy*));
+	if (spbuf == NULL) {
+		ipseclog((LOG_DEBUG, "key_spddump: No more memory.\n"));
+		error = ENOMEM;
+		goto end;
+	}
+	lck_mtx_lock(sadb_mutex);
+	/* search SPD entry, make list. */
+	sp_ptr = spbuf;
 	for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
 		LIST_FOREACH(sp, &sptree[dir], chain) {
+			if (cnt == bufcount)	
+				break;		/* buffer full */
+			*sp_ptr++ = sp;
+			sp->refcnt++;
 			cnt++;
 		}
 	}
+	lck_mtx_unlock(sadb_mutex);
 
-	if (cnt == 0)
-		return key_senderror(so, m, ENOENT);
-
-	for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
-		LIST_FOREACH(sp, &sptree[dir], chain) {
-			--cnt;
-			n = key_setdumpsp(sp, SADB_X_SPDDUMP, cnt,
-			    mhp->msg->sadb_msg_pid);
+	if (cnt == 0) {
+		error = ENOENT;
+		goto end;
+	}
+	
+	sp_ptr = spbuf;
+	while (cnt) {
+		--cnt;
+		n = key_setdumpsp(*sp_ptr++, SADB_X_SPDDUMP, cnt,
+			mhp->msg->sadb_msg_pid);
 
-			if (n)
-				key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
-		}
+		if (n)
+			key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
 	}
+	
+	lck_mtx_lock(sadb_mutex);
+	while (sp_ptr > spbuf)
+		key_freesp(*(--sp_ptr), KEY_SADB_LOCKED);
+	lck_mtx_unlock(sadb_mutex);
+	
+end:	
+	if (spbuf)
+		KFREE(spbuf);
+	if (error)
+		return key_senderror(so, m, error);
 
 	m_freem(m);
 	return 0;
+
 }
 
 static struct mbuf *
@@ -2478,8 +2728,6 @@ key_setdumpsp(sp, type, seq, pid)
 {
 	struct mbuf *result = NULL, *m;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	m = key_setsadbmsg(type, 0, SADB_SATYPE_UNSPEC, seq, pid, sp->refcnt);
 	if (!m)
 		goto fail;
@@ -2536,8 +2784,6 @@ key_getspreqmsglen(sp)
 {
 	u_int tlen;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	tlen = sizeof(struct sadb_x_policy);
 
 	/* if is the policy for ipsec ? */
@@ -2574,14 +2820,13 @@ static int
 key_spdexpire(sp)
 	struct secpolicy *sp;
 {
-	int s;
 	struct mbuf *result = NULL, *m;
 	int len;
 	int error = -1;
 	struct sadb_lifetime *lt;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (sp == NULL)
 		panic("key_spdexpire: NULL pointer is passed.\n");
@@ -2688,8 +2933,6 @@ key_newsah(saidx)
 {
 	struct secashead *newsah;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
 	/* sanity check */
 	if (saidx == NULL)
 		panic("key_newsaidx: NULL pointer is passed.\n");
@@ -2699,6 +2942,28 @@ key_newsah(saidx)
 		return NULL;
 
 	bcopy(saidx, &newsah->saidx, sizeof(newsah->saidx));
+	
+	/* remove the ports */
+	switch (saidx->src.ss_family) {
+	        case AF_INET:
+			((struct sockaddr_in *)(&newsah->saidx.src))->sin_port = IPSEC_PORT_ANY;
+			break;
+        	case AF_INET6:
+		        ((struct sockaddr_in6 *)(&newsah->saidx.src))->sin6_port = IPSEC_PORT_ANY;
+			break;
+	        default:
+			break;
+	}
+	switch (saidx->dst.ss_family) {
+        	case AF_INET:
+		  ((struct sockaddr_in *)(&newsah->saidx.dst))->sin_port = IPSEC_PORT_ANY;
+			break;
+        	case AF_INET6:
+			((struct sockaddr_in6 *)(&newsah->saidx.dst))->sin6_port = IPSEC_PORT_ANY;
+			break;
+        	default:
+			break;
+        }
 
 	/* add to saidxtree */
 	newsah->state = SADB_SASTATE_MATURE;
@@ -2716,7 +2981,6 @@ key_delsah(sah)
 {
 	struct secasvar *sav, *nextsav;
 	u_int stateidx, state;
-	int s;
 	int zombie = 0;
 
 	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
@@ -2725,8 +2989,6 @@ key_delsah(sah)
 	if (sah == NULL)
 		panic("key_delsah: NULL pointer is passed.\n");
 
-	s = splnet();	/*called from softclock()*/
-
 	/* searching all SA registerd in the secindex. */
 	for (stateidx = 0;
 	     stateidx < _ARRAYLEN(saorder_state_any);
@@ -2748,7 +3010,7 @@ key_delsah(sah)
 			/* sanity check */
 			KEY_CHKSASTATE(state, sav->state, "key_delsah");
 
-			key_freesav(sav);
+			key_freesav(sav, KEY_SADB_LOCKED);
 
 			/* remove back pointer */
 			sav->sah = NULL;
@@ -2757,10 +3019,8 @@ key_delsah(sah)
 	}
 
 	/* don't delete sah only if there are savs. */
-	if (zombie) {
-		splx(s);
+	if (zombie)
 		return;
-	}
 
 	if (sah->sa_route.ro_rt) {
 		rtfree(sah->sa_route.ro_rt);
@@ -2773,7 +3033,6 @@ key_delsah(sah)
 
 	KFREE(sah);
 
-	splx(s);
 	return;
 }
 
@@ -2805,11 +3064,16 @@ key_newsav(m, mhp, sah, errp)
 	if (m == NULL || mhp == NULL || mhp->msg == NULL || sah == NULL)
 		panic("key_newsa: NULL pointer is passed.\n");
 
-	KMALLOC(newsav, struct secasvar *, sizeof(struct secasvar));
+	KMALLOC_NOWAIT(newsav, struct secasvar *, sizeof(struct secasvar));
 	if (newsav == NULL) {
-		ipseclog((LOG_DEBUG, "key_newsa: No more memory.\n"));
-		*errp = ENOBUFS;
-		return NULL;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(newsav, struct secasvar *, sizeof(struct secasvar));
+		lck_mtx_lock(sadb_mutex);
+		if (newsav == NULL) {
+			ipseclog((LOG_DEBUG, "key_newsa: No more memory.\n"));
+			*errp = ENOBUFS;
+			return NULL;
+		}
 	}
 	bzero((caddr_t)newsav, sizeof(struct secasvar));
 
@@ -2871,6 +3135,7 @@ key_newsav(m, mhp, sah, errp)
 	newsav->state = SADB_SASTATE_LARVAL;
 	LIST_INSERT_TAIL(&sah->savtree[SADB_SASTATE_LARVAL], newsav,
 			secasvar, chain);
+	ipsec_sav_count++;
 
 	return newsav;
 }
@@ -2882,18 +3147,20 @@ static void
 key_delsav(sav)
 	struct secasvar *sav;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	
 	/* sanity check */
 	if (sav == NULL)
 		panic("key_delsav: NULL pointer is passed.\n");
 
 	if (sav->refcnt > 0)
 		return;		/* can't free */
-
+	
 	/* remove from SA header */
 	if (__LIST_CHAINED(sav))
 		LIST_REMOVE(sav, chain);
+	ipsec_sav_count--;
 		
 	if (sav->spihash.le_prev || sav->spihash.le_next)
 		LIST_REMOVE(sav, spihash);
@@ -3008,7 +3275,7 @@ key_setspi(sav, spi)
 	struct secasvar *sav;
 	u_int32_t spi;
 {
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	sav->spi = spi;
 	if (sav->spihash.le_prev || sav->spihash.le_next)
 		LIST_REMOVE(sav, spihash);
@@ -3030,6 +3297,7 @@ key_getsavbyspi(sah, spi)
 	struct secasvar *sav, *match;
 	u_int stateidx, state, matchidx;
 			
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	match = NULL;
 	matchidx = _ARRAYLEN(saorder_state_alive);
 	LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) {
@@ -3088,6 +3356,7 @@ key_setsaval(sav, m, mhp)
 	sav->lft_s = NULL;
 	sav->remote_ike_port = 0;
 	sav->natt_last_activity = natt_now;
+	sav->natt_encapsulated_src_port = 0;
 
 	/* SA */
 	if (mhp->ext[SADB_EXT_SA] != NULL) {
@@ -3117,6 +3386,16 @@ key_setsaval(sav, m, mhp)
 			}
 			sav->remote_ike_port = ((struct sadb_sa_2*)(sa0))->sadb_sa_natt_port;
 		}
+		
+		/*
+		 * Verify if SADB_X_EXT_NATT_MULTIPLEUSERS flag is set that
+		 * SADB_X_EXT_NATT is set and SADB_X_EXT_NATT_KEEPALIVE is not 
+		 * set (we're not behind nat) - otherwise clear it.
+		 */
+		if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0)
+			if ((sav->flags & SADB_X_EXT_NATT) == 0 ||
+				(sav->flags & SADB_X_EXT_NATT_KEEPALIVE) != 0)
+				sav->flags &= ~SADB_X_EXT_NATT_MULTIPLEUSERS;
 
 		/* replay window */
 		if ((sa0->sadb_sa_flags & SADB_X_EXT_OLD) == 0) {
@@ -3224,11 +3503,16 @@ key_setsaval(sav, m, mhp)
 			sav->ivlen = (*algo->ivlen)(algo, sav);
 		if (sav->ivlen == 0)
 			break;
-		KMALLOC(sav->iv, caddr_t, sav->ivlen);
+		KMALLOC_NOWAIT(sav->iv, caddr_t, sav->ivlen);
 		if (sav->iv == 0) {
-			ipseclog((LOG_DEBUG, "key_setsaval: No more memory.\n"));
-			error = ENOBUFS;
-			goto fail;
+			lck_mtx_unlock(sadb_mutex);
+			KMALLOC_WAIT(sav->iv, caddr_t, sav->ivlen);
+			lck_mtx_lock(sadb_mutex);
+			if (sav->iv == 0) {
+				ipseclog((LOG_DEBUG, "key_setsaval: No more memory.\n"));
+				error = ENOBUFS;
+				goto fail;
+			}
 		}
 
 		/* initialize */
@@ -3249,12 +3533,18 @@ key_setsaval(sav, m, mhp)
 	sav->created = tv.tv_sec;
 
 	/* make lifetime for CURRENT */
-	KMALLOC(sav->lft_c, struct sadb_lifetime *,
+	KMALLOC_NOWAIT(sav->lft_c, struct sadb_lifetime *,
 	    sizeof(struct sadb_lifetime));
 	if (sav->lft_c == NULL) {
-		ipseclog((LOG_DEBUG, "key_setsaval: No more memory.\n"));
-		error = ENOBUFS;
-		goto fail;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(sav->lft_c, struct sadb_lifetime *,
+	    	sizeof(struct sadb_lifetime));
+	    lck_mtx_lock(sadb_mutex);
+		if (sav->lft_c == NULL) {
+			ipseclog((LOG_DEBUG, "key_setsaval: No more memory.\n"));
+			error = ENOBUFS;
+			goto fail;
+		}
 	}
 
 	microtime(&tv);
@@ -3549,9 +3839,7 @@ key_setdumpsa(sav, type, satype, seq, pid)
 		SADB_EXT_KEY_ENCRYPT, SADB_EXT_IDENTITY_SRC,
 		SADB_EXT_IDENTITY_DST, SADB_EXT_SENSITIVITY,
 	};
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	m = key_setsadbmsg(type, 0, satype, seq, pid, sav->refcnt);
 	if (m == NULL)
 		goto fail;
@@ -3928,10 +4216,16 @@ key_newbuf(src, len)
 {
 	caddr_t new;
 
-	KMALLOC(new, caddr_t, len);
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+	KMALLOC_NOWAIT(new, caddr_t, len);
 	if (new == NULL) {
-		ipseclog((LOG_DEBUG, "key_newbuf: No more memory.\n"));
-		return NULL;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(new, caddr_t, len);
+		lck_mtx_lock(sadb_mutex);
+		if (new == NULL) {
+			ipseclog((LOG_DEBUG, "key_newbuf: No more memory.\n"));
+			return NULL;
+		}
 	}
 	bcopy(src, new, len);
 
@@ -4068,8 +4362,7 @@ key_cmpsaidx(saidx0, saidx1, flag)
 	} else {
 
 		/* CMP_MODE_REQID, CMP_REQID, CMP_HEAD */
-		if (flag == CMP_MODE_REQID
-		  ||flag == CMP_REQID) {
+		if (flag & CMP_REQID) {
 			/*
 			 * If reqid of SPD is non-zero, unique SA is required.
 			 * The result must be of same reqid in this case.
@@ -4078,18 +4371,18 @@ key_cmpsaidx(saidx0, saidx1, flag)
 				return 0;
 		}
 
-		if (flag == CMP_MODE_REQID) {
+		if (flag & CMP_MODE) {
 			if (saidx0->mode != IPSEC_MODE_ANY
 			 && saidx0->mode != saidx1->mode)
 				return 0;
 		}
 
 		if (key_sockaddrcmp((struct sockaddr *)&saidx0->src,
-				(struct sockaddr *)&saidx1->src, 0) != 0) {
+				    (struct sockaddr *)&saidx1->src, flag & CMP_PORT ? 1 : 0) != 0) {
 			return 0;
 		}
 		if (key_sockaddrcmp((struct sockaddr *)&saidx0->dst,
-				(struct sockaddr *)&saidx1->dst, 0) != 0) {
+				    (struct sockaddr *)&saidx1->dst, flag & CMP_PORT ? 1 : 0) != 0) {
 			return 0;
 		}
 	}
@@ -4276,6 +4569,7 @@ key_sockaddrcmp(sa1, sa2, port)
 		    satosin6(sa1)->sin6_port != satosin6(sa2)->sin6_port) {
 			return 1;
 		}
+		break;
 	default:
 		if (bcmp(sa1, sa2, sa1->sa_len) != 0)
 			return 1;
@@ -4334,226 +4628,260 @@ void
 key_timehandler(void)
 {
 	u_int dir;
-	int s;
 	struct timeval tv;
-
+	struct secpolicy **spbuf = NULL, **spptr = NULL;
+	struct secasvar **savexbuf = NULL, **savexptr = NULL;
+	struct secasvar **savkabuf = NULL, **savkaptr = NULL;
+	int spbufcount = 0, savbufcount = 0, spcount = 0, savexcount = 0, savkacount = 0, cnt;
+	
 	microtime(&tv);
 
+	/* pre-allocate buffers before taking the lock */
+	/* if allocation failures occur - portions of the processing will be skipped */
+	if ((spbufcount = ipsec_policy_count) != 0) {
+		spbufcount += 256;
+		KMALLOC_WAIT(spbuf, struct secpolicy **, spbufcount * sizeof(struct secpolicy *));
+		if (spbuf)
+			spptr = spbuf;
+	}
+	if ((savbufcount = ipsec_sav_count) != 0) {
+		savbufcount += 512;
+		KMALLOC_WAIT(savexbuf, struct secasvar **, savbufcount * sizeof(struct secasvar *));
+		if (savexbuf)
+			savexptr = savexbuf;
+		KMALLOC_WAIT(savkabuf, struct secasvar **, savbufcount * sizeof(struct secasvar *));
+		if (savkabuf)
+			savkaptr = savkabuf;
+	}
 	lck_mtx_lock(sadb_mutex);
 	/* SPD */
-    {
-	struct secpolicy *sp, *nextsp;
+	if (spbuf) {
 
-	for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
-		for (sp = LIST_FIRST(&sptree[dir]);
-		     sp != NULL;
-		     sp = nextsp) {
+		struct secpolicy *sp, *nextsp;
 
-			nextsp = LIST_NEXT(sp, chain);
+		for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+			for (sp = LIST_FIRST(&sptree[dir]);
+			     sp != NULL;
+			     sp = nextsp) {
 
-			if (sp->state == IPSEC_SPSTATE_DEAD) {
-				key_freesp(sp);
-				continue;
-			}
+				nextsp = LIST_NEXT(sp, chain);
 
-			if (sp->lifetime == 0 && sp->validtime == 0)
-				continue;
+				if (sp->state == IPSEC_SPSTATE_DEAD) {
+					key_freesp(sp, KEY_SADB_LOCKED);
+					continue;
+				}
 
-			/* the deletion will occur next time */
-			if ((sp->lifetime
-			  && tv.tv_sec - sp->created > sp->lifetime)
-			 || (sp->validtime
-			  && tv.tv_sec - sp->lastused > sp->validtime)) {
-				sp->state = IPSEC_SPSTATE_DEAD;
-				key_spdexpire(sp);
-				continue;
+				if (sp->lifetime == 0 && sp->validtime == 0)
+					continue;
+				if (spbuf && spcount < spbufcount) {
+					/* the deletion will occur next time */
+					if ((sp->lifetime
+					     && tv.tv_sec - sp->created > sp->lifetime)
+					    || (sp->validtime
+						&& tv.tv_sec - sp->lastused > sp->validtime)) {
+						//key_spdexpire(sp);
+						sp->state = IPSEC_SPSTATE_DEAD;
+						sp->refcnt++;
+						*spptr++ = sp;
+						spcount++;
+					}
+				}
 			}
 		}
 	}
-    }
 
 	/* SAD */
-    {
-	struct secashead *sah, *nextsah;
-	struct secasvar *sav, *nextsav;
-
-	for (sah = LIST_FIRST(&sahtree);
-	     sah != NULL;
-	     sah = nextsah) {
-
-		nextsah = LIST_NEXT(sah, chain);
-
-		/* if sah has been dead, then delete it and process next sah. */
-		if (sah->state == SADB_SASTATE_DEAD) {
-			key_delsah(sah);
-			continue;
-		}
-
-		/* if LARVAL entry doesn't become MATURE, delete it. */
-		for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_LARVAL]);
-		     sav != NULL;
-		     sav = nextsav) {
-
-			nextsav = LIST_NEXT(sav, chain);
-
-			if (tv.tv_sec - sav->created > key_larval_lifetime) {
-				key_freesav(sav);
-			}
-		}
-		
-		/*
-		 * If this is a NAT traversal SA with no activity,
-		 * we need to send a keep alive.
-		 *
-		 * Performed outside of the loop before so we will
-		 * only ever send one keepalive. The first SA on
-		 * the list is the one that will be used for sending
-		 * traffic, so this is the one we use for determining
-		 * when to send the keepalive.
-		 */
-		sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_MATURE]);
-		if (natt_keepalive_interval && sav && (sav->flags & SADB_X_EXT_NATT_KEEPALIVE) != 0 &&
-			(natt_now - sav->natt_last_activity) >= natt_keepalive_interval) {
-			ipsec_send_natt_keepalive(sav);
-		}
-		
-		/*
-		 * check MATURE entry to start to send expire message
-		 * whether or not.
-		 */
-		for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_MATURE]);
-		     sav != NULL;
-		     sav = nextsav) {
-
-			nextsav = LIST_NEXT(sav, chain);
-
-			/* we don't need to check. */
-			if (sav->lft_s == NULL)
-				continue;
-
-			/* sanity check */
-			if (sav->lft_c == NULL) {
-				ipseclog((LOG_DEBUG,"key_timehandler: "
-					"There is no CURRENT time, why?\n"));
+	if (savbufcount != 0) {
+		struct secashead *sah, *nextsah;
+		struct secasvar *sav, *nextsav;
+	
+		for (sah = LIST_FIRST(&sahtree);
+			 sah != NULL;
+			 sah = nextsah) {
+	
+			nextsah = LIST_NEXT(sah, chain);
+	
+			/* if sah has been dead, then delete it and process next sah. */
+			if (sah->state == SADB_SASTATE_DEAD) {
+				key_delsah(sah);
 				continue;
 			}
-
-			/* check SOFT lifetime */
-			if (sav->lft_s->sadb_lifetime_addtime != 0
-			 && tv.tv_sec - sav->created > sav->lft_s->sadb_lifetime_addtime) {
+	
+			/* if LARVAL entry doesn't become MATURE, delete it. */
+			for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_LARVAL]);
+				 sav != NULL;
+				 sav = nextsav) {
+	
+				nextsav = LIST_NEXT(sav, chain);
+	
+				if (tv.tv_sec - sav->created > key_larval_lifetime) {
+					key_freesav(sav, KEY_SADB_LOCKED);
+				}
+			}
+			
+			/*
+			 * If this is a NAT traversal SA with no activity,
+			 * we need to send a keep alive.
+			 *
+			 * Performed outside of the loop before so we will
+			 * only ever send one keepalive. The first SA on
+			 * the list is the one that will be used for sending
+			 * traffic, so this is the one we use for determining
+			 * when to send the keepalive.
+			 */
+			if (savkabuf && savkacount < savbufcount) {
+				sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_MATURE]);	//%%% should we check dying list if this is empty???
+				if (natt_keepalive_interval && sav && (sav->flags & SADB_X_EXT_NATT_KEEPALIVE) != 0 &&
+					(natt_now - sav->natt_last_activity) >= natt_keepalive_interval) {
+					//ipsec_send_natt_keepalive(sav);
+					sav->refcnt++;
+					*savkaptr++ = sav;
+					savkacount++;
+				}
+			}
+			
+			/*
+			 * check MATURE entry to start to send expire message
+			 * whether or not.
+			 */
+			for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_MATURE]);
+				 sav != NULL;
+				 sav = nextsav) {
+	
+				nextsav = LIST_NEXT(sav, chain);
+	
+				/* we don't need to check. */
+				if (sav->lft_s == NULL)
+					continue;
+	
+				/* sanity check */
+				if (sav->lft_c == NULL) {
+					ipseclog((LOG_DEBUG,"key_timehandler: "
+						"There is no CURRENT time, why?\n"));
+					continue;
+				}
+	
+				/* check SOFT lifetime */
+				if (sav->lft_s->sadb_lifetime_addtime != 0
+				 && tv.tv_sec - sav->created > sav->lft_s->sadb_lifetime_addtime) {
+					/*
+					 * check the SA if it has been used.
+					 * when it hasn't been used, delete it.
+					 * i don't think such SA will be used.
+					 */
+					if (sav->lft_c->sadb_lifetime_usetime == 0) {
+						key_sa_chgstate(sav, SADB_SASTATE_DEAD);
+						key_freesav(sav, KEY_SADB_LOCKED);
+						sav = NULL;
+					} else if (savexbuf && savexcount < savbufcount) {
+						key_sa_chgstate(sav, SADB_SASTATE_DYING);	
+						sav->refcnt++;
+						*savexptr++ = sav;
+						savexcount++;
+					}
+				}
+	
+				/* check SOFT lifetime by bytes */
 				/*
-				 * check the SA if it has been used.
-				 * when it hasn't been used, delete it.
-				 * i don't think such SA will be used.
+				 * XXX I don't know the way to delete this SA
+				 * when new SA is installed.  Caution when it's
+				 * installed too big lifetime by time.
 				 */
-				if (sav->lft_c->sadb_lifetime_usetime == 0) {
-					key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-					key_freesav(sav);
-					sav = NULL;
-				} else {
-					key_sa_chgstate(sav, SADB_SASTATE_DYING);
+				else if (savexbuf && savexcount < savbufcount
+					  && sav->lft_s->sadb_lifetime_bytes != 0
+					  && sav->lft_s->sadb_lifetime_bytes < sav->lft_c->sadb_lifetime_bytes) {
+	
 					/*
 					 * XXX If we keep to send expire
 					 * message in the status of
 					 * DYING. Do remove below code.
 					 */
-					key_expire(sav);
+					//key_expire(sav);
+					key_sa_chgstate(sav, SADB_SASTATE_DYING);
+					sav->refcnt++;
+					*savexptr++ = sav;
+					savexcount++;
 				}
 			}
-
-			/* check SOFT lifetime by bytes */
-			/*
-			 * XXX I don't know the way to delete this SA
-			 * when new SA is installed.  Caution when it's
-			 * installed too big lifetime by time.
-			 */
-			else if (sav->lft_s->sadb_lifetime_bytes != 0
-			      && sav->lft_s->sadb_lifetime_bytes < sav->lft_c->sadb_lifetime_bytes) {
-
-				key_sa_chgstate(sav, SADB_SASTATE_DYING);
-				/*
-				 * XXX If we keep to send expire
-				 * message in the status of
-				 * DYING. Do remove below code.
-				 */
-				key_expire(sav);
-			}
-		}
-
-		/* check DYING entry to change status to DEAD. */
-		for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_DYING]);
-		     sav != NULL;
-		     sav = nextsav) {
-
-			nextsav = LIST_NEXT(sav, chain);
-
-			/* we don't need to check. */
-			if (sav->lft_h == NULL)
-				continue;
-
-			/* sanity check */
-			if (sav->lft_c == NULL) {
-				ipseclog((LOG_DEBUG, "key_timehandler: "
-					"There is no CURRENT time, why?\n"));
-				continue;
-			}
-
-			if (sav->lft_h->sadb_lifetime_addtime != 0
-			 && tv.tv_sec - sav->created > sav->lft_h->sadb_lifetime_addtime) {
-				key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-				key_freesav(sav);
-				sav = NULL;
-			}
+	
+			/* check DYING entry to change status to DEAD. */
+			for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_DYING]);
+				 sav != NULL;
+				 sav = nextsav) {
+	
+				nextsav = LIST_NEXT(sav, chain);
+	
+				/* we don't need to check. */
+				if (sav->lft_h == NULL)
+					continue;
+	
+				/* sanity check */
+				if (sav->lft_c == NULL) {
+					ipseclog((LOG_DEBUG, "key_timehandler: "
+						"There is no CURRENT time, why?\n"));
+					continue;
+				}
+	
+				if (sav->lft_h->sadb_lifetime_addtime != 0
+				 && tv.tv_sec - sav->created > sav->lft_h->sadb_lifetime_addtime) {
+					key_sa_chgstate(sav, SADB_SASTATE_DEAD);
+					key_freesav(sav, KEY_SADB_LOCKED);
+					sav = NULL;
+				}
 #if 0	/* XXX Should we keep to send expire message until HARD lifetime ? */
-			else if (sav->lft_s != NULL
-			      && sav->lft_s->sadb_lifetime_addtime != 0
-			      && tv.tv_sec - sav->created > sav->lft_s->sadb_lifetime_addtime) {
-				/*
-				 * XXX: should be checked to be
-				 * installed the valid SA.
-				 */
-
-				/*
-				 * If there is no SA then sending
-				 * expire message.
-				 */
-				key_expire(sav);
-			}
+				else if (savbuf && savexcount < savbufcount
+					  && sav->lft_s != NULL
+					  && sav->lft_s->sadb_lifetime_addtime != 0
+					  && tv.tv_sec - sav->created > sav->lft_s->sadb_lifetime_addtime) {
+					/*
+					 * XXX: should be checked to be
+					 * installed the valid SA.
+					 */
+	
+					/*
+					 * If there is no SA then sending
+					 * expire message.
+					 */
+					//key_expire(sav);
+					sav->refcnt++;
+					*savexptr++ = sav;
+					savexcount++;
+				}
 #endif
-			/* check HARD lifetime by bytes */
-			else if (sav->lft_h->sadb_lifetime_bytes != 0
-			      && sav->lft_h->sadb_lifetime_bytes < sav->lft_c->sadb_lifetime_bytes) {
-				key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-				key_freesav(sav);
-				sav = NULL;
+				/* check HARD lifetime by bytes */
+				else if (sav->lft_h->sadb_lifetime_bytes != 0
+					  && sav->lft_h->sadb_lifetime_bytes < sav->lft_c->sadb_lifetime_bytes) {
+					key_sa_chgstate(sav, SADB_SASTATE_DEAD);
+					key_freesav(sav, KEY_SADB_LOCKED);
+					sav = NULL;
+				}
 			}
-		}
-
-		/* delete entry in DEAD */
-		for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_DEAD]);
-		     sav != NULL;
-		     sav = nextsav) {
-
-			nextsav = LIST_NEXT(sav, chain);
-
-			/* sanity check */
-			if (sav->state != SADB_SASTATE_DEAD) {
-				ipseclog((LOG_DEBUG, "key_timehandler: "
-					"invalid sav->state "
-					"(queue: %d SA: %d): "
-					"kill it anyway\n",
-					SADB_SASTATE_DEAD, sav->state));
+	
+			/* delete entry in DEAD */
+			for (sav = LIST_FIRST(&sah->savtree[SADB_SASTATE_DEAD]);
+				 sav != NULL;
+				 sav = nextsav) {
+	
+				nextsav = LIST_NEXT(sav, chain);
+	
+				/* sanity check */
+				if (sav->state != SADB_SASTATE_DEAD) {
+					ipseclog((LOG_DEBUG, "key_timehandler: "
+						"invalid sav->state "
+						"(queue: %d SA: %d): "
+						"kill it anyway\n",
+						SADB_SASTATE_DEAD, sav->state));
+				}
+	
+				/*
+				 * do not call key_freesav() here.
+				 * sav should already be freed, and sav->refcnt
+				 * shows other references to sav
+				 * (such as from SPD).
+				 */
 			}
-
-			/*
-			 * do not call key_freesav() here.
-			 * sav should already be freed, and sav->refcnt
-			 * shows other references to sav
-			 * (such as from SPD).
-			 */
 		}
-	}
-    }
+   }
 
 #ifndef IPSEC_NONBLOCK_ACQUIRE
 	/* ACQ tree */
@@ -4602,6 +4930,44 @@ key_timehandler(void)
 	natt_now++;
 
 	lck_mtx_unlock(sadb_mutex);
+
+	/* send messages outside of sadb_mutex */
+	if (spbuf && spcount > 0) {
+		cnt = spcount;
+		while (cnt--)
+			key_spdexpire(*(--spptr));
+	}
+	if (savkabuf && savkacount > 0) {
+		cnt = savkacount;
+		while (cnt--)
+			ipsec_send_natt_keepalive(*(--savkaptr));
+	}
+	if (savexbuf && savexcount > 0) {
+		cnt = savexcount;
+		while (cnt--)
+			key_expire(*(--savexptr));
+	}
+	
+	/* decrement ref counts and free buffers */
+	lck_mtx_lock(sadb_mutex);
+	if (spbuf) {
+		while (spcount--)
+			key_freesp(*spptr++, KEY_SADB_LOCKED);
+		KFREE(spbuf);
+	}
+	if (savkabuf) {
+		while (savkacount--)
+			key_freesav(*savkaptr++, KEY_SADB_LOCKED);
+		KFREE(savkabuf);
+	}
+	if (savexbuf) {
+		while (savexcount--)
+			key_freesav(*savexptr++, KEY_SADB_LOCKED);
+		KFREE(savexbuf);
+	}
+	lck_mtx_unlock(sadb_mutex);
+
+	
 #ifndef IPSEC_DEBUG2
 	/* do exchange to tick time !! */
 	(void)timeout((void *)key_timehandler, (void *)0, hz);
@@ -4644,13 +5010,14 @@ key_randomfill(p, l)
 	void *p;
 	size_t l;
 {
-	size_t n;
-	u_long v;
-	static int warn = 1;
 #ifdef __APPLE__
 
 	read_random(p, (u_int)l);
 #else
+	size_t n;
+	u_long v;
+	static int warn = 1;
+
 	n = 0;
 	n = (size_t)read_random(p, (u_int)l);
 	/* last resort */
@@ -4747,8 +5114,8 @@ key_getspi(so, m, mhp)
 	u_int32_t reqid;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_getspi: NULL pointer is passed.\n");
@@ -4817,16 +5184,21 @@ key_getspi(so, m, mhp)
 	/* XXX boundary check against sa_len */
 	KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
 
+	lck_mtx_lock(sadb_mutex);
+	
 	/* SPI allocation */
 	spi = key_do_getnewspi((struct sadb_spirange *)mhp->ext[SADB_EXT_SPIRANGE],
 	                       &saidx);
-	if (spi == 0)
+	if (spi == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, EINVAL);
+	}
 
 	/* get a SA index */
 	if ((newsah = key_getsah(&saidx)) == NULL) {
 		/* create a new SA index */
 		if ((newsah = key_newsah(&saidx)) == NULL) {
+			lck_mtx_unlock(sadb_mutex);
 			ipseclog((LOG_DEBUG, "key_getspi: No more memory.\n"));
 			return key_senderror(so, m, ENOBUFS);
 		}
@@ -4837,6 +5209,7 @@ key_getspi(so, m, mhp)
 	newsav = key_newsav(m, mhp, newsah, &error);
 	if (newsav == NULL) {
 		/* XXX don't free new SA index allocated in above. */
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
 
@@ -4857,6 +5230,8 @@ key_getspi(so, m, mhp)
     	}
 #endif
 
+	lck_mtx_unlock(sadb_mutex);
+	
     {
 	struct mbuf *n, *nn;
 	struct sadb_sa *m_sa;
@@ -4939,49 +5314,53 @@ key_do_getnewspi(spirange, saidx)
 	struct secasindex *saidx;
 {
 	u_int32_t newspi;
-	u_int32_t min, max;
+	u_int32_t keymin, keymax;
 	int count = key_spi_trycnt;
 
 	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* set spi range to allocate */
 	if (spirange != NULL) {
-		min = spirange->sadb_spirange_min;
-		max = spirange->sadb_spirange_max;
+		keymin = spirange->sadb_spirange_min;
+		keymax = spirange->sadb_spirange_max;
 	} else {
-		min = key_spi_minval;
-		max = key_spi_maxval;
+		keymin = key_spi_minval;
+		keymax = key_spi_maxval;
 	}
 	/* IPCOMP needs 2-byte SPI */
 	if (saidx->proto == IPPROTO_IPCOMP) {
 		u_int32_t t;
-		if (min >= 0x10000)
-			min = 0xffff;
-		if (max >= 0x10000)
-			max = 0xffff;
-		if (min > max) {
-			t = min; min = max; max = t;
+		if (keymin >= 0x10000)
+			keymin = 0xffff;
+		if (keymax >= 0x10000)
+			keymax = 0xffff;
+		if (keymin > keymax) {
+			t = keymin; keymin = keymax; keymax = t;
 		}
 	}
 
-	if (min == max) {
-		if (key_checkspidup(saidx, min) != NULL) {
-			ipseclog((LOG_DEBUG, "key_do_getnewspi: SPI %u exists already.\n", min));
+	if (keymin == keymax) {
+		if (key_checkspidup(saidx, keymin) != NULL) {
+			ipseclog((LOG_DEBUG, "key_do_getnewspi: SPI %u exists already.\n", keymin));
 			return 0;
 		}
 
 		count--; /* taking one cost. */
-		newspi = min;
+		newspi = keymin;
 
 	} else {
+	
+		u_long range = keymax - keymin + 1;  /* overflow value of zero means full range */
 
 		/* init SPI */
 		newspi = 0;
 
 		/* when requesting to allocate spi ranged */
 		while (count--) {
+			u_long rand_val = key_random();
+			
 			/* generate pseudo-random SPI value ranged. */
-			newspi = min + (key_random() % (max - min + 1));
+			newspi = (range == 0 ? rand_val : keymin + (rand_val % range));
 
 			if (key_checkspidup(saidx, newspi) == NULL)
 				break;
@@ -5029,8 +5408,8 @@ key_update(so, m, mhp)
 	u_int32_t reqid;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_update: NULL pointer is passed.\n");
@@ -5077,8 +5456,11 @@ key_update(so, m, mhp)
 	/* XXX boundary check against sa_len */
 	KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
 
+	lck_mtx_lock(sadb_mutex);
+	
 	/* get a SA header */
 	if ((sah = key_getsah(&saidx)) == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_update: no SA index found.\n"));
 		return key_senderror(so, m, ENOENT);
 	}
@@ -5086,13 +5468,16 @@ key_update(so, m, mhp)
 	/* set spidx if there */
 	/* XXX rewrite */
 	error = key_setident(sah, m, mhp);
-	if (error)
+	if (error) {
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
+	}
 
 	/* find a SA with sequence number. */
 #if IPSEC_DOSEQCHECK
 	if (mhp->msg->sadb_msg_seq != 0
 	 && (sav = key_getsavbyseq(sah, mhp->msg->sadb_msg_seq)) == NULL) {
+	 	lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG,
 		    "key_update: no larval SA with sequence %u exists.\n",
 		    mhp->msg->sadb_msg_seq));
@@ -5100,6 +5485,7 @@ key_update(so, m, mhp)
 	}
 #else
 	if ((sav = key_getsavbyspi(sah, sa0->sadb_sa_spi)) == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG,
 		    "key_update: no such a SA found (spi:%u)\n",
 		    (u_int32_t)ntohl(sa0->sadb_sa_spi)));
@@ -5109,6 +5495,7 @@ key_update(so, m, mhp)
 
 	/* validity check */
 	if (sav->sah->saidx.proto != proto) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG,
 		    "key_update: protocol mismatched (DB=%u param=%u)\n",
 		    sav->sah->saidx.proto, proto));
@@ -5116,6 +5503,7 @@ key_update(so, m, mhp)
 	}
 #if IPSEC_DOSEQCHECK
 	if (sav->spi != sa0->sadb_sa_spi) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG,
 		    "key_update: SPI mismatched (DB:%u param:%u)\n",
 		    (u_int32_t)ntohl(sav->spi),
@@ -5124,6 +5512,7 @@ key_update(so, m, mhp)
 	}
 #endif
 	if (sav->pid != mhp->msg->sadb_msg_pid) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG,
 		    "key_update: pid mismatched (DB:%u param:%u)\n",
 		    sav->pid, mhp->msg->sadb_msg_pid));
@@ -5133,16 +5522,29 @@ key_update(so, m, mhp)
 	/* copy sav values */
 	error = key_setsaval(sav, m, mhp);
 	if (error) {
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_LOCKED);
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
+	
+	/*
+	 * Verify if SADB_X_EXT_NATT_MULTIPLEUSERS flag is set that
+	 * this SA is for transport mode - otherwise clear it.
+	 */
+	if ((sav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0 &&
+		(sav->sah->saidx.mode != IPSEC_MODE_TRANSPORT ||
+		sav->sah->saidx.src.ss_family != AF_INET))
+		sav->flags &= ~SADB_X_EXT_NATT_MULTIPLEUSERS;
 
 	/* check SA values to be mature. */
 	if ((error = key_mature(sav)) != 0) {
-		key_freesav(sav);
+		key_freesav(sav, KEY_SADB_LOCKED);
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
-
+	
+	lck_mtx_unlock(sadb_mutex);
+	
     {
 	struct mbuf *n;
 
@@ -5228,8 +5630,8 @@ key_add(so, m, mhp)
 	u_int32_t reqid;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_add: NULL pointer is passed.\n");
@@ -5276,10 +5678,13 @@ key_add(so, m, mhp)
 	/* XXX boundary check against sa_len */
 	KEY_SETSECASIDX(proto, mode, reqid, src0 + 1, dst0 + 1, &saidx);
 
+	lck_mtx_lock(sadb_mutex);
+	
 	/* get a SA header */
 	if ((newsah = key_getsah(&saidx)) == NULL) {
 		/* create a new SA header */
 		if ((newsah = key_newsah(&saidx)) == NULL) {
+			lck_mtx_unlock(sadb_mutex);
 			ipseclog((LOG_DEBUG, "key_add: No more memory.\n"));
 			return key_senderror(so, m, ENOBUFS);
 		}
@@ -5289,26 +5694,41 @@ key_add(so, m, mhp)
 	/* XXX rewrite */
 	error = key_setident(newsah, m, mhp);
 	if (error) {
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
 
 	/* create new SA entry. */
-	/* We can create new SA only if SPI is differenct. */
+	/* We can create new SA only if SPI is different. */
 	if (key_getsavbyspi(newsah, sa0->sadb_sa_spi)) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_add: SA already exists.\n"));
 		return key_senderror(so, m, EEXIST);
 	}
 	newsav = key_newsav(m, mhp, newsah, &error);
 	if (newsav == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
 
+	/*
+	 * Verify if SADB_X_EXT_NATT_MULTIPLEUSERS flag is set that
+	 * this SA is for transport mode - otherwise clear it.
+	 */
+	if ((newsav->flags & SADB_X_EXT_NATT_MULTIPLEUSERS) != 0 &&
+		(newsah->saidx.mode != IPSEC_MODE_TRANSPORT ||
+		newsah->saidx.dst.ss_family != AF_INET))
+		newsav->flags &= ~SADB_X_EXT_NATT_MULTIPLEUSERS;
+
 	/* check SA values to be mature. */
 	if ((error = key_mature(newsav)) != 0) {
-		key_freesav(newsav);
+		key_freesav(newsav, KEY_SADB_LOCKED);
+		lck_mtx_unlock(sadb_mutex);
 		return key_senderror(so, m, error);
 	}
 
+	lck_mtx_unlock(sadb_mutex);
+	
 	/*
 	 * don't call key_freesav() here, as we would like to keep the SA
 	 * in the database on success.
@@ -5382,17 +5802,27 @@ key_setident(sah, m, mhp)
 	}
 
 	/* make structure */
-	KMALLOC(sah->idents, struct sadb_ident *, idsrclen);
+	KMALLOC_NOWAIT(sah->idents, struct sadb_ident *, idsrclen);
 	if (sah->idents == NULL) {
-		ipseclog((LOG_DEBUG, "key_setident: No more memory.\n"));
-		return ENOBUFS;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(sah->idents, struct sadb_ident *, idsrclen);
+		lck_mtx_lock(sadb_mutex);
+		if (sah->idents == NULL) {
+			ipseclog((LOG_DEBUG, "key_setident: No more memory.\n"));
+			return ENOBUFS;
+		}
 	}
-	KMALLOC(sah->identd, struct sadb_ident *, iddstlen);
+	KMALLOC_NOWAIT(sah->identd, struct sadb_ident *, iddstlen);
 	if (sah->identd == NULL) {
-		KFREE(sah->idents);
-		sah->idents = NULL;
-		ipseclog((LOG_DEBUG, "key_setident: No more memory.\n"));
-		return ENOBUFS;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(sah->identd, struct sadb_ident *, iddstlen);
+		lck_mtx_lock(sadb_mutex);
+		if (sah->identd == NULL) {
+			KFREE(sah->idents);
+			sah->idents = NULL;
+			ipseclog((LOG_DEBUG, "key_setident: No more memory.\n"));
+			return ENOBUFS;
+		}
 	}
 	bcopy(idsrc, sah->idents, idsrclen);
 	bcopy(iddst, sah->identd, iddstlen);
@@ -5464,8 +5894,8 @@ key_delete(so, m, mhp)
 	struct secasvar *sav = NULL;
 	u_int16_t proto;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_delete: NULL pointer is passed.\n");
@@ -5488,6 +5918,8 @@ key_delete(so, m, mhp)
 		return key_senderror(so, m, EINVAL);
 	}
 
+	lck_mtx_lock(sadb_mutex);
+	
 	if (mhp->ext[SADB_EXT_SA] == NULL) {
 		/*
 		 * Caller wants us to delete all non-LARVAL SAs
@@ -5495,8 +5927,10 @@ key_delete(so, m, mhp)
 		 * IKE INITIAL-CONTACT.
 		 */
 		ipseclog((LOG_DEBUG, "key_delete: doing delete all.\n"));
-		return key_delete_all(so, m, mhp, proto);
+		/* key_delete_all will unlock sadb_mutex  */
+		return key_delete_all(so, m, mhp, proto);	
 	} else if (mhp->extlen[SADB_EXT_SA] < sizeof(struct sadb_sa)) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_delete: invalid message is passed.\n"));
 		return key_senderror(so, m, EINVAL);
 	}
@@ -5521,12 +5955,15 @@ key_delete(so, m, mhp)
 			break;
 	}
 	if (sah == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_delete: no SA found.\n"));
 		return key_senderror(so, m, ENOENT);
 	}
 
 	key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-	key_freesav(sav);
+	key_freesav(sav, KEY_SADB_LOCKED);
+	
+	lck_mtx_unlock(sadb_mutex);
 	sav = NULL;
 
     {
@@ -5604,10 +6041,12 @@ key_delete_all(so, m, mhp, proto)
 				}
 				
 				key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-				key_freesav(sav);
+				key_freesav(sav, KEY_SADB_LOCKED);
 			}
 		}
 	}
+	lck_mtx_unlock(sadb_mutex);
+	
     {
 	struct mbuf *n;
 	struct sadb_msg *newmsg;
@@ -5658,8 +6097,8 @@ key_get(so, m, mhp)
 	struct secasvar *sav = NULL;
 	u_int16_t proto;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_get: NULL pointer is passed.\n");
@@ -5690,6 +6129,8 @@ key_get(so, m, mhp)
 	/* XXX boundary check against sa_len */
 	KEY_SETSECASIDX(proto, IPSEC_MODE_ANY, 0, src0 + 1, dst0 + 1, &saidx);
 
+	lck_mtx_lock(sadb_mutex);
+	
 	/* get a SA header */
 	LIST_FOREACH(sah, &sahtree, chain) {
 		if (sah->state == SADB_SASTATE_DEAD)
@@ -5703,6 +6144,7 @@ key_get(so, m, mhp)
 			break;
 	}
 	if (sah == NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_get: no SA found.\n"));
 		return key_senderror(so, m, ENOENT);
 	}
@@ -5713,13 +6155,18 @@ key_get(so, m, mhp)
 
 	/* map proto to satype */
 	if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_get: there was invalid proto in SAD.\n"));
 		return key_senderror(so, m, EINVAL);
 	}
+	lck_mtx_unlock(sadb_mutex);
 
 	/* create new sadb_msg to reply. */
 	n = key_setdumpsa(sav, SADB_GET, satype, mhp->msg->sadb_msg_seq,
 	    mhp->msg->sadb_msg_pid);
+	   
+	
+	
 	if (!n)
 		return key_senderror(so, m, ENOBUFS);
 
@@ -5837,7 +6284,7 @@ key_getcomb_ah()
 	struct sadb_comb *comb;
 	const struct ah_algorithm *algo;
 	struct mbuf *m;
-	int min;
+	int keymin;
 	int i;
 	const int l = PFKEY_ALIGN8(sizeof(struct sadb_comb));
 
@@ -5855,9 +6302,9 @@ key_getcomb_ah()
 		if (algo->keymax < ipsec_ah_keymin)
 			continue;
 		if (algo->keymin < ipsec_ah_keymin)
-			min = ipsec_ah_keymin;
+			keymin = ipsec_ah_keymin;
 		else
-			min = algo->keymin;
+			keymin = algo->keymin;
 
 		if (!m) {
 #if DIAGNOSTIC
@@ -5879,7 +6326,7 @@ key_getcomb_ah()
 		bzero(comb, sizeof(*comb));
 		key_getcomb_setlifetime(comb);
 		comb->sadb_comb_auth = i;
-		comb->sadb_comb_auth_minbits = min;
+		comb->sadb_comb_auth_minbits = keymin;
 		comb->sadb_comb_auth_maxbits = algo->keymax;
 	}
 
@@ -6013,8 +6460,8 @@ key_acquire(saidx, sp)
 	int error = -1;
 	u_int32_t seq;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (saidx == NULL)
 		panic("key_acquire: NULL pointer is passed.\n");
@@ -6029,6 +6476,7 @@ key_acquire(saidx, sp)
 	 * managed with ACQUIRING list.
 	 */
 	/* get a entry to check whether sending message or not. */
+	lck_mtx_lock(sadb_mutex);
 	if ((newacq = key_getacq(saidx)) != NULL) {
 		if (key_blockacq_count < newacq->count) {
 			/* reset counter and do send message. */
@@ -6036,21 +6484,22 @@ key_acquire(saidx, sp)
 		} else {
 			/* increment counter and do nothing. */
 			newacq->count++;
+			lck_mtx_unlock(sadb_mutex);
 			return 0;
 		}
 	} else {
 		/* make new entry for blocking to send SADB_ACQUIRE. */
-		if ((newacq = key_newacq(saidx)) == NULL)
+		if ((newacq = key_newacq(saidx)) == NULL) {
+			lck_mtx_unlock(sadb_mutex);
 			return ENOBUFS;
+		}
 
 		/* add to acqtree */
 		LIST_INSERT_HEAD(&acqtree, newacq, chain);
 	}
-#endif
-
-
-#ifndef IPSEC_NONBLOCK_ACQUIRE
 	seq = newacq->seq;
+	lck_mtx_unlock(sadb_mutex);
+
 #else
 	seq = (acq_seq = (acq_seq == ~0 ? 1 : ++acq_seq));
 #endif
@@ -6191,10 +6640,15 @@ key_newacq(saidx)
 	struct timeval tv;
 
 	/* get new entry */
-	KMALLOC(newacq, struct secacq *, sizeof(struct secacq));
+	KMALLOC_NOWAIT(newacq, struct secacq *, sizeof(struct secacq));
 	if (newacq == NULL) {
-		ipseclog((LOG_DEBUG, "key_newacq: No more memory.\n"));
-		return NULL;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(newacq, struct secacq *, sizeof(struct secacq));
+		lck_mtx_lock(sadb_mutex);
+			if (newacq == NULL) {
+			ipseclog((LOG_DEBUG, "key_newacq: No more memory.\n"));
+			return NULL;
+		}
 	}
 	bzero(newacq, sizeof(*newacq));
 
@@ -6249,10 +6703,15 @@ key_newspacq(spidx)
 	struct timeval tv;
 
 	/* get new entry */
-	KMALLOC(acq, struct secspacq *, sizeof(struct secspacq));
+	KMALLOC_NOWAIT(acq, struct secspacq *, sizeof(struct secspacq));
 	if (acq == NULL) {
-		ipseclog((LOG_DEBUG, "key_newspacq: No more memory.\n"));
-		return NULL;
+		lck_mtx_unlock(sadb_mutex);
+		KMALLOC_WAIT(acq, struct secspacq *, sizeof(struct secspacq));
+		lck_mtx_lock(sadb_mutex);
+		if (acq == NULL) {
+			ipseclog((LOG_DEBUG, "key_newspacq: No more memory.\n"));
+			return NULL;
+		}
 	}
 	bzero(acq, sizeof(*acq));
 
@@ -6307,7 +6766,6 @@ key_acquire2(so, m, mhp)
 	u_int16_t proto;
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
@@ -6319,6 +6777,8 @@ key_acquire2(so, m, mhp)
 	 * message is equal to the size of sadb_msg structure.
 	 * We do not raise error even if error occurred in this function.
 	 */
+	 lck_mtx_lock(sadb_mutex);
+
 	if (mhp->msg->sadb_msg_len == PFKEY_UNIT64(sizeof(struct sadb_msg))) {
 #ifndef IPSEC_NONBLOCK_ACQUIRE
 		struct secacq *acq;
@@ -6326,6 +6786,7 @@ key_acquire2(so, m, mhp)
 
 		/* check sequence number */
 		if (mhp->msg->sadb_msg_seq == 0) {
+			lck_mtx_unlock(sadb_mutex);
 			ipseclog((LOG_DEBUG, "key_acquire2: must specify sequence number.\n"));
 			m_freem(m);
 			return 0;
@@ -6336,6 +6797,7 @@ key_acquire2(so, m, mhp)
 			 * the specified larval SA is already gone, or we got
 			 * a bogus sequence number.  we can silently ignore it.
 			 */
+			lck_mtx_unlock(sadb_mutex);
 			m_freem(m);
 			return 0;
 		}
@@ -6345,6 +6807,7 @@ key_acquire2(so, m, mhp)
 		acq->created = tv.tv_sec;
 		acq->count = 0;
 #endif
+		lck_mtx_unlock(sadb_mutex);
 		m_freem(m);
 		return 0;
 	}
@@ -6355,6 +6818,7 @@ key_acquire2(so, m, mhp)
 
 	/* map satype to proto */
 	if ((proto = key_satype2proto(mhp->msg->sadb_msg_satype)) == 0) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_acquire2: invalid satype is passed.\n"));
 		return key_senderror(so, m, EINVAL);
 	}
@@ -6363,6 +6827,7 @@ key_acquire2(so, m, mhp)
 	    mhp->ext[SADB_EXT_ADDRESS_DST] == NULL ||
 	    mhp->ext[SADB_EXT_PROPOSAL] == NULL) {
 		/* error */
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_acquire2: invalid message is passed.\n"));
 		return key_senderror(so, m, EINVAL);
 	}
@@ -6370,6 +6835,7 @@ key_acquire2(so, m, mhp)
 	    mhp->extlen[SADB_EXT_ADDRESS_DST] < sizeof(struct sadb_address) ||
 	    mhp->extlen[SADB_EXT_PROPOSAL] < sizeof(struct sadb_prop)) {
 		/* error */
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_acquire2: invalid message is passed.\n"));
 		return key_senderror(so, m, EINVAL);
 	}
@@ -6384,14 +6850,15 @@ key_acquire2(so, m, mhp)
 	LIST_FOREACH(sah, &sahtree, chain) {
 		if (sah->state == SADB_SASTATE_DEAD)
 			continue;
-		if (key_cmpsaidx(&sah->saidx, &saidx, CMP_MODE_REQID))
+		if (key_cmpsaidx(&sah->saidx, &saidx, CMP_MODE | CMP_REQID))
 			break;
 	}
 	if (sah != NULL) {
+		lck_mtx_unlock(sadb_mutex);
 		ipseclog((LOG_DEBUG, "key_acquire2: a SA exists already.\n"));
 		return key_senderror(so, m, EEXIST);
 	}
-
+	lck_mtx_unlock(sadb_mutex);
 	error = key_acquire(&saidx, NULL);
 	if (error != 0) {
 		ipseclog((LOG_DEBUG, "key_acquire2: error %d returned "
@@ -6404,7 +6871,7 @@ key_acquire2(so, m, mhp)
 
 /*
  * SADB_REGISTER processing.
- * If SATYPE_UNSPEC has been passed as satype, only return sabd_supported.
+ * If SATYPE_UNSPEC has been passed as satype, only return sadb_supported.
  * receive
  *   <base>
  * from the ikmpd, and register a socket to send PF_KEY messages,
@@ -6422,9 +6889,7 @@ key_register(so, m, mhp)
 	const struct sadb_msghdr *mhp;
 {
 	struct secreg *reg, *newreg = 0;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_register: NULL pointer is passed.\n");
@@ -6433,26 +6898,29 @@ key_register(so, m, mhp)
 	if (mhp->msg->sadb_msg_satype >= sizeof(regtree)/sizeof(regtree[0]))
 		return key_senderror(so, m, EINVAL);
 
-	/* When SATYPE_UNSPEC is specified, only return sabd_supported. */
+	/* When SATYPE_UNSPEC is specified, only return sadb_supported. */
 	if (mhp->msg->sadb_msg_satype == SADB_SATYPE_UNSPEC)
 		goto setmsg;
 
+	/* create regnode */
+	KMALLOC_WAIT(newreg, struct secreg *, sizeof(*newreg));
+        if (newreg == NULL) {
+	  ipseclog((LOG_DEBUG, "key_register: No more memory.\n"));
+	  return key_senderror(so, m, ENOBUFS);
+        }
+        bzero((caddr_t)newreg, sizeof(*newreg));
+
+	lck_mtx_lock(sadb_mutex);
 	/* check whether existing or not */
 	LIST_FOREACH(reg, &regtree[mhp->msg->sadb_msg_satype], chain) {
 		if (reg->so == so) {
+			lck_mtx_unlock(sadb_mutex);
 			ipseclog((LOG_DEBUG, "key_register: socket exists already.\n"));
+			KFREE(newreg);
 			return key_senderror(so, m, EEXIST);
 		}
 	}
 
-	/* create regnode */
-	KMALLOC(newreg, struct secreg *, sizeof(*newreg));
-	if (newreg == NULL) {
-		ipseclog((LOG_DEBUG, "key_register: No more memory.\n"));
-		return key_senderror(so, m, ENOBUFS);
-	}
-	bzero((caddr_t)newreg, sizeof(*newreg));
-
 	socket_lock(so, 1);
 	newreg->so = so;
 	((struct keycb *)sotorawcb(so))->kp_registered++;
@@ -6460,7 +6928,7 @@ key_register(so, m, mhp)
 
 	/* add regnode to regtree. */
 	LIST_INSERT_HEAD(&regtree[mhp->msg->sadb_msg_satype], newreg, chain);
-
+	lck_mtx_unlock(sadb_mutex);
   setmsg:
     {
 	struct mbuf *n;
@@ -6589,9 +7057,7 @@ key_freereg(so)
 {
 	struct secreg *reg;
 	int i;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL)
 		panic("key_freereg: NULL pointer is passed.\n");
@@ -6601,6 +7067,7 @@ key_freereg(so)
 	 * check all type of SA, because there is a potential that
 	 * one socket is registered to multiple type of SA.
 	 */
+	lck_mtx_lock(sadb_mutex);
 	for (i = 0; i <= SADB_SATYPE_MAX; i++) {
 		LIST_FOREACH(reg, &regtree[i], chain) {
 			if (reg->so == so
@@ -6611,7 +7078,7 @@ key_freereg(so)
 			}
 		}
 	}
-	
+	lck_mtx_unlock(sadb_mutex);
 	return;
 }
 
@@ -6629,15 +7096,14 @@ static int
 key_expire(sav)
 	struct secasvar *sav;
 {
-	int s;
 	int satype;
 	struct mbuf *result = NULL, *m;
 	int len;
 	int error = -1;
 	struct sadb_lifetime *lt;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (sav == NULL)
 		panic("key_expire: NULL pointer is passed.\n");
@@ -6733,13 +7199,11 @@ key_expire(sav)
 	mtod(result, struct sadb_msg *)->sadb_msg_len =
 	    PFKEY_UNIT64(result->m_pkthdr.len);
 
-	splx(s);
 	return key_sendup_mbuf(NULL, result, KEY_SENDUP_REGISTERED);
 
  fail:
 	if (result)
 		m_freem(result);
-	splx(s);
 	return error;
 }
 
@@ -6767,9 +7231,7 @@ key_flush(so, m, mhp)
 	u_int16_t proto;
 	u_int8_t state;
 	u_int stateidx;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_flush: NULL pointer is passed.\n");
@@ -6780,6 +7242,8 @@ key_flush(so, m, mhp)
 		return key_senderror(so, m, EINVAL);
 	}
 
+	lck_mtx_lock(sadb_mutex);
+	
 	/* no SATYPE specified, i.e. flushing all SA. */
 	for (sah = LIST_FIRST(&sahtree);
 	     sah != NULL;
@@ -6801,13 +7265,14 @@ key_flush(so, m, mhp)
 				nextsav = LIST_NEXT(sav, chain);
 
 				key_sa_chgstate(sav, SADB_SASTATE_DEAD);
-				key_freesav(sav);
+				key_freesav(sav, KEY_SADB_LOCKED);
 			}
 		}
 
 		sah->state = SADB_SASTATE_DEAD;
 	}
-
+	lck_mtx_unlock(sadb_mutex);
+	
 	if (m->m_len < sizeof(struct sadb_msg) ||
 	    sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
 		ipseclog((LOG_DEBUG, "key_flush: No more memory.\n"));
@@ -6837,6 +7302,12 @@ key_flush(so, m, mhp)
  *
  * m will always be freed.
  */
+ 
+struct sav_dump_elem {
+	struct secasvar *sav;
+	u_int8_t satype;
+};
+
 static int
 key_dump(so, m, mhp)
 	struct socket *so;
@@ -6845,16 +7316,17 @@ key_dump(so, m, mhp)
 {
 	struct secashead *sah;
 	struct secasvar *sav;
+	struct sav_dump_elem *savbuf = NULL, *elem_ptr;
 	u_int16_t proto;
 	u_int stateidx;
 	u_int8_t satype;
 	u_int8_t state;
-	int cnt;
-	struct sadb_msg *newmsg;
+	int cnt = 0, cnt2, bufcount;
 	struct mbuf *n;
+	int error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_dump: NULL pointer is passed.\n");
@@ -6865,54 +7337,87 @@ key_dump(so, m, mhp)
 		return key_senderror(so, m, EINVAL);
 	}
 
+	if ((bufcount = ipsec_sav_count) <= 0) {
+		error = ENOENT;
+		goto end;
+	}
+	bufcount += 512;	/* extra */
+	KMALLOC_WAIT(savbuf, struct sav_dump_elem*, bufcount * sizeof(struct sav_dump_elem));
+	if (savbuf == NULL) {
+		ipseclog((LOG_DEBUG, "key_dump: No more memory.\n"));
+		error = ENOMEM;
+		goto end;
+	}
+
 	/* count sav entries to be sent to the userland. */
-	cnt = 0;
+	lck_mtx_lock(sadb_mutex);
+	elem_ptr = savbuf;
 	LIST_FOREACH(sah, &sahtree, chain) {
 		if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
 		 && proto != sah->saidx.proto)
 			continue;
+		
+		/* map proto to satype */
+		if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
+			lck_mtx_unlock(sadb_mutex);
+			ipseclog((LOG_DEBUG, "key_dump: there was invalid proto in SAD.\n"));
+			error = EINVAL;
+			goto end;
+		}
 
 		for (stateidx = 0;
 		     stateidx < _ARRAYLEN(saorder_state_any);
 		     stateidx++) {
 			state = saorder_state_any[stateidx];
 			LIST_FOREACH(sav, &sah->savtree[state], chain) {
-				cnt++;
+				if (cnt == bufcount)
+					break;		/* out of buffer space */
+				elem_ptr->sav = sav;
+				elem_ptr->satype = satype;
+				sav->refcnt++;
+				elem_ptr++;
+				cnt++;				
 			}
 		}
 	}
+	lck_mtx_unlock(sadb_mutex);
 
-	if (cnt == 0)
-		return key_senderror(so, m, ENOENT);
+	if (cnt == 0) {
+		error = ENOENT;
+		goto end;
+	}
 
 	/* send this to the userland, one at a time. */
-	newmsg = NULL;
-	LIST_FOREACH(sah, &sahtree, chain) {
-		if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
-		 && proto != sah->saidx.proto)
-			continue;
-
-		/* map proto to satype */
-		if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
-			ipseclog((LOG_DEBUG, "key_dump: there was invalid proto in SAD.\n"));
-			return key_senderror(so, m, EINVAL);
+	elem_ptr = savbuf;
+	cnt2 = cnt;
+	while (cnt2) {
+		n = key_setdumpsa(elem_ptr->sav, SADB_DUMP, elem_ptr->satype,
+			--cnt2, mhp->msg->sadb_msg_pid);
+		
+		if (!n) {
+			error = ENOBUFS;
+			goto end;
 		}
 
-		for (stateidx = 0;
-		     stateidx < _ARRAYLEN(saorder_state_any);
-		     stateidx++) {
-			state = saorder_state_any[stateidx];
-			LIST_FOREACH(sav, &sah->savtree[state], chain) {
-				n = key_setdumpsa(sav, SADB_DUMP, satype,
-				    --cnt, mhp->msg->sadb_msg_pid);
-				if (!n)
-					return key_senderror(so, m, ENOBUFS);
+		key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
+		elem_ptr++;
+	}
 
-				key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
-			}
+end:
+	if (savbuf) {
+		if (cnt) {
+			elem_ptr = savbuf;
+			lck_mtx_lock(sadb_mutex);
+			while (cnt--)
+				key_freesav((elem_ptr++)->sav, KEY_SADB_LOCKED);
+			lck_mtx_unlock(sadb_mutex);
 		}
+		KFREE(savbuf);
 	}
 
+	if (error)
+		return key_senderror(so, m, error);
+
 	m_freem(m);
 	return 0;
 }
@@ -6929,9 +7434,7 @@ key_promisc(so, m, mhp)
 	const struct sadb_msghdr *mhp;
 {
 	int olen;
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	
 	/* sanity check */
 	if (so == NULL || m == NULL || mhp == NULL || mhp->msg == NULL)
 		panic("key_promisc: NULL pointer is passed.\n");
@@ -7026,8 +7529,8 @@ key_parse(m, so)
 	int error;
 	int target;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	/* sanity check */
 	if (m == NULL || so == NULL)
 		panic("key_parse: NULL pointer is passed.\n");
@@ -7050,7 +7553,7 @@ key_parse(m, so)
 	if ((m->m_flags & M_PKTHDR) == 0 ||
 	    m->m_pkthdr.len != m->m_pkthdr.len) {
 		ipseclog((LOG_DEBUG, "key_parse: invalid message length.\n"));
-		pfkeystat.out_invlen++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invlen);
 		error = EINVAL;
 		goto senderror;
 	}
@@ -7059,7 +7562,7 @@ key_parse(m, so)
 		ipseclog((LOG_DEBUG,
 		    "key_parse: PF_KEY version %u is mismatched.\n",
 		    msg->sadb_msg_version));
-		pfkeystat.out_invver++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invver);
 		error = EINVAL;
 		goto senderror;
 	}
@@ -7067,7 +7570,7 @@ key_parse(m, so)
 	if (msg->sadb_msg_type > SADB_MAX) {
 		ipseclog((LOG_DEBUG, "key_parse: invalid type %u is passed.\n",
 		    msg->sadb_msg_type));
-		pfkeystat.out_invmsgtype++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invmsgtype);
 		error = EINVAL;
 		goto senderror;
 	}
@@ -7124,7 +7627,7 @@ key_parse(m, so)
 		case SADB_EXPIRE:
 			ipseclog((LOG_DEBUG, "key_parse: must specify satype "
 			    "when msg type=%u.\n", msg->sadb_msg_type));
-			pfkeystat.out_invsatype++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invsatype);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7143,7 +7646,7 @@ key_parse(m, so)
 		case SADB_X_SPDDELETE2:
 			ipseclog((LOG_DEBUG, "key_parse: illegal satype=%u\n",
 			    msg->sadb_msg_type));
-			pfkeystat.out_invsatype++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invsatype);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7154,7 +7657,7 @@ key_parse(m, so)
 	case SADB_SATYPE_MIP:
 		ipseclog((LOG_DEBUG, "key_parse: type %u isn't supported.\n",
 		    msg->sadb_msg_satype));
-		pfkeystat.out_invsatype++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invsatype);
 		error = EOPNOTSUPP;
 		goto senderror;
 	case 1:	/* XXX: What does it do? */
@@ -7164,7 +7667,7 @@ key_parse(m, so)
 	default:
 		ipseclog((LOG_DEBUG, "key_parse: invalid type %u is passed.\n",
 		    msg->sadb_msg_satype));
-		pfkeystat.out_invsatype++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invsatype);
 		error = EINVAL;
 		goto senderror;
 	}
@@ -7181,7 +7684,7 @@ key_parse(m, so)
 		/* check upper layer protocol */
 		if (src0->sadb_address_proto != dst0->sadb_address_proto) {
 			ipseclog((LOG_DEBUG, "key_parse: upper layer protocol mismatched.\n"));
-			pfkeystat.out_invaddr++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7190,7 +7693,7 @@ key_parse(m, so)
 		if (PFKEY_ADDR_SADDR(src0)->sa_family !=
 		    PFKEY_ADDR_SADDR(dst0)->sa_family) {
 			ipseclog((LOG_DEBUG, "key_parse: address family mismatched.\n"));
-			pfkeystat.out_invaddr++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7198,7 +7701,7 @@ key_parse(m, so)
 		    PFKEY_ADDR_SADDR(dst0)->sa_len) {
 			ipseclog((LOG_DEBUG,
 			    "key_parse: address struct size mismatched.\n"));
-			pfkeystat.out_invaddr++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7207,7 +7710,7 @@ key_parse(m, so)
 		case AF_INET:
 			if (PFKEY_ADDR_SADDR(src0)->sa_len !=
 			    sizeof(struct sockaddr_in)) {
-				pfkeystat.out_invaddr++;
+				PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 				error = EINVAL;
 				goto senderror;
 			}
@@ -7215,7 +7718,7 @@ key_parse(m, so)
 		case AF_INET6:
 			if (PFKEY_ADDR_SADDR(src0)->sa_len !=
 			    sizeof(struct sockaddr_in6)) {
-				pfkeystat.out_invaddr++;
+				PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 				error = EINVAL;
 				goto senderror;
 			}
@@ -7223,7 +7726,7 @@ key_parse(m, so)
 		default:
 			ipseclog((LOG_DEBUG,
 			    "key_parse: unsupported address family.\n"));
-			pfkeystat.out_invaddr++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 			error = EAFNOSUPPORT;
 			goto senderror;
 		}
@@ -7245,7 +7748,7 @@ key_parse(m, so)
 		    dst0->sadb_address_prefixlen > plen) {
 			ipseclog((LOG_DEBUG,
 			    "key_parse: illegal prefixlen.\n"));
-			pfkeystat.out_invaddr++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invaddr);
 			error = EINVAL;
 			goto senderror;
 		}
@@ -7258,7 +7761,7 @@ key_parse(m, so)
 
 	if (msg->sadb_msg_type >= sizeof(key_typesw)/sizeof(key_typesw[0]) ||
 	    key_typesw[msg->sadb_msg_type] == NULL) {
-		pfkeystat.out_invmsgtype++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invmsgtype);
 		error = EINVAL;
 		goto senderror;
 	}
@@ -7278,8 +7781,8 @@ key_senderror(so, m, code)
 {
 	struct sadb_msg *msg;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+	
 	if (m->m_len < sizeof(struct sadb_msg))
 		panic("invalid mbuf passed to key_senderror");
 
@@ -7356,7 +7859,7 @@ key_align(m, mhp)
 				    "key_align: duplicate ext_type %u "
 				    "is passed.\n", ext->sadb_ext_type));
 				m_freem(m);
-				pfkeystat.out_dupext++;
+				PFKEY_STAT_INCREMENT(pfkeystat.out_dupext);
 				return EINVAL;
 			}
 			break;
@@ -7365,7 +7868,7 @@ key_align(m, mhp)
 			    "key_align: invalid ext_type %u is passed.\n",
 			    ext->sadb_ext_type));
 			m_freem(m);
-			pfkeystat.out_invexttype++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invexttype);
 			return EINVAL;
 		}
 
@@ -7373,7 +7876,7 @@ key_align(m, mhp)
 
 		if (key_validate_ext(ext, extlen)) {
 			m_freem(m);
-			pfkeystat.out_invlen++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_invlen);
 			return EINVAL;
 		}
 
@@ -7391,7 +7894,7 @@ key_align(m, mhp)
 
 	if (off != end) {
 		m_freem(m);
-		pfkeystat.out_invlen++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invlen);
 		return EINVAL;
 	}
 
@@ -7463,16 +7966,18 @@ key_domain_init()
 	int i;
 
 	bzero((caddr_t)&key_cb, sizeof(key_cb));
-
+	
 	for (i = 0; i < IPSEC_DIR_MAX; i++) {
 		LIST_INIT(&sptree[i]);
 	}
+	ipsec_policy_count = 0;
 
 	LIST_INIT(&sahtree);
 
 	for (i = 0; i <= SADB_SATYPE_MAX; i++) {
 		LIST_INIT(&regtree[i]);
 	}
+	ipsec_sav_count = 0;
 
 #ifndef IPSEC_NONBLOCK_ACQUIRE
 	LIST_INIT(&acqtree);
@@ -7512,13 +8017,12 @@ key_domain_init()
  * xxx more checks to be provided
  */
 int
-key_checktunnelsanity(sav, family, src, dst)
-	struct secasvar *sav;
-	u_int family;
-	caddr_t src;
-	caddr_t dst;
+key_checktunnelsanity(
+	struct secasvar *sav,
+	__unused u_int family,
+	__unused caddr_t src,
+	__unused caddr_t dst)
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	/* sanity check */
 	if (sav->sah == NULL)
@@ -7529,78 +8033,14 @@ key_checktunnelsanity(sav, family, src, dst)
 	return 1;
 }
 
-#if 0
-#define hostnamelen	strlen(hostname)
-
-/*
- * Get FQDN for the host.
- * If the administrator configured hostname (by hostname(1)) without
- * domain name, returns nothing.
- */
-static const char *
-key_getfqdn()
-{
-	int i;
-	int hasdot;
-	static char fqdn[MAXHOSTNAMELEN + 1];
-
-	if (!hostnamelen)
-		return NULL;
-
-	/* check if it comes with domain name. */
-	hasdot = 0;
-	for (i = 0; i < hostnamelen; i++) {
-		if (hostname[i] == '.')
-			hasdot++;
-	}
-	if (!hasdot)
-		return NULL;
-
-	/* NOTE: hostname may not be NUL-terminated. */
-	bzero(fqdn, sizeof(fqdn));
-	bcopy(hostname, fqdn, hostnamelen);
-	fqdn[hostnamelen] = '\0';
-	return fqdn;
-}
-
-/*
- * get username@FQDN for the host/user.
- */
-static const char *
-key_getuserfqdn()
-{
-	const char *host;
-	static char userfqdn[MAXHOSTNAMELEN + MAXLOGNAME + 2];
-	struct proc *p = curproc;
-	char *q;
-
-	if (!p || !p->p_pgrp || !p->p_pgrp->pg_session)
-		return NULL;
-	if (!(host = key_getfqdn()))
-		return NULL;
-
-	/* NOTE: s_login may not be-NUL terminated. */
-	bzero(userfqdn, sizeof(userfqdn));
-	bcopy(p->p_pgrp->pg_session->s_login, userfqdn, MAXLOGNAME);
-	userfqdn[MAXLOGNAME] = '\0';	/* safeguard */
-	q = userfqdn + strlen(userfqdn);
-	*q++ = '@';
-	bcopy(host, q, strlen(host));
-	q += strlen(host);
-	*q++ = '\0';
-
-	return userfqdn;
-}
-#endif
-
 /* record data transfer on SA, and update timestamps */
 void
 key_sa_recordxfer(sav, m)
 	struct secasvar *sav;
 	struct mbuf *m;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
+	
 	if (!sav)
 		panic("key_sa_recordxfer called with sav == NULL");
 	if (!m)
@@ -7608,6 +8048,7 @@ key_sa_recordxfer(sav, m)
 	if (!sav->lft_c)
 		return;
 
+	lck_mtx_lock(sadb_mutex);
 	/*
 	 * XXX Currently, there is a difference of bytes size
 	 * between inbound and outbound processing.
@@ -7640,7 +8081,8 @@ key_sa_recordxfer(sav, m)
 	sav->lft_c->sadb_lifetime_usetime = tv.tv_sec;
 	/* XXX check for expires? */
     }
-
+	lck_mtx_unlock(sadb_mutex);
+	
 	return;
 }
 
@@ -7651,7 +8093,7 @@ key_sa_routechange(dst)
 {
 	struct secashead *sah;
 	struct route *ro;
-
+	
 	lck_mtx_lock(sadb_mutex);
 	LIST_FOREACH(sah, &sahtree, chain) {
 		ro = &sah->sa_route;
@@ -7671,7 +8113,6 @@ key_sa_chgstate(sav, state)
 	struct secasvar *sav;
 	u_int8_t state;
 {
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 
 	if (sav == NULL)
 		panic("key_sa_chgstate called with sav == NULL");
@@ -7679,23 +8120,25 @@ key_sa_chgstate(sav, state)
 	if (sav->state == state)
 		return;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+		
 	if (__LIST_CHAINED(sav))
 		LIST_REMOVE(sav, chain);
 
 	sav->state = state;
 	LIST_INSERT_HEAD(&sav->sah->savtree[state], sav, chain);
+	
 }
 
 void
 key_sa_stir_iv(sav)
 	struct secasvar *sav;
 {
-
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
-
+	lck_mtx_lock(sadb_mutex);
 	if (!sav->iv)
 		panic("key_sa_stir_iv called with sav == NULL");
 	key_randomfill(sav->iv, sav->ivlen);
+	lck_mtx_unlock(sadb_mutex);
 }
 
 /* XXX too much? */
diff --git a/bsd/netkey/key.h b/bsd/netkey/key.h
index ce509fa9f..985b347fe 100644
--- a/bsd/netkey/key.h
+++ b/bsd/netkey/key.h
@@ -35,6 +35,9 @@
 
 #ifdef KERNEL_PRIVATE
 
+#define KEY_SADB_UNLOCKED	0
+#define KEY_SADB_LOCKED		1
+
 extern struct key_cb key_cb;
 
 struct secpolicy;
@@ -47,14 +50,17 @@ struct sadb_msg;
 struct sadb_x_policy;
 
 extern struct secpolicy *key_allocsp(struct secpolicyindex *, u_int);
+extern struct secasvar *key_allocsa_policy(struct secasindex *);
 extern struct secpolicy *key_gettunnel(struct sockaddr *,
 	struct sockaddr *, struct sockaddr *, struct sockaddr *);
-extern int key_checkrequest(struct ipsecrequest *isr, struct secasindex *);
+extern int key_checkrequest(struct ipsecrequest *isr, struct secasindex *,
+	struct secasvar **sav);
 extern struct secasvar *key_allocsa(u_int, caddr_t, caddr_t,
 					u_int, u_int32_t);
-extern void key_freesp(struct secpolicy *);
+extern u_int16_t key_natt_get_translated_port(struct secasvar *);
+extern void key_freesp(struct secpolicy *, int);
 extern void key_freeso(struct socket *);
-extern void key_freesav(struct secasvar *);
+extern void key_freesav(struct secasvar *, int);
 extern struct secpolicy *key_newsp(void);
 extern struct secpolicy *key_msg2sp(struct sadb_x_policy *, size_t, int *);
 extern struct mbuf *key_sp2msg(struct secpolicy *);
diff --git a/bsd/netkey/key_debug.c b/bsd/netkey/key_debug.c
index 5c3acffdc..054edecb1 100644
--- a/bsd/netkey/key_debug.c
+++ b/bsd/netkey/key_debug.c
@@ -470,6 +470,7 @@ kdebug_secpolicy(sp)
 
 	switch (sp->policy) {
 	case IPSEC_POLICY_DISCARD:
+	case IPSEC_POLICY_GENERATE:
 		printf("  type=discard }\n");
 		break;
 	case IPSEC_POLICY_NONE:
@@ -483,8 +484,6 @@ kdebug_secpolicy(sp)
 			printf("  level=%u\n", isr->level);
 			kdebug_secasindex(&isr->saidx);
 
-			if (isr->sav != NULL)
-				kdebug_secasv(isr->sav);
 		}
 		printf("  }\n");
 	    }
diff --git a/bsd/netkey/key_debug.h b/bsd/netkey/key_debug.h
index a9f823d77..761571725 100644
--- a/bsd/netkey/key_debug.h
+++ b/bsd/netkey/key_debug.h
@@ -52,8 +52,12 @@
 #define KEYDEBUG_IPSEC_DATA	(KEYDEBUG_IPSEC | KEYDEBUG_DATA)
 #define KEYDEBUG_IPSEC_DUMP	(KEYDEBUG_IPSEC | KEYDEBUG_DUMP)
 
+#if 0
 #define KEYDEBUG(lev,arg) \
 	do { if ((key_debug_level & (lev)) == (lev)) { arg; } } while (0)
+#else
+#define KEYDEBUG(lev,arg)
+#endif
 
 struct sadb_msg;
 struct sadb_ext;
@@ -83,7 +87,6 @@ extern void kdebug_sockaddr(struct sockaddr *);
 extern void ipsec_hexdump(caddr_t, int);
 extern void ipsec_bindump(caddr_t, int);
 
-#define KEYDEBUG(lev,arg)
 
 #endif /* _NETKEY_KEY_DEBUG_H_ */
 
diff --git a/bsd/netkey/keydb.c b/bsd/netkey/keydb.c
index 8da09b877..362ce530f 100644
--- a/bsd/netkey/keydb.c
+++ b/bsd/netkey/keydb.c
@@ -49,9 +49,11 @@
 
 #include <net/net_osdep.h>
 
+extern lck_mtx_t  *sadb_mutex;
+
 MALLOC_DEFINE(M_SECA, "key mgmt", "security associations, key management");
 
-static void keydb_delsecasvar(struct secasvar *);
+// static void keydb_delsecasvar(struct secasvar *); // not used
 
 /*
  * secpolicy management
@@ -61,6 +63,8 @@ keydb_newsecpolicy()
 {
 	struct secpolicy *p;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+
 	p = (struct secpolicy *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
 	if (!p)
 		return p;
@@ -85,8 +89,15 @@ keydb_newsecashead()
 	struct secashead *p;
 	int i;
 
-	p = (struct secashead *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
-	if (!p)
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+
+	p = (struct secashead *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT);
+	if (!p) {
+		lck_mtx_unlock(sadb_mutex);
+		p = (struct secashead *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
+		lck_mtx_lock(sadb_mutex);
+	}
+	if (!p) 
 		return p;
 	bzero(p, sizeof(*p));
 	for (i = 0; i < sizeof(p->savtree)/sizeof(p->savtree[0]); i++)
@@ -94,6 +105,7 @@ keydb_newsecashead()
 	return p;
 }
 
+#if 0
 void
 keydb_delsecashead(p)
 	struct secashead *p;
@@ -102,7 +114,9 @@ keydb_delsecashead(p)
 	_FREE(p, M_SECA);
 }
 
-/*
+
+
+/* 
  * secasvar management (reference counted)
  */
 struct secasvar *
@@ -110,6 +124,8 @@ keydb_newsecasvar()
 {
 	struct secasvar *p;
 
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_NOTOWNED);
+
 	p = (struct secasvar *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
 	if (!p)
 		return p;
@@ -122,25 +138,23 @@ void
 keydb_refsecasvar(p)
 	struct secasvar *p;
 {
-	int s;
 
-	s = splnet();
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+
 	p->refcnt++;
-	splx(s);
 }
 
 void
 keydb_freesecasvar(p)
 	struct secasvar *p;
 {
-	int s;
 
-	s = splnet();
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+
 	p->refcnt--;
 	/* negative refcnt will cause panic intentionally */
 	if (p->refcnt <= 0)
 		keydb_delsecasvar(p);
-	splx(s);
 }
 
 static void
@@ -153,6 +167,7 @@ keydb_delsecasvar(p)
 
 	_FREE(p, M_SECA);
 }
+#endif
 
 /*
  * secreplay management
@@ -162,17 +177,29 @@ keydb_newsecreplay(wsize)
 	size_t wsize;
 {
 	struct secreplay *p;
-
-	p = (struct secreplay *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
+	
+	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
+
+	p = (struct secreplay *)_MALLOC(sizeof(*p), M_SECA, M_NOWAIT);
+	if (!p) {
+		lck_mtx_unlock(sadb_mutex);
+		p = (struct secreplay *)_MALLOC(sizeof(*p), M_SECA, M_WAITOK);
+		lck_mtx_lock(sadb_mutex);
+	}
 	if (!p)
 		return p;
 
 	bzero(p, sizeof(*p));
 	if (wsize != 0) {
-		p->bitmap = (caddr_t)_MALLOC(wsize, M_SECA, M_WAITOK);
+		p->bitmap = (caddr_t)_MALLOC(wsize, M_SECA, M_NOWAIT);
 		if (!p->bitmap) {
-			_FREE(p, M_SECA);
-			return NULL;
+			lck_mtx_unlock(sadb_mutex);
+			p->bitmap = (caddr_t)_MALLOC(wsize, M_SECA, M_WAITOK);
+			lck_mtx_lock(sadb_mutex);
+			if (!p->bitmap) {
+				_FREE(p, M_SECA);
+				return NULL;
+			}
 		}
 		bzero(p->bitmap, wsize);
 	}
@@ -190,7 +217,8 @@ keydb_delsecreplay(p)
 	_FREE(p, M_SECA);
 }
 
-/*
+#if 0
+/*	NOT USED
  * secreg management
  */
 struct secreg *
@@ -211,3 +239,4 @@ keydb_delsecreg(p)
 
 	_FREE(p, M_SECA);
 }
+#endif
diff --git a/bsd/netkey/keydb.h b/bsd/netkey/keydb.h
index 80b16054f..53cddffce 100644
--- a/bsd/netkey/keydb.h
+++ b/bsd/netkey/keydb.h
@@ -101,6 +101,7 @@ struct secasvar {
 	/* Nat Traversal related bits */
 	u_int32_t	natt_last_activity;
 	u_int16_t	remote_ike_port;
+	u_int16_t	natt_encapsulated_src_port;	/* network byte order */
 };
 
 /* replay prevention */
@@ -148,17 +149,17 @@ extern struct secpolicy *keydb_newsecpolicy(void);
 extern void keydb_delsecpolicy(struct secpolicy *);
 /* secashead */
 extern struct secashead *keydb_newsecashead(void);
-extern void keydb_delsecashead(struct secashead *);
+// extern void keydb_delsecashead(struct secashead *);	// not used
 /* secasvar */
-extern struct secasvar *keydb_newsecasvar(void);
-extern void keydb_refsecasvar(struct secasvar *);
-extern void keydb_freesecasvar(struct secasvar *);
+// extern struct secasvar *keydb_newsecasvar(void);		// not used
+// extern void keydb_refsecasvar(struct secasvar *);	// not used
+// extern void keydb_freesecasvar(struct secasvar *);	// not used
 /* secreplay */
 extern struct secreplay *keydb_newsecreplay(size_t);
 extern void keydb_delsecreplay(struct secreplay *);
 /* secreg */
-extern struct secreg *keydb_newsecreg(void);
-extern void keydb_delsecreg(struct secreg *);
+// extern struct secreg *keydb_newsecreg(void);			// not used
+// extern void keydb_delsecreg(struct secreg *);		// not used
 
 #endif /* KERNEL_PRIVATE */
 #endif /* KERNEL */
diff --git a/bsd/netkey/keysock.c b/bsd/netkey/keysock.c
index dcf0b5e68..08f515e1c 100644
--- a/bsd/netkey/keysock.c
+++ b/bsd/netkey/keysock.c
@@ -56,16 +56,18 @@
 #include <netkey/key_debug.h>
 
 extern lck_mtx_t *raw_mtx;
-extern lck_mtx_t *sadb_mutex;
-extern void key_init(void);
+extern void key_init(void) __attribute__((section("__TEXT, initcode")));
 
-struct sockaddr key_dst = { 2, PF_KEY, };
-struct sockaddr key_src = { 2, PF_KEY, };
+struct sockaddr key_dst = { 2, PF_KEY, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,} };
+struct sockaddr key_src = { 2, PF_KEY, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,} };
 
 static int key_sendup0(struct rawcb *, struct mbuf *, int);
 
 struct pfkeystat pfkeystat;
 
+
+extern lck_mtx_t *pfkey_stat_mutex;
+
 /*
  * key_output()
  */
@@ -98,16 +100,17 @@ key_output(m, va_alist)
 		panic("key_output: NULL pointer was passed.\n");
 
 	socket_unlock(so, 0);
-	lck_mtx_lock(sadb_mutex);
+	lck_mtx_lock(pfkey_stat_mutex);
 	pfkeystat.out_total++;
 	pfkeystat.out_bytes += m->m_pkthdr.len;
+	lck_mtx_unlock(pfkey_stat_mutex);
 
 	len = m->m_pkthdr.len;
 	if (len < sizeof(struct sadb_msg)) {
 #if IPSEC_DEBUG
 		printf("key_output: Invalid message length.\n");
 #endif
-		pfkeystat.out_tooshort++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_tooshort);
 		error = EINVAL;
 		goto end;
 	}
@@ -117,7 +120,7 @@ key_output(m, va_alist)
 #if IPSEC_DEBUG
 			printf("key_output: can't pullup mbuf\n");
 #endif
-			pfkeystat.out_nomem++;
+			PFKEY_STAT_INCREMENT(pfkeystat.out_nomem);
 			error = ENOBUFS;
 			goto end;
 		}
@@ -131,12 +134,12 @@ key_output(m, va_alist)
 #endif /* defined(IPSEC_DEBUG) */
 
 	msg = mtod(m, struct sadb_msg *);
-	pfkeystat.out_msgtype[msg->sadb_msg_type]++;
+	PFKEY_STAT_INCREMENT(pfkeystat.out_msgtype[msg->sadb_msg_type]);
 	if (len != PFKEY_UNUNIT64(msg->sadb_msg_len)) {
 #if IPSEC_DEBUG
 		printf("key_output: Invalid message length.\n");
 #endif
-		pfkeystat.out_invlen++;
+		PFKEY_STAT_INCREMENT(pfkeystat.out_invlen);
 		error = EINVAL;
 		goto end;
 	}
@@ -147,7 +150,6 @@ key_output(m, va_alist)
 end:
 	if (m)
 		m_freem(m);
-	lck_mtx_unlock(sadb_mutex);
 	socket_lock(so, 0);
 	return error;
 }
@@ -163,7 +165,6 @@ key_sendup0(rp, m, promisc)
 {
 	int error;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	if (promisc) {
 		struct sadb_msg *pmsg;
 
@@ -174,7 +175,7 @@ key_sendup0(rp, m, promisc)
 #if IPSEC_DEBUG
 			printf("key_sendup0: cannot pullup\n");
 #endif
-			pfkeystat.in_nomem++;
+			PFKEY_STAT_INCREMENT(pfkeystat.in_nomem);
 			m_freem(m);
 			return ENOBUFS;
 		}
@@ -187,7 +188,7 @@ key_sendup0(rp, m, promisc)
 		pmsg->sadb_msg_len = PFKEY_UNIT64(m->m_pkthdr.len);
 		/* pid and seq? */
 
-		pfkeystat.in_msgtype[pmsg->sadb_msg_type]++;
+		PFKEY_STAT_INCREMENT(pfkeystat.in_msgtype[pmsg->sadb_msg_type]);
 	}
 
 	if (!sbappendaddr(&rp->rcb_socket->so_rcv, (struct sockaddr *)&key_src,
@@ -195,7 +196,7 @@ key_sendup0(rp, m, promisc)
 #if IPSEC_DEBUG
 		printf("key_sendup0: sbappendaddr failed\n");
 #endif
-		pfkeystat.in_nomem++;
+		PFKEY_STAT_INCREMENT(pfkeystat.in_nomem);
 	}
 	else {
 		sorwakeup(rp->rcb_socket);
@@ -217,19 +218,20 @@ key_sendup_mbuf(so, m, target)
 	struct rawcb *rp;
 	int error = 0;
 
-	lck_mtx_assert(sadb_mutex, LCK_MTX_ASSERT_OWNED);
 	if (m == NULL)
 		panic("key_sendup_mbuf: NULL pointer was passed.\n");
 	if (so == NULL && target == KEY_SENDUP_ONE)
 		panic("key_sendup_mbuf: NULL pointer was passed.\n");
 
+	lck_mtx_lock(pfkey_stat_mutex);
 	pfkeystat.in_total++;
 	pfkeystat.in_bytes += m->m_pkthdr.len;
+	lck_mtx_unlock(pfkey_stat_mutex);
 	if (m->m_len < sizeof(struct sadb_msg)) {
 #if 1
 		m = m_pullup(m, sizeof(struct sadb_msg));
 		if (m == NULL) {
-			pfkeystat.in_nomem++;
+			PFKEY_STAT_INCREMENT(pfkeystat.in_nomem);
 			return ENOBUFS;
 		}
 #else
@@ -239,7 +241,7 @@ key_sendup_mbuf(so, m, target)
 	if (m->m_len >= sizeof(struct sadb_msg)) {
 		struct sadb_msg *msg;
 		msg = mtod(m, struct sadb_msg *);
-		pfkeystat.in_msgtype[msg->sadb_msg_type]++;
+		PFKEY_STAT_INCREMENT(pfkeystat.in_msgtype[msg->sadb_msg_type]);
 	}
 	
 	lck_mtx_lock(raw_mtx);
@@ -286,7 +288,7 @@ key_sendup_mbuf(so, m, target)
 				sendup++;
 			break;
 		}
-		pfkeystat.in_msgtarget[target]++;
+		PFKEY_STAT_INCREMENT(pfkeystat.in_msgtarget[target]);
 
 		if (!sendup) {
 			socket_unlock(rp->rcb_socket, 1);
@@ -300,7 +302,7 @@ key_sendup_mbuf(so, m, target)
 			printf("key_sendup: m_copy fail\n");
 #endif
 			m_freem(m);
-			pfkeystat.in_nomem++;
+			PFKEY_STAT_INCREMENT(pfkeystat.in_nomem);
 			socket_unlock(rp->rcb_socket, 1);
 			lck_mtx_unlock(raw_mtx);
 			return ENOBUFS;
@@ -422,9 +424,7 @@ key_detach(struct socket *so)
 			key_cb.key_count--;
 		key_cb.any_count--;
 		socket_unlock(so, 0);
-		lck_mtx_lock(sadb_mutex);
 		key_freereg(so);
-		lck_mtx_unlock(sadb_mutex);
 		socket_lock(so, 0);
 	}
 	error = raw_usrreqs.pru_detach(so);
@@ -503,7 +503,7 @@ struct pr_usrreqs key_usrreqs = {
 };
 
 /* sysctl */
-SYSCTL_NODE(_net, PF_KEY, key, CTLFLAG_RW, 0, "Key Family");
+SYSCTL_NODE(_net, PF_KEY, key, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "Key Family");
 
 /*
  * Definitions of protocols supported in the KEY domain.
@@ -513,20 +513,21 @@ extern struct domain keydomain;
 
 struct protosw keysw[] = {
 { SOCK_RAW,	&keydomain,	PF_KEY_V2,	PR_ATOMIC|PR_ADDR,
-  0,		key_output,	raw_ctlinput,	0,
-  0,
-  key_init,	0,		0,		0,
-  0,
+  NULL,		key_output,	raw_ctlinput,	NULL,
+  NULL,
+  key_init,	NULL,		NULL,		NULL,
+  NULL,
   &key_usrreqs,
-  0,		0,		0,
+  NULL,		NULL,		NULL,
+  { NULL, NULL }, NULL, { 0 }
 }
 };
 
-struct domain keydomain =
-    { PF_KEY, "key", key_domain_init, 0, 0,
-      keysw, 0,
-      0,0,
-      sizeof(struct key_cb), 0
-    };
+struct domain keydomain = { PF_KEY, "key", key_domain_init, NULL, NULL,
+      keysw, NULL,
+      NULL, 0,
+      sizeof(struct key_cb), 0, 0,
+      NULL, 0, { 0, 0}
+};
 
 DOMAIN_SET(key);
diff --git a/bsd/netkey/keysock.h b/bsd/netkey/keysock.h
index c055f2488..413e281d5 100644
--- a/bsd/netkey/keysock.h
+++ b/bsd/netkey/keysock.h
@@ -62,6 +62,10 @@ struct pfkeystat {
 #define KEY_SENDUP_ALL		1
 #define KEY_SENDUP_REGISTERED	2
 
+#define PFKEY_STAT_INCREMENT(x)	 \
+	{lck_mtx_lock(pfkey_stat_mutex); (x)++; lck_mtx_unlock(pfkey_stat_mutex);}
+
+
 #ifdef KERNEL_PRIVATE
 #ifdef KERNEL
 struct keycb {
diff --git a/bsd/nfs/Makefile b/bsd/nfs/Makefile
index fa4b4198d..9464a92ae 100644
--- a/bsd/nfs/Makefile
+++ b/bsd/nfs/Makefile
@@ -21,8 +21,8 @@ EXPINC_SUBDIRS_I386 = \
 
 DATAFILES = \
 	krpc.h nfs.h nfsdiskless.h nfsm_subs.h nfsmount.h nfsnode.h \
-	nfs_lock.h \
-	nfsproto.h nfsrtt.h nfsrvcache.h rpcv2.h xdr_subs.h
+	nfs_lock.h nfs_gss.h \
+	nfsproto.h nfsrvcache.h rpcv2.h xdr_subs.h
 
 
 INSTALL_MI_LIST	= ${DATAFILES}
diff --git a/bsd/nfs/krpc.h b/bsd/nfs/krpc.h
index 5d69fd6f6..16fde5248 100644
--- a/bsd/nfs/krpc.h
+++ b/bsd/nfs/krpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 
diff --git a/bsd/nfs/krpc_subr.c b/bsd/nfs/krpc_subr.c
index 694b152f5..b1d597105 100644
--- a/bsd/nfs/krpc_subr.c
+++ b/bsd/nfs/krpc_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -210,10 +216,11 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 	mbuf_t m, nam, mhead;
 	struct rpc_call *call;
 	struct rpc_reply *reply;
-	int error, timo, secs, len;
+	int error, timo, secs;
+	size_t len;
 	static u_int32_t xid = ~0xFF;
 	u_int16_t tport;
-	int maxpacket = 1<<16;
+	size_t maxpacket = 1<<16;
 
 	/*
 	 * Validate address family.
@@ -229,7 +236,7 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 	 * Create socket and set its recieve timeout.
 	 */
 	if ((error = sock_socket(AF_INET, sotype, 0, 0, 0, &so)))
-		goto out;
+		goto out1;
 
 	{
 		struct timeval tv;
@@ -408,7 +415,7 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 				if (!error && readlen < aio.iov_len) {
 				    /* only log a message if we got a partial word */
 				    if (readlen != 0)
-					    printf("short receive (%d/%d) from server " IP_FORMAT "\n",
+					    printf("short receive (%ld/%ld) from server " IP_FORMAT "\n",
 						 readlen, sizeof(u_long), IP_LIST(&(sin->sin_addr.s_addr)));
 				    error = EPIPE;
 				}
@@ -420,7 +427,7 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 				 * and forcing a disconnect/reconnect is all I can do.
 				 */
 				if (len > maxpacket) {
-				    printf("impossible packet length (%d) from server %s\n",
+				    printf("impossible packet length (%ld) from server " IP_FORMAT "\n",
 					len, IP_LIST(&(sin->sin_addr.s_addr)));
 				    error = EFBIG;
 				    goto out;
@@ -431,8 +438,8 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 				    error = sock_receivembuf(so, NULL, &m, MSG_WAITALL, &readlen);
 				} while (error == EWOULDBLOCK);
 
-				if (!error && (len > (int)readlen)) {
-				    printf("short receive (%d/%d) from server %s\n",
+				if (!error && (len > readlen)) {
+				    printf("short receive (%ld/%ld) from server " IP_FORMAT "\n",
 					readlen, len, IP_LIST(&(sin->sin_addr.s_addr)));
 				    error = EPIPE;
 				}
@@ -550,8 +557,9 @@ krpc_call(sa, sotype, prog, vers, func, data, from_p)
 	/* result */
 	*data = m;
  out:
+	sock_close(so);
+out1:
 	if (nam) mbuf_freem(nam);
 	if (mhead) mbuf_freem(mhead);
-	sock_close(so);
 	return error;
 }
diff --git a/bsd/nfs/nfs.h b/bsd/nfs/nfs.h
index b6da2fbaa..4ff6d43f7 100644
--- a/bsd/nfs/nfs.h
+++ b/bsd/nfs/nfs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -70,43 +76,40 @@
  * Tunable constants for nfs
  */
 
-#define	NFS_MAXIOVEC	34
 #define NFS_TICKINTVL	5		/* Desired time for a tick (msec) */
 #define NFS_HZ		(hz / nfs_ticks) /* Ticks/sec */
+__private_extern__ int nfs_ticks;
 #define	NFS_TIMEO	(1 * NFS_HZ)	/* Default timeout = 1 second */
 #define	NFS_MINTIMEO	(1 * NFS_HZ)	/* Min timeout to use */
 #define	NFS_MAXTIMEO	(60 * NFS_HZ)	/* Max timeout to backoff to */
 #define	NFS_MINIDEMTIMEO (5 * NFS_HZ)	/* Min timeout for non-idempotent ops*/
 #define	NFS_MAXREXMIT	100		/* Stop counting after this many */
-#define	NFS_MAXWINDOW	1024		/* Max number of outstanding requests */
 #define	NFS_RETRANS	10		/* Num of retrans for soft mounts */
-#define	NFS_TRYLATERDEL	15		/* Initial try later delay (sec) */
+#define	NFS_TRYLATERDEL	4		/* Initial try later delay (sec) */
 #define	NFS_MAXGRPS	16		/* Max. size of groups list */
-#ifndef NFS_MINATTRTIMO
 #define	NFS_MINATTRTIMO 5		/* Attribute cache timeout in sec */
-#endif
-#ifndef NFS_MAXATTRTIMO
 #define	NFS_MAXATTRTIMO 60
-#endif
-#ifndef NFS_MINDIRATTRTIMO
 #define	NFS_MINDIRATTRTIMO 5		/* directory attribute cache timeout in sec */
-#endif
-#ifndef NFS_MAXDIRATTRTIMO
 #define	NFS_MAXDIRATTRTIMO 60
-#endif
-#define	NFS_IOSIZE	(256 * 1024)	/* suggested I/O size */
-#define	NFS_WSIZE	16384		/* Def. write data size <= 16K */
-#define	NFS_RSIZE	16384		/* Def. read data size <= 16K */
+#define	NFS_IOSIZE	(1024 * 1024)	/* suggested I/O size */
+#define	NFS_RWSIZE	32768		/* Def. read/write data size <= 32K */
+#define	NFS_WSIZE	NFS_RWSIZE	/* Def. write data size <= 32K */
+#define	NFS_RSIZE	NFS_RWSIZE	/* Def. read data size <= 32K */
 #define	NFS_DGRAM_WSIZE	8192		/* UDP Def. write data size <= 8K */
 #define	NFS_DGRAM_RSIZE	8192		/* UDP Def. read data size <= 8K */
 #define NFS_READDIRSIZE	8192		/* Def. readdir size */
-#define	NFS_DEFRAHEAD	4		/* Def. read ahead # blocks */
-#define	NFS_MAXRAHEAD	16		/* Max. read ahead # blocks */
-#define	NFS_MAXUIDHASH	64		/* Max. # of hashed uid entries/mp */
-#define	NFS_MAXASYNCDAEMON 	32	/* Max. number async_daemons runnable */
-#define NFS_MAXGATHERDELAY	100	/* Max. write gather delay (msec) */
-#ifndef NFS_GATHERDELAY
-#define NFS_GATHERDELAY		10	/* Default write gather delay (msec) */
+#define	NFS_DEFRAHEAD	16		/* Def. read ahead # blocks */
+#define	NFS_MAXRAHEAD	128		/* Max. read ahead # blocks */
+#define	NFS_DEFMAXASYNCWRITES 	128	/* Def. max # concurrent async write RPCs */
+#define	NFS_DEFASYNCTHREAD 	16	/* Def. # nfsiod threads */
+#define	NFS_MAXASYNCTHREAD 	64	/* max # nfsiod threads */
+#define	NFS_ASYNCTHREADMAXIDLE	60	/* Seconds before idle nfsiods are reaped */
+#define	NFS_DEFSTATFSRATELIMIT 	10	/* Def. max # statfs RPCs per second */
+#define NFS_REQUESTDELAY	10	/* ms interval to check request queue */
+#define NFSRV_DEADSOCKDELAY	5	/* Seconds before dead sockets are reaped */
+#define NFSRV_MAXWGATHERDELAY	100	/* Max. write gather delay (msec) */
+#ifndef NFSRV_WGATHERDELAY
+#define NFSRV_WGATHERDELAY	1	/* Default write gather delay (msec) */
 #endif
 #define	NFS_DIRBLKSIZ	4096		/* Must be a multiple of DIRBLKSIZ */
 #if defined(KERNEL) && !defined(DIRBLKSIZ)
@@ -114,28 +117,19 @@
  					/* can't be larger than NFS_FABLKSIZE */
 #endif
 
+/* default values for unresponsive mount timeouts */
+#define NFS_TPRINTF_INITIAL_DELAY	12
+#define NFS_TPRINTF_DELAY		30
+
 /*
  * Oddballs
  */
-#define	NMOD(a)		((a) % nfs_asyncdaemons)
 #define NFS_CMPFH(n, f, s) \
 	((n)->n_fhsize == (s) && !bcmp((caddr_t)(n)->n_fhp, (caddr_t)(f), (s)))
-#define NFS_ISV3(v)	(VFSTONFS(vnode_mount(v))->nm_flag & NFSMNT_NFSV3)
 #define NFS_SRVMAXDATA(n) \
-		(((n)->nd_flag & ND_NFSV3) ? (((n)->nd_nam2) ? \
+		(((n)->nd_vers == NFS_VER3) ? (((n)->nd_nam2) ? \
 		 NFS_MAXDGRAMDATA : NFS_MAXDATA) : NFS_V2MAXDATA)
 
-/*
- * XXX
- * The NB_INVAFTERWRITE flag should be set to whatever is required by the
- * buffer cache code to say "Invalidate the block after it is written back".
- */
-#ifdef __FreeBSD__
-#define	NB_INVAFTERWRITE	NB_NOCACHE
-#else
-#define	NB_INVAFTERWRITE	NB_INVAL
-#endif
-
 /*
  * The IO_METASYNC flag should be implemented for local file systems.
  * (Until then, it is nothin at all.)
@@ -157,12 +151,11 @@
 #define NFS_NODEALLOC	512
 #define NFS_MNTALLOC	512
 #define NFS_SVCALLOC	256
-#define NFS_UIDALLOC	128
 
 /*
  * Arguments to mount NFS
  */
-#define NFS_ARGSVERSION	4		/* change when nfs_args changes */
+#define NFS_ARGSVERSION	5		/* change when nfs_args changes */
 struct nfs_args {
 	int		version;	/* args structure version number */
 	struct sockaddr	*addr;		/* file server address */
@@ -187,6 +180,33 @@ struct nfs_args {
 	int		acregmax;	/* reg file max attr cache timeout */
 	int		acdirmin;	/* dir min attr cache timeout */
 	int		acdirmax;	/* dir max attr cache timeout */
+	/* NFS_ARGSVERSION 4 ends here */
+	uint32_t	auth;		/* security mechanism flavor */
+};
+struct nfs_args4 {
+	int		version;	/* args structure version number */
+	struct sockaddr	*addr;		/* file server address */
+	int		addrlen;	/* length of address */
+	int		sotype;		/* Socket type */
+	int		proto;		/* and Protocol */
+	u_char		*fh;		/* File handle to be mounted */
+	int		fhsize;		/* Size, in bytes, of fh */
+	int		flags;		/* flags */
+	int		wsize;		/* write size in bytes */
+	int		rsize;		/* read size in bytes */
+	int		readdirsize;	/* readdir size in bytes */
+	int		timeo;		/* initial timeout in .1 secs */
+	int		retrans;	/* times to retry send */
+	int		maxgrouplist;	/* Max. size of group list */
+	int		readahead;	/* # of blocks to readahead */
+	int		leaseterm;	/* obsolete: Term (sec) of lease */
+	int		deadthresh;	/* obsolete: Retrans threshold */
+	char		*hostname;	/* server's name */
+	/* NFS_ARGSVERSION 3 ends here */
+	int		acregmin;	/* reg file min attr cache timeout */
+	int		acregmax;	/* reg file max attr cache timeout */
+	int		acdirmin;	/* dir min attr cache timeout */
+	int		acdirmax;	/* dir max attr cache timeout */
 };
 
 struct nfs_args3 {
@@ -210,7 +230,6 @@ struct nfs_args3 {
 	char		*hostname;	/* server's name */
 };
 
-// LP64todo - should this move?
 #ifdef KERNEL
 /* LP64 version of nfs_args.  all pointers and longs
  * grow when we're dealing with a 64-bit process.
@@ -240,6 +259,33 @@ struct user_nfs_args {
 	int		acregmax;	/* reg file max attr cache timeout */
 	int		acdirmin;	/* dir min attr cache timeout */
 	int		acdirmax;	/* dir max attr cache timeout */
+	/* NFS_ARGSVERSION 4 ends here */
+	uint32_t	auth;		/* security mechanism flavor */
+};
+struct user_nfs_args4 {
+	int		version;	/* args structure version number */
+	user_addr_t	addr __attribute((aligned(8)));		/* file server address */
+	int		addrlen;	/* length of address */
+	int		sotype;		/* Socket type */
+	int		proto;		/* and Protocol */
+	user_addr_t	fh __attribute((aligned(8)));		/* File handle to be mounted */
+	int		fhsize;		/* Size, in bytes, of fh */
+	int		flags;		/* flags */
+	int		wsize;		/* write size in bytes */
+	int		rsize;		/* read size in bytes */
+	int		readdirsize;	/* readdir size in bytes */
+	int		timeo;		/* initial timeout in .1 secs */
+	int		retrans;	/* times to retry send */
+	int		maxgrouplist;	/* Max. size of group list */
+	int		readahead;	/* # of blocks to readahead */
+	int		leaseterm;	/* obsolete: Term (sec) of lease */
+	int		deadthresh;	/* obsolete: Retrans threshold */
+	user_addr_t	hostname __attribute((aligned(8)));	/* server's name */
+	/* NFS_ARGSVERSION 3 ends here */
+	int		acregmin;	/* reg file min attr cache timeout */
+	int		acregmax;	/* reg file max attr cache timeout */
+	int		acdirmin;	/* dir min attr cache timeout */
+	int		acdirmax;	/* dir max attr cache timeout */
 };
 struct user_nfs_args3 {
 	int		version;	/* args structure version number */
@@ -275,50 +321,27 @@ struct user_nfs_args3 {
 #define	NFSMNT_MAXGRPS		0x00000020  /* set maximum grouplist size */
 #define	NFSMNT_INT		0x00000040  /* allow interrupts on hard mount */
 #define	NFSMNT_NOCONN		0x00000080  /* Don't Connect the socket */
+#define	NFSMNT_NONEGNAMECACHE	0x00000100  /* Don't do negative name caching */
 #define	NFSMNT_NFSV3		0x00000200  /* Use NFS Version 3 protocol */
-#define	NFSMNT_KERB		0x00000400  /* Use Kerberos authentication */
+#define	NFSMNT_NFSV4		0x00000400  /* Use NFS Version 4 protocol */
 #define	NFSMNT_DUMBTIMR		0x00000800  /* Don't estimate rtt dynamically */
+// #define	NFSMNT_UNUSED	0x00001000  /* unused */
 #define	NFSMNT_READAHEAD	0x00002000  /* set read ahead */
+// #define	NFSMNT_UNUSED	0x00004000  /* unused */
 #define	NFSMNT_RESVPORT		0x00008000  /* Allocate a reserved port */
 #define	NFSMNT_RDIRPLUS		0x00010000  /* Use Readdirplus for V3 */
 #define	NFSMNT_READDIRSIZE	0x00020000  /* Set readdir size */
 #define	NFSMNT_NOLOCKS		0x00040000  /* don't support file locking */
+#define	NFSMNT_LOCALLOCKS	0x00080000  /* do file locking locally on client */
 #define	NFSMNT_ACREGMIN		0x00100000  /* reg min attr cache timeout */
 #define	NFSMNT_ACREGMAX		0x00200000  /* reg max attr cache timeout */
 #define	NFSMNT_ACDIRMIN		0x00400000  /* dir min attr cache timeout */
 #define	NFSMNT_ACDIRMAX		0x00800000  /* dir max attr cache timeout */
+#define	NFSMNT_SECFLAVOR	0x01000000  /* Use security flavor */
+#define	NFSMNT_SECGIVEN		0x02000000  /* A sec= mount option was given */
 
 /*
- * NFS mount state flags (nm_state)
- */
-#define NFSSTA_LOCKTIMEO	0x00002000  /* experienced a lock req timeout */
-#define	NFSSTA_MOUNTED		0x00004000  /* completely mounted */
-#define NFSSTA_LOCKSWORK	0x00008000  /* lock ops have worked. */
-#define NFSSTA_TIMEO		0x00010000  /* experienced a timeout. */
-#define NFSSTA_FORCE		0x00020000  /* doing a forced unmount. */
-#define NFSSTA_HASWRITEVERF	0x00040000  /* Has write verifier for V3 */
-#define NFSSTA_GOTPATHCONF	0x00080000  /* Got the V3 pathconf info */
-#define NFSSTA_GOTFSINFO	0x00100000  /* Got the V3 fsinfo */
-#define	NFSSTA_MNTD		0x00200000  /* Mnt server for mnt point */
-#define	NFSSTA_SNDLOCK		0x01000000  /* Send socket lock */
-#define	NFSSTA_WANTSND		0x02000000  /* Want above */
-#define	NFSSTA_RCVLOCK		0x04000000  /* Rcv socket lock */
-#define	NFSSTA_WANTRCV		0x08000000  /* Want above */
-#define	NFSSTA_WAITAUTH		0x10000000  /* Wait for authentication */
-#define	NFSSTA_HASAUTH		0x20000000  /* Has authenticator */
-#define	NFSSTA_WANTAUTH		0x40000000  /* Wants an authenticator */
-#define	NFSSTA_AUTHERR		0x80000000  /* Authentication error */
-
-/*
- * NFS mount pathconf info flags (nm_fsinfo.pcflags)
- */
-#define NFSPCINFO_NOTRUNC		0x01
-#define NFSPCINFO_CHOWN_RESTRICTED	0x02
-#define NFSPCINFO_CASE_INSENSITIVE	0x04
-#define NFSPCINFO_CASE_PRESERVING	0x08
-
-/*
- * Structures for the nfssvc(2) syscall. Not that anyone but nfsd and mount_nfs
+ * Structures for the nfssvc(2) syscall. Not that anyone but nfsd
  * should ever try and use it.
  */
 struct nfsd_args {
@@ -327,7 +350,6 @@ struct nfsd_args {
 	int	namelen;	/* Length of name */
 };
 
-// LP64todo - should this move?
 #ifdef KERNEL
 /* LP64 version of nfsd_args.  all pointers and longs
  * grow when we're dealing with a 64-bit process.
@@ -341,31 +363,6 @@ struct user_nfsd_args {
 
 #endif // KERNEL
 
-struct nfsd_srvargs {
-	struct nfsd	*nsd_nfsd;	/* Pointer to in kernel nfsd struct */
-	uid_t		nsd_uid;	/* Effective uid mapped to cred */
-	u_long		nsd_haddr;	/* Ip address of client */
-	struct ucred	nsd_cr;		/* Cred. uid maps to */
-	int		nsd_authlen;	/* Length of auth string (ret) */
-	u_char		*nsd_authstr;	/* Auth string (ret) */
-	int		nsd_verflen;	/* and the verfier */
-	u_char		*nsd_verfstr;
-	struct timeval	nsd_timestamp;	/* timestamp from verifier */
-	u_long		nsd_ttl;	/* credential ttl (sec) */
-	NFSKERBKEY_T	nsd_key;	/* Session key */
-};
-
-struct nfsd_cargs {
-	char		*ncd_dirp;	/* Mount dir path */
-	uid_t		ncd_authuid;	/* Effective uid */
-	int		ncd_authtype;	/* Type of authenticator */
-	int		ncd_authlen;	/* Length of authenticator string */
-	u_char		*ncd_authstr;	/* Authenticator string */
-	int		ncd_verflen;	/* and the verifier */
-	u_char		*ncd_verfstr;
-	NFSKERBKEY_T	ncd_key;	/* Session key */
-};
-
 /*
  * NFS Server File Handle structures
  */
@@ -385,24 +382,37 @@ struct nfs_exphandle {
 #define NXHF_INVALIDFH		0x0001		/* file handle is invalid */
 
 #define	NFS_MAX_FID_SIZE	(NFS_MAX_FH_SIZE - sizeof(struct nfs_exphandle))
+#define	NFSV4_MAX_FID_SIZE	(NFSV4_MAX_FH_SIZE - sizeof(struct nfs_exphandle))
+#define	NFSV3_MAX_FID_SIZE	(NFSV3_MAX_FH_SIZE - sizeof(struct nfs_exphandle))
 #define	NFSV2_MAX_FID_SIZE	(NFSV2_MAX_FH_SIZE - sizeof(struct nfs_exphandle))
 
 /* NFS server internal view of fhandle_t */
+/* The first sizeof(fhandle_t) bytes must match what goes into fhandle_t. */
+/* (fhp is used to allow use of an external buffer) */
 struct nfs_filehandle {
-	int			nfh_len;	/* total length of file handle */
+	uint32_t		nfh_len;	/* total length of file handle */
 	struct nfs_exphandle	nfh_xh;		/* export handle */
 	unsigned char		nfh_fid[NFS_MAX_FID_SIZE]; /* File ID */
+	unsigned char		*nfh_fhp;	/* pointer to file handle */
 };
 
 /*
  * NFS export data structures
  */
 
+/* Structure to hold an array of security flavors */
+#define NX_MAX_SEC_FLAVORS 5
+struct nfs_sec {
+	int count;
+	uint32_t flavors[NX_MAX_SEC_FLAVORS];
+}; 
+
 struct nfs_export_net_args {
 	uint32_t		nxna_flags;	/* export flags */
 	struct xucred		nxna_cred;	/* mapped credential for root/all user */
 	struct sockaddr_storage	nxna_addr;	/* net address to which exported */
 	struct sockaddr_storage	nxna_mask;	/* mask for net address */
+	struct nfs_sec		nxna_sec;	/* security mechanism flavors */
 };
 
 struct nfs_export_args {
@@ -435,14 +445,69 @@ struct user_nfs_export_args {
 #define NXA_ADD			0x0002	/* add the specified export(s) */
 #define NXA_REPLACE		0x0003	/* delete and add the specified export(s) */
 #define NXA_DELETE_ALL		0x0004	/* delete all exports */
+#define NXA_OFFLINE		0x0008	/* export is offline */
 
 /* export option flags */
 #define NX_READONLY		0x0001	/* exported read-only */
 #define NX_DEFAULTEXPORT	0x0002	/* exported to the world */
 #define NX_MAPROOT		0x0004	/* map root access to anon credential */
 #define NX_MAPALL		0x0008	/* map all access to anon credential */
-#define NX_KERB			0x0010	/* exported with Kerberos uid mapping */
 #define NX_32BITCLIENTS		0x0020	/* restrict directory cookies to 32 bits */
+#define NX_OFFLINE		0x0040	/* export is offline */
+
+/*
+ * fs.nfs sysctl(3) export stats record structures
+ */
+#define NFS_EXPORT_STAT_REC_VERSION 1	/* export stat record version */
+#define NFS_USER_STAT_REC_VERSION 1     /* active user list record version */
+
+/* descriptor describing following records */
+struct nfs_export_stat_desc
+{
+	uint32_t rec_vers;		/* version of export stat records */
+	uint32_t rec_count;		/* total record count */
+}__attribute__((__packed__));
+
+/* export stat record containing path and stat counters */
+struct nfs_export_stat_rec {
+	char     path[RPCMNT_PATHLEN + 1];
+	uint64_t ops;		/* Count of NFS Requests received for this export */
+	uint64_t bytes_read;	/* Count of bytes read from this export */
+	uint64_t bytes_written;	/* Count of bytes written to this export */
+}__attribute__((__packed__));
+
+/* Active user list stat buffer descriptor */
+struct nfs_user_stat_desc
+{
+	uint32_t rec_vers;	/* version of active user stat records */
+	uint32_t rec_count;	/* total record count */
+}__attribute__((__packed__));
+
+/* Active user list user stat record format */
+struct nfs_user_stat_user_rec {
+	u_char			rec_type;
+	uid_t			uid;
+	struct sockaddr_storage	sock;
+	uint64_t		ops;
+	uint64_t		bytes_read;
+	uint64_t		bytes_written;
+	uint32_t		tm_start;
+	uint32_t		tm_last;
+}__attribute__((__packed__));
+
+/* Active user list path record format */
+struct nfs_user_stat_path_rec {
+	u_char	rec_type;
+	char	path[RPCMNT_PATHLEN + 1];
+}__attribute__((__packed__));
+
+/* Defines for rec_type field of
+ * nfs_user_stat_rec & nfs_user_stat_rec
+ * data structures
+ */
+#define NFS_USER_STAT_USER_REC  0
+#define NFS_USER_STAT_PATH_REC  1
+
 
 #ifdef KERNEL
 struct nfs_exportfs;
@@ -450,6 +515,7 @@ struct nfs_exportfs;
 struct nfs_export_options {
 	uint32_t		nxo_flags;	/* export options */
 	kauth_cred_t		nxo_cred;	/* mapped credential */
+	struct nfs_sec		nxo_sec;	/* security mechanism flavors */
 };
 
 /* Network address lookup element and individual export options */
@@ -458,6 +524,74 @@ struct nfs_netopt {
 	struct nfs_export_options	no_opt;		/* export options */
 };
 
+/* statistic counters for each exported directory
+ *
+ * Since 64-bit atomic operations are not available on 32-bit platforms,
+ * 64-bit counters are implemented using 32-bit integers and 32-bit
+ * atomic operations
+ */
+typedef struct nfsstatcount64 {
+	uint32_t	hi;
+	uint32_t	lo;
+} nfsstatcount64;
+
+struct nfs_export_stat_counters {
+	struct nfsstatcount64 ops;		/* Count of NFS Requests received for this export  */
+	struct nfsstatcount64 bytes_read;	/* Count of bytes read from this export */
+	struct nfsstatcount64 bytes_written;	/* Count of bytes written to his export */
+}__attribute__((__packed__));
+
+/* Macro for updating nfs export stat counters */
+#define NFSStatAdd64(PTR, VAL) \
+	do { \
+		uint32_t NFSSA_OldValue = \
+		OSAddAtomic((VAL), (SInt32*)&(PTR)->lo); \
+		if ((NFSSA_OldValue + (VAL)) < NFSSA_OldValue) \
+			OSAddAtomic(1, (SInt32*)&(PTR)->hi); \
+	} while (0)
+
+/* Some defines for dealing with active user list stats */
+#define NFSRV_USER_STAT_DEF_MAX_NODES 1024	/* default active user list size limit */
+#define NFSRV_USER_STAT_DEF_IDLE_SEC  7200	/* default idle seconds (node no longer considered active) */
+
+/* active user list globals */
+__private_extern__ uint32_t nfsrv_user_stat_enabled;		/* enable/disable active user list */
+__private_extern__ uint32_t nfsrv_user_stat_node_count;		/* current count of user stat nodes */
+__private_extern__ uint32_t nfsrv_user_stat_max_idle_sec;	/* idle seconds (node no longer considered active) */
+__private_extern__ uint32_t nfsrv_user_stat_max_nodes;		/* active user list size limit */
+__private_extern__ lck_grp_t *nfsrv_active_user_mutex_group;
+
+/* An active user node represented in the kernel */
+struct nfs_user_stat_node {
+	TAILQ_ENTRY(nfs_user_stat_node)	lru_link;
+	LIST_ENTRY(nfs_user_stat_node)	hash_link;
+	uid_t			uid;
+	struct sockaddr_storage	sock;
+	uint64_t		ops;
+	uint64_t		bytes_read;
+	uint64_t		bytes_written;
+	uint32_t		tm_start;
+	uint32_t		tm_last;
+};
+
+/* Hash table for active user nodes */
+#define NFS_USER_STAT_HASH_SIZE 16	/* MUST be a power of 2 */
+#define NFS_USER_STAT_HASH(userhashtbl, uid) \
+		&((userhashtbl)[(uid) & (NFS_USER_STAT_HASH_SIZE - 1)])
+
+TAILQ_HEAD(nfs_user_stat_lru_head, nfs_user_stat_node);
+LIST_HEAD(nfs_user_stat_hashtbl_head, nfs_user_stat_node);
+
+/* Active user list data structure */
+/* One per exported directory */
+struct nfs_active_user_list {
+	struct nfs_user_stat_lru_head		user_lru;
+	struct nfs_user_stat_hashtbl_head	user_hashtbl[NFS_USER_STAT_HASH_SIZE];
+	uint32_t				node_count;
+	lck_mtx_t user_mutex;
+};
+
+
 /* Network export information */
 /* one of these for each exported directory */
 struct nfs_export {
@@ -472,6 +606,9 @@ struct nfs_export {
 	struct nfs_export_options	nx_defopt;	/* default options */
 	uint32_t			nx_expcnt;	/* # exports in table */
 	struct radix_node_head		*nx_rtable[AF_MAX+1]; /* table of exports (netopts) */
+	struct nfs_export_stat_counters	nx_stats;	/* statistic counters for this exported directory */
+	struct nfs_active_user_list	nx_user_list;	/* Active User List for this exported directory */
+	struct timeval			nx_exptime;	/* time of export for write verifier */
 };
 
 /* NFS exported file system info */
@@ -483,14 +620,34 @@ struct nfs_exportfs {
 	LIST_HEAD(,nfs_export)		nxfs_exports;	/* list of exports for this file system */
 };
 
-extern LIST_HEAD(nfsexpfslist, nfs_exportfs) nfs_exports;
-extern lck_rw_t nfs_export_rwlock;  // lock for export data structures
-#define	NFSEXPHASHVAL(FSID, EXPID)	\
+__private_extern__ LIST_HEAD(nfsrv_expfs_list, nfs_exportfs) nfsrv_exports;
+__private_extern__ lck_rw_t nfsrv_export_rwlock;  // lock for export data structures
+#define	NFSRVEXPHASHVAL(FSID, EXPID)	\
 	(((FSID) >> 24) ^ ((FSID) >> 16) ^ ((FSID) >> 8) ^ (EXPID))
-#define	NFSEXPHASH(FSID, EXPID)	\
-	(&nfsexphashtbl[NFSEXPHASHVAL((FSID),(EXPID)) & nfsexphash])
-extern LIST_HEAD(nfsexphashhead, nfs_export) *nfsexphashtbl;
-extern u_long nfsexphash;
+#define	NFSRVEXPHASH(FSID, EXPID)	\
+	(&nfsrv_export_hashtbl[NFSRVEXPHASHVAL((FSID),(EXPID)) & nfsrv_export_hash])
+__private_extern__ LIST_HEAD(nfsrv_export_hashhead, nfs_export) *nfsrv_export_hashtbl;
+__private_extern__ u_long nfsrv_export_hash;
+
+/*
+ * NFS server file mod fsevents
+ */
+struct nfsrv_fmod {
+	LIST_ENTRY(nfsrv_fmod)	fm_link;
+	vnode_t			fm_vp;
+	struct vfs_context	fm_context;
+	uint64_t		fm_deadline;
+};
+
+#define NFSRVFMODHASHSZ	128
+#define NFSRVFMODHASH(vp) (((u_long) vp) & nfsrv_fmod_hash)
+__private_extern__ LIST_HEAD(nfsrv_fmod_hashhead, nfsrv_fmod) *nfsrv_fmod_hashtbl;
+__private_extern__ u_long nfsrv_fmod_hash;
+__private_extern__ lck_mtx_t *nfsrv_fmod_mutex;
+__private_extern__ int nfsrv_fmod_pending, nfsrv_fsevents_enabled;
+__private_extern__ int nfsrv_async, nfsrv_reqcache_size, nfsrv_sock_max_rec_queue_length;
+
+__private_extern__ struct nfsstats nfsstats;
 
 #endif // KERNEL
 
@@ -540,137 +697,194 @@ struct nfsstats {
 /*
  * Flags for nfssvc() system call.
  */
-#define	NFSSVC_BIOD	0x002
 #define	NFSSVC_NFSD	0x004
 #define	NFSSVC_ADDSOCK	0x008
-#define	NFSSVC_AUTHIN	0x010
-#define	NFSSVC_GOTAUTH	0x040
-#define	NFSSVC_AUTHINFAIL 0x080
-#define	NFSSVC_MNTD	0x100
 #define	NFSSVC_EXPORT	0x200
 
 /*
  * Flags for nfsclnt() system call.
  */
 #define NFSCLNT_LOCKDANS	0x200
-#define NFSCLNT_LOCKDFD		0x400
-#define NFSCLNT_LOCKDWAIT	0x800
 
 /*
  * fs.nfs sysctl(3) identifiers
  */
-#define NFS_NFSSTATS	1		/* struct: struct nfsstats */
-#define NFS_NFSPRIVPORT	2		/* int: prohibit nfs to resvports */
-
-#define FS_NFS_NAMES { \
-		       { 0, 0 }, \
-		       { "nfsstats", CTLTYPE_STRUCT }, \
-		       { "nfsprivport", CTLTYPE_INT }, \
-}
+#define NFS_NFSSTATS	1	/* struct: struct nfsstats */
+#define NFS_EXPORTSTATS 3	/* gets exported directory stats */
+#define NFS_USERSTATS	4	/* gets exported directory active user stats */
+#define NFS_USERCOUNT	5	/* gets current count of active nfs users */
 
-#ifndef NFS_MUIDHASHSIZ
-#define NFS_MUIDHASHSIZ	63	/* Tune the size of nfsmount with this */
-#endif
 #ifndef NFS_WDELAYHASHSIZ
 #define	NFS_WDELAYHASHSIZ 16	/* and with this */
 #endif
 
-/*
- * The set of signals the interrupt an I/O in progress for NFSMNT_INT mounts.
- * What should be in this set is open to debate, but I believe that since
- * I/O system calls on ufs are never interrupted by signals the set should
- * be minimal. My reasoning is that many current programs that use signals
- * such as SIGALRM will not expect file I/O system calls to be interrupted
- * by them and break.
- */
 #ifdef KERNEL
 #include <sys/kernel_types.h>
+#include <kern/thread_call.h>
+#include <sys/kdebug.h>
+
+#define NFS_KERNEL_DEBUG KERNEL_DEBUG
+
+/* kernel debug trace macros */
+#define FSDBG(A, B, C, D, E) \
+	NFS_KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_NONE, \
+		(int)(B), (int)(C), (int)(D), (int)(E), 0)
+#define FSDBG_TOP(A, B, C, D, E) \
+	NFS_KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_START, \
+		(int)(B), (int)(C), (int)(D), (int)(E), 0)
+#define FSDBG_BOT(A, B, C, D, E) \
+	NFS_KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_END, \
+		(int)(B), (int)(C), (int)(D), (int)(E), 0)
 
 #ifdef MALLOC_DECLARE
 MALLOC_DECLARE(M_NFSREQ);
 MALLOC_DECLARE(M_NFSMNT);
 MALLOC_DECLARE(M_NFSDIROFF);
 MALLOC_DECLARE(M_NFSRVDESC);
-MALLOC_DECLARE(M_NFSUID);
 MALLOC_DECLARE(M_NFSD);
 MALLOC_DECLARE(M_NFSBIGFH);
 #endif
 
-struct uio; struct vnode_attr; struct nameidata;	/* XXX */
+struct uio; struct vnode_attr; struct nameidata; struct dqblk;	/* XXX */
 struct nfsbuf;
 struct nfs_vattr;
+struct nfs_fsattr;
 struct nfsnode;
+typedef struct nfsnode * nfsnode_t;
+struct nfsreq;
 
+/*
+ * The set of signals the interrupt an I/O in progress for NFSMNT_INT mounts.
+ * What should be in this set is open to debate, but I believe that since
+ * I/O system calls on ufs are never interrupted by signals the set should
+ * be minimal. My reasoning is that many current programs that use signals
+ * such as SIGALRM will not expect file I/O system calls to be interrupted
+ * by them and break.
+ */
 #define	NFSINT_SIGMASK	(sigmask(SIGINT)|sigmask(SIGTERM)|sigmask(SIGKILL)| \
 			 sigmask(SIGHUP)|sigmask(SIGQUIT))
 
-__private_extern__ int nfs_mbuf_mlen, nfs_mbuf_mhlen,
-	nfs_mbuf_minclsize, nfs_mbuf_mclbytes;
+__private_extern__ size_t nfs_mbuf_mhlen, nfs_mbuf_minclsize;
+
+/*
+ * NFS mbuf chain structure used for managing the building/dissection of RPCs
+ */
+struct nfsm_chain {
+	mbuf_t		nmc_mhead;	/* mbuf chain head */
+	mbuf_t		nmc_mcur;	/* current mbuf */
+	caddr_t		nmc_ptr;	/* pointer into current mbuf */
+	uint32_t	nmc_left;	/* bytes remaining in current mbuf */
+	uint32_t	nmc_flags;	/* flags for this nfsm_chain */
+};
+#define NFSM_CHAIN_FLAG_ADD_CLUSTERS	0x1	/* always add mbuf clusters */
 
 /*
- * Socket errors ignored for connectionless sockets??
- * For now, ignore them all
+ * Each retransmission of an RPCSEC_GSS request
+ * has an additional sequence number.
  */
-#define	NFSIGNORE_SOERROR(s, e) \
-		((e) != EINTR && (e) != ERESTART && (e) != EWOULDBLOCK && \
-		 (e) != EIO && ((s)) != SOCK_STREAM)
+struct gss_seq {
+	SLIST_ENTRY(gss_seq)	gss_seqnext;
+	uint32_t		gss_seqnum;
+};
 
 /*
- * Nfs outstanding request list element
+ * async NFS request callback info
+ */
+struct nfsreq_cbinfo {
+	void			(*rcb_func)(struct nfsreq *);	/* async request callback function */
+	struct nfsbuf		*rcb_bp;			/* buffer I/O RPC is for */
+	uint32_t		rcb_args[2];			/* additional callback args */
+};
+
+/*
+ * NFS outstanding request list element
  */
 struct nfsreq {
-	TAILQ_ENTRY(nfsreq) r_chain;
-	mbuf_t		r_mreq;
-	mbuf_t		r_mrep;
-	mbuf_t		r_md;
-	caddr_t		r_dpos;
-	struct nfsmount *r_nmp;
-	vnode_t		r_vp;
-	u_long		r_xid;
-	int		r_flags;	/* flags on request, see below */
-	int		r_retry;	/* max retransmission count */
-	int		r_rexmit;	/* current retrans count */
-	int		r_timer;	/* tick counter on reply */
-	u_int32_t	r_procnum;	/* NFS procedure number */
-	int		r_rtt;		/* RTT for rpc */
-	proc_t		r_procp;	/* Proc that did I/O system call */
-	long		r_lastmsg;	/* time of last tprintf */
+	lck_mtx_t		r_mtx;		/* NFS request mutex */
+	TAILQ_ENTRY(nfsreq)	r_chain;	/* request queue chain */
+	TAILQ_ENTRY(nfsreq)	r_achain;	/* mount's async I/O request queue chain */
+	TAILQ_ENTRY(nfsreq)	r_rchain;	/* mount's async I/O resend queue chain */
+	TAILQ_ENTRY(nfsreq)	r_cchain;	/* mount's cwnd queue chain */
+	mbuf_t			r_mrest;	/* request body mbufs */
+	mbuf_t			r_mhead;	/* request header mbufs */
+	struct nfsm_chain	r_nmrep;	/* reply mbufs */
+	nfsnode_t		r_np;		/* NFS node */
+	struct nfsmount		*r_nmp;		/* NFS mount point */
+	uint64_t		r_xid;		/* RPC transaction ID */
+	uint32_t		r_procnum;	/* NFS procedure number */
+	u_long			r_mreqlen;	/* request length */
+	int			r_flags;	/* flags on request, see below */
+	int			r_lflags;	/* flags protected by list mutex, see below */
+	int			r_refs;		/* # outstanding references */
+	uint8_t			r_delay;	/* delay to use for jukebox error */
+	uint8_t			r_retry;	/* max retransmission count */
+	uint8_t			r_rexmit;	/* current retrans count */
+	int			r_rtt;		/* RTT for rpc */
+	thread_t		r_thread;	/* thread that did I/O system call */
+	kauth_cred_t		r_cred;		/* credential used for request */
+	long			r_start;	/* request start time */
+	long			r_lastmsg;	/* time of last tprintf */
+	long			r_resendtime;	/* time of next jukebox error resend */
+	struct nfs_gss_clnt_ctx	*r_gss_ctx;	/* RPCSEC_GSS context */
+	SLIST_HEAD(, gss_seq)	r_gss_seqlist;	/* RPCSEC_GSS sequence numbers */
+	uint32_t		r_gss_argoff;	/* RPCSEC_GSS offset to args */
+	uint32_t		r_gss_arglen;	/* RPCSEC_GSS arg length */
+	int			r_error;	/* request error */
+	struct nfsreq_cbinfo	r_callback;	/* callback info */
 };
 
 /*
  * Queue head for nfsreq's
  */
-extern TAILQ_HEAD(nfs_reqq, nfsreq) nfs_reqq;
+TAILQ_HEAD(nfs_reqqhead, nfsreq);
+__private_extern__ struct nfs_reqqhead nfs_reqq;
+__private_extern__ lck_grp_t *nfs_request_grp;
+
+#define R_XID32(x)	((x) & 0xffffffff)
+
+#define NFSREQNOLIST ((struct nfsreq *)0xdeadbeef)	/* sentinel value for nfsreq lists */
 
 /* Flag values for r_flags */
 #define R_TIMING	0x0001		/* timing request (in mntp) */
-#define R_SENT		0x0002		/* request has been sent */
-#define R_SOFTTERM	0x0004		/* soft mnt, too many retries */
-#define R_INTR		0x0008		/* intr mnt, signal pending */
-#define R_SOCKERR	0x0010		/* Fatal error on socket */
+#define R_CWND		0x0002		/* request accounted for in congestion window */
+#define R_SOFTTERM	0x0004		/* request terminated (e.g. soft mnt) */
+#define R_RESTART	0x0008		/* RPC should be restarted. */
+#define R_INITTED	0x0010		/* request has been initialized */
 #define R_TPRINTFMSG	0x0020		/* Did a tprintf msg. */
 #define R_MUSTRESEND	0x0040		/* Must resend request */
-#define R_BUSY		0x0100		/* Locked. */
-#define R_WAITING	0x0200		/* Someone waiting for lock. */
-#define R_RESENDERR	0x0400		/* resend failed. */
-
-/*
- * A list of nfssvc_sock structures is maintained with all the sockets
- * that require service by the nfsd.
- * The nfsuid structs hang off of the nfssvc_sock structs in both lru
- * and uid hash lists.
- */
-#ifndef NFS_UIDHASHSIZ
-#define	NFS_UIDHASHSIZ	13	/* Tune the size of nfssvc_sock with this */
-#endif
-#define	NUIDHASH(sock, uid) \
-	(&(sock)->ns_uidhashtbl[(uid) % NFS_UIDHASHSIZ])
-#define	NWDELAYHASH(sock, f) \
-	(&(sock)->ns_wdelayhashtbl[(*((u_long *)(f))) % NFS_WDELAYHASHSIZ])
-#define	NMUIDHASH(nmp, uid) \
-	(&(nmp)->nm_uidhashtbl[(uid) % NFS_MUIDHASHSIZ])
-#define	NFSNOHASH(fhsum) \
-	(&nfsnodehashtbl[(fhsum) & nfsnodehash])
+#define R_ALLOCATED	0x0080		/* request was allocated */
+#define R_SENT		0x0100		/* request has been sent */
+#define R_WAITSENT	0x0200		/* someone is waiting for request to be sent */
+#define R_RESENDERR	0x0400		/* resend failed */
+#define R_JBTPRINTFMSG	0x0800		/* Did a tprintf msg for jukebox error */
+#define R_ASYNC		0x1000		/* async request */
+#define R_ASYNCWAIT	0x2000		/* async request now being waited on */
+#define R_RESENDQ	0x4000		/* async request currently on resendq */
+
+#define R_SETUP		0x8000		/* a setup RPC - during (re)connection */
+#define R_OPTMASK	0x8000		/* mask of all RPC option flags */
+
+/* Flag values for r_lflags */
+#define RL_BUSY		0x0001		/* Locked. */
+#define RL_WAITING	0x0002		/* Someone waiting for lock. */
+#define RL_QUEUED	0x0004		/* request is on the queue */
+
+__private_extern__ u_long nfs_xid, nfs_xidwrap;
+__private_extern__ int nfs_iosize, nfs_access_cache_timeout, nfs_allow_async, nfs_statfs_rate_limit;
+__private_extern__ int nfs_lockd_mounts, nfs_lockd_request_sent;
+__private_extern__ int nfs_tprintf_initial_delay, nfs_tprintf_delay;
+__private_extern__ int nfsiod_thread_count, nfsiod_thread_max, nfs_max_async_writes;
+
+#define NFSIOD_MAX	(MIN(nfsiod_thread_max, NFS_MAXASYNCTHREAD))
+
+struct nfs_dulookup {
+	int du_flags;			/* state of ._ lookup */
+#define NFS_DULOOKUP_DOIT	0x1
+#define NFS_DULOOKUP_INPROG	0x2
+	struct componentname du_cn;	/* ._ name being looked up */
+	struct nfsreq du_req;		/* NFS request for lookup */
+	char du_smallname[48];		/* buffer for small names */
+};
 
 /*
  * Network address hash list element
@@ -680,36 +894,14 @@ union nethostaddr {
 	mbuf_t had_nam;
 };
 
-struct nfsuid {
-	TAILQ_ENTRY(nfsuid) nu_lru;	/* LRU chain */
-	LIST_ENTRY(nfsuid) nu_hash;	/* Hash list */
-	int		nu_flag;	/* Flags */
-	union nethostaddr nu_haddr;	/* Host addr. for dgram sockets */
-	kauth_cred_t	nu_cr;		/* Cred uid mapped to */
-	int		nu_expire;	/* Expiry time (sec) */
-	struct timeval	nu_timestamp;	/* Kerb. timestamp */
-	u_long		nu_nickname;	/* Nickname on server */
-	NFSKERBKEY_T	nu_key;		/* and session key */
-};
-
-#define	nu_inetaddr	nu_haddr.had_inetaddr
-#define	nu_nam		nu_haddr.had_nam
-/* Bits for nu_flag */
-#define	NU_INETADDR	0x1
-#define NU_NAM		0x2
-#define NU_NETFAM(u)	(((u)->nu_flag & NU_INETADDR) ? AF_INET : AF_ISO)
-
-#ifdef notyet
-/* XXX CSM 12/2/97 When/if we merge queue.h */
-struct nfsrv_rec {
-	STAILQ_ENTRY(nfsrv_rec) nr_link;
-	struct sockaddr	*nr_address;
-	mbuf_t		nr_packet;
-};
-#endif
-
-struct nfssvc_sock {
-	TAILQ_ENTRY(nfssvc_sock) ns_chain;	/* List of all nfssvc_sock's */
+/*
+ * One nfsrv_sock structure is maintained for each socket the
+ * server is servicing requests on.
+ */
+struct nfsrv_sock {
+	TAILQ_ENTRY(nfsrv_sock) ns_chain;	/* List of all nfsrv_sock's */
+	TAILQ_ENTRY(nfsrv_sock) ns_svcq;	/* List of sockets needing servicing */
+	TAILQ_ENTRY(nfsrv_sock) ns_wgq;		/* List of sockets with a pending write gather */
 	lck_rw_t	ns_rwlock;		/* lock for most fields */
 	socket_t	ns_so;
 	mbuf_t		ns_nam;
@@ -722,54 +914,62 @@ struct nfssvc_sock {
 	int		ns_sotype;
 	int		ns_cc;
 	int		ns_reclen;
-	int		ns_numuids;
+	int		ns_reccnt;
 	u_long		ns_sref;
 	time_t		ns_timestamp;		/* socket timestamp */
 	lck_mtx_t	ns_wgmutex;		/* mutex for write gather fields */
 	u_quad_t	ns_wgtime;		/* next Write deadline (usec) */
 	LIST_HEAD(, nfsrv_descript) ns_tq;	/* Write gather lists */
-	LIST_HEAD(nfsrvw_delayhash, nfsrv_descript) ns_wdelayhashtbl[NFS_WDELAYHASHSIZ];
-	TAILQ_HEAD(, nfsuid) ns_uidlruhead;
-	LIST_HEAD(, nfsuid) ns_uidhashtbl[NFS_UIDHASHSIZ];
+	LIST_HEAD(nfsrv_wg_delayhash, nfsrv_descript) ns_wdelayhashtbl[NFS_WDELAYHASHSIZ];
 };
 
 /* Bits for "ns_flag" */
-#define	SLP_VALID	0x01 /* nfs sock valid */
-#define	SLP_DOREC	0x02 /* nfs sock has received data to process */
-#define	SLP_NEEDQ	0x04 /* network socket has data to receive */
-#define	SLP_DISCONN	0x08 /* socket needs to be zapped */
-#define	SLP_GETSTREAM	0x10 /* currently in nfsrv_getstream() */
-#define	SLP_LASTFRAG	0x20 /* on last fragment of RPC record */
-#define SLP_ALLFLAGS	0xff
-
-extern TAILQ_HEAD(nfssvc_sockhead, nfssvc_sock) nfssvc_sockhead, nfssvc_deadsockhead;
-
-/* locks for nfssvc_sock's */
-extern lck_grp_attr_t *nfs_slp_group_attr;
-extern lck_attr_t *nfs_slp_lock_attr;
-extern lck_grp_t *nfs_slp_rwlock_group;
-extern lck_grp_t *nfs_slp_mutex_group;
+#define	SLP_VALID	0x0001 /* nfs sock valid */
+#define	SLP_DOREC	0x0002 /* nfs sock has received data to process */
+#define	SLP_NEEDQ	0x0004 /* network socket has data to receive */
+#define	SLP_DISCONN	0x0008 /* socket needs to be zapped */
+#define	SLP_GETSTREAM	0x0010 /* currently in nfsrv_getstream() */
+#define	SLP_LASTFRAG	0x0020 /* on last fragment of RPC record */
+#define	SLP_DOWRITES	0x0040 /* nfs sock has gathered writes to service */
+#define SLP_WORKTODO	0x004e /* mask of all "work to do" flags */
+#define SLP_ALLFLAGS	0x007f
+#define	SLP_WAITQ	0x4000 /* nfs sock is on the wait queue */
+#define	SLP_WORKQ	0x8000 /* nfs sock is on the work queue */
+#define	SLP_QUEUED	0xc000 /* nfs sock is on a queue */
+
+#define SLPNOLIST ((struct nfsrv_sock *)0xdeadbeef)	/* sentinel value for sockets not in the nfsrv_sockwg list */
+
+__private_extern__ struct nfsrv_sock *nfsrv_udpsock;
+
+/*
+ * global NFS server socket lists:
+ *
+ * nfsrv_socklist - list of all sockets (ns_chain)
+ * nfsrv_sockwait - sockets w/new data waiting to be worked on (ns_svcq)
+ * nfsrv_sockwork - sockets being worked on which may have more work to do (ns_svcq)
+ * nfsrv_sockwg - sockets with pending write gather input (ns_wgq)
+ */
+__private_extern__ TAILQ_HEAD(nfsrv_sockhead, nfsrv_sock) nfsrv_socklist, nfsrv_deadsocklist,
+						nfsrv_sockwg, nfsrv_sockwait, nfsrv_sockwork;
+
+/* lock groups for nfsrv_sock's */
+__private_extern__ lck_grp_t *nfsrv_slp_rwlock_group;
+__private_extern__ lck_grp_t *nfsrv_slp_mutex_group;
 
 /*
  * One of these structures is allocated for each nfsd.
  */
 struct nfsd {
-	TAILQ_ENTRY(nfsd) nfsd_chain;	/* List of all nfsd's */
-	int		nfsd_flag;	/* NFSD_ flags */
-	struct nfssvc_sock *nfsd_slp;	/* Current socket */
-	int		nfsd_authlen;	/* Authenticator len */
-	u_char		nfsd_authstr[RPCAUTH_MAXSIZ]; /* Authenticator data */
-	int		nfsd_verflen;	/* and the Verifier */
-	u_char		nfsd_verfstr[RPCVERF_MAXSIZ];
-	proc_t		nfsd_procp;	/* Proc ptr */
-	struct nfsrv_descript *nfsd_nd;	/* Associated nfsrv_descript */
+	TAILQ_ENTRY(nfsd)	nfsd_chain;	/* List of all nfsd's */
+	TAILQ_ENTRY(nfsd)	nfsd_queue;	/* List of waiting nfsd's */
+	int			nfsd_flag;	/* NFSD_ flags */
+	struct nfsrv_sock	*nfsd_slp;	/* Current socket */
+	struct nfsrv_descript	*nfsd_nd;	/* Associated nfsrv_descript */
 };
 
 /* Bits for "nfsd_flag" */
 #define	NFSD_WAITING	0x01
 #define	NFSD_REQINPROG	0x02
-#define	NFSD_NEEDAUTH	0x04
-#define	NFSD_AUTHFAIL	0x08
 
 /*
  * This structure is used by the server for describing each request.
@@ -780,242 +980,220 @@ struct nfsrv_descript {
 	off_t			nd_off;		/* Start byte offset */
 	off_t			nd_eoff;	/* and end byte offset */
 	LIST_ENTRY(nfsrv_descript) nd_hash;	/* Hash list */
-	LIST_ENTRY(nfsrv_descript) nd_tq;		/* and timer list */
+	LIST_ENTRY(nfsrv_descript) nd_tq;	/* and timer list */
 	LIST_HEAD(,nfsrv_descript) nd_coalesce;	/* coalesced writes */
-	mbuf_t			nd_mrep;	/* Request mbuf list */
-	mbuf_t			nd_md;		/* Current dissect mbuf */
-	mbuf_t			nd_mreq;	/* Reply mbuf list */
+	struct nfsm_chain	nd_nmreq;	/* Request mbuf chain */
+	mbuf_t			nd_mrep;	/* Reply mbuf list (WG) */
 	mbuf_t			nd_nam;		/* and socket addr */
 	mbuf_t			nd_nam2;	/* return socket addr */
-	caddr_t			nd_dpos;	/* Current dissect pos */
 	u_int32_t		nd_procnum;	/* RPC # */
 	int			nd_stable;	/* storage type */
-	int			nd_flag;	/* nd_flag */
+	int			nd_vers;	/* NFS version */
 	int			nd_len;		/* Length of this write */
 	int			nd_repstat;	/* Reply status */
 	u_long			nd_retxid;	/* Reply xid */
 	struct timeval		nd_starttime;	/* Time RPC initiated */
 	struct nfs_filehandle	nd_fh;		/* File handle */
+	uint32_t		nd_sec;		/* Security flavor */
+	struct nfs_gss_svc_ctx	*nd_gss_context;/* RPCSEC_GSS context */
+	uint32_t		nd_gss_seqnum;	/* RPCSEC_GSS seq num */
+	mbuf_t			nd_gss_mb;	/* RPCSEC_GSS results mbuf */
 	kauth_cred_t		nd_cr;		/* Credentials */
 };
 
-/* Bits for "nd_flag" */
-#define ND_NFSV3	0x08
-#define ND_KERBNICK	0x20
-#define ND_KERBFULL	0x40
-#define ND_KERBAUTH	(ND_KERBNICK | ND_KERBFULL)
-
-extern TAILQ_HEAD(nfsd_head, nfsd) nfsd_head;
-extern int nfsd_head_flag;
-#define	NFSD_CHECKSLP	0x01
+__private_extern__ TAILQ_HEAD(nfsd_head, nfsd) nfsd_head, nfsd_queue;
 
-/*
- * These macros compare nfsrv_descript structures.
- */
-#define NFSW_CONTIG(o, n) \
-		(((o)->nd_eoff >= (n)->nd_off) && \
-		 ((o)->nd_fh.nfh_len == (n)->nd_fh.nfh_len) && \
-		 !bcmp((caddr_t)&(o)->nd_fh, (caddr_t)&(n)->nd_fh, (o)->nd_fh.nfh_len))
+/* mutex for nfs server */
+__private_extern__ lck_mtx_t *nfsd_mutex;
+__private_extern__ int nfsd_thread_count, nfsd_thread_max;
 
-#define NFSW_SAMECRED(o, n) \
-	(((o)->nd_flag & ND_KERBAUTH) == ((n)->nd_flag & ND_KERBAUTH) && \
- 	 !bcmp((caddr_t)(o)->nd_cr, (caddr_t)(n)->nd_cr, \
-		sizeof (struct ucred)))
+/* request list mutex */
+__private_extern__ lck_mtx_t *nfs_request_mutex;
+__private_extern__ int nfs_request_timer_on;
 
-/* mutex for nfs server */
-extern lck_grp_t * nfsd_lck_grp;
-extern lck_grp_attr_t * nfsd_lck_grp_attr;
-extern lck_attr_t * nfsd_lck_attr;
-extern lck_mtx_t *nfsd_mutex;
+/* nfs timer call structures */
+__private_extern__ thread_call_t	nfs_request_timer_call;
+__private_extern__ thread_call_t	nfs_buf_timer_call;
+__private_extern__ thread_call_t	nfsrv_deadsock_timer_call;
+__private_extern__ thread_call_t	nfsrv_fmod_timer_call;
 
-extern int nfs_numnfsd, nfsd_waiting;
+__BEGIN_DECLS
 
-/*
- * Defines for WebNFS
- */
+nfstype	vtonfs_type(enum vtype, int);
+enum vtype nfstov_type(nfstype, int);
+int	vtonfsv2_mode(enum vtype, mode_t);
 
-#define WEBNFS_ESC_CHAR		'%'
-#define WEBNFS_SPECCHAR_START	0x80
+void	nfs_mbuf_init(void);
 
-#define WEBNFS_NATIVE_CHAR	0x80
-/*
- * ..
- * Possibly more here in the future.
- */
+void	nfs_nhinit(void);
+void	nfs_nhinit_finish(void);
+u_long	nfs_hash(u_char *, int);
 
-/*
- * Macro for converting escape characters in WebNFS pathnames.
- * Should really be in libkern.
- */
-#define ISHEX(c) \
-	((((c) >= 'a') && ((c) <= 'f')) || \
-	 (((c) >= 'A') && ((c) <= 'F')) || \
-	 (((c) >= '0') && ((c) <= '9')))
-#define HEXTOC(c) \
-	((c) >= 'a' ? ((c) - ('a' - 10)) : \
-	    ((c) >= 'A' ? ((c) - ('A' - 10)) : ((c) - '0')))
-#define HEXSTRTOI(p) \
-	((HEXTOC(p[0]) << 4) + HEXTOC(p[1]))
+int	nfs4_setclientid(struct nfsmount *);
+void	nfs4_renew_timer(void *, void *);
+int	nfs_connect(struct nfsmount *);
+void	nfs_disconnect(struct nfsmount *);
+void	nfs_mount_sock_thread_wake(struct nfsmount *);
 
-__BEGIN_DECLS
+int	nfs_getattr(nfsnode_t, struct nfs_vattr *, vfs_context_t, int);
+int	nfs_getattrcache(nfsnode_t, struct nfs_vattr *, int);
+int	nfs_loadattrcache(nfsnode_t, struct nfs_vattr *, u_int64_t *, int);
+int	nfs_attrcachetimeout(nfsnode_t);
 
-int	nfs_init(struct vfsconf *vfsp);
-void	nfs_mbuf_init(void);
-int	nfs_reply(struct nfsreq *);
-int	nfs_getreq(struct nfsrv_descript *,struct nfsd *,int);
-int	nfs_send(socket_t, mbuf_t, mbuf_t, struct nfsreq *);
-int	nfs_rephead(int, struct nfsrv_descript *, struct nfssvc_sock *,
-			 int, mbuf_t *, mbuf_t *, caddr_t *);
+int	nfs_buf_page_inval(vnode_t vp, off_t offset);
+int	nfs_vinvalbuf(vnode_t, int, vfs_context_t, int);
+int	nfs_vinvalbuf2(vnode_t, int, thread_t, kauth_cred_t, int);
+
+int	nfs_request_create(nfsnode_t, mount_t, struct nfsm_chain *, int, thread_t, kauth_cred_t, struct nfsreq **);
+void	nfs_request_destroy(struct nfsreq *);
+void	nfs_request_ref(struct nfsreq *, int);
+void	nfs_request_rele(struct nfsreq *);
+int	nfs_request_add_header(struct nfsreq *);
+int	nfs_request_send(struct nfsreq *, int);
+void	nfs_request_wait(struct nfsreq *);
+int	nfs_request_finish(struct nfsreq *, struct nfsm_chain *, int *);
+int	nfs_request(nfsnode_t, mount_t, struct nfsm_chain *, int, vfs_context_t, struct nfsm_chain *, u_int64_t *, int *);
+int	nfs_request2(nfsnode_t, mount_t, struct nfsm_chain *, int, thread_t, kauth_cred_t, int, struct nfsm_chain *, u_int64_t *, int *);
+int	nfs_request_async(nfsnode_t, mount_t, struct nfsm_chain *, int, thread_t, kauth_cred_t, struct nfsreq_cbinfo *cb, struct nfsreq **);
+int	nfs_request_async_finish(struct nfsreq *, struct nfsm_chain *, u_int64_t *, int *);
+void	nfs_request_async_cancel(struct nfsreq *);
+void	nfs_request_timer(void *, void *);
+int	nfs_sigintr(struct nfsmount *, struct nfsreq *, thread_t, int);
+int	nfs_noremotehang(thread_t);
+
+int	nfs_send(struct nfsreq *, int);
 int	nfs_sndlock(struct nfsreq *);
 void	nfs_sndunlock(struct nfsreq *);
-int	nfs_vinvalbuf(vnode_t, int, struct ucred *, proc_t, int);
-int	nfs_buf_page_inval(vnode_t vp, off_t offset);
-int	nfs_readrpc(vnode_t, struct uio *, struct ucred *, proc_t);
-int	nfs_writerpc(vnode_t, struct uio *, struct ucred *, proc_t, int *, uint64_t *);
-int	nfs_readdirrpc(vnode_t, struct uio *, struct ucred *, proc_t);
-int	nfs_readdirplusrpc(vnode_t, struct uio *, struct ucred *, proc_t);
-int	nfs_asyncio(struct nfsbuf *, struct ucred *);
-int	nfs_doio(struct nfsbuf *, struct ucred *, proc_t);
-int	nfs_readlinkrpc(vnode_t, struct uio *, struct ucred *, proc_t);
-int	nfs_sigintr(struct nfsmount *, struct nfsreq *, proc_t);
-int	nfsm_disct(mbuf_t *, caddr_t *, int, int, caddr_t *);
-void	nfsm_srvfattr(struct nfsrv_descript *, struct vnode_attr *, 
-			   struct nfs_fattr *);
-void	nfsm_srvwcc(struct nfsrv_descript *, int, struct vnode_attr *, int,
-			 struct vnode_attr *, mbuf_t *, char **);
-void	nfsm_srvpostopattr(struct nfsrv_descript *, int, struct vnode_attr *,
-				mbuf_t *, char **);
-int	netaddr_match(int, union nethostaddr *, mbuf_t);
-int	nfs_request(vnode_t, mount_t, mbuf_t, int, proc_t,
-			 struct ucred *, mbuf_t *, mbuf_t *,
-			 caddr_t *, u_int64_t *);
-int	nfs_parsefattr(mbuf_t *, caddr_t *, int, struct nfs_vattr *);
-int	nfs_loadattrcache(struct nfsnode *, struct nfs_vattr *, u_int64_t *, int);
-int	nfsm_path_mbuftond(mbuf_t *, caddr_t *, int, int, int *, struct nameidata *);
-int	nfs_namei(struct nfsrv_descript *, struct vfs_context *, struct nameidata *,
-			struct nfs_filehandle *, mbuf_t, int, vnode_t *,
-			struct nfs_export **, struct nfs_export_options **);
-void	nfsm_adj(mbuf_t, int, int);
-int	nfsm_mbuftouio(mbuf_t *, struct uio *, int, caddr_t *);
-void	nfsrv_initcache(void);
-int	nfs_getauth(struct nfsmount *, struct nfsreq *, struct ucred *, 
-			 char **, int *, char *, int *, NFSKERBKEY_T);
-int	nfs_getnickauth(struct nfsmount *, struct ucred *, char **, 
-			     int *, char *, int);
-int	nfs_savenickauth(struct nfsmount *, struct ucred *, int, 
-			      NFSKERBKEY_T, mbuf_t *, char **,
-			      mbuf_t);
-int	nfs_adv(mbuf_t *, caddr_t *, int, int);
-void	nfs_nhinit(void);
-void	nfs_timer_funnel(void*);
-void	nfs_timer(void*);
-u_long	nfs_hash(u_char *, int);
-int	nfsrv_dorec(struct nfssvc_sock *, struct nfsd *, 
-			 struct nfsrv_descript **);
-int	nfsrv_getcache(struct nfsrv_descript *, struct nfssvc_sock *,
-			    mbuf_t *);
-void	nfsrv_updatecache(struct nfsrv_descript *, int, mbuf_t);
+
+int	nfs_lookitup(nfsnode_t, char *, int, vfs_context_t, nfsnode_t *);
+void	nfs_dulookup_init(struct nfs_dulookup *, nfsnode_t, const char *, int);
+void	nfs_dulookup_start(struct nfs_dulookup *, nfsnode_t, vfs_context_t);
+void	nfs_dulookup_finish(struct nfs_dulookup *, nfsnode_t, vfs_context_t);
+
+int	nfs_parsefattr(struct nfsm_chain *, int, struct nfs_vattr *);
+int	nfs4_parsefattr(struct nfsm_chain *, struct nfs_fsattr *, struct nfs_vattr *, fhandle_t *, struct dqblk *);
+void	nfs_vattr_set_supported(uint32_t *, struct vnode_attr *);
+void	nfs3_pathconf_cache(struct nfsmount *, struct nfs_fsattr *);
+int	nfs_node_mode_slot(nfsnode_t, uid_t, int);
+
+int	nfs_vnop_access(struct vnop_access_args *);
+
+int	nfs3_vnop_open(struct vnop_open_args *);
+int	nfs3_vnop_close(struct vnop_close_args *);
+
+int	nfs4_vnop_create(struct vnop_create_args *);
+int	nfs4_vnop_mknod(struct vnop_mknod_args *);
+int	nfs4_vnop_open(struct vnop_open_args *);
+int	nfs4_vnop_close(struct vnop_close_args *);
+int	nfs4_vnop_getattr(struct vnop_getattr_args *);
+int	nfs4_vnop_link(struct vnop_link_args *);
+int	nfs4_vnop_mkdir(struct vnop_mkdir_args *);
+int	nfs4_vnop_rmdir(struct vnop_rmdir_args *);
+int	nfs4_vnop_symlink(struct vnop_symlink_args *);
+int	nfs4_vnop_advlock(struct vnop_advlock_args *ap);
+
+int	nfs_read_rpc(nfsnode_t, struct uio *, vfs_context_t);
+int	nfs_write_rpc(nfsnode_t, struct uio *, vfs_context_t, int *, uint64_t *);
+int	nfs_write_rpc2(nfsnode_t, struct uio *, thread_t, kauth_cred_t, int *, uint64_t *);
+
+int	nfs3_access_rpc(nfsnode_t, u_long *, vfs_context_t);
+int	nfs4_access_rpc(nfsnode_t, u_long *, vfs_context_t);
+int	nfs3_getattr_rpc(nfsnode_t, mount_t, u_char *, size_t, vfs_context_t, struct nfs_vattr *, u_int64_t *);
+int	nfs4_getattr_rpc(nfsnode_t, mount_t, u_char *, size_t, vfs_context_t, struct nfs_vattr *, u_int64_t *);
+int	nfs3_setattr_rpc(nfsnode_t, struct vnode_attr *, vfs_context_t, int);
+int	nfs4_setattr_rpc(nfsnode_t, struct vnode_attr *, vfs_context_t, int);
+int	nfs3_read_rpc_async(nfsnode_t, off_t, size_t, thread_t, kauth_cred_t, struct nfsreq_cbinfo *, struct nfsreq **);
+int	nfs4_read_rpc_async(nfsnode_t, off_t, size_t, thread_t, kauth_cred_t, struct nfsreq_cbinfo *, struct nfsreq **);
+int	nfs3_read_rpc_async_finish(nfsnode_t, struct nfsreq *, struct uio *, size_t *, int *);
+int	nfs4_read_rpc_async_finish(nfsnode_t, struct nfsreq *, struct uio *, size_t *, int *);
+int	nfs3_write_rpc_async(nfsnode_t, struct uio *, size_t, thread_t, kauth_cred_t, int, struct nfsreq_cbinfo *, struct nfsreq **);
+int	nfs4_write_rpc_async(nfsnode_t, struct uio *, size_t, thread_t, kauth_cred_t, int, struct nfsreq_cbinfo *, struct nfsreq **);
+int	nfs3_write_rpc_async_finish(nfsnode_t, struct nfsreq *, int *, size_t *, uint64_t *);
+int	nfs4_write_rpc_async_finish(nfsnode_t, struct nfsreq *, int *, size_t *, uint64_t *);
+int	nfs3_readdir_rpc(nfsnode_t, struct uio *, vfs_context_t);
+int	nfs3_readdirplus_rpc(nfsnode_t, struct uio *, vfs_context_t);
+int	nfs4_readdir_rpc(nfsnode_t, struct uio *, vfs_context_t);
+int	nfs3_readlink_rpc(nfsnode_t, char *, uint32_t *, vfs_context_t);
+int	nfs4_readlink_rpc(nfsnode_t, char *, uint32_t *, vfs_context_t);
+int	nfs3_commit_rpc(nfsnode_t, u_int64_t, u_int64_t, kauth_cred_t);
+int	nfs4_commit_rpc(nfsnode_t, u_int64_t, u_int64_t, kauth_cred_t);
+int	nfs3_lookup_rpc_async(nfsnode_t, char *, int, vfs_context_t, struct nfsreq **);
+int	nfs4_lookup_rpc_async(nfsnode_t, char *, int, vfs_context_t, struct nfsreq **);
+int	nfs3_lookup_rpc_async_finish(nfsnode_t, vfs_context_t, struct nfsreq *, u_int64_t *, fhandle_t *, struct nfs_vattr *);
+int	nfs4_lookup_rpc_async_finish(nfsnode_t, vfs_context_t, struct nfsreq *, u_int64_t *, fhandle_t *, struct nfs_vattr *);
+int	nfs3_remove_rpc(nfsnode_t, char *, int, thread_t, kauth_cred_t);
+int	nfs4_remove_rpc(nfsnode_t, char *, int, thread_t, kauth_cred_t);
+int	nfs3_rename_rpc(nfsnode_t, char *, int, nfsnode_t, char *, int, vfs_context_t);
+int	nfs4_rename_rpc(nfsnode_t, char *, int, nfsnode_t, char *, int, vfs_context_t);
+int	nfs3_pathconf_rpc(nfsnode_t, struct nfs_fsattr *, vfs_context_t);
+int	nfs4_pathconf_rpc(nfsnode_t, struct nfs_fsattr *, vfs_context_t);
+
+void	nfsrv_active_user_list_reclaim(void);
 void	nfsrv_cleancache(void);
-int	nfs_bind_resv_thread_wake(void);
-int	nfs_connect(struct nfsmount *, struct nfsreq *);
-void	nfs_disconnect(struct nfsmount *);
-int	nfs_getattr_no_vnode(mount_t,u_char *,int,struct ucred *,proc_t,struct nfs_vattr *,u_int64_t *);
-int	nfs_getattr(vnode_t vp, struct nfs_vattr *nvap, struct ucred *cred, proc_t p);
-int	nfs_getattrcache(vnode_t, struct nfs_vattr *);
-int	nfs_attrcachetimeout(vnode_t);
-int	nfsm_strtmbuf(mbuf_t *, char **, char *, long);
-int	nfs_bioread(vnode_t, struct uio *, int, struct ucred *, proc_t);
-int	nfsm_uiotombuf(struct uio *, mbuf_t *, int, caddr_t *);
-void	nfsrv_init(int);
-int	nfs_commit(vnode_t vp, u_quad_t offset, u_int32_t count,
-			struct ucred *cred, proc_t procp);
-int	nfs_flushcommits(vnode_t, proc_t, int);
-int	nfs_flush(vnode_t,int,struct ucred *,proc_t,int);
-int	nfsrv_errmap(struct nfsrv_descript *, int);
-void	nfsrvw_sort(gid_t *, int);
-void	nfsrv_setcred(struct ucred *, struct ucred *);
-int	nfs_buf_write(struct nfsbuf *);
-void	nfsrv_wakenfsd(struct nfssvc_sock *slp);
-int	nfsrv_writegather(struct nfsrv_descript **, struct nfssvc_sock *,
-			       proc_t, mbuf_t *);
-int	nfs_fsinfo(struct nfsmount *, vnode_t, struct ucred *, proc_t p);
-int	nfs_pathconfrpc(vnode_t, struct nfsv3_pathconf *, kauth_cred_t, proc_t);
-void	nfs_pathconf_cache(struct nfsmount *, struct nfsv3_pathconf *);
-
-int	nfsrv3_access(struct nfsrv_descript *nfsd, 
-			   struct nfssvc_sock *slp,
-			   proc_t procp, mbuf_t *mrq);
-int	nfsrv_commit(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_create(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_credcheck(struct nfsrv_descript *, struct nfs_export *,
+void	nfsrv_cleanup(void);
+int	nfsrv_credcheck(struct nfsrv_descript *, vfs_context_t, struct nfs_export *,
 			struct nfs_export_options *);
-int	nfsrv_export(struct user_nfs_export_args *, struct vfs_context *);
-int	nfsrv_fhmatch(struct nfs_filehandle *fh1, struct nfs_filehandle *fh2);
-int	nfsrv_fhtovp(struct nfs_filehandle *, mbuf_t, int, vnode_t *,
+void	nfsrv_deadsock_timer(void *, void *);
+int	nfsrv_dorec(struct nfsrv_sock *, struct nfsd *, struct nfsrv_descript **);
+int	nfsrv_errmap(struct nfsrv_descript *, int);
+int	nfsrv_export(struct user_nfs_export_args *, vfs_context_t);
+int	nfsrv_fhmatch(struct nfs_filehandle *, struct nfs_filehandle *);
+int	nfsrv_fhtovp(struct nfs_filehandle *, struct nfsrv_descript *, vnode_t *,
 			struct nfs_export **, struct nfs_export_options **);
-int	nfs_ispublicfh(struct nfs_filehandle *);
-int	nfsrv_fsinfo(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_getattr(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			   proc_t procp, mbuf_t *mrq);
-int	nfsrv_link(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			proc_t procp, mbuf_t *mrq);
-int	nfsrv_lookup(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_mkdir(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			 proc_t procp, mbuf_t *mrq);
-int	nfsrv_mknod(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			 proc_t procp, mbuf_t *mrq);
-int	nfsrv_noop(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			proc_t procp, mbuf_t *mrq);
-int	nfsrv_null(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			proc_t procp, mbuf_t *mrq);
-int	nfsrv_pathconf(struct nfsrv_descript *nfsd,
-			    struct nfssvc_sock *slp, proc_t procp,
-			    mbuf_t *mrq);
-void	nfsrv_rcv(socket_t, caddr_t arg, int waitflag);
-void	nfsrv_rcv_locked(socket_t, struct nfssvc_sock *slp, int waitflag);
-int	nfsrv_read(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			proc_t procp, mbuf_t *mrq);
-int	nfsrv_readdir(struct nfsrv_descript *nfsd, 
-			   struct nfssvc_sock *slp,
-			   proc_t procp, mbuf_t *mrq);
-int	nfsrv_readdirplus(struct nfsrv_descript *nfsd,
-			       struct nfssvc_sock *slp, proc_t procp,
-			       mbuf_t *mrq);
-int	nfsrv_readlink(struct nfsrv_descript *nfsd,
-			    struct nfssvc_sock *slp, proc_t procp,
-			    mbuf_t *mrq);
-int	nfsrv_remove(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_rename(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_rmdir(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			 proc_t procp, mbuf_t *mrq);
-int	nfsrv_setattr(struct nfsrv_descript *nfsd, 
-			   struct nfssvc_sock *slp,
-			   proc_t procp, mbuf_t *mrq);
-void	nfsrv_slpderef(struct nfssvc_sock *slp);
-void	nfsrv_slpfree(struct nfssvc_sock *slp);
-int	nfsrv_statfs(struct nfsrv_descript *nfsd, 
-			  struct nfssvc_sock *slp,
-			  proc_t procp, mbuf_t *mrq);
-int	nfsrv_symlink(struct nfsrv_descript *nfsd, 
-			   struct nfssvc_sock *slp,
-			   proc_t procp, mbuf_t *mrq);
-int	nfsrv_write(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
-			 proc_t procp, mbuf_t *mrq);
-int	nfsrv_vptofh( struct nfs_export *, int, struct nfs_filehandle *,
-			vnode_t, struct vfs_context *, struct nfs_filehandle *);
-
-void	nfs_up(struct nfsmount *, proc_t, int, const char *);
-void	nfs_down(struct nfsmount *, proc_t, int, int, const char *);
-
+void	nfsrv_fmod_timer(void *, void *);
+int	nfsrv_getcache(struct nfsrv_descript *, struct nfsrv_sock *, mbuf_t *);
+void	nfsrv_group_sort(gid_t *, int);
+void	nfsrv_init(void);
+void	nfsrv_initcache(void);
+int	nfsrv_is_initialized(void);
+int	nfsrv_namei(struct nfsrv_descript *, vfs_context_t, struct nameidata *,
+			struct nfs_filehandle *, vnode_t *,
+			struct nfs_export **, struct nfs_export_options **);
+void	nfsrv_rcv(socket_t, caddr_t, int);
+void	nfsrv_rcv_locked(socket_t, struct nfsrv_sock *, int);
+int	nfsrv_rephead(struct nfsrv_descript *, struct nfsrv_sock *, struct nfsm_chain *, size_t);
+int	nfsrv_send(struct nfsrv_sock *, mbuf_t, mbuf_t);
+void	nfsrv_updatecache(struct nfsrv_descript *, int, mbuf_t);
+void	nfsrv_update_user_stat(struct nfs_export *, struct nfsrv_descript *, uid_t, u_int, u_int, u_int);
+int	nfsrv_vptofh(struct nfs_export *, int, struct nfs_filehandle *,
+			vnode_t, vfs_context_t, struct nfs_filehandle *);
+void	nfsrv_wakenfsd(struct nfsrv_sock *);
+void	nfsrv_wg_timer(void *, void *);
+int	nfsrv_writegather(struct nfsrv_descript **, struct nfsrv_sock *,
+			       vfs_context_t, mbuf_t *);
+
+int	nfsrv_access(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_commit(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_create(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_fsinfo(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_getattr(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_link(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_lookup(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_mkdir(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_mknod(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_noop(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_null(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_pathconf(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_read(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_readdir(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_readdirplus(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_readlink(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_remove(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_rename(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_rmdir(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_setattr(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_statfs(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_symlink(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+int	nfsrv_write(struct nfsrv_descript *, struct nfsrv_sock *, vfs_context_t, mbuf_t *);
+
+void	nfs_interval_timer_start(thread_call_t, int);
+void	nfs_up(struct nfsmount *, thread_t, int, const char *);
+void	nfs_down(struct nfsmount *, thread_t, int, int, const char *);
+
+int	nfs_mountroot(void);
 struct nfs_diskless;
-int	nfs_boot_init(struct nfs_diskless *nd, proc_t procp);
-int	nfs_boot_getfh(struct nfs_diskless *nd, proc_t procp, int v3, int sotype);
+int	nfs_boot_init(struct nfs_diskless *);
+int	nfs_boot_getfh(struct nfs_diskless *, int, int);
 
 __END_DECLS
 
diff --git a/bsd/nfs/nfs4_subs.c b/bsd/nfs/nfs4_subs.c
new file mode 100644
index 000000000..977748800
--- /dev/null
+++ b/bsd/nfs/nfs4_subs.c
@@ -0,0 +1,797 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * miscellaneous support functions for NFSv4
+ */
+#include <sys/param.h>
+#include <sys/proc.h>
+#include <sys/kauth.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/mount_internal.h>
+#include <sys/vnode_internal.h>
+#include <sys/kpi_mbuf.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/malloc.h>
+#include <sys/syscall.h>
+#include <sys/ubc_internal.h>
+#include <sys/fcntl.h>
+#include <sys/quota.h>
+#include <sys/uio_internal.h>
+#include <sys/domain.h>
+#include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
+
+#include <sys/vm.h>
+#include <sys/vmparam.h>
+
+#include <sys/time.h>
+#include <kern/clock.h>
+
+#include <nfs/rpcv2.h>
+#include <nfs/nfsproto.h>
+#include <nfs/nfs.h>
+#include <nfs/nfsnode.h>
+#include <nfs/xdr_subs.h>
+#include <nfs/nfsm_subs.h>
+#include <nfs/nfs_gss.h>
+#include <nfs/nfsmount.h>
+#include <nfs/nfs_lock.h>
+
+#include <miscfs/specfs/specdev.h>
+
+#include <netinet/in.h>
+#include <net/kpi_interface.h>
+
+
+/*
+ * NFSv4 SETCLIENTID
+ */
+int
+nfs4_setclientid(struct nfsmount *nmp)
+{
+	struct sockaddr *saddr;
+	uint64_t verifier;
+	char id[128];
+	int idlen, len, error = 0, status, numops;
+	u_int64_t xid;
+	vfs_context_t ctx;
+	thread_t thd;
+	kauth_cred_t cred;
+	struct nfsm_chain nmreq, nmrep;
+
+	static uint8_t en0addr[6];
+	static uint8_t en0addr_set = 0;
+
+	lck_mtx_lock(nfs_request_mutex);
+	if (!en0addr_set) {
+		ifnet_t interface = NULL;
+		error = ifnet_find_by_name("en0", &interface);
+		if (!error)
+			error = ifnet_lladdr_copy_bytes(interface, en0addr, sizeof(en0addr));
+		if (error)
+			printf("nfs4_setclientid: error getting en0 address, %d\n", error);
+		if (!error)
+			en0addr_set = 1;
+		error = 0;
+		if (interface)
+			ifnet_release(interface);
+	}
+	lck_mtx_unlock(nfs_request_mutex);
+
+	ctx = vfs_context_kernel(); /* XXX */
+	thd = vfs_context_thread(ctx);
+	cred = vfs_context_ucred(ctx);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	/* ID: en0_address + server_address */
+	idlen = len = sizeof(en0addr);
+	bcopy(en0addr, &id[0], len);
+	saddr = mbuf_data(nmp->nm_nam);
+	len = min(saddr->sa_len, sizeof(id)-idlen);
+	bcopy(saddr, &id[idlen], len);
+	idlen += len;
+
+	// SETCLIENTID
+	numops = 1;
+	nfsm_chain_build_alloc_init(error, &nmreq, 14 * NFSX_UNSIGNED + idlen);
+	nfsm_chain_add_compound_header(error, &nmreq, "setclientid", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SETCLIENTID);
+	/* nfs_client_id4  client; */
+	nfsm_chain_add_64(error, &nmreq, nmp->nm_mounttime);
+	nfsm_chain_add_32(error, &nmreq, idlen);
+	nfsm_chain_add_opaque(error, &nmreq, id, idlen);
+	/* cb_client4      callback; */
+	/* We don't provide callback info yet */
+	nfsm_chain_add_32(error, &nmreq, 0); /* callback program */
+	nfsm_chain_add_string(error, &nmreq, "", 0); /* callback r_netid */
+	nfsm_chain_add_string(error, &nmreq, "", 0); /* callback r_addr */
+	nfsm_chain_add_32(error, &nmreq, 0); /* callback_ident */
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request2(NULL, nmp->nm_mountp, &nmreq, NFSPROC4_COMPOUND, thd, cred, R_SETUP, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SETCLIENTID);
+	if (error == NFSERR_CLID_INUSE)
+		printf("nfs4_setclientid: client ID in use?\n");
+	nfsmout_if(error);
+	nfsm_chain_get_64(error, &nmrep, nmp->nm_clientid);
+	nfsm_chain_get_64(error, &nmrep, verifier);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	// SETCLIENTID_CONFIRM
+	numops = 1;
+	nfsm_chain_build_alloc_init(error, &nmreq, 13 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "setclientid_confirm", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SETCLIENTID_CONFIRM);
+	nfsm_chain_add_64(error, &nmreq, nmp->nm_clientid);
+	nfsm_chain_add_64(error, &nmreq, verifier);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request2(NULL, nmp->nm_mountp, &nmreq, NFSPROC4_COMPOUND, thd, cred, R_SETUP, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SETCLIENTID_CONFIRM);
+	if (error)
+		printf("nfs4_setclientid: confirm error %d\n", error);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	if (error)
+		printf("nfs4_setclientid failed, %d\n", error);
+	return (error);
+}
+
+/*
+ * periodic timer to renew lease state on server
+ */
+void
+nfs4_renew_timer(void *param0, __unused void *param1)
+{
+	struct nfsmount *nmp = param0;
+	int error = 0, status, numops, interval;
+	u_int64_t xid;
+	vfs_context_t ctx;
+	struct nfsm_chain nmreq, nmrep;
+
+	ctx = vfs_context_kernel(); /* XXX */
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// RENEW
+	numops = 1;
+	nfsm_chain_build_alloc_init(error, &nmreq, 8 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "renew", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RENEW);
+	nfsm_chain_add_64(error, &nmreq, nmp->nm_clientid);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(NULL, nmp->nm_mountp, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RENEW);
+nfsmout:
+	if (error)
+		printf("nfs4_renew_timer: error %d\n", error);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	interval = nmp->nm_fsattr.nfsa_lease / (error ? 4 : 2);
+	if (interval < 1)
+		interval = 1;
+	nfs_interval_timer_start(nmp->nm_renew_timer, interval * 1000);
+}
+
+/*
+ * Set a vnode attr's supported bits according to the given bitmap
+ */
+void
+nfs_vattr_set_supported(uint32_t *bitmap, struct vnode_attr *vap)
+{
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TYPE))
+		VATTR_SET_SUPPORTED(vap, va_type);
+	// if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CHANGE))
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SIZE))
+		VATTR_SET_SUPPORTED(vap, va_data_size);
+	// if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_NAMED_ATTR))
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FSID))
+		VATTR_SET_SUPPORTED(vap, va_fsid);
+//	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ACL))
+//		VATTR_SET_SUPPORTED(vap, va_acl);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ARCHIVE))
+		VATTR_SET_SUPPORTED(vap, va_flags);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILEID))
+		VATTR_SET_SUPPORTED(vap, va_fileid);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_HIDDEN))
+		VATTR_SET_SUPPORTED(vap, va_flags);
+	// if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MIMETYPE))
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MODE))
+		VATTR_SET_SUPPORTED(vap, va_mode);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_NUMLINKS))
+		VATTR_SET_SUPPORTED(vap, va_nlink);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER))
+		VATTR_SET_SUPPORTED(vap, va_uid);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER_GROUP))
+		VATTR_SET_SUPPORTED(vap, va_gid);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_RAWDEV))
+		VATTR_SET_SUPPORTED(vap, va_rdev);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SPACE_USED))
+		VATTR_SET_SUPPORTED(vap, va_total_alloc);
+	// if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SYSTEM))
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_ACCESS))
+		VATTR_SET_SUPPORTED(vap, va_access_time);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_BACKUP))
+		VATTR_SET_SUPPORTED(vap, va_backup_time);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_CREATE))
+		VATTR_SET_SUPPORTED(vap, va_create_time);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_METADATA))
+		VATTR_SET_SUPPORTED(vap, va_change_time);
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_MODIFY))
+		VATTR_SET_SUPPORTED(vap, va_modify_time);
+}
+
+/*
+ * Parse the attributes that are in the mbuf list and store them in
+ * the given structures.
+ */
+int
+nfs4_parsefattr(
+	struct nfsm_chain *nmc,
+	struct nfs_fsattr *nfsap,
+	struct nfs_vattr *nvap,
+	fhandle_t *fhp,
+	struct dqblk *dqbp)
+{
+	int error = 0, attrbytes;
+	uint32_t val, val2, val3, i, j;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN], len;
+	char *s;
+	struct nfs_fsattr nfsa_dummy;
+	struct nfs_vattr nva_dummy;
+	struct dqblk dqb_dummy;
+
+	/* if not interested in some values... throw 'em into a local dummy variable */
+	if (!nfsap)
+		nfsap = &nfsa_dummy;
+	if (!nvap)
+		nvap = &nva_dummy;
+	if (!dqbp)
+		dqbp = &dqb_dummy;
+
+	attrbytes = val = val2 = val3 = 0;
+
+	len = NFS_ATTR_BITMAP_LEN;
+	nfsm_chain_get_bitmap(error, nmc, bitmap, len);
+	/* add bits to object/fs attr bitmaps */
+	for (i=0; i < NFS_ATTR_BITMAP_LEN; i++) {
+		nvap->nva_bitmap[i] |= bitmap[i] & nfs_object_attr_bitmap[i];
+		nfsap->nfsa_bitmap[i] |= bitmap[i] & nfs_fs_attr_bitmap[i];
+	}
+
+	nfsm_chain_get_32(error, nmc, attrbytes);
+	nfsmout_if(error);
+
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SUPPORTED_ATTRS)) {
+		len = NFS_ATTR_BITMAP_LEN;
+		nfsm_chain_get_bitmap(error, nmc, nfsap->nfsa_supp_attr, len);
+		attrbytes -= (len + 1) * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TYPE)) {
+		nfsm_chain_get_32(error, nmc, val);
+		nvap->nva_type = nfstov_type(val, NFS_VER4);
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FH_EXPIRE_TYPE)) {
+		nfsm_chain_get_32(error, nmc, val);
+		nfsmout_if(error);
+		if (val != NFS_FH_PERSISTENT)
+			printf("nfs: warning: non-persistent file handles!\n");
+		if (val & ~0xff)
+			printf("nfs: warning unknown fh type: 0x%x\n", val);
+		nfsap->nfsa_flags &= ~NFS_FSFLAG_FHTYPE_MASK;
+		nfsap->nfsa_flags |= val << 24;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CHANGE)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_change);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SIZE)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_size);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_LINK_SUPPORT)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_LINK;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_LINK;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SYMLINK_SUPPORT)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_SYMLINK;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_SYMLINK;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_NAMED_ATTR)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nvap->nva_flags |= NFS_FFLAG_NAMED_ATTR;
+		else
+			nvap->nva_flags &= ~NFS_FFLAG_NAMED_ATTR;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FSID)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_fsid.major);
+		nfsm_chain_get_64(error, nmc, nvap->nva_fsid.minor);
+		attrbytes -= 4 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_UNIQUE_HANDLES)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_UNIQUE_FH;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_UNIQUE_FH;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_LEASE_TIME)) {
+		nfsm_chain_get_32(error, nmc, nfsap->nfsa_lease);
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_RDATTR_ERROR)) {
+		nfsm_chain_get_32(error, nmc, error);
+		attrbytes -= NFSX_UNSIGNED;
+		nfsmout_if(error);
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ACL)) { /* skip for now */
+		nfsm_chain_get_32(error, nmc, val); /* ACE count */
+		for (i=0; !error && (i < val); i++) {
+			nfsm_chain_adv(error, nmc, 3 * NFSX_UNSIGNED);
+			nfsm_chain_get_32(error, nmc, val2); /* string length */
+			nfsm_chain_adv(error, nmc, nfsm_rndup(val2));
+			attrbytes -= 4*NFSX_UNSIGNED + nfsm_rndup(val2);
+			nfsm_assert(error, (attrbytes >= 0), EBADRPC);
+		}
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ACLSUPPORT)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_ACL;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_ACL;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ARCHIVE)) { /* SF_ARCHIVED */
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nvap->nva_flags |= NFS_FFLAG_ARCHIVED;
+		else
+			nvap->nva_flags &= ~NFS_FFLAG_ARCHIVED;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CANSETTIME)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_SET_TIME;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_SET_TIME;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CASE_INSENSITIVE)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_CASE_INSENSITIVE;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_CASE_INSENSITIVE;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CASE_PRESERVING)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_CASE_PRESERVING;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_CASE_PRESERVING;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_CHOWN_RESTRICTED)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_CHOWN_RESTRICTED;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_CHOWN_RESTRICTED;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILEHANDLE)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (fhp) {
+			fhp->fh_len = val;
+			nfsm_chain_get_opaque(error, nmc, nfsm_rndup(val), fhp->fh_data);
+		} else {
+			nfsm_chain_adv(error, nmc, nfsm_rndup(val));
+		}
+		attrbytes -= NFSX_UNSIGNED + nfsm_rndup(val);
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILEID)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_fileid);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILES_AVAIL)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_files_avail);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILES_FREE)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_files_free);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FILES_TOTAL)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_files_total);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_FS_LOCATIONS)) { /* skip for now */
+		nfsm_chain_get_32(error, nmc, val); /* root path length */
+		nfsm_chain_adv(error, nmc, nfsm_rndup(val)); /* root path */
+		attrbytes -= (2 * NFSX_UNSIGNED) + nfsm_rndup(val);
+		nfsm_chain_get_32(error, nmc, val); /* location count */
+		for (i=0; !error && (i < val); i++) {
+			nfsm_chain_get_32(error, nmc, val2); /* server string length */
+			nfsm_chain_adv(error, nmc, nfsm_rndup(val2)); /* server string */
+			attrbytes -= (2 * NFSX_UNSIGNED) + nfsm_rndup(val2);
+			nfsm_chain_get_32(error, nmc, val2); /* pathname component count */
+			for (j=0; !error && (j < val2); j++) {
+				nfsm_chain_get_32(error, nmc, val3); /* component length */
+				nfsm_chain_adv(error, nmc, nfsm_rndup(val3)); /* component */
+				attrbytes -= NFSX_UNSIGNED + nfsm_rndup(val3);
+				nfsm_assert(error, (attrbytes >= 0), EBADRPC);
+			}
+			nfsm_assert(error, (attrbytes >= 0), EBADRPC);
+		}
+		nfsm_assert(error, (attrbytes >= 0), EBADRPC);
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_HIDDEN)) { /* UF_HIDDEN */
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nvap->nva_flags |= NFS_FFLAG_HIDDEN;
+		else
+			nvap->nva_flags &= ~NFS_FFLAG_HIDDEN;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_HOMOGENEOUS)) {
+		/* XXX If NOT homogeneous, we may need to clear flags on the mount */
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_HOMOGENEOUS;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_HOMOGENEOUS;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MAXFILESIZE)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_maxfilesize);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MAXLINK)) {
+		nfsm_chain_get_32(error, nmc, nvap->nva_maxlink);
+		if (!error && (nfsap->nfsa_maxlink > INT32_MAX))
+			nfsap->nfsa_maxlink = INT32_MAX;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MAXNAME)) {
+		nfsm_chain_get_32(error, nmc, nfsap->nfsa_maxname);
+		if (!error && (nfsap->nfsa_maxname > INT32_MAX))
+			nfsap->nfsa_maxname = INT32_MAX;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MAXREAD)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_maxread);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MAXWRITE)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_maxwrite);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MIMETYPE)) {
+		nfsm_chain_get_32(error, nmc, val);
+		nfsm_chain_adv(error, nmc, nfsm_rndup(val));
+		attrbytes -= NFSX_UNSIGNED + nfsm_rndup(val);
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MODE)) {
+		nfsm_chain_get_32(error, nmc, nvap->nva_mode);
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_NO_TRUNC)) {
+		nfsm_chain_get_32(error, nmc, val);
+		if (val)
+			nfsap->nfsa_flags |= NFS_FSFLAG_NO_TRUNC;
+		else
+			nfsap->nfsa_flags &= ~NFS_FSFLAG_NO_TRUNC;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_NUMLINKS)) {
+		nfsm_chain_get_32(error, nmc, val);
+		nvap->nva_nlink = val;
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER)) { /* XXX ugly hack for now */
+		nfsm_chain_get_32(error, nmc, len);
+		nfsm_chain_get_opaque_pointer(error, nmc, len, s);
+		attrbytes -= NFSX_UNSIGNED + nfsm_rndup(len);
+		nfsmout_if(error);
+		if ((*s >= '0') && (*s <= '9'))
+			nvap->nva_uid = strtol(s, NULL, 10);
+		else if (!strncmp(s, "nobody@", 7))
+			nvap->nva_uid = -2;
+		else if (!strncmp(s, "root@", 5))
+			nvap->nva_uid = 0;
+		else
+			nvap->nva_uid = 99; /* unknown */
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER_GROUP)) { /* XXX ugly hack for now */
+		nfsm_chain_get_32(error, nmc, len);
+		nfsm_chain_get_opaque_pointer(error, nmc, len, s);
+		attrbytes -= NFSX_UNSIGNED + nfsm_rndup(len);
+		nfsmout_if(error);
+		if ((*s >= '0') && (*s <= '9'))
+			nvap->nva_gid = strtol(s, NULL, 10);
+		else if (!strncmp(s, "nobody@", 7))
+			nvap->nva_gid = -2;
+		else if (!strncmp(s, "root@", 5))
+			nvap->nva_uid = 0;
+		else
+			nvap->nva_gid = 99; /* unknown */
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_QUOTA_AVAIL_HARD)) {
+		nfsm_chain_get_64(error, nmc, dqbp->dqb_bhardlimit);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_QUOTA_AVAIL_SOFT)) {
+		nfsm_chain_get_64(error, nmc, dqbp->dqb_bsoftlimit);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_QUOTA_USED)) {
+		nfsm_chain_get_64(error, nmc, dqbp->dqb_curbytes);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_RAWDEV)) {
+		nfsm_chain_get_32(error, nmc, nvap->nva_rawdev.specdata1);
+		nfsm_chain_get_32(error, nmc, nvap->nva_rawdev.specdata2);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SPACE_AVAIL)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_space_avail);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SPACE_FREE)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_space_free);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SPACE_TOTAL)) {
+		nfsm_chain_get_64(error, nmc, nfsap->nfsa_space_total);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SPACE_USED)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_bytes);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SYSTEM)) {
+		/* we'd support this if we had a flag to map it to... */
+		nfsm_chain_adv(error, nmc, NFSX_UNSIGNED);
+		attrbytes -= NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_ACCESS)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_timesec[NFSTIME_ACCESS]);
+		nfsm_chain_get_32(error, nmc, nvap->nva_timensec[NFSTIME_ACCESS]);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_ACCESS_SET)) {
+		nfsm_chain_adv(error, nmc, 4*NFSX_UNSIGNED); /* just skip it */
+		attrbytes -= 4 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_BACKUP)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_timesec[NFSTIME_BACKUP]);
+		nfsm_chain_get_32(error, nmc, nvap->nva_timensec[NFSTIME_BACKUP]);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_CREATE)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_timesec[NFSTIME_CREATE]);
+		nfsm_chain_get_32(error, nmc, nvap->nva_timensec[NFSTIME_CREATE]);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_DELTA)) { /* skip for now */
+		nfsm_chain_adv(error, nmc, 3*NFSX_UNSIGNED);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_METADATA)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_timesec[NFSTIME_CHANGE]);
+		nfsm_chain_get_32(error, nmc, nvap->nva_timensec[NFSTIME_CHANGE]);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_MODIFY)) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_timesec[NFSTIME_MODIFY]);
+		nfsm_chain_get_32(error, nmc, nvap->nva_timensec[NFSTIME_MODIFY]);
+		attrbytes -= 3 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_MODIFY_SET)) {
+		nfsm_chain_adv(error, nmc, 4*NFSX_UNSIGNED); /* just skip it */
+		attrbytes -= 4 * NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MOUNTED_ON_FILEID)) { /* skip for now */
+		nfsm_chain_adv(error, nmc, 2*NFSX_UNSIGNED);
+		attrbytes -= 2 * NFSX_UNSIGNED;
+	}
+	/* advance over any leftover attrbytes */
+	nfsm_assert(error, (attrbytes >= 0), EBADRPC);
+	nfsm_chain_adv(error, nmc, nfsm_rndup(attrbytes));
+nfsmout:
+	return (error);
+}
+
+/*
+ * Add an NFSv4 "sattr" structure to an mbuf chain
+ */
+int
+nfsm_chain_add_fattr4_f(struct nfsm_chain *nmc, struct vnode_attr *vap, struct nfsmount *nmp)
+{
+	int error = 0, attrbytes, slen, i;
+	uint32_t *pattrbytes;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN];
+	char s[32];
+
+	/*
+	 * Do this in two passes.
+	 * First calculate the bitmap, then pack
+	 * everything together and set the size.
+	 */
+
+	NFS_CLEAR_ATTRIBUTES(bitmap);
+	if (VATTR_IS_ACTIVE(vap, va_data_size))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_SIZE);
+	if (VATTR_IS_ACTIVE(vap, va_acl)) {
+		// NFS_BITMAP_SET(bitmap, NFS_FATTR_ACL)
+	}
+	if (VATTR_IS_ACTIVE(vap, va_flags)) {
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_ARCHIVE);
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_HIDDEN);
+	}
+	// NFS_BITMAP_SET(bitmap, NFS_FATTR_MIMETYPE)
+	if (VATTR_IS_ACTIVE(vap, va_mode))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_MODE);
+	if (VATTR_IS_ACTIVE(vap, va_uid))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_OWNER);
+	if (VATTR_IS_ACTIVE(vap, va_gid))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_OWNER_GROUP);
+	// NFS_BITMAP_SET(bitmap, NFS_FATTR_SYSTEM)
+	if (vap->va_vaflags & VA_UTIMES_NULL) {
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_ACCESS_SET);
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_MODIFY_SET);
+	} else {
+		if (VATTR_IS_ACTIVE(vap, va_access_time))
+			NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_ACCESS_SET);
+		if (VATTR_IS_ACTIVE(vap, va_modify_time))
+			NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_MODIFY_SET);
+	}
+	if (VATTR_IS_ACTIVE(vap, va_backup_time))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_BACKUP);
+	if (VATTR_IS_ACTIVE(vap, va_create_time))
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_TIME_CREATE);
+	/* and limit to what is supported by server */
+	for (i=0; i < NFS_ATTR_BITMAP_LEN; i++)
+		bitmap[i] &= nmp->nm_fsattr.nfsa_supp_attr[i];
+
+	/*
+	 * Now pack it all together:
+	 *     BITMAP, #BYTES, ATTRS
+	 * Keep a pointer to the length so we can set it later.
+	 */
+	nfsm_chain_add_bitmap(error, nmc, bitmap, NFS_ATTR_BITMAP_LEN);
+	attrbytes = 0;
+	nfsm_chain_add_32(error, nmc, attrbytes);
+	pattrbytes = (uint32_t*)(nmc->nmc_ptr - NFSX_UNSIGNED);
+
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_SIZE)) {
+		nfsm_chain_add_64(error, nmc, vap->va_data_size);
+		attrbytes += 2*NFSX_UNSIGNED;
+	}
+	// NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ACL)
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_ARCHIVE)) {
+		nfsm_chain_add_32(error, nmc, (vap->va_flags & SF_ARCHIVED) ? 1 : 0);
+		attrbytes += NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_HIDDEN)) {
+		nfsm_chain_add_32(error, nmc, (vap->va_flags & UF_HIDDEN) ? 1 : 0);
+		attrbytes += NFSX_UNSIGNED;
+	}
+	// NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MIMETYPE)
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_MODE)) {
+		nfsm_chain_add_32(error, nmc, vap->va_mode);
+		attrbytes += NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER)) {
+		slen = snprintf(s, sizeof(s), "%d", vap->va_uid);
+		nfsm_chain_add_string(error, nmc, s, slen);
+		attrbytes += NFSX_UNSIGNED + nfsm_rndup(slen);
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_OWNER_GROUP)) {
+		slen = snprintf(s, sizeof(s), "%d", vap->va_gid);
+		nfsm_chain_add_string(error, nmc, s, slen);
+		attrbytes += NFSX_UNSIGNED + nfsm_rndup(slen);
+	}
+	// NFS_BITMAP_SET(bitmap, NFS_FATTR_SYSTEM)
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_ACCESS_SET)) {
+		if (vap->va_vaflags & VA_UTIMES_NULL) {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_SERVER);
+			attrbytes += NFSX_UNSIGNED;
+		} else {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_CLIENT);
+			nfsm_chain_add_64(error, nmc, vap->va_access_time.tv_sec);
+			nfsm_chain_add_32(error, nmc, vap->va_access_time.tv_nsec);
+			attrbytes += 4*NFSX_UNSIGNED;
+		}
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_BACKUP)) {
+		nfsm_chain_add_64(error, nmc, vap->va_backup_time.tv_sec);
+		nfsm_chain_add_32(error, nmc, vap->va_backup_time.tv_nsec);
+		attrbytes += 3*NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_CREATE)) {
+		nfsm_chain_add_64(error, nmc, vap->va_create_time.tv_sec);
+		nfsm_chain_add_32(error, nmc, vap->va_create_time.tv_nsec);
+		attrbytes += 3*NFSX_UNSIGNED;
+	}
+	if (NFS_BITMAP_ISSET(bitmap, NFS_FATTR_TIME_MODIFY_SET)) {
+		if (vap->va_vaflags & VA_UTIMES_NULL) {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_SERVER);
+			attrbytes += NFSX_UNSIGNED;
+		} else {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_CLIENT);
+			nfsm_chain_add_64(error, nmc, vap->va_modify_time.tv_sec);
+			nfsm_chain_add_32(error, nmc, vap->va_modify_time.tv_nsec);
+			attrbytes += 4*NFSX_UNSIGNED;
+		}
+	}
+	nfsmout_if(error);
+	/* Now, set the attribute data length */
+	*pattrbytes = txdr_unsigned(attrbytes);
+nfsmout:
+	return (error);
+}
+
diff --git a/bsd/nfs/nfs4_vnops.c b/bsd/nfs/nfs4_vnops.c
new file mode 100644
index 000000000..d8247eafc
--- /dev/null
+++ b/bsd/nfs/nfs4_vnops.c
@@ -0,0 +1,2151 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * vnode op calls for NFS version 4
+ */
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/resourcevar.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/mount_internal.h>
+#include <sys/malloc.h>
+#include <sys/kpi_mbuf.h>
+#include <sys/conf.h>
+#include <sys/vnode_internal.h>
+#include <sys/dirent.h>
+#include <sys/fcntl.h>
+#include <sys/lockf.h>
+#include <sys/ubc_internal.h>
+#include <sys/attr.h>
+#include <sys/signalvar.h>
+#include <sys/uio_internal.h>
+
+#include <vfs/vfs_support.h>
+
+#include <sys/vm.h>
+
+#include <sys/time.h>
+#include <kern/clock.h>
+#include <libkern/OSAtomic.h>
+
+#include <miscfs/fifofs/fifo.h>
+#include <miscfs/specfs/specdev.h>
+
+#include <nfs/rpcv2.h>
+#include <nfs/nfsproto.h>
+#include <nfs/nfs.h>
+#include <nfs/nfsnode.h>
+#include <nfs/nfs_gss.h>
+#include <nfs/nfsmount.h>
+#include <nfs/nfs_lock.h>
+#include <nfs/xdr_subs.h>
+#include <nfs/nfsm_subs.h>
+
+#include <net/if.h>
+#include <netinet/in.h>
+#include <netinet/in_var.h>
+#include <vm/vm_kern.h>
+
+#include <kern/task.h>
+#include <kern/sched_prim.h>
+
+
+int
+nfs4_access_rpc(nfsnode_t np, u_long *mode, vfs_context_t ctx)
+{
+	int error = 0, status, numops, slot;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+	struct timeval now;
+	uint32_t access, supported = 0, missing;
+	struct nfsmount *nmp = NFSTONMP(np);
+	int nfsvers = nmp->nm_vers;
+	uid_t uid;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	numops = 3; // PUTFH + ACCESS + GETATTR
+	nfsm_chain_build_alloc_init(error, &nmreq, 17 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "access", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_ACCESS);
+	nfsm_chain_add_32(error, &nmreq, *mode);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_ACCESS);
+	nfsm_chain_get_32(error, &nmrep, supported);
+	nfsm_chain_get_32(error, &nmrep, access);
+	nfsmout_if(error);
+	if ((missing = (*mode & ~supported))) {
+		/* missing support for something(s) we wanted */
+		if (missing & NFS_ACCESS_DELETE) {
+			/*
+			 * If the server doesn't report DELETE (possible
+			 * on UNIX systems), we'll assume that it is OK
+			 * and just let any subsequent delete action fail
+			 * if it really isn't deletable.
+			 */
+			access |= NFS_ACCESS_DELETE;
+		}
+	}
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	nfsmout_if(error);
+
+	uid = kauth_cred_getuid(vfs_context_ucred(ctx));
+	slot = nfs_node_mode_slot(np, uid, 1);
+	np->n_modeuid[slot] = uid;
+	microuptime(&now);
+	np->n_modestamp[slot] = now.tv_sec;
+	np->n_mode[slot] = access;
+
+	/* pass back the mode returned with this request */
+	*mode = np->n_mode[slot];
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_getattr_rpc(
+	nfsnode_t np,
+	mount_t mp,
+	u_char *fhp,
+	size_t fhsize,
+	vfs_context_t ctx,
+	struct nfs_vattr *nvap,
+	u_int64_t *xidp)
+{
+	struct nfsmount *nmp = mp ? VFSTONFS(mp) : NFSTONMP(np);
+	int error = 0, status, nfsvers, numops;
+	struct nfsm_chain nmreq, nmrep;
+
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	numops = 2; // PUTFH + GETATTR
+	nfsm_chain_build_alloc_init(error, &nmreq, 15 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "getattr", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, fhp, fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, mp, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, xidp, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nvap->nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, NULL, nvap, NULL, NULL);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_readlink_rpc(nfsnode_t np, char *buf, uint32_t *buflenp, vfs_context_t ctx)
+{
+	struct nfsmount *nmp;
+	int error = 0, lockerror = ENOENT, status, numops;
+	uint32_t len = 0;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return (ENXIO);
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	numops = 3; // PUTFH + GETATTR + READLINK
+	nfsm_chain_build_alloc_init(error, &nmreq, 16 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "readlink", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, NFS_VER4, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_READLINK);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, NFS_VER4, NULL, &xid);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_READLINK);
+	nfsm_chain_get_32(error, &nmrep, len);
+	nfsmout_if(error);
+	if (len >= *buflenp) {
+		if (np->n_size && (np->n_size < *buflenp))
+			len = np->n_size;
+		else
+			len = *buflenp - 1;
+	}
+	nfsm_chain_get_opaque(error, &nmrep, len, buf);
+	if (!error)
+		*buflenp = len;
+nfsmout:
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_read_rpc_async(
+	nfsnode_t np,
+	off_t offset,
+	size_t len,
+	thread_t thd,
+	kauth_cred_t cred,
+	struct nfsreq_cbinfo *cb,
+	struct nfsreq **reqp)
+{
+	struct nfsmount *nmp;
+	int error = 0, nfsvers, numops;
+	struct nfsm_chain nmreq;
+
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+
+	// PUTFH + READ + GETATTR
+	numops = 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 22 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "read", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_READ);
+
+	/* XXX use special stateid for now */
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+
+	nfsm_chain_add_64(error, &nmreq, offset);
+	nfsm_chain_add_32(error, &nmreq, len);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request_async(np, NULL, &nmreq, NFSPROC4_COMPOUND, thd, cred, cb, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs4_read_rpc_async_finish(
+	nfsnode_t np,
+	struct nfsreq *req,
+	struct uio *uiop,
+	size_t *lenp,
+	int *eofp)
+{
+	struct nfsmount *nmp;
+	int error = 0, lockerror, nfsvers, numops, status, eof = 0;
+	size_t retlen = 0;
+	u_int64_t xid;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		nfs_request_async_cancel(req);
+		return (ENXIO);
+	}
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	if (error == EINPROGRESS) /* async request restarted */
+		return (error);
+
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_READ);
+	nfsm_chain_get_32(error, &nmrep, eof);
+	nfsm_chain_get_32(error, &nmrep, retlen);
+	if (!error) {
+		*lenp = MIN(retlen, *lenp);
+		error = nfsm_chain_get_uio(&nmrep, *lenp, uiop);
+	}
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	if (eofp) {
+		if (!eof && !retlen)
+			eof = 1;
+		*eofp = eof;
+	}
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_write_rpc_async(
+	nfsnode_t np,
+	struct uio *uiop,
+	size_t len,
+	thread_t thd,
+	kauth_cred_t cred,
+	int iomode,
+	struct nfsreq_cbinfo *cb,
+	struct nfsreq **reqp)
+{
+	struct nfsmount *nmp;
+	int error = 0, nfsvers, numops;
+	off_t offset;
+	struct nfsm_chain nmreq;
+
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	offset = uiop->uio_offset;
+
+	nfsm_chain_null(&nmreq);
+
+	// PUTFH + WRITE + GETATTR
+	numops = 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 25 * NFSX_UNSIGNED + len);
+	nfsm_chain_add_compound_header(error, &nmreq, "write", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_WRITE);
+
+	/* XXX use special stateid for now */
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+	nfsm_chain_add_32(error, &nmreq, 0xffffffff);
+
+	nfsm_chain_add_64(error, &nmreq, uiop->uio_offset);
+	nfsm_chain_add_32(error, &nmreq, iomode);
+	nfsm_chain_add_32(error, &nmreq, len);
+	if (!error)
+		error = nfsm_chain_add_uio(&nmreq, uiop, len);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+
+	error = nfs_request_async(np, NULL, &nmreq, NFSPROC4_COMPOUND, thd, cred, cb, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs4_write_rpc_async_finish(
+	nfsnode_t np,
+	struct nfsreq *req,
+	int *iomodep,
+	size_t *rlenp,
+	uint64_t *wverfp)
+{
+	struct nfsmount *nmp;
+	int error = 0, lockerror = ENOENT, nfsvers, numops, status;
+	int committed = NFS_WRITE_FILESYNC;
+	size_t rlen = 0;
+	u_int64_t xid, wverf;
+	mount_t mp;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		nfs_request_async_cancel(req);
+		return (ENXIO);
+	}
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	if (error == EINPROGRESS) /* async request restarted */
+		return (error);
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		error = ENXIO;
+	if (!error && (lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_WRITE);
+	nfsm_chain_get_32(error, &nmrep, rlen);
+	nfsmout_if(error);
+	*rlenp = rlen;
+	if (rlen <= 0)
+		error = NFSERR_IO;
+	nfsm_chain_get_32(error, &nmrep, committed);
+	nfsm_chain_get_64(error, &nmrep, wverf);
+	nfsmout_if(error);
+	if (wverfp)
+		*wverfp = wverf;
+	lck_mtx_lock(&nmp->nm_lock);
+	if (!(nmp->nm_state & NFSSTA_HASWRITEVERF)) {
+		nmp->nm_verf = wverf;
+		nmp->nm_state |= NFSSTA_HASWRITEVERF;
+	} else if (nmp->nm_verf != wverf) {
+		nmp->nm_verf = wverf;
+	}
+	lck_mtx_unlock(&nmp->nm_lock);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+nfsmout:
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmrep);
+	if ((committed != NFS_WRITE_FILESYNC) && nfs_allow_async &&
+	    ((mp = NFSTOMP(np))) && (vfs_flags(mp) & MNT_ASYNC))
+		committed = NFS_WRITE_FILESYNC;
+	*iomodep = committed;
+	return (error);
+}
+
+int
+nfs4_remove_rpc(
+	nfsnode_t dnp,
+	char *name,
+	int namelen,
+	thread_t thd,
+	kauth_cred_t cred)
+{
+	int error = 0, remove_error = 0, status;
+	struct nfsmount *nmp;
+	int nfsvers, numops;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH, REMOVE, GETATTR
+	numops = 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 17 * NFSX_UNSIGNED + namelen);
+	nfsm_chain_add_compound_header(error, &nmreq, "remove", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_REMOVE);
+	nfsm_chain_add_string(error, &nmreq, name, namelen);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+
+	error = nfs_request2(dnp, NULL, &nmreq, NFSPROC4_COMPOUND, thd, cred, 0, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_REMOVE);
+	remove_error = error;
+	nfsm_chain_check_change_info(error, &nmrep, dnp);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, dnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(dnp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	dnp->n_flag |= NMODIFIED;
+
+	return (remove_error);
+}
+
+int
+nfs4_rename_rpc(
+	nfsnode_t fdnp,
+	char *fnameptr,
+	int fnamelen,
+	nfsnode_t tdnp,
+	char *tnameptr,
+	int tnamelen,
+	vfs_context_t ctx)
+{
+	int error = 0, status, nfsvers, numops;
+	struct nfsmount *nmp;
+	u_int64_t xid, savedxid;
+	struct nfsm_chain nmreq, nmrep;
+
+	nmp = NFSTONMP(fdnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH(FROM), SAVEFH, PUTFH(TO), RENAME, GETATTR(TO), RESTOREFH, GETATTR(FROM)
+	numops = 7;
+	nfsm_chain_build_alloc_init(error, &nmreq, 30 * NFSX_UNSIGNED + fnamelen + tnamelen);
+	nfsm_chain_add_compound_header(error, &nmreq, "rename", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, fdnp->n_fhp, fdnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SAVEFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, tdnp->n_fhp, tdnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RENAME);
+	nfsm_chain_add_string(error, &nmreq, fnameptr, fnamelen);
+	nfsm_chain_add_string(error, &nmreq, tnameptr, tnamelen);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RESTOREFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+
+	error = nfs_request(fdnp, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SAVEFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RENAME);
+	nfsm_chain_check_change_info(error, &nmrep, fdnp);
+	nfsm_chain_check_change_info(error, &nmrep, tdnp);
+	/* directory attributes: if we don't get them, make sure to invalidate */
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	savedxid = xid;
+	nfsm_chain_loadattr(error, &nmrep, tdnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(tdnp);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RESTOREFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	xid = savedxid;
+	nfsm_chain_loadattr(error, &nmrep, fdnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(fdnp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	fdnp->n_flag |= NMODIFIED;
+	tdnp->n_flag |= NMODIFIED;
+	/* Kludge: Map EEXIST => 0 assuming that it is a reply to a retry. */
+	if (error == EEXIST)
+		error = 0;
+	return (error);
+}
+
+/*
+ * NFS V4 readdir RPC.
+ */
+#define	DIRHDSIZ	((int)(sizeof(struct dirent) - (MAXNAMLEN + 1)))
+int
+nfs4_readdir_rpc(nfsnode_t dnp, struct uio *uiop, vfs_context_t ctx)
+{
+	size_t len, tlen, skiplen, left;
+	struct dirent *dp = NULL;
+	vnode_t newvp;
+	nfsuint64 *cookiep;
+	struct componentname cn, *cnp = &cn;
+	nfsuint64 cookie;
+	struct nfsmount *nmp;
+	nfsnode_t np;
+	int error = 0, lockerror, status, more_entries = 1, blksiz = 0, bigenough = 1;
+	int nfsvers, rdirplus, nmreaddirsize, nmrsize, eof, i, numops;
+	u_int64_t xid, savexid;
+	struct nfs_vattr nvattr;
+	struct nfsm_chain nmreq, nmrep;
+	char *cp;
+	const char *tag;
+	uint32_t entry_attrs[NFS_ATTR_BITMAP_LEN];
+	fhandle_t fh;
+
+#if DIAGNOSTIC
+	/* XXX limitation based on need to adjust uio */
+	if (uiop->uio_iovcnt != 1 || (uiop->uio_offset & (DIRBLKSIZ - 1)) ||
+		(uio_uio_resid(uiop) & (DIRBLKSIZ - 1)))
+		panic("nfs4_readdir_rpc: bad uio");
+#endif
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+	nmreaddirsize = nmp->nm_readdirsize;
+	nmrsize = nmp->nm_rsize;
+	rdirplus = (nmp->nm_flag & NFSMNT_RDIRPLUS) ? 1 : 0;
+
+	bzero(cnp, sizeof(*cnp));
+	newvp = NULLVP;
+
+	/*
+	 * Set up attribute request for entries.
+	 * For READDIRPLUS functionality, get everything.
+	 * Otherwise, just get what we need for struct dirent.
+	 */
+	if (rdirplus) {
+		tag = "READDIRPLUS";
+		for (i=0; i < NFS_ATTR_BITMAP_LEN; i++)
+			entry_attrs[i] =
+				nfs_getattr_bitmap[i] &
+				nmp->nm_fsattr.nfsa_supp_attr[i];
+		NFS_BITMAP_SET(entry_attrs, NFS_FATTR_FILEHANDLE);
+	} else {
+		tag = "READDIR";
+		NFS_CLEAR_ATTRIBUTES(entry_attrs);
+		NFS_BITMAP_SET(entry_attrs, NFS_FATTR_TYPE);
+		NFS_BITMAP_SET(entry_attrs, NFS_FATTR_FILEID);
+	}
+	/* XXX NFS_BITMAP_SET(entry_attrs, NFS_FATTR_MOUNTED_ON_FILEID); */
+	NFS_BITMAP_SET(entry_attrs, NFS_FATTR_RDATTR_ERROR);
+
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+		return (lockerror);
+
+	/*
+	 * If there is no cookie, assume directory was stale.
+	 */
+	cookiep = nfs_getcookie(dnp, uiop->uio_offset, 0);
+	if (cookiep)
+		cookie = *cookiep;
+	else {
+		nfs_unlock(dnp);
+		return (NFSERR_BAD_COOKIE);
+	}
+
+	/*
+	 * The NFS client is responsible for the "." and ".."
+	 * entries in the directory.  So, we put them at the top.
+	 */
+	if ((uiop->uio_offset == 0) &&
+	    ((2*(4 + DIRHDSIZ)) <= uio_uio_resid(uiop))) {
+		/* add "." entry */
+		len = 2;
+		tlen = nfsm_rndup(len);
+		// LP64todo - fix this!
+		dp = (struct dirent *) CAST_DOWN(caddr_t, uio_iov_base(uiop));
+		dp->d_fileno = dnp->n_vattr.nva_fileid;
+		dp->d_namlen = len;
+		dp->d_reclen = tlen + DIRHDSIZ;
+		dp->d_type = DT_DIR;
+		strlcpy(dp->d_name, ".", len);
+		blksiz += dp->d_reclen;
+		if (blksiz == DIRBLKSIZ)
+			blksiz = 0;
+		uiop->uio_offset += DIRHDSIZ + tlen;
+		uio_iov_base_add(uiop, DIRHDSIZ + tlen);
+		uio_uio_resid_add(uiop, -(DIRHDSIZ + tlen));
+		uio_iov_len_add(uiop, -(DIRHDSIZ + tlen));
+		/* add ".." entry */
+		len = 3;
+		tlen = nfsm_rndup(len);
+		// LP64todo - fix this!
+		dp = (struct dirent *) CAST_DOWN(caddr_t, uio_iov_base(uiop));
+		if (dnp->n_parent)
+			dp->d_fileno = VTONFS(dnp->n_parent)->n_vattr.nva_fileid;
+		else
+			dp->d_fileno = dnp->n_vattr.nva_fileid;
+		dp->d_namlen = len;
+		dp->d_reclen = tlen + DIRHDSIZ;
+		dp->d_type = DT_DIR;
+		strlcpy(dp->d_name, "..", len);
+		blksiz += dp->d_reclen;
+		if (blksiz == DIRBLKSIZ)
+			blksiz = 0;
+		uiop->uio_offset += DIRHDSIZ + tlen;
+		uio_iov_base_add(uiop, DIRHDSIZ + tlen);
+		uio_uio_resid_add(uiop, -(DIRHDSIZ + tlen));
+		uio_iov_len_add(uiop, -(DIRHDSIZ + tlen));
+		cookie.nfsuquad[0] = 0;
+		cookie.nfsuquad[1] = 2;
+	}
+
+	/*
+	 * Loop around doing readdir rpc's of size nm_readdirsize
+	 * truncated to a multiple of DIRBLKSIZ.
+	 * The stopping criteria is EOF or buffer full.
+	 */
+	while (more_entries && bigenough) {
+		nfsm_chain_null(&nmreq);
+		nfsm_chain_null(&nmrep);
+		nfsm_assert(error, NFSTONMP(dnp), ENXIO);
+
+		numops = 3; // PUTFH + GETATTR + READDIR
+		nfsm_chain_build_alloc_init(error, &nmreq, 26 * NFSX_UNSIGNED);
+		nfsm_chain_add_compound_header(error, &nmreq, tag, numops);
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+		nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+		nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+			NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_READDIR);
+		/* opaque values don't need swapping, but as long */
+		/* as we are consistent about it, it should be ok */
+		nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[0]);
+		if ((cookie.nfsuquad[0] == 0) && (cookie.nfsuquad[1] <= 2))
+			nfsm_chain_add_32(error, &nmreq, 0);
+		else
+			nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[1]);
+		nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[0]);
+		nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[1]);
+		nfsm_chain_add_32(error, &nmreq, nmreaddirsize);
+		nfsm_chain_add_32(error, &nmreq, nmrsize);
+		nfsm_chain_add_bitmap(error, &nmreq, entry_attrs, NFS_ATTR_BITMAP_LEN);
+		nfsm_chain_build_done(error, &nmreq);
+		nfsm_assert(error, (numops == 0), EPROTO);
+		nfs_unlock(dnp);
+		nfsmout_if(error);
+		error = nfs_request(dnp, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+			error = lockerror;
+		savexid = xid;
+		nfsm_chain_skip_tag(error, &nmrep);
+		nfsm_chain_get_32(error, &nmrep, numops);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+		nfsm_chain_loadattr(error, &nmrep, dnp, nfsvers, NULL, &xid);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_READDIR);
+		nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[0]);
+		nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[1]);
+		nfsm_chain_get_32(error, &nmrep, more_entries);
+		nfs_unlock(dnp);
+		nfsmout_if(error);
+
+		/* Loop through the entries, massaging them into "dirent" form. */
+		/* If READDIRPLUS, also create the vnodes. */
+		while (more_entries && bigenough) {
+			/* Entry: COOKIE, NAME, FATTR */
+			nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[0]);
+			nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[1]);
+			nfsm_chain_get_32(error, &nmrep, len);
+			nfsmout_if(error);
+			/* Note: NFS supports longer names, but struct dirent doesn't */
+			/* so we just truncate the names to fit */
+			if (len <= 0) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+			if (len > MAXNAMLEN) {
+				skiplen = len - MAXNAMLEN;
+				len = MAXNAMLEN;
+			} else {
+				skiplen = 0;
+			}
+			tlen = nfsm_rndup(len);
+			if (tlen == len)
+				tlen += 4;	/* To ensure null termination */
+			left = DIRBLKSIZ - blksiz;
+			if ((tlen + DIRHDSIZ) > left) {
+				dp->d_reclen += left;
+				uio_iov_base_add(uiop, left);
+				uio_iov_len_add(uiop, -left);
+				uiop->uio_offset += left;
+				uio_uio_resid_add(uiop, -left);
+				blksiz = 0;
+			}
+			if ((tlen + DIRHDSIZ) > uio_uio_resid(uiop)) {
+				bigenough = 0;
+				break;
+			}
+			// LP64todo - fix this!
+			dp = (struct dirent *) CAST_DOWN(caddr_t, uio_iov_base(uiop));
+			dp->d_fileno = 0;
+			dp->d_namlen = len;
+			dp->d_reclen = tlen + DIRHDSIZ;
+			dp->d_type = DT_UNKNOWN;
+			blksiz += dp->d_reclen;
+			if (blksiz == DIRBLKSIZ)
+				blksiz = 0;
+			uiop->uio_offset += DIRHDSIZ;
+#if LP64KERN
+			uio_uio_resid_add(uiop, -((int64_t)DIRHDSIZ));
+			uio_iov_len_add(uiop, -((int64_t)DIRHDSIZ));
+#else
+			uio_uio_resid_add(uiop, -((int)DIRHDSIZ));
+			uio_iov_len_add(uiop, -((int)DIRHDSIZ));
+#endif
+			uio_iov_base_add(uiop, DIRHDSIZ);
+			// LP64todo - fix this!
+			cnp->cn_nameptr = CAST_DOWN(caddr_t, uio_iov_base(uiop));
+			cnp->cn_namelen = len;
+			error = nfsm_chain_get_uio(&nmrep, len, uiop);
+			if (skiplen)
+				nfsm_chain_adv(error, &nmrep,
+					nfsm_rndup(len + skiplen) - nfsm_rndup(len));
+			nfsmout_if(error);
+			NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+			error = nfs4_parsefattr(&nmrep, NULL, &nvattr, &fh, NULL);
+			if (error && NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_RDATTR_ERROR)) {
+				/* OK, we didn't get attributes, whatever... */
+				NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+				error = 0;
+			}
+			nfsm_chain_get_32(error, &nmrep, more_entries);
+			nfsmout_if(error);
+
+			cp = CAST_DOWN(caddr_t, uio_iov_base(uiop));
+			tlen -= len;
+			*cp = '\0';
+			uio_iov_base_add(uiop, tlen);
+			uio_iov_len_add(uiop, -tlen);
+			uiop->uio_offset += tlen;
+			uio_uio_resid_add(uiop, -tlen);
+
+			/*
+			 * Skip any "." and ".." entries returned from server.
+			 * (Actually, just leave it in place with d_fileno == 0.)
+			 */
+			if ((cnp->cn_nameptr[0] == '.') &&
+			    ((len == 1) || ((len == 2) && (cnp->cn_nameptr[1] == '.')))) {
+				/* clear the name too */
+				dp->d_namlen = 0;
+				dp->d_name[0] = '\0';
+				continue;
+			}
+
+			if (NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_TYPE))
+				dp->d_type = IFTODT(VTTOIF(nvattr.nva_type));
+			if (NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_FILEID))
+				dp->d_fileno = (int)nvattr.nva_fileid;
+			if (rdirplus && NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_FILEHANDLE) &&
+			    !NFS_CMPFH(dnp, fh.fh_data, fh.fh_len)) {
+				cnp->cn_hash = 0;
+				error = nfs_nget(NFSTOMP(dnp), dnp, cnp,
+						fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+				if (!error) {
+					nfs_unlock(np);
+					vnode_put(NFSTOV(np));
+				}
+			}
+			nfsmout_if(error);
+		}
+		/* If at end of rpc data, get the eof boolean */
+		if (!more_entries) {
+			nfsm_chain_get_32(error, &nmrep, eof);
+			if (!error)
+				more_entries = (eof == 0);
+		}
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+			error = lockerror;
+		nfsmout_if(error);
+		nfsm_chain_cleanup(&nmrep);
+	}
+	nfs_unlock(dnp);
+	/*
+	 * Fill last record, iff any, out to a multiple of DIRBLKSIZ
+	 * by increasing d_reclen for the last record.
+	 */
+	if (blksiz > 0) {
+		left = DIRBLKSIZ - blksiz;
+		dp->d_reclen += left;
+		uio_iov_base_add(uiop, left);
+		uio_iov_len_add(uiop, -left);
+		uiop->uio_offset += left;
+		uio_uio_resid_add(uiop, -left);
+	}
+
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	/*
+	 * We are now either at the end of the directory or have filled the
+	 * block.
+	 */
+	if (bigenough)
+		dnp->n_direofoffset = uiop->uio_offset;
+	else {
+		if (uio_uio_resid(uiop) > 0)
+			printf("EEK! nfs4_readdir_rpc resid > 0\n");
+		cookiep = nfs_getcookie(dnp, uiop->uio_offset, 1);
+		if (cookiep)
+			*cookiep = cookie;
+	}
+
+	nfs_unlock(dnp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_lookup_rpc_async(
+	nfsnode_t dnp,
+	char *name,
+	int namelen,
+	vfs_context_t ctx,
+	struct nfsreq **reqp)
+{
+	int error = 0, isdotdot = 0, getattrs = 1, nfsvers, numops;
+	struct nfsm_chain nmreq;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN];
+	struct nfsmount *nmp;
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	if ((name[0] == '.') && (name[1] == '.') && (namelen == 2))
+		isdotdot = 1;
+
+	nfsm_chain_null(&nmreq);
+
+	// PUTFH, GETATTR, LOOKUP(P), GETATTR (FH)
+	numops = getattrs ? 4 : 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 20 * NFSX_UNSIGNED + namelen);
+	nfsm_chain_add_compound_header(error, &nmreq, "lookup", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	if (isdotdot) {
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_LOOKUPP);
+	} else {
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_LOOKUP);
+		nfsm_chain_add_string(error, &nmreq, name, namelen);
+	}
+	if (getattrs) {
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+		NFS_COPY_ATTRIBUTES(nfs_getattr_bitmap, bitmap);
+		NFS_BITMAP_SET(bitmap, NFS_FATTR_FILEHANDLE);
+		nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+			NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	}
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC4_COMPOUND,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs4_lookup_rpc_async_finish(
+	nfsnode_t dnp,
+	__unused vfs_context_t ctx,
+	struct nfsreq *req,
+	u_int64_t *xidp,
+	fhandle_t *fhp,
+	struct nfs_vattr *nvap)
+{
+	int error = 0, status, nfsvers, numops;
+	uint32_t val = 0;
+	u_int64_t xid;
+	struct nfsmount *nmp;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(dnp);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	if (xidp)
+		*xidp = xid;
+	nfsm_chain_loadattr(error, &nmrep, dnp, nfsvers, NULL, &xid);
+
+	// nfsm_chain_op_check(error, &nmrep, (isdotdot ? NFS_OP_LOOKUPP : NFS_OP_LOOKUP));
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsm_assert(error, (val == NFS_OP_LOOKUPP) || (val == NFS_OP_LOOKUP), EBADRPC);
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsm_assert(error, (val == NFS_OK), val);
+
+	nfsmout_if(error || !fhp || !nvap);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nvap->nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, NULL, nvap, fhp, NULL);
+	if (!NFS_BITMAP_ISSET(nvap->nva_bitmap, NFS_FATTR_FILEHANDLE)) {
+		error = EBADRPC;
+		goto nfsmout;
+	}
+nfsmout:
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_commit_rpc(
+	nfsnode_t np,
+	u_int64_t offset,
+	u_int64_t count,
+	kauth_cred_t cred)
+{
+	struct nfsmount *nmp;
+	int error = 0, lockerror, status, nfsvers, numops;
+	u_int64_t xid, wverf;
+	uint32_t count32;
+	struct nfsm_chain nmreq, nmrep;
+
+	nmp = NFSTONMP(np);
+	FSDBG(521, np, offset, count, nmp ? nmp->nm_state : 0);
+	if (!nmp)
+		return (ENXIO);
+	if (!(nmp->nm_state & NFSSTA_HASWRITEVERF))
+		return (0);
+	nfsvers = nmp->nm_vers;
+
+	if (count > UINT32_MAX)
+		count32 = 0;
+	else
+		count32 = count;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH, COMMIT, GETATTR
+	numops = 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 19 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "commit", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_COMMIT);
+	nfsm_chain_add_64(error, &nmreq, offset);
+	nfsm_chain_add_32(error, &nmreq, count32);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request2(np, NULL, &nmreq, NFSPROC4_COMPOUND,
+			current_thread(), cred, 0, &nmrep, &xid, &status);
+
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_COMMIT);
+	nfsm_chain_get_64(error, &nmrep, wverf);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsmout_if(error);
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_verf != wverf) {
+		nmp->nm_verf = wverf;
+		error = NFSERR_STALEWRITEVERF;
+	}
+	lck_mtx_unlock(&nmp->nm_lock);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_pathconf_rpc(
+	nfsnode_t np,
+	struct nfs_fsattr *nfsap,
+	vfs_context_t ctx)
+{
+	u_int64_t xid;
+	int error = 0, lockerror, status, nfsvers, numops;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsmount *nmp = NFSTONMP(np);
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN];
+	struct nfs_vattr nvattr;
+
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	/* NFSv4: fetch "pathconf" info for this node */
+	numops = 2; // PUTFH + GETATTR
+	nfsm_chain_build_alloc_init(error, &nmreq, 16 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "pathconf", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS_COPY_ATTRIBUTES(nfs_getattr_bitmap, bitmap);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_MAXLINK);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_MAXNAME);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_NO_TRUNC);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_CHOWN_RESTRICTED);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_CASE_INSENSITIVE);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_CASE_PRESERVING);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, nfsap, &nvattr, NULL, NULL);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfs_loadattrcache(np, &nvattr, &xid, 0);
+	if (!lockerror)
+		nfs_unlock(np);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_vnop_getattr(
+	struct vnop_getattr_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_vp;
+		struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	struct vnode_attr *vap = ap->a_vap;
+	struct nfs_vattr nva;
+	int error;
+
+	error = nfs_getattr(VTONFS(ap->a_vp), &nva, ap->a_context, 0);
+	if (error)
+		return (error);
+
+	/* copy what we have in nva to *a_vap */
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_RAWDEV)) {
+		dev_t rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2);
+		VATTR_RETURN(vap, va_rdev, rdev);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_NUMLINKS))
+		VATTR_RETURN(vap, va_nlink, nva.nva_nlink);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_SIZE))
+		VATTR_RETURN(vap, va_data_size, nva.nva_size);
+	// VATTR_RETURN(vap, va_data_alloc, ???);
+	// VATTR_RETURN(vap, va_total_size, ???);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_SPACE_USED))
+		VATTR_RETURN(vap, va_total_alloc, nva.nva_bytes);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_OWNER))
+		VATTR_RETURN(vap, va_uid, nva.nva_uid);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_OWNER_GROUP))
+		VATTR_RETURN(vap, va_gid, nva.nva_gid);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_MODE))
+		VATTR_RETURN(vap, va_mode, nva.nva_mode);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_ARCHIVE) ||
+	    NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_HIDDEN)) {
+		uint32_t flags = 0;
+		if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_ARCHIVE))
+			flags |= SF_ARCHIVED;
+		if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_HIDDEN))
+			flags |= UF_HIDDEN;
+		VATTR_RETURN(vap, va_flags, flags);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TIME_CREATE)) {
+		vap->va_create_time.tv_sec = nva.nva_timesec[NFSTIME_CREATE];
+		vap->va_create_time.tv_nsec = nva.nva_timensec[NFSTIME_CREATE];
+		VATTR_SET_SUPPORTED(vap, va_create_time);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TIME_ACCESS)) {
+		vap->va_access_time.tv_sec = nva.nva_timesec[NFSTIME_ACCESS];
+		vap->va_access_time.tv_nsec = nva.nva_timensec[NFSTIME_ACCESS];
+		VATTR_SET_SUPPORTED(vap, va_access_time);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TIME_MODIFY)) {
+		vap->va_modify_time.tv_sec = nva.nva_timesec[NFSTIME_MODIFY];
+		vap->va_modify_time.tv_nsec = nva.nva_timensec[NFSTIME_MODIFY];
+		VATTR_SET_SUPPORTED(vap, va_modify_time);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TIME_METADATA)) {
+		vap->va_change_time.tv_sec = nva.nva_timesec[NFSTIME_CHANGE];
+		vap->va_change_time.tv_nsec = nva.nva_timensec[NFSTIME_CHANGE];
+		VATTR_SET_SUPPORTED(vap, va_change_time);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TIME_BACKUP)) {
+		vap->va_backup_time.tv_sec = nva.nva_timesec[NFSTIME_BACKUP];
+		vap->va_backup_time.tv_nsec = nva.nva_timensec[NFSTIME_BACKUP];
+		VATTR_SET_SUPPORTED(vap, va_backup_time);
+	}
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_FILEID))
+		VATTR_RETURN(vap, va_fileid, nva.nva_fileid);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_TYPE))
+		VATTR_RETURN(vap, va_type, nva.nva_type);
+	if (NFS_BITMAP_ISSET(nva.nva_bitmap, NFS_FATTR_CHANGE))
+		VATTR_RETURN(vap, va_filerev, nva.nva_change);
+
+	// other attrs we might support someday:
+	// VATTR_RETURN(vap, va_encoding, ??? /* potentially unnormalized UTF-8? */);
+	// struct kauth_acl *va_acl;	/* access control list */
+	// guid_t	va_uuuid;	/* file owner UUID */
+	// guid_t	va_guuid;	/* file group UUID */
+
+	return (error);
+}
+
+int
+nfs4_setattr_rpc(
+	nfsnode_t np,
+	struct vnode_attr *vap,
+	vfs_context_t ctx,
+	int alreadylocked)
+{
+	struct nfsmount *nmp = NFSTONMP(np);
+	int error = 0, lockerror = ENOENT, status, nfsvers, numops;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN], bmlen, stateid;
+
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	if (VATTR_IS_ACTIVE(vap, va_flags) && (vap->va_flags & ~(SF_ARCHIVED|UF_HIDDEN))) {
+		/* we don't support setting unsupported flags (duh!) */
+		if (vap->va_active & ~VNODE_ATTR_va_flags)
+			return (EINVAL);	/* return EINVAL if other attributes also set */
+		else
+			return (ENOTSUP);	/* return ENOTSUP for chflags(2) */
+	}
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH, SETATTR, GETATTR
+	numops = 3;
+	nfsm_chain_build_alloc_init(error, &nmreq, 40 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "setattr", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SETATTR);
+	if (VATTR_IS_ACTIVE(vap, va_data_size))
+		stateid = 0xffffffff; /* XXX use the special stateid for now */
+	else
+		stateid = 0;
+	nfsm_chain_add_32(error, &nmreq, stateid);
+	nfsm_chain_add_32(error, &nmreq, stateid);
+	nfsm_chain_add_32(error, &nmreq, stateid);
+	nfsm_chain_add_32(error, &nmreq, stateid);
+	nfsm_chain_add_fattr4(error, &nmreq, vap, nmp);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	if (!alreadylocked && ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))))
+		error = lockerror;
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SETATTR);
+	bmlen = NFS_ATTR_BITMAP_LEN;
+	nfsm_chain_get_bitmap(error, &nmrep, bitmap, bmlen);
+	nfsmout_if(error);
+	nfs_vattr_set_supported(bitmap, vap);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(np);
+nfsmout:
+	if (!alreadylocked && !lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+int
+nfs4_vnop_open(struct vnop_open_args *ap)
+{
+	return nfs3_vnop_open(ap);
+}
+
+int
+nfs4_vnop_close(struct vnop_close_args *ap)
+{
+	return nfs3_vnop_close(ap);
+}
+
+int
+nfs4_vnop_advlock(__unused struct vnop_advlock_args *ap)
+{
+	return (ENOSYS);
+}
+
+/*
+ * Note: the NFSv4 CREATE RPC is for everything EXCEPT regular files.
+ * Files are created using the NFSv4 OPEN RPC.  So we must open the
+ * file to create it and then close it immediately.
+ */
+int
+nfs4_vnop_create(
+	struct vnop_create_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t *a_vpp;
+		struct componentname *a_cnp;
+		struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	vfs_context_t ctx = ap->a_context;
+	struct componentname *cnp = ap->a_cnp;
+	struct vnode_attr *vap = ap->a_vap;
+	vnode_t dvp = ap->a_dvp;
+	vnode_t *vpp = ap->a_vpp;
+	struct nfsmount *nmp;
+	struct nfs_vattr nvattr, dnvattr;
+	int error = 0, create_error = EIO, lockerror = ENOENT, status;
+	int nfsvers, numops;
+	u_int64_t xid, savedxid = 0;
+	nfsnode_t dnp = VTONFS(dvp);
+	nfsnode_t np = NULL;
+	vnode_t newvp = NULL;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN], bmlen;
+	uint32_t seqid, stateid[4], rflags, delegation, val;
+	fhandle_t fh;
+	struct nfsreq *req = NULL;
+	struct nfs_dulookup dul;
+
+	static uint32_t nfs4_open_owner_hack = 0;
+
+	nmp = VTONMP(dvp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	seqid = stateid[0] = stateid[1] = stateid[2] = stateid[3] = 0;
+	rflags = 0;
+
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH, SAVEFH, OPEN(CREATE), GETATTR(FH), RESTOREFH, GETATTR
+	numops = 6;
+	nfsm_chain_build_alloc_init(error, &nmreq, 53 * NFSX_UNSIGNED + cnp->cn_namelen);
+	nfsm_chain_add_compound_header(error, &nmreq, "create", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SAVEFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_OPEN);
+	nfsm_chain_add_32(error, &nmreq, seqid);
+	seqid++;
+	nfsm_chain_add_32(error, &nmreq, NFS_OPEN_SHARE_ACCESS_BOTH);
+	nfsm_chain_add_32(error, &nmreq, NFS_OPEN_SHARE_DENY_NONE);
+	nfsm_chain_add_64(error, &nmreq, nmp->nm_clientid); // open_owner4.clientid
+	OSAddAtomic(1, (SInt32*)&nfs4_open_owner_hack);
+	nfsm_chain_add_32(error, &nmreq, sizeof(nfs4_open_owner_hack));
+	nfsm_chain_add_opaque(error, &nmreq, &nfs4_open_owner_hack, sizeof(nfs4_open_owner_hack)); // open_owner4.owner
+	// openflag4
+	nfsm_chain_add_32(error, &nmreq, NFS_OPEN_CREATE);
+	nfsm_chain_add_32(error, &nmreq, NFS_CREATE_UNCHECKED); // XXX exclusive/guarded
+	nfsm_chain_add_fattr4(error, &nmreq, vap, nmp);
+	// open_claim4
+	nfsm_chain_add_32(error, &nmreq, NFS_CLAIM_NULL);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS_COPY_ATTRIBUTES(nfs_getattr_bitmap, bitmap);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_FILEHANDLE);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RESTOREFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC4_COMPOUND,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+	if (!error) {
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	}
+	savedxid = xid;
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SAVEFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_OPEN);
+	nfsm_chain_get_32(error, &nmrep, stateid[0]);
+	nfsm_chain_get_32(error, &nmrep, stateid[1]);
+	nfsm_chain_get_32(error, &nmrep, stateid[2]);
+	nfsm_chain_get_32(error, &nmrep, stateid[3]);
+	nfsm_chain_check_change_info(error, &nmrep, dnp);
+	nfsm_chain_get_32(error, &nmrep, rflags);
+	bmlen = NFS_ATTR_BITMAP_LEN;
+	nfsm_chain_get_bitmap(error, &nmrep, bitmap, bmlen);
+	nfsm_chain_get_32(error, &nmrep, delegation);
+	if (!error)
+		switch (delegation) {
+		case NFS_OPEN_DELEGATE_NONE:
+			break;
+		case NFS_OPEN_DELEGATE_READ:
+			printf("nfs4_vnop_create: read delegation?\n");
+			nfsm_chain_adv(error, &nmrep, 5*NFSX_UNSIGNED);
+			// ACE:
+			nfsm_chain_adv(error, &nmrep, 3 * NFSX_UNSIGNED);
+			nfsm_chain_get_32(error, &nmrep, val); /* string length */
+			nfsm_chain_adv(error, &nmrep, nfsm_rndup(val));
+			break;
+		case NFS_OPEN_DELEGATE_WRITE:
+			printf("nfs4_vnop_create: write delegation?\n");
+			nfsm_chain_adv(error, &nmrep, 5*NFSX_UNSIGNED);
+			nfsm_chain_adv(error, &nmrep, 3*NFSX_UNSIGNED);
+			// ACE:
+			nfsm_chain_adv(error, &nmrep, 3 * NFSX_UNSIGNED);
+			nfsm_chain_get_32(error, &nmrep, val); /* string length */
+			nfsm_chain_adv(error, &nmrep, nfsm_rndup(val));
+			break;
+		default:
+			error = EBADRPC;
+			break;
+		}
+	/* At this point if we have no error, the object was created. */
+	/* if we don't get attributes, then we should lookitup. */
+	create_error = error;
+	nfsmout_if(error);
+	nfs_vattr_set_supported(bitmap, vap);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, NULL, &nvattr, &fh, NULL);
+	nfsmout_if(error);
+	if (!NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_FILEHANDLE)) {
+		printf("nfs: open/create didn't return filehandle?\n");
+		error = EBADRPC;
+		goto nfsmout;
+	}
+	/* directory attributes: if we don't get them, make sure to invalidate */
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RESTOREFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_chain_loadattr(error, &nmrep, dnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(dnp);
+
+	if (rflags & NFS_OPEN_RESULT_CONFIRM) {
+		nfsm_chain_cleanup(&nmreq);
+		nfsm_chain_cleanup(&nmrep);
+		// PUTFH, OPEN_CONFIRM, GETATTR
+		numops = 3;
+		nfsm_chain_build_alloc_init(error, &nmreq, 23 * NFSX_UNSIGNED);
+		nfsm_chain_add_compound_header(error, &nmreq, "create_confirm", numops);
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+		nfsm_chain_add_fh(error, &nmreq, nfsvers, fh.fh_data, fh.fh_len);
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_OPEN_CONFIRM);
+		nfsm_chain_add_32(error, &nmreq, stateid[0]);
+		nfsm_chain_add_32(error, &nmreq, stateid[1]);
+		nfsm_chain_add_32(error, &nmreq, stateid[2]);
+		nfsm_chain_add_32(error, &nmreq, stateid[3]);
+		nfsm_chain_add_32(error, &nmreq, seqid);
+		seqid++;
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+		nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+			NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+		nfsm_chain_build_done(error, &nmreq);
+		nfsm_assert(error, (numops == 0), EPROTO);
+		nfsmout_if(error);
+		error = nfs_request(dnp, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+		nfsm_chain_skip_tag(error, &nmrep);
+		nfsm_chain_get_32(error, &nmrep, numops);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_OPEN_CONFIRM);
+		nfsm_chain_get_32(error, &nmrep, stateid[0]);
+		nfsm_chain_get_32(error, &nmrep, stateid[1]);
+		nfsm_chain_get_32(error, &nmrep, stateid[2]);
+		nfsm_chain_get_32(error, &nmrep, stateid[3]);
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+		nfsmout_if(error);
+		NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+		error = nfs4_parsefattr(&nmrep, NULL, &nvattr, NULL, NULL);
+		nfsmout_if(error);
+		savedxid = xid;
+	}
+	nfsmout_if(error);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	// PUTFH, CLOSE
+	numops = 2;
+	nfsm_chain_build_alloc_init(error, &nmreq, 19 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "create_close", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, fh.fh_data, fh.fh_len);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_CLOSE);
+	nfsm_chain_add_32(error, &nmreq, seqid);
+	seqid++;
+	nfsm_chain_add_32(error, &nmreq, stateid[0]);
+	nfsm_chain_add_32(error, &nmreq, stateid[1]);
+	nfsm_chain_add_32(error, &nmreq, stateid[2]);
+	nfsm_chain_add_32(error, &nmreq, stateid[3]);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(dnp, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_CLOSE);
+	nfsm_chain_get_32(error, &nmrep, stateid[0]);
+	nfsm_chain_get_32(error, &nmrep, stateid[1]);
+	nfsm_chain_get_32(error, &nmrep, stateid[2]);
+	nfsm_chain_get_32(error, &nmrep, stateid[3]);
+	if (error)
+		printf("nfs4_vnop_create: close error %d\n", error);
+
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	if (!lockerror) {
+		if (!create_error && (dnp->n_flag & NNEGNCENTRIES)) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(dvp);
+		}
+		dnp->n_flag |= NMODIFIED;
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(dvp);
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
+		}
+	}
+
+	if (!error && fh.fh_len) {
+		/* create the vnode with the filehandle and attributes */
+		xid = savedxid;
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+		if (!error)
+			newvp = NFSTOV(np);
+	}
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
+
+	/*
+	 * Kludge: Map EEXIST => 0 assuming that you have a reply to a retry
+	 * if we can succeed in looking up the object.
+	 */
+	if ((create_error == EEXIST) || (!create_error && !newvp)) {
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
+		if (!error) {
+			newvp = NFSTOV(np);
+			if (vnode_vtype(newvp) != VLNK)
+				error = EEXIST;
+		}
+	}
+	if (!lockerror)
+		nfs_unlock(dnp);
+	if (error) {
+		if (newvp) {
+			nfs_unlock(np);
+			vnode_put(newvp);
+		}
+	} else {
+		nfs_unlock(np);
+		*vpp = newvp;
+	}
+	return (error);
+}
+
+/*
+ * Note: the NFSv4 CREATE RPC is for everything EXCEPT regular files.
+ */
+static int
+nfs4_create_rpc(
+	vfs_context_t ctx,
+	nfsnode_t dnp,
+	struct componentname *cnp,
+	struct vnode_attr *vap,
+	int type,
+	char *link,
+	nfsnode_t *npp)
+{
+	struct nfsmount *nmp;
+	struct nfs_vattr nvattr, dnvattr;
+	int error = 0, create_error = EIO, lockerror = ENOENT, status;
+	int nfsvers, numops;
+	u_int64_t xid, savedxid = 0;
+	nfsnode_t np = NULL;
+	vnode_t newvp = NULL;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN], bmlen;
+	const char *tag;
+	nfs_specdata sd;
+	fhandle_t fh;
+	struct nfsreq *req = NULL;
+	struct nfs_dulookup dul;
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	sd.specdata1 = sd.specdata2 = 0;
+
+	switch (type) {
+	case NFLNK:
+		tag = "symlink";
+		break;
+	case NFBLK:
+	case NFCHR:
+		tag = "mknod";
+		if (!VATTR_IS_ACTIVE(vap, va_rdev))
+			return (EINVAL);
+		sd.specdata1 = major(vap->va_rdev);
+		sd.specdata2 = minor(vap->va_rdev);
+		break;
+	case NFSOCK:
+	case NFFIFO:
+		tag = "mknod";
+		break;
+	case NFDIR:
+		tag = "mkdir";
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH, SAVEFH, CREATE, GETATTR(FH), RESTOREFH, GETATTR
+	numops = 6;
+	nfsm_chain_build_alloc_init(error, &nmreq, 66 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, tag, numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SAVEFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_CREATE);
+	nfsm_chain_add_32(error, &nmreq, type);
+	if (type == NFLNK) {
+		nfsm_chain_add_string(error, &nmreq, link, strlen(link));
+	} else if ((type == NFBLK) || (type == NFCHR)) {
+		nfsm_chain_add_32(error, &nmreq, sd.specdata1);
+		nfsm_chain_add_32(error, &nmreq, sd.specdata2);
+	}
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	nfsm_chain_add_fattr4(error, &nmreq, vap, nmp);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS_COPY_ATTRIBUTES(nfs_getattr_bitmap, bitmap);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_FILEHANDLE);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RESTOREFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC4_COMPOUND,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+	if (!error) {
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	}
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SAVEFH);
+	nfsmout_if(error);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_CREATE);
+	nfsm_chain_check_change_info(error, &nmrep, dnp);
+	bmlen = NFS_ATTR_BITMAP_LEN;
+	nfsm_chain_get_bitmap(error, &nmrep, bitmap, bmlen);
+	/* At this point if we have no error, the object was created. */
+	/* if we don't get attributes, then we should lookitup. */
+	create_error = error;
+	nfsmout_if(error);
+	nfs_vattr_set_supported(bitmap, vap);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, NULL, &nvattr, &fh, NULL);
+	nfsmout_if(error);
+	if (!NFS_BITMAP_ISSET(nvattr.nva_bitmap, NFS_FATTR_FILEHANDLE)) {
+		printf("nfs: create/%s didn't return filehandle?\n", tag);
+		error = EBADRPC;
+		goto nfsmout;
+	}
+	/* directory attributes: if we don't get them, make sure to invalidate */
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RESTOREFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	savedxid = xid;
+	nfsm_chain_loadattr(error, &nmrep, dnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(dnp);
+
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	if (!lockerror) {
+		if (!create_error && (dnp->n_flag & NNEGNCENTRIES)) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(NFSTOV(dnp));
+		}
+		dnp->n_flag |= NMODIFIED;
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(NFSTOV(dnp));
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
+		}
+	}
+
+	if (!error && fh.fh_len) {
+		/* create the vnode with the filehandle and attributes */
+		xid = savedxid;
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+		if (!error)
+			newvp = NFSTOV(np);
+	}
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
+
+	/*
+	 * Kludge: Map EEXIST => 0 assuming that you have a reply to a retry
+	 * if we can succeed in looking up the object.
+	 */
+	if ((create_error == EEXIST) || (!create_error && !newvp)) {
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
+		if (!error) {
+			newvp = NFSTOV(np);
+			if (vnode_vtype(newvp) != VLNK)
+				error = EEXIST;
+		}
+	}
+	if (!lockerror)
+		nfs_unlock(dnp);
+	if (error) {
+		if (newvp) {
+			nfs_unlock(np);
+			vnode_put(newvp);
+		}
+	} else {
+		nfs_unlock(np);
+		*npp = np;
+	}
+	return (error);
+}
+
+int
+nfs4_vnop_mknod(
+	struct vnop_mknod_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t *a_vpp;
+		struct componentname *a_cnp;
+		struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	nfsnode_t np = NULL;
+	struct nfsmount *nmp;
+	int error;
+
+	nmp = VTONMP(ap->a_dvp);
+	if (!nmp)
+		return (ENXIO);
+
+	if (!VATTR_IS_ACTIVE(ap->a_vap, va_type))
+		return (EINVAL);
+	switch (ap->a_vap->va_type) {
+	case VBLK:
+	case VCHR:
+	case VFIFO:
+	case VSOCK:
+		break;
+	default:
+		return (ENOTSUP);
+	}
+
+	error = nfs4_create_rpc(ap->a_context, VTONFS(ap->a_dvp), ap->a_cnp, ap->a_vap,
+			vtonfs_type(ap->a_vap->va_type, nmp->nm_vers), NULL, &np);
+	if (!error)
+		*ap->a_vpp = NFSTOV(np);
+	return (error);
+}
+
+int
+nfs4_vnop_mkdir(
+	struct vnop_mkdir_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t *a_vpp;
+		struct componentname *a_cnp;
+		struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	nfsnode_t np = NULL;
+	int error;
+
+	error = nfs4_create_rpc(ap->a_context, VTONFS(ap->a_dvp), ap->a_cnp, ap->a_vap,
+			NFDIR, NULL, &np);
+	if (!error)
+		*ap->a_vpp = NFSTOV(np);
+	return (error);
+}
+
+int
+nfs4_vnop_symlink(
+	struct vnop_symlink_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t *a_vpp;
+		struct componentname *a_cnp;
+		struct vnode_attr *a_vap;
+		char *a_target;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	nfsnode_t np = NULL;
+	int error;
+
+	error = nfs4_create_rpc(ap->a_context, VTONFS(ap->a_dvp), ap->a_cnp, ap->a_vap,
+			NFLNK, ap->a_target, &np);
+	if (!error)
+		*ap->a_vpp = NFSTOV(np);
+	return (error);
+}
+
+int
+nfs4_vnop_link(
+	struct vnop_link_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_vp;
+		vnode_t a_tdvp;
+		struct componentname *a_cnp;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	vfs_context_t ctx = ap->a_context;
+	vnode_t vp = ap->a_vp;
+	vnode_t tdvp = ap->a_tdvp;
+	struct componentname *cnp = ap->a_cnp;
+	int error = 0, status;
+	struct nfsmount *nmp;
+	nfsnode_t np = VTONFS(vp);
+	nfsnode_t tdnp = VTONFS(tdvp);
+	int nfsvers, numops;
+	u_int64_t xid, savedxid;
+	struct nfsm_chain nmreq, nmrep;
+
+	if (vnode_mount(vp) != vnode_mount(tdvp))
+		return (EXDEV);
+
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	/*
+	 * Push all writes to the server, so that the attribute cache
+	 * doesn't get "out of sync" with the server.
+	 * XXX There should be a better way!
+	 */
+	nfs_flush(np, MNT_WAIT, vfs_context_thread(ctx), V_IGNORE_WRITEERR);
+
+	error = nfs_lock2(tdnp, np, NFS_NODE_LOCK_EXCLUSIVE);
+	if (error)
+		return (error);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH(SOURCE), SAVEFH, PUTFH(DIR), LINK, GETATTR(DIR), RESTOREFH, GETATTR
+	numops = 7;
+	nfsm_chain_build_alloc_init(error, &nmreq, 29 * NFSX_UNSIGNED + cnp->cn_namelen);
+	nfsm_chain_add_compound_header(error, &nmreq, "link", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_SAVEFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, tdnp->n_fhp, tdnp->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_LINK);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_RESTOREFH);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, nfs_getattr_bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(tdnp, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_SAVEFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_LINK);
+	nfsm_chain_check_change_info(error, &nmrep, tdnp);
+	/* directory attributes: if we don't get them, make sure to invalidate */
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	savedxid = xid;
+	nfsm_chain_loadattr(error, &nmrep, tdnp, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(tdnp);
+	/* link attributes: if we don't get them, make sure to invalidate */
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_RESTOREFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	xid = savedxid;
+	nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	if (error)
+		NATTRINVALIDATE(np);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	tdnp->n_flag |= NMODIFIED;
+	/* Kludge: Map EEXIST => 0 assuming that it is a reply to a retry. */
+	if (error == EEXIST)
+		error = 0;
+	if (!error && (tdnp->n_flag & NNEGNCENTRIES)) {
+		tdnp->n_flag &= ~NNEGNCENTRIES;
+		cache_purge_negatives(tdvp);
+	}
+	nfs_unlock2(tdnp, np);
+	return (error);
+}
+
+int
+nfs4_vnop_rmdir(
+	struct vnop_rmdir_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t a_vp;
+		struct componentname *a_cnp;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	vfs_context_t ctx = ap->a_context;
+	vnode_t vp = ap->a_vp;
+	vnode_t dvp = ap->a_dvp;
+	struct componentname *cnp = ap->a_cnp;
+	int error = 0;
+	nfsnode_t np = VTONFS(vp);
+	nfsnode_t dnp = VTONFS(dvp);
+	struct nfs_vattr dnvattr;
+	struct nfs_dulookup dul;
+
+	if (vnode_vtype(vp) != VDIR)
+		return (EINVAL);
+
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	if ((error = nfs_lock2(dnp, np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
+
+	nfs_dulookup_start(&dul, dnp, ctx);
+
+	error = nfs4_remove_rpc(dnp, cnp->cn_nameptr, cnp->cn_namelen,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx));
+
+	cache_purge(vp);
+	if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+		if (NFS_CHANGED_NC(NFS_VER4, dnp, &dnvattr)) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge(dvp);
+			NFS_CHANGED_UPDATE_NC(NFS_VER4, dnp, &dnvattr);
+		}
+	}
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
+	nfs_unlock2(dnp, np);
+
+	/*
+	 * Kludge: Map ENOENT => 0 assuming that you have a reply to a retry.
+	 */
+	if (error == ENOENT)
+		error = 0;
+	if (!error) {
+		/*
+		 * remove nfsnode from hash now so we can't accidentally find it
+		 * again if another object gets created with the same filehandle
+		 * before this vnode gets reclaimed
+		 */
+		lck_mtx_lock(nfs_node_hash_mutex);
+		if (np->n_hflag & NHHASHED) {
+			LIST_REMOVE(np, n_hash);
+			np->n_hflag &= ~NHHASHED;
+			FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
+		}
+		lck_mtx_unlock(nfs_node_hash_mutex);
+	}
+	return (error);
+}
+
diff --git a/bsd/nfs/nfs_bio.c b/bsd/nfs/nfs_bio.c
index 42b2326a1..8b2ab3e1f 100644
--- a/bsd/nfs/nfs_bio.c
+++ b/bsd/nfs/nfs_bio.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -69,7 +75,6 @@
 #include <sys/dirent.h>
 #include <sys/mount_internal.h>
 #include <sys/kernel.h>
-#include <sys/sysctl.h>
 #include <sys/ubc_internal.h>
 #include <sys/uio_internal.h>
 
@@ -80,29 +85,18 @@
 #include <kern/clock.h>
 #include <libkern/OSAtomic.h>
 #include <kern/kalloc.h>
+#include <kern/thread_call.h>
 
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/nfsnode.h>
 #include <sys/buf_internal.h>
+#include <libkern/OSAtomic.h>
 
-#include <sys/kdebug.h>
-
-#define FSDBG(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_NONE, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_TOP(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_START, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_BOT(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_END, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-
-extern int nfs_numasync;
-extern int nfs_ioddelwri;
-extern struct nfsstats nfsstats;
+kern_return_t	thread_terminate(thread_t); /* XXX */
 
 #define	NFSBUFHASH(np, lbn)	\
 	(&nfsbufhashtbl[((long)(np) / sizeof(*(np)) + (int)(lbn)) & nfsbufhash])
@@ -112,14 +106,13 @@ u_long nfsbufhash;
 int nfsbufcnt, nfsbufmin, nfsbufmax, nfsbufmetacnt, nfsbufmetamax;
 int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer;
 int nfs_nbdwrite;
-time_t nfsbuffreeuptimestamp;
+int nfs_buf_timer_on = 0;
+thread_t nfsbufdelwrithd = NULL;
 
 lck_grp_t *nfs_buf_lck_grp;
-lck_grp_attr_t *nfs_buf_lck_grp_attr;
-lck_attr_t *nfs_buf_lck_attr;
 lck_mtx_t *nfs_buf_mutex;
 
-#define NFSBUFWRITE_THROTTLE	9
+#define NFSBUF_FREE_PERIOD	30	/* seconds */
 #define NFSBUF_LRU_STALE	120
 #define NFSBUF_META_STALE	240
 
@@ -129,9 +122,9 @@ lck_mtx_t *nfs_buf_mutex;
 #define META_TO_FREEUP			3
 /* total number of nfsbufs nfs_buf_freeup() should attempt to free */
 #define TOTAL_TO_FREEUP			(LRU_TO_FREEUP+META_TO_FREEUP)
-/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list when called from nfs_timer() */
+/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffree list when called from timer */
 #define LRU_FREEUP_FRAC_ON_TIMER	8
-/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list when called from nfs_timer() */
+/* fraction of nfsbufs nfs_buf_freeup() should attempt to free from nfsbuffreemeta list when called from timer */
 #define META_FREEUP_FRAC_ON_TIMER	16
 /* fraction of total nfsbufs that nfsbuffreecnt should exceed before bothering to call nfs_buf_freeup() */
 #define LRU_FREEUP_MIN_FRAC		4
@@ -153,21 +146,17 @@ lck_mtx_t *nfs_buf_mutex;
 void
 nfs_nbinit(void)
 {
-	nfs_buf_lck_grp_attr = lck_grp_attr_alloc_init();
-	nfs_buf_lck_grp = lck_grp_alloc_init("nfs_buf", nfs_buf_lck_grp_attr);
-
-	nfs_buf_lck_attr = lck_attr_alloc_init();
-
-	nfs_buf_mutex = lck_mtx_alloc_init(nfs_buf_lck_grp, nfs_buf_lck_attr);
+	nfs_buf_lck_grp = lck_grp_alloc_init("nfs_buf", LCK_GRP_ATTR_NULL);
+	nfs_buf_mutex = lck_mtx_alloc_init(nfs_buf_lck_grp, LCK_ATTR_NULL);
 
 	nfsbufcnt = nfsbufmetacnt =
 	nfsbuffreecnt = nfsbuffreemetacnt = nfsbufdelwricnt = 0;
 	nfsbufmin = 128;
-	nfsbufmax = (sane_size >> PAGE_SHIFT) / 4;
-	nfsbufmetamax = (sane_size >> PAGE_SHIFT) / 16;
+	/* size nfsbufmax to cover at most half sane_size (w/default buf size) */
+	nfsbufmax = (sane_size >> PAGE_SHIFT) / (2 * (NFS_RWSIZE >> PAGE_SHIFT));
+	nfsbufmetamax = nfsbufmax / 4;
 	nfsneedbuffer = 0;
 	nfs_nbdwrite = 0;
-	nfsbuffreeuptimestamp = 0;
 
 	nfsbufhashtbl = hashinit(nfsbufmax/4, M_TEMP, &nfsbufhash);
 	TAILQ_INIT(&nfsbuffree);
@@ -176,6 +165,26 @@ nfs_nbinit(void)
 
 }
 
+/*
+ * Check periodically for stale/unused nfs bufs
+ */
+void
+nfs_buf_timer(__unused void *param0, __unused void *param1)
+{
+	nfs_buf_freeup(1);
+
+	lck_mtx_lock(nfs_buf_mutex);
+	if (nfsbufcnt <= nfsbufmin) {
+		nfs_buf_timer_on = 0;
+		lck_mtx_unlock(nfs_buf_mutex);
+		return;
+	}
+	lck_mtx_unlock(nfs_buf_mutex);
+
+	nfs_interval_timer_start(nfs_buf_timer_call,
+		NFSBUF_FREE_PERIOD * 1000);
+}
+
 /*
  * try to free up some excess, unused nfsbufs
  */
@@ -192,7 +201,6 @@ nfs_buf_freeup(int timer)
 	lck_mtx_lock(nfs_buf_mutex);
 
 	microuptime(&now);
-	nfsbuffreeuptimestamp = now.tv_sec;
 
 	FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, 0);
 
@@ -207,13 +215,13 @@ nfs_buf_freeup(int timer)
 		    (fbp->nb_timestamp + (2*NFSBUF_LRU_STALE)) > now.tv_sec)
 			break;
 		nfs_buf_remfree(fbp);
-		/* disassociate buffer from any vnode */
-		if (fbp->nb_vp) {
+		/* disassociate buffer from any nfsnode */
+		if (fbp->nb_np) {
 			if (fbp->nb_vnbufs.le_next != NFSNOLIST) {
 				LIST_REMOVE(fbp, nb_vnbufs);
 				fbp->nb_vnbufs.le_next = NFSNOLIST;
 			}
-			fbp->nb_vp = NULL;
+			fbp->nb_np = NULL;
 		}
 		LIST_REMOVE(fbp, nb_hash);
 		TAILQ_INSERT_TAIL(&nfsbuffreeup, fbp, nb_free);
@@ -231,13 +239,13 @@ nfs_buf_freeup(int timer)
 		    (fbp->nb_timestamp + (2*NFSBUF_META_STALE)) > now.tv_sec)
 			break;
 		nfs_buf_remfree(fbp);
-		/* disassociate buffer from any vnode */
-		if (fbp->nb_vp) {
+		/* disassociate buffer from any nfsnode */
+		if (fbp->nb_np) {
 			if (fbp->nb_vnbufs.le_next != NFSNOLIST) {
 				LIST_REMOVE(fbp, nb_vnbufs);
 				fbp->nb_vnbufs.le_next = NFSNOLIST;
 			}
-			fbp->nb_vp = NULL;
+			fbp->nb_np = NULL;
 		}
 		LIST_REMOVE(fbp, nb_hash);
 		TAILQ_INSERT_TAIL(&nfsbuffreeup, fbp, nb_free);
@@ -246,19 +254,17 @@ nfs_buf_freeup(int timer)
 	}
 
 	FSDBG(320, nfsbufcnt, nfsbuffreecnt, nfsbuffreemetacnt, 0);
-	NFSBUFCNTCHK(1);
+	NFSBUFCNTCHK();
 
 	lck_mtx_unlock(nfs_buf_mutex);
 
 	while ((fbp = TAILQ_FIRST(&nfsbuffreeup))) {
 		TAILQ_REMOVE(&nfsbuffreeup, fbp, nb_free);
 		/* nuke any creds */
-		if (IS_VALID_CRED(fbp->nb_rcred)) {
+		if (IS_VALID_CRED(fbp->nb_rcred))
 			kauth_cred_unref(&fbp->nb_rcred);
-		}
-		if (IS_VALID_CRED(fbp->nb_wcred)) {
+		if (IS_VALID_CRED(fbp->nb_wcred))
 			kauth_cred_unref(&fbp->nb_wcred);
-		}
 		/* if buf was NB_META, dump buffer */
 		if (ISSET(fbp->nb_flags, NB_META) && fbp->nb_data)
 			kfree(fbp->nb_data, fbp->nb_bufsize);
@@ -287,18 +293,18 @@ nfs_buf_remfree(struct nfsbuf *bp)
 		TAILQ_REMOVE(&nfsbuffree, bp, nb_free);
 	}
 	bp->nb_free.tqe_next = NFSNOLIST;
-	NFSBUFCNTCHK(1);
+	NFSBUFCNTCHK();
 }
 
 /*
  * check for existence of nfsbuf in cache
  */
 boolean_t
-nfs_buf_is_incore(vnode_t vp, daddr64_t blkno)
+nfs_buf_is_incore(nfsnode_t np, daddr64_t blkno)
 {
 	boolean_t rv;
 	lck_mtx_lock(nfs_buf_mutex);
-	if (nfs_buf_incore(vp, blkno))
+	if (nfs_buf_incore(np, blkno))
 		rv = TRUE;
 	else
 		rv = FALSE;
@@ -310,14 +316,14 @@ nfs_buf_is_incore(vnode_t vp, daddr64_t blkno)
  * return incore buffer (must be called with nfs_buf_mutex held)
  */
 struct nfsbuf *
-nfs_buf_incore(vnode_t vp, daddr64_t blkno)
+nfs_buf_incore(nfsnode_t np, daddr64_t blkno)
 {
 	/* Search hash chain */
-	struct nfsbuf * bp = NFSBUFHASH(VTONFS(vp), blkno)->lh_first;
+	struct nfsbuf * bp = NFSBUFHASH(np, blkno)->lh_first;
 	for (; bp != NULL; bp = bp->nb_hash.le_next)
-		if (bp->nb_lblkno == blkno && bp->nb_vp == vp) {
+		if ((bp->nb_lblkno == blkno) && (bp->nb_np == np)) {
 			if (!ISSET(bp->nb_flags, NB_INVAL)) {
-				FSDBG(547, bp, blkno, bp->nb_flags, bp->nb_vp);
+				FSDBG(547, bp, blkno, bp->nb_flags, bp->nb_np);
 				return (bp);
 			}
 		}
@@ -335,11 +341,15 @@ nfs_buf_incore(vnode_t vp, daddr64_t blkno)
 int
 nfs_buf_page_inval(vnode_t vp, off_t offset)
 {
+	struct nfsmount *nmp = VTONMP(vp);
 	struct nfsbuf *bp;
 	int error = 0;
 
+	if (!nmp)
+		return (ENXIO);
+
 	lck_mtx_lock(nfs_buf_mutex);
-	bp = nfs_buf_incore(vp, ubc_offtoblk(vp, offset));
+	bp = nfs_buf_incore(VTONFS(vp), (daddr64_t)(offset / nmp->nm_biosize));
 	if (!bp)
 		goto out;
 	FSDBG(325, bp, bp->nb_flags, bp->nb_dirtyoff, bp->nb_dirtyend);
@@ -380,14 +390,14 @@ nfs_buf_upl_setup(struct nfsbuf *bp)
 		return (0);
 
 	upl_flags = UPL_PRECIOUS;
-	if (! ISSET(bp->nb_flags, NB_READ)) {
+	if (!ISSET(bp->nb_flags, NB_READ)) {
 		/*
 		 * We're doing a "write", so we intend to modify
 		 * the pages we're gathering.
 		 */
 		upl_flags |= UPL_WILL_MODIFY;
 	}
-	kret = ubc_create_upl(bp->nb_vp, NBOFF(bp), bp->nb_bufsize,
+	kret = ubc_create_upl(NFSTOV(bp->nb_np), NBOFF(bp), bp->nb_bufsize,
 				&upl, NULL, upl_flags);
 	if (kret == KERN_INVALID_ARGUMENT) {
 		/* vm object probably doesn't exist any more */
@@ -400,7 +410,7 @@ nfs_buf_upl_setup(struct nfsbuf *bp)
 		return (EIO);
 	}
 
-	FSDBG(538, bp, NBOFF(bp), bp->nb_bufsize, bp->nb_vp);
+	FSDBG(538, bp, NBOFF(bp), bp->nb_bufsize, bp->nb_np);
 
 	bp->nb_pagelist = upl;
 	SET(bp->nb_flags, NB_PAGELIST);
@@ -422,7 +432,7 @@ nfs_buf_upl_check(struct nfsbuf *bp)
 		return;
 
 	npages = round_page_32(bp->nb_bufsize) / PAGE_SIZE;
-	filesize = ubc_getsize(bp->nb_vp);
+	filesize = ubc_getsize(NFSTOV(bp->nb_np));
 	fileoffset = NBOFF(bp);
 	if (fileoffset < filesize)
 		SET(bp->nb_flags, NB_CACHE);
@@ -441,11 +451,8 @@ nfs_buf_upl_check(struct nfsbuf *bp)
 			continue;
 		}
 		NBPGVALID_SET(bp,i);
-		if (upl_dirty_page(pl, i)) {
+		if (upl_dirty_page(pl, i))
 			NBPGDIRTY_SET(bp, i);
-			if (!ISSET(bp->nb_flags, NB_WASDIRTY))
-				SET(bp->nb_flags, NB_WASDIRTY);
-		}
 	}
 	fileoffset = NBOFF(bp);
 	if (ISSET(bp->nb_flags, NB_CACHE)) {
@@ -464,7 +471,7 @@ nfs_buf_upl_check(struct nfsbuf *bp)
  * make sure that a buffer is mapped
  * (must NOT be called with nfs_buf_mutex held)
  */
-static int
+int
 nfs_buf_map(struct nfsbuf *bp)
 {
 	kern_return_t kret;
@@ -483,37 +490,6 @@ nfs_buf_map(struct nfsbuf *bp)
 	return (0);
 }
 
-/*
- * check range of pages in nfsbuf's UPL for validity
- */
-static int
-nfs_buf_upl_valid_range(struct nfsbuf *bp, int off, int size)
-{
-	off_t fileoffset, filesize;
-	int pg, lastpg;
-	upl_page_info_t *pl;
-
-	if (!ISSET(bp->nb_flags, NB_PAGELIST))
-		return (0);
-	pl = ubc_upl_pageinfo(bp->nb_pagelist);
-
-	size += off & PAGE_MASK;
-	off &= ~PAGE_MASK;
-	fileoffset = NBOFF(bp);
-	filesize = VTONFS(bp->nb_vp)->n_size;
-	if ((fileoffset + off + size) > filesize)
-		size = filesize - (fileoffset + off);
-
-	pg = off/PAGE_SIZE;
-	lastpg = (off + size - 1)/PAGE_SIZE;
-	while (pg <= lastpg) {
-		if (!upl_valid_page(pl, pg))
-			return (0);
-		pg++;
-	}
-	return (1);
-}
-
 /*
  * normalize an nfsbuf's valid range
  *
@@ -523,8 +499,8 @@ nfs_buf_upl_valid_range(struct nfsbuf *bp, int off, int size)
  * here as we extend the valid range through all of the
  * contiguous valid pages.
  */
-static void
-nfs_buf_normalize_valid_range(struct nfsnode *np, struct nfsbuf *bp)
+void
+nfs_buf_normalize_valid_range(nfsnode_t np, struct nfsbuf *bp)
 {
 	int pg, npg;
 	/* pull validoff back to start of contiguous valid page range */
@@ -544,35 +520,25 @@ nfs_buf_normalize_valid_range(struct nfsnode *np, struct nfsbuf *bp)
 }
 
 /*
- * try to push out some delayed/uncommitted writes
- * ("locked" indicates whether nfs_buf_mutex is already held)
+ * process some entries on the delayed write queue
+ * (must be called with nfs_buf_mutex held)
  */
 static void
-nfs_buf_delwri_push(int locked)
+nfs_buf_delwri_service(void)
 {
 	struct nfsbuf *bp;
-	int i, error;
-
-	if (TAILQ_EMPTY(&nfsbufdelwri))
-		return;
+	nfsnode_t np;
+	int error, i = 0;
 
-	/* first try to tell the nfsiods to do it */
-	if (nfs_asyncio(NULL, NULL) == 0)
-		return;
-
-	/* otherwise, try to do some of the work ourselves */
-	i = 0;
-	if (!locked)
-		lck_mtx_lock(nfs_buf_mutex);
 	while (i < 8 && (bp = TAILQ_FIRST(&nfsbufdelwri)) != NULL) {
-		struct nfsnode *np = VTONFS(bp->nb_vp);
+		np = bp->nb_np;
 		nfs_buf_remfree(bp);
 		nfs_buf_refget(bp);
 		while ((error = nfs_buf_acquire(bp, 0, 0, 0)) == EAGAIN);
 		nfs_buf_refrele(bp);
 		if (error)
 			break;
-		if (!bp->nb_vp) {
+		if (!bp->nb_np) {
 			/* buffer is no longer valid */
 			nfs_buf_drop(bp);
 			continue;
@@ -585,7 +551,7 @@ nfs_buf_delwri_push(int locked)
 			nfsbufdelwricnt++;
 			nfs_buf_drop(bp);
 			lck_mtx_unlock(nfs_buf_mutex);
-			nfs_flushcommits(np->n_vnode, NULL, 1);
+			nfs_flushcommits(np, 1);
 		} else {
 			SET(bp->nb_flags, NB_ASYNC);
 			lck_mtx_unlock(nfs_buf_mutex);
@@ -594,6 +560,46 @@ nfs_buf_delwri_push(int locked)
 		i++;
 		lck_mtx_lock(nfs_buf_mutex);
 	}
+}
+
+/*
+ * thread to service the delayed write queue when asked
+ */
+static void
+nfs_buf_delwri_thread(__unused void *arg, __unused wait_result_t wr)
+{
+	struct timespec ts = { 30, 0 };
+	int error = 0;
+
+	lck_mtx_lock(nfs_buf_mutex);
+	while (!error) {
+		nfs_buf_delwri_service();
+		error = msleep(&nfsbufdelwrithd, nfs_buf_mutex, 0, "nfsbufdelwri", &ts);
+	}
+	nfsbufdelwrithd = NULL;
+	lck_mtx_unlock(nfs_buf_mutex);
+	thread_terminate(nfsbufdelwrithd);
+}
+
+/*
+ * try to push out some delayed/uncommitted writes
+ * ("locked" indicates whether nfs_buf_mutex is already held)
+ */
+static void
+nfs_buf_delwri_push(int locked)
+{
+	if (TAILQ_EMPTY(&nfsbufdelwri))
+		return;
+	if (!locked)
+		lck_mtx_lock(nfs_buf_mutex);
+	/* wake up the delayed write service thread */
+	if (nfsbufdelwrithd)
+		wakeup(&nfsbufdelwrithd);
+	else if (kernel_thread_start(nfs_buf_delwri_thread, NULL, &nfsbufdelwrithd) == KERN_SUCCESS)
+		thread_deallocate(nfsbufdelwrithd);
+	/* otherwise, try to do some of the work ourselves */
+	if (!nfsbufdelwrithd)
+		nfs_buf_delwri_service();
 	if (!locked)
 		lck_mtx_unlock(nfs_buf_mutex);
 }
@@ -619,23 +625,23 @@ nfs_buf_delwri_push(int locked)
  */
 int
 nfs_buf_get(
-	vnode_t vp,
+	nfsnode_t np,
 	daddr64_t blkno,
 	int size,
-	proc_t p,
+	thread_t thd,
 	int flags,
 	struct nfsbuf **bpp)
 {
-	struct nfsnode *np = VTONFS(vp);
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
+	vnode_t vp = NFSTOV(np);
+	struct nfsmount *nmp = VTONMP(vp);
 	struct nfsbuf *bp;
-	int biosize, bufsize;
+	int bufsize;
 	int slpflag = PCATCH;
 	int operation = (flags & NBLK_OPMASK);
 	int error = 0;
 	struct timespec ts;
 
-	FSDBG_TOP(541, vp, blkno, size, flags);
+	FSDBG_TOP(541, np, blkno, size, flags);
 	*bpp = NULL;
 
 	bufsize = size;
@@ -643,53 +649,54 @@ nfs_buf_get(
 		panic("nfs_buf_get: buffer larger than NFS_MAXBSIZE requested");
 
 	if (!nmp) {
-		FSDBG_BOT(541, vp, blkno, 0, ENXIO);
+		FSDBG_BOT(541, np, blkno, 0, ENXIO);
 		return (ENXIO);
 	}
-	biosize = nmp->nm_biosize;
 
-	if (UBCINVALID(vp) || !UBCINFOEXISTS(vp)) {
+	if (!UBCINFOEXISTS(vp)) {
 		operation = NBLK_META;
-	} else if (bufsize < biosize) {
+	} else if (bufsize < nmp->nm_biosize) {
 		/* reg files should always have biosize blocks */
-		bufsize = biosize;
+		bufsize = nmp->nm_biosize;
 	}
 
 	/* if NBLK_WRITE, check for too many delayed/uncommitted writes */
-	if ((operation == NBLK_WRITE) && (nfs_nbdwrite > ((nfsbufcnt*3)/4))) {
-		FSDBG_TOP(542, vp, blkno, nfs_nbdwrite, ((nfsbufcnt*3)/4));
+	if ((operation == NBLK_WRITE) && (nfs_nbdwrite > NFS_A_LOT_OF_DELAYED_WRITES)) {
+		FSDBG_TOP(542, np, blkno, nfs_nbdwrite, NFS_A_LOT_OF_DELAYED_WRITES);
 
 		/* poke the delwri list */
 		nfs_buf_delwri_push(0);
 
 		/* sleep to let other threads run... */
 		tsleep(&nfs_nbdwrite, PCATCH, "nfs_nbdwrite", 1);
-		FSDBG_BOT(542, vp, blkno, nfs_nbdwrite, ((nfsbufcnt*3)/4));
+		FSDBG_BOT(542, np, blkno, nfs_nbdwrite, NFS_A_LOT_OF_DELAYED_WRITES);
 	}
 
 loop:
 	lck_mtx_lock(nfs_buf_mutex);
 
 	/* check for existence of nfsbuf in cache */
-	if ((bp = nfs_buf_incore(vp, blkno))) {
+	if ((bp = nfs_buf_incore(np, blkno))) {
 		/* if busy, set wanted and wait */
 		if (ISSET(bp->nb_lflags, NBL_BUSY)) {
 			if (flags & NBLK_NOWAIT) {
 				lck_mtx_unlock(nfs_buf_mutex);
-				FSDBG_BOT(541, vp, blkno, bp, 0xbcbcbcbc);
+				FSDBG_BOT(541, np, blkno, bp, 0xbcbcbcbc);
 				return (0);
 			}
-			FSDBG_TOP(543, vp, blkno, bp, bp->nb_flags);
+			FSDBG_TOP(543, np, blkno, bp, bp->nb_flags);
 			SET(bp->nb_lflags, NBL_WANTED);
 
 			ts.tv_sec = 2;
 			ts.tv_nsec = 0;
-			msleep(bp, nfs_buf_mutex, slpflag|(PRIBIO+1)|PDROP,
-				"nfsbufget", (slpflag == PCATCH) ? 0 : &ts);
+			error = msleep(bp, nfs_buf_mutex, slpflag|(PRIBIO+1)|PDROP,
+					"nfsbufget", (slpflag == PCATCH) ? NULL : &ts);
+			if (error == EWOULDBLOCK)
+				error = 0;
 			slpflag = 0;
-			FSDBG_BOT(543, vp, blkno, bp, bp->nb_flags);
-			if ((error = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p))) {
-				FSDBG_BOT(541, vp, blkno, 0, error);
+			FSDBG_BOT(543, np, blkno, bp, bp->nb_flags);
+			if (error || ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0)))) {
+				FSDBG_BOT(541, np, blkno, 0, error);
 				return (error);
 			}
 			goto loop;
@@ -707,7 +714,7 @@ loop:
 
 	if (flags & NBLK_ONLYVALID) {
 		lck_mtx_unlock(nfs_buf_mutex);
-		FSDBG_BOT(541, vp, blkno, 0, 0x0000cace);
+		FSDBG_BOT(541, np, blkno, 0, 0x0000cace);
 		return (0);
 	}
 
@@ -764,27 +771,25 @@ loop:
 
 	if (bp) {
 		/* we have a buffer to reuse */
-		FSDBG(544, vp, blkno, bp, bp->nb_flags);
+		FSDBG(544, np, blkno, bp, bp->nb_flags);
 		nfs_buf_remfree(bp);
 		if (ISSET(bp->nb_flags, NB_DELWRI))
 			panic("nfs_buf_get: delwri");
 		SET(bp->nb_lflags, NBL_BUSY);
-		/* disassociate buffer from previous vnode */
-		if (bp->nb_vp) {
+		/* disassociate buffer from previous nfsnode */
+		if (bp->nb_np) {
 			if (bp->nb_vnbufs.le_next != NFSNOLIST) {
 				LIST_REMOVE(bp, nb_vnbufs);
 				bp->nb_vnbufs.le_next = NFSNOLIST;
 			}
-			bp->nb_vp = NULL;
+			bp->nb_np = NULL;
 		}
 		LIST_REMOVE(bp, nb_hash);
 		/* nuke any creds we're holding */
-		if (IS_VALID_CRED(bp->nb_rcred)) {
+		if (IS_VALID_CRED(bp->nb_rcred))
 			kauth_cred_unref(&bp->nb_rcred);
-		}
-		if (IS_VALID_CRED(bp->nb_wcred)) {
+		if (IS_VALID_CRED(bp->nb_wcred))
 			kauth_cred_unref(&bp->nb_wcred);
-		}
 		/* if buf will no longer be NB_META, dump old buffer */
 		if (operation == NBLK_META) {
 			if (!ISSET(bp->nb_flags, NB_META))
@@ -811,31 +816,41 @@ loop:
 			MALLOC(bp, struct nfsbuf *, sizeof(struct nfsbuf), M_TEMP, M_WAITOK);
 			if (!bp) {
 				lck_mtx_unlock(nfs_buf_mutex);
-				FSDBG_BOT(541, vp, blkno, 0, error);
+				FSDBG_BOT(541, np, blkno, 0, error);
 				return (ENOMEM);
 			}
 			nfsbufcnt++;
+
+			/*
+			 * If any excess bufs, make sure the timer
+			 * is running to free them up later.
+			 */
+			if (nfsbufcnt > nfsbufmin && !nfs_buf_timer_on) {
+				nfs_buf_timer_on = 1;
+				nfs_interval_timer_start(nfs_buf_timer_call,
+					NFSBUF_FREE_PERIOD * 1000);
+			}
+
 			if (operation == NBLK_META)
 				nfsbufmetacnt++;
-			NFSBUFCNTCHK(1);
+			NFSBUFCNTCHK();
 			/* init nfsbuf */
 			bzero(bp, sizeof(*bp));
 			bp->nb_free.tqe_next = NFSNOLIST;
 			bp->nb_validoff = bp->nb_validend = -1;
-			FSDBG(545, vp, blkno, bp, 0);
+			FSDBG(545, np, blkno, bp, 0);
 		} else {
 			/* too many bufs... wait for buffers to free up */
-			FSDBG_TOP(546, vp, blkno, nfsbufcnt, nfsbufmax);
+			FSDBG_TOP(546, np, blkno, nfsbufcnt, nfsbufmax);
 
 			/* poke the delwri list */
 			nfs_buf_delwri_push(1);
 
 			nfsneedbuffer = 1;
-			msleep(&nfsneedbuffer, nfs_buf_mutex, PCATCH|PDROP,
-				"nfsbufget", 0);
-			FSDBG_BOT(546, vp, blkno, nfsbufcnt, nfsbufmax);
-			if ((error = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p))) {
-				FSDBG_BOT(541, vp, blkno, 0, error);
+			error = msleep(&nfsneedbuffer, nfs_buf_mutex, PCATCH|PDROP, "nfsbufget", NULL);
+			FSDBG_BOT(546, np, blkno, nfsbufcnt, nfsbufmax);
+			if (error || ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0)))) {
+				FSDBG_BOT(541, np, blkno, 0, error);
 				return (error);
 			}
 			goto loop;
@@ -848,8 +863,8 @@ loop:
 	bp->nb_lblkno = blkno;
 	/* insert buf in hash */
 	LIST_INSERT_HEAD(NFSBUFHASH(np, blkno), bp, nb_hash);
-	/* associate buffer with new vnode */
-	bp->nb_vp = vp;
+	/* associate buffer with new nfsnode */
+	bp->nb_np = np;
 	LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
 
 buffer_setup:
@@ -873,11 +888,11 @@ buffer_setup:
 			bp->nb_data = kalloc(bufsize);
 		if (!bp->nb_data) {
 			/* Ack! couldn't allocate the data buffer! */
-			/* cleanup buffer and return error */
+			/* clean up buffer and return error */
 			lck_mtx_lock(nfs_buf_mutex);
 			LIST_REMOVE(bp, nb_vnbufs);
 			bp->nb_vnbufs.le_next = NFSNOLIST;
-			bp->nb_vp = NULL;
+			bp->nb_np = NULL;
 			/* invalidate usage timestamp to allow immediate freeing */
 			NBUFSTAMPINVALIDATE(bp);
 			if (bp->nb_free.tqe_next != NFSNOLIST)
@@ -885,7 +900,7 @@ buffer_setup:
 			TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free);
 			nfsbuffreecnt++;
 			lck_mtx_unlock(nfs_buf_mutex);
-			FSDBG_BOT(541, vp, blkno, 0xb00, ENOMEM);
+			FSDBG_BOT(541, np, blkno, 0xb00, ENOMEM);
 			return (ENOMEM);
 		}
 		bp->nb_bufsize = bufsize;
@@ -908,15 +923,15 @@ buffer_setup:
 		bp->nb_validoff = bp->nb_validend = -1;
 
 		if (UBCINFOEXISTS(vp)) {
-			/* setup upl */
+			/* set up upl */
 			if (nfs_buf_upl_setup(bp)) {
 				/* unable to create upl */
 				/* vm object must no longer exist */
-				/* cleanup buffer and return error */
+				/* clean up buffer and return error */
 				lck_mtx_lock(nfs_buf_mutex);
 				LIST_REMOVE(bp, nb_vnbufs);
 				bp->nb_vnbufs.le_next = NFSNOLIST;
-				bp->nb_vp = NULL;
+				bp->nb_np = NULL;
 				/* invalidate usage timestamp to allow immediate freeing */
 				NBUFSTAMPINVALIDATE(bp);
 				if (bp->nb_free.tqe_next != NFSNOLIST)
@@ -924,7 +939,7 @@ buffer_setup:
 				TAILQ_INSERT_HEAD(&nfsbuffree, bp, nb_free);
 				nfsbuffreecnt++;
 				lck_mtx_unlock(nfs_buf_mutex);
-				FSDBG_BOT(541, vp, blkno, 0x2bc, EIO);
+				FSDBG_BOT(541, np, blkno, 0x2bc, EIO);
 				return (EIO);
 			}
 			nfs_buf_upl_check(bp);
@@ -937,7 +952,7 @@ buffer_setup:
 
 	*bpp = bp;
 
-	FSDBG_BOT(541, vp, blkno, bp, bp->nb_flags);
+	FSDBG_BOT(541, np, blkno, bp, bp->nb_flags);
 
 	return (0);
 }
@@ -945,7 +960,8 @@ buffer_setup:
 void
 nfs_buf_release(struct nfsbuf *bp, int freeup)
 {
-	vnode_t vp = bp->nb_vp;
+	nfsnode_t np = bp->nb_np;
+	vnode_t vp;
 	struct timeval now;
 	int wakeup_needbuffer, wakeup_buffer, wakeup_nbdwrite;
 
@@ -953,7 +969,8 @@ nfs_buf_release(struct nfsbuf *bp, int freeup)
 	FSDBG(548, bp->nb_validoff, bp->nb_validend, bp->nb_dirtyoff, bp->nb_dirtyend);
 	FSDBG(548, bp->nb_valid, 0, bp->nb_dirty, 0);
 
-	if (UBCINFOEXISTS(vp) && bp->nb_bufsize) {
+	vp = np ? NFSTOV(np) : NULL;
+	if (vp && UBCINFOEXISTS(vp) && bp->nb_bufsize) {
 		int upl_flags;
 		upl_t upl;
 		int i, rv;
@@ -973,10 +990,13 @@ nfs_buf_release(struct nfsbuf *bp, int freeup)
 				panic("ubc_upl_unmap failed");
 			bp->nb_data = NULL;
 		}
-		/* abort pages if error, invalid, or non-needcommit nocache */
-		if ((bp->nb_flags & (NB_ERROR | NB_INVAL)) ||
-		    ((bp->nb_flags & NB_NOCACHE) && !(bp->nb_flags & (NB_NEEDCOMMIT | NB_DELWRI)))) {
-			if (bp->nb_flags & (NB_READ | NB_INVAL | NB_NOCACHE))
+		/*
+		 * Abort the pages on error or: if this is an invalid or
+		 * non-needcommit nocache buffer AND no pages are dirty.
+		 */
+		if (ISSET(bp->nb_flags, NB_ERROR) || (!bp->nb_dirty && (ISSET(bp->nb_flags, NB_INVAL) ||
+		    (ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, (NB_NEEDCOMMIT | NB_DELWRI)))))) {
+			if (ISSET(bp->nb_flags, (NB_READ | NB_INVAL | NB_NOCACHE)))
 				upl_flags = UPL_ABORT_DUMP_PAGES;
 			else
 				upl_flags = 0;
@@ -1003,10 +1023,10 @@ nfs_buf_release(struct nfsbuf *bp, int freeup)
 		}
 pagelist_cleanup_done:
 		/* was this the last buffer in the file? */
-		if (NBOFF(bp) + bp->nb_bufsize > (off_t)(VTONFS(vp)->n_size)) {
+		if (NBOFF(bp) + bp->nb_bufsize > (off_t)(np->n_size)) {
 			/* if so, invalidate all pages of last buffer past EOF */
 			off_t start, end;
-			start = trunc_page_64(VTONFS(vp)->n_size) + PAGE_SIZE_64;
+			start = trunc_page_64(np->n_size) + PAGE_SIZE_64;
 			end = trunc_page_64(NBOFF(bp) + bp->nb_bufsize);
 			if (end > start) {
 				if (!(rv = ubc_sync_range(vp, start, end, UBC_INVALIDATE)))
@@ -1038,18 +1058,18 @@ pagelist_cleanup_done:
 		SET(bp->nb_flags, NB_INVAL);
 
 	if ((bp->nb_bufsize <= 0) || ISSET(bp->nb_flags, NB_INVAL)) {
-		/* If it's invalid or empty, dissociate it from its vnode */
+		/* If it's invalid or empty, dissociate it from its nfsnode */
 		if (bp->nb_vnbufs.le_next != NFSNOLIST) {
 			LIST_REMOVE(bp, nb_vnbufs);
 			bp->nb_vnbufs.le_next = NFSNOLIST;
 		}
-		bp->nb_vp = NULL;
+		bp->nb_np = NULL;
 		/* if this was a delayed write, wakeup anyone */
 		/* waiting for delayed writes to complete */
 		if (ISSET(bp->nb_flags, NB_DELWRI)) {
 			CLR(bp->nb_flags, NB_DELWRI);
-			OSAddAtomic(-1, (SInt32*)&nfs_nbdwrite);
-			NFSBUFCNTCHK(1);
+			nfs_nbdwrite--;
+			NFSBUFCNTCHK();
 			wakeup_nbdwrite = 1;
 		}
 		/* invalidate usage timestamp to allow immediate freeing */
@@ -1088,10 +1108,10 @@ pagelist_cleanup_done:
 		}
 	}
 
-	NFSBUFCNTCHK(1);
+	NFSBUFCNTCHK();
 
 	/* Unlock the buffer. */
-	CLR(bp->nb_flags, (NB_ASYNC | NB_STABLE | NB_IOD));
+	CLR(bp->nb_flags, (NB_ASYNC | NB_STABLE));
 	CLR(bp->nb_lflags, NBL_BUSY);
 
 	FSDBG_BOT(548, bp, NBOFF(bp), bp->nb_flags, bp->nb_data);
@@ -1120,7 +1140,7 @@ nfs_buf_iowait(struct nfsbuf *bp)
 	lck_mtx_lock(nfs_buf_mutex);
 
 	while (!ISSET(bp->nb_flags, NB_DONE))
-		msleep(bp, nfs_buf_mutex, PRIBIO + 1, "nfs_buf_iowait", 0);
+		msleep(bp, nfs_buf_mutex, PRIBIO + 1, "nfs_buf_iowait", NULL);
 
 	lck_mtx_unlock(nfs_buf_mutex);
 
@@ -1146,11 +1166,6 @@ nfs_buf_iodone(struct nfsbuf *bp)
 
 	if (ISSET(bp->nb_flags, NB_DONE))
 		panic("nfs_buf_iodone already");
-	/*
-	 * I/O was done, so don't believe
-	 * the DIRTY state from VM anymore
-	 */
-	CLR(bp->nb_flags, NB_WASDIRTY);
 
 	if (!ISSET(bp->nb_flags, NB_READ)) {
 		CLR(bp->nb_flags, NB_WRITEINPROG);
@@ -1158,7 +1173,7 @@ nfs_buf_iodone(struct nfsbuf *bp)
 		 * vnode_writedone() takes care of waking up
 		 * any throttled write operations
 		 */
-		vnode_writedone(bp->nb_vp);
+		vnode_writedone(NFSTOV(bp->nb_np));
 	}
 	if (ISSET(bp->nb_flags, NB_ASYNC)) {	/* if async, release it */
 		SET(bp->nb_flags, NB_DONE);		/* note that it's done */
@@ -1175,9 +1190,9 @@ nfs_buf_iodone(struct nfsbuf *bp)
 }
 
 void
-nfs_buf_write_delayed(struct nfsbuf *bp, proc_t p)
+nfs_buf_write_delayed(struct nfsbuf *bp)
 {
-	vnode_t vp = bp->nb_vp;
+	nfsnode_t np = bp->nb_np;
 
 	FSDBG_TOP(551, bp, NBOFF(bp), bp->nb_flags, 0);
 	FSDBG(551, bp, bp->nb_dirtyoff, bp->nb_dirtyend, bp->nb_dirty);
@@ -1185,20 +1200,17 @@ nfs_buf_write_delayed(struct nfsbuf *bp, proc_t p)
 	/*
 	 * If the block hasn't been seen before:
 	 *	(1) Mark it as having been seen,
-	 *	(2) Charge for the write.
-	 *	(3) Make sure it's on its vnode's correct block list,
+	 *	(2) Make sure it's on its node's correct block list,
 	 */
 	if (!ISSET(bp->nb_flags, NB_DELWRI)) {
 		SET(bp->nb_flags, NB_DELWRI);
-		if (p && p->p_stats) 
-			p->p_stats->p_ru.ru_oublock++;		/* XXX */
-		OSAddAtomic(1, (SInt32*)&nfs_nbdwrite);
-		NFSBUFCNTCHK(0);
 		/* move to dirty list */
 		lck_mtx_lock(nfs_buf_mutex);
+		nfs_nbdwrite++;
+		NFSBUFCNTCHK();
 		if (bp->nb_vnbufs.le_next != NFSNOLIST)
 			LIST_REMOVE(bp, nb_vnbufs);
-		LIST_INSERT_HEAD(&VTONFS(vp)->n_dirtyblkhd, bp, nb_vnbufs);
+		LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
 		lck_mtx_unlock(nfs_buf_mutex);
 	}
 
@@ -1206,24 +1218,27 @@ nfs_buf_write_delayed(struct nfsbuf *bp, proc_t p)
 	 * If the vnode has "too many" write operations in progress
 	 * wait for them to finish the IO
 	 */
-	(void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, "nfs_buf_write_delayed");
+	vnode_waitforwrites(NFSTOV(np), VNODE_ASYNC_THROTTLE, 0, 0, "nfs_buf_write_delayed");
+
+	/* the file is in a modified state, so make sure the flag's set */
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
+	np->n_flag |= NMODIFIED;
+	nfs_unlock(np);
 
 	/*
-	 * If we have too many delayed write buffers, 
-	 * more than we can "safely" handle, just fall back to
-	 * doing the async write
+	 * If we have too many delayed write buffers,
+	 * just fall back to doing the async write.
 	 */
 	if (nfs_nbdwrite < 0)
 		panic("nfs_buf_write_delayed: Negative nfs_nbdwrite");
-
-	if (nfs_nbdwrite > ((nfsbufcnt/4)*3)) {
+	if (nfs_nbdwrite > NFS_A_LOT_OF_DELAYED_WRITES) {
 		/* issue async write */
 		SET(bp->nb_flags, NB_ASYNC);
 		nfs_buf_write(bp);
 		FSDBG_BOT(551, bp, NBOFF(bp), bp->nb_flags, bp->nb_error);
 		return;
 	}
-	 
+
 	/* Otherwise, the "write" is done, so mark and release the buffer. */
 	SET(bp->nb_flags, NB_DONE);
 	nfs_buf_release(bp, 1);
@@ -1237,21 +1252,26 @@ nfs_buf_write_delayed(struct nfsbuf *bp, proc_t p)
  * the needcommit flag.
  */
 void
-nfs_buf_check_write_verifier(struct nfsnode *np, struct nfsbuf *bp)
+nfs_buf_check_write_verifier(nfsnode_t np, struct nfsbuf *bp)
 {
 	struct nfsmount *nmp;
 
 	if (!ISSET(bp->nb_flags, NB_NEEDCOMMIT))
 		return;
 
-	nmp = VFSTONFS(vnode_mount(NFSTOV(np)));
-	if (!nmp || (bp->nb_verf == nmp->nm_verf))
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return;
+	if (!ISSET(bp->nb_flags, NB_STALEWVERF) && (bp->nb_verf == nmp->nm_verf))
 		return;
 
-	/* write verifier changed, clear commit flag */
-	bp->nb_flags &= ~NB_NEEDCOMMIT;
+	/* write verifier changed, clear commit/wverf flags */
+	CLR(bp->nb_flags, (NB_NEEDCOMMIT | NB_STALEWVERF));
+	bp->nb_verf = 0;
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
 	np->n_needcommitcnt--;
 	CHECK_NEEDCOMMITCNT(np);
+	nfs_unlock(np);
 }
 
 /*
@@ -1293,15 +1313,15 @@ nfs_buf_acquire(struct nfsbuf *bp, int flags, int slpflag, int slptimeo)
 			return (EBUSY);
 	        SET(bp->nb_lflags, NBL_WANTED);
 
-			ts.tv_sec = (slptimeo/100);
-			/* the hz value is 100; which leads to 10ms */
-			ts.tv_nsec = (slptimeo % 100) * 10  * NSEC_PER_USEC * 1000;
+		ts.tv_sec = (slptimeo/100);
+		/* the hz value is 100; which leads to 10ms */
+		ts.tv_nsec = (slptimeo % 100) * 10  * NSEC_PER_USEC * 1000;
 
-			error = msleep(bp, nfs_buf_mutex, slpflag | (PRIBIO + 1),
-				"nfs_buf_acquire", &ts);
-			if (error)
-				return (error);
-			return (EAGAIN);
+		error = msleep(bp, nfs_buf_mutex, slpflag | (PRIBIO + 1),
+			"nfs_buf_acquire", &ts);
+		if (error)
+			return (error);
+		return (EAGAIN);
 	}
 	if (flags & NBAC_REMOVE)
 	        nfs_buf_remfree(bp);
@@ -1322,10 +1342,7 @@ nfs_buf_drop(struct nfsbuf *bp)
 	if (!ISSET(bp->nb_lflags, NBL_BUSY))
 		panic("nfs_buf_drop: buffer not busy!");
 	if (ISSET(bp->nb_lflags, NBL_WANTED)) {
-	        /*	
-		 * delay the actual wakeup until after we
-		 * clear NBL_BUSY and we've dropped nfs_buf_mutex
-		 */
+	        /* delay the actual wakeup until after we clear NBL_BUSY */
 		need_wakeup = 1;
 	}
 	/* Unlock the buffer. */
@@ -1341,7 +1358,7 @@ nfs_buf_drop(struct nfsbuf *bp)
  * (must be called with nfs_buf_mutex held)
  */
 int
-nfs_buf_iterprepare(struct nfsnode *np, struct nfsbuflists *iterheadp, int flags)
+nfs_buf_iterprepare(nfsnode_t np, struct nfsbuflists *iterheadp, int flags)
 {
 	struct nfsbuflists *listheadp;
 
@@ -1357,7 +1374,7 @@ nfs_buf_iterprepare(struct nfsnode *np, struct nfsbuflists *iterheadp, int flags
 
 	while (np->n_bufiterflags & NBI_ITER) 	{
 	        np->n_bufiterflags |= NBI_ITERWANT;
-		msleep(&np->n_bufiterflags, nfs_buf_mutex, 0, "nfs_buf_iterprepare", 0);	
+		msleep(&np->n_bufiterflags, nfs_buf_mutex, 0, "nfs_buf_iterprepare", NULL);
 	}
 	if (LIST_EMPTY(listheadp)) {
 	        LIST_INIT(iterheadp);
@@ -1373,12 +1390,12 @@ nfs_buf_iterprepare(struct nfsnode *np, struct nfsbuflists *iterheadp, int flags
 }
 
 /*
- * cleanup after iterating over an nfsnode's buffer list
+ * clean up after iterating over an nfsnode's buffer list
  * this lock protects the queue manipulation
  * (must be called with nfs_buf_mutex held)
  */
 void
-nfs_buf_itercomplete(struct nfsnode *np, struct nfsbuflists *iterheadp, int flags)
+nfs_buf_itercomplete(nfsnode_t np, struct nfsbuflists *iterheadp, int flags)
 {
 	struct nfsbuflists * listheadp;
 	struct nfsbuf *bp;
@@ -1403,243 +1420,606 @@ nfs_buf_itercomplete(struct nfsnode *np, struct nfsbuflists *iterheadp, int flag
 
 
 /*
- * Vnode op for read using bio
- * Any similarity to readip() is purely coincidental
+ * Read an NFS buffer for a file.
  */
 int
-nfs_bioread(
-	vnode_t vp,
-	struct uio *uio,
-	__unused int ioflag,
-	kauth_cred_t cred,
-	proc_t p)
+nfs_buf_read(struct nfsbuf *bp)
 {
-	struct nfsnode *np = VTONFS(vp);
-	int biosize;
-	off_t diff;
-	struct nfsbuf *bp = NULL, *rabp;
-	struct nfs_vattr nvattr;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-	daddr64_t lbn, rabn, lastrabn = -1, tlbn;
-	int bufsize;
-	int nra, error = 0, n = 0, on = 0;
-	caddr_t dp;
-	struct dirent *direntp = NULL;
-	enum vtype vtype;
-	int nocachereadahead = 0;
+	int error = 0;
+	nfsnode_t np;
+	thread_t thd;
+	kauth_cred_t cred;
 
-	FSDBG_TOP(514, vp, uio->uio_offset, uio_uio_resid(uio), ioflag);
+	np = bp->nb_np;
+	cred = bp->nb_rcred;
+	if (IS_VALID_CRED(cred))
+		kauth_cred_ref(cred);
+	thd = ISSET(bp->nb_flags, NB_ASYNC) ? NULL : current_thread();
 
-#if DIAGNOSTIC
-	if (uio->uio_rw != UIO_READ)
-		panic("nfs_read mode");
-#endif
-	if (uio_uio_resid(uio) == 0) {
-		FSDBG_BOT(514, vp, 0xd1e0001, 0, 0);
-		return (0);
-	}
-	if (uio->uio_offset < 0) {
-		FSDBG_BOT(514, vp, 0xd1e0002, 0, EINVAL);
-		return (EINVAL);
-	}
+	/* sanity checks */
+	if (!ISSET(bp->nb_flags, NB_READ))
+		panic("nfs_buf_read: !NB_READ");
+	if (ISSET(bp->nb_flags, NB_DONE))
+		CLR(bp->nb_flags, NB_DONE);
 
-	biosize = nmp->nm_biosize;
-	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		nfs_fsinfo(nmp, vp, cred, p);
+	NFS_BUF_MAP(bp);
 
-	vtype = vnode_vtype(vp);
+	OSAddAtomic(1, (SInt32 *)&nfsstats.read_bios);
+
+	error = nfs_buf_read_rpc(bp, thd, cred);
 	/*
-	 * For nfs, cache consistency can only be maintained approximately.
-	 * Although RFC1094 does not specify the criteria, the following is
-	 * believed to be compatible with the reference port.
-	 * For nfs:
-	 * If the file's modify time on the server has changed since the
-	 * last read rpc or you have written to the file,
-	 * you may have lost data cache consistency with the
-	 * server, so flush all of the file's data out of the cache.
-	 * Then force a getattr rpc to ensure that you have up to date
-	 * attributes.
-	 * NB: This implies that cache data can be read when up to
-	 * NFS_MAXATTRTIMEO seconds out of date. If you find that you need
-	 * current attributes this could be forced by setting calling
-	 * NATTRINVALIDATE() before the nfs_getattr() call.
+	 * For async I/O, the callbacks will finish up the
+	 * read.  Otherwise, the read has already been finished.
 	 */
-	if (np->n_flag & NNEEDINVALIDATE) {
-		np->n_flag &= ~NNEEDINVALIDATE;
-		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cred, p, 1);
-	}
-	if (np->n_flag & NMODIFIED) {
-		if (vtype != VREG) {
-			if (vtype != VDIR)
-				panic("nfs: bioread, not dir");
-			nfs_invaldir(vp);
-			error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-			if (error) {
-				FSDBG_BOT(514, vp, 0xd1e0003, 0, error);
-				return (error);
-			}
-		}
-		NATTRINVALIDATE(np);
-		error = nfs_getattr(vp, &nvattr, cred, p);
-		if (error) {
-			FSDBG_BOT(514, vp, 0xd1e0004, 0, error);
-			return (error);
+
+	if (IS_VALID_CRED(cred))
+		kauth_cred_unref(&cred);
+	return (error);
+}
+
+/*
+ * finish the reading of a buffer
+ */
+void
+nfs_buf_read_finish(struct nfsbuf *bp)
+{
+	nfsnode_t np = bp->nb_np;
+	struct nfsmount *nmp;
+
+	if (!ISSET(bp->nb_flags, NB_ERROR)) {
+		/* update valid range */
+		bp->nb_validoff = 0;
+		bp->nb_validend = bp->nb_endio;
+		if (bp->nb_endio < bp->nb_bufsize) { 
+			/*
+			 * The read may be short because we have unflushed writes
+			 * that are extending the file size and the reads hit the
+			 * (old) EOF on the server.  So, just make sure nb_validend
+			 * correctly tracks EOF.
+			 * Note that the missing data should have already been zeroed
+			 * in nfs_buf_read_rpc_finish().
+			 */
+			off_t boff = NBOFF(bp);
+			if ((off_t)np->n_size >= (boff + bp->nb_bufsize))
+				bp->nb_validend = bp->nb_bufsize;
+			else if ((off_t)np->n_size >= boff)
+				bp->nb_validend = np->n_size - boff;
+			else
+				bp->nb_validend = 0;
 		}
-		if (vtype == VDIR) {
-			/* if directory changed, purge any name cache entries */
-			if (nfstimespeccmp(&np->n_ncmtime, &nvattr.nva_mtime, !=))
-				cache_purge(vp);
-			np->n_ncmtime = nvattr.nva_mtime;
+		if ((nmp = NFSTONMP(np)) && (nmp->nm_vers == NFS_VER2) &&
+		    ((NBOFF(bp) + bp->nb_validend) > 0x100000000LL))
+			bp->nb_validend = 0x100000000LL - NBOFF(bp);
+		bp->nb_valid = (1 << (round_page_32(bp->nb_validend) / PAGE_SIZE)) - 1;
+		if (bp->nb_validend & PAGE_MASK) {
+			/* zero-fill remainder of last page */
+			bzero(bp->nb_data + bp->nb_validend, bp->nb_bufsize - bp->nb_validend);
 		}
-		np->n_mtime = nvattr.nva_mtime;
-	} else {
-		error = nfs_getattr(vp, &nvattr, cred, p);
-		if (error) {
-			FSDBG_BOT(514, vp, 0xd1e0005, 0, error);
+	}
+	nfs_buf_iodone(bp);
+}
+
+/*
+ * initiate the NFS READ RPC(s) for a buffer
+ */
+int
+nfs_buf_read_rpc(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
+{
+	struct nfsmount *nmp;
+	nfsnode_t np = bp->nb_np;
+	int error = 0, nfsvers, async;
+	int offset, length, nmrsize, nrpcs, len;
+	off_t boff;
+	struct nfsreq *req;
+	struct nfsreq_cbinfo cb;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		bp->nb_error = error = ENXIO;
+		SET(bp->nb_flags, NB_ERROR);
+		nfs_buf_iodone(bp);
+		return (error);
+	}
+	nfsvers = nmp->nm_vers;
+	nmrsize = nmp->nm_rsize;
+
+	boff = NBOFF(bp);
+	offset = 0;
+	length = bp->nb_bufsize;
+
+	if (nfsvers == NFS_VER2) {
+		if (boff > 0xffffffffLL) {
+			bp->nb_error = error = EFBIG;
+			SET(bp->nb_flags, NB_ERROR);
+			nfs_buf_iodone(bp);
 			return (error);
 		}
-		if (nfstimespeccmp(&np->n_mtime, &nvattr.nva_mtime, !=)) {
-			if (vtype == VDIR) {
-				nfs_invaldir(vp);
-				/* purge name cache entries */
-				if (nfstimespeccmp(&np->n_ncmtime, &nvattr.nva_mtime, !=))
-					cache_purge(vp);
-			}
-			error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-			if (error) {
-				FSDBG_BOT(514, vp, 0xd1e0006, 0, error);
-				return (error);
-			}
-			if (vtype == VDIR)
-				np->n_ncmtime = nvattr.nva_mtime;
-			np->n_mtime = nvattr.nva_mtime;
-		}
+		if ((boff + length - 1) > 0xffffffffLL)
+			length = 0x100000000LL - boff;
 	}
 
-	if (vnode_isnocache(vp)) {
-		if (!(np->n_flag & NNOCACHE)) {
-			if (NVALIDBUFS(np)) {
-				error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-				if (error) {
-					FSDBG_BOT(514, vp, 0xd1e000a, 0, error);
-					return (error);
-				}
-			}
-			np->n_flag |= NNOCACHE;
-		}
-	} else if (np->n_flag & NNOCACHE) {
-		np->n_flag &= ~NNOCACHE;
+	/* Note: Can only do async I/O if nfsiods are configured. */
+	async = (bp->nb_flags & NB_ASYNC);
+	cb.rcb_func = async ? nfs_buf_read_rpc_finish : NULL;
+	cb.rcb_bp = bp;
+
+	bp->nb_offio = bp->nb_endio = 0;
+	bp->nb_rpcs = nrpcs = (length + nmrsize - 1) / nmrsize;
+	if (async && (nrpcs > 1)) {
+		SET(bp->nb_flags, NB_MULTASYNCRPC);
+	} else {
+		CLR(bp->nb_flags, NB_MULTASYNCRPC);
 	}
 
-	do {
-	    if (np->n_flag & NNOCACHE) {
-		switch (vtype) {
-		case VREG:
-			/*
-			 * If we have only a block or so to read,
-			 *     just do the rpc directly.
-			 * If we have a couple blocks or more to read,
-			 *     then we'll take advantage of readahead within
-			 *     this loop to try to fetch all the data in parallel
-			 */
-			if (!nocachereadahead && (uio_uio_resid(uio) < 2*biosize)) {
-				error = nfs_readrpc(vp, uio, cred, p);
-				FSDBG_BOT(514, vp, uio->uio_offset, uio_uio_resid(uio), error);
-				return (error);
-			}
-			nocachereadahead = 1;
+	while (length > 0) {
+		if (ISSET(bp->nb_flags, NB_ERROR)) {
+			error = bp->nb_error;
 			break;
-		case VLNK:
-			error = nfs_readlinkrpc(vp, uio, cred, p);
-			FSDBG_BOT(514, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			return (error);
-		case VDIR:
+		}
+		len = (length > nmrsize) ? nmrsize : length;
+		cb.rcb_args[0] = offset;
+		cb.rcb_args[1] = len;
+		req = NULL;
+		error = nmp->nm_funcs->nf_read_rpc_async(np, boff + offset, len, thd, cred, &cb, &req);
+		if (error)
 			break;
-		default:
-			printf(" NFSNOCACHE: type %x unexpected\n", vtype);
-		};
-	    }
-	    switch (vtype) {
-	    case VREG:
-		lbn = uio->uio_offset / biosize;
+		offset += len;
+		length -= len;
+		if (async)
+			continue;
+		nfs_buf_read_rpc_finish(req);
+		if (ISSET(bp->nb_flags, NB_ERROR)) {
+			error = bp->nb_error;
+			break;
+		}
+	}
 
+	if (length > 0) {
 		/*
-		 * Copy directly from any cached pages without grabbing the bufs.
-		 *
-		 * Note: for "nocache" reads, we don't copy directly from UBC
-		 * because any cached pages will be for readahead buffers that
-		 * need to be invalidated anyway before we finish this request.
+		 * Something bad happened while trying to send the RPC(s).
+		 * Wait for any outstanding requests to complete.
 		 */
-		if (!(np->n_flag & NNOCACHE) &&
-			(uio->uio_segflg == UIO_USERSPACE32 || 
-			 uio->uio_segflg == UIO_USERSPACE64 ||
-			 uio->uio_segflg == UIO_USERSPACE)) {
-			// LP64todo - fix this!
-			int io_resid = uio_uio_resid(uio);
-			diff = np->n_size - uio->uio_offset;
-			if (diff < io_resid)
-				io_resid = diff;
-			if (io_resid > 0) {
-				error = cluster_copy_ubc_data(vp, uio, &io_resid, 0);
-				if (error) {
-					FSDBG_BOT(514, vp, uio->uio_offset, 0xcacefeed, error);
-					return (error);
-				}
-			}
-			/* count any biocache reads that we just copied directly */
-			if (lbn != uio->uio_offset / biosize) {
-				OSAddAtomic((uio->uio_offset / biosize) - lbn, (SInt32*)&nfsstats.biocache_reads);
-				FSDBG(514, vp, 0xcacefeed, uio->uio_offset, error);
+		bp->nb_error = error;
+		SET(bp->nb_flags, NB_ERROR);
+		if (ISSET(bp->nb_flags, NB_MULTASYNCRPC)) {
+			nrpcs = (length + nmrsize - 1) / nmrsize;
+			lck_mtx_lock(nfs_buf_mutex);
+			bp->nb_rpcs -= nrpcs;
+			if (bp->nb_rpcs == 0) {
+				/* No RPCs left, so the buffer's done */
+				lck_mtx_unlock(nfs_buf_mutex);
+				nfs_buf_iodone(bp);
+			} else {
+				/* wait for the last RPC to mark it done */
+				while (bp->nb_rpcs > 0)
+					msleep(&bp->nb_rpcs, nfs_buf_mutex, 0,
+						"nfs_buf_read_rpc_cancel", NULL);
+				lck_mtx_unlock(nfs_buf_mutex);
 			}
+		} else {
+			nfs_buf_iodone(bp);
 		}
+	}
 
-		lbn = uio->uio_offset / biosize;
-		on = uio->uio_offset % biosize;
+	return (error);
+}
 
-		/*
-		 * Start the read ahead(s), as required.
-		 */
-		if (nfs_numasync > 0 && nmp->nm_readahead > 0) {
-			for (nra = 0; nra < nmp->nm_readahead; nra++) {
-				rabn = lbn + 1 + nra;
-				if (rabn <= lastrabn) {
-					/* we've already (tried to) read this block */
-					/* no need to try it again... */
-					continue;
-				}
-				lastrabn = rabn;
-				if ((off_t)rabn * biosize >= (off_t)np->n_size)
-					break;
-				if ((np->n_flag & NNOCACHE) &&
-				   (((off_t)rabn * biosize) >= (uio->uio_offset + uio_uio_resid(uio))))
-					/* for uncached readahead, don't go beyond end of request */
-					break;
-				/* check if block exists and is valid. */
-				error = nfs_buf_get(vp, rabn, biosize, p, NBLK_READ|NBLK_NOWAIT, &rabp);
+/*
+ * finish up an NFS READ RPC on a buffer
+ */
+void
+nfs_buf_read_rpc_finish(struct nfsreq *req)
+{
+	struct nfsmount *nmp;
+	size_t rlen;
+	struct nfsreq_cbinfo cb;
+	struct nfsbuf *bp;
+	int error = 0, nfsvers, offset, length, eof = 0, multasyncrpc, finished;
+	void *wakeme = NULL;
+	struct nfsreq *rreq = NULL;
+	nfsnode_t np;
+	thread_t thd;
+	kauth_cred_t cred;
+	struct uio uio;
+	struct iovec_32 io;
+
+finish:
+	np = req->r_np;
+	thd = req->r_thread;
+	cred = req->r_cred;
+	if (IS_VALID_CRED(cred))
+		kauth_cred_ref(cred);
+	cb = req->r_callback;
+	bp = cb.rcb_bp;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error = ENXIO;
+	}
+	if (error || ISSET(bp->nb_flags, NB_ERROR)) {
+		/* just drop it */
+		nfs_request_async_cancel(req);
+		goto out;
+	}
+
+	nfsvers = nmp->nm_vers;
+	offset = cb.rcb_args[0];
+	rlen = length = cb.rcb_args[1];
+
+	uio.uio_iovs.iov32p = &io;
+	uio.uio_iovcnt = 1;
+	uio.uio_rw = UIO_READ;
+#if 1	/* LP64todo - can't use new segment flags until the drivers are ready */
+	uio.uio_segflg = UIO_SYSSPACE;
+#else
+	uio.uio_segflg = UIO_SYSSPACE32;
+#endif
+	io.iov_len = length;
+	uio_uio_resid_set(&uio, io.iov_len);
+	uio.uio_offset = NBOFF(bp) + offset;
+	io.iov_base = (uintptr_t) bp->nb_data + offset;
+
+	/* finish the RPC */
+	error = nmp->nm_funcs->nf_read_rpc_async_finish(np, req, &uio, &rlen, &eof);
+	if ((error == EINPROGRESS) && cb.rcb_func) {
+		/* async request restarted */
+		if (IS_VALID_CRED(cred))
+			kauth_cred_unref(&cred);
+		return;
+	}
+
+	if (error) {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error;
+		goto out;
+	}
+
+	if ((rlen > 0) && (bp->nb_endio < (offset + (int)rlen)))
+		bp->nb_endio = offset + rlen;
+
+	if ((nfsvers == NFS_VER2) || eof || (rlen == 0)) {
+		/* zero out the remaining data (up to EOF) */
+		off_t rpcrem, eofrem, rem;
+		rpcrem = (length - rlen);
+		eofrem = np->n_size - (NBOFF(bp) + offset + rlen);
+		rem = (rpcrem < eofrem) ? rpcrem : eofrem;
+		if (rem > 0)
+			bzero(bp->nb_data + offset + rlen, rem);
+	} else if (((int)rlen < length) && !ISSET(bp->nb_flags, NB_ERROR)) {
+		/*
+		 * short read
+		 *
+		 * We haven't hit EOF and we didn't get all the data
+		 * requested, so we need to issue another read for the rest.
+		 * (Don't bother if the buffer already hit an error.)
+		 */
+		offset += rlen;
+		length -= rlen;
+		cb.rcb_args[0] = offset;
+		cb.rcb_args[1] = length;
+		error = nmp->nm_funcs->nf_read_rpc_async(np, offset, length, thd, cred, &cb, &rreq);
+		if (!error) {
+			if (IS_VALID_CRED(cred))
+				kauth_cred_unref(&cred);
+			if (!cb.rcb_func) {
+				/* if !async we'll need to wait for this RPC to finish */
+				req = rreq;
+				goto finish;
+			}
+			/*
+			 * We're done here.
+			 * Outstanding RPC count is unchanged.
+			 * Callback will be called when RPC is done.
+			 */
+			return;
+		}
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error;
+	}
+
+out:
+	if (IS_VALID_CRED(cred))
+		kauth_cred_unref(&cred);
+
+	/*
+	 * Decrement outstanding RPC count on buffer
+	 * and call nfs_buf_read_finish on last RPC.
+	 *
+	 * (Note: when there are multiple async RPCs issued for a
+	 * buffer we need nfs_buffer_mutex to avoid problems when
+	 * aborting a partially-initiated set of RPCs)
+	 */
+
+	multasyncrpc = ISSET(bp->nb_flags, NB_MULTASYNCRPC);
+	if (multasyncrpc)
+		lck_mtx_lock(nfs_buf_mutex);
+
+	bp->nb_rpcs--;
+	finished = (bp->nb_rpcs == 0);
+
+	if (multasyncrpc)
+		lck_mtx_unlock(nfs_buf_mutex);
+
+	if (finished) {
+		if (multasyncrpc)
+			wakeme = &bp->nb_rpcs;
+		nfs_buf_read_finish(bp);
+		if (wakeme)
+			wakeup(wakeme);
+	}
+}
+
+/*
+ * Do buffer readahead.
+ * Initiate async I/O to read buffers not in cache.
+ */
+static int
+nfs_buf_readahead(nfsnode_t np, int ioflag, daddr64_t *rabnp, daddr64_t lastrabn, thread_t thd, kauth_cred_t cred)
+{
+	struct nfsmount *nmp = NFSTONMP(np);
+	struct nfsbuf *bp;
+	int error = 0, nra;
+
+	if (!nmp)
+		return (ENXIO);
+	if (nmp->nm_readahead <= 0)
+		return (0);
+	if (*rabnp > lastrabn)
+		return (0);
+
+	for (nra = 0; (nra < nmp->nm_readahead) && (*rabnp <= lastrabn); nra++, *rabnp = *rabnp + 1) {
+		/* check if block exists and is valid. */
+		error = nfs_buf_get(np, *rabnp, nmp->nm_biosize, thd, NBLK_READ|NBLK_NOWAIT, &bp);
+		if (error)
+			break;
+		if (!bp)
+			continue;
+		if ((ioflag & IO_NOCACHE) && ISSET(bp->nb_flags, NB_CACHE) &&
+		    !bp->nb_dirty && !ISSET(bp->nb_flags, (NB_DELWRI|NB_NCRDAHEAD))) {
+			CLR(bp->nb_flags, NB_CACHE);
+			bp->nb_valid = 0;
+			bp->nb_validoff = bp->nb_validend = -1;
+		}
+		if ((bp->nb_dirtyend <= 0) && !bp->nb_dirty &&
+		    !ISSET(bp->nb_flags, (NB_CACHE|NB_DELWRI))) {
+			SET(bp->nb_flags, (NB_READ|NB_ASYNC));
+			if (ioflag & IO_NOCACHE)
+				SET(bp->nb_flags, NB_NCRDAHEAD);
+			if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
+				kauth_cred_ref(cred);
+				bp->nb_rcred = cred;
+			}
+			if ((error = nfs_buf_read(bp)))
+				break;
+			continue;
+		}
+		nfs_buf_release(bp, 1);
+	}
+	return (error);
+}
+
+/*
+ * NFS buffer I/O for reading files/directories.
+ */
+int
+nfs_bioread(nfsnode_t np, struct uio *uio, int ioflag, int *eofflag, vfs_context_t ctx)
+{
+	vnode_t vp = NFSTOV(np);
+	struct nfsbuf *bp = NULL;
+	struct nfs_vattr nvattr;
+	struct nfsmount *nmp = VTONMP(vp);
+	daddr64_t lbn, rabn = 0, lastrabn, maxrabn = -1, tlbn;
+	off_t diff;
+	int error = 0, n = 0, on = 0;
+	int nfsvers, biosize;
+	caddr_t dp;
+	struct dirent *direntp = NULL;
+	enum vtype vtype;
+	thread_t thd;
+	kauth_cred_t cred;
+
+	FSDBG_TOP(514, np, uio->uio_offset, uio_uio_resid(uio), ioflag);
+
+	if (uio_uio_resid(uio) == 0) {
+		FSDBG_BOT(514, np, 0xd1e0001, 0, 0);
+		return (0);
+	}
+	if (uio->uio_offset < 0) {
+		FSDBG_BOT(514, np, 0xd1e0002, 0, EINVAL);
+		return (EINVAL);
+	}
+
+	nfsvers = nmp->nm_vers;
+	biosize = nmp->nm_biosize;
+	thd = vfs_context_thread(ctx);
+	cred = vfs_context_ucred(ctx);
+
+	vtype = vnode_vtype(vp);
+	if ((vtype != VREG) && (vtype != VDIR)) {
+		printf("nfs_bioread: type %x unexpected\n", vtype);
+		FSDBG_BOT(514, np, 0xd1e0016, 0, EINVAL);
+		return (EINVAL);
+	}
+
+	/*
+	 * For nfs, cache consistency can only be maintained approximately.
+	 * Although RFC1094 does not specify the criteria, the following is
+	 * believed to be compatible with the reference port.
+	 * For nfs:
+	 * If the file's modify time on the server has changed since the
+	 * last read rpc or you have written to the file,
+	 * you may have lost data cache consistency with the
+	 * server, so flush all of the file's data out of the cache.
+	 * Then force a getattr rpc to ensure that you have up to date
+	 * attributes.
+	 * NB: This implies that cache data can be read when up to
+	 * NFS_MAXATTRTIMEO seconds out of date. If you find that you need
+	 * current attributes this could be forced by calling
+	 * NATTRINVALIDATE() before the nfs_getattr() call.
+	 */
+
+	if (ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 0);
+
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))) {
+		FSDBG_BOT(514, np, 0xd1e0222, 0, error);
+		return (error);
+	}
+
+	if (np->n_flag & NNEEDINVALIDATE) {
+		np->n_flag &= ~NNEEDINVALIDATE;
+		nfs_unlock(np);
+		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, ctx, 1);
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))) {
+			FSDBG_BOT(514, np, 0xd1e0322, 0, error);
+			return (error);
+		}
+	}
+
+	if (np->n_flag & NMODIFIED) {
+		if (vtype == VDIR) {
+			nfs_invaldir(np);
+			nfs_unlock(np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			if (!error)
+				error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+			if (error) {
+				FSDBG_BOT(514, np, 0xd1e0003, 0, error);
+				return (error);
+			}
+		}
+		NATTRINVALIDATE(np);
+		error = nfs_getattr(np, &nvattr, ctx, 1);
+		if (error) {
+			nfs_unlock(np);
+			FSDBG_BOT(514, np, 0xd1e0004, 0, error);
+			return (error);
+		}
+		if (vtype == VDIR) {
+			/* if directory changed, purge any name cache entries */
+			if (NFS_CHANGED_NC(nfsvers, np, &nvattr))
+				cache_purge(vp);
+			NFS_CHANGED_UPDATE_NC(nfsvers, np, &nvattr);
+		}
+		NFS_CHANGED_UPDATE(nfsvers, np, &nvattr);
+	} else {
+		error = nfs_getattr(np, &nvattr, ctx, 1);
+		if (error) {
+			nfs_unlock(np);
+			FSDBG_BOT(514, np, 0xd1e0005, 0, error);
+			return (error);
+		}
+		if (NFS_CHANGED(nfsvers, np, &nvattr)) {
+			if (vtype == VDIR) {
+				nfs_invaldir(np);
+				/* purge name cache entries */
+				if (NFS_CHANGED_NC(nfsvers, np, &nvattr))
+					cache_purge(vp);
+			}
+			nfs_unlock(np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			if (!error)
+				error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+			if (error) {
+				FSDBG_BOT(514, np, 0xd1e0006, 0, error);
+				return (error);
+			}
+			if (vtype == VDIR)
+				NFS_CHANGED_UPDATE_NC(nfsvers, np, &nvattr);
+			NFS_CHANGED_UPDATE(nfsvers, np, &nvattr);
+		}
+	}
+
+	nfs_unlock(np);
+
+	if (vtype == VREG) {
+		if ((ioflag & IO_NOCACHE) && (uio_uio_resid(uio) < (2*biosize))) {
+			/* We have only a block or so to read, just do the rpc directly. */
+			error = nfs_read_rpc(np, uio, ctx);
+			FSDBG_BOT(514, np, uio->uio_offset, uio_uio_resid(uio), error);
+			return (error);
+		}
+		/*
+		 * set up readahead - which may be limited by:
+		 * + current request length (for IO_NOCACHE)
+		 * + readahead setting
+		 * + file size
+		 */
+		if (nmp->nm_readahead > 0) {
+			off_t end = uio->uio_offset + uio_uio_resid(uio);
+			if (end > (off_t)np->n_size)
+				end = np->n_size;
+			rabn = uio->uio_offset / biosize;
+			maxrabn = (end - 1) / biosize;
+			if (!(ioflag & IO_NOCACHE) &&
+			    (!rabn || (rabn == np->n_lastread) || (rabn == (np->n_lastread+1)))) {
+				maxrabn += nmp->nm_readahead;
+				if ((maxrabn * biosize) >= (off_t)np->n_size)
+					maxrabn = ((off_t)np->n_size - 1)/biosize;
+			}
+		} else {
+			rabn = maxrabn = 0;
+		}
+	}
+
+	do {
+
+	    if (vtype == VREG) {
+		nfs_data_lock(np, NFS_NODE_LOCK_SHARED);
+		lbn = uio->uio_offset / biosize;
+
+		/*
+		 * Copy directly from any cached pages without grabbing the bufs.
+		 *
+		 * Note: for "nocache" reads, we don't copy directly from UBC
+		 * because any cached pages will be for readahead buffers that
+		 * need to be invalidated anyway before we finish this request.
+		 */
+		if (!(ioflag & IO_NOCACHE) &&
+			(uio->uio_segflg == UIO_USERSPACE32 ||
+			 uio->uio_segflg == UIO_USERSPACE64 ||
+			 uio->uio_segflg == UIO_USERSPACE)) {
+			// LP64todo - fix this!
+			int io_resid = uio_uio_resid(uio);
+			diff = np->n_size - uio->uio_offset;
+			if (diff < io_resid)
+				io_resid = diff;
+			if (io_resid > 0) {
+				error = cluster_copy_ubc_data(vp, uio, &io_resid, 0);
 				if (error) {
-					FSDBG_BOT(514, vp, 0xd1e000b, 1, error);
+					nfs_data_unlock(np);
+					FSDBG_BOT(514, np, uio->uio_offset, 0xcacefeed, error);
 					return (error);
 				}
-				if (!rabp)
-					continue;
-				if (nfs_buf_upl_valid_range(rabp, 0, rabp->nb_bufsize)) {
-					nfs_buf_release(rabp, 1);
-					continue;
-				}
-				if (!ISSET(rabp->nb_flags, (NB_CACHE|NB_DELWRI))) {
-					SET(rabp->nb_flags, (NB_READ|NB_ASYNC));
-					if (nfs_asyncio(rabp, cred)) {
-						SET(rabp->nb_flags, (NB_INVAL|NB_ERROR));
-						rabp->nb_error = EIO;
-						nfs_buf_release(rabp, 1);
-					}
-				} else
-					nfs_buf_release(rabp, 1);
+			}
+			/* count any biocache reads that we just copied directly */
+			if (lbn != (uio->uio_offset / biosize)) {
+				OSAddAtomic((uio->uio_offset / biosize) - lbn, (SInt32*)&nfsstats.biocache_reads);
+				FSDBG(514, np, 0xcacefeed, uio->uio_offset, error);
+			}
+		}
+
+		lbn = uio->uio_offset / biosize;
+		on = uio->uio_offset % biosize;
+		np->n_lastread = (uio->uio_offset - 1) / biosize;
+
+		/* adjust readahead block number, if necessary */
+		if (rabn < lbn)
+			rabn = lbn;
+		lastrabn = MIN(maxrabn, lbn + nmp->nm_readahead);
+		if (rabn <= lastrabn) { /* start readaheads */
+			error = nfs_buf_readahead(np, ioflag, &rabn, lastrabn, thd, cred);
+			if (error) {
+				nfs_data_unlock(np);
+				FSDBG_BOT(514, np, 0xd1e000b, 1, error);
+				return (error);
 			}
 		}
 
 		if ((uio_uio_resid(uio) <= 0) || (uio->uio_offset >= (off_t)np->n_size)) {
-			FSDBG_BOT(514, vp, uio->uio_offset, uio_uio_resid(uio), 0xaaaaaaaa);
+			nfs_data_unlock(np);
+			FSDBG_BOT(514, np, uio->uio_offset, uio_uio_resid(uio), 0xaaaaaaaa);
 			return (0);
 		}
 
@@ -1652,17 +2032,37 @@ nfs_bioread(
 		 * as required.
 		 */
 again:
-		bufsize = biosize;
 		// LP64todo - fix this!
-		n = min((unsigned)(bufsize - on), uio_uio_resid(uio));
+		n = min((unsigned)(biosize - on), uio_uio_resid(uio));
 		diff = np->n_size - uio->uio_offset;
 		if (diff < n)
 			n = diff;
 
-		error = nfs_buf_get(vp, lbn, bufsize, p, NBLK_READ, &bp);
+		error = nfs_buf_get(np, lbn, biosize, thd, NBLK_READ, &bp);
 		if (error) {
-			FSDBG_BOT(514, vp, 0xd1e000c, 0, EINTR);
-			return (EINTR);
+			nfs_data_unlock(np);
+			FSDBG_BOT(514, np, 0xd1e000c, 0, error);
+			return (error);
+		}
+
+		if ((ioflag & IO_NOCACHE) && ISSET(bp->nb_flags, NB_CACHE)) {
+			/*
+			 * IO_NOCACHE found a cached buffer.
+			 * Flush the buffer if it's dirty.
+			 * Invalidate the data if it wasn't just read
+			 * in as part of a "nocache readahead".
+			 */
+			if (bp->nb_dirty || (bp->nb_dirtyend > 0)) {
+				/* so write the buffer out and try again */
+				SET(bp->nb_flags, NB_NOCACHE);
+				goto flushbuffer;
+			}
+			if (!ISSET(bp->nb_flags, NB_NCRDAHEAD)) {
+				CLR(bp->nb_flags, NB_CACHE);
+				bp->nb_valid = 0;
+			} else {
+				CLR(bp->nb_flags, NB_NCRDAHEAD);
+			}
 		}
 
 		/* if any pages are valid... */
@@ -1694,24 +2094,27 @@ again:
 			}
 
 			/* there are invalid pages in the read range */
-			if ((dirtypg > firstpg) && (dirtypg < lastpg)) {
-				/* there are also dirty page(s) in the range, */
+			if (((dirtypg > firstpg) && (dirtypg < lastpg)) ||
+			    (((firstpg*PAGE_SIZE) < bp->nb_dirtyend) && (((lastpg+1)*PAGE_SIZE) > bp->nb_dirtyoff))) {
+				/* there are also dirty page(s) (or range) in the read range, */
 				/* so write the buffer out and try again */
+flushbuffer:
 				CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
 				SET(bp->nb_flags, NB_ASYNC);
-				if (!IS_VALID_CRED(bp->nb_wcred))  {
+				if (!IS_VALID_CRED(bp->nb_wcred)) {
 					kauth_cred_ref(cred);
 					bp->nb_wcred = cred;
 				}
 				error = nfs_buf_write(bp);
 				if (error) {
-					FSDBG_BOT(514, vp, 0xd1e000d, 0, error);
+					nfs_data_unlock(np);
+					FSDBG_BOT(514, np, 0xd1e000d, 0, error);
 					return (error);
 				}
 				goto again;
 			}
 			if (!bp->nb_dirty && bp->nb_dirtyend <= 0 &&
-			    (lastpg - firstpg + 1) > (bufsize/PAGE_SIZE)/2) {
+			    (lastpg - firstpg + 1) > (biosize/PAGE_SIZE)/2) {
 				/* we need to read in more than half the buffer and the */
 				/* buffer's not dirty, so just fetch the whole buffer */
 				bp->nb_valid = 0;
@@ -1721,20 +2124,21 @@ again:
 				char uio_buf[ UIO_SIZEOF(1) ];
 				
 				NFS_BUF_MAP(bp);
-				auio = uio_createwithbuffer(1, (NBOFF(bp) + firstpg * PAGE_SIZE_64), 
+				auio = uio_createwithbuffer(1, (NBOFF(bp) + firstpg * PAGE_SIZE_64),
 						UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf));
 				if (!auio) {
 					error = ENOMEM;
 				} else {
-					uio_addiov(auio, CAST_USER_ADDR_T((bp->nb_data + firstpg * PAGE_SIZE)), 
+					uio_addiov(auio, CAST_USER_ADDR_T((bp->nb_data + firstpg * PAGE_SIZE)),
 							((lastpg - firstpg + 1) * PAGE_SIZE));
-					error = nfs_readrpc(vp, auio, cred, p);
+					error = nfs_read_rpc(np, auio, ctx);
 				}
 				if (error) {
-					if (np->n_flag & NNOCACHE)
+					if (ioflag & IO_NOCACHE)
 						SET(bp->nb_flags, NB_NOCACHE);
 					nfs_buf_release(bp, 1);
-					FSDBG_BOT(514, vp, 0xd1e000e, 0, error);
+					nfs_data_unlock(np);
+					FSDBG_BOT(514, np, 0xd1e000e, 0, error);
 					return (error);
 				}
 				/* Make sure that the valid range is set to cover this read. */
@@ -1753,14 +2157,17 @@ again:
 		}
 		/* if no pages are valid, read the whole block */
 		if (!bp->nb_valid) {
+			if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
+				kauth_cred_ref(cred);
+				bp->nb_rcred = cred;
+			}
 			SET(bp->nb_flags, NB_READ);
 			CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
-			error = nfs_doio(bp, cred, p);
+			error = nfs_buf_read(bp);
 			if (error) {
-				if (np->n_flag & NNOCACHE)
-					SET(bp->nb_flags, NB_NOCACHE);
+				nfs_data_unlock(np);
 				nfs_buf_release(bp, 1);
-				FSDBG_BOT(514, vp, 0xd1e000f, 0, error);
+				FSDBG_BOT(514, np, 0xd1e000f, 0, error);
 				return (error);
 			}
 		}
@@ -1773,50 +2180,37 @@ buffer_ready:
 		}
 		if (n > 0)
 			NFS_BUF_MAP(bp);
-		break;
-	    case VLNK:
-		OSAddAtomic(1, (SInt32*)&nfsstats.biocache_readlinks);
-		error = nfs_buf_get(vp, 0, NFS_MAXPATHLEN, p, NBLK_READ, &bp);
-		if (error) {
-			FSDBG_BOT(514, vp, 0xd1e0010, 0, error);
-			return (error);
-		}
-		if (!ISSET(bp->nb_flags, NB_CACHE)) {
-			SET(bp->nb_flags, NB_READ);
-			error = nfs_doio(bp, cred, p);
-			if (error) {
-				SET(bp->nb_flags, NB_ERROR);
-				nfs_buf_release(bp, 1);
-				FSDBG_BOT(514, vp, 0xd1e0011, 0, error);
-				return (error);
-			}
-		}
-		// LP64todo - fix this!
-		n = min(uio_uio_resid(uio), bp->nb_validend);
-		on = 0;
-		break;
-	    case VDIR:
+	    } else if (vtype == VDIR) {
 		OSAddAtomic(1, (SInt32*)&nfsstats.biocache_readdirs);
-		if (np->n_direofoffset && uio->uio_offset >= np->n_direofoffset) {
-			FSDBG_BOT(514, vp, 0xde0f0001, 0, 0);
+		error = nfs_lock(np, NFS_NODE_LOCK_SHARED);
+		if (error || (np->n_direofoffset && (uio->uio_offset >= np->n_direofoffset))) {
+			if (!error)
+				nfs_unlock(np);
+			if (eofflag)
+				*eofflag = 1;
+			FSDBG_BOT(514, np, 0xde0f0001, 0, 0);
 			return (0);
 		}
+		nfs_unlock(np);
 		lbn = uio->uio_offset / NFS_DIRBLKSIZ;
 		on = uio->uio_offset & (NFS_DIRBLKSIZ - 1);
-		error = nfs_buf_get(vp, lbn, NFS_DIRBLKSIZ, p, NBLK_READ, &bp);
+		error = nfs_buf_get(np, lbn, NFS_DIRBLKSIZ, thd, NBLK_READ, &bp);
 		if (error) {
-			FSDBG_BOT(514, vp, 0xd1e0012, 0, error);
+			FSDBG_BOT(514, np, 0xd1e0012, 0, error);
 			return (error);
 		}
 		if (!ISSET(bp->nb_flags, NB_CACHE)) {
 		    SET(bp->nb_flags, NB_READ);
-		    error = nfs_doio(bp, cred, p);
-		    if (error) {
+		    error = nfs_buf_readdir(bp, ctx);
+		    if (error)
 			nfs_buf_release(bp, 1);
-		    }
 		    while (error == NFSERR_BAD_COOKIE) {
-			nfs_invaldir(vp);
-			error = nfs_vinvalbuf(vp, 0, cred, p, 1);
+			error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+			if (!error) {
+				nfs_invaldir(np);
+				nfs_unlock(np);
+			}
+			error = nfs_vinvalbuf(vp, 0, ctx, 1);
 			/*
 			 * Yuck! The directory has been modified on the
 			 * server. The only way to get the block is by
@@ -1824,25 +2218,34 @@ buffer_ready:
 			 * offset cookies.
 			 */
 			for (tlbn = 0; tlbn <= lbn && !error; tlbn++) {
+			    if ((error = nfs_lock(np, NFS_NODE_LOCK_SHARED)))
+				    break;
 			    if (np->n_direofoffset
 				&& (tlbn * NFS_DIRBLKSIZ) >= np->n_direofoffset) {
-				    FSDBG_BOT(514, vp, 0xde0f0002, 0, 0);
+				    nfs_unlock(np);
+				    if (eofflag)
+					    *eofflag = 1;
+				    FSDBG_BOT(514, np, 0xde0f0002, 0, 0);
 				    return (0);
 			    }
-			    error = nfs_buf_get(vp, tlbn, NFS_DIRBLKSIZ, p, NBLK_READ, &bp);
+			    nfs_unlock(np);
+			    error = nfs_buf_get(np, tlbn, NFS_DIRBLKSIZ, thd, NBLK_READ, &bp);
 			    if (error) {
-				    FSDBG_BOT(514, vp, 0xd1e0013, 0, error);
+				    FSDBG_BOT(514, np, 0xd1e0013, 0, error);
 				    return (error);
 			    }
 			    if (!ISSET(bp->nb_flags, NB_CACHE)) {
 				    SET(bp->nb_flags, NB_READ);
-				    error = nfs_doio(bp, cred, p);
+				    error = nfs_buf_readdir(bp, ctx);
 				    /*
 				     * no error + NB_INVAL == directory EOF,
 				     * use the block.
 				     */
-				    if (error == 0 && (bp->nb_flags & NB_INVAL))
+				    if (error == 0 && ISSET(bp->nb_flags, NB_INVAL)) {
+					    if (eofflag)
+						    *eofflag = 1;
 					    break;
+				    }
 			    }
 			    /*
 			     * An error will throw away the block and the
@@ -1850,7 +2253,7 @@ buffer_ready:
 			     * is not the block we want, we throw away the
 			     * block and go for the next one via the for loop.
 			     */
-			    if (error || tlbn < lbn)
+			    if (error || (tlbn < lbn))
 				    nfs_buf_release(bp, 1);
 			}
 		    }
@@ -1860,38 +2263,10 @@ buffer_ready:
 		     * we give up.
 		     */
 		    if (error) {
-		        FSDBG_BOT(514, vp, 0xd1e0014, 0, error);
+		        FSDBG_BOT(514, np, 0xd1e0014, 0, error);
 			return (error);
 		    }
 		}
-
-		/*
-		 * If not eof and read aheads are enabled, start one.
-		 * (You need the current block first, so that you have the
-		 *  directory offset cookie of the next block.)
-		 */
-		if (nfs_numasync > 0 && nmp->nm_readahead > 0 &&
-		    (np->n_direofoffset == 0 ||
-		    (lbn + 1) * NFS_DIRBLKSIZ < np->n_direofoffset) &&
-		    !nfs_buf_is_incore(vp, lbn + 1)) {
-			error = nfs_buf_get(vp, lbn + 1, NFS_DIRBLKSIZ, p, NBLK_READ|NBLK_NOWAIT, &rabp);
-			if (error) {
-		            FSDBG_BOT(514, vp, 0xd1e0015, 0, error);
-			    return (error);
-			}
-			if (rabp) {
-			    if (!ISSET(rabp->nb_flags, (NB_CACHE))) {
-				SET(rabp->nb_flags, (NB_READ | NB_ASYNC));
-				if (nfs_asyncio(rabp, cred)) {
-				    SET(rabp->nb_flags, (NB_INVAL|NB_ERROR));
-				    rabp->nb_error = EIO;
-				    nfs_buf_release(rabp, 1);
-				}
-			    } else {
-				nfs_buf_release(rabp, 1);
-			    }
-			}
-		}
 		/*
 		 * Make sure we use a signed variant of min() since
 		 * the second term may be negative.
@@ -1903,9 +2278,14 @@ buffer_ready:
 		 * np->n_direofoffset and chop it off as an
 		 * extra step right here.
 		 */
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_SHARED))) {
+			FSDBG_BOT(514, np, 0xd1e0115, 0, error);
+			return (error);
+		}
 		if (np->n_direofoffset &&
 		    n > np->n_direofoffset - uio->uio_offset)
 			n = np->n_direofoffset - uio->uio_offset;
+		nfs_unlock(np);
 		/*
 		 * Make sure that we return an integral number of entries so
 		 * that any subsequent calls will start copying from the start
@@ -1923,582 +2303,1179 @@ buffer_ready:
 			if (dp > (bp->nb_data + on + n))
 				n = (dp - direntp->d_reclen) - (bp->nb_data + on);
 		}
-		break;
-	    default:
-		printf("nfs_bioread: type %x unexpected\n", vtype);
-		FSDBG_BOT(514, vp, 0xd1e0016, 0, EINVAL);
-		return (EINVAL);
-	    };
+	    }
 
-	    if (n > 0) {
+	    if (n > 0)
 		error = uiomove(bp->nb_data + on, (int)n, uio);
-	    }
-	    switch (vtype) {
-	    case VREG:
-		if (np->n_flag & NNOCACHE)
+
+	    if (vtype == VREG) {
+		if (ioflag & IO_NOCACHE)
 			SET(bp->nb_flags, NB_NOCACHE);
-		break;
-	    case VLNK:
-		n = 0;
-		break;
-	    case VDIR:
-		break;
-	    default:
-		break;
+		nfs_buf_release(bp, 1);
+		nfs_data_unlock(np);
+		np->n_lastread = (uio->uio_offset - 1) / biosize;
+	    } else {
+		nfs_buf_release(bp, 1);
 	    }
-	    nfs_buf_release(bp, 1);
 	} while (error == 0 && uio_uio_resid(uio) > 0 && n > 0);
-	FSDBG_BOT(514, vp, uio->uio_offset, uio_uio_resid(uio), error);
+	FSDBG_BOT(514, np, uio->uio_offset, uio_uio_resid(uio), error);
 	return (error);
 }
 
+/*
+ * limit the number of outstanding async I/O writes
+ */
+static int
+nfs_async_write_start(struct nfsmount *nmp)
+{
+	int error = 0, slpflag = (nmp->nm_flag & NFSMNT_INT) ? PCATCH : 0;
+	struct timespec ts = {1, 0};
+
+	if (nfs_max_async_writes <= 0)
+		return (0);
+	lck_mtx_lock(&nmp->nm_lock);
+	while (!error && (nfs_max_async_writes > 0) && (nmp->nm_asyncwrites >= nfs_max_async_writes)) {
+		if ((error = nfs_sigintr(nmp, NULL, current_thread(), 1)))
+			break;
+		error = msleep(&nmp->nm_asyncwrites, &nmp->nm_lock, slpflag|(PZERO-1), "nfsasyncwrites", &ts);
+		if (error == EWOULDBLOCK)
+			error = 0;
+	}
+	if (!error)
+		nmp->nm_asyncwrites++;
+	lck_mtx_unlock(&nmp->nm_lock);
+	return (error);
+}
+static void
+nfs_async_write_done(struct nfsmount *nmp)
+{
+	if (nmp->nm_asyncwrites <= 0)
+		return;
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_asyncwrites-- >= nfs_max_async_writes)
+		wakeup(&nmp->nm_asyncwrites);
+	lck_mtx_unlock(&nmp->nm_lock);
+}
 
 /*
- * Vnode op for write using bio
+ * write (or commit) the given NFS buffer
+ *
+ * Commit the buffer if we can.
+ * Write out any dirty range.
+ * If any dirty pages remain, write them out.
+ * Mark buffer done.
+ *
+ * For async requests, all the work beyond sending the initial
+ * write RPC is handled in the RPC callback(s).
  */
 int
-nfs_write(ap)
-	struct vnop_write_args /* {
-		struct vnodeop_desc *a_desc;
-		vnode_t a_vp;
-		struct uio *a_uio;
-		int a_ioflag;
-		vfs_context_t a_context;
-	} */ *ap;
+nfs_buf_write(struct nfsbuf *bp)
 {
-	struct uio *uio = ap->a_uio;
-	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
-	proc_t p;
+	int error = 0, oldflags, async;
+	nfsnode_t np;
+	thread_t thd;
 	kauth_cred_t cred;
-	int ioflag = ap->a_ioflag;
-	struct nfsbuf *bp;
-	struct nfs_vattr nvattr;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-	daddr64_t lbn;
-	int biosize, bufsize;
-	int n, on, error = 0;
-	off_t boff, start, end, cureof;
-	struct iovec_32 iov;
-	struct uio auio;
-
-	FSDBG_TOP(515, vp, uio->uio_offset, uio_uio_resid(uio), ioflag);
-
-#if DIAGNOSTIC
-	if (uio->uio_rw != UIO_WRITE)
-		panic("nfs_write mode");
-	if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg))
-		panic("nfs_write proc");
-#endif
+	proc_t p = current_proc();
+	int iomode, doff, dend, firstpg, lastpg;
+	uint32_t pagemask;
 
-	p = vfs_context_proc(ap->a_context);
-	cred = vfs_context_ucred(ap->a_context);
+	FSDBG_TOP(553, bp, NBOFF(bp), bp->nb_flags, 0);
 
-	if (vnode_vtype(vp) != VREG)
-		return (EIO);
+	if (!ISSET(bp->nb_lflags, NBL_BUSY))
+		panic("nfs_buf_write: buffer is not busy???");
 
-	np->n_flag |= NWRBUSY;
+	np = bp->nb_np;
+	async = ISSET(bp->nb_flags, NB_ASYNC);
+	oldflags = bp->nb_flags;
 
-	if (np->n_flag & NNEEDINVALIDATE) {
-		np->n_flag &= ~NNEEDINVALIDATE;
-		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cred, p, 1);
+	CLR(bp->nb_flags, (NB_READ|NB_DONE|NB_ERROR|NB_DELWRI));
+	if (ISSET(oldflags, NB_DELWRI)) {
+		lck_mtx_lock(nfs_buf_mutex);
+		nfs_nbdwrite--;
+		NFSBUFCNTCHK();
+		lck_mtx_unlock(nfs_buf_mutex);
+		wakeup(&nfs_nbdwrite);
 	}
-	if (np->n_flag & NWRITEERR) {
-		np->n_flag &= ~(NWRITEERR | NWRBUSY);
-		FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), np->n_error);
-		return (np->n_error);
+
+	/* move to clean list */
+	if (ISSET(oldflags, (NB_ASYNC|NB_DELWRI))) {
+		lck_mtx_lock(nfs_buf_mutex);
+		if (bp->nb_vnbufs.le_next != NFSNOLIST)
+			LIST_REMOVE(bp, nb_vnbufs);
+		LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
+		lck_mtx_unlock(nfs_buf_mutex);
 	}
+	vnode_startwrite(NFSTOV(np));
 
-	biosize = nmp->nm_biosize;
-	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		nfs_fsinfo(nmp, vp, cred, p);
+	if (p && p->p_stats)
+		OSIncrementAtomic(&p->p_stats->p_ru.ru_oublock);
 
-	if (ioflag & (IO_APPEND | IO_SYNC)) {
-		if (np->n_flag & NMODIFIED) {
-			NATTRINVALIDATE(np);
-			error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-			if (error) {
-				np->n_flag &= ~NWRBUSY;
-				FSDBG_BOT(515, vp, uio->uio_offset, 0x10bad01, error);
-				return (error);
-			}
-		}
-		if (ioflag & IO_APPEND) {
-			NATTRINVALIDATE(np);
-			error = nfs_getattr(vp, &nvattr, cred, p);
+	cred = bp->nb_wcred;
+	if (!IS_VALID_CRED(cred) && ISSET(bp->nb_flags, NB_READ))
+		cred = bp->nb_rcred;  /* shouldn't really happen, but... */
+	if (IS_VALID_CRED(cred))
+		kauth_cred_ref(cred);
+	thd = async ? NULL : current_thread();
+
+	/* We need to make sure the pages are locked before doing I/O.  */
+	if (!ISSET(bp->nb_flags, NB_META) && UBCINFOEXISTS(NFSTOV(np))) {
+		if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
+			error = nfs_buf_upl_setup(bp);
 			if (error) {
-				np->n_flag &= ~NWRBUSY;
-				FSDBG_BOT(515, vp, uio->uio_offset, 0x10bad02, error);
-				return (error);
+				printf("nfs_buf_write: upl create failed %d\n", error);
+				SET(bp->nb_flags, NB_ERROR);
+				bp->nb_error = error = EIO;
+				nfs_buf_iodone(bp);
+				goto out;
 			}
-			uio->uio_offset = np->n_size;
+			nfs_buf_upl_check(bp);
 		}
 	}
-	if (uio->uio_offset < 0) {
-		np->n_flag &= ~NWRBUSY;
-		FSDBG_BOT(515, vp, uio->uio_offset, 0xbad0ff, EINVAL);
-		return (EINVAL);
-	}
-	if (uio_uio_resid(uio) == 0) {
-		np->n_flag &= ~NWRBUSY;
-		FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), 0);
-		return (0);
-	}
 
-	if (vnode_isnocache(vp)) {
-		if (!(np->n_flag & NNOCACHE)) {
-			if (NVALIDBUFS(np)) {
-				error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-				if (error) {
-					np->n_flag &= ~NWRBUSY;
-					FSDBG_BOT(515, vp, 0, 0, error);
-					return (error);
-				}
+	/* If NB_NEEDCOMMIT is set, a commit RPC may do the trick. */
+	if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
+		nfs_buf_check_write_verifier(np, bp);
+	if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+		struct nfsmount *nmp = NFSTONMP(np);
+		if (!nmp) {
+			SET(bp->nb_flags, NB_ERROR);
+			bp->nb_error = error = EIO;
+			nfs_buf_iodone(bp);
+			goto out;
+		}
+		SET(bp->nb_flags, NB_WRITEINPROG);
+		error = nmp->nm_funcs->nf_commit_rpc(np, NBOFF(bp) + bp->nb_dirtyoff,
+				bp->nb_dirtyend - bp->nb_dirtyoff, bp->nb_wcred);
+		CLR(bp->nb_flags, NB_WRITEINPROG);
+		if (error) {
+			if (error != NFSERR_STALEWRITEVERF) {
+				SET(bp->nb_flags, NB_ERROR);
+				bp->nb_error = error;
 			}
-			np->n_flag |= NNOCACHE;
+			nfs_buf_iodone(bp);
+			goto out;
+		}
+		bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+		CLR(bp->nb_flags, NB_NEEDCOMMIT);
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		np->n_needcommitcnt--;
+		CHECK_NEEDCOMMITCNT(np);
+		nfs_unlock(np);
+	}
+	if (!error && (bp->nb_dirtyend > 0)) {
+		/* sanity check the dirty range */
+		if (NBOFF(bp) + bp->nb_dirtyend > (off_t) np->n_size) {
+			bp->nb_dirtyend = np->n_size - NBOFF(bp);
+			if (bp->nb_dirtyoff >= bp->nb_dirtyend)
+				bp->nb_dirtyoff = bp->nb_dirtyend = 0;
 		}
-	} else if (np->n_flag & NNOCACHE) {
-		np->n_flag &= ~NNOCACHE;
 	}
+	if (!error && (bp->nb_dirtyend > 0)) {
+		/* there's a dirty range that needs to be written out */
+		NFS_BUF_MAP(bp);
+
+		doff = bp->nb_dirtyoff;
+		dend = bp->nb_dirtyend;
+
+		/* if doff page is dirty, move doff to start of page */
+		if (NBPGDIRTY(bp, doff / PAGE_SIZE))
+			doff -= doff & PAGE_MASK;
+		/* try to expand write range to include preceding dirty pages */
+		if (!(doff & PAGE_MASK))
+			while ((doff > 0) && NBPGDIRTY(bp, (doff - 1) / PAGE_SIZE))
+				doff -= PAGE_SIZE;
+		/* if dend page is dirty, move dend to start of next page */
+		if ((dend & PAGE_MASK) && NBPGDIRTY(bp, dend / PAGE_SIZE))
+			dend = round_page_32(dend);
+		/* try to expand write range to include trailing dirty pages */
+		if (!(dend & PAGE_MASK))
+			while ((dend < bp->nb_bufsize) && NBPGDIRTY(bp, dend / PAGE_SIZE))
+				dend += PAGE_SIZE;
+		/* make sure to keep dend clipped to EOF */
+		if ((NBOFF(bp) + dend) > (off_t) np->n_size)
+			dend = np->n_size - NBOFF(bp);
+		/* calculate range of complete pages being written */
+		firstpg = round_page_32(doff) / PAGE_SIZE;
+		lastpg = (trunc_page_32(dend) - 1) / PAGE_SIZE;
+		/* calculate mask for that page range */
+		pagemask = ((1 << (lastpg + 1)) - 1) & ~((1 << firstpg) - 1);
 
-	do {
-		OSAddAtomic(1, (SInt32*)&nfsstats.biocache_writes);
-		lbn = uio->uio_offset / biosize;
-		on = uio->uio_offset % biosize;
-		// LP64todo - fix this
-		n = min((unsigned)(biosize - on), uio_uio_resid(uio));
-again:
-		bufsize = biosize;
 		/*
-		 * Get a cache block for writing.  The range to be written is
-		 * (off..off+n) within the block.  We ensure that the block
-		 * either has no dirty region or that the given range is
-		 * contiguous with the existing dirty region.
+		 * compare page mask to nb_dirty; if there are other dirty pages
+		 * then write FILESYNC; otherwise, write UNSTABLE if async and
+		 * not needcommit/stable; otherwise write FILESYNC
 		 */
-		error = nfs_buf_get(vp, lbn, bufsize, p, NBLK_WRITE, &bp);
-		if (error) {
-			np->n_flag &= ~NWRBUSY;
-			FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			return (error);
-		}
-		/* map the block because we know we're going to write to it */
-		NFS_BUF_MAP(bp);
+		if (bp->nb_dirty & ~pagemask)
+			iomode = NFS_WRITE_FILESYNC;
+		else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_STABLE)) == NB_ASYNC)
+			iomode = NFS_WRITE_UNSTABLE;
+		else
+			iomode = NFS_WRITE_FILESYNC;
 
-		if (np->n_flag & NNOCACHE)
-			SET(bp->nb_flags, NB_NOCACHE);
+		/* write the whole contiguous dirty range */
+		bp->nb_offio = doff;
+		bp->nb_endio = dend;
 
-		if (!IS_VALID_CRED(bp->nb_wcred)) {
-			kauth_cred_ref(cred);
-			bp->nb_wcred = cred;
-		}
+		OSAddAtomic(1, (SInt32 *)&nfsstats.write_bios);
 
+		SET(bp->nb_flags, NB_WRITEINPROG);
+		error = nfs_buf_write_rpc(bp, iomode, thd, cred);
 		/*
-		 * If there's already a dirty range AND dirty pages in this block we
-		 * need to send a commit AND write the dirty pages before continuing.
-		 *
-		 * If there's already a dirty range OR dirty pages in this block
-		 * and the new write range is not contiguous with the existing range,
-		 * then force the buffer to be written out now.
-		 * (We used to just extend the dirty range to cover the valid,
-		 * but unwritten, data in between also.  But writing ranges
-		 * of data that weren't actually written by an application
-		 * risks overwriting some other client's data with stale data
-		 * that's just masquerading as new written data.)
+		 * For async I/O, the callbacks will finish up the
+		 * write and push out any dirty pages.  Otherwise,
+		 * the write has already been finished and any dirty
+		 * pages pushed out.
 		 */
-		if (bp->nb_dirtyend > 0) {
-		    if (on > bp->nb_dirtyend || (on + n) < bp->nb_dirtyoff || bp->nb_dirty) {
-			FSDBG(515, vp, uio->uio_offset, bp, 0xd15c001);
-			/* write/commit buffer "synchronously" */
-			/* (NB_STABLE indicates that data writes should be FILESYNC) */
-			CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
-			SET(bp->nb_flags, (NB_ASYNC | NB_STABLE));
-			error = nfs_buf_write(bp);
-			if (error) {
-			    np->n_flag &= ~NWRBUSY;
-			    FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			    return (error);
+	} else {
+		if (!error && bp->nb_dirty) /* write out any dirty pages */
+			error = nfs_buf_write_dirty_pages(bp, thd, cred);
+		nfs_buf_iodone(bp);
+	}
+	/* note: bp is still valid only for !async case */
+out:
+	if (!async) {
+		error = nfs_buf_iowait(bp);
+		/* move to clean list */
+		if (oldflags & NB_DELWRI) {
+			lck_mtx_lock(nfs_buf_mutex);
+			if (bp->nb_vnbufs.le_next != NFSNOLIST)
+				LIST_REMOVE(bp, nb_vnbufs);
+			LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
+			lck_mtx_unlock(nfs_buf_mutex);
+		}
+		FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, error);
+		nfs_buf_release(bp, 1);
+		/* check if we need to invalidate (and we can) */
+		if ((np->n_flag & NNEEDINVALIDATE) &&
+		    !(np->n_bflag & (NBINVALINPROG|NBFLUSHINPROG))) {
+			int invalidate = 0;
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			if (np->n_flag & NNEEDINVALIDATE) {
+				invalidate = 1;
+				np->n_flag &= ~NNEEDINVALIDATE;
 			}
-			goto again;
-		    }
-		} else if (bp->nb_dirty) {
-		    int firstpg, lastpg;
-		    u_int32_t pagemask;
-		    /* calculate write range pagemask */
-		    firstpg = on/PAGE_SIZE;
-		    lastpg = (on+n-1)/PAGE_SIZE;
-		    pagemask = ((1 << (lastpg+1)) - 1) & ~((1 << firstpg) - 1);
-		    /* check if there are dirty pages outside the write range */
-		    if (bp->nb_dirty & ~pagemask) {
-			FSDBG(515, vp, uio->uio_offset, bp, 0xd15c002);
-			/* write/commit buffer "synchronously" */
-			/* (NB_STABLE indicates that data writes should be FILESYNC) */
-			CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
-			SET(bp->nb_flags, (NB_ASYNC | NB_STABLE));
-			error = nfs_buf_write(bp);
-			if (error) {
-			    np->n_flag &= ~NWRBUSY;
-			    FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			    return (error);
+			nfs_unlock(np);
+			if (invalidate) {
+				/*
+				 * There was a write error and we need to
+				 * invalidate attrs and flush buffers in
+				 * order to sync up with the server.
+				 * (if this write was extending the file,
+				 * we may no longer know the correct size)
+				 *
+				 * But we couldn't call vinvalbuf while holding
+				 * the buffer busy.  So we call vinvalbuf() after
+				 * releasing the buffer.
+				 */
+				nfs_vinvalbuf2(NFSTOV(np), V_SAVE|V_IGNORE_WRITEERR, thd, cred, 1);
 			}
-			goto again;
-		    }
-		    /* if the first or last pages are already dirty */
-		    /* make sure that the dirty range encompasses those pages */
-		    if (NBPGDIRTY(bp,firstpg) || NBPGDIRTY(bp,lastpg)) {
-			FSDBG(515, vp, uio->uio_offset, bp, 0xd15c003);
-		    	bp->nb_dirtyoff = min(on, firstpg * PAGE_SIZE);
-			if (NBPGDIRTY(bp,lastpg)) {
-			    bp->nb_dirtyend = (lastpg+1) * PAGE_SIZE;
-			    /* clip to EOF */
-			    if (NBOFF(bp) + bp->nb_dirtyend > (off_t)np->n_size)
-				    bp->nb_dirtyend = np->n_size - NBOFF(bp);
-			} else
-			    bp->nb_dirtyend = on+n;
-		    }
 		}
+	}
+
+	if (IS_VALID_CRED(cred))
+		kauth_cred_unref(&cred);
+	return (error);
+}
 
+/*
+ * finish the writing of a buffer
+ */
+void
+nfs_buf_write_finish(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
+{
+	nfsnode_t np = bp->nb_np;
+	int error = (bp->nb_flags & NB_ERROR) ? bp->nb_error : 0;
+	int firstpg, lastpg;
+	uint32_t pagemask;
+
+	if ((error == EINTR) || (error == ERESTART)) {
+		CLR(bp->nb_flags, NB_ERROR);
+		SET(bp->nb_flags, NB_EINTR);
+	}
+
+	if (!error) {
+		/* calculate range of complete pages being written */
+		firstpg = round_page_32(bp->nb_offio) / PAGE_SIZE;
+		lastpg = (trunc_page_32(bp->nb_endio) - 1) / PAGE_SIZE;
+		/* calculate mask for that page range written */
+		pagemask = ((1 << (lastpg + 1)) - 1) & ~((1 << firstpg) - 1);
+		/* clear dirty bits for pages we've written */
+		bp->nb_dirty &= ~pagemask;
+	}
+
+	/* manage needcommit state */
+	if (!error && (bp->nb_commitlevel == NFS_WRITE_UNSTABLE)) {
+		if (!ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			np->n_needcommitcnt++;
+			nfs_unlock(np);
+			SET(bp->nb_flags, NB_NEEDCOMMIT);
+		}
+		/* make sure nb_dirtyoff/nb_dirtyend reflect actual range written */
+		bp->nb_dirtyoff = bp->nb_offio;
+		bp->nb_dirtyend = bp->nb_endio;
+	} else if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		np->n_needcommitcnt--;
+		CHECK_NEEDCOMMITCNT(np);
+		nfs_unlock(np);
+		CLR(bp->nb_flags, NB_NEEDCOMMIT);
+	}
+
+	CLR(bp->nb_flags, NB_WRITEINPROG);
+
+	/*
+	 * For an unstable write, the buffer is still treated as dirty until
+	 * a commit (or stable (re)write) is performed.  Buffers needing only
+	 * a commit are marked with the NB_DELWRI and NB_NEEDCOMMIT flags.
+	 *
+	 * If the write was interrupted we set NB_EINTR.  Don't set NB_ERROR
+	 * because that would cause the buffer to be dropped.  The buffer is
+	 * still valid and simply needs to be written again.
+	 */
+	if ((error == EINTR) || (error == ERESTART) || (!error && (bp->nb_flags & NB_NEEDCOMMIT))) {
+		CLR(bp->nb_flags, NB_INVAL);
+		if (!ISSET(bp->nb_flags, NB_DELWRI)) {
+			SET(bp->nb_flags, NB_DELWRI);
+			lck_mtx_lock(nfs_buf_mutex);
+			nfs_nbdwrite++;
+			NFSBUFCNTCHK();
+			lck_mtx_unlock(nfs_buf_mutex);
+		}
 		/*
-		 * Are we extending the size of the file with this write?
-		 * If so, update file size now that we have the block.
-		 * If there was a partial buf at the old eof, validate
-		 * and zero the new bytes. 
+		 * Since for the NB_ASYNC case, we've reassigned the buffer to the
+		 * clean list, we have to reassign it back to the dirty one. Ugh.
 		 */
-		cureof = (off_t)np->n_size;
-		if (uio->uio_offset + n > (off_t)np->n_size) {
-			struct nfsbuf *eofbp = NULL;
-			daddr64_t eofbn = np->n_size / biosize;
-			int eofoff = np->n_size % biosize;
-			int neweofoff = (uio->uio_offset + n) % biosize;
+		if (ISSET(bp->nb_flags, NB_ASYNC)) {
+			/* move to dirty list */
+			lck_mtx_lock(nfs_buf_mutex);
+			if (bp->nb_vnbufs.le_next != NFSNOLIST)
+				LIST_REMOVE(bp, nb_vnbufs);
+			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
+			lck_mtx_unlock(nfs_buf_mutex);
+		}
+	} else {
+		/* either there's an error or we don't need to commit */
+		if (error) {
+			/*
+			 * There was a write error and we need to invalidate
+			 * attrs and flush buffers in order to sync up with the
+			 * server.  (if this write was extending the file, we
+			 * may no longer know the correct size)
+			 *
+			 * But we can't call vinvalbuf while holding this
+			 * buffer busy.  Set a flag to do it after releasing
+			 * the buffer.
+			 */
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			np->n_error = error;
+			np->n_flag |= (NWRITEERR | NNEEDINVALIDATE);
+			NATTRINVALIDATE(np);
+			nfs_unlock(np);
+		}
+		/* clear the dirty range */
+		bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+	}
 
-			FSDBG(515, 0xb1ffa000, uio->uio_offset + n, eofoff, neweofoff);
+	if (!error && bp->nb_dirty)
+		nfs_buf_write_dirty_pages(bp, thd, cred);
+	nfs_buf_iodone(bp);
+}
 
-			if (eofoff && (eofbn < lbn)) {
-				error = nfs_buf_get(vp, eofbn, biosize, p, NBLK_WRITE|NBLK_ONLYVALID, &eofbp);
-				if (error) {
-					np->n_flag &= ~NWRBUSY;
-					FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-					return (error);
-				}
-			}
+/*
+ * write out any pages marked dirty in a buffer
+ *
+ * We do use unstable writes and follow up with a commit.
+ * If we catch the write verifier changing we'll restart
+ * do the writes filesync.
+ */
+int
+nfs_buf_write_dirty_pages(struct nfsbuf *bp, thread_t thd, kauth_cred_t cred)
+{
+	nfsnode_t np = bp->nb_np;
+	struct nfsmount *nmp = NFSTONMP(np);
+	int error = 0, commit, iomode, iomode2, len, pg, count, npages, off;
+	uint32_t dirty = bp->nb_dirty;
+	uint64_t wverf;
+	struct uio uio;
+	struct iovec_32 io;
 
-			/* if we're extending within the same last block */
-			/* and the block is flagged as being cached... */
-			if ((lbn == eofbn) && ISSET(bp->nb_flags, NB_CACHE)) {
-				/* ...check that all pages in buffer are valid */
-				int endpg = ((neweofoff ? neweofoff : biosize) - 1)/PAGE_SIZE;
-				u_int32_t pagemask;
-				/* pagemask only has to extend to last page being written to */
-				pagemask = (1 << (endpg+1)) - 1;
-				FSDBG(515, 0xb1ffa001, bp->nb_valid, pagemask, 0);
-				if ((bp->nb_valid & pagemask) != pagemask) {
-					/* zerofill any hole */
-					if (on > bp->nb_validend) {
-						int i;
-						for (i=bp->nb_validend/PAGE_SIZE; i <= (on - 1)/PAGE_SIZE; i++)
-							NBPGVALID_SET(bp, i);
-						NFS_BUF_MAP(bp);
-						FSDBG(516, bp, bp->nb_validend, on - bp->nb_validend, 0xf01e);
-						bzero((char *)bp->nb_data + bp->nb_validend,
-							on - bp->nb_validend);
-					}
-					/* zerofill any trailing data in the last page */
-					if (neweofoff) {
-						NFS_BUF_MAP(bp);
-						FSDBG(516, bp, neweofoff, PAGE_SIZE - (neweofoff & PAGE_MASK), 0xe0f);
-						bzero((char *)bp->nb_data + neweofoff,
-							PAGE_SIZE - (neweofoff & PAGE_MASK));
-					}
-				}
-			}
-			np->n_flag |= NMODIFIED;
-			np->n_size = uio->uio_offset + n;
-			ubc_setsize(vp, (off_t)np->n_size); /* XXX errors */
-			if (eofbp) {
-				/*
-				 * We may need to zero any previously invalid data
-				 * after the old EOF in the previous EOF buffer.
-				 *
-				 * For the old last page, don't zero bytes if there
-				 * are invalid bytes in that page (i.e. the page isn't
-				 * currently valid).
-				 * For pages after the old last page, zero them and
-				 * mark them as valid.
-				 */
-				char *d;
-				int i;
-				if (np->n_flag & NNOCACHE)
-					SET(eofbp->nb_flags, NB_NOCACHE);
-				NFS_BUF_MAP(eofbp);
-				FSDBG(516, eofbp, eofoff, biosize - eofoff, 0xe0fff01e);
-				d = eofbp->nb_data;
-				i = eofoff/PAGE_SIZE;
-				while (eofoff < biosize) {
-					int poff = eofoff & PAGE_MASK;
-					if (!poff || NBPGVALID(eofbp,i)) {
-						bzero(d + eofoff, PAGE_SIZE - poff);
-						NBPGVALID_SET(eofbp, i);
-					}
-					if (bp->nb_validend == eofoff)
-						bp->nb_validend += PAGE_SIZE - poff;
-					eofoff += PAGE_SIZE - poff;
-					i++;
-				}
-				nfs_buf_release(eofbp, 1);
+	if (!bp->nb_dirty)
+		return (0);
+
+	/* there are pages marked dirty that need to be written out */
+	OSAddAtomic(1, (SInt32 *)&nfsstats.write_bios);
+	NFS_BUF_MAP(bp);
+	SET(bp->nb_flags, NB_WRITEINPROG);
+	npages = bp->nb_bufsize / PAGE_SIZE;
+	iomode = NFS_WRITE_UNSTABLE;
+
+	uio.uio_iovs.iov32p = &io;
+	uio.uio_iovcnt = 1;
+	uio.uio_rw = UIO_WRITE;
+#if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
+	uio.uio_segflg = UIO_SYSSPACE;
+#else
+	uio.uio_segflg = UIO_SYSSPACE32;
+#endif
+
+again:
+	dirty = bp->nb_dirty;
+	wverf = bp->nb_verf;
+	commit = NFS_WRITE_FILESYNC;
+	for (pg = 0; pg < npages; pg++) {
+		if (!NBPGDIRTY(bp, pg))
+			continue;
+		count = 1;
+		while (((pg + count) < npages) && NBPGDIRTY(bp, pg + count))
+			count++;
+		/* write count pages starting with page pg */
+		off = pg * PAGE_SIZE;
+		len = count * PAGE_SIZE;
+		/* clip writes to EOF */
+		if (NBOFF(bp) + off + len > (off_t) np->n_size)
+			len -= (NBOFF(bp) + off + len) - np->n_size;
+		if (len > 0) {
+			iomode2 = iomode;
+			io.iov_len = len;
+			uio_uio_resid_set(&uio, io.iov_len);
+			uio.uio_offset = NBOFF(bp) + off;
+			io.iov_base = (uintptr_t) bp->nb_data + off;
+			error = nfs_write_rpc2(np, &uio, thd, cred, &iomode2, &bp->nb_verf);
+			if (error)
+				break;
+			if (iomode2 < commit) /* Retain the lowest commitment level returned. */
+				commit = iomode2;
+			if ((commit != NFS_WRITE_FILESYNC) && (wverf != bp->nb_verf)) {
+				/* verifier changed, redo all the writes filesync */
+				iomode = NFS_WRITE_FILESYNC;
+				goto again;
 			}
 		}
+		/* clear dirty bits */
+		while (count--) {
+			dirty &= ~(1 << pg);
+			if (count) /* leave pg on last page */
+				pg++;
+		}
+	}
+	CLR(bp->nb_flags, NB_WRITEINPROG);
+
+	if (!error && (commit != NFS_WRITE_FILESYNC)) {
+		error = nmp->nm_funcs->nf_commit_rpc(np, NBOFF(bp), bp->nb_bufsize, cred);
+		if (error == NFSERR_STALEWRITEVERF) {
+			/* verifier changed, so we need to restart all the writes */
+			iomode = NFS_WRITE_FILESYNC;
+			goto again;
+		}
+	}
+	if (!error) {
+		bp->nb_dirty = dirty;
+	} else {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error;
+	}
+	return (error);
+}
+
+/*
+ * initiate the NFS WRITE RPC(s) for a buffer
+ */
+int
+nfs_buf_write_rpc(struct nfsbuf *bp, int iomode, thread_t thd, kauth_cred_t cred)
+{
+	struct nfsmount *nmp;
+	nfsnode_t np = bp->nb_np;
+	int error = 0, nfsvers, async;
+	int offset, length, nmwsize, nrpcs, len;
+	struct nfsreq *req;
+	struct nfsreq_cbinfo cb;
+	struct uio uio;
+	struct iovec_32 io;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		bp->nb_error = error = ENXIO;
+		SET(bp->nb_flags, NB_ERROR);
+		nfs_buf_iodone(bp);
+		return (error);
+	}
+	nfsvers = nmp->nm_vers;
+	nmwsize = nmp->nm_wsize;
+
+	offset = bp->nb_offio;
+	length = bp->nb_endio - bp->nb_offio;
+
+	/* Note: Can only do async I/O if nfsiods are configured. */
+	async = (bp->nb_flags & NB_ASYNC) && (NFSIOD_MAX > 0);
+	bp->nb_commitlevel = NFS_WRITE_FILESYNC;
+	cb.rcb_func = async ? nfs_buf_write_rpc_finish : NULL;
+	cb.rcb_bp = bp;
+
+	if ((nfsvers == NFS_VER2) && ((NBOFF(bp) + bp->nb_endio) > 0xffffffffLL)) {
+		bp->nb_error = error = EFBIG;
+		SET(bp->nb_flags, NB_ERROR);
+		nfs_buf_iodone(bp);
+		return (error);
+	}
+
+	uio.uio_iovs.iov32p = &io;
+	uio.uio_iovcnt = 1;
+	uio.uio_rw = UIO_WRITE;
+#if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
+	uio.uio_segflg = UIO_SYSSPACE;
+#else
+	uio.uio_segflg = UIO_SYSSPACE32;
+#endif
+	io.iov_len = length;
+	uio_uio_resid_set(&uio, io.iov_len);
+	uio.uio_offset = NBOFF(bp) + offset;
+	io.iov_base = (uintptr_t) bp->nb_data + offset;
+
+	bp->nb_rpcs = nrpcs = (length + nmwsize - 1) / nmwsize;
+	if (async && (nrpcs > 1)) {
+		SET(bp->nb_flags, NB_MULTASYNCRPC);
+	} else {
+		CLR(bp->nb_flags, NB_MULTASYNCRPC);
+	}
+
+	while (length > 0) {
+		if (ISSET(bp->nb_flags, NB_ERROR)) {
+			error = bp->nb_error;
+			break;
+		}
+		len = (length > nmwsize) ? nmwsize : length;
+		cb.rcb_args[0] = offset;
+		cb.rcb_args[1] = len;
+		if (async && ((error = nfs_async_write_start(nmp))))
+			break;
+		req = NULL;
+		error = nmp->nm_funcs->nf_write_rpc_async(np, &uio, len, thd, cred,
+				iomode, &cb, &req);
+		if (error) {
+			if (async)
+				nfs_async_write_done(nmp);
+			break;
+		}
+		offset += len;
+		length -= len;
+		if (async)
+			continue;
+		nfs_buf_write_rpc_finish(req);
+	}
+
+	if (length > 0) {
 		/*
-		 * If dirtyend exceeds file size, chop it down.  This should
-		 * not occur unless there is a race.
-		 */
-		if (NBOFF(bp) + bp->nb_dirtyend > (off_t)np->n_size)
-			bp->nb_dirtyend = np->n_size - NBOFF(bp);
-		/*
-		 * UBC doesn't handle partial pages, so we need to make sure
-		 * that any pages left in the page cache are completely valid.
-		 *
-		 * Writes that are smaller than a block are delayed if they
-		 * don't extend to the end of the block.
-		 *
-		 * If the block isn't (completely) cached, we may need to read
-		 * in some parts of pages that aren't covered by the write.
-		 * If the write offset (on) isn't page aligned, we'll need to
-		 * read the start of the first page being written to.  Likewise,
-		 * if the offset of the end of the write (on+n) isn't page aligned,
-		 * we'll need to read the end of the last page being written to.
-		 *
-		 * Notes:
-		 * We don't want to read anything we're just going to write over.
-		 * We don't want to issue multiple I/Os if we don't have to
-		 *   (because they're synchronous rpcs).
-		 * We don't want to read anything we already have modified in the
-		 *   page cache.
+		 * Something bad happened while trying to send the RPCs.
+		 * Wait for any outstanding requests to complete.
 		 */
-		if (!ISSET(bp->nb_flags, NB_CACHE) && n < biosize) {
-			int firstpg, lastpg, dirtypg;
-			int firstpgoff, lastpgoff;
-			start = end = -1;
-			firstpg = on/PAGE_SIZE;
-			firstpgoff = on & PAGE_MASK;
-			lastpg = (on+n-1)/PAGE_SIZE;
-			lastpgoff = (on+n) & PAGE_MASK;
-			if (firstpgoff && !NBPGVALID(bp,firstpg)) {
-				/* need to read start of first page */
-				start = firstpg * PAGE_SIZE;
-				end = start + firstpgoff;
-			}
-			if (lastpgoff && !NBPGVALID(bp,lastpg)) {
-				/* need to read end of last page */
-				if (start < 0)
-					start = (lastpg * PAGE_SIZE) + lastpgoff;
-				end = (lastpg + 1) * PAGE_SIZE;
+		bp->nb_error = error;
+		SET(bp->nb_flags, NB_ERROR);
+		if (ISSET(bp->nb_flags, NB_MULTASYNCRPC)) {
+			nrpcs = (length + nmwsize - 1) / nmwsize;
+			lck_mtx_lock(nfs_buf_mutex);
+			bp->nb_rpcs -= nrpcs;
+			if (bp->nb_rpcs == 0) {
+				/* No RPCs left, so the buffer's done */
+				lck_mtx_unlock(nfs_buf_mutex);
+				nfs_buf_write_finish(bp, thd, cred);
+			} else {
+				/* wait for the last RPC to mark it done */
+				while (bp->nb_rpcs > 0)
+					msleep(&bp->nb_rpcs, nfs_buf_mutex, 0,
+						"nfs_buf_write_rpc_cancel", NULL);
+				lck_mtx_unlock(nfs_buf_mutex);
 			}
-			if (end > start) {
-				/* need to read the data in range: start...end-1 */
-
-				/* first, check for dirty pages in between */
-				/* if there are, we'll have to do two reads because */
-				/* we don't want to overwrite the dirty pages. */
-				for (dirtypg=start/PAGE_SIZE; dirtypg <= (end-1)/PAGE_SIZE; dirtypg++)
-					if (NBPGDIRTY(bp,dirtypg))
-						break;
-
-				/* if start is at beginning of page, try */
-				/* to get any preceeding pages as well. */
-				if (!(start & PAGE_MASK)) {
-					/* stop at next dirty/valid page or start of block */
-					for (; start > 0; start-=PAGE_SIZE)
-						if (NBPGVALID(bp,((start-1)/PAGE_SIZE)))
-							break;
-				}
+		} else {
+			nfs_buf_write_finish(bp, thd, cred);
+		}
+	}
 
-				NFS_BUF_MAP(bp);
-				/* setup uio for read(s) */
-				boff = NBOFF(bp);
-				auio.uio_iovs.iov32p = &iov;
-				auio.uio_iovcnt = 1;
+	return (error);
+}
+
+/*
+ * finish up an NFS WRITE RPC on a buffer
+ */
+void
+nfs_buf_write_rpc_finish(struct nfsreq *req)
+{
+	int error = 0, nfsvers, offset, length, multasyncrpc, finished;
+	int committed = NFS_WRITE_FILESYNC;
+	uint64_t wverf = 0;
+	size_t rlen;
+	void *wakeme = NULL;
+	struct nfsreq_cbinfo cb;
+	struct nfsreq *wreq = NULL;
+	struct nfsbuf *bp;
+	struct nfsmount *nmp;
+	nfsnode_t np;
+	thread_t thd;
+	kauth_cred_t cred;
+	struct uio uio;
+	struct iovec_32 io;
+
+finish:
+	np = req->r_np;
+	thd = req->r_thread;
+	cred = req->r_cred;
+	if (IS_VALID_CRED(cred))
+		kauth_cred_ref(cred);
+	cb = req->r_callback;
+	bp = cb.rcb_bp;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error = ENXIO;
+	}
+	if (error || ISSET(bp->nb_flags, NB_ERROR)) {
+		/* just drop it */
+		nfs_request_async_cancel(req);
+		goto out;
+	}
+	nfsvers = nmp->nm_vers;
+
+	offset = cb.rcb_args[0];
+	rlen = length = cb.rcb_args[1];
+
+	/* finish the RPC */
+	error = nmp->nm_funcs->nf_write_rpc_async_finish(np, req, &committed, &rlen, &wverf);
+	if ((error == EINPROGRESS) && cb.rcb_func) {
+		/* async request restarted */
+		if (IS_VALID_CRED(cred))
+			kauth_cred_unref(&cred);
+		return;
+	}
+
+	if (error) {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error;
+	}
+	if (error || (nfsvers == NFS_VER2))
+		goto out;
+	if (rlen <= 0) {
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error = EIO;
+		goto out;
+	}
+
+	/* save lowest commit level returned */
+	if (committed < bp->nb_commitlevel)
+		bp->nb_commitlevel = committed;
+
+	/* check the write verifier */
+	if (!bp->nb_verf) {
+		bp->nb_verf = wverf;
+	} else if (bp->nb_verf != wverf) {
+		/* verifier changed, so buffer will need to be rewritten */
+		bp->nb_flags |= NB_STALEWVERF;
+		bp->nb_commitlevel = NFS_WRITE_UNSTABLE;
+		bp->nb_verf = wverf;
+	}
+
+	/*
+	 * check for a short write
+	 *
+	 * If the server didn't write all the data, then we
+	 * need to issue another write for the rest of it.
+	 * (Don't bother if the buffer hit an error or stale wverf.)
+	 */
+	if (((int)rlen < length) && !(bp->nb_flags & (NB_STALEWVERF|NB_ERROR))) {
+		offset += rlen;
+		length -= rlen;
+
+		uio.uio_iovs.iov32p = &io;
+		uio.uio_iovcnt = 1;
+		uio.uio_rw = UIO_WRITE;
 #if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
-				auio.uio_segflg = UIO_SYSSPACE;
+		uio.uio_segflg = UIO_SYSSPACE;
 #else
-				auio.uio_segflg = UIO_SYSSPACE32;
-#endif 
-				auio.uio_rw = UIO_READ;
-
-				if (dirtypg <= (end-1)/PAGE_SIZE) {
-					/* there's a dirty page in the way, so just do two reads */
-					/* we'll read the preceding data here */
-					auio.uio_offset = boff + start;
-					iov.iov_len = on - start;
-					uio_uio_resid_set(&auio, iov.iov_len);
-					iov.iov_base = (uintptr_t) bp->nb_data + start;
-					error = nfs_readrpc(vp, &auio, cred, p);
-					if (error) {
-						bp->nb_error = error;
-						SET(bp->nb_flags, NB_ERROR);
-						printf("nfs_write: readrpc %d", error);
-					}
-					if (uio_uio_resid(&auio) > 0) {
-						FSDBG(516, bp, iov.iov_base - bp->nb_data, uio_uio_resid(&auio), 0xd00dee01);
-						// LP64todo - fix this
-						bzero((caddr_t)iov.iov_base, uio_uio_resid(&auio));
-					}
-					/* update validoff/validend if necessary */
-					if ((bp->nb_validoff < 0) || (bp->nb_validoff > start))
-						bp->nb_validoff = start;
-					if ((bp->nb_validend < 0) || (bp->nb_validend < on))
-						bp->nb_validend = on;
-					if ((off_t)np->n_size > boff + bp->nb_validend)
-						bp->nb_validend = min(np->n_size - (boff + start), biosize);
-					/* validate any pages before the write offset */
-					for (; start < on/PAGE_SIZE; start+=PAGE_SIZE)
-						NBPGVALID_SET(bp, start/PAGE_SIZE);
-					/* adjust start to read any trailing data */
-					start = on+n;
-				}
+		uio.uio_segflg = UIO_SYSSPACE32;
+#endif
+		io.iov_len = length;
+		uio_uio_resid_set(&uio, io.iov_len);
+		uio.uio_offset = NBOFF(bp) + offset;
+		io.iov_base = (uintptr_t) bp->nb_data + offset;
 
-				/* if end is at end of page, try to */
-				/* get any following pages as well. */
-				if (!(end & PAGE_MASK)) {
-					/* stop at next valid page or end of block */
-					for (; end < bufsize; end+=PAGE_SIZE)
-						if (NBPGVALID(bp,end/PAGE_SIZE))
-							break;
-				}
+		cb.rcb_args[0] = offset;
+		cb.rcb_args[1] = length;
 
-				if (((boff+start) >= cureof) || ((start >= on) && ((boff + on + n) >= cureof))) {
-					/*
-					 * Either this entire read is beyond the current EOF
-					 * or the range that we won't be modifying (on+n...end)
-					 * is all beyond the current EOF.
-					 * No need to make a trip across the network to
-					 * read nothing.  So, just zero the buffer instead.
-					 */
-					FSDBG(516, bp, start, end - start, 0xd00dee00);
-					bzero(bp->nb_data + start, end - start);
-				} else {
-					/* now we'll read the (rest of the) data */
-					auio.uio_offset = boff + start;
-					iov.iov_len = end - start;
-					uio_uio_resid_set(&auio, iov.iov_len);
-					iov.iov_base = (uintptr_t) (bp->nb_data + start);
-					error = nfs_readrpc(vp, &auio, cred, p);
-					if (error) {
-						bp->nb_error = error;
-						SET(bp->nb_flags, NB_ERROR);
-						printf("nfs_write: readrpc %d", error);
-					}
-					if (uio_uio_resid(&auio) > 0) {
-						FSDBG(516, bp, iov.iov_base - bp->nb_data, uio_uio_resid(&auio), 0xd00dee02);
-						// LP64todo - fix this
-						bzero((caddr_t)iov.iov_base, uio_uio_resid(&auio));
-					}
-				}
-				/* update validoff/validend if necessary */
-				if ((bp->nb_validoff < 0) || (bp->nb_validoff > start))
-					bp->nb_validoff = start;
-				if ((bp->nb_validend < 0) || (bp->nb_validend < end))
-					bp->nb_validend = end;
-				if ((off_t)np->n_size > boff + bp->nb_validend)
-					bp->nb_validend = min(np->n_size - (boff + start), biosize);
-				/* validate any pages before the write offset's page */
-				for (; start < trunc_page_32(on); start+=PAGE_SIZE)
-					NBPGVALID_SET(bp, start/PAGE_SIZE);
-				/* validate any pages after the range of pages being written to */
-				for (; (end - 1) > round_page_32(on+n-1); end-=PAGE_SIZE)
-					NBPGVALID_SET(bp, (end-1)/PAGE_SIZE);
-				/* Note: pages being written to will be validated when written */
+		error = nmp->nm_funcs->nf_write_rpc_async(np, &uio, length, thd, cred,
+				NFS_WRITE_FILESYNC, &cb, &wreq);
+		if (!error) {
+			if (IS_VALID_CRED(cred))
+				kauth_cred_unref(&cred);
+			if (!cb.rcb_func) {
+				/* if !async we'll need to wait for this RPC to finish */
+				req = wreq;
+				goto finish;
 			}
+			/*
+			 * We're done here.
+			 * Outstanding RPC count is unchanged.
+			 * Callback will be called when RPC is done.
+			 */
+			return;
 		}
+		SET(bp->nb_flags, NB_ERROR);
+		bp->nb_error = error;
+	}
 
-		if (ISSET(bp->nb_flags, NB_ERROR)) {
-			error = bp->nb_error;
-			nfs_buf_release(bp, 1);
-			np->n_flag &= ~NWRBUSY;
-			FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			return (error);
-		}
+out:
+	if (cb.rcb_func)
+		nfs_async_write_done(nmp);
+	/*
+	 * Decrement outstanding RPC count on buffer
+	 * and call nfs_buf_write_finish on last RPC.
+	 *
+	 * (Note: when there are multiple async RPCs issued for a
+	 * buffer we need nfs_buffer_mutex to avoid problems when
+	 * aborting a partially-initiated set of RPCs)
+	 */
+	multasyncrpc = ISSET(bp->nb_flags, NB_MULTASYNCRPC);
+	if (multasyncrpc)
+		lck_mtx_lock(nfs_buf_mutex);
+
+	bp->nb_rpcs--;
+	finished = (bp->nb_rpcs == 0);
 
+	if (multasyncrpc)
+		lck_mtx_unlock(nfs_buf_mutex);
+
+	if (finished) {
+		if (multasyncrpc)
+			wakeme = &bp->nb_rpcs;
+		nfs_buf_write_finish(bp, thd, cred);
+		if (wakeme)
+			wakeup(wakeme);
+	}
+
+	if (IS_VALID_CRED(cred))
+		kauth_cred_unref(&cred);
+}
+
+/*
+ * Send commit(s) for the given node's "needcommit" buffers 
+ */
+int
+nfs_flushcommits(nfsnode_t np, int nowait)
+{
+	struct nfsmount *nmp;
+	struct nfsbuf *bp;
+	struct nfsbuflists blist, commitlist;
+	int error = 0, retv, wcred_set, flags, dirty;
+	u_quad_t off, endoff, toff;
+	u_int32_t count;
+	kauth_cred_t wcred = NULL;
+
+	FSDBG_TOP(557, np, 0, 0, 0);
+
+	/*
+	 * A nb_flags == (NB_DELWRI | NB_NEEDCOMMIT) block has been written to the
+	 * server, but nas not been committed to stable storage on the server
+	 * yet. The byte range is worked out for as many nfsbufs as we can handle
+	 * and the commit rpc is done.
+	 */
+	if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
+		error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+		if (error)
+			goto done;
 		np->n_flag |= NMODIFIED;
+		nfs_unlock(np);
+	}
 
-		NFS_BUF_MAP(bp);
-		error = uiomove((char *)bp->nb_data + on, n, uio);
-		if (error) {
-			SET(bp->nb_flags, NB_ERROR);
-			nfs_buf_release(bp, 1);
-			np->n_flag &= ~NWRBUSY;
-			FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
-			return (error);
-		}
+	off = (u_quad_t)-1;
+	endoff = 0;
+	wcred_set = 0;
+	LIST_INIT(&commitlist);
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		error = ENXIO;
+		goto done;
+	}
+	if (nmp->nm_vers == NFS_VER2) {
+		error = EINVAL;
+		goto done;
+	}
+
+	flags = NBI_DIRTY;
+	if (nowait)
+		flags |= NBI_NOWAIT;
+	lck_mtx_lock(nfs_buf_mutex);
+	if (!nfs_buf_iterprepare(np, &blist, flags)) {
+		while ((bp = LIST_FIRST(&blist))) {
+			LIST_REMOVE(bp, nb_vnbufs);
+			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
+			error = nfs_buf_acquire(bp, NBAC_NOWAIT, 0, 0);
+			if (error)
+				continue;
+			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
+				nfs_buf_check_write_verifier(np, bp);
+			if (((bp->nb_flags & (NB_DELWRI | NB_NEEDCOMMIT))
+				!= (NB_DELWRI | NB_NEEDCOMMIT))) {
+				nfs_buf_drop(bp);
+				continue;
+			}
+			nfs_buf_remfree(bp);
+			lck_mtx_unlock(nfs_buf_mutex);
+			/*
+			 * we need a upl to see if the page has been
+			 * dirtied (think mmap) since the unstable write, and
+			 * also to prevent vm from paging it during our commit rpc
+			 */
+			if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
+				retv = nfs_buf_upl_setup(bp);
+				if (retv) {
+					/* unable to create upl */
+					/* vm object must no longer exist */
+					/* this could be fatal if we need */
+					/* to write the data again, we'll see...  */
+					printf("nfs_flushcommits: upl create failed %d\n", retv);
+					bp->nb_valid = bp->nb_dirty = 0;
+				}
+			}
+			nfs_buf_upl_check(bp);
+			lck_mtx_lock(nfs_buf_mutex);
+
+			FSDBG(557, bp, bp->nb_flags, bp->nb_valid, bp->nb_dirty);
+			FSDBG(557, bp->nb_validoff, bp->nb_validend,
+			      bp->nb_dirtyoff, bp->nb_dirtyend);
 
-		/* validate any pages written to */
-		start = on & ~PAGE_MASK;
-		for (; start < on+n; start += PAGE_SIZE) {
-			NBPGVALID_SET(bp, start/PAGE_SIZE);
 			/*
-			 * This may seem a little weird, but we don't actually set the
-			 * dirty bits for writes.  This is because we keep the dirty range
-			 * in the nb_dirtyoff/nb_dirtyend fields.  Also, particularly for
-			 * delayed writes, when we give the pages back to the VM we don't
-			 * want to keep them marked dirty, because when we later write the
-			 * buffer we won't be able to tell which pages were written dirty
-			 * and which pages were mmapped and dirtied.
+			 * We used to check for dirty pages here; if there were any
+			 * we'd abort the commit and force the entire buffer to be
+			 * written again.
+			 *
+			 * Instead of doing that, we now go ahead and commit the dirty
+			 * range, and then leave the buffer around with dirty pages
+			 * that will be written out later.
 			 */
+
+			/*
+			 * Work out if all buffers are using the same cred
+			 * so we can deal with them all with one commit.
+			 *
+			 * Note: creds in bp's must be obtained by kauth_cred_ref
+			 * on the same original cred in order for them to be equal.
+			 */
+			if (wcred_set == 0) {
+				wcred = bp->nb_wcred;
+				if (!IS_VALID_CRED(wcred))
+					panic("nfs: needcommit w/out wcred");
+				wcred_set = 1;
+			} else if ((wcred_set == 1) && wcred != bp->nb_wcred) {
+				wcred_set = -1;
+			}
+			SET(bp->nb_flags, NB_WRITEINPROG);
+
+			/*
+			 * A list of these buffers is kept so that the
+			 * second loop knows which buffers have actually
+			 * been committed. This is necessary, since there
+			 * may be a race between the commit rpc and new
+			 * uncommitted writes on the file.
+			 */
+			LIST_REMOVE(bp, nb_vnbufs);
+			LIST_INSERT_HEAD(&commitlist, bp, nb_vnbufs);
+			toff = NBOFF(bp) + bp->nb_dirtyoff;
+			if (toff < off)
+				off = toff;
+			toff += (u_quad_t)(bp->nb_dirtyend - bp->nb_dirtyoff);
+			if (toff > endoff)
+				endoff = toff;
+		}
+		nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
+	}
+	lck_mtx_unlock(nfs_buf_mutex);
+
+	if (LIST_EMPTY(&commitlist)) {
+		error = ENOBUFS;
+		goto done;
+	}
+
+	/*
+	 * Commit data on the server, as required.
+	 * If all bufs are using the same wcred, then use that with
+	 * one call for all of them, otherwise commit each one
+	 * separately.
+	 */
+	if (wcred_set == 1) {
+		/*
+		 * Note, it's possible the commit range could be >2^32-1.
+		 * If it is, we'll send one commit that covers the whole file.
+		 */
+		if ((endoff - off) > 0xffffffff)
+			count = 0;
+		else
+			count = (endoff - off);
+		retv = nmp->nm_funcs->nf_commit_rpc(np, off, count, wcred);
+	} else {
+		retv = 0;
+		LIST_FOREACH(bp, &commitlist, nb_vnbufs) {
+			toff = NBOFF(bp) + bp->nb_dirtyoff;
+			count = bp->nb_dirtyend - bp->nb_dirtyoff;
+			retv = nmp->nm_funcs->nf_commit_rpc(np, toff, count, bp->nb_wcred);
+			if (retv)
+				break;
 		}
-		if (bp->nb_dirtyend > 0) {
-			bp->nb_dirtyoff = min(on, bp->nb_dirtyoff);
-			bp->nb_dirtyend = max((on + n), bp->nb_dirtyend);
-		} else {
-			bp->nb_dirtyoff = on;
-			bp->nb_dirtyend = on + n;
+	}
+
+	/*
+	 * Now, either mark the blocks I/O done or mark the
+	 * blocks dirty, depending on whether the commit
+	 * succeeded.
+	 */
+	while ((bp = LIST_FIRST(&commitlist))) {
+		LIST_REMOVE(bp, nb_vnbufs);
+		FSDBG(557, bp, retv, bp->nb_flags, bp->nb_dirty);
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		CLR(bp->nb_flags, (NB_NEEDCOMMIT | NB_WRITEINPROG));
+		np->n_needcommitcnt--;
+		CHECK_NEEDCOMMITCNT(np);
+		nfs_unlock(np);
+
+		if (retv) {
+			/* move back to dirty list */
+			lck_mtx_lock(nfs_buf_mutex);
+			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
+			lck_mtx_unlock(nfs_buf_mutex);
+			nfs_buf_release(bp, 1);
+			continue;
 		}
-		if (bp->nb_validend <= 0 || bp->nb_validend < bp->nb_dirtyoff ||
-		    bp->nb_validoff > bp->nb_dirtyend) {
-			bp->nb_validoff = bp->nb_dirtyoff;
-			bp->nb_validend = bp->nb_dirtyend;
-		} else {
-			bp->nb_validoff = min(bp->nb_validoff, bp->nb_dirtyoff);
-			bp->nb_validend = max(bp->nb_validend, bp->nb_dirtyend);
+
+		vnode_startwrite(NFSTOV(np));
+		if (ISSET(bp->nb_flags, NB_DELWRI)) {
+			lck_mtx_lock(nfs_buf_mutex);
+			nfs_nbdwrite--;
+			NFSBUFCNTCHK();
+			lck_mtx_unlock(nfs_buf_mutex);
+			wakeup(&nfs_nbdwrite);
 		}
-		if (!ISSET(bp->nb_flags, NB_CACHE))
-			nfs_buf_normalize_valid_range(np, bp);
+		CLR(bp->nb_flags, (NB_READ|NB_DONE|NB_ERROR|NB_DELWRI));
+		/* if block still has dirty pages, we don't want it to */
+		/* be released in nfs_buf_iodone().  So, don't set NB_ASYNC. */
+		if (!(dirty = bp->nb_dirty))
+			SET(bp->nb_flags, NB_ASYNC);
+		else
+			CLR(bp->nb_flags, NB_ASYNC);
 
-		/*
-		 * Since this block is being modified, it must be written
-		 * again and not just committed.
-		 */
-		if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
-			np->n_needcommitcnt--;
-			CHECK_NEEDCOMMITCNT(np);
+		/* move to clean list */
+		lck_mtx_lock(nfs_buf_mutex);
+		LIST_INSERT_HEAD(&np->n_cleanblkhd, bp, nb_vnbufs);
+		lck_mtx_unlock(nfs_buf_mutex);
+
+		bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+
+		nfs_buf_iodone(bp);
+		if (dirty) {
+			/* throw it back in as a delayed write buffer */
+			CLR(bp->nb_flags, NB_DONE);
+			nfs_buf_write_delayed(bp);
 		}
-		CLR(bp->nb_flags, NB_NEEDCOMMIT);
+	}
 
-		if (ioflag & IO_SYNC) {
-			bp->nb_proc = p;
-			error = nfs_buf_write(bp);
-			if (error) {
-				np->n_flag &= ~NWRBUSY;
-				FSDBG_BOT(515, vp, uio->uio_offset,
-					uio_uio_resid(uio), error);
-				return (error);
+done:
+	FSDBG_BOT(557, np, 0, 0, error);
+	return (error);
+}
+
+/*
+ * Flush all the blocks associated with a vnode.
+ * 	Walk through the buffer pool and push any dirty pages
+ *	associated with the vnode.
+ */
+int
+nfs_flush(nfsnode_t np, int waitfor, thread_t thd, int ignore_writeerr)
+{
+	struct nfsbuf *bp;
+	struct nfsbuflists blist;
+	struct nfsmount *nmp = NFSTONMP(np);
+	int error = 0, error2, slptimeo = 0, slpflag = 0;
+	int nfsvers, flags, passone = 1;
+
+	FSDBG_TOP(517, np, waitfor, ignore_writeerr, 0);
+
+	if (!nmp) {
+		error = ENXIO;
+		goto out;
+	}
+	nfsvers = nmp->nm_vers;
+	if (nmp->nm_flag & NFSMNT_INT)
+		slpflag = PCATCH;
+
+	if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		np->n_flag |= NMODIFIED;
+		nfs_unlock(np);
+	}
+
+	lck_mtx_lock(nfs_buf_mutex);
+	while (np->n_bflag & NBFLUSHINPROG) {
+		np->n_bflag |= NBFLUSHWANT;
+		error = msleep(&np->n_bflag, nfs_buf_mutex, slpflag, "nfs_flush", NULL);
+		if (error) {
+			lck_mtx_unlock(nfs_buf_mutex);
+			goto out;
+		}
+	}
+	np->n_bflag |= NBFLUSHINPROG;
+
+	/*
+	 * On the first pass, start async/unstable writes on all
+	 * delayed write buffers.  Then wait for all writes to complete
+	 * and call nfs_flushcommits() to commit any uncommitted buffers.
+	 * On all subsequent passes, start STABLE writes on any remaining
+	 * dirty buffers.  Then wait for all writes to complete.
+	 */
+again:
+	FSDBG(518, LIST_FIRST(&np->n_dirtyblkhd), np->n_flag, 0, 0);
+	if (!NFSTONMP(np)) {
+		lck_mtx_unlock(nfs_buf_mutex);
+		error = ENXIO;
+		goto done;
+	}
+
+	/* Start/do any write(s) that are required. */
+	if (!nfs_buf_iterprepare(np, &blist, NBI_DIRTY)) {
+		while ((bp = LIST_FIRST(&blist))) {
+			LIST_REMOVE(bp, nb_vnbufs);
+			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
+			flags = (passone || (waitfor != MNT_WAIT)) ? NBAC_NOWAIT : 0;
+			if (flags != NBAC_NOWAIT)
+				nfs_buf_refget(bp);
+			while ((error = nfs_buf_acquire(bp, flags, slpflag, slptimeo))) {
+				FSDBG(524, bp, flags, bp->nb_lflags, bp->nb_flags);
+				if (error == EBUSY)
+					break;
+				if (error) {
+					error2 = nfs_sigintr(NFSTONMP(np), NULL, thd, 0);
+					if (error2) {
+						if (flags != NBAC_NOWAIT)
+							nfs_buf_refrele(bp);
+						nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
+						lck_mtx_unlock(nfs_buf_mutex);
+						error = error2;
+						goto done;
+					}
+					if (slpflag == PCATCH) {
+						slpflag = 0;
+						slptimeo = 2 * hz;
+					}
+				}
+			}
+			if (flags != NBAC_NOWAIT)
+				nfs_buf_refrele(bp);
+			if (error == EBUSY)
+				continue;
+			if (!bp->nb_np) {
+				/* buffer is no longer valid */
+				nfs_buf_drop(bp);
+				continue;
+			}
+			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
+				nfs_buf_check_write_verifier(np, bp);
+			if (!ISSET(bp->nb_flags, NB_DELWRI)) {
+				/* buffer is no longer dirty */
+				nfs_buf_drop(bp);
+				continue;
+			}
+			FSDBG(525, bp, passone, bp->nb_lflags, bp->nb_flags);
+			if ((passone || (waitfor != MNT_WAIT)) &&
+			    ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+				nfs_buf_drop(bp);
+				continue;
+			}
+			nfs_buf_remfree(bp);
+			lck_mtx_unlock(nfs_buf_mutex);
+			if (ISSET(bp->nb_flags, NB_ERROR)) {
+				nfs_lock(np, NFS_NODE_LOCK_FORCE);
+				np->n_error = bp->nb_error ? bp->nb_error : EIO;
+				np->n_flag |= NWRITEERR;
+				nfs_unlock(np);
+				nfs_buf_release(bp, 1);
+				lck_mtx_lock(nfs_buf_mutex);
+				continue;
+			}
+			SET(bp->nb_flags, NB_ASYNC);
+			if (!passone) {
+				/* NB_STABLE forces this to be written FILESYNC */
+				SET(bp->nb_flags, NB_STABLE);
+			}
+			nfs_buf_write(bp);
+			lck_mtx_lock(nfs_buf_mutex);
+		}
+		nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
+	}
+	lck_mtx_unlock(nfs_buf_mutex);
+
+	if (waitfor == MNT_WAIT) {
+	        while ((error = vnode_waitforwrites(NFSTOV(np), 0, slpflag, slptimeo, "nfsflush"))) {
+		        error2 = nfs_sigintr(NFSTONMP(np), NULL, thd, 0);
+			if (error2) {
+			        error = error2;
+				goto done;
+			}
+			if (slpflag == PCATCH) {
+				slpflag = 0;
+				slptimeo = 2 * hz;
 			}
-		} else if (((n + on) == biosize) || (np->n_flag & NNOCACHE)) {
-			bp->nb_proc = NULL;
-			SET(bp->nb_flags, NB_ASYNC);
-			nfs_buf_write(bp);
-		} else
-			nfs_buf_write_delayed(bp, p);
+		}
+	}
 
-		if (np->n_needcommitcnt > (nfsbufcnt/16))
-		        nfs_flushcommits(vp, p, 1);
+	if (nfsvers != NFS_VER2) {
+		/* loop while it looks like there are still buffers to be */
+		/* commited and nfs_flushcommits() seems to be handling them. */
+		while (np->n_needcommitcnt)
+			if (nfs_flushcommits(np, 0))
+				break;
+	}
 
-	} while (uio_uio_resid(uio) > 0 && n > 0);
+	if (passone) {
+		passone = 0;
+		if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			np->n_flag |= NMODIFIED;
+			nfs_unlock(np);
+		}
+		lck_mtx_lock(nfs_buf_mutex);
+		goto again;
+	}
 
-	if (np->n_flag & NNOCACHE) {
-		/* make sure all the buffers are flushed out */
-		error = nfs_flush(vp, MNT_WAIT, cred, p, 0);
+	if (waitfor == MNT_WAIT) {
+		if (!LIST_EMPTY(&np->n_dirtyblkhd)) {
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			np->n_flag |= NMODIFIED;
+			nfs_unlock(np);
+		}
+		lck_mtx_lock(nfs_buf_mutex);
+		if (!LIST_EMPTY(&np->n_dirtyblkhd))
+			goto again;
+		lck_mtx_unlock(nfs_buf_mutex);
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		/* if we have no dirty blocks, we can clear the modified flag */
+		if (!np->n_wrbusy)
+			np->n_flag &= ~NMODIFIED;
+	} else {
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
 	}
 
-	np->n_flag &= ~NWRBUSY;
-	FSDBG_BOT(515, vp, uio->uio_offset, uio_uio_resid(uio), error);
+	FSDBG(526, np->n_flag, np->n_error, 0, 0);
+	if (!ignore_writeerr && (np->n_flag & NWRITEERR)) {
+		error = np->n_error;
+		np->n_flag &= ~NWRITEERR;
+	}
+	nfs_unlock(np);
+done:
+	lck_mtx_lock(nfs_buf_mutex);
+	flags = np->n_bflag;
+	np->n_bflag &= ~(NBFLUSHINPROG|NBFLUSHWANT);
+	lck_mtx_unlock(nfs_buf_mutex);
+	if (flags & NBFLUSHWANT)
+		wakeup(&np->n_bflag);
+out:
+	FSDBG_BOT(517, np, error, ignore_writeerr, 0);
 	return (error);
 }
 
@@ -2508,25 +3485,20 @@ again:
  */
 static int
 nfs_vinvalbuf_internal(
-	vnode_t vp,
+	nfsnode_t np,
 	int flags,
+	thread_t thd,
 	kauth_cred_t cred,
-	proc_t p,
 	int slpflag,
 	int slptimeo)
 {
 	struct nfsbuf *bp;
 	struct nfsbuflists blist;
 	int list, error = 0;
-	struct nfsnode *np = VTONFS(vp);
 
 	if (flags & V_SAVE) {
-		if ((error = nfs_flush(vp, MNT_WAIT, cred, p,
-					(flags & V_IGNORE_WRITEERR))))
+		if ((error = nfs_flush(np, MNT_WAIT, thd, (flags & V_IGNORE_WRITEERR))))
 			return (error);
-		if (!LIST_EMPTY(&np->n_dirtyblkhd))
-			panic("nfs_vinvalbuf: dirty bufs (vp 0x%x, bp 0x%x)",
-				vp, LIST_FIRST(&np->n_dirtyblkhd));
 	}
 
 	lck_mtx_lock(nfs_buf_mutex);
@@ -2545,9 +3517,9 @@ nfs_vinvalbuf_internal(
 				LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
 			nfs_buf_refget(bp);
 			while ((error = nfs_buf_acquire(bp, NBAC_REMOVE, slpflag, slptimeo))) {
-				FSDBG(556, vp, bp, NBOFF(bp), bp->nb_flags);
+				FSDBG(556, np, bp, NBOFF(bp), bp->nb_flags);
 				if (error != EAGAIN) {
-					FSDBG(554, vp, bp, -1, error);
+					FSDBG(554, np, bp, -1, error);
 					nfs_buf_refrele(bp);
 					nfs_buf_itercomplete(np, &blist, list);
 					lck_mtx_unlock(nfs_buf_mutex);
@@ -2555,11 +3527,11 @@ nfs_vinvalbuf_internal(
 				}
 			}
 			nfs_buf_refrele(bp);
-			FSDBG(554, vp, bp, NBOFF(bp), bp->nb_flags);
+			FSDBG(554, np, bp, NBOFF(bp), bp->nb_flags);
 			lck_mtx_unlock(nfs_buf_mutex);
-			if ((flags & V_SAVE) && UBCINFOEXISTS(vp) && bp->nb_vp &&
+			if ((flags & V_SAVE) && UBCINFOEXISTS(NFSTOV(np)) && bp->nb_np &&
 			    (NBOFF(bp) < (off_t)np->n_size)) {
-				/* XXX extra paranoia: make sure we're not */
+				/* extra paranoia: make sure we're not */
 				/* somehow leaving any dirty data around */
 				int mustwrite = 0;
 				int end = (NBOFF(bp) + bp->nb_bufsize > (off_t)np->n_size) ?
@@ -2576,20 +3548,26 @@ nfs_vinvalbuf_internal(
 				}
 				nfs_buf_upl_check(bp);
 				/* check for any dirty data before the EOF */
-				if (bp->nb_dirtyend && bp->nb_dirtyoff < end) {
+				if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < end)) {
 					/* clip dirty range to EOF */
-					if (bp->nb_dirtyend > end)
+					if (bp->nb_dirtyend > end) {
 						bp->nb_dirtyend = end;
-					mustwrite++;
+						if (bp->nb_dirtyoff >= bp->nb_dirtyend)
+							bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+					}
+					if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < end))
+						mustwrite++;
 				}
 				bp->nb_dirty &= (1 << (round_page_32(end)/PAGE_SIZE)) - 1;
+				if (bp->nb_dirty)
+					mustwrite++;
 				/* also make sure we'll have a credential to do the write */
 				if (mustwrite && !IS_VALID_CRED(bp->nb_wcred) && !IS_VALID_CRED(cred)) {
 					printf("nfs_vinvalbuf: found dirty buffer with no write creds\n");
 					mustwrite = 0;
 				}
 				if (mustwrite) {
-					FSDBG(554, vp, bp, 0xd00dee, bp->nb_flags);
+					FSDBG(554, np, bp, 0xd00dee, bp->nb_flags);
 					if (!ISSET(bp->nb_flags, NB_PAGELIST))
 						panic("nfs_vinvalbuf: dirty buffer without upl");
 					/* gotta write out dirty data before invalidating */
@@ -2605,8 +3583,10 @@ nfs_vinvalbuf_internal(
 					// Note: bp has been released
 					if (error) {
 						FSDBG(554, bp, 0xd00dee, 0xbad, error);
+						nfs_lock(np, NFS_NODE_LOCK_FORCE);
 						np->n_error = error;
 						np->n_flag |= NWRITEERR;
+						nfs_unlock(np);
 						/*
 						 * There was a write error and we need to
 						 * invalidate attrs to sync with server.
@@ -2627,10 +3607,15 @@ nfs_vinvalbuf_internal(
 		}
 		nfs_buf_itercomplete(np, &blist, list);
 	}
+	if (!LIST_EMPTY(&(np)->n_dirtyblkhd) || !LIST_EMPTY(&(np)->n_cleanblkhd))
+		panic("nfs_vinvalbuf: flush/inval failed");
 	lck_mtx_unlock(nfs_buf_mutex);
+	if (!(flags & V_SAVE)) {
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		np->n_flag &= ~NMODIFIED;
+		nfs_unlock(np);
+	}
 	NFS_BUF_FREEUP();
-	if (NVALIDBUFS(np))
-		panic("nfs_vinvalbuf: flush failed");
 	return (0);
 }
 
@@ -2640,21 +3625,22 @@ nfs_vinvalbuf_internal(
  * doing the flush, just wait for completion.
  */
 int
-nfs_vinvalbuf(
-	vnode_t vp,
-	int flags,
-	kauth_cred_t cred,
-	proc_t p,
-	int intrflg)
+nfs_vinvalbuf(vnode_t vp, int flags, vfs_context_t ctx, int intrflg)
+{
+	return nfs_vinvalbuf2(vp, flags, vfs_context_thread(ctx), vfs_context_ucred(ctx), intrflg);
+}
+
+int
+nfs_vinvalbuf2(vnode_t vp, int flags, thread_t thd, kauth_cred_t cred, int intrflg)
 {
-	struct nfsnode *np = VTONFS(vp);
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-	int error = 0, slpflag, slptimeo;
+	nfsnode_t np = VTONFS(vp);
+	struct nfsmount *nmp = VTONMP(vp);
+	int error, rv, slpflag, slptimeo, nflags;
 	off_t size;
 
-	FSDBG_TOP(554, vp, flags, intrflg, 0);
+	FSDBG_TOP(554, np, flags, intrflg, 0);
 
-	if (nmp && ((nmp->nm_flag & NFSMNT_INT) == 0))
+	if (nmp && !(nmp->nm_flag & NFSMNT_INT))
 		intrflg = 0;
 	if (intrflg) {
 		slpflag = PCATCH;
@@ -2663,636 +3649,190 @@ nfs_vinvalbuf(
 		slpflag = 0;
 		slptimeo = 0;
 	}
-	/*
-	 * First wait for any other process doing a flush to complete.
-	 */
-	while (np->n_flag & NFLUSHINPROG) {
-		np->n_flag |= NFLUSHWANT;
-		FSDBG_TOP(555, vp, flags, intrflg, np->n_flag);
-		error = tsleep((caddr_t)&np->n_flag, PRIBIO + 2, "nfsvinval", slptimeo);
-		FSDBG_BOT(555, vp, flags, intrflg, np->n_flag);
-		if (error && (error = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p))) {
-			FSDBG_BOT(554, vp, flags, intrflg, error);
-			return (error);
-		}
-	}
 
-	/*
-	 * Now, flush as required.
-	 */
-	np->n_flag |= NFLUSHINPROG;
-	error = nfs_vinvalbuf_internal(vp, flags, cred, p, slpflag, 0);
-	while (error) {
-		FSDBG(554, vp, 0, 0, error);
-		error = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p);
+	/* First wait for any other process doing a flush to complete.  */
+	lck_mtx_lock(nfs_buf_mutex);
+	while (np->n_bflag & NBINVALINPROG) {
+		np->n_bflag |= NBINVALWANT;
+		error = msleep(&np->n_bflag, nfs_buf_mutex, slpflag, "nfs_vinvalbuf", NULL);
 		if (error) {
-			np->n_flag &= ~NFLUSHINPROG;
-			if (np->n_flag & NFLUSHWANT) {
-				np->n_flag &= ~NFLUSHWANT;
-				wakeup((caddr_t)&np->n_flag);
-			}
-			FSDBG_BOT(554, vp, flags, intrflg, error);
+			lck_mtx_unlock(nfs_buf_mutex);
 			return (error);
 		}
-		error = nfs_vinvalbuf_internal(vp, flags, cred, p, 0, slptimeo);
 	}
-	np->n_flag &= ~(NMODIFIED | NFLUSHINPROG);
-	if (np->n_flag & NFLUSHWANT) {
-		np->n_flag &= ~NFLUSHWANT;
-		wakeup((caddr_t)&np->n_flag);
+	np->n_bflag |= NBINVALINPROG;
+	lck_mtx_unlock(nfs_buf_mutex);
+
+	/* Now, flush as required.  */
+	error = nfs_vinvalbuf_internal(np, flags, thd, cred, slpflag, 0);
+	while (error) {
+		FSDBG(554, np, 0, 0, error);
+		if ((error = nfs_sigintr(VTONMP(vp), NULL, thd, 0)))
+			goto done;
+		error = nfs_vinvalbuf_internal(np, flags, thd, cred, 0, slptimeo);
 	}
-	/*
-	 * get the pages out of vm also
-	 */
-	if (UBCINFOEXISTS(vp) && (size = ubc_getsize(vp))) {
-		int rv = ubc_sync_range(vp, 0, size, UBC_PUSHALL | UBC_INVALIDATE);
-		if (!rv)
+
+	/* get the pages out of vm also */
+	if (UBCINFOEXISTS(vp) && (size = ubc_getsize(vp)))
+		if (!(rv = ubc_sync_range(vp, 0, size, UBC_PUSHALL | UBC_SYNC | UBC_INVALIDATE)))
 			panic("nfs_vinvalbuf(): ubc_sync_range failed!");
-	}
+done:
+	lck_mtx_lock(nfs_buf_mutex);
+	nflags = np->n_bflag;
+	np->n_bflag &= ~(NBINVALINPROG|NBINVALWANT);
+	lck_mtx_unlock(nfs_buf_mutex);
+	if (nflags & NBINVALWANT)
+		wakeup(&np->n_bflag);
 
-	FSDBG_BOT(554, vp, flags, intrflg, 0);
-	return (0);
+	FSDBG_BOT(554, np, flags, intrflg, error);
+	return (error);
 }
 
 /*
- * Initiate asynchronous I/O. Return an error if no nfsiods are available.
- * This is mainly to avoid queueing async I/O requests when the nfsiods
- * are all hung on a dead server.
+ * Add an async I/O request to the mount's async I/O queue and make
+ * sure that an nfsiod will service it.
  */
-int
-nfs_asyncio(bp, cred)
-	struct nfsbuf *bp;
-	kauth_cred_t cred;
+void
+nfs_asyncio_finish(struct nfsreq *req)
 {
 	struct nfsmount *nmp;
-	int i;
-	int gotiod;
-	int slpflag = 0;
-	int slptimeo = 0;
-	int error, error2;
-	void *wakeme = NULL;
-	struct timespec ts;
+	struct nfsiod *niod;
+	int started = 0;
 
-	if (nfs_numasync == 0)
-		return (EIO);
-
-	FSDBG_TOP(552, bp, bp ? NBOFF(bp) : 0, bp ? bp->nb_flags : 0, 0);
-
-	nmp = ((bp != NULL) ? VFSTONFS(vnode_mount(bp->nb_vp)) : NULL);
+	FSDBG_TOP(552, nmp, 0, 0, 0);
 again:
-	if (nmp && nmp->nm_flag & NFSMNT_INT)
-		slpflag = PCATCH;
-	gotiod = FALSE;
-
-	lck_mtx_lock(nfs_iod_mutex);
-
-	/* no nfsbuf means tell nfsiod to process delwri list */
-	if (!bp)
-		nfs_ioddelwri = 1;
-
-	/*
-	 * Find a free iod to process this request.
-	 */
-	for (i = 0; i < NFS_MAXASYNCDAEMON; i++)
-		if (nfs_iodwant[i]) {
+	if (((nmp = req->r_nmp)) == NULL)
+		return;
+	lck_mtx_lock(nfsiod_mutex);
+	niod = nmp->nm_niod;
+
+	/* grab an nfsiod if we don't have one already */
+	if (!niod) {
+		niod = TAILQ_FIRST(&nfsiodfree);
+		if (niod) {
+			TAILQ_REMOVE(&nfsiodfree, niod, niod_link);
+			TAILQ_INSERT_TAIL(&nfsiodwork, niod, niod_link);
+			niod->niod_nmp = nmp;
+		} else if (((nfsiod_thread_count < NFSIOD_MAX) || (nfsiod_thread_count <= 0)) && (started < 4)) {
 			/*
-			 * Found one, so wake it up and tell it which
-			 * mount to process.
+			 * Try starting a new thread.
+			 * We may try a couple times if other callers
+			 * get the new threads before we do.
 			 */
-			nfs_iodwant[i] = NULL;
-			nfs_iodmount[i] = nmp;
-			if (nmp)
-				nmp->nm_bufqiods++;
-			wakeme = &nfs_iodwant[i];
-			gotiod = TRUE;
-			break;
+			lck_mtx_unlock(nfsiod_mutex);
+			started++;
+			if (!nfsiod_start())
+				goto again;
+			lck_mtx_lock(nfsiod_mutex);
 		}
-
-	/* if we're just poking the delwri list, we're done */
-	if (!bp) {
-		lck_mtx_unlock(nfs_iod_mutex);
-		if (wakeme)
-			wakeup(wakeme);
-		FSDBG_BOT(552, bp, 0x10101010, wakeme, 0);
-		return (0);
 	}
 
-	/*
-	 * If none are free, we may already have an iod working on this mount
-	 * point.  If so, it will process our request.
-	 */
-	if (!gotiod) {
-		if (nmp->nm_bufqiods > 0) {
-			gotiod = TRUE;
+	if (req->r_achain.tqe_next == NFSREQNOLIST)
+		TAILQ_INSERT_TAIL(&nmp->nm_iodq, req, r_achain);
+
+	/* If this mount doesn't already have an nfsiod working on it... */
+	if (!nmp->nm_niod) {
+		if (niod) { /* give it the nfsiod we just grabbed */
+			nmp->nm_niod = niod;
+			lck_mtx_unlock(nfsiod_mutex);
+			wakeup(niod);
+		} else if (nfsiod_thread_count > 0) {
+			/* just queue it up on nfsiod mounts queue */
+			TAILQ_INSERT_TAIL(&nfsiodmounts, nmp, nm_iodlink);
+			lck_mtx_unlock(nfsiod_mutex);
+		} else {
+			printf("nfs_asyncio(): no nfsiods? %d %d (%d)\n", nfsiod_thread_count, NFSIOD_MAX, started);
+			lck_mtx_unlock(nfsiod_mutex);
+			/* we have no other option but to be persistent */
+			started = 0;
+			goto again;
 		}
+	} else {
+		lck_mtx_unlock(nfsiod_mutex);
 	}
 
-	/*
-	 * If we have an iod which can process the request, then queue
-	 * the buffer.
-	 */
-	FSDBG(552, bp, gotiod, i, nmp->nm_bufqiods);
-	if (gotiod) {
-		/*
-		 * Ensure that the queue never grows too large.
-		 */
-		while (nmp->nm_bufqlen >= 2*nfs_numasync) {
-			if (ISSET(bp->nb_flags, NB_IOD)) {
-				/* An nfsiod is attempting this async operation so */
-				/* we must not fall asleep on the bufq because we */
-				/* could be waiting on ourself.  Just return error */
-				/* and we'll do this operation syncrhonously. */
-				goto out;
-			}
-			FSDBG(552, bp, nmp->nm_bufqlen, 2*nfs_numasync, -1);
-			nmp->nm_bufqwant = TRUE;
-
-			ts.tv_sec = (slptimeo/100);
-			/* the hz value is 100; which leads to 10ms */
-			ts.tv_nsec = (slptimeo % 100) * 10  * NSEC_PER_USEC * 1000;
-
-			error = msleep(&nmp->nm_bufq, nfs_iod_mutex, slpflag | PRIBIO,
-				       "nfsaio", &ts);
-			if (error) {
-				error2 = nfs_sigintr(nmp, NULL, bp->nb_proc);
-				if (error2) {
-					lck_mtx_unlock(nfs_iod_mutex);
-					FSDBG_BOT(552, bp, NBOFF(bp), bp->nb_flags, error2);
-					return (error2);
-				}
-				if (slpflag == PCATCH) {
-					slpflag = 0;
-					slptimeo = 2 * hz;
-				}
-			}
-			/*
-			 * We might have lost our iod while sleeping,
-			 * so check and loop if nescessary.
-			 */
-			if (nmp->nm_bufqiods == 0) {
-				lck_mtx_unlock(nfs_iod_mutex);
-				goto again;
-			}
-		}
+	FSDBG_BOT(552, nmp, 0, 0, 0);
+}
 
-		if (ISSET(bp->nb_flags, NB_READ)) {
-			if (!IS_VALID_CRED(bp->nb_rcred) && IS_VALID_CRED(cred)) {
-				kauth_cred_ref(cred);
-				bp->nb_rcred = cred;
-			}
-		} else {
-			SET(bp->nb_flags, NB_WRITEINPROG);
-			if (!IS_VALID_CRED(bp->nb_wcred) && IS_VALID_CRED(cred)) {
-				kauth_cred_ref(cred);
-				bp->nb_wcred = cred;
-			}
-		}
+/*
+ * queue up async I/O request for resend
+ */
+void
+nfs_asyncio_resend(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_nmp;
 
-		TAILQ_INSERT_TAIL(&nmp->nm_bufq, bp, nb_free);
-		nmp->nm_bufqlen++;
-		lck_mtx_unlock(nfs_iod_mutex);
-		if (wakeme)
-			wakeup(wakeme);
-		FSDBG_BOT(552, bp, NBOFF(bp), bp->nb_flags, 0);
-		return (0);
+	if (!nmp)
+		return;
+	nfs_gss_clnt_rpcdone(req);
+	lck_mtx_lock(&nmp->nm_lock);
+	if (req->r_rchain.tqe_next == NFSREQNOLIST) {
+		TAILQ_INSERT_TAIL(&nmp->nm_resendq, req, r_rchain);
+		req->r_flags |= R_RESENDQ;
 	}
-
-out:
-	lck_mtx_unlock(nfs_iod_mutex);
-	/*
-	 * All the iods are busy on other mounts, so return EIO to
-	 * force the caller to process the i/o synchronously.
-	 */
-	FSDBG_BOT(552, bp, NBOFF(bp), bp->nb_flags, EIO);
-	return (EIO);
+	nfs_mount_sock_thread_wake(nmp);
+	lck_mtx_unlock(&nmp->nm_lock);
 }
 
 /*
- * Do an I/O operation to/from a cache block. This may be called
- * synchronously or from an nfsiod.
+ * Read an NFS buffer for a directory.
  */
 int
-nfs_doio(struct nfsbuf *bp, kauth_cred_t cr, proc_t p)
+nfs_buf_readdir(struct nfsbuf *bp, vfs_context_t ctx)
 {
-	struct uio *uiop;
+	nfsnode_t np;
 	vnode_t vp;
-	struct nfsnode *np;
 	struct nfsmount *nmp;
-	int error = 0, diff, len, iomode, invalidate = 0;
+	int error = 0, nfsvers;
 	struct uio uio;
 	struct iovec_32 io;
-	enum vtype vtype;
 
-	vp = bp->nb_vp;
-	vtype = vnode_vtype(vp);
-	np = VTONFS(vp);
-	nmp = VFSTONFS(vnode_mount(vp));
-	uiop = &uio;
-	uiop->uio_iovs.iov32p = &io;
-	uiop->uio_iovcnt = 1;
+	np = bp->nb_np;
+	vp = NFSTOV(np);
+	nmp = VTONMP(vp);
+	nfsvers = nmp->nm_vers;
+	uio.uio_iovs.iov32p = &io;
+	uio.uio_iovcnt = 1;
 #if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
-	uiop->uio_segflg = UIO_SYSSPACE;
+	uio.uio_segflg = UIO_SYSSPACE;
 #else
-	uiop->uio_segflg = UIO_SYSSPACE32;
-#endif 
+	uio.uio_segflg = UIO_SYSSPACE32;
+#endif
 
-	/*
-	 * we've decided to perform I/O for this block,
-	 * so we couldn't possibly NB_DONE.  So, clear it.
-	 */
-	if (ISSET(bp->nb_flags, NB_DONE)) {
-		if (!ISSET(bp->nb_flags, NB_ASYNC))
-			panic("nfs_doio: done and not async");
+	/* sanity check */
+	if (ISSET(bp->nb_flags, NB_DONE))
 		CLR(bp->nb_flags, NB_DONE);
-	}
-	FSDBG_TOP(256, np->n_size, NBOFF(bp), bp->nb_bufsize, bp->nb_flags);
-	FSDBG(257, bp->nb_validoff, bp->nb_validend, bp->nb_dirtyoff,
-	      bp->nb_dirtyend);
-
-	if (ISSET(bp->nb_flags, NB_READ)) {
-	    if (vtype == VREG)
-		    NFS_BUF_MAP(bp);
-	    io.iov_len = bp->nb_bufsize;
-	    uio_uio_resid_set(uiop, io.iov_len);
-	    io.iov_base = (uintptr_t) bp->nb_data;
-	    uiop->uio_rw = UIO_READ;
-	    switch (vtype) {
-	    case VREG:
-		uiop->uio_offset = NBOFF(bp);
-		OSAddAtomic(1, (SInt32*)&nfsstats.read_bios);
-		error = nfs_readrpc(vp, uiop, cr, p);
-		FSDBG(262, np->n_size, NBOFF(bp), uio_uio_resid(uiop), error);
-		if (!error) {
-		    /* update valid range */
-		    bp->nb_validoff = 0;
-		    if (uio_uio_resid(uiop) != 0) {
-			/*
-			 * If len > 0, there is a hole in the file and
-			 * no writes after the hole have been pushed to
-			 * the server yet.
-			 * Just zero fill the rest of the valid area.
-			 */
-			// LP64todo - fix this
-			diff = bp->nb_bufsize - uio_uio_resid(uiop);
-			len = np->n_size - (NBOFF(bp) + diff);
-			if (len > 0) {
-				// LP64todo - fix this
-				len = min(len, uio_uio_resid(uiop));
-				bzero((char *)bp->nb_data + diff, len);
-				bp->nb_validend = diff + len;
-				FSDBG(258, diff, len, 0, 1);
-			} else
-				bp->nb_validend = diff;
-		    } else
-				bp->nb_validend = bp->nb_bufsize;
-		    bp->nb_valid = (1 << (round_page_32(bp->nb_validend)/PAGE_SIZE)) - 1;
-		    if (bp->nb_validend & PAGE_MASK) {
-			    /* valid range ends in the middle of a page so we */
-			    /* need to zero-fill any invalid data at the end */
-			    /* of the last page */
-			    bzero((caddr_t)(bp->nb_data + bp->nb_validend),
-			          bp->nb_bufsize - bp->nb_validend);
-			    FSDBG(258, bp->nb_validend,
-			          bp->nb_bufsize - bp->nb_validend, 0, 2);
-		    }
-		}
-		break;
-	    case VLNK:
-		uiop->uio_offset = (off_t)0;
-		OSAddAtomic(1, (SInt32*)&nfsstats.readlink_bios);
-		error = nfs_readlinkrpc(vp, uiop, cr, p);
-		if (!error) {
-			bp->nb_validoff = 0;
-			bp->nb_validend = uiop->uio_offset;
-		}
-		break;
-	    case VDIR:
-		OSAddAtomic(1, (SInt32*)&nfsstats.readdir_bios);
-		uiop->uio_offset = NBOFF(bp);
-		if (!(nmp->nm_flag & NFSMNT_NFSV3))
-			nmp->nm_flag &= ~NFSMNT_RDIRPLUS; /* dk@farm.org */
-		if (nmp->nm_flag & NFSMNT_RDIRPLUS) {
-			error = nfs_readdirplusrpc(vp, uiop, cr, p);
-			if (error == NFSERR_NOTSUPP)
-				nmp->nm_flag &= ~NFSMNT_RDIRPLUS;
-		}
-		if ((nmp->nm_flag & NFSMNT_RDIRPLUS) == 0)
-			error = nfs_readdirrpc(vp, uiop, cr, p);
-		if (!error) {
-			bp->nb_validoff = 0;
-			bp->nb_validend = uiop->uio_offset - NBOFF(bp);
-			bp->nb_valid = (1 << (round_page_32(bp->nb_validend)/PAGE_SIZE)) - 1;
-		}
-		break;
-	    default:
-		printf("nfs_doio: type %x unexpected\n", vtype);
-		break;
-	    };
-	    if (error) {
-		SET(bp->nb_flags, NB_ERROR);
-		bp->nb_error = error;
-	    }
-
-	} else {
-	    /* we're doing a write */
-	    int doff, dend = 0;
-
-	    /* We need to make sure the pages are locked before doing I/O.  */
-	    if (!ISSET(bp->nb_flags, NB_META) && UBCINFOEXISTS(vp)) {
-		if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
-		    error = nfs_buf_upl_setup(bp);
-		    if (error) {
-			printf("nfs_doio: upl create failed %d\n", error);
-			SET(bp->nb_flags, NB_ERROR);
-			bp->nb_error = EIO;
-			return (EIO);
-		    }
-		    nfs_buf_upl_check(bp);
-		}
-	    }
-
-	    if (ISSET(bp->nb_flags, NB_WASDIRTY)) {
-		FSDBG(256, bp, NBOFF(bp), bp->nb_dirty, 0xd00dee);
-		/*
-		 * There are pages marked dirty that need to be written out.
-		 *
-		 * We don't want to just combine the write range with the
-		 * range of pages that are dirty because that could cause us
-		 * to write data that wasn't actually written to.
-		 * We also don't want to write data more than once.
-		 *
-		 * If the dirty range just needs to be committed, we do that.
-		 * Otherwise, we write the dirty range and clear the dirty bits
-		 * for any COMPLETE pages covered by that range.
-		 * If there are dirty pages left after that, we write out the
-		 * parts that we haven't written yet.
-		 */
-	    }
-
-	    /*
-	     * If NB_NEEDCOMMIT is set, a commit rpc may do the trick. If not
-	     * an actual write will have to be done.
-	     * If NB_WRITEINPROG is already set, then push it with a write anyhow.
-	     */
-	    if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
-	    	nfs_buf_check_write_verifier(np, bp);
-	    if ((bp->nb_flags & (NB_NEEDCOMMIT | NB_WRITEINPROG)) == NB_NEEDCOMMIT) {
-		doff = NBOFF(bp) + bp->nb_dirtyoff;
-		SET(bp->nb_flags, NB_WRITEINPROG);
-		error = nfs_commit(vp, doff, bp->nb_dirtyend - bp->nb_dirtyoff,
-				bp->nb_wcred, bp->nb_proc);
-		CLR(bp->nb_flags, NB_WRITEINPROG);
-		if (!error) {
-		    bp->nb_dirtyoff = bp->nb_dirtyend = 0;
-		    CLR(bp->nb_flags, NB_NEEDCOMMIT);
-		    np->n_needcommitcnt--;
-		    CHECK_NEEDCOMMITCNT(np);
-		}
-	    }
-
-	    if (!error && bp->nb_dirtyend > 0) {
-		/* there's a dirty range that needs to be written out */
-		u_int32_t pagemask;
-		int firstpg, lastpg;
-
-		if (NBOFF(bp) + bp->nb_dirtyend > (off_t)np->n_size)
-		    bp->nb_dirtyend = np->n_size - NBOFF(bp);
-
-		NFS_BUF_MAP(bp);
-
-		doff = bp->nb_dirtyoff;
-		dend = bp->nb_dirtyend;
-
-		/* if doff page is dirty, move doff to start of page */
-		if (NBPGDIRTY(bp,doff/PAGE_SIZE))
-		    doff -= doff & PAGE_MASK;
-		/* try to expand write range to include preceding dirty pages */
-		if (!(doff & PAGE_MASK))
-		    while (doff > 0 && NBPGDIRTY(bp,(doff-1)/PAGE_SIZE))
-		    	doff -= PAGE_SIZE;
-		/* if dend page is dirty, move dend to start of next page */
-		if ((dend & PAGE_MASK) && NBPGDIRTY(bp,dend/PAGE_SIZE))
-		    dend = round_page_32(dend);
-		/* try to expand write range to include trailing dirty pages */
-		if (!(dend & PAGE_MASK))
-		    while (dend < bp->nb_bufsize && NBPGDIRTY(bp,dend/PAGE_SIZE))
-		    	dend += PAGE_SIZE;
-		/* make sure to keep dend clipped to EOF */
-		if (NBOFF(bp) + dend > (off_t)np->n_size)
-		    dend = np->n_size - NBOFF(bp);
-		/* calculate range of complete pages being written */
-		firstpg = round_page_32(doff) / PAGE_SIZE;
-		lastpg = (trunc_page_32(dend) - 1)/ PAGE_SIZE;
-		/* calculate mask for that page range */
-		pagemask = ((1 << (lastpg+1)) - 1) & ~((1 << firstpg) - 1);
-
-		/* compare page mask to nb_dirty; if there are other dirty pages */
-		/* then write FILESYNC; otherwise, write UNSTABLE if async and */
-		/* not needcommit/stable; otherwise write FILESYNC */
-		if (bp->nb_dirty & ~pagemask)
-		    iomode = NFSV3WRITE_FILESYNC;
-		else if ((bp->nb_flags & (NB_ASYNC | NB_NEEDCOMMIT | NB_STABLE)) == NB_ASYNC)
-		    iomode = NFSV3WRITE_UNSTABLE;
-		else
-		    iomode = NFSV3WRITE_FILESYNC;
 
-		/* write the dirty range */
-		io.iov_len = dend - doff;
-		uio_uio_resid_set(uiop, io.iov_len);
-		uiop->uio_offset = NBOFF(bp) + doff;
-		io.iov_base = (uintptr_t) bp->nb_data + doff;
-		uiop->uio_rw = UIO_WRITE;
+	uio.uio_rw = UIO_READ;
+	io.iov_len = bp->nb_bufsize;
+	uio_uio_resid_set(&uio, io.iov_len);
+	io.iov_base = (uintptr_t) bp->nb_data;
+	uio.uio_offset = NBOFF(bp);
 
-		OSAddAtomic(1, (SInt32*)&nfsstats.write_bios);
-
-		SET(bp->nb_flags, NB_WRITEINPROG);
-		error = nfs_writerpc(vp, uiop, cr, p, &iomode, &bp->nb_verf);
-		/* clear dirty bits for pages we've written */
-		if (!error)
-		    bp->nb_dirty &= ~pagemask;
-		/* set/clear needcommit flag */
-		if (!error && iomode == NFSV3WRITE_UNSTABLE) {
-		    if (!ISSET(bp->nb_flags, NB_NEEDCOMMIT))
-			np->n_needcommitcnt++;
-		    SET(bp->nb_flags, NB_NEEDCOMMIT);
-		    /* make sure nb_dirtyoff/nb_dirtyend reflect actual range written */
-		    bp->nb_dirtyoff = doff;
-		    bp->nb_dirtyend = dend;
-		} else {
-		    if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
-			np->n_needcommitcnt--;
-			CHECK_NEEDCOMMITCNT(np);
-		    }
-		    CLR(bp->nb_flags, NB_NEEDCOMMIT);
-		}
-		CLR(bp->nb_flags, NB_WRITEINPROG);
-		/*
-		 * For an interrupted write, the buffer is still valid and the write
-		 * hasn't been pushed to the server yet, so we can't set NB_ERROR and
-		 * report the interruption by setting NB_EINTR.  For the NB_ASYNC case,
-		 * NB_EINTR is not relevant.
-		 *
-		 * For the case of a V3 write rpc not being committed to stable
-		 * storage, the block is still dirty and requires either a commit rpc
-		 * or another write rpc with iomode == NFSV3WRITE_FILESYNC before the
-		 * block is reused. This is indicated by setting the NB_DELWRI and
-		 * NB_NEEDCOMMIT flags.
-		 */
-		if (error == EINTR || (!error && bp->nb_flags & NB_NEEDCOMMIT)) {
-		    CLR(bp->nb_flags, NB_INVAL);
-		    if (!ISSET(bp->nb_flags, NB_DELWRI)) {
-			SET(bp->nb_flags, NB_DELWRI);
-			OSAddAtomic(1, (SInt32*)&nfs_nbdwrite);
-			NFSBUFCNTCHK(0);
-		    }
-		    FSDBG(261, bp->nb_validoff, bp->nb_validend,
-			  bp->nb_bufsize, 0);
-		    /*
-		     * Since for the NB_ASYNC case, nfs_bwrite() has
-		     * reassigned the buffer to the clean list, we have to
-		     * reassign it back to the dirty one. Ugh.
-		     */
-		    if (ISSET(bp->nb_flags, NB_ASYNC)) {
-			/* move to dirty list */
-			lck_mtx_lock(nfs_buf_mutex);
-			if (bp->nb_vnbufs.le_next != NFSNOLIST)
-			    LIST_REMOVE(bp, nb_vnbufs);
-			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
-			lck_mtx_unlock(nfs_buf_mutex);
-		    } else {
-			SET(bp->nb_flags, NB_EINTR);
-		    }
-		} else {
-			/* either there's an error or we don't need to commit */
-			if (error) {
-			    SET(bp->nb_flags, NB_ERROR);
-			    bp->nb_error = np->n_error = error;
-			    np->n_flag |= NWRITEERR;
-			    /*
-			     * There was a write error and we need to
-			     * invalidate attrs and flush buffers in
-			     * order to sync up with the server.
-			     * (if this write was extending the file,
-			     * we may no longer know the correct size)
-			     *
-			     * But we can't call vinvalbuf while holding
-			     * this buffer busy.  Set a flag to do it after
-			     * releasing the buffer.
-			     *
-			     * Note we can only invalidate in this function
-			     * if this is an async write and so the iodone
-			     * below will release the buffer.  Also, we
-			     * shouldn't call vinvalbuf from nfsiod because
-			     * that may deadlock waiting for the completion
-			     * of writes that are queued up behind this one.
-			     */
-			    if (ISSET(bp->nb_flags, NB_ASYNC) &&
-			        !ISSET(bp->nb_flags, NB_IOD)) {
-				    invalidate = 1;
-			    } else {
-				    /* invalidate later */
-				    np->n_flag |= NNEEDINVALIDATE;
-			    }
-			    NATTRINVALIDATE(np);
+	OSAddAtomic(1, (SInt32*)&nfsstats.readdir_bios);
+	if (nfsvers < NFS_VER4) {
+		if (nmp->nm_flag & NFSMNT_RDIRPLUS) {
+			error = nfs3_readdirplus_rpc(np, &uio, ctx);
+			if (error == NFSERR_NOTSUPP) {
+				lck_mtx_lock(&nmp->nm_lock);
+				nmp->nm_flag &= ~NFSMNT_RDIRPLUS;
+				lck_mtx_unlock(&nmp->nm_lock);
 			}
-			/* clear the dirty range */
-			bp->nb_dirtyoff = bp->nb_dirtyend = 0;
-		}
-	    }
-
-	    if (!error && bp->nb_dirty) {
-		/* there are pages marked dirty that need to be written out */
-		int pg, count, npages, off;
-
-		OSAddAtomic(1, (SInt32*)&nfsstats.write_bios);
-
-		NFS_BUF_MAP(bp);
-
-		/*
-		 * we do these writes synchronously because we can't really
-		 * support the unstable/needommit method.  We could write
-		 * them unstable, clear the dirty bits, and then commit the
-		 * whole block later, but if we need to rewrite the data, we
-		 * won't have any idea which pages were written because that
-		 * info can't be stored in the nb_dirtyoff/nb_dirtyend.  We
-		 * also can't leave the dirty bits set because then we wouldn't
-		 * be able to tell if the pages were re-dirtied between the end
-		 * of the write and the commit.
-		 */
-		iomode = NFSV3WRITE_FILESYNC;
-		uiop->uio_rw = UIO_WRITE;
-
-		SET(bp->nb_flags, NB_WRITEINPROG);
-		npages = bp->nb_bufsize/PAGE_SIZE;
-		for (pg=0; pg < npages; pg++) {
-		    if (!NBPGDIRTY(bp,pg))
-		    	continue;
-		    count = 1;
-		    while (((pg+count) < npages) && NBPGDIRTY(bp,pg+count))
-			    count++;
-		    /* write count pages starting with page pg */
-		    off = pg * PAGE_SIZE;
-		    len = count * PAGE_SIZE;
-
-		    /* clip writes to EOF */
-		    if (NBOFF(bp) + off + len > (off_t)np->n_size)
-		    	len -= (NBOFF(bp) + off + len) - np->n_size;
-		    if (len > 0) {
-			io.iov_len = len;
-			uio_uio_resid_set(uiop, io.iov_len);
-			uiop->uio_offset = NBOFF(bp) + off;
-			io.iov_base = (uintptr_t) bp->nb_data + off;
-			error = nfs_writerpc(vp, uiop, cr, p, &iomode, &bp->nb_verf);
-			if (error)
-			    break;
-		    }
-		    /* clear dirty bits */
-		    while (count--) {
-			   bp->nb_dirty &= ~(1 << pg);
-			   /* leave pg on last page */
-			   if (count) pg++;
-		    }
-		}
-		if (!error) {
-		    if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
-			np->n_needcommitcnt--;
-			CHECK_NEEDCOMMITCNT(np);
-		    }
-		    CLR(bp->nb_flags, NB_NEEDCOMMIT);
 		}
-		CLR(bp->nb_flags, NB_WRITEINPROG);
-		FSDBG_BOT(256, bp->nb_validoff, bp->nb_validend, bp->nb_bufsize,
-			  np->n_size);
-	    }
-
-	    if (error) {
+		if (!(nmp->nm_flag & NFSMNT_RDIRPLUS))
+			error = nfs3_readdir_rpc(np, &uio, ctx);
+	} else {
+		error = nfs4_readdir_rpc(np, &uio, ctx);
+	}
+	if (error) {
 		SET(bp->nb_flags, NB_ERROR);
 		bp->nb_error = error;
-	    }
+	} else {
+		bp->nb_validoff = 0;
+		bp->nb_validend = uio.uio_offset - NBOFF(bp);
+		bp->nb_valid = (1 << (round_page_32(bp->nb_validend)/PAGE_SIZE)) - 1;
 	}
 
-	FSDBG_BOT(256, bp->nb_validoff, bp->nb_validend, bp->nb_bufsize, error);
-
 	nfs_buf_iodone(bp);
-
-	if (invalidate) {
-		/*
-		 * There was a write error and we need to
-		 * invalidate attrs and flush buffers in
-		 * order to sync up with the server.
-		 * (if this write was extending the file,
-		 * we may no longer know the correct size)
-		 *
-		 * But we couldn't call vinvalbuf while holding
-		 * the buffer busy.  So we call vinvalbuf() after
-		 * releasing the buffer.
-		 *
-		 * Note: we don't bother calling nfs_vinvalbuf() if
-		 * there's already a flush in progress.
-		 */
-		if (!(np->n_flag & NFLUSHINPROG))
-			nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cr, p, 1);
-	}
-
 	return (error);
 }
diff --git a/bsd/nfs/nfs_boot.c b/bsd/nfs/nfs_boot.c
index 2eaf21273..cfe0d5df6 100644
--- a/bsd/nfs/nfs_boot.c
+++ b/bsd/nfs/nfs_boot.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -121,12 +127,12 @@
 
 #if NETHER == 0
 
-int nfs_boot_init(struct nfs_diskless *nd, proc_t procp)
+int nfs_boot_init(__unused struct nfs_diskless *nd)
 {
 	panic("nfs_boot_init: no ether");
 }
 
-int nfs_boot_getfh(struct nfs_diskless *nd, proc_t procp, int v3, int sotype)
+int nfs_boot_getfh(__unused struct nfs_diskless *nd, __unused int v3, __unused int sotype)
 {
 	panic("nfs_boot_getfh: no ether");
 }
@@ -183,7 +189,7 @@ netboot_rootpath(struct in_addr * server_ip,
  * Called with an empty nfs_diskless struct to be filled in.
  */
 int
-nfs_boot_init(struct nfs_diskless *nd, __unused proc_t procp)
+nfs_boot_init(struct nfs_diskless *nd)
 {
 	struct sockaddr_in 	bp_sin;
 	boolean_t		do_bpwhoami = TRUE;
@@ -193,7 +199,7 @@ nfs_boot_init(struct nfs_diskless *nd, __unused proc_t procp)
 	struct sockaddr_in *	sin_p;
 
 	/* make sure mbuf constants are set up */
-	if (!nfs_mbuf_mlen)
+	if (!nfs_mbuf_mhlen)
 		nfs_mbuf_init();
 
 	/* by this point, networking must already have been configured */
@@ -280,7 +286,7 @@ nfs_boot_init(struct nfs_diskless *nd, __unused proc_t procp)
 			snprintf(check_path, MAXPATHLEN, "%s/private", nd->nd_root.ndm_path);
 			if ((nd->nd_root.ndm_saddr.sin_addr.s_addr 
 			     == nd->nd_private.ndm_saddr.sin_addr.s_addr)
-			    && (strcmp(check_path, nd->nd_private.ndm_path) == 0)) {
+			    && (strncmp(check_path, nd->nd_private.ndm_path, MAXPATHLEN) == 0)) {
 				/* private path is prefix of root path, don't mount */
 				nd->nd_private.ndm_saddr.sin_addr.s_addr = 0;
 			}
@@ -304,7 +310,7 @@ failed:
  * with file handles to be filled in.
  */
 int
-nfs_boot_getfh(struct nfs_diskless *nd, __unused proc_t procp, int v3, int sotype)
+nfs_boot_getfh(struct nfs_diskless *nd, int v3, int sotype)
 {
 	int error = 0;
 
@@ -368,7 +374,7 @@ get_file_handle(ndmntp)
  * initialize the pkthdr length field.
  */
 static int
-mbuf_get_with_len(int msg_len, mbuf_t *m)
+mbuf_get_with_len(size_t msg_len, mbuf_t *m)
 {
 	int error;
 	error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, m);
@@ -444,8 +450,8 @@ bp_whoami(bpsin, my_ip, gw_ip)
 	struct bp_inaddr *bia;
 	mbuf_t m;
 	struct sockaddr_in sin;
-	int error, msg_len;
-	int cn_len, dn_len;
+	int error;
+	size_t msg_len, cn_len, dn_len;
 	u_char *p;
 	long *lp;
 
@@ -489,20 +495,20 @@ bp_whoami(bpsin, my_ip, gw_ip)
 	lp = mbuf_data(m);
 
 	/* bootparam server port (also grab from address). */
-	if (msg_len < (int)sizeof(*lp))
+	if (msg_len < sizeof(*lp))
 		goto bad;
 	msg_len -= sizeof(*lp);
 	bpsin->sin_port = htons((short)ntohl(*lp++));
 	bpsin->sin_addr.s_addr = sin.sin_addr.s_addr;
 
 	/* length of encapsulated results */
-	if (msg_len < (ntohl(*lp) + (int)sizeof(*lp)))
+	if (msg_len < (ntohl(*lp) + sizeof(*lp)))
 		goto bad;
 	msg_len = ntohl(*lp++);
-	p = (char*)lp;
+	p = (u_char*)lp;
 
 	/* client name */
-	if (msg_len < (int)sizeof(*str))
+	if (msg_len < sizeof(*str))
 		goto bad;
 	str = (struct rpc_string *)p;
 	cn_len = ntohl(str->len);
@@ -517,7 +523,7 @@ bp_whoami(bpsin, my_ip, gw_ip)
 	msg_len -= RPC_STR_SIZE(cn_len);
 
 	/* domain name */
-	if (msg_len < (int)sizeof(*str))
+	if (msg_len < sizeof(*str))
 		goto bad;
 	str = (struct rpc_string *)p;
 	dn_len = ntohl(str->len);
@@ -532,7 +538,7 @@ bp_whoami(bpsin, my_ip, gw_ip)
 	msg_len -= RPC_STR_SIZE(dn_len);
 
 	/* gateway address */
-	if (msg_len < (int)sizeof(*bia))
+	if (msg_len < sizeof(*bia))
 		goto bad;
 	bia = (struct bp_inaddr *)p;
 	if (bia->atype != htonl(1))
diff --git a/bsd/nfs/nfs_gss.c b/bsd/nfs/nfs_gss.c
new file mode 100644
index 000000000..62cc01a47
--- /dev/null
+++ b/bsd/nfs/nfs_gss.c
@@ -0,0 +1,3255 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*************
+ * These functions implement RPCSEC_GSS security for the NFS client and server.
+ * The code is specific to the use of Kerberos v5 and the use of DES MAC MD5
+ * protection as described in Internet RFC 2203 and 2623.
+ *
+ * In contrast to the original AUTH_SYS authentication, RPCSEC_GSS is stateful.
+ * It requires the client and server negotiate a secure connection as part of a
+ * security context. The context state is maintained in client and server structures.
+ * On the client side, each user of an NFS mount is assigned their own context,
+ * identified by UID, on their first use of the mount, and it persists until the
+ * unmount or until the context is renewed.  Each user context has a corresponding
+ * server context which the server maintains until the client destroys it, or
+ * until the context expires.
+ *
+ * The client and server contexts are set up dynamically.  When a user attempts
+ * to send an NFS request, if there is no context for the user, then one is
+ * set up via an exchange of NFS null procedure calls as described in RFC 2203.
+ * During this exchange, the client and server pass a security token that is
+ * forwarded via Mach upcall to the gssd, which invokes the GSS-API to authenticate
+ * the user to the server (and vice-versa). The client and server also receive
+ * a unique session key that can be used to digitally sign the credentials and
+ * verifier or optionally to provide data integrity and/or privacy.
+ *
+ * Once the context is complete, the client and server enter a normal data
+ * exchange phase - beginning with the NFS request that prompted the context
+ * creation. During this phase, the client's RPC header contains an RPCSEC_GSS
+ * credential and verifier, and the server returns a verifier as well.
+ * For simple authentication, the verifier contains a signed checksum of the
+ * RPC header, including the credential.  The server's verifier has a signed
+ * checksum of the current sequence number.
+ *
+ * Each client call contains a sequence number that nominally increases by one
+ * on each request.  The sequence number is intended to prevent replay attacks.
+ * Since the protocol can be used over UDP, there is some allowance for
+ * out-of-sequence requests, so the server checks whether the sequence numbers
+ * are within a sequence "window". If a sequence number is outside the lower
+ * bound of the window, the server silently drops the request. This has some
+ * implications for retransmission. If a request needs to be retransmitted, the
+ * client must bump the sequence number even if the request XID is unchanged.
+ *
+ * When the NFS mount is unmounted, the client sends a "destroy" credential
+ * to delete the server's context for each user of the mount. Since it's
+ * possible for the client to crash or disconnect without sending the destroy
+ * message, the server has a thread that reaps contexts that have been idle
+ * too long.
+ */
+
+#include <stdint.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/kauth.h>
+#include <sys/kernel.h>
+#include <sys/mount_internal.h>
+#include <sys/vnode.h>
+#include <sys/ubc.h>
+#include <sys/malloc.h>
+#include <sys/kpi_mbuf.h>
+
+#include <kern/host.h>
+#include <libkern/libkern.h>
+
+#include <mach/task.h>
+#include <mach/task_special_ports.h>
+#include <mach/host_priv.h>
+#include <mach/thread_act.h>
+#include <mach/mig_errors.h>
+#include <mach/vm_map.h>
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+#include <gssd/gssd_mach.h>
+
+#include <nfs/rpcv2.h>
+#include <nfs/nfsproto.h>
+#include <nfs/nfs.h>
+#include <nfs/nfsnode.h>
+#include <nfs/nfs_gss.h>
+#include <nfs/nfsmount.h>
+#include <nfs/xdr_subs.h>
+#include <nfs/nfsm_subs.h>
+#include <nfs/nfs_gss.h>
+
+#define NFS_GSS_MACH_MAX_RETRIES 3
+
+#if NFSSERVER
+u_long nfs_gss_svc_ctx_hash;
+struct nfs_gss_svc_ctx_hashhead *nfs_gss_svc_ctx_hashtbl;
+lck_mtx_t *nfs_gss_svc_ctx_mutex;
+lck_grp_t *nfs_gss_svc_grp;
+#endif /* NFSSERVER */
+
+#if NFSCLIENT
+lck_grp_t *nfs_gss_clnt_grp;
+#endif /* NFSCLIENT */
+
+/*
+ * These octet strings are used to encode/decode ASN.1 tokens
+ * in the RPCSEC_GSS verifiers.
+ */
+static u_char krb5_tokhead[] = { 0x60, 0x23 };
+static u_char krb5_mech[] = { 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x12, 0x01, 0x02, 0x02 };
+static u_char krb5_mic[]  = { 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff };
+static u_char krb5_wrap[] = { 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff };
+static u_char iv0[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; // DES MAC Initialization Vector
+
+/*
+ * The size of the Kerberos v5 ASN.1 token
+ * in the verifier.
+ *
+ * Note that the second octet of the krb5_tokhead (0x23) is a
+ * DER-encoded size field that has variable length.  If the size
+ * is 128 bytes or greater, then it uses two bytes, three bytes
+ * if 65536 or greater, and so on.  Since the MIC tokens are
+ * separate from the data, the size is always the same: 35 bytes (0x23).
+ * However, the wrap token is different. Its size field includes the
+ * size of the token + the encrypted data that follows. So the size
+ * field may be two, three or four bytes.
+ */
+#define KRB5_SZ_TOKHEAD sizeof(krb5_tokhead)
+#define KRB5_SZ_MECH	sizeof(krb5_mech)
+#define KRB5_SZ_ALG	sizeof(krb5_mic) // 8 - same as krb5_wrap
+#define KRB5_SZ_SEQ	8
+#define KRB5_SZ_CKSUM	8
+#define KRB5_SZ_EXTRA	3  // a wrap token may be longer by up to this many octets
+#define KRB5_SZ_TOKEN	(KRB5_SZ_TOKHEAD + KRB5_SZ_MECH + KRB5_SZ_ALG + KRB5_SZ_SEQ + KRB5_SZ_CKSUM)
+#define KRB5_SZ_TOKMAX	(KRB5_SZ_TOKEN + KRB5_SZ_EXTRA)
+
+#if NFSCLIENT
+static int	nfs_gss_clnt_ctx_find(struct nfsreq *);
+static int	nfs_gss_clnt_ctx_failover(struct nfsreq *);
+static int	nfs_gss_clnt_ctx_init(struct nfsreq *, struct nfs_gss_clnt_ctx *);
+static int	nfs_gss_clnt_ctx_callserver(struct nfsreq *, struct nfs_gss_clnt_ctx *);
+static char	*nfs_gss_clnt_svcname(struct nfsmount *);
+static int	nfs_gss_clnt_gssd_upcall(struct nfsreq *, struct nfs_gss_clnt_ctx *);
+static void	nfs_gss_clnt_ctx_remove(struct nfsmount *, struct nfs_gss_clnt_ctx *);
+static int	nfs_gss_clnt_ctx_delay(struct nfsreq *, int *);
+#endif /* NFSCLIENT */
+
+#if NFSSERVER
+static struct nfs_gss_svc_ctx *nfs_gss_svc_ctx_find(uint32_t);
+static void	nfs_gss_svc_ctx_insert(struct nfs_gss_svc_ctx *);
+static void	nfs_gss_svc_ctx_timer(void *, void *);
+static int	nfs_gss_svc_gssd_upcall(struct nfs_gss_svc_ctx *);
+static int	nfs_gss_svc_seqnum_valid(struct nfs_gss_svc_ctx *, uint32_t);
+#endif /* NFSSERVER */
+
+static void	task_release_special_port(mach_port_t);
+static mach_port_t task_copy_special_port(mach_port_t);
+static void	nfs_gss_mach_alloc_buffer(u_char *, uint32_t, vm_map_copy_t *);
+static int	nfs_gss_mach_vmcopyout(vm_map_copy_t, uint32_t, u_char *);
+static int	nfs_gss_token_get(des_key_schedule, u_char *, u_char *, int, uint32_t *, u_char *);
+static int	nfs_gss_token_put(des_key_schedule, u_char *, u_char *, int, int, u_char *);
+static int	nfs_gss_der_length_size(int);
+static void	nfs_gss_der_length_put(u_char **, int);
+static int	nfs_gss_der_length_get(u_char **);
+static int	nfs_gss_mchain_length(mbuf_t);
+static int	nfs_gss_append_chain(struct nfsm_chain *, mbuf_t);
+static void	nfs_gss_nfsm_chain(struct nfsm_chain *, mbuf_t);
+static void	nfs_gss_cksum_mchain(des_key_schedule, mbuf_t, u_char *, int, int, u_char *);
+static void	nfs_gss_cksum_chain(des_key_schedule, struct nfsm_chain *, u_char *, int, int, u_char *);
+static void	nfs_gss_cksum_rep(des_key_schedule, uint32_t, u_char *);
+static void	nfs_gss_encrypt_mchain(u_char *, mbuf_t, int, int, int);
+static void	nfs_gss_encrypt_chain(u_char *, struct nfsm_chain *, int, int, int);
+static DES_LONG	des_cbc_cksum(des_cblock *, des_cblock *, long, des_key_schedule, des_cblock *);
+static void	des_cbc_encrypt(des_cblock *, des_cblock *, long, des_key_schedule,
+			des_cblock *, des_cblock *, int);
+
+#if NFSSERVER
+thread_call_t nfs_gss_svc_ctx_timer_call;
+int nfs_gss_timer_on = 0;
+uint32_t nfs_gss_ctx_count = 0;
+const uint32_t nfs_gss_ctx_max = GSS_SVC_MAXCONTEXTS;
+#endif /* NFSSERVER */
+
+/*
+ * Initialization when NFS starts
+ */
+void
+nfs_gss_init(void)
+{
+#if NFSCLIENT
+	nfs_gss_clnt_grp = lck_grp_alloc_init("rpcsec_gss_clnt", LCK_GRP_ATTR_NULL);
+#endif /* NFSCLIENT */
+
+#if NFSSERVER
+	nfs_gss_svc_grp  = lck_grp_alloc_init("rpcsec_gss_svc",  LCK_GRP_ATTR_NULL);
+
+	nfs_gss_svc_ctx_hashtbl = hashinit(SVC_CTX_HASHSZ, M_TEMP, &nfs_gss_svc_ctx_hash);
+	nfs_gss_svc_ctx_mutex = lck_mtx_alloc_init(nfs_gss_svc_grp, LCK_ATTR_NULL);
+
+	nfs_gss_svc_ctx_timer_call = thread_call_allocate(nfs_gss_svc_ctx_timer, NULL);
+#endif /* NFSSERVER */
+}
+
+#if NFSCLIENT
+
+/*
+ * Find the context for a particular user.
+ *
+ * If the context doesn't already exist
+ * then create a new context for this user.
+ *
+ * Note that the code allows superuser (uid == 0)
+ * to adopt the context of another user.
+ */
+static int
+nfs_gss_clnt_ctx_find(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfs_gss_clnt_ctx *cp;
+	uid_t uid = kauth_cred_getuid(req->r_cred);
+	int error = 0;
+	int retrycnt = 0;
+
+retry:	
+	lck_mtx_lock(&nmp->nm_lock);
+	TAILQ_FOREACH(cp, &nmp->nm_gsscl, gss_clnt_entries) {
+		if (cp->gss_clnt_uid == uid) {
+			if (cp->gss_clnt_flags & GSS_CTX_INVAL)
+				continue;
+			lck_mtx_unlock(&nmp->nm_lock);
+			nfs_gss_clnt_ctx_ref(req, cp);
+			return (0);
+		}
+	}
+
+	if (uid == 0) {
+		/*
+		 * If superuser is trying to get access, then co-opt
+		 * the first valid context in the list.
+		 * XXX Ultimately, we need to allow superuser to
+		 * go ahead and attempt to set up its own context
+		 * in case one is set up for it.
+		 */
+		TAILQ_FOREACH(cp, &nmp->nm_gsscl, gss_clnt_entries) {
+			if (!(cp->gss_clnt_flags & GSS_CTX_INVAL)) {
+				lck_mtx_unlock(&nmp->nm_lock);
+				nfs_gss_clnt_ctx_ref(req, cp);
+				return (0);
+			}
+		}
+	}
+
+	/*
+	 * Not found - create a new context
+	 */
+
+	/*
+	 * If the thread is async, then it cannot get
+	 * kerberos creds and set up a proper context.
+	 * If no sec= mount option is given, attempt
+	 * to failover to sec=sys.
+	 */
+	if (req->r_thread == NULL) {
+		if ((nmp->nm_flag & NFSMNT_SECGIVEN) == 0) {
+			error = nfs_gss_clnt_ctx_failover(req);
+		} else {
+			printf("nfs_gss_clnt_ctx_find: no context for async\n");
+			error = EAUTH;
+		}
+
+		lck_mtx_unlock(&nmp->nm_lock);
+		return (error);
+	}
+		
+
+	MALLOC(cp, struct nfs_gss_clnt_ctx *, sizeof(*cp), M_TEMP, M_WAITOK|M_ZERO);
+	if (cp == NULL) {
+		lck_mtx_unlock(&nmp->nm_lock);
+		return (ENOMEM);
+	}
+
+	cp->gss_clnt_uid = uid;
+	cp->gss_clnt_mtx = lck_mtx_alloc_init(nfs_gss_clnt_grp, LCK_ATTR_NULL);
+	cp->gss_clnt_thread = current_thread();
+	nfs_gss_clnt_ctx_ref(req, cp);
+	TAILQ_INSERT_TAIL(&nmp->nm_gsscl, cp, gss_clnt_entries);
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	error = nfs_gss_clnt_ctx_init(req, cp);
+	if (error)
+		nfs_gss_clnt_ctx_unref(req);
+
+	if (error == ENEEDAUTH) {
+		error = nfs_gss_clnt_ctx_delay(req, &retrycnt);
+		if (!error)
+			goto retry;
+	}
+
+	/*
+	 * If we failed to set up a Kerberos context for this
+	 * user and no sec= mount option was given then set
+	 * up a dummy context that allows this user to attempt
+	 * sec=sys calls.
+	 */
+	if (error && (nmp->nm_flag & NFSMNT_SECGIVEN) == 0) {
+		lck_mtx_lock(&nmp->nm_lock);
+		error = nfs_gss_clnt_ctx_failover(req);
+		lck_mtx_unlock(&nmp->nm_lock);
+	}
+
+	return (error);
+}
+
+/*
+ * Set up a dummy context to allow the use of sec=sys
+ * for this user, if the server allows sec=sys.
+ * The context is valid for GSS_CLNT_SYS_VALID seconds,
+ * so that the user will periodically attempt to fail back
+ * and get a real credential.
+ *
+ * Assumes context list (nm_lock) is locked
+ */
+static int
+nfs_gss_clnt_ctx_failover(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfs_gss_clnt_ctx *cp;
+	uid_t uid = kauth_cred_getuid(req->r_cred);
+	struct timeval now;
+
+	MALLOC(cp, struct nfs_gss_clnt_ctx *, sizeof(*cp), M_TEMP, M_WAITOK|M_ZERO);
+	if (cp == NULL)
+		return (ENOMEM);
+
+	cp->gss_clnt_service = RPCSEC_GSS_SVC_SYS;
+	cp->gss_clnt_uid = uid;
+	cp->gss_clnt_mtx = lck_mtx_alloc_init(nfs_gss_clnt_grp, LCK_ATTR_NULL);
+	microuptime(&now);
+	cp->gss_clnt_ctime = now.tv_sec;	// time stamp
+	nfs_gss_clnt_ctx_ref(req, cp);
+	TAILQ_INSERT_TAIL(&nmp->nm_gsscl, cp, gss_clnt_entries);
+
+	return (0);
+}
+
+/*
+ * Inserts an RPCSEC_GSS credential into an RPC header.
+ * After the credential is inserted, the code continues
+ * to build the verifier which contains a signed checksum
+ * of the RPC header.
+ */
+int
+nfs_gss_clnt_cred_put(struct nfsreq *req, struct nfsm_chain *nmc, mbuf_t args)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfs_gss_clnt_ctx *cp;
+	uint32_t seqnum = 0;
+	int error = 0;
+	int slpflag = 0;
+	int start, len, offset = 0;
+	int pad, toklen;
+	struct nfsm_chain nmc_tmp;
+	struct gss_seq *gsp;
+	u_char tokbuf[KRB5_SZ_TOKMAX];
+	u_char cksum[8];
+	struct timeval now;
+
+retry:
+	if (req->r_gss_ctx == NULL) {
+		/*
+		 * Find the context for this user.
+		 * If no context is found, one will
+		 * be created.
+		 */
+		error = nfs_gss_clnt_ctx_find(req);
+		if (error)
+			return (error);
+	}
+	cp = req->r_gss_ctx;
+
+	/*
+	 * If it's a dummy context for a user that's using
+	 * a fallback to sec=sys, then just return an error
+	 * so rpchead can encode an RPCAUTH_UNIX cred.
+	 */
+	if (cp->gss_clnt_service == RPCSEC_GSS_SVC_SYS) {
+		/*
+		 * The dummy context is valid for just
+		 * GSS_CLNT_SYS_VALID seconds.  If the context
+		 * is older than this, mark it invalid and try
+		 * again to get a real one.
+		 */
+		lck_mtx_lock(cp->gss_clnt_mtx);
+		microuptime(&now);
+		if (now.tv_sec > cp->gss_clnt_ctime + GSS_CLNT_SYS_VALID) {
+			cp->gss_clnt_flags |= GSS_CTX_INVAL;
+			lck_mtx_unlock(cp->gss_clnt_mtx);
+			nfs_gss_clnt_ctx_unref(req);
+			goto retry;
+		}
+		lck_mtx_unlock(cp->gss_clnt_mtx);
+		return (ENEEDAUTH);
+	}
+
+	/*
+	 * If the context thread isn't null, then the context isn't
+	 * yet complete and is for the exclusive use of the thread
+	 * doing the context setup. Wait until the context thread
+	 * is null.
+	 */
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	if (cp->gss_clnt_thread && cp->gss_clnt_thread != current_thread()) {
+		cp->gss_clnt_flags |= GSS_NEEDCTX;
+		slpflag = (PZERO-1) | PDROP | (((nmp->nm_flag & NFSMNT_INT) && req->r_thread) ? PCATCH : 0);
+		msleep(cp, cp->gss_clnt_mtx, slpflag, "ctxwait", NULL);
+		if ((error = nfs_sigintr(nmp, req, req->r_thread, 0)))
+			return (error);
+		nfs_gss_clnt_ctx_unref(req);
+		goto retry;
+	}
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+
+	if (cp->gss_clnt_flags & GSS_CTX_COMPLETE) {
+		/*
+		 * Get a sequence number for this request.
+		 * Check whether the oldest request in the window is complete.
+		 * If it's still pending, then wait until it's done before
+		 * we allocate a new sequence number and allow this request
+		 * to proceed.
+		 */
+		lck_mtx_lock(cp->gss_clnt_mtx);
+		while (win_getbit(cp->gss_clnt_seqbits, 
+			((cp->gss_clnt_seqnum - cp->gss_clnt_seqwin) + 1) % cp->gss_clnt_seqwin)) {
+			cp->gss_clnt_flags |= GSS_NEEDSEQ;
+			slpflag = (PZERO-1) | (((nmp->nm_flag & NFSMNT_INT) && req->r_thread) ? PCATCH : 0);
+			msleep(cp, cp->gss_clnt_mtx, slpflag, "seqwin", NULL);
+			if ((error = nfs_sigintr(nmp, req, req->r_thread, 0))) {
+				lck_mtx_unlock(cp->gss_clnt_mtx);
+				return (error);
+			}
+			if (cp->gss_clnt_flags & GSS_CTX_INVAL) {
+				/* Renewed while while we were waiting */
+				lck_mtx_unlock(cp->gss_clnt_mtx);
+				nfs_gss_clnt_ctx_unref(req);
+				goto retry;
+			}
+		}
+		seqnum = ++cp->gss_clnt_seqnum;
+		win_setbit(cp->gss_clnt_seqbits, seqnum % cp->gss_clnt_seqwin);
+		lck_mtx_unlock(cp->gss_clnt_mtx);
+
+		MALLOC(gsp, struct gss_seq *, sizeof(*gsp), M_TEMP, M_WAITOK|M_ZERO);
+		if (gsp == NULL)
+			return (ENOMEM);
+		gsp->gss_seqnum = seqnum;
+		SLIST_INSERT_HEAD(&req->r_gss_seqlist, gsp, gss_seqnext);
+	}
+
+	/* Insert the credential */
+	nfsm_chain_add_32(error, nmc, RPCSEC_GSS);
+	nfsm_chain_add_32(error, nmc, 5 * NFSX_UNSIGNED + cp->gss_clnt_handle_len);
+	nfsm_chain_add_32(error, nmc, RPCSEC_GSS_VERS_1);
+	nfsm_chain_add_32(error, nmc, cp->gss_clnt_proc);
+	nfsm_chain_add_32(error, nmc, seqnum);
+	nfsm_chain_add_32(error, nmc, cp->gss_clnt_service);
+	nfsm_chain_add_32(error, nmc, cp->gss_clnt_handle_len);
+	nfsm_chain_add_opaque(error, nmc, cp->gss_clnt_handle, cp->gss_clnt_handle_len);
+
+	/*
+	 * Now add the verifier
+	 */
+	if (cp->gss_clnt_proc == RPCSEC_GSS_INIT ||
+		cp->gss_clnt_proc == RPCSEC_GSS_CONTINUE_INIT) {
+		/*
+		 * If the context is still being created
+		 * then use a null verifier.
+		 */
+		nfsm_chain_add_32(error, nmc, RPCAUTH_NULL);	// flavor
+		nfsm_chain_add_32(error, nmc, 0);		// length
+		nfsm_chain_build_done(error, nmc);
+		if (!error)
+			nfs_gss_append_chain(nmc, args);
+		return (error);
+	}
+
+	offset = nmp->nm_sotype == SOCK_STREAM ? NFSX_UNSIGNED : 0; // record mark
+	nfsm_chain_build_done(error, nmc);
+	nfs_gss_cksum_chain(cp->gss_clnt_sched, nmc, krb5_mic, offset, 0, cksum);
+
+	toklen = nfs_gss_token_put(cp->gss_clnt_sched, krb5_mic, tokbuf, 1, 0, cksum);
+	nfsm_chain_add_32(error, nmc, RPCSEC_GSS);	// flavor
+	nfsm_chain_add_32(error, nmc, toklen);		// length
+	nfsm_chain_add_opaque(error, nmc, tokbuf, toklen);
+	nfsm_chain_build_done(error, nmc);
+	if (error)
+		return (error);
+
+	/*
+	 * Now we may have to compute integrity or encrypt the call args
+	 * per RFC 2203 Section 5.3.2
+	 */
+	switch (cp->gss_clnt_service) {
+	case RPCSEC_GSS_SVC_NONE:
+		nfs_gss_append_chain(nmc, args);
+		break;
+	case RPCSEC_GSS_SVC_INTEGRITY:
+		len = nfs_gss_mchain_length(args);	// Find args length
+		req->r_gss_arglen = len;		// Stash the args len
+		len += NFSX_UNSIGNED;			// Add seqnum length
+		nfsm_chain_add_32(error, nmc, len);	// and insert it
+		start = nfsm_chain_offset(nmc);
+		nfsm_chain_add_32(error, nmc, seqnum);	// Insert seqnum
+		req->r_gss_argoff = nfsm_chain_offset(nmc); // Offset to args
+		nfsm_chain_build_done(error, nmc);
+		if (error)
+			return (error);
+		nfs_gss_append_chain(nmc, args);	// Append the args mbufs
+
+		/* Now compute a checksum over the seqnum + args */
+		nfs_gss_cksum_chain(cp->gss_clnt_sched, nmc, krb5_mic, start, len, cksum);
+
+		/* Insert it into a token and append to the request */
+		toklen = nfs_gss_token_put(cp->gss_clnt_sched, krb5_mic, tokbuf, 1, 0, cksum);
+		nfsm_chain_finish_mbuf(error, nmc);	// force checksum into new mbuf
+		nfsm_chain_add_32(error, nmc, toklen);
+		nfsm_chain_add_opaque(error, nmc, tokbuf, toklen);
+		nfsm_chain_build_done(error, nmc);
+		break;
+	case RPCSEC_GSS_SVC_PRIVACY:
+		/* Prepend a new mbuf with the confounder & sequence number */
+		nfsm_chain_build_alloc_init(error, &nmc_tmp, 3 * NFSX_UNSIGNED);
+		nfsm_chain_add_32(error, &nmc_tmp, random());	// confounder bytes 1-4
+		nfsm_chain_add_32(error, &nmc_tmp, random());	// confounder bytes 4-8
+		nfsm_chain_add_32(error, &nmc_tmp, seqnum);
+		nfsm_chain_build_done(error, &nmc_tmp);
+		if (error)
+			return (error);
+		nfs_gss_append_chain(&nmc_tmp, args);		// Append the args mbufs
+
+		len = nfs_gss_mchain_length(args);		// Find args length
+		len += 3 * NFSX_UNSIGNED;			// add confounder & seqnum
+		req->r_gss_arglen = len;			// Stash length
+
+		/*
+		 * Append a pad trailer - per RFC 1964 section 1.2.2.3
+		 * Since XDR data is always 32-bit aligned, it
+		 * needs to be padded either by 4 bytes or 8 bytes.
+		 */
+		nfsm_chain_finish_mbuf(error, &nmc_tmp);	// force padding into new mbuf
+		if (len % 8 > 0) {
+			nfsm_chain_add_32(error, &nmc_tmp, 0x04040404);
+			len += NFSX_UNSIGNED;
+		} else {
+			nfsm_chain_add_32(error, &nmc_tmp, 0x08080808);
+			nfsm_chain_add_32(error, &nmc_tmp, 0x08080808);
+			len +=  2 * NFSX_UNSIGNED;
+		}
+		nfsm_chain_build_done(error, &nmc_tmp);
+
+		/* Now compute a checksum over the confounder + seqnum + args */
+		nfs_gss_cksum_chain(cp->gss_clnt_sched, &nmc_tmp, krb5_wrap, 0, len, cksum);
+
+		/* Insert it into a token */
+		toklen = nfs_gss_token_put(cp->gss_clnt_sched, krb5_wrap, tokbuf, 1, len, cksum);
+		nfsm_chain_add_32(error, nmc, toklen + len);	// token + args length
+		nfsm_chain_add_opaque_nopad(error, nmc, tokbuf, toklen);
+		req->r_gss_argoff = nfsm_chain_offset(nmc);	// Stash offset
+		nfsm_chain_build_done(error, nmc);
+		if (error)
+			return (error);
+		nfs_gss_append_chain(nmc, nmc_tmp.nmc_mhead);	// Append the args mbufs
+
+		/* Finally, encrypt the args */
+		nfs_gss_encrypt_chain(cp->gss_clnt_skey, &nmc_tmp, 0, len, DES_ENCRYPT);
+
+		/* Add null XDR pad if the ASN.1 token misaligned the data */
+		pad = nfsm_pad(toklen + len);
+		if (pad > 0) {
+			nfsm_chain_add_opaque_nopad(error, nmc, iv0, pad);
+			nfsm_chain_build_done(error, nmc);
+		}
+		break;
+	}
+
+	return (error);
+}
+
+/*
+ * When receiving a reply, the client checks the verifier
+ * returned by the server. Check that the verifier is the
+ * correct type, then extract the sequence number checksum
+ * from the token in the credential and compare it with a
+ * computed checksum of the sequence number in the request
+ * that was sent.
+ */
+int
+nfs_gss_clnt_verf_get(
+	struct nfsreq *req,
+	struct nfsm_chain *nmc,
+	uint32_t verftype,
+	uint32_t verflen,
+	uint32_t *accepted_statusp)
+{
+	u_char tokbuf[KRB5_SZ_TOKMAX];
+	u_char cksum1[8], cksum2[8];
+	uint32_t seqnum = 0;
+	struct nfs_gss_clnt_ctx *cp = req->r_gss_ctx;
+	struct nfsm_chain nmc_tmp;
+	struct gss_seq *gsp;
+	uint32_t reslen, start, cksumlen, toklen;
+	int error = 0;
+
+	reslen = cksumlen = 0;
+	*accepted_statusp = 0;
+
+	if (cp == NULL)
+		return (EAUTH);
+	/*
+	 * If it's not an RPCSEC_GSS verifier, then it has to
+	 * be a null verifier that resulted from either
+	 * a CONTINUE_NEEDED reply during context setup or
+	 * from the reply to an AUTH_UNIX call from a dummy
+	 * context that resulted from a fallback to sec=sys.
+	 */
+	if (verftype != RPCSEC_GSS) {
+		if (verftype != RPCAUTH_NULL)
+			return (EAUTH);
+		if (cp->gss_clnt_flags & GSS_CTX_COMPLETE &&
+			cp->gss_clnt_service != RPCSEC_GSS_SVC_SYS)
+			return (EAUTH);
+		if (verflen > 0)
+			nfsm_chain_adv(error, nmc, nfsm_rndup(verflen));
+		nfsm_chain_get_32(error, nmc, *accepted_statusp);
+		return (error);
+	}
+
+	if (verflen != KRB5_SZ_TOKEN)
+		return (EAUTH);
+
+	/*
+	 * If we received an RPCSEC_GSS verifier but the
+	 * context isn't yet complete, then it must be
+	 * the context complete message from the server.
+	 * The verifier will contain an encrypted checksum
+	 * of the window but we don't have the session key
+	 * yet so we can't decrypt it. Stash the verifier
+	 * and check it later in nfs_gss_clnt_ctx_init() when
+	 * the context is complete.
+	 */
+	if (!(cp->gss_clnt_flags & GSS_CTX_COMPLETE)) {
+		MALLOC(cp->gss_clnt_verf, u_char *, verflen, M_TEMP, M_WAITOK|M_ZERO);
+		if (cp->gss_clnt_verf == NULL)
+			return (ENOMEM);
+		nfsm_chain_get_opaque(error, nmc, verflen, cp->gss_clnt_verf);
+		nfsm_chain_get_32(error, nmc, *accepted_statusp);
+		return (error);
+	}
+
+	/*
+	 * Get the 8 octet sequence number
+	 * checksum out of the verifier token.
+	 */
+	nfsm_chain_get_opaque(error, nmc, verflen, tokbuf);
+	if (error)
+		goto nfsmout;
+	error = nfs_gss_token_get(cp->gss_clnt_sched, krb5_mic, tokbuf, 0, NULL, cksum1);
+	if (error)
+		goto nfsmout;
+
+	/*
+	 * Search the request sequence numbers for this reply, starting
+	 * with the most recent, looking for a checksum that matches
+	 * the one in the verifier returned by the server.
+	 */
+	SLIST_FOREACH(gsp, &req->r_gss_seqlist, gss_seqnext) {
+		nfs_gss_cksum_rep(cp->gss_clnt_sched, gsp->gss_seqnum, cksum2);
+		if (bcmp(cksum1, cksum2, 8) == 0)
+			break;
+	}
+	if (gsp == NULL)
+		return (EAUTH);
+
+	/*
+	 * Get the RPC accepted status
+	 */
+	nfsm_chain_get_32(error, nmc, *accepted_statusp);
+	if (*accepted_statusp != RPC_SUCCESS)
+		return (0);
+
+	/*
+	 * Now we may have to check integrity or decrypt the results
+	 * per RFC 2203 Section 5.3.2
+	 */
+	switch (cp->gss_clnt_service) {
+	case RPCSEC_GSS_SVC_NONE:
+		/* nothing to do */
+		break;
+	case RPCSEC_GSS_SVC_INTEGRITY:
+		/*
+		 * Here's what we expect in the integrity results:
+		 *
+		 * - length of seq num + results (4 bytes)
+		 * - sequence number (4 bytes)
+		 * - results (variable bytes)
+		 * - length of checksum token (37)
+		 * - checksum of seqnum + results (37 bytes)
+		 */
+		nfsm_chain_get_32(error, nmc, reslen);		// length of results
+		if (reslen > NFS_MAXPACKET) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+
+		/* Compute a checksum over the sequence number + results */
+		start = nfsm_chain_offset(nmc);
+		nfs_gss_cksum_chain(cp->gss_clnt_sched, nmc, krb5_mic, start, reslen, cksum1);
+
+		/*
+		 * Get the sequence number prepended to the results
+		 * and compare it against the list in the request.
+		 */
+		nfsm_chain_get_32(error, nmc, seqnum);
+		SLIST_FOREACH(gsp, &req->r_gss_seqlist, gss_seqnext) {
+			if (seqnum == gsp->gss_seqnum)
+				break;
+		}
+		if (gsp == NULL) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+
+		/*
+		 * Advance to the end of the results and
+		 * fetch the checksum computed by the server.
+		 */
+		nmc_tmp = *nmc;	
+		reslen -= NFSX_UNSIGNED;			// already skipped seqnum
+		nfsm_chain_adv(error, &nmc_tmp, reslen);	// skip over the results
+		nfsm_chain_get_32(error, &nmc_tmp, cksumlen);	// length of checksum
+		if (cksumlen != KRB5_SZ_TOKEN) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+		nfsm_chain_get_opaque(error, &nmc_tmp, cksumlen, tokbuf);
+		if (error)
+			goto nfsmout;
+		error = nfs_gss_token_get(cp->gss_clnt_sched, krb5_mic, tokbuf, 0,
+			NULL, cksum2);
+		if (error)
+			goto nfsmout;
+
+		/* Verify that the checksums are the same */
+		if (bcmp(cksum1, cksum2, 8) != 0) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+		break;
+	case RPCSEC_GSS_SVC_PRIVACY:
+		/*
+		 * Here's what we expect in the privacy results:
+		 *
+		 * - length of confounder + seq num + token + results
+		 * - wrap token (37-40 bytes)
+		 * - confounder (8 bytes)
+		 * - sequence number (4 bytes)
+		 * - results (encrypted)
+		 */
+		nfsm_chain_get_32(error, nmc, reslen);		// length of results
+		if (reslen > NFS_MAXPACKET) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+
+		/* Get the token that prepends the encrypted results */
+		nfsm_chain_get_opaque(error, nmc, KRB5_SZ_TOKMAX, tokbuf);
+		if (error)
+			goto nfsmout;
+		error = nfs_gss_token_get(cp->gss_clnt_sched, krb5_wrap, tokbuf, 0,
+			&toklen, cksum1);
+		if (error)
+			goto nfsmout;
+		nfsm_chain_reverse(nmc, nfsm_pad(toklen));
+		reslen -= toklen;				// size of confounder + seqnum + results
+
+		/* decrypt the confounder + sequence number + results */
+		start = nfsm_chain_offset(nmc);
+		nfs_gss_encrypt_chain(cp->gss_clnt_skey, nmc, start, reslen, DES_DECRYPT);
+
+		/* Compute a checksum over the confounder + sequence number + results */
+		nfs_gss_cksum_chain(cp->gss_clnt_sched, nmc, krb5_wrap, start, reslen, cksum2);
+
+		/* Verify that the checksums are the same */
+		if (bcmp(cksum1, cksum2, 8) != 0) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+
+		nfsm_chain_adv(error, nmc, 8);	// skip over the confounder
+
+		/*
+		 * Get the sequence number prepended to the results
+		 * and compare it against the list in the request.
+		 */
+		nfsm_chain_get_32(error, nmc, seqnum);
+		SLIST_FOREACH(gsp, &req->r_gss_seqlist, gss_seqnext) {
+			if (seqnum == gsp->gss_seqnum)
+				break;
+		}
+		if (gsp == NULL) {
+			error = EBADRPC;
+			goto nfsmout;
+		}
+
+		break;
+	}
+nfsmout:
+	return (error);
+}
+
+/*
+ * An RPCSEC_GSS request with no integrity or privacy consists
+ * of just the header mbufs followed by the arg mbufs.
+ *
+ * However, integrity or privacy both trailer mbufs to the args,
+ * which means we have to do some work to restore the arg mbuf
+ * chain to its previous state in case we need to retransmit.
+ *
+ * The location and length of the args is marked by two fields
+ * in the request structure: r_gss_argoff and r_gss_arglen,
+ * which are stashed when the NFS request is built.
+ */ 
+int
+nfs_gss_clnt_args_restore(struct nfsreq *req)
+{
+	struct nfs_gss_clnt_ctx *cp = req->r_gss_ctx;
+	struct nfsm_chain mchain, *nmc = &mchain;
+	int len, error = 0;
+
+	if (cp == NULL) 
+		return (EAUTH);
+
+	if ((cp->gss_clnt_flags & GSS_CTX_COMPLETE) == 0)
+		return (ENEEDAUTH);
+
+	nfsm_chain_dissect_init(error, nmc, req->r_mhead);	// start at RPC header
+	nfsm_chain_adv(error, nmc, req->r_gss_argoff);		// advance to args
+	if (error)
+		return (error);
+
+	switch (cp->gss_clnt_service) {
+	case RPCSEC_GSS_SVC_NONE:
+		/* nothing to do */
+		break;
+	case RPCSEC_GSS_SVC_INTEGRITY:
+		/*
+		 * All we have to do here is remove the appended checksum mbufs.
+		 * We know that the checksum starts in a new mbuf beyond the end
+		 * of the args.
+		 */
+		nfsm_chain_adv(error, nmc, req->r_gss_arglen);	// adv to last args mbuf
+		if (error)
+			return (error);
+
+		mbuf_freem(mbuf_next(nmc->nmc_mcur));		// free the cksum mbuf
+		error = mbuf_setnext(nmc->nmc_mcur, NULL);
+		break;
+	case RPCSEC_GSS_SVC_PRIVACY:
+		/*
+		 * The args are encrypted along with prepended confounders and seqnum.
+		 * First we decrypt, the confounder, seqnum and args then skip to the
+		 * final mbuf of the args.
+		 * The arglen includes 8 bytes of confounder and 4 bytes of seqnum.
+		 * Finally, we remove between 4 and 8 bytes of encryption padding
+		 * as well as any alignment padding in the trailing mbuf.
+		 */
+		len = req->r_gss_arglen;
+		len += len % 8 > 0 ? 4 : 8;			// add DES padding length
+		nfs_gss_encrypt_chain(cp->gss_clnt_skey, nmc,
+			req->r_gss_argoff, len, DES_DECRYPT);
+		nfsm_chain_adv(error, nmc, req->r_gss_arglen);
+		if (error)
+			return (error);
+		mbuf_freem(mbuf_next(nmc->nmc_mcur));		// free the pad mbuf
+		error = mbuf_setnext(nmc->nmc_mcur, NULL);
+		break;
+	}
+
+	return (error);
+}
+
+/*
+ * This function sets up  a new context on the client.
+ * Context setup alternates upcalls to the gssd with NFS nullproc calls
+ * to the server.  Each of these calls exchanges an opaque token, obtained
+ * via the gssd's calls into the GSS-API on either the client or the server.
+ * This cycle of calls ends when the client's upcall to the gssd and the
+ * server's response both return GSS_S_COMPLETE.  At this point, the client
+ * should have its session key and a handle that it can use to refer to its
+ * new context on the server.
+ */
+static int
+nfs_gss_clnt_ctx_init(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	int client_complete = 0;
+	int server_complete = 0;
+	u_char cksum1[8], cksum2[8];
+	int error = 0;
+	struct timeval now;
+
+	/* Initialize a new client context */
+
+	cp->gss_clnt_svcname = nfs_gss_clnt_svcname(nmp);
+	if (cp->gss_clnt_svcname == NULL) {
+		error = EAUTH;
+		goto nfsmout;
+	}
+	cp->gss_clnt_proc = RPCSEC_GSS_INIT;
+
+	cp->gss_clnt_service =
+		nmp->nm_auth == RPCAUTH_KRB5  ? RPCSEC_GSS_SVC_NONE :
+		nmp->nm_auth == RPCAUTH_KRB5I ? RPCSEC_GSS_SVC_INTEGRITY :
+		nmp->nm_auth == RPCAUTH_KRB5P ? RPCSEC_GSS_SVC_PRIVACY : 0;
+
+	/*
+	 * Now loop around alternating gss_init_sec_context and
+	 * gss_accept_sec_context upcalls to the gssd on the client
+	 * and server side until the context is complete - or fails.
+	 */
+	for (;;) {
+
+		/* Upcall to the gss_init_sec_context in the gssd */
+		error = nfs_gss_clnt_gssd_upcall(req, cp);
+		if (error)
+			goto nfsmout;
+
+		if (cp->gss_clnt_major == GSS_S_COMPLETE) {
+			client_complete = 1;
+			if (server_complete)
+				break;
+		} else if (cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) {
+			error = EAUTH;
+			goto nfsmout;
+		}
+
+		/*
+		 * Pass the token to the server.
+		 */
+		error = nfs_gss_clnt_ctx_callserver(req, cp);
+		if (error)
+			goto nfsmout;
+
+		if (cp->gss_clnt_major == GSS_S_COMPLETE) {
+			server_complete = 1;
+			if (client_complete)
+				break;
+		} else if (cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) {
+			error = EAUTH;
+			goto nfsmout;
+		}
+
+		cp->gss_clnt_proc = RPCSEC_GSS_CONTINUE_INIT;
+	}
+
+	/*
+	 * The context is apparently established successfully
+	 */
+	cp->gss_clnt_flags |= GSS_CTX_COMPLETE;
+	cp->gss_clnt_proc = RPCSEC_GSS_DATA;
+	microuptime(&now);
+	cp->gss_clnt_ctime = now.tv_sec;	// time stamp
+
+	/*
+	 * Construct a key schedule from our shiny new session key
+	 */
+	error = des_key_sched((des_cblock *) cp->gss_clnt_skey, cp->gss_clnt_sched);
+	if (error) {
+		error = EAUTH;
+		goto nfsmout;
+	}
+
+	/*
+	 * Compute checksum of the server's window
+	 */
+	nfs_gss_cksum_rep(cp->gss_clnt_sched, cp->gss_clnt_seqwin, cksum1);
+
+	/*
+	 * and see if it matches the one in the
+	 * verifier the server returned.
+	 */
+	error = nfs_gss_token_get(cp->gss_clnt_sched, krb5_mic, cp->gss_clnt_verf, 0,
+		NULL, cksum2);
+	FREE(cp->gss_clnt_verf, M_TEMP);
+	cp->gss_clnt_verf = NULL;
+
+	if (error || bcmp(cksum1, cksum2, 8) != 0) {
+		error = EAUTH;
+		goto nfsmout;
+	}
+
+	/*
+	 * Set an initial sequence number somewhat randomized.
+	 * Start small so we don't overflow GSS_MAXSEQ too quickly.
+	 * Add the size of the sequence window so seqbits arithmetic
+	 * doesn't go negative.
+	 */
+	cp->gss_clnt_seqnum = (random() & 0xffff) + cp->gss_clnt_seqwin;
+
+	/*
+	 * Allocate a bitmap to keep track of which requests
+	 * are pending within the sequence number window.
+	 */
+	MALLOC(cp->gss_clnt_seqbits, uint32_t *,
+		nfsm_rndup((cp->gss_clnt_seqwin + 7) / 8), M_TEMP, M_WAITOK|M_ZERO);
+	if (cp->gss_clnt_seqbits == NULL)
+		error = EAUTH;
+nfsmout:
+	/*
+	 * If there's an error, just mark it as invalid.
+	 * It will be removed when the reference count
+	 * drops to zero.
+	 */
+	if (error)
+		cp->gss_clnt_flags |= GSS_CTX_INVAL;
+
+	/*
+	 * Wake any threads waiting to use the context
+	 */
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	cp->gss_clnt_thread = NULL;
+	if (cp->gss_clnt_flags & GSS_NEEDCTX) {
+		cp->gss_clnt_flags &= ~GSS_NEEDCTX;
+		wakeup(cp);
+	}
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+
+	return (error);
+}
+
+/*
+ * Call the NFS server using a null procedure for context setup.
+ * Even though it's a null procedure and nominally has no arguments
+ * RFC 2203 requires that the GSS-API token be passed as an argument
+ * and received as a reply.
+ */
+static int
+nfs_gss_clnt_ctx_callserver(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfsm_chain nmreq, nmrep;
+	int error = 0, status;
+	u_int64_t xid;
+	int sz;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+	sz = NFSX_UNSIGNED + nfsm_rndup(cp->gss_clnt_tokenlen);
+	nfsm_chain_build_alloc_init(error, &nmreq, sz);
+	nfsm_chain_add_32(error, &nmreq, cp->gss_clnt_tokenlen);
+	nfsm_chain_add_opaque(error, &nmreq, cp->gss_clnt_token, cp->gss_clnt_tokenlen);
+	nfsm_chain_build_done(error, &nmreq);
+	if (error)
+		goto nfsmout;
+
+	/* Call the server */
+	error = nfs_request2(NULL, nmp->nm_mountp, &nmreq, NFSPROC_NULL,
+			req->r_thread, req->r_cred, 0, &nmrep, &xid, &status);
+	if (cp->gss_clnt_token != NULL) {
+		FREE(cp->gss_clnt_token, M_TEMP);
+		cp->gss_clnt_token = NULL;
+	}
+	if (!error)
+		error = status;
+	if (error)
+		goto nfsmout;
+
+	/* Get the server's reply */
+
+	nfsm_chain_get_32(error, &nmrep, cp->gss_clnt_handle_len);
+	if (cp->gss_clnt_handle != NULL)
+		FREE(cp->gss_clnt_handle, M_TEMP);
+	if (cp->gss_clnt_handle_len > 0) {
+		MALLOC(cp->gss_clnt_handle, u_char *, cp->gss_clnt_handle_len, M_TEMP, M_WAITOK);
+		if (cp->gss_clnt_handle == NULL) {
+			error = ENOMEM;
+			goto nfsmout;
+		}
+		nfsm_chain_get_opaque(error, &nmrep, cp->gss_clnt_handle_len, cp->gss_clnt_handle);
+	}
+	nfsm_chain_get_32(error, &nmrep, cp->gss_clnt_major);
+	nfsm_chain_get_32(error, &nmrep, cp->gss_clnt_minor);
+	nfsm_chain_get_32(error, &nmrep, cp->gss_clnt_seqwin);
+	nfsm_chain_get_32(error, &nmrep, cp->gss_clnt_tokenlen);
+	if (error)
+		goto nfsmout;
+	if (cp->gss_clnt_tokenlen > 0) {
+		MALLOC(cp->gss_clnt_token, u_char *, cp->gss_clnt_tokenlen, M_TEMP, M_WAITOK);
+		if (cp->gss_clnt_token == NULL) {
+			error = ENOMEM;
+			goto nfsmout;
+		}
+		nfsm_chain_get_opaque(error, &nmrep, cp->gss_clnt_tokenlen, cp->gss_clnt_token);
+	}
+
+	/*
+	 * Make sure any unusual errors are expanded and logged by gssd
+	 */
+	if (cp->gss_clnt_major != GSS_S_COMPLETE &&
+	    cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) {
+		char who[] = "server";
+
+		(void) mach_gss_log_error(
+			cp->gss_clnt_mport,
+			vfs_statfs(nmp->nm_mountp)->f_mntfromname,
+			cp->gss_clnt_uid,
+			who,
+			cp->gss_clnt_major,
+			cp->gss_clnt_minor);
+	}
+
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	return (error);
+}
+
+/*
+ * Ugly hack to get the service principal from the f_mntfromname field in 
+ * the statfs struct. We assume a format of server:path. We don't currently
+ * support url's or other bizarre formats like path@server. A better solution
+ * here might be to allow passing the service principal down in the mount args.
+ * For kerberos we just use the default realm. 
+ */
+static char *
+nfs_gss_clnt_svcname(struct nfsmount *nmp)
+{
+	char *svcname, *d;
+	char* mntfromhere = &vfs_statfs(nmp->nm_mountp)->f_mntfromname[0];
+	int len;
+
+	len = strlen(mntfromhere) + 5; /* "nfs/" plus null */
+	MALLOC(svcname, char *, len, M_TEMP, M_NOWAIT);
+	if (svcname == NULL)
+		return (NULL);
+	strlcpy(svcname, "nfs/", len);
+	strlcat(svcname, mntfromhere, len);
+	d = strchr(svcname, ':');
+	if (d)
+		*d = '\0';
+
+	return (svcname);
+}
+
+/*
+ * Make an upcall to the gssd using Mach RPC
+ * The upcall is made using a task special port.
+ * This allows launchd to fire up the gssd in the
+ * user's session.  This is important, since gssd
+ * must have access to the user's credential cache.
+ */
+static int
+nfs_gss_clnt_gssd_upcall(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp)
+{
+	kern_return_t kr;
+	byte_buffer okey = NULL;
+	uint32_t skeylen = 0;
+	int retry_cnt = 0;
+	vm_map_copy_t itoken = NULL;
+	byte_buffer otoken = NULL;
+	int error = 0;
+	char uprinc[1];
+
+	/*
+	 * NFS currently only supports default principals or
+	 * principals based on the uid of the caller.
+	 *
+	 * N.B. Note we define a one character array for the principal
+	 * so that we can hold an empty string required by mach, since 
+	 * the kernel is being compiled with -Wwrite-strings.
+	 */
+	uprinc[0] = '\0';
+	if (cp->gss_clnt_mport == NULL) {
+		kr = task_get_gssd_port(get_threadtask(req->r_thread), &cp->gss_clnt_mport);
+		if (kr != KERN_SUCCESS) {
+			printf("nfs_gss_clnt_gssd_upcall: can't get gssd port, status %d\n", kr);
+			return (EAUTH);
+		}
+		if (!IPC_PORT_VALID(cp->gss_clnt_mport)) {
+			printf("nfs_gss_clnt_gssd_upcall: gssd port not valid\n");
+			cp->gss_clnt_mport = NULL;
+			return (EAUTH);
+		}
+	}
+
+	if (cp->gss_clnt_tokenlen > 0)
+		nfs_gss_mach_alloc_buffer(cp->gss_clnt_token, cp->gss_clnt_tokenlen, &itoken);
+
+retry:
+	kr = mach_gss_init_sec_context(
+		cp->gss_clnt_mport,
+		KRB5_MECH,
+		(byte_buffer) itoken, (mach_msg_type_number_t) cp->gss_clnt_tokenlen,
+		cp->gss_clnt_uid,
+		uprinc,
+		cp->gss_clnt_svcname,
+		GSSD_MUTUAL_FLAG | GSSD_NO_UI,
+		&cp->gss_clnt_gssd_verf,
+		&cp->gss_clnt_context,
+		&cp->gss_clnt_cred_handle,
+		&okey,  (mach_msg_type_number_t *) &skeylen,
+		&otoken, (mach_msg_type_number_t *) &cp->gss_clnt_tokenlen,
+		&cp->gss_clnt_major,
+		&cp->gss_clnt_minor);
+
+	if (kr != 0) {
+		printf("nfs_gss_clnt_gssd_upcall: mach_gss_init_sec_context failed: %x\n", kr);
+		if (kr == MIG_SERVER_DIED && cp->gss_clnt_cred_handle == 0 &&
+			retry_cnt++ < NFS_GSS_MACH_MAX_RETRIES)
+			goto retry;
+		task_release_special_port(cp->gss_clnt_mport);
+		cp->gss_clnt_mport = NULL;
+		return (EAUTH);
+	}
+
+	/*
+	 * Make sure any unusual errors are expanded and logged by gssd
+	 */
+	if (cp->gss_clnt_major != GSS_S_COMPLETE &&
+	    cp->gss_clnt_major != GSS_S_CONTINUE_NEEDED) {
+		char who[] = "client";
+
+		(void) mach_gss_log_error(
+			cp->gss_clnt_mport,
+			vfs_statfs(req->r_nmp->nm_mountp)->f_mntfromname,
+			cp->gss_clnt_uid,
+			who,
+			cp->gss_clnt_major,
+			cp->gss_clnt_minor);
+	}
+
+	if (skeylen > 0) {
+		if (skeylen != SKEYLEN) {
+			printf("nfs_gss_clnt_gssd_upcall: bad key length (%d)\n", skeylen);
+			return (EAUTH);
+		}
+		error = nfs_gss_mach_vmcopyout((vm_map_copy_t) okey, skeylen, cp->gss_clnt_skey);
+		if (error)
+			return (EAUTH);
+	}
+
+	if (cp->gss_clnt_tokenlen > 0) {
+		MALLOC(cp->gss_clnt_token, u_char *, cp->gss_clnt_tokenlen, M_TEMP, M_WAITOK);
+		if (cp->gss_clnt_token == NULL)
+			return (ENOMEM);
+		error = nfs_gss_mach_vmcopyout((vm_map_copy_t) otoken, cp->gss_clnt_tokenlen,
+			cp->gss_clnt_token);
+		if (error)
+			return (EAUTH);
+	}
+
+	return (0);
+}
+
+/*
+ * Invoked at the completion of an RPC call that uses an RPCSEC_GSS
+ * credential. The sequence number window that the server returns
+ * at context setup indicates the maximum number of client calls that
+ * can be outstanding on a context. The client maintains a bitmap that
+ * represents the server's window.  Each pending request has a bit set
+ * in the window bitmap.  When a reply comes in or times out, we reset
+ * the bit in the bitmap and if there are any other threads waiting for
+ * a context slot we notify the waiting thread(s).
+ *
+ * Note that if a request is retransmitted, it will have a single XID
+ * but it may be associated with multiple sequence numbers.  So we
+ * may have to reset multiple sequence number bits in the window bitmap.
+ */
+void
+nfs_gss_clnt_rpcdone(struct nfsreq *req)
+{
+	struct nfs_gss_clnt_ctx *cp = req->r_gss_ctx;
+	struct gss_seq *gsp, *ngsp;
+	int i = 0;
+
+	if (cp == NULL || !(cp->gss_clnt_flags & GSS_CTX_COMPLETE))
+		return;	// no context - don't bother
+	/*
+	 * Reset the bit for this request in the
+	 * sequence number window to indicate it's done.
+	 * We do this even if the request timed out.
+	 */
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	gsp = SLIST_FIRST(&req->r_gss_seqlist);
+	if (gsp && gsp->gss_seqnum > (cp->gss_clnt_seqnum - cp->gss_clnt_seqwin))
+		win_resetbit(cp->gss_clnt_seqbits,
+			gsp->gss_seqnum % cp->gss_clnt_seqwin);
+
+	/*
+	 * Limit the seqnum list to GSS_CLNT_SEQLISTMAX entries
+	 */
+	SLIST_FOREACH_SAFE(gsp, &req->r_gss_seqlist, gss_seqnext, ngsp) {
+		if (++i > GSS_CLNT_SEQLISTMAX) {
+			SLIST_REMOVE(&req->r_gss_seqlist, gsp, gss_seq, gss_seqnext);
+			FREE(gsp, M_TEMP);
+		}
+	}
+
+	/*
+	 * If there's a thread waiting for
+	 * the window to advance, wake it up.
+	 */
+	if (cp->gss_clnt_flags & GSS_NEEDSEQ) {
+		cp->gss_clnt_flags &= ~GSS_NEEDSEQ;
+		wakeup(cp);
+	}
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+}
+
+/*
+ * Create a reference to a context from a request
+ * and bump the reference count
+ */
+void
+nfs_gss_clnt_ctx_ref(struct nfsreq *req, struct nfs_gss_clnt_ctx *cp)
+{
+	req->r_gss_ctx = cp;
+
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	cp->gss_clnt_refcnt++;
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+}
+
+/*
+ * Remove a context reference from a request
+ * If the reference count drops to zero, and the
+ * context is invalid, destroy the context
+ */
+void
+nfs_gss_clnt_ctx_unref(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfs_gss_clnt_ctx *cp = req->r_gss_ctx;
+
+	if (cp == NULL)
+		return;
+
+	req->r_gss_ctx = NULL;
+
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	if (--cp->gss_clnt_refcnt == 0
+		&& cp->gss_clnt_flags & GSS_CTX_INVAL) {
+		lck_mtx_unlock(cp->gss_clnt_mtx);
+
+		if (nmp)
+			lck_mtx_lock(&nmp->nm_lock);
+		nfs_gss_clnt_ctx_remove(nmp, cp);
+		if (nmp)
+			lck_mtx_unlock(&nmp->nm_lock);
+
+		return;
+	}
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+}
+
+/*
+ * Remove a context
+ */
+static void
+nfs_gss_clnt_ctx_remove(struct nfsmount *nmp, struct nfs_gss_clnt_ctx *cp)
+{
+	/*
+	 * If dequeueing, assume nmp->nm_lock is held
+	 */
+	if (nmp != NULL)
+		TAILQ_REMOVE(&nmp->nm_gsscl, cp, gss_clnt_entries);
+
+	if (cp->gss_clnt_mport)
+		task_release_special_port(cp->gss_clnt_mport);
+	if (cp->gss_clnt_mtx)
+		lck_mtx_destroy(cp->gss_clnt_mtx, nfs_gss_clnt_grp);
+	if (cp->gss_clnt_handle)
+		FREE(cp->gss_clnt_handle, M_TEMP);
+	if (cp->gss_clnt_seqbits)
+		FREE(cp->gss_clnt_seqbits, M_TEMP);
+	if (cp->gss_clnt_token)
+		FREE(cp->gss_clnt_token, M_TEMP);
+	if (cp->gss_clnt_svcname)
+		FREE(cp->gss_clnt_svcname, M_TEMP);
+	FREE(cp, M_TEMP);
+}
+
+/*
+ * The context for a user is invalid.
+ * Mark the context as invalid, then
+ * create a new context.
+ */
+int
+nfs_gss_clnt_ctx_renew(struct nfsreq *req)
+{
+	struct nfs_gss_clnt_ctx *cp = req->r_gss_ctx;
+	struct nfsmount *nmp = req->r_nmp;
+	struct nfs_gss_clnt_ctx *ncp;
+	int error = 0;
+	uid_t saved_uid;
+	mach_port_t saved_mport;
+	int retrycnt = 0;
+
+	if (cp == NULL || !(cp->gss_clnt_flags & GSS_CTX_COMPLETE))
+		return (0);
+
+	lck_mtx_lock(cp->gss_clnt_mtx);
+	if (cp->gss_clnt_flags & GSS_CTX_INVAL) {
+		lck_mtx_unlock(cp->gss_clnt_mtx);
+		nfs_gss_clnt_ctx_unref(req);
+		return (0);	// already being renewed
+	}
+	saved_uid = cp->gss_clnt_uid;
+	saved_mport = task_copy_special_port(cp->gss_clnt_mport);
+
+	/* Remove the old context */
+	lck_mtx_lock(&nmp->nm_lock);
+	cp->gss_clnt_flags |= GSS_CTX_INVAL;
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	/*
+	 * If there's a thread waiting
+	 * in the old context, wake it up.
+	 */
+	if (cp->gss_clnt_flags & (GSS_NEEDCTX | GSS_NEEDSEQ)) {
+		cp->gss_clnt_flags &= ~GSS_NEEDSEQ;
+		wakeup(cp);
+	}
+	lck_mtx_unlock(cp->gss_clnt_mtx);
+
+retry:
+	/*
+	 * Create a new context
+	 */
+	MALLOC(ncp, struct nfs_gss_clnt_ctx *, sizeof(*ncp),
+		M_TEMP, M_WAITOK|M_ZERO);
+	if (ncp == NULL) {
+		return (ENOMEM);
+	}
+
+	ncp->gss_clnt_uid = saved_uid;
+	ncp->gss_clnt_mport = task_copy_special_port(saved_mport); // re-use the gssd port
+	ncp->gss_clnt_mtx = lck_mtx_alloc_init(nfs_gss_clnt_grp, LCK_ATTR_NULL);
+	ncp->gss_clnt_thread = current_thread();
+	lck_mtx_lock(&nmp->nm_lock);
+	TAILQ_INSERT_TAIL(&nmp->nm_gsscl, ncp, gss_clnt_entries);
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	/* Adjust reference counts to new and old context */
+	nfs_gss_clnt_ctx_unref(req);
+	nfs_gss_clnt_ctx_ref(req, ncp);
+
+	error = nfs_gss_clnt_ctx_init(req, ncp); // Initialize new context
+	if (error == ENEEDAUTH) {
+		error = nfs_gss_clnt_ctx_delay(req, &retrycnt);
+		if (!error)
+			goto retry;
+	}
+
+	task_release_special_port(saved_mport);
+	if (error)
+		nfs_gss_clnt_ctx_unref(req);
+
+	return (error);
+}
+
+/*
+ * Destroy all the contexts associated with a mount.
+ * The contexts are also destroyed by the server.
+ */
+void
+nfs_gss_clnt_ctx_unmount(struct nfsmount *nmp, int mntflags)
+{
+	struct nfs_gss_clnt_ctx *cp;
+	struct ucred temp_cred;
+	kauth_cred_t cred;
+	struct nfsm_chain nmreq, nmrep;
+	u_int64_t xid;
+	int error, status;
+	struct nfsreq req;
+
+	bzero((caddr_t) &temp_cred, sizeof(temp_cred));
+	temp_cred.cr_ngroups = 1;
+	req.r_nmp = nmp;
+
+	for (;;) {
+		lck_mtx_lock(&nmp->nm_lock);
+		cp = TAILQ_FIRST(&nmp->nm_gsscl);
+		lck_mtx_unlock(&nmp->nm_lock);
+		if (cp == NULL)
+			break;
+
+		nfs_gss_clnt_ctx_ref(&req, cp);
+
+		/*
+		 * Tell the server to destroy its context.
+		 * But don't bother if it's a forced unmount
+		 * or if it's a dummy sec=sys context.
+		 */
+		if (!(mntflags & MNT_FORCE) && cp->gss_clnt_service != RPCSEC_GSS_SVC_SYS) {
+			temp_cred.cr_uid = cp->gss_clnt_uid;
+			cred = kauth_cred_create(&temp_cred);
+			cp->gss_clnt_proc = RPCSEC_GSS_DESTROY;
+
+			error = 0;
+			nfsm_chain_null(&nmreq);
+			nfsm_chain_null(&nmrep);
+			nfsm_chain_build_alloc_init(error, &nmreq, 0);
+			nfsm_chain_build_done(error, &nmreq);
+			if (!error)
+				nfs_request2(NULL, nmp->nm_mountp, &nmreq, NFSPROC_NULL,
+					current_thread(), cred, 0, &nmrep, &xid, &status);
+			nfsm_chain_cleanup(&nmreq);
+			nfsm_chain_cleanup(&nmrep);
+			kauth_cred_unref(&cred);
+		}
+
+		/*
+		 * Mark the context invalid then drop
+		 * the reference to remove it if its
+		 * refcount is zero.
+		 */
+		cp->gss_clnt_flags |= GSS_CTX_INVAL;
+		nfs_gss_clnt_ctx_unref(&req);
+	}
+}
+
+/*
+ * If we get a failure in trying to establish a context we need to wait a 
+ * little while to see if the server is feeling better. In our case this is
+ * probably a failure in directory services not coming up in a timely fashion.
+ * This routine sort of mimics receiving a jukebox error.
+ */
+static int
+nfs_gss_clnt_ctx_delay(struct nfsreq *req, int *retry)
+{
+	int timeo = (1 << *retry) * NFS_TRYLATERDEL;
+	int error = 0;
+	struct nfsmount *nmp = req->r_nmp;
+	struct timeval now;
+	time_t waituntil;
+
+	if ((nmp->nm_flag & NFSMNT_SOFT) && *retry > nmp->nm_retry)
+		return (ETIMEDOUT);
+	if (timeo > 60)
+		timeo = 60;
+
+	microuptime(&now);
+	waituntil = now.tv_sec + timeo;
+	while (now.tv_sec < waituntil) {
+		tsleep(&lbolt, PSOCK, "nfs_gss_clnt_ctx_delay", 0);
+		error = nfs_sigintr(nmp, req, current_thread(), 0);
+		if (error)
+			break;
+		microuptime(&now);
+	}
+	*retry += 1;
+
+	return (error);
+}
+
+
+#endif /* NFSCLIENT */
+
+/*************
+ *
+ * Server functions
+ */
+
+#if NFSSERVER
+
+/*
+ * Find a server context based on a handle value received
+ * in an RPCSEC_GSS credential.
+ */
+static struct nfs_gss_svc_ctx *
+nfs_gss_svc_ctx_find(uint32_t handle)
+{
+	struct nfs_gss_svc_ctx_hashhead *head;
+	struct nfs_gss_svc_ctx *cp;
+	
+	head = &nfs_gss_svc_ctx_hashtbl[SVC_CTX_HASH(handle)];
+
+	lck_mtx_lock(nfs_gss_svc_ctx_mutex);
+	LIST_FOREACH(cp, head, gss_svc_entries)
+		if (cp->gss_svc_handle == handle)
+			break;
+	lck_mtx_unlock(nfs_gss_svc_ctx_mutex);
+
+	return (cp);
+}
+
+/*
+ * Insert a new server context into the hash table
+ * and start the context reap thread if necessary.
+ */
+static void
+nfs_gss_svc_ctx_insert(struct nfs_gss_svc_ctx *cp)
+{
+	struct nfs_gss_svc_ctx_hashhead *head;
+	
+	head = &nfs_gss_svc_ctx_hashtbl[SVC_CTX_HASH(cp->gss_svc_handle)];
+
+	lck_mtx_lock(nfs_gss_svc_ctx_mutex);
+	LIST_INSERT_HEAD(head, cp, gss_svc_entries);
+	nfs_gss_ctx_count++;
+
+	if (!nfs_gss_timer_on) {
+		nfs_gss_timer_on = 1;
+		nfs_interval_timer_start(nfs_gss_svc_ctx_timer_call,
+			GSS_TIMER_PERIOD * MSECS_PER_SEC);
+	}
+	lck_mtx_unlock(nfs_gss_svc_ctx_mutex);
+}
+
+/*
+ * This function is called via the kernel's callout
+ * mechanism.  It runs only when there are
+ * cached RPCSEC_GSS contexts.
+ */
+void
+nfs_gss_svc_ctx_timer(__unused void *param1, __unused void *param2)
+{
+	struct nfs_gss_svc_ctx_hashhead *head;
+	struct nfs_gss_svc_ctx *cp, *next;
+	uint64_t timenow;
+	int contexts = 0;
+	int i;
+
+	lck_mtx_lock(nfs_gss_svc_ctx_mutex);
+	clock_get_uptime(&timenow);
+
+	/*
+	 * Scan all the hash chains
+	 * Assume nfs_gss_svc_ctx_mutex is held
+	 */
+	for (i = 0; i < SVC_CTX_HASHSZ; i++) {
+		/*
+		 * For each hash chain, look for entries
+		 * that haven't been used in a while.
+		 */
+		head = &nfs_gss_svc_ctx_hashtbl[i];
+		for (cp = LIST_FIRST(head); cp; cp = next) {
+			contexts++;
+			next = LIST_NEXT(cp, gss_svc_entries);
+			if (timenow > cp->gss_svc_expiretime) {
+				/*
+				 * A stale context - remove it
+				 */
+				LIST_REMOVE(cp, gss_svc_entries);
+				if (cp->gss_svc_seqbits)
+					FREE(cp->gss_svc_seqbits, M_TEMP);
+				lck_mtx_destroy(cp->gss_svc_mtx, nfs_gss_svc_grp);
+				FREE(cp, M_TEMP);
+				contexts--;
+			}
+		}
+	}
+
+	nfs_gss_ctx_count = contexts;
+
+	/*
+	 * If there are still some cached contexts left,
+	 * set up another callout to check on them later.
+	 */
+	nfs_gss_timer_on = nfs_gss_ctx_count > 0;
+	if (nfs_gss_timer_on)
+		nfs_interval_timer_start(nfs_gss_svc_ctx_timer_call,
+			GSS_TIMER_PERIOD * MSECS_PER_SEC);
+
+	lck_mtx_unlock(nfs_gss_svc_ctx_mutex);
+}
+
+/*
+ * Here the server receives an RPCSEC_GSS credential in an
+ * RPC call header.  First there's some checking to make sure
+ * the credential is appropriate - whether the context is still
+ * being set up, or is complete.  Then we use the handle to find
+ * the server's context and validate the verifier, which contains
+ * a signed checksum of the RPC header. If the verifier checks
+ * out, we extract the user's UID and groups from the context
+ * and use it to set up a UNIX credential for the user's request.
+ */
+int
+nfs_gss_svc_cred_get(struct nfsrv_descript *nd, struct nfsm_chain *nmc)
+{
+	uint32_t vers, proc, seqnum, service;
+	uint32_t handle, handle_len;
+	struct nfs_gss_svc_ctx *cp = NULL;
+	uint32_t flavor = 0, verflen = 0;
+	int error = 0;
+	uint32_t arglen, start, toklen, cksumlen;
+	u_char tokbuf[KRB5_SZ_TOKMAX];
+	u_char cksum1[8], cksum2[8];
+	struct nfsm_chain nmc_tmp;
+
+	vers = proc = seqnum = service = handle_len = 0;
+	arglen = cksumlen = 0;
+
+	nfsm_chain_get_32(error, nmc, vers);
+	if (vers != RPCSEC_GSS_VERS_1) {
+		error = NFSERR_AUTHERR | AUTH_REJECTCRED;
+		goto nfsmout;
+	}
+
+	nfsm_chain_get_32(error, nmc, proc);
+	nfsm_chain_get_32(error, nmc, seqnum);
+	nfsm_chain_get_32(error, nmc, service);
+	nfsm_chain_get_32(error, nmc, handle_len);
+	if (error)
+		goto nfsmout;
+
+	/*
+	 * Make sure context setup/destroy is being done with a nullproc
+	 */
+	if (proc != RPCSEC_GSS_DATA && nd->nd_procnum != NFSPROC_NULL) {
+		error = NFSERR_AUTHERR | RPCSEC_GSS_CREDPROBLEM;
+		goto nfsmout;
+	}
+
+	/*
+	 * If the sequence number is greater than the max
+	 * allowable, reject and have the client init a
+	 * new context.
+	 */
+	if (seqnum > GSS_MAXSEQ) {
+		error = NFSERR_AUTHERR | RPCSEC_GSS_CTXPROBLEM;
+		goto nfsmout;
+	}
+
+	nd->nd_sec = 
+		service == RPCSEC_GSS_SVC_NONE ?      RPCAUTH_KRB5 :
+		service == RPCSEC_GSS_SVC_INTEGRITY ? RPCAUTH_KRB5I :
+		service == RPCSEC_GSS_SVC_PRIVACY ?   RPCAUTH_KRB5P : 0;
+
+	if (proc == RPCSEC_GSS_INIT) {
+		/*
+		 * Limit the total number of contexts
+		 */
+		if (nfs_gss_ctx_count > nfs_gss_ctx_max) {
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CTXPROBLEM;
+			goto nfsmout;
+		}
+
+		/*
+		 * Set up a new context
+		 */
+		MALLOC(cp, struct nfs_gss_svc_ctx *, sizeof(*cp), M_TEMP, M_WAITOK|M_ZERO);
+		if (cp == NULL) {
+			error = ENOMEM;
+			goto nfsmout;
+		}
+	} else {
+
+		/*
+		 * Use the handle to find the context
+		 */
+		if (handle_len != sizeof(handle)) {
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CREDPROBLEM;
+			goto nfsmout;
+		}
+		nfsm_chain_get_32(error, nmc, handle);
+		if (error)
+			goto nfsmout;
+		cp = nfs_gss_svc_ctx_find(handle);
+		if (cp == NULL) {
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CTXPROBLEM;
+			goto nfsmout;
+		}
+	}
+
+	cp->gss_svc_proc = proc;
+
+	if (proc == RPCSEC_GSS_DATA || proc == RPCSEC_GSS_DESTROY) {
+		struct ucred temp_cred;
+
+		if (cp->gss_svc_seqwin == 0) {
+			/*
+			 * Context isn't complete
+			 */
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CTXPROBLEM;
+			goto nfsmout;
+		}
+
+		if (!nfs_gss_svc_seqnum_valid(cp, seqnum)) {
+			/*
+			 * Sequence number is bad
+			 */
+			error = EINVAL;	// drop the request
+			goto nfsmout;
+		}
+
+		/* Now compute the client's call header checksum */
+		nfs_gss_cksum_chain(cp->gss_svc_sched, nmc, krb5_mic, 0, 0, cksum1);
+
+		/*
+		 * Validate the verifier.
+		 * The verifier contains an encrypted checksum
+		 * of the call header from the XID up to and
+		 * including the credential.  We compute the
+		 * checksum and compare it with what came in
+		 * the verifier.
+		 */
+		nfsm_chain_get_32(error, nmc, flavor);
+		nfsm_chain_get_32(error, nmc, verflen);
+		if (flavor != RPCSEC_GSS || verflen != KRB5_SZ_TOKEN)
+			error = NFSERR_AUTHERR | AUTH_BADVERF;
+		nfsm_chain_get_opaque(error, nmc, verflen, tokbuf);
+		if (error)
+			goto nfsmout;
+
+		/* Get the checksum from the token inside the verifier */
+		error = nfs_gss_token_get(cp->gss_svc_sched, krb5_mic, tokbuf, 1,
+			NULL, cksum2);
+		if (error)
+			goto nfsmout;
+
+		if (bcmp(cksum1, cksum2, 8) != 0) {
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CTXPROBLEM;
+			goto nfsmout;
+		}
+
+		nd->nd_gss_seqnum = seqnum;
+
+		/*
+		 * Set up the user's cred
+		 */
+		bzero(&temp_cred, sizeof(temp_cred));
+		temp_cred.cr_uid = cp->gss_svc_uid;
+		bcopy(cp->gss_svc_gids, temp_cred.cr_groups,
+				sizeof(gid_t) * cp->gss_svc_ngroups);
+		temp_cred.cr_ngroups = cp->gss_svc_ngroups;
+
+		nd->nd_cr = kauth_cred_create(&temp_cred);
+		if (nd->nd_cr == NULL) {
+			error = ENOMEM;
+			goto nfsmout;
+		}
+		clock_interval_to_deadline(GSS_CTX_EXPIRE, NSEC_PER_SEC,
+			&cp->gss_svc_expiretime);
+
+		/*
+		 * If the call arguments are integrity or privacy protected
+		 * then we need to check them here.
+		 */
+		switch (service) {
+		case RPCSEC_GSS_SVC_NONE:
+			/* nothing to do */
+			break;
+		case RPCSEC_GSS_SVC_INTEGRITY:
+			/*
+			 * Here's what we expect in the integrity call args:
+			 *
+			 * - length of seq num + call args (4 bytes)
+			 * - sequence number (4 bytes)
+			 * - call args (variable bytes)
+			 * - length of checksum token (37)
+			 * - checksum of seqnum + call args (37 bytes)
+			 */
+			nfsm_chain_get_32(error, nmc, arglen);		// length of args
+			if (arglen > NFS_MAXPACKET) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+
+			/* Compute the checksum over the call args */
+			start = nfsm_chain_offset(nmc);
+			nfs_gss_cksum_chain(cp->gss_svc_sched, nmc, krb5_mic, start, arglen, cksum1);
+	
+			/*
+			 * Get the sequence number prepended to the args
+			 * and compare it against the one sent in the
+			 * call credential.
+			 */
+			nfsm_chain_get_32(error, nmc, seqnum);
+			if (seqnum != nd->nd_gss_seqnum) {
+				error = EBADRPC;			// returns as GARBAGEARGS
+				goto nfsmout;
+			}
+	
+			/*
+			 * Advance to the end of the args and
+			 * fetch the checksum computed by the client.
+			 */
+			nmc_tmp = *nmc;	
+			arglen -= NFSX_UNSIGNED;			// skipped seqnum
+			nfsm_chain_adv(error, &nmc_tmp, arglen);	// skip args
+			nfsm_chain_get_32(error, &nmc_tmp, cksumlen);	// length of checksum
+			if (cksumlen != KRB5_SZ_TOKEN) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+			nfsm_chain_get_opaque(error, &nmc_tmp, cksumlen, tokbuf);
+			if (error)
+				goto nfsmout;
+			error = nfs_gss_token_get(cp->gss_svc_sched, krb5_mic, tokbuf, 1,
+				NULL, cksum2);
+	
+			/* Verify that the checksums are the same */
+			if (error || bcmp(cksum1, cksum2, 8) != 0) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+			break;
+		case RPCSEC_GSS_SVC_PRIVACY:
+			/*
+			 * Here's what we expect in the privacy call args:
+			 *
+			 * - length of confounder + seq num + token + call args
+			 * - wrap token (37-40 bytes)
+			 * - confounder (8 bytes)
+			 * - sequence number (4 bytes)
+			 * - call args (encrypted)
+			 */
+			nfsm_chain_get_32(error, nmc, arglen);		// length of args
+			if (arglen > NFS_MAXPACKET) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+	
+			/* Get the token that prepends the encrypted args */
+			nfsm_chain_get_opaque(error, nmc, KRB5_SZ_TOKMAX, tokbuf);
+			if (error)
+				goto nfsmout;
+			error = nfs_gss_token_get(cp->gss_svc_sched, krb5_wrap, tokbuf, 1,
+				&toklen, cksum1);
+			if (error)
+				goto nfsmout;
+			nfsm_chain_reverse(nmc, nfsm_pad(toklen));
+	
+			/* decrypt the 8 byte confounder + seqnum + args */
+			start = nfsm_chain_offset(nmc);
+			arglen -= toklen;
+			nfs_gss_encrypt_chain(cp->gss_svc_skey, nmc, start, arglen, DES_DECRYPT);
+	
+			/* Compute a checksum over the sequence number + results */
+			nfs_gss_cksum_chain(cp->gss_svc_sched, nmc, krb5_wrap, start, arglen, cksum2);
+	
+			/* Verify that the checksums are the same */
+			if (bcmp(cksum1, cksum2, 8) != 0) {
+				error = EBADRPC;
+				goto nfsmout;
+			}
+
+			/*
+			 * Get the sequence number prepended to the args
+			 * and compare it against the one sent in the
+			 * call credential.
+			 */
+			nfsm_chain_adv(error, nmc, 8);			// skip over the confounder
+			nfsm_chain_get_32(error, nmc, seqnum);
+			if (seqnum != nd->nd_gss_seqnum) {
+				error = EBADRPC;			// returns as GARBAGEARGS
+				goto nfsmout;
+			}
+			break;
+		}
+	} else {
+		/*
+		 * If the proc is RPCSEC_GSS_INIT or RPCSEC_GSS_CONTINUE_INIT
+		 * then we expect a null verifier.
+		 */
+		nfsm_chain_get_32(error, nmc, flavor);
+		nfsm_chain_get_32(error, nmc, verflen);
+		if (error || flavor != RPCAUTH_NULL || verflen > 0)
+			error = NFSERR_AUTHERR | RPCSEC_GSS_CREDPROBLEM;
+		if (error)
+			goto nfsmout;
+	}
+
+	nd->nd_gss_context = cp;
+nfsmout:
+	return (error);
+}
+
+/*
+ * Insert the server's verifier into the RPC reply header.
+ * It contains a signed checksum of the sequence number that
+ * was received in the RPC call.
+ * Then go on to add integrity or privacy if necessary.
+ */
+int
+nfs_gss_svc_verf_put(struct nfsrv_descript *nd, struct nfsm_chain *nmc)
+{
+	struct nfs_gss_svc_ctx *cp;
+	int error = 0;
+	u_char tokbuf[KRB5_SZ_TOKEN];
+	int toklen;
+	u_char cksum[8];
+
+	cp = nd->nd_gss_context;
+
+	if (cp->gss_svc_major != GSS_S_COMPLETE) {
+		/*
+		 * If the context isn't yet complete
+		 * then return a null verifier.
+		 */
+		nfsm_chain_add_32(error, nmc, RPCAUTH_NULL);
+		nfsm_chain_add_32(error, nmc, 0);
+		return (error);
+	}
+
+	/*
+	 * Compute checksum of the request seq number
+	 * If it's the final reply of context setup
+	 * then return the checksum of the context
+	 * window size.
+	 */
+	if (cp->gss_svc_proc == RPCSEC_GSS_INIT ||
+	    cp->gss_svc_proc == RPCSEC_GSS_CONTINUE_INIT)
+		nfs_gss_cksum_rep(cp->gss_svc_sched, cp->gss_svc_seqwin, cksum);
+	else
+		nfs_gss_cksum_rep(cp->gss_svc_sched, nd->nd_gss_seqnum, cksum);
+	/*
+	 * Now wrap it in a token and add
+	 * the verifier to the reply.
+	 */
+	toklen = nfs_gss_token_put(cp->gss_svc_sched, krb5_mic, tokbuf, 0, 0, cksum);
+	nfsm_chain_add_32(error, nmc, RPCSEC_GSS);
+	nfsm_chain_add_32(error, nmc, toklen);
+	nfsm_chain_add_opaque(error, nmc, tokbuf, toklen);
+
+	return (error);
+}
+
+/*
+ * The results aren't available yet, but if they need to be
+ * checksummed for integrity protection or encrypted, then
+ * we can record the start offset here, insert a place-holder
+ * for the results length, as well as the sequence number.
+ * The rest of the work is done later by nfs_gss_svc_protect_reply()
+ * when the results are available.
+ */
+int
+nfs_gss_svc_prepare_reply(struct nfsrv_descript *nd, struct nfsm_chain *nmc)
+{
+	struct nfs_gss_svc_ctx *cp = nd->nd_gss_context;
+	int error = 0;
+
+	if (cp->gss_svc_proc == RPCSEC_GSS_INIT ||
+	    cp->gss_svc_proc == RPCSEC_GSS_CONTINUE_INIT)
+		return (0);
+
+	switch (nd->nd_sec) {
+	case RPCAUTH_KRB5:
+		/* Nothing to do */
+		break;
+	case RPCAUTH_KRB5I:
+		nd->nd_gss_mb = nmc->nmc_mcur;			// record current mbuf
+		nfsm_chain_finish_mbuf(error, nmc);		// split the chain here
+		nfsm_chain_add_32(error, nmc, nd->nd_gss_seqnum); // req sequence number
+		break;
+	case RPCAUTH_KRB5P:
+		nd->nd_gss_mb = nmc->nmc_mcur;			// record current mbuf
+		nfsm_chain_finish_mbuf(error, nmc);		// split the chain here
+		nfsm_chain_add_32(error, nmc, random());	// confounder bytes 1-4
+		nfsm_chain_add_32(error, nmc, random());	// confounder bytes 5-8
+		nfsm_chain_add_32(error, nmc, nd->nd_gss_seqnum); // req sequence number
+		break;
+	}
+
+	return (error);
+}
+
+/*
+ * The results are checksummed or encrypted for return to the client
+ */
+int
+nfs_gss_svc_protect_reply(struct nfsrv_descript *nd, mbuf_t mrep)
+{
+	struct nfs_gss_svc_ctx *cp = nd->nd_gss_context;
+	struct nfsm_chain nmrep_res, *nmc_res = &nmrep_res;
+	struct nfsm_chain nmrep_pre, *nmc_pre = &nmrep_pre;
+	mbuf_t mb, results;
+	uint32_t reslen;
+	u_char tokbuf[KRB5_SZ_TOKMAX];
+	int pad, toklen;
+	u_char cksum[8];
+	int error = 0;
+
+	/*
+	 * Using a reference to the mbuf where we previously split the reply
+	 * mbuf chain, we split the mbuf chain argument into two mbuf chains,
+	 * one that allows us to prepend a length field or token, (nmc_pre)
+	 * and the second which holds just the results that we're going to
+	 * checksum and/or encrypt.  When we're done, we join the chains back
+	 * together.
+	 */
+	nfs_gss_nfsm_chain(nmc_res, mrep);		// set up the results chain
+	mb = nd->nd_gss_mb;				// the mbuf where we split
+	results = mbuf_next(mb);			// first mbuf in the results
+	reslen = nfs_gss_mchain_length(results);	// length of results
+	error = mbuf_setnext(mb, NULL);			// disconnect the chains
+	if (error)
+		return (error);
+	nfs_gss_nfsm_chain(nmc_pre, mb);		// set up the prepend chain
+
+	if (nd->nd_sec == RPCAUTH_KRB5I) {
+		nfsm_chain_add_32(error, nmc_pre, reslen);
+		nfsm_chain_build_done(error, nmc_pre);
+		if (error)
+			return (error);
+		nfs_gss_append_chain(nmc_pre, results);	// Append the results mbufs
+
+		/* Now compute the checksum over the results data */
+		nfs_gss_cksum_mchain(cp->gss_svc_sched, results, krb5_mic, 0, reslen, cksum);
+
+		/* Put it into a token and append to the request */
+		toklen = nfs_gss_token_put(cp->gss_svc_sched, krb5_mic, tokbuf, 0, 0, cksum);
+		nfsm_chain_add_32(error, nmc_res, toklen);
+		nfsm_chain_add_opaque(error, nmc_res, tokbuf, toklen);
+		nfsm_chain_build_done(error, nmc_res);
+	} else {
+		/* RPCAUTH_KRB5P */
+		/*
+		 * Append a pad trailer - per RFC 1964 section 1.2.2.3
+		 * Since XDR data is always 32-bit aligned, it
+		 * needs to be padded either by 4 bytes or 8 bytes.
+		 */
+		if (reslen % 8 > 0) {
+			nfsm_chain_add_32(error, nmc_res, 0x04040404);
+			reslen += NFSX_UNSIGNED;
+		} else {
+			nfsm_chain_add_32(error, nmc_res, 0x08080808);
+			nfsm_chain_add_32(error, nmc_res, 0x08080808);
+			reslen +=  2 * NFSX_UNSIGNED;
+		}
+		nfsm_chain_build_done(error, nmc_res);
+
+		/* Now compute the checksum over the results data */
+		nfs_gss_cksum_mchain(cp->gss_svc_sched, results, krb5_wrap, 0, reslen, cksum);
+
+		/* Put it into a token and insert in the reply */
+		toklen = nfs_gss_token_put(cp->gss_svc_sched, krb5_wrap, tokbuf, 0, reslen, cksum);
+		nfsm_chain_add_32(error, nmc_pre, toklen + reslen);
+		nfsm_chain_add_opaque_nopad(error, nmc_pre, tokbuf, toklen);
+		nfsm_chain_build_done(error, nmc_pre);
+		if (error)
+			return (error);
+		nfs_gss_append_chain(nmc_pre, results);	// Append the results mbufs
+
+		/* Encrypt the confounder + seqnum + results */
+		nfs_gss_encrypt_mchain(cp->gss_svc_skey, results, 0, reslen, DES_ENCRYPT);
+
+		/* Add null XDR pad if the ASN.1 token misaligned the data */
+		pad = nfsm_pad(toklen + reslen);
+		if (pad > 0) {
+			nfsm_chain_add_opaque_nopad(error, nmc_pre, iv0, pad);
+			nfsm_chain_build_done(error, nmc_pre);
+		}
+	}
+
+	return (error);
+}
+
+/*
+ * This function handles the context setup calls from the client.
+ * Essentially, it implements the NFS null procedure calls when
+ * an RPCSEC_GSS credential is used.
+ * This is the context maintenance function.  It creates and
+ * destroys server contexts at the whim of the client.
+ * During context creation, it receives GSS-API tokens from the
+ * client, passes them up to gssd, and returns a received token
+ * back to the client in the null procedure reply.
+ */
+int
+nfs_gss_svc_ctx_init(struct nfsrv_descript *nd, struct nfsrv_sock *slp, mbuf_t *mrepp)
+{
+	struct nfs_gss_svc_ctx *cp = NULL;
+	uint32_t handle = 0;
+	int error = 0;
+	int autherr = 0;
+	struct nfsm_chain *nmreq, nmrep;
+	int sz;
+
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	*mrepp = NULL;
+	cp = nd->nd_gss_context;
+	nd->nd_repstat = 0;
+
+	switch (cp->gss_svc_proc) {
+	case RPCSEC_GSS_INIT:
+		/*
+		 * Give the client a random handle so that
+		 * if we reboot it's unlikely the client
+		 * will get a bad context match.
+		 * Make sure it's not zero, or already assigned.
+		 */
+		do {
+			handle = random();
+		} while (nfs_gss_svc_ctx_find(handle) != NULL || handle == 0);
+		cp->gss_svc_handle = handle;
+		cp->gss_svc_mtx = lck_mtx_alloc_init(nfs_gss_svc_grp, LCK_ATTR_NULL);
+		clock_interval_to_deadline(GSS_CTX_PEND, NSEC_PER_SEC,
+			&cp->gss_svc_expiretime);
+
+		nfs_gss_svc_ctx_insert(cp);
+
+		/* FALLTHRU */
+
+	case RPCSEC_GSS_CONTINUE_INIT:
+		/* Get the token from the request */
+		nfsm_chain_get_32(error, nmreq, cp->gss_svc_tokenlen);
+		if (cp->gss_svc_tokenlen == 0) {
+			autherr = RPCSEC_GSS_CREDPROBLEM;
+			break;
+		}
+		MALLOC(cp->gss_svc_token, u_char *, cp->gss_svc_tokenlen, M_TEMP, M_WAITOK);
+		if (cp->gss_svc_token == NULL) {
+			autherr = RPCSEC_GSS_CREDPROBLEM;
+			break;
+		}
+		nfsm_chain_get_opaque(error, nmreq, cp->gss_svc_tokenlen, cp->gss_svc_token);
+
+		/* Use the token in a gss_accept_sec_context upcall */
+		error = nfs_gss_svc_gssd_upcall(cp);
+		if (error) {
+			autherr = RPCSEC_GSS_CREDPROBLEM;
+			if (error == EAUTH)
+				error = 0;
+			break;
+		}
+
+		/*
+		 * If the context isn't complete, pass the new token
+		 * back to the client for another round.
+		 */
+		if (cp->gss_svc_major != GSS_S_COMPLETE)
+			break;
+
+		/*
+		 * Now the server context is complete.
+		 * Finish setup.
+		 */
+		clock_interval_to_deadline(GSS_CTX_EXPIRE, NSEC_PER_SEC,
+			&cp->gss_svc_expiretime);
+		cp->gss_svc_seqwin = GSS_SVC_SEQWINDOW;
+		MALLOC(cp->gss_svc_seqbits, uint32_t *,
+			nfsm_rndup((cp->gss_svc_seqwin + 7) / 8), M_TEMP, M_WAITOK|M_ZERO);
+		if (cp->gss_svc_seqbits == NULL) {
+			autherr = RPCSEC_GSS_CREDPROBLEM;
+			break;
+		}
+
+		/*
+		 * Generate a key schedule from our shiny new DES key
+		 */
+		error = des_key_sched((des_cblock *) cp->gss_svc_skey, cp->gss_svc_sched);
+		if (error) {
+			autherr = RPCSEC_GSS_CREDPROBLEM;
+			error = 0;
+			break;
+		}
+		break;
+
+	case RPCSEC_GSS_DATA:
+		/* Just a nullproc ping - do nothing */
+		break;
+
+	case RPCSEC_GSS_DESTROY:
+		/*
+		 * Don't destroy the context immediately because
+		 * other active requests might still be using it.
+		 * Instead, schedule it for destruction after
+		 * GSS_CTX_PEND time has elapsed.
+		 */
+		cp = nfs_gss_svc_ctx_find(cp->gss_svc_handle);
+		if (cp != NULL) {
+			cp->gss_svc_handle = 0;	// so it can't be found
+			lck_mtx_lock(cp->gss_svc_mtx);
+			clock_interval_to_deadline(GSS_CTX_PEND, NSEC_PER_SEC,
+				&cp->gss_svc_expiretime);
+			lck_mtx_unlock(cp->gss_svc_mtx);
+		}
+		break;
+	default:
+		autherr = RPCSEC_GSS_CREDPROBLEM;
+		break;
+	}
+
+	/* Now build the reply  */
+
+	if (nd->nd_repstat == 0)
+		nd->nd_repstat = autherr ? (NFSERR_AUTHERR | autherr) : NFSERR_RETVOID;
+	sz = 7 * NFSX_UNSIGNED + nfsm_rndup(cp->gss_svc_tokenlen); // size of results
+	error = nfsrv_rephead(nd, slp, &nmrep, sz);
+	*mrepp = nmrep.nmc_mhead;
+	if (error || autherr)
+		goto nfsmout;
+
+	if (cp->gss_svc_proc == RPCSEC_GSS_INIT ||
+	    cp->gss_svc_proc == RPCSEC_GSS_CONTINUE_INIT) {
+		nfsm_chain_add_32(error, &nmrep, sizeof(cp->gss_svc_handle));
+		nfsm_chain_add_32(error, &nmrep, cp->gss_svc_handle);
+	
+		nfsm_chain_add_32(error, &nmrep, cp->gss_svc_major);
+		nfsm_chain_add_32(error, &nmrep, cp->gss_svc_minor);
+		nfsm_chain_add_32(error, &nmrep, cp->gss_svc_seqwin);
+	
+		nfsm_chain_add_32(error, &nmrep, cp->gss_svc_tokenlen);
+		nfsm_chain_add_opaque(error, &nmrep, cp->gss_svc_token, cp->gss_svc_tokenlen);
+		if (cp->gss_svc_token != NULL) {
+			FREE(cp->gss_svc_token, M_TEMP);
+			cp->gss_svc_token = NULL;
+		}
+	}
+
+nfsmout:
+	if (autherr != 0) {
+		LIST_REMOVE(cp, gss_svc_entries);
+		if (cp->gss_svc_seqbits != NULL)
+			FREE(cp->gss_svc_seqbits, M_TEMP);
+		if (cp->gss_svc_token != NULL)
+			FREE(cp->gss_svc_token, M_TEMP);
+		lck_mtx_destroy(cp->gss_svc_mtx, nfs_gss_svc_grp);
+		FREE(cp, M_TEMP);
+	}
+
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
+	return (error);
+}
+
+/*
+ * This is almost a mirror-image of the client side upcall.
+ * It passes and receives a token, but invokes gss_accept_sec_context.
+ * If it's the final call of the context setup, then gssd also returns
+ * the session key and the user's UID.
+ */
+static int
+nfs_gss_svc_gssd_upcall(struct nfs_gss_svc_ctx *cp)
+{
+	kern_return_t kr;
+	mach_port_t mp;
+	int retry_cnt = 0;
+	byte_buffer okey = NULL;
+	uint32_t skeylen = 0;
+	vm_map_copy_t itoken = NULL;
+	byte_buffer otoken = NULL;
+	int error = 0;
+	char svcname[] = "nfs";
+
+	kr = task_get_gssd_port(get_threadtask(current_thread()), &mp);
+	if (kr != KERN_SUCCESS) {
+		printf("nfs_gss_svc_gssd_upcall: can't get gssd port, status 0x%08x\n", kr);
+		return (EAUTH);
+	}
+	if (!IPC_PORT_VALID(mp)) {
+		printf("nfs_gss_svc_gssd_upcall: gssd port not valid\n");
+		return (EAUTH);
+	}
+
+	if (cp->gss_svc_tokenlen > 0)
+		nfs_gss_mach_alloc_buffer(cp->gss_svc_token, cp->gss_svc_tokenlen, &itoken);
+
+retry:
+	kr = mach_gss_accept_sec_context(
+		mp,
+		(byte_buffer) itoken, (mach_msg_type_number_t) cp->gss_svc_tokenlen,
+		svcname,
+		0,
+		&cp->gss_svc_gssd_verf,
+		&cp->gss_svc_context,
+		&cp->gss_svc_cred_handle,
+		&cp->gss_svc_uid,
+		cp->gss_svc_gids,
+		&cp->gss_svc_ngroups,
+		&okey, (mach_msg_type_number_t *) &skeylen,
+		&otoken, (mach_msg_type_number_t *) &cp->gss_svc_tokenlen,
+		&cp->gss_svc_major,
+		&cp->gss_svc_minor);
+
+	if (kr != KERN_SUCCESS) { 
+		printf("nfs_gss_svc_gssd_upcall failed: %d\n", kr);
+		if (kr == MIG_SERVER_DIED && cp->gss_svc_context == 0 &&
+			retry_cnt++ < NFS_GSS_MACH_MAX_RETRIES)
+			goto retry;
+		task_release_special_port(mp);
+		return (EAUTH);
+	}
+
+	task_release_special_port(mp);
+	if (skeylen > 0) {
+		if (skeylen != SKEYLEN) {
+			printf("nfs_gss_svc_gssd_upcall: bad key length (%d)\n", skeylen);
+			return (EAUTH);
+		}
+		error = nfs_gss_mach_vmcopyout((vm_map_copy_t) okey, skeylen, cp->gss_svc_skey);
+		if (error)
+			return (EAUTH);
+	}
+
+	if (cp->gss_svc_tokenlen > 0) {
+		MALLOC(cp->gss_svc_token, u_char *, cp->gss_svc_tokenlen, M_TEMP, M_WAITOK);
+		if (cp->gss_svc_token == NULL)
+			return (ENOMEM);
+		error = nfs_gss_mach_vmcopyout((vm_map_copy_t) otoken, cp->gss_svc_tokenlen,
+			cp->gss_svc_token);
+		if (error)
+			return (EAUTH);
+	}
+
+	return (kr);
+}
+
+/*
+ * Validate the sequence number in the credential as described
+ * in RFC 2203 Section 5.3.3.1
+ *
+ * Here the window of valid sequence numbers is represented by
+ * a bitmap.  As each sequence number is received, its bit is
+ * set in the bitmap.  An invalid sequence number lies below
+ * the lower bound of the window, or is within the window but
+ * has its bit already set.
+ */
+static int
+nfs_gss_svc_seqnum_valid(struct nfs_gss_svc_ctx *cp, uint32_t seq)
+{
+	uint32_t *bits = cp->gss_svc_seqbits;
+	uint32_t win = cp->gss_svc_seqwin;
+	uint32_t i;
+
+	lck_mtx_lock(cp->gss_svc_mtx);
+
+	/*
+	 * If greater than the window upper bound,
+	 * move the window up, and set the bit.
+	 */
+	if (seq > cp->gss_svc_seqmax) {
+		if (seq - cp->gss_svc_seqmax > win)
+			bzero(bits, nfsm_rndup((win + 7) / 8));
+		else
+			for (i = cp->gss_svc_seqmax + 1; i < seq; i++)
+				win_resetbit(bits, i % win);
+		win_setbit(bits, seq % win);
+		cp->gss_svc_seqmax = seq;
+		lck_mtx_unlock(cp->gss_svc_mtx);
+		return (1);
+	}
+
+	/*
+	 * Invalid if below the lower bound of the window
+	 */
+	if (seq <= cp->gss_svc_seqmax - win) {
+		lck_mtx_unlock(cp->gss_svc_mtx);
+		return (0);
+	}
+
+	/*
+	 * In the window, invalid if the bit is already set
+	 */
+	if (win_getbit(bits, seq % win)) {
+		lck_mtx_unlock(cp->gss_svc_mtx);
+		return (0);
+	}
+	win_setbit(bits, seq % win);
+	lck_mtx_unlock(cp->gss_svc_mtx);
+	return (1);
+}
+
+/*
+ * Called at NFS server shutdown - destroy all contexts
+ */
+void
+nfs_gss_svc_cleanup(void)
+{
+	struct nfs_gss_svc_ctx_hashhead *head;
+	struct nfs_gss_svc_ctx *cp, *ncp;
+	int i;
+	
+	lck_mtx_lock(nfs_gss_svc_ctx_mutex);
+
+	/*
+	 * Run through all the buckets
+	 */
+	for (i = 0; i < SVC_CTX_HASHSZ; i++) {
+		/*
+		 * Remove and free all entries in the bucket
+		 */
+		head = &nfs_gss_svc_ctx_hashtbl[i];
+		LIST_FOREACH_SAFE(cp, head, gss_svc_entries, ncp) {
+			LIST_REMOVE(cp, gss_svc_entries);
+			if (cp->gss_svc_seqbits)
+				FREE(cp->gss_svc_seqbits, M_TEMP);
+			lck_mtx_destroy(cp->gss_svc_mtx, nfs_gss_svc_grp);
+			FREE(cp, M_TEMP);
+		}
+	}
+
+	lck_mtx_unlock(nfs_gss_svc_ctx_mutex);
+}
+
+#endif /* NFSSERVER */
+
+
+/*************
+ * The following functions are used by both client and server.
+ */
+
+/*
+ * Release a task special port that was obtained by task_get_special_port
+ * or one of its macros (task_get_gssd_port in this case).
+ * This really should be in a public kpi. 
+ */
+
+/* This should be in a public header if this routine is not */
+extern void ipc_port_release_send(ipc_port_t);
+extern ipc_port_t ipc_port_copy_send(ipc_port_t);
+
+static void
+task_release_special_port(mach_port_t mp)
+{
+
+	ipc_port_release_send(mp);
+}
+
+static mach_port_t
+task_copy_special_port(mach_port_t mp)
+{
+	return ipc_port_copy_send(mp);
+}
+
+/*
+ * The token that is sent and received in the gssd upcall
+ * has unbounded variable length.  Mach RPC does not pass
+ * the token in-line.  Instead it uses page mapping to handle
+ * these parameters.  This function allocates a VM buffer
+ * to hold the token for an upcall and copies the token
+ * (received from the client) into it.  The VM buffer is
+ * marked with a src_destroy flag so that the upcall will
+ * automatically de-allocate the buffer when the upcall is
+ * complete.
+ */
+static void
+nfs_gss_mach_alloc_buffer(u_char *buf, uint32_t buflen, vm_map_copy_t *addr)
+{
+	kern_return_t kr;
+	vm_offset_t kmem_buf;
+	vm_size_t tbuflen;
+
+	*addr = NULL;
+	if (buf == NULL || buflen == 0)
+		return;
+
+	tbuflen = round_page(buflen);
+	kr = vm_allocate(ipc_kernel_map, &kmem_buf, tbuflen, VM_FLAGS_ANYWHERE);
+	if (kr != 0) {
+		printf("nfs_gss_mach_alloc_buffer: vm_allocate failed\n");
+		return;
+	}
+
+	kr = vm_map_wire(ipc_kernel_map, vm_map_trunc_page(kmem_buf),
+		vm_map_round_page(kmem_buf + tbuflen),
+		VM_PROT_READ|VM_PROT_WRITE, FALSE);
+
+	bcopy(buf, (void *) kmem_buf, buflen);
+
+	kr = vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(kmem_buf),
+		vm_map_round_page(kmem_buf + tbuflen), FALSE);
+	if (kr != 0) {
+		printf("nfs_gss_mach_alloc_buffer: vm_map_unwire failed\n");
+		return;
+	}
+
+	kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t) kmem_buf,
+		(vm_map_size_t) buflen, TRUE, addr);
+	if (kr != 0) {
+		printf("nfs_gss_mach_alloc_buffer: vm_map_copyin failed\n");
+		return;
+	}
+
+	if (buflen != tbuflen)
+		kmem_free(ipc_kernel_map, kmem_buf + buflen, tbuflen - buflen);
+}
+
+/*
+ * Here we handle a token received from the gssd via an upcall.
+ * The received token resides in an allocate VM buffer.
+ * We copy the token out of this buffer to a chunk of malloc'ed
+ * memory of the right size, then de-allocate the VM buffer.
+ */
+static int
+nfs_gss_mach_vmcopyout(vm_map_copy_t in, uint32_t len, u_char *out)
+{
+	vm_map_offset_t map_data;
+	vm_offset_t data;
+	int error;
+
+	error = vm_map_copyout(ipc_kernel_map, &map_data, in);
+	if (error)
+		return (error);
+
+	data = CAST_DOWN(vm_offset_t, map_data);
+	bcopy((void *) data, out, len);
+	vm_deallocate(ipc_kernel_map, data, len);
+
+	return (0);
+}
+
+/*
+ * Encode an ASN.1 token to be wrapped in an RPCSEC_GSS verifier.
+ * Returns the size of the token, since it contains a variable
+ * length DER encoded size field.
+ */
+static int
+nfs_gss_token_put(
+	des_key_schedule sched,
+	u_char *alg,
+	u_char *p,
+	int initiator,
+	int datalen,
+	u_char *cksum)
+{
+	static uint32_t seqnum = 0;
+	u_char *psave = p;
+	u_char plain[8];
+	int toklen, i;
+
+	/*
+	 * Fill in the token header: 2 octets.
+	 * This is 0x06 - an ASN.1 tag for APPLICATION, 0, SEQUENCE
+	 * followed by the length of the token: 35 + 0 octets for a
+	 * MIC token, or 35 + encrypted octets for a wrap token;
+	 */
+	*p++ = 0x060;
+	toklen = KRB5_SZ_MECH + KRB5_SZ_ALG + KRB5_SZ_SEQ + KRB5_SZ_CKSUM;
+	nfs_gss_der_length_put(&p, toklen + datalen);
+
+	/*
+	 * Fill in the DER encoded mech OID for Kerberos v5.
+	 * This represents the Kerberos OID 1.2.840.113554.1.2.2
+	 * described in RFC 2623, section 4.2
+	 */
+	bcopy(krb5_mech, p, sizeof(krb5_mech));
+	p += sizeof(krb5_mech);
+
+	/*
+	 * Now at the token described in RFC 1964, section 1.2.1
+	 * Fill in the token ID, integrity algorithm indicator,
+	 * for DES MAC MD5, and four filler octets.
+	 * The alg string encodes the bytes to represent either
+	 * a MIC token or a WRAP token for Kerberos.
+	 */
+	bcopy(alg, p, KRB5_SZ_ALG);
+	p += KRB5_SZ_ALG;
+
+	/*
+	 * Now encode the sequence number according to
+	 * RFC 1964, section 1.2.1.2 which dictates 4 octets
+	 * of sequence number followed by 4 bytes of direction
+	 * indicator: 0x00 for initiator or 0xff for acceptor.
+	 * We DES CBC encrypt the sequence number using the first
+	 * 8 octets of the checksum field as an initialization
+	 * vector.
+	 * Note that this sequence number is not at all related
+	 * to the RPCSEC_GSS protocol sequence number.  This
+	 * number is private to the ASN.1 token.  The only
+	 * requirement is that it not be repeated in case the
+	 * server has replay detection on, which normally should
+	 * not be the case, since RFC 2203 section 5.2.3 says that
+	 * replay detection and sequence checking must be turned off.
+	 */
+	seqnum++;
+	for (i = 0; i < 4; i++)
+		plain[i] = (u_char) ((seqnum >> (i * 8)) & 0xff);
+	for (i = 4; i < 8; i++)
+		plain[i] = initiator ? 0x00 : 0xff;
+	des_cbc_encrypt((des_cblock *) plain, (des_cblock *) p, 8,
+		sched, (des_cblock *) cksum, NULL, DES_ENCRYPT);
+	p += 8;
+
+	/*
+	 * Finally, append 8 octets of DES MAC MD5
+	 * checksum of the alg + plaintext data.
+	 * The plaintext could be an RPC call header,
+	 * the window value, or a sequence number.
+	 */
+	bcopy(cksum, p, 8);
+	p += 8;
+
+	return (p - psave);
+}
+
+/*
+ * Determine size of ASN.1 DER length
+ */
+static int
+nfs_gss_der_length_size(int len)
+{
+	return
+		len < (1 <<  7) ? 1 :
+		len < (1 <<  8) ? 2 :
+		len < (1 << 16) ? 3 :
+		len < (1 << 24) ? 4 : 5;
+}
+
+/*
+ * Encode an ASN.1 DER length field
+ */
+static void
+nfs_gss_der_length_put(u_char **pp, int len)
+{
+	int sz = nfs_gss_der_length_size(len);
+	u_char *p = *pp;
+
+	if (sz == 1) {
+		*p++ = (u_char) len;
+	} else {
+		*p++ = (u_char) ((sz-1) | 0x80);
+		sz -= 1;
+		while (sz--)
+			*p++ = (u_char) ((len >> (sz * 8)) & 0xff);
+	}
+
+	*pp = p;
+}
+
+/*
+ * Decode an ASN.1 DER length field
+ */
+static int
+nfs_gss_der_length_get(u_char **pp)
+{
+	u_char *p = *pp;
+	uint32_t flen, len = 0;
+
+	flen = *p & 0x7f;
+
+	if ((*p++ & 0x80) == 0)
+		len = flen;
+	else {
+		if (flen > sizeof(uint32_t))
+			return (-1);
+		while (flen--)
+			len = (len << 8) + *p++;
+	}
+	*pp = p;
+	return (len);
+}
+
+/*
+ * Decode an ASN.1 token from an RPCSEC_GSS verifier.
+ */
+static int
+nfs_gss_token_get(
+	des_key_schedule sched,
+	u_char *alg,
+	u_char *p,
+	int initiator,
+	uint32_t *len,
+	u_char *cksum)
+{
+	u_char d, plain[8];
+	u_char *psave = p;
+	int seqnum, i;
+
+	/*
+	 * Check that we have a valid token header
+	 */
+	if (*p++ != 0x60)
+		return (AUTH_BADCRED);
+	(void) nfs_gss_der_length_get(&p);	// ignore the size
+
+	/*
+	 * Check that we have the DER encoded Kerberos v5 mech OID
+	 */
+	if (bcmp(p, krb5_mech, sizeof(krb5_mech) != 0))
+		return (AUTH_BADCRED);
+	p += sizeof(krb5_mech);
+
+	/*
+	 * Now check the token ID, DES MAC MD5 algorithm
+	 * indicator, and filler octets.
+	 */
+	if (bcmp(p, alg, KRB5_SZ_ALG) != 0)
+		return (AUTH_BADCRED);
+	p += KRB5_SZ_ALG;
+
+	/*
+	 * Now decrypt the sequence number.
+	 * Note that the DES CBC decryption uses the first 8 octets
+	 * of the checksum field as an initialization vector (p + 8).
+	 * Per RFC 2203 section 5.2.2 we don't check the sequence number
+	 * in the ASN.1 token because the RPCSEC_GSS protocol has its
+	 * own sequence number described in section 5.3.3.1
+	 */
+	seqnum = 0;
+	des_cbc_encrypt((des_cblock *) p, (des_cblock *) plain, 8,
+		sched, (des_cblock *) (p + 8), NULL, DES_DECRYPT);
+	p += 8;
+	for (i = 0; i < 4; i++)
+		seqnum |= plain[i] << (i * 8);
+
+	/*
+	 * Make sure the direction
+	 * indicator octets are correct.
+	 */
+	d = initiator ? 0x00 : 0xff;
+	for (i = 4; i < 8; i++)
+		if (plain[i] != d)
+			return (AUTH_BADCRED);
+
+	/*
+	 * Finally, get the checksum
+	 */
+	bcopy(p, cksum, 8);
+	p += 8;
+
+	if (len != NULL)
+		*len = p - psave;
+
+	return (0);
+}
+
+/*
+ * Return the number of bytes in an mbuf chain.
+ */
+static int
+nfs_gss_mchain_length(mbuf_t mhead)
+{
+	mbuf_t mb;
+	int len = 0;
+
+	for (mb = mhead; mb; mb = mbuf_next(mb))
+		len += mbuf_len(mb);
+
+	return (len);
+}
+
+/*
+ * Append an args or results mbuf chain to the header chain
+ */
+static int
+nfs_gss_append_chain(struct nfsm_chain *nmc, mbuf_t mc)
+{
+	int error = 0;
+	mbuf_t mb, tail;
+
+	/* Connect the mbuf chains */
+	error = mbuf_setnext(nmc->nmc_mcur, mc);
+	if (error)
+		return (error);
+
+	/* Find the last mbuf in the chain */
+	tail = NULL;
+	for (mb = mc; mb; mb = mbuf_next(mb))
+		tail = mb;
+
+	nmc->nmc_mcur = tail;
+	nmc->nmc_ptr = (caddr_t) mbuf_data(tail) + mbuf_len(tail);
+	nmc->nmc_left = mbuf_trailingspace(tail);
+
+	return (0);
+}
+
+/*
+ * Convert an mbuf chain to an NFS mbuf chain
+ */
+static void
+nfs_gss_nfsm_chain(struct nfsm_chain *nmc, mbuf_t mc)
+{
+	mbuf_t mb, tail;
+
+	/* Find the last mbuf in the chain */
+	tail = NULL;
+	for (mb = mc; mb; mb = mbuf_next(mb))
+		tail = mb;
+
+	nmc->nmc_mhead = mc;
+	nmc->nmc_mcur = tail;
+	nmc->nmc_ptr = (caddr_t) mbuf_data(tail) + mbuf_len(tail);
+	nmc->nmc_left = mbuf_trailingspace(tail);
+	nmc->nmc_flags = 0;
+}
+
+
+/*
+ * Compute a checksum over an mbuf chain.
+ * Start building an MD5 digest at the given offset and keep
+ * going until the end of data in the current mbuf is reached.
+ * Then convert the 16 byte MD5 digest to an 8 byte DES CBC
+ * checksum.
+ */
+static void
+nfs_gss_cksum_mchain(
+	des_key_schedule sched,
+	mbuf_t mhead,
+	u_char *alg,
+	int offset,
+	int len,
+	u_char *cksum)
+{
+	mbuf_t mb;
+	u_char *ptr;
+	int left, bytes;
+	MD5_CTX context;
+	u_char digest[16];
+
+	MD5Init(&context);
+
+	/*
+	 * Logically prepend the first 8 bytes of the algorithm
+	 * field as required by RFC 1964, section 1.2.1.1
+	 */
+	MD5Update(&context, alg, KRB5_SZ_ALG);
+
+	/*
+	 * Move down the mbuf chain until we reach the given
+	 * byte offset, then start MD5 on the mbuf data until
+	 * we've done len bytes.
+	 */
+
+	for (mb = mhead; mb && len > 0; mb = mbuf_next(mb)) {
+		ptr  = mbuf_data(mb);
+		left = mbuf_len(mb);
+		if (offset >= left) {
+			/* Offset not yet reached */
+			offset -= left;
+			continue;
+		}
+		/* At or beyond offset - checksum data */
+		ptr += offset;
+		left -= offset;
+		offset = 0;
+			
+		bytes = left < len ? left : len;
+		if (bytes > 0)
+			MD5Update(&context, ptr, bytes);
+		len -= bytes;
+	}
+
+	MD5Final(digest, &context);
+
+	/*
+	 * Now get the DES CBC checksum for the digest.
+	 */
+	(void) des_cbc_cksum((des_cblock *) digest, (des_cblock *) cksum,
+		sizeof(digest), sched, (des_cblock *) iv0);
+}
+
+/*
+ * Compute a checksum over an NFS mbuf chain.
+ * Start building an MD5 digest at the given offset and keep
+ * going until the end of data in the current mbuf is reached.
+ * Then convert the 16 byte MD5 digest to an 8 byte DES CBC
+ * checksum.
+ */
+static void
+nfs_gss_cksum_chain(
+	des_key_schedule sched,
+	struct nfsm_chain *nmc,
+	u_char *alg,
+	int offset,
+	int len,
+	u_char *cksum)
+{
+	/*
+	 * If the length parameter is zero, then we need
+	 * to use the length from the offset to the current
+	 * encode/decode offset.
+	 */
+	if (len == 0)
+		len = nfsm_chain_offset(nmc) - offset;
+
+	return (nfs_gss_cksum_mchain(sched, nmc->nmc_mhead, alg, offset, len, cksum));
+}
+
+/*
+ * Compute a checksum of the sequence number (or sequence window)
+ * of an RPCSEC_GSS reply.
+ */
+static void
+nfs_gss_cksum_rep(des_key_schedule sched, uint32_t seqnum, u_char *cksum)
+{
+	MD5_CTX context;
+	u_char digest[16];
+	uint32_t val = htonl(seqnum);
+
+	MD5Init(&context);
+
+	/*
+	 * Logically prepend the first 8 bytes of the MIC
+	 * token as required by RFC 1964, section 1.2.1.1
+	 */
+	MD5Update(&context, krb5_mic, KRB5_SZ_ALG);
+
+	/*
+	 * Compute the digest of the seqnum in network order
+	 */
+	MD5Update(&context, (u_char *) &val, 4);
+	MD5Final(digest, &context);
+
+	/*
+	 * Now get the DES CBC checksum for the digest.
+	 */
+	(void) des_cbc_cksum((des_cblock *) digest, (des_cblock *) cksum,
+		sizeof(digest), sched, (des_cblock *) iv0);
+}
+
+/*
+ * Encrypt or decrypt data in an mbuf chain with des-cbc.
+ */
+static void
+nfs_gss_encrypt_mchain(
+	u_char *key,
+	mbuf_t mhead,
+	int offset,
+	int len,
+	int encrypt)
+{
+	des_key_schedule sched;
+	mbuf_t mb, mbn;
+	u_char *ptr, *nptr;
+	u_char tmp[8], ivec[8];
+	int i, left, left8, remain;
+
+	/*
+	 * Make the key schedule per RFC 1964 section 1.2.2.3
+	 */
+	for (i = 0; i < 8; i++)
+		tmp[i] = key[i] ^ 0xf0;
+	bzero(ivec, 8);
+
+	(void) des_key_sched((des_cblock *) tmp, sched);
+
+	/*
+	 * Move down the mbuf chain until we reach the given
+	 * byte offset, then start encrypting the mbuf data until
+	 * we've done len bytes.
+	 */
+
+	for (mb = mhead; mb && len > 0; mb = mbn) {
+		mbn  = mbuf_next(mb);
+		ptr  = mbuf_data(mb);
+		left = mbuf_len(mb);
+		if (offset >= left) {
+			/* Offset not yet reached */
+			offset -= left;
+			continue;
+		}
+		/* At or beyond offset - encrypt data */
+		ptr += offset;
+		left -= offset;
+		offset = 0;
+			
+		/*
+		 * DES CBC has to encrypt 8 bytes at a time.
+		 * If the number of bytes to be encrypted in this
+		 * mbuf isn't some multiple of 8 bytes, encrypt all
+		 * the 8 byte blocks, then combine the remaining
+		 * bytes with enough from the next mbuf to make up
+		 * an 8 byte block and encrypt that block separately,
+		 * i.e. that block is split across two mbufs.
+		 */
+		remain = left % 8;
+		left8 = left - remain;
+		left = left8 < len ? left8 : len;
+		if (left > 0) {
+			des_cbc_encrypt((des_cblock *) ptr, (des_cblock *) ptr, left, sched,
+				(des_cblock *) ivec, (des_cblock *) ivec, encrypt);
+			len -= left;
+		}
+
+		if (mbn && remain > 0) {
+			nptr = mbuf_data(mbn);
+			offset = 8 - remain;
+			bcopy(ptr + left, tmp, remain);		// grab from this mbuf
+			bcopy(nptr, tmp + remain, offset);	// grab from next mbuf
+			des_cbc_encrypt((des_cblock *) tmp, (des_cblock *) tmp, 8, sched,
+				(des_cblock *) ivec, (des_cblock *) ivec, encrypt);
+			bcopy(tmp, ptr + left, remain);		// return to this mbuf
+			bcopy(tmp + remain, nptr, offset);	// return to next mbuf
+			len -= 8;
+		}
+	}
+}
+
+/*
+ * Encrypt or decrypt data in an NFS mbuf chain with des-cbc.
+ */
+static void
+nfs_gss_encrypt_chain(
+	u_char *key,
+	struct nfsm_chain *nmc,
+	int offset,
+	int len,
+	int encrypt)
+{
+	/*
+	 * If the length parameter is zero, then we need
+	 * to use the length from the offset to the current
+	 * encode/decode offset.
+	 */
+	if (len == 0)
+		len = nfsm_chain_offset(nmc) - offset;
+
+	return (nfs_gss_encrypt_mchain(key, nmc->nmc_mhead, offset, len, encrypt));
+}
+
+/*
+ * XXX This function borrowed from OpenBSD.
+ * It will likely be moved into kernel crypto.
+ */
+static DES_LONG
+des_cbc_cksum(input, output, length, schedule, ivec)
+	des_cblock (*input);
+	des_cblock (*output);
+	long length;
+	des_key_schedule schedule;
+	des_cblock (*ivec);
+{
+	register unsigned long tout0,tout1,tin0,tin1;
+	register long l=length;
+	unsigned long tin[2];
+	unsigned char *in,*out,*iv;
+
+	in=(unsigned char *)input;
+	out=(unsigned char *)output;
+	iv=(unsigned char *)ivec;
+
+	c2l(iv,tout0);
+	c2l(iv,tout1);
+	for (; l>0; l-=8) {
+		if (l >= 8) {
+			c2l(in,tin0);
+			c2l(in,tin1);
+		} else
+			c2ln(in,tin0,tin1,l);
+			
+		tin0^=tout0; tin[0]=tin0;
+		tin1^=tout1; tin[1]=tin1;
+		des_encrypt1((DES_LONG *)tin,schedule,DES_ENCRYPT);
+		/* fix 15/10/91 eay - thanks to keithr@sco.COM */
+		tout0=tin[0];
+		tout1=tin[1];
+	}
+	if (out != NULL) {
+		l2c(tout0,out);
+		l2c(tout1,out);
+	}
+	tout0=tin0=tin1=tin[0]=tin[1]=0;
+	return(tout1);
+}
+
+/*
+ * XXX This function borrowed from OpenBSD.
+ * It will likely be moved into kernel crypto.
+ */
+static void
+des_cbc_encrypt(input, output, length, schedule, ivec, retvec, encrypt)
+	des_cblock (*input);
+	des_cblock (*output);
+	long length;
+	des_key_schedule schedule;
+	des_cblock (*ivec);
+	des_cblock (*retvec);
+	int encrypt;
+{
+	register unsigned long tin0,tin1;
+	register unsigned long tout0,tout1,xor0,xor1;
+	register unsigned char *in,*out,*retval;
+	register long l=length;
+	unsigned long tin[2];
+	unsigned char *iv;
+	tin0 = tin1 = 0;
+
+	in=(unsigned char *)input;
+	out=(unsigned char *)output;
+	retval=(unsigned char *)retvec;
+	iv=(unsigned char *)ivec;
+
+	if (encrypt) {
+		c2l(iv,tout0);
+		c2l(iv,tout1);
+		for (l-=8; l>=0; l-=8) {
+			c2l(in,tin0);
+			c2l(in,tin1);
+			tin0^=tout0; tin[0]=tin0;
+			tin1^=tout1; tin[1]=tin1;
+			des_encrypt1((DES_LONG *)tin,schedule,DES_ENCRYPT);
+			tout0=tin[0]; l2c(tout0,out);
+			tout1=tin[1]; l2c(tout1,out);
+		}
+		if (l != -8) {
+			c2ln(in,tin0,tin1,l+8);
+			tin0^=tout0; tin[0]=tin0;
+			tin1^=tout1; tin[1]=tin1;
+			des_encrypt1((DES_LONG *)tin,schedule,DES_ENCRYPT);
+			tout0=tin[0]; l2c(tout0,out);
+			tout1=tin[1]; l2c(tout1,out);
+		}
+		if (retval) {
+			l2c(tout0,retval);
+			l2c(tout1,retval);
+		}
+	} else {
+		c2l(iv,xor0);
+		c2l(iv,xor1);
+		for (l-=8; l>=0; l-=8) {
+			c2l(in,tin0); tin[0]=tin0;
+			c2l(in,tin1); tin[1]=tin1;
+			des_encrypt1((DES_LONG *)tin,schedule,DES_DECRYPT);
+			tout0=tin[0]^xor0;
+			tout1=tin[1]^xor1;
+			l2c(tout0,out);
+			l2c(tout1,out);
+			xor0=tin0;
+			xor1=tin1;
+		}
+		if (l != -8) {
+			c2l(in,tin0); tin[0]=tin0;
+			c2l(in,tin1); tin[1]=tin1;
+			des_encrypt1((DES_LONG *)tin,schedule,DES_DECRYPT);
+			tout0=tin[0]^xor0;
+			tout1=tin[1]^xor1;
+			l2cn(tout0,tout1,out,l+8);
+		/*	xor0=tin0;
+			xor1=tin1; */
+		}
+		if (retval) {
+			l2c(tin0,retval);
+			l2c(tin1,retval);
+		}
+	}
+	tin0=tin1=tout0=tout1=xor0=xor1=0;
+	tin[0]=tin[1]=0;
+}
diff --git a/bsd/nfs/nfs_gss.h b/bsd/nfs/nfs_gss.h
new file mode 100644
index 000000000..a3536bd29
--- /dev/null
+++ b/bsd/nfs/nfs_gss.h
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _NFS_NFS_GSS_H_
+#define _NFS_NFS_GSS_H_
+
+#include <gssd/gssd_mach.h>
+#include <libkern/crypto/md5.h>
+#include <crypto/des/des_locl.h>
+#include <sys/param.h>
+
+#define RPCSEC_GSS			6
+#define	RPCSEC_GSS_VERS_1		1
+
+enum rpcsec_gss_proc {
+	RPCSEC_GSS_DATA			= 0,
+	RPCSEC_GSS_INIT			= 1,
+	RPCSEC_GSS_CONTINUE_INIT	= 2,
+	RPCSEC_GSS_DESTROY		= 3
+};
+
+enum rpcsec_gss_service {
+	RPCSEC_GSS_SVC_NONE		= 1,	// sec=krb5
+	RPCSEC_GSS_SVC_INTEGRITY	= 2,	// sec=krb5i
+	RPCSEC_GSS_SVC_PRIVACY		= 3,	// sec=krb5p
+	RPCSEC_GSS_SVC_SYS		= 4	// sec=sys (fallback)
+};
+
+/*
+ * GSS-API things
+ */
+#define GSS_S_COMPLETE			0
+#define GSS_S_CONTINUE_NEEDED		1
+
+#define GSS_MAXSEQ			0x80000000	// The biggest sequence number
+#define GSS_SVC_MAXCONTEXTS		500000		// Max contexts supported
+#define GSS_SVC_SEQWINDOW		256		// Server's sequence window
+#define GSS_CLNT_SEQLISTMAX		32		// Max length of req seq num list
+#define GSS_CLNT_SYS_VALID		300		// Valid time (sec) for failover ctx
+
+#define SKEYLEN	8			// length of DES key
+
+/*
+ * The client's RPCSEC_GSS context information
+ */
+struct nfs_gss_clnt_ctx {
+	lck_mtx_t		*gss_clnt_mtx;
+	thread_t		gss_clnt_thread;	// Thread creating context
+	TAILQ_ENTRY(nfs_gss_clnt_ctx)	gss_clnt_entries;
+	uint32_t		gss_clnt_flags;		// Flag bits - see below
+	uint32_t		gss_clnt_refcnt;	// Reference count
+	uid_t			gss_clnt_uid;		// Owner of this context
+	uint32_t		gss_clnt_proc;		// Current GSS proc for cred
+	uint32_t		gss_clnt_seqnum;	// GSS sequence number
+	uint32_t		gss_clnt_service;	// Indicates krb5, krb5i or krb5p
+	u_char			*gss_clnt_handle;	// Identifies server context
+	uint32_t		gss_clnt_handle_len;	// Size of server's ctx handle
+	time_t			gss_clnt_ctime;		// When context was created
+	uint32_t		gss_clnt_seqwin;	// Server's seq num window
+	uint32_t		*gss_clnt_seqbits;	// Bitmap to track seq numbers in use
+	mach_port_t		gss_clnt_mport;		// Mach port for gssd upcall
+	u_char			*gss_clnt_verf;		// RPC verifier from server
+	uint64_t		gss_clnt_gssd_verf;	// Verifier from gssd
+	char			*gss_clnt_svcname;	// Service name e.g. "nfs/big.apple.com"
+	uint32_t		gss_clnt_cred_handle;	// Opaque cred handle from gssd
+	uint32_t		gss_clnt_context;	// Opaque context handle from gssd
+	u_char			*gss_clnt_token;	// GSS token exchanged via gssd & server
+	uint32_t		gss_clnt_tokenlen;	// Length of token
+	u_char			gss_clnt_skey[SKEYLEN];	// Context session key (DES)
+	des_key_schedule	gss_clnt_sched;		// Schedule derived from key
+	uint32_t		gss_clnt_major;		// GSS major result from gssd or server
+	uint32_t		gss_clnt_minor;		// GSS minor result from gssd or server
+};
+
+/*
+ * gss_clnt_flags
+ */
+#define GSS_CTX_COMPLETE	0x00000001	// Context is complete
+#define GSS_CTX_INVAL		0x00000002	// Context is invalid
+#define GSS_NEEDSEQ		0x00000004	// Need a sequence number
+#define GSS_NEEDCTX		0x00000008	// Need the context
+
+/*
+ * The server's RPCSEC_GSS context information
+ */
+struct nfs_gss_svc_ctx {
+	lck_mtx_t		*gss_svc_mtx;
+	LIST_ENTRY(nfs_gss_svc_ctx)	gss_svc_entries;
+	uint32_t		gss_svc_handle;		// Identifies server context to client
+	uint32_t		gss_svc_proc;		// Current GSS proc from cred
+	uid_t			gss_svc_uid;		// UID of this user
+	gid_t			gss_svc_gids[NGROUPS];	// GIDs of this user
+	uint32_t		gss_svc_ngroups;	// Count of gids
+	uint64_t		gss_svc_expiretime;	// Delete ctx if we exceed this
+	uint32_t		gss_svc_seqmax;		// Current max GSS sequence number
+	uint32_t		gss_svc_seqwin;		// GSS sequence number window
+	uint32_t		*gss_svc_seqbits;	// Bitmap to track seq numbers
+	uint64_t		gss_svc_gssd_verf;	// Verifier from gssd
+	uint32_t		gss_svc_cred_handle;	// Opaque cred handle from gssd
+	uint32_t		gss_svc_context;	// Opaque context handle from gssd
+	u_char			*gss_svc_token;		// GSS token exchanged via gssd & client
+	uint32_t		gss_svc_tokenlen;	// Length of token
+	u_char			gss_svc_skey[SKEYLEN];	// Context session key (DES)
+	des_key_schedule	gss_svc_sched;		// Schedule derived from key
+	uint32_t		gss_svc_major;		// GSS major result from gssd
+	uint32_t		gss_svc_minor;		// GSS minor result from gssd
+};
+
+#define SVC_CTX_HASHSZ	64
+#define SVC_CTX_HASH(handle)	((handle) % SVC_CTX_HASHSZ)
+LIST_HEAD(nfs_gss_svc_ctx_hashhead, nfs_gss_svc_ctx);
+
+/*
+ * Macros to manipulate bits in the sequence window
+ */
+#define win_getbit(bits, bit)      ((bits[(bit) / 32] &   (1 << (bit) % 32)) != 0)
+#define win_setbit(bits, bit)   do { bits[(bit) / 32] |=  (1 << (bit) % 32); } while (0)
+#define win_resetbit(bits, bit) do { bits[(bit) / 32] &= ~(1 << (bit) % 32); } while (0)
+
+/*
+ * Server context stale times
+ */
+#define GSS_CTX_PEND		5 		// seconds
+#define GSS_CTX_EXPIRE		(8 * 3600)	// seconds
+#define GSS_TIMER_PERIOD	300		// seconds
+#define MSECS_PER_SEC		1000
+
+__BEGIN_DECLS
+
+void	nfs_gss_init(void);
+int	nfs_gss_clnt_cred_put(struct nfsreq *, struct nfsm_chain *, mbuf_t);
+int	nfs_gss_clnt_verf_get(struct nfsreq *, struct nfsm_chain *,
+		uint32_t, uint32_t, uint32_t *);
+void	nfs_gss_clnt_rpcdone(struct nfsreq *);
+int	nfs_gss_clnt_args_restore(struct nfsreq *);
+int	nfs_gss_clnt_ctx_renew(struct nfsreq *);
+void	nfs_gss_clnt_ctx_ref(struct nfsreq *, struct nfs_gss_clnt_ctx *);
+void	nfs_gss_clnt_ctx_unref(struct nfsreq *);
+void	nfs_gss_clnt_ctx_unmount(struct nfsmount *, int);
+int	nfs_gss_svc_cred_get(struct nfsrv_descript *, struct nfsm_chain *);
+int	nfs_gss_svc_verf_put(struct nfsrv_descript *, struct nfsm_chain *);
+int	nfs_gss_svc_ctx_init(struct nfsrv_descript *, struct nfsrv_sock *, mbuf_t *);
+int	nfs_gss_svc_prepare_reply(struct nfsrv_descript *, struct nfsm_chain *);
+int	nfs_gss_svc_protect_reply(struct nfsrv_descript *, mbuf_t);
+void	nfs_gss_svc_cleanup(void);
+
+__END_DECLS
+#endif /* _NFS_NFS_GSS_H_ */
diff --git a/bsd/nfs/nfs_lock.c b/bsd/nfs/nfs_lock.c
index 762c140b0..c767ae523 100644
--- a/bsd/nfs/nfs_lock.c
+++ b/bsd/nfs/nfs_lock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
@@ -68,6 +74,7 @@
 #include <sys/vnode_internal.h>
 
 #include <kern/thread.h>
+#include <kern/host.h>
 
 #include <machine/limits.h>
 
@@ -76,28 +83,26 @@
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/nfsnode.h>
 #include <nfs/nfs_lock.h>
 
-#define OFF_MAX QUAD_MAX
+#include <mach/host_priv.h>
+#include <mach/mig_errors.h>
+#include <mach/host_special_ports.h>
+#include <lockd/lockd_mach.h>
 
-/*
- * globals for managing the lockd fifo
- */
-vnode_t nfslockdvnode = 0;
-int nfslockdwaiting = 0;
-time_t nfslockdstarttimeout = 0;
-int nfslockdfifolock = 0;
-#define NFSLOCKDFIFOLOCK_LOCKED	1
-#define NFSLOCKDFIFOLOCK_WANT	2
+extern void ipc_port_release_send(ipc_port_t);
+
+#define OFF_MAX QUAD_MAX
 
 /*
  * pending lock request messages are kept in this queue which is
  * kept sorted by transaction ID (xid).
  */
-uint64_t nfs_lockxid = 0;
-LOCKD_MSG_QUEUE nfs_pendlockq;
+static uint64_t nfs_lockxid = 0;
+static LOCKD_MSG_QUEUE nfs_pendlockq;
 
 /*
  * This structure is used to identify processes which have acquired NFS locks.
@@ -117,10 +122,12 @@ struct nfs_lock_pid {
 #define NFS_LOCK_PID_HASH_SIZE		64	// XXX tune me
 #define	NFS_LOCK_PID_HASH(pid)	\
 	(&nfs_lock_pid_hash_tbl[(pid) & nfs_lock_pid_hash])
-LIST_HEAD(, nfs_lock_pid) *nfs_lock_pid_hash_tbl;
-TAILQ_HEAD(, nfs_lock_pid) nfs_lock_pid_lru;
-u_long nfs_lock_pid_hash;
-int nfs_lock_pid_lock;
+static LIST_HEAD(, nfs_lock_pid) *nfs_lock_pid_hash_tbl;
+static TAILQ_HEAD(, nfs_lock_pid) nfs_lock_pid_lru;
+static u_long nfs_lock_pid_hash, nfs_lock_pid_hash_trusted;
+
+static lck_grp_t *nfs_lock_lck_grp;
+static lck_mtx_t *nfs_lock_mutex;
 
 
 /*
@@ -130,14 +137,63 @@ void
 nfs_lockinit(void)
 {
 	TAILQ_INIT(&nfs_pendlockq);
-	nfs_lock_pid_lock = 0;
+	nfs_lock_pid_hash_trusted = 1;
 	nfs_lock_pid_hash_tbl = hashinit(NFS_LOCK_PID_HASH_SIZE,
 					 M_TEMP, &nfs_lock_pid_hash);
 	TAILQ_INIT(&nfs_lock_pid_lru);
+
+	nfs_lock_lck_grp = lck_grp_alloc_init("nfs_lock", LCK_GRP_ATTR_NULL);
+	nfs_lock_mutex = lck_mtx_alloc_init(nfs_lock_lck_grp, LCK_ATTR_NULL);
+}
+
+/*
+ * change the count of NFS mounts that may need to make lockd requests
+ *
+ * If the mount count drops to zero, then send a shutdown request to
+ * lockd if we've sent any requests to it.
+ */
+void
+nfs_lockd_mount_change(int i)
+{
+	mach_port_t lockd_port = IPC_PORT_NULL;
+	kern_return_t kr;
+	int send_shutdown;
+
+	lck_mtx_lock(nfs_lock_mutex);
+
+	nfs_lockd_mounts += i;
+
+	/* send a shutdown request if there are no more lockd mounts */
+	send_shutdown = ((nfs_lockd_mounts == 0) && nfs_lockd_request_sent);
+	if (send_shutdown)
+		nfs_lockd_request_sent = 0;
+
+	lck_mtx_unlock(nfs_lock_mutex);
+
+	if (!send_shutdown)
+		return;
+
+	/*
+	 * Let lockd know that it is no longer need for any NFS mounts
+	 */
+	kr = host_get_lockd_port(host_priv_self(), &lockd_port);
+	if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(lockd_port)) {
+		printf("nfs_lockd_mount_change: shutdown couldn't get port, kr %d, port %s\n",
+			kr, (lockd_port == IPC_PORT_NULL) ? "NULL" :
+			(lockd_port == IPC_PORT_DEAD) ? "DEAD" : "VALID");
+		return;
+	}
+
+	kr = lockd_shutdown(lockd_port);
+	if (kr != KERN_SUCCESS)
+		printf("nfs_lockd_mount_change: shutdown %d\n", kr);
+
+	ipc_port_release_send(lockd_port);
 }
 
 /*
  * insert a lock request message into the pending queue
+ * (nfs_lock_mutex must be held)
  */
 static inline void
 nfs_lockdmsg_enqueue(LOCKD_MSG_REQUEST *msgreq)
@@ -163,6 +219,7 @@ nfs_lockdmsg_enqueue(LOCKD_MSG_REQUEST *msgreq)
 
 /*
  * remove a lock request message from the pending queue
+ * (nfs_lock_mutex must be held)
  */
 static inline void
 nfs_lockdmsg_dequeue(LOCKD_MSG_REQUEST *msgreq)
@@ -179,6 +236,8 @@ nfs_lockdmsg_dequeue(LOCKD_MSG_REQUEST *msgreq)
  * However, this may not be the case if there are blocked requests.  We may
  * want to move blocked requests to a separate queue (but that'll complicate
  * duplicate xid checking).
+ *
+ * (nfs_lock_mutex must be held)
  */
 static inline LOCKD_MSG_REQUEST *
 nfs_lockdmsg_find_by_xid(uint64_t lockxid)
@@ -236,6 +295,8 @@ nfs_lockdmsg_compare_to_answer(LOCKD_MSG_REQUEST *msgreq, struct lockd_ans *ansp
  * However, this may not be the case if there are blocked requests.  We may
  * want to move blocked requests to a separate queue (but that'll complicate
  * duplicate xid checking).
+ *
+ * (nfs_lock_mutex must be held)
  */
 static inline LOCKD_MSG_REQUEST *
 nfs_lockdmsg_find_by_answer(struct lockd_ans *ansp)
@@ -253,6 +314,7 @@ nfs_lockdmsg_find_by_answer(struct lockd_ans *ansp)
 
 /*
  * return the next unique lock request transaction ID
+ * (nfs_lock_mutex must be held)
  */
 static inline uint64_t
 nfs_lockxid_get(void)
@@ -294,36 +356,30 @@ nfs_lockxid_get(void)
  * add the entry if it is not found.
  *
  * (Also, if adding, try to clean up some stale entries.)
+ * (nfs_lock_mutex must be held)
  */
 static int
-nfs_lock_pid_check(proc_t p, int addflag, vnode_t vp)
+nfs_lock_pid_check(proc_t p, int addflag)
 {
-	struct nfs_lock_pid *lp, *lplru, *lplru_next;
-	proc_t plru;
+	struct nfs_lock_pid *lp, *lplru, *lplru_next, *mlp;
+	TAILQ_HEAD(, nfs_lock_pid) nfs_lock_pid_free;
+	proc_t plru = PROC_NULL;
+	pid_t pid;
 	int error = 0;
 	struct timeval now;
 
-	/* lock hash */
-loop:
-	if (nfs_lock_pid_lock) {
-		struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-		while (nfs_lock_pid_lock) {
-			nfs_lock_pid_lock = -1;
-			tsleep(&nfs_lock_pid_lock, PCATCH, "nfslockpid", 0);
-			if ((error = nfs_sigintr(nmp, NULL, p)))
-				return (error);
-		}
-		goto loop;
-	}
-	nfs_lock_pid_lock = 1;
+	TAILQ_INIT(&nfs_lock_pid_free);
+	mlp = NULL;
 
+loop:
 	/* Search hash chain */
+	pid = proc_pid(p);
 	error = ENOENT;
-	lp = NFS_LOCK_PID_HASH(proc_pid(p))->lh_first;
+	lp = NFS_LOCK_PID_HASH(pid)->lh_first;
 	for (; lp != NULL; lp = lp->lp_hash.le_next)
-		if (lp->lp_pid == proc_pid(p)) {
+		if (lp->lp_pid == pid) {
 			/* found pid... */
-			if (timevalcmp(&lp->lp_pid_start, &p->p_stats->p_start, ==)) {
+			if (timevalcmp(&lp->lp_pid_start, &p->p_start, ==)) {
 				/* ...and it's valid */
 				/* move to tail of LRU */
 				TAILQ_REMOVE(&nfs_lock_pid_lru, lp, lp_lru);
@@ -343,8 +399,12 @@ loop:
 			break;
 		}
 
-	/* if we didn't find it (valid) and we've been asked to add it */
-	if ((error == ENOENT) && addflag) {
+	/* if we didn't find it (valid), use any newly allocated one */
+	if (!lp)
+		lp = mlp;
+
+	/* if we don't have an lp and we've been asked to add it */
+	if ((error == ENOENT) && addflag && !lp) {
 		/* scan lru list for invalid, stale entries to reuse/free */
 		int lrucnt = 0;
 		microuptime(&now);
@@ -359,19 +419,27 @@ loop:
 			}
 			/* remove entry from LRU, and check if it's still in use */
 			TAILQ_REMOVE(&nfs_lock_pid_lru, lplru, lp_lru);
-			if (!lplru->lp_valid || !(plru = pfind(lplru->lp_pid)) ||
-			    timevalcmp(&lplru->lp_pid_start, &plru->p_stats->p_start, !=)) {
+			if (!lplru->lp_valid || !(plru = proc_find(lplru->lp_pid)) ||
+			    timevalcmp(&lplru->lp_pid_start, &plru->p_start, !=)) {
+				if (plru != PROC_NULL) {
+					proc_rele(plru);
+					plru = PROC_NULL;
+				}
 				/* no longer in use */
 				LIST_REMOVE(lplru, lp_hash);
 				if (!lp) {
 					/* we'll reuse this one */
 					lp = lplru;
 				} else {
-					/* we can free this one */
-					FREE(lplru, M_TEMP);
+					/* queue it up for freeing */
+					TAILQ_INSERT_HEAD(&nfs_lock_pid_free, lplru, lp_lru);
 				}
 			} else {
 				/* still in use */
+				if (plru != PROC_NULL) {
+					proc_rele(plru);
+					plru = PROC_NULL;
+				}
 				lplru->lp_time = now.tv_sec;
 				TAILQ_INSERT_TAIL(&nfs_lock_pid_lru, lplru, lp_lru);
 			}
@@ -381,80 +449,144 @@ loop:
 		}
 		if (!lp) {
 			/* we need to allocate a new one */
-			MALLOC(lp, struct nfs_lock_pid *, sizeof(struct nfs_lock_pid),
+			lck_mtx_unlock(nfs_lock_mutex);
+			MALLOC(mlp, struct nfs_lock_pid *, sizeof(struct nfs_lock_pid),
 				M_TEMP, M_WAITOK | M_ZERO);
-		}
-		if (!lp) {
+			lck_mtx_lock(nfs_lock_mutex);
+			if (mlp) /* make sure somebody hasn't already added this guy */
+				goto loop;
 			error = ENOMEM;
-		} else {
-			/* (re)initialize nfs_lock_pid info */
-			lp->lp_pid = proc_pid(p);
-			lp->lp_pid_start = p->p_stats->p_start;
-			/* insert pid in hash */
-			LIST_INSERT_HEAD(NFS_LOCK_PID_HASH(lp->lp_pid), lp, lp_hash);
-			lp->lp_valid = 1;
-			lp->lp_time = now.tv_sec;
-			TAILQ_INSERT_TAIL(&nfs_lock_pid_lru, lp, lp_lru);
-			error = 0;
 		}
 	}
+	if ((error == ENOENT) && addflag && lp) {
+		/* (re)initialize nfs_lock_pid info */
+		lp->lp_pid = pid;
+		lp->lp_pid_start = p->p_start;
+		/* insert pid in hash */
+		LIST_INSERT_HEAD(NFS_LOCK_PID_HASH(lp->lp_pid), lp, lp_hash);
+		lp->lp_valid = 1;
+		lp->lp_time = now.tv_sec;
+		TAILQ_INSERT_TAIL(&nfs_lock_pid_lru, lp, lp_lru);
+		error = 0;
+	}
 
-	/* unlock hash */
-	if (nfs_lock_pid_lock < 0) {
-		nfs_lock_pid_lock = 0;
-		wakeup(&nfs_lock_pid_lock);
-	} else
-		nfs_lock_pid_lock = 0;
+	if ((mlp && (lp != mlp)) || TAILQ_FIRST(&nfs_lock_pid_free)) {
+		lck_mtx_unlock(nfs_lock_mutex);
+		if (mlp && (lp != mlp)) {
+			/* we didn't need this one, so we can free it */
+			FREE(mlp, M_TEMP);
+		}
+		/* free up any stale entries */
+		while ((lp = TAILQ_FIRST(&nfs_lock_pid_free))) {
+			TAILQ_REMOVE(&nfs_lock_pid_free, lp, lp_lru);
+			FREE(lp, M_TEMP);
+		}
+		lck_mtx_lock(nfs_lock_mutex);
+	}
 
 	return (error);
 }
 
+#define MACH_MAX_TRIES 3
+
+static int
+send_request(LOCKD_MSG *msg, int interruptable)
+{
+	kern_return_t kr;
+	int retries = 0;
+	mach_port_t lockd_port = IPC_PORT_NULL;
+
+	kr = host_get_lockd_port(host_priv_self(), &lockd_port);
+	if (kr != KERN_SUCCESS || !IPC_PORT_VALID(lockd_port))
+		return (ENOTSUP);
+
+	do {
+		/* In the kernel all mach messaging is interruptable */
+		do {
+			kr = lockd_request(
+				lockd_port,
+				msg->lm_version,
+				msg->lm_flags,
+				msg->lm_xid,
+				msg->lm_fl.l_start,
+				msg->lm_fl.l_len,
+				msg->lm_fl.l_pid,
+				msg->lm_fl.l_type,
+				msg->lm_fl.l_whence,
+				(uint32_t *)&msg->lm_addr,
+				(uint32_t *)&msg->lm_cred,
+				msg->lm_fh_len,
+				msg->lm_fh);
+			if (kr != KERN_SUCCESS)
+				printf("lockd_request received %d!\n", kr);
+		} while (!interruptable && kr == MACH_SEND_INTERRUPTED);
+	} while (kr == MIG_SERVER_DIED && retries++ < MACH_MAX_TRIES);
+
+	ipc_port_release_send(lockd_port);
+	switch (kr) {
+	case MACH_SEND_INTERRUPTED: 
+		return (EINTR);
+	default:
+		/*
+		 * Other MACH or MIG errors we will retry. Eventually
+		 * we will call nfs_down and allow the user to disable 
+		 * locking.
+		 */
+		return (EAGAIN);
+	}
+	return (kr);
+}
+				
 
 /*
- * nfs_advlock --
- *      NFS advisory byte-level locks.
+ * NFS advisory byte-level locks (client)
  */
 int
-nfs_dolock(struct vnop_advlock_args *ap)
-/* struct vnop_advlock_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	caddr_t a_id;
-	int a_op;
-	struct flock *a_fl;
-	int a_flags;
-	vfs_context_t a_context;
-}; */
+nfs3_vnop_advlock(
+	struct vnop_advlock_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_vp;
+		caddr_t a_id;
+		int a_op;
+		struct flock *a_fl;
+		int a_flags;
+		vfs_context_t a_context;
+	} */ *ap)
 {
+	vfs_context_t ctx;
+	proc_t p;
 	LOCKD_MSG_REQUEST msgreq;
 	LOCKD_MSG *msg;
-	vnode_t vp, wvp;
-	struct nfsnode *np;
-	int error, error1;
+	vnode_t vp;
+	nfsnode_t np;
+	int error, error2;
+	int interruptable;
 	struct flock *fl;
-	int fmode, ioflg;
 	struct nfsmount *nmp;
 	struct nfs_vattr nvattr;
 	off_t start, end;
 	struct timeval now;
 	int timeo, endtime, lastmsg, wentdown = 0;
-	int lockpidcheck;
-	kauth_cred_t cred;
-	proc_t p;
+	int lockpidcheck, nfsvers;
 	struct sockaddr *saddr;
+	struct timespec ts;
 
-	p = vfs_context_proc(ap->a_context);
-	cred = vfs_context_ucred(ap->a_context);
-
+	ctx = ap->a_context;
+	p = vfs_context_proc(ctx);
 	vp = ap->a_vp;
 	fl = ap->a_fl;
 	np = VTONFS(vp);
 
-	nmp = VFSTONFS(vnode_mount(vp));
+	nmp = VTONMP(vp);
 	if (!nmp)
 		return (ENXIO);
-	if (nmp->nm_flag & NFSMNT_NOLOCKS)
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_flag & NFSMNT_NOLOCKS) {
+		lck_mtx_unlock(&nmp->nm_lock);
 		return (ENOTSUP);
+	}
+	nfsvers = nmp->nm_vers;
+	lck_mtx_unlock(&nmp->nm_lock);
 
 	/*
 	 * The NLM protocol doesn't allow the server to return an error
@@ -475,75 +607,20 @@ nfs_dolock(struct vnop_advlock_args *ap)
 		    (fl->l_len < 0 && fl->l_start + fl->l_len < 0))
 			return (EINVAL);
 	}
-	/*
-	 * If daemon is running take a ref on its fifo vnode
-	 */
-	if (!(wvp = nfslockdvnode)) {
-		if (!nfslockdwaiting && !nfslockdstarttimeout)
-			return (ENOTSUP);
-		/*
-		 * Don't wake lock daemon if it hasn't been started yet and
-		 * this is an unlock request (since we couldn't possibly
-		 * actually have a lock on the file).  This could be an
-		 * uninformed unlock request due to closef()'s behavior of doing
-		 * unlocks on all files if a process has had a lock on ANY file.
-		 */
-		if (!nfslockdvnode && (fl->l_type == F_UNLCK))
-			return (EINVAL);
-		microuptime(&now);
-		if (nfslockdwaiting) {
-			/* wake up lock daemon */
-			nfslockdstarttimeout = now.tv_sec + 60;
-			(void)wakeup((void *)&nfslockdwaiting);
-		}
-		/* wait on nfslockdvnode for a while to allow daemon to start */
-		while (!nfslockdvnode && (now.tv_sec < nfslockdstarttimeout)) {
-			error = tsleep((void *)&nfslockdvnode, PCATCH | PUSER, "lockdstart", 2*hz);
-			if (error && (error != EWOULDBLOCK))
-				return (error);
-			/* check that we still have our mount... */
-			/* ...and that we still support locks */
-			nmp = VFSTONFS(vnode_mount(vp));
-			if (!nmp)
-				return (ENXIO);
-			if (nmp->nm_flag & NFSMNT_NOLOCKS)
-				return (ENOTSUP);
-			if (!error)
-				break;
-			microuptime(&now);
-		}
-		/*
-		 * check for nfslockdvnode
-		 * If it hasn't started by now, there's a problem.
-		 */
-		if (!(wvp = nfslockdvnode))
-			return (ENOTSUP);
-	}
-	error = vnode_getwithref(wvp);
-	if (error)
-		return (ENOTSUP);
-	error = vnode_ref(wvp);
-	if (error) {
-		vnode_put(wvp);
-		return (ENOTSUP);
-	}
+
+	lck_mtx_lock(nfs_lock_mutex);
 
 	/*
 	 * Need to check if this process has successfully acquired an NFS lock before.
 	 * If not, and this is an unlock request we can simply return success here.
 	 */
-	lockpidcheck = nfs_lock_pid_check(p, 0, vp);
+	lockpidcheck = nfs_lock_pid_check(p, 0);
+	lck_mtx_unlock(nfs_lock_mutex);
 	if (lockpidcheck) {
-		if (lockpidcheck != ENOENT) {
-			vnode_rele(wvp);
-			vnode_put(wvp);
+		if (lockpidcheck != ENOENT)
 			return (lockpidcheck);
-		}
-		if (ap->a_op == F_UNLCK) {
-			vnode_rele(wvp);
-			vnode_put(wvp);
+		if ((ap->a_op == F_UNLCK) && nfs_lock_pid_hash_trusted)
 			return (0);
-		}
 	}
 
 	/*
@@ -565,28 +642,31 @@ nfs_dolock(struct vnop_advlock_args *ap)
 	case SEEK_END:
 		/* need to flush, and refetch attributes to make */
 		/* sure we have the correct end of file offset   */
+		error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+		if (error)
+			return (error);
+		NATTRINVALIDATE(np);
 		if (np->n_flag & NMODIFIED) {
-			NATTRINVALIDATE(np);
-			error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-			if (error) {
-				vnode_rele(wvp);
-				vnode_put(wvp);
+			nfs_unlock(np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			if (error)
 				return (error);
-			}
-		}
-		NATTRINVALIDATE(np);
+		} else
+			nfs_unlock(np);
 
-		error = nfs_getattr(vp, &nvattr, cred, p);
+		error = nfs_getattr(np, &nvattr, ctx, 0);
+		nfs_data_lock(np, NFS_NODE_LOCK_SHARED);
+		if (!error)
+			error = nfs_lock(np, NFS_NODE_LOCK_SHARED);
 		if (error) {
-			vnode_rele(wvp);
-			vnode_put(wvp);
+			nfs_data_unlock(np);
 			return (error);
 		}
 		start = np->n_size + fl->l_start;
+		nfs_unlock(np);
+		nfs_data_unlock(np);
 		break;
 	default:
-		vnode_rele(wvp);
-		vnode_put(wvp);
 		return (EINVAL);
 	}
 	if (fl->l_len == 0)
@@ -597,17 +677,12 @@ nfs_dolock(struct vnop_advlock_args *ap)
 		end = start - 1;
 		start += fl->l_len;
 	}
-	if (start < 0) {
-		vnode_rele(wvp);
-		vnode_put(wvp);
+	if (start < 0)
 		return (EINVAL);
-	}
-	if (!NFS_ISV3(vp) &&
-	    ((start >= 0x80000000) || (end >= 0x80000000))) {
-		vnode_rele(wvp);
-		vnode_put(wvp);
+
+	if ((nfsvers == NFS_VER2) &&
+	    ((start >= 0x80000000) || (end >= 0x80000000)))
 		return (EINVAL);
-	}
 
 	/*
 	 * Fill in the information structure.
@@ -623,73 +698,48 @@ nfs_dolock(struct vnop_advlock_args *ap)
 	msg->lm_fl.l_start = start;
 	if (end != -1)
 		msg->lm_fl.l_len = end - start + 1;
-	msg->lm_fl.l_pid = proc_pid(p);
+	msg->lm_fl.l_pid = vfs_context_pid(ctx);
 
 	if (ap->a_flags & F_WAIT)
 		msg->lm_flags |= LOCKD_MSG_BLOCK;
 	if (ap->a_op == F_GETLK)
 		msg->lm_flags |= LOCKD_MSG_TEST;
 
-	nmp = VFSTONFS(vnode_mount(vp));
-	if (!nmp) {
-		vnode_rele(wvp);
-		vnode_put(wvp);
+	nmp = VTONMP(vp);
+	if (!nmp)
 		return (ENXIO);
-	}
 
+	lck_mtx_lock(&nmp->nm_lock);
 	saddr = mbuf_data(nmp->nm_nam);
 	bcopy(saddr, &msg->lm_addr, min(sizeof msg->lm_addr, saddr->sa_len));
-	msg->lm_fh_len = NFS_ISV3(vp) ? VTONFS(vp)->n_fhsize : NFSX_V2FH;
-	bcopy(VTONFS(vp)->n_fhp, msg->lm_fh, msg->lm_fh_len);
-	if (NFS_ISV3(vp))
+	msg->lm_fh_len = (nfsvers == NFS_VER2) ? NFSX_V2FH : np->n_fhsize;
+	bcopy(np->n_fhp, msg->lm_fh, msg->lm_fh_len);
+	if (nfsvers == NFS_VER3)
 		msg->lm_flags |= LOCKD_MSG_NFSV3;
-	cru2x(cred, &msg->lm_cred);
+	cru2x(vfs_context_ucred(ctx), &msg->lm_cred);
 
 	microuptime(&now);
 	lastmsg = now.tv_sec - ((nmp->nm_tprintf_delay) - (nmp->nm_tprintf_initial_delay));
+	interruptable = nmp->nm_flag & NFSMNT_INT;
+	lck_mtx_unlock(&nmp->nm_lock);
 
-	fmode = FFLAGS(O_WRONLY);
-	if ((error = VNOP_OPEN(wvp, fmode, ap->a_context))) {
-		vnode_rele(wvp);
-		vnode_put(wvp);
-		return (error);
-	}
-	vnode_lock(wvp);
-	++wvp->v_writecount;
-	vnode_unlock(wvp);
+	lck_mtx_lock(nfs_lock_mutex);
 
 	/* allocate unique xid */
 	msg->lm_xid = nfs_lockxid_get();
 	nfs_lockdmsg_enqueue(&msgreq);
 
-	timeo = 2*hz;
-#define IO_NOMACCHECK 0;
-	ioflg = IO_UNIT | IO_NOMACCHECK;
-	for (;;) {
-		error = 0;
-		while (nfslockdfifolock & NFSLOCKDFIFOLOCK_LOCKED) {
-			nfslockdfifolock |= NFSLOCKDFIFOLOCK_WANT;
-			error = tsleep((void *)&nfslockdfifolock,
-					PCATCH | PUSER, "lockdfifo", 20*hz);
-			if (error)
-				break;
-		}
-		if (error)
-			break;
-		nfslockdfifolock |= NFSLOCKDFIFOLOCK_LOCKED;
-
-		error = vn_rdwr(UIO_WRITE, wvp, (caddr_t)msg, sizeof(*msg), 0,
-		    UIO_SYSSPACE32, ioflg, proc_ucred(kernproc), NULL, p);
+	timeo = 2;
 
-		nfslockdfifolock &= ~NFSLOCKDFIFOLOCK_LOCKED;
-		if (nfslockdfifolock & NFSLOCKDFIFOLOCK_WANT) {
-			nfslockdfifolock &= ~NFSLOCKDFIFOLOCK_WANT;
-			wakeup((void *)&nfslockdfifolock);
-		}
+	for (;;) {
+		nfs_lockd_request_sent = 1;
 
-		if (error && (((ioflg & IO_NDELAY) == 0) || error != EAGAIN)) {
+		/* need to drop nfs_lock_mutex while calling send_request() */
+		lck_mtx_unlock(nfs_lock_mutex);
+		error = send_request(msg, interruptable);
+		lck_mtx_lock(nfs_lock_mutex);
+		if (error && error != EAGAIN)
 			break;
-		}
 
 		/*
 		 * Always wait for an answer.  Not waiting for unlocks could
@@ -703,18 +753,19 @@ nfs_dolock(struct vnop_advlock_args *ap)
 		 * at 2 and double each timeout with a max of 60 seconds.
 		 *
 		 * In order to maintain responsiveness, we pass a small timeout
-		 * to tsleep and calculate the timeouts ourselves.  This allows
+		 * to msleep and calculate the timeouts ourselves.  This allows
 		 * us to pick up on mount changes quicker.
 		 */
 wait_for_granted:
 		error = EWOULDBLOCK;
+		ts.tv_sec = 2;
+		ts.tv_nsec = 0;
 		microuptime(&now);
-		if ((timeo/hz) > 0)
-			endtime = now.tv_sec + timeo/hz;
-		else
-			endtime = now.tv_sec + 1;
+		endtime = now.tv_sec + timeo;
 		while (now.tv_sec < endtime) {
-			error = tsleep((void *)&msgreq, PCATCH | PUSER, "lockd", 2*hz);
+			error = error2 = 0;
+			if (!msgreq.lmr_answered)
+				error = msleep(&msgreq, nfs_lock_mutex, PCATCH | PUSER, "lockd", &ts);
 			if (msgreq.lmr_answered) {
 				/*
 				 * Note: it's possible to have a lock granted at
@@ -723,58 +774,66 @@ wait_for_granted:
 				 * error from this request or we might not unlock the
 				 * lock that's been granted.
 				 */
-				error = 0;
+				nmp = VTONMP(vp);
+				if ((msgreq.lmr_errno == ENOTSUP) && nmp &&
+				    (nmp->nm_state & NFSSTA_LOCKSWORK)) {
+					/*
+					 * We have evidence that locks work, yet lockd
+					 * returned ENOTSUP.  This is probably because
+					 * it was unable to contact the server's lockd
+					 * to send it the request.
+					 *
+					 * Because we know locks work, we'll consider
+					 * this failure to be a timeout.
+					 */
+					error = EWOULDBLOCK;
+				} else {
+					error = 0;
+				}
 				break;
 			}
 			if (error != EWOULDBLOCK)
 				break;
 			/* check that we still have our mount... */
 			/* ...and that we still support locks */
-			nmp = VFSTONFS(vnode_mount(vp));
-			if (!nmp || (nmp->nm_flag & NFSMNT_NOLOCKS))
-				break;
-			/*
-			 * If the mount is hung and we've requested not to hang
-			 * on remote filesystems, then bail now.
-			 */
-			if ((p != NULL) && ((proc_noremotehang(p)) != 0) &&
-			    ((nmp->nm_state & (NFSSTA_TIMEO|NFSSTA_LOCKTIMEO)) != 0)) {
+			nmp = VTONMP(vp);
+			if ((error2 = nfs_sigintr(nmp, NULL, vfs_context_thread(ctx), 0))) {
+				error = error2;
 				if (fl->l_type == F_UNLCK)
-					printf("nfs_dolock: aborting unlock request "
-					    "due to timeout (noremotehang)\n");
-				error = EIO;
+					printf("nfs_vnop_advlock: aborting unlock request, error %d\n", error);
+				break;
+			}
+			lck_mtx_lock(&nmp->nm_lock);
+			if (nmp->nm_flag & NFSMNT_NOLOCKS) {
+				lck_mtx_unlock(&nmp->nm_lock);
 				break;
 			}
+			interruptable = nmp->nm_flag & NFSMNT_INT;
+			lck_mtx_unlock(&nmp->nm_lock);
 			microuptime(&now);
 		}
 		if (error) {
 			/* check that we still have our mount... */
-			nmp = VFSTONFS(vnode_mount(vp));
-			if (!nmp) {
-				if (error == EWOULDBLOCK)
-					error = ENXIO;
-				break;
+			nmp = VTONMP(vp);
+			if ((error2 = nfs_sigintr(nmp, NULL, vfs_context_thread(ctx), 0))) {
+				error = error2;
+				if (error2 != EINTR) {
+					if (fl->l_type == F_UNLCK)
+						printf("nfs_vnop_advlock: aborting unlock request, error %d\n", error);
+					break;
+				}
 			}
 			/* ...and that we still support locks */
+			lck_mtx_lock(&nmp->nm_lock);
 			if (nmp->nm_flag & NFSMNT_NOLOCKS) {
 				if (error == EWOULDBLOCK)
 					error = ENOTSUP;
+				lck_mtx_unlock(&nmp->nm_lock);
 				break;
 			}
-			if ((error == ENOTSUP) &&
-			    (nmp->nm_state & NFSSTA_LOCKSWORK)) {
-				/*
-				 * We have evidence that locks work, yet lockd
-				 * returned ENOTSUP.  This is probably because
-				 * it was unable to contact the server's lockd to
-				 * send it the request.
-				 *
-				 * Because we know locks work, we'll consider
-				 * this failure to be a timeout.
-				 */
-				error = EWOULDBLOCK;
-			}
+			interruptable = nmp->nm_flag & NFSMNT_INT;
 			if (error != EWOULDBLOCK) {
+				lck_mtx_unlock(&nmp->nm_lock);
 				/*
 				 * We're going to bail on this request.
 				 * If we were a blocked lock request, send a cancel.
@@ -790,34 +849,25 @@ wait_for_granted:
 					msgreq.lmr_errno = 0;
 					msgreq.lmr_answered = 0;
 					/* reset timeout */
-					timeo = 2*hz;
+					timeo = 2;
 					/* send cancel request */
 					continue;
 				}
 				break;
 			}
 
-			/*
-			 * If the mount is hung and we've requested not to hang
-			 * on remote filesystems, then bail now.
-			 */
-			if ((p != NULL) && ((proc_noremotehang(p)) != 0) &&
-			    ((nmp->nm_state & (NFSSTA_TIMEO|NFSSTA_LOCKTIMEO)) != 0)) {
-				if (fl->l_type == F_UNLCK)
-					printf("nfs_dolock: aborting unlock request "
-					    "due to timeout (noremotehang)\n");
-				error = EIO;
-				break;
-			}
 			/* warn if we're not getting any response */
 			microuptime(&now);
 			if ((msgreq.lmr_errno != EINPROGRESS) &&
 			    (nmp->nm_tprintf_initial_delay != 0) &&
 			    ((lastmsg + nmp->nm_tprintf_delay) < now.tv_sec)) {
+				lck_mtx_unlock(&nmp->nm_lock);
 				lastmsg = now.tv_sec;
-				nfs_down(nmp, p, 0, NFSSTA_LOCKTIMEO, "lockd not responding");
+				nfs_down(nmp, vfs_context_thread(ctx), 0, NFSSTA_LOCKTIMEO, "lockd not responding");
 				wentdown = 1;
-			}
+			} else
+				lck_mtx_unlock(&nmp->nm_lock);
+
 			if (msgreq.lmr_errno == EINPROGRESS) {
 				/*
 				 * We've got a blocked lock request that we are
@@ -837,31 +887,29 @@ wait_for_granted:
 				msgreq.lmr_saved_errno = msgreq.lmr_errno;
 				msgreq.lmr_errno = 0;
 				msgreq.lmr_answered = 0;
-				timeo = 2*hz;
+				timeo = 2;
 				/* send cancel then resend request */
 				continue;
 			}
 			/*
-			 * We timed out, so we will rewrite the request
-			 * to the fifo, but only if it isn't already full.
+			 * We timed out, so we will resend the request.
 			 */
-			ioflg |= IO_NDELAY;
 			timeo *= 2;
-			if (timeo > 60*hz)
-				timeo = 60*hz;
+			if (timeo > 60)
+				timeo = 60;
 			/* resend request */
 			continue;
 		}
 
 		/* we got a reponse, so the server's lockd is OK */
-		nfs_up(VFSTONFS(vnode_mount(vp)), p, NFSSTA_LOCKTIMEO,
+		nfs_up(VTONMP(vp), vfs_context_thread(ctx), NFSSTA_LOCKTIMEO,
 			wentdown ? "lockd alive again" : NULL);
 		wentdown = 0;
 
 		if (msgreq.lmr_errno == EINPROGRESS) {
 			/* got NLM_BLOCKED response */
 			/* need to wait for NLM_GRANTED */
-			timeo = 60*hz;
+			timeo = 60;
 			msgreq.lmr_answered = 0;
 			goto wait_for_granted;
 		}
@@ -880,7 +928,7 @@ wait_for_granted:
 			msgreq.lmr_saved_errno = 0;
 			msgreq.lmr_errno = 0;
 			msgreq.lmr_answered = 0;
-			timeo = 2*hz;
+			timeo = 2;
 			/* resend request */
 			continue;
 		}
@@ -892,9 +940,8 @@ wait_for_granted:
 				fl->l_start = msg->lm_fl.l_start;
 				fl->l_len = msg->lm_fl.l_len;
 				fl->l_whence = SEEK_SET;
-			} else {
+			} else
 				fl->l_type = F_UNLCK;
-			}
 		}
 
 		/*
@@ -908,31 +955,56 @@ wait_for_granted:
 		} else
 			error = msgreq.lmr_errno;
 
+		nmp = VTONMP(vp);
+		if ((error == ENOTSUP) && nmp && !(nmp->nm_state & NFSSTA_LOCKSWORK)) {
+			/*
+			 * We have NO evidence that locks work and lockd
+			 * returned ENOTSUP.  Let's take this as a hint
+			 * that locks aren't supported and disable them
+			 * for this mount.
+			 */
+			lck_mtx_lock(&nmp->nm_lock);
+			nmp->nm_flag |= NFSMNT_NOLOCKS;
+			nmp->nm_state &= ~NFSSTA_LOCKTIMEO;
+			lck_mtx_unlock(&nmp->nm_lock);
+			printf("lockd returned ENOTSUP, disabling locks for nfs server: %s\n",
+				vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+		}
 		if (!error) {
 			/* record that NFS file locking has worked on this mount */
-			nmp = VFSTONFS(vnode_mount(vp));
-			if (nmp && !(nmp->nm_state & NFSSTA_LOCKSWORK))
-				nmp->nm_state |= NFSSTA_LOCKSWORK;
+			if (nmp) {
+				lck_mtx_lock(&nmp->nm_lock);
+				if (!(nmp->nm_state & NFSSTA_LOCKSWORK))
+					nmp->nm_state |= NFSSTA_LOCKSWORK;
+				lck_mtx_unlock(&nmp->nm_lock);
+			}
 			/*
 			 * If we successfully acquired a lock, make sure this pid
 			 * is in the nfs_lock_pid hash table so we know we can't
 			 * short-circuit unlock requests.
 			 */
 			if ((lockpidcheck == ENOENT) &&
-			    ((ap->a_op == F_SETLK) || (ap->a_op == F_SETLKW)))
-				nfs_lock_pid_check(p, 1, vp);
-	
+			    ((ap->a_op == F_SETLK) || (ap->a_op == F_SETLKW))) {
+				error = nfs_lock_pid_check(p, 1);
+				if (error) {
+					/*
+					 * We couldn't add the pid to the table,
+					 * so we can no longer trust that a pid
+					 * not in the table has no locks.
+					 */
+					nfs_lock_pid_hash_trusted = 0;
+					printf("nfs_vnop_advlock: pid add failed - no longer trusted\n");
+				}
+			}
 		}
 		break;
 	}
 
 	nfs_lockdmsg_dequeue(&msgreq);
 
-	error1 = VNOP_CLOSE(wvp, FWRITE, ap->a_context);
-	vnode_rele(wvp);
-	vnode_put(wvp);
-	/* prefer any previous 'error' to our vn_close 'error1'. */
-	return (error != 0 ? error : error1);
+	lck_mtx_unlock(nfs_lock_mutex);
+
+	return (error);
 }
 
 /*
@@ -954,6 +1026,8 @@ nfslockdans(proc_t p, struct lockd_ans *ansp)
 	if (ansp->la_version != LOCKD_ANS_VERSION)
 		return (EINVAL);
 
+	lck_mtx_lock(nfs_lock_mutex);
+
 	/* try to find the lockd message by transaction id (cookie) */
 	msgreq = nfs_lockdmsg_find_by_xid(ansp->la_xid);
 	if (ansp->la_flags & LOCKD_ANS_GRANTED) {
@@ -972,8 +1046,10 @@ nfslockdans(proc_t p, struct lockd_ans *ansp)
 		if (msgreq && (msgreq->lmr_msg.lm_flags & LOCKD_MSG_CANCEL))
 			msgreq = NULL;
 	}
-	if (!msgreq)
+	if (!msgreq) {
+		lck_mtx_unlock(nfs_lock_mutex);
 		return (EPIPE);
+	}
 
 	msgreq->lmr_errno = ansp->la_errno;
 	if ((msgreq->lmr_msg.lm_flags & LOCKD_MSG_TEST) && msgreq->lmr_errno == 0) {
@@ -991,70 +1067,9 @@ nfslockdans(proc_t p, struct lockd_ans *ansp)
 	}
 
 	msgreq->lmr_answered = 1;
-	(void)wakeup((void *)msgreq);
+	lck_mtx_unlock(nfs_lock_mutex);
+	wakeup(msgreq);
 
 	return (0);
 }
 
-/*
- * nfslockdfd --
- *      NFS advisory byte-level locks: fifo file# from the lock daemon.
- */
-int
-nfslockdfd(proc_t p, int fd)
-{
-	int error;
-	vnode_t vp, oldvp;
-
-	error = proc_suser(p);
-	if (error)
-		return (error);
-	if (fd < 0) {
-		vp = NULL;
-	} else {
-		error = file_vnode(fd, &vp);
-		if (error)
-			return (error);
-		error = vnode_getwithref(vp);
-		if (error)
-			return (error);
-		error = vnode_ref(vp);
-		if (error) {
-			vnode_put(vp);
-			return (error);
-		}
-	}
-	oldvp = nfslockdvnode;
-	nfslockdvnode = vp;
-	if (oldvp) {
-		vnode_rele(oldvp);
-	}
-	(void)wakeup((void *)&nfslockdvnode);
-	if (vp) {
-		vnode_put(vp);
-	}
-	return (0);
-}
-
-/*
- * nfslockdwait --
- *      lock daemon waiting for lock request
- */
-int
-nfslockdwait(proc_t p)
-{
-	int error;
-
-	error = proc_suser(p);
-	if (error)
-		return (error);
-	if (nfslockdwaiting || nfslockdvnode)
-		return (EBUSY);
-
-	nfslockdstarttimeout = 0;
-	nfslockdwaiting = 1;
-	tsleep((void *)&nfslockdwaiting, PCATCH | PUSER, "lockd", 0);
-	nfslockdwaiting = 0;
-
-	return (0);
-}
diff --git a/bsd/nfs/nfs_lock.h b/bsd/nfs/nfs_lock.h
index 512408454..0737615c1 100644
--- a/bsd/nfs/nfs_lock.h
+++ b/bsd/nfs/nfs_lock.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1998 Berkeley Software Design, Inc. All rights reserved.
@@ -62,14 +68,7 @@
  * If a structure changes, you must bump the version number.
  */
 
-#include <nfs/nfsproto.h>
-
-/*
- * The fifo where the kernel writes requests for locks on remote NFS files,
- * and where lockd reads these requests.  Note this is no longer hardwired
- * in the kernel binary - lockd passes the file descriptor down via nfsclnt()
- */
-#define	_PATH_LCKFIFO	"/var/run/nfslockd"
+#include <sys/mount.h>
 
 /*
  * The structure that the kernel hands lockd for each lock request.
@@ -83,7 +82,7 @@ typedef struct nfs_lock_msg {
 	struct sockaddr_storage lm_addr;		/* The address. */
 	int			lm_fh_len;		/* The file handle length. */
 	struct xucred		lm_cred;		/* user cred for lock req */
-	u_int8_t		lm_fh[NFS_SMALLFH];	/* The file handle. */
+	u_int8_t		lm_fh[NFSV3_MAX_FH_SIZE]; /* The file handle. */
 } LOCKD_MSG;
 
 /* lm_flags */
@@ -118,7 +117,7 @@ struct lockd_ans {
 	off_t		la_start;		/* lock starting offset */
 	off_t		la_len;			/* lock length */
 	int 		la_fh_len;		/* The file handle length. */
-	u_int8_t	la_fh[NFS_SMALLFH];	/* The file handle. */
+	u_int8_t	la_fh[NFSV3_MAX_FH_SIZE];/* The file handle. */
 };
 
 /* la_flags */
@@ -129,12 +128,9 @@ struct lockd_ans {
 
 #ifdef KERNEL
 void	nfs_lockinit(void);
-int	nfs_dolock(struct vnop_advlock_args *ap);
+void	nfs_lockd_mount_change(int);
+int	nfs3_vnop_advlock(struct vnop_advlock_args *ap);
 int	nfslockdans(proc_t p, struct lockd_ans *ansp);
-int	nfslockdfd(proc_t p, int fd);
-int	nfslockdwait(proc_t p);
 
-extern vnode_t nfslockdvnode;
-extern int nfslockdwaiting;
 #endif
 #endif /* __APPLE_API_PRIVATE */
diff --git a/bsd/nfs/nfs_node.c b/bsd/nfs/nfs_node.c
index 6070f5a7b..e42d5022d 100644
--- a/bsd/nfs/nfs_node.c
+++ b/bsd/nfs/nfs_node.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -73,14 +79,16 @@
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 #include <nfs/nfsnode.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 
-LIST_HEAD(nfsnodehashhead, nfsnode) *nfsnodehashtbl;
-u_long nfsnodehash;
+#define	NFSNOHASH(fhsum) \
+	(&nfsnodehashtbl[(fhsum) & nfsnodehash])
+static LIST_HEAD(nfsnodehashhead, nfsnode) *nfsnodehashtbl;
+static u_long nfsnodehash;
 
-lck_grp_t * nfs_node_hash_lck_grp;
-lck_grp_attr_t * nfs_node_hash_lck_grp_attr;
-lck_attr_t * nfs_node_hash_lck_attr;
+static lck_grp_t *nfs_node_hash_lck_grp;
+static lck_grp_t *nfs_node_lck_grp;
 lck_mtx_t *nfs_node_hash_mutex;
 
 /*
@@ -90,14 +98,18 @@ lck_mtx_t *nfs_node_hash_mutex;
 void
 nfs_nhinit(void)
 {
-	nfsnodehashtbl = hashinit(desiredvnodes, M_NFSNODE, &nfsnodehash);
-
-	nfs_node_hash_lck_grp_attr = lck_grp_attr_alloc_init();
-	nfs_node_hash_lck_grp = lck_grp_alloc_init("nfs_node_hash", nfs_node_hash_lck_grp_attr);
-
-	nfs_node_hash_lck_attr = lck_attr_alloc_init();
+	nfs_node_hash_lck_grp = lck_grp_alloc_init("nfs_node_hash", LCK_GRP_ATTR_NULL);
+	nfs_node_hash_mutex = lck_mtx_alloc_init(nfs_node_hash_lck_grp, LCK_ATTR_NULL);
+	nfs_node_lck_grp = lck_grp_alloc_init("nfs_node", LCK_GRP_ATTR_NULL);
+}
 
-	nfs_node_hash_mutex = lck_mtx_alloc_init(nfs_node_hash_lck_grp, nfs_node_hash_lck_attr);
+void
+nfs_nhinit_finish(void)
+{
+	lck_mtx_lock(nfs_node_hash_mutex);
+	if (!nfsnodehashtbl)
+		nfsnodehashtbl = hashinit(desiredvnodes, M_NFSNODE, &nfsnodehash);
+	lck_mtx_unlock(nfs_node_hash_mutex);
 }
 
 /*
@@ -123,42 +135,50 @@ nfs_hash(u_char *fhp, int fhsize)
  */
 int
 nfs_nget(
-	mount_t mntp,
-	vnode_t dvp,
+	mount_t mp,
+	nfsnode_t dnp,
 	struct componentname *cnp,
 	u_char *fhp,
 	int fhsize,
 	struct nfs_vattr *nvap,
 	u_int64_t *xidp,
 	int flags,
-	struct nfsnode **npp)
+	nfsnode_t *npp)
 {
-	struct nfsnode *np;
+	nfsnode_t np;
 	struct nfsnodehashhead *nhpp;
-	vnode_t vp, nvp;
-	int error;
-	mount_t mp;
+	vnode_t vp;
+	int error, nfsvers;
+	mount_t mp2;
 	struct vnode_fsparam vfsp;
 	uint32_t vid;
 
+	FSDBG_TOP(263, mp, dnp, flags, npp);
+
 	/* Check for unmount in progress */
-	if (!mntp || (mntp->mnt_kern_flag & MNTK_UNMOUNT)) {
-		*npp = 0;
-		return (!mntp ? ENXIO : EPERM);
+	if (!mp || (mp->mnt_kern_flag & MNTK_FRCUNMOUNT)) {
+		*npp = NULL;
+		error = ENXIO;
+		FSDBG_BOT(263, mp, dnp, 0xd1e, error);
+		return (error);
 	}
+	nfsvers = VFSTONFS(mp)->nm_vers;
 
 	nhpp = NFSNOHASH(nfs_hash(fhp, fhsize));
 loop:
 	lck_mtx_lock(nfs_node_hash_mutex);
 	for (np = nhpp->lh_first; np != 0; np = np->n_hash.le_next) {
-		mp = (np->n_flag & NINIT) ? np->n_mount : vnode_mount(NFSTOV(np));
-		if (mntp != mp || np->n_fhsize != fhsize ||
+		mp2 = (np->n_hflag & NHINIT) ? np->n_mount : NFSTOMP(np);
+		if (mp != mp2 || np->n_fhsize != fhsize ||
 		    bcmp(fhp, np->n_fhp, fhsize))
 			continue;
-		/* if the node is still being initialized, sleep on it */
-		if (np->n_flag & NINIT) {
-			np->n_flag |= NWINIT;
-			msleep(np, nfs_node_hash_mutex, PDROP | PINOD, "nfs_nget", 0);
+		FSDBG(263, dnp, np, np->n_flag, 0xcace0000);
+		/* if the node is locked, sleep on it */
+		if (np->n_hflag & NHLOCKED) {
+			np->n_hflag |= NHLOCKWANT;
+			FSDBG(263, dnp, np, np->n_flag, 0xcace2222);
+			msleep(np, nfs_node_hash_mutex, PDROP | PINOD, "nfs_nget", NULL);
+			FSDBG(263, dnp, np, np->n_flag, 0xcace3333);
 			goto loop;
 		}
 		vp = NFSTOV(np);
@@ -169,34 +189,57 @@ loop:
 			 * If vnode is being reclaimed or has already
 			 * changed identity, no need to wait.
 			 */
+			FSDBG_BOT(263, dnp, *npp, 0xcace0d1e, error);
 			return (error);
-		} 
+		}
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))) {
+			/* this only fails if the node is now unhashed */
+			/* so let's see if we can find/create it again */
+			FSDBG(263, dnp, *npp, 0xcaced1e2, error);
+			vnode_put(vp);
+			goto loop;
+		}
 		/* update attributes */
 		error = nfs_loadattrcache(np, nvap, xidp, 0);
 		if (error) {
+			nfs_unlock(np);
 			vnode_put(vp);
 		} else {
-			if (dvp && cnp && (flags & NG_MAKEENTRY))
-				cache_enter(dvp, vp, cnp);
+			if (dnp && cnp && (flags & NG_MAKEENTRY))
+				cache_enter(NFSTOV(dnp), vp, cnp);
 			*npp = np;
 		}
+		FSDBG_BOT(263, dnp, *npp, 0xcace0000, error);
 		return(error);
 	}
 
+	FSDBG(263, mp, dnp, npp, 0xaaaaaaaa);
+
 	/*
 	 * allocate and initialize nfsnode and stick it in the hash
 	 * before calling getnewvnode().  Anyone finding it in the
 	 * hash before initialization is complete will wait for it.
 	 */
-	MALLOC_ZONE(np, struct nfsnode *, sizeof *np, M_NFSNODE, M_WAITOK);
+	MALLOC_ZONE(np, nfsnode_t, sizeof *np, M_NFSNODE, M_WAITOK);
 	if (!np) {
 		lck_mtx_unlock(nfs_node_hash_mutex);
 		*npp = 0;
+		FSDBG_BOT(263, dnp, *npp, 0x80000001, ENOMEM);
 		return (ENOMEM);
 	}
-	bzero((caddr_t)np, sizeof *np);
-	np->n_flag |= NINIT;
-	np->n_mount = mntp;
+	bzero(np, sizeof *np);
+	np->n_hflag |= (NHINIT | NHLOCKED);
+	np->n_mount = mp;
+
+	if (dnp && cnp && ((cnp->cn_namelen != 2) ||
+	    (cnp->cn_nameptr[0] != '.') || (cnp->cn_nameptr[1] != '.'))) {
+		vnode_t dvp = NFSTOV(dnp);
+		if (!vnode_get(dvp)) {
+			if (!vnode_ref(dvp))
+				np->n_parent = dvp;
+			vnode_put(dvp);
+		}
+	}
 
 	/* setup node's file handle */
 	if (fhsize > NFS_SMALLFH) {
@@ -206,6 +249,7 @@ loop:
 			lck_mtx_unlock(nfs_node_hash_mutex);
 			FREE_ZONE(np, sizeof *np, M_NFSNODE);
 			*npp = 0;
+			FSDBG_BOT(263, dnp, *npp, 0x80000002, ENOMEM);
 			return (ENOMEM);
 		}
 	} else {
@@ -215,8 +259,14 @@ loop:
 	np->n_fhsize = fhsize;
 
 	/* Insert the nfsnode in the hash queue for its new file handle */
-	np->n_flag |= NHASHED;
 	LIST_INSERT_HEAD(nhpp, np, n_hash);
+	np->n_hflag |= NHHASHED;
+	FSDBG(266, 0, np, np->n_flag, np->n_hflag);
+
+	/* lock the new nfsnode */
+	lck_rw_init(&np->n_lock, nfs_node_lck_grp, LCK_ATTR_NULL);
+	lck_rw_init(&np->n_datalock, nfs_node_lck_grp, LCK_ATTR_NULL);
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
 
 	/* release lock on hash table */
 	lck_mtx_unlock(nfs_node_hash_mutex);
@@ -224,131 +274,237 @@ loop:
 	/* do initial loading of attributes */
 	error = nfs_loadattrcache(np, nvap, xidp, 1);
 	if (error) {
+		FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
+		nfs_unlock(np);
 		lck_mtx_lock(nfs_node_hash_mutex);
 		LIST_REMOVE(np, n_hash);
-		np->n_flag &= ~(NHASHED|NINIT);
-		if (np->n_flag & NWINIT) {
-			np->n_flag &= ~NWINIT;
-			wakeup((caddr_t)np);
+		np->n_hflag &= ~(NHHASHED|NHINIT|NHLOCKED);
+		if (np->n_hflag & NHLOCKWANT) {
+			np->n_hflag &= ~NHLOCKWANT;
+			wakeup(np);
 		}
 		lck_mtx_unlock(nfs_node_hash_mutex);
+		if (np->n_parent) {
+			if (!vnode_get(np->n_parent)) {
+				vnode_rele(np->n_parent);
+				vnode_put(np->n_parent);
+			}
+			np->n_parent = NULL;
+		}
+		lck_rw_destroy(&np->n_lock, nfs_node_lck_grp);
+		lck_rw_destroy(&np->n_datalock, nfs_node_lck_grp);
 		if (np->n_fhsize > NFS_SMALLFH)
 			FREE_ZONE(np->n_fhp, np->n_fhsize, M_NFSBIGFH);
 		FREE_ZONE(np, sizeof *np, M_NFSNODE);
 		*npp = 0;
+		FSDBG_BOT(263, dnp, *npp, 0x80000003, error);
 		return (error);
 	}
-	np->n_mtime = nvap->nva_mtime;
+	NFS_CHANGED_UPDATE(nfsvers, np, nvap);
 	if (nvap->nva_type == VDIR)
-		np->n_ncmtime = nvap->nva_mtime;
+		NFS_CHANGED_UPDATE_NC(nfsvers, np, nvap);
 	NMODEINVALIDATE(np);
 
 	/* now, attempt to get a new vnode */
-	vfsp.vnfs_mp = mntp;
+	vfsp.vnfs_mp = mp;
 	vfsp.vnfs_vtype = nvap->nva_type;
 	vfsp.vnfs_str = "nfs";
-	vfsp.vnfs_dvp = dvp;
+	vfsp.vnfs_dvp = dnp ? NFSTOV(dnp) : NULL;
 	vfsp.vnfs_fsnode = np;
-	if (nvap->nva_type == VFIFO)
-		vfsp.vnfs_vops = fifo_nfsv2nodeop_p;
-	else if (nvap->nva_type == VBLK || nvap->nva_type == VCHR)
-		vfsp.vnfs_vops = spec_nfsv2nodeop_p;
-	else
-		vfsp.vnfs_vops = nfsv2_vnodeop_p;
+	if (nfsvers == NFS_VER4) {
+#if FIFO
+		if (nvap->nva_type == VFIFO)
+			vfsp.vnfs_vops = fifo_nfsv4nodeop_p;
+		else
+#endif /* FIFO */
+		if (nvap->nva_type == VBLK || nvap->nva_type == VCHR)
+			vfsp.vnfs_vops = spec_nfsv4nodeop_p;
+		else
+			vfsp.vnfs_vops = nfsv4_vnodeop_p;
+	} else {
+#if FIFO
+		if (nvap->nva_type == VFIFO)
+			vfsp.vnfs_vops = fifo_nfsv2nodeop_p;
+		else
+#endif /* FIFO */
+		if (nvap->nva_type == VBLK || nvap->nva_type == VCHR)
+			vfsp.vnfs_vops = spec_nfsv2nodeop_p;
+		else
+			vfsp.vnfs_vops = nfsv2_vnodeop_p;
+	}
 	vfsp.vnfs_markroot = (flags & NG_MARKROOT) ? 1 : 0;
 	vfsp.vnfs_marksystem = 0;
 	vfsp.vnfs_rdev = 0;
 	vfsp.vnfs_filesize = nvap->nva_size;
 	vfsp.vnfs_cnp = cnp;
-	if (dvp && cnp && (flags & NG_MAKEENTRY))
-		vfsp.vnfs_flags = 0;
-	else
-		vfsp.vnfs_flags = VNFS_NOCACHE;
-	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &nvp);
+	vfsp.vnfs_flags = VNFS_ADDFSREF;
+	if (!dnp || !cnp || !(flags & NG_MAKEENTRY))
+		vfsp.vnfs_flags |= VNFS_NOCACHE;
+
+	error = vnode_create(VNCREATE_FLAVOR, VCREATESIZE, &vfsp, &np->n_vnode);
 	if (error) {
+		FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
+		nfs_unlock(np);
 		lck_mtx_lock(nfs_node_hash_mutex);
 		LIST_REMOVE(np, n_hash);
-		np->n_flag &= ~(NHASHED|NINIT);
-		if (np->n_flag & NWINIT) {
-			np->n_flag &= ~NWINIT;
-			wakeup((caddr_t)np);
+		np->n_hflag &= ~(NHHASHED|NHINIT|NHLOCKED);
+		if (np->n_hflag & NHLOCKWANT) {
+			np->n_hflag &= ~NHLOCKWANT;
+			wakeup(np);
 		}
 		lck_mtx_unlock(nfs_node_hash_mutex);
+		if (np->n_parent) {
+			if (!vnode_get(np->n_parent)) {
+				vnode_rele(np->n_parent);
+				vnode_put(np->n_parent);
+			}
+			np->n_parent = NULL;
+		}
+		lck_rw_destroy(&np->n_lock, nfs_node_lck_grp);
+		lck_rw_destroy(&np->n_datalock, nfs_node_lck_grp);
 		if (np->n_fhsize > NFS_SMALLFH)
 			FREE_ZONE(np->n_fhp, np->n_fhsize, M_NFSBIGFH);
 		FREE_ZONE(np, sizeof *np, M_NFSNODE);
 		*npp = 0;
+		FSDBG_BOT(263, dnp, *npp, 0x80000004, error);
 		return (error);
 	}
-	vp = nvp;
-	np->n_vnode = vp;
-	vnode_addfsref(vp);
-	vnode_settag(vp, VT_NFS); // XXX shouldn't this be a vnode_create() parameter?
-	*npp = np;
+	vp = np->n_vnode;
+	vnode_settag(vp, VT_NFS);
 	/* node is now initialized */
 
 	/* check if anyone's waiting on this node */
 	lck_mtx_lock(nfs_node_hash_mutex);
-	np->n_flag &= ~NINIT;
-	if (np->n_flag & NWINIT) {
-		np->n_flag &= ~NWINIT;
-		wakeup((caddr_t)np);
+	np->n_hflag &= ~(NHINIT|NHLOCKED);
+	if (np->n_hflag & NHLOCKWANT) {
+		np->n_hflag &= ~NHLOCKWANT;
+		wakeup(np);
 	}
 	lck_mtx_unlock(nfs_node_hash_mutex);
 
+	*npp = np;
+
+	FSDBG_BOT(263, dnp, vp, *npp, error);
 	return (error);
 }
 
 
 int
-nfs_inactive(ap)
+nfs_vnop_inactive(ap)
 	struct vnop_inactive_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		vfs_context_t a_context;
 	} */ *ap;
 {
-	register struct nfsnode *np;
-	register struct sillyrename *sp;
+	vnode_t vp;
+	nfsnode_t np;
+	struct nfs_sillyrename *nsp;
+	struct nfs_vattr nvattr;
+	int unhash, attrerr;
 
+	vp = ap->a_vp;
 	np = VTONFS(ap->a_vp);
-	if (vnode_vtype(ap->a_vp) != VDIR) {
-		sp = np->n_sillyrename;
-		np->n_sillyrename = (struct sillyrename *)0;
+
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
+
+	if (vnode_vtype(vp) != VDIR) {
+		nsp = np->n_sillyrename;
+		np->n_sillyrename = NULL;
 	} else
-		sp = (struct sillyrename *)0;
+		nsp = NULL;
 
-	if (sp) {
-		/*
-		 * Remove the silly file that was rename'd earlier
-		 */
-#if DIAGNOSTIC
-		kprintf("nfs_inactive removing %s, dvp=%x, a_vp=%x, ap=%x, np=%x, sp=%x\n",
-			&sp->s_name[0], (unsigned)sp->s_dvp, (unsigned)ap->a_vp, (unsigned)ap,
-			(unsigned)np, (unsigned)sp);
-#endif
-		nfs_vinvalbuf(ap->a_vp, 0, sp->s_cred, vfs_context_proc(ap->a_context), 1);
-		np->n_size = 0;
-		ubc_setsize(ap->a_vp, (off_t)0);
-		nfs_removeit(sp);
+	FSDBG_TOP(264, vp, np, np->n_flag, nsp);
+
+	if (!nsp) {
+		/* no silly file to clean up... */
+		/* clear all flags other than these */
+		np->n_flag &= (NMODIFIED);
+		nfs_unlock(np);
+		FSDBG_BOT(264, vp, np, np->n_flag, 0);
+		return (0);
+	}
+
+	/* Remove the silly file that was rename'd earlier */
+
+	/* flush all the buffers */
+	nfs_unlock(np);
+	nfs_vinvalbuf2(vp, V_SAVE, vfs_context_thread(ap->a_context), nsp->nsr_cred, 1);
+
+	/* purge the name cache to deter others from finding it */
+	cache_purge(vp);
+
+	/* try to get the latest attributes */
+	attrerr = nfs_getattr(np, &nvattr, ap->a_context, 0);
+
+	/* Check if we should remove it from the node hash. */
+	/* Leave it if inuse or it has multiple hard links. */
+	if (vnode_isinuse(vp, 0) || (!attrerr && (nvattr.nva_nlink > 1))) {
+		unhash = 0;
+	} else {
+		unhash = 1;
+		ubc_setsize(vp, 0);
+	}
+
+	/* grab node lock on this node and the directory */
+	nfs_lock2(nsp->nsr_dnp, np, NFS_NODE_LOCK_FORCE);
+
+	/* lock the node while we remove the silly file */
+	lck_mtx_lock(nfs_node_hash_mutex);
+	while (np->n_hflag & NHLOCKED) {
+		np->n_hflag |= NHLOCKWANT;
+		msleep(np, nfs_node_hash_mutex, PINOD, "nfs_inactive", NULL);
+	}
+	np->n_hflag |= NHLOCKED;
+	lck_mtx_unlock(nfs_node_hash_mutex);
+
+	/* purge again in case it was looked up while we were locking */
+	cache_purge(vp);
+
+	FSDBG(264, np, np->n_size, np->n_vattr.nva_size, 0xf00d00f1);
+
+	/* now remove the silly file */
+	nfs_removeit(nsp);
+
+	/* clear all flags other than these */
+	np->n_flag &= (NMODIFIED);
+	nfs_unlock2(nsp->nsr_dnp, np);
+
+	if (unhash && vnode_isinuse(vp, 0)) {
+		/* vnode now inuse after silly remove? */
+		unhash = 0;
+		ubc_setsize(vp, np->n_size);
+	}
+
+	lck_mtx_lock(nfs_node_hash_mutex);
+	if (unhash) {
 		/*
 		 * remove nfsnode from hash now so we can't accidentally find it
 		 * again if another object gets created with the same filehandle
 		 * before this vnode gets reclaimed
 		 */
-		lck_mtx_lock(nfs_node_hash_mutex);
-		LIST_REMOVE(np, n_hash);
-		np->n_flag &= ~NHASHED;
-		lck_mtx_unlock(nfs_node_hash_mutex);
-		if (IS_VALID_CRED(sp->s_cred)) {
-			kauth_cred_unref(&sp->s_cred);
+		if (np->n_hflag & NHHASHED) {
+			LIST_REMOVE(np, n_hash);
+			np->n_hflag &= ~NHHASHED;
+			FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
 		}
-		vnode_rele(sp->s_dvp);
-		FREE_ZONE((caddr_t)sp, sizeof (struct sillyrename), M_NFSREQ);
-		vnode_recycle(ap->a_vp);
+		vnode_recycle(vp);
 	}
-	/* clear all flags other than these */
-	np->n_flag &= (NMODIFIED | NFLUSHINPROG | NFLUSHWANT | NHASHED);
+	/* unlock the node */
+	np->n_hflag &= ~NHLOCKED;
+	if (np->n_hflag & NHLOCKWANT) {
+		np->n_hflag &= ~NHLOCKWANT;
+		wakeup(np);
+	}
+	lck_mtx_unlock(nfs_node_hash_mutex);
+
+	/* cleanup sillyrename info */
+	if (nsp->nsr_cred != NOCRED)
+		kauth_cred_unref(&nsp->nsr_cred);
+	vnode_rele(NFSTOV(nsp->nsr_dnp));
+	FREE_ZONE(nsp, sizeof(*nsp), M_NFSREQ);
+
+	FSDBG_BOT(264, vp, np, np->n_flag, 0);
 	return (0);
 }
 
@@ -356,7 +512,7 @@ nfs_inactive(ap)
  * Reclaim an nfsnode so that it can be used for other purposes.
  */
 int
-nfs_reclaim(ap)
+nfs_vnop_reclaim(ap)
 	struct vnop_reclaim_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -364,23 +520,31 @@ nfs_reclaim(ap)
 	} */ *ap;
 {
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
 	struct nfsdmap *dp, *dp2;
 
+	FSDBG_TOP(265, vp, np, np->n_flag, 0);
+
+	lck_mtx_lock(nfs_node_hash_mutex);
+
+	if ((vnode_vtype(vp) != VDIR) && np->n_sillyrename)
+		printf("nfs_reclaim: leaving unlinked file %s\n", np->n_sillyrename->nsr_name);
+
 	vnode_removefsref(vp);
 
-	if (np->n_flag & NHASHED) {
-		lck_mtx_lock(nfs_node_hash_mutex);
+	if (np->n_hflag & NHHASHED) {
 		LIST_REMOVE(np, n_hash);
-		np->n_flag &= ~NHASHED;
-		lck_mtx_unlock(nfs_node_hash_mutex);
+		np->n_hflag &= ~NHHASHED;
+		FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
 	}
+	lck_mtx_unlock(nfs_node_hash_mutex);
 
 	/*
 	 * Free up any directory cookie structures and
 	 * large file handle structures that might be associated with
 	 * this nfs node.
 	 */
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
 	if (vnode_vtype(vp) == VDIR) {
 		dp = np->n_cookies.lh_first;
 		while (dp) {
@@ -393,9 +557,259 @@ nfs_reclaim(ap)
 	if (np->n_fhsize > NFS_SMALLFH) {
 		FREE_ZONE(np->n_fhp, np->n_fhsize, M_NFSBIGFH);
 	}
+
+	nfs_unlock(np);
 	vnode_clearfsnode(vp);
 
+	if (np->n_parent) {
+		if (!vnode_get(np->n_parent)) {
+			vnode_rele(np->n_parent);
+			vnode_put(np->n_parent);
+		}
+		np->n_parent = NULL;
+	}
+
+	lck_rw_destroy(&np->n_lock, nfs_node_lck_grp);
+	lck_rw_destroy(&np->n_datalock, nfs_node_lck_grp);
+
+	FSDBG_BOT(265, vp, np, np->n_flag, 0xd1ed1e);
 	FREE_ZONE(np, sizeof(struct nfsnode), M_NFSNODE);
 	return (0);
 }
 
+/*
+ * Acquire an NFS node lock
+ */
+int
+nfs_lock(nfsnode_t np, int locktype)
+{
+	FSDBG_TOP(268, np, locktype, np->n_lockowner, 0);
+	if (locktype == NFS_NODE_LOCK_SHARED) {
+		lck_rw_lock_shared(&np->n_lock);
+	} else {
+		lck_rw_lock_exclusive(&np->n_lock);
+		np->n_lockowner = current_thread();
+	}
+	if ((locktype != NFS_NODE_LOCK_FORCE) && !(np->n_hflag && NHHASHED)) {
+		FSDBG_BOT(268, np, 0xdead, np->n_lockowner, 0);
+		nfs_unlock(np);
+		return (ENOENT);
+	}
+	FSDBG_BOT(268, np, locktype, np->n_lockowner, 0);
+	return (0);
+}
+
+/*
+ * Release an NFS node lock
+ */
+void
+nfs_unlock(nfsnode_t np)
+{
+	FSDBG(269, np, np->n_lockowner, current_thread(), 0);
+	np->n_lockowner = NULL;
+	lck_rw_done(&np->n_lock);
+}
+
+/*
+ * Acquire 2 NFS node locks
+ *   - locks taken in order given (assumed to be parent-child order)
+ *   - both or neither of the locks are taken
+ *   - only one lock taken per node (dup nodes are skipped)
+ */
+int
+nfs_lock2(nfsnode_t np1, nfsnode_t np2, int locktype)
+{
+	int error;
+
+	if ((error = nfs_lock(np1, locktype)))
+		return (error);
+	if (np1 == np2)
+		return (error);
+	if ((error = nfs_lock(np2, locktype)))
+		nfs_unlock(np1);
+	return (error);
+}
+
+/*
+ * Unlock a couple of NFS nodes
+ */
+void
+nfs_unlock2(nfsnode_t np1, nfsnode_t np2)
+{
+	nfs_unlock(np1);
+	if (np1 != np2)
+		nfs_unlock(np2);
+}
+
+/*
+ * Acquire 4 NFS node locks
+ *   - fdnp/fnp and tdnp/tnp locks taken in order given
+ *   - otherwise locks taken in node address order.
+ *   - all or none of the locks are taken
+ *   - only one lock taken per node (dup nodes are skipped)
+ *   - some of the node pointers may be null
+ */
+int
+nfs_lock4(nfsnode_t fdnp, nfsnode_t fnp, nfsnode_t tdnp, nfsnode_t tnp, int locktype)
+{
+	nfsnode_t list[4];
+	int i, lcnt = 0, error;
+
+	if (fdnp == tdnp) {
+		list[lcnt++] = fdnp;
+	} else if (fdnp->n_parent && (tdnp == VTONFS(fdnp->n_parent))) {
+		list[lcnt++] = tdnp;
+		list[lcnt++] = fdnp;
+	} else if (tdnp->n_parent && (fdnp == VTONFS(tdnp->n_parent))) {
+		list[lcnt++] = fdnp;
+		list[lcnt++] = tdnp;
+	} else if (fdnp < tdnp) {
+		list[lcnt++] = fdnp;
+		list[lcnt++] = tdnp;
+	} else {
+		list[lcnt++] = tdnp;
+		list[lcnt++] = fdnp;
+	}
+
+	if (!tnp || (fnp == tnp) || (tnp == fdnp)) {
+		list[lcnt++] = fnp;
+	} else if (fnp < tnp) {
+		list[lcnt++] = fnp;
+		list[lcnt++] = tnp;
+	} else {
+		list[lcnt++] = tnp;
+		list[lcnt++] = fnp;
+	}
+
+	/* Now we can lock using list[0 - lcnt-1] */
+	for (i = 0; i < lcnt; ++i) {
+		if (list[i])
+			if ((error = nfs_lock(list[i], locktype))) {
+				/* Drop any locks we acquired. */
+				while (--i >= 0) {
+					if (list[i])
+						nfs_unlock(list[i]);
+				}
+				return (error);
+			}
+	}
+	return (0);
+}
+
+/*
+ * Unlock a group of NFS nodes
+ */
+void
+nfs_unlock4(nfsnode_t np1, nfsnode_t np2, nfsnode_t np3, nfsnode_t np4)
+{
+	nfsnode_t list[4];
+	int i, k = 0;
+	
+	if (np1) {
+		nfs_unlock(np1);
+		list[k++] = np1;
+	}
+	if (np2) {
+		for (i = 0; i < k; ++i)
+			if (list[i] == np2)
+				goto skip2;
+		nfs_unlock(np2);
+		list[k++] = np2;
+	}
+skip2:
+	if (np3) {
+		for (i = 0; i < k; ++i)
+			if (list[i] == np3)
+				goto skip3;
+		nfs_unlock(np3);
+		list[k++] = np3;
+	}
+skip3:
+	if (np4) {
+		for (i = 0; i < k; ++i)
+			if (list[i] == np4)
+				return;
+		nfs_unlock(np4);
+	}
+}
+
+/*
+ * Acquire an NFS node data lock
+ */
+void
+nfs_data_lock(nfsnode_t np, int locktype)
+{
+	nfs_data_lock2(np, locktype, 1);
+}
+void
+nfs_data_lock2(nfsnode_t np, int locktype, int updatesize)
+{
+	FSDBG_TOP(270, np, locktype, np->n_datalockowner, 0);
+	if (locktype == NFS_NODE_LOCK_SHARED) {
+		if (updatesize && ISSET(np->n_flag, NUPDATESIZE))
+			nfs_data_update_size(np, 0);
+		lck_rw_lock_shared(&np->n_datalock);
+	} else {
+		lck_rw_lock_exclusive(&np->n_datalock);
+		np->n_datalockowner = current_thread();
+		if (updatesize && ISSET(np->n_flag, NUPDATESIZE))
+			nfs_data_update_size(np, 1);
+	}
+	FSDBG_BOT(270, np, locktype, np->n_datalockowner, 0);
+}
+
+/*
+ * Release an NFS node data lock
+ */
+void
+nfs_data_unlock(nfsnode_t np)
+{
+	nfs_data_unlock2(np, 1);
+}
+void
+nfs_data_unlock2(nfsnode_t np, int updatesize)
+{
+	int mine = (np->n_datalockowner == current_thread());
+	FSDBG_TOP(271, np, np->n_datalockowner, current_thread(), 0);
+	if (updatesize && mine && ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 1);
+	np->n_datalockowner = NULL;
+	lck_rw_done(&np->n_datalock);
+	if (updatesize && !mine && ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 0);
+	FSDBG_BOT(271, np, np->n_datalockowner, current_thread(), 0);
+}
+
+
+/*
+ * update an NFS node's size
+ */
+void
+nfs_data_update_size(nfsnode_t np, int datalocked)
+{
+	int error;
+
+	FSDBG_TOP(272, np, np->n_flag, np->n_size, np->n_newsize);
+	if (!datalocked) {
+		nfs_data_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+		/* grabbing data lock will automatically update size */
+		nfs_data_unlock(np);
+		FSDBG_BOT(272, np, np->n_flag, np->n_size, np->n_newsize);
+		return;
+	}
+	error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+	if (error || !ISSET(np->n_flag, NUPDATESIZE)) {
+		if (!error)
+			nfs_unlock(np);
+		FSDBG_BOT(272, np, np->n_flag, np->n_size, np->n_newsize);
+		return;
+	}
+	CLR(np->n_flag, NUPDATESIZE);
+	np->n_size = np->n_newsize;
+	/* make sure we invalidate buffers the next chance we get */
+	SET(np->n_flag, NNEEDINVALIDATE);
+	nfs_unlock(np);
+	ubc_setsize(NFSTOV(np), (off_t)np->n_size); /* XXX error? */
+	FSDBG_BOT(272, np, np->n_flag, np->n_size, np->n_newsize);
+}
+
diff --git a/bsd/nfs/nfs_serv.c b/bsd/nfs/nfs_serv.c
index 2d7a33992..b7be99468 100644
--- a/bsd/nfs/nfs_serv.c
+++ b/bsd/nfs/nfs_serv.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,26 +65,6 @@
  * FreeBSD-Id: nfs_serv.c,v 1.52 1997/10/28 15:59:05 bde Exp $
  */
 
-/*
- * nfs version 2 and 3 server calls to vnode ops
- * - these routines generally have 3 phases
- *   1 - break down and validate rpc request in mbuf list
- *   2 - do the vnode ops for the request
- *       (surprisingly ?? many are very similar to syscalls in vfs_syscalls.c)
- *   3 - build the rpc reply in an mbuf list
- *   nb:
- *	- do not mix the phases, since the nfsm_?? macros can return failures
- *	  on a bad rpc or similar and do not do any vnode_rele()s or vnode_put()s
- *
- *      - the nfsm_reply() macro generates an nfs rpc reply with the nfs
- *	error number iff error != 0 whereas
- *	returning an error from the server function implies a fatal error
- *	such as a badly constructed rpc request that should be dropped without
- *	a reply.
- *	For Version 3, nfsm_reply() does not return for the error case, since
- *	most version 3 rpcs return more than the status for error cases.
- */
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
@@ -93,11 +79,12 @@
 #include <sys/dirent.h>
 #include <sys/stat.h>
 #include <sys/kernel.h>
-#include <sys/sysctl.h>
 #include <sys/ubc.h>
 #include <sys/vnode_internal.h>
 #include <sys/uio_internal.h>
 #include <libkern/OSAtomic.h>
+#include <sys/fsevents.h>
+#include <kern/thread_call.h>
 
 #include <sys/vm.h>
 #include <sys/vmparam.h>
@@ -107,78 +94,207 @@
 #include <nfs/nfs.h>
 #include <nfs/xdr_subs.h>
 #include <nfs/nfsm_subs.h>
+#include <nfs/nfsrvcache.h>
+#include <nfs/nfs_gss.h>
 
-nfstype nfsv3_type[9] = { NFNON, NFREG, NFDIR, NFBLK, NFCHR, NFLNK, NFSOCK,
-		      NFFIFO, NFNON };
-#ifndef NFS_NOSERVER 
-nfstype nfsv2_type[9] = { NFNON, NFREG, NFDIR, NFBLK, NFCHR, NFLNK, NFNON,
-		      NFCHR, NFNON };
-/* Global vars */
-extern u_long nfs_xdrneg1;
-extern u_long nfs_false, nfs_true;
-extern enum vtype nv3tov_type[8];
-extern struct nfsstats nfsstats;
-
-int nfsrvw_procrastinate = NFS_GATHERDELAY * 1000;
-int nfsrvw_procrastinate_v3 = 0;
-
-int nfs_async = 0;
-#ifdef notyet
-/* XXX CSM 11/25/97 Upgrade sysctl.h someday */
-SYSCTL_INT(_vfs_nfs, OID_AUTO, async, CTLFLAG_RW, &nfs_async, 0, "");
-#endif
+#if NFSSERVER
+
+/*
+ * NFS server globals
+ */
+
+int nfsd_thread_count = 0;
+int nfsd_thread_max = 0;
+lck_grp_t *nfsd_lck_grp;
+lck_mtx_t *nfsd_mutex;
+struct nfsd_head nfsd_head, nfsd_queue;
+
+lck_grp_t *nfsrv_slp_rwlock_group;
+lck_grp_t *nfsrv_slp_mutex_group;
+struct nfsrv_sockhead nfsrv_socklist, nfsrv_deadsocklist, nfsrv_sockwg,
+			nfsrv_sockwait, nfsrv_sockwork;
+struct nfsrv_sock *nfsrv_udpsock = NULL;
+
+/* NFS exports */
+struct nfsrv_expfs_list nfsrv_exports;
+struct nfsrv_export_hashhead *nfsrv_export_hashtbl;
+u_long nfsrv_export_hash;
+lck_grp_t *nfsrv_export_rwlock_group;
+lck_rw_t nfsrv_export_rwlock;
+
+/* NFS server file modification event generator */
+struct nfsrv_fmod_hashhead *nfsrv_fmod_hashtbl;
+u_long nfsrv_fmod_hash;
+lck_grp_t *nfsrv_fmod_grp;
+lck_mtx_t *nfsrv_fmod_mutex;
+static int nfsrv_fmod_timer_on = 0;
+
+int nfsrv_fsevents_enabled = 1;
+
+/* NFS server timers */
+thread_call_t	nfsrv_fmod_timer_call;
+thread_call_t	nfsrv_deadsock_timer_call;
+thread_call_t	nfsrv_wg_timer_call;
+int nfsrv_wg_timer_on;
+
+/* globals for the active user list */
+uint32_t nfsrv_user_stat_enabled = 1;
+uint32_t nfsrv_user_stat_node_count = 0;
+uint32_t nfsrv_user_stat_max_idle_sec = NFSRV_USER_STAT_DEF_IDLE_SEC;
+uint32_t nfsrv_user_stat_max_nodes = NFSRV_USER_STAT_DEF_MAX_NODES;
+lck_grp_t *nfsrv_active_user_mutex_group;
+
+int nfsrv_wg_delay = NFSRV_WGATHERDELAY * 1000;
+int nfsrv_wg_delay_v3 = 0;
+
+int nfsrv_async = 0;
 
 static int nfsrv_authorize(vnode_t,vnode_t,kauth_action_t,vfs_context_t,struct nfs_export_options*,int);
-static void nfsrvw_coalesce(struct nfsrv_descript *, struct nfsrv_descript *);
+static int nfsrv_wg_coalesce(struct nfsrv_descript *, struct nfsrv_descript *);
+
+extern void IOSleep(int);
+
+/*
+ * Initialize the data structures for the server.
+ */
+
+#define NFSRV_NOT_INITIALIZED	0
+#define NFSRV_INITIALIZING	1
+#define NFSRV_INITIALIZED	2
+static volatile UInt32 nfsrv_initted = NFSRV_NOT_INITIALIZED;
 
-#define THREAD_SAFE_FS(VP)  \
-        ((VP)->v_mount ? (VP)->v_mount->mnt_vtable->vfc_threadsafe : 0)
+int
+nfsrv_is_initialized(void)
+{
+	return (nfsrv_initted == NFSRV_INITIALIZED);
+}
+
+void
+nfsrv_init(void)
+{
+	/* make sure we init only once */
+	if (!OSCompareAndSwap(NFSRV_NOT_INITIALIZED, NFSRV_INITIALIZING, &nfsrv_initted)) {
+		/* wait until initialization is complete */
+		while (!nfsrv_is_initialized())
+			IOSleep(500);
+		return;
+	}
+
+	if (sizeof (struct nfsrv_sock) > NFS_SVCALLOC) {
+		printf("struct nfsrv_sock bloated (> %dbytes)\n",NFS_SVCALLOC);
+		printf("Try reducing NFS_UIDHASHSIZ\n");
+	}
+
+	/* init nfsd mutex */
+	nfsd_lck_grp = lck_grp_alloc_init("nfsd", LCK_GRP_ATTR_NULL);
+	nfsd_mutex = lck_mtx_alloc_init(nfsd_lck_grp, LCK_ATTR_NULL);
+
+	/* init slp rwlock */
+	nfsrv_slp_rwlock_group = lck_grp_alloc_init("nfsrv-slp-rwlock", LCK_GRP_ATTR_NULL);
+	nfsrv_slp_mutex_group  = lck_grp_alloc_init("nfsrv-slp-mutex", LCK_GRP_ATTR_NULL);
+
+	/* init export data structures */
+	nfsrv_export_hashtbl = hashinit(8, M_TEMP, &nfsrv_export_hash);
+	LIST_INIT(&nfsrv_exports);
+	nfsrv_export_rwlock_group = lck_grp_alloc_init("nfsrv-export-rwlock", LCK_GRP_ATTR_NULL);
+	lck_rw_init(&nfsrv_export_rwlock, nfsrv_export_rwlock_group, LCK_ATTR_NULL);
+
+	/* init active user list mutex structures */
+	nfsrv_active_user_mutex_group = lck_grp_alloc_init("nfs-active-user-mutex", LCK_GRP_ATTR_NULL);
+
+	/* init nfs server request cache mutex */
+	nfsrv_reqcache_lck_grp = lck_grp_alloc_init("nfsrv_reqcache", LCK_GRP_ATTR_NULL);
+	nfsrv_reqcache_mutex = lck_mtx_alloc_init(nfsrv_reqcache_lck_grp, LCK_ATTR_NULL);
+
+	/* init NFS server file modified event generation */
+	nfsrv_fmod_hashtbl = hashinit(NFSRVFMODHASHSZ, M_TEMP, &nfsrv_fmod_hash);
+	nfsrv_fmod_grp = lck_grp_alloc_init("nfsrv_fmod", LCK_GRP_ATTR_NULL);
+	nfsrv_fmod_mutex = lck_mtx_alloc_init(nfsrv_fmod_grp, LCK_ATTR_NULL);
+
+	/* initialize NFS server timer callouts */
+	nfsrv_fmod_timer_call = thread_call_allocate(nfsrv_fmod_timer, NULL);
+	nfsrv_deadsock_timer_call = thread_call_allocate(nfsrv_deadsock_timer, NULL);
+	nfsrv_wg_timer_call = thread_call_allocate(nfsrv_wg_timer, NULL);
+
+	/* Init server data structures */
+	TAILQ_INIT(&nfsrv_socklist);
+	TAILQ_INIT(&nfsrv_sockwait);
+	TAILQ_INIT(&nfsrv_sockwork);
+	TAILQ_INIT(&nfsrv_deadsocklist);
+	TAILQ_INIT(&nfsrv_sockwg);
+	TAILQ_INIT(&nfsd_head);
+	TAILQ_INIT(&nfsd_queue);
+	nfsrv_udpsock = NULL;
+
+	/* initialization complete */
+	nfsrv_initted = NFSRV_INITIALIZED;
+}
+
+
+/*
+ *
+ * NFS version 2 and 3 server request processing functions
+ *
+ * These functions take the following parameters:
+ *
+ *      struct nfsrv_descript *nd - the NFS request descriptor
+ *      struct nfsrv_sock *slp    - the NFS socket the request came in on
+ *      vfs_context_t ctx         - VFS context
+ *      mbuf_t *mrepp             - pointer to hold the reply mbuf list
+ *
+ * These routines generally have 3 phases:
+ *
+ *   1 - break down and validate the RPC request in the mbuf chain
+ *       provided in nd->nd_nmreq.
+ *   2 - perform the vnode operations for the request
+ *       (many are very similar to syscalls in vfs_syscalls.c and
+ *       should therefore be kept in sync with those implementations)
+ *   3 - build the RPC reply in an mbuf chain (nmrep) and return the mbuf chain
+ *
+ */
 
 /*
  * nfs v3 access service
  */
 int
-nfsrv3_access(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_access(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
+	struct nfsm_chain *nmreq, nmrep;
 	vnode_t vp;
+	int error, attrerr;
+	struct vnode_attr vattr;
 	struct nfs_filehandle nfh;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, getret;
-	char *cp2;
-	mbuf_t mb, mreq, mb2;
-	struct vnode_attr vattr, *vap = &vattr;
 	u_long nfsmode;
 	kauth_action_t testaction;
-	struct vfs_context context;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
 
-	nfsm_srvmtofh(&nfh);
-	nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
-	nfsmode = fxdr_unsigned(u_long, *tl);
+	error = 0;
+	attrerr = ENOENT;
+	nfsmode = 0;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	*mrepp = NULL;
+	vp = NULL;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, NFS_VER3, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, nfsmode);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
 
 	/*
 	 * Each NFS mode bit is tested separately.
@@ -188,7 +304,7 @@ nfsrv3_access(nfsd, slp, procp, mrq)
 	 *     an NFS-specific interface to the vnode_authorize code to
 	 *     obtain good performance in the optimistic mode.
 	 */
-	if (nfsmode & NFSV3ACCESS_READ) {
+	if (nfsmode & NFS_ACCESS_READ) {
 		if (vnode_isdir(vp)) {
 			testaction =
 			    KAUTH_VNODE_LIST_DIRECTORY |
@@ -198,14 +314,14 @@ nfsrv3_access(nfsd, slp, procp, mrq)
 			    KAUTH_VNODE_READ_DATA |
 			    KAUTH_VNODE_READ_EXTATTRIBUTES;
 		}
-		if (nfsrv_authorize(vp, NULL, testaction, &context, nxo, 0))
-			nfsmode &= ~NFSV3ACCESS_READ;
+		if (nfsrv_authorize(vp, NULL, testaction, ctx, nxo, 0))
+			nfsmode &= ~NFS_ACCESS_READ;
 	}
-	if ((nfsmode & NFSV3ACCESS_LOOKUP) &&
+	if ((nfsmode & NFS_ACCESS_LOOKUP) &&
 	    (!vnode_isdir(vp) ||
-	    nfsrv_authorize(vp, NULL, KAUTH_VNODE_SEARCH, &context, nxo, 0)))
-		nfsmode &= ~NFSV3ACCESS_LOOKUP;
-	if (nfsmode & NFSV3ACCESS_MODIFY) {
+	    nfsrv_authorize(vp, NULL, KAUTH_VNODE_SEARCH, ctx, nxo, 0)))
+		nfsmode &= ~NFS_ACCESS_LOOKUP;
+	if (nfsmode & NFS_ACCESS_MODIFY) {
 		if (vnode_isdir(vp)) {
 			testaction =
 			    KAUTH_VNODE_ADD_FILE |
@@ -213,12 +329,12 @@ nfsrv3_access(nfsd, slp, procp, mrq)
 			    KAUTH_VNODE_DELETE_CHILD;
 		} else {
 			testaction =
-                           KAUTH_VNODE_WRITE_DATA;
+			    KAUTH_VNODE_WRITE_DATA;
 		}
-		if (nfsrv_authorize(vp, NULL, testaction, &context, nxo, 0))
-			nfsmode &= ~NFSV3ACCESS_MODIFY;
+		if (nfsrv_authorize(vp, NULL, testaction, ctx, nxo, 0))
+			nfsmode &= ~NFS_ACCESS_MODIFY;
 	}
-	if (nfsmode & NFSV3ACCESS_EXTEND) {
+	if (nfsmode & NFS_ACCESS_EXTEND) {
 		if (vnode_isdir(vp)) {
 			testaction =
 			    KAUTH_VNODE_ADD_FILE |
@@ -228,12 +344,12 @@ nfsrv3_access(nfsd, slp, procp, mrq)
 			    KAUTH_VNODE_WRITE_DATA |
 			    KAUTH_VNODE_APPEND_DATA;
 		}
-		if (nfsrv_authorize(vp, NULL, testaction, &context, nxo, 0))
-			nfsmode &= ~NFSV3ACCESS_EXTEND;
+		if (nfsrv_authorize(vp, NULL, testaction, ctx, nxo, 0))
+			nfsmode &= ~NFS_ACCESS_EXTEND;
 	}
 
 	/*
-	 * Note concerning NFSV3ACCESS_DELETE:
+	 * Note concerning NFS_ACCESS_DELETE:
 	 * For hard links, the answer may be wrong if the vnode
 	 * has multiple parents with different permissions.
 	 * Also, some clients (e.g. MacOSX 10.3) may incorrectly
@@ -243,19 +359,33 @@ nfsrv3_access(nfsd, slp, procp, mrq)
 	 * something it really can't.
 	 */
 
-	if ((nfsmode & NFSV3ACCESS_EXECUTE) &&
+	if ((nfsmode & NFS_ACCESS_EXECUTE) &&
 	    (vnode_isdir(vp) ||
-	    nfsrv_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, &context, nxo, 0)))
-		nfsmode &= ~NFSV3ACCESS_EXECUTE;
-
-	nfsm_srv_vattr_init(vap, 1);
-	getret = vnode_getattr(vp, vap, &context);
-	vnode_put(vp);
-	nfsm_reply(NFSX_POSTOPATTR(1) + NFSX_UNSIGNED);
-	nfsm_srvpostop_attr(getret, vap);
-	nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-	*tl = txdr_unsigned(nfsmode);
+	    nfsrv_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, ctx, nxo, 0)))
+		nfsmode &= ~NFS_ACCESS_EXECUTE;
+
+	/* get postop attributes */
+	nfsm_srv_vattr_init(&vattr, NFS_VER3);
+	attrerr = vnode_getattr(vp, &vattr, ctx);
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(NFS_VER3) + NFSX_UNSIGNED);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &vattr);
+	if (!nd->nd_repstat)
+		nfsm_chain_add_32(error, &nmrep, nfsmode);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (vp)
+		vnode_put(vp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -263,53 +393,61 @@ nfsmout:
  * nfs getattr service
  */
 int
-nfsrv_getattr(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_getattr(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct nfs_fattr *fp;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
+	struct nfsm_chain *nmreq, nmrep;
+	struct vnode_attr vattr;
 	vnode_t vp;
+	int error;
 	struct nfs_filehandle nfh;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	struct vfs_context context;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
 
-	nfsm_srvmtofh(&nfh);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(0);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(0);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	*mrepp = NULL;
+	vp = NULL;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
 
-	nfsm_srv_vattr_init(vap, v3);
-	error = vnode_getattr(vp, vap, &context);
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	nfsm_srv_vattr_init(&vattr, nd->nd_vers);
+	error = vnode_getattr(vp, &vattr, ctx);
 	vnode_put(vp);
-	nfsm_reply(NFSX_FATTR(v3));
-	if (error)
-		return (0);
-	nfsm_build(fp, struct nfs_fattr *, NFSX_FATTR(v3));
-	nfsm_srvfillattr(vap, fp);
+	vp = NULL;
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_FATTR(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_if(nd->nd_repstat);
+	error = nfsm_chain_add_fattr(nd, &nmrep, &vattr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (vp)
+		vnode_put(vp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -317,149 +455,125 @@ nfsmout:
  * nfs setattr service
  */
 int
-nfsrv_setattr(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_setattr(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct vnode_attr preat;
-	struct vnode_attr postat;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
-	struct nfsv2_sattr *sp;
-	struct nfs_fattr *fp;
+	struct nfsm_chain *nmreq, nmrep;
+	struct vnode_attr preattr, postattr;
+	struct vnode_attr vattr, *vap = &vattr;
 	vnode_t vp;
-	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, preat_ret = 1, postat_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3), gcheck = 0;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	struct timespec guard;
-	struct vfs_context context;
+	int error, preattrerr, postattrerr, gcheck;
+	struct nfs_filehandle nfh;
+	struct timespec guard = { 0, 0 };
 	kauth_action_t action;
 	uid_t saved_uid;
 
-	nfsm_srvmtofh(&nfh);
+	error = 0;
+	preattrerr = postattrerr = ENOENT;
+	gcheck = 0;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	*mrepp = NULL;
+	vp = NULL;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+
 	VATTR_INIT(vap);
-	if (v3) {
-		nfsm_srvsattr(vap);
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		gcheck = fxdr_unsigned(int, *tl);
-		if (gcheck) {
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-			fxdr_nfsv3time(tl, &guard);
-		}
-	} else {
-		nfsm_dissect(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		/*
-		 * Nah nah nah nah na nah
-		 * There is a bug in the Sun client that puts 0xffff in the mode
-		 * field of sattr when it should put in 0xffffffff. The u_short
-		 * doesn't sign extend.
-		 * --> check the low order 2 bytes for 0xffff
-		 */
-		if ((fxdr_unsigned(int, sp->sa_mode) & 0xffff) != 0xffff)
-			VATTR_SET(vap, va_mode, nfstov_mode(sp->sa_mode));
-		if (sp->sa_uid != nfs_xdrneg1)
-			VATTR_SET(vap, va_uid, fxdr_unsigned(uid_t, sp->sa_uid));
-		if (sp->sa_gid != nfs_xdrneg1)
-			VATTR_SET(vap, va_gid, fxdr_unsigned(gid_t, sp->sa_gid));
-		if (sp->sa_size != nfs_xdrneg1)
-			VATTR_SET(vap, va_data_size, fxdr_unsigned(u_quad_t, sp->sa_size));
-		if (sp->sa_atime.nfsv2_sec != nfs_xdrneg1) {
-			fxdr_nfsv2time(&sp->sa_atime, &vap->va_access_time);
-			VATTR_SET_ACTIVE(vap, va_access_time);
-		}
-		if (sp->sa_mtime.nfsv2_sec != nfs_xdrneg1) {
-			fxdr_nfsv2time(&sp->sa_mtime, &vap->va_modify_time);
-			VATTR_SET_ACTIVE(vap, va_modify_time);
-		}
+	error = nfsm_chain_get_sattr(nd, nmreq, vap);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_get_32(error, nmreq, gcheck);
+		if (gcheck)
+			nfsm_chain_get_time(error, nmreq, nd->nd_vers, guard.tv_sec, guard.tv_nsec);
 	}
+	nfsmerr_if(error);
 
 	/*
 	 * Save the original credential UID in case they are
 	 * mapped and we need to map the IDs in the attributes.
 	 */
-	saved_uid = kauth_cred_getuid(nfsd->nd_cr);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
 
 	/*
 	 * Now that we have all the fields, lets do it.
 	 */
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(preat_ret, &preat, postat_ret, &postat);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(preat_ret, &preat, postat_ret, &postat);
-		return (0);
-	}
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
 
-	if (v3) {
-		nfsm_srv_pre_vattr_init(&preat, v3);
-		error = preat_ret = vnode_getattr(vp, &preat, &context);
-		if (!error && gcheck && VATTR_IS_SUPPORTED(&preat, va_change_time) &&
-			(preat.va_change_time.tv_sec != guard.tv_sec ||
-			 preat.va_change_time.tv_nsec != guard.tv_nsec))
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_pre_vattr_init(&preattr);
+		error = preattrerr = vnode_getattr(vp, &preattr, ctx);
+		if (!error && gcheck && VATTR_IS_SUPPORTED(&preattr, va_change_time) &&
+			(preattr.va_change_time.tv_sec != guard.tv_sec ||
+			 preattr.va_change_time.tv_nsec != guard.tv_nsec))
 			error = NFSERR_NOT_SYNC;
-		if (!preat_ret && !VATTR_ALL_SUPPORTED(&preat))
-			preat_ret = 1;
-		if (error) {
-			vnode_put(vp);
-			nfsm_reply(NFSX_WCCDATA(v3));
-			nfsm_srvwcc_data(preat_ret, &preat, postat_ret, &postat);
-			return (0);
-		}
+		if (!preattrerr && !VATTR_ALL_SUPPORTED(&preattr))
+			preattrerr = ENOENT;
+		nfsmerr_if(error);
 	}
 
 	/*
 	 * If the credentials were mapped, we should
 	 * map the same values in the attributes.
 	 */
-	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nfsd->nd_cr) != saved_uid)) {
+	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nd->nd_cr) != saved_uid)) {
 		int ismember;
-		VATTR_SET(vap, va_uid, kauth_cred_getuid(nfsd->nd_cr));
-		if (kauth_cred_ismember_gid(nfsd->nd_cr, vap->va_gid, &ismember) || !ismember)
-			VATTR_SET(vap, va_gid, kauth_cred_getgid(nfsd->nd_cr));
+		VATTR_SET(vap, va_uid, kauth_cred_getuid(nd->nd_cr));
+		if (kauth_cred_ismember_gid(nd->nd_cr, vap->va_gid, &ismember) || !ismember)
+			VATTR_SET(vap, va_gid, kauth_cred_getgid(nd->nd_cr));
 	}
 
-	/*
-	 * Authorize the attribute changes.
-	 */
-	if (((error = vnode_authattr(vp, vap, &action, &context))) ||
-	    ((error = nfsrv_authorize(vp, NULL, action, &context, nxo, 0))))
-		goto out;
-	error = vnode_setattr(vp, vap, &context);
+	/* Authorize the attribute changes.  */
+	error = vnode_authattr(vp, vap, &action, ctx);
+	if (!error)
+		error = nfsrv_authorize(vp, NULL, action, ctx, nxo, 0);
 
-	nfsm_srv_vattr_init(&postat, v3);
-	postat_ret = vnode_getattr(vp, &postat, &context);
+	/* set the new attributes */
 	if (!error)
-		error = postat_ret;
-out:
-	vnode_put(vp);
-	nfsm_reply(NFSX_WCCORFATTR(v3));
-	if (v3) {
-		nfsm_srvwcc_data(preat_ret, &preat, postat_ret, &postat);
-		return (0);
-	} else {
-		nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-		nfsm_srvfillattr(&postat, fp);
+		error = vnode_setattr(vp, vap, ctx);
+
+	if (!error || (nd->nd_vers == NFS_VER3)) {
+		nfsm_srv_vattr_init(&postattr, nd->nd_vers);
+		postattrerr = vnode_getattr(vp, &postattr, ctx);
+		if (!error)
+			error = postattrerr;
 	}
+
+nfsmerr:
+	if (vp)
+		vnode_put(vp);
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_WCCORFATTR(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			preattrerr, &preattr, postattrerr, &postattr);
+	else
+		error = nfsm_chain_add_fattr(nd, &nmrep, &postattr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -467,130 +581,93 @@ nfsmout:
  * nfs lookup rpc
  */
 int
-nfsrv_lookup(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_lookup(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct nfs_fattr *fp;
-	struct nameidata nd, *ndp = &nd;
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	struct nameidata ind;
-#endif
+	struct nameidata ni, *nip = &ni;
 	vnode_t vp, dirp = NULL;
 	struct nfs_filehandle dnfh, nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	caddr_t cp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len, dirattr_ret = 1, isdotdot;
-	int v3 = (nfsd->nd_flag & ND_NFSV3), pubflag;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
+	int error, attrerr, dirattrerr, isdotdot;
+	uint32_t len;
+	uid_t saved_uid;
 	struct vnode_attr va, dirattr, *vap = &va;
-	struct vfs_context context;
-
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
-	nfsm_srvmtofh(&dnfh);
-	nfsm_srvnamesiz(len, v3);
-
-	pubflag = nfs_ispublicfh(&dnfh);
-
-	nd.ni_cnd.cn_nameiop = LOOKUP;
-	nd.ni_cnd.cn_flags = LOCKLEAF;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, pubflag, &len, &nd);
-	isdotdot = ((len == 2) && (nd.ni_cnd.cn_pnbuf[0] == '.') && (nd.ni_cnd.cn_pnbuf[1] == '.'));
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &dnfh, nam, pubflag, &dirp, &nx, &nxo);
-
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	if (!error && pubflag) {
-		if (vnode_vtype(nd.ni_vp) == VDIR && nfs_pub.np_index != NULL) {
-			/*
-			 * Setup call to lookup() to see if we can find
-			 * the index file. Arguably, this doesn't belong
-			 * in a kernel.. Ugh.
-			 */
-			ind = nd;
-			ind.ni_pathlen = strlen(nfs_pub.np_index);
-			ind.ni_cnd.cn_nameptr = ind.ni_cnd.cn_pnbuf =
-			    nfs_pub.np_index;
-			ind.ni_startdir = nd.ni_vp;
-			ind.ni_usedvp   = nd.ni_vp;
-
-			if (!(error = lookup(&ind))) {
-				/*
-				 * Found an index file. Get rid of
-				 * the old references.
-				 */
-				if (dirp)	
-					vnode_put(dirp);
-				dirp = nd.ni_vp;
-				vnode_put(nd.ni_startdir);
-				ndp = &ind;
-			} else
-				error = 0;
-		}
-		/*
-		 * If the public filehandle was used, check that this lookup
-		 * didn't result in a filehandle outside the publicly exported
-		 * filesystem.
-		 */
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	attrerr = dirattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, dnfh.nfh_fhp, dnfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+
+	ni.ni_cnd.cn_nameiop = LOOKUP;
+	ni.ni_cnd.cn_flags = LOCKLEAF;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	isdotdot = ((len == 2) && (ni.ni_cnd.cn_pnbuf[0] == '.') && (ni.ni_cnd.cn_pnbuf[1] == '.'));
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &dnfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
 
-		if (!error && vnode_mount(ndp->ni_vp) != nfs_pub.np_mount) {
-			vnode_put(nd.ni_vp);
-			nameidone(&nd);
-			error = EPERM;
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
 		}
 	}
-#endif
 
 	if (dirp) {
-		if (v3) {
-			nfsm_srv_vattr_init(&dirattr, v3);
-			dirattr_ret = vnode_getattr(dirp, &dirattr, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_vattr_init(&dirattr, NFS_VER3);
+			dirattrerr = vnode_getattr(dirp, &dirattr, ctx);
 		}
 		vnode_put(dirp);
 	}
+	nfsmerr_if(error);
 
-	if (error) {
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(dirattr_ret, &dirattr);
-		return (0);
-	}
-	nameidone(&nd);
+	nameidone(&ni);
 
-	vp = ndp->ni_vp;
-	error = nfsrv_vptofh(nx, !v3, (isdotdot ? &dnfh : NULL), vp, &context, &nfh);
+	vp = nip->ni_vp;
+	error = nfsrv_vptofh(nx, nd->nd_vers, (isdotdot ? &dnfh : NULL), vp, ctx, &nfh);
 	if (!error) {
-		nfsm_srv_vattr_init(vap, v3);
-		error = vnode_getattr(vp, vap, &context);
+		nfsm_srv_vattr_init(vap, nd->nd_vers);
+		attrerr = vnode_getattr(vp, vap, ctx);
 	}
 	vnode_put(vp);
-	nfsm_reply(NFSX_SRVFH(v3, &nfh) + NFSX_POSTOPORFATTR(v3) + NFSX_POSTOPATTR(v3));
-	if (error) {
-		nfsm_srvpostop_attr(dirattr_ret, &dirattr);
-		return (0);
-	}
-	nfsm_srvfhtom(&nfh, v3);
-	if (v3) {
-		nfsm_srvpostop_attr(0, vap);
-		nfsm_srvpostop_attr(dirattr_ret, &dirattr);
-	} else {
-		nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-		nfsm_srvfillattr(vap, fp);
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_SRVFH(nd->nd_vers, &nfh) +
+			NFSX_POSTOPORFATTR(nd->nd_vers) + NFSX_POSTOPATTR(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	if (nd->nd_repstat) {
+		if (nd->nd_vers == NFS_VER3)
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, dirattrerr, &dirattr);
+		goto nfsmout;
+	}
+	nfsm_chain_add_fh(error, &nmrep, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, vap);
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, dirattrerr, &dirattr);
+	} else if (!error) {
+		error = nfsm_chain_add_fattr(nd, &nmrep, vap);
 	}
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -598,162 +675,126 @@ nfsmout:
  * nfs readlink service
  */
 int
-nfsrv_readlink(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_readlink(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	mbuf_t mp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, i, tlen, len, getret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mb2, mp2, mp3, mreq;
+	int error, mpcnt, tlen, len, attrerr;
 	vnode_t vp;
-	struct vnode_attr attr;
+	struct vnode_attr vattr;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
+	struct nfsm_chain *nmreq, nmrep;
+	mbuf_t mpath, mp;
 	uio_t uiop = NULL;
 	char uio_buf[ UIO_SIZEOF(4) ];
 	char *uio_bufp = &uio_buf[0];
 	int uio_buflen = UIO_SIZEOF(4);
-	int mblen;
-	struct vfs_context context;
-
-	nfsm_srvmtofh(&nfh);
-	len = 0;
-	i = 0;
 
-	mp2 = mp3 = NULL;
+	error = 0;
+	attrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	mpath = NULL;
 	vp = NULL;
-	while (len < NFS_MAXPATHLEN) {
-		mp = NULL;
-		if ((error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mp)))
-			goto out;
-		mblen = mbuf_maxlen(mp);
-		mbuf_setlen(mp, mblen);
-		if (len == 0)
-			mp3 = mp2 = mp;
-		else {
-			if ((error = mbuf_setnext(mp2, mp))) {
-				mbuf_free(mp);
-				goto out;
-			}
-			mp2 = mp;
-		}
-		if ((len + mblen) > NFS_MAXPATHLEN) {
-			mbuf_setlen(mp, NFS_MAXPATHLEN - len);
-			len = NFS_MAXPATHLEN;
-		} else
-			len += mblen;
-  		i++;
-	}
-	if (i > 4) {
-		uio_buflen = UIO_SIZEOF(i);
+	len = NFS_MAXPATHLEN;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+
+	/* get mbuf list to hold symlink path */
+	error = nfsm_mbuf_get_list(len, &mpath, &mpcnt);
+	nfsmerr_if(error);
+	if (mpcnt > 4) {
+		uio_buflen = UIO_SIZEOF(mpcnt);
 		MALLOC(uio_bufp, char*, uio_buflen, M_TEMP, M_WAITOK);
-		if (!uio_bufp) {
+		if (!uio_bufp)
 			error = ENOMEM;
-			mbuf_freem(mp3);
-			nfsm_reply(2 * NFSX_UNSIGNED);
-			nfsm_srvpostop_attr(1, NULL);
-			return (0);
-		}
+		nfsmerr_if(error);
 	}
-	uiop = uio_createwithbuffer(i, 0, UIO_SYSSPACE, UIO_READ, uio_bufp, uio_buflen);
-	if (!uiop) {
+	uiop = uio_createwithbuffer(mpcnt, 0, UIO_SYSSPACE, UIO_READ, uio_bufp, uio_buflen);
+	if (!uiop)
 		error = ENOMEM;
-		mbuf_freem(mp3);
-		if (uio_bufp != &uio_buf[0]) {
-			FREE(uio_bufp, M_TEMP);
-			uio_bufp = &uio_buf[0];
-		}
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
-	mp = mp3;
-	while (mp) {
+	nfsmerr_if(error);
+
+	for (mp = mpath; mp; mp = mbuf_next(mp))
 		uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(mp)), mbuf_len(mp));
-		mp = mbuf_next(mp);
-	}
 
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		mbuf_freem(mp3);
-		if (uio_bufp != &uio_buf[0]) {
-			FREE(uio_bufp, M_TEMP);
-			uio_bufp = &uio_buf[0];
-		}
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		mbuf_freem(mp3);
-		if (uio_bufp != &uio_buf[0]) {
-			FREE(uio_bufp, M_TEMP);
-			uio_bufp = &uio_buf[0];
-		}
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
 	if (vnode_vtype(vp) != VLNK) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EINVAL;
 		else
 			error = ENXIO;
-		goto out;
 	}
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
-	if ((error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, &context, nxo, 0)))
-		goto out;
-	error = VNOP_READLINK(vp, uiop, &context);
-out:
+	if (!error)
+		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, ctx, nxo, 0);
+	if (!error)
+		error = VNOP_READLINK(vp, uiop, ctx);
 	if (vp) {
-		if (v3) {
-			nfsm_srv_vattr_init(&attr, v3);
-			getret = vnode_getattr(vp, &attr, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_vattr_init(&vattr, NFS_VER3);
+			attrerr = vnode_getattr(vp, &vattr, ctx);
 		}
 		vnode_put(vp);
+		vp = NULL;
 	}
 	if (error) {
-		mbuf_freem(mp3);
-		mp3 = NULL;
-	}
-	nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_UNSIGNED);
-	if (v3) {
-		nfsm_srvpostop_attr(getret, &attr);
-		if (error) {
-			if (uio_bufp != &uio_buf[0])
-				FREE(uio_bufp, M_TEMP);
-			return (0);
-		}
-	}
-	if (!error) {
-		if (uiop && (uio_resid(uiop) > 0)) {
-			// LP64todo - fix this
-			len -= uio_resid(uiop);
-			tlen = nfsm_rndup(len);
-			nfsm_adj(mp3, NFS_MAXPATHLEN-tlen, tlen-len);
-		}
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-		*tl = txdr_unsigned(len);
-		mbuf_setnext(mb, mp3);
-	}
+		mbuf_freem(mpath);
+		mpath = NULL;
+	}
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers) + NFSX_UNSIGNED);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &vattr);
+	if (error || nd->nd_repstat) {
+		nfsm_chain_build_done(error, &nmrep);
+		goto nfsmout;
+	}
+	if (uiop && (uio_resid(uiop) > 0)) {
+		// LP64todo - fix this
+		len -= uio_resid(uiop);
+		tlen = nfsm_rndup(len);
+		nfsm_adj(mpath, NFS_MAXPATHLEN-tlen, tlen-len);
+	}
+	nfsm_chain_add_32(error, &nmrep, len);
+	nfsm_chain_build_done(error, &nmrep);
+	nfsmout_if(error);
+	error = mbuf_setnext(nmrep.nmc_mcur, mpath);
+	if (!error)
+		mpath = NULL;
 nfsmout:
+	if (vp)
+		vnode_put(vp);
+	if (mpath)
+		mbuf_freem(mpath);
 	if (uio_bufp != &uio_buf[0])
 		FREE(uio_bufp, M_TEMP);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -761,178 +802,96 @@ nfsmout:
  * nfs read service
  */
 int
-nfsrv_read(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_read(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	mbuf_t m;
-	struct nfs_fattr *fp;
-	u_long *tl;
-	long t1;
-	int i;
-	caddr_t bpos;
-	int error = 0, count, len, left, siz, tlen, getret;
-	int v3 = (nfsd->nd_flag & ND_NFSV3), reqlen, maxlen;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	mbuf_t m2;
+	int error, attrerr, mreadcnt;
+	uint32_t reqlen, maxlen, count, len, tlen, left;
+	mbuf_t mread, m;
 	vnode_t vp;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
 	uio_t uiop = NULL;
 	char *uio_bufp = NULL;
-	struct vnode_attr va, *vap = &va;
+	struct vnode_attr vattr, *vap = &vattr;
 	off_t off;
+	uid_t saved_uid;
 	char uio_buf[ UIO_SIZEOF(0) ];
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
 
-	nfsm_srvmtofh(&nfh);
-	if (v3) {
-		nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-		fxdr_hyper(tl, &off);
-	} else {
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		off = (off_t)fxdr_unsigned(u_long, *tl);
-	}
-	nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-	reqlen = fxdr_unsigned(u_long, *tl);
-	maxlen = NFS_SRVMAXDATA(nfsd);
+	error = 0;
+	attrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	mread = NULL;
+	vp = NULL;
+	len = reqlen = 0;
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_get_64(error, nmreq, off);
+	else
+		nfsm_chain_get_32(error, nmreq, off);
+	nfsm_chain_get_32(error, nmreq, reqlen);
+	maxlen = NFS_SRVMAXDATA(nd);
 	if (reqlen > maxlen)
 		reqlen = maxlen;
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
 
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(1, NULL);
-		return (0);
-	}
 	if (vnode_vtype(vp) != VREG) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EINVAL;
 		else
 			error = (vnode_vtype(vp) == VDIR) ? EISDIR : EACCES;
 	}
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
 	if (!error) {
-	    if ((error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, &context, nxo, 1)))
-		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, &context, nxo, 1);
+	    if ((error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, ctx, nxo, 1)))
+		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_EXECUTE, ctx, nxo, 1);
 	}
-	nfsm_srv_vattr_init(vap, v3);
-	getret = vnode_getattr(vp, vap, &context);
+	nfsm_srv_vattr_init(vap, nd->nd_vers);
+	attrerr = vnode_getattr(vp, vap, ctx);
 	if (!error)
-		error = getret;
-	if (error) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(getret, vap);
-		return (0);
-	}
+		error = attrerr;
+	nfsmerr_if(error);
+
 	if ((u_quad_t)off >= vap->va_data_size)
 		count = 0;
 	else if (((u_quad_t)off + reqlen) > vap->va_data_size)
 		count = nfsm_rndup(vap->va_data_size - off);
 	else
 		count = reqlen;
-	nfsm_reply(NFSX_POSTOPORFATTR(v3) + 3 * NFSX_UNSIGNED+nfsm_rndup(count));
-	if (v3) {
-		nfsm_build(tl, u_long *, NFSX_V3FATTR + 4 * NFSX_UNSIGNED);
-		*tl++ = nfs_true;
-		fp = (struct nfs_fattr *)tl;
-		tl += (NFSX_V3FATTR / sizeof (u_long));
-	} else {
-		nfsm_build(tl, u_long *, NFSX_V2FATTR + NFSX_UNSIGNED);
-		fp = (struct nfs_fattr *)tl;
-		tl += (NFSX_V2FATTR / sizeof (u_long));
-	}
+
 	len = left = count;
 	if (count > 0) {
-		/*
-		 * Generate the mbuf list with the uio_iov ref. to it.
-		 */
-		i = 0;
-		m = m2 = mb;
-		while (left > 0) {
-			siz = min(mbuf_trailingspace(m), left);
-			if (siz > 0) {
-				left -= siz;
-				i++;
-			}
-			if (left > 0) {
-				m = NULL;
-				if ((error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &m)))
-					goto errorexit;
-				mbuf_setnext(m2, m);
-				m2 = m;
-			}
-		}
-		MALLOC(uio_bufp, char *, UIO_SIZEOF(i), M_TEMP, M_WAITOK);
-		if (!uio_bufp) {
-			error = ENOMEM;
-			goto errorexit;
-		}
-		uiop = uio_createwithbuffer(i, off, UIO_SYSSPACE, UIO_READ, 
-					uio_bufp, UIO_SIZEOF(i));
-		if (!uiop) {
+		/* get mbuf list to hold read data */
+		error = nfsm_mbuf_get_list(count, &mread, &mreadcnt);
+		nfsmerr_if(error);
+		MALLOC(uio_bufp, char *, UIO_SIZEOF(mreadcnt), M_TEMP, M_WAITOK);
+		if (uio_bufp)
+			uiop = uio_createwithbuffer(mreadcnt, off, UIO_SYSSPACE,
+					UIO_READ, uio_bufp, UIO_SIZEOF(mreadcnt));
+		if (!uio_bufp || !uiop) {
 			error = ENOMEM;
 			goto errorexit;
 		}
-		m = mb;
-		left = count;
-		i = 0;
-		while (left > 0) {
-			if (m == NULL)
-				panic("nfsrv_read iov");
-			siz = min(mbuf_trailingspace(m), left);
-			if (siz > 0) {
-				tlen = mbuf_len(m);
-				uio_addiov(uiop, CAST_USER_ADDR_T((char *)mbuf_data(m) + tlen), siz);
-				mbuf_setlen(m, tlen + siz);
-				left -= siz;
-				i++;
-			}
-			m = mbuf_next(m);
-		}
-		error = VNOP_READ(vp, uiop, IO_NODELOCKED, &context);
-		off = uio_offset(uiop);
-errorexit:
-		/*
-		 * This may seem a little weird that we drop the whole
-		 * successful read if we get an error on the getattr.
-		 * The reason is because we've already set up the reply
-		 * to have postop attrs and omitting these optional bits
-		 * would require shifting all the data in the reply.
-		 *
-		 * It would be more correct if we would simply drop the
-		 * postop attrs if the getattr fails.  We might be able to
-		 * do that easier if we allocated separate mbufs for the data.
-		 */
-		nfsm_srv_vattr_init(vap, v3);
-		if (error || (getret = vnode_getattr(vp, vap, &context))) {
-			if (!error)
-				error = getret;
-			mbuf_freem(mreq);
-			vnode_put(vp);
-			nfsm_reply(NFSX_POSTOPATTR(v3));
-			nfsm_srvpostop_attr(getret, vap);
-			if (uio_bufp != NULL) {
-				FREE(uio_bufp, M_TEMP);
-			}
-			return (0);
-		}
+		for (m = mread; m; m = mbuf_next(m))
+			uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(m)), mbuf_len(m));
+		error = VNOP_READ(vp, uiop, IO_NODELOCKED, ctx);
 	} else {
 		uiop = uio_createwithbuffer(0, 0, UIO_SYSSPACE, UIO_READ, &uio_buf[0], sizeof(uio_buf));
 		if (!uiop) {
@@ -940,422 +899,573 @@ errorexit:
 			goto errorexit;
 		}
 	}
+
+errorexit:
+	if (!error || (nd->nd_vers == NFS_VER3)) {
+		nfsm_srv_vattr_init(vap, nd->nd_vers);
+		attrerr = vnode_getattr(vp, vap, ctx);
+		if (!error && (nd->nd_vers == NFS_VER2))
+			error = attrerr; /* NFSv2 must have attributes to return */
+	}
+	nfsmerr_if(error);
+
 	vnode_put(vp);
-	nfsm_srvfillattr(vap, fp);
+	vp = NULL;
+
+	/* trim off any data not actually read */
 	// LP64todo - fix this
 	len -= uio_resid(uiop);
 	tlen = nfsm_rndup(len);
 	if (count != tlen || tlen != len)
-		nfsm_adj(mb, count - tlen, tlen - len);
-	if (v3) {
-		*tl++ = txdr_unsigned(len);
-		if (len < reqlen)
-			*tl++ = nfs_true;
-		else
-			*tl++ = nfs_false;
+		nfsm_adj(mread, count - tlen, tlen - len);
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPORFATTR(nd->nd_vers) + 3 * NFSX_UNSIGNED);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, vap);
+	if (error || nd->nd_repstat) {
+		nfsm_chain_build_done(error, &nmrep);
+		goto nfsmout;
+	}
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_32(error, &nmrep, len);
+		nfsm_chain_add_32(error, &nmrep, (len < reqlen) ? TRUE : FALSE);
+	} else {
+		error = nfsm_chain_add_fattr(nd, &nmrep, vap);
 	}
-	*tl = txdr_unsigned(len);
+	nfsm_chain_add_32(error, &nmrep, len);
+	nfsm_chain_build_done(error, &nmrep);
+	nfsmout_if(error);
+	error = mbuf_setnext(nmrep.nmc_mcur, mread);
+	if (!error)
+		mread = NULL;
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.bytes_read, len);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, saved_uid, 1, len, 0);
 nfsmout:
-	if (uio_bufp != NULL) {
+	if (vp)
+		vnode_put(vp);
+	if (mread)
+		mbuf_freem(mread);
+	if (uio_bufp != NULL)
 		FREE(uio_bufp, M_TEMP);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
 	}
 	return (error);
 }
 
+/*
+ * NFS File modification reporting
+ *
+ * When the contents of a file are changed, a "content modified"
+ * fsevent needs to be issued.  Normally this would be done at
+ * file close time.  This is difficult for NFS because the protocol
+ * has no "close" operation.  The client sends a stream of write
+ * requests that just stop.  So we keep a hash table full of
+ * vnodes that have been written to recently, and issue a
+ * "content modified" fsevent only if there are no writes to
+ * a vnode for nfsrv_fmod_pendtime milliseconds.
+ */
+int nfsrv_fmod_pending;		/* count of vnodes being written to */
+int nfsrv_fmod_pendtime = 1000;	/* msec to wait */
+int nfsrv_fmod_min_interval = 100;	/* msec min interval between callbacks */
+
+/*
+ * This function is called via the kernel's callout
+ * mechanism.  Calls are made only when there are
+ * vnodes pending a fsevent creation, and no more
+ * frequently than every nfsrv_fmod_min_interval ms.
+ */
+void
+nfsrv_fmod_timer(__unused void *param0, __unused void *param1)
+{
+	struct nfsrv_fmod_hashhead *head;
+	struct nfsrv_fmod *fp, *nfp;
+	uint64_t timenow, next_deadline;
+	int interval = 0;
+	int i;
+
+	lck_mtx_lock(nfsrv_fmod_mutex);
+	clock_get_uptime(&timenow);
+	clock_interval_to_deadline(nfsrv_fmod_pendtime, 1000 * 1000,
+		&next_deadline);
+
+	/*
+	 * Scan all the hash chains
+	 */
+	for (i = 0; i < NFSRVFMODHASHSZ; i++) {
+		/*
+		 * For each hash chain, look for an entry
+		 * that has exceeded the deadline.
+		 */
+		head = &nfsrv_fmod_hashtbl[i];
+		LIST_FOREACH(fp, head, fm_link) {
+			if (timenow >= fp->fm_deadline)
+				break;
+			if (fp->fm_deadline < next_deadline)
+				next_deadline = fp->fm_deadline;
+		}
+
+		/*
+		 * If we have an entry that's exceeded the
+		 * deadline, then the same is true for all
+		 * following entries in the chain, since they're
+		 * sorted in time order.
+		 */
+		while (fp) {
+			/*
+			 * Fire off the content modified fsevent for each
+			 * entry, remove it from the list, and free it.
+			 */
+#if CONFIG_FSE
+			if (nfsrv_fsevents_enabled)
+				add_fsevent(FSE_CONTENT_MODIFIED, &fp->fm_context,
+					FSE_ARG_VNODE, fp->fm_vp,
+					FSE_ARG_DONE);
+#endif
+			vnode_put(fp->fm_vp);
+			kauth_cred_unref(&fp->fm_context.vc_ucred);
+			nfp = LIST_NEXT(fp, fm_link);
+			LIST_REMOVE(fp, fm_link);
+			FREE(fp, M_TEMP);
+			nfsrv_fmod_pending--;
+			fp = nfp;
+		}
+	}
+
+	/*
+	 * If there are still pending entries, set up another
+	 * callout to handle them later. Set the timeout deadline
+	 * so that the callout happens when the oldest pending
+	 * entry is ready to send its fsevent.
+	 */
+	if (nfsrv_fmod_pending > 0) {
+		interval = (next_deadline - timenow) / (1000 * 1000);
+		if (interval < nfsrv_fmod_min_interval)
+			interval = nfsrv_fmod_min_interval;
+	}
+
+	nfsrv_fmod_timer_on = interval > 0;
+	if (nfsrv_fmod_timer_on)
+		nfs_interval_timer_start(nfsrv_fmod_timer_call, interval);
+
+	lck_mtx_unlock(nfsrv_fmod_mutex);
+}
+
+#if CONFIG_FSE
+/*
+ * When a vnode has been written to, enter it in the hash
+ * table of vnodes pending creation of an fsevent. If the
+ * callout timer isn't already running, schedule a callback
+ * for nfsrv_fmod_pendtime msec from now.
+ */
+static void
+nfsrv_modified(vnode_t vp, vfs_context_t ctx)
+{
+	uint64_t deadline;
+	struct nfsrv_fmod *fp;
+	struct nfsrv_fmod_hashhead *head;
+
+	lck_mtx_lock(nfsrv_fmod_mutex);
+
+	/*
+	 * Compute the time in the future when the
+	 * content modified fsevent is to be issued.
+	 */
+	clock_interval_to_deadline(nfsrv_fmod_pendtime, 1000 * 1000, &deadline);
+
+	/*
+	 * Check if there's already a file content change fsevent
+	 * pending for this vnode.  If there is, update its
+	 * timestamp and make sure it's at the front of the hash chain.
+	 */
+	head = &nfsrv_fmod_hashtbl[NFSRVFMODHASH(vp)];
+	LIST_FOREACH(fp, head, fm_link) {
+		if (vp == fp->fm_vp) {
+			fp->fm_deadline = deadline;
+			if (fp != LIST_FIRST(head)) {
+				LIST_REMOVE(fp, fm_link);
+				LIST_INSERT_HEAD(head, fp, fm_link);
+			}
+			lck_mtx_unlock(nfsrv_fmod_mutex);
+			return;
+		}
+	}
+
+	/*
+	 * First content change fsevent for this vnode.
+	 * Allocate a new file mod entry and add it
+	 * on the front of the hash chain.
+	 */
+	if (vnode_get(vp) != 0)
+		goto done;
+	MALLOC(fp, struct nfsrv_fmod *, sizeof(*fp), M_TEMP, M_WAITOK);
+	if (fp == NULL) {
+		vnode_put(vp);
+		goto done;
+	}
+	fp->fm_vp = vp;
+	kauth_cred_ref(vfs_context_ucred(ctx));
+	fp->fm_context = *ctx;
+	fp->fm_deadline = deadline;
+	LIST_INSERT_HEAD(head, fp, fm_link);
+
+	/*
+	 * If added to an empty hash table, then set the
+	 * callout timer to go off after nfsrv_fmod_pendtime.
+	 */
+	nfsrv_fmod_pending++;
+	if (!nfsrv_fmod_timer_on) {
+		nfsrv_fmod_timer_on = 1;
+		nfs_interval_timer_start(nfsrv_fmod_timer_call,
+			nfsrv_fmod_pendtime);
+	}
+done:
+	lck_mtx_unlock(nfsrv_fmod_mutex);
+	return;
+}
+#endif /* CONFIG_FSE */
+
 /*
  * nfs write service
  */
 int
-nfsrv_write(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_write(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	int i, count;
-	mbuf_t mp;
-	struct nfs_fattr *fp;
-	struct vnode_attr va, forat;
-	struct vnode_attr *vap = &va;
-	u_long *tl;
-	long t1;
-	caddr_t bpos, tpos;
-	int error = 0, len, forat_ret = 1;
-	int ioflags, aftat_ret = 1, retlen, zeroing, adjust, tlen;
-	int stable = NFSV3WRITE_FILESYNC;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
+	struct vnode_attr preattr, postattr;
+	int error, preattrerr, postattrerr;
+	int ioflags, len, retlen;
+	int mlen, mcount;
+	int stable = NFS_WRITE_FILESYNC;
+	mbuf_t m;
 	vnode_t vp;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	uio_t uiop;
-	off_t off;
+	uio_t uiop = NULL;
 	char *uio_bufp = NULL;
-	struct vfs_context context;
+	off_t off;
+	uid_t saved_uid;
+	struct nfsm_chain *nmreq, nmrep;
 
-	if (mrep == NULL) {
-		*mrq = NULL;
+	if (nd->nd_nmreq.nmc_mhead == NULL) {
+		*mrepp = NULL;
 		return (0);
 	}
-	nfsm_srvmtofh(&nfh);
-	if (v3) {
-		nfsm_dissect(tl, u_long *, 5 * NFSX_UNSIGNED);
-		fxdr_hyper(tl, &off);
-		tl += 3;
-		stable = fxdr_unsigned(int, *tl++);
+
+	error = 0;
+	preattrerr = postattrerr = ENOENT;
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
+	len = retlen = 0;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_get_64(error, nmreq, off);
+		nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, nmreq, stable);
 	} else {
-		nfsm_dissect(tl, u_long *, 4 * NFSX_UNSIGNED);
-		off = (off_t)fxdr_unsigned(u_long, *++tl);
-		tl += 2;
-		if (nfs_async)
-	    		stable = NFSV3WRITE_UNSTABLE;
+		nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, nmreq, off);
+		nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		if (nfsrv_async)
+	    		stable = NFS_WRITE_UNSTABLE;
 	}
-	retlen = len = fxdr_unsigned(long, *tl);
-	count = i = 0;
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsmerr_if(error);
+	retlen = len;
 
 	/*
 	 * For NFS Version 2, it is not obvious what a write of zero length
 	 * should do, but I might as well be consistent with Version 3,
 	 * which is to return ok so long as there are no permission problems.
 	 */
+
 	if (len > 0) {
-	    zeroing = 1;
-	    mp = mrep;
-	    while (mp) {
-		if (mp == md) {
-			zeroing = 0;
-			tpos = mbuf_data(mp);
-			tlen = mbuf_len(mp);
-			adjust = dpos - tpos;
-			tlen -= adjust;
-			mbuf_setlen(mp, tlen);
-			if (tlen > 0 && adjust > 0) {
-				tpos += adjust;
-				if ((error = mbuf_setdata(mp, tpos, tlen))) {
-					nfsm_reply(2 * NFSX_UNSIGNED);
-					nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-					return (0);
-				}
-			}
-		}
-		if (zeroing)
-			mbuf_setlen(mp, 0);
-		else if ((tlen = mbuf_len(mp)) > 0) {
-			i += tlen;
-			if (i > len) {
-				mbuf_setlen(mp, tlen - (i - len));
-				zeroing	= 1;
-			}
-			if (mbuf_len(mp) > 0)
-				count++;
-		}
-		mp = mbuf_next(mp);
-	    }
+		error = nfsm_chain_trim_data(nmreq, len, &mlen);
+		nfsmerr_if(error);
+	} else {
+		mlen = 0;
 	}
-	if (len > NFS_MAXDATA || len < 0 || i < len) {
+	if ((len > NFS_MAXDATA) || (len < 0) || (mlen < len)) {
 		error = EIO;
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		return (0);
-	}
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		return (0);
+		goto nfsmerr;
 	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
 
-	if (v3) {
-		nfsm_srv_pre_vattr_init(&forat, v3);
-		forat_ret = vnode_getattr(vp, &forat, &context);
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_pre_vattr_init(&preattr);
+		preattrerr = vnode_getattr(vp, &preattr, ctx);
 	}
 	if (vnode_vtype(vp) != VREG) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EINVAL;
 		else
 			error = (vnode_vtype(vp) == VDIR) ? EISDIR : EACCES;
 	}
-	if (!error) {
-		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, &context, nxo, 1);
-	}
-	if (error) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		return (0);
-	}
+	if (!error)
+		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, ctx, nxo, 1);
+	nfsmerr_if(error);
 
 	if (len > 0) {
-	    MALLOC(uio_bufp, char *, UIO_SIZEOF(count), M_TEMP, M_WAITOK);
-	    if (!uio_bufp) {
-		error = ENOMEM;
-		vnode_put(vp);
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		return (0);
-	    }
-	    uiop = uio_createwithbuffer(count, off, UIO_SYSSPACE, UIO_WRITE, uio_bufp, UIO_SIZEOF(count));
-	    if (!uiop) {
-		error = ENOMEM;
-		vnode_put(vp);
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		if (uio_bufp != NULL) {
-		    FREE(uio_bufp, M_TEMP);
-		}
-		return (0);
-	    }
-	    mp = mrep;
-	    while (mp) {
-		if ((tlen = mbuf_len(mp)) > 0)
-		    uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(mp)), tlen);
-		mp = mbuf_next(mp);
-	    }
+		for (mcount=0, m=nmreq->nmc_mcur; m; m = mbuf_next(m))
+			if (mbuf_len(m) > 0)
+				mcount++;
+		MALLOC(uio_bufp, char *, UIO_SIZEOF(mcount), M_TEMP, M_WAITOK);
+		if (uio_bufp)
+			uiop = uio_createwithbuffer(mcount, off, UIO_SYSSPACE, UIO_WRITE, uio_bufp, UIO_SIZEOF(mcount));
+		if (!uio_bufp || !uiop)
+			error = ENOMEM;
+		nfsmerr_if(error);
+		for (m = nmreq->nmc_mcur; m; m = mbuf_next(m))
+			if ((mlen = mbuf_len(m)) > 0)
+				uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(m)), mlen);
+		/*
+		 * XXX The IO_METASYNC flag indicates that all metadata (and not just
+		 * enough to ensure data integrity) mus be written to stable storage
+		 * synchronously.  (IO_METASYNC is not yet implemented in 4.4BSD-Lite.)
+		 */
+		if (stable == NFS_WRITE_UNSTABLE)
+			ioflags = IO_NODELOCKED;
+		else if (stable == NFS_WRITE_DATASYNC)
+			ioflags = (IO_SYNC | IO_NODELOCKED);
+		else
+			ioflags = (IO_METASYNC | IO_SYNC | IO_NODELOCKED);
 
-	    /*
-	     * XXX
-	     * The IO_METASYNC flag indicates that all metadata (and not just
-	     * enough to ensure data integrity) mus be written to stable storage
-	     * synchronously.
-	     * (IO_METASYNC is not yet implemented in 4.4BSD-Lite.)
-	     */
-	    if (stable == NFSV3WRITE_UNSTABLE)
-		ioflags = IO_NODELOCKED;
-	    else if (stable == NFSV3WRITE_DATASYNC)
-		ioflags = (IO_SYNC | IO_NODELOCKED);
-	    else
-		ioflags = (IO_METASYNC | IO_SYNC | IO_NODELOCKED);
-	
-	    error = VNOP_WRITE(vp, uiop, ioflags, &context);
-	    OSAddAtomic(1, (SInt32*)(SInt32*)&nfsstats.srvvop_writes);
+		error = VNOP_WRITE(vp, uiop, ioflags, ctx);
+		OSAddAtomic(1, (SInt32*)&nfsstats.srvvop_writes);
+
+		/* update export stats */
+		NFSStatAdd64(&nx->nx_stats.bytes_written, len);
+
+		/* update active user stats */
+		nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, len);
+
+#if CONFIG_FSE
+		if (nfsrv_fsevents_enabled && !error && need_fsevent(FSE_CONTENT_MODIFIED, vp))
+			nfsrv_modified(vp, ctx);
+#endif
 	}
-	nfsm_srv_vattr_init(vap, v3);
-	aftat_ret = vnode_getattr(vp, vap, &context);
+	nfsm_srv_vattr_init(&postattr, nd->nd_vers);
+	postattrerr = vnode_getattr(vp, &postattr, ctx);
+	if (!error && (nd->nd_vers == NFS_VER2))
+		error = postattrerr; /* NFSv2 must have attributes to return */
 	vnode_put(vp);
-	if (!error)
-		error = aftat_ret;
-	nfsm_reply(NFSX_PREOPATTR(v3) + NFSX_POSTOPORFATTR(v3) +
-		2 * NFSX_UNSIGNED + NFSX_WRITEVERF(v3));
-	if (v3) {
-		nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, vap);
-		if (error) {
-			if (uio_bufp != NULL) {
-				FREE(uio_bufp, M_TEMP);
-			}
-			return (0);
-		}
-		nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
-		*tl++ = txdr_unsigned(retlen);
-		/*
-		 * If nfs_async is set, then pretend the write was FILESYNC.
-		 */
-		if (stable == NFSV3WRITE_UNSTABLE && !nfs_async)
-			*tl++ = txdr_unsigned(stable);
+	vp = NULL;
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_PREOPATTR(nd->nd_vers) +
+			NFSX_POSTOPORFATTR(nd->nd_vers) + 2 * NFSX_UNSIGNED +
+			NFSX_WRITEVERF(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			preattrerr, &preattr, postattrerr, &postattr);
+		nfsmout_if(error || nd->nd_repstat);
+		nfsm_chain_add_32(error, &nmrep, retlen);
+		/* If nfsrv_async is set, then pretend the write was FILESYNC. */
+		if ((stable == NFS_WRITE_UNSTABLE) && !nfsrv_async)
+			nfsm_chain_add_32(error, &nmrep, stable);
 		else
-			*tl++ = txdr_unsigned(NFSV3WRITE_FILESYNC);
+			nfsm_chain_add_32(error, &nmrep, NFS_WRITE_FILESYNC);
 		/* write verifier */
-		*tl++ = txdr_unsigned(boottime_sec());
-		*tl = txdr_unsigned(0);
+		nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_sec);
+		nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_usec);
 	} else {
-		nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-		nfsm_srvfillattr(vap, fp);
+		error = nfsm_chain_add_fattr(nd, &nmrep, &postattr);
 	}
 nfsmout:
-	if (uio_bufp != NULL) {
+	nfsm_chain_build_done(error, &nmrep);
+	if (vp)
+		vnode_put(vp);
+	if (uio_bufp != NULL)
 		FREE(uio_bufp, M_TEMP);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
 	}
 	return (error);
 }
 
 /*
  * NFS write service with write gathering support. Called when
- * nfsrvw_procrastinate > 0.
+ * nfsrv_wg_delay > 0.
  * See: Chet Juszczak, "Improving the Write Performance of an NFS Server",
  * in Proc. of the Winter 1994 Usenix Conference, pg. 247-259, San Franscisco,
  * Jan. 1994.
  */
+
+#define	NWDELAYHASH(sock, f) \
+	(&(sock)->ns_wdelayhashtbl[(*((u_long *)(f))) % NFS_WDELAYHASHSIZ])
+/* These macros compare nfsrv_descript structures.  */
+#define NFSW_CONTIG(o, n) \
+		(((o)->nd_eoff >= (n)->nd_off) && nfsrv_fhmatch(&(o)->nd_fh, &(n)->nd_fh))
+/*
+ * XXX The following is an incorrect comparison; it fails to take into account
+ * XXX scoping of MAC labels, but we currently lack KPI for credential
+ * XXX comparisons.
+ */
+#define NFSW_SAMECRED(o, n) \
+	(!bcmp((caddr_t)(o)->nd_cr, (caddr_t)(n)->nd_cr, \
+		sizeof (struct ucred)))
+
 int
-nfsrv_writegather(ndp, slp, procp, mrq)
-	struct nfsrv_descript **ndp;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_writegather(
+	struct nfsrv_descript **ndp,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mp;
-	struct nfsrv_descript *wp, *nfsd, *owp, *swp;
+	struct nfsrv_descript *nd, *wp, *owp, *swp;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	struct nfs_fattr *fp;
-	int i;
-	struct nfsrvw_delayhash *wpp;
-	kauth_cred_t cred;
-	struct vnode_attr va, forat;
-	u_long *tl;
-	long t1;
-	caddr_t bpos, dpos, tpos;
-	int error = 0, len, forat_ret = 1;
-	int ioflags, aftat_ret = 1, adjust, v3, zeroing, tlen;
-	char *cp2;
-	mbuf_t mb, mb2, mreq, mrep, md;
+	struct nfsrv_wg_delayhash *wpp;
+	uid_t saved_uid;
+	struct vnode_attr preattr, postattr;
+	int error, mlen, i, ioflags, tlen;
+	int preattrerr, postattrerr;
 	vnode_t vp;
+	mbuf_t m;
 	uio_t uiop = NULL;
 	char *uio_bufp = NULL;
 	u_quad_t cur_usec;
 	struct timeval now;
-	struct vfs_context context;
-
-	context.vc_proc = procp;
+	struct nfsm_chain *nmreq, nmrep;
 
-#ifndef nolint
-	i = 0;
-	len = 0;
-#endif
+	error = 0;
+	preattrerr = postattrerr = ENOENT;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
 
-	*mrq = NULL;
+	*mrepp = NULL;
 	if (*ndp) {
-	    nfsd = *ndp;
+	    nd = *ndp;
 	    *ndp = NULL;
-	    mrep = nfsd->nd_mrep;
-	    md = nfsd->nd_md;
-	    dpos = nfsd->nd_dpos;
-	    cred = nfsd->nd_cr;
-	    context.vc_ucred = cred;
-	    v3 = (nfsd->nd_flag & ND_NFSV3);
-	    LIST_INIT(&nfsd->nd_coalesce);
-	    nfsd->nd_mreq = NULL;
-	    nfsd->nd_stable = NFSV3WRITE_FILESYNC;
+	    nmreq = &nd->nd_nmreq;
+	    LIST_INIT(&nd->nd_coalesce);
+	    nd->nd_mrep = NULL;
+	    nd->nd_stable = NFS_WRITE_FILESYNC;
 	    microuptime(&now);
 	    cur_usec = (u_quad_t)now.tv_sec * 1000000 + (u_quad_t)now.tv_usec;
-	    nfsd->nd_time = cur_usec +
-		(v3 ? nfsrvw_procrastinate_v3 : nfsrvw_procrastinate);
-    
-	    /*
-	     * Now, get the write header..
-	     */
-	    nfsm_srvmtofh(&nfsd->nd_fh);
+	    nd->nd_time = cur_usec +
+		((nd->nd_vers == NFS_VER3) ? nfsrv_wg_delay_v3 : nfsrv_wg_delay);
+
+	    /* Now, get the write header... */
+	    nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nd->nd_fh.nfh_fhp, nd->nd_fh.nfh_len);
 	    /* XXX shouldn't we be checking for invalid FHs before doing any more work? */
-	    if (v3) {
-		nfsm_dissect(tl, u_long *, 5 * NFSX_UNSIGNED);
-		fxdr_hyper(tl, &nfsd->nd_off);
-		tl += 3;
-		nfsd->nd_stable = fxdr_unsigned(int, *tl++);
+	    nfsmerr_if(error);
+	    if (nd->nd_vers == NFS_VER3) {
+		    nfsm_chain_get_64(error, nmreq, nd->nd_off);
+		    nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		    nfsm_chain_get_32(error, nmreq, nd->nd_stable);
 	    } else {
-		nfsm_dissect(tl, u_long *, 4 * NFSX_UNSIGNED);
-		nfsd->nd_off = (off_t)fxdr_unsigned(u_long, *++tl);
-		tl += 2;
-		if (nfs_async)
-			nfsd->nd_stable = NFSV3WRITE_UNSTABLE;
+		    nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		    nfsm_chain_get_32(error, nmreq, nd->nd_off);
+		    nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		    if (nfsrv_async)
+			    nd->nd_stable = NFS_WRITE_UNSTABLE;
 	    }
-	    len = fxdr_unsigned(long, *tl);
-	    nfsd->nd_len = len;
-	    nfsd->nd_eoff = nfsd->nd_off + len;
-    
-	    /*
-	     * Trim the header out of the mbuf list and trim off any trailing
-	     * junk so that the mbuf list has only the write data.
-	     */
-	    zeroing = 1;
-	    i = 0;
-	    mp = mrep;
-	    while (mp) {
-		if (mp == md) {
-		    zeroing = 0;
-		    tpos = mbuf_data(mp);
-		    tlen = mbuf_len(mp);
-		    adjust = dpos - tpos;
-		    tlen -= adjust;
-		    mbuf_setlen(mp, tlen);
-		    if (tlen > 0 && adjust > 0) {
-			tpos += adjust;
-			if ((error = mbuf_setdata(mp, tpos, tlen)))
-				goto nfsmout;
-		    }
-		}
-		if (zeroing)
-		    mbuf_setlen(mp, 0);
-		else {
-		    tlen = mbuf_len(mp);
-		    i += tlen;
-		    if (i > len) {
-			mbuf_setlen(mp, tlen - (i - len));
-			zeroing = 1;
-		    }
-		}
-		mp = mbuf_next(mp);
+	    nfsm_chain_get_32(error, nmreq, nd->nd_len);
+	    nfsmerr_if(error);
+	    nd->nd_eoff = nd->nd_off + nd->nd_len;
+
+	    if (nd->nd_len > 0) {
+		    error = nfsm_chain_trim_data(nmreq, nd->nd_len, &mlen);
+		    nfsmerr_if(error);
+	    } else {
+		mlen = 0;
 	    }
-	    if (len > NFS_MAXDATA || len < 0  || i < len) {
-nfsmout:
-		mbuf_freem(mrep);
-		mrep = NULL;
+
+	    if ((nd->nd_len > NFS_MAXDATA) || (nd->nd_len < 0)  || (mlen < nd->nd_len)) {
 		error = EIO;
-		nfsm_writereply(2 * NFSX_UNSIGNED, v3);
-		if (v3)
-		    nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, &va);
-		nfsd->nd_mreq = mreq;
-		nfsd->nd_mrep = NULL;
-		nfsd->nd_time = 1;
+nfsmerr:
+		nd->nd_repstat = error;
+		error = nfsrv_rephead(nd, slp, &nmrep, NFSX_WCCDATA(nd->nd_vers));
+		if (!error) {
+		    nd->nd_mrep = nmrep.nmc_mhead;
+		    if (nd->nd_vers == NFS_VER3)
+			nfsm_chain_add_wcc_data(error, nd, &nmrep,
+				preattrerr, &preattr, postattrerr, &postattr);
+		}
+		nfsm_chain_build_done(error, &nmrep);
+		nd->nd_time = 1;
 	    }
-    
+
 	    /*
 	     * Add this entry to the hash and time queues.
 	     */
 	    lck_mtx_lock(&slp->ns_wgmutex);
 	    owp = NULL;
 	    wp = slp->ns_tq.lh_first;
-	    while (wp && wp->nd_time < nfsd->nd_time) {
+	    while (wp && wp->nd_time < nd->nd_time) {
 		owp = wp;
 		wp = wp->nd_tq.le_next;
 	    }
 	    if (owp) {
-		LIST_INSERT_AFTER(owp, nfsd, nd_tq);
+		LIST_INSERT_AFTER(owp, nd, nd_tq);
 	    } else {
-		LIST_INSERT_HEAD(&slp->ns_tq, nfsd, nd_tq);
+		LIST_INSERT_HEAD(&slp->ns_tq, nd, nd_tq);
 	    }
-	    if (nfsd->nd_mrep) {
-		wpp = NWDELAYHASH(slp, nfsd->nd_fh.nfh_fid);
+	    if (!error) {
+		wpp = NWDELAYHASH(slp, nd->nd_fh.nfh_fid);
 		owp = NULL;
 		wp = wpp->lh_first;
-		while (wp && !nfsrv_fhmatch(&nfsd->nd_fh, &wp->nd_fh)) {
+		while (wp && !nfsrv_fhmatch(&nd->nd_fh, &wp->nd_fh)) {
 		    owp = wp;
 		    wp = wp->nd_hash.le_next;
 		}
-		while (wp && (wp->nd_off < nfsd->nd_off) &&
-		    nfsrv_fhmatch(&nfsd->nd_fh, &wp->nd_fh)) {
+		while (wp && (wp->nd_off < nd->nd_off) &&
+		    nfsrv_fhmatch(&nd->nd_fh, &wp->nd_fh)) {
 		    owp = wp;
 		    wp = wp->nd_hash.le_next;
 		}
 		if (owp) {
-		    LIST_INSERT_AFTER(owp, nfsd, nd_hash);
-
+		    LIST_INSERT_AFTER(owp, nd, nd_hash);
 		    /*
 		     * Search the hash list for overlapping entries and
 		     * coalesce.
 		     */
-		    for(; nfsd && NFSW_CONTIG(owp, nfsd); nfsd = wp) {
-			wp = nfsd->nd_hash.le_next;
-			if (NFSW_SAMECRED(owp, nfsd))
-			    nfsrvw_coalesce(owp, nfsd);
+		    for(; nd && NFSW_CONTIG(owp, nd); nd = wp) {
+			wp = nd->nd_hash.le_next;
+			if (NFSW_SAMECRED(owp, nd))
+			    nfsrv_wg_coalesce(owp, nd);
 		    }
 		} else {
-		    LIST_INSERT_HEAD(wpp, nfsd, nd_hash);
+		    LIST_INSERT_HEAD(wpp, nd, nd_hash);
 		}
 	    }
 	} else {
 	    lck_mtx_lock(&slp->ns_wgmutex);
 	}
-    
+
 	/*
 	 * Now, do VNOP_WRITE()s for any one(s) that need to be done now
 	 * and generate the associated reply mbuf list(s).
@@ -1363,85 +1473,89 @@ nfsmout:
 loop1:
 	microuptime(&now);
 	cur_usec = (u_quad_t)now.tv_sec * 1000000 + (u_quad_t)now.tv_usec;
-	for (nfsd = slp->ns_tq.lh_first; nfsd; nfsd = owp) {
-		owp = nfsd->nd_tq.le_next;
-		if (nfsd->nd_time > cur_usec)
+	for (nd = slp->ns_tq.lh_first; nd; nd = owp) {
+		owp = nd->nd_tq.le_next;
+		if (nd->nd_time > cur_usec)
 		    break;
-		if (nfsd->nd_mreq)
+		if (nd->nd_mrep)
 		    continue;
-		LIST_REMOVE(nfsd, nd_tq);
-		LIST_REMOVE(nfsd, nd_hash);
-		mrep = nfsd->nd_mrep;
-		nfsd->nd_mrep = NULL;
-		v3 = (nfsd->nd_flag & ND_NFSV3);
-		forat_ret = aftat_ret = 1;
-		error = nfsrv_fhtovp(&nfsd->nd_fh, nfsd->nd_nam, TRUE, &vp, &nx, &nxo);
+		LIST_REMOVE(nd, nd_tq);
+		LIST_REMOVE(nd, nd_hash);
+		nmreq = &nd->nd_nmreq;
+		preattrerr = postattrerr = ENOENT;
+
+		/* save the incoming uid before mapping, */	
+		/* for updating active user stats later */
+		saved_uid = kauth_cred_getuid(nd->nd_cr);
+
+		error = nfsrv_fhtovp(&nd->nd_fh, nd, &vp, &nx, &nxo);
 		if (!error) {
-			error = nfsrv_credcheck(nfsd, nx, nxo);
-			if (error)
-				vnode_put(vp);
+		    /* update per-export stats */
+		    NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+		    error = nfsrv_credcheck(nd, ctx, nx, nxo);
+		    if (error)
+		    	vnode_put(vp);
 		}
-		cred = nfsd->nd_cr;
-		context.vc_ucred = cred;
 		if (!error) {
-		    if (v3) {
-			nfsm_srv_pre_vattr_init(&forat, v3);
-			forat_ret = vnode_getattr(vp, &forat, &context);
+		    if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&preattr);
+			preattrerr = vnode_getattr(vp, &preattr, ctx);
 		    }
 		    if (vnode_vtype(vp) != VREG) {
-			if (v3)
+			if (nd->nd_vers == NFS_VER3)
 			    error = EINVAL;
 			else
 			    error = (vnode_vtype(vp) == VDIR) ? EISDIR : EACCES;
 		    }
 		} else
 		    vp = NULL;
-		if (!error) {
-		    error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, &context, nxo, 1);
-		}
-    
-		if (nfsd->nd_stable == NFSV3WRITE_UNSTABLE)
+		if (!error)
+		    error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA, ctx, nxo, 1);
+
+		if (nd->nd_stable == NFS_WRITE_UNSTABLE)
 		    ioflags = IO_NODELOCKED;
-		else if (nfsd->nd_stable == NFSV3WRITE_DATASYNC)
+		else if (nd->nd_stable == NFS_WRITE_DATASYNC)
 		    ioflags = (IO_SYNC | IO_NODELOCKED);
 		else
 		    ioflags = (IO_METASYNC | IO_SYNC | IO_NODELOCKED);
 
-		if (!error && ((nfsd->nd_eoff - nfsd->nd_off) > 0)) {
-		    mp = mrep;
-		    i = 0;
-		    while (mp) {
-			if (mbuf_len(mp) > 0)
+		if (!error && ((nd->nd_eoff - nd->nd_off) > 0)) {
+		    for (i=0, m=nmreq->nmc_mhead; m; m = mbuf_next(m))
+			if (mbuf_len(m) > 0)
 			    i++;
-			mp = mbuf_next(mp);
-		    }
 
 		    MALLOC(uio_bufp, char *, UIO_SIZEOF(i), M_TEMP, M_WAITOK);
 		    if (uio_bufp)
-			uiop = uio_createwithbuffer(i, nfsd->nd_off, UIO_SYSSPACE,
+			uiop = uio_createwithbuffer(i, nd->nd_off, UIO_SYSSPACE,
 						UIO_WRITE, uio_bufp, UIO_SIZEOF(i));
 		    if (!uio_bufp || !uiop)
 			error = ENOMEM;
 		    if (!error) {
-			mp = mrep;
-			while (mp) {
-			    if ((tlen = mbuf_len(mp)) > 0)
-				uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(mp)), tlen);
-			    mp = mbuf_next(mp);
-			}
-			error = VNOP_WRITE(vp, uiop, ioflags, &context);
+			for (m = nmreq->nmc_mhead; m; m = mbuf_next(m))
+			    if ((tlen = mbuf_len(m)) > 0)
+				uio_addiov(uiop, CAST_USER_ADDR_T((caddr_t)mbuf_data(m)), tlen);
+			error = VNOP_WRITE(vp, uiop, ioflags, ctx);
 			OSAddAtomic(1, (SInt32*)&nfsstats.srvvop_writes);
+
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.bytes_written, nd->nd_len);
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, nd->nd_len);
+
+#if CONFIG_FSE
+			if (nfsrv_fsevents_enabled && !error && need_fsevent(FSE_CONTENT_MODIFIED, vp))
+				nfsrv_modified(vp, ctx);
+#endif
 		    }
 		    if (uio_bufp) {
 			FREE(uio_bufp, M_TEMP);
 			uio_bufp = NULL;
 		    }
 		}
-		mbuf_freem(mrep);
-		mrep = NULL;
 		if (vp) {
-		    nfsm_srv_pre_vattr_init(&va, v3);
-		    aftat_ret = vnode_getattr(vp, &va, &context);
+		    nfsm_srv_vattr_init(&postattr, nd->nd_vers);
+		    postattrerr = vnode_getattr(vp, &postattr, ctx);
 		    vnode_put(vp);
 		}
 
@@ -1449,47 +1563,49 @@ loop1:
 		 * Loop around generating replies for all write rpcs that have
 		 * now been completed.
 		 */
-		swp = nfsd;
+		swp = nd;
 		do {
 		    if (error) {
-			nfsm_writereply(NFSX_WCCDATA(v3), v3);
-			if (v3) {
-			    nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, &va);
+			nd->nd_repstat = error;
+			error = nfsrv_rephead(nd, slp, &nmrep, NFSX_WCCDATA(nd->nd_vers));
+			if (!error && (nd->nd_vers == NFS_VER3)) {
+			    nfsm_chain_add_wcc_data(error, nd, &nmrep,
+				    preattrerr, &preattr, postattrerr, &postattr);
 			}
 		    } else {
-			nfsm_writereply(NFSX_PREOPATTR(v3) +
-			    NFSX_POSTOPORFATTR(v3) + 2 * NFSX_UNSIGNED +
-			    NFSX_WRITEVERF(v3), v3);
-			if (v3) {
-			    nfsm_srvwcc_data(forat_ret, &forat, aftat_ret, &va);
-			    nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
-			    *tl++ = txdr_unsigned(nfsd->nd_len);
-			    *tl++ = txdr_unsigned(swp->nd_stable);
+			nd->nd_repstat = error;
+			error = nfsrv_rephead(nd, slp, &nmrep, NFSX_PREOPATTR(nd->nd_vers) +
+			    NFSX_POSTOPORFATTR(nd->nd_vers) + 2 * NFSX_UNSIGNED +
+			    NFSX_WRITEVERF(nd->nd_vers));
+			if (!error && (nd->nd_vers == NFS_VER3)) {
+			    nfsm_chain_add_wcc_data(error, nd, &nmrep,
+				    preattrerr, &preattr, postattrerr, &postattr);
+			    nfsm_chain_add_32(error, &nmrep, nd->nd_len);
+			    nfsm_chain_add_32(error, &nmrep, nd->nd_stable);
 			    /* write verifier */
-			    *tl++ = txdr_unsigned(boottime_sec());
-			    *tl = txdr_unsigned(0);
-			} else {
-			    nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-			    nfsm_srvfillattr(&va, fp);
+			    nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_sec);
+			    nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_usec);
+			} else if (!error) {
+			    error = nfsm_chain_add_fattr(nd, &nmrep, &postattr);
 			}
 		    }
-		    nfsd->nd_mreq = mreq;
-		    if (nfsd->nd_mrep)
-			panic("nfsrv_write: nd_mrep not free");
+		    nfsm_chain_build_done(error, &nmrep);
+		    nfsmerr_if(error);
+		    nd->nd_mrep = nmrep.nmc_mhead;
 
 		    /*
 		     * Done. Put it at the head of the timer queue so that
 		     * the final phase can return the reply.
 		     */
-		    if (nfsd != swp) {
-			nfsd->nd_time = 1;
-			LIST_INSERT_HEAD(&slp->ns_tq, nfsd, nd_tq);
+		    if (nd != swp) {
+			nd->nd_time = 1;
+			LIST_INSERT_HEAD(&slp->ns_tq, nd, nd_tq);
 		    }
-		    nfsd = swp->nd_coalesce.lh_first;
-		    if (nfsd) {
-			LIST_REMOVE(nfsd, nd_tq);
+		    nd = swp->nd_coalesce.lh_first;
+		    if (nd) {
+			LIST_REMOVE(nd, nd_tq);
 		    }
-		} while (nfsd);
+		} while (nd);
 		swp->nd_time = 1;
 		LIST_INSERT_HEAD(&slp->ns_tq, swp, nd_tq);
 		goto loop1;
@@ -1498,82 +1614,150 @@ loop1:
 	/*
 	 * Search for a reply to return.
 	 */
-	for (nfsd = slp->ns_tq.lh_first; nfsd; nfsd = nfsd->nd_tq.le_next)
-		if (nfsd->nd_mreq) {
-		    LIST_REMOVE(nfsd, nd_tq);
-		    *mrq = nfsd->nd_mreq;
-		    *ndp = nfsd;
+	for (nd = slp->ns_tq.lh_first; nd; nd = nd->nd_tq.le_next)
+		if (nd->nd_mrep) {
+		    LIST_REMOVE(nd, nd_tq);
+		    *mrepp = nd->nd_mrep;
+		    *ndp = nd;
 		    break;
 		}
-	slp->ns_wgtime = slp->ns_tq.lh_first ? slp->ns_tq.lh_first->nd_time : 0;
-	lck_mtx_unlock(&slp->ns_wgmutex);
+	slp->ns_wgtime = slp->ns_tq.lh_first ? slp->ns_tq.lh_first->nd_time : 0;
+	lck_mtx_unlock(&slp->ns_wgmutex);
+
+	/*
+	 * If we've just created a write pending gather,
+	 * start the timer to check on it soon to make sure
+	 * the write will be completed.
+	 *
+	 * Add/Remove the socket in the nfsrv_sockwg queue as needed.
+	 */
+	lck_mtx_lock(nfsd_mutex);
+	if (slp->ns_wgtime) {
+		if (slp->ns_wgq.tqe_next == SLPNOLIST) {
+			TAILQ_INSERT_HEAD(&nfsrv_sockwg, slp, ns_wgq);
+		}
+		if (!nfsrv_wg_timer_on) {
+			nfsrv_wg_timer_on = 1;
+			nfs_interval_timer_start(nfsrv_wg_timer_call,
+				NFSRV_WGATHERDELAY);
+		}
+	} else if (slp->ns_wgq.tqe_next != SLPNOLIST) {
+		TAILQ_REMOVE(&nfsrv_sockwg, slp, ns_wgq);
+		slp->ns_wgq.tqe_next = SLPNOLIST;
+	}
+	lck_mtx_unlock(nfsd_mutex);
+
 	return (0);
 }
 
 /*
- * Coalesce the write request nfsd into owp. To do this we must:
- * - remove nfsd from the queues
- * - merge nfsd->nd_mrep into owp->nd_mrep
+ * Coalesce the write request nd into owp. To do this we must:
+ * - remove nd from the queues
+ * - merge nd->nd_nmreq into owp->nd_nmreq
  * - update the nd_eoff and nd_stable for owp
- * - put nfsd on owp's nd_coalesce list
+ * - put nd on owp's nd_coalesce list
  */
-static void
-nfsrvw_coalesce(
-        struct nfsrv_descript *owp,
-        struct nfsrv_descript *nfsd)
+static int
+nfsrv_wg_coalesce(struct nfsrv_descript *owp, struct nfsrv_descript *nd)
 {
-        int overlap, error;
-        mbuf_t mp, mpnext;
+	int overlap, error;
+	mbuf_t mp, mpnext;
 	struct nfsrv_descript *p;
 
-        LIST_REMOVE(nfsd, nd_hash);
-        LIST_REMOVE(nfsd, nd_tq);
-        if (owp->nd_eoff < nfsd->nd_eoff) {
-            overlap = owp->nd_eoff - nfsd->nd_off;
-            if (overlap < 0)
-                panic("nfsrv_coalesce: bad off");
-            if (overlap > 0)
-                mbuf_adj(nfsd->nd_mrep, overlap);
-            mp = owp->nd_mrep;
-            while ((mpnext = mbuf_next(mp)))
-                mp = mpnext;
-            error = mbuf_setnext(mp, nfsd->nd_mrep);
-	    if (error)
-	    	panic("nfsrvw_coalesce: mbuf_setnext failed: %d", error);
-            owp->nd_eoff = nfsd->nd_eoff;
-        } else {
-            mbuf_freem(nfsd->nd_mrep);
-	}
-        nfsd->nd_mrep = NULL;
-        if (nfsd->nd_stable == NFSV3WRITE_FILESYNC)
-            owp->nd_stable = NFSV3WRITE_FILESYNC;
-        else if (nfsd->nd_stable == NFSV3WRITE_DATASYNC &&
-            owp->nd_stable == NFSV3WRITE_UNSTABLE)
-            owp->nd_stable = NFSV3WRITE_DATASYNC;
-        LIST_INSERT_HEAD(&owp->nd_coalesce, nfsd, nd_tq);
+	LIST_REMOVE(nd, nd_hash);
+	LIST_REMOVE(nd, nd_tq);
+	if (owp->nd_eoff < nd->nd_eoff) {
+		overlap = owp->nd_eoff - nd->nd_off;
+		if (overlap < 0)
+			return (EIO);
+		if (overlap > 0)
+			mbuf_adj(nd->nd_nmreq.nmc_mhead, overlap);
+		mp = owp->nd_nmreq.nmc_mhead;
+		while ((mpnext = mbuf_next(mp)))
+			mp = mpnext;
+		error = mbuf_setnext(mp, nd->nd_nmreq.nmc_mhead);
+		if (error)
+			return (error);
+		owp->nd_eoff = nd->nd_eoff;
+	} else {
+		mbuf_freem(nd->nd_nmreq.nmc_mhead);
+	}
+	nd->nd_nmreq.nmc_mhead = NULL;
+	nd->nd_nmreq.nmc_mcur = NULL;
+	if (nd->nd_stable == NFS_WRITE_FILESYNC)
+		owp->nd_stable = NFS_WRITE_FILESYNC;
+	else if ((nd->nd_stable == NFS_WRITE_DATASYNC) &&
+		 (owp->nd_stable == NFS_WRITE_UNSTABLE))
+		owp->nd_stable = NFS_WRITE_DATASYNC;
+	LIST_INSERT_HEAD(&owp->nd_coalesce, nd, nd_tq);
 
 	/*
-	 * If nfsd had anything else coalesced into it, transfer them
+	 * If nd had anything else coalesced into it, transfer them
 	 * to owp, otherwise their replies will never get sent.
 	 */
-	for (p = nfsd->nd_coalesce.lh_first; p;
-	     p = nfsd->nd_coalesce.lh_first) {
-	    LIST_REMOVE(p, nd_tq);
-	    LIST_INSERT_HEAD(&owp->nd_coalesce, p, nd_tq);
+	while ((p = nd->nd_coalesce.lh_first)) {
+		LIST_REMOVE(p, nd_tq);
+		LIST_INSERT_HEAD(&owp->nd_coalesce, p, nd_tq);
+	}
+	return (0);
+}
+
+/*
+ * Scan the write gathering queues for writes that need to be
+ * completed now.
+ */
+void
+nfsrv_wg_timer(__unused void *param0, __unused void *param1)
+{
+	struct timeval now;
+	uint64_t cur_usec, next_usec;
+	int interval;
+	struct nfsrv_sock *slp;
+	int writes_pending = 0;
+
+	microuptime(&now);
+	cur_usec = (uint64_t)now.tv_sec * 1000000 + (uint64_t)now.tv_usec;
+	next_usec = cur_usec + (NFSRV_WGATHERDELAY * 1000);
+
+	lck_mtx_lock(nfsd_mutex);
+	TAILQ_FOREACH(slp, &nfsrv_sockwg, ns_wgq) {
+		if (slp->ns_wgtime) {
+			writes_pending++;
+			if (slp->ns_wgtime <= cur_usec) {
+				lck_rw_lock_exclusive(&slp->ns_rwlock);
+				slp->ns_flag |= SLP_DOWRITES;
+				lck_rw_done(&slp->ns_rwlock);
+				nfsrv_wakenfsd(slp);
+				continue;
+			}
+			if (slp->ns_wgtime < next_usec)
+				next_usec = slp->ns_wgtime;
+		}
+	}
+
+	if (writes_pending == 0) {
+		nfsrv_wg_timer_on = 0;
+		lck_mtx_unlock(nfsd_mutex);
+		return;
 	}
+	lck_mtx_unlock(nfsd_mutex);
+
+	/*
+	 * Return the number of msec to wait again
+	 */
+	interval = (next_usec - cur_usec) / 1000;
+	if (interval < 1)
+		interval = 1;
+	nfs_interval_timer_start(nfsrv_wg_timer_call, interval);
 }
 
 /*
  * Sort the group list in increasing numerical order.
  * (Insertion sort by Chris Torek, who was grossed out by the bubble sort
  *  that used to be here.)
- *
- * XXX ILLEGAL
  */
 void
-nfsrvw_sort(list, num)
-        gid_t *list;
-        int num;
+nfsrv_group_sort(gid_t *list, int num)
 {
 	int i, j;
 	gid_t v;
@@ -1588,123 +1772,98 @@ nfsrvw_sort(list, num)
 	}
 }
 
-/*
- * copy credentials making sure that the result can be compared with bcmp().
- *
- * NOTE:	This function is only intended to operate on a real input
- *		credential and a template output credential; the template
- *		ouptut credential is intended to then be used as an argument
- *		to kauth_cred_create() - AND NEVER REFERENCED OTHERWISE.
- */
-void
-nfsrv_setcred(kauth_cred_t incred, kauth_cred_t outcred)
-{
-	int i;
-
-	bzero((caddr_t)outcred, sizeof (*outcred));
-	outcred->cr_uid = kauth_cred_getuid(incred);
-	outcred->cr_ngroups = incred->cr_ngroups;
-	for (i = 0; i < incred->cr_ngroups; i++)
-		outcred->cr_groups[i] = incred->cr_groups[i];
-	nfsrvw_sort(outcred->cr_groups, outcred->cr_ngroups);
-}
-
 /*
  * nfs create service
  * now does a truncate to 0 length via. setattr if it already exists
  */
 int
-nfsrv_create(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_create(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct nfs_fattr *fp;
-	struct vnode_attr dirfor, diraft, postat;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
-	struct nfsv2_sattr *sp;
-	u_long *tl;
-	struct nameidata nd;
-	caddr_t cp;
-	long t1;
-	caddr_t bpos;
-	int error = 0, rdev, len, tsize, dirfor_ret = 1, diraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3), how, exclusive_flag = 0;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	vnode_t vp, dvp, dirp = NULL;
+	struct vnode_attr dpreattr, dpostattr, postattr;
+	struct vnode_attr va, *vap = &va;
+	struct nameidata ni;
+	int error, rdev, dpreattrerr, dpostattrerr, postattrerr;
+	int how, exclusive_flag;
+	uint32_t len;
+	vnode_t vp, dvp, dirp;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
 	u_quad_t tempsize;
 	u_char cverf[NFSX_V3CREATEVERF];
-	struct vfs_context context;
 	uid_t saved_uid;
-
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	dpreattrerr = dpostattrerr = postattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = dvp = dirp = NULL;
+	exclusive_flag = 0;
+	ni.ni_cnd.cn_nameiop = 0;
+	rdev = 0;
 
 	/*
 	 * Save the original credential UID in case they are
 	 * mapped and we need to map the IDs in the attributes.
 	 */
-	saved_uid = kauth_cred_getuid(nfsd->nd_cr);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
 
-#ifndef nolint
-	rdev = 0;
-#endif
-	nd.ni_cnd.cn_nameiop = 0;
-	vp = dvp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, v3);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
 
-	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+	ni.ni_cnd.cn_nameiop = CREATE;
+	ni.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
+
 	if (error) {
-		nd.ni_cnd.cn_nameiop = 0;
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		if (dirp)
-			vnode_put(dirp);
-		return (0);
+		ni.ni_cnd.cn_nameiop = 0;
+		goto nfsmerr;
 	}
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
 
+	dvp = ni.ni_dvp;
+	vp = ni.ni_vp;
 	VATTR_INIT(vap);
 
-	if (v3) {
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		how = fxdr_unsigned(int, *tl);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_get_32(error, nmreq, how);
+		nfsmerr_if(error);
 		switch (how) {
-		case NFSV3CREATE_GUARDED:
+		case NFS_CREATE_GUARDED:
 			if (vp) {
 				error = EEXIST;
 				break;
 			}
-		case NFSV3CREATE_UNCHECKED:
-			nfsm_srvsattr(vap);
+		case NFS_CREATE_UNCHECKED:
+			error = nfsm_chain_get_sattr(nd, nmreq, vap);
 			break;
-		case NFSV3CREATE_EXCLUSIVE:
-			nfsm_dissect(cp, caddr_t, NFSX_V3CREATEVERF);
-			bcopy(cp, cverf, NFSX_V3CREATEVERF);
+		case NFS_CREATE_EXCLUSIVE:
+			nfsm_chain_get_opaque(error, nmreq, NFSX_V3CREATEVERF, cverf);
 			exclusive_flag = 1;
 			if (vp == NULL)
 				VATTR_SET(vap, va_mode, 0);
@@ -1714,28 +1873,25 @@ nfsrv_create(nfsd, slp, procp, mrq)
 	} else {
 	        enum vtype v_type;
 
-		nfsm_dissect(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		v_type = IFTOVT(fxdr_unsigned(u_long, sp->sa_mode));
+		error = nfsm_chain_get_sattr(nd, nmreq, vap);
+		nfsmerr_if(error);
+		v_type = vap->va_type;
 		if (v_type == VNON)
 			v_type = VREG;
 		VATTR_SET(vap, va_type, v_type);
-		VATTR_SET(vap, va_mode, nfstov_mode(sp->sa_mode));
 
 		switch (v_type) {
-		case VREG:
-			tsize = fxdr_unsigned(long, sp->sa_size);
-			if (tsize != -1)
-				VATTR_SET(vap, va_data_size, (u_quad_t)tsize);
-			break;
 		case VCHR:
 		case VBLK:
 		case VFIFO:
-			rdev = fxdr_unsigned(long, sp->sa_size);
+			rdev = vap->va_data_size;
+			VATTR_CLEAR_ACTIVE(vap, va_data_size);
 			break;
 		default:
 			break;
 		};
 	}
+	nfsmerr_if(error);
 
 	/*
 	 * If it doesn't exist, create it
@@ -1749,15 +1905,15 @@ nfsrv_create(nfsd, slp, procp, mrq)
 		 * If the credentials were mapped, we should
 		 * map the same values in the attributes.
 		 */
-		if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nfsd->nd_cr) != saved_uid)) {
+		if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nd->nd_cr) != saved_uid)) {
 			int ismember;
-			VATTR_SET(vap, va_uid, kauth_cred_getuid(nfsd->nd_cr));
-			if (kauth_cred_ismember_gid(nfsd->nd_cr, vap->va_gid, &ismember) || !ismember)
-				VATTR_SET(vap, va_gid, kauth_cred_getgid(nfsd->nd_cr));
+			VATTR_SET(vap, va_uid, kauth_cred_getuid(nd->nd_cr));
+			if (kauth_cred_ismember_gid(nd->nd_cr, vap->va_gid, &ismember) || !ismember)
+				VATTR_SET(vap, va_gid, kauth_cred_getgid(nd->nd_cr));
 		}
 
 		/* authorize before creating */
-		error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context, nxo, 0);
+		error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx, nxo, 0);
 
 		/* construct ACL and handle inheritance */
 		if (!error) {
@@ -1765,7 +1921,7 @@ nfsrv_create(nfsd, slp, procp, mrq)
 			    NULL,
 			    &xacl,
 			    0 /* !isdir */,
-			    &context);
+			    ctx);
 
 			if (!error && xacl != NULL)
 			        VATTR_SET(vap, va_acl, xacl);
@@ -1775,32 +1931,32 @@ nfsrv_create(nfsd, slp, procp, mrq)
 
 		/* validate new-file security information */
 		if (!error) {
-			error = vnode_authattr_new(dvp, vap, 0, &context);
+			error = vnode_authattr_new(dvp, vap, 0, ctx);
 			if (error && (VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid))) {
 				/*
-				 * Most NFS servers just ignore the UID/GID attributes, so we   
+				 * Most NFS servers just ignore the UID/GID attributes, so we
 				 * try ignoring them if that'll help the request succeed.
 				 */
 				VATTR_CLEAR_ACTIVE(vap, va_uid);
 				VATTR_CLEAR_ACTIVE(vap, va_gid);
-				error = vnode_authattr_new(dvp, vap, 0, &context);
+				error = vnode_authattr_new(dvp, vap, 0, ctx);
 			}
 		}
 
 		if (vap->va_type == VREG || vap->va_type == VSOCK) {
 
 			if (!error)
-				error = VNOP_CREATE(dvp, &vp, &nd.ni_cnd, vap, &context);
-  
- 			if (!error && !VATTR_ALL_SUPPORTED(vap))
+				error = VNOP_CREATE(dvp, &vp, &ni.ni_cnd, vap, ctx);
+
+			if (!error && !VATTR_ALL_SUPPORTED(vap))
 			        /*
 				 * If some of the requested attributes weren't handled by the VNOP,
 				 * use our fallback code.
 				 */
- 				error = vnode_setattr_fallback(vp, vap, &context);
+				error = vnode_setattr_fallback(vp, vap, ctx);
 
- 			if (xacl != NULL)
- 				kauth_acl_free(xacl);
+			if (xacl != NULL)
+				kauth_acl_free(xacl);
 
 			if (!error) {
 				if (exclusive_flag) {
@@ -1811,46 +1967,52 @@ nfsrv_create(nfsd, slp, procp, mrq)
 					VATTR_SET_ACTIVE(vap, va_access_time);
 					// skip authorization, as this is an
 					// NFS internal implementation detail.
-					error = vnode_setattr(vp, vap, &context);
+					error = vnode_setattr(vp, vap, ctx);
+				}
+
+#if CONFIG_FSE
+				if (nfsrv_fsevents_enabled && need_fsevent(FSE_CREATE_FILE, vp)) {
+				        add_fsevent(FSE_CREATE_FILE, ctx,
+						    FSE_ARG_VNODE, vp,
+						    FSE_ARG_DONE);
 				}
+#endif
 			}
 
 		} else if (vap->va_type == VCHR || vap->va_type == VBLK ||
 			vap->va_type == VFIFO) {
 			if (vap->va_type == VCHR && rdev == (int)0xffffffff)
 				VATTR_SET(vap, va_type, VFIFO);
-			if (vap->va_type != VFIFO &&
-			    (error = suser(nfsd->nd_cr, (u_short *)0))) {
-				nfsm_reply(0);
-			} else
-				VATTR_SET(vap, va_rdev, (dev_t)rdev);
+			if (vap->va_type != VFIFO) {
+				error = suser(nd->nd_cr, NULL);
+				nfsmerr_if(error);
+			}
+			VATTR_SET(vap, va_rdev, (dev_t)rdev);
 
-			error = VNOP_MKNOD(dvp, &vp, &nd.ni_cnd, vap, &context);
+			error = VNOP_MKNOD(dvp, &vp, &ni.ni_cnd, vap, ctx);
 
- 			if (xacl != NULL)
- 				kauth_acl_free(xacl);
+			if (xacl != NULL)
+				kauth_acl_free(xacl);
+
+			nfsmerr_if(error);
 
-			if (error) {
-				nfsm_reply(0);
-			}
 			if (vp) {
 				vnode_recycle(vp);
 				vnode_put(vp);
 				vp = NULL;
 			}
-			nd.ni_cnd.cn_nameiop = LOOKUP;
-			nd.ni_cnd.cn_flags &= ~LOCKPARENT;
-			nd.ni_cnd.cn_context = &context;
-			nd.ni_startdir = dvp;
-			nd.ni_usedvp   = dvp;
-			error = lookup(&nd);
+			ni.ni_cnd.cn_nameiop = LOOKUP;
+			ni.ni_cnd.cn_flags &= ~LOCKPARENT;
+			ni.ni_cnd.cn_context = ctx;
+			ni.ni_startdir = dvp;
+			ni.ni_usedvp   = dvp;
+			error = lookup(&ni);
 			if (!error) {
-			        if (nd.ni_cnd.cn_flags & ISSYMLINK)
-				        error = EINVAL;
-				vp = nd.ni_vp;
+				if (ni.ni_cnd.cn_flags & ISSYMLINK)
+					error = EINVAL;
+				vp = ni.ni_vp;
 			}
-			if (error)
-				nfsm_reply(0);
+			nfsmerr_if(error);
 		} else {
 			error = ENXIO;
 		}
@@ -1858,8 +2020,8 @@ nfsrv_create(nfsd, slp, procp, mrq)
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
-		nd.ni_cnd.cn_nameiop = 0;
+		nameidone(&ni);
+		ni.ni_cnd.cn_nameiop = 0;
 
 		vnode_put(dvp);
 	} else {
@@ -1867,62 +2029,72 @@ nfsrv_create(nfsd, slp, procp, mrq)
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-	        nameidone(&nd);
-		nd.ni_cnd.cn_nameiop = 0;
+	        nameidone(&ni);
+		ni.ni_cnd.cn_nameiop = 0;
 
 		vnode_put(dvp);
 
 		if (!error && VATTR_IS_ACTIVE(vap, va_data_size)) {
 			error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_WRITE_DATA,
-			    &context, nxo, 0);
+			    ctx, nxo, 0);
 			if (!error) {
 				tempsize = vap->va_data_size;
 				VATTR_INIT(vap);
 				VATTR_SET(vap, va_data_size, tempsize);
-				error = vnode_setattr(vp, vap, &context);
+				error = vnode_setattr(vp, vap, ctx);
 			}
 		}
 	}
 	if (!error) {
-		error = nfsrv_vptofh(nx, !v3, NULL, vp, &context, &nfh);
+		error = nfsrv_vptofh(nx, nd->nd_vers, NULL, vp, ctx, &nfh);
 		if (!error) {
-			nfsm_srv_vattr_init(&postat, v3);
-			error = vnode_getattr(vp, &postat, &context);
+			nfsm_srv_vattr_init(&postattr, nd->nd_vers);
+			postattrerr = vnode_getattr(vp, &postattr, ctx);
+			if (nd->nd_vers == NFS_VER2)
+				error = postattrerr;
 		}
 	}
 	if (vp)
 	        vnode_put(vp);
 
-	if (v3) {
+	if (nd->nd_vers == NFS_VER3) {
 		if (exclusive_flag && !error &&
-			bcmp(cverf, (caddr_t)&postat.va_access_time, NFSX_V3CREATEVERF))
+		    bcmp(cverf, &postattr.va_access_time, NFSX_V3CREATEVERF))
 			error = EEXIST;
-		nfsm_srv_vattr_init(&diraft, v3);
-		diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, NFS_VER3);
+		dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
 		dirp = NULL;
 	}
-	nfsm_reply(NFSX_SRVFH(v3, &nfh) + NFSX_FATTR(v3) + NFSX_WCCDATA(v3));
 
-	if (v3) {
-		if (!error) {
-			nfsm_srvpostop_fh(&nfh);
-			nfsm_srvpostop_attr(0, &postat);
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_SRVFH(nd->nd_vers, &nfh) +
+			NFSX_FATTR(nd->nd_vers) + NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		if (!nd->nd_repstat) {
+			nfsm_chain_add_postop_fh(error, &nmrep, nfh.nfh_fhp, nfh.nfh_len);
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, postattrerr, &postattr);
 		}
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 	} else {
-		nfsm_srvfhtom(&nfh, v3);
-		nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-		nfsm_srvfillattr(&postat, fp);
+		nfsm_chain_add_fh(error, &nmrep, NFS_VER2, nfh.nfh_fhp, nfh.nfh_len);
+		if (!error)
+			error = nfsm_chain_add_fattr(nd, &nmrep, &postattr);
 	}
-	return (0);
 nfsmout:
-	if (nd.ni_cnd.cn_nameiop) {
+	nfsm_chain_build_done(error, &nmrep);
+	if (ni.ni_cnd.cn_nameiop) {
 	        /*
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
+		nameidone(&ni);
 
 		if (vp)
 			vnode_put(vp);
@@ -1930,6 +2102,10 @@ nfsmout:
 	}
 	if (dirp)
 		vnode_put(dirp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -1937,85 +2113,87 @@ nfsmout:
  * nfs v3 mknod service
  */
 int
-nfsrv_mknod(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_mknod(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct vnode_attr dirfor, diraft, postat;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
-	u_long *tl;
-	struct nameidata nd;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len, dirfor_ret = 1, diraft_ret = 1;
+	struct vnode_attr dpreattr, dpostattr, postattr;
+	struct vnode_attr va, *vap = &va;
+	struct nameidata ni;
+	int error, dpreattrerr, dpostattrerr, postattrerr;
+	uint32_t len;
 	u_long major, minor;
 	enum vtype vtyp;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	vnode_t vp, dvp, dirp = NULL;
+	vnode_t vp, dvp, dirp;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	struct vfs_context hacked_context;	/* XXX should we have this? */
-	struct vfs_context context;
 	uid_t saved_uid;
-        kauth_acl_t xacl = NULL;
+	kauth_acl_t xacl = NULL;
+	struct nfsm_chain *nmreq, nmrep;
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	dpreattrerr = dpostattrerr = postattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = dvp = dirp = NULL;
+	ni.ni_cnd.cn_nameiop = 0;
 
 	/*
 	 * Save the original credential UID in case they are
 	 * mapped and we need to map the IDs in the attributes.
 	 */
-	saved_uid = kauth_cred_getuid(nfsd->nd_cr);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
 
-	vp = dvp = NULL;
-	nd.ni_cnd.cn_nameiop = 0;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, 1);
+	nfsm_chain_get_fh_ptr(error, nmreq, NFS_VER3, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
 
-	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
-	error = nfsm_path_mbuftond(&md, &dpos, 1, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+	ni.ni_cnd.cn_nameiop = CREATE;
+	ni.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-		nfsm_srv_pre_vattr_init(&dirfor, 1);
-		dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+		nfsm_srv_pre_vattr_init(&dpreattr);
+		dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 	}
 	if (error) {
-		nd.ni_cnd.cn_nameiop = 0;
-		nfsm_reply(NFSX_WCCDATA(1));
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		if (dirp)
-			vnode_put(dirp);
-		return (0);
+		ni.ni_cnd.cn_nameiop = 0;
+		goto nfsmerr;
 	}
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
 
-	nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-	vtyp = nfsv3tov_type(*tl);
-	if (vtyp != VCHR && vtyp != VBLK && vtyp != VSOCK && vtyp != VFIFO) {
+	dvp = ni.ni_dvp;
+	vp = ni.ni_vp;
+
+	nfsm_chain_get_32(error, nmreq, vtyp);
+	nfsmerr_if(error);
+	vtyp = nfstov_type(vtyp, NFS_VER3);
+	if (!error && (vtyp != VCHR) && (vtyp != VBLK) && (vtyp != VSOCK) && (vtyp != VFIFO)) {
 		error = NFSERR_BADTYPE;
 		goto out;
 	}
-	VATTR_INIT(vap);
-	nfsm_srvsattr(vap);
 
-	if (vtyp == VCHR || vtyp == VBLK) {
-		nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-		major = fxdr_unsigned(u_long, *tl++);
-		minor = fxdr_unsigned(u_long, *tl);
+	VATTR_INIT(vap);
+	error = nfsm_chain_get_sattr(nd, nmreq, vap);
+	if ((vtyp == VCHR) || (vtyp == VBLK)) {
+		nfsm_chain_get_32(error, nmreq, major);
+		nfsm_chain_get_32(error, nmreq, minor);
+		nfsmerr_if(error);
 		VATTR_SET(vap, va_rdev, makedev(major, minor));
 	}
+	nfsmerr_if(error);
 
 	/*
 	 * If it doesn't exist, create it.
@@ -2030,15 +2208,15 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 	 * If the credentials were mapped, we should
 	 * map the same values in the attributes.
 	 */
-	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nfsd->nd_cr) != saved_uid)) {
+	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nd->nd_cr) != saved_uid)) {
 		int ismember;
-		VATTR_SET(vap, va_uid, kauth_cred_getuid(nfsd->nd_cr));
-		if (kauth_cred_ismember_gid(nfsd->nd_cr, vap->va_gid, &ismember) || !ismember)
-			VATTR_SET(vap, va_gid, kauth_cred_getgid(nfsd->nd_cr));
+		VATTR_SET(vap, va_uid, kauth_cred_getuid(nd->nd_cr));
+		if (kauth_cred_ismember_gid(nd->nd_cr, vap->va_gid, &ismember) || !ismember)
+			VATTR_SET(vap, va_gid, kauth_cred_getgid(nd->nd_cr));
 	}
 
 	/* authorize before creating */
-	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context, nxo, 0);
+	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx, nxo, 0);
 
 	/* construct ACL and handle inheritance */
 	if (!error) {
@@ -2046,7 +2224,7 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 		    NULL,
 		    &xacl,
 		    0 /* !isdir */,
-		    &context);
+		    ctx);
 
 		if (!error && xacl != NULL)
 		        VATTR_SET(vap, va_acl, xacl);
@@ -2056,54 +2234,50 @@ nfsrv_mknod(nfsd, slp, procp, mrq)
 
 	/* validate new-file security information */
 	if (!error) {
-		error = vnode_authattr_new(dvp, vap, 0, &context);
+		error = vnode_authattr_new(dvp, vap, 0, ctx);
 		if (error && (VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid))) {
 			/*
-			 * Most NFS servers just ignore the UID/GID attributes, so we   
+			 * Most NFS servers just ignore the UID/GID attributes, so we
 			 * try ignoring them if that'll help the request succeed.
 			 */
 			VATTR_CLEAR_ACTIVE(vap, va_uid);
 			VATTR_CLEAR_ACTIVE(vap, va_gid);
-			error = vnode_authattr_new(dvp, vap, 0, &context);
+			error = vnode_authattr_new(dvp, vap, 0, ctx);
 		}
 	}
+	if (error)
+		goto out1;
 
 	if (vtyp == VSOCK) {
-		error = VNOP_CREATE(dvp, &vp, &nd.ni_cnd, vap, &context);
+		error = VNOP_CREATE(dvp, &vp, &ni.ni_cnd, vap, ctx);
 
 		if (!error && !VATTR_ALL_SUPPORTED(vap))
 		        /*
 			 * If some of the requested attributes weren't handled by the VNOP,
 			 * use our fallback code.
 			 */
-			error = vnode_setattr_fallback(vp, vap, &context);
+			error = vnode_setattr_fallback(vp, vap, ctx);
 	} else {
-		if (vtyp != VFIFO && (error = suser(nfsd->nd_cr, (u_short *)0))) {
+		if (vtyp != VFIFO && (error = suser(nd->nd_cr, (u_short *)0)))
 			goto out1;
-		}
-		if ((error = VNOP_MKNOD(dvp, &vp, &nd.ni_cnd, vap, &context))) {
+		if ((error = VNOP_MKNOD(dvp, &vp, &ni.ni_cnd, vap, ctx)))
 			goto out1;
-		}
 		if (vp) {
 			vnode_recycle(vp);
 			vnode_put(vp);
 			vp = NULL;
 		}
-		hacked_context.vc_proc = procp;
-		hacked_context.vc_ucred = kauth_cred_proc_ref(procp);
-
-		nd.ni_cnd.cn_nameiop = LOOKUP;
-		nd.ni_cnd.cn_flags &= ~LOCKPARENT;
-		nd.ni_cnd.cn_context = &hacked_context;
-		nd.ni_startdir = dvp;
-		nd.ni_usedvp   = dvp;
-		error = lookup(&nd);
+		ni.ni_cnd.cn_nameiop = LOOKUP;
+		ni.ni_cnd.cn_flags &= ~LOCKPARENT;
+		ni.ni_cnd.cn_context = vfs_context_current();
+		ni.ni_startdir = dvp;
+		ni.ni_usedvp   = dvp;
+		error = lookup(&ni);
 		if (!error) {
-		        vp = nd.ni_vp;
-			if (nd.ni_cnd.cn_flags & ISSYMLINK)
+		        vp = ni.ni_vp;
+			if (ni.ni_cnd.cn_flags & ISSYMLINK)
 			        error = EINVAL;
 		}
-		kauth_cred_unref(&hacked_context.vc_ucred);
 	}
 out1:
 	if (xacl != NULL)
@@ -2113,47 +2287,66 @@ out:
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
-	nd.ni_cnd.cn_nameiop = 0;
+	nameidone(&ni);
+	ni.ni_cnd.cn_nameiop = 0;
 
 	vnode_put(dvp);
+	dvp = NULL;
 
 	if (!error) {
-		error = nfsrv_vptofh(nx, 0, NULL, vp, &context, &nfh);
+		error = nfsrv_vptofh(nx, NFS_VER3, NULL, vp, ctx, &nfh);
 		if (!error) {
-			nfsm_srv_vattr_init(&postat, 1);
-			error = vnode_getattr(vp, &postat, &context);
+			nfsm_srv_vattr_init(&postattr, NFS_VER3);
+			postattrerr = vnode_getattr(vp, &postattr, ctx);
 		}
 	}
-	if (vp)
+	if (vp) {
 		vnode_put(vp);
+		vp = NULL;
+	}
 
-	nfsm_srv_vattr_init(&diraft, 1);
-	diraft_ret = vnode_getattr(dirp, &diraft, &context);
+	nfsm_srv_vattr_init(&dpostattr, NFS_VER3);
+	dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 	vnode_put(dirp);
 	dirp = NULL;
 
-	nfsm_reply(NFSX_SRVFH(1, &nfh) + NFSX_POSTOPATTR(1) + NFSX_WCCDATA(1));
-	if (!error) {
-		nfsm_srvpostop_fh(&nfh);
-		nfsm_srvpostop_attr(0, &postat);
-	}
-	nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-	return (0);
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_SRVFH(NFS_VER3, &nfh) +
+			NFSX_POSTOPATTR(NFS_VER3) + NFSX_WCCDATA(NFS_VER3));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (!nd->nd_repstat) {
+		nfsm_chain_add_postop_fh(error, &nmrep, nfh.nfh_fhp, nfh.nfh_len);
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, postattrerr, &postattr);
+	}
+	nfsm_chain_add_wcc_data(error, nd, &nmrep,
+		dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 nfsmout:
-	if (nd.ni_cnd.cn_nameiop) {
+	nfsm_chain_build_done(error, &nmrep);
+	if (ni.ni_cnd.cn_nameiop) {
 	        /*
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
+		nameidone(&ni);
 
 		if (vp)
 			vnode_put(vp);
 		vnode_put(dvp);
 	}
+	if (dvp)
+		vnode_put(dvp);
+	if (vp)
+		vnode_put(vp);
 	if (dirp)
 		vnode_put(dirp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -2161,54 +2354,61 @@ nfsmout:
  * nfs remove service
  */
 int
-nfsrv_remove(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_remove(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct nameidata nd;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len, dirfor_ret = 1, diraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mreq;
+	struct nameidata ni;
+	int error, dpreattrerr, dpostattrerr;
+	uint32_t len;
+	uid_t saved_uid;
 	vnode_t vp, dvp, dirp = NULL;
-	struct vnode_attr dirfor, diraft;
+	struct vnode_attr dpreattr, dpostattr;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	struct vfs_context context;
-
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
-	dvp = vp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, v3);
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	dpreattrerr = dpostattrerr = ENOENT;
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+	dvp = vp = dirp = NULL;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+
+	ni.ni_cnd.cn_nameiop = DELETE;
+	ni.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
 
-	nd.ni_cnd.cn_nameiop = DELETE;
-	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-	        if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+	        if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
+
 	if (!error) {
-		dvp = nd.ni_dvp;
-		vp = nd.ni_vp;
+		dvp = ni.ni_dvp;
+		vp = ni.ni_vp;
 
 		if (vnode_vtype(vp) == VDIR)
 			error = EPERM;		/* POSIX */
@@ -2218,31 +2418,70 @@ nfsrv_remove(nfsd, slp, procp, mrq)
 			 */
 			error = EBUSY;
 		else
-			error = nfsrv_authorize(vp, dvp, KAUTH_VNODE_DELETE, &context, nxo, 0);
+			error = nfsrv_authorize(vp, dvp, KAUTH_VNODE_DELETE, ctx, nxo, 0);
 
-		if (!error)
-			error = VNOP_REMOVE(dvp, vp, &nd.ni_cnd, 0, &context);
+		if (!error) {
+#if CONFIG_FSE
+			char     *path = NULL;
+			int       plen;
+			fse_info  finfo;
+	
+			if (nfsrv_fsevents_enabled && need_fsevent(FSE_DELETE, dvp)) {
+				plen = MAXPATHLEN;
+				if ((path = get_pathbuff()) && !vn_getpath(vp, path, &plen)) {
+					get_fse_info(vp, &finfo, ctx);
+				} else if (path) {
+					release_pathbuff(path);
+					path = NULL;
+				}
+			}
+#endif
+		    	error = VNOP_REMOVE(dvp, vp, &ni.ni_cnd, 0, ctx);
+	
+#if CONFIG_FSE
+			if (path) {
+				if (!error)
+					add_fsevent(FSE_DELETE, ctx,
+						    FSE_ARG_STRING, plen, path,
+						    FSE_ARG_FINFO, &finfo,
+						    FSE_ARG_DONE);
+			        release_pathbuff(path);
+			}
+#endif
+		}
 
 		/*
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
+		nameidone(&ni);
 
 		vnode_put(vp);
 	        vnode_put(dvp);
 	}
+
+nfsmerr:
 	if (dirp) {
-		nfsm_srv_vattr_init(&diraft, v3);
-	        diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, nd->nd_vers);
+	        dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
 	}
-	nfsm_reply(NFSX_WCCDATA(v3));
-	if (v3) {
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		return (0);
-	}
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -2250,98 +2489,97 @@ nfsmout:
  * nfs rename service
  */
 int
-nfsrv_rename(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_rename(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
 	kauth_cred_t saved_cred = NULL;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, fromlen, tolen;
-	int fdirfor_ret = 1, fdiraft_ret = 1;
-	int tdirfor_ret = 1, tdiraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2, *frompath = NULL, *topath = NULL;
-	mbuf_t mb, mreq;
-	struct nameidata fromnd, tond;
-	vnode_t fvp, tvp, tdvp, fdvp, fdirp = NULL;
-	vnode_t tdirp = NULL;
-	struct vnode_attr fdirfor, fdiraft, tdirfor, tdiraft;
+	uid_t saved_uid;
+	int error;
+	uint32_t fromlen, tolen;
+	int fdpreattrerr, fdpostattrerr;
+	int tdpreattrerr, tdpostattrerr;
+	char *frompath = NULL, *topath = NULL;
+	struct nameidata fromni, toni;
+	vnode_t fvp, tvp, tdvp, fdvp, fdirp, tdirp;
+	struct vnode_attr fdpreattr, fdpostattr;
+	struct vnode_attr tdpreattr, tdpostattr;
 	struct nfs_filehandle fnfh, tnfh;
 	struct nfs_export *fnx, *tnx;
 	struct nfs_export_options *fnxo, *tnxo;
 	enum vtype fvtype, tvtype;
 	int holding_mntlock;
 	mount_t locked_mp;
-	struct vfs_context context;
-
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
-#ifndef nolint
-	fvp = (vnode_t)0;
+	struct nfsm_chain *nmreq, nmrep;
+	char *from_name, *to_name;
+#if CONFIG_FSE
+	int from_len, to_len;
+	fse_info from_finfo, to_finfo;
 #endif
+	u_char didstats = 0;
+	const char *oname;
+
+	error = 0;
+	fdpreattrerr = fdpostattrerr = ENOENT;
+	tdpreattrerr = tdpostattrerr = ENOENT;
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+	frompath = topath = NULL;
+	fdirp = tdirp = NULL;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
 
 	/*
-	 * these need to be set before
-	 * calling any nfsm_xxxx macros
-	 * since they may take us out
-	 * through the error path
+	 * these need to be set before calling any code
+	 * that they may take us out through the error path.
 	 */
 	holding_mntlock = 0;
 	fvp = tvp = NULL;
 	fdvp = tdvp = NULL;
 	locked_mp = NULL;
 
-	nfsm_srvmtofh(&fnfh);
-	nfsm_srvnamesiz(fromlen, v3);
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &fromlen, &fromnd);
-	if (error) {
-		nfsm_reply(0);
-		return (0);
-	}
-	frompath = fromnd.ni_cnd.cn_pnbuf;
-	nfsm_srvmtofh(&tnfh);
-	nfsm_strsiz(tolen, NFS_MAXNAMLEN, v3);
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &tolen, &tond);
-	if (error) {
-		nfsm_reply(0);
-		FREE_ZONE(frompath, MAXPATHLEN, M_NAMEI);
-		return (0);
-	}
-	topath = tond.ni_cnd.cn_pnbuf;
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, fnfh.nfh_fhp, fnfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, fromlen);
+	nfsm_name_len_check(error, nd, fromlen);
+	nfsmerr_if(error);
+	error = nfsm_chain_get_path_namei(nmreq, fromlen, &fromni);
+	nfsmerr_if(error);
+	frompath = fromni.ni_cnd.cn_pnbuf;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, tnfh.nfh_fhp, tnfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, tolen);
+	nfsm_name_len_check(error, nd, tolen);
+	nfsmerr_if(error);
+	error = nfsm_chain_get_path_namei(nmreq, tolen, &toni);
+	nfsmerr_if(error);
+	topath = toni.ni_cnd.cn_pnbuf;
 
 	/*
 	 * Remember our original uid so that we can reset cr_uid before
-	 * the second nfs_namei() call, in case it is remapped.
+	 * the second nfsrv_namei() call, in case it is remapped.
 	 */
-	saved_cred = nfsd->nd_cr;
+	saved_cred = nd->nd_cr;
 	kauth_cred_ref(saved_cred);
 retry:
-	fromnd.ni_cnd.cn_nameiop = DELETE;
-	fromnd.ni_cnd.cn_flags = WANTPARENT;
+	fromni.ni_cnd.cn_nameiop = DELETE;
+	fromni.ni_cnd.cn_flags = WANTPARENT;
 
-	fromnd.ni_cnd.cn_pnbuf = frompath;
+	fromni.ni_cnd.cn_pnbuf = frompath;
 	frompath = NULL;
-	fromnd.ni_cnd.cn_pnlen = MAXPATHLEN;
-	fromnd.ni_cnd.cn_flags |= HASBUF;
+	fromni.ni_cnd.cn_pnlen = MAXPATHLEN;
+	fromni.ni_cnd.cn_flags |= HASBUF;
 
-	error = nfs_namei(nfsd, &context, &fromnd, &fnfh, nam, FALSE, &fdirp, &fnx, &fnxo);
+	error = nfsrv_namei(nd, ctx, &fromni, &fnfh, &fdirp, &fnx, &fnxo);
 	if (error)
 		goto out;
-	fdvp = fromnd.ni_dvp;
-	fvp  = fromnd.ni_vp;
+	fdvp = fromni.ni_dvp;
+	fvp  = fromni.ni_vp;
 
 	if (fdirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&fdirfor, v3);
-			fdirfor_ret = vnode_getattr(fdirp, &fdirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&fdpreattr);
+			fdpreattrerr = vnode_getattr(fdirp, &fdpreattr, ctx);
 		} else {
 			vnode_put(fdirp);
 			fdirp = NULL;
@@ -2350,43 +2588,56 @@ retry:
 	fvtype = vnode_vtype(fvp);
 
 	/* reset credential if it was remapped */
-	if (nfsd->nd_cr != saved_cred) {
+	if (nd->nd_cr != saved_cred) {
 		kauth_cred_ref(saved_cred);
-		kauth_cred_unref(&nfsd->nd_cr);
-		nfsd->nd_cr = saved_cred;
+		kauth_cred_unref(&nd->nd_cr);
+		ctx->vc_ucred = nd->nd_cr = saved_cred;
 	}
 
-	tond.ni_cnd.cn_nameiop = RENAME;
-	tond.ni_cnd.cn_flags = WANTPARENT;
+	toni.ni_cnd.cn_nameiop = RENAME;
+	toni.ni_cnd.cn_flags = WANTPARENT;
 
-	tond.ni_cnd.cn_pnbuf = topath;
+	toni.ni_cnd.cn_pnbuf = topath;
 	topath = NULL;
-	tond.ni_cnd.cn_pnlen = MAXPATHLEN;
-	tond.ni_cnd.cn_flags |= HASBUF;
+	toni.ni_cnd.cn_pnlen = MAXPATHLEN;
+	toni.ni_cnd.cn_flags |= HASBUF;
 
 	if (fvtype == VDIR)
-		tond.ni_cnd.cn_flags |= WILLBEDIR;
+		toni.ni_cnd.cn_flags |= WILLBEDIR;
 
-	error = nfs_namei(nfsd, &context, &tond, &tnfh, nam, FALSE, &tdirp, &tnx, &tnxo);
+	tnx = NULL;
+	error = nfsrv_namei(nd, ctx, &toni, &tnfh, &tdirp, &tnx, &tnxo);
 	if (error) {
 		/*
 		 * Translate error code for rename("dir1", "dir2/.").
 		 */
 	        if (error == EISDIR && fvtype == VDIR) {
-		        if (v3)
+		        if (nd->nd_vers == NFS_VER3)
 			        error = EINVAL;
 			else
 			        error = ENOTEMPTY;
 		}
 		goto out;
 	}
-	tdvp = tond.ni_dvp;
-	tvp  = tond.ni_vp;
+	tdvp = toni.ni_dvp;
+	tvp  = toni.ni_vp;
+
+	if (!didstats) {
+		/* update export stats once only */
+		if (tnx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&tnx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(tnx, nd, saved_uid, 1, 0, 0);
+			didstats = 1;
+		}
+	}
 
 	if (tdirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&tdirfor, v3);
-			tdirfor_ret = vnode_getattr(tdirp, &tdirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&tdpreattr);
+			tdpreattrerr = vnode_getattr(tdirp, &tdpreattr, ctx);
 		} else {
 			vnode_put(tdirp);
 			tdirp = NULL;
@@ -2397,20 +2648,20 @@ retry:
 		tvtype = vnode_vtype(tvp);
 
 		if (fvtype == VDIR && tvtype != VDIR) {
-			if (v3)
+			if (nd->nd_vers == NFS_VER3)
 				error = EEXIST;
 			else
 				error = EISDIR;
 			goto out;
 		} else if (fvtype != VDIR && tvtype == VDIR) {
-			if (v3)
+			if (nd->nd_vers == NFS_VER3)
 				error = EEXIST;
 			else
 				error = ENOTDIR;
 			goto out;
 		}
 		if (tvtype == VDIR && vnode_mountedhere(tvp)) {
-			if (v3)
+			if (nd->nd_vers == NFS_VER3)
 				error = EXDEV;
 			else
 				error = ENOTEMPTY;
@@ -2418,7 +2669,7 @@ retry:
 		}
 	}
 	if (fvp == tdvp) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EINVAL;
 		else
 			error = ENOTEMPTY;
@@ -2449,24 +2700,24 @@ retry:
 		}
 		if (moving) {
 			/* moving out of fdvp, must have delete rights */
-			if ((error = nfsrv_authorize(fvp, fdvp, KAUTH_VNODE_DELETE, &context, fnxo, 0)) != 0)
+			if ((error = nfsrv_authorize(fvp, fdvp, KAUTH_VNODE_DELETE, ctx, fnxo, 0)) != 0)
 				goto auth_exit;
 			/* moving into tdvp or tvp, must have rights to add */
 			if ((error = nfsrv_authorize(((tvp != NULL) && vnode_isdir(tvp)) ? tvp : tdvp,
-				 NULL, 
+				 NULL,
 				 vnode_isdir(fvp) ? KAUTH_VNODE_ADD_SUBDIRECTORY : KAUTH_VNODE_ADD_FILE,
-				 &context, tnxo, 0)) != 0)
+				 ctx, tnxo, 0)) != 0)
 				goto auth_exit;
 		} else {
 			/* node staying in same directory, must be allowed to add new name */
 			if ((error = nfsrv_authorize(fdvp, NULL,
 				 vnode_isdir(fvp) ? KAUTH_VNODE_ADD_SUBDIRECTORY : KAUTH_VNODE_ADD_FILE,
-				 &context, fnxo, 0)) != 0)
+				 ctx, fnxo, 0)) != 0)
 				goto auth_exit;
 		}
 		/* overwriting tvp */
 		if ((tvp != NULL) && !vnode_isdir(tvp) &&
-		    ((error = nfsrv_authorize(tvp, tdvp, KAUTH_VNODE_DELETE, &context, tnxo, 0)) != 0))
+		    ((error = nfsrv_authorize(tvp, tdvp, KAUTH_VNODE_DELETE, ctx, tnxo, 0)) != 0))
 			goto auth_exit;
 
 		/* XXX more checks? */
@@ -2479,7 +2730,7 @@ auth_exit:
 
 	if ((vnode_mount(fvp) != vnode_mount(tdvp)) ||
 	    (tvp && (vnode_mount(fvp) != vnode_mount(tvp)))) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EXDEV;
 		else
 			error = ENOTEMPTY;
@@ -2501,14 +2752,14 @@ auth_exit:
 	 *    o tvp
 	 */
 	if (tdvp->v_parent == fvp) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EXDEV;
 		else
 			error = ENOTEMPTY;
 		goto out;
 	}
 	if (fvtype == VDIR && vnode_mountedhere(fvp)) {
-		if (v3)
+		if (nd->nd_vers == NFS_VER3)
 			error = EXDEV;
 		else
 			error = ENOTEMPTY;
@@ -2532,9 +2783,9 @@ auth_exit:
 	 * that correct behaviour then is just to remove the source (link)
 	 */
 	if ((fvp == tvp) && (fdvp == tdvp)) {
-		if (fromnd.ni_cnd.cn_namelen == tond.ni_cnd.cn_namelen &&
-	       	    !bcmp(fromnd.ni_cnd.cn_nameptr, tond.ni_cnd.cn_nameptr,
-			  fromnd.ni_cnd.cn_namelen)) {
+		if (fromni.ni_cnd.cn_namelen == toni.ni_cnd.cn_namelen &&
+	       	    !bcmp(fromni.ni_cnd.cn_nameptr, toni.ni_cnd.cn_nameptr,
+			  fromni.ni_cnd.cn_namelen)) {
 			goto out;
 		}
 	}
@@ -2572,31 +2823,31 @@ auth_exit:
 			locked_mp = vnode_mount(fvp);
 			mount_ref(locked_mp, 0);
 
-			/* make a copy of to path to pass to nfs_namei() again */
+			/* make a copy of to path to pass to nfsrv_namei() again */
 			MALLOC_ZONE(topath, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
 			if (topath)
-				bcopy(tond.ni_cnd.cn_pnbuf, topath, tolen + 1);
+				bcopy(toni.ni_cnd.cn_pnbuf, topath, tolen + 1);
 
 			/*
 			 * nameidone has to happen before we vnode_put(tdvp)
 			 * since it may need to release the fs_nodelock on the tdvp
 			 */
-			nameidone(&tond);
+			nameidone(&toni);
 
 			if (tvp)
 			        vnode_put(tvp);
 			vnode_put(tdvp);
 
-			/* make a copy of from path to pass to nfs_namei() again */
+			/* make a copy of from path to pass to nfsrv_namei() again */
 			MALLOC_ZONE(frompath, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
 			if (frompath)
-				bcopy(fromnd.ni_cnd.cn_pnbuf, frompath, fromlen + 1);
+				bcopy(fromni.ni_cnd.cn_pnbuf, frompath, fromlen + 1);
 
 			/*
 			 * nameidone has to happen before we vnode_put(fdvp)
 			 * since it may need to release the fs_nodelock on the fdvp
 			 */
-			nameidone(&fromnd);
+			nameidone(&fromni);
 
 			vnode_put(fvp);
 			vnode_put(fdvp);
@@ -2615,7 +2866,7 @@ auth_exit:
 			fvp = tvp = NULL;
 			fdvp = tdvp = NULL;
 
-			fdirfor_ret = tdirfor_ret = 1;
+			fdpreattrerr = tdpreattrerr = ENOENT;
 
 			if (!topath || !frompath) {
 				/* we couldn't allocate a path, so bail */
@@ -2623,12 +2874,19 @@ auth_exit:
 				goto out;
 			}
 
+			/* reset credential if it was remapped */
+			if (nd->nd_cr != saved_cred) {
+				kauth_cred_ref(saved_cred);
+				kauth_cred_unref(&nd->nd_cr);
+				ctx->vc_ucred = nd->nd_cr = saved_cred;
+			}
+
 			goto retry;
 		}
 	} else {
 	        /*
 		 * when we dropped the iocounts to take
-		 * the lock, we allowed the identity of 
+		 * the lock, we allowed the identity of
 		 * the various vnodes to change... if they did,
 		 * we may no longer be dealing with a rename
 		 * that reshapes the tree... once we're holding
@@ -2644,13 +2902,71 @@ auth_exit:
 	}
 
 	// save these off so we can later verify that fvp is the same
-	char *oname;
 	vnode_t oparent;
 	oname   = fvp->v_name;
 	oparent = fvp->v_parent;
 
-	error = VNOP_RENAME(fromnd.ni_dvp, fromnd.ni_vp, &fromnd.ni_cnd,
-			    tond.ni_dvp, tond.ni_vp, &tond.ni_cnd, &context);
+	/*
+	 * If generating an fsevent, then
+	 * stash any pre-rename info we may need.
+	 */
+#if CONFIG_FSE
+	if (nfsrv_fsevents_enabled && need_fsevent(FSE_RENAME, fvp)) {
+	        get_fse_info(fvp, &from_finfo, ctx);
+		if (tvp)
+		        get_fse_info(tvp, &to_finfo, ctx);
+
+	        from_name = get_pathbuff();
+		from_len = MAXPATHLEN;
+		if (from_name && vn_getpath(fdvp, from_name, &from_len)) {
+			release_pathbuff(from_name);
+			from_name = NULL;
+		} else if ((from_len + 1 + fromni.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+			// if the path is not just "/", then append a "/"
+			if (from_len > 2) {
+				from_name[from_len-1] = '/';
+			} else {
+				from_len--;
+			}
+			strlcpy(&from_name[from_len], fromni.ni_cnd.cn_nameptr, MAXPATHLEN-from_len);
+			from_len += fromni.ni_cnd.cn_namelen + 1;
+			from_name[from_len] = '\0';
+		}
+
+		to_name = from_name ? get_pathbuff() : NULL;
+		to_len = MAXPATHLEN;
+
+		if (!to_name) {
+			if (from_name) {
+				release_pathbuff(from_name);
+				from_name = NULL;
+			}
+		} else if (vn_getpath(tdvp, to_name, &to_len)) {
+			release_pathbuff(from_name);
+			release_pathbuff(to_name);
+			from_name = to_name = NULL;
+		} else if ((to_len + 1 + toni.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+			// if the path is not just "/", then append a "/"
+			if (to_len > 2) {
+				to_name[to_len-1] = '/';
+			} else {
+				to_len--;
+			}
+			strlcpy(&to_name[to_len], toni.ni_cnd.cn_nameptr, MAXPATHLEN-to_len);
+			to_len += toni.ni_cnd.cn_namelen + 1;
+			to_name[to_len] = '\0';
+		}
+	} else {
+	        from_name = NULL;
+	        to_name   = NULL;
+	}
+#else /* CONFIG_FSE */
+	from_name = NULL;
+	to_name   = NULL;
+#endif /* CONFIG_FSE */
+
+	error = VNOP_RENAME(fromni.ni_dvp, fromni.ni_vp, &fromni.ni_cnd,
+			    toni.ni_dvp, toni.ni_vp, &toni.ni_cnd, ctx);
 	/*
 	 * fix up name & parent pointers.  note that we first	
 	 * check that fvp has the same name/parent pointers it
@@ -2662,8 +2978,38 @@ auth_exit:
 		update_flags = VNODE_UPDATE_NAME;
 		if (fdvp != tdvp)
 			update_flags |= VNODE_UPDATE_PARENT;
-		vnode_update_identity(fvp, tdvp, tond.ni_cnd.cn_nameptr, tond.ni_cnd.cn_namelen, tond.ni_cnd.cn_hash, update_flags);
+		vnode_update_identity(fvp, tdvp, toni.ni_cnd.cn_nameptr,
+			toni.ni_cnd.cn_namelen, toni.ni_cnd.cn_hash, update_flags);
+	}
+
+	/*
+	 * If the rename is OK and we've got the paths
+	 * then add an fsevent.
+	 */
+#if CONFIG_FSE
+	if (nfsrv_fsevents_enabled && !error && from_name && to_name) {
+	        if (tvp) {
+		        add_fsevent(FSE_RENAME, ctx,
+				    FSE_ARG_STRING, from_len, from_name,
+				    FSE_ARG_FINFO, &from_finfo,
+				    FSE_ARG_STRING, to_len, to_name,
+				    FSE_ARG_FINFO, &to_finfo,
+				    FSE_ARG_DONE);
+		} else {
+		        add_fsevent(FSE_RENAME, ctx,
+				    FSE_ARG_STRING, from_len, from_name,
+				    FSE_ARG_FINFO, &from_finfo,
+				    FSE_ARG_STRING, to_len, to_name,
+				    FSE_ARG_DONE);
+		}
 	}
+	if (from_name)
+	        release_pathbuff(from_name);
+	if (to_name)
+	        release_pathbuff(to_name);
+#endif /* CONFIG_FSE */
+	from_name = to_name = NULL;
+		
 out:
 	if (holding_mntlock) {
 	        mount_unlock_renames(locked_mp);
@@ -2675,7 +3021,7 @@ out:
 		 * nameidone has to happen before we vnode_put(tdvp)
 		 * since it may need to release the fs_nodelock on the tdvp
 		 */
-		nameidone(&tond);
+		nameidone(&toni);
 		if (tvp)
 		        vnode_put(tvp);
 	        vnode_put(tdvp);
@@ -2687,7 +3033,7 @@ out:
 		 * nameidone has to happen before we vnode_put(fdvp)
 		 * since it may need to release the fs_nodelock on the fdvp
 		 */
-		nameidone(&fromnd);
+		nameidone(&fromni);
 
 		if (fvp)
 		        vnode_put(fvp);
@@ -2696,31 +3042,33 @@ out:
 		fdvp = NULL;
 	}
 	if (fdirp) {
-		nfsm_srv_vattr_init(&fdiraft, v3);
-		fdiraft_ret = vnode_getattr(fdirp, &fdiraft, &context);
+		nfsm_srv_vattr_init(&fdpostattr, nd->nd_vers);
+		fdpostattrerr = vnode_getattr(fdirp, &fdpostattr, ctx);
 		vnode_put(fdirp);
 		fdirp = NULL;
 	}
 	if (tdirp) {
-		nfsm_srv_vattr_init(&tdiraft, v3);
-		tdiraft_ret = vnode_getattr(tdirp, &tdiraft, &context);
+		nfsm_srv_vattr_init(&tdpostattr, nd->nd_vers);
+		tdpostattrerr = vnode_getattr(tdirp, &tdpostattr, ctx);
 		vnode_put(tdirp);
 		tdirp = NULL;
 	}
-	nfsm_reply(2 * NFSX_WCCDATA(v3));
-	if (v3) {
-		nfsm_srvwcc_data(fdirfor_ret, &fdirfor, fdiraft_ret, &fdiraft);
-		nfsm_srvwcc_data(tdirfor_ret, &tdirfor, tdiraft_ret, &tdiraft);
-	}
-	if (frompath)
-		FREE_ZONE(frompath, MAXPATHLEN, M_NAMEI);
-	if (topath)
-		FREE_ZONE(topath, MAXPATHLEN, M_NAMEI);
-	if (saved_cred)
-		kauth_cred_unref(&saved_cred);
-	return (0);
 
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, 2 * NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			fdpreattrerr, &fdpreattr, fdpostattrerr, &fdpostattr);
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			tdpreattrerr, &tdpreattr, tdpostattrerr, &tdpostattr);
+	}
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
 	if (holding_mntlock) {
 	        mount_unlock_renames(locked_mp);
 		mount_drop(locked_mp, 0);
@@ -2730,7 +3078,7 @@ nfsmout:
 		 * nameidone has to happen before we vnode_put(tdvp)
 		 * since it may need to release the fs_nodelock on the tdvp
 		 */
-		nameidone(&tond);
+		nameidone(&toni);
 
 		if (tvp)
 		        vnode_put(tvp);
@@ -2741,7 +3089,7 @@ nfsmout:
 		 * nameidone has to happen before we vnode_put(fdvp)
 		 * since it may need to release the fs_nodelock on the fdvp
 		 */
-       		nameidone(&fromnd);
+		nameidone(&fromni);
 
 		if (fvp)
 		        vnode_put(fvp);
@@ -2757,6 +3105,10 @@ nfsmout:
 		FREE_ZONE(topath, MAXPATHLEN, M_NAMEI);
 	if (saved_cred)
 		kauth_cred_unref(&saved_cred);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -2764,118 +3116,152 @@ nfsmout:
  * nfs link service
  */
 int
-nfsrv_link(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_link(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct nameidata nd;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len, dirfor_ret = 1, diraft_ret = 1;
-	int getret = 1, v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mreq;
-	vnode_t vp, xp, dvp, dirp = NULL;
-	struct vnode_attr dirfor, diraft, at;
+	struct nameidata ni;
+	int error, dpreattrerr, dpostattrerr, attrerr;
+	uint32_t len = 0;
+	vnode_t vp, xp, dvp, dirp;
+	struct vnode_attr dpreattr, dpostattr, attr;
 	struct nfs_filehandle nfh, dnfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	struct vfs_context context;
-
-	vp = xp = dvp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvmtofh(&dnfh);
-	nfsm_srvnamesiz(len, v3);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_WCCDATA(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_WCCDATA(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		return (0);
-	}
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	dpreattrerr = dpostattrerr = attrerr = ENOENT;
+	vp = xp = dvp = dirp = NULL;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, dnfh.nfh_fhp, dnfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
 
 	/* we're not allowed to link to directories... */
 	if (vnode_vtype(vp) == VDIR) {
 		error = EPERM;		/* POSIX */
-		goto out1;
+		goto out;
 	}
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-
-  	/* ...or to anything that kauth doesn't want us to (eg. immutable items) */
-  	if ((error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LINKTARGET, &context, nxo, 0)) != 0)
- 		goto out1;
+	/* ...or to anything that kauth doesn't want us to (eg. immutable items) */
+	if ((error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LINKTARGET, ctx, nxo, 0)) != 0)
+		goto out;
 
-	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
+	ni.ni_cnd.cn_nameiop = CREATE;
+	ni.ni_cnd.cn_flags = LOCKPARENT;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
 	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &dnfh, nam, FALSE, &dirp, &nx, &nxo);
+		error = nfsrv_namei(nd, ctx, &ni, &dnfh, &dirp, &nx, &nxo);
 	if (dirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
 	if (error)
-		goto out1;
-	dvp = nd.ni_dvp;
-	xp = nd.ni_vp;
+		goto out;
+	dvp = ni.ni_dvp;
+	xp = ni.ni_vp;
 
 	if (xp != NULL)
 		error = EEXIST;
 	else if (vnode_mount(vp) != vnode_mount(dvp))
 		error = EXDEV;
-	else 
-		error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context, nxo, 0);
+	else
+		error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx, nxo, 0);
 
 	if (!error)
-		error = VNOP_LINK(vp, dvp, &nd.ni_cnd, &context);
+		error = VNOP_LINK(vp, dvp, &ni.ni_cnd, ctx);
+
+#if CONFIG_FSE
+	if (nfsrv_fsevents_enabled && !error && need_fsevent(FSE_CREATE_FILE, dvp)) {
+		char *target_path = NULL;
+		int plen;
+		fse_info finfo;
+
+		/* build the path to the new link file */
+		plen = MAXPATHLEN;
+		if ((target_path = get_pathbuff()) && !vn_getpath(dvp, target_path, &plen)) {
+			if ((plen + 1 + ni.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+				target_path[plen-1] = '/';
+				strlcpy(&target_path[plen], ni.ni_cnd.cn_nameptr, MAXPATHLEN-plen);
+				plen += ni.ni_cnd.cn_namelen;
+			}
+			if (get_fse_info(vp, &finfo, ctx) == 0)
+				add_fsevent(FSE_CREATE_FILE, ctx,
+					    FSE_ARG_STRING, plen, target_path,
+					    FSE_ARG_FINFO, &finfo,
+					    FSE_ARG_DONE);
+		}
+		if (target_path)
+			release_pathbuff(target_path);
+	}
+#endif
 
-        /*
+	/*
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
+	nameidone(&ni);
 
 	if (xp)
 		vnode_put(xp);
 	vnode_put(dvp);
-out1:
-	if (v3) {
-		nfsm_srv_vattr_init(&at, v3);
-		getret = vnode_getattr(vp, &at, &context);
+out:
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_vattr_init(&attr, NFS_VER3);
+		attrerr = vnode_getattr(vp, &attr, ctx);
 	}
 	if (dirp) {
-		nfsm_srv_vattr_init(&diraft, v3);
-		diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, nd->nd_vers);
+		dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
+		dirp = NULL;
 	}
 	vnode_put(vp);
+	vp = NULL;
 
-	nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_WCCDATA(v3));
-	if (v3) {
-		nfsm_srvpostop_attr(getret, &at);
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		return (0);
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers) + NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 	}
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (vp)
+		vnode_put(vp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -2883,100 +3269,98 @@ nfsmout:
  * nfs symbolic link service
  */
 int
-nfsrv_symlink(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_symlink(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct vnode_attr dirfor, diraft, postat;
-	struct nameidata nd;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
-	u_long *tl;
-	long t1;
-	struct nfsv2_sattr *sp;
-	char *bpos, *linkdata = NULL, *cp2;
-	int error = 0, len, linkdatalen;
-	int dirfor_ret = 1, diraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	mbuf_t mb, mreq, mb2;
-	vnode_t vp, dvp, dirp = NULL;
+	struct vnode_attr dpreattr, dpostattr, postattr;
+	struct vnode_attr va, *vap = &va;
+	struct nameidata ni;
+	int error, dpreattrerr, dpostattrerr, postattrerr;
+	uint32_t len, linkdatalen;
+	uid_t saved_uid;
+	char *linkdata;
+	vnode_t vp, dvp, dirp;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	uio_t auio;
+	uio_t auio = NULL;
 	char uio_buf[ UIO_SIZEOF(1) ];
-	struct vfs_context context;
-	uid_t saved_uid;
+	struct nfsm_chain *nmreq, nmrep;
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	dpreattrerr = dpostattrerr = postattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	linkdata = NULL;
+	dirp = NULL;
 
 	/*
 	 * Save the original credential UID in case they are
 	 * mapped and we need to map the IDs in the attributes.
 	 */
-	saved_uid = kauth_cred_getuid(nfsd->nd_cr);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
 
-	nd.ni_cnd.cn_nameiop = 0;
+	ni.ni_cnd.cn_nameiop = 0;
 	vp = dvp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, v3);
 
-	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+
+	ni.ni_cnd.cn_nameiop = CREATE;
+	ni.ni_cnd.cn_flags = LOCKPARENT;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
 	if (error) {
-		nd.ni_cnd.cn_nameiop = 0;
+		ni.ni_cnd.cn_nameiop = 0;
 		goto out1;
 	}
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
+	dvp = ni.ni_dvp;
+	vp = ni.ni_vp;
 
 	VATTR_INIT(vap);
-	if (v3)
-		nfsm_srvsattr(vap);
-	nfsm_strsiz(linkdatalen, NFS_MAXPATHLEN, v3);
+	if (nd->nd_vers == NFS_VER3)
+		error = nfsm_chain_get_sattr(nd, nmreq, vap);
+	nfsm_chain_get_32(error, nmreq, linkdatalen);
+	if (!error && (((nd->nd_vers == NFS_VER2) && (linkdatalen > NFS_MAXPATHLEN)) ||
+			((nd->nd_vers == NFS_VER3) && (linkdatalen > MAXPATHLEN))))
+		error = NFSERR_NAMETOL;
+	nfsmerr_if(error);
 	MALLOC(linkdata, caddr_t, linkdatalen + 1, M_TEMP, M_WAITOK);
-	if (!linkdata) {
-		nameidone(&nd);
-		nd.ni_cnd.cn_nameiop = 0;
-		vnode_put(nd.ni_dvp);
-		vnode_put(nd.ni_vp);
-		error = ENOMEM;
-		goto out;
-	}
-	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
-				&uio_buf[0], sizeof(uio_buf));
-	if (!auio) {
-		nameidone(&nd);
-		nd.ni_cnd.cn_nameiop = 0;
-		vnode_put(nd.ni_dvp);
-		vnode_put(nd.ni_vp);
+	if (linkdata)
+		auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
+					&uio_buf[0], sizeof(uio_buf));
+	if (!linkdata || !auio) {
 		error = ENOMEM;
 		goto out;
 	}
 	uio_addiov(auio, CAST_USER_ADDR_T(linkdata), linkdatalen);
-	nfsm_mtouio(auio, linkdatalen);
-	if (!v3) {
-		nfsm_dissect(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		VATTR_SET(vap, va_mode, fxdr_unsigned(u_short, sp->sa_mode));
-	}
+	error = nfsm_chain_get_uio(nmreq, linkdatalen, auio);
+	if (!error && (nd->nd_vers == NFS_VER2))
+		error = nfsm_chain_get_sattr(nd, nmreq, vap);
+	nfsmerr_if(error);
 	*(linkdata + linkdatalen) = '\0';
 	if (vp) {
 		error = EEXIST;
@@ -2987,90 +3371,109 @@ nfsrv_symlink(nfsd, slp, procp, mrq)
 	 * If the credentials were mapped, we should
 	 * map the same values in the attributes.
 	 */
-	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nfsd->nd_cr) != saved_uid)) {
+	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nd->nd_cr) != saved_uid)) {
 		int ismember;
-		VATTR_SET(vap, va_uid, kauth_cred_getuid(nfsd->nd_cr));
-		if (kauth_cred_ismember_gid(nfsd->nd_cr, vap->va_gid, &ismember) || !ismember)
-			VATTR_SET(vap, va_gid, kauth_cred_getgid(nfsd->nd_cr));
+		VATTR_SET(vap, va_uid, kauth_cred_getuid(nd->nd_cr));
+		if (kauth_cred_ismember_gid(nd->nd_cr, vap->va_gid, &ismember) || !ismember)
+			VATTR_SET(vap, va_gid, kauth_cred_getgid(nd->nd_cr));
 	}
 	VATTR_SET(vap, va_type, VLNK);
 	VATTR_CLEAR_ACTIVE(vap, va_data_size);
 	VATTR_CLEAR_ACTIVE(vap, va_access_time);
 
 	/* authorize before creating */
-	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context, nxo, 0);
+	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx, nxo, 0);
 
 	/* validate given attributes */
 	if (!error) {
-		error = vnode_authattr_new(dvp, vap, 0, &context);
+		error = vnode_authattr_new(dvp, vap, 0, ctx);
 		if (error && (VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid))) {
 			/*
-			 * Most NFS servers just ignore the UID/GID attributes, so we   
+			 * Most NFS servers just ignore the UID/GID attributes, so we
 			 * try ignoring them if that'll help the request succeed.
 			 */
 			VATTR_CLEAR_ACTIVE(vap, va_uid);
 			VATTR_CLEAR_ACTIVE(vap, va_gid);
-			error = vnode_authattr_new(dvp, vap, 0, &context);
+			error = vnode_authattr_new(dvp, vap, 0, ctx);
 		}
 	}
 	if (!error)
-		error = VNOP_SYMLINK(dvp, &vp, &nd.ni_cnd, vap, linkdata, &context);
+		error = VNOP_SYMLINK(dvp, &vp, &ni.ni_cnd, vap, linkdata, ctx);
 
-	if (!error && v3) {
+	if (!error && (nd->nd_vers == NFS_VER3)) {
 		if (vp == NULL) {
-			nd.ni_cnd.cn_nameiop = LOOKUP;
-			nd.ni_cnd.cn_flags &= ~(LOCKPARENT | FOLLOW);
-			nd.ni_cnd.cn_flags |= (NOFOLLOW | LOCKLEAF);
-			nd.ni_cnd.cn_context = &context;
-			nd.ni_startdir = dvp;
-			nd.ni_usedvp   = dvp;
-			error = lookup(&nd);
+			ni.ni_cnd.cn_nameiop = LOOKUP;
+			ni.ni_cnd.cn_flags &= ~(LOCKPARENT | FOLLOW);
+			ni.ni_cnd.cn_flags |= (NOFOLLOW | LOCKLEAF);
+			ni.ni_cnd.cn_context = ctx;
+			ni.ni_startdir = dvp;
+			ni.ni_usedvp   = dvp;
+			error = lookup(&ni);
 			if (!error)
-			        vp = nd.ni_vp;
+			        vp = ni.ni_vp;
 		}
 		if (!error) {
-			error = nfsrv_vptofh(nx, !v3, NULL, vp, &context, &nfh);
+			error = nfsrv_vptofh(nx, NFS_VER3, NULL, vp, ctx, &nfh);
 			if (!error) {
-				nfsm_srv_vattr_init(&postat, v3);
-				error = vnode_getattr(vp, &postat, &context);
+				nfsm_srv_vattr_init(&postattr, NFS_VER3);
+				postattrerr = vnode_getattr(vp, &postattr, ctx);
 			}
 		}
 	}
+
+#if CONFIG_FSE
+	if (nfsrv_fsevents_enabled && !error && vp) {
+		add_fsevent(FSE_CREATE_FILE, ctx,
+			    FSE_ARG_VNODE, vp,
+			    FSE_ARG_DONE);
+	}
+#endif
 out:
-        /*
+	/*
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
-	nd.ni_cnd.cn_nameiop = 0;
-
+	nameidone(&ni);
+	ni.ni_cnd.cn_nameiop = 0;
 	if (vp)
 	        vnode_put(vp);
 	vnode_put(dvp);
 out1:
-	if (linkdata)
+	if (linkdata) {
 		FREE(linkdata, M_TEMP);
+		linkdata = NULL;
+	}
 	if (dirp) {
-		nfsm_srv_vattr_init(&diraft, v3);
-		diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, nd->nd_vers);
+		dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
+		dirp = NULL;
 	}
-	nfsm_reply(NFSX_SRVFH(v3, &nfh) + NFSX_POSTOPATTR(v3) + NFSX_WCCDATA(v3));
-	if (v3) {
-		if (!error) {
-			nfsm_srvpostop_fh(&nfh);
-			nfsm_srvpostop_attr(0, &postat);
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_SRVFH(nd->nd_vers, &nfh) +
+			NFSX_POSTOPATTR(nd->nd_vers) + NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		if (!nd->nd_repstat) {
+			nfsm_chain_add_postop_fh(error, &nmrep, nfh.nfh_fhp, nfh.nfh_len);
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, postattrerr, &postattr);
 		}
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 	}
-	return (0);
 nfsmout:
-	if (nd.ni_cnd.cn_nameiop) {
+	nfsm_chain_build_done(error, &nmrep);
+	if (ni.ni_cnd.cn_nameiop) {
 	        /*
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
+		nameidone(&ni);
 
 		if (vp)
 			vnode_put(vp);
@@ -3080,6 +3483,10 @@ nfsmout:
 		vnode_put(dirp);
 	if (linkdata)
 		FREE(linkdata, M_TEMP);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -3087,83 +3494,76 @@ nfsmout:
  * nfs mkdir service
  */
 int
-nfsrv_mkdir(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_mkdir(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct vnode_attr dirfor, diraft, postat;
-	struct vnode_attr va;
-	struct vnode_attr *vap = &va;
-	struct nfs_fattr *fp;
-	struct nameidata nd;
-	caddr_t cp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len;
-	int dirfor_ret = 1, diraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
-	vnode_t vp, dvp, dirp = NULL;
+	struct vnode_attr dpreattr, dpostattr, postattr;
+	struct vnode_attr va, *vap = &va;
+	struct nameidata ni;
+	int error, dpreattrerr, dpostattrerr, postattrerr;
+	uint32_t len;
+	vnode_t vp, dvp, dirp;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	struct vfs_context context;
 	uid_t saved_uid;
-        kauth_acl_t xacl = NULL;
+	kauth_acl_t xacl = NULL;
+	struct nfsm_chain *nmreq, nmrep;
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	dpreattrerr = dpostattrerr = postattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
 
 	/*
 	 * Save the original credential UID in case they are
 	 * mapped and we need to map the IDs in the attributes.
 	 */
-	saved_uid = kauth_cred_getuid(nfsd->nd_cr);
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
 
-	nd.ni_cnd.cn_nameiop = 0;
-	vp = dvp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, v3);
+	ni.ni_cnd.cn_nameiop = 0;
+	vp = dvp = dirp = NULL;
 
-	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+
+	ni.ni_cnd.cn_nameiop = CREATE;
+	ni.ni_cnd.cn_flags = LOCKPARENT;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-		if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+		if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
 	if (error) {
-		nd.ni_cnd.cn_nameiop = 0;
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		if (dirp)
-			vnode_put(dirp);
-		return (0);
+		ni.ni_cnd.cn_nameiop = 0;
+		goto nfsmerr;
 	}
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
+	dvp = ni.ni_dvp;
+	vp = ni.ni_vp;
 
 	VATTR_INIT(vap);
-	if (v3) {
-		nfsm_srvsattr(vap);
-	} else {
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		VATTR_SET(vap, va_mode, nfstov_mode(*tl++));
-	}
+	error = nfsm_chain_get_sattr(nd, nmreq, vap);
+	nfsmerr_if(error);
 	VATTR_SET(vap, va_type, VDIR);
 
 	if (vp != NULL) {
@@ -3171,8 +3571,7 @@ nfsrv_mkdir(nfsd, slp, procp, mrq)
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-	        nameidone(&nd);
-
+	        nameidone(&ni);
 		vnode_put(dvp);
 		vnode_put(vp);
 		error = EEXIST;
@@ -3183,22 +3582,22 @@ nfsrv_mkdir(nfsd, slp, procp, mrq)
 	 * If the credentials were mapped, we should
 	 * map the same values in the attributes.
 	 */
-	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nfsd->nd_cr) != saved_uid)) {
+	if ((vap->va_uid == saved_uid) && (kauth_cred_getuid(nd->nd_cr) != saved_uid)) {
 		int ismember;
-		VATTR_SET(vap, va_uid, kauth_cred_getuid(nfsd->nd_cr));
-		if (kauth_cred_ismember_gid(nfsd->nd_cr, vap->va_gid, &ismember) || !ismember)
-			VATTR_SET(vap, va_gid, kauth_cred_getgid(nfsd->nd_cr));
+		VATTR_SET(vap, va_uid, kauth_cred_getuid(nd->nd_cr));
+		if (kauth_cred_ismember_gid(nd->nd_cr, vap->va_gid, &ismember) || !ismember)
+			VATTR_SET(vap, va_gid, kauth_cred_getgid(nd->nd_cr));
 	}
 
-	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_SUBDIRECTORY, &context, nxo, 0);
+	error = nfsrv_authorize(dvp, NULL, KAUTH_VNODE_ADD_SUBDIRECTORY, ctx, nxo, 0);
 
-  	/* construct ACL and handle inheritance */
+	/* construct ACL and handle inheritance */
 	if (!error) {
 		error = kauth_acl_inherit(dvp,
 		    NULL,
 		    &xacl,	/* isdir */
 		    1,
-		    &context);
+		    ctx);
 		
 		if (!error && xacl != NULL)
 		        VATTR_SET(vap, va_acl, xacl);
@@ -3206,38 +3605,45 @@ nfsrv_mkdir(nfsd, slp, procp, mrq)
 	VATTR_CLEAR_ACTIVE(vap, va_data_size);
 	VATTR_CLEAR_ACTIVE(vap, va_access_time);
 
-  	/* validate new-file security information */
+	/* validate new-file security information */
 	if (!error) {
-		error = vnode_authattr_new(dvp, vap, 0, &context);
+		error = vnode_authattr_new(dvp, vap, 0, ctx);
 		if (error && (VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid))) {
 			/*
-			 * Most NFS servers just ignore the UID/GID attributes, so we   
+			 * Most NFS servers just ignore the UID/GID attributes, so we
 			 * try ignoring them if that'll help the request succeed.
 			 */
 			VATTR_CLEAR_ACTIVE(vap, va_uid);
 			VATTR_CLEAR_ACTIVE(vap, va_gid);
-			error = vnode_authattr_new(dvp, vap, 0, &context);
+			error = vnode_authattr_new(dvp, vap, 0, ctx);
 		}
 	}
 
 	if (!error)
-		error = VNOP_MKDIR(dvp, &vp, &nd.ni_cnd, vap, &context);
+		error = VNOP_MKDIR(dvp, &vp, &ni.ni_cnd, vap, ctx);
+
+#if CONFIG_FSE
+	if (nfsrv_fsevents_enabled && !error)
+		add_fsevent(FSE_CREATE_DIR, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
+#endif
 
 	if (!error && !VATTR_ALL_SUPPORTED(vap))
 	        /*
 		 * If some of the requested attributes weren't handled by the VNOP,
 		 * use our fallback code.
 		 */
-		error = vnode_setattr_fallback(vp, vap, &context);
+		error = vnode_setattr_fallback(vp, vap, ctx);
 
 	if (xacl != NULL)
 		kauth_acl_free(xacl);
 	
 	if (!error) {
-		error = nfsrv_vptofh(nx, !v3, NULL, vp, &context, &nfh);
+		error = nfsrv_vptofh(nx, nd->nd_vers, NULL, vp, ctx, &nfh);
 		if (!error) {
-			nfsm_srv_vattr_init(&postat, v3);
-			error = vnode_getattr(vp, &postat, &context);
+			nfsm_srv_vattr_init(&postattr, nd->nd_vers);
+			postattrerr = vnode_getattr(vp, &postattr, ctx);
+			if (nd->nd_vers == NFS_VER2)
+				error = postattrerr;
 		}
 		vnode_put(vp);
 		vp = NULL;
@@ -3246,43 +3652,56 @@ nfsrv_mkdir(nfsd, slp, procp, mrq)
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
-
+	nameidone(&ni);
 	vnode_put(dvp);
 out:
-	nd.ni_cnd.cn_nameiop = 0;
+	ni.ni_cnd.cn_nameiop = 0;
 
 	if (dirp) {
-		nfsm_srv_vattr_init(&diraft, v3);
-		diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, nd->nd_vers);
+		dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
+		dirp = NULL;
 	}
-	nfsm_reply(NFSX_SRVFH(v3, &nfh) + NFSX_POSTOPATTR(v3) + NFSX_WCCDATA(v3));
-	if (v3) {
-		if (!error) {
-			nfsm_srvpostop_fh(&nfh);
-			nfsm_srvpostop_attr(0, &postat);
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_SRVFH(nd->nd_vers, &nfh) +
+			NFSX_POSTOPATTR(nd->nd_vers) + NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3) {
+		if (!nd->nd_repstat) {
+			nfsm_chain_add_postop_fh(error, &nmrep, nfh.nfh_fhp, nfh.nfh_len);
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, postattrerr, &postattr);
 		}
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 	} else {
-		nfsm_srvfhtom(&nfh, v3);
-		nfsm_build(fp, struct nfs_fattr *, NFSX_V2FATTR);
-		nfsm_srvfillattr(&postat, fp);
+		nfsm_chain_add_fh(error, &nmrep, NFS_VER2, nfh.nfh_fhp, nfh.nfh_len);
+		if (!error)
+			error = nfsm_chain_add_fattr(nd, &nmrep, &postattr);
 	}
-	return (0);
 nfsmout:
-	if (nd.ni_cnd.cn_nameiop) {
+	nfsm_chain_build_done(error, &nmrep);
+	if (ni.ni_cnd.cn_nameiop) {
 	        /*
 		 * nameidone has to happen before we vnode_put(dvp)
 		 * since it may need to release the fs_nodelock on the dvp
 		 */
-		nameidone(&nd);
+		nameidone(&ni);
 		vnode_put(dvp);
 		if (vp)
 			vnode_put(vp);
 	}
 	if (dirp)
 		vnode_put(dirp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -3290,61 +3709,62 @@ nfsmout:
  * nfs rmdir service
  */
 int
-nfsrv_rmdir(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_rmdir(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, len;
-	int dirfor_ret = 1, diraft_ret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mreq;
-	vnode_t vp, dvp, dirp = NULL;
-	struct vnode_attr dirfor, diraft;
+	int error, dpreattrerr, dpostattrerr;
+	uint32_t len;
+	uid_t saved_uid;
+	vnode_t vp, dvp, dirp;
+	struct vnode_attr dpreattr, dpostattr;
 	struct nfs_filehandle nfh;
-	struct nfs_export *nx;
+	struct nfs_export *nx = NULL;
 	struct nfs_export_options *nxo;
-	struct nameidata nd;
-	struct vfs_context context;
+	struct nameidata ni;
+	struct nfsm_chain *nmreq, nmrep;
 
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	dpreattrerr = dpostattrerr = ENOENT;
+	saved_uid = kauth_cred_getuid(nd->nd_cr);
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
 
-	vp = dvp = NULL;
-	nfsm_srvmtofh(&nfh);
-	nfsm_srvnamesiz(len, v3);
+	vp = dvp = dirp = NULL;
 
-	nd.ni_cnd.cn_nameiop = DELETE;
-	nd.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
-	error = nfsm_path_mbuftond(&md, &dpos, v3, FALSE, &len, &nd);
-	if (!error)
-		error = nfs_namei(nfsd, &context, &nd, &nfh, nam, FALSE, &dirp, &nx, &nxo);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_32(error, nmreq, len);
+	nfsm_name_len_check(error, nd, len);
+	nfsmerr_if(error);
+
+	ni.ni_cnd.cn_nameiop = DELETE;
+	ni.ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
+	error = nfsm_chain_get_path_namei(nmreq, len, &ni);
+	if (!error) {
+		error = nfsrv_namei(nd, ctx, &ni, &nfh, &dirp, &nx, &nxo);
+		if (nx != NULL) {
+			/* update export stats */
+			NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+			/* update active user stats */
+			nfsrv_update_user_stat(nx, nd, saved_uid, 1, 0, 0);
+		}
+	}
 	if (dirp) {
-	        if (v3) {
-			nfsm_srv_pre_vattr_init(&dirfor, v3);
-			dirfor_ret = vnode_getattr(dirp, &dirfor, &context);
+	        if (nd->nd_vers == NFS_VER3) {
+			nfsm_srv_pre_vattr_init(&dpreattr);
+			dpreattrerr = vnode_getattr(dirp, &dpreattr, ctx);
 		} else {
 			vnode_put(dirp);
 			dirp = NULL;
 		}
 	}
-	if (error) {
-		nfsm_reply(NFSX_WCCDATA(v3));
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		if (dirp)
-			vnode_put(dirp);
-		return (0);
-	}
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
+	nfsmerr_if(error);
+
+	dvp = ni.ni_dvp;
+	vp = ni.ni_vp;
 
 	if (vnode_vtype(vp) != VDIR) {
 		error = ENOTDIR;
@@ -3363,30 +3783,72 @@ nfsrv_rmdir(nfsd, slp, procp, mrq)
 	if (vnode_isvroot(vp))
 		error = EBUSY;
 	if (!error)
-		error = nfsrv_authorize(vp, dvp, KAUTH_VNODE_DELETE, &context, nxo, 0);
-	if (!error)
-		error = VNOP_RMDIR(dvp, vp, &nd.ni_cnd, &context);
+		error = nfsrv_authorize(vp, dvp, KAUTH_VNODE_DELETE, ctx, nxo, 0);
+	if (!error) {
+#if CONFIG_FSE
+		char     *path = NULL;
+		int       plen;
+		fse_info  finfo;
+		
+		if (nfsrv_fsevents_enabled && need_fsevent(FSE_DELETE, dvp)) {
+			plen = MAXPATHLEN;
+		        if ((path = get_pathbuff()) && !vn_getpath(vp, path, &plen)) {
+				get_fse_info(vp, &finfo, ctx);
+			} else if (path) {
+				release_pathbuff(path);
+				path = NULL;
+			}
+		}
+#endif /* CONFIG_FSE */
+
+		error = VNOP_RMDIR(dvp, vp, &ni.ni_cnd, ctx);
+
+#if CONFIG_FSE
+		if (path) {
+			if (!error)
+				add_fsevent(FSE_DELETE, ctx,
+					FSE_ARG_STRING, plen, path,
+					FSE_ARG_FINFO, &finfo,
+					FSE_ARG_DONE);
+		        release_pathbuff(path);
+		}
+#endif /* CONFIG_FSE */
+	}
 out:
 	/*
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
+	nameidone(&ni);
 
 	vnode_put(dvp);
 	vnode_put(vp);
 
 	if (dirp) {
-		nfsm_srv_vattr_init(&diraft, v3);
-		diraft_ret = vnode_getattr(dirp, &diraft, &context);
+		nfsm_srv_vattr_init(&dpostattr, nd->nd_vers);
+		dpostattrerr = vnode_getattr(dirp, &dpostattr, ctx);
 		vnode_put(dirp);
+		dirp = NULL;
 	}
-	nfsm_reply(NFSX_WCCDATA(v3));
-	if (v3) {
-		nfsm_srvwcc_data(dirfor_ret, &dirfor, diraft_ret, &diraft);
-		return (0);
-	}
+
+nfsmerr:
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_WCCDATA(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_wcc_data(error, nd, &nmrep,
+			dpreattrerr, &dpreattr, dpostattrerr, &dpostattr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (dirp)
+		vnode_put(dirp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -3417,152 +3879,124 @@ nfsmout:
  *      the EOF flag.  For readdirplus, the maxcount is the same, and the
  *      dircount includes all that except for the entry attributes and handles.
  */
-
 int
-nfsrv_readdir(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_readdir(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	char *bp, *be;
-	mbuf_t mp;
 	struct direntry *dp;
-	caddr_t cp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	mbuf_t mb, mb2, mreq, mp2;
-	char *cpos, *cend, *cp2, *rbuf;
+	char *cpos, *cend, *rbuf;
 	vnode_t vp;
-	struct vnode_attr at;
+	struct vnode_attr attr;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	uio_t auio;
+	uio_t auio = NULL;
 	char uio_buf[ UIO_SIZEOF(1) ];
-	int len, nlen, rem, xfer, tsiz, i, error = 0, getret = 1;
-	int siz, count, fullsiz, eofflag, nentries = 0;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
+	int len, nlen, rem, xfer, error, attrerr;
+	int siz, count, fullsiz, eofflag, nentries;
 	u_quad_t off, toff, verf;
-	nfsuint64 tquad;
 	int vnopflag;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	attrerr = ENOENT;
+	nentries = 0;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	rbuf = NULL;
+	vp = NULL;
 
 	vnopflag = VNODE_READDIR_EXTENDED | VNODE_READDIR_REQSEEKOFF;
 
-	nfsm_srvmtofh(&nfh);
-	if (v3) {
-		nfsm_dissect(tl, u_long *, 5 * NFSX_UNSIGNED);
-		fxdr_hyper(tl, &toff);
-		tl += 2;
-		fxdr_hyper(tl, &verf);
-		tl += 2;
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_get_64(error, nmreq, toff);
+		nfsm_chain_get_64(error, nmreq, verf);
 	} else {
-		nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-		toff = fxdr_unsigned(u_quad_t, *tl++);
+		nfsm_chain_get_32(error, nmreq, toff);
 	}
+	nfsm_chain_get_32(error, nmreq, count);
+	nfsmerr_if(error);
+
 	off = toff;
-	count = fxdr_unsigned(int, *tl);
 	siz = ((count + DIRBLKSIZ - 1) & ~(DIRBLKSIZ - 1));
-	xfer = NFS_SRVMAXDATA(nfsd);
+	xfer = NFS_SRVMAXDATA(nd);
 	if (siz > xfer)
 		siz = xfer;
 	fullsiz = siz;
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
-	if (!v3 || (nxo->nxo_flags & NX_32BITCLIENTS))
+
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	if ((nd->nd_vers == NFS_VER2) || (nxo->nxo_flags & NX_32BITCLIENTS))
 		vnopflag |= VNODE_READDIR_SEEKOFF32;
-	if (v3) {
-		nfsm_srv_vattr_init(&at, v3);
-		error = getret = vnode_getattr(vp, &at, &context);
-		if (!error && toff && verf && verf != at.va_filerev)
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_vattr_init(&attr, NFS_VER3);
+		error = attrerr = vnode_getattr(vp, &attr, ctx);
+		if (!error && toff && verf && (verf != attr.va_filerev))
 			error = NFSERR_BAD_COOKIE;
 	}
 	if (!error)
-		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LIST_DIRECTORY, &context, nxo, 0);
-	if (error) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
+		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LIST_DIRECTORY, ctx, nxo, 0);
+	nfsmerr_if(error);
+
 	MALLOC(rbuf, caddr_t, siz, M_TEMP, M_WAITOK);
-	if (!rbuf) {
-		error = ENOMEM;
-		vnode_put(vp);
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
+	if (rbuf)
+		auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
 				    &uio_buf[0], sizeof(uio_buf));
-	if (!auio) {
+	if (!rbuf || !auio) {
 		error = ENOMEM;
-		FREE(rbuf, M_TEMP);
-		vnode_put(vp);
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
+		goto nfsmerr;
 	}
 again:
 	uio_reset(auio, off, UIO_SYSSPACE, UIO_READ);
 	uio_addiov(auio, CAST_USER_ADDR_T(rbuf), fullsiz);
-
 	eofflag = 0;
-	error = VNOP_READDIR(vp, auio, vnopflag, &eofflag, &nentries, &context);
+	error = VNOP_READDIR(vp, auio, vnopflag, &eofflag, &nentries, ctx);
 	off = uio_offset(auio);
 
-	if (v3) {
-		nfsm_srv_vattr_init(&at, v3);
-		getret = vnode_getattr(vp, &at, &context);
-		if (!error)
-			error = getret;
-	}
-	if (error) {
-		vnode_put(vp);
-		FREE(rbuf, M_TEMP);
-		nfsm_reply(NFSX_POSTOPATTR(v3));
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_vattr_init(&attr, NFS_VER3);
+		attrerr = vnode_getattr(vp, &attr, ctx);
 	}
+	nfsmerr_if(error);
+
 	if (uio_resid(auio) != 0) {
 		// LP64todo - fix this
 		siz -= uio_resid(auio);
 
-		/*
-		 * If nothing read, return eof
-		 * rpc reply
-		 */
+		/* If nothing read, return empty reply with eof set */
 		if (siz == 0) {
 			vnode_put(vp);
-			nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_COOKIEVERF(v3) +
-				2 * NFSX_UNSIGNED);
-			if (v3) {
-				nfsm_srvpostop_attr(getret, &at);
-				nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
-				txdr_hyper(&at.va_filerev, tl);
-				tl += 2;
-			} else
-				nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = nfs_false;
-			*tl = nfs_true;
+			vp = NULL;
 			FREE(rbuf, M_TEMP);
-			return (0);
+			/* assemble reply */
+			nd->nd_repstat = error;
+			error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers) +
+					NFSX_COOKIEVERF(nd->nd_vers) + 2 * NFSX_UNSIGNED);
+			nfsmout_if(error);
+			*mrepp = nmrep.nmc_mhead;
+			nfsmout_on_status(nd, error);
+			if (nd->nd_vers == NFS_VER3) {
+				nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+				nfsm_chain_add_64(error, &nmrep, attr.va_filerev);
+			}
+			nfsm_chain_add_32(error, &nmrep, FALSE);
+			nfsm_chain_add_32(error, &nmrep, TRUE);
+			nfsm_chain_build_done(error, &nmrep);
+			return (error);
 		}
 	}
 
@@ -3573,182 +4007,139 @@ again:
 	cpos = rbuf;
 	cend = rbuf + siz;
 	dp = (struct direntry *)cpos;
-	while (dp->d_fileno == 0 && cpos < cend && nentries > 0) {
+	while ((dp->d_fileno == 0) && (cpos < cend) && (nentries > 0)) {
 		cpos += dp->d_reclen;
 		dp = (struct direntry *)cpos;
 		nentries--;
 	}
-	if (cpos >= cend || nentries == 0) {
+	if ((cpos >= cend) || (nentries == 0)) {
 		toff = off;
 		siz = fullsiz;
 		goto again;
 	}
 
 	vnode_put(vp);
-	nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_COOKIEVERF(v3) + siz);
-	if (v3) {
-		len = NFSX_V3POSTOPATTR + NFSX_V3COOKIEVERF + 2 * NFSX_UNSIGNED;
-		nfsm_srvpostop_attr(getret, &at);
-		nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-		txdr_hyper(&at.va_filerev, tl);
-	} else
-		len = 2 * NFSX_UNSIGNED;
-	mp = mp2 = mb;
-	bp = bpos;
-	be = bp + mbuf_trailingspace(mp);
+	vp = NULL;
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers) +
+			NFSX_COOKIEVERF(nd->nd_vers) + siz);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nmrep.nmc_flags |= NFSM_CHAIN_FLAG_ADD_CLUSTERS;
+
+	len = 2 * NFSX_UNSIGNED;
+	if (nd->nd_vers == NFS_VER3) {
+		len += NFSX_V3POSTOPATTR + NFSX_V3COOKIEVERF;
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+		nfsm_chain_add_64(error, &nmrep, attr.va_filerev);
+		nfsmerr_if(error);
+	}
 
 	/* Loop through the records and build reply */
-	while (cpos < cend && nentries > 0) {
+	while ((cpos < cend) && (nentries > 0)) {
 		if (dp->d_fileno != 0) {
 			nlen = dp->d_namlen;
-			if (!v3 && (nlen > NFS_MAXNAMLEN))
+			if ((nd->nd_vers == NFS_VER2) && (nlen > NFS_MAXNAMLEN))
 				nlen = NFS_MAXNAMLEN;
 			rem = nfsm_rndup(nlen)-nlen;
 			len += (4 * NFSX_UNSIGNED + nlen + rem);
-			if (v3)
+			if (nd->nd_vers == NFS_VER3)
 				len += 2 * NFSX_UNSIGNED;
 			if (len > count) {
 				eofflag = 0;
 				break;
 			}
-			/*
-			 * Build the directory record xdr from
-			 * the direntry entry.
-			 */
-			nfsm_clget;
-			*tl = nfs_true;
-			bp += NFSX_UNSIGNED;
-			nfsm_clget;
-			if (v3) {
-				txdr_hyper(&dp->d_fileno, &tquad);
-				*tl = tquad.nfsuquad[0];
-				bp += NFSX_UNSIGNED;
-				nfsm_clget;
-				*tl = tquad.nfsuquad[1];
-				bp += NFSX_UNSIGNED;
+			/* Build the directory record xdr from the direntry. */
+			nfsm_chain_add_32(error, &nmrep, TRUE);
+			if (nd->nd_vers == NFS_VER3) {
+				nfsm_chain_add_64(error, &nmrep, dp->d_fileno);
 			} else {
-				*tl = txdr_unsigned(dp->d_fileno);
-				bp += NFSX_UNSIGNED;
-			}
-			nfsm_clget;
-			*tl = txdr_unsigned(nlen);
-			bp += NFSX_UNSIGNED;
-
-			/* And loop around copying the name */
-			xfer = nlen;
-			cp = dp->d_name;
-			while (xfer > 0) {
-				nfsm_clget;
-				if ((bp+xfer) > be)
-					tsiz = be-bp;
-				else
-					tsiz = xfer;
-				bcopy(cp, bp, tsiz);
-				bp += tsiz;
-				xfer -= tsiz;
-				if (xfer > 0)
-					cp += tsiz;
+				nfsm_chain_add_32(error, &nmrep, dp->d_fileno);
 			}
-			/* And null pad to a long boundary */
-			for (i = 0; i < rem; i++)
-				*bp++ = '\0';
-
-			/* Finish off the record with the cookie */
-			nfsm_clget;
-			if (v3) {
+			nfsm_chain_add_string(error, &nmrep, dp->d_name, nlen);
+			if (nd->nd_vers == NFS_VER3) {
 				if (vnopflag & VNODE_READDIR_SEEKOFF32)
 					dp->d_seekoff &= 0x00000000ffffffffULL;
-				txdr_hyper(&dp->d_seekoff, &tquad);
-				*tl = tquad.nfsuquad[0];
-				bp += NFSX_UNSIGNED;
-				nfsm_clget;
-				*tl = tquad.nfsuquad[1];
-				bp += NFSX_UNSIGNED;
+				nfsm_chain_add_64(error, &nmrep, dp->d_seekoff);
 			} else {
-				*tl = txdr_unsigned(dp->d_seekoff);
-				bp += NFSX_UNSIGNED;
+				nfsm_chain_add_32(error, &nmrep, dp->d_seekoff);
 			}
+			nfsmerr_if(error);
 		}
 		cpos += dp->d_reclen;
 		dp = (struct direntry *)cpos;
 		nentries--;
 	}
-	nfsm_clget;
-	*tl = nfs_false;
-	bp += NFSX_UNSIGNED;
-	nfsm_clget;
-	if (eofflag)
-		*tl = nfs_true;
-	else
-		*tl = nfs_false;
-	bp += NFSX_UNSIGNED;
-	if (mp != mb) {
-		if (bp < be)
-			mbuf_setlen(mp, bp - (char*)mbuf_data(mp));
-	} else
-		mbuf_setlen(mp, mbuf_len(mp) + (bp - bpos));
+	nfsm_chain_add_32(error, &nmrep, FALSE);
+	nfsm_chain_add_32(error, &nmrep, eofflag ? TRUE : FALSE);
 	FREE(rbuf, M_TEMP);
+	goto nfsmout;
+nfsmerr:
+	if (rbuf)
+		FREE(rbuf, M_TEMP);
+	if (vp)
+		vnode_put(vp);
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
-struct flrep {
-	nfsuint64	fl_off;
-	u_long		fl_postopok;
-	u_long		fl_fattr[NFSX_V3FATTR / sizeof (u_long)];
-	u_long		fl_fhok;
-	u_long		fl_fhsize;
-	u_long		fl_nfh[NFSX_V3FHMAX / sizeof (u_long)];
-};
-
 int
-nfsrv_readdirplus(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_readdirplus(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	char *bp, *be;
-	mbuf_t mp;
 	struct direntry *dp;
-	caddr_t cp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	mbuf_t mb, mb2, mreq, mp2;
-	char *cpos, *cend, *cp2, *rbuf;
+	char *cpos, *cend, *rbuf;
 	vnode_t vp, nvp;
-	struct flrep fl;
-	struct nfs_filehandle dnfh, *nfhp = (struct nfs_filehandle *)&fl.fl_fhsize;
-	u_long fhsize;
+	struct nfs_filehandle dnfh, nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	uio_t auio;
+	uio_t auio = NULL;
 	char uio_buf[ UIO_SIZEOF(1) ];
-	struct vnode_attr va, at, *vap = &va;
-	struct nfs_fattr *fp;
-	int len, nlen, rem, xfer, tsiz, i, error = 0, getret = 1;
-	int siz, dircount, maxcount, fullsiz, eofflag, dirlen, nentries = 0, isdotdot;
+	struct vnode_attr attr, va, *vap = &va;
+	int len, nlen, rem, xfer, error, attrerr, gotfh, gotattr;
+	int siz, dircount, maxcount, fullsiz, eofflag, dirlen, nentries, isdotdot;
 	u_quad_t off, toff, verf;
-	nfsuint64 tquad;
 	int vnopflag;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
+
+	error = 0;
+	attrerr = ENOENT;
+	nentries = 0;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	rbuf = NULL;
+	vp = NULL;
+	dircount = maxcount = 0;
 
 	vnopflag = VNODE_READDIR_EXTENDED | VNODE_READDIR_REQSEEKOFF;
-	vp = NULL;
-	nfsm_srvmtofh(&dnfh);
-	nfsm_dissect(tl, u_long *, 6 * NFSX_UNSIGNED);
-	fxdr_hyper(tl, &toff);
-	tl += 2;
-	fxdr_hyper(tl, &verf);
-	tl += 2;
-	dircount = fxdr_unsigned(int, *tl++);
-	maxcount = fxdr_unsigned(int, *tl);
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, dnfh.nfh_fhp, dnfh.nfh_len);
+	nfsm_chain_get_64(error, nmreq, toff);
+	nfsm_chain_get_64(error, nmreq, verf);
+	nfsm_chain_get_32(error, nmreq, dircount);
+	nfsm_chain_get_32(error, nmreq, maxcount);
+	nfsmerr_if(error);
+
 	off = toff;
-	xfer = NFS_SRVMAXDATA(nfsd);
+	xfer = NFS_SRVMAXDATA(nd);
 	dircount = ((dircount + DIRBLKSIZ - 1) & ~(DIRBLKSIZ - 1));
 	if (dircount > xfer)
 		dircount = xfer;
@@ -3757,94 +4148,70 @@ nfsrv_readdirplus(nfsd, slp, procp, mrq)
 	if (maxcount > xfer)
 		maxcount = xfer;
 
-	if ((error = nfsrv_fhtovp(&dnfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = nfsrv_fhtovp(&dnfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
 	if (nxo->nxo_flags & NX_32BITCLIENTS)
 		vnopflag |= VNODE_READDIR_SEEKOFF32;
-	nfsm_srv_vattr_init(&at, 1);
-	error = getret = vnode_getattr(vp, &at, &context);
-	if (!error && toff && verf && verf != at.va_filerev)
+
+	nfsm_srv_vattr_init(&attr, NFS_VER3);
+	error = attrerr = vnode_getattr(vp, &attr, ctx);
+	if (!error && toff && verf && (verf != attr.va_filerev))
 		error = NFSERR_BAD_COOKIE;
 	if (!error)
-		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LIST_DIRECTORY, &context, nxo, 0);
-	if (error) {
-		vnode_put(vp);
-		vp = NULL;
-		nfsm_reply(NFSX_V3POSTOPATTR);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
+		error = nfsrv_authorize(vp, NULL, KAUTH_VNODE_LIST_DIRECTORY, ctx, nxo, 0);
+	nfsmerr_if(error);
+
 	MALLOC(rbuf, caddr_t, siz, M_TEMP, M_WAITOK);
-	if (!rbuf) {
-		error = ENOMEM;
-		vnode_put(vp);
-		vp = NULL;
-		nfsm_reply(NFSX_V3POSTOPATTR);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
+	if (rbuf)
+		auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
 				    &uio_buf[0], sizeof(uio_buf));
-	if (!auio) {
+	if (!rbuf || !auio) {
 		error = ENOMEM;
-		FREE(rbuf, M_TEMP);
-		vnode_put(vp);
-		vp = NULL;
-		nfsm_reply(NFSX_V3POSTOPATTR);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
+		goto nfsmerr;
 	}
+
 again:
 	uio_reset(auio, off, UIO_SYSSPACE, UIO_READ);
 	uio_addiov(auio, CAST_USER_ADDR_T(rbuf), fullsiz);
 	eofflag = 0;
-	error = VNOP_READDIR(vp, auio, vnopflag, &eofflag, &nentries, &context);
+	error = VNOP_READDIR(vp, auio, vnopflag, &eofflag, &nentries, ctx);
 	off = uio_offset(auio);
-	nfsm_srv_vattr_init(&at, 1);
-	getret = vnode_getattr(vp, &at, &context);
+	nfsm_srv_vattr_init(&attr, NFS_VER3);
+	attrerr = vnode_getattr(vp, &attr, ctx);
+	nfsmerr_if(error);
 
-	if (!error)
-		error = getret;
-	if (error) {
-		vnode_put(vp);
-		vp = NULL;
-		FREE(rbuf, M_TEMP);
-		nfsm_reply(NFSX_V3POSTOPATTR);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
 	if (uio_resid(auio) != 0) {
 		// LP64todo - fix this
 		siz -= uio_resid(auio);
 
-		/*
-		 * If nothing read, return eof
-		 * rpc reply
-		 */
+		/* If nothing read, return empty reply with eof set */
 		if (siz == 0) {
 			vnode_put(vp);
 			vp = NULL;
-			nfsm_reply(NFSX_V3POSTOPATTR + NFSX_V3COOKIEVERF +
-				2 * NFSX_UNSIGNED);
-			nfsm_srvpostop_attr(getret, &at);
-			nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
-			txdr_hyper(&at.va_filerev, tl);
-			tl += 2;
-			*tl++ = nfs_false;
-			*tl = nfs_true;
 			FREE(rbuf, M_TEMP);
-			return (0);
+			/* assemble reply */
+			nd->nd_repstat = error;
+			error = nfsrv_rephead(nd, slp, &nmrep, NFSX_V3POSTOPATTR +
+					NFSX_V3COOKIEVERF + 2 * NFSX_UNSIGNED);
+			nfsmout_if(error);
+			*mrepp = nmrep.nmc_mhead;
+			nfsmout_on_status(nd, error);
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+			nfsm_chain_add_64(error, &nmrep, attr.va_filerev);
+			nfsm_chain_add_32(error, &nmrep, FALSE);
+			nfsm_chain_add_32(error, &nmrep, TRUE);
+			nfsm_chain_build_done(error, &nmrep);
+			return (error);
 		}
 	}
 
@@ -3855,12 +4222,12 @@ again:
 	cpos = rbuf;
 	cend = rbuf + siz;
 	dp = (struct direntry *)cpos;
-	while (dp->d_fileno == 0 && cpos < cend && nentries > 0) {
+	while ((dp->d_fileno == 0) && (cpos < cend) && (nentries > 0)) {
 		cpos += dp->d_reclen;
 		dp = (struct direntry *)cpos;
 		nentries--;
 	}
-	if (cpos >= cend || nentries == 0) {
+	if ((cpos >= cend) || (nentries == 0)) {
 		toff = off;
 		siz = fullsiz;
 		goto again;
@@ -3870,54 +4237,47 @@ again:
 	 * Probe one of the directory entries to see if the filesystem
 	 * supports VGET.
 	 */
-	if ((error = VFS_VGET(vnode_mount(vp), (ino64_t)dp->d_fileno, &nvp, &context))) {
+	if ((error = VFS_VGET(vnode_mount(vp), (ino64_t)dp->d_fileno, &nvp, ctx))) {
 		if (error == ENOTSUP) /* let others get passed back */
-			error = NFSERR_NOTSUPP; 
-		vnode_put(vp);
-		vp = NULL;
-		FREE(rbuf, M_TEMP);
-		nfsm_reply(NFSX_V3POSTOPATTR);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
+			error = NFSERR_NOTSUPP;
+		goto nfsmerr;
 	}
 	vnode_put(nvp);
-	    
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, maxcount);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nmrep.nmc_flags |= NFSM_CHAIN_FLAG_ADD_CLUSTERS;
+
 	dirlen = len = NFSX_V3POSTOPATTR + NFSX_V3COOKIEVERF + 2 * NFSX_UNSIGNED;
-	nfsm_reply(maxcount);
-	nfsm_srvpostop_attr(getret, &at);
-	nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-	txdr_hyper(&at.va_filerev, tl);
-	mp = mp2 = mb;
-	bp = bpos;
-	be = bp + mbuf_trailingspace(mp);
+	nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+	nfsm_chain_add_64(error, &nmrep, attr.va_filerev);
+	nfsmerr_if(error);
 
 	/* Loop through the records and build reply */
-	while (cpos < cend && nentries > 0) {
+	while ((cpos < cend) && (nentries > 0)) {
 		if (dp->d_fileno != 0) {
 			nlen = dp->d_namlen;
 			rem = nfsm_rndup(nlen)-nlen;
+			gotfh = gotattr = 1;
 
-			/* 
-			 * Got to get the vnode for lookup per entry.
-			 */
-			if (VFS_VGET(vnode_mount(vp), (ino64_t)dp->d_fileno, &nvp, &context))
-				goto invalid;
-			isdotdot = ((dp->d_namlen == 2) &&
-				    (dp->d_name[0] == '.') && (dp->d_name[1] == '.'));
-			if (nfsrv_vptofh(nx, 0, (isdotdot ? &dnfh : NULL), nvp, &context, nfhp)) {
-				// XXX file handle is optional, so we should be able to
-				// XXX return this entry without the file handle
-				vnode_put(nvp);
-				goto invalid;
-			}
-			nfsm_srv_vattr_init(vap, 1);
-			if (vnode_getattr(nvp, vap, &context)) {
-				// XXX attributes are optional, so we should be able to
-				// XXX return this entry without the attributes
+			/* Got to get the vnode for lookup per entry. */
+			if (VFS_VGET(vnode_mount(vp), (ino64_t)dp->d_fileno, &nvp, ctx)) {
+				/* Can't get the vnode... so no fh or attrs */
+				gotfh = gotattr = 0;
+			} else {
+				isdotdot = ((dp->d_namlen == 2) &&
+					    (dp->d_name[0] == '.') && (dp->d_name[1] == '.'));
+				if (nfsrv_vptofh(nx, 0, (isdotdot ? &dnfh : NULL), nvp, ctx, &nfh))
+					gotfh = 0;
+				nfsm_srv_vattr_init(vap, NFS_VER3);
+				if (vnode_getattr(nvp, vap, ctx))
+					gotattr = 0;
 				vnode_put(nvp);
-				goto invalid;
 			}
-			vnode_put(nvp);
 
 			/*
 			 * If either the dircount or maxcount will be
@@ -3925,106 +4285,58 @@ again:
 			 * are calculated conservatively, including all
 			 * XDR overheads.
 			 */
-			len += (8 * NFSX_UNSIGNED + nlen + rem + nfsm_rndup(nfhp->nfh_len) +
-				NFSX_V3POSTOPATTR);
-			dirlen += (6 * NFSX_UNSIGNED + nlen + rem);
+			len += 8 * NFSX_UNSIGNED + nlen + rem;
+			if (gotattr)
+				len += NFSX_V3FATTR;
+			if (gotfh)
+				len += NFSX_UNSIGNED + nfsm_rndup(nfh.nfh_len);
+			dirlen += 6 * NFSX_UNSIGNED + nlen + rem;
 			if ((len > maxcount) || (dirlen > dircount)) {
 				eofflag = 0;
 				break;
 			}
 
-			/*
-			 * Build the directory record xdr from
-			 * the direntry entry.
-			 */
-			fp = (struct nfs_fattr *)&fl.fl_fattr;
-			nfsm_srvfillattr(vap, fp);
-			fhsize = nfhp->nfh_len;
-			fl.fl_fhsize = txdr_unsigned(fhsize);
-			fl.fl_fhok = nfs_true;
-			fl.fl_postopok = nfs_true;
+			/* Build the directory record xdr from the direntry. */
+			nfsm_chain_add_32(error, &nmrep, TRUE);
+			nfsm_chain_add_64(error, &nmrep, dp->d_fileno);
+			nfsm_chain_add_string(error, &nmrep, dp->d_name, nlen);
 			if (vnopflag & VNODE_READDIR_SEEKOFF32)
 				dp->d_seekoff &= 0x00000000ffffffffULL;
-			txdr_hyper(&dp->d_seekoff, &fl.fl_off);
-
-			nfsm_clget;
-			*tl = nfs_true;
-			bp += NFSX_UNSIGNED;
-
-			nfsm_clget;
-			txdr_hyper(&dp->d_fileno, &tquad);
-			*tl = tquad.nfsuquad[0];
-			bp += NFSX_UNSIGNED;
-			nfsm_clget;
-			*tl = tquad.nfsuquad[1];
-			bp += NFSX_UNSIGNED;
-
-			nfsm_clget;
-			*tl = txdr_unsigned(nlen);
-			bp += NFSX_UNSIGNED;
-
-			/* And loop around copying the name */
-			xfer = nlen;
-			cp = dp->d_name;
-			while (xfer > 0) {
-				nfsm_clget;
-				if ((bp + xfer) > be)
-					tsiz = be - bp;
-				else
-					tsiz = xfer;
-				bcopy(cp, bp, tsiz);
-				bp += tsiz;
-				xfer -= tsiz;
-				if (xfer > 0)
-					cp += tsiz;
-			}
-			/* And null pad to a long boundary */
-			for (i = 0; i < rem; i++)
-				*bp++ = '\0';
-	
-			/*
-			 * Now copy the flrep structure out.
-			 */
-			xfer = sizeof(struct flrep) - sizeof(fl.fl_nfh) + fhsize;
-			cp = (caddr_t)&fl;
-			while (xfer > 0) {
-				nfsm_clget;
-				if ((bp + xfer) > be)
-					tsiz = be - bp;
-				else
-					tsiz = xfer;
-				bcopy(cp, bp, tsiz);
-				bp += tsiz;
-				xfer -= tsiz;
-				if (xfer > 0)
-					cp += tsiz;
-			}
+			nfsm_chain_add_64(error, &nmrep, dp->d_seekoff);
+			nfsm_chain_add_postop_attr(error, nd, &nmrep, (gotattr ? 0 : ENOENT), vap);
+			if (gotfh)
+				nfsm_chain_add_postop_fh(error, &nmrep, nfh.nfh_fhp, nfh.nfh_len);
+			else
+				nfsm_chain_add_32(error, &nmrep, FALSE);
+			nfsmerr_if(error);
 		}
-invalid:
 		cpos += dp->d_reclen;
 		dp = (struct direntry *)cpos;
 		nentries--;
 	}
 	vnode_put(vp);
 	vp = NULL;
-	nfsm_clget;
-	*tl = nfs_false;
-	bp += NFSX_UNSIGNED;
-	nfsm_clget;
-	if (eofflag)
-		*tl = nfs_true;
-	else
-		*tl = nfs_false;
-	bp += NFSX_UNSIGNED;
-	if (mp != mb) {
-		if (bp < be)
-			mbuf_setlen(mp, bp - (char*)mbuf_data(mp));
-	} else
-		mbuf_setlen(mp, mbuf_len(mp) + (bp - bpos));
+	nfsm_chain_add_32(error, &nmrep, FALSE);
+	nfsm_chain_add_32(error, &nmrep, eofflag ? TRUE : FALSE);
 	FREE(rbuf, M_TEMP);
+	goto nfsmout;
+nfsmerr:
+	if (rbuf)
+		FREE(rbuf, M_TEMP);
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_V3POSTOPATTR);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
 	if (vp)
 		vnode_put(vp);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -4032,68 +4344,79 @@ nfsmout:
  * nfs commit service
  */
 int
-nfsrv_commit(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_commit(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	struct vnode_attr bfor, aft;
 	vnode_t vp;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, for_ret = 1, aft_ret = 1, count;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
+	int error, preattrerr, postattrerr, count;
+	struct vnode_attr preattr, postattr;
 	u_quad_t off;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
 
-	nfsm_srvmtofh(&nfh);
-	nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
+	error = 0;
+	preattrerr = postattrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
 
 	/*
 	 * XXX At this time VNOP_FSYNC() does not accept offset and byte
-	 * count parameters, so these arguments are useless (someday maybe).
+	 * count parameters, so those arguments are useless (someday maybe).
 	 */
-	fxdr_hyper(tl, &off);
-	tl += 2;
-	count = fxdr_unsigned(int, *tl);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(for_ret, &bfor, aft_ret, &aft);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
+
+	nfsm_chain_get_fh_ptr(error, nmreq, NFS_VER3, nfh.nfh_fhp, nfh.nfh_len);
+	nfsm_chain_get_64(error, nmreq, off);
+	nfsm_chain_get_32(error, nmreq, count);
+	nfsmerr_if(error);
+
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	nfsm_srv_pre_vattr_init(&preattr);
+	preattrerr = vnode_getattr(vp, &preattr, ctx);
+
+	error = VNOP_FSYNC(vp, MNT_WAIT, ctx);
+
+	nfsm_srv_vattr_init(&postattr, 1);
+	postattrerr = vnode_getattr(vp, &postattr, ctx);
+
+nfsmerr:
+	if (vp)
 		vnode_put(vp);
-		nfsm_reply(2 * NFSX_UNSIGNED);
-		nfsm_srvwcc_data(for_ret, &bfor, aft_ret, &aft);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
 
-	nfsm_srv_pre_vattr_init(&bfor, 1);
-	for_ret = vnode_getattr(vp, &bfor, &context);
-	error = VNOP_FSYNC(vp, MNT_WAIT, &context);
-	nfsm_srv_vattr_init(&aft, 1);
-	aft_ret = vnode_getattr(vp, &aft, &context);
-	vnode_put(vp);
-	nfsm_reply(NFSX_V3WCCDATA + NFSX_V3WRITEVERF);
-	nfsm_srvwcc_data(for_ret, &bfor, aft_ret, &aft);
-	if (!error) {
-		nfsm_build(tl, u_long *, NFSX_V3WRITEVERF);
-		*tl++ = txdr_unsigned(boottime_sec());
-		*tl = txdr_unsigned(0);
-	} else
-		return (0);
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_V3WCCDATA + NFSX_V3WRITEVERF);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nfsm_chain_add_wcc_data(error, nd, &nmrep,
+		preattrerr, &preattr, postattrerr, &postattr);
+	if (!nd->nd_repstat) {
+		nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_sec);
+		nfsm_chain_add_32(error, &nmrep, nx->nx_exptime.tv_usec);
+	}
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -4101,83 +4424,91 @@ nfsmout:
  * nfs statfs service
  */
 int
-nfsrv_statfs(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_statfs(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
 	struct vfs_attr va;
-	struct nfs_statfs *sfp;
-	u_long *tl;
-	long t1;
-	caddr_t bpos;
-	int error = 0, getret = 1;
-	int v3 = (nfsd->nd_flag & ND_NFSV3);
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
+	int error, attrerr;
 	vnode_t vp;
-	struct vnode_attr at;
+	struct vnode_attr attr;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	u_quad_t tval;
 	off_t blksize;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
 
-	nfsm_srvmtofh(&nfh);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	attrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
+	blksize = 512;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
 
 	VFSATTR_INIT(&va);
 	VFSATTR_WANTED(&va, f_blocks);
 	VFSATTR_WANTED(&va, f_bavail);
 	VFSATTR_WANTED(&va, f_files);
 	VFSATTR_WANTED(&va, f_ffree);
-	error = vfs_getattr(vnode_mount(vp), &va, &context);
+	error = vfs_getattr(vnode_mount(vp), &va, ctx);
 	blksize = vnode_mount(vp)->mnt_vfsstat.f_bsize;
-	nfsm_srv_vattr_init(&at, v3);
-	getret = vnode_getattr(vp, &at, &context);
-	vnode_put(vp);
-	nfsm_reply(NFSX_POSTOPATTR(v3) + NFSX_STATFS(v3));
-	if (v3)
-		nfsm_srvpostop_attr(getret, &at);
-	if (error)
-		return (0);
-	nfsm_build(sfp, struct nfs_statfs *, NFSX_STATFS(v3));
-	if (v3) {
-		tval = (u_quad_t)(va.f_blocks * blksize);
-		txdr_hyper(&tval, &sfp->sf_tbytes);
-		tval = (u_quad_t)(va.f_bfree * blksize);
-		txdr_hyper(&tval, &sfp->sf_fbytes);
-		tval = (u_quad_t)(va.f_bavail * blksize);
-		txdr_hyper(&tval, &sfp->sf_abytes);
-		txdr_hyper(&va.f_files, &sfp->sf_tfiles);
-		txdr_hyper(&va.f_ffree, &sfp->sf_ffiles);
-		txdr_hyper(&va.f_ffree, &sfp->sf_afiles);
-		sfp->sf_invarsec = 0;
+
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_srv_vattr_init(&attr, nd->nd_vers);
+		attrerr = vnode_getattr(vp, &attr, ctx);
+	}
+
+nfsmerr:
+	if (vp)
+		vnode_put(vp);
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_POSTOPATTR(nd->nd_vers) + NFSX_STATFS(nd->nd_vers));
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	if (nd->nd_vers == NFS_VER3)
+		nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+	nfsmout_if(nd->nd_repstat);
+
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_64(error, &nmrep, va.f_blocks * blksize);
+		nfsm_chain_add_64(error, &nmrep, va.f_bfree * blksize);
+		nfsm_chain_add_64(error, &nmrep, va.f_bavail * blksize);
+		nfsm_chain_add_64(error, &nmrep, va.f_files);
+		nfsm_chain_add_64(error, &nmrep, va.f_ffree);
+		nfsm_chain_add_64(error, &nmrep, va.f_ffree);
+		nfsm_chain_add_32(error, &nmrep, 0); /* invarsec */
 	} else {
-		sfp->sf_tsize = txdr_unsigned(NFS_V2MAXDATA);
-		sfp->sf_bsize = txdr_unsigned((unsigned)blksize);
-		sfp->sf_blocks = txdr_unsigned((unsigned)va.f_blocks);
-		sfp->sf_bfree = txdr_unsigned((unsigned)va.f_bfree);
-		sfp->sf_bavail = txdr_unsigned((unsigned)va.f_bavail);
+		nfsm_chain_add_32(error, &nmrep, NFS_V2MAXDATA);
+		nfsm_chain_add_32(error, &nmrep, blksize);
+		nfsm_chain_add_32(error, &nmrep, va.f_blocks);
+		nfsm_chain_add_32(error, &nmrep, va.f_bfree);
+		nfsm_chain_add_32(error, &nmrep, va.f_bavail);
 	}
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -4185,54 +4516,58 @@ nfsmout:
  * nfs fsinfo service
  */
 int
-nfsrv_fsinfo(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_fsinfo(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	u_long *tl;
-	struct nfsv3_fsinfo *sip;
-	long t1;
-	caddr_t bpos;
-	int error = 0, getret = 1, prefsize, maxsize;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
+	int error, attrerr, prefsize, maxsize;
 	vnode_t vp;
-	struct vnode_attr at;
+	struct vnode_attr attr;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
 
-	nfsm_srvmtofh(&nfh);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
+	error = 0;
+	attrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
+
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	nfsm_srv_vattr_init(&attr, NFS_VER3);
+	attrerr = vnode_getattr(vp, &attr, ctx);
+
+nfsmerr:
+	if (vp)
 		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
 
-	nfsm_srv_vattr_init(&at, 1);
-	getret = vnode_getattr(vp, &at, &context);
-	vnode_put(vp);
-	nfsm_reply(NFSX_V3POSTOPATTR + NFSX_V3FSINFO);
-	nfsm_srvpostop_attr(getret, &at);
-	nfsm_build(sip, struct nfsv3_fsinfo *, NFSX_V3FSINFO);
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_V3POSTOPATTR + NFSX_V3FSINFO);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+	nfsmout_if(nd->nd_repstat);
 
 	/*
-	 * XXX
-	 * There should be file system VFS OP(s) to get this information.
+	 * XXX There should be file system VFS OP(s) to get this information.
 	 * For now, assume our usual NFS defaults.
 	 */
 	if (slp->ns_sotype == SOCK_DGRAM) {
@@ -4240,21 +4575,28 @@ nfsrv_fsinfo(nfsd, slp, procp, mrq)
 		prefsize = NFS_PREFDGRAMDATA;
 	} else
 		maxsize = prefsize = NFS_MAXDATA;
-	sip->fs_rtmax = txdr_unsigned(maxsize);
-	sip->fs_rtpref = txdr_unsigned(prefsize);
-	sip->fs_rtmult = txdr_unsigned(NFS_FABLKSIZE);
-	sip->fs_wtmax = txdr_unsigned(maxsize);
-	sip->fs_wtpref = txdr_unsigned(prefsize);
-	sip->fs_wtmult = txdr_unsigned(NFS_FABLKSIZE);
-	sip->fs_dtpref = txdr_unsigned(prefsize);
-	sip->fs_maxfilesize.nfsuquad[0] = 0xffffffff;
-	sip->fs_maxfilesize.nfsuquad[1] = 0xffffffff;
-	sip->fs_timedelta.nfsv3_sec = 0;
-	sip->fs_timedelta.nfsv3_nsec = txdr_unsigned(1);
-	sip->fs_properties = txdr_unsigned(NFSV3FSINFO_LINK |
-		NFSV3FSINFO_SYMLINK | NFSV3FSINFO_HOMOGENEOUS |
-		NFSV3FSINFO_CANSETTIME);
+
+	nfsm_chain_add_32(error, &nmrep, maxsize);
+	nfsm_chain_add_32(error, &nmrep, prefsize);
+	nfsm_chain_add_32(error, &nmrep, NFS_FABLKSIZE);
+	nfsm_chain_add_32(error, &nmrep, maxsize);
+	nfsm_chain_add_32(error, &nmrep, prefsize);
+	nfsm_chain_add_32(error, &nmrep, NFS_FABLKSIZE);
+	nfsm_chain_add_32(error, &nmrep, prefsize);
+	nfsm_chain_add_64(error, &nmrep, 0xffffffffffffffffULL);
+	nfsm_chain_add_32(error, &nmrep, 0);
+	nfsm_chain_add_32(error, &nmrep, 1);
+	/* XXX link/symlink support should be taken from volume capabilities */
+	nfsm_chain_add_32(error, &nmrep,
+		NFSV3FSINFO_LINK | NFSV3FSINFO_SYMLINK |
+		NFSV3FSINFO_HOMOGENEOUS | NFSV3FSINFO_CANSETTIME);
+
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -4262,73 +4604,82 @@ nfsmout:
  * nfs pathconf service
  */
 int
-nfsrv_pathconf(nfsd, slp, procp, mrq)
-	struct nfsrv_descript *nfsd;
-	struct nfssvc_sock *slp;
-	proc_t procp;
-	mbuf_t *mrq;
+nfsrv_pathconf(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep, md = nfsd->nd_md;
-	mbuf_t nam = nfsd->nd_nam;
-	caddr_t dpos = nfsd->nd_dpos;
-	u_long *tl;
-	struct nfsv3_pathconf *pc;
-	long t1;
-	caddr_t bpos;
-	int error = 0, getret = 1, linkmax, namemax;
+	int error, attrerr, linkmax, namemax;
 	int chownres, notrunc, case_sensitive, case_preserving;
-	char *cp2;
-	mbuf_t mb, mb2, mreq;
 	vnode_t vp;
-	struct vnode_attr at;
+	struct vnode_attr attr;
 	struct nfs_filehandle nfh;
 	struct nfs_export *nx;
 	struct nfs_export_options *nxo;
-	struct vfs_context context;
+	struct nfsm_chain *nmreq, nmrep;
 
-	nfsm_srvmtofh(&nfh);
-	if ((error = nfsrv_fhtovp(&nfh, nam, TRUE, &vp, &nx, &nxo))) {
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	if ((error = nfsrv_credcheck(nfsd, nx, nxo))) {
-		vnode_put(vp);
-		nfsm_reply(NFSX_UNSIGNED);
-		nfsm_srvpostop_attr(getret, &at);
-		return (0);
-	}
-	context.vc_proc = procp;
-	context.vc_ucred = nfsd->nd_cr;
+	error = 0;
+	attrerr = ENOENT;
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_null(&nmrep);
+	vp = NULL;
 
-	error = VNOP_PATHCONF(vp, _PC_LINK_MAX, &linkmax, &context);
+	nfsm_chain_get_fh_ptr(error, nmreq, nd->nd_vers, nfh.nfh_fhp, nfh.nfh_len);
+	nfsmerr_if(error);
+	error = nfsrv_fhtovp(&nfh, nd, &vp, &nx, &nxo);
+	nfsmerr_if(error);
+
+	/* update export stats */
+	NFSStatAdd64(&nx->nx_stats.ops, 1);
+
+	/* update active user stats */
+	nfsrv_update_user_stat(nx, nd, kauth_cred_getuid(nd->nd_cr), 1, 0, 0);
+
+	error = nfsrv_credcheck(nd, ctx, nx, nxo);
+	nfsmerr_if(error);
+
+	error = VNOP_PATHCONF(vp, _PC_LINK_MAX, &linkmax, ctx);
 	if (!error)
-		error = VNOP_PATHCONF(vp, _PC_NAME_MAX, &namemax, &context);
+		error = VNOP_PATHCONF(vp, _PC_NAME_MAX, &namemax, ctx);
 	if (!error)
-		error = VNOP_PATHCONF(vp, _PC_CHOWN_RESTRICTED, &chownres, &context);
+		error = VNOP_PATHCONF(vp, _PC_CHOWN_RESTRICTED, &chownres, ctx);
 	if (!error)
-		error = VNOP_PATHCONF(vp, _PC_NO_TRUNC, &notrunc, &context);
+		error = VNOP_PATHCONF(vp, _PC_NO_TRUNC, &notrunc, ctx);
 	if (!error)
-		error = VNOP_PATHCONF(vp, _PC_CASE_SENSITIVE, &case_sensitive, &context);
+		error = VNOP_PATHCONF(vp, _PC_CASE_SENSITIVE, &case_sensitive, ctx);
 	if (!error)
-		error = VNOP_PATHCONF(vp, _PC_CASE_PRESERVING, &case_preserving, &context);
-	nfsm_srv_vattr_init(&at, 1);
-	getret = vnode_getattr(vp, &at, &context);
-	vnode_put(vp);
-	nfsm_reply(NFSX_V3POSTOPATTR + NFSX_V3PATHCONF);
-	nfsm_srvpostop_attr(getret, &at);
-	if (error)
-		return (0);
-	nfsm_build(pc, struct nfsv3_pathconf *, NFSX_V3PATHCONF);
+		error = VNOP_PATHCONF(vp, _PC_CASE_PRESERVING, &case_preserving, ctx);
+
+	nfsm_srv_vattr_init(&attr, NFS_VER3);
+	attrerr = vnode_getattr(vp, &attr, ctx);
 
-	pc->pc_linkmax = txdr_unsigned(linkmax);
-	pc->pc_namemax = txdr_unsigned(namemax);
-	pc->pc_notrunc = txdr_unsigned(notrunc);
-	pc->pc_chownrestricted = txdr_unsigned(chownres);
-	pc->pc_caseinsensitive = txdr_unsigned(!case_sensitive);
-	pc->pc_casepreserving = txdr_unsigned(case_preserving);
+nfsmerr:
+	if (vp)
+		vnode_put(vp);
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, NFSX_V3POSTOPATTR + NFSX_V3PATHCONF);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
+	nfsmout_on_status(nd, error);
+	nfsm_chain_add_postop_attr(error, nd, &nmrep, attrerr, &attr);
+	nfsmout_if(nd->nd_repstat);
+
+	nfsm_chain_add_32(error, &nmrep, linkmax);
+	nfsm_chain_add_32(error, &nmrep, namemax);
+	nfsm_chain_add_32(error, &nmrep, notrunc);
+	nfsm_chain_add_32(error, &nmrep, chownres);
+	nfsm_chain_add_32(error, &nmrep, !case_sensitive);
+	nfsm_chain_add_32(error, &nmrep, case_preserving);
 
 nfsmout:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
 	return (error);
 }
 
@@ -4338,19 +4689,34 @@ nfsmout:
 /* ARGSUSED */
 int
 nfsrv_null(
-	struct nfsrv_descript *nfsd,
-	struct nfssvc_sock *slp,
-	__unused proc_t procp,
-	mbuf_t *mrq)
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	__unused vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep;
-	caddr_t bpos;
 	int error = NFSERR_RETVOID;
-	mbuf_t mb, mreq;
+	struct nfsm_chain nmrep;
+
+	/*
+	 * RPCSEC_GSS context setup ?
+	 */
+	if (nd->nd_gss_context)
+		return(nfs_gss_svc_ctx_init(nd, slp, mrepp));
 
-	nfsm_reply(0);
+	nfsm_chain_null(&nmrep);
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, 0);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
 nfsmout:
-	return (0);
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
+	return (error);
 }
 
 /*
@@ -4359,25 +4725,64 @@ nfsmout:
 /* ARGSUSED */
 int
 nfsrv_noop(
-	struct nfsrv_descript *nfsd,
-	struct nfssvc_sock *slp,
-	__unused proc_t procp,
-	mbuf_t *mrq)
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	__unused vfs_context_t ctx,
+	mbuf_t *mrepp)
 {
-	mbuf_t mrep = nfsd->nd_mrep;
-	caddr_t bpos;
 	int error;
-	mbuf_t mb, mreq;
+	struct nfsm_chain nmrep;
+
+	nfsm_chain_null(&nmrep);
 
-	if (nfsd->nd_repstat)
-		error = nfsd->nd_repstat;
+	if (nd->nd_repstat)
+		error = nd->nd_repstat;
 	else
 		error = EPROCUNAVAIL;
-	nfsm_reply(0);
+
+	/* assemble reply */
+	nd->nd_repstat = error;
+	error = nfsrv_rephead(nd, slp, &nmrep, 0);
+	nfsmout_if(error);
+	*mrepp = nmrep.nmc_mhead;
 nfsmout:
-	return (0);
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		nfsm_chain_cleanup(&nmrep);
+		*mrepp = NULL;
+	}
+	return (error);
 }
 
+int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
+				    struct nfsrv_sock *slp,
+				    vfs_context_t ctx,
+				    mbuf_t *mrepp) = {
+	nfsrv_null,
+	nfsrv_getattr,
+	nfsrv_setattr,
+	nfsrv_lookup,
+	nfsrv_access,
+	nfsrv_readlink,
+	nfsrv_read,
+	nfsrv_write,
+	nfsrv_create,
+	nfsrv_mkdir,
+	nfsrv_symlink,
+	nfsrv_mknod,
+	nfsrv_remove,
+	nfsrv_rmdir,
+	nfsrv_rename,
+	nfsrv_link,
+	nfsrv_readdir,
+	nfsrv_readdirplus,
+	nfsrv_statfs,
+	nfsrv_fsinfo,
+	nfsrv_pathconf,
+	nfsrv_commit,
+	nfsrv_noop
+};
+
 /*
  * Perform access checking for vnodes obtained from file handles that would
  * refer to files already opened by a Unix client. You cannot just use
@@ -4397,9 +4802,9 @@ nfsrv_authorize(
 	vnode_t vp,
 	vnode_t dvp,
 	kauth_action_t action,
-	vfs_context_t context,
+	vfs_context_t ctx,
 	struct nfs_export_options *nxo,
-        int override)
+	int override)
 {
 	struct vnode_attr vattr;
 	int error;
@@ -4419,19 +4824,20 @@ nfsrv_authorize(
 			}
 		}
 	}
-	error = vnode_authorize(vp, dvp, action, context);
-        /*
-         * Allow certain operations for the owner (reads and writes
-         * on files that are already open). Picking up from FreeBSD.
-         */
+	error = vnode_authorize(vp, dvp, action, ctx);
+	/*
+	 * Allow certain operations for the owner (reads and writes
+	 * on files that are already open). Picking up from FreeBSD.
+	 */
 	if (override && (error == EACCES)) {
 		VATTR_INIT(&vattr);
 		VATTR_WANTED(&vattr, va_uid);
-		if ((vnode_getattr(vp, &vattr, context) == 0) &&
-		    (kauth_cred_getuid(vfs_context_ucred(context)) == vattr.va_uid))
+		if ((vnode_getattr(vp, &vattr, ctx) == 0) &&
+		    (kauth_cred_getuid(vfs_context_ucred(ctx)) == vattr.va_uid))
 			error = 0;
 	}
-        return error;
+	return error;
 }
-#endif /* NFS_NOSERVER */
+
+#endif /* NFSSERVER */
 
diff --git a/bsd/nfs/nfs_socket.c b/bsd/nfs/nfs_socket.c
index 62dbe3680..4c21d2115 100644
--- a/bsd/nfs/nfs_socket.c
+++ b/bsd/nfs/nfs_socket.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -84,6 +90,7 @@
 #include <kern/clock.h>
 #include <kern/task.h>
 #include <kern/thread.h>
+#include <kern/thread_call.h>
 #include <sys/user.h>
 
 #include <netinet/in.h>
@@ -94,21 +101,37 @@
 #include <nfs/nfs.h>
 #include <nfs/xdr_subs.h>
 #include <nfs/nfsm_subs.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/nfsnode.h>
-#include <nfs/nfsrtt.h>
 
-#include <sys/kdebug.h>
+/* XXX */
+boolean_t	current_thread_aborted(void);
+kern_return_t	thread_terminate(thread_t);
+
+
+#if NFSSERVER
+int nfsrv_sock_max_rec_queue_length = 128; /* max # RPC records queued on (UDP) socket */
+
+static int nfsrv_getstream(struct nfsrv_sock *,int);
+static int nfsrv_getreq(struct nfsrv_descript *);
+extern int nfsv3_procid[NFS_NPROCS];
+#endif /* NFSSERVER */
+
+#if NFSCLIENT
+
+static int	nfs_connect_setup(struct nfsmount *);
+static void	nfs_reqdequeue(struct nfsreq *);
+static void	nfs_udp_rcv(socket_t, void*, int);
+static void	nfs_tcp_rcv(socket_t, void*, int);
+static void	nfs_request_match_reply(struct nfsmount *, mbuf_t);
+static void	nfs_softterm(struct nfsreq *);
 
-#define FSDBG(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_NONE, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_TOP(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_START, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_BOT(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_END, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
+#ifdef NFS_SOCKET_DEBUGGING
+#define NFS_SOCK_DBG(X)	printf X
+#else
+#define NFS_SOCK_DBG(X)
+#endif
 
 /*
  * Estimate rto for an nfs rpc sent via. an unreliable datagram.
@@ -129,17 +152,6 @@
 	  ((((n)->nm_srtt[t-1] + 7) >> 3) + (n)->nm_sdrtt[t-1] + 1)))
 #define	NFS_SRTT(r)	(r)->r_nmp->nm_srtt[proct[(r)->r_procnum] - 1]
 #define	NFS_SDRTT(r)	(r)->r_nmp->nm_sdrtt[proct[(r)->r_procnum] - 1]
-/*
- * External data, mostly RPC constants in XDR form
- */
-extern u_long rpc_reply, rpc_msgdenied, rpc_mismatch, rpc_vers, rpc_auth_unix,
-	rpc_msgaccepted, rpc_call, rpc_autherr,
-	rpc_auth_kerb;
-extern u_long nfs_prog;
-extern struct nfsstats nfsstats;
-extern int nfsv3_procid[NFS_NPROCS];
-extern int nfs_ticks;
-extern u_long nfs_xidwrap;
 
 /*
  * Defines which timer to use for the procnum.
@@ -169,362 +181,283 @@ static int proct[NFS_NPROCS] = {
 #define	NFS_CWNDSCALE	256
 #define	NFS_MAXCWND	(NFS_CWNDSCALE * 32)
 static int nfs_backoff[8] = { 2, 4, 8, 16, 32, 64, 128, 256, };
-int nfsrtton = 0;
-struct nfsrtt nfsrtt;
-
-static int	nfs_rcvlock(struct nfsreq *);
-static void	nfs_rcvunlock(struct nfsreq *);
-static int	nfs_receive(struct nfsreq *rep, mbuf_t *mp);
-static int	nfs_reconnect(struct nfsreq *rep);
-static void	nfs_repdequeue(struct nfsreq *rep);
-
-/* XXX */
-boolean_t	current_thread_aborted(void);
-kern_return_t	thread_terminate(thread_t);
-
-#ifndef NFS_NOSERVER 
-static int	nfsrv_getstream(struct nfssvc_sock *,int);
-
-int (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
-				    struct nfssvc_sock *slp,
-				    proc_t procp,
-				    mbuf_t *mreqp) = {
-	nfsrv_null,
-	nfsrv_getattr,
-	nfsrv_setattr,
-	nfsrv_lookup,
-	nfsrv3_access,
-	nfsrv_readlink,
-	nfsrv_read,
-	nfsrv_write,
-	nfsrv_create,
-	nfsrv_mkdir,
-	nfsrv_symlink,
-	nfsrv_mknod,
-	nfsrv_remove,
-	nfsrv_rmdir,
-	nfsrv_rename,
-	nfsrv_link,
-	nfsrv_readdir,
-	nfsrv_readdirplus,
-	nfsrv_statfs,
-	nfsrv_fsinfo,
-	nfsrv_pathconf,
-	nfsrv_commit,
-	nfsrv_noop
-};
-#endif /* NFS_NOSERVER */
-
-
-/*
- * attempt to bind a socket to a reserved port
- */
-static int
-nfs_bind_resv(struct nfsmount *nmp)
-{
-	socket_t so = nmp->nm_so;
-	struct sockaddr_in sin;
-	int error;
-	u_short tport;
-
-	if (!so)
-		return (EINVAL);
-
-	sin.sin_len = sizeof (struct sockaddr_in);
-	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = INADDR_ANY;
-	tport = IPPORT_RESERVED - 1;
-	sin.sin_port = htons(tport);
-
-	while (((error = sock_bind(so, (struct sockaddr *) &sin)) == EADDRINUSE) &&
-	       (--tport > IPPORT_RESERVED / 2))
-		sin.sin_port = htons(tport);
-	return (error);
-}
-
-/*
- * variables for managing the nfs_bind_resv_thread
- */
-int nfs_resv_mounts = 0;
-static int nfs_bind_resv_thread_state = 0;
-#define NFS_BIND_RESV_THREAD_STATE_INITTED	1
-#define NFS_BIND_RESV_THREAD_STATE_RUNNING	2
-lck_grp_t *nfs_bind_resv_lck_grp;
-lck_grp_attr_t *nfs_bind_resv_lck_grp_attr;
-lck_attr_t *nfs_bind_resv_lck_attr;
-lck_mtx_t *nfs_bind_resv_mutex;
-struct nfs_bind_resv_request {
-	TAILQ_ENTRY(nfs_bind_resv_request) brr_chain;
-	struct nfsmount *brr_nmp;
-	int brr_error;
-};
-static TAILQ_HEAD(, nfs_bind_resv_request) nfs_bind_resv_request_queue;
-
-/*
- * thread to handle any reserved port bind requests
- */
-static void
-nfs_bind_resv_thread(void)
-{
-	struct nfs_bind_resv_request *brreq;
-
-	nfs_bind_resv_thread_state = NFS_BIND_RESV_THREAD_STATE_RUNNING;
-
-	while (nfs_resv_mounts > 0) {
-		lck_mtx_lock(nfs_bind_resv_mutex);
-		while ((brreq = TAILQ_FIRST(&nfs_bind_resv_request_queue))) {
-			TAILQ_REMOVE(&nfs_bind_resv_request_queue, brreq, brr_chain);
-			lck_mtx_unlock(nfs_bind_resv_mutex);
-			brreq->brr_error = nfs_bind_resv(brreq->brr_nmp);
-			wakeup(brreq);
-			lck_mtx_lock(nfs_bind_resv_mutex);
-		}
-		msleep((caddr_t)&nfs_bind_resv_request_queue,
-				nfs_bind_resv_mutex, PSOCK | PDROP,
-				"nfs_bind_resv_request_queue", 0);
-	}
-
-	nfs_bind_resv_thread_state = NFS_BIND_RESV_THREAD_STATE_INITTED;
-	(void) thread_terminate(current_thread());
-}
-
-int
-nfs_bind_resv_thread_wake(void)
-{
-	if (nfs_bind_resv_thread_state < NFS_BIND_RESV_THREAD_STATE_RUNNING)
-		return (EIO);
-	wakeup(&nfs_bind_resv_request_queue);
-	return (0);
-}
-
-/*
- * underprivileged procs call this to request nfs_bind_resv_thread
- * to perform the reserved port binding for them.
- */
-static int
-nfs_bind_resv_nopriv(struct nfsmount *nmp)
-{
-	struct nfs_bind_resv_request brreq;
-	int error;
-
-	if (nfs_bind_resv_thread_state < NFS_BIND_RESV_THREAD_STATE_RUNNING) {
-		if (nfs_bind_resv_thread_state < NFS_BIND_RESV_THREAD_STATE_INITTED) {
-			nfs_bind_resv_lck_grp_attr = lck_grp_attr_alloc_init();
-			nfs_bind_resv_lck_grp = lck_grp_alloc_init("nfs_bind_resv", nfs_bind_resv_lck_grp_attr);
-			nfs_bind_resv_lck_attr = lck_attr_alloc_init();
-			nfs_bind_resv_mutex = lck_mtx_alloc_init(nfs_bind_resv_lck_grp, nfs_bind_resv_lck_attr);
-			TAILQ_INIT(&nfs_bind_resv_request_queue);
-			nfs_bind_resv_thread_state = NFS_BIND_RESV_THREAD_STATE_INITTED;
-		}
-		kernel_thread(kernel_task, nfs_bind_resv_thread);
-		nfs_bind_resv_thread_state = NFS_BIND_RESV_THREAD_STATE_RUNNING;
-	}
-
-	brreq.brr_nmp = nmp;
-	brreq.brr_error = 0;
-
-	lck_mtx_lock(nfs_bind_resv_mutex);
-	TAILQ_INSERT_TAIL(&nfs_bind_resv_request_queue, &brreq, brr_chain);
-	lck_mtx_unlock(nfs_bind_resv_mutex);
-
-	error = nfs_bind_resv_thread_wake();
-	if (error) {
-		TAILQ_REMOVE(&nfs_bind_resv_request_queue, &brreq, brr_chain);
-		/* Note: we might be able to simply restart the thread */
-		return (error);
-	}
-
-	tsleep((caddr_t)&brreq, PSOCK, "nfsbindresv", 0);
-
-	return (brreq.brr_error);
-}
 
 /*
- * Initialize sockets and congestion for a new NFS connection.
- * We do not free the sockaddr if error.
+ * Initialize socket state and perform setup for a new NFS connection.
  */
 int
-nfs_connect(
-	struct nfsmount *nmp,
-	__unused struct nfsreq *rep)
+nfs_connect(struct nfsmount *nmp)
 {
 	socket_t so;
-	int error, rcvreserve, sndreserve;
+	int error, on = 1, proto;
+	sock_upcall upcall;
 	struct sockaddr *saddr;
+	struct sockaddr_in sin;
 	struct timeval timeo;
+	u_short tport;
 
-	nmp->nm_so = 0;
+	lck_mtx_lock(&nmp->nm_lock);
+	nmp->nm_sockflags |= NMSOCK_CONNECTING;
 	saddr = mbuf_data(nmp->nm_nam);
+	upcall = (nmp->nm_sotype == SOCK_STREAM) ? nfs_tcp_rcv : nfs_udp_rcv;
+	lck_mtx_unlock(&nmp->nm_lock);
 	error = sock_socket(saddr->sa_family, nmp->nm_sotype,
-						nmp->nm_soproto, 0, 0, &nmp->nm_so);
-	if (error) {
+			    nmp->nm_soproto, upcall, nmp, &nmp->nm_so);
+	if (error)
 		goto bad;
-	}
+	lck_mtx_lock(&nmp->nm_lock);
 	so = nmp->nm_so;
 
 	/*
 	 * Some servers require that the client port be a reserved port number.
 	 */
 	if (saddr->sa_family == AF_INET && (nmp->nm_flag & NFSMNT_RESVPORT)) {
-		proc_t p;
-		/*
-		 * sobind() requires current_proc() to have superuser privs.
-		 * If this bind is part of a reconnect, and the current proc
-		 * doesn't have superuser privs, we hand the sobind() off to
-		 * a kernel thread to process.
-		 */
-		if ((nmp->nm_state & NFSSTA_MOUNTED) &&
-		    (p = current_proc()) && suser(kauth_cred_get(), 0)) {
-			/* request nfs_bind_resv_thread() to do bind */
-			error = nfs_bind_resv_nopriv(nmp);
-		} else {
-			error = nfs_bind_resv(nmp);
-		}
+		lck_mtx_unlock(&nmp->nm_lock);
+		sin.sin_len = sizeof (struct sockaddr_in);
+		sin.sin_family = AF_INET;
+		sin.sin_addr.s_addr = INADDR_ANY;
+		tport = IPPORT_RESERVED - 1;
+		sin.sin_port = htons(tport);
+		while (((error = sock_bind(so, (struct sockaddr *) &sin)) == EADDRINUSE) &&
+		       (--tport > IPPORT_RESERVED / 2))
+			sin.sin_port = htons(tport);
 		if (error)
 			goto bad;
+		lck_mtx_lock(&nmp->nm_lock);
 	}
 
 	/*
 	 * Protocols that do not require connections may be optionally left
-	 * unconnected for servers that reply from a port other than NFS_PORT.
+	 * unconnected for servers that reply from a different address/port.
 	 */
 	if (nmp->nm_flag & NFSMNT_NOCONN) {
 		if (nmp->nm_sotype == SOCK_STREAM) {
 			error = ENOTCONN;
+			lck_mtx_unlock(&nmp->nm_lock);
 			goto bad;
 		}
 	} else {
-		struct timeval	tv;
-		tv.tv_sec = 2;
-		tv.tv_usec = 0;
+		int tocnt = 0, optlen = sizeof(error);
+		struct timespec ts = { 2, 0 };
+
+		lck_mtx_unlock(&nmp->nm_lock);
 		error = sock_connect(so, mbuf_data(nmp->nm_nam), MSG_DONTWAIT);
-		if (error && error != EINPROGRESS) {
+		if (error && (error != EINPROGRESS))
 			goto bad;
-		}
-		
-		while ((error = sock_connectwait(so, &tv)) == EINPROGRESS) {
-			if (rep && (error = nfs_sigintr(nmp, rep, rep->r_procp))) {
-				goto bad;
+		lck_mtx_lock(&nmp->nm_lock);
+		while (!sock_isconnected(so)) {
+			if (tocnt++ == 15) /* log a warning if connect is taking a while */
+				log(LOG_INFO, "nfs_connect: socket connect taking a while for %s\n",
+					vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+			/* check for error on socket */
+			sock_getsockopt(so, SOL_SOCKET, SO_ERROR, &error, &optlen);
+			if (error) {
+				log(LOG_INFO, "nfs_connect: socket error %d for %s\n",
+					error, vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+				break;
+			}
+			if (tocnt > 60) {
+				/* abort if this is taking too long */
+				error = ENOTCONN;
+				break;
 			}
+			if ((error = nfs_sigintr(nmp, NULL, current_thread(), 1)))
+				break;
+			error = msleep(&nmp->nm_so, &nmp->nm_lock, PSOCK, "nfs_socket_connect", &ts);
+			if (error == EWOULDBLOCK)
+				error = 0;
+			if (error)
+				break;
+		}
+		if (tocnt > 15)
+			log(LOG_INFO, "nfs_connect: socket connect %s for %s\n",
+				error ? "aborted" : "completed",
+				vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+		if (error) {
+			lck_mtx_unlock(&nmp->nm_lock);
+			goto bad;
 		}
 	}
-	
+
 	/*
-	 * Always time out on recieve, this allows us to reconnect the
-	 * socket to deal with network changes.
+	 * Set socket send/receive timeouts
+	 * - Receive timeout shouldn't matter because all receives are performed
+	 *   in the socket upcall non-blocking.
+	 * - Send timeout should allow us to react to a blocked socket.
+	 *   Soft mounts will want to abort sooner.
 	 */
 	timeo.tv_usec = 0;
-	timeo.tv_sec = 2;
-	error = sock_setsockopt(so, SOL_SOCKET, SO_RCVTIMEO, &timeo, sizeof(timeo));
-	if (nmp->nm_flag & (NFSMNT_SOFT | NFSMNT_INT)) {
-		timeo.tv_sec = 5;
-	} else {
-		timeo.tv_sec = 0;
+	timeo.tv_sec = (nmp->nm_flag & NFSMNT_SOFT) ? 10 : 60;
+	error |= sock_setsockopt(so, SOL_SOCKET, SO_RCVTIMEO, &timeo, sizeof(timeo));
+	error |= sock_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO, &timeo, sizeof(timeo));
+	if (error) {
+		log(LOG_INFO, "nfs_connect: socket timeout setting errors for %s\n",
+			vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+		error = 0;
 	}
-	error = sock_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO, &timeo, sizeof(timeo));
-	
-	if (nmp->nm_sotype == SOCK_DGRAM) {
-		sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 3;
-		rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR) *
-			(nmp->nm_readahead > 0 ? nmp->nm_readahead + 1 : 2);
-	} else if (nmp->nm_sotype == SOCK_SEQPACKET) {
-		sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 3;
-		rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR) *
-			(nmp->nm_readahead > 0 ? nmp->nm_readahead + 1 : 2);
-	} else {
-		int proto;
-		int on = 1;
-		
-		sock_gettype(so, NULL, NULL, &proto);
-		if (nmp->nm_sotype != SOCK_STREAM)
-			panic("nfscon sotype");
 
-		// Assume that SOCK_STREAM always requires a connection
+	if (nmp->nm_sotype == SOCK_STREAM) {
+		/* Assume that SOCK_STREAM always requires a connection */
 		sock_setsockopt(so, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on));
-		
-		if (proto == IPPROTO_TCP) {
+		/* set nodelay for TCP */
+		sock_gettype(so, NULL, NULL, &proto);
+		if (proto == IPPROTO_TCP)
 			sock_setsockopt(so, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
-		}
-
-		sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR + sizeof (u_long)) * 3;
-		rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR + sizeof (u_long)) *
-				(nmp->nm_readahead > 0 ? nmp->nm_readahead + 1 : 2);
 	}
 
-	if (sndreserve > NFS_MAXSOCKBUF)
-		sndreserve = NFS_MAXSOCKBUF;
-	if (rcvreserve > NFS_MAXSOCKBUF)
-		rcvreserve = NFS_MAXSOCKBUF;
-	error = sock_setsockopt(so, SOL_SOCKET, SO_SNDBUF, &sndreserve, sizeof(sndreserve));
-	if (error) {
-		goto bad;
+	if (nmp->nm_sotype == SOCK_DGRAM) { /* set socket buffer sizes for UDP */
+		int reserve = NFS_UDPSOCKBUF;
+		error |= sock_setsockopt(so, SOL_SOCKET, SO_SNDBUF, &reserve, sizeof(reserve));
+		error |= sock_setsockopt(so, SOL_SOCKET, SO_RCVBUF, &reserve, sizeof(reserve));
+		if (error) {
+			log(LOG_INFO, "nfs_connect: socket buffer setting errors for %s\n",
+				vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+			error = 0;
+		}
 	}
-	error = sock_setsockopt(so, SOL_SOCKET, SO_RCVBUF, &rcvreserve, sizeof(rcvreserve));
+
+	/* set SO_NOADDRERR to detect network changes ASAP */
+	error = sock_setsockopt(so, SOL_SOCKET, SO_NOADDRERR, &on, sizeof(on));
 	if (error) {
+		lck_mtx_unlock(&nmp->nm_lock);
 		goto bad;
 	}
 
-	sock_nointerrupt(so, 1);
+	if (!(nmp->nm_flag & NFSMNT_INT))
+		sock_nointerrupt(so, 1);
 
-	/* Initialize other non-zero congestion variables */
+	/* Initialize socket state variables */
 	nmp->nm_srtt[0] = nmp->nm_srtt[1] = nmp->nm_srtt[2] =
 		nmp->nm_srtt[3] = (NFS_TIMEO << 3);
 	nmp->nm_sdrtt[0] = nmp->nm_sdrtt[1] = nmp->nm_sdrtt[2] =
 		nmp->nm_sdrtt[3] = 0;
-	nmp->nm_cwnd = NFS_MAXCWND / 2;	    /* Initial send window */
-	nmp->nm_sent = 0;
-	FSDBG(529, nmp, nmp->nm_state, nmp->nm_soflags, nmp->nm_cwnd);
-	nmp->nm_timeouts = 0;
-	return (0);
-
+	if (nmp->nm_sotype == SOCK_DGRAM) {
+		/* XXX do we really want to reset this on each reconnect? */
+		nmp->nm_cwnd = NFS_MAXCWND / 2;	    /* Initial send window */
+		nmp->nm_sent = 0;
+	} else if (nmp->nm_sotype == SOCK_STREAM) {
+		nmp->nm_markerleft = sizeof(nmp->nm_fragleft);
+		nmp->nm_fragleft = nmp->nm_reclen = 0;
+		nmp->nm_timeouts = 0;
+	}
+	nmp->nm_sockflags &= ~NMSOCK_CONNECTING;
+	nmp->nm_sockflags |= NMSOCK_SETUP;
+	FSDBG(529, nmp, nmp->nm_state, nmp->nm_flag, nmp->nm_cwnd);
+	lck_mtx_unlock(&nmp->nm_lock);
+	error = nfs_connect_setup(nmp);
 bad:
-	nfs_disconnect(nmp);
+	lck_mtx_lock(&nmp->nm_lock);
+	nmp->nm_sockflags &= ~(NMSOCK_CONNECTING|NMSOCK_SETUP);
+	if (!error) {
+		nmp->nm_sockflags |= NMSOCK_READY;
+		wakeup(&nmp->nm_sockflags);
+	}
+	lck_mtx_unlock(&nmp->nm_lock);
+	if (error)
+		nfs_disconnect(nmp);
+	return (error);
+}
+
+/* setup & confirm socket connection is functional */
+static int
+nfs_connect_setup(struct nfsmount *nmp)
+{
+	struct nfsm_chain nmreq, nmrep;
+	int error = 0, status;
+	u_int64_t xid;
+
+	if (nmp->nm_vers >= NFS_VER4) {
+		error = nfs4_setclientid(nmp);
+	} else {
+		/* verify connection's OK by sending a NULL request */
+		nfsm_chain_null(&nmreq);
+		nfsm_chain_null(&nmrep);
+		nfsm_chain_build_alloc_init(error, &nmreq, 0);
+		nfsm_chain_build_done(error, &nmreq);
+		nfsmout_if(error);
+		error = nfs_request2(NULL, nmp->nm_mountp, &nmreq, NFSPROC_NULL,
+				current_thread(), NULL, R_SETUP, &nmrep, &xid, &status);
+		if (!error)
+			error = status;
+nfsmout:
+		nfsm_chain_cleanup(&nmreq);
+		nfsm_chain_cleanup(&nmrep);
+	}
 	return (error);
 }
 
 /*
- * Reconnect routine:
- * Called when a connection is broken on a reliable protocol.
- * - clean up the old socket
+ * NFS socket reconnect routine:
+ * Called when a connection is broken.
+ * - disconnect the old socket
  * - nfs_connect() again
  * - set R_MUSTRESEND for all outstanding requests on mount point
  * If this fails the mount point is DEAD!
- * nb: Must be called with the nfs_sndlock() set on the mount point.
  */
 static int
-nfs_reconnect(struct nfsreq *rep)
+nfs_reconnect(struct nfsmount *nmp)
 {
-	struct nfsreq *rp;
-	struct nfsmount *nmp = rep->r_nmp;
-	int error;
+	struct nfsreq *rq;
+	struct timeval now;
+	thread_t thd = current_thread();
+	int error, lastmsg, wentdown = 0;
+
+	microuptime(&now);
+	lastmsg = now.tv_sec - (nmp->nm_tprintf_delay - nmp->nm_tprintf_initial_delay);
 
 	nfs_disconnect(nmp);
-	while ((error = nfs_connect(nmp, rep))) {
+
+	while ((error = nfs_connect(nmp))) {
 		if (error == EINTR || error == ERESTART)
 			return (EINTR);
 		if (error == EIO)
 			return (EIO);
-		nfs_down(rep->r_nmp, rep->r_procp, error, NFSSTA_TIMEO,
-			"can not connect");
-		rep->r_flags |= R_TPRINTFMSG;
+		microuptime(&now);
+		if ((lastmsg + nmp->nm_tprintf_delay) < now.tv_sec) {
+			lastmsg = now.tv_sec;
+			nfs_down(nmp, thd, error, NFSSTA_TIMEO, "can not connect");
+			wentdown = 1;
+		}
+		lck_mtx_lock(&nmp->nm_lock);
 		if (!(nmp->nm_state & NFSSTA_MOUNTED)) {
 			/* we're not yet completely mounted and */
 			/* we can't reconnect, so we fail */
+			lck_mtx_unlock(&nmp->nm_lock);
+			return (error);
+		}
+		if ((error = nfs_sigintr(nmp, NULL, thd, 1))) {
+			lck_mtx_unlock(&nmp->nm_lock);
 			return (error);
 		}
-		if ((error = nfs_sigintr(rep->r_nmp, rep, rep->r_procp)))
+		lck_mtx_unlock(&nmp->nm_lock);
+		tsleep(&lbolt, PSOCK, "nfs_reconnect_delay", 0);
+		if ((error = nfs_sigintr(nmp, NULL, thd, 0)))
 			return (error);
-		tsleep((caddr_t)&lbolt, PSOCK, "nfscon", 0);
 	}
 
+	if (wentdown)
+		nfs_up(nmp, thd, NFSSTA_TIMEO, "connected");
+
 	/*
-	 * Loop through outstanding request list and fix up all requests
-	 * on old socket.
+	 * Loop through outstanding request list and mark all requests
+	 * as needing a resend.  (Though nfs_need_reconnect() probably
+	 * marked them all already.)
 	 */
-	TAILQ_FOREACH(rp, &nfs_reqq, r_chain) {
-		if (rp->r_nmp == nmp)
-			rp->r_flags |= R_MUSTRESEND;
+	lck_mtx_lock(nfs_request_mutex);
+	TAILQ_FOREACH(rq, &nfs_reqq, r_chain) {
+		if (rq->r_nmp == nmp) {
+			lck_mtx_lock(&rq->r_mtx);
+			if (!rq->r_error && !rq->r_nmrep.nmc_mhead && !(rq->r_flags & R_MUSTRESEND)) {
+				rq->r_flags |= R_MUSTRESEND;
+				rq->r_rtt = -1;
+				wakeup(rq);
+				if ((rq->r_flags & (R_ASYNC|R_ASYNCWAIT)) == R_ASYNC)
+					nfs_asyncio_resend(rq);
+			}
+			lck_mtx_unlock(&rq->r_mtx);
+		}
 	}
+	lck_mtx_unlock(nfs_request_mutex);
 	return (0);
 }
 
@@ -536,1150 +469,1676 @@ nfs_disconnect(struct nfsmount *nmp)
 {
 	socket_t so;
 
+	lck_mtx_lock(&nmp->nm_lock);
+	if ((nmp->nm_sotype == SOCK_STREAM) && nmp->nm_m) {
+		mbuf_freem(nmp->nm_m);
+		nmp->nm_m = nmp->nm_mlast = NULL;
+	}
 	if (nmp->nm_so) {
 		so = nmp->nm_so;
-		nmp->nm_so = 0;
-		sock_shutdown(so, 2);
+		nmp->nm_so = NULL;
+		lck_mtx_unlock(&nmp->nm_lock);
+		sock_shutdown(so, SHUT_RDWR);
 		sock_close(so);
+	} else {
+		lck_mtx_unlock(&nmp->nm_lock);
 	}
 }
 
 /*
- * This is the nfs send routine. For connection based socket types, it
- * must be called with an nfs_sndlock() on the socket.
- * "rep == NULL" indicates that it has been called from a server.
- * For the client side:
- * - return EINTR if the RPC is terminated, 0 otherwise
- * - set R_MUSTRESEND if the send fails for any reason
- * - do any cleanup required by recoverable socket errors (???)
- * For the server side:
- * - return EINTR or ERESTART if interrupted by a signal
- * - return EPIPE if a connection is lost for connection based sockets (TCP...)
- * - do any cleanup required by recoverable socket errors (???)
+ * mark an NFS mount as needing a reconnect/resends.
  */
-int
-nfs_send(so, nam, top, rep)
-	socket_t so;
-	mbuf_t nam;
-	mbuf_t top;
-	struct nfsreq *rep;
+static void
+nfs_need_reconnect(struct nfsmount *nmp)
 {
-	struct sockaddr *sendnam;
-	int error, error2, sotype, flags;
-	u_long xidqueued = 0;
-	struct nfsreq *rp;
-	char savenametolog[MAXPATHLEN];
-	struct msghdr msg;
-	
-	if (rep) {
-		error = nfs_sigintr(rep->r_nmp, rep, rep->r_procp);
-		if (error) {
-			mbuf_freem(top);
-			return (error);
-		}
-		if ((so = rep->r_nmp->nm_so) == NULL) {
-			rep->r_flags |= R_MUSTRESEND;
-			mbuf_freem(top);
-			return (0);
-		}
-		rep->r_flags &= ~R_MUSTRESEND;
-		TAILQ_FOREACH(rp, &nfs_reqq, r_chain)
-			if (rp == rep)
-				break;
-		if (rp)
-			xidqueued = rp->r_xid;
-	}
-	sock_gettype(so, NULL, &sotype, NULL);
-	if ((sotype == SOCK_STREAM) || (sock_isconnected(so)) ||
-	    (nam == 0))
-		sendnam = (struct sockaddr *)0;
-	else
-		sendnam = mbuf_data(nam);
+	struct nfsreq *rq;
 
-	if (sotype == SOCK_SEQPACKET)
-		flags = MSG_EOR;
-	else
-		flags = 0;
+	lck_mtx_lock(&nmp->nm_lock);
+	nmp->nm_sockflags &= ~(NMSOCK_READY|NMSOCK_SETUP);
+	lck_mtx_unlock(&nmp->nm_lock);
 
-	/* 
-	 * Save the name here in case mount point goes away if we block.
-	 * The name is using local stack and is large, but don't
-	 * want to block if we malloc.
+	/*
+	 * Loop through outstanding request list and
+	 * mark all requests as needing a resend.
 	 */
-	if (rep)
-		strncpy(savenametolog,
-			vfs_statfs(rep->r_nmp->nm_mountp)->f_mntfromname,
-			MAXPATHLEN - 1);
-	bzero(&msg, sizeof(msg));
-	msg.msg_name = (caddr_t)sendnam;
-	msg.msg_namelen = sendnam == 0 ? 0 : sendnam->sa_len;
-	error = sock_sendmbuf(so, &msg, top, flags, NULL);
-
-	if (error) {
-		if (rep) {
-			if (xidqueued) {
-				TAILQ_FOREACH(rp, &nfs_reqq, r_chain)
-					if (rp == rep && rp->r_xid == xidqueued)
-						break;
-				if (!rp)
-					panic("nfs_send: error %d xid %x gone",
-					      error, xidqueued);
-			}
-			log(LOG_INFO, "nfs send error %d for server %s\n",
-			    error, savenametolog);
-			/*
-			 * Deal with errors for the client side.
-			 */
-			error2 = nfs_sigintr(rep->r_nmp, rep, rep->r_procp);
-			if (error2) {
-				error = error2;
-			} else {
-				rep->r_flags |= R_MUSTRESEND;
+	lck_mtx_lock(nfs_request_mutex);
+	TAILQ_FOREACH(rq, &nfs_reqq, r_chain) {
+		if (rq->r_nmp == nmp) {
+			lck_mtx_lock(&rq->r_mtx);
+			if (!rq->r_error && !rq->r_nmrep.nmc_mhead && !(rq->r_flags & R_MUSTRESEND)) {
+				rq->r_flags |= R_MUSTRESEND;
+				rq->r_rtt = -1;
+				wakeup(rq);
+				if ((rq->r_flags & (R_ASYNC|R_ASYNCWAIT)) == R_ASYNC)
+					nfs_asyncio_resend(rq);
 			}
-		} else
-			log(LOG_INFO, "nfsd send error %d\n", error);
-
-		/*
-		 * Handle any recoverable (soft) socket errors here. (???)
-		 */
-		if (error != EINTR && error != ERESTART && error != EIO &&
-			error != EWOULDBLOCK && error != EPIPE) {
-			error = 0;
+			lck_mtx_unlock(&rq->r_mtx);
 		}
 	}
-	return (error);
+	lck_mtx_unlock(nfs_request_mutex);
 }
 
 /*
- * Receive a Sun RPC Request/Reply. For SOCK_DGRAM, the work is all
- * done by soreceive(), but for SOCK_STREAM we must deal with the Record
- * Mark and consolidate the data into a new mbuf list.
- * nb: Sometimes TCP passes the data up to soreceive() in long lists of
- *     small mbufs.
- * For SOCK_STREAM we must be very careful to read an entire record once
- * we have read any of it, even if the system call has been interrupted.
+ * thread to handle miscellaneous async NFS socket work (reconnects/resends)
  */
-static int
-nfs_receive(struct nfsreq *rep, mbuf_t *mp)
+static void
+nfs_mount_sock_thread(void *arg, __unused wait_result_t wr)
 {
-	socket_t so;
-	struct iovec_32 aio;
-	mbuf_t m, mlast;
-	u_long len, fraglen;
-	int error, error2, sotype;
-	proc_t p = current_proc();	/* XXX */
-	struct msghdr msg;
-	size_t rcvlen;
-	int lastfragment;
+	struct nfsmount *nmp = arg;
+	struct timespec ts = { 30, 0 };
+	thread_t thd = current_thread();
+	struct nfsreq *req;
+	struct timeval now;
+	int error, dofinish, force;
 
-	/*
-	 * Set up arguments for soreceive()
-	 */
-	*mp = NULL;
-	sotype = rep->r_nmp->nm_sotype;
+	lck_mtx_lock(&nmp->nm_lock);
 
-	/*
-	 * For reliable protocols, lock against other senders/receivers
-	 * in case a reconnect is necessary.
-	 * For SOCK_STREAM, first get the Record Mark to find out how much
-	 * more there is to get.
-	 * We must lock the socket against other receivers
-	 * until we have an entire rpc request/reply.
-	 */
-	if (sotype != SOCK_DGRAM) {
-		error = nfs_sndlock(rep);
-		if (error)
-			return (error);
-tryagain:
-		/*
-		 * Check for fatal errors and resending request.
-		 */
-		/*
-		 * Ugh: If a reconnect attempt just happened, nm_so
-		 * would have changed. NULL indicates a failed
-		 * attempt that has essentially shut down this
-		 * mount point.
-		 */
-		if ((error = nfs_sigintr(rep->r_nmp, rep, p)) || rep->r_mrep) {
-			nfs_sndunlock(rep);
-			if (error)
-				return (error);
-			return (EINTR);
-		}
-		so = rep->r_nmp->nm_so;
-		if (!so) {
-			error = nfs_reconnect(rep);
-			if (error) {
-				nfs_sndunlock(rep);
-				return (error);
-			}
-			goto tryagain;
-		}
-		while (rep->r_flags & R_MUSTRESEND) {
-			error = mbuf_copym(rep->r_mreq, 0, MBUF_COPYALL, MBUF_WAITOK, &m);
-			if (!error) {
-				OSAddAtomic(1, (SInt32*)&nfsstats.rpcretries);
-				error = nfs_send(so, rep->r_nmp->nm_nam, m, rep);
-			}
-			/*
-			 * we also hold rcv lock so rep is still
-			 * legit this point
-			 */
-			if (error) {
-				if (error == EINTR || error == ERESTART ||
-				    (error = nfs_reconnect(rep))) {
-					nfs_sndunlock(rep);
-					return (error);
-				}
-				goto tryagain;
+	while (!(nmp->nm_sockflags & NMSOCK_READY) || !TAILQ_EMPTY(&nmp->nm_resendq)) {
+		if (nmp->nm_sockflags & NMSOCK_UNMOUNT)
+			break;
+		force = (nmp->nm_state & NFSSTA_FORCE);
+		/* do reconnect, if necessary */
+		if (!(nmp->nm_sockflags & NMSOCK_READY) && !force) {
+			if (nmp->nm_reconnect_start <= 0) {
+				microuptime(&now);
+				nmp->nm_reconnect_start = now.tv_sec;
 			}
+			lck_mtx_unlock(&nmp->nm_lock);
+			NFS_SOCK_DBG(("nfs reconnect %s\n", vfs_statfs(nmp->nm_mountp)->f_mntfromname));
+			if ((error = nfs_reconnect(nmp)))
+				printf("nfs_reconnect failed %d for %s\n", error,
+					vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+			else
+				nmp->nm_reconnect_start = 0;
+			lck_mtx_lock(&nmp->nm_lock);
 		}
-		nfs_sndunlock(rep);
-		if (sotype == SOCK_STREAM) {
-			error = 0;
-			len = 0;
-			lastfragment = 0;
-			mlast = NULL;
-			while (!error && !lastfragment) {
-				aio.iov_base = (uintptr_t) &fraglen;
-				aio.iov_len = sizeof(u_long);
-				bzero(&msg, sizeof(msg));
-				msg.msg_iov = (struct iovec *) &aio;
-				msg.msg_iovlen = 1;
-				do {
-				   error = sock_receive(so, &msg, MSG_WAITALL, &rcvlen);
-				   if (!rep->r_nmp) /* if unmounted then bailout */
-					goto shutout;
-				   if (error == EWOULDBLOCK && rep) {
-					error2 = nfs_sigintr(rep->r_nmp, rep, p);
-					if (error2)
-						error = error2;
-				   }
-				} while (error == EWOULDBLOCK);
-				if (!error && rcvlen < aio.iov_len) {
-				    /* only log a message if we got a partial word */
-				    if (rcvlen != 0)
-					    log(LOG_INFO,
-						 "short receive (%d/%d) from nfs server %s\n",
-						 rcvlen, sizeof(u_long),
-						 vfs_statfs(rep->r_nmp->nm_mountp)->f_mntfromname);
-				    error = EPIPE;
-				}
-				if (error)
-					goto errout;
-				lastfragment = ntohl(fraglen) & 0x80000000;
-				fraglen = ntohl(fraglen) & ~0x80000000;
-				len += fraglen;
-				/*
-				 * This is SERIOUS! We are out of sync with the sender
-				 * and forcing a disconnect/reconnect is all I can do.
-				 */
-				if (len > NFS_MAXPACKET) {
-				    log(LOG_ERR, "%s (%d) from nfs server %s\n",
-					"impossible RPC record length", len,
-					vfs_statfs(rep->r_nmp->nm_mountp)->f_mntfromname);
-				    error = EFBIG;
-				    goto errout;
-				}
-
-				m = NULL;
-				do {
-				    rcvlen = fraglen;
-				    error = sock_receivembuf(so, NULL, &m, MSG_WAITALL, &rcvlen);
-				    if (!rep->r_nmp) /* if unmounted then bailout */ {
-					goto shutout;
-				    }
-				} while (error == EWOULDBLOCK || error == EINTR ||
-					 error == ERESTART);
-
-				if (!error && fraglen > rcvlen) {
-				    log(LOG_INFO,
-					"short receive (%d/%d) from nfs server %s\n",
-					rcvlen, fraglen,
-					vfs_statfs(rep->r_nmp->nm_mountp)->f_mntfromname);
-				    error = EPIPE;
-				    mbuf_freem(m);
-				}
-				if (!error) {
-					if (!*mp) {
-						*mp = m;
-						mlast = m;
-					} else {
-						error = mbuf_setnext(mlast, m);
-						if (error) {
-							printf("nfs_receive: mbuf_setnext failed %d\n", error);
-							mbuf_freem(m);
-						}
-					}
-					while (mbuf_next(mlast))
-						mlast = mbuf_next(mlast);
-				}
+		/* do resends, if necessary/possible */
+		while (((nmp->nm_sockflags & NMSOCK_READY) || force) && ((req = TAILQ_FIRST(&nmp->nm_resendq)))) {
+			if (req->r_resendtime)
+				microuptime(&now);
+			while (req && !force && req->r_resendtime && (now.tv_sec < req->r_resendtime))
+				req = TAILQ_NEXT(req, r_rchain);
+			if (!req)
+				break;
+			TAILQ_REMOVE(&nmp->nm_resendq, req, r_rchain);
+			req->r_rchain.tqe_next = NFSREQNOLIST;
+			lck_mtx_unlock(&nmp->nm_lock);
+			lck_mtx_lock(&req->r_mtx);
+			if (req->r_error || req->r_nmrep.nmc_mhead) {
+				dofinish = req->r_callback.rcb_func && !(req->r_flags & R_WAITSENT);
+				req->r_flags &= ~R_RESENDQ;
+				wakeup(req);
+				lck_mtx_unlock(&req->r_mtx);
+				if (dofinish)
+					nfs_asyncio_finish(req);
+				lck_mtx_lock(&nmp->nm_lock);
+				continue;
 			}
-		} else {
-			bzero(&msg, sizeof(msg));
-			do {
-			    rcvlen = 100000000;
-			    error = sock_receivembuf(so, &msg, mp, 0, &rcvlen);
-			    if (!rep->r_nmp) /* if unmounted then bailout */ {
-				goto shutout;
- 			    }   
-			    if (error == EWOULDBLOCK && rep) {
-				error2 = nfs_sigintr(rep->r_nmp, rep, p);
-				if (error2) {
-					return (error2);
+			if ((req->r_flags & R_RESTART) || req->r_gss_ctx) {
+				req->r_flags &= ~R_RESTART;
+				req->r_resendtime = 0;
+				lck_mtx_unlock(&req->r_mtx);
+				/* async RPCs on GSS mounts need to be rebuilt and resent. */
+				nfs_reqdequeue(req);
+				if (req->r_gss_ctx) {
+					nfs_gss_clnt_rpcdone(req);
+					error = nfs_gss_clnt_args_restore(req);
+					if (error == ENEEDAUTH)
+						req->r_xid = 0;
 				}
-			    }
-			} while (error == EWOULDBLOCK);
-
-			if ((msg.msg_flags & MSG_EOR) == 0)
-				printf("Egad!!\n");
-			if (!error && *mp == NULL)
-				error = EPIPE;
-			len = rcvlen;
-		}
-errout:
-		if (error && error != EINTR && error != ERESTART) {
-			mbuf_freem(*mp);
-			*mp = NULL;
-			if (error != EPIPE)
-				log(LOG_INFO,
-				    "receive error %d from nfs server %s\n", error,
-				    vfs_statfs(rep->r_nmp->nm_mountp)->f_mntfromname);
-			error = nfs_sndlock(rep);
-			if (!error) {
-				error = nfs_reconnect(rep);
+				NFS_SOCK_DBG(("nfs async%s restart: p %d x 0x%llx f 0x%x rtt %d\n",
+					req->r_gss_ctx ? " gss" : "", req->r_procnum, req->r_xid,
+					req->r_flags, req->r_rtt));
+				error = !req->r_nmp ? ENXIO : 0;	/* unmounted? */
 				if (!error)
-					goto tryagain;
-				nfs_sndunlock(rep);
+					error = nfs_sigintr(nmp, req, req->r_thread, 0);
+				if (!error)
+					error = nfs_request_add_header(req);
+				if (!error)
+					error = nfs_request_send(req, 0);
+				lck_mtx_lock(&req->r_mtx);
+				if (req->r_rchain.tqe_next == NFSREQNOLIST)
+					req->r_flags &= ~R_RESENDQ;
+				if (error)
+					req->r_error = error;
+				wakeup(req);
+				dofinish = error && req->r_callback.rcb_func && !(req->r_flags & R_WAITSENT);
+				lck_mtx_unlock(&req->r_mtx);
+				if (dofinish)
+					nfs_asyncio_finish(req);
+				lck_mtx_lock(&nmp->nm_lock);
+				error = 0;
+				continue;
 			}
-		}
-	} else {
-		/*
-		 * We could have failed while rebinding the datagram socket
-		 * so we need to attempt to rebind here.
-		 */
-		if ((so = rep->r_nmp->nm_so) == NULL) {
-			error = nfs_sndlock(rep);
+			NFS_SOCK_DBG(("nfs async resend: p %d x 0x%llx f 0x%x rtt %d\n",
+				req->r_procnum, req->r_xid, req->r_flags, req->r_rtt));
+			error = !req->r_nmp ? ENXIO : 0;	/* unmounted? */
+			if (!error)
+				error = nfs_sigintr(nmp, req, req->r_thread, 0);
 			if (!error) {
-				error = nfs_reconnect(rep);
-				nfs_sndunlock(rep);
-			}
-			if (error)
-				return (error);
-			if (!rep->r_nmp) /* if unmounted then bailout */
-				return (ENXIO);
-			so = rep->r_nmp->nm_so;
-		}
-		bzero(&msg, sizeof(msg));
-		len = 0;
-		do {
-			rcvlen = 1000000;
-			error = sock_receivembuf(so, &msg, mp, 0, &rcvlen);
-			if (!rep->r_nmp) /* if unmounted then bailout */
-				goto shutout;
-			if (error) {
-				error2 = nfs_sigintr(rep->r_nmp, rep, p);
-				if (error2) {
-					error = error2;
-					goto shutout;
-				}
-			}
-			/* Reconnect for all errors.  We may be receiving
-			 * soft/hard/blocking errors because of a network
-			 * change.
-			 * XXX: we should rate limit or delay this
-			 * to once every N attempts or something.
-			 * although TCP doesn't seem to.
-			 */
-			if (error) {
-				error2 = nfs_sndlock(rep);
-				if (!error2) {
-					error2 = nfs_reconnect(rep);
-					if (error2)
-						error = error2;
-					else if (!rep->r_nmp) /* if unmounted then bailout */
-						error = ENXIO;
-					else
-						so = rep->r_nmp->nm_so;
-					nfs_sndunlock(rep);
-				} else {
-					error = error2;
+				lck_mtx_unlock(&req->r_mtx);
+				error = nfs_send(req, 0);
+				lck_mtx_lock(&req->r_mtx);
+				if (!error) {
+					if (req->r_rchain.tqe_next == NFSREQNOLIST)
+						req->r_flags &= ~R_RESENDQ;
+					wakeup(req);
+					lck_mtx_unlock(&req->r_mtx);
+					lck_mtx_lock(&nmp->nm_lock);
+					continue;
 				}
 			}
-		} while (error == EWOULDBLOCK);
-	}
-shutout:
-	if (error) {
-		mbuf_freem(*mp);
-		*mp = NULL;
+			req->r_error = error;
+			if (req->r_rchain.tqe_next == NFSREQNOLIST)
+				req->r_flags &= ~R_RESENDQ;
+			wakeup(req);
+			dofinish = req->r_callback.rcb_func && !(req->r_flags & R_WAITSENT);
+			lck_mtx_unlock(&req->r_mtx);
+			if (dofinish)
+				nfs_asyncio_finish(req);
+			lck_mtx_lock(&nmp->nm_lock);
+		}
+		if (nmp->nm_sockflags & NMSOCK_READY) {
+			ts.tv_sec = TAILQ_EMPTY(&nmp->nm_resendq) ? 30 : 1;
+			msleep(&nmp->nm_sockthd, &nmp->nm_lock, PSOCK, "nfssockthread", &ts);
+		} else if (force)
+			break;
 	}
-	return (error);
+
+	if (nmp->nm_sockthd == thd)
+		nmp->nm_sockthd = NULL;
+	lck_mtx_unlock(&nmp->nm_lock);
+	wakeup(&nmp->nm_sockthd);
+	thread_terminate(thd);
+}
+
+/* start or wake a mount's socket thread */
+void
+nfs_mount_sock_thread_wake(struct nfsmount *nmp)
+{
+	if (nmp->nm_sockthd)
+		wakeup(&nmp->nm_sockthd);
+	else if (kernel_thread_start(nfs_mount_sock_thread, nmp, &nmp->nm_sockthd) == KERN_SUCCESS)
+		thread_deallocate(nmp->nm_sockthd);
 }
 
 /*
- * Implement receipt of reply on a socket.
- * We must search through the list of received datagrams matching them
- * with outstanding requests using the xid, until ours is found.
+ * The NFS client send routine.
+ *
+ * Send the given NFS request out the mount's socket.
+ * Holds nfs_sndlock() for the duration of this call.
+ *
+ * - check for request termination (sigintr)
+ * - perform reconnect, if necessary
+ * - UDP: check the congestion window
+ * - make a copy of the request to send
+ * - UDP: update the congestion window
+ * - send the request
+ *
+ * If sent successfully, R_MUSTRESEND and R_RESENDERR are cleared.
+ * rexmit count is also updated if this isn't the first send.
+ *
+ * If the send is not successful, make sure R_MUSTRESEND is set.
+ * If this wasn't the first transmit, set R_RESENDERR.
+ * Also, undo any UDP congestion window changes made.
+ *
+ * If the error appears to indicate that the socket should
+ * be reconnected, mark the socket for reconnection.
+ *
+ * Only return errors when the request should be aborted.
  */
-/* ARGSUSED */
 int
-nfs_reply(myrep)
-	struct nfsreq *myrep;
+nfs_send(struct nfsreq *req, int wait)
 {
-	struct nfsreq *rep;
-	struct nfsmount *nmp = myrep->r_nmp;
-	long t1;
-	mbuf_t mrep, md;
-	u_long rxid, *tl;
-	caddr_t dpos, cp2;
-	int error;
+	struct nfsmount *nmp;
+	socket_t so;
+	int error, error2, sotype, rexmit, slpflag = PSOCK, needrecon;
+	struct msghdr msg;
+	struct sockaddr *sendnam;
+	mbuf_t mreqcopy;
+	size_t sentlen = 0;
+	struct timespec ts = { 2, 0 };
 
-	/*
-	 * Loop around until we get our own reply
-	 */
-	for (;;) {
-		/*
-		 * Lock against other receivers so that I don't get stuck in
-		 * sbwait() after someone else has received my reply for me.
-		 * Also necessary for connection based protocols to avoid
-		 * race conditions during a reconnect.
-		 * If nfs_rcvlock() returns EALREADY, that means that
-		 * the reply has already been recieved by another
-		 * process and we can return immediately.  In this
-		 * case, the lock is not taken to avoid races with
-		 * other processes.
-		 */
-		error = nfs_rcvlock(myrep);
-		if (error == EALREADY)
+again:
+	error = nfs_sndlock(req);
+	if (error)
+		return (error);
+
+	error = nfs_sigintr(req->r_nmp, req, req->r_thread, 0);
+	if (error) {
+		nfs_sndunlock(req);
+		return (error);
+	}
+	nmp = req->r_nmp;
+	sotype = nmp->nm_sotype;
+
+	if ((req->r_flags & R_SETUP) && !(nmp->nm_sockflags & NMSOCK_SETUP)) {
+		/* a setup RPC but we're not in SETUP... must need reconnect */
+		nfs_sndunlock(req);
+		return (EPIPE);
+	}
+
+	/* If the socket needs reconnection, do that now. */
+	/* wait until socket is ready - unless this request is part of setup */
+	lck_mtx_lock(&nmp->nm_lock);
+	if (!(nmp->nm_sockflags & NMSOCK_READY) &&
+	    !((nmp->nm_sockflags & NMSOCK_SETUP) && (req->r_flags & R_SETUP))) {
+		if (nmp->nm_flag & NFSMNT_INT)
+			slpflag |= PCATCH;
+		lck_mtx_unlock(&nmp->nm_lock);
+		nfs_sndunlock(req);
+		if (!wait) {
+			lck_mtx_lock(&req->r_mtx);
+			req->r_flags |= R_MUSTRESEND;
+			req->r_rtt = 0;
+			lck_mtx_unlock(&req->r_mtx);
 			return (0);
+		}
+		NFS_SOCK_DBG(("nfs_send: 0x%llx wait reconnect\n", req->r_xid));
+		lck_mtx_lock(&req->r_mtx);
+		req->r_flags &= ~R_MUSTRESEND;
+		req->r_rtt = 0;
+		lck_mtx_unlock(&req->r_mtx);
+		lck_mtx_lock(&nmp->nm_lock);
+		while (!(nmp->nm_sockflags & NMSOCK_READY)) {
+			/* don't bother waiting if the socket thread won't be reconnecting it */
+			if (nmp->nm_state & NFSSTA_FORCE) {
+				error = EIO;
+				break;
+			}
+			/* make sure socket thread is running, then wait */
+			nfs_mount_sock_thread_wake(nmp);
+			if ((error = nfs_sigintr(req->r_nmp, req, req->r_thread, 1)))
+				break;
+			error = msleep(req, &nmp->nm_lock, slpflag, "nfsconnectwait", &ts);
+			if (error == EWOULDBLOCK)
+				error = 0;
+			if ((error == EINTR) || (error == ERESTART))
+				break;
+		}
+		lck_mtx_unlock(&nmp->nm_lock);
 		if (error)
 			return (error);
-		
+		goto again;
+	}
+	so = nmp->nm_so;
+	lck_mtx_unlock(&nmp->nm_lock);
+	if (!so) {
+		nfs_sndunlock(req);
+		lck_mtx_lock(&req->r_mtx);
+		req->r_flags |= R_MUSTRESEND;
+		req->r_rtt = 0;
+		lck_mtx_unlock(&req->r_mtx);
+		return (0);
+	}
+
+	lck_mtx_lock(&req->r_mtx);
+	rexmit = (req->r_flags & R_SENT);
+
+	if (sotype == SOCK_DGRAM) {
+		lck_mtx_lock(&nmp->nm_lock);
+		if (!(req->r_flags & R_CWND) && (nmp->nm_sent >= nmp->nm_cwnd)) {
+			/* if we can't send this out yet, wait on the cwnd queue */
+			slpflag = ((nmp->nm_flag & NFSMNT_INT) && req->r_thread) ? PCATCH : 0;
+			lck_mtx_unlock(&nmp->nm_lock);
+			nfs_sndunlock(req);
+			req->r_flags |= R_MUSTRESEND;
+			lck_mtx_unlock(&req->r_mtx);
+			if (!wait) {
+				req->r_rtt = 0;
+				return (0);
+			}
+			lck_mtx_lock(&nmp->nm_lock);
+			while (nmp->nm_sent >= nmp->nm_cwnd) {
+				if ((error = nfs_sigintr(req->r_nmp, req, req->r_thread, 1)))
+					break;
+				TAILQ_INSERT_TAIL(&nmp->nm_cwndq, req, r_cchain);
+				error = msleep(req, &nmp->nm_lock, slpflag | (PZERO - 1), "nfswaitcwnd", &ts);
+				if ((req->r_cchain.tqe_next != NFSREQNOLIST)) {
+					TAILQ_REMOVE(&nmp->nm_cwndq, req, r_cchain);
+					req->r_cchain.tqe_next = NFSREQNOLIST;
+				}
+				if ((error == EINTR) || (error == ERESTART))
+					break;
+			}
+			lck_mtx_unlock(&nmp->nm_lock);
+			if ((error == EINTR) || (error == ERESTART))
+				return (error);
+			goto again;
+		}
 		/*
-		 * If we slept after putting bits otw, then reply may have
-		 * arrived.  In which case returning is required, or we
-		 * would hang trying to nfs_receive an already received reply.
+		 * We update these *before* the send to avoid racing
+		 * against others who may be looking to send requests.
 		 */
-		if (myrep->r_mrep != NULL) {
-			nfs_rcvunlock(myrep);
-			FSDBG(530, myrep->r_xid, myrep, myrep->r_nmp, -1);
-			return (0);
+		if (!rexmit) {
+			/* first transmit */
+			req->r_flags |= R_CWND;
+			nmp->nm_sent += NFS_CWNDSCALE;
+		} else {
+			/*
+			 * When retransmitting, turn timing off
+			 * and divide congestion window by 2. 
+			 */
+			req->r_flags &= ~R_TIMING;
+			nmp->nm_cwnd >>= 1;
+			if (nmp->nm_cwnd < NFS_CWNDSCALE)
+				nmp->nm_cwnd = NFS_CWNDSCALE;
 		}
+		lck_mtx_unlock(&nmp->nm_lock);
+	}
+
+	req->r_flags &= ~R_MUSTRESEND;
+	lck_mtx_unlock(&req->r_mtx);
+
+	error = mbuf_copym(req->r_mhead, 0, MBUF_COPYALL,
+			wait ? MBUF_WAITOK : MBUF_DONTWAIT, &mreqcopy);
+	if (error) {
+		if (wait)
+			log(LOG_INFO, "nfs_send: mbuf copy failed %d\n", error);
+		nfs_sndunlock(req);
+		lck_mtx_lock(&req->r_mtx);
+		req->r_flags |= R_MUSTRESEND;
+		req->r_rtt = 0;
+		lck_mtx_unlock(&req->r_mtx);
+		return (0);
+	}
+
+	bzero(&msg, sizeof(msg));
+	if (nmp->nm_nam && (sotype != SOCK_STREAM) && !sock_isconnected(so)) {
+		if ((sendnam = mbuf_data(nmp->nm_nam))) {
+			msg.msg_name = (caddr_t)sendnam;
+			msg.msg_namelen = sendnam->sa_len;
+		}
+	}
+	error = sock_sendmbuf(so, &msg, mreqcopy, 0, &sentlen);
+#ifdef NFS_SOCKET_DEBUGGING
+	if (error || (sentlen != req->r_mreqlen))
+		NFS_SOCK_DBG(("nfs_send: 0x%llx sent %d/%d error %d\n",
+			req->r_xid, (int)sentlen, (int)req->r_mreqlen, error));
+#endif
+	if (!error && (sentlen != req->r_mreqlen))
+		error = EWOULDBLOCK;
+	needrecon = ((sotype == SOCK_STREAM) && sentlen && (sentlen != req->r_mreqlen));
+
+	lck_mtx_lock(&req->r_mtx);
+	req->r_rtt = 0;
+	if (rexmit && (++req->r_rexmit > NFS_MAXREXMIT))
+		req->r_rexmit = NFS_MAXREXMIT;
+
+	if (!error) {
+		/* SUCCESS */
+		req->r_flags &= ~R_RESENDERR;
+		if (rexmit)
+			OSAddAtomic(1, (SInt32*)&nfsstats.rpcretries);
+		req->r_flags |= R_SENT;
+		if (req->r_flags & R_WAITSENT) {
+			req->r_flags &= ~R_WAITSENT;
+			wakeup(req);
+		}
+		nfs_sndunlock(req);
+		lck_mtx_unlock(&req->r_mtx);
+		return (0);
+	}
+
+	/* send failed */
+	req->r_flags |= R_MUSTRESEND;
+	if (rexmit)
+		req->r_flags |= R_RESENDERR;
+	if ((error == EINTR) || (error == ERESTART))
+		req->r_error = error;
+	lck_mtx_unlock(&req->r_mtx);
+
+	if (sotype == SOCK_DGRAM) {
 		/*
-		 * Get the next Rpc reply off the socket. Assume myrep->r_nmp
-		 * is still intact by checks done in nfs_rcvlock.
+		 * Note: even though a first send may fail, we consider
+		 * the request sent for congestion window purposes.
+		 * So we don't need to undo any of the changes made above.
 		 */
-		error = nfs_receive(myrep, &mrep);
 		/*
-		 * Bailout asap if nfsmount struct gone (unmounted). 
+		 * Socket errors ignored for connectionless sockets??
+		 * For now, ignore them all
 		 */
-		if (!myrep->r_nmp) {
-			FSDBG(530, myrep->r_xid, myrep, nmp, -2);
-			if (mrep)
-				mbuf_freem(mrep);
-			return (ENXIO);
+		if ((error != EINTR) && (error != ERESTART) &&
+		    (error != EWOULDBLOCK) && (error != EIO)) {
+			int clearerror = 0, optlen = sizeof(clearerror);
+			sock_getsockopt(so, SOL_SOCKET, SO_ERROR, &clearerror, &optlen);
+#ifdef NFS_SOCKET_DEBUGGING
+			if (clearerror)
+				NFS_SOCK_DBG(("nfs_send: ignoring UDP socket error %d so %d\n",
+					error, clearerror));
+#endif
 		}
-		if (error) {
-			FSDBG(530, myrep->r_xid, myrep, nmp, error);
-			nfs_rcvunlock(myrep);
-
-			/* Bailout asap if nfsmount struct gone (unmounted). */
-			if (!myrep->r_nmp) {
-				if (mrep)
-					mbuf_freem(mrep);
-				return (ENXIO);
-			}
+	}
+
+	/* check if it appears we should reconnect the socket */
+	switch (error) {
+	case EWOULDBLOCK:
+		/* if send timed out, reconnect if on TCP */
+		if (sotype != SOCK_STREAM)
+			break;
+	case EPIPE:
+	case EADDRNOTAVAIL:
+	case ENETDOWN:
+	case ENETUNREACH:
+	case ENETRESET:
+	case ECONNABORTED:
+	case ECONNRESET:
+	case ENOTCONN:
+	case ESHUTDOWN:
+	case ECONNREFUSED:
+	case EHOSTDOWN:
+	case EHOSTUNREACH:
+		needrecon = 1;
+		break;
+	}
+	if (needrecon) { /* mark socket as needing reconnect */
+		NFS_SOCK_DBG(("nfs_send: 0x%llx need reconnect %d\n", req->r_xid, error));
+		nfs_need_reconnect(nmp);
+	}
+
+	nfs_sndunlock(req);
+
+	/*
+	 * Don't log some errors:
+	 * EPIPE errors may be common with servers that drop idle connections.
+	 * EADDRNOTAVAIL may occur on network transitions.
+	 * ENOTCONN may occur under some network conditions.
+	 */
+	if ((error == EPIPE) || (error == EADDRNOTAVAIL) || (error == ENOTCONN))
+		error = 0;
+	if (error && (error != EINTR) && (error != ERESTART))
+		log(LOG_INFO, "nfs send error %d for server %s\n", error,
+			!req->r_nmp ? "<unmounted>" :
+			vfs_statfs(req->r_nmp->nm_mountp)->f_mntfromname);
+
+	/* prefer request termination error over other errors */
+	error2 = nfs_sigintr(req->r_nmp, req, req->r_thread, 0);
+	if (error2)
+		error = error2;
+
+	/* only allow the following errors to be returned */
+	if ((error != EINTR) && (error != ERESTART) && (error != EIO) &&
+	    (error != ENXIO) && (error != ETIMEDOUT))
+		error = 0;
+	return (error);
+}
+
+/*
+ * NFS client socket upcalls
+ *
+ * Pull RPC replies out of an NFS mount's socket and match them
+ * up with the pending request.
+ *
+ * The datagram code is simple because we always get whole
+ * messages out of the socket.
+ *
+ * The stream code is more involved because we have to parse
+ * the RPC records out of the stream.
+ */
+
+/* NFS client UDP socket upcall */
+static void
+nfs_udp_rcv(socket_t so, void *arg, __unused int waitflag)
+{
+	struct nfsmount *nmp = arg;
+	size_t rcvlen;
+	mbuf_t m;
+	int error = 0;
+
+	if (nmp->nm_sockflags & NMSOCK_CONNECTING) {
+		wakeup(&nmp->nm_so);
+		return;
+	}
+
+	/* make sure we're on the current socket */
+	if (nmp->nm_so != so)
+		return;
+
+	do {
+		m = NULL;
+		rcvlen = 1000000;
+		error = sock_receivembuf(so, NULL, &m, MSG_DONTWAIT, &rcvlen);
+		if (m)
+			nfs_request_match_reply(nmp, m);
+	} while (m && !error);
+
+	if (error && (error != EWOULDBLOCK)) {
+		/* problems with the socket... mark for reconnection */
+		NFS_SOCK_DBG(("nfs_udp_rcv: need reconnect %d\n", error));
+		nfs_need_reconnect(nmp);
+	}
+}
+
+/* NFS client TCP socket upcall */
+static void
+nfs_tcp_rcv(socket_t so, void *arg, __unused int waitflag)
+{
+	struct nfsmount *nmp = arg;
+	struct iovec_32 aio;
+	struct msghdr msg;
+	size_t rcvlen;
+	mbuf_t m;
+	int error = 0;
+	int recv;
 
+	if (nmp->nm_sockflags & NMSOCK_CONNECTING) {
+		wakeup(&nmp->nm_so);
+		return;
+	}
+
+	/* make sure we're on the current socket */
+	if (nmp->nm_so != so)
+		return;
+
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_sockflags & NMSOCK_UPCALL) {
+		/* upcall is already receiving data - just return */
+		lck_mtx_unlock(&nmp->nm_lock);
+		return;
+	}
+	nmp->nm_sockflags |= NMSOCK_UPCALL;
+
+nextfrag:
+	recv = 0;
+
+	/* read the TCP RPC record marker */
+	while (!error && nmp->nm_markerleft) {
+		aio.iov_base = (uintptr_t)((char*)&nmp->nm_fragleft +
+				sizeof(nmp->nm_fragleft) - nmp->nm_markerleft);
+		aio.iov_len = nmp->nm_markerleft;
+		bzero(&msg, sizeof(msg));
+		msg.msg_iov = (struct iovec *) &aio;
+		msg.msg_iovlen = 1;
+		lck_mtx_unlock(&nmp->nm_lock);
+		error = sock_receive(so, &msg, MSG_DONTWAIT, &rcvlen);
+		lck_mtx_lock(&nmp->nm_lock);
+		if (error || !rcvlen)
+			break;
+		recv = 1;
+		nmp->nm_markerleft -= rcvlen;
+		if (nmp->nm_markerleft)
+			continue;
+		/* record marker complete */
+		nmp->nm_fragleft = ntohl(nmp->nm_fragleft);
+		if (nmp->nm_fragleft & 0x80000000) {
+			nmp->nm_sockflags |= NMSOCK_LASTFRAG;
+			nmp->nm_fragleft &= ~0x80000000;
+		}
+		nmp->nm_reclen += nmp->nm_fragleft;
+		if (nmp->nm_reclen > NFS_MAXPACKET) {
 			/*
-			 * Ignore routing errors on connectionless protocols??
+			 * This is SERIOUS! We are out of sync with the sender
+			 * and forcing a disconnect/reconnect is all I can do.
 			 */
-			if (NFSIGNORE_SOERROR(nmp->nm_sotype, error)) {
-				if (nmp->nm_so) {
-					int clearerror;
-					int optlen = sizeof(clearerror);
-					sock_getsockopt(nmp->nm_so, SOL_SOCKET, SO_ERROR, &clearerror, &optlen);
-				}
-				continue;
+			log(LOG_ERR, "%s (%d) from nfs server %s\n",
+				"impossible RPC record length", nmp->nm_reclen,
+				vfs_statfs(nmp->nm_mountp)->f_mntfromname);
+			error = EFBIG;
+		}
+	}
+
+	/* read the TCP RPC record fragment */
+	while (!error && !nmp->nm_markerleft && nmp->nm_fragleft) {
+		m = NULL;
+		rcvlen = nmp->nm_fragleft;
+		lck_mtx_unlock(&nmp->nm_lock);
+		error = sock_receivembuf(so, NULL, &m, MSG_DONTWAIT, &rcvlen);
+		lck_mtx_lock(&nmp->nm_lock);
+		if (error || !rcvlen || !m)
+			break;
+		recv = 1;
+		/* append mbufs to list */
+		nmp->nm_fragleft -= rcvlen;
+		if (!nmp->nm_m) {
+			nmp->nm_m = m;
+		} else {
+			error = mbuf_setnext(nmp->nm_mlast, m);
+			if (error) {
+				printf("nfs_tcp_rcv: mbuf_setnext failed %d\n", error);
+				mbuf_freem(m);
+				break;
 			}
-			if (mrep)
-				mbuf_freem(mrep);
-			return (error);
 		}
+		while (mbuf_next(m))
+			m = mbuf_next(m);
+		nmp->nm_mlast = m;
+	}
+
+	/* done reading fragment? */
+	m = NULL;
+	if (!error && !nmp->nm_markerleft && !nmp->nm_fragleft) {
+		/* reset socket fragment parsing state */
+		nmp->nm_markerleft = sizeof(nmp->nm_fragleft);
+		if (nmp->nm_sockflags & NMSOCK_LASTFRAG) {
+			/* RPC record complete */
+			m = nmp->nm_m;
+			/* reset socket record parsing state */
+			nmp->nm_reclen = 0;
+			nmp->nm_m = nmp->nm_mlast = NULL;
+			nmp->nm_sockflags &= ~NMSOCK_LASTFRAG;
+		}
+	}
 
-		/*
-		 * We assume all is fine, but if we did not have an error
-                 * and mrep is 0, better not dereference it. nfs_receive
-                 * calls soreceive which carefully sets error=0 when it got
-                 * errors on sbwait (tsleep). In most cases, I assume that's 
-                 * so we could go back again. In tcp case, EPIPE is returned.
-                 * In udp, case nfs_receive gets back here with no error and no
-                 * mrep. Is the right fix to have soreceive check for process
-                 * aborted after sbwait and return something non-zero? Should
-                 * nfs_receive give an EPIPE?  Too risky to play with those
-                 * two this late in game for a shutdown problem. Instead,
-                 * just check here and get out. (ekn)
-		 */
-		if (!mrep) {
-			nfs_rcvunlock(myrep);
-                        FSDBG(530, myrep->r_xid, myrep, nmp, -3);
-                        return (ENXIO); /* sounds good */
-                }
-                        
-		/*
-		 * Get the xid and check that it is an rpc reply
-		 */
-		md = mrep;
-		dpos = mbuf_data(md);
-		nfsm_dissect(tl, u_long *, 2*NFSX_UNSIGNED);
-		rxid = *tl++;
-		if (*tl != rpc_reply) {
-			OSAddAtomic(1, (SInt32*)&nfsstats.rpcinvalid);
-			mbuf_freem(mrep);
-nfsmout:
-			if (nmp->nm_state & NFSSTA_RCVLOCK)
-				nfs_rcvunlock(myrep);
+	if (m) { /* match completed response with request */
+		lck_mtx_unlock(&nmp->nm_lock);
+		nfs_request_match_reply(nmp, m);
+		lck_mtx_lock(&nmp->nm_lock);
+	}
+
+	/* loop if we've been making error-free progress */
+	if (!error && recv)
+		goto nextfrag;
+
+	nmp->nm_sockflags &= ~NMSOCK_UPCALL;
+	lck_mtx_unlock(&nmp->nm_lock);
+#ifdef NFS_SOCKET_DEBUGGING
+	if (!recv && (error != EWOULDBLOCK))
+		NFS_SOCK_DBG(("nfs_tcp_rcv: got nothing, error %d, got FIN?\n", error));
+#endif
+	/* note: no error and no data indicates server closed its end */
+	if ((error != EWOULDBLOCK) && (error || !recv)) {
+		/* problems with the socket... mark for reconnection */
+		NFS_SOCK_DBG(("nfs_tcp_rcv: need reconnect %d\n", error));
+		nfs_need_reconnect(nmp);
+	}
+}
+
+/*
+ * "poke" a socket to try to provoke any pending errors
+ */
+static void
+nfs_sock_poke(struct nfsmount *nmp)
+{
+	struct iovec_32 aio;
+	struct msghdr msg;
+	size_t len;
+	int error = 0;
+	int dummy;
+
+	lck_mtx_lock(&nmp->nm_lock);
+	if ((nmp->nm_sockflags & NMSOCK_UNMOUNT) || !nmp->nm_so) {
+		lck_mtx_unlock(&nmp->nm_lock);
+		return;
+	}
+	lck_mtx_unlock(&nmp->nm_lock);
+	aio.iov_base = (uintptr_t)&dummy;
+	aio.iov_len = 0;
+	len = 0;
+	bzero(&msg, sizeof(msg));
+	msg.msg_iov = (struct iovec *) &aio;
+	msg.msg_iovlen = 1;
+	error = sock_send(nmp->nm_so, &msg, MSG_DONTWAIT, &len);
+	NFS_SOCK_DBG(("nfs_sock_poke: error %d\n", error));
+}
+
+/*
+ * Match an RPC reply with the corresponding request
+ */
+static void
+nfs_request_match_reply(struct nfsmount *nmp, mbuf_t mrep)
+{
+	struct nfsreq *req;
+	struct nfsm_chain nmrep;
+	u_long reply = 0, rxid = 0;
+	long t1;
+	int error = 0, asyncioq, asyncgss;
+
+	/* Get the xid and check that it is an rpc reply */
+	nfsm_chain_dissect_init(error, &nmrep, mrep);
+	nfsm_chain_get_32(error, &nmrep, rxid);
+	nfsm_chain_get_32(error, &nmrep, reply);
+	if (error || (reply != RPC_REPLY)) {
+		OSAddAtomic(1, (SInt32*)&nfsstats.rpcinvalid);
+		mbuf_freem(mrep);
+		return;
+	}
+
+	/*
+	 * Loop through the request list to match up the reply
+	 * Iff no match, just drop it.
+	 */
+	lck_mtx_lock(nfs_request_mutex);
+	TAILQ_FOREACH(req, &nfs_reqq, r_chain) {
+		if (req->r_nmrep.nmc_mhead || (rxid != R_XID32(req->r_xid)))
+			continue;
+		/* looks like we have it, grab lock and double check */
+		lck_mtx_lock(&req->r_mtx);
+		if (req->r_nmrep.nmc_mhead || (rxid != R_XID32(req->r_xid))) {
+			lck_mtx_unlock(&req->r_mtx);
 			continue;
 		}
-
+		/* Found it.. */
+		req->r_nmrep = nmrep;
+		lck_mtx_lock(&nmp->nm_lock);
+		if (nmp->nm_sotype == SOCK_DGRAM) {
+			/*
+			 * Update congestion window.
+			 * Do the additive increase of one rpc/rtt.
+			 */
+			FSDBG(530, R_XID32(req->r_xid), req, nmp->nm_sent, nmp->nm_cwnd);
+			if (nmp->nm_cwnd <= nmp->nm_sent) {
+				nmp->nm_cwnd +=
+				   ((NFS_CWNDSCALE * NFS_CWNDSCALE) +
+				    (nmp->nm_cwnd >> 1)) / nmp->nm_cwnd;
+				if (nmp->nm_cwnd > NFS_MAXCWND)
+					nmp->nm_cwnd = NFS_MAXCWND;
+			}
+			if (req->r_flags & R_CWND) {
+				nmp->nm_sent -= NFS_CWNDSCALE;
+				req->r_flags &= ~R_CWND;
+			}
+			if ((nmp->nm_sent < nmp->nm_cwnd) && !TAILQ_EMPTY(&nmp->nm_cwndq)) {
+				/* congestion window is open, poke the cwnd queue */
+				struct nfsreq *req2 = TAILQ_FIRST(&nmp->nm_cwndq);
+				TAILQ_REMOVE(&nmp->nm_cwndq, req2, r_cchain);
+				req2->r_cchain.tqe_next = NFSREQNOLIST;
+				wakeup(req2);
+			}
+		}
 		/*
-		 * Loop through the request list to match up the reply
-		 * Iff no match, just drop the datagram
+		 * Update rtt using a gain of 0.125 on the mean
+		 * and a gain of 0.25 on the deviation.
 		 */
-		TAILQ_FOREACH(rep, &nfs_reqq, r_chain) {
-			if (rep->r_mrep == NULL && rxid == rep->r_xid) {
-				/* Found it.. */
-				rep->r_mrep = mrep;
-				rep->r_md = md;
-				rep->r_dpos = dpos;
-				/*
-				 * If we're tracking the round trip time
-				 * then we update the circular log here
-				 * with the stats from our current request.
-				 */
-				if (nfsrtton) {
-					struct rttl *rt;
-
-					rt = &nfsrtt.rttl[nfsrtt.pos];
-					rt->proc = rep->r_procnum;
-					rt->rto = NFS_RTO(nmp, proct[rep->r_procnum]);
-					rt->sent = nmp->nm_sent;
-					rt->cwnd = nmp->nm_cwnd;
-					if (proct[rep->r_procnum] == 0)
-						panic("nfs_reply: proct[%d] is zero", rep->r_procnum);
-					rt->srtt = nmp->nm_srtt[proct[rep->r_procnum] - 1];
-					rt->sdrtt = nmp->nm_sdrtt[proct[rep->r_procnum] - 1];
-					rt->fsid = vfs_statfs(nmp->nm_mountp)->f_fsid;
-					microtime(&rt->tstamp); // XXX unused
-					if (rep->r_flags & R_TIMING)
-						rt->rtt = rep->r_rtt;
-					else
-						rt->rtt = 1000000;
-					nfsrtt.pos = (nfsrtt.pos + 1) % NFSRTTLOGSIZ;
-				}
-				/*
-				 * Update congestion window.
-				 * Do the additive increase of
-				 * one rpc/rtt.
-				 */
-				FSDBG(530, rep->r_xid, rep, nmp->nm_sent,
-				      nmp->nm_cwnd);
-				if (nmp->nm_cwnd <= nmp->nm_sent) {
-					nmp->nm_cwnd +=
-					   (NFS_CWNDSCALE * NFS_CWNDSCALE +
-					   (nmp->nm_cwnd >> 1)) / nmp->nm_cwnd;
-					if (nmp->nm_cwnd > NFS_MAXCWND)
-						nmp->nm_cwnd = NFS_MAXCWND;
-				}
-                                if (rep->r_flags & R_SENT) {
-                                    rep->r_flags &= ~R_SENT;
-                                    nmp->nm_sent -= NFS_CWNDSCALE;
-                               }
+		if (req->r_flags & R_TIMING) {
+			/*
+			 * Since the timer resolution of
+			 * NFS_HZ is so course, it can often
+			 * result in r_rtt == 0. Since
+			 * r_rtt == N means that the actual
+			 * rtt is between N+dt and N+2-dt ticks,
+			 * add 1.
+			 */
+			if (proct[req->r_procnum] == 0)
+				panic("nfs_request_match_reply: proct[%d] is zero", req->r_procnum);
+			t1 = req->r_rtt + 1;
+			t1 -= (NFS_SRTT(req) >> 3);
+			NFS_SRTT(req) += t1;
+			if (t1 < 0)
+				t1 = -t1;
+			t1 -= (NFS_SDRTT(req) >> 2);
+			NFS_SDRTT(req) += t1;
+		}
+		nmp->nm_timeouts = 0;
+		lck_mtx_unlock(&nmp->nm_lock);
+		/* signal anyone waiting on this request */
+		wakeup(req);
+		asyncioq = (req->r_callback.rcb_func != NULL);
+		if ((asyncgss = ((req->r_gss_ctx != NULL) && ((req->r_flags & (R_ASYNC|R_ASYNCWAIT|R_ALLOCATED)) == (R_ASYNC|R_ALLOCATED)))))
+			nfs_request_ref(req, 1);
+		lck_mtx_unlock(&req->r_mtx);
+		lck_mtx_unlock(nfs_request_mutex);
+		if (asyncgss) {
+			nfs_gss_clnt_rpcdone(req);
+			nfs_request_rele(req);
+		}
+		/* if it's an async RPC with a callback, queue it up */
+		if (asyncioq)
+			nfs_asyncio_finish(req);
+		break;
+	}
+
+	if (!req) {
+		/* not matched to a request, so drop it. */
+		lck_mtx_unlock(nfs_request_mutex);
+		OSAddAtomic(1, (SInt32*)&nfsstats.rpcunexpected);
+		mbuf_freem(mrep);
+	}
+}
+
+/*
+ * Wait for the reply for a given request...
+ * ...potentially resending the request if necessary.
+ */
+static int
+nfs_wait_reply(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_nmp;
+	struct timespec ts = { 30, 0 };
+	int error = 0, slpflag;
+
+	if ((nmp->nm_flag & NFSMNT_INT) && req->r_thread)
+		slpflag = PCATCH;
+	else
+		slpflag = 0;
+
+	lck_mtx_lock(&req->r_mtx);
+	while (!req->r_nmrep.nmc_mhead) {
+		if ((error = nfs_sigintr(nmp, req, req->r_thread, 0)))
+			break;
+		if (((error = req->r_error)) || req->r_nmrep.nmc_mhead)
+			break;
+		/* check if we need to resend */
+		if (req->r_flags & R_MUSTRESEND) {
+			NFS_SOCK_DBG(("nfs wait resend: p %d x 0x%llx f 0x%x rtt %d\n",
+				req->r_procnum, req->r_xid, req->r_flags, req->r_rtt));
+			lck_mtx_unlock(&req->r_mtx);
+			if (req->r_gss_ctx) {
 				/*
-				 * Update rtt using a gain of 0.125 on the mean
-				 * and a gain of 0.25 on the deviation.
+				 * It's an RPCSEC_GSS mount.
+				 * Can't just resend the original request
+				 * without bumping the cred sequence number.
+				 * Go back and re-build the request.
 				 */
-				if (rep->r_flags & R_TIMING) {
-					/*
-					 * Since the timer resolution of
-					 * NFS_HZ is so course, it can often
-					 * result in r_rtt == 0. Since
-					 * r_rtt == N means that the actual
-					 * rtt is between N+dt and N+2-dt ticks,
-					 * add 1.
-					 */
-					if (proct[rep->r_procnum] == 0)
-						panic("nfs_reply: proct[%d] is zero", rep->r_procnum);
-					t1 = rep->r_rtt + 1;
-					t1 -= (NFS_SRTT(rep) >> 3);
-					NFS_SRTT(rep) += t1;
-					if (t1 < 0)
-						t1 = -t1;
-					t1 -= (NFS_SDRTT(rep) >> 2);
-					NFS_SDRTT(rep) += t1;
-				}
-				nmp->nm_timeouts = 0;
-				break;
+				return (EAGAIN);
 			}
+			error = nfs_send(req, 1);
+			lck_mtx_lock(&req->r_mtx);
+			NFS_SOCK_DBG(("nfs wait resend: p %d x 0x%llx f 0x%x rtt %d err %d\n",
+				req->r_procnum, req->r_xid, req->r_flags, req->r_rtt, error));
+			if (error)
+				break;
+			if (((error = req->r_error)) || req->r_nmrep.nmc_mhead)
+				break;
 		}
-		nfs_rcvunlock(myrep);
-		/*
-		 * If not matched to a request, drop it.
-		 * If it's mine, get out.
-		 */
-		if (rep == 0) {
-			OSAddAtomic(1, (SInt32*)&nfsstats.rpcunexpected);
-			mbuf_freem(mrep);
-		} else if (rep == myrep) {
-			if (rep->r_mrep == NULL)
-				panic("nfs_reply: nil r_mrep");
-			return (0);
-		}
-		FSDBG(530, myrep->r_xid, myrep, rep,
-		      rep ? rep->r_xid : myrep->r_flags);
+		/* need to poll if we're P_NOREMOTEHANG */
+		if (nfs_noremotehang(req->r_thread))
+			ts.tv_sec = 1;
+		error = msleep(req, &req->r_mtx, slpflag | (PZERO - 1), "nfswaitreply", &ts);
+		if (error == EWOULDBLOCK)
+			error = 0;
+		if ((error == EINTR) || (error == ERESTART))
+			break;
 	}
+	lck_mtx_unlock(&req->r_mtx);
+
+	return (error);
 }
 
 /*
- * nfs_request - goes something like this
- *	- fill in request struct
- *	- links it into list
- *	- calls nfs_send() for first transmit
- *	- calls nfs_receive() to get reply
- *	- break down rpc header and return with nfs reply pointed to
- *	  by mrep or error
- * nb: always frees up mreq mbuf list
+ * An NFS request goes something like this:
+ * (nb: always frees up mreq mbuf list)
+ * nfs_request_create()
+ *	- allocates a request struct if one is not provided
+ *	- initial fill-in of the request struct
+ * nfs_request_add_header()
+ *	- add the RPC header
+ * nfs_request_send()
+ *	- link it into list
+ *	- call nfs_send() for first transmit
+ * nfs_request_wait()
+ *	- call nfs_wait_reply() to wait for the reply
+ * nfs_request_finish()
+ *	- break down rpc header and return with error or nfs reply
+ *	  pointed to by nmrep.
+ * nfs_request_rele()
+ * nfs_request_destroy()
+ *      - clean up the request struct
+ *      - free the request struct if it was allocated by nfs_request_create()
+ */
+
+/*
+ * Set up an NFS request struct (allocating if no request passed in).
  */
 int
-nfs_request(vp, mp, mrest, procnum, procp, cred, mrp, mdp, dposp, xidp)
-	vnode_t vp;
-	mount_t mp;
-	mbuf_t mrest;
-	int procnum;
-	proc_t procp;
-	kauth_cred_t cred;
-	mbuf_t *mrp;
-	mbuf_t *mdp;
-	caddr_t *dposp;
-	u_int64_t *xidp;
+nfs_request_create(
+	nfsnode_t np,
+	mount_t mp,	/* used only if !np */
+	struct nfsm_chain *nmrest,
+	int procnum,
+	thread_t thd,
+	kauth_cred_t cred,
+	struct nfsreq **reqp)
 {
-	mbuf_t m, mrep, m2;
-	struct nfsreq re, *rep;
-	u_long *tl;
-	int i;
+	struct nfsreq *req, *newreq = NULL;
 	struct nfsmount *nmp;
-	mbuf_t md, mheadend;
-	char nickv[RPCX_NICKVERF];
-	time_t waituntil;
-	caddr_t dpos, cp2;
-	int t1, error = 0, mrest_len, auth_len, auth_type;
-	int trylater_delay = NFS_TRYLATERDEL, failed_auth = 0;
-	int verf_len, verf_type;
-	u_long xid;
-	char *auth_str, *verf_str;
-	NFSKERBKEY_T key;		/* save session key */
-	int nmsotype;
-	struct timeval now;
 
-	if (mrp)
-		*mrp = NULL;
-	if (xidp)
-		*xidp = 0;
-	nmp = VFSTONFS(mp);
+	req = *reqp;
+	if (!req) {
+		/* allocate a new NFS request structure */
+		MALLOC_ZONE(newreq, struct nfsreq*, sizeof(*newreq), M_NFSREQ, M_WAITOK);
+		if (!newreq) {
+			mbuf_freem(nmrest->nmc_mhead);
+			nmrest->nmc_mhead = NULL;
+			return (ENOMEM);
+		}
+		req = newreq;
+	}
 
-	rep = &re;
+	bzero(req, sizeof(*req));
+	if (req == newreq)
+		req->r_flags = R_ALLOCATED;
 
-	if (vp)
-		nmp = VFSTONFS(vnode_mount(vp));
-	if (nmp == NULL ||
-	    (nmp->nm_state & (NFSSTA_FORCE|NFSSTA_TIMEO)) ==
+	nmp = VFSTONFS(np ? NFSTOMP(np) : mp);
+	if (!nmp) {
+		if (newreq)
+			FREE_ZONE(newreq, sizeof(*newreq), M_NFSREQ);
+		return (ENXIO);
+	}
+	lck_mtx_lock(&nmp->nm_lock);
+	if ((nmp->nm_state & (NFSSTA_FORCE|NFSSTA_TIMEO)) ==
 	    (NFSSTA_FORCE|NFSSTA_TIMEO)) {
-		mbuf_freem(mrest);
+		lck_mtx_unlock(&nmp->nm_lock);
+		mbuf_freem(nmrest->nmc_mhead);
+		nmrest->nmc_mhead = NULL;
+		if (newreq)
+			FREE_ZONE(newreq, sizeof(*newreq), M_NFSREQ);
 		return (ENXIO);
 	}
-	nmsotype = nmp->nm_sotype;
 
-	FSDBG_TOP(531, vp, procnum, nmp, rep);
+	if ((nmp->nm_vers != NFS_VER4) && (procnum >= 0) && (procnum < NFS_NPROCS))
+		OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[procnum]);
+	if ((nmp->nm_vers == NFS_VER4) && (procnum != NFSPROC4_COMPOUND) && (procnum != NFSPROC4_NULL))
+		panic("nfs_request: invalid NFSv4 RPC request %d\n", procnum);
 
-	rep->r_nmp = nmp;
-	rep->r_vp = vp;
-	rep->r_procp = procp;
-	rep->r_procnum = procnum;
-	microuptime(&now);
-	rep->r_lastmsg = now.tv_sec -
-	    ((nmp->nm_tprintf_delay) - (nmp->nm_tprintf_initial_delay));
-	i = 0;
-	m = mrest;
-	while (m) {
-		i += mbuf_len(m);
-		m = mbuf_next(m);
+	lck_mtx_init(&req->r_mtx, nfs_request_grp, LCK_ATTR_NULL);
+	req->r_nmp = nmp;
+	req->r_np = np;
+	req->r_thread = thd;
+	if (IS_VALID_CRED(cred)) {
+		kauth_cred_ref(cred);
+		req->r_cred = cred;
 	}
-	mrest_len = i;
+	req->r_procnum = procnum;
+	if (proct[procnum] > 0)
+		req->r_flags |= R_TIMING;
+	req->r_nmrep.nmc_mhead = NULL;
+	SLIST_INIT(&req->r_gss_seqlist);
+	req->r_achain.tqe_next = NFSREQNOLIST;
+	req->r_rchain.tqe_next = NFSREQNOLIST;
+	req->r_cchain.tqe_next = NFSREQNOLIST;
+
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	/* move the request mbuf chain to the nfsreq */
+	req->r_mrest = nmrest->nmc_mhead;
+	nmrest->nmc_mhead = NULL;
+
+	req->r_flags |= R_INITTED;
+	req->r_refs = 1;
+	if (newreq)
+		*reqp = req;
+	return (0);
+}
+
+/*
+ * Clean up and free an NFS request structure.
+ */
+void
+nfs_request_destroy(struct nfsreq *req)
+{
+	struct nfsmount *nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp;
+	struct gss_seq *gsp, *ngsp;
+	struct timespec ts = { 1, 0 };
+
+	if (!req || !(req->r_flags & R_INITTED))
+		return;
+	req->r_flags &= ~R_INITTED;
+	if (req->r_lflags & RL_QUEUED)
+		nfs_reqdequeue(req);
+	if (req->r_achain.tqe_next != NFSREQNOLIST) {
+		/* still on an async I/O queue? */
+		lck_mtx_lock(nfsiod_mutex);
+		if (nmp && (req->r_achain.tqe_next != NFSREQNOLIST)) {
+			TAILQ_REMOVE(&nmp->nm_iodq, req, r_achain);
+			req->r_achain.tqe_next = NFSREQNOLIST;
+		}
+		lck_mtx_unlock(nfsiod_mutex);
+	}
+	if (nmp) {
+		lck_mtx_lock(&nmp->nm_lock);
+		if (req->r_rchain.tqe_next != NFSREQNOLIST) {
+			TAILQ_REMOVE(&nmp->nm_resendq, req, r_rchain);
+			req->r_rchain.tqe_next = NFSREQNOLIST;
+			req->r_flags &= ~R_RESENDQ;
+		}
+		if (req->r_cchain.tqe_next != NFSREQNOLIST) {
+			TAILQ_REMOVE(&nmp->nm_cwndq, req, r_cchain);
+			req->r_cchain.tqe_next = NFSREQNOLIST;
+		}
+		lck_mtx_unlock(&nmp->nm_lock);
+	}
+	lck_mtx_lock(&req->r_mtx);
+	while (req->r_flags & R_RESENDQ)
+		msleep(req, &req->r_mtx, (PZERO - 1), "nfsresendqwait", &ts);
+	lck_mtx_unlock(&req->r_mtx);
+	if (req->r_mhead)
+		mbuf_freem(req->r_mhead);
+	else if (req->r_mrest)
+		mbuf_freem(req->r_mrest);
+	if (req->r_nmrep.nmc_mhead)
+		mbuf_freem(req->r_nmrep.nmc_mhead);
+	if (IS_VALID_CRED(req->r_cred))
+		kauth_cred_unref(&req->r_cred);
+	if (req->r_gss_ctx)
+		nfs_gss_clnt_rpcdone(req);
+	SLIST_FOREACH_SAFE(gsp, &req->r_gss_seqlist, gss_seqnext, ngsp)
+		FREE(gsp, M_TEMP);
+	if (req->r_gss_ctx)
+		nfs_gss_clnt_ctx_unref(req);
+
+	lck_mtx_destroy(&req->r_mtx, nfs_request_grp);
+	if (req->r_flags & R_ALLOCATED)
+		FREE_ZONE(req, sizeof(*req), M_NFSREQ);
+}
+
+void
+nfs_request_ref(struct nfsreq *req, int locked)
+{
+	if (!locked)
+		lck_mtx_lock(&req->r_mtx);
+	if (req->r_refs <= 0)
+		panic("nfsreq reference error");
+	req->r_refs++;
+	if (!locked)
+		lck_mtx_unlock(&req->r_mtx);
+}
+
+void
+nfs_request_rele(struct nfsreq *req)
+{
+	int destroy;
+
+	lck_mtx_lock(&req->r_mtx);
+	if (req->r_refs <= 0)
+		panic("nfsreq reference underflow");
+	req->r_refs--;
+	destroy = (req->r_refs == 0);
+	lck_mtx_unlock(&req->r_mtx);
+	if (destroy)
+		nfs_request_destroy(req);
+}
+
+
+/*
+ * Add an (updated) RPC header with authorization to an NFS request.
+ */
+int
+nfs_request_add_header(struct nfsreq *req)
+{
+	struct nfsmount *nmp;
+	int error = 0, auth_len = 0;
+	mbuf_t m;
+
+	/* free up any previous header */
+	if ((m = req->r_mhead)) {
+		while (m && (m != req->r_mrest))
+			m = mbuf_free(m);
+		req->r_mhead = NULL;
+	}
+
+	nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp;
+	if (!nmp)
+		return (ENXIO);
+
+	if (!req->r_cred) /* RPCAUTH_NULL */
+		auth_len = 0;
+	else switch (nmp->nm_auth) {
+		case RPCAUTH_UNIX:
+			if (req->r_cred->cr_ngroups < 1)
+				return (EINVAL);
+			auth_len = ((((req->r_cred->cr_ngroups - 1) > nmp->nm_numgrps) ?
+				nmp->nm_numgrps : (req->r_cred->cr_ngroups - 1)) << 2) +
+				5 * NFSX_UNSIGNED;
+			break;
+		case RPCAUTH_KRB5:
+		case RPCAUTH_KRB5I:
+		case RPCAUTH_KRB5P:
+			auth_len = 5 * NFSX_UNSIGNED + 0; // zero context handle for now
+			break;
+		}
+
+	error = nfsm_rpchead(req, auth_len, req->r_mrest, &req->r_xid, &req->r_mhead);
+	if (error)
+		return (error);
+
+	req->r_mreqlen = mbuf_pkthdr_len(req->r_mhead);
+	nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp;
+	if (!nmp)
+		return (ENXIO);
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_flag & NFSMNT_SOFT)
+		req->r_retry = nmp->nm_retry;
+	else
+		req->r_retry = NFS_MAXREXMIT + 1;	/* past clip limit */
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	return (error);
+}
+
+
+/*
+ * Queue an NFS request up and send it out.
+ */
+int
+nfs_request_send(struct nfsreq *req, int wait)
+{
+	struct nfsmount *nmp;
+	struct timeval now;
+
+	lck_mtx_lock(nfs_request_mutex);
 
-	/*
-	 * Get the RPC header with authorization.
-	 */
-kerbauth:
-	nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
+	nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp;
 	if (!nmp) {
-		FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-		mbuf_freem(mrest);
+		lck_mtx_unlock(nfs_request_mutex);
 		return (ENXIO);
 	}
-	verf_str = auth_str = (char *)0;
-	if (nmp->nm_flag & NFSMNT_KERB) {
-		verf_str = nickv;
-		verf_len = sizeof (nickv);
-		auth_type = RPCAUTH_KERB4;
-		bzero((caddr_t)key, sizeof (key));
-		if (failed_auth || nfs_getnickauth(nmp, cred, &auth_str,
-			&auth_len, verf_str, verf_len)) {
-			nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
-			if (!nmp) {
-				FSDBG_BOT(531, 2, vp, error, rep);
-				mbuf_freem(mrest);
-				return (ENXIO);
-			}
-			error = nfs_getauth(nmp, rep, cred, &auth_str,
-				&auth_len, verf_str, &verf_len, key);
-			nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
-			if (!error && !nmp)
-				error = ENXIO;
-			if (error) {
-				FSDBG_BOT(531, 2, vp, error, rep);
-				mbuf_freem(mrest);
-				return (error);
-			}
-		}
-	} else {
-		auth_type = RPCAUTH_UNIX;
-		if (cred->cr_ngroups < 1)
-			panic("nfsreq nogrps");
-		auth_len = ((((cred->cr_ngroups - 1) > nmp->nm_numgrps) ?
-			nmp->nm_numgrps : (cred->cr_ngroups - 1)) << 2) +
-			5 * NFSX_UNSIGNED;
-	}
-	error = nfsm_rpchead(cred, nmp->nm_flag, procnum, auth_type, auth_len,
-	     auth_str, verf_len, verf_str, mrest, mrest_len, &mheadend, &xid, &m);
-	if (auth_str)
-		_FREE(auth_str, M_TEMP);
-	if (error) {
-		mbuf_freem(mrest);
-		FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-		return (error);
-	}
-	if (xidp)
-		*xidp = ntohl(xid) + ((u_int64_t)nfs_xidwrap << 32);
 
-	/*
-	 * For stream protocols, insert a Sun RPC Record Mark.
-	 */
-	if (nmsotype == SOCK_STREAM) {
-		error = mbuf_prepend(&m, NFSX_UNSIGNED, MBUF_WAITOK);
-		if (error) {
-			mbuf_freem(m);
-			FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-			return (error);
-		}
-		*((u_long*)mbuf_data(m)) =
-			htonl(0x80000000 | (mbuf_pkthdr_len(m) - NFSX_UNSIGNED));
+	microuptime(&now);
+	if (!req->r_start) {
+		req->r_start = now.tv_sec;
+		req->r_lastmsg = now.tv_sec -
+		    ((nmp->nm_tprintf_delay) - (nmp->nm_tprintf_initial_delay));
 	}
-	rep->r_mreq = m;
-	rep->r_xid = xid;
-tryagain:
-	nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
-	if (nmp && (nmp->nm_flag & NFSMNT_SOFT))
-		rep->r_retry = nmp->nm_retry;
-	else
-		rep->r_retry = NFS_MAXREXMIT + 1;	/* past clip limit */
-	rep->r_rtt = rep->r_rexmit = 0;
-	if (proct[procnum] > 0)
-		rep->r_flags = R_TIMING;
-	else
-		rep->r_flags = 0;
-	rep->r_mrep = NULL;
 
-	/*
-	 * Do the client side RPC.
-	 */
 	OSAddAtomic(1, (SInt32*)&nfsstats.rpcrequests);
+
 	/*
 	 * Chain request into list of outstanding requests. Be sure
 	 * to put it LAST so timer finds oldest requests first.
+	 * Make sure that the request queue timer is running
+	 * to check for possible request timeout.
 	 */
-	TAILQ_INSERT_TAIL(&nfs_reqq, rep, r_chain);
+	TAILQ_INSERT_TAIL(&nfs_reqq, req, r_chain);
+	req->r_lflags |= RL_QUEUED;
+	if (!nfs_request_timer_on) {
+		nfs_request_timer_on = 1;
+		nfs_interval_timer_start(nfs_request_timer_call,
+			NFS_REQUESTDELAY);
+	}
+	lck_mtx_unlock(nfs_request_mutex);
 
-	/*
-	 * If backing off another request or avoiding congestion, don't
-	 * send this one now but let timer do it. If not timing a request,
-	 * do it now.
-	 */
-	if (nmp && nmp->nm_so && (nmp->nm_sotype != SOCK_DGRAM ||
-			   (nmp->nm_flag & NFSMNT_DUMBTIMR) ||
-			   nmp->nm_sent < nmp->nm_cwnd)) {
-		int connrequired = (nmp->nm_sotype == SOCK_STREAM);
+	/* Send the request... */
+	return (nfs_send(req, wait));
+}
+
+/*
+ * Call nfs_wait_reply() to wait for the reply.
+ */
+void
+nfs_request_wait(struct nfsreq *req)
+{
+	req->r_error = nfs_wait_reply(req);
+}
 
-		if (connrequired)
-			error = nfs_sndlock(rep);
+/*
+ * Finish up an NFS request by dequeueing it and
+ * doing the initial NFS request reply processing.
+ */
+int
+nfs_request_finish(
+	struct nfsreq *req,
+	struct nfsm_chain *nmrepp,
+	int *status)
+{
+	struct nfsmount *nmp;
+	mbuf_t mrep;
+	int verf_type = 0;
+	uint32_t verf_len = 0;
+	uint32_t reply_status = 0;
+	uint32_t rejected_status = 0;
+	uint32_t auth_status = 0;
+	uint32_t accepted_status = 0;
+	struct nfsm_chain nmrep;
+	int error, auth;
 
-		/*
-		 * Set the R_SENT before doing the send in case another thread
-		 * processes the reply before the nfs_send returns here
-		 */
-		if (!error) {
-			if ((rep->r_flags & R_MUSTRESEND) == 0) {
-				FSDBG(531, rep->r_xid, rep, nmp->nm_sent,
-				      nmp->nm_cwnd);
-				nmp->nm_sent += NFS_CWNDSCALE;
-				rep->r_flags |= R_SENT;
-			}
+	error = req->r_error;
 
-			error = mbuf_copym(m, 0, MBUF_COPYALL, MBUF_WAITOK, &m2);
-			if (!error)
-				error = nfs_send(nmp->nm_so, nmp->nm_nam, m2, rep);
-			if (connrequired)
-				nfs_sndunlock(rep);
-		}
-		nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
-		if (error) {
-			if (nmp)
-				nmp->nm_sent -= NFS_CWNDSCALE;
-			rep->r_flags &= ~R_SENT;
-		}
-	} else {
-		rep->r_rtt = -1;
-	}
+	if (nmrepp)
+		nmrepp->nmc_mhead = NULL;
 
-	/*
-	 * Wait for the reply from our send or the timer's.
-	 */
-	if (!error || error == EPIPE)
-		error = nfs_reply(rep);
+	/* RPC done, unlink the request. */
+	nfs_reqdequeue(req);
 
-	/*
-	 * RPC done, unlink the request.
-	 */
-	nfs_repdequeue(rep);
+	mrep = req->r_nmrep.nmc_mhead;
 
-	nmp = vp ? VFSTONFS(vnode_mount(vp)) : rep->r_nmp;
+	nmp = req->r_np ? NFSTONMP(req->r_np) : req->r_nmp;
 
 	/*
 	 * Decrement the outstanding request count.
 	 */
-	if (rep->r_flags & R_SENT) {
-		rep->r_flags &= ~R_SENT;	/* paranoia */
-		if (nmp) {
-			FSDBG(531, rep->r_xid, rep, nmp->nm_sent, nmp->nm_cwnd);
-			nmp->nm_sent -= NFS_CWNDSCALE;
+	if (req->r_flags & R_CWND) {
+		req->r_flags &= ~R_CWND;
+		lck_mtx_lock(&nmp->nm_lock);
+		FSDBG(273, R_XID32(req->r_xid), req, nmp->nm_sent, nmp->nm_cwnd);
+		nmp->nm_sent -= NFS_CWNDSCALE;
+		if ((nmp->nm_sent < nmp->nm_cwnd) && !TAILQ_EMPTY(&nmp->nm_cwndq)) {
+			/* congestion window is open, poke the cwnd queue */
+			struct nfsreq *req2 = TAILQ_FIRST(&nmp->nm_cwndq);
+			TAILQ_REMOVE(&nmp->nm_cwndq, req2, r_cchain);
+			req2->r_cchain.tqe_next = NFSREQNOLIST;
+			wakeup(req2);
 		}
+		lck_mtx_unlock(&nmp->nm_lock);
 	}
 
-	/*
-	 * If there was a successful reply and a tprintf msg.
-	 * tprintf a response.
-	 */
-	if (!error)
-		nfs_up(nmp, procp, NFSSTA_TIMEO,
-			(rep->r_flags & R_TPRINTFMSG) ? "is alive again" : NULL);
-	mrep = rep->r_mrep;
-	md = rep->r_md;
-	dpos = rep->r_dpos;
-	if (!error && !nmp)
-		error = ENXIO;
-	if (error) {
-		mbuf_freem(rep->r_mreq);
-		FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-		return (error);
+	if (req->r_gss_ctx) {	// Using gss cred ?
+		/*
+		 * If the request had an RPCSEC_GSS credential
+		 * then reset its sequence number bit in the
+		 * request window.
+		 */
+		nfs_gss_clnt_rpcdone(req);
+
+		/*
+		 * If we need to re-send, go back and re-build the
+		 * request based on a new sequence number.
+		 * Note that we're using the original XID.
+		 */
+		if (error == EAGAIN) {
+			req->r_error = 0;
+			if (mrep)
+				mbuf_freem(mrep);
+			error = nfs_gss_clnt_args_restore(req);	// remove any trailer mbufs
+			req->r_nmrep.nmc_mhead = NULL;
+			req->r_flags |= R_RESTART;
+			if (error == ENEEDAUTH) {
+				req->r_xid = 0;		// get a new XID
+				error = 0;
+			}
+			goto nfsmout;
+		}
 	}
 
 	/*
-	 * break down the rpc header and check if ok
+	 * If there was a successful reply, make sure to mark the mount as up.
+	 * If a tprintf message was given (or if this is a timed-out soft mount)
+	 * then post a tprintf message indicating the server is alive again.
 	 */
-	nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-	if (*tl++ == rpc_msgdenied) {
-		if (*tl == rpc_mismatch)
-			error = EOPNOTSUPP;
-		else if ((nmp->nm_flag & NFSMNT_KERB) && *tl++ == rpc_autherr) {
-			if (!failed_auth) {
-				failed_auth++;
-				error = mbuf_setnext(mheadend, NULL);
-				mbuf_freem(mrep);
-				mbuf_freem(rep->r_mreq);
-				if (!error)
-					goto kerbauth;
-				printf("nfs_request: mbuf_setnext failed\n");
-			} else
-				error = EAUTH;
-		} else
-			error = EACCES;
-		mbuf_freem(mrep);
-		mbuf_freem(rep->r_mreq);
-		FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-		return (error);
+	if (!error) {
+		if ((req->r_flags & R_TPRINTFMSG) ||
+		    (nmp && (nmp->nm_flag & NFSMNT_SOFT) &&
+		     ((nmp->nm_state & (NFSSTA_TIMEO|NFSSTA_FORCE)) == NFSSTA_TIMEO)))
+			nfs_up(nmp, req->r_thread, NFSSTA_TIMEO, "is alive again");
+		else
+			nfs_up(nmp, req->r_thread, NFSSTA_TIMEO, NULL);
 	}
+	if (!error && !nmp)
+		error = ENXIO;
+	nfsmout_if(error);
 
 	/*
-	 * Grab any Kerberos verifier, otherwise just throw it away.
+	 * break down the RPC header and check if ok
 	 */
-	verf_type = fxdr_unsigned(int, *tl++);
-	i = fxdr_unsigned(int, *tl);
-	if ((nmp->nm_flag & NFSMNT_KERB) && verf_type == RPCAUTH_KERB4) {
-		error = nfs_savenickauth(nmp, cred, i, key, &md, &dpos, mrep);
-		if (error)
+	nmrep = req->r_nmrep;
+	nfsm_chain_get_32(error, &nmrep, reply_status);
+	nfsmout_if(error);
+	if (reply_status == RPC_MSGDENIED) {
+		nfsm_chain_get_32(error, &nmrep, rejected_status);
+		nfsmout_if(error);
+		if (rejected_status == RPC_MISMATCH) {
+			error = ENOTSUP;
 			goto nfsmout;
-	} else if (i > 0)
-		nfsm_adv(nfsm_rndup(i));
-	nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-	/* 0 == ok */
-	if (*tl == 0) {
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		if (*tl != 0) {
-			error = fxdr_unsigned(int, *tl);
-			if ((nmp->nm_flag & NFSMNT_NFSV3) &&
-				error == NFSERR_TRYLATER) {
-				mbuf_freem(mrep);
-				error = 0;
-				microuptime(&now);
-				waituntil = now.tv_sec + trylater_delay;
-				while (now.tv_sec < waituntil) {
-					tsleep((caddr_t)&lbolt, PSOCK, "nfstrylater", 0);
-					microuptime(&now);
-				}
-				trylater_delay *= 2;
-				if (trylater_delay > 60)
-					trylater_delay = 60;
-				goto tryagain;
+		}
+		nfsm_chain_get_32(error, &nmrep, auth_status);
+		nfsmout_if(error);
+		switch (auth_status) {
+		case RPCSEC_GSS_CREDPROBLEM:
+		case RPCSEC_GSS_CTXPROBLEM:
+			/*
+			 * An RPCSEC_GSS cred or context problem.
+			 * We can't use it anymore.
+			 * Restore the args, renew the context
+			 * and set up for a resend.
+			 */
+			error = nfs_gss_clnt_args_restore(req);
+			if (error && error != ENEEDAUTH)
+				break;
+			
+			if (!error) {
+				error = nfs_gss_clnt_ctx_renew(req);
+				if (error)
+					break;
 			}
+			mbuf_freem(mrep);
+			req->r_nmrep.nmc_mhead = NULL;
+			req->r_xid = 0;		// get a new XID
+			req->r_flags |= R_RESTART;
+			goto nfsmout;
+		default:
+			error = EACCES;
+			break;
+		}
+		goto nfsmout;
+	}
+
+	/* Now check the verifier */
+	nfsm_chain_get_32(error, &nmrep, verf_type); // verifier flavor
+	nfsm_chain_get_32(error, &nmrep, verf_len);  // verifier length
+	nfsmout_if(error);
+
+	auth = !req->r_cred ? RPCAUTH_NULL : nmp->nm_auth;
+	switch (auth) {
+	case RPCAUTH_NULL:
+	case RPCAUTH_UNIX:
+		/* Any AUTH_UNIX verifier is ignored */
+		if (verf_len > 0)
+			nfsm_chain_adv(error, &nmrep, nfsm_rndup(verf_len));
+		nfsm_chain_get_32(error, &nmrep, accepted_status);
+		break;
+	case RPCAUTH_KRB5:
+	case RPCAUTH_KRB5I:
+	case RPCAUTH_KRB5P:
+		error = nfs_gss_clnt_verf_get(req, &nmrep,
+			verf_type, verf_len, &accepted_status);
+		break;
+	}
+	nfsmout_if(error);
+
+	switch (accepted_status) {
+	case RPC_SUCCESS:
+		if (req->r_procnum == NFSPROC_NULL) {
+			/*
+			 * The NFS null procedure is unique,
+			 * in not returning an NFS status.
+			 */
+			*status = NFS_OK;
+		} else {
+			nfsm_chain_get_32(error, &nmrep, *status);
+			nfsmout_if(error);
+		}
 
+		if ((nmp->nm_vers != NFS_VER2) && (*status == NFSERR_TRYLATER)) {
 			/*
-			 * If the File Handle was stale, invalidate the
-			 * lookup cache, just in case.
+			 * It's a JUKEBOX error - delay and try again
 			 */
-			if ((error == ESTALE) && vp)
-				cache_purge(vp);
-			if (nmp->nm_flag & NFSMNT_NFSV3) {
-				*mrp = mrep;
-				*mdp = md;
-				*dposp = dpos;
-				error |= NFSERR_RETERR;
+			int delay, slpflag = (nmp->nm_flag & NFSMNT_INT) ? PCATCH : 0;
+
+			mbuf_freem(mrep);
+			req->r_nmrep.nmc_mhead = NULL;
+			if ((req->r_delay >= 30) && !(nmp->nm_state & NFSSTA_MOUNTED)) {
+				/* we're not yet completely mounted and */
+				/* we can't complete an RPC, so we fail */
+				OSAddAtomic(1, (SInt32*)&nfsstats.rpctimeouts);
+				nfs_softterm(req);
+				error = req->r_error;
+				goto nfsmout;
+			}
+			req->r_delay = !req->r_delay ? NFS_TRYLATERDEL : (req->r_delay * 2);
+			if (req->r_delay > 30)
+				req->r_delay = 30;
+			if (nmp->nm_tprintf_initial_delay && (req->r_delay == 30)) {
+				nfs_down(req->r_nmp, req->r_thread, 0, NFSSTA_JUKEBOXTIMEO,
+					"resource temporarily unavailable (jukebox)");
+				req->r_flags |= R_JBTPRINTFMSG;
+			}
+			delay = req->r_delay;
+			if (req->r_callback.rcb_func) {
+				struct timeval now;
+				microuptime(&now);
+				req->r_resendtime = now.tv_sec + delay;
 			} else {
-				mbuf_freem(mrep);
-				error &= ~NFSERR_RETERR;
+				do {
+					if ((error = nfs_sigintr(req->r_nmp, req, req->r_thread, 0)))
+						return (error);
+					tsleep(&lbolt, PSOCK|slpflag, "nfs_jukebox_trylater", 0);
+				} while (--delay > 0);
 			}
-			mbuf_freem(rep->r_mreq);
-			FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
-			return (error);
+			req->r_xid = 0;			// get a new XID
+			req->r_flags |= R_RESTART;
+			req->r_start = 0;
+			FSDBG(273, R_XID32(req->r_xid), nmp, req, NFSERR_TRYLATER);
+			return (0);
 		}
 
-		*mrp = mrep;
-		*mdp = md;
-		*dposp = dpos;
-		mbuf_freem(rep->r_mreq);
-		FSDBG_BOT(531, 0xf0f0f0f0, rep->r_xid, nmp, rep);
-		return (0);
+		if (req->r_flags & R_JBTPRINTFMSG)
+			nfs_up(nmp, req->r_thread, NFSSTA_JUKEBOXTIMEO, "resource available again");
+
+		if (*status == NFS_OK) {
+			/*
+			 * Successful NFS request
+			 */
+			*nmrepp = nmrep;
+			req->r_nmrep.nmc_mhead = NULL;
+			break;
+		}
+		/* Got an NFS error of some kind */
+
+		/*
+		 * If the File Handle was stale, invalidate the
+		 * lookup cache, just in case.
+		 */
+		if ((*status == ESTALE) && req->r_np)
+			cache_purge(NFSTOV(req->r_np));
+		if (nmp->nm_vers == NFS_VER2)
+			mbuf_freem(mrep);
+		else
+			*nmrepp = nmrep;
+		req->r_nmrep.nmc_mhead = NULL;
+		error = 0;
+		break;
+	case RPC_PROGUNAVAIL:
+		error = EPROGUNAVAIL;
+		break;
+	case RPC_PROGMISMATCH:
+		error = ERPCMISMATCH;
+		break;
+	case RPC_PROCUNAVAIL:
+		error = EPROCUNAVAIL;
+		break;
+	case RPC_GARBAGE:
+		error = EBADRPC;
+		break;
+	case RPC_SYSTEM_ERR:
+	default:
+		error = EIO;
+		break;
 	}
-	mbuf_freem(mrep);
-	error = EPROTONOSUPPORT;
 nfsmout:
-	mbuf_freem(rep->r_mreq);
-	FSDBG_BOT(531, error, rep->r_xid, nmp, rep);
+	if (!error && (req->r_flags & R_JBTPRINTFMSG))
+		nfs_up(nmp, req->r_thread, NFSSTA_JUKEBOXTIMEO, NULL);
+	FSDBG(273, R_XID32(req->r_xid), nmp, req,
+		(!error && (*status == NFS_OK)) ? 0xf0f0f0f0 : error);
 	return (error);
 }
 
-#ifndef NFS_NOSERVER
+
 /*
- * Generate the rpc reply header
- * siz arg. is used to decide if adding a cluster is worthwhile
+ * Perform an NFS request synchronously.
  */
+
 int
-nfs_rephead(siz, nd, slp, err, mrq, mbp, bposp)
-	int siz;
-	struct nfsrv_descript *nd;
-	struct nfssvc_sock *slp;
-	int err;
-	mbuf_t *mrq;
-	mbuf_t *mbp;
-	caddr_t *bposp;
+nfs_request(
+	nfsnode_t np,
+	mount_t mp,	/* used only if !np */
+	struct nfsm_chain *nmrest,
+	int procnum,
+	vfs_context_t ctx,
+	struct nfsm_chain *nmrepp,
+	u_int64_t *xidp,
+	int *status)
 {
-	u_long *tl;
-	mbuf_t mreq;
-	caddr_t bpos;
-	mbuf_t mb, mb2;
-	int error, mlen;
-
-	/*
-	 * If this is a big reply, use a cluster else
-	 * try and leave leading space for the lower level headers.
-	 */
-	siz += RPC_REPLYSIZ;
-	if (siz >= nfs_mbuf_minclsize) {
-		error = mbuf_getpacket(MBUF_WAITOK, &mreq);
-	} else {
-		error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &mreq);
-	}
-	if (error) {
-		/* unable to allocate packet */
-		/* XXX nfsstat? */
-		return (error);
-	}
-	mb = mreq;
-	tl = mbuf_data(mreq);
-	mlen = 6 * NFSX_UNSIGNED;
-	if (siz < nfs_mbuf_minclsize) {
-		/* leave space for lower level headers */
-		tl += 80/sizeof(*tl);  /* XXX max_hdr? XXX */
-		mbuf_setdata(mreq, tl, mlen);
-	} else {
-		mbuf_setlen(mreq, mlen);
-	}
-	bpos = ((caddr_t)tl) + mlen;
-	*tl++ = txdr_unsigned(nd->nd_retxid);
-	*tl++ = rpc_reply;
-	if (err == ERPCMISMATCH || (err & NFSERR_AUTHERR)) {
-		*tl++ = rpc_msgdenied;
-		if (err & NFSERR_AUTHERR) {
-			*tl++ = rpc_autherr;
-			*tl = txdr_unsigned(err & ~NFSERR_AUTHERR);
-			mlen -= NFSX_UNSIGNED;
-			mbuf_setlen(mreq, mlen);
-			bpos -= NFSX_UNSIGNED;
-		} else {
-			*tl++ = rpc_mismatch;
-			*tl++ = txdr_unsigned(RPC_VER2);
-			*tl = txdr_unsigned(RPC_VER2);
-		}
-	} else {
-		*tl++ = rpc_msgaccepted;
+	return nfs_request2(np, mp, nmrest, procnum,
+		vfs_context_thread(ctx), vfs_context_ucred(ctx),
+		0, nmrepp, xidp, status);
+}
 
-		/*
-		 * For Kerberos authentication, we must send the nickname
-		 * verifier back, otherwise just RPCAUTH_NULL.
-		 */
-		if (nd->nd_flag & ND_KERBFULL) {
-		    struct nfsuid *nuidp;
-		    struct timeval ktvin, ktvout;
-		    uid_t uid = kauth_cred_getuid(nd->nd_cr);
-
-		    lck_rw_lock_shared(&slp->ns_rwlock);
-		    for (nuidp = NUIDHASH(slp, uid)->lh_first;
-			nuidp != 0; nuidp = nuidp->nu_hash.le_next) {
-			if (kauth_cred_getuid(nuidp->nu_cr) == uid &&
-			    (!nd->nd_nam2 || netaddr_match(NU_NETFAM(nuidp),
-			     &nuidp->nu_haddr, nd->nd_nam2)))
-			    break;
-		    }
-		    if (nuidp) {
-			ktvin.tv_sec =
-			    txdr_unsigned(nuidp->nu_timestamp.tv_sec - 1);
-			ktvin.tv_usec =
-			    txdr_unsigned(nuidp->nu_timestamp.tv_usec);
+int
+nfs_request2(
+	nfsnode_t np,
+	mount_t mp,	/* used only if !np */
+	struct nfsm_chain *nmrest,
+	int procnum,
+	thread_t thd,
+	kauth_cred_t cred,
+	int flags,
+	struct nfsm_chain *nmrepp,
+	u_int64_t *xidp,
+	int *status)
+{
+	struct nfsreq rq, *req = &rq;
+	int error;
 
-			/*
-			 * Encrypt the timestamp in ecb mode using the
-			 * session key.
-			 */
-#if NFSKERB
-			XXX
-#endif
+	if ((error = nfs_request_create(np, mp, nmrest, procnum, thd, cred, &req)))
+		return (error);
+	req->r_flags |= (flags & R_OPTMASK);
 
-			*tl++ = rpc_auth_kerb;
-			*tl++ = txdr_unsigned(3 * NFSX_UNSIGNED);
-			*tl = ktvout.tv_sec;
-			nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED);
-			*tl++ = ktvout.tv_usec;
-			*tl++ = txdr_unsigned(kauth_cred_getuid(nuidp->nu_cr));
-		    } else {
-			*tl++ = 0;
-			*tl++ = 0;
-		    }
-		    lck_rw_done(&slp->ns_rwlock);
-		} else {
-			*tl++ = 0;
-			*tl++ = 0;
-		}
-		switch (err) {
-		case EPROGUNAVAIL:
-			*tl = txdr_unsigned(RPC_PROGUNAVAIL);
+	FSDBG_TOP(273, R_XID32(req->r_xid), np, procnum, 0);
+	do {
+		req->r_error = 0;
+		req->r_flags &= ~R_RESTART;
+		if ((error = nfs_request_add_header(req)))
 			break;
-		case EPROGMISMATCH:
-			*tl = txdr_unsigned(RPC_PROGMISMATCH);
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			// XXX hard coded versions
-			*tl++ = txdr_unsigned(2);
-			*tl = txdr_unsigned(3);
-			break;
-		case EPROCUNAVAIL:
-			*tl = txdr_unsigned(RPC_PROCUNAVAIL);
-			break;
-		case EBADRPC:
-			*tl = txdr_unsigned(RPC_GARBAGE);
+		if (xidp)
+			*xidp = req->r_xid;
+		if ((error = nfs_request_send(req, 1)))
 			break;
-		default:
-			*tl = 0;
-			if (err != NFSERR_RETVOID) {
-				nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-				if (err)
-				    *tl = txdr_unsigned(nfsrv_errmap(nd, err));
-				else
-				    *tl = 0;
-			}
+		nfs_request_wait(req);
+		if ((error = nfs_request_finish(req, nmrepp, status)))
 			break;
-		}
-	}
+	} while (req->r_flags & R_RESTART);
 
-	if (mrq != NULL)
-		*mrq = mreq;
-	*mbp = mb;
-	*bposp = bpos;
-	if (err != 0 && err != NFSERR_RETVOID) {
-		OSAddAtomic(1, (SInt32*)&nfsstats.srvrpc_errs);
-	}
-	return (0);
+	FSDBG_BOT(273, R_XID32(req->r_xid), np, procnum, error);
+	nfs_request_rele(req);
+	return (error);
 }
 
+/*
+ * Create and start an asynchronous NFS request.
+ */
+int
+nfs_request_async(
+	nfsnode_t np,
+	mount_t mp,	/* used only if !np */
+	struct nfsm_chain *nmrest,
+	int procnum,
+	thread_t thd,
+	kauth_cred_t cred,
+	struct nfsreq_cbinfo *cb,
+	struct nfsreq **reqp)
+{
+	struct nfsreq *req;
+	int error, sent;
 
-#endif /* NFS_NOSERVER */
-
+	error = nfs_request_create(np, mp, nmrest, procnum, thd, cred, reqp);
+	req = *reqp;
+	FSDBG(274, (req ? R_XID32(req->r_xid) : 0), np, procnum, error);
+	if (error)
+		return (error);
+	req->r_flags |= R_ASYNC;
+	if (cb)
+		req->r_callback = *cb;
+	error = nfs_request_add_header(req);
+	if (!error) {
+		req->r_flags |= R_WAITSENT;
+		if (req->r_callback.rcb_func)
+			nfs_request_ref(req, 0);
+		error = nfs_request_send(req, 1);
+		lck_mtx_lock(&req->r_mtx);
+		if (!error && !(req->r_flags & R_SENT) && req->r_callback.rcb_func) {
+			/* make sure to wait until this async I/O request gets sent */
+			int slpflag = (req->r_nmp && (req->r_nmp->nm_flag & NFSMNT_INT) && req->r_thread) ? PCATCH : 0;
+			struct timespec ts = { 2, 0 };
+			while (!error && !(req->r_flags & R_SENT)) {
+				if ((error = nfs_sigintr(req->r_nmp, req, req->r_thread, 0)))
+					break;
+				error = msleep(req, &req->r_mtx, slpflag | (PZERO - 1), "nfswaitsent", &ts);
+				if (error == EWOULDBLOCK)
+					error = 0;
+			}
+		}
+		sent = req->r_flags & R_SENT;
+		lck_mtx_unlock(&req->r_mtx);
+		if (error && req->r_callback.rcb_func && !sent)
+			nfs_request_rele(req);
+	}
+	FSDBG(274, R_XID32(req->r_xid), np, procnum, error);
+	if (error || req->r_callback.rcb_func)
+		nfs_request_rele(req);
+	return (error);
+}
 
 /*
- * From FreeBSD 1.58, a Matt Dillon fix...
- * Flag a request as being about to terminate.
- * The nm_sent count is decremented now to avoid deadlocks when the process
- * in soreceive() hasn't yet managed to send its own request.
+ * Wait for and finish an asynchronous NFS request.
  */
-static void
-nfs_softterm(struct nfsreq *rep)
+int
+nfs_request_async_finish(
+	struct nfsreq *req,
+	struct nfsm_chain *nmrepp,
+	u_int64_t *xidp,
+	int *status)
 {
-
-	rep->r_flags |= R_SOFTTERM;
-	if (rep->r_flags & R_SENT) {
-		FSDBG(532, rep->r_xid, rep, rep->r_nmp->nm_sent,
-		      rep->r_nmp->nm_cwnd);
-		rep->r_nmp->nm_sent -= NFS_CWNDSCALE;
-		rep->r_flags &= ~R_SENT;
+	int error, asyncio = req->r_callback.rcb_func ? 1 : 0;
+
+	lck_mtx_lock(&req->r_mtx);
+	if (!asyncio)
+		req->r_flags |= R_ASYNCWAIT;
+	while (req->r_flags & R_RESENDQ) /* wait until the request is off the resend queue */
+		msleep(req, &req->r_mtx, PZERO-1, "nfsresendqwait", NULL);
+	lck_mtx_unlock(&req->r_mtx);
+
+	nfs_request_wait(req);
+	error = nfs_request_finish(req, nmrepp, status);
+
+	while (!error && (req->r_flags & R_RESTART)) {
+		if (asyncio && req->r_resendtime) {  /* send later */
+			lck_mtx_lock(&req->r_mtx);
+			nfs_asyncio_resend(req);
+			lck_mtx_unlock(&req->r_mtx);
+			return (EINPROGRESS);
+		}
+		req->r_error = 0;
+		req->r_flags &= ~R_RESTART;
+		if ((error = nfs_request_add_header(req)))
+			break;
+		if ((error = nfs_request_send(req, !asyncio)))
+			break;
+		if (asyncio)
+			return (EINPROGRESS);
+		nfs_request_wait(req);
+		if ((error = nfs_request_finish(req, nmrepp, status)))
+			break;
 	}
+	if (xidp)
+		*xidp = req->r_xid;
+
+	FSDBG(275, R_XID32(req->r_xid), req->r_np, req->r_procnum, error);
+	nfs_request_rele(req);
+	return (error);
 }
 
+/*
+ * Cancel a pending asynchronous NFS request.
+ */
 void
-nfs_timer_funnel(void * arg)
+nfs_request_async_cancel(struct nfsreq *req)
 {
-	(void) thread_funnel_set(kernel_flock, TRUE);
-	nfs_timer(arg);
-	(void) thread_funnel_set(kernel_flock, FALSE);
-
+	nfs_reqdequeue(req);
+	FSDBG(275, R_XID32(req->r_xid), req->r_np, req->r_procnum, 0xD1ED1E);
+	nfs_request_rele(req);
 }
 
 /*
- * Ensure rep isn't in use by the timer, then dequeue it.
+ * Flag a request as being terminated.
  */
 static void
-nfs_repdequeue(struct nfsreq *rep)
+nfs_softterm(struct nfsreq *req)
 {
+	struct nfsmount *nmp = req->r_nmp;
+	req->r_flags |= R_SOFTTERM;
+	req->r_error = ETIMEDOUT;
+	if (!(req->r_flags & R_CWND) || !nmp)
+		return;
+	/* update congestion window */
+	req->r_flags &= ~R_CWND;
+	lck_mtx_lock(&nmp->nm_lock);
+	FSDBG(532, R_XID32(req->r_xid), req, nmp->nm_sent, nmp->nm_cwnd);
+	nmp->nm_sent -= NFS_CWNDSCALE;
+	if ((nmp->nm_sent < nmp->nm_cwnd) && !TAILQ_EMPTY(&nmp->nm_cwndq)) {
+		/* congestion window is open, poke the cwnd queue */
+		struct nfsreq *req2 = TAILQ_FIRST(&nmp->nm_cwndq);
+		TAILQ_REMOVE(&nmp->nm_cwndq, req2, r_cchain);
+		req2->r_cchain.tqe_next = NFSREQNOLIST;
+		wakeup(req2);
+	}
+	lck_mtx_unlock(&nmp->nm_lock);
+}
 
-	while ((rep->r_flags & R_BUSY)) {
-		rep->r_flags |= R_WAITING;
-		tsleep(rep, PSOCK, "repdeq", 0);
+/*
+ * Ensure req isn't in use by the timer, then dequeue it.
+ */
+static void
+nfs_reqdequeue(struct nfsreq *req)
+{
+	lck_mtx_lock(nfs_request_mutex);
+	while (req->r_lflags & RL_BUSY) {
+		req->r_lflags |= RL_WAITING;
+		msleep(&req->r_lflags, nfs_request_mutex, PSOCK, "reqdeq", NULL);
+	}
+	if (req->r_lflags & RL_QUEUED) {
+		TAILQ_REMOVE(&nfs_reqq, req, r_chain);
+		req->r_lflags &= ~RL_QUEUED;
 	}
-	TAILQ_REMOVE(&nfs_reqq, rep, r_chain);
+	lck_mtx_unlock(nfs_request_mutex);
 }
 
 /*
@@ -1687,292 +2146,314 @@ nfs_repdequeue(struct nfsreq *rep)
  * free()'d out from under it.
  */
 static void
-nfs_repbusy(struct nfsreq *rep)
+nfs_reqbusy(struct nfsreq *req)
 {
-
-	if ((rep->r_flags & R_BUSY))
-		panic("rep locked");
-	rep->r_flags |= R_BUSY;
+	if (req->r_lflags & RL_BUSY)
+		panic("req locked");
+	req->r_lflags |= RL_BUSY;
 }
 
 /*
  * Unbusy the nfsreq passed in, return the next nfsreq in the chain busied.
  */
 static struct nfsreq *
-nfs_repnext(struct nfsreq *rep)
+nfs_reqnext(struct nfsreq *req)
 {
-	struct nfsreq * nextrep;
+	struct nfsreq * nextreq;
 
-	if (rep == NULL)
+	if (req == NULL)
 		return (NULL);
 	/*
 	 * We need to get and busy the next req before signalling the
 	 * current one, otherwise wakeup() may block us and we'll race to
 	 * grab the next req.
 	 */
-	nextrep = TAILQ_NEXT(rep, r_chain);
-	if (nextrep != NULL)
-		nfs_repbusy(nextrep);
+	nextreq = TAILQ_NEXT(req, r_chain);
+	if (nextreq != NULL)
+		nfs_reqbusy(nextreq);
 	/* unbusy and signal. */
-	rep->r_flags &= ~R_BUSY;
-	if ((rep->r_flags & R_WAITING)) {
-		rep->r_flags &= ~R_WAITING;
-		wakeup(rep);
+	req->r_lflags &= ~RL_BUSY;
+	if (req->r_lflags & RL_WAITING) {
+		req->r_lflags &= ~RL_WAITING;
+		wakeup(&req->r_lflags);
 	}
-	return (nextrep);
+	return (nextreq);
 }
 
 /*
- * Nfs timer routine
- * Scan the nfsreq list and retranmit any requests that have timed out
- * To avoid retransmission attempts on STREAM sockets (in the future) make
- * sure to set the r_retry field to 0 (implies nm_retry == 0).
+ * NFS request queue timer routine
+ *
+ * Scan the NFS request queue for any requests that have timed out.
+ *
+ * Alert the system of unresponsive servers.
+ * Mark expired requests on soft mounts as terminated.
+ * For UDP, mark/signal requests for retransmission.
  */
 void
-nfs_timer(__unused void *arg)
+nfs_request_timer(__unused void *param0, __unused void *param1)
 {
-	struct nfsreq *rep;
-	mbuf_t m;
-	socket_t so;
+	struct nfsreq *req;
 	struct nfsmount *nmp;
-	int timeo;
-	int error;
-#ifndef NFS_NOSERVER
-	struct nfssvc_sock *slp;
-	u_quad_t cur_usec;
-#endif /* NFS_NOSERVER */
-	int flags, rexmit, cwnd, sent;
-	u_long xid;
+	int timeo, maxtime, finish_asyncio, error;
 	struct timeval now;
+	TAILQ_HEAD(nfs_mount_pokeq, nfsmount) nfs_mount_poke_queue;
+
+	lck_mtx_lock(nfs_request_mutex);
+	req = TAILQ_FIRST(&nfs_reqq);
+	if (req == NULL) {	/* no requests - turn timer off */
+		nfs_request_timer_on = 0;
+		lck_mtx_unlock(nfs_request_mutex);
+		return;
+	}
+
+	nfs_reqbusy(req);
+	TAILQ_INIT(&nfs_mount_poke_queue);
 
-	rep = TAILQ_FIRST(&nfs_reqq);
-	if (rep != NULL)
-		nfs_repbusy(rep);
 	microuptime(&now);
-	for ( ; rep != NULL ; rep = nfs_repnext(rep)) {
-		nmp = rep->r_nmp;
+	for ( ; req != NULL ; req = nfs_reqnext(req)) {
+		nmp = req->r_nmp;
 		if (!nmp) /* unmounted */
-		    continue;
-		if (rep->r_mrep || (rep->r_flags & R_SOFTTERM))
 			continue;
-		if (nfs_sigintr(nmp, rep, rep->r_procp))
+		if (req->r_error || req->r_nmrep.nmc_mhead)
 			continue;
-		if (nmp->nm_tprintf_initial_delay != 0 &&
-		    (rep->r_rexmit > 2 || (rep->r_flags & R_RESENDERR)) &&
-		    rep->r_lastmsg + nmp->nm_tprintf_delay < now.tv_sec) {
-			rep->r_lastmsg = now.tv_sec;
-			nfs_down(rep->r_nmp, rep->r_procp, 0, NFSSTA_TIMEO,
+		if ((error = nfs_sigintr(nmp, req, req->r_thread, 0))) {
+			if (req->r_callback.rcb_func != NULL) {
+				/* async I/O RPC needs to be finished */
+				lck_mtx_lock(&req->r_mtx);
+				req->r_error = error;
+				finish_asyncio = !(req->r_flags & R_WAITSENT);
+				wakeup(req);
+				lck_mtx_unlock(&req->r_mtx);
+				if (finish_asyncio)
+					nfs_asyncio_finish(req);
+			}
+			continue;
+		}
+
+		lck_mtx_lock(&req->r_mtx);
+
+		if (nmp->nm_tprintf_initial_delay &&
+		    ((req->r_rexmit > 2) || (req->r_flags & R_RESENDERR)) &&
+		    ((req->r_lastmsg + nmp->nm_tprintf_delay) < now.tv_sec)) {
+			req->r_lastmsg = now.tv_sec;
+			nfs_down(req->r_nmp, req->r_thread, 0, NFSSTA_TIMEO,
 				"not responding");
-			rep->r_flags |= R_TPRINTFMSG;
+			req->r_flags |= R_TPRINTFMSG;
+			lck_mtx_lock(&nmp->nm_lock);
 			if (!(nmp->nm_state & NFSSTA_MOUNTED)) {
+				lck_mtx_unlock(&nmp->nm_lock);
 				/* we're not yet completely mounted and */
 				/* we can't complete an RPC, so we fail */
 				OSAddAtomic(1, (SInt32*)&nfsstats.rpctimeouts);
-				nfs_softterm(rep);
+				nfs_softterm(req);
+				finish_asyncio = ((req->r_callback.rcb_func != NULL) && !(req->r_flags & R_WAITSENT));
+				wakeup(req);
+				lck_mtx_unlock(&req->r_mtx);
+				if (finish_asyncio)
+					nfs_asyncio_finish(req);
 				continue;
 			}
+			lck_mtx_unlock(&nmp->nm_lock);
 		}
-		if (rep->r_rtt >= 0) {
-			rep->r_rtt++;
-			if (nmp->nm_flag & NFSMNT_DUMBTIMR)
-				timeo = nmp->nm_timeo;
-			else
-				timeo = NFS_RTO(nmp, proct[rep->r_procnum]);
-			/* ensure 62.5 ms floor */
-			while (16 * timeo < hz)
-			    timeo *= 2;
-			if (nmp->nm_timeouts > 0)
-				timeo *= nfs_backoff[nmp->nm_timeouts - 1];
-			if (rep->r_rtt <= timeo)
-				continue;
-			if (nmp->nm_timeouts < 8)
-				nmp->nm_timeouts++;
-		}
+
 		/*
-		 * Check for too many retransmits.  This is never true for
-		 * 'hard' mounts because we set r_retry to NFS_MAXREXMIT + 1
-		 * and never allow r_rexmit to be more than NFS_MAXREXMIT.
+		 * Put a reasonable limit on the maximum timeout,
+		 * and reduce that limit when soft mounts get timeouts or are in reconnect.
 		 */
-		if (rep->r_rexmit >= rep->r_retry) {	/* too many */
-			OSAddAtomic(1, (SInt32*)&nfsstats.rpctimeouts);
-			nfs_softterm(rep);
-			continue;
-		}
-		if (nmp->nm_sotype != SOCK_DGRAM) {
-			if (++rep->r_rexmit > NFS_MAXREXMIT)
-				rep->r_rexmit = NFS_MAXREXMIT;
-			continue;
-		}
-		if ((so = nmp->nm_so) == NULL)
-			continue;
+		if (!(nmp->nm_flag & NFSMNT_SOFT))
+			maxtime = NFS_MAXTIMEO;
+		else if ((req->r_flags & R_SETUP) || ((nmp->nm_reconnect_start <= 0) || ((now.tv_sec - nmp->nm_reconnect_start) < 8)))
+			maxtime = (NFS_MAXTIMEO / (nmp->nm_timeouts+1))/2;
+		else
+			maxtime = NFS_MINTIMEO/4;
 
 		/*
-		 * If there is enough space and the window allows..
-		 *	Resend it
-		 * Set r_rtt to -1 in case we fail to send it now.
+		 * Check for request timeout.
 		 */
-		rep->r_rtt = -1;
-		if (((nmp->nm_flag & NFSMNT_DUMBTIMR) ||
-		    (rep->r_flags & R_SENT) ||
-		    nmp->nm_sent < nmp->nm_cwnd) &&
-		   (mbuf_copym(rep->r_mreq, 0, MBUF_COPYALL, MBUF_DONTWAIT, &m) == 0)){
-			struct msghdr	msg;
-			/*
-			 * Iff first send, start timing
-			 * else turn timing off, backoff timer
-			 * and divide congestion window by 2.
-			 * We update these *before* the send to avoid
-			 * racing against receiving the reply.
-			 * We save them so we can restore them on send error.
-			 */
-			flags = rep->r_flags;
-			rexmit = rep->r_rexmit;
-			cwnd = nmp->nm_cwnd;
-			sent = nmp->nm_sent;
-			xid = rep->r_xid;
-			if (rep->r_flags & R_SENT) {
-				rep->r_flags &= ~R_TIMING;
-				if (++rep->r_rexmit > NFS_MAXREXMIT)
-					rep->r_rexmit = NFS_MAXREXMIT;
-				nmp->nm_cwnd >>= 1;
-				if (nmp->nm_cwnd < NFS_CWNDSCALE)
-					nmp->nm_cwnd = NFS_CWNDSCALE;
-				OSAddAtomic(1, (SInt32*)&nfsstats.rpcretries);
+		if (req->r_rtt >= 0) {
+			req->r_rtt++;
+			lck_mtx_lock(&nmp->nm_lock);
+			if (req->r_flags & R_RESENDERR) {
+				/* with resend errors, retry every few seconds */
+				timeo = 4*hz;
 			} else {
-				rep->r_flags |= R_SENT;
-				nmp->nm_sent += NFS_CWNDSCALE;
-			}
-			FSDBG(535, xid, rep, nmp->nm_sent, nmp->nm_cwnd);
+				if (req->r_procnum == NFSPROC_NULL && req->r_gss_ctx != NULL)
+					timeo = NFS_MINIDEMTIMEO; // gss context setup
+				else if (nmp->nm_flag & NFSMNT_DUMBTIMR)
+					timeo = nmp->nm_timeo;
+				else
+					timeo = NFS_RTO(nmp, proct[req->r_procnum]);
 
-	 		bzero(&msg, sizeof(msg));
-			if ((nmp->nm_flag & NFSMNT_NOCONN) == NFSMNT_NOCONN) {
-				msg.msg_name = mbuf_data(nmp->nm_nam);
-				msg.msg_namelen = mbuf_len(nmp->nm_nam);
+				/* ensure 62.5 ms floor */
+				while (16 * timeo < hz)
+					timeo *= 2;
+				if (nmp->nm_timeouts > 0)
+					timeo *= nfs_backoff[nmp->nm_timeouts - 1];
+			}
+			/* limit timeout to max */
+			if (timeo > maxtime)
+				timeo = maxtime;
+			if (req->r_rtt <= timeo) {
+				lck_mtx_unlock(&nmp->nm_lock);
+				lck_mtx_unlock(&req->r_mtx);
+				continue;
 			}
-			error = sock_sendmbuf(so, &msg, m, MSG_DONTWAIT, NULL);
+			/* The request has timed out */
+			NFS_SOCK_DBG(("nfs timeout: proc %d %d xid %llx rtt %d to %d # %d, t %ld/%d\n",
+				req->r_procnum, proct[req->r_procnum],
+				req->r_xid, req->r_rtt, timeo, nmp->nm_timeouts,
+				(now.tv_sec - req->r_start)*NFS_HZ, maxtime));
+			if (nmp->nm_timeouts < 8)
+				nmp->nm_timeouts++;
+			/* if it's been a few seconds, try poking the socket */
+			if ((nmp->nm_sotype == SOCK_STREAM) &&
+			    ((now.tv_sec - req->r_start) >= 3) &&
+			    !(nmp->nm_sockflags & NMSOCK_POKE)) {
+				nmp->nm_sockflags |= NMSOCK_POKE;
+				TAILQ_INSERT_TAIL(&nfs_mount_poke_queue, nmp, nm_pokeq);
+			}
+			lck_mtx_unlock(&nmp->nm_lock);
+		}
 
-			FSDBG(535, xid, error, sent, cwnd);
+		/* For soft mounts (& SETUPs), check for too many retransmits/timeout. */
+		if (((nmp->nm_flag & NFSMNT_SOFT) || (req->r_flags & R_SETUP)) &&
+		    ((req->r_rexmit >= req->r_retry) || /* too many */
+		     ((now.tv_sec - req->r_start)*NFS_HZ > maxtime))) { /* too long */
+			OSAddAtomic(1, (SInt32*)&nfsstats.rpctimeouts);
+			lck_mtx_lock(&nmp->nm_lock);
+			if (!(nmp->nm_state & NFSSTA_TIMEO)) {
+				lck_mtx_unlock(&nmp->nm_lock);
+				/* make sure we note the unresponsive server */
+				/* (maxtime may be less than tprintf delay) */
+				nfs_down(req->r_nmp, req->r_thread, 0, NFSSTA_TIMEO,
+					"not responding");
+				req->r_lastmsg = now.tv_sec;
+				req->r_flags |= R_TPRINTFMSG;
+			} else {
+				lck_mtx_unlock(&nmp->nm_lock);
+			}
+			NFS_SOCK_DBG(("nfs timer TERMINATE: p %d x 0x%llx f 0x%x rtt %d t %ld\n",
+				req->r_procnum, req->r_xid, req->r_flags, req->r_rtt,
+				now.tv_sec - req->r_start));
+			nfs_softterm(req);
+			finish_asyncio = ((req->r_callback.rcb_func != NULL) && !(req->r_flags & R_WAITSENT));
+			wakeup(req);
+			lck_mtx_unlock(&req->r_mtx);
+			if (finish_asyncio)
+				nfs_asyncio_finish(req);
+			continue;
+		}
 
-			if (error) {
-				if (error == EWOULDBLOCK) {
-					rep->r_flags = flags;
-					rep->r_rexmit = rexmit;
-					nmp->nm_cwnd = cwnd;
-					nmp->nm_sent = sent;
-					rep->r_xid = xid;
-				}
-				else {
-					if (NFSIGNORE_SOERROR(nmp->nm_sotype, error)) {
-						int clearerror;
-						int optlen = sizeof(clearerror);
-						sock_getsockopt(nmp->nm_so, SOL_SOCKET, SO_ERROR, &clearerror, &optlen);
-					}
-					rep->r_flags  = flags | R_RESENDERR;
-					rep->r_rexmit = rexmit;
-					nmp->nm_cwnd = cwnd;
-					nmp->nm_sent = sent;
-					if (flags & R_SENT)
-						OSAddAtomic(-1, (SInt32*)&nfsstats.rpcretries);
-				}
-			} else
-				rep->r_rtt = 0;
+		/* for TCP, only resend if explicitly requested */
+		if ((nmp->nm_sotype == SOCK_STREAM) && !(req->r_flags & R_MUSTRESEND)) {
+			if (++req->r_rexmit > NFS_MAXREXMIT)
+				req->r_rexmit = NFS_MAXREXMIT;
+			req->r_rtt = 0;
+			lck_mtx_unlock(&req->r_mtx);
+			continue;
 		}
-	}
-	microuptime(&now);
-#ifndef NFS_NOSERVER
-	/*
-	 * Scan the write gathering queues for writes that need to be
-	 * completed now.
-	 */
-	cur_usec = (u_quad_t)now.tv_sec * 1000000 + (u_quad_t)now.tv_usec;
-	lck_mtx_lock(nfsd_mutex);
-	TAILQ_FOREACH(slp, &nfssvc_sockhead, ns_chain) {
-	    if (slp->ns_wgtime && (slp->ns_wgtime <= cur_usec))
-		nfsrv_wakenfsd(slp);
-	}
-	while ((slp = TAILQ_FIRST(&nfssvc_deadsockhead))) {
-		if ((slp->ns_timestamp + 5) > now.tv_sec)
-			break;
-		TAILQ_REMOVE(&nfssvc_deadsockhead, slp, ns_chain);
-		nfsrv_slpfree(slp);
-	}
-	lck_mtx_unlock(nfsd_mutex);
-#endif /* NFS_NOSERVER */
 
-	if (nfsbuffreeuptimestamp + 30 <= now.tv_sec) {
 		/*
-		 * We haven't called nfs_buf_freeup() in a little while.
-		 * So, see if we can free up any stale/unused bufs now.
+		 * The request needs to be (re)sent.  Kick the requester to resend it.
+		 * (unless it's already marked as needing a resend)
 		 */
-		nfs_buf_freeup(1);
-	}
-
-	timeout(nfs_timer_funnel, (void *)0, nfs_ticks);
-
+		if ((req->r_flags & R_MUSTRESEND) && (req->r_rtt == -1)) {
+			lck_mtx_unlock(&req->r_mtx);
+			continue;
+		}
+		NFS_SOCK_DBG(("nfs timer mark resend: p %d x 0x%llx f 0x%x rtt %d\n",
+			req->r_procnum, req->r_xid, req->r_flags, req->r_rtt));
+		req->r_flags |= R_MUSTRESEND;
+		req->r_rtt = -1;
+		wakeup(req);
+		if ((req->r_flags & (R_ASYNC|R_ASYNCWAIT)) == R_ASYNC)
+			nfs_asyncio_resend(req);
+		lck_mtx_unlock(&req->r_mtx);
+	}
+
+	lck_mtx_unlock(nfs_request_mutex);
+
+	/* poke any sockets */
+	while ((nmp = TAILQ_FIRST(&nfs_mount_poke_queue))) {
+		TAILQ_REMOVE(&nfs_mount_poke_queue, nmp, nm_pokeq);
+		nfs_sock_poke(nmp);
+		lck_mtx_lock(&nmp->nm_lock);
+		nmp->nm_sockflags &= ~NMSOCK_POKE;
+		if (!(nmp->nm_state & NFSSTA_MOUNTED))
+			wakeup(&nmp->nm_sockflags);
+		lck_mtx_unlock(&nmp->nm_lock);
+	}
+
+	nfs_interval_timer_start(nfs_request_timer_call, NFS_REQUESTDELAY);
 }
 
+/*
+ * check a thread's proc for the "noremotehang" flag.
+ */
+int
+nfs_noremotehang(thread_t thd)
+{
+	proc_t p = thd ? get_bsdthreadtask_info(thd) : NULL;
+	return (p && proc_noremotehang(p));
+}
 
 /*
  * Test for a termination condition pending on the process.
  * This is used to determine if we need to bail on a mount.
- * EIO is returned if there has been a soft timeout.
+ * ETIMEDOUT is returned if there has been a soft timeout.
  * EINTR is returned if there is a signal pending that is not being ignored
  * and the mount is interruptable, or if we are a thread that is in the process
  * of cancellation (also SIGKILL posted).
  */
 int
-nfs_sigintr(nmp, rep, p)
-	struct nfsmount *nmp;
-	struct nfsreq *rep;
-	proc_t p;
+nfs_sigintr(struct nfsmount *nmp, struct nfsreq *req, thread_t thd, int nmplocked)
 {
-	sigset_t pending_sigs;
-	int context_good = 0;
-	struct nfsmount *repnmp;
-	extern proc_t kernproc;
+	int error = 0;
 
 	if (nmp == NULL)
 		return (ENXIO);
-	if (rep != NULL) {
-		repnmp = rep->r_nmp;
-		/* we've had a forced unmount. */
-		if (repnmp == NULL)
-			return (ENXIO);
-		/* request has timed out on a 'soft' mount. */
-		if (rep->r_flags & R_SOFTTERM)
-			return (EIO);
-		/*
-		 * We're in the progress of a force unmount and there's
-		 * been a timeout we're dead and fail IO.
-		 */
-		if ((repnmp->nm_state & (NFSSTA_FORCE|NFSSTA_TIMEO)) ==
-		   (NFSSTA_FORCE|NFSSTA_TIMEO))
-			return (EIO);
+
+	if (req && (req->r_flags & R_SOFTTERM))
+		return (ETIMEDOUT); /* request has been terminated. */
+
+	/*
+	 * If we're in the progress of a force unmount and there's
+	 * been a timeout, we're dead and fail IO.
+	 */
+	if (!nmplocked)
+		lck_mtx_lock(&nmp->nm_lock);
+	if ((nmp->nm_state & NFSSTA_FORCE) &&
+	    (nmp->nm_state & (NFSSTA_TIMEO|NFSSTA_JUKEBOXTIMEO|NFSSTA_LOCKTIMEO))) {
+		error = EIO;
+	} else if (nmp->nm_mountp->mnt_kern_flag & MNTK_FRCUNMOUNT) {
 		/* Someone is unmounting us, go soft and mark it. */
-		if (repnmp->nm_mountp->mnt_kern_flag & MNTK_FRCUNMOUNT) {
-			repnmp->nm_flag |= NFSMNT_SOFT;
-			nmp->nm_state |= NFSSTA_FORCE;
-		}
-		/*
-		 * If the mount is hung and we've requested not to hang
-		 * on remote filesystems, then bail now.
-		 */
-		if (p != NULL && (proc_noremotehang(p)) != 0 &&
-		    (repnmp->nm_state & NFSSTA_TIMEO) != 0)
-			return (EIO);
+		nmp->nm_flag |= NFSMNT_SOFT;
+		nmp->nm_state |= NFSSTA_FORCE;
 	}
-	/* XXX: is this valid?  this probably should be an assertion. */
-	if (p == NULL)
+
+	/*
+	 * If the mount is hung and we've requested not to hang
+	 * on remote filesystems, then bail now.
+	 */
+	if (!error && (nmp->nm_state & NFSSTA_TIMEO) && nfs_noremotehang(thd))
+		error = EIO;
+
+	if (!nmplocked)
+		lck_mtx_unlock(&nmp->nm_lock);
+	if (error)
+		return (error);
+
+	/* may not have a thread for async I/O */
+	if (thd == NULL)
 		return (0);
 
-	/* Is this thread belongs to kernel task; then abort check  is not needed */
-	if ((current_proc() != kernproc) && current_thread_aborted()) {
+	/* If this thread belongs to kernel task; then abort check is not needed */
+	if ((current_proc() != kernproc) && current_thread_aborted())
 		return (EINTR);
-	}
-	/* mask off thread and process blocked signals. */
 
-	pending_sigs = proc_pendingsignals(p, NFSINT_SIGMASK);
-	if (pending_sigs && (nmp->nm_flag & NFSMNT_INT) != 0)
+	/* mask off thread and process blocked signals. */
+	if ((nmp->nm_flag & NFSMNT_INT) &&
+	    proc_pendingsignals(get_bsdthreadtask_info(thd), NFSINT_SIGMASK))
 		return (EINTR);
 	return (0);
 }
@@ -1984,162 +2465,232 @@ nfs_sigintr(nmp, rep, p)
  * in progress when a reconnect is necessary.
  */
 int
-nfs_sndlock(rep)
-	struct nfsreq *rep;
+nfs_sndlock(struct nfsreq *req)
 {
+	struct nfsmount *nmp = req->r_nmp;
 	int *statep;
-	proc_t p;
-	int error, slpflag = 0, slptimeo = 0;
+	int error = 0, slpflag = 0;
+	struct timespec ts = { 0, 0 };
 
-	if (rep->r_nmp == NULL)
+	if (nmp == NULL)
 		return (ENXIO);
-	statep = &rep->r_nmp->nm_state;
 
-	p = rep->r_procp;
-	if (rep->r_nmp->nm_flag & NFSMNT_INT)
+	lck_mtx_lock(&nmp->nm_lock);
+	statep = &nmp->nm_state;
+
+	if ((nmp->nm_flag & NFSMNT_INT) && req->r_thread)
 		slpflag = PCATCH;
-	while (*statep & NFSSTA_SNDLOCK) {
-		error = nfs_sigintr(rep->r_nmp, rep, p);
-		if (error)
-			return (error);
+	while (!error && (*statep & NFSSTA_SNDLOCK)) {
+		if ((error = nfs_sigintr(nmp, req, req->r_thread, 1)))
+			break;
 		*statep |= NFSSTA_WANTSND;
-		if (p != NULL && (proc_noremotehang(p)) != 0)
-			slptimeo = hz;
-		tsleep((caddr_t)statep, slpflag | (PZERO - 1), "nfsndlck", slptimeo);
+		if (nfs_noremotehang(req->r_thread))
+			ts.tv_sec = 1;
+		error = msleep(statep, &nmp->nm_lock, slpflag | (PZERO - 1), "nfsndlck", &ts);
+		if (error == EWOULDBLOCK)
+			error = 0;
 		if (slpflag == PCATCH) {
 			slpflag = 0;
-			slptimeo = 2 * hz;
+			ts.tv_sec = 2;
 		}
-		/*
-		 * Make sure while we slept that the mountpoint didn't go away.
-		 * nfs_sigintr and callers expect it in tact.
-		 */
-		if (!rep->r_nmp) 
-			return (ENXIO); /* don't have lock until out of loop */
 	}
-	*statep |= NFSSTA_SNDLOCK;
-	return (0);
+	if (!error)
+		*statep |= NFSSTA_SNDLOCK;
+	lck_mtx_unlock(&nmp->nm_lock);
+	return (error);
 }
 
 /*
  * Unlock the stream socket for others.
  */
 void
-nfs_sndunlock(rep)
-	struct nfsreq *rep;
+nfs_sndunlock(struct nfsreq *req)
 {
-	int *statep;
+	struct nfsmount *nmp = req->r_nmp;
+	int *statep, wake = 0;
 
-	if (rep->r_nmp == NULL)
+	if (nmp == NULL)
 		return;
-	statep = &rep->r_nmp->nm_state;
+	lck_mtx_lock(&nmp->nm_lock);
+	statep = &nmp->nm_state;
 	if ((*statep & NFSSTA_SNDLOCK) == 0)
 		panic("nfs sndunlock");
 	*statep &= ~NFSSTA_SNDLOCK;
 	if (*statep & NFSSTA_WANTSND) {
 		*statep &= ~NFSSTA_WANTSND;
-		wakeup((caddr_t)statep);
+		wake = 1;
 	}
+	lck_mtx_unlock(&nmp->nm_lock);
+	if (wake)
+		wakeup(statep);
 }
 
-static int
-nfs_rcvlock(struct nfsreq *rep)
+#endif /* NFSCLIENT */
+
+#if NFSSERVER
+
+/*
+ * Generate the rpc reply header
+ * siz arg. is used to decide if adding a cluster is worthwhile
+ */
+int
+nfsrv_rephead(
+	struct nfsrv_descript *nd,
+	__unused struct nfsrv_sock *slp,
+	struct nfsm_chain *nmrepp,
+	size_t siz)
 {
-	int *statep;
-	int error, slpflag, slptimeo = 0;
+	mbuf_t mrep;
+	u_long *tl;
+	struct nfsm_chain nmrep;
+	int err, error;
 
-	/* make sure we still have our mountpoint */
-	if (!rep->r_nmp) {
-		if (rep->r_mrep != NULL)
-			return (EALREADY);
-		return (ENXIO);
-	}
+	err = nd->nd_repstat;
+	if (err && (nd->nd_vers == NFS_VER2))
+		siz = 0;
 
-	statep = &rep->r_nmp->nm_state;
-	FSDBG_TOP(534, rep->r_xid, rep, rep->r_nmp, *statep);
-	if (rep->r_nmp->nm_flag & NFSMNT_INT)
-		slpflag = PCATCH;
-	else
-		slpflag = 0;
-	while (*statep & NFSSTA_RCVLOCK) {
-		if ((error = nfs_sigintr(rep->r_nmp, rep, rep->r_procp))) {
-			FSDBG_BOT(534, rep->r_xid, rep, rep->r_nmp, 0x100);
-			return (error);
-		} else if (rep->r_mrep != NULL) {
-			/*
-			 * Don't bother sleeping if reply already arrived
-			 */
-			FSDBG_BOT(534, rep->r_xid, rep, rep->r_nmp, 0x101);
-			return (EALREADY);
+	/*
+	 * If this is a big reply, use a cluster else
+	 * try and leave leading space for the lower level headers.
+	 */
+	siz += RPC_REPLYSIZ;
+	if (siz >= nfs_mbuf_minclsize) {
+		error = mbuf_getpacket(MBUF_WAITOK, &mrep);
+	} else {
+		error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &mrep);
+	}
+	if (error) {
+		/* unable to allocate packet */
+		/* XXX should we keep statistics for these errors? */
+		return (error);
+	}
+	if (siz < nfs_mbuf_minclsize) {
+		/* leave space for lower level headers */
+		tl = mbuf_data(mrep);
+		tl += 80/sizeof(*tl);  /* XXX max_hdr? XXX */
+		mbuf_setdata(mrep, tl, 6 * NFSX_UNSIGNED);
+	}
+	nfsm_chain_init(&nmrep, mrep);
+	nfsm_chain_add_32(error, &nmrep, nd->nd_retxid);
+	nfsm_chain_add_32(error, &nmrep, RPC_REPLY);
+	if (err == ERPCMISMATCH || (err & NFSERR_AUTHERR)) {
+		nfsm_chain_add_32(error, &nmrep, RPC_MSGDENIED);
+		if (err & NFSERR_AUTHERR) {
+			nfsm_chain_add_32(error, &nmrep, RPC_AUTHERR);
+			nfsm_chain_add_32(error, &nmrep, (err & ~NFSERR_AUTHERR));
+		} else {
+			nfsm_chain_add_32(error, &nmrep, RPC_MISMATCH);
+			nfsm_chain_add_32(error, &nmrep, RPC_VER2);
+			nfsm_chain_add_32(error, &nmrep, RPC_VER2);
 		}
-		FSDBG(534, rep->r_xid, rep, rep->r_nmp, 0x102);
-		*statep |= NFSSTA_WANTRCV;
-		/*
-		 * We need to poll if we're P_NOREMOTEHANG so that we
-		 * call nfs_sigintr periodically above.
-		 */
-		if (rep->r_procp != NULL &&
-		    (proc_noremotehang(rep->r_procp)) != 0)
-			slptimeo = hz;
-		tsleep((caddr_t)statep, slpflag | (PZERO - 1), "nfsrcvlk", slptimeo);
-		if (slpflag == PCATCH) {
-			slpflag = 0;
-			slptimeo = 2 * hz;
+	} else {
+		/* reply status */
+		nfsm_chain_add_32(error, &nmrep, RPC_MSGACCEPTED);
+		if (nd->nd_gss_context != NULL) {
+			/* RPCSEC_GSS verifier */
+			error = nfs_gss_svc_verf_put(nd, &nmrep);
+			if (error) {
+				nfsm_chain_add_32(error, &nmrep, RPC_SYSTEM_ERR);
+				goto done;
+			}
+		} else {
+			/* RPCAUTH_NULL verifier */
+			nfsm_chain_add_32(error, &nmrep, RPCAUTH_NULL);
+			nfsm_chain_add_32(error, &nmrep, 0);
 		}
-		/*
-		 * Make sure while we slept that the mountpoint didn't go away.
-		 * nfs_sigintr and caller nfs_reply expect it intact.
-		 */
-		if (!rep->r_nmp)  {
-			FSDBG_BOT(534, rep->r_xid, rep, rep->r_nmp, 0x103);
-			return (ENXIO); /* don't have lock until out of loop */
+		/* accepted status */
+		switch (err) {
+		case EPROGUNAVAIL:
+			nfsm_chain_add_32(error, &nmrep, RPC_PROGUNAVAIL);
+			break;
+		case EPROGMISMATCH:
+			nfsm_chain_add_32(error, &nmrep, RPC_PROGMISMATCH);
+			/* XXX hard coded versions? */
+			nfsm_chain_add_32(error, &nmrep, NFS_VER2);
+			nfsm_chain_add_32(error, &nmrep, NFS_VER3);
+			break;
+		case EPROCUNAVAIL:
+			nfsm_chain_add_32(error, &nmrep, RPC_PROCUNAVAIL);
+			break;
+		case EBADRPC:
+			nfsm_chain_add_32(error, &nmrep, RPC_GARBAGE);
+			break;
+		default:
+			nfsm_chain_add_32(error, &nmrep, RPC_SUCCESS);
+			if (nd->nd_gss_context != NULL)
+				error = nfs_gss_svc_prepare_reply(nd, &nmrep);
+			if (err != NFSERR_RETVOID)
+				nfsm_chain_add_32(error, &nmrep,
+					(err ? nfsrv_errmap(nd, err) : 0));
+			break;
 		}
 	}
-	/*
-	 * nfs_reply will handle it if reply already arrived.
-	 * (We may have slept or been preempted).
-	 */
-	FSDBG_BOT(534, rep->r_xid, rep, rep->r_nmp, *statep);
-	*statep |= NFSSTA_RCVLOCK;
+
+done:
+	nfsm_chain_build_done(error, &nmrep);
+	if (error) {
+		/* error composing reply header */
+		/* XXX should we keep statistics for these errors? */
+		mbuf_freem(mrep);
+		return (error);
+	}
+
+	*nmrepp = nmrep;
+	if ((err != 0) && (err != NFSERR_RETVOID))
+		OSAddAtomic(1, (SInt32*)&nfsstats.srvrpc_errs);
 	return (0);
 }
 
 /*
- * Unlock the stream socket for others.
+ * The nfs server send routine.
+ *
+ * - return EINTR or ERESTART if interrupted by a signal
+ * - return EPIPE if a connection is lost for connection based sockets (TCP...)
+ * - do any cleanup required by recoverable socket errors (???)
  */
-static void
-nfs_rcvunlock(struct nfsreq *rep)
+int
+nfsrv_send(struct nfsrv_sock *slp, mbuf_t nam, mbuf_t top)
 {
-	int *statep;
-	
-	if (rep->r_nmp == NULL)
-		return;
-	statep = &rep->r_nmp->nm_state;
+	int error;
+	socket_t so = slp->ns_so;
+	struct sockaddr *sendnam;
+	struct msghdr msg;
 
-	FSDBG(533, statep, *statep, 0, 0);
-	if ((*statep & NFSSTA_RCVLOCK) == 0)
-		panic("nfs rcvunlock");
-	*statep &= ~NFSSTA_RCVLOCK;
-	if (*statep & NFSSTA_WANTRCV) {
-		*statep &= ~NFSSTA_WANTRCV;
-		wakeup((caddr_t)statep);
+	bzero(&msg, sizeof(msg));
+	if (nam && !sock_isconnected(so) && (slp->ns_sotype != SOCK_STREAM)) {
+		if ((sendnam = mbuf_data(nam))) {
+			msg.msg_name = (caddr_t)sendnam;
+			msg.msg_namelen = sendnam->sa_len;
+		}
 	}
-}
+	error = sock_sendmbuf(so, &msg, top, 0, NULL);
+	if (!error)
+		return (0);
+	log(LOG_INFO, "nfsd send error %d\n", error);
+
+	if ((error == EWOULDBLOCK) && (slp->ns_sotype == SOCK_STREAM))
+		error = EPIPE;  /* zap TCP sockets if they time out on send */
 
+	/* Handle any recoverable (soft) socket errors here. (???) */
+	if (error != EINTR && error != ERESTART && error != EIO &&
+		error != EWOULDBLOCK && error != EPIPE)
+		error = 0;
+
+	return (error);
+}
 
-#ifndef NFS_NOSERVER
 /*
  * Socket upcall routine for the nfsd sockets.
- * The caddr_t arg is a pointer to the "struct nfssvc_sock".
+ * The caddr_t arg is a pointer to the "struct nfsrv_sock".
  * Essentially do as much as possible non-blocking, else punt and it will
  * be called with MBUF_WAITOK from an nfsd.
  */
 void
 nfsrv_rcv(socket_t so, caddr_t arg, int waitflag)
 {
-	struct nfssvc_sock *slp = (struct nfssvc_sock *)arg;
+	struct nfsrv_sock *slp = (struct nfsrv_sock *)arg;
 
-	if (!nfs_numnfsd || !(slp->ns_flag & SLP_VALID))
+	if (!nfsd_thread_count || !(slp->ns_flag & SLP_VALID))
 		return;
 
 	lck_rw_lock_exclusive(&slp->ns_rwlock);
@@ -2147,7 +2698,7 @@ nfsrv_rcv(socket_t so, caddr_t arg, int waitflag)
 	/* Note: ns_rwlock gets dropped when called with MBUF_DONTWAIT */
 }
 void
-nfsrv_rcv_locked(socket_t so, struct nfssvc_sock *slp, int waitflag)
+nfsrv_rcv_locked(socket_t so, struct nfsrv_sock *slp, int waitflag)
 {
 	mbuf_t m, mp, mhck, m2;
 	int ns_flag=0, error;
@@ -2187,7 +2738,7 @@ nfsrv_rcv_locked(socket_t so, struct nfssvc_sock *slp, int waitflag)
 		error = sock_receivembuf(so, NULL, &mp, MSG_DONTWAIT, &bytes_read);
 		if (error || mp == NULL) {
 			if (error == EWOULDBLOCK)
-				ns_flag = SLP_NEEDQ;
+				ns_flag = (waitflag == MBUF_DONTWAIT) ? SLP_NEEDQ : 0;
 			else
 				ns_flag = SLP_DISCONN;
 			goto dorecs;
@@ -2217,6 +2768,12 @@ nfsrv_rcv_locked(socket_t so, struct nfssvc_sock *slp, int waitflag)
 		}
 	} else {
 		struct sockaddr_storage	nam;
+
+		if (slp->ns_reccnt >= nfsrv_sock_max_rec_queue_length) {
+			/* already have max # RPC records queued on this socket */
+			ns_flag = SLP_NEEDQ;
+			goto dorecs;
+		}
 		
 		bzero(&msg, sizeof(msg));
 		msg.msg_name = (caddr_t)&nam;
@@ -2241,29 +2798,14 @@ nfsrv_rcv_locked(socket_t so, struct nfssvc_sock *slp, int waitflag)
 				}
 				if (slp->ns_recend)
 					mbuf_setnextpkt(slp->ns_recend, m);
-				else
+				else {
 					slp->ns_rec = m;
+					slp->ns_flag |= SLP_DOREC;
+				}
 				slp->ns_recend = m;
 				mbuf_setnextpkt(m, NULL);
+				slp->ns_reccnt++;
 			}
-#if 0
-			if (error) {
-				/*
-				 * This may be needed in the future to support
-				 * non-byte-stream connection-oriented protocols
-				 * such as SCTP.
-				 */
-				/*
-				 * This (slp->ns_sotype == SOCK_STREAM) should really
-				 * be a check for PR_CONNREQUIRED.
-				 */
-				if ((slp->ns_sotype == SOCK_STREAM)
-					&& error != EWOULDBLOCK) {
-					ns_flag = SLP_DISCONN;
-					goto dorecs;
-				}
-			}
-#endif
 		} while (mp);
 	}
 
@@ -2274,9 +2816,9 @@ dorecs:
 	if (ns_flag)
 		slp->ns_flag |= ns_flag;
 	if (waitflag == MBUF_DONTWAIT) {
-		int wake = (slp->ns_rec || (slp->ns_flag & (SLP_NEEDQ | SLP_DISCONN)));
+		int wake = (slp->ns_flag & SLP_WORKTODO);
 		lck_rw_done(&slp->ns_rwlock);
-		if (wake && nfs_numnfsd) {
+		if (wake && nfsd_thread_count) {
 			lck_mtx_lock(nfsd_mutex);
 			nfsrv_wakenfsd(slp);
 			lck_mtx_unlock(nfsd_mutex);
@@ -2290,9 +2832,7 @@ dorecs:
  * can sleep.
  */
 static int
-nfsrv_getstream(slp, waitflag)
-	struct nfssvc_sock *slp;
-	int waitflag;
+nfsrv_getstream(struct nfsrv_sock *slp, int waitflag)
 {
 	mbuf_t m;
 	char *cp1, *cp2, *mdata;
@@ -2426,8 +2966,10 @@ nfsrv_getstream(slp, waitflag)
 	    if (slp->ns_flag & SLP_LASTFRAG) {
 		if (slp->ns_recend)
 		    mbuf_setnextpkt(slp->ns_recend, slp->ns_frag);
-		else
+		else {
 		    slp->ns_rec = slp->ns_frag;
+		    slp->ns_flag |= SLP_DOREC;
+		}
 		slp->ns_recend = slp->ns_frag;
 		slp->ns_frag = NULL;
 	    }
@@ -2438,18 +2980,18 @@ nfsrv_getstream(slp, waitflag)
  * Parse an RPC header.
  */
 int
-nfsrv_dorec(slp, nfsd, ndp)
-	struct nfssvc_sock *slp;
-	struct nfsd *nfsd;
-	struct nfsrv_descript **ndp;
+nfsrv_dorec(
+	struct nfsrv_sock *slp,
+	struct nfsd *nfsd,
+	struct nfsrv_descript **ndp)
 {
 	mbuf_t m;
 	mbuf_t nam;
 	struct nfsrv_descript *nd;
-	int error;
+	int error = 0;
 
 	*ndp = NULL;
-	if ((slp->ns_flag & SLP_VALID) == 0 || (slp->ns_rec == NULL))
+	if (!(slp->ns_flag & (SLP_VALID|SLP_DOREC)) || (slp->ns_rec == NULL))
 		return (ENOBUFS);
 	MALLOC_ZONE(nd, struct nfsrv_descript *,
 			sizeof (struct nfsrv_descript), M_NFSRVDESC, M_WAITOK);
@@ -2459,8 +3001,11 @@ nfsrv_dorec(slp, nfsd, ndp)
 	slp->ns_rec = mbuf_nextpkt(m);
 	if (slp->ns_rec)
 		mbuf_setnextpkt(m, NULL);
-	else
+	else {
+		slp->ns_flag &= ~SLP_DOREC;
 		slp->ns_recend = NULL;
+	}
+	slp->ns_reccnt--;
 	if (mbuf_type(m) == MBUF_TYPE_SONAME) {
 		nam = m;
 		m = mbuf_next(m);
@@ -2468,16 +3013,17 @@ nfsrv_dorec(slp, nfsd, ndp)
 			panic("nfsrv_dorec: mbuf_setnext failed %d\n", error);
 	} else
 		nam = NULL;
-	nd->nd_md = nd->nd_mrep = m;
 	nd->nd_nam2 = nam;
-	nd->nd_dpos = mbuf_data(m);
-	error = nfs_getreq(nd, nfsd, TRUE);
+	nfsm_chain_dissect_init(error, &nd->nd_nmreq, m);
+	if (!error)
+		error = nfsrv_getreq(nd);
 	if (error) {
 		if (nam)
 			mbuf_freem(nam);
-		FREE_ZONE((caddr_t)nd,	sizeof *nd, M_NFSRVDESC);
+		FREE_ZONE(nd, sizeof(*nd), M_NFSRVDESC);
 		return (error);
 	}
+	nd->nd_mrep = NULL;
 	*ndp = nd;
 	nfsd->nfsd_nd = nd;
 	return (0);
@@ -2488,270 +3034,150 @@ nfsrv_dorec(slp, nfsd, ndp)
  * - verify it
  * - fill in the cred struct.
  */
-int
-nfs_getreq(nd, nfsd, has_header)
-	struct nfsrv_descript *nd;
-	struct nfsd *nfsd;
-	int has_header;
+static int
+nfsrv_getreq(struct nfsrv_descript *nd)
 {
+	struct nfsm_chain *nmreq;
 	int len, i;
-	u_long *tl;
-	long t1;
-	uio_t uiop;
-	caddr_t dpos, cp2, cp;
 	u_long nfsvers, auth_type;
-	uid_t nickuid;
-	int error = 0, ticklen;
-	mbuf_t mrep, md;
-	struct nfsuid *nuidp;
+	int error = 0;
 	uid_t user_id;
 	gid_t group_id;
 	int ngroups;
 	struct ucred temp_cred;
-	struct timeval tvin, tvout, now;
-	char uio_buf[ UIO_SIZEOF(1) ];
-#if 0				/* until encrypted keys are implemented */
-	NFSKERBKEYSCHED_T keys;	/* stores key schedule */
-#endif
+	uint32_t val;
 
 	nd->nd_cr = NULL;
-
-	mrep = nd->nd_mrep;
-	md = nd->nd_md;
-	dpos = nd->nd_dpos;
-	if (has_header) {
-		nfsm_dissect(tl, u_long *, 10 * NFSX_UNSIGNED);
-		nd->nd_retxid = fxdr_unsigned(u_long, *tl++);
-		if (*tl++ != rpc_call) {
-			mbuf_freem(mrep);
-			return (EBADRPC);
-		}
-	} else
-		nfsm_dissect(tl, u_long *, 8 * NFSX_UNSIGNED);
+	nd->nd_gss_context = NULL;
+	nd->nd_gss_seqnum = 0;
+	nd->nd_gss_mb = NULL;
+
+	user_id = group_id = -2;
+	val = auth_type = len = 0;
+
+	nmreq = &nd->nd_nmreq;
+	nfsm_chain_get_32(error, nmreq, nd->nd_retxid);	// XID
+	nfsm_chain_get_32(error, nmreq, val);		// RPC Call
+	if (!error && (val != RPC_CALL))
+		error = EBADRPC;
+	nfsmout_if(error);
 	nd->nd_repstat = 0;
-	nd->nd_flag = 0;
-	if (*tl++ != rpc_vers) {
+	nfsm_chain_get_32(error, nmreq, val);	// RPC Version
+	nfsmout_if(error);
+	if (val != RPC_VER2) {
 		nd->nd_repstat = ERPCMISMATCH;
 		nd->nd_procnum = NFSPROC_NOOP;
 		return (0);
 	}
-	if (*tl != nfs_prog) {
+	nfsm_chain_get_32(error, nmreq, val);	// RPC Program Number
+	nfsmout_if(error);
+	if (val != NFS_PROG) {
 		nd->nd_repstat = EPROGUNAVAIL;
 		nd->nd_procnum = NFSPROC_NOOP;
 		return (0);
 	}
-	tl++;
-	nfsvers = fxdr_unsigned(u_long, *tl++);
+	nfsm_chain_get_32(error, nmreq, nfsvers);// NFS Version Number
+	nfsmout_if(error);
 	if ((nfsvers < NFS_VER2) || (nfsvers > NFS_VER3)) {
 		nd->nd_repstat = EPROGMISMATCH;
 		nd->nd_procnum = NFSPROC_NOOP;
 		return (0);
 	}
-	else if (nfsvers == NFS_VER3)
-		nd->nd_flag = ND_NFSV3;
-	nd->nd_procnum = fxdr_unsigned(u_long, *tl++);
-	if (nd->nd_procnum == NFSPROC_NULL)
-		return (0);
+	nd->nd_vers = nfsvers;
+	nfsm_chain_get_32(error, nmreq, nd->nd_procnum);// NFS Procedure Number
+	nfsmout_if(error);
 	if ((nd->nd_procnum >= NFS_NPROCS) ||
-		(!nd->nd_flag && nd->nd_procnum > NFSV2PROC_STATFS)) {
+		((nd->nd_vers == NFS_VER2) && (nd->nd_procnum > NFSV2PROC_STATFS))) {
 		nd->nd_repstat = EPROCUNAVAIL;
 		nd->nd_procnum = NFSPROC_NOOP;
 		return (0);
 	}
-	if ((nd->nd_flag & ND_NFSV3) == 0)
+	if (nfsvers != NFS_VER3)
 		nd->nd_procnum = nfsv3_procid[nd->nd_procnum];
-	auth_type = *tl++;
-	len = fxdr_unsigned(int, *tl++);
-	if (len < 0 || len > RPCAUTH_MAXSIZ) {
-		mbuf_freem(mrep);
-		return (EBADRPC);
-	}
-
-	nd->nd_flag &= ~ND_KERBAUTH;
-	/*
-	 * Handle auth_unix or auth_kerb.
-	 */
-	if (auth_type == rpc_auth_unix) {
-		len = fxdr_unsigned(int, *++tl);
-		if (len < 0 || len > NFS_MAXNAMLEN) {
-			mbuf_freem(mrep);
-			return (EBADRPC);
-		}
+	nfsm_chain_get_32(error, nmreq, auth_type);	// Auth Flavor
+	nfsm_chain_get_32(error, nmreq, len);		// Auth Length
+	if (!error && (len < 0 || len > RPCAUTH_MAXSIZ))
+		error = EBADRPC;
+	nfsmout_if(error);
+
+	/* Handle authentication */
+	if (auth_type == RPCAUTH_UNIX) {
+		if (nd->nd_procnum == NFSPROC_NULL)
+			return (0);
+		nd->nd_sec = RPCAUTH_UNIX;
+		nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);	// skip stamp
+		nfsm_chain_get_32(error, nmreq, len);		// hostname length
+		if (len < 0 || len > NFS_MAXNAMLEN)
+			error = EBADRPC;
+		nfsm_chain_adv(error, nmreq, nfsm_rndup(len));	// skip hostname
+		nfsmout_if(error);
+
+		/* create a temporary credential using the bits from the wire */
 		bzero(&temp_cred, sizeof(temp_cred));
-		nfsm_adv(nfsm_rndup(len));
-		nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-		user_id = fxdr_unsigned(uid_t, *tl++);
-		group_id = fxdr_unsigned(gid_t, *tl++);
+		nfsm_chain_get_32(error, nmreq, user_id);
+		nfsm_chain_get_32(error, nmreq, group_id);
 		temp_cred.cr_groups[0] = group_id;
-		len = fxdr_unsigned(int, *tl);
-		if (len < 0 || len > RPCAUTH_UNIXGIDS) {
-			mbuf_freem(mrep);
-			return (EBADRPC);
-		}
-		nfsm_dissect(tl, u_long *, (len + 2) * NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, nmreq, len);		// extra GID count
+		if ((len < 0) || (len > RPCAUTH_UNIXGIDS))
+			error = EBADRPC;
+		nfsmout_if(error);
 		for (i = 1; i <= len; i++)
-		    if (i < NGROUPS)
-			temp_cred.cr_groups[i] = fxdr_unsigned(gid_t, *tl++);
-		    else
-			tl++;
+			if (i < NGROUPS)
+				nfsm_chain_get_32(error, nmreq, temp_cred.cr_groups[i]);
+			else
+				nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);
+		nfsmout_if(error);
 		ngroups = (len >= NGROUPS) ? NGROUPS : (len + 1);
 		if (ngroups > 1)
-		    nfsrvw_sort(&temp_cred.cr_groups[0], ngroups);
-		len = fxdr_unsigned(int, *++tl);
-		if (len < 0 || len > RPCAUTH_MAXSIZ) {
-			mbuf_freem(mrep);
-			return (EBADRPC);
-		}
+			nfsrv_group_sort(&temp_cred.cr_groups[0], ngroups);
+		nfsm_chain_adv(error, nmreq, NFSX_UNSIGNED);	// verifier flavor (should be AUTH_NONE)
+		nfsm_chain_get_32(error, nmreq, len);		// verifier length
+		if (len < 0 || len > RPCAUTH_MAXSIZ)
+			error = EBADRPC;
+		if (len > 0)
+			nfsm_chain_adv(error, nmreq, nfsm_rndup(len));
+
+		/* request creation of a real credential */
 		temp_cred.cr_uid = user_id;
 		temp_cred.cr_ngroups = ngroups;
-		nd->nd_cr = kauth_cred_create(&temp_cred); 
+		nd->nd_cr = kauth_cred_create(&temp_cred);
 		if (nd->nd_cr == NULL) {
 			nd->nd_repstat = ENOMEM;
 			nd->nd_procnum = NFSPROC_NOOP;
 			return (0);
 		}
-		if (len > 0)
-			nfsm_adv(nfsm_rndup(len));
-	} else if (auth_type == rpc_auth_kerb) {
-		switch (fxdr_unsigned(int, *tl++)) {
-		case RPCAKN_FULLNAME:
-			ticklen = fxdr_unsigned(int, *tl);
-			*((u_long *)nfsd->nfsd_authstr) = *tl;
-			uiop = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
-						&uio_buf[0], sizeof(uio_buf));
-			if (!uiop) {
-				nd->nd_repstat = ENOMEM;
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-
-			// LP64todo - fix this
-			nfsd->nfsd_authlen = (nfsm_rndup(ticklen) + (NFSX_UNSIGNED * 2));
-			if ((nfsm_rndup(ticklen) + NFSX_UNSIGNED) > (len - 2 * NFSX_UNSIGNED)) {
-				mbuf_freem(mrep);
-				return (EBADRPC);
-			}
-			uio_addiov(uiop, CAST_USER_ADDR_T(&nfsd->nfsd_authstr[4]), RPCAUTH_MAXSIZ - 4);
-			// LP64todo - fix this
-			nfsm_mtouio(uiop, uio_resid(uiop));
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-			if (*tl++ != rpc_auth_kerb ||
-				fxdr_unsigned(int, *tl) != 4 * NFSX_UNSIGNED) {
-				printf("Bad kerb verifier\n");
-				nd->nd_repstat = (NFSERR_AUTHERR|AUTH_BADVERF);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			nfsm_dissect(cp, caddr_t, 4 * NFSX_UNSIGNED);
-			tl = (u_long *)cp;
-			if (fxdr_unsigned(int, *tl) != RPCAKN_FULLNAME) {
-				printf("Not fullname kerb verifier\n");
-				nd->nd_repstat = (NFSERR_AUTHERR|AUTH_BADVERF);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			cp += NFSX_UNSIGNED;
-			bcopy(cp, nfsd->nfsd_verfstr, 3 * NFSX_UNSIGNED);
-			nfsd->nfsd_verflen = 3 * NFSX_UNSIGNED;
-			nd->nd_flag |= ND_KERBFULL;
-			nfsd->nfsd_flag |= NFSD_NEEDAUTH;
-			break;
-		case RPCAKN_NICKNAME:
-			if (len != 2 * NFSX_UNSIGNED) {
-				printf("Kerb nickname short\n");
-				nd->nd_repstat = (NFSERR_AUTHERR|AUTH_BADCRED);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			nickuid = fxdr_unsigned(uid_t, *tl);
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-			if (*tl++ != rpc_auth_kerb ||
-				fxdr_unsigned(int, *tl) != 3 * NFSX_UNSIGNED) {
-				printf("Kerb nick verifier bad\n");
-				nd->nd_repstat = (NFSERR_AUTHERR|AUTH_BADVERF);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-			tvin.tv_sec = *tl++;
-			tvin.tv_usec = *tl;
-
-			for (nuidp = NUIDHASH(nfsd->nfsd_slp,nickuid)->lh_first;
-			    nuidp != 0; nuidp = nuidp->nu_hash.le_next) {
-				if (kauth_cred_getuid(nuidp->nu_cr) == nickuid &&
-				    (!nd->nd_nam2 ||
-				     netaddr_match(NU_NETFAM(nuidp),
-				      &nuidp->nu_haddr, nd->nd_nam2)))
-					break;
-			}
-			if (!nuidp) {
-				nd->nd_repstat =
-					(NFSERR_AUTHERR|AUTH_REJECTCRED);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-
-			/*
-			 * Now, decrypt the timestamp using the session key
-			 * and validate it.
-			 */
-#if NFSKERB
-			XXX
-#endif
-
-			tvout.tv_sec = fxdr_unsigned(long, tvout.tv_sec);
-			tvout.tv_usec = fxdr_unsigned(long, tvout.tv_usec);
-			microtime(&now);
-			if (nuidp->nu_expire < now.tv_sec ||
-			    nuidp->nu_timestamp.tv_sec > tvout.tv_sec ||
-			    (nuidp->nu_timestamp.tv_sec == tvout.tv_sec &&
-			     nuidp->nu_timestamp.tv_usec > tvout.tv_usec)) {
-				nuidp->nu_expire = 0;
-				nd->nd_repstat =
-				    (NFSERR_AUTHERR|AUTH_REJECTVERF);
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			bzero(&temp_cred, sizeof(temp_cred));
-			ngroups = nuidp->nu_cr->cr_ngroups;
-			for (i = 0; i < ngroups; i++)
-				temp_cred.cr_groups[i] = nuidp->nu_cr->cr_groups[i];
-			if (ngroups > 1)
-				nfsrvw_sort(&temp_cred.cr_groups[0], ngroups);
-
-			temp_cred.cr_uid = kauth_cred_getuid(nuidp->nu_cr);
-			temp_cred.cr_ngroups = ngroups;
-			nd->nd_cr = kauth_cred_create(&temp_cred); 
-			if (!nd->nd_cr) {
-				nd->nd_repstat = ENOMEM;
-				nd->nd_procnum = NFSPROC_NOOP;
-				return (0);
-			}
-			nd->nd_flag |= ND_KERBNICK;
-		};
+	} else if (auth_type == RPCSEC_GSS) {
+		error = nfs_gss_svc_cred_get(nd, nmreq);
+		if (error) {
+			if (error == EINVAL)
+				goto nfsmout;	// drop the request
+			nd->nd_repstat = error;
+			nd->nd_procnum = NFSPROC_NOOP;
+			return (0);
+		}
 	} else {
+		if (nd->nd_procnum == NFSPROC_NULL)	// assume it's AUTH_NONE
+			return (0);
 		nd->nd_repstat = (NFSERR_AUTHERR | AUTH_REJECTCRED);
 		nd->nd_procnum = NFSPROC_NOOP;
 		return (0);
 	}
-
-	nd->nd_md = md;
-	nd->nd_dpos = dpos;
 	return (0);
 nfsmout:
 	if (IS_VALID_CRED(nd->nd_cr))
 		kauth_cred_unref(&nd->nd_cr);
+	nfsm_chain_cleanup(nmreq);
 	return (error);
 }
 
 /*
  * Search for a sleeping nfsd and wake it up.
- * SIDE EFFECT: If none found, set NFSD_CHECKSLP flag, so that one of the
- * running nfsds will go look for the work in the nfssvc_sock list.
+ * SIDE EFFECT: If none found, make sure the socket is queued up so that one
+ * of the running nfsds will go look for the work in the nfsrv_sockwait list.
  * Note: Must be called with nfsd_mutex held.
  */
 void
-nfsrv_wakenfsd(struct nfssvc_sock *slp)
+nfsrv_wakenfsd(struct nfsrv_sock *slp)
 {
 	struct nfsd *nd;
 
@@ -2759,36 +3185,32 @@ nfsrv_wakenfsd(struct nfssvc_sock *slp)
 		return;
 
 	lck_rw_lock_exclusive(&slp->ns_rwlock);
-
-	if (nfsd_waiting) {
-		TAILQ_FOREACH(nd, &nfsd_head, nfsd_chain) {
-			if (nd->nfsd_flag & NFSD_WAITING) {
-				nd->nfsd_flag &= ~NFSD_WAITING;
-				if (nd->nfsd_slp)
-					panic("nfsd wakeup");
-				slp->ns_sref++;
-				nd->nfsd_slp = slp;
-				lck_rw_done(&slp->ns_rwlock);
-				wakeup((caddr_t)nd);
-				return;
-			}
-		}
+	/* if there's work to do on this socket, make sure it's queued up */
+	if ((slp->ns_flag & SLP_WORKTODO) && !(slp->ns_flag & SLP_QUEUED)) {
+		TAILQ_INSERT_TAIL(&nfsrv_sockwait, slp, ns_svcq);
+		slp->ns_flag |= SLP_WAITQ;
 	}
-
-	slp->ns_flag |= SLP_DOREC;
-
 	lck_rw_done(&slp->ns_rwlock);
 
-	nfsd_head_flag |= NFSD_CHECKSLP;
+	/* wake up a waiting nfsd, if possible */
+	nd = TAILQ_FIRST(&nfsd_queue);
+	if (!nd)
+		return;
+
+	TAILQ_REMOVE(&nfsd_queue, nd, nfsd_queue);
+	nd->nfsd_flag &= ~NFSD_WAITING;
+	wakeup(nd);
 }
-#endif /* NFS_NOSERVER */
+
+#endif /* NFSSERVER */
 
 static int
-nfs_msg(proc_t p,
+nfs_msg(thread_t thd,
 	const char *server,
 	const char *msg,
 	int error)
 {
+	proc_t p = thd ? get_bsdthreadtask_info(thd) : NULL;
 	tpr_t tpr;
 
 	if (p)
@@ -2796,8 +3218,7 @@ nfs_msg(proc_t p,
 	else
 		tpr = NULL;
 	if (error)
-		tprintf(tpr, "nfs server %s: %s, error %d\n", server, msg,
-		    error);
+		tprintf(tpr, "nfs server %s: %s, error %d\n", server, msg, error);
 	else
 		tprintf(tpr, "nfs server %s: %s\n", server, msg);
 	tprintf_close(tpr);
@@ -2805,43 +3226,53 @@ nfs_msg(proc_t p,
 }
 
 void
-nfs_down(nmp, proc, error, flags, msg)
-	struct nfsmount *nmp;
-	proc_t proc;
-	int error, flags;
-	const char *msg;
+nfs_down(struct nfsmount *nmp, thread_t thd, int error, int flags, const char *msg)
 {
+	int ostate;
+
 	if (nmp == NULL)
 		return;
-	if ((flags & NFSSTA_TIMEO) && !(nmp->nm_state & NFSSTA_TIMEO)) {
-		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESP, 0);
+
+	lck_mtx_lock(&nmp->nm_lock);
+	ostate = nmp->nm_state;
+	if ((flags & NFSSTA_TIMEO) && !(ostate & NFSSTA_TIMEO))
 		nmp->nm_state |= NFSSTA_TIMEO;
-	}
-	if ((flags & NFSSTA_LOCKTIMEO) && !(nmp->nm_state & NFSSTA_LOCKTIMEO)) {
-		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESPLOCK, 0);
+	if ((flags & NFSSTA_LOCKTIMEO) && !(ostate & NFSSTA_LOCKTIMEO))
 		nmp->nm_state |= NFSSTA_LOCKTIMEO;
-	}
-	nfs_msg(proc, vfs_statfs(nmp->nm_mountp)->f_mntfromname, msg, error);
+	if ((flags & NFSSTA_JUKEBOXTIMEO) && !(ostate & NFSSTA_JUKEBOXTIMEO))
+		nmp->nm_state |= NFSSTA_JUKEBOXTIMEO;
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	if (!(ostate & (NFSSTA_TIMEO|NFSSTA_LOCKTIMEO|NFSSTA_JUKEBOXTIMEO)))
+		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESP, 0);
+
+	nfs_msg(thd, vfs_statfs(nmp->nm_mountp)->f_mntfromname, msg, error);
 }
 
 void
-nfs_up(nmp, proc, flags, msg)
-	struct nfsmount *nmp;
-	proc_t proc;
-	int flags;
-	const char *msg;
+nfs_up(struct nfsmount *nmp, thread_t thd, int flags, const char *msg)
 {
+	int ostate, state;
+
 	if (nmp == NULL)
 		return;
+
 	if (msg)
-		nfs_msg(proc, vfs_statfs(nmp->nm_mountp)->f_mntfromname, msg, 0);
-	if ((flags & NFSSTA_TIMEO) && (nmp->nm_state & NFSSTA_TIMEO)) {
+		nfs_msg(thd, vfs_statfs(nmp->nm_mountp)->f_mntfromname, msg, 0);
+
+	lck_mtx_lock(&nmp->nm_lock);
+	ostate = nmp->nm_state;
+	if ((flags & NFSSTA_TIMEO) && (ostate & NFSSTA_TIMEO))
 		nmp->nm_state &= ~NFSSTA_TIMEO;
-		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESP, 1);
-	}
-	if ((flags & NFSSTA_LOCKTIMEO) && (nmp->nm_state & NFSSTA_LOCKTIMEO)) {
+	if ((flags & NFSSTA_LOCKTIMEO) && (ostate & NFSSTA_LOCKTIMEO))
 		nmp->nm_state &= ~NFSSTA_LOCKTIMEO;
-		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESPLOCK, 1);
-	}
+	if ((flags & NFSSTA_JUKEBOXTIMEO) && (ostate & NFSSTA_JUKEBOXTIMEO))
+		nmp->nm_state &= ~NFSSTA_JUKEBOXTIMEO;
+	state = nmp->nm_state;
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	if ((ostate & (NFSSTA_TIMEO|NFSSTA_LOCKTIMEO|NFSSTA_JUKEBOXTIMEO)) &&
+	    !(state & (NFSSTA_TIMEO|NFSSTA_LOCKTIMEO|NFSSTA_JUKEBOXTIMEO)))
+		vfs_event_signal(&vfs_statfs(nmp->nm_mountp)->f_fsid, VQ_NOTRESP, 1);
 }
 
diff --git a/bsd/nfs/nfs_srvcache.c b/bsd/nfs/nfs_srvcache.c
index c15362dad..1b28bdcfb 100644
--- a/bsd/nfs/nfs_srvcache.c
+++ b/bsd/nfs/nfs_srvcache.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,7 +65,7 @@
  * FreeBSD-Id: nfs_srvcache.c,v 1.15 1997/10/12 20:25:46 phk Exp $
  */
 
-#ifndef NFS_NOSERVER 
+#if NFSSERVER
 /*
  * Reference: Chet Juszczak, "Improving the Performance and Correctness
  *		of an NFS Server", in Proc. Winter 1989 USENIX Conference,
@@ -74,37 +80,27 @@
 #include <sys/kpi_mbuf.h>
 #include <sys/malloc.h>
 #include <sys/socket.h>
-#include <sys/socketvar.h>	/* for dup_sockaddr */
 #include <libkern/OSAtomic.h>
 
 #include <netinet/in.h>
-#if ISO
-#include <netiso/iso.h>
-#endif
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 #include <nfs/nfsrvcache.h>
 
-extern struct nfsstats nfsstats;
 extern int nfsv2_procid[NFS_NPROCS];
-long numnfsrvcache;
-static long desirednfsrvcache = NFSRVCACHESIZ;
+static int nfsrv_reqcache_count;
+int nfsrv_reqcache_size = NFSRVCACHESIZ;
 
 #define	NFSRCHASH(xid) \
-	(&nfsrvhashtbl[((xid) + ((xid) >> 24)) & nfsrvhash])
-LIST_HEAD(nfsrvhash, nfsrvcache) *nfsrvhashtbl;
-TAILQ_HEAD(nfsrvlru, nfsrvcache) nfsrvlruhead;
-u_long nfsrvhash;
+	(&nfsrv_reqcache_hashtbl[((xid) + ((xid) >> 24)) & nfsrv_reqcache_hash])
+LIST_HEAD(nfsrv_reqcache_hash, nfsrvcache) *nfsrv_reqcache_hashtbl;
+TAILQ_HEAD(nfsrv_reqcache_lru, nfsrvcache) nfsrv_reqcache_lruhead;
+u_long nfsrv_reqcache_hash;
 
 lck_grp_t *nfsrv_reqcache_lck_grp;
-lck_grp_attr_t *nfsrv_reqcache_lck_grp_attr;
-lck_attr_t *nfsrv_reqcache_lck_attr;
 lck_mtx_t *nfsrv_reqcache_mutex;
 
-#define	NETFAMILY(rp) \
-		(((rp)->rc_flag & RC_INETADDR) ? AF_INET : AF_ISO)
-
 /*
  * Static array that defines which nfs rpc's are nonidempotent
  */
@@ -160,16 +156,44 @@ static int nfsv2_repstat[NFS_NPROCS] = {
  * Initialize the server request cache list
  */
 void
-nfsrv_initcache()
+nfsrv_initcache(void)
+{
+	if (nfsrv_reqcache_size <= 0)
+		return;
+
+	lck_mtx_lock(nfsrv_reqcache_mutex);
+	/* init nfs server request cache hash table */
+	nfsrv_reqcache_hashtbl = hashinit(nfsrv_reqcache_size, M_NFSD, &nfsrv_reqcache_hash);
+	TAILQ_INIT(&nfsrv_reqcache_lruhead);
+	lck_mtx_unlock(nfsrv_reqcache_mutex);
+}
+
+/*
+ * This function compares two net addresses by family and returns TRUE
+ * if they are the same host.
+ * If there is any doubt, return FALSE.
+ * The AF_INET family is handled as a special case so that address mbufs
+ * don't need to be saved to store "struct in_addr", which is only 4 bytes.
+ */
+static int
+netaddr_match(
+	int family,
+	union nethostaddr *haddr,
+	mbuf_t nam)
 {
-	/* init nfs server request cache mutex */
-	nfsrv_reqcache_lck_grp_attr = lck_grp_attr_alloc_init();
-	nfsrv_reqcache_lck_grp = lck_grp_alloc_init("nfsrv_reqcache", nfsrv_reqcache_lck_grp_attr);
-	nfsrv_reqcache_lck_attr = lck_attr_alloc_init();
-	nfsrv_reqcache_mutex = lck_mtx_alloc_init(nfsrv_reqcache_lck_grp, nfsrv_reqcache_lck_attr);
+	struct sockaddr_in *inetaddr;
 
-	nfsrvhashtbl = hashinit(desirednfsrvcache, M_NFSD, &nfsrvhash);
-	TAILQ_INIT(&nfsrvlruhead);
+	switch (family) {
+	case AF_INET:
+		inetaddr = mbuf_data(nam);
+		if (inetaddr->sin_family == AF_INET &&
+		    inetaddr->sin_addr.s_addr == haddr->had_inetaddr)
+			return (1);
+		break;
+	default:
+		break;
+	};
+	return (0);
 }
 
 /*
@@ -187,15 +211,14 @@ nfsrv_initcache()
  * Update/add new request at end of lru list
  */
 int
-nfsrv_getcache(nd, slp, repp)
-	struct nfsrv_descript *nd;
-	struct nfssvc_sock *slp;
-	mbuf_t *repp;
+nfsrv_getcache(
+	struct nfsrv_descript *nd,
+	struct nfsrv_sock *slp,
+	mbuf_t *mrepp)
 {
 	struct nfsrvcache *rp;
-	mbuf_t mb;
+	struct nfsm_chain nmrep;
 	struct sockaddr_in *saddr;
-	caddr_t bpos;
 	int ret, error;
 
 	/*
@@ -209,17 +232,17 @@ loop:
 	for (rp = NFSRCHASH(nd->nd_retxid)->lh_first; rp != 0;
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
-		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+		netaddr_match(AF_INET, &rp->rc_haddr, nd->nd_nam)) {
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
-				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
+				msleep(rp, nfsrv_reqcache_mutex, PZERO-1, "nfsrc", NULL);
 				goto loop;
 			}
 			rp->rc_flag |= RC_LOCKED;
 			/* If not at end of LRU chain, move it there */
 			if (rp->rc_lru.tqe_next) {
-				TAILQ_REMOVE(&nfsrvlruhead, rp, rc_lru);
-				TAILQ_INSERT_TAIL(&nfsrvlruhead, rp, rc_lru);
+				TAILQ_REMOVE(&nfsrv_reqcache_lruhead, rp, rc_lru);
+				TAILQ_INSERT_TAIL(&nfsrv_reqcache_lruhead, rp, rc_lru);
 			}
 			if (rp->rc_state == RC_UNUSED)
 				panic("nfsrv cache");
@@ -228,11 +251,19 @@ loop:
 				ret = RC_DROPIT;
 			} else if (rp->rc_flag & RC_REPSTATUS) {
 				OSAddAtomic(1, (SInt32*)&nfsstats.srvcache_nonidemdonehits);
-				nfs_rephead(0, nd, slp, rp->rc_status, repp, &mb, &bpos);
-				ret = RC_REPLY;
+				nd->nd_repstat = rp->rc_status;
+				error = nfsrv_rephead(nd, slp, &nmrep, 0);
+				if (error) {
+					printf("nfsrv cache: reply alloc failed for nonidem request hit\n");
+					ret = RC_DROPIT;
+					*mrepp = NULL;
+				} else {
+					ret = RC_REPLY;
+					*mrepp = nmrep.nmc_mhead;
+				}
 			} else if (rp->rc_flag & RC_REPMBUF) {
 				OSAddAtomic(1, (SInt32*)&nfsstats.srvcache_nonidemdonehits);
-				error = mbuf_copym(rp->rc_reply, 0, MBUF_COPYALL, MBUF_WAITOK, repp);
+				error = mbuf_copym(rp->rc_reply, 0, MBUF_COPYALL, MBUF_WAITOK, mrepp);
 				if (error) {
 					printf("nfsrv cache: reply copym failed for nonidem request hit\n");
 					ret = RC_DROPIT;
@@ -247,19 +278,19 @@ loop:
 			rp->rc_flag &= ~RC_LOCKED;
 			if (rp->rc_flag & RC_WANTED) {
 				rp->rc_flag &= ~RC_WANTED;
-				wakeup((caddr_t)rp);
+				wakeup(rp);
 			}
 			lck_mtx_unlock(nfsrv_reqcache_mutex);
 			return (ret);
 		}
 	}
 	OSAddAtomic(1, (SInt32*)&nfsstats.srvcache_misses);
-	if (numnfsrvcache < desirednfsrvcache) {
+	if (nfsrv_reqcache_count < nfsrv_reqcache_size) {
 		/* try to allocate a new entry */
 		MALLOC(rp, struct nfsrvcache *, sizeof *rp, M_NFSD, M_WAITOK);
 		if (rp) {
 			bzero((char *)rp, sizeof *rp);
-			numnfsrvcache++;
+			nfsrv_reqcache_count++;
 			rp->rc_flag = RC_LOCKED;
 		}
 	} else {
@@ -267,7 +298,7 @@ loop:
 	}
 	if (!rp) {
 		/* try to reuse the least recently used entry */
-		rp = nfsrvlruhead.tqh_first;
+		rp = nfsrv_reqcache_lruhead.tqh_first;
 		if (!rp) {
 			/* no entry to reuse? */
 			/* OK, we just won't be able to cache this request */
@@ -276,19 +307,19 @@ loop:
 		}
 		while ((rp->rc_flag & RC_LOCKED) != 0) {
 			rp->rc_flag |= RC_WANTED;
-			(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
-			rp = nfsrvlruhead.tqh_first;
+			msleep(rp, nfsrv_reqcache_mutex, PZERO-1, "nfsrc", NULL);
+			rp = nfsrv_reqcache_lruhead.tqh_first;
 		}
 		rp->rc_flag |= RC_LOCKED;
 		LIST_REMOVE(rp, rc_hash);
-		TAILQ_REMOVE(&nfsrvlruhead, rp, rc_lru);
+		TAILQ_REMOVE(&nfsrv_reqcache_lruhead, rp, rc_lru);
 		if (rp->rc_flag & RC_REPMBUF)
 			mbuf_freem(rp->rc_reply);
 		if (rp->rc_flag & RC_NAM)
 			mbuf_freem(rp->rc_nam);
 		rp->rc_flag &= (RC_LOCKED | RC_WANTED);
 	}
-	TAILQ_INSERT_TAIL(&nfsrvlruhead, rp, rc_lru);
+	TAILQ_INSERT_TAIL(&nfsrv_reqcache_lruhead, rp, rc_lru);
 	rp->rc_state = RC_INPROG;
 	rp->rc_xid = nd->nd_retxid;
 	saddr = mbuf_data(nd->nd_nam);
@@ -297,7 +328,6 @@ loop:
 		rp->rc_flag |= RC_INETADDR;
 		rp->rc_inetaddr = saddr->sin_addr.s_addr;
 		break;
-	case AF_ISO:
 	default:
 		error = mbuf_copym(nd->nd_nam, 0, MBUF_COPYALL, MBUF_WAITOK, &rp->rc_nam);
 		if (error)
@@ -311,7 +341,7 @@ loop:
 	rp->rc_flag &= ~RC_LOCKED;
 	if (rp->rc_flag & RC_WANTED) {
 		rp->rc_flag &= ~RC_WANTED;
-		wakeup((caddr_t)rp);
+		wakeup(rp);
 	}
 	lck_mtx_unlock(nfsrv_reqcache_mutex);
 	return (RC_DOIT);
@@ -321,10 +351,10 @@ loop:
  * Update a request cache entry after the rpc has been done
  */
 void
-nfsrv_updatecache(nd, repvalid, repmbuf)
-	struct nfsrv_descript *nd;
-	int repvalid;
-	mbuf_t repmbuf;
+nfsrv_updatecache(
+	struct nfsrv_descript *nd,
+	int repvalid,
+	mbuf_t repmbuf)
 {
 	struct nfsrvcache *rp;
 	int error;
@@ -336,20 +366,32 @@ loop:
 	for (rp = NFSRCHASH(nd->nd_retxid)->lh_first; rp != 0;
 	    rp = rp->rc_hash.le_next) {
 	    if (nd->nd_retxid == rp->rc_xid && nd->nd_procnum == rp->rc_proc &&
-		netaddr_match(NETFAMILY(rp), &rp->rc_haddr, nd->nd_nam)) {
+		netaddr_match(AF_INET, &rp->rc_haddr, nd->nd_nam)) {
 			if ((rp->rc_flag & RC_LOCKED) != 0) {
 				rp->rc_flag |= RC_WANTED;
-				(void) tsleep((caddr_t)rp, PZERO-1, "nfsrc", 0);
+				msleep(rp, nfsrv_reqcache_mutex, PZERO-1, "nfsrc", NULL);
 				goto loop;
 			}
 			rp->rc_flag |= RC_LOCKED;
+                        if (rp->rc_state == RC_DONE) {
+                                /*
+                                 * This can occur if the cache is too small.
+                                 * Retransmits of the same request aren't
+                                 * dropped so we may see the operation
+                                 * complete more then once.
+                                 */
+                                if (rp->rc_flag & RC_REPMBUF) {
+                                        mbuf_freem(rp->rc_reply);
+                                        rp->rc_flag &= ~RC_REPMBUF;
+                                }
+			}
 			rp->rc_state = RC_DONE;
 			/*
 			 * If we have a valid reply update status and save
 			 * the reply for non-idempotent rpc's.
 			 */
 			if (repvalid && nonidempotent[nd->nd_procnum]) {
-				if ((nd->nd_flag & ND_NFSV3) == 0 &&
+				if ((nd->nd_vers == NFS_VER2) &&
 				  nfsv2_repstat[nfsv2_procid[nd->nd_procnum]]) {
 					rp->rc_status = nd->nd_repstat;
 					rp->rc_flag |= RC_REPSTATUS;
@@ -362,7 +404,7 @@ loop:
 			rp->rc_flag &= ~RC_LOCKED;
 			if (rp->rc_flag & RC_WANTED) {
 				rp->rc_flag &= ~RC_WANTED;
-				wakeup((caddr_t)rp);
+				wakeup(rp);
 			}
 			lck_mtx_unlock(nfsrv_reqcache_mutex);
 			return;
@@ -375,19 +417,20 @@ loop:
  * Clean out the cache. Called when the last nfsd terminates.
  */
 void
-nfsrv_cleancache()
+nfsrv_cleancache(void)
 {
 	struct nfsrvcache *rp, *nextrp;
 
 	lck_mtx_lock(nfsrv_reqcache_mutex);
-	for (rp = nfsrvlruhead.tqh_first; rp != 0; rp = nextrp) {
+	for (rp = nfsrv_reqcache_lruhead.tqh_first; rp != 0; rp = nextrp) {
 		nextrp = rp->rc_lru.tqe_next;
 		LIST_REMOVE(rp, rc_hash);
-		TAILQ_REMOVE(&nfsrvlruhead, rp, rc_lru);
+		TAILQ_REMOVE(&nfsrv_reqcache_lruhead, rp, rc_lru);
 		_FREE(rp, M_NFSD);
 	}
-	numnfsrvcache = 0;
+	nfsrv_reqcache_count = 0;
+	FREE(nfsrv_reqcache_hashtbl, M_TEMP);
 	lck_mtx_unlock(nfsrv_reqcache_mutex);
 }
 
-#endif /* NFS_NOSERVER */
+#endif /* NFSSERVER */
diff --git a/bsd/nfs/nfs_subs.c b/bsd/nfs/nfs_subs.c
index 54975e71a..f0fd596a9 100644
--- a/bsd/nfs/nfs_subs.c
+++ b/bsd/nfs/nfs_subs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -76,12 +82,12 @@
 #include <sys/stat.h>
 #include <sys/malloc.h>
 #include <sys/syscall.h>
-#include <sys/sysctl.h>
 #include <sys/ubc_internal.h>
 #include <sys/fcntl.h>
 #include <sys/uio_internal.h>
 #include <sys/domain.h>
 #include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
 
 #include <sys/vm.h>
 #include <sys/vmparam.h>
@@ -95,80 +101,97 @@
 #include <nfs/nfsnode.h>
 #include <nfs/xdr_subs.h>
 #include <nfs/nfsm_subs.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
-#include <nfs/nfsrtt.h>
 #include <nfs/nfs_lock.h>
 
 #include <miscfs/specfs/specdev.h>
 
 #include <netinet/in.h>
-#if ISO
-#include <netiso/iso.h>
-#endif
-
-#include <sys/kdebug.h>
-
-SYSCTL_DECL(_vfs_generic);
-SYSCTL_NODE(_vfs_generic, OID_AUTO, nfs, CTLFLAG_RW, 0, "nfs hinge");
-
-#define FSDBG(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_NONE, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_TOP(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_START, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_BOT(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_END, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
+#include <net/kpi_interface.h>
+
 /*
- * Data items converted to xdr at startup, since they are constant
- * This is kinda hokey, but may save a little time doing byte swaps
+ * NFS globals
  */
-u_long nfs_xdrneg1;
-u_long rpc_call, rpc_vers, rpc_reply, rpc_msgdenied, rpc_autherr,
-	rpc_mismatch, rpc_auth_unix, rpc_msgaccepted,
-	rpc_auth_kerb;
-u_long nfs_prog, nfs_true, nfs_false;
-__private_extern__ int nfs_mbuf_mlen = 0, nfs_mbuf_mhlen = 0,
-	nfs_mbuf_minclsize = 0, nfs_mbuf_mclbytes = 0;
-
-/* And other global data */
-static u_long nfs_xid = 0;
-u_long nfs_xidwrap = 0;		/* to build a (non-wwrapping) 64 bit xid */
-static enum vtype nv2tov_type[8]= {
-	VNON, VREG, VDIR, VBLK, VCHR, VLNK, VNON,  VNON 
-};
-enum vtype nv3tov_type[8]= {
-	VNON, VREG, VDIR, VBLK, VCHR, VLNK, VSOCK, VFIFO
-};
+struct nfsstats	nfsstats;
+size_t nfs_mbuf_mhlen = 0, nfs_mbuf_minclsize = 0;
+
+/*
+ * functions to convert between NFS and VFS types
+ */
+nfstype
+vtonfs_type(enum vtype vtype, int nfsvers)
+{
+	switch (vtype) {
+	case VNON:
+		return NFNON;
+	case VREG:
+		return NFREG;
+	case VDIR:
+		return NFDIR;
+	case VBLK:
+		return NFBLK;
+	case VCHR:
+		return NFCHR;
+	case VLNK:
+		return NFLNK;
+	case VSOCK:
+		if (nfsvers > NFS_VER2)
+			return NFSOCK;
+	case VFIFO:
+		if (nfsvers > NFS_VER2)
+			return NFFIFO;
+	case VBAD:
+	case VSTR:
+	case VCPLX:
+	default:
+		return NFNON;
+	}
+}
+
+enum vtype
+nfstov_type(nfstype nvtype, int nfsvers)
+{
+	switch (nvtype) {
+	case NFNON:
+		return VNON;
+	case NFREG:
+		return VREG;
+	case NFDIR:
+		return VDIR;
+	case NFBLK:
+		return VBLK;
+	case NFCHR:
+		return VCHR;
+	case NFLNK:
+		return VLNK;
+	case NFSOCK:
+		if (nfsvers > NFS_VER2)
+			return VSOCK;
+	case NFFIFO:
+		if (nfsvers > NFS_VER2)
+			return VFIFO;
+	case NFATTRDIR:
+		if (nfsvers > NFS_VER3)
+			return VDIR;
+	case NFNAMEDATTR:
+		if (nfsvers > NFS_VER3)
+			return VREG;
+	default:
+		return VNON;
+	}
+}
+
+int
+vtonfsv2_mode(enum vtype vtype, mode_t m)
+{
+	if (vtype == VFIFO)
+		return vnode_makeimode(VCHR, m);
+	return vnode_makeimode(vtype, m);
+}
+
+#if NFSSERVER
 
-int nfs_mount_type;
-int nfs_ticks;
-
-lck_grp_t *nfsd_lck_grp;
-lck_grp_attr_t *nfsd_lck_grp_attr;
-lck_attr_t *nfsd_lck_attr;
-lck_mtx_t *nfsd_mutex;
-
-lck_grp_attr_t *nfs_slp_group_attr;
-lck_attr_t *nfs_slp_lock_attr;
-lck_grp_t *nfs_slp_rwlock_group;
-lck_grp_t *nfs_slp_mutex_group;
-
-struct nfs_reqq nfs_reqq;
-struct nfssvc_sockhead nfssvc_sockhead, nfssvc_deadsockhead;
-struct nfsd_head nfsd_head;
-int nfsd_head_flag;
-
-struct nfsexpfslist nfs_exports;
-struct nfsexphashhead *nfsexphashtbl;
-u_long nfsexphash;
-lck_grp_attr_t *nfs_export_group_attr;
-lck_attr_t *nfs_export_lock_attr;
-lck_grp_t *nfs_export_rwlock_group;
-lck_rw_t nfs_export_rwlock;
-
-#ifndef NFS_NOSERVER
 /*
  * Mapping of old NFS Version 2 RPC numbers to generic numbers.
  */
@@ -198,7 +221,8 @@ int nfsv3_procid[NFS_NPROCS] = {
 	NFSPROC_NOOP
 };
 
-#endif /* NFS_NOSERVER */
+#endif /* NFSSERVER */
+
 /*
  * and the reverse mapping from generic to Version 2 procedure numbers
  */
@@ -228,1089 +252,888 @@ int nfsv2_procid[NFS_NPROCS] = {
 	NFSV2PROC_NOOP
 };
 
-#ifndef NFS_NOSERVER
+
 /*
- * Maps errno values to nfs error numbers.
- * Use NFSERR_IO as the catch all for ones not specifically defined in
- * RFC 1094.
+ * initialize NFS's cache of mbuf constants
  */
-static u_char nfsrv_v2errmap[ELAST] = {
-  NFSERR_PERM,	NFSERR_NOENT,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_NXIO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_ACCES,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_EXIST,	NFSERR_IO,	NFSERR_NODEV,	NFSERR_NOTDIR,
-  NFSERR_ISDIR,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_FBIG,	NFSERR_NOSPC,	NFSERR_IO,	NFSERR_ROFS,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_NAMETOL,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_NOTEMPTY, NFSERR_IO,	NFSERR_IO,	NFSERR_DQUOT,	NFSERR_STALE,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
-};
+void
+nfs_mbuf_init(void)
+{
+	struct mbuf_stat ms;
+
+	mbuf_stats(&ms);
+	nfs_mbuf_mhlen = ms.mhlen;
+	nfs_mbuf_minclsize = ms.minclsize;
+}
+
+#if NFSSERVER
 
 /*
- * Maps errno values to nfs error numbers.
- * Although it is not obvious whether or not NFS clients really care if
- * a returned error value is in the specified list for the procedure, the
- * safest thing to do is filter them appropriately. For Version 2, the
- * X/Open XNFS document is the only specification that defines error values
- * for each RPC (The RFC simply lists all possible error values for all RPCs),
- * so I have decided to not do this for Version 2.
- * The first entry is the default error return and the rest are the valid
- * errors for that RPC in increasing numeric order.
+ * allocate a list of mbufs to hold the given amount of data
  */
-static short nfsv3err_null[] = {
-	0,
-	0,
-};
+int
+nfsm_mbuf_get_list(size_t size, mbuf_t *mp, int *mbcnt)
+{
+	int error, cnt;
+	mbuf_t mhead, mlast, m;
+	size_t len, mlen;
 
-static short nfsv3err_getattr[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	error = cnt = 0;
+	mhead = mlast = NULL;
+	len = 0;
 
-static short nfsv3err_setattr[] = {
-	NFSERR_IO,
-	NFSERR_PERM,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_INVAL,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOT_SYNC,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	while (len < size) {
+		nfsm_mbuf_get(error, &m, (size - len));
+		if (error)
+			break;
+		if (!mhead)
+			mhead = m;
+		if (mlast && ((error = mbuf_setnext(mlast, m)))) {
+			mbuf_free(m);
+			break;
+		}
+		mlen = mbuf_maxlen(m);
+		if ((len + mlen) > size)
+			mlen = size - len;
+		mbuf_setlen(m, mlen);
+		len += mlen;
+		cnt++;
+		mlast = m;
+	}
 
-static short nfsv3err_lookup[] = {
-	NFSERR_IO,
-	NFSERR_NOENT,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_NOTDIR,
-	NFSERR_NAMETOL,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	if (!error) {
+		*mp = mhead;
+		*mbcnt = cnt;
+	}
+	return (error);
+}
 
-static short nfsv3err_access[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
+#endif /* NFSSERVER */
 
-static short nfsv3err_readlink[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_INVAL,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
+/*
+ * nfsm_chain_new_mbuf()
+ *
+ * Add a new mbuf to the given chain.
+ */
+int
+nfsm_chain_new_mbuf(struct nfsm_chain *nmc, size_t sizehint)
+{
+	mbuf_t mb;
+	int error = 0;
 
-static short nfsv3err_read[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_NXIO,
-	NFSERR_ACCES,
-	NFSERR_INVAL,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	if (nmc->nmc_flags & NFSM_CHAIN_FLAG_ADD_CLUSTERS)
+		sizehint = nfs_mbuf_minclsize;
 
-static short nfsv3err_write[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_INVAL,
-	NFSERR_FBIG,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	/* allocate a new mbuf */
+	nfsm_mbuf_get(error, &mb, sizehint);
+	if (error)
+		return (error);
+	if (mb == NULL)
+		panic("got NULL mbuf?");
+
+	/* do we have a current mbuf? */
+	if (nmc->nmc_mcur) {
+		/* first cap off current mbuf */
+		mbuf_setlen(nmc->nmc_mcur, nmc->nmc_ptr - (caddr_t)mbuf_data(nmc->nmc_mcur));
+		/* then append the new mbuf */
+		error = mbuf_setnext(nmc->nmc_mcur, mb);
+		if (error) {
+			mbuf_free(mb);
+			return (error);
+		}
+	}
 
-static short nfsv3err_create[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_NOTDIR,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	/* set up for using the new mbuf */
+	nmc->nmc_mcur = mb;
+	nmc->nmc_ptr = mbuf_data(mb);
+	nmc->nmc_left = mbuf_trailingspace(mb);
 
-static short nfsv3err_mkdir[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_NOTDIR,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
+	return (0);
+}
 
-static short nfsv3err_symlink[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_NOTDIR,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
+/*
+ * nfsm_chain_add_opaque_f()
+ *
+ * Add "len" bytes of opaque data pointed to by "buf" to the given chain.
+ */
+int
+nfsm_chain_add_opaque_f(struct nfsm_chain *nmc, const u_char *buf, uint32_t len)
+{
+	uint32_t paddedlen, tlen;
+	int error;
 
-static short nfsv3err_mknod[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_NOTDIR,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	NFSERR_BADTYPE,
-	0,
-};
+	paddedlen = nfsm_rndup(len);
 
-static short nfsv3err_remove[] = {
-	NFSERR_IO,
-	NFSERR_NOENT,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_NOTDIR,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_rmdir[] = {
-	NFSERR_IO,
-	NFSERR_NOENT,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_NOTDIR,
-	NFSERR_INVAL,
-	NFSERR_ROFS,
-	NFSERR_NAMETOL,
-	NFSERR_NOTEMPTY,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_rename[] = {
-	NFSERR_IO,
-	NFSERR_NOENT,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_XDEV,
-	NFSERR_NOTDIR,
-	NFSERR_ISDIR,
-	NFSERR_INVAL,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_MLINK,
-	NFSERR_NAMETOL,
-	NFSERR_NOTEMPTY,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_link[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_EXIST,
-	NFSERR_XDEV,
-	NFSERR_NOTDIR,
-	NFSERR_INVAL,
-	NFSERR_NOSPC,
-	NFSERR_ROFS,
-	NFSERR_MLINK,
-	NFSERR_NAMETOL,
-	NFSERR_DQUOT,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_NOTSUPP,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_readdir[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_NOTDIR,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_BAD_COOKIE,
-	NFSERR_TOOSMALL,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_readdirplus[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_ACCES,
-	NFSERR_NOTDIR,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_BAD_COOKIE,
-	NFSERR_NOTSUPP,
-	NFSERR_TOOSMALL,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_fsstat[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_fsinfo[] = {
-	NFSERR_STALE,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_pathconf[] = {
-	NFSERR_STALE,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short nfsv3err_commit[] = {
-	NFSERR_IO,
-	NFSERR_IO,
-	NFSERR_STALE,
-	NFSERR_BADHANDLE,
-	NFSERR_SERVERFAULT,
-	0,
-};
-
-static short *nfsrv_v3errmap[] = {
-	nfsv3err_null,
-	nfsv3err_getattr,
-	nfsv3err_setattr,
-	nfsv3err_lookup,
-	nfsv3err_access,
-	nfsv3err_readlink,
-	nfsv3err_read,
-	nfsv3err_write,
-	nfsv3err_create,
-	nfsv3err_mkdir,
-	nfsv3err_symlink,
-	nfsv3err_mknod,
-	nfsv3err_remove,
-	nfsv3err_rmdir,
-	nfsv3err_rename,
-	nfsv3err_link,
-	nfsv3err_readdir,
-	nfsv3err_readdirplus,
-	nfsv3err_fsstat,
-	nfsv3err_fsinfo,
-	nfsv3err_pathconf,
-	nfsv3err_commit,
-};
-
-#endif /* NFS_NOSERVER */
-
-extern struct nfsrtt nfsrtt;
-extern struct nfsstats nfsstats;
-extern nfstype nfsv2_type[9];
-extern nfstype nfsv3_type[9];
-extern struct nfsnodehashhead *nfsnodehashtbl;
-extern u_long nfsnodehash;
-
-
-LIST_HEAD(nfsnodehashhead, nfsnode);
+	while (paddedlen) {
+		if (!nmc->nmc_left) {
+			error = nfsm_chain_new_mbuf(nmc, paddedlen);
+			if (error)
+				return (error);
+		}
+		tlen = MIN(nmc->nmc_left, paddedlen);
+		if (tlen) {
+			if (len) {
+				if (tlen > len)
+					tlen = len;
+				bcopy(buf, nmc->nmc_ptr, tlen);
+			} else {
+				bzero(nmc->nmc_ptr, tlen);
+			}
+			nmc->nmc_ptr += tlen;
+			nmc->nmc_left -= tlen;
+			paddedlen -= tlen;
+			if (len) {
+				buf += tlen;
+				len -= tlen;
+			}
+		}
+	}
+	return (0);
+}
 
 /*
- * Create the header for an rpc request packet
- * The hsiz is the size of the rest of the nfs request header.
- * (just used to decide if a cluster is a good idea)
+ * nfsm_chain_add_opaque_nopad_f()
+ *
+ * Add "len" bytes of opaque data pointed to by "buf" to the given chain.
+ * Do not XDR pad.
  */
 int
-nfsm_reqh(int hsiz, caddr_t *bposp, mbuf_t *mbp)
+nfsm_chain_add_opaque_nopad_f(struct nfsm_chain *nmc, const u_char *buf, uint32_t len)
 {
+	uint32_t tlen;
 	int error;
 
-	*mbp = NULL;
-	if (hsiz >= nfs_mbuf_minclsize)
-		error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, mbp);
-	else
-		error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, mbp);
-	if (error)
-		return (error);
-	*bposp = mbuf_data(*mbp);
+	while (len > 0) {
+		if (nmc->nmc_left <= 0) {
+			error = nfsm_chain_new_mbuf(nmc, len);
+			if (error)
+				return (error);
+		}
+		tlen = MIN(nmc->nmc_left, len);
+		bcopy(buf, nmc->nmc_ptr, tlen);
+		nmc->nmc_ptr += tlen;
+		nmc->nmc_left -= tlen;
+		len -= tlen;
+		buf += tlen;
+	}
 	return (0);
 }
 
 /*
- * Build the RPC header and fill in the authorization info.
- * The authorization string argument is only used when the credentials
- * come from outside of the kernel.
- * Returns the head of the mbuf list.
+ * nfsm_chain_add_uio()
+ *
+ * Add "len" bytes of data from "uio" to the given chain.
  */
 int
-nfsm_rpchead(cr, nmflag, procid, auth_type, auth_len, auth_str, verf_len,
-	verf_str, mrest, mrest_len, mbp, xidp, mreqp)
-	kauth_cred_t cr;
-	int nmflag;
-	int procid;
-	int auth_type;
-	int auth_len;
-	char *auth_str;
-	int verf_len;
-	char *verf_str;
-	mbuf_t mrest;
-	int mrest_len;
-	mbuf_t *mbp;
-	u_long *xidp;
-	mbuf_t *mreqp;
+nfsm_chain_add_uio(struct nfsm_chain *nmc, struct uio *uiop, uint32_t len)
 {
-	mbuf_t mb;
-	u_long *tl;
-	caddr_t bpos;
-	int i, error, len;
-	mbuf_t mreq, mb2;
-	int siz, grpsiz, authsiz, mlen;
-	struct timeval tv;
-
-	authsiz = nfsm_rndup(auth_len);
-	len = authsiz + 10 * NFSX_UNSIGNED;
-	if (len >= nfs_mbuf_minclsize) {
-		error = mbuf_getpacket(MBUF_WAITOK, &mb);
-	} else {
-		error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &mb);
-		if (!error) {
-			if (len < nfs_mbuf_mhlen)
-				mbuf_align_32(mb, len);
-			else
-				mbuf_align_32(mb, 8 * NFSX_UNSIGNED);
-		}
-	}
-	if (error) {
-		/* unable to allocate packet */
-		/* XXX nfsstat? */
-		return (error);
-	}
-	mreq = mb;
-	bpos = mbuf_data(mb);
-
-	/*
-	 * First the RPC header.
-	 */
-	nfsm_build(tl, u_long *, 8 * NFSX_UNSIGNED);
-
-	/*
-	 * derive initial xid from system time
-	 */
-	if (!nfs_xid) {
-		/*
-		 * Note: it's OK if this code inits nfs_xid to 0 (for example,
-		 * due to a broken clock) because we immediately increment it
-		 * and we guarantee to never use xid 0.  So, nfs_xid should only
-		 * ever be 0 the first time this function is called.
-		 */
-		microtime(&tv);
-		nfs_xid = tv.tv_sec << 12;
-	}
-	/*
-	 * Skip zero xid if it should ever happen.
-	 */
-	if (++nfs_xid == 0) {
-		nfs_xidwrap++;
-		nfs_xid++;
-	}
+	uint32_t paddedlen, tlen;
+	int error;
 
-	*tl++ = *xidp = txdr_unsigned(nfs_xid);
-	*tl++ = rpc_call;
-	*tl++ = rpc_vers;
-	*tl++ = txdr_unsigned(NFS_PROG);
-	if (nmflag & NFSMNT_NFSV3)
-		*tl++ = txdr_unsigned(NFS_VER3);
-	else
-		*tl++ = txdr_unsigned(NFS_VER2);
-	if (nmflag & NFSMNT_NFSV3)
-		*tl++ = txdr_unsigned(procid);
-	else
-		*tl++ = txdr_unsigned(nfsv2_procid[procid]);
+	paddedlen = nfsm_rndup(len);
 
-	/*
-	 * And then the authorization cred.
-	 */
-	*tl++ = txdr_unsigned(auth_type);
-	*tl = txdr_unsigned(authsiz);
-	switch (auth_type) {
-	case RPCAUTH_UNIX:
-		nfsm_build(tl, u_long *, auth_len);
-		*tl++ = 0;		/* stamp ?? */
-		*tl++ = 0;		/* NULL hostname */
-		*tl++ = txdr_unsigned(kauth_cred_getuid(cr));
-		*tl++ = txdr_unsigned(cr->cr_groups[0]);
-		grpsiz = (auth_len >> 2) - 5;
-		*tl++ = txdr_unsigned(grpsiz);
-		for (i = 1; i <= grpsiz; i++)
-			*tl++ = txdr_unsigned(cr->cr_groups[i]);
-		break;
-	case RPCAUTH_KERB4:
-		siz = auth_len;
-		mlen = mbuf_len(mb);
-		while (siz > 0) {
-			if (mbuf_trailingspace(mb) == 0) {
-				mb2 = NULL;
-				if (siz >= nfs_mbuf_minclsize)
-					error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mb2);
-				else
-					error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mb2);
-				if (!error)
-					error = mbuf_setnext(mb, mb2);
-				if (error) {
-					mbuf_freem(mreq);
-					return (error);
-				}
-				mb = mb2;
-				mlen = 0;
-				bpos = mbuf_data(mb);
-			}
-			i = min(siz, mbuf_trailingspace(mb));
-			bcopy(auth_str, bpos, i);
-			mlen += i;
-			mbuf_setlen(mb, mlen);
-			auth_str += i;
-			bpos += i;
-			siz -= i;
-		}
-		if ((siz = (nfsm_rndup(auth_len) - auth_len)) > 0) {
-			for (i = 0; i < siz; i++)
-				*bpos++ = '\0';
-			mlen += siz;
-			mbuf_setlen(mb, mlen);
+	while (paddedlen) {
+		if (!nmc->nmc_left) {
+			error = nfsm_chain_new_mbuf(nmc, paddedlen);
+			if (error)
+				return (error);
 		}
-		break;
-	};
-
-	/*
-	 * And the verifier...
-	 */
-	nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-	if (verf_str) {
-		mlen = mbuf_len(mb);
-		*tl++ = txdr_unsigned(RPCAUTH_KERB4);
-		*tl = txdr_unsigned(verf_len);
-		siz = verf_len;
-		while (siz > 0) {
-			if (mbuf_trailingspace(mb) == 0) {
-				mb2 = NULL;
-				if (siz >= nfs_mbuf_minclsize)
-					error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mb2);
-				else
-					error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mb2);
-				if (!error)
-					error = mbuf_setnext(mb, mb2);
-				if (error) {
-					mbuf_freem(mreq);
-					return (error);
-				}
-				mb = mb2;
-				mlen = 0;
-				bpos = mbuf_data(mb);
+		tlen = MIN(nmc->nmc_left, paddedlen);
+		if (tlen) {
+			if (len) {
+				if (tlen > len)
+					tlen = len;
+				uiomove(nmc->nmc_ptr, tlen, uiop);
+			} else {
+				bzero(nmc->nmc_ptr, tlen);
 			}
-			i = min(siz, mbuf_trailingspace(mb));
-			bcopy(verf_str, bpos, i);
-			mlen += i;
-			mbuf_setlen(mb, mlen);
-			verf_str += i;
-			bpos += i;
-			siz -= i;
-		}
-		if ((siz = (nfsm_rndup(verf_len) - verf_len)) > 0) {
-			for (i = 0; i < siz; i++)
-				*bpos++ = '\0';
-			mlen += siz;
-			mbuf_setlen(mb, mlen);
+			nmc->nmc_ptr += tlen;
+			nmc->nmc_left -= tlen;
+			paddedlen -= tlen;
+			if (len)
+				len -= tlen;
 		}
-	} else {
-		*tl++ = txdr_unsigned(RPCAUTH_NULL);
-		*tl = 0;
-	}
-	error = mbuf_pkthdr_setrcvif(mreq, 0);
-	if (!error)
-		error = mbuf_setnext(mb, mrest);
-	if (error) {
-		mbuf_freem(mreq);
-		return (error);
 	}
-	mbuf_pkthdr_setlen(mreq, authsiz + 10 * NFSX_UNSIGNED + mrest_len);
-	*mbp = mb;
-	*mreqp = mreq;
 	return (0);
 }
 
 /*
- * copies mbuf chain to the uio scatter/gather list
+ * Find the length of the NFS mbuf chain
+ * up to the current encoding/decoding offset.
  */
 int
-nfsm_mbuftouio(mrep, uiop, siz, dpos)
-	mbuf_t *mrep;
-	struct uio *uiop;
-	int siz;
-	caddr_t *dpos;
+nfsm_chain_offset(struct nfsm_chain *nmc)
 {
-	char *mbufcp, *uiocp;
-	int xfer, left, len;
-	mbuf_t mp;
-	long uiosiz, rem;
-	int error = 0;
+	mbuf_t mb;
+	int len = 0;
 
-	mp = *mrep;
-	mbufcp = *dpos;
-	len = (caddr_t)mbuf_data(mp) + mbuf_len(mp) - mbufcp;
-	rem = nfsm_rndup(siz)-siz;
-	while (siz > 0) {
-		if (uiop->uio_iovcnt <= 0 || uiop->uio_iovs.iov32p == NULL)
-			return (EFBIG);
-		// LP64todo - fix this!
-		left = uio_iov_len(uiop);
-		uiocp = CAST_DOWN(caddr_t, uio_iov_base(uiop));
-		if (left > siz)
-			left = siz;
-		uiosiz = left;
-		while (left > 0) {
-			while (len == 0) {
-				mp = mbuf_next(mp);
-				if (mp == NULL)
-					return (EBADRPC);
-				mbufcp = mbuf_data(mp);
-				len = mbuf_len(mp);
-			}
-			xfer = (left > len) ? len : left;
-			if (UIO_SEG_IS_USER_SPACE(uiop->uio_segflg))
-				copyout(mbufcp, CAST_USER_ADDR_T(uiocp), xfer);
-			else
-				bcopy(mbufcp, uiocp, xfer);
-			left -= xfer;
-			len -= xfer;
-			mbufcp += xfer;
-			uiocp += xfer;
-			uiop->uio_offset += xfer;
-			uio_uio_resid_add(uiop, -xfer);
-		}
-		if (uio_iov_len(uiop) <= (size_t)siz) {
-			uiop->uio_iovcnt--;
-			uio_next_iov(uiop);
-		} else {
-			uio_iov_base_add(uiop, uiosiz);
-			uio_iov_len_add(uiop, -uiosiz);
-		}
-		siz -= uiosiz;
-	}
-	*dpos = mbufcp;
-	*mrep = mp;
-	if (rem > 0) {
-		if (len < rem)
-			error = nfs_adv(mrep, dpos, rem, len);
-		else
-			*dpos += rem;
+	for (mb = nmc->nmc_mhead; mb; mb = mbuf_next(mb)) {
+		if (mb == nmc->nmc_mcur)
+			return (len + (nmc->nmc_ptr - (caddr_t) mbuf_data(mb)));
+		len += mbuf_len(mb);
 	}
-	return (error);
+
+	return (len);
 }
 
 /*
- * copies a uio scatter/gather list to an mbuf chain.
- * NOTE: can ony handle iovcnt == 1
+ * nfsm_chain_advance()
+ *
+ * Advance an nfsm_chain by "len" bytes.
  */
 int
-nfsm_uiotombuf(uiop, mq, siz, bpos)
-	struct uio *uiop;
-	mbuf_t *mq;
-	int siz;
-	caddr_t *bpos;
+nfsm_chain_advance(struct nfsm_chain *nmc, uint32_t len)
 {
-	char *uiocp;
-	mbuf_t mp, mp2;
-	int xfer, left, mlen, mplen;
-	int uiosiz, clflg, rem, error;
-	char *cp;
-
-	if (uiop->uio_iovcnt != 1)
-		panic("nfsm_uiotombuf: iovcnt != 1");
+	mbuf_t mb;
 
-	if (siz > nfs_mbuf_mlen)		/* or should it >= MCLBYTES ?? */
-		clflg = 1;
-	else
-		clflg = 0;
-	rem = nfsm_rndup(siz)-siz;
-	mp = mp2 = *mq;
-	mplen = mbuf_len(mp);
-	while (siz > 0) {
-		// LP64todo - fix this!
-		left = uio_iov_len(uiop);
-		uiocp = CAST_DOWN(caddr_t, uio_iov_base(uiop));
-		if (left > siz)
-			left = siz;
-		uiosiz = left;
-		while (left > 0) {
-			mlen = mbuf_trailingspace(mp);
-			if (mlen == 0) {
-				mp = NULL;
-				if (clflg)
-					error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mp);
-				else
-					error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mp);
-				if (!error)
-					error = mbuf_setnext(mp2, mp);
-				if (error)
-					return (error);
-				mplen = 0;
-				mp2 = mp;
-				mlen = mbuf_trailingspace(mp);
-			}
-			xfer = (left > mlen) ? mlen : left;
-			if (UIO_SEG_IS_USER_SPACE(uiop->uio_segflg))
-				copyin(CAST_USER_ADDR_T(uiocp), (caddr_t)mbuf_data(mp) + mplen, xfer);
-			else
-				bcopy(uiocp, (caddr_t)mbuf_data(mp) + mplen, xfer);
-			mplen += xfer;
-			mbuf_setlen(mp, mplen);
-			left -= xfer;
-			uiocp += xfer;
-			uiop->uio_offset += xfer;
-			uio_uio_resid_add(uiop, -xfer);
-		}
-		uio_iov_base_add(uiop, uiosiz);
-		uio_iov_len_add(uiop, -uiosiz);
-		siz -= uiosiz;
-	}
-	if (rem > 0) {
-		if (rem > mbuf_trailingspace(mp)) {
-			error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mp);
-			if (!error)
-				error = mbuf_setnext(mp2, mp);
-			if (error)
-				return (error);
-			mplen = 0;
-		}
-		cp = (caddr_t)mbuf_data(mp) + mplen;
-		for (left = 0; left < rem; left++)
-			*cp++ = '\0';
-		mplen += rem;
-		mbuf_setlen(mp, mplen);
-		*bpos = cp;
-	} else {
-		*bpos = (caddr_t)mbuf_data(mp) + mplen;
+	while (len) {
+		if (nmc->nmc_left >= len) {
+			nmc->nmc_left -= len;
+			nmc->nmc_ptr += len;
+			return (0);
+		}
+		len -= nmc->nmc_left;
+		nmc->nmc_mcur = mb = mbuf_next(nmc->nmc_mcur);
+		if (!mb)
+			return (EBADRPC);
+		nmc->nmc_ptr = mbuf_data(mb);
+		nmc->nmc_left = mbuf_len(mb);
 	}
-	*mq = mp;
+
 	return (0);
 }
 
 /*
- * Help break down an mbuf chain by setting the first siz bytes contiguous
- * pointed to by returned val.
- * This is used by the macros nfsm_dissect and nfsm_dissecton for tough
- * cases. (The macros use the vars. dpos and dpos2)
+ * nfsm_chain_reverse()
+ *
+ * Reverse decode offset in an nfsm_chain by "len" bytes.
  */
 int
-nfsm_disct(mdp, dposp, siz, left, cp2)
-	mbuf_t *mdp;
-	caddr_t *dposp;
-	int siz;
-	int left;
-	caddr_t *cp2;
+nfsm_chain_reverse(struct nfsm_chain *nmc, uint32_t len)
 {
-	mbuf_t mp, mp2;
-	int siz2, xfer, error, mp2len;
-	caddr_t p, mp2data;
-
-	mp = *mdp;
-	while (left == 0) {
-		*mdp = mp = mbuf_next(mp);
-		if (mp == NULL)
-			return (EBADRPC);
-		left = mbuf_len(mp);
-		*dposp = mbuf_data(mp);
+	uint32_t mlen, new_offset;
+	int error = 0;
+
+	mlen = nmc->nmc_ptr - (caddr_t) mbuf_data(nmc->nmc_mcur);
+	if (len <= mlen) {
+		nmc->nmc_ptr -= len;
+		nmc->nmc_left += len;
+		return (0);
 	}
-	if (left >= siz) {
-		*cp2 = *dposp;
-		*dposp += siz;
-	} else if (mbuf_next(mp) == NULL) {
-		return (EBADRPC);
-	} else if (siz > nfs_mbuf_mhlen) {
-		panic("nfs S too big");
-	} else {
-		error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mp2);
-		if (error)
+
+	new_offset = nfsm_chain_offset(nmc) - len;
+	nfsm_chain_dissect_init(error, nmc, nmc->nmc_mhead);
+	if (error)
+		return (error);
+
+	return (nfsm_chain_advance(nmc, new_offset));
+}
+
+/*
+ * nfsm_chain_get_opaque_pointer_f()
+ *
+ * Return a pointer to the next "len" bytes of contiguous data in
+ * the mbuf chain.  If the next "len" bytes are not contiguous, we
+ * try to manipulate the mbuf chain so that it is.
+ *
+ * The nfsm_chain is advanced by nfsm_rndup("len") bytes.
+ */
+int
+nfsm_chain_get_opaque_pointer_f(struct nfsm_chain *nmc, uint32_t len, u_char **pptr)
+{
+	mbuf_t mbcur, mb;
+	uint32_t left, need, mblen, cplen, padlen;
+	u_char *ptr;
+	int error = 0;
+
+	/* move to next mbuf with data */
+	while (nmc->nmc_mcur && (nmc->nmc_left == 0)) {
+		mb = mbuf_next(nmc->nmc_mcur);
+		nmc->nmc_mcur = mb;
+		if (!mb)
+			break;
+		nmc->nmc_ptr = mbuf_data(mb);
+		nmc->nmc_left = mbuf_len(mb);
+	}
+	/* check if we've run out of data */
+	if (!nmc->nmc_mcur)
+		return (EBADRPC);
+
+	/* do we already have a contiguous buffer? */
+	if (nmc->nmc_left >= len) {
+		/* the returned pointer will be the current pointer */
+		*pptr = (u_char*)nmc->nmc_ptr;
+		error = nfsm_chain_advance(nmc, nfsm_rndup(len));
+		return (error);
+	}
+
+	padlen = nfsm_rndup(len) - len;
+
+	/* we need (len - left) more bytes */
+	mbcur = nmc->nmc_mcur;
+	left = nmc->nmc_left;
+	need = len - left;
+
+	if (need > mbuf_trailingspace(mbcur)) {
+		/*
+		 * The needed bytes won't fit in the current mbuf so we'll
+		 * allocate a new mbuf to hold the contiguous range of data.
+		 */
+		nfsm_mbuf_get(error, &mb, len);
+		if (error)
 			return (error);
-		error = mbuf_setnext(mp2, mbuf_next(mp));
+		/* double check that this mbuf can hold all the data */
+		if (mbuf_maxlen(mb) < len) {
+			mbuf_free(mb);
+			return (EOVERFLOW);
+		}
+
+		/* the returned pointer will be the new mbuf's data pointer */
+		*pptr = ptr = mbuf_data(mb);
+
+		/* copy "left" bytes to the new mbuf */
+		bcopy(nmc->nmc_ptr, ptr, left);
+		ptr += left;
+		mbuf_setlen(mb, left);
+
+		/* insert the new mbuf between the current and next mbufs */
+		error = mbuf_setnext(mb, mbuf_next(mbcur));
 		if (!error)
-			error = mbuf_setnext(mp, mp2);
+			error = mbuf_setnext(mbcur, mb);
 		if (error) {
-			mbuf_free(mp2);
+			mbuf_free(mb);
 			return (error);
 		}
-		mbuf_setlen(mp, mbuf_len(mp) - left);
-		mp = mp2;
-		*cp2 = p = mbuf_data(mp);
-		bcopy(*dposp, p, left);		/* Copy what was left */
-		siz2 = siz-left;
-		p += left;
-		mp2 = mbuf_next(mp);
-		mp2data = mbuf_data(mp2);
-		mp2len = mbuf_len(mp2);
-		/* Loop around copying up the siz2 bytes */
-		while (siz2 > 0) {
-			if (mp2 == NULL)
-				return (EBADRPC);
-			xfer = (siz2 > mp2len) ? mp2len : siz2;
-			if (xfer > 0) {
-				bcopy(mp2data, p, xfer);
-				mp2data += xfer;
-				mp2len -= xfer;
-				mbuf_setdata(mp2, mp2data, mp2len);
-				p += xfer;
-				siz2 -= xfer;
-			}
-			if (siz2 > 0) {
-				mp2 = mbuf_next(mp2);
-				mp2data = mbuf_data(mp2);
-				mp2len = mbuf_len(mp2);
+
+		/* reduce current mbuf's length by "left" */
+		mbuf_setlen(mbcur, mbuf_len(mbcur) - left);
+
+		/*
+		 * update nmc's state to point at the end of the mbuf
+		 * where the needed data will be copied to.
+		 */
+		nmc->nmc_mcur = mbcur = mb;
+		nmc->nmc_left = 0;
+		nmc->nmc_ptr = (caddr_t)ptr;
+	} else {
+		/* The rest of the data will fit in this mbuf. */
+
+		/* the returned pointer will be the current pointer */
+		*pptr = (u_char*)nmc->nmc_ptr;
+
+		/*
+		 * update nmc's state to point at the end of the mbuf
+		 * where the needed data will be copied to.
+		 */
+		nmc->nmc_ptr += left;
+		nmc->nmc_left = 0;
+	}
+
+	/*
+	 * move the next "need" bytes into the current
+	 * mbuf from the mbufs that follow
+	 */
+
+	/* extend current mbuf length */
+	mbuf_setlen(mbcur, mbuf_len(mbcur) + need);
+
+	/* mb follows mbufs we're copying/compacting data from */
+	mb = mbuf_next(mbcur);
+
+	while (need && mb) {
+		/* copy as much as we need/can */
+		ptr = mbuf_data(mb);
+		mblen = mbuf_len(mb);
+		cplen = MIN(mblen, need);
+		if (cplen) {
+			bcopy(ptr, nmc->nmc_ptr, cplen);
+			/*
+			 * update the mbuf's pointer and length to reflect that
+			 * the data was shifted to an earlier mbuf in the chain
+			 */
+			error = mbuf_setdata(mb, ptr + cplen, mblen - cplen);
+			if (error) {
+				mbuf_setlen(mbcur, mbuf_len(mbcur) - need);
+				return (error);
 			}
+			/* update pointer/need */
+			nmc->nmc_ptr += cplen;
+			need -= cplen;
 		}
-		mbuf_setlen(mp, siz);
-		*mdp = mp2;
-		*dposp = mp2data;
+		/* if more needed, go to next mbuf */
+		if (need)
+			mb = mbuf_next(mb);
 	}
-	return (0);
+
+	/* did we run out of data in the mbuf chain? */
+	if (need) {
+		mbuf_setlen(mbcur, mbuf_len(mbcur) - need);
+		return (EBADRPC);
+	}
+
+	/*
+	 * update nmc's state to point after this contiguous data
+	 *
+	 * "mb" points to the last mbuf we copied data from so we
+	 * just set nmc to point at whatever remains in that mbuf.
+	 */
+	nmc->nmc_mcur = mb;
+	nmc->nmc_ptr = mbuf_data(mb);
+	nmc->nmc_left = mbuf_len(mb);
+
+	/* move past any padding */
+	if (padlen)
+		error = nfsm_chain_advance(nmc, padlen);
+
+	return (error);
 }
 
 /*
- * Advance the position in the mbuf chain.
+ * nfsm_chain_get_opaque_f()
+ *
+ * Read the next "len" bytes in the chain into "buf".
+ * The nfsm_chain is advanced by nfsm_rndup("len") bytes.
  */
 int
-nfs_adv(mdp, dposp, offs, left)
-	mbuf_t *mdp;
-	caddr_t *dposp;
-	int offs;
-	int left;
+nfsm_chain_get_opaque_f(struct nfsm_chain *nmc, uint32_t len, u_char *buf)
 {
-	mbuf_t m;
-	int s;
-
-	m = *mdp;
-	s = left;
-	while (s < offs) {
-		offs -= s;
-		m = mbuf_next(m);
-		if (m == NULL)
-			return (EBADRPC);
-		s = mbuf_len(m);
+	uint32_t cplen, padlen;
+	int error = 0;
+
+	padlen = nfsm_rndup(len) - len;
+
+	/* loop through mbufs copying all the data we need */
+	while (len && nmc->nmc_mcur) {
+		/* copy as much as we need/can */
+		cplen = MIN(nmc->nmc_left, len);
+		if (cplen) {
+			bcopy(nmc->nmc_ptr, buf, cplen);
+			nmc->nmc_ptr += cplen;
+			nmc->nmc_left -= cplen;
+			buf += cplen;
+			len -= cplen;
+		}
+		/* if more needed, go to next mbuf */
+		if (len) {
+			mbuf_t mb = mbuf_next(nmc->nmc_mcur);
+			nmc->nmc_mcur = mb;
+			nmc->nmc_ptr = mb ? mbuf_data(mb) : NULL;
+			nmc->nmc_left = mb ? mbuf_len(mb) : 0;
+		}
 	}
-	*mdp = m;
-	*dposp = (caddr_t)mbuf_data(m) + offs;
-	return (0);
+
+	/* did we run out of data in the mbuf chain? */
+	if (len)
+		return (EBADRPC);
+
+	if (padlen)
+		nfsm_chain_adv(error, nmc, padlen);
+
+	return (error);
 }
 
 /*
- * Copy a string into mbufs for the hard cases...
+ * nfsm_chain_get_uio()
+ *
+ * Read the next "len" bytes in the chain into the given uio.
+ * The nfsm_chain is advanced by nfsm_rndup("len") bytes.
  */
 int
-nfsm_strtmbuf(mb, bpos, cp, siz)
-	mbuf_t *mb;
-	char **bpos;
-	char *cp;
-	long siz;
+nfsm_chain_get_uio(struct nfsm_chain *nmc, uint32_t len, struct uio *uiop)
 {
-	mbuf_t m1 = NULL, m2;
-	long left, xfer, len, tlen, mlen;
-	u_long *tl;
-	int putsize, error;
-
-	putsize = 1;
-	m2 = *mb;
-	left = mbuf_trailingspace(m2);
-	if (left >= NFSX_UNSIGNED) {
-		tl = ((u_long *)(*bpos));
-		*tl++ = txdr_unsigned(siz);
-		putsize = 0;
-		left -= NFSX_UNSIGNED;
-		len = mbuf_len(m2);
-		len += NFSX_UNSIGNED;
-		mbuf_setlen(m2, len);
-		if (left > 0) {
-			bcopy(cp, (caddr_t) tl, left);
-			siz -= left;
-			cp += left;
-			len += left;
-			mbuf_setlen(m2, len);
-			left = 0;
-		}
-	}
-	/* Loop around adding mbufs */
-	while (siz > 0) {
-		m1 = NULL;
-		if (siz > nfs_mbuf_mlen)
-			error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &m1);
-		else
-			error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &m1);
-		if (!error)
-			error = mbuf_setnext(m2, m1);
-		if (error)
-			return (error);
-		mlen = mbuf_maxlen(m1);
-		mbuf_setlen(m1, mlen);
-		m2 = m1;
-		tl = mbuf_data(m1);
-		tlen = 0;
-		if (putsize) {
-			*tl++ = txdr_unsigned(siz);
-			mlen -= NFSX_UNSIGNED;
-			mbuf_setlen(m1, mlen);
-			tlen = NFSX_UNSIGNED;
-			putsize = 0;
-		}
-		if (siz < mlen) {
-			len = nfsm_rndup(siz);
-			xfer = siz;
-			if (xfer < len)
-				*(tl+(xfer>>2)) = 0;
+	uint32_t cplen, padlen;
+	int error = 0;
+
+	padlen = nfsm_rndup(len) - len;
+
+	/* loop through mbufs copying all the data we need */
+	while (len && nmc->nmc_mcur) {
+		/* copy as much as we need/can */
+		cplen = MIN(nmc->nmc_left, len);
+		if (cplen) {
+			error = uiomove(nmc->nmc_ptr, cplen, uiop);
+			if (error)
+				return (error);
+			nmc->nmc_ptr += cplen;
+			nmc->nmc_left -= cplen;
+			len -= cplen;
+		}
+		/* if more needed, go to next mbuf */
+		if (len) {
+			mbuf_t mb = mbuf_next(nmc->nmc_mcur);
+			nmc->nmc_mcur = mb;
+			nmc->nmc_ptr = mb ? mbuf_data(mb) : NULL;
+			nmc->nmc_left = mb ? mbuf_len(mb) : 0;
+		}
+	}
+
+	/* did we run out of data in the mbuf chain? */
+	if (len)
+		return (EBADRPC);
+
+	if (padlen)
+		nfsm_chain_adv(error, nmc, padlen);
+
+	return (error);
+}
+
+#if NFSCLIENT
+
+/*
+ * Add an NFSv2 "sattr" structure to an mbuf chain
+ */
+int
+nfsm_chain_add_v2sattr_f(struct nfsm_chain *nmc, struct vnode_attr *vap, uint32_t szrdev)
+{
+	int error = 0;
+
+	nfsm_chain_add_32(error, nmc, vtonfsv2_mode(vap->va_type,
+		(VATTR_IS_ACTIVE(vap, va_mode) ? vap->va_mode : 0600)));
+	nfsm_chain_add_32(error, nmc,
+		VATTR_IS_ACTIVE(vap, va_uid) ? vap->va_uid : (uint32_t)-1);
+	nfsm_chain_add_32(error, nmc,
+		VATTR_IS_ACTIVE(vap, va_gid) ? vap->va_gid : (uint32_t)-1);
+	nfsm_chain_add_32(error, nmc, szrdev);
+	nfsm_chain_add_v2time(error, nmc,
+		VATTR_IS_ACTIVE(vap, va_access_time) ?
+		&vap->va_access_time : NULL);
+	nfsm_chain_add_v2time(error, nmc,
+		VATTR_IS_ACTIVE(vap, va_modify_time) ?
+		&vap->va_modify_time : NULL);
+
+	return (error);
+}
+
+/*
+ * Add an NFSv3 "sattr" structure to an mbuf chain
+ */
+int
+nfsm_chain_add_v3sattr_f(struct nfsm_chain *nmc, struct vnode_attr *vap)
+{
+	int error = 0;
+
+	if (VATTR_IS_ACTIVE(vap, va_mode)) {
+		nfsm_chain_add_32(error, nmc, TRUE);
+		nfsm_chain_add_32(error, nmc, vap->va_mode);
+	} else {
+		nfsm_chain_add_32(error, nmc, FALSE);
+	}
+	if (VATTR_IS_ACTIVE(vap, va_uid)) {
+		nfsm_chain_add_32(error, nmc, TRUE);
+		nfsm_chain_add_32(error, nmc, vap->va_uid);
+	} else {
+		nfsm_chain_add_32(error, nmc, FALSE);
+	}
+	if (VATTR_IS_ACTIVE(vap, va_gid)) {
+		nfsm_chain_add_32(error, nmc, TRUE);
+		nfsm_chain_add_32(error, nmc, vap->va_gid);
+	} else {
+		nfsm_chain_add_32(error, nmc, FALSE);
+	}
+	if (VATTR_IS_ACTIVE(vap, va_data_size)) {
+		nfsm_chain_add_32(error, nmc, TRUE);
+		nfsm_chain_add_64(error, nmc, vap->va_data_size);
+	} else {
+		nfsm_chain_add_32(error, nmc, FALSE);
+	}
+	if (vap->va_vaflags & VA_UTIMES_NULL) {
+		nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_SERVER);
+		nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_SERVER);
+	} else {
+		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_CLIENT);
+			nfsm_chain_add_32(error, nmc, vap->va_access_time.tv_sec);
+			nfsm_chain_add_32(error, nmc, vap->va_access_time.tv_nsec);
+		} else {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_DONT_CHANGE);
+		}
+		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
+			nfsm_chain_add_32(error, nmc, NFS_TIME_SET_TO_CLIENT);
+			nfsm_chain_add_32(error, nmc, vap->va_modify_time.tv_sec);
+			nfsm_chain_add_32(error, nmc, vap->va_modify_time.tv_nsec);
 		} else {
-			xfer = len = mlen;
+			nfsm_chain_add_32(error, nmc, NFS_TIME_DONT_CHANGE);
 		}
-		bcopy(cp, (caddr_t) tl, xfer);
-		mbuf_setlen(m1, len + tlen);
-		siz -= xfer;
-		cp += xfer;
 	}
-	*mb = m1;
-	*bpos = (caddr_t)mbuf_data(m1) + mbuf_len(m1);
-	return (0);
+
+	return (error);
 }
 
+
 /*
- * Called once to initialize data structures...
+ * nfsm_chain_get_fh_attr()
+ *
+ * Get the file handle and attributes from an mbuf chain. (NFSv2/v3)
  */
 int
-nfs_init(struct vfsconf *vfsp)
+nfsm_chain_get_fh_attr(
+	struct nfsm_chain *nmc,
+	nfsnode_t dnp,
+	vfs_context_t ctx,
+	int nfsvers,
+	uint64_t *xidp,
+	fhandle_t *fhp,
+	struct nfs_vattr *nvap)
 {
-	int i;
+	int error = 0, gotfh, gotattr;
 
-	/*
-	 * Check to see if major data structures haven't bloated.
-	 */
-	if (sizeof (struct nfsnode) > NFS_NODEALLOC) {
-		printf("struct nfsnode bloated (> %dbytes)\n", NFS_NODEALLOC);
-		printf("Try reducing NFS_SMALLFH\n");
-	}
-	if (sizeof (struct nfsmount) > NFS_MNTALLOC) {
-		printf("struct nfsmount bloated (> %dbytes)\n", NFS_MNTALLOC);
-		printf("Try reducing NFS_MUIDHASHSIZ\n");
-	}
-	if (sizeof (struct nfssvc_sock) > NFS_SVCALLOC) {
-		printf("struct nfssvc_sock bloated (> %dbytes)\n",NFS_SVCALLOC);
-		printf("Try reducing NFS_UIDHASHSIZ\n");
-	}
-	if (sizeof (struct nfsuid) > NFS_UIDALLOC) {
-		printf("struct nfsuid bloated (> %dbytes)\n",NFS_UIDALLOC);
-		printf("Try unionizing the nu_nickname and nu_flag fields\n");
-	}
-
-	nfs_mount_type = vfsp->vfc_typenum;
-	nfsrtt.pos = 0;
-	rpc_vers = txdr_unsigned(RPC_VER2);
-	rpc_call = txdr_unsigned(RPC_CALL);
-	rpc_reply = txdr_unsigned(RPC_REPLY);
-	rpc_msgdenied = txdr_unsigned(RPC_MSGDENIED);
-	rpc_msgaccepted = txdr_unsigned(RPC_MSGACCEPTED);
-	rpc_mismatch = txdr_unsigned(RPC_MISMATCH);
-	rpc_autherr = txdr_unsigned(RPC_AUTHERR);
-	rpc_auth_unix = txdr_unsigned(RPCAUTH_UNIX);
-	rpc_auth_kerb = txdr_unsigned(RPCAUTH_KERB4);
-	nfs_prog = txdr_unsigned(NFS_PROG);
-	nfs_true = txdr_unsigned(TRUE);
-	nfs_false = txdr_unsigned(FALSE);
-	nfs_xdrneg1 = txdr_unsigned(-1);
-
-	nfs_ticks = (hz * NFS_TICKINTVL + 500) / 1000;
-	if (nfs_ticks < 1)
-		nfs_ticks = 1;
-	/* Ensure async daemons disabled */
-	for (i = 0; i < NFS_MAXASYNCDAEMON; i++) {
-		nfs_iodwant[i] = NULL;
-		nfs_iodmount[i] = (struct nfsmount *)0;
-	}
-	/* init nfsiod mutex */
-	nfs_iod_lck_grp_attr = lck_grp_attr_alloc_init();
-	nfs_iod_lck_grp = lck_grp_alloc_init("nfs_iod", nfs_iod_lck_grp_attr);
-	nfs_iod_lck_attr = lck_attr_alloc_init();
-	nfs_iod_mutex = lck_mtx_alloc_init(nfs_iod_lck_grp, nfs_iod_lck_attr);
-
-	nfs_nbinit();			/* Init the nfsbuf table */
-	nfs_nhinit();			/* Init the nfsnode table */
-	nfs_lockinit();			/* Init the nfs lock state */
-
-#ifndef NFS_NOSERVER
-	/* init nfsd mutex */
-	nfsd_lck_grp_attr = lck_grp_attr_alloc_init();
-	nfsd_lck_grp = lck_grp_alloc_init("nfsd", nfsd_lck_grp_attr);
-	nfsd_lck_attr = lck_attr_alloc_init();
-	nfsd_mutex = lck_mtx_alloc_init(nfsd_lck_grp, nfsd_lck_attr);
-
-	/* init slp rwlock */
-	nfs_slp_lock_attr    = lck_attr_alloc_init();
-	nfs_slp_group_attr   = lck_grp_attr_alloc_init();
-	nfs_slp_rwlock_group = lck_grp_alloc_init("nfs-slp-rwlock", nfs_slp_group_attr);
-	nfs_slp_mutex_group  = lck_grp_alloc_init("nfs-slp-mutex", nfs_slp_group_attr);
-
-	/* init export data structures */
-	nfsexphashtbl = hashinit(8, M_TEMP, &nfsexphash);
-	LIST_INIT(&nfs_exports);
-	nfs_export_lock_attr    = lck_attr_alloc_init();
-	nfs_export_group_attr   = lck_grp_attr_alloc_init();
-	nfs_export_rwlock_group = lck_grp_alloc_init("nfs-export-rwlock", nfs_export_group_attr);
-	lck_rw_init(&nfs_export_rwlock, nfs_export_rwlock_group, nfs_export_lock_attr);
-
-	lck_mtx_lock(nfsd_mutex);
-	nfsrv_init(0);			/* Init server data structures */
-	nfsrv_initcache();		/* Init the server request cache */
-	lck_mtx_unlock(nfsd_mutex);
-#endif
+	gotfh = gotattr = 1;
 
-	/*
-	 * Initialize reply list and start timer
-	 */
-	TAILQ_INIT(&nfs_reqq);
+	if (nfsvers == NFS_VER3) /* check for file handle */
+		nfsm_chain_get_32(error, nmc, gotfh);
+	if (!error && gotfh) /* get file handle */
+		nfsm_chain_get_fh(error, nmc, nfsvers, fhp);
+	else
+		fhp->fh_len = 0;
+	if (nfsvers == NFS_VER3) /* check for file attributes */
+		nfsm_chain_get_32(error, nmc, gotattr);
+	nfsmout_if(error);
+	if (gotattr) {
+		if (!gotfh) /* skip attributes */
+			nfsm_chain_adv(error, nmc, NFSX_V3FATTR);
+		else /* get attributes */
+			error = nfs_parsefattr(nmc, nfsvers, nvap);
+	} else if (gotfh) {
+		/* we need valid attributes in order to call nfs_nget() */
+		if (nfs3_getattr_rpc(NULL, NFSTOMP(dnp), fhp->fh_data, fhp->fh_len, ctx, nvap, xidp)) {
+			gotattr = 0;
+			fhp->fh_len = 0;
+		}
+	}
+nfsmout:
+	return (error);
+}
 
-	nfs_timer(0);
+/*
+ * Get and process NFSv3 WCC data from an mbuf chain
+ */
+int
+nfsm_chain_get_wcc_data_f(
+	struct nfsm_chain *nmc,
+	nfsnode_t np,
+	struct timespec *premtime,
+	int *newpostattr,
+	u_int64_t *xidp)
+{
+	int error = 0;
+	uint32_t flag = 0;
+
+	nfsm_chain_get_32(error, nmc, flag);
+	if (!error && flag) {
+		nfsm_chain_adv(error, nmc, 2 * NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, nmc, premtime->tv_sec);
+		nfsm_chain_get_32(error, nmc, premtime->tv_nsec);
+		nfsm_chain_adv(error, nmc, 2 * NFSX_UNSIGNED);
+	} else {
+		premtime->tv_sec = 0;
+		premtime->tv_nsec = 0;
+	}
+	nfsm_chain_postop_attr_update_flag(error, nmc, np, *newpostattr, xidp);
 
-	vfsp->vfc_refcount++; /* make us non-unloadable */
-	return (0);
+	return (error);
 }
 
 /*
- * initialize NFS's cache of mbuf constants
+ * Build the RPC header and fill in the authorization info.
+ * Returns the head of the mbuf list and the xid.
  */
-void
-nfs_mbuf_init(void)
+
+int
+nfsm_rpchead(
+	struct nfsreq *req,
+	int auth_len,
+	mbuf_t mrest,
+	u_int64_t *xidp,
+	mbuf_t *mreqp)
 {
-	struct mbuf_stat ms;
+	struct nfsmount *nmp = req->r_nmp;
+	int nfsvers = nmp->nm_vers;
+	int proc = ((nfsvers == NFS_VER2) ? nfsv2_procid[req->r_procnum] : (int)req->r_procnum);
+	int auth_type = (!auth_len && !req->r_cred) ? RPCAUTH_NULL : nmp->nm_auth;
 
-	mbuf_stats(&ms);
-	nfs_mbuf_mlen = ms.mlen;
-	nfs_mbuf_mhlen = ms.mhlen;
-	nfs_mbuf_minclsize = ms.minclsize;
-	nfs_mbuf_mclbytes = ms.mclbytes;
+	return nfsm_rpchead2(nmp->nm_sotype, NFS_PROG, nfsvers, proc,
+		auth_type, auth_len, req->r_cred, req, mrest, xidp, mreqp);
+}
+
+int
+nfsm_rpchead2(int sotype, int prog, int vers, int proc, int auth_type, int auth_len,
+	kauth_cred_t cred, struct nfsreq *req, mbuf_t mrest, u_int64_t *xidp, mbuf_t *mreqp)
+{
+	mbuf_t mreq, mb;
+	int error, i, grpsiz, authsiz, reqlen;
+	size_t headlen;
+	struct timeval tv;
+	struct nfsm_chain nmreq;
+
+	/* allocate the packet */
+	authsiz = nfsm_rndup(auth_len);
+	headlen = authsiz + 10 * NFSX_UNSIGNED;
+	if (sotype == SOCK_STREAM) /* also include room for any RPC Record Mark */
+		headlen += NFSX_UNSIGNED;
+	if (headlen >= nfs_mbuf_minclsize) {
+		error = mbuf_getpacket(MBUF_WAITOK, &mreq);
+	} else {
+		error = mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &mreq);
+		if (!error) {
+			if (headlen < nfs_mbuf_mhlen)
+				mbuf_align_32(mreq, headlen);
+			else
+				mbuf_align_32(mreq, 8 * NFSX_UNSIGNED);
+		}
+	}
+	if (error) {
+		/* unable to allocate packet */
+		/* XXX should we keep statistics for these errors? */
+		return (error);
+	}
+
+	/*
+	 * If the caller gave us a non-zero XID then use it because
+	 * it may be a higher-level resend with a GSSAPI credential.
+	 * Otherwise, allocate a new one.
+	 */
+	if (*xidp == 0) {
+		lck_mtx_lock(nfs_request_mutex);
+		if (!nfs_xid) {
+			/*
+			 * Derive initial xid from system time.
+			 *
+			 * Note: it's OK if this code inits nfs_xid to 0 (for example,
+			 * due to a broken clock) because we immediately increment it
+			 * and we guarantee to never use xid 0.  So, nfs_xid should only
+			 * ever be 0 the first time this function is called.
+			 */
+			microtime(&tv);
+			nfs_xid = tv.tv_sec << 12;
+		}
+		if (++nfs_xid == 0) {
+			/* Skip zero xid if it should ever happen. */
+			nfs_xidwrap++;
+			nfs_xid++;
+		}
+		*xidp = nfs_xid + ((u_int64_t)nfs_xidwrap << 32);
+		lck_mtx_unlock(nfs_request_mutex);
+	}
+
+	/* build the header(s) */
+	nmreq.nmc_mcur = nmreq.nmc_mhead = mreq;
+	nmreq.nmc_ptr = mbuf_data(nmreq.nmc_mcur);
+	nmreq.nmc_left = mbuf_trailingspace(nmreq.nmc_mcur);
+
+	/* First, if it's a TCP stream insert space for an RPC record mark */
+	if (sotype == SOCK_STREAM)
+		nfsm_chain_add_32(error, &nmreq, 0);
+
+	/* Then the RPC header. */
+	nfsm_chain_add_32(error, &nmreq, (*xidp & 0xffffffff));
+	nfsm_chain_add_32(error, &nmreq, RPC_CALL);
+	nfsm_chain_add_32(error, &nmreq, RPC_VER2);
+	nfsm_chain_add_32(error, &nmreq, prog);
+	nfsm_chain_add_32(error, &nmreq, vers);
+	nfsm_chain_add_32(error, &nmreq, proc);
+
+add_cred:
+	switch (auth_type) {
+	case RPCAUTH_NULL:
+		nfsm_chain_add_32(error, &nmreq, RPCAUTH_NULL); /* auth */
+		nfsm_chain_add_32(error, &nmreq, 0);		/* length */
+		nfsm_chain_add_32(error, &nmreq, RPCAUTH_NULL);	/* verf */
+		nfsm_chain_add_32(error, &nmreq, 0);		/* length */
+		nfsm_chain_build_done(error, &nmreq);
+		break;
+	case RPCAUTH_UNIX:
+		nfsm_chain_add_32(error, &nmreq, RPCAUTH_UNIX);
+		nfsm_chain_add_32(error, &nmreq, authsiz);
+		nfsm_chain_add_32(error, &nmreq, 0);	/* stamp */
+		nfsm_chain_add_32(error, &nmreq, 0);	/* zero-length hostname */
+		nfsm_chain_add_32(error, &nmreq, kauth_cred_getuid(cred));	/* UID */
+		nfsm_chain_add_32(error, &nmreq, cred->cr_groups[0]);	/* GID */
+		grpsiz = (auth_len >> 2) - 5;
+		nfsm_chain_add_32(error, &nmreq, grpsiz);/* additional GIDs */
+		for (i = 1; i <= grpsiz; i++)
+			nfsm_chain_add_32(error, &nmreq, cred->cr_groups[i]);
+
+		/* And the verifier... */
+		nfsm_chain_add_32(error, &nmreq, RPCAUTH_NULL);	/* flavor */
+		nfsm_chain_add_32(error, &nmreq, 0);		/* length */
+		nfsm_chain_build_done(error, &nmreq);
+
+		/* Append the args mbufs */
+		if (!error)
+			error = mbuf_setnext(nmreq.nmc_mcur, mrest);
+		break;
+	case RPCAUTH_KRB5:
+	case RPCAUTH_KRB5I:
+	case RPCAUTH_KRB5P:
+		error = nfs_gss_clnt_cred_put(req, &nmreq, mrest);
+		if (error == ENEEDAUTH) {
+			/*
+			 * Use sec=sys for this user
+			 */
+			error = 0;
+			auth_type = RPCAUTH_UNIX;
+			goto add_cred;
+		}
+		break;
+	};
+
+	/* finish setting up the packet */
+	if (!error)
+		error = mbuf_pkthdr_setrcvif(mreq, 0);
+
+	if (error) {
+		mbuf_freem(mreq);
+		return (error);
+	}
+
+	/* Calculate the size of the request */
+	reqlen = 0;
+	for (mb = nmreq.nmc_mhead; mb; mb = mbuf_next(mb))
+		reqlen += mbuf_len(mb);
+
+	mbuf_pkthdr_setlen(mreq, reqlen);
+
+	/*
+	 * If the request goes on a TCP stream,
+	 * set its size in the RPC record mark.
+	 * The record mark count doesn't include itself
+	 * and the last fragment bit is set.
+	 */
+	if (sotype == SOCK_STREAM)
+		nfsm_chain_set_recmark(error, &nmreq,
+			(reqlen - NFSX_UNSIGNED) | 0x80000000);
+
+	*mreqp = mreq;
+	return (0);
 }
 
 /*
- * Parse the attributes that are in the mbuf list and store them in *nvap.
+ * Parse an NFS file attribute structure out of an mbuf chain.
  */
 int
-nfs_parsefattr(mbuf_t *mdp, caddr_t *dposp, int v3, struct nfs_vattr *nvap)
+nfs_parsefattr(struct nfsm_chain *nmc, int nfsvers, struct nfs_vattr *nvap)
 {
-	struct nfs_fattr *fp;
-	long t1;
-	caddr_t cp2;
-	int error = 0, rdev;
-	mbuf_t md;
+	int error = 0;
 	enum vtype vtype;
 	u_short vmode;
+	uint32_t val, val2;
+	dev_t rdev;
 
-	md = *mdp;
-	t1 = ((caddr_t)mbuf_data(md) + mbuf_len(md)) - *dposp;
-	if ((error = nfsm_disct(mdp, dposp, NFSX_FATTR(v3), t1, &cp2))) {
-		return (error);
-	}
-	fp = (struct nfs_fattr *)cp2;
-	if (v3) {
-		vtype = nfsv3tov_type(fp->fa_type);
-		vmode = fxdr_unsigned(u_short, fp->fa_mode);
-		rdev = makedev(fxdr_unsigned(int, fp->fa3_rdev.specdata1),
-			fxdr_unsigned(int, fp->fa3_rdev.specdata2));
+	val = val2 = 0;
+
+	nfsm_chain_get_32(error, nmc, vtype);
+	nfsm_chain_get_32(error, nmc, vmode);
+	nfsmout_if(error);
+
+	if (nfsvers == NFS_VER3) {
+		nvap->nva_type = nfstov_type(vtype, nfsvers);
 	} else {
-		vtype = nfsv2tov_type(fp->fa_type);
-		vmode = fxdr_unsigned(u_short, fp->fa_mode);
 		/*
-		 * XXX
-		 *
 		 * The duplicate information returned in fa_type and fa_mode
 		 * is an ambiguity in the NFS version 2 protocol.
 		 *
 		 * VREG should be taken literally as a regular file.  If a
-		 * server intents to return some type information differently
+		 * server intends to return some type information differently
 		 * in the upper bits of the mode field (e.g. for sockets, or
 		 * FIFOs), NFSv2 mandates fa_type to be VNON.  Anyway, we
 		 * leave the examination of the mode bits even in the VREG
@@ -1319,44 +1142,62 @@ nfs_parsefattr(mbuf_t *mdp, caddr_t *dposp, int v3, struct nfs_vattr *nvap)
 		 * fa_mode (and failing that, trust the va_type field).
 		 *
 		 * NFSv3 cleared the issue, and requires fa_mode to not
-		 * contain any type information (while also introduing sockets
-		 * and FIFOs for fa_type).
+		 * contain any type information (while also introducing
+		 * sockets and FIFOs for fa_type).
 		 */
-		if (vtype == VNON || (vtype == VREG && (vmode & S_IFMT) != 0))
+		vtype = nfstov_type(vtype, nfsvers);
+		if ((vtype == VNON) || ((vtype == VREG) && ((vmode & S_IFMT) != 0)))
 			vtype = IFTOVT(vmode);
-		rdev = fxdr_unsigned(long, fp->fa2_rdev);
-		/*
-		 * Really ugly NFSv2 kludge.
-		 */
-		if (vtype == VCHR && rdev == (int)0xffffffff)
-			vtype = VFIFO;
+		nvap->nva_type = vtype;
 	}
 
-	nvap->nva_type = vtype;
 	nvap->nva_mode = (vmode & 07777);
-	nvap->nva_rdev = (dev_t)rdev;
-	nvap->nva_nlink = (uint64_t)fxdr_unsigned(u_long, fp->fa_nlink);
-	nvap->nva_uid = fxdr_unsigned(uid_t, fp->fa_uid);
-	nvap->nva_gid = fxdr_unsigned(gid_t, fp->fa_gid);
-	if (v3) {
-		fxdr_hyper(&fp->fa3_size, &nvap->nva_size);
-		nvap->nva_blocksize = 16*1024;
-		fxdr_hyper(&fp->fa3_used, &nvap->nva_bytes);
-		fxdr_hyper(&fp->fa3_fileid, &nvap->nva_fileid);
-		fxdr_nfsv3time(&fp->fa3_atime, &nvap->nva_atime);
-		fxdr_nfsv3time(&fp->fa3_mtime, &nvap->nva_mtime);
-		fxdr_nfsv3time(&fp->fa3_ctime, &nvap->nva_ctime);
-	} else {
-		nvap->nva_size = fxdr_unsigned(u_long, fp->fa2_size);
-		nvap->nva_blocksize = fxdr_unsigned(long, fp->fa2_blocksize);
-		nvap->nva_bytes = fxdr_unsigned(long, fp->fa2_blocks) * NFS_FABLKSIZE;
-		nvap->nva_fileid = (uint64_t)fxdr_unsigned(u_long, fp->fa2_fileid);
-		fxdr_nfsv2time(&fp->fa2_atime, &nvap->nva_atime);
-		fxdr_nfsv2time(&fp->fa2_mtime, &nvap->nva_mtime);
-		fxdr_nfsv2time(&fp->fa2_ctime, &nvap->nva_ctime);
-	}
 
-	return (0);
+	nfsm_chain_get_32(error, nmc, nvap->nva_nlink);
+	nfsm_chain_get_32(error, nmc, nvap->nva_uid);
+	nfsm_chain_get_32(error, nmc, nvap->nva_gid);
+
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_get_64(error, nmc, nvap->nva_size);
+		nfsm_chain_get_64(error, nmc, nvap->nva_bytes);
+		nfsm_chain_get_32(error, nmc, nvap->nva_rawdev.specdata1);
+		nfsm_chain_get_32(error, nmc, nvap->nva_rawdev.specdata2);
+		nfsmout_if(error);
+		nfsm_chain_get_64(error, nmc, nvap->nva_fsid.major);
+		nvap->nva_fsid.minor = 0;
+		nfsm_chain_get_64(error, nmc, nvap->nva_fileid);
+	} else {
+		nfsm_chain_get_32(error, nmc, nvap->nva_size);
+		nfsm_chain_adv(error, nmc, NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, nmc, rdev);
+		nfsmout_if(error);
+		nvap->nva_rawdev.specdata1 = major(rdev);
+		nvap->nva_rawdev.specdata2 = minor(rdev);
+		nfsm_chain_get_32(error, nmc, val); /* blocks */
+		nfsmout_if(error);
+		nvap->nva_bytes = val * NFS_FABLKSIZE;
+		nfsm_chain_get_32(error, nmc, val);
+		nfsmout_if(error);
+		nvap->nva_fsid.major = (uint64_t)val;
+		nvap->nva_fsid.minor = 0;
+		nfsm_chain_get_32(error, nmc, val);
+		nfsmout_if(error);
+		nvap->nva_fileid = (uint64_t)val;
+		/* Really ugly NFSv2 kludge. */
+		if ((vtype == VCHR) && (rdev == (dev_t)0xffffffff))
+			nvap->nva_type = VFIFO;
+	}
+	nfsm_chain_get_time(error, nmc, nfsvers,
+		nvap->nva_timesec[NFSTIME_ACCESS],
+		nvap->nva_timensec[NFSTIME_ACCESS]);
+	nfsm_chain_get_time(error, nmc, nfsvers,
+		nvap->nva_timesec[NFSTIME_MODIFY],
+		nvap->nva_timensec[NFSTIME_MODIFY]);
+	nfsm_chain_get_time(error, nmc, nfsvers,
+		nvap->nva_timesec[NFSTIME_CHANGE],
+		nvap->nva_timensec[NFSTIME_CHANGE]);
+nfsmout:
+	return (error);
 }
 
 /*
@@ -1370,7 +1211,7 @@ nfs_parsefattr(mbuf_t *mdp, caddr_t *dposp, int v3, struct nfs_vattr *nvap)
  */
 int
 nfs_loadattrcache(
-	struct nfsnode *np,
+	nfsnode_t np,
 	struct nfs_vattr *nvap,
 	u_int64_t *xidp,
 	int dontshrink)
@@ -1380,7 +1221,7 @@ nfs_loadattrcache(
 	struct timeval now;
 	struct nfs_vattr *npnvap;
 
-	if (np->n_flag & NINIT) {
+	if (np->n_hflag & NHINIT) {
 		vp = NULL;
 		mp = np->n_mount;
 	} else {
@@ -1388,18 +1229,18 @@ nfs_loadattrcache(
 		mp = vnode_mount(vp);
 	}
 
-	FSDBG_TOP(527, vp, np, *xidp >> 32, *xidp);
+	FSDBG_TOP(527, np, vp, *xidp >> 32, *xidp);
 
 	if (!VFSTONFS(mp)) {
 		FSDBG_BOT(527, ENXIO, 1, 0, *xidp);
-		return (ENXIO); 
+		return (ENXIO);
 	}
 
 	if (*xidp < np->n_xid) {
 		/*
 		 * We have already updated attributes with a response from
 		 * a later request.  The attributes we have here are probably
-		 * stale so we drop them (just return).  However, our 
+		 * stale so we drop them (just return).  However, our
 		 * out-of-order receipt could be correct - if the requests were
 		 * processed out of order at the server.  Given the uncertainty
 		 * we invalidate our cached attributes.  *xidp is zeroed here
@@ -1445,45 +1286,46 @@ nfs_loadattrcache(
 	np->n_xid = *xidp;
 
 	npnvap = &np->n_vattr;
-	nvap->nva_fsid = vfs_statfs(mp)->f_fsid.val[0];
 	bcopy((caddr_t)nvap, (caddr_t)npnvap, sizeof(*nvap));
 
-	if (vp) {
-		if (nvap->nva_size != np->n_size) {
-			FSDBG(527, vp, nvap->nva_size, np->n_size,
-			      (nvap->nva_type == VREG) |
-			      (np->n_flag & NMODIFIED ? 6 : 4));
-			if (nvap->nva_type == VREG) {
-				u_quad_t orig_size = np->n_size;
-				if (np->n_flag & NMODIFIED) {
-					if (nvap->nva_size < np->n_size)
-						nvap->nva_size = np->n_size;
-					else
-						np->n_size = nvap->nva_size;
-				} else
-					np->n_size = nvap->nva_size;
-				if (!UBCINFOEXISTS(vp) ||
-				    (dontshrink && np->n_size < (u_quad_t)ubc_getsize(vp))) {
-					nvap->nva_size = np->n_size = orig_size;
-					NATTRINVALIDATE(np);
-				} else {
-					ubc_setsize(vp, (off_t)np->n_size); /* XXX */
-				}
-			} else
-				np->n_size = nvap->nva_size;
+	if (nvap->nva_size != np->n_size) {
+		/*
+		 * n_size is protected by the data lock, so we need to
+		 * defer updating it until it's safe.  We save the new size
+		 * and set a flag and it'll get updated the next time we get/drop
+		 * the data lock or the next time we do a getattr.
+		 */
+		np->n_newsize = nvap->nva_size;
+		FSDBG(527, np, nvap->nva_size, np->n_size, (nvap->nva_type == VREG) | (np->n_flag & NMODIFIED ? 6 : 4));
+		SET(np->n_flag, NUPDATESIZE);
+		if (vp && (nvap->nva_type == VREG)) {
+			if (!UBCINFOEXISTS(vp) || (dontshrink && (np->n_newsize < np->n_size))) {
+				/* asked not to shrink, so stick with current size */
+				FSDBG(527, np, np->n_size, np->n_vattr.nva_size, 0xf00d0001);
+				nvap->nva_size = np->n_size;
+				CLR(np->n_flag, NUPDATESIZE);
+				NATTRINVALIDATE(np);
+			} else if ((np->n_flag & NMODIFIED) && (nvap->nva_size < np->n_size)) {
+				/* if we've modified, use larger size */
+				FSDBG(527, np, np->n_size, np->n_vattr.nva_size, 0xf00d0002);
+				nvap->nva_size = np->n_size;
+				CLR(np->n_flag, NUPDATESIZE);
+			}
 		}
-	} else {
-		np->n_size = nvap->nva_size;
 	}
 
 	if (np->n_flag & NCHG) {
-		if (np->n_flag & NACC)
-			nvap->nva_atime = np->n_atim;
-		if (np->n_flag & NUPD)
-			nvap->nva_mtime = np->n_mtim;
+		if (np->n_flag & NACC) {
+			nvap->nva_timesec[NFSTIME_ACCESS] = np->n_atim.tv_sec;
+			nvap->nva_timensec[NFSTIME_ACCESS] = np->n_atim.tv_nsec;
+		}
+		if (np->n_flag & NUPD) {
+			nvap->nva_timesec[NFSTIME_MODIFY] = np->n_mtim.tv_sec;
+			nvap->nva_timensec[NFSTIME_MODIFY] = np->n_mtim.tv_nsec;
+		}
 	}
 
-	FSDBG_BOT(527, 0, np, 0, *xidp);
+	FSDBG_BOT(527, 0, np, np->n_size, *xidp);
 	return (0);
 }
 
@@ -1492,17 +1334,16 @@ nfs_loadattrcache(
  * how recently the file has been modified.
  */
 int
-nfs_attrcachetimeout(vnode_t vp)
+nfs_attrcachetimeout(nfsnode_t np)
 {
-	struct nfsnode *np = VTONFS(vp);
 	struct nfsmount *nmp;
 	struct timeval now;
 	int isdir, timeo;
 
-	if (!(nmp = VFSTONFS(vnode_mount(vp))))
+	if (!(nmp = NFSTONMP(np)))
 		return (0);
 
-	isdir = vnode_isdir(vp);
+	isdir = vnode_isdir(NFSTOV(np));
 
 	if ((np)->n_flag & NMODIFIED)
 		timeo = isdir ? nmp->nm_acdirmin : nmp->nm_acregmin;
@@ -1533,87 +1374,216 @@ nfs_attrcachetimeout(vnode_t vp)
  * otherwise return an error
  */
 int
-nfs_getattrcache(vp, nvaper)
-	vnode_t vp;
-	struct nfs_vattr *nvaper;
+nfs_getattrcache(nfsnode_t np, struct nfs_vattr *nvaper, int alreadylocked)
 {
-	struct nfsnode *np = VTONFS(vp);
 	struct nfs_vattr *nvap;
 	struct timeval nowup;
 	int32_t timeo;
 
+	if (!alreadylocked && nfs_lock(np, NFS_NODE_LOCK_SHARED)) {
+		FSDBG(528, np, 0, 0xffffff00, ENOENT);
+		OSAddAtomic(1, (SInt32*)&nfsstats.attrcache_misses);
+		return (ENOENT);
+	}
+
 	if (!NATTRVALID(np)) {
-		FSDBG(528, vp, 0, 0, 0);
+		if (!alreadylocked)
+			nfs_unlock(np);
+		FSDBG(528, np, 0, 0xffffff01, ENOENT);
 		OSAddAtomic(1, (SInt32*)&nfsstats.attrcache_misses);
 		return (ENOENT);
 	}
 
-	timeo = nfs_attrcachetimeout(vp);
+	timeo = nfs_attrcachetimeout(np);
 
 	microuptime(&nowup);
 	if ((nowup.tv_sec - np->n_attrstamp) >= timeo) {
-		FSDBG(528, vp, 0, 0, 1);
+		if (!alreadylocked)
+			nfs_unlock(np);
+		FSDBG(528, np, 0, 0xffffff02, ENOENT);
 		OSAddAtomic(1, (SInt32*)&nfsstats.attrcache_misses);
 		return (ENOENT);
 	}
-	FSDBG(528, vp, 0, 0, 2);
-	OSAddAtomic(1, (SInt32*)&nfsstats.attrcache_hits);
+
 	nvap = &np->n_vattr;
+	FSDBG(528, np, nvap->nva_size, np->n_size, 0xcace);
+	OSAddAtomic(1, (SInt32*)&nfsstats.attrcache_hits);
 
 	if (nvap->nva_size != np->n_size) {
-		FSDBG(528, vp, nvap->nva_size, np->n_size,
-		      (nvap->nva_type == VREG) |
-		      (np->n_flag & NMODIFIED ? 6 : 4));
-		if (nvap->nva_type == VREG) {
-			if (np->n_flag & NMODIFIED) {
-				if (nvap->nva_size < np->n_size)
-					nvap->nva_size = np->n_size;
-				else
-					np->n_size = nvap->nva_size;
-			} else
-				np->n_size = nvap->nva_size;
-			ubc_setsize(vp, (off_t)np->n_size); /* XXX */
-		} else
-			np->n_size = nvap->nva_size;
+		/*
+		 * n_size is protected by the data lock, so we need to
+		 * defer updating it until it's safe.  We save the new size
+		 * and set a flag and it'll get updated the next time we get/drop
+		 * the data lock or the next time we do a getattr.
+		 */
+		if (!alreadylocked) {
+			/* need to upgrade shared lock to exclusive */
+			if (lck_rw_lock_shared_to_exclusive(&np->n_lock) == FALSE)
+				lck_rw_lock_exclusive(&np->n_lock);
+		}
+		np->n_newsize = nvap->nva_size;
+		FSDBG(528, np, nvap->nva_size, np->n_size, (nvap->nva_type == VREG) | (np->n_flag & NMODIFIED ? 6 : 4));
+		SET(np->n_flag, NUPDATESIZE);
+		if ((nvap->nva_type == VREG) && (np->n_flag & NMODIFIED) &&
+		    (nvap->nva_size < np->n_size)) {
+			/* if we've modified, use larger size */
+			nvap->nva_size = np->n_size;
+			CLR(np->n_flag, NUPDATESIZE);
+		}
 	}
 
 	bcopy((caddr_t)nvap, (caddr_t)nvaper, sizeof(struct nfs_vattr));
 	if (np->n_flag & NCHG) {
-		if (np->n_flag & NACC)
-			nvaper->nva_atime = np->n_atim;
-		if (np->n_flag & NUPD)
-			nvaper->nva_mtime = np->n_mtim;
+		if (np->n_flag & NACC) {
+			nvaper->nva_timesec[NFSTIME_ACCESS] = np->n_atim.tv_sec;
+			nvaper->nva_timensec[NFSTIME_ACCESS] = np->n_atim.tv_nsec;
+		}
+		if (np->n_flag & NUPD) {
+			nvaper->nva_timesec[NFSTIME_MODIFY] = np->n_mtim.tv_sec;
+			nvaper->nva_timensec[NFSTIME_MODIFY] = np->n_mtim.tv_nsec;
+		}
 	}
+	if (!alreadylocked)
+		nfs_unlock(np);
 	return (0);
 }
 
-#ifndef NFS_NOSERVER
+
+static nfsuint64 nfs_nullcookie = { { 0, 0 } };
+/*
+ * This function finds the directory cookie that corresponds to the
+ * logical byte offset given.
+ */
+nfsuint64 *
+nfs_getcookie(nfsnode_t dnp, off_t off, int add)
+{
+	struct nfsdmap *dp, *dp2;
+	int pos;
+
+	pos = off / NFS_DIRBLKSIZ;
+	if (pos == 0)
+		return (&nfs_nullcookie);
+	pos--;
+	dp = dnp->n_cookies.lh_first;
+	if (!dp) {
+		if (add) {
+			MALLOC_ZONE(dp, struct nfsdmap *, sizeof(struct nfsdmap),
+					M_NFSDIROFF, M_WAITOK);
+			if (!dp)
+				return ((nfsuint64 *)0);
+			dp->ndm_eocookie = 0;
+			LIST_INSERT_HEAD(&dnp->n_cookies, dp, ndm_list);
+		} else
+			return ((nfsuint64 *)0);
+	}
+	while (pos >= NFSNUMCOOKIES) {
+		pos -= NFSNUMCOOKIES;
+		if (dp->ndm_list.le_next) {
+			if (!add && dp->ndm_eocookie < NFSNUMCOOKIES &&
+				pos >= dp->ndm_eocookie)
+				return ((nfsuint64 *)0);
+			dp = dp->ndm_list.le_next;
+		} else if (add) {
+			MALLOC_ZONE(dp2, struct nfsdmap *, sizeof(struct nfsdmap),
+					M_NFSDIROFF, M_WAITOK);
+			if (!dp2)
+				return ((nfsuint64 *)0);
+			dp2->ndm_eocookie = 0;
+			LIST_INSERT_AFTER(dp, dp2, ndm_list);
+			dp = dp2;
+		} else
+			return ((nfsuint64 *)0);
+	}
+	if (pos >= dp->ndm_eocookie) {
+		if (add)
+			dp->ndm_eocookie = pos + 1;
+		else
+			return ((nfsuint64 *)0);
+	}
+	return (&dp->ndm_cookies[pos]);
+}
+
+/*
+ * Invalidate cached directory information, except for the actual directory
+ * blocks (which are invalidated separately).
+ * Done mainly to avoid the use of stale offset cookies.
+ */
+void
+nfs_invaldir(nfsnode_t dnp)
+{
+	if (vnode_vtype(NFSTOV(dnp)) != VDIR) {
+		printf("nfs: invaldir not dir\n");
+		return;
+	}
+	dnp->n_direofoffset = 0;
+	dnp->n_cookieverf.nfsuquad[0] = 0;
+	dnp->n_cookieverf.nfsuquad[1] = 0;
+	if (dnp->n_cookies.lh_first)
+		dnp->n_cookies.lh_first->ndm_eocookie = 0;
+}
+
+#endif /* NFSCLIENT */
+
+/*
+ * Schedule a callout thread to run an NFS timer function
+ * interval milliseconds in the future.
+ */
+void
+nfs_interval_timer_start(thread_call_t call, int interval)
+{
+	uint64_t deadline;
+
+	clock_interval_to_deadline(interval, 1000 * 1000, &deadline);
+	thread_call_enter_delayed(call, deadline);
+}
+
+
+#if NFSSERVER
+
+static void nfsrv_init_user_list(struct nfs_active_user_list *);
+static void nfsrv_free_user_list(struct nfs_active_user_list *);
+
+/*
+ * add NFSv3 WCC data to an mbuf chain
+ */
+int
+nfsm_chain_add_wcc_data_f(
+	struct nfsrv_descript *nd,
+	struct nfsm_chain *nmc,
+	int preattrerr,
+	struct vnode_attr *prevap,
+	int postattrerr,
+	struct vnode_attr *postvap)
+{
+	int error = 0;
+
+	if (preattrerr) {
+		nfsm_chain_add_32(error, nmc, FALSE);
+	} else {
+		nfsm_chain_add_32(error, nmc, TRUE);
+		nfsm_chain_add_64(error, nmc, prevap->va_data_size);
+		nfsm_chain_add_time(error, nmc, NFS_VER3, &prevap->va_modify_time);
+		nfsm_chain_add_time(error, nmc, NFS_VER3, &prevap->va_change_time);
+	}
+	nfsm_chain_add_postop_attr(error, nd, nmc, postattrerr, postvap);
+
+	return (error);
+}
+
 /*
  * Extract a lookup path from the given mbufs and store it in
  * a newly allocated buffer saved in the given nameidata structure.
- * exptected string length given as *lenp and final string length
- * (after any WebNFS processing) is returned in *lenp.
  */
 int
-nfsm_path_mbuftond(
-	mbuf_t *mdp,
-	caddr_t *dposp,
-	__unused int v3,
-	__unused int pubflag,
-	int* lenp,
-	struct nameidata *ndp)
+nfsm_chain_get_path_namei(
+	struct nfsm_chain *nmc,
+	uint32_t len,
+	struct nameidata *nip)
 {
-	int i, len, len2, rem, error = 0;
-	mbuf_t md;
-	char *fromcp, *tocp;
-	struct componentname *cnp = &ndp->ni_cnd;
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	int webcnt = 0, digitcnt = 0;
-	char hexdigits[2];
-#endif
-
-	len = *lenp;
+	struct componentname *cnp = &nip->ni_cnd;
+	int error = 0;
+	char *cp;
+
 	if (len > (MAXPATHLEN - 1))
 		return (ENAMETOOLONG);
 
@@ -1627,157 +1597,43 @@ nfsm_path_mbuftond(
 	cnp->cn_pnlen = MAXPATHLEN;
 	cnp->cn_flags |= HASBUF;
 
-	/*
-	 * Copy the name from the mbuf list to the string
-	 *
-	 * Along the way, take note of any WebNFS characters
-	 * and convert any % escapes.
-	 */
-	fromcp = *dposp;
-	tocp = cnp->cn_pnbuf;
-	md = *mdp;
-	rem = (caddr_t)mbuf_data(md) + mbuf_len(md) - fromcp;
-	for (i = 1; i <= len; i++) {
-		while (rem == 0) {
-			md = mbuf_next(md);
-			if (md == NULL) {
-				error = EBADRPC;
-				goto out;
-			}
-			fromcp = mbuf_data(md);
-			rem = mbuf_len(md);
-		}
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-		if (pubflag) {
-			if ((i == 1) && ((unsigned char)*fromcp >= WEBNFS_SPECCHAR_START)) {
-				switch ((unsigned char)*fromcp) {
-				case WEBNFS_NATIVE_CHAR:
-					/*
-					 * 'Native' path for us is the same
-					 * as a path according to the NFS spec,
-					 * just skip the escape char.
-					 */
-					webcnt++;
-					fromcp++;
-					rem--;
-					/* next iteration of for loop */
-					continue;
-				/*
-				 * More may be added in the future, range 0x80-0xff.
-				 * Don't currently support security query lookup (0x81).
-				 */
-				default:
-					error = EIO;
-					goto out;
-				}
-			}
-			if (digitcnt) {
-				/* We're expecting hex digits */
-				if (!ISHEX(*fromcp)) {
-					error = ENOENT;
-					goto out;
-				}
-				digitcnt--;
-				hexdigits[digitcnt ? 0 : 1] = *fromcp++;
-				if (!digitcnt)
-					*tocp++ = HEXSTRTOI(hexdigits);
-				rem--;
-				/* next iteration of for loop */
-				continue;
-			} else if (*fromcp == WEBNFS_ESC_CHAR) {
-				/*
-				 * We can't really look at the next couple
-				 * bytes here safely/easily, so we note that
-				 * the next two characters should be hex
-				 * digits and later save them in hexdigits[].
-				 * When we've got both, we'll convert it.
-				 */
-				digitcnt = 2;
-				webcnt += 2;
-				fromcp++;
-				rem--;
-				/* next iteration of for loop */
-				continue;
-			}
-		}
-		if (*fromcp == '\0' || (!pubflag && *fromcp == '/'))
-#else
-		if (*fromcp == '\0' || *fromcp == '/')
-#endif
-		{
-			error = EACCES;
-			goto out;
-		}
-		*tocp++ = *fromcp++;
-		rem--;
-	}
-	*tocp = '\0';
-	*mdp = md;
-	*dposp = fromcp;
-	len2 = nfsm_rndup(len)-len;
-	if (len2 > 0) {
-		if (rem >= len2)
-			*dposp += len2;
-		else if ((error = nfs_adv(mdp, dposp, len2, rem)) != 0)
-			goto out;
-	}
-
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	if (pubflag) {
-		if (digitcnt) {
-			/* The string ended in the middle of an escape! */
-			error = ENOENT;
-			goto out;
-		}
-		len -= webcnt;
-	}
-#endif
+	/* Copy the name from the mbuf list to the string */
+	cp = cnp->cn_pnbuf;
+	nfsm_chain_get_opaque(error, nmc, len, cp);
+	if (error)
+		goto out;
+	cnp->cn_pnbuf[len] = '\0';
 
+	/* sanity check the string */
+	if ((strlen(cp) != len) || strchr(cp, '/'))
+		error = EACCES;
 out:
 	if (error) {
 		if (cnp->cn_pnbuf)
 			FREE_ZONE(cnp->cn_pnbuf, MAXPATHLEN, M_NAMEI);
 		cnp->cn_flags &= ~HASBUF;
 	} else {
-		ndp->ni_pathlen = len;
-		*lenp = len;
+		nip->ni_pathlen = len;
 	}
 	return (error);
 }
 
 /*
  * Set up nameidata for a lookup() call and do it.
- *
- * If pubflag is set, this call is done for a lookup operation on the
- * public filehandle. In that case we allow crossing mountpoints and
- * absolute pathnames. However, the caller is expected to check that
- * the lookup result is within the public fs, and deny access if
- * it is not.
  */
 int
-nfs_namei(
-	struct nfsrv_descript *nfsd,
-	struct vfs_context *ctx,
-	struct nameidata *ndp,
+nfsrv_namei(
+	struct nfsrv_descript *nd,
+	vfs_context_t ctx,
+	struct nameidata *nip,
 	struct nfs_filehandle *nfhp,
-	mbuf_t nam,
-	int pubflag,
 	vnode_t *retdirp,
 	struct nfs_export **nxp,
 	struct nfs_export_options **nxop)
 {
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	char *cp;
-	uio_t auio;
-	char uio_buf[ UIO_SIZEOF(1) ];
-	int linklen, olen = ndp->ni_pathlen;
-#endif
 	vnode_t dp;
 	int error;
-	struct componentname *cnp = &ndp->ni_cnd;
+	struct componentname *cnp = &nip->ni_cnd;
 	char *tmppn;
 
 	*retdirp = NULL;
@@ -1785,163 +1641,54 @@ nfs_namei(
 	/*
 	 * Extract and set starting directory.
 	 */
-	error = nfsrv_fhtovp(nfhp, nam, pubflag, &dp, nxp, nxop);
+	error = nfsrv_fhtovp(nfhp, nd, &dp, nxp, nxop);
 	if (error)
 		goto out;
-	error = nfsrv_credcheck(nfsd, *nxp, *nxop);
+	error = nfsrv_credcheck(nd, ctx, *nxp, *nxop);
 	if (error || (vnode_vtype(dp) != VDIR)) {
 		vnode_put(dp);
 		error = ENOTDIR;
 		goto out;
 	}
+	*retdirp = dp;
 
-	ctx->vc_ucred = nfsd->nd_cr;
-	ndp->ni_cnd.cn_context = ctx;
+	nip->ni_cnd.cn_context = ctx;
 
 	if (*nxop && ((*nxop)->nxo_flags & NX_READONLY))
 		cnp->cn_flags |= RDONLY;
 
-	*retdirp = dp;
-
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	if (pubflag) {
-		ndp->ni_rootdir = rootvnode;
-		ndp->ni_loopcnt = 0;
-		if (cnp->cn_pnbuf[0] == '/') {
-			vnode_put(dp);
-			dp = rootvnode;
-			error = vnode_get(dp);
-			if (error) {
-				*retdirp = NULL;
-				goto out;
-			}
-		}
-	} else {
-		cnp->cn_flags |= NOCROSSMOUNT;
-	}
-#else
 	cnp->cn_flags |= NOCROSSMOUNT;
-#endif
-
-	ndp->ni_usedvp = dp;
-
-    for (;;) {
 	cnp->cn_nameptr = cnp->cn_pnbuf;
-	ndp->ni_startdir = dp;
+	nip->ni_usedvp = nip->ni_startdir = dp;
+
 	/*
 	 * And call lookup() to do the real work
 	 */
-	error = lookup(ndp);
+	error = lookup(nip);
 	if (error)
-		break;
-	/*
-	 * Check for encountering a symbolic link
-	 */
-	if ((cnp->cn_flags & ISSYMLINK) == 0) {
-		return (0);
-	} else {
+		goto out;
+
+	/* Check for encountering a symbolic link */
+	if (cnp->cn_flags & ISSYMLINK) {
 	        if ((cnp->cn_flags & FSNODELOCKHELD)) {
 		        cnp->cn_flags &= ~FSNODELOCKHELD;
-			unlock_fsnode(ndp->ni_dvp, NULL);
-		}
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-		if (!pubflag) {
-#endif
-		        if (cnp->cn_flags & (LOCKPARENT | WANTPARENT))
-			        vnode_put(ndp->ni_dvp);
-			if (ndp->ni_vp) {
-			        vnode_put(ndp->ni_vp);
-				ndp->ni_vp = NULL;
-			}
-			error = EINVAL;
-			break;
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-		}
-
-		if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
-			vnode_put(ndp->ni_vp);
-			ndp->ni_vp = NULL;
-			error = ELOOP;
-			break;
-		}
-		/* XXX assert(olen <= MAXPATHLEN - 1); */
-		if (ndp->ni_pathlen > 1) {
-			MALLOC_ZONE(cp, char *, MAXPATHLEN, M_NAMEI, M_WAITOK);
-			if (!cp) {
-				vnode_put(ndp->ni_vp);
-				ndp->ni_vp = NULL;
-				error = ENOMEM;
-				break;
-			}
-		} else {
-			cp = cnp->cn_pnbuf;
-		}
-		auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, 
-					&uio_buf[0], sizeof(uio_buf));
-		if (!auio) {
-			vnode_put(ndp->ni_vp);
-			ndp->ni_vp = NULL;
-			if (ndp->ni_pathlen > 1)
-				FREE_ZONE(cp, MAXPATHLEN, M_NAMEI);
-			error = ENOMEM;
-			break;
-		}
-		uio_addiov(auio, CAST_USER_ADDR_T(cp), MAXPATHLEN);
-		error = VNOP_READLINK(ndp->ni_vp, auio, cnp->cn_context);
-		if (error) {
-badlink:
-			vnode_put(ndp->ni_vp);
-			ndp->ni_vp = NULL;
-			if (ndp->ni_pathlen > 1)
-				FREE_ZONE(cp, MAXPATHLEN, M_NAMEI);
-			break;
+			unlock_fsnode(nip->ni_dvp, NULL);
 		}
-		linklen = MAXPATHLEN - uio_resid(auio);
-		if (linklen == 0) {
-			error = ENOENT;
-			goto badlink;
-		}
-		if (linklen + ndp->ni_pathlen >= MAXPATHLEN) {
-			error = ENAMETOOLONG;
-			goto badlink;
-		}
-		if (ndp->ni_pathlen > 1) {
-			long len = cnp->cn_pnlen;
-			tmppn = cnp->cn_pnbuf;
-			cnp->cn_pnbuf = cp;
-			cnp->cn_pnlen = olen + 1;
-			bcopy(ndp->ni_next, cp + linklen, ndp->ni_pathlen);
-			FREE_ZONE(tmppn, len, M_NAMEI);
-		} else
-			cnp->cn_pnbuf[linklen] = '\0';
-		ndp->ni_pathlen += linklen;
-
-		vnode_put(ndp->ni_vp);
-		dp = ndp->ni_dvp;
-		ndp->ni_dvp = NULL;
-
-		/*
-		 * Check if root directory should replace current directory.
-		 */
-		if (cnp->cn_pnbuf[0] == '/') {
-			vnode_put(dp);
-			dp = ndp->ni_rootdir;
-			error = vnode_get(dp);
-			if (error)
-				break;
+		if (cnp->cn_flags & (LOCKPARENT | WANTPARENT))
+			vnode_put(nip->ni_dvp);
+		if (nip->ni_vp) {
+			vnode_put(nip->ni_vp);
+			nip->ni_vp = NULL;
 		}
-#endif
+		error = EINVAL;
 	}
-   }
 out:
-	tmppn = cnp->cn_pnbuf;
-	cnp->cn_pnbuf = NULL;
-	cnp->cn_flags &= ~HASBUF;
-	FREE_ZONE(tmppn, cnp->cn_pnlen, M_NAMEI);
-
+	if (error) {
+		tmppn = cnp->cn_pnbuf;
+		cnp->cn_pnbuf = NULL;
+		cnp->cn_flags &= ~HASBUF;
+		FREE_ZONE(tmppn, cnp->cn_pnlen, M_NAMEI);
+	}
 	return (error);
 }
 
@@ -1950,10 +1697,7 @@ out:
  * boundary and only trims off the back end
  */
 void
-nfsm_adj(mp, len, nul)
-	mbuf_t mp;
-	int len;
-	int nul;
+nfsm_adj(mbuf_t mp, int len, int nul)
 {
 	mbuf_t m, mnext;
 	int count, i, mlen;
@@ -2013,107 +1757,218 @@ nfsm_adj(mp, len, nul)
 }
 
 /*
- * Make these functions instead of macros, so that the kernel text size
- * doesn't get too big...
+ * Trim the header out of the mbuf list and trim off any trailing
+ * junk so that the mbuf list has only the write data.
  */
-void
-nfsm_srvwcc(nfsd, before_ret, before_vap, after_ret, after_vap, mbp, bposp)
-	struct nfsrv_descript *nfsd;
-	int before_ret;
-	struct vnode_attr *before_vap;
-	int after_ret;
-	struct vnode_attr *after_vap;
-	mbuf_t *mbp;
-	char **bposp;
+int
+nfsm_chain_trim_data(struct nfsm_chain *nmc, int len, int *mlen)
 {
-	mbuf_t mb = *mbp, mb2;
-	char *bpos = *bposp;
-	u_long *tl;
+	int cnt = 0, dlen, adjust;
+	caddr_t data;
+	mbuf_t m;
 
-	if (before_ret) {
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-		*tl = nfs_false;
-	} else {
-		nfsm_build(tl, u_long *, 7 * NFSX_UNSIGNED);
-		*tl++ = nfs_true;
-		txdr_hyper(&(before_vap->va_data_size), tl);
-		tl += 2;
-		txdr_nfsv3time(&(before_vap->va_modify_time), tl);
-		tl += 2;
-		txdr_nfsv3time(&(before_vap->va_change_time), tl);
-	}
-	*bposp = bpos;
-	*mbp = mb;
-	nfsm_srvpostopattr(nfsd, after_ret, after_vap, mbp, bposp);
-}
+	if (mlen)
+		*mlen = 0;
 
-void
-nfsm_srvpostopattr(nfsd, after_ret, after_vap, mbp, bposp)
-	struct nfsrv_descript *nfsd;
-	int after_ret;
-	struct vnode_attr *after_vap;
-	mbuf_t *mbp;
-	char **bposp;
-{
-	mbuf_t mb = *mbp, mb2;
-	char *bpos = *bposp;
-	u_long *tl;
-	struct nfs_fattr *fp;
-
-	if (after_ret) {
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-		*tl = nfs_false;
-	} else {
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED + NFSX_V3FATTR);
-		*tl++ = nfs_true;
-		fp = (struct nfs_fattr *)tl;
-		nfsm_srvfattr(nfsd, after_vap, fp);
+	/* trim header */
+	for (m = nmc->nmc_mhead; m && (m != nmc->nmc_mcur); m = mbuf_next(m))
+		mbuf_setlen(m, 0);
+	if (!m)
+		return (EIO);
+
+	/* trim current mbuf */
+	data = mbuf_data(m);
+	dlen = mbuf_len(m);
+	adjust = nmc->nmc_ptr - data;
+	dlen -= adjust;
+	if ((dlen > 0) && (adjust > 0)) {
+		if (mbuf_setdata(m, nmc->nmc_ptr, dlen))
+			return(EIO);
+	} else
+		mbuf_setlen(m, dlen);
+
+	/* skip next len bytes  */
+	for (; m && (cnt < len); m = mbuf_next(m)) {
+		dlen = mbuf_len(m);
+		cnt += dlen;
+		if (cnt > len) {
+			/* truncate to end of data */
+			mbuf_setlen(m, dlen - (cnt - len));
+			if (m == nmc->nmc_mcur)
+				nmc->nmc_left -= (cnt - len);
+			cnt = len;
+		}
 	}
-	*mbp = mb;
-	*bposp = bpos;
+	if (mlen)
+		*mlen = cnt;
+
+	/* trim any trailing data */
+	if (m == nmc->nmc_mcur)
+		nmc->nmc_left = 0;
+	for (; m; m = mbuf_next(m))
+		mbuf_setlen(m, 0);
+
+	return (0);
 }
 
-void
-nfsm_srvfattr(nfsd, vap, fp)
-	struct nfsrv_descript *nfsd;
-	struct vnode_attr *vap;
-	struct nfs_fattr *fp;
+int
+nfsm_chain_add_fattr(
+	struct nfsrv_descript *nd,
+	struct nfsm_chain *nmc,
+	struct vnode_attr *vap)
 {
+	int error = 0;
 
 	// XXX Should we assert here that all fields are supported?
 
-	fp->fa_nlink = txdr_unsigned(vap->va_nlink);
-	fp->fa_uid = txdr_unsigned(vap->va_uid);
-	fp->fa_gid = txdr_unsigned(vap->va_gid);
-	if (nfsd->nd_flag & ND_NFSV3) {
-		fp->fa_type = vtonfsv3_type(vap->va_type);
-		fp->fa_mode = vtonfsv3_mode(vap->va_mode);
-		txdr_hyper(&vap->va_data_size, &fp->fa3_size);
-		txdr_hyper(&vap->va_data_alloc, &fp->fa3_used);
-		fp->fa3_rdev.specdata1 = txdr_unsigned(major(vap->va_rdev));
-		fp->fa3_rdev.specdata2 = txdr_unsigned(minor(vap->va_rdev));
-		fp->fa3_fsid.nfsuquad[0] = 0;
-		fp->fa3_fsid.nfsuquad[1] = txdr_unsigned(vap->va_fsid);
-		txdr_hyper(&vap->va_fileid, &fp->fa3_fileid);
-		txdr_nfsv3time(&vap->va_access_time, &fp->fa3_atime);
-		txdr_nfsv3time(&vap->va_modify_time, &fp->fa3_mtime);
-		txdr_nfsv3time(&vap->va_change_time, &fp->fa3_ctime);
+	nfsm_chain_add_32(error, nmc, vtonfs_type(vap->va_type, nd->nd_vers));
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_32(error, nmc, vap->va_mode & 07777);
+	} else {
+		nfsm_chain_add_32(error, nmc, vtonfsv2_mode(vap->va_type, vap->va_mode));
+	}
+	nfsm_chain_add_32(error, nmc, vap->va_nlink);
+	nfsm_chain_add_32(error, nmc, vap->va_uid);
+	nfsm_chain_add_32(error, nmc, vap->va_gid);
+	if (nd->nd_vers == NFS_VER3) {
+		nfsm_chain_add_64(error, nmc, vap->va_data_size);
+		nfsm_chain_add_64(error, nmc, vap->va_data_alloc);
+		nfsm_chain_add_32(error, nmc, major(vap->va_rdev));
+		nfsm_chain_add_32(error, nmc, minor(vap->va_rdev));
+		nfsm_chain_add_64(error, nmc, vap->va_fsid);
+		nfsm_chain_add_64(error, nmc, vap->va_fileid);
 	} else {
-		fp->fa_type = vtonfsv2_type(vap->va_type);
-		fp->fa_mode = vtonfsv2_mode(vap->va_type, vap->va_mode);
-		fp->fa2_size = txdr_unsigned(vap->va_data_size);
-		fp->fa2_blocksize = txdr_unsigned(vap->va_iosize);
+		nfsm_chain_add_32(error, nmc, vap->va_data_size);
+		nfsm_chain_add_32(error, nmc, NFS_FABLKSIZE);
 		if (vap->va_type == VFIFO)
-			fp->fa2_rdev = 0xffffffff;
+			nfsm_chain_add_32(error, nmc, 0xffffffff);
 		else
-			fp->fa2_rdev = txdr_unsigned(vap->va_rdev);
-		fp->fa2_blocks = txdr_unsigned(vap->va_data_alloc / NFS_FABLKSIZE);
-		fp->fa2_fsid = txdr_unsigned(vap->va_fsid);
-		fp->fa2_fileid = txdr_unsigned(vap->va_fileid);
-		txdr_nfsv2time(&vap->va_access_time, &fp->fa2_atime);
-		txdr_nfsv2time(&vap->va_modify_time, &fp->fa2_mtime);
-		txdr_nfsv2time(&vap->va_change_time, &fp->fa2_ctime);
+			nfsm_chain_add_32(error, nmc, vap->va_rdev);
+		nfsm_chain_add_32(error, nmc, vap->va_data_alloc / NFS_FABLKSIZE);
+		nfsm_chain_add_32(error, nmc, vap->va_fsid);
+		nfsm_chain_add_32(error, nmc, vap->va_fileid);
+	}
+	nfsm_chain_add_time(error, nmc, nd->nd_vers, &vap->va_access_time);
+	nfsm_chain_add_time(error, nmc, nd->nd_vers, &vap->va_modify_time);
+	nfsm_chain_add_time(error, nmc, nd->nd_vers, &vap->va_change_time);
+
+	return (error);
+}
+
+int
+nfsm_chain_get_sattr(
+	struct nfsrv_descript *nd,
+	struct nfsm_chain *nmc,
+	struct vnode_attr *vap)
+{
+	int error = 0, nullflag = 0;
+	uint32_t val = 0;
+	uint64_t val64;
+	struct timespec now;
+
+	if (nd->nd_vers == NFS_VER2) {
+		/*
+		 * There is/was a bug in the Sun client that puts 0xffff in the mode
+		 * field of sattr when it should put in 0xffffffff.  The u_short
+		 * doesn't sign extend.  So check the low order 2 bytes for 0xffff.
+		 */
+		nfsm_chain_get_32(error, nmc, val);
+		if ((val & 0xffff) != 0xffff) {
+			VATTR_SET(vap, va_mode, val & 07777);
+			/* save the "type" bits for NFSv2 create */
+			VATTR_SET(vap, va_type, IFTOVT(val));
+			VATTR_CLEAR_ACTIVE(vap, va_type);
+		}
+		nfsm_chain_get_32(error, nmc, val);
+		if (val != (uint32_t)-1)
+			VATTR_SET(vap, va_uid, val);
+		nfsm_chain_get_32(error, nmc, val);
+		if (val != (uint32_t)-1)
+			VATTR_SET(vap, va_gid, val);
+		/* save the "size" bits for NFSv2 create (even if they appear unset) */
+		nfsm_chain_get_32(error, nmc, val);
+		VATTR_SET(vap, va_data_size, val);
+		if (val == (uint32_t)-1)
+			VATTR_CLEAR_ACTIVE(vap, va_data_size);
+		nfsm_chain_get_time(error, nmc, NFS_VER2,
+			vap->va_access_time.tv_sec,
+			vap->va_access_time.tv_nsec);
+		if (vap->va_access_time.tv_sec != -1)
+			VATTR_SET_ACTIVE(vap, va_access_time);
+		nfsm_chain_get_time(error, nmc, NFS_VER2,
+			vap->va_modify_time.tv_sec,
+			vap->va_modify_time.tv_nsec);
+		if (vap->va_modify_time.tv_sec != -1)
+			VATTR_SET_ACTIVE(vap, va_modify_time);
+		return (error);
+	}
+
+	/* NFSv3 */
+	nfsm_chain_get_32(error, nmc, val);
+	if (val) {
+		nfsm_chain_get_32(error, nmc, val);
+		VATTR_SET(vap, va_mode, val & 07777);
+	}
+	nfsm_chain_get_32(error, nmc, val);
+	if (val) {
+		nfsm_chain_get_32(error, nmc, val);
+		VATTR_SET(vap, va_uid, val);
+	}
+	nfsm_chain_get_32(error, nmc, val);
+	if (val) {
+		nfsm_chain_get_32(error, nmc, val);
+		VATTR_SET(vap, va_gid, val);
+	}
+	nfsm_chain_get_32(error, nmc, val);
+	if (val) {
+		nfsm_chain_get_64(error, nmc, val64);
+		VATTR_SET(vap, va_data_size, val64);
+	}
+	nanotime(&now);
+	nfsm_chain_get_32(error, nmc, val);
+	switch (val) {
+	case NFS_TIME_SET_TO_CLIENT:
+		nfsm_chain_get_time(error, nmc, nd->nd_vers,
+			vap->va_access_time.tv_sec,
+			vap->va_access_time.tv_nsec);
+		VATTR_SET_ACTIVE(vap, va_access_time);
+		break;
+	case NFS_TIME_SET_TO_SERVER:
+		VATTR_SET(vap, va_access_time, now);
+		nullflag = VA_UTIMES_NULL;
+		break;
+	}
+	nfsm_chain_get_32(error, nmc, val);
+	switch (val) {
+	case NFS_TIME_SET_TO_CLIENT:
+		nfsm_chain_get_time(error, nmc, nd->nd_vers,
+			vap->va_modify_time.tv_sec,
+			vap->va_modify_time.tv_nsec);
+		VATTR_SET_ACTIVE(vap, va_modify_time);
+		break;
+	case NFS_TIME_SET_TO_SERVER:
+		VATTR_SET(vap, va_modify_time, now);
+		vap->va_vaflags |= nullflag;
+		break;
 	}
+
+	return (error);
+}
+
+/*
+ * Compare two security flavor structs
+ */
+static int
+nfsrv_cmp_secflavs(struct nfs_sec *sf1, struct nfs_sec *sf2)
+{
+	int i;
+
+	if (sf1->count != sf2->count)
+		return 1;
+	for (i = 0; i < sf1->count; i++)
+		if (sf1->flavors[i] != sf2->flavors[i])
+			return 1;
+	return 0;
 }
 
 /*
@@ -2124,7 +1979,7 @@ static int
 nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 {
 	struct nfs_export_net_args nxna;
-	struct nfs_netopt *no;
+	struct nfs_netopt *no, *rn_no;
 	struct radix_node_head *rnh;
 	struct radix_node *rn;
 	struct sockaddr *saddr, *smask;
@@ -2147,9 +2002,8 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 			temp_cred.cr_ngroups = nxna.nxna_cred.cr_ngroups;
 			for (i=0; i < nxna.nxna_cred.cr_ngroups && i < NGROUPS; i++)
 				temp_cred.cr_groups[i] = nxna.nxna_cred.cr_groups[i];
-
 			cred = kauth_cred_create(&temp_cred);
-			if (!cred)
+			if (!IS_VALID_CRED(cred))
 				return (ENOMEM);
 		} else {
 			cred = NOCRED;
@@ -2157,11 +2011,15 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 
 		if (nxna.nxna_addr.ss_len == 0) {
 			/* No address means this is a default/world export */
-			if (nx->nx_flags & NX_DEFAULTEXPORT)
+			if (nx->nx_flags & NX_DEFAULTEXPORT) {
+			        if (IS_VALID_CRED(cred))
+				        kauth_cred_unref(&cred);
 				return (EEXIST);
+			}
 			nx->nx_flags |= NX_DEFAULTEXPORT;
 			nx->nx_defopt.nxo_flags = nxna.nxna_flags;
 			nx->nx_defopt.nxo_cred = cred;
+			bcopy(&nxna.nxna_sec, &nx->nx_defopt.nxo_sec, sizeof(struct nfs_sec));
 			nx->nx_expcnt++;
 			continue;
 		}
@@ -2169,11 +2027,15 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 		i = sizeof(struct nfs_netopt);
 		i += nxna.nxna_addr.ss_len + nxna.nxna_mask.ss_len;
 		MALLOC(no, struct nfs_netopt *, i, M_NETADDR, M_WAITOK);
-		if (!no)
+		if (!no) {
+			if (IS_VALID_CRED(cred))
+				kauth_cred_unref(&cred);
 			return (ENOMEM);
+		}
 		bzero(no, sizeof(struct nfs_netopt));
 		no->no_opt.nxo_flags = nxna.nxna_flags;
 		no->no_opt.nxo_cred = cred;
+		bcopy(&nxna.nxna_sec, &no->no_opt.nxo_sec, sizeof(struct nfs_sec));
 
 		saddr = (struct sockaddr *)(no + 1);
 		bcopy(&nxna.nxna_addr, saddr, nxna.nxna_addr.ss_len);
@@ -2209,13 +2071,20 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 			 * the entry already exists. To check for this case, we
 			 * look up the entry to see if it is there. If so, we
 			 * do not need to make a new entry but do continue.
+			 *
+			 * XXX should this be rnh_lookup() instead?
 			 */
 			int matched = 0;
 			rn = (*rnh->rnh_matchaddr)((caddr_t)saddr, rnh);
+			rn_no = (struct nfs_netopt *)rn;
 			if (rn != 0 && (rn->rn_flags & RNF_ROOT) == 0 &&
-			    (((struct nfs_netopt *)rn)->no_opt.nxo_flags == nxna.nxna_flags)) {
-				kauth_cred_t cred2 = ((struct nfs_netopt *)rn)->no_opt.nxo_cred;
-				if (cred && cred2 && (cred->cr_uid == cred2->cr_uid) &&
+			    (rn_no->no_opt.nxo_flags == nxna.nxna_flags) &&
+			    (!nfsrv_cmp_secflavs(&rn_no->no_opt.nxo_sec, &nxna.nxna_sec))) {
+				kauth_cred_t cred2 = rn_no->no_opt.nxo_cred;
+				if (cred == cred2) {
+					/* creds are same (or both NULL) */
+					matched = 1;
+				} else if (cred && cred2 && (cred->cr_uid == cred2->cr_uid) &&
 				    (cred->cr_ngroups == cred2->cr_ngroups)) {
 					for (i=0; i < cred2->cr_ngroups && i < NGROUPS; i++)
 						if (cred->cr_groups[i] != cred2->cr_groups[i])
@@ -2238,7 +2107,7 @@ nfsrv_hang_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 }
 
 /*
- * In order to properly track an export's netopt count, we need to pass 
+ * In order to properly track an export's netopt count, we need to pass
  * an additional argument to nfsrv_free_netopt() so that it can decrement
  * the export's netopt count.
  */
@@ -2266,27 +2135,86 @@ nfsrv_free_netopt(struct radix_node *rn, void *w)
 /*
  * Free the net address hash lists that are hanging off the mount points.
  */
-static void
-nfsrv_free_addrlist(struct nfs_export *nx)
+static int
+nfsrv_free_addrlist(struct nfs_export *nx, struct user_nfs_export_args *unxa)
 {
-	int i;
+	struct nfs_export_net_args nxna;
 	struct radix_node_head *rnh;
+	struct radix_node *rn;
 	struct nfsrv_free_netopt_arg fna;
+	struct nfs_netopt *nno;
+	user_addr_t uaddr;
+	unsigned int net;
+	int i, error;
+
+	if (!unxa || !unxa->nxa_netcount) {
+		/* delete everything */
+		for (i = 0; i <= AF_MAX; i++)
+			if ( (rnh = nx->nx_rtable[i]) ) {
+				fna.rnh = rnh;
+				fna.cnt = &nx->nx_expcnt;
+				(*rnh->rnh_walktree)(rnh, nfsrv_free_netopt, (caddr_t)&fna);
+				_FREE((caddr_t)rnh, M_RTABLE);
+				nx->nx_rtable[i] = 0;
+			}
+		return (0);
+	}
+
+	/* delete only the exports specified */
+	uaddr = unxa->nxa_nets;
+	for (net = 0; net < unxa->nxa_netcount; net++, uaddr += sizeof(nxna)) {
+		error = copyin(uaddr, &nxna, sizeof(nxna));
+		if (error)
+			return (error);
+
+		if (nxna.nxna_addr.ss_len == 0) {
+			/* No address means this is a default/world export */
+			if (nx->nx_flags & NX_DEFAULTEXPORT) {
+				nx->nx_flags &= ~NX_DEFAULTEXPORT;
+				if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
+					kauth_cred_unref(&nx->nx_defopt.nxo_cred);
+				}
+				nx->nx_expcnt--;
+			}
+			continue;
+		}
+
+		if ((rnh = nx->nx_rtable[nxna.nxna_addr.ss_family]) == 0) {
+			/* AF not initialized? */
+			if (!(unxa->nxa_flags & NXA_ADD))
+				printf("nfsrv_free_addrlist: address not found (0)\n");
+			continue;
+		}
 
-	for (i = 0; i <= AF_MAX; i++)
-		if ( (rnh = nx->nx_rtable[i]) ) {
-			fna.rnh = rnh;
-			fna.cnt = &nx->nx_expcnt;
-			(*rnh->rnh_walktree)(rnh, nfsrv_free_netopt, (caddr_t)&fna);
+		rn = (*rnh->rnh_lookup)(&nxna.nxna_addr,
+			nxna.nxna_mask.ss_len ? &nxna.nxna_mask : NULL, rnh);
+		if (!rn || (rn->rn_flags & RNF_ROOT)) {
+			if (!(unxa->nxa_flags & NXA_ADD))
+				printf("nfsrv_free_addrlist: address not found (1)\n");
+			continue;
+		}
+
+		(*rnh->rnh_deladdr)(rn->rn_key, rn->rn_mask, rnh);
+		nno = (struct nfs_netopt *)rn;
+		if (IS_VALID_CRED(nno->no_opt.nxo_cred))
+			kauth_cred_unref(&nno->no_opt.nxo_cred);
+		_FREE((caddr_t)rn, M_NETADDR);
+
+		nx->nx_expcnt--;
+		if (nx->nx_expcnt == ((nx->nx_flags & NX_DEFAULTEXPORT) ? 1 : 0)) {
+			/* no more entries in rnh, so free it up */
 			_FREE((caddr_t)rnh, M_RTABLE);
-			nx->nx_rtable[i] = 0;
+			nx->nx_rtable[nxna.nxna_addr.ss_family] = 0;
 		}
+	}
+
+	return (0);
 }
 
 void enablequotas(struct mount *mp, vfs_context_t ctx); // XXX
 
 int
-nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
+nfsrv_export(struct user_nfs_export_args *unxa, vfs_context_t ctx)
 {
 	int error = 0, pathlen;
 	struct nfs_exportfs *nxfs, *nxfs2, *nxfs3;
@@ -2294,27 +2222,29 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 	struct nfs_filehandle nfh;
 	struct nameidata mnd, xnd;
 	vnode_t mvp = NULL, xvp = NULL;
-	mount_t mp;
+	mount_t mp = NULL;
 	char path[MAXPATHLEN];
 	int expisroot;
 
 	if (unxa->nxa_flags & NXA_DELETE_ALL) {
 		/* delete all exports on all file systems */
-		lck_rw_lock_exclusive(&nfs_export_rwlock);
-		while ((nxfs = LIST_FIRST(&nfs_exports))) {
+		lck_rw_lock_exclusive(&nfsrv_export_rwlock);
+		while ((nxfs = LIST_FIRST(&nfsrv_exports))) {
 			mp = vfs_getvfs_by_mntonname(nxfs->nxfs_path);
 			if (mp)
-				mp->mnt_flag &= ~MNT_EXPORTED;
+				vfs_clearflags(mp, MNT_EXPORTED);
 			/* delete all exports on this file system */
 			while ((nx = LIST_FIRST(&nxfs->nxfs_exports))) {
 				LIST_REMOVE(nx, nx_next);
 				LIST_REMOVE(nx, nx_hash);
 				/* delete all netopts for this export */
-				nfsrv_free_addrlist(nx);
+				nfsrv_free_addrlist(nx, NULL);
 				nx->nx_flags &= ~NX_DEFAULTEXPORT;
 				if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
 					kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 				}
+				/* free active user list for this export */
+				nfsrv_free_user_list(&nx->nx_user_list);
 				FREE(nx->nx_path, M_TEMP);
 				FREE(nx, M_TEMP);
 			}
@@ -2322,7 +2252,7 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 			FREE(nxfs->nxfs_path, M_TEMP);
 			FREE(nxfs, M_TEMP);
 		}
-		lck_rw_done(&nfs_export_rwlock);
+		lck_rw_done(&nfsrv_export_rwlock);
 		return (0);
 	}
 
@@ -2330,36 +2260,39 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 	if (error)
 		return (error);
 
-	lck_rw_lock_exclusive(&nfs_export_rwlock);
+	lck_rw_lock_exclusive(&nfsrv_export_rwlock);
 
 	// first check if we've already got an exportfs with the given ID
-	LIST_FOREACH(nxfs, &nfs_exports, nxfs_next) {
+	LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next) {
 		if (nxfs->nxfs_id == unxa->nxa_fsid)
 			break;
 	}
 	if (nxfs) {
 		/* verify exported FS path matches given path */
-		if (strcmp(path, nxfs->nxfs_path)) {
+		if (strncmp(path, nxfs->nxfs_path, MAXPATHLEN)) {
 			error = EEXIST;
 			goto unlock_out;
 		}
-		mp = vfs_getvfs_by_mntonname(nxfs->nxfs_path);
-		/* find exported FS root vnode */
-		NDINIT(&mnd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, 
-			UIO_SYSSPACE, nxfs->nxfs_path, ctx);
-		error = namei(&mnd);
-		if (error)
-			goto unlock_out;
-		mvp = mnd.ni_vp;
-		/* make sure it's (still) the root of a file system */
-		if ((mvp->v_flag & VROOT) == 0) {
-			error = EINVAL;
-			goto out;
-		}
-		/* sanity check: this should be same mount */
-		if (mp != vnode_mount(mvp)) {
-			error = EINVAL;
-			goto out;
+		if ((unxa->nxa_flags & (NXA_ADD|NXA_OFFLINE)) == NXA_ADD) {
+			/* if adding, verify that the mount is still what we expect */
+			mp = vfs_getvfs_by_mntonname(nxfs->nxfs_path);
+			/* find exported FS root vnode */
+			NDINIT(&mnd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
+				UIO_SYSSPACE, CAST_USER_ADDR_T(nxfs->nxfs_path), ctx);
+			error = namei(&mnd);
+			if (error)
+				goto unlock_out;
+			mvp = mnd.ni_vp;
+			/* make sure it's (still) the root of a file system */
+			if (!vnode_isvroot(mvp)) {
+				error = EINVAL;
+				goto out;
+			}
+			/* sanity check: this should be same mount */
+			if (mp != vnode_mount(mvp)) {
+				error = EINVAL;
+				goto out;
+			}
 		}
 	} else {
 		/* no current exported file system with that ID */
@@ -2369,26 +2302,36 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 		}
 
 		/* find exported FS root vnode */
-		NDINIT(&mnd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, 
-			UIO_SYSSPACE, path, ctx);
+		NDINIT(&mnd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
+			UIO_SYSSPACE, CAST_USER_ADDR_T(path), ctx);
 		error = namei(&mnd);
-		if (error)
-			goto unlock_out;
-		mvp = mnd.ni_vp;
-		/* make sure it's the root of a file system */
-		if ((mvp->v_flag & VROOT) == 0) {
-			error = EINVAL;
-			goto out;
-		}
-		mp = vnode_mount(mvp);
+		if (error) {
+			if (!(unxa->nxa_flags & NXA_OFFLINE))
+				goto unlock_out;
+		} else {
+			mvp = mnd.ni_vp;
+			/* make sure it's the root of a file system */
+			if (!vnode_isvroot(mvp)) {
+				/* bail if not marked offline */
+				if (!(unxa->nxa_flags & NXA_OFFLINE)) {
+					error = EINVAL;
+					goto out;
+				}
+				vnode_put(mvp);
+				nameidone(&mnd);
+				mvp = NULL;
+			} else {
+				mp = vnode_mount(mvp);
 
-		/* make sure the file system is NFS-exportable */
-		nfh.nfh_len = NFS_MAX_FID_SIZE;
-		error = VFS_VPTOFH(mvp, &nfh.nfh_len, &nfh.nfh_fid[0], NULL);
-		if (!error && (nfh.nfh_len > (int)NFS_MAX_FID_SIZE))
-			error = EIO;
-		if (error)
-			goto out;
+				/* make sure the file system is NFS-exportable */
+				nfh.nfh_len = NFSV3_MAX_FID_SIZE;
+				error = VFS_VPTOFH(mvp, (int*)&nfh.nfh_len, &nfh.nfh_fid[0], NULL);
+				if (!error && (nfh.nfh_len > (int)NFSV3_MAX_FID_SIZE))
+					error = EIO;
+				if (error)
+					goto out;
+			}
+		}
 
 		/* add an exportfs for it */
 		MALLOC(nxfs, struct nfs_exportfs *, sizeof(struct nfs_exportfs), M_TEMP, M_WAITOK);
@@ -2407,8 +2350,8 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 		bcopy(path, nxfs->nxfs_path, pathlen);
 		/* insert into list in reverse-sorted order */
 		nxfs3 = NULL;
-		LIST_FOREACH(nxfs2, &nfs_exports, nxfs_next) {
-			if (strcmp(nxfs->nxfs_path, nxfs2->nxfs_path) > 0)
+		LIST_FOREACH(nxfs2, &nfsrv_exports, nxfs_next) {
+			if (strncmp(nxfs->nxfs_path, nxfs2->nxfs_path, MAXPATHLEN) > 0)
 				break;
 			nxfs3 = nxfs2;
 		}
@@ -2417,10 +2360,11 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 		else if (nxfs3)
 			LIST_INSERT_AFTER(nxfs3, nxfs, nxfs_next);
 		else
-			LIST_INSERT_HEAD(&nfs_exports, nxfs, nxfs_next);
+			LIST_INSERT_HEAD(&nfsrv_exports, nxfs, nxfs_next);
 
 		/* make sure any quotas are enabled before we export the file system */
-		enablequotas(mp, ctx);
+		if (mp)
+			enablequotas(mp, ctx);
 	}
 
 	if (unxa->nxa_exppath) {
@@ -2433,7 +2377,7 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 		}
 		if (nx) {
 			/* verify exported FS path matches given path */
-			if (strcmp(path, nx->nx_path)) {
+			if (strncmp(path, nx->nx_path, MAXPATHLEN)) {
 				error = EEXIST;
 				goto out;
 			}
@@ -2452,6 +2396,7 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 			bzero(nx, sizeof(struct nfs_export));
 			nx->nx_id = unxa->nxa_expid;
 			nx->nx_fs = nxfs;
+			microtime(&nx->nx_exptime);
 			MALLOC(nx->nx_path, char*, pathlen, M_TEMP, M_WAITOK);
 			if (!nx->nx_path) {
 				error = ENOMEM;
@@ -2460,10 +2405,12 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 				goto out1;
 			}
 			bcopy(path, nx->nx_path, pathlen);
+			/* initialize the active user list */
+			nfsrv_init_user_list(&nx->nx_user_list);
 			/* insert into list in reverse-sorted order */
 			nx3 = NULL;
 			LIST_FOREACH(nx2, &nxfs->nxfs_exports, nx_next) {
-				if (strcmp(nx->nx_path, nx2->nx_path) > 0)
+				if (strncmp(nx->nx_path, nx2->nx_path, MAXPATHLEN) > 0)
 					break;
 				nx3 = nx2;
 			}
@@ -2474,10 +2421,10 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 			else
 				LIST_INSERT_HEAD(&nxfs->nxfs_exports, nx, nx_next);
 			/* insert into hash */
-			LIST_INSERT_HEAD(NFSEXPHASH(nxfs->nxfs_id, nx->nx_id), nx, nx_hash);
+			LIST_INSERT_HEAD(NFSRVEXPHASH(nxfs->nxfs_id, nx->nx_id), nx, nx_hash);
 
 			/*
-			 * We don't allow nested exports.  Check if the new entry
+			 * We don't allow/support nested exports.  Check if the new entry
 			 * nests with the entries before and after or if there's an
 			 * entry for the file system root and subdirs.
 			 */
@@ -2504,54 +2451,76 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 					error = EINVAL;
 			}
 			if (error) {
-				printf("nfsrv_export: attempt to register nested exports: %s/%s\n",
+				/*
+				 * Don't actually return an error because mountd is
+				 * probably about to delete the conflicting export.
+				 * This can happen when a new export momentarily conflicts
+				 * with an old export while the transition is being made.
+				 * Theoretically, mountd could be written to avoid this
+				 * transient situation - but it would greatly increase the
+				 * complexity of mountd for very little overall benefit.
+				 */
+				printf("nfsrv_export: warning: nested exports: %s/%s\n",
 					nxfs->nxfs_path, nx->nx_path);
-				goto out1;
-			}
-
-			/* find export root vnode */
-			if (!nx->nx_path[0] || ((nx->nx_path[0] == '.') && !nx->nx_path[1])) {
-				/* exporting file system's root directory */
-				xvp = mvp;
-				vnode_get(xvp);
-			} else {
-				xnd.ni_cnd.cn_nameiop = LOOKUP;
-				xnd.ni_cnd.cn_flags = LOCKLEAF;
-				xnd.ni_pathlen = pathlen - 1;
-				xnd.ni_cnd.cn_nameptr = xnd.ni_cnd.cn_pnbuf = path;
-				xnd.ni_startdir = mvp;
-				xnd.ni_usedvp   = mvp;
-				xnd.ni_cnd.cn_context = ctx;
-				error = lookup(&xnd);
-				if (error)
-					goto out1;
-				xvp = xnd.ni_vp;
+				error = 0;
 			}
-
-			if (vnode_vtype(xvp) != VDIR) {
-				error = EINVAL;
-				vnode_put(xvp);
-				goto out1;
-			}
-
-			/* grab file handle */
+			nx->nx_fh.nfh_xh.nxh_flags = NXHF_INVALIDFH;
+		}
+		/* make sure file handle is set up */
+		if ((nx->nx_fh.nfh_xh.nxh_version != htonl(NFS_FH_VERSION)) ||
+		    (nx->nx_fh.nfh_xh.nxh_flags & NXHF_INVALIDFH)) {
+			/* try to set up export root file handle */
 			nx->nx_fh.nfh_xh.nxh_version = htonl(NFS_FH_VERSION);
 			nx->nx_fh.nfh_xh.nxh_fsid = htonl(nx->nx_fs->nxfs_id);
 			nx->nx_fh.nfh_xh.nxh_expid = htonl(nx->nx_id);
 			nx->nx_fh.nfh_xh.nxh_flags = 0;
 			nx->nx_fh.nfh_xh.nxh_reserved = 0;
-			nx->nx_fh.nfh_len = NFS_MAX_FID_SIZE;
-			error = VFS_VPTOFH(xvp, &nx->nx_fh.nfh_len, &nx->nx_fh.nfh_fid[0], NULL);
-			if (!error && (nx->nx_fh.nfh_len > (int)NFS_MAX_FID_SIZE)) {
-				error = EIO;
+			nx->nx_fh.nfh_fhp = (u_char*)&nx->nx_fh.nfh_xh;
+			bzero(&nx->nx_fh.nfh_fid[0], NFSV2_MAX_FID_SIZE);
+			if (mvp) {
+				/* find export root vnode */
+				if (!nx->nx_path[0] || ((nx->nx_path[0] == '.') && !nx->nx_path[1])) {
+					/* exporting file system's root directory */
+					xvp = mvp;
+					vnode_get(xvp);
+				} else {
+					xnd.ni_cnd.cn_nameiop = LOOKUP;
+					xnd.ni_cnd.cn_flags = LOCKLEAF;
+					xnd.ni_pathlen = pathlen - 1;
+					xnd.ni_cnd.cn_nameptr = xnd.ni_cnd.cn_pnbuf = path;
+					xnd.ni_startdir = mvp;
+					xnd.ni_usedvp   = mvp;
+					xnd.ni_cnd.cn_context = ctx;
+					error = lookup(&xnd);
+					if (error)
+						goto out1;
+					xvp = xnd.ni_vp;
+				}
+
+				if (vnode_vtype(xvp) != VDIR) {
+					error = EINVAL;
+					vnode_put(xvp);
+					goto out1;
+				}
+
+				/* grab file handle */
+				nx->nx_fh.nfh_len = NFSV3_MAX_FID_SIZE;
+				error = VFS_VPTOFH(xvp, (int*)&nx->nx_fh.nfh_len, &nx->nx_fh.nfh_fid[0], NULL);
+				if (!error && (nx->nx_fh.nfh_len > (int)NFSV3_MAX_FID_SIZE)) {
+					error = EIO;
+				} else {
+					nx->nx_fh.nfh_xh.nxh_fidlen = nx->nx_fh.nfh_len;
+					nx->nx_fh.nfh_len += sizeof(nx->nx_fh.nfh_xh);
+				}
+
+				vnode_put(xvp);
+				if (error)
+					goto out1;
 			} else {
-				nx->nx_fh.nfh_xh.nxh_fidlen = nx->nx_fh.nfh_len;
-				nx->nx_fh.nfh_len += sizeof(nx->nx_fh.nfh_xh);
+				nx->nx_fh.nfh_xh.nxh_flags = NXHF_INVALIDFH;
+				nx->nx_fh.nfh_xh.nxh_fidlen = 0;
+				nx->nx_fh.nfh_len = sizeof(nx->nx_fh.nfh_xh);
 			}
-
-			vnode_put(xvp);
-			if (error)
-				goto out1;
 		}
 	} else {
 		nx = NULL;
@@ -2565,28 +2534,43 @@ nfsrv_export(struct user_nfs_export_args *unxa, struct vfs_context *ctx)
 				LIST_REMOVE(nx, nx_next);
 				LIST_REMOVE(nx, nx_hash);
 				/* delete all netopts for this export */
-				nfsrv_free_addrlist(nx);
+				nfsrv_free_addrlist(nx, NULL);
 				nx->nx_flags &= ~NX_DEFAULTEXPORT;
 				if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
 					kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 				}
+				/* delete active user list for this export */
+				nfsrv_free_user_list(&nx->nx_user_list);
 				FREE(nx->nx_path, M_TEMP);
 				FREE(nx, M_TEMP);
 			}
 			goto out1;
-		} else {
+		} else if (!unxa->nxa_netcount) {
 			/* delete all netopts for this export */
-			nfsrv_free_addrlist(nx);
+			nfsrv_free_addrlist(nx, NULL);
 			nx->nx_flags &= ~NX_DEFAULTEXPORT;
 			if (IS_VALID_CRED(nx->nx_defopt.nxo_cred)) {
 				kauth_cred_unref(&nx->nx_defopt.nxo_cred);
 			}
+		} else {
+			/* delete only the netopts for the given addresses */
+			error = nfsrv_free_addrlist(nx, unxa);
+			if (error)
+				goto out1;
 		}
 	}
 	if (unxa->nxa_flags & NXA_ADD) {
+		/* 
+		 * If going offline set the export time so that when
+		 * coming back on line we will present a new write verifier
+		 * to the client.
+		 */
+		if (unxa->nxa_flags & NXA_OFFLINE)
+			microtime(&nx->nx_exptime);
+
 		error = nfsrv_hang_addrlist(nx, unxa);
-		if (!error)
-			mp->mnt_flag |= MNT_EXPORTED;
+		if (!error && mp)
+			vfs_setflags(mp, MNT_EXPORTED);
 	}
 
 out1:
@@ -2594,6 +2578,8 @@ out1:
 		/* export has no export options */
 		LIST_REMOVE(nx, nx_next);
 		LIST_REMOVE(nx, nx_hash);
+		/* delete active user list for this export */
+		nfsrv_free_user_list(&nx->nx_user_list);
 		FREE(nx->nx_path, M_TEMP);
 		FREE(nx, M_TEMP);
 	}
@@ -2602,7 +2588,8 @@ out1:
 		LIST_REMOVE(nxfs, nxfs_next);
 		FREE(nxfs->nxfs_path, M_TEMP);
 		FREE(nxfs, M_TEMP);
-		mp->mnt_flag &= ~MNT_EXPORTED;
+		if (mp)
+			vfs_clearflags(mp, MNT_EXPORTED);
 	}
 
 out:
@@ -2611,7 +2598,7 @@ out:
 		nameidone(&mnd);
 	}
 unlock_out:
-	lck_rw_done(&nfs_export_rwlock);
+	lck_rw_done(&nfsrv_export_rwlock);
 	return (error);
 }
 
@@ -2646,12 +2633,13 @@ nfsrv_export_lookup(struct nfs_export *nx, mbuf_t nam)
 static struct nfs_export *
 nfsrv_fhtoexport(struct nfs_filehandle *nfhp)
 {
+	struct nfs_exphandle *nxh = (struct nfs_exphandle*)nfhp->nfh_fhp;
 	struct nfs_export *nx;
 	uint32_t fsid, expid;
 
-	fsid = ntohl(nfhp->nfh_xh.nxh_fsid);
-	expid = ntohl(nfhp->nfh_xh.nxh_expid);
-	nx = NFSEXPHASH(fsid, expid)->lh_first;
+	fsid = ntohl(nxh->nxh_fsid);
+	expid = ntohl(nxh->nxh_expid);
+	nx = NFSRVEXPHASH(fsid, expid)->lh_first;
 	for (; nx; nx = LIST_NEXT(nx, nx_hash)) {
 		if (nx->nx_fs->nxfs_id != fsid)
 			continue;
@@ -2668,57 +2656,92 @@ nfsrv_fhtoexport(struct nfs_filehandle *nfhp)
 int
 nfsrv_fhtovp(
 	struct nfs_filehandle *nfhp,
-	mbuf_t nam,
-	__unused int pubflag,
+	struct nfsrv_descript *nd,
 	vnode_t *vpp,
 	struct nfs_export **nxp,
 	struct nfs_export_options **nxop)
 {
+	struct nfs_exphandle *nxh = (struct nfs_exphandle*)nfhp->nfh_fhp;
+	struct nfs_export_options *nxo;
+	u_char *fidp;
 	int error;
 	struct mount *mp;
+	mbuf_t nam = NULL;
 	uint32_t v;
+	int i, valid;
 
 	*vpp = NULL;
 	*nxp = NULL;
 	*nxop = NULL;
 
-	v = ntohl(nfhp->nfh_xh.nxh_version);
+	if (nd != NULL)
+		nam = nd->nd_nam;
+
+	v = ntohl(nxh->nxh_version);
 	if (v != NFS_FH_VERSION) {
 		/* file handle format not supported */
 		return (ESTALE);
 	}
-	if (nfhp->nfh_len > NFS_MAX_FH_SIZE)
+	if (nfhp->nfh_len > NFSV3_MAX_FH_SIZE)
 		return (EBADRPC);
-	if (nfhp->nfh_len < (int)sizeof(nfhp->nfh_xh))
+	if (nfhp->nfh_len < (int)sizeof(struct nfs_exphandle))
 		return (ESTALE);
-	v = ntohs(nfhp->nfh_xh.nxh_flags);
+	v = ntohs(nxh->nxh_flags);
 	if (v & NXHF_INVALIDFH)
 		return (ESTALE);
 
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	if (nfs_ispublicfh(nfhp)) {
-		if (!pubflag || !nfs_pub.np_valid)
-			return (ESTALE);
-		nfhp = &nfs_pub.np_handle;
-	}
-#endif
-
 	*nxp = nfsrv_fhtoexport(nfhp);
 	if (!*nxp)
 		return (ESTALE);
 
 	/* Get the export option structure for this <export, client> tuple. */
-	*nxop = nfsrv_export_lookup(*nxp, nam);
+	*nxop = nxo = nfsrv_export_lookup(*nxp, nam);
 	if (nam && (*nxop == NULL))
 		return (EACCES);
 
+	if (nd != NULL) {
+		/* Validate the security flavor of the request */
+		for (i = 0, valid = 0; i < nxo->nxo_sec.count; i++) {
+			if (nd->nd_sec == nxo->nxo_sec.flavors[i]) {
+				valid = 1;
+				break;
+			}
+		}
+		if (!valid) {
+			/*
+			 * RFC 2623 section 2.3.2 recommends no authentication
+			 * requirement for certain NFS procedures used for mounting.
+			 * This allows an unauthenticated superuser on the client
+			 * to do mounts for the benefit of authenticated users.
+			 */
+			if (nd->nd_vers == NFS_VER2)
+				if (nd->nd_procnum == NFSV2PROC_GETATTR ||
+				    nd->nd_procnum == NFSV2PROC_STATFS)
+					valid = 1;
+			if (nd->nd_vers == NFS_VER3)
+				if (nd->nd_procnum == NFSPROC_FSINFO)
+					valid = 1;
+
+			if (!valid)
+				return (NFSERR_AUTHERR | AUTH_REJECTCRED);
+		}
+	}
+
+	if (nxo && (nxo->nxo_flags & NX_OFFLINE))
+		return ((nd->nd_vers == NFS_VER2) ? ESTALE : NFSERR_TRYLATER);
+
 	/* find mount structure */
 	mp = vfs_getvfs_by_mntonname((*nxp)->nx_fs->nxfs_path);
-	if (!mp)
-		return (ESTALE);
+	if (!mp) {
+		/*
+		 * We have an export, but no mount?
+		 * Perhaps the export just hasn't been marked offline yet.
+		 */
+		return ((nd->nd_vers == NFS_VER2) ? ESTALE : NFSERR_TRYLATER);
+	}
 
-	error = VFS_FHTOVP(mp, nfhp->nfh_xh.nxh_fidlen, &nfhp->nfh_fid[0], vpp, NULL);
+	fidp = nfhp->nfh_fhp + sizeof(*nxh);
+	error = VFS_FHTOVP(mp, nxh->nxh_fidlen, fidp, vpp, NULL);
 	if (error)
 		return (error);
 	/* vnode pointer should be good at this point or ... */
@@ -2728,47 +2751,28 @@ nfsrv_fhtovp(
 }
 
 /*
- * nfsrv_credcheck() - check/map credentials according to given export options
+ * nfsrv_credcheck() - check/map credentials according
+ * to given export options.
  */
 int
 nfsrv_credcheck(
-	struct nfsrv_descript *nfsd,
+	struct nfsrv_descript *nd,
+	vfs_context_t ctx,
 	__unused struct nfs_export *nx,
 	struct nfs_export_options *nxo)
 {
 	if (nxo && nxo->nxo_cred) {
 		if ((nxo->nxo_flags & NX_MAPALL) ||
-		    ((nxo->nxo_flags & NX_MAPROOT) && !suser(nfsd->nd_cr, NULL))) {
+		    ((nxo->nxo_flags & NX_MAPROOT) && !suser(nd->nd_cr, NULL))) {
 			kauth_cred_ref(nxo->nxo_cred);
-			kauth_cred_unref(&nfsd->nd_cr);
-			nfsd->nd_cr = nxo->nxo_cred;
+			kauth_cred_unref(&nd->nd_cr);
+			nd->nd_cr = nxo->nxo_cred;
 		}
 	}
+	ctx->vc_ucred = nd->nd_cr;
 	return (0);
 }
 
-
-/*
- * WebNFS: check if a filehandle is a public filehandle. For v3, this
- * means a length of 0, for v2 it means all zeroes. nfsm_srvmtofh has
- * transformed this to all zeroes in both cases, so check for it.
- */
-int
-nfs_ispublicfh(struct nfs_filehandle *nfhp)
-{
-	char *cp = (char *)nfhp;
-	unsigned int i;
-
-	if (nfhp->nfh_len == 0)
-		return (TRUE);
-	if (nfhp->nfh_len != NFSX_V2FH)
-		return (FALSE);
-	for (i = 0; i < NFSX_V2FH; i++)
-		if (*cp++ != 0)
-			return (FALSE);
-	return (TRUE);
-}
-
 /*
  * nfsrv_vptofh() - convert vnode to file handle for given export
  *
@@ -2780,216 +2784,729 @@ nfs_ispublicfh(struct nfs_filehandle *nfhp)
 int
 nfsrv_vptofh(
 	struct nfs_export *nx,
-	int v2,
+	int nfsvers,
 	struct nfs_filehandle *dnfhp,
 	vnode_t vp,
-	struct vfs_context *ctx,
+	vfs_context_t ctx,
 	struct nfs_filehandle *nfhp)
 {
 	int error;
+	uint32_t maxfidsize;
 
+	nfhp->nfh_fhp = (u_char*)&nfhp->nfh_xh;
 	nfhp->nfh_xh.nxh_version = htonl(NFS_FH_VERSION);
 	nfhp->nfh_xh.nxh_fsid = htonl(nx->nx_fs->nxfs_id);
 	nfhp->nfh_xh.nxh_expid = htonl(nx->nx_id);
 	nfhp->nfh_xh.nxh_flags = 0;
 	nfhp->nfh_xh.nxh_reserved = 0;
 
-	if (v2)
+	if (nfsvers == NFS_VER2)
 		bzero(&nfhp->nfh_fid[0], NFSV2_MAX_FID_SIZE);
 
 	/* if directory FH matches export root, return invalid FH */
 	if (dnfhp && nfsrv_fhmatch(dnfhp, &nx->nx_fh)) {
-		nfhp->nfh_len = v2 ? NFSX_V2FH : sizeof(nfhp->nfh_xh);
+		if (nfsvers == NFS_VER2)
+			nfhp->nfh_len = NFSX_V2FH;
+		else
+			nfhp->nfh_len = sizeof(nfhp->nfh_xh);
 		nfhp->nfh_xh.nxh_fidlen = 0;
 		nfhp->nfh_xh.nxh_flags = htons(NXHF_INVALIDFH);
 		return (0);
 	}
 
-	nfhp->nfh_len = v2 ? NFSV2_MAX_FID_SIZE : NFS_MAX_FID_SIZE;
-	error = VFS_VPTOFH(vp, &nfhp->nfh_len, &nfhp->nfh_fid[0], ctx);
+	if (nfsvers == NFS_VER2)
+		maxfidsize = NFSV2_MAX_FID_SIZE;
+	else
+		maxfidsize = NFSV3_MAX_FID_SIZE;
+	nfhp->nfh_len = maxfidsize;
+
+	error = VFS_VPTOFH(vp, (int*)&nfhp->nfh_len, &nfhp->nfh_fid[0], ctx);
 	if (error)
 		return (error);
-	if (nfhp->nfh_len > (int)(v2 ? NFSV2_MAX_FID_SIZE : NFS_MAX_FID_SIZE))
+	if (nfhp->nfh_len > maxfidsize)
 		return (EOVERFLOW);
 	nfhp->nfh_xh.nxh_fidlen = nfhp->nfh_len;
 	nfhp->nfh_len += sizeof(nfhp->nfh_xh);
-	if (v2 && (nfhp->nfh_len < NFSX_V2FH))
+	if ((nfsvers == NFS_VER2) && (nfhp->nfh_len < NFSX_V2FH))
 		nfhp->nfh_len = NFSX_V2FH;
 
 	return (0);
 }
 
+/*
+ * Compare two file handles to see it they're the same.
+ * Note that we don't use nfh_len because that may include
+ * padding in an NFSv2 file handle.
+ */
 int
 nfsrv_fhmatch(struct nfs_filehandle *fh1, struct nfs_filehandle *fh2)
 {
+	struct nfs_exphandle *nxh1, *nxh2;
 	int len1, len2;
 
-	len1 = sizeof(fh1->nfh_xh) + fh1->nfh_xh.nxh_fidlen;
-	len2 = sizeof(fh2->nfh_xh) + fh2->nfh_xh.nxh_fidlen;
+	nxh1 = (struct nfs_exphandle *)fh1->nfh_fhp;
+	nxh2 = (struct nfs_exphandle *)fh2->nfh_fhp;
+	len1 = sizeof(fh1->nfh_xh) + nxh1->nxh_fidlen;
+	len2 = sizeof(fh2->nfh_xh) + nxh2->nxh_fidlen;
 	if (len1 != len2)
 		return (0);
-	if (bcmp(&fh1->nfh_xh, &fh2->nfh_xh, len1))
+	if (bcmp(nxh1, nxh2, len1))
 		return (0);
 	return (1);
 }
-  
-#endif /* NFS_NOSERVER */
+
 /*
- * This function compares two net addresses by family and returns TRUE
- * if they are the same host.
- * If there is any doubt, return FALSE.
- * The AF_INET family is handled as a special case so that address mbufs
- * don't need to be saved to store "struct in_addr", which is only 4 bytes.
+ * Functions for dealing with active user lists
  */
-int
-netaddr_match(family, haddr, nam)
-	int family;
-	union nethostaddr *haddr;
-	mbuf_t nam;
+
+/*
+ * Compare address fields of two sockaddr_storage structures.
+ * Returns zero if they match.
+ */
+static int
+nfsrv_cmp_sockaddr(struct sockaddr_storage *sock1, struct sockaddr_storage *sock2)
 {
-	struct sockaddr_in *inetaddr;
-
-	switch (family) {
-	case AF_INET:
-		inetaddr = mbuf_data(nam);
-		if (inetaddr->sin_family == AF_INET &&
-		    inetaddr->sin_addr.s_addr == haddr->had_inetaddr)
-			return (1);
-		break;
-#if ISO
-	case AF_ISO:
-	    {
-		struct sockaddr_iso *isoaddr1, *isoaddr2;
-
-		isoaddr1 = mbuf_data(nam);
-		isoaddr2 = mbuf_data(haddr->had_nam);
-		if (isoaddr1->siso_family == AF_ISO &&
-		    isoaddr1->siso_nlen > 0 &&
-		    isoaddr1->siso_nlen == isoaddr2->siso_nlen &&
-		    SAME_ISOADDR(isoaddr1, isoaddr2))
-			return (1);
-		break;
-	    }
-#endif	/* ISO */
-	default:
-		break;
-	};
-	return (0);
+	struct sockaddr_in	*ipv4_sock1, *ipv4_sock2;
+	struct sockaddr_in6	*ipv6_sock1, *ipv6_sock2;
+
+	/* check for valid parameters */
+	if (sock1 == NULL || sock2 == NULL)
+		return 1;
+
+	/* check address length */
+	if (sock1->ss_len != sock2->ss_len)
+		return 1;
+
+	/* Check address family */
+	if (sock1->ss_family != sock2->ss_family)
+		return 1;
+
+	if (sock1->ss_family == AF_INET) {
+		/* IPv4 */
+		ipv4_sock1 = (struct sockaddr_in *)sock1;
+		ipv4_sock2 = (struct sockaddr_in *)sock2;
+
+		if (!bcmp(&ipv4_sock1->sin_addr, &ipv4_sock2->sin_addr, sizeof(struct in_addr)))
+			return 0;
+	} else {
+		/* IPv6 */
+		ipv6_sock1 = (struct sockaddr_in6 *)sock1;
+		ipv6_sock2 = (struct sockaddr_in6 *)sock2;
+
+		if (!bcmp(&ipv6_sock1->sin6_addr, &ipv6_sock2->sin6_addr, sizeof(struct in6_addr)))
+			return 0;
+	}
+	return 1;
 }
 
-static nfsuint64 nfs_nullcookie = { { 0, 0 } };
 /*
- * This function finds the directory cookie that corresponds to the
- * logical byte offset given.
+ * Search the hash table for a user node with a matching IP address and uid field.
+ * If found, the node's tm_last timestamp is updated and the node is returned.
+ *
+ * If not found, a new node is allocated (or reclaimed via LRU), initialized, and returned.
+ * Returns NULL if a new node could not be allcoated.
+ *
+ * The list's user_mutex lock MUST be held.
  */
-nfsuint64 *
-nfs_getcookie(np, off, add)
-	struct nfsnode *np;
-	off_t off;
-	int add;
+static struct nfs_user_stat_node *
+nfsrv_get_user_stat_node(struct nfs_active_user_list *list, struct sockaddr_storage *sock, uid_t uid)
 {
-	struct nfsdmap *dp, *dp2;
-	int pos;
+	struct nfs_user_stat_node		*unode;
+	struct timeval				now;
+	struct nfs_user_stat_hashtbl_head	*head;
+
+	/* seach the hash table */
+	head = NFS_USER_STAT_HASH(list->user_hashtbl, uid);
+	LIST_FOREACH(unode, head, hash_link) {
+		if (uid == unode->uid && nfsrv_cmp_sockaddr(sock, &unode->sock) == 0) {
+			/* found matching node */
+			break;
+		}
+	}
 
-	pos = off / NFS_DIRBLKSIZ;
-	if (pos == 0) {
-#if DIAGNOSTIC
-		if (add)
-			panic("nfs getcookie add at 0");
-#endif
-		return (&nfs_nullcookie);
+	if (unode) {
+		/* found node in the hash table, now update lru position */
+		TAILQ_REMOVE(&list->user_lru, unode, lru_link);
+		TAILQ_INSERT_TAIL(&list->user_lru, unode, lru_link);
+
+		/* update time stamp */
+		microtime(&now);
+		unode->tm_last = (uint32_t)now.tv_sec;
+		return unode;
 	}
-	pos--;
-	dp = np->n_cookies.lh_first;
-	if (!dp) {
-		if (add) {
-			MALLOC_ZONE(dp, struct nfsdmap *, sizeof(struct nfsdmap),
-					M_NFSDIROFF, M_WAITOK);
-			if (!dp)
-				return ((nfsuint64 *)0);
-			dp->ndm_eocookie = 0;
-			LIST_INSERT_HEAD(&np->n_cookies, dp, ndm_list);
-		} else
-			return ((nfsuint64 *)0);
+
+	if (list->node_count < nfsrv_user_stat_max_nodes) {
+		/* Allocate a new node */
+		MALLOC(unode, struct nfs_user_stat_node *, sizeof(struct nfs_user_stat_node),
+			M_TEMP, M_WAITOK | M_ZERO);
+
+		if (!unode)
+			return NULL;
+
+		/* increment node count */
+		OSAddAtomic(1, (SInt32*)&nfsrv_user_stat_node_count);
+		list->node_count++;
+	} else {
+		/* reuse the oldest node in the lru list */
+		unode = TAILQ_FIRST(&list->user_lru);
+
+		if (!unode)
+			return NULL;
+
+		/* Remove the node */
+		TAILQ_REMOVE(&list->user_lru, unode, lru_link);
+		LIST_REMOVE(unode, hash_link);
 	}
-	while (pos >= NFSNUMCOOKIES) {
-		pos -= NFSNUMCOOKIES;
-		if (dp->ndm_list.le_next) {
-			if (!add && dp->ndm_eocookie < NFSNUMCOOKIES &&
-				pos >= dp->ndm_eocookie)
-				return ((nfsuint64 *)0);
-			dp = dp->ndm_list.le_next;
-		} else if (add) {
-			MALLOC_ZONE(dp2, struct nfsdmap *, sizeof(struct nfsdmap),
-					M_NFSDIROFF, M_WAITOK);
-			if (!dp2)
-				return ((nfsuint64 *)0);
-			dp2->ndm_eocookie = 0;
-			LIST_INSERT_AFTER(dp, dp2, ndm_list);
-			dp = dp2;
-		} else
-			return ((nfsuint64 *)0);
+
+	/* Initialize the node */
+	unode->uid = uid;
+	bcopy(sock, &unode->sock, sock->ss_len);
+	microtime(&now);
+	unode->ops = 0;
+	unode->bytes_read = 0;
+	unode->bytes_written = 0;
+	unode->tm_start = (uint32_t)now.tv_sec;
+	unode->tm_last = (uint32_t)now.tv_sec;
+
+	/* insert the node  */
+	TAILQ_INSERT_TAIL(&list->user_lru, unode, lru_link);
+	LIST_INSERT_HEAD(head, unode, hash_link);
+
+	return unode;
+}
+
+void
+nfsrv_update_user_stat(struct nfs_export *nx, struct nfsrv_descript *nd, uid_t uid, u_int ops, u_int rd_bytes, u_int wr_bytes)
+{
+	struct nfs_user_stat_node	*unode;
+	struct nfs_active_user_list	*ulist;
+	struct sockaddr_storage		*sock_stor;
+
+	if ((!nfsrv_user_stat_enabled) || (!nx) || (!nd) || (!nd->nd_nam))
+		return;
+
+	sock_stor = (struct sockaddr_storage *)mbuf_data(nd->nd_nam);
+
+	/* check address family before going any further */
+	if ((sock_stor->ss_family != AF_INET) && (sock_stor->ss_family != AF_INET6))
+		return;
+
+	ulist = &nx->nx_user_list;
+
+	/* lock the active user list */
+	lck_mtx_lock(&ulist->user_mutex);
+
+	/* get the user node */
+	unode = nfsrv_get_user_stat_node(ulist, sock_stor, uid);
+
+	if (!unode) {
+		lck_mtx_unlock(&ulist->user_mutex);
+		return;
 	}
-	if (pos >= dp->ndm_eocookie) {
-		if (add)
-			dp->ndm_eocookie = pos + 1;
-		else
-			return ((nfsuint64 *)0);
+
+	/* update counters */
+	unode->ops += ops;
+	unode->bytes_read += rd_bytes;
+	unode->bytes_written += wr_bytes;
+
+	/* done */
+	lck_mtx_unlock(&ulist->user_mutex);
+}
+
+/* initialize an active user list */
+static void
+nfsrv_init_user_list(struct nfs_active_user_list *ulist)
+{
+	uint i;
+
+	/* initialize the lru */
+	TAILQ_INIT(&ulist->user_lru);
+
+	/* initialize the hash table */
+	for(i = 0; i < NFS_USER_STAT_HASH_SIZE; i++)
+		LIST_INIT(&ulist->user_hashtbl[i]);
+	ulist->node_count = 0;
+
+	lck_mtx_init(&ulist->user_mutex, nfsrv_active_user_mutex_group, LCK_ATTR_NULL);
+}
+
+/* Free all nodes in an active user list */
+static void
+nfsrv_free_user_list(struct nfs_active_user_list *ulist)
+{
+	struct nfs_user_stat_node *unode;
+
+	if (!ulist)
+		return;
+
+	while ((unode = TAILQ_FIRST(&ulist->user_lru))) {
+		/* Remove node and free */
+		TAILQ_REMOVE(&ulist->user_lru, unode, lru_link);
+		LIST_REMOVE(unode, hash_link);
+		FREE(unode, M_TEMP);
+
+		/* decrement node count */
+		OSAddAtomic(-1, (SInt32*)&nfsrv_user_stat_node_count);
 	}
-	return (&dp->ndm_cookies[pos]);
+	ulist->node_count = 0;
+
+	lck_mtx_destroy(&ulist->user_mutex, nfsrv_active_user_mutex_group);
 }
 
-/*
- * Invalidate cached directory information, except for the actual directory
- * blocks (which are invalidated separately).
- * Done mainly to avoid the use of stale offset cookies.
- */
+/* Reclaim old expired user nodes from active user lists. */
 void
-nfs_invaldir(vp)
-	vnode_t vp;
+nfsrv_active_user_list_reclaim(void)
 {
-	struct nfsnode *np = VTONFS(vp);
-
-#if DIAGNOSTIC
-	if (vnode_vtype(vp) != VDIR)
-		panic("nfs: invaldir not dir");
-#endif
-	np->n_direofoffset = 0;
-	np->n_cookieverf.nfsuquad[0] = 0;
-	np->n_cookieverf.nfsuquad[1] = 0;
-	if (np->n_cookies.lh_first)
-		np->n_cookies.lh_first->ndm_eocookie = 0;
+	struct nfs_exportfs			*nxfs;
+	struct nfs_export			*nx;
+	struct nfs_active_user_list		*ulist;
+	struct nfs_user_stat_hashtbl_head	oldlist;
+	struct nfs_user_stat_node		*unode, *unode_next;
+	struct timeval				now;
+	uint32_t				tstale;
+
+	LIST_INIT(&oldlist);
+
+	lck_rw_lock_shared(&nfsrv_export_rwlock);
+	microtime(&now);
+	tstale = now.tv_sec - nfsrv_user_stat_max_idle_sec;
+	LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next) {
+		LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next) {
+			/* Scan through all user nodes of this export */
+			ulist = &nx->nx_user_list;
+			lck_mtx_lock(&ulist->user_mutex);
+			for (unode = TAILQ_FIRST(&ulist->user_lru); unode; unode = unode_next) {
+				unode_next = TAILQ_NEXT(unode, lru_link);
+
+				/* check if this node has expired */
+				if (unode->tm_last >= tstale)
+					break;
+
+				/* Remove node from the active user list */
+				TAILQ_REMOVE(&ulist->user_lru, unode, lru_link);
+				LIST_REMOVE(unode, hash_link);
+
+				/* Add node to temp list */
+				LIST_INSERT_HEAD(&oldlist, unode, hash_link);
+
+				/* decrement node count */
+				OSAddAtomic(-1, (SInt32*)&nfsrv_user_stat_node_count);
+				ulist->node_count--;
+			}
+			/* can unlock this export's list now */
+			lck_mtx_unlock(&ulist->user_mutex);
+		}
+	}
+        lck_rw_done(&nfsrv_export_rwlock);
+
+	/* Free expired nodes */
+        while ((unode = LIST_FIRST(&oldlist))) {
+		LIST_REMOVE(unode, hash_link);
+		FREE(unode, M_TEMP);
+	}
 }
 
-#ifndef NFS_NOSERVER
 /*
- * Map errnos to NFS error numbers. For Version 3 also filter out error
- * numbers not specified for the associated procedure.
+ * Maps errno values to nfs error numbers.
+ * Use NFSERR_IO as the catch all for ones not specifically defined in
+ * RFC 1094.
  */
-int
-nfsrv_errmap(nd, err)
-	struct nfsrv_descript *nd;
-	int err;
-{
-	short *defaulterrp, *errp;
+static u_char nfsrv_v2errmap[] = {
+  NFSERR_PERM,	NFSERR_NOENT,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_NXIO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_ACCES,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_EXIST,	NFSERR_IO,	NFSERR_NODEV,	NFSERR_NOTDIR,
+  NFSERR_ISDIR,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_FBIG,	NFSERR_NOSPC,	NFSERR_IO,	NFSERR_ROFS,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_IO,	NFSERR_IO,	NFSERR_NAMETOL,	NFSERR_IO,	NFSERR_IO,
+  NFSERR_NOTEMPTY, NFSERR_IO,	NFSERR_IO,	NFSERR_DQUOT,	NFSERR_STALE,
+};
 
-	if (nd->nd_flag & ND_NFSV3) {
-	    if (nd->nd_procnum <= NFSPROC_COMMIT) {
-		errp = defaulterrp = nfsrv_v3errmap[nd->nd_procnum];
-		while (*++errp) {
-			if (*errp == err)
-				return (err);
-			else if (*errp > err)
-				break;
-		}
-		return ((int)*defaulterrp);
-	    } else
-		return (err & 0xffff);
+/*
+ * Maps errno values to nfs error numbers.
+ * Although it is not obvious whether or not NFS clients really care if
+ * a returned error value is in the specified list for the procedure, the
+ * safest thing to do is filter them appropriately. For Version 2, the
+ * X/Open XNFS document is the only specification that defines error values
+ * for each RPC (The RFC simply lists all possible error values for all RPCs),
+ * so I have decided to not do this for Version 2.
+ * The first entry is the default error return and the rest are the valid
+ * errors for that RPC in increasing numeric order.
+ */
+static short nfsv3err_null[] = {
+	0,
+	0,
+};
+
+static short nfsv3err_getattr[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_setattr[] = {
+	NFSERR_IO,
+	NFSERR_PERM,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_INVAL,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOT_SYNC,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_lookup[] = {
+	NFSERR_IO,
+	NFSERR_NOENT,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_NOTDIR,
+	NFSERR_NAMETOL,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_access[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_readlink[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_INVAL,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_read[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_NXIO,
+	NFSERR_ACCES,
+	NFSERR_INVAL,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_write[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_INVAL,
+	NFSERR_FBIG,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_create[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_NOTDIR,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_mkdir[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_NOTDIR,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_symlink[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_NOTDIR,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_mknod[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_NOTDIR,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_BADTYPE,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_remove[] = {
+	NFSERR_IO,
+	NFSERR_NOENT,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_NOTDIR,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_rmdir[] = {
+	NFSERR_IO,
+	NFSERR_NOENT,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_NOTDIR,
+	NFSERR_INVAL,
+	NFSERR_ROFS,
+	NFSERR_NAMETOL,
+	NFSERR_NOTEMPTY,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_rename[] = {
+	NFSERR_IO,
+	NFSERR_NOENT,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_XDEV,
+	NFSERR_NOTDIR,
+	NFSERR_ISDIR,
+	NFSERR_INVAL,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_MLINK,
+	NFSERR_NAMETOL,
+	NFSERR_NOTEMPTY,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_link[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_EXIST,
+	NFSERR_XDEV,
+	NFSERR_NOTDIR,
+	NFSERR_INVAL,
+	NFSERR_NOSPC,
+	NFSERR_ROFS,
+	NFSERR_MLINK,
+	NFSERR_NAMETOL,
+	NFSERR_DQUOT,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_NOTSUPP,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_readdir[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_NOTDIR,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_BAD_COOKIE,
+	NFSERR_TOOSMALL,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_readdirplus[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_ACCES,
+	NFSERR_NOTDIR,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_BAD_COOKIE,
+	NFSERR_NOTSUPP,
+	NFSERR_TOOSMALL,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_fsstat[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_fsinfo[] = {
+	NFSERR_STALE,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_pathconf[] = {
+	NFSERR_STALE,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short nfsv3err_commit[] = {
+	NFSERR_IO,
+	NFSERR_IO,
+	NFSERR_STALE,
+	NFSERR_BADHANDLE,
+	NFSERR_SERVERFAULT,
+	NFSERR_TRYLATER,
+	0,
+};
+
+static short *nfsrv_v3errmap[] = {
+	nfsv3err_null,
+	nfsv3err_getattr,
+	nfsv3err_setattr,
+	nfsv3err_lookup,
+	nfsv3err_access,
+	nfsv3err_readlink,
+	nfsv3err_read,
+	nfsv3err_write,
+	nfsv3err_create,
+	nfsv3err_mkdir,
+	nfsv3err_symlink,
+	nfsv3err_mknod,
+	nfsv3err_remove,
+	nfsv3err_rmdir,
+	nfsv3err_rename,
+	nfsv3err_link,
+	nfsv3err_readdir,
+	nfsv3err_readdirplus,
+	nfsv3err_fsstat,
+	nfsv3err_fsinfo,
+	nfsv3err_pathconf,
+	nfsv3err_commit,
+};
+
+/*
+ * Map errnos to NFS error numbers. For Version 3 also filter out error
+ * numbers not specified for the associated procedure.
+ */
+int
+nfsrv_errmap(struct nfsrv_descript *nd, int err)
+{
+	short *defaulterrp, *errp;
+
+	if (nd->nd_vers == NFS_VER2) {
+		if (err <= (int)sizeof(nfsrv_v2errmap))
+			return ((int)nfsrv_v2errmap[err - 1]);
+		return (NFSERR_IO);
+	}
+	/* NFSv3 */
+	if (nd->nd_procnum > NFSPROC_COMMIT)
+		return (err & 0xffff);
+	errp = defaulterrp = nfsrv_v3errmap[nd->nd_procnum];
+	while (*++errp) {
+		if (*errp == err)
+			return (err);
+		else if (*errp > err)
+			break;
 	}
-	if (err <= ELAST)
-		return ((int)nfsrv_v2errmap[err - 1]);
-	return (NFSERR_IO);
+	return ((int)*defaulterrp);
 }
 
-#endif /* NFS_NOSERVER */
+#endif /* NFSSERVER */
 
diff --git a/bsd/nfs/nfs_syscalls.c b/bsd/nfs/nfs_syscalls.c
index a3d7d1376..1a4f5bb0e 100644
--- a/bsd/nfs/nfs_syscalls.c
+++ b/bsd/nfs/nfs_syscalls.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,13 +64,15 @@
  *	@(#)nfs_syscalls.c	8.5 (Berkeley) 3/30/95
  * FreeBSD-Id: nfs_syscalls.c,v 1.32 1997/11/07 08:53:25 phk Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
-/* XXX CSM 11/25/97 FreeBSD's generated syscall prototypes */
-#ifdef notyet
-#include <sys/sysproto.h>
-#endif
 #include <sys/kernel.h>
 #include <sys/file_internal.h>
 #include <sys/filedesc.h>
@@ -88,66 +96,279 @@
 #include <sys/user.h>
 #include <sys/sysproto.h>
 #include <sys/kpi_socket.h>
+#include <sys/fsevents.h>
 #include <libkern/OSAtomic.h>
+#include <kern/thread_call.h>
+#include <kern/task.h>
 
 #include <bsm/audit_kernel.h>
 
 #include <netinet/in.h>
 #include <netinet/tcp.h>
-#if ISO
-#include <netiso/iso.h>
-#endif
 #include <nfs/xdr_subs.h>
 #include <nfs/rpcv2.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 #include <nfs/nfsm_subs.h>
 #include <nfs/nfsrvcache.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/nfsnode.h>
-#include <nfs/nfsrtt.h>
 #include <nfs/nfs_lock.h>
-
-extern void unix_syscall_return(int);
-
-/* Global defs. */
-extern int (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
-					    struct nfssvc_sock *slp,
-					    proc_t procp,
-					    mbuf_t *mreqp);
-extern int nfs_numasync;
-extern int nfs_ioddelwri;
-extern int nfsrtton;
-extern struct nfsstats nfsstats;
-extern int nfsrvw_procrastinate;
-extern int nfsrvw_procrastinate_v3;
-
-struct nfssvc_sock *nfs_udpsock, *nfs_cltpsock;
-static int nuidhash_max = NFS_MAXUIDHASH;
-
-static void	nfsrv_zapsock(struct nfssvc_sock *slp);
-static int	nfssvc_iod(proc_t);
-static int	nfskerb_clientd(struct nfsmount *, struct nfsd_cargs *, int, user_addr_t, proc_t);
-
-static int nfs_asyncdaemon[NFS_MAXASYNCDAEMON];
-
-#ifndef NFS_NOSERVER
-int nfsd_waiting = 0;
-static struct nfsdrt nfsdrt;
-int nfs_numnfsd = 0;
-static void	nfsd_rt(int sotype, struct nfsrv_descript *nd, int cacherep);
-static int	nfssvc_addsock(socket_t, mbuf_t, proc_t);
-static int	nfssvc_nfsd(struct nfsd_srvargs *,user_addr_t, proc_t);
-static int	nfssvc_export(user_addr_t, proc_t);
-
-static int nfs_privport = 0;
-/* XXX CSM 11/25/97 Upgrade sysctl.h someday */
-#ifdef notyet
-SYSCTL_INT(_vfs_nfs, NFS_NFSPRIVPORT, nfs_privport, CTLFLAG_RW, &nfs_privport, 0, "");
-SYSCTL_INT(_vfs_nfs, OID_AUTO, gatherdelay, CTLFLAG_RW, &nfsrvw_procrastinate, 0, "");
-SYSCTL_INT(_vfs_nfs, OID_AUTO, gatherdelay_v3, CTLFLAG_RW, &nfsrvw_procrastinate_v3, 0, "");
+#if CONFIG_MACF
+#include <security/mac_framework.h>
 #endif
 
+kern_return_t	thread_terminate(thread_t); /* XXX */
+
+#if NFSSERVER
+
+extern int (*nfsrv_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
+					    struct nfsrv_sock *slp,
+					    vfs_context_t ctx,
+					    mbuf_t *mrepp);
+extern int nfsrv_wg_delay;
+extern int nfsrv_wg_delay_v3;
+
+static int nfsrv_require_resv_port = 0;
+static int nfsrv_deadsock_timer_on = 0;
+
+static int	nfssvc_addsock(socket_t, mbuf_t);
+static int	nfssvc_nfsd(void);
+static int	nfssvc_export(user_addr_t);
+
+static void	nfsrv_zapsock(struct nfsrv_sock *slp);
+static void	nfsrv_slpderef(struct nfsrv_sock *);
+static void	nfsrv_slpfree(struct nfsrv_sock *);
+
+#endif /* NFSSERVER */
+
+/*
+ * sysctl stuff
+ */
+SYSCTL_DECL(_vfs_generic);
+SYSCTL_NODE(_vfs_generic, OID_AUTO, nfs, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "nfs hinge");
+
+#if NFSCLIENT
+SYSCTL_NODE(_vfs_generic_nfs, OID_AUTO, client, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "nfs client hinge");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, initialdowndelay, CTLFLAG_RW, &nfs_tprintf_initial_delay, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, nextdowndelay, CTLFLAG_RW, &nfs_tprintf_delay, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, iosize, CTLFLAG_RW, &nfs_iosize, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, access_cache_timeout, CTLFLAG_RW, &nfs_access_cache_timeout, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, allow_async, CTLFLAG_RW, &nfs_allow_async, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, statfs_rate_limit, CTLFLAG_RW, &nfs_statfs_rate_limit, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, nfsiod_thread_max, CTLFLAG_RW, &nfsiod_thread_max, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, nfsiod_thread_count, CTLFLAG_RD, &nfsiod_thread_count, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, lockd_mounts, CTLFLAG_RD, &nfs_lockd_mounts, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_client, OID_AUTO, max_async_writes, CTLFLAG_RW, &nfs_max_async_writes, 0, "");
+#endif /* NFSCLIENT */
+
+#if NFSSERVER
+SYSCTL_NODE(_vfs_generic_nfs, OID_AUTO, server, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "nfs server hinge");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, wg_delay, CTLFLAG_RW, &nfsrv_wg_delay, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, wg_delay_v3, CTLFLAG_RW, &nfsrv_wg_delay_v3, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, require_resv_port, CTLFLAG_RW, &nfsrv_require_resv_port, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, async, CTLFLAG_RW, &nfsrv_async, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, reqcache_size, CTLFLAG_RW, &nfsrv_reqcache_size, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, request_queue_length, CTLFLAG_RW, &nfsrv_sock_max_rec_queue_length, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, user_stats, CTLFLAG_RW, &nfsrv_user_stat_enabled, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, fsevents, CTLFLAG_RW, &nfsrv_fsevents_enabled, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, nfsd_thread_max, CTLFLAG_RW, &nfsd_thread_max, 0, "");
+SYSCTL_INT(_vfs_generic_nfs_server, OID_AUTO, nfsd_thread_count, CTLFLAG_RD, &nfsd_thread_count, 0, "");
+#endif /* NFSSERVER */
+
+
+#if NFSCLIENT
+
+int
+nfsclnt(proc_t p, struct nfsclnt_args *uap, __unused int *retval)
+{
+	struct lockd_ans la;
+	int error;
+
+	if (uap->flag == NFSCLNT_LOCKDANS) {
+		error = copyin(uap->argp, &la, sizeof(la));
+		return (error != 0 ? error : nfslockdans(p, &la));
+	}
+	return EINVAL;
+}
+
+/*
+ * Asynchronous I/O threads for client NFS.
+ * They do read-ahead and write-behind operations on the block I/O cache.
+ *
+ * The pool of up to nfsiod_thread_max threads is launched on demand and exit
+ * when unused for a while.  There are as many nfsiod structs as there are
+ * nfsiod threads; however there's no strict tie between a thread and a struct.
+ * Each thread puts an nfsiod on the free list and sleeps on it.  When it wakes
+ * up, it removes the next struct nfsiod from the queue and services it.  Then
+ * it will put the struct at the head of free list and sleep on it.
+ * Async requests will pull the next struct nfsiod from the head of the free list,
+ * put it on the work queue, and wake whatever thread is waiting on that struct.
+ */
+static int nfsiod_continue(int);
+
+/*
+ * nfsiod thread exit routine
+ *
+ * Must be called with nfsiod_mutex held so that the
+ * decision to terminate is atomic with the termination.
+ */
+static void
+nfsiod_terminate(struct nfsiod *niod)
+{
+	nfsiod_thread_count--;
+	lck_mtx_unlock(nfsiod_mutex);
+	if (niod)
+		FREE(niod, M_TEMP);
+	else
+		printf("nfsiod: terminating without niod\n");
+	thread_terminate(current_thread());
+	/*NOTREACHED*/
+}
+
+/* nfsiod thread startup routine */
+static void
+nfsiod_thread(void)
+{
+	struct nfsiod *niod;
+	int error;
+
+	MALLOC(niod, struct nfsiod *, sizeof(struct nfsiod), M_TEMP, M_WAITOK);
+	if (!niod) {
+		lck_mtx_lock(nfsiod_mutex);
+		nfsiod_thread_count--;
+		lck_mtx_unlock(nfsiod_mutex);
+		thread_terminate(current_thread());
+		/*NOTREACHED*/
+	}
+	bzero(niod, sizeof(*niod));
+	lck_mtx_lock(nfsiod_mutex);
+	TAILQ_INSERT_HEAD(&nfsiodfree, niod, niod_link);
+	wakeup(current_thread());
+	error = msleep0(niod, nfsiod_mutex, PWAIT | PDROP, "nfsiod", NFS_ASYNCTHREADMAXIDLE*hz, nfsiod_continue);
+	/* shouldn't return... so we have an error */
+	/* remove an old nfsiod struct and terminate */
+	lck_mtx_lock(nfsiod_mutex);
+	if ((niod = TAILQ_LAST(&nfsiodfree, nfsiodlist)))
+		TAILQ_REMOVE(&nfsiodfree, niod, niod_link);
+	nfsiod_terminate(niod);
+	/*NOTREACHED*/
+}
+
+/*
+ * Start up another nfsiod thread.
+ * (unless we're already maxed out and there are nfsiods running)
+ */
+int
+nfsiod_start(void)
+{
+	thread_t thd;
+
+	lck_mtx_lock(nfsiod_mutex);
+	if ((nfsiod_thread_count >= NFSIOD_MAX) && (nfsiod_thread_count > 0)) {
+		lck_mtx_unlock(nfsiod_mutex);
+		return (EBUSY);
+	}
+	nfsiod_thread_count++;
+	thd = kernel_thread(kernel_task, nfsiod_thread);
+	/* wait for the thread to complete startup */
+	msleep(thd, nfsiod_mutex, PWAIT | PDROP, "nfsiodw", NULL);
+	return (0);
+}
+
+/*
+ * Continuation for Asynchronous I/O threads for NFS client.
+ *
+ * Grab an nfsiod struct to work on, do some work, then drop it
+ */
+static int
+nfsiod_continue(int error)
+{
+	struct nfsiod *niod;
+	struct nfsmount *nmp;
+	struct nfsreq *req, *treq;
+	struct nfs_reqqhead iodq;
+	int morework;
+
+	lck_mtx_lock(nfsiod_mutex);
+	niod = TAILQ_FIRST(&nfsiodwork);
+	if (!niod) {
+		/* there's no work queued up */
+		if (error != EWOULDBLOCK)
+			printf("nfsiod: error %d work %p\n", error, niod);
+		/* remove an old nfsiod struct and terminate */
+		if ((niod = TAILQ_LAST(&nfsiodfree, nfsiodlist)))
+			TAILQ_REMOVE(&nfsiodfree, niod, niod_link);
+		nfsiod_terminate(niod);
+		/*NOTREACHED*/
+	}
+	TAILQ_REMOVE(&nfsiodwork, niod, niod_link);
+
+worktodo:
+	while ((nmp = niod->niod_nmp)) {
+		/* 
+		 * Service this mount's async I/O queue.
+		 *
+		 * In order to ensure some level of fairness between mounts,
+		 * we grab all the work up front before processing it so any
+		 * new work that arrives will be serviced on a subsequent
+		 * iteration - and we have a chance to see if other work needs
+		 * to be done (e.g. the delayed write queue needs to be pushed
+		 * or other mounts are waiting for an nfsiod).
+		 */
+		/* grab the current contents of the queue */
+		TAILQ_INIT(&iodq);
+		TAILQ_CONCAT(&iodq, &nmp->nm_iodq, r_achain);
+		lck_mtx_unlock(nfsiod_mutex);
+
+		/* process the queue */
+		TAILQ_FOREACH_SAFE(req, &iodq, r_achain, treq) {
+			TAILQ_REMOVE(&iodq, req, r_achain);
+			req->r_achain.tqe_next = NFSREQNOLIST;
+			req->r_callback.rcb_func(req);
+		}
+
+		/* now check if there's more/other work to be done */
+		lck_mtx_lock(nfsiod_mutex);
+		morework = !TAILQ_EMPTY(&nmp->nm_iodq);
+		if (!morework || !TAILQ_EMPTY(&nfsiodmounts)) {
+			/* we're going to stop working on this mount */
+			if (morework) /* mount still needs more work so queue it up */
+				TAILQ_INSERT_TAIL(&nfsiodmounts, nmp, nm_iodlink);
+			nmp->nm_niod = NULL;
+			niod->niod_nmp = NULL;
+		}
+	}
+
+	/* loop if there's still a mount to work on */
+	if (!niod->niod_nmp && !TAILQ_EMPTY(&nfsiodmounts)) {
+		niod->niod_nmp = TAILQ_FIRST(&nfsiodmounts);
+		TAILQ_REMOVE(&nfsiodmounts, niod->niod_nmp, nm_iodlink);
+	}
+	if (niod->niod_nmp)
+		goto worktodo;
+
+	/* queue ourselves back up - if there aren't too many threads running */
+	if (nfsiod_thread_count <= NFSIOD_MAX) {
+		TAILQ_INSERT_HEAD(&nfsiodfree, niod, niod_link);
+		error = msleep0(niod, nfsiod_mutex, PWAIT | PDROP, "nfsiod", NFS_ASYNCTHREADMAXIDLE*hz, nfsiod_continue);
+		/* shouldn't return... so we have an error */
+		/* remove an old nfsiod struct and terminate */
+		lck_mtx_lock(nfsiod_mutex);
+		if ((niod = TAILQ_LAST(&nfsiodfree, nfsiodlist)))
+			TAILQ_REMOVE(&nfsiodfree, niod, niod_link);
+	}
+	nfsiod_terminate(niod);
+	/*NOTREACHED*/
+	return (0);
+}
+
+#endif /* NFSCLIENT */
+
+
+#if NFSSERVER
+
 /*
  * NFS server system calls
  * getfh() lives here too, but maybe should move to kern/vfs_syscalls.c
@@ -163,15 +384,11 @@ getfh(proc_t p, struct getfh_args *uap, __unused int *retval)
 	struct nfs_filehandle nfh;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
 	char path[MAXPATHLEN], *ptr;
 	u_int pathlen;
 	struct nfs_exportfs *nxfs;
 	struct nfs_export *nx;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	/*
 	 * Must be super user
 	 */
@@ -183,8 +400,11 @@ getfh(proc_t p, struct getfh_args *uap, __unused int *retval)
 	if (error)
 		return (error);
 
+	if (!nfsrv_is_initialized())
+		return (EINVAL);
+
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, 
-			UIO_SYSSPACE, path, &context);
+			UIO_SYSSPACE, CAST_USER_ADDR_T(path), vfs_context_current());
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -193,10 +413,10 @@ getfh(proc_t p, struct getfh_args *uap, __unused int *retval)
 	vp = nd.ni_vp;
 
 	// find exportfs that matches f_mntonname
-	lck_rw_lock_shared(&nfs_export_rwlock);
+	lck_rw_lock_shared(&nfsrv_export_rwlock);
 	ptr = vnode_mount(vp)->mnt_vfsstat.f_mntonname;
-	LIST_FOREACH(nxfs, &nfs_exports, nxfs_next) {
-		if (!strcmp(nxfs->nxfs_path, ptr))
+	LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next) {
+		if (!strncmp(nxfs->nxfs_path, ptr, MAXPATHLEN))
 			break;
 	}
 	if (!nxfs || strncmp(nxfs->nxfs_path, path, strlen(nxfs->nxfs_path))) {
@@ -225,15 +445,16 @@ getfh(proc_t p, struct getfh_args *uap, __unused int *retval)
 	nfh.nfh_xh.nxh_expid = htonl(nx->nx_id);
 	nfh.nfh_xh.nxh_flags = 0;
 	nfh.nfh_xh.nxh_reserved = 0;
-	nfh.nfh_len = NFS_MAX_FID_SIZE;
-	error = VFS_VPTOFH(vp, &nfh.nfh_len, &nfh.nfh_fid[0], NULL);
-	if (nfh.nfh_len > (int)NFS_MAX_FID_SIZE)
+	nfh.nfh_len = NFSV3_MAX_FID_SIZE;
+	error = VFS_VPTOFH(vp, (int*)&nfh.nfh_len, &nfh.nfh_fid[0], NULL);
+	if (nfh.nfh_len > (int)NFSV3_MAX_FID_SIZE)
 		error = EOVERFLOW;
 	nfh.nfh_xh.nxh_fidlen = nfh.nfh_len;
 	nfh.nfh_len += sizeof(nfh.nfh_xh);
+	nfh.nfh_fhp = (u_char*)&nfh.nfh_xh;
 
 out:
-	lck_rw_done(&nfs_export_rwlock);
+	lck_rw_done(&nfsrv_export_rwlock);
 	vnode_put(vp);
 	if (error)
 		return (error);
@@ -241,8 +462,6 @@ out:
 	return (error);
 }
 
-#endif /* NFS_NOSERVER */
-
 extern struct fileops vnops;
 
 /*
@@ -265,50 +484,44 @@ fhopen( proc_t p,
 	struct fileproc *fp, *nfp;
 	int fmode, error, type;
 	int indx;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	kauth_action_t action;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);
-
 	/*
 	 * Must be super user
 	 */
-	error = suser(context.vc_ucred, 0);
+	error = suser(vfs_context_ucred(ctx), 0);
 	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
 		return (error);
 	}
 
+	if (!nfsrv_is_initialized()) {
+		return (EINVAL);
+	}
+
 	fmode = FFLAGS(uap->flags);
 	/* why not allow a non-read/write open for our lockd? */
-	if (((fmode & (FREAD | FWRITE)) == 0) || (fmode & O_CREAT)) {
-		kauth_cred_unref(&context.vc_ucred);
+	if (((fmode & (FREAD | FWRITE)) == 0) || (fmode & O_CREAT))
 		return (EINVAL);
-	}
 
 	error = copyin(uap->u_fhp, &nfh.nfh_len, sizeof(nfh.nfh_len));
-	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
+	if (error)
 		return (error);
-	}
 	if ((nfh.nfh_len < (int)sizeof(struct nfs_exphandle)) ||
-	    (nfh.nfh_len > (int)NFS_MAX_FH_SIZE)) {
-		kauth_cred_unref(&context.vc_ucred);
+	    (nfh.nfh_len > (int)NFSV3_MAX_FH_SIZE))
 		return (EINVAL);
-	}
 	error = copyin(uap->u_fhp, &nfh, sizeof(nfh.nfh_len) + nfh.nfh_len);
-	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
+	if (error)
 		return (error);
-	}
+	nfh.nfh_fhp = (u_char*)&nfh.nfh_xh;
 
-	lck_rw_lock_shared(&nfs_export_rwlock);
+	lck_rw_lock_shared(&nfsrv_export_rwlock);
 	/* now give me my vnode, it gets returned to me with a reference */
-	error = nfsrv_fhtovp(&nfh, NULL, TRUE, &vp, &nx, &nxo);
-	lck_rw_done(&nfs_export_rwlock);
+	error = nfsrv_fhtovp(&nfh, NULL, &vp, &nx, &nxo);
+	lck_rw_done(&nfsrv_export_rwlock);
 	if (error) {
-		kauth_cred_unref(&context.vc_ucred);
+		if (error == NFSERR_TRYLATER)
+			error = EAGAIN; // XXX EBUSY? Or just leave as TRYLATER?
 		return (error);
 	}
 
@@ -339,10 +552,10 @@ fhopen( proc_t p,
 		action |= KAUTH_VNODE_READ_DATA;
 	if (fmode & (FWRITE | O_TRUNC))
 		action |= KAUTH_VNODE_WRITE_DATA;
-	if ((error = vnode_authorize(vp, NULL, action, &context)) != 0)
+	if ((error = vnode_authorize(vp, NULL, action, ctx)) != 0)
 		goto bad;
 
-	if ((error = VNOP_OPEN(vp, fmode, &context)))
+	if ((error = VNOP_OPEN(vp, fmode, ctx)))
 		goto bad;
 	if ((error = vnode_ref_ext(vp, fmode)))
 		goto bad;
@@ -352,8 +565,8 @@ fhopen( proc_t p,
 	 */
 
 	// starting here... error paths should call vn_close/vnode_put
-	if ((error = falloc(p, &nfp, &indx)) != 0) {
-		vn_close(vp, fmode & FMASK, context.vc_ucred, p);
+	if ((error = falloc(p, &nfp, &indx, ctx)) != 0) {
+		vn_close(vp, fmode & FMASK, ctx);
 		goto bad;
 	}
 	fp = nfp;
@@ -375,10 +588,13 @@ fhopen( proc_t p,
 		type = F_FLOCK;
 		if ((fmode & FNONBLOCK) == 0)
 			type |= F_WAIT;
-		if ((error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, type, &context))) {
-			vn_close(vp, fp->f_fglob->fg_flag, fp->f_fglob->fg_cred, p);
+		if ((error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, type, ctx))) {
+			struct vfs_context context = *vfs_context_current();
+			/* Modify local copy (to not damage thread copy) */
+			context.vc_ucred = fp->f_fglob->fg_cred;
+
+			vn_close(vp, fp->f_fglob->fg_flag, &context);
 			fp_free(p, indx, fp);
-			kauth_cred_unref(&context.vc_ucred);
 			return (error);
 		}
 		fp->f_fglob->fg_flag |= FHASLOCK;
@@ -392,40 +608,22 @@ fhopen( proc_t p,
 	proc_fdunlock(p);
 
 	*retval = indx;
-	kauth_cred_unref(&context.vc_ucred);
 	return (0);
 
 bad:
 	vnode_put(vp);
-	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
 /*
- * Nfs server psuedo system call for the nfsd's
- * Based on the flag value it either:
- * - adds a socket to the selection list
- * - remains in the kernel as an nfsd
- * - remains in the kernel as an nfsiod
+ * NFS server pseudo system call
  */
 int
 nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 {
-#ifndef NFS_NOSERVER
-	struct nameidata nd;
 	mbuf_t nam;
 	struct user_nfsd_args user_nfsdarg;
-	struct nfsd_srvargs nfsd_srvargs, *nsd = &nfsd_srvargs;
-	struct nfsd_cargs ncd;
-	struct nfsd *nfsd;
-	struct nfssvc_sock *slp;
-	struct nfsuid *nuidp;
-	struct nfsmount *nmp;
-	struct timeval now;
 	socket_t so;
-	struct vfs_context context;
-	struct ucred temp_cred;
-#endif /* NFS_NOSERVER */
 	int error;
 
 	AUDIT_ARG(cmd, uap->flag);
@@ -434,44 +632,18 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 	 * Must be super user
 	 */
 	error = proc_suser(p);
-	if(error)
+	if (error)
 		return (error);
-	if (uap->flag & NFSSVC_BIOD)
-		error = nfssvc_iod(p);
-#ifdef NFS_NOSERVER
-	else
-		error = ENXIO;
-#else /* !NFS_NOSERVER */
-	else if (uap->flag & NFSSVC_MNTD) {
-
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
-
-		error = copyin(uap->argp, (caddr_t)&ncd, sizeof (ncd));
-		if (error)
-			return (error);
-
-		NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1, 
-			(proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-			CAST_USER_ADDR_T(ncd.ncd_dirp), &context);
-		error = namei(&nd);
-		if (error)
-			return (error);
-		nameidone(&nd);
+#if CONFIG_MACF
+	error = mac_system_check_nfsd(kauth_cred_get());
+	if (error)
+		return (error);
+#endif
 
-		if (vnode_isvroot(nd.ni_vp) == 0)
-			error = EINVAL;
-		nmp = VFSTONFS(vnode_mount(nd.ni_vp));
-		vnode_put(nd.ni_vp);
-		if (error)
-			return (error);
+	/* make sure NFS server data structures have been initialized */
+	nfsrv_init();
 
-		if ((nmp->nm_state & NFSSTA_MNTD) &&
-			(uap->flag & NFSSVC_GOTAUTH) == 0)
-			return (0);
-		nmp->nm_state |= NFSSTA_MNTD;
-		error = nfskerb_clientd(nmp, &ncd, uap->flag, uap->argp, p);
-	} else if (uap->flag & NFSSVC_ADDSOCK) {
+	if (uap->flag & NFSSVC_ADDSOCK) {
 		if (IS_64BIT_PROCESS(p)) {
 			error = copyin(uap->argp, (caddr_t)&user_nfsdarg, sizeof(user_nfsdarg));
 		} else {
@@ -505,336 +677,114 @@ nfssvc(proc_t p, struct nfssvc_args *uap, __unused int *retval)
 		 * to keep the socket from being closed when nfsd closes its
 		 * file descriptor for it.
 		 */
-		error = nfssvc_addsock(so, nam, p);
+		error = nfssvc_addsock(so, nam);
 		/* drop the iocount file_socket() grabbed on the file descriptor */
 		file_drop(user_nfsdarg.sock);
 	} else if (uap->flag & NFSSVC_NFSD) {
-		error = copyin(uap->argp, (caddr_t)nsd, sizeof (*nsd));
-		if (error)
-			return (error);
-
-		if ((uap->flag & NFSSVC_AUTHIN) && ((nfsd = nsd->nsd_nfsd)) &&
-			(nfsd->nfsd_slp->ns_flag & SLP_VALID)) {
-			slp = nfsd->nfsd_slp;
-
-			/*
-			 * First check to see if another nfsd has already
-			 * added this credential.
-			 */
-			for (nuidp = NUIDHASH(slp,nsd->nsd_cr.cr_uid)->lh_first;
-			    nuidp != 0; nuidp = nuidp->nu_hash.le_next) {
-				if (kauth_cred_getuid(nuidp->nu_cr) == nsd->nsd_cr.cr_uid &&
-				    (!nfsd->nfsd_nd->nd_nam2 ||
-				     netaddr_match(NU_NETFAM(nuidp),
-				     &nuidp->nu_haddr, nfsd->nfsd_nd->nd_nam2)))
-					break;
-			}
-			if (nuidp) {
-			    nfsrv_setcred(nuidp->nu_cr,&temp_cred);
-			    kauth_cred_unref(&nfsd->nfsd_nd->nd_cr);
-			    nfsd->nfsd_nd->nd_cr = kauth_cred_create(&temp_cred);
-			    nfsd->nfsd_nd->nd_flag |= ND_KERBFULL;
-			} else {
-			    /*
-			     * Nope, so we will.
-			     */
-			    if (slp->ns_numuids < nuidhash_max) {
-				slp->ns_numuids++;
-				nuidp = (struct nfsuid *)
-				   _MALLOC_ZONE(sizeof (struct nfsuid),
-							M_NFSUID, M_WAITOK);
-			    } else
-				nuidp = (struct nfsuid *)0;
-			    if ((slp->ns_flag & SLP_VALID) == 0) {
-				if (nuidp) {
-				    FREE_ZONE((caddr_t)nuidp,
-					sizeof (struct nfsuid), M_NFSUID);
-				    slp->ns_numuids--;
-				}
-			    } else {
-				if (nuidp == (struct nfsuid *)0) {
-				    nuidp = slp->ns_uidlruhead.tqh_first;
-				    if (!nuidp)
-					return (ENOMEM);
-				    LIST_REMOVE(nuidp, nu_hash);
-				    TAILQ_REMOVE(&slp->ns_uidlruhead, nuidp,
-					nu_lru);
-				    if (nuidp->nu_flag & NU_NAM)
-					mbuf_freem(nuidp->nu_nam);
-				    kauth_cred_unref(&nuidp->nu_cr);
-				}
-				nuidp->nu_flag = 0;
-
-				if (nsd->nsd_cr.cr_ngroups > NGROUPS)
-				    nsd->nsd_cr.cr_ngroups = NGROUPS;
-
-				nfsrv_setcred(&nsd->nsd_cr, &temp_cred);
-				nuidp->nu_cr = kauth_cred_create(&temp_cred);
-
-				if (!nuidp->nu_cr) {
-					FREE_ZONE(nuidp, sizeof(struct nfsuid), M_NFSUID);
-					slp->ns_numuids--;
-					return (ENOMEM);
-				}
-				nuidp->nu_timestamp = nsd->nsd_timestamp;
-				microtime(&now);
-				nuidp->nu_expire = now.tv_sec + nsd->nsd_ttl;
-				/*
-				 * and save the session key in nu_key.
-				 */
-				bcopy(nsd->nsd_key, nuidp->nu_key,
-				    sizeof (nsd->nsd_key));
-				if (nfsd->nfsd_nd->nd_nam2) {
-				    struct sockaddr_in *saddr;
-
-				    saddr = mbuf_data(nfsd->nfsd_nd->nd_nam2);
-				    switch (saddr->sin_family) {
-				    case AF_INET:
-					nuidp->nu_flag |= NU_INETADDR;
-					nuidp->nu_inetaddr =
-					     saddr->sin_addr.s_addr;
-					break;
-				    case AF_ISO:
-				    default:
-					nuidp->nu_flag |= NU_NAM;
-					error = mbuf_copym(nfsd->nfsd_nd->nd_nam2, 0,
-							MBUF_COPYALL, MBUF_WAITOK,
-							&nuidp->nu_nam);
-					if (error) {
-						kauth_cred_unref(&nuidp->nu_cr);
-						FREE_ZONE(nuidp, sizeof(struct nfsuid), M_NFSUID);
-						slp->ns_numuids--;
-						return (error);
-					}
-					break;
-				    };
-				}
-				TAILQ_INSERT_TAIL(&slp->ns_uidlruhead, nuidp,
-					nu_lru);
-				LIST_INSERT_HEAD(NUIDHASH(slp, nsd->nsd_uid),
-					nuidp, nu_hash);
-				nfsrv_setcred(nuidp->nu_cr,&temp_cred);
-				kauth_cred_unref(&nfsd->nfsd_nd->nd_cr);
-				nfsd->nfsd_nd->nd_cr = kauth_cred_create(&temp_cred);
-				nfsd->nfsd_nd->nd_flag |= ND_KERBFULL;
-			    }
-			}
-		}
-		if ((uap->flag & NFSSVC_AUTHINFAIL) && (nfsd = nsd->nsd_nfsd))
-			nfsd->nfsd_flag |= NFSD_AUTHFAIL;
-		error = nfssvc_nfsd(nsd, uap->argp, p);
+		error = nfssvc_nfsd();
 	} else if (uap->flag & NFSSVC_EXPORT) {
-		error = nfssvc_export(uap->argp, p);
+		error = nfssvc_export(uap->argp);
 	} else {
 		error = EINVAL;
 	}
-#endif /* NFS_NOSERVER */
 	if (error == EINTR || error == ERESTART)
 		error = 0;
 	return (error);
 }
 
-/*
- * NFSKERB client helper daemon.
- * Gets authorization strings for "kerb" mounts.
- */
-static int
-nfskerb_clientd(
-	struct nfsmount *nmp,
-	struct nfsd_cargs *ncd,
-	int flag,
-	user_addr_t argp,
-	proc_t p)
-{
-	struct nfsuid *nuidp, *nnuidp;
-	int error = 0;
-	struct nfsreq *rp;
-	struct timeval now;
-
-	/*
-	 * First initialize some variables
-	 */
-	microtime(&now);
-
-	/*
-	 * If an authorization string is being passed in, get it.
-	 */
-	if ((flag & NFSSVC_GOTAUTH) && (nmp->nm_state & NFSSTA_MOUNTED) &&
-	    ((nmp->nm_state & NFSSTA_WAITAUTH) == 0)) {
-	    if (nmp->nm_state & NFSSTA_HASAUTH)
-		panic("cld kerb");
-	    if ((flag & NFSSVC_AUTHINFAIL) == 0) {
-		if (ncd->ncd_authlen <= nmp->nm_authlen &&
-		    ncd->ncd_verflen <= nmp->nm_verflen &&
-		    !copyin(CAST_USER_ADDR_T(ncd->ncd_authstr),nmp->nm_authstr,ncd->ncd_authlen)&&
-		    !copyin(CAST_USER_ADDR_T(ncd->ncd_verfstr),nmp->nm_verfstr,ncd->ncd_verflen)){
-		    nmp->nm_authtype = ncd->ncd_authtype;
-		    nmp->nm_authlen = ncd->ncd_authlen;
-		    nmp->nm_verflen = ncd->ncd_verflen;
-#if NFSKERB
-		    nmp->nm_key = ncd->ncd_key;
-#endif
-		} else
-		    nmp->nm_state |= NFSSTA_AUTHERR;
-	    } else
-		nmp->nm_state |= NFSSTA_AUTHERR;
-	    nmp->nm_state |= NFSSTA_HASAUTH;
-	    wakeup((caddr_t)&nmp->nm_authlen);
-	} else {
-	    nmp->nm_state |= NFSSTA_WAITAUTH;
-	}
-
-	/*
-	 * Loop every second updating queue until there is a termination sig.
-	 */
-	while (nmp->nm_state & NFSSTA_MOUNTED) {
-	    /* Get an authorization string, if required. */
-	    if ((nmp->nm_state & (NFSSTA_WAITAUTH | NFSSTA_HASAUTH)) == 0) {
-		ncd->ncd_authuid = nmp->nm_authuid;
-		if (copyout((caddr_t)ncd, argp, sizeof (struct nfsd_cargs)))
-			nmp->nm_state |= NFSSTA_WAITAUTH;
-		else
-			return (ENEEDAUTH);
-	    }
-	    /* Wait a bit (no pun) and do it again. */
-	    if ((nmp->nm_state & NFSSTA_MOUNTED) &&
-		(nmp->nm_state & (NFSSTA_WAITAUTH | NFSSTA_HASAUTH))) {
-		    error = tsleep((caddr_t)&nmp->nm_authstr, PSOCK | PCATCH,
-			"nfskrbtimr", hz / 3);
-		    if (error == EINTR || error == ERESTART)
-			    dounmount(nmp->nm_mountp, 0, NULL, 0, p);
-	    }
-	}
-
-	/*
-	 * Finally, we can free up the mount structure.
-	 */
-	for (nuidp = nmp->nm_uidlruhead.tqh_first; nuidp != 0; nuidp = nnuidp) {
-		nnuidp = nuidp->nu_lru.tqe_next;
-		LIST_REMOVE(nuidp, nu_hash);
-		TAILQ_REMOVE(&nmp->nm_uidlruhead, nuidp, nu_lru);
-		kauth_cred_unref(&nuidp->nu_cr);
-		FREE_ZONE((caddr_t)nuidp, sizeof (struct nfsuid), M_NFSUID);
-	}
-	/*
-	 * Loop through outstanding request list and remove dangling
-	 * references to defunct nfsmount struct
-	 */
-	for (rp = nfs_reqq.tqh_first; rp; rp = rp->r_chain.tqe_next)
-		if (rp->r_nmp == nmp)
-			rp->r_nmp = (struct nfsmount *)0;
-	/* Need to wake up any rcvlock waiters so they notice the unmount. */
-	if (nmp->nm_state & NFSSTA_WANTRCV) {
-		nmp->nm_state &= ~NFSSTA_WANTRCV;
-		wakeup(&nmp->nm_state);
-	}
-	FREE_ZONE((caddr_t)nmp, sizeof (struct nfsmount), M_NFSMNT);
-	if (error == EWOULDBLOCK)
-		error = 0;
-	return (error);
-}
-
-#ifndef NFS_NOSERVER
 /*
  * Adds a socket to the list for servicing by nfsds.
  */
 static int
-nfssvc_addsock(
-	socket_t so,
-	mbuf_t mynam,
-	__unused proc_t p)
+nfssvc_addsock(socket_t so, mbuf_t mynam)
 {
-	int siz;
-	struct nfssvc_sock *slp;
-	struct nfssvc_sock *tslp = NULL;
-	int error, sodomain, sotype, soprotocol, on = 1;
+	struct nfsrv_sock *slp;
+	int error = 0, sodomain, sotype, soprotocol, on = 1;
 	struct timeval timeo;
 
 	/* make sure mbuf constants are set up */
-	if (!nfs_mbuf_mlen)
+	if (!nfs_mbuf_mhlen)
 		nfs_mbuf_init();
 
 	sock_gettype(so, &sodomain, &sotype, &soprotocol);
 
-	/*
-	 * Add it to the list, as required.
-	 */
-	if (soprotocol == IPPROTO_UDP) {
-		tslp = nfs_udpsock;
-		if (!tslp || (tslp->ns_flag & SLP_VALID)) {
-			mbuf_freem(mynam);
-			return (EPERM);
-		}
-#if ISO
-	} else if (soprotocol == ISOPROTO_CLTP) {
-		tslp = nfs_cltpsock;
-		if (!tslp || (tslp->ns_flag & SLP_VALID)) {
-			mbuf_freem(mynam);
-			return (EPERM);
-		}
-#endif /* ISO */
-	}
-	/* reserve buffer space for 2 maximally-sized packets */
-	siz = NFS_MAXPACKET;
-	if (sotype == SOCK_STREAM)
-		siz += sizeof (u_long);
-	siz *= 2;
-	if (siz > NFS_MAXSOCKBUF)
-		siz = NFS_MAXSOCKBUF;
-	if ((error = sock_setsockopt(so, SOL_SOCKET, SO_SNDBUF, &siz, sizeof(siz))) ||
-	    (error = sock_setsockopt(so, SOL_SOCKET, SO_RCVBUF, &siz, sizeof(siz)))) {
+	/* There should be only one UDP socket */
+	if ((soprotocol == IPPROTO_UDP) && nfsrv_udpsock) {
 		mbuf_freem(mynam);
-		return (error);
+		return (EEXIST);
 	}
 
-	/*
-	 * Set protocol specific options { for now TCP only } and
-	 * reserve some space. For datagram sockets, this can get called
-	 * repeatedly for the same socket, but that isn't harmful.
-	 */
-	if (sotype == SOCK_STREAM) {
+	/* Set protocol options and reserve some space (for UDP). */
+	if (sotype == SOCK_STREAM)
 		sock_setsockopt(so, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on));
-	}
-	if (sodomain == AF_INET && soprotocol == IPPROTO_TCP) {
+	if ((sodomain == AF_INET) && (soprotocol == IPPROTO_TCP))
 		sock_setsockopt(so, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
+	if (sotype == SOCK_DGRAM) { /* set socket buffer sizes for UDP */
+		int reserve = NFS_UDPSOCKBUF;
+		error |= sock_setsockopt(so, SOL_SOCKET, SO_SNDBUF, &reserve, sizeof(reserve));
+		error |= sock_setsockopt(so, SOL_SOCKET, SO_RCVBUF, &reserve, sizeof(reserve));
+		if (error) {
+			log(LOG_INFO, "nfssvc_addsock: UDP socket buffer setting error(s) %d\n", error);
+			error = 0;
+		}
 	}
-
 	sock_nointerrupt(so, 0);
 
+	/*
+	 * Set socket send/receive timeouts.
+	 * Receive timeout shouldn't matter, but setting the send timeout
+	 * will make sure that an unresponsive client can't hang the server.
+	 */
 	timeo.tv_usec = 0;
-	timeo.tv_sec = 0;
-	error = sock_setsockopt(so, SOL_SOCKET, SO_RCVTIMEO, &timeo, sizeof(timeo));
-	error = sock_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO, &timeo, sizeof(timeo));
+	timeo.tv_sec = 1;
+	error |= sock_setsockopt(so, SOL_SOCKET, SO_RCVTIMEO, &timeo, sizeof(timeo));
+	timeo.tv_sec = 30;
+	error |= sock_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO, &timeo, sizeof(timeo));
+	if (error) {
+		log(LOG_INFO, "nfssvc_addsock: socket timeout setting error(s) %d\n", error);
+		error = 0;
+	}
 
-	if (tslp) {
-		slp = tslp;
-		lck_mtx_lock(nfsd_mutex);
-	} else {
-		MALLOC(slp, struct nfssvc_sock *, sizeof(struct nfssvc_sock),
-				M_NFSSVC, M_WAITOK);
-		if (!slp) {
+	MALLOC(slp, struct nfsrv_sock *, sizeof(struct nfsrv_sock), M_NFSSVC, M_WAITOK);
+	if (!slp) {
+		mbuf_freem(mynam);
+		return (ENOMEM);
+	}
+	bzero((caddr_t)slp, sizeof (struct nfsrv_sock));
+	lck_rw_init(&slp->ns_rwlock, nfsrv_slp_rwlock_group, LCK_ATTR_NULL);
+	lck_mtx_init(&slp->ns_wgmutex, nfsrv_slp_mutex_group, LCK_ATTR_NULL);
+
+	lck_mtx_lock(nfsd_mutex);
+
+	if (soprotocol == IPPROTO_UDP) {
+		/* There should be only one UDP socket */
+		if (nfsrv_udpsock) {
+			lck_mtx_unlock(nfsd_mutex);
+			nfsrv_slpfree(slp);
 			mbuf_freem(mynam);
-			return (ENOMEM);
+			return (EEXIST);
 		}
-		bzero((caddr_t)slp, sizeof (struct nfssvc_sock));
-		lck_rw_init(&slp->ns_rwlock, nfs_slp_rwlock_group, nfs_slp_lock_attr);
-		lck_mtx_init(&slp->ns_wgmutex, nfs_slp_mutex_group, nfs_slp_lock_attr);
-		TAILQ_INIT(&slp->ns_uidlruhead);
-		lck_mtx_lock(nfsd_mutex);
-		TAILQ_INSERT_TAIL(&nfssvc_sockhead, slp, ns_chain);
+		nfsrv_udpsock = slp;
 	}
 
+	/* add the socket to the list */
+	TAILQ_INSERT_TAIL(&nfsrv_socklist, slp, ns_chain);
+
 	sock_retain(so); /* grab a retain count on the socket */
 	slp->ns_so = so;
 	slp->ns_sotype = sotype;
 	slp->ns_nam = mynam;
 
+	/* set up the socket upcall */
 	socket_lock(so, 1);
 	so->so_upcallarg = (caddr_t)slp;
 	so->so_upcall = nfsrv_rcv;
-	so->so_rcv.sb_flags |= SB_UPCALL; /* required for freebsd merge */
+	so->so_rcv.sb_flags |= SB_UPCALL;
 	socket_unlock(so, 1);
 
+	/* mark that the socket is not in the nfsrv_sockwg list */
+	slp->ns_wgq.tqe_next = SLPNOLIST;
+	
 	slp->ns_flag = SLP_VALID | SLP_NEEDQ;
 
 	nfsrv_wakenfsd(slp);
@@ -844,88 +794,145 @@ nfssvc_addsock(
 }
 
 /*
- * Called by nfssvc() for nfsds. Just loops around servicing rpc requests
- * until it is killed by a signal.
+ * nfssvc_nfsd()
+ *
+ * nfsd theory of operation:
+ *
+ * The first nfsd thread stays in user mode accepting new TCP connections
+ * which are then added via the "addsock" call.  The rest of the nfsd threads
+ * simply call into the kernel and remain there in a loop handling NFS
+ * requests until killed by a signal.
+ * 
+ * There's a list of nfsd threads (nfsd_head).
+ * There's an nfsd queue that contains only those nfsds that are
+ *   waiting for work to do (nfsd_queue).
+ *
+ * There's a list of all NFS sockets (nfsrv_socklist) and two queues for
+ *   managing the work on the sockets:
+ *   nfsrv_sockwait - sockets w/new data waiting to be worked on
+ *   nfsrv_sockwork - sockets being worked on which may have more work to do
+ *   nfsrv_sockwg -- sockets which have pending write gather data
+ * When a socket receives data, if it is not currently queued, it
+ *   will be placed at the end of the "wait" queue.
+ * Whenever a socket needs servicing we make sure it is queued and
+ *   wake up a waiting nfsd (if there is one).
+ *
+ * nfsds will service at most 8 requests from the same socket before
+ *   defecting to work on another socket.
+ * nfsds will defect immediately if there are any sockets in the "wait" queue
+ * nfsds looking for a socket to work on check the "wait" queue first and
+ *   then check the "work" queue.
+ * When an nfsd starts working on a socket, it removes it from the head of
+ *   the queue it's currently on and moves it to the end of the "work" queue.
+ * When nfsds are checking the queues for work, any sockets found not to 
+ *   have any work are simply dropped from the queue.
+ *
  */
 static int
-nfssvc_nfsd(nsd, argp, p)
-	struct nfsd_srvargs *nsd;
-	user_addr_t argp;
-	proc_t p;
+nfssvc_nfsd(void)
 {
-	mbuf_t m, mreq;
-	struct nfssvc_sock *slp;
-	struct nfsd *nfsd = nsd->nsd_nfsd;
+	mbuf_t m, mrep;
+	struct nfsrv_sock *slp;
+	struct nfsd *nfsd;
 	struct nfsrv_descript *nd = NULL;
 	int error = 0, cacherep, writes_todo;
-	int siz, procrastinate;
+	int siz, procrastinate, opcnt = 0;
 	u_quad_t cur_usec;
 	struct timeval now;
-	boolean_t funnel_state;
+	struct vfs_context context;
 
 #ifndef nolint
 	cacherep = RC_DOIT;
 	writes_todo = 0;
 #endif
-	if (nfsd == (struct nfsd *)0) {
-		MALLOC(nfsd, struct nfsd *, sizeof(struct nfsd), M_NFSD, M_WAITOK);
-		if (!nfsd)
-			return (ENOMEM);
-		nsd->nsd_nfsd = nfsd;
-		bzero((caddr_t)nfsd, sizeof (struct nfsd));
-		nfsd->nfsd_procp = p;
-		lck_mtx_lock(nfsd_mutex);
-		TAILQ_INSERT_TAIL(&nfsd_head, nfsd, nfsd_chain);
-		nfs_numnfsd++;
-		lck_mtx_unlock(nfsd_mutex);
-	}
 
-	funnel_state = thread_funnel_set(kernel_flock, FALSE);
+	MALLOC(nfsd, struct nfsd *, sizeof(struct nfsd), M_NFSD, M_WAITOK);
+	if (!nfsd)
+		return (ENOMEM);
+	bzero(nfsd, sizeof(struct nfsd));
+	lck_mtx_lock(nfsd_mutex);
+	if (nfsd_thread_count++ == 0)
+		nfsrv_initcache();		/* Init the server request cache */
+	TAILQ_INSERT_TAIL(&nfsd_head, nfsd, nfsd_chain);
+	lck_mtx_unlock(nfsd_mutex);
+
+	context.vc_thread = current_thread();
 
 	/*
 	 * Loop getting rpc requests until SIGKILL.
 	 */
 	for (;;) {
-		if ((nfsd->nfsd_flag & NFSD_REQINPROG) == 0) {
+		if (nfsd_thread_max <= 0) {
+			/* NFS server shutting down, get out ASAP */
+			error = EINTR;
+			slp = nfsd->nfsd_slp;
+		} else if (nfsd->nfsd_flag & NFSD_REQINPROG) {
+			/* already have some work to do */
+			error = 0;
+			slp = nfsd->nfsd_slp;
+		} else {
+			/* need to find work to do */
+			error = 0;
 			lck_mtx_lock(nfsd_mutex);
-			while ((nfsd->nfsd_slp == NULL) && !(nfsd_head_flag & NFSD_CHECKSLP)) {
+			while (!nfsd->nfsd_slp && TAILQ_EMPTY(&nfsrv_sockwait) && TAILQ_EMPTY(&nfsrv_sockwork)) {
+				if (nfsd_thread_count > nfsd_thread_max) {
+					/*
+					 * If we have no socket and there are more
+					 * nfsd threads than configured, let's exit.
+					 */
+					error = 0;
+					goto done;
+				}
 				nfsd->nfsd_flag |= NFSD_WAITING;
-				nfsd_waiting++;
-				error = msleep(nfsd, nfsd_mutex, PSOCK | PCATCH, "nfsd", 0);
-				nfsd_waiting--;
+				TAILQ_INSERT_HEAD(&nfsd_queue, nfsd, nfsd_queue);
+				error = msleep(nfsd, nfsd_mutex, PSOCK | PCATCH, "nfsd", NULL);
 				if (error) {
-					lck_mtx_unlock(nfsd_mutex);
+					if (nfsd->nfsd_flag & NFSD_WAITING) {
+						TAILQ_REMOVE(&nfsd_queue, nfsd, nfsd_queue);
+						nfsd->nfsd_flag &= ~NFSD_WAITING;
+					}
 					goto done;
 				}
 			}
-			if ((nfsd->nfsd_slp == NULL) && (nfsd_head_flag & NFSD_CHECKSLP)) {
-				TAILQ_FOREACH(slp, &nfssvc_sockhead, ns_chain) {
-				    lck_rw_lock_shared(&slp->ns_rwlock);
-				    if ((slp->ns_flag & (SLP_VALID | SLP_DOREC))
-					== (SLP_VALID | SLP_DOREC)) {
-					    if (lck_rw_lock_shared_to_exclusive(&slp->ns_rwlock)) {
-						/* upgrade failed and we lost the lock; take exclusive and recheck */
-						lck_rw_lock_exclusive(&slp->ns_rwlock);
-						if ((slp->ns_flag & (SLP_VALID | SLP_DOREC))
-						    != (SLP_VALID | SLP_DOREC)) {
-						    /* flags no longer set, so skip this socket */
-						    lck_rw_done(&slp->ns_rwlock);
-						    continue;
-						}
-					    }
-					    slp->ns_flag &= ~SLP_DOREC;
-					    slp->ns_sref++;
-					    nfsd->nfsd_slp = slp;
-					    lck_rw_done(&slp->ns_rwlock);
-					    break;
-				    }
-				    lck_rw_done(&slp->ns_rwlock);
+			slp = nfsd->nfsd_slp;
+			if (!slp && !TAILQ_EMPTY(&nfsrv_sockwait)) {
+				/* look for a socket to work on in the wait queue */
+				while ((slp = TAILQ_FIRST(&nfsrv_sockwait))) {
+					lck_rw_lock_exclusive(&slp->ns_rwlock);
+					/* remove from the head of the queue */
+					TAILQ_REMOVE(&nfsrv_sockwait, slp, ns_svcq);
+					slp->ns_flag &= ~SLP_WAITQ;
+					if ((slp->ns_flag & SLP_VALID) && (slp->ns_flag & SLP_WORKTODO))
+						break;
+					/* nothing to do, so skip this socket */
+					lck_rw_done(&slp->ns_rwlock);
 				}
-				if (slp == 0)
-					nfsd_head_flag &= ~NFSD_CHECKSLP;
+			}
+			if (!slp && !TAILQ_EMPTY(&nfsrv_sockwork)) {
+				/* look for a socket to work on in the work queue */
+				while ((slp = TAILQ_FIRST(&nfsrv_sockwork))) {
+					lck_rw_lock_exclusive(&slp->ns_rwlock);
+					/* remove from the head of the queue */
+					TAILQ_REMOVE(&nfsrv_sockwork, slp, ns_svcq);
+					slp->ns_flag &= ~SLP_WORKQ;
+					if ((slp->ns_flag & SLP_VALID) && (slp->ns_flag & SLP_WORKTODO))
+						break;
+					/* nothing to do, so skip this socket */
+					lck_rw_done(&slp->ns_rwlock);
+				}
+			}
+			if (!nfsd->nfsd_slp && slp) {
+				/* we found a socket to work on, grab a reference */
+				slp->ns_sref++;
+				nfsd->nfsd_slp = slp;
+				opcnt = 0;
+				/* and put it at the back of the work queue */
+				TAILQ_INSERT_TAIL(&nfsrv_sockwork, slp, ns_svcq);
+				slp->ns_flag |= SLP_WORKQ;
+				lck_rw_done(&slp->ns_rwlock);
 			}
 			lck_mtx_unlock(nfsd_mutex);
-			if ((slp = nfsd->nfsd_slp) == NULL)
+			if (!slp)
 				continue;
 			lck_rw_lock_exclusive(&slp->ns_rwlock);
 			if (slp->ns_flag & SLP_VALID) {
@@ -936,37 +943,42 @@ nfssvc_nfsd(nsd, argp, p)
 				if (slp->ns_flag & SLP_DISCONN)
 					nfsrv_zapsock(slp);
 				error = nfsrv_dorec(slp, nfsd, &nd);
-				microuptime(&now);
-				cur_usec = (u_quad_t)now.tv_sec * 1000000 +
-					(u_quad_t)now.tv_usec;
-				if (error && slp->ns_wgtime && (slp->ns_wgtime <= cur_usec)) {
-					error = 0;
-					cacherep = RC_DOIT;
-					writes_todo = 1;
-				} else
-					writes_todo = 0;
+				if (error == EINVAL) {	// RPCSEC_GSS drop
+					if (slp->ns_sotype == SOCK_STREAM)
+						nfsrv_zapsock(slp); // drop connection
+				}
+				writes_todo = 0;
+				if (error && (slp->ns_wgtime || (slp->ns_flag & SLP_DOWRITES))) {
+					microuptime(&now);
+					cur_usec = (u_quad_t)now.tv_sec * 1000000 +
+						(u_quad_t)now.tv_usec;
+					if (slp->ns_wgtime <= cur_usec) {
+						error = 0;
+						cacherep = RC_DOIT;
+						writes_todo = 1;
+					}
+					slp->ns_flag &= ~SLP_DOWRITES;
+				}
 				nfsd->nfsd_flag |= NFSD_REQINPROG;
 			}
 			lck_rw_done(&slp->ns_rwlock);
-		} else {
-			error = 0;
-			slp = nfsd->nfsd_slp;
 		}
-		if (error || (slp->ns_flag & SLP_VALID) == 0) {
+		if (error || (slp && !(slp->ns_flag & SLP_VALID))) {
 			if (nd) {
-				if (nd->nd_mrep)
-					mbuf_freem(nd->nd_mrep);
+				nfsm_chain_cleanup(&nd->nd_nmreq);
 				if (nd->nd_nam2)
 					mbuf_freem(nd->nd_nam2);
 				if (IS_VALID_CRED(nd->nd_cr))
 					kauth_cred_unref(&nd->nd_cr);
-				FREE_ZONE((caddr_t)nd,
-						sizeof *nd, M_NFSRVDESC);
+				FREE_ZONE(nd, sizeof(*nd), M_NFSRVDESC);
 				nd = NULL;
 			}
 			nfsd->nfsd_slp = NULL;
 			nfsd->nfsd_flag &= ~NFSD_REQINPROG;
-			nfsrv_slpderef(slp);
+			if (slp)
+				nfsrv_slpderef(slp);
+			if (nfsd_thread_max <= 0)
+				break;
 			continue;
 		}
 		if (nd) {
@@ -976,33 +988,10 @@ nfssvc_nfsd(nsd, argp, p)
 		    else
 			nd->nd_nam = slp->ns_nam;
 
-		    /*
-		     * Check to see if authorization is needed.
-		     */
-		    if (nfsd->nfsd_flag & NFSD_NEEDAUTH) {
-			nfsd->nfsd_flag &= ~NFSD_NEEDAUTH;
-			nsd->nsd_haddr = ((struct sockaddr_in *)mbuf_data(nd->nd_nam))->sin_addr.s_addr;
-			nsd->nsd_authlen = nfsd->nfsd_authlen;
-			nsd->nsd_verflen = nfsd->nfsd_verflen;
-			if (!copyout(nfsd->nfsd_authstr,CAST_USER_ADDR_T(nsd->nsd_authstr),
-				nfsd->nfsd_authlen) &&
-			    !copyout(nfsd->nfsd_verfstr, CAST_USER_ADDR_T(nsd->nsd_verfstr),
-				nfsd->nfsd_verflen) &&
-			    !copyout((caddr_t)nsd, argp, sizeof (*nsd))) {
-			    thread_funnel_set(kernel_flock, funnel_state);
-			    return (ENEEDAUTH);
-			}
-			cacherep = RC_DROPIT;
-		    } else
-			cacherep = nfsrv_getcache(nd, slp, &mreq);
-
-		    if (nfsd->nfsd_flag & NFSD_AUTHFAIL) {
-			nfsd->nfsd_flag &= ~NFSD_AUTHFAIL;
-			nd->nd_procnum = NFSPROC_NOOP;
-			nd->nd_repstat = (NFSERR_AUTHERR | AUTH_TOOWEAK);
-			cacherep = RC_DOIT;
-		    } else if (nfs_privport) {
-			/* Check if source port is privileged */
+		    cacherep = nfsrv_getcache(nd, slp, &mrep);
+
+		    if (nfsrv_require_resv_port) {
+			/* Check if source port is a reserved port */
 			u_short port;
 			struct sockaddr *nam = mbuf_data(nd->nd_nam);
 			struct sockaddr_in *sin;
@@ -1024,42 +1013,68 @@ nfssvc_nfsd(nsd, argp, p)
 		}
 
 		/*
-		 * Loop to get all the write rpc relies that have been
+		 * Loop to get all the write RPC replies that have been
 		 * gathered together.
 		 */
 		do {
 		    switch (cacherep) {
 		    case RC_DOIT:
-			if (nd && (nd->nd_flag & ND_NFSV3))
-			    procrastinate = nfsrvw_procrastinate_v3;
+			if (nd && (nd->nd_vers == NFS_VER3))
+			    procrastinate = nfsrv_wg_delay_v3;
 			else
-			    procrastinate = nfsrvw_procrastinate;
-			lck_rw_lock_shared(&nfs_export_rwlock);
+			    procrastinate = nfsrv_wg_delay;
+			lck_rw_lock_shared(&nfsrv_export_rwlock);
+			context.vc_ucred = NULL;
 			if (writes_todo || ((nd->nd_procnum == NFSPROC_WRITE) && (procrastinate > 0)))
-			    error = nfsrv_writegather(&nd, slp, nfsd->nfsd_procp, &mreq);
-			else {
-			    error = (*(nfsrv3_procs[nd->nd_procnum]))(nd, slp, nfsd->nfsd_procp, &mreq);
-			    if (mreq == NULL)
-			    	nd->nd_mrep = NULL;
-			}
-			lck_rw_done(&nfs_export_rwlock);
-			if (mreq == NULL)
+				error = nfsrv_writegather(&nd, slp, &context, &mrep);
+			else
+				error = (*(nfsrv_procs[nd->nd_procnum]))(nd, slp, &context, &mrep);
+			lck_rw_done(&nfsrv_export_rwlock);
+			if (mrep == NULL) {
+				/*
+				 * If this is a stream socket and we are not going
+				 * to send a reply we better close the connection
+				 * so the client doesn't hang.
+				 */
+				if (error && slp->ns_sotype == SOCK_STREAM) {
+					lck_rw_lock_exclusive(&slp->ns_rwlock);
+					nfsrv_zapsock(slp);
+					lck_rw_done(&slp->ns_rwlock);
+					printf("NFS server: NULL reply from proc = %d error = %d\n",
+						nd->nd_procnum, error);
+				}
 				break;
+
+			}
 			if (error) {
 				OSAddAtomic(1, (SInt32*)&nfsstats.srv_errs);
-				nfsrv_updatecache(nd, FALSE, mreq);
+				nfsrv_updatecache(nd, FALSE, mrep);
 				if (nd->nd_nam2) {
 					mbuf_freem(nd->nd_nam2);
 					nd->nd_nam2 = NULL;
 				}
-				nd->nd_mrep = NULL;
 				break;
 			}
 			OSAddAtomic(1, (SInt32*)&nfsstats.srvrpccnt[nd->nd_procnum]);
-			nfsrv_updatecache(nd, TRUE, mreq);
-			nd->nd_mrep = NULL;
+			nfsrv_updatecache(nd, TRUE, mrep);
+			/* FALLTHRU */
+
 		    case RC_REPLY:
-			m = mreq;
+			if (nd->nd_gss_mb != NULL) {	// It's RPCSEC_GSS
+				/*
+				 * Need to checksum or encrypt the reply
+				 */
+				error = nfs_gss_svc_protect_reply(nd, mrep);
+				if (error) {
+				    	mbuf_freem(mrep);
+					break;
+				}
+			}
+
+			/*
+			 * Get the total size of the reply
+			 */
+			m = mrep;
 			siz = 0;
 			while (m) {
 				siz += mbuf_len(m);
@@ -1069,7 +1084,7 @@ nfssvc_nfsd(nsd, argp, p)
 				printf("mbuf siz=%d\n",siz);
 				panic("Bad nfs svc reply");
 			}
-			m = mreq;
+			m = mrep;
 			mbuf_pkthdr_setlen(m, siz);
 			error = mbuf_pkthdr_setrcvif(m, NULL);
 			if (error)
@@ -1085,7 +1100,7 @@ nfssvc_nfsd(nsd, argp, p)
 			}
 			if (!error) {
 				if (slp->ns_flag & SLP_VALID) {
-				    error = nfs_send(slp->ns_so, nd->nd_nam2, m, NULL);
+				    error = nfsrv_send(slp, nd->nd_nam2, m);
 				} else {
 				    error = EPIPE;
 				    mbuf_freem(m);
@@ -1093,46 +1108,39 @@ nfssvc_nfsd(nsd, argp, p)
 			} else {
 				mbuf_freem(m);
 			}
-			mreq = NULL;
-			if (nfsrtton)
-				nfsd_rt(slp->ns_sotype, nd, cacherep);
+			mrep = NULL;
 			if (nd->nd_nam2) {
 				mbuf_freem(nd->nd_nam2);
 				nd->nd_nam2 = NULL;
 			}
-			if (nd->nd_mrep) {
-				mbuf_freem(nd->nd_mrep);
-				nd->nd_mrep = NULL;
-			}
 			if (error == EPIPE) {
 				lck_rw_lock_exclusive(&slp->ns_rwlock);
 				nfsrv_zapsock(slp);
 				lck_rw_done(&slp->ns_rwlock);
 			}
 			if (error == EINTR || error == ERESTART) {
+				nfsm_chain_cleanup(&nd->nd_nmreq);
 				if (IS_VALID_CRED(nd->nd_cr))
 					kauth_cred_unref(&nd->nd_cr);
-				FREE_ZONE((caddr_t)nd, sizeof *nd, M_NFSRVDESC);
+				FREE_ZONE(nd, sizeof(*nd), M_NFSRVDESC);
 				nfsrv_slpderef(slp);
+				lck_mtx_lock(nfsd_mutex);
 				goto done;
 			}
 			break;
 		    case RC_DROPIT:
-			if (nfsrtton)
-				nfsd_rt(slp->ns_sotype, nd, cacherep);
-			mbuf_freem(nd->nd_mrep);
 			mbuf_freem(nd->nd_nam2);
-			nd->nd_mrep = nd->nd_nam2 = NULL;
+			nd->nd_nam2 = NULL;
 			break;
 		    };
+		    opcnt++;
 		    if (nd) {
-			if (nd->nd_mrep)
-				mbuf_freem(nd->nd_mrep);
+			nfsm_chain_cleanup(&nd->nd_nmreq);
 			if (nd->nd_nam2)
 				mbuf_freem(nd->nd_nam2);
 			if (IS_VALID_CRED(nd->nd_cr))
 				kauth_cred_unref(&nd->nd_cr);
-			FREE_ZONE((caddr_t)nd, sizeof *nd, M_NFSRVDESC);
+			FREE_ZONE(nd, sizeof(*nd), M_NFSRVDESC);
 			nd = NULL;
 		    }
 
@@ -1140,48 +1148,53 @@ nfssvc_nfsd(nsd, argp, p)
 		     * Check to see if there are outstanding writes that
 		     * need to be serviced.
 		     */
-		    microuptime(&now);
-		    cur_usec = (u_quad_t)now.tv_sec * 1000000 +
-			(u_quad_t)now.tv_usec;
-		    if (slp->ns_wgtime && (slp->ns_wgtime <= cur_usec)) {
-			cacherep = RC_DOIT;
-			writes_todo = 1;
-		    } else {
-			writes_todo = 0;
+		    writes_todo = 0;
+		    if (slp->ns_wgtime) {
+			microuptime(&now);
+			cur_usec = (u_quad_t)now.tv_sec * 1000000 +
+				(u_quad_t)now.tv_usec;
+			if (slp->ns_wgtime <= cur_usec) {
+			    cacherep = RC_DOIT;
+			    writes_todo = 1;
+			}
 		    }
 		} while (writes_todo);
-		lck_rw_lock_exclusive(&slp->ns_rwlock);
-		if (nfsrv_dorec(slp, nfsd, &nd)) {
+
+		nd = NULL;
+		if (TAILQ_EMPTY(&nfsrv_sockwait) && (opcnt < 8)) {
+			lck_rw_lock_exclusive(&slp->ns_rwlock);
+			error = nfsrv_dorec(slp, nfsd, &nd);
+			if (error == EINVAL) {	// RPCSEC_GSS drop
+				if (slp->ns_sotype == SOCK_STREAM)
+					nfsrv_zapsock(slp); // drop connection
+			}
 			lck_rw_done(&slp->ns_rwlock);
+		}
+		if (!nd) {
+			/* drop our reference on the socket */
 			nfsd->nfsd_flag &= ~NFSD_REQINPROG;
 			nfsd->nfsd_slp = NULL;
 			nfsrv_slpderef(slp);
-		} else {
-			lck_rw_done(&slp->ns_rwlock);
 		}
 	}
-done:
-	thread_funnel_set(kernel_flock, funnel_state);
 	lck_mtx_lock(nfsd_mutex);
+done:
 	TAILQ_REMOVE(&nfsd_head, nfsd, nfsd_chain);
 	FREE(nfsd, M_NFSD);
-	nsd->nsd_nfsd = (struct nfsd *)0;
-	if (--nfs_numnfsd == 0)
-		nfsrv_init(TRUE);	/* Reinitialize everything */
+	if (--nfsd_thread_count == 0)
+		nfsrv_cleanup();
 	lck_mtx_unlock(nfsd_mutex);
 	return (error);
 }
 
 static int
-nfssvc_export(user_addr_t argp, proc_t p)
+nfssvc_export(user_addr_t argp)
 {
 	int error = 0, is_64bit;
 	struct user_nfs_export_args unxa;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	is_64bit = IS_64BIT_PROCESS(p);
+	is_64bit = IS_64BIT_PROCESS(vfs_context_proc(ctx));
 
 	/* copy in pointers to path and export args */
 	if (is_64bit) {
@@ -1203,221 +1216,20 @@ nfssvc_export(user_addr_t argp, proc_t p)
 	if (error)
 		return (error);
 
-	error = nfsrv_export(&unxa, &context);
+	error = nfsrv_export(&unxa, ctx);
 
 	return (error);
 }
 
-#endif /* NFS_NOSERVER */
-
-int nfs_defect = 0;
-/* XXX CSM 11/25/97 Upgrade sysctl.h someday */
-#ifdef notyet
-SYSCTL_INT(_vfs_nfs, OID_AUTO, defect, CTLFLAG_RW, &nfs_defect, 0, "");
-#endif
-
-int
-nfsclnt(proc_t p, struct nfsclnt_args *uap, __unused int *retval)
-{
-	struct lockd_ans la;
-	int error;
-
-	if (uap->flag == NFSCLNT_LOCKDWAIT) {
-		return (nfslockdwait(p));
-	}
-	if (uap->flag == NFSCLNT_LOCKDANS) {
-		error = copyin(uap->argp, &la, sizeof(la));
-		return (error != 0 ? error : nfslockdans(p, &la));
-	}
-	if (uap->flag == NFSCLNT_LOCKDFD)
-		return (nfslockdfd(p, CAST_DOWN(int, uap->argp)));
-	return EINVAL;
-}
-
-
-static int nfssvc_iod_continue(int);
-
 /*
- * Asynchronous I/O daemons for client nfs.
- * They do read-ahead and write-behind operations on the block I/O cache.
- * Never returns unless it fails or gets killed.
- */
-static int
-nfssvc_iod(__unused proc_t p)
-{
-	register int i, myiod;
-	struct uthread *ut;
-
-	/*
-	 * Assign my position or return error if too many already running
-	 */
-	myiod = -1;
-	for (i = 0; i < NFS_MAXASYNCDAEMON; i++)
-		if (nfs_asyncdaemon[i] == 0) {
-			nfs_asyncdaemon[i]++;
-			myiod = i;
-			break;
-		}
-	if (myiod == -1)
-		return (EBUSY);
-	nfs_numasync++;
-
-	/* stuff myiod into uthread to get off local stack for continuation */
-
-	ut = (struct uthread *)get_bsdthread_info(current_thread());
-	ut->uu_state.uu_nfs_myiod = myiod;  /* squirrel away for continuation */
-
-	nfssvc_iod_continue(0);
-	/* NOTREACHED */
-	return (0);
-}
-
-/*
- * Continuation for Asynchronous I/O daemons for client nfs.
- */
-static int
-nfssvc_iod_continue(int error)
-{
-	register struct nfsbuf *bp;
-	register int i, myiod;
-	struct nfsmount *nmp;
-	struct uthread *ut;
-	proc_t p;
-	int exiterror = 0;
-
-	/*
-	 * real myiod is stored in uthread, recover it
-	 */
-	ut = (struct uthread *)get_bsdthread_info(current_thread());
-	myiod = ut->uu_state.uu_nfs_myiod;
-	p = current_proc(); // XXX
-
-	/*
-	 * Just loop around doin our stuff until SIGKILL
-	 *  - actually we don't loop with continuations...
-	 */
-	lck_mtx_lock(nfs_iod_mutex);
-	for (;;) {
-	    while (((nmp = nfs_iodmount[myiod]) == NULL
-		    || nmp->nm_bufq.tqh_first == NULL)
-		   && error == 0 && nfs_ioddelwri == 0) {
-		if (nmp)
-		    nmp->nm_bufqiods--;
-		nfs_iodwant[myiod] = p; // XXX this doesn't need to be a proc_t
-		nfs_iodmount[myiod] = NULL;
-		error = msleep0((caddr_t)&nfs_iodwant[myiod], nfs_iod_mutex,
-			PWAIT | PCATCH | PDROP, "nfsidl", 0, nfssvc_iod_continue);
-		lck_mtx_lock(nfs_iod_mutex);
-	    }
-	    if (error && !exiterror && nmp && (nmp->nm_bufqiods == 1) &&
-	        !TAILQ_EMPTY(&nmp->nm_bufq)) {
-		/*
-		 * Finish processing the queued buffers before exitting.
-		 * Decrement the iod count now to make sure nfs_asyncio()
-		 * doesn't keep queueing up more work.
-		 */
-		nmp->nm_bufqiods--;
-		exiterror = error;
-		error = 0;
-	    }
-	    if (error) {
-		nfs_asyncdaemon[myiod] = 0;
-		if (nmp && !exiterror)
-			nmp->nm_bufqiods--;
-		nfs_iodwant[myiod] = NULL;
-		nfs_iodmount[myiod] = NULL;
-		lck_mtx_unlock(nfs_iod_mutex);
-		nfs_numasync--;
-		if (error == EINTR || error == ERESTART)
-		  error = 0;
-		unix_syscall_return(error);
-	    }
-	    if (nmp != NULL) {
-		while ((bp = TAILQ_FIRST(&nmp->nm_bufq)) != NULL) {
-		    /* Take one off the front of the list */
-		    TAILQ_REMOVE(&nmp->nm_bufq, bp, nb_free);
-		    bp->nb_free.tqe_next = NFSNOLIST;
-		    nmp->nm_bufqlen--;
-		    if (nmp->nm_bufqwant && nmp->nm_bufqlen < 2 * nfs_numasync) {
-			nmp->nm_bufqwant = FALSE;
-			lck_mtx_unlock(nfs_iod_mutex);
-			wakeup(&nmp->nm_bufq);
-		    } else {
-			lck_mtx_unlock(nfs_iod_mutex);
-		    }
-
-		    SET(bp->nb_flags, NB_IOD);
-		    if (ISSET(bp->nb_flags, NB_READ))
-			nfs_doio(bp, bp->nb_rcred, NULL);
-		    else
-			nfs_doio(bp, bp->nb_wcred, NULL);
-
-		    lck_mtx_lock(nfs_iod_mutex);
-		    /*
-		     * If there are more than one iod on this mount, then defect
-		     * so that the iods can be shared out fairly between the mounts
-		     */
-		    if (!exiterror && nfs_defect && nmp->nm_bufqiods > 1) {
-			nfs_iodmount[myiod] = NULL;
-			nmp->nm_bufqiods--;
-			break;
-		    }
-		}
-	    }
-	    lck_mtx_unlock(nfs_iod_mutex);
-
-	    if (nfs_ioddelwri) {
-		i = 0;
-		nfs_ioddelwri = 0;
-		lck_mtx_lock(nfs_buf_mutex);
-		while (i < 8 && (bp = TAILQ_FIRST(&nfsbufdelwri)) != NULL) {
-			struct nfsnode *np = VTONFS(bp->nb_vp);
-			nfs_buf_remfree(bp);
-			nfs_buf_refget(bp);
-			while ((error = nfs_buf_acquire(bp, 0, 0, 0)) == EAGAIN);
-			nfs_buf_refrele(bp);
-			if (error)
-				break;
-			if (!bp->nb_vp) {
-				/* buffer is no longer valid */
-				nfs_buf_drop(bp);
-				continue;
-			}
-			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
-				nfs_buf_check_write_verifier(np, bp);
-			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
-				/* put buffer at end of delwri list */
-				TAILQ_INSERT_TAIL(&nfsbufdelwri, bp, nb_free);
-				nfsbufdelwricnt++;
-				nfs_buf_drop(bp);
-				lck_mtx_unlock(nfs_buf_mutex);
-				nfs_flushcommits(np->n_vnode, NULL, 1);
-			} else {
-				SET(bp->nb_flags, (NB_ASYNC | NB_IOD));
-				lck_mtx_unlock(nfs_buf_mutex);
-				nfs_buf_write(bp);
-			}
-			i++;
-			lck_mtx_lock(nfs_buf_mutex);
-		}
-		lck_mtx_unlock(nfs_buf_mutex);
-	    }
-
-	    lck_mtx_lock(nfs_iod_mutex);
-	    if (exiterror)
-	    	error = exiterror;
-	}
-}
-
-/*
- * Shut down a socket associated with an nfssvc_sock structure.
+ * Shut down a socket associated with an nfsrv_sock structure.
  * Should be called with the send lock set, if required.
  * The trick here is to increment the sref at the start, so that the nfsds
  * will stop using it and clear ns_flag at the end so that it will not be
  * reassigned during cleanup.
  */
 static void
-nfsrv_zapsock(struct nfssvc_sock *slp)
+nfsrv_zapsock(struct nfsrv_sock *slp)
 {
 	socket_t so;
 
@@ -1441,236 +1253,12 @@ nfsrv_zapsock(struct nfssvc_sock *slp)
 	sock_shutdown(so, SHUT_RDWR);
 }
 
-/*
- * Get an authorization string for the uid by having the mount_nfs sitting
- * on this mount point porpous out of the kernel and do it.
- */
-int
-nfs_getauth(nmp, rep, cred, auth_str, auth_len, verf_str, verf_len, key)
-	register struct nfsmount *nmp;
-	struct nfsreq *rep;
-	kauth_cred_t cred;
-	char **auth_str;
-	int *auth_len;
-	char *verf_str;
-	int *verf_len;
-	NFSKERBKEY_T key;		/* return session key */
-{
-	int error = 0;
-
-	while ((nmp->nm_state & NFSSTA_WAITAUTH) == 0) {
-		nmp->nm_state |= NFSSTA_WANTAUTH;
-		(void) tsleep((caddr_t)&nmp->nm_authtype, PSOCK,
-			"nfsauth1", 2 * hz);
-		error = nfs_sigintr(nmp, rep, rep->r_procp);
-		if (error) {
-			nmp->nm_state &= ~NFSSTA_WANTAUTH;
-			return (error);
-		}
-	}
-	nmp->nm_state &= ~NFSSTA_WANTAUTH;
-	MALLOC(*auth_str, char *, RPCAUTH_MAXSIZ, M_TEMP, M_WAITOK);
-	if (!*auth_str)
-		return (ENOMEM);
-	nmp->nm_authstr = *auth_str;
-	nmp->nm_authlen = RPCAUTH_MAXSIZ;
-	nmp->nm_verfstr = verf_str;
-	nmp->nm_verflen = *verf_len;
-	nmp->nm_authuid = kauth_cred_getuid(cred);
-	nmp->nm_state &= ~NFSSTA_WAITAUTH;
-	wakeup((caddr_t)&nmp->nm_authstr);
-
-	/*
-	 * And wait for mount_nfs to do its stuff.
-	 */
-	while ((nmp->nm_state & NFSSTA_HASAUTH) == 0 && error == 0) {
-		(void) tsleep((caddr_t)&nmp->nm_authlen, PSOCK,
-			"nfsauth2", 2 * hz);
-		error = nfs_sigintr(nmp, rep, rep->r_procp);
-	}
-	if (nmp->nm_state & NFSSTA_AUTHERR) {
-		nmp->nm_state &= ~NFSSTA_AUTHERR;
-		error = EAUTH;
-	}
-	if (error)
-		FREE(*auth_str, M_TEMP);
-	else {
-		*auth_len = nmp->nm_authlen;
-		*verf_len = nmp->nm_verflen;
-		bcopy((caddr_t)nmp->nm_key, (caddr_t)key, sizeof (key));
-	}
-	nmp->nm_state &= ~NFSSTA_HASAUTH;
-	nmp->nm_state |= NFSSTA_WAITAUTH;
-	if (nmp->nm_state & NFSSTA_WANTAUTH) {
-		nmp->nm_state &= ~NFSSTA_WANTAUTH;
-		wakeup((caddr_t)&nmp->nm_authtype);
-	}
-	return (error);
-}
-
-/*
- * Get a nickname authenticator and verifier.
- */
-int
-nfs_getnickauth(
-	struct nfsmount *nmp,
-	kauth_cred_t cred,
-	char **auth_str,
-	int *auth_len,
-	char *verf_str,
-	__unused int verf_len)
-{
-	register struct nfsuid *nuidp;
-	register u_long *nickp, *verfp;
-	struct timeval ktvin, ktvout, now;
-
-#if DIAGNOSTIC
-	if (verf_len < (4 * NFSX_UNSIGNED))
-		panic("nfs_getnickauth verf too small");
-#endif
-	for (nuidp = NMUIDHASH(nmp, kauth_cred_getuid(cred))->lh_first;
-	    nuidp != 0; nuidp = nuidp->nu_hash.le_next) {
-		if (kauth_cred_getuid(nuidp->nu_cr) == kauth_cred_getuid(cred))
-			break;
-	}
-	microtime(&now);
-	if (!nuidp || nuidp->nu_expire < now.tv_sec)
-		return (EACCES);
-
-	MALLOC(nickp, u_long *, 2 * NFSX_UNSIGNED, M_TEMP, M_WAITOK);
-	if (!nickp)
-		return (ENOMEM);
-
-	/*
-	 * Move to the end of the lru list (end of lru == most recently used).
-	 */
-	TAILQ_REMOVE(&nmp->nm_uidlruhead, nuidp, nu_lru);
-	TAILQ_INSERT_TAIL(&nmp->nm_uidlruhead, nuidp, nu_lru);
-
-	*nickp++ = txdr_unsigned(RPCAKN_NICKNAME);
-	*nickp = txdr_unsigned(nuidp->nu_nickname);
-	*auth_str = (char *)nickp;
-	*auth_len = 2 * NFSX_UNSIGNED;
-
-	/*
-	 * Now we must encrypt the verifier and package it up.
-	 */
-	verfp = (u_long *)verf_str;
-	*verfp++ = txdr_unsigned(RPCAKN_NICKNAME);
-	microtime(&now);
-	if (now.tv_sec > nuidp->nu_timestamp.tv_sec ||
-	    (now.tv_sec == nuidp->nu_timestamp.tv_sec &&
-	     now.tv_usec > nuidp->nu_timestamp.tv_usec))
-		nuidp->nu_timestamp = now;
-	else
-		nuidp->nu_timestamp.tv_usec++;
-	ktvin.tv_sec = txdr_unsigned(nuidp->nu_timestamp.tv_sec);
-	ktvin.tv_usec = txdr_unsigned(nuidp->nu_timestamp.tv_usec);
-
-	/*
-	 * Now encrypt the timestamp verifier in ecb mode using the session
-	 * key.
-	 */
-#if NFSKERB
-	XXX
-#endif
-
-	*verfp++ = ktvout.tv_sec;
-	*verfp++ = ktvout.tv_usec;
-	*verfp = 0;
-	return (0);
-}
-
-/*
- * Save the current nickname in a hash list entry on the mount point.
- */
-int
-nfs_savenickauth(nmp, cred, len, key, mdp, dposp, mrep)
-	register struct nfsmount *nmp;
-	kauth_cred_t cred;
-	int len;
-	NFSKERBKEY_T key;
-	mbuf_t *mdp;
-	char **dposp;
-	mbuf_t mrep;
-{
-	register struct nfsuid *nuidp;
-	register u_long *tl;
-	register long t1;
-	mbuf_t md = *mdp;
-	struct timeval ktvin, ktvout, now;
-	u_long nick;
-	char *dpos = *dposp, *cp2;
-	int deltasec, error = 0;
-
-	if (len == (3 * NFSX_UNSIGNED)) {
-		nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-		ktvin.tv_sec = *tl++;
-		ktvin.tv_usec = *tl++;
-		nick = fxdr_unsigned(u_long, *tl);
-
-		/*
-		 * Decrypt the timestamp in ecb mode.
-		 */
-#if NFSKERB
-		XXX
-#endif
-		ktvout.tv_sec = fxdr_unsigned(long, ktvout.tv_sec);
-		ktvout.tv_usec = fxdr_unsigned(long, ktvout.tv_usec);
-		microtime(&now);
-		deltasec = now.tv_sec - ktvout.tv_sec;
-		if (deltasec < 0)
-			deltasec = -deltasec;
-		/*
-		 * If ok, add it to the hash list for the mount point.
-		 */
-		if (deltasec <= NFS_KERBCLOCKSKEW) {
-			if (nmp->nm_numuids < nuidhash_max) {
-				nmp->nm_numuids++;
-				MALLOC_ZONE(nuidp, struct nfsuid *,
-						sizeof (struct nfsuid),
-							M_NFSUID, M_WAITOK);
-			} else {
-				nuidp = NULL;
-			}
-			if (!nuidp) {
-				nuidp = nmp->nm_uidlruhead.tqh_first;
-				if (!nuidp) {
-					error = ENOMEM;
-					goto nfsmout;
-				}
-				LIST_REMOVE(nuidp, nu_hash);
-				TAILQ_REMOVE(&nmp->nm_uidlruhead, nuidp, nu_lru);
-				kauth_cred_unref(&nuidp->nu_cr);
-			}
-			nuidp->nu_flag = 0;
-			kauth_cred_ref(cred);
-			nuidp->nu_cr = cred;
-			nuidp->nu_expire = now.tv_sec + NFS_KERBTTL;
-			nuidp->nu_timestamp = ktvout;
-			nuidp->nu_nickname = nick;
-			bcopy(key, nuidp->nu_key, sizeof (key));
-			TAILQ_INSERT_TAIL(&nmp->nm_uidlruhead, nuidp, nu_lru);
-			LIST_INSERT_HEAD(NMUIDHASH(nmp, kauth_cred_getuid(cred)),
-				nuidp, nu_hash);
-		}
-	} else
-		nfsm_adv(nfsm_rndup(len));
-nfsmout:
-	*mdp = md;
-	*dposp = dpos;
-	return (error);
-}
-
-#ifndef NFS_NOSERVER
-
 /*
  * cleanup and release a server socket structure.
  */
-void
-nfsrv_slpfree(struct nfssvc_sock *slp)
+static void
+nfsrv_slpfree(struct nfsrv_sock *slp)
 {
-	struct nfsuid *nuidp, *nnuidp;
 	struct nfsrv_descript *nwp, *nnwp;
 
 	if (slp->ns_so) {
@@ -1683,31 +1271,27 @@ nfsrv_slpfree(struct nfssvc_sock *slp)
 		mbuf_freem(slp->ns_raw);
 	if (slp->ns_rec)
 		mbuf_freem(slp->ns_rec);
-	slp->ns_nam = slp->ns_raw = slp->ns_rec = NULL;
-
-	for (nuidp = slp->ns_uidlruhead.tqh_first; nuidp != 0;
-	    nuidp = nnuidp) {
-		nnuidp = nuidp->nu_lru.tqe_next;
-		LIST_REMOVE(nuidp, nu_hash);
-		TAILQ_REMOVE(&slp->ns_uidlruhead, nuidp, nu_lru);
-		if (nuidp->nu_flag & NU_NAM)
-			mbuf_freem(nuidp->nu_nam);
-		kauth_cred_unref(&nuidp->nu_cr);
-		FREE_ZONE((caddr_t)nuidp,
-				sizeof (struct nfsuid), M_NFSUID);
-	}
+	if (slp->ns_frag)
+		mbuf_freem(slp->ns_frag);
+	slp->ns_nam = slp->ns_raw = slp->ns_rec = slp->ns_frag = NULL;
+	slp->ns_reccnt = 0;
 
 	for (nwp = slp->ns_tq.lh_first; nwp; nwp = nnwp) {
 		nnwp = nwp->nd_tq.le_next;
 		LIST_REMOVE(nwp, nd_tq);
+		nfsm_chain_cleanup(&nwp->nd_nmreq);
+		if (nwp->nd_mrep)
+			mbuf_freem(nwp->nd_mrep);
+		if (nwp->nd_nam2)
+			mbuf_freem(nwp->nd_nam2);
 		if (IS_VALID_CRED(nwp->nd_cr))
 			kauth_cred_unref(&nwp->nd_cr);
-		FREE_ZONE((caddr_t)nwp, sizeof *nwp, M_NFSRVDESC);
+		FREE_ZONE(nwp, sizeof(*nwp), M_NFSRVDESC);
 	}
 	LIST_INIT(&slp->ns_tq);
 
-	lck_rw_destroy(&slp->ns_rwlock, nfs_slp_rwlock_group);
-	lck_mtx_destroy(&slp->ns_wgmutex, nfs_slp_mutex_group);
+	lck_rw_destroy(&slp->ns_rwlock, nfsrv_slp_rwlock_group);
+	lck_mtx_destroy(&slp->ns_wgmutex, nfsrv_slp_mutex_group);
 	FREE(slp, M_NFSSVC);
 }
 
@@ -1716,142 +1300,171 @@ nfsrv_slpfree(struct nfssvc_sock *slp)
  * is no longer valid, you can throw it away.
  */
 void
-nfsrv_slpderef(struct nfssvc_sock *slp)
+nfsrv_slpderef(struct nfsrv_sock *slp)
 {
 	struct timeval now;
 
 	lck_mtx_lock(nfsd_mutex);
 	lck_rw_lock_exclusive(&slp->ns_rwlock);
 	slp->ns_sref--;
+
 	if (slp->ns_sref || (slp->ns_flag & SLP_VALID)) {
+		if ((slp->ns_flag & SLP_QUEUED) && !(slp->ns_flag & SLP_WORKTODO)) {
+			/* remove socket from queue since there's no work */
+			if (slp->ns_flag & SLP_WAITQ)
+				TAILQ_REMOVE(&nfsrv_sockwait, slp, ns_svcq);
+			else
+				TAILQ_REMOVE(&nfsrv_sockwork, slp, ns_svcq);
+			slp->ns_flag &= ~SLP_QUEUED;
+		}
 		lck_rw_done(&slp->ns_rwlock);
 		lck_mtx_unlock(nfsd_mutex);
 		return;
 	}
 
-	/* queue the socket up for deletion */
+	/* This socket is no longer valid, so we'll get rid of it */
+
+	if (slp->ns_flag & SLP_QUEUED) {
+		if (slp->ns_flag & SLP_WAITQ)
+			TAILQ_REMOVE(&nfsrv_sockwait, slp, ns_svcq);
+		else
+			TAILQ_REMOVE(&nfsrv_sockwork, slp, ns_svcq);
+		slp->ns_flag &= ~SLP_QUEUED;
+	}
+
+	/*
+	 * Queue the socket up for deletion
+	 * and start the timer to delete it
+	 * after it has been in limbo for
+	 * a while.
+	 */
 	microuptime(&now);
 	slp->ns_timestamp = now.tv_sec;
-	TAILQ_REMOVE(&nfssvc_sockhead, slp, ns_chain);
-	TAILQ_INSERT_TAIL(&nfssvc_deadsockhead, slp, ns_chain);
+	TAILQ_REMOVE(&nfsrv_socklist, slp, ns_chain);
+	TAILQ_INSERT_TAIL(&nfsrv_deadsocklist, slp, ns_chain);
+	if (!nfsrv_deadsock_timer_on) {
+		nfsrv_deadsock_timer_on = 1;
+		nfs_interval_timer_start(nfsrv_deadsock_timer_call,
+				NFSRV_DEADSOCKDELAY * 1000);
+	}
+
 	lck_rw_done(&slp->ns_rwlock);
-	if (slp == nfs_udpsock)
-		nfs_udpsock = NULL;
-#if ISO
-	else if (slp == nfs_cltpsock)
-		nfs_cltpsock = NULL;
-#endif
+	/* now remove from the write gather socket list */ 
+	if (slp->ns_wgq.tqe_next != SLPNOLIST) {
+		TAILQ_REMOVE(&nfsrv_sockwg, slp, ns_wgq);
+		slp->ns_wgq.tqe_next = SLPNOLIST;
+	}
 	lck_mtx_unlock(nfsd_mutex);
 }
 
-
 /*
- * Initialize the data structures for the server.
- * Handshake with any new nfsds starting up to avoid any chance of
- * corruption.
+ * Check periodically for dead sockets pending delete.
+ * If a socket has been dead for more than NFSRV_DEADSOCKDELAY
+ * seconds then we assume it's safe to free.
  */
 void
-nfsrv_init(terminating)
-	int terminating;
+nfsrv_deadsock_timer(__unused void *param0, __unused void *param1)
 {
-	struct nfssvc_sock *slp, *nslp;
+	struct nfsrv_sock *slp;
 	struct timeval now;
+	time_t time_to_wait;
 
-	if (terminating) {
-		microuptime(&now);
-		for (slp = TAILQ_FIRST(&nfssvc_sockhead); slp != 0; slp = nslp) {
-			nslp = TAILQ_NEXT(slp, ns_chain);
-			if (slp->ns_flag & SLP_VALID) {
-				lck_rw_lock_exclusive(&slp->ns_rwlock);
-				nfsrv_zapsock(slp);
-				lck_rw_done(&slp->ns_rwlock);
-			}
-			/* queue the socket up for deletion */
-			slp->ns_timestamp = now.tv_sec;
-			TAILQ_REMOVE(&nfssvc_sockhead, slp, ns_chain);
-			TAILQ_INSERT_TAIL(&nfssvc_deadsockhead, slp, ns_chain);
-			if (slp == nfs_udpsock)
-				nfs_udpsock = NULL;
-#if ISO
-			else if (slp == nfs_cltpsock)
-				nfs_cltpsock = NULL;
-#endif
-		}
-		nfsrv_cleancache();	/* And clear out server cache */
-/* XXX Revisit when enabling WebNFS */
-#ifdef WEBNFS_ENABLED
-	} else
-		nfs_pub.np_valid = 0;
-#else
-	}
-#endif
+	microuptime(&now);
+	lck_mtx_lock(nfsd_mutex);
 
-	if (!terminating) {
-		TAILQ_INIT(&nfssvc_sockhead);
-		TAILQ_INIT(&nfssvc_deadsockhead);
-		TAILQ_INIT(&nfsd_head);
-		nfsd_head_flag &= ~NFSD_CHECKSLP;
+	while ((slp = TAILQ_FIRST(&nfsrv_deadsocklist))) {
+		if ((slp->ns_timestamp + NFSRV_DEADSOCKDELAY) > now.tv_sec)
+			break;
+		TAILQ_REMOVE(&nfsrv_deadsocklist, slp, ns_chain);
+		nfsrv_slpfree(slp);
 	}
-
-	MALLOC(nfs_udpsock, struct nfssvc_sock *, sizeof(struct nfssvc_sock),
-			M_NFSSVC, M_WAITOK);
-	if (nfs_udpsock) {
-		bzero((caddr_t)nfs_udpsock, sizeof (struct nfssvc_sock));
-		lck_rw_init(&nfs_udpsock->ns_rwlock, nfs_slp_rwlock_group, nfs_slp_lock_attr);
-		lck_mtx_init(&nfs_udpsock->ns_wgmutex, nfs_slp_mutex_group, nfs_slp_lock_attr);
-		TAILQ_INIT(&nfs_udpsock->ns_uidlruhead);
-		TAILQ_INSERT_HEAD(&nfssvc_sockhead, nfs_udpsock, ns_chain);
-	} else {
-		printf("nfsrv_init() failed to allocate UDP socket\n");
+	if (TAILQ_EMPTY(&nfsrv_deadsocklist)) {
+		nfsrv_deadsock_timer_on = 0;
+		lck_mtx_unlock(nfsd_mutex);
+		return;
 	}
+	time_to_wait = (slp->ns_timestamp + NFSRV_DEADSOCKDELAY) - now.tv_sec;
+	if (time_to_wait < 1)
+		time_to_wait = 1;
 
-#if ISO
-	MALLOC(nfs_cltpsock, struct nfssvc_sock *, sizeof(struct nfssvc_sock),
-			M_NFSSVC, M_WAITOK);
-	if (nfs_cltpsock) {
-		bzero((caddr_t)nfs_cltpsock, sizeof (struct nfssvc_sock));
-		lck_rw_init(&nfs_cltpsock->ns_rwlock, nfs_slp_rwlock_group, nfs_slp_lock_attr);
-		lck_mtx_init(&nfs_cltpsock->ns_wgmutex, nfs_slp_mutex_group, nfs_slp_lock_attr);
-		TAILQ_INIT(&nfs_cltpsock->ns_uidlruhead);
-		TAILQ_INSERT_TAIL(&nfssvc_sockhead, nfs_cltpsock, ns_chain);
-	} else {
-		printf("nfsrv_init() failed to allocate CLTP socket\n");
-	}
-#endif
+	lck_mtx_unlock(nfsd_mutex);
+
+	nfs_interval_timer_start(nfsrv_deadsock_timer_call,
+		time_to_wait * 1000);
 }
 
 /*
- * Add entries to the server monitor log.
+ * Clean up the data structures for the server.
  */
-static void
-nfsd_rt(sotype, nd, cacherep)
-	int sotype;
-	register struct nfsrv_descript *nd;
-	int cacherep;
+void
+nfsrv_cleanup(void)
 {
-	register struct drt *rt;
+	struct nfsrv_sock *slp, *nslp;
 	struct timeval now;
+	struct nfsrv_fmod *fp, *nfp;
+	int i;
 
-	rt = &nfsdrt.drt[nfsdrt.pos];
-	if (cacherep == RC_DOIT)
-		rt->flag = 0;
-	else if (cacherep == RC_REPLY)
-		rt->flag = DRT_CACHEREPLY;
-	else
-		rt->flag = DRT_CACHEDROP;
-	if (sotype == SOCK_STREAM)
-		rt->flag |= DRT_TCP;
-	else if (nd->nd_flag & ND_NFSV3)
-		rt->flag |= DRT_NFSV3;
-	rt->proc = nd->nd_procnum;
-	if (((struct sockaddr *)mbuf_data(nd->nd_nam))->sa_family == AF_INET)
-	    rt->ipadr = ((struct sockaddr_in *)mbuf_data(nd->nd_nam))->sin_addr.s_addr;
-	else
-	    rt->ipadr = INADDR_ANY;
 	microuptime(&now);
-	rt->resptime = ((now.tv_sec - nd->nd_starttime.tv_sec) * 1000000) +
-		(now.tv_usec - nd->nd_starttime.tv_usec);
-	microtime(&rt->tstamp); // XXX unused
-	nfsdrt.pos = (nfsdrt.pos + 1) % NFSRTTLOGSIZ;
+	for (slp = TAILQ_FIRST(&nfsrv_socklist); slp != 0; slp = nslp) {
+		nslp = TAILQ_NEXT(slp, ns_chain);
+		if (slp->ns_flag & SLP_VALID) {
+			lck_rw_lock_exclusive(&slp->ns_rwlock);
+			nfsrv_zapsock(slp);
+			lck_rw_done(&slp->ns_rwlock);
+		}
+		if (slp->ns_flag & SLP_QUEUED) {
+			if (slp->ns_flag & SLP_WAITQ)
+				TAILQ_REMOVE(&nfsrv_sockwait, slp, ns_svcq);
+			else
+				TAILQ_REMOVE(&nfsrv_sockwork, slp, ns_svcq);
+			slp->ns_flag &= ~SLP_QUEUED;
+		}
+		if (slp->ns_wgq.tqe_next != SLPNOLIST) {
+			TAILQ_REMOVE(&nfsrv_sockwg, slp, ns_wgq);
+			slp->ns_wgq.tqe_next = SLPNOLIST;
+		}
+		/* queue the socket up for deletion */
+		slp->ns_timestamp = now.tv_sec;
+		TAILQ_REMOVE(&nfsrv_socklist, slp, ns_chain);
+		TAILQ_INSERT_TAIL(&nfsrv_deadsocklist, slp, ns_chain);
+		if (!nfsrv_deadsock_timer_on) {
+			nfsrv_deadsock_timer_on = 1;
+			nfs_interval_timer_start(nfsrv_deadsock_timer_call,
+				NFSRV_DEADSOCKDELAY * 1000);
+		}
+	}
+
+	/*
+	 * Flush pending file write fsevents
+	 */
+	lck_mtx_lock(nfsrv_fmod_mutex);
+	for (i = 0; i < NFSRVFMODHASHSZ; i++) {
+		for (fp = LIST_FIRST(&nfsrv_fmod_hashtbl[i]); fp; fp = nfp) {
+			/*
+			 * Fire off the content modified fsevent for each
+			 * entry, remove it from the list, and free it.
+			 */
+#if CONFIG_FSE
+			if (nfsrv_fsevents_enabled)
+				add_fsevent(FSE_CONTENT_MODIFIED, &fp->fm_context,
+						FSE_ARG_VNODE, fp->fm_vp,
+						FSE_ARG_DONE);
+#endif
+			vnode_put(fp->fm_vp);
+			kauth_cred_unref(&fp->fm_context.vc_ucred);
+			nfp = LIST_NEXT(fp, fm_link);
+			LIST_REMOVE(fp, fm_link);
+			FREE(fp, M_TEMP);
+		}
+	}
+	nfsrv_fmod_pending = 0;
+	lck_mtx_unlock(nfsrv_fmod_mutex);
+
+	nfs_gss_svc_cleanup();	/* Remove any RPCSEC_GSS contexts */
+
+	nfsrv_cleancache();	/* And clear out server cache */
+
+	nfsrv_udpsock = NULL;
 }
+
 #endif /* NFS_NOSERVER */
diff --git a/bsd/nfs/nfs_vfsops.c b/bsd/nfs/nfs_vfsops.c
index d8e5cb69d..7e2e1ee83 100644
--- a/bsd/nfs/nfs_vfsops.c
+++ b/bsd/nfs/nfs_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,6 +64,12 @@
  *	@(#)nfs_vfsops.c	8.12 (Berkeley) 5/20/95
  * FreeBSD-Id: nfs_vfsops.c,v 1.52 1997/11/12 05:42:21 julian Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -75,6 +87,7 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/fcntl.h>
+#include <sys/quota.h>
 #include <libkern/OSAtomic.h>
 
 #include <sys/vm.h>
@@ -89,209 +102,376 @@
 #include <netinet/in.h>
 
 #include <nfs/rpcv2.h>
+#include <nfs/krpc.h>
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 #include <nfs/nfsnode.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/xdr_subs.h>
 #include <nfs/nfsm_subs.h>
 #include <nfs/nfsdiskless.h>
 #include <nfs/nfs_lock.h>
-
-extern int	nfs_mountroot(void);
-
-extern int	nfs_ticks;
-extern int	nfs_mount_type;
-extern int	nfs_resv_mounts;
-
-struct nfsstats	nfsstats;
-static int nfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
-/* XXX CSM 11/25/97 Upgrade sysctl.h someday */
-#ifdef notyet
-SYSCTL_NODE(_vfs, MOUNT_NFS, nfs, CTLFLAG_RW, 0, "NFS filesystem");
-SYSCTL_STRUCT(_vfs_nfs, NFS_NFSSTATS, nfsstats, CTLFLAG_RD,
-	&nfsstats, nfsstats, "");
+#if CONFIG_MACF
+#include <security/mac_framework.h>
 #endif
 
-SYSCTL_DECL(_vfs_generic_nfs);
-SYSCTL_NODE(_vfs_generic_nfs, OID_AUTO, client, CTLFLAG_RW, 0,
-    "nfs client hinge");
-/* how long NFS will wait before signalling vfs that it's down. */
-static int nfs_tprintf_initial_delay = NFS_TPRINTF_INITIAL_DELAY;
-SYSCTL_INT(_vfs_generic_nfs_client, NFS_TPRINTF_INITIAL_DELAY,
-    initialdowndelay, CTLFLAG_RW, &nfs_tprintf_initial_delay, 0, "");
-/* how long between console messages "nfs server foo not responding" */
-static int nfs_tprintf_delay = NFS_TPRINTF_DELAY;
-SYSCTL_INT(_vfs_generic_nfs_client, NFS_TPRINTF_DELAY,
-    nextdowndelay, CTLFLAG_RW, &nfs_tprintf_delay, 0, "");
-
-static int	nfs_biosize(struct nfsmount *);
-static int	mountnfs(struct user_nfs_args *,mount_t,mbuf_t,proc_t,vnode_t *);
-static int	nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context);
-static int	nfs_start(mount_t mp, int flags, vfs_context_t context);
-static int	nfs_unmount(mount_t mp, int mntflags, vfs_context_t context);
-static int	nfs_root(mount_t mp, vnode_t *vpp, vfs_context_t context);
-static int	nfs_statfs(mount_t mp, struct vfsstatfs *sbp, vfs_context_t context);
-static int	nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context);
-static int	nfs_sync( mount_t mp, int waitfor, vfs_context_t context);
-static int	nfs_vptofh(vnode_t vp, int *fhlenp, unsigned char *fhp, vfs_context_t context);
-static int	nfs_fhtovp(mount_t mp, int fhlen, unsigned char *fhp, vnode_t *vpp, vfs_context_t context);
-static int	nfs_vget(mount_t , ino64_t, vnode_t *, vfs_context_t context);
+#include <pexpert/pexpert.h>
 
+/*
+ * NFS client globals
+ */
+
+int nfs_ticks;
+static lck_grp_t *nfs_mount_grp;
+uint32_t nfs_fs_attr_bitmap[NFS_ATTR_BITMAP_LEN];
+uint32_t nfs_object_attr_bitmap[NFS_ATTR_BITMAP_LEN];
+uint32_t nfs_getattr_bitmap[NFS_ATTR_BITMAP_LEN];
+
+/* NFS requests */
+struct nfs_reqqhead nfs_reqq;
+lck_grp_t *nfs_request_grp;
+lck_mtx_t *nfs_request_mutex;
+thread_call_t nfs_request_timer_call;
+int nfs_request_timer_on;
+u_long nfs_xid = 0;
+u_long nfs_xidwrap = 0;		/* to build a (non-wrapping) 64 bit xid */
+
+thread_call_t nfs_buf_timer_call;
+
+/* nfsiod */
+lck_grp_t *nfsiod_lck_grp;
+lck_mtx_t *nfsiod_mutex;
+struct nfsiodlist nfsiodfree, nfsiodwork;
+struct nfsiodmountlist nfsiodmounts;
+int nfsiod_thread_count = 0;
+int nfsiod_thread_max = NFS_DEFASYNCTHREAD;
+int nfs_max_async_writes = NFS_DEFMAXASYNCWRITES;
+
+int nfs_iosize = NFS_IOSIZE;
+int nfs_access_cache_timeout = NFS_MAXATTRTIMO;
+int nfs_allow_async = 0;
+int nfs_statfs_rate_limit = NFS_DEFSTATFSRATELIMIT;
+int nfs_lockd_mounts = 0;
+int nfs_lockd_request_sent = 0;
+
+int nfs_tprintf_initial_delay = NFS_TPRINTF_INITIAL_DELAY;
+int nfs_tprintf_delay = NFS_TPRINTF_DELAY;
+
+
+static int	mountnfs(struct user_nfs_args *,mount_t,mbuf_t,vfs_context_t,vnode_t *);
+static int	nfs_mount_diskless(struct nfs_dlmount *, const char *, int, vnode_t *, mount_t *, vfs_context_t);
+#if !defined(NO_MOUNT_PRIVATE)
+static int	nfs_mount_diskless_private(struct nfs_dlmount *, const char *, int, vnode_t *, mount_t *, vfs_context_t);
+#endif /* NO_MOUNT_PRIVATE */
 
 /*
- * nfs vfs operations.
+ * NFS VFS operations.
  */
+static int	nfs_vfs_mount(mount_t, vnode_t, user_addr_t, vfs_context_t);
+static int	nfs_vfs_start(mount_t, int, vfs_context_t);
+static int	nfs_vfs_unmount(mount_t, int, vfs_context_t);
+static int	nfs_vfs_root(mount_t, vnode_t *, vfs_context_t);
+static int	nfs_vfs_quotactl(mount_t, int, uid_t, caddr_t, vfs_context_t);
+static int	nfs_vfs_getattr(mount_t, struct vfs_attr *, vfs_context_t);
+static int	nfs_vfs_sync(mount_t, int, vfs_context_t);
+static int	nfs_vfs_vget(mount_t, ino64_t, vnode_t *, vfs_context_t);
+static int	nfs_vfs_vptofh(vnode_t, int *, unsigned char *, vfs_context_t);
+static int	nfs_vfs_fhtovp(mount_t, int, unsigned char *, vnode_t *, vfs_context_t);
+static int	nfs_vfs_init(struct vfsconf *);
+static int	nfs_vfs_sysctl(int *, u_int, user_addr_t, size_t *, user_addr_t, size_t, vfs_context_t);
+
 struct vfsops nfs_vfsops = {
-	nfs_mount,
-	nfs_start,
-	nfs_unmount,
-	nfs_root,
-	NULL,		/* quotactl */
+	nfs_vfs_mount,
+	nfs_vfs_start,
+	nfs_vfs_unmount,
+	nfs_vfs_root,
+	nfs_vfs_quotactl,
 	nfs_vfs_getattr,
-	nfs_sync,
-	nfs_vget,
-	nfs_fhtovp,
-	nfs_vptofh,
-	nfs_init,
-	nfs_sysctl,
-	NULL		/* setattr */
+	nfs_vfs_sync,
+	nfs_vfs_vget,
+	nfs_vfs_fhtovp,
+	nfs_vfs_vptofh,
+	nfs_vfs_init,
+	nfs_vfs_sysctl,
+	NULL,		/* setattr */
+	{ NULL,		/* reserved */
+	  NULL,		/* reserved */
+	  NULL,		/* reserved */
+	  NULL,		/* reserved */
+	  NULL,		/* reserved */
+	  NULL,		/* reserved */
+	  NULL }	/* reserved */
 };
 
 
-static int
-nfs_mount_diskless(struct nfs_dlmount *, const char *, int, vnode_t *, mount_t *);
-#if !defined(NO_MOUNT_PRIVATE)
-static int
-nfs_mount_diskless_private(struct nfs_dlmount *, const char *, int, vnode_t *, mount_t *);
-#endif /* NO_MOUNT_PRIVATE */
+/*
+ * version-specific NFS functions
+ */
+static int nfs3_mount(struct nfsmount *, vfs_context_t, struct user_nfs_args *, nfsnode_t *);
+static int nfs4_mount(struct nfsmount *, vfs_context_t, struct user_nfs_args *, nfsnode_t *);
+static int nfs3_update_statfs(struct nfsmount *, vfs_context_t);
+static int nfs4_update_statfs(struct nfsmount *, vfs_context_t);
+#if !QUOTA
+#define nfs3_getquota	NULL
+#define nfs4_getquota	NULL
+#else
+static int nfs3_getquota(struct nfsmount *, vfs_context_t, u_long, int, struct dqblk *);
+static int nfs4_getquota(struct nfsmount *, vfs_context_t, u_long, int, struct dqblk *);
+#endif
 
+struct nfs_funcs nfs3_funcs = {
+	nfs3_mount,
+	nfs3_update_statfs,
+	nfs3_getquota,
+	nfs3_access_rpc,
+	nfs3_getattr_rpc,
+	nfs3_setattr_rpc,
+	nfs3_read_rpc_async,
+	nfs3_read_rpc_async_finish,
+	nfs3_readlink_rpc,
+	nfs3_write_rpc_async,
+	nfs3_write_rpc_async_finish,
+	nfs3_commit_rpc,
+	nfs3_lookup_rpc_async,
+	nfs3_lookup_rpc_async_finish,
+	nfs3_remove_rpc,
+	nfs3_rename_rpc
+	};
+struct nfs_funcs nfs4_funcs = {
+	nfs4_mount,
+	nfs4_update_statfs,
+	nfs4_getquota,
+	nfs4_access_rpc,
+	nfs4_getattr_rpc,
+	nfs4_setattr_rpc,
+	nfs4_read_rpc_async,
+	nfs4_read_rpc_async_finish,
+	nfs4_readlink_rpc,
+	nfs4_write_rpc_async,
+	nfs4_write_rpc_async_finish,
+	nfs4_commit_rpc,
+	nfs4_lookup_rpc_async,
+	nfs4_lookup_rpc_async_finish,
+	nfs4_remove_rpc,
+	nfs4_rename_rpc
+	};
+
+/*
+ * Called once to initialize data structures...
+ */
 static int
-nfs_biosize(struct nfsmount *nmp)
+nfs_vfs_init(struct vfsconf *vfsp)
 {
-	int iosize;
+	int i;
 
 	/*
-	 * Calculate the size used for io buffers.  Use the larger
-	 * of the two sizes to minimise nfs requests but make sure
-	 * that it is at least one VM page to avoid wasting buffer
-	 * space and to allow easy mmapping of I/O buffers.
-	 * The read/write rpc calls handle the splitting up of
-	 * buffers into multiple requests if the buffer size is
-	 * larger than the I/O size.
+	 * Check to see if major data structures haven't bloated.
 	 */
-	iosize = max(nmp->nm_rsize, nmp->nm_wsize);
-	if (iosize < PAGE_SIZE)
-		iosize = PAGE_SIZE;
-	return (trunc_page_32(iosize));
+	if (sizeof (struct nfsnode) > NFS_NODEALLOC) {
+		printf("struct nfsnode bloated (> %dbytes)\n", NFS_NODEALLOC);
+		printf("Try reducing NFS_SMALLFH\n");
+	}
+	if (sizeof (struct nfsmount) > NFS_MNTALLOC) {
+		printf("struct nfsmount bloated (> %dbytes)\n", NFS_MNTALLOC);
+		printf("Try reducing NFS_MUIDHASHSIZ\n");
+	}
+
+	nfs_ticks = (hz * NFS_TICKINTVL + 500) / 1000;
+	if (nfs_ticks < 1)
+		nfs_ticks = 1;
+
+	/* init async I/O thread pool state */
+	TAILQ_INIT(&nfsiodfree);
+	TAILQ_INIT(&nfsiodwork);
+	TAILQ_INIT(&nfsiodmounts);
+	nfsiod_lck_grp = lck_grp_alloc_init("nfsiod", LCK_GRP_ATTR_NULL);
+	nfsiod_mutex = lck_mtx_alloc_init(nfsiod_lck_grp, LCK_ATTR_NULL);
+
+	/* init mount lock group */
+	nfs_mount_grp = lck_grp_alloc_init("nfs_mount", LCK_GRP_ATTR_NULL);
+
+	/* init request list mutex */
+	nfs_request_grp = lck_grp_alloc_init("nfs_request", LCK_GRP_ATTR_NULL);
+	nfs_request_mutex = lck_mtx_alloc_init(nfs_request_grp, LCK_ATTR_NULL);
+
+	/* initialize NFS request list */
+	TAILQ_INIT(&nfs_reqq);
+
+	nfs_nbinit();			/* Init the nfsbuf table */
+	nfs_nhinit();			/* Init the nfsnode table */
+	nfs_lockinit();			/* Init the nfs lock state */
+	nfs_gss_init();			/* Init RPCSEC_GSS security */
+
+	/* NFSv4 stuff */
+	NFS4_PER_FS_ATTRIBUTES(nfs_fs_attr_bitmap);
+	NFS4_PER_OBJECT_ATTRIBUTES(nfs_object_attr_bitmap);
+	NFS4_DEFAULT_ATTRIBUTES(nfs_getattr_bitmap);
+	for (i=0; i < NFS_ATTR_BITMAP_LEN; i++)
+		nfs_getattr_bitmap[i] &= nfs_object_attr_bitmap[i];
+
+	/* initialize NFS timer callouts */
+	nfs_request_timer_call = thread_call_allocate(nfs_request_timer, NULL);
+	nfs_buf_timer_call = thread_call_allocate(nfs_buf_timer, NULL);
+
+	vfsp->vfc_refcount++; /* make us non-unloadable */
+	return (0);
 }
 
 /*
  * nfs statfs call
  */
-int
-nfs_statfs(mount_t mp, struct vfsstatfs *sbp, vfs_context_t context)
+static int
+nfs3_update_statfs(struct nfsmount *nmp, vfs_context_t ctx)
 {
-	proc_t p = vfs_context_proc(context);
-	vnode_t vp;
-	struct nfs_statfs *sfp;
-	caddr_t cp;
-	u_long *tl;
-	long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	struct nfsmount *nmp = VFSTONFS(mp);
-	int error = 0, v3 = (nmp->nm_flag & NFSMNT_NFSV3), retattr;
-	mbuf_t mreq, mrep, md, mb, mb2;
+	nfsnode_t np;
+	int error = 0, lockerror, status, nfsvers;
 	u_int64_t xid;
-	kauth_cred_t cred;
-	struct ucred temp_cred;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t val;
 
-#ifndef nolint
-	sfp = (struct nfs_statfs *)0;
-#endif
-	vp = nmp->nm_dvp;
-	if ((error = vnode_get(vp)))
+	nfsvers = nmp->nm_vers;
+	np = nmp->nm_dnp;
+	if ((error = vnode_get(NFSTOV(np))))
 		return(error);
 
-	bzero(&temp_cred, sizeof(temp_cred));
-	temp_cred.cr_ngroups = 1;
-	cred = kauth_cred_create(&temp_cred);
-
-	if (v3 && (nmp->nm_state & NFSSTA_GOTFSINFO) == 0)
-		nfs_fsinfo(nmp, vp, cred, p);
-	nfsm_reqhead(NFSX_FH(v3));
-	if (error) {
-		kauth_cred_unref(&cred);
-		vnode_put(vp);
-		return (error);
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_FSSTAT, ctx,
+		   &nmrep, &xid, &status);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	if (!error)
+		error = status;
+	nfsm_assert(error, NFSTONMP(np), ENXIO);
+	nfsmout_if(error);
+	lck_mtx_lock(&nmp->nm_lock);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_TOTAL);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_FREE);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_AVAIL);
+	if (nfsvers == NFS_VER3) {
+		NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_FILES_AVAIL);
+		NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_FILES_TOTAL);
+		NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_FILES_FREE);
+		nmp->nm_fsattr.nfsa_bsize = NFS_FABLKSIZE;
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_space_total);
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_space_free);
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_space_avail);
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_files_total);
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_files_free);
+		nfsm_chain_get_64(error, &nmrep, nmp->nm_fsattr.nfsa_files_avail);
+		// skip invarsec
+	} else {
+		nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED); // skip tsize?
+		nfsm_chain_get_32(error, &nmrep, nmp->nm_fsattr.nfsa_bsize);
+		nfsm_chain_get_32(error, &nmrep, val);
+		nfsmout_if(error);
+		if (nmp->nm_fsattr.nfsa_bsize <= 0)
+			nmp->nm_fsattr.nfsa_bsize = NFS_FABLKSIZE;
+		nmp->nm_fsattr.nfsa_space_total = (uint64_t)val * nmp->nm_fsattr.nfsa_bsize;
+		nfsm_chain_get_32(error, &nmrep, val);
+		nfsmout_if(error);
+		nmp->nm_fsattr.nfsa_space_free = (uint64_t)val * nmp->nm_fsattr.nfsa_bsize;
+		nfsm_chain_get_32(error, &nmrep, val);
+		nfsmout_if(error);
+		nmp->nm_fsattr.nfsa_space_avail = (uint64_t)val * nmp->nm_fsattr.nfsa_bsize;
 	}
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_FSSTAT]);
-	nfsm_fhtom(vp, v3);
-	nfsm_request(vp, NFSPROC_FSSTAT, p, cred, &xid);
-	if (v3 && mrep)
-		nfsm_postop_attr_update(vp, v3, retattr, &xid);
-	nfsm_dissect(sfp, struct nfs_statfs *, NFSX_STATFS(v3));
-
-	sbp->f_flags = nmp->nm_flag;
-	sbp->f_iosize = NFS_IOSIZE;
-	if (v3) {
-		/*
-		 * Adjust block size to get total block count to fit in a long.
-		 * If we can't increase block size enough, clamp to max long.
-		 */
-		u_quad_t tquad, tquad2, bsize;
-		bsize = NFS_FABLKSIZE;
-
-		fxdr_hyper(&sfp->sf_tbytes, &tquad);
-		tquad /= bsize;
-		while ((tquad & ~0x7fffffff) && (bsize < 0x40000000)) {
-			bsize <<= 1;
-			tquad >>= 1;
-		}
-		sbp->f_blocks = (tquad & ~0x7fffffff) ? 0x7fffffff : (long)tquad;
-
-		fxdr_hyper(&sfp->sf_fbytes, &tquad);
-		tquad /= bsize;
-		sbp->f_bfree = (tquad & ~0x7fffffff) ? 0x7fffffff : (long)tquad;
+	lck_mtx_unlock(&nmp->nm_lock);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	vnode_put(NFSTOV(np));
+	return (error);
+}
 
-		fxdr_hyper(&sfp->sf_abytes, &tquad);
-		tquad /= bsize;
-		sbp->f_bavail = (tquad & ~0x7fffffff) ? 0x7fffffff : (long)tquad;
+static int
+nfs4_update_statfs(struct nfsmount *nmp, vfs_context_t ctx)
+{
+	nfsnode_t np;
+	int error = 0, lockerror, status, nfsvers, numops;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN];
+	struct nfs_vattr nvattr;
 
-		sbp->f_bsize = (long)bsize;
+	nfsvers = nmp->nm_vers;
+	np = nmp->nm_dnp;
+	if ((error = vnode_get(NFSTOV(np))))
+		return(error);
 
-		/* adjust file slots too... */
-		fxdr_hyper(&sfp->sf_tfiles, &tquad);
-		fxdr_hyper(&sfp->sf_ffiles, &tquad2);
-		while (tquad & ~0x7fffffff) {
-			tquad >>= 1;
-			tquad2 >>= 1;
-		}
-		sbp->f_files = tquad;
-		sbp->f_ffree = tquad2;
-	} else {
-		sbp->f_bsize = fxdr_unsigned(long, sfp->sf_bsize);
-		sbp->f_blocks = fxdr_unsigned(long, sfp->sf_blocks);
-		sbp->f_bfree = fxdr_unsigned(long, sfp->sf_bfree);
-		sbp->f_bavail = fxdr_unsigned(long, sfp->sf_bavail);
-		sbp->f_files = 0;
-		sbp->f_ffree = 0;
-	}
-	nfsm_reqdone;
- 	kauth_cred_unref(&cred);
-	vnode_put(vp);
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH + GETATTR
+	numops = 2;
+	nfsm_chain_build_alloc_init(error, &nmreq, 15 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "statfs", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS_COPY_ATTRIBUTES(nfs_getattr_bitmap, bitmap);
+	NFS4_STATFS_ATTRIBUTES(bitmap);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_assert(error, NFSTONMP(np), ENXIO);
+	nfsmout_if(error);
+	lck_mtx_lock(&nmp->nm_lock);
+	NFS_CLEAR_ATTRIBUTES(nvattr.nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, &nmp->nm_fsattr, &nvattr, NULL, NULL);
+	lck_mtx_unlock(&nmp->nm_lock);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	if (!error)
+		nfs_loadattrcache(np, &nvattr, &xid, 0);
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_assert(error, NFSTONMP(np), ENXIO);
+	nfsmout_if(error);
+	nmp->nm_fsattr.nfsa_bsize = NFS_FABLKSIZE;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	vnode_put(NFSTOV(np));
 	return (error);
 }
 
+
 /*
- * The nfs_statfs code is complicated, and used by mountnfs(), so leave it as-is
- * and handle VFS_GETATTR by calling nfs_statfs and copying fields.
+ * The NFS VFS_GETATTR function: "statfs"-type information is retrieved
+ * using the nf_update_statfs() function, and other attributes are cobbled
+ * together from whatever sources we can (getattr, fsinfo, pathconf).
  */
 static int
-nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
+nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t ctx)
 {
-	int error = 0;
-	
+	struct nfsmount *nmp;
+	uint32_t bsize;
+	int error = 0, nfsvers;
+
+	if (!(nmp = VFSTONFS(mp)))
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
 	if (VFSATTR_IS_ACTIVE(fsap, f_bsize)  ||
 	    VFSATTR_IS_ACTIVE(fsap, f_iosize) ||
 	    VFSATTR_IS_ACTIVE(fsap, f_blocks) ||
@@ -300,32 +480,67 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 	    VFSATTR_IS_ACTIVE(fsap, f_bused)  ||
 	    VFSATTR_IS_ACTIVE(fsap, f_files)  ||
 	    VFSATTR_IS_ACTIVE(fsap, f_ffree)) {
-		struct vfsstatfs sb;
+		int statfsrate = nfs_statfs_rate_limit;
+		int refresh = 1;
 
-		error = nfs_statfs(mp, &sb, context);
-		if (!error) {
-			VFSATTR_RETURN(fsap, f_bsize, sb.f_bsize);
-			VFSATTR_RETURN(fsap, f_iosize, sb.f_iosize);
-			VFSATTR_RETURN(fsap, f_blocks, sb.f_blocks);
-			VFSATTR_RETURN(fsap, f_bfree, sb.f_bfree);
-			VFSATTR_RETURN(fsap, f_bavail, sb.f_bavail);
-			VFSATTR_RETURN(fsap, f_bused, sb.f_blocks - sb.f_bfree);
-			VFSATTR_RETURN(fsap, f_files, sb.f_files);
-			VFSATTR_RETURN(fsap, f_ffree, sb.f_ffree);
+		/*
+		 * Are we rate-limiting statfs RPCs?
+		 * (Treat values less than 1 or greater than 1,000,000 as no limit.)
+		 */
+		if ((statfsrate > 0) && (statfsrate < 1000000)) {
+			struct timeval now;
+			uint32_t stamp;
+
+			microuptime(&now);
+			lck_mtx_lock(&nmp->nm_lock);
+			stamp = (now.tv_sec * statfsrate) + (now.tv_usec / (1000000/statfsrate));
+			if (stamp != nmp->nm_fsattrstamp) {
+				refresh = 1;
+				nmp->nm_fsattrstamp = stamp;
+			} else {
+				refresh = 0;
+			}
+			lck_mtx_unlock(&nmp->nm_lock);
 		}
+
+		if (refresh)
+			error = nmp->nm_funcs->nf_update_statfs(nmp, ctx);
+		if ((error == ESTALE) || (error == ETIMEDOUT))
+			error = 0;
+		if (error)
+			return (error);
+
+		lck_mtx_lock(&nmp->nm_lock);
+		VFSATTR_RETURN(fsap, f_iosize, nfs_iosize);
+		VFSATTR_RETURN(fsap, f_bsize, nmp->nm_fsattr.nfsa_bsize);
+		bsize = nmp->nm_fsattr.nfsa_bsize;
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_TOTAL))
+			VFSATTR_RETURN(fsap, f_blocks, nmp->nm_fsattr.nfsa_space_total / bsize);
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_FREE))
+			VFSATTR_RETURN(fsap, f_bfree, nmp->nm_fsattr.nfsa_space_free / bsize);
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_AVAIL))
+			VFSATTR_RETURN(fsap, f_bavail, nmp->nm_fsattr.nfsa_space_avail / bsize);
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_TOTAL) &&
+		    NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SPACE_FREE))
+			VFSATTR_RETURN(fsap, f_bused,
+				(nmp->nm_fsattr.nfsa_space_total / bsize) -
+				(nmp->nm_fsattr.nfsa_space_free / bsize));
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_FILES_TOTAL))
+			VFSATTR_RETURN(fsap, f_files, nmp->nm_fsattr.nfsa_files_total);
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_FILES_FREE))
+			VFSATTR_RETURN(fsap, f_ffree, nmp->nm_fsattr.nfsa_files_free);
+		lck_mtx_unlock(&nmp->nm_lock);
 	}
 
 	if (VFSATTR_IS_ACTIVE(fsap, f_capabilities)) {
-		struct nfsmount *nmp;
-		struct nfsv3_pathconf pc;
 		u_int32_t caps, valid;
-		vnode_t vp;
-		int v3;
+		nfsnode_t np;
 
-		if (!(nmp = VFSTONFS(mp)))
-			return (ENXIO);
-		vp = nmp->nm_dvp;
-		v3 = (nmp->nm_flag & NFSMNT_NFSV3);
+		nfsm_assert(error, VFSTONFS(mp), ENXIO);
+		if (error)
+			return (error);
+		np = nmp->nm_dnp;
+		lck_mtx_lock(&nmp->nm_lock);
 
 		/*
 		 * The capabilities[] array defines what this volume supports.
@@ -337,59 +552,41 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 		 * present or not.
 		 */
 		caps = valid = 0;
-		if (v3) {
-			/* try to get fsinfo if we haven't already */
-			if (!(nmp->nm_state & NFSSTA_GOTFSINFO)) {
-				nfs_fsinfo(nmp, vp, vfs_context_ucred(context),
-					vfs_context_proc(context));
-				if (!(nmp = VFSTONFS(vnode_mount(vp))))
-					return (ENXIO);
-			}
-			if (nmp->nm_state & NFSSTA_GOTFSINFO) {
-				/* fsinfo indicates (non)support of links and symlinks */
-				valid |= VOL_CAP_FMT_SYMBOLICLINKS |
-					 VOL_CAP_FMT_HARDLINKS;
-				if (nmp->nm_fsinfo.fsproperties & NFSV3FSINFO_SYMLINK)
-					caps |= VOL_CAP_FMT_SYMBOLICLINKS;
-				if (nmp->nm_fsinfo.fsproperties & NFSV3FSINFO_LINK)
-					caps |= VOL_CAP_FMT_HARDLINKS;
-				/* if fsinfo indicates all pathconf info is the same, */
-				/* we can use it to report case attributes */
-				if ((nmp->nm_fsinfo.fsproperties & NFSV3FSINFO_HOMOGENEOUS) &&
-				    !(nmp->nm_state & NFSSTA_GOTPATHCONF)) {
-					/* no cached pathconf info, try to get now */
-					error = nfs_pathconfrpc(vp, &pc,
-							vfs_context_ucred(context),
-							vfs_context_proc(context));
-					if (!(nmp = VFSTONFS(vnode_mount(vp))))
-						return (ENXIO);
-					if (!error) {
-						/* all files have the same pathconf info, */
-						/* so cache a copy of the results */
-						nfs_pathconf_cache(nmp, &pc);
-					}
-				}
-				if (nmp->nm_state & NFSSTA_GOTPATHCONF) {
-					valid |= VOL_CAP_FMT_CASE_SENSITIVE |
-						 VOL_CAP_FMT_CASE_PRESERVING;
-					if (!(nmp->nm_fsinfo.pcflags &
-						NFSPCINFO_CASE_INSENSITIVE))
-						caps |= VOL_CAP_FMT_CASE_SENSITIVE;
-					if (nmp->nm_fsinfo.pcflags &
-						NFSPCINFO_CASE_PRESERVING)
-						caps |= VOL_CAP_FMT_CASE_PRESERVING;
-				}
-				/* Is server's max file size at least 2TB? */
-				if (nmp->nm_fsinfo.maxfilesize >= 0x20000000000ULL)
-					caps |= VOL_CAP_FMT_2TB_FILESIZE;
-			} else {
-				/*
-				 * NFSv3 supports 64 bits of file size.
-				 * Without FSINFO from the server, we'll
-				 * just assume maxfilesize >= 2TB
-				 */
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SYMLINK_SUPPORT)) {
+			valid |= VOL_CAP_FMT_SYMBOLICLINKS;
+			if (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_SYMLINK)
+				caps |= VOL_CAP_FMT_SYMBOLICLINKS;
+		}
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_LINK_SUPPORT)) {
+			valid |= VOL_CAP_FMT_HARDLINKS;
+			if (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_LINK)
+				caps |= VOL_CAP_FMT_HARDLINKS;
+		}
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CASE_INSENSITIVE)) {
+			valid |= VOL_CAP_FMT_CASE_SENSITIVE;
+			if (!(nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_CASE_INSENSITIVE))
+				caps |= VOL_CAP_FMT_CASE_SENSITIVE;
+		}
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CASE_PRESERVING)) {
+			valid |= VOL_CAP_FMT_CASE_PRESERVING;
+			if (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_CASE_PRESERVING)
+				caps |= VOL_CAP_FMT_CASE_PRESERVING;
+		}
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXFILESIZE)) {
+			/* Is server's max file size at least 2TB? */
+			if (nmp->nm_fsattr.nfsa_maxfilesize >= 0x20000000000ULL)
 				caps |= VOL_CAP_FMT_2TB_FILESIZE;
-			}
+		} else if (nfsvers >= NFS_VER3) {
+			/*
+			 * NFSv3 and up supports 64 bits of file size.
+			 * So, we'll just assume maxfilesize >= 2TB
+			 */
+			caps |= VOL_CAP_FMT_2TB_FILESIZE;
+		}
+		if (nfsvers >= NFS_VER4) {
+			caps |= VOL_CAP_FMT_HIDDEN_FILES;
+			valid |= VOL_CAP_FMT_HIDDEN_FILES;
+			// VOL_CAP_FMT_OPENDENYMODES
 		}
 		fsap->f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] =
 			// VOL_CAP_FMT_PERSISTENTOBJECTIDS |
@@ -404,6 +601,8 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 			// VOL_CAP_FMT_CASE_PRESERVING |
 			// VOL_CAP_FMT_FAST_STATFS |
 			// VOL_CAP_FMT_2TB_FILESIZE |
+			// VOL_CAP_FMT_OPENDENYMODES |
+			// VOL_CAP_FMT_HIDDEN_FILES |
 			caps;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_FORMAT] =
 			VOL_CAP_FMT_PERSISTENTOBJECTIDS |
@@ -418,6 +617,8 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 			// VOL_CAP_FMT_CASE_PRESERVING |
 			VOL_CAP_FMT_FAST_STATFS |
 			VOL_CAP_FMT_2TB_FILESIZE |
+			// VOL_CAP_FMT_OPENDENYMODES |
+			// VOL_CAP_FMT_HIDDEN_FILES |
 			valid;
 
 		/*
@@ -433,8 +634,13 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 		 * looks like lock ops have worked, we do report that we support them.
 		 */
 		caps = valid = 0;
-		if ((!nfslockdvnode && !nfslockdwaiting) ||
-		    (nmp->nm_flag & NFSMNT_NOLOCKS)) {
+		if (nfsvers >= NFS_VER4) {
+			caps = VOL_CAP_INT_ADVLOCK | VOL_CAP_INT_FLOCK;
+			valid = VOL_CAP_INT_ADVLOCK | VOL_CAP_INT_FLOCK;
+			// VOL_CAP_INT_EXTENDED_SECURITY
+			// VOL_CAP_INT_NAMEDSTREAMS
+			// VOL_CAP_INT_EXTENDED_ATTR
+		} else if ((nmp->nm_flag & NFSMNT_NOLOCKS)) {
 			/* locks disabled on this mount, so they definitely won't work */
 			valid = VOL_CAP_INT_ADVLOCK | VOL_CAP_INT_FLOCK;
 		} else if (nmp->nm_state & NFSSTA_LOCKSWORK) {
@@ -454,6 +660,9 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 			// VOL_CAP_INT_FLOCK |
 			// VOL_CAP_INT_EXTENDED_SECURITY |
 			// VOL_CAP_INT_USERACCESS |
+			// VOL_CAP_INT_MANLOCK |
+			// VOL_CAP_INT_NAMEDSTREAMS |
+			// VOL_CAP_INT_EXTENDED_ATTR |
 			caps;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] =
 			VOL_CAP_INT_SEARCHFS |
@@ -468,6 +677,9 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 			// VOL_CAP_INT_FLOCK |
 			// VOL_CAP_INT_EXTENDED_SECURITY |
 			// VOL_CAP_INT_USERACCESS |
+			// VOL_CAP_INT_MANLOCK |
+			// VOL_CAP_INT_NAMEDSTREAMS |
+			// VOL_CAP_INT_EXTENDED_ATTR |
 			valid;
 
 		fsap->f_capabilities.capabilities[VOL_CAPABILITIES_RESERVED1] = 0;
@@ -477,6 +689,7 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_RESERVED2] = 0;
 
 		VFSATTR_SET_SUPPORTED(fsap, f_capabilities);
+		lck_mtx_unlock(&nmp->nm_lock);
 	}
 
 	if (VFSATTR_IS_ACTIVE(fsap, f_attributes)) {
@@ -503,65 +716,99 @@ nfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, vfs_context_t context)
 /*
  * nfs version 3 fsinfo rpc call
  */
-int
-nfs_fsinfo(nmp, vp, cred, p)
-	struct nfsmount *nmp;
-	vnode_t vp;
-	kauth_cred_t cred;
-	proc_t p;
+static int
+nfs3_fsinfo(struct nfsmount *nmp, nfsnode_t np, vfs_context_t ctx)
 {
-	struct nfsv3_fsinfo *fsp;
-	caddr_t cp;
-	long t1, t2;
-	u_long *tl;
-	int prefsize, maxsize;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, retattr;
-	mbuf_t mreq, mrep, md, mb, mb2;
+	int error = 0, lockerror, status, prefsize, maxsize, nmlocked = 0;
 	u_int64_t xid;
-
-	nfsm_reqhead(NFSX_FH(1));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_FSINFO]);
-	nfsm_fhtom(vp, 1);
-	nfsm_request(vp, NFSPROC_FSINFO, p, cred, &xid);
-	if (mrep) {
-		nfsm_postop_attr_update(vp, 1, retattr, &xid);
-	}
-	if (!error) {
-		nfsm_dissect(fsp, struct nfsv3_fsinfo *, NFSX_V3FSINFO);
-		prefsize = fxdr_unsigned(u_long, fsp->fs_wtpref);
-		if (prefsize < nmp->nm_wsize)
-			nmp->nm_wsize = (prefsize + NFS_FABLKSIZE - 1) &
-				~(NFS_FABLKSIZE - 1);
-		maxsize = fxdr_unsigned(u_long, fsp->fs_wtmax);
-		if (maxsize < nmp->nm_wsize) {
-			nmp->nm_wsize = maxsize & ~(NFS_FABLKSIZE - 1);
-			if (nmp->nm_wsize == 0)
-				nmp->nm_wsize = maxsize;
-		}
-		prefsize = fxdr_unsigned(u_long, fsp->fs_rtpref);
-		if (prefsize < nmp->nm_rsize)
-			nmp->nm_rsize = (prefsize + NFS_FABLKSIZE - 1) &
-				~(NFS_FABLKSIZE - 1);
-		maxsize = fxdr_unsigned(u_long, fsp->fs_rtmax);
-		if (maxsize < nmp->nm_rsize) {
-			nmp->nm_rsize = maxsize & ~(NFS_FABLKSIZE - 1);
-			if (nmp->nm_rsize == 0)
-				nmp->nm_rsize = maxsize;
-		}
-		prefsize = fxdr_unsigned(u_long, fsp->fs_dtpref);
-		if (prefsize < nmp->nm_readdirsize)
-			nmp->nm_readdirsize = prefsize;
-		if (maxsize < nmp->nm_readdirsize) {
-			nmp->nm_readdirsize = maxsize;
-		}
-		fxdr_hyper(&fsp->fs_maxfilesize, &nmp->nm_fsinfo.maxfilesize);
-		nmp->nm_fsinfo.fsproperties = fxdr_unsigned(u_long, fsp->fs_properties);
-		nmp->nm_state |= NFSSTA_GOTFSINFO;
+	uint32_t val;
+	struct nfsm_chain nmreq, nmrep;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(nmp->nm_vers));
+	nfsm_chain_add_fh(error, &nmreq, nmp->nm_vers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_FSINFO, ctx,
+			&nmrep, &xid, &status);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	if (!error)
+		error = status;
+	nfsmout_if(error);
+
+	lck_mtx_lock(&nmp->nm_lock);
+	nmlocked = 1;
+
+	nfsm_chain_get_32(error, &nmrep, maxsize);
+	nfsm_chain_get_32(error, &nmrep, prefsize);
+	nfsmout_if(error);
+	nmp->nm_fsattr.nfsa_maxread = maxsize;
+	if (prefsize < nmp->nm_rsize)
+		nmp->nm_rsize = (prefsize + NFS_FABLKSIZE - 1) &
+			~(NFS_FABLKSIZE - 1);
+	if (maxsize < nmp->nm_rsize) {
+		nmp->nm_rsize = maxsize & ~(NFS_FABLKSIZE - 1);
+		if (nmp->nm_rsize == 0)
+			nmp->nm_rsize = maxsize;
+	}
+	nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED); // skip rtmult
+
+	nfsm_chain_get_32(error, &nmrep, maxsize);
+	nfsm_chain_get_32(error, &nmrep, prefsize);
+	nfsmout_if(error);
+	nmp->nm_fsattr.nfsa_maxwrite = maxsize;
+	if (prefsize < nmp->nm_wsize)
+		nmp->nm_wsize = (prefsize + NFS_FABLKSIZE - 1) &
+			~(NFS_FABLKSIZE - 1);
+	if (maxsize < nmp->nm_wsize) {
+		nmp->nm_wsize = maxsize & ~(NFS_FABLKSIZE - 1);
+		if (nmp->nm_wsize == 0)
+			nmp->nm_wsize = maxsize;
 	}
-	nfsm_reqdone;
+	nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED); // skip wtmult
+
+	nfsm_chain_get_32(error, &nmrep, prefsize);
+	nfsmout_if(error);
+	if (prefsize < nmp->nm_readdirsize)
+		nmp->nm_readdirsize = prefsize;
+	if (maxsize < nmp->nm_readdirsize)
+		nmp->nm_readdirsize = maxsize;
+
+	nfsm_chain_get_64(error, &nmrep, maxsize);
+	nmp->nm_fsattr.nfsa_maxfilesize = maxsize;
+
+	nfsm_chain_adv(error, &nmrep, 2 * NFSX_UNSIGNED); // skip time_delta
+
+	/* convert FS properties to our own flags */
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsmout_if(error);
+	if (val & NFSV3FSINFO_LINK)
+		nmp->nm_fsattr.nfsa_flags |= NFS_FSFLAG_LINK;
+	if (val & NFSV3FSINFO_SYMLINK)
+		nmp->nm_fsattr.nfsa_flags |= NFS_FSFLAG_SYMLINK;
+	if (val & NFSV3FSINFO_HOMOGENEOUS)
+		nmp->nm_fsattr.nfsa_flags |= NFS_FSFLAG_HOMOGENEOUS;
+	if (val & NFSV3FSINFO_CANSETTIME)
+		nmp->nm_fsattr.nfsa_flags |= NFS_FSFLAG_SET_TIME;
+	nmp->nm_state |= NFSSTA_GOTFSINFO;
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXREAD);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXWRITE);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXFILESIZE);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_LINK_SUPPORT);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_SYMLINK_SUPPORT);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_HOMOGENEOUS);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CANSETTIME);
+nfsmout:
+	if (nmlocked)
+		lck_mtx_unlock(&nmp->nm_lock);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
@@ -579,29 +826,27 @@ nfs_fsinfo(nmp, vp, cred, p)
  * - build the rootfs mount point and call mountnfs() to do the rest.
  */
 int
-nfs_mountroot()
+nfs_mountroot(void)
 {
 	struct nfs_diskless nd;
 	struct nfs_vattr nvattr;
-	mount_t mp;
-	vnode_t vp;
-	proc_t procp;
+	mount_t mp = NULL;
+	vnode_t vp = NULL;
+	vfs_context_t ctx;
 	int error;
 #if !defined(NO_MOUNT_PRIVATE)
-	mount_t mppriv;
-	vnode_t vppriv;
+	mount_t mppriv = NULL;
+	vnode_t vppriv = NULL;
 #endif /* NO_MOUNT_PRIVATE */
 	int v3, sotype;
 
-	procp = current_proc(); /* XXX */
-
 	/*
 	 * Call nfs_boot_init() to fill in the nfs_diskless struct.
 	 * Note: networking must already have been configured before
 	 * we're called.
 	 */
 	bzero((caddr_t) &nd, sizeof(nd));
-	error = nfs_boot_init(&nd, procp);
+	error = nfs_boot_init(&nd);
 	if (error) {
 		panic("nfs_boot_init failed with %d\n", error);
 	}
@@ -614,14 +859,14 @@ nfs_mountroot()
 	sotype = SOCK_STREAM;
 
 tryagain:
-	error = nfs_boot_getfh(&nd, procp, v3, sotype);
+	error = nfs_boot_getfh(&nd, v3, sotype);
 	if (error) {
 		if (error == EHOSTDOWN || error == EHOSTUNREACH) {
 			if (nd.nd_root.ndm_path)
-				FREE_ZONE(nd.nd_root.ndm_path, 
+				FREE_ZONE(nd.nd_root.ndm_path,
 					  MAXPATHLEN, M_NAMEI);
 			if (nd.nd_private.ndm_path)
-				FREE_ZONE(nd.nd_private.ndm_path, 
+				FREE_ZONE(nd.nd_private.ndm_path,
 					  MAXPATHLEN, M_NAMEI);
 			return (error);
 		}
@@ -640,16 +885,37 @@ tryagain:
 			sotype = SOCK_DGRAM;
 			goto tryagain;
 		}
-		panic("nfs_boot_getfh(v2,UDP) failed with %d\n", error);
+		switch(error) {
+		case EPROGUNAVAIL:
+			panic("nfs_boot_getfh(v2,UDP) failed: NFS server mountd not responding - check server configuration: %s", PE_boot_args());
+		case EACCES:
+		case EPERM:
+			panic("nfs_boot_getfh(v2,UDP) failed: NFS server refused mount - check server configuration: %s", PE_boot_args());
+		default:
+			panic("nfs_boot_getfh(v2,UDP) failed with %d: %s", error, PE_boot_args());
+		}
 	}
 
+	ctx = vfs_context_kernel();
+
 	/*
 	 * Create the root mount point.
 	 */
 #if !defined(NO_MOUNT_PRIVATE)
-	if ((error = nfs_mount_diskless(&nd.nd_root, "/", MNT_RDONLY|MNT_ROOTFS, &vp, &mp)))
+	{
+		//PWC hack until we have a real "mount" tool to remount root rw
+		int rw_root=0;
+		int flags = MNT_ROOTFS|MNT_RDONLY;
+		PE_parse_boot_arg("-rwroot_hack", &rw_root);
+		if(rw_root)
+		{
+			flags = MNT_ROOTFS;
+			kprintf("-rwroot_hack in effect: mounting root fs read/write\n");
+		}
+				
+	if ((error = nfs_mount_diskless(&nd.nd_root, "/", flags, &vp, &mp, ctx)))
 #else
-	if ((error = nfs_mount_diskless(&nd.nd_root, "/", MNT_ROOTFS, &vp, &mp)))
+	if ((error = nfs_mount_diskless(&nd.nd_root, "/", MNT_ROOTFS, &vp, &mp, ctx)))
 #endif /* NO_MOUNT_PRIVATE */
 	{
 		if (v3) {
@@ -667,7 +933,8 @@ tryagain:
 			sotype = SOCK_DGRAM;
 			goto tryagain;
 		}
-		panic("nfs_mount_diskless(v2,UDP) root failed with %d\n", error);
+		panic("nfs_mount_diskless(v2,UDP) root failed with %d: %s\n", error, PE_boot_args());
+	}
 	}
 	printf("root on %s\n", (char *)&nd.nd_root.ndm_host);
 
@@ -678,12 +945,12 @@ tryagain:
 #if !defined(NO_MOUNT_PRIVATE)
 	if (nd.nd_private.ndm_saddr.sin_addr.s_addr) {
 	    error = nfs_mount_diskless_private(&nd.nd_private, "/private",
-					       0, &vppriv, &mppriv);
+					       0, &vppriv, &mppriv, ctx);
 	    if (error) {
 		panic("nfs_mount_diskless private failed with %d\n", error);
 	    }
 	    printf("private on %s\n", (char *)&nd.nd_private.ndm_host);
-	    
+
 	    vfs_unbusy(mppriv);
 	    mount_list_add(mppriv);
 	}
@@ -696,7 +963,7 @@ tryagain:
 		FREE_ZONE(nd.nd_private.ndm_path, MAXPATHLEN, M_NAMEI);
 
 	/* Get root attributes (for the time). */
-	error = nfs_getattr(vp, &nvattr, kauth_cred_get(), procp);
+	error = nfs_getattr(VTONFS(vp), &nvattr, ctx, 0);
 	if (error) panic("nfs_mountroot: getattr for root");
 	return (0);
 }
@@ -710,15 +977,13 @@ nfs_mount_diskless(
 	const char *mntname,
 	int mntflag,
 	vnode_t *vpp,
-	mount_t *mpp)
+	mount_t *mpp,
+	vfs_context_t ctx)
 {
 	struct user_nfs_args args;
 	mount_t mp;
 	mbuf_t m;
 	int error;
-	proc_t procp;
-
-	procp = current_proc(); /* XXX */
 
 	if ((error = vfs_rootmountalloc("nfs", ndmntp->ndm_host, &mp))) {
 		printf("nfs_mount_diskless: NFS not configured");
@@ -747,8 +1012,8 @@ nfs_mount_diskless(
 		return (error);
 	}
 	mbuf_setlen(m, ndmntp->ndm_saddr.sin_len);
-	bcopy((caddr_t)args.addr, mbuf_data(m), ndmntp->ndm_saddr.sin_len);
-	if ((error = mountnfs(&args, mp, m, procp, vpp))) {
+	bcopy(&ndmntp->ndm_saddr, mbuf_data(m), ndmntp->ndm_saddr.sin_len);
+	if ((error = mountnfs(&args, mp, m, ctx, vpp))) {
 		printf("nfs_mountroot: mount %s failed: %d\n", mntname, error);
 		// XXX vfs_rootmountfailed(mp);
 		mount_list_lock();
@@ -756,6 +1021,9 @@ nfs_mount_diskless(
 		mount_list_unlock();
 		vfs_unbusy(mp);
 		mount_lock_destroy(mp);
+#if CONFIG_MACF
+		mac_mount_label_destroy(mp);
+#endif
 		FREE_ZONE(mp, sizeof(struct mount), M_MOUNT);
 		return (error);
 	}
@@ -774,7 +1042,8 @@ nfs_mount_diskless_private(
 	const char *mntname,
 	int mntflag,
 	vnode_t *vpp,
-	mount_t *mpp)
+	mount_t *mpp,
+	vfs_context_t ctx)
 {
 	struct user_nfs_args args;
 	mount_t mp;
@@ -784,11 +1053,8 @@ nfs_mount_diskless_private(
 	struct vfstable *vfsp;
 	struct nameidata nd;
 	vnode_t vp;
-	struct vfs_context context;
 
 	procp = current_proc(); /* XXX */
-	context.vc_proc = procp;
-	context.vc_ucred = kauth_cred_get();
 
 	{
 	/*
@@ -805,7 +1071,7 @@ nfs_mount_diskless_private(
 		error = vnode_ref(rootvnode);
 		if (error) {
 			printf("nfs_mountroot: vnode_ref() failed on root vnode!\n");
-			return (error);
+			goto out;
 		}
 		fdp->fd_cdir = rootvnode;
 		fdp->fd_rdir = NULL;
@@ -815,10 +1081,10 @@ nfs_mount_diskless_private(
 	 * Get vnode to be covered
 	 */
 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF, UIO_SYSSPACE32,
-	    mntname, &context);
+	    CAST_USER_ADDR_T(mntname), ctx);
 	if ((error = namei(&nd))) {
 		printf("nfs_mountroot: private namei failed!\n");
-		return (error);
+		goto out;
 	}
 	{
 		/* undo vnode_ref() in mimic main()! */
@@ -827,26 +1093,29 @@ nfs_mount_diskless_private(
 	nameidone(&nd);
 	vp = nd.ni_vp;
 
-	if ((error = VNOP_FSYNC(vp, MNT_WAIT, &context)) ||
+	if ((error = VNOP_FSYNC(vp, MNT_WAIT, ctx)) ||
 	    (error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0))) {
 		vnode_put(vp);
-		return (error);
+		goto out;
 	}
 	if (vnode_vtype(vp) != VDIR) {
 		vnode_put(vp);
-		return (ENOTDIR);
+		error = ENOTDIR;
+		goto out;
 	}
 	for (vfsp = vfsconf; vfsp; vfsp = vfsp->vfc_next)
-		if (!strcmp(vfsp->vfc_name, "nfs"))
+		if (!strncmp(vfsp->vfc_name, "nfs", sizeof(vfsp->vfc_name)))
 			break;
 	if (vfsp == NULL) {
 		printf("nfs_mountroot: private NFS not configured\n");
 		vnode_put(vp);
-		return (ENODEV);
+		error = ENODEV;
+		goto out;
 	}
 	if (vnode_mountedhere(vp) != NULL) {
 		vnode_put(vp);
-		return (EBUSY);
+		error = EBUSY;
+		goto out;
 	}
 
 	/*
@@ -856,13 +1125,17 @@ nfs_mount_diskless_private(
 	if (!mp) {
 		printf("nfs_mountroot: unable to allocate mount structure\n");
 		vnode_put(vp);
-		return (ENOMEM);
+		error = ENOMEM;
+		goto out;
 	}
 	bzero((char *)mp, (u_long)sizeof(struct mount));
 
 	/* Initialize the default IO constraints */
 	mp->mnt_maxreadcnt = mp->mnt_maxwritecnt = MAXPHYS;
 	mp->mnt_segreadcnt = mp->mnt_segwritecnt = 32;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
 
 	mount_lock_init(mp);
 	TAILQ_INIT(&mp->mnt_vnodelist);
@@ -884,6 +1157,10 @@ nfs_mount_diskless_private(
 	mp->mnt_vfsstat.f_owner = kauth_cred_getuid(kauth_cred_get());
 	(void) copystr(mntname, mp->mnt_vfsstat.f_mntonname, MNAMELEN - 1, 0);
 	(void) copystr(ndmntp->ndm_host, mp->mnt_vfsstat.f_mntfromname, MNAMELEN - 1, 0);
+#if CONFIG_MACF
+	mac_mount_label_init(mp);
+	mac_mount_label_associate(ctx, mp);
+#endif
 
 	/* Initialize mount args. */
 	bzero((caddr_t) &args, sizeof(args));
@@ -900,24 +1177,28 @@ nfs_mount_diskless_private(
 	error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_SONAME, &m);
 	if (error) {
 		printf("nfs_mount_diskless_private: mbuf_get(soname) failed");
-		return (error);
+		goto out;
 	}
 	mbuf_setlen(m, ndmntp->ndm_saddr.sin_len);
-	bcopy((caddr_t)args.addr, mbuf_data(m), ndmntp->ndm_saddr.sin_len);
-	if ((error = mountnfs(&args, mp, m, procp, &vp))) {
+	bcopy(&ndmntp->ndm_saddr, mbuf_data(m), ndmntp->ndm_saddr.sin_len);
+	if ((error = mountnfs(&args, mp, m, ctx, &vp))) {
 		printf("nfs_mountroot: mount %s failed: %d\n", mntname, error);
 		mount_list_lock();
 		vfsp->vfc_refcount--;
 		mount_list_unlock();
 		vfs_unbusy(mp);
 		mount_lock_destroy(mp);
+#if CONFIG_MACF
+		mac_mount_label_destroy(mp);
+#endif
 		FREE_ZONE(mp, sizeof (struct mount), M_MOUNT);
-		return (error);
+		goto out;
 	}
 
 	*mpp = mp;
 	*vpp = vp;
-	return (0);
+out:
+	return (error);
 }
 #endif /* NO_MOUNT_PRIVATE */
 
@@ -927,9 +1208,8 @@ nfs_mount_diskless_private(
  * mount system call
  */
 static int
-nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context)
+nfs_vfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t ctx)
 {
-	proc_t p = vfs_context_proc(context);
 	int error, argsvers;
 	struct user_nfs_args args;
 	struct nfs_args tempargs;
@@ -944,13 +1224,19 @@ nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context)
 
 	switch (argsvers) {
 	case 3:
-		if (vfs_context_is64bit(context))
+		if (vfs_context_is64bit(ctx))
 			error = copyin(data, (caddr_t)&args, sizeof (struct user_nfs_args3));
 		else
 			error = copyin(data, (caddr_t)&tempargs, sizeof (struct nfs_args3));
 		break;
 	case 4:
-		if (vfs_context_is64bit(context))
+		if (vfs_context_is64bit(ctx))
+			error = copyin(data, (caddr_t)&args, sizeof (struct user_nfs_args4));
+		else
+			error = copyin(data, (caddr_t)&tempargs, sizeof (struct nfs_args4));
+		break;
+	case 5:
+		if (vfs_context_is64bit(ctx))
 			error = copyin(data, (caddr_t)&args, sizeof (args));
 		else
 			error = copyin(data, (caddr_t)&tempargs, sizeof (tempargs));
@@ -961,7 +1247,7 @@ nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context)
 	if (error)
 		return (error);
 
-	if (!vfs_context_is64bit(context)) {
+	if (!vfs_context_is64bit(ctx)) {
 		args.version = tempargs.version;
 		args.addrlen = tempargs.addrlen;
 		args.sotype = tempargs.sotype;
@@ -986,13 +1272,17 @@ nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context)
 			args.acdirmin = tempargs.acdirmin;
 			args.acdirmax = tempargs.acdirmax;
 		}
+		if (argsvers >= 5)
+			args.auth = tempargs.auth;
 	}
 
 	if (args.fhsize < 0 || args.fhsize > NFSX_V3FHMAX)
 		return (EINVAL);
-	error = copyin(args.fh, (caddr_t)nfh, args.fhsize);
-	if (error)
-		return (error);
+	if (args.fhsize > 0) {
+		error = copyin(args.fh, (caddr_t)nfh, args.fhsize);
+		if (error)
+			return (error);
+	}
 
 	mntfrom = &vfs_statfs(mp)->f_mntfromname[0];
 	error = copyinstr(args.hostname, mntfrom, MAXPATHLEN-1, &len);
@@ -1006,30 +1296,217 @@ nfs_mount(mount_t mp, vnode_t vp, user_addr_t data, vfs_context_t context)
 		return (error);
 
 	args.fh = CAST_USER_ADDR_T(&nfh[0]);
-	error = mountnfs(&args, mp, nam, p, &vp);
+	error = mountnfs(&args, mp, nam, ctx, &vp);
 	return (error);
 }
 
 /*
  * Common code for mount and mountroot
  */
+
+static int
+nfs3_mount(
+	struct nfsmount *nmp,
+	vfs_context_t ctx,
+	struct user_nfs_args *argp,
+	nfsnode_t *npp)
+{
+	int error = 0;
+	struct nfs_vattr nvattr;
+	u_int64_t xid;
+	u_char *fhp;
+
+	*npp = NULL;
+
+	/*
+	 * Get file attributes for the mountpoint.  These are needed
+	 * in order to properly create the root vnode.
+	 */
+	// LP64todo - fix CAST_DOWN of argp->fh
+	fhp = CAST_DOWN(u_char *, argp->fh);
+	error = nfs3_getattr_rpc(NULL, nmp->nm_mountp, fhp, argp->fhsize,
+			ctx, &nvattr, &xid);
+	if (error)
+		goto out;
+
+	error = nfs_nget(nmp->nm_mountp, NULL, NULL, fhp, argp->fhsize,
+			&nvattr, &xid, NG_MARKROOT, npp);
+	if (*npp)
+		nfs_unlock(*npp);
+	if (error)
+		goto out;
+
+	/*
+	 * Try to make sure we have all the general info from the server.
+	 */
+	if (nmp->nm_vers == NFS_VER2) {
+		NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXNAME);
+		nmp->nm_fsattr.nfsa_maxname = NFS_MAXNAMLEN;
+	} else if (nmp->nm_vers == NFS_VER3) {
+		/* get the NFSv3 FSINFO */
+		error = nfs3_fsinfo(nmp, *npp, ctx);
+		if (error)
+			goto out;
+		/* If the server indicates all pathconf info is */
+		/* the same, grab a copy of that info now */
+		if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_HOMOGENEOUS) &&
+		    (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_HOMOGENEOUS)) {
+			struct nfs_fsattr nfsa;
+			if (!nfs3_pathconf_rpc(*npp, &nfsa, ctx)) {
+				/* cache a copy of the results */
+				lck_mtx_lock(&nmp->nm_lock);
+				nfs3_pathconf_cache(nmp, &nfsa);
+				lck_mtx_unlock(&nmp->nm_lock);
+			}
+		}
+	}
+out:
+	if (*npp && error) {
+		vnode_put(NFSTOV(*npp));
+		*npp = NULL;
+	}
+	return (error);
+}
+
+static int
+nfs4_mount(
+	struct nfsmount *nmp,
+	vfs_context_t ctx,
+	__unused struct user_nfs_args *argp,
+	nfsnode_t *npp)
+{
+	struct nfsm_chain nmreq, nmrep;
+	int error = 0, numops, status, interval;
+	char *path = &vfs_statfs(nmp->nm_mountp)->f_mntfromname[0];
+	char *name, *nextname;
+	fhandle_t fh;
+	struct nfs_vattr nvattr;
+	u_int64_t xid;
+	struct timeval now;
+
+	*npp = NULL;
+	fh.fh_len = 0;
+	microtime(&now);
+	nmp->nm_mounttime = ((uint64_t)now.tv_sec << 32) | now.tv_usec;
+
+	/* look up path to get fh and attrs for mount point root */
+	numops = 2; // PUTROOTFH + LOOKUP* + GETATTR
+	while (*path && (*path != '/'))
+		path++;
+	name = path;
+	while (*name) {
+		while (*name && (*name == '/'))
+			name++;
+		if (!*name)
+			break;
+		nextname = name;
+		while (*nextname && (*nextname != '/'))
+			nextname++;
+		numops++;
+		name = nextname;
+	}
+	nfsm_chain_build_alloc_init(error, &nmreq, 25 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "mount", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTROOTFH);
+	// (LOOKUP)*
+	name = path;
+	while (*name) {
+		while (*name && (*name == '/'))
+			name++;
+		if (!*name)
+			break;
+		nextname = name;
+		while (*nextname && (*nextname != '/'))
+			nextname++;
+		numops--;
+		nfsm_chain_add_32(error, &nmreq, NFS_OP_LOOKUP);
+		nfsm_chain_add_string(error, &nmreq, name, nextname - name);
+		name = nextname;
+	}
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS4_DEFAULT_ATTRIBUTES(nmp->nm_fsattr.nfsa_supp_attr);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_supp_attr, NFS_FATTR_FILEHANDLE);
+	nfsm_chain_add_bitmap(error, &nmreq, nmp->nm_fsattr.nfsa_supp_attr, NFS_ATTR_BITMAP_LEN);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request(NULL, nmp->nm_mountp, &nmreq, NFSPROC4_COMPOUND, ctx, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTROOTFH);
+	name = path;
+	while (*name) {
+		while (*name && (*name == '/'))
+			name++;
+		if (!*name)
+			break;
+		nextname = name;
+		while (*nextname && (*nextname != '/'))
+			nextname++;
+		nfsm_chain_op_check(error, &nmrep, NFS_OP_LOOKUP);
+		name = nextname;
+	}
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsmout_if(error);
+	NFS_CLEAR_ATTRIBUTES(nmp->nm_fsattr.nfsa_bitmap);
+	NFS_CLEAR_ATTRIBUTES(&nvattr.nva_bitmap);
+	error = nfs4_parsefattr(&nmrep, &nmp->nm_fsattr, &nvattr, &fh, NULL);
+	if (!error && !NFS_BITMAP_ISSET(&nvattr.nva_bitmap, NFS_FATTR_FILEHANDLE)) {
+		printf("nfs: mount didn't return filehandle?\n");
+		error = EBADRPC;
+	}
+	nfsmout_if(error);
+
+	error = nfs_nget(nmp->nm_mountp, NULL, NULL, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MARKROOT, npp);
+	nfsmout_if(error);
+
+	/* XXX local locking for now */
+	vfs_setlocklocal(nmp->nm_mountp);
+
+	/* adjust I/O sizes to server limits */
+	if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXREAD)) {
+		if (nmp->nm_fsattr.nfsa_maxread < (uint64_t)nmp->nm_rsize) {
+			nmp->nm_rsize = nmp->nm_fsattr.nfsa_maxread & ~(NFS_FABLKSIZE - 1);
+			if (nmp->nm_rsize == 0)
+				nmp->nm_rsize = nmp->nm_fsattr.nfsa_maxread;
+		}
+	}
+	if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXWRITE)) {
+		if (nmp->nm_fsattr.nfsa_maxwrite < (uint64_t)nmp->nm_wsize) {
+			nmp->nm_wsize = nmp->nm_fsattr.nfsa_maxwrite & ~(NFS_FABLKSIZE - 1);
+			if (nmp->nm_wsize == 0)
+				nmp->nm_wsize = nmp->nm_fsattr.nfsa_maxwrite;
+		}
+	}
+
+	/* set up lease renew timer */
+	nmp->nm_renew_timer = thread_call_allocate(nfs4_renew_timer, nmp);
+	interval = nmp->nm_fsattr.nfsa_lease / 2;
+	if (interval < 1)
+		interval = 1;
+	nfs_interval_timer_start(nmp->nm_renew_timer, interval * 1000);
+
+nfsmout:
+	if (*npp)
+		nfs_unlock(*npp);
+	return (error);
+}
+
 static int
 mountnfs(
 	struct user_nfs_args *argp,
 	mount_t mp,
 	mbuf_t nam,
-	proc_t p,
+	vfs_context_t ctx,
 	vnode_t *vpp)
 {
 	struct nfsmount *nmp;
-	struct nfsnode *np;
-	int error, maxio;
-	struct nfs_vattr nvattrs;
-	struct vfs_context context; /* XXX get from caller? */
-	u_int64_t xid;
-
-	/* up front because of reference */
-	context.vc_ucred = kauth_cred_proc_ref(p);
+	nfsnode_t np;
+	int error, maxio, iosize;
+	struct vfsstatfs *sbp;
+	struct timespec ts = { 1, 0 };
 
 	/*
 	 * Silently clear NFSMNT_NOCONN if it's a TCP mount, it makes
@@ -1037,28 +1514,32 @@ mountnfs(
 	 */
 	if (argp->sotype == SOCK_STREAM)
 		argp->flags &= ~NFSMNT_NOCONN;
-        
+
 	if (vfs_flags(mp) & MNT_UPDATE) {
 		nmp = VFSTONFS(mp);
 		/* update paths, file handles, etc, here	XXX */
 		mbuf_freem(nam);
-		kauth_cred_unref(&context.vc_ucred);
 		return (0);
 	} else {
 		MALLOC_ZONE(nmp, struct nfsmount *,
 				sizeof (struct nfsmount), M_NFSMNT, M_WAITOK);
 		if (!nmp) {
 			mbuf_freem(nam);
-			kauth_cred_unref(&context.vc_ucred);
 			return (ENOMEM);
 		}
 		bzero((caddr_t)nmp, sizeof (struct nfsmount));
-		TAILQ_INIT(&nmp->nm_uidlruhead);
-		TAILQ_INIT(&nmp->nm_bufq);
+		lck_mtx_init(&nmp->nm_lock, nfs_mount_grp, LCK_ATTR_NULL);
+		TAILQ_INIT(&nmp->nm_resendq);
+		TAILQ_INIT(&nmp->nm_iodq);
+		TAILQ_INIT(&nmp->nm_gsscl);
 		vfs_setfsprivate(mp, nmp);
+
+		nfs_nhinit_finish();
 	}
+	lck_mtx_lock(&nmp->nm_lock);
 
 	/* setup defaults */
+	nmp->nm_vers = NFS_VER2;
 	nmp->nm_timeo = NFS_TIMEO;
 	nmp->nm_retry = NFS_RETRANS;
 	if (argp->sotype == SOCK_DGRAM) {
@@ -1081,6 +1562,7 @@ mountnfs(
 	nmp->nm_acregmax = NFS_MAXATTRTIMO;
 	nmp->nm_acdirmin = NFS_MINDIRATTRTIMO;
 	nmp->nm_acdirmax = NFS_MAXDIRATTRTIMO;
+	nmp->nm_auth = RPCAUTH_SYS;
 
 	vfs_getnewfsid(mp);
 	nmp->nm_mountp = mp;
@@ -1088,6 +1570,19 @@ mountnfs(
 	nmp->nm_flag = argp->flags;
 	nmp->nm_nam = nam;
 
+	if (argp->flags & NFSMNT_NFSV4) {
+		nmp->nm_vers = NFS_VER4;
+		/* NFSv4 is only allowed over TCP. */
+		if (argp->sotype != SOCK_STREAM) {
+			error = EINVAL;
+			goto bad;
+		}
+	} else if (argp->flags & NFSMNT_NFSV3)
+		nmp->nm_vers = NFS_VER3;
+
+	if (nmp->nm_vers == NFS_VER2)
+		nmp->nm_flag &= ~NFSMNT_RDIRPLUS;
+
 	if ((argp->flags & NFSMNT_TIMEO) && argp->timeo > 0) {
 		nmp->nm_timeo = (argp->timeo * NFS_HZ + 5) / 10;
 		if (nmp->nm_timeo < NFS_MINTIMEO)
@@ -1102,7 +1597,7 @@ mountnfs(
 			nmp->nm_retry = NFS_MAXREXMIT;
 	}
 
-	if (argp->flags & NFSMNT_NFSV3) {
+	if (nmp->nm_vers != NFS_VER2) {
 		if (argp->sotype == SOCK_DGRAM)
 			maxio = NFS_MAXDGRAMDATA;
 		else
@@ -1148,6 +1643,12 @@ mountnfs(
 	if ((argp->flags & NFSMNT_READAHEAD) && argp->readahead >= 0 &&
 		argp->readahead <= NFS_MAXRAHEAD)
 		nmp->nm_readahead = argp->readahead;
+	if (argp->flags & NFSMNT_READAHEAD)
+		nmp->nm_readahead = argp->readahead;
+	if (nmp->nm_readahead < 0)
+		nmp->nm_readahead = 0;
+	else if (nmp->nm_readahead > NFS_MAXRAHEAD)
+		nmp->nm_readahead = NFS_MAXRAHEAD;
 
 	if (argp->version >= 4) {
 		if ((argp->flags & NFSMNT_ACREGMIN) && argp->acregmin >= 0)
@@ -1163,74 +1664,101 @@ mountnfs(
 		if (nmp->nm_acdirmin > nmp->nm_acdirmax)
 			nmp->nm_acdirmin = nmp->nm_acdirmax;
 	}
+	if (argp->version >= 5) {
+		if (argp->flags & NFSMNT_SECFLAVOR) {
+			/*
+			 * Check for valid security flavor
+			 */
+			switch (argp->auth) {
+			case RPCAUTH_SYS:
+			case RPCAUTH_KRB5:
+			case RPCAUTH_KRB5I:
+			case RPCAUTH_KRB5P:
+				nmp->nm_auth = argp->auth;
+				break;
+			default:
+				error = EINVAL;
+				goto bad;
+			}
+		}
+	}
 
-	/* Set up the sockets and per-host congestion */
+	/* set up the version-specific function tables */
+	if (nmp->nm_vers < NFS_VER4)
+		nmp->nm_funcs = &nfs3_funcs;
+	else
+		nmp->nm_funcs = &nfs4_funcs;
+
+	/* Set up the sockets and related info */
 	nmp->nm_sotype = argp->sotype;
 	nmp->nm_soproto = argp->proto;
+	if (nmp->nm_sotype == SOCK_DGRAM)
+		TAILQ_INIT(&nmp->nm_cwndq);
+
+	lck_mtx_unlock(&nmp->nm_lock);
 
 	/* make sure mbuf constants are set up */
-	if (!nfs_mbuf_mlen)
+	if (!nfs_mbuf_mhlen)
 		nfs_mbuf_init();
 
-	/*
-	 * For Connection based sockets (TCP,...) defer the connect until
-	 * the first request, in case the server is not responding.
-	 */
-	if (nmp->nm_sotype == SOCK_DGRAM &&
-		(error = nfs_connect(nmp, (struct nfsreq *)0)))
+	/* NFS does its own node locking */
+	mp->mnt_vtable->vfc_threadsafe = TRUE;
+
+	/* set up the socket */
+	if ((error = nfs_connect(nmp)))
 		goto bad;
 
 	/*
-	 * Get file attributes for the mountpoint.  These are needed
-	 * in order to properly create the root vnode.
+	 * Get the root node/attributes from the NFS server and
+	 * do any basic, version-specific setup.
 	 */
-	// LP64todo - fix CAST_DOWN of argp->fh
-	error = nfs_getattr_no_vnode(mp, CAST_DOWN(caddr_t, argp->fh), argp->fhsize,
-			context.vc_ucred, p, &nvattrs, &xid);
-	if (error) {
-		/*
-		 * we got problems... we couldn't get the attributes
-		 * from the NFS server... so the mount fails.
-		 */
+	error = nmp->nm_funcs->nf_mount(nmp, ctx, argp, &np);
+	if (error)
 		goto bad;
-	}
 
 	/*
-	 * A reference count is needed on the nfsnode representing the
+	 * A reference count is needed on the node representing the
 	 * remote root.  If this object is not persistent, then backward
 	 * traversals of the mount point (i.e. "..") will not work if
-	 * the nfsnode gets flushed out of the cache. UFS does not have
-	 * this problem, because one can identify root inodes by their
-	 * number == ROOTINO (2).
+	 * the node gets flushed out of the cache.
 	 */
-	error = nfs_nget(mp, NULL, NULL, CAST_DOWN(caddr_t, argp->fh), argp->fhsize,
-			&nvattrs, &xid, NG_MARKROOT, &np);
+	nmp->nm_dnp = np;
+	*vpp = NFSTOV(np);
+	/* get usecount and drop iocount */
+	error = vnode_ref(*vpp);
+	vnode_put(*vpp);
 	if (error)
 		goto bad;
 
 	/*
-	 * save this vnode pointer. That way nfs_unmount()
-	 * does not need to call nfs_nget() just get it to drop
-	 * this vnode reference.
+	 * Do statfs to ensure static info gets set to reasonable values.
 	 */
-	nmp->nm_dvp = *vpp = NFSTOV(np);
-	/* get usecount and drop iocount */
-	error = vnode_ref(*vpp);
-	if (error) {
-		vnode_put(*vpp);
+	if ((error = nmp->nm_funcs->nf_update_statfs(nmp, ctx)))
 		goto bad;
-	}
-	vnode_put(*vpp);
+	sbp = vfs_statfs(mp);
+	sbp->f_bsize = nmp->nm_fsattr.nfsa_bsize;
+	sbp->f_blocks = nmp->nm_fsattr.nfsa_space_total / sbp->f_bsize;
+	sbp->f_bfree = nmp->nm_fsattr.nfsa_space_free / sbp->f_bsize;
+	sbp->f_bavail = nmp->nm_fsattr.nfsa_space_avail / sbp->f_bsize;
+	sbp->f_bused = (nmp->nm_fsattr.nfsa_space_total / sbp->f_bsize) -
+			(nmp->nm_fsattr.nfsa_space_free / sbp->f_bsize);
+	sbp->f_files = nmp->nm_fsattr.nfsa_files_total;
+	sbp->f_ffree = nmp->nm_fsattr.nfsa_files_free;
+	sbp->f_iosize = nfs_iosize;
 
 	/*
-	 * Set the mount point's block I/O size.
-	 * We really need to do this after we get info back from
-	 * the server about what its preferred I/O sizes are.
+	 * Calculate the size used for I/O buffers.  Use the larger
+	 * of the two sizes to minimise NFS requests but make sure
+	 * that it is at least one VM page to avoid wasting buffer
+	 * space and to allow easy mmapping of I/O buffers.
+	 * The read/write RPC calls handle the splitting up of
+	 * buffers into multiple requests if the buffer size is
+	 * larger than the I/O size.
 	 */
-	if (nmp->nm_flag & NFSMNT_NFSV3)
-		nfs_fsinfo(nmp, *vpp, context.vc_ucred, p);
-	nmp->nm_biosize = nfs_biosize(nmp);
-	vfs_statfs(mp)->f_iosize = NFS_IOSIZE;
+	iosize = max(nmp->nm_rsize, nmp->nm_wsize);
+	if (iosize < PAGE_SIZE)
+		iosize = PAGE_SIZE;
+	nmp->nm_biosize = trunc_page_32(iosize);
 
 	/*
 	 * V3 mounts give us a (relatively) reliable remote access(2)
@@ -1239,25 +1767,40 @@ mountnfs(
 	 * XXX this may not be the best way to go, as the granularity
 	 *     offered isn't a good match to our needs.
 	 */
-	if (nmp->nm_flag & NFSMNT_NFSV3)
+	if (nmp->nm_vers != NFS_VER2)
 		vfs_setauthopaqueaccess(mp);
 
-	/*
-	 * Do statfs to ensure static info gets set to reasonable values.
-	 */
-	context.vc_proc = p;
-	nfs_statfs(mp, vfs_statfs(mp), &context);
+	if (nmp->nm_flag & NFSMNT_LOCALLOCKS)
+		vfs_setlocklocal(nmp->nm_mountp);
+	if (!(nmp->nm_flag & (NFSMNT_NOLOCKS|NFSMNT_LOCALLOCKS)))
+		nfs_lockd_mount_change(1);
 
-	if (nmp->nm_flag & NFSMNT_RESVPORT)
-		nfs_resv_mounts++;
+	lck_mtx_lock(&nmp->nm_lock);
 	nmp->nm_state |= NFSSTA_MOUNTED;
-	kauth_cred_unref(&context.vc_ucred);
+	lck_mtx_unlock(&nmp->nm_lock);
 	return (0);
 bad:
+	/* mark the socket for termination */
+	lck_mtx_lock(&nmp->nm_lock);
+	nmp->nm_sockflags |= NMSOCK_UNMOUNT;
+	/* wait for any socket poking to complete */
+	while (nmp->nm_sockflags & NMSOCK_POKE)
+		msleep(&nmp->nm_sockflags, &nmp->nm_lock, PZERO-1, "nfswaitpoke", &ts);
+	/* wait for the socket thread to terminate */
+	while (nmp->nm_sockthd) {
+		wakeup(&nmp->nm_sockthd);
+		msleep(&nmp->nm_sockthd, &nmp->nm_lock, PZERO-1, "nfswaitsockthd", &ts);
+	}
+	/* tear down the socket */
+	lck_mtx_unlock(&nmp->nm_lock);
 	nfs_disconnect(nmp);
+	if (nmp->nm_renew_timer) {
+		thread_call_cancel(nmp->nm_renew_timer);
+		thread_call_free(nmp->nm_renew_timer);
+	}
+	lck_mtx_destroy(&nmp->nm_lock, nfs_mount_grp);
 	FREE_ZONE((caddr_t)nmp, sizeof (struct nfsmount), M_NFSMNT);
 	mbuf_freem(nam);
-	kauth_cred_unref(&context.vc_ucred);
 	return (error);
 }
 
@@ -1266,16 +1809,20 @@ bad:
  * unmount system call
  */
 static int
-nfs_unmount(
+nfs_vfs_unmount(
 	mount_t mp,
 	int mntflags,
-	__unused vfs_context_t context)
+	__unused vfs_context_t ctx)
 {
-	register struct nfsmount *nmp;
+	struct nfsmount *nmp;
 	vnode_t vp;
-	int error, flags = 0;
+	int error, flags = 0, docallback;
+	struct nfsreq *req, *treq;
+	struct nfs_reqqhead iodq;
+	struct timespec ts = { 1, 0 };
 
 	nmp = VFSTONFS(mp);
+	lck_mtx_lock(&nmp->nm_lock);
 	/*
 	 * During a force unmount we want to...
 	 *   Mark that we are doing a force unmount.
@@ -1294,10 +1841,11 @@ nfs_unmount(
 	 * - Close the socket
 	 * - Free up the data structures
 	 */
-	vp = nmp->nm_dvp;
+	vp = NFSTOV(nmp->nm_dnp);
+	lck_mtx_unlock(&nmp->nm_lock);
 	
 	/*
-	 * vflush will check for busy vnodes on mountpoint. 
+	 * vflush will check for busy vnodes on mountpoint.
 	 * Will do the right thing for MNT_FORCE. That is, we should
 	 * not get EBUSY back.
 	 */
@@ -1312,39 +1860,103 @@ nfs_unmount(
 	if (error)
 		return (error);
 
+	lck_mtx_lock(&nmp->nm_lock);
 	nmp->nm_state &= ~NFSSTA_MOUNTED;
-	if (nmp->nm_flag & NFSMNT_RESVPORT) {
-		if (--nfs_resv_mounts == 0)
-			nfs_bind_resv_thread_wake();
-	}
+	lck_mtx_unlock(&nmp->nm_lock);
 
 	/*
 	 * Release the root vnode reference held by mountnfs()
 	 */
 	vnode_rele(vp);
 
-	(void)vflush(mp, NULLVP, FORCECLOSE);
+	vflush(mp, NULLVP, FORCECLOSE);
+
+	/*
+	 * Destroy any RPCSEC_GSS contexts
+	 */
+	if (!TAILQ_EMPTY(&nmp->nm_gsscl))
+		nfs_gss_clnt_ctx_unmount(nmp, mntflags);
+
 	vfs_setfsprivate(mp, 0); /* don't want to end up using stale vp */
 
+	/* mark the socket for termination */
+	lck_mtx_lock(&nmp->nm_lock);
+	nmp->nm_sockflags |= NMSOCK_UNMOUNT;
+
+	/* wait for any socket poking to complete */
+	while (nmp->nm_sockflags & NMSOCK_POKE)
+		msleep(&nmp->nm_sockflags, &nmp->nm_lock, PZERO-1, "nfswaitpoke", &ts);
+
+	/* wait for the socket thread to terminate */
+	while (nmp->nm_sockthd) {
+		wakeup(&nmp->nm_sockthd);
+		msleep(&nmp->nm_sockthd, &nmp->nm_lock, PZERO-1, "nfswaitsockthd", &ts);
+	}
+
+	/* tear down the socket */
+	lck_mtx_unlock(&nmp->nm_lock);
 	nfs_disconnect(nmp);
+	lck_mtx_lock(&nmp->nm_lock);
+
+	/* cancel any renew timer */
+	if (nmp->nm_renew_timer) {
+		thread_call_cancel(nmp->nm_renew_timer);
+		thread_call_free(nmp->nm_renew_timer);
+	}
+
 	mbuf_freem(nmp->nm_nam);
+	lck_mtx_unlock(&nmp->nm_lock);
 
-	if ((nmp->nm_flag & NFSMNT_KERB) == 0) {
-		struct nfsreq *rp;
-		/*
-		 * Loop through outstanding request list and remove dangling
-		 * references to defunct nfsmount struct
-		 */
-		for (rp = nfs_reqq.tqh_first; rp; rp = rp->r_chain.tqe_next)
-			if (rp->r_nmp == nmp)
-				rp->r_nmp = (struct nfsmount *)0;
-		/* Need to wake up any rcvlock waiters so they notice the unmount. */
-		if (nmp->nm_state & NFSSTA_WANTRCV) {
-			nmp->nm_state &= ~NFSSTA_WANTRCV;
-			wakeup(&nmp->nm_state);
+	if (!(nmp->nm_flag & (NFSMNT_NOLOCKS|NFSMNT_LOCALLOCKS)))
+		nfs_lockd_mount_change(-1);
+
+	/*
+	 * Loop through outstanding request list and remove dangling
+	 * references to defunct nfsmount struct
+	 */
+	TAILQ_INIT(&iodq);
+	lck_mtx_lock(nfs_request_mutex);
+	TAILQ_FOREACH(req, &nfs_reqq, r_chain) {
+		if (req->r_nmp == nmp) {
+			lck_mtx_lock(&req->r_mtx);
+			req->r_nmp = NULL;
+			lck_mtx_unlock(&req->r_mtx);
+			if (req->r_callback.rcb_func) {
+				/* async I/O RPC needs to be finished */
+				lck_mtx_lock(nfsiod_mutex);
+				if (req->r_achain.tqe_next == NFSREQNOLIST)
+					TAILQ_INSERT_TAIL(&iodq, req, r_achain);
+				lck_mtx_unlock(nfsiod_mutex);
+			}
+			lck_mtx_lock(&nmp->nm_lock);
+			if (req->r_rchain.tqe_next != NFSREQNOLIST) {
+				TAILQ_REMOVE(&nmp->nm_resendq, req, r_rchain);
+				req->r_rchain.tqe_next = NFSREQNOLIST;
+				req->r_flags &= ~R_RESENDQ;
+			}
+			lck_mtx_unlock(&nmp->nm_lock);
+			wakeup(req);
 		}
-		FREE_ZONE((caddr_t)nmp, sizeof (struct nfsmount), M_NFSMNT);
 	}
+	lck_mtx_unlock(nfs_request_mutex);
+
+	/* finish any async I/O RPCs queued up */
+	lck_mtx_lock(nfsiod_mutex);
+	TAILQ_CONCAT(&iodq, &nmp->nm_iodq, r_achain);
+	lck_mtx_unlock(nfsiod_mutex);
+	TAILQ_FOREACH_SAFE(req, &iodq, r_achain, treq) {
+		TAILQ_REMOVE(&iodq, req, r_achain);
+		req->r_achain.tqe_next = NFSREQNOLIST;
+		lck_mtx_lock(&req->r_mtx);
+		req->r_error = ENXIO;
+		docallback = !(req->r_flags & R_WAITSENT);
+		lck_mtx_unlock(&req->r_mtx);
+		if (docallback)
+			req->r_callback.rcb_func(req);
+	}
+
+	lck_mtx_destroy(&nmp->nm_lock, nfs_mount_grp);
+	FREE_ZONE((caddr_t)nmp, sizeof (struct nfsmount), M_NFSMNT);
 	return (0);
 }
 
@@ -1352,7 +1964,7 @@ nfs_unmount(
  * Return root of a filesystem
  */
 static int
-nfs_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
+nfs_vfs_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t ctx)
 {
 	vnode_t vp;
 	struct nfsmount *nmp;
@@ -1360,7 +1972,7 @@ nfs_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
 	u_long vpid;
 
 	nmp = VFSTONFS(mp);
-	vp = nmp->nm_dvp;
+	vp = NFSTOV(nmp->nm_dnp);
 	vpid = vnode_vid(vp);
 	while ((error = vnode_getwithvid(vp, vpid))) {
 		/* vnode_get() may return ENOENT if the dir changes. */
@@ -1373,14 +1985,352 @@ nfs_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t context)
 	return (0);
 }
 
+/*
+ * Do operations associated with quotas
+ */
+#if !QUOTA
+static int
+nfs_vfs_quotactl(
+	__unused mount_t mp,
+	__unused int cmds,
+	__unused uid_t uid,
+	__unused caddr_t datap,
+	__unused vfs_context_t context)
+{
+	return (ENOTSUP);
+}
+#else
+static int
+nfs_aux_request(
+	struct nfsmount *nmp,
+	thread_t thd,
+	struct sockaddr_in *saddr,
+	mbuf_t mreq,
+	uint32_t xid,
+	int timeo,
+	struct nfsm_chain *nmrep)
+{
+	int error = 0, on = 1, try, sendat = 2;
+	socket_t so = NULL;
+	struct timeval tv = { 1, 0 };
+	mbuf_t m, mrep = NULL;
+	struct msghdr msg;
+	uint32_t rxid, reply, reply_status, rejected_status;
+	uint32_t verf_type, verf_len, accepted_status;
+	size_t readlen;
+
+	/* create socket and set options */
+	if (((error = sock_socket(saddr->sin_family, SOCK_DGRAM, IPPROTO_UDP, NULL, NULL, &so))) ||
+	    ((error = sock_setsockopt(so, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)))) ||
+	    ((error = sock_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)))) ||
+	    ((error = sock_setsockopt(so, SOL_SOCKET, SO_NOADDRERR, &on, sizeof(on)))))
+		goto nfsmout;
+
+	for (try=0; try < timeo; try++) {
+		if ((error = nfs_sigintr(nmp, NULL, thd, 0)))
+			break;
+		if (!try || (try == sendat)) {
+			/* send the request (resending periodically) */
+			if ((error = mbuf_copym(mreq, 0, MBUF_COPYALL, MBUF_WAITOK, &m)))
+				goto nfsmout;
+			bzero(&msg, sizeof(msg));
+			msg.msg_name = saddr;
+			msg.msg_namelen = saddr->sin_len;
+			if ((error = sock_sendmbuf(so, &msg, m, 0, NULL)))
+				goto nfsmout;
+			sendat *= 2;
+			if (sendat > 30)
+				sendat = 30;
+		}
+		/* wait for the response */
+		readlen = 1<<18;
+		bzero(&msg, sizeof(msg));
+		error = sock_receivembuf(so, &msg, &mrep, 0, &readlen);
+		if (error == EWOULDBLOCK)
+			continue;
+		nfsmout_if(error);
+		/* parse the response */
+		nfsm_chain_dissect_init(error, nmrep, mrep);
+		nfsm_chain_get_32(error, nmrep, rxid);
+		nfsm_chain_get_32(error, nmrep, reply);
+		nfsmout_if(error);
+		if ((rxid != xid) || (reply != RPC_REPLY))
+			error = EBADRPC;
+		nfsm_chain_get_32(error, nmrep, reply_status);
+		nfsmout_if(error);
+		if (reply_status == RPC_MSGDENIED) {
+			nfsm_chain_get_32(error, nmrep, rejected_status);
+			nfsmout_if(error);
+			error = (rejected_status == RPC_MISMATCH) ? ENOTSUP : EACCES;
+			goto nfsmout;
+		}
+		nfsm_chain_get_32(error, nmrep, verf_type); /* verifier flavor */
+		nfsm_chain_get_32(error, nmrep, verf_len); /* verifier length */
+		nfsmout_if(error);
+		if (verf_len)
+			nfsm_chain_adv(error, nmrep, nfsm_rndup(verf_len));
+		nfsm_chain_get_32(error, nmrep, accepted_status);
+		nfsm_assert(error, (accepted_status == RPC_SUCCESS), EIO);
+		break;
+	}
+nfsmout:
+	if (so) {
+		sock_shutdown(so, SHUT_RDWR);
+		sock_close(so);
+	}
+	mbuf_freem(mreq);
+	return (error);
+}
+
+static int
+nfs3_getquota(struct nfsmount *nmp, vfs_context_t ctx, u_long id, int type, struct dqblk *dqb)
+{
+	int error = 0, auth_len, slen, timeo;
+	int rqvers = (type == GRPQUOTA) ? RPCRQUOTA_EXT_VER : RPCRQUOTA_VER;
+	thread_t thd = vfs_context_thread(ctx);
+	kauth_cred_t cred = vfs_context_ucred(ctx);
+	char *path;
+	uint64_t xid = 0;
+	struct nfsm_chain nmreq, nmrep;
+	mbuf_t mreq;
+	uint32_t val = 0, bsize;
+	struct sockaddr *nam = mbuf_data(nmp->nm_nam);
+	struct sockaddr_in saddr;
+	struct timeval now;
+
+	bcopy(nam, &saddr, min(sizeof(saddr), nam->sa_len));
+	auth_len = ((((cred->cr_ngroups - 1) > nmp->nm_numgrps) ?
+			nmp->nm_numgrps : (cred->cr_ngroups - 1)) << 2) +
+			5 * NFSX_UNSIGNED;
+	timeo = (nmp->nm_flag & NFSMNT_SOFT) ? 10 : 60;
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	/* check if we have a recently cached rquota port */
+	if (nmp->nm_rqport) {
+		microuptime(&now);
+		if ((nmp->nm_rqportstamp + 60) >= (uint32_t)now.tv_sec)
+			goto got_rqport;
+	}
+
+	/* send portmap request to get rquota port */
+	saddr.sin_port = htons(PMAPPORT);
+	nfsm_chain_build_alloc_init(error, &nmreq, 4*NFSX_UNSIGNED);
+	nfsm_chain_add_32(error, &nmreq, RPCPROG_RQUOTA);
+	nfsm_chain_add_32(error, &nmreq, rqvers);
+	nfsm_chain_add_32(error, &nmreq, IPPROTO_UDP);
+	nfsm_chain_add_32(error, &nmreq, 0);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfsm_rpchead2(SOCK_DGRAM, PMAPPROG, PMAPVERS, PMAPPROC_GETPORT,
+			RPCAUTH_SYS, auth_len, cred, NULL, nmreq.nmc_mhead, &xid, &mreq);
+	nfsmout_if(error);
+	nmreq.nmc_mhead = NULL;
+	error = nfs_aux_request(nmp, thd, &saddr, mreq, R_XID32(xid), timeo, &nmrep);
+	nfsmout_if(error);
+
+	/* grab rquota port from portmap response */
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsmout_if(error);
+	nmp->nm_rqport = val;
+	microuptime(&now);
+	nmp->nm_rqportstamp = now.tv_sec;
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	xid = 0;
+
+got_rqport:
+	/* rquota request */
+	saddr.sin_port = htons(nmp->nm_rqport);
+	path = &vfs_statfs(nmp->nm_mountp)->f_mntfromname[0];
+	while (*path && (*path != '/'))
+		path++;
+	slen = strlen(path);
+	nfsm_chain_build_alloc_init(error, &nmreq, 3 * NFSX_UNSIGNED + nfsm_rndup(slen));
+	nfsm_chain_add_string(error, &nmreq, path, slen);
+	if (type == GRPQUOTA)
+		nfsm_chain_add_32(error, &nmreq, type);
+	nfsm_chain_add_32(error, &nmreq, id);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfsm_rpchead2(SOCK_DGRAM, RPCPROG_RQUOTA, rqvers, RPCRQUOTA_GET,
+			RPCAUTH_SYS, auth_len, cred, NULL, nmreq.nmc_mhead, &xid, &mreq);
+	nfsmout_if(error);
+	nmreq.nmc_mhead = NULL;
+	error = nfs_aux_request(nmp, thd, &saddr, mreq, R_XID32(xid), timeo, &nmrep);
+	nfsmout_if(error);
+
+	/* parse rquota response */
+	nfsm_chain_get_32(error, &nmrep, val);
+	if (!error && (val != RQUOTA_STAT_OK)) {
+		if (val == RQUOTA_STAT_NOQUOTA)
+			error = ENOENT;
+		else if (val == RQUOTA_STAT_EPERM)
+			error = EPERM;
+		else
+			error = EIO;
+	}
+	nfsm_chain_get_32(error, &nmrep, bsize);
+	nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED);
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsmout_if(error);
+	dqb->dqb_bhardlimit = (uint64_t)val * bsize;
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsmout_if(error);
+	dqb->dqb_bsoftlimit = (uint64_t)val * bsize;
+	nfsm_chain_get_32(error, &nmrep, val);
+	nfsmout_if(error);
+	dqb->dqb_curbytes = (uint64_t)val * bsize;
+	nfsm_chain_get_32(error, &nmrep, dqb->dqb_ihardlimit);
+	nfsm_chain_get_32(error, &nmrep, dqb->dqb_isoftlimit);
+	nfsm_chain_get_32(error, &nmrep, dqb->dqb_curinodes);
+	nfsm_chain_get_32(error, &nmrep, dqb->dqb_btime);
+	nfsm_chain_get_32(error, &nmrep, dqb->dqb_itime);
+	nfsmout_if(error);
+	dqb->dqb_id = id;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+static int
+nfs4_getquota(struct nfsmount *nmp, vfs_context_t ctx, u_long id, int type, struct dqblk *dqb)
+{
+	nfsnode_t np;
+	int error = 0, status, nfsvers, numops;
+	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+	uint32_t bitmap[NFS_ATTR_BITMAP_LEN];
+	thread_t thd = vfs_context_thread(ctx);
+	kauth_cred_t cred = vfs_context_ucred(ctx);
+
+	if (type != USRQUOTA)  /* NFSv4 only supports user quotas */
+		return (ENOTSUP);
+
+	/* first check that the server supports any of the quota attributes */
+	if (!NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_supp_attr, NFS_FATTR_QUOTA_AVAIL_HARD) &&
+	    !NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_supp_attr, NFS_FATTR_QUOTA_AVAIL_SOFT) &&
+	    !NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_supp_attr, NFS_FATTR_QUOTA_USED))
+		return (ENOTSUP);
+
+	/*
+	 * The credential passed to the server needs to have
+	 * an effective uid that matches the given uid.
+	 */
+	if (id != kauth_cred_getuid(cred)) {
+		struct ucred temp_cred;
+		bzero(&temp_cred, sizeof(temp_cred));
+		temp_cred.cr_uid = id;
+		temp_cred.cr_ngroups = cred->cr_ngroups;
+		bcopy(cred->cr_groups, temp_cred.cr_groups, sizeof(temp_cred.cr_groups));
+		cred = kauth_cred_create(&temp_cred);
+		if (!IS_VALID_CRED(cred))
+			return (ENOMEM);
+	} else {
+		kauth_cred_ref(cred);
+	}
+
+	nfsvers = nmp->nm_vers;
+	np = nmp->nm_dnp;
+	if ((error = vnode_get(NFSTOV(np)))) {
+		kauth_cred_unref(&cred);
+		return(error);
+	}
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	// PUTFH + GETATTR
+	numops = 2;
+	nfsm_chain_build_alloc_init(error, &nmreq, 15 * NFSX_UNSIGNED);
+	nfsm_chain_add_compound_header(error, &nmreq, "quota", numops);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_PUTFH);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	numops--;
+	nfsm_chain_add_32(error, &nmreq, NFS_OP_GETATTR);
+	NFS_CLEAR_ATTRIBUTES(bitmap);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_QUOTA_AVAIL_HARD);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_QUOTA_AVAIL_SOFT);
+	NFS_BITMAP_SET(bitmap, NFS_FATTR_QUOTA_USED);
+	nfsm_chain_add_bitmap_masked(error, &nmreq, bitmap,
+		NFS_ATTR_BITMAP_LEN, nmp->nm_fsattr.nfsa_supp_attr);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsm_assert(error, (numops == 0), EPROTO);
+	nfsmout_if(error);
+	error = nfs_request2(np, NULL, &nmreq, NFSPROC4_COMPOUND, thd, cred, 0, &nmrep, &xid, &status);
+	nfsm_chain_skip_tag(error, &nmrep);
+	nfsm_chain_get_32(error, &nmrep, numops);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_PUTFH);
+	nfsm_chain_op_check(error, &nmrep, NFS_OP_GETATTR);
+	nfsm_assert(error, NFSTONMP(np), ENXIO);
+	nfsmout_if(error);
+	error = nfs4_parsefattr(&nmrep, NULL, NULL, NULL, dqb);
+	nfsmout_if(error);
+	nfsm_assert(error, NFSTONMP(np), ENXIO);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	vnode_put(NFSTOV(np));
+	kauth_cred_unref(&cred);
+	return (error);
+}
+
+static int
+nfs_vfs_quotactl(mount_t mp, int cmds, uid_t uid, caddr_t datap, vfs_context_t ctx)
+{
+	struct nfsmount *nmp;
+	int cmd, type, error, nfsvers;
+	uid_t ruid = vfs_context_ucred(ctx)->cr_ruid;
+	struct dqblk *dqb = (struct dqblk*)datap;
+
+	if (!(nmp = VFSTONFS(mp)))
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	if (uid == ~0U)
+		uid = ruid;
+
+	/* we can only support Q_GETQUOTA */
+	cmd = cmds >> SUBCMDSHIFT;
+	switch (cmd) {
+	case Q_GETQUOTA:
+		break;
+	case Q_QUOTAON:
+	case Q_QUOTAOFF:
+	case Q_SETQUOTA:
+	case Q_SETUSE:
+	case Q_SYNC:
+	case Q_QUOTASTAT:
+		return (ENOTSUP);
+	default:
+		return (EINVAL);
+	}
+
+	type = cmds & SUBCMDMASK;
+	if ((u_int)type >= MAXQUOTAS)
+		return (EINVAL);
+	if ((uid != ruid) && ((error = vfs_context_suser(ctx))))
+		return (error);
+
+	if (vfs_busy(mp, LK_NOWAIT))
+		return (0);
+	bzero(dqb, sizeof(*dqb));
+	error = nmp->nm_funcs->nf_getquota(nmp, ctx, uid, type, dqb);
+	vfs_unbusy(mp);
+	return (error);
+}
+#endif
+
 /*
  * Flush out the buffer cache
  */
 
 struct nfs_sync_cargs {
-        vfs_context_t context;
-        int    waitfor;
-        int    error;
+	thread_t	thd;
+	int		waitfor;
+	int		error;
 };
 
 static int
@@ -1391,12 +2341,12 @@ nfs_sync_callout(vnode_t vp, void *arg)
 
 	if (LIST_EMPTY(&VTONFS(vp)->n_dirtyblkhd))
 		return (VNODE_RETURNED);
-	if (VTONFS(vp)->n_flag & NWRBUSY)
+	if (VTONFS(vp)->n_wrbusy > 0)
+		return (VNODE_RETURNED);
+	if (VTONFS(vp)->n_bflag & (NBFLUSHINPROG|NBINVALINPROG))
 		return (VNODE_RETURNED);
 
-	error = nfs_flush(vp, cargs->waitfor,
-			vfs_context_ucred(cargs->context),
-			vfs_context_proc(cargs->context), 0);
+	error = nfs_flush(VTONFS(vp), cargs->waitfor, cargs->thd, 0);
 	if (error)
 		cargs->error = error;
 
@@ -1404,12 +2354,12 @@ nfs_sync_callout(vnode_t vp, void *arg)
 }
 
 static int
-nfs_sync(mount_t mp, int waitfor, vfs_context_t context)
+nfs_vfs_sync(mount_t mp, int waitfor, vfs_context_t ctx)
 {
 	struct nfs_sync_cargs cargs;
 
 	cargs.waitfor = waitfor;
-	cargs.context = context;
+	cargs.thd = vfs_context_thread(ctx);
 	cargs.error = 0;
 
 	vnode_iterate(mp, 0, nfs_sync_callout, &cargs);
@@ -1423,11 +2373,11 @@ nfs_sync(mount_t mp, int waitfor, vfs_context_t context)
  */
 /*ARGSUSED*/
 static int
-nfs_vget(
+nfs_vfs_vget(
 	__unused mount_t mp,
 	__unused ino64_t ino,
 	__unused vnode_t *vpp,
-	__unused vfs_context_t context)
+	__unused vfs_context_t ctx)
 {
 
 	return (ENOTSUP);
@@ -1438,12 +2388,12 @@ nfs_vget(
  */
 /*ARGSUSED*/
 static int
-nfs_fhtovp(
+nfs_vfs_fhtovp(
 	__unused mount_t mp,
 	__unused int fhlen,
 	__unused unsigned char *fhp,
 	__unused vnode_t *vpp,
-	__unused vfs_context_t context)
+	__unused vfs_context_t ctx)
 {
 
 	return (ENOTSUP);
@@ -1454,11 +2404,11 @@ nfs_fhtovp(
  */
 /*ARGSUSED*/
 static int
-nfs_vptofh(
+nfs_vfs_vptofh(
 	__unused vnode_t vp,
 	__unused int *fhlenp,
 	__unused unsigned char *fhp,
-	__unused vfs_context_t context)
+	__unused vfs_context_t ctx)
 {
 
 	return (ENOTSUP);
@@ -1469,10 +2419,10 @@ nfs_vptofh(
  */
 /*ARGSUSED*/
 static int
-nfs_start(
+nfs_vfs_start(
 	__unused mount_t mp,
 	__unused int flags,
-	__unused vfs_context_t context)
+	__unused vfs_context_t ctx)
 {
 
 	return (0);
@@ -1482,8 +2432,8 @@ nfs_start(
  * Do that sysctl thang...
  */
 static int
-nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-           user_addr_t newp, size_t newlen, vfs_context_t context)
+nfs_vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
+           user_addr_t newp, size_t newlen, vfs_context_t ctx)
 {
 	int error = 0, val;
 	struct sysctl_req *req = NULL;
@@ -1493,14 +2443,27 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	struct nfsmount *nmp = NULL;
 	struct vfsquery vq;
 	boolean_t is_64_bit;
+#if NFSSERVER
+	struct nfs_exportfs *nxfs;
+	struct nfs_export *nx;
+	struct nfs_active_user_list *ulist;
+	struct nfs_export_stat_desc stat_desc;
+	struct nfs_export_stat_rec statrec;
+	struct nfs_user_stat_node *unode, *unode_next;
+	struct nfs_user_stat_desc ustat_desc;
+	struct nfs_user_stat_user_rec ustat_rec;
+	struct nfs_user_stat_path_rec upath_rec;
+	uint bytes_avail, bytes_total, recs_copied;
+	uint numExports, totlen, pos, numRecs, count;
+#endif /* NFSSERVER */
 
 	/*
 	 * All names at this level are terminal.
 	 */
-	if(namelen > 1)
-		return ENOTDIR;	/* overloaded */
+	if (namelen > 1)
+		return (ENOTDIR);	/* overloaded */
 
-	is_64_bit = vfs_context_is64bit(context);
+	is_64_bit = vfs_context_is64bit(ctx);
 
 	/* common code for "new style" VFS_CTL sysctl, get the mount. */
 	switch (name[0]) {
@@ -1513,8 +2476,7 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 			if (error)
 				 return (error);
 			mp = vfs_getvfs(&user_vc.vc_fsid);
-		} 
-		else {
+		} else {
 			error = SYSCTL_IN(req, &vc, sizeof(vc));
 			if (error)
 				return (error);
@@ -1530,8 +2492,7 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 		if (is_64_bit) {
 			req->newptr = user_vc.vc_ptr;
 			req->newlen = (size_t)user_vc.vc_len;
-		}
-		else {
+		} else {
 			req->newptr = CAST_USER_ADDR_T(vc.vc_ptr);
 			req->newlen = vc.vc_len;
 		}
@@ -1539,30 +2500,272 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 
 	switch(name[0]) {
 	case NFS_NFSSTATS:
-		if(!oldp) {
+		if (!oldp) {
 			*oldlenp = sizeof nfsstats;
-			return 0;
+			return (0);
 		}
 
-		if(*oldlenp < sizeof nfsstats) {
+		if (*oldlenp < sizeof nfsstats) {
 			*oldlenp = sizeof nfsstats;
-			return ENOMEM;
+			return (ENOMEM);
 		}
 
 		error = copyout(&nfsstats, oldp, sizeof nfsstats);
 		if (error)
 			return (error);
 
-		if(newp && newlen != sizeof nfsstats)
-			return EINVAL;
+		if (newp && newlen != sizeof nfsstats)
+			return (EINVAL);
 
-		if(newp) {
+		if (newp)
 			return copyin(newp, &nfsstats, sizeof nfsstats);
+		return (0);
+#if NFSSERVER
+	case NFS_EXPORTSTATS:
+		/* setup export stat descriptor */
+		stat_desc.rec_vers = NFS_EXPORT_STAT_REC_VERSION;
+
+		if (!nfsrv_is_initialized()) {
+			stat_desc.rec_count = 0;
+			if (oldp && (*oldlenp >= sizeof(struct nfs_export_stat_desc)))
+				error = copyout(&stat_desc, oldp, sizeof(struct nfs_export_stat_desc));
+			*oldlenp = sizeof(struct nfs_export_stat_desc);
+			return (error);
+		}
+
+		/* Count the number of exported directories */
+		lck_rw_lock_shared(&nfsrv_export_rwlock);
+		numExports = 0;
+		LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next)
+			LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next)
+					numExports += 1;
+
+		/* update stat descriptor's export record count */
+		stat_desc.rec_count = numExports;
+
+		/* calculate total size of required buffer */
+		totlen = sizeof(struct nfs_export_stat_desc) + (numExports * sizeof(struct nfs_export_stat_rec));
+
+		/* Check caller's buffer */
+		if (oldp == 0) {
+			lck_rw_done(&nfsrv_export_rwlock);
+			/* indicate required buffer len */
+			*oldlenp = totlen;
+			return (0);
+		}
+
+		/* We require the caller's buffer to be at least large enough to hold the descriptor */
+		if (*oldlenp < sizeof(struct nfs_export_stat_desc)) {
+			lck_rw_done(&nfsrv_export_rwlock);
+			/* indicate required buffer len */
+			*oldlenp = totlen;
+			return (ENOMEM);
+		}
+
+		/* indicate required buffer len */
+		*oldlenp = totlen;
+
+		/* check if export table is empty */
+		if (!numExports) {
+			lck_rw_done(&nfsrv_export_rwlock);
+			error = copyout(&stat_desc, oldp, sizeof(struct nfs_export_stat_desc));
+			return (error);
+		}
+
+		/* calculate how many actual export stat records fit into caller's buffer */
+		numRecs = (*oldlenp - sizeof(struct nfs_export_stat_desc)) / sizeof(struct nfs_export_stat_rec);
+
+		if (!numRecs) {
+			/* caller's buffer can only accomodate descriptor */
+			lck_rw_done(&nfsrv_export_rwlock);
+			stat_desc.rec_count = 0;
+			error = copyout(&stat_desc, oldp, sizeof(struct nfs_export_stat_desc));
+			return (error);
+		}
+
+		/* adjust to actual number of records to copyout to caller's buffer */
+		if (numRecs > numExports)
+			numRecs = numExports;
+
+		/* set actual number of records we are returning */
+		stat_desc.rec_count = numRecs;
+
+		/* first copy out the stat descriptor */
+		pos = 0;
+		error = copyout(&stat_desc, oldp + pos, sizeof(struct nfs_export_stat_desc));
+		if (error) {
+			lck_rw_done(&nfsrv_export_rwlock);
+			return (error);
+		}
+		pos += sizeof(struct nfs_export_stat_desc);
+
+		/* Loop through exported directories */
+		count = 0;
+		LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next) {
+			LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next) {
+
+				if (count >= numRecs)
+					break;
+
+				/* build exported filesystem path */
+				snprintf(statrec.path, sizeof(statrec.path), "%s%s%s",
+					nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""),
+					nx->nx_path);
+
+				/* build the 64-bit export stat counters */
+				statrec.ops = ((uint64_t)nx->nx_stats.ops.hi << 32) |
+						nx->nx_stats.ops.lo;
+				statrec.bytes_read = ((uint64_t)nx->nx_stats.bytes_read.hi << 32) |
+						nx->nx_stats.bytes_read.lo;
+				statrec.bytes_written = ((uint64_t)nx->nx_stats.bytes_written.hi << 32) |
+						nx->nx_stats.bytes_written.lo;
+				error = copyout(&statrec, oldp + pos, sizeof(statrec));
+				if (error) {
+					lck_rw_done(&nfsrv_export_rwlock);
+					return (error);
+				}
+				/* advance buffer position */
+				pos += sizeof(statrec);
+			}
+		}
+		lck_rw_done(&nfsrv_export_rwlock);
+		break;
+	case NFS_USERSTATS:
+		/* init structures used for copying out of kernel */
+		ustat_desc.rec_vers = NFS_USER_STAT_REC_VERSION;
+		ustat_rec.rec_type = NFS_USER_STAT_USER_REC;
+		upath_rec.rec_type = NFS_USER_STAT_PATH_REC;
+
+		/* initialize counters */
+		bytes_total = sizeof(struct nfs_user_stat_desc);
+		bytes_avail  = *oldlenp;
+		recs_copied = 0;
+
+		if (!nfsrv_is_initialized()) /* NFS server not initialized, so no stats */
+			goto ustat_skip;
+
+		/* reclaim old expired user nodes */
+		nfsrv_active_user_list_reclaim();
+
+		/* reserve space for the buffer descriptor */
+		if (bytes_avail >= sizeof(struct nfs_user_stat_desc))
+			bytes_avail -= sizeof(struct nfs_user_stat_desc);
+		else
+			bytes_avail = 0;
+
+		/* put buffer position past the buffer descriptor */
+		pos = sizeof(struct nfs_user_stat_desc);
+
+		/* Loop through exported directories */
+		lck_rw_lock_shared(&nfsrv_export_rwlock);
+		LIST_FOREACH(nxfs, &nfsrv_exports, nxfs_next) {
+			LIST_FOREACH(nx, &nxfs->nxfs_exports, nx_next) {
+				/* copy out path */
+				if (bytes_avail >= sizeof(struct nfs_user_stat_path_rec)) {
+					snprintf(upath_rec.path, sizeof(upath_rec.path), "%s%s%s",
+					    nxfs->nxfs_path, ((nxfs->nxfs_path[1] && nx->nx_path[0]) ? "/" : ""),
+					    nx->nx_path);
+
+					error = copyout(&upath_rec, oldp + pos, sizeof(struct nfs_user_stat_path_rec));
+					if (error) {
+						/* punt */
+						goto ustat_done;
+					}
+
+					pos += sizeof(struct nfs_user_stat_path_rec);
+					bytes_avail -= sizeof(struct nfs_user_stat_path_rec);
+					recs_copied++;
+				}
+				else {
+					/* Caller's buffer is exhausted */
+					bytes_avail = 0;
+				}
+
+				bytes_total += sizeof(struct nfs_user_stat_path_rec);
+
+				/* Scan through all user nodes of this export */
+				ulist = &nx->nx_user_list;
+				lck_mtx_lock(&ulist->user_mutex);
+				for (unode = TAILQ_FIRST(&ulist->user_lru); unode; unode = unode_next) {
+					unode_next = TAILQ_NEXT(unode, lru_link);
+
+					/* copy out node if there is space */
+					if (bytes_avail >= sizeof(struct nfs_user_stat_user_rec)) {
+						/* prepare a user stat rec for copying out */
+						ustat_rec.uid = unode->uid;
+						bcopy(&unode->sock, &ustat_rec.sock, unode->sock.ss_len);
+						ustat_rec.ops = unode->ops;
+						ustat_rec.bytes_read = unode->bytes_read;
+						ustat_rec.bytes_written = unode->bytes_written;
+						ustat_rec.tm_start = unode->tm_start;
+						ustat_rec.tm_last = unode->tm_last;
+
+						error = copyout(&ustat_rec, oldp + pos, sizeof(struct nfs_user_stat_user_rec));
+
+						if (error) {
+							/* punt */
+							lck_mtx_unlock(&ulist->user_mutex);
+							goto ustat_done;
+						}
+
+						pos += sizeof(struct nfs_user_stat_user_rec);
+						bytes_avail -= sizeof(struct nfs_user_stat_user_rec);
+						recs_copied++;
+					}
+					else {
+						/* Caller's buffer is exhausted */
+						bytes_avail = 0;
+					}
+					bytes_total += sizeof(struct nfs_user_stat_user_rec);
+				}
+				/* can unlock this export's list now */
+				lck_mtx_unlock(&ulist->user_mutex);
+			}
+		}
+
+ustat_done:
+		/* unlock the export table */
+		lck_rw_done(&nfsrv_export_rwlock);
+
+ustat_skip:
+		/* indicate number of actual records copied */
+		ustat_desc.rec_count = recs_copied;
+
+		if (!error) {
+			/* check if there was enough room for the buffer descriptor */
+			if (*oldlenp >= sizeof(struct nfs_user_stat_desc))
+				error = copyout(&ustat_desc, oldp, sizeof(struct nfs_user_stat_desc));
+			else
+				error = ENOMEM;
+
+			/* always indicate required buffer size */
+			*oldlenp = bytes_total;
+		}
+		break;
+	case NFS_USERCOUNT:
+		if (!oldp) {
+			*oldlenp = sizeof(nfsrv_user_stat_node_count);
+			return (0);
+		}
+
+		if (*oldlenp < sizeof(nfsrv_user_stat_node_count)) {
+			*oldlenp = sizeof(nfsrv_user_stat_node_count);
+			return (ENOMEM);
 		}
-		return 0;
+
+		if (nfsrv_is_initialized()) {
+			/* reclaim old expired user nodes */
+			nfsrv_active_user_list_reclaim();
+		}
+
+		error = copyout(&nfsrv_user_stat_node_count, oldp, sizeof(nfsrv_user_stat_node_count));
+		break;
+#endif /* NFSSERVER */
 	case VFS_CTL_NOLOCKS:
-		val = (nmp->nm_flag & NFSMNT_NOLOCKS) ? 1 : 0;
  		if (req->oldptr != USER_ADDR_NULL) {
+			lck_mtx_lock(&nmp->nm_lock);
+			val = (nmp->nm_flag & NFSMNT_NOLOCKS) ? 1 : 0;
+			lck_mtx_unlock(&nmp->nm_lock);
  			error = SYSCTL_OUT(req, &val, sizeof(val));
  			if (error)
  				return (error);
@@ -1571,34 +2774,52 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
  			error = SYSCTL_IN(req, &val, sizeof(val));
  			if (error)
  				return (error);
-			if (val)
+			lck_mtx_lock(&nmp->nm_lock);
+			if (nmp->nm_flag & NFSMNT_LOCALLOCKS) {
+				/* can't toggle locks when using local locks */
+				error = EINVAL;
+			} else if (val) {
+				if (!(nmp->nm_flag & NFSMNT_NOLOCKS))
+					nfs_lockd_mount_change(-1);
 				nmp->nm_flag |= NFSMNT_NOLOCKS;
-			else
+				nmp->nm_state &= ~NFSSTA_LOCKTIMEO;
+			} else {
+				if (nmp->nm_flag & NFSMNT_NOLOCKS)
+					nfs_lockd_mount_change(1);
 				nmp->nm_flag &= ~NFSMNT_NOLOCKS;
+			}
+			lck_mtx_unlock(&nmp->nm_lock);
  		}
 		break;
 	case VFS_CTL_QUERY:
-		if (nmp->nm_state & NFSSTA_TIMEO)
+		lck_mtx_lock(&nmp->nm_lock);
+		if (nmp->nm_state & (NFSSTA_TIMEO|NFSSTA_JUKEBOXTIMEO))
 			vq.vq_flags |= VQ_NOTRESP;
-		if (!(nmp->nm_flag & NFSMNT_NOLOCKS) &&
+		if (!(nmp->nm_flag & (NFSMNT_NOLOCKS|NFSMNT_LOCALLOCKS)) &&
 		    (nmp->nm_state & NFSSTA_LOCKTIMEO))
-			vq.vq_flags |= VQ_NOTRESPLOCK;
+			vq.vq_flags |= VQ_NOTRESP;
+		lck_mtx_unlock(&nmp->nm_lock);
 		error = SYSCTL_OUT(req, &vq, sizeof(vq));
 		break;
  	case VFS_CTL_TIMEO:
  		if (req->oldptr != USER_ADDR_NULL) {
- 			error = SYSCTL_OUT(req, &nmp->nm_tprintf_initial_delay,
- 			    sizeof(nmp->nm_tprintf_initial_delay));
+			lck_mtx_lock(&nmp->nm_lock);
+			val = nmp->nm_tprintf_initial_delay;
+			lck_mtx_unlock(&nmp->nm_lock);
+ 			error = SYSCTL_OUT(req, &val, sizeof(val));
  			if (error)
  				return (error);
  		}
  		if (req->newptr != USER_ADDR_NULL) {
- 			error = SYSCTL_IN(req, &nmp->nm_tprintf_initial_delay,
- 			    sizeof(nmp->nm_tprintf_initial_delay));
+ 			error = SYSCTL_IN(req, &val, sizeof(val));
  			if (error)
  				return (error);
- 			if (nmp->nm_tprintf_initial_delay < 0)
+			lck_mtx_lock(&nmp->nm_lock);
+ 			if (val < 0)
  				nmp->nm_tprintf_initial_delay = 0;
+			else
+				nmp->nm_tprintf_initial_delay = val;
+			lck_mtx_unlock(&nmp->nm_lock);
  		}
 		break;
 	default:
@@ -1606,4 +2827,3 @@ nfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	}
 	return (error);
 }
-
diff --git a/bsd/nfs/nfs_vnops.c b/bsd/nfs/nfs_vnops.c
index 97ae28c41..47c461076 100644
--- a/bsd/nfs/nfs_vnops.c
+++ b/bsd/nfs/nfs_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -86,7 +92,7 @@
 
 #include <sys/vm.h>
 
-#include <sys/time.h> 
+#include <sys/time.h>
 #include <kern/clock.h>
 #include <libkern/OSAtomic.h>
 
@@ -97,6 +103,7 @@
 #include <nfs/nfsproto.h>
 #include <nfs/nfs.h>
 #include <nfs/nfsnode.h>
+#include <nfs/nfs_gss.h>
 #include <nfs/nfsmount.h>
 #include <nfs/nfs_lock.h>
 #include <nfs/xdr_subs.h>
@@ -109,105 +116,130 @@
 
 #include <kern/task.h>
 #include <kern/sched_prim.h>
-
-#include <sys/kdebug.h>
-
-#define FSDBG(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_NONE, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_TOP(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_START, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-#define FSDBG_BOT(A, B, C, D, E) \
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, (A))) | DBG_FUNC_END, \
-		(int)(B), (int)(C), (int)(D), (int)(E), 0)
-
-static int	nfsspec_read(struct vnop_read_args *);
-static int	nfsspec_write(struct vnop_write_args *);
-static int	nfsfifo_read(struct vnop_read_args *);
-static int	nfsfifo_write(struct vnop_write_args *);
-static int	nfsspec_close(struct vnop_close_args *);
-static int	nfsfifo_close(struct vnop_close_args *);
-static int	nfs_ioctl(struct vnop_ioctl_args *);
-static int	nfs_select(struct vnop_select_args *);
-static int	nfs_setattrrpc(vnode_t,struct vnode_attr *,kauth_cred_t,proc_t);
-static	int	nfs_lookup(struct vnop_lookup_args *);
-static	int	nfs_create(struct vnop_create_args *);
-static	int	nfs_mknod(struct vnop_mknod_args *);
-static	int	nfs_open(struct vnop_open_args *);
-static	int	nfs_close(struct vnop_close_args *);
-static	int	nfs_access(struct vnop_access_args *);
-static	int	nfs_vnop_getattr(struct vnop_getattr_args *);
-static	int	nfs_setattr(struct vnop_setattr_args *);
-static	int	nfs_read(struct vnop_read_args *);
-static	int	nfs_mmap(struct vnop_mmap_args *);
-static	int	nfs_fsync(struct vnop_fsync_args *);
-static	int	nfs_remove(struct vnop_remove_args *);
-static	int	nfs_link(struct vnop_link_args *);
-static	int	nfs_rename(struct vnop_rename_args *);
-static	int	nfs_mkdir(struct vnop_mkdir_args *);
-static	int	nfs_rmdir(struct vnop_rmdir_args *);
-static	int	nfs_symlink(struct vnop_symlink_args *);
-static	int	nfs_readdir(struct vnop_readdir_args *);
-static	int	nfs_lookitup(vnode_t,char *,int,kauth_cred_t,proc_t,struct nfsnode **);
-static	int	nfs_sillyrename(vnode_t,vnode_t,struct componentname *,kauth_cred_t,proc_t);
-static int	nfs_readlink(struct vnop_readlink_args *);
-static int	nfs_pathconf(struct vnop_pathconf_args *);
-static int	nfs_advlock(struct vnop_advlock_args *);
-static	int	nfs_pagein(struct vnop_pagein_args *);
-static	int	nfs_pageout(struct vnop_pageout_args *);
-static	int nfs_blktooff(struct vnop_blktooff_args *);
-static	int nfs_offtoblk(struct vnop_offtoblk_args *);
-static	int nfs_blockmap(struct vnop_blockmap_args *);
+#include <libkern/OSAtomic.h>
 
 /*
- * Global vfs data structures for nfs
+ * NFS vnode ops
  */
+static int	nfs_vnop_lookup(struct vnop_lookup_args *);
+static int	nfsspec_vnop_read(struct vnop_read_args *);
+static int	nfsspec_vnop_write(struct vnop_write_args *);
+static int	nfsspec_vnop_close(struct vnop_close_args *);
+#if FIFO
+static int	nfsfifo_vnop_read(struct vnop_read_args *);
+static int	nfsfifo_vnop_write(struct vnop_write_args *);
+static int	nfsfifo_vnop_close(struct vnop_close_args *);
+#endif
+static int	nfs_vnop_ioctl(struct vnop_ioctl_args *);
+static int	nfs_vnop_select(struct vnop_select_args *);
+static int	nfs_vnop_setattr(struct vnop_setattr_args *);
+static int	nfs_vnop_read(struct vnop_read_args *);
+static int	nfs_vnop_mmap(struct vnop_mmap_args *);
+static int	nfs_vnop_fsync(struct vnop_fsync_args *);
+static int	nfs_vnop_remove(struct vnop_remove_args *);
+static int	nfs_vnop_rename(struct vnop_rename_args *);
+static int	nfs_vnop_readdir(struct vnop_readdir_args *);
+static int	nfs_vnop_readlink(struct vnop_readlink_args *);
+static int	nfs_vnop_pathconf(struct vnop_pathconf_args *);
+static int	nfs_vnop_pagein(struct vnop_pagein_args *);
+static int	nfs_vnop_pageout(struct vnop_pageout_args *);
+static int	nfs_vnop_blktooff(struct vnop_blktooff_args *);
+static int	nfs_vnop_offtoblk(struct vnop_offtoblk_args *);
+static int	nfs_vnop_blockmap(struct vnop_blockmap_args *);
+
+static int	nfs3_vnop_create(struct vnop_create_args *);
+static int	nfs3_vnop_mknod(struct vnop_mknod_args *);
+static int	nfs3_vnop_getattr(struct vnop_getattr_args *);
+static int	nfs3_vnop_link(struct vnop_link_args *);
+static int	nfs3_vnop_mkdir(struct vnop_mkdir_args *);
+static int	nfs3_vnop_rmdir(struct vnop_rmdir_args *);
+static int	nfs3_vnop_symlink(struct vnop_symlink_args *);
+
 vnop_t **nfsv2_vnodeop_p;
 static struct vnodeopv_entry_desc nfsv2_vnodeop_entries[] = {
 	{ &vnop_default_desc, (vnop_t *)vn_default_error },
-	{ &vnop_lookup_desc, (vnop_t *)nfs_lookup },		/* lookup */
-	{ &vnop_create_desc, (vnop_t *)nfs_create },		/* create */
-	{ &vnop_mknod_desc, (vnop_t *)nfs_mknod },		/* mknod */
-	{ &vnop_open_desc, (vnop_t *)nfs_open },		/* open */
-	{ &vnop_close_desc, (vnop_t *)nfs_close },		/* close */
-	{ &vnop_access_desc, (vnop_t *)nfs_access },		/* access */
-	{ &vnop_getattr_desc, (vnop_t *)nfs_vnop_getattr },	/* getattr */
-	{ &vnop_setattr_desc, (vnop_t *)nfs_setattr },		/* setattr */
-	{ &vnop_read_desc, (vnop_t *)nfs_read },		/* read */
-	{ &vnop_write_desc, (vnop_t *)nfs_write },		/* write */
-	{ &vnop_ioctl_desc, (vnop_t *)nfs_ioctl },		/* ioctl */
-	{ &vnop_select_desc, (vnop_t *)nfs_select },		/* select */
-	{ &vnop_revoke_desc, (vnop_t *)nfs_revoke },		/* revoke */
-	{ &vnop_mmap_desc, (vnop_t *)nfs_mmap },		/* mmap */
-	{ &vnop_fsync_desc, (vnop_t *)nfs_fsync },		/* fsync */
-	{ &vnop_remove_desc, (vnop_t *)nfs_remove },		/* remove */
-	{ &vnop_link_desc, (vnop_t *)nfs_link },		/* link */
-	{ &vnop_rename_desc, (vnop_t *)nfs_rename },		/* rename */
-	{ &vnop_mkdir_desc, (vnop_t *)nfs_mkdir },		/* mkdir */
-	{ &vnop_rmdir_desc, (vnop_t *)nfs_rmdir },		/* rmdir */
-	{ &vnop_symlink_desc, (vnop_t *)nfs_symlink },		/* symlink */
-	{ &vnop_readdir_desc, (vnop_t *)nfs_readdir },		/* readdir */
-	{ &vnop_readlink_desc, (vnop_t *)nfs_readlink },	/* readlink */
-	{ &vnop_inactive_desc, (vnop_t *)nfs_inactive },	/* inactive */
-	{ &vnop_reclaim_desc, (vnop_t *)nfs_reclaim },		/* reclaim */
+	{ &vnop_lookup_desc, (vnop_t *)nfs_vnop_lookup },	/* lookup */
+	{ &vnop_create_desc, (vnop_t *)nfs3_vnop_create },	/* create */
+	{ &vnop_mknod_desc, (vnop_t *)nfs3_vnop_mknod },	/* mknod */
+	{ &vnop_open_desc, (vnop_t *)nfs3_vnop_open },		/* open */
+	{ &vnop_close_desc, (vnop_t *)nfs3_vnop_close },	/* close */
+	{ &vnop_access_desc, (vnop_t *)nfs_vnop_access },	/* access */
+	{ &vnop_getattr_desc, (vnop_t *)nfs3_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfs_vnop_read },		/* read */
+	{ &vnop_write_desc, (vnop_t *)nfs_vnop_write },		/* write */
+	{ &vnop_ioctl_desc, (vnop_t *)nfs_vnop_ioctl },		/* ioctl */
+	{ &vnop_select_desc, (vnop_t *)nfs_vnop_select },	/* select */
+	{ &vnop_revoke_desc, (vnop_t *)nfs_vnop_revoke },	/* revoke */
+	{ &vnop_mmap_desc, (vnop_t *)nfs_vnop_mmap },		/* mmap */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
+	{ &vnop_remove_desc, (vnop_t *)nfs_vnop_remove },	/* remove */
+	{ &vnop_link_desc, (vnop_t *)nfs3_vnop_link },		/* link */
+	{ &vnop_rename_desc, (vnop_t *)nfs_vnop_rename },	/* rename */
+	{ &vnop_mkdir_desc, (vnop_t *)nfs3_vnop_mkdir },	/* mkdir */
+	{ &vnop_rmdir_desc, (vnop_t *)nfs3_vnop_rmdir },	/* rmdir */
+	{ &vnop_symlink_desc, (vnop_t *)nfs3_vnop_symlink },	/* symlink */
+	{ &vnop_readdir_desc, (vnop_t *)nfs_vnop_readdir },	/* readdir */
+	{ &vnop_readlink_desc, (vnop_t *)nfs_vnop_readlink },	/* readlink */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
 	{ &vnop_strategy_desc, (vnop_t *)err_strategy },	/* strategy */
-	{ &vnop_pathconf_desc, (vnop_t *)nfs_pathconf },	/* pathconf */
-	{ &vnop_advlock_desc, (vnop_t *)nfs_advlock },		/* advlock */
+	{ &vnop_pathconf_desc, (vnop_t *)nfs_vnop_pathconf },	/* pathconf */
+	{ &vnop_advlock_desc, (vnop_t *)nfs3_vnop_advlock },	/* advlock */
 	{ &vnop_bwrite_desc, (vnop_t *)err_bwrite },		/* bwrite */
-	{ &vnop_pagein_desc, (vnop_t *)nfs_pagein },		/* Pagein */
-	{ &vnop_pageout_desc, (vnop_t *)nfs_pageout },		/* Pageout */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
 	{ &vnop_copyfile_desc, (vnop_t *)err_copyfile },	/* Copyfile */
-	{ &vnop_blktooff_desc, (vnop_t *)nfs_blktooff },	/* blktooff */
-	{ &vnop_offtoblk_desc, (vnop_t *)nfs_offtoblk },	/* offtoblk */
-	{ &vnop_blockmap_desc, (vnop_t *)nfs_blockmap },	/* blockmap */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
 	{ NULL, NULL }
 };
 struct vnodeopv_desc nfsv2_vnodeop_opv_desc =
 	{ &nfsv2_vnodeop_p, nfsv2_vnodeop_entries };
-#ifdef __FreeBSD__
-VNODEOP_SET(nfsv2_vnodeop_opv_desc);
-#endif
+
+vnop_t **nfsv4_vnodeop_p;
+static struct vnodeopv_entry_desc nfsv4_vnodeop_entries[] = {
+	{ &vnop_default_desc, (vnop_t *)vn_default_error },
+	{ &vnop_lookup_desc, (vnop_t *)nfs_vnop_lookup },	/* lookup */
+	{ &vnop_create_desc, (vnop_t *)nfs4_vnop_create },	/* create */
+	{ &vnop_mknod_desc, (vnop_t *)nfs4_vnop_mknod },	/* mknod */
+	{ &vnop_open_desc, (vnop_t *)nfs4_vnop_open },		/* open */
+	{ &vnop_close_desc, (vnop_t *)nfs4_vnop_close },	/* close */
+	{ &vnop_access_desc, (vnop_t *)nfs_vnop_access },	/* access */
+	{ &vnop_getattr_desc, (vnop_t *)nfs4_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfs_vnop_read },		/* read */
+	{ &vnop_write_desc, (vnop_t *)nfs_vnop_write },		/* write */
+	{ &vnop_ioctl_desc, (vnop_t *)nfs_vnop_ioctl },		/* ioctl */
+	{ &vnop_select_desc, (vnop_t *)nfs_vnop_select },	/* select */
+	{ &vnop_revoke_desc, (vnop_t *)nfs_vnop_revoke },	/* revoke */
+	{ &vnop_mmap_desc, (vnop_t *)nfs_vnop_mmap },		/* mmap */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
+	{ &vnop_remove_desc, (vnop_t *)nfs_vnop_remove },	/* remove */
+	{ &vnop_link_desc, (vnop_t *)nfs4_vnop_link },		/* link */
+	{ &vnop_rename_desc, (vnop_t *)nfs_vnop_rename },	/* rename */
+	{ &vnop_mkdir_desc, (vnop_t *)nfs4_vnop_mkdir },	/* mkdir */
+	{ &vnop_rmdir_desc, (vnop_t *)nfs4_vnop_rmdir },	/* rmdir */
+	{ &vnop_symlink_desc, (vnop_t *)nfs4_vnop_symlink },	/* symlink */
+	{ &vnop_readdir_desc, (vnop_t *)nfs_vnop_readdir },	/* readdir */
+	{ &vnop_readlink_desc, (vnop_t *)nfs_vnop_readlink },	/* readlink */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
+	{ &vnop_strategy_desc, (vnop_t *)err_strategy },	/* strategy */
+	{ &vnop_pathconf_desc, (vnop_t *)nfs_vnop_pathconf },	/* pathconf */
+	{ &vnop_advlock_desc, (vnop_t *)nfs4_vnop_advlock },	/* advlock */
+	{ &vnop_bwrite_desc, (vnop_t *)err_bwrite },		/* bwrite */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
+	{ &vnop_copyfile_desc, (vnop_t *)err_copyfile },	/* Copyfile */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
+	{ NULL, NULL }
+};
+struct vnodeopv_desc nfsv4_vnodeop_opv_desc =
+	{ &nfsv4_vnodeop_p, nfsv4_vnodeop_entries };
 
 /*
  * Special device vnode ops
@@ -219,16 +251,16 @@ static struct vnodeopv_entry_desc spec_nfsv2nodeop_entries[] = {
 	{ &vnop_create_desc, (vnop_t *)spec_create },		/* create */
 	{ &vnop_mknod_desc, (vnop_t *)spec_mknod },		/* mknod */
 	{ &vnop_open_desc, (vnop_t *)spec_open },		/* open */
-	{ &vnop_close_desc, (vnop_t *)nfsspec_close },		/* close */
-	{ &vnop_getattr_desc, (vnop_t *)nfs_vnop_getattr },	/* getattr */
-	{ &vnop_setattr_desc, (vnop_t *)nfs_setattr },		/* setattr */
-	{ &vnop_read_desc, (vnop_t *)nfsspec_read },		/* read */
-	{ &vnop_write_desc, (vnop_t *)nfsspec_write },		/* write */
+	{ &vnop_close_desc, (vnop_t *)nfsspec_vnop_close },	/* close */
+	{ &vnop_getattr_desc, (vnop_t *)nfs3_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfsspec_vnop_read },	/* read */
+	{ &vnop_write_desc, (vnop_t *)nfsspec_vnop_write },	/* write */
 	{ &vnop_ioctl_desc, (vnop_t *)spec_ioctl },		/* ioctl */
 	{ &vnop_select_desc, (vnop_t *)spec_select },		/* select */
 	{ &vnop_revoke_desc, (vnop_t *)spec_revoke },		/* revoke */
 	{ &vnop_mmap_desc, (vnop_t *)spec_mmap },		/* mmap */
-	{ &vnop_fsync_desc, (vnop_t *)nfs_fsync },		/* fsync */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
 	{ &vnop_remove_desc, (vnop_t *)spec_remove },		/* remove */
 	{ &vnop_link_desc, (vnop_t *)spec_link },		/* link */
 	{ &vnop_rename_desc, (vnop_t *)spec_rename },		/* rename */
@@ -237,25 +269,63 @@ static struct vnodeopv_entry_desc spec_nfsv2nodeop_entries[] = {
 	{ &vnop_symlink_desc, (vnop_t *)spec_symlink },		/* symlink */
 	{ &vnop_readdir_desc, (vnop_t *)spec_readdir },		/* readdir */
 	{ &vnop_readlink_desc, (vnop_t *)spec_readlink },	/* readlink */
-	{ &vnop_inactive_desc, (vnop_t *)nfs_inactive },	/* inactive */
-	{ &vnop_reclaim_desc, (vnop_t *)nfs_reclaim },		/* reclaim */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
 	{ &vnop_strategy_desc, (vnop_t *)spec_strategy },	/* strategy */
 	{ &vnop_pathconf_desc, (vnop_t *)spec_pathconf },	/* pathconf */
 	{ &vnop_advlock_desc, (vnop_t *)spec_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (vnop_t *)vn_bwrite },		/* bwrite */
-	{ &vnop_pagein_desc, (vnop_t *)nfs_pagein },		/* Pagein */
-	{ &vnop_pageout_desc, (vnop_t *)nfs_pageout },		/* Pageout */
-	{ &vnop_blktooff_desc, (vnop_t *)nfs_blktooff },	/* blktooff */
-	{ &vnop_offtoblk_desc, (vnop_t *)nfs_offtoblk },	/* offtoblk */
-	{ &vnop_blockmap_desc, (vnop_t *)nfs_blockmap },	/* blockmap */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
 	{ NULL, NULL }
 };
 struct vnodeopv_desc spec_nfsv2nodeop_opv_desc =
 	{ &spec_nfsv2nodeop_p, spec_nfsv2nodeop_entries };
-#ifdef __FreeBSD__
-VNODEOP_SET(spec_nfsv2nodeop_opv_desc);
-#endif
+vnop_t **spec_nfsv4nodeop_p;
+static struct vnodeopv_entry_desc spec_nfsv4nodeop_entries[] = {
+	{ &vnop_default_desc, (vnop_t *)vn_default_error },
+	{ &vnop_lookup_desc, (vnop_t *)spec_lookup },		/* lookup */
+	{ &vnop_create_desc, (vnop_t *)spec_create },		/* create */
+	{ &vnop_mknod_desc, (vnop_t *)spec_mknod },		/* mknod */
+	{ &vnop_open_desc, (vnop_t *)spec_open },		/* open */
+	{ &vnop_close_desc, (vnop_t *)nfsspec_vnop_close },	/* close */
+	{ &vnop_getattr_desc, (vnop_t *)nfs4_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfsspec_vnop_read },	/* read */
+	{ &vnop_write_desc, (vnop_t *)nfsspec_vnop_write },	/* write */
+	{ &vnop_ioctl_desc, (vnop_t *)spec_ioctl },		/* ioctl */
+	{ &vnop_select_desc, (vnop_t *)spec_select },		/* select */
+	{ &vnop_revoke_desc, (vnop_t *)spec_revoke },		/* revoke */
+	{ &vnop_mmap_desc, (vnop_t *)spec_mmap },		/* mmap */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
+	{ &vnop_remove_desc, (vnop_t *)spec_remove },		/* remove */
+	{ &vnop_link_desc, (vnop_t *)spec_link },		/* link */
+	{ &vnop_rename_desc, (vnop_t *)spec_rename },		/* rename */
+	{ &vnop_mkdir_desc, (vnop_t *)spec_mkdir },		/* mkdir */
+	{ &vnop_rmdir_desc, (vnop_t *)spec_rmdir },		/* rmdir */
+	{ &vnop_symlink_desc, (vnop_t *)spec_symlink },		/* symlink */
+	{ &vnop_readdir_desc, (vnop_t *)spec_readdir },		/* readdir */
+	{ &vnop_readlink_desc, (vnop_t *)spec_readlink },	/* readlink */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
+	{ &vnop_strategy_desc, (vnop_t *)spec_strategy },	/* strategy */
+	{ &vnop_pathconf_desc, (vnop_t *)spec_pathconf },	/* pathconf */
+	{ &vnop_advlock_desc, (vnop_t *)spec_advlock },		/* advlock */
+	{ &vnop_bwrite_desc, (vnop_t *)vn_bwrite },		/* bwrite */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
+	{ NULL, NULL }
+};
+struct vnodeopv_desc spec_nfsv4nodeop_opv_desc =
+	{ &spec_nfsv4nodeop_p, spec_nfsv4nodeop_entries };
 
+#if FIFO
 vnop_t **fifo_nfsv2nodeop_p;
 static struct vnodeopv_entry_desc fifo_nfsv2nodeop_entries[] = {
 	{ &vnop_default_desc, (vnop_t *)vn_default_error },
@@ -263,16 +333,16 @@ static struct vnodeopv_entry_desc fifo_nfsv2nodeop_entries[] = {
 	{ &vnop_create_desc, (vnop_t *)fifo_create },		/* create */
 	{ &vnop_mknod_desc, (vnop_t *)fifo_mknod },		/* mknod */
 	{ &vnop_open_desc, (vnop_t *)fifo_open },		/* open */
-	{ &vnop_close_desc, (vnop_t *)nfsfifo_close },		/* close */
-	{ &vnop_getattr_desc, (vnop_t *)nfs_vnop_getattr },	/* getattr */
-	{ &vnop_setattr_desc, (vnop_t *)nfs_setattr },		/* setattr */
-	{ &vnop_read_desc, (vnop_t *)nfsfifo_read },		/* read */
-	{ &vnop_write_desc, (vnop_t *)nfsfifo_write },		/* write */
+	{ &vnop_close_desc, (vnop_t *)nfsfifo_vnop_close },	/* close */
+	{ &vnop_getattr_desc, (vnop_t *)nfs3_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfsfifo_vnop_read },	/* read */
+	{ &vnop_write_desc, (vnop_t *)nfsfifo_vnop_write },	/* write */
 	{ &vnop_ioctl_desc, (vnop_t *)fifo_ioctl },		/* ioctl */
 	{ &vnop_select_desc, (vnop_t *)fifo_select },		/* select */
 	{ &vnop_revoke_desc, (vnop_t *)fifo_revoke },		/* revoke */
 	{ &vnop_mmap_desc, (vnop_t *)fifo_mmap },		/* mmap */
-	{ &vnop_fsync_desc, (vnop_t *)nfs_fsync },		/* fsync */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
 	{ &vnop_remove_desc, (vnop_t *)fifo_remove },		/* remove */
 	{ &vnop_link_desc, (vnop_t *)fifo_link },		/* link */
 	{ &vnop_rename_desc, (vnop_t *)fifo_rename },		/* rename */
@@ -281,438 +351,364 @@ static struct vnodeopv_entry_desc fifo_nfsv2nodeop_entries[] = {
 	{ &vnop_symlink_desc, (vnop_t *)fifo_symlink },		/* symlink */
 	{ &vnop_readdir_desc, (vnop_t *)fifo_readdir },		/* readdir */
 	{ &vnop_readlink_desc, (vnop_t *)fifo_readlink },	/* readlink */
-	{ &vnop_inactive_desc, (vnop_t *)nfs_inactive },	/* inactive */
-	{ &vnop_reclaim_desc, (vnop_t *)nfs_reclaim },		/* reclaim */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
 	{ &vnop_strategy_desc, (vnop_t *)fifo_strategy },	/* strategy */
 	{ &vnop_pathconf_desc, (vnop_t *)fifo_pathconf },	/* pathconf */
 	{ &vnop_advlock_desc, (vnop_t *)fifo_advlock },		/* advlock */
 	{ &vnop_bwrite_desc, (vnop_t *)vn_bwrite },		/* bwrite */
-	{ &vnop_pagein_desc, (vnop_t *)nfs_pagein },		/* Pagein */
-	{ &vnop_pageout_desc, (vnop_t *)nfs_pageout },		/* Pageout */
-	{ &vnop_blktooff_desc, (vnop_t *)nfs_blktooff },	/* blktooff */
-	{ &vnop_offtoblk_desc, (vnop_t *)nfs_offtoblk },	/* offtoblk */
-	{ &vnop_blockmap_desc, (vnop_t *)nfs_blockmap },	/* blockmap */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
 	{ NULL, NULL }
 };
 struct vnodeopv_desc fifo_nfsv2nodeop_opv_desc =
 	{ &fifo_nfsv2nodeop_p, fifo_nfsv2nodeop_entries };
-#ifdef __FreeBSD__
-VNODEOP_SET(fifo_nfsv2nodeop_opv_desc);
-#endif
-
-static int	nfs_mknodrpc(vnode_t dvp, vnode_t *vpp,
-				struct componentname *cnp,
-				struct vnode_attr *vap,
-				kauth_cred_t cred, proc_t p);
-static int	nfs_removerpc(vnode_t dvp, char *name, int namelen,
-				kauth_cred_t cred, proc_t proc);
-static int	nfs_renamerpc(vnode_t fdvp, char *fnameptr,
-				int fnamelen, vnode_t tdvp,
-				char *tnameptr, int tnamelen,
-				kauth_cred_t cred, proc_t proc);
-
-/*
- * Global variables
- */
-extern u_long nfs_xdrneg1;
-extern u_long nfs_true, nfs_false;
-extern struct nfsstats nfsstats;
-extern nfstype nfsv3_type[9];
-proc_t nfs_iodwant[NFS_MAXASYNCDAEMON];
-struct nfsmount *nfs_iodmount[NFS_MAXASYNCDAEMON];
-
-lck_grp_t *nfs_iod_lck_grp;
-lck_grp_attr_t *nfs_iod_lck_grp_attr;
-lck_attr_t *nfs_iod_lck_attr;
-lck_mtx_t *nfs_iod_mutex;
-
-int nfs_numasync = 0;
-int nfs_ioddelwri = 0;
-
-#define	DIRHDSIZ	(sizeof (struct dirent) - (MAXNAMLEN + 1))
-
-static int	nfsaccess_cache_timeout = NFS_MAXATTRTIMO;
-/* SYSCTL_INT(_vfs_nfs, OID_AUTO, access_cache_timeout, CTLFLAG_RW,
-           &nfsaccess_cache_timeout, 0, "NFS ACCESS cache timeout");
-*/
-#define	NFSV3ACCESS_ALL (NFSV3ACCESS_READ | NFSV3ACCESS_MODIFY		\
-			 | NFSV3ACCESS_EXTEND | NFSV3ACCESS_EXECUTE	\
-			 | NFSV3ACCESS_DELETE | NFSV3ACCESS_LOOKUP)
-                         
-
-/* 
- * the following are needed only by nfs_pageout to know how to handle errors
- * see nfs_pageout comments on explanation of actions.
- * the errors here are copied from errno.h and errors returned by servers
- * are expected to match the same numbers here. If not, our actions maybe
- * erroneous.
- */
-enum actiontype {NOACTION, DUMP, DUMPANDLOG, RETRY, RETRYWITHSLEEP, SEVER};
-
-static int errorcount[ELAST+1]; /* better be zeros when initialized */
-
-static const short errortooutcome[ELAST+1] = {
-	NOACTION,
-	DUMP,			/* EPERM	1	Operation not permitted */
-	DUMP,			/* ENOENT	2	No such file or directory */
-	DUMPANDLOG,		/* ESRCH	3	No such process */
-	RETRY,			/* EINTR 	4	Interrupted system call */
-	DUMP,			/* EIO		5	Input/output error */
-	DUMP,			/* ENXIO	6	Device not configured */
-	DUMPANDLOG,		/* E2BIG	7	Argument list too long */
-	DUMPANDLOG,		/* ENOEXEC	8	Exec format error */
-	DUMPANDLOG,		/* EBADF	9	Bad file descriptor */
-	DUMPANDLOG,		/* ECHILD	10	No child processes */
-	DUMPANDLOG,		/* EDEADLK	11	Resource deadlock avoided - was EAGAIN */
-	RETRY,			/* ENOMEM	12	Cannot allocate memory */
-	DUMP,			/* EACCES	13	Permission denied */
-	DUMPANDLOG,		/* EFAULT	14	Bad address */
-	DUMPANDLOG,		/* ENOTBLK	15	POSIX - Block device required */
-	RETRY,			/* EBUSY	16	Device busy */
-	DUMP,			/* EEXIST	17	File exists */
-	DUMP,			/* EXDEV	18	Cross-device link */
-	DUMP,			/* ENODEV	19	Operation not supported by device */
-	DUMP,			/* ENOTDIR	20	Not a directory */
-	DUMP,			/* EISDIR 	21	Is a directory */
-	DUMP,			/* EINVAL	22	Invalid argument */
-	DUMPANDLOG,		/* ENFILE	23	Too many open files in system */
-	DUMPANDLOG,		/* EMFILE	24	Too many open files */
-	DUMPANDLOG,		/* ENOTTY	25	Inappropriate ioctl for device */
-	DUMPANDLOG,		/* ETXTBSY	26	Text file busy - POSIX */
-	DUMP,			/* EFBIG	27	File too large */
-	DUMP,			/* ENOSPC	28	No space left on device */
-	DUMPANDLOG,		/* ESPIPE	29	Illegal seek */
-	DUMP,			/* EROFS	30	Read-only file system */
-	DUMP,			/* EMLINK	31	Too many links */
-	RETRY,			/* EPIPE	32	Broken pipe */
-	/* math software */
-	DUMPANDLOG,		/* EDOM				33	Numerical argument out of domain */
-	DUMPANDLOG,		/* ERANGE			34	Result too large */
-	RETRY,			/* EAGAIN/EWOULDBLOCK	35	Resource temporarily unavailable */
-	DUMPANDLOG,		/* EINPROGRESS		36	Operation now in progress */
-	DUMPANDLOG,		/* EALREADY			37	Operation already in progress */
-	/* ipc/network software -- argument errors */
-	DUMPANDLOG,		/* ENOTSOC			38	Socket operation on non-socket */
-	DUMPANDLOG,		/* EDESTADDRREQ		39	Destination address required */
-	DUMPANDLOG,		/* EMSGSIZE			40	Message too long */
-	DUMPANDLOG,		/* EPROTOTYPE		41	Protocol wrong type for socket */
-	DUMPANDLOG,		/* ENOPROTOOPT		42	Protocol not available */
-	DUMPANDLOG,		/* EPROTONOSUPPORT	43	Protocol not supported */
-	DUMPANDLOG,		/* ESOCKTNOSUPPORT	44	Socket type not supported */
-	DUMPANDLOG,		/* ENOTSUP			45	Operation not supported */
-	DUMPANDLOG,		/* EPFNOSUPPORT		46	Protocol family not supported */
-	DUMPANDLOG,		/* EAFNOSUPPORT		47	Address family not supported by protocol family */
-	DUMPANDLOG,		/* EADDRINUSE		48	Address already in use */
-	DUMPANDLOG,		/* EADDRNOTAVAIL	49	Can't assign requested address */
-	/* ipc/network software -- operational errors */
-	RETRY,			/* ENETDOWN			50	Network is down */
-	RETRY,			/* ENETUNREACH		51	Network is unreachable */
-	RETRY,			/* ENETRESET		52	Network dropped connection on reset */
-	RETRY,			/* ECONNABORTED		53	Software caused connection abort */
-	RETRY,			/* ECONNRESET		54	Connection reset by peer */
-	RETRY,			/* ENOBUFS			55	No buffer space available */
-	RETRY,			/* EISCONN			56	Socket is already connected */
-	RETRY,			/* ENOTCONN			57	Socket is not connected */
-	RETRY,			/* ESHUTDOWN		58	Can't send after socket shutdown */
-	RETRY,			/* ETOOMANYREFS		59	Too many references: can't splice */
-	RETRY,			/* ETIMEDOUT		60	Operation timed out */
-	RETRY,			/* ECONNREFUSED		61	Connection refused */
-
-	DUMPANDLOG,		/* ELOOP			62	Too many levels of symbolic links */
-	DUMP,			/* ENAMETOOLONG		63	File name too long */
-	RETRY,			/* EHOSTDOWN		64	Host is down */ 
-	RETRY,			/* EHOSTUNREACH		65	No route to host */
-	DUMP,			/* ENOTEMPTY		66	Directory not empty */
-	/* quotas & mush */     
-	DUMPANDLOG,		/* PROCLIM			67	Too many processes */
-	DUMPANDLOG,		/* EUSERS			68	Too many users */
-	DUMPANDLOG,		/* EDQUOT			69	Disc quota exceeded */   
-	/* Network File System */
-	DUMP,			/* ESTALE			70	Stale NFS file handle */
-	DUMP,			/* EREMOTE			71	Too many levels of remote in path */
-	DUMPANDLOG,		/* EBADRPC			72	RPC struct is bad */
-	DUMPANDLOG,		/* ERPCMISMATCH		73	RPC version wrong */
-	DUMPANDLOG,		/* EPROGUNAVAIL		74	RPC prog. not avail */
-	DUMPANDLOG,		/* EPROGMISMATCH	75	Program version wrong */
-	DUMPANDLOG,		/* EPROCUNAVAIL		76	Bad procedure for program */
 
-	DUMPANDLOG,		/* ENOLCK			77	No locks available */
-	DUMPANDLOG,		/* ENOSYS			78	Function not implemented */
-	DUMPANDLOG,		/* EFTYPE			79	Inappropriate file type or format */  
-	DUMPANDLOG,		/* EAUTH			80	Authentication error */
-	DUMPANDLOG,		/* ENEEDAUTH		81	Need authenticator */
-	/* Intelligent device errors */
-	DUMPANDLOG,		/* EPWROFF			82	Device power is off */
-	DUMPANDLOG,		/* EDEVERR			83	Device error, e.g. paper out */
-	DUMPANDLOG,		/* EOVERFLOW		84	Value too large to be stored in data type */
-	/* Program loading errors */
-	DUMPANDLOG,		/* EBADEXEC			85	Bad executable */
-	DUMPANDLOG,		/* EBADARCH			86	Bad CPU type in executable */
-	DUMPANDLOG,		/* ESHLIBVERS		87	Shared library version mismatch */
-	DUMPANDLOG,		/* EBADMACHO		88	Malformed Macho file */
+vnop_t **fifo_nfsv4nodeop_p;
+static struct vnodeopv_entry_desc fifo_nfsv4nodeop_entries[] = {
+	{ &vnop_default_desc, (vnop_t *)vn_default_error },
+	{ &vnop_lookup_desc, (vnop_t *)fifo_lookup },		/* lookup */
+	{ &vnop_create_desc, (vnop_t *)fifo_create },		/* create */
+	{ &vnop_mknod_desc, (vnop_t *)fifo_mknod },		/* mknod */
+	{ &vnop_open_desc, (vnop_t *)fifo_open },		/* open */
+	{ &vnop_close_desc, (vnop_t *)nfsfifo_vnop_close },	/* close */
+	{ &vnop_getattr_desc, (vnop_t *)nfs4_vnop_getattr },	/* getattr */
+	{ &vnop_setattr_desc, (vnop_t *)nfs_vnop_setattr },	/* setattr */
+	{ &vnop_read_desc, (vnop_t *)nfsfifo_vnop_read },	/* read */
+	{ &vnop_write_desc, (vnop_t *)nfsfifo_vnop_write },	/* write */
+	{ &vnop_ioctl_desc, (vnop_t *)fifo_ioctl },		/* ioctl */
+	{ &vnop_select_desc, (vnop_t *)fifo_select },		/* select */
+	{ &vnop_revoke_desc, (vnop_t *)fifo_revoke },		/* revoke */
+	{ &vnop_mmap_desc, (vnop_t *)fifo_mmap },		/* mmap */
+	{ &vnop_fsync_desc, (vnop_t *)nfs_vnop_fsync },		/* fsync */
+	{ &vnop_remove_desc, (vnop_t *)fifo_remove },		/* remove */
+	{ &vnop_link_desc, (vnop_t *)fifo_link },		/* link */
+	{ &vnop_rename_desc, (vnop_t *)fifo_rename },		/* rename */
+	{ &vnop_mkdir_desc, (vnop_t *)fifo_mkdir },		/* mkdir */
+	{ &vnop_rmdir_desc, (vnop_t *)fifo_rmdir },		/* rmdir */
+	{ &vnop_symlink_desc, (vnop_t *)fifo_symlink },		/* symlink */
+	{ &vnop_readdir_desc, (vnop_t *)fifo_readdir },		/* readdir */
+	{ &vnop_readlink_desc, (vnop_t *)fifo_readlink },	/* readlink */
+	{ &vnop_inactive_desc, (vnop_t *)nfs_vnop_inactive },	/* inactive */
+	{ &vnop_reclaim_desc, (vnop_t *)nfs_vnop_reclaim },	/* reclaim */
+	{ &vnop_strategy_desc, (vnop_t *)fifo_strategy },	/* strategy */
+	{ &vnop_pathconf_desc, (vnop_t *)fifo_pathconf },	/* pathconf */
+	{ &vnop_advlock_desc, (vnop_t *)fifo_advlock },		/* advlock */
+	{ &vnop_bwrite_desc, (vnop_t *)vn_bwrite },		/* bwrite */
+	{ &vnop_pagein_desc, (vnop_t *)nfs_vnop_pagein },	/* Pagein */
+	{ &vnop_pageout_desc, (vnop_t *)nfs_vnop_pageout },	/* Pageout */
+	{ &vnop_blktooff_desc, (vnop_t *)nfs_vnop_blktooff },	/* blktooff */
+	{ &vnop_offtoblk_desc, (vnop_t *)nfs_vnop_offtoblk },	/* offtoblk */
+	{ &vnop_blockmap_desc, (vnop_t *)nfs_vnop_blockmap },	/* blockmap */
+	{ NULL, NULL }
 };
+struct vnodeopv_desc fifo_nfsv4nodeop_opv_desc =
+	{ &fifo_nfsv4nodeop_p, fifo_nfsv4nodeop_entries };
+#endif /* FIFO */
 
 
-static short
-nfs_pageouterrorhandler(int error)
+static int	nfs_sillyrename(nfsnode_t,nfsnode_t,struct componentname *,vfs_context_t);
+
+/*
+ * Find the slot in the access cache for this UID.
+ * If adding and no existing slot is found, reuse slots in FIFO order.
+ * The index of the next slot to use is kept in the last entry of the n_mode array.
+ */
+int
+nfs_node_mode_slot(nfsnode_t np, uid_t uid, int add)
 {
-	if (error > ELAST) 
-		return(DUMP);
-	else 
-		return(errortooutcome[error]);
+	int slot;
+
+	for (slot=0; slot < NFS_ACCESS_CACHE_SIZE; slot++)
+		if (np->n_modeuid[slot] == uid)
+			break;
+	if (slot == NFS_ACCESS_CACHE_SIZE) {
+		if (!add)
+			return (-1);
+		slot = np->n_mode[NFS_ACCESS_CACHE_SIZE];
+		np->n_mode[NFS_ACCESS_CACHE_SIZE] = (slot + 1) % NFS_ACCESS_CACHE_SIZE;
+	}
+	return (slot);
 }
 
-static int
-nfs3_access_otw(vnode_t vp,  
-		int wmode,
-		proc_t p,
-		kauth_cred_t cred)  
+int
+nfs3_access_rpc(nfsnode_t np, u_long *mode, vfs_context_t ctx)
 {
-	const int v3 = 1;
-	u_long *tl;
-	int error = 0, attrflag;
-
-	mbuf_t mreq, mrep, md, mb, mb2;
-	caddr_t bpos, dpos, cp2;
-	register long t1, t2;
-	register caddr_t cp;
-	u_int32_t rmode;
-	struct nfsnode *np = VTONFS(vp);
+	int error = 0, status, slot;
+	uint32_t access = 0;
 	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
 	struct timeval now;
+	uid_t uid;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(NFS_VER3) + NFSX_UNSIGNED);
+	nfsm_chain_add_fh(error, &nmreq, NFS_VER3, np->n_fhp, np->n_fhsize);
+	nfsm_chain_add_32(error, &nmreq, *mode);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_ACCESS, ctx,
+			&nmrep, &xid, &status);
+	nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!error)
+		error = status;
+	nfsm_chain_get_32(error, &nmrep, access);
+	nfsmout_if(error);
 
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED);
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_ACCESS]);   
-	nfsm_fhtom(vp, v3);
-	nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-	*tl = txdr_unsigned(wmode);
-	nfsm_request(vp, NFSPROC_ACCESS, p, cred, &xid);
-	if (mrep) {
-		nfsm_postop_attr_update(vp, 1, attrflag, &xid);
-	}
-	if (!error) {
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		rmode = fxdr_unsigned(u_int32_t, *tl);
-		np->n_mode = rmode;
-		np->n_modeuid = kauth_cred_getuid(cred);
-		microuptime(&now);
-		np->n_modestamp = now.tv_sec;
-	}
-	nfsm_reqdone;
-	return error;
+	uid = kauth_cred_getuid(vfs_context_ucred(ctx));
+	slot = nfs_node_mode_slot(np, uid, 1);
+	np->n_modeuid[slot] = uid;
+	microuptime(&now);
+	np->n_modestamp[slot] = now.tv_sec;
+	np->n_mode[slot] = access;
+
+	/*
+	 * If we asked for DELETE but didn't get it, the server
+	 * may simply not support returning that bit (possible
+	 * on UNIX systems).  So, we'll assume that it is OK,
+	 * and just let any subsequent delete action fail if it
+	 * really isn't deletable.
+	 */
+	if ((*mode & NFS_ACCESS_DELETE) &&
+	    !(np->n_mode[slot] & NFS_ACCESS_DELETE))
+		np->n_mode[slot] |= NFS_ACCESS_DELETE;
+	/* pass back the mode returned with this request */
+	*mode = np->n_mode[slot];
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
 }
 
 /*
- * nfs access vnode op.
- * For nfs version 2, just return ok. File accesses may fail later.
- * For nfs version 3, use the access rpc to check accessibility. If file modes
+ * NFS access vnode op.
+ * For NFS version 2, just return ok. File accesses may fail later.
+ * For NFS version 3+, use the access RPC to check accessibility. If file modes
  * are changed on the server, accesses might still fail later.
  */
-static int
-nfs_access(ap)
+int
+nfs_vnop_access(
 	struct vnop_access_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
-		int a_mode;
+		int a_action;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
-	int error = 0, dorpc;
+	int error = 0, slot, dorpc;
 	u_long mode, wmode;
-	int v3 = NFS_ISV3(vp);
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
+	struct nfsmount *nmp;
+	int nfsvers;
 	struct timeval now;
-	kauth_cred_t cred;
+	uid_t uid;
+
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	if (nfsvers == NFS_VER2) {
+		if ((ap->a_action & KAUTH_VNODE_WRITE_RIGHTS) &&
+		    vfs_isrdonly(vnode_mount(vp)))
+			return (EROFS);
+		return (0);
+	}
 
 	/*
-	 * For nfs v3, do an access rpc, otherwise you are stuck emulating
+	 * For NFS v3, do an access rpc, otherwise you are stuck emulating
 	 * ufs_access() locally using the vattr. This may not be correct,
 	 * since the server may apply other access criteria such as
 	 * client uid-->server uid mapping that we do not know about, but
 	 * this is better than just returning anything that is lying about
 	 * in the cache.
 	 */
-	if (v3) {
-		/*
-		 * Convert KAUTH primitives to NFS access rights.
-		 */
-		mode = 0;
-		if (vnode_isdir(vp)) {
-			/* directory */
-			if (ap->a_action &
-			    (KAUTH_VNODE_LIST_DIRECTORY |
-			    KAUTH_VNODE_READ_EXTATTRIBUTES))
-				mode |= NFSV3ACCESS_READ;
-			if (ap->a_action & KAUTH_VNODE_SEARCH)
-				mode |= NFSV3ACCESS_LOOKUP;
-			if (ap->a_action &
-			    (KAUTH_VNODE_ADD_FILE |
-			    KAUTH_VNODE_ADD_SUBDIRECTORY))
-				mode |= NFSV3ACCESS_MODIFY | NFSV3ACCESS_EXTEND;
-			if (ap->a_action & KAUTH_VNODE_DELETE_CHILD)
-				mode |= NFSV3ACCESS_MODIFY;
-		} else {
-			/* file */
-			if (ap->a_action &
-			    (KAUTH_VNODE_READ_DATA |
-			    KAUTH_VNODE_READ_EXTATTRIBUTES))
-				mode |= NFSV3ACCESS_READ;
-			if (ap->a_action & KAUTH_VNODE_WRITE_DATA)
-				mode |= NFSV3ACCESS_MODIFY | NFSV3ACCESS_EXTEND;
-			if (ap->a_action & KAUTH_VNODE_APPEND_DATA)
-				mode |= NFSV3ACCESS_EXTEND;
-			if (ap->a_action & KAUTH_VNODE_EXECUTE)
-				mode |= NFSV3ACCESS_EXECUTE;
-		}
-		/* common */
-		if (ap->a_action & KAUTH_VNODE_DELETE)
-			mode |= NFSV3ACCESS_DELETE;
-		if (ap->a_action &
-		    (KAUTH_VNODE_WRITE_ATTRIBUTES |
-		    KAUTH_VNODE_WRITE_EXTATTRIBUTES |
-		    KAUTH_VNODE_WRITE_SECURITY))
-			mode |= NFSV3ACCESS_MODIFY;
-		/* XXX this is pretty dubious */
-		if (ap->a_action & KAUTH_VNODE_CHANGE_OWNER)
-			mode |= NFSV3ACCESS_MODIFY;
-
-		/* if caching, always ask for every right */
-		if (nfsaccess_cache_timeout > 0) {
-			wmode = NFSV3ACCESS_READ | NFSV3ACCESS_MODIFY |
-				NFSV3ACCESS_EXTEND | NFSV3ACCESS_EXECUTE |
-				NFSV3ACCESS_DELETE | NFSV3ACCESS_LOOKUP;
-		} else
-			wmode = mode;
-                
-		cred = vfs_context_ucred(ap->a_context);
 
-		/*
-		 * Does our cached result allow us to give a definite yes to
-		 * this request?
-		 */     
-		dorpc = 1;
-		if (NMODEVALID(np)) {
-			microuptime(&now);
-			if ((now.tv_sec < (np->n_modestamp + nfsaccess_cache_timeout)) &&
-			    (kauth_cred_getuid(cred) == np->n_modeuid) &&
-			    ((np->n_mode & mode) == mode)) {
-				/* OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_hits); */
-				dorpc = 0;
-			}
-		}
-		if (dorpc) {
-			/* Either a no, or a don't know.  Go to the wire. */
-			/* OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_misses); */
-			error = nfs3_access_otw(vp, wmode, vfs_context_proc(ap->a_context), cred);
-		}
-		if (!error) {
-			/*
-			 * If we asked for DELETE but didn't get it, the server
-			 * may simply not support returning that bit (possible
-			 * on UNIX systems).  So, we'll assume that it is OK,
-			 * and just let any subsequent delete action fail if it
-			 * really isn't deletable.
-			 */
-			if ((mode & NFSV3ACCESS_DELETE) &&
-			    !(np->n_mode & NFSV3ACCESS_DELETE))
-				np->n_mode |= NFSV3ACCESS_DELETE;
-			if ((np->n_mode & mode) != mode)
-				error = EACCES;
-		}
+	/*
+	 * Convert KAUTH primitives to NFS access rights.
+	 */
+	mode = 0;
+	if (vnode_isdir(vp)) {
+		/* directory */
+		if (ap->a_action &
+		    (KAUTH_VNODE_LIST_DIRECTORY |
+		    KAUTH_VNODE_READ_EXTATTRIBUTES))
+			mode |= NFS_ACCESS_READ;
+		if (ap->a_action & KAUTH_VNODE_SEARCH)
+			mode |= NFS_ACCESS_LOOKUP;
+		if (ap->a_action &
+		    (KAUTH_VNODE_ADD_FILE |
+		    KAUTH_VNODE_ADD_SUBDIRECTORY))
+			mode |= NFS_ACCESS_MODIFY | NFS_ACCESS_EXTEND;
+		if (ap->a_action & KAUTH_VNODE_DELETE_CHILD)
+			mode |= NFS_ACCESS_MODIFY;
 	} else {
-		/* v2 */
-		if ((ap->a_action & KAUTH_VNODE_WRITE_RIGHTS) && vfs_isrdonly(vnode_mount(vp))) {
-			error = EROFS;
-		} else {
-			error = 0;
+		/* file */
+		if (ap->a_action &
+		    (KAUTH_VNODE_READ_DATA |
+		    KAUTH_VNODE_READ_EXTATTRIBUTES))
+			mode |= NFS_ACCESS_READ;
+		if (ap->a_action & KAUTH_VNODE_WRITE_DATA)
+			mode |= NFS_ACCESS_MODIFY | NFS_ACCESS_EXTEND;
+		if (ap->a_action & KAUTH_VNODE_APPEND_DATA)
+			mode |= NFS_ACCESS_EXTEND;
+		if (ap->a_action & KAUTH_VNODE_EXECUTE)
+			mode |= NFS_ACCESS_EXECUTE;
+	}
+	/* common */
+	if (ap->a_action & KAUTH_VNODE_DELETE)
+		mode |= NFS_ACCESS_DELETE;
+	if (ap->a_action &
+	    (KAUTH_VNODE_WRITE_ATTRIBUTES |
+	    KAUTH_VNODE_WRITE_EXTATTRIBUTES |
+	    KAUTH_VNODE_WRITE_SECURITY))
+		mode |= NFS_ACCESS_MODIFY;
+	/* XXX this is pretty dubious */
+	if (ap->a_action & KAUTH_VNODE_CHANGE_OWNER)
+		mode |= NFS_ACCESS_MODIFY;
+
+	/* if caching, always ask for every right */
+	if (nfs_access_cache_timeout > 0) {
+		wmode = NFS_ACCESS_READ | NFS_ACCESS_MODIFY |
+			NFS_ACCESS_EXTEND | NFS_ACCESS_EXECUTE |
+			NFS_ACCESS_DELETE | NFS_ACCESS_LOOKUP;
+	} else {
+		wmode = mode;
+	}
+
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
+
+	/*
+	 * Does our cached result allow us to give a definite yes to
+	 * this request?
+	 */
+	uid = kauth_cred_getuid(vfs_context_ucred(ctx));
+	slot = nfs_node_mode_slot(np, uid, 0);
+	dorpc = 1;
+	if (NMODEVALID(np, slot)) {
+		microuptime(&now);
+		if ((now.tv_sec < (np->n_modestamp[slot] + nfs_access_cache_timeout)) &&
+		    ((np->n_mode[slot] & mode) == mode)) {
+			/* OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_hits); */
+			dorpc = 0;
+			wmode = np->n_mode[slot];
 		}
 	}
+	if (dorpc) {
+		/* Either a no, or a don't know.  Go to the wire. */
+		/* OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_misses); */
+		error = nmp->nm_funcs->nf_access_rpc(np, &wmode, ctx);
+	}
+	if (!error && ((wmode & mode) != mode))
+		error = EACCES;
+	nfs_unlock(np);
 
 	return (error);
 }
 
 /*
- * nfs open vnode op
- * Check to see if the type is ok
- * and that deletion is not in progress.
- * For paged in text files, you will need to flush the page cache
- * if consistency is lost.
+ * NFS open vnode op
  */
-/* ARGSUSED */
-
-static int
-nfs_open(ap)
+int
+nfs3_vnop_open(
 	struct vnop_open_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_mode;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
+	struct nfsmount *nmp;
 	struct nfs_vattr nvattr;
-	kauth_cred_t cred;
-	proc_t p;
 	enum vtype vtype;
-	int error;
+	int error, nfsvers;
+
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
 	vtype = vnode_vtype(vp);
-	if (vtype != VREG && vtype != VDIR && vtype != VLNK) {
+	if ((vtype != VREG) && (vtype != VDIR) && (vtype != VLNK))
 		return (EACCES);
-	}
-
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
-
+	if (ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 0);
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	if (np->n_flag & NNEEDINVALIDATE) {
 		np->n_flag &= ~NNEEDINVALIDATE;
-		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cred, p, 1);
+		nfs_unlock(np);
+		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, ctx, 1);
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+			return (error);
 	}
 	if (np->n_flag & NMODIFIED) {
-		if ((error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1)) == EINTR)
+		nfs_unlock(np);
+		if ((error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1)) == EINTR)
+			return (error);
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
 			return (error);
-		NATTRINVALIDATE(np);
 		if (vtype == VDIR)
 			np->n_direofoffset = 0;
-		error = nfs_getattr(vp, &nvattr, cred, p);
-		if (error)
+		NATTRINVALIDATE(np); /* For Open/Close consistency */
+		error = nfs_getattr(np, &nvattr, ctx, 1);
+		if (error) {
+			nfs_unlock(np);
 			return (error);
+		}
 		if (vtype == VDIR) {
 			/* if directory changed, purge any name cache entries */
-			if (nfstimespeccmp(&np->n_ncmtime, &nvattr.nva_mtime, !=))
+			if (NFS_CHANGED_NC(nfsvers, np, &nvattr)) {
+				np->n_flag &= ~NNEGNCENTRIES;
 				cache_purge(vp);
-			np->n_ncmtime = nvattr.nva_mtime;
+			}
+			NFS_CHANGED_UPDATE_NC(nfsvers, np, &nvattr);
 		}
-		np->n_mtime = nvattr.nva_mtime;
+		NFS_CHANGED_UPDATE(nfsvers, np, &nvattr);
 	} else {
-		error = nfs_getattr(vp, &nvattr, cred, p);
-		if (error)
+		NATTRINVALIDATE(np); /* For Open/Close consistency */
+		error = nfs_getattr(np, &nvattr, ctx, 1);
+		if (error) {
+			nfs_unlock(np);
 			return (error);
-		if (nfstimespeccmp(&np->n_mtime, &nvattr.nva_mtime, !=)) {
+		}
+		if (NFS_CHANGED(nfsvers, np, &nvattr)) {
 			if (vtype == VDIR) {
 				np->n_direofoffset = 0;
-				nfs_invaldir(vp);
+				nfs_invaldir(np);
 				/* purge name cache entries */
-				if (nfstimespeccmp(&np->n_ncmtime, &nvattr.nva_mtime, !=))
+				if (NFS_CHANGED_NC(nfsvers, np, &nvattr)) {
+					np->n_flag &= ~NNEGNCENTRIES;
 					cache_purge(vp);
+				}
 			}
-			if ((error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1)) == EINTR)
+			nfs_unlock(np);
+			if ((error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1)) == EINTR)
+				return (error);
+			if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
 				return (error);
 			if (vtype == VDIR)
-				np->n_ncmtime = nvattr.nva_mtime;
-			np->n_mtime = nvattr.nva_mtime;
+				NFS_CHANGED_UPDATE_NC(nfsvers, np, &nvattr);
+			NFS_CHANGED_UPDATE(nfsvers, np, &nvattr);
 		}
 	}
-	NATTRINVALIDATE(np); /* For Open/Close consistency */
+	nfs_unlock(np);
 	return (0);
 }
 
 /*
- * nfs close vnode op
+ * NFS close vnode op
  * What an NFS client should do upon close after writing is a debatable issue.
  * Most NFS clients push delayed writes to the server upon close, basically for
  * two reasons:
@@ -733,180 +729,135 @@ nfs_open(ap)
  * for NFS Version 3 - flush dirty buffers to the server but don't invalidate
  *                     them.
  */
-/* ARGSUSED */
-static int
-nfs_close(ap)
+int
+nfs3_vnop_close(
 	struct vnop_close_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_fflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
 	struct nfsmount *nmp;
-	kauth_cred_t cred;
-	proc_t p;
+	int nfsvers;
 	int error = 0;
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	if (vnode_vtype(vp) != VREG)
+		return (0);
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
-	if (vnode_vtype(vp) == VREG) {
-#if DIAGNOSTIC
-	    register struct sillyrename *sp = np->n_sillyrename;
-	    if (sp)
-                kprintf("nfs_close: %s, dvp=%x, vp=%x, ap=%x, np=%x, sp=%x\n",
-                	&sp->s_name[0], (unsigned)(sp->s_dvp), (unsigned)vp,
-                	(unsigned)ap, (unsigned)np, (unsigned)sp);
-#endif
-	    nmp = VFSTONFS(vnode_mount(vp));
-	    if (!nmp)
-	   	return (ENXIO);
-	    if (np->n_flag & NNEEDINVALIDATE) {
+	if (ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 0);
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
+	if (np->n_flag & NNEEDINVALIDATE) {
 		np->n_flag &= ~NNEEDINVALIDATE;
-		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cred, p, 1);
-	    }
-	    if (np->n_flag & NMODIFIED) {
-		if (NFS_ISV3(vp)) {
-		    error = nfs_flush(vp, MNT_WAIT, cred, p, 0);
-                    /*
-                     * We cannot clear the NMODIFIED bit in np->n_flag due to
-                     * potential races with other processes
-		     * NMODIFIED is a hint
-                     */
-		    /* np->n_flag &= ~NMODIFIED; */
-		} else {
-		    error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-		}
+		nfs_unlock(np);
+		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, ctx, 1);
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+			return (error);
+	}
+	if (np->n_flag & NMODIFIED) {
+		nfs_unlock(np);
+		if (nfsvers != NFS_VER2)
+			error = nfs_flush(np, MNT_WAIT, vfs_context_thread(ctx), 0);
+		else
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+		if (error)
+			return (error);
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
 		NATTRINVALIDATE(np);
-	    }
-	    if (np->n_flag & NWRITEERR) {
+	}
+	if (np->n_flag & NWRITEERR) {
 		np->n_flag &= ~NWRITEERR;
 		error = np->n_error;
-	    }
 	}
+	nfs_unlock(np);
 	return (error);
 }
 
 
 int
-nfs_getattr_no_vnode(
+nfs3_getattr_rpc(
+	nfsnode_t np,
 	mount_t mp,
 	u_char *fhp,
-	int fhsize,
-	kauth_cred_t cred,
-	proc_t p,
+	size_t fhsize,
+	vfs_context_t ctx,
 	struct nfs_vattr *nvap,
 	u_int64_t *xidp)
 {
-	mbuf_t mreq, mrep, md, mb, mb2;
-	caddr_t bpos, dpos;
-	int t2;
-	u_long *tl;
-	caddr_t cp;
-	struct nfsmount *nmp = VFSTONFS(mp);
-	int v3 = (nmp->nm_flag & NFSMNT_NFSV3);
-	int hsiz;
-	int error = 0;
+	struct nfsmount *nmp = mp ? VFSTONFS(mp) : NFSTONMP(np);
+	int error = 0, status, nfsvers;
+	struct nfsm_chain nmreq, nmrep;
 
-	// XXX fix this to use macros once the macros get cleaned up
-	//nfsm_reqhead(NFSX_FH(v3));
-		hsiz = NFSX_FH(v3);
-		mb = NULL;
-		if (hsiz >= nfs_mbuf_minclsize)
-			error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mb);
-		else
-			error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mb);
-		if (error)
-			return (error);
-		bpos = mbuf_data(mb);
-		mreq = mb;
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_GETATTR]);
-	//nfsm_fhtom(vp, v3);
-	      if (v3) {
-			t2 = nfsm_rndup(fhsize) + NFSX_UNSIGNED;
-			if (t2 <= mbuf_trailingspace(mb)) {
-				nfsm_build(tl, u_long *, t2);
-				*tl++ = txdr_unsigned(fhsize);
-				*(tl + ((t2>>2) - 2)) = 0;
-				bcopy((caddr_t)fhp,(caddr_t)tl, fhsize);
-			} else if ((t2 = nfsm_strtmbuf(&mb, &bpos, (caddr_t)fhp, fhsize))) {
-				error = t2;
-				mbuf_freem(mreq);
-				goto nfsmout;
-			}
-		} else {
-			nfsm_build(cp, caddr_t, NFSX_V2FH);
-			bcopy((caddr_t)fhp, cp, NFSX_V2FH);
-		}
-	//nfsm_request(vp, NFSPROC_GETATTR, p, cred, xidp);
-		if ((error = nfs_request(NULL, mp, mreq, NFSPROC_GETATTR, p, cred, &mrep, &md, &dpos, xidp))) {
-			if (error & NFSERR_RETERR)
-				error &= ~NFSERR_RETERR;
-			else
-				goto nfsmout;
-		}
-	if (!error) {
-		//nfsm_loadattr(vp, nvap, xidp);
-		error = nfs_parsefattr(&md, &dpos, v3, nvap);
-		if (error) {
-			mbuf_freem(mrep);
-			goto nfsmout;
-		}
-	}
-	nfsm_reqdone;
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(nfsvers));
+	if (nfsvers != NFS_VER2)
+		nfsm_chain_add_32(error, &nmreq, fhsize);
+	nfsm_chain_add_opaque(error, &nmreq, fhp, fhsize);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, mp, &nmreq, NFSPROC_GETATTR, ctx,
+			&nmrep, xidp, &status);
+	if (!error)
+		error = status;
+	nfsmout_if(error);
+	error = nfs_parsefattr(&nmrep, nfsvers, nvap);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
-/*
- * nfs getattr call from vfs.
- */
+
 int
-nfs_getattr(
-	vnode_t vp,
-	struct nfs_vattr *nvap,
-	kauth_cred_t cred,
-	proc_t p)
+nfs_getattr(nfsnode_t np, struct nfs_vattr *nvap, vfs_context_t ctx, int alreadylocked)
 {
-	struct nfsnode *np = VTONFS(vp);
-	caddr_t cp;
-	u_long *tl;
-	int t1, t2;
-	caddr_t bpos, dpos;
-	int error = 0;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
+	struct nfsmount *nmp;
+	int error = 0, lockerror = ENOENT, nfsvers, avoidfloods;
 	u_int64_t xid;
-	int avoidfloods;
 
 	FSDBG_TOP(513, np->n_size, np, np->n_vattr.nva_size, np->n_flag);
 
-	/*
-	 * Update local times for special files.
-	 */
-	if (np->n_flag & (NACC | NUPD))
+	/* Update local times for special files. */
+	if (np->n_flag & (NACC | NUPD)) {
+		if (!alreadylocked)
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
 		np->n_flag |= NCHG;
+		if (!alreadylocked)
+			nfs_unlock(np);
+	}
+	/* Update size, if necessary */
+	if (!alreadylocked && ISSET(np->n_flag, NUPDATESIZE))
+		nfs_data_update_size(np, 0);
+
 	/*
 	 * First look in the cache.
 	 */
-	if ((error = nfs_getattrcache(vp, nvap)) == 0) {
-		FSDBG_BOT(513, np->n_size, 0, np->n_vattr.nva_size, np->n_flag);
-		return (0);
-	}
-	if (error != ENOENT) {
-		FSDBG_BOT(513, np->n_size, error, np->n_vattr.nva_size,
-			  np->n_flag);
-		return (error);
-	}
+	if ((error = nfs_getattrcache(np, nvap, alreadylocked)) == 0)
+		goto nfsmout;
+	if (error != ENOENT)
+		goto nfsmout;
 
-	if (!VFSTONFS(vnode_mount(vp))) {
-		FSDBG_BOT(513, np->n_size, ENXIO, np->n_vattr.nva_size, np->n_flag);
-		return (ENXIO);
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		error = ENXIO;
+		goto nfsmout;
 	}
-	v3 = NFS_ISV3(vp);
-	error = 0;
+	nfsvers = nmp->nm_vers;
 
 	/*
 	 * Try to get both the attributes and access info by making an
@@ -914,67 +865,79 @@ nfs_getattr(
 	 * But don't bother if we aren't caching access info or if the
 	 * attributes returned wouldn't be cached.
 	 */
-	if (v3 && (nfsaccess_cache_timeout > 0) &&
-	    (nfs_attrcachetimeout(vp) > 0)) {
-		/*  OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_misses); */
-		if ((error = nfs3_access_otw(vp, NFSV3ACCESS_ALL, p, cred)))
-			return (error);
-		if ((error = nfs_getattrcache(vp, nvap)) == 0)
-			return (0);
-		if (error != ENOENT)
-			return (error);
-		error = 0;
+	if ((nfsvers != NFS_VER2) && (nfs_access_cache_timeout > 0)) {
+		if (!alreadylocked && ((error = lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))))
+			goto nfsmout;
+		if (nfs_attrcachetimeout(np) > 0) {
+			/*  OSAddAtomic(1, (SInt32*)&nfsstats.accesscache_misses); */
+			u_long mode = NFS_ACCESS_ALL;
+			error = nmp->nm_funcs->nf_access_rpc(np, &mode, ctx);
+			if (error)
+				goto nfsmout;
+			if ((error = nfs_getattrcache(np, nvap, 1)) == 0)
+				goto nfsmout;
+			if (error != ENOENT)
+				goto nfsmout;
+			error = 0;
+		}
+	} else if (!alreadylocked) {
+		error = lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+		nfsmout_if(error);
 	}
 	avoidfloods = 0;
 tryagain:
-	nfsm_reqhead(NFSX_FH(v3));
-	if (error) {
-		FSDBG_BOT(513, np->n_size, error, np->n_vattr.nva_size, np->n_flag);
-		return (error);
+	error = nmp->nm_funcs->nf_getattr_rpc(np, NULL, np->n_fhp, np->n_fhsize, ctx, nvap, &xid);
+	nfsmout_if(error);
+	error = nfs_loadattrcache(np, nvap, &xid, 0);
+	nfsmout_if(error);
+	if (!xid) { /* out-of-order rpc - attributes were dropped */
+		FSDBG(513, -1, np, np->n_xid >> 32, np->n_xid);
+		if (avoidfloods++ < 100)
+			goto tryagain;
+		/* avoidfloods>1 is bizarre.  at 100 pull the plug */
+		panic("nfs_getattr: getattr flood\n");
 	}
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_GETATTR]);
-	nfsm_fhtom(vp, v3);
-	nfsm_request(vp, NFSPROC_GETATTR, p, cred, &xid);
-	if (!error) {
-		nfsm_loadattr(vp, v3, nvap, &xid);
-		if (!xid) { /* out-of-order rpc - attributes were dropped */
-			mbuf_freem(mrep);
-			mrep = NULL;
-			FSDBG(513, -1, np, np->n_xid << 32, np->n_xid);
-			if (avoidfloods++ < 100)
-				goto tryagain;
-			/*
-			 * avoidfloods>1 is bizarre.  at 100 pull the plug
-			 */
-			panic("nfs_getattr: getattr flood\n");
-		}
-		if (nfstimespeccmp(&np->n_mtime, &nvap->nva_mtime, !=)) {
-			enum vtype vtype = vnode_vtype(vp);
-			FSDBG(513, -1, np, -1, vp);
-			if (vtype == VDIR) {
-				nfs_invaldir(vp);
-				/* purge name cache entries */
-				if (nfstimespeccmp(&np->n_ncmtime, &nvap->nva_mtime, !=))
-					cache_purge(vp);
+	if (NFS_CHANGED(nfsvers, np, nvap)) {
+		vnode_t vp = NFSTOV(np);
+		enum vtype vtype = vnode_vtype(vp);
+		FSDBG(513, -1, np, -1, np);
+		if (vtype == VDIR) {
+			nfs_invaldir(np);
+			/* purge name cache entries */
+			if (NFS_CHANGED_NC(nfsvers, np, nvap)) {
+				np->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(vp);
 			}
-			error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
+		}
+		if (!alreadylocked) {
+			nfs_unlock(np);
+			lockerror = ENOENT;
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
 			FSDBG(513, -1, np, -2, error);
+			if (!error)
+				error = lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
 			if (!error) {
 				if (vtype == VDIR)
-					np->n_ncmtime = nvap->nva_mtime;
-				np->n_mtime = nvap->nva_mtime;
+					NFS_CHANGED_UPDATE_NC(nfsvers, np, nvap);
+				NFS_CHANGED_UPDATE(nfsvers, np, nvap);
 			}
+		} else {
+			/* invalidate later */
+			np->n_flag |= NNEEDINVALIDATE;
 		}
 	}
-	nfsm_reqdone;
-
-	FSDBG_BOT(513, np->n_size, -1, np->n_vattr.nva_size, error);
+nfsmout:
+	if (!lockerror)
+		nfs_unlock(np);
+	FSDBG_BOT(513, np->n_size, error, np->n_vattr.nva_size, np->n_flag);
 	return (error);
 }
 
-
+/*
+ * NFS getattr call from vfs.
+ */
 static int
-nfs_vnop_getattr(
+nfs3_vnop_getattr(
 	struct vnop_getattr_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -985,107 +948,92 @@ nfs_vnop_getattr(
 	int error;
 	struct nfs_vattr nva;
 	struct vnode_attr *vap = ap->a_vap;
+	dev_t rdev;
 
-	error = nfs_getattr(ap->a_vp, &nva,
-		vfs_context_ucred(ap->a_context),
-		vfs_context_proc(ap->a_context));
+	error = nfs_getattr(VTONFS(ap->a_vp), &nva, ap->a_context, 0);
 	if (error)
 		return (error);
 
 	/* copy nva to *a_vap */
- 	VATTR_RETURN(vap, va_type, nva.nva_type);
- 	VATTR_RETURN(vap, va_mode, nva.nva_mode);
- 	VATTR_RETURN(vap, va_rdev, nva.nva_rdev);
- 	VATTR_RETURN(vap, va_uid, nva.nva_uid);
- 	VATTR_RETURN(vap, va_gid, nva.nva_gid);
- 	VATTR_RETURN(vap, va_nlink, nva.nva_nlink);
- 	VATTR_RETURN(vap, va_fileid, nva.nva_fileid);
- 	VATTR_RETURN(vap, va_data_size, nva.nva_size);
- 	VATTR_RETURN(vap, va_data_alloc, nva.nva_bytes);
- 	VATTR_RETURN(vap, va_iosize, nva.nva_blocksize);  /* should this just be f_iosize? */
- 	VATTR_RETURN(vap, va_fsid, nva.nva_fsid);
- 	vap->va_access_time.tv_sec = nva.nva_atime.tv_sec;
- 	vap->va_access_time.tv_nsec = nva.nva_atime.tv_nsec;
- 	VATTR_SET_SUPPORTED(vap, va_access_time);
- 	vap->va_modify_time.tv_sec = nva.nva_mtime.tv_sec;
- 	vap->va_modify_time.tv_nsec = nva.nva_mtime.tv_nsec;
- 	VATTR_SET_SUPPORTED(vap, va_modify_time);
- 	vap->va_change_time.tv_sec = nva.nva_ctime.tv_sec;
- 	vap->va_change_time.tv_nsec = nva.nva_ctime.tv_nsec;
- 	VATTR_SET_SUPPORTED(vap, va_change_time);
+	VATTR_RETURN(vap, va_type, nva.nva_type);
+	VATTR_RETURN(vap, va_mode, nva.nva_mode);
+	rdev = makedev(nva.nva_rawdev.specdata1, nva.nva_rawdev.specdata2);
+	VATTR_RETURN(vap, va_rdev, rdev);
+	VATTR_RETURN(vap, va_uid, nva.nva_uid);
+	VATTR_RETURN(vap, va_gid, nva.nva_gid);
+	VATTR_RETURN(vap, va_nlink, nva.nva_nlink);
+	VATTR_RETURN(vap, va_fileid, nva.nva_fileid);
+	VATTR_RETURN(vap, va_data_size, nva.nva_size);
+	VATTR_RETURN(vap, va_data_alloc, nva.nva_bytes);
+	VATTR_RETURN(vap, va_iosize, nfs_iosize);
+	vap->va_access_time.tv_sec = nva.nva_timesec[NFSTIME_ACCESS];
+	vap->va_access_time.tv_nsec = nva.nva_timensec[NFSTIME_ACCESS];
+	VATTR_SET_SUPPORTED(vap, va_access_time);
+	vap->va_modify_time.tv_sec = nva.nva_timesec[NFSTIME_MODIFY];
+	vap->va_modify_time.tv_nsec = nva.nva_timensec[NFSTIME_MODIFY];
+	VATTR_SET_SUPPORTED(vap, va_modify_time);
+	vap->va_change_time.tv_sec = nva.nva_timesec[NFSTIME_CHANGE];
+	vap->va_change_time.tv_nsec = nva.nva_timensec[NFSTIME_CHANGE];
+	VATTR_SET_SUPPORTED(vap, va_change_time);
 
+	// VATTR_RETURN(vap, va_encoding, 0xffff /* kTextEncodingUnknown */);
 	return (error);
 }
 
 /*
- * nfs setattr call.
+ * NFS setattr call.
  */
 static int
-nfs_setattr(ap)
+nfs_vnop_setattr(
 	struct vnop_setattr_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
 	struct nfsmount *nmp;
 	struct vnode_attr *vap = ap->a_vap;
 	int error = 0;
-	int biosize;
-	u_quad_t tsize;
-	kauth_cred_t cred;
-	proc_t p;
-
-#ifndef nolint
-	tsize = (u_quad_t)0;
-#endif
-	nmp = VFSTONFS(vnode_mount(vp));
+	int biosize, nfsvers;
+	u_quad_t origsize;
+	struct nfs_dulookup dul;
+	nfsnode_t dnp = NULL;
+	vnode_t dvp = NULL;
+	const char *vname = NULL;
+
+	nmp = VTONMP(vp);
 	if (!nmp)
 		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 	biosize = nmp->nm_biosize;
 
-	/* Setting of flags is not supported. */
-	if (VATTR_IS_ACTIVE(vap, va_flags))
-		return (ENOTSUP);
-
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
-
-	VATTR_SET_SUPPORTED(vap, va_mode);
-	VATTR_SET_SUPPORTED(vap, va_uid);
-	VATTR_SET_SUPPORTED(vap, va_gid);
-	VATTR_SET_SUPPORTED(vap, va_data_size);
-	VATTR_SET_SUPPORTED(vap, va_access_time);
-	VATTR_SET_SUPPORTED(vap, va_modify_time);
-
 	/* Disallow write attempts if the filesystem is mounted read-only. */
-	if ((VATTR_IS_ACTIVE(vap, va_flags) || VATTR_IS_ACTIVE(vap, va_mode) ||
-	     VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid) ||
-	     VATTR_IS_ACTIVE(vap, va_access_time) ||
-	     VATTR_IS_ACTIVE(vap, va_modify_time)) &&
-	    vnode_vfsisrdonly(vp))
+	if (vnode_vfsisrdonly(vp))
 		return (EROFS);
 
+	origsize = np->n_size;
 	if (VATTR_IS_ACTIVE(vap, va_data_size)) {
- 		switch (vnode_vtype(vp)) {
- 		case VDIR:
- 			return (EISDIR);
- 		case VCHR:
- 		case VBLK:
- 		case VSOCK:
- 		case VFIFO:
+		switch (vnode_vtype(vp)) {
+		case VDIR:
+			return (EISDIR);
+		case VCHR:
+		case VBLK:
+		case VSOCK:
+		case VFIFO:
 			if (!VATTR_IS_ACTIVE(vap, va_modify_time) &&
 			    !VATTR_IS_ACTIVE(vap, va_access_time) &&
 			    !VATTR_IS_ACTIVE(vap, va_mode) &&
 			    !VATTR_IS_ACTIVE(vap, va_uid) &&
-			    !VATTR_IS_ACTIVE(vap, va_gid))
+			    !VATTR_IS_ACTIVE(vap, va_gid)) {
 				return (0);
+			}
 			VATTR_CLEAR_ACTIVE(vap, va_data_size);
- 			break;
- 		default:
+			break;
+		default:
 			/*
 			 * Disallow write attempts if the filesystem is
 			 * mounted read-only.
@@ -1094,28 +1042,31 @@ nfs_setattr(ap)
 				return (EROFS);
 			FSDBG_TOP(512, np->n_size, vap->va_data_size,
 				  np->n_vattr.nva_size, np->n_flag);
-			if (np->n_flag & NMODIFIED) {
- 				if (vap->va_data_size == 0)
- 					error = nfs_vinvalbuf(vp, 0, cred, p, 1);
- 				else
- 					error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-	 			if (error) {
-					printf("nfs_setattr: nfs_vinvalbuf %d\n", error);
-					FSDBG_BOT(512, np->n_size, vap->va_data_size,
-						  np->n_vattr.nva_size, -1);
- 					return (error);
-				}
-			} else if (np->n_size > vap->va_data_size) { /* shrinking? */
+			/* clear NNEEDINVALIDATE, if set */
+			if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+				return (error);
+			if (np->n_flag & NNEEDINVALIDATE)
+				np->n_flag &= ~NNEEDINVALIDATE;
+			nfs_unlock(np);
+			/* flush everything */
+			error = nfs_vinvalbuf(vp, (vap->va_data_size ? V_SAVE : 0) , ctx, 1);
+			if (error) {
+				printf("nfs_setattr: nfs_vinvalbuf %d\n", error);
+				FSDBG_BOT(512, np->n_size, vap->va_data_size, np->n_vattr.nva_size, -1);
+				return (error);
+			}
+			nfs_data_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+			if (np->n_size > vap->va_data_size) { /* shrinking? */
 				daddr64_t obn, bn;
 				int neweofoff, mustwrite;
 				struct nfsbuf *bp;
 
 				obn = (np->n_size - 1) / biosize;
-				bn = vap->va_data_size / biosize; 
+				bn = vap->va_data_size / biosize;
 				for ( ; obn >= bn; obn--) {
-					if (!nfs_buf_is_incore(vp, obn))
+					if (!nfs_buf_is_incore(np, obn))
 						continue;
-					error = nfs_buf_get(vp, obn, biosize, 0, NBLK_READ, &bp);
+					error = nfs_buf_get(np, obn, biosize, NULL, NBLK_READ, &bp);
 					if (error)
 						continue;
 					if (obn != bn) {
@@ -1127,11 +1078,15 @@ nfs_setattr(ap)
 					mustwrite = 0;
 					neweofoff = vap->va_data_size - NBOFF(bp);
 					/* check for any dirty data before the new EOF */
-					if (bp->nb_dirtyend && bp->nb_dirtyoff < neweofoff) {
+					if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < neweofoff)) {
 						/* clip dirty range to EOF */
-						if (bp->nb_dirtyend > neweofoff)
+						if (bp->nb_dirtyend > neweofoff) {
 							bp->nb_dirtyend = neweofoff;
-						mustwrite++;
+							if (bp->nb_dirtyoff >= bp->nb_dirtyend)
+								bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+						}
+						if ((bp->nb_dirtyend > 0) && (bp->nb_dirtyoff < neweofoff))
+							mustwrite++;
 					}
 					bp->nb_dirty &= (1 << round_page_32(neweofoff)/PAGE_SIZE) - 1;
 					if (bp->nb_dirty)
@@ -1148,6 +1103,7 @@ nfs_setattr(ap)
 					CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL | NB_ASYNC | NB_READ));
 					SET(bp->nb_flags, NB_STABLE | NB_NOCACHE);
 					if (!IS_VALID_CRED(bp->nb_wcred)) {
+						kauth_cred_t cred = vfs_context_ucred(ctx);
 						kauth_cred_ref(cred);
 						bp->nb_wcred = cred;
 					}
@@ -1155,6 +1111,7 @@ nfs_setattr(ap)
 					// Note: bp has been released
 					if (error) {
 						FSDBG(512, bp, 0xd00dee, 0xbad, error);
+						nfs_lock(np, NFS_NODE_LOCK_FORCE);
 						np->n_error = error;
 						np->n_flag |= NWRITEERR;
 						/*
@@ -1165,300 +1122,352 @@ nfs_setattr(ap)
 						 * we may no longer know the correct size)
 						 */
 						NATTRINVALIDATE(np);
-						nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cred, p, 1);
+						nfs_unlock(np);
+						nfs_data_unlock(np);
+						nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, ctx, 1);
+						nfs_data_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
 						error = 0;
 					}
 				}
 			}
- 			tsize = np->n_size;
+			if (vap->va_data_size != np->n_size)
+				ubc_setsize(vp, (off_t)vap->va_data_size); /* XXX error? */
+			origsize = np->n_size;
 			np->n_size = np->n_vattr.nva_size = vap->va_data_size;
-			ubc_setsize(vp, (off_t)vap->va_data_size); /* XXX error? */
-  		}
-	} else if ((VATTR_IS_ACTIVE(vap, va_modify_time) ||
-		    VATTR_IS_ACTIVE(vap, va_access_time)) &&
-		   (np->n_flag & NMODIFIED) && (vnode_vtype(vp) == VREG)) {
-		error = nfs_vinvalbuf(vp, V_SAVE, cred, p, 1);
-		if (error == EINTR)
+			CLR(np->n_flag, NUPDATESIZE);
+			FSDBG(512, np, np->n_size, np->n_vattr.nva_size, 0xf00d0001);
+		}
+	} else if (VATTR_IS_ACTIVE(vap, va_modify_time) ||
+		    VATTR_IS_ACTIVE(vap, va_access_time) ||
+		    (vap->va_vaflags & VA_UTIMES_NULL)) {
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_SHARED)))
 			return (error);
+		if ((np->n_flag & NMODIFIED) && (vnode_vtype(vp) == VREG)) {
+			nfs_unlock(np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			if (error == EINTR)
+				return (error);
+		} else {
+			nfs_unlock(np);
+		}
 	}
-	if (VATTR_IS_ACTIVE(vap, va_mode)) {
+	if (VATTR_IS_ACTIVE(vap, va_mode) ||
+	    VATTR_IS_ACTIVE(vap, va_uid) ||
+	    VATTR_IS_ACTIVE(vap, va_gid)) {
+		if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))) {
+			if (VATTR_IS_ACTIVE(vap, va_data_size))
+				nfs_data_unlock(np);
+			return (error);
+		}
 		NMODEINVALIDATE(np);
+		nfs_unlock(np);
+		dvp = vnode_getparent(vp);
+		vname = vnode_getname(vp);
+		dnp = (dvp && vname) ? VTONFS(dvp) : NULL;
+		if (dnp) {
+			error = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE);
+			if (error) {
+				dnp = NULL;
+				error = 0;
+			}
+		}
+		if (dnp) {
+			nfs_dulookup_init(&dul, dnp, vname, strlen(vname));
+			nfs_dulookup_start(&dul, dnp, ctx);
+		}
+	}
+
+	error = nmp->nm_funcs->nf_setattr_rpc(np, vap, ctx, 0);
+
+	if (VATTR_IS_ACTIVE(vap, va_mode) ||
+	    VATTR_IS_ACTIVE(vap, va_uid) ||
+	    VATTR_IS_ACTIVE(vap, va_gid)) {
+		if (dnp) {
+			nfs_dulookup_finish(&dul, dnp, ctx);
+			nfs_unlock(dnp);
+		}
+		if (dvp != NULLVP)
+			vnode_put(dvp);
+		if (vname != NULL)
+			vnode_putname(vname);
 	}
-	error = nfs_setattrrpc(vp, vap, cred, p);
-	FSDBG_BOT(512, np->n_size, vap->va_data_size, np->n_vattr.nva_size, error);
-	if (error && VATTR_IS_ACTIVE(vap, va_data_size)) {
-		/* make every effort to resync file size w/ server... */
-		int err; /* preserve "error" for return */
 
-		np->n_size = np->n_vattr.nva_size = tsize;
-		ubc_setsize(vp, (off_t)np->n_size); /* XXX check error */
-		vap->va_data_size = tsize;
-		err = nfs_setattrrpc(vp, vap, cred, p);
-		printf("nfs_setattr: nfs_setattrrpc %d %d\n", error, err);
+	FSDBG_BOT(512, np->n_size, vap->va_data_size, np->n_vattr.nva_size, error);
+	if (VATTR_IS_ACTIVE(vap, va_data_size)) {
+		if (error && (origsize != np->n_size)) {
+			/* make every effort to resync file size w/ server... */
+			int err; /* preserve "error" for return */
+			np->n_size = np->n_vattr.nva_size = origsize;
+			CLR(np->n_flag, NUPDATESIZE);
+			FSDBG(512, np, np->n_size, np->n_vattr.nva_size, 0xf00d0002);
+			ubc_setsize(vp, (off_t)np->n_size); /* XXX check error */
+			vap->va_data_size = origsize;
+			err = nmp->nm_funcs->nf_setattr_rpc(np, vap, ctx, 0);
+			if (err)
+				printf("nfs_vnop_setattr: nfs%d_setattr_rpc %d %d\n", nfsvers, error, err);
+		}
+		nfs_data_unlock(np);
 	}
 	return (error);
 }
 
 /*
- * Do an nfs setattr rpc.
+ * Do an NFS setattr RPC.
  */
-static int
-nfs_setattrrpc(vp, vap, cred, procp)
-	vnode_t vp;
-	struct vnode_attr *vap;
-	kauth_cred_t cred;
-	proc_t procp;
+int
+nfs3_setattr_rpc(
+	nfsnode_t np,
+	struct vnode_attr *vap,
+	vfs_context_t ctx,
+	int alreadylocked)
 {
-	register struct nfsv2_sattr *sp;
-	register caddr_t cp;
-	register long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	u_long *tl;
-	int error = 0, wccpostattr = 0;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
+	struct nfsmount *nmp = NFSTONMP(np);
+	int error = 0, lockerror = ENOENT, status, wccpostattr = 0, nfsvers;
 	u_int64_t xid;
-	struct timeval now;
+	struct nfsm_chain nmreq, nmrep;
 
-	if (!VFSTONFS(vnode_mount(vp)))
+	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(vp);
+	nfsvers = nmp->nm_vers;
 
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_SATTR(v3));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_SETATTR]);
-	nfsm_fhtom(vp, v3);
-	if (v3) {
+	VATTR_SET_SUPPORTED(vap, va_mode);
+	VATTR_SET_SUPPORTED(vap, va_uid);
+	VATTR_SET_SUPPORTED(vap, va_gid);
+	VATTR_SET_SUPPORTED(vap, va_data_size);
+	VATTR_SET_SUPPORTED(vap, va_access_time);
+	VATTR_SET_SUPPORTED(vap, va_modify_time);
+
+	if (VATTR_IS_ACTIVE(vap, va_flags)) {
+		if (vap->va_flags) {	/* we don't support setting flags */
+			if (vap->va_active & ~VNODE_ATTR_va_flags)
+				return (EINVAL);	/* return EINVAL if other attributes also set */
+			else
+				return (ENOTSUP);	/* return ENOTSUP for chflags(2) */
+		}
+		/* no flags set, so we'll just ignore it */
+		if (!(vap->va_active & ~VNODE_ATTR_va_flags))
+			return (0); /* no (other) attributes to set, so nothing to do */
+	}
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + NFSX_SATTR(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	if (nfsvers == NFS_VER3) {
 		if (VATTR_IS_ACTIVE(vap, va_mode)) {
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = nfs_true;
-			*tl = txdr_unsigned(vap->va_mode);
+			nfsm_chain_add_32(error, &nmreq, TRUE);
+			nfsm_chain_add_32(error, &nmreq, vap->va_mode);
 		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = nfs_false;
+			nfsm_chain_add_32(error, &nmreq, FALSE);
 		}
 		if (VATTR_IS_ACTIVE(vap, va_uid)) {
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = nfs_true;
-			*tl = txdr_unsigned(vap->va_uid);
+			nfsm_chain_add_32(error, &nmreq, TRUE);
+			nfsm_chain_add_32(error, &nmreq, vap->va_uid);
 		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = nfs_false;
+			nfsm_chain_add_32(error, &nmreq, FALSE);
 		}
 		if (VATTR_IS_ACTIVE(vap, va_gid)) {
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = nfs_true;
-			*tl = txdr_unsigned(vap->va_gid);
+			nfsm_chain_add_32(error, &nmreq, TRUE);
+			nfsm_chain_add_32(error, &nmreq, vap->va_gid);
 		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = nfs_false;
+			nfsm_chain_add_32(error, &nmreq, FALSE);
 		}
 		if (VATTR_IS_ACTIVE(vap, va_data_size)) {
-			nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED);
-			*tl++ = nfs_true;
-			txdr_hyper(&vap->va_data_size, tl);
+			nfsm_chain_add_32(error, &nmreq, TRUE);
+			nfsm_chain_add_64(error, &nmreq, vap->va_data_size);
 		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = nfs_false;
+			nfsm_chain_add_32(error, &nmreq, FALSE);
 		}
-		microtime(&now);
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			if (vap->va_access_time.tv_sec != now.tv_sec) {
-				nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED);
-				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
-				txdr_nfsv3time(&vap->va_access_time, tl);
+		if (vap->va_vaflags & VA_UTIMES_NULL) {
+			nfsm_chain_add_32(error, &nmreq, NFS_TIME_SET_TO_SERVER);
+			nfsm_chain_add_32(error, &nmreq, NFS_TIME_SET_TO_SERVER);
+		} else {
+			if (VATTR_IS_ACTIVE(vap, va_access_time)) {
+				nfsm_chain_add_32(error, &nmreq, NFS_TIME_SET_TO_CLIENT);
+				nfsm_chain_add_32(error, &nmreq, vap->va_access_time.tv_sec);
+				nfsm_chain_add_32(error, &nmreq, vap->va_access_time.tv_nsec);
 			} else {
-				nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-				*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER);
+				nfsm_chain_add_32(error, &nmreq, NFS_TIME_DONT_CHANGE);
 			}
-		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
-		}
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			if (vap->va_modify_time.tv_sec != now.tv_sec) {
-				nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED);
-				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
-				txdr_nfsv3time(&vap->va_modify_time, tl);
+			if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
+				nfsm_chain_add_32(error, &nmreq, NFS_TIME_SET_TO_CLIENT);
+				nfsm_chain_add_32(error, &nmreq, vap->va_modify_time.tv_sec);
+				nfsm_chain_add_32(error, &nmreq, vap->va_modify_time.tv_nsec);
 			} else {
-				nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-				*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER);
+				nfsm_chain_add_32(error, &nmreq, NFS_TIME_DONT_CHANGE);
 			}
-		} else {
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-			*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
 		}
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-		*tl = nfs_false;
+		nfsm_chain_add_32(error, &nmreq, FALSE);
 	} else {
-		struct timespec neg1time = { -1, -1 };
-		nfsm_build(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		if (VATTR_IS_ACTIVE(vap, va_mode))
-			sp->sa_mode = vtonfsv2_mode(vnode_vtype(vp), vap->va_mode);
-		else
-			sp->sa_mode = nfs_xdrneg1;
-		if (VATTR_IS_ACTIVE(vap, va_uid))
-			sp->sa_uid = txdr_unsigned(vap->va_uid);
-		else
-			sp->sa_uid = nfs_xdrneg1;
-		if (VATTR_IS_ACTIVE(vap, va_gid))
-			sp->sa_gid = txdr_unsigned(vap->va_gid);
-		else
-			sp->sa_gid = nfs_xdrneg1;
-		if (VATTR_IS_ACTIVE(vap, va_data_size))
-			sp->sa_size = txdr_unsigned(vap->va_data_size);
-		else
-			sp->sa_size = nfs_xdrneg1;
+		nfsm_chain_add_32(error, &nmreq, VATTR_IS_ACTIVE(vap, va_mode) ?
+			vtonfsv2_mode(vnode_vtype(NFSTOV(np)), vap->va_mode) : -1);
+		nfsm_chain_add_32(error, &nmreq, VATTR_IS_ACTIVE(vap, va_uid) ?
+			vap->va_uid : (uint32_t)-1);
+		nfsm_chain_add_32(error, &nmreq, VATTR_IS_ACTIVE(vap, va_gid) ?
+			vap->va_gid : (uint32_t)-1);
+		nfsm_chain_add_32(error, &nmreq, VATTR_IS_ACTIVE(vap, va_data_size) ?
+			vap->va_data_size : (uint32_t)-1);
 		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			txdr_nfsv2time(&vap->va_access_time, &sp->sa_atime);
+			nfsm_chain_add_32(error, &nmreq, vap->va_access_time.tv_sec);
+			nfsm_chain_add_32(error, &nmreq, (vap->va_access_time.tv_nsec != -1) ?
+				((uint32_t)vap->va_access_time.tv_nsec / 1000) : 0xffffffff);
 		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_atime);
+			nfsm_chain_add_32(error, &nmreq, -1);
+			nfsm_chain_add_32(error, &nmreq, -1);
 		}
 		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			txdr_nfsv2time(&vap->va_modify_time, &sp->sa_mtime);
+			nfsm_chain_add_32(error, &nmreq, vap->va_modify_time.tv_sec);
+			nfsm_chain_add_32(error, &nmreq, (vap->va_modify_time.tv_nsec != -1) ?
+				((uint32_t)vap->va_modify_time.tv_nsec / 1000) : 0xffffffff);
 		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_mtime);
+			nfsm_chain_add_32(error, &nmreq, -1);
+			nfsm_chain_add_32(error, &nmreq, -1);
 		}
 	}
-	nfsm_request(vp, NFSPROC_SETATTR, procp, cred, &xid);
-	if (v3) {
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_SETATTR, ctx,
+			&nmrep, &xid, &status);
+	if (!alreadylocked && ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))))
+		error = lockerror;
+	if (nfsvers == NFS_VER3) {
 		struct timespec premtime = { 0, 0 };
-		if (mrep) {
-			nfsm_wcc_data(vp, &premtime, wccpostattr, &xid);
-		}
+		nfsm_chain_get_wcc_data(error, &nmrep, np, &premtime, &wccpostattr, &xid);
+		nfsmout_if(error);
 		/* if file hadn't changed, update cached mtime */
-		if (nfstimespeccmp(&VTONFS(vp)->n_mtime, &premtime, ==)) {
-			VTONFS(vp)->n_mtime = VTONFS(vp)->n_vattr.nva_mtime;
-		}
+		if (nfstimespeccmp(&np->n_mtime, &premtime, ==))
+			NFS_CHANGED_UPDATE(nfsvers, np, &np->n_vattr);
 		/* if directory hadn't changed, update namecache mtime */
-		if ((vnode_vtype(vp) == VDIR) &&
-		    nfstimespeccmp(&VTONFS(vp)->n_ncmtime, &premtime, ==)) {
-			VTONFS(vp)->n_ncmtime = VTONFS(vp)->n_vattr.nva_mtime;
-		}
+		if ((vnode_vtype(NFSTOV(np)) == VDIR) &&
+		    nfstimespeccmp(&np->n_ncmtime, &premtime, ==))
+			NFS_CHANGED_UPDATE_NC(nfsvers, np, &np->n_vattr);
 		if (!wccpostattr)
-			NATTRINVALIDATE(VTONFS(vp));
+			NATTRINVALIDATE(np);
+		error = status;
 	} else {
-		if (mrep) {
-			nfsm_loadattr(vp, v3, NULL, &xid);
-		}
+		if (!error)
+			error = status;
+		nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
 	}
-	nfsm_reqdone;
+nfsmout:
+	if (!alreadylocked && !lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
 /*
- * nfs lookup call, one step at a time...
+ * NFS lookup call, one step at a time...
  * First look in cache
- * If not found, unlock the directory nfsnode and do the rpc
+ * If not found, unlock the directory nfsnode and do the RPC
  */
 static int
-nfs_lookup(ap)
+nfs_vnop_lookup(
 	struct vnop_lookup_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
 		vnode_t *a_vpp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	struct componentname *cnp = ap->a_cnp;
 	vnode_t dvp = ap->a_dvp;
 	vnode_t *vpp = ap->a_vpp;
 	int flags = cnp->cn_flags;
 	vnode_t newvp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	long len;
-	u_char *fhp;
-	struct nfsnode *dnp, *np;
-	int wantparent, error, attrflag, dattrflag, fhsize, fhisdvp;
-	int v3 = NFS_ISV3(dvp);
-	u_int64_t xid, dxid;
+	nfsnode_t dnp, np;
+	struct nfsmount *nmp;
+	mount_t mp;
+	int nfsvers, error, lockerror = ENOENT, isdot, isdotdot, negnamecache;
+	u_int64_t xid;
 	struct nfs_vattr nvattr;
-	kauth_cred_t cred;
-	proc_t p;
 	int ngflags;
+	struct vnop_access_args naa;
+	fhandle_t fh;
+	struct nfsreq rq, *req = &rq;
 
 	*vpp = NULLVP;
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
-
-	wantparent = flags & (LOCKPARENT|WANTPARENT);
 	dnp = VTONFS(dvp);
 
-	error = nfs_getattr(dvp, &nvattr, cred, p);
+	mp = vnode_mount(dvp);
+	nmp = VFSTONFS(mp);
+	if (!nmp) {
+		error = ENXIO;
+		goto error_return;
+	}
+	nfsvers = nmp->nm_vers;
+	negnamecache = !(nmp->nm_flag & NFSMNT_NONEGNAMECACHE);
+
+	error = lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE);
+	if (!error)
+		error = nfs_getattr(dnp, &nvattr, ctx, 1);
 	if (error)
 		goto error_return;
-	if (nfstimespeccmp(&dnp->n_ncmtime, &nvattr.nva_mtime, !=)) {
+	if (NFS_CHANGED_NC(nfsvers, dnp, &nvattr)) {
 		/*
 		 * This directory has changed on us.
 		 * Purge any name cache entries.
 		 */
+		dnp->n_flag &= ~NNEGNCENTRIES;
 		cache_purge(dvp);
-		dnp->n_ncmtime = nvattr.nva_mtime;
+		NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &nvattr);
 	}
 
 	error = cache_lookup(dvp, vpp, cnp);
 	switch (error) {
 	case ENOENT:
-		/* negative cache entry same as cache miss */
-		error = 0;
-		/* FALLTHROUGH */
+		/* negative cache entry */
+		goto error_return;
 	case 0:
 		/* cache miss */
 		break;
 	case -1:
 		/* cache hit, not really an error */
-	{
-		struct vnop_access_args naa;
-
 		OSAddAtomic(1, (SInt32*)&nfsstats.lookupcache_hits);
 
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
+
 		/* check for directory access */
 		naa.a_vp = dvp;
 		naa.a_action = KAUTH_VNODE_SEARCH;
-		naa.a_context = ap->a_context;
+		naa.a_context = ctx;
 
 		/* compute actual success/failure based on accessibility */
-		error = nfs_access(&naa);
-	}
+		error = nfs_vnop_access(&naa);
 		/* FALLTHROUGH */
 	default:
 		/* unexpected error from cache_lookup */
 		goto error_return;
 	}
-	
-	/* check for lookup of "." */
-	if ((cnp->cn_nameptr[0] == '.') && (cnp->cn_namelen == 1)) {
-		/* skip lookup, we know who we are */
-		fhisdvp = 1;
-		fhp = NULL;
-		fhsize = 0;
-		mrep = NULL;
+
+	/* skip lookup, if we know who we are: "." or ".." */
+	isdot = isdotdot = 0;
+	if (cnp->cn_nameptr[0] == '.') {
+		if (cnp->cn_namelen == 1)
+			isdot = 1;
+		if ((cnp->cn_namelen == 2) && (cnp->cn_nameptr[1] == '.'))
+			isdotdot = 1;
+	}
+	if (isdotdot || isdot) {
+		fh.fh_len = 0;
 		goto found;
 	}
 
 	/* do we know this name is too long? */
-	if (v3) {
-		/* For NFSv3: need uniform pathconf info to test pc_namemax */
-		struct nfsmount *nmp = VFSTONFS(vnode_mount(dvp));
-		if (!nmp) {
-			error = ENXIO;
-			goto error_return;
-		}
-		if (((nmp->nm_state & (NFSSTA_GOTFSINFO|NFSSTA_GOTPATHCONF)) ==
-			(NFSSTA_GOTFSINFO|NFSSTA_GOTPATHCONF)) &&
-		     (nmp->nm_fsinfo.fsproperties & NFSV3FSINFO_HOMOGENEOUS) &&
-		     (cnp->cn_namelen > (long)nmp->nm_fsinfo.namemax)) {
-			error = ENAMETOOLONG;
-			goto error_return;
-		}
-	} else if (cnp->cn_namelen > NFS_MAXNAMLEN) {
+	nmp = VTONMP(dvp);
+	if (!nmp) {
+		error = ENXIO;
+		goto error_return;
+	}
+	if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXNAME) &&
+	     (cnp->cn_namelen > (long)nmp->nm_fsattr.nfsa_maxname)) {
 		error = ENAMETOOLONG;
 		goto error_return;
 	}
@@ -1467,117 +1476,76 @@ nfs_lookup(ap)
 	newvp = NULLVP;
 
 	OSAddAtomic(1, (SInt32*)&nfsstats.lookupcache_misses);
-	len = cnp->cn_namelen;
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED + nfsm_rndup(len));
-	if (error)
-		goto error_return;
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_LOOKUP]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, len, NFS_MAXNAMLEN, v3);
-	/* nfsm_request for NFSv2 causes you to goto to nfsmout upon errors */
-	nfsm_request(dvp, NFSPROC_LOOKUP, p, cred, &xid); 
 
-	if (error) {
-		if (mrep) {
-			nfsm_postop_attr_update(dvp, v3, dattrflag, &xid);
-			mbuf_freem(mrep);
-		}
-		goto nfsmout;
-	}
+	error = nmp->nm_funcs->nf_lookup_rpc_async(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &req);
+	nfsmout_if(error);
+	error = nmp->nm_funcs->nf_lookup_rpc_async_finish(dnp, ctx, req, &xid, &fh, &nvattr);
+	nfsmout_if(error);
 
-	/* get the filehandle */
-	nfsm_getfh(fhp, fhsize, v3);
 	/* is the file handle the same as this directory's file handle? */
-	fhisdvp = NFS_CMPFH(dnp, fhp, fhsize);
-
-	/* get attributes */
-	if (v3) {
-		dxid = xid;
-		nfsm_postop_attr_get(v3, attrflag, &nvattr);
-		nfsm_postop_attr_update(dvp, v3, dattrflag, &dxid);
-		if (!attrflag && (!fhisdvp || !dattrflag)) {
-			/* We need valid attributes in order */
-			/* to call nfs_nget/vnode_create().  */
-			error = nfs_getattr_no_vnode(vnode_mount(dvp),
-					fhp, fhsize, cred, p, &nvattr, &xid);
-			if (error) {
-				mbuf_freem(mrep);
+	isdot = NFS_CMPFH(dnp, fh.fh_data, fh.fh_len);
+
+found:
+
+	if (flags & ISLASTCN) {
+		switch (cnp->cn_nameiop) {
+		case DELETE:
+			cnp->cn_flags &= ~MAKEENTRY;
+			break;
+		case RENAME:
+			cnp->cn_flags &= ~MAKEENTRY;
+			if (isdot) {
+				error = EISDIR;
 				goto error_return;
 			}
+			break;
 		}
-	} else {
-		nfsm_attr_get(v3, &nvattr);
 	}
 
-found:
-
-	/*
-	 * Handle RENAME case...
-	 */
-	if (cnp->cn_nameiop == RENAME && wantparent && (flags & ISLASTCN)) {
-		if (fhisdvp) {
-			mbuf_freem(mrep);
-			error = EISDIR;
-			goto error_return;
-		}
-		error = nfs_nget(vnode_mount(dvp), dvp, cnp, fhp, fhsize,
-				&nvattr, &xid, 0, &np);
-		if (error) {
-			mbuf_freem(mrep);
+	if (isdotdot) {
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
+		newvp = vnode_getparent(dvp);
+		if (!newvp) {
+			error = ENOENT;
 			goto error_return;
 		}
-		*vpp = NFSTOV(np);
-		mbuf_freem(mrep);
-
-		goto error_return;
-	}
-
-	if ((cnp->cn_flags & MAKEENTRY) &&
-	    (cnp->cn_nameiop != DELETE || !(flags & ISLASTCN)))
-		ngflags = NG_MAKEENTRY;
-	else
-		ngflags = 0;
-
-	if (fhisdvp) {
+	} else if (isdot) {
 		error = vnode_get(dvp);
-		if (error) {
-			mbuf_freem(mrep);
+		if (error)
 			goto error_return;
-		}
 		newvp = dvp;
-		/* test fhp to see if we have valid attributes in nvattr */
-		if (fhp && (dnp->n_xid <= xid)) {
-			error = nfs_loadattrcache(dnp, &nvattr, &xid, 0);
-			if (error) {
-				vnode_put(dvp);
-				mbuf_freem(mrep);
-				goto error_return;
-			}
-		}
+		if (fh.fh_len && (dnp->n_xid <= xid))
+			nfs_loadattrcache(dnp, &nvattr, &xid, 0);
 	} else {
-		error = nfs_nget(vnode_mount(dvp), dvp, cnp, fhp, fhsize,
-				&nvattr, &xid, ngflags, &np);
-		if (error) {
-			mbuf_freem(mrep);
+		ngflags = (cnp->cn_flags & MAKEENTRY) ? NG_MAKEENTRY : 0;
+		error = nfs_nget(mp, dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, ngflags, &np);
+		if (error)
 			goto error_return;
-		}
 		newvp = NFSTOV(np);
+		nfs_unlock(np);
 	}
 	*vpp = newvp;
-//	if (error == 0 && *vpp != NULL && *vpp != dvp)
-//		nfs_unlock(VTONFS(*vpp));
 
-	nfsm_reqdone;
+nfsmout:
 	if (error) {
-		if ((cnp->cn_nameiop == CREATE || cnp->cn_nameiop == RENAME) &&
-		    (flags & ISLASTCN) && error == ENOENT) {
+		if (((cnp->cn_nameiop == CREATE) || (cnp->cn_nameiop == RENAME)) &&
+		    (flags & ISLASTCN) && (error == ENOENT)) {
 			if (vnode_mount(dvp) && vnode_vfsisrdonly(dvp))
 				error = EROFS;
 			else
 				error = EJUSTRETURN;
 		}
 	}
+	if ((error == ENOENT) && (cnp->cn_flags & MAKEENTRY) &&
+	    (cnp->cn_nameiop != CREATE) && negnamecache) {
+		/* add a negative entry in the name cache */
+		cache_enter(dvp, NULL, cnp);
+		dnp->n_flag |= NNEGNCENTRIES;
+	}
 error_return:
+	if (!lockerror)
+		nfs_unlock(dnp);
 	if (error && *vpp) {
 	        vnode_put(*vpp);
 		*vpp = NULLVP;
@@ -1586,368 +1554,1156 @@ error_return:
 }
 
 /*
- * nfs read call.
+ * NFS read call.
  * Just call nfs_bioread() to do the work.
  */
 static int
-nfs_read(ap)
+nfs_vnop_read(
 	struct vnop_read_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		int a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	if (vnode_vtype(ap->a_vp) != VREG)
 		return (EPERM);
-	return (nfs_bioread(ap->a_vp, ap->a_uio, ap->a_ioflag,
-		vfs_context_ucred(ap->a_context),
-		vfs_context_proc(ap->a_context)));
+	return (nfs_bioread(VTONFS(ap->a_vp), ap->a_uio, ap->a_ioflag, NULL, ap->a_context));
 }
 
 
 /*
- * nfs readlink call
+ * NFS readlink call
  */
 static int
-nfs_readlink(ap)
+nfs_vnop_readlink(
 	struct vnop_readlink_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	if (vnode_vtype(ap->a_vp) != VLNK)
+	vfs_context_t ctx = ap->a_context;
+	nfsnode_t np = VTONFS(ap->a_vp);
+	struct nfsmount *nmp;
+	int error = 0, lockerror, nfsvers, changed = 0, n;
+	uint32_t buflen;
+	struct uio *uio = ap->a_uio;
+	struct nfs_vattr nvattr;
+	struct nfsbuf *bp = NULL;
+
+	if (vnode_vtype(ap->a_vp) != VLNK)
 		return (EPERM);
-	return (nfs_bioread(ap->a_vp, ap->a_uio, 0,
-		vfs_context_ucred(ap->a_context),
-		vfs_context_proc(ap->a_context)));
+
+	if (uio_uio_resid(uio) == 0)
+		return (0);
+	if (uio->uio_offset < 0)
+		return (EINVAL);
+
+	nmp = VTONMP(ap->a_vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	error = lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+	if (!error)
+		error = nfs_getattr(np, &nvattr, ctx, 1);
+	if (error) {
+		if (!lockerror)
+			nfs_unlock(np);
+		FSDBG(531, np, 0xd1e0001, 0, error);
+		return (error);
+	}
+	if (NFS_CHANGED(nfsvers, np, &nvattr)) {
+		/* link changed, so just ignore NB_CACHE */
+		changed = 1;
+		NFS_CHANGED_UPDATE(nfsvers, np, &nvattr);
+	}
+	nfs_unlock(np);
+
+	OSAddAtomic(1, (SInt32*)&nfsstats.biocache_readlinks);
+	error = nfs_buf_get(np, 0, NFS_MAXPATHLEN, vfs_context_thread(ctx), NBLK_READ, &bp);
+	if (error) {
+		FSDBG(531, np, 0xd1e0002, 0, error);
+		return (error);
+	}
+	if (changed)
+		CLR(bp->nb_flags, NB_CACHE);
+	if (!ISSET(bp->nb_flags, NB_CACHE)) {
+		SET(bp->nb_flags, NB_READ);
+		CLR(bp->nb_flags, NB_DONE);
+		OSAddAtomic(1, (SInt32*)&nfsstats.readlink_bios);
+		buflen = bp->nb_bufsize;
+		error = nmp->nm_funcs->nf_readlink_rpc(np, bp->nb_data, &buflen, ctx);
+		if (error) {
+			SET(bp->nb_flags, NB_ERROR);
+			bp->nb_error = error;
+		} else {
+			bp->nb_validoff = 0;
+			bp->nb_validend = buflen;
+		}
+		nfs_buf_iodone(bp);
+	}
+	if (!error) {
+		// LP64todo - fix this!
+		n = min(uio_uio_resid(uio), bp->nb_validend);
+		if (n > 0)
+			error = uiomove(bp->nb_data, n, uio);
+	}
+	FSDBG(531, np, bp->nb_validend, 0, error);
+	nfs_buf_release(bp, 1);
+	return (error);
 }
 
 /*
- * Do a readlink rpc.
- * Called by nfs_doio() from below the buffer cache.
+ * Do a readlink RPC.
  */
 int
-nfs_readlinkrpc(
-	vnode_t vp,
-	struct uio *uiop,
-	kauth_cred_t cred,
-	proc_t p)
+nfs3_readlink_rpc(nfsnode_t np, char *buf, uint32_t *buflenp, vfs_context_t ctx)
 {
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, len, attrflag;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
+	struct nfsmount *nmp;
+	int error = 0, lockerror = ENOENT, nfsvers, status;
+	uint32_t len;
 	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
 
-	if (!VFSTONFS(vnode_mount(vp)))
+	nmp = NFSTONMP(np);
+	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(vp);
-
-	nfsm_reqhead(NFSX_FH(v3));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_READLINK]);
-	nfsm_fhtom(vp, v3);
-	nfsm_request(vp, NFSPROC_READLINK, p, cred, &xid);
-	if (v3 && mrep)
-		nfsm_postop_attr_update(vp, v3, attrflag, &xid);
-	if (!error) {
-		nfsm_strsiz(len, NFS_MAXPATHLEN, v3);
-		if (len >= NFS_MAXPATHLEN) {
-			struct nfsnode *np = VTONFS(vp);
-#if DIAGNOSTIC
-                        if (!np)
-                                panic("nfs_readlinkrpc: null np");
-#endif  
-                        if (np->n_size && np->n_size < NFS_MAXPATHLEN)
-                                len = np->n_size;
-                }
-		nfsm_mtouio(uiop, len);
-	}
-	nfsm_reqdone;
+	nfsvers = nmp->nm_vers;
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_READLINK, ctx,
+			&nmrep, &xid, &status);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!error)
+		error = status;
+	nfsm_chain_get_32(error, &nmrep, len);
+	nfsmout_if(error);
+	if ((nfsvers == NFS_VER2) && (len > *buflenp)) {
+		error = EBADRPC;
+		goto nfsmout;
+	}
+	if (len >= *buflenp) {
+		if (np->n_size && (np->n_size < *buflenp))
+			len = np->n_size;
+		else
+			len = *buflenp - 1;
+	}
+	nfsm_chain_get_opaque(error, &nmrep, len, buf);
+	if (!error)
+		*buflenp = len;
+nfsmout:
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
 /*
- * nfs read rpc call
+ * NFS read RPC call
  * Ditto above
  */
 int
-nfs_readrpc(
-	vnode_t vp,
-	struct uio *uiop,
-	kauth_cred_t cred,
-	proc_t p)
+nfs_read_rpc(nfsnode_t np, struct uio *uiop, vfs_context_t ctx)
 {
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	mbuf_t mreq, mrep, md, mb, mb2;
 	struct nfsmount *nmp;
-	int error = 0, len, retlen, tsiz, eof = 0, attrflag;
-	int v3, nmrsize;
-	u_int64_t xid;
+	int error = 0, nfsvers, eof = 0;
+	size_t nmrsize, len, retlen, tsiz;
+	off_t txoffset;
+	struct nfsreq rq, *req = &rq;
 
-	FSDBG_TOP(536, vp, uiop->uio_offset, uio_uio_resid(uiop), 0);
-	nmp = VFSTONFS(vnode_mount(vp));
+	FSDBG_TOP(536, np, uiop->uio_offset, uio_uio_resid(uiop), 0);
+	nmp = NFSTONMP(np);
 	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(vp);
+	nfsvers = nmp->nm_vers;
 	nmrsize = nmp->nm_rsize;
 
 	// LP64todo - fix this
 	tsiz = uio_uio_resid(uiop);
-        if (((u_int64_t)uiop->uio_offset + (unsigned int)tsiz > 0xffffffff) && !v3) {
-		FSDBG_BOT(536, vp, uiop->uio_offset, uio_uio_resid(uiop), EFBIG);
+	if (((u_int64_t)uiop->uio_offset + (unsigned int)tsiz > 0xffffffff) && (nfsvers == NFS_VER2)) {
+		FSDBG_BOT(536, np, uiop->uio_offset, uio_uio_resid(uiop), EFBIG);
 		return (EFBIG);
 	}
+
+	txoffset = uiop->uio_offset;
+
 	while (tsiz > 0) {
-		len = (tsiz > nmrsize) ? nmrsize : tsiz;
-		nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED * 3);
+		len = retlen = (tsiz > nmrsize) ? nmrsize : tsiz;
+		FSDBG(536, np, txoffset, len, 0);
+		error = nmp->nm_funcs->nf_read_rpc_async(np, txoffset, len,
+				vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+		if (!error)
+			error = nmp->nm_funcs->nf_read_rpc_async_finish(np, req, uiop, &retlen, &eof);
 		if (error)
 			break;
-		OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_READ]);
-		nfsm_fhtom(vp, v3);
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED * 3);
-		if (v3) {
-			txdr_hyper(&uiop->uio_offset, tl);
-			*(tl + 2) = txdr_unsigned(len);
-		} else {
-			*tl++ = txdr_unsigned(uiop->uio_offset);
-			*tl++ = txdr_unsigned(len);
-			*tl = 0;
+		txoffset += retlen;
+		tsiz -= retlen;
+		if (nfsvers != NFS_VER2) {
+			if (eof || (retlen == 0))
+				tsiz = 0;
+		} else if (retlen < len)
+			tsiz = 0;
+	}
+
+	FSDBG_BOT(536, np, eof, uio_uio_resid(uiop), error);
+	return (error);
+}
+
+int
+nfs3_read_rpc_async(
+	nfsnode_t np,
+	off_t offset,
+	size_t len,
+	thread_t thd,
+	kauth_cred_t cred,
+	struct nfsreq_cbinfo *cb,
+	struct nfsreq **reqp)
+{
+	struct nfsmount *nmp;
+	int error = 0, nfsvers;
+	struct nfsm_chain nmreq;
+
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(nfsvers) + 3 * NFSX_UNSIGNED);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_add_64(error, &nmreq, offset);
+		nfsm_chain_add_32(error, &nmreq, len);
+	} else {
+		nfsm_chain_add_32(error, &nmreq, offset);
+		nfsm_chain_add_32(error, &nmreq, len);
+		nfsm_chain_add_32(error, &nmreq, 0);
+	}
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request_async(np, NULL, &nmreq, NFSPROC_READ, thd, cred, cb, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs3_read_rpc_async_finish(
+	nfsnode_t np,
+	struct nfsreq *req,
+	struct uio *uiop,
+	size_t *lenp,
+	int *eofp)
+{
+	int error = 0, lockerror, nfsvers, status, eof = 0;
+	size_t retlen = 0;
+	uint64_t xid;
+	struct nfsmount *nmp;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		nfs_request_async_cancel(req);
+		return (ENXIO);
+	}
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	if (error == EINPROGRESS) /* async request restarted */
+		return (error);
+
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!error)
+		error = status;
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED);
+		nfsm_chain_get_32(error, &nmrep, eof);
+	} else {
+		nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+	}
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_chain_get_32(error, &nmrep, retlen);
+	if ((nfsvers == NFS_VER2) && (retlen > *lenp))
+		error = EBADRPC;
+	nfsmout_if(error);
+	error = nfsm_chain_get_uio(&nmrep, MIN(retlen, *lenp), uiop);
+	if (eofp) {
+		if (nfsvers == NFS_VER3) {
+			if (!eof && !retlen)
+				eof = 1;
+		} else if (retlen < *lenp) {
+			eof = 1;
 		}
-		FSDBG(536, vp, uiop->uio_offset, len, 0);
-		nfsm_request(vp, NFSPROC_READ, p, cred, &xid);
-		if (v3) {
-			if (mrep) {
-				nfsm_postop_attr_update(vp, v3, attrflag, &xid);
+		*eofp = eof;
+	}
+	*lenp = MIN(retlen, *lenp);
+nfsmout:
+	nfsm_chain_cleanup(&nmrep);
+	return (error);
+}
+
+/*
+ * NFS write call
+ */
+int
+nfs_vnop_write(
+	struct vnop_write_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_vp;
+		struct uio *a_uio;
+		int a_ioflag;
+		vfs_context_t a_context;
+	} */ *ap)
+{
+	vfs_context_t ctx = ap->a_context;
+	struct uio *uio = ap->a_uio;
+	vnode_t vp = ap->a_vp;
+	nfsnode_t np = VTONFS(vp);
+	int ioflag = ap->a_ioflag;
+	struct nfsbuf *bp;
+	struct nfs_vattr nvattr;
+	struct nfsmount *nmp = VTONMP(vp);
+	daddr64_t lbn;
+	int biosize;
+	int n, on, error = 0;
+	off_t boff, start, end;
+	struct iovec_32 iov;
+	struct uio auio;
+	thread_t thd;
+	kauth_cred_t cred;
+
+	FSDBG_TOP(515, np, uio->uio_offset, uio_uio_resid(uio), ioflag);
+
+	if (vnode_vtype(vp) != VREG) {
+		FSDBG_BOT(515, np, uio->uio_offset, uio_uio_resid(uio), EIO);
+		return (EIO);
+	}
+
+	thd = vfs_context_thread(ctx);
+	cred = vfs_context_ucred(ctx);
+
+	nfs_data_lock(np, NFS_NODE_LOCK_SHARED);
+
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))) {
+		nfs_data_unlock(np);
+		FSDBG_BOT(515, np, uio->uio_offset, uio_uio_resid(uio), error);
+		return (error);
+	}
+	np->n_wrbusy++;
+
+	if (np->n_flag & NWRITEERR) {
+		error = np->n_error;
+		np->n_flag &= ~NWRITEERR;
+	}
+	if (np->n_flag & NNEEDINVALIDATE) {
+		np->n_flag &= ~NNEEDINVALIDATE;
+		nfs_unlock(np);
+		nfs_data_unlock(np);
+		nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, ctx, 1);
+		nfs_data_lock(np, NFS_NODE_LOCK_SHARED);
+		if (error || ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE))))
+			goto out;
+	}
+	if (error) {
+		nfs_unlock(np);
+		goto out;
+	}
+
+	biosize = nmp->nm_biosize;
+
+	if (ioflag & (IO_APPEND | IO_SYNC)) {
+		if (np->n_flag & NMODIFIED) {
+			NATTRINVALIDATE(np);
+			nfs_unlock(np);
+			nfs_data_unlock(np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			nfs_data_lock(np, NFS_NODE_LOCK_SHARED);
+			if (error || ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))) {
+				FSDBG(515, np, uio->uio_offset, 0x10bad01, error);
+				goto out;
 			}
-			if (error) {
-				mbuf_freem(mrep);
-				goto nfsmout;
+		}
+		if (ioflag & IO_APPEND) {
+			NATTRINVALIDATE(np);
+			nfs_unlock(np);
+			nfs_data_unlock(np);
+			error = nfs_getattr(np, &nvattr, ctx, 0);
+			/* we'll be extending the file, so take the data lock exclusive */
+			nfs_data_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+			if (error || ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))) {
+				FSDBG(515, np, uio->uio_offset, 0x10bad02, error);
+				goto out;
+			}
+			uio->uio_offset = np->n_size;
+		}
+	}
+	if (uio->uio_offset < 0) {
+		nfs_unlock(np);
+		error = EINVAL;
+		FSDBG_BOT(515, np, uio->uio_offset, 0xbad0ff, error);
+		goto out;
+	}
+	if (uio_uio_resid(uio) == 0) {
+		nfs_unlock(np);
+		goto out;
+	}
+
+	nfs_unlock(np);
+
+	if (((uio->uio_offset + uio_uio_resid(uio)) > (off_t)np->n_size) && !(ioflag & IO_APPEND)) {
+		/* it looks like we'll be extending the file, so take the data lock exclusive */
+		nfs_data_unlock(np);
+		nfs_data_lock(np, NFS_NODE_LOCK_EXCLUSIVE);
+	}
+
+	do {
+		OSAddAtomic(1, (SInt32*)&nfsstats.biocache_writes);
+		lbn = uio->uio_offset / biosize;
+		on = uio->uio_offset % biosize;
+		// LP64todo - fix this
+		n = min((unsigned)(biosize - on), uio_uio_resid(uio));
+again:
+		/*
+		 * Get a cache block for writing.  The range to be written is
+		 * (off..off+n) within the block.  We ensure that the block
+		 * either has no dirty region or that the given range is
+		 * contiguous with the existing dirty region.
+		 */
+		error = nfs_buf_get(np, lbn, biosize, thd, NBLK_WRITE, &bp);
+		if (error)
+			goto out;
+		/* map the block because we know we're going to write to it */
+		NFS_BUF_MAP(bp);
+
+		if (ioflag & IO_NOCACHE)
+			SET(bp->nb_flags, NB_NOCACHE);
+
+		if (!IS_VALID_CRED(bp->nb_wcred)) {
+			kauth_cred_ref(cred);
+			bp->nb_wcred = cred;
+		}
+
+		/*
+		 * If there's already a dirty range AND dirty pages in this block we
+		 * need to send a commit AND write the dirty pages before continuing.
+		 *
+		 * If there's already a dirty range OR dirty pages in this block
+		 * and the new write range is not contiguous with the existing range,
+		 * then force the buffer to be written out now.
+		 * (We used to just extend the dirty range to cover the valid,
+		 * but unwritten, data in between also.  But writing ranges
+		 * of data that weren't actually written by an application
+		 * risks overwriting some other client's data with stale data
+		 * that's just masquerading as new written data.)
+		 */
+		if (bp->nb_dirtyend > 0) {
+		    if (on > bp->nb_dirtyend || (on + n) < bp->nb_dirtyoff || bp->nb_dirty) {
+			FSDBG(515, np, uio->uio_offset, bp, 0xd15c001);
+			/* write/commit buffer "synchronously" */
+			/* (NB_STABLE indicates that data writes should be FILESYNC) */
+			CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
+			SET(bp->nb_flags, (NB_ASYNC | NB_STABLE));
+			error = nfs_buf_write(bp);
+			if (error)
+			    goto out;
+			goto again;
+		    }
+		} else if (bp->nb_dirty) {
+		    int firstpg, lastpg;
+		    u_int32_t pagemask;
+		    /* calculate write range pagemask */
+		    firstpg = on/PAGE_SIZE;
+		    lastpg = (on+n-1)/PAGE_SIZE;
+		    pagemask = ((1 << (lastpg+1)) - 1) & ~((1 << firstpg) - 1);
+		    /* check if there are dirty pages outside the write range */
+		    if (bp->nb_dirty & ~pagemask) {
+			FSDBG(515, np, uio->uio_offset, bp, 0xd15c002);
+			/* write/commit buffer "synchronously" */
+			/* (NB_STABLE indicates that data writes should be FILESYNC) */
+			CLR(bp->nb_flags, (NB_DONE | NB_ERROR | NB_INVAL));
+			SET(bp->nb_flags, (NB_ASYNC | NB_STABLE));
+			error = nfs_buf_write(bp);
+			if (error)
+			    goto out;
+			goto again;
+		    }
+		    /* if the first or last pages are already dirty */
+		    /* make sure that the dirty range encompasses those pages */
+		    if (NBPGDIRTY(bp,firstpg) || NBPGDIRTY(bp,lastpg)) {
+			FSDBG(515, np, uio->uio_offset, bp, 0xd15c003);
+		    	bp->nb_dirtyoff = min(on, firstpg * PAGE_SIZE);
+			if (NBPGDIRTY(bp,lastpg)) {
+			    bp->nb_dirtyend = (lastpg+1) * PAGE_SIZE;
+			    /* clip to EOF */
+			    if (NBOFF(bp) + bp->nb_dirtyend > (off_t)np->n_size) {
+				    bp->nb_dirtyend = np->n_size - NBOFF(bp);
+				    if (bp->nb_dirtyoff >= bp->nb_dirtyend)
+					    bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+			    }
+			} else
+			    bp->nb_dirtyend = on+n;
+		    }
+		}
+
+		/*
+		 * Are we extending the size of the file with this write?
+		 * If so, update file size now that we have the block.
+		 * If there was a partial buf at the old eof, validate
+		 * and zero the new bytes.
+		 */
+		if ((uio->uio_offset + n) > (off_t)np->n_size) {
+			struct nfsbuf *eofbp = NULL;
+			daddr64_t eofbn = np->n_size / biosize;
+			int eofoff = np->n_size % biosize;
+			int neweofoff = (uio->uio_offset + n) % biosize;
+
+			FSDBG(515, 0xb1ffa000, uio->uio_offset + n, eofoff, neweofoff);
+
+			if (eofoff && (eofbn < lbn) &&
+			    ((error = nfs_buf_get(np, eofbn, biosize, thd, NBLK_WRITE|NBLK_ONLYVALID, &eofbp))))
+				goto out;
+
+			/* if we're extending within the same last block */
+			/* and the block is flagged as being cached... */
+			if ((lbn == eofbn) && ISSET(bp->nb_flags, NB_CACHE)) {
+				/* ...check that all pages in buffer are valid */
+				int endpg = ((neweofoff ? neweofoff : biosize) - 1)/PAGE_SIZE;
+				u_int32_t pagemask;
+				/* pagemask only has to extend to last page being written to */
+				pagemask = (1 << (endpg+1)) - 1;
+				FSDBG(515, 0xb1ffa001, bp->nb_valid, pagemask, 0);
+				if ((bp->nb_valid & pagemask) != pagemask) {
+					/* zerofill any hole */
+					if (on > bp->nb_validend) {
+						int i;
+						for (i=bp->nb_validend/PAGE_SIZE; i <= (on - 1)/PAGE_SIZE; i++)
+							NBPGVALID_SET(bp, i);
+						NFS_BUF_MAP(bp);
+						FSDBG(516, bp, bp->nb_validend, on - bp->nb_validend, 0xf01e);
+						bzero((char *)bp->nb_data + bp->nb_validend,
+							on - bp->nb_validend);
+					}
+					/* zerofill any trailing data in the last page */
+					if (neweofoff) {
+						NFS_BUF_MAP(bp);
+						FSDBG(516, bp, neweofoff, PAGE_SIZE - (neweofoff & PAGE_MASK), 0xe0f);
+						bzero((char *)bp->nb_data + neweofoff,
+							PAGE_SIZE - (neweofoff & PAGE_MASK));
+					}
+				}
+			}
+			np->n_size = uio->uio_offset + n;
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			CLR(np->n_flag, NUPDATESIZE);
+			np->n_flag |= NMODIFIED;
+			nfs_unlock(np);
+			FSDBG(516, np, np->n_size, np->n_vattr.nva_size, 0xf00d0001);
+			ubc_setsize(vp, (off_t)np->n_size); /* XXX errors */
+			if (eofbp) {
+				/*
+				 * We may need to zero any previously invalid data
+				 * after the old EOF in the previous EOF buffer.
+				 *
+				 * For the old last page, don't zero bytes if there
+				 * are invalid bytes in that page (i.e. the page isn't
+				 * currently valid).
+				 * For pages after the old last page, zero them and
+				 * mark them as valid.
+				 */
+				char *d;
+				int i;
+				if (ioflag & IO_NOCACHE)
+					SET(eofbp->nb_flags, NB_NOCACHE);
+				NFS_BUF_MAP(eofbp);
+				FSDBG(516, eofbp, eofoff, biosize - eofoff, 0xe0fff01e);
+				d = eofbp->nb_data;
+				i = eofoff/PAGE_SIZE;
+				while (eofoff < biosize) {
+					int poff = eofoff & PAGE_MASK;
+					if (!poff || NBPGVALID(eofbp,i)) {
+						bzero(d + eofoff, PAGE_SIZE - poff);
+						NBPGVALID_SET(eofbp, i);
+					}
+					if (bp->nb_validend == eofoff)
+						bp->nb_validend += PAGE_SIZE - poff;
+					eofoff += PAGE_SIZE - poff;
+					i++;
+				}
+				nfs_buf_release(eofbp, 1);
 			}
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-			eof = fxdr_unsigned(int, *(tl + 1));
+		}
+		/*
+		 * If dirtyend exceeds file size, chop it down.  This should
+		 * not occur unless there is a race.
+		 */
+		if (NBOFF(bp) + bp->nb_dirtyend > (off_t)np->n_size) {
+			bp->nb_dirtyend = np->n_size - NBOFF(bp);
+			if (bp->nb_dirtyoff >= bp->nb_dirtyend)
+				bp->nb_dirtyoff = bp->nb_dirtyend = 0;
+		}
+		/*
+		 * UBC doesn't handle partial pages, so we need to make sure
+		 * that any pages left in the page cache are completely valid.
+		 *
+		 * Writes that are smaller than a block are delayed if they
+		 * don't extend to the end of the block.
+		 *
+		 * If the block isn't (completely) cached, we may need to read
+		 * in some parts of pages that aren't covered by the write.
+		 * If the write offset (on) isn't page aligned, we'll need to
+		 * read the start of the first page being written to.  Likewise,
+		 * if the offset of the end of the write (on+n) isn't page aligned,
+		 * we'll need to read the end of the last page being written to.
+		 *
+		 * Notes:
+		 * We don't want to read anything we're just going to write over.
+		 * We don't want to issue multiple I/Os if we don't have to
+		 *   (because they're synchronous rpcs).
+		 * We don't want to read anything we already have modified in the
+		 *   page cache.
+		 */
+		if (!ISSET(bp->nb_flags, NB_NOCACHE) && !ISSET(bp->nb_flags, NB_CACHE) && (n < biosize)) {
+			int firstpg, lastpg, dirtypg;
+			int firstpgoff, lastpgoff;
+			start = end = -1;
+			firstpg = on/PAGE_SIZE;
+			firstpgoff = on & PAGE_MASK;
+			lastpg = (on+n-1)/PAGE_SIZE;
+			lastpgoff = (on+n) & PAGE_MASK;
+			if (firstpgoff && !NBPGVALID(bp,firstpg)) {
+				/* need to read start of first page */
+				start = firstpg * PAGE_SIZE;
+				end = start + firstpgoff;
+			}
+			if (lastpgoff && !NBPGVALID(bp,lastpg)) {
+				/* need to read end of last page */
+				if (start < 0)
+					start = (lastpg * PAGE_SIZE) + lastpgoff;
+				end = (lastpg + 1) * PAGE_SIZE;
+			}
+			if (end > start) {
+				/* need to read the data in range: start...end-1 */
+
+				/* first, check for dirty pages in between */
+				/* if there are, we'll have to do two reads because */
+				/* we don't want to overwrite the dirty pages. */
+				for (dirtypg=start/PAGE_SIZE; dirtypg <= (end-1)/PAGE_SIZE; dirtypg++)
+					if (NBPGDIRTY(bp,dirtypg))
+						break;
+
+				/* if start is at beginning of page, try */
+				/* to get any preceeding pages as well. */
+				if (!(start & PAGE_MASK)) {
+					/* stop at next dirty/valid page or start of block */
+					for (; start > 0; start-=PAGE_SIZE)
+						if (NBPGVALID(bp,((start-1)/PAGE_SIZE)))
+							break;
+				}
+
+				NFS_BUF_MAP(bp);
+				/* setup uio for read(s) */
+				boff = NBOFF(bp);
+				auio.uio_iovs.iov32p = &iov;
+				auio.uio_iovcnt = 1;
+#if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
+				auio.uio_segflg = UIO_SYSSPACE;
+#else
+				auio.uio_segflg = UIO_SYSSPACE32;
+#endif
+				auio.uio_rw = UIO_READ;
+
+				if (dirtypg <= (end-1)/PAGE_SIZE) {
+					/* there's a dirty page in the way, so just do two reads */
+					/* we'll read the preceding data here */
+					auio.uio_offset = boff + start;
+					iov.iov_len = on - start;
+					uio_uio_resid_set(&auio, iov.iov_len);
+					iov.iov_base = (uintptr_t) bp->nb_data + start;
+					error = nfs_read_rpc(np, &auio, ctx);
+					if (error) /* couldn't read the data, so treat buffer as NOCACHE */
+						SET(bp->nb_flags, (NB_NOCACHE|NB_STABLE));
+					if (uio_uio_resid(&auio) > 0) {
+						FSDBG(516, bp, (caddr_t)iov.iov_base - bp->nb_data, uio_uio_resid(&auio), 0xd00dee01);
+						// LP64todo - fix this
+						bzero((caddr_t)iov.iov_base, uio_uio_resid(&auio));
+					}
+					if (!error) {
+						/* update validoff/validend if necessary */
+						if ((bp->nb_validoff < 0) || (bp->nb_validoff > start))
+							bp->nb_validoff = start;
+						if ((bp->nb_validend < 0) || (bp->nb_validend < on))
+							bp->nb_validend = on;
+						if ((off_t)np->n_size > boff + bp->nb_validend)
+							bp->nb_validend = min(np->n_size - (boff + start), biosize);
+						/* validate any pages before the write offset */
+						for (; start < on/PAGE_SIZE; start+=PAGE_SIZE)
+							NBPGVALID_SET(bp, start/PAGE_SIZE);
+					}
+					/* adjust start to read any trailing data */
+					start = on+n;
+				}
+
+				/* if end is at end of page, try to */
+				/* get any following pages as well. */
+				if (!(end & PAGE_MASK)) {
+					/* stop at next valid page or end of block */
+					for (; end < biosize; end+=PAGE_SIZE)
+						if (NBPGVALID(bp,end/PAGE_SIZE))
+							break;
+				}
+
+				if (((boff+start) >= (off_t)np->n_size) ||
+				    ((start >= on) && ((boff + on + n) >= (off_t)np->n_size))) {
+					/*
+					 * Either this entire read is beyond the current EOF
+					 * or the range that we won't be modifying (on+n...end)
+					 * is all beyond the current EOF.
+					 * No need to make a trip across the network to
+					 * read nothing.  So, just zero the buffer instead.
+					 */
+					FSDBG(516, bp, start, end - start, 0xd00dee00);
+					bzero(bp->nb_data + start, end - start);
+					error = 0;
+				} else if (!ISSET(bp->nb_flags, NB_NOCACHE)) {
+					/* now we'll read the (rest of the) data */
+					auio.uio_offset = boff + start;
+					iov.iov_len = end - start;
+					uio_uio_resid_set(&auio, iov.iov_len);
+					iov.iov_base = (uintptr_t) (bp->nb_data + start);
+					error = nfs_read_rpc(np, &auio, ctx);
+					if (error) /* couldn't read the data, so treat buffer as NOCACHE */
+						SET(bp->nb_flags, (NB_NOCACHE|NB_STABLE));
+					if (uio_uio_resid(&auio) > 0) {
+						FSDBG(516, bp, (caddr_t)iov.iov_base - bp->nb_data, uio_uio_resid(&auio), 0xd00dee02);
+						// LP64todo - fix this
+						bzero((caddr_t)iov.iov_base, uio_uio_resid(&auio));
+					}
+				}
+				if (!error) {
+					/* update validoff/validend if necessary */
+					if ((bp->nb_validoff < 0) || (bp->nb_validoff > start))
+						bp->nb_validoff = start;
+					if ((bp->nb_validend < 0) || (bp->nb_validend < end))
+						bp->nb_validend = end;
+					if ((off_t)np->n_size > boff + bp->nb_validend)
+						bp->nb_validend = min(np->n_size - (boff + start), biosize);
+					/* validate any pages before the write offset's page */
+					for (; start < trunc_page_32(on); start+=PAGE_SIZE)
+						NBPGVALID_SET(bp, start/PAGE_SIZE);
+					/* validate any pages after the range of pages being written to */
+					for (; (end - 1) > round_page_32(on+n-1); end-=PAGE_SIZE)
+						NBPGVALID_SET(bp, (end-1)/PAGE_SIZE);
+				}
+				/* Note: pages being written to will be validated when written */
+			}
+		}
+
+		if (ISSET(bp->nb_flags, NB_ERROR)) {
+			error = bp->nb_error;
+			nfs_buf_release(bp, 1);
+			goto out;
+		}
+
+		nfs_lock(np, NFS_NODE_LOCK_FORCE);
+		np->n_flag |= NMODIFIED;
+		nfs_unlock(np);
+
+		NFS_BUF_MAP(bp);
+		error = uiomove((char *)bp->nb_data + on, n, uio);
+		if (error) {
+			SET(bp->nb_flags, NB_ERROR);
+			nfs_buf_release(bp, 1);
+			goto out;
+		}
+
+		/* validate any pages written to */
+		start = on & ~PAGE_MASK;
+		for (; start < on+n; start += PAGE_SIZE) {
+			NBPGVALID_SET(bp, start/PAGE_SIZE);
+			/*
+			 * This may seem a little weird, but we don't actually set the
+			 * dirty bits for writes.  This is because we keep the dirty range
+			 * in the nb_dirtyoff/nb_dirtyend fields.  Also, particularly for
+			 * delayed writes, when we give the pages back to the VM we don't
+			 * want to keep them marked dirty, because when we later write the
+			 * buffer we won't be able to tell which pages were written dirty
+			 * and which pages were mmapped and dirtied.
+			 */
+		}
+		if (bp->nb_dirtyend > 0) {
+			bp->nb_dirtyoff = min(on, bp->nb_dirtyoff);
+			bp->nb_dirtyend = max((on + n), bp->nb_dirtyend);
+		} else {
+			bp->nb_dirtyoff = on;
+			bp->nb_dirtyend = on + n;
+		}
+		if (bp->nb_validend <= 0 || bp->nb_validend < bp->nb_dirtyoff ||
+		    bp->nb_validoff > bp->nb_dirtyend) {
+			bp->nb_validoff = bp->nb_dirtyoff;
+			bp->nb_validend = bp->nb_dirtyend;
 		} else {
-			if (mrep) {
-				nfsm_loadattr(vp, v3, NULL, &xid);
+			bp->nb_validoff = min(bp->nb_validoff, bp->nb_dirtyoff);
+			bp->nb_validend = max(bp->nb_validend, bp->nb_dirtyend);
+		}
+		if (!ISSET(bp->nb_flags, NB_CACHE))
+			nfs_buf_normalize_valid_range(np, bp);
+
+		/*
+		 * Since this block is being modified, it must be written
+		 * again and not just committed.
+		 */
+		if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
+			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
+				np->n_needcommitcnt--;
+				CHECK_NEEDCOMMITCNT(np);
 			}
+			CLR(bp->nb_flags, NB_NEEDCOMMIT);
+			nfs_unlock(np);
 		}
-		if (mrep) {
-			nfsm_strsiz(retlen, nmrsize, 0);
-			nfsm_mtouio(uiop, retlen);
-			mbuf_freem(mrep);
+
+		if (ioflag & IO_SYNC) {
+			error = nfs_buf_write(bp);
+			if (error)
+				goto out;
+		} else if (((n + on) == biosize) || (ioflag & IO_NOCACHE) || ISSET(bp->nb_flags, NB_NOCACHE)) {
+			SET(bp->nb_flags, NB_ASYNC);
+			error = nfs_buf_write(bp);
+			if (error)
+				goto out;
 		} else {
-			retlen = 0;
+			/* If the block wasn't already delayed: charge for the write */
+			if (!ISSET(bp->nb_flags, NB_DELWRI)) {
+				proc_t p = vfs_context_proc(ctx);
+				if (p && p->p_stats)
+					OSIncrementAtomic(&p->p_stats->p_ru.ru_oublock);
+			}
+			nfs_buf_write_delayed(bp);
 		}
-		tsiz -= retlen;
-		if (v3) {
-			if (eof || retlen == 0)
-				tsiz = 0;
-		} else if (retlen < len)
-			tsiz = 0;
-	}
-nfsmout:
-	FSDBG_BOT(536, vp, eof, uio_uio_resid(uiop), error);
+		if (np->n_needcommitcnt >= NFS_A_LOT_OF_NEEDCOMMITS)
+		        nfs_flushcommits(np, 1);
+
+	} while (uio_uio_resid(uio) > 0 && n > 0);
+
+out:
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
+	np->n_wrbusy--;
+	nfs_unlock(np);
+	nfs_data_unlock(np);
+	FSDBG_BOT(515, np, uio->uio_offset, uio_uio_resid(uio), error);
 	return (error);
 }
 
+
 /*
- * nfs write call
+ * NFS write call
  */
 int
-nfs_writerpc(
-	vnode_t vp,
+nfs_write_rpc(
+	nfsnode_t np,
 	struct uio *uiop,
+	vfs_context_t ctx,
+	int *iomodep,
+	uint64_t *wverfp)
+{
+	return nfs_write_rpc2(np, uiop, vfs_context_thread(ctx), vfs_context_ucred(ctx), iomodep, wverfp);
+}
+
+int
+nfs_write_rpc2(
+	nfsnode_t np,
+	struct uio *uiop,
+	thread_t thd,
 	kauth_cred_t cred,
-	proc_t p,
-	int *iomode,
+	int *iomodep,
 	uint64_t *wverfp)
 {
-	register u_long *tl;
-	register caddr_t cp;
-	register int t1, t2, backup;
-	caddr_t bpos, dpos, cp2;
-	mbuf_t mreq, mrep, md, mb, mb2;
 	struct nfsmount *nmp;
-	int error = 0, len, tsiz, updatemtime = 0, wccpostattr = 0, rlen, commit;
-	int v3, committed = NFSV3WRITE_FILESYNC;
-	u_int64_t xid, wverf;
-	mount_t mp;
+	int error = 0, nfsvers, restart;
+	int backup, wverfset, commit, committed;
+	uint64_t wverf = 0, wverf2;
+	size_t nmwsize, totalsize, tsiz, len, rlen;
+	struct nfsreq rq, *req = &rq;
 
 #if DIAGNOSTIC
+	/* XXX limitation based on need to back up uio on short write */
 	if (uiop->uio_iovcnt != 1)
-		panic("nfs_writerpc: iovcnt > 1");
+		panic("nfs3_write_rpc: iovcnt > 1");
 #endif
-	FSDBG_TOP(537, vp, uiop->uio_offset, uio_uio_resid(uiop), *iomode);
-	nmp = VFSTONFS(vnode_mount(vp));
+	FSDBG_TOP(537, np, uiop->uio_offset, uio_uio_resid(uiop), *iomodep);
+	nmp = NFSTONMP(np);
 	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(vp);
+	nfsvers = nmp->nm_vers;
+	nmwsize = nmp->nm_wsize;
+
+	restart = wverfset = 0;
+	committed = NFS_WRITE_FILESYNC;
+
 	// LP64todo - fix this
-	tsiz = uio_uio_resid(uiop);
-        if (((u_int64_t)uiop->uio_offset + (unsigned int)tsiz > 0xffffffff) && !v3) {
-		FSDBG_BOT(537, vp, uiop->uio_offset, uio_uio_resid(uiop), EFBIG);
+	totalsize = tsiz = uio_uio_resid(uiop);
+	if (((u_int64_t)uiop->uio_offset + (unsigned int)tsiz > 0xffffffff) && (nfsvers == NFS_VER2)) {
+		FSDBG_BOT(537, np, uiop->uio_offset, uio_uio_resid(uiop), EFBIG);
 		return (EFBIG);
 	}
+
 	while (tsiz > 0) {
-		nmp = VFSTONFS(vnode_mount(vp));
-		if (!nmp) {
+		len = (tsiz > nmwsize) ? nmwsize : tsiz;
+		FSDBG(537, np, uiop->uio_offset, len, 0);
+		error = nmp->nm_funcs->nf_write_rpc_async(np, uiop, len, thd, cred, *iomodep, NULL, &req);
+		if (!error)
+			error = nmp->nm_funcs->nf_write_rpc_async_finish(np, req, &commit, &rlen, &wverf2);
+		nmp = NFSTONMP(np);
+		if (!nmp)
 			error = ENXIO;
-			break;
-		}
-		len = (tsiz > nmp->nm_wsize) ? nmp->nm_wsize : tsiz;
-		nfsm_reqhead(NFSX_FH(v3) + 5 * NFSX_UNSIGNED + nfsm_rndup(len));
 		if (error)
 			break;
-		OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_WRITE]);
-		nfsm_fhtom(vp, v3);
-		if (v3) {
-			nfsm_build(tl, u_long *, 5 * NFSX_UNSIGNED);
-			txdr_hyper(&uiop->uio_offset, tl);
-			tl += 2;
-			*tl++ = txdr_unsigned(len);
-			*tl++ = txdr_unsigned(*iomode);
-		} else {
-			nfsm_build(tl, u_long *, 4 * NFSX_UNSIGNED);
-			*++tl = txdr_unsigned(uiop->uio_offset);
-			tl += 2;
+		if (nfsvers == NFS_VER2) {
+			tsiz -= len;
+			continue;
 		}
-		*tl = txdr_unsigned(len);
-		FSDBG(537, vp, uiop->uio_offset, len, 0);
-		nfsm_uiotom(uiop, len);
-		nfsm_request(vp, NFSPROC_WRITE, p, cred, &xid);
-		nmp = VFSTONFS(vnode_mount(vp));
-		if (!nmp)
-			error = ENXIO;
-		if (v3) {
-			if (mrep) {
-				struct timespec premtime;
-				nfsm_wcc_data(vp, &premtime, wccpostattr, &xid);
-				if (nfstimespeccmp(&VTONFS(vp)->n_mtime, &premtime, ==))
-					updatemtime = 1;
-			}
-			if (!error) {
-				nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED +
-					NFSX_V3WRITEVERF);
-				rlen = fxdr_unsigned(int, *tl++);
-				if (rlen <= 0) {
-					error = NFSERR_IO;
-					break;
-				} else if (rlen < len) {
-					backup = len - rlen;
-					uio_iov_base_add(uiop, -backup);
-					uio_iov_len_add(uiop, backup);
-					uiop->uio_offset -= backup;
-					uio_uio_resid_add(uiop, backup);
-					len = rlen;
-				}
-				commit = fxdr_unsigned(int, *tl++);
 
-				/*
-				 * Return the lowest committment level
-				 * obtained by any of the RPCs.
-				 */
-				if (committed == NFSV3WRITE_FILESYNC)
-					committed = commit;
-				else if (committed == NFSV3WRITE_DATASYNC &&
-					commit == NFSV3WRITE_UNSTABLE)
-					committed = commit;
-			        fxdr_hyper(tl, &wverf);
-				if (wverfp)
-					*wverfp = wverf;
-				if ((nmp->nm_state & NFSSTA_HASWRITEVERF) == 0) {
-				    nmp->nm_verf = wverf;
-				    nmp->nm_state |= NFSSTA_HASWRITEVERF;
-				} else if (wverf != nmp->nm_verf) {
-				    nmp->nm_verf = wverf;
-				}
-			}
-		} else {
-			if (mrep) {
-				nfsm_loadattr(vp, v3, NULL, &xid);
-			}
+		/* check for a short write */
+		if (rlen < len) {
+			backup = len - rlen;
+			uio_iov_base_add(uiop, -backup);
+			uio_iov_len_add(uiop, backup);
+			uiop->uio_offset -= backup;
+			uio_uio_resid_add(uiop, backup);
+			len = rlen;
 		}
 
-		if (updatemtime)
-			VTONFS(vp)->n_mtime = VTONFS(vp)->n_vattr.nva_mtime;
-		mbuf_freem(mrep);
-		/*
-		 * we seem to have a case where we end up looping on shutdown
-		 * and taking down nfs servers.  For V3, error cases, there is
-		 * no way to terminate loop, if the len was 0, meaning,
-		 * nmp->nm_wsize was trashed. FreeBSD has this fix in it.
-		 * Let's try it.
-		 */
-		if (error)
-			break;
+		/* return lowest commit level returned */
+		if (commit < committed)
+			committed = commit;
+
 		tsiz -= len;
+
+		/* check write verifier */
+		if (!wverfset) {
+			wverf = wverf2;
+			wverfset = 1;
+		} else if (wverf != wverf2) {
+			/* verifier changed, so we need to restart all the writes */
+			if (++restart > 10) {
+				/* give up after too many restarts */
+				error = EIO;
+				break;
+			}
+			backup = totalsize - tsiz;
+			uio_iov_base_add(uiop, -backup);
+			uio_iov_len_add(uiop, backup);
+			uiop->uio_offset -= backup;
+			uio_uio_resid_add(uiop, backup);
+			committed = NFS_WRITE_FILESYNC;
+			wverfset = 0;
+			tsiz = totalsize;
+		}
+	}
+	if (wverfset && wverfp)
+		*wverfp = wverf;
+	*iomodep = committed;
+	if (error)
+		uio_uio_resid_set(uiop, tsiz);
+	FSDBG_BOT(537, np, committed, uio_uio_resid(uiop), error);
+	return (error);
+}
+
+int
+nfs3_write_rpc_async(
+	nfsnode_t np,
+	struct uio *uiop,
+	size_t len,
+	thread_t thd,
+	kauth_cred_t cred,
+	int iomode,
+	struct nfsreq_cbinfo *cb,
+	struct nfsreq **reqp)
+{
+	struct nfsmount *nmp;
+	int error = 0, nfsvers;
+	off_t offset;
+	struct nfsm_chain nmreq;
+
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	offset = uiop->uio_offset;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + 5 * NFSX_UNSIGNED + nfsm_rndup(len));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_add_64(error, &nmreq, offset);
+		nfsm_chain_add_32(error, &nmreq, len);
+		nfsm_chain_add_32(error, &nmreq, iomode);
+	} else {
+		nfsm_chain_add_32(error, &nmreq, 0);
+		nfsm_chain_add_32(error, &nmreq, offset);
+		nfsm_chain_add_32(error, &nmreq, 0);
+	}
+	nfsm_chain_add_32(error, &nmreq, len);
+	nfsmout_if(error);
+	error = nfsm_chain_add_uio(&nmreq, uiop, len);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request_async(np, NULL, &nmreq, NFSPROC_WRITE, thd, cred, cb, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs3_write_rpc_async_finish(
+	nfsnode_t np,
+	struct nfsreq *req,
+	int *iomodep,
+	size_t *rlenp,
+	uint64_t *wverfp)
+{
+	struct nfsmount *nmp;
+	int error = 0, lockerror = ENOENT, nfsvers, status;
+	int updatemtime = 0, wccpostattr = 0, rlen, committed = NFS_WRITE_FILESYNC;
+	u_int64_t xid, wverf;
+	mount_t mp;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(np);
+	if (!nmp) {
+		nfs_request_async_cancel(req);
+		return (ENXIO);
+	}
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	if (error == EINPROGRESS) /* async request restarted */
+		return (error);
+	nmp = NFSTONMP(np);
+	if (!nmp)
+		error = ENXIO;
+	if (!error && (lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	if (nfsvers == NFS_VER3) {
+		struct timespec premtime = { 0, 0 };
+		nfsm_chain_get_wcc_data(error, &nmrep, np, &premtime, &wccpostattr, &xid);
+		if (nfstimespeccmp(&np->n_mtime, &premtime, ==))
+			updatemtime = 1;
+		if (!error)
+			error = status;
+		nfsm_chain_get_32(error, &nmrep, rlen);
+		nfsmout_if(error);
+		*rlenp = rlen;
+		if (rlen <= 0)
+			error = NFSERR_IO;
+		nfsm_chain_get_32(error, &nmrep, committed);
+		nfsm_chain_get_64(error, &nmrep, wverf);
+		nfsmout_if(error);
+		if (wverfp)
+			*wverfp = wverf;
+		lck_mtx_lock(&nmp->nm_lock);
+		if (!(nmp->nm_state & NFSSTA_HASWRITEVERF)) {
+			nmp->nm_verf = wverf;
+			nmp->nm_state |= NFSSTA_HASWRITEVERF;
+		} else if (nmp->nm_verf != wverf) {
+			nmp->nm_verf = wverf;
+		}
+		lck_mtx_unlock(&nmp->nm_lock);
+	} else {
+		if (!error)
+			error = status;
+		nfsm_chain_loadattr(error, &nmrep, np, nfsvers, NULL, &xid);
+		nfsmout_if(error);
 	}
+	if (updatemtime)
+		NFS_CHANGED_UPDATE(nfsvers, np, &np->n_vattr);
 nfsmout:
-        if ((mp = vnode_mount(vp)) && (vfs_flags(mp) & MNT_ASYNC))
-		committed = NFSV3WRITE_FILESYNC;
-        *iomode = committed;
-	if (error)
-		uio_uio_resid_set(uiop, tsiz);
-	FSDBG_BOT(537, vp, committed, uio_uio_resid(uiop), error);
+	if (!lockerror)
+		nfs_unlock(np);
+	nfsm_chain_cleanup(&nmrep);
+	if ((committed != NFS_WRITE_FILESYNC) && nfs_allow_async &&
+	    ((mp = NFSTOMP(np))) && (vfs_flags(mp) & MNT_ASYNC))
+		committed = NFS_WRITE_FILESYNC;
+	*iomodep = committed;
 	return (error);
 }
 
 /*
- * nfs mknod rpc
- * For NFS v2 this is a kludge. Use a create rpc but with the IFMT bits of the
+ * NFS mknod vnode op
+ *
+ * For NFS v2 this is a kludge. Use a create RPC but with the IFMT bits of the
  * mode set to specify the file type and the size field for rdev.
  */
 static int
-nfs_mknodrpc(
-	vnode_t dvp,
-	vnode_t *vpp,
-	struct componentname *cnp,
-	struct vnode_attr *vap,
-	kauth_cred_t cred,
-	proc_t p)
+nfs3_vnop_mknod(
+	struct vnop_mknod_args /* {
+		struct vnodeop_desc *a_desc;
+		vnode_t a_dvp;
+		vnode_t *a_vpp;
+		struct componentname *a_cnp;
+		struct vnode_attr *a_vap;
+		vfs_context_t a_context;
+	} */ *ap)
 {
-	register struct nfsv2_sattr *sp;
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	vnode_t newvp = (vnode_t)0;
-	struct nfsnode *np = (struct nfsnode *)0;
-	struct nfs_vattr nvattr;
-	char *cp2;
-	caddr_t bpos, dpos;
-	int error = 0, wccpostattr = 0, gotvp = 0;
+	vnode_t dvp = ap->a_dvp;
+	vnode_t *vpp = ap->a_vpp;
+	struct componentname *cnp = ap->a_cnp;
+	struct vnode_attr *vap = ap->a_vap;
+	vfs_context_t ctx = ap->a_context;
+	vnode_t newvp = NULL;
+	nfsnode_t np = NULL;
+	struct nfsmount *nmp;
+	nfsnode_t dnp = VTONFS(dvp);
+	struct nfs_vattr nvattr, dnvattr;
+	fhandle_t fh;
+	int error = 0, lockerror = ENOENT, status, wccpostattr = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
 	u_long rdev;
-	u_int64_t xid;
-	int v3 = NFS_ISV3(dvp);
-	int gotuid, gotgid;
+	u_int64_t xid, dxid;
+	int nfsvers, gotuid, gotgid;
+	struct nfsm_chain nmreq, nmrep;
+
+	nmp = VTONMP(dvp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
 	if (!VATTR_IS_ACTIVE(vap, va_type))
 		return (EINVAL);
 	if (vap->va_type == VCHR || vap->va_type == VBLK) {
 		if (!VATTR_IS_ACTIVE(vap, va_rdev))
 			return (EINVAL);
-		rdev = txdr_unsigned(vap->va_rdev);
+		rdev = vap->va_rdev;
 	} else if (vap->va_type == VFIFO || vap->va_type == VSOCK)
 		rdev = 0xffffffff;
 	else {
 		return (ENOTSUP);
 	}
-	nfsm_reqhead(NFSX_FH(v3) + 4 * NFSX_UNSIGNED +
-		nfsm_rndup(cnp->cn_namelen) + NFSX_SATTR(v3));
-	if (error)
-		return (error);
+	if ((nfsvers == NFS_VER2) && (cnp->cn_namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
 	VATTR_SET_SUPPORTED(vap, va_mode);
 	VATTR_SET_SUPPORTED(vap, va_uid);
@@ -1958,105 +2714,100 @@ nfs_mknodrpc(
 	gotuid = VATTR_IS_ACTIVE(vap, va_uid);
 	gotgid = VATTR_IS_ACTIVE(vap, va_gid);
 
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_MKNOD]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN, v3);
-	if (v3) {
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
-		*tl++ = vtonfsv3_type(vap->va_type);
-		nfsm_v3sattr(vap);
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + 4 * NFSX_UNSIGNED +
+		nfsm_rndup(cnp->cn_namelen) + NFSX_SATTR(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_add_32(error, &nmreq, vtonfs_type(vap->va_type, nfsvers));
+		nfsm_chain_add_v3sattr(error, &nmreq, vap);
 		if (vap->va_type == VCHR || vap->va_type == VBLK) {
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = txdr_unsigned(major(vap->va_rdev));
-			*tl = txdr_unsigned(minor(vap->va_rdev));
+			nfsm_chain_add_32(error, &nmreq, major(vap->va_rdev));
+			nfsm_chain_add_32(error, &nmreq, minor(vap->va_rdev));
 		}
 	} else {
-		struct timespec neg1time = { -1, -1 };
-		nfsm_build(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		sp->sa_mode = vtonfsv2_mode(vap->va_type,
-			(VATTR_IS_ACTIVE(vap, va_mode) ? vap->va_mode : 0600));
-		sp->sa_uid = gotuid ? (u_long)txdr_unsigned(vap->va_uid) : nfs_xdrneg1;
-		sp->sa_gid = gotgid ? (u_long)txdr_unsigned(vap->va_gid) : nfs_xdrneg1;
-		sp->sa_size = rdev;
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			txdr_nfsv2time(&vap->va_access_time, &sp->sa_atime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_atime);
-		}
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			txdr_nfsv2time(&vap->va_modify_time, &sp->sa_mtime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_mtime);
-		}
+		nfsm_chain_add_v2sattr(error, &nmreq, vap, rdev);
 	}
-	nfsm_request(dvp, NFSPROC_MKNOD, p, cred, &xid);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request(dnp, NULL, &nmreq, NFSPROC_MKNOD, ctx, &nmrep, &xid, &status);
+
 	/* XXX no EEXIST kludge here? */
-	if (!error) {
-		nfsm_mtofh(dvp, cnp, newvp, v3, &xid, gotvp);
-		if (!gotvp) {
-			error = nfs_lookitup(dvp, cnp->cn_nameptr,
-			    cnp->cn_namelen, cred, p, &np);
-			if (!error)
-				newvp = NFSTOV(np);
+	dxid = xid;
+	if (!error && !status) {
+		if (dnp->n_flag & NNEGNCENTRIES) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(dvp);
+		}
+		error = nfsm_chain_get_fh_attr(&nmrep, dnp, ctx, nfsvers, &xid, &fh, &nvattr);
+	}
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &dxid);
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	if (!lockerror) {
+		dnp->n_flag |= NMODIFIED;
+		/* if directory hadn't changed, update namecache mtime */
+		if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
+		if (!wccpostattr)
+			NATTRINVALIDATE(dnp);
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(dvp);
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
 		}
 	}
-	if (v3 && mrep)
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &xid);
+
+	if (!error && fh.fh_len)
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+	if (!error && !np)
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
+	if (!error && np)
+		newvp = NFSTOV(np);
+	if (!lockerror)
+		nfs_unlock(dnp);
+
 	if (!error && (gotuid || gotgid) &&
-	    (!newvp || nfs_getattrcache(newvp, &nvattr) ||
+	    (!newvp || nfs_getattrcache(np, &nvattr, 1) ||
 	     (gotuid && (nvattr.nva_uid != vap->va_uid)) ||
 	     (gotgid && (nvattr.nva_gid != vap->va_gid)))) {
 		/* clear ID bits if server didn't use them (or we can't tell) */
 		VATTR_CLEAR_SUPPORTED(vap, va_uid);
 		VATTR_CLEAR_SUPPORTED(vap, va_gid);
 	}
-	nfsm_reqdone;
 	if (error) {
-		if (newvp)
+		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
+		}
 	} else {
 		*vpp = newvp;
+		nfs_unlock(np);
 	}
-	VTONFS(dvp)->n_flag |= NMODIFIED;
-	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
-	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
-	return (error);
-}
-
-/*
- * nfs mknod vop
- * just call nfs_mknodrpc() to do the work.
- */
-/* ARGSUSED */
-static int
-nfs_mknod(ap)
-	struct vnop_mknod_args /* {
-		struct vnodeop_desc *a_desc;
-		vnode_t a_dvp;
-		vnode_t *a_vpp;
-		struct componentname *a_cnp;
-		struct vnode_attr *a_vap;
-		vfs_context_t a_context;
-	} */ *ap;
-{
-	int error;
-
-	error = nfs_mknodrpc(ap->a_dvp, ap->a_vpp, ap->a_cnp, ap->a_vap,
-			vfs_context_ucred(ap->a_context),
-			vfs_context_proc(ap->a_context));
-
 	return (error);
 }
 
 static u_long create_verf;
 /*
- * nfs file create call
+ * NFS file create call
  */
 static int
-nfs_create(ap)
+nfs3_vnop_create(
 	struct vnop_create_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
@@ -2064,39 +2815,34 @@ nfs_create(ap)
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t dvp = ap->a_dvp;
 	struct vnode_attr *vap = ap->a_vap;
 	struct componentname *cnp = ap->a_cnp;
-	struct nfs_vattr nvattr;
-	struct nfsv2_sattr *sp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	struct nfsnode *np = (struct nfsnode *)0;
-	vnode_t newvp = (vnode_t)0;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0, gotvp = 0, fmode = 0;
+	struct nfs_vattr nvattr, dnvattr;
+	fhandle_t fh;
+	nfsnode_t np = NULL;
+	struct nfsmount *nmp;
+	nfsnode_t dnp = VTONFS(dvp);
+	vnode_t newvp = NULL;
+	int error = 0, lockerror = ENOENT, status, wccpostattr = 0, fmode = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3 = NFS_ISV3(dvp);
-	int gotuid, gotgid;
-	u_int64_t xid;
-	kauth_cred_t cred;
-	proc_t p;
-
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	int nfsvers, gotuid, gotgid;
+	u_int64_t xid, dxid;
+	uint32_t val;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsreq *req;
+	struct nfs_dulookup dul;
 
-	if (!VATTR_IS_ACTIVE(vap, va_type))
-		return (EINVAL);
+	nmp = VTONMP(dvp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
-	/*
-	 * Oops, not for me..
-	 */
-	if (vap->va_type == VSOCK)
-		return (nfs_mknodrpc(dvp, ap->a_vpp, cnp, vap, cred, p));
+	if ((nfsvers == NFS_VER2) && (cnp->cn_namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
 	VATTR_SET_SUPPORTED(vap, va_mode);
 	VATTR_SET_SUPPORTED(vap, va_uid);
@@ -2109,112 +2855,143 @@ nfs_create(ap)
 
 	if (vap->va_vaflags & VA_EXCLUSIVE)
 		fmode |= O_EXCL;
+
 again:
-	nfsm_reqhead(NFSX_FH(v3) + 2 * NFSX_UNSIGNED +
-		nfsm_rndup(cnp->cn_namelen) + NFSX_SATTR(v3));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_CREATE]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN, v3);
-	if (v3) {
-		nfsm_build(tl, u_long *, NFSX_UNSIGNED);
+	req = NULL;
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + 2 * NFSX_UNSIGNED +
+		nfsm_rndup(cnp->cn_namelen) + NFSX_SATTR(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	if (nfsvers == NFS_VER3) {
 		if (fmode & O_EXCL) {
-		    *tl = txdr_unsigned(NFSV3CREATE_EXCLUSIVE);
-		    nfsm_build(tl, u_long *, NFSX_V3CREATEVERF);
-		    if (!TAILQ_EMPTY(&in_ifaddrhead))
-			*tl++ = IA_SIN(in_ifaddrhead.tqh_first)->sin_addr.s_addr;
-		    else
-			*tl++ = create_verf;
-		    *tl = ++create_verf;
+			nfsm_chain_add_32(error, &nmreq, NFS_CREATE_EXCLUSIVE);
+			if (!TAILQ_EMPTY(&in_ifaddrhead))
+				val = IA_SIN(in_ifaddrhead.tqh_first)->sin_addr.s_addr;
+			else
+				val = create_verf;
+			nfsm_chain_add_32(error, &nmreq, val);
+			++create_verf;
+			nfsm_chain_add_32(error, &nmreq, create_verf);
 		} else {
-		    *tl = txdr_unsigned(NFSV3CREATE_UNCHECKED);
-		    nfsm_v3sattr(vap);
+			nfsm_chain_add_32(error, &nmreq, NFS_CREATE_UNCHECKED);
+			nfsm_chain_add_v3sattr(error, &nmreq, vap);
 		}
 	} else {
-		struct timespec neg1time = { -1, -1 };
-		nfsm_build(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		sp->sa_mode = vtonfsv2_mode(vap->va_type,
-			(VATTR_IS_ACTIVE(vap, va_mode) ? vap->va_mode : 0600));
-		sp->sa_uid = gotuid ? (u_long)txdr_unsigned(vap->va_uid) : nfs_xdrneg1;
-		sp->sa_gid = gotgid ? (u_long)txdr_unsigned(vap->va_gid) : nfs_xdrneg1;
-		sp->sa_size = 0;
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			txdr_nfsv2time(&vap->va_access_time, &sp->sa_atime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_atime);
-		}
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			txdr_nfsv2time(&vap->va_modify_time, &sp->sa_mtime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_mtime);
-		}
+		nfsm_chain_add_v2sattr(error, &nmreq, vap, 0);
 	}
-	nfsm_request(dvp, NFSPROC_CREATE, p, cred, &xid);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC_CREATE,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
 	if (!error) {
-		nfsm_mtofh(dvp, cnp, newvp, v3, &xid, gotvp);
-		if (!gotvp) {
-			error = nfs_lookitup(dvp, cnp->cn_nameptr,
-			    cnp->cn_namelen, cred, p, &np);
-			if (!error)
-				newvp = NFSTOV(np);
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	}
+
+	dxid = xid;
+	if (!error && !status) {
+		if (dnp->n_flag & NNEGNCENTRIES) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(dvp);
+		}
+		error = nfsm_chain_get_fh_attr(&nmrep, dnp, ctx, nfsvers, &xid, &fh, &nvattr);
+	}
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &dxid);
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	if (!lockerror) {
+		dnp->n_flag |= NMODIFIED;
+		/* if directory hadn't changed, update namecache mtime */
+		if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
+		if (!wccpostattr)
+			NATTRINVALIDATE(dnp);
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(dvp);
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
 		}
 	}
-	if (v3 && mrep)
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &xid);
-	nfsm_reqdone;
+
+	if (!error && fh.fh_len)
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+	if (!error && !np)
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
+	if (!error && np)
+		newvp = NFSTOV(np);
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
+	if (!lockerror)
+		nfs_unlock(dnp);
+
 	if (error) {
-		if (v3 && (fmode & O_EXCL) && error == NFSERR_NOTSUPP) {
+		if ((nfsvers == NFS_VER3) && (fmode & O_EXCL) && (error == NFSERR_NOTSUPP)) {
 			fmode &= ~O_EXCL;
 			goto again;
 		}
-		if (newvp)
+		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
-	} else if (v3 && (fmode & O_EXCL)) {
-		error = nfs_setattrrpc(newvp, vap, cred, p);
+		}
+	} else if ((nfsvers == NFS_VER3) && (fmode & O_EXCL)) {
+		error = nfs3_setattr_rpc(np, vap, ctx, 1);
 		if (error && (gotuid || gotgid)) {
 			/* it's possible the server didn't like our attempt to set IDs. */
 			/* so, let's try it again without those */
 			VATTR_CLEAR_ACTIVE(vap, va_uid);
 			VATTR_CLEAR_ACTIVE(vap, va_gid);
-			error = nfs_setattrrpc(newvp, vap, cred, p);
+			error = nfs3_setattr_rpc(np, vap, ctx, 1);
 		}
-		if (error)
+		if (error) {
+			nfs_unlock(np);
 			vnode_put(newvp);
+		}
 	}
-	if (!error) {
+	if (!error)
 		*ap->a_vpp = newvp;
-	}
-	VTONFS(dvp)->n_flag |= NMODIFIED;
-	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
-	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
 	if (!error && (gotuid || gotgid) &&
-	    (!newvp || nfs_getattrcache(newvp, &nvattr) ||
+	    (!newvp || nfs_getattrcache(np, &nvattr, 1) ||
 	     (gotuid && (nvattr.nva_uid != vap->va_uid)) ||
 	     (gotgid && (nvattr.nva_gid != vap->va_gid)))) {
 		/* clear ID bits if server didn't use them (or we can't tell) */
 		VATTR_CLEAR_SUPPORTED(vap, va_uid);
 		VATTR_CLEAR_SUPPORTED(vap, va_gid);
 	}
+	if (!error)
+		nfs_unlock(np);
 	return (error);
 }
 
 /*
- * nfs file remove call
- * To try and make nfs semantics closer to ufs semantics, a file that has
+ * NFS file remove call
+ * To try and make NFS semantics closer to UFS semantics, a file that has
  * other processes using the vnode is renamed instead of removed and then
  * removed later on the last close.
  * - If vnode_isinuse()
  *	  If a rename is not already in the works
  *	     call nfs_sillyrename() to set it up
  *     else
- *	  do the remove rpc
+ *	  do the remove RPC
  */
 static int
-nfs_remove(ap)
+nfs_vnop_remove(
 	struct vnop_remove_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
@@ -2222,45 +2999,91 @@ nfs_remove(ap)
 		struct componentname *a_cnp;
 		int a_flags;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
 	vnode_t dvp = ap->a_dvp;
 	struct componentname *cnp = ap->a_cnp;
-	struct nfsnode *np = VTONFS(vp);
-	int error = 0, gofree = 0;
+	nfsnode_t dnp = VTONFS(dvp);
+	nfsnode_t np = VTONFS(vp);
+	int error = 0, nfsvers, inuse, gotattr = 0, flushed = 0, setsize = 0;
 	struct nfs_vattr nvattr;
-	kauth_cred_t cred;
-	proc_t p;
+	struct nfsmount *nmp;
+	struct nfs_dulookup dul;
+
+	/* XXX prevent removing a sillyrenamed file? */
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+again_relock:
+	error = nfs_lock2(dnp, np, NFS_NODE_LOCK_EXCLUSIVE);
+	if (error)
+		return (error);
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	/* lock the node while we remove the file */
+	lck_mtx_lock(nfs_node_hash_mutex);
+	while (np->n_hflag & NHLOCKED) {
+		np->n_hflag |= NHLOCKWANT;
+		msleep(np, nfs_node_hash_mutex, PINOD, "nfs_remove", NULL);
+	}
+	np->n_hflag |= NHLOCKED;
+	lck_mtx_unlock(nfs_node_hash_mutex);
 
-	gofree = vnode_isinuse(vp, 0) ? 0 : 1;
-	if ((ap->a_flags & VNODE_REMOVE_NODELETEBUSY) && !gofree) {
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+again:
+	inuse = vnode_isinuse(vp, 0);
+	if ((ap->a_flags & VNODE_REMOVE_NODELETEBUSY) && inuse) {
 		/* Caller requested Carbon delete semantics, but file is busy */
-		return (EBUSY);
+		error = EBUSY;
+		goto out;
 	}
-	if (gofree || (np->n_sillyrename &&
-		nfs_getattr(vp, &nvattr, cred, p) == 0 &&
-		nvattr.nva_nlink > 1)) {
+	if (inuse && !gotattr) {
+		if (nfs_getattr(np, &nvattr, ctx, 1))
+			nvattr.nva_nlink = 1;
+		gotattr = 1;
+		goto again;
+	}
+	if (!inuse || (np->n_sillyrename && (nvattr.nva_nlink > 1))) {
+
+		if (!inuse && !flushed) { /* flush all the buffers first */
+			/* unlock the node */
+			lck_mtx_lock(nfs_node_hash_mutex);
+			np->n_hflag &= ~NHLOCKED;
+			if (np->n_hflag & NHLOCKWANT) {
+				np->n_hflag &= ~NHLOCKWANT;
+				wakeup(np);
+			}
+			lck_mtx_unlock(nfs_node_hash_mutex);
+			nfs_unlock2(dnp, np);
+			error = nfs_vinvalbuf(vp, V_SAVE, ctx, 1);
+			FSDBG(260, np, np->n_size, np->n_vattr.nva_size, 0xf00d0011);
+			flushed = 1;
+			if (error == EINTR) {
+				nfs_lock(np, NFS_NODE_LOCK_FORCE);
+				NATTRINVALIDATE(np);
+				nfs_unlock(np);
+				return (error);
+			}
+			goto again_relock;
+		}
+
 		/*
 		 * Purge the name cache so that the chance of a lookup for
 		 * the name succeeding while the remove is in progress is
 		 * minimized.
 		 */
 		cache_purge(vp);
-		/*
-		 * throw away biocache buffers, mainly to avoid
-		 * unnecessary delayed writes later.
-		 */
-		error = nfs_vinvalbuf(vp, 0, cred, p, 1);
-		np->n_size = 0;
-		ubc_setsize(vp, (off_t)0); /* XXX check error */
+
+		nfs_dulookup_start(&dul, dnp, ctx);
+
 		/* Do the rpc */
-		if (error != EINTR)
-			error = nfs_removerpc(dvp, cnp->cn_nameptr,
-				cnp->cn_namelen, cred, p);
+		error = nmp->nm_funcs->nf_remove_rpc(dnp, cnp->cn_nameptr, cnp->cn_namelen,
+				vfs_context_thread(ctx), vfs_context_ucred(ctx));
+
 		/*
 		 * Kludge City: If the first reply to the remove rpc is lost..
 		 *   the reply to the retransmitted request will be ENOENT
@@ -2269,89 +3092,132 @@ nfs_remove(ap)
 		 */
 		if (error == ENOENT)
 			error = 0;
-		if (!error) {
+
+		if (!error && !inuse && !np->n_sillyrename) {
 			/*
+			 * removal succeeded, it's not in use, and not silly renamed so
 			 * remove nfsnode from hash now so we can't accidentally find it
 			 * again if another object gets created with the same filehandle
 			 * before this vnode gets reclaimed
 			 */
 			lck_mtx_lock(nfs_node_hash_mutex);
-			LIST_REMOVE(np, n_hash);
-			np->n_flag &= ~NHASHED;
+			if (np->n_hflag & NHHASHED) {
+				LIST_REMOVE(np, n_hash);
+				np->n_hflag &= ~NHHASHED;
+				FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
+			}
 			lck_mtx_unlock(nfs_node_hash_mutex);
-		}
-		if (!error && !np->n_sillyrename) {
-			/* clear flags now: won't get nfs_inactive for recycled vnode */
+			/* clear flags now: won't get nfs_vnop_inactive for recycled vnode */
 			/* clear all flags other than these */
-			np->n_flag &= (NMODIFIED | NFLUSHINPROG | NFLUSHWANT | NHASHED);
+			np->n_flag &= (NMODIFIED);
 			vnode_recycle(vp);
+			NATTRINVALIDATE(np);
+			setsize = 1;
+		} else {
+			NATTRINVALIDATE(np);
 		}
 	} else if (!np->n_sillyrename) {
-		error = nfs_sillyrename(dvp, vp, cnp, cred, p);
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_sillyrename(dnp, np, cnp, ctx);
+		NATTRINVALIDATE(np);
+	} else {
+		NATTRINVALIDATE(np);
+		nfs_dulookup_start(&dul, dnp, ctx);
 	}
-	NATTRINVALIDATE(np);
 
+	if (!nfs_getattr(dnp, &nvattr, ctx, 1)) {
+		if (NFS_CHANGED_NC(nfsvers, dnp, &nvattr)) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge(dvp);
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &nvattr);
+		}
+	}
+	nfs_dulookup_finish(&dul, dnp, ctx);
+out:
+	/* unlock the node */
+	lck_mtx_lock(nfs_node_hash_mutex);
+	np->n_hflag &= ~NHLOCKED;
+	if (np->n_hflag & NHLOCKWANT) {
+		np->n_hflag &= ~NHLOCKWANT;
+		wakeup(np);
+	}
+	lck_mtx_unlock(nfs_node_hash_mutex);
+	nfs_unlock2(dnp, np);
+	if (setsize)
+		ubc_setsize(vp, 0);
 	return (error);
 }
 
 /*
- * nfs file remove rpc called from nfs_inactive
+ * NFS silly-renamed file removal function called from nfs_vnop_inactive
  */
 int
-nfs_removeit(struct sillyrename *sp)
+nfs_removeit(struct nfs_sillyrename *nsp)
 {
-	return (nfs_removerpc(sp->s_dvp, sp->s_name, sp->s_namlen, sp->s_cred, NULL));
+	struct nfsmount *nmp = NFSTONMP(nsp->nsr_dnp);
+	if (!nmp)
+		return (ENXIO);
+	return nmp->nm_funcs->nf_remove_rpc(nsp->nsr_dnp, nsp->nsr_name, nsp->nsr_namlen, NULL, nsp->nsr_cred);
 }
 
 /*
- * Nfs remove rpc, called from nfs_remove() and nfs_removeit().
+ * NFS remove rpc, called from nfs_remove() and nfs_removeit().
  */
-static int
-nfs_removerpc(dvp, name, namelen, cred, proc)
-	vnode_t dvp;
-	char *name;
-	int namelen;
-	kauth_cred_t cred;
-	proc_t proc;
+int
+nfs3_remove_rpc(
+	nfsnode_t dnp,
+	char *name,
+	int namelen,
+	thread_t thd,
+	kauth_cred_t cred)
 {
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0;
+	int error = 0, status, wccpostattr = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
+	struct nfsmount *nmp;
+	int nfsvers;
 	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
 
-	if (!VFSTONFS(vnode_mount(dvp)))
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(dvp);
+	nfsvers = nmp->nm_vers;
+	if ((nfsvers == NFS_VER2) && (namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED + nfsm_rndup(namelen));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_REMOVE]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(name, namelen, NFS_MAXNAMLEN, v3);
-	nfsm_request(dvp, NFSPROC_REMOVE, proc, cred, &xid);
-	if (v3 && mrep)
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &xid);
-	nfsm_reqdone;
-	VTONFS(dvp)->n_flag |= NMODIFIED;
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + NFSX_UNSIGNED + nfsm_rndup(namelen));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, name, namelen);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+
+	error = nfs_request2(dnp, NULL, &nmreq, NFSPROC_REMOVE, thd, cred, 0, &nmrep, &xid, &status);
+
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &xid);
+	dnp->n_flag |= NMODIFIED;
 	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
+	if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+		NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
 	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
+		NATTRINVALIDATE(dnp);
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
 /*
- * nfs file rename call
+ * NFS file rename call
  */
 static int
-nfs_rename(ap)
+nfs_vnop_rename(
 	struct vnop_rename_args  /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_fdvp;
@@ -2361,25 +3227,51 @@ nfs_rename(ap)
 		vnode_t a_tvp;
 		struct componentname *a_tcnp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	vnode_t fvp = ap->a_fvp;
-	vnode_t tvp = ap->a_tvp;
+	vfs_context_t ctx = ap->a_context;
 	vnode_t fdvp = ap->a_fdvp;
+	vnode_t fvp = ap->a_fvp;
 	vnode_t tdvp = ap->a_tdvp;
+	vnode_t tvp = ap->a_tvp;
+	nfsnode_t fdnp, fnp, tdnp, tnp;
 	struct componentname *tcnp = ap->a_tcnp;
 	struct componentname *fcnp = ap->a_fcnp;
-	int error, inuse=0;
+	int error, nfsvers, inuse=0, tvprecycle=0, locked=0;
 	mount_t fmp, tdmp, tmp;
-	struct nfsnode *tnp;
-	kauth_cred_t cred;
-	proc_t p;
-
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	struct nfs_vattr nvattr;
+	struct nfsmount *nmp;
+	struct nfs_dulookup fdul, tdul;
 
+	fdnp = VTONFS(fdvp);
+	fnp = VTONFS(fvp);
+	tdnp = VTONFS(tdvp);
 	tnp = tvp ? VTONFS(tvp) : NULL;
 
+	nmp = NFSTONMP(fdnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	error = nfs_lock4(fdnp, fnp, tdnp, tnp, NFS_NODE_LOCK_EXCLUSIVE);
+	if (error)
+		return (error);
+
+	if (tvp && (tvp != fvp)) {
+		/* lock the node while we rename over the existing file */
+		lck_mtx_lock(nfs_node_hash_mutex);
+		while (tnp->n_hflag & NHLOCKED) {
+			tnp->n_hflag |= NHLOCKWANT;
+			msleep(tnp, nfs_node_hash_mutex, PINOD, "nfs_rename", NULL);
+		}
+		tnp->n_hflag |= NHLOCKED;
+		lck_mtx_unlock(nfs_node_hash_mutex);
+		locked = 1;
+	}
+
+	nfs_dulookup_init(&fdul, fdnp, fcnp->cn_nameptr, fcnp->cn_namelen);
+	nfs_dulookup_init(&tdul, tdnp, tcnp->cn_nameptr, tcnp->cn_namelen);
+
 	/* Check for cross-device rename */
 	fmp = vnode_mount(fvp);
 	tmp = tvp ? vnode_mount(tvp) : NULL;
@@ -2389,6 +3281,8 @@ nfs_rename(ap)
 		goto out;
 	}
 
+	/* XXX prevent renaming from/over a sillyrenamed file? */
+
 	/*
 	 * If the tvp exists and is in use, sillyrename it before doing the
 	 * rename of the new file over it.
@@ -2396,11 +3290,11 @@ nfs_rename(ap)
 	 * Don't sillyrename if source and target are same vnode (hard
 	 * links or case-variants)
 	 */
-	if (tvp && tvp != fvp) {
+	if (tvp && (tvp != fvp))
 		inuse = vnode_isinuse(tvp, 0);
-	}
-	if (inuse && !tnp->n_sillyrename && vnode_vtype(tvp) != VDIR) {
-		if  ((error = nfs_sillyrename(tdvp, tvp, tcnp, cred, p))) {
+	if (inuse && !tnp->n_sillyrename && (vnode_vtype(tvp) != VDIR)) {
+		error = nfs_sillyrename(tdnp, tnp, tcnp, ctx);
+		if (error) {
 			/* sillyrename failed. Instead of pressing on, return error */
 			goto out; /* should not be ENOENT. */
 		} else {
@@ -2409,8 +3303,11 @@ nfs_rename(ap)
 		}
 	}
 
-	error = nfs_renamerpc(fdvp, fcnp->cn_nameptr, fcnp->cn_namelen,
-		tdvp, tcnp->cn_nameptr, tcnp->cn_namelen, cred, p);
+	nfs_dulookup_start(&fdul, fdnp, ctx);
+	nfs_dulookup_start(&tdul, tdnp, ctx);
+
+	error = nmp->nm_funcs->nf_rename_rpc(fdnp, fcnp->cn_nameptr, fcnp->cn_namelen,
+			tdnp, tcnp->cn_nameptr, tcnp->cn_namelen, ctx);
 
 	/*
 	 * Kludge: Map ENOENT => 0 assuming that it is a reply to a retry.
@@ -2418,169 +3315,237 @@ nfs_rename(ap)
 	if (error == ENOENT)
 		error = 0;
 
-	if (!error && tvp && tvp != fvp && !tnp->n_sillyrename) {
-		/*
-		 * remove nfsnode from hash now so we can't accidentally find it
-		 * again if another object gets created with the same filehandle
-		 * before this vnode gets reclaimed
-		 */
+	if (tvp && (tvp != fvp) && !tnp->n_sillyrename) {
+		tvprecycle = (!error && !vnode_isinuse(tvp, 0) &&
+		    (nfs_getattrcache(tnp, &nvattr, 1) || (nvattr.nva_nlink == 1)));
 		lck_mtx_lock(nfs_node_hash_mutex);
-		LIST_REMOVE(tnp, n_hash);
-		tnp->n_flag &= ~NHASHED;
+		if (tvprecycle && (tnp->n_hflag & NHHASHED)) {
+			/*
+			 * remove nfsnode from hash now so we can't accidentally find it
+			 * again if another object gets created with the same filehandle
+			 * before this vnode gets reclaimed
+			 */
+			LIST_REMOVE(tnp, n_hash);
+			tnp->n_hflag &= ~NHHASHED;
+			FSDBG(266, 0, tnp, tnp->n_flag, 0xb1eb1e);
+		}
 		lck_mtx_unlock(nfs_node_hash_mutex);
 	}
-	
+
 	/* purge the old name cache entries and enter the new one */
 	cache_purge(fvp);
 	if (tvp) {
 		cache_purge(tvp);
-		if (!error && !tnp->n_sillyrename) {
-			/* clear flags now: won't get nfs_inactive for recycled vnode */
+		if (tvprecycle) {
+			/* clear flags now: won't get nfs_vnop_inactive for recycled vnode */
 			/* clear all flags other than these */
-			tnp->n_flag &= (NMODIFIED | NFLUSHINPROG | NFLUSHWANT | NHASHED);
+			tnp->n_flag &= (NMODIFIED);
 			vnode_recycle(tvp);
 		}
 	}
-	if (!error)
+	if (!error) {
+		if (tdnp->n_flag & NNEGNCENTRIES) {
+			tdnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(tdvp);
+		}
 		cache_enter(tdvp, fvp, tcnp);
-
+		if (tdvp != fdvp) {	/* update parent pointer */
+			if (fnp->n_parent && !vnode_get(fnp->n_parent)) {
+				/* remove ref from old parent */
+				vnode_rele(fnp->n_parent);
+				vnode_put(fnp->n_parent);
+			}
+			fnp->n_parent = tdvp;
+			if (tdvp && !vnode_get(tdvp)) {
+				/* add ref to new parent */
+				vnode_ref(tdvp);
+				vnode_put(tdvp);
+			} else {
+				fnp->n_parent = NULL;
+			}
+		}
+	}
 out:
-	/*
-	 * Kludge: Map ENOENT => 0 assuming that it is a reply to a retry.
-	 */
-	if (error == ENOENT)
-		error = 0;
+	if (!nfs_getattr(fdnp, &nvattr, ctx, 1)) {
+		if (NFS_CHANGED_NC(nfsvers, fdnp, &nvattr)) {
+			fdnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge(fdvp);
+			NFS_CHANGED_UPDATE_NC(nfsvers, fdnp, &nvattr);
+		}
+	}
+	if (!nfs_getattr(tdnp, &nvattr, ctx, 1)) {
+		if (NFS_CHANGED_NC(nfsvers, tdnp, &nvattr)) {
+			tdnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge(tdvp);
+			NFS_CHANGED_UPDATE_NC(nfsvers, tdnp, &nvattr);
+		}
+	}
+	nfs_dulookup_finish(&fdul, fdnp, ctx);
+	nfs_dulookup_finish(&tdul, tdnp, ctx);
+	if (locked) {
+		/* unlock node */
+		lck_mtx_lock(nfs_node_hash_mutex);
+		tnp->n_hflag &= ~NHLOCKED;
+		if (tnp->n_hflag & NHLOCKWANT) {
+			tnp->n_hflag &= ~NHLOCKWANT;
+			wakeup(tnp);
+		}
+		lck_mtx_unlock(nfs_node_hash_mutex);
+	}
+	nfs_unlock4(fdnp, fnp, tdnp, tnp);
 	return (error);
 }
 
 /*
- * Do an nfs rename rpc. Called from nfs_rename() and nfs_sillyrename().
+ * Do an NFS rename rpc. Called from nfs_vnop_rename() and nfs_sillyrename().
  */
-static int
-nfs_renamerpc(fdvp, fnameptr, fnamelen, tdvp, tnameptr, tnamelen, cred, proc)
-	vnode_t fdvp;
-	char *fnameptr;
-	int fnamelen;
-	vnode_t tdvp;
-	char *tnameptr;
-	int tnamelen;
-	kauth_cred_t cred;
-	proc_t proc;
+int
+nfs3_rename_rpc(
+	nfsnode_t fdnp,
+	char *fnameptr,
+	int fnamelen,
+	nfsnode_t tdnp,
+	char *tnameptr,
+	int tnamelen,
+	vfs_context_t ctx)
 {
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, fwccpostattr = 0, twccpostattr = 0;
+	int error = 0, status, fwccpostattr = 0, twccpostattr = 0;
 	struct timespec fpremtime = { 0, 0 }, tpremtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
-	u_int64_t xid;
+	struct nfsmount *nmp;
+	int nfsvers;
+	u_int64_t xid, txid;
+	struct nfsm_chain nmreq, nmrep;
 
-	if (!VFSTONFS(vnode_mount(fdvp)))
+	nmp = NFSTONMP(fdnp);
+	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(fdvp);
-
-	nfsm_reqhead((NFSX_FH(v3) + NFSX_UNSIGNED)*2 + nfsm_rndup(fnamelen) +
-		      nfsm_rndup(tnamelen));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_RENAME]);
-	nfsm_fhtom(fdvp, v3);
-	nfsm_strtom(fnameptr, fnamelen, NFS_MAXNAMLEN, v3);
-	nfsm_fhtom(tdvp, v3);
-	nfsm_strtom(tnameptr, tnamelen, NFS_MAXNAMLEN, v3);
-	nfsm_request(fdvp, NFSPROC_RENAME, proc, cred, &xid);
-	if (v3 && mrep) {
-		u_int64_t txid = xid;
-
-		nfsm_wcc_data(fdvp, &fpremtime, fwccpostattr, &xid);
-		nfsm_wcc_data(tdvp, &tpremtime, twccpostattr, &txid);
-	}
-	nfsm_reqdone;
-	VTONFS(fdvp)->n_flag |= NMODIFIED;
+	nfsvers = nmp->nm_vers;
+	if ((nfsvers == NFS_VER2) &&
+	    ((fnamelen > NFS_MAXNAMLEN) || (tnamelen > NFS_MAXNAMLEN)))
+		return (ENAMETOOLONG);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		(NFSX_FH(nfsvers) + NFSX_UNSIGNED) * 2 +
+		nfsm_rndup(fnamelen) + nfsm_rndup(tnamelen));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, fdnp->n_fhp, fdnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, fnameptr, fnamelen);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, tdnp->n_fhp, tdnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, tnameptr, tnamelen);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+
+	error = nfs_request(fdnp, NULL, &nmreq, NFSPROC_RENAME, ctx, &nmrep, &xid, &status);
+
+	if (nfsvers == NFS_VER3) {
+		txid = xid;
+		nfsm_chain_get_wcc_data(error, &nmrep, fdnp, &fpremtime, &fwccpostattr, &xid);
+		nfsm_chain_get_wcc_data(error, &nmrep, tdnp, &tpremtime, &twccpostattr, &txid);
+	}
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	fdnp->n_flag |= NMODIFIED;
 	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(fdvp)->n_ncmtime, &fpremtime, ==))
-		VTONFS(fdvp)->n_ncmtime = VTONFS(fdvp)->n_vattr.nva_mtime;
+	if (nfstimespeccmp(&fdnp->n_ncmtime, &fpremtime, ==))
+		NFS_CHANGED_UPDATE_NC(nfsvers, fdnp, &fdnp->n_vattr);
 	if (!fwccpostattr)
-		NATTRINVALIDATE(VTONFS(fdvp));
-	VTONFS(tdvp)->n_flag |= NMODIFIED;
+		NATTRINVALIDATE(fdnp);
+	tdnp->n_flag |= NMODIFIED;
 	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(tdvp)->n_ncmtime, &tpremtime, ==))
-		VTONFS(tdvp)->n_ncmtime = VTONFS(tdvp)->n_vattr.nva_mtime;
+	if (nfstimespeccmp(&tdnp->n_ncmtime, &tpremtime, ==))
+		NFS_CHANGED_UPDATE_NC(nfsvers, tdnp, &tdnp->n_vattr);
 	if (!twccpostattr)
-		NATTRINVALIDATE(VTONFS(tdvp));
+		NATTRINVALIDATE(tdnp);
 	return (error);
 }
 
 /*
- * nfs hard link create call
+ * NFS hard link create call
  */
 static int
-nfs_link(ap)
+nfs3_vnop_link(
 	struct vnop_link_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		vnode_t a_tdvp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
 	vnode_t tdvp = ap->a_tdvp;
 	struct componentname *cnp = ap->a_cnp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0, attrflag = 0;
+	int error = 0, status, wccpostattr = 0, attrflag = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3;
-	u_int64_t xid;
-	kauth_cred_t cred;
-	proc_t p;
+	struct nfsmount *nmp;
+	nfsnode_t np = VTONFS(vp);
+	nfsnode_t tdnp = VTONFS(tdvp);
+	int nfsvers;
+	u_int64_t xid, txid;
+	struct nfsm_chain nmreq, nmrep;
 
-	if (vnode_mount(vp) != vnode_mount(tdvp)) {
+	if (vnode_mount(vp) != vnode_mount(tdvp))
 		return (EXDEV);
-	}
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
-
-	v3 = NFS_ISV3(vp);
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+	if ((nfsvers == NFS_VER2) && (cnp->cn_namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
 	/*
 	 * Push all writes to the server, so that the attribute cache
 	 * doesn't get "out of sync" with the server.
 	 * XXX There should be a better way!
 	 */
-	nfs_flush(vp, MNT_WAIT, cred, p, 0);
+	nfs_flush(np, MNT_WAIT, vfs_context_thread(ctx), V_IGNORE_WRITEERR);
 
-	nfsm_reqhead(NFSX_FH(v3)*2 + NFSX_UNSIGNED + nfsm_rndup(cnp->cn_namelen));
+	error = nfs_lock2(tdnp, np, NFS_NODE_LOCK_EXCLUSIVE);
 	if (error)
 		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_LINK]);
-	nfsm_fhtom(vp, v3);
-	nfsm_fhtom(tdvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN, v3);
-	nfsm_request(vp, NFSPROC_LINK, p, cred, &xid);
-	if (v3 && mrep) {
-		u_int64_t txid = xid;
 
-		nfsm_postop_attr_update(vp, v3, attrflag, &xid);
-		nfsm_wcc_data(tdvp, &premtime, wccpostattr, &txid);
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers)*2 + NFSX_UNSIGNED + nfsm_rndup(cnp->cn_namelen));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, tdnp->n_fhp, tdnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_LINK, ctx,
+			&nmrep, &xid, &status);
+	if (nfsvers == NFS_VER3) {
+		txid = xid;
+		nfsm_chain_postop_attr_update_flag(error, &nmrep, np, attrflag, &xid);
+		nfsm_chain_get_wcc_data(error, &nmrep, tdnp, &premtime, &wccpostattr, &txid);
 	}
-	nfsm_reqdone;
-
-	VTONFS(tdvp)->n_flag |= NMODIFIED;
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+	tdnp->n_flag |= NMODIFIED;
 	if (!attrflag)
-		NATTRINVALIDATE(VTONFS(vp));
+		NATTRINVALIDATE(np);
 	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(tdvp)->n_ncmtime, &premtime, ==))
-		VTONFS(tdvp)->n_ncmtime = VTONFS(tdvp)->n_vattr.nva_mtime;
+	if (nfstimespeccmp(&tdnp->n_ncmtime, &premtime, ==))
+		NFS_CHANGED_UPDATE_NC(nfsvers, tdnp, &tdnp->n_vattr);
 	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(tdvp));
+		NATTRINVALIDATE(tdnp);
+	if (!error && (tdnp->n_flag & NNEGNCENTRIES)) {
+		tdnp->n_flag &= ~NNEGNCENTRIES;
+		cache_purge_negatives(tdvp);
+	}
+	nfs_unlock2(tdnp, np);
 	/*
 	 * Kludge: Map EEXIST => 0 assuming that it is a reply to a retry.
 	 */
@@ -2590,10 +3555,10 @@ nfs_link(ap)
 }
 
 /*
- * nfs symbolic link create call
+ * NFS symbolic link create call
  */
 static int
-nfs_symlink(ap)
+nfs3_vnop_symlink(
 	struct vnop_symlink_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
@@ -2602,36 +3567,35 @@ nfs_symlink(ap)
 		struct vnode_attr *a_vap;
 		char *a_target;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t dvp = ap->a_dvp;
 	struct vnode_attr *vap = ap->a_vap;
 	struct componentname *cnp = ap->a_cnp;
-	struct nfs_vattr nvattr;
-	struct nfsv2_sattr *sp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int slen, error = 0, wccpostattr = 0, gotvp = 0;
+	struct nfs_vattr nvattr, dnvattr;
+	fhandle_t fh;
+	int slen, error = 0, lockerror = ENOENT, status, wccpostattr = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	vnode_t newvp = (vnode_t)0;
-	int v3 = NFS_ISV3(dvp);
-	int gotuid, gotgid;
-	u_int64_t xid;
-	kauth_cred_t cred;
-	proc_t p;
-	struct nfsnode *np = NULL;
+	vnode_t newvp = NULL;
+	int nfsvers, gotuid, gotgid;
+	u_int64_t xid, dxid;
+	nfsnode_t np = NULL;
+	nfsnode_t dnp = VTONFS(dvp);
+	struct nfsmount *nmp;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsreq *req = NULL;
+	struct nfs_dulookup dul;
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	nmp = VTONMP(dvp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
 	slen = strlen(ap->a_target);
-	nfsm_reqhead(NFSX_FH(v3) + 2*NFSX_UNSIGNED +
-	    nfsm_rndup(cnp->cn_namelen) + nfsm_rndup(slen) + NFSX_SATTR(v3));
-	if (error)
-		return (error);
+	if ((nfsvers == NFS_VER2) &&
+	    ((cnp->cn_namelen > NFS_MAXNAMLEN) || (slen > NFS_MAXPATHLEN)))
+		return (ENAMETOOLONG);
 
 	VATTR_SET_SUPPORTED(vap, va_mode);
 	VATTR_SET_SUPPORTED(vap, va_uid);
@@ -2642,67 +3606,97 @@ nfs_symlink(ap)
 	gotuid = VATTR_IS_ACTIVE(vap, va_uid);
 	gotgid = VATTR_IS_ACTIVE(vap, va_gid);
 
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_SYMLINK]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN, v3);
-	if (v3) {
-		nfsm_v3sattr(vap);
-	}
-	nfsm_strtom(ap->a_target, slen, NFS_MAXPATHLEN, v3);
-	if (!v3) {
-		struct timespec neg1time = { -1, -1 };
-		nfsm_build(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		sp->sa_mode = vtonfsv2_mode(VLNK,
-			(VATTR_IS_ACTIVE(vap, va_mode) ? vap->va_mode : 0600));
-		sp->sa_uid = gotuid ? (u_long)txdr_unsigned(vap->va_uid) : nfs_xdrneg1;
-		sp->sa_gid = gotgid ? (u_long)txdr_unsigned(vap->va_gid) : nfs_xdrneg1;
-		sp->sa_size = nfs_xdrneg1;
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			txdr_nfsv2time(&vap->va_access_time, &sp->sa_atime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_atime);
-		}
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			txdr_nfsv2time(&vap->va_modify_time, &sp->sa_mtime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_mtime);
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + 2 * NFSX_UNSIGNED +
+		nfsm_rndup(cnp->cn_namelen) + nfsm_rndup(slen) + NFSX_SATTR(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_add_v3sattr(error, &nmreq, vap);
+	nfsm_chain_add_string(error, &nmreq, ap->a_target, slen);
+	if (nfsvers == NFS_VER2)
+		nfsm_chain_add_v2sattr(error, &nmreq, vap, -1);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC_SYMLINK,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+	if (!error) {
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	}
+
+	dxid = xid;
+	if (!error && !status) {
+		if (dnp->n_flag & NNEGNCENTRIES) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(dvp);
 		}
+		if (nfsvers == NFS_VER3)
+			error = nfsm_chain_get_fh_attr(&nmrep, dnp, ctx, nfsvers, &xid, &fh, &nvattr);
+		else
+			fh.fh_len = 0;
 	}
-	nfsm_request(dvp, NFSPROC_SYMLINK, p, cred, &xid);
-	if (v3 && mrep) {
-		u_int64_t dxid = xid;
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &dxid);
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 
-		if (!error)
-			nfsm_mtofh(dvp, cnp, newvp, v3, &xid, gotvp);
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &dxid);
+	if (!lockerror) {
+		dnp->n_flag |= NMODIFIED;
+		/* if directory hadn't changed, update namecache mtime */
+		if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
+		if (!wccpostattr)
+			NATTRINVALIDATE(dnp);
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(dvp);
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
+		}
 	}
-	nfsm_reqdone;
 
-	VTONFS(dvp)->n_flag |= NMODIFIED;
-	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
-	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
+	if (!error && fh.fh_len)
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+	if (!error && np)
+		newvp = NFSTOV(np);
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
 
 	/*
 	 * Kludge: Map EEXIST => 0 assuming that you have a reply to a retry
 	 * if we can succeed in looking up the symlink.
 	 */
-	if ((error == EEXIST) || (!error && !gotvp)) {
+	if ((error == EEXIST) || (!error && !newvp)) {
 		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
 			newvp = NULL;
 		}
-		error = nfs_lookitup(dvp, cnp->cn_nameptr, cnp->cn_namelen, cred, p, &np);
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
 		if (!error) {
 			newvp = NFSTOV(np);
 			if (vnode_vtype(newvp) != VLNK)
 				error = EEXIST;
 		}
 	}
+	if (!lockerror)
+		nfs_unlock(dnp);
 	if (!error && (gotuid || gotgid) &&
-	    (!newvp || nfs_getattrcache(newvp, &nvattr) ||
+	    (!newvp || nfs_getattrcache(np, &nvattr, 1) ||
 	     (gotuid && (nvattr.nva_uid != vap->va_uid)) ||
 	     (gotgid && (nvattr.nva_gid != vap->va_gid)))) {
 		/* clear ID bits if server didn't use them (or we can't tell) */
@@ -2710,19 +3704,22 @@ nfs_symlink(ap)
 		VATTR_CLEAR_SUPPORTED(vap, va_gid);
 	}
 	if (error) {
-		if (newvp)
+		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
+		}
 	} else {
+		nfs_unlock(np);
 		*ap->a_vpp = newvp;
 	}
 	return (error);
 }
 
 /*
- * nfs make dir call
+ * NFS make dir call
  */
 static int
-nfs_mkdir(ap)
+nfs3_vnop_mkdir(
 	struct vnop_mkdir_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
@@ -2730,37 +3727,32 @@ nfs_mkdir(ap)
 		struct componentname *a_cnp;
 		struct vnode_attr *a_vap;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t dvp = ap->a_dvp;
 	struct vnode_attr *vap = ap->a_vap;
 	struct componentname *cnp = ap->a_cnp;
-	struct nfs_vattr nvattr;
-	struct nfsv2_sattr *sp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	int len;
-	struct nfsnode *np = (struct nfsnode *)0;
-	vnode_t newvp = (vnode_t)0;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0;
+	struct nfs_vattr nvattr, dnvattr;
+	nfsnode_t np = NULL;
+	struct nfsmount *nmp;
+	nfsnode_t dnp = VTONFS(dvp);
+	vnode_t newvp = NULL;
+	int error = 0, lockerror = ENOENT, status, wccpostattr = 0;
 	struct timespec premtime = { 0, 0 };
-	int gotvp = 0;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3 = NFS_ISV3(dvp);
-	int gotuid, gotgid;
+	int nfsvers, gotuid, gotgid;
 	u_int64_t xid, dxid;
-	kauth_cred_t cred;
-	proc_t p;
-
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	fhandle_t fh;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsreq *req = NULL;
+	struct nfs_dulookup dul;
 
-	len = cnp->cn_namelen;
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED + nfsm_rndup(len) + NFSX_SATTR(v3));
-	if (error)
-		return (error);
+	nmp = VTONMP(dvp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+	if ((nfsvers == NFS_VER2) && (cnp->cn_namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
 	VATTR_SET_SUPPORTED(vap, va_mode);
 	VATTR_SET_SUPPORTED(vap, va_uid);
@@ -2771,61 +3763,93 @@ nfs_mkdir(ap)
 	gotuid = VATTR_IS_ACTIVE(vap, va_uid);
 	gotgid = VATTR_IS_ACTIVE(vap, va_gid);
 
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_MKDIR]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, len, NFS_MAXNAMLEN, v3);
-	if (v3) {
-		nfsm_v3sattr(vap);
-	} else {
-		struct timespec neg1time = { -1, -1 };
-		nfsm_build(sp, struct nfsv2_sattr *, NFSX_V2SATTR);
-		sp->sa_mode = vtonfsv2_mode(VDIR,
-			(VATTR_IS_ACTIVE(vap, va_mode) ? vap->va_mode : 0600));
-		sp->sa_uid = gotuid ? (u_long)txdr_unsigned(vap->va_uid) : nfs_xdrneg1;
-		sp->sa_gid = gotgid ? (u_long)txdr_unsigned(vap->va_gid) : nfs_xdrneg1;
-		sp->sa_size = nfs_xdrneg1;
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) {
-			txdr_nfsv2time(&vap->va_access_time, &sp->sa_atime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_atime);
-		}
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) {
-			txdr_nfsv2time(&vap->va_modify_time, &sp->sa_mtime);
-		} else {
-			txdr_nfsv2time(&neg1time, &sp->sa_mtime);
-		}
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + NFSX_UNSIGNED +
+		nfsm_rndup(cnp->cn_namelen) + NFSX_SATTR(nfsvers));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_add_v3sattr(error, &nmreq, vap);
+	else
+		nfsm_chain_add_v2sattr(error, &nmreq, vap, -1);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC_MKDIR,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+	if (!error) {
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
 	}
-	nfsm_request(dvp, NFSPROC_MKDIR, p, cred, &xid);
+
 	dxid = xid;
+	if (!error && !status) {
+		if (dnp->n_flag & NNEGNCENTRIES) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge_negatives(dvp);
+		}
+		error = nfsm_chain_get_fh_attr(&nmrep, dnp, ctx, nfsvers, &xid, &fh, &nvattr);
+	}
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &dxid);
 	if (!error)
-		nfsm_mtofh(dvp, cnp, newvp, v3, &xid, gotvp);
-	if (v3 && mrep)
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &dxid);
-	nfsm_reqdone;
-	VTONFS(dvp)->n_flag |= NMODIFIED;
-	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
-	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	if (!lockerror) {
+		dnp->n_flag |= NMODIFIED;
+		/* if directory hadn't changed, update namecache mtime */
+		if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
+		if (!wccpostattr)
+			NATTRINVALIDATE(dnp);
+		if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+			if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+				dnp->n_flag &= ~NNEGNCENTRIES;
+				cache_purge(dvp);
+				NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+			}
+		}
+	}
+
+	if (!error && fh.fh_len)
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len, &nvattr, &xid, NG_MAKEENTRY, &np);
+	if (!error && np)
+		newvp = NFSTOV(np);
+
+	nfs_dulookup_finish(&dul, dnp, ctx);
+
 	/*
 	 * Kludge: Map EEXIST => 0 assuming that you have a reply to a retry
 	 * if we can succeed in looking up the directory.
 	 */
-	if (error == EEXIST || (!error && !gotvp)) {
+	if (error == EEXIST || (!error && !newvp)) {
 		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
 			newvp = NULL;
 		}
-		error = nfs_lookitup(dvp, cnp->cn_nameptr, len, cred, p, &np);
+		error = nfs_lookitup(dnp, cnp->cn_nameptr, cnp->cn_namelen, ctx, &np);
 		if (!error) {
 			newvp = NFSTOV(np);
 			if (vnode_vtype(newvp) != VDIR)
 				error = EEXIST;
 		}
 	}
+	if (!lockerror)
+		nfs_unlock(dnp);
 	if (!error && (gotuid || gotgid) &&
-	    (!newvp || nfs_getattrcache(newvp, &nvattr) ||
+	    (!newvp || nfs_getattrcache(np, &nvattr, 1) ||
 	     (gotuid && (nvattr.nva_uid != vap->va_uid)) ||
 	     (gotgid && (nvattr.nva_gid != vap->va_gid)))) {
 		/* clear ID bits if server didn't use them (or we can't tell) */
@@ -2833,62 +3857,100 @@ nfs_mkdir(ap)
 		VATTR_CLEAR_SUPPORTED(vap, va_gid);
 	}
 	if (error) {
-		if (newvp)
+		if (newvp) {
+			nfs_unlock(np);
 			vnode_put(newvp);
+		}
 	} else {
+		nfs_unlock(np);
 		*ap->a_vpp = newvp;
 	}
 	return (error);
 }
 
 /*
- * nfs remove directory call
+ * NFS remove directory call
  */
 static int
-nfs_rmdir(ap)
+nfs3_vnop_rmdir(
 	struct vnop_rmdir_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_dvp;
 		vnode_t a_vp;
 		struct componentname *a_cnp;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
 	vnode_t dvp = ap->a_dvp;
 	struct componentname *cnp = ap->a_cnp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0;
+	int error = 0, status, wccpostattr = 0;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
-	int v3 = NFS_ISV3(dvp);
+	struct nfsmount *nmp;
+	nfsnode_t np = VTONFS(vp);
+	nfsnode_t dnp = VTONFS(dvp);
+	struct nfs_vattr dnvattr;
+	int nfsvers;
 	u_int64_t xid;
-	kauth_cred_t cred;
-	proc_t p;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsreq *req = NULL;
+	struct nfs_dulookup dul;
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+	if ((nfsvers == NFS_VER2) && (cnp->cn_namelen > NFS_MAXNAMLEN))
+		return (ENAMETOOLONG);
 
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED + nfsm_rndup(cnp->cn_namelen));
-	if (error)
+	nfs_dulookup_init(&dul, dnp, cnp->cn_nameptr, cnp->cn_namelen);
+
+	if ((error = nfs_lock2(dnp, np, NFS_NODE_LOCK_EXCLUSIVE)))
 		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_RMDIR]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(cnp->cn_nameptr, cnp->cn_namelen, NFS_MAXNAMLEN, v3);
-	nfsm_request(dvp, NFSPROC_RMDIR, p, cred, &xid);
-	if (v3 && mrep)
-		nfsm_wcc_data(dvp, &premtime, wccpostattr, &xid);
-	nfsm_reqdone;
-	VTONFS(dvp)->n_flag |= NMODIFIED;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + NFSX_UNSIGNED + nfsm_rndup(cnp->cn_namelen));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, cnp->cn_nameptr, cnp->cn_namelen);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC_RMDIR,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, &req);
+	if (!error) {
+		nfs_dulookup_start(&dul, dnp, ctx);
+		error = nfs_request_async_finish(req, &nmrep, &xid, &status);
+	}
+
+	if (nfsvers == NFS_VER3)
+		nfsm_chain_get_wcc_data(error, &nmrep, dnp, &premtime, &wccpostattr, &xid);
+	if (!error)
+		error = status;
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
+
+	dnp->n_flag |= NMODIFIED;
 	/* if directory hadn't changed, update namecache mtime */
-	if (nfstimespeccmp(&VTONFS(dvp)->n_ncmtime, &premtime, ==))
-		VTONFS(dvp)->n_ncmtime = VTONFS(dvp)->n_vattr.nva_mtime;
+	if (nfstimespeccmp(&dnp->n_ncmtime, &premtime, ==))
+		NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnp->n_vattr);
 	if (!wccpostattr)
-		NATTRINVALIDATE(VTONFS(dvp));
+		NATTRINVALIDATE(dnp);
 	cache_purge(vp);
+	if (!nfs_getattr(dnp, &dnvattr, ctx, 1)) {
+		if (NFS_CHANGED_NC(nfsvers, dnp, &dnvattr)) {
+			dnp->n_flag &= ~NNEGNCENTRIES;
+			cache_purge(dvp);
+			NFS_CHANGED_UPDATE_NC(nfsvers, dnp, &dnvattr);
+		}
+	}
+	nfs_dulookup_finish(&dul, dnp, ctx);
+	nfs_unlock2(dnp, np);
+
 	/*
 	 * Kludge: Map ENOENT => 0 assuming that you have a reply to a retry.
 	 */
@@ -2901,18 +3963,21 @@ nfs_rmdir(ap)
 		 * before this vnode gets reclaimed
 		 */
 		lck_mtx_lock(nfs_node_hash_mutex);
-		LIST_REMOVE(VTONFS(vp), n_hash);
-		VTONFS(vp)->n_flag &= ~NHASHED;
+		if (np->n_hflag & NHHASHED) {
+			LIST_REMOVE(np, n_hash);
+			np->n_hflag &= ~NHHASHED;
+			FSDBG(266, 0, np, np->n_flag, 0xb1eb1e);
+		}
 		lck_mtx_unlock(nfs_node_hash_mutex);
 	}
 	return (error);
 }
 
 /*
- * nfs readdir call
+ * NFS readdir call
  */
 static int
-nfs_readdir(ap)
+nfs_vnop_readdir(
 	struct vnop_readdir_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -2921,45 +3986,57 @@ nfs_readdir(ap)
 		int *a_ncookies;
 		u_long **a_cookies;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
+	vfs_context_t ctx = ap->a_context;
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
+	struct nfsmount *nmp;
 	struct uio *uio = ap->a_uio;
-	int tresid, error;
+	int tresid, error, nfsvers;
 	struct nfs_vattr nvattr;
-	kauth_cred_t cred;
-	proc_t p;
 
 	if (vnode_vtype(vp) != VDIR)
 		return (EPERM);
 
-	cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
+	nmp = VTONMP(vp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 
 	/*
 	 * First, check for hit on the EOF offset cache
 	 */
 	if (np->n_direofoffset > 0 && uio->uio_offset >= np->n_direofoffset &&
 	    (np->n_flag & NMODIFIED) == 0) {
-		if (!nfs_getattr(vp, &nvattr, cred, p)) {
-			if (nfstimespeccmp(&np->n_mtime, &nvattr.nva_mtime, ==)) {
+		if (!nfs_getattr(np, &nvattr, ctx, 1)) {
+			if (!NFS_CHANGED(nfsvers, np, &nvattr)) {
+				nfs_unlock(np);
 				OSAddAtomic(1, (SInt32*)&nfsstats.direofcache_hits);
+				if (ap->a_eofflag)
+					*ap->a_eofflag = 1;
 				return (0);
 			}
-			if (nfstimespeccmp(&np->n_ncmtime, &nvattr.nva_mtime, !=)) {
+			if (NFS_CHANGED_NC(nfsvers, np, &nvattr)) {
 				/* directory changed, purge any name cache entries */
+				np->n_flag &= ~NNEGNCENTRIES;
 				cache_purge(vp);
 			}
 		}
 	}
+	nfs_unlock(np);
+	if (ap->a_eofflag)
+		*ap->a_eofflag = 0;
 
 	/*
 	 * Call nfs_bioread() to do the real work.
 	 */
 	// LP64todo - fix this
 	tresid = uio_uio_resid(uio);
-	error = nfs_bioread(vp, uio, 0, cred, p);
+	error = nfs_bioread(np, uio, 0, ap->a_eofflag, ctx);
 
 	if (!error && uio_uio_resid(uio) == tresid)
 		OSAddAtomic(1, (SInt32*)&nfsstats.direofcache_misses);
@@ -2967,113 +4044,112 @@ nfs_readdir(ap)
 }
 
 /*
- * Readdir rpc call.
- * Called from below the buffer cache by nfs_doio().
+ * Readdir RPC call.
+ * Called from below the buffer cache by nfs_buf_readdir().
  */
+#define	DIRHDSIZ	((int)(sizeof(struct dirent) - (MAXNAMLEN + 1)))
 int
-nfs_readdirrpc(
-	vnode_t vp,
-	struct uio *uiop,
-	kauth_cred_t cred,
-	proc_t p)
+nfs3_readdir_rpc(nfsnode_t dnp, struct uio *uiop, vfs_context_t ctx)
 {
-	register int len, skiplen, left;
-	register struct dirent *dp;
-	register u_long *tl;
-	register caddr_t cp;
-	register long t1, t2;
-	register nfsuint64 *cookiep;
-	caddr_t bpos, dpos, cp2;
-	mbuf_t mreq, mrep, md, mb, mb2;
+	int len, skiplen, left;
+	struct dirent *dp = NULL;
+	nfsuint64 *cookiep;
 	nfsuint64 cookie;
 	struct nfsmount *nmp;
-	struct nfsnode *dnp = VTONFS(vp);
 	u_quad_t fileno;
-	int error = 0, tlen, more_dirs = 1, blksiz = 0, bigenough = 1;
-	int attrflag;
-	int v3, nmreaddirsize;
+	int error = 0, lockerror, status, tlen, more_dirs = 1, blksiz = 0, bigenough = 1, eof;
+	int nfsvers, nmreaddirsize;
 	u_int64_t xid;
+	struct nfsm_chain nmreq, nmrep;
+	char *cp;
 
-#ifndef nolint
-	dp = (struct dirent *)0;
-#endif
 #if DIAGNOSTIC
-	if (uiop->uio_iovcnt != 1 || (uiop->uio_offset & (NFS_DIRBLKSIZ - 1)) ||
-		(uio_uio_resid(uiop) & (NFS_DIRBLKSIZ - 1)))
+	/* XXX limitation based on need to adjust uio */
+	if (uiop->uio_iovcnt != 1 || (uiop->uio_offset & (DIRBLKSIZ - 1)) ||
+		(uio_uio_resid(uiop) & (DIRBLKSIZ - 1)))
 		panic("nfs_readdirrpc: bad uio");
 #endif
-	nmp = VFSTONFS(vnode_mount(vp));
+	nmp = NFSTONMP(dnp);
 	if (!nmp)
 		return (ENXIO);
-	v3 = NFS_ISV3(vp);
+	nfsvers = nmp->nm_vers;
 	nmreaddirsize = nmp->nm_readdirsize;
 
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+		return (lockerror);
+
 	/*
 	 * If there is no cookie, assume directory was stale.
 	 */
 	cookiep = nfs_getcookie(dnp, uiop->uio_offset, 0);
 	if (cookiep)
 		cookie = *cookiep;
-	else
+	else {
+		nfs_unlock(dnp);
 		return (NFSERR_BAD_COOKIE);
+	}
+
 	/*
 	 * Loop around doing readdir rpc's of size nm_readdirsize
 	 * truncated to a multiple of DIRBLKSIZ.
 	 * The stopping criteria is EOF or buffer full.
 	 */
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
 	while (more_dirs && bigenough) {
-		nfsm_reqhead(NFSX_FH(v3) + NFSX_READDIR(v3));
-		if (error)
-			goto nfsmout;
-		OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_READDIR]);
-		nfsm_fhtom(vp, v3);
-		if (v3) {
-			nfsm_build(tl, u_long *, 5 * NFSX_UNSIGNED);
-			*tl++ = cookie.nfsuquad[0];
-			*tl++ = cookie.nfsuquad[1];
-			*tl++ = dnp->n_cookieverf.nfsuquad[0];
-			*tl++ = dnp->n_cookieverf.nfsuquad[1];
+		nfsm_chain_build_alloc_init(error, &nmreq,
+			NFSX_FH(nfsvers) + NFSX_READDIR(nfsvers));
+		nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+		if (nfsvers == NFS_VER3) {
+			/* opaque values don't need swapping, but as long */
+			/* as we are consistent about it, it should be ok */
+			nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[0]);
+			nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[1]);
+			nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[0]);
+			nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[1]);
 		} else {
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED);
-			*tl++ = cookie.nfsuquad[0];
+			nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[0]);
 		}
-		*tl = txdr_unsigned(nmreaddirsize);
-		nfsm_request(vp, NFSPROC_READDIR, p, cred, &xid);
-		if (v3) {
-			if (mrep) {
-				nfsm_postop_attr_update(vp, v3, attrflag, &xid);
-			}
-			if (!error) {
-				nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-				dnp->n_cookieverf.nfsuquad[0] = *tl++;
-				dnp->n_cookieverf.nfsuquad[1] = *tl;
-			} else {
-				mbuf_freem(mrep);
-				goto nfsmout;
-			}
-		} else if (!mrep) {
-			// XXX assert error?
-			goto nfsmout;
+		nfsm_chain_add_32(error, &nmreq, nmreaddirsize);
+		nfsm_chain_build_done(error, &nmreq);
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
+		nfsmout_if(error);
+
+		error = nfs_request(dnp, NULL, &nmreq, NFSPROC_READDIR, ctx,
+				&nmrep, &xid, &status);
+
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+			error = lockerror;
+
+		if (nfsvers == NFS_VER3)
+			nfsm_chain_postop_attr_update(error, &nmrep, dnp, &xid);
+		if (!error)
+			error = status;
+		if (nfsvers == NFS_VER3) {
+			nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[0]);
+			nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[1]);
+		}
+		nfsm_chain_get_32(error, &nmrep, more_dirs);
+
+		if (!lockerror) {
+			nfs_unlock(dnp);
+			lockerror = ENOENT;
 		}
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-		more_dirs = fxdr_unsigned(int, *tl);
-	
+		nfsmout_if(error);
+
 		/* loop thru the dir entries, doctoring them to 4bsd form */
 		while (more_dirs && bigenough) {
-			if (v3) {
-				nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-				fxdr_hyper(tl, &fileno);
-				len = fxdr_unsigned(int, *(tl + 2));
-			} else {
-				nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
-				fileno = fxdr_unsigned(u_quad_t, *tl++);
-				len = fxdr_unsigned(int, *tl);
-			}
+			if (nfsvers == NFS_VER3)
+				nfsm_chain_get_64(error, &nmrep, fileno);
+			else
+				nfsm_chain_get_32(error, &nmrep, fileno);
+			nfsm_chain_get_32(error, &nmrep, len);
+			nfsmout_if(error);
 			/* Note: v3 supports longer names, but struct dirent doesn't */
 			/* so we just truncate the names to fit */
 			if (len <= 0) {
 				error = EBADRPC;
-				mbuf_freem(mrep);
 				goto nfsmout;
 			}
 			if (len > MAXNAMLEN) {
@@ -3086,7 +4162,7 @@ nfs_readdirrpc(
 			if (tlen == len)
 				tlen += 4;	/* To ensure null termination */
 			left = DIRBLKSIZ - blksiz;
-			if ((tlen + (int)DIRHDSIZ) > left) {
+			if ((tlen + DIRHDSIZ) > left) {
 				dp->d_reclen += left;
 				uio_iov_base_add(uiop, left);
 				uio_iov_len_add(uiop, -left);
@@ -3094,7 +4170,7 @@ nfs_readdirrpc(
 				uio_uio_resid_add(uiop, -left);
 				blksiz = 0;
 			}
-			if ((tlen + (int)DIRHDSIZ) > uio_uio_resid(uiop))
+			if ((tlen + DIRHDSIZ) > uio_uio_resid(uiop))
 				bigenough = 0;
 			if (bigenough) {
 				// LP64todo - fix this!
@@ -3115,7 +4191,8 @@ nfs_readdirrpc(
 				uio_iov_len_add(uiop, -((int)DIRHDSIZ));
 #endif
 				uio_iov_base_add(uiop, DIRHDSIZ);
-				nfsm_mtouio(uiop, len);
+				error = nfsm_chain_get_uio(&nmrep, len, uiop);
+				nfsmout_if(error);
 				// LP64todo - fix this!
 				cp = CAST_DOWN(caddr_t, uio_iov_base(uiop));
 				tlen -= len;
@@ -3124,34 +4201,40 @@ nfs_readdirrpc(
 				uio_iov_len_add(uiop, -tlen);
 				uiop->uio_offset += tlen;
 				uio_uio_resid_add(uiop, -tlen);
+				if (skiplen)
+					nfsm_chain_adv(error, &nmrep,
+						nfsm_rndup(len + skiplen) - nfsm_rndup(len));
 			} else {
-				nfsm_adv(nfsm_rndup(len));
-			}
-			if (skiplen)
-				nfsm_adv(nfsm_rndup(skiplen));
-			if (v3) {
-				nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-			} else {
-				nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED);
+				nfsm_chain_adv(error, &nmrep, nfsm_rndup(len + skiplen));
 			}
 			if (bigenough) {
-				cookie.nfsuquad[0] = *tl++;
-				if (v3)
-					cookie.nfsuquad[1] = *tl++;
-			} else if (v3)
-				tl += 2;
+				nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[0]);
+				if (nfsvers == NFS_VER3)
+					nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[1]);
+			} else if (nfsvers == NFS_VER3)
+				nfsm_chain_adv(error, &nmrep, 2 * NFSX_UNSIGNED);
 			else
-				tl++;
-			more_dirs = fxdr_unsigned(int, *tl);
+				nfsm_chain_adv(error, &nmrep, NFSX_UNSIGNED);
+			nfsm_chain_get_32(error, &nmrep, more_dirs);
+			nfsmout_if(error);
 		}
 		/*
 		 * If at end of rpc data, get the eof boolean
 		 */
 		if (!more_dirs) {
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-			more_dirs = (fxdr_unsigned(int, *tl) == 0);
+			nfsm_chain_get_32(error, &nmrep, eof);
+			if (!error)
+				more_dirs = (eof == 0);
 		}
-		mbuf_freem(mrep);
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+			error = lockerror;
+		nfsmout_if(error);
+		nfsm_chain_cleanup(&nmrep);
+		nfsm_chain_null(&nmreq);
+	}
+	if (!lockerror) {
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
 	}
 	/*
 	 * Fill last record, iff any, out to a multiple of DIRBLKSIZ
@@ -3166,6 +4249,10 @@ nfs_readdirrpc(
 		uio_uio_resid_add(uiop, -left);
 	}
 
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
 	/*
 	 * We are now either at the end of the directory or have filled the
 	 * block.
@@ -3179,7 +4266,12 @@ nfs_readdirrpc(
 		if (cookiep)
 			*cookiep = cookie;
 	}
+
 nfsmout:
+	if (!lockerror)
+		nfs_unlock(dnp);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
@@ -3187,99 +4279,110 @@ nfsmout:
  * NFS V3 readdir plus RPC. Used in place of nfs_readdirrpc().
  */
 int
-nfs_readdirplusrpc(
-	vnode_t vp,
-	struct uio *uiop,
-	kauth_cred_t cred,
-	proc_t p)
+nfs3_readdirplus_rpc(nfsnode_t dnp, struct uio *uiop, vfs_context_t ctx)
 {
-	int len, skiplen, left;
-	struct dirent *dp;
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
+	size_t len, tlen, skiplen, left;
+	struct dirent *dp = NULL;
 	vnode_t newvp;
 	nfsuint64 *cookiep;
-	caddr_t bpos, dpos, cp2;
-	mbuf_t mreq, mrep, md, mb, mb2;
 	struct componentname cn, *cnp = &cn;
 	nfsuint64 cookie;
 	struct nfsmount *nmp;
-	struct nfsnode *dnp = VTONFS(vp), *np;
+	nfsnode_t np;
 	u_char *fhp;
 	u_quad_t fileno;
-	int error = 0, tlen, more_dirs = 1, blksiz = 0, doit, bigenough = 1, i;
-	int attrflag, fhsize, nmreaddirsize, nmrsize;
+	int error = 0, lockerror, status, more_dirs = 1, blksiz = 0, doit, bigenough = 1;
+	int nfsvers, nmreaddirsize, nmrsize, attrflag, eof;
+	size_t fhsize;
 	u_int64_t xid, savexid;
 	struct nfs_vattr nvattr;
+	struct nfsm_chain nmreq, nmrep;
+	char *cp;
 
-#ifndef nolint
-	dp = (struct dirent *)0;
-#endif
 #if DIAGNOSTIC
+	/* XXX limitation based on need to adjust uio */
 	if (uiop->uio_iovcnt != 1 || (uiop->uio_offset & (DIRBLKSIZ - 1)) ||
 		(uio_uio_resid(uiop) & (DIRBLKSIZ - 1)))
-		panic("nfs_readdirplusrpc: bad uio");
+		panic("nfs3_readdirplus_rpc: bad uio");
 #endif
-	nmp = VFSTONFS(vnode_mount(vp));
+	nmp = NFSTONMP(dnp);
 	if (!nmp)
 		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 	nmreaddirsize = nmp->nm_readdirsize;
 	nmrsize = nmp->nm_rsize;
 
 	bzero(cnp, sizeof(*cnp));
 	newvp = NULLVP;
 
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+		return (lockerror);
+
 	/*
 	 * If there is no cookie, assume directory was stale.
 	 */
 	cookiep = nfs_getcookie(dnp, uiop->uio_offset, 0);
 	if (cookiep)
 		cookie = *cookiep;
-	else
+	else {
+		nfs_unlock(dnp);
 		return (NFSERR_BAD_COOKIE);
+	}
+
 	/*
 	 * Loop around doing readdir rpc's of size nm_readdirsize
 	 * truncated to a multiple of DIRBLKSIZ.
 	 * The stopping criteria is EOF or buffer full.
 	 */
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
 	while (more_dirs && bigenough) {
-		nfsm_reqhead(NFSX_FH(1) + 6 * NFSX_UNSIGNED);
-		if (error)
-			goto nfsmout;
-		OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_READDIRPLUS]);
-		nfsm_fhtom(vp, 1);
- 		nfsm_build(tl, u_long *, 6 * NFSX_UNSIGNED);
-		*tl++ = cookie.nfsuquad[0];
-		*tl++ = cookie.nfsuquad[1];
-		*tl++ = dnp->n_cookieverf.nfsuquad[0];
-		*tl++ = dnp->n_cookieverf.nfsuquad[1];
-		*tl++ = txdr_unsigned(nmreaddirsize);
-		*tl = txdr_unsigned(nmrsize);
-		nfsm_request(vp, NFSPROC_READDIRPLUS, p, cred, &xid);
+		nfsm_chain_build_alloc_init(error, &nmreq,
+			NFSX_FH(NFS_VER3) + 6 * NFSX_UNSIGNED);
+		nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+		/* opaque values don't need swapping, but as long */
+		/* as we are consistent about it, it should be ok */
+		nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[0]);
+		nfsm_chain_add_32(error, &nmreq, cookie.nfsuquad[1]);
+		nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[0]);
+		nfsm_chain_add_32(error, &nmreq, dnp->n_cookieverf.nfsuquad[1]);
+		nfsm_chain_add_32(error, &nmreq, nmreaddirsize);
+		nfsm_chain_add_32(error, &nmreq, nmrsize);
+		nfsm_chain_build_done(error, &nmreq);
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
+		nfsmout_if(error);
+
+		error = nfs_request(dnp, NULL, &nmreq, NFSPROC_READDIRPLUS, ctx,
+				&nmrep, &xid, &status);
+
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+			error = lockerror;
+
 		savexid = xid;
-		if (mrep) {
-			nfsm_postop_attr_update(vp, 1, attrflag, &xid);
-		}
-		if (error) {
-			mbuf_freem(mrep);
-			goto nfsmout;
+		nfsm_chain_postop_attr_update(error, &nmrep, dnp, &xid);
+		if (!error)
+			error = status;
+		nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[0]);
+		nfsm_chain_get_32(error, &nmrep, dnp->n_cookieverf.nfsuquad[1]);
+		nfsm_chain_get_32(error, &nmrep, more_dirs);
+
+		if (!lockerror) {
+			nfs_unlock(dnp);
+			lockerror = ENOENT;
 		}
-		nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-		dnp->n_cookieverf.nfsuquad[0] = *tl++;
-		dnp->n_cookieverf.nfsuquad[1] = *tl++;
-		more_dirs = fxdr_unsigned(int, *tl);
+		nfsmout_if(error);
+		nfsmout_if(error);
 
 		/* loop thru the dir entries, doctoring them to 4bsd form */
 		while (more_dirs && bigenough) {
-			nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
-			fxdr_hyper(tl, &fileno);
-			len = fxdr_unsigned(int, *(tl + 2));
+			nfsm_chain_get_64(error, &nmrep, fileno);
+			nfsm_chain_get_32(error, &nmrep, len);
+			nfsmout_if(error);
 			/* Note: v3 supports longer names, but struct dirent doesn't */
 			/* so we just truncate the names to fit */
 			if (len <= 0) {
 				error = EBADRPC;
-				mbuf_freem(mrep);
 				goto nfsmout;
 			}
 			if (len > MAXNAMLEN) {
@@ -3290,9 +4393,9 @@ nfs_readdirplusrpc(
 			}
 			tlen = nfsm_rndup(len);
 			if (tlen == len)
-				tlen += 4;	/* To ensure null termination*/
+				tlen += 4;	/* To ensure null termination */
 			left = DIRBLKSIZ - blksiz;
-			if ((tlen + (int)DIRHDSIZ) > left) {
+			if ((tlen + DIRHDSIZ) > left) {
 				dp->d_reclen += left;
 				uio_iov_base_add(uiop, left);
 				uio_iov_len_add(uiop, -left);
@@ -3300,7 +4403,7 @@ nfs_readdirplusrpc(
 				uio_uio_resid_add(uiop, -left);
 				blksiz = 0;
 			}
-			if ((tlen + (int)DIRHDSIZ) > uio_uio_resid(uiop))
+			if ((tlen + DIRHDSIZ) > uio_uio_resid(uiop))
 				bigenough = 0;
 			if (bigenough) {
 				// LP64todo - fix this!
@@ -3324,7 +4427,8 @@ nfs_readdirplusrpc(
 				// LP64todo - fix this!
 				cnp->cn_nameptr = CAST_DOWN(caddr_t, uio_iov_base(uiop));
 				cnp->cn_namelen = len;
-				nfsm_mtouio(uiop, len);
+				error = nfsm_chain_get_uio(&nmrep, len, uiop);
+				nfsmout_if(error);
 				cp = CAST_DOWN(caddr_t, uio_iov_base(uiop));
 				tlen -= len;
 				*cp = '\0';
@@ -3332,39 +4436,43 @@ nfs_readdirplusrpc(
 				uio_iov_len_add(uiop, -tlen);
 				uiop->uio_offset += tlen;
 				uio_uio_resid_add(uiop, -tlen);
+				if (skiplen)
+					nfsm_chain_adv(error, &nmrep,
+						nfsm_rndup(len + skiplen) - nfsm_rndup(len));
 			} else {
-				nfsm_adv(nfsm_rndup(len));
+				nfsm_chain_adv(error, &nmrep, nfsm_rndup(len + skiplen));
 			}
-			if (skiplen)
-				nfsm_adv(nfsm_rndup(skiplen));
-			nfsm_dissect(tl, u_long *, 3 * NFSX_UNSIGNED);
 			if (bigenough) {
-				cookie.nfsuquad[0] = *tl++;
-				cookie.nfsuquad[1] = *tl++;
+				nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[0]);
+				nfsm_chain_get_32(error, &nmrep, cookie.nfsuquad[1]);
 			} else
-				tl += 2;
+				nfsm_chain_adv(error, &nmrep, 2 * NFSX_UNSIGNED);
 
-			/*
-			 * Since the attributes are before the file handle
-			 * (sigh), we must skip over the attributes and then
-			 * come back and get them.
-			 */
-			attrflag = fxdr_unsigned(int, *tl);
+			nfsm_chain_get_32(error, &nmrep, attrflag);
+			nfsmout_if(error);
 			if (attrflag) {
 			    /* grab attributes */
-			    nfsm_attr_get(1, &nvattr);
+			    error = nfs_parsefattr(&nmrep, NFS_VER3, &nvattr);
+			    nfsmout_if(error);
 			    dp->d_type = IFTODT(VTTOIF(nvattr.nva_type));
 			    /* check for file handle */
-			    nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-			    doit = fxdr_unsigned(int, *tl);
+			    nfsm_chain_get_32(error, &nmrep, doit);
+			    nfsmout_if(error);
 			    if (doit) {
-				nfsm_getfh(fhp, fhsize, 1);
+				nfsm_chain_get_fh_ptr(error, &nmrep, NFS_VER3, fhp, fhsize);
+				nfsmout_if(error);
 				if (NFS_CMPFH(dnp, fhp, fhsize)) {
-				    error = vnode_ref(vp);
+				    error = vnode_ref(NFSTOV(dnp));
 				    if (error) {
 					doit = 0;
 				    } else {
-					newvp = vp;
+					if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+					    error = lockerror;
+					if (error) {
+					    vnode_rele(NFSTOV(dnp));
+					    goto nfsmout;
+					}
+					newvp = NFSTOV(dnp);
 					np = dnp;
 				    }
 				} else if (!bigenough ||
@@ -3385,9 +4493,8 @@ nfs_readdirplusrpc(
 				} else {
 				    cnp->cn_hash = 0;
 
-				    error = nfs_nget(vnode_mount(vp), vp, cnp,
-				    		fhp, fhsize, &nvattr, &xid,
-						NG_MAKEENTRY, &np);
+				    error = nfs_nget(NFSTOMP(dnp), dnp, cnp,
+				    		fhp, fhsize, &nvattr, &xid, NG_MAKEENTRY, &np);
 				    if (error)
 					doit = 0;
 				    else
@@ -3402,31 +4509,40 @@ nfs_readdirplusrpc(
 			    }
 			} else {
 			    /* Just skip over the file handle */
-			    nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-			    i = fxdr_unsigned(int, *tl);
-			    nfsm_adv(nfsm_rndup(i));
+			    nfsm_chain_get_32(error, &nmrep, fhsize);
+			    nfsm_chain_adv(error, &nmrep, nfsm_rndup(fhsize));
 			}
 			if (newvp != NULLVP) {
-			    if (newvp == vp)
+			    nfs_unlock(np);
+			    if (newvp == NFSTOV(dnp))
 				vnode_rele(newvp);
 			    else
 				vnode_put(newvp);
 			    newvp = NULLVP;
 			}
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-			more_dirs = fxdr_unsigned(int, *tl);
+			nfsm_chain_get_32(error, &nmrep, more_dirs);
+			nfsmout_if(error);
 		}
 		/*
 		 * If at end of rpc data, get the eof boolean
 		 */
 		if (!more_dirs) {
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED);
-			more_dirs = (fxdr_unsigned(int, *tl) == 0);
+			nfsm_chain_get_32(error, &nmrep, eof);
+			if (!error)
+				more_dirs = (eof == 0);
 		}
-		mbuf_freem(mrep);
+		if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_SHARED)))
+			error = lockerror;
+		nfsmout_if(error);
+		nfsm_chain_cleanup(&nmrep);
+		nfsm_chain_null(&nmreq);
+	}
+	if (!lockerror) {
+		nfs_unlock(dnp);
+		lockerror = ENOENT;
 	}
 	/*
-	 * Fill last record, iff any, out to a multiple of NFS_DIRBLKSIZ
+	 * Fill last record, iff any, out to a multiple of DIRBLKSIZ
 	 * by increasing d_reclen for the last record.
 	 */
 	if (blksiz > 0) {
@@ -3438,6 +4554,10 @@ nfs_readdirplusrpc(
 		uio_uio_resid_add(uiop, -left);
 	}
 
+	if ((lockerror = nfs_lock(dnp, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsmout_if(error);
+
 	/*
 	 * We are now either at the end of the directory or have filled the
 	 * block.
@@ -3446,110 +4566,184 @@ nfs_readdirplusrpc(
 		dnp->n_direofoffset = uiop->uio_offset;
 	else {
 		if (uio_uio_resid(uiop) > 0)
-			printf("EEK! readdirplusrpc resid > 0\n");
+			printf("EEK! readdirplus_rpc resid > 0\n");
 		cookiep = nfs_getcookie(dnp, uiop->uio_offset, 1);
 		if (cookiep)
 			*cookiep = cookie;
 	}
+
 nfsmout:
+	if (!lockerror)
+		nfs_unlock(dnp);
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
 /*
  * Silly rename. To make the NFS filesystem that is stateless look a little
  * more like the "ufs" a remove of an active vnode is translated to a rename
- * to a funny looking filename that is removed by nfs_inactive on the
+ * to a funny looking filename that is removed by nfs_vnop_inactive on the
  * nfsnode. There is the potential for another process on a different client
- * to create the same funny name between the nfs_lookitup() fails and the
- * nfs_rename() completes, but...
+ * to create the same funny name between when the lookitup() fails and the
+ * rename() completes, but...
  */
 
-/* format of "random" names and next name to try */
-/* (note: shouldn't exceed size of sillyrename.s_name) */
-static char sillyrename_name[] = ".nfsAAA%04x4.4";
+/* format of "random" silly names - includes a number and pid */
+/* (note: shouldn't exceed size of nfs_sillyrename.nsr_name) */
+#define NFS_SILLYNAME_FORMAT ".nfs.%08x.%04x"
+/* starting from zero isn't silly enough */
+static uint32_t nfs_sillyrename_number = 0x20051025;
 
 static int
 nfs_sillyrename(
-	vnode_t dvp,
-	vnode_t vp,
+	nfsnode_t dnp,
+	nfsnode_t np,
 	struct componentname *cnp,
-	kauth_cred_t cred,
-	proc_t p)
+	vfs_context_t ctx)
 {
-	register struct sillyrename *sp;
-	struct nfsnode *np;
+	struct nfs_sillyrename *nsp;
 	int error;
 	short pid;
-	kauth_cred_t tmpcred;
-	int i, j, k;
+	kauth_cred_t cred;
+	uint32_t num;
+	struct nfsmount *nmp;
 
-	cache_purge(vp);
-	np = VTONFS(vp);
-#if DIAGNOSTIC
-	if (vnode_vtype(vp) == VDIR)
-		panic("nfs_sillyrename: dir");
-#endif
-	MALLOC_ZONE(sp, struct sillyrename *,
-			sizeof (struct sillyrename), M_NFSREQ, M_WAITOK);
-	if (!sp)
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+
+	cache_purge(NFSTOV(np));
+
+	MALLOC_ZONE(nsp, struct nfs_sillyrename *,
+			sizeof (struct nfs_sillyrename), M_NFSREQ, M_WAITOK);
+	if (!nsp)
 		return (ENOMEM);
+	cred = vfs_context_ucred(ctx);
 	kauth_cred_ref(cred);
-	sp->s_cred = cred;
-	sp->s_dvp = dvp;
-	error = vnode_ref(dvp);
+	nsp->nsr_cred = cred;
+	nsp->nsr_dnp = dnp;
+	error = vnode_ref(NFSTOV(dnp));
 	if (error)
 		goto bad_norele;
 
 	/* Fudge together a funny name */
-	pid = proc_pid(p);
-	sp->s_namlen = sprintf(sp->s_name, sillyrename_name, pid);
+	pid = vfs_context_pid(ctx);
+	num = OSAddAtomic(1, (SInt32*)&nfs_sillyrename_number);
+	nsp->nsr_namlen = snprintf(nsp->nsr_name, sizeof(nsp->nsr_name),
+				NFS_SILLYNAME_FORMAT, num, (pid & 0xffff));
+	if (nsp->nsr_namlen >= (int)sizeof(nsp->nsr_name))
+		nsp->nsr_namlen = sizeof(nsp->nsr_name) - 1;
 
 	/* Try lookitups until we get one that isn't there */
-	i = j = k = 0;
-	while (nfs_lookitup(dvp, sp->s_name, sp->s_namlen, sp->s_cred, p, NULL) == 0) {
-		if (sp->s_name[4]++ >= 'z')
-			sp->s_name[4] = 'A';
-		if (++i > ('z' - 'A' + 1)) {
-			i = 0;
-			if (sp->s_name[5]++ >= 'z')
-				sp->s_name[5] = 'A';
-			if (++j > ('z' - 'A' + 1)) {
-				j = 0;
-				if (sp->s_name[6]++ >= 'z')
-					sp->s_name[6] = 'A';
-				if (++k > ('z' - 'A' + 1)) {
-					error = EINVAL;
-					goto bad;
-				}
-			}
-		}
-	}
-	/* make note of next "random" name to try */
-	if ((sillyrename_name[4] = (sp->s_name[4] + 1)) > 'z') {
-		sillyrename_name[4] = 'A';
-		if ((sillyrename_name[5] = (sp->s_name[5] + 1)) > 'z') {
-			sillyrename_name[5] = 'A';
-			if ((sillyrename_name[6] = (sp->s_name[6] + 1)) > 'z')
-				sillyrename_name[6] = 'A';
-		}
+	while (nfs_lookitup(dnp, nsp->nsr_name, nsp->nsr_namlen, ctx, NULL) == 0) {
+		num = OSAddAtomic(1, (SInt32*)&nfs_sillyrename_number);
+		nsp->nsr_namlen = snprintf(nsp->nsr_name, sizeof(nsp->nsr_name),
+					NFS_SILLYNAME_FORMAT, num, (pid & 0xffff));
+		if (nsp->nsr_namlen >= (int)sizeof(nsp->nsr_name))
+			nsp->nsr_namlen = sizeof(nsp->nsr_name) - 1;
 	}
+
 	/* now, do the rename */
-	error = nfs_renamerpc(dvp, cnp->cn_nameptr, cnp->cn_namelen,
-				dvp, sp->s_name, sp->s_namlen, sp->s_cred, p);
+	error = nmp->nm_funcs->nf_rename_rpc(dnp, cnp->cn_nameptr, cnp->cn_namelen,
+					dnp, nsp->nsr_name, nsp->nsr_namlen, ctx);
+	if (!error && (dnp->n_flag & NNEGNCENTRIES)) {
+		dnp->n_flag &= ~NNEGNCENTRIES;
+		cache_purge_negatives(NFSTOV(dnp));
+	}
+	FSDBG(267, dnp, np, num, error);
 	if (error)
 		goto bad;
-	error = nfs_lookitup(dvp, sp->s_name, sp->s_namlen, sp->s_cred, p, &np);
-#if DIAGNOSTIC
-	kprintf("sillyrename: %s, vp=%x, np=%x, dvp=%x\n",
-		&sp->s_name[0], (unsigned)vp, (unsigned)np, (unsigned)dvp);
-#endif
-	np->n_sillyrename = sp;
+	error = nfs_lookitup(dnp, nsp->nsr_name, nsp->nsr_namlen, ctx, &np);
+	np->n_sillyrename = nsp;
 	return (0);
 bad:
-	vnode_rele(sp->s_dvp);
+	vnode_rele(NFSTOV(dnp));
 bad_norele:
-	kauth_cred_unref(&sp->s_cred);
-	FREE_ZONE((caddr_t)sp, sizeof (struct sillyrename), M_NFSREQ);
+	nsp->nsr_cred = NOCRED;
+	kauth_cred_unref(&cred);
+	FREE_ZONE(nsp, sizeof(*nsp), M_NFSREQ);
+	return (error);
+}
+
+int
+nfs3_lookup_rpc_async(
+	nfsnode_t dnp,
+	char *name,
+	int namelen,
+	vfs_context_t ctx,
+	struct nfsreq **reqp)
+{
+	struct nfsmount *nmp;
+	struct nfsm_chain nmreq;
+	int error = 0, nfsvers;
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmreq);
+
+	nfsm_chain_build_alloc_init(error, &nmreq,
+		NFSX_FH(nfsvers) + NFSX_UNSIGNED + nfsm_rndup(namelen));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, dnp->n_fhp, dnp->n_fhsize);
+	nfsm_chain_add_string(error, &nmreq, name, namelen);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request_async(dnp, NULL, &nmreq, NFSPROC_LOOKUP,
+			vfs_context_thread(ctx), vfs_context_ucred(ctx), NULL, reqp);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	return (error);
+}
+
+int
+nfs3_lookup_rpc_async_finish(
+	nfsnode_t dnp,
+	vfs_context_t ctx,
+	struct nfsreq *req,
+	u_int64_t *xidp,
+	fhandle_t *fhp,
+	struct nfs_vattr *nvap)
+{
+	int error = 0, status, nfsvers, attrflag;
+	u_int64_t xid;
+	struct nfsmount *nmp;
+	struct nfsm_chain nmrep;
+
+	nmp = NFSTONMP(dnp);
+	nfsvers = nmp->nm_vers;
+
+	nfsm_chain_null(&nmrep);
+
+	error = nfs_request_async_finish(req, &nmrep, xidp, &status);
+
+	xid = *xidp;
+	if (error || status) {
+		if (nfsvers == NFS_VER3)
+			nfsm_chain_postop_attr_update(error, &nmrep, dnp, &xid);
+		if (!error)
+			error = status;
+		goto nfsmout;
+	}
+
+	nfsmout_if(error || !fhp || !nvap);
+
+	/* get the file handle */
+	nfsm_chain_get_fh(error, &nmrep, nfsvers, fhp);
+
+	/* get the attributes */
+	if (nfsvers == NFS_VER3) {
+		nfsm_chain_postop_attr_get(error, &nmrep, attrflag, nvap);
+		nfsm_chain_postop_attr_update(error, &nmrep, dnp, &xid);
+		if (!error && !attrflag)
+			error = nfs3_getattr_rpc(NULL, NFSTOMP(dnp), fhp->fh_data, fhp->fh_len, ctx, nvap, xidp);
+	} else {
+		error = nfs_parsefattr(&nmrep, nfsvers, nvap);
+	}
+nfsmout:
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
@@ -3561,181 +4755,242 @@ bad_norele:
  *			handled too
  * *npp != NULL --> update the file handle in the vnode
  */
-static int
-nfs_lookitup(dvp, name, len, cred, procp, npp)
-	vnode_t dvp;
-	char *name;
-	int len;
-	kauth_cred_t cred;
-	proc_t procp;
-	struct nfsnode **npp;
+int
+nfs_lookitup(
+	nfsnode_t dnp,
+	char *name,
+	int namelen,
+	vfs_context_t ctx,
+	nfsnode_t *npp)
+{
+	int error = 0;
+	nfsnode_t np, newnp = NULL;
+	u_int64_t xid;
+	fhandle_t fh;
+	struct nfsmount *nmp;
+	struct nfs_vattr nvattr;
+	struct nfsreq rq, *req = &rq;
+
+	nmp = NFSTONMP(dnp);
+	if (!nmp)
+		return (ENXIO);
+
+	if (NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXNAME) &&
+	    (namelen > (long)nmp->nm_fsattr.nfsa_maxname))
+		return (ENAMETOOLONG);
+
+	/* check for lookup of "." */
+	if ((name[0] == '.') && (namelen == 1)) {
+		/* skip lookup, we know who we are */
+		fh.fh_len = 0;
+		newnp = dnp;
+		goto nfsmout;
+	}
+
+	error = nmp->nm_funcs->nf_lookup_rpc_async(dnp, name, namelen, ctx, &req);
+	nfsmout_if(error);
+	error = nmp->nm_funcs->nf_lookup_rpc_async_finish(dnp, ctx, req, &xid, &fh, &nvattr);
+	nfsmout_if(!npp || error);
+
+	if (*npp) {
+		np = *npp;
+		if (fh.fh_len != np->n_fhsize) {
+			u_char *oldbuf = (np->n_fhsize > NFS_SMALLFH) ? np->n_fhp : NULL;
+			if (fh.fh_len > NFS_SMALLFH) {
+				MALLOC_ZONE(np->n_fhp, u_char *, fh.fh_len, M_NFSBIGFH, M_WAITOK);
+				if (!np->n_fhp) {
+				    np->n_fhp = oldbuf;
+				    error = ENOMEM;
+				    goto nfsmout;
+				}
+			} else {
+				np->n_fhp = &np->n_fh[0];
+			}
+			if (oldbuf)
+				FREE_ZONE(oldbuf, np->n_fhsize, M_NFSBIGFH);
+		}
+		bcopy(fh.fh_data, np->n_fhp, fh.fh_len);
+		np->n_fhsize = fh.fh_len;
+		error = nfs_loadattrcache(np, &nvattr, &xid, 0);
+		nfsmout_if(error);
+		newnp = np;
+	} else if (NFS_CMPFH(dnp, fh.fh_data, fh.fh_len)) {
+		if (dnp->n_xid <= xid)
+			error = nfs_loadattrcache(dnp, &nvattr, &xid, 0);
+		nfsmout_if(error);
+		newnp = dnp;
+	} else {
+		struct componentname cn, *cnp = &cn;
+		bzero(cnp, sizeof(*cnp));
+		cnp->cn_nameptr = name;
+		cnp->cn_namelen = namelen;
+		error = nfs_nget(NFSTOMP(dnp), dnp, cnp, fh.fh_data, fh.fh_len,
+			    &nvattr, &xid, NG_MAKEENTRY, &np);
+		nfsmout_if(error);
+		newnp = np;
+	}
+
+nfsmout:
+	if (npp && !*npp && !error)
+		*npp = newnp;
+	return (error);
+}
+
+/*
+ * set up and initialize a "._" file lookup structure used for
+ * performing async lookups.
+ */
+void
+nfs_dulookup_init(struct nfs_dulookup *dulp, nfsnode_t dnp, const char *name, int namelen)
+{
+	int error, du_namelen;
+	vnode_t du_vp;
+
+	/* check for ._ file in name cache */
+	dulp->du_flags = 0;
+	bzero(&dulp->du_cn, sizeof(dulp->du_cn));
+	du_namelen = namelen + 2;
+	if ((namelen >= 2) && (name[0] == '.') && (name[1] == '_'))
+		return;
+	if (du_namelen >= (int)sizeof(dulp->du_smallname))
+		MALLOC(dulp->du_cn.cn_nameptr, char *, du_namelen + 1, M_TEMP, M_WAITOK);
+	else
+		dulp->du_cn.cn_nameptr = dulp->du_smallname;
+	if (!dulp->du_cn.cn_nameptr)
+		return;
+	dulp->du_cn.cn_namelen = du_namelen;
+	snprintf(dulp->du_cn.cn_nameptr, du_namelen + 1, "._%s", name);
+	dulp->du_cn.cn_nameptr[du_namelen] = '\0';
+
+	error = cache_lookup(NFSTOV(dnp), &du_vp, &dulp->du_cn);
+	if (error == -1)
+		vnode_put(du_vp);
+	else if (!error)
+		dulp->du_flags |= NFS_DULOOKUP_DOIT;
+	else if (dulp->du_cn.cn_nameptr != dulp->du_smallname)
+		FREE(dulp->du_cn.cn_nameptr, M_TEMP);
+}
+
+/*
+ * start an async "._" file lookup request
+ */
+void
+nfs_dulookup_start(struct nfs_dulookup *dulp, nfsnode_t dnp, vfs_context_t ctx)
+{
+	struct nfsmount *nmp = NFSTONMP(dnp);
+	struct nfsreq *req = &dulp->du_req;
+
+	if (!nmp || !(dulp->du_flags & NFS_DULOOKUP_DOIT))
+		return;
+	if (!nmp->nm_funcs->nf_lookup_rpc_async(dnp, dulp->du_cn.cn_nameptr,
+			dulp->du_cn.cn_namelen, ctx, &req))
+		dulp->du_flags |= NFS_DULOOKUP_INPROG;
+}
+
+/*
+ * finish an async "._" file lookup request and clean up the structure
+ */
+void
+nfs_dulookup_finish(struct nfs_dulookup *dulp, nfsnode_t dnp, vfs_context_t ctx)
 {
-	u_long *tl;
-	caddr_t cp;
-	long t1, t2;
-	vnode_t newvp = (vnode_t)0;
-	struct nfsnode *np, *dnp = VTONFS(dvp);
-	caddr_t bpos, dpos, cp2;
-	int error = 0, fhlen, attrflag;
-	mbuf_t mreq, mrep, md, mb, mb2;
-	u_char *nfhp;
-	int v3;
-	u_int64_t xid, dxid, savedxid;
+	struct nfsmount *nmp = NFSTONMP(dnp);
+	int error;
+	nfsnode_t du_np;
+	u_int64_t xid;
+	fhandle_t fh;
 	struct nfs_vattr nvattr;
 
-	if (!VFSTONFS(vnode_mount(dvp)))
-		return (ENXIO);
-	v3 = NFS_ISV3(dvp);
+	if (!nmp || !(dulp->du_flags & NFS_DULOOKUP_INPROG))
+		goto out;
 
-	nfsm_reqhead(NFSX_FH(v3) + NFSX_UNSIGNED + nfsm_rndup(len));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_LOOKUP]);
-	nfsm_fhtom(dvp, v3);
-	nfsm_strtom(name, len, NFS_MAXNAMLEN, v3);
-	nfsm_request(dvp, NFSPROC_LOOKUP, procp, cred, &xid);
-	if (npp && !error) {
-		savedxid = xid;
-		nfsm_getfh(nfhp, fhlen, v3);
-		/* get attributes */
-		if (v3) {
-			nfsm_postop_attr_get(v3, attrflag, &nvattr);
-			if (!attrflag) {
-				/* We need valid attributes in order */
-				/* to call nfs_nget/vnode_create().  */
-				error = nfs_getattr_no_vnode(vnode_mount(dvp),
-						nfhp, fhlen, cred, procp, &nvattr, &xid);
-				if (error) {
-					mbuf_freem(mrep);
-					goto nfsmout;
-				}
-			}
-			dxid = savedxid;
-			nfsm_postop_attr_update(dvp, v3, attrflag, &dxid);
-		} else {
-			nfsm_attr_get(v3, &nvattr);
-		}
-		if (*npp) {
-		    np = *npp;
-		    if (fhlen != np->n_fhsize) {
-			u_char *oldbuf = (np->n_fhsize > NFS_SMALLFH) ? np->n_fhp : NULL;
-			if (fhlen > NFS_SMALLFH) {
-			    MALLOC_ZONE(np->n_fhp, u_char *, fhlen, M_NFSBIGFH, M_WAITOK);
-			    if (!np->n_fhp) {
-				np->n_fhp = oldbuf;
-				error = ENOMEM;
-				mbuf_freem(mrep);
-				goto nfsmout;
-			    }
-			} else {
-			    np->n_fhp = &np->n_fh[0];
-			}
-			if (oldbuf) {
-			    FREE_ZONE(oldbuf, np->n_fhsize, M_NFSBIGFH);
-			}
-		    }
-		    bcopy(nfhp, np->n_fhp, fhlen);
-		    np->n_fhsize = fhlen;
-		    newvp = NFSTOV(np);
-		    error = nfs_loadattrcache(np, &nvattr, &xid, 0);
-		    if (error) {
-			mbuf_freem(mrep);
-			goto nfsmout;
-		    }
-		} else if (NFS_CMPFH(dnp, nfhp, fhlen)) {
-		    newvp = dvp;
-		    if (dnp->n_xid <= savedxid) {
-			dxid = savedxid;
-			error = nfs_loadattrcache(dnp, &nvattr, &dxid, 0);
-			if (error) {
-			    mbuf_freem(mrep);
-			    goto nfsmout;
-			}
-		    }
-		} else {
-		    struct componentname cn, *cnp = &cn;
-		    bzero(cnp, sizeof(*cnp));
-		    cnp->cn_nameptr = name;
-		    cnp->cn_namelen = len;
-
-		    error = nfs_nget(vnode_mount(dvp), dvp, cnp, nfhp, fhlen,
-				&nvattr, &xid, NG_MAKEENTRY, &np);
-		    if (error) {
-			mbuf_freem(mrep);
-			return (error);
-		    }
-		    newvp = NFSTOV(np);
+	error = nmp->nm_funcs->nf_lookup_rpc_async_finish(dnp, ctx, &dulp->du_req, &xid, &fh, &nvattr);
+	dulp->du_flags &= ~NFS_DULOOKUP_INPROG;
+	if (error == ENOENT) {
+		/* add a negative entry in the name cache */
+		cache_enter(NFSTOV(dnp), NULL, &dulp->du_cn);
+		dnp->n_flag |= NNEGNCENTRIES;
+	} else if (!error) {
+		error = nfs_nget(NFSTOMP(dnp), dnp, &dulp->du_cn, fh.fh_data, fh.fh_len,
+			    &nvattr, &xid, NG_MAKEENTRY, &du_np);
+		if (!error) {
+			nfs_unlock(du_np);
+			vnode_put(NFSTOV(du_np));
 		}
 	}
-	nfsm_reqdone;
-	if (npp && *npp == NULL) {
-		if (error) {
-			if (newvp) {
-				if (newvp == dvp)
-					vnode_rele(newvp);
-				else
-					vnode_put(newvp);
-			}
-		} else
-			*npp = np;
-	}
-	return (error);
+out:
+	if (dulp->du_flags & NFS_DULOOKUP_INPROG)
+		nfs_request_async_cancel(&dulp->du_req);
+	if (dulp->du_cn.cn_nameptr && (dulp->du_cn.cn_nameptr != dulp->du_smallname))
+		FREE(dulp->du_cn.cn_nameptr, M_TEMP);
 }
 
+
 /*
- * Nfs Version 3 commit rpc
+ * NFS Version 3 commit RPC
  */
 int
-nfs_commit(vp, offset, count, cred, procp)
-	vnode_t vp;
-	u_quad_t offset;
-	u_int32_t count;
-	kauth_cred_t cred;
-	proc_t procp;
+nfs3_commit_rpc(
+	nfsnode_t np,
+	u_int64_t offset,
+	u_int64_t count,
+	kauth_cred_t cred)
 {
-	caddr_t cp;
-	u_long *tl;
-	int t1, t2;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-	caddr_t bpos, dpos, cp2;
-	int error = 0, wccpostattr = 0;
+	struct nfsmount *nmp;
+	int error = 0, lockerror, status, wccpostattr = 0, nfsvers;
 	struct timespec premtime = { 0, 0 };
-	mbuf_t mreq, mrep, md, mb, mb2;
 	u_int64_t xid, wverf;
+	uint32_t count32;
+	struct nfsm_chain nmreq, nmrep;
 
-	FSDBG(521, vp, offset, count, nmp->nm_state);
+	nmp = NFSTONMP(np);
+	FSDBG(521, np, offset, count, nmp ? nmp->nm_state : 0);
 	if (!nmp)
 		return (ENXIO);
-	if ((nmp->nm_state & NFSSTA_HASWRITEVERF) == 0)
+	if (!(nmp->nm_state & NFSSTA_HASWRITEVERF))
 		return (0);
-	nfsm_reqhead(NFSX_FH(1));
-	if (error)
-		return (error);
-	OSAddAtomic(1, (SInt32*)&nfsstats.rpccnt[NFSPROC_COMMIT]);
-	nfsm_fhtom(vp, 1);
-	nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED);
-	txdr_hyper(&offset, tl);
-	tl += 2;
-	*tl = txdr_unsigned(count);
-	nfsm_request(vp, NFSPROC_COMMIT, procp, cred, &xid);
-	if (mrep) {
-		nfsm_wcc_data(vp, &premtime, wccpostattr, &xid);
-		/* XXX can we do anything useful with the wcc info? */
-	}
-	if (!error) {
-		nfsm_dissect(tl, u_long *, NFSX_V3WRITEVERF);
-		fxdr_hyper(tl, &wverf);
-		if (wverf != nmp->nm_verf) {
-			nmp->nm_verf = wverf;
-			error = NFSERR_STALEWRITEVERF;
-		}
+	nfsvers = nmp->nm_vers;
+
+	if (count > UINT32_MAX)
+		count32 = 0;
+	else
+		count32 = count;
+
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
+
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(NFS_VER3));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_add_64(error, &nmreq, offset);
+	nfsm_chain_add_32(error, &nmreq, count32);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request2(np, NULL, &nmreq, NFSPROC_COMMIT,
+			current_thread(), cred, 0, &nmrep, &xid, &status);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	/* can we do anything useful with the wcc info? */
+	nfsm_chain_get_wcc_data(error, &nmrep, np, &premtime, &wccpostattr, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	if (!error)
+		error = status;
+	nfsm_chain_get_64(error, &nmrep, wverf);
+	nfsmout_if(error);
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_verf != wverf) {
+		nmp->nm_verf = wverf;
+		error = NFSERR_STALEWRITEVERF;
 	}
-	nfsm_reqdone;
+	lck_mtx_unlock(&nmp->nm_lock);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
+
 static int
-nfs_blockmap(
+nfs_vnop_blockmap(
 	__unused struct vnop_blockmap_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -3757,467 +5012,113 @@ nfs_blockmap(
  */
 /*ARGSUSED*/
 static int
-nfs_mmap(
+nfs_vnop_mmap(
 	__unused struct vnop_mmap_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_fflags;
-		kauth_cred_t a_cred;
-		proc_t a_p;
+		vfs_context_t a_context;
 	} */ *ap)
 {
-
 	return (EINVAL);
 }
 
 /*
- * fsync vnode op. Just call nfs_flush() with commit == 1.
+ * fsync vnode op. Just call nfs_flush().
  */
 /* ARGSUSED */
 static int
-nfs_fsync(ap)
+nfs_vnop_fsync(
 	struct vnop_fsync_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_waitfor;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	kauth_cred_t cred = vfs_context_ucred(ap->a_context);
-	proc_t p = vfs_context_proc(ap->a_context);
-	struct nfsnode *np = VTONFS(ap->a_vp);
-	int error;
-
-	np->n_flag |= NWRBUSY;
-	error = nfs_flush(ap->a_vp, ap->a_waitfor, cred, p, 0);
-	np->n_flag &= ~NWRBUSY;
-	return (error);
+	return (nfs_flush(VTONFS(ap->a_vp), ap->a_waitfor, vfs_context_thread(ap->a_context), 0));
 }
- 
-int
-nfs_flushcommits(vnode_t vp, proc_t p, int nowait)
-{
-	struct nfsnode *np = VTONFS(vp);
-	struct nfsbuf *bp;
-	struct nfsbuflists blist, commitlist;
-	int error = 0, retv, wcred_set, flags;
-	u_quad_t off, endoff, toff;
-	u_int32_t count;
-	kauth_cred_t wcred = NULL;
-
-	FSDBG_TOP(557, vp, np, 0, 0);
 
-	/*
-	 * A nb_flags == (NB_DELWRI | NB_NEEDCOMMIT) block has been written to the
-	 * server, but nas not been committed to stable storage on the server
-	 * yet. The byte range is worked out for as many nfsbufs as we can handle
-	 * and the commit rpc is done.
-	 */
-	if (!LIST_EMPTY(&np->n_dirtyblkhd))
-		np->n_flag |= NMODIFIED;
-
-	off = (u_quad_t)-1;
-	endoff = 0;
-	wcred_set = 0;
-	LIST_INIT(&commitlist);
-
-	if (!VFSTONFS(vnode_mount(vp))) {
-		error = ENXIO;
-		goto done;
-	}
-	if (!NFS_ISV3(vp)) {
-		error = EINVAL;
-		goto done;
-	}
-
-	flags = NBI_DIRTY;
-	if (nowait)
-		flags |= NBI_NOWAIT;
-	lck_mtx_lock(nfs_buf_mutex);
-	if (!nfs_buf_iterprepare(np, &blist, flags)) {
-		while ((bp = LIST_FIRST(&blist))) {
-			LIST_REMOVE(bp, nb_vnbufs);
-			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
-			error = nfs_buf_acquire(bp, NBAC_NOWAIT, 0, 0);
-			if (error)
-				continue;
-			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
-				nfs_buf_check_write_verifier(np, bp);
-			if (((bp->nb_flags & (NB_DELWRI | NB_NEEDCOMMIT))
-				!= (NB_DELWRI | NB_NEEDCOMMIT))) {
-				nfs_buf_drop(bp);
-				continue;
-			}
-			nfs_buf_remfree(bp);
-			lck_mtx_unlock(nfs_buf_mutex);
-			/*
-			 * we need a upl to see if the page has been
-			 * dirtied (think mmap) since the unstable write, and
-			 * also to prevent vm from paging it during our commit rpc
-			 */
-			if (!ISSET(bp->nb_flags, NB_PAGELIST)) {
-				retv = nfs_buf_upl_setup(bp);
-				if (retv) {
-					/* unable to create upl */
-					/* vm object must no longer exist */
-					/* this could be fatal if we need */
-					/* to write the data again, we'll see...  */
-					printf("nfs_flushcommits: upl create failed %d\n", retv);
-					bp->nb_valid = bp->nb_dirty = 0;
-				}
-			}
-			nfs_buf_upl_check(bp);
-			lck_mtx_lock(nfs_buf_mutex);
-
-			FSDBG(557, bp, bp->nb_flags, bp->nb_valid, bp->nb_dirty);
-			FSDBG(557, bp->nb_validoff, bp->nb_validend,
-			      bp->nb_dirtyoff, bp->nb_dirtyend);
-
-			/*
-			 * We used to check for dirty pages here; if there were any
-			 * we'd abort the commit and force the entire buffer to be
-			 * written again.
-			 *
-			 * Instead of doing that, we now go ahead and commit the dirty
-			 * range, and then leave the buffer around with dirty pages
-			 * that will be written out later.
-			 */
-
-			/*
-			 * Work out if all buffers are using the same cred
-			 * so we can deal with them all with one commit.
-			 *
-			 * XXX creds in bp's must be obtained by kauth_cred_ref on
-			 *     the same original cred in order for them to be equal.
-			 */
-			if (wcred_set == 0) {
-				wcred = bp->nb_wcred;
-				if (!IS_VALID_CRED(wcred))
-					panic("nfs: needcommit w/out wcred");
-				wcred_set = 1;
-			} else if ((wcred_set == 1) && wcred != bp->nb_wcred) {
-				wcred_set = -1;
-			}
-			SET(bp->nb_flags, NB_WRITEINPROG);
-
-			/*
-			 * A list of these buffers is kept so that the
-			 * second loop knows which buffers have actually
-			 * been committed. This is necessary, since there
-			 * may be a race between the commit rpc and new
-			 * uncommitted writes on the file.
-			 */
-			LIST_REMOVE(bp, nb_vnbufs);
-			LIST_INSERT_HEAD(&commitlist, bp, nb_vnbufs);
-			toff = NBOFF(bp) + bp->nb_dirtyoff;
-			if (toff < off)
-				off = toff;
-			toff += (u_quad_t)(bp->nb_dirtyend - bp->nb_dirtyoff);
-			if (toff > endoff)
-				endoff = toff;
-		}
-		nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
-	}
-	lck_mtx_unlock(nfs_buf_mutex);
-
-	if (LIST_EMPTY(&commitlist)) {
-		error = ENOBUFS;
-		goto done;
-	}
-
-	/*
-	 * Commit data on the server, as required.
-	 * If all bufs are using the same wcred, then use that with
-	 * one call for all of them, otherwise commit each one
-	 * separately.
-	 */
-	if (wcred_set == 1) {
-		/*
-		 * Note, it's possible the commit range could be >2^32-1.
-		 * If it is, we'll send one commit that covers the whole file.
-		 */
-		if ((endoff - off) > 0xffffffff)
-			count = 0; 
-		else
-			count = (endoff - off); 
-		retv = nfs_commit(vp, off, count, wcred, p);
-	} else {
-		retv = 0;
-		LIST_FOREACH(bp, &commitlist, nb_vnbufs) {
-			toff = NBOFF(bp) + bp->nb_dirtyoff;
-			count = bp->nb_dirtyend - bp->nb_dirtyoff;
-			retv = nfs_commit(vp, toff, count, bp->nb_wcred, p);
-			if (retv)
-				break;
-		}
-	}
-
-	/*
-	 * Now, either mark the blocks I/O done or mark the
-	 * blocks dirty, depending on whether the commit
-	 * succeeded.
-	 */
-	while ((bp = LIST_FIRST(&commitlist))) {
-		LIST_REMOVE(bp, nb_vnbufs);
-		FSDBG(557, bp, retv, bp->nb_flags, bp->nb_dirty);
-		CLR(bp->nb_flags, (NB_NEEDCOMMIT | NB_WRITEINPROG));
-		np->n_needcommitcnt--;
-		CHECK_NEEDCOMMITCNT(np);
-
-		if (retv) {
-			/* move back to dirty list */
-			lck_mtx_lock(nfs_buf_mutex);
-			LIST_INSERT_HEAD(&VTONFS(vp)->n_dirtyblkhd, bp, nb_vnbufs);
-			lck_mtx_unlock(nfs_buf_mutex);
-			nfs_buf_release(bp, 1);
-			continue;
-		}
-
-		vnode_startwrite(vp);
-		if (ISSET(bp->nb_flags, NB_DELWRI)) {
-			OSAddAtomic(-1, (SInt32*)&nfs_nbdwrite);
-			NFSBUFCNTCHK(0);
-			wakeup(&nfs_nbdwrite);
-		}
-		CLR(bp->nb_flags, (NB_READ|NB_DONE|NB_ERROR|NB_DELWRI));
-		/* if block still has dirty pages, we don't want it to */
-		/* be released in nfs_buf_iodone().  So, don't set NB_ASYNC. */
-		if (!bp->nb_dirty)
-			SET(bp->nb_flags, NB_ASYNC);
-
-		/* move to clean list */
-		lck_mtx_lock(nfs_buf_mutex);
-		LIST_INSERT_HEAD(&VTONFS(vp)->n_cleanblkhd, bp, nb_vnbufs);
-		lck_mtx_unlock(nfs_buf_mutex);
-
-		bp->nb_dirtyoff = bp->nb_dirtyend = 0;
-
-		nfs_buf_iodone(bp);
-		if (bp->nb_dirty) {
-			/* throw it back in as a delayed write buffer */
-			CLR(bp->nb_flags, NB_DONE);
-			nfs_buf_write_delayed(bp, p);
-		}
-	}
-
-done:
-	FSDBG_BOT(557, vp, np, 0, error);
-	return (error);
-}
 
 /*
- * Flush all the blocks associated with a vnode.
- * 	Walk through the buffer pool and push any dirty pages
- *	associated with the vnode.
+ * Do an NFS pathconf RPC.
  */
 int
-nfs_flush(
-	vnode_t vp,
-	int waitfor,
-	__unused kauth_cred_t cred,
-	proc_t p,
-	int ignore_writeerr)
+nfs3_pathconf_rpc(
+	nfsnode_t np,
+	struct nfs_fsattr *nfsap,
+	vfs_context_t ctx)
 {
-	struct nfsnode *np = VTONFS(vp);
-	struct nfsbuf *bp;
-	struct nfsbuflists blist;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
-	int error = 0, error2, slptimeo = 0, slpflag = 0;
-	int flags, passone = 1;
-
-	FSDBG_TOP(517, vp, np, waitfor, 0);
-
-	if (!nmp) {
-		error = ENXIO;
-		goto done;
-	}
-	if (nmp->nm_flag & NFSMNT_INT)
-		slpflag = PCATCH;
-
-	/*
-	 * On the first pass, start async/unstable writes on all
-	 * delayed write buffers.  Then wait for all writes to complete
-	 * and call nfs_flushcommits() to commit any uncommitted buffers.
-	 * On all subsequent passes, start STABLE writes on any remaining
-	 * dirty buffers.  Then wait for all writes to complete.
-	 */
-again:
-	lck_mtx_lock(nfs_buf_mutex);
-	FSDBG(518, LIST_FIRST(&np->n_dirtyblkhd), np->n_flag, 0, 0);
-	if (!LIST_EMPTY(&np->n_dirtyblkhd))
-		np->n_flag |= NMODIFIED;
-	if (!VFSTONFS(vnode_mount(vp))) {
-		lck_mtx_unlock(nfs_buf_mutex);
-		error = ENXIO;
-		goto done;
-	}
-
-	/* Start/do any write(s) that are required. */
-	if (!nfs_buf_iterprepare(np, &blist, NBI_DIRTY)) {
-		while ((bp = LIST_FIRST(&blist))) {
-			LIST_REMOVE(bp, nb_vnbufs);
-			LIST_INSERT_HEAD(&np->n_dirtyblkhd, bp, nb_vnbufs);
-			flags = (passone || (waitfor != MNT_WAIT)) ? NBAC_NOWAIT : 0;
-			if (flags != NBAC_NOWAIT)
-				nfs_buf_refget(bp);
-			while ((error = nfs_buf_acquire(bp, flags, slpflag, slptimeo))) {
-				FSDBG(524, bp, flags, bp->nb_lflags, bp->nb_flags);
-				if (error == EBUSY)
-					break;
-				if (error) {
-					error2 = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p);
-					if (error2) {
-						if (flags != NBAC_NOWAIT)
-							nfs_buf_refrele(bp);
-						nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
-						lck_mtx_unlock(nfs_buf_mutex);
-						error = error2;
-						goto done;
-					}
-					if (slpflag == PCATCH) {
-						slpflag = 0;
-						slptimeo = 2 * hz;
-					}
-				}
-			}
-			if (flags != NBAC_NOWAIT)
-				nfs_buf_refrele(bp);
-			if (error == EBUSY)
-				continue;
-			if (!bp->nb_vp) {
-				/* buffer is no longer valid */
-				nfs_buf_drop(bp);
-				continue;
-			}
-			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT))
-				nfs_buf_check_write_verifier(np, bp);
-			if (!ISSET(bp->nb_flags, NB_DELWRI))
-				panic("nfs_flush: not dirty");
-			FSDBG(525, bp, passone, bp->nb_lflags, bp->nb_flags);
-			if ((passone || (waitfor != MNT_WAIT)) &&
-			    ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
-				nfs_buf_drop(bp);
-				continue;
-			}
-			nfs_buf_remfree(bp);
-			lck_mtx_unlock(nfs_buf_mutex);
-			if (ISSET(bp->nb_flags, NB_ERROR)) {
-				np->n_error = bp->nb_error ? bp->nb_error : EIO;
-				np->n_flag |= NWRITEERR;
-				nfs_buf_release(bp, 1);
-				lck_mtx_lock(nfs_buf_mutex);
-				continue;
-			}
-			SET(bp->nb_flags, NB_ASYNC);
-			if (!passone) {
-				/* NB_STABLE forces this to be written FILESYNC */
-				SET(bp->nb_flags, NB_STABLE);
-			}
-			nfs_buf_write(bp);
-			lck_mtx_lock(nfs_buf_mutex);
-		}
-		nfs_buf_itercomplete(np, &blist, NBI_DIRTY);
-	}
-	lck_mtx_unlock(nfs_buf_mutex);
-
-	if (waitfor == MNT_WAIT) {
-	        while ((error = vnode_waitforwrites(vp, 0, slpflag, slptimeo, "nfsflush"))) {
-		        error2 = nfs_sigintr(VFSTONFS(vnode_mount(vp)), NULL, p);
-			if (error2) {
-			        error = error2;
-				goto done;
-			}
-			if (slpflag == PCATCH) {
-				slpflag = 0;
-				slptimeo = 2 * hz;
-			}
-		}
-	}
-
-	if (NFS_ISV3(vp)) {
-		/* loop while it looks like there are still buffers to be */
-		/* commited and nfs_flushcommits() seems to be handling them. */
-		while (np->n_needcommitcnt)
-			if (nfs_flushcommits(vp, p, 0))
-				break;
-	}
-
-	if (passone) {
-		passone = 0;
-		goto again;
-	}
-
-	if (waitfor == MNT_WAIT) {
-		if (!LIST_EMPTY(&np->n_dirtyblkhd))
-			goto again;
-		/* if we have no dirty blocks, we can clear the modified flag */
-		np->n_flag &= ~NMODIFIED;
-	}
+	u_int64_t xid;
+	int error = 0, lockerror, status, nfsvers;
+	struct nfsm_chain nmreq, nmrep;
+	struct nfsmount *nmp = NFSTONMP(np);
+	uint32_t val = 0;
 
-	FSDBG(526, np->n_flag, np->n_error, 0, 0);
-	if (!ignore_writeerr && (np->n_flag & NWRITEERR)) {
-		error = np->n_error;
-		np->n_flag &= ~NWRITEERR;
-	}
-done:
-	FSDBG_BOT(517, vp, np, error, 0);
-	return (error);
-}
+	if (!nmp)
+		return (ENXIO);
+	nfsvers = nmp->nm_vers;
 
-/*
- * Do an nfs pathconf rpc.
- */
-int
-nfs_pathconfrpc(
-	vnode_t vp,
-	struct nfsv3_pathconf *pc,
-	kauth_cred_t cred,
-	proc_t procp)
-{
-	mbuf_t mreq, mrep, md, mb, mb2;
-	caddr_t bpos, dpos, cp, cp2;
-	int32_t t1, t2;
-	u_long *tl;
-	u_int64_t xid;
-	int attrflag, error = 0;
-	struct nfsv3_pathconf *mpc;
+	nfsm_chain_null(&nmreq);
+	nfsm_chain_null(&nmrep);
 
 	/* fetch pathconf info from server */
-	nfsm_reqhead(NFSX_FH(1));
-	if (error)
-		return (error);
-	nfsm_fhtom(vp, 1);
-	nfsm_request(vp, NFSPROC_PATHCONF, procp, cred, &xid);
-	nfsm_postop_attr_update(vp, 1, attrflag, &xid);
-	if (!error) {
-		nfsm_dissect(mpc, struct nfsv3_pathconf *, NFSX_V3PATHCONF);
-		pc->pc_linkmax = fxdr_unsigned(long, mpc->pc_linkmax);
-		pc->pc_namemax = fxdr_unsigned(long, mpc->pc_namemax);
-		pc->pc_chownrestricted = fxdr_unsigned(long, mpc->pc_chownrestricted);
-		pc->pc_notrunc = fxdr_unsigned(long, mpc->pc_notrunc);
-		pc->pc_caseinsensitive = fxdr_unsigned(long, mpc->pc_caseinsensitive);
-		pc->pc_casepreserving = fxdr_unsigned(long, mpc->pc_casepreserving);
-	}
-	nfsm_reqdone;
-
+	nfsm_chain_build_alloc_init(error, &nmreq, NFSX_FH(NFS_VER3));
+	nfsm_chain_add_fh(error, &nmreq, nfsvers, np->n_fhp, np->n_fhsize);
+	nfsm_chain_build_done(error, &nmreq);
+	nfsmout_if(error);
+	error = nfs_request(np, NULL, &nmreq, NFSPROC_PATHCONF, ctx,
+			&nmrep, &xid, &status);
+	if ((lockerror = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		error = lockerror;
+	nfsm_chain_postop_attr_update(error, &nmrep, np, &xid);
+	if (!lockerror)
+		nfs_unlock(np);
+	if (!error)
+		error = status;
+	nfsm_chain_get_32(error, &nmrep, nfsap->nfsa_maxlink);
+	nfsm_chain_get_32(error, &nmrep, nfsap->nfsa_maxname);
+	nfsm_chain_get_32(error, &nmrep, val);
+	if (val)
+		nfsap->nfsa_flags |= NFS_FSFLAG_NO_TRUNC;
+	nfsm_chain_get_32(error, &nmrep, val);
+	if (val)
+		nfsap->nfsa_flags |= NFS_FSFLAG_CHOWN_RESTRICTED;
+	nfsm_chain_get_32(error, &nmrep, val);
+	if (val)
+		nfsap->nfsa_flags |= NFS_FSFLAG_CASE_INSENSITIVE;
+	nfsm_chain_get_32(error, &nmrep, val);
+	if (val)
+		nfsap->nfsa_flags |= NFS_FSFLAG_CASE_PRESERVING;
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_MAXLINK);
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_MAXNAME);
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_NO_TRUNC);
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_CHOWN_RESTRICTED);
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_CASE_INSENSITIVE);
+	NFS_BITMAP_SET(nfsap->nfsa_bitmap, NFS_FATTR_CASE_PRESERVING);
+nfsmout:
+	nfsm_chain_cleanup(&nmreq);
+	nfsm_chain_cleanup(&nmrep);
 	return (error);
 }
 
+/* save pathconf info for NFSv3 mount */
 void
-nfs_pathconf_cache(struct nfsmount *nmp, struct nfsv3_pathconf *pc)
+nfs3_pathconf_cache(struct nfsmount *nmp, struct nfs_fsattr *nfsap)
 {
+	nmp->nm_fsattr.nfsa_maxlink = nfsap->nfsa_maxlink;
+	nmp->nm_fsattr.nfsa_maxname = nfsap->nfsa_maxname;
+	nmp->nm_fsattr.nfsa_flags |= nfsap->nfsa_flags & NFS_FSFLAG_NO_TRUNC;
+	nmp->nm_fsattr.nfsa_flags |= nfsap->nfsa_flags & NFS_FSFLAG_CHOWN_RESTRICTED;
+	nmp->nm_fsattr.nfsa_flags |= nfsap->nfsa_flags & NFS_FSFLAG_CASE_INSENSITIVE;
+	nmp->nm_fsattr.nfsa_flags |= nfsap->nfsa_flags & NFS_FSFLAG_CASE_PRESERVING;
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXLINK);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXNAME);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_NO_TRUNC);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CHOWN_RESTRICTED);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CASE_INSENSITIVE);
+	NFS_BITMAP_SET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_CASE_PRESERVING);
 	nmp->nm_state |= NFSSTA_GOTPATHCONF;
-	nmp->nm_fsinfo.linkmax = pc->pc_linkmax;
-	nmp->nm_fsinfo.namemax = pc->pc_namemax;
-	nmp->nm_fsinfo.pcflags = 0;
-	if (pc->pc_notrunc)
-		nmp->nm_fsinfo.pcflags |= NFSPCINFO_NOTRUNC;
-	if (pc->pc_chownrestricted)
-		nmp->nm_fsinfo.pcflags |= NFSPCINFO_CHOWN_RESTRICTED;
-	if (pc->pc_caseinsensitive)
-		nmp->nm_fsinfo.pcflags |= NFSPCINFO_CASE_INSENSITIVE;
-	if (pc->pc_casepreserving)
-		nmp->nm_fsinfo.pcflags |= NFSPCINFO_CASE_PRESERVING;
 }
 
 /*
@@ -4228,25 +5129,26 @@ nfs_pathconf_cache(struct nfsmount *nmp, struct nfsv3_pathconf *pc)
  */
 /* ARGSUSED */
 static int
-nfs_pathconf(ap)
+nfs_vnop_pathconf(
 	struct vnop_pathconf_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_name;
 		register_t *a_retval;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	vnode_t vp = ap->a_vp;
+	nfsnode_t np = VTONFS(vp);
 	struct nfsmount *nmp;
-	struct nfsv3_pathconf pc;
-	int error = 0, cached;
+	struct nfs_fsattr nfsa, *nfsap;
+	int error = 0;
+	uint64_t maxFileSize;
+	uint nbits;
 
-	nmp = VFSTONFS(vnode_mount(vp));
+	nmp = VTONMP(vp);
 	if (!nmp)
 		return (ENXIO);
-	if (!NFS_ISV3(vp))
-		return (EINVAL);
 
 	switch (ap->a_name) {
 	case _PC_LINK_MAX:
@@ -4256,205 +5158,166 @@ nfs_pathconf(ap)
 	case _PC_CASE_SENSITIVE:
 	case _PC_CASE_PRESERVING:
 		break;
+	case _PC_FILESIZEBITS:
+		if (nmp->nm_vers == NFS_VER2) {
+			*ap->a_retval = 32;
+			return (0);
+		}
+		break;
 	default:
 		/* don't bother contacting the server if we know the answer */
 		return (EINVAL);
 	}
 
-	if (!(nmp->nm_state & NFSSTA_GOTPATHCONF)) {
+	if (nmp->nm_vers == NFS_VER2)
+		return (EINVAL);
+
+	lck_mtx_lock(&nmp->nm_lock);
+	if (nmp->nm_vers == NFS_VER3) {
+		if (!(nmp->nm_state & NFSSTA_GOTPATHCONF)) {
+			/* no pathconf info cached */
+			lck_mtx_unlock(&nmp->nm_lock);
+			NFS_CLEAR_ATTRIBUTES(nfsa.nfsa_bitmap);
+			error = nfs3_pathconf_rpc(np, &nfsa, ap->a_context);
+			if (error)
+				return (error);
+			nmp = VTONMP(vp);
+			if (!nmp)
+				return (ENXIO);
+			lck_mtx_lock(&nmp->nm_lock);
+			if (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_HOMOGENEOUS) {
+				/* all files have the same pathconf info, */
+				/* so cache a copy of the results */
+				nfs3_pathconf_cache(nmp, &nfsa);
+			}
+			nfsap = &nfsa;
+		} else {
+			nfsap = &nmp->nm_fsattr;
+		}
+	} else if (!(nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_HOMOGENEOUS)) {
 		/* no pathconf info cached */
-		kauth_cred_t cred = vfs_context_ucred(ap->a_context);
-		proc_t p = vfs_context_proc(ap->a_context);
-		error = nfs_pathconfrpc(vp, &pc, cred, p);
+		lck_mtx_unlock(&nmp->nm_lock);
+		NFS_CLEAR_ATTRIBUTES(nfsa.nfsa_bitmap);
+		error = nfs4_pathconf_rpc(np, &nfsa, ap->a_context);
 		if (error)
 			return (error);
-		nmp = VFSTONFS(vnode_mount(vp));
+		nmp = VTONMP(vp);
 		if (!nmp)
 			return (ENXIO);
-		if (!(nmp->nm_state & NFSSTA_GOTFSINFO)) {
-			nfs_fsinfo(nmp, vp, cred, p);
-			nmp = VFSTONFS(vnode_mount(vp));
-			if (!nmp)
-				return (ENXIO);
-		}
-		if ((nmp->nm_state & NFSSTA_GOTFSINFO) &&
-		    (nmp->nm_fsinfo.fsproperties & NFSV3FSINFO_HOMOGENEOUS)) {
-			/* all files have the same pathconf info, */
-			/* so cache a copy of the results */
-			nfs_pathconf_cache(nmp, &pc);
-		}
+		lck_mtx_lock(&nmp->nm_lock);
+		nfsap = &nfsa;
+	} else {
+		nfsap = &nmp->nm_fsattr;
 	}
 
-	cached = (nmp->nm_state & NFSSTA_GOTPATHCONF);
-
 	switch (ap->a_name) {
 	case _PC_LINK_MAX:
-		*ap->a_retval = cached ? nmp->nm_fsinfo.linkmax : pc.pc_linkmax;
-		break;
-	case _PC_NAME_MAX:
-		*ap->a_retval = cached ? nmp->nm_fsinfo.namemax : pc.pc_namemax;
-		break;
-	case _PC_CHOWN_RESTRICTED:
-		if (cached)
-			*ap->a_retval = (nmp->nm_fsinfo.pcflags & NFSPCINFO_CHOWN_RESTRICTED) ? 1 : 0;
-		else
-			*ap->a_retval = pc.pc_chownrestricted;
-		break;
-	case _PC_NO_TRUNC:
-		if (cached)
-			*ap->a_retval = (nmp->nm_fsinfo.pcflags & NFSPCINFO_NOTRUNC) ? 1 : 0;
-		else
-			*ap->a_retval = pc.pc_notrunc;
-		break;
-	case _PC_CASE_SENSITIVE:
-		if (cached)
-			*ap->a_retval = (nmp->nm_fsinfo.pcflags & NFSPCINFO_CASE_INSENSITIVE) ? 0 : 1;
-		else
-			*ap->a_retval = !pc.pc_caseinsensitive;
-		break;
-	case _PC_CASE_PRESERVING:
-		if (cached)
-			*ap->a_retval = (nmp->nm_fsinfo.pcflags & NFSPCINFO_CASE_PRESERVING) ? 1 : 0;
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_MAXLINK))
+			*ap->a_retval = nfsap->nfsa_maxlink;
+		else if ((nmp->nm_vers == NFS_VER4) && NFS_BITMAP_ISSET(np->n_vattr.nva_bitmap, NFS_FATTR_MAXLINK))
+			*ap->a_retval = np->n_vattr.nva_maxlink;
 		else
-			*ap->a_retval = pc.pc_casepreserving;
+			error = EINVAL;
 		break;
-	default:
-		error = EINVAL;
-	}
-
-	return (error);
-}
-
-/*
- * NFS advisory byte-level locks (client)
- */
-static int
-nfs_advlock(ap)
-	struct vnop_advlock_args /* {
-		struct vnodeop_desc *a_desc;
-		vnode_t a_vp;
-		caddr_t a_id;
-		int a_op;
-		struct flock *a_fl;
-		int a_flags;
-		vfs_context_t a_context;
-	} */ *ap;
-{
-	return (nfs_dolock(ap));
-}
-
-/*
- * write (or commit) the given NFS buffer
- */
-int
-nfs_buf_write(struct nfsbuf *bp)
-{
-	int oldflags = bp->nb_flags, rv = 0;
-	vnode_t vp = bp->nb_vp;
-	struct nfsnode *np = VTONFS(vp);
-	kauth_cred_t cr;
-	proc_t p = current_proc(); // XXX
-
-	FSDBG_TOP(553, bp, NBOFF(bp), bp->nb_flags, 0);
-
-	if (!ISSET(bp->nb_lflags, NBL_BUSY))
-		panic("nfs_buf_write: buffer is not busy???");
-
-	CLR(bp->nb_flags, (NB_READ|NB_DONE|NB_ERROR|NB_DELWRI));
-	if (ISSET(oldflags, NB_DELWRI)) {
-		OSAddAtomic(-1, (SInt32*)&nfs_nbdwrite);
-		NFSBUFCNTCHK(0);
-		wakeup(&nfs_nbdwrite);
-	}
-
-	/* move to clean list */
-	if (ISSET(oldflags, (NB_ASYNC|NB_DELWRI))) {
-		lck_mtx_lock(nfs_buf_mutex);
-		if (bp->nb_vnbufs.le_next != NFSNOLIST)
-			LIST_REMOVE(bp, nb_vnbufs);
-		LIST_INSERT_HEAD(&VTONFS(vp)->n_cleanblkhd, bp, nb_vnbufs);
-		lck_mtx_unlock(nfs_buf_mutex);
-	}
-	vnode_startwrite(vp);
-
-	if (p && p->p_stats)
-		p->p_stats->p_ru.ru_oublock++;
-
-	/*
-	 * For async requests when nfsiod(s) are running, queue the request by
-	 * calling nfs_asyncio(), otherwise just all nfs_doio() to do the request.
-	 */
-	if (ISSET(bp->nb_flags, NB_ASYNC))
-		p = NULL;
-	if (ISSET(bp->nb_flags, NB_READ))
-		cr = bp->nb_rcred;
-	else
-		cr = bp->nb_wcred;
-	if (!ISSET(bp->nb_flags, NB_ASYNC) || nfs_asyncio(bp, NOCRED))
-		rv = nfs_doio(bp, cr, p);
-
-	if ((oldflags & NB_ASYNC) == 0) {
-		rv = nfs_buf_iowait(bp);
-		/* move to clean list */
-		if (oldflags & NB_DELWRI) {
-			lck_mtx_lock(nfs_buf_mutex);
-			if (bp->nb_vnbufs.le_next != NFSNOLIST)
-				LIST_REMOVE(bp, nb_vnbufs);
-			LIST_INSERT_HEAD(&VTONFS(vp)->n_cleanblkhd, bp, nb_vnbufs);
-			lck_mtx_unlock(nfs_buf_mutex);
+	case _PC_NAME_MAX:
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_MAXNAME))
+			*ap->a_retval = nfsap->nfsa_maxname;
+		else
+			error = EINVAL;
+		break;
+	case _PC_CHOWN_RESTRICTED:
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_CHOWN_RESTRICTED))
+			*ap->a_retval = (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_CHOWN_RESTRICTED) ? 200112 /* _POSIX_CHOWN_RESTRICTED */ : 0;
+		else
+			error = EINVAL;
+		break;
+	case _PC_NO_TRUNC:
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_NO_TRUNC))
+			*ap->a_retval = (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_NO_TRUNC) ? 200112 /* _POSIX_NO_TRUNC */ : 0;
+		else
+			error = EINVAL;
+		break;
+	case _PC_CASE_SENSITIVE:
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_CASE_INSENSITIVE))
+			*ap->a_retval = (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_CASE_INSENSITIVE) ? 0 : 1;
+		else
+			error = EINVAL;
+		break;
+	case _PC_CASE_PRESERVING:
+		if (NFS_BITMAP_ISSET(nfsap->nfsa_bitmap, NFS_FATTR_CASE_PRESERVING))
+			*ap->a_retval = (nmp->nm_fsattr.nfsa_flags & NFS_FSFLAG_CASE_PRESERVING) ? 1 : 0;
+		else
+			error = EINVAL;
+		break;
+	case _PC_FILESIZEBITS:
+		if (!NFS_BITMAP_ISSET(nmp->nm_fsattr.nfsa_bitmap, NFS_FATTR_MAXFILESIZE)) {
+			*ap->a_retval = 64;
+			error = 0;
+			break;
 		}
-		oldflags = bp->nb_flags;
-		FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, rv);
-		if (IS_VALID_CRED(cr)) {
-			kauth_cred_ref(cr);
+		maxFileSize = nmp->nm_fsattr.nfsa_maxfilesize;
+		nbits = 1;
+		if (maxFileSize & 0xffffffff00000000ULL) {
+			nbits += 32;
+			maxFileSize >>= 32;
 		}
-		nfs_buf_release(bp, 1);
-		if (ISSET(oldflags, NB_ERROR) && !(np->n_flag & NFLUSHINPROG)) {
-			/*
-			 * There was a write error and we need to
-			 * invalidate attrs and flush buffers in
-			 * order to sync up with the server.
-			 * (if this write was extending the file,
-			 * we may no longer know the correct size)
-			 *
-			 * But we couldn't call vinvalbuf while holding
-			 * the buffer busy.  So we call vinvalbuf() after
-			 * releasing the buffer.
-			 */
-			nfs_vinvalbuf(vp, V_SAVE|V_IGNORE_WRITEERR, cr, p, 1);
+		if (maxFileSize & 0xffff0000) {
+			nbits += 16;
+			maxFileSize >>= 16;
+		}
+		if (maxFileSize & 0xff00) {
+			nbits += 8;
+			maxFileSize >>= 8;
+		}
+		if (maxFileSize & 0xf0) {
+			nbits += 4;
+			maxFileSize >>= 4;
 		}
-		if (IS_VALID_CRED(cr))
-			kauth_cred_unref(&cr);
-		return (rv);
-	} 
+		if (maxFileSize & 0xc) {
+			nbits += 2;
+			maxFileSize >>= 2;
+		}
+		if (maxFileSize & 0x2) {
+			nbits += 1;
+		}
+		*ap->a_retval = nbits;
+		break;
+	default:
+		error = EINVAL;
+	}
 
-	FSDBG_BOT(553, bp, NBOFF(bp), bp->nb_flags, rv);
-	return (rv);
+	lck_mtx_unlock(&nmp->nm_lock);
+
+	return (error);
 }
 
 /*
  * Read wrapper for special devices.
  */
 static int
-nfsspec_read(ap)
+nfsspec_vnop_read(
 	struct vnop_read_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		int a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	register struct nfsnode *np = VTONFS(ap->a_vp);
+	nfsnode_t np = VTONFS(ap->a_vp);
 	struct timeval now;
+	int error;
 
 	/*
 	 * Set access flag.
 	 */
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	np->n_flag |= NACC;
 	microtime(&now);
 	np->n_atim.tv_sec = now.tv_sec;
 	np->n_atim.tv_nsec = now.tv_usec * 1000;
+	nfs_unlock(np);
 	return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_read), ap));
 }
 
@@ -4462,25 +5325,29 @@ nfsspec_read(ap)
  * Write wrapper for special devices.
  */
 static int
-nfsspec_write(ap)
+nfsspec_vnop_write(
 	struct vnop_write_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		int a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	register struct nfsnode *np = VTONFS(ap->a_vp);
+	nfsnode_t np = VTONFS(ap->a_vp);
 	struct timeval now;
+	int error;
 
 	/*
 	 * Set update flag.
 	 */
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	np->n_flag |= NUPD;
 	microtime(&now);
 	np->n_mtim.tv_sec = now.tv_sec;
 	np->n_mtim.tv_nsec = now.tv_usec * 1000;
+	nfs_unlock(np);
 	return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_write), ap));
 }
 
@@ -4490,19 +5357,22 @@ nfsspec_write(ap)
  * Update the times on the nfsnode then do device close.
  */
 static int
-nfsspec_close(ap)
+nfsspec_vnop_close(
 	struct vnop_close_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_fflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
 	struct vnode_attr vattr;
 	mount_t mp;
+	int error;
 
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	if (np->n_flag & (NACC | NUPD)) {
 		np->n_flag |= NCHG;
 		if (!vnode_isinuse(vp, 1) && (mp = vnode_mount(vp)) && !vfs_isrdonly(mp)) {
@@ -4515,37 +5385,47 @@ nfsspec_close(ap)
 				vattr.va_modify_time = np->n_mtim;
 				VATTR_SET_ACTIVE(&vattr, va_modify_time);
 			}
+			nfs_unlock(np);
 			vnode_setattr(vp, &vattr, ap->a_context);
+		} else {
+			nfs_unlock(np);
 		}
+	} else {
+		nfs_unlock(np);
 	}
 	return (VOCALL(spec_vnodeop_p, VOFFSET(vnop_close), ap));
 }
 
+#if FIFO
 extern vnop_t **fifo_vnodeop_p;
 
 /*
  * Read wrapper for fifos.
  */
 static int
-nfsfifo_read(ap)
+nfsfifo_vnop_read(
 	struct vnop_read_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		int a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	register struct nfsnode *np = VTONFS(ap->a_vp);
+	nfsnode_t np = VTONFS(ap->a_vp);
 	struct timeval now;
+	int error;
 
 	/*
 	 * Set access flag.
 	 */
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	np->n_flag |= NACC;
 	microtime(&now);
 	np->n_atim.tv_sec = now.tv_sec;
 	np->n_atim.tv_nsec = now.tv_usec * 1000;
+	nfs_unlock(np);
 	return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_read), ap));
 }
 
@@ -4553,25 +5433,29 @@ nfsfifo_read(ap)
  * Write wrapper for fifos.
  */
 static int
-nfsfifo_write(ap)
+nfsfifo_vnop_write(
 	struct vnop_write_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		struct uio *a_uio;
 		int a_ioflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
-	register struct nfsnode *np = VTONFS(ap->a_vp);
+	nfsnode_t np = VTONFS(ap->a_vp);
 	struct timeval now;
+	int error;
 
 	/*
 	 * Set update flag.
 	 */
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	np->n_flag |= NUPD;
 	microtime(&now);
 	np->n_mtim.tv_sec = now.tv_sec;
 	np->n_mtim.tv_nsec = now.tv_usec * 1000;
+	nfs_unlock(np);
 	return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_write), ap));
 }
 
@@ -4581,20 +5465,23 @@ nfsfifo_write(ap)
  * Update the times on the nfsnode then do fifo close.
  */
 static int
-nfsfifo_close(ap)
+nfsfifo_vnop_close(
 	struct vnop_close_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_fflag;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	vnode_t vp = ap->a_vp;
-	struct nfsnode *np = VTONFS(vp);
+	nfsnode_t np = VTONFS(vp);
 	struct vnode_attr vattr;
 	struct timeval now;
 	mount_t mp;
+	int error;
 
+	if ((error = nfs_lock(np, NFS_NODE_LOCK_EXCLUSIVE)))
+		return (error);
 	if (np->n_flag & (NACC | NUPD)) {
 		microtime(&now);
 		if (np->n_flag & NACC) {
@@ -4616,23 +5503,28 @@ nfsfifo_close(ap)
 				vattr.va_modify_time = np->n_mtim;
 				VATTR_SET_ACTIVE(&vattr, va_modify_time);
 			}
+			nfs_unlock(np);
 			vnode_setattr(vp, &vattr, ap->a_context);
+		} else {
+			nfs_unlock(np);
 		}
+	} else {
+		nfs_unlock(np);
 	}
 	return (VOCALL(fifo_vnodeop_p, VOFFSET(vnop_close), ap));
 }
+#endif /* FIFO */
 
 /*ARGSUSED*/
 static int
-nfs_ioctl(
+nfs_vnop_ioctl(
 	__unused struct vnop_ioctl_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		u_long a_command;
 		caddr_t a_data;
 		int a_fflag;
-		kauth_cred_t a_cred;
-		proc_t a_p;
+		vfs_context_t a_context;
 	} */ *ap)
 {
 
@@ -4645,15 +5537,14 @@ nfs_ioctl(
 
 /*ARGSUSED*/
 static int
-nfs_select(
+nfs_vnop_select(
 	__unused struct vnop_select_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		int a_which;
 		int a_fflags;
-		kauth_cred_t a_cred;
 		void *a_wql;
-		proc_t a_p;
+		vfs_context_t a_context;
 	} */ *ap)
 {
 
@@ -4664,12 +5555,12 @@ nfs_select(
 }
 
 /*
- * Vnode op for pagein using getblk_pages
- * derived from nfs_bioread()
- * No read aheads are started from pagein operation
+ * vnode OP for pagein using UPL
+ *
+ * No buffer I/O, just RPCs straight into the mapped pages.
  */
 static int
-nfs_pagein(ap)
+nfs_vnop_pagein(
 	struct vnop_pagein_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -4679,18 +5570,19 @@ nfs_pagein(ap)
 		size_t a_size;
 		int a_flags;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	vnode_t vp = ap->a_vp;
 	upl_t pl = ap->a_pl;
-	size_t size= ap->a_size;
+	size_t size = ap->a_size;
 	off_t f_offset = ap->a_f_offset;
 	vm_offset_t pl_offset = ap->a_pl_offset;
-	int flags  = ap->a_flags;
+	int flags = ap->a_flags;
+	thread_t thd;
 	kauth_cred_t cred;
-	proc_t p;
-	struct nfsnode *np = VTONFS(vp);
-	int biosize, xsize, iosize;
+	nfsnode_t np = VTONFS(vp);
+	size_t nmrsize, iosize, txsize, rxsize, retsize;
+	off_t txoffset;
 	struct nfsmount *nmp;
 	int error = 0;
 	vm_offset_t ioaddr;
@@ -4699,118 +5591,119 @@ nfs_pagein(ap)
 	struct uio * uio = &auio;
 	int nofreeupl = flags & UPL_NOCOMMIT;
 	upl_page_info_t *plinfo;
+#define MAXPAGINGREQS	16	/* max outstanding RPCs for pagein/pageout */
+	struct nfsreq *req[MAXPAGINGREQS];
+	int nextsend, nextwait;
 
-	FSDBG(322, vp, f_offset, size, flags);
+	FSDBG(322, np, f_offset, size, flags);
 	if (pl == (upl_t)NULL)
 		panic("nfs_pagein: no upl");
 
-	if (UBCINVALID(vp)) {
-		printf("nfs_pagein: invalid vnode 0x%x", (int)vp);
-		if (!nofreeupl)
-			(void) ubc_upl_abort(pl, 0); 
-		return (EPERM);
-	}
-	UBCINFOCHECK("nfs_pagein", vp);
-
 	if (size <= 0) {
-		printf("nfs_pagein: invalid size %d", size);
+		printf("nfs_pagein: invalid size %ld", size);
 		if (!nofreeupl)
-			(void) ubc_upl_abort(pl, 0); 
+			(void) ubc_upl_abort(pl, 0);
 		return (EINVAL);
 	}
 	if (f_offset < 0 || f_offset >= (off_t)np->n_size || (f_offset & PAGE_MASK_64)) {
 		if (!nofreeupl)
-			ubc_upl_abort_range(pl, pl_offset, size, 
+			ubc_upl_abort_range(pl, pl_offset, size,
 				UPL_ABORT_ERROR | UPL_ABORT_FREE_ON_EMPTY);
 		return (EINVAL);
 	}
 
+	thd = vfs_context_thread(ap->a_context);
 	cred = ubc_getcred(vp);
 	if (!IS_VALID_CRED(cred))
 		cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
 
 	auio.uio_offset = f_offset;
 #if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
 	auio.uio_segflg = UIO_SYSSPACE;
 #else
 	auio.uio_segflg = UIO_SYSSPACE32;
-#endif 
+#endif
 	auio.uio_rw = UIO_READ;
-	auio.uio_procp = p;
+	auio.uio_procp = vfs_context_proc(ap->a_context);
 
-	nmp = VFSTONFS(vnode_mount(vp));
+	nmp = VTONMP(vp);
 	if (!nmp) {
 		if (!nofreeupl)
-			ubc_upl_abort_range(pl, pl_offset, size, 
+			ubc_upl_abort_range(pl, pl_offset, size,
 				UPL_ABORT_ERROR | UPL_ABORT_FREE_ON_EMPTY);
 		return (ENXIO);
 	}
-	biosize = nmp->nm_biosize;
-	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		nfs_fsinfo(nmp, vp, cred, p);
+	nmrsize = nmp->nm_rsize;
 
 	plinfo = ubc_upl_pageinfo(pl);
 	ubc_upl_map(pl, &ioaddr);
 	ioaddr += pl_offset;
-	xsize = size;
+	txsize = rxsize = size;
+	txoffset = f_offset;
 
+	bzero(req, sizeof(req));
+	nextsend = nextwait = 0;
 	do {
-		/*
-		 * It would be nice to be able to issue all these requests
-		 * in parallel instead of waiting for each one to complete
-		 * before sending the next one.
-		 * XXX Should we align these requests to block boundaries?
-		 */
-		iosize = min(biosize, xsize);
-		aiov.iov_len  = iosize;
-		aiov.iov_base = (uintptr_t)ioaddr;
-		auio.uio_iovs.iov32p = &aiov;
-		auio.uio_iovcnt = 1;
-		uio_uio_resid_set(&auio, iosize);
-
-		FSDBG(322, uio->uio_offset, uio_uio_resid(uio), ioaddr, xsize);
-		/*
-		 * With UBC we get here only when the file data is not in the VM
-		 * page cache, so go ahead and read in.
-		 */
+		/* send requests while we need to and have available slots */
+		while ((txsize > 0) && (req[nextsend] == NULL)) {
+			iosize = MIN(nmrsize, txsize);
+			if ((error = nmp->nm_funcs->nf_read_rpc_async(np, txoffset, iosize, thd, cred, NULL, &req[nextsend]))) {
+				req[nextsend] = NULL;
+				break;
+			}
+			txoffset += iosize;
+			txsize -= iosize;
+			nextsend = (nextsend + 1) % MAXPAGINGREQS;
+		}
+		/* wait while we need to and break out if more requests to send */
+		while ((rxsize > 0) && req[nextwait]) {
+			iosize = retsize = MIN(nmrsize, rxsize);
+			aiov.iov_len  = iosize;
+			aiov.iov_base = (uintptr_t)ioaddr;
+			auio.uio_iovs.iov32p = &aiov;
+			auio.uio_iovcnt = 1;
+			uio_uio_resid_set(&auio, iosize);
+			FSDBG(322, uio->uio_offset, uio_uio_resid(uio), ioaddr, rxsize);
 #ifdef UPL_DEBUG
-		upl_ubc_alias_set(pl, current_thread(), 2);
+			upl_ubc_alias_set(pl, current_thread(), 2);
 #endif /* UPL_DEBUG */
-		OSAddAtomic(1, (SInt32*)&nfsstats.pageins);
-
-		error = nfs_readrpc(vp, uio, cred, p);
-
-		if (!error) {
-			if (uio_uio_resid(uio)) {
-				/*
-				 * If uio_resid > 0, there is a hole in the file
-				 * and no writes after the hole have been pushed
-				 * to the server yet... or we're at the EOF
-				 * Just zero fill the rest of the valid area.
-				 */
+			OSAddAtomic(1, (SInt32*)&nfsstats.pageins);
+			error = nmp->nm_funcs->nf_read_rpc_async_finish(np, req[nextwait], uio, &retsize, NULL);
+			req[nextwait] = NULL;
+			nextwait = (nextwait + 1) % MAXPAGINGREQS;
+			if (error) {
+				FSDBG(322, uio->uio_offset, uio_uio_resid(uio), error, -1);
+				break;
+			}
+			if (retsize < iosize) {
+				/* Just zero fill the rest of the valid area. */
 				// LP64todo - fix this
-				int zcnt = uio_uio_resid(uio);
-				int zoff = iosize - zcnt;
-				bzero((char *)ioaddr + zoff, zcnt);
-
-				FSDBG(324, uio->uio_offset, zoff, zcnt, ioaddr);
+				int zcnt = iosize - retsize;
+				bzero((char *)ioaddr + retsize, zcnt);
+				FSDBG(324, uio->uio_offset, retsize, zcnt, ioaddr);
 				uio->uio_offset += zcnt;
 			}
 			ioaddr += iosize;	
-			xsize  -= iosize;
-		} else {
-			FSDBG(322, uio->uio_offset, uio_uio_resid(uio), error, -1);
+			rxsize -= iosize;
+			if (txsize)
+				break;
 		}
-
-		nmp = VFSTONFS(vnode_mount(vp));
-	} while (error == 0 && xsize > 0);
+	} while (!error && (txsize || rxsize));
 
 	ubc_upl_unmap(pl);
 
+	if (error) {
+		/* cancel any outstanding requests */
+		while (req[nextwait]) {
+			nfs_request_async_cancel(req[nextwait]);
+			req[nextwait] = NULL;
+			nextwait = (nextwait + 1) % MAXPAGINGREQS;
+		}
+	}
+
 	if (!nofreeupl) {
-		if (error) 
-			ubc_upl_abort_range(pl, pl_offset, size, 
+		if (error)
+			ubc_upl_abort_range(pl, pl_offset, size,
 					    UPL_ABORT_ERROR |
 					    UPL_ABORT_FREE_ON_EMPTY);
 		else
@@ -4823,12 +5716,134 @@ nfs_pagein(ap)
 
 
 /*
- * Vnode op for pageout using UPL
- * Derived from nfs_write()
+ * the following are needed only by nfs_pageout to know how to handle errors
+ * see nfs_pageout comments on explanation of actions.
+ * the errors here are copied from errno.h and errors returned by servers
+ * are expected to match the same numbers here. If not, our actions maybe
+ * erroneous.
+ */
+enum actiontype {NOACTION, DUMP, DUMPANDLOG, RETRY, RETRYWITHSLEEP, SEVER};
+#define NFS_ELAST 88
+static u_char errorcount[NFS_ELAST+1]; /* better be zeros when initialized */
+static const char errortooutcome[NFS_ELAST+1] = {
+	NOACTION,
+	DUMP,			/* EPERM	1	Operation not permitted */
+	DUMP,			/* ENOENT	2	No such file or directory */
+	DUMPANDLOG,		/* ESRCH	3	No such process */
+	RETRY,			/* EINTR 	4	Interrupted system call */
+	DUMP,			/* EIO		5	Input/output error */
+	DUMP,			/* ENXIO	6	Device not configured */
+	DUMPANDLOG,		/* E2BIG	7	Argument list too long */
+	DUMPANDLOG,		/* ENOEXEC	8	Exec format error */
+	DUMPANDLOG,		/* EBADF	9	Bad file descriptor */
+	DUMPANDLOG,		/* ECHILD	10	No child processes */
+	DUMPANDLOG,		/* EDEADLK	11	Resource deadlock avoided - was EAGAIN */
+	RETRY,			/* ENOMEM	12	Cannot allocate memory */
+	DUMP,			/* EACCES	13	Permission denied */
+	DUMPANDLOG,		/* EFAULT	14	Bad address */
+	DUMPANDLOG,		/* ENOTBLK	15	POSIX - Block device required */
+	RETRY,			/* EBUSY	16	Device busy */
+	DUMP,			/* EEXIST	17	File exists */
+	DUMP,			/* EXDEV	18	Cross-device link */
+	DUMP,			/* ENODEV	19	Operation not supported by device */
+	DUMP,			/* ENOTDIR	20	Not a directory */
+	DUMP,			/* EISDIR 	21	Is a directory */
+	DUMP,			/* EINVAL	22	Invalid argument */
+	DUMPANDLOG,		/* ENFILE	23	Too many open files in system */
+	DUMPANDLOG,		/* EMFILE	24	Too many open files */
+	DUMPANDLOG,		/* ENOTTY	25	Inappropriate ioctl for device */
+	DUMPANDLOG,		/* ETXTBSY	26	Text file busy - POSIX */
+	DUMP,			/* EFBIG	27	File too large */
+	DUMP,			/* ENOSPC	28	No space left on device */
+	DUMPANDLOG,		/* ESPIPE	29	Illegal seek */
+	DUMP,			/* EROFS	30	Read-only file system */
+	DUMP,			/* EMLINK	31	Too many links */
+	RETRY,			/* EPIPE	32	Broken pipe */
+	/* math software */
+	DUMPANDLOG,		/* EDOM				33	Numerical argument out of domain */
+	DUMPANDLOG,		/* ERANGE			34	Result too large */
+	RETRY,			/* EAGAIN/EWOULDBLOCK	35	Resource temporarily unavailable */
+	DUMPANDLOG,		/* EINPROGRESS		36	Operation now in progress */
+	DUMPANDLOG,		/* EALREADY			37	Operation already in progress */
+	/* ipc/network software -- argument errors */
+	DUMPANDLOG,		/* ENOTSOC			38	Socket operation on non-socket */
+	DUMPANDLOG,		/* EDESTADDRREQ		39	Destination address required */
+	DUMPANDLOG,		/* EMSGSIZE			40	Message too long */
+	DUMPANDLOG,		/* EPROTOTYPE		41	Protocol wrong type for socket */
+	DUMPANDLOG,		/* ENOPROTOOPT		42	Protocol not available */
+	DUMPANDLOG,		/* EPROTONOSUPPORT	43	Protocol not supported */
+	DUMPANDLOG,		/* ESOCKTNOSUPPORT	44	Socket type not supported */
+	DUMPANDLOG,		/* ENOTSUP			45	Operation not supported */
+	DUMPANDLOG,		/* EPFNOSUPPORT		46	Protocol family not supported */
+	DUMPANDLOG,		/* EAFNOSUPPORT		47	Address family not supported by protocol family */
+	DUMPANDLOG,		/* EADDRINUSE		48	Address already in use */
+	DUMPANDLOG,		/* EADDRNOTAVAIL	49	Can't assign requested address */
+	/* ipc/network software -- operational errors */
+	RETRY,			/* ENETDOWN			50	Network is down */
+	RETRY,			/* ENETUNREACH		51	Network is unreachable */
+	RETRY,			/* ENETRESET		52	Network dropped connection on reset */
+	RETRY,			/* ECONNABORTED		53	Software caused connection abort */
+	RETRY,			/* ECONNRESET		54	Connection reset by peer */
+	RETRY,			/* ENOBUFS			55	No buffer space available */
+	RETRY,			/* EISCONN			56	Socket is already connected */
+	RETRY,			/* ENOTCONN			57	Socket is not connected */
+	RETRY,			/* ESHUTDOWN		58	Can't send after socket shutdown */
+	RETRY,			/* ETOOMANYREFS		59	Too many references: can't splice */
+	RETRY,			/* ETIMEDOUT		60	Operation timed out */
+	RETRY,			/* ECONNREFUSED		61	Connection refused */
+
+	DUMPANDLOG,		/* ELOOP			62	Too many levels of symbolic links */
+	DUMP,			/* ENAMETOOLONG		63	File name too long */
+	RETRY,			/* EHOSTDOWN		64	Host is down */
+	RETRY,			/* EHOSTUNREACH		65	No route to host */
+	DUMP,			/* ENOTEMPTY		66	Directory not empty */
+	/* quotas & mush */
+	DUMPANDLOG,		/* PROCLIM			67	Too many processes */
+	DUMPANDLOG,		/* EUSERS			68	Too many users */
+	DUMPANDLOG,		/* EDQUOT			69	Disc quota exceeded */
+	/* Network File System */
+	DUMP,			/* ESTALE			70	Stale NFS file handle */
+	DUMP,			/* EREMOTE			71	Too many levels of remote in path */
+	DUMPANDLOG,		/* EBADRPC			72	RPC struct is bad */
+	DUMPANDLOG,		/* ERPCMISMATCH		73	RPC version wrong */
+	DUMPANDLOG,		/* EPROGUNAVAIL		74	RPC prog. not avail */
+	DUMPANDLOG,		/* EPROGMISMATCH	75	Program version wrong */
+	DUMPANDLOG,		/* EPROCUNAVAIL		76	Bad procedure for program */
+
+	DUMPANDLOG,		/* ENOLCK			77	No locks available */
+	DUMPANDLOG,		/* ENOSYS			78	Function not implemented */
+	DUMPANDLOG,		/* EFTYPE			79	Inappropriate file type or format */
+	DUMPANDLOG,		/* EAUTH			80	Authentication error */
+	DUMPANDLOG,		/* ENEEDAUTH		81	Need authenticator */
+	/* Intelligent device errors */
+	DUMPANDLOG,		/* EPWROFF			82	Device power is off */
+	DUMPANDLOG,		/* EDEVERR			83	Device error, e.g. paper out */
+	DUMPANDLOG,		/* EOVERFLOW		84	Value too large to be stored in data type */
+	/* Program loading errors */
+	DUMPANDLOG,		/* EBADEXEC			85	Bad executable */
+	DUMPANDLOG,		/* EBADARCH			86	Bad CPU type in executable */
+	DUMPANDLOG,		/* ESHLIBVERS		87	Shared library version mismatch */
+	DUMPANDLOG,		/* EBADMACHO		88	Malformed Macho file */
+};
+
+static char
+nfs_pageouterrorhandler(int error)
+{
+	if (error > NFS_ELAST)
+		return(DUMP);
+	else
+		return(errortooutcome[error]);
+}
+
+
+/*
+ * vnode OP for pageout using UPL
+ *
+ * No buffer I/O, just RPCs straight from the mapped pages.
  * File size changes are not permitted in pageout.
  */
 static int
-nfs_pageout(ap)
+nfs_vnop_pageout(
 	struct vnop_pageout_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
@@ -4838,45 +5853,40 @@ nfs_pageout(ap)
 		size_t a_size;
 		int a_flags;
 		vfs_context_t a_context;
-	} */ *ap;
+	} */ *ap)
 {
 	vnode_t vp = ap->a_vp;
 	upl_t pl = ap->a_pl;
-	size_t size= ap->a_size;
+	size_t size = ap->a_size;
 	off_t f_offset = ap->a_f_offset;
 	vm_offset_t pl_offset = ap->a_pl_offset;
-	int flags  = ap->a_flags;
-	struct nfsnode *np = VTONFS(vp);
+	int flags = ap->a_flags;
+	nfsnode_t np = VTONFS(vp);
+	thread_t thd;
 	kauth_cred_t cred;
-	proc_t p;
 	struct nfsbuf *bp;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
+	struct nfsmount *nmp = VTONMP(vp);
 	daddr64_t lbn;
 	int error = 0, iomode;
-	off_t off;
-	vm_offset_t ioaddr;
+	off_t off, txoffset, rxoffset;
+	vm_offset_t ioaddr, txaddr, rxaddr;
 	struct uio	auio;
 	struct iovec_32	aiov;
 	int nofreeupl = flags & UPL_NOCOMMIT;
-	size_t biosize, iosize, pgsize, xsize;
+	size_t nmwsize, biosize, iosize, pgsize, txsize, rxsize, xsize, remsize;
+	struct nfsreq *req[MAXPAGINGREQS];
+	int nextsend, nextwait, wverfset, commit, restart = 0;
+	uint64_t wverf, wverf2;
 
 	FSDBG(323, f_offset, size, pl, pl_offset);
 
 	if (pl == (upl_t)NULL)
 		panic("nfs_pageout: no upl");
 
-	if (UBCINVALID(vp)) {
-		printf("nfs_pageout: invalid vnode 0x%x", (int)vp);
-		if (!nofreeupl)
-			ubc_upl_abort(pl, 0); 
-		return (EIO);
-	}
-	UBCINFOCHECK("nfs_pageout", vp);
-
 	if (size <= 0) {
-		printf("nfs_pageout: invalid size %d", size);
+		printf("nfs_pageout: invalid size %ld", size);
 		if (!nofreeupl)
-			ubc_upl_abort(pl, 0); 
+			ubc_upl_abort(pl, 0);
 		return (EINVAL);
 	}
 
@@ -4886,6 +5896,9 @@ nfs_pageout(ap)
 		return (ENXIO);
 	}
 	biosize = nmp->nm_biosize;
+	nmwsize = nmp->nm_wsize;
+
+	nfs_data_lock2(np, NFS_NODE_LOCK_SHARED, 0);
 
 	/*
 	 * Check to see whether the buffer is incore.
@@ -4897,15 +5910,16 @@ nfs_pageout(ap)
 		xsize = biosize - (off % biosize);
 		if (off + xsize > f_offset + size)
 			xsize = f_offset + size - off;
-		lbn = ubc_offtoblk(vp, off);
+		lbn = (daddr64_t)(off / biosize);
 		lck_mtx_lock(nfs_buf_mutex);
-		if ((bp = nfs_buf_incore(vp, lbn))) {
+		if ((bp = nfs_buf_incore(np, lbn))) {
 			FSDBG(323, off, bp, bp->nb_lflags, bp->nb_flags);
 			if (nfs_buf_acquire(bp, NBAC_NOWAIT, 0, 0)) {
 				lck_mtx_unlock(nfs_buf_mutex);
+				nfs_data_unlock2(np, 0);
 				/* no panic. just tell vm we are busy */
 				if (!nofreeupl)
-					ubc_upl_abort(pl, 0); 
+					ubc_upl_abort(pl, 0);
 				return (EBUSY);
 			}
 			if (bp->nb_dirtyend > 0) {
@@ -4936,11 +5950,12 @@ nfs_pageout(ap)
 				if ((bp->nb_dirtyoff < start) &&
 				    (bp->nb_dirtyend > end)) {
 				    /* not gonna be able to clip the dirty region */
-				    FSDBG(323, vp, bp, 0xd00deebc, EBUSY);
+				    FSDBG(323, np, bp, 0xd00deebc, EBUSY);
 				    nfs_buf_drop(bp);
 				    lck_mtx_unlock(nfs_buf_mutex);
+				    nfs_data_unlock2(np, 0);
 				    if (!nofreeupl)
-					ubc_upl_abort(pl, 0); 
+					ubc_upl_abort(pl, 0);
 				    return (EBUSY);
 				}
 				if ((bp->nb_dirtyoff < start) ||
@@ -4960,34 +5975,39 @@ nfs_pageout(ap)
 			nfs_buf_remfree(bp);
 			lck_mtx_unlock(nfs_buf_mutex);
 			SET(bp->nb_flags, NB_INVAL);
+			nfs_lock(np, NFS_NODE_LOCK_FORCE);
 			if (ISSET(bp->nb_flags, NB_NEEDCOMMIT)) {
 				CLR(bp->nb_flags, NB_NEEDCOMMIT);
 				np->n_needcommitcnt--;
 				CHECK_NEEDCOMMITCNT(np);
 			}
+			nfs_unlock(np);
 			nfs_buf_release(bp, 1);
 		} else {
 			lck_mtx_unlock(nfs_buf_mutex);
 		}
 	}
 
+	thd = vfs_context_thread(ap->a_context);
 	cred = ubc_getcred(vp);
 	if (!IS_VALID_CRED(cred))
 		cred = vfs_context_ucred(ap->a_context);
-	p = vfs_context_proc(ap->a_context);
 
+	nfs_lock(np, NFS_NODE_LOCK_FORCE);
 	if (np->n_flag & NWRITEERR) {
-		np->n_flag &= ~NWRITEERR;
+		error = np->n_error;
+		nfs_unlock(np);
+		nfs_data_unlock2(np, 0);
 		if (!nofreeupl)
 			ubc_upl_abort_range(pl, pl_offset, size,
 					    UPL_ABORT_FREE_ON_EMPTY);
-		return (np->n_error);
+		return (error);
 	}
-	if ((nmp->nm_flag & NFSMNT_NFSV3) && !(nmp->nm_state & NFSSTA_GOTFSINFO))
-		nfs_fsinfo(nmp, vp, cred, p);
+	nfs_unlock(np);
 
 	if (f_offset < 0 || f_offset >= (off_t)np->n_size ||
 	    f_offset & PAGE_MASK_64 || size & PAGE_MASK_64) {
+		nfs_data_unlock2(np, 0);
 		if (!nofreeupl)
 			ubc_upl_abort_range(pl, pl_offset, size,
 					    UPL_ABORT_FREE_ON_EMPTY);
@@ -5003,14 +6023,11 @@ nfs_pageout(ap)
 		xsize = size;
 
 	pgsize = round_page_64(xsize);
-	if (size > pgsize) {
-		if (!nofreeupl)
-			ubc_upl_abort_range(pl, pl_offset + pgsize,
-					    size - pgsize,
-					    UPL_ABORT_FREE_ON_EMPTY);
-	}
+	if ((size > pgsize) && !nofreeupl)
+		ubc_upl_abort_range(pl, pl_offset + pgsize, size - pgsize,
+				    UPL_ABORT_FREE_ON_EMPTY);
 
-	/* 
+	/*
 	 * check for partial page and clear the
 	 * contents past end of the file before
 	 * releasing it in the VM page cache
@@ -5020,59 +6037,145 @@ nfs_pageout(ap)
 		bzero((caddr_t)(ioaddr + io), size - io);
 		FSDBG(321, np->n_size, f_offset, f_offset + io, size - io);
 	}
+	nfs_data_unlock2(np, 0);
 
-	auio.uio_offset = f_offset;
 #if 1   /* LP64todo - can't use new segment flags until the drivers are ready */
 	auio.uio_segflg = UIO_SYSSPACE;
 #else
 	auio.uio_segflg = UIO_SYSSPACE32;
-#endif 
-	auio.uio_rw = UIO_READ;
-	auio.uio_procp = p;
+#endif
+	auio.uio_rw = UIO_WRITE;
+	auio.uio_procp = vfs_context_proc(ap->a_context);
 
+tryagain:
+	wverf = wverf2 = wverfset = 0;
+	txsize = rxsize = xsize;
+	txoffset = rxoffset = f_offset;
+	txaddr = rxaddr = ioaddr;
+	commit = NFS_WRITE_FILESYNC;
+
+	bzero(req, sizeof(req));
+	nextsend = nextwait = 0;
 	do {
-		/*
-		 * It would be nice to be able to issue all these requests
-		 * in parallel instead of waiting for each one to complete
-		 * before sending the next one.
-		 * XXX Should we align these requests to block boundaries?
-		 */
-		iosize = min(biosize, xsize);
-		uio_uio_resid_set(&auio, iosize);
-		aiov.iov_len = iosize;
-		aiov.iov_base = (uintptr_t)ioaddr;
-		auio.uio_iovs.iov32p = &aiov;
-		auio.uio_iovcnt = 1;
-
-		FSDBG(323, auio.uio_offset, uio_uio_resid(&auio), ioaddr, xsize);
-		OSAddAtomic(1, (SInt32*)&nfsstats.pageouts);
-
-		vnode_startwrite(vp);
-
-		/* NMODIFIED would be set here if doing unstable writes */
-		iomode = NFSV3WRITE_FILESYNC;
-		error = nfs_writerpc(vp, &auio, cred, p, &iomode, NULL);
-		vnode_writedone(vp);
-		if (error)
-			goto cleanup;
-		/* Note: no need to check uio_resid, because */
-		/* it'll only be set if there was an error. */
-		ioaddr += iosize;
-		xsize -= iosize;
-	} while (xsize > 0);
-
-cleanup:
+		/* send requests while we need to and have available slots */
+		while ((txsize > 0) && (req[nextsend] == NULL)) {
+			iosize = MIN(nmwsize, txsize);
+			aiov.iov_len = iosize;
+			aiov.iov_base = (uintptr_t)txaddr;
+			auio.uio_iovs.iov32p = &aiov;
+			auio.uio_iovcnt = 1;
+			auio.uio_offset = txoffset;
+			uio_uio_resid_set(&auio, iosize);
+			FSDBG(323, auio.uio_offset, iosize, txaddr, txsize);
+			OSAddAtomic(1, (SInt32*)&nfsstats.pageouts);
+			vnode_startwrite(vp);
+			iomode = NFS_WRITE_UNSTABLE;
+			if ((error = nmp->nm_funcs->nf_write_rpc_async(np, &auio, iosize, thd, cred, iomode, NULL, &req[nextsend]))) {
+				req[nextsend] = NULL;
+				vnode_writedone(vp);
+				break;
+			}
+			txaddr += iosize;
+			txoffset += iosize;
+			txsize -= iosize;
+			nextsend = (nextsend + 1) % MAXPAGINGREQS;
+		}
+		/* wait while we need to and break out if more requests to send */
+		while ((rxsize > 0) && req[nextwait]) {
+			iosize = remsize = MIN(nmwsize, rxsize);
+			error = nmp->nm_funcs->nf_write_rpc_async_finish(np, req[nextwait], &iomode, &iosize, &wverf2);
+			req[nextwait] = NULL;
+			nextwait = (nextwait + 1) % MAXPAGINGREQS;
+			vnode_writedone(vp);
+			if (error) {
+				FSDBG(323, rxoffset, rxsize, error, -1);
+				break;
+			}
+			if (!wverfset) {
+				wverf = wverf2;
+				wverfset = 1;
+			} else if (wverf != wverf2) {
+				/* verifier changed, so we need to restart all the writes */
+				restart++;
+				goto cancel;
+			}
+			/* Retain the lowest commitment level returned. */
+			if (iomode < commit)
+				commit = iomode;
+			rxaddr += iosize;	
+			rxoffset += iosize;	
+			rxsize -= iosize;
+			remsize -= iosize;
+			if (remsize > 0) {
+				/* need to try sending the remainder */
+				iosize = remsize;
+				aiov.iov_len = remsize;
+				aiov.iov_base = (uintptr_t)rxaddr;
+				auio.uio_iovs.iov32p = &aiov;
+				auio.uio_iovcnt = 1;
+				auio.uio_offset = rxoffset;
+				uio_uio_resid_set(&auio, remsize);
+				iomode = NFS_WRITE_UNSTABLE;
+				error = nfs_write_rpc2(np, &auio, thd, cred, &iomode, &wverf2);
+				if (error) {
+					FSDBG(323, rxoffset, rxsize, error, -1);
+					break;
+				}
+				if (wverf != wverf2) {
+					/* verifier changed, so we need to restart all the writes */
+					restart++;
+					goto cancel;
+				}
+				if (iomode < commit)
+					commit = iomode;
+				rxaddr += iosize;	
+				rxoffset += iosize;	
+				rxsize -= iosize;
+			}
+			if (txsize)
+				break;
+		}
+	} while (!error && (txsize || rxsize));
+
+	restart = 0;
+
+	if (!error && (commit != NFS_WRITE_FILESYNC)) {
+		error = nmp->nm_funcs->nf_commit_rpc(np, f_offset, xsize, cred);
+		if (error == NFSERR_STALEWRITEVERF) {
+			restart++;
+			error = EIO;
+		}
+	}
+
+	if (error) {
+cancel:
+		/* cancel any outstanding requests */
+		while (req[nextwait]) {
+			nfs_request_async_cancel(req[nextwait]);
+			req[nextwait] = NULL;
+			nextwait = (nextwait + 1) % MAXPAGINGREQS;
+			vnode_writedone(vp);
+		}
+		if (restart) {
+			if (restart <= 10)
+				goto tryagain;
+			printf("nfs_pageout: too many restarts, aborting.\n");
+			FSDBG(323, f_offset, xsize, ERESTART, -1);
+		}
+	}
+
 	ubc_upl_unmap(pl);
+
 	/*
 	 * We've had several different solutions on what to do when the pageout
-	 * gets an error. If we don't handle it, and return an error to the 
-	 * caller, vm, it will retry . This can end in endless looping 
+	 * gets an error. If we don't handle it, and return an error to the
+	 * caller, vm, it will retry . This can end in endless looping
 	 * between vm and here doing retries of the same page. Doing a dump
 	 * back to vm, will get it out of vm's knowledge and we lose whatever
 	 * data existed. This is risky, but in some cases necessary. For
 	 * example, the initial fix here was to do that for ESTALE. In that case
-	 * the server is telling us that the file is no longer the same. We 
-	 * would not want to keep paging out to that. We also saw some 151 
+	 * the server is telling us that the file is no longer the same. We
+	 * would not want to keep paging out to that. We also saw some 151
 	 * errors from Auspex server and NFSv3 can return errors higher than
 	 * ELAST. Those along with NFS known server errors we will "dump" from
 	 * vm.  Errors we don't expect to occur, we dump and log for further
@@ -5086,11 +6189,10 @@ cleanup:
 	 * have some kind of persistant store when the vm cannot dump nor keep
 	 * retrying as a solution, but this would be a file architectural change
 	 */
-	  
 	if (!nofreeupl) { /* otherwise stacked file system has to handle this */
 		if (error) {
-			int abortflags = 0; 
-			short action = nfs_pageouterrorhandler(error);
+			int abortflags = 0;
+			char action = nfs_pageouterrorhandler(error);
 			
 			switch (action) {
 				case DUMP:
@@ -5098,10 +6200,11 @@ cleanup:
 					break;
 				case DUMPANDLOG:
 					abortflags = UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY;
-					if (error <= ELAST &&
-					    (errorcount[error] % 100 == 0)) 
-						printf("nfs_pageout: unexpected error %d. dumping vm page\n", error);
-					errorcount[error]++;
+					if (error <= NFS_ELAST) {
+						if ((errorcount[error] % 100) == 0)
+							printf("nfs_pageout: unexpected error %d. dumping vm page\n", error);
+						errorcount[error]++;
+					}
 					break;
 				case RETRY:
 					abortflags = UPL_ABORT_FREE_ON_EMPTY;
@@ -5116,31 +6219,32 @@ cleanup:
 					printf("nfs_pageout: action %d not expected\n", action);
 					break;
 			}
-				
-			ubc_upl_abort_range(pl, pl_offset, size, abortflags);
+
+			ubc_upl_abort_range(pl, pl_offset, pgsize, abortflags);
 			/* return error in all cases above */
-			
-		} else 
+
+		} else {
 			ubc_upl_commit_range(pl, pl_offset, pgsize,
 					     UPL_COMMIT_CLEAR_DIRTY |
 					     UPL_COMMIT_FREE_ON_EMPTY);
+		}
 	}
 	return (error);
 }
 
 /* Blktooff derives file offset given a logical block number */
 static int
-nfs_blktooff(ap)
+nfs_vnop_blktooff(
 	struct vnop_blktooff_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		daddr64_t a_lblkno;
 		off_t *a_offset;
-	} */ *ap;
+	} */ *ap)
 {
 	int biosize;
 	vnode_t vp = ap->a_vp;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
+	struct nfsmount *nmp = VTONMP(vp);
 
 	if (!nmp)
 		return (ENXIO);
@@ -5152,17 +6256,17 @@ nfs_blktooff(ap)
 }
 
 static int
-nfs_offtoblk(ap)
+nfs_vnop_offtoblk(
 	struct vnop_offtoblk_args /* {
 		struct vnodeop_desc *a_desc;
 		vnode_t a_vp;
 		off_t a_offset;
 		daddr64_t *a_lblkno;
-	} */ *ap;
+	} */ *ap)
 {
 	int biosize;
 	vnode_t vp = ap->a_vp;
-	struct nfsmount *nmp = VFSTONFS(vnode_mount(vp));
+	struct nfsmount *nmp = VTONMP(vp);
 
 	if (!nmp)
 		return (ENXIO);
diff --git a/bsd/nfs/nfsdiskless.h b/bsd/nfs/nfsdiskless.h
index c5292026d..fd33d5e20 100644
--- a/bsd/nfs/nfsdiskless.h
+++ b/bsd/nfs/nfsdiskless.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/nfs/nfsm_subs.h b/bsd/nfs/nfsm_subs.h
index 1e8de1dd5..3d6d36b65 100644
--- a/bsd/nfs/nfsm_subs.h
+++ b/bsd/nfs/nfsm_subs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -66,540 +72,604 @@
 #include <sys/appleapiopts.h>
 
 #ifdef __APPLE_API_PRIVATE
+
+int nfsm_rpchead(struct nfsreq *, int, mbuf_t, u_int64_t *, mbuf_t *);
+int nfsm_rpchead2(int, int, int, int, int, int, kauth_cred_t, struct nfsreq *, mbuf_t, u_int64_t *, mbuf_t *);
+
+int nfsm_chain_new_mbuf(struct nfsm_chain *, size_t);
+int nfsm_chain_add_opaque_f(struct nfsm_chain *, const u_char *, uint32_t);
+int nfsm_chain_add_opaque_nopad_f(struct nfsm_chain *, const u_char *, uint32_t);
+int nfsm_chain_add_uio(struct nfsm_chain *, struct uio *, uint32_t);
+int nfsm_chain_add_fattr4_f(struct nfsm_chain *, struct vnode_attr *, struct nfsmount *);
+int nfsm_chain_add_v2sattr_f(struct nfsm_chain *, struct vnode_attr *, uint32_t);
+int nfsm_chain_add_v3sattr_f(struct nfsm_chain *, struct vnode_attr *);
+
+int nfsm_chain_advance(struct nfsm_chain *, uint32_t);
+int nfsm_chain_offset(struct nfsm_chain *);
+int nfsm_chain_reverse(struct nfsm_chain *, uint32_t);
+int nfsm_chain_get_opaque_pointer_f(struct nfsm_chain *, uint32_t, u_char **);
+int nfsm_chain_get_opaque_f(struct nfsm_chain *, uint32_t, u_char *);
+int nfsm_chain_get_uio(struct nfsm_chain *, uint32_t, struct uio *);
+int nfsm_chain_get_fh_attr(struct nfsm_chain *, nfsnode_t,
+	vfs_context_t, int, uint64_t *, fhandle_t *, struct nfs_vattr *);
+int nfsm_chain_get_wcc_data_f(struct nfsm_chain *, nfsnode_t, struct timespec *, int *, u_int64_t *);
+
+#if NFSSERVER
+void nfsm_adj(mbuf_t, int, int);
+int nfsm_mbuf_get_list(size_t, mbuf_t *, int *);
+
+int nfsm_chain_add_fattr(struct nfsrv_descript *, struct nfsm_chain *, struct vnode_attr *);
+int nfsm_chain_add_wcc_data_f(struct nfsrv_descript *, struct nfsm_chain *, int,
+	struct vnode_attr *, int, struct vnode_attr *);
+int nfsm_chain_get_path_namei(struct nfsm_chain *, uint32_t, struct nameidata *);
+int nfsm_chain_get_sattr(struct nfsrv_descript *, struct nfsm_chain *, struct vnode_attr *);
+int nfsm_chain_trim_data(struct nfsm_chain *, int, int *);
+#endif /* NFSSERVER */
+
+/* check name length */
+#define nfsm_name_len_check(E, ND, LEN) \
+	do { \
+		if (E) break; \
+		if (((ND)->nd_vers == NFS_VER2) && ((LEN) > NFS_MAXNAMLEN)) \
+			(E) = NFSERR_NAMETOL; \
+		if ((LEN) <= 0) \
+			error = EBADRPC; \
+	} while (0)
+
+#define nfsm_assert(E, COND, ERR) \
+	do { \
+		if (E) break; \
+		if (!(COND)) \
+			(E) = (ERR); \
+	} while (0)
+
+/* Initialize a vnode_attr to retrieve attributes for the NFS server. */
+#define nfsm_srv_vattr_init(VAP, VERS) \
+	do { \
+		VATTR_INIT(VAP); \
+		VATTR_WANTED((VAP), va_type); \
+		VATTR_WANTED((VAP), va_mode); \
+		VATTR_WANTED((VAP), va_nlink); \
+		VATTR_WANTED((VAP), va_uid); \
+		VATTR_WANTED((VAP), va_gid); \
+		VATTR_WANTED((VAP), va_data_size); \
+		VATTR_WANTED((VAP), va_data_alloc); \
+		VATTR_WANTED((VAP), va_rdev); \
+		VATTR_WANTED((VAP), va_fsid); \
+		VATTR_WANTED((VAP), va_fileid); \
+		VATTR_WANTED((VAP), va_access_time); \
+		VATTR_WANTED((VAP), va_modify_time); \
+		VATTR_WANTED((VAP), va_change_time); \
+		if ((VERS) == NFS_VER2) \
+			VATTR_WANTED((VAP), va_iosize); \
+	} while (0)
+
+/* Initialize a vnode_attr to retrieve pre-operation attributes for the NFS server. */
+#define nfsm_srv_pre_vattr_init(VAP) \
+	do { \
+		VATTR_INIT(VAP); \
+		VATTR_WANTED((VAP), va_data_size); \
+		VATTR_WANTED((VAP), va_modify_time); \
+		VATTR_WANTED((VAP), va_change_time); \
+	} while (0)
+
+/* round up to a multiple of 4 */
+#define nfsm_rndup(a)	(((a)+3)&(~0x3))
+
+#define nfsm_pad(a)	(nfsm_rndup(a) - (a))
+
 /*
- * These macros do strange and peculiar things to mbuf chains for
- * the assistance of the nfs code. To attempt to use them for any
- * other purpose will be dangerous. (they make weird assumptions)
+ * control flow macros:
+ * go to the appropriate label on condition
  */
+#define nfsmout_if(E)	do { if (E) goto nfsmout; } while (0)
+#define nfsmerr_if(E)	do { if (E) goto nfsmerr; } while (0)
 
 /*
- * First define what the actual subs. return
+ * For NFS v2 errors and EBADRPC, the reply contains only the error.
+ * This macro is used to skip any reply building code and go straight
+ * to nfsmout instead.
  */
-int nfsm_reqh(int hsiz, caddr_t *bposp, mbuf_t *mbp);
-int nfsm_rpchead(struct ucred *cr, int nmflag, int procid,
-			       int auth_type, int auth_len, char *auth_str,
-			       int verf_len, char *verf_str,
-			       mbuf_t mrest, int mrest_len,
-			       mbuf_t *mbp, u_long *xidp, mbuf_t *mreqp);
+#define nfsmout_on_status(ND, E) \
+	do { \
+		if (((ND)->nd_repstat == EBADRPC) || \
+		    ((ND)->nd_repstat && ((ND)->nd_vers == NFS_VER2))) { \
+			(E) = 0; \
+			goto nfsmout; \
+		} \
+	} while (0)
+
+/* initialize an mbuf chain */
+#define nfsm_chain_null(NMC) \
+	do { \
+		(NMC)->nmc_mhead = (NMC)->nmc_mcur = NULL; \
+		(NMC)->nmc_ptr = NULL; \
+	} while (0)
+
+/* cleanup an mbuf chain */
+#define nfsm_chain_cleanup(NMC) \
+	do { \
+		if (!(NMC)->nmc_mhead) break; \
+		mbuf_freem((NMC)->nmc_mhead); \
+		nfsm_chain_null(NMC); \
+	} while (0)
+
+/* get an mbuf given a size hint */
+#define nfsm_mbuf_get(E, MBP, SIZEHINT) \
+	do { \
+		*(MBP) = NULL; \
+		if ((size_t)(SIZEHINT) >= nfs_mbuf_minclsize) \
+			(E) = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, (MBP)); \
+		else \
+			(E) = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, (MBP)); \
+	} while (0)
+
 
 /*
- * Now for the macros that do the simple stuff and call the functions
- * for the hard stuff.
- * These macros use several vars. declared in nfsm_reqhead and these
- * vars. must not be used elsewhere unless you are careful not to corrupt
- * them. The vars. starting with pN and tN (N=1,2,3,..) are temporaries
- * that may be used so long as the value is not expected to retained
- * after a macro.
- * I know, this is kind of dorkey, but it makes the actual op functions
- * fairly clean and deals with the mess caused by the xdr discriminating
- * unions.
+ * macros for building NFS mbuf chains
  */
 
-#define	nfsm_build(a,c,s) \
-		{ if ((s) > mbuf_trailingspace(mb)) { \
-			int __nfsm_error; \
-			__nfsm_error = mbuf_get(MBUF_WAITOK, MBUF_TYPE_DATA, &mb2); \
-			if (__nfsm_error) \
-				panic("nfsm_build mbuf_get error %d", __nfsm_error); \
-			if ((s) > mbuf_maxlen(mb2)) \
-				panic("nfsm_build size error"); \
-			__nfsm_error = mbuf_setnext(mb, mb2); \
-			if (__nfsm_error) \
-				panic("nfsm_build mbuf_setnext error %d", __nfsm_error); \
-			mb = mb2; \
-			bpos = mbuf_data(mb); \
-		} \
-		(a) = (c)(bpos); \
-		mbuf_setlen(mb, (mbuf_len(mb) + (s))); \
-		bpos += (s); }
-
-#define	nfsm_dissect(a, c, s) \
-		{ t1 = ((caddr_t)mbuf_data(md)) + mbuf_len(md) - dpos; \
-		if (t1 >= (s)) { \
-			(a) = (c)(dpos); \
-			dpos += (s); \
-		} else if ((t1 = nfsm_disct(&md, &dpos, (s), t1, &cp2))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
-		} else { \
-			(a) = (c)cp2; \
-		} }
-
-#define nfsm_fhtom(v, v3) \
-	      { if (v3) { \
-			t2 = nfsm_rndup(VTONFS(v)->n_fhsize) + NFSX_UNSIGNED; \
-			if (t2 <= mbuf_trailingspace(mb)) { \
-				nfsm_build(tl, u_long *, t2); \
-				*tl++ = txdr_unsigned(VTONFS(v)->n_fhsize); \
-				*(tl + ((t2>>2) - 2)) = 0; \
-				bcopy((caddr_t)VTONFS(v)->n_fhp,(caddr_t)tl, \
-					VTONFS(v)->n_fhsize); \
-			} else if ((t2 = nfsm_strtmbuf(&mb, &bpos, \
-				(caddr_t)VTONFS(v)->n_fhp, VTONFS(v)->n_fhsize))) { \
-				error = t2; \
-				mbuf_freem(mreq); \
-				goto nfsmout; \
-			} \
-		} else { \
-			nfsm_build(cp, caddr_t, NFSX_V2FH); \
-			bcopy((caddr_t)VTONFS(v)->n_fhp, cp, NFSX_V2FH); \
-		} }
-
-#define nfsm_srvfhtom(f, v3) \
-		{ if (v3) { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED + (unsigned)(f)->nfh_len); \
-			*tl++ = txdr_unsigned((f)->nfh_len); \
-			bcopy((caddr_t)&(f)->nfh_xh, (caddr_t)tl, (f)->nfh_len); \
-		} else { \
-			nfsm_build(cp, caddr_t, NFSX_V2FH); \
-			bcopy((caddr_t)&(f)->nfh_xh, cp, NFSX_V2FH); \
-		} }
-
-#define nfsm_srvpostop_fh(f) \
-		{ nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED + (unsigned)(f)->nfh_len); \
-		*tl++ = nfs_true; \
-		*tl++ = txdr_unsigned((f)->nfh_len); \
-		bcopy((caddr_t)&(f)->nfh_xh, (caddr_t)tl, (f)->nfh_len); \
-		}
-
-#define nfsm_mtofh(d, cnp, v, v3, xp, f) \
-		{ \
-		struct nfsnode *ttnp; u_char *ttfhp = NULL; \
-		int ttfhsize = 0, ttgotfh = 1, ttgotattr = 1, ttgotnode = 0; \
-		struct nfs_vattr ttvattr; \
-		(v) = NULL; \
-		/* XXX would be nice to not bail to nfsmout on error */ \
-		if (v3) { /* check for file handle */ \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			ttgotfh = fxdr_unsigned(int, *tl); \
-		} \
-		if (ttgotfh) { \
-			/* get file handle */ \
-			nfsm_getfh(ttfhp, ttfhsize, (v3)); \
-		} \
-		if (v3) { /* check for attributes */ \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			ttgotattr = fxdr_unsigned(int, *tl); \
+/* prepare an mbuf chain for building starting with the given mbuf */
+#define nfsm_chain_init(NMC, MB) \
+	do { \
+		(NMC)->nmc_mhead = (MB); \
+		(NMC)->nmc_mcur = (NMC)->nmc_mhead; \
+		(NMC)->nmc_ptr = mbuf_data((NMC)->nmc_mcur); \
+		(NMC)->nmc_left = mbuf_trailingspace((NMC)->nmc_mcur); \
+		(NMC)->nmc_flags = 0; \
+	} while (0)
+
+/* prepare an mbuf chain for building starting with a newly allocated mbuf */
+#define nfsm_chain_build_alloc_init(E, NMC, SIZEHINT) \
+	do { \
+		mbuf_t ncbimb; \
+		nfsm_mbuf_get((E), &ncbimb, (SIZEHINT)); \
+		if (E) break; \
+		nfsm_chain_init((NMC), ncbimb); \
+	} while (0)
+
+/* done building an mbuf chain */
+#define nfsm_chain_build_done(E, NMC) \
+	do { \
+		if ((E) || !(NMC)->nmc_mcur) break; \
+		/* cap off current mbuf */ \
+		mbuf_setlen((NMC)->nmc_mcur, \
+			(NMC)->nmc_ptr - (caddr_t)mbuf_data((NMC)->nmc_mcur)); \
+	} while (0)
+
+/*
+ *  set the TCP record mark at the head of an mbuf chain -
+ *  assumes 4 bytes are already allocated in the head mbuf
+ */
+#define nfsm_chain_set_recmark(E, NMC, VAL) \
+	do { \
+		if (E) break; \
+		*((uint32_t*)mbuf_data((NMC)->nmc_mhead)) \
+			= txdr_unsigned(VAL); \
+	} while (0)
+
+/* make sure there's room for size bytes in current mbuf */
+#define nfsm_chain_check_size(E, NMC, SIZE) \
+	do { \
+		if (E) break; \
+		if ((NMC)->nmc_left < (SIZE)) { \
+			(E) = nfsm_chain_new_mbuf((NMC), (SIZE)); \
+			if (!(E) && ((NMC)->nmc_left < (SIZE))) \
+				(E) = ENOMEM; \
 		} \
-		/* get attributes */ \
-		if (ttgotattr) { \
-			if (!ttgotfh) { \
-				nfsm_adv(NFSX_V3FATTR); \
-			} else { \
-				nfsm_attr_get(v3, &ttvattr); \
-			} \
-		} else if (ttgotfh) { \
-			/* We need valid attributes in order */ \
-			/* to call nfs_nget/vnode_create().  */ \
-			t1 = nfs_getattr_no_vnode(vnode_mount(d), \
-				ttfhp, ttfhsize, cred, p, &ttvattr, xp); \
-			if (t1) \
-				ttgotattr = 0; \
+	} while (0)
+
+/* add a 32bit value to an mbuf chain extending if necessary */
+#define nfsm_chain_add_32(E, NMC, VAL) \
+	do { \
+		nfsm_chain_check_size((E), (NMC), NFSX_UNSIGNED); \
+		if (E) break; \
+		*((uint32_t*)(NMC)->nmc_ptr) = txdr_unsigned(VAL); \
+		(NMC)->nmc_ptr += NFSX_UNSIGNED; \
+		(NMC)->nmc_left -= NFSX_UNSIGNED; \
+	} while (0)
+
+/* add a 64bit value to an mbuf chain */
+#define nfsm_chain_add_64(E, NMC, VAL) \
+	do { \
+		uint64_t __tmp64; \
+		nfsm_chain_check_size((E), (NMC), 2 * NFSX_UNSIGNED); \
+		if (E) break; \
+		__tmp64 = (VAL); \
+		txdr_hyper(&__tmp64, (NMC)->nmc_ptr); \
+		(NMC)->nmc_ptr += 2 * NFSX_UNSIGNED; \
+		(NMC)->nmc_left -= 2 * NFSX_UNSIGNED; \
+	} while (0)
+
+/* zero the last 4 bytes for a range of opaque */
+/* data to make sure any pad bytes will be zero. */
+#define nfsm_chain_zero_opaque_pad(BUF, LEN) \
+	*(((uint32_t*)(BUF))+((nfsm_rndup(LEN)>>2)-1)) = 0
+
+/* add buffer of opaque data to an mbuf chain */
+#define nfsm_chain_add_opaque(E, NMC, BUF, LEN) \
+	do { \
+		uint32_t rndlen = nfsm_rndup(LEN); \
+		if (E) break; \
+		if ((NMC)->nmc_left < rndlen) { \
+			(E) = nfsm_chain_add_opaque_f((NMC), (const u_char*)(BUF), (LEN)); \
+			break; \
 		} \
-		if (ttgotfh && ttgotattr) { \
-			int ttngflags = NG_MAKEENTRY; \
-			if ((t1 = nfs_nget(vnode_mount(d), d, cnp, ttfhp, ttfhsize, \
-					&ttvattr, xp, ttngflags, &ttnp))) { \
-				error = t1; \
-				ttgotnode = 0; \
-			} else { \
-				ttgotnode = 1; \
-				(v) = NFSTOV(ttnp); \
-			} \
+		nfsm_chain_zero_opaque_pad((NMC)->nmc_ptr, (LEN)); \
+		bcopy((BUF), (NMC)->nmc_ptr, (LEN)); \
+		(NMC)->nmc_ptr += rndlen; \
+		(NMC)->nmc_left -= rndlen; \
+	} while (0)
+
+/* add buffer of opaque data to an mbuf chain without padding */
+#define nfsm_chain_add_opaque_nopad(E, NMC, BUF, LEN) \
+	do { \
+		if (E) break; \
+		if ((NMC)->nmc_left < (uint32_t) (LEN)) { \
+			(E) = nfsm_chain_add_opaque_nopad_f((NMC), (const u_char*)(BUF), (LEN)); \
+			break; \
 		} \
-		(f) = ttgotnode;  \
-		}
-
-#define nfsm_getfh(f, s, v3) \
-		{ if (v3) { \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			if (((s) = fxdr_unsigned(int, *tl)) <= 0 || \
-				(s) > NFSX_V3FHMAX) { \
-				mbuf_freem(mrep); \
-				error = EBADRPC; \
-				goto nfsmout; \
-			} \
+		bcopy((BUF), (NMC)->nmc_ptr, (LEN)); \
+		(NMC)->nmc_ptr += (LEN); \
+		(NMC)->nmc_left -= (LEN); \
+	} while (0)
+
+/* finish an mbuf in a chain to allow subsequent insertion */
+#define nfsm_chain_finish_mbuf(E, NMC) \
+	do { \
+		if (E) break; \
+		mbuf_setlen((NMC)->nmc_mcur, \
+			(NMC)->nmc_ptr - (caddr_t)mbuf_data((NMC)->nmc_mcur)); \
+		(NMC)->nmc_left = 0; \
+	} while (0)
+
+/* add a file handle to an mbuf chain */
+#define nfsm_chain_add_fh(E, NMC, VERS, FHP, FHLEN) \
+	do { \
+		if (E) break; \
+		if ((VERS) != NFS_VER2) \
+			nfsm_chain_add_32((E), (NMC), (FHLEN)); \
+		nfsm_chain_add_opaque((E), (NMC), (FHP), (FHLEN)); \
+	} while (0)
+
+/* add a string to an mbuf chain */
+#define nfsm_chain_add_string(E, NMC, STR, LEN) \
+	do { \
+		nfsm_chain_add_32((E), (NMC), (LEN)); \
+		nfsm_chain_add_opaque((E), (NMC), (STR), (LEN)); \
+	} while (0)
+
+/* add an NFSv2 time to an mbuf chain */
+#define nfsm_chain_add_v2time(E, NMC, TVP) \
+	do { \
+		if (TVP) { \
+			nfsm_chain_add_32((E), (NMC), (TVP)->tv_sec); \
+			nfsm_chain_add_32((E), (NMC), ((TVP)->tv_nsec != -1) ? \
+				((uint32_t)(TVP)->tv_nsec / 1000) : 0xffffffff); \
 		} else { \
-			(s) = NFSX_V2FH; \
-		} \
-		nfsm_dissect((f), u_char *, nfsm_rndup(s)); }
-
-#define	nfsm_loadattr(v, v3, a, x) \
-		{ struct nfs_vattr ttvattr; \
-		if ((t1 = nfs_parsefattr(&md, &dpos, v3, &ttvattr))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
+			/* no time... use -1 */ \
+			nfsm_chain_add_32((E), (NMC), -1); \
+			nfsm_chain_add_32((E), (NMC), -1); \
 		} \
-		if ((t1 = nfs_loadattrcache(VTONFS(v), &ttvattr, (x), 0))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
+	} while (0)
+
+/* add an NFSv3 time to an mbuf chain */
+#define nfsm_chain_add_v3time(E, NMC, TVP) \
+	do { \
+		nfsm_chain_add_32((E), (NMC), (TVP)->tv_sec); \
+		nfsm_chain_add_32((E), (NMC), (TVP)->tv_nsec); \
+	} while (0)
+
+/* add an NFS v2 or v3 time to an mbuf chain */
+#define nfsm_chain_add_time(E, NMC, VERS, TVP) \
+	do { \
+		if ((VERS) == NFS_VER2) { \
+			nfsm_chain_add_v2time((E), (NMC), (TVP)); \
+		} else { \
+			nfsm_chain_add_v3time((E), (NMC), (TVP)); \
 		} \
-		if (a) { \
-			bcopy(&ttvattr, (a), sizeof(ttvattr)); \
+	} while (0)
+
+/* add an NFSv3 postop file handle to an mbuf chain */
+#define nfsm_chain_add_postop_fh(E, NMC, FHP, FHLEN) \
+	do { \
+		nfsm_chain_add_32((E), (NMC), TRUE); \
+		nfsm_chain_add_fh((E), (NMC), NFS_VER3, (FHP), (FHLEN)); \
+	} while (0)
+
+/* add NFSv3 postop attributes to an mbuf chain */
+#define nfsm_chain_add_postop_attr(E, ND, NMC, ATTRERR, VAP) \
+	do { \
+		if (E) break; \
+		if (ATTRERR) { \
+			nfsm_chain_add_32((E), (NMC), FALSE); \
+			break; \
 		} \
-		}
+		nfsm_chain_add_32((E), (NMC), TRUE); \
+		if (E) break; \
+		(E) = nfsm_chain_add_fattr((ND), (NMC), (VAP)); \
+	} while (0)
+
+/* Add an NFSv2 "sattr" structure to an mbuf chain */
+#define nfsm_chain_add_v2sattr(E, NMC, VAP, SZRDEV) \
+	do { \
+		if (E) break; \
+		(E) = nfsm_chain_add_v2sattr_f((NMC), (VAP), (SZRDEV)); \
+	} while (0)
+
+/* Add an NFSv3 "sattr" structure to an mbuf chain */
+#define nfsm_chain_add_v3sattr(E, NMC, VAP) \
+	do { \
+		if (E) break; \
+		(E) = nfsm_chain_add_v3sattr_f((NMC), (VAP)); \
+	} while (0)
+
+/* Add an NFSv4 "fattr" structure to an mbuf chain */
+#define nfsm_chain_add_fattr4(E, NMC, VAP, NMP) \
+	do { \
+		if (E) break; \
+		(E) = nfsm_chain_add_fattr4_f((NMC), (VAP), (NMP)); \
+	} while (0)
+
+/* add NFSv3 WCC data to an mbuf chain */
+#define nfsm_chain_add_wcc_data(E, ND, NMC, PREERR, PREVAP, POSTERR, POSTVAP) \
+	do { \
+		if (E) break; \
+		(E) = nfsm_chain_add_wcc_data_f((ND), (NMC), \
+			(PREERR), (PREVAP), (POSTERR), (POSTVAP)); \
+	} while (0)
+
+/* add NFSv4 COMPOUND header */
+#define nfsm_chain_add_compound_header(E, NMC, TAG, NUMOPS) \
+	do { \
+		if ((TAG) && strlen(TAG)) \
+			nfsm_chain_add_string((E), (NMC), (TAG), strlen(TAG)); \
+		else \
+			nfsm_chain_add_32((E), (NMC), 0); \
+		nfsm_chain_add_32((E), (NMC), 0); /*minorversion*/ \
+		nfsm_chain_add_32((E), (NMC), (NUMOPS)); \
+	} while (0)
+
+/* add NFSv4 attr bitmap */
+#define nfsm_chain_add_bitmap(E, NMC, B, LEN) \
+	do { \
+		int __i; \
+		nfsm_chain_add_32((E), (NMC), (LEN)); \
+		for (__i=0; __i < (LEN); __i++) \
+			nfsm_chain_add_32((E), (NMC), (B)[__i]); \
+	} while (0)
+
+/* add NFSv4 attr bitmap masked with the given mask */
+#define nfsm_chain_add_bitmap_masked(E, NMC, B, LEN, MASK) \
+	do { \
+		int __i; \
+		nfsm_chain_add_32((E), (NMC), (LEN)); \
+		for (__i=0; __i < (LEN); __i++) \
+			nfsm_chain_add_32((E), (NMC), ((B)[__i] & (MASK)[__i])); \
+	} while (0)
 
-#define	nfsm_attr_get(v3, vap) \
-		{ \
-		if ((t1 = nfs_parsefattr(&md, &dpos, v3, vap))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
-		} \
-		}
-
-#define	nfsm_postop_attr_get(v3, f, vap) \
-		{ \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (((f) = fxdr_unsigned(int, *tl))) { \
-			if ((t1 = nfs_parsefattr(&md, &dpos, v3, vap))) { \
-				error = t1; \
-				(f) = 0; \
-				mbuf_freem(mrep); \
-				goto nfsmout; \
-			} \
-		} }
-
-#define	nfsm_postop_attr_update(v, v3, f, x) \
-		{ \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (((f) = fxdr_unsigned(int, *tl))) { \
-			struct nfs_vattr ttvattr; \
-			if ((t1 = nfs_parsefattr(&md, &dpos, v3, &ttvattr))) { \
-				error = t1; \
-				(f) = 0; \
-				mbuf_freem(mrep); \
-				goto nfsmout; \
-			} \
-			if ((t1 = nfs_loadattrcache(VTONFS(v), &ttvattr, (x), 1))) { \
-				error = t1; \
-				(f) = 0; \
-				mbuf_freem(mrep); \
-				goto nfsmout; \
-			} \
-			if (*(x) == 0) \
-				(f) = 0; \
-		} }
-
-#define	nfsm_wcc_data(v, premtime, newpostattr, x) \
-		{ \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (*tl == nfs_true) { \
-			nfsm_dissect(tl, u_long *, 6 * NFSX_UNSIGNED); \
-			(premtime)->tv_sec = fxdr_unsigned(time_t, *(tl + 2)); \
-			(premtime)->tv_nsec = fxdr_unsigned(time_t, *(tl + 3)); \
-		} else { \
-			(premtime)->tv_sec = 0; \
-			(premtime)->tv_nsec = 0; \
-		} \
-		nfsm_postop_attr_update((v), 1, (newpostattr), (x)); \
-		}
-
-#define nfsm_v3sattr(vap) \
-		{\
-		struct timeval now; \
-		if (VATTR_IS_ACTIVE(vap, va_mode)) { \
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			*tl++ = nfs_true; \
-			*tl = txdr_unsigned(vap->va_mode); \
-		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = nfs_false; \
+/*
+ * macros for dissecting NFS mbuf chains
+ */
+
+/* prepare an mbuf chain for dissection starting with the given mbuf */
+#define nfsm_chain_dissect_init(E, NMC, H) \
+	do { \
+		if (!(H)) { \
+			(E) = EINVAL; \
+			break; \
 		} \
-		if (VATTR_IS_ACTIVE(vap, va_uid)) { \
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			*tl++ = nfs_true; \
-			*tl = txdr_unsigned(vap->va_uid); \
+		(NMC)->nmc_mcur = (NMC)->nmc_mhead = (H); \
+		(NMC)->nmc_ptr = mbuf_data(H); \
+		(NMC)->nmc_left = mbuf_len(H); \
+	} while (0)
+
+/* skip a number of bytes in an mbuf chain */
+#define nfsm_chain_adv(E, NMC, LEN) \
+	do { \
+		if (E) break; \
+		if ((NMC)->nmc_left >= (uint32_t)(LEN)) { \
+			(NMC)->nmc_left -= (LEN); \
+			(NMC)->nmc_ptr += (LEN); \
 		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = nfs_false; \
+			(E) = nfsm_chain_advance((NMC), (LEN)); \
 		} \
-		if (VATTR_IS_ACTIVE(vap, va_gid)) { \
-			nfsm_build(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			*tl++ = nfs_true; \
-			*tl = txdr_unsigned(vap->va_gid); \
+	} while (0)
+
+/* get a 32bit value from an mbuf chain */
+#define nfsm_chain_get_32(E, NMC, LVAL) \
+	do { \
+		uint32_t __tmp32, *__tmpptr; \
+		if (E) break; \
+		if ((NMC)->nmc_left >= NFSX_UNSIGNED) { \
+			__tmpptr = (uint32_t*)(NMC)->nmc_ptr; \
+			(NMC)->nmc_left -= NFSX_UNSIGNED; \
+			(NMC)->nmc_ptr += NFSX_UNSIGNED; \
 		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = nfs_false; \
+			__tmpptr = &__tmp32; \
+			(E) = nfsm_chain_get_opaque_f((NMC), NFSX_UNSIGNED, (u_char*)__tmpptr); \
+			if (E) break; \
 		} \
-		if (VATTR_IS_ACTIVE(vap, va_data_size)) { \
-			nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED); \
-			*tl++ = nfs_true; \
-			txdr_hyper(&vap->va_data_size, tl); \
+		(LVAL) = fxdr_unsigned(uint32_t, *__tmpptr); \
+	} while (0)
+
+/* get a 64bit value from an mbuf chain */
+#define nfsm_chain_get_64(E, NMC, LVAL) \
+	do { \
+		uint64_t __tmp64, *__tmpptr; \
+		if (E) break; \
+		if ((NMC)->nmc_left >= 2 * NFSX_UNSIGNED) { \
+			__tmpptr = (uint64_t*)(NMC)->nmc_ptr; \
+			(NMC)->nmc_left -= 2 * NFSX_UNSIGNED; \
+			(NMC)->nmc_ptr += 2 * NFSX_UNSIGNED; \
 		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = nfs_false; \
+			__tmpptr = &__tmp64; \
+			(E) = nfsm_chain_get_opaque_f((NMC), 2 * NFSX_UNSIGNED, (u_char*)__tmpptr); \
+			if (E) break; \
 		} \
-		microtime(&now); \
-		if (VATTR_IS_ACTIVE(vap, va_access_time)) { \
-			if (vap->va_access_time.tv_sec != now.tv_sec) { \
-				nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED); \
-				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT); \
-				txdr_nfsv3time(&vap->va_access_time, tl); \
-			} else { \
-				nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-				*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER); \
-			} \
+		fxdr_hyper(__tmpptr, &(LVAL)); \
+	} while (0)
+
+/* get a pointer to the next consecutive bytes in an mbuf chain */
+#define nfsm_chain_get_opaque_pointer(E, NMC, LEN, PTR) \
+	do { \
+		if (E) break; \
+		if ((NMC)->nmc_left >= (uint32_t)(LEN)) { \
+			(PTR) = (void*)(NMC)->nmc_ptr; \
+			(NMC)->nmc_left -= nfsm_rndup(LEN); \
+			(NMC)->nmc_ptr += nfsm_rndup(LEN); \
 		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE); \
+			(E) = nfsm_chain_get_opaque_pointer_f((NMC), (LEN), (u_char**)&(PTR)); \
 		} \
-		if (VATTR_IS_ACTIVE(vap, va_modify_time)) { \
-			if (vap->va_modify_time.tv_sec != now.tv_sec) { \
-				nfsm_build(tl, u_long *, 3 * NFSX_UNSIGNED); \
-				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT); \
-				txdr_nfsv3time(&vap->va_modify_time, tl); \
-			} else { \
-				nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-				*tl = txdr_unsigned(NFSV3SATTRTIME_TOSERVER); \
-			} \
+	} while (0)
+
+/* copy the next consecutive bytes of opaque data from an mbuf chain */
+#define nfsm_chain_get_opaque(E, NMC, LEN, PTR) \
+	do { \
+		if (E) break; \
+		if ((NMC)->nmc_left >= (LEN)) { \
+			u_char *__tmpptr = (u_char*)(NMC)->nmc_ptr; \
+			(NMC)->nmc_left -= nfsm_rndup(LEN); \
+			(NMC)->nmc_ptr += nfsm_rndup(LEN); \
+			bcopy(__tmpptr, (PTR), (LEN)); \
 		} else { \
-			nfsm_build(tl, u_long *, NFSX_UNSIGNED); \
-			*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE); \
+			(E) = nfsm_chain_get_opaque_f((NMC), (LEN), (u_char*)(PTR)); \
 		} \
-		}
-
-#define	nfsm_strsiz(s,m,v3) \
-		{ nfsm_dissect(tl,u_long *,NFSX_UNSIGNED); \
-		(s) = fxdr_unsigned(long,*tl); \
-		if (!(v3) && ((s) > (m))) { \
-			mbuf_freem(mrep); \
-			error = EBADRPC; \
-			goto nfsmout; \
-		} }
-
-#define	nfsm_srvstrsiz(s,m) \
-		{ nfsm_dissect(tl,u_long *,NFSX_UNSIGNED); \
-		if (((s) = fxdr_unsigned(long,*tl)) > (m) || (s) <= 0) { \
-			error = EBADRPC; \
-			nfsm_reply(0); \
-		} }
-
-#define	nfsm_srvnamesiz(s,v3) \
-		{ nfsm_dissect(tl,u_long *,NFSX_UNSIGNED); \
-		(s) = fxdr_unsigned(long,*tl); \
-		if (!(v3) && ((s) > NFS_MAXNAMLEN)) \
-			error = NFSERR_NAMETOL; \
-		if ((s) <= 0) \
-			error = EBADRPC; \
-		if (error) \
-			nfsm_reply(0); \
-		}
-
-#define nfsm_mtouio(p,s) \
-		if ((s) > 0 && \
-		   (t1 = nfsm_mbuftouio(&md,(p),(s),&dpos))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
-		}
-
-#define nfsm_uiotom(p,s) \
-		if ((t1 = nfsm_uiotombuf((p),&mb,(s),&bpos))) { \
-			error = t1; \
-			mbuf_freem(mreq); \
-			goto nfsmout; \
-		}
-
-#define	nfsm_reqhead(s) \
-		error = nfsm_reqh((s), &bpos, &mreq); \
-		mb = mreq;
-
-#define nfsm_reqdone	mbuf_freem(mrep); \
-		nfsmout:
-
-#define nfsm_rndup(a)	(((a)+3)&(~0x3))
-
-#define	nfsm_request(v, t, p, c, x)	\
-		if ((error = nfs_request((v), vnode_mount(v), mreq, (t), (p), \
-		   (c), &mrep, &md, &dpos, (x)))) { \
-			if (error & NFSERR_RETERR) \
-				error &= ~NFSERR_RETERR; \
-			else \
-				goto nfsmout; \
-                }
-
-#define	nfsm_strtom(a,s,m,v3) \
-		if (!(v3) && ((s) > (m))) { \
-			mbuf_freem(mreq); \
-			error = ENAMETOOLONG; \
-			goto nfsmout; \
-		} \
-		t2 = nfsm_rndup(s)+NFSX_UNSIGNED; \
-		if (t2 <= mbuf_trailingspace(mb)) { \
-			nfsm_build(tl,u_long *,t2); \
-			*tl++ = txdr_unsigned(s); \
-			*(tl+((t2>>2)-2)) = 0; \
-			bcopy((caddr_t)(a), (caddr_t)tl, (s)); \
-		} else if ((t2 = nfsm_strtmbuf(&mb, &bpos, (a), (s)))) { \
-			error = t2; \
-			mbuf_freem(mreq); \
-			goto nfsmout; \
-		}
-
-#define	nfsm_srvdone \
-		nfsmout: \
-		return(error)
+	} while (0)
 
-#define	nfsm_reply(s) \
-		{ \
-		nfsd->nd_repstat = error; \
-		if (error && !(nfsd->nd_flag & ND_NFSV3)) \
-		   nfs_rephead(0, nfsd, slp, error, mrq, &mb, &bpos); \
+/* get the size of and a pointer to a file handle in an mbuf chain */
+#define nfsm_chain_get_fh_ptr(E, NMC, VERS, FHP, FHSIZE) \
+	do { \
+		if ((VERS) != NFS_VER2) \
+			nfsm_chain_get_32((E), (NMC), (FHSIZE)); \
 		else \
-		   nfs_rephead((s), nfsd, slp, error, mrq, &mb, &bpos); \
-		mbuf_freem(mrep); \
-		mrep = NULL; \
-		mreq = *mrq; \
-		if (error && (!(nfsd->nd_flag & ND_NFSV3) || \
-			error == EBADRPC)) { \
-			error = 0; \
-			goto nfsmout; \
-		} \
-		}
-
-#define	nfsm_writereply(s, v3) \
-		{ \
-		nfsd->nd_repstat = error; \
-		if (error && !(v3)) \
-		   nfs_rephead(0, nfsd, slp, error, &mreq, &mb, &bpos); \
+			(FHSIZE) = NFSX_V2FH;\
+		nfsm_chain_get_opaque_pointer((E), (NMC), (FHSIZE), (FHP));\
+	} while (0)
+
+/* get the size of and data for a file handle in an mbuf chain */
+#define nfsm_chain_get_fh(E, NMC, VERS, FHP) \
+	do { \
+		if ((VERS) != NFS_VER2) \
+			nfsm_chain_get_32((E), (NMC), (FHP)->fh_len); \
 		else \
-		   nfs_rephead((s), nfsd, slp, error, &mreq, &mb, &bpos); \
-		}
-
-#define	nfsm_adv(s) \
-		{ t1 = ((caddr_t)mbuf_data(md)) + mbuf_len(md) - dpos; \
-		if (t1 >= (s)) { \
-			dpos += (s); \
-		} else if ((t1 = nfs_adv(&md, &dpos, (s), t1))) { \
-			error = t1; \
-			mbuf_freem(mrep); \
-			goto nfsmout; \
-		} }
-
-#define nfsm_srvmtofh(f) \
-		{ \
-		if (nfsd->nd_flag & ND_NFSV3) { \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			(f)->nfh_len = fxdr_unsigned(int, *tl); \
-			if (((f)->nfh_len < (int)sizeof(struct nfs_exphandle)) || \
-			    ((f)->nfh_len > NFSX_V3FHMAX)) { \
-				error = EBADRPC; \
-				nfsm_reply(0); \
-			} \
-		} else { \
-			(f)->nfh_len = NFSX_V2FH; \
-		} \
-		nfsm_dissect(tl, u_long *, (f)->nfh_len); \
-		bcopy((caddr_t)tl, (caddr_t)&(f)->nfh_xh, (f)->nfh_len); \
-		}
-
-#define	nfsm_clget \
-		if (bp >= be) { \
-			int __nfsm_error, __nfsm_len; \
-			if (mp == mb) \
-				mbuf_setlen(mp, mbuf_len(mp) + bp - bpos); \
-			mp = NULL; \
-			__nfsm_error = mbuf_mclget(MBUF_WAITOK, MBUF_TYPE_DATA, &mp); \
-			if (__nfsm_error) \
-				panic("nfsm_clget: mbuf_mclget error %d", __nfsm_error); \
-			__nfsm_len = mbuf_maxlen(mp); \
-			mbuf_setlen(mp, __nfsm_len); \
-			__nfsm_error = mbuf_setnext(mp2, mp); \
-			if (__nfsm_error) \
-				panic("nfsm_clget: mbuf_setnext error %d", __nfsm_error); \
-			mp2 = mp; \
-			bp = mbuf_data(mp); \
-			be = bp + __nfsm_len; \
-		} \
-		tl = (u_long *)bp
-
-#define	nfsm_srv_vattr_init(vap, v3) \
-		{ \
-		VATTR_INIT(vap); \
-		VATTR_WANTED((vap), va_type); \
-		VATTR_WANTED((vap), va_mode); \
-		VATTR_WANTED((vap), va_nlink); \
-		VATTR_WANTED((vap), va_uid); \
-		VATTR_WANTED((vap), va_gid); \
-		VATTR_WANTED((vap), va_data_size); \
-		VATTR_WANTED((vap), va_data_alloc); \
-		VATTR_WANTED((vap), va_rdev); \
-		VATTR_WANTED((vap), va_fsid); \
-		VATTR_WANTED((vap), va_fileid); \
-		VATTR_WANTED((vap), va_access_time); \
-		VATTR_WANTED((vap), va_modify_time); \
-		VATTR_WANTED((vap), va_change_time); \
-		if (!v3) VATTR_WANTED((vap), va_iosize); \
-		}
-
-#define	nfsm_srv_pre_vattr_init(vap, v3) \
-		{ \
-		VATTR_INIT(vap); \
-		VATTR_WANTED((vap), va_data_size); \
-		VATTR_WANTED((vap), va_modify_time); \
-		VATTR_WANTED((vap), va_change_time); \
-		}
-
-#define	nfsm_srvfillattr(a, f) \
-		nfsm_srvfattr(nfsd, (a), (f))
-
-#define nfsm_srvwcc_data(br, b, ar, a) \
-		nfsm_srvwcc(nfsd, (br), (b), (ar), (a), &mb, &bpos)
-
-#define nfsm_srvpostop_attr(r, a) \
-		nfsm_srvpostopattr(nfsd, (r), (a), &mb, &bpos)
-
-#define nfsm_srvsattr(a) \
-		{ \
-		struct timespec now; \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (*tl == nfs_true) { \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			VATTR_SET(a, va_mode, nfstov_mode(*tl)); \
-		} \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (*tl == nfs_true) { \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			VATTR_SET(a, va_uid, fxdr_unsigned(uid_t, *tl)); \
-		} \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (*tl == nfs_true) { \
-			nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-			VATTR_SET(a, va_gid, fxdr_unsigned(gid_t, *tl)); \
+			(FHP)->fh_len = NFSX_V2FH;\
+		nfsm_chain_get_opaque((E), (NMC), (uint32_t)(FHP)->fh_len, (FHP)->fh_data);\
+		if (E) \
+			(FHP)->fh_len = 0;\
+	} while (0)
+
+/* get an NFS v2 or v3 time from an mbuf chain */
+#define nfsm_chain_get_time(E, NMC, VERS, TSEC, TNSEC) \
+	do { \
+		nfsm_chain_get_32((E), (NMC), (TSEC)); \
+		nfsm_chain_get_32((E), (NMC), (TNSEC)); \
+		if ((E) || ((VERS) != NFS_VER2)) break; \
+		if ((uint32_t)(TNSEC) == 0xffffffff) \
+			(TNSEC) = 0; \
+		else \
+			(TNSEC) *= 1000; \
+	} while (0)
+
+/* get postop attributes from an mbuf chain */
+#define nfsm_chain_postop_attr_get(E, NMC, F, VAP) \
+	do { \
+		(F) = 0; \
+		if ((E) || !(NMC)->nmc_mhead) break; \
+		nfsm_chain_get_32((E), (NMC), (F)); \
+		if ((E) || !(F)) break; \
+		if (((E) = nfs_parsefattr((NMC), NFS_VER3, (VAP)))) \
+			(F) = 0; \
+	} while (0)
+
+/* update a node's attribute cache with postop attributes from an mbuf chain */
+/* (F returns whether the attributes were updated or not) */
+#define nfsm_chain_postop_attr_update_flag(E, NMC, NP, F, X) \
+	do { \
+		struct nfs_vattr ttvattr; \
+		nfsm_chain_postop_attr_get((E), (NMC), (F), &ttvattr); \
+		if ((E) || !(F)) break; \
+		if (((E) = nfs_loadattrcache((NP), &ttvattr, (X), 1))) { \
+			(F) = 0; \
+			break; \
 		} \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		if (*tl == nfs_true) { \
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			fxdr_hyper(tl, &(a)->va_data_size); \
-			VATTR_SET_ACTIVE(a, va_data_size); \
+		if (*(X) == 0) \
+			(F) = 0; \
+	} while (0)
+
+/* update a node's attribute cache with postop attributes from an mbuf chain */
+#define nfsm_chain_postop_attr_update(E, NMC, NP, X) \
+	do { \
+		int __dummy_flag = 0; \
+		nfsm_chain_postop_attr_update_flag((E), (NMC), (NP), __dummy_flag, (X)); \
+	} while (0)
+
+/* get and process NFSv3 WCC data from an mbuf chain */
+#define nfsm_chain_get_wcc_data(E, NMC, NP, PREMTIME, NEWPOSTATTR, X) \
+	do { \
+		if (E) break; \
+		(E) = nfsm_chain_get_wcc_data_f((NMC), (NP), (PREMTIME), (NEWPOSTATTR), (X)); \
+	} while (0)
+
+/* update a node's attribute cache with attributes from an mbuf chain */
+#define nfsm_chain_loadattr(E, NMC, NP, VERS, A, X) \
+	do { \
+		struct nfs_vattr ttvattr, *ttnvap; \
+		if (E) break; \
+		ttnvap = (A) ? (A) : &ttvattr; \
+		if ((VERS) == NFS_VER4) { \
+			NFS_CLEAR_ATTRIBUTES(ttnvap->nva_bitmap); \
+			(E) = nfs4_parsefattr((NMC), NULL, ttnvap, NULL, NULL); \
+		} else { \
+			(E) = nfs_parsefattr((NMC), (VERS), ttnvap); \
 		} \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		nanotime(&now); \
-		switch (fxdr_unsigned(int, *tl)) { \
-		case NFSV3SATTRTIME_TOCLIENT: \
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			fxdr_nfsv3time(tl, &(a)->va_access_time); \
-			VATTR_SET_ACTIVE(a, va_access_time); \
-			break; \
-		case NFSV3SATTRTIME_TOSERVER: \
-			VATTR_SET(a, va_access_time, now); \
-			break; \
-		}; \
-		nfsm_dissect(tl, u_long *, NFSX_UNSIGNED); \
-		switch (fxdr_unsigned(int, *tl)) { \
-		case NFSV3SATTRTIME_TOCLIENT: \
-			nfsm_dissect(tl, u_long *, 2 * NFSX_UNSIGNED); \
-			fxdr_nfsv3time(tl, &(a)->va_modify_time); \
-			VATTR_SET_ACTIVE(a, va_modify_time); \
-			break; \
-		case NFSV3SATTRTIME_TOSERVER: \
-			VATTR_SET(a, va_modify_time, now); \
-			break; \
-		}; }
+		if (E) break; \
+		(E) = nfs_loadattrcache((NP), ttnvap, (X), 0); \
+	} while (0)
+
+/* get NFSv4 attr bitmap */
+#define nfsm_chain_get_bitmap(E, NMC, B, LEN) \
+	do { \
+		uint32_t __len, __i; \
+		nfsm_chain_get_32((E), (NMC), __len); \
+		if (E) break; \
+		for (__i=0; __i < MIN(__len, (LEN)); __i++) \
+			nfsm_chain_get_32((E), (NMC), (B)[__i]); \
+		if (E) break; \
+		for (; __i < (LEN); __i++) \
+			(B)[__i] = 0; \
+		(LEN) = __len; \
+	} while (0)
+
+#define nfsm_chain_skip_tag(E, NMC) \
+	do { \
+		uint32_t __val; \
+		nfsm_chain_get_32((E), (NMC), __val); \
+		nfsm_chain_adv((E), (NMC), nfsm_rndup(__val)); \
+	} while (0)
+
+#define nfsm_chain_op_check(E, NMC, OP) \
+	do { \
+		uint32_t __val = 0; \
+		nfsm_chain_get_32((E), (NMC), __val); \
+		nfsm_assert((E), (__val == (OP)), EBADRPC); \
+		nfsm_chain_get_32((E), (NMC), __val); \
+		nfsm_assert((E), (__val == NFS_OK), __val); \
+	} while (0)
+
+#define nfsm_chain_check_change_info(E, NMC, DNP) \
+	do { \
+		uint64_t __ci_before, __ci_after; \
+		uint32_t __ci_atomic = 0; \
+		nfsm_chain_get_32((E), (NMC), __ci_atomic); \
+		nfsm_chain_get_64((E), (NMC), __ci_before); \
+		nfsm_chain_get_64((E), (NMC), __ci_after); \
+		if (E) break; \
+		if (__ci_atomic && (__ci_before == (DNP)->n_ncchange)) \
+			(DNP)->n_ncchange = __ci_after; \
+		else \
+			cache_purge(NFSTOV(DNP)); \
+	} while (0)
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* _NFS_NFSM_SUBS_H_ */
diff --git a/bsd/nfs/nfsmount.h b/bsd/nfs/nfsmount.h
index ce6490b48..cfbdec71a 100644
--- a/bsd/nfs/nfsmount.h
+++ b/bsd/nfs/nfsmount.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -66,28 +72,81 @@
 #include <sys/appleapiopts.h>
 
 #ifdef __APPLE_API_PRIVATE
+
+/*
+ * NFS mount file system attributes
+ */
+struct nfs_fsattr {
+	uint32_t	nfsa_flags;		/* file system flags */
+	uint32_t	nfsa_lease;		/* lease time in seconds */
+	uint32_t	nfsa_maxname;		/* maximum filename size */
+	uint32_t	nfsa_maxlink;		/* maximum # links */
+	uint32_t	nfsa_bsize;		/* block size */
+	uint32_t	nfsa_pad;		/* UNUSED */
+	uint64_t	nfsa_maxfilesize;	/* maximum file size */
+	uint64_t	nfsa_maxread;		/* maximum read size */
+	uint64_t	nfsa_maxwrite;		/* maximum write size */
+	uint64_t	nfsa_files_avail;	/* file slots available */
+	uint64_t	nfsa_files_free;	/* file slots free */
+	uint64_t	nfsa_files_total;	/* file slots total */
+	uint64_t	nfsa_space_avail;	/* disk space available */
+	uint64_t	nfsa_space_free;	/* disk space free */
+	uint64_t	nfsa_space_total;	/* disk space total */
+	uint32_t	nfsa_supp_attr[NFS_ATTR_BITMAP_LEN]; /* attributes supported on this file system */
+	uint32_t	nfsa_bitmap[NFS_ATTR_BITMAP_LEN]; /* valid attributes */
+};
+#define NFS_FSFLAG_LINK			0x00000001
+#define NFS_FSFLAG_SYMLINK		0x00000002
+#define NFS_FSFLAG_UNIQUE_FH		0x00000004
+#define NFS_FSFLAG_ACL			0x00000008
+#define NFS_FSFLAG_SET_TIME		0x00000010
+#define NFS_FSFLAG_CASE_INSENSITIVE	0x00000020
+#define NFS_FSFLAG_CASE_PRESERVING	0x00000040
+#define NFS_FSFLAG_CHOWN_RESTRICTED	0x00000080
+#define NFS_FSFLAG_HOMOGENEOUS		0x00000100
+#define NFS_FSFLAG_NO_TRUNC		0x00000200
+#define NFS_FSFLAG_FHTYPE_MASK		0xFF000000
+#define NFS_FSFLAG_FHTYPE_SHIFT		24
+
+/*
+ * function table for calling version-specific NFS functions
+ */
+struct nfs_funcs {
+	int	(*nf_mount)(struct nfsmount *, vfs_context_t, struct user_nfs_args *, nfsnode_t *);
+	int	(*nf_update_statfs)(struct nfsmount *, vfs_context_t);
+	int	(*nf_getquota)(struct nfsmount *, vfs_context_t, u_long, int, struct dqblk *);
+	int	(*nf_access_rpc)(nfsnode_t, u_long *, vfs_context_t);
+	int	(*nf_getattr_rpc)(nfsnode_t, mount_t, u_char *, size_t, vfs_context_t, struct nfs_vattr *, u_int64_t *);
+	int	(*nf_setattr_rpc)(nfsnode_t, struct vnode_attr *, vfs_context_t, int);
+	int	(*nf_read_rpc_async)(nfsnode_t, off_t, size_t, thread_t, kauth_cred_t, struct nfsreq_cbinfo *, struct nfsreq **);
+	int	(*nf_read_rpc_async_finish)(nfsnode_t, struct nfsreq *, struct uio *, size_t *, int *);
+	int	(*nf_readlink_rpc)(nfsnode_t, char *, uint32_t *, vfs_context_t);
+	int	(*nf_write_rpc_async)(nfsnode_t, struct uio *, size_t, thread_t, kauth_cred_t, int, struct nfsreq_cbinfo *, struct nfsreq **);
+	int	(*nf_write_rpc_async_finish)(nfsnode_t, struct nfsreq *, int *, size_t *, uint64_t *);
+	int	(*nf_commit_rpc)(nfsnode_t, uint64_t, uint64_t, kauth_cred_t);
+	int	(*nf_lookup_rpc_async)(nfsnode_t, char *, int, vfs_context_t, struct nfsreq **);
+	int	(*nf_lookup_rpc_async_finish)(nfsnode_t, vfs_context_t, struct nfsreq *, u_int64_t *, fhandle_t *, struct nfs_vattr *);
+	int	(*nf_remove_rpc)(nfsnode_t, char *, int, thread_t, kauth_cred_t);
+	int	(*nf_rename_rpc)(nfsnode_t, char *, int, nfsnode_t, char *, int, vfs_context_t);
+};
+
 /*
  * Mount structure.
  * One allocated on every NFS mount.
  * Holds NFS specific information for mount.
  */
-struct	nfsmount {
+struct nfsmount {
+	lck_mtx_t nm_lock;		/* nfs mount lock */
 	int	nm_flag;		/* Flags for soft/hard... */
 	int	nm_state;		/* Internal state flags */
-	mount_t	nm_mountp;		/* Vfs structure for this filesystem */
+	int	nm_vers;		/* NFS version */
+	struct nfs_funcs *nm_funcs;	/* version-specific functions */
+	mount_t	nm_mountp;		/* VFS structure for this filesystem */
+	nfsnode_t nm_dnp;		/* root directory nfsnode pointer */
 	int	nm_numgrps;		/* Max. size of groupslist */
-	struct vnode *nm_dvp;		/* root directory vnode pointer */
-	socket_t nm_so;			/* Rpc socket */
-	int	nm_sotype;		/* Type of socket */
-	int	nm_soproto;		/* and protocol */
-	mbuf_t	nm_nam;			/* Addr of server */
+	TAILQ_HEAD(, nfs_gss_clnt_ctx) nm_gsscl; /* GSS user contexts */
 	int	nm_timeo;		/* Init timer for NFSMNT_DUMBTIMR */
 	int	nm_retry;		/* Max retries */
-	int	nm_srtt[4];		/* Timers for rpcs */
-	int	nm_sdrtt[4];
-	int	nm_sent;		/* Request send count */
-	int	nm_cwnd;		/* Request send window */
-	int	nm_timeouts;		/* Request timeouts */
 	int	nm_rsize;		/* Max size of read rpc */
 	int	nm_wsize;		/* Max size of write rpc */
 	int	nm_biosize;		/* buffer I/O size */
@@ -97,46 +156,107 @@ struct	nfsmount {
 	int	nm_acregmax;		/* reg file max attr cache timeout */
 	int	nm_acdirmin;		/* dir min attr cache timeout */
 	int	nm_acdirmax;		/* dir max attr cache timeout */
-	uid_t	nm_authuid;		/* Uid for authenticator */
-	int	nm_authtype;		/* Authenticator type */
-	int	nm_authlen;		/* and length */
-	char	*nm_authstr;		/* Authenticator string */
-	char	*nm_verfstr;		/* and the verifier */
-	int	nm_verflen;
-	uint64_t nm_verf;		/* V3 write verifier */
-	NFSKERBKEY_T nm_key;		/* and the session key */
-	int	nm_numuids;		/* Number of nfsuid mappings */
-	TAILQ_HEAD(, nfsuid) nm_uidlruhead; /* Lists of nfsuid mappings */
-	LIST_HEAD(, nfsuid) nm_uidhashtbl[NFS_MUIDHASHSIZ];
-	TAILQ_HEAD(, nfsbuf) nm_bufq;	/* async io buffer queue */
-	short	nm_bufqlen;		/* number of buffers in queue */
-	short	nm_bufqwant;		/* process wants to add to the queue */
-	int	nm_bufqiods;		/* number of iods processing queue */
+	uint32_t nm_auth;		/* security mechanism flavor */
+	/* mount info */
+	uint32_t nm_fsattrstamp;	/* timestamp for fs attrs */
+	struct nfs_fsattr nm_fsattr;	/* file system attributes */
+	uint64_t nm_verf;		/* v3/v4 write verifier */
+	union {
+	    struct {			/* v2/v3 specific fields */
+		u_short rqport;		/* cached rquota port */
+		uint32_t rqportstamp;	/* timestamp of rquota port */
+	    } v3;
+	    struct {			/* v4 specific fields */
+		uint64_t clientid;	/* client ID */
+		uint64_t mounttime;	/* mount verifier */
+		thread_call_t renew_timer; /* RENEW timer call */
+	    } v4;
+	} nm_un;
+	/* async I/O queue */
+	struct nfs_reqqhead nm_resendq;	/* async I/O resend queue */
+	struct nfs_reqqhead nm_iodq;	/* async I/O request queue */
+	struct nfsiod *nm_niod;		/* nfsiod processing this mount */
+	TAILQ_ENTRY(nfsmount) nm_iodlink; /* chain of mounts awaiting nfsiod */
+	int	nm_asyncwrites;		/* outstanding async I/O writes */
+	/* socket state */
+	int	nm_sotype;		/* Type of socket */
+	int	nm_soproto;		/* and protocol */
+	mbuf_t	nm_nam;			/* Address of server */
+	u_short nm_sockflags;		/* socket state flags */
+	socket_t nm_so;			/* RPC socket */
+	int	nm_reconnect_start;	/* reconnect start time */
 	int	nm_tprintf_initial_delay;	/* delay first "server down" */
 	int	nm_tprintf_delay;	/* delay between "server down" */
-	struct {			/* fsinfo & (homogenous) pathconf info */
-		u_int64_t maxfilesize;	/* max size of a file */
-		u_long	linkmax;	/* max # hard links to an object */
-		u_long	namemax;	/* max length of filename component */
-		u_char	pcflags;	/* boolean pathconf properties */
-		u_char	fsproperties;	/* fsinfo properties */
-	} nm_fsinfo;
+	int	nm_srtt[4];		/* Timers for RPCs */
+	int	nm_sdrtt[4];
+	int	nm_timeouts;		/* Request timeouts */
+	union {
+		struct {
+			int sent;	/* Request send count */
+			int cwnd;	/* Request congestion window */
+			struct nfs_reqqhead cwndq; /* requests waiting on cwnd */
+		} udp;
+		struct {
+			u_int32_t mleft;/* marker bytes remaining */
+			u_int32_t fleft;/* fragment bytes remaining */
+			u_int32_t len;	/* length of RPC record */
+			mbuf_t m;	/* mbufs for current record */
+			mbuf_t mlast;
+		} tcp;
+	} nm_sockstate;
+	TAILQ_ENTRY(nfsmount) nm_pokeq;	/* mount poke queue chain */
+	thread_t nm_sockthd;		/* socket thread for this mount */
 };
 
+/*
+ * NFS mount state flags (nm_state)
+ */
+#define NFSSTA_JUKEBOXTIMEO	0x00001000  /* experienced a jukebox timeout */
+#define NFSSTA_LOCKTIMEO	0x00002000  /* experienced a lock req timeout */
+#define NFSSTA_MOUNTED		0x00004000  /* completely mounted */
+#define NFSSTA_LOCKSWORK	0x00008000  /* lock ops have worked. */
+#define NFSSTA_TIMEO		0x00010000  /* experienced a timeout. */
+#define NFSSTA_FORCE		0x00020000  /* doing a forced unmount. */
+#define NFSSTA_HASWRITEVERF	0x00040000  /* Has write verifier for V3 */
+#define NFSSTA_GOTPATHCONF	0x00080000  /* Got the V3 pathconf info */
+#define NFSSTA_GOTFSINFO	0x00100000  /* Got the V3 fsinfo */
+#define NFSSTA_SNDLOCK		0x01000000  /* Send socket lock */
+#define NFSSTA_WANTSND		0x02000000  /* Want above */
+
+/* flags for nm_sockflags */
+#define NMSOCK_READY		0x0001	/* socket is ready for use */
+#define NMSOCK_CONNECTING	0x0002	/* socket is being connect()ed */
+#define NMSOCK_SETUP		0x0004	/* socket/connection is being set up */
+#define NMSOCK_UNMOUNT		0x0008	/* unmounted, no more socket activity */
+#define NMSOCK_LASTFRAG		0x0010	/* on last fragment of RPC record */
+#define NMSOCK_POKE		0x0020	/* socket needs to be poked */
+#define NMSOCK_UPCALL		0x0040	/* socket upcall in progress */
+
+/* aliases for socket state variables */
+#define nm_sent		nm_sockstate.udp.sent
+#define nm_cwnd		nm_sockstate.udp.cwnd
+#define nm_cwndq	nm_sockstate.udp.cwndq
+#define nm_markerleft	nm_sockstate.tcp.mleft
+#define nm_fragleft	nm_sockstate.tcp.fleft
+#define nm_reclen	nm_sockstate.tcp.len
+#define nm_m		nm_sockstate.tcp.m
+#define nm_mlast	nm_sockstate.tcp.mlast
+
+/* aliases for version-specific fields */
+#define nm_rqport	nm_un.v3.rqport
+#define nm_rqportstamp	nm_un.v3.rqportstamp
+#define nm_clientid	nm_un.v4.clientid
+#define nm_mounttime	nm_un.v4.mounttime
+#define nm_renew_timer	nm_un.v4.renew_timer
 
 #if defined(KERNEL)
 /*
- * Convert mount_t to struct nfsmount*
+ * Macros to convert from various things to mount structures.
  */
 #define VFSTONFS(mp)	((mp) ? ((struct nfsmount *)vfs_fsprivate(mp)) : NULL)
-
-#ifndef NFS_TPRINTF_INITIAL_DELAY
-#define NFS_TPRINTF_INITIAL_DELAY	12
-#endif
-
-#ifndef NFS_TPRINTF_DELAY
-#define NFS_TPRINTF_DELAY		30
-#endif
+#define VTONMP(vp)	VFSTONFS(vnode_mount(vp))
+#define NFSTONMP(np)	VTONMP(NFSTOV(np))
+#define NFSTOMP(np)	(vnode_mount(NFSTOV(np)))
 
 #endif /* KERNEL */
 
diff --git a/bsd/nfs/nfsnode.h b/bsd/nfs/nfsnode.h
index ba9a7deb6..794841e72 100644
--- a/bsd/nfs/nfsnode.h
+++ b/bsd/nfs/nfsnode.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -72,13 +78,13 @@
 
 /*
  * Silly rename structure that hangs off the nfsnode until the name
- * can be removed by nfs_inactive()
+ * can be removed by nfs_vnop_inactive()
  */
-struct sillyrename {
-	struct	ucred *s_cred;
-	vnode_t	s_dvp;
-	long	s_namlen;
-	char	s_name[20];
+struct nfs_sillyrename {
+	kauth_cred_t	nsr_cred;
+	struct nfsnode	*nsr_dnp;
+	int		nsr_namlen;
+	char		nsr_name[20];
 };
 
 /*
@@ -102,7 +108,7 @@ struct nfsdmap {
  */
 struct nfsbuf {
 	LIST_ENTRY(nfsbuf)	nb_hash;	/* hash chain */
-	LIST_ENTRY(nfsbuf)	nb_vnbufs;	/* vnode's nfsbuf chain */
+	LIST_ENTRY(nfsbuf)	nb_vnbufs;	/* nfsnode's nfsbuf chain */
 	TAILQ_ENTRY(nfsbuf)	nb_free;	/* free list position if not active. */
 	volatile long		nb_flags;	/* NB_* flags. */
 	volatile long		nb_lflags;	/* NBL_* flags. */
@@ -110,6 +116,7 @@ struct nfsbuf {
 	long			nb_bufsize;	/* buffer size */
 	daddr64_t		nb_lblkno;	/* logical block number. */
 	uint64_t		nb_verf;	/* V3 write verifier */
+	int			nb_commitlevel;	/* lowest write commit level */
 	time_t			nb_timestamp;	/* buffer timestamp */
 	int			nb_error;	/* errno value. */
 	u_int32_t		nb_valid;	/* valid pages in buf */
@@ -118,18 +125,23 @@ struct nfsbuf {
 	int			nb_validend;	/* offset of end of valid region. */
 	int			nb_dirtyoff;	/* offset in buffer of dirty region. */
 	int			nb_dirtyend;	/* offset of end of dirty region. */
+	int			nb_offio;	/* offset in buffer of I/O region. */
+	int			nb_endio;	/* offset of end of I/O region. */
+	int			nb_rpcs;	/* Count of RPCs remaining for this buffer. */
 	caddr_t			nb_data;	/* mapped buffer */
-	vnode_t			nb_vp;		/* device vnode */
-	proc_t			nb_proc;	/* associated proc; NULL if kernel. */
-	struct ucred *		nb_rcred;	/* read credentials reference */
-	struct ucred *		nb_wcred;	/* write credentials reference */
+	nfsnode_t		nb_np;		/* nfsnode buffer belongs to */
+	kauth_cred_t		nb_rcred;	/* read credentials reference */
+	kauth_cred_t		nb_wcred;	/* write credentials reference */
 	void *			nb_pagelist;	/* upl */
 };
 
 #define NFS_MAXBSIZE	(32 * PAGE_SIZE)	/* valid/dirty page masks limit buffer size */
 
+#define NFS_A_LOT_OF_NEEDCOMMITS	256			/* max# uncommitted buffers for a node */
+#define NFS_A_LOT_OF_DELAYED_WRITES	MAX(nfsbufcnt/8,512)	/* max# "delwri" buffers in system */
+
 /*
- * These flags are kept in b_lflags... 
+ * These flags are kept in b_lflags...
  * nfs_buf_mutex must be held before examining/updating
  */
 #define	NBL_BUSY	0x00000001	/* I/O in progress. */
@@ -140,23 +152,24 @@ struct nfsbuf {
  * very similar to the B_* flags for struct buf.
  * nfs_buf_mutex is not needed to examine/update these.
  */
-#define	NB_NEEDCOMMIT	0x00000002	/* Append-write in progress. */
+#define	NB_STALEWVERF	0x00000001	/* write verifier changed on us */
+#define	NB_NEEDCOMMIT	0x00000002	/* buffer needs to be committed */
 #define	NB_ASYNC	0x00000004	/* Start I/O, do not wait. */
-#define	NB_CACHE	0x00000020	/* Bread found us in the cache. */
-#define	NB_STABLE	0x00000040	/* write FILESYNC not UNSTABLE. */
-#define	NB_DELWRI	0x00000080	/* Delay I/O until buffer reused. */
+#define	NB_CACHE	0x00000020	/* buffer data found in the cache */
+#define	NB_STABLE	0x00000040	/* write FILESYNC not UNSTABLE */
+#define	NB_DELWRI	0x00000080	/* delayed write: dirty range needs to be written */
 #define	NB_DONE		0x00000200	/* I/O completed. */
 #define	NB_EINTR	0x00000400	/* I/O was interrupted */
 #define	NB_ERROR	0x00000800	/* I/O error occurred. */
-#define	NB_WASDIRTY	0x00001000	/* page was found dirty in the VM cache */
 #define	NB_INVAL	0x00002000	/* Does not contain valid info. */
+#define	NB_NCRDAHEAD	0x00004000	/* "nocache readahead" data */
 #define	NB_NOCACHE	0x00008000	/* Do not cache block after use. */
+#define	NB_WRITE	0x00000000	/* Write buffer (pseudo flag). */
 #define	NB_READ		0x00100000	/* Read buffer. */
+#define	NB_MULTASYNCRPC	0x00200000	/* multiple async RPCs issued for buffer */
 #define	NB_PAGELIST	0x00400000	/* Buffer describes pagelist I/O. */
-#define	NB_WRITE	0x00000000	/* Write buffer (pseudo flag). */
 #define	NB_WRITEINPROG	0x01000000	/* Write in progress. */
 #define	NB_META		0x40000000	/* buffer contains meta-data. */
-#define	NB_IOD		0x80000000	/* buffer being handled by nfsiod. */
 
 /* Flags for operation type in nfs_buf_get() */
 #define	NBLK_READ	0x00000001	/* buffer for read */
@@ -199,16 +212,15 @@ TAILQ_HEAD(nfsbuffreehead, nfsbuf);
 
 #define NFSNOLIST ((struct nfsbuf *)0xdeadbeef)
 
-extern lck_mtx_t *nfs_buf_mutex;
-extern int nfsbufcnt, nfsbufmin, nfsbufmax, nfsbufmetacnt, nfsbufmetamax;
-extern int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer;
-extern int nfs_nbdwrite;
-extern struct nfsbuffreehead nfsbuffree, nfsbufdelwri;
-extern time_t nfsbuffreeuptimestamp;
+__private_extern__ lck_mtx_t *nfs_buf_mutex;
+__private_extern__ int nfsbufcnt, nfsbufmin, nfsbufmax, nfsbufmetacnt, nfsbufmetamax;
+__private_extern__ int nfsbuffreecnt, nfsbuffreemetacnt, nfsbufdelwricnt, nfsneedbuffer;
+__private_extern__ int nfs_nbdwrite;
+__private_extern__ struct nfsbuffreehead nfsbuffree, nfsbufdelwri;
 
-#define NFSBUFCNTCHK(locked) \
+#ifdef NFSBUFDEBUG
+#define NFSBUFCNTCHK() \
 	do { \
-	if (!locked) lck_mtx_lock(nfs_buf_mutex); \
 	if (	(nfsbufcnt < 0) || \
 		(nfsbufcnt > nfsbufmax) || \
 		(nfsbufmetacnt < 0) || \
@@ -231,78 +243,154 @@ extern time_t nfsbuffreeuptimestamp;
 		panic("nfsbuf count error: max %d meta %d cnt %d meta %d free %d meta %d delwr %d bdw %d\n", \
 			nfsbufmax, nfsbufmetamax, nfsbufcnt, nfsbufmetacnt, nfsbuffreecnt, nfsbuffreemetacnt, \
 			nfsbufdelwricnt, nfs_nbdwrite); \
-	if (!locked) lck_mtx_unlock(nfs_buf_mutex); \
+	} while (0)
+#else
+#define NFSBUFCNTCHK()
+#endif
+
+/*
+ * NFS vnode attribute structure
+ */
+#define NFSTIME_ACCESS	0	/* time of last access */
+#define NFSTIME_MODIFY	1	/* time of last modification */
+#define NFSTIME_CHANGE	2	/* time file changed */
+#define NFSTIME_CREATE	3	/* time file created */
+#define NFSTIME_BACKUP	4	/* time of last backup */
+#define NFSTIME_COUNT	5
+
+#define NFS_COMPARE_MTIME(TVP, NVAP, CMP) \
+	(((TVP)->tv_sec == (NVAP)->nva_timesec[NFSTIME_MODIFY]) ?	\
+	 ((TVP)->tv_nsec CMP (NVAP)->nva_timensec[NFSTIME_MODIFY]) :	\
+	 ((TVP)->tv_sec CMP (NVAP)->nva_timesec[NFSTIME_MODIFY]))
+#define NFS_COPY_TIME(TVP, NVAP, WHICH) \
+	do { \
+	(TVP)->tv_sec = (NVAP)->nva_timesec[NFSTIME_##WHICH]; \
+	(TVP)->tv_nsec = (NVAP)->nva_timensec[NFSTIME_##WHICH]; \
 	} while (0)
 
 struct nfs_vattr {
 	enum vtype	nva_type;	/* vnode type (for create) */
-	u_short		nva_mode;	/* files access mode and type */ 
-	dev_t		nva_rdev;	/* device the special file represents */
+	uint32_t	nva_mode;	/* files access mode (and type) */
 	uid_t		nva_uid;	/* owner user id */
 	gid_t		nva_gid;	/* owner group id */
-	uint32_t	nva_fsid;	/* file system id (dev for now) */
-	uint64_t	nva_nlink;	/* number of references to file */ 
+	nfs_specdata	nva_rawdev;	/* device the special file represents */
+	uint32_t	nva_flags;	/* file flags */
+	uint32_t	nva_maxlink;	/* maximum # of links (v4) */
+	uint64_t	nva_nlink;	/* number of references to file */
 	uint64_t	nva_fileid;	/* file id */
+	nfs_fsid	nva_fsid;	/* file system id */
 	uint64_t	nva_size;	/* file size in bytes */
 	uint64_t	nva_bytes;	/* bytes of disk space held by file */
-	uint32_t	nva_blocksize;	/* blocksize preferred for i/o */
-	struct timespec	nva_atime;	/* time of last access */
-	struct timespec	nva_mtime;	/* time of last modification */
-	struct timespec	nva_ctime;	/* time file changed */
+	uint64_t	nva_change;	/* change attribute */
+	int64_t		nva_timesec[NFSTIME_COUNT];
+	int32_t		nva_timensec[NFSTIME_COUNT];
+	uint32_t 	nva_bitmap[NFS_ATTR_BITMAP_LEN]; /* attributes that are valid */
 };
 
+#define NFS_FFLAG_ARCHIVED	0x0001
+#define NFS_FFLAG_HIDDEN	0x0002
+#define NFS_FFLAG_NAMED_ATTR	0x0004	/* file has named attributes */
+
+/*
+ * macros for detecting node changes
+ *
+ * These macros help us determine if a file has been changed on the server and
+ * thus whether or not we need to invalidate any cached data.
+ *
+ * For NFSv2/v3, the modification time is used.
+ * For NFSv4, the change attribute is used.
+ */
+#define NFS_CHANGED(VERS, NP, NVAP) \
+		(((VERS) >= NFS_VER4) ? \
+			((NP)->n_change != (NVAP)->nva_change) : \
+			NFS_COMPARE_MTIME(&(NP)->n_mtime, (NVAP), !=))
+#define NFS_CHANGED_NC(VERS, NP, NVAP) \
+		(((VERS) >= NFS_VER4) ? \
+			((NP)->n_ncchange != (NVAP)->nva_change) : \
+			NFS_COMPARE_MTIME(&(NP)->n_ncmtime, (NVAP), !=))
+#define NFS_CHANGED_UPDATE(VERS, NP, NVAP) \
+	do { \
+		if ((VERS) >= NFS_VER4) \
+			(NP)->n_change = (NVAP)->nva_change; \
+		else \
+			NFS_COPY_TIME(&(NP)->n_mtime, (NVAP), MODIFY); \
+	} while (0)
+#define NFS_CHANGED_UPDATE_NC(VERS, NP, NVAP) \
+	do { \
+		if ((VERS) >= NFS_VER4) \
+			(NP)->n_ncchange = (NVAP)->nva_change; \
+		else \
+			NFS_COPY_TIME(&(NP)->n_ncmtime, (NVAP), MODIFY); \
+	} while (0)
+
 /*
- * The nfsnode is the nfs equivalent to ufs's inode. Any similarity
- * is purely coincidental.
- * There is a unique nfsnode allocated for each active file,
- * each current directory, each mounted-on file, text file, and the root.
+ * The nfsnode is the NFS equivalent of an inode.
+ * There is a unique nfsnode for each NFS vnode.
  * An nfsnode is 'named' by its file handle. (nget/nfs_node.c)
- * If this structure exceeds 256 bytes (it is currently 256 using 4.4BSD-Lite
- * type definitions), file handles of > 32 bytes should probably be split out
- * into a separate MALLOC()'d data structure. (Reduce the size of nfsnode.n_fh
- * by changing the definition in nfsproto.h of NFS_SMALLFH.)
  * NB: Hopefully the current order of the fields is such that everything will
  *     be well aligned and, therefore, tightly packed.
  */
+
+#define NFS_ACCESS_CACHE_SIZE	3
+
 struct nfsnode {
+	lck_rw_t		n_lock;		/* nfs node lock */
+	void			*n_lockowner;	/* nfs node lock owner (exclusive) */
+	lck_rw_t		n_datalock;	/* nfs node data lock */
+	void			*n_datalockowner;/* nfs node data lock owner (exclusive) */
 	LIST_ENTRY(nfsnode)	n_hash;		/* Hash chain */
 	u_quad_t		n_size;		/* Current size of file */
+	u_quad_t		n_newsize;	/* new size of file (pending update) */
+	u_int64_t		n_xid;		/* last xid to loadattr */
 	struct nfs_vattr	n_vattr;	/* Vnode attribute cache */
 	time_t			n_attrstamp;	/* Attr. cache timestamp */
-        u_int32_t               n_mode;         /* ACCESS mode cache */
-        uid_t                   n_modeuid;      /* credentials having mode */
-        time_t                  n_modestamp;    /* mode cache timestamp */
-	struct timespec		n_mtime;	/* Prev modify time. */
-	struct timespec		n_ncmtime;	/* namecache modify time. */
-	u_char			*n_fhp;		/* NFS File Handle */
+	u_int8_t		n_mode[NFS_ACCESS_CACHE_SIZE+1];	/* ACCESS mode cache */
+	uid_t                   n_modeuid[NFS_ACCESS_CACHE_SIZE];	/* credentials having mode */
+	time_t                  n_modestamp[NFS_ACCESS_CACHE_SIZE];	/* mode cache timestamp */
 	union {
-		vnode_t		n_vp;		/* associated vnode */
-		mount_t		n_mp;		/* associated mount (NINIT) */
-	} n_un0;
-	struct lockf		*n_lockf;	/* Locking record of file */
+	    struct {
+		struct timespec	n3_mtime;	/* Prev modify time. */
+		struct timespec	n3_ncmtime;	/* namecache modify time. */
+	    } v3;
+	    struct {
+		uint64_t	n4_change;	/* prev change attribute */
+		uint64_t	n4_ncchange;	/* namecache change attribute */
+	    } v4;
+	} n_un4;
+	vnode_t			n_parent;	/* this node's parent */
+	u_char			*n_fhp;		/* NFS File Handle */
+	vnode_t			n_vnode;	/* associated vnode */
+	mount_t			n_mount;	/* associated mount (NHINIT) */
 	int			n_error;	/* Save write error value */
 	union {
 		struct timespec	nf_atim;	/* Special file times */
 		nfsuint64	nd_cookieverf;	/* Cookie verifier (dir only) */
 	} n_un1;
 	union {
-		struct timespec	nf_mtim;
+		struct timespec	nf_mtim;	/* Special file times */
+		daddr64_t	nf_lastread;	/* last block# read from (for readahead) */
 		off_t		nd_direof;	/* Dir. EOF offset cache */
 	} n_un2;
 	union {
-		struct sillyrename *nf_silly;	/* Ptr to silly rename struct */
+		struct nfs_sillyrename *nf_silly;/* Ptr to silly rename struct */
 		LIST_HEAD(, nfsdmap) nd_cook;	/* cookies */
 	} n_un3;
-	short			n_fhsize;	/* size in bytes, of fh */
-	short			n_flag;		/* Flag for locking.. */
+	u_short			n_fhsize;	/* size in bytes, of fh */
+	u_short			n_flag;		/* node flags */
+	u_short			n_hflag;	/* node hash flags */
+	u_short			n_bflag;	/* node buffer flags */
 	u_char			n_fh[NFS_SMALLFH];/* Small File Handle */
-	u_int64_t		n_xid;		/* last xid to loadattr */
 	struct nfsbuflists	n_cleanblkhd;	/* clean blocklist head */
 	struct nfsbuflists	n_dirtyblkhd;	/* dirty blocklist head */
+	int			n_wrbusy;	/* # threads in write/fsync */
 	int			n_needcommitcnt;/* # bufs that need committing */
 	int			n_bufiterflags;	/* buf iterator flags */
 };
 
+#define NFS_NODE_LOCK_SHARED	1
+#define NFS_NODE_LOCK_EXCLUSIVE	2
+#define NFS_NODE_LOCK_FORCE	3
+
 #define nfstimespeccmp(tvp, uvp, cmp)		\
 	(((tvp)->tv_sec == (uvp)->tv_sec) ?	\
 	 ((tvp)->tv_nsec cmp (uvp)->tv_nsec) :	\
@@ -316,39 +404,59 @@ struct nfsnode {
 		} \
 	} while (0)
 
-#define n_vnode		n_un0.n_vp
-#define n_mount		n_un0.n_mp
-#define n_atim		n_un1.nf_atim
-#define n_mtim		n_un2.nf_mtim
-#define n_sillyrename	n_un3.nf_silly
-#define n_cookieverf	n_un1.nd_cookieverf
-#define n_direofoffset	n_un2.nd_direof
-#define n_cookies	n_un3.nd_cook
+#define n_atim			n_un1.nf_atim
+#define n_mtim			n_un2.nf_mtim
+#define n_lastread		n_un2.nf_lastread
+#define n_sillyrename		n_un3.nf_silly
+#define n_cookieverf		n_un1.nd_cookieverf
+#define n_direofoffset		n_un2.nd_direof
+#define n_cookies		n_un3.nd_cook
+#define n_mtime			n_un4.v3.n3_mtime
+#define n_ncmtime		n_un4.v3.n3_ncmtime
+#define n_change		n_un4.v4.n4_change
+#define n_ncchange		n_un4.v4.n4_ncchange
 
 /*
  * Flags for n_flag
  */
-#define	NFLUSHWANT	0x0001	/* Want wakeup from a flush in prog. */
-#define	NFLUSHINPROG	0x0002	/* Avoid multiple calls to vinvalbuf() */
+#define	NUPDATESIZE	0x0001	/* size of file needs updating */
 #define	NMODIFIED	0x0004	/* Might have a modified buffer in bio */
 #define	NWRITEERR	0x0008	/* Flag write errors so close will know */
 #define	NNEEDINVALIDATE	0x0010	/* need to call vinvalbuf() */
-#define	NNOCACHE	0x0020	/* all bufs are uncached */
-#define	NWRBUSY		0x0040	/* node in write/fsync */
 #define	NACC		0x0100	/* Special file accessed */
 #define	NUPD		0x0200	/* Special file updated */
 #define	NCHG		0x0400	/* Special file times changed */
-#define NHASHED		0x1000  /* someone wants to lock */
-#define NINIT		0x2000  /* node is being initialized */
-#define NWINIT		0x4000  /* someone waiting for init to complete */
+#define	NNEGNCENTRIES	0x0800	/* directory has negative name cache entries */
 
+/*
+ * Flags for n_hflag
+ * Note: protected by nfs_node_hash_mutex
+ */
+#define NHHASHED	0x0001  /* node is in hash table */
+#define NHINIT		0x0002  /* node is being initialized */
+#define NHLOCKED	0x0004  /* node is locked (initting or deleting) */
+#define NHLOCKWANT	0x0008  /* someone wants to lock */
+
+/*
+ * Flags for n_bflag
+ * Note: protected by nfs_buf_mutex
+ */
+#define	NBFLUSHINPROG	0x0001	/* Avoid multiple calls to nfs_flush() */
+#define	NBFLUSHWANT	0x0002	/* waiting for nfs_flush() to complete */
+#define	NBINVALINPROG	0x0004	/* Avoid multiple calls to nfs_vinvalbuf() */
+#define	NBINVALWANT	0x0008	/* waiting for nfs_vinvalbuf() to complete */
+
+/* attr/mode timestamp macros */
 #define NATTRVALID(np)		((np)->n_attrstamp != ~0)
 #define NATTRINVALIDATE(np)	((np)->n_attrstamp = ~0)
-#define NMODEVALID(np)		((np)->n_modestamp != ~0)
-#define NMODEINVALIDATE(np)	((np)->n_modestamp = ~0)
-
-#define NVALIDBUFS(np)	(!LIST_EMPTY(&(np)->n_dirtyblkhd) || \
- 			 !LIST_EMPTY(&(np)->n_cleanblkhd))
+#define NMODEVALID(np, slot)	(((slot) >= 0) && ((slot) < 3) && ((np)->n_modestamp[(slot)] != ~0))
+#define NMODEINVALIDATE(np) \
+	do { \
+		(np)->n_modestamp[0] = ~0; \
+		(np)->n_modestamp[1] = ~0; \
+		(np)->n_modestamp[2] = ~0; \
+		(np)->n_mode[3] = 0; \
+	} while (0)
 
 /*
  * NFS-specific flags for nfs_vinvalbuf/nfs_flush
@@ -364,21 +472,24 @@ struct nfsnode {
 /*
  * Convert between nfsnode pointers and vnode pointers
  */
-#define VTONFS(vp)	((struct nfsnode *)vnode_fsnode(vp))
+#define VTONFS(vp)	((nfsnode_t)vnode_fsnode(vp))
 #define NFSTOV(np)	((np)->n_vnode)
 
 /* nfsnode hash table mutex */
-extern lck_mtx_t *nfs_node_hash_mutex;
+__private_extern__ lck_mtx_t *nfs_node_hash_mutex;
 
 /*
  * nfsiod structures
  */
-extern proc_t nfs_iodwant[NFS_MAXASYNCDAEMON];
-extern struct nfsmount *nfs_iodmount[NFS_MAXASYNCDAEMON];
-extern lck_grp_t *nfs_iod_lck_grp;
-extern lck_grp_attr_t *nfs_iod_lck_grp_attr;
-extern lck_attr_t *nfs_iod_lck_attr;
-extern lck_mtx_t *nfs_iod_mutex;
+struct nfsiod {
+	TAILQ_ENTRY(nfsiod)	niod_link;	/* List of nfsiods */
+	struct nfsmount *	niod_nmp;	/* mount point for this nfsiod */
+};
+TAILQ_HEAD(nfsiodlist, nfsiod);
+TAILQ_HEAD(nfsiodmountlist, nfsmount);
+__private_extern__ struct nfsiodlist nfsiodfree, nfsiodwork;
+__private_extern__ struct nfsiodmountlist nfsiodmounts;
+__private_extern__ lck_mtx_t *nfsiod_mutex;
 
 #if defined(KERNEL)
 
@@ -386,43 +497,78 @@ typedef int     vnop_t(void *);
 extern	vnop_t	**fifo_nfsv2nodeop_p;
 extern	vnop_t	**nfsv2_vnodeop_p;
 extern	vnop_t	**spec_nfsv2nodeop_p;
+extern	vnop_t	**fifo_nfsv4nodeop_p;
+extern	vnop_t	**nfsv4_vnodeop_p;
+extern	vnop_t	**spec_nfsv4nodeop_p;
 
 /*
  * Prototypes for NFS vnode operations
  */
-int	nfs_write(struct vnop_write_args *);
-#define nfs_revoke nop_revoke
-#define nfs_seek ((int (*)(struct  vnop_seek_args *))nullop) //XXXdead?
-int	nfs_inactive(struct vnop_inactive_args *);
-int	nfs_reclaim(struct vnop_reclaim_args *);
-
+int	nfs_vnop_write(struct vnop_write_args *);
+#define nfs_vnop_revoke nop_revoke
+int	nfs_vnop_inactive(struct vnop_inactive_args *);
+int	nfs_vnop_reclaim(struct vnop_reclaim_args *);
+
+int nfs_lock(nfsnode_t, int);
+void nfs_unlock(nfsnode_t);
+int nfs_lock2(nfsnode_t, nfsnode_t, int);
+void nfs_unlock2(nfsnode_t, nfsnode_t);
+int nfs_lock4(nfsnode_t, nfsnode_t, nfsnode_t, nfsnode_t, int);
+void nfs_unlock4(nfsnode_t, nfsnode_t, nfsnode_t, nfsnode_t);
+void nfs_data_lock(nfsnode_t, int);
+void nfs_data_lock2(nfsnode_t, int, int);
+void nfs_data_unlock(nfsnode_t);
+void nfs_data_unlock2(nfsnode_t, int);
+void nfs_data_update_size(nfsnode_t, int);
 
 /* other stuff */
-int	nfs_removeit(struct sillyrename *);
-int	nfs_nget(mount_t,vnode_t,struct componentname *,u_char *,int,struct nfs_vattr *,u_int64_t *,int,struct nfsnode **);
-nfsuint64 *nfs_getcookie(struct nfsnode *, off_t, int);
-void nfs_invaldir(vnode_t);
+int nfs_removeit(struct nfs_sillyrename *);
+int nfs_nget(mount_t,nfsnode_t,struct componentname *,u_char *,int,struct nfs_vattr *,u_int64_t *,int,nfsnode_t*);
+nfsuint64 *nfs_getcookie(nfsnode_t, off_t, int);
+void nfs_invaldir(nfsnode_t);
 
 /* nfsbuf functions */
 void nfs_nbinit(void);
+void nfs_buf_timer(void *, void *);
 void nfs_buf_remfree(struct nfsbuf *);
-boolean_t nfs_buf_is_incore(vnode_t, daddr64_t);
-struct nfsbuf * nfs_buf_incore(vnode_t, daddr64_t);
-int nfs_buf_get(vnode_t, daddr64_t, int, proc_t, int, struct nfsbuf **);
+boolean_t nfs_buf_is_incore(nfsnode_t, daddr64_t);
+struct nfsbuf * nfs_buf_incore(nfsnode_t, daddr64_t);
+int nfs_buf_get(nfsnode_t, daddr64_t, int, thread_t, int, struct nfsbuf **);
 int nfs_buf_upl_setup(struct nfsbuf *bp);
 void nfs_buf_upl_check(struct nfsbuf *bp);
+void nfs_buf_normalize_valid_range(nfsnode_t, struct nfsbuf *);
+int nfs_buf_map(struct nfsbuf *);
 void nfs_buf_release(struct nfsbuf *, int);
 int nfs_buf_iowait(struct nfsbuf *);
 void nfs_buf_iodone(struct nfsbuf *);
-void nfs_buf_write_delayed(struct nfsbuf *, proc_t);
-void nfs_buf_check_write_verifier(struct nfsnode *, struct nfsbuf *);
+void nfs_buf_write_delayed(struct nfsbuf *);
+void nfs_buf_check_write_verifier(nfsnode_t, struct nfsbuf *);
 void nfs_buf_freeup(int);
 void nfs_buf_refget(struct nfsbuf *bp);
 void nfs_buf_refrele(struct nfsbuf *bp);
 void nfs_buf_drop(struct nfsbuf *);
 errno_t nfs_buf_acquire(struct nfsbuf *, int, int, int);
-int nfs_buf_iterprepare(struct nfsnode *, struct nfsbuflists *, int);
-void nfs_buf_itercomplete(struct nfsnode *, struct nfsbuflists *, int);
+int nfs_buf_iterprepare(nfsnode_t, struct nfsbuflists *, int);
+void nfs_buf_itercomplete(nfsnode_t, struct nfsbuflists *, int);
+
+int nfs_bioread(nfsnode_t, struct uio *, int, int *, vfs_context_t);
+int nfs_buf_readdir(struct nfsbuf *, vfs_context_t);
+int nfs_buf_read(struct nfsbuf *);
+void nfs_buf_read_finish(struct nfsbuf *);
+int nfs_buf_read_rpc(struct nfsbuf *, thread_t, kauth_cred_t);
+void nfs_buf_read_rpc_finish(struct nfsreq *);
+int nfs_buf_write(struct nfsbuf *);
+void nfs_buf_write_finish(struct nfsbuf *, thread_t, kauth_cred_t);
+int nfs_buf_write_rpc(struct nfsbuf *, int, thread_t, kauth_cred_t);
+void nfs_buf_write_rpc_finish(struct nfsreq *);
+int nfs_buf_write_dirty_pages(struct nfsbuf *, thread_t, kauth_cred_t);
+
+int nfs_flushcommits(nfsnode_t, int);
+int nfs_flush(nfsnode_t, int, thread_t, int);
+
+int nfsiod_start(void);
+void nfs_asyncio_finish(struct nfsreq *);
+void nfs_asyncio_resend(struct nfsreq *);
 
 #endif /* KERNEL */
 
diff --git a/bsd/nfs/nfsproto.h b/bsd/nfs/nfsproto.h
index d44115245..47ba3faf1 100644
--- a/bsd/nfs/nfsproto.h
+++ b/bsd/nfs/nfsproto.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -65,15 +71,11 @@
 #include <sys/appleapiopts.h>
 
 #ifdef __APPLE_API_PRIVATE
-/*
- * nfs definitions as per the Version 2 and 3 specs
- */
 
 /*
- * Constants as defined in the Sun NFS Version 2 and 3 specs.
- * "NFS: Network File System Protocol Specification" RFC1094
- * and in the "NFS: Network File System Version 3 Protocol
- * Specification"
+ * NFS definitions per the various NFS protocol specs:
+ * Version 2 (RFC 1094), Version 3 (RFC 1813), and Version 4 (RFC 3530)
+ * and various protocol-related implementation definitions.
  */
 
 /* Only define these if nfs_prot.h hasn't been included */
@@ -83,51 +85,94 @@
 #define	NFS_PROG	100003
 #define NFS_VER2	2
 #define	NFS_VER3	3
+#define	NFS_VER4	4
 #define NFS_V2MAXDATA	8192
 #define	NFS_MAXDGRAMDATA 16384
 #define	NFS_PREFDGRAMDATA 8192
-#define	NFS_MAXDATA	(60*1024) // XXX not ready for 64K-128K
+#define	NFS_MAXDATA	(64*1024) // XXX not ready for >64K
 #define	NFS_MAXPATHLEN	1024
 #define	NFS_MAXNAMLEN	255
 #define	NFS_MAXPKTHDR	404
 #define NFS_MAXPACKET	(NFS_MAXPKTHDR + NFS_MAXDATA)
 #define	NFS_MINPACKET	20
-#define	NFS_MAXSOCKBUF	(224*1024)
+#define	NFS_UDPSOCKBUF	(224*1024)
 #define	NFS_FABLKSIZE	512	/* Size in bytes of a block wrt fa_blocks */
 
-/* Stat numbers for rpc returns (version 2 and 3) */
-#define	NFS_OK			0
-#define	NFSERR_PERM		1
-#define	NFSERR_NOENT		2
-#define	NFSERR_IO		5
-#define	NFSERR_NXIO		6
-#define	NFSERR_ACCES		13
-#define	NFSERR_EXIST		17
-#define	NFSERR_XDEV		18	/* Version 3 only */
-#define	NFSERR_NODEV		19
-#define	NFSERR_NOTDIR		20
-#define	NFSERR_ISDIR		21
-#define	NFSERR_INVAL		22	/* Version 3 only */
-#define	NFSERR_FBIG		27
-#define	NFSERR_NOSPC		28
-#define	NFSERR_ROFS		30
-#define	NFSERR_MLINK		31	/* Version 3 only */
-#define	NFSERR_NAMETOL		63
-#define	NFSERR_NOTEMPTY		66
-#define	NFSERR_DQUOT		69
-#define	NFSERR_STALE		70
-#define	NFSERR_REMOTE		71	/* Version 3 only */
-#define	NFSERR_WFLUSH		99	/* Version 2 only */
-#define	NFSERR_BADHANDLE	10001	/* The rest Version 3 only */
-#define	NFSERR_NOT_SYNC		10002
-#define	NFSERR_BAD_COOKIE	10003
-#define	NFSERR_NOTSUPP		10004
-#define	NFSERR_TOOSMALL		10005
-#define	NFSERR_SERVERFAULT	10006
-#define	NFSERR_BADTYPE		10007
-#define	NFSERR_JUKEBOX		10008
-#define NFSERR_TRYLATER		NFSERR_JUKEBOX
-#define	NFSERR_STALEWRITEVERF	30001	/* Fake return for nfs_commit() */
+/* Stat numbers for NFS RPC returns */
+#define NFS_OK				0
+#define NFSERR_PERM			1
+#define NFSERR_NOENT			2
+#define NFSERR_IO			5
+#define NFSERR_NXIO			6
+#define NFSERR_ACCES			13
+#define NFSERR_EXIST			17
+#define NFSERR_XDEV			18	/* Version 3 only */
+#define NFSERR_NODEV			19
+#define NFSERR_NOTDIR			20
+#define NFSERR_ISDIR			21
+#define NFSERR_INVAL			22	/* Version 3 only */
+#define NFSERR_FBIG			27
+#define NFSERR_NOSPC			28
+#define NFSERR_ROFS			30
+#define NFSERR_MLINK			31	/* Version 3 only */
+#define NFSERR_NAMETOL			63
+#define NFSERR_NOTEMPTY			66
+#define NFSERR_DQUOT			69
+#define NFSERR_STALE			70
+#define NFSERR_REMOTE			71	/* Version 3 only */
+#define NFSERR_WFLUSH			99	/* Version 2 only */
+#define NFSERR_BADHANDLE		10001	/* The rest Version 3 only */
+#define NFSERR_NOT_SYNC			10002
+#define NFSERR_BAD_COOKIE		10003
+#define NFSERR_NOTSUPP			10004
+#define NFSERR_TOOSMALL			10005
+#define NFSERR_SERVERFAULT		10006
+#define NFSERR_BADTYPE			10007
+#define NFSERR_JUKEBOX			10008
+#define NFSERR_TRYLATER			NFSERR_JUKEBOX
+#define NFSERR_DELAY			NFSERR_JUKEBOX
+#define NFSERR_SAME			10009	/* The rest Version 4 only */
+#define NFSERR_DENIED			10010
+#define NFSERR_EXPIRED			10011
+#define NFSERR_LOCKED			10012
+#define NFSERR_GRACE			10013
+#define NFSERR_FHEXPIRED		10014
+#define NFSERR_SHARE_DENIED		10015
+#define NFSERR_WRONGSEC			10016
+#define NFSERR_CLID_INUSE		10017
+#define NFSERR_RESOURCE			10018
+#define NFSERR_MOVED			10019
+#define NFSERR_NOFILEHANDLE		10020
+#define NFSERR_MINOR_VERS_MISMATCH	10021
+#define NFSERR_STALE_CLIENTID		10022
+#define NFSERR_STALE_STATEID		10023
+#define NFSERR_OLD_STATEID		10024
+#define NFSERR_BAD_STATEID		10025
+#define NFSERR_BAD_SEQID		10026
+#define NFSERR_NOT_SAME			10027
+#define NFSERR_LOCK_RANGE		10028
+#define NFSERR_SYMLINK			10029
+#define NFSERR_RESTOREFH		10030
+#define NFSERR_LEASE_MOVED		10031
+#define NFSERR_ATTRNOTSUPP		10032
+#define NFSERR_NO_GRACE			10033
+#define NFSERR_RECLAIM_BAD		10034
+#define NFSERR_RECLAIM_CONFLICT		10035
+#define NFSERR_BADXDR			10036
+#define NFSERR_LOCKS_HELD		10037
+#define NFSERR_OPENMODE			10038
+#define NFSERR_BADOWNER			10039
+#define NFSERR_BADCHAR			10040
+#define NFSERR_BADNAME			10041
+#define NFSERR_BAD_RANGE		10042
+#define NFSERR_LOCK_NOTSUPP		10043
+#define NFSERR_OP_ILLEGAL		10044
+#define NFSERR_DEADLOCK			10045
+#define NFSERR_FILE_OPEN		10046
+#define NFSERR_ADMIN_REVOKED		10047
+#define NFSERR_CB_PATH_DOWN		10048
+
+#define NFSERR_STALEWRITEVERF		30001	/* Fake return for nfs_commit() */
 
 #define NFSERR_RETVOID		0x20000000 /* Return void, not error */
 #define NFSERR_AUTHERR		0x40000000 /* Mark an authentication error */
@@ -158,23 +203,27 @@
 #define NFSX_V3FSINFO		48
 #define NFSX_V3PATHCONF		24
 
-/* variants for both versions */
-#define NFSX_FH(v3)		((v3) ? (NFSX_V3FHMAX + NFSX_UNSIGNED) : \
-					NFSX_V2FH)
-#define NFSX_SRVFH(v3,FH)	((v3) ? (FH)->nfh_len : NFSX_V2FH)
-#define	NFSX_FATTR(v3)		((v3) ? NFSX_V3FATTR : NFSX_V2FATTR)
-#define NFSX_PREOPATTR(v3)	((v3) ? (7 * NFSX_UNSIGNED) : 0)
-#define NFSX_POSTOPATTR(v3)	((v3) ? (NFSX_V3FATTR + NFSX_UNSIGNED) : 0)
-#define NFSX_POSTOPORFATTR(v3)	((v3) ? (NFSX_V3FATTR + NFSX_UNSIGNED) : \
-					NFSX_V2FATTR)
-#define NFSX_WCCDATA(v3)	((v3) ? NFSX_V3WCCDATA : 0)
-#define NFSX_WCCORFATTR(v3)	((v3) ? NFSX_V3WCCDATA : NFSX_V2FATTR)
-#define	NFSX_SATTR(v3)		((v3) ? NFSX_V3SATTR : NFSX_V2SATTR)
-#define	NFSX_COOKIEVERF(v3)	((v3) ? NFSX_V3COOKIEVERF : 0)
-#define	NFSX_WRITEVERF(v3)	((v3) ? NFSX_V3WRITEVERF : 0)
-#define NFSX_READDIR(v3)	((v3) ? (5 * NFSX_UNSIGNED) : \
+/* specific to NFS Version 4 */
+#define NFS4_FHSIZE		128
+#define NFS4_VERIFIER_SIZE	8
+#define NFS4_OPAQUE_LIMIT	1024
+
+/* variants for multiple versions */
+#define NFSX_FH(V)		(((V) == NFS_VER2) ? NFSX_V2FH : (NFSX_UNSIGNED + \
+				 (((V) == NFS_VER3) ? NFSX_V3FHMAX : NFS4_FHSIZE)))
+#define NFSX_SRVFH(V,FH)	(((V) == NFS_VER2) ? NFSX_V2FH : (FH)->nfh_len)
+#define	NFSX_FATTR(V)		(((V) == NFS_VER3) ? NFSX_V3FATTR : NFSX_V2FATTR)
+#define NFSX_PREOPATTR(V)	(((V) == NFS_VER3) ? (7 * NFSX_UNSIGNED) : 0)
+#define NFSX_POSTOPATTR(V)	(((V) == NFS_VER3) ? (NFSX_V3FATTR + NFSX_UNSIGNED) : 0)
+#define NFSX_POSTOPORFATTR(V)	(((V) == NFS_VER3) ? (NFSX_V3FATTR + NFSX_UNSIGNED) : NFSX_V2FATTR)
+#define NFSX_WCCDATA(V)		(((V) == NFS_VER3) ? NFSX_V3WCCDATA : 0)
+#define NFSX_WCCORFATTR(V)	(((V) == NFS_VER3) ? NFSX_V3WCCDATA : NFSX_V2FATTR)
+#define	NFSX_SATTR(V)		(((V) == NFS_VER3) ? NFSX_V3SATTR : NFSX_V2SATTR)
+#define	NFSX_COOKIEVERF(V)	(((V) == NFS_VER3) ? NFSX_V3COOKIEVERF : 0)
+#define	NFSX_WRITEVERF(V)	(((V) == NFS_VER3) ? NFSX_V3WRITEVERF : 0)
+#define NFSX_READDIR(V)		(((V) == NFS_VER3) ? (5 * NFSX_UNSIGNED) : \
 					(2 * NFSX_UNSIGNED))
-#define	NFSX_STATFS(v3)		((v3) ? NFSX_V3STATFS : NFSX_V2STATFS)
+#define	NFSX_STATFS(V)		(((V) == NFS_VER3) ? NFSX_V3STATFS : NFSX_V2STATFS)
 
 /* Only define these if nfs_prot.h hasn't been included */
 #ifndef NFS_PROGRAM
@@ -232,72 +281,473 @@
 /*
  * Constants used by the Version 3 protocol for various RPCs
  */
-#define NFSV3SATTRTIME_DONTCHANGE	0
-#define NFSV3SATTRTIME_TOSERVER		1
-#define NFSV3SATTRTIME_TOCLIENT		2
-
-#define NFSV3ACCESS_READ		0x01
-#define NFSV3ACCESS_LOOKUP		0x02
-#define NFSV3ACCESS_MODIFY		0x04
-#define NFSV3ACCESS_EXTEND		0x08
-#define NFSV3ACCESS_DELETE		0x10
-#define NFSV3ACCESS_EXECUTE		0x20
-
-#define NFSV3WRITE_UNSTABLE		0
-#define NFSV3WRITE_DATASYNC		1
-#define NFSV3WRITE_FILESYNC		2
-
-#define NFSV3CREATE_UNCHECKED		0
-#define NFSV3CREATE_GUARDED		1
-#define NFSV3CREATE_EXCLUSIVE		2
 
 #define NFSV3FSINFO_LINK		0x01
 #define NFSV3FSINFO_SYMLINK		0x02
 #define NFSV3FSINFO_HOMOGENEOUS		0x08
 #define NFSV3FSINFO_CANSETTIME		0x10
 
-/* Conversion macros */
-#define	vtonfsv2_mode(t,m) \
-		txdr_unsigned(((t) == VFIFO) ? vnode_makeimode(VCHR, (m)) : \
-				vnode_makeimode((t), (m)))
-#define vtonfsv3_mode(m)	txdr_unsigned((m) & 07777)
-#define	nfstov_mode(a)		(fxdr_unsigned(u_short, (a))&07777)
-#define	vtonfsv2_type(a)	txdr_unsigned(nfsv2_type[((long)(a))])
-#define	vtonfsv3_type(a)	txdr_unsigned(nfsv3_type[((long)(a))])
-#define	nfsv2tov_type(a)	nv2tov_type[fxdr_unsigned(u_long,(a))&0x7]
-#define	nfsv3tov_type(a)	nv3tov_type[fxdr_unsigned(u_long,(a))&0x7]
+/* time setting constants */
+#define NFS_TIME_DONT_CHANGE		0
+#define NFS_TIME_SET_TO_SERVER		1
+#define NFS_TIME_SET_TO_CLIENT		2
+
+/* access() constants */
+#define NFS_ACCESS_READ			0x01
+#define NFS_ACCESS_LOOKUP		0x02
+#define NFS_ACCESS_MODIFY		0x04
+#define NFS_ACCESS_EXTEND		0x08
+#define NFS_ACCESS_DELETE		0x10
+#define NFS_ACCESS_EXECUTE		0x20
+#define	NFS_ACCESS_ALL (NFS_ACCESS_READ | NFS_ACCESS_MODIFY		\
+			 | NFS_ACCESS_EXTEND | NFS_ACCESS_EXECUTE	\
+			 | NFS_ACCESS_DELETE | NFS_ACCESS_LOOKUP)
+
+/* NFS WRITE how constants */
+#define NFS_WRITE_UNSTABLE		0
+#define NFS_WRITE_DATASYNC		1
+#define NFS_WRITE_FILESYNC		2
+
+/* NFS CREATE types */
+#define NFS_CREATE_UNCHECKED		0
+#define NFS_CREATE_GUARDED		1
+#define NFS_CREATE_EXCLUSIVE		2
 
 /* Only define these if nfs_prot.h hasn't been included */
 #ifndef NFS_PROGRAM
-
-/* File types */
+/* NFS object types */
 typedef enum { NFNON=0, NFREG=1, NFDIR=2, NFBLK=3, NFCHR=4, NFLNK=5,
-	NFSOCK=6, NFFIFO=7 } nfstype;
+	NFSOCK=6, NFFIFO=7, NFATTRDIR=8, NFNAMEDATTR=9 } nfstype;
 #endif /* !NFS_PROGRAM */
 
-/* Structs for common parts of the rpc's */
 /*
- * File Handle (32 bytes for version 2), variable up to 64 for version 3.
+ * File Handle (32 bytes for version 2), variable up to 64 for version 3
+ * and variable up to 128 bytes for version 4.
  * File Handles of up to NFS_SMALLFH in size are stored directly in the
  * nfs node, whereas larger ones are malloc'd. (This never happens when
- * NFS_SMALLFH is set to 64.)
+ * NFS_SMALLFH is set to the largest size.)
  * NFS_SMALLFH should be in the range of 32 to 64 and be divisible by 4.
  */
 #ifndef NFS_SMALLFH
 #define NFS_SMALLFH	64
 #endif
 
-struct nfsv2_time {
-	u_long	nfsv2_sec;
-	u_long	nfsv2_usec;
-};
-typedef struct nfsv2_time	nfstime2;
+/*
+ * NFS attribute management stuff
+ */
+#define NFS_ATTR_BITMAP_LEN	2
+#define NFS_BITMAP_SET(A, I)	(((uint32_t *)(A))[(I)/32] |= 1<<((I)%32))
+#define NFS_BITMAP_CLR(A, I)	(((uint32_t *)(A))[(I)/32] &= ~(1<<((I)%32)))
+#define NFS_BITMAP_ISSET(A, I)	(((uint32_t *)(A))[(I)/32] & (1<<((I)%32)))
+
+__private_extern__ uint32_t nfs_fs_attr_bitmap[NFS_ATTR_BITMAP_LEN];
+__private_extern__ uint32_t nfs_object_attr_bitmap[NFS_ATTR_BITMAP_LEN];
+__private_extern__ uint32_t nfs_getattr_bitmap[NFS_ATTR_BITMAP_LEN];
+
+#define NFS_CLEAR_ATTRIBUTES(A) \
+	do { \
+	int __i; \
+	for (__i=0; __i < NFS_ATTR_BITMAP_LEN; __i++) \
+		((uint32_t*)(A))[__i] = 0; \
+	} while (0)
+
+#define NFS_COPY_ATTRIBUTES(SRC, DST) \
+	do { \
+	int __i; \
+	for (__i=0; __i < NFS_ATTR_BITMAP_LEN; __i++) \
+		((uint32_t*)(DST))[__i] = ((uint32_t*)(SRC))[__i]; \
+	} while (0)
+
+/* NFS attributes */
+#define NFS_FATTR_SUPPORTED_ATTRS		0
+#define NFS_FATTR_TYPE				1
+#define NFS_FATTR_FH_EXPIRE_TYPE		2
+#define NFS_FATTR_CHANGE			3
+#define NFS_FATTR_SIZE				4
+#define NFS_FATTR_LINK_SUPPORT			5
+#define NFS_FATTR_SYMLINK_SUPPORT		6
+#define NFS_FATTR_NAMED_ATTR			7
+#define NFS_FATTR_FSID				8
+#define NFS_FATTR_UNIQUE_HANDLES		9
+#define NFS_FATTR_LEASE_TIME			10
+#define NFS_FATTR_RDATTR_ERROR			11
+#define NFS_FATTR_FILEHANDLE			19
+#define NFS_FATTR_ACL				12
+#define NFS_FATTR_ACLSUPPORT			13
+#define NFS_FATTR_ARCHIVE			14
+#define NFS_FATTR_CANSETTIME			15
+#define NFS_FATTR_CASE_INSENSITIVE		16
+#define NFS_FATTR_CASE_PRESERVING		17
+#define NFS_FATTR_CHOWN_RESTRICTED		18
+#define NFS_FATTR_FILEID			20
+#define NFS_FATTR_FILES_AVAIL			21
+#define NFS_FATTR_FILES_FREE			22
+#define NFS_FATTR_FILES_TOTAL			23
+#define NFS_FATTR_FS_LOCATIONS			24
+#define NFS_FATTR_HIDDEN			25
+#define NFS_FATTR_HOMOGENEOUS			26
+#define NFS_FATTR_MAXFILESIZE			27
+#define NFS_FATTR_MAXLINK			28
+#define NFS_FATTR_MAXNAME			29
+#define NFS_FATTR_MAXREAD			30
+#define NFS_FATTR_MAXWRITE			31
+#define NFS_FATTR_MIMETYPE			32
+#define NFS_FATTR_MODE				33
+#define NFS_FATTR_NO_TRUNC			34
+#define NFS_FATTR_NUMLINKS			35
+#define NFS_FATTR_OWNER				36
+#define NFS_FATTR_OWNER_GROUP			37
+#define NFS_FATTR_QUOTA_AVAIL_HARD		38
+#define NFS_FATTR_QUOTA_AVAIL_SOFT		39
+#define NFS_FATTR_QUOTA_USED			40
+#define NFS_FATTR_RAWDEV			41
+#define NFS_FATTR_SPACE_AVAIL			42
+#define NFS_FATTR_SPACE_FREE			43
+#define NFS_FATTR_SPACE_TOTAL			44
+#define NFS_FATTR_SPACE_USED			45
+#define NFS_FATTR_SYSTEM			46
+#define NFS_FATTR_TIME_ACCESS			47
+#define NFS_FATTR_TIME_ACCESS_SET		48
+#define NFS_FATTR_TIME_BACKUP			49
+#define NFS_FATTR_TIME_CREATE			50
+#define NFS_FATTR_TIME_DELTA			51
+#define NFS_FATTR_TIME_METADATA			52
+#define NFS_FATTR_TIME_MODIFY			53
+#define NFS_FATTR_TIME_MODIFY_SET		54
+#define NFS_FATTR_MOUNTED_ON_FILEID		55
+
+#define NFS4_ALL_ATTRIBUTES(A) \
+	do { \
+	/* required: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_SUPPORTED_ATTRS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FH_EXPIRE_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHANGE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SYMLINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NAMED_ATTR); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FSID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_UNIQUE_HANDLES); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LEASE_TIME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_RDATTR_ERROR); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILEHANDLE); \
+	/* optional: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_ACL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_ACLSUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_ARCHIVE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CANSETTIME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_INSENSITIVE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_PRESERVING); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHOWN_RESTRICTED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILEID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FS_LOCATIONS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_HIDDEN); \
+	NFS_BITMAP_SET((A), NFS_FATTR_HOMOGENEOUS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXFILESIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXLINK); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXNAME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXREAD); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXWRITE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MIMETYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MODE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NO_TRUNC); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NUMLINKS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER_GROUP); \
+	NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_AVAIL_HARD); \
+	NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_AVAIL_SOFT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_USED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_RAWDEV); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_USED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SYSTEM); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_ACCESS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_ACCESS_SET); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_BACKUP); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_CREATE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_DELTA); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_METADATA); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_MODIFY); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_MODIFY_SET); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MOUNTED_ON_FILEID); \
+	} while (0)
+
+#define NFS4_PER_OBJECT_ATTRIBUTES(A) \
+	do { \
+	/* required: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHANGE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NAMED_ATTR); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FSID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_RDATTR_ERROR); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILEHANDLE); \
+	/* optional: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_ACL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_ARCHIVE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILEID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_HIDDEN); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXLINK); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MIMETYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MODE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NUMLINKS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER_GROUP); \
+	NFS_BITMAP_SET((A), NFS_FATTR_RAWDEV); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_USED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SYSTEM); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_ACCESS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_BACKUP); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_CREATE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_METADATA); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_MODIFY); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MOUNTED_ON_FILEID); \
+	} while (0)
+
+#define NFS4_PER_FS_ATTRIBUTES(A) \
+	do { \
+	/* required: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_SUPPORTED_ATTRS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FH_EXPIRE_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SYMLINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_UNIQUE_HANDLES); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LEASE_TIME); \
+	/* optional: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_ACLSUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CANSETTIME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_INSENSITIVE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_PRESERVING); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHOWN_RESTRICTED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FS_LOCATIONS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_HOMOGENEOUS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXFILESIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXNAME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXREAD); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXWRITE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NO_TRUNC); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_DELTA); \
+	} while (0)
+
+#define NFS4_DEFAULT_ATTRIBUTES(A) \
+	do { \
+	/* required: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_SUPPORTED_ATTRS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FH_EXPIRE_TYPE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHANGE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SYMLINK_SUPPORT); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NAMED_ATTR); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FSID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_UNIQUE_HANDLES); \
+	NFS_BITMAP_SET((A), NFS_FATTR_LEASE_TIME); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_RDATTR_ERROR); */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_FILEHANDLE); */ \
+	/* optional: */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_ACL); */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_ACLSUPPORT); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_ARCHIVE); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_CANSETTIME); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_INSENSITIVE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CASE_PRESERVING); \
+	NFS_BITMAP_SET((A), NFS_FATTR_CHOWN_RESTRICTED); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILEID); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_TOTAL); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_FS_LOCATIONS); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_HIDDEN); \
+	NFS_BITMAP_SET((A), NFS_FATTR_HOMOGENEOUS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXFILESIZE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXLINK); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXNAME); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXREAD); \
+	NFS_BITMAP_SET((A), NFS_FATTR_MAXWRITE); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_MIMETYPE); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_MODE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NO_TRUNC); \
+	NFS_BITMAP_SET((A), NFS_FATTR_NUMLINKS); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER); \
+	NFS_BITMAP_SET((A), NFS_FATTR_OWNER_GROUP); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_AVAIL_HARD); */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_AVAIL_SOFT); */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_QUOTA_USED); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_RAWDEV); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_USED); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_SYSTEM); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_ACCESS); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_TIME_ACCESS_SET); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_BACKUP); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_CREATE); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_TIME_DELTA); */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_METADATA); \
+	NFS_BITMAP_SET((A), NFS_FATTR_TIME_MODIFY); \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_TIME_MODIFY_SET); */ \
+	/* NFS_BITMAP_SET((A), NFS_FATTR_MOUNTED_ON_FILEID); */ \
+	} while (0)
+
+/* attributes requested when we want to do a "statfs" */
+#define NFS4_STATFS_ATTRIBUTES(A) \
+	do { \
+	/* optional: */ \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_FILES_TOTAL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_AVAIL); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_FREE); \
+	NFS_BITMAP_SET((A), NFS_FATTR_SPACE_TOTAL); \
+	} while (0)
+
+/*
+ * NFS OPEN constants
+ */
+/* open type */
+#define NFS_OPEN_NOCREATE			0
+#define NFS_OPEN_CREATE				1
+/* delegation space limit */
+#define NFS_LIMIT_SIZE				1
+#define NFS_LIMIT_BLOCKS			2
+/* access/deny modes */
+#define NFS_OPEN_SHARE_ACCESS_READ		0x00000001
+#define NFS_OPEN_SHARE_ACCESS_WRITE		0x00000002
+#define NFS_OPEN_SHARE_ACCESS_BOTH		0x00000003
+#define NFS_OPEN_SHARE_DENY_NONE		0x00000000
+#define NFS_OPEN_SHARE_DENY_READ		0x00000001
+#define NFS_OPEN_SHARE_DENY_WRITE		0x00000002
+#define NFS_OPEN_SHARE_DENY_BOTH		0x00000003
+/* delegation types */
+#define NFS_OPEN_DELEGATE_NONE			0
+#define NFS_OPEN_DELEGATE_READ			1
+#define NFS_OPEN_DELEGATE_WRITE			2
+/* delegation claim types */
+#define NFS_CLAIM_NULL				0
+#define NFS_CLAIM_PREVIOUS			1
+#define NFS_CLAIM_DELEGATE_CUR			2
+#define NFS_CLAIM_DELEGATE_PREV			3
+/* open result flags */
+#define NFS_OPEN_RESULT_CONFIRM			0x00000002
+#define NFS_OPEN_RESULT_LOCKTYPE_POSIX		0x00000004
+/* NFS lock types */
+#define NFS_LOCK_TYPE_READ			1
+#define NFS_LOCK_TYPE_WRITE			2
+#define NFS_LOCK_TYPE_READW			3 /* "blocking" */
+#define NFS_LOCK_TYPE_WRITEW			4 /* "blocking" */
+
+/* NFSv4 RPC procedures */
+#define NFSPROC4_NULL				0
+#define NFSPROC4_COMPOUND			1
+
+/* NFSv4 opcodes */
+#define NFS_OP_ACCESS				3
+#define NFS_OP_CLOSE				4
+#define NFS_OP_COMMIT				5
+#define NFS_OP_CREATE				6
+#define NFS_OP_DELEGPURGE			7
+#define NFS_OP_DELEGRETURN			8
+#define NFS_OP_GETATTR				9
+#define NFS_OP_GETFH				10
+#define NFS_OP_LINK				11
+#define NFS_OP_LOCK				12
+#define NFS_OP_LOCKT				13
+#define NFS_OP_LOCKU				14
+#define NFS_OP_LOOKUP				15
+#define NFS_OP_LOOKUPP				16
+#define NFS_OP_NVERIFY				17
+#define NFS_OP_OPEN				18
+#define NFS_OP_OPENATTR				19
+#define NFS_OP_OPEN_CONFIRM			20
+#define NFS_OP_OPEN_DOWNGRADE			21
+#define NFS_OP_PUTFH				22
+#define NFS_OP_PUTPUBFH				23
+#define NFS_OP_PUTROOTFH			24
+#define NFS_OP_READ				25
+#define NFS_OP_READDIR				26
+#define NFS_OP_READLINK				27
+#define NFS_OP_REMOVE				28
+#define NFS_OP_RENAME				29
+#define NFS_OP_RENEW				30
+#define NFS_OP_RESTOREFH			31
+#define NFS_OP_SAVEFH				32
+#define NFS_OP_SECINFO				33
+#define NFS_OP_SETATTR				34
+#define NFS_OP_SETCLIENTID			35
+#define NFS_OP_SETCLIENTID_CONFIRM		36
+#define NFS_OP_VERIFY				37
+#define NFS_OP_WRITE				38
+#define NFS_OP_RELEASE_LOCKOWNER		39
+#define NFS_OP_ILLEGAL				10044
+/* NFSv4 callback opcodes */
+#define NFS_OP_CB_GETATTR			3
+#define NFS_OP_CB_RECALL			4
+#define NFS_OP_CB_ILLEGAL			10044
+
+/* NFSv4 file handle type flags */
+#define NFS_FH_PERSISTENT			0x00000000
+#define NFS_FH_NOEXPIRE_WITH_OPEN		0x00000001
+#define NFS_FH_VOLATILE_ANY			0x00000002
+#define NFS_FH_VOL_MIGRATION			0x00000004
+#define NFS_FH_VOL_RENAME			0x00000008
+
+/*
+ * NFSv4 ACL constants
+ */
+/* ACE support mask bits */
+#define NFS_ACL_SUPPORT_ALLOW_ACL		0x00000001
+#define NFS_ACL_SUPPORT_DENY_ACL		0x00000002
+#define NFS_ACL_SUPPORT_AUDIT_ACL		0x00000004
+#define NFS_ACL_SUPPORT_ALARM_ACL		0x00000008
+/* ACE types */
+#define NFS_ACE_ACCESS_ALLOWED_ACE_TYPE		0x00000000
+#define NFS_ACE_ACCESS_DENIED_ACE_TYPE		0x00000001
+#define NFS_ACE_SYSTEM_AUDIT_ACE_TYPE		0x00000002
+#define NFS_ACE_SYSTEM_ALARM_ACE_TYPE		0x00000003
+/* ACE flags */
+#define NFS_ACE_FILE_INHERIT_ACE		0x00000001
+#define NFS_ACE_DIRECTORY_INHERIT_ACE		0x00000002
+#define NFS_ACE_NO_PROPAGATE_INHERIT_ACE	0x00000004
+#define NFS_ACE_INHERIT_ONLY_ACE		0x00000008
+#define NFS_ACE_SUCCESSFUL_ACCESS_ACE_FLAG	0x00000010
+#define NFS_ACE_FAILED_ACCESS_ACE_FLAG		0x00000020
+#define NFS_ACE_IDENTIFIER_GROUP		0x00000040
+/* ACE mask flags */
+#define NFS_ACE_READ_DATA			0x00000001
+#define NFS_ACE_LIST_DIRECTORY			0x00000001
+#define NFS_ACE_WRITE_DATA			0x00000002
+#define NFS_ACE_ADD_FILE			0x00000002
+#define NFS_ACE_APPEND_DATA			0x00000004
+#define NFS_ACE_ADD_SUBDIRECTORY		0x00000004
+#define NFS_ACE_READ_NAMED_ATTRS		0x00000008
+#define NFS_ACE_WRITE_NAMED_ATTRS		0x00000010
+#define NFS_ACE_EXECUTE				0x00000020
+#define NFS_ACE_DELETE_CHILD			0x00000040
+#define NFS_ACE_READ_ATTRIBUTES			0x00000080
+#define NFS_ACE_WRITE_ATTRIBUTES		0x00000100
+#define NFS_ACE_DELETE				0x00010000
+#define NFS_ACE_READ_ACL			0x00020000
+#define NFS_ACE_WRITE_ACL			0x00040000
+#define NFS_ACE_WRITE_OWNER			0x00080000
+#define NFS_ACE_SYNCHRONIZE			0x00100000
+#define NFS_ACE_GENERIC_READ			0x00120081
+#define NFS_ACE_GENERIC_WRITE			0x00160106
+#define NFS_ACE_GENERIC_EXECUTE			0x001200A0
 
-struct nfsv3_time {
-	u_long	nfsv3_sec;
-	u_long	nfsv3_nsec;
-};
-typedef struct nfsv3_time	nfstime3;
 
 /*
  * Quads are defined as arrays of 2 longs to ensure dense packing for the
@@ -318,73 +768,32 @@ union nfs_quadconvert {
 typedef union nfs_quadconvert	nfsquad_t;
 
 /*
- * NFS Version 3 special file number.
+ * special data/attribute associated with NFBLK/NFCHR
  */
-struct nfsv3_spec {
-	u_long	specdata1;
-	u_long	specdata2;
+struct nfs_specdata {
+	uint32_t specdata1;	/* major device number */
+	uint32_t specdata2;	/* minor device number */
 };
-typedef	struct nfsv3_spec	nfsv3spec;
+typedef struct nfs_specdata nfs_specdata;
 
 /*
- * File attributes and setable attributes. These structures cover both
- * NFS version 2 and the version 3 protocol. Note that the union is only
- * used so that one pointer can refer to both variants. These structures
- * go out on the wire and must be densely packed, so no quad data types
- * are used. (all fields are longs or u_longs or structures of same)
- * NB: You can't do sizeof(struct nfs_fattr), you must use the
- *     NFSX_FATTR(v3) macro.
+ * an "fsid" large enough to hold an NFSv4 fsid.
  */
-struct nfs_fattr {
-	u_long	fa_type;
-	u_long	fa_mode;
-	u_long	fa_nlink;
-	u_long	fa_uid;
-	u_long	fa_gid;
-	union {
-		struct {
-			u_long		nfsv2fa_size;
-			u_long		nfsv2fa_blocksize;
-			u_long		nfsv2fa_rdev;
-			u_long		nfsv2fa_blocks;
-			u_long		nfsv2fa_fsid;
-			u_long		nfsv2fa_fileid;
-			nfstime2	nfsv2fa_atime;
-			nfstime2	nfsv2fa_mtime;
-			nfstime2	nfsv2fa_ctime;
-		} fa_nfsv2;
-		struct {
-			nfsuint64	nfsv3fa_size;
-			nfsuint64	nfsv3fa_used;
-			nfsv3spec	nfsv3fa_rdev;
-			nfsuint64	nfsv3fa_fsid;
-			nfsuint64	nfsv3fa_fileid;
-			nfstime3	nfsv3fa_atime;
-			nfstime3	nfsv3fa_mtime;
-			nfstime3	nfsv3fa_ctime;
-		} fa_nfsv3;
-	} fa_un;
+struct nfs_fsid {
+	uint64_t major;
+	uint64_t minor;
 };
+typedef struct nfs_fsid nfs_fsid;
 
-/* and some ugly defines for accessing union components */
-#define	fa2_size		fa_un.fa_nfsv2.nfsv2fa_size
-#define	fa2_blocksize		fa_un.fa_nfsv2.nfsv2fa_blocksize
-#define	fa2_rdev		fa_un.fa_nfsv2.nfsv2fa_rdev
-#define	fa2_blocks		fa_un.fa_nfsv2.nfsv2fa_blocks
-#define	fa2_fsid		fa_un.fa_nfsv2.nfsv2fa_fsid
-#define	fa2_fileid		fa_un.fa_nfsv2.nfsv2fa_fileid
-#define	fa2_atime		fa_un.fa_nfsv2.nfsv2fa_atime
-#define	fa2_mtime		fa_un.fa_nfsv2.nfsv2fa_mtime
-#define	fa2_ctime		fa_un.fa_nfsv2.nfsv2fa_ctime
-#define	fa3_size		fa_un.fa_nfsv3.nfsv3fa_size
-#define	fa3_used		fa_un.fa_nfsv3.nfsv3fa_used
-#define	fa3_rdev		fa_un.fa_nfsv3.nfsv3fa_rdev
-#define	fa3_fsid		fa_un.fa_nfsv3.nfsv3fa_fsid
-#define	fa3_fileid		fa_un.fa_nfsv3.nfsv3fa_fileid
-#define	fa3_atime		fa_un.fa_nfsv3.nfsv3fa_atime
-#define	fa3_mtime		fa_un.fa_nfsv3.nfsv3fa_mtime
-#define	fa3_ctime		fa_un.fa_nfsv3.nfsv3fa_ctime
 
+/*
+ * remove these once we're sure nobody's using them
+ */
+struct nfsv2_time {
+	u_long	nfsv2_sec;
+	u_long	nfsv2_usec;
+};
+typedef struct nfsv2_time	nfstime2;
 struct nfsv2_sattr {
 	u_long		sa_mode;
 	u_long		sa_uid;
@@ -394,61 +803,5 @@ struct nfsv2_sattr {
 	nfstime2	sa_mtime;
 };
 
-struct nfs_statfs {
-	union {
-		struct {
-			u_long		nfsv2sf_tsize;
-			u_long		nfsv2sf_bsize;
-			u_long		nfsv2sf_blocks;
-			u_long		nfsv2sf_bfree;
-			u_long		nfsv2sf_bavail;
-		} sf_nfsv2;
-		struct {
-			nfsuint64	nfsv3sf_tbytes;
-			nfsuint64	nfsv3sf_fbytes;
-			nfsuint64	nfsv3sf_abytes;
-			nfsuint64	nfsv3sf_tfiles;
-			nfsuint64	nfsv3sf_ffiles;
-			nfsuint64	nfsv3sf_afiles;
-			u_long		nfsv3sf_invarsec;
-		} sf_nfsv3;
-	} sf_un;
-};
-
-#define sf_tsize	sf_un.sf_nfsv2.nfsv2sf_tsize
-#define sf_bsize	sf_un.sf_nfsv2.nfsv2sf_bsize
-#define sf_blocks	sf_un.sf_nfsv2.nfsv2sf_blocks
-#define sf_bfree	sf_un.sf_nfsv2.nfsv2sf_bfree
-#define sf_bavail	sf_un.sf_nfsv2.nfsv2sf_bavail
-#define sf_tbytes	sf_un.sf_nfsv3.nfsv3sf_tbytes
-#define sf_fbytes	sf_un.sf_nfsv3.nfsv3sf_fbytes
-#define sf_abytes	sf_un.sf_nfsv3.nfsv3sf_abytes
-#define sf_tfiles	sf_un.sf_nfsv3.nfsv3sf_tfiles
-#define sf_ffiles	sf_un.sf_nfsv3.nfsv3sf_ffiles
-#define sf_afiles	sf_un.sf_nfsv3.nfsv3sf_afiles
-#define sf_invarsec	sf_un.sf_nfsv3.nfsv3sf_invarsec
-
-struct nfsv3_fsinfo {
-	u_long		fs_rtmax;
-	u_long		fs_rtpref;
-	u_long		fs_rtmult;
-	u_long		fs_wtmax;
-	u_long		fs_wtpref;
-	u_long		fs_wtmult;
-	u_long		fs_dtpref;
-	nfsuint64	fs_maxfilesize;
-	nfstime3	fs_timedelta;
-	u_long		fs_properties;
-};
-
-struct nfsv3_pathconf {
-	u_long		pc_linkmax;
-	u_long		pc_namemax;
-	u_long		pc_notrunc;
-	u_long		pc_chownrestricted;
-	u_long		pc_caseinsensitive;
-	u_long		pc_casepreserving;
-};
-
 #endif /* __APPLE_API_PRIVATE */
 #endif /* _NFS_NFSPROTO_H_ */
diff --git a/bsd/nfs/nfsrtt.h b/bsd/nfs/nfsrtt.h
deleted file mode 100644
index 8e00d78ca..000000000
--- a/bsd/nfs/nfsrtt.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-/*
- * Copyright (c) 1992, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * Rick Macklem at The University of Guelph.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)nfsrtt.h	8.2 (Berkeley) 3/30/95
- * FreeBSD-Id: nfsrtt.h,v 1.8 1997/06/03 17:22:47 dfr Exp $
- */
-
-
-#ifndef _NFS_NFSRTT_H_
-#define _NFS_NFSRTT_H_
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_PRIVATE
-/*
- * Definitions for performance monitor.
- * The client and server logging are turned on by setting the global
- * constant "nfsrtton" to 1.
- */
-#define	NFSRTTLOGSIZ	128
-
-/*
- * Circular log of client side rpc activity. Each log entry is for one
- * rpc filled in upon completion. (ie. in order of completion)
- * The "pos" is the table index for the "next" entry, therefore the
- * list goes from nfsrtt.rttl[pos] --> nfsrtt.rttl[pos - 1] in
- * chronological order of completion.
- */
-struct nfsrtt {
-	int pos;			/* Position in array for next entry */
-	struct rttl {
-		u_int32_t proc;		/* NFS procedure number */
-		int	rtt;		/* Measured round trip time */
-		int	rto;		/* Round Trip Timeout */
-		int	sent;		/* # rpcs in progress */
-		int	cwnd;		/* Send window */
-		int	srtt;		/* Ave Round Trip Time */
-		int	sdrtt;		/* Ave mean deviation of RTT */
-		fsid_t	fsid;		/* Fsid for mount point */
-		struct timeval tstamp;	/* Timestamp of log entry */
-	} rttl[NFSRTTLOGSIZ];
-};
-
-/*
- * And definitions for server side performance monitor.
- * The log organization is the same as above except it is filled in at the
- * time the server sends the rpc reply.
- */
-
-/*
- * Bits for the flags field.
- */
-#define	DRT_TCP		0x02	/* Client used TCP transport */
-#define	DRT_CACHEREPLY	0x04	/* Reply was from recent request cache */
-#define	DRT_CACHEDROP	0x08	/* Rpc request dropped, due to recent reply */
-#define DRT_NFSV3	0x10	/* Rpc used NFS Version 3 */
-
-/*
- * Server log structure
- * NB: ipadr == INADDR_ANY indicates a client using a non IP protocol.
- *	(ISO perhaps?)
- */
-struct nfsdrt {
-	int pos;			/* Position of next log entry */
-	struct drt {
-		int	flag;		/* Bits as defined above */
-		u_int32_t proc;		/* NFS procedure number */
-		u_long	ipadr;		/* IP address of client */
-		int	resptime;	/* Response time (usec) */
-		struct timeval tstamp;	/* Timestamp of log entry */
-	} drt[NFSRTTLOGSIZ];
-};
-
-#endif /* __APPLE_API_PRIVATE */
-#endif /* _NFS_NFSRTT_H_ */
diff --git a/bsd/nfs/nfsrvcache.h b/bsd/nfs/nfsrvcache.h
index b18671041..481dcab81 100644
--- a/bsd/nfs/nfsrvcache.h
+++ b/bsd/nfs/nfsrvcache.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -110,5 +116,8 @@ struct nfsrvcache {
 #define	RC_INETADDR	0x20
 #define	RC_NAM		0x40
 
+__private_extern__ lck_grp_t *nfsrv_reqcache_lck_grp;
+__private_extern__ lck_mtx_t *nfsrv_reqcache_mutex;
+
 #endif /* __APPLE_API_PRIVATE */
 #endif /* _NFS_NFSRVCACHE_H_ */
diff --git a/bsd/nfs/rpcv2.h b/bsd/nfs/rpcv2.h
index 7bd0cca92..510f5110b 100644
--- a/bsd/nfs/rpcv2.h
+++ b/bsd/nfs/rpcv2.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -77,10 +83,14 @@
 /* Authentication */
 #define	RPCAUTH_NULL	0
 #define	RPCAUTH_UNIX	1
+#define RPCAUTH_SYS	RPCAUTH_UNIX
 #define	RPCAUTH_SHORT	2
 #define RPCAUTH_KERB4	4
+#define RPCAUTH_KRB5	390003
+#define RPCAUTH_KRB5I	390004
+#define RPCAUTH_KRB5P	390005
+
 #define	RPCAUTH_MAXSIZ	400
-#define	RPCVERF_MAXSIZ	12	/* For Kerb, can actually be 400 */
 #define	RPCAUTH_UNIXGIDS 16
 
 /*
@@ -94,6 +104,7 @@
 #define	RPC_REPLY	1
 #define	RPC_MSGACCEPTED	0
 #define	RPC_MSGDENIED	1
+#define	RPC_SUCCESS	0
 #define	RPC_PROGUNAVAIL	1
 #define	RPC_PROGMISMATCH	2
 #define	RPC_PROCUNAVAIL	3
@@ -103,11 +114,20 @@
 #define	RPC_AUTHERR	1
 
 /* Authentication failures */
-#define	AUTH_BADCRED	1
-#define	AUTH_REJECTCRED	2
-#define	AUTH_BADVERF	3
-#define	AUTH_REJECTVERF	4
-#define	AUTH_TOOWEAK	5		/* Give em wheaties */
+#define	AUTH_BADCRED		1
+#define	AUTH_REJECTCRED		2
+#define	AUTH_BADVERF		3
+#define	AUTH_REJECTVERF		4
+#define	AUTH_TOOWEAK		5	/* Give em wheaties */
+#define AUTH_INVALIDRESP	6
+#define AUTH_FAILED		7
+#define AUTH_KERB_GENERIC	8
+#define AUTH_TIMEEXPIRE		9
+#define AUTH_TKT_FILE		10
+#define AUTH_DECODE		11
+#define AUTH_NET_ADDR		12
+#define RPCSEC_GSS_CREDPROBLEM	13
+#define RPCSEC_GSS_CTXPROBLEM	14
 
 /* Sizes of rpc header parts */
 #define	RPC_SIZ		24
@@ -126,45 +146,13 @@
 #define	RPCMNT_PATHLEN	1024
 #define	RPCPROG_NFS	100003
 
-/*
- * Structures used for RPCAUTH_KERB4.
- */
-struct nfsrpc_fullverf {
-	u_long		t1;
-	u_long		t2;
-	u_long		w2;
-};
-
-struct nfsrpc_fullblock {
-	u_long		t1;
-	u_long		t2;
-	u_long		w1;
-	u_long		w2;
-};
-
-struct nfsrpc_nickverf {
-	u_long			kind;
-	struct nfsrpc_fullverf	verf;
-};
-
-/*
- * and their sizes in bytes.. If sizeof (struct nfsrpc_xx) != these
- * constants, well then things will break in mount_nfs and nfsd.
- */
-#define RPCX_FULLVERF	12
-#define RPCX_FULLBLOCK	16
-#define RPCX_NICKVERF	16
-
-#if NFSKERB
-XXX
-#else
-typedef u_char			NFSKERBKEY_T[2];
-typedef u_char			NFSKERBKEYSCHED_T[2];
-#endif
-#define NFS_KERBSRV	"rcmd"		/* Kerberos Service for NFS */
-#define NFS_KERBTTL	(30 * 60)	/* Credential ttl (sec) */
-#define NFS_KERBCLOCKSKEW (5 * 60)	/* Clock skew (sec) */
-#define NFS_KERBW1(t)	(*((u_long *)(&((t).dat[((t).length + 3) & ~0x3]))))
+#define	RPCPROG_RQUOTA	100011
+#define	RPCRQUOTA_VER		1
+#define	RPCRQUOTA_EXT_VER	2
+#define	RPCRQUOTA_GET		1
+#define	RQUOTA_STAT_OK		1
+#define	RQUOTA_STAT_NOQUOTA	2
+#define	RQUOTA_STAT_EPERM	3
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* _NFS_RPCV2_H_ */
diff --git a/bsd/nfs/xdr_subs.h b/bsd/nfs/xdr_subs.h
index f6bc3748e..be5c6444c 100644
--- a/bsd/nfs/xdr_subs.h
+++ b/bsd/nfs/xdr_subs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -81,30 +87,6 @@
 #define	fxdr_unsigned(t, v)	((t)ntohl((long)(v)))
 #define	txdr_unsigned(v)	(htonl((long)(v)))
 
-#define	fxdr_nfsv2time(f, t) { \
-	(t)->tv_sec = ntohl(((struct nfsv2_time *)(f))->nfsv2_sec); \
-	if (((struct nfsv2_time *)(f))->nfsv2_usec != 0xffffffff) \
-		(t)->tv_nsec = 1000 * ntohl(((struct nfsv2_time *)(f))->nfsv2_usec); \
-	else \
-		(t)->tv_nsec = 0; \
-}
-#define	txdr_nfsv2time(f, t) { \
-	((struct nfsv2_time *)(t))->nfsv2_sec = htonl((f)->tv_sec); \
-	if ((f)->tv_nsec != -1) \
-		((struct nfsv2_time *)(t))->nfsv2_usec = htonl((f)->tv_nsec / 1000); \
-	else \
-		((struct nfsv2_time *)(t))->nfsv2_usec = 0xffffffff; \
-}
-
-#define	fxdr_nfsv3time(f, t) { \
-	(t)->tv_sec = ntohl(((struct nfsv3_time *)(f))->nfsv3_sec); \
-	(t)->tv_nsec = ntohl(((struct nfsv3_time *)(f))->nfsv3_nsec); \
-}
-#define	txdr_nfsv3time(f, t) { \
-	((struct nfsv3_time *)(t))->nfsv3_sec = htonl((f)->tv_sec); \
-	((struct nfsv3_time *)(t))->nfsv3_nsec = htonl((f)->tv_nsec); \
-}
-
 #define	fxdr_hyper(f, t) { \
 	((long *)(t))[_QUAD_HIGHWORD] = ntohl(((long *)(f))[0]); \
 	((long *)(t))[_QUAD_LOWWORD] = ntohl(((long *)(f))[1]); \
diff --git a/bsd/ppc/Makefile b/bsd/ppc/Makefile
index 633b7a521..42d55dcc1 100644
--- a/bsd/ppc/Makefile
+++ b/bsd/ppc/Makefile
@@ -8,14 +8,15 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 DATAFILES = \
-	 endian.h param.h profile.h \
+	 endian.h fasttrap_isa.h param.h profile.h \
 	setjmp.h signal.h  \
-	types.h ucontext.h  vmparam.h _types.h
+	types.h vmparam.h _structs.h _types.h _param.h
 
 KERNELFILES = \
+	disklabel.h \
 	 endian.h param.h profile.h \
 	   signal.h  \
-	 types.h  vmparam.h _types.h
+	 types.h  vmparam.h _structs.h _types.h _param.h
 
 INSTALL_MD_LIST = ${DATAFILES}
 INSTALL_MD_LCL_LIST = ${DATAFILES} disklabel.h
diff --git a/bsd/ppc/_param.h b/bsd/ppc/_param.h
new file mode 100644
index 000000000..1408c6fcc
--- /dev/null
+++ b/bsd/ppc/_param.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef	_PPC__PARAM_H_
+#define	_PPC__PARAM_H_
+
+#include <ppc/_types.h>
+
+/*
+ * Round p (pointer or byte index) up to a correctly-aligned value for all
+ * data types (int, long, ...).   The result is unsigned int and must be
+ * cast to any desired pointer type.
+ */
+#define	__DARWIN_ALIGNBYTES	(sizeof(__darwin_size_t) - 1)
+#define	__DARWIN_ALIGN(p)	((__darwin_size_t)((char *)(p) + __DARWIN_ALIGNBYTES) &~ __DARWIN_ALIGNBYTES)
+
+#endif	/* _PPC__PARAM_H_ */
diff --git a/bsd/ppc/_structs.h b/bsd/ppc/_structs.h
new file mode 100644
index 000000000..0d7fd4c9e
--- /dev/null
+++ b/bsd/ppc/_structs.h
@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/cdefs.h>
+
+#ifdef __need_mcontext_t
+#ifndef __need_struct_mcontext
+#define __need_struct_mcontext
+#endif /* __need_struct_mcontext */
+#endif /* __need_mcontext_t */
+
+#ifdef __need_mcontext64_t
+#ifndef __need_struct_mcontext64
+#define __need_struct_mcontext64
+#endif /* __need_struct_mcontext64 */
+#endif /* __need_mcontext64_t */
+
+#if defined(__need_struct_mcontext) || defined(__need_struct_mcontext64)
+#include <mach/ppc/_structs.h>
+#endif /* __need_struct_mcontext || __need_struct_mcontext64 */
+
+#ifdef __need_struct_mcontext
+#undef __need_struct_mcontext
+#ifndef _STRUCT_MCONTEXT
+#if __DARWIN_UNIX03
+#define	_STRUCT_MCONTEXT	struct __darwin_mcontext
+_STRUCT_MCONTEXT
+{
+	_STRUCT_PPC_EXCEPTION_STATE	__es;
+	_STRUCT_PPC_THREAD_STATE	__ss;
+	_STRUCT_PPC_FLOAT_STATE		__fs;
+	_STRUCT_PPC_VECTOR_STATE	__vs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_MCONTEXT	struct mcontext
+_STRUCT_MCONTEXT
+{
+	_STRUCT_PPC_EXCEPTION_STATE	es;
+	_STRUCT_PPC_THREAD_STATE	ss;
+	_STRUCT_PPC_FLOAT_STATE		fs;
+	_STRUCT_PPC_VECTOR_STATE	vs;
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_MCONTEXT */
+#endif /* __need_struct_mcontext */
+
+#ifdef __need_struct_mcontext64
+#undef __need_struct_mcontext64
+#ifndef _STRUCT_MCONTEXT64
+#if __DARWIN_UNIX03
+#define _STRUCT_MCONTEXT64	struct __darwin_mcontext64
+_STRUCT_MCONTEXT64
+{
+	_STRUCT_PPC_EXCEPTION_STATE64	__es;
+	_STRUCT_PPC_THREAD_STATE64	__ss;
+	_STRUCT_PPC_FLOAT_STATE		__fs;
+	_STRUCT_PPC_VECTOR_STATE	__vs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_MCONTEXT64	struct mcontext64
+_STRUCT_MCONTEXT64
+{
+	_STRUCT_PPC_EXCEPTION_STATE64	es;
+	_STRUCT_PPC_THREAD_STATE64	ss;
+	_STRUCT_PPC_FLOAT_STATE		fs;
+	_STRUCT_PPC_VECTOR_STATE	vs;
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_MCONTEXT64 */
+#endif /* __need_struct_mcontext64 */
+
+#ifdef __need_mcontext_t
+#undef __need_mcontext_t
+#ifndef _MCONTEXT_T
+#define _MCONTEXT_T
+typedef _STRUCT_MCONTEXT	*mcontext_t;
+#endif /* _MCONTEXT_T */
+#endif /* __need_mcontext_t */
+
+#ifdef __need_mcontext64_t
+#undef __need_mcontext64_t
+#ifndef _MCONTEXT64_T
+#define _MCONTEXT64_T
+typedef _STRUCT_MCONTEXT64	*mcontext64_t;
+#endif /* _MCONTEXT64_T */
+#endif /* __need_mcontext64_t */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef PPC_MCONTEXT_SIZE
+#define PPC_MCONTEXT_SIZE	(PPC_THREAD_STATE_COUNT + PPC_FLOAT_STATE_COUNT + PPC_EXCEPTION_STATE_COUNT + PPC_VECTOR_STATE_COUNT) * sizeof(int)
+#endif /* PPC_MCONTEXT_SIZE */
+#ifndef PPC_MCONTEXT64_SIZE
+#define PPC_MCONTEXT64_SIZE	(PPC_THREAD_STATE64_COUNT + PPC_FLOAT_STATE_COUNT + PPC_EXCEPTION_STATE_COUNT + PPC_VECTOR_STATE_COUNT) * sizeof(int)
+#endif /* PPC_MCONTEXT64_SIZE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+
+/*
+ * LP64todo - Have to decide how to handle this.
+ * For now, just duplicate the 32-bit context as the generic one.
+ */
+#ifdef __need_struct_sigcontext
+#undef __need_struct_sigcontext
+#ifndef _STRUCT_SIGCONTEXT
+#if __DARWIN_UNIX03		/* signal.h needs struct sigcontext visible */
+#define _STRUCT_SIGCONTEXT	struct __darwin_sigcontext
+_STRUCT_SIGCONTEXT
+{
+    int		__sc_onstack;	/* sigstack state to restore */
+    int		__sc_mask;	/* signal mask to restore */
+    int		__sc_ir;	/* pc */
+    int		__sc_psw;	/* processor status word */
+    int		__sc_sp;	/* stack pointer if sc_regs == NULL */
+    void	*__sc_regs;	/* (kernel private) saved state */
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_SIGCONTEXT	struct sigcontext
+_STRUCT_SIGCONTEXT
+{
+    int		sc_onstack;	/* sigstack state to restore */
+    int		sc_mask;	/* signal mask to restore */
+    int		sc_ir;		/* pc */
+    int		sc_psw;		/* processor status word */
+    int		sc_sp;		/* stack pointer if sc_regs == NULL */
+    void	*sc_regs;	/* (kernel private) saved state */
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_SIGCONTEXT */
+#endif /* __need_struct_sigcontext */
+
+/*
+ * Information pushed on stack when a signal is delivered.
+ * This is used by the kernel to restore state following
+ * execution of the signal handler.  It is also made available
+ * to the handler to allow it to properly restore state if
+ * a non-standard exit is performed.
+ */
+#ifdef __need_struct_sigcontext32
+#undef __need_struct_sigcontext32
+#ifndef _STRUCT_SIGCONTEXT32
+#if __DARWIN_UNIX03
+#define _STRUCT_SIGCONTEXT32	struct __darwin_sigcontext32
+_STRUCT_SIGCONTEXT32
+{
+    int		__sc_onstack;	/* sigstack state to restore */
+    int		__sc_mask;	/* signal mask to restore */
+    int		__sc_ir;	/* pc */
+    int		__sc_psw;	/* processor status word */
+    int		__sc_sp;	/* stack pointer if sc_regs == NULL */
+    void	*__sc_regs;	/* (kernel private) saved state */
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_SIGCONTEXT32	struct sigcontext32
+_STRUCT_SIGCONTEXT32
+{
+    int		sc_onstack;	/* sigstack state to restore */
+    int		sc_mask;	/* signal mask to restore */
+    int		sc_ir;		/* pc */
+    int		sc_psw;		/* processor status word */
+    int		sc_sp;		/* stack pointer if sc_regs == NULL */
+    void	*sc_regs;	/* (kernel private) saved state */
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_SIGCONTEXT32 */
+#endif /* __need_struct_sigcontext32 */
+
+#ifdef __need_struct_sigcontext64
+#undef __need_struct_sigcontext64
+#ifndef _STRUCT_SIGCONTEXT64
+#if __DARWIN_UNIX03
+#define _STRUCT_SIGCONTEXT64	struct __darwin_sigcontext64
+_STRUCT_SIGCONTEXT64
+{
+    int		__sc_onstack;	/* sigstack state to restore */
+    int		__sc_mask;	/* signal mask to restore */
+    long long	__sc_ir;	/* pc */
+    long long	__sc_psw;	/* processor status word */
+    long long	__sc_sp;	/* stack pointer if sc_regs == NULL */
+    void	*__sc_regs;	/* (kernel private) saved state */
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_SIGCONTEXT64	struct sigcontext64
+_STRUCT_SIGCONTEXT64
+{
+    int		sc_onstack;	/* sigstack state to restore */
+    int		sc_mask;	/* signal mask to restore */
+    long long	sc_ir;		/* pc */
+    long long	sc_psw;		/* processor status word */
+    long long	sc_sp;		/* stack pointer if sc_regs == NULL */
+    void	*sc_regs;	/* (kernel private) saved state */
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* _STRUCT_SIGCONTEXT64 */
+#endif /* __need_struct_sigcontext64 */
diff --git a/bsd/ppc/_types.h b/bsd/ppc/_types.h
index c607bba87..4b7855988 100644
--- a/bsd/ppc/_types.h
+++ b/bsd/ppc/_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_BSD_PPC__TYPES_H_
 #define	_BSD_PPC__TYPES_H_
@@ -86,15 +92,11 @@ typedef __SIZE_TYPE__		__darwin_size_t;	/* sizeof() */
 typedef unsigned long		__darwin_size_t;	/* sizeof() */
 #endif
 
-#ifdef KERNEL
-typedef char *			__darwin_va_list;	/* va_list */
-#else /* !KERNEL */
 #if (__GNUC__ > 2)
 typedef __builtin_va_list	__darwin_va_list;	/* va_list */
 #else
 typedef char *			__darwin_va_list;	/* va_list */
 #endif
-#endif /* KERNEL */
 
 #if defined(__GNUC__) && defined(__WCHAR_TYPE__)
 typedef __WCHAR_TYPE__		__darwin_wchar_t;	/* wchar_t */
diff --git a/bsd/ppc/decodePPC.h b/bsd/ppc/decodePPC.h
new file mode 100644
index 000000000..8fb4756f6
--- /dev/null
+++ b/bsd/ppc/decodePPC.h
@@ -0,0 +1,919 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+struct dcdtab {
+
+	uint8_t		dcdFlgs;			/* Flags needed to decode */
+#define dcdStep 0x80				/* Step to next table entry on non-match */
+#define dcdJump 0x40				/* Jump to new entry in table. Index is in dcdMatch. */
+#define dcdMask 0x0F				/* Index into mask table.  0 matches everything */
+
+	uint8_t		dcdType;			/* Instruction type */
+#define diINV  0x00
+#define diTRP  0x01
+#define diSC   0x02
+#define diRFI  0x03
+#define diB	   0x04
+#define diBC   0x05
+#define diBLR  0x06
+#define diBCTR 0x07
+#define diOR   0x08
+#define diSPR  0x09
+#define diCMN  0x0A
+#define diPRV  0x0B
+
+	uint16_t	dcdMatch;			/* Extended op code to match */
+};
+
+typedef struct dcdtab dcdtab;
+
+static uint16_t masktab[] = {0x0000, 0x0003, 0x001C, 0x001E, 0x003E, /* Table of extended op masks */
+	0x003F, 0x03FE, 0x03FF, 0x07FC, 0x07FE, 0x07FF};
+
+static dcdtab insts[] = {
+    { 0x40,      0,     64 },         //    0 Maj op =  0, jump to entry 64
+    { 0x00,  diINV, 0x0000 },         //    1 Maj op =  1, invalid
+    { 0x00,  diTRP, 0x0000 },         //    2 Maj op =  2, tdi
+    { 0x00,  diTRP, 0x0000 },         //    3 Maj op =  3, twi
+    { 0x40,      0,     65 },         //    4 Maj op =  4, jump to entry 65
+    { 0x00,  diINV, 0x0000 },         //    5 Maj op =  5, invalid
+    { 0x00,  diINV, 0x0000 },         //    6 Maj op =  6, invalid
+    { 0x00,  diCMN, 0x0000 },         //    7 Maj op =  7, mulli
+    { 0x00,  diCMN, 0x0000 },         //    8 Maj op =  8, subfic
+    { 0x00,  diINV, 0x0000 },         //    9 Maj op =  9, invalid
+    { 0x00,  diCMN, 0x0000 },         //   10 Maj op = 10, cmpli
+    { 0x00,  diCMN, 0x0000 },         //   11 Maj op = 11, cmpi
+    { 0x00,  diCMN, 0x0000 },         //   12 Maj op = 12, addic
+    { 0x00,  diCMN, 0x0000 },         //   13 Maj op = 13, addic.
+    { 0x00,  diCMN, 0x0000 },         //   14 Maj op = 14, addi
+    { 0x00,  diCMN, 0x0000 },         //   15 Maj op = 15, addis
+    { 0x00,   diBC, 0x0000 },         //   16 Maj op = 16, bc
+    { 0x00,   diSC, 0x0000 },         //   17 Maj op = 17, sc
+    { 0x00,    diB, 0x0000 },         //   18 Maj op = 18, b
+    { 0x40,      0,    209 },         //   19 Maj op = 19, jump to entry 209
+    { 0x00,  diCMN, 0x0000 },         //   20 Maj op = 20, rlwimi
+    { 0x00,  diCMN, 0x0000 },         //   21 Maj op = 21, rlwinm
+    { 0x00,  diINV, 0x0000 },         //   22 Maj op = 22, invalid
+    { 0x00,  diCMN, 0x0000 },         //   23 Maj op = 23, rlwnm
+    { 0x00,   diOR, 0x0000 },         //   24 Maj op = 24, ori
+    { 0x00,  diCMN, 0x0000 },         //   25 Maj op = 25, oris
+    { 0x00,  diCMN, 0x0000 },         //   26 Maj op = 26, xori
+    { 0x00,  diCMN, 0x0000 },         //   27 Maj op = 27, xoris
+    { 0x00,  diCMN, 0x0000 },         //   28 Maj op = 28, andi.
+    { 0x00,  diCMN, 0x0000 },         //   29 Maj op = 29, andis.
+    { 0x40,      0,    224 },         //   30 Maj op = 30, jump to entry 224
+    { 0x40,      0,    230 },         //   31 Maj op = 31, jump to entry 230
+    { 0x00,  diCMN, 0x0000 },         //   32 Maj op = 32, lwz
+    { 0x00,  diCMN, 0x0000 },         //   33 Maj op = 33, lwzu
+    { 0x00,  diCMN, 0x0000 },         //   34 Maj op = 34, lbz
+    { 0x00,  diCMN, 0x0000 },         //   35 Maj op = 35, lbzu
+    { 0x00,  diCMN, 0x0000 },         //   36 Maj op = 36, stw
+    { 0x00,  diCMN, 0x0000 },         //   37 Maj op = 37, stwu
+    { 0x00,  diCMN, 0x0000 },         //   38 Maj op = 38, stb
+    { 0x00,  diCMN, 0x0000 },         //   39 Maj op = 39, stbu
+    { 0x00,  diCMN, 0x0000 },         //   40 Maj op = 40, lhz
+    { 0x00,  diCMN, 0x0000 },         //   41 Maj op = 41, lhzu
+    { 0x00,  diCMN, 0x0000 },         //   42 Maj op = 42, lha
+    { 0x00,  diCMN, 0x0000 },         //   43 Maj op = 43, lhau
+    { 0x00,  diCMN, 0x0000 },         //   44 Maj op = 44, sth
+    { 0x00,  diCMN, 0x0000 },         //   45 Maj op = 45, sthu
+    { 0x00,  diCMN, 0x0000 },         //   46 Maj op = 46, lmw
+    { 0x00,  diCMN, 0x0000 },         //   47 Maj op = 47, stmw
+    { 0x00,  diCMN, 0x0000 },         //   48 Maj op = 48, lfs
+    { 0x00,  diCMN, 0x0000 },         //   49 Maj op = 49, lfsu
+    { 0x00,  diCMN, 0x0000 },         //   50 Maj op = 50, lfd
+    { 0x00,  diCMN, 0x0000 },         //   51 Maj op = 51, lfdu
+    { 0x00,  diCMN, 0x0000 },         //   52 Maj op = 52, stfs
+    { 0x00,  diCMN, 0x0000 },         //   53 Maj op = 53, stfsu
+    { 0x00,  diCMN, 0x0000 },         //   54 Maj op = 54, stfd
+    { 0x00,  diCMN, 0x0000 },         //   55 Maj op = 55, stfdu
+    { 0x00,  diINV, 0x0000 },         //   56 Maj op = 56, invalid
+    { 0x00,  diINV, 0x0000 },         //   57 Maj op = 57, invalid
+    { 0x40,      0,    365 },         //   58 Maj op = 58, jump to entry 365
+    { 0x40,      0,    368 },         //   59 Maj op = 59, jump to entry 368
+    { 0x00,  diINV, 0x0000 },         //   60 Maj op = 60, invalid
+    { 0x00,  diINV, 0x0000 },         //   61 Maj op = 61, invalid
+    { 0x40,      0,    378 },         //   62 Maj op = 62, jump to entry 378
+    { 0x40,      0,    380 },         //   63 Maj op = 63, jump to entry 380
+    { 0x09,  diCMN, 0x0200 },         //   64 Maj op =  0, mask = 07FE, xop = 0x0200 ( 256) - attn
+    { 0x85,  diCMN, 0x0020 },         //   65 Maj op =  4, mask = 003F, xop = 0x0020 (  32) - vmhaddshs
+    { 0x85,  diCMN, 0x0021 },         //   66 Maj op =  4, mask = 003F, xop = 0x0021 (  33) - vmhraddshs
+    { 0x85,  diCMN, 0x0022 },         //   67 Maj op =  4, mask = 003F, xop = 0x0022 (  34) - vmladduhm
+    { 0x85,  diCMN, 0x0024 },         //   68 Maj op =  4, mask = 003F, xop = 0x0024 (  36) - vmsumubm
+    { 0x85,  diCMN, 0x0025 },         //   69 Maj op =  4, mask = 003F, xop = 0x0025 (  37) - vmsummbm
+    { 0x85,  diCMN, 0x0026 },         //   70 Maj op =  4, mask = 003F, xop = 0x0026 (  38) - vmsumuhm
+    { 0x85,  diCMN, 0x0027 },         //   71 Maj op =  4, mask = 003F, xop = 0x0027 (  39) - vmsumuhs
+    { 0x85,  diCMN, 0x0028 },         //   72 Maj op =  4, mask = 003F, xop = 0x0028 (  40) - vmsumshm
+    { 0x85,  diCMN, 0x0029 },         //   73 Maj op =  4, mask = 003F, xop = 0x0029 (  41) - vmsumshs
+    { 0x85,  diCMN, 0x002A },         //   74 Maj op =  4, mask = 003F, xop = 0x002A (  42) - vsel
+    { 0x85,  diCMN, 0x002B },         //   75 Maj op =  4, mask = 003F, xop = 0x002B (  43) - vperm
+    { 0x85,  diCMN, 0x002C },         //   76 Maj op =  4, mask = 003F, xop = 0x002C (  44) - vsldoi
+    { 0x85,  diCMN, 0x002E },         //   77 Maj op =  4, mask = 003F, xop = 0x002E (  46) - vmaddfp
+    { 0x85,  diCMN, 0x002F },         //   78 Maj op =  4, mask = 003F, xop = 0x002F (  47) - vnmsubfp
+    { 0x87,  diCMN, 0x0006 },         //   79 Maj op =  4, mask = 03FF, xop = 0x0006 (   6) - vcmpequb
+    { 0x87,  diCMN, 0x0046 },         //   80 Maj op =  4, mask = 03FF, xop = 0x0046 (  70) - vcmpequh
+    { 0x87,  diCMN, 0x0086 },         //   81 Maj op =  4, mask = 03FF, xop = 0x0086 ( 134) - vcmpequw
+    { 0x87,  diCMN, 0x00C6 },         //   82 Maj op =  4, mask = 03FF, xop = 0x00C6 ( 198) - vcmpeqfp
+    { 0x87,  diCMN, 0x01C6 },         //   83 Maj op =  4, mask = 03FF, xop = 0x01C6 ( 454) - vcmpgefp
+    { 0x87,  diCMN, 0x0206 },         //   84 Maj op =  4, mask = 03FF, xop = 0x0206 ( 518) - vcmpgtub
+    { 0x87,  diCMN, 0x0246 },         //   85 Maj op =  4, mask = 03FF, xop = 0x0246 ( 582) - vcmpgtuh
+    { 0x87,  diCMN, 0x0286 },         //   86 Maj op =  4, mask = 03FF, xop = 0x0286 ( 646) - vcmpgtuw
+    { 0x87,  diCMN, 0x02C6 },         //   87 Maj op =  4, mask = 03FF, xop = 0x02C6 ( 710) - vcmpgtfp
+    { 0x87,  diCMN, 0x0306 },         //   88 Maj op =  4, mask = 03FF, xop = 0x0306 ( 774) - vcmpgtsb
+    { 0x87,  diCMN, 0x0346 },         //   89 Maj op =  4, mask = 03FF, xop = 0x0346 ( 838) - vcmpgtsh
+    { 0x87,  diCMN, 0x0386 },         //   90 Maj op =  4, mask = 03FF, xop = 0x0386 ( 902) - vcmpgtsw
+    { 0x87,  diCMN, 0x03C6 },         //   91 Maj op =  4, mask = 03FF, xop = 0x03C6 ( 966) - vcmpbfp
+    { 0x8A,  diCMN, 0x0000 },         //   92 Maj op =  4, mask = 07FF, xop = 0x0000 (   0) - vaddubm
+    { 0x8A,  diCMN, 0x0002 },         //   93 Maj op =  4, mask = 07FF, xop = 0x0002 (   2) - vmaxub
+    { 0x8A,  diCMN, 0x0004 },         //   94 Maj op =  4, mask = 07FF, xop = 0x0004 (   4) - vrlb
+    { 0x8A,  diCMN, 0x0008 },         //   95 Maj op =  4, mask = 07FF, xop = 0x0008 (   8) - vmuloub
+    { 0x8A,  diCMN, 0x000A },         //   96 Maj op =  4, mask = 07FF, xop = 0x000A (  10) - vaddfp
+    { 0x8A,  diCMN, 0x000C },         //   97 Maj op =  4, mask = 07FF, xop = 0x000C (  12) - vmrghb
+    { 0x8A,  diCMN, 0x000E },         //   98 Maj op =  4, mask = 07FF, xop = 0x000E (  14) - vpkuhum
+    { 0x8A,  diCMN, 0x0040 },         //   99 Maj op =  4, mask = 07FF, xop = 0x0040 (  64) - vadduhm
+    { 0x8A,  diCMN, 0x0042 },         //  100 Maj op =  4, mask = 07FF, xop = 0x0042 (  66) - vmaxuh
+    { 0x8A,  diCMN, 0x0044 },         //  101 Maj op =  4, mask = 07FF, xop = 0x0044 (  68) - vrlh
+    { 0x8A,  diCMN, 0x0048 },         //  102 Maj op =  4, mask = 07FF, xop = 0x0048 (  72) - vmulouh
+    { 0x8A,  diCMN, 0x004A },         //  103 Maj op =  4, mask = 07FF, xop = 0x004A (  74) - vsubfp
+    { 0x8A,  diCMN, 0x004C },         //  104 Maj op =  4, mask = 07FF, xop = 0x004C (  76) - vmrghh
+    { 0x8A,  diCMN, 0x004E },         //  105 Maj op =  4, mask = 07FF, xop = 0x004E (  78) - vpkuwum
+    { 0x8A,  diCMN, 0x0080 },         //  106 Maj op =  4, mask = 07FF, xop = 0x0080 ( 128) - vadduwm
+    { 0x8A,  diCMN, 0x0082 },         //  107 Maj op =  4, mask = 07FF, xop = 0x0082 ( 130) - vmaxuw
+    { 0x8A,  diCMN, 0x0084 },         //  108 Maj op =  4, mask = 07FF, xop = 0x0084 ( 132) - vrlw
+    { 0x8A,  diCMN, 0x008C },         //  109 Maj op =  4, mask = 07FF, xop = 0x008C ( 140) - vmrghw
+    { 0x8A,  diCMN, 0x008E },         //  110 Maj op =  4, mask = 07FF, xop = 0x008E ( 142) - vpkuhus
+    { 0x8A,  diCMN, 0x00CE },         //  111 Maj op =  4, mask = 07FF, xop = 0x00CE ( 206) - vpkuwus
+    { 0x8A,  diCMN, 0x0102 },         //  112 Maj op =  4, mask = 07FF, xop = 0x0102 ( 258) - vmaxsb
+    { 0x8A,  diCMN, 0x0104 },         //  113 Maj op =  4, mask = 07FF, xop = 0x0104 ( 260) - vslb
+    { 0x8A,  diCMN, 0x0108 },         //  114 Maj op =  4, mask = 07FF, xop = 0x0108 ( 264) - vmulosb
+    { 0x8A,  diCMN, 0x010A },         //  115 Maj op =  4, mask = 07FF, xop = 0x010A ( 266) - vrefp
+    { 0x8A,  diCMN, 0x010C },         //  116 Maj op =  4, mask = 07FF, xop = 0x010C ( 268) - vmrglb
+    { 0x8A,  diCMN, 0x010E },         //  117 Maj op =  4, mask = 07FF, xop = 0x010E ( 270) - vpkshus
+    { 0x8A,  diCMN, 0x0142 },         //  118 Maj op =  4, mask = 07FF, xop = 0x0142 ( 322) - vmaxsh
+    { 0x8A,  diCMN, 0x0144 },         //  119 Maj op =  4, mask = 07FF, xop = 0x0144 ( 324) - vslh
+    { 0x8A,  diCMN, 0x0148 },         //  120 Maj op =  4, mask = 07FF, xop = 0x0148 ( 328) - vmulosh
+    { 0x8A,  diCMN, 0x014A },         //  121 Maj op =  4, mask = 07FF, xop = 0x014A ( 330) - vrsqrtefp
+    { 0x8A,  diCMN, 0x014C },         //  122 Maj op =  4, mask = 07FF, xop = 0x014C ( 332) - vmrglh
+    { 0x8A,  diCMN, 0x014E },         //  123 Maj op =  4, mask = 07FF, xop = 0x014E ( 334) - vpkswus
+    { 0x8A,  diCMN, 0x0180 },         //  124 Maj op =  4, mask = 07FF, xop = 0x0180 ( 384) - vaddcuw
+    { 0x8A,  diCMN, 0x0182 },         //  125 Maj op =  4, mask = 07FF, xop = 0x0182 ( 386) - vmaxsw
+    { 0x8A,  diCMN, 0x0184 },         //  126 Maj op =  4, mask = 07FF, xop = 0x0184 ( 388) - vslw
+    { 0x8A,  diCMN, 0x018A },         //  127 Maj op =  4, mask = 07FF, xop = 0x018A ( 394) - vexptefp
+    { 0x8A,  diCMN, 0x018C },         //  128 Maj op =  4, mask = 07FF, xop = 0x018C ( 396) - vmrglw
+    { 0x8A,  diCMN, 0x018E },         //  129 Maj op =  4, mask = 07FF, xop = 0x018E ( 398) - vpkshss
+    { 0x8A,  diCMN, 0x01C4 },         //  130 Maj op =  4, mask = 07FF, xop = 0x01C4 ( 452) - vsl
+    { 0x8A,  diCMN, 0x01CA },         //  131 Maj op =  4, mask = 07FF, xop = 0x01CA ( 458) - vlogefp
+    { 0x8A,  diCMN, 0x01CE },         //  132 Maj op =  4, mask = 07FF, xop = 0x01CE ( 462) - vpkswss
+    { 0x8A,  diCMN, 0x0200 },         //  133 Maj op =  4, mask = 07FF, xop = 0x0200 ( 512) - vaddubs
+    { 0x8A,  diCMN, 0x0202 },         //  134 Maj op =  4, mask = 07FF, xop = 0x0202 ( 514) - vminub
+    { 0x8A,  diCMN, 0x0204 },         //  135 Maj op =  4, mask = 07FF, xop = 0x0204 ( 516) - vsrb
+    { 0x8A,  diCMN, 0x0208 },         //  136 Maj op =  4, mask = 07FF, xop = 0x0208 ( 520) - vmuleub
+    { 0x8A,  diCMN, 0x020A },         //  137 Maj op =  4, mask = 07FF, xop = 0x020A ( 522) - vrfin
+    { 0x8A,  diCMN, 0x020C },         //  138 Maj op =  4, mask = 07FF, xop = 0x020C ( 524) - vspltb
+    { 0x8A,  diCMN, 0x020E },         //  139 Maj op =  4, mask = 07FF, xop = 0x020E ( 526) - vupkhsb
+    { 0x8A,  diCMN, 0x0240 },         //  140 Maj op =  4, mask = 07FF, xop = 0x0240 ( 576) - vadduhs
+    { 0x8A,  diCMN, 0x0242 },         //  141 Maj op =  4, mask = 07FF, xop = 0x0242 ( 578) - vminuh
+    { 0x8A,  diCMN, 0x0244 },         //  142 Maj op =  4, mask = 07FF, xop = 0x0244 ( 580) - vsrh
+    { 0x8A,  diCMN, 0x0248 },         //  143 Maj op =  4, mask = 07FF, xop = 0x0248 ( 584) - vmuleuh
+    { 0x8A,  diCMN, 0x024A },         //  144 Maj op =  4, mask = 07FF, xop = 0x024A ( 586) - vrfiz
+    { 0x8A,  diCMN, 0x024C },         //  145 Maj op =  4, mask = 07FF, xop = 0x024C ( 588) - vsplth
+    { 0x8A,  diCMN, 0x024E },         //  146 Maj op =  4, mask = 07FF, xop = 0x024E ( 590) - vupkhsh
+    { 0x8A,  diCMN, 0x0280 },         //  147 Maj op =  4, mask = 07FF, xop = 0x0280 ( 640) - vadduws
+    { 0x8A,  diCMN, 0x0282 },         //  148 Maj op =  4, mask = 07FF, xop = 0x0282 ( 642) - vminuw
+    { 0x8A,  diCMN, 0x0284 },         //  149 Maj op =  4, mask = 07FF, xop = 0x0284 ( 644) - vsrw
+    { 0x8A,  diCMN, 0x028A },         //  150 Maj op =  4, mask = 07FF, xop = 0x028A ( 650) - vrfip
+    { 0x8A,  diCMN, 0x028C },         //  151 Maj op =  4, mask = 07FF, xop = 0x028C ( 652) - vspltw
+    { 0x8A,  diCMN, 0x028E },         //  152 Maj op =  4, mask = 07FF, xop = 0x028E ( 654) - vupklsb
+    { 0x8A,  diCMN, 0x02C4 },         //  153 Maj op =  4, mask = 07FF, xop = 0x02C4 ( 708) - vsr
+    { 0x8A,  diCMN, 0x02CA },         //  154 Maj op =  4, mask = 07FF, xop = 0x02CA ( 714) - vrfim
+    { 0x8A,  diCMN, 0x02CE },         //  155 Maj op =  4, mask = 07FF, xop = 0x02CE ( 718) - vupklsh
+    { 0x8A,  diCMN, 0x0300 },         //  156 Maj op =  4, mask = 07FF, xop = 0x0300 ( 768) - vaddsbs
+    { 0x8A,  diCMN, 0x0302 },         //  157 Maj op =  4, mask = 07FF, xop = 0x0302 ( 770) - vminsb
+    { 0x8A,  diCMN, 0x0304 },         //  158 Maj op =  4, mask = 07FF, xop = 0x0304 ( 772) - vsrab
+    { 0x8A,  diCMN, 0x0308 },         //  159 Maj op =  4, mask = 07FF, xop = 0x0308 ( 776) - vmulesb
+    { 0x8A,  diCMN, 0x030A },         //  160 Maj op =  4, mask = 07FF, xop = 0x030A ( 778) - vcfux
+    { 0x8A,  diCMN, 0x030C },         //  161 Maj op =  4, mask = 07FF, xop = 0x030C ( 780) - vspltisb
+    { 0x8A,  diCMN, 0x030E },         //  162 Maj op =  4, mask = 07FF, xop = 0x030E ( 782) - vpkpx
+    { 0x8A,  diCMN, 0x0340 },         //  163 Maj op =  4, mask = 07FF, xop = 0x0340 ( 832) - vaddshs
+    { 0x8A,  diCMN, 0x0342 },         //  164 Maj op =  4, mask = 07FF, xop = 0x0342 ( 834) - vminsh
+    { 0x8A,  diCMN, 0x0344 },         //  165 Maj op =  4, mask = 07FF, xop = 0x0344 ( 836) - vsrah
+    { 0x8A,  diCMN, 0x0348 },         //  166 Maj op =  4, mask = 07FF, xop = 0x0348 ( 840) - vmulesh
+    { 0x8A,  diCMN, 0x034A },         //  167 Maj op =  4, mask = 07FF, xop = 0x034A ( 842) - vcfsx
+    { 0x8A,  diCMN, 0x034C },         //  168 Maj op =  4, mask = 07FF, xop = 0x034C ( 844) - vspltish
+    { 0x8A,  diCMN, 0x034E },         //  169 Maj op =  4, mask = 07FF, xop = 0x034E ( 846) - vupkhpx
+    { 0x8A,  diCMN, 0x0380 },         //  170 Maj op =  4, mask = 07FF, xop = 0x0380 ( 896) - vaddsws
+    { 0x8A,  diCMN, 0x0382 },         //  171 Maj op =  4, mask = 07FF, xop = 0x0382 ( 898) - vminsw
+    { 0x8A,  diCMN, 0x0384 },         //  172 Maj op =  4, mask = 07FF, xop = 0x0384 ( 900) - vsraw
+    { 0x8A,  diCMN, 0x038A },         //  173 Maj op =  4, mask = 07FF, xop = 0x038A ( 906) - vctuxs
+    { 0x8A,  diCMN, 0x038C },         //  174 Maj op =  4, mask = 07FF, xop = 0x038C ( 908) - vspltisw
+    { 0x8A,  diCMN, 0x03CA },         //  175 Maj op =  4, mask = 07FF, xop = 0x03CA ( 970) - vctsxs
+    { 0x8A,  diCMN, 0x03CE },         //  176 Maj op =  4, mask = 07FF, xop = 0x03CE ( 974) - vupklpx
+    { 0x8A,  diCMN, 0x0400 },         //  177 Maj op =  4, mask = 07FF, xop = 0x0400 (1024) - vsububm
+    { 0x8A,  diCMN, 0x0402 },         //  178 Maj op =  4, mask = 07FF, xop = 0x0402 (1026) - vavgub
+    { 0x8A,  diCMN, 0x0404 },         //  179 Maj op =  4, mask = 07FF, xop = 0x0404 (1028) - vand
+    { 0x8A,  diCMN, 0x040A },         //  180 Maj op =  4, mask = 07FF, xop = 0x040A (1034) - vmaxfp
+    { 0x8A,  diCMN, 0x040C },         //  181 Maj op =  4, mask = 07FF, xop = 0x040C (1036) - vslo
+    { 0x8A,  diCMN, 0x0440 },         //  182 Maj op =  4, mask = 07FF, xop = 0x0440 (1088) - vsubuhm
+    { 0x8A,  diCMN, 0x0442 },         //  183 Maj op =  4, mask = 07FF, xop = 0x0442 (1090) - vavguh
+    { 0x8A,  diCMN, 0x0444 },         //  184 Maj op =  4, mask = 07FF, xop = 0x0444 (1092) - vandc
+    { 0x8A,  diCMN, 0x044A },         //  185 Maj op =  4, mask = 07FF, xop = 0x044A (1098) - vminfp
+    { 0x8A,  diCMN, 0x044C },         //  186 Maj op =  4, mask = 07FF, xop = 0x044C (1100) - vsro
+    { 0x8A,  diCMN, 0x0480 },         //  187 Maj op =  4, mask = 07FF, xop = 0x0480 (1152) - vsubuwm
+    { 0x8A,  diCMN, 0x0482 },         //  188 Maj op =  4, mask = 07FF, xop = 0x0482 (1154) - vavguw
+    { 0x8A,  diCMN, 0x0484 },         //  189 Maj op =  4, mask = 07FF, xop = 0x0484 (1156) - vor
+    { 0x8A,  diCMN, 0x04C4 },         //  190 Maj op =  4, mask = 07FF, xop = 0x04C4 (1220) - vxor
+    { 0x8A,  diCMN, 0x0502 },         //  191 Maj op =  4, mask = 07FF, xop = 0x0502 (1282) - vavgsb
+    { 0x8A,  diCMN, 0x0504 },         //  192 Maj op =  4, mask = 07FF, xop = 0x0504 (1284) - vnor
+    { 0x8A,  diCMN, 0x0542 },         //  193 Maj op =  4, mask = 07FF, xop = 0x0542 (1346) - vavgsh
+    { 0x8A,  diCMN, 0x0580 },         //  194 Maj op =  4, mask = 07FF, xop = 0x0580 (1408) - vsubcuw
+    { 0x8A,  diCMN, 0x0582 },         //  195 Maj op =  4, mask = 07FF, xop = 0x0582 (1410) - vavgsw
+    { 0x8A,  diCMN, 0x0600 },         //  196 Maj op =  4, mask = 07FF, xop = 0x0600 (1536) - vsububs
+    { 0x8A,  diCMN, 0x0604 },         //  197 Maj op =  4, mask = 07FF, xop = 0x0604 (1540) - mfvscr
+    { 0x8A,  diCMN, 0x0608 },         //  198 Maj op =  4, mask = 07FF, xop = 0x0608 (1544) - vsum4ubs
+    { 0x8A,  diCMN, 0x0640 },         //  199 Maj op =  4, mask = 07FF, xop = 0x0640 (1600) - vsubuhs
+    { 0x8A,  diCMN, 0x0644 },         //  200 Maj op =  4, mask = 07FF, xop = 0x0644 (1604) - mtvscr
+    { 0x8A,  diCMN, 0x0648 },         //  201 Maj op =  4, mask = 07FF, xop = 0x0648 (1608) - vsum4shs
+    { 0x8A,  diCMN, 0x0680 },         //  202 Maj op =  4, mask = 07FF, xop = 0x0680 (1664) - vsubuws
+    { 0x8A,  diCMN, 0x0688 },         //  203 Maj op =  4, mask = 07FF, xop = 0x0688 (1672) - vsum2sws
+    { 0x8A,  diCMN, 0x0700 },         //  204 Maj op =  4, mask = 07FF, xop = 0x0700 (1792) - vsubsbs
+    { 0x8A,  diCMN, 0x0708 },         //  205 Maj op =  4, mask = 07FF, xop = 0x0708 (1800) - vsum4sbs
+    { 0x8A,  diCMN, 0x0740 },         //  206 Maj op =  4, mask = 07FF, xop = 0x0740 (1856) - vsubshs
+    { 0x8A,  diCMN, 0x0780 },         //  207 Maj op =  4, mask = 07FF, xop = 0x0780 (1920) - vsubsws
+    { 0x0A,  diCMN, 0x0788 },         //  208 Maj op =  4, mask = 07FF, xop = 0x0788 (1928) - vsumsws
+    { 0x89,  diCMN, 0x0000 },         //  209 Maj op = 19, mask = 07FE, xop = 0x0000 (   0) - mcrf
+    { 0x89,  diBLR, 0x0020 },         //  210 Maj op = 19, mask = 07FE, xop = 0x0020 (  16) - bclr
+    { 0x89,  diPRV, 0x0024 },         //  211 Maj op = 19, mask = 07FE, xop = 0x0024 (  18) - rfid
+    { 0x89,  diCMN, 0x0042 },         //  212 Maj op = 19, mask = 07FE, xop = 0x0042 (  33) - crnor
+    { 0x89,  diPRV, 0x0064 },         //  213 Maj op = 19, mask = 07FE, xop = 0x0064 (  50) - rfi
+    { 0x89,  diCMN, 0x0102 },         //  214 Maj op = 19, mask = 07FE, xop = 0x0102 ( 129) - crandc
+    { 0x89,  diCMN, 0x012C },         //  215 Maj op = 19, mask = 07FE, xop = 0x012C ( 150) - isync
+    { 0x89,  diCMN, 0x0182 },         //  216 Maj op = 19, mask = 07FE, xop = 0x0182 ( 193) - crxor
+    { 0x89,  diCMN, 0x01C2 },         //  217 Maj op = 19, mask = 07FE, xop = 0x01C2 ( 225) - crnand
+    { 0x89,  diCMN, 0x0202 },         //  218 Maj op = 19, mask = 07FE, xop = 0x0202 ( 257) - crand
+    { 0x89,  diPRV, 0x0224 },         //  219 Maj op = 19, mask = 07FE, xop = 0x0224 ( 274) - hrfid
+    { 0x89,  diCMN, 0x0242 },         //  220 Maj op = 19, mask = 07FE, xop = 0x0242 ( 289) - creqv
+    { 0x89,  diCMN, 0x0342 },         //  221 Maj op = 19, mask = 07FE, xop = 0x0342 ( 417) - crorc
+    { 0x89,  diCMN, 0x0382 },         //  222 Maj op = 19, mask = 07FE, xop = 0x0382 ( 449) - cror
+    { 0x09, diBCTR, 0x0420 },         //  223 Maj op = 19, mask = 07FE, xop = 0x0420 ( 528) - bctr
+    { 0x82,  diCMN, 0x0000 },         //  224 Maj op = 30, mask = 001C, xop = 0x0000 (   0) - rldicl
+    { 0x82,  diCMN, 0x0004 },         //  225 Maj op = 30, mask = 001C, xop = 0x0004 (   1) - rldicr
+    { 0x82,  diCMN, 0x0008 },         //  226 Maj op = 30, mask = 001C, xop = 0x0008 (   2) - rldic
+    { 0x82,  diCMN, 0x000C },         //  227 Maj op = 30, mask = 001C, xop = 0x000C (   3) - rldimi
+    { 0x83,  diCMN, 0x0010 },         //  228 Maj op = 30, mask = 001E, xop = 0x0010 (   8) - rldcl
+    { 0x03,  diCMN, 0x0012 },         //  229 Maj op = 30, mask = 001E, xop = 0x0012 (   9) - rldcr
+    { 0x86,  diCMN, 0x0010 },         //  230 Maj op = 31, mask = 03FE, xop = 0x0010 (   8) - subfc
+    { 0x86,  diCMN, 0x0012 },         //  231 Maj op = 31, mask = 03FE, xop = 0x0012 (   9) - mulhdu
+    { 0x86,  diCMN, 0x0014 },         //  232 Maj op = 31, mask = 03FE, xop = 0x0014 (  10) - addc
+    { 0x86,  diCMN, 0x0016 },         //  233 Maj op = 31, mask = 03FE, xop = 0x0016 (  11) - mulhwu
+    { 0x86,  diCMN, 0x0050 },         //  234 Maj op = 31, mask = 03FE, xop = 0x0050 (  40) - subf
+    { 0x86,  diCMN, 0x0092 },         //  235 Maj op = 31, mask = 03FE, xop = 0x0092 (  73) - mulhd
+    { 0x86,  diCMN, 0x0096 },         //  236 Maj op = 31, mask = 03FE, xop = 0x0096 (  75) - mulhw
+    { 0x86,  diCMN, 0x00D0 },         //  237 Maj op = 31, mask = 03FE, xop = 0x00D0 ( 104) - neg
+    { 0x86,  diCMN, 0x0110 },         //  238 Maj op = 31, mask = 03FE, xop = 0x0110 ( 136) - subfe
+    { 0x86,  diCMN, 0x0114 },         //  239 Maj op = 31, mask = 03FE, xop = 0x0114 ( 138) - adde
+    { 0x86,  diCMN, 0x0190 },         //  240 Maj op = 31, mask = 03FE, xop = 0x0190 ( 200) - subfze
+    { 0x86,  diCMN, 0x0194 },         //  241 Maj op = 31, mask = 03FE, xop = 0x0194 ( 202) - addze
+    { 0x86,  diCMN, 0x01D0 },         //  242 Maj op = 31, mask = 03FE, xop = 0x01D0 ( 232) - subfme
+    { 0x86,  diCMN, 0x01D2 },         //  243 Maj op = 31, mask = 03FE, xop = 0x01D2 ( 233) - mulld
+    { 0x86,  diCMN, 0x01D4 },         //  244 Maj op = 31, mask = 03FE, xop = 0x01D4 ( 234) - addme
+    { 0x86,  diCMN, 0x01D6 },         //  245 Maj op = 31, mask = 03FE, xop = 0x01D6 ( 235) - mullw
+    { 0x86,  diCMN, 0x0214 },         //  246 Maj op = 31, mask = 03FE, xop = 0x0214 ( 266) - add
+    { 0x86,  diCMN, 0x0392 },         //  247 Maj op = 31, mask = 03FE, xop = 0x0392 ( 457) - divdu
+    { 0x86,  diCMN, 0x0396 },         //  248 Maj op = 31, mask = 03FE, xop = 0x0396 ( 459) - divwu
+    { 0x86,  diCMN, 0x03D2 },         //  249 Maj op = 31, mask = 03FE, xop = 0x03D2 ( 489) - divd
+    { 0x86,  diCMN, 0x03D6 },         //  250 Maj op = 31, mask = 03FE, xop = 0x03D6 ( 491) - divw
+    { 0x88,  diCMN, 0x0674 },         //  251 Maj op = 31, mask = 07FC, xop = 0x0674 ( 413) - sradi
+    { 0x89,  diCMN, 0x0000 },         //  252 Maj op = 31, mask = 07FE, xop = 0x0000 (   0) - cmp
+    { 0x89,  diTRP, 0x0008 },         //  253 Maj op = 31, mask = 07FE, xop = 0x0008 (   4) - tw
+    { 0x89,  diCMN, 0x000C },         //  254 Maj op = 31, mask = 07FE, xop = 0x000C (   6) - lvsl
+    { 0x89,  diCMN, 0x000E },         //  255 Maj op = 31, mask = 07FE, xop = 0x000E (   7) - lvebx
+    { 0x89,  diCMN, 0x0026 },         //  256 Maj op = 31, mask = 07FE, xop = 0x0026 (  19) - mfcr
+    { 0x89,  diCMN, 0x0028 },         //  257 Maj op = 31, mask = 07FE, xop = 0x0028 (  20) - lwarx
+    { 0x89,  diCMN, 0x002A },         //  258 Maj op = 31, mask = 07FE, xop = 0x002A (  21) - ldx
+    { 0x89,  diCMN, 0x002E },         //  259 Maj op = 31, mask = 07FE, xop = 0x002E (  23) - lwzx
+    { 0x89,  diCMN, 0x0030 },         //  260 Maj op = 31, mask = 07FE, xop = 0x0030 (  24) - slw
+    { 0x89,  diCMN, 0x0034 },         //  261 Maj op = 31, mask = 07FE, xop = 0x0034 (  26) - cntlzw
+    { 0x89,  diCMN, 0x0036 },         //  262 Maj op = 31, mask = 07FE, xop = 0x0036 (  27) - sld
+    { 0x89,  diCMN, 0x0038 },         //  263 Maj op = 31, mask = 07FE, xop = 0x0038 (  28) - and
+    { 0x89,  diCMN, 0x0040 },         //  264 Maj op = 31, mask = 07FE, xop = 0x0040 (  32) - cmpl
+    { 0x89,  diCMN, 0x004C },         //  265 Maj op = 31, mask = 07FE, xop = 0x004C (  38) - lvsr
+    { 0x89,  diCMN, 0x004E },         //  266 Maj op = 31, mask = 07FE, xop = 0x004E (  39) - lvehx
+    { 0x89,  diCMN, 0x006A },         //  267 Maj op = 31, mask = 07FE, xop = 0x006A (  53) - ldux
+    { 0x89,  diCMN, 0x006C },         //  268 Maj op = 31, mask = 07FE, xop = 0x006C (  54) - dcbst
+    { 0x89,  diCMN, 0x006E },         //  269 Maj op = 31, mask = 07FE, xop = 0x006E (  55) - lwzux
+    { 0x89,  diCMN, 0x0074 },         //  270 Maj op = 31, mask = 07FE, xop = 0x0074 (  58) - cntlzd
+    { 0x89,  diCMN, 0x0078 },         //  271 Maj op = 31, mask = 07FE, xop = 0x0078 (  60) - andc
+    { 0x89,  diTRP, 0x0088 },         //  272 Maj op = 31, mask = 07FE, xop = 0x0088 (  68) - td
+    { 0x89,  diCMN, 0x008E },         //  273 Maj op = 31, mask = 07FE, xop = 0x008E (  71) - lvewx
+    { 0x89,  diPRV, 0x00A6 },         //  274 Maj op = 31, mask = 07FE, xop = 0x00A6 (  83) - mfmsr
+    { 0x89,  diCMN, 0x00A8 },         //  275 Maj op = 31, mask = 07FE, xop = 0x00A8 (  84) - ldarx
+    { 0x89,  diCMN, 0x00AC },         //  276 Maj op = 31, mask = 07FE, xop = 0x00AC (  86) - dcbf
+    { 0x89,  diCMN, 0x00AE },         //  277 Maj op = 31, mask = 07FE, xop = 0x00AE (  87) - lbzx
+    { 0x89,  diCMN, 0x00CE },         //  278 Maj op = 31, mask = 07FE, xop = 0x00CE ( 103) - lvx
+    { 0x89,  diCMN, 0x00EE },         //  279 Maj op = 31, mask = 07FE, xop = 0x00EE ( 119) - lbzux
+    { 0x89,  diCMN, 0x00F8 },         //  280 Maj op = 31, mask = 07FE, xop = 0x00F8 ( 124) - nor
+    { 0x89,  diCMN, 0x010E },         //  281 Maj op = 31, mask = 07FE, xop = 0x010E ( 135) - stvebx
+    { 0x89,  diCMN, 0x0120 },         //  282 Maj op = 31, mask = 07FE, xop = 0x0120 ( 144) - mtcrf
+    { 0x89,  diPRV, 0x0124 },         //  283 Maj op = 31, mask = 07FE, xop = 0x0124 ( 146) - mtmsr
+    { 0x89,  diCMN, 0x012A },         //  284 Maj op = 31, mask = 07FE, xop = 0x012A ( 149) - stdx
+    { 0x89,  diCMN, 0x012C },         //  285 Maj op = 31, mask = 07FE, xop = 0x012C ( 150) - stwcx
+    { 0x89,  diCMN, 0x012E },         //  286 Maj op = 31, mask = 07FE, xop = 0x012E ( 151) - stwx
+    { 0x89,  diCMN, 0x014E },         //  287 Maj op = 31, mask = 07FE, xop = 0x014E ( 167) - stvehx
+    { 0x89,  diPRV, 0x0164 },         //  288 Maj op = 31, mask = 07FE, xop = 0x0164 ( 178) - mtmsrd
+    { 0x89,  diCMN, 0x016A },         //  289 Maj op = 31, mask = 07FE, xop = 0x016A ( 181) - stdux
+    { 0x89,  diCMN, 0x016E },         //  290 Maj op = 31, mask = 07FE, xop = 0x016E ( 183) - stwux
+    { 0x89,  diCMN, 0x018E },         //  291 Maj op = 31, mask = 07FE, xop = 0x018E ( 199) - stvewx
+    { 0x89,  diCMN, 0x01A4 },         //  292 Maj op = 31, mask = 07FE, xop = 0x01A4 ( 210) - mtsr
+    { 0x89,  diCMN, 0x01AC },         //  293 Maj op = 31, mask = 07FE, xop = 0x01AC ( 214) - stdcx.
+    { 0x89,  diCMN, 0x01AE },         //  294 Maj op = 31, mask = 07FE, xop = 0x01AE ( 215) - stbx
+    { 0x89,  diCMN, 0x01CE },         //  295 Maj op = 31, mask = 07FE, xop = 0x01CE ( 231) - stvx
+    { 0x89,  diPRV, 0x01E4 },         //  296 Maj op = 31, mask = 07FE, xop = 0x01E4 ( 242) - mtsrin
+    { 0x89,  diCMN, 0x01EC },         //  297 Maj op = 31, mask = 07FE, xop = 0x01EC ( 246) - dcbtst
+    { 0x89,  diCMN, 0x01EE },         //  298 Maj op = 31, mask = 07FE, xop = 0x01EE ( 247) - stbux
+    { 0x89,  diPRV, 0x0224 },         //  299 Maj op = 31, mask = 07FE, xop = 0x0224 ( 274) - tlbiel
+    { 0x89,  diCMN, 0x022C },         //  300 Maj op = 31, mask = 07FE, xop = 0x022C ( 278) - dcbt
+    { 0x89,  diCMN, 0x022E },         //  301 Maj op = 31, mask = 07FE, xop = 0x022E ( 279) - lhzx
+    { 0x89,  diCMN, 0x0238 },         //  302 Maj op = 31, mask = 07FE, xop = 0x0238 ( 284) - eqv
+    { 0x89,  diPRV, 0x0264 },         //  303 Maj op = 31, mask = 07FE, xop = 0x0264 ( 306) - tlbie
+    { 0x89,  diPRV, 0x026C },         //  304 Maj op = 31, mask = 07FE, xop = 0x026C ( 310) - eciwx
+    { 0x89,  diCMN, 0x026E },         //  305 Maj op = 31, mask = 07FE, xop = 0x026E ( 311) - lhzux
+    { 0x89,  diCMN, 0x0278 },         //  306 Maj op = 31, mask = 07FE, xop = 0x0278 ( 316) - xor
+    { 0x89,  diSPR, 0x02A6 },         //  307 Maj op = 31, mask = 07FE, xop = 0x02A6 ( 339) - mfspr
+    { 0x89,  diCMN, 0x02AA },         //  308 Maj op = 31, mask = 07FE, xop = 0x02AA ( 341) - lwax
+    { 0x89,  diCMN, 0x02AC },         //  309 Maj op = 31, mask = 07FE, xop = 0x02AC ( 342) - dst
+    { 0x89,  diCMN, 0x02AE },         //  310 Maj op = 31, mask = 07FE, xop = 0x02AE ( 343) - lhax
+    { 0x89,  diCMN, 0x02CE },         //  311 Maj op = 31, mask = 07FE, xop = 0x02CE ( 359) - lvxl
+    { 0x89,  diPRV, 0x02E4 },         //  312 Maj op = 31, mask = 07FE, xop = 0x02E4 ( 370) - tlbia
+    { 0x89,  diCMN, 0x02E6 },         //  313 Maj op = 31, mask = 07FE, xop = 0x02E6 ( 371) - mftb
+    { 0x89,  diCMN, 0x02EA },         //  314 Maj op = 31, mask = 07FE, xop = 0x02EA ( 373) - lwaux
+    { 0x89,  diCMN, 0x02EC },         //  315 Maj op = 31, mask = 07FE, xop = 0x02EC ( 374) - dstst
+    { 0x89,  diCMN, 0x02EE },         //  316 Maj op = 31, mask = 07FE, xop = 0x02EE ( 375) - lhaux
+    { 0x89,  diPRV, 0x0324 },         //  317 Maj op = 31, mask = 07FE, xop = 0x0324 ( 402) - slbmte
+    { 0x89,  diCMN, 0x032E },         //  318 Maj op = 31, mask = 07FE, xop = 0x032E ( 407) - sthx
+    { 0x89,  diCMN, 0x0338 },         //  319 Maj op = 31, mask = 07FE, xop = 0x0338 ( 412) - orc
+    { 0x89,  diPRV, 0x0364 },         //  320 Maj op = 31, mask = 07FE, xop = 0x0364 ( 434) - slbie
+    { 0x89,  diPRV, 0x036C },         //  321 Maj op = 31, mask = 07FE, xop = 0x036C ( 438) - ecowx
+    { 0x89,  diCMN, 0x036E },         //  322 Maj op = 31, mask = 07FE, xop = 0x036E ( 439) - sthux
+    { 0x89,   diOR, 0x0378 },         //  323 Maj op = 31, mask = 07FE, xop = 0x0378 ( 444) - or
+    { 0x89,  diSPR, 0x03A6 },         //  324 Maj op = 31, mask = 07FE, xop = 0x03A6 ( 467) - mtspr
+    { 0x89,  diCMN, 0x03B8 },         //  325 Maj op = 31, mask = 07FE, xop = 0x03B8 ( 476) - nand
+    { 0x89,  diCMN, 0x03CE },         //  326 Maj op = 31, mask = 07FE, xop = 0x03CE ( 487) - stvxl
+    { 0x89,  diPRV, 0x03E4 },         //  327 Maj op = 31, mask = 07FE, xop = 0x03E4 ( 498) - slbia
+    { 0x89,  diCMN, 0x0400 },         //  328 Maj op = 31, mask = 07FE, xop = 0x0400 ( 512) - mcrxr
+    { 0x89,  diCMN, 0x042A },         //  329 Maj op = 31, mask = 07FE, xop = 0x042A ( 533) - lswx
+    { 0x89,  diCMN, 0x042C },         //  330 Maj op = 31, mask = 07FE, xop = 0x042C ( 534) - lwbrx
+    { 0x89,  diCMN, 0x042E },         //  331 Maj op = 31, mask = 07FE, xop = 0x042E ( 535) - lfsx
+    { 0x89,  diCMN, 0x0430 },         //  332 Maj op = 31, mask = 07FE, xop = 0x0430 ( 536) - srw
+    { 0x89,  diCMN, 0x0436 },         //  333 Maj op = 31, mask = 07FE, xop = 0x0436 ( 539) - srd
+    { 0x89,  diPRV, 0x046C },         //  334 Maj op = 31, mask = 07FE, xop = 0x046C ( 566) - tlbsync
+    { 0x89,  diCMN, 0x046E },         //  335 Maj op = 31, mask = 07FE, xop = 0x046E ( 567) - lfsux
+    { 0x89,  diPRV, 0x04A6 },         //  336 Maj op = 31, mask = 07FE, xop = 0x04A6 ( 595) - mfsr
+    { 0x89,  diCMN, 0x04AA },         //  337 Maj op = 31, mask = 07FE, xop = 0x04AA ( 597) - lswi
+    { 0x89,  diCMN, 0x04AC },         //  338 Maj op = 31, mask = 07FE, xop = 0x04AC ( 598) - sync
+    { 0x89,  diCMN, 0x04AE },         //  339 Maj op = 31, mask = 07FE, xop = 0x04AE ( 599) - lfdx
+    { 0x89,  diCMN, 0x04EE },         //  340 Maj op = 31, mask = 07FE, xop = 0x04EE ( 631) - lfdux
+    { 0x89,  diPRV, 0x0526 },         //  341 Maj op = 31, mask = 07FE, xop = 0x0526 ( 659) - mfsrin
+    { 0x89,  diCMN, 0x052A },         //  342 Maj op = 31, mask = 07FE, xop = 0x052A ( 661) - stswx
+    { 0x89,  diCMN, 0x052C },         //  343 Maj op = 31, mask = 07FE, xop = 0x052C ( 662) - stwbrx
+    { 0x89,  diCMN, 0x052E },         //  344 Maj op = 31, mask = 07FE, xop = 0x052E ( 663) - stfsx
+    { 0x89,  diCMN, 0x056E },         //  345 Maj op = 31, mask = 07FE, xop = 0x056E ( 695) - stfsux
+    { 0x89,  diCMN, 0x05AA },         //  346 Maj op = 31, mask = 07FE, xop = 0x05AA ( 725) - stswi
+    { 0x89,  diCMN, 0x05AE },         //  347 Maj op = 31, mask = 07FE, xop = 0x05AE ( 727) - stfdx
+    { 0x89,  diCMN, 0x05EC },         //  348 Maj op = 31, mask = 07FE, xop = 0x05EC ( 758) - dcba
+    { 0x89,  diCMN, 0x05EE },         //  349 Maj op = 31, mask = 07FE, xop = 0x05EE ( 759) - stfdux
+    { 0x89,  diCMN, 0x062C },         //  350 Maj op = 31, mask = 07FE, xop = 0x062C ( 790) - lhbrx
+    { 0x89,  diCMN, 0x0630 },         //  351 Maj op = 31, mask = 07FE, xop = 0x0630 ( 792) - sraw
+    { 0x89,  diCMN, 0x0634 },         //  352 Maj op = 31, mask = 07FE, xop = 0x0634 ( 794) - srad
+    { 0x89,  diCMN, 0x066C },         //  353 Maj op = 31, mask = 07FE, xop = 0x066C ( 822) - dss
+    { 0x89,  diCMN, 0x0670 },         //  354 Maj op = 31, mask = 07FE, xop = 0x0670 ( 824) - srawi
+    { 0x89,  diPRV, 0x06A6 },         //  355 Maj op = 31, mask = 07FE, xop = 0x06A6 ( 851) - slbmfev
+    { 0x89,  diCMN, 0x06AC },         //  356 Maj op = 31, mask = 07FE, xop = 0x06AC ( 854) - eieio
+    { 0x89,  diPRV, 0x0726 },         //  357 Maj op = 31, mask = 07FE, xop = 0x0726 ( 915) - slbmfee
+    { 0x89,  diCMN, 0x072C },         //  358 Maj op = 31, mask = 07FE, xop = 0x072C ( 918) - sthbrx
+    { 0x89,  diCMN, 0x0734 },         //  359 Maj op = 31, mask = 07FE, xop = 0x0734 ( 922) - extsh
+    { 0x89,  diCMN, 0x0774 },         //  360 Maj op = 31, mask = 07FE, xop = 0x0774 ( 954) - extsb
+    { 0x89,  diCMN, 0x07AC },         //  361 Maj op = 31, mask = 07FE, xop = 0x07AC ( 982) - icbi
+    { 0x89,  diCMN, 0x07AE },         //  362 Maj op = 31, mask = 07FE, xop = 0x07AE ( 983) - stfiwx
+    { 0x89,  diCMN, 0x07B4 },         //  363 Maj op = 31, mask = 07FE, xop = 0x07B4 ( 986) - extsw
+    { 0x09,  diCMN, 0x07EC },         //  364 Maj op = 31, mask = 07FE, xop = 0x07EC (1014) - dcbz
+    { 0x81,  diCMN, 0x0000 },         //  365 Maj op = 58, mask = 0003, xop = 0x0000 (   0) - ld
+    { 0x81,  diCMN, 0x0001 },         //  366 Maj op = 58, mask = 0003, xop = 0x0001 (   1) - ldu
+    { 0x01,  diCMN, 0x0002 },         //  367 Maj op = 58, mask = 0003, xop = 0x0002 (   2) - lwa
+    { 0x84,  diCMN, 0x0024 },         //  368 Maj op = 59, mask = 003E, xop = 0x0024 (  18) - fdivs
+    { 0x84,  diCMN, 0x0028 },         //  369 Maj op = 59, mask = 003E, xop = 0x0028 (  20) - fsubs
+    { 0x84,  diCMN, 0x002A },         //  370 Maj op = 59, mask = 003E, xop = 0x002A (  21) - fadds
+    { 0x84,  diCMN, 0x002C },         //  371 Maj op = 59, mask = 003E, xop = 0x002C (  22) - fsqrts
+    { 0x84,  diCMN, 0x0030 },         //  372 Maj op = 59, mask = 003E, xop = 0x0030 (  24) - fres
+    { 0x84,  diCMN, 0x0032 },         //  373 Maj op = 59, mask = 003E, xop = 0x0032 (  25) - fmuls
+    { 0x84,  diCMN, 0x0038 },         //  374 Maj op = 59, mask = 003E, xop = 0x0038 (  28) - fmsubs
+    { 0x84,  diCMN, 0x003A },         //  375 Maj op = 59, mask = 003E, xop = 0x003A (  29) - fmadds
+    { 0x84,  diCMN, 0x003C },         //  376 Maj op = 59, mask = 003E, xop = 0x003C (  30) - fnmsubs
+    { 0x04,  diCMN, 0x003E },         //  377 Maj op = 59, mask = 003E, xop = 0x003E (  31) - fnmadds
+    { 0x81,  diCMN, 0x0000 },         //  378 Maj op = 62, mask = 0003, xop = 0x0000 (   0) - std
+    { 0x01,  diCMN, 0x0001 },         //  379 Maj op = 62, mask = 0003, xop = 0x0001 (   1) - stdu
+    { 0x84,  diCMN, 0x0024 },         //  380 Maj op = 63, mask = 003E, xop = 0x0024 (  18) - fdiv
+    { 0x84,  diCMN, 0x0028 },         //  381 Maj op = 63, mask = 003E, xop = 0x0028 (  20) - fsub
+    { 0x84,  diCMN, 0x002A },         //  382 Maj op = 63, mask = 003E, xop = 0x002A (  21) - fadd
+    { 0x84,  diCMN, 0x002C },         //  383 Maj op = 63, mask = 003E, xop = 0x002C (  22) - fsqrt
+    { 0x84,  diCMN, 0x002E },         //  384 Maj op = 63, mask = 003E, xop = 0x002E (  23) - fsel
+    { 0x84,  diCMN, 0x0032 },         //  385 Maj op = 63, mask = 003E, xop = 0x0032 (  25) - fmul
+    { 0x84,  diCMN, 0x0034 },         //  386 Maj op = 63, mask = 003E, xop = 0x0034 (  26) - frsqrte
+    { 0x84,  diCMN, 0x0038 },         //  387 Maj op = 63, mask = 003E, xop = 0x0038 (  28) - fmsub
+    { 0x84,  diCMN, 0x003A },         //  388 Maj op = 63, mask = 003E, xop = 0x003A (  29) - fmadd
+    { 0x84,  diCMN, 0x003C },         //  389 Maj op = 63, mask = 003E, xop = 0x003C (  30) - fnmsub
+    { 0x84,  diCMN, 0x003E },         //  390 Maj op = 63, mask = 003E, xop = 0x003E (  31) - fnmadd
+    { 0x89,  diCMN, 0x0000 },         //  391 Maj op = 63, mask = 07FE, xop = 0x0000 (   0) - fcmpu
+    { 0x89,  diCMN, 0x0018 },         //  392 Maj op = 63, mask = 07FE, xop = 0x0018 (  12) - frsp
+    { 0x89,  diCMN, 0x001C },         //  393 Maj op = 63, mask = 07FE, xop = 0x001C (  14) - fctiw
+    { 0x89,  diCMN, 0x001E },         //  394 Maj op = 63, mask = 07FE, xop = 0x001E (  15) - fctiwz
+    { 0x89,  diCMN, 0x0040 },         //  395 Maj op = 63, mask = 07FE, xop = 0x0040 (  32) - fcmpo
+    { 0x89,  diCMN, 0x004C },         //  396 Maj op = 63, mask = 07FE, xop = 0x004C (  38) - mtfsb1
+    { 0x89,  diCMN, 0x0050 },         //  397 Maj op = 63, mask = 07FE, xop = 0x0050 (  40) - fneg
+    { 0x89,  diCMN, 0x0080 },         //  398 Maj op = 63, mask = 07FE, xop = 0x0080 (  64) - mcrfs
+    { 0x89,  diCMN, 0x008C },         //  399 Maj op = 63, mask = 07FE, xop = 0x008C (  70) - mtfsb0
+    { 0x89,  diCMN, 0x0090 },         //  400 Maj op = 63, mask = 07FE, xop = 0x0090 (  72) - fmr
+    { 0x89,  diCMN, 0x010C },         //  401 Maj op = 63, mask = 07FE, xop = 0x010C ( 134) - mtfsfi
+    { 0x89,  diCMN, 0x0110 },         //  402 Maj op = 63, mask = 07FE, xop = 0x0110 ( 136) - fnabs
+    { 0x89,  diCMN, 0x0210 },         //  403 Maj op = 63, mask = 07FE, xop = 0x0210 ( 264) - fabs
+    { 0x89,  diCMN, 0x048E },         //  404 Maj op = 63, mask = 07FE, xop = 0x048E ( 583) - mffs
+    { 0x89,  diCMN, 0x058E },         //  405 Maj op = 63, mask = 07FE, xop = 0x058E ( 711) - mtfsf
+    { 0x89,  diCMN, 0x065C },         //  406 Maj op = 63, mask = 07FE, xop = 0x065C ( 814) - fctid
+    { 0x89,  diCMN, 0x065E },         //  407 Maj op = 63, mask = 07FE, xop = 0x065E ( 815) - fctidz
+    { 0x09,  diCMN, 0x069C },         //  408 Maj op = 63, mask = 07FE, xop = 0x069C ( 846) - fcfid
+};
+
+#ifdef __decodePPC_debug__
+char *instname[] = {
+    "Jump entry...",
+    "Invalid",
+    "tdi",
+    "twi",
+    "Jump entry...",
+    "Invalid",
+    "Invalid",
+    "mulli",
+    "subfic",
+    "Invalid",
+    "cmpli",
+    "cmpi",
+    "addic",
+    "addic.",
+    "addi",
+    "addis",
+    "bc",
+    "sc",
+    "b",
+    "Jump entry...",
+    "rlwimi",
+    "rlwinm",
+    "Invalid",
+    "rlwnm",
+    "ori",
+    "oris",
+    "xori",
+    "xoris",
+    "andi.",
+    "andis.",
+    "Jump entry...",
+    "Jump entry...",
+    "lwz",
+    "lwzu",
+    "lbz",
+    "lbzu",
+    "stw",
+    "stwu",
+    "stb",
+    "stbu",
+    "lhz",
+    "lhzu",
+    "lha",
+    "lhau",
+    "sth",
+    "sthu",
+    "lmw",
+    "stmw",
+    "lfs",
+    "lfsu",
+    "lfd",
+    "lfdu",
+    "stfs",
+    "stfsu",
+    "stfd",
+    "stfdu",
+    "Invalid",
+    "Invalid",
+    "Jump entry...",
+    "Jump entry...",
+    "Invalid",
+    "Invalid",
+    "Jump entry...",
+    "Jump entry...",
+    "attn",
+    "vmhaddshs",
+    "vmhraddshs",
+    "vmladduhm",
+    "vmsumubm",
+    "vmsummbm",
+    "vmsumuhm",
+    "vmsumuhs",
+    "vmsumshm",
+    "vmsumshs",
+    "vsel",
+    "vperm",
+    "vsldoi",
+    "vmaddfp",
+    "vnmsubfp",
+    "vcmpequb",
+    "vcmpequh",
+    "vcmpequw",
+    "vcmpeqfp",
+    "vcmpgefp",
+    "vcmpgtub",
+    "vcmpgtuh",
+    "vcmpgtuw",
+    "vcmpgtfp",
+    "vcmpgtsb",
+    "vcmpgtsh",
+    "vcmpgtsw",
+    "vcmpbfp",
+    "vaddubm",
+    "vmaxub",
+    "vrlb",
+    "vmuloub",
+    "vaddfp",
+    "vmrghb",
+    "vpkuhum",
+    "vadduhm",
+    "vmaxuh",
+    "vrlh",
+    "vmulouh",
+    "vsubfp",
+    "vmrghh",
+    "vpkuwum",
+    "vadduwm",
+    "vmaxuw",
+    "vrlw",
+    "vmrghw",
+    "vpkuhus",
+    "vpkuwus",
+    "vmaxsb",
+    "vslb",
+    "vmulosb",
+    "vrefp",
+    "vmrglb",
+    "vpkshus",
+    "vmaxsh",
+    "vslh",
+    "vmulosh",
+    "vrsqrtefp",
+    "vmrglh",
+    "vpkswus",
+    "vaddcuw",
+    "vmaxsw",
+    "vslw",
+    "vexptefp",
+    "vmrglw",
+    "vpkshss",
+    "vsl",
+    "vlogefp",
+    "vpkswss",
+    "vaddubs",
+    "vminub",
+    "vsrb",
+    "vmuleub",
+    "vrfin",
+    "vspltb",
+    "vupkhsb",
+    "vadduhs",
+    "vminuh",
+    "vsrh",
+    "vmuleuh",
+    "vrfiz",
+    "vsplth",
+    "vupkhsh",
+    "vadduws",
+    "vminuw",
+    "vsrw",
+    "vrfip",
+    "vspltw",
+    "vupklsb",
+    "vsr",
+    "vrfim",
+    "vupklsh",
+    "vaddsbs",
+    "vminsb",
+    "vsrab",
+    "vmulesb",
+    "vcfux",
+    "vspltisb",
+    "vpkpx",
+    "vaddshs",
+    "vminsh",
+    "vsrah",
+    "vmulesh",
+    "vcfsx",
+    "vspltish",
+    "vupkhpx",
+    "vaddsws",
+    "vminsw",
+    "vsraw",
+    "vctuxs",
+    "vspltisw",
+    "vctsxs",
+    "vupklpx",
+    "vsububm",
+    "vavgub",
+    "vand",
+    "vmaxfp",
+    "vslo",
+    "vsubuhm",
+    "vavguh",
+    "vandc",
+    "vminfp",
+    "vsro",
+    "vsubuwm",
+    "vavguw",
+    "vor",
+    "vxor",
+    "vavgsb",
+    "vnor",
+    "vavgsh",
+    "vsubcuw",
+    "vavgsw",
+    "vsububs",
+    "mfvscr",
+    "vsum4ubs",
+    "vsubuhs",
+    "mtvscr",
+    "vsum4shs",
+    "vsubuws",
+    "vsum2sws",
+    "vsubsbs",
+    "vsum4sbs",
+    "vsubshs",
+    "vsubsws",
+    "vsumsws",
+    "mcrf",
+    "bclr",
+    "rfid",
+    "crnor",
+    "rfi",
+    "crandc",
+    "isync",
+    "crxor",
+    "crnand",
+    "crand",
+    "hrfid",
+    "creqv",
+    "crorc",
+    "cror",
+    "bctr",
+    "rldicl",
+    "rldicr",
+    "rldic",
+    "rldimi",
+    "rldcl",
+    "rldcr",
+    "subfc",
+    "mulhdu",
+    "addc",
+    "mulhwu",
+    "subf",
+    "mulhd",
+    "mulhw",
+    "neg",
+    "subfe",
+    "adde",
+    "subfze",
+    "addze",
+    "subfme",
+    "mulld",
+    "addme",
+    "mullw",
+    "add",
+    "divdu",
+    "divwu",
+    "divd",
+    "divw",
+    "sradi",
+    "cmp",
+    "tw",
+    "lvsl",
+    "lvebx",
+    "mfcr",
+    "lwarx",
+    "ldx",
+    "lwzx",
+    "slw",
+    "cntlzw",
+    "sld",
+    "and",
+    "cmpl",
+    "lvsr",
+    "lvehx",
+    "ldux",
+    "dcbst",
+    "lwzux",
+    "cntlzd",
+    "andc",
+    "td",
+    "lvewx",
+    "mfmsr",
+    "ldarx",
+    "dcbf",
+    "lbzx",
+    "lvx",
+    "lbzux",
+    "nor",
+    "stvebx",
+    "mtcrf",
+    "mtmsr",
+    "stdx",
+    "stwcx",
+    "stwx",
+    "stvehx",
+    "mtmsrd",
+    "stdux",
+    "stwux",
+    "stvewx",
+    "mtsr",
+    "stdcx.",
+    "stbx",
+    "stvx",
+    "mtsrin",
+    "dcbtst",
+    "stbux",
+    "tlbiel",
+    "dcbt",
+    "lhzx",
+    "eqv",
+    "tlbie",
+    "eciwx",
+    "lhzux",
+    "xor",
+    "mfspr",
+    "lwax",
+    "dst",
+    "lhax",
+    "lvxl",
+    "tlbia",
+    "mftb",
+    "lwaux",
+    "dstst",
+    "lhaux",
+    "slbmte",
+    "sthx",
+    "orc",
+    "slbie",
+    "ecowx",
+    "sthux",
+    "or",
+    "mtspr",
+    "nand",
+    "stvxl",
+    "slbia",
+    "mcrxr",
+    "lswx",
+    "lwbrx",
+    "lfsx",
+    "srw",
+    "srd",
+    "tlbsync",
+    "lfsux",
+    "mfsr",
+    "lswi",
+    "sync",
+    "lfdx",
+    "lfdux",
+    "mfsrin",
+    "stswx",
+    "stwbrx",
+    "stfsx",
+    "stfsux",
+    "stswi",
+    "stfdx",
+    "dcba",
+    "stfdux",
+    "lhbrx",
+    "sraw",
+    "srad",
+    "dss",
+    "srawi",
+    "slbmfev",
+    "eieio",
+    "slbmfee",
+    "sthbrx",
+    "extsh",
+    "extsb",
+    "icbi",
+    "stfiwx",
+    "extsw",
+    "dcbz",
+    "ld",
+    "ldu",
+    "lwa",
+    "fdivs",
+    "fsubs",
+    "fadds",
+    "fsqrts",
+    "fres",
+    "fmuls",
+    "fmsubs",
+    "fmadds",
+    "fnmsubs",
+    "fnmadds",
+    "std",
+    "stdu",
+    "fdiv",
+    "fsub",
+    "fadd",
+    "fsqrt",
+    "fsel",
+    "fmul",
+    "frsqrte",
+    "fmsub",
+    "fmadd",
+    "fnmsub",
+    "fnmadd",
+    "fcmpu",
+    "frsp",
+    "fctiw",
+    "fctiwz",
+    "fcmpo",
+    "mtfsb1",
+    "fneg",
+    "mcrfs",
+    "mtfsb0",
+    "fmr",
+    "mtfsfi",
+    "fnabs",
+    "fabs",
+    "mffs",
+    "mtfsf",
+    "fctid",
+    "fctidz",
+    "fcfid",
+};
+#endif
+
+static dcdtab dcdfail = { 0x00,	 diINV, 0x0000 };	// Decode failed
+
+static uint32_t sprtbl[] = {
+    0xCCC03274,						// spr    0 to   31
+    0x00000000,						// spr   32 to   63
+    0x00000000,						// spr   64 to   95
+    0x00000000,						// spr   96 to  127
+    0x00000080,						// spr  128 to  159
+    0x00000000,						// spr  160 to  191
+    0x00000000,						// spr  192 to  223
+    0x00000000,						// spr  224 to  255
+    0x9000FCAD,						// spr  256 to  287
+    0x0000C3F3,						// spr  288 to  319
+    0x00000000,						// spr  320 to  351
+    0x00000000,						// spr  352 to  383
+    0x00000000,						// spr  384 to  415
+    0x00000000,						// spr  416 to  447
+    0x00000000,						// spr  448 to  479
+    0x00000000,						// spr  480 to  511
+    0x0000FFFF,						// spr  512 to  543
+    0x00000000,						// spr  544 to  575
+    0x00000000,						// spr  576 to  607
+    0x00000000,						// spr  608 to  639
+    0x00000000,						// spr  640 to  671
+    0x00000000,						// spr  672 to  703
+    0x00000000,						// spr  704 to  735
+    0x00000000,						// spr  736 to  767
+    0x3FFF3FFF,						// spr  768 to  799
+    0x00000000,						// spr  800 to  831
+    0x00000000,						// spr  832 to  863
+    0x00000000,						// spr  864 to  895
+    0x00000000,						// spr  896 to  927
+    0xE1FFE1FF,						// spr  928 to  959
+    0x0000FE80,						// spr  960 to  991
+    0x0000FFFF,						// spr  992 to 1023
+};
diff --git a/bsd/ppc/disklabel.h b/bsd/ppc/disklabel.h
index 9d97865f0..f7cfe1155 100644
--- a/bsd/ppc/disklabel.h
+++ b/bsd/ppc/disklabel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 
diff --git a/bsd/ppc/endian.h b/bsd/ppc/endian.h
index 72808459e..c6929f117 100644
--- a/bsd/ppc/endian.h
+++ b/bsd/ppc/endian.h
@@ -1,46 +1,58 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All rights reserved.
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1987, 1991, 1993
@@ -80,6 +92,8 @@
 #ifndef _PPC_ENDIAN_H_
 #define	_PPC_ENDIAN_H_
 
+#include <sys/cdefs.h>
+
 /*
  * Define the order of 32-bit words in 64-bit words.
  */
@@ -96,7 +110,7 @@
 
 #define	__DARWIN_BYTE_ORDER	__DARWIN_BIG_ENDIAN
 
-#if	defined(KERNEL) || !defined(_POSIX_C_SOURCE)
+#if	defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 
 #define	LITTLE_ENDIAN	__DARWIN_LITTLE_ENDIAN
 #define	BIG_ENDIAN	__DARWIN_BIG_ENDIAN
@@ -106,5 +120,5 @@
 
 #include <sys/_endian.h>
 
-#endif /* defined(KERNEL) || !defined(_POSIX_C_SOURCE) */
+#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
 #endif /* !_PPC_ENDIAN_H_ */
diff --git a/bsd/ppc/exec.h b/bsd/ppc/exec.h
index 86024c6d6..471543a1d 100644
--- a/bsd/ppc/exec.h
+++ b/bsd/ppc/exec.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1994, The University of Utah and
diff --git a/bsd/ppc/fasttrap_isa.h b/bsd/ppc/fasttrap_isa.h
new file mode 100644
index 000000000..b4a2cb4c2
--- /dev/null
+++ b/bsd/ppc/fasttrap_isa.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_FASTTRAP_ISA_H
+#define	_FASTTRAP_ISA_H
+
+/* #pragma ident	"@(#)fasttrap_isa.h	1.4	05/06/08 SMI" */
+
+#include <sys/types.h>
+#if defined(__APPLE__)
+#include <stdint.h>
+#endif 
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef	uint32_t		fasttrap_instr_t;
+
+typedef struct fasttrap_machtp {
+	fasttrap_instr_t	ftmt_instr;     /* Original instruction */
+	int32_t				ftmt_trgt;		/* Offset or absolute address */
+	uint8_t				ftmt_type;		/* Emulation function type */
+#define ftmtNOP 0
+#define ftmtCommon 1
+#define ftmtB 2
+#define ftmtBC 3
+#define ftmtBLR 4
+#define ftmtBCTR 5
+	uint8_t				ftmt_bo;		/* Branch options */
+	uint8_t				ftmt_bi;		/* Condition bit */
+	uint8_t				ftmt_flgs;		/* Flags */
+#define ftmtAbs 2
+#define ftmtLink 1
+} fasttrap_machtp_t;
+
+#define	ftt_instr	ftt_mtp.ftmt_instr
+#define	ftt_trgt	ftt_mtp.ftmt_trgt
+#define	ftt_type	ftt_mtp.ftmt_type
+#define	ftt_bo		ftt_mtp.ftmt_bo
+#define	ftt_bi		ftt_mtp.ftmt_bi
+#define	ftt_flgs	ftt_mtp.ftmt_flgs
+
+#define	FASTTRAP_INSTR 0x0FFFDDDD
+#define T_DTRACE_RET (0x2E * 4)
+
+#define	FASTTRAP_RETURN_AFRAMES		7
+#define	FASTTRAP_ENTRY_AFRAMES		7
+#define	FASTTRAP_OFFSET_AFRAMES		6
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _FASTTRAP_ISA_H */
diff --git a/bsd/ppc/param.h b/bsd/ppc/param.h
index adcdd4e23..a434e3c4c 100644
--- a/bsd/ppc/param.h
+++ b/bsd/ppc/param.h
@@ -1,36 +1,44 @@
 /*
- * Copyright (c) 2000-2004, 2007 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1993,1995 NeXT Computer, Inc. All Rights Reserved */
 
 #ifndef	_PPC_PARAM_H_
 #define	_PPC_PARAM_H_
 
+#include <ppc/_param.h>
+
 /*
  * Round p (pointer or byte index) up to a correctly-aligned value for all
- * data types (int, long, ...).   The result is unsigned long and must be
+ * data types (int, long, ...).   The result is unsigned int and must be
  * cast to any desired pointer type.
  */
-#define	ALIGNBYTES	(sizeof(unsigned long) - 1)
-#define	ALIGN(p)	((unsigned long)((char *)(p) + ALIGNBYTES) &~ ALIGNBYTES)
+#define	ALIGNBYTES	__DARWIN_ALIGNBYTES
+#define	ALIGN(p)	__DARWIN_ALIGN(p)
 
 #define	NBPG		4096		/* bytes/page */
 #define	PGOFSET		(NBPG-1)	/* byte offset into page */
diff --git a/bsd/ppc/profile.h b/bsd/ppc/profile.h
index a7263bd42..7be38b3a9 100644
--- a/bsd/ppc/profile.h
+++ b/bsd/ppc/profile.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997, Apple Computer, Inc. All rights reserved.
@@ -37,7 +43,7 @@
  * PPC platfom, can't do splhigh/splx as those are C routines and can
  * recursively invoke mcount.
  */
-extern unsigned long disable_ee();
+extern unsigned long disable_ee(void);
 extern void restore_ee(unsigned long smsr);
 
 #define MCOUNT_INIT		register unsigned long smsr;
diff --git a/bsd/ppc/psl.h b/bsd/ppc/psl.h
index 7e081d882..14abec125 100644
--- a/bsd/ppc/psl.h
+++ b/bsd/ppc/psl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1993 NeXT Computer, Inc.  All rights reserved.
  *
diff --git a/bsd/ppc/ptrace.h b/bsd/ppc/ptrace.h
index 769354c3f..be9af6886 100644
--- a/bsd/ppc/ptrace.h
+++ b/bsd/ppc/ptrace.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992, 1993
diff --git a/bsd/ppc/reboot.h b/bsd/ppc/reboot.h
index 576b8658c..75e3a7656 100644
--- a/bsd/ppc/reboot.h
+++ b/bsd/ppc/reboot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #ifndef	_BSD_PPC_REBOOT_H_
diff --git a/bsd/ppc/reg.h b/bsd/ppc/reg.h
index b45306d2c..0449be6df 100644
--- a/bsd/ppc/reg.h
+++ b/bsd/ppc/reg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1993, NeXT Computer, Inc.
diff --git a/bsd/ppc/setjmp.h b/bsd/ppc/setjmp.h
index cb9c7cd33..27eb59ab0 100644
--- a/bsd/ppc/setjmp.h
+++ b/bsd/ppc/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
  *
@@ -29,13 +35,26 @@
 #define _BSD_PPC_SETJMP_H_
 
 #include <sys/cdefs.h>
-#include <machine/signal.h>
+
+#define __need_struct_sigcontext
+#if defined(KERNEL)
+#define __need_struct_sigcontext32
+#define __need_struct_sigcontext64
+#endif /* KERNEL */
+#include <ppc/_structs.h>
 
 struct _jmp_buf {
-	struct sigcontext	sigcontext; /* kernel state preserved by set/longjmp */
+#if __DARWIN_UNIX03
+	_STRUCT_SIGCONTEXT	__sigcontext; /* kernel state preserved by set/longjmp */
+	unsigned int __vmask __attribute__((aligned(8))); /* vector mask register */
+	unsigned int __vreg[32 * 4] __attribute__((aligned(16)));
+		/* 32 128-bit vector registers */
+#else /* !__DARWIN_UNIX03 */
+	_STRUCT_SIGCONTEXT	sigcontext; /* kernel state preserved by set/longjmp */
 	unsigned int vmask __attribute__((aligned(8))); /* vector mask register */
 	unsigned int vreg[32 * 4] __attribute__((aligned(16)));
 		/* 32 128-bit vector registers */
+#endif /* __DARWIN_UNIX03 */
 };
 
 /*
@@ -61,7 +80,7 @@ struct _jmp_buf {
 #endif
 
 #if defined(KERNEL)
-typedef struct sigcontext32 jmp_buf32[1];
+typedef _STRUCT_SIGCONTEXT32 jmp_buf32[1];
 typedef struct __sigjmp_buf32 {
 		int __storage[_JBLEN32 + 1] __attribute__((aligned(8)));
 		} sigjmp_buf32[1];
@@ -75,7 +94,7 @@ typedef struct __sigjmp_buf64 {
  * JMM - have to decide how the kernel will deal with this.
  * For now, hard-code the 32-bit types.
  */
-typedef struct sigcontext32 jmp_buf[1];
+typedef _STRUCT_SIGCONTEXT32 jmp_buf[1];
 typedef struct __sigjmp_buf32 sigjmp_buf[1];
 
 #else
@@ -84,17 +103,17 @@ typedef int sigjmp_buf[_JBLEN + 1];
 #endif
 
 __BEGIN_DECLS
-extern int setjmp(jmp_buf env);
-extern void longjmp(jmp_buf env, int val);
+int	setjmp(jmp_buf);
+void	longjmp(jmp_buf, int);
 
 #ifndef _ANSI_SOURCE
-int	_setjmp(jmp_buf env);
-void	_longjmp(jmp_buf, int val);
-int sigsetjmp(sigjmp_buf env, int val);
-void siglongjmp(sigjmp_buf env, int val);
+int	_setjmp(jmp_buf);
+void	_longjmp(jmp_buf, int);
+int	sigsetjmp(sigjmp_buf, int);
+void	siglongjmp(sigjmp_buf, int);
 #endif /* _ANSI_SOURCE  */
 
-#if !defined(_ANSI_SOURCE) && !defined(_POSIX_C_SOURCE)
+#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 void	longjmperror(void);
 #endif /* neither ANSI nor POSIX */
 __END_DECLS
diff --git a/bsd/ppc/signal.h b/bsd/ppc/signal.h
index fee82c365..31af83a02 100644
--- a/bsd/ppc/signal.h
+++ b/bsd/ppc/signal.h
@@ -1,35 +1,53 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992, 1993 NeXT Computer, Inc.
  */
 
-#ifndef	_PPC_SIGNAL_
-#define	_PPC_SIGNAL_ 1
+#ifndef	_PPC_SIGNAL_H_
+#define	_PPC_SIGNAL_H_ 1
+
+#include <sys/cdefs.h>
+
+#ifndef _ANSI_SOURCE
+
+typedef int sig_atomic_t; 
 
 #include <sys/appleapiopts.h>
 
 #ifdef __APPLE_API_OBSOLETE
-typedef int sig_atomic_t; 
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+
+#define __need_struct_sigcontext
+#define __need_struct_sigcontext32
+#define __need_struct_sigcontext64
+#include <ppc/_structs.h>
 
 /*
  * Machine-dependant flags used in sigvec call.
@@ -55,46 +73,11 @@ typedef enum {
 	REGS_SAVED_ALL			/* All registers */
 } regs_saved_t;
 
-
-/*
- * Information pushed on stack when a signal is delivered.
- * This is used by the kernel to restore state following
- * execution of the signal handler.  It is also made available
- * to the handler to allow it to properly restore state if
- * a non-standard exit is performed.
- */
-struct sigcontext32 {
-    int		sc_onstack;     /* sigstack state to restore */
-    int		sc_mask;        /* signal mask to restore */
-    int		sc_ir;			/* pc */
-    int		sc_psw;         /* processor status word */
-    int		sc_sp;      	/* stack pointer if sc_regs == NULL */
-    void	*sc_regs;		/* (kernel private) saved state */
-};
-
-struct sigcontext64 {
-    int		sc_onstack;     /* sigstack state to restore */
-    int		sc_mask;        /* signal mask to restore */
-    long long	sc_ir;		/* pc */
-    long long	sc_psw;         /* processor status word */
-    long long	sc_sp;      	/* stack pointer if sc_regs == NULL */
-    void	*sc_regs;	/* (kernel private) saved state */
-};
-
-/*
- * LP64todo - Have to decide how to handle this.
- * For now, just duplicate the 32-bit context as the generic one.
- */
-struct sigcontext {
-    int		sc_onstack;     /* sigstack state to restore */
-    int		sc_mask;        /* signal mask to restore */
-    int		sc_ir;			/* pc */
-    int		sc_psw;         /* processor status word */
-    int		sc_sp;      	/* stack pointer if sc_regs == NULL */
-    void	*sc_regs;		/* (kernel private) saved state */
-};
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #endif /* __APPLE_API_OBSOLETE */
 
-#endif /* _PPC_SIGNAL_ */
+#endif /* _ANSI_SOURCE */
+
+#endif /* _PPC_SIGNAL_H_ */
 
diff --git a/bsd/ppc/types.h b/bsd/ppc/types.h
index 543bfed53..dca303748 100644
--- a/bsd/ppc/types.h
+++ b/bsd/ppc/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright 1995 NeXT Computer, Inc. All rights reserved.
@@ -103,12 +109,11 @@ typedef __darwin_intptr_t	intptr_t;
 typedef unsigned long		uintptr_t;
 #endif
 
+#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 /* with LP64 support pointers and longs from user address space may vary */
 /* in size depending on the type of process (currently 32 or 64-bit, but */
-/* may change in the future).  These types are used for  reserving the largest */
+/* may change in the future).  These types are used for reserving the largest */
 /* possible size. */
-// LP64todo - typedef mach_vm_address_t	user_addr_t;	/* varying length pointers from user space */ 
-// LP64todo - typedef mach_vm_size_t		user_size_t;	/* varying length values from user space (unsigned) */
 typedef u_int64_t	user_addr_t;	
 typedef u_int64_t	user_size_t;	
 typedef int64_t		user_ssize_t;	
@@ -117,6 +122,7 @@ typedef u_int64_t	user_ulong_t;
 typedef int64_t		user_time_t;
 #define USER_ADDR_NULL	((user_addr_t) 0)
 #define CAST_USER_ADDR_T(a_ptr)   ((user_addr_t)((uintptr_t)(a_ptr)))
+#endif /* !_ANSI_SOURCE && (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /* This defines the size of syscall arguments after copying into the kernel: */
 typedef u_int64_t	syscall_arg_t;
diff --git a/bsd/ppc/ucontext.h b/bsd/ppc/ucontext.h
index ab434a1d4..5c391c283 100644
--- a/bsd/ppc/ucontext.h
+++ b/bsd/ppc/ucontext.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _PPC_UCONTEXT_H_
@@ -26,7 +32,7 @@
 
 #include <mach/ppc/_types.h>
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct mcontext {
 	struct ppc_exception_state	es;
 	struct ppc_thread_state		ss;
@@ -34,21 +40,21 @@ struct mcontext {
 	struct ppc_vector_state		vs;
 };
 #define PPC_MCONTEXT_SIZE	(PPC_THREAD_STATE_COUNT + PPC_FLOAT_STATE_COUNT + PPC_EXCEPTION_STATE_COUNT + PPC_VECTOR_STATE_COUNT) * sizeof(int)
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_mcontext {
 	struct __darwin_ppc_exception_state	es;
 	struct __darwin_ppc_thread_state	ss;
 	struct __darwin_ppc_float_state		fs;
 	struct __darwin_ppc_vector_state	vs;
 };
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 
 #ifndef _MCONTEXT_T
 #define _MCONTEXT_T
 typedef __darwin_mcontext_t		mcontext_t;
 #endif
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 struct mcontext64 {
 	struct ppc_exception_state64	es;
 	struct ppc_thread_state64	ss;
@@ -62,6 +68,6 @@ struct mcontext64 {
 typedef struct mcontext64  * mcontext64_t;
 #endif
 
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #endif /* _PPC_UCONTEXT_H_ */
diff --git a/bsd/ppc/vmparam.h b/bsd/ppc/vmparam.h
index 013608b64..8e682fcdf 100644
--- a/bsd/ppc/vmparam.h
+++ b/bsd/ppc/vmparam.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_BSD_PPC_VMPARAM_H_
@@ -31,13 +37,15 @@
  * put the default 64-bit stack at the max address
  * (minus one 32-bit address space for other incidentals)
  */
-#define	USRSTACK64	(MACH_VM_MAX_ADDRESS - VM_MAX_ADDRESS)
+#define	USRSTACK64 (0x00007FFF5FC00000ULL)
 
 /*
  * Virtual memory related constants, all in bytes
  */
 #ifndef DFLDSIZ
-#define	DFLDSIZ		(6*1024*1024)		/* initial data size limit */
+#define	DFLDSIZ		(RLIM_INFINITY)		/* initial data size limit */
+// XXX Not enforced
+//#define DFLDSIZ	(6*1024*1024)		/* initial data size limit */
 #endif
 #ifndef MAXDSIZ
 #define	MAXDSIZ		(RLIM_INFINITY)		/* max data size */
diff --git a/bsd/sys/Makefile b/bsd/sys/Makefile
index 5e10aaed1..3324fe92a 100644
--- a/bsd/sys/Makefile
+++ b/bsd/sys/Makefile
@@ -19,82 +19,102 @@ EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
-# In both the framework PrivateHeader area and /usr/include/sys
+# Installs header file for user level -  
+#	  $(DSTROOT)/System/Library/Frameworks/System.framework/Headers
+#	  $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
+#	  $(DSTROOT)/usr/include/
 DATAFILES = \
 	appleapiopts.h 	acct.h aio.h attr.h \
 	buf.h cdefs.h conf.h \
-	dir.h dirent.h disk.h dkstat.h  \
-	errno.h ev.h event.h fcntl.h file.h filedesc.h filio.h gmon.h \
+	dir.h dirent.h disk.h dkstat.h dtrace.h dtrace_glue.h dtrace_impl.h \
+	errno.h ev.h event.h fasttrap.h fasttrap_isa.h fcntl.h file.h filedesc.h filio.h gmon.h \
 	ioccom.h ioctl.h \
 	ioctl_compat.h ipc.h kernel.h kernel_types.h kern_event.h loadable_fs.h lock.h lockf.h \
-	kauth.h kdebug.h md5.h kern_control.h malloc.h \
+	kauth.h kdebug.h kern_control.h kern_memorystatus.h lctx.h malloc.h \
 	mbuf.h mman.h mount.h msg.h msgbuf.h mtio.h netport.h param.h paths.h pipe.h poll.h \
-	proc.h  ptrace.h queue.h quota.h random.h reboot.h resource.h resourcevar.h \
+	proc.h  proc_info.h ptrace.h queue.h quota.h random.h reboot.h resource.h resourcevar.h \
+	sbuf.h posix_sem.h posix_shm.h sdt.h \
 	select.h sem.h semaphore.h shm.h signal.h signalvar.h socket.h socketvar.h sockio.h stat.h \
 	syscall.h sysctl.h syslimits.h syslog.h sys_domain.h termios.h  time.h \
 	timeb.h times.h tprintf.h trace.h tty.h  ttychars.h ttycom.h \
 	ttydefaults.h ttydev.h  types.h ubc.h ucontext.h ucred.h uio.h un.h unistd.h unpcb.h \
-	user.h utfconv.h utsname.h vadvise.h vcmd.h  version.h \
+	user.h utfconv.h utsname.h vadvise.h vcmd.h \
 	vm.h vmmeter.h vmparam.h vnioctl.h vnode.h vnode_if.h vstat.h wait.h xattr.h \
-	_types.h _endian.h domain.h protosw.h 
+	_select.h _structs.h _types.h _endian.h domain.h protosw.h \
+	spawn.h
 
-# Only in the framework PrivateHeader area
+# Installs header file for Apple internal use in user level - 
+#	  $(DSTROOT)/System/Library/Frameworks/System.framework/PrivateHeaders
 PRIVATE_DATAFILES = \
+	codesign.h \
 	disklabel.h \
+	fslog.h \
 	ipcs.h \
-	sem_internal.h \
 	shm_internal.h \
+	spawn_internal.h \
 	ux_exception.h \
-	ktrace.h \
 	proc_info.h \
 	vnioctl.h
 
-# KERNELFILES will appear only in the kernel framework
+# Installs header file for kernel extensions - 
+#	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/Headers
+#	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
 KERNELFILES = \
 	appleapiopts.h 	 attr.h \
 	buf.h cdefs.h conf.h \
-	dir.h dirent.h disk.h dkstat.h  \
+	dir.h dirent.h disk.h disklabel.h dkstat.h  \
 	errno.h ev.h event.h fcntl.h file.h filedesc.h filio.h \
-	ioccom.h ioctl.h \
-	ioctl_compat.h  kernel.h kernel_types.h kern_event.h  lock.h lockf.h \
-	kauth.h kdebug.h  md5.h kern_control.h malloc.h namei.h \
+	ioccom.h ioctl.h ipc.h \
+	ioctl_compat.h  kernel.h kernel_types.h kern_event.h lctx.h lock.h lockf.h \
+	kauth.h kdebug.h  md5.h kern_control.h kern_memorystatus.h imgact.h malloc.h namei.h \
 	mman.h mbuf.h mount.h  mtio.h netport.h param.h paths.h \
 	proc.h  queue.h quota.h random.h resource.h resourcevar.h \
+	sbuf.h posix_sem.h posix_shm.h sem.h shm.h \
 	select.h signal.h socket.h socketvar.h sockio.h stat.h \
 	sysctl.h syslimits.h syslog.h systm.h sys_domain.h  time.h \
 	types.h ubc.h ucontext.h ucred.h uio.h un.h unistd.h unpcb.h \
-	utfconv.h  version.h \
-	vm.h vmparam.h vnode.h vnode_if.h xattr.h \
-	_types.h _endian.h protosw.h domain.h \
+	utfconv.h vm.h vmparam.h vnode.h vnode_if.h xattr.h \
+	_select.h _structs.h _types.h _endian.h protosw.h domain.h \
 	kpi_mbuf.h kpi_socket.h kpi_socketfilter.h \
-	ttycom.h termios.h
+	ttycom.h termios.h msg.h \
+	pipe.h sysproto.h semaphore.h wait.h \
+	spawn.h
+# The last line was added to export needed headers for the MAC calls
+# whose source is outside of the xnu/bsd tree.
 
 
-# Only in the private kernel framework
+# Installs header file for Apple internal use for kernel extensions - 
+#	  $(DSTROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders
 PRIVATE_KERNELFILES = \
 	disktab.h \
+	fbt.h \
 	file_internal.h \
+	fslog.h \
 	mach_swapon.h \
 	msgbuf.h \
 	eventvar.h \
 	mount_internal.h \
 	proc_internal.h \
-	ptrace_internal.h \
 	vnode_internal.h \
+	sem_internal.h \
+	shm_internal.h \
 	signalvar.h \
 	tty.h  ttychars.h \
 	ttydefaults.h ttydev.h \
 	user.h \
 	ubc_internal.h \
 	uio_internal.h \
-	vfs_context.h 
+	vfs_context.h \
+	vmmeter.h \
+	spawn_internal.h
 	
 
+# /System/Library/Frameworks/System.framework/Headers and /usr/include
 INSTALL_MI_LIST	= ${DATAFILES}
 
 INSTALL_MI_DIR = sys
 
-EXPORT_MI_LIST	= ${KERNELFILES} ${PRIVATE_KERNELFILES} syscall.h ktrace.h linker_set.h bsdtask_info.h
+EXPORT_MI_LIST	= ${KERNELFILES} ${PRIVATE_KERNELFILES} syscall.h linker_set.h bsdtask_info.h
 
 EXPORT_MI_DIR = sys
 
@@ -102,11 +122,9 @@ EXPORT_MI_DIR = sys
 INSTALL_MI_LCL_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
 
 # /System/Library/Frameworks/Kernel.framework/PrivateHeaders
-
 INSTALL_KF_MI_LCL_LIST =  ${KERNELFILES} ${PRIVATE_KERNELFILES}
 
 # /System/Library/Frameworks/Kernel.framework/Headers
-
 INSTALL_KF_MI_LIST = ${KERNELFILES}
 
 include $(MakeInc_rule)
diff --git a/bsd/sys/_endian.h b/bsd/sys/_endian.h
index 6e34f2e70..e1ec25078 100644
--- a/bsd/sys/_endian.h
+++ b/bsd/sys/_endian.h
@@ -1,46 +1,58 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004, 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All rights reserved.
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1987, 1991, 1993
@@ -83,29 +95,45 @@
 /*
  * Macros for network/external number representation conversion.
  */
-#if !defined(__ASSEMBLER__) 
 
-#include <stdint.h>
-#include <libkern/OSByteOrder.h>
- 
+#if defined(lint)
+
 __BEGIN_DECLS
-uint16_t	ntohs(uint16_t);
-uint16_t	htons(uint16_t);
-uint32_t	ntohl(uint32_t);
-uint32_t	htonl(uint32_t);
+__uint16_t	ntohs(__uint16_t);
+__uint16_t	htons(__uint16_t);
+__uint32_t	ntohl(__uint32_t);
+__uint32_t	htonl(__uint32_t);
 __END_DECLS
-#endif /* !defined(__ASSEMBLER__) */
 
-#define ntohs(x)	OSSwapBigToHostInt16(x)
-#define htons(x)	OSSwapHostToBigInt16(x)
+#elif __DARWIN_BYTE_ORDER == __DARWIN_BIG_ENDIAN
+
+#define ntohl(x)        (x)
+#define ntohs(x)        (x)
+#define htonl(x)        (x)
+#define htons(x)        (x)
+
+#if     defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
+#define NTOHL(x)        (x)
+#define NTOHS(x)        (x)
+#define HTONL(x)        (x)
+#define HTONS(x)        (x)
+#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
+
+#else	/* __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN */
+
+#include <libkern/_OSByteOrder.h>
+
+#define ntohs(x)	__DARWIN_OSSwapInt16(x)
+#define htons(x)	__DARWIN_OSSwapInt16(x)
 
-#define ntohl(x)	OSSwapBigToHostInt32(x)
-#define htonl(x)	OSSwapHostToBigInt32(x)
+#define ntohl(x)	__DARWIN_OSSwapInt32(x)
+#define htonl(x)	__DARWIN_OSSwapInt32(x)
 
-#if	defined(KERNEL) || !defined(_POSIX_C_SOURCE)
-#define	NTOHL(x)	(x) = ntohl((u_long)x)
-#define	NTOHS(x)	(x) = ntohs((u_short)x)
-#define	HTONL(x)	(x) = htonl((u_long)x)
-#define	HTONS(x)	(x) = htons((u_short)x)
-#endif /* defined(KERNEL) || !defined(_POSIX_C_SOURCE) */
+#if	defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
+#define	NTOHL(x)	(x) = ntohl((__uint32_t)x)
+#define	NTOHS(x)	(x) = ntohs((__uint16_t)x)
+#define	HTONL(x)	(x) = htonl((__uint32_t)x)
+#define	HTONS(x)	(x) = htons((__uint16_t)x)
+#endif /* defined(KERNEL) || (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
+#endif /* __DARWIN_BYTE_ORDER */
 #endif /* !_SYS__ENDIAN_H_ */
diff --git a/bsd/sys/_select.h b/bsd/sys/_select.h
new file mode 100644
index 000000000..6f709210c
--- /dev/null
+++ b/bsd/sys/_select.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2005, 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * This is called from sys/select.h and sys/time.h for the common prototype
+ * of select().  Setting _DARWIN_C_SOURCE or _DARWIN_UNLIMITED_SELECT uses
+ * the version of select() that does not place a limit on the first argument
+ * (nfds).  In the UNIX conformance case, values of nfds greater than
+ * FD_SETSIZE will return an error of EINVAL.
+ */
+#ifndef _SYS__SELECT_H_
+#define	_SYS__SELECT_H_
+
+int	 select(int, fd_set * __restrict, fd_set * __restrict,
+		fd_set * __restrict, struct timeval * __restrict)
+#if defined(_DARWIN_C_SOURCE) || defined(_DARWIN_UNLIMITED_SELECT)
+		__DARWIN_EXTSN_C(select)
+#else /* !_DARWIN_C_SOURCE && !_DARWIN_UNLIMITED_SELECT */
+#  if defined(__LP64__) && !__DARWIN_NON_CANCELABLE
+		__DARWIN_1050(select)
+#  else /* !__LP64__ || __DARWIN_NON_CANCELABLE */
+		__DARWIN_ALIAS_C(select)
+#  endif /* __LP64__ && !__DARWIN_NON_CANCELABLE */
+#endif /* _DARWIN_C_SOURCE || _DARWIN_UNLIMITED_SELECT */
+		;
+
+#endif /* !_SYS__SELECT_H_ */
diff --git a/bsd/sys/_structs.h b/bsd/sys/_structs.h
new file mode 100644
index 000000000..e0792b73f
--- /dev/null
+++ b/bsd/sys/_structs.h
@@ -0,0 +1,254 @@
+/*
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <sys/cdefs.h>
+#include <sys/_types.h>
+
+#ifdef __need_ucontext_t
+#ifndef __need_struct_ucontext
+#define __need_struct_ucontext
+#endif /* __need_struct_ucontext */
+#endif /* __need_ucontext_t */
+
+#ifdef __need_ucontext64_t
+#ifndef __need_struct_ucontext64
+#define __need_struct_ucontext64
+#endif /* __need_struct_ucontext64 */
+#endif /* __need_ucontext64_t */
+
+#ifdef __need_struct_ucontext
+#ifndef __need_struct_mcontext
+#define __need_struct_mcontext
+#endif /* __need_struct_mcontext */
+#endif /* __need_struct_ucontext */
+
+#ifdef __need_struct_ucontext64
+#ifndef __need_struct_mcontext64
+#define __need_struct_mcontext64
+#endif /* __need_struct_mcontext64 */
+#endif /* __need_struct_ucontext64 */
+
+#if defined(__need_struct_mcontext) || defined(__need_struct_mcontext64)
+#include <machine/_structs.h>
+#endif /* __need_struct_mcontext || __need_struct_mcontext64 */
+
+#if defined(__need_stack_t) || defined(__need_struct_ucontext) || defined(__need_struct_ucontext64)
+#ifndef __need_struct_sigaltstack
+#define __need_struct_sigaltstack
+#endif /* __need_struct_sigaltstack */
+#endif /* __need_stack_t || __need_struct_ucontext || __need_struct_ucontext64 */
+
+#ifdef __need_struct_sigaltstack
+#undef __need_struct_sigaltstack
+/* Structure used in sigaltstack call. */
+#ifndef _STRUCT_SIGALTSTACK
+#if __DARWIN_UNIX03
+#define _STRUCT_SIGALTSTACK	struct __darwin_sigaltstack
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_SIGALTSTACK	struct sigaltstack
+#endif /* __DARWIN_UNIX03 */
+_STRUCT_SIGALTSTACK
+{
+	void		*ss_sp;		/* signal stack base */
+	__darwin_size_t ss_size;	/* signal stack length */
+	int		ss_flags;	/* SA_DISABLE and/or SA_ONSTACK */
+};
+#endif /* _STRUCT_SIGALTSTACK */
+#endif /* __need_struct_sigaltstack */
+
+#ifdef __need_struct_timespec
+#undef __need_struct_timespec
+#ifndef _STRUCT_TIMESPEC
+#define _STRUCT_TIMESPEC	struct timespec
+_STRUCT_TIMESPEC
+{
+	__darwin_time_t	tv_sec;
+	long		tv_nsec;
+};
+#endif /* _STRUCT_TIMESPEC */
+#endif /* __need_struct_timespec */
+
+#ifdef __need_struct_timeval
+#undef __need_struct_timeval
+#ifndef _STRUCT_TIMEVAL
+#define _STRUCT_TIMEVAL		struct timeval
+_STRUCT_TIMEVAL
+{
+	__darwin_time_t		tv_sec;		/* seconds */
+	__darwin_suseconds_t	tv_usec;	/* and microseconds */
+};
+#endif /* _STRUCT_TIMEVAL */
+#endif /* __need_struct_timeval */
+
+#ifdef __need_struct_timeval32
+#undef __need_struct_timeval32
+#ifndef _STRUCT_TIMEVAL32
+#define _STRUCT_TIMEVAL32	struct timeval32
+_STRUCT_TIMEVAL32
+{
+	__int32_t		tv_sec;		/* seconds */
+	__int32_t		tv_usec;	/* and microseconds */
+};
+#endif /* _STRUCT_TIMEVAL32 */
+#endif /* __need_struct_timeval32 */
+
+#ifdef __need_struct_ucontext
+#undef __need_struct_ucontext
+#ifndef _STRUCT_UCONTEXT
+#if __DARWIN_UNIX03
+#define _STRUCT_UCONTEXT	struct __darwin_ucontext
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_UCONTEXT	struct ucontext
+#endif /* __DARWIN_UNIX03 */
+_STRUCT_UCONTEXT
+{
+	int			uc_onstack;
+	__darwin_sigset_t	uc_sigmask;	/* signal mask used by this context */
+	_STRUCT_SIGALTSTACK 	uc_stack;	/* stack used by this context */
+	_STRUCT_UCONTEXT	*uc_link;	/* pointer to resuming context */
+	__darwin_size_t		uc_mcsize;	/* size of the machine context passed in */
+	_STRUCT_MCONTEXT	*uc_mcontext;	/* pointer to machine specific context */
+#ifdef _XOPEN_SOURCE
+	_STRUCT_MCONTEXT	__mcontext_data;
+#endif /* _XOPEN_SOURCE */
+};
+#endif /* _STRUCT_UCONTEXT */
+#endif /* __need_struct_ucontext */
+
+#ifdef __need_struct_ucontext64
+#undef __need_struct_ucontext64
+#ifndef _STRUCT_UCONTEXT64
+#if __DARWIN_UNIX03
+#define _STRUCT_UCONTEXT64	struct __darwin_ucontext64
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_UCONTEXT64	struct ucontext64
+#endif /* __DARWIN_UNIX03 */
+_STRUCT_UCONTEXT64
+{
+	int			uc_onstack;
+	__darwin_sigset_t	uc_sigmask;	/* signal mask used by this context */
+	_STRUCT_SIGALTSTACK 	uc_stack;	/* stack used by this context */
+	_STRUCT_UCONTEXT64	*uc_link;	/* pointer to resuming context */
+	__darwin_size_t		uc_mcsize;	/* size of the machine context passed in */
+	_STRUCT_MCONTEXT64	*uc_mcontext64;	/* pointer to machine specific context */
+};
+#endif /* _STRUCT_UCONTEXT64 */
+#endif /* __need_struct_ucontext64 */
+
+#ifdef KERNEL
+/* LP64 version of struct timespec.  time_t is a long and must grow when 
+ * we're dealing with a 64-bit process.
+ * WARNING - keep in sync with struct timespec
+ */
+#ifdef __need_struct_user_timespec
+#undef __need_struct_user_timespec
+#ifndef _STRUCT_USER_TIMESPEC
+#define _STRUCT_USER_TIMESPEC	struct user_timespec
+_STRUCT_USER_TIMESPEC
+{
+	user_time_t	tv_sec;		/* seconds */
+	__int64_t	tv_nsec __attribute((aligned(8)));	/* and nanoseconds */
+};
+#endif /* _STRUCT_USER_TIMESPEC */
+#endif /* __need_struct_user_timespec */
+#endif	/* KERNEL */
+
+#ifdef __need_fd_set
+#undef __need_fd_set
+#ifndef _FD_SET
+#define _FD_SET
+/*
+ * Select uses bit masks of file descriptors in longs.  These macros
+ * manipulate such bit fields (the filesystem macros use chars).  The
+ * extra protection here is to permit application redefinition above
+ * the default size.
+ */
+#ifdef FD_SETSIZE
+#define	__DARWIN_FD_SETSIZE	FD_SETSIZE
+#else /* !FD_SETSIZE */
+#define	__DARWIN_FD_SETSIZE	1024
+#endif /* FD_SETSIZE */
+#define	__DARWIN_NBBY		8				/* bits in a byte */
+#define __DARWIN_NFDBITS	(sizeof(__int32_t) * __DARWIN_NBBY) /* bits per mask */
+#define	__DARWIN_howmany(x, y) (((x) + ((y) - 1)) / (y))	/* # y's == x bits? */
+
+__BEGIN_DECLS
+typedef	struct fd_set {
+	__int32_t	fds_bits[__DARWIN_howmany(__DARWIN_FD_SETSIZE, __DARWIN_NFDBITS)];
+} fd_set;
+__END_DECLS
+
+/* This inline avoids argument side-effect issues with FD_ISSET() */
+static __inline int
+__darwin_fd_isset(int _n, struct fd_set *_p)
+{
+	return (_p->fds_bits[_n/__DARWIN_NFDBITS] & (1<<(_n % __DARWIN_NFDBITS)));
+}
+
+#define	__DARWIN_FD_SET(n, p)	do { int __fd = (n); ((p)->fds_bits[__fd/__DARWIN_NFDBITS] |= (1<<(__fd % __DARWIN_NFDBITS))); } while(0)
+#define	__DARWIN_FD_CLR(n, p)	do { int __fd = (n); ((p)->fds_bits[__fd/__DARWIN_NFDBITS] &= ~(1<<(__fd % __DARWIN_NFDBITS))); } while(0)
+#define	__DARWIN_FD_ISSET(n, p)	__darwin_fd_isset((n), (p))
+
+#if __GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3
+/*
+ * Use the built-in bzero function instead of the library version so that
+ * we do not pollute the namespace or introduce prototype warnings.
+ */
+#define	__DARWIN_FD_ZERO(p)	__builtin_bzero(p, sizeof(*(p)))
+#else
+#define	__DARWIN_FD_ZERO(p)	bzero(p, sizeof(*(p)))
+#endif
+
+#define	__DARWIN_FD_COPY(f, t)	bcopy(f, t, sizeof(*(f)))
+#endif /* _FD_SET */
+#endif /* __need_fd_set */
+
+#ifdef __need_stack_t
+#undef __need_stack_t
+#ifndef _STACK_T
+#define _STACK_T
+typedef _STRUCT_SIGALTSTACK	stack_t; /* [???] signal stack */
+#endif /* _STACK_T */
+#endif /* __need_stack_t */
+
+#ifdef __need_ucontext_t
+#undef __need_ucontext_t
+/* user context */
+#ifndef _UCONTEXT_T
+#define _UCONTEXT_T
+typedef _STRUCT_UCONTEXT	ucontext_t;	/* [???] user context */
+#endif /* _UCONTEXT_T */
+#endif /* __need_ucontext_t */
+
+#ifdef __need_ucontext64_t
+#undef __need_ucontext64_t
+#ifndef _UCONTEXT64_T
+#define _UCONTEXT64_T
+typedef _STRUCT_UCONTEXT64	ucontext64_t;	/* [???] user context */
+#endif /* _UCONTEXT64_T */
+#endif /* __need_ucontext64_t */
diff --git a/bsd/sys/_types.h b/bsd/sys/_types.h
index addbb39b1..64dd43578 100644
--- a/bsd/sys/_types.h
+++ b/bsd/sys/_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS__TYPES_H_
@@ -26,14 +32,6 @@
 #include <sys/cdefs.h>
 #include <machine/_types.h>
 
-/* Forward references */
-#ifndef _POSIX_C_SOURCE
-struct mcontext;
-struct mcontext64;
-#else /* _POSIX_C_SOURCE */
-struct __darwin_mcontext;
-#endif /* _POSIX_C_SOURCE */
-
 /* pthread opaque structures */
 #if defined(__LP64__)
 #define __PTHREAD_SIZE__           1168
@@ -100,15 +98,14 @@ typedef unsigned int	__darwin_fsblkcnt_t;	/* Used by statvfs and fstatvfs */
 typedef unsigned int	__darwin_fsfilcnt_t;	/* Used by statvfs and fstatvfs */
 typedef __uint32_t	__darwin_gid_t;		/* [???] process and group IDs */
 typedef __uint32_t	__darwin_id_t;		/* [XSI] pid_t, uid_t, or gid_t*/
+typedef __uint64_t	__darwin_ino64_t;	/* [???] Used for 64 bit inodes */
+#if __DARWIN_64_BIT_INO_T
+typedef __darwin_ino64_t __darwin_ino_t;	/* [???] Used for inodes */
+#else /* !__DARWIN_64_BIT_INO_T */
 typedef __uint32_t	__darwin_ino_t;		/* [???] Used for inodes */
+#endif /* __DARWIN_64_BIT_INO_T */
 typedef __darwin_natural_t __darwin_mach_port_name_t; /* Used by mach */
 typedef __darwin_mach_port_name_t __darwin_mach_port_t; /* Used by mach */
-#ifndef _POSIX_C_SOURCE
-typedef struct mcontext *__darwin_mcontext_t;	/* [???] machine context */
-typedef struct mcontext64 *__darwin_mcontext64_t; /* [???] machine context */
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_mcontext *__darwin_mcontext_t; /* [???] machine context */
-#endif /* _POSIX_C_SOURCE */
 typedef __uint16_t	__darwin_mode_t;	/* [???] Some file attributes */
 typedef __int64_t	__darwin_off_t;		/* [???] Used for file sizes */
 typedef __int32_t	__darwin_pid_t;		/* [???] process and group IDs */
@@ -137,59 +134,6 @@ typedef __uint32_t	__darwin_uid_t;		/* [???] user IDs */
 typedef __uint32_t	__darwin_useconds_t;	/* [???] microseconds */
 typedef	unsigned char	__darwin_uuid_t[16];
 
-/* Structure used in sigaltstack call. */
-#ifndef _POSIX_C_SOURCE
-struct	sigaltstack
-#else /* _POSIX_C_SOURCE */
-struct	__darwin_sigaltstack
-#endif /* _POSIX_C_SOURCE */
-{
-	void	*ss_sp;			/* signal stack base */
-	__darwin_size_t ss_size;	/* signal stack length */
-	int	ss_flags;		/* SA_DISABLE and/or SA_ONSTACK */
-};
-#ifndef _POSIX_C_SOURCE
-typedef struct sigaltstack __darwin_stack_t;	/* [???] signal stack */
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_sigaltstack __darwin_stack_t; /* [???] signal stack */
-#endif /* _POSIX_C_SOURCE */
-
-/* user context */
-#ifndef _POSIX_C_SOURCE
-struct ucontext
-#else /* _POSIX_C_SOURCE */
-struct __darwin_ucontext
-#endif /* _POSIX_C_SOURCE */
-{
-	int		uc_onstack;
-	__darwin_sigset_t	uc_sigmask;	/* signal mask used by this context */
-	__darwin_stack_t 	uc_stack;	/* stack used by this context */
-#ifndef _POSIX_C_SOURCE
-	struct ucontext	*uc_link;		/* pointer to resuming context */
-#else /* _POSIX_C_SOURCE */
-	struct __darwin_ucontext *uc_link;	/* pointer to resuming context */
-#endif /* _POSIX_C_SOURCE */
-	__darwin_size_t	uc_mcsize;		/* size of the machine context passed in */
-	__darwin_mcontext_t	uc_mcontext;	/* pointer to machine specific context */
-};
-#ifndef _POSIX_C_SOURCE
-typedef struct ucontext __darwin_ucontext_t;	/* [???] user context */
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ucontext __darwin_ucontext_t; /* [???] user context */
-#endif /* _POSIX_C_SOURCE */
-
-#ifndef _POSIX_C_SOURCE
-struct ucontext64 {
-	int		uc_onstack;
-	__darwin_sigset_t	uc_sigmask;	/* signal mask used by this context */
-	__darwin_stack_t 	uc_stack;	/* stack used by this context */
-	struct ucontext64 *uc_link;		/* pointer to resuming context */
-	__darwin_size_t	uc_mcsize;		/* size of the machine context passed in */
-	__darwin_mcontext64_t uc_mcontext64;	/* pointer to machine specific context */
-};
-typedef struct ucontext64 __darwin_ucontext64_t; /* [???] user context */
-#endif /* _POSIX_C_SOURCE */
-
 #ifdef KERNEL
 #ifndef offsetof
 #define offsetof(type, member)	((size_t)(&((type *)0)->member))
diff --git a/bsd/sys/acct.h b/bsd/sys/acct.h
index 1cd61259b..050ccc3f9 100644
--- a/bsd/sys/acct.h
+++ b/bsd/sys/acct.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -76,7 +82,7 @@ struct acct {
 	comp_t	  ac_utime;	/* user time */
 	comp_t	  ac_stime;	/* system time */
 	comp_t	  ac_etime;	/* elapsed time */
-	time_t	  ac_btime;	/* starting time */
+	u_int32_t ac_btime;	/* starting time */
 	uid_t	  ac_uid;	/* user id */
 	gid_t	  ac_gid;	/* group id */
 	u_int16_t ac_mem;	/* average memory usage */
diff --git a/bsd/sys/aio.h b/bsd/sys/aio.h
index a8a149866..938583373 100644
--- a/bsd/sys/aio.h
+++ b/bsd/sys/aio.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  *	File:	sys/aio.h
@@ -32,29 +38,49 @@
 #define	_SYS_AIO_H_
 
 #include <sys/signal.h>
+#include <sys/_types.h>
 #include <sys/cdefs.h>
 
+/*
+ * [XSI] Inclusion of the <aio.h> header may make visible symbols defined
+ * in the headers <fcntl.h>, <signal.h>, <sys/types.h>, and <time.h>.
+ * 
+ * In our case, this is limited to struct timespec, off_t and ssize_t.
+ */
+#define __need_struct_timespec
+#include <sys/_structs.h>
+
+#ifndef _OFF_T
+typedef __darwin_off_t	off_t;
+#define _OFF_T
+#endif
+
+#ifndef	_SSIZE_T
+#define	_SSIZE_T
+typedef	__darwin_ssize_t	ssize_t;
+#endif
+
+
 struct aiocb {
-	int					aio_fildes;		/* File descriptor */
-	off_t				aio_offset;		/* File offset */
-	volatile void		*aio_buf;		/* Location of buffer */
-	size_t				aio_nbytes;		/* Length of transfer */
-	int					aio_reqprio;	/* Request priority offset */
-	struct sigevent		aio_sigevent;	/* Signal number and value */
-	int					aio_lio_opcode;	/* Operation to be performed */
+	int		aio_fildes;		/* File descriptor */
+	off_t		aio_offset;		/* File offset */
+	volatile void	*aio_buf;		/* Location of buffer */
+	size_t		aio_nbytes;		/* Length of transfer */
+	int		aio_reqprio;		/* Request priority offset */
+	struct sigevent	aio_sigevent;		/* Signal number and value */
+	int		aio_lio_opcode;		/* Operation to be performed */
 };
 
-// LP64todo - should this move?
 #ifdef KERNEL
 
 struct user_aiocb {
-	int					aio_fildes;		/* File descriptor */
-	off_t				aio_offset;		/* File offset */
-	user_addr_t			aio_buf __attribute((aligned(8)));		/* Location of buffer */
-	user_size_t			aio_nbytes;		/* Length of transfer */
-	int					aio_reqprio;	/* Request priority offset */
+	int		aio_fildes;		/* File descriptor */
+	off_t		aio_offset;		/* File offset */
+	user_addr_t	aio_buf __attribute((aligned(8)));		/* Location of buffer */
+	user_size_t	aio_nbytes;		/* Length of transfer */
+	int		aio_reqprio;	/* Request priority offset */
 	struct user_sigevent aio_sigevent __attribute((aligned(8)));	/* Signal number and value */
-	int					aio_lio_opcode;	/* Operation to be performed */
+	int		aio_lio_opcode;		/* Operation to be performed */
 };
 
 #endif // KERNEL
@@ -215,7 +241,7 @@ ssize_t	aio_return( struct aiocb * aiocbp );
  */
 int		aio_suspend( const struct aiocb *const aiocblist[], 
 					 int nent,
-	 				 const struct timespec * timeoutp );
+	 				 const struct timespec * timeoutp ) __DARWIN_ALIAS_C(aio_suspend);
 	 				 
 /*
  * Write aiocbp->aio_nbytes to the file associated with aiocbp->aio_fildes from 
diff --git a/bsd/sys/aio_kern.h b/bsd/sys/aio_kern.h
index 09401f0b3..33ae502c9 100644
--- a/bsd/sys/aio_kern.h
+++ b/bsd/sys/aio_kern.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  *	File:	sys/aio_kern.h
@@ -83,7 +89,7 @@ __private_extern__ void
 _aio_create_worker_threads(int num);
 
 __private_extern__ void		
-aio_init(void);
+aio_init(void) __attribute__((section("__TEXT, initcode")));
 
 task_t						
 get_aiotask(void);
diff --git a/bsd/sys/appleapiopts.h b/bsd/sys/appleapiopts.h
index 2ecde9dee..20557019b 100644
--- a/bsd/sys/appleapiopts.h
+++ b/bsd/sys/appleapiopts.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __SYS_APPLEAPIOPTS_H__
diff --git a/bsd/sys/attr.h b/bsd/sys/attr.h
index e2b97fb67..ca87e7052 100644
--- a/bsd/sys/attr.h
+++ b/bsd/sys/attr.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -177,6 +183,23 @@ typedef struct vol_capabilities_attr {
  * size upto 2TB.  This bit does not necessarily mean that the file 
  * system does not support file size more than 2TB.   This bit does 
  * not mean that the currently available space on the volume is 2TB. 
+ *
+ * VOL_CAP_FMT_OPENDENYMODES: When set, the volume supports open deny
+ * modes (e.g. "open for read write, deny write"; effectively, mandatory
+ * file locking based on open modes).
+ *
+ * VOL_CAP_FMT_HIDDEN_FILES: When set, the volume supports the UF_HIDDEN
+ * file flag, and the UF_HIDDEN flag is mapped to that volume's native
+ * "hidden" or "invisible" bit (which may be the invisible bit from the
+ * Finder Info extended attribute).
+ *
+ * VOL_CAP_FMT_PATH_FROM_ID:  When set, the volume supports the ability
+ * to derive a pathname to the root of the file system given only the
+ * id of an object.  This also implies that object ids on this file
+ * system are persistent and not recycled.  This is a very specialized
+ * capability and it is assumed that most file systems will not support
+ * it.  Its use is for legacy non-posix APIs like ResolveFileIDRef.
+ * 
  */
 #define VOL_CAP_FMT_PERSISTENTOBJECTIDS		0x00000001
 #define VOL_CAP_FMT_SYMBOLICLINKS 		0x00000002
@@ -190,6 +213,9 @@ typedef struct vol_capabilities_attr {
 #define VOL_CAP_FMT_CASE_PRESERVING		0x00000200
 #define VOL_CAP_FMT_FAST_STATFS 		0x00000400
 #define VOL_CAP_FMT_2TB_FILESIZE		0x00000800
+#define VOL_CAP_FMT_OPENDENYMODES		0x00001000
+#define VOL_CAP_FMT_HIDDEN_FILES		0x00002000
+#define VOL_CAP_FMT_PATH_FROM_ID		0x00004000
 
 
 /*
@@ -233,6 +259,21 @@ typedef struct vol_capabilities_attr {
  * style locks via vnop_advlock.  This includes the O_EXLOCK and O_SHLOCK
  * flags of the open(2) call.
  *
+ * VOL_CAP_INT_EXTENDED_SECURITY: When set, the volume implements
+ * extended security (ACLs).
+ *
+ * VOL_CAP_INT_USERACCESS:  When set, the volume supports the
+ * ATTR_CMN_USERACCESS attribute (used to get the user's access
+ * mode to the file).
+ *
+ * VOL_CAP_INT_MANLOCK: When set, the volume supports AFP-style
+ * mandatory byte range locks via an ioctl().
+ *
+ * VOL_CAP_INT_EXTENDED_ATTR: When set, the volume implements
+ * native extended attribues.
+ *
+ * VOL_CAP_INT_NAMEDSTREAMS: When set, the volume supports
+ * native named streams.
  */
 #define VOL_CAP_INT_SEARCHFS			0x00000001
 #define VOL_CAP_INT_ATTRLIST			0x00000002
@@ -246,6 +287,9 @@ typedef struct vol_capabilities_attr {
 #define VOL_CAP_INT_FLOCK			0x00000200
 #define VOL_CAP_INT_EXTENDED_SECURITY		0x00000400
 #define VOL_CAP_INT_USERACCESS			0x00000800
+#define VOL_CAP_INT_MANLOCK			0x00001000
+#define VOL_CAP_INT_NAMEDSTREAMS		0x00002000
+#define VOL_CAP_INT_EXTENDED_ATTR		0x00004000
 
 typedef struct vol_attributes_attr {
 	attribute_set_t validattr;
@@ -275,8 +319,10 @@ typedef struct vol_attributes_attr {
 #define ATTR_CMN_EXTENDED_SECURITY		0x00400000
 #define ATTR_CMN_UUID				0x00800000
 #define ATTR_CMN_GRPUUID			0x01000000
+#define ATTR_CMN_FILEID				0x02000000
+#define ATTR_CMN_PARENTID			0x04000000
 
-#define ATTR_CMN_VALIDMASK			0x003FFFFF
+#define ATTR_CMN_VALIDMASK			0x07FFFFFF
 #define ATTR_CMN_SETMASK			0x01C7FF00
 #define ATTR_CMN_VOLSETMASK			0x00006700
 
@@ -376,7 +422,6 @@ struct fssearchblock {
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with fssearchblock
  */
-// LP64todo - should this move?
 
 struct user_fssearchblock {
 	user_addr_t         returnattrs;
diff --git a/bsd/sys/bsdtask_info.h b/bsd/sys/bsdtask_info.h
index a18c56fc8..0da659b82 100644
--- a/bsd/sys/bsdtask_info.h
+++ b/bsd/sys/bsdtask_info.h
@@ -2,23 +2,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_BSDTASK_INFO_H
@@ -57,6 +63,7 @@ struct proc_threadinfo_internal {
 	int32_t			pth_curpri;		/* cur priority*/
 	int32_t			pth_priority;		/*  priority*/
 	int32_t			pth_maxpriority;		/* max priority*/
+	char *			pth_name[64];		/* thread name, if any */
 };
 
 
@@ -80,9 +87,9 @@ struct proc_regioninfo_internal {
 	uint32_t		pri_private_pages_resident;
 	uint32_t		pri_shared_pages_resident;
 	uint32_t		pri_obj_id;
+	uint32_t		pri_depth;
 	uint64_t		pri_address;
 	uint64_t		pri_size;
-	uint32_t		pri_depth;
 };
 
 #ifdef  MACH_KERNEL_PRIVATE
@@ -99,10 +106,10 @@ extern uint32_t vnode_vid(void *vp);
 
 extern int fill_procregioninfo(task_t t, uint64_t arg, struct proc_regioninfo_internal *pinfo, uint32_t *vp, uint32_t *vid);
 void fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo);
-int fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo);
+int fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo, void *, int *);
 int fill_taskthreadlist(task_t task, void * buffer, int thcount);
 int get_numthreads(task_t);
-
+void bsd_threadcdir(void * uth, void *vptr, int *vidp);
 
 #endif /*_SYS_BSDTASK_INFO_H */
 
diff --git a/bsd/sys/buf.h b/bsd/sys/buf.h
index 91aa77cf7..5cb9c7f9c 100644
--- a/bsd/sys/buf.h
+++ b/bsd/sys/buf.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -78,6 +84,9 @@
 #define B_CLUSTER	0x00000040	/* UPL based I/O generated by cluster layer */
 #define	B_PAGEIO	0x00000080	/* Page in/out */
 #define	B_META		0x00000100	/* buffer contains meta-data. */
+#define	B_RAW		0x00000200	/* Set by physio for raw transfers. */
+#define	B_FUA		0x00000400	/* Write-through disk cache(if supported) */
+#define B_PASSIVE	0x00000800	/* PASSIVE I/Os are ignored by THROTTLE I/O */
 /*
  * make sure to check when adding flags that
  * that the new flags don't overlap the definitions
@@ -111,6 +120,18 @@ void	buf_markdelayed(buf_t);
  */
 void	buf_markeintr(buf_t);
 
+/*
+ * mark the buffer associated with buf_t
+ * for write through disk cache if device supports
+ */
+void	buf_markfua(buf_t);
+
+/*
+ * returns 1 if the buffer associated with buf_t
+ * is set for write through disk cache... 0 if it does not
+ */
+int	buf_fua(buf_t);
+
 /*
  * returns 1 if the buffer associated with buf_t
  * contains valid data... 0 if it does not
@@ -171,19 +192,19 @@ void	buf_seterror(buf_t, errno_t);
 
 /*
  * set specified flags on buf_t
- * B_LOCKED/B_NOCACHE/B_ASYNC/B_READ/B_WRITE/B_PAGEIO
+ * B_LOCKED/B_NOCACHE/B_ASYNC/B_READ/B_WRITE/B_PAGEIO/B_FUA
  */
 void	buf_setflags(buf_t, int32_t);
 
 /*
  * clear specified flags on buf_t
- * B_LOCKED/B_NOCACHE/B_ASYNC/B_READ/B_WRITE/B_PAGEIO
+ * B_LOCKED/B_NOCACHE/B_ASYNC/B_READ/B_WRITE/B_PAGEIO/B_FUA
  */
 void	buf_clearflags(buf_t, int32_t);
 
 /*
  * return external flags associated with buf_t
- * B_CLUSTER/B_PHYS/B_LOCKED/B_DELWRI/B_ASYNC/B_READ/B_WRITE/B_META/B_PAGEIO
+ * B_CLUSTER/B_PHYS/B_LOCKED/B_DELWRI/B_ASYNC/B_READ/B_WRITE/B_META/B_PAGEIO/B_FUA
  */
 int32_t	buf_flags(buf_t);
 
@@ -391,8 +412,11 @@ int	buf_invalidateblks(vnode_t, int, int, int);
  */
 #define BUF_SKIP_NONLOCKED	0x01
 #define BUF_SKIP_LOCKED		0x02
+#define BUF_SCAN_CLEAN		0x04	/* scan only the clean buffers */
+#define BUF_SCAN_DIRTY		0x08	/* scan only the dirty buffers */
+#define BUF_NOTIFY_BUSY		0x10	/* notify the caller about the busy pages during the scan */
 
-void	buf_flushdirtyblks(vnode_t, int, int, char *);
+void	buf_flushdirtyblks(vnode_t, int, int, const char *);
 void	buf_iterate(vnode_t, int (*)(buf_t, void *), int, void *);
 
 #define	BUF_RETURNED		0
diff --git a/bsd/sys/buf_internal.h b/bsd/sys/buf_internal.h
index ec3a84dcc..bbe22779d 100644
--- a/bsd/sys/buf_internal.h
+++ b/bsd/sys/buf_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -96,9 +102,9 @@ struct buf {
 	daddr64_t	b_lblkno;	/* Logical block number. */
 	daddr64_t	b_blkno;	/* Underlying physical block number. */
 	void	(*b_iodone)(buf_t, void *);	/* Function to call upon completion. */
-	vnode_t	b_vp;			/* Device vnode. */
-	struct	ucred *b_rcred;		/* Read credentials reference. */
-	struct	ucred *b_wcred;		/* Write credentials reference. */
+	vnode_t	b_vp;			/* File vnode for data, device vnode for metadata. */
+	kauth_cred_t b_rcred;		/* Read credentials reference. */
+	kauth_cred_t b_wcred;		/* Write credentials reference. */
 	void *	b_upl;			/* Pointer to UPL */
 	buf_t	b_real_bp;		/* used to track bp generated through cluster_bp */
 	TAILQ_ENTRY(buf)	b_act;	/* Device driver queue when active */
@@ -125,6 +131,7 @@ struct buf {
 #define b_trans_head b_freelist.tqe_prev
 #define b_trans_next b_freelist.tqe_next
 #define b_iostate    b_rcred
+#define b_cliodone   b_wcred
 
 /*
  * These flags are kept in b_lflags... 
@@ -133,17 +140,18 @@ struct buf {
 #define	BL_BUSY		0x00000001	/* I/O in progress. */
 #define	BL_WANTED	0x00000002	/* Process wants this buffer. */
 #define BL_IOBUF	0x00000004	/* buffer allocated via 'buf_alloc' */
-
+#define BL_CALLDONE	0x00000008	/* callback routine on B_CALL bp has completed */
 
 /*
  * mask used by buf_flags... these are the readable external flags
  */
-#define BUF_X_RDFLAGS (B_CLUSTER | B_PHYS | B_LOCKED | B_DELWRI | B_ASYNC |\
-                       B_READ | B_WRITE | B_META | B_PAGEIO)
+#define BUF_X_RDFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\
+		       B_META | B_CLUSTER | B_DELWRI | B_FUA | B_PASSIVE)
 /*
  * mask used by buf_clearflags/buf_setflags... these are the writable external flags
  */
-#define BUF_X_WRFLAGS (B_LOCKED | B_NOCACHE | B_ASYNC | B_READ | B_WRITE | B_PAGEIO)
+#define BUF_X_WRFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\
+		       B_NOCACHE | B_FUA | B_PASSIVE)
 
 /*
  * These flags are kept in b_flags... access is lockless
@@ -160,7 +168,7 @@ struct buf {
 #define	B_AGE		0x00200000	/* Move to age queue when I/O done. */
 #define B_FILTER	0x00400000	/* call b_iodone from biodone as an in-line filter */
 #define	B_CALL		0x00800000	/* Call b_iodone from biodone, assumes b_iodone consumes bp */
-#define	B_RAW		0x01000000	/* Set by physio for raw transfers. */
+#define B_EOT		0x01000000	/* last buffer in a transaction list created by cluster_io */
 #define	B_WASDIRTY	0x02000000	/* page was found dirty in the VM cache */
 #define	B_HDRALLOC	0x04000000	/* zone allocated buffer header */
 #define	B_ZALLOC	0x08000000	/* b_datap is zalloc()ed */
@@ -185,11 +193,11 @@ struct buf {
 #define B_NOBUFF	0x04	/* Do not allocate struct buf */
 
 
-extern int niobuf;		/* The number of IO buffer headers for cluster IO */
-extern int nbuf;		/* The number of buffer headers */
-extern struct buf *buf;		/* The buffer headers. */
+extern int niobuf_headers;		/* The number of IO buffer headers for cluster IO */
+extern int nbuf_headers;		/* The number of buffer headers */
 extern int max_nbuf_headers;		/* The max number of buffer headers */
 extern int nbuf_hashelements;		/* The number of elements in bufhash */
+extern struct buf *buf_headers;		/* The buffer headers. */
 
 
 /*
@@ -211,7 +219,7 @@ buf_t	alloc_io_buf(vnode_t, int);
 void	free_io_buf(buf_t);
 
 int	allocbuf(struct buf *, int);
-void	bufinit(void);
+void	bufinit(void) __attribute__((section("__TEXT, initcode")));
 
 void	buf_setfilter(buf_t, void (*)(buf_t, void *), void *, void **, void **);
 
@@ -223,7 +231,12 @@ void	buf_setfilter(buf_t, void (*)(buf_t, void *), void *, void **, void **);
 #define BAC_SKIP_NONLOCKED	0x04	/* Don't return LOCKED buffers */
 #define BAC_SKIP_LOCKED		0x08	/* Only return LOCKED buffers */
 
-void	cluster_init(void);
+void	buf_list_lock(void);
+void	buf_list_unlock(void);
+
+void	buf_biowait_callback(buf_t);
+
+void	cluster_init(void) __attribute__((section("__TEXT, initcode")));
 void	buf_drop(buf_t);
 errno_t	buf_acquire(buf_t, int, int, int);
 
diff --git a/bsd/sys/callout.h b/bsd/sys/callout.h
index 61c4dee0b..cb4e17d78 100644
--- a/bsd/sys/callout.h
+++ b/bsd/sys/callout.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1990, 1993
diff --git a/bsd/sys/cdefs.h b/bsd/sys/cdefs.h
index 46607547e..c4881ef33 100644
--- a/bsd/sys/cdefs.h
+++ b/bsd/sys/cdefs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright 1995 NeXT Computer, Inc. All rights reserved. */
 /*
@@ -209,42 +215,159 @@
 #endif
 
 /*
- * The __DARWIN_ALIAS macros is used to do symbol renaming, 
- * they allow old code to use the old symbol thus maintiang binary 
- * compatability while new code can use a new improved version of the 
- * same function.
+ * COMPILATION ENVIRONMENTS
+ *
+ * DEFAULT	By default newly complied code will get POSIX APIs plus
+ *		Apple API extensions in scope.
+ *
+ *		Most users will use this compilation environment to avoid
+ *		behavioural differences between 32 and 64 bit code.
+ *
+ * LEGACY	Defining _NONSTD_SOURCE will get pre-POSIX APIs plus Apple
+ *		API extensions in scope.
+ *
+ *		This is generally equivalent to the Tiger release compilation
+ *		environment, except that it cannot be applied to 64 bit code;
+ *		its use is discouraged.
+ *
+ *		We expect this environment to be deprecated in the future.
+ *
+ * STRICT	Defining _POSIX_C_SOURCE or _XOPEN_SOURCE restricts the
+ *		available APIs to exactly the set of APIs defined by the
+ *		corresponding standard, based on the value defined.
  *
- * By default newly complied code will actually get the same symbols
- * that the old code did.  Defining any of _APPLE_C_SOURCE, _XOPEN_SOURCE,
- * or _POSIX_C_SOURCE will give you the new symbols.  Defining _XOPEN_SOURCE
- * or _POSIX_C_SOURCE also restricts the avilable symbols to a subset of
- * Apple's APIs.
+ *		A correct, portable definition for _POSIX_C_SOURCE is 200112L.
+ *		A correct, portable definition for _XOPEN_SOURCE is 600L.
+ *
+ *		Apple API extensions are not visible in this environment,
+ *		which can cause Apple specific code to fail to compile,
+ *		or behave incorrectly if prototypes are not in scope or
+ *		warnings about missing prototypes are not enabled or ignored.
+ *
+ * In any compilation environment, for correct symbol resolution to occur,
+ * function prototypes must be in scope.  It is recommended that all Apple
+ * tools users add etiher the "-Wall" or "-Wimplicit-function-declaration"
+ * compiler flags to their projects to be warned when a function is being
+ * used without a prototype in scope.
+ */
+
+/*
+ * The __DARWIN_ALIAS macros are used to do symbol renaming; they allow
+ * legacy code to use the old symbol, thus maintiang binary compatability
+ * while new code can use a standards compliant version of the same function.
  *
  * __DARWIN_ALIAS is used by itself if the function signature has not
  * changed, it is used along with a #ifdef check for __DARWIN_UNIX03
  * if the signature has changed.  Because the __LP64__ enviroment
  * only supports UNIX03 sementics it causes __DARWIN_UNIX03 to be
  * defined, but causes __DARWIN_ALIAS to do no symbol mangling.
+ *
+ * As a special case, when XCode is used to target a specific version of the
+ * OS, the manifest constant __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__
+ * will be defined by the compiler, with the digits representing major version
+ * time 100 + minor version times 10 (e.g. 10.5 := 1050).  If we are targetting
+ * pre-10.5, and it is the default compilation environment, revert the
+ * compilation environment to pre-__DARWIN_UNIX03.
  */
-
 #if !defined(__DARWIN_UNIX03)
-#if defined(_APPLE_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_C_SOURCE) || defined(__LP64__)
-#if defined(_NONSTD_SOURCE)
-#error "Can't define both _NONSTD_SOURCE and any of _APPLE_C_SOURCE, _XOPEN_SOURCE, _POSIX_C_SOURCE, or __LP64__"
-#endif /* _NONSTD_SOURCE */
-#define __DARWIN_UNIX03	1
-#elif defined(_NONSTD_SOURCE)
-#define __DARWIN_UNIX03	0
-#else /* default */
-#define __DARWIN_UNIX03	0
-#endif /* _APPLE_C_SOURCE || _XOPEN_SOURCE || _POSIX_C_SOURCE || __LP64__ */
+#  if defined(_DARWIN_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_C_SOURCE) || defined(__LP64__) || (defined(__arm__) && !defined(KERNEL))
+#    if defined(_NONSTD_SOURCE)
+#      error "Can't define both _NONSTD_SOURCE and any of _DARWIN_C_SOURCE, _XOPEN_SOURCE, _POSIX_C_SOURCE, or __LP64__"
+#    endif /* _NONSTD_SOURCE */
+#    define __DARWIN_UNIX03	1
+#  elif defined(_NONSTD_SOURCE) || defined(KERNEL)
+#    define __DARWIN_UNIX03	0
+#  else /* default */
+#    if defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && ((__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__-0) < 1050)
+#      define __DARWIN_UNIX03	0
+#    else /* __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 */
+#      define __DARWIN_UNIX03	1
+#    endif /* __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1050 */
+#  endif /* _DARWIN_C_SOURCE || _XOPEN_SOURCE || _POSIX_C_SOURCE || __LP64__ */
 #endif /* !__DARWIN_UNIX03 */
 
-#if __DARWIN_UNIX03 && !defined(__LP64__)
-#define __DARWIN_ALIAS(sym) __asm("_" __STRING(sym) "$UNIX2003")
-#else
-#define __DARWIN_ALIAS(sym)
-#endif
+#if !defined(__DARWIN_64_BIT_INO_T)
+#  if defined(_DARWIN_USE_64_BIT_INODE)
+#    define __DARWIN_64_BIT_INO_T 1
+#  elif defined(_DARWIN_NO_64_BIT_INODE) || defined(KERNEL)
+#    define __DARWIN_64_BIT_INO_T 0
+#  else /* default */
+#    define __DARWIN_64_BIT_INO_T 0
+#  endif
+#endif /* !__DARWIN_64_BIT_INO_T */
+
+#if !defined(__DARWIN_NON_CANCELABLE)
+#  if defined(KERNEL)
+#    define __DARWIN_NON_CANCELABLE 0
+#  else /* default */
+#    define __DARWIN_NON_CANCELABLE 0
+#  endif
+#endif /* !__DARWIN_NON_CANCELABLE */
+
+#if !defined(__DARWIN_VERS_1050)
+#  if !defined(KERNEL) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && ((__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__-0) >= 1050)
+#    define __DARWIN_VERS_1050 1
+#  else /* default */
+#    define __DARWIN_VERS_1050 0
+#  endif
+#endif /* !__DARWIN_NON_CANCELABLE */
+
+/*
+ * symbol suffixes used for symbol versioning
+ */
+#if __DARWIN_UNIX03
+#  if !defined(__LP64__) && !defined(__arm__)
+#    define __DARWIN_SUF_UNIX03		"$UNIX2003"
+#    define __DARWIN_SUF_UNIX03_SET	1
+#  else /* __LP64__ || __arm__ */
+#    define __DARWIN_SUF_UNIX03		/* nothing */
+#    define __DARWIN_SUF_UNIX03_SET	0
+#  endif /* !__LP64__ && !__arm__ */
+
+#  if __DARWIN_64_BIT_INO_T
+#    define __DARWIN_SUF_64_BIT_INO_T	"$INODE64"
+#  else /* !__DARWIN_64_BIT_INO_T */
+#    define __DARWIN_SUF_64_BIT_INO_T	/* nothing */
+#  endif /* __DARWIN_UNIX03 */
+
+#  if __DARWIN_NON_CANCELABLE
+#    define __DARWIN_SUF_NON_CANCELABLE	"$NOCANCEL"
+#  else /* !__DARWIN_NON_CANCELABLE */
+#    define __DARWIN_SUF_NON_CANCELABLE	/* nothing */
+#  endif /* __DARWIN_NON_CANCELABLE */
+
+#  if __DARWIN_VERS_1050
+#    define __DARWIN_SUF_1050		"$1050"
+#  else /* !__DARWIN_VERS_1050 */
+#    define __DARWIN_SUF_1050		/* nothing */
+#  endif /* __DARWIN_VERS_1050 */
+
+#else /* !__DARWIN_UNIX03 */
+#  define __DARWIN_SUF_UNIX03		/* nothing */
+#  define __DARWIN_SUF_UNIX03_SET	0
+#  define __DARWIN_SUF_64_BIT_INO_T	/* nothing */
+#  define __DARWIN_SUF_NON_CANCELABLE	/* nothing */
+#  define __DARWIN_SUF_1050		/* nothing */
+#endif /* __DARWIN_UNIX03 */
+
+#define __DARWIN_SUF_EXTSN		"$DARWIN_EXTSN"
+
+/*
+ * symbol versioning macros
+ */
+#define __DARWIN_ALIAS(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_UNIX03)
+#define __DARWIN_ALIAS_C(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_NON_CANCELABLE __DARWIN_SUF_UNIX03)
+#define __DARWIN_ALIAS_I(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_64_BIT_INO_T __DARWIN_SUF_UNIX03)
+#define __DARWIN_INODE64(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_64_BIT_INO_T)
+
+#define __DARWIN_1050(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_1050)
+#define __DARWIN_1050ALIAS(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_1050 __DARWIN_SUF_UNIX03)
+#define __DARWIN_1050ALIAS_C(sym)	__asm("_" __STRING(sym) __DARWIN_SUF_1050 __DARWIN_SUF_NON_CANCELABLE __DARWIN_SUF_UNIX03)
+#define __DARWIN_1050ALIAS_I(sym)	__asm("_" __STRING(sym) __DARWIN_SUF_1050 __DARWIN_SUF_64_BIT_INO_T __DARWIN_SUF_UNIX03)
+#define __DARWIN_1050INODE64(sym)	__asm("_" __STRING(sym) __DARWIN_SUF_1050 __DARWIN_SUF_64_BIT_INO_T)
+
+#define __DARWIN_EXTSN(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_EXTSN)
+#define __DARWIN_EXTSN_C(sym)		__asm("_" __STRING(sym) __DARWIN_SUF_EXTSN __DARWIN_SUF_NON_CANCELABLE)
 
 
 /*
@@ -327,7 +450,7 @@
 #   define	__DARWIN_LDBL_COMPAT2(x) /* nothing */
 #   define	__DARWIN_LONG_DOUBLE_IS_DOUBLE	1
 #  endif
-#elif defined(__i386__) || defined(__ppc64__) || defined(__x86_64__)
+#elif defined(__i386__) || defined(__ppc64__) || defined(__x86_64__) || defined (__arm__)
 #  define	__DARWIN_LDBL_COMPAT(x)	/* nothing */
 #  define	__DARWIN_LDBL_COMPAT2(x) /* nothing */
 #  define	__DARWIN_LONG_DOUBLE_IS_DOUBLE	0
@@ -335,4 +458,41 @@
 #  error Unknown architecture
 #endif
 
+/*
+ * Deprecation macro
+ */
+#if __GNUC__ >= 3
+#define __deprecated __attribute__((deprecated))
+#else
+#define __deprecated /* nothing */
+#endif
+
+/*****************************************
+ *  Public darwin-specific feature macros
+ *****************************************/
+
+/*
+ * _DARWIN_FEATURE_LONG_DOUBLE_IS_DOUBLE indicates when the long double type
+ * is the same as the double type (ppc only)
+ */
+#if __DARWIN_LONG_DOUBLE_IS_DOUBLE
+#define _DARWIN_FEATURE_LONG_DOUBLE_IS_DOUBLE	1
+#endif
+
+/*
+ * _DARWIN_FEATURE_UNIX_CONFORMANCE indicates whether UNIX conformance is on,
+ * and specifies the conformance level (3 is SUSv3)
+ */
+#if __DARWIN_UNIX03
+#define _DARWIN_FEATURE_UNIX_CONFORMANCE	3
+#endif
+
+/*
+ * _DARWIN_FEATURE_64_BIT_INODE indicates that the ino_t type is 64-bit, and
+ * structures modified for 64-bit inodes (like struct stat) will be used.
+ */
+#if __DARWIN_64_BIT_INO_T
+#define _DARWIN_FEATURE_64_BIT_INODE		1
+#endif
+
 #endif /* !_CDEFS_H_ */
diff --git a/bsd/sys/clist.h b/bsd/sys/clist.h
index 5503f994a..98aa9427b 100644
--- a/bsd/sys/clist.h
+++ b/bsd/sys/clist.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/codesign.h b/bsd/sys/codesign.h
new file mode 100644
index 000000000..f8fd235c9
--- /dev/null
+++ b/bsd/sys/codesign.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _SYS_CODESIGN_H_
+#define _SYS_CODESIGN_H_
+
+#include <sys/types.h>
+
+/* code signing attributes of a process */
+#define	CS_VALID		0x0001	/* dynamically valid */
+#define	CS_HARD			0x0100	/* don't load invalid pages */
+#define	CS_KILL			0x0200	/* kill process if it becomes invalid */
+#define CS_EXEC_SET_HARD	0x1000	/* set CS_HARD on any exec'ed process */
+#define CS_EXEC_SET_KILL	0x2000	/* set CS_KILL on any exec'ed process */
+
+/* csops  operations */
+#define	CS_OPS_STATUS		0	/* return status */
+#define	CS_OPS_MARKINVALID	1	/* invalidate process */
+#define	CS_OPS_MARKHARD		2	/* set HARD flag */
+#define	CS_OPS_MARKKILL		3	/* set KILL flag (sticky) */
+#define	CS_OPS_PIDPATH		4	/* get executable's pathname */
+#define	CS_OPS_CDHASH		5	/* get code directory hash */
+
+#ifndef KERNEL
+
+__BEGIN_DECLS
+
+/* code sign operations */
+int csops(pid_t pid, unsigned int  ops, void * useraddr, size_t usersize);
+
+__END_DECLS
+
+#endif /* ! KERNEL */
+
+#endif /* _SYS_CODESIGN_H_ */
diff --git a/bsd/sys/conf.h b/bsd/sys/conf.h
index 7dd5ad281..319dd704e 100644
--- a/bsd/sys/conf.h
+++ b/bsd/sys/conf.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/dir.h b/bsd/sys/dir.h
index 112b8526c..921ce9526 100644
--- a/bsd/sys/dir.h
+++ b/bsd/sys/dir.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/sys/dirent.h b/bsd/sys/dirent.h
index 7a0732739..bef3ffb23 100644
--- a/bsd/sys/dirent.h
+++ b/bsd/sys/dirent.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -77,10 +83,12 @@ typedef	__darwin_ino_t	ino_t;		/* inode number */
 #define _INO_T
 #endif
 
+
 #define __DARWIN_MAXNAMLEN	255
 
 #pragma pack(4)
 
+#if !__DARWIN_64_BIT_INO_T
 struct dirent {
 	ino_t d_ino;			/* file number of entry */
 	__uint16_t d_reclen;		/* length of this record */
@@ -88,25 +96,32 @@ struct dirent {
 	__uint8_t  d_namlen;		/* length of string in d_name */
 	char d_name[__DARWIN_MAXNAMLEN + 1];	/* name must be no longer than this */
 };
+#endif /* !__DARWIN_64_BIT_INO_T */
 
 #pragma pack()
 
-#ifdef KERNEL
-#include <sys/kernel_types.h>
+#define __DARWIN_MAXPATHLEN	1024
+
+#define __DARWIN_STRUCT_DIRENTRY { \
+	__uint64_t  d_ino;      /* file number of entry */ \
+	__uint64_t  d_seekoff;  /* seek offset (optional, used by servers) */ \
+	__uint16_t  d_reclen;   /* length of this record */ \
+	__uint16_t  d_namlen;   /* length of string in d_name */ \
+	__uint8_t   d_type;     /* file type, see below */ \
+	char      d_name[__DARWIN_MAXPATHLEN]; /* entry name (up to MAXPATHLEN bytes) */ \
+}
 
+#if __DARWIN_64_BIT_INO_T
+struct dirent __DARWIN_STRUCT_DIRENTRY;
+#endif /* __DARWIN_64_BIT_INO_T */
+
+#ifdef KERNEL
 /* Extended directory entry */
-struct direntry{
-	ino64_t     d_ino;      /* file number of entry */
-	__uint64_t  d_seekoff;   /* seek offset (optional, used by servers) */
-	__uint16_t  d_reclen;   /* length of this record */
-	__uint16_t  d_namlen;   /* length of string in d_name */
-	__uint8_t   d_type;     /* file type, see below */
-	u_char      d_name[MAXPATHLEN - 1]; /* entry name (up to MAXPATHLEN - 1 bytes) */
-};
+struct direntry __DARWIN_STRUCT_DIRENTRY;
 #endif
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	d_fileno	d_ino		/* backward compatibility */
 #define	MAXNAMLEN	__DARWIN_MAXNAMLEN
 /*
diff --git a/bsd/sys/dis_tables.h b/bsd/sys/dis_tables.h
new file mode 100644
index 000000000..fcbc46b06
--- /dev/null
+++ b/bsd/sys/dis_tables.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1989 Carnegie-Mellon University
+ * Copyright (c) 1988 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef	_SYS_DIS_TABLES_H_
+#define _SYS_DIS_TABLES_H_
+
+/*
+ * Machine dependent constants
+ */
+
+#include <machine/dis_tables.h>
+
+#endif	/* _SYS_DIS_TABLES_H_ */
diff --git a/bsd/sys/disk.h b/bsd/sys/disk.h
index f7e627b64..b36277729 100644
--- a/bsd/sys/disk.h
+++ b/bsd/sys/disk.h
@@ -1,31 +1,36 @@
 /*
- * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_SYS_DISK_H_
 #define	_SYS_DISK_H_
 
+#include <stdint.h>
 #include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/cdefs.h>
 
 /*
  * Definitions
@@ -59,8 +64,6 @@
  * DKIOCGETMAXSEGMENTADDRESSABLEBITCOUNT get maximum segment width in bits
  */
 
-#pragma pack(4)
-
 typedef struct
 {
     char path[128];
@@ -68,55 +71,58 @@ typedef struct
 
 typedef struct
 {
-    u_int64_t              blockCount;
-    u_int32_t              blockSize;
+    uint64_t               blockCount;
+    uint32_t               blockSize;
 
-    u_int8_t               reserved0096[4];        /* reserved, clear to zero */
+    uint8_t                reserved0096[4];        /* reserved, clear to zero */
 } dk_format_capacity_t;
 
-/* LP64todo: not 64-bit clean */
 typedef struct
 {
     dk_format_capacity_t * capacities;
-    u_int32_t              capacitiesCount;        /* use zero to probe count */
+    uint32_t               capacitiesCount;        /* use zero to probe count */
 
-    u_int8_t               reserved0064[8];        /* reserved, clear to zero */
+#ifdef __LP64__
+    uint8_t                reserved0096[4];        /* reserved, clear to zero */
+#else /* !__LP64__ */
+    uint8_t                reserved0064[8];        /* reserved, clear to zero */
+#endif /* !__LP64__ */
 } dk_format_capacities_t;
 
-#pragma pack()
-
 #define DKIOCEJECT                            _IO('d', 21)
 #define DKIOCSYNCHRONIZECACHE                 _IO('d', 22)
 
 #define DKIOCFORMAT                           _IOW('d', 26, dk_format_capacity_t)
 #define DKIOCGETFORMATCAPACITIES              _IOWR('d', 26, dk_format_capacities_t)
 
-#define DKIOCGETBLOCKSIZE                     _IOR('d', 24, u_int32_t)
-#define DKIOCGETBLOCKCOUNT                    _IOR('d', 25, u_int64_t)
+#define DKIOCGETBLOCKSIZE                     _IOR('d', 24, uint32_t)
+#define DKIOCGETBLOCKCOUNT                    _IOR('d', 25, uint64_t)
 #define DKIOCGETFIRMWAREPATH                  _IOR('d', 28, dk_firmware_path_t)
 
-#define DKIOCISFORMATTED                      _IOR('d', 23, u_int32_t)
-#define DKIOCISWRITABLE                       _IOR('d', 29, u_int32_t)
+#define DKIOCISFORMATTED                      _IOR('d', 23, uint32_t)
+#define DKIOCISWRITABLE                       _IOR('d', 29, uint32_t)
 
-#define DKIOCGETMAXBLOCKCOUNTREAD             _IOR('d', 64, u_int64_t)
-#define DKIOCGETMAXBLOCKCOUNTWRITE            _IOR('d', 65, u_int64_t)
-#define DKIOCGETMAXBYTECOUNTREAD              _IOR('d', 70, u_int64_t)
-#define DKIOCGETMAXBYTECOUNTWRITE             _IOR('d', 71, u_int64_t)
+#define DKIOCGETMAXBLOCKCOUNTREAD             _IOR('d', 64, uint64_t)
+#define DKIOCGETMAXBLOCKCOUNTWRITE            _IOR('d', 65, uint64_t)
+#define DKIOCGETMAXBYTECOUNTREAD              _IOR('d', 70, uint64_t)
+#define DKIOCGETMAXBYTECOUNTWRITE             _IOR('d', 71, uint64_t)
 
-#define DKIOCGETMAXSEGMENTCOUNTREAD           _IOR('d', 66, u_int64_t)
-#define DKIOCGETMAXSEGMENTCOUNTWRITE          _IOR('d', 67, u_int64_t)
-#define DKIOCGETMAXSEGMENTBYTECOUNTREAD       _IOR('d', 68, u_int64_t)
-#define DKIOCGETMAXSEGMENTBYTECOUNTWRITE      _IOR('d', 69, u_int64_t)
+#define DKIOCGETMAXSEGMENTCOUNTREAD           _IOR('d', 66, uint64_t)
+#define DKIOCGETMAXSEGMENTCOUNTWRITE          _IOR('d', 67, uint64_t)
+#define DKIOCGETMAXSEGMENTBYTECOUNTREAD       _IOR('d', 68, uint64_t)
+#define DKIOCGETMAXSEGMENTBYTECOUNTWRITE      _IOR('d', 69, uint64_t)
 
-#define DKIOCGETMINSEGMENTALIGNMENTBYTECOUNT  _IOR('d', 74, u_int64_t)
-#define DKIOCGETMAXSEGMENTADDRESSABLEBITCOUNT _IOR('d', 75, u_int64_t)
+#define DKIOCGETMINSEGMENTALIGNMENTBYTECOUNT  _IOR('d', 74, uint64_t)
+#define DKIOCGETMAXSEGMENTADDRESSABLEBITCOUNT _IOR('d', 75, uint64_t)
 
 #ifdef KERNEL
-#define DKIOCGETBLOCKCOUNT32                  _IOR('d', 25, u_int32_t)
-#define DKIOCSETBLOCKSIZE                     _IOW('d', 24, u_int32_t)
-#define DKIOCGETBSDUNIT                       _IOR('d', 27, u_int32_t)
-#define DKIOCISVIRTUAL                        _IOR('d', 72, u_int32_t)
-#define DKIOCGETBASE                          _IOR('d', 73, u_int64_t)
+#define DK_FEATURE_FORCE_UNIT_ACCESS          0x00000001
+#define DKIOCGETBLOCKCOUNT32                  _IOR('d', 25, uint32_t)
+#define DKIOCSETBLOCKSIZE                     _IOW('d', 24, uint32_t)
+#define DKIOCGETBSDUNIT                       _IOR('d', 27, uint32_t)
+#define DKIOCISVIRTUAL                        _IOR('d', 72, uint32_t)
+#define DKIOCGETBASE                          _IOR('d', 73, uint64_t)
+#define DKIOCGETFEATURES                      _IOR('d', 76, uint32_t)
 #endif /* KERNEL */
 
 #endif	/* _SYS_DISK_H_ */
diff --git a/bsd/sys/disklabel.h b/bsd/sys/disklabel.h
index 14eb3d746..a56c354b7 100644
--- a/bsd/sys/disklabel.h
+++ b/bsd/sys/disklabel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/sys/disktab.h b/bsd/sys/disktab.h
index 4df1f84f7..a7efed39a 100644
--- a/bsd/sys/disktab.h
+++ b/bsd/sys/disktab.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * HISTORY:
diff --git a/bsd/sys/dkstat.h b/bsd/sys/dkstat.h
index fa0060f6f..4fe84af4c 100644
--- a/bsd/sys/dkstat.h
+++ b/bsd/sys/dkstat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/dmap.h b/bsd/sys/dmap.h
index 3234c4d5d..a58593098 100644
--- a/bsd/sys/dmap.h
+++ b/bsd/sys/dmap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/domain.h b/bsd/sys/domain.h
index 3a5833774..9d4fd75b5 100644
--- a/bsd/sys/domain.h
+++ b/bsd/sys/domain.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -82,7 +88,7 @@ struct	mbuf;
 
 struct	domain {
 	int	dom_family;		/* AF_xxx */
-	char	*dom_name;
+	const char	*dom_name;
 	void	(*dom_init)(void);	/* initialize domain data structures */
 	int	(*dom_externalize)(struct mbuf *);
 					/* externalize access rights */
diff --git a/bsd/sys/dtrace.h b/bsd/sys/dtrace.h
new file mode 100644
index 000000000..7785b9b34
--- /dev/null
+++ b/bsd/sys/dtrace.h
@@ -0,0 +1,2397 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_DTRACE_H
+#define _SYS_DTRACE_H
+
+/* #pragma ident	"@(#)dtrace.h	1.32	06/08/07 SMI" */
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+/*
+ * DTrace Dynamic Tracing Software: Kernel Interfaces
+ *
+ * Note: The contents of this file are private to the implementation of the
+ * Solaris system and DTrace subsystem and are subject to change at any time
+ * without notice.  Applications and drivers using these interfaces will fail
+ * to run on future releases.  These interfaces should not be used for any
+ * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
+ * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
+ */
+
+#ifndef _ASM
+
+#if !defined(__APPLE__)
+#include <sys/types.h>
+#include <sys/modctl.h>
+#include <sys/processor.h>
+#include <sys/systm.h>
+#include <sys/ctf_api.h>
+#include <sys/cyclic.h>
+#include <sys/int_limits.h>
+#else /* is Apple Mac OS X */
+
+#ifdef KERNEL
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+#endif
+
+#if defined(__BIG_ENDIAN__)
+#if !defined(_BIG_ENDIAN)
+#define _BIG_ENDIAN /* Solaris vs. Darwin */
+#endif
+#elif defined(__LITTLE_ENDIAN__)
+#if !defined(_LITTLE_ENDIAN)
+#define _LITTLE_ENDIAN /* Solaris vs. Darwin */
+#endif
+#else
+#error Unknown endian-ness
+#endif
+
+#include <sys/types.h>
+#include <stdint.h>
+
+#ifndef NULL
+#define NULL ((void *)0) /* quiets many warnings */
+#endif
+
+#define SEC			1
+#define MILLISEC	1000
+#define MICROSEC	1000000
+#define NANOSEC		1000000000
+
+#define S_ROUND(x, a)   ((x) + (((a) ? (a) : 1) - 1) & ~(((a) ? (a) : 1) - 1))
+#define P2ROUNDUP(x, align)             (-(-(x) & -(align)))
+
+#define	CTF_MODEL_ILP32	1	/* object data model is ILP32 */
+#define	CTF_MODEL_NATIVE	CTF_MODEL_ILP32
+
+typedef uint8_t		uchar_t;
+typedef uint16_t	ushort_t;
+typedef uint32_t	uint_t;
+typedef uint32_t	ulong_t;
+typedef uint64_t	u_longlong_t;
+typedef int64_t		longlong_t;
+typedef int64_t		off64_t;
+typedef int			processorid_t;
+typedef int64_t		hrtime_t;
+
+typedef enum { B_FALSE = 0, B_TRUE = 1 } _dtrace_boolean;
+
+struct modctl; /* In lieu of Solaris <sys/modctl.h> */
+/* NOTHING */  /* In lieu of Solaris <sys/processor.h> */
+#include <sys/ioctl.h> /* In lieu of Solaris <sys/systm.h> */
+#ifdef KERNEL
+/* NOTHING */ /* In lieu of Solaris <sys/ctf_api.h> */
+#else
+/* In lieu of Solaris <sys/ctf_api.h> */
+typedef struct ctf_file ctf_file_t;
+typedef long ctf_id_t;
+#endif
+/* NOTHING */ /* In lieu of Solaris <sys/cyclic.h> */
+/* NOTHING */ /* In lieu of Solaris <sys/int_limits.h> */
+
+typedef uint32_t        zoneid_t;
+
+#include <sys/dtrace_glue.h>
+
+#include <stdarg.h> 
+typedef va_list __va_list;
+
+#define proc_t struct proc
+#endif /* __APPLE__ */
+
+/*
+ * DTrace Universal Constants and Typedefs
+ */
+#define	DTRACE_CPUALL		-1	/* all CPUs */
+#define	DTRACE_IDNONE		0	/* invalid probe identifier */
+#define	DTRACE_EPIDNONE		0	/* invalid enabled probe identifier */
+#define	DTRACE_AGGIDNONE	0	/* invalid aggregation identifier */
+#define	DTRACE_AGGVARIDNONE	0	/* invalid aggregation variable ID */
+#define	DTRACE_CACHEIDNONE	0	/* invalid predicate cache */
+#define	DTRACE_PROVNONE		0	/* invalid provider identifier */
+#define	DTRACE_METAPROVNONE	0	/* invalid meta-provider identifier */
+#define	DTRACE_ARGNONE		-1	/* invalid argument index */
+
+#define	DTRACE_PROVNAMELEN	64
+#define	DTRACE_MODNAMELEN	64
+#define	DTRACE_FUNCNAMELEN	128
+#define	DTRACE_NAMELEN		64
+#define	DTRACE_FULLNAMELEN	(DTRACE_PROVNAMELEN + DTRACE_MODNAMELEN + \
+				DTRACE_FUNCNAMELEN + DTRACE_NAMELEN + 4)
+#define	DTRACE_ARGTYPELEN	128
+
+typedef uint32_t dtrace_id_t;		/* probe identifier */
+typedef uint32_t dtrace_epid_t;		/* enabled probe identifier */
+typedef uint32_t dtrace_aggid_t;	/* aggregation identifier */
+typedef int64_t dtrace_aggvarid_t;	/* aggregation variable identifier */
+typedef uint16_t dtrace_actkind_t;	/* action kind */
+typedef int64_t dtrace_optval_t;	/* option value */
+typedef uint32_t dtrace_cacheid_t;	/* predicate cache identifier */
+
+typedef enum dtrace_probespec {
+        DTRACE_PROBESPEC_NONE = -1,
+        DTRACE_PROBESPEC_PROVIDER = 0,
+        DTRACE_PROBESPEC_MOD,
+        DTRACE_PROBESPEC_FUNC,
+        DTRACE_PROBESPEC_NAME
+} dtrace_probespec_t;
+
+/*
+ * DTrace Intermediate Format (DIF)
+ *
+ * The following definitions describe the DTrace Intermediate Format (DIF), a
+ * a RISC-like instruction set and program encoding used to represent
+ * predicates and actions that can be bound to DTrace probes.  The constants
+ * below defining the number of available registers are suggested minimums; the
+ * compiler should use DTRACEIOC_CONF to dynamically obtain the number of
+ * registers provided by the current DTrace implementation.
+ */
+#define	DIF_VERSION_1	1		/* DIF version 1: Solaris 10 Beta */
+#define	DIF_VERSION_2	2		/* DIF version 2: Solaris 10 FCS */
+#define	DIF_VERSION	DIF_VERSION_2	/* latest DIF instruction set version */
+#define	DIF_DIR_NREGS	8		/* number of DIF integer registers */
+#define	DIF_DTR_NREGS	8		/* number of DIF tuple registers */
+
+#define	DIF_OP_OR	1		/* or	r1, r2, rd */
+#define	DIF_OP_XOR	2		/* xor	r1, r2, rd */
+#define	DIF_OP_AND	3		/* and	r1, r2, rd */
+#define	DIF_OP_SLL	4		/* sll	r1, r2, rd */
+#define	DIF_OP_SRL	5		/* srl	r1, r2, rd */
+#define	DIF_OP_SUB	6		/* sub	r1, r2, rd */
+#define	DIF_OP_ADD	7		/* add	r1, r2, rd */
+#define	DIF_OP_MUL	8		/* mul	r1, r2, rd */
+#define	DIF_OP_SDIV	9		/* sdiv	r1, r2, rd */
+#define	DIF_OP_UDIV	10		/* udiv r1, r2, rd */
+#define	DIF_OP_SREM	11		/* srem r1, r2, rd */
+#define	DIF_OP_UREM	12		/* urem r1, r2, rd */
+#define	DIF_OP_NOT	13		/* not	r1, rd */
+#define	DIF_OP_MOV	14		/* mov	r1, rd */
+#define	DIF_OP_CMP	15		/* cmp	r1, r2 */
+#define	DIF_OP_TST	16		/* tst  r1 */
+#define	DIF_OP_BA	17		/* ba	label */
+#define	DIF_OP_BE	18		/* be	label */
+#define	DIF_OP_BNE	19		/* bne	label */
+#define	DIF_OP_BG	20		/* bg	label */
+#define	DIF_OP_BGU	21		/* bgu	label */
+#define	DIF_OP_BGE	22		/* bge	label */
+#define	DIF_OP_BGEU	23		/* bgeu	label */
+#define	DIF_OP_BL	24		/* bl	label */
+#define	DIF_OP_BLU	25		/* blu	label */
+#define	DIF_OP_BLE	26		/* ble	label */
+#define	DIF_OP_BLEU	27		/* bleu	label */
+#define	DIF_OP_LDSB	28		/* ldsb	[r1], rd */
+#define	DIF_OP_LDSH	29		/* ldsh	[r1], rd */
+#define	DIF_OP_LDSW	30		/* ldsw [r1], rd */
+#define	DIF_OP_LDUB	31		/* ldub	[r1], rd */
+#define	DIF_OP_LDUH	32		/* lduh	[r1], rd */
+#define	DIF_OP_LDUW	33		/* lduw	[r1], rd */
+#define	DIF_OP_LDX	34		/* ldx	[r1], rd */
+#define	DIF_OP_RET	35		/* ret	rd */
+#define	DIF_OP_NOP	36		/* nop */
+#define	DIF_OP_SETX	37		/* setx	intindex, rd */
+#define	DIF_OP_SETS	38		/* sets strindex, rd */
+#define	DIF_OP_SCMP	39		/* scmp	r1, r2 */
+#define	DIF_OP_LDGA	40		/* ldga	var, ri, rd */
+#define	DIF_OP_LDGS	41		/* ldgs var, rd */
+#define	DIF_OP_STGS	42		/* stgs var, rs */
+#define	DIF_OP_LDTA	43		/* ldta var, ri, rd */
+#define	DIF_OP_LDTS	44		/* ldts var, rd */
+#define	DIF_OP_STTS	45		/* stts var, rs */
+#define	DIF_OP_SRA	46		/* sra	r1, r2, rd */
+#define	DIF_OP_CALL	47		/* call	subr, rd */
+#define	DIF_OP_PUSHTR	48		/* pushtr type, rs, rr */
+#define	DIF_OP_PUSHTV	49		/* pushtv type, rs, rv */
+#define	DIF_OP_POPTS	50		/* popts */
+#define	DIF_OP_FLUSHTS	51		/* flushts */
+#define	DIF_OP_LDGAA	52		/* ldgaa var, rd */
+#define	DIF_OP_LDTAA	53		/* ldtaa var, rd */
+#define	DIF_OP_STGAA	54		/* stgaa var, rs */
+#define	DIF_OP_STTAA	55		/* sttaa var, rs */
+#define	DIF_OP_LDLS	56		/* ldls	var, rd */
+#define	DIF_OP_STLS	57		/* stls	var, rs */
+#define	DIF_OP_ALLOCS	58		/* allocs r1, rd */
+#define	DIF_OP_COPYS	59		/* copys  r1, r2, rd */
+#define	DIF_OP_STB	60		/* stb	r1, [rd] */
+#define	DIF_OP_STH	61		/* sth	r1, [rd] */
+#define	DIF_OP_STW	62		/* stw	r1, [rd] */
+#define	DIF_OP_STX	63		/* stx	r1, [rd] */
+#define	DIF_OP_ULDSB	64		/* uldsb [r1], rd */
+#define	DIF_OP_ULDSH	65		/* uldsh [r1], rd */
+#define	DIF_OP_ULDSW	66		/* uldsw [r1], rd */
+#define	DIF_OP_ULDUB	67		/* uldub [r1], rd */
+#define	DIF_OP_ULDUH	68		/* ulduh [r1], rd */
+#define	DIF_OP_ULDUW	69		/* ulduw [r1], rd */
+#define	DIF_OP_ULDX	70		/* uldx  [r1], rd */
+#define	DIF_OP_RLDSB	71		/* rldsb [r1], rd */
+#define	DIF_OP_RLDSH	72		/* rldsh [r1], rd */
+#define	DIF_OP_RLDSW	73		/* rldsw [r1], rd */
+#define	DIF_OP_RLDUB	74		/* rldub [r1], rd */
+#define	DIF_OP_RLDUH	75		/* rlduh [r1], rd */
+#define	DIF_OP_RLDUW	76		/* rlduw [r1], rd */
+#define	DIF_OP_RLDX	77		/* rldx  [r1], rd */
+#define	DIF_OP_XLATE	78		/* xlate xlrindex, rd */
+#define	DIF_OP_XLARG	79		/* xlarg xlrindex, rd */
+
+#define	DIF_INTOFF_MAX		0xffff	/* highest integer table offset */
+#define	DIF_STROFF_MAX		0xffff	/* highest string table offset */
+#define	DIF_REGISTER_MAX	0xff	/* highest register number */
+#define	DIF_VARIABLE_MAX	0xffff	/* highest variable identifier */
+#define	DIF_SUBROUTINE_MAX	0xffff	/* highest subroutine code */
+
+#define	DIF_VAR_ARRAY_MIN	0x0000	/* lowest numbered array variable */
+#define	DIF_VAR_ARRAY_UBASE	0x0080	/* lowest user-defined array */
+#define	DIF_VAR_ARRAY_MAX	0x00ff	/* highest numbered array variable */
+
+#define	DIF_VAR_OTHER_MIN	0x0100	/* lowest numbered scalar or assc */
+#define	DIF_VAR_OTHER_UBASE	0x0500	/* lowest user-defined scalar or assc */
+#define	DIF_VAR_OTHER_MAX	0xffff	/* highest numbered scalar or assc */
+
+#define	DIF_VAR_ARGS		0x0000	/* arguments array */
+#define	DIF_VAR_REGS		0x0001	/* registers array */
+#define	DIF_VAR_UREGS		0x0002	/* user registers array */
+#define	DIF_VAR_CURTHREAD	0x0100	/* thread pointer */
+#define	DIF_VAR_TIMESTAMP	0x0101	/* timestamp */
+#define	DIF_VAR_VTIMESTAMP	0x0102	/* virtual timestamp */
+#define	DIF_VAR_IPL		0x0103	/* interrupt priority level */
+#define	DIF_VAR_EPID		0x0104	/* enabled probe ID */
+#define	DIF_VAR_ID		0x0105	/* probe ID */
+#define	DIF_VAR_ARG0		0x0106	/* first argument */
+#define	DIF_VAR_ARG1		0x0107	/* second argument */
+#define	DIF_VAR_ARG2		0x0108	/* third argument */
+#define	DIF_VAR_ARG3		0x0109	/* fourth argument */
+#define	DIF_VAR_ARG4		0x010a	/* fifth argument */
+#define	DIF_VAR_ARG5		0x010b	/* sixth argument */
+#define	DIF_VAR_ARG6		0x010c	/* seventh argument */
+#define	DIF_VAR_ARG7		0x010d	/* eighth argument */
+#define	DIF_VAR_ARG8		0x010e	/* ninth argument */
+#define	DIF_VAR_ARG9		0x010f	/* tenth argument */
+#define	DIF_VAR_STACKDEPTH	0x0110	/* stack depth */
+#define	DIF_VAR_CALLER		0x0111	/* caller */
+#define	DIF_VAR_PROBEPROV	0x0112	/* probe provider */
+#define	DIF_VAR_PROBEMOD	0x0113	/* probe module */
+#define	DIF_VAR_PROBEFUNC	0x0114	/* probe function */
+#define	DIF_VAR_PROBENAME	0x0115	/* probe name */
+#define	DIF_VAR_PID		0x0116	/* process ID */
+#define	DIF_VAR_TID		0x0117	/* (per-process) thread ID */
+#define	DIF_VAR_EXECNAME	0x0118	/* name of executable */
+#define	DIF_VAR_ZONENAME	0x0119	/* zone name associated with process */
+#define	DIF_VAR_WALLTIMESTAMP	0x011a	/* wall-clock timestamp */
+#define	DIF_VAR_USTACKDEPTH	0x011b	/* user-land stack depth */
+#define	DIF_VAR_UCALLER		0x011c	/* user-level caller */
+#define	DIF_VAR_PPID		0x011d	/* parent process ID */
+#define	DIF_VAR_UID		0x011e	/* process user ID */
+#define	DIF_VAR_GID		0x011f	/* process group ID */
+#define	DIF_VAR_ERRNO		0x0120	/* thread errno */
+
+#define	DIF_SUBR_RAND			0
+#define	DIF_SUBR_MUTEX_OWNED		1
+#define	DIF_SUBR_MUTEX_OWNER		2
+#define	DIF_SUBR_MUTEX_TYPE_ADAPTIVE	3
+#define	DIF_SUBR_MUTEX_TYPE_SPIN	4
+#define	DIF_SUBR_RW_READ_HELD		5
+#define	DIF_SUBR_RW_WRITE_HELD		6
+#define	DIF_SUBR_RW_ISWRITER		7
+#define	DIF_SUBR_COPYIN			8
+#define	DIF_SUBR_COPYINSTR		9
+#define	DIF_SUBR_SPECULATION		10
+#define	DIF_SUBR_PROGENYOF		11
+#define	DIF_SUBR_STRLEN			12
+#define	DIF_SUBR_COPYOUT		13
+#define	DIF_SUBR_COPYOUTSTR		14
+#define	DIF_SUBR_ALLOCA			15
+#define	DIF_SUBR_BCOPY			16
+#define	DIF_SUBR_COPYINTO		17
+#define	DIF_SUBR_MSGDSIZE		18
+#define	DIF_SUBR_MSGSIZE		19
+#define	DIF_SUBR_GETMAJOR		20
+#define	DIF_SUBR_GETMINOR		21
+#define	DIF_SUBR_DDI_PATHNAME		22
+#define	DIF_SUBR_STRJOIN		23
+#define	DIF_SUBR_LLTOSTR		24
+#define	DIF_SUBR_BASENAME		25
+#define	DIF_SUBR_DIRNAME		26
+#define	DIF_SUBR_CLEANPATH		27
+#define	DIF_SUBR_STRCHR			28
+#define	DIF_SUBR_STRRCHR		29
+#define	DIF_SUBR_STRSTR			30
+#define	DIF_SUBR_STRTOK			31
+#define	DIF_SUBR_SUBSTR			32
+#define	DIF_SUBR_INDEX			33
+#define	DIF_SUBR_RINDEX			34
+#define DIF_SUBR_CHUD			35
+
+#define DIF_SUBR_MAX            35      /* max subroutine value */
+
+typedef uint32_t dif_instr_t;
+
+#define DIF_INSTR_OP(i)                 (((i) >> 24) & 0xff)
+#define DIF_INSTR_R1(i)                 (((i) >> 16) & 0xff)
+#define DIF_INSTR_R2(i)                 (((i) >>  8) & 0xff)
+#define DIF_INSTR_RD(i)                 ((i) & 0xff)
+#define DIF_INSTR_RS(i)                 ((i) & 0xff)
+#define DIF_INSTR_LABEL(i)              ((i) & 0xffffff)
+#define DIF_INSTR_VAR(i)                (((i) >>  8) & 0xffff)
+#define DIF_INSTR_INTEGER(i)            (((i) >>  8) & 0xffff)
+#define DIF_INSTR_STRING(i)             (((i) >>  8) & 0xffff)
+#define DIF_INSTR_SUBR(i)               (((i) >>  8) & 0xffff)
+#define DIF_INSTR_TYPE(i)               (((i) >> 16) & 0xff)
+#define DIF_INSTR_XLREF(i)              (((i) >>  8) & 0xffff)
+
+#define DIF_INSTR_FMT(op, r1, r2, d) \
+        (((op) << 24) | ((r1) << 16) | ((r2) << 8) | (d))
+
+#define DIF_INSTR_NOT(r1, d)            (DIF_INSTR_FMT(DIF_OP_NOT, r1, 0, d))
+#define DIF_INSTR_MOV(r1, d)            (DIF_INSTR_FMT(DIF_OP_MOV, r1, 0, d))
+#define DIF_INSTR_CMP(op, r1, r2)       (DIF_INSTR_FMT(op, r1, r2, 0))
+#define DIF_INSTR_TST(r1)               (DIF_INSTR_FMT(DIF_OP_TST, r1, 0, 0))
+#define DIF_INSTR_BRANCH(op, label)     (((op) << 24) | (label))
+#define DIF_INSTR_LOAD(op, r1, d)       (DIF_INSTR_FMT(op, r1, 0, d))
+#define DIF_INSTR_STORE(op, r1, d)      (DIF_INSTR_FMT(op, r1, 0, d))
+#define DIF_INSTR_SETX(i, d)            ((DIF_OP_SETX << 24) | ((i) << 8) | (d))
+#define DIF_INSTR_SETS(s, d)            ((DIF_OP_SETS << 24) | ((s) << 8) | (d))
+#define DIF_INSTR_RET(d)                (DIF_INSTR_FMT(DIF_OP_RET, 0, 0, d))
+#define DIF_INSTR_NOP                   (DIF_OP_NOP << 24)
+#define DIF_INSTR_LDA(op, v, r, d)      (DIF_INSTR_FMT(op, v, r, d))
+#define DIF_INSTR_LDV(op, v, d)         (((op) << 24) | ((v) << 8) | (d))
+#define DIF_INSTR_STV(op, v, rs)        (((op) << 24) | ((v) << 8) | (rs))
+#define DIF_INSTR_CALL(s, d)            ((DIF_OP_CALL << 24) | ((s) << 8) | (d))
+#define DIF_INSTR_PUSHTS(op, t, r2, rs) (DIF_INSTR_FMT(op, t, r2, rs))
+#define DIF_INSTR_POPTS                 (DIF_OP_POPTS << 24)
+#define DIF_INSTR_FLUSHTS               (DIF_OP_FLUSHTS << 24)
+#define DIF_INSTR_ALLOCS(r1, d)         (DIF_INSTR_FMT(DIF_OP_ALLOCS, r1, 0, d))
+#define DIF_INSTR_COPYS(r1, r2, d)      (DIF_INSTR_FMT(DIF_OP_COPYS, r1, r2, d))
+#define DIF_INSTR_XLATE(op, r, d)       (((op) << 24) | ((r) << 8) | (d))
+
+#define DIF_REG_R0      0               /* %r0 is always set to zero */
+
+/*
+ * A DTrace Intermediate Format Type (DIF Type) is used to represent the types
+ * of variables, function and associative array arguments, and the return type
+ * for each DIF object (shown below).  It contains a description of the type,
+ * its size in bytes, and a module identifier.
+ */
+typedef struct dtrace_diftype {
+        uint8_t dtdt_kind;              /* type kind (see below) */
+        uint8_t dtdt_ckind;             /* type kind in CTF */
+        uint8_t dtdt_flags;             /* type flags (see below) */
+        uint8_t dtdt_pad;               /* reserved for future use */
+        uint32_t dtdt_size;             /* type size in bytes (unless string) */
+} dtrace_diftype_t;
+
+#define DIF_TYPE_CTF            0       /* type is a CTF type */
+#define DIF_TYPE_STRING         1       /* type is a D string */
+
+#define DIF_TF_BYREF            0x1     /* type is passed by reference */
+
+/*
+ * A DTrace Intermediate Format variable record is used to describe each of the
+ * variables referenced by a given DIF object.  It contains an integer variable
+ * identifier along with variable scope and properties, as shown below.  The
+ * size of this structure must be sizeof (int) aligned.
+ */
+typedef struct dtrace_difv {
+        uint32_t dtdv_name;             /* variable name index in dtdo_strtab */
+        uint32_t dtdv_id;               /* variable reference identifier */
+        uint8_t dtdv_kind;              /* variable kind (see below) */
+        uint8_t dtdv_scope;             /* variable scope (see below) */
+        uint16_t dtdv_flags;            /* variable flags (see below) */
+        dtrace_diftype_t dtdv_type;     /* variable type (see above) */
+} dtrace_difv_t;
+
+#define DIFV_KIND_ARRAY         0       /* variable is an array of quantities */
+#define DIFV_KIND_SCALAR        1       /* variable is a scalar quantity */
+
+#define DIFV_SCOPE_GLOBAL       0       /* variable has global scope */
+#define DIFV_SCOPE_THREAD       1       /* variable has thread scope */
+#define DIFV_SCOPE_LOCAL        2       /* variable has local scope */
+
+#define DIFV_F_REF              0x1     /* variable is referenced by DIFO */
+#define DIFV_F_MOD              0x2     /* variable is written by DIFO */
+
+/*
+ * DTrace Actions
+ *
+ * The upper byte determines the class of the action; the low bytes determines
+ * the specific action within that class.  The classes of actions are as
+ * follows:
+ *
+ *   [ no class ]                  <= May record process- or kernel-related data
+ *   DTRACEACT_PROC                <= Only records process-related data
+ *   DTRACEACT_PROC_DESTRUCTIVE    <= Potentially destructive to processes
+ *   DTRACEACT_KERNEL              <= Only records kernel-related data
+ *   DTRACEACT_KERNEL_DESTRUCTIVE  <= Potentially destructive to the kernel
+ *   DTRACEACT_SPECULATIVE         <= Speculation-related action
+ *   DTRACEACT_AGGREGATION         <= Aggregating action
+ */
+#define DTRACEACT_NONE                  0       /* no action */
+#define DTRACEACT_DIFEXPR               1       /* action is DIF expression */
+#define DTRACEACT_EXIT                  2       /* exit() action */
+#define DTRACEACT_PRINTF                3       /* printf() action */
+#define DTRACEACT_PRINTA                4       /* printa() action */
+#define DTRACEACT_LIBACT                5       /* library-controlled action */
+
+#define DTRACEACT_PROC                  0x0100
+#define DTRACEACT_USTACK                (DTRACEACT_PROC + 1)
+#define DTRACEACT_JSTACK                (DTRACEACT_PROC + 2)
+#define DTRACEACT_USYM                  (DTRACEACT_PROC + 3)
+#define DTRACEACT_UMOD                  (DTRACEACT_PROC + 4)
+#define DTRACEACT_UADDR                 (DTRACEACT_PROC + 5)
+
+#define DTRACEACT_PROC_DESTRUCTIVE      0x0200
+#define DTRACEACT_STOP                  (DTRACEACT_PROC_DESTRUCTIVE + 1)
+#define DTRACEACT_RAISE                 (DTRACEACT_PROC_DESTRUCTIVE + 2)
+#define DTRACEACT_SYSTEM                (DTRACEACT_PROC_DESTRUCTIVE + 3)
+#define DTRACEACT_FREOPEN               (DTRACEACT_PROC_DESTRUCTIVE + 4)
+
+#define DTRACEACT_PROC_CONTROL          0x0300
+
+#define DTRACEACT_KERNEL                0x0400
+#define DTRACEACT_STACK                 (DTRACEACT_KERNEL + 1)
+#define DTRACEACT_SYM                   (DTRACEACT_KERNEL + 2)
+#define DTRACEACT_MOD                   (DTRACEACT_KERNEL + 3)
+
+#define DTRACEACT_KERNEL_DESTRUCTIVE    0x0500
+#define DTRACEACT_BREAKPOINT            (DTRACEACT_KERNEL_DESTRUCTIVE + 1)
+#define DTRACEACT_PANIC                 (DTRACEACT_KERNEL_DESTRUCTIVE + 2)
+#define DTRACEACT_CHILL                 (DTRACEACT_KERNEL_DESTRUCTIVE + 3)
+
+#define DTRACEACT_SPECULATIVE           0x0600
+#define DTRACEACT_SPECULATE             (DTRACEACT_SPECULATIVE + 1)
+#define DTRACEACT_COMMIT                (DTRACEACT_SPECULATIVE + 2)
+#define DTRACEACT_DISCARD               (DTRACEACT_SPECULATIVE + 3)
+
+#define DTRACEACT_CLASS(x)              ((x) & 0xff00)
+
+#define DTRACEACT_ISDESTRUCTIVE(x)      \
+        (DTRACEACT_CLASS(x) == DTRACEACT_PROC_DESTRUCTIVE || \
+        DTRACEACT_CLASS(x) == DTRACEACT_KERNEL_DESTRUCTIVE)
+
+#define DTRACEACT_ISSPECULATIVE(x)      \
+        (DTRACEACT_CLASS(x) == DTRACEACT_SPECULATIVE)
+
+#define DTRACEACT_ISPRINTFLIKE(x)       \
+        ((x) == DTRACEACT_PRINTF || (x) == DTRACEACT_PRINTA || \
+        (x) == DTRACEACT_SYSTEM || (x) == DTRACEACT_FREOPEN)
+
+/*
+ * DTrace Aggregating Actions
+ *
+ * These are functions f(x) for which the following is true:
+ *
+ *    f(f(x_0) U f(x_1) U ... U f(x_n)) = f(x_0 U x_1 U ... U x_n)
+ *
+ * where x_n is a set of arbitrary data.  Aggregating actions are in their own
+ * DTrace action class, DTTRACEACT_AGGREGATION.  The macros provided here allow
+ * for easier processing of the aggregation argument and data payload for a few
+ * aggregating actions (notably:  quantize(), lquantize(), and ustack()).
+ */
+#define DTRACEACT_AGGREGATION           0x0700
+#define DTRACEAGG_COUNT                 (DTRACEACT_AGGREGATION + 1)
+#define DTRACEAGG_MIN                   (DTRACEACT_AGGREGATION + 2)
+#define DTRACEAGG_MAX                   (DTRACEACT_AGGREGATION + 3)
+#define DTRACEAGG_AVG                   (DTRACEACT_AGGREGATION + 4)
+#define DTRACEAGG_SUM                   (DTRACEACT_AGGREGATION + 5)
+#define DTRACEAGG_STDDEV                (DTRACEACT_AGGREGATION + 6)
+#define DTRACEAGG_QUANTIZE              (DTRACEACT_AGGREGATION + 7)
+#define DTRACEAGG_LQUANTIZE             (DTRACEACT_AGGREGATION + 8)
+
+#define DTRACEACT_ISAGG(x)              \
+        (DTRACEACT_CLASS(x) == DTRACEACT_AGGREGATION)
+
+#define DTRACE_QUANTIZE_NBUCKETS        \
+        (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1)
+
+#define DTRACE_QUANTIZE_ZEROBUCKET      ((sizeof (uint64_t) * NBBY) - 1)
+
+#define DTRACE_QUANTIZE_BUCKETVAL(buck)                                 \
+        (int64_t)((buck) < DTRACE_QUANTIZE_ZEROBUCKET ?                 \
+        -(1LL << (DTRACE_QUANTIZE_ZEROBUCKET - 1 - (buck))) :           \
+        (buck) == DTRACE_QUANTIZE_ZEROBUCKET ? 0 :                      \
+        1LL << ((buck) - DTRACE_QUANTIZE_ZEROBUCKET - 1))
+
+#define DTRACE_LQUANTIZE_STEPSHIFT              48
+#define DTRACE_LQUANTIZE_STEPMASK               ((uint64_t)UINT16_MAX << 48)
+#define DTRACE_LQUANTIZE_LEVELSHIFT             32
+#define DTRACE_LQUANTIZE_LEVELMASK              ((uint64_t)UINT16_MAX << 32)
+#define DTRACE_LQUANTIZE_BASESHIFT              0
+#define DTRACE_LQUANTIZE_BASEMASK               UINT32_MAX
+
+#define DTRACE_LQUANTIZE_STEP(x)                \
+        (uint16_t)(((x) & DTRACE_LQUANTIZE_STEPMASK) >> \
+        DTRACE_LQUANTIZE_STEPSHIFT)
+
+#define DTRACE_LQUANTIZE_LEVELS(x)              \
+        (uint16_t)(((x) & DTRACE_LQUANTIZE_LEVELMASK) >> \
+        DTRACE_LQUANTIZE_LEVELSHIFT)
+
+#define DTRACE_LQUANTIZE_BASE(x)                \
+        (int32_t)(((x) & DTRACE_LQUANTIZE_BASEMASK) >> \
+        DTRACE_LQUANTIZE_BASESHIFT)
+
+#define DTRACE_USTACK_NFRAMES(x)        (uint32_t)((x) & UINT32_MAX)
+#define DTRACE_USTACK_STRSIZE(x)        (uint32_t)((x) >> 32)
+#define DTRACE_USTACK_ARG(x, y)         \
+        ((((uint64_t)(y)) << 32) | ((x) & UINT32_MAX))
+
+#ifndef _LP64
+#ifndef _LITTLE_ENDIAN
+#define DTRACE_PTR(type, name)  uint32_t name##pad; type *name
+#else
+#define DTRACE_PTR(type, name)  type *name; uint32_t name##pad
+#endif
+#else
+#define DTRACE_PTR(type, name)  type *name
+#endif
+
+/*
+ * DTrace Object Format (DOF)
+ *
+ * DTrace programs can be persistently encoded in the DOF format so that they
+ * may be embedded in other programs (for example, in an ELF file) or in the
+ * dtrace driver configuration file for use in anonymous tracing.  The DOF
+ * format is versioned and extensible so that it can be revised and so that
+ * internal data structures can be modified or extended compatibly.  All DOF
+ * structures use fixed-size types, so the 32-bit and 64-bit representations
+ * are identical and consumers can use either data model transparently.
+ *
+ * The file layout is structured as follows:
+ *
+ * +---------------+-------------------+----- ... ----+---- ... ------+
+ * |   dof_hdr_t   |  dof_sec_t[ ... ] |   loadable   | non-loadable  |
+ * | (file header) | (section headers) | section data | section data  |
+ * +---------------+-------------------+----- ... ----+---- ... ------+
+ * |<------------ dof_hdr.dofh_loadsz --------------->|               |
+ * |<------------ dof_hdr.dofh_filesz ------------------------------->|
+ *
+ * The file header stores meta-data including a magic number, data model for
+ * the instrumentation, data encoding, and properties of the DIF code within.
+ * The header describes its own size and the size of the section headers.  By
+ * convention, an array of section headers follows the file header, and then
+ * the data for all loadable sections and unloadable sections.  This permits
+ * consumer code to easily download the headers and all loadable data into the
+ * DTrace driver in one contiguous chunk, omitting other extraneous sections.
+ *
+ * The section headers describe the size, offset, alignment, and section type
+ * for each section.  Sections are described using a set of #defines that tell
+ * the consumer what kind of data is expected.  Sections can contain links to
+ * other sections by storing a dof_secidx_t, an index into the section header
+ * array, inside of the section data structures.  The section header includes
+ * an entry size so that sections with data arrays can grow their structures.
+ *
+ * The DOF data itself can contain many snippets of DIF (i.e. >1 DIFOs), which
+ * are represented themselves as a collection of related DOF sections.  This
+ * permits us to change the set of sections associated with a DIFO over time,
+ * and also permits us to encode DIFOs that contain different sets of sections.
+ * When a DOF section wants to refer to a DIFO, it stores the dof_secidx_t of a
+ * section of type DOF_SECT_DIFOHDR.  This section's data is then an array of
+ * dof_secidx_t's which in turn denote the sections associated with this DIFO.
+ *
+ * This loose coupling of the file structure (header and sections) to the
+ * structure of the DTrace program itself (ECB descriptions, action
+ * descriptions, and DIFOs) permits activities such as relocation processing
+ * to occur in a single pass without having to understand D program structure.
+ *
+ * Finally, strings are always stored in ELF-style string tables along with a
+ * string table section index and string table offset.  Therefore strings in
+ * DOF are always arbitrary-length and not bound to the current implementation.
+ */
+
+#define DOF_ID_SIZE     16      /* total size of dofh_ident[] in bytes */
+
+typedef struct dof_hdr {
+        uint8_t dofh_ident[DOF_ID_SIZE]; /* identification bytes (see below) */
+        uint32_t dofh_flags;            /* file attribute flags (if any) */
+        uint32_t dofh_hdrsize;          /* size of file header in bytes */
+        uint32_t dofh_secsize;          /* size of section header in bytes */
+        uint32_t dofh_secnum;           /* number of section headers */
+        uint64_t dofh_secoff;           /* file offset of section headers */
+        uint64_t dofh_loadsz;           /* file size of loadable portion */
+        uint64_t dofh_filesz;           /* file size of entire DOF file */
+        uint64_t dofh_pad;              /* reserved for future use */
+} dof_hdr_t;
+
+#define DOF_ID_MAG0     0       /* first byte of magic number */
+#define DOF_ID_MAG1     1       /* second byte of magic number */
+#define DOF_ID_MAG2     2       /* third byte of magic number */
+#define DOF_ID_MAG3     3       /* fourth byte of magic number */
+#define DOF_ID_MODEL    4       /* DOF data model (see below) */
+#define DOF_ID_ENCODING 5       /* DOF data encoding (see below) */
+#define DOF_ID_VERSION  6       /* DOF file format major version (see below) */
+#define DOF_ID_DIFVERS  7       /* DIF instruction set version */
+#define DOF_ID_DIFIREG  8       /* DIF integer registers used by compiler */
+#define DOF_ID_DIFTREG  9       /* DIF tuple registers used by compiler */
+#define DOF_ID_PAD      10      /* start of padding bytes (all zeroes) */
+
+#define DOF_MAG_MAG0    0x7F    /* DOF_ID_MAG[0-3] */
+#define DOF_MAG_MAG1    'D'
+#define DOF_MAG_MAG2    'O'
+#define DOF_MAG_MAG3    'F'
+
+#define DOF_MAG_STRING  "\177DOF"
+#define DOF_MAG_STRLEN  4
+
+#define DOF_MODEL_NONE  0       /* DOF_ID_MODEL */
+#define DOF_MODEL_ILP32 1
+#define DOF_MODEL_LP64  2
+
+#ifdef _LP64
+#define DOF_MODEL_NATIVE        DOF_MODEL_LP64
+#else
+#define DOF_MODEL_NATIVE        DOF_MODEL_ILP32
+#endif
+
+#define DOF_ENCODE_NONE 0       /* DOF_ID_ENCODING */
+#define DOF_ENCODE_LSB  1
+#define DOF_ENCODE_MSB  2
+
+#ifdef _BIG_ENDIAN
+#define DOF_ENCODE_NATIVE       DOF_ENCODE_MSB
+#else
+#define DOF_ENCODE_NATIVE       DOF_ENCODE_LSB
+#endif
+
+#define	DOF_VERSION_1	1	/* DOF version 1: Solaris 10 FCS */
+#define	DOF_VERSION_2	2	/* DOF version 2: Solaris Express 6/06 */
+#define	DOF_VERSION_3	3	/* DOF version 3: Minimum version for Leopard */
+#define	DOF_VERSION	DOF_VERSION_3	/* Latest DOF version */
+
+#define DOF_FL_VALID    0       /* mask of all valid dofh_flags bits */
+
+typedef uint32_t dof_secidx_t;  /* section header table index type */
+typedef uint32_t dof_stridx_t;  /* string table index type */
+
+#define DOF_SECIDX_NONE (-1U)   /* null value for section indices */
+#define DOF_STRIDX_NONE (-1U)   /* null value for string indices */
+
+typedef struct dof_sec {
+        uint32_t dofs_type;     /* section type (see below) */
+        uint32_t dofs_align;    /* section data memory alignment */
+        uint32_t dofs_flags;    /* section flags (if any) */
+        uint32_t dofs_entsize;  /* size of section entry (if table) */
+        uint64_t dofs_offset;   /* offset of section data within file */
+        uint64_t dofs_size;     /* size of section data in bytes */
+} dof_sec_t;
+
+#define	DOF_SECT_NONE		0	/* null section */
+#define	DOF_SECT_COMMENTS	1	/* compiler comments */
+#define	DOF_SECT_SOURCE		2	/* D program source code */
+#define	DOF_SECT_ECBDESC	3	/* dof_ecbdesc_t */
+#define	DOF_SECT_PROBEDESC	4	/* dof_probedesc_t */
+#define	DOF_SECT_ACTDESC	5	/* dof_actdesc_t array */
+#define	DOF_SECT_DIFOHDR	6	/* dof_difohdr_t (variable length) */
+#define	DOF_SECT_DIF		7	/* uint32_t array of byte code */
+#define	DOF_SECT_STRTAB		8	/* string table */
+#define	DOF_SECT_VARTAB		9	/* dtrace_difv_t array */
+#define	DOF_SECT_RELTAB		10	/* dof_relodesc_t array */
+#define	DOF_SECT_TYPTAB		11	/* dtrace_diftype_t array */
+#define	DOF_SECT_URELHDR	12	/* dof_relohdr_t (user relocations) */
+#define	DOF_SECT_KRELHDR	13	/* dof_relohdr_t (kernel relocations) */
+#define	DOF_SECT_OPTDESC	14	/* dof_optdesc_t array */
+#define	DOF_SECT_PROVIDER	15	/* dof_provider_t */
+#define	DOF_SECT_PROBES		16	/* dof_probe_t array */
+#define	DOF_SECT_PRARGS		17	/* uint8_t array (probe arg mappings) */
+#define	DOF_SECT_PROFFS		18	/* uint32_t array (probe arg offsets) */
+#define	DOF_SECT_INTTAB		19	/* uint64_t array */
+#define	DOF_SECT_UTSNAME	20	/* struct utsname */
+#define	DOF_SECT_XLTAB		21	/* dof_xlref_t array */
+#define	DOF_SECT_XLMEMBERS	22	/* dof_xlmember_t array */
+#define	DOF_SECT_XLIMPORT	23	/* dof_xlator_t */
+#define	DOF_SECT_XLEXPORT	24	/* dof_xlator_t */
+#define	DOF_SECT_PREXPORT	25	/* dof_secidx_t array (exported objs) */
+#define	DOF_SECT_PRENOFFS	26	/* uint32_t array (enabled offsets) */
+
+#define DOF_SECF_LOAD           1       /* section should be loaded */
+
+typedef struct dof_ecbdesc {
+        dof_secidx_t dofe_probes;       /* link to DOF_SECT_PROBEDESC */
+        dof_secidx_t dofe_pred;         /* link to DOF_SECT_DIFOHDR */
+        dof_secidx_t dofe_actions;      /* link to DOF_SECT_ACTDESC */
+        uint32_t dofe_pad;              /* reserved for future use */
+        uint64_t dofe_uarg;             /* user-supplied library argument */
+} dof_ecbdesc_t;
+
+typedef struct dof_probedesc {
+        dof_secidx_t dofp_strtab;       /* link to DOF_SECT_STRTAB section */
+        dof_stridx_t dofp_provider;     /* provider string */
+        dof_stridx_t dofp_mod;          /* module string */
+        dof_stridx_t dofp_func;         /* function string */
+        dof_stridx_t dofp_name;         /* name string */
+        uint32_t dofp_id;               /* probe identifier (or zero) */
+} dof_probedesc_t;
+
+typedef struct dof_actdesc {
+        dof_secidx_t dofa_difo;         /* link to DOF_SECT_DIFOHDR */
+        dof_secidx_t dofa_strtab;       /* link to DOF_SECT_STRTAB section */
+        uint32_t dofa_kind;             /* action kind (DTRACEACT_* constant) */
+        uint32_t dofa_ntuple;           /* number of subsequent tuple actions */
+        uint64_t dofa_arg;              /* kind-specific argument */
+        uint64_t dofa_uarg;             /* user-supplied argument */
+} dof_actdesc_t;
+
+typedef struct dof_difohdr {
+        dtrace_diftype_t dofd_rtype;    /* return type for this fragment */
+        dof_secidx_t dofd_links[1];     /* variable length array of indices */
+} dof_difohdr_t;
+
+typedef struct dof_relohdr {
+        dof_secidx_t dofr_strtab;       /* link to DOF_SECT_STRTAB for names */
+        dof_secidx_t dofr_relsec;       /* link to DOF_SECT_RELTAB for relos */
+        dof_secidx_t dofr_tgtsec;       /* link to section we are relocating */
+} dof_relohdr_t;
+
+typedef struct dof_relodesc {
+        dof_stridx_t dofr_name;         /* string name of relocation symbol */
+        uint32_t dofr_type;             /* relo type (DOF_RELO_* constant) */
+        uint64_t dofr_offset;           /* byte offset for relocation */
+        uint64_t dofr_data;             /* additional type-specific data */
+} dof_relodesc_t;
+
+#define DOF_RELO_NONE   0               /* empty relocation entry */
+#define DOF_RELO_SETX   1               /* relocate setx value */
+
+typedef struct dof_optdesc {
+        uint32_t dofo_option;           /* option identifier */
+        dof_secidx_t dofo_strtab;       /* string table, if string option */
+        uint64_t dofo_value;            /* option value or string index */
+} dof_optdesc_t;
+
+typedef uint32_t dof_attr_t;            /* encoded stability attributes */
+
+#define DOF_ATTR(n, d, c)       (((n) << 24) | ((d) << 16) | ((c) << 8))
+#define DOF_ATTR_NAME(a)        (((a) >> 24) & 0xff)
+#define DOF_ATTR_DATA(a)        (((a) >> 16) & 0xff)
+#define DOF_ATTR_CLASS(a)       (((a) >>  8) & 0xff)
+
+typedef struct dof_provider {
+	dof_secidx_t dofpv_strtab;	/* link to DOF_SECT_STRTAB section */
+	dof_secidx_t dofpv_probes;	/* link to DOF_SECT_PROBES section */
+	dof_secidx_t dofpv_prargs;	/* link to DOF_SECT_PRARGS section */
+	dof_secidx_t dofpv_proffs;	/* link to DOF_SECT_PROFFS section */
+	dof_stridx_t dofpv_name;	/* provider name string */
+	dof_attr_t dofpv_provattr;	/* provider attributes */
+	dof_attr_t dofpv_modattr;	/* module attributes */
+	dof_attr_t dofpv_funcattr;	/* function attributes */
+	dof_attr_t dofpv_nameattr;	/* name attributes */
+	dof_attr_t dofpv_argsattr;	/* args attributes */
+	dof_secidx_t dofpv_prenoffs;	/* link to DOF_SECT_PRENOFFS section */
+} dof_provider_t;
+
+typedef struct dof_probe {
+	uint64_t dofpr_addr;		/* probe base address or offset */
+	dof_stridx_t dofpr_func;	/* probe function string */
+	dof_stridx_t dofpr_name;	/* probe name string */
+	dof_stridx_t dofpr_nargv;	/* native argument type strings */
+	dof_stridx_t dofpr_xargv;	/* translated argument type strings */
+	uint32_t dofpr_argidx;		/* index of first argument mapping */
+	uint32_t dofpr_offidx;		/* index of first offset entry */
+	uint8_t dofpr_nargc;		/* native argument count */
+	uint8_t dofpr_xargc;		/* translated argument count */
+	uint16_t dofpr_noffs;		/* number of offset entries for probe */
+	uint32_t dofpr_enoffidx;	/* index of first is-enabled offset */
+	uint16_t dofpr_nenoffs;		/* number of is-enabled offsets */
+	uint16_t dofpr_pad1;		/* reserved for future use */
+	uint32_t dofpr_pad2;		/* reserved for future use */
+} dof_probe_t;
+
+typedef struct dof_xlator {
+        dof_secidx_t dofxl_members;     /* link to DOF_SECT_XLMEMBERS section */
+        dof_secidx_t dofxl_strtab;      /* link to DOF_SECT_STRTAB section */
+        dof_stridx_t dofxl_argv;        /* input parameter type strings */
+        uint32_t dofxl_argc;            /* input parameter list length */
+        dof_stridx_t dofxl_type;        /* output type string name */
+        dof_attr_t dofxl_attr;          /* output stability attributes */
+} dof_xlator_t;
+
+typedef struct dof_xlmember {
+        dof_secidx_t dofxm_difo;        /* member link to DOF_SECT_DIFOHDR */
+        dof_stridx_t dofxm_name;        /* member name */
+        dtrace_diftype_t dofxm_type;    /* member type */
+} dof_xlmember_t;
+
+typedef struct dof_xlref {
+        dof_secidx_t dofxr_xlator;      /* link to DOF_SECT_XLATORS section */
+        uint32_t dofxr_member;          /* index of referenced dof_xlmember */
+        uint32_t dofxr_argn;            /* index of argument for DIF_OP_XLARG */
+} dof_xlref_t;
+
+/*
+ * DTrace Intermediate Format Object (DIFO)
+ *
+ * A DIFO is used to store the compiled DIF for a D expression, its return
+ * type, and its string and variable tables.  The string table is a single
+ * buffer of character data into which sets instructions and variable
+ * references can reference strings using a byte offset.  The variable table
+ * is an array of dtrace_difv_t structures that describe the name and type of
+ * each variable and the id used in the DIF code.  This structure is described
+ * above in the DIF section of this header file.  The DIFO is used at both
+ * user-level (in the library) and in the kernel, but the structure is never
+ * passed between the two: the DOF structures form the only interface.  As a
+ * result, the definition can change depending on the presence of _KERNEL.
+ */
+typedef struct dtrace_difo {
+        dif_instr_t *dtdo_buf;          /* instruction buffer */
+        uint64_t *dtdo_inttab;          /* integer table (optional) */
+        char *dtdo_strtab;              /* string table (optional) */
+        dtrace_difv_t *dtdo_vartab;     /* variable table (optional) */
+        uint_t dtdo_len;                /* length of instruction buffer */
+        uint_t dtdo_intlen;             /* length of integer table */
+        uint_t dtdo_strlen;             /* length of string table */
+        uint_t dtdo_varlen;             /* length of variable table */
+        dtrace_diftype_t dtdo_rtype;    /* return type */
+        uint_t dtdo_refcnt;             /* owner reference count */
+        uint_t dtdo_destructive;        /* invokes destructive subroutines */
+#ifndef _KERNEL
+        dof_relodesc_t *dtdo_kreltab;   /* kernel relocations */
+        dof_relodesc_t *dtdo_ureltab;   /* user relocations */
+        struct dt_node **dtdo_xlmtab;   /* translator references */
+        uint_t dtdo_krelen;             /* length of krelo table */
+        uint_t dtdo_urelen;             /* length of urelo table */
+        uint_t dtdo_xlmlen;             /* length of translator table */
+#endif
+} dtrace_difo_t;
+
+/*
+ * DTrace Enabling Description Structures
+ *
+ * When DTrace is tracking the description of a DTrace enabling entity (probe,
+ * predicate, action, ECB, record, etc.), it does so in a description
+ * structure.  These structures all end in "desc", and are used at both
+ * user-level and in the kernel -- but (with the exception of
+ * dtrace_probedesc_t) they are never passed between them.  Typically,
+ * user-level will use the description structures when assembling an enabling.
+ * It will then distill those description structures into a DOF object (see
+ * above), and send it into the kernel.  The kernel will again use the
+ * description structures to create a description of the enabling as it reads
+ * the DOF.  When the description is complete, the enabling will be actually
+ * created -- turning it into the structures that represent the enabling
+ * instead of merely describing it.  Not surprisingly, the description
+ * structures bear a strong resemblance to the DOF structures that act as their
+ * conduit.
+ */
+struct dtrace_predicate;
+
+typedef struct dtrace_probedesc {
+        dtrace_id_t dtpd_id;                    /* probe identifier */
+        char dtpd_provider[DTRACE_PROVNAMELEN]; /* probe provider name */
+        char dtpd_mod[DTRACE_MODNAMELEN];       /* probe module name */
+        char dtpd_func[DTRACE_FUNCNAMELEN];     /* probe function name */
+        char dtpd_name[DTRACE_NAMELEN];         /* probe name */
+} dtrace_probedesc_t;
+
+typedef struct dtrace_repldesc {
+        dtrace_probedesc_t dtrpd_match;         /* probe descr. to match */
+        dtrace_probedesc_t dtrpd_create;        /* probe descr. to create */
+} dtrace_repldesc_t;
+
+typedef struct dtrace_preddesc {
+        dtrace_difo_t *dtpdd_difo;              /* pointer to DIF object */
+        struct dtrace_predicate *dtpdd_predicate; /* pointer to predicate */
+} dtrace_preddesc_t;
+
+typedef struct dtrace_actdesc {
+        dtrace_difo_t *dtad_difo;               /* pointer to DIF object */
+        struct dtrace_actdesc *dtad_next;       /* next action */
+        dtrace_actkind_t dtad_kind;             /* kind of action */
+        uint32_t dtad_ntuple;                   /* number in tuple */
+        uint64_t dtad_arg;                      /* action argument */
+        uint64_t dtad_uarg;                     /* user argument */
+        int dtad_refcnt;                        /* reference count */
+} dtrace_actdesc_t;
+
+typedef struct dtrace_ecbdesc {
+        dtrace_actdesc_t *dted_action;          /* action description(s) */
+        dtrace_preddesc_t dted_pred;            /* predicate description */
+        dtrace_probedesc_t dted_probe;          /* probe description */
+        uint64_t dted_uarg;                     /* library argument */
+        int dted_refcnt;                        /* reference count */
+} dtrace_ecbdesc_t;
+
+/*
+ * DTrace Metadata Description Structures
+ *
+ * DTrace separates the trace data stream from the metadata stream.  The only
+ * metadata tokens placed in the data stream are enabled probe identifiers
+ * (EPIDs) or (in the case of aggregations) aggregation identifiers.  In order
+ * to determine the structure of the data, DTrace consumers pass the token to
+ * the kernel, and receive in return a corresponding description of the enabled
+ * probe (via the dtrace_eprobedesc structure) or the aggregation (via the
+ * dtrace_aggdesc structure).  Both of these structures are expressed in terms
+ * of record descriptions (via the dtrace_recdesc structure) that describe the
+ * exact structure of the data.  Some record descriptions may also contain a
+ * format identifier; this additional bit of metadata can be retrieved from the
+ * kernel, for which a format description is returned via the dtrace_fmtdesc
+ * structure.  Note that all four of these structures must be bitness-neutral
+ * to allow for a 32-bit DTrace consumer on a 64-bit kernel.
+ */
+typedef struct dtrace_recdesc {
+        dtrace_actkind_t dtrd_action;           /* kind of action */
+        uint32_t dtrd_size;                     /* size of record */
+        uint32_t dtrd_offset;                   /* offset in ECB's data */
+        uint16_t dtrd_alignment;                /* required alignment */
+        uint16_t dtrd_format;                   /* format, if any */
+        uint64_t dtrd_arg;                      /* action argument */
+        uint64_t dtrd_uarg;                     /* user argument */
+} dtrace_recdesc_t;
+
+typedef struct dtrace_eprobedesc {
+        dtrace_epid_t dtepd_epid;               /* enabled probe ID */
+        dtrace_id_t dtepd_probeid;              /* probe ID */
+        uint64_t dtepd_uarg;                    /* library argument */
+        uint32_t dtepd_size;                    /* total size */
+        int dtepd_nrecs;                        /* number of records */
+        dtrace_recdesc_t dtepd_rec[1];          /* records themselves */
+} dtrace_eprobedesc_t;
+
+typedef struct dtrace_aggdesc {
+	DTRACE_PTR(char, dtagd_name);		/* not filled in by kernel */
+	dtrace_aggvarid_t dtagd_varid;		/* not filled in by kernel */
+	int dtagd_flags;			/* not filled in by kernel */
+	dtrace_aggid_t dtagd_id;		/* aggregation ID */
+	dtrace_epid_t dtagd_epid;		/* enabled probe ID */
+	uint32_t dtagd_size;			/* size in bytes */
+	int dtagd_nrecs;			/* number of records */
+	uint32_t dtagd_pad;			/* explicit padding */
+	dtrace_recdesc_t dtagd_rec[1];		/* record descriptions */
+} dtrace_aggdesc_t;
+
+typedef struct dtrace_fmtdesc {
+        DTRACE_PTR(char, dtfd_string);          /* format string */
+        int dtfd_length;                        /* length of format string */
+        uint16_t dtfd_format;                   /* format identifier */
+} dtrace_fmtdesc_t;
+
+#define DTRACE_SIZEOF_EPROBEDESC(desc)                          \
+        (sizeof (dtrace_eprobedesc_t) + ((desc)->dtepd_nrecs ?  \
+        (((desc)->dtepd_nrecs - 1) * sizeof (dtrace_recdesc_t)) : 0))
+
+#define DTRACE_SIZEOF_AGGDESC(desc)                             \
+        (sizeof (dtrace_aggdesc_t) + ((desc)->dtagd_nrecs ?     \
+        (((desc)->dtagd_nrecs - 1) * sizeof (dtrace_recdesc_t)) : 0))
+
+/*
+ * DTrace Option Interface
+ *
+ * Run-time DTrace options are set and retrieved via DOF_SECT_OPTDESC sections
+ * in a DOF image.  The dof_optdesc structure contains an option identifier and
+ * an option value.  The valid option identifiers are found below; the mapping
+ * between option identifiers and option identifying strings is maintained at
+ * user-level.  Note that the value of DTRACEOPT_UNSET is such that all of the
+ * following are potentially valid option values:  all positive integers, zero
+ * and negative one.  Some options (notably "bufpolicy" and "bufresize") take
+ * predefined tokens as their values; these are defined with
+ * DTRACEOPT_{option}_{token}.
+ */
+#define	DTRACEOPT_BUFSIZE	0	/* buffer size */
+#define	DTRACEOPT_BUFPOLICY	1	/* buffer policy */
+#define	DTRACEOPT_DYNVARSIZE	2	/* dynamic variable size */
+#define	DTRACEOPT_AGGSIZE	3	/* aggregation size */
+#define	DTRACEOPT_SPECSIZE	4	/* speculation size */
+#define	DTRACEOPT_NSPEC		5	/* number of speculations */
+#define	DTRACEOPT_STRSIZE	6	/* string size */
+#define	DTRACEOPT_CLEANRATE	7	/* dynvar cleaning rate */
+#define	DTRACEOPT_CPU		8	/* CPU to trace */
+#define	DTRACEOPT_BUFRESIZE	9	/* buffer resizing policy */
+#define	DTRACEOPT_GRABANON	10	/* grab anonymous state, if any */
+#define	DTRACEOPT_FLOWINDENT	11	/* indent function entry/return */
+#define	DTRACEOPT_QUIET		12	/* only output explicitly traced data */
+#define	DTRACEOPT_STACKFRAMES	13	/* number of stack frames */
+#define	DTRACEOPT_USTACKFRAMES	14	/* number of user stack frames */
+#define	DTRACEOPT_AGGRATE	15	/* aggregation snapshot rate */
+#define	DTRACEOPT_SWITCHRATE	16	/* buffer switching rate */
+#define	DTRACEOPT_STATUSRATE	17	/* status rate */
+#define	DTRACEOPT_DESTRUCTIVE	18	/* destructive actions allowed */
+#define	DTRACEOPT_STACKINDENT	19	/* output indent for stack traces */
+#define	DTRACEOPT_RAWBYTES	20	/* always print bytes in raw form */
+#define	DTRACEOPT_JSTACKFRAMES	21	/* number of jstack() frames */
+#define	DTRACEOPT_JSTACKSTRSIZE	22	/* size of jstack() string table */
+#define	DTRACEOPT_AGGSORTKEY	23	/* sort aggregations by key */
+#define	DTRACEOPT_AGGSORTREV	24	/* reverse-sort aggregations */
+#define	DTRACEOPT_AGGSORTPOS	25	/* agg. position to sort on */
+#define	DTRACEOPT_AGGSORTKEYPOS	26	/* agg. key position to sort on */
+#if defined(__APPLE__)
+#define DTRACEOPT_STACKSYMBOLS  27      /* clear to prevent stack symbolication */
+#define DTRACEOPT_MAX           28      /* number of options */
+#else
+#define DTRACEOPT_MAX           27      /* number of options */
+#endif
+
+#define	DTRACEOPT_UNSET		(dtrace_optval_t)-2	/* unset option */
+
+#define	DTRACEOPT_BUFPOLICY_RING	0	/* ring buffer */
+#define	DTRACEOPT_BUFPOLICY_FILL	1	/* fill buffer, then stop */
+#define	DTRACEOPT_BUFPOLICY_SWITCH	2	/* switch buffers */
+
+#define DTRACEOPT_BUFRESIZE_AUTO        0       /* automatic resizing */
+#define DTRACEOPT_BUFRESIZE_MANUAL      1       /* manual resizing */
+
+/*
+ * DTrace Buffer Interface
+ *
+ * In order to get a snapshot of the principal or aggregation buffer,
+ * user-level passes a buffer description to the kernel with the dtrace_bufdesc
+ * structure.  This describes which CPU user-level is interested in, and
+ * where user-level wishes the kernel to snapshot the buffer to (the
+ * dtbd_data field).  The kernel uses the same structure to pass back some
+ * information regarding the buffer:  the size of data actually copied out, the
+ * number of drops, the number of errors, and the offset of the oldest record.
+ * If the buffer policy is a "switch" policy, taking a snapshot of the
+ * principal buffer has the additional effect of switching the active and
+ * inactive buffers.  Taking a snapshot of the aggregation buffer _always_ has
+ * the additional effect of switching the active and inactive buffers.
+ */
+typedef struct dtrace_bufdesc {
+        uint64_t dtbd_size;                     /* size of buffer */
+        uint32_t dtbd_cpu;                      /* CPU or DTRACE_CPUALL */
+        uint32_t dtbd_errors;                   /* number of errors */
+        uint64_t dtbd_drops;                    /* number of drops */
+        DTRACE_PTR(char, dtbd_data);            /* data */
+        uint64_t dtbd_oldest;                   /* offset of oldest record */
+} dtrace_bufdesc_t;
+
+/*
+ * DTrace Status
+ *
+ * The status of DTrace is relayed via the dtrace_status structure.  This
+ * structure contains members to count drops other than the capacity drops
+ * available via the buffer interface (see above).  This consists of dynamic
+ * drops (including capacity dynamic drops, rinsing drops and dirty drops), and
+ * speculative drops (including capacity speculative drops, drops due to busy
+ * speculative buffers and drops due to unavailable speculative buffers).
+ * Additionally, the status structure contains a field to indicate the number
+ * of "fill"-policy buffers have been filled and a boolean field to indicate
+ * that exit() has been called.  If the dtst_exiting field is non-zero, no
+ * further data will be generated until tracing is stopped (at which time any
+ * enablings of the END action will be processed); if user-level sees that
+ * this field is non-zero, tracing should be stopped as soon as possible.
+ */
+typedef struct dtrace_status {
+        uint64_t dtst_dyndrops;                 /* dynamic drops */
+        uint64_t dtst_dyndrops_rinsing;         /* dyn drops due to rinsing */
+        uint64_t dtst_dyndrops_dirty;           /* dyn drops due to dirty */
+        uint64_t dtst_specdrops;                /* speculative drops */
+        uint64_t dtst_specdrops_busy;           /* spec drops due to busy */
+        uint64_t dtst_specdrops_unavail;        /* spec drops due to unavail */
+        uint64_t dtst_errors;                   /* total errors */
+        uint64_t dtst_filled;                   /* number of filled bufs */
+        uint64_t dtst_stkstroverflows;          /* stack string tab overflows */
+        uint64_t dtst_dblerrors;                /* errors in ERROR probes */
+        char dtst_killed;                       /* non-zero if killed */
+        char dtst_exiting;                      /* non-zero if exit() called */
+        char dtst_pad[6];                       /* pad out to 64-bit align */
+} dtrace_status_t;
+
+/*
+ * DTrace Configuration
+ *
+ * User-level may need to understand some elements of the kernel DTrace
+ * configuration in order to generate correct DIF.  This information is
+ * conveyed via the dtrace_conf structure.
+ */
+typedef struct dtrace_conf {
+        uint_t dtc_difversion;                  /* supported DIF version */
+        uint_t dtc_difintregs;                  /* # of DIF integer registers */
+        uint_t dtc_diftupregs;                  /* # of DIF tuple registers */
+        uint_t dtc_ctfmodel;                    /* CTF data model */
+        uint_t dtc_pad[8];                      /* reserved for future use */
+} dtrace_conf_t;
+
+/*
+ * DTrace Faults
+ *
+ * The constants below DTRACEFLT_LIBRARY indicate probe processing faults;
+ * constants at or above DTRACEFLT_LIBRARY indicate faults in probe
+ * postprocessing at user-level.  Probe processing faults induce an ERROR
+ * probe and are replicated in unistd.d to allow users' ERROR probes to decode
+ * the error condition using thse symbolic labels.
+ */
+#define DTRACEFLT_UNKNOWN               0       /* Unknown fault */
+#define DTRACEFLT_BADADDR               1       /* Bad address */
+#define DTRACEFLT_BADALIGN              2       /* Bad alignment */
+#define DTRACEFLT_ILLOP                 3       /* Illegal operation */
+#define DTRACEFLT_DIVZERO               4       /* Divide-by-zero */
+#define DTRACEFLT_NOSCRATCH             5       /* Out of scratch space */
+#define DTRACEFLT_KPRIV                 6       /* Illegal kernel access */
+#define DTRACEFLT_UPRIV                 7       /* Illegal user access */
+#define DTRACEFLT_TUPOFLOW              8       /* Tuple stack overflow */
+
+#define DTRACEFLT_LIBRARY               1000    /* Library-level fault */
+
+/*
+ * DTrace Argument Types
+ *
+ * Because it would waste both space and time, argument types do not reside
+ * with the probe.  In order to determine argument types for args[X]
+ * variables, the D compiler queries for argument types on a probe-by-probe
+ * basis.  (This optimizes for the common case that arguments are either not
+ * used or used in an untyped fashion.)  Typed arguments are specified with a
+ * string of the type name in the dtragd_native member of the argument
+ * description structure.  Typed arguments may be further translated to types
+ * of greater stability; the provider indicates such a translated argument by
+ * filling in the dtargd_xlate member with the string of the translated type.
+ * Finally, the provider may indicate which argument value a given argument
+ * maps to by setting the dtargd_mapping member -- allowing a single argument
+ * to map to multiple args[X] variables.
+ */
+typedef struct dtrace_argdesc {
+        dtrace_id_t dtargd_id;                  /* probe identifier */
+        int dtargd_ndx;                         /* arg number (-1 iff none) */
+        int dtargd_mapping;                     /* value mapping */
+        char dtargd_native[DTRACE_ARGTYPELEN];  /* native type name */
+        char dtargd_xlate[DTRACE_ARGTYPELEN];   /* translated type name */
+} dtrace_argdesc_t;
+
+/*
+ * DTrace Stability Attributes
+ *
+ * Each DTrace provider advertises the name and data stability of each of its
+ * probe description components, as well as its architectural dependencies.
+ * The D compiler can query the provider attributes (dtrace_pattr_t below) in
+ * order to compute the properties of an input program and report them.
+ */
+typedef uint8_t dtrace_stability_t;     /* stability code (see attributes(5)) */
+typedef uint8_t dtrace_class_t;         /* architectural dependency class */
+
+#define	DTRACE_STABILITY_INTERNAL	0	/* private to DTrace itself */
+#define	DTRACE_STABILITY_PRIVATE	1	/* private to Sun (see docs) */
+#define	DTRACE_STABILITY_OBSOLETE	2	/* scheduled for removal */
+#define	DTRACE_STABILITY_EXTERNAL	3	/* not controlled by Sun */
+#define	DTRACE_STABILITY_UNSTABLE	4	/* new or rapidly changing */
+#define	DTRACE_STABILITY_EVOLVING	5	/* less rapidly changing */
+#define	DTRACE_STABILITY_STABLE		6	/* mature interface from Sun */
+#define	DTRACE_STABILITY_STANDARD	7	/* industry standard */
+#define	DTRACE_STABILITY_MAX		7	/* maximum valid stability */
+
+#define	DTRACE_CLASS_UNKNOWN	0	/* unknown architectural dependency */
+#define	DTRACE_CLASS_CPU	1	/* CPU-module-specific */
+#define	DTRACE_CLASS_PLATFORM	2	/* platform-specific (uname -i) */
+#define	DTRACE_CLASS_GROUP	3	/* hardware-group-specific (uname -m) */
+#define	DTRACE_CLASS_ISA	4	/* ISA-specific (uname -p) */
+#define	DTRACE_CLASS_COMMON	5	/* common to all systems */
+#define	DTRACE_CLASS_MAX	5	/* maximum valid class */
+
+#define	DTRACE_PRIV_NONE	0x0000
+#define	DTRACE_PRIV_KERNEL	0x0001
+#define	DTRACE_PRIV_USER	0x0002
+#define	DTRACE_PRIV_PROC	0x0004
+#define	DTRACE_PRIV_OWNER	0x0008
+#define	DTRACE_PRIV_ZONEOWNER	0x0010
+
+#define	DTRACE_PRIV_ALL	\
+	(DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER | \
+	DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER)
+
+typedef struct dtrace_ppriv {
+	uint32_t dtpp_flags;			/* privilege flags */
+	uid_t dtpp_uid;				/* user ID */
+	zoneid_t dtpp_zoneid;			/* zone ID */
+} dtrace_ppriv_t;
+
+typedef struct dtrace_attribute {
+        dtrace_stability_t dtat_name;           /* entity name stability */
+        dtrace_stability_t dtat_data;           /* entity data stability */
+        dtrace_class_t dtat_class;              /* entity data dependency */
+} dtrace_attribute_t;
+
+typedef struct dtrace_pattr {
+        dtrace_attribute_t dtpa_provider;       /* provider attributes */
+        dtrace_attribute_t dtpa_mod;            /* module attributes */
+        dtrace_attribute_t dtpa_func;           /* function attributes */
+        dtrace_attribute_t dtpa_name;           /* name attributes */
+        dtrace_attribute_t dtpa_args;           /* args[] attributes */
+} dtrace_pattr_t;
+
+typedef struct dtrace_providerdesc {
+        char dtvd_name[DTRACE_PROVNAMELEN];     /* provider name */
+        dtrace_pattr_t dtvd_attr;               /* stability attributes */
+        dtrace_ppriv_t dtvd_priv;               /* privileges required */
+} dtrace_providerdesc_t;
+
+/*
+ * DTrace Pseudodevice Interface
+ *
+ * DTrace is controlled through ioctl(2)'s to the in-kernel dtrace:dtrace
+ * pseudodevice driver.  These ioctls comprise the user-kernel interface to
+ * DTrace.
+ */
+#if !defined(__APPLE__)
+#define DTRACEIOC               (('d' << 24) | ('t' << 16) | ('r' << 8))
+#define DTRACEIOC_PROVIDER      (DTRACEIOC | 1)         /* provider query */
+#define DTRACEIOC_PROBES        (DTRACEIOC | 2)         /* probe query */
+#define DTRACEIOC_BUFSNAP       (DTRACEIOC | 4)         /* snapshot buffer */
+#define DTRACEIOC_PROBEMATCH    (DTRACEIOC | 5)         /* match probes */
+#define DTRACEIOC_ENABLE        (DTRACEIOC | 6)         /* enable probes */
+#define DTRACEIOC_AGGSNAP       (DTRACEIOC | 7)         /* snapshot agg. */
+#define DTRACEIOC_EPROBE        (DTRACEIOC | 8)         /* get eprobe desc. */
+#define DTRACEIOC_PROBEARG      (DTRACEIOC | 9)         /* get probe arg */
+#define DTRACEIOC_CONF          (DTRACEIOC | 10)        /* get config. */
+#define DTRACEIOC_STATUS        (DTRACEIOC | 11)        /* get status */
+#define DTRACEIOC_GO            (DTRACEIOC | 12)        /* start tracing */
+#define DTRACEIOC_STOP          (DTRACEIOC | 13)        /* stop tracing */
+#define DTRACEIOC_AGGDESC       (DTRACEIOC | 15)        /* get agg. desc. */
+#define DTRACEIOC_FORMAT        (DTRACEIOC | 16)        /* get format str */
+#define DTRACEIOC_DOFGET        (DTRACEIOC | 17)        /* get DOF */
+#define DTRACEIOC_REPLICATE     (DTRACEIOC | 18)        /* replicate enab */
+#else
+/* coding this as IOC_VOID allows this driver to handle its own copyin/copuout */
+#define DTRACEIOC               _IO('d',0)
+#define DTRACEIOC_PROVIDER      (DTRACEIOC | 1)         /* provider query */
+#define DTRACEIOC_PROBES        (DTRACEIOC | 2)         /* probe query */
+#define DTRACEIOC_BUFSNAP       (DTRACEIOC | 4)         /* snapshot buffer */
+#define DTRACEIOC_PROBEMATCH    (DTRACEIOC | 5)         /* match probes */
+#define DTRACEIOC_ENABLE        (DTRACEIOC | 6)         /* enable probes */
+#define DTRACEIOC_AGGSNAP       (DTRACEIOC | 7)         /* snapshot agg. */
+#define DTRACEIOC_EPROBE        (DTRACEIOC | 8)         /* get eprobe desc. */
+#define DTRACEIOC_PROBEARG      (DTRACEIOC | 9)         /* get probe arg */
+#define DTRACEIOC_CONF          (DTRACEIOC | 10)        /* get config. */
+#define DTRACEIOC_STATUS        (DTRACEIOC | 11)        /* get status */
+#define DTRACEIOC_GO            (DTRACEIOC | 12)        /* start tracing */
+#define DTRACEIOC_STOP          (DTRACEIOC | 13)        /* stop tracing */
+#define DTRACEIOC_AGGDESC       (DTRACEIOC | 15)        /* get agg. desc. */
+#define DTRACEIOC_FORMAT        (DTRACEIOC | 16)        /* get format str */
+#define DTRACEIOC_DOFGET        (DTRACEIOC | 17)        /* get DOF */
+#define DTRACEIOC_REPLICATE     (DTRACEIOC | 18)        /* replicate enab */
+#endif /* __APPLE__ */
+
+/*
+ * DTrace Helpers
+ *
+ * In general, DTrace establishes probes in processes and takes actions on
+ * processes without knowing their specific user-level structures.  Instead of
+ * existing in the framework, process-specific knowledge is contained by the
+ * enabling D program -- which can apply process-specific knowledge by making
+ * appropriate use of DTrace primitives like copyin() and copyinstr() to
+ * operate on user-level data.  However, there may exist some specific probes
+ * of particular semantic relevance that the application developer may wish to
+ * explicitly export.  For example, an application may wish to export a probe
+ * at the point that it begins and ends certain well-defined transactions.  In
+ * addition to providing probes, programs may wish to offer assistance for
+ * certain actions.  For example, in highly dynamic environments (e.g., Java),
+ * it may be difficult to obtain a stack trace in terms of meaningful symbol
+ * names (the translation from instruction addresses to corresponding symbol
+ * names may only be possible in situ); these environments may wish to define
+ * a series of actions to be applied in situ to obtain a meaningful stack
+ * trace.
+ *
+ * These two mechanisms -- user-level statically defined tracing and assisting
+ * DTrace actions -- are provided via DTrace _helpers_.  Helpers are specified
+ * via DOF, but unlike enabling DOF, helper DOF may contain definitions of
+ * providers, probes and their arguments.  If a helper wishes to provide
+ * action assistance, probe descriptions and corresponding DIF actions may be
+ * specified in the helper DOF.  For such helper actions, however, the probe
+ * description describes the specific helper:  all DTrace helpers have the
+ * provider name "dtrace" and the module name "helper", and the name of the
+ * helper is contained in the function name (for example, the ustack() helper
+ * is named "ustack").  Any helper-specific name may be contained in the name
+ * (for example, if a helper were to have a constructor, it might be named
+ * "dtrace:helper:<helper>:init").  Helper actions are only called when the
+ * action that they are helping is taken.  Helper actions may only return DIF
+ * expressions, and may only call the following subroutines:
+ *
+ *    alloca()      <= Allocates memory out of the consumer's scratch space
+ *    bcopy()       <= Copies memory to scratch space
+ *    copyin()      <= Copies memory from user-level into consumer's scratch
+ *    copyinto()    <= Copies memory into a specific location in scratch
+ *    copyinstr()   <= Copies a string into a specific location in scratch
+ *
+ * Helper actions may only access the following built-in variables:
+ *
+ *    curthread     <= Current kthread_t pointer
+ *    tid           <= Current thread identifier
+ *    pid           <= Current process identifier
+ *    ppid          <= Parent process identifier
+ *    uid           <= Current user ID
+ *    gid           <= Current group ID
+ *    execname      <= Current executable name
+ *    zonename      <= Current zone name
+ *
+ * Helper actions may not manipulate or allocate dynamic variables, but they
+ * may have clause-local and statically-allocated global variables.  The
+ * helper action variable state is specific to the helper action -- variables
+ * used by the helper action may not be accessed outside of the helper
+ * action, and the helper action may not access variables that like outside
+ * of it.  Helper actions may not load from kernel memory at-large; they are
+ * restricting to loading current user state (via copyin() and variants) and
+ * scratch space.  As with probe enablings, helper actions are executed in
+ * program order.  The result of the helper action is the result of the last
+ * executing helper expression.
+ *
+ * Helpers -- composed of either providers/probes or probes/actions (or both)
+ * -- are added by opening the "helper" minor node, and issuing an ioctl(2)
+ * (DTRACEHIOC_ADDDOF) that specifies the dof_helper_t structure. This
+ * encapsulates the name and base address of the user-level library or
+ * executable publishing the helpers and probes as well as the DOF that
+ * contains the definitions of those helpers and probes.
+ *
+ * The DTRACEHIOC_ADD and DTRACEHIOC_REMOVE are left in place for legacy
+ * helpers and should no longer be used.  No other ioctls are valid on the
+ * helper minor node.
+ */
+#if !defined(__APPLE__)
+#define DTRACEHIOC              (('d' << 24) | ('t' << 16) | ('h' << 8))
+#define DTRACEHIOC_ADD          (DTRACEHIOC | 1)        /* add helper */
+#define DTRACEHIOC_REMOVE       (DTRACEHIOC | 2)        /* remove helper */
+#define DTRACEHIOC_ADDDOF       (DTRACEHIOC | 3)        /* add helper DOF */
+#else
+#define DTRACEHIOC_REMOVE	_IO('h', 2)			/* remove helper */
+#define DTRACEHIOC_ADDDOF	_IOW('h', 4, user_addr_t)	/* add helper DOF */
+#endif /* __APPLE__ */
+
+typedef struct dof_helper {
+        char dofhp_mod[DTRACE_MODNAMELEN];      /* executable or library name */
+        uint64_t dofhp_addr;                    /* base address of object */
+        uint64_t dofhp_dof;                     /* address of helper DOF */
+} dof_helper_t;
+
+#if defined(__APPLE__)
+/*
+ * This structure is used to register one or more dof_helper_t(s).
+ * For counts greater than one, malloc the structure as if the
+ * dofiod_helpers field was "count" sized. The kernel will copyin
+ * data of size:
+ *
+ * sizeof(dof_ioctl_data_t) + ((count - 1) * sizeof(dof_helper_t))
+ */
+typedef struct dof_ioctl_data {
+	/*
+	 * This field must be 64 bits to keep the alignment the same
+	 * when 64 bit user procs are sending data to 32 bit xnu
+	 */
+	uint64_t dofiod_count;
+	dof_helper_t dofiod_helpers[1];
+} dof_ioctl_data_t;
+
+#define DOF_IOCTL_DATA_T_SIZE(count) (sizeof(dof_ioctl_data_t) + ((count - 1) * sizeof(dof_helper_t)))
+
+#endif
+
+#define DTRACEMNR_DTRACE        "dtrace"        /* node for DTrace ops */
+#define DTRACEMNR_HELPER        "dtracehelper"  /* node for helpers */
+#define DTRACEMNRN_DTRACE       0               /* minor for DTrace ops */
+#define DTRACEMNRN_HELPER       1               /* minor for helpers */
+#define DTRACEMNRN_CLONE        2               /* first clone minor */
+
+#ifdef _KERNEL
+
+/*
+ * DTrace Provider API
+ *
+ * The following functions are implemented by the DTrace framework and are
+ * used to implement separate in-kernel DTrace providers.  Common functions
+ * are provided in uts/common/os/dtrace.c.  ISA-dependent subroutines are
+ * defined in uts/<isa>/dtrace/dtrace_asm.s or uts/<isa>/dtrace/dtrace_isa.c.
+ *
+ * The provider API has two halves:  the API that the providers consume from
+ * DTrace, and the API that providers make available to DTrace.
+ *
+ * 1 Framework-to-Provider API
+ *
+ * 1.1  Overview
+ *
+ * The Framework-to-Provider API is represented by the dtrace_pops structure
+ * that the provider passes to the framework when registering itself.  This
+ * structure consists of the following members:
+ *
+ *   dtps_provide()          <-- Provide all probes, all modules
+ *   dtps_provide_module()   <-- Provide all probes in specified module
+ *   dtps_enable()           <-- Enable specified probe
+ *   dtps_disable()          <-- Disable specified probe
+ *   dtps_suspend()          <-- Suspend specified probe
+ *   dtps_resume()           <-- Resume specified probe
+ *   dtps_getargdesc()       <-- Get the argument description for args[X]
+ *   dtps_getargval()        <-- Get the value for an argX or args[X] variable
+ *   dtps_usermode()         <-- Find out if the probe was fired in user mode
+ *   dtps_destroy()          <-- Destroy all state associated with this probe
+ *
+ * 1.2  void dtps_provide(void *arg, const dtrace_probedesc_t *spec)
+ *
+ * 1.2.1  Overview
+ *
+ *   Called to indicate that the provider should provide all probes.  If the
+ *   specified description is non-NULL, dtps_provide() is being called because
+ *   no probe matched a specified probe -- if the provider has the ability to
+ *   create custom probes, it may wish to create a probe that matches the
+ *   specified description.
+ *
+ * 1.2.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is a pointer to a probe description that the provider may
+ *   wish to consider when creating custom probes.  The provider is expected to
+ *   call back into the DTrace framework via dtrace_probe_create() to create
+ *   any necessary probes.  dtps_provide() may be called even if the provider
+ *   has made available all probes; the provider should check the return value
+ *   of dtrace_probe_create() to handle this case.  Note that the provider need
+ *   not implement both dtps_provide() and dtps_provide_module(); see
+ *   "Arguments and Notes" for dtrace_register(), below.
+ *
+ * 1.2.3  Return value
+ *
+ *   None.
+ *
+ * 1.2.4  Caller's context
+ *
+ *   dtps_provide() is typically called from open() or ioctl() context, but may
+ *   be called from other contexts as well.  The DTrace framework is locked in
+ *   such a way that providers may not register or unregister.  This means that
+ *   the provider may not call any DTrace API that affects its registration with
+ *   the framework, including dtrace_register(), dtrace_unregister(),
+ *   dtrace_invalidate(), and dtrace_condense().  However, the context is such
+ *   that the provider may (and indeed, is expected to) call probe-related
+ *   DTrace routines, including dtrace_probe_create(), dtrace_probe_lookup(),
+ *   and dtrace_probe_arg().
+ *
+ * 1.3  void dtps_provide_module(void *arg, struct modctl *mp)
+ *
+ * 1.3.1  Overview
+ *
+ *   Called to indicate that the provider should provide all probes in the
+ *   specified module.
+ *
+ * 1.3.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is a pointer to a modctl structure that indicates the
+ *   module for which probes should be created.
+ *
+ * 1.3.3  Return value
+ *
+ *   None.
+ *
+ * 1.3.4  Caller's context
+ *
+ *   dtps_provide_module() may be called from open() or ioctl() context, but
+ *   may also be called from a module loading context.  mod_lock is held, and
+ *   the DTrace framework is locked in such a way that providers may not
+ *   register or unregister.  This means that the provider may not call any
+ *   DTrace API that affects its registration with the framework, including
+ *   dtrace_register(), dtrace_unregister(), dtrace_invalidate(), and
+ *   dtrace_condense().  However, the context is such that the provider may (and
+ *   indeed, is expected to) call probe-related DTrace routines, including
+ *   dtrace_probe_create(), dtrace_probe_lookup(), and dtrace_probe_arg().  Note
+ *   that the provider need not implement both dtps_provide() and
+ *   dtps_provide_module(); see "Arguments and Notes" for dtrace_register(),
+ *   below.
+ *
+ * 1.4  void dtps_enable(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.4.1  Overview
+ *
+ *   Called to enable the specified probe.
+ *
+ * 1.4.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is the identifier of the probe to be enabled.  The third
+ *   argument is the probe argument as passed to dtrace_probe_create().
+ *   dtps_enable() will be called when a probe transitions from not being
+ *   enabled at all to having one or more ECB.  The number of ECBs associated
+ *   with the probe may change without subsequent calls into the provider.
+ *   When the number of ECBs drops to zero, the provider will be explicitly
+ *   told to disable the probe via dtps_disable().  dtrace_probe() should never
+ *   be called for a probe identifier that hasn't been explicitly enabled via
+ *   dtps_enable().
+ *
+ * 1.4.3  Return value
+ *
+ *   None.
+ *
+ * 1.4.4  Caller's context
+ *
+ *   The DTrace framework is locked in such a way that it may not be called
+ *   back into at all.  cpu_lock is held.  mod_lock is not held and may not
+ *   be acquired.
+ *
+ * 1.5  void dtps_disable(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.5.1  Overview
+ *
+ *   Called to disable the specified probe.
+ *
+ * 1.5.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is the identifier of the probe to be disabled.  The third
+ *   argument is the probe argument as passed to dtrace_probe_create().
+ *   dtps_disable() will be called when a probe transitions from being enabled
+ *   to having zero ECBs.  dtrace_probe() should never be called for a probe
+ *   identifier that has been explicitly enabled via dtps_disable().
+ *
+ * 1.5.3  Return value
+ *
+ *   None.
+ *
+ * 1.5.4  Caller's context
+ *
+ *   The DTrace framework is locked in such a way that it may not be called
+ *   back into at all.  cpu_lock is held.  mod_lock is not held and may not
+ *   be acquired.
+ *
+ * 1.6  void dtps_suspend(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.6.1  Overview
+ *
+ *   Called to suspend the specified enabled probe.  This entry point is for
+ *   providers that may need to suspend some or all of their probes when CPUs
+ *   are being powered on or when the boot monitor is being entered for a
+ *   prolonged period of time.
+ *
+ * 1.6.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is the identifier of the probe to be suspended.  The
+ *   third argument is the probe argument as passed to dtrace_probe_create().
+ *   dtps_suspend will only be called on an enabled probe.  Providers that
+ *   provide a dtps_suspend entry point will want to take roughly the action
+ *   that it takes for dtps_disable.
+ *
+ * 1.6.3  Return value
+ *
+ *   None.
+ *
+ * 1.6.4  Caller's context
+ *
+ *   Interrupts are disabled.  The DTrace framework is in a state such that the
+ *   specified probe cannot be disabled or destroyed for the duration of
+ *   dtps_suspend().  As interrupts are disabled, the provider is afforded
+ *   little latitude; the provider is expected to do no more than a store to
+ *   memory.
+ *
+ * 1.7  void dtps_resume(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.7.1  Overview
+ *
+ *   Called to resume the specified enabled probe.  This entry point is for
+ *   providers that may need to resume some or all of their probes after the
+ *   completion of an event that induced a call to dtps_suspend().
+ *
+ * 1.7.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is the identifier of the probe to be resumed.  The
+ *   third argument is the probe argument as passed to dtrace_probe_create().
+ *   dtps_resume will only be called on an enabled probe.  Providers that
+ *   provide a dtps_resume entry point will want to take roughly the action
+ *   that it takes for dtps_enable.
+ *
+ * 1.7.3  Return value
+ *
+ *   None.
+ *
+ * 1.7.4  Caller's context
+ *
+ *   Interrupts are disabled.  The DTrace framework is in a state such that the
+ *   specified probe cannot be disabled or destroyed for the duration of
+ *   dtps_resume().  As interrupts are disabled, the provider is afforded
+ *   little latitude; the provider is expected to do no more than a store to
+ *   memory.
+ *
+ * 1.8  void dtps_getargdesc(void *arg, dtrace_id_t id, void *parg,
+ *           dtrace_argdesc_t *desc)
+ *
+ * 1.8.1  Overview
+ *
+ *   Called to retrieve the argument description for an args[X] variable.
+ *
+ * 1.8.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register(). The
+ *   second argument is the identifier of the current probe. The third
+ *   argument is the probe argument as passed to dtrace_probe_create(). The
+ *   fourth argument is a pointer to the argument description.  This
+ *   description is both an input and output parameter:  it contains the
+ *   index of the desired argument in the dtargd_ndx field, and expects
+ *   the other fields to be filled in upon return.  If there is no argument
+ *   corresponding to the specified index, the dtargd_ndx field should be set
+ *   to DTRACE_ARGNONE.
+ *
+ * 1.8.3  Return value
+ *
+ *   None.  The dtargd_ndx, dtargd_native, dtargd_xlate and dtargd_mapping
+ *   members of the dtrace_argdesc_t structure are all output values.
+ *
+ * 1.8.4  Caller's context
+ *
+ *   dtps_getargdesc() is called from ioctl() context. mod_lock is held, and
+ *   the DTrace framework is locked in such a way that providers may not
+ *   register or unregister.  This means that the provider may not call any
+ *   DTrace API that affects its registration with the framework, including
+ *   dtrace_register(), dtrace_unregister(), dtrace_invalidate(), and
+ *   dtrace_condense().
+ *
+ * 1.9  uint64_t dtps_getargval(void *arg, dtrace_id_t id, void *parg,
+ *               int argno, int aframes)
+ *
+ * 1.9.1  Overview
+ *
+ *   Called to retrieve a value for an argX or args[X] variable.
+ *
+ * 1.9.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register(). The
+ *   second argument is the identifier of the current probe. The third
+ *   argument is the probe argument as passed to dtrace_probe_create(). The
+ *   fourth argument is the number of the argument (the X in the example in
+ *   1.9.1). The fifth argument is the number of stack frames that were used
+ *   to get from the actual place in the code that fired the probe to
+ *   dtrace_probe() itself, the so-called artificial frames. This argument may
+ *   be used to descend an appropriate number of frames to find the correct
+ *   values. If this entry point is left NULL, the dtrace_getarg() built-in
+ *   function is used.
+ *
+ * 1.9.3  Return value
+ *
+ *   The value of the argument.
+ *
+ * 1.9.4  Caller's context
+ *
+ *   This is called from within dtrace_probe() meaning that interrupts
+ *   are disabled. No locks should be taken within this entry point.
+ *
+ * 1.10  int dtps_usermode(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.10.1  Overview
+ *
+ *   Called to determine if the probe was fired in a user context.
+ *
+ * 1.10.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register(). The
+ *   second argument is the identifier of the current probe. The third
+ *   argument is the probe argument as passed to dtrace_probe_create().  This
+ *   entry point must not be left NULL for providers whose probes allow for
+ *   mixed mode tracing, that is to say those probes that can fire during
+ *   kernel- _or_ user-mode execution
+ *
+ * 1.10.3  Return value
+ *
+ *   A boolean value.
+ *
+ * 1.10.4  Caller's context
+ *
+ *   This is called from within dtrace_probe() meaning that interrupts
+ *   are disabled. No locks should be taken within this entry point.
+ *
+ * 1.11 void dtps_destroy(void *arg, dtrace_id_t id, void *parg)
+ *
+ * 1.11.1 Overview
+ *
+ *   Called to destroy the specified probe.
+ *
+ * 1.11.2 Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_register().  The
+ *   second argument is the identifier of the probe to be destroyed.  The third
+ *   argument is the probe argument as passed to dtrace_probe_create().  The
+ *   provider should free all state associated with the probe.  The framework
+ *   guarantees that dtps_destroy() is only called for probes that have either
+ *   been disabled via dtps_disable() or were never enabled via dtps_enable().
+ *   Once dtps_disable() has been called for a probe, no further call will be
+ *   made specifying the probe.
+ *
+ * 1.11.3 Return value
+ *
+ *   None.
+ *
+ * 1.11.4 Caller's context
+ *
+ *   The DTrace framework is locked in such a way that it may not be called
+ *   back into at all.  mod_lock is held.  cpu_lock is not held, and may not be
+ *   acquired.
+ *
+ *
+ * 2 Provider-to-Framework API
+ *
+ * 2.1  Overview
+ *
+ * The Provider-to-Framework API provides the mechanism for the provider to
+ * register itself with the DTrace framework, to create probes, to lookup
+ * probes and (most importantly) to fire probes.  The Provider-to-Framework
+ * consists of:
+ *
+ *   dtrace_register()       <-- Register a provider with the DTrace framework
+ *   dtrace_unregister()     <-- Remove a provider's DTrace registration
+ *   dtrace_invalidate()     <-- Invalidate the specified provider
+ *   dtrace_condense()       <-- Remove a provider's unenabled probes
+ *   dtrace_attached()       <-- Indicates whether or not DTrace has attached
+ *   dtrace_probe_create()   <-- Create a DTrace probe
+ *   dtrace_probe_lookup()   <-- Lookup a DTrace probe based on its name
+ *   dtrace_probe_arg()      <-- Return the probe argument for a specific probe
+ *   dtrace_probe()          <-- Fire the specified probe
+ *
+ * 2.2  int dtrace_register(const char *name, const dtrace_pattr_t *pap,
+ *          uint32_t priv, cred_t *cr, const dtrace_pops_t *pops, void *arg,
+ *          dtrace_provider_id_t *idp)
+ *
+ * 2.2.1  Overview
+ *
+ *   dtrace_register() registers the calling provider with the DTrace
+ *   framework.  It should generally be called by DTrace providers in their
+ *   attach(9E) entry point.
+ *
+ * 2.2.2  Arguments and Notes
+ *
+ *   The first argument is the name of the provider.  The second argument is a
+ *   pointer to the stability attributes for the provider.  The third argument
+ *   is the privilege flags for the provider, and must be some combination of:
+ *
+ *     DTRACE_PRIV_NONE     <= All users may enable probes from this provider
+ *
+ *     DTRACE_PRIV_PROC     <= Any user with privilege of PRIV_DTRACE_PROC may
+ *                             enable probes from this provider
+ *
+ *     DTRACE_PRIV_USER     <= Any user with privilege of PRIV_DTRACE_USER may
+ *                             enable probes from this provider
+ *
+ *     DTRACE_PRIV_KERNEL   <= Any user with privilege of PRIV_DTRACE_KERNEL
+ *                             may enable probes from this provider
+ *
+ *     DTRACE_PRIV_OWNER    <= This flag places an additional constraint on
+ *                             the privilege requirements above. These probes
+ *                             require either (a) a user ID matching the user
+ *                             ID of the cred passed in the fourth argument
+ *                             or (b) the PRIV_PROC_OWNER privilege.
+ *
+ *     DTRACE_PRIV_ZONEOWNER<= This flag places an additional constraint on
+ *                             the privilege requirements above. These probes
+ *                             require either (a) a zone ID matching the zone
+ *                             ID of the cred passed in the fourth argument
+ *                             or (b) the PRIV_PROC_ZONE privilege.
+ *
+ *   Note that these flags designate the _visibility_ of the probes, not
+ *   the conditions under which they may or may not fire.
+ *
+ *   The fourth argument is the credential that is associated with the
+ *   provider.  This argument should be NULL if the privilege flags don't
+ *   include DTRACE_PRIV_OWNER or DTRACE_PRIV_ZONEOWNER.  If non-NULL, the
+ *   framework stashes the uid and zoneid represented by this credential
+ *   for use at probe-time, in implicit predicates.  These limit visibility
+ *   of the probes to users and/or zones which have sufficient privilege to
+ *   access them.
+ *
+ *   The fifth argument is a DTrace provider operations vector, which provides
+ *   the implementation for the Framework-to-Provider API.  (See Section 1,
+ *   above.)  This must be non-NULL, and each member must be non-NULL.  The
+ *   exceptions to this are (1) the dtps_provide() and dtps_provide_module()
+ *   members (if the provider so desires, _one_ of these members may be left
+ *   NULL -- denoting that the provider only implements the other) and (2)
+ *   the dtps_suspend() and dtps_resume() members, which must either both be
+ *   NULL or both be non-NULL.
+ *
+ *   The sixth argument is a cookie to be specified as the first argument for
+ *   each function in the Framework-to-Provider API.  This argument may have
+ *   any value.
+ *
+ *   The final argument is a pointer to dtrace_provider_id_t.  If
+ *   dtrace_register() successfully completes, the provider identifier will be
+ *   stored in the memory pointed to be this argument.  This argument must be
+ *   non-NULL.
+ *
+ * 2.2.3  Return value
+ *
+ *   On success, dtrace_register() returns 0 and stores the new provider's
+ *   identifier into the memory pointed to by the idp argument.  On failure,
+ *   dtrace_register() returns an errno:
+ *
+ *     EINVAL   The arguments passed to dtrace_register() were somehow invalid.
+ *              This may because a parameter that must be non-NULL was NULL,
+ *              because the name was invalid (either empty or an illegal
+ *              provider name) or because the attributes were invalid.
+ *
+ *   No other failure code is returned.
+ *
+ * 2.2.4  Caller's context
+ *
+ *   dtrace_register() may induce calls to dtrace_provide(); the provider must
+ *   hold no locks across dtrace_register() that may also be acquired by
+ *   dtrace_provide().  cpu_lock and mod_lock must not be held.
+ *
+ * 2.3  int dtrace_unregister(dtrace_provider_t id)
+ *
+ * 2.3.1  Overview
+ *
+ *   Unregisters the specified provider from the DTrace framework.  It should
+ *   generally be called by DTrace providers in their detach(9E) entry point.
+ *
+ * 2.3.2  Arguments and Notes
+ *
+ *   The only argument is the provider identifier, as returned from a
+ *   successful call to dtrace_register().  As a result of calling
+ *   dtrace_unregister(), the DTrace framework will call back into the provider
+ *   via the dtps_destroy() entry point.  Once dtrace_unregister() successfully
+ *   completes, however, the DTrace framework will no longer make calls through
+ *   the Framework-to-Provider API.
+ *
+ * 2.3.3  Return value
+ *
+ *   On success, dtrace_unregister returns 0.  On failure, dtrace_unregister()
+ *   returns an errno:
+ *
+ *     EBUSY    There are currently processes that have the DTrace pseudodevice
+ *              open, or there exists an anonymous enabling that hasn't yet
+ *              been claimed.
+ *
+ *   No other failure code is returned.
+ *
+ * 2.3.4  Caller's context
+ *
+ *   Because a call to dtrace_unregister() may induce calls through the
+ *   Framework-to-Provider API, the caller may not hold any lock across
+ *   dtrace_register() that is also acquired in any of the Framework-to-
+ *   Provider API functions.  Additionally, mod_lock may not be held.
+ *
+ * 2.4  void dtrace_invalidate(dtrace_provider_id_t id)
+ *
+ * 2.4.1  Overview
+ *
+ *   Invalidates the specified provider.  All subsequent probe lookups for the
+ *   specified provider will fail, but its probes will not be removed.
+ *
+ * 2.4.2  Arguments and note
+ *
+ *   The only argument is the provider identifier, as returned from a
+ *   successful call to dtrace_register().  In general, a provider's probes
+ *   always remain valid; dtrace_invalidate() is a mechanism for invalidating
+ *   an entire provider, regardless of whether or not probes are enabled or
+ *   not.  Note that dtrace_invalidate() will _not_ prevent already enabled
+ *   probes from firing -- it will merely prevent any new enablings of the
+ *   provider's probes.
+ *
+ * 2.5 int dtrace_condense(dtrace_provider_id_t id)
+ *
+ * 2.5.1  Overview
+ *
+ *   Removes all the unenabled probes for the given provider. This function is
+ *   not unlike dtrace_unregister(), except that it doesn't remove the
+ *   provider just as many of its associated probes as it can.
+ *
+ * 2.5.2  Arguments and Notes
+ *
+ *   As with dtrace_unregister(), the sole argument is the provider identifier
+ *   as returned from a successful call to dtrace_register().  As a result of
+ *   calling dtrace_condense(), the DTrace framework will call back into the
+ *   given provider's dtps_destroy() entry point for each of the provider's
+ *   unenabled probes.
+ *
+ * 2.5.3  Return value
+ *
+ *   Currently, dtrace_condense() always returns 0.  However, consumers of this
+ *   function should check the return value as appropriate; its behavior may
+ *   change in the future.
+ *
+ * 2.5.4  Caller's context
+ *
+ *   As with dtrace_unregister(), the caller may not hold any lock across
+ *   dtrace_condense() that is also acquired in the provider's entry points.
+ *   Also, mod_lock may not be held.
+ *
+ * 2.6 int dtrace_attached()
+ *
+ * 2.6.1  Overview
+ *
+ *   Indicates whether or not DTrace has attached.
+ *
+ * 2.6.2  Arguments and Notes
+ *
+ *   For most providers, DTrace makes initial contact beyond registration.
+ *   That is, once a provider has registered with DTrace, it waits to hear
+ *   from DTrace to create probes.  However, some providers may wish to
+ *   proactively create probes without first being told by DTrace to do so.
+ *   If providers wish to do this, they must first call dtrace_attached() to
+ *   determine if DTrace itself has attached.  If dtrace_attached() returns 0,
+ *   the provider must not make any other Provider-to-Framework API call.
+ *
+ * 2.6.3  Return value
+ *
+ *   dtrace_attached() returns 1 if DTrace has attached, 0 otherwise.
+ *
+ * 2.7  int dtrace_probe_create(dtrace_provider_t id, const char *mod,
+ *          const char *func, const char *name, int aframes, void *arg)
+ *
+ * 2.7.1  Overview
+ *
+ *   Creates a probe with specified module name, function name, and name.
+ *
+ * 2.7.2  Arguments and Notes
+ *
+ *   The first argument is the provider identifier, as returned from a
+ *   successful call to dtrace_register().  The second, third, and fourth
+ *   arguments are the module name, function name, and probe name,
+ *   respectively.  Of these, module name and function name may both be NULL
+ *   (in which case the probe is considered to be unanchored), or they may both
+ *   be non-NULL.  The name must be non-NULL, and must point to a non-empty
+ *   string.
+ *
+ *   The fifth argument is the number of artificial stack frames that will be
+ *   found on the stack when dtrace_probe() is called for the new probe.  These
+ *   artificial frames will be automatically be pruned should the stack() or
+ *   stackdepth() functions be called as part of one of the probe's ECBs.  If
+ *   the parameter doesn't add an artificial frame, this parameter should be
+ *   zero.
+ *
+ *   The final argument is a probe argument that will be passed back to the
+ *   provider when a probe-specific operation is called.  (e.g., via
+ *   dtps_enable(), dtps_disable(), etc.)
+ *
+ *   Note that it is up to the provider to be sure that the probe that it
+ *   creates does not already exist -- if the provider is unsure of the probe's
+ *   existence, it should assure its absence with dtrace_probe_lookup() before
+ *   calling dtrace_probe_create().
+ *
+ * 2.7.3  Return value
+ *
+ *   dtrace_probe_create() always succeeds, and always returns the identifier
+ *   of the newly-created probe.
+ *
+ * 2.7.4  Caller's context
+ *
+ *   While dtrace_probe_create() is generally expected to be called from
+ *   dtps_provide() and/or dtps_provide_module(), it may be called from other
+ *   non-DTrace contexts.  Neither cpu_lock nor mod_lock may be held.
+ *
+ * 2.8  dtrace_id_t dtrace_probe_lookup(dtrace_provider_t id, const char *mod,
+ *          const char *func, const char *name)
+ *
+ * 2.8.1  Overview
+ *
+ *   Looks up a probe based on provdider and one or more of module name,
+ *   function name and probe name.
+ *
+ * 2.8.2  Arguments and Notes
+ *
+ *   The first argument is the provider identifier, as returned from a
+ *   successful call to dtrace_register().  The second, third, and fourth
+ *   arguments are the module name, function name, and probe name,
+ *   respectively.  Any of these may be NULL; dtrace_probe_lookup() will return
+ *   the identifier of the first probe that is provided by the specified
+ *   provider and matches all of the non-NULL matching criteria.
+ *   dtrace_probe_lookup() is generally used by a provider to be check the
+ *   existence of a probe before creating it with dtrace_probe_create().
+ *
+ * 2.8.3  Return value
+ *
+ *   If the probe exists, returns its identifier.  If the probe does not exist,
+ *   return DTRACE_IDNONE.
+ *
+ * 2.8.4  Caller's context
+ *
+ *   While dtrace_probe_lookup() is generally expected to be called from
+ *   dtps_provide() and/or dtps_provide_module(), it may also be called from
+ *   other non-DTrace contexts.  Neither cpu_lock nor mod_lock may be held.
+ *
+ * 2.9  void *dtrace_probe_arg(dtrace_provider_t id, dtrace_id_t probe)
+ *
+ * 2.9.1  Overview
+ *
+ *   Returns the probe argument associated with the specified probe.
+ *
+ * 2.9.2  Arguments and Notes
+ *
+ *   The first argument is the provider identifier, as returned from a
+ *   successful call to dtrace_register().  The second argument is a probe
+ *   identifier, as returned from dtrace_probe_lookup() or
+ *   dtrace_probe_create().  This is useful if a probe has multiple
+ *   provider-specific components to it:  the provider can create the probe
+ *   once with provider-specific state, and then add to the state by looking
+ *   up the probe based on probe identifier.
+ *
+ * 2.9.3  Return value
+ *
+ *   Returns the argument associated with the specified probe.  If the
+ *   specified probe does not exist, or if the specified probe is not provided
+ *   by the specified provider, NULL is returned.
+ *
+ * 2.9.4  Caller's context
+ *
+ *   While dtrace_probe_arg() is generally expected to be called from
+ *   dtps_provide() and/or dtps_provide_module(), it may also be called from
+ *   other non-DTrace contexts.  Neither cpu_lock nor mod_lock may be held.
+ *
+ * 2.10  void dtrace_probe(dtrace_id_t probe, uintptr_t arg0, uintptr_t arg1,
+ *              uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
+ *
+ * 2.10.1  Overview
+ *
+ *   The epicenter of DTrace:  fires the specified probes with the specified
+ *   arguments.
+ *
+ * 2.10.2  Arguments and Notes
+ *
+ *   The first argument is a probe identifier as returned by
+ *   dtrace_probe_create() or dtrace_probe_lookup().  The second through sixth
+ *   arguments are the values to which the D variables "arg0" through "arg4"
+ *   will be mapped.
+ *
+ *   dtrace_probe() should be called whenever the specified probe has fired --
+ *   however the provider defines it.
+ *
+ * 2.10.3  Return value
+ *
+ *   None.
+ *
+ * 2.10.4  Caller's context
+ *
+ *   dtrace_probe() may be called in virtually any context:  kernel, user,
+ *   interrupt, high-level interrupt, with arbitrary adaptive locks held, with
+ *   dispatcher locks held, with interrupts disabled, etc.  The only latitude
+ *   that must be afforded to DTrace is the ability to make calls within
+ *   itself (and to its in-kernel subroutines) and the ability to access
+ *   arbitrary (but mapped) memory.  On some platforms, this constrains
+ *   context.  For example, on UltraSPARC, dtrace_probe() cannot be called
+ *   from any context in which TL is greater than zero.  dtrace_probe() may
+ *   also not be called from any routine which may be called by dtrace_probe()
+ *   -- which includes functions in the DTrace framework and some in-kernel
+ *   DTrace subroutines.  All such functions "dtrace_"; providers that
+ *   instrument the kernel arbitrarily should be sure to not instrument these
+ *   routines.
+ */
+typedef struct dtrace_pops {
+        void (*dtps_provide)(void *arg, const dtrace_probedesc_t *spec);
+        void (*dtps_provide_module)(void *arg, struct modctl *mp);
+        void (*dtps_enable)(void *arg, dtrace_id_t id, void *parg);
+        void (*dtps_disable)(void *arg, dtrace_id_t id, void *parg);
+        void (*dtps_suspend)(void *arg, dtrace_id_t id, void *parg);
+        void (*dtps_resume)(void *arg, dtrace_id_t id, void *parg);
+        void (*dtps_getargdesc)(void *arg, dtrace_id_t id, void *parg,
+            dtrace_argdesc_t *desc);
+        uint64_t (*dtps_getargval)(void *arg, dtrace_id_t id, void *parg,
+            int argno, int aframes);
+        int (*dtps_usermode)(void *arg, dtrace_id_t id, void *parg);
+        void (*dtps_destroy)(void *arg, dtrace_id_t id, void *parg);
+} dtrace_pops_t;
+
+typedef uintptr_t       dtrace_provider_id_t;
+
+extern int dtrace_register(const char *, const dtrace_pattr_t *, uint32_t,
+    cred_t *, const dtrace_pops_t *, void *, dtrace_provider_id_t *);
+extern int dtrace_unregister(dtrace_provider_id_t);
+extern int dtrace_condense(dtrace_provider_id_t);
+extern void dtrace_invalidate(dtrace_provider_id_t);
+extern dtrace_id_t dtrace_probe_lookup(dtrace_provider_id_t, const char *,
+    const char *, const char *);
+extern dtrace_id_t dtrace_probe_create(dtrace_provider_id_t, const char *,
+    const char *, const char *, int, void *);
+extern void *dtrace_probe_arg(dtrace_provider_id_t, dtrace_id_t);
+#if !defined(__APPLE__)
+extern void dtrace_probe(dtrace_id_t, uintptr_t arg0, uintptr_t arg1,
+    uintptr_t arg2, uintptr_t arg3, uintptr_t arg4);
+#else
+extern void dtrace_probe(dtrace_id_t, uint64_t arg0, uint64_t arg1,
+    uint64_t arg2, uint64_t arg3, uint64_t arg4);
+#endif /* __APPLE__ */
+
+/*
+ * DTrace Meta Provider API
+ *
+ * The following functions are implemented by the DTrace framework and are
+ * used to implement meta providers. Meta providers plug into the DTrace
+ * framework and are used to instantiate new providers on the fly. At
+ * present, there is only one type of meta provider and only one meta
+ * provider may be registered with the DTrace framework at a time. The
+ * sole meta provider type provides user-land static tracing facilities
+ * by taking meta probe descriptions and adding a corresponding provider
+ * into the DTrace framework.
+ *
+ * 1 Framework-to-Provider
+ *
+ * 1.1 Overview
+ *
+ * The Framework-to-Provider API is represented by the dtrace_mops structure
+ * that the meta provider passes to the framework when registering itself as
+ * a meta provider. This structure consists of the following members:
+ *
+ *   dtms_create_probe()        <-- Add a new probe to a created provider
+ *   dtms_provide_pid()         <-- Create a new provider for a given process
+ *   dtms_remove_pid()          <-- Remove a previously created provider
+ *
+ * 1.2  void dtms_create_probe(void *arg, void *parg,
+ *           dtrace_helper_probedesc_t *probedesc);
+ *
+ * 1.2.1  Overview
+ *
+ *   Called by the DTrace framework to create a new probe in a provider
+ *   created by this meta provider.
+ *
+ * 1.2.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_meta_register().
+ *   The second argument is the provider cookie for the associated provider;
+ *   this is obtained from the return value of dtms_provide_pid(). The third
+ *   argument is the helper probe description.
+ *
+ * 1.2.3  Return value
+ *
+ *   None
+ *
+ * 1.2.4  Caller's context
+ *
+ *   dtms_create_probe() is called from either ioctl() or module load context.
+ *   The DTrace framework is locked in such a way that meta providers may not
+ *   register or unregister. This means that the meta provider cannot call
+ *   dtrace_meta_register() or dtrace_meta_unregister(). However, the context is
+ *   such that the provider may (and is expected to) call provider-related
+ *   DTrace provider APIs including dtrace_probe_create().
+ *
+ * 1.3  void *dtms_provide_pid(void *arg, dtrace_meta_provider_t *mprov,
+ *            pid_t pid)
+ *
+ * 1.3.1  Overview
+ *
+ *   Called by the DTrace framework to instantiate a new provider given the
+ *   description of the provider and probes in the mprov argument. The
+ *   meta provider should call dtrace_register() to insert the new provider
+ *   into the DTrace framework.
+ *
+ * 1.3.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_meta_register().
+ *   The second argument is a pointer to a structure describing the new
+ *   helper provider. The third argument is the process identifier for
+ *   process associated with this new provider. Note that the name of the
+ *   provider as passed to dtrace_register() should be the contatenation of
+ *   the dtmpb_provname member of the mprov argument and the processs
+ *   identifier as a string.
+ *
+ * 1.3.3  Return value
+ *
+ *   The cookie for the provider that the meta provider creates. This is
+ *   the same value that it passed to dtrace_register().
+ *
+ * 1.3.4  Caller's context
+ *
+ *   dtms_provide_pid() is called from either ioctl() or module load context.
+ *   The DTrace framework is locked in such a way that meta providers may not
+ *   register or unregister. This means that the meta provider cannot call
+ *   dtrace_meta_register() or dtrace_meta_unregister(). However, the context
+ *   is such that the provider may -- and is expected to --  call
+ *   provider-related DTrace provider APIs including dtrace_register().
+ *
+ * 1.4  void dtms_remove_pid(void *arg, dtrace_meta_provider_t *mprov,
+ *           pid_t pid)
+ *
+ * 1.4.1  Overview
+ *
+ *   Called by the DTrace framework to remove a provider that had previously
+ *   been instantiated via the dtms_provide_pid() entry point. The meta
+ *   provider need not remove the provider immediately, but this entry
+ *   point indicates that the provider should be removed as soon as possible
+ *   using the dtrace_unregister() API.
+ *
+ * 1.4.2  Arguments and notes
+ *
+ *   The first argument is the cookie as passed to dtrace_meta_register().
+ *   The second argument is a pointer to a structure describing the helper
+ *   provider. The third argument is the process identifier for process
+ *   associated with this new provider.
+ *
+ * 1.4.3  Return value
+ *
+ *   None
+ *
+ * 1.4.4  Caller's context
+ *
+ *   dtms_remove_pid() is called from either ioctl() or exit() context.
+ *   The DTrace framework is locked in such a way that meta providers may not
+ *   register or unregister. This means that the meta provider cannot call
+ *   dtrace_meta_register() or dtrace_meta_unregister(). However, the context
+ *   is such that the provider may -- and is expected to -- call
+ *   provider-related DTrace provider APIs including dtrace_unregister().
+ */
+typedef struct dtrace_helper_probedesc {
+	char *dthpb_mod;			/* probe module */
+	char *dthpb_func; 			/* probe function */
+	char *dthpb_name; 			/* probe name */
+	uint64_t dthpb_base;			/* base address */
+#if !defined(__APPLE__)
+	uint32_t *dthpb_offs;			/* offsets array */
+	uint32_t *dthpb_enoffs;			/* is-enabled offsets array */
+#else
+	int32_t *dthpb_offs;			/* (signed) offsets array */
+	int32_t *dthpb_enoffs;			/* (signed) is-enabled offsets array */
+#endif
+	uint32_t dthpb_noffs;			/* offsets count */
+	uint32_t dthpb_nenoffs;			/* is-enabled offsets count */
+	uint8_t *dthpb_args;			/* argument mapping array */
+	uint8_t dthpb_xargc;			/* translated argument count */
+	uint8_t dthpb_nargc;			/* native argument count */
+	char *dthpb_xtypes;			/* translated types strings */
+	char *dthpb_ntypes;			/* native types strings */
+} dtrace_helper_probedesc_t;
+
+typedef struct dtrace_helper_provdesc {
+        char *dthpv_provname;                   /* provider name */
+        dtrace_pattr_t dthpv_pattr;             /* stability attributes */
+} dtrace_helper_provdesc_t;
+
+typedef struct dtrace_mops {
+        void (*dtms_create_probe)(void *, void *, dtrace_helper_probedesc_t *);
+        void *(*dtms_provide_pid)(void *, dtrace_helper_provdesc_t *, pid_t);
+        void (*dtms_remove_pid)(void *, dtrace_helper_provdesc_t *, pid_t);
+} dtrace_mops_t;
+
+typedef uintptr_t       dtrace_meta_provider_id_t;
+
+extern int dtrace_meta_register(const char *, const dtrace_mops_t *, void *,
+    dtrace_meta_provider_id_t *);
+extern int dtrace_meta_unregister(dtrace_meta_provider_id_t);
+
+/*
+ * DTrace Kernel Hooks
+ *
+ * The following functions are implemented by the base kernel and form a set of
+ * hooks used by the DTrace framework.  DTrace hooks are implemented in either
+ * uts/common/os/dtrace_subr.c, an ISA-specific assembly file, or in a
+ * uts/<platform>/os/dtrace_subr.c corresponding to each hardware platform.
+ */
+
+typedef enum dtrace_vtime_state {
+        DTRACE_VTIME_INACTIVE = 0,      /* No DTrace, no TNF */
+        DTRACE_VTIME_ACTIVE,            /* DTrace virtual time, no TNF */
+        DTRACE_VTIME_INACTIVE_TNF,      /* No DTrace, TNF active */
+        DTRACE_VTIME_ACTIVE_TNF         /* DTrace virtual time _and_ TNF */
+} dtrace_vtime_state_t;
+
+extern dtrace_vtime_state_t dtrace_vtime_active;
+extern void dtrace_vtime_switch(kthread_t *next);
+extern void dtrace_vtime_enable_tnf(void);
+extern void dtrace_vtime_disable_tnf(void);
+extern void dtrace_vtime_enable(void);
+extern void dtrace_vtime_disable(void);
+
+#if defined (__ppc__) || defined (__ppc64__)
+extern int (*dtrace_pid_probe_ptr)(ppc_saved_state_t *regs);
+extern int (*dtrace_return_probe_ptr)(ppc_saved_state_t* regs);
+#elif defined (__i386__) || defined(__x86_64__)
+extern int (*dtrace_pid_probe_ptr)(x86_saved_state_t *regs);
+extern int (*dtrace_return_probe_ptr)(x86_saved_state_t* regs);
+#elif defined (__arm__)
+extern int (*dtrace_pid_probe_ptr)(struct arm_saved_state *regs);
+extern int (*dtrace_return_probe_ptr)(struct arm_saved_state* regs);
+#else
+#error architecture not supported
+#endif
+ 
+extern void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *);
+extern void (*dtrace_fasttrap_exec_ptr)(proc_t *);
+extern void (*dtrace_fasttrap_exit_ptr)(proc_t *);
+extern void dtrace_fasttrap_fork(proc_t *, proc_t *);
+
+typedef uintptr_t dtrace_icookie_t;
+typedef void (*dtrace_xcall_t)(void *);
+
+extern dtrace_icookie_t dtrace_interrupt_disable(void);
+extern void dtrace_interrupt_enable(dtrace_icookie_t);
+
+extern void dtrace_membar_producer(void);
+extern void dtrace_membar_consumer(void);
+
+extern void (*dtrace_cpu_init)(processorid_t);
+extern void (*dtrace_modload)(struct modctl *);
+extern void (*dtrace_modunload)(struct modctl *);
+extern void (*dtrace_helpers_cleanup)(proc_t*);
+extern void (*dtrace_helpers_fork)(proc_t *parent, proc_t *child);
+extern void (*dtrace_cpustart_init)(void);
+extern void (*dtrace_cpustart_fini)(void);
+
+extern void (*dtrace_kreloc_init)(void);
+extern void (*dtrace_kreloc_fini)(void);
+
+extern void (*dtrace_debugger_init)(void);
+extern void (*dtrace_debugger_fini)(void);
+extern dtrace_cacheid_t dtrace_predcache_id;
+
+extern hrtime_t dtrace_gethrtime(void);
+extern void dtrace_sync(void);
+extern void dtrace_toxic_ranges(void (*)(uintptr_t, uintptr_t));
+extern void dtrace_xcall(processorid_t, dtrace_xcall_t, void *);
+extern void dtrace_vpanic(const char *, __va_list);
+extern void dtrace_panic(const char *, ...);
+
+extern int dtrace_safe_defer_signal(void);
+extern void dtrace_safe_synchronous_signal(void);
+
+#if defined(__i386__) || defined(__x86_64__)
+extern int dtrace_instr_size(uchar_t *instr);
+extern int dtrace_instr_size_isa(uchar_t *, model_t, int *);
+extern void dtrace_invop_add(int (*)(uintptr_t, uintptr_t *, uintptr_t));
+extern void dtrace_invop_remove(int (*)(uintptr_t, uintptr_t *, uintptr_t));
+extern void dtrace_invop_callsite(void);
+#endif
+
+#ifdef __sparc
+extern int dtrace_blksuword32(uintptr_t, uint32_t *, int);
+extern void dtrace_getfsr(uint64_t *);
+#endif
+
+#if defined(__APPLE__)
+#if defined (__ppc__) || defined (__ppc64__)
+extern void dtrace_invop_add(int (*)(uintptr_t, uintptr_t *, uintptr_t));
+extern void dtrace_invop_remove(int (*)(uintptr_t, uintptr_t *, uintptr_t));
+#endif
+#undef proc_t
+#endif /* __APPLE__ */
+
+#define DTRACE_CPUFLAG_ISSET(flag) \
+        (cpu_core[CPU->cpu_id].cpuc_dtrace_flags & (flag))
+
+#define DTRACE_CPUFLAG_SET(flag) \
+        (cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= (flag))
+
+#define DTRACE_CPUFLAG_CLEAR(flag) \
+        (cpu_core[CPU->cpu_id].cpuc_dtrace_flags &= ~(flag))
+
+#endif /* _KERNEL */
+
+#endif  /* _ASM */
+
+#if defined(__i386__) || defined(__x86_64__)
+
+#define DTRACE_INVOP_PUSHL_EBP          1
+#define DTRACE_INVOP_POPL_EBP           2
+#define DTRACE_INVOP_LEAVE              3
+#define DTRACE_INVOP_NOP                4
+#define DTRACE_INVOP_RET                5
+
+#endif
+
+#if defined(__APPLE__)
+#if defined (__ppc__) || defined (__ppc64__)
+#define DTRACE_INVOP_NOP                4
+#define DTRACE_INVOP_RET                5
+#define DTRACE_INVOP_BCTR               6
+#define DTRACE_INVOP_TAILJUMP           7
+#endif
+#endif /* __APPLE__ */
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif  /* _SYS_DTRACE_H */
diff --git a/bsd/sys/dtrace_glue.h b/bsd/sys/dtrace_glue.h
new file mode 100644
index 000000000..754a272dc
--- /dev/null
+++ b/bsd/sys/dtrace_glue.h
@@ -0,0 +1,503 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _DTRACE_GLUE_H
+#define _DTRACE_GLUE_H
+
+#ifdef KERNEL_BUILD
+
+#include <libkern/libkern.h>
+#include <kern/lock.h>
+#include <kern/locks.h>
+#include <kern/thread_call.h>
+#include <machine/machine_routines.h>
+#include <sys/syslog.h>
+#include <sys/ucred.h>
+#include <stdarg.h>
+#include <mach/kmod.h>
+#include <libkern/OSAtomic.h>
+
+#ifdef QUIET_PLEASE
+  #ifndef NULL
+  #define NULL ((void *)0) /* quiets many warnings */
+  #endif
+#endif
+
+/*
+ * cmn_err
+ */
+#define	CE_CONT		0	/* continuation		*/
+#define	CE_NOTE		1	/* notice		*/
+#define	CE_WARN		2	/* warning		*/
+#define	CE_PANIC	3	/* panic		*/
+#define	CE_IGNORE	4	/* print nothing	*/
+
+extern void cmn_err( int, const char *, ... );
+
+/*
+ * pid/proc
+ */
+
+typedef struct proc SUN_PROC_T; /* Solaris proc_t is the struct. Darwin's proc_t is a pointer to it. */
+#define proc_t SUN_PROC_T /* replace all the original uses of (Solaris) proc_t */
+#define curproc ((struct proc *)current_proc()) /* Called from probe context, must blacklist */
+
+proc_t* sprlock(pid_t pid);
+void sprunlock(proc_t *p);
+
+/*
+ * uread/uwrite
+ */
+
+int uread(proc_t *p, void *buf, user_size_t len, user_addr_t a);
+int uwrite(proc_t *p, void *buf, user_size_t len, user_addr_t a);
+
+/*
+ * fuword / suword
+ */
+
+int fuword8(user_addr_t, uint8_t *);
+int fuword16(user_addr_t, uint16_t *);
+int fuword32(user_addr_t, uint32_t *);
+int fuword64(user_addr_t, uint64_t *);
+
+void fuword8_noerr(user_addr_t, uint8_t *);
+void fuword16_noerr(user_addr_t, uint16_t *);
+void fuword32_noerr(user_addr_t, uint32_t *);
+void fuword64_noerr(user_addr_t, uint64_t *);
+
+int suword64(user_addr_t, uint64_t value);
+int suword32(user_addr_t, uint32_t value);
+int suword16(user_addr_t, uint16_t value);
+int suword8(user_addr_t, uint8_t value);
+
+/*
+ * cpuvar
+ */
+extern lck_mtx_t cpu_lock;
+extern lck_mtx_t mod_lock;
+
+/*
+ * Per-CPU data.
+ */
+typedef struct cpu {
+	processorid_t   cpu_id;                    /* CPU number */
+	struct cpu      *cpu_next;                 /* next existing CPU */
+	lck_rw_t        cpu_ft_lock;               /* DTrace: fasttrap lock */
+	uintptr_t       cpu_dtrace_caller;         /* DTrace: caller, if any */
+	hrtime_t        cpu_dtrace_chillmark;      /* DTrace: chill mark time */
+	hrtime_t        cpu_dtrace_chilled;        /* DTrace: total chill time */
+	boolean_t       cpu_dtrace_invop_underway; /* DTrace gaurds against invalid op re-entrancy */
+} cpu_t;
+
+extern cpu_t *cpu_list;
+
+/*
+ * The cpu_core structure consists of per-CPU state available in any context.
+ * On some architectures, this may mean that the page(s) containing the
+ * NCPU-sized array of cpu_core structures must be locked in the TLB -- it
+ * is up to the platform to assure that this is performed properly.  Note that
+ * the structure is sized to avoid false sharing.
+ */
+#define	CPU_CACHE_COHERENCE_SIZE	64
+#define	CPUC_SIZE		(sizeof (uint16_t))
+#define	CPUC_PADSIZE		CPU_CACHE_COHERENCE_SIZE - CPUC_SIZE
+
+typedef struct cpu_core {
+	uint16_t	cpuc_dtrace_flags;      /* DTrace flags */
+	uint8_t		cpuc_pad[CPUC_PADSIZE];	/* padding */
+	uint64_t	cpuc_dtrace_illval;     /* DTrace illegal value */
+	lck_mtx_t	cpuc_pid_lock;          /* DTrace pid provider lock */
+} cpu_core_t;
+
+extern cpu_core_t *cpu_core; /* XXX TLB lockdown? */
+extern unsigned int real_ncpus;
+extern int cpu_number(void); /* XXX #include <kern/cpu_number.h>. Called from probe context, must blacklist. */
+
+#define	CPU		(&(cpu_list[cpu_number()]))	/* Pointer to current CPU */
+#define	CPU_ON_INTR(cpup) ml_at_interrupt_context() /* always invoked on current cpu */
+#define NCPU	real_ncpus
+
+/*
+ * Routines used to register interest in cpu's being added to or removed
+ * from the system.
+ */
+typedef enum {
+	CPU_INIT,
+	CPU_CONFIG,
+	CPU_UNCONFIG,
+	CPU_ON,
+	CPU_OFF,
+	CPU_CPUPART_IN,
+	CPU_CPUPART_OUT
+} cpu_setup_t;
+
+typedef int cpu_setup_func_t(cpu_setup_t, int, void *);
+
+extern void register_cpu_setup_func(cpu_setup_func_t *, void *);
+extern void unregister_cpu_setup_func(cpu_setup_func_t *, void *);
+
+/*
+ * CPU_DTRACE
+ */
+
+/*
+ * DTrace flags.
+ */
+#define	CPU_DTRACE_NOFAULT	0x0001	/* Don't fault */
+#define	CPU_DTRACE_DROP		0x0002	/* Drop this ECB */
+#define	CPU_DTRACE_BADADDR	0x0004	/* DTrace fault: bad address */
+#define	CPU_DTRACE_BADALIGN	0x0008	/* DTrace fault: bad alignment */
+#define	CPU_DTRACE_DIVZERO	0x0010	/* DTrace fault: divide by zero */
+#define	CPU_DTRACE_ILLOP	0x0020	/* DTrace fault: illegal operation */
+#define	CPU_DTRACE_NOSCRATCH	0x0040	/* DTrace fault: out of scratch */
+#define	CPU_DTRACE_KPRIV	0x0080	/* DTrace fault: bad kernel access */
+#define	CPU_DTRACE_UPRIV	0x0100	/* DTrace fault: bad user access */
+#define	CPU_DTRACE_TUPOFLOW	0x0200	/* DTrace fault: tuple stack overflow */
+#if defined(__sparc)
+//#define	CPU_DTRACE_FAKERESTORE	0x0400	/* pid provider hint to getreg */
+#endif
+#define CPU_DTRACE_USTACK_FP	0x0400  /* pid provider hint to ustack() */
+#define	CPU_DTRACE_ENTRY	0x0800	/* pid provider hint to ustack() */
+
+#define	CPU_DTRACE_FAULT	(CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \
+				CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \
+				CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \
+				CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW)
+#define	CPU_DTRACE_ERROR	(CPU_DTRACE_FAULT | CPU_DTRACE_DROP)
+
+/*
+ * cred_t
+ */
+/* Privileges */
+#define PRIV_DTRACE_KERNEL        3
+#define PRIV_DTRACE_PROC          4
+#define PRIV_DTRACE_USER          5
+#define PRIV_PROC_OWNER          30
+#define PRIV_PROC_ZONE           35
+#define	PRIV_ALL			(-1)	/* All privileges required */
+
+/* Privilege sets */
+#define PRIV_EFFECTIVE            0
+
+typedef struct ucred cred_t;
+#define cr_suid cr_svuid
+#define cr_sgid cr_svgid
+
+extern cred_t *dtrace_CRED(void); /* Safe to call from probe context. */
+#define CRED() kauth_cred_get() /* Can't be called from probe context! */
+extern int PRIV_POLICY_CHOICE(void *, int, int);
+extern int PRIV_POLICY_ONLY(void *, int, int);
+extern gid_t crgetgid(const cred_t *);
+extern uid_t crgetuid(const cred_t *);
+#define crgetzoneid(x) ((zoneid_t)0)
+
+#define crhold(a) {}
+#define crfree(a) {}
+
+/*
+ * "cyclic"
+ */
+#define	CY_LOW_LEVEL		0
+#define	CY_LOCK_LEVEL		1
+#define	CY_HIGH_LEVEL		2
+#define	CY_SOFT_LEVELS		2
+#define	CY_LEVELS			3
+
+typedef uintptr_t cyclic_id_t;
+typedef cyclic_id_t *cyclic_id_list_t;
+typedef uint16_t cyc_level_t;
+typedef void (*cyc_func_t)(void *);
+
+#define	CYCLIC_NONE		((cyclic_id_t)0)
+
+typedef struct cyc_time {
+	hrtime_t cyt_when;
+	hrtime_t cyt_interval;
+} cyc_time_t;
+
+typedef struct cyc_handler {
+	cyc_func_t cyh_func;
+	void *cyh_arg;
+	cyc_level_t cyh_level;
+} cyc_handler_t;
+
+typedef struct cyc_omni_handler {
+	void (*cyo_online)(void *, cpu_t *, cyc_handler_t *, cyc_time_t *);
+	void (*cyo_offline)(void *, cpu_t *, void *);
+	void *cyo_arg;
+} cyc_omni_handler_t;
+
+extern cyclic_id_t cyclic_add(cyc_handler_t *, cyc_time_t *);
+extern void cyclic_remove(cyclic_id_t);
+
+extern cyclic_id_list_t cyclic_add_omni(cyc_omni_handler_t *);
+extern void cyclic_remove_omni(cyclic_id_list_t);
+
+extern cyclic_id_t cyclic_timer_add(cyc_handler_t *, cyc_time_t *);
+extern void cyclic_timer_remove(cyclic_id_t);
+
+/*
+ * timeout / untimeout (converted to dtrace_timeout / dtrace_untimeout due to name collision)
+ */
+
+thread_call_t dtrace_timeout(void (*func)(void *, void *), void* arg, uint64_t nanos);
+
+/*
+ * ddi
+ */
+
+#define DDI_SUCCESS			0
+#define DDI_FAILURE			-1
+
+#define	DDI_DEV_T_NONE	((dev_t)-1)
+#define	DDI_DEV_T_ANY	((dev_t)-2)
+#define	DDI_MAJOR_T_UNKNOWN	((major_t)0)
+
+#define DDI_PSEUDO "ddi_pseudo"
+
+typedef enum {
+	DDI_ATTACH = 0,
+	DDI_RESUME = 1,
+	DDI_PM_RESUME = 2
+} ddi_attach_cmd_t;
+
+typedef enum {
+	DDI_DETACH = 0,
+	DDI_SUSPEND = 1,
+	DDI_PM_SUSPEND = 2,
+	DDI_HOTPLUG_DETACH = 3		/* detach, don't try to auto-unconfig */
+} ddi_detach_cmd_t;
+
+#define	DDI_PROP_SUCCESS	0
+
+#define DDI_PROP_DONTPASS   1
+typedef uint_t major_t;
+typedef uint_t minor_t;
+
+typedef struct __dev_info *dev_info_t;
+
+extern void ddi_report_dev(dev_info_t *);
+extern int ddi_soft_state_init(void **, size_t, size_t);
+extern void *ddi_get_soft_state(void *, int);
+extern int ddi_soft_state_free(void *, int);
+extern int ddi_soft_state_zalloc(void *, int);
+extern void ddi_soft_state_fini(void **);
+
+int ddi_getprop(dev_t dev, dev_info_t *dip, int flags, const char *name, int defvalue);
+
+extern int ddi_prop_free(void *);
+extern int ddi_prop_lookup_int_array(dev_t, dev_info_t *, uint_t, char *, int **, uint_t *);
+
+extern int ddi_driver_major(dev_info_t *);
+
+extern int ddi_create_minor_node(dev_info_t *, const char *, int, minor_t, const char *, int);
+extern void ddi_remove_minor_node(dev_info_t *, char *);
+
+extern major_t getemajor(dev_t);
+extern minor_t getminor(dev_t);
+
+extern int _dtrace_dev;
+extern dev_t makedevice(major_t, minor_t);
+
+/*
+ * Kernel Debug Interface
+ */
+
+typedef enum kdi_dtrace_set {
+	KDI_DTSET_DTRACE_ACTIVATE,
+	KDI_DTSET_DTRACE_DEACTIVATE,
+	KDI_DTSET_KMDB_BPT_ACTIVATE,
+	KDI_DTSET_KMDB_BPT_DEACTIVATE
+} kdi_dtrace_set_t;
+
+extern int kdi_dtrace_set(kdi_dtrace_set_t);
+extern void debug_enter(char *);
+
+/*
+ * DTrace specific zone allocation
+ */
+
+/*
+ * To break dtrace memory usage out in a trackable
+ * fashion, uncomment the #define below. This will
+ * enable emulation of the general kalloc.XXX zones
+ * for most dtrace allocations. (kalloc.large is not
+ * emulated)
+ *
+ * #define DTRACE_MEMORY_ZONES 1
+ *
+ */
+
+#if defined(DTRACE_MEMORY_ZONES)
+void dtrace_alloc_init(void);
+void *dtrace_alloc(vm_size_t);
+void dtrace_free(void *, vm_size_t);
+#endif
+
+/*
+ * kmem
+ */
+
+#define KM_SLEEP	0x00000000
+#define KM_NOSLEEP	0x00000001
+
+typedef struct vmem vmem_t;
+typedef struct kmem_cache kmem_cache_t;
+
+#define kmem_alloc dt_kmem_alloc /* Avoid clash with Darwin's kmem_alloc */
+#define kmem_free dt_kmem_free /* Avoid clash with Darwin's kmem_free */
+#define kmem_zalloc dt_kmem_zalloc /* Avoid clash with Darwin's kmem_zalloc */
+extern void *dt_kmem_alloc(size_t, int);
+extern void dt_kmem_free(void *, size_t);
+extern void *dt_kmem_zalloc(size_t, int);
+
+extern void *dt_kmem_alloc_aligned(size_t, size_t, int);
+extern void *dt_kmem_zalloc_aligned(size_t, size_t, int);
+extern void dt_kmem_free_aligned(void*, size_t);
+
+extern kmem_cache_t *
+kmem_cache_create(char *, size_t, size_t, int (*)(void *, void *, int),
+	void (*)(void *, void *), void (*)(void *), void *, vmem_t *, int);
+extern void *kmem_cache_alloc(kmem_cache_t *, int);
+extern void kmem_cache_free(kmem_cache_t *, void *);
+extern void kmem_cache_destroy(kmem_cache_t *);
+
+/*
+ * kthread
+ */
+
+typedef struct _kthread kthread_t; /* For dtrace_vtime_switch(), dtrace_panicked and dtrace_errthread */
+
+/*
+ * Loadable Modules
+ */
+
+decl_simple_lock_data(extern,kmod_lock)
+
+/* Want to use Darwin's kmod_info in place of the Solaris modctl.
+   Can't typedef since the (many) usages in the code are "struct modctl *" */
+extern kmod_info_t *kmod;
+#define modctl kmod_info
+
+#define mod_modname name
+#define mod_loadcnt id
+#define mod_next next
+#define mod_loaded info_version /* XXX Is always > 0, hence TRUE */
+#define modules kmod
+
+/*
+ * proc
+ */
+
+#define DATAMODEL_MASK  0x0FF00000
+
+#define DATAMODEL_ILP32 0x00100000
+#define DATAMODEL_LP64  0x00200000
+
+#define DATAMODEL_NONE  0
+
+#if     defined(__LP64__)
+#define DATAMODEL_NATIVE        DATAMODEL_LP64
+#else
+#define DATAMODEL_NATIVE        DATAMODEL_ILP32
+#endif  /* __LP64__ */
+
+typedef unsigned int model_t; /* For dtrace_instr_size_isa() prototype in <sys/dtrace.h> */
+
+/*
+ * taskq
+ */
+
+#define	TQ_SLEEP	0x00	/* Can block for memory */
+
+typedef uint_t pri_t;
+typedef struct taskq taskq_t;
+typedef void (task_func_t)(void *);
+typedef uintptr_t taskqid_t;
+
+extern taskq_t	*taskq_create(const char *, int, pri_t, int, int, uint_t);
+extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
+extern void	taskq_destroy(taskq_t *);
+
+extern pri_t maxclsyspri;
+
+/*
+ * vmem
+ */
+
+#define	VMC_IDENTIFIER	0x00040000	/* not backed by memory */
+#define	VM_SLEEP	0x00000000	/* same as KM_SLEEP */
+#define	VM_BESTFIT	0x00000100
+
+extern void *vmem_alloc(vmem_t *, size_t, int);
+extern vmem_t *vmem_create(const char *, void *, size_t, size_t, void *,
+					void *, vmem_t *, size_t, int);
+extern void vmem_destroy(vmem_t *);
+extern void vmem_free(vmem_t *vmp, void *vaddr, size_t size);
+
+/*
+ * Atomic
+ */
+
+static inline void atomic_add_32( uint32_t *theValue, int32_t theAmount )
+{
+	(void)OSAddAtomic( theAmount, (SInt32 *)theValue );
+}
+
+/*
+ * Miscellaneous
+ */
+
+typedef uintptr_t pc_t;
+typedef uintptr_t greg_t; /* For dtrace_impl.h prototype of dtrace_getfp() */
+extern struct regs *find_user_regs( thread_t thread);
+extern vm_offset_t dtrace_get_cpu_int_stack_top(void);
+extern vm_offset_t max_valid_stack_address(void); /* kern/thread.h */
+extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); /* machine/pmap.h */
+
+extern volatile int panicwait; /* kern/debug.c */
+#define panic_quiesce (panicwait)
+
+#define	IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
+
+extern void delay( int ); /* kern/clock.h */
+
+extern int vuprintf(const char *, va_list);
+
+extern boolean_t dtxnu_is_RAM_page(ppnum_t);
+
+extern hrtime_t dtrace_abs_to_nano(uint64_t);
+
+__private_extern__ char * strstr(const char *, const char *);
+
+#undef proc_t
+
+#endif /* KERNEL_BUILD */
+#endif /* _DTRACE_GLUE_H */
+
diff --git a/bsd/sys/dtrace_impl.h b/bsd/sys/dtrace_impl.h
new file mode 100644
index 000000000..4d75ea53d
--- /dev/null
+++ b/bsd/sys/dtrace_impl.h
@@ -0,0 +1,1343 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_DTRACE_IMPL_H
+#define	_SYS_DTRACE_IMPL_H
+
+/* #pragma ident	"@(#)dtrace_impl.h	1.21	06/05/19 SMI" */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * DTrace Dynamic Tracing Software: Kernel Implementation Interfaces
+ *
+ * Note: The contents of this file are private to the implementation of the
+ * Solaris system and DTrace subsystem and are subject to change at any time
+ * without notice.  Applications and drivers using these interfaces will fail
+ * to run on future releases.  These interfaces should not be used for any
+ * purpose except those expressly outlined in dtrace(7D) and libdtrace(3LIB).
+ * Please refer to the "Solaris Dynamic Tracing Guide" for more information.
+ */
+
+#include <sys/dtrace.h>
+
+/*
+ * DTrace Implementation Constants and Typedefs
+ */
+#define	DTRACE_MAXPROPLEN		128
+#define	DTRACE_DYNVAR_CHUNKSIZE		256
+
+struct dtrace_probe;
+struct dtrace_ecb;
+struct dtrace_predicate;
+struct dtrace_action;
+struct dtrace_provider;
+struct dtrace_state;
+
+typedef struct dtrace_probe dtrace_probe_t;
+typedef struct dtrace_ecb dtrace_ecb_t;
+typedef struct dtrace_predicate dtrace_predicate_t;
+typedef struct dtrace_action dtrace_action_t;
+typedef struct dtrace_provider dtrace_provider_t;
+typedef struct dtrace_meta dtrace_meta_t;
+typedef struct dtrace_state dtrace_state_t;
+typedef uint32_t dtrace_optid_t;
+typedef uint32_t dtrace_specid_t;
+typedef uint64_t dtrace_genid_t;
+
+/*
+ * DTrace Probes
+ *
+ * The probe is the fundamental unit of the DTrace architecture.  Probes are
+ * created by DTrace providers, and managed by the DTrace framework.  A probe
+ * is identified by a unique <provider, module, function, name> tuple, and has
+ * a unique probe identifier assigned to it.  (Some probes are not associated
+ * with a specific point in text; these are called _unanchored probes_ and have
+ * no module or function associated with them.)  Probes are represented as a
+ * dtrace_probe structure.  To allow quick lookups based on each element of the
+ * probe tuple, probes are hashed by each of provider, module, function and
+ * name.  (If a lookup is performed based on a regular expression, a
+ * dtrace_probekey is prepared, and a linear search is performed.) Each probe
+ * is additionally pointed to by a linear array indexed by its identifier.  The
+ * identifier is the provider's mechanism for indicating to the DTrace
+ * framework that a probe has fired:  the identifier is passed as the first
+ * argument to dtrace_probe(), where it is then mapped into the corresponding
+ * dtrace_probe structure.  From the dtrace_probe structure, dtrace_probe() can
+ * iterate over the probe's list of enabling control blocks; see "DTrace
+ * Enabling Control Blocks", below.)
+ */
+struct dtrace_probe {
+	dtrace_id_t dtpr_id;			/* probe identifier */
+	dtrace_ecb_t *dtpr_ecb;			/* ECB list; see below */
+	dtrace_ecb_t *dtpr_ecb_last;		/* last ECB in list */
+	void *dtpr_arg;				/* provider argument */
+	dtrace_cacheid_t dtpr_predcache;	/* predicate cache ID */
+	int dtpr_aframes;			/* artificial frames */
+	dtrace_provider_t *dtpr_provider;	/* pointer to provider */
+	char *dtpr_mod;				/* probe's module name */
+	char *dtpr_func;			/* probe's function name */
+	char *dtpr_name;			/* probe's name */
+	dtrace_probe_t *dtpr_nextmod;		/* next in module hash */
+	dtrace_probe_t *dtpr_prevmod;		/* previous in module hash */
+	dtrace_probe_t *dtpr_nextfunc;		/* next in function hash */
+	dtrace_probe_t *dtpr_prevfunc;		/* previous in function hash */
+	dtrace_probe_t *dtpr_nextname;		/* next in name hash */
+	dtrace_probe_t *dtpr_prevname;		/* previous in name hash */
+	dtrace_genid_t dtpr_gen;		/* probe generation ID */
+};
+
+typedef int dtrace_probekey_f(const char *, const char *, int);
+
+typedef struct dtrace_probekey {
+	const char *dtpk_prov;			/* provider name to match */
+	dtrace_probekey_f *dtpk_pmatch;		/* provider matching function */
+	const char *dtpk_mod;			/* module name to match */
+	dtrace_probekey_f *dtpk_mmatch;		/* module matching function */
+	const char *dtpk_func;			/* func name to match */
+	dtrace_probekey_f *dtpk_fmatch;		/* func matching function */
+	const char *dtpk_name;			/* name to match */
+	dtrace_probekey_f *dtpk_nmatch;		/* name matching function */
+	dtrace_id_t dtpk_id;			/* identifier to match */
+} dtrace_probekey_t;
+
+typedef struct dtrace_hashbucket {
+	struct dtrace_hashbucket *dthb_next;	/* next on hash chain */
+	dtrace_probe_t *dthb_chain;		/* chain of probes */
+	int dthb_len;				/* number of probes here */
+} dtrace_hashbucket_t;
+
+typedef struct dtrace_hash {
+	dtrace_hashbucket_t **dth_tab;		/* hash table */
+	int dth_size;				/* size of hash table */
+	int dth_mask;				/* mask to index into table */
+	int dth_nbuckets;			/* total number of buckets */
+	uintptr_t dth_nextoffs;			/* offset of next in probe */
+	uintptr_t dth_prevoffs;			/* offset of prev in probe */
+	uintptr_t dth_stroffs;			/* offset of str in probe */
+} dtrace_hash_t;
+
+/*
+ * DTrace Enabling Control Blocks
+ *
+ * When a provider wishes to fire a probe, it calls into dtrace_probe(),
+ * passing the probe identifier as the first argument.  As described above,
+ * dtrace_probe() maps the identifier into a pointer to a dtrace_probe_t
+ * structure.  This structure contains information about the probe, and a
+ * pointer to the list of Enabling Control Blocks (ECBs).  Each ECB points to
+ * DTrace consumer state, and contains an optional predicate, and a list of
+ * actions.  (Shown schematically below.)  The ECB abstraction allows a single
+ * probe to be multiplexed across disjoint consumers, or across disjoint
+ * enablings of a single probe within one consumer.
+ *
+ *   Enabling Control Block
+ *        dtrace_ecb_t
+ * +------------------------+
+ * | dtrace_epid_t ---------+--------------> Enabled Probe ID (EPID)
+ * | dtrace_state_t * ------+--------------> State associated with this ECB
+ * | dtrace_predicate_t * --+---------+
+ * | dtrace_action_t * -----+----+    |
+ * | dtrace_ecb_t * ---+    |    |    |       Predicate (if any)
+ * +-------------------+----+    |    |       dtrace_predicate_t
+ *                     |         |    +---> +--------------------+
+ *                     |         |          | dtrace_difo_t * ---+----> DIFO
+ *                     |         |          +--------------------+
+ *                     |         |
+ *            Next ECB |         |           Action
+ *            (if any) |         |       dtrace_action_t
+ *                     :         +--> +-------------------+
+ *                     :              | dtrace_actkind_t -+------> kind
+ *                     v              | dtrace_difo_t * --+------> DIFO (if any)
+ *                                    | dtrace_recdesc_t -+------> record descr.
+ *                                    | dtrace_action_t * +------+
+ *                                    +-------------------+      |
+ *                                                               | Next action
+ *                               +-------------------------------+  (if any)
+ *                               |
+ *                               |           Action
+ *                               |       dtrace_action_t
+ *                               +--> +-------------------+
+ *                                    | dtrace_actkind_t -+------> kind
+ *                                    | dtrace_difo_t * --+------> DIFO (if any)
+ *                                    | dtrace_action_t * +------+
+ *                                    +-------------------+      |
+ *                                                               | Next action
+ *                               +-------------------------------+  (if any)
+ *                               |
+ *                               :
+ *                               v
+ *
+ *
+ * dtrace_probe() iterates over the ECB list.  If the ECB needs less space
+ * than is available in the principal buffer, the ECB is processed:  if the
+ * predicate is non-NULL, the DIF object is executed.  If the result is
+ * non-zero, the action list is processed, with each action being executed
+ * accordingly.  When the action list has been completely executed, processing
+ * advances to the next ECB.  processing advances to the next ECB.  If the
+ * result is non-zero; For each ECB, it first determines the The ECB
+ * abstraction allows disjoint consumers to multiplex on single probes.
+ */
+struct dtrace_ecb {
+	dtrace_epid_t dte_epid;			/* enabled probe ID */
+	uint32_t dte_alignment;			/* required alignment */
+	size_t dte_needed;			/* bytes needed */
+	size_t dte_size;			/* total size of payload */
+	dtrace_predicate_t *dte_predicate;	/* predicate, if any */
+	dtrace_action_t *dte_action;		/* actions, if any */
+	dtrace_ecb_t *dte_next;			/* next ECB on probe */
+	dtrace_state_t *dte_state;		/* pointer to state */
+	uint32_t dte_cond;			/* security condition */
+	dtrace_probe_t *dte_probe;		/* pointer to probe */
+	dtrace_action_t *dte_action_last;	/* last action on ECB */
+	uint64_t dte_uarg;			/* library argument */
+};
+
+struct dtrace_predicate {
+	dtrace_difo_t *dtp_difo;		/* DIF object */
+	dtrace_cacheid_t dtp_cacheid;		/* cache identifier */
+	int dtp_refcnt;				/* reference count */
+};
+
+struct dtrace_action {
+	dtrace_actkind_t dta_kind;		/* kind of action */
+	uint16_t dta_intuple;			/* boolean:  in aggregation */
+	uint32_t dta_refcnt;			/* reference count */
+	dtrace_difo_t *dta_difo;		/* pointer to DIFO */
+	dtrace_recdesc_t dta_rec;		/* record description */
+	dtrace_action_t *dta_prev;		/* previous action */
+	dtrace_action_t *dta_next;		/* next action */
+};
+
+typedef struct dtrace_aggregation {
+	dtrace_action_t dtag_action;		/* action; must be first */
+	dtrace_aggid_t dtag_id;			/* identifier */
+	dtrace_ecb_t *dtag_ecb;			/* corresponding ECB */
+	dtrace_action_t *dtag_first;		/* first action in tuple */
+	uint32_t dtag_base;			/* base of aggregation */
+	uint8_t dtag_hasarg;			/* boolean:  has argument */
+	uint64_t dtag_initial;			/* initial value */
+	void (*dtag_aggregate)(uint64_t *, uint64_t, uint64_t);
+} dtrace_aggregation_t;
+
+/*
+ * DTrace Buffers
+ *
+ * Principal buffers, aggregation buffers, and speculative buffers are all
+ * managed with the dtrace_buffer structure.  By default, this structure
+ * includes twin data buffers -- dtb_tomax and dtb_xamot -- that serve as the
+ * active and passive buffers, respectively.  For speculative buffers,
+ * dtb_xamot will be NULL; for "ring" and "fill" buffers, dtb_xamot will point
+ * to a scratch buffer.  For all buffer types, the dtrace_buffer structure is
+ * always allocated on a per-CPU basis; a single dtrace_buffer structure is
+ * never shared among CPUs.  (That is, there is never true sharing of the
+ * dtrace_buffer structure; to prevent false sharing of the structure, it must
+ * always be aligned to the coherence granularity -- generally 64 bytes.)
+ *
+ * One of the critical design decisions of DTrace is that a given ECB always
+ * stores the same quantity and type of data.  This is done to assure that the
+ * only metadata required for an ECB's traced data is the EPID.  That is, from
+ * the EPID, the consumer can determine the data layout.  (The data buffer
+ * layout is shown schematically below.)  By assuring that one can determine
+ * data layout from the EPID, the metadata stream can be separated from the
+ * data stream -- simplifying the data stream enormously.
+ *
+ *      base of data buffer --->  +------+--------------------+------+
+ *                                | EPID | data               | EPID |
+ *                                +------+--------+------+----+------+
+ *                                | data          | EPID | data      |
+ *                                +---------------+------+-----------+
+ *                                | data, cont.                      |
+ *                                +------+--------------------+------+
+ *                                | EPID | data               |      |
+ *                                +------+--------------------+      |
+ *                                |                ||                |
+ *                                |                ||                |
+ *                                |                \/                |
+ *                                :                                  :
+ *                                .                                  .
+ *                                .                                  .
+ *                                .                                  .
+ *                                :                                  :
+ *                                |                                  |
+ *     limit of data buffer --->  +----------------------------------+
+ *
+ * When evaluating an ECB, dtrace_probe() determines if the ECB's needs of the
+ * principal buffer (both scratch and payload) exceed the available space.  If
+ * the ECB's needs exceed available space (and if the principal buffer policy
+ * is the default "switch" policy), the ECB is dropped, the buffer's drop count
+ * is incremented, and processing advances to the next ECB.  If the ECB's needs
+ * can be met with the available space, the ECB is processed, but the offset in
+ * the principal buffer is only advanced if the ECB completes processing
+ * without error.
+ *
+ * When a buffer is to be switched (either because the buffer is the principal
+ * buffer with a "switch" policy or because it is an aggregation buffer), a
+ * cross call is issued to the CPU associated with the buffer.  In the cross
+ * call context, interrupts are disabled, and the active and the inactive
+ * buffers are atomically switched.  This involves switching the data pointers,
+ * copying the various state fields (offset, drops, errors, etc.) into their
+ * inactive equivalents, and clearing the state fields.  Because interrupts are
+ * disabled during this procedure, the switch is guaranteed to appear atomic to
+ * dtrace_probe().
+ *
+ * DTrace Ring Buffering
+ *
+ * To process a ring buffer correctly, one must know the oldest valid record.
+ * Processing starts at the oldest record in the buffer and continues until
+ * the end of the buffer is reached.  Processing then resumes starting with
+ * the record stored at offset 0 in the buffer, and continues until the
+ * youngest record is processed.  If trace records are of a fixed-length,
+ * determining the oldest record is trivial:
+ *
+ *   - If the ring buffer has not wrapped, the oldest record is the record
+ *     stored at offset 0.
+ *
+ *   - If the ring buffer has wrapped, the oldest record is the record stored
+ *     at the current offset.
+ *
+ * With variable length records, however, just knowing the current offset
+ * doesn't suffice for determining the oldest valid record:  assuming that one
+ * allows for arbitrary data, one has no way of searching forward from the
+ * current offset to find the oldest valid record.  (That is, one has no way
+ * of separating data from metadata.) It would be possible to simply refuse to
+ * process any data in the ring buffer between the current offset and the
+ * limit, but this leaves (potentially) an enormous amount of otherwise valid
+ * data unprocessed.
+ *
+ * To effect ring buffering, we track two offsets in the buffer:  the current
+ * offset and the _wrapped_ offset.  If a request is made to reserve some
+ * amount of data, and the buffer has wrapped, the wrapped offset is
+ * incremented until the wrapped offset minus the current offset is greater
+ * than or equal to the reserve request.  This is done by repeatedly looking
+ * up the ECB corresponding to the EPID at the current wrapped offset, and
+ * incrementing the wrapped offset by the size of the data payload
+ * corresponding to that ECB.  If this offset is greater than or equal to the
+ * limit of the data buffer, the wrapped offset is set to 0.  Thus, the
+ * current offset effectively "chases" the wrapped offset around the buffer.
+ * Schematically:
+ *
+ *      base of data buffer --->  +------+--------------------+------+
+ *                                | EPID | data               | EPID |
+ *                                +------+--------+------+----+------+
+ *                                | data          | EPID | data      |
+ *                                +---------------+------+-----------+
+ *                                | data, cont.                      |
+ *                                +------+---------------------------+
+ *                                | EPID | data                      |
+ *           current offset --->  +------+---------------------------+
+ *                                | invalid data                     |
+ *           wrapped offset --->  +------+--------------------+------+
+ *                                | EPID | data               | EPID |
+ *                                +------+--------+------+----+------+
+ *                                | data          | EPID | data      |
+ *                                +---------------+------+-----------+
+ *                                :                                  :
+ *                                .                                  .
+ *                                .        ... valid data ...        .
+ *                                .                                  .
+ *                                :                                  :
+ *                                +------+-------------+------+------+
+ *                                | EPID | data        | EPID | data |
+ *                                +------+------------++------+------+
+ *                                | data, cont.       | leftover     |
+ *     limit of data buffer --->  +-------------------+--------------+
+ *
+ * If the amount of requested buffer space exceeds the amount of space
+ * available between the current offset and the end of the buffer:
+ *
+ *  (1)  all words in the data buffer between the current offset and the limit
+ *       of the data buffer (marked "leftover", above) are set to
+ *       DTRACE_EPIDNONE
+ *
+ *  (2)  the wrapped offset is set to zero
+ *
+ *  (3)  the iteration process described above occurs until the wrapped offset
+ *       is greater than the amount of desired space.
+ *
+ * The wrapped offset is implemented by (re-)using the inactive offset.
+ * In a "switch" buffer policy, the inactive offset stores the offset in
+ * the inactive buffer; in a "ring" buffer policy, it stores the wrapped
+ * offset.
+ *
+ * DTrace Scratch Buffering
+ *
+ * Some ECBs may wish to allocate dynamically-sized temporary scratch memory.
+ * To accommodate such requests easily, scratch memory may be allocated in
+ * the buffer beyond the current offset plus the needed memory of the current
+ * ECB.  If there isn't sufficient room in the buffer for the requested amount
+ * of scratch space, the allocation fails and an error is generated.  Scratch
+ * memory is tracked in the dtrace_mstate_t and is automatically freed when
+ * the ECB ceases processing.  Note that ring buffers cannot allocate their
+ * scratch from the principal buffer -- lest they needlessly overwrite older,
+ * valid data.  Ring buffers therefore have their own dedicated scratch buffer
+ * from which scratch is allocated.
+ */
+#define	DTRACEBUF_RING		0x0001		/* bufpolicy set to "ring" */
+#define	DTRACEBUF_FILL		0x0002		/* bufpolicy set to "fill" */
+#define	DTRACEBUF_NOSWITCH	0x0004		/* do not switch buffer */
+#define	DTRACEBUF_WRAPPED	0x0008		/* ring buffer has wrapped */
+#define	DTRACEBUF_DROPPED	0x0010		/* drops occurred */
+#define	DTRACEBUF_ERROR		0x0020		/* errors occurred */
+#define	DTRACEBUF_FULL		0x0040		/* "fill" buffer is full */
+#define	DTRACEBUF_CONSUMED	0x0080		/* buffer has been consumed */
+#define	DTRACEBUF_INACTIVE	0x0100		/* buffer is not yet active */
+
+typedef struct dtrace_buffer {
+	uint64_t dtb_offset;			/* current offset in buffer */
+	uint64_t dtb_size;			/* size of buffer */
+	uint32_t dtb_flags;			/* flags */
+	uint32_t dtb_drops;			/* number of drops */
+	caddr_t dtb_tomax;			/* active buffer */
+	caddr_t dtb_xamot;			/* inactive buffer */
+	uint32_t dtb_xamot_flags;		/* inactive flags */
+	uint32_t dtb_xamot_drops;		/* drops in inactive buffer */
+	uint64_t dtb_xamot_offset;		/* offset in inactive buffer */
+	uint32_t dtb_errors;			/* number of errors */
+	uint32_t dtb_xamot_errors;		/* errors in inactive buffer */
+#ifndef _LP64
+	uint64_t dtb_pad1;
+#endif
+} dtrace_buffer_t;
+
+/*
+ * DTrace Aggregation Buffers
+ *
+ * Aggregation buffers use much of the same mechanism as described above
+ * ("DTrace Buffers").  However, because an aggregation is fundamentally a
+ * hash, there exists dynamic metadata associated with an aggregation buffer
+ * that is not associated with other kinds of buffers.  This aggregation
+ * metadata is _only_ relevant for the in-kernel implementation of
+ * aggregations; it is not actually relevant to user-level consumers.  To do
+ * this, we allocate dynamic aggregation data (hash keys and hash buckets)
+ * starting below the _limit_ of the buffer, and we allocate data from the
+ * _base_ of the buffer.  When the aggregation buffer is copied out, _only_ the
+ * data is copied out; the metadata is simply discarded.  Schematically,
+ * aggregation buffers look like:
+ *
+ *      base of data buffer --->  +-------+------+-----------+-------+
+ *                                | aggid | key  | value     | aggid |
+ *                                +-------+------+-----------+-------+
+ *                                | key                              |
+ *                                +-------+-------+-----+------------+
+ *                                | value | aggid | key | value      |
+ *                                +-------+------++-----+------+-----+
+ *                                | aggid | key  | value       |     |
+ *                                +-------+------+-------------+     |
+ *                                |                ||                |
+ *                                |                ||                |
+ *                                |                \/                |
+ *                                :                                  :
+ *                                .                                  .
+ *                                .                                  .
+ *                                .                                  .
+ *                                :                                  :
+ *                                |                /\                |
+ *                                |                ||   +------------+
+ *                                |                ||   |            |
+ *                                +---------------------+            |
+ *                                | hash keys                        |
+ *                                | (dtrace_aggkey structures)       |
+ *                                |                                  |
+ *                                +----------------------------------+
+ *                                | hash buckets                     |
+ *                                | (dtrace_aggbuffer structure)     |
+ *                                |                                  |
+ *     limit of data buffer --->  +----------------------------------+
+ *
+ *
+ * As implied above, just as we assure that ECBs always store a constant
+ * amount of data, we assure that a given aggregation -- identified by its
+ * aggregation ID -- always stores data of a constant quantity and type.
+ * As with EPIDs, this allows the aggregation ID to serve as the metadata for a
+ * given record.
+ *
+ * Note that the size of the dtrace_aggkey structure must be sizeof (uintptr_t)
+ * aligned.  (If this the structure changes such that this becomes false, an
+ * assertion will fail in dtrace_aggregate().)
+ */
+typedef struct dtrace_aggkey {
+	uint32_t dtak_hashval;			/* hash value */
+	uint32_t dtak_action:4;			/* action -- 4 bits */
+	uint32_t dtak_size:28;			/* size -- 28 bits */
+	caddr_t dtak_data;			/* data pointer */
+	struct dtrace_aggkey *dtak_next;	/* next in hash chain */
+} dtrace_aggkey_t;
+
+typedef struct dtrace_aggbuffer {
+	uintptr_t dtagb_hashsize;		/* number of buckets */
+	uintptr_t dtagb_free;			/* free list of keys */
+	dtrace_aggkey_t **dtagb_hash;		/* hash table */
+} dtrace_aggbuffer_t;
+
+/*
+ * DTrace Speculations
+ *
+ * Speculations have a per-CPU buffer and a global state.  Once a speculation
+ * buffer has been comitted or discarded, it cannot be reused until all CPUs
+ * have taken the same action (commit or discard) on their respective
+ * speculative buffer.  However, because DTrace probes may execute in arbitrary
+ * context, other CPUs cannot simply be cross-called at probe firing time to
+ * perform the necessary commit or discard.  The speculation states thus
+ * optimize for the case that a speculative buffer is only active on one CPU at
+ * the time of a commit() or discard() -- for if this is the case, other CPUs
+ * need not take action, and the speculation is immediately available for
+ * reuse.  If the speculation is active on multiple CPUs, it must be
+ * asynchronously cleaned -- potentially leading to a higher rate of dirty
+ * speculative drops.  The speculation states are as follows:
+ *
+ *  DTRACESPEC_INACTIVE       <= Initial state; inactive speculation
+ *  DTRACESPEC_ACTIVE         <= Allocated, but not yet speculatively traced to
+ *  DTRACESPEC_ACTIVEONE      <= Speculatively traced to on one CPU
+ *  DTRACESPEC_ACTIVEMANY     <= Speculatively traced to on more than one CPU
+ *  DTRACESPEC_COMMITTING     <= Currently being commited on one CPU
+ *  DTRACESPEC_COMMITTINGMANY <= Currently being commited on many CPUs
+ *  DTRACESPEC_DISCARDING     <= Currently being discarded on many CPUs
+ *
+ * The state transition diagram is as follows:
+ *
+ *     +----------------------------------------------------------+
+ *     |                                                          |
+ *     |                      +------------+                      |
+ *     |  +-------------------| COMMITTING |<-----------------+   |
+ *     |  |                   +------------+                  |   |
+ *     |  | copied spec.            ^             commit() on |   | discard() on
+ *     |  | into principal          |              active CPU |   | active CPU
+ *     |  |                         | commit()                |   |
+ *     V  V                         |                         |   |
+ * +----------+                 +--------+                +-----------+
+ * | INACTIVE |---------------->| ACTIVE |--------------->| ACTIVEONE |
+ * +----------+  speculation()  +--------+  speculate()   +-----------+
+ *     ^  ^                         |                         |   |
+ *     |  |                         | discard()               |   |
+ *     |  | asynchronously          |            discard() on |   | speculate()
+ *     |  | cleaned                 V            inactive CPU |   | on inactive
+ *     |  |                   +------------+                  |   | CPU
+ *     |  +-------------------| DISCARDING |<-----------------+   |
+ *     |                      +------------+                      |
+ *     | asynchronously             ^                             |
+ *     | copied spec.               |       discard()             |
+ *     | into principal             +------------------------+    |
+ *     |                                                     |    V
+ *  +----------------+             commit()              +------------+
+ *  | COMMITTINGMANY |<----------------------------------| ACTIVEMANY |
+ *  +----------------+                                   +------------+
+ */
+typedef enum dtrace_speculation_state {
+	DTRACESPEC_INACTIVE = 0,
+	DTRACESPEC_ACTIVE,
+	DTRACESPEC_ACTIVEONE,
+	DTRACESPEC_ACTIVEMANY,
+	DTRACESPEC_COMMITTING,
+	DTRACESPEC_COMMITTINGMANY,
+	DTRACESPEC_DISCARDING
+} dtrace_speculation_state_t;
+
+typedef struct dtrace_speculation {
+	dtrace_speculation_state_t dtsp_state;	/* current speculation state */
+	int dtsp_cleaning;			/* non-zero if being cleaned */
+	dtrace_buffer_t *dtsp_buffer;		/* speculative buffer */
+} dtrace_speculation_t;
+
+/*
+ * DTrace Dynamic Variables
+ *
+ * The dynamic variable problem is obviously decomposed into two subproblems:
+ * allocating new dynamic storage, and freeing old dynamic storage.  The
+ * presence of the second problem makes the first much more complicated -- or
+ * rather, the absence of the second renders the first trivial.  This is the
+ * case with aggregations, for which there is effectively no deallocation of
+ * dynamic storage.  (Or more accurately, all dynamic storage is deallocated
+ * when a snapshot is taken of the aggregation.)  As DTrace dynamic variables
+ * allow for both dynamic allocation and dynamic deallocation, the
+ * implementation of dynamic variables is quite a bit more complicated than
+ * that of their aggregation kin.
+ *
+ * We observe that allocating new dynamic storage is tricky only because the
+ * size can vary -- the allocation problem is much easier if allocation sizes
+ * are uniform.  We further observe that in D, the size of dynamic variables is
+ * actually _not_ dynamic -- dynamic variable sizes may be determined by static
+ * analysis of DIF text.  (This is true even of putatively dynamically-sized
+ * objects like strings and stacks, the sizes of which are dictated by the
+ * "stringsize" and "stackframes" variables, respectively.)  We exploit this by
+ * performing this analysis on all DIF before enabling any probes.  For each
+ * dynamic load or store, we calculate the dynamically-allocated size plus the
+ * size of the dtrace_dynvar structure plus the storage required to key the
+ * data.  For all DIF, we take the largest value and dub it the _chunksize_.
+ * We then divide dynamic memory into two parts:  a hash table that is wide
+ * enough to have every chunk in its own bucket, and a larger region of equal
+ * chunksize units.  Whenever we wish to dynamically allocate a variable, we
+ * always allocate a single chunk of memory.  Depending on the uniformity of
+ * allocation, this will waste some amount of memory -- but it eliminates the
+ * non-determinism inherent in traditional heap fragmentation.
+ *
+ * Dynamic objects are allocated by storing a non-zero value to them; they are
+ * deallocated by storing a zero value to them.  Dynamic variables are
+ * complicated enormously by being shared between CPUs.  In particular,
+ * consider the following scenario:
+ *
+ *                 CPU A                                 CPU B
+ *  +---------------------------------+   +---------------------------------+
+ *  |                                 |   |                                 |
+ *  | allocates dynamic object a[123] |   |                                 |
+ *  | by storing the value 345 to it  |   |                                 |
+ *  |                               --------->                              |
+ *  |                                 |   | wishing to load from object     |
+ *  |                                 |   | a[123], performs lookup in      |
+ *  |                                 |   | dynamic variable space          |
+ *  |                               <---------                              |
+ *  | deallocates object a[123] by    |   |                                 |
+ *  | storing 0 to it                 |   |                                 |
+ *  |                                 |   |                                 |
+ *  | allocates dynamic object b[567] |   | performs load from a[123]       |
+ *  | by storing the value 789 to it  |   |                                 |
+ *  :                                 :   :                                 :
+ *  .                                 .   .                                 .
+ *
+ * This is obviously a race in the D program, but there are nonetheless only
+ * two valid values for CPU B's load from a[123]:  345 or 0.  Most importantly,
+ * CPU B may _not_ see the value 789 for a[123].
+ *
+ * There are essentially two ways to deal with this:
+ *
+ *  (1)  Explicitly spin-lock variables.  That is, if CPU B wishes to load
+ *       from a[123], it needs to lock a[123] and hold the lock for the
+ *       duration that it wishes to manipulate it.
+ *
+ *  (2)  Avoid reusing freed chunks until it is known that no CPU is referring
+ *       to them.
+ *
+ * The implementation of (1) is rife with complexity, because it requires the
+ * user of a dynamic variable to explicitly decree when they are done using it.
+ * Were all variables by value, this perhaps wouldn't be debilitating -- but
+ * dynamic variables of non-scalar types are tracked by reference.  That is, if
+ * a dynamic variable is, say, a string, and that variable is to be traced to,
+ * say, the principal buffer, the DIF emulation code returns to the main
+ * dtrace_probe() loop a pointer to the underlying storage, not the contents of
+ * the storage.  Further, code calling on DIF emulation would have to be aware
+ * that the DIF emulation has returned a reference to a dynamic variable that
+ * has been potentially locked.  The variable would have to be unlocked after
+ * the main dtrace_probe() loop is finished with the variable, and the main
+ * dtrace_probe() loop would have to be careful to not call any further DIF
+ * emulation while the variable is locked to avoid deadlock.  More generally,
+ * if one were to implement (1), DIF emulation code dealing with dynamic
+ * variables could only deal with one dynamic variable at a time (lest deadlock
+ * result).  To sum, (1) exports too much subtlety to the users of dynamic
+ * variables -- increasing maintenance burden and imposing serious constraints
+ * on future DTrace development.
+ *
+ * The implementation of (2) is also complex, but the complexity is more
+ * manageable.  We need to be sure that when a variable is deallocated, it is
+ * not placed on a traditional free list, but rather on a _dirty_ list.  Once a
+ * variable is on a dirty list, it cannot be found by CPUs performing a
+ * subsequent lookup of the variable -- but it may still be in use by other
+ * CPUs.  To assure that all CPUs that may be seeing the old variable have
+ * cleared out of probe context, a dtrace_sync() can be issued.  Once the
+ * dtrace_sync() has completed, it can be known that all CPUs are done
+ * manipulating the dynamic variable -- the dirty list can be atomically
+ * appended to the free list.  Unfortunately, there's a slight hiccup in this
+ * mechanism:  dtrace_sync() may not be issued from probe context.  The
+ * dtrace_sync() must be therefore issued asynchronously from non-probe
+ * context.  For this we rely on the DTrace cleaner, a cyclic that runs at the
+ * "cleanrate" frequency.  To ease this implementation, we define several chunk
+ * lists:
+ *
+ *   - Dirty.  Deallocated chunks, not yet cleaned.  Not available.
+ *
+ *   - Rinsing.  Formerly dirty chunks that are currently being asynchronously
+ *     cleaned.  Not available, but will be shortly.  Dynamic variable
+ *     allocation may not spin or block for availability, however.
+ *
+ *   - Clean.  Clean chunks, ready for allocation -- but not on the free list.
+ *
+ *   - Free.  Available for allocation.
+ *
+ * Moreover, to avoid absurd contention, _each_ of these lists is implemented
+ * on a per-CPU basis.  This is only for performance, not correctness; chunks
+ * may be allocated from another CPU's free list.  The algorithm for allocation
+ * then is this:
+ *
+ *   (1)  Attempt to atomically allocate from current CPU's free list.  If list
+ *        is non-empty and allocation is successful, allocation is complete.
+ *
+ *   (2)  If the clean list is non-empty, atomically move it to the free list,
+ *        and reattempt (1).
+ *
+ *   (3)  If the dynamic variable space is in the CLEAN state, look for free
+ *        and clean lists on other CPUs by setting the current CPU to the next
+ *        CPU, and reattempting (1).  If the next CPU is the current CPU (that
+ *        is, if all CPUs have been checked), atomically switch the state of
+ *        the dynamic variable space based on the following:
+ *
+ *        - If no free chunks were found and no dirty chunks were found,
+ *          atomically set the state to EMPTY.
+ *
+ *        - If dirty chunks were found, atomically set the state to DIRTY.
+ *
+ *        - If rinsing chunks were found, atomically set the state to RINSING.
+ *
+ *   (4)  Based on state of dynamic variable space state, increment appropriate
+ *        counter to indicate dynamic drops (if in EMPTY state) vs. dynamic
+ *        dirty drops (if in DIRTY state) vs. dynamic rinsing drops (if in
+ *        RINSING state).  Fail the allocation.
+ *
+ * The cleaning cyclic operates with the following algorithm:  for all CPUs
+ * with a non-empty dirty list, atomically move the dirty list to the rinsing
+ * list.  Perform a dtrace_sync().  For all CPUs with a non-empty rinsing list,
+ * atomically move the rinsing list to the clean list.  Perform another
+ * dtrace_sync().  By this point, all CPUs have seen the new clean list; the
+ * state of the dynamic variable space can be restored to CLEAN.
+ *
+ * There exist two final races that merit explanation.  The first is a simple
+ * allocation race:
+ *
+ *                 CPU A                                 CPU B
+ *  +---------------------------------+   +---------------------------------+
+ *  |                                 |   |                                 |
+ *  | allocates dynamic object a[123] |   | allocates dynamic object a[123] |
+ *  | by storing the value 345 to it  |   | by storing the value 567 to it  |
+ *  |                                 |   |                                 |
+ *  :                                 :   :                                 :
+ *  .                                 .   .                                 .
+ *
+ * Again, this is a race in the D program.  It can be resolved by having a[123]
+ * hold the value 345 or a[123] hold the value 567 -- but it must be true that
+ * a[123] have only _one_ of these values.  (That is, the racing CPUs may not
+ * put the same element twice on the same hash chain.)  This is resolved
+ * simply:  before the allocation is undertaken, the start of the new chunk's
+ * hash chain is noted.  Later, after the allocation is complete, the hash
+ * chain is atomically switched to point to the new element.  If this fails
+ * (because of either concurrent allocations or an allocation concurrent with a
+ * deletion), the newly allocated chunk is deallocated to the dirty list, and
+ * the whole process of looking up (and potentially allocating) the dynamic
+ * variable is reattempted.
+ *
+ * The final race is a simple deallocation race:
+ *
+ *                 CPU A                                 CPU B
+ *  +---------------------------------+   +---------------------------------+
+ *  |                                 |   |                                 |
+ *  | deallocates dynamic object      |   | deallocates dynamic object      |
+ *  | a[123] by storing the value 0   |   | a[123] by storing the value 0   |
+ *  | to it                           |   | to it                           |
+ *  |                                 |   |                                 |
+ *  :                                 :   :                                 :
+ *  .                                 .   .                                 .
+ *
+ * Once again, this is a race in the D program, but it is one that we must
+ * handle without corrupting the underlying data structures.  Because
+ * deallocations require the deletion of a chunk from the middle of a hash
+ * chain, we cannot use a single-word atomic operation to remove it.  For this,
+ * we add a spin lock to the hash buckets that is _only_ used for deallocations
+ * (allocation races are handled as above).  Further, this spin lock is _only_
+ * held for the duration of the delete; before control is returned to the DIF
+ * emulation code, the hash bucket is unlocked.
+ */
+typedef struct dtrace_key {
+	uint64_t dttk_value;			/* data value or data pointer */
+	uint64_t dttk_size;			/* 0 if by-val, >0 if by-ref */
+} dtrace_key_t;
+
+typedef struct dtrace_tuple {
+	uint32_t dtt_nkeys;			/* number of keys in tuple */
+	uint32_t dtt_pad;			/* padding */
+	dtrace_key_t dtt_key[1];		/* array of tuple keys */
+} dtrace_tuple_t;
+
+typedef struct dtrace_dynvar {
+	uint64_t dtdv_hashval;			/* hash value -- 0 if free */
+	struct dtrace_dynvar *dtdv_next;	/* next on list or hash chain */
+	void *dtdv_data;			/* pointer to data */
+	dtrace_tuple_t dtdv_tuple;		/* tuple key */
+} dtrace_dynvar_t;
+
+typedef enum dtrace_dynvar_op {
+	DTRACE_DYNVAR_ALLOC,
+	DTRACE_DYNVAR_NOALLOC,
+	DTRACE_DYNVAR_DEALLOC
+} dtrace_dynvar_op_t;
+
+typedef struct dtrace_dynhash {
+	dtrace_dynvar_t *dtdh_chain;		/* hash chain for this bucket */
+	uintptr_t dtdh_lock;			/* deallocation lock */
+#ifdef _LP64
+	uintptr_t dtdh_pad[6];			/* pad to avoid false sharing */
+#else
+	uintptr_t dtdh_pad[14];			/* pad to avoid false sharing */
+#endif
+} dtrace_dynhash_t;
+
+typedef struct dtrace_dstate_percpu {
+	dtrace_dynvar_t *dtdsc_free;		/* free list for this CPU */
+	dtrace_dynvar_t *dtdsc_dirty;		/* dirty list for this CPU */
+	dtrace_dynvar_t *dtdsc_rinsing;		/* rinsing list for this CPU */
+	dtrace_dynvar_t *dtdsc_clean;		/* clean list for this CPU */
+	uint64_t dtdsc_drops;			/* number of capacity drops */
+	uint64_t dtdsc_dirty_drops;		/* number of dirty drops */
+	uint64_t dtdsc_rinsing_drops;		/* number of rinsing drops */
+#ifdef _LP64
+	uint64_t dtdsc_pad;			/* pad to avoid false sharing */
+#else
+	uint64_t dtdsc_pad[2];			/* pad to avoid false sharing */
+#endif
+} dtrace_dstate_percpu_t;
+
+typedef enum dtrace_dstate_state {
+	DTRACE_DSTATE_CLEAN = 0,
+	DTRACE_DSTATE_EMPTY,
+	DTRACE_DSTATE_DIRTY,
+	DTRACE_DSTATE_RINSING
+} dtrace_dstate_state_t;
+
+typedef struct dtrace_dstate {
+	void *dtds_base;			/* base of dynamic var. space */
+	size_t dtds_size;			/* size of dynamic var. space */
+	size_t dtds_hashsize;			/* number of buckets in hash */
+	size_t dtds_chunksize;			/* size of each chunk */
+	dtrace_dynhash_t *dtds_hash;		/* pointer to hash table */
+	dtrace_dstate_state_t dtds_state;	/* current dynamic var. state */
+	dtrace_dstate_percpu_t *dtds_percpu;	/* per-CPU dyn. var. state */
+} dtrace_dstate_t;
+
+/*
+ * DTrace Variable State
+ *
+ * The DTrace variable state tracks user-defined variables in its dtrace_vstate
+ * structure.  Each DTrace consumer has exactly one dtrace_vstate structure,
+ * but some dtrace_vstate structures may exist without a corresponding DTrace
+ * consumer (see "DTrace Helpers", below).  As described in <sys/dtrace.h>,
+ * user-defined variables can have one of three scopes:
+ *
+ *  DIFV_SCOPE_GLOBAL  =>  global scope
+ *  DIFV_SCOPE_THREAD  =>  thread-local scope (i.e. "self->" variables)
+ *  DIFV_SCOPE_LOCAL   =>  clause-local scope (i.e. "this->" variables)
+ *
+ * The variable state tracks variables by both their scope and their allocation
+ * type:
+ *
+ *  - The dtvs_globals and dtvs_locals members each point to an array of
+ *    dtrace_statvar structures.  These structures contain both the variable
+ *    metadata (dtrace_difv structures) and the underlying storage for all
+ *    statically allocated variables, including statically allocated
+ *    DIFV_SCOPE_GLOBAL variables and all DIFV_SCOPE_LOCAL variables.
+ *
+ *  - The dtvs_tlocals member points to an array of dtrace_difv structures for
+ *    DIFV_SCOPE_THREAD variables.  As such, this array tracks _only_ the
+ *    variable metadata for DIFV_SCOPE_THREAD variables; the underlying storage
+ *    is allocated out of the dynamic variable space.
+ *
+ *  - The dtvs_dynvars member is the dynamic variable state associated with the
+ *    variable state.  The dynamic variable state (described in "DTrace Dynamic
+ *    Variables", above) tracks all DIFV_SCOPE_THREAD variables and all
+ *    dynamically-allocated DIFV_SCOPE_GLOBAL variables.
+ */
+typedef struct dtrace_statvar {
+	uint64_t dtsv_data;			/* data or pointer to it */
+	size_t dtsv_size;			/* size of pointed-to data */
+	int dtsv_refcnt;			/* reference count */
+	dtrace_difv_t dtsv_var;			/* variable metadata */
+} dtrace_statvar_t;
+
+typedef struct dtrace_vstate {
+	dtrace_state_t *dtvs_state;		/* back pointer to state */
+	dtrace_statvar_t **dtvs_globals;	/* statically-allocated glbls */
+	int dtvs_nglobals;			/* number of globals */
+	dtrace_difv_t *dtvs_tlocals;		/* thread-local metadata */
+	int dtvs_ntlocals;			/* number of thread-locals */
+	dtrace_statvar_t **dtvs_locals;		/* clause-local data */
+	int dtvs_nlocals;			/* number of clause-locals */
+	dtrace_dstate_t dtvs_dynvars;		/* dynamic variable state */
+} dtrace_vstate_t;
+
+/*
+ * DTrace Machine State
+ *
+ * In the process of processing a fired probe, DTrace needs to track and/or
+ * cache some per-CPU state associated with that particular firing.  This is
+ * state that is always discarded after the probe firing has completed, and
+ * much of it is not specific to any DTrace consumer, remaining valid across
+ * all ECBs.  This state is tracked in the dtrace_mstate structure.
+ */
+#define	DTRACE_MSTATE_ARGS		0x00000001
+#define	DTRACE_MSTATE_PROBE		0x00000002
+#define	DTRACE_MSTATE_EPID		0x00000004
+#define	DTRACE_MSTATE_TIMESTAMP		0x00000008
+#define	DTRACE_MSTATE_STACKDEPTH	0x00000010
+#define	DTRACE_MSTATE_CALLER		0x00000020
+#define	DTRACE_MSTATE_IPL		0x00000040
+#define	DTRACE_MSTATE_FLTOFFS		0x00000080
+#define	DTRACE_MSTATE_WALLTIMESTAMP	0x00000100
+#define	DTRACE_MSTATE_USTACKDEPTH	0x00000200
+#define	DTRACE_MSTATE_UCALLER		0x00000400
+
+typedef struct dtrace_mstate {
+	uintptr_t dtms_scratch_base;		/* base of scratch space */
+	uintptr_t dtms_scratch_ptr;		/* current scratch pointer */
+	size_t dtms_scratch_size;		/* scratch size */
+	uint32_t dtms_present;			/* variables that are present */
+	uint64_t dtms_arg[5];			/* cached arguments */
+	dtrace_epid_t dtms_epid;		/* current EPID */
+	uint64_t dtms_timestamp;		/* cached timestamp */
+	hrtime_t dtms_walltimestamp;		/* cached wall timestamp */
+	int dtms_stackdepth;			/* cached stackdepth */
+	int dtms_ustackdepth;			/* cached ustackdepth */
+	struct dtrace_probe *dtms_probe;	/* current probe */
+	uintptr_t dtms_caller;			/* cached caller */
+	uint64_t dtms_ucaller;			/* cached user-level caller */
+	int dtms_ipl;				/* cached interrupt pri lev */
+	int dtms_fltoffs;			/* faulting DIFO offset */
+	uintptr_t dtms_strtok;			/* saved strtok() pointer */
+} dtrace_mstate_t;
+
+#define	DTRACE_COND_OWNER	0x1
+#define	DTRACE_COND_USERMODE	0x2
+#define	DTRACE_COND_ZONEOWNER	0x4
+
+#define	DTRACE_PROBEKEY_MAXDEPTH	8	/* max glob recursion depth */
+
+/*
+ * DTrace Activity
+ *
+ * Each DTrace consumer is in one of several states, which (for purposes of
+ * avoiding yet-another overloading of the noun "state") we call the current
+ * _activity_.  The activity transitions on dtrace_go() (from DTRACIOCGO), on
+ * dtrace_stop() (from DTRACIOCSTOP) and on the exit() action.  Activities may
+ * only transition in one direction; the activity transition diagram is a
+ * directed acyclic graph.  The activity transition diagram is as follows:
+ *
+ *
+ * +----------+                   +--------+                   +--------+
+ * | INACTIVE |------------------>| WARMUP |------------------>| ACTIVE |
+ * +----------+   dtrace_go(),    +--------+   dtrace_go(),    +--------+
+ *                before BEGIN        |        after BEGIN       |  |  |
+ *                                    |                          |  |  |
+ *                      exit() action |                          |  |  |
+ *                     from BEGIN ECB |                          |  |  |
+ *                                    |                          |  |  |
+ *                                    v                          |  |  |
+ *                               +----------+     exit() action  |  |  |
+ * +-----------------------------| DRAINING |<-------------------+  |  |
+ * |                             +----------+                       |  |
+ * |                                  |                             |  |
+ * |                   dtrace_stop(), |                             |  |
+ * |                     before END   |                             |  |
+ * |                                  |                             |  |
+ * |                                  v                             |  |
+ * | +---------+                 +----------+                       |  |
+ * | | STOPPED |<----------------| COOLDOWN |<----------------------+  |
+ * | +---------+  dtrace_stop(), +----------+     dtrace_stop(),       |
+ * |                after END                       before END         |
+ * |                                                                   |
+ * |                              +--------+                           |
+ * +----------------------------->| KILLED |<--------------------------+
+ *       deadman timeout or       +--------+     deadman timeout or
+ *        killed consumer                         killed consumer
+ *
+ * Note that once a DTrace consumer has stopped tracing, there is no way to
+ * restart it; if a DTrace consumer wishes to restart tracing, it must reopen
+ * the DTrace pseudodevice.
+ */
+typedef enum dtrace_activity {
+	DTRACE_ACTIVITY_INACTIVE = 0,		/* not yet running */
+	DTRACE_ACTIVITY_WARMUP,			/* while starting */
+	DTRACE_ACTIVITY_ACTIVE,			/* running */
+	DTRACE_ACTIVITY_DRAINING,		/* before stopping */
+	DTRACE_ACTIVITY_COOLDOWN,		/* while stopping */
+	DTRACE_ACTIVITY_STOPPED,		/* after stopping */
+	DTRACE_ACTIVITY_KILLED			/* killed */
+} dtrace_activity_t;
+
+/*
+ * DTrace dof modes
+ *
+ * DTrace has four "dof modes". They are:
+ *
+ * DTRACE_DOF_MODE_NEVER	Never load any dof, period.
+ * DTRACE_DOF_MODE_LAZY_ON	Defer loading dof until later
+ * DTRACE_DOF_MODE_LAZY_OFF	Load all deferred dof now, and any new dof 
+ * DTRACE_DOF_MODE_NON_LAZY	Load all dof immediately.
+ *
+ * It is legal to transition between the two lazy modes. The NEVER and
+ * NON_LAZY modes are permanent, and must not change once set.
+ *
+ * The current dof mode is kept in dtrace_dof_mode, which is protected by the
+ * dtrace_dof_mode_lock. This is a RW lock, reads require shared access, writes
+ * require exclusive access. Because NEVER and NON_LAZY are permanent states,
+ * it is legal to test for those modes without holding the dof mode lock.
+ *
+ * Lock ordering is dof mode lock before any dtrace lock, and before the
+ * process p_dtrace_sprlock. In general, other locks should not be held when
+ * taking the dof mode lock. Acquiring the dof mode lock in exclusive mode
+ * will block process fork, exec, and exit, so it should be held exclusive
+ * for as short a time as possible.
+ */
+
+#define DTRACE_DOF_MODE_NEVER 		0
+#define DTRACE_DOF_MODE_LAZY_ON		1
+#define DTRACE_DOF_MODE_LAZY_OFF	2
+#define DTRACE_DOF_MODE_NON_LAZY	3
+
+/*
+ * DTrace Helper Implementation
+ *
+ * A description of the helper architecture may be found in <sys/dtrace.h>.
+ * Each process contains a pointer to its helpers in its p_dtrace_helpers
+ * member.  This is a pointer to a dtrace_helpers structure, which contains an
+ * array of pointers to dtrace_helper structures, helper variable state (shared
+ * among a process's helpers) and a generation count.  (The generation count is
+ * used to provide an identifier when a helper is added so that it may be
+ * subsequently removed.)  The dtrace_helper structure is self-explanatory,
+ * containing pointers to the objects needed to execute the helper.  Note that
+ * helpers are _duplicated_ across fork(2), and destroyed on exec(2).  No more
+ * than dtrace_helpers_max are allowed per-process.
+ */
+#define	DTRACE_HELPER_ACTION_USTACK	0
+#define	DTRACE_NHELPER_ACTIONS		1
+
+typedef struct dtrace_helper_action {
+	int dtha_generation;			/* helper action generation */
+	int dtha_nactions;			/* number of actions */
+	dtrace_difo_t *dtha_predicate;		/* helper action predicate */
+	dtrace_difo_t **dtha_actions;		/* array of actions */
+	struct dtrace_helper_action *dtha_next;	/* next helper action */
+} dtrace_helper_action_t;
+
+typedef struct dtrace_helper_provider {
+	int dthp_generation;			/* helper provider generation */
+	uint32_t dthp_ref;			/* reference count */
+	dof_helper_t dthp_prov;			/* DOF w/ provider and probes */
+} dtrace_helper_provider_t;
+
+typedef struct dtrace_helpers {
+	dtrace_helper_action_t **dthps_actions;	/* array of helper actions */
+	dtrace_vstate_t dthps_vstate;		/* helper action var. state */
+	dtrace_helper_provider_t **dthps_provs;	/* array of providers */
+	uint_t dthps_nprovs;			/* count of providers */
+	uint_t dthps_maxprovs;			/* provider array size */
+	int dthps_generation;			/* current generation */
+	pid_t dthps_pid;			/* pid of associated proc */
+	int dthps_deferred;			/* helper in deferred list */
+	struct dtrace_helpers *dthps_next;	/* next pointer */
+	struct dtrace_helpers *dthps_prev;	/* prev pointer */
+} dtrace_helpers_t;
+
+/*
+ * DTrace Helper Action Tracing
+ *
+ * Debugging helper actions can be arduous.  To ease the development and
+ * debugging of helpers, DTrace contains a tracing-framework-within-a-tracing-
+ * framework: helper tracing.  If dtrace_helptrace_enabled is non-zero (which
+ * it is by default on DEBUG kernels), all helper activity will be traced to a
+ * global, in-kernel ring buffer.  Each entry includes a pointer to the specific
+ * helper, the location within the helper, and a trace of all local variables.
+ * The ring buffer may be displayed in a human-readable format with the
+ * ::dtrace_helptrace mdb(1) dcmd.
+ */
+#define	DTRACE_HELPTRACE_NEXT	(-1)
+#define	DTRACE_HELPTRACE_DONE	(-2)
+#define	DTRACE_HELPTRACE_ERR	(-3)
+
+typedef struct dtrace_helptrace {
+	dtrace_helper_action_t	*dtht_helper;	/* helper action */
+	int dtht_where;				/* where in helper action */
+	int dtht_nlocals;			/* number of locals */
+	int dtht_fault;				/* type of fault (if any) */
+	int dtht_fltoffs;			/* DIF offset */
+	uint64_t dtht_illval;			/* faulting value */
+	uint64_t dtht_locals[1];		/* local variables */
+} dtrace_helptrace_t;
+
+/*
+ * DTrace Credentials
+ *
+ * In probe context, we have limited flexibility to examine the credentials
+ * of the DTrace consumer that created a particular enabling.  We use
+ * the Least Privilege interfaces to cache the consumer's cred pointer and
+ * some facts about that credential in a dtrace_cred_t structure. These
+ * can limit the consumer's breadth of visibility and what actions the
+ * consumer may take.
+ */
+#define	DTRACE_CRV_ALLPROC		0x01
+#define	DTRACE_CRV_KERNEL		0x02
+#define	DTRACE_CRV_ALLZONE		0x04
+
+#define	DTRACE_CRV_ALL		(DTRACE_CRV_ALLPROC | DTRACE_CRV_KERNEL | \
+	DTRACE_CRV_ALLZONE)
+
+#define	DTRACE_CRA_PROC				0x0001
+#define	DTRACE_CRA_PROC_CONTROL			0x0002
+#define	DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER	0x0004
+#define	DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE	0x0008
+#define	DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG	0x0010
+#define	DTRACE_CRA_KERNEL			0x0020
+#define	DTRACE_CRA_KERNEL_DESTRUCTIVE		0x0040
+
+#define	DTRACE_CRA_ALL		(DTRACE_CRA_PROC | \
+	DTRACE_CRA_PROC_CONTROL | \
+	DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER | \
+	DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE | \
+	DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG | \
+	DTRACE_CRA_KERNEL | \
+	DTRACE_CRA_KERNEL_DESTRUCTIVE)
+
+typedef struct dtrace_cred {
+	cred_t			*dcr_cred;
+	uint8_t			dcr_destructive;
+	uint8_t			dcr_visible;
+	uint16_t		dcr_action;
+} dtrace_cred_t;
+
+/*
+ * DTrace Consumer State
+ *
+ * Each DTrace consumer has an associated dtrace_state structure that contains
+ * its in-kernel DTrace state -- including options, credentials, statistics and
+ * pointers to ECBs, buffers, speculations and formats.  A dtrace_state
+ * structure is also allocated for anonymous enablings.  When anonymous state
+ * is grabbed, the grabbing consumers dts_anon pointer is set to the grabbed
+ * dtrace_state structure.
+ */
+struct dtrace_state {
+	dev_t dts_dev;				/* device */
+	int dts_necbs;				/* total number of ECBs */
+	dtrace_ecb_t **dts_ecbs;		/* array of ECBs */
+	dtrace_epid_t dts_epid;			/* next EPID to allocate */
+	size_t dts_needed;			/* greatest needed space */
+	struct dtrace_state *dts_anon;		/* anon. state, if grabbed */
+	dtrace_activity_t dts_activity;		/* current activity */
+	dtrace_vstate_t dts_vstate;		/* variable state */
+	dtrace_buffer_t *dts_buffer;		/* principal buffer */
+	dtrace_buffer_t *dts_aggbuffer;		/* aggregation buffer */
+	dtrace_speculation_t *dts_speculations;	/* speculation array */
+	int dts_nspeculations;			/* number of speculations */
+	int dts_naggregations;			/* number of aggregations */
+	dtrace_aggregation_t **dts_aggregations; /* aggregation array */
+	vmem_t *dts_aggid_arena;		/* arena for aggregation IDs */
+	uint64_t dts_errors;			/* total number of errors */
+	uint32_t dts_speculations_busy;		/* number of spec. busy */
+	uint32_t dts_speculations_unavail;	/* number of spec unavail */
+	uint32_t dts_stkstroverflows;		/* stack string tab overflows */
+	uint32_t dts_dblerrors;			/* errors in ERROR probes */
+	uint32_t dts_reserve;			/* space reserved for END */
+	hrtime_t dts_laststatus;		/* time of last status */
+	cyclic_id_t dts_cleaner;		/* cleaning cyclic */
+	cyclic_id_t dts_deadman;		/* deadman cyclic */
+	hrtime_t dts_alive;			/* time last alive */
+	char dts_speculates;			/* boolean: has speculations */
+	char dts_destructive;			/* boolean: has dest. actions */
+	int dts_nformats;			/* number of formats */
+	char **dts_formats;			/* format string array */
+	dtrace_optval_t dts_options[DTRACEOPT_MAX]; /* options */
+	dtrace_cred_t dts_cred;			/* credentials */
+	size_t dts_nretained;			/* number of retained enabs */
+#if defined(__APPLE__)
+	uint64_t dts_arg_error_illval;
+#endif /* __APPLE__ */
+};
+
+struct dtrace_provider {
+	dtrace_pattr_t dtpv_attr;		/* provider attributes */
+	dtrace_ppriv_t dtpv_priv;		/* provider privileges */
+	dtrace_pops_t dtpv_pops;		/* provider operations */
+	char *dtpv_name;			/* provider name */
+	void *dtpv_arg;				/* provider argument */
+	uint_t dtpv_defunct;			/* boolean: defunct provider */
+	struct dtrace_provider *dtpv_next;	/* next provider */
+};
+
+struct dtrace_meta {
+	dtrace_mops_t dtm_mops;			/* meta provider operations */
+	char *dtm_name;				/* meta provider name */
+	void *dtm_arg;				/* meta provider user arg */
+	uint64_t dtm_count;			/* no. of associated provs. */
+};
+
+/*
+ * DTrace Enablings
+ *
+ * A dtrace_enabling structure is used to track a collection of ECB
+ * descriptions -- before they have been turned into actual ECBs.  This is
+ * created as a result of DOF processing, and is generally used to generate
+ * ECBs immediately thereafter.  However, enablings are also generally
+ * retained should the probes they describe be created at a later time; as
+ * each new module or provider registers with the framework, the retained
+ * enablings are reevaluated, with any new match resulting in new ECBs.  To
+ * prevent probes from being matched more than once, the enabling tracks the
+ * last probe generation matched, and only matches probes from subsequent
+ * generations.
+ */
+typedef struct dtrace_enabling {
+	dtrace_ecbdesc_t **dten_desc;		/* all ECB descriptions */
+	int dten_ndesc;				/* number of ECB descriptions */
+	int dten_maxdesc;			/* size of ECB array */
+	dtrace_vstate_t *dten_vstate;		/* associated variable state */
+	dtrace_genid_t dten_probegen;		/* matched probe generation */
+	dtrace_ecbdesc_t *dten_current;		/* current ECB description */
+	int dten_error;				/* current error value */
+	int dten_primed;			/* boolean: set if primed */
+	struct dtrace_enabling *dten_prev;	/* previous enabling */
+	struct dtrace_enabling *dten_next;	/* next enabling */
+} dtrace_enabling_t;
+
+/*
+ * DTrace Anonymous Enablings
+ *
+ * Anonymous enablings are DTrace enablings that are not associated with a
+ * controlling process, but rather derive their enabling from DOF stored as
+ * properties in the dtrace.conf file.  If there is an anonymous enabling, a
+ * DTrace consumer state and enabling are created on attach.  The state may be
+ * subsequently grabbed by the first consumer specifying the "grabanon"
+ * option.  As long as an anonymous DTrace enabling exists, dtrace(7D) will
+ * refuse to unload.
+ */
+typedef struct dtrace_anon {
+	dtrace_state_t *dta_state;		/* DTrace consumer state */
+	dtrace_enabling_t *dta_enabling;	/* pointer to enabling */
+	processorid_t dta_beganon;		/* which CPU BEGIN ran on */
+} dtrace_anon_t;
+
+/*
+ * DTrace Error Debugging
+ */
+#ifdef DEBUG
+#define	DTRACE_ERRDEBUG
+#endif
+
+#ifdef DTRACE_ERRDEBUG
+
+typedef struct dtrace_errhash {
+	const char	*dter_msg;	/* error message */
+	int		dter_count;	/* number of times seen */
+} dtrace_errhash_t;
+
+#define	DTRACE_ERRHASHSZ	256	/* must be > number of err msgs */
+
+#endif	/* DTRACE_ERRDEBUG */
+
+/*
+ * DTrace Toxic Ranges
+ *
+ * DTrace supports safe loads from probe context; if the address turns out to
+ * be invalid, a bit will be set by the kernel indicating that DTrace
+ * encountered a memory error, and DTrace will propagate the error to the user
+ * accordingly.  However, there may exist some regions of memory in which an
+ * arbitrary load can change system state, and from which it is impossible to
+ * recover from such a load after it has been attempted.  Examples of this may
+ * include memory in which programmable I/O registers are mapped (for which a
+ * read may have some implications for the device) or (in the specific case of
+ * UltraSPARC-I and -II) the virtual address hole.  The platform is required
+ * to make DTrace aware of these toxic ranges; DTrace will then check that
+ * target addresses are not in a toxic range before attempting to issue a
+ * safe load.
+ */
+typedef struct dtrace_toxrange {
+	uintptr_t	dtt_base;		/* base of toxic range */
+	uintptr_t	dtt_limit;		/* limit of toxic range */
+} dtrace_toxrange_t;
+
+extern uint64_t dtrace_getarg(int, int);
+extern greg_t dtrace_getfp(void);
+extern int dtrace_getipl(void);
+extern uintptr_t dtrace_caller(int);
+extern uint32_t dtrace_cas32(uint32_t *, uint32_t, uint32_t);
+extern void *dtrace_casptr(void *, void *, void *);
+#if !defined(__APPLE__)
+extern void dtrace_copyin(uintptr_t, uintptr_t, size_t);
+extern void dtrace_copyinstr(uintptr_t, uintptr_t, size_t);
+extern void dtrace_copyout(uintptr_t, uintptr_t, size_t);
+extern void dtrace_copyoutstr(uintptr_t, uintptr_t, size_t);
+#else
+extern void dtrace_copyin(user_addr_t, uintptr_t, size_t);
+extern void dtrace_copyinstr(user_addr_t, uintptr_t, size_t);
+extern void dtrace_copyout(uintptr_t, user_addr_t, size_t);
+extern void dtrace_copyoutstr(uintptr_t, user_addr_t, size_t);
+#endif /* __APPLE__ */
+extern void dtrace_getpcstack(pc_t *, int, int, uint32_t *);
+#if !defined(__APPLE__)
+extern ulong_t dtrace_getreg(struct regs *, uint_t);
+#else
+extern uint64_t dtrace_getreg(struct regs *, uint_t);
+#endif /* __APPLE__ */
+extern int dtrace_getstackdepth(int);
+extern void dtrace_getupcstack(uint64_t *, int);
+extern void dtrace_getufpstack(uint64_t *, uint64_t *, int);
+extern int dtrace_getustackdepth(void);
+extern uintptr_t dtrace_fulword(void *);
+#if !defined(__APPLE__)
+extern uint8_t dtrace_fuword8(void *);
+extern uint16_t dtrace_fuword16(void *);
+extern uint32_t dtrace_fuword32(void *);
+extern uint64_t dtrace_fuword64(void *);
+extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
+    int, uintptr_t);
+#else
+extern uint8_t dtrace_fuword8(user_addr_t);
+extern uint16_t dtrace_fuword16(user_addr_t);
+extern uint32_t dtrace_fuword32(user_addr_t);
+extern uint64_t dtrace_fuword64(user_addr_t);
+extern void dtrace_probe_error(dtrace_state_t *, dtrace_epid_t, int, int,
+    int, uint64_t);
+#endif /* __APPLE__ */
+extern int dtrace_assfail(const char *, const char *, int);
+extern int dtrace_attached(void);
+extern hrtime_t dtrace_gethrestime(void);
+
+#ifdef __sparc
+extern void dtrace_flush_windows(void);
+extern void dtrace_flush_user_windows(void);
+extern uint_t dtrace_getotherwin(void);
+extern uint_t dtrace_getfprs(void);
+#else
+extern void dtrace_copy(uintptr_t, uintptr_t, size_t);
+extern void dtrace_copystr(uintptr_t, uintptr_t, size_t);
+#endif
+
+/*
+ * DTrace Assertions
+ *
+ * DTrace calls ASSERT from probe context.  To assure that a failed ASSERT
+ * does not induce a markedly more catastrophic failure (e.g., one from which
+ * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
+ * may safely be called from probe context.  This header file must thus be
+ * included by any DTrace component that calls ASSERT from probe context, and
+ * _only_ by those components.  (The only exception to this is kernel
+ * debugging infrastructure at user-level that doesn't depend on calling
+ * ASSERT.)
+ */
+#undef ASSERT
+#ifdef DEBUG
+#define	ASSERT(EX)	((void)((EX) || \
+			dtrace_assfail(#EX, __FILE__, __LINE__)))
+#else
+#define	ASSERT(X)	((void)0)
+#endif
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_DTRACE_IMPL_H */
+
diff --git a/bsd/sys/dtrace_ptss.h b/bsd/sys/dtrace_ptss.h
new file mode 100644
index 000000000..58454dacf
--- /dev/null
+++ b/bsd/sys/dtrace_ptss.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _DTRACE_PTSS_H_
+#define	_DTRACE_PTSS_H_
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * The pid provider needs a small per thread scratch space,
+ * in the address space of the user task. This code is used to
+ * manage that space.
+ *
+ * High level design:
+ *
+ * To avoid serialization, this is a (mostly) lockless allocator. If
+ * a new page has to be allocated, the process's sprlock will be acquired.
+ * 
+ * NOTE: The dtrace copyin/copyout code is still the shared code that
+ * can handle unmapped pages, so the scratch space isn't wired for now.
+ * * Each page in user space is wired. It cannot be paged out, because
+ * * dtrace's copyin/copyout is only guaranteed to handle pages already
+ * * in memory. 
+ *
+ * Each page in user space is represented by a dt_ptss_page. Page entries
+ * are chained. Once allocated, a page is not freed until dtrace "cleans up"
+ * that process.
+ *
+ * Clean up works like this:
+ *
+ * At process exit, free all kernel allocated memory, but ignore user pages.
+ * At process exec, free all kernel allocated memory, but ignore user pages.
+ * At process fork, free user pages copied from parent, and do not allocate kernel memory.
+ *
+ * This is making the assumption that it is faster to let the bulk vm_map
+ * operations in exec/exit do their work, instead of explicit page free(s)
+ * via mach_vm_deallocate.
+ *
+ * As each page is allocated, its entries are chained and added to the
+ * free_list. To claim an entry, cas it off the list. When a thread exits,
+ * cas its entry onto the list. We could potentially optimize this by
+ * keeping a head/tail, and cas'ing the frees to the tail instead of the
+ * head. Without evidence to support such a need, it seems better to keep
+ * things simple for now.
+ */
+
+#define DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD 64
+#define DTRACE_PTSS_ENTRIES_PER_PAGE (PAGE_SIZE / DTRACE_PTSS_SCRATCH_SPACE_PER_THREAD)
+
+struct dtrace_ptss_page_entry {
+	struct dtrace_ptss_page_entry*	next;
+	user_addr_t			addr;
+};
+
+struct dtrace_ptss_page {
+	struct dtrace_ptss_page*       next;
+	struct dtrace_ptss_page_entry  entries[DTRACE_PTSS_ENTRIES_PER_PAGE]; 
+};
+
+struct dtrace_ptss_page_entry*	dtrace_ptss_claim_entry(struct proc* p); /* sprlock not held */
+struct dtrace_ptss_page_entry*	dtrace_ptss_claim_entry_locked(struct proc* p); /* sprlock held */
+void				dtrace_ptss_release_entry(struct proc* p, struct dtrace_ptss_page_entry* e);
+
+struct dtrace_ptss_page*	dtrace_ptss_allocate_page(struct proc* p);
+void				dtrace_ptss_free_page(struct proc* p, struct dtrace_ptss_page* ptss_page);
+
+void				dtrace_ptss_enable(struct proc* p);
+void				dtrace_ptss_exec_exit(struct proc* p);
+void				dtrace_ptss_fork(struct proc* parent, struct proc* child);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _DTRACE_PTSS_H_ */
+
diff --git a/bsd/sys/errno.h b/bsd/sys/errno.h
index b19b04da4..a11877c52 100644
--- a/bsd/sys/errno.h
+++ b/bsd/sys/errno.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -90,10 +96,10 @@ __END_DECLS
 #define	ENOMEM		12		/* Cannot allocate memory */
 #define	EACCES		13		/* Permission denied */
 #define	EFAULT		14		/* Bad address */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ENOTBLK		15		/* Block device required */
 #endif
-#define	EBUSY		16		/* Device busy */
+#define	EBUSY		16		/* Device / Resource busy */
 #define	EEXIST		17		/* File exists */
 #define	EXDEV		18		/* Cross-device link */
 #define	ENODEV		19		/* Operation not supported by device */
@@ -128,9 +134,9 @@ __END_DECLS
 #define	EPROTOTYPE	41		/* Protocol wrong type for socket */
 #define	ENOPROTOOPT	42		/* Protocol not available */
 #define	EPROTONOSUPPORT	43		/* Protocol not supported */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ESOCKTNOSUPPORT	44		/* Socket type not supported */
-#endif /* ! _POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define ENOTSUP		45		/* Operation not supported */
 #if !__DARWIN_UNIX03 && !defined(KERNEL)
 /*
@@ -144,9 +150,9 @@ __END_DECLS
 #define	EOPNOTSUPP	 ENOTSUP	/* Operation not supported on socket */
 #endif /* !__DARWIN_UNIX03 && !KERNEL */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	EPFNOSUPPORT	46		/* Protocol family not supported */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	EAFNOSUPPORT	47		/* Address family not supported by protocol family */
 #define	EADDRINUSE	48		/* Address already in use */
 #define	EADDRNOTAVAIL	49		/* Can't assign requested address */
@@ -160,10 +166,10 @@ __END_DECLS
 #define	ENOBUFS		55		/* No buffer space available */
 #define	EISCONN		56		/* Socket is already connected */
 #define	ENOTCONN	57		/* Socket is not connected */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ESHUTDOWN	58		/* Can't send after socket shutdown */
 #define	ETOOMANYREFS	59		/* Too many references: can't splice */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	ETIMEDOUT	60		/* Operation timed out */
 #define	ECONNREFUSED	61		/* Connection refused */
 
@@ -171,34 +177,34 @@ __END_DECLS
 #define	ENAMETOOLONG	63		/* File name too long */
 
 /* should be rearranged */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	EHOSTDOWN	64		/* Host is down */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	EHOSTUNREACH	65		/* No route to host */
 #define	ENOTEMPTY	66		/* Directory not empty */
 
 /* quotas & mush */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	EPROCLIM	67		/* Too many processes */
 #define	EUSERS		68		/* Too many users */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	EDQUOT		69		/* Disc quota exceeded */
 
 /* Network File System */
 #define	ESTALE		70		/* Stale NFS file handle */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	EREMOTE		71		/* Too many levels of remote in path */
 #define	EBADRPC		72		/* RPC struct is bad */
 #define	ERPCMISMATCH	73		/* RPC version wrong */
 #define	EPROGUNAVAIL	74		/* RPC prog. not avail */
 #define	EPROGMISMATCH	75		/* Program version wrong */
 #define	EPROCUNAVAIL	76		/* Bad procedure for program */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #define	ENOLCK		77		/* No locks available */
 #define	ENOSYS		78		/* Function not implemented */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	EFTYPE		79		/* Inappropriate file type or format */
 #define	EAUTH		80		/* Authentication error */
 #define	ENEEDAUTH	81		/* Need authenticator */
@@ -206,26 +212,26 @@ __END_DECLS
 /* Intelligent device errors */
 #define	EPWROFF		82	/* Device power is off */
 #define	EDEVERR		83	/* Device error, e.g. paper out */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #define	EOVERFLOW	84		/* Value too large to be stored in data type */
 
 /* Program loading errors */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define EBADEXEC	85	/* Bad executable */
 #define EBADARCH	86	/* Bad CPU type in executable */
 #define ESHLIBVERS	87	/* Shared library version mismatch */
 #define EBADMACHO	88	/* Malformed Macho file */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #define	ECANCELED	89		/* Operation canceled */
 
 #define EIDRM		90		/* Identifier removed */
 #define ENOMSG		91		/* No message of desired type */   
 #define EILSEQ		92		/* Illegal byte sequence */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define ENOATTR		93		/* Attribute not found */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #define EBADMSG		94		/* Bad message */
 #define EMULTIHOP	95		/* Reserved */
@@ -241,9 +247,11 @@ __END_DECLS
 #define	EOPNOTSUPP	102		/* Operation not supported on socket */
 #endif /* __DARWIN_UNIX03 || KERNEL */
 
-#ifndef _POSIX_C_SOURCE
-#define	ELAST		102		/* Must be equal largest errno */
-#endif /* _POSIX_C_SOURCE */
+#define ENOPOLICY	103		/* No such policy registered */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define	ELAST		103		/* Must be equal largest errno */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifdef KERNEL
 /* pseudo-errors returned inside kernel to modify return to process */
diff --git a/bsd/sys/ev.h b/bsd/sys/ev.h
index 39c4aeb61..8000e9900 100644
--- a/bsd/sys/ev.h
+++ b/bsd/sys/ev.h
@@ -1,30 +1,35 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998 Apple Computer, Inc. All rights reserved */
 
 #ifndef _SYS_EV_H_
 #define _SYS_EV_H_
 
-#if !defined(__LP64__)
 
 #include <sys/appleapiopts.h>
 
@@ -62,12 +67,46 @@ typedef struct eventreq *er_t;
 #define EV_TIMEOUT 0x20000
 #define EV_DMASK   0xffffff00
 
+
+#ifndef KERNEL
+
+__BEGIN_DECLS
+int	modwatch(er_t, int);
+int	watchevent(er_t, int);
+int	waitevent(er_t, struct timeval *);
+__END_DECLS
+
+#endif
+
+
 #ifdef BSD_KERNEL_PRIVATE
 
+
+
+struct eventreq32 {
+  int      er_type;
+  int      er_handle;
+  uint32_t er_data;
+  int      er_rcnt;
+  int      er_wcnt;
+  int      er_ecnt;
+  int      er_eventbits;
+};
+
+struct eventreq64 {
+  int      er_type;
+  int      er_handle;
+  user_addr_t er_data;
+  int      er_rcnt;
+  int      er_wcnt;
+  int      er_ecnt;
+  int      er_eventbits;
+};
+
 struct eventqelt {
   TAILQ_ENTRY(eventqelt)  ee_slist;
   TAILQ_ENTRY(eventqelt)  ee_plist;
-  struct eventreq  ee_req;
+  struct eventreq64  ee_req;
   struct proc *    ee_proc;
   u_int            ee_flags;
 #define EV_QUEUED	0x01
@@ -77,6 +116,4 @@ struct eventqelt {
 int waitevent_close(struct proc *p, struct fileproc *);
 #endif /* BSD_KERNEL_PRIVATE */
 
-#endif /* !defined(__LP64__) */
-
 #endif /* _SYS_EV_H_ */
diff --git a/bsd/sys/event.h b/bsd/sys/event.h
index 72d71ceca..d08051ca2 100644
--- a/bsd/sys/event.h
+++ b/bsd/sys/event.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
@@ -112,6 +118,7 @@ struct user_kevent {
 #define EV_DELETE	0x0002		/* delete event from kq */
 #define EV_ENABLE	0x0004		/* enable event */
 #define EV_DISABLE	0x0008		/* disable event (not reported) */
+#define EV_RECEIPT	0x0040		/* force EV_ERROR on success, data == 0 */
 
 /* flags */
 #define EV_ONESHOT	0x0010		/* only report one occurrence */
@@ -166,12 +173,20 @@ struct user_kevent {
 
 /*
  * data/hint fflags for EVFILT_PROC, shared with userspace
+ *
+ * Please note that EVFILT_PROC and EVFILT_SIGNAL share the same knote list
+ * that hangs off the proc structure. They also both play games with the hint
+ * passed to KNOTE(). If NOTE_SIGNAL is passed as a hint, then the lower bits
+ * of the hint contain the signal. IF NOTE_FORK is passed, then the lower bits
+ * contain the PID of the child.
  */
 #define	NOTE_EXIT	0x80000000		/* process exited */
 #define	NOTE_FORK	0x40000000		/* process forked */
 #define	NOTE_EXEC	0x20000000		/* process exec'd */
-#define	NOTE_PCTRLMASK	0xf0000000		/* mask for hint bits */
-#define	NOTE_PDATAMASK	0x000fffff		/* mask for pid */
+#define	NOTE_REAP	0x10000000		/* process reaped */
+#define	NOTE_SIGNAL	0x08000000		/* shared with EVFILT_SIGNAL */
+#define	NOTE_PDATAMASK	0x000fffff		/* mask for pid/signal */
+#define	NOTE_PCTRLMASK	(~NOTE_PDATAMASK)
 
 /*
  * data/hint fflags for EVFILT_TIMER, shared with userspace.
@@ -186,6 +201,10 @@ struct user_kevent {
 #define NOTE_ABSOLUTE	0x00000008		/* absolute timeout        */
 						/* ... implicit EV_ONESHOT */
  
+/*
+ * DEPRECATED!!!!!!!!!
+ * NOTE_TRACK, NOTE_TRACKERR, and NOTE_CHILD are no longer supported as of 10.5
+ */
 /* additional flags for EVFILT_PROC */
 #define	NOTE_TRACK	0x00000001		/* follow across forks */
 #define	NOTE_TRACKERR	0x00000002		/* could not track child */
@@ -209,12 +228,6 @@ SLIST_HEAD(klist, knote);
 MALLOC_DECLARE(M_KQUEUE);
 #endif
 
-/*
- * Flag indicating hint is a signal.  Used by EVFILT_SIGNAL, and also
- * shared by EVFILT_PROC  (all knotes attached to p->p_klist)
- */
-#define NOTE_SIGNAL	0x08000000
-
 TAILQ_HEAD(kqtailq, knote);	/* a list of "queued" events */
 
 struct knote {
@@ -261,6 +274,7 @@ struct filterops {
 struct proc;
 
 SLIST_HEAD(klist, knote);
+extern void	knote_init(void) __attribute__((section("__TEXT, initcode")));
 extern void	klist_init(struct klist *list);
 
 #define KNOTE(list, hint)	knote(list, hint)
diff --git a/bsd/sys/eventvar.h b/bsd/sys/eventvar.h
index 8206a7201..9694af109 100644
--- a/bsd/sys/eventvar.h
+++ b/bsd/sys/eventvar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1999,2000 Jonathan Lemon <jlemon@FreeBSD.org>
@@ -64,7 +70,8 @@ struct kqueue {
 	struct kqtailq	kq_head;		/* list of queued events */
 	struct kqtailq	kq_inprocess;		/* list of in-process events */
 	struct selinfo	kq_sel;		/* parent select/kqueue info */
-	struct filedesc	*kq_fdp;
+	struct proc	*kq_p;		/* process containing kqueue */
+	int		kq_level;	/* nesting level */
 
 #define KQ_SEL		0x01
 #define KQ_SLEEP	0x02
@@ -72,7 +79,7 @@ struct kqueue {
 };
 
 extern struct kqueue *kqueue_alloc(struct proc *);
-extern void kqueue_dealloc(struct kqueue *, struct proc *);
+extern void kqueue_dealloc(struct kqueue *);
 
 typedef int (*kevent_callback_t)(struct kqueue *, struct kevent *, void *);
 typedef void (*kevent_continue_t)(struct kqueue *, void *, int);
diff --git a/bsd/sys/exec.h b/bsd/sys/exec.h
index e3b9d6e5b..ee79964a3 100644
--- a/bsd/sys/exec.h
+++ b/bsd/sys/exec.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/fasttrap.h b/bsd/sys/fasttrap.h
new file mode 100644
index 000000000..64f08842e
--- /dev/null
+++ b/bsd/sys/fasttrap.h
@@ -0,0 +1,115 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_SYS_FASTTRAP_H
+#define	_SYS_FASTTRAP_H
+
+/* #pragma ident	"@(#)fasttrap.h	1.5	06/03/30 SMI" */
+
+#include <sys/fasttrap_isa.h>
+#include <sys/dtrace.h>
+#include <sys/types.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#if !defined(__APPLE__)
+#define	FASTTRAPIOC		(('m' << 24) | ('r' << 16) | ('f' << 8))
+#else
+#define	FASTTRAPIOC		_IO('d', 0)
+#endif /* __APPLE__ */
+#define	FASTTRAPIOC_MAKEPROBE	(FASTTRAPIOC | 1)
+#define	FASTTRAPIOC_GETINSTR	(FASTTRAPIOC | 2)
+
+typedef enum fasttrap_probe_type {
+	DTFTP_NONE = 0,
+	DTFTP_ENTRY,
+	DTFTP_RETURN,
+	DTFTP_OFFSETS,
+	DTFTP_POST_OFFSETS,
+	DTFTP_IS_ENABLED
+} fasttrap_probe_type_t;
+
+#if defined(__APPLE__)
+typedef enum fasttrap_provider_type {
+	DTFTP_PROVIDER_NONE = 0,
+	DTFTP_PROVIDER_USDT, 
+	DTFTP_PROVIDER_PID,
+	DTFTP_PROVIDER_OBJC,
+	DTFTP_PROVIDER_ONESHOT
+} fasttrap_provider_type_t;
+
+/* Moved from fasttrap.c */
+#define	FASTTRAP_PID_NAME		"pid"
+#define	FASTTRAP_OBJC_NAME		"objc"
+#define	FASTTRAP_ONESHOT_NAME		"oneshot"
+
+#endif
+
+typedef struct fasttrap_probe_spec {
+	pid_t				ftps_pid;
+#if defined(__APPLE__)
+	fasttrap_provider_type_t	ftps_provider_type;
+	fasttrap_probe_type_t		ftps_probe_type;
+#endif
+	char				ftps_func[DTRACE_FUNCNAMELEN];
+	char				ftps_mod[DTRACE_MODNAMELEN];
+
+	uint64_t			ftps_pc;
+	uint64_t			ftps_size;
+	uint64_t			ftps_noffs;
+	uint64_t			ftps_offs[1];
+} fasttrap_probe_spec_t;
+
+typedef struct fasttrap_instr_query {
+	uint64_t		ftiq_pc;
+	pid_t			ftiq_pid;
+	fasttrap_instr_t	ftiq_instr;
+} fasttrap_instr_query_t;
+
+/*
+ * To support the fasttrap provider from very early in a process's life,
+ * the run-time linker, ld.so.1, has a program header of type PT_SUNWDTRACE
+ * which points to a data object which must be PT_SUNWDTRACE_SIZE bytes.
+ * This structure mimics the fasttrap provider section of the ulwp_t structure.
+ * When the fasttrap provider is changed to require new or different
+ * instructions, the data object in ld.so.1 and the thread initializers in libc
+ * (libc_init() and _thrp_create()) need to be updated to include the new
+ * instructions, and PT_SUNWDTRACE needs to be changed to a new unique number
+ * (while the old value gets assigned something like PT_SUNWDTRACE_1). Since the
+ * linker must be backward compatible with old Solaris releases, it must have
+ * program headers for each of the PT_SUNWDTRACE versions. The kernel's
+ * elfexec() function only has to look for the latest version of the
+ * PT_SUNWDTRACE program header.
+ */
+#define	PT_SUNWDTRACE_SIZE	FASTTRAP_SUNWDTRACE_SIZE
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_FASTTRAP_H */
diff --git a/bsd/sys/fasttrap_impl.h b/bsd/sys/fasttrap_impl.h
new file mode 100644
index 000000000..8625c6494
--- /dev/null
+++ b/bsd/sys/fasttrap_impl.h
@@ -0,0 +1,222 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef	_FASTTRAP_IMPL_H
+#define	_FASTTRAP_IMPL_H
+
+/*
+ * #pragma ident	"@(#)fasttrap_impl.h	1.12	06/06/12 SMI"
+ */
+
+#include <sys/types.h>
+#include <sys/dtrace.h>
+#include <sys/proc.h>
+#include <sys/user.h>
+#include <sys/fasttrap.h>
+#include <sys/fasttrap_isa.h>
+
+#define proc_t struct proc
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * Fasttrap Providers, Probes and Tracepoints
+ *
+ * Each Solaris process can have multiple providers -- the pid provider as
+ * well as any number of user-level statically defined tracing (USDT)
+ * providers. Those providers are each represented by a fasttrap_provider_t.
+ * All providers for a given process have a pointer to a shared
+ * fasttrap_proc_t. The fasttrap_proc_t has two states: active or defunct.
+ * It becomes defunct when the process performs an exit or an exec.
+ *
+ * Each probe is represented by a fasttrap_probe_t which has a pointer to
+ * its associated provider as well as a list of fasttrap_id_tp_t structures
+ * which are tuples combining a fasttrap_id_t and a fasttrap_tracepoint_t.
+ * A fasttrap_tracepoint_t represents the actual point of instrumentation
+ * and it contains two lists of fasttrap_id_t structures (to be fired pre-
+ * and post-instruction emulation) that identify the probes attached to the
+ * tracepoint. Tracepoints also have a pointer to the fasttrap_proc_t for the
+ * process they trace which is used when looking up a tracepoint both at
+ * probe fire time and when enabling and disabling probes.
+ *
+ * It's important to note that probes are preallocated with the necessary
+ * number of tracepoints, but that tracepoints can be shared by probes and
+ * swapped between probes. If a probe's preallocated tracepoint is enabled
+ * (and, therefore, the associated probe is enabled), and that probe is
+ * then disabled, ownership of that tracepoint may be exchanged for an
+ * unused tracepoint belonging to another probe that was attached to the
+ * enabled tracepoint.
+ */
+
+/*
+ * APPLE NOTE: All kmutex_t's have been converted to lck_mtx_t
+ */
+
+typedef struct fasttrap_proc {
+	pid_t ftpc_pid;				/* process ID for this proc */
+	uint_t ftpc_defunct;			/* denotes a lame duck proc */
+	uint64_t ftpc_count;			/* reference count */
+	lck_mtx_t ftpc_mtx;			/* proc lock */
+	struct fasttrap_proc *ftpc_next;	/* next proc in hash chain */
+} fasttrap_proc_t;
+
+typedef struct fasttrap_provider {
+	pid_t ftp_pid;				/* process ID for this prov */
+	fasttrap_provider_type_t ftp_provider_type;	/* type of this provider (usdt, pid, objc, oneshot) */
+	char ftp_name[DTRACE_PROVNAMELEN];	/* prov name (w/o the pid) */
+	dtrace_provider_id_t ftp_provid;	/* DTrace provider handle */
+	uint_t ftp_marked;			/* mark for possible removal */
+	uint_t ftp_retired;			/* mark when retired */
+	lck_mtx_t ftp_mtx;			/* provider lock */
+	lck_mtx_t ftp_cmtx;			/* lock on creating probes */
+	uint64_t ftp_rcount;			/* enabled probes ref count */
+	uint64_t ftp_ccount;			/* consumers creating probes */
+	uint64_t ftp_mcount;			/* meta provider count */
+	fasttrap_proc_t *ftp_proc;		/* shared proc for all provs */
+	struct fasttrap_provider *ftp_next;	/* next prov in hash chain */
+} fasttrap_provider_t;
+
+typedef struct fasttrap_id fasttrap_id_t;
+typedef struct fasttrap_probe fasttrap_probe_t;
+typedef struct fasttrap_tracepoint fasttrap_tracepoint_t;
+
+struct fasttrap_id {
+	fasttrap_probe_t *fti_probe;		/* referrring probe */
+	fasttrap_id_t *fti_next;		/* enabled probe list on tp */
+	fasttrap_probe_type_t fti_ptype;	/* probe type */
+};
+
+typedef struct fasttrap_id_tp {
+	fasttrap_id_t fit_id;
+	fasttrap_tracepoint_t *fit_tp;
+} fasttrap_id_tp_t;
+
+struct fasttrap_probe {
+	dtrace_id_t ftp_id;			/* DTrace probe identifier */
+	pid_t ftp_pid;				/* pid for this probe */
+	fasttrap_provider_t *ftp_prov;		/* this probe's provider */
+	user_addr_t ftp_faddr;			/* associated function's addr */
+	size_t ftp_fsize;			/* associated function's size */
+	uint64_t ftp_gen;			/* modification generation */
+	uint64_t ftp_ntps;			/* number of tracepoints */
+	uint8_t *ftp_argmap;			/* native to translated args */
+	uint8_t ftp_nargs;			/* translated argument count */
+	uint8_t ftp_enabled;			/* is this probe enabled */
+	char *ftp_xtypes;			/* translated types index */
+	char *ftp_ntypes;			/* native types index */
+	fasttrap_id_tp_t ftp_tps[1];		/* flexible array */
+};
+
+#define	FASTTRAP_ID_INDEX(id)	\
+((fasttrap_id_tp_t *)(((char *)(id) - offsetof(fasttrap_id_tp_t, fit_id))) - \
+&(id)->fti_probe->ftp_tps[0])
+
+struct fasttrap_tracepoint {
+	fasttrap_proc_t *ftt_proc;		/* associated process struct */
+	user_addr_t ftt_pc;			/* address of tracepoint */
+	pid_t ftt_pid;				/* pid of tracepoint */
+	fasttrap_machtp_t ftt_mtp;		/* ISA-specific portion */
+	fasttrap_id_t *ftt_ids;			/* NULL-terminated list */
+	fasttrap_id_t *ftt_retids;		/* NULL-terminated list */
+	fasttrap_tracepoint_t *ftt_next;	/* link in global hash */
+};
+
+typedef struct fasttrap_bucket {
+	lck_mtx_t ftb_mtx;			/* bucket lock */
+	void *ftb_data;				/* data payload */
+
+	uint8_t ftb_pad[64 - sizeof (lck_mtx_t) - sizeof (void *)];
+} fasttrap_bucket_t;
+
+typedef struct fasttrap_hash {
+	ulong_t fth_nent;			/* power-of-2 num. of entries */
+	ulong_t fth_mask;			/* fth_nent - 1 */
+	fasttrap_bucket_t *fth_table;		/* array of buckets */
+} fasttrap_hash_t;
+
+/*
+ * If at some future point these assembly functions become observable by
+ * DTrace, then these defines should become separate functions so that the
+ * fasttrap provider doesn't trigger probes during internal operations.
+ */
+#define	fasttrap_copyout	copyout
+#define	fasttrap_fuword32	fuword32
+#define	fasttrap_suword32	suword32
+
+#if defined __APPLE__
+/*
+ * xnu runs in 32 bit mode even when supporting 64 bit processes. We need
+ * to make size explicit.
+ */
+#define	fasttrap_fuword64	fuword64
+#define	fasttrap_suword64	suword64
+#define fasttrap_fuword64_noerr	fuword64_noerr
+#define fasttrap_fuword32_noerr	fuword32_noerr
+#else
+#define	fasttrap_fulword	fulword
+#define	fasttrap_sulword	sulword
+#endif
+
+extern void fasttrap_sigtrap(proc_t *, uthread_t, user_addr_t);
+
+extern dtrace_id_t 		fasttrap_probe_id;
+extern fasttrap_hash_t		fasttrap_tpoints;
+
+#define	FASTTRAP_TPOINTS_INDEX(pid, pc) \
+	(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints.fth_mask)
+
+/*
+ * Must be implemented by fasttrap_isa.c
+ */
+extern int fasttrap_tracepoint_init(proc_t *, fasttrap_tracepoint_t *,
+    user_addr_t, fasttrap_probe_type_t);
+extern int fasttrap_tracepoint_install(proc_t *, fasttrap_tracepoint_t *);
+extern int fasttrap_tracepoint_remove(proc_t *, fasttrap_tracepoint_t *);
+
+#if defined (__ppc__) || defined (__ppc64__)
+extern int fasttrap_pid_probe(ppc_saved_state_t *regs);
+extern int fasttrap_return_probe(ppc_saved_state_t* regs);
+#elif defined (__i386__) || defined(__x86_64__)
+extern int fasttrap_pid_probe(x86_saved_state_t *regs);
+extern int fasttrap_return_probe(x86_saved_state_t* regs);
+#elif defined(__arm__)
+extern int fasttrap_pid_probe(struct arm_saved_state *rp); /* so very, very, very broken... */
+#else
+#error architecture not supported
+#endif
+
+extern uint64_t fasttrap_pid_getarg(void *, dtrace_id_t, void *, int, int);
+extern uint64_t fasttrap_usdt_getarg(void *, dtrace_id_t, void *, int, int);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#undef proc_t
+
+#endif	/* _FASTTRAP_IMPL_H */
diff --git a/bsd/sys/fasttrap_isa.h b/bsd/sys/fasttrap_isa.h
new file mode 100644
index 000000000..0d952e47a
--- /dev/null
+++ b/bsd/sys/fasttrap_isa.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1989 Carnegie-Mellon University
+ * Copyright (c) 1988 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+/*
+ * HISTORY
+ */
+
+#ifndef	_SYS_FASTTRAP_ISA_H_
+#define _SYS_FASTTRAP_ISA_H_
+
+/*
+ * Machine dependent constants
+ */
+
+#include <machine/fasttrap_isa.h>
+
+#endif	/* _SYS_FASTTRAP_ISA_H_ */
diff --git a/bsd/sys/fbt.h b/bsd/sys/fbt.h
new file mode 100644
index 000000000..8121ebbbe
--- /dev/null
+++ b/bsd/sys/fbt.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _FBT_H
+#define _FBT_H
+
+#if defined (__ppc__) || defined (__ppc64__)
+typedef uint32_t machine_inst_t;
+#elif defined(__i386__) || defined (__x86_64__)
+typedef uint8_t machine_inst_t;
+#elif defined(__arm__)
+typedef uint32_t machine_inst_t; /* XXX is this correct? Thumb? */
+#else
+#error Unknown Architecture
+#endif
+
+#define MAX_FBTP_NAME_CHARS 16
+
+typedef struct fbt_probe {
+	struct fbt_probe *fbtp_hashnext;
+	machine_inst_t	*fbtp_patchpoint;
+	int8_t			fbtp_rval;
+	machine_inst_t	fbtp_patchval;
+	machine_inst_t	fbtp_savedval;
+	uintptr_t		fbtp_roffset;
+	dtrace_id_t		fbtp_id;
+	char			fbtp_name[MAX_FBTP_NAME_CHARS];
+	struct modctl	*fbtp_ctl;
+	int		fbtp_loadcnt;
+	int		fbtp_symndx;
+	int		fbtp_primary;
+	struct fbt_probe *fbtp_next;
+} fbt_probe_t;
+
+extern int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t);
+extern int fbt_invop(uintptr_t, uintptr_t *, uintptr_t);
+extern void fbt_provide_module(void *, struct modctl *);
+
+#endif /* _FBT_H */
diff --git a/bsd/sys/fcntl.h b/bsd/sys/fcntl.h
index 2ceeb325b..200b3e7ac 100644
--- a/bsd/sys/fcntl.h
+++ b/bsd/sys/fcntl.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -114,20 +120,20 @@ typedef __darwin_pid_t	pid_t;
  * FREAD and FWRITE are excluded from the #ifdef KERNEL so that TIOCFLUSH,
  * which was documented to use FREAD/FWRITE, continues to work.
  */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	FREAD		0x0001
 #define	FWRITE		0x0002
 #endif
 #define	O_NONBLOCK	0x0004		/* no delay */
 #define	O_APPEND	0x0008		/* set append mode */
 #define	O_SYNC		0x0080		/* synchronous writes */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	O_SHLOCK	0x0010		/* open with shared file lock */
 #define	O_EXLOCK	0x0020		/* open with exclusive file lock */
 #define	O_ASYNC		0x0040		/* signal pgrp when data ready */
 #define	O_FSYNC		O_SYNC		/* source compatibility: do not use */
 #define O_NOFOLLOW  0x0100      /* don't follow symlinks */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	O_CREAT		0x0200		/* create if nonexistant */
 #define	O_TRUNC		0x0400		/* truncate to zero length */
 #define	O_EXCL		0x0800		/* error if already exists */
@@ -136,7 +142,7 @@ typedef __darwin_pid_t	pid_t;
 #define	FDEFER		0x2000		/* defer for next gc pass */
 #define	FHASLOCK	0x4000		/* descriptor holds advisory lock */
 #endif
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	O_EVTONLY	0x8000		/* descriptor requested for event notifications only */
 #endif
 
@@ -144,12 +150,18 @@ typedef __darwin_pid_t	pid_t;
 #define	FWASWRITTEN	0x10000		/* descriptor was written */
 #endif
 
-#ifndef _POSIX_C_SOURCE
-#define O_DIRECTORY    0x100000
+#define	O_NOCTTY	0x20000		/* don't assign controlling terminal */
+
+#ifdef KERNEL
+#define FNOCACHE	0x40000		/* fcntl(F_NOCACHE, 1) */
+#define FNORDAHEAD	0x80000		/* fcntl(F_RDAHEAD, 0) */
+#endif
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define O_DIRECTORY	0x100000
+#define O_SYMLINK	0x200000	/* allow open of a symlink */
 #endif
 
-/* defined by POSIX 1003.1; BSD default, so no bit required */
-#define	O_NOCTTY	0		/* don't assign controlling terminal */
 //#define	O_SYNC  /* ??? POSIX: Write according to synchronized I/O file integrity completion */
 
 #ifdef KERNEL
@@ -168,7 +180,7 @@ typedef __darwin_pid_t	pid_t;
  * and by fcntl.  We retain the F* names for the kernel f_flags field
  * and for backward compatibility for fcntl.
  */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	FAPPEND		O_APPEND	/* kernel/compat */
 #define	FASYNC		O_ASYNC		/* kernel/compat */
 #define	FFSYNC		O_FSYNC		/* kernel */
@@ -181,7 +193,7 @@ typedef __darwin_pid_t	pid_t;
  * Flags used for copyfile(2)
  */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define CPF_OVERWRITE 1
 #define CPF_IGNORE_MODE 2
 #define CPF_MASK (CPF_OVERWRITE|CPF_IGNORE_MODE)
@@ -202,26 +214,37 @@ typedef __darwin_pid_t	pid_t;
 #define	F_GETLK		7		/* get record locking information */
 #define	F_SETLK		8		/* set record locking information */
 #define	F_SETLKW	9		/* F_SETLK; wait if blocked */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define F_CHKCLEAN      41              /* Used for regression test */
 #define F_PREALLOCATE   42		/* Preallocate storage */
 #define F_SETSIZE       43		/* Truncate a file without zeroing space */	
 #define F_RDADVISE      44              /* Issue an advisory read async with no copy to user */
-#define F_RDAHEAD       45              /* turn read ahead off/on */
+#define F_RDAHEAD       45              /* turn read ahead off/on for this fd */
 #define F_READBOOTSTRAP 46              /* Read bootstrap from disk */
 #define F_WRITEBOOTSTRAP 47             /* Write bootstrap on disk */
-#define F_NOCACHE       48              /* turning data caching off/on */
+#define F_NOCACHE       48              /* turn data caching off/on for this fd */
 #define F_LOG2PHYS	49		/* file offset to device offset */
 #define F_GETPATH       50              /* return the full path of the fd */
 #define F_FULLFSYNC     51		/* fsync + ask the drive to flush to the media */
 #define F_PATHPKG_CHECK 52              /* find which component (if any) is a package */
 #define F_FREEZE_FS     53              /* "freeze" all fs operations */
 #define F_THAW_FS       54              /* "thaw" all fs operations */
-#define F_GLOBAL_NOCACHE 55		/* turn data caching off/on (globally) for this file */
+#define	F_GLOBAL_NOCACHE 55		/* turn data caching off/on (globally) for this file */
+
+#ifdef PRIVATE
+#define	F_OPENFROM	56		/* SPI: open a file relative to fd (must be a dir) */
+#define	F_UNLINKFROM	57		/* SPI: open a file relative to fd (must be a dir) */
+#define	F_CHECK_OPENEVT 58		/* SPI: if a process is marked OPENEVT, or in O_EVTONLY on opens of this vnode */
+#endif /* PRIVATE */
+
+#define F_ADDSIGS	59		/* add detached signatures */
+
+#define F_MARKDEPENDENCY 60             /* this process hosts the device supporting the fs backing this fd */
 
 // FS-specific fcntl()'s numbers begin at 0x00010000 and go up
 #define FCNTL_FS_SPECIFIC_BASE  0x00010000
-#endif /* _POSIX_C_SOURCE */
+
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /* file descriptor flags (F_GETFD, F_SETFD) */
 #define	FD_CLOEXEC	1		/* close-on-exec flag */
@@ -234,6 +257,7 @@ typedef __darwin_pid_t	pid_t;
 #define	F_WAIT		0x010		/* Wait until lock is granted */
 #define	F_FLOCK		0x020	 	/* Use flock(2) semantics for lock */
 #define	F_POSIX		0x040	 	/* Use POSIX semantics for lock */
+#define	F_PROV		0x080		/* Non-coelesced provisional lock */
 #endif
 
 /*
@@ -260,7 +284,7 @@ typedef __darwin_pid_t	pid_t;
 #define	S_IFREG		0100000		/* [XSI] regular */
 #define	S_IFLNK		0120000		/* [XSI] symbolic link */
 #define	S_IFSOCK	0140000		/* [XSI] socket */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	S_IFWHT		0160000		/* whiteout */
 #define S_IFXATTR	0200000		/* extended attribute */
 #endif
@@ -286,7 +310,7 @@ typedef __darwin_pid_t	pid_t;
 #define	S_ISGID		0002000		/* [XSI] set group id on execution */
 #define	S_ISVTX		0001000		/* [XSI] directory restrcted delete */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	S_ISTXT		S_ISVTX		/* sticky bit: not supported */
 #define	S_IREAD		S_IRUSR		/* backward compatability */
 #define	S_IWRITE	S_IWUSR		/* backward compatability */
@@ -294,7 +318,7 @@ typedef __darwin_pid_t	pid_t;
 #endif
 #endif	/* !S_IFMT */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* allocate flags (F_PREALLOCATE) */
 
 #define F_ALLOCATECONTIG  0x00000002    /* allocate contigious space */
@@ -305,7 +329,7 @@ typedef __darwin_pid_t	pid_t;
 #define F_PEOFPOSMODE 3			/* Make it past all of the SEEK pos modes so that */
 					/* we can keep them in sync should we desire */	
 #define F_VOLPOSMODE	4		/* specify volume starting postion */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Advisory file segment locking data type -
@@ -320,7 +344,7 @@ struct flock {
 };
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * advisory file read data type -
  * information passed by user to system
@@ -330,6 +354,27 @@ struct radvisory {
        int     ra_count;
 };
 
+/*
+ * detached code signatures data type -
+ * information passed by user to system
+ * used by F_ADDSIGS
+ */
+typedef struct fsignatures {
+	off_t		fs_file_start;
+	void		*fs_blob_start;
+	size_t		fs_blob_size;
+} fsignatures_t;
+#ifdef KERNEL
+/* LP64 version of fsignatures.  all pointers 
+ * grow when we're dealing with a 64-bit process.
+ * WARNING - keep in sync with fsignatures
+ */
+typedef struct user_fsignatures {
+	off_t		fs_file_start;
+	user_addr_t	fs_blob_start;
+	user_size_t	fs_blob_size;
+} user_fsignatures_t;
+#endif /* KERNEL */
 
 /* lock operations for flock(2) */
 #define	LOCK_SH		0x01		/* shared file lock */
@@ -356,7 +401,6 @@ typedef struct fbootstraptransfer {
 } fbootstraptransfer_t;
 
 
-// LP64todo - should this move?
 #ifdef KERNEL
 /* LP64 version of fbootstraptransfer.  all pointers 
  * grow when we're dealing with a 64-bit process.
@@ -399,11 +443,38 @@ struct log2phys {
 
 #define	O_POPUP	   0x80000000   /* force window to popup on open */
 #define	O_ALERT	   0x20000000	/* small, clean popup window */
-#endif /* _POSIX_C_SOURCE */
+
+#ifdef PRIVATE
+/*
+ * SPI: Argument data for F_OPENFROM
+ */
+struct fopenfrom {
+	unsigned int	o_flags;	/* same as open(2) */
+	mode_t		o_mode;		/* same as open(2) */
+	char *		o_pathname;	/* relative pathname */
+};
+
+#ifdef KERNEL
+/*
+ * LP64 version of fopenfrom.  Memory pointers 
+ * grow when we're dealing with a 64-bit process.
+ *
+ * WARNING - keep in sync with fopenfrom (above)
+ */
+struct user_fopenfrom {
+	unsigned int	o_flags;
+	mode_t		o_mode;
+	user_addr_t	o_pathname;
+};
+#endif /* KERNEL */
+
+#endif /* PRIVATE */
+
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifndef KERNEL
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifndef _FILESEC_T
 struct _filesec;
 typedef struct _filesec	*filesec_t;
@@ -424,13 +495,13 @@ typedef enum {
 
 /* XXX backwards compatibility */
 #define FILESEC_GUID FILESEC_UUID
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 __BEGIN_DECLS
-int	open(const char *, int, ...);
-int	creat(const char *, mode_t);
-int	fcntl(int, int, ...);
-#ifndef _POSIX_C_SOURCE
+int	open(const char *, int, ...) __DARWIN_ALIAS_C(open);
+int	creat(const char *, mode_t) __DARWIN_ALIAS_C(creat);
+int	fcntl(int, int, ...) __DARWIN_ALIAS_C(fcntl);
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int	openx_np(const char *, int, filesec_t);
 int	flock(int, int);
 filesec_t filesec_init(void);
@@ -438,11 +509,10 @@ filesec_t filesec_dup(filesec_t);
 void	filesec_free(filesec_t);
 int	filesec_get_property(filesec_t, filesec_property_t, void *);
 int	filesec_set_property(filesec_t, filesec_property_t, const void *);
-int	filesec_unset_property(filesec_t, filesec_property_t);
 int	filesec_query_property(filesec_t, filesec_property_t, int *);
 #define _FILESEC_UNSET_PROPERTY	((void *)0)
 #define _FILESEC_REMOVE_ACL	((void *)1)
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 __END_DECLS
 #endif
 
diff --git a/bsd/sys/file.h b/bsd/sys/file.h
index 492d64aca..9bf925acb 100644
--- a/bsd/sys/file.h
+++ b/bsd/sys/file.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -70,6 +76,12 @@
 #include <sys/kernel_types.h>
 #endif
 
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
+struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
+
 #pragma pack(4)
 
 /* for the compat sake;  */
@@ -79,7 +91,7 @@ struct extern_file {
 	short	f_type;		/* descriptor type */
 	short	f_count;	/* reference count */
 	short	f_msgcount;	/* references from message queue */
-	struct	ucred *f_cred;	/* credentials associated with descriptor */
+	kauth_cred_t f_cred;	/* credentials associated with descriptor */
 	void * f_ops;
 	off_t	f_offset;
 	caddr_t	f_data;		/* vnode or socket or SHM or semaphore */
diff --git a/bsd/sys/file_internal.h b/bsd/sys/file_internal.h
index a4ed40ce0..ea0813377 100644
--- a/bsd/sys/file_internal.h
+++ b/bsd/sys/file_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -82,7 +88,7 @@ struct file;
  * One entry for each open kernel vnode and socket.
  */
 struct fileproc {
-	int32_t	f_flags;
+	unsigned int f_flags;
 	int32_t f_iocount;
 	struct fileglob * f_fglob;
 	void *	f_waddr;
@@ -100,31 +106,32 @@ struct fileproc {
 #define FP_AIOISSUED	0x0080
 #define FP_WAITEVENT	0x0100
 
+#define FP_VALID_FLAGS (FP_INCREATE | FP_INCLOSE | FP_INSELECT | FP_INCHRREAD | FP_WRITTEN | FP_WRITTEN | FP_CLOSING | FP_WAITCLOSE | FP_AIOISSUED | FP_WAITEVENT)
+
+
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
+struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
 
-/* defns of close_internal */
-#define CLOSEINT_LOCKED     1
-#define CLOSEINT_WAITONCLOSE 2
-#define CLOSEINT_NOFDRELSE  4
-#define CLOSEINT_NOFDNOREF  8
-  
 /* file types */
 typedef enum {
-	DTYPE_VNODE     = 1,    /* file */
-	DTYPE_SOCKET,           /* communications endpoint */
-	DTYPE_PSXSHM,           /* POSIX Shared memory */
-	DTYPE_PSXSEM,           /* POSIX Semaphores */
-	DTYPE_KQUEUE,           /* kqueue */
-	DTYPE_PIPE,             /* pipe */
-	DTYPE_FSEVENTS          /* fsevents */
+	DTYPE_VNODE 	= 1,	/* file */
+	DTYPE_SOCKET,		/* communications endpoint */
+	DTYPE_PSXSHM,		/* POSIX Shared memory */
+	DTYPE_PSXSEM,		/* POSIX Semaphores */
+	DTYPE_KQUEUE,		/* kqueue */
+	DTYPE_PIPE,		/* pipe */
+	DTYPE_FSEVENTS		/* fsevents */
 } file_type_t;
 
 /* defines for fg_lflags */
-#define FG_TERM        0x01    /* the fileglob is terminating .. */
-#define FG_INSMSGQ     0x02    /* insert to msgqueue pending .. */
-#define FG_WINSMSGQ    0x04    /* wait for the fielglob is in msgque */
-#define FG_RMMSGQ      0x08    /* the fileglob is being removed from msgqueue */
-#define FG_WRMMSGQ     0x10    /* wait for the fileglob to  be removed from msgqueue */
-
+#define FG_TERM 	0x01	/* the fileglob is terminating .. */
+#define FG_INSMSGQ 	0x02	/* insert to msgqueue pending .. */
+#define FG_WINSMSGQ	0x04 	/* wait for the fielglob is in msgque */
+#define FG_RMMSGQ	0x08 	/* the fileglob is being removed from msgqueue */
+#define FG_WRMMSGQ	0x10 	/* wait for the fileglob to  be removed from msgqueue */
 
 struct fileglob {
 	LIST_ENTRY(fileglob) f_list;/* list of active files */
@@ -133,24 +140,22 @@ struct fileglob {
 	file_type_t fg_type;		/* descriptor type */
 	int32_t	fg_count;	/* reference count */
 	int32_t	fg_msgcount;	/* references from message queue */
-	struct	ucred *fg_cred;	/* credentials associated with descriptor */
+	kauth_cred_t fg_cred;	/* credentials associated with descriptor */
 	struct	fileops {
-		int	(*fo_read)	__P((struct fileproc *fp, struct uio *uio,
-					    struct ucred *cred, int flags,
-					    struct proc *p));
-		int	(*fo_write)	__P((struct fileproc *fp, struct uio *uio,
-					    struct ucred *cred, int flags,
-					    struct proc *p));
+		int	(*fo_read)	(struct fileproc *fp, struct uio *uio,
+					 int flags, vfs_context_t ctx);
+		int	(*fo_write)	(struct fileproc *fp, struct uio *uio,
+					 int flags, vfs_context_t ctx);
 #define	FOF_OFFSET	0x00000001	/* offset supplied to vn_write */
 #define FOF_PCRED	0x00000002	/* cred from proc, not current thread */
-		int	(*fo_ioctl)	__P((struct fileproc *fp, u_long com,
-					    caddr_t data, struct proc *p));
-		int	(*fo_select)	__P((struct fileproc *fp, int which,
-						void *wql, struct proc *p));
-		int	(*fo_close)	__P((struct fileglob *fg, struct proc *p));
-		int	(*fo_kqfilter)	__P((struct fileproc *fp, struct knote *kn,
-					     struct proc *p));
-		int	(*fo_drain)	(struct fileproc *fp, struct proc *p);
+		int	(*fo_ioctl)	(struct fileproc *fp, u_long com,
+					 caddr_t data, vfs_context_t ctx);
+		int	(*fo_select)	(struct fileproc *fp, int which,
+					 void *wql, vfs_context_t ctx);
+		int	(*fo_close)	(struct fileglob *fg, vfs_context_t ctx);
+		int	(*fo_kqfilter)	(struct fileproc *fp, struct knote *kn,
+					 vfs_context_t ctx);
+		int	(*fo_drain)	(struct fileproc *fp, vfs_context_t ctx);
 	} *fg_ops;
 	off_t	fg_offset;
 	caddr_t	fg_data;		/* vnode or socket or SHM or semaphore */
@@ -158,6 +163,9 @@ struct fileglob {
 	int32_t fg_lflags;		/* file global flags */
 	unsigned int fg_lockpc[4];
 	unsigned int fg_unlockpc[4];
+#if CONFIG_MACF
+	struct label *fg_label;  /* JMM - use the one in the cred? */
+#endif
 };
 
 #ifdef __APPLE_API_PRIVATE
@@ -171,20 +179,14 @@ extern int nfiles;			/* actual number of open files */
 
 
 __BEGIN_DECLS
-int fo_read(struct fileproc *fp, struct uio *uio,
-	struct ucred *cred, int flags, struct proc *p);
-int fo_write(struct fileproc *fp, struct uio *uio,
-	struct ucred *cred, int flags, struct proc *p);
-int fo_ioctl(struct fileproc *fp, u_long com, caddr_t data,
-	struct proc *p);
-int fo_select(struct fileproc *fp, int which, void *wql,
-	struct proc *p);
-int fo_close(struct fileglob *fg, struct proc *p);
-int fo_kqfilter(struct fileproc *fp, struct knote *kn,
-	struct proc *p);
+int fo_read(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx);
+int fo_write(struct fileproc *fp, struct uio *uio, int flags,
+		vfs_context_t ctx);
+int fo_ioctl(struct fileproc *fp, u_long com, caddr_t data, vfs_context_t ctx);
+int fo_select(struct fileproc *fp, int which, void *wql, vfs_context_t ctx);
+int fo_close(struct fileglob *fg, vfs_context_t ctx);
+int fo_kqfilter(struct fileproc *fp, struct knote *kn, vfs_context_t ctx);
 void fileproc_drain(proc_t, struct fileproc *);
-void fp_setflags(proc_t, struct fileproc *, int);
-void fp_clearflags(proc_t, struct fileproc *, int);
 int fp_drop(struct proc *p, int fd, struct fileproc *fp, int locked);
 int fp_drop_written(proc_t p, int fd, struct fileproc *fp);
 int fp_drop_event(proc_t p, int fd, struct fileproc *fp);
@@ -205,7 +207,6 @@ int fp_getfvpandvid(struct proc *p, int fd, struct fileproc **resultfp, struct v
 struct socket;
 int fp_getfsock(struct proc *p, int fd, struct fileproc **resultfp, struct socket  **results);
 int fp_lookup(struct proc *p, int fd, struct fileproc **resultfp, int locked);
-int close_internal(struct proc *p, int fd, struct fileproc *fp, int flags);
 int closef_locked(struct fileproc *fp, struct fileglob *fg, struct proc *p);
 void fg_insertuipc(struct fileglob * fg);
 void fg_removeuipc(struct fileglob * fg);
diff --git a/bsd/sys/filedesc.h b/bsd/sys/filedesc.h
index 59300496c..b9839f3e9 100644
--- a/bsd/sys/filedesc.h
+++ b/bsd/sys/filedesc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -86,10 +92,10 @@ struct filedesc {
 	struct	vnode *fd_cdir;		/* current directory */
 	struct	vnode *fd_rdir;		/* root directory */
 	int	fd_nfiles;		/* number of open files allocated */
-	u_short	fd_lastfile;		/* high-water mark of fd_ofiles */
-	u_short	fd_freefile;		/* approx. next free file */
+	int	fd_lastfile;		/* high-water mark of fd_ofiles */
+	int	fd_freefile;		/* approx. next free file */
 	u_short	fd_cmask;		/* mask for file creation */
-	u_short	fd_refcnt;		/* reference count */
+	u_long	fd_refcnt;		/* reference count */
 
 	int     fd_knlistsize;          /* size of knlist */
 	struct  klist *fd_knlist;       /* list of attached knotes */
@@ -114,7 +120,9 @@ struct filedesc {
 
 #ifdef KERNEL
 #define UF_RESVWAIT	0x10		/* close in progress */
-#endif
+#define UF_VALID_FLAGS	(UF_EXCLOSE| UF_RESERVED | UF_CLOSING | UF_RESVWAIT)
+#endif /* KERNEL */
+
 /*
  * Storage required per open file descriptor.
  */
@@ -126,21 +134,20 @@ struct filedesc {
  */
 extern int	dupfdopen(struct filedesc *fdp,
 				int indx, int dfd, int mode, int error);
-extern int	fdalloc(struct proc *p, int want, int *result);
-extern void	fdrelse(struct proc *p, int fd);
-extern int	fdavail(struct proc *p, int n);
+extern int	fdalloc(proc_t p, int want, int *result);
+extern void	fdrelse(proc_t p, int fd);
+extern int	fdavail(proc_t p, int n);
 #define		fdfile(p, fd)					\
 			(&(p)->p_fd->fd_ofiles[(fd)])
 #define		fdflags(p, fd)					\
 			(&(p)->p_fd->fd_ofileflags[(fd)])
-extern int	falloc(struct proc *p,
-				struct fileproc **resultfp, int *resultfd);
+extern int	falloc(proc_t p, struct fileproc **resultfp, int *resultfd, vfs_context_t ctx);
 extern void	ffree(struct file *fp);
 
 #ifdef __APPLE_API_PRIVATE
-extern struct	filedesc *fdcopy(struct proc *p);
-extern void	fdfree(struct proc *p);
-extern void	fdexec(struct proc *p);
+extern struct	filedesc *fdcopy(proc_t p, struct vnode *uth_cdir);
+extern void	fdfree(proc_t p);
+extern void	fdexec(proc_t p);
 #endif /* __APPLE_API_PRIVATE */
 
 #endif /* KERNEL */
diff --git a/bsd/sys/filio.h b/bsd/sys/filio.h
index 10465cc8e..de132c60d 100644
--- a/bsd/sys/filio.h
+++ b/bsd/sys/filio.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/fsctl.h b/bsd/sys/fsctl.h
index 0d2033f06..53bdbf31e 100644
--- a/bsd/sys/fsctl.h
+++ b/bsd/sys/fsctl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/fsevents.h b/bsd/sys/fsevents.h
index 91b9f5958..b8508c577 100644
--- a/bsd/sys/fsevents.h
+++ b/bsd/sys/fsevents.h
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 #ifndef FSEVENT_H
 #define FSEVENT_H 1
 
@@ -12,12 +39,31 @@
 #define FSE_FINDER_INFO_CHANGED  6
 #define FSE_CREATE_DIR           7
 #define FSE_CHOWN                8
+#define FSE_XATTR_MODIFIED       9
+#define FSE_XATTR_REMOVED       10
 
-#define FSE_MAX_EVENTS           9
+#define FSE_MAX_EVENTS          11
 #define FSE_ALL_EVENTS         998
 
 #define FSE_EVENTS_DROPPED     999
 
+//
+// These defines only apply if you've asked for extended
+// type info.  In that case the type field's low 12-bits
+// contain the basic types defined above and the top 4
+// bits contain flags that indicate if an event had other
+// events combined with it or if it represents a directory
+// that has dropped events below it.
+//
+#define FSE_TYPE_MASK          0x0fff
+#define FSE_FLAG_MASK          0xf000
+#define FSE_FLAG_SHIFT         12
+#define FSE_GET_FLAGS(type)    (((type) >> 12) & 0x000f)
+
+#define FSE_COMBINED_EVENTS          0x0001
+#define FSE_CONTAINS_DROPPED_EVENTS  0x0002
+
+
 // Actions for each event type
 #define FSE_IGNORE    0
 #define FSE_REPORT    1
@@ -37,45 +83,75 @@
 #define FSE_ARG_DEV      0x0009   // next arg is the file's dev_t
 #define FSE_ARG_MODE     0x000a   // next arg is the file's mode (as an int32, file type only)
 #define FSE_ARG_GID      0x000b   // next arg is the file's gid (gid_t)
-#define FSE_ARG_FINFO    0x000c   // kernel internal only
+#define FSE_ARG_FINFO    0x000c   // next arg is a packed finfo (dev, ino, mode, uid, gid)
 #define FSE_ARG_DONE     0xb33f   // no more arguments
 
 #define FSE_MAX_ARGS     12
 
+//
+// These are special bits that be set in the 32-bit mode
+// field that /dev/fsevents provides.
+//
+#define FSE_MODE_HLINK         (1 << 31)    // notification is for a hard-link
+#define FSE_MODE_LAST_HLINK    (1 << 30)    // link count == 0 on a hard-link delete 
+
 
 // ioctl's on /dev/fsevents
+#if defined(__x86_64__) || defined(__ppc64__)
+typedef struct fsevent_clone_args {
+    int8_t  *event_list;
+    int32_t  num_events;
+    int32_t  event_queue_depth;
+    int32_t *fd;
+} fsevent_clone_args;
+#else
 typedef struct fsevent_clone_args {
     int8_t  *event_list;
+    int32_t  pad1;
     int32_t  num_events;
     int32_t  event_queue_depth;
     int32_t *fd;
+    int32_t  pad2;
 } fsevent_clone_args;
+#endif
 
-#define	FSEVENTS_CLONE	_IOW('s', 1, fsevent_clone_args)
+#define	FSEVENTS_CLONE		_IOW('s', 1, fsevent_clone_args)
 
 
 // ioctl's on the cloned fd
+#if defined(__x86_64__) || defined(__ppc64__)
+typedef struct fsevent_dev_filter_args {
+    uint32_t  num_devices;
+    dev_t    *devices;
+} fsevent_dev_filter_args;
+#else
 typedef struct fsevent_dev_filter_args {
     uint32_t  num_devices;
     dev_t    *devices;
+    int32_t   pad1;
 } fsevent_dev_filter_args;
+#endif
 
-#define	FSEVENTS_DEVICE_FILTER	_IOW('s', 100, fsevent_dev_filter_args)
+#define	FSEVENTS_DEVICE_FILTER		_IOW('s', 100, fsevent_dev_filter_args)
+#define	FSEVENTS_WANT_COMPACT_EVENTS	_IO('s', 101)
+#define	FSEVENTS_WANT_EXTENDED_INFO	_IO('s', 102)
 
 
 #ifdef KERNEL
 
+void fsevents_init(void);
 int  need_fsevent(int type, vnode_t vp);
 int  add_fsevent(int type, vfs_context_t, ...);
 void fsevent_unmount(struct mount *mp);
 
 // misc utility functions for fsevent info and pathbuffers...
 typedef struct fse_info {
+    ino64_t    ino;
     dev_t      dev;
-    ino_t      ino;
     int32_t    mode;   // note: this is not a mode_t (it's 32-bits, not 16)
     uid_t      uid;
     gid_t      gid;
+    uint64_t   nlink;  // only filled in if the vnode is marked as a hardlink
 } fse_info;
 
 int   get_fse_info(struct vnode *vp, fse_info *fse, vfs_context_t ctx);
diff --git a/bsd/sys/fslog.h b/bsd/sys/fslog.h
new file mode 100644
index 000000000..c1bee8c64
--- /dev/null
+++ b/bsd/sys/fslog.h
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef	_FSLOG_H_
+#define _FSLOG_H_
+
+#include <sys/syslog.h>
+#include <sys/mount.h>
+#include <machine/limits.h>
+
+#ifdef KERNEL
+/* Log file system related error in key-value format identified by Apple 
+ * system log (ASL) facility.  The key-value pairs are string pointers 
+ * (char *) and are provided as variable arguments list.  A NULL value 
+ * indicates end of the list.
+ *
+ * Keys can not contain '[', ']', space, and newline.  Values can not 
+ * contain '[', ']', and newline.  If any key-value contains any of the 
+ * reserved characters, the behavior is undefined.  The caller of the 
+ * function should escape any occurrences of '[' and ']' by prefixing
+ * it with '\'.
+ * 
+ * The function takes a message ID which can be used to logically group
+ * different ASL messages.  Messages in same logical group have same message
+ * ID and have information to describe order of the message --- first, 
+ * middle, or last.
+ *
+ * The following message IDs have special meaning - 
+ * FSLOG_MSG_FIRST - This message is the first message in its logical
+ *	group.  This generates a unique message ID, creates two key-value 
+ *	pairs with message ID and order of the message as "First".
+ * FSLOG_MSG_LAST - This is really a MASK which should be logically OR'ed 
+ *	with message ID to indicate the last message for a logical group.  
+ *	This also creates two key-value pairs with message ID and order of 
+ *	message as "Last".
+ * FSLOG_MSG_SINGLE - This signifies that the message is the only message
+ * 	in its logical group.  Therefore no extra key-values are generated 
+ *	for this option.
+ * For all other values of message IDs, it regards them as intermediate 
+ * message and generates two key-value pairs with message ID and order of
+ * message as "Middle".
+ * 
+ * Returns - 
+ *	Message ID of the ASL message printed.  The caller should use
+ * 	this value to print intermediate messages or end the logical message
+ *	group.
+ *	For FSLOG_MSG_SINGLE option, it returns FSLOG_MSG_SINGLE. 
+ */
+unsigned long fslog_err(unsigned long msg_id, ... );
+
+/* Reserved message IDs to determine message order */
+#define FSLOG_MSG_SINGLE	ULONG_MAX
+#define FSLOG_MSG_FIRST		0x0
+#define FSLOG_MSG_LAST		(~(ULONG_MAX >> 1)) 
+
+#ifdef BSD_KERNEL_PRIVATE
+
+/* Log information about runtime file system corruption detected */
+void fslog_fs_corrupt(struct mount *mnt);
+
+/* Log information about IO error detected */
+void fslog_io_error(const buf_t bp);
+
+#endif /* BSD_KERNEL_PRIVATE */
+#endif /* KERNEL */
+
+/* Keys used by FSLog */
+#define FSLOG_KEY_FACILITY	"Facility"	/* Facility generating messages */
+#define FSLOG_KEY_LEVEL		"Level"		/* Priority level */
+#define FSLOG_KEY_MSG_ID	"FSLogMsgID"	/* Message ID */
+#define FSLOG_KEY_MSG_ORDER	"FSLogMsgOrder"	/* Message Order */
+#define FSLOG_KEY_READ_UID	"ReadUID"	/* Allow read access to this UID only */
+
+/* Values for message order (FSLOG_KEY_MSG_ORDER) */
+#define FSLOG_VAL_ORDER_FIRST	"First"		
+#define FSLOG_VAL_ORDER_MIDDLE	"Middle"
+#define FSLOG_VAL_ORDER_LAST 	"Last"
+
+/* Values used by FSLog */
+#define FSLOG_VAL_FACILITY	"com.apple.system.fs" /* Facility generating messages */
+#define FSLOG_VAL_LEVEL		LOG_ERR		/* Priority level */
+#define FSLOG_VAL_READ_UID	0		/* Allow read access to root only */
+
+/* Keys for IO/FS logging using FSLog */
+#define	FSLOG_KEY_ERR_TYPE	"ErrType"	/* Type of problem (IO, FS Corruption) */
+#define	FSLOG_KEY_ERRNO		"ErrNo"		/* Error number (Integer) */
+#define FSLOG_KEY_IOTYPE	"IOType"	/* Type of IO (Read/Write) */
+#define FSLOG_KEY_PHYS_BLKNUM	"PBlkNum"	/* Physical block number */
+#define FSLOG_KEY_LOG_BLKNUM	"LBlkNum"	/* Logical block number */
+#define FSLOG_KEY_DEVNODE	"DevNode"	/* Device node (f_mntfromname) */
+#define FSLOG_KEY_PATH		"Path"		/* File system path */
+#define FSLOG_KEY_MNTPT		"MountPt"	/* Mount point */
+
+/* Values for type of error (FSLOG_KEY_ERR_TYPE) */
+#define FSLOG_VAL_ERR_TYPE_IO	"IO"		/* IO error */
+#define FSLOG_VAL_ERR_TYPE_FS	"FS"		/* FS error */
+
+/* Values for type of operation (FSLOG_KEY_IOTYPE) */
+#define FSLOG_VAL_IOTYPE_READ	"Read"
+#define FSLOG_VAL_IOTYPE_WRITE	"Write"
+
+#endif /* !_FSLOG_H_ */
diff --git a/bsd/sys/gmon.h b/bsd/sys/gmon.h
index b368a1fd2..75e478c53 100644
--- a/bsd/sys/gmon.h
+++ b/bsd/sys/gmon.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000, 2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -57,20 +63,38 @@
 
 #ifndef _SYS_GMON_H_
 #define _SYS_GMON_H_
+#include <stdint.h>
 
 /*
  * Structure prepended to gmon.out profiling data file.
  */
 struct gmonhdr {
-	u_long	lpc;		/* base pc address of sample buffer */
-	u_long	hpc;		/* max pc address of sampled buffer */
-	int	ncnt;		/* size of sample buffer (plus this header) */
-	int	version;	/* version number */
-	int	profrate;	/* profiling clock rate */
-	int	spare[3];	/* reserved */
+	uint32_t lpc;		/* base pc address of sample buffer */
+	uint32_t hpc;		/* max pc address of sampled buffer */
+	uint32_t ncnt;		/* size of sample buffer (plus this header) */
+	int32_t version;	/* version number */
+	int32_t profrate;	/* profiling clock rate */
+	int32_t spare[3];	/* reserved */
 };
 #define GMONVERSION	0x00051879
 
+struct gmonhdr_64 {
+	uint64_t lpc;		/* base pc address of sample buffer */
+	uint64_t hpc;		/* max pc address of sampled buffer */
+	uint32_t ncnt;		/* size of sample buffer (plus this header) */
+	int32_t version;	/* version number */
+	int32_t profrate;	/* profiling clock rate */
+	int32_t spare[3];	/* reserved */
+};
+
+typedef struct
+#ifndef __LP64__
+        gmonhdr
+#else
+        gmonhdr_64
+#endif
+gmonhdr_t;
+
 /*
  * histogram counters are unsigned shorts (according to the kernel).
  */
@@ -119,22 +143,51 @@ struct gmonhdr {
 #define MAXARCS		((1 << (8 * sizeof(HISTCOUNTER))) - 2)
 
 struct tostruct {
-	u_long	selfpc;
-	long	count;
-	u_short	link;
-	u_short order;
+	uint32_t	selfpc;
+	int32_t		count;
+	uint16_t	link;
+	uint16_t	order;
 };
 
+struct tostruct_64 {
+	uint64_t	selfpc;
+	int32_t		count;
+	uint16_t	link;
+	uint16_t	order;
+};
+
+typedef struct
+#ifndef __LP64__
+        tostruct
+#else
+        tostruct_64
+#endif
+tostruct_t;
+
 /*
  * a raw arc, with pointers to the calling site and 
  * the called site and a count.
  */
 struct rawarc {
-	u_long	raw_frompc;
-	u_long	raw_selfpc;
-	long	raw_count;
+	uint32_t	raw_frompc;
+	uint32_t	raw_selfpc;
+	int32_t		raw_count;
+};
+
+struct rawarc_64 {
+	uint64_t	raw_frompc;
+	uint64_t	raw_selfpc;
+	int32_t		raw_count;
 };
 
+typedef struct
+#ifndef __LP64__
+        rawarc
+#else
+        rawarc_64
+#endif
+rawarc_t;
+
 /*
  * general rounding functions.
  */
@@ -150,7 +203,7 @@ struct gmonparam {
 	u_long		kcountsize;
 	u_short		*froms;
 	u_long		fromssize;
-	struct tostruct	*tos;
+	tostruct_t	*tos;
 	u_long		tossize;
 	long		tolimit;
 	u_long		lowpc;
@@ -177,21 +230,38 @@ extern struct gmonparam _gmonparam;
 #define	GPROF_TOS	3	/* struct: destination/count structure */
 #define	GPROF_GMONPARAM	4	/* struct: profiling parameters (see above) */
 
+
+/*
+ * Declarations for various profiling related functions from
+ * bsd/kern/subr_prof.c
+ */
+#ifdef GPROF
+#ifdef XNU_KERNEL_PRIVATE
+
+void kmstartup(void);
+void cfreemem(caddr_t, int);  /* Currently only a stub function. */
+void mcount(u_long, u_long);
+
+#endif /* XNU_KERNEL_PRIVATE */
+#endif /* GPROF */
+
+
 /*
  * In order to support more information than in the original mon.out and
  * gmon.out files there is an alternate gmon.out file format.  The alternate
  * gmon.out file format starts with a magic number then separates the
- * information with gmon_data structs.
+ * information with gmon_data_t's.
  */
 #define GMON_MAGIC 0xbeefbabe
-struct gmon_data {
-    unsigned long type; /* constant for type of data following this struct */
-    unsigned long size; /* size in bytes of the data following this struct */
-};
+#define GMON_MAGIC_64 0xbeefbabf
+typedef struct gmon_data {
+    uint32_t type; /* constant for type of data following this struct */
+    uint32_t size; /* size in bytes of the data following this struct */
+} gmon_data_t;
 
 /*
  * The GMONTYPE_SAMPLES gmon_data.type is for the histogram counters described
- * above and has a struct gmonhdr followed by the counters.
+ * above and has a gmonhdr_t followed by the counters.
  */
 #define GMONTYPE_SAMPLES	1
 /*
@@ -202,24 +272,54 @@ struct gmon_data {
  * The GMONTYPE_ARCS_ORDERS gmon_data.type is for the raw arcs with a call
  * order field.  The order is the order is a sequence number for the order each
  * call site was executed.  Raw_order values start at 1 not zero.  Other than
- * the raw_order field this is the same information as in the struct rawarc.
+ * the raw_order field this is the same information as in the rawarc_t.
  */
 #define GMONTYPE_ARCS_ORDERS	3
 struct rawarc_order {
-    unsigned long	raw_frompc;
-    unsigned long	raw_selfpc;
-    unsigned long	raw_count;
-    unsigned long	raw_order;
+    uint32_t	raw_frompc;
+    uint32_t	raw_selfpc;
+    uint32_t	raw_count;
+    uint32_t	raw_order;
+
+}; struct rawarc_order_64 {
+    uint64_t	raw_frompc;
+    uint64_t	raw_selfpc;
+    uint32_t	raw_count;
+    uint32_t	raw_order;
 };
+
+typedef struct
+#ifndef __LP64__
+        rawarc_order
+#else
+        rawarc_order_64
+#endif
+rawarc_order_t;
+
 /*
  * The GMONTYPE_DYLD_STATE gmon_data.type is for the dynamic link editor state
  * of the program.
- * The informations starts with an unsigned long with the count of states:
+ * The informations starts with an uint32_t with the count of states:
  *      image_count
  * Then each state follows in the file.  The state is made up of 
- *      image_header (the address where dyld loaded this image)
  *      vmaddr_slide (the amount dyld slid this image from it's vmaddress)
  *      name (the file name dyld loaded this image from)
+ * The vmaddr_slide is a 32-bit value for 32-bit programs and 64-bit value for
+ * 64-bit programs.
  */
 #define GMONTYPE_DYLD_STATE     4
+
+/*
+ * The GMONTYPE_DYLD2_STATE gmon_data.type is for the dynamic link editor state
+ * of the program.
+ * The informations starts with an uint32_t with the count of states:
+ *      image_count
+ * Then each state follows in the file.  The state is made up of 
+ *      image_header (the address where dyld loaded this image)
+ *      name (the file name dyld loaded this image from)
+ * The image_header is a 32-bit value for 32-bit programs and 64-bit value for
+ * 64-bit programs.
+ */
+#define GMONTYPE_DYLD2_STATE     5
+
 #endif /* !_SYS_GMON_H_ */
diff --git a/bsd/sys/imageboot.h b/bsd/sys/imageboot.h
new file mode 100644
index 000000000..9ab02b5ab
--- /dev/null
+++ b/bsd/sys/imageboot.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _IMAGEBOOT_H_
+#define _IMAGEBOOT_H_
+
+int imageboot_needed(void);
+int imageboot_setup(void);
+
+#endif
diff --git a/bsd/sys/imgact.h b/bsd/sys/imgact.h
index 58deac06c..1bed0a7c5 100644
--- a/bsd/sys/imgact.h
+++ b/bsd/sys/imgact.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1993, David Greenman
@@ -51,11 +57,18 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 #ifndef _SYS_IMGACT_H_
 #define	_SYS_IMGACT_H_
 
 #define	IMG_SHSIZE	512	/* largest shell interpreter, in bytes */
 
+struct label;
 struct proc;
 struct nameidata;
 
@@ -63,9 +76,12 @@ struct image_params {
 	user_addr_t	ip_user_fname;		/* argument */
 	user_addr_t	ip_user_argv;		/* argument */
 	user_addr_t	ip_user_envv;		/* argument */
+	int		ip_seg;			/* segment for arguments */
 	struct vnode	*ip_vp;			/* file */
 	struct vnode_attr	*ip_vattr;	/* run file attributes */
 	struct vnode_attr	*ip_origvattr;	/* invocation file attributes */
+	cpu_type_t	ip_origcputype;		/* cputype of invocation file */
+	cpu_subtype_t	ip_origcpusubtype;	/* subtype of invocation file */
 	char		*ip_vdata;		/* file data (up to one page) */
 	int		ip_flags;		/* image flags */
 	int		ip_argc;		/* argument count */
@@ -81,10 +97,17 @@ struct image_params {
 
 	/* Next two fields are for support of architecture translation... */
 	char		*ip_p_comm;		/* optional alt p->p_comm */
-	char		*ip_tws_cache_name;	/* task working set cache */
 	struct vfs_context	*ip_vfs_context;	/* VFS context */
 	struct nameidata *ip_ndp;		/* current nameidata */
 	thread_t	ip_vfork_thread;	/* thread created, if vfork */
+
+	struct label	*ip_execlabelp;		/* label of the executable */
+	struct label	*ip_scriptlabelp;	/* label of the script */
+	unsigned int	ip_csflags;		/* code signing flags */
+	int		ip_no_trans;		/* allow suid/sgid transition?*/
+	void		*ip_px_sa;
+	void		*ip_px_sfa;
+	void		*ip_px_spa;
 };
 
 /*
@@ -92,11 +115,7 @@ struct image_params {
  */
 #define	IMGPF_NONE	0x00000000		/* No flags */
 #define	IMGPF_INTERPRET	0x00000001		/* Interpreter invoked */
-#if defined (__i386__) || defined(__x86_64__)
-#define	IMGPF_POWERPC	0x00000002		/* ppc mode */
-#else
-#define	IMGPF_RESERVED1	0x00000002		/* reserved */
-#endif
+#define	IMGPF_POWERPC	0x00000002		/* ppc mode for x86 */
 #define	IMGPF_WAS_64BIT	0x00000004		/* exec from a 64Bit binary */
 #define	IMGPF_IS_64BIT	0x00000008		/* exec to a 64Bit binary */
 
diff --git a/bsd/sys/ioccom.h b/bsd/sys/ioccom.h
index 00e87d9aa..eebf1e745 100644
--- a/bsd/sys/ioccom.h
+++ b/bsd/sys/ioccom.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/ioctl.h b/bsd/sys/ioctl.h
index 006b04d44..f8fa38926 100644
--- a/bsd/sys/ioctl.h
+++ b/bsd/sys/ioctl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/ioctl_compat.h b/bsd/sys/ioctl_compat.h
index 11d837db0..55537e558 100644
--- a/bsd/sys/ioctl_compat.h
+++ b/bsd/sys/ioctl_compat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/sys/ipc.h b/bsd/sys/ipc.h
index 13479cf6a..14bed3fb7 100644
--- a/bsd/sys/ipc.h
+++ b/bsd/sys/ipc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 University of Utah.
@@ -105,12 +111,13 @@ typedef	__int32_t	key_t;
  * legacy interface there for binary compatibility only.  Currently, we
  * are only forcing this for programs requesting standards conformance.
  */
-#if defined(__POSIX_C_SOURCE) || defined(kernel) || defined(__LP64__)
+#if __DARWIN_UNIX03 || defined(KERNEL)
 /*
  * [XSI] Information used in determining permission to perform an IPC
  * operation
  */
-struct __ipc_perm_new {
+struct ipc_perm
+{
 	uid_t		uid;		/* [XSI] Owner's user ID */
 	gid_t		gid;		/* [XSI] Owner's group ID */
 	uid_t		cuid;		/* [XSI] Creator's user ID */
@@ -119,12 +126,12 @@ struct __ipc_perm_new {
 	unsigned short	_seq;		/* Reserved for internal use */
 	key_t		_key;		/* Reserved for internal use */
 };
-#define	ipc_perm	__ipc_perm_new
-#else	/* !_POSIX_C_SOURCE */
+#define	__ipc_perm_new	ipc_perm
+#else	/* !__DARWIN_UNIX03 */
 #define	ipc_perm	__ipc_perm_old
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
-#if !defined(__POSIX_C_SOURCE) && !defined(__LP64__)
+#if !__DARWIN_UNIX03
 /*
  * Legacy structure; this structure is maintained for binary backward
  * compatability with previous versions of the interface.  New code
@@ -139,7 +146,7 @@ struct __ipc_perm_old {
 	__uint16_t	seq;		/* Reserved for internal use */
 	key_t		key;		/* Reserved for internal use */
 };
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
 /*
  * [XSI] Definitions shall be provided for the following constants:
@@ -159,14 +166,14 @@ struct __ipc_perm_old {
 #define	IPC_STAT	2		/* Get options */
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 
 /* common mode bits */
 #define	IPC_R		000400		/* Read permission */
 #define	IPC_W		000200		/* Write/alter permission */
 #define	IPC_M		010000		/* Modify control info permission */
 
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 #ifdef BSD_KERNEL_PRIVATE
@@ -178,7 +185,7 @@ struct __ipc_perm_old {
 /* Macros to convert between ipc ids and array indices or sequence ids */
 #define	IPCID_TO_IX(id)		((id) & 0xffff)
 #define	IPCID_TO_SEQ(id)	(((id) >> 16) & 0xffff)
-#define	IXSEQ_TO_IPCID(ix,perm)	(((perm.seq) << 16) | (ix & 0xffff))
+#define	IXSEQ_TO_IPCID(ix,perm)	(((perm._seq) << 16L) | (ix & 0xffff))
 
 struct ucred;
 
diff --git a/bsd/sys/ipcs.h b/bsd/sys/ipcs.h
index 7592dc951..26cf1dcfa 100644
--- a/bsd/sys/ipcs.h
+++ b/bsd/sys/ipcs.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * NOTE: Internal ipcs.h header; all interfaces are private; if you want this
diff --git a/bsd/sys/kauth.h b/bsd/sys/kauth.h
index aa12d1668..8b41ac018 100644
--- a/bsd/sys/kauth.h
+++ b/bsd/sys/kauth.h
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #ifndef _SYS_KAUTH_H
@@ -25,6 +37,7 @@
 
 #include <sys/appleapiopts.h>
 #include <sys/cdefs.h>
+#include <mach/boolean.h>
 
 #ifdef __APPLE_API_EVOLVING
 
@@ -62,7 +75,11 @@ typedef struct {
 #define KAUTH_NTSID_SIZE(_s)	(KAUTH_NTSID_HDRSIZE + ((_s)->sid_authcount * sizeof(u_int32_t)))
 
 /*
- * External lookup message payload
+ * External lookup message payload; this structure is shared between the
+ * kernel group membership resolver, and the user space group membership
+ * resolver daemon, and is use to communicate resolution requests from the
+ * kernel to user space, and the result of that request from user space to
+ * the kernel.
  */
 struct kauth_identity_extlookup {
 	u_int32_t	el_seqno;	/* request sequence number */
@@ -176,17 +193,32 @@ extern kauth_cred_t kauth_cred_proc_ref(proc_t procp);
 extern kauth_cred_t kauth_cred_alloc(void);
 extern kauth_cred_t kauth_cred_create(kauth_cred_t cred);
 extern void	kauth_cred_ref(kauth_cred_t _cred);
-extern void	kauth_cred_rele(kauth_cred_t _cred);
+/* Use kauth_cred_unref(), not kauth_cred_rele() */
+extern void	kauth_cred_rele(kauth_cred_t _cred) __deprecated;
 extern kauth_cred_t kauth_cred_dup(kauth_cred_t cred);
 extern kauth_cred_t kauth_cred_copy_real(kauth_cred_t cred);
 extern void	kauth_cred_unref(kauth_cred_t *_cred);
-extern kauth_cred_t	kauth_cred_setuid(kauth_cred_t cred, uid_t uid);
-extern kauth_cred_t	kauth_cred_seteuid(kauth_cred_t cred, uid_t euid);
-extern kauth_cred_t	kauth_cred_setgid(kauth_cred_t cred, gid_t gid);
-extern kauth_cred_t	kauth_cred_setegid(kauth_cred_t cred, gid_t egid);
+extern kauth_cred_t	kauth_cred_setresuid(kauth_cred_t cred, uid_t ruid, uid_t euid, uid_t svuid, uid_t gmuid);
+extern kauth_cred_t	kauth_cred_setresgid(kauth_cred_t cred, gid_t rgid, gid_t egid, gid_t svgid);
 extern kauth_cred_t kauth_cred_setuidgid(kauth_cred_t cred, uid_t uid, gid_t gid);
 extern kauth_cred_t kauth_cred_setsvuidgid(kauth_cred_t cred, uid_t uid, gid_t gid);
 extern kauth_cred_t	kauth_cred_setgroups(kauth_cred_t cred, gid_t *groups, int groupcount, uid_t gmuid);
+struct uthread;
+extern void	kauth_cred_uthread_update(struct uthread *, proc_t);
+#if CONFIG_MACF
+struct label;
+extern kauth_cred_t	kauth_cred_label_update(kauth_cred_t cred, struct label *label);
+extern int kauth_proc_label_update(struct proc *p, struct label *label);
+extern int kauth_proc_label_update_execve(struct proc *p, struct vfs_context *ctx, struct vnode *vp, struct label *scriptlabel, struct label *execlabel);
+#else
+/* this is a temp hack to cover us when MAC is not built in a kernel configuration. 
+ * Since we cannot build our export list based on the kernel configuration we need
+ * to define a stub. 
+ */
+extern kauth_cred_t	kauth_cred_label_update(kauth_cred_t cred, void *label);
+extern int kauth_proc_label_update(struct proc *p, void *label);
+#endif
+
 extern kauth_cred_t kauth_cred_find(kauth_cred_t cred);
 extern int	kauth_cred_getgroups(gid_t *_groups, int *_groupcount);
 extern int	kauth_cred_assume(uid_t _uid);
@@ -206,12 +238,16 @@ extern int      kauth_cred_gid2ntsid(gid_t _gid, ntsid_t *_sidp);
 extern int      kauth_cred_guid2ntsid(guid_t *_guid, ntsid_t *_sidp);
 extern int	kauth_cred_ismember_gid(kauth_cred_t _cred, gid_t _gid, int *_resultp);
 extern int	kauth_cred_ismember_guid(kauth_cred_t _cred, guid_t *_guidp, int *_resultp);
+extern int	kauth_cred_gid_subset(kauth_cred_t _cred1, kauth_cred_t _cred2, int *_resultp);
+
+struct auditinfo;
+extern kauth_cred_t kauth_cred_setauditinfo(kauth_cred_t, struct auditinfo *);
 
 extern int	kauth_cred_supplementary_register(const char *name, int *ident);
 extern int	kauth_cred_supplementary_add(kauth_cred_t cred, int ident, const void *data, size_t datasize);
 extern int	kauth_cred_supplementary_remove(kauth_cred_t cred, int ident);
 
-/* NOT KPI - fast path for in-kernel code only */
+/* currently only exported in unsupported for use by seatbelt */
 extern int	kauth_cred_issuser(kauth_cred_t _cred);
 
 
@@ -256,6 +292,12 @@ struct kauth_ace {
 #define KAUTH_ACE_ONLY_INHERIT		(1<<8)
 #define KAUTH_ACE_SUCCESS		(1<<9)	/* not implemented (AUDIT/ALARM) */
 #define KAUTH_ACE_FAILURE		(1<<10)	/* not implemented (AUDIT/ALARM) */
+/* All flag bits controlling ACE inheritance */
+#define KAUTH_ACE_INHERIT_CONTROL_FLAGS		\
+		(KAUTH_ACE_FILE_INHERIT |	\
+		 KAUTH_ACE_DIRECTORY_INHERIT |	\
+		 KAUTH_ACE_LIMIT_INHERIT |	\
+		 KAUTH_ACE_ONLY_INHERIT)
 	kauth_ace_rights_t ace_rights;		/* scope specific */
 	/* These rights are never tested, but may be present in an ACL */
 #define KAUTH_ACE_GENERIC_ALL		(1<<21) 
@@ -368,6 +410,12 @@ typedef struct kauth_filesec *kauth_filesec_t;
 #define	KAUTH_ENDIAN_HOST	0x00000001	/* set host endianness */
 #define	KAUTH_ENDIAN_DISK	0x00000002	/* set disk endianness */
 
+#endif /* KERNEL || <sys/acl.h> */
+
+
+#ifdef KERNEL
+
+/* KPI */
 __BEGIN_DECLS
 kauth_filesec_t	kauth_filesec_alloc(int size);
 void		kauth_filesec_free(kauth_filesec_t fsp);
@@ -375,10 +423,6 @@ int		kauth_copyinfilesec(user_addr_t xsecurity, kauth_filesec_t *xsecdestpp);
  void		kauth_filesec_acl_setendian(int, kauth_filesec_t, kauth_acl_t);
 __END_DECLS	
 
-#endif /* KERNEL || <sys/acl.h> */
-
-
-#ifdef KERNEL
 /*
  * Scope management.
  */
@@ -409,6 +453,7 @@ struct kauth_acl_eval {
 	kauth_ace_rights_t	ae_requested;
 	kauth_ace_rights_t	ae_residual;
 	int			ae_result;
+        boolean_t		ae_found_deny;
 	int			ae_options;
 #define KAUTH_AEVAL_IS_OWNER	(1<<0)		/* authorizing operation for owner */
 #define KAUTH_AEVAL_IN_GROUP	(1<<1)		/* authorizing operation for groupmember */
@@ -479,9 +524,10 @@ __END_DECLS
 #define KAUTH_FILEOP_OPEN			1
 #define KAUTH_FILEOP_CLOSE			2
 #define KAUTH_FILEOP_RENAME			3
-#define KAUTH_FILEOP_EXCHANGE		4
+#define KAUTH_FILEOP_EXCHANGE			4
 #define KAUTH_FILEOP_LINK			5
 #define KAUTH_FILEOP_EXEC			6
+#define KAUTH_FILEOP_DELETE			7
 
 /*
  * arguments passed to KAUTH_FILEOP_OPEN listeners
@@ -503,6 +549,9 @@ __END_DECLS
  * arguments passed to KAUTH_FILEOP_EXEC listeners
  *		arg0 is pointer to vnode (vnode *) for executable.
  *		arg1 is pointer to path (char *) to executable.
+ * arguments passed to KAUTH_FILEOP_DELETE listeners
+ *		arg0 is pointer to vnode (vnode *) of file/dir that was deleted.
+ *		arg1 is pointer to path (char *) of file/dir that was deleted.
  */
  
 /* Flag values returned to close listeners. */
@@ -578,6 +627,27 @@ __END_DECLS
  */
 #define KAUTH_VNODE_NOIMMUTABLE			(1<<30)
 
+
+/*
+ * fake right that is composed by the following...
+ * vnode must have search for owner, group and world allowed
+ * plus there must be no deny modes present for SEARCH... this fake
+ * right is used by the fast lookup path to avoid checking
+ * for an exact match on the last credential to lookup
+ * the component being acted on
+ */
+#define KAUTH_VNODE_SEARCHBYANYONE		(1<<29)
+
+
+/*
+ * when passed as an 'action' to "vnode_uncache_authorized_actions"
+ * it indicates that all of the cached authorizations for that
+ * vnode should be invalidated 
+ */
+#define	KAUTH_INVALIDATE_CACHED_RIGHTS		((kauth_action_t)~0)
+
+
+
 /* The expansions of the GENERIC bits at evaluation time */
 #define KAUTH_VNODE_GENERIC_READ_BITS	(KAUTH_VNODE_READ_DATA |		\
 					KAUTH_VNODE_READ_ATTRIBUTES |		\
@@ -629,7 +699,7 @@ __END_DECLS
 # ifndef _FN_KPRINTF
 #  define	_FN_KPRINTF
 void kprintf(const char *fmt, ...);
-# endif
+# endif	/* !_FN_KPRINTF */
 # define KAUTH_DEBUG_ENABLE
 # define K_UUID_FMT "%08x:%08x:%08x:%08x"
 # define K_UUID_ARG(_u) *(int *)&_u.g_guid[0],*(int *)&_u.g_guid[4],*(int *)&_u.g_guid[8],*(int *)&_u.g_guid[12]
@@ -643,24 +713,23 @@ void kprintf(const char *fmt, ...);
 		    __PRETTY_FUNCTION__, __LINE__ ,					\
 		    ##args);								\
 	} while(0)
-#else
+#else	/* !0 */
 # define KAUTH_DEBUG(fmt, args...)		do { } while (0)
 # define VFS_DEBUG(ctx, vp, fmt, args...)	do { } while(0)
-#endif
+#endif	/* !0 */
 
 /*
  * Initialisation.
  */
 extern lck_grp_t *kauth_lck_grp;
 __BEGIN_DECLS
-extern void	kauth_init(void);
-extern void	kauth_identity_init(void);
-extern void	kauth_groups_init(void);
-extern void	kauth_cred_init(void);
-extern void	kauth_resolver_init(void);
+extern void	kauth_init(void) __attribute__((section("__TEXT, initcode")));
+extern void	kauth_identity_init(void) __attribute__((section("__TEXT, initcode")));
+extern void	kauth_groups_init(void) __attribute__((section("__TEXT, initcode")));
+extern void	kauth_cred_init(void) __attribute__((section("__TEXT, initcode")));
+extern void	kauth_resolver_init(void) __attribute__((section("__TEXT, initcode")));
 __END_DECLS
-#endif
+#endif	/* KERNEL */
 
 #endif /* __APPLE_API_EVOLVING */
 #endif /* _SYS_KAUTH_H */
-
diff --git a/bsd/sys/kdebug.h b/bsd/sys/kdebug.h
index 3544ecd90..b8f903856 100644
--- a/bsd/sys/kdebug.h
+++ b/bsd/sys/kdebug.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* 	Copyright (c) 1997 Apple Computer, Inc.  All rights reserved. 
@@ -49,6 +55,8 @@ __BEGIN_DECLS
 #define DBG_PAGEIN_FAULT      2
 #define DBG_COW_FAULT         3
 #define DBG_CACHE_HIT_FAULT   4
+#define DBG_NZF_PAGE_FAULT    5
+#define DBG_GUARD_FAULT	      6	
 
 
 /* The debug code consists of the following 
@@ -98,9 +106,11 @@ __BEGIN_DECLS
 #define	DBG_MACH_IHDLR		0x10	/* Interrupt Handlers */
 #define	DBG_MACH_IPC		0x20	/* Inter Process Comm */
 #define	DBG_MACH_VM		0x30	/* Virtual Memory */
+#define	DBG_MACH_LEAKS		0x31    /* alloc/free */
 #define	DBG_MACH_SCHED		0x40	/* Scheduler */
 #define	DBG_MACH_MSGID_INVALID	0x50	/* Messages - invalid */
 #define DBG_MACH_LOCKS		0x60	/* new lock APIs */
+#define DBG_MACH_PMAP		0x70	/* pmap */
 
 /* Codes for Scheduler (DBG_MACH_SCHED) */     
 #define MACH_SCHED              0x0     /* Scheduler */
@@ -112,7 +122,20 @@ __BEGIN_DECLS
 #define MACH_MAKE_RUNNABLE      0x6     /* make thread runnable */
 #define	MACH_PROMOTE			0x7		/* promoted due to resource */
 #define	MACH_DEMOTE				0x8		/* promotion undone */
-#define MACH_PREBLOCK_MUTEX		0x9		/* preblocking on mutex */
+#define MACH_IDLE				0x9		/* processor idling */
+
+/* Codes for pmap (DBG_MACH_PMAP) */     
+#define PMAP__CREATE		0x0
+#define PMAP__DESTROY		0x1
+#define PMAP__PROTECT		0x2
+#define PMAP__PAGE_PROTECT	0x3
+#define PMAP__ENTER		0x4
+#define PMAP__REMOVE		0x5
+#define PMAP__NEST		0x6
+#define PMAP__UNNEST		0x7
+#define PMAP__FLUSH_TLBS	0x8
+#define PMAP__UPDATE_INTERRUPT	0x9
+#define PMAP__ATTRIBUTE_CLEAR	0xa
 
 /* **** The Kernel Debug Sub Classes for Network (DBG_NETWORK) **** */
 #define DBG_NETIP	1	/* Internet Protocol */
@@ -208,11 +231,13 @@ __BEGIN_DECLS
 #define DBG_DKRW      2       /* reads and writes to the disk */
 #define DBG_FSVN      3       /* vnode operations (inc. locking/unlocking) */
 #define DBG_FSLOOOKUP 4       /* namei and other lookup-related operations */
+#define DBG_JOURNAL   5       /* journaling operations */
 
 /* The Kernel Debug Sub Classes for BSD */
-#define	DBG_BSD_EXCP_SC	0x0C	/* System Calls */
+#define	DBG_BSD_EXCP_SC		0x0C	/* System Calls */
 #define	DBG_BSD_AIO		0x0D	/* aio (POSIX async IO) */
-#define DBG_BSD_SC_EXTENDED_INFO 0x0E    /* System Calls, extended info */
+#define DBG_BSD_SC_EXTENDED_INFO 0x0E	/* System Calls, extended info */
+#define DBG_BSD_SC_EXTENDED_INFO2 0x0F	/* System Calls, extended info */
 
 /* The Kernel Debug Sub Classes for DBG_TRACE */
 #define DBG_TRACE_DATA      0
@@ -232,11 +257,8 @@ __BEGIN_DECLS
 #define DKIO_META	0x08
 #define DKIO_PAGING	0x10
 
-/* The Kernel Debug Modifiers for the DBG_IOCPUPM sub-class */
-#define DCPM_PSTATE		0x0001
-#define DCPM_IDLE_CSTATE	0x0002
-#define DCPM_IDLE_HALT		0x0003
-#define DCPM_IDLE_LOOP		0x0004
+/* Codes for Application Sub Classes */
+#define DBG_APP_SAMBA	128
 
 /**********************************************************************/
 
@@ -259,6 +281,17 @@ __BEGIN_DECLS
 #define APPSDBG_CODE(SubClass,code) KDBG_CODE(DBG_APPS, SubClass, code)
 #define CPUPM_CODE(code) IOKDBG_CODE(DBG_IOCPUPM, code)
 
+#define KMEM_ALLOC_CODE MACHDBG_CODE(DBG_MACH_LEAKS, 0)
+#define KMEM_ALLOC_CODE_2 MACHDBG_CODE(DBG_MACH_LEAKS, 1)
+#define KMEM_FREE_CODE MACHDBG_CODE(DBG_MACH_LEAKS, 2)
+#define KMEM_FREE_CODE_2 MACHDBG_CODE(DBG_MACH_LEAKS, 3)
+#define ZALLOC_CODE MACHDBG_CODE(DBG_MACH_LEAKS, 4)
+#define ZALLOC_CODE_2 MACHDBG_CODE(DBG_MACH_LEAKS, 5)
+#define ZFREE_CODE MACHDBG_CODE(DBG_MACH_LEAKS, 6)
+#define ZFREE_CODE_2 MACHDBG_CODE(DBG_MACH_LEAKS, 7)
+
+#define PMAP_CODE(code) MACHDBG_CODE(DBG_MACH_PMAP, code)
+
 /*   Usage:
 * kernel_debug((KDBG_CODE(DBG_NETWORK, DNET_PROTOCOL, 51) | DBG_FUNC_START), 
 *	offset, 0, 0, 0,0) 
@@ -291,6 +324,8 @@ extern unsigned int kdebug_enable;
 #define KDEBUG_ENABLE_ENTROPY 0x2
 #define KDEBUG_ENABLE_CHUD    0x4
 
+#if	(!defined(NO_KDEBUG))
+
 #define KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e)    \
 do {					\
     if (kdebug_enable)			\
@@ -303,6 +338,14 @@ do {					\
         kernel_debug1(x,a,b,c,d,e);	\
 } while(0)
 
+#else
+
+#define KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e)
+#define KERNEL_DEBUG_CONSTANT1(x,a,b,c,d,e)
+
+#define __kdebug_constant_only __unused
+#endif
+
 extern void kernel_debug(unsigned int debugid, unsigned int arg1, unsigned int arg2, unsigned int arg3,  unsigned int arg4, unsigned int arg5);
 
 extern void kernel_debug1(unsigned int debugid, unsigned int arg1, unsigned int arg2, unsigned int arg3,  unsigned int arg4, unsigned int arg5);
@@ -315,30 +358,34 @@ extern void kdbg_trace_data(struct proc *proc, long *arg_pid);
 
 extern void kdbg_trace_string(struct proc *proc, long *arg1, long *arg2, long *arg3, long *arg4);
 
-#if	KDEBUG
+#if	(KDEBUG && (!defined(NO_KDEBUG)))
 
 #define KERNEL_DEBUG(x,a,b,c,d,e)	\
 do {					\
     if (kdebug_enable)			\
-        kernel_debug(x,a,b,c,d,e);	\
+        kernel_debug((unsigned int)x, (unsigned int)a, (unsigned int)b, \
+		     (unsigned int)c, (unsigned int)d, (unsigned int)e); \
 } while(0)
 
 #define KERNEL_DEBUG1(x,a,b,c,d,e)	\
 do {					\
     if (kdebug_enable)			\
-        kernel_debug1(x,a,b,c,d,e);	\
+        kernel_debug1((unsigned int)x, (unsigned int)a, (unsigned int)b, \
+		      (unsigned int)c, (unsigned int)d, (unsigned int)e); \
 } while(0)
 
 #define __kdebug_only
 
 #else
 
-#define KERNEL_DEBUG(x,a,b,c,d,e)
-#define KERNEL_DEBUG1(x,a,b,c,d,e)
+#define KERNEL_DEBUG(x,a,b,c,d,e) do {} while (0)
+#define KERNEL_DEBUG1(x,a,b,c,d,e) do {} while (0)
 
 #define __kdebug_only __unused
 #endif
 
+void start_kern_tracing(unsigned int);
+
 #endif /* __APPLE_API_UNSTABLE */
 __END_DECLS
 
@@ -350,13 +397,13 @@ __END_DECLS
  */
 
 typedef struct {
-uint64_t	timestamp;
-unsigned int	arg1;
-unsigned int	arg2;
-unsigned int	arg3;
-unsigned int	arg4;
-unsigned int	arg5;       /* will hold current thread */
-unsigned int	debugid;
+	uint64_t	timestamp;
+	unsigned int	arg1;
+	unsigned int	arg2;
+	unsigned int	arg3;
+	unsigned int	arg4;
+	unsigned int	arg5;       /* will hold current thread */
+	unsigned int	debugid;
 } kd_buf;
 
 #define KDBG_TIMESTAMP_MASK 0x00ffffffffffffffULL
@@ -430,31 +477,6 @@ typedef struct
 /* Minimum value allowed when setting decrementer ticks */
 #define KDBG_MINRTCDEC  2500
 
-
-/* PCSAMPLES control operations */
-#define PCSAMPLE_DISABLE   1
-#define PCSAMPLE_SETNUMBUF 2
-#define PCSAMPLE_GETNUMBUF 3
-#define PCSAMPLE_SETUP	   4
-#define PCSAMPLE_REMOVE	   5
-#define	PCSAMPLE_READBUF   6
-#define	PCSAMPLE_SETREG    7
-#define PCSAMPLE_COMM      8
-
-#define MAX_PCSAMPLES    1000000     /* Maximum number of pc's in a single buffer */
-
-
-extern unsigned int pcsample_enable;
-
-typedef struct
-{
-    int npcbufs;
-    int bufsize;
-    int enable;
-	unsigned int pcsample_beg;
-	unsigned int pcsample_end;
-} pcinfo_t;
-
 #endif /* __APPLE_API_PRIVATE */
 #endif	/* PRIVATE */
 
diff --git a/bsd/sys/kern_control.h b/bsd/sys/kern_control.h
index f032e79a2..0e83895b4 100644
--- a/bsd/sys/kern_control.h
+++ b/bsd/sys/kern_control.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kern_control.h
@@ -188,11 +194,13 @@ typedef void * kern_ctl_ref;
 #define CTL_DATA_NOWAKEUP	0x1
 /*!
 	@defined CTL_DATA_EOR
-    @discussion The CTL_DATA_NOWAKEUP flag can be used for the enqueue
+    @discussion The CTL_DATA_EOR flag can be used for the enqueue
     	data and enqueue mbuf functions to mark the end of a record.
 */
 #define CTL_DATA_EOR		0x2
 
+__BEGIN_DECLS
+
 /*!
 	@typedef ctl_connect_func
 	@discussion The ctl_connect_func is used to receive
@@ -206,8 +214,13 @@ typedef void * kern_ctl_ref;
 		If CTL_FLAG_REG_ID_UNIT was not set when the kernel control was 
 		registered, sc_unit is the dynamically allocated unit number of 
 		the new kernel control instance that is used for this connection.
-	@param unitinfo A place for the kernel control to store a pointer to
-		per-connection data.
+	@param unitinfo A placeholder for a pointer to the optional user-defined
+		private data associated with this kernel control instance.  This
+		opaque info will be provided to the user when the rest of the
+		callback routines are executed.  For example, it can be used
+		to pass a pointer to an instance-specific data structure in
+		order for the user to keep track of the states related to this
+		kernel control instance.
  */
 typedef errno_t (*ctl_connect_func)(kern_ctl_ref kctlref, 
 									struct sockaddr_ctl *sac,
@@ -224,8 +237,8 @@ typedef errno_t (*ctl_connect_func)(kern_ctl_ref kctlref,
 		disconnected from.
 	@param unit The unit number of the kernel control instance the client has
 		disconnected from.  
-	@param unitinfo The unitinfo value specified by the connect function
-		when the client connected.
+	@param unitinfo The user-defined private data initialized by the
+		ctl_connect_func callback.
  */
 typedef errno_t (*ctl_disconnect_func)(kern_ctl_ref kctlref, u_int32_t unit, void *unitinfo);
 
@@ -236,8 +249,8 @@ typedef errno_t (*ctl_disconnect_func)(kern_ctl_ref kctlref, u_int32_t unit, voi
 	@param kctlref The control ref of the kernel control.
 	@param unit The unit number of the kernel control instance the client has
 		connected to.
-	@param unitinfo The unitinfo value specified by the connect function
-		when the client connected.
+	@param unitinfo The user-defined private data initialized by the
+		ctl_connect_func callback.
 	@param m The data sent by the client to the kernel control in an
 		mbuf chain.
 	@param flags The flags specified by the client when calling
@@ -252,8 +265,8 @@ typedef errno_t (*ctl_send_func)(kern_ctl_ref kctlref, u_int32_t unit, void *uni
 		calls for the SYSPROTO_CONTROL option level.
 	@param kctlref The control ref of the kernel control.
 	@param unit The unit number of the kernel control instance.
-	@param unitinfo The unitinfo value specified by the connect function
-		when the client connected.
+	@param unitinfo The user-defined private data initialized by the
+		ctl_connect_func callback.
 	@param opt The socket option.
 	@param data A pointer to the socket option data. The data has
 		already been copied in to the kernel for you.
@@ -274,8 +287,8 @@ typedef errno_t (*ctl_setopt_func)(kern_ctl_ref kctlref, u_int32_t unit, void *u
 		return an error.
 	@param kctlref The control ref of the kernel control.
 	@param unit The unit number of the kernel control instance.
-	@param unitinfo The unitinfo value specified by the connect function
-		when the client connected.
+	@param unitinfo The user-defined private data initialized by the
+		ctl_connect_func callback.
 	@param opt The socket option.
 	@param data A buffer to copy the results in to. May be NULL, see
 		discussion.
@@ -414,13 +427,13 @@ ctl_enqueuembuf(kern_ctl_ref kctlref, u_int32_t unit, mbuf_t m, u_int32_t flags)
 	@param kctlref The control reference of the kernel control.
 	@param unit The unit number of the kernel control instance.
 	@param space The address where to return the current space available 
-	@result 0 - Data was enqueued to be read by the client.
+	@result 0 - Success; the amount of space is returned to caller.
 		EINVAL - Invalid parameters.
  */
 errno_t 
 ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space);
 
-
+__END_DECLS
 #endif /* KERNEL */
 
 #endif /* KPI_KERN_CONTROL_H */
diff --git a/bsd/sys/kern_event.h b/bsd/sys/kern_event.h
index ca40cc713..7a658ac9a 100644
--- a/bsd/sys/kern_event.h
+++ b/bsd/sys/kern_event.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /*!
@@ -35,7 +41,7 @@
 
 #define KEVENTS_ON  1
 #define KEV_SNDSPACE (4 * 1024)
-#define KEV_RECVSPACE (8 * 1024)
+#define KEV_RECVSPACE (32 * 1024)
 
 #define KEV_ANY_VENDOR    0
 #define KEV_ANY_CLASS     0
@@ -82,6 +88,12 @@
 */
 #define KEV_APPLESHARE_CLASS	4
 
+/*!
+	@defined KEV_FIREWALL_CLASS
+	@discussion Firewall kernel event class.
+*/
+#define KEV_FIREWALL_CLASS	5
+
 /*!
 	@struct kern_event_msg
 	@discussion This structure is prepended to all kernel events. This structure
@@ -104,22 +116,21 @@
 		KEV_MSG_HEADER_SIZE.
 */
 struct kern_event_msg {
-     u_long	total_size;		/* Size of entire event msg */
-     u_long	vendor_code;	/* For non-Apple extensibility */
-     u_long	kev_class;		/* Layer of event source */
-     u_long	kev_subclass;	/* Component within layer */
-     u_long	id;				/* Monotonically increasing value */
-     u_long	event_code;		/* unique code */
-     u_long	event_data[1];	/* One or more data longwords */
-
+	u_int32_t	total_size;	/* Size of entire event msg */
+	u_int32_t	vendor_code;	/* For non-Apple extensibility */
+	u_int32_t	kev_class;	/* Layer of event source */
+	u_int32_t	kev_subclass;	/* Component within layer */
+	u_int32_t	id;		/* Monotonically increasing value */
+	u_int32_t	event_code;	/* unique code */
+	u_int32_t	event_data[1];	/* One or more data words */
 };
 
 /*!
 	@defined KEV_MSG_HEADER_SIZE
-    @discussion Size of the header portion of the kern_event_msg structure. This
-    	accounts for everything right up to event_data. The size of the data can
-    	be found by subtracting KEV_MSG_HEADER_SIZE from the total size from the
-    	kern_event_msg.
+	@discussion Size of the header portion of the kern_event_msg structure. This
+		accounts for everything right up to event_data. The size of the data can
+		be found by subtracting KEV_MSG_HEADER_SIZE from the total size from the
+		kern_event_msg.
 */
 #define KEV_MSG_HEADER_SIZE (offsetof(struct kern_event_msg, event_data[0]))
 
@@ -140,9 +151,9 @@ struct kern_event_msg {
 		subclass.
 */
 struct kev_request {
-     u_long	vendor_code;
-     u_long	kev_class;
-     u_long	kev_subclass;
+	u_int32_t	vendor_code;
+	u_int32_t	kev_class;
+	u_int32_t	kev_subclass;
 };
 
 /*!
@@ -152,6 +163,8 @@ struct kev_request {
 */
 #define KEV_VENDOR_CODE_MAX_STR_LEN	200
 
+#pragma pack(4)
+
 /*!
 	@struct kev_vendor_code
 	@discussion This structure is used with the SIOCGKEVVENDOR ioctl to convert
@@ -162,17 +175,18 @@ struct kev_request {
 	@field vendor_string A bundle style identifier.
 */
 struct kev_vendor_code {
-     u_long	vendor_code;
-     char	vendor_string[KEV_VENDOR_CODE_MAX_STR_LEN];
+	u_int32_t	vendor_code;
+	char		vendor_string[KEV_VENDOR_CODE_MAX_STR_LEN];
 };
 
+#pragma pack()
 
 /*!
 	@defined SIOCGKEVID
 	@discussion Retrieve the current event id. Each event generated will have
 		a new idea. The next event to be generated will have an id of id+1.
 */
-#define SIOCGKEVID	_IOR('e', 1, u_long)
+#define SIOCGKEVID	_IOR('e', 1, u_int32_t)
 
 /*!
 	@defined SIOCSKEVFILT
@@ -209,12 +223,12 @@ struct kev_vendor_code {
 	@field data_ptr A pointer to data.
 */
 struct kev_d_vectors {
-     u_long	data_length;	/* Length of the event data */
-     void	*data_ptr;    /* Pointer to event data */
+	u_int32_t	data_length;	/* Length of the event data */
+	void		*data_ptr;	/* Pointer to event data */
 };
 
 /*!
-	@struct kev_d_vectors
+	@struct kev_msg
 	@discussion This structure is used when posting a kernel event.
 	@field vendor_code The vendor code assigned by kev_vendor_code_find.
 	@field kev_class The event's class.
@@ -224,11 +238,11 @@ struct kev_d_vectors {
 		the kernel event.
 */
 struct kev_msg {
-     u_long	       vendor_code;     /* For non-Apple extensibility */
-     u_long	       kev_class;	/* Layer of event source */
-     u_long	       kev_subclass;    /* Component within layer    */
-     u_long	       event_code;      /* The event code        */
-     struct kev_d_vectors  dv[N_KEV_VECTORS];      /* Up to n data vectors  */
+	u_int32_t		vendor_code;		/* For non-Apple extensibility	*/
+	u_int32_t		kev_class;		/* Layer of event source	*/
+	u_int32_t		kev_subclass;		/* Component within layer	*/
+	u_int32_t		event_code;		/* The event code		*/
+	struct kev_d_vectors	dv[N_KEV_VECTORS];	/* Up to n data vectors		*/
 };
 
 /*!
@@ -238,12 +252,12 @@ struct kev_msg {
 		that string. Vendor codes will remain the same until the machine is
 		rebooted.
 	@param vendor_string A bundle style vendor identifier (i.e. com.apple).
-	@param vender_code Upon return, a unique vendor code for use when posting
+	@param vendor_code Upon return, a unique vendor code for use when posting
 		kernel events.
 	@result May return ENOMEM if memory constraints prevent allocation of a new
 		vendor code.
  */
-errno_t	kev_vendor_code_find(const char *vendor_string, u_long *vender_code);
+errno_t	kev_vendor_code_find(const char *vendor_string, u_int32_t *vendor_code);
 
 /*!
 	@function kev_msg_post
@@ -262,16 +276,16 @@ errno_t kev_msg_post(struct kev_msg *event_msg);
  * Internal version of kev_post_msg. Allows posting Apple vendor code kernel
  * events.
  */
-int		kev_post_msg(struct kev_msg *event);
+int	kev_post_msg(struct kev_msg *event);
 
 LIST_HEAD(kern_event_head, kern_event_pcb);
 
 struct  kern_event_pcb {
-     LIST_ENTRY(kern_event_pcb) ev_link;     /* glue on list of all PCBs */
-     struct  socket *ev_socket;     /* pointer back to socket */
-     u_long	    vendor_code_filter;
-     u_long	    class_filter;
-     u_long	    subclass_filter;
+	LIST_ENTRY(kern_event_pcb) ev_link;	/* glue on list of all PCBs */
+	struct socket	*ev_socket;		/* pointer back to socket */
+	u_int32_t	vendor_code_filter;
+	u_int32_t	class_filter;
+	u_int32_t	subclass_filter;
 };
 
 #define sotoevpcb(so)   ((struct kern_event_pcb *)((so)->so_pcb))
diff --git a/bsd/sys/kern_memorystatus.h b/bsd/sys/kern_memorystatus.h
new file mode 100644
index 000000000..f94c4a7a7
--- /dev/null
+++ b/bsd/sys/kern_memorystatus.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*!
+	@header kern_memorystatus.h
+	This header defines a kernel event subclass for the OSMemoryNotification API
+ */
+
+#ifndef SYS_KERN_MEMORYSTATUS_H
+#define SYS_KERN_MEMORYSTATUS_H
+
+/*
+ * Define Memory Status event subclass.
+ * Subclass of KEV_SYSTEM_CLASS
+ */
+
+/*!
+	@defined KEV_MEMORYSTATUS_SUBCLASS
+	@discussion The kernel event subclass for memory status events.
+*/
+#define KEV_MEMORYSTATUS_SUBCLASS        3
+
+enum {
+	kMemoryStatusLevelAny = -1,
+	kMemoryStatusLevelNormal = 0,
+	kMemoryStatusLevelWarning = 1,
+	kMemoryStatusLevelUrgent = 2,
+	kMemoryStatusLevelCritical = 3
+};
+
+#ifdef KERNEL
+extern void kern_memorystatus_init(void) __attribute__((section("__TEXT, initcode")));
+
+extern int kern_memorystatus_wakeup;
+extern int kern_memorystatus_level;
+
+#endif /* KERNEL */
+#endif /* SYS_KERN_MEMORYSTATUS_H */
diff --git a/bsd/sys/kernel.h b/bsd/sys/kernel.h
index c7a2c0b77..6b9c7baef 100644
--- a/bsd/sys/kernel.h
+++ b/bsd/sys/kernel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/kernel_types.h b/bsd/sys/kernel_types.h
index c4d9a60be..a285994de 100644
--- a/bsd/sys/kernel_types.h
+++ b/bsd/sys/kernel_types.h
@@ -16,8 +16,6 @@
 typedef int errno_t;
 typedef int64_t daddr64_t;
 
-typedef	int64_t	ino64_t;
-
 #ifndef BSD_BUILD
 struct buf;
 typedef struct buf * buf_t;
diff --git a/bsd/sys/kpi_mbuf.h b/bsd/sys/kpi_mbuf.h
index 5a1d26d8e..4e2b57329 100644
--- a/bsd/sys/kpi_mbuf.h
+++ b/bsd/sys/kpi_mbuf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_mbuf.h
@@ -45,7 +51,7 @@
 /*!
 	@enum mbuf_flags_t
 	@abstract Constants defining mbuf flags. Only the flags listed below
-		can be set or retreieved.
+		can be set or retrieved.
 	@constant MBUF_EXT Indicates this mbuf has external data.
 	@constant MBUF_PKTHDR Indicates this mbuf has a packet header.
 	@constant MBUF_EOR Indicates this mbuf is the end of a record.
@@ -251,20 +257,23 @@ struct mbuf_stat {
 /* Parameter for m_copym to copy all bytes */
 #define	MBUF_COPYALL	1000000000
 
+__BEGIN_DECLS
 /* Data access */
 /*!
 	@function mbuf_data
 	@discussion Returns a pointer to the start of data in this mbuf.
 		There may be additional data on chained mbufs. The data you're
-		looking for may not be contiguous if it spans more than one
-		mbuf. Use mbuf_len to determine the lenght of data available in
-		this mbuf. If a data structure you want to access stradles two
-		mbufs in a chain, either use mbuf_pullup to get the data
-		contiguous in one mbuf or copy the pieces of data from each mbuf
-		in to a contiguous buffer. Using mbuf_pullup has the advantage
-		of not having to copy the data. On the other hand, if you don't
-		make sure there is space in the mbuf, mbuf_pullup may fail and
-		free the mbuf.
+		looking for may not be virtually contiguous if it spans more
+		than one mbuf.  In addition, data that is virtually contiguous
+		might not be represented by physically contiguous pages; see
+		further comments in mbuf_data_to_physical.  Use mbuf_len to
+		determine the lenght of data available in this mbuf. If a data
+		structure you want to access stradles two mbufs in a chain,
+		either use mbuf_pullup to get the data contiguous in one mbuf
+		or copy the pieces of data from each mbuf in to a contiguous
+		buffer. Using mbuf_pullup has the advantage of not having to
+		copy the data. On the other hand, if you don't make sure there
+		is space in the mbuf, mbuf_pullup may fail and free the mbuf.
 	@param mbuf The mbuf.
 	@result A pointer to the data in the mbuf.
  */
@@ -317,7 +326,14 @@ errno_t		mbuf_align_32(mbuf_t mbuf, size_t len);
 	@discussion mbuf_data_to_physical is a replacement for mcl_to_paddr.
 		Given a pointer returned from mbuf_data of mbuf_datastart,
 		mbuf_data_to_physical will return the phyical address for that
-		block of data.
+		block of data.  Note that even though the data is in virtually
+		contiguous span, the underlying physical pages might not be
+		physically contiguous.  Because of this, callers must ensure
+		to call this routine for each page boundary.  Device drivers
+		that deal with DMA are strongly encouraged to utilize the
+		IOMbufNaturalMemoryCursor and walk down the list of vectors
+		instead of using this interface to obtain the physical address.
+		Use of this routine is therefore discouraged.
 	@param ptr A pointer to data stored in an mbuf.
 	@result The 64 bit physical address of the mbuf data or NULL if ptr
 		does not point to data stored in an mbuf.
@@ -349,18 +365,87 @@ errno_t		mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t* mbuf);
  */
 errno_t		mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t* mbuf);
 
+/*!
+	@function mbuf_attachcluster
+	@discussion Attach an external buffer as a cluster for an mbuf.  If mbuf
+		points to a NULL mbuf_t, an mbuf will be allocated for you.  If
+		mbuf points to a non-NULL mbuf_t, the user-supplied mbuf will
+		be used instead.  The caller is responsible for allocating the
+		external buffer by calling mbuf_alloccluster().
+	@param how Blocking or non-blocking.
+	@param type The type of the mbuf if mbuf is non-NULL; otherwise ignored.
+	@param mbuf Pointer to the address of the mbuf; if NULL, an mbuf will
+		be allocated, otherwise, it must point to a valid mbuf address.
+		If the user-supplied mbuf is already attached to a cluster, the
+		current cluster will be freed before the mbuf gets attached to
+		the supplied external buffer.  Note that this routine may return
+		a different mbuf_t than the one you passed in.
+	@param extbuf Address of the external buffer.
+	@param extfree Free routine for the external buffer; the caller is
+		required to defined a routine that will be invoked when the
+		mbuf is freed.
+	@param extsize Size of the external buffer.
+	@param extarg Private value that will be passed to the free routine
+		when it is called at the time the mbuf is freed.
+	@result 0 on success
+		EINVAL - Invalid parameter
+		ENOMEM - Not enough memory available
+ */
+errno_t
+mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf,
+    caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t),
+    size_t extsize, caddr_t extarg);
+
+/*!
+	@function mbuf_alloccluster
+	@discussion Allocate a cluster that can be later attached to an
+		mbuf by calling mbuf_attachcluster().  The allocated cluster
+		can also be freed (without being attached to an mbuf) by
+		calling mbuf_freecluster().  At the moment this routine
+		will either return a cluster of 2048, 4096 or 16384 bytes
+		depending on the requested size.  Note that clusters greater
+		than 4096 bytes might not be available in all configurations;
+		the caller must additionally check for ENOTSUP (see below).
+	@param how Blocking or non-blocking.
+	@param size Pointer to size of requested cluster.  Sizes up to 2048
+		will be rounded up to 2048; sizes greater than 2048 and up
+		to 4096 will be rounded up to 4096.  Sizes greater than 4096
+		will be rounded up to 16384.
+	@param addr Pointer to the address of the requested cluster.
+	@result 0 on success or ENOMEM if failure.  If the caller requests
+		greater than 4096 bytes and the system is unable to fulfill
+		the request due to the lack of jumbo clusters support based
+		on the configuration, this routine will return ENOTSUP.
+		In this case, the caller is advised to use 4096 bytes or
+		smaller during subseqent requests.
+ */
+errno_t mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr);
+
+/*!
+	@function mbuf_freecluster
+	@discussion Free a cluster that was previously allocated by a call
+		to mbuf_alloccluster().  The caller must pass the actual
+		size of the cluster as returned by mbuf_alloccluster(),
+		which at this point must be either 2048, 4096 or 16384 bytes.
+	@param addr The address of the cluster.
+	@param size The actual size of the cluster.
+ */
+void mbuf_freecluster(caddr_t addr, size_t size);
 
 /*!
 	@function mbuf_getcluster
 	@discussion Allocate a cluster of the requested size and attach it to
-		an mbuf for use as external data. If mbuf points to a NULL mbuf_t, 
-		an mbuf will be allocated for you. If mbuf points to a non-NULL mbuf_t,
-		mbuf_getcluster may return a different mbuf_t than the one you
-		passed in. 
+		an mbuf for use as external data. If mbuf points to a NULL
+		mbuf_t, an mbuf will be allocated for you. If mbuf points to
+		a non-NULL mbuf_t, mbuf_getcluster may return a different
+		mbuf_t than the one you passed in.
 	@param how Blocking or non-blocking.
 	@param type The type of the mbuf.
-	@param size The size of the cluster to be allocated. Supported sizes for a 
-		cluster are be 2048 or 4096. Any other value with return EINVAL.
+	@param size The size of the cluster to be allocated. Supported sizes
+		for a cluster are be 2048, 4096, or 16384. Any other value
+		with return EINVAL.  Note that clusters greater than 4096
+		bytes might not be available in all configurations; the
+		caller must additionally check for ENOTSUP (see below).
 	@param mbuf The mbuf the cluster will be attached to.
 	@result 0 on success, errno error on failure. If you specified NULL
 		for the mbuf, any intermediate mbuf that may have been allocated
@@ -368,6 +453,11 @@ errno_t		mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t* mbuf);
 		mbuf_mclget will not free it.
 		EINVAL - Invalid parameter
 		ENOMEM - Not enough memory available
+		ENOTSUP - The caller had requested greater than 4096 bytes
+		    cluster and the system is unable to fulfill it due to the
+		    lack of jumbo clusters support based on the configuration.
+		    In this case, the caller is advised to use 4096 bytes or
+		    smaller during subsequent requests.
  */
 errno_t		mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf);
 
@@ -390,28 +480,80 @@ errno_t		mbuf_mclget(mbuf_how_t how, mbuf_type_t type, mbuf_t* mbuf);
 
 /*!
 	@function mbuf_allocpacket
-	@discussion Allocate an mbuf chain to store a single packet of the requested length. 
-		According to the requested length, a chain of mbufs will be created. The mbuf type 
-		will be set to MBUF_TYPE_DATA. The caller may specify the maximum number of 
-		buffer 
+	@discussion Allocate an mbuf chain to store a single packet of the
+		requested length.  According to the requested length, a chain
+		of mbufs will be created. The mbuf type will be set to
+		MBUF_TYPE_DATA. The caller may specify the maximum number of
+		buffer.
 	@param how Blocking or non-blocking
 	@param packetlen The total length of the packet mbuf to be allocated.
 		The length must be greater than zero.
-	@param maxchunks An input/output pointer to the maximum number of mbufs segments making up the chain. 
-		On input if maxchunks is zero, or the value pointed to by maxchunks is zero, 
-		the packet will be made of as many buffer segments as necessary to fit the length.
-		The allocation will fail with ENOBUFS if the number of segments requested is too small and 
-		the sum of the maximum size of each individual segment is less than the packet length.
-		On output, if the allocation succeed and maxchunks is non zero, it will point to 
-		the actual number of segments allocated.
+	@param maxchunks An input/output pointer to the maximum number of mbufs
+		segments making up the chain.  On input, if maxchunks is NULL,
+		or the value pointed to by maxchunks is zero, the packet will
+		be made up of as few or as many buffer segments as necessary
+		to fit the length.  The allocation will fail with ENOBUFS if
+		the number of segments requested is too small and the sum of
+		the maximum size of each individual segment is less than the
+		packet length.  On output, if the allocation succeed and
+		maxchunks is non-NULL, it will point to the actual number
+		of segments allocated.
+		Additional notes for packetlen greater than 4096 bytes:
+		the caller may pass a non-NULL maxchunks and initialize it
+		with zero such that upon success, it can find out whether
+		or not the system configuration allows for larger than
+		4096 bytes cluster allocations, by checking on the value
+		pointed to by maxchunks.  E.g. a request for 9018 bytes may
+		result in 1 chunk when jumbo clusters are available, or
+		3 chunks otherwise.
 	@param Upon success, *mbuf will be a reference to the new mbuf.
 	@result Returns 0 upon success or the following error code:
 		EINVAL - Invalid parameter
 		ENOMEM - Not enough memory available
-		ENOBUFS - Buffers not big enough for the maximum number of chunks requested
+		ENOBUFS - Buffers not big enough for the maximum number of
+		    chunks requested
 */
 errno_t	mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int * maxchunks, mbuf_t *mbuf);
 
+/*!
+	@function mbuf_allocpacket_list
+	@discussion Allocate a linked list of packets.  According to the
+		requested length, each packet will made of a chain of one
+		or more mbufs.  The mbuf type will be set to MBUF_TYPE_DATA.
+		The caller may specify the maximum number of element for
+		each mbuf chain making up a packet.
+	@param numpkts Number of packets to allocate
+	@param how Blocking or non-blocking
+	@param packetlen The total length of the packet mbuf to be allocated.
+		The length must be greater than zero.
+	@param maxchunks An input/output pointer to the maximum number of
+		mbufs segments making up the chain.  On input, if maxchunks is
+		zero, or the value pointed to by maxchunks is zero, the packet
+		will be made of as few or as many buffer segments as necessary
+		to fit the length.  The allocation will fail with ENOBUFS if
+		the number of segments requested is too small and the sum of
+		the maximum size of each individual segment is less than the
+		packet length.  On output, if the allocation succeed and
+		maxchunks is non zero, it will point to the actual number
+		of segments allocated.
+		Additional notes for packetlen greater than 4096 bytes:
+		the caller may pass a non-NULL maxchunks and initialize it
+		with zero such that upon success, it can find out whether
+		or not the system configuration allows for larger than
+		4096 bytes cluster allocations, by checking on the value
+		pointed to by maxchunks.  E.g. a request for 9018 bytes may
+		result in 1 chunk when jumbo clusters are available, or
+		3 chunks otherwise.
+	@param Upon success, *mbuf will be a reference to the new mbuf.
+	@result Returns 0 upon success or the following error code:
+		EINVAL - Invalid parameter
+		ENOMEM - Not enough memory available
+		ENOBUFS - Buffers not big enough for the maximum number of
+		    chunks requested
+*/
+errno_t	mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int * maxchunks, mbuf_t *mbuf);
+
+
 /*!
 	@function mbuf_getpacket
 	@discussion Allocate an mbuf, allocate and attach a cluster, and set
@@ -441,7 +583,7 @@ void		mbuf_freem(mbuf_t mbuf);
 /*!
 	@function mbuf_freem_list
 	@discussion Frees linked list of mbuf chains. Walks through
-		mnextpackt and does the equivalent of mbuf_mfreem to each.
+		mnextpackt and does the equivalent of mbuf_freem to each.
 	@param mbuf The first mbuf in the linked list to free.
 	@result The number of mbufs freed.
  */
@@ -454,7 +596,7 @@ int			mbuf_freem_list(mbuf_t mbuf);
 	@param mbuf The mbuf.
 	@result The number of unused bytes at the start of the mbuf.
  */
-size_t		mbuf_leadingspace(mbuf_t mbuf);
+size_t		mbuf_leadingspace(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_trailingspace
@@ -463,13 +605,17 @@ size_t		mbuf_leadingspace(mbuf_t mbuf);
 	@param mbuf The mbuf.
 	@result The number of unused bytes following the current data.
  */
-size_t		mbuf_trailingspace(mbuf_t mbuf);
+size_t		mbuf_trailingspace(const mbuf_t mbuf);
 
 /* Manipulation */
 
 /*!
 	@function mbuf_copym
-	@discussion Copies len bytes from offset from src to a new mbuf.
+	@discussion Copies len bytes from offset from src to a new mbuf.  If
+	    the original mbuf contains a packet header, the new mbuf will
+	    contain similar packet header except for any tags which may be
+	    associated with the original mbuf.  mbuf_dup() should be used
+	    instead if tags are to be copied to the new mbuf.
 	@param src The source mbuf.
 	@param offset The offset in the mbuf to start copying from.
 	@param len The the number of bytes to copy.
@@ -477,18 +623,21 @@ size_t		mbuf_trailingspace(mbuf_t mbuf);
 	@param new_mbuf Upon success, the newly allocated mbuf.
 	@result 0 upon success otherwise the errno error.
  */
-errno_t		mbuf_copym(mbuf_t src, size_t offset, size_t len,
+errno_t		mbuf_copym(const mbuf_t src, size_t offset, size_t len,
 						mbuf_how_t how, mbuf_t* new_mbuf);
 
 /*!
 	@function mbuf_dup
-	@discussion Exactly duplicates an mbuf chain.
+	@discussion Exactly duplicates an mbuf chain.  If the original mbuf
+	    contains a packet header (including tags), the new mbuf will have
+	    the same packet header contents and a copy of each tag associated
+	    with the original mbuf.
 	@param src The source mbuf.
 	@param how Blocking or non-blocking.
 	@param new_mbuf Upon success, the newly allocated mbuf.
 	@result 0 upon success otherwise the errno error.
  */
-errno_t		mbuf_dup(mbuf_t src, mbuf_how_t how, mbuf_t* new_mbuf);
+errno_t		mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t* new_mbuf);
 
 /*!
 	@function mbuf_prepend
@@ -562,10 +711,22 @@ errno_t		mbuf_pulldown(mbuf_t src, size_t *offset, size_t length, mbuf_t *locati
 		of the mbuf chain.
 	@param mbuf The mbuf chain to trim.
 	@param len The number of bytes to trim from the mbuf chain.
-	@result 0 upon success otherwise the errno error.
  */
 void		mbuf_adj(mbuf_t mbuf, int len);
 
+/*!
+	@function mbuf_adjustlen
+	@discussion Adds amount to the mbuf len. Verifies that the new
+		length is valid (greater than or equal to zero and less than
+		maximum amount of data that may be stored in the mbuf). This
+		function will not adjust the packet header length field or
+		affect any other mbufs in a chain.
+	@param mbuf The mbuf to adjust.
+	@param amount The number of bytes increment the length by.
+	@result 0 upon success otherwise the errno error.
+ */
+errno_t		mbuf_adjustlen(mbuf_t mbuf, int amount);
+
 /*!
 	@function mbuf_copydata
 	@discussion Copies data out of an mbuf in to a specified buffer. If
@@ -578,7 +739,7 @@ void		mbuf_adj(mbuf_t mbuf, int len);
 		copied.
 	@result 0 upon success otherwise the errno error.
  */
-errno_t		mbuf_copydata(mbuf_t mbuf, size_t offset, size_t length, void* out_data);
+errno_t		mbuf_copydata(const mbuf_t mbuf, size_t offset, size_t length, void* out_data);
 
 /*!
 	@function mbuf_copyback
@@ -605,24 +766,6 @@ errno_t		mbuf_copydata(mbuf_t mbuf, size_t offset, size_t length, void* out_data
 errno_t		mbuf_copyback(mbuf_t mbuf, size_t offset, size_t length,
 						  const void *data, mbuf_how_t how);
 
-#ifdef KERNEL_PRIVATE
-/*!
-	@function mbuf_mclref
-	@discussion Incrememnt the reference count of the cluster.
-	@param mbuf The mbuf with the cluster to increment the refcount of.
-	@result 0 upon success otherwise the errno error.
- */
-int			mbuf_mclref(mbuf_t mbuf);
-
-/*!
-	@function mbuf_mclunref
-	@discussion Decrement the reference count of the cluster.
-	@param mbuf The mbuf with the cluster to decrement the refcount of.
-	@result 0 upon success otherwise the errno error.
- */
-int			mbuf_mclunref(mbuf_t mbuf);
-#endif
-
 /*!
 	@function mbuf_mclhasreference
 	@discussion Check if a cluster of an mbuf is referenced by another mbuf.
@@ -642,7 +785,7 @@ int			mbuf_mclhasreference(mbuf_t mbuf);
 	@param mbuf The mbuf.
 	@result The next mbuf in the chain.
  */
-mbuf_t		mbuf_next(mbuf_t mbuf);
+mbuf_t		mbuf_next(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_setnext
@@ -659,7 +802,7 @@ errno_t		mbuf_setnext(mbuf_t mbuf, mbuf_t next);
 	@param mbuf The mbuf.
 	@result The nextpkt.
  */
-mbuf_t		mbuf_nextpkt(mbuf_t mbuf);
+mbuf_t		mbuf_nextpkt(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_setnextpkt
@@ -675,7 +818,7 @@ void		mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt);
 	@param mbuf The mbuf.
 	@result The length.
  */
-size_t		mbuf_len(mbuf_t mbuf);
+size_t		mbuf_len(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_setlen
@@ -696,7 +839,7 @@ void		mbuf_setlen(mbuf_t mbuf, size_t len);
 	@param mbuf The mbuf.
 	@result The maximum lenght of data for this mbuf.
  */
-size_t		mbuf_maxlen(mbuf_t mbuf);
+size_t		mbuf_maxlen(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_type
@@ -704,7 +847,7 @@ size_t		mbuf_maxlen(mbuf_t mbuf);
 	@param mbuf The mbuf.
 	@result The type.
  */
-mbuf_type_t	mbuf_type(mbuf_t mbuf);
+mbuf_type_t	mbuf_type(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_settype
@@ -721,7 +864,7 @@ errno_t		mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type);
 	@param mbuf The mbuf.
 	@result The flags.
  */
-mbuf_flags_t	mbuf_flags(mbuf_t mbuf);
+mbuf_flags_t	mbuf_flags(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_setflags
@@ -751,7 +894,7 @@ errno_t		mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags,
 	@param mbuf The mbuf to which the packet header will be copied.
 	@result 0 upon success otherwise the errno error.
  */
-errno_t		mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src);
+errno_t		mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src);
 
 /*!
 	@function mbuf_pkthdr_len
@@ -760,27 +903,38 @@ errno_t		mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src);
 		be changed.
 	@result The length, in bytes, of the packet.
  */
-size_t		mbuf_pkthdr_len(mbuf_t mbuf);
+size_t		mbuf_pkthdr_len(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_pkthdr_setlen
 	@discussion Sets the length of the packet in the packet header.
 	@param mbuf The mbuf containing the packet header.
 	@param len The new length of the packet.
-	@result 0 upon success otherwise the errno error.
  */
 void		mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len);
 
+/*!
+	@function mbuf_pkthdr_adjustlen
+	@discussion Adjusts the length of the packet in the packet header.
+	@param mbuf The mbuf containing the packet header.
+	@param amount The number of bytes to adjust the packet header length
+		field by.
+ */
+void		mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount);
+
 /*!
 	@function mbuf_pkthdr_rcvif
-	@discussion Returns a reference to the interface the packet was
-		received on. Increments the reference count of the interface
-		before returning. Caller is responsible for releasing
-		the reference by calling ifnet_release.
+	@discussion Returns the interface the packet was received on. This
+		funciton does not modify the reference count of the interface.
+		The interface is only valid for as long as the mbuf is not freed
+		and the rcvif for the mbuf is not changed. Take a reference on
+		the interface that you will release later before doing any of
+		the following: free the mbuf, change the rcvif, pass the mbuf to
+		any function that may free the mbuf or change the rcvif.
 	@param mbuf The mbuf containing the packet header.
 	@result A reference to the interface.
  */
-ifnet_t		mbuf_pkthdr_rcvif(mbuf_t mbuf);
+ifnet_t		mbuf_pkthdr_rcvif(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_pkthdr_setrcvif
@@ -789,7 +943,7 @@ ifnet_t		mbuf_pkthdr_rcvif(mbuf_t mbuf);
 	@param ifnet A reference to an interface.
 	@result 0 upon success otherwise the errno error.
  */
-errno_t		mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet);
+errno_t		mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifp);
 
 /*!
 	@function mbuf_pkthdr_header
@@ -797,7 +951,7 @@ errno_t		mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet);
 	@param mbuf The mbuf containing the packet header.
 	@result A pointer to the packet header.
  */
-void*		mbuf_pkthdr_header(mbuf_t mbuf);
+void*		mbuf_pkthdr_header(const mbuf_t mbuf);
 
 /*!
 	@function mbuf_pkthdr_setheader
@@ -807,40 +961,6 @@ void*		mbuf_pkthdr_header(mbuf_t mbuf);
 	@result 0 upon success otherwise the errno error.
  */
 void		mbuf_pkthdr_setheader(mbuf_t mbuf, void* header);
-#ifdef KERNEL_PRIVATE
-
-/* mbuf aux data */
-
-/*!
-	@function mbuf_aux_add
-	@discussion Adds auxiliary data in the form of an mbuf.
-	@param mbuf The mbuf to add aux data to.
-	@param family The protocol family of the aux data to add.
-	@param type The mbuf type of the aux data to add.
-	@param aux_mbuf The aux mbuf allocated for you.
-	@result 0 upon success otherwise the errno error.
- */
-errno_t		mbuf_aux_add(mbuf_t mbuf, int family, mbuf_type_t type, mbuf_t *aux_mbuf);
-
-/*!
-	@function mbuf_aux_find
-	@discussion Finds auxiliary data attached to an mbuf.
-	@param mbuf The mbuf to find aux data on.
-	@param family The protocol family of the aux data to add.
-	@param type The mbuf type of the aux data to add.
-	@result The aux data mbuf or NULL if there isn't one.
- */
-mbuf_t		mbuf_aux_find(mbuf_t mbuf, int family, mbuf_type_t type);
-
-/*!
-	@function mbuf_aux_delete
-	@discussion Free an mbuf used as aux data and disassosciate it from
-		the mbuf.
-	@param mbuf The mbuf to find aux data on.
-	@param aux The aux data to free.
- */
-void		mbuf_aux_delete(mbuf_t mbuf, mbuf_t aux);
-#endif /* KERNEL_PRIVATE */
 
 /* Checksums */
 
@@ -932,7 +1052,7 @@ errno_t		mbuf_get_vlan_tag(mbuf_t mbuf, u_int16_t *vlan);
 errno_t		mbuf_clear_vlan_tag(mbuf_t mbuf);
 
 #ifdef KERNEL_PRIVATE
-/*!
+/*
 	@function mbuf_set_csum_requested
 	@discussion This function is used by the stack to indicate which
 		checksums should be calculated in hardware. The stack normally
@@ -987,7 +1107,7 @@ errno_t		mbuf_set_csum_performed(mbuf_t mbuf,
 				mbuf_csum_performed_flags_t flags, u_int32_t value);
 
 #ifdef KERNEL_PRIVATE
-/*!
+/*
 	@function mbuf_get_csum_performed
 	@discussion This is used by the stack to determine which checksums
 		were calculated in hardware on the inbound path.
@@ -1011,6 +1131,64 @@ errno_t		mbuf_get_csum_performed(mbuf_t mbuf,
  */
 errno_t		mbuf_clear_csum_performed(mbuf_t mbuf);
 
+/*!
+	@function mbuf_inet_cksum
+	@discussions Calculates 16-bit 1's complement Internet checksum of the
+		transport segment with or without the pseudo header checksum
+		of a given IPv4 packet.  If the caller specifies a non-zero
+		transport protocol, the checksum returned will also include
+		the pseudo header checksum for the corresponding transport
+		header.  Otherwise, no header parsing will be done and the
+		caller may use this to calculate the Internet checksum of
+		an arbitrary span of data.
+
+		This routine does not modify the contents of the packet.  If
+		the caller specifies a non-zero protocol and/or offset, the
+		routine expects the complete protocol header to be present
+		at the beginning of the first mbuf.
+	@param mbuf The mbuf (or chain of mbufs) containing the packet.
+	@param protocol A zero or non-zero value.  A non-zero value specifies
+		the transport protocol used for pseudo header checksum.
+	@param offset A zero or non-zero value; if the latter, it specifies
+		the offset of the transport header from the beginning of mbuf.
+	@param length The total (non-zero) length of the transport segment.
+	@param csum Pointer to the checksum variable; upon success, this
+		routine will return the calculated Internet checksum through
+		this variable.  The caller must set it to a non-NULL value.
+	@result 0 upon success otherwise the errno error.
+ */
+errno_t		mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset,
+		    u_int32_t length, u_int16_t *csum);
+
+/*!
+	@function mbuf_inet6_cksum
+	@discussions Calculates 16-bit 1's complement Internet checksum of the
+		transport segment with or without the pseudo header checksum
+		of a given IPv6 packet.  If the caller specifies a non-zero
+		transport protocol, the checksum returned will also include
+		the pseudo header checksum for the corresponding transport
+		header.  Otherwise, no header parsing will be done and the
+		caller may use this to calculate the Internet checksum of
+		an arbitrary span of data.
+
+		This routine does not modify the contents of the packet.  If
+		the caller specifies a non-zero protocol and/or offset, the
+		routine expects the complete protocol header(s) to be present
+		at the beginning of the first mbuf.
+	@param mbuf The mbuf (or chain of mbufs) containing the packet.
+	@param protocol A zero or non-zero value.  A non-zero value specifies
+		the transport protocol used for pseudo header checksum.
+	@param offset A zero or non-zero value; if the latter, it specifies
+		the offset of the transport header from the beginning of mbuf.
+	@param length The total (non-zero) length of the transport segment.
+	@param csum Pointer to the checksum variable; upon success, this
+		routine will return the calculated Internet checksum through
+		this variable.  The caller must set it to a non-NULL value.
+	@result 0 upon success otherwise the errno error.
+ */
+errno_t		mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset,
+		    u_int32_t length, u_int16_t *csum);
+
 /* mbuf tags */
 
 /*!
@@ -1123,5 +1301,5 @@ void		mbuf_stats(struct mbuf_stat* stats);
 	} \
 }
 
-
+__END_DECLS
 #endif
diff --git a/bsd/sys/kpi_socket.h b/bsd/sys/kpi_socket.h
index 452264dcd..1e63945cf 100644
--- a/bsd/sys/kpi_socket.h
+++ b/bsd/sys/kpi_socket.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_socket.h
@@ -35,21 +41,23 @@
 #include <sys/kernel_types.h>
 #include <sys/socket.h>
 
+__BEGIN_DECLS
+
 struct timeval;
 
 /*!
 	@typedef sock_upcall
-	
+
 	@discussion sock_upcall is used by a socket to notify an in kernel
 		client that data is waiting. Instead of making blocking calls in
 		the kernel, a client can specify an upcall which will be called
 		when data is available or the socket is ready for sending.
-		
+
 		Calls to your upcall function are not serialized and may be
 		called concurrently from multiple threads in the kernel.
-		
+
 		Your upcall function will be called when:
-		
+
 	@param so A reference to the socket that's ready.
 	@param cookie The cookie passed in when the socket was created.
 	@param waitf Indicates whether or not it's safe to block.
@@ -78,8 +86,7 @@ typedef void (*sock_upcall)(socket_t so, void* cookie, int waitf);
 	@result 0 on success otherwise the errno error.
  */
 errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen,
-					int flags, sock_upcall callback, void* cookie,
-					socket_t *new_so);
+    int flags, sock_upcall callback, void* cookie, socket_t *new_so);
 
 /*!
 	@function sock_bind
@@ -108,15 +115,15 @@ errno_t sock_bind(socket_t so, const struct sockaddr *to);
 errno_t sock_connect(socket_t so, const struct sockaddr *to, int flags);
 
 #ifdef KERNEL_PRIVATE
-/*!
+/*
 	This function was added to support NFS. NFS does something funny,
 	setting a short timeout and checking to see if it should abort the
 	connect every two seconds. Ideally, NFS would use the upcall to be
 	notified when the connect is complete.
-	
+
 	If you feel you need to use this function, please contact us to
 	explain why.
-	
+
 	@function sock_connectwait
 	@discussion Allows a caller to wait on a socket connect.
 	@param so The socket being connected.
@@ -126,7 +133,7 @@ errno_t sock_connect(socket_t so, const struct sockaddr *to, int flags);
 		specified.
  */
 errno_t sock_connectwait(socket_t so, const struct timeval *tv);
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
 /*!
 	@function sock_getpeername
@@ -137,7 +144,8 @@ errno_t sock_connectwait(socket_t so, const struct timeval *tv);
 	@param peernamelen Length of storage for the peer name.
 	@result 0 on success otherwise the errno error.
  */
-errno_t sock_getpeername(socket_t so, struct sockaddr *peername, int peernamelen);
+errno_t sock_getpeername(socket_t so, struct sockaddr *peername,
+    int peernamelen);
 
 /*!
 	@function sock_getsockname
@@ -148,7 +156,8 @@ errno_t sock_getpeername(socket_t so, struct sockaddr *peername, int peernamelen
 	@param socknamelen Length of storage for the socket name.
 	@result 0 on success otherwise the errno error.
  */
-errno_t sock_getsockname(socket_t so, struct sockaddr *sockname, int socknamelen);
+errno_t sock_getsockname(socket_t so, struct sockaddr *sockname,
+    int socknamelen);
 
 /*!
 	@function sock_getsockopt
@@ -160,7 +169,8 @@ errno_t sock_getsockname(socket_t so, struct sockaddr *sockname, int socknamelen
 	@param optlen The length of optval, returns the actual length.
 	@result 0 on success otherwise the errno error.
  */
-errno_t sock_getsockopt(socket_t so, int level, int optname, void *optval, int *optlen);
+errno_t sock_getsockopt(socket_t so, int level, int optname, void *optval,
+    int *optlen);
 
 /*!
 	@function sock_ioctl
@@ -182,7 +192,8 @@ errno_t sock_ioctl(socket_t so, unsigned long request, void *argp);
 	@param optlen The length of optval.
 	@result 0 on success otherwise the errno error.
  */
-errno_t sock_setsockopt(socket_t so, int level, int optname, const void *optval, int optlen);
+errno_t sock_setsockopt(socket_t so, int level, int optname, const void *optval,
+    int optlen);
 
 /*!
 	@function sock_listen
@@ -206,7 +217,8 @@ errno_t sock_listen(socket_t so, int backlog);
 	@result 0 on success, EWOULDBLOCK if non-blocking and operation
 		would cause the thread to block, otherwise the errno error.
  */
-errno_t sock_receive(socket_t so, struct msghdr *msg, int flags, size_t *recvdlen);
+errno_t sock_receive(socket_t so, struct msghdr *msg, int flags,
+    size_t *recvdlen);
 
 /*!
 	@function sock_receivembuf
@@ -226,7 +238,8 @@ errno_t sock_receive(socket_t so, struct msghdr *msg, int flags, size_t *recvdle
 	@result 0 on success, EWOULDBLOCK if non-blocking and operation
 		would cause the thread to block, otherwise the errno error.
  */
-errno_t sock_receivembuf(socket_t so, struct msghdr *msg, mbuf_t *data, int flags, size_t *recvlen);
+errno_t sock_receivembuf(socket_t so, struct msghdr *msg, mbuf_t *data,
+    int flags, size_t *recvlen);
 
 /*!
 	@function sock_send
@@ -240,7 +253,8 @@ errno_t sock_receivembuf(socket_t so, struct msghdr *msg, mbuf_t *data, int flag
 	@result 0 on success, EWOULDBLOCK if non-blocking and operation
 		would cause the thread to block, otherwise the errno error.
  */
-errno_t sock_send(socket_t so, const struct msghdr *msg, int flags, size_t *sentlen);
+errno_t sock_send(socket_t so, const struct msghdr *msg, int flags,
+    size_t *sentlen);
 
 /*!
 	@function sock_sendmbuf
@@ -256,7 +270,8 @@ errno_t sock_send(socket_t so, const struct msghdr *msg, int flags, size_t *sent
 		would cause the thread to block, otherwise the errno error.
 		Regardless of return value, the mbuf chain 'data' will be freed.
  */
-errno_t sock_sendmbuf(socket_t so, const struct msghdr *msg, mbuf_t data, int flags, size_t *sentlen);
+errno_t sock_sendmbuf(socket_t so, const struct msghdr *msg, mbuf_t data,
+    int flags, size_t *sentlen);
 
 /*!
 	@function sock_shutdown
@@ -283,7 +298,7 @@ errno_t sock_shutdown(socket_t so, int how);
 	@result 0 on success otherwise the errno error.
  */
 errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback,
-			   void* cookie, socket_t *new_so);
+    void* cookie, socket_t *new_so);
 
 /*!
 	@function sock_close
@@ -295,7 +310,8 @@ errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback,
  */
 void	sock_close(socket_t so);
 
-/*!
+#ifdef KERNEL_PRIVATE
+/*
 	@function sock_retain
 	@discussion Prevents the socket from closing
 	@param so The socket to close.  Increment a retain count on the
@@ -307,7 +323,7 @@ void	sock_close(socket_t so);
  */
 void	sock_retain(socket_t so);
 
-/*!
+/*
 	@function sock_release
 	@discussion Decrement the retain count and close the socket if the
 		retain count reaches zero.
@@ -316,6 +332,7 @@ void	sock_retain(socket_t so);
 		count is reached, this will call sock_close to close the socket.
  */
 void	sock_release(socket_t so);
+#endif /* KERNEL_PRIVATE */
 
 /*!
 	@function sock_setpriv
@@ -340,7 +357,7 @@ int sock_isconnected(socket_t so);
 	@discussion Returns whether or not the socket is non-blocking. In
 		the context of this KPI, non-blocking means that functions to
 		perform operations on a socket will not wait for completion.
-		
+
 		To enable or disable blocking, use the FIONBIO ioctl. The
 		parameter is an int. If the int is zero, the socket will block.
 		If the parameter is non-zero, the socket will not block.
@@ -363,7 +380,7 @@ int sock_isnonblocking(socket_t so);
 errno_t sock_gettype(socket_t so, int *domain, int *type, int *protocol);
 
 #ifdef KERNEL_PRIVATE
-/*!
+/*
 	@function sock_nointerrupt
 	@discussion Disables interrupt on socket buffers (sets SB_NOINTR on
 		send and receive socket buffers).
@@ -372,5 +389,50 @@ errno_t sock_gettype(socket_t so, int *domain, int *type, int *protocol);
 	@result 0 on success otherwise the errno error.
  */
 errno_t sock_nointerrupt(socket_t so, int on);
-#endif KERNEL_PRIVATE
-#endif __KPI_SOCKET__
+
+/*
+	@function sock_getlistener
+	@discussion Retrieves the listening socket of a pre-accepted socket,
+		i.e. a socket which is still in the incomplete/completed list.
+		Once a socket has been accepted, the information pertaining
+		to its listener is no longer available.  Therefore, modules
+		interested in finding out the listening socket should install
+		the appropriate socket filter callback (sf_attach) which gets
+		invoked prior to the socket being fully accepted, and call
+		this routine at such a time to obtain the listener.  Callers
+		are guaranteed that the listener socket will not go away
+		during the sf_attach callback, and therefore the value is
+		safe to be used only in that callback context.  Callers should
+		therefore take note that the listening socket's lock will be
+		held throughout the duration of the callback.
+	@param so The pre-accepted socket.
+	@result Non-NULL value which indicates the listening socket; otherwise,
+		NULL if the socket is not in the incomplete/completed list
+		of a listener.
+ */
+socket_t sock_getlistener(socket_t so);
+
+/*
+	@function sock_getaddr
+	@discussion Retrieves the local or remote address of a socket.
+		This is a composite of sock_getpeername and sock_getsockname,
+		except that the allocated socket address is returned to the
+		caller, and that the caller is reponsible for calling
+		sock_freeaddr once finished with it.
+	@param so The socket.
+	@param psockname Pointer to the storage for the socket name.
+	@param peername 0 for local address, and non-zero for peer address.
+	@result 0 on success otherwise the errno error.
+ */
+errno_t sock_getaddr(socket_t so, struct sockaddr **psockname, int peername);
+
+/*
+	@function sock_freeaddr
+	@discussion Frees the socket address allocated by sock_getaddr.
+	@param sockname The socket name to be freed.
+ */
+void sock_freeaddr(struct sockaddr *sockname);
+#endif /* KERNEL_PRIVATE */
+
+__END_DECLS
+#endif /* __KPI_SOCKET__ */
diff --git a/bsd/sys/kpi_socketfilter.h b/bsd/sys/kpi_socketfilter.h
index a467d5569..5be9ed377 100644
--- a/bsd/sys/kpi_socketfilter.h
+++ b/bsd/sys/kpi_socketfilter.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*!
 	@header kpi_socketfilter.h
@@ -54,16 +60,19 @@ struct sockaddr;
 /*!
 	@enum sflt_flags
 	@abstract Constants defining mbuf flags. Only the flags listed below
-		can be set or retreieved.
+		can be set or retrieved.
 	@constant SFLT_GLOBAL Indicates this socket filter should be
 		attached to all new sockets when they're created.
 	@constant SFLT_PROG Indicates this socket filter should be attached
 		only when request by the application using the SO_NKE socket
 		option.
+	@constant SFLT_EXTENDED	Indicates that this socket filter utilizes
+		the extended fields within the sflt_filter structure.
 */
 enum {
 	SFLT_GLOBAL		= 0x01,
-	SFLT_PROG		= 0x02
+	SFLT_PROG		= 0x02,
+	SFLT_EXTENDED		= 0x04
 };
 typedef	u_int32_t	sflt_flags;
 
@@ -77,7 +86,7 @@ typedef	u_int32_t	sflt_handle;
 /*!
 	@enum sflt_event_t
 	@abstract Events notify a filter of state changes and other various
-		events related to the socket. These events can not be prevented
+		events related to the socket. These events cannot be prevented
 		or intercepted, only observed.
 	@constant sock_evt_connected Indicates this socket has moved to the
 		connected state.
@@ -89,22 +98,25 @@ typedef	u_int32_t	sflt_handle;
 		connection have been shutdown. The param will point to an
 		integer that indicates the direction that has been shutdown. See
 		'man 2 shutdown' for more information.
-	@constant sock_evt_cantrecvmore Indicates the socket can not receive
+	@constant sock_evt_cantrecvmore Indicates the socket cannot receive
 		more data.
-	@constant sock_evt_cantsendmore Indicates the socket can not send
+	@constant sock_evt_cantsendmore Indicates the socket cannot send
 		more data.
 	@constant sock_evt_closing Indicates the socket is closing.
+	@constant sock_evt_bound Indicates this socket has moved to the
+		bound state (only for PF_INET/PF_INET6 domain).
 */
 enum {
 	sock_evt_connecting		= 1,
 	sock_evt_connected		= 2,
-	sock_evt_disconnecting	= 3,
-	sock_evt_disconnected	= 4,
+	sock_evt_disconnecting		= 3,
+	sock_evt_disconnected		= 4,
 	sock_evt_flush_read		= 5,
 	sock_evt_shutdown		= 6, /* param points to an integer specifying how (read, write, or both) see man 2 shutdown */
-	sock_evt_cantrecvmore	= 7,
-	sock_evt_cantsendmore	= 8,
-	sock_evt_closing		= 9
+	sock_evt_cantrecvmore		= 7,
+	sock_evt_cantsendmore		= 8,
+	sock_evt_closing		= 9,
+	sock_evt_bound			= 10
 };
 typedef	u_int32_t	sflt_event_t;
 
@@ -124,6 +136,8 @@ enum {
 };
 typedef	u_int32_t	sflt_data_flag_t;
 
+__BEGIN_DECLS
+
 /*!
 	@typedef sf_unregistered_func
 	
@@ -385,6 +399,10 @@ typedef	errno_t	(*sf_listen_func)(void *cookie, socket_t so);
 	
 	@discussion sf_ioctl_func is called before performing an ioctl
 		on a socket.
+
+		All undefined ioctls are reserved for future use by Apple. If
+		you need to communicate with your kext using an ioctl, please
+		use SIOCSIFKPI and SIOCGIFKPI.
 	@param cookie Cookie value specified when the filter attach was
 		called.
 	@param so The socket the filter is attached to.
@@ -397,6 +415,39 @@ typedef	errno_t	(*sf_listen_func)(void *cookie, socket_t so);
 typedef	errno_t	(*sf_ioctl_func)(void *cookie, socket_t so,
 					u_int32_t request, const char* argp);
 
+/*!
+	@typedef sf_accept_func
+
+	@discussion sf_accept_func is called after a socket is dequeued
+		off the completed (incoming) connection list and before
+		the file descriptor is associated with it.  A filter can
+		utilize this callback to intercept the accepted socket
+		in order to examine it, prior to returning the socket to
+		the caller of accept.  Such a filter may also choose to
+		discard the accepted socket if it wishes to do so.
+	@param cookie Cookie value specified when the filter attach was called.
+	@param so_listen The listening socket.
+	@param so The socket that is about to be accepted.
+	@param local The local address of the about to be accepted socket.
+	@param remote The remote address of the about to be accepted socket.
+	@result Return:
+		0 - The caller will continue with normal processing of accept.
+		EJUSTRETURN - The to be accepted socket will be disconnected
+		    prior to being returned to the caller of accept.  No further
+		    control or data operations on the socket will be allowed.
+		    This is the recommended return value as it has the least
+		    amount of impact, especially to applications which don't
+		    check the error value returned by accept.
+		Anything Else - The to be accepted socket will be closed and
+		    the error will be returned to the caller of accept.
+		    Note that socket filter developers are advised to exercise
+		    caution when returning non-zero values to the caller,
+		    since some applications don't check the error value
+		    returned by accept and therefore risk breakage.
+ */
+typedef errno_t (*sf_accept_func)(void *cookie, socket_t so_listen, socket_t so,
+    const struct sockaddr *local, const struct sockaddr *remote);
+
 /*!
 	@struct sflt_filter
 	@discussion This structure is used to define a socket filter.
@@ -406,7 +457,9 @@ typedef	errno_t	(*sf_ioctl_func)(void *cookie, socket_t so,
 		option.
 	@field sf_flags Indicate whether this filter should be attached to
 		all new sockets or just those that request the filter be
-		attached using the SO_NKE socket option.
+		attached using the SO_NKE socket option. If this filter
+		utilizes the socket filter extension fields, it must also
+		set SFLT_EXTENDED.
 	@field sf_name A name used for debug purposes.
 	@field sf_unregistered Your function for being notified when your
 		filter has been unregistered.
@@ -419,23 +472,31 @@ typedef	errno_t	(*sf_ioctl_func)(void *cookie, socket_t so,
 		null.
 	@field sf_connect_in Your function for handling inbound
 		connections. May be null.
-	@field sf_connect_in Your function for handling outbound
+	@field sf_connect_out Your function for handling outbound
 		connections. May be null.
 	@field sf_bind Your function for handling binds. May be null.
 	@field sf_setoption Your function for handling setsockopt. May be null.
 	@field sf_getoption Your function for handling getsockopt. May be null.
 	@field sf_listen Your function for handling listen. May be null.
 	@field sf_ioctl Your function for handling ioctls. May be null.
+	@field sf_len Length of socket filter extension structure; developers
+		must initialize this to sizeof sflt_filter_ext structure.
+		This field and all fields following it will only be valid
+		if SFLT_EXTENDED flag is set in sf_flags field.
+	@field sf_ext_accept Your function for handling inbound connections
+		at accept time.  May be null.
+	@field sf_ext_rsvd Reserved for future use; you must initialize
+		the reserved fields with zeroes.
 */
 struct sflt_filter {
-	sflt_handle				sf_handle;
-	int						sf_flags;
-	char*					sf_name;
-	
-	sf_unregistered_func	sf_unregistered;
+	sflt_handle			sf_handle;
+	int				sf_flags;
+	char				*sf_name;
+
+	sf_unregistered_func		sf_unregistered;
 	sf_attach_func			sf_attach;
 	sf_detach_func			sf_detach;
-	
+
 	sf_notify_func			sf_notify;
 	sf_getpeername_func		sf_getpeername;
 	sf_getsockname_func		sf_getsockname;
@@ -448,6 +509,18 @@ struct sflt_filter {
 	sf_getoption_func		sf_getoption;
 	sf_listen_func			sf_listen;
 	sf_ioctl_func			sf_ioctl;
+	/*
+	 * The following are valid only if SFLT_EXTENDED flag is set.
+	 * Initialize sf_ext_len to sizeof sflt_filter_ext structure.
+	 * Filters must also initialize reserved fields with zeroes.
+	 */
+	struct sflt_filter_ext {
+		unsigned int		sf_ext_len;
+		sf_accept_func		sf_ext_accept;
+		void			*sf_ext_rsvd[5];	/* Reserved */
+	} sf_ext;
+#define	sf_len		sf_ext.sf_ext_len
+#define	sf_accept	sf_ext.sf_ext_accept
 };
 
 /*!
@@ -482,7 +555,7 @@ errno_t sflt_unregister(sflt_handle handle);
 	@param handle The handle of the registered filter to be attached.
 	@result 0 on success otherwise the errno error.
  */
-errno_t	sflt_attach(socket_t socket, sflt_handle);
+errno_t	sflt_attach(socket_t so, sflt_handle);
 
 /*!
 	@function sflt_detach
@@ -491,7 +564,7 @@ errno_t	sflt_attach(socket_t socket, sflt_handle);
 	@param handle The handle of the registered filter to be detached.
 	@result 0 on success otherwise the errno error.
  */
-errno_t	sflt_detach(socket_t socket, sflt_handle);
+errno_t	sflt_detach(socket_t so, sflt_handle);
 
 /* Functions for manipulating sockets */
 /*
@@ -550,7 +623,7 @@ typedef u_int8_t sockopt_dir;
 
 /*!
 	@function sockopt_direction
-	@discussion Retreives the direction of the socket option (Get or
+	@discussion Retrieves the direction of the socket option (Get or
 		Set).
 	@param sopt The socket option.
 	@result sock_opt_get or sock_opt_set.
@@ -559,7 +632,7 @@ sockopt_dir	sockopt_direction(sockopt_t sopt);
 
 /*!
 	@function sockopt_level
-	@discussion Retreives the socket option level. (SOL_SOCKET, etc).
+	@discussion Retrieves the socket option level. (SOL_SOCKET, etc).
 	@param sopt The socket option.
 	@result The socket option level. See man 2 setsockopt
  */
@@ -567,7 +640,7 @@ int	sockopt_level(sockopt_t sopt);
 
 /*!
 	@function sockopt_name
-	@discussion Retreives the socket option name. (SO_SNDBUF, etc).
+	@discussion Retrieves the socket option name. (SO_SNDBUF, etc).
 	@param sopt The socket option.
 	@result The socket option name. See man 2 setsockopt
  */
@@ -575,7 +648,7 @@ int	sockopt_name(sockopt_t sopt);
 
 /*!
 	@function sockopt_valsize
-	@discussion Retreives the size of the socket option data.
+	@discussion Retrieves the size of the socket option data.
 	@param sopt The socket option.
 	@result The length, in bytes, of the data.
  */
@@ -601,3 +674,5 @@ errno_t	sockopt_copyin(sockopt_t sopt, void *data, size_t length);
  */
 errno_t	sockopt_copyout(sockopt_t sopt, void *data, size_t length);
 
+__END_DECLS
+#endif
diff --git a/bsd/sys/ktrace.h b/bsd/sys/ktrace.h
deleted file mode 100644
index 61787233d..000000000
--- a/bsd/sys/ktrace.h
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-/*
- * Copyright (c) 1988, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)ktrace.h	8.1 (Berkeley) 6/2/93
- * $FreeBSD: src/sys/sys/ktrace.h,v 1.19.2.3 2001/01/06 09:58:23 alfred Exp $
- */
-
-#ifndef _SYS_KTRACE_H_
-#define	_SYS_KTRACE_H_
-
-#include <sys/appleapiopts.h>
-
-#ifdef MACH_KERNEL_PRIVATE
-
-void ktrsyscall(void *, int, int, void *);
-void ktrsysret(void *, int, int, int);
-
-#else
-#ifdef __APPLE_API_UNSTABLE
-/*
- * operations to ktrace system call  (KTROP(op))
- */
-#define KTROP_SET		0	/* set trace points */
-#define KTROP_CLEAR		1	/* clear trace points */
-#define KTROP_CLEARFILE		2	/* stop all tracing to file */
-#define	KTROP(o)		((o)&3)	/* macro to extract operation */
-/*
- * flags (ORed in with operation)
- */
-#define KTRFLAG_DESCEND		4	/* perform op on all children too */
-
-/*
- * ktrace record header
- *
- * LP64todo: not 64-bit safe
- */
-struct ktr_header {
-	int	ktr_len;		/* length of buf */
-	short	ktr_type;		/* trace record type */
-	pid_t	ktr_pid;		/* process id */
-	char	ktr_comm[MAXCOMLEN+1];	/* command name */
-	struct	timeval ktr_time;	/* timestamp */
-	caddr_t	ktr_buf;
-};
-
-/*
- * Test for kernel trace point
- */
-#define KTRPOINT(p, type)	\
-	(((p)->p_traceflag & ((1<<(type))|KTRFAC_ACTIVE)) == (1<<(type)))
-
-/*
- * ktrace record types
- */
-
-/*
- * KTR_SYSCALL - system call record
- */
-#define KTR_SYSCALL	1
-struct ktr_syscall {
-	short	ktr_code;		/* syscall number */
-	short	ktr_narg;		/* number of arguments */
-	/*
-	 * followed by ktr_narg register_t
-	 */
-	u_int64_t	ktr_args[1];
-};
-
-/*
- * KTR_SYSRET - return from system call record
- */
-#define KTR_SYSRET	2
-struct ktr_sysret {
-	short	ktr_code;
-	short	ktr_eosys;
-	int	ktr_error;
-	register_t	ktr_retval;
-};
-
-/*
- * KTR_NAMEI - namei record
- */
-#define KTR_NAMEI	3
-	/* record contains pathname */
-
-/*
- * KTR_GENIO - trace generic process i/o
- */
-#define KTR_GENIO	4
-struct ktr_genio {
-	int	ktr_fd;
-	enum	uio_rw ktr_rw;
-	/*
-	 * followed by data successfully read/written
-	 */
-};
-
-/*
- * KTR_PSIG - trace processed signal
- */
-#define	KTR_PSIG	5
-struct ktr_psig {
-	int	signo;
-	sig_t	action;
-	int	code;
-	sigset_t mask;
-};
-
-/*
- * KTR_CSW - trace context switches
- */
-#define KTR_CSW		6
-struct ktr_csw {
-	int	out;	/* 1 if switch out, 0 if switch in */
-	int	user;	/* 1 if usermode (ivcsw), 0 if kernel (vcsw) */
-};
-
-/*
- * KTR_USER - data comming from userland
- */
-#define	KTR_USER_MAXLEN	2048	/* maximum length of passed data */
-#define KTR_USER	7
-
-/*
- * kernel trace points (in p_traceflag)
- */
-#define KTRFAC_MASK	0x00ffffff
-#define KTRFAC_SYSCALL	(1<<KTR_SYSCALL)
-#define KTRFAC_SYSRET	(1<<KTR_SYSRET)
-#define KTRFAC_NAMEI	(1<<KTR_NAMEI)
-#define KTRFAC_GENIO	(1<<KTR_GENIO)
-#define	KTRFAC_PSIG	(1<<KTR_PSIG)
-#define KTRFAC_CSW	(1<<KTR_CSW)
-#define KTRFAC_USER	(1<<KTR_USER)
-/*
- * trace flags (also in p_traceflags)
- */
-#define KTRFAC_ROOT	0x80000000	/* root set this trace */
-#define KTRFAC_INHERIT	0x40000000	/* pass trace flags to children */
-#define KTRFAC_ACTIVE	0x20000000	/* ktrace logging in progress, ignore */
-
-
-#ifdef	KERNEL
-#ifdef __APPLE_API_PRIVATE
-void	ktrnamei(struct vnode *,char *);
-void	ktrcsw(struct vnode *, int, int);
-void	ktrpsig(struct vnode *, int, sig_t, sigset_t *, int);
-void	ktrgenio(struct vnode *, int, enum uio_rw, struct uio *, int);
-void	ktrsyscall(struct proc *, int, int, syscall_arg_t args[]);
-void	ktrsysret(struct proc *, int, int, register_t);
-#endif /* __APPLE_API_PRIVATE */
-#else
-
-#include <sys/cdefs.h>
-
-__BEGIN_DECLS
-int	ktrace(const char *, int, int, pid_t);
-int	utrace(const void *, size_t);
-__END_DECLS
-
-#endif	/* !KERNEL */
-
-#endif /* __APPLE_API_UNSTABLE */
-#endif /* !MACH_KERNEL_PRIVATE */
-#endif /* !_SYS_KTRACE_H_ */
-
diff --git a/bsd/sys/lctx.h b/bsd/sys/lctx.h
new file mode 100644
index 000000000..fa374a0e8
--- /dev/null
+++ b/bsd/sys/lctx.h
@@ -0,0 +1,28 @@
+/*
+ * $Id: lctx.h,v 1.1.6.2 2006/03/03 23:20:46 msteil Exp $
+ */
+
+#ifndef _SYS_LCTX_H_
+#define _SYS_LCTX_H_
+
+#include <sys/syscall.h>
+
+#ifndef KERNEL
+static __inline pid_t
+getlcid(pid_t pid)
+{
+        return (syscall(SYS_getlcid, pid));
+}
+
+static __inline int
+setlcid(pid_t pid, pid_t lcid)
+{
+        return (syscall(SYS_setlcid, pid, lcid));
+}
+#endif
+
+#define LCID_PROC_SELF  (0)
+#define LCID_REMOVE     (-1)
+#define LCID_CREATE     (0)
+
+#endif	/* !_SYS_LCTX_H_ */
diff --git a/bsd/sys/linker_set.h b/bsd/sys/linker_set.h
index 25776d91a..45dd7742d 100644
--- a/bsd/sys/linker_set.h
+++ b/bsd/sys/linker_set.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1999 John D. Polstra
@@ -50,55 +56,163 @@
 #define _SYS_LINKER_SET_H_
 
 #include <sys/appleapiopts.h>
-
 #if !defined(KERNEL) || defined(__APPLE_API_PRIVATE)
+
 /*
  * The following macros are used to declare global sets of objects, which
- * are collected by the linker into a `struct linker_set' as defined below.
- * For ELF, this is done by constructing a separate segment for each set.
- * For a.out, it is done automatically by the linker.
+ * are collected by the linker into a `linker set' as defined below.
+ * For Mach-O, this is done by constructing a separate segment inside the
+ * __DATA section for each set.  The contents of this segment are an array
+ * of pointers to the objects in the set.
+ * 
+ * Note that due to limitations of the Mach-O format, there cannot
+ * be more than 255 sections in a segment, so linker set usage should be
+ * conserved.  Set names may not exceed 16 characters.
  */
 
-#define __ELF__
-#ifdef __ELF__
+#ifdef KERNEL
+# define MACH_KERNEL	1
+# include "mach-o/loader.h"
+typedef int _ls_size_t;
+# ifndef _KERN_MACH_HEADER_
+extern void *getsectdatafromheader(struct mach_header *, const char *, const char *, _ls_size_t *);
+extern struct mach_header _mh_execute_header;
+# endif
+#else
+# include <mach-o/ldsyms.h>
+# include <mach-o/getsect.h>
+# include <mach-o/loader.h>
+typedef unsigned long  _ls_size_t;
+#endif
 
-#define MAKE_SET(seg, set, sym)						\
-	static void const * const __set_##set##_sym_##sym = &sym;	\
-	__asm(".section  seg, " #set ""); 	\
-	__asm(".long " #sym);						
 
-/*	__asm(".previous") */
+/*
+ * Private macros, not to be used outside this header file.
+ *
+ * The objective of this macro stack is to produce the following output,
+ * given SET and SYM as arguments:
+ *
+ *  void const * __set_SET_sym_SYM __attribute__((section("__DATA,SET"))) = & SYM
+ */
+#ifdef __LS_VA_STRINGIFY__
+#  undef __LS_VA_STRINGIFY__
+#endif
+#ifdef __LS_VA_STRCONCAT__
+#  undef __LS_VA_STRCONCAT__
+#endif
+#define __LS_VA_STRINGIFY(_x...)	#_x
+#define __LS_VA_STRCONCAT(_x,_y)	__LS_VA_STRINGIFY(_x,_y)
+#define __LINKER_MAKE_SET(_set, _sym)					\
+	/*__unused*/ /*static*/ void const * /*const*/ __set_##_set##_sym_##_sym		\
+	__attribute__ ((section(__LS_VA_STRCONCAT(__DATA,_set)))) = (void *)&_sym
+/* the line above is very fragile - if your compiler breaks linker sets,
+   just play around with "static", "const" etc. :-) */
 
+/*
+ * Public macros.
+ */
+#define LINKER_SET_ENTRY(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
 
-#define TEXT_SET(set, sym) MAKE_SET(__TEXT, set, sym)
-#define DATA_SET(set, sym) MAKE_SET(__DATA, set, sym)
-#define BSS_SET(set, sym)  MAKE_SET(__BSS,  set, sym)
-#define ABS_SET(set, sym)  MAKE_SET(__ABS,  set, sym)
+/*
+ * FreeBSD compatibility.
+ */
+#ifdef __APPLE_API_OBSOLETE
+# define TEXT_SET(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
+# define DATA_SET(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
+# define BSS_SET(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
+# define ABS_SET(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
+# define SET_ENTRY(_set, _sym)	__LINKER_MAKE_SET(_set, _sym)
+#endif /* __APPLE_API_OBSOLETE */
 
-#else
+/*
+ * Extended linker set API.
+ *
+ * Since linker sets are per-object-file, and we may have multiple
+ * object files, we need to be able to specify which object's set
+ * to scan.
+ *
+ * The set itself is a contiguous array of pointers to the objects
+ * within the set.
+ */
 
 /*
- * NB: the constants defined below must match those defined in
- * nlist.h.  Since their calculation requires arithmetic, we
- * can't name them symbolically (e.g., 7 is N_DATA | N_EXT).
+ * Public interface.
+ *
+ * void **LINKER_SET_OBJECT_BEGIN(_object, _set)
+ *	Preferred interface to linker_set_object_begin(), takes set name unquoted.
+ * void **LINKER_SET_OBJECT_LIMIT(_object, _set)
+ *	Preferred interface to linker_set_object_begin(), takes set name unquoted.
+ * LINKER_SET_OBJECT_FOREACH(_object, (set_member_type **)_pvar, _set)
+ *	Iterates over the members of _set within _object.  Since the set contains
+ *	pointers to its elements, for a set of elements of type etyp, _pvar must
+ *	be (etyp **).
+ * set_member_type **LINKER_SET_OBJECT_ITEM(_object, _set, _i)
+ *	Returns a pointer to the _i'th element of _set within _object.
+ *
+ * void **LINKER_SET_BEGIN(_set)
+ * void **LINKER_SET_LIMINT(_set)
+ * LINKER_SET_FOREACH((set_member_type **)_pvar, _set)
+ * set_member_type **LINKER_SET_ITEM(_set, _i)
+ *	These versions implicitly reference the kernel/application object.
  */
-#define MAKE_SET(set, sym, type) \
-	static void const * const __set_##set##_sym_##sym = &sym; \
-	__asm(".stabs \"_" #set "\", " #type ", 0, 0, _" #sym)
 
-#define TEXT_SET(set, sym) MAKE_SET(set, sym, 5)
-#define DATA_SET(set, sym) MAKE_SET(set, sym, 7)
-#define BSS_SET(set, sym)  MAKE_SET(set, sym, 9)
-#define ABS_SET(set, sym)  MAKE_SET(set, sym, 3)
+#define LINKER_SET_OBJECT_BEGIN(_object, _set)	__linker_set_object_begin(_object, _set)
+#define LINKER_SET_OBJECT_LIMIT(_object, _set)	__linker_set_object_limit(_object, _set)
 
-#endif
+#define LINKER_SET_OBJECT_FOREACH(_object, _pvar, _set)			\
+	for ((void **)_pvar = LINKER_SET_OBJECT_BEGIN(_object, _set);		\
+     	     (void **)_pvar < LINKER_SET_OBJECT_LIMIT(_object, _set);	\
+	     ((void **)_pvar)++)
+
+#define LINKER_SET_OBJECT_ITEM(_object, _set, _i)			\
+	((LINKER_SET_OBJECT_BEGIN(_object, _set))[_i])
+
+#define LINKER_SET_BEGIN(_set)						\
+		LINKER_SET_OBJECT_BEGIN((struct mach_header *)&_mh_execute_header, _set)
+#define LINKER_SET_LIMIT(_set)						\
+		LINKER_SET_OBJECT_LIMIT((struct mach_header *)&_mh_execute_header, _set)
+#define LINKER_SET_FOREACH(_pvar, _set)					\
+	LINKER_SET_OBJECT_FOREACH((struct mach_header *)&_mh_execute_header, _pvar, _set)
+#define LINKER_SET_ITEM(_set, _i)					\
+      	LINKER_SET_OBJECT_ITEM((struct mach_header *)&_mh_execute_header, _set, _i)
+
+/*
+ * Implementation.
+ *
+ * void **__linker_set_object_begin(_header, _set)
+ *	Returns a pointer to the first pointer in the linker set.
+ * void **__linker_set_object_limi(_header, _set)
+ *	Returns an upper bound to the linker set (base + size).
+ */
+
+static __inline void **
+__linker_set_object_begin(struct mach_header *_header, const char *_set)
+     __attribute__((__const__));
+static __inline void **
+__linker_set_object_begin(struct mach_header *_header, const char *_set)
+{
+	void *_set_begin;
+	_ls_size_t _size;
+
+	_set_begin = getsectdatafromheader(_header, "__DATA", _set, &_size);
+	return((void **)_set_begin);
+}
+
+static __inline void **
+__linker_set_object_limit(struct mach_header *_header, const char *_set)
+     __attribute__((__const__));
+static __inline void **
+__linker_set_object_limit(struct mach_header *_header, const char *_set)
+{
+	void *_set_begin;
+	_ls_size_t _size;
+
+	_set_begin = getsectdatafromheader(_header, "__DATA", _set, &_size);
+	return((void **)((uintptr_t)_set_begin + _size));
+}
 
-struct linker_set {
-	int		ls_length;
-	const void	*ls_items[1];		/* really ls_length of them,
-						 * trailing NULL */
-};
 #endif /* !KERNEL || __APPLE_API_PRIVATE */
 
 #endif /* _SYS_LINKER_SET_H_ */
 
+
diff --git a/bsd/sys/loadable_fs.h b/bsd/sys/loadable_fs.h
index 95f1c727d..4006f46a4 100644
--- a/bsd/sys/loadable_fs.h
+++ b/bsd/sys/loadable_fs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)loadable_fs.h	2.0	26/06/90	(c) 1990 NeXT	*/
 
diff --git a/bsd/sys/lock.h b/bsd/sys/lock.h
index ec098b61a..4e2c537dc 100644
--- a/bsd/sys/lock.h
+++ b/bsd/sys/lock.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /* 
@@ -72,149 +78,4 @@
 
 #endif /* KERNEL */
 
-#ifdef BSD_KERNEL_PRIVATE
-/*
- * The general lock structure.  Provides for multiple shared locks,
- * upgrading from shared to exclusive, and sleeping until the lock
- * can be gained. The simple locks are defined in <machine/param.h>.
- */
-struct lock__bsd__ {
-	void * lk_interlock[10];		/* lock on remaining fields */
-	u_int	lk_flags;		/* see below */
-	int	lk_sharecount;		/* # of accepted shared locks */
-	int	lk_waitcount;		/* # of processes sleeping for lock */
-	short	lk_exclusivecount;	/* # of recursive exclusive locks */
-	short	lk_prio;		/* priority at which to sleep */
-	const char	*lk_wmesg;	/* resource sleeping (for tsleep) */
-	int	lk_timo;		/* maximum sleep time (for tsleep) */
-	pid_t	lk_lockholder;		/* pid of exclusive lock holder */
-	void	*lk_lockthread;		/* thread which acquired excl lock */
-};
-
-// LP64todo - should this move?
-
-/* LP64 version of lock__bsd__.  all pointers 
- * grow when we're dealing with a 64-bit process.
- * WARNING - keep in sync with lock__bsd__
- */
-
-struct user_lock__bsd__ {
-	user_addr_t	lk_interlock[10];   /* lock on remaining fields */
-	u_int		lk_flags;			/* see below */
-	int			lk_sharecount;		/* # of accepted shared locks */
-	int			lk_waitcount;		/* # of processes sleeping for lock */
-	short		lk_exclusivecount;	/* # of recursive exclusive locks */
-	short		lk_prio;			/* priority at which to sleep */
-	user_addr_t lk_wmesg;           /* resource sleeping (for tsleep) */
-	int			lk_timo;			/* maximum sleep time (for tsleep) */
-	pid_t		lk_lockholder;		/* pid of exclusive lock holder */
-	user_addr_t	lk_lockthread;		/* thread which acquired excl lock */
-};
-
-/*
- * Lock request types:
- *   LK_SHARED - get one of many possible shared locks. If a process
- *	holding an exclusive lock requests a shared lock, the exclusive
- *	lock(s) will be downgraded to shared locks.
- *   LK_EXCLUSIVE - stop further shared locks, when they are cleared,
- *	grant a pending upgrade if it exists, then grant an exclusive
- *	lock. Only one exclusive lock may exist at a time, except that
- *	a process holding an exclusive lock may get additional exclusive
- *	locks if it explicitly sets the LK_CANRECURSE flag in the lock
- *	request, or if the LK_CANRECUSE flag was set when the lock was
- *	initialized.
- *   LK_UPGRADE - the process must hold a shared lock that it wants to
- *	have upgraded to an exclusive lock. Other processes may get
- *	exclusive access to the resource between the time that the upgrade
- *	is requested and the time that it is granted.
- *   LK_EXCLUPGRADE - the process must hold a shared lock that it wants to
- *	have upgraded to an exclusive lock. If the request succeeds, no
- *	other processes will have gotten exclusive access to the resource
- *	between the time that the upgrade is requested and the time that
- *	it is granted. However, if another process has already requested
- *	an upgrade, the request will fail (see error returns below).
- *   LK_DOWNGRADE - the process must hold an exclusive lock that it wants
- *	to have downgraded to a shared lock. If the process holds multiple
- *	(recursive) exclusive locks, they will all be downgraded to shared
- *	locks.
- *   LK_RELEASE - release one instance of a lock.
- *   LK_DRAIN - wait for all activity on the lock to end, then mark it
- *	decommissioned. This feature is used before freeing a lock that
- *	is part of a piece of memory that is about to be freed.
- *
- * These are flags that are passed to the lockmgr routine.
- */
-#define LK_TYPE_MASK	0x0000000f	/* type of lock sought */
-#define LK_SHARED	0x00000001	/* shared lock */
-#define LK_EXCLUSIVE	0x00000002	/* exclusive lock */
-#define LK_UPGRADE	0x00000003	/* shared-to-exclusive upgrade */
-#define LK_EXCLUPGRADE	0x00000004	/* first shared-to-exclusive upgrade */
-#define LK_DOWNGRADE	0x00000005	/* exclusive-to-shared downgrade */
-#define LK_RELEASE	0x00000006	/* release any type of lock */
-#define LK_DRAIN	0x00000007	/* wait for all lock activity to end */
-/*
- * External lock flags.
- *
- * The first three flags may be set in lock_init to set their mode permanently,
- * or passed in as arguments to the lock manager. The LK_REENABLE flag may be
- * set only at the release of a lock obtained by drain.
- */
-#define LK_EXTFLG_MASK	0x00000070	/* mask of external flags */
-#define LK_NOWAIT	0x00000010	/* do not sleep to await lock */
-#define LK_SLEEPFAIL	0x00000020	/* sleep, then return failure */
-#define LK_CANRECURSE	0x00000040	/* allow recursive exclusive lock */
-#define LK_REENABLE	0x00000080	/* lock is be reenabled after drain */
-/*
- * Internal lock flags.
- *
- * These flags are used internally to the lock manager.
- */
-#define LK_WANT_UPGRADE	0x00000100	/* waiting for share-to-excl upgrade */
-#define LK_WANT_EXCL	0x00000200	/* exclusive lock sought */
-#define LK_HAVE_EXCL	0x00000400	/* exclusive lock obtained */
-#define LK_WAITDRAIN	0x00000800	/* process waiting for lock to drain */
-#define LK_DRAINING	0x00004000	/* lock is being drained */
-#define LK_DRAINED	0x00008000	/* lock has been decommissioned */
-/*
- * Control flags
- *
- * Non-persistent external flags.
- */
-#define LK_INTERLOCK	0x00010000	/* unlock passed simple lock after
-					   getting lk_interlock */
-#define LK_RETRY	0x00020000	/* vn_lock: retry until locked */
-
-/*
- * Lock return status.
- *
- * Successfully obtained locks return 0. Locks will always succeed
- * unless one of the following is true:
- *	LK_FORCEUPGRADE is requested and some other process has already
- *	    requested a lock upgrade (returns EBUSY).
- *	LK_WAIT is set and a sleep would be required (returns EBUSY).
- *	LK_SLEEPFAIL is set and a sleep was done (returns ENOLCK).
- *	PCATCH is set in lock priority and a signal arrives (returns
- *	    either EINTR or ERESTART if system calls is to be restarted).
- *	Non-null lock timeout and timeout expires (returns EWOULDBLOCK).
- * A failed lock attempt always returns a non-zero error value. No lock
- * is held after an error return (in particular, a failed LK_UPGRADE
- * or LK_FORCEUPGRADE will have released its shared access lock).
- */
-
-/*
- * Indicator that no process holds exclusive lock
- */
-#define LK_KERNPROC ((pid_t) -2)
-#define LK_NOPROC ((pid_t) -1)
-
-struct proc;
-
-void	lockinit(struct lock__bsd__ *, int prio, const char *wmesg, int timo,
-			int flags);
-int	lockmgr(struct lock__bsd__ *, u_int flags,
-			void *, struct proc *p);
-int	lockstatus(struct lock__bsd__ *);
-
-#endif /* BSD_KERNEL_PRIVATE  */
-
 #endif	/* _SYS_LOCK_H_ */
diff --git a/bsd/sys/lockf.h b/bsd/sys/lockf.h
index 1c4e21a95..f5f8ad03a 100644
--- a/bsd/sys/lockf.h
+++ b/bsd/sys/lockf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1991, 1993
@@ -70,8 +76,9 @@ MALLOC_DECLARE(M_LOCKF);
 /*
  * The lockf structure is a kernel structure which contains the information
  * associated with a byte range lock.  The lockf structures are linked into
- * the inode structure. Locks are sorted by the starting byte of the lock for
- * efficiency.
+ * the vnode structure.  Locks are sorted by the starting byte of the lock for
+ * efficiency after they have been committed; uncommitted locks are on the list
+ * head so they may quickly be accessed, and are both short lived and transient.
  */
 TAILQ_HEAD(locklist, lockf);
 
@@ -97,7 +104,9 @@ struct lockf {
 
 __BEGIN_DECLS
 
-int	 lf_advlock(struct vnop_advlock_args *);
+int	lf_advlock(struct vnop_advlock_args *);
+int	lf_assert(struct vnop_advlock_args *, void **);
+void	lf_commit(void *, int);
 
 #ifdef LOCKF_DEBUG
 void	lf_print(char *, struct lockf *);
diff --git a/bsd/sys/lockstat.h b/bsd/sys/lockstat.h
new file mode 100644
index 000000000..72d3ca1c8
--- /dev/null
+++ b/bsd/sys/lockstat.h
@@ -0,0 +1,231 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 1997-2003 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_LOCKSTAT_H
+#define	_SYS_LOCKSTAT_H
+#endif
+
+/* #pragma ident	"@(#)lockstat.h	1.6	05/06/08 SMI" */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/*
+ * Spin locks - we have less variants
+ */
+#define	LS_LCK_SPIN_LOCK_ACQUIRE	0
+#define	LS_LCK_SPIN_LOCK_SPIN		1
+#define	LS_LCK_SPIN_UNLOCK_RELEASE	2
+/*
+ * Mutexes can also have interlock-spin events, which are
+ * unique to our lock implementation.
+ */
+#define	LS_LCK_MTX_LOCK_ACQUIRE			3
+#define	LS_LCK_MTX_LOCK_BLOCK			5
+#define	LS_LCK_MTX_LOCK_SPIN			6
+#define	LS_LCK_MTX_LOCK_ILK_SPIN		7
+#define	LS_LCK_MTX_TRY_LOCK_ACQUIRE		8
+#define	LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE	9
+#define	LS_LCK_MTX_UNLOCK_RELEASE		10
+/*
+ * Although theoretically deprecated, there's still lots of uses of mutex_lock() style locks.
+ * Block and spin events share code with lck_mtx_lock() and friends, so we only need events
+ * for the acquire and release events.
+ */
+#define LS_MUTEX_LOCK_ACQUIRE			11
+#define LS_MUTEX_TRY_LOCK_ACQUIRE		12
+#define	LS_MUTEX_TRY_SPIN_ACQUIRE		13
+#define	LS_MUTEX_UNLOCK_RELEASE			14
+#define	LS_MUTEX_LOCK_SPIN_ACQUIRE		15
+
+#define	LS_MUTEX_CONVERT_SPIN_ACQUIRE		16
+/*
+ * Provide a parallel set for indirect mutexes
+ */
+#define	LS_LCK_MTX_EXT_LOCK_ACQUIRE			17
+#define	LS_LCK_MTX_EXT_LOCK_BLOCK			18
+#define	LS_LCK_MTX_EXT_LOCK_SPIN			19
+#define	LS_LCK_MTX_EXT_LOCK_ILK_SPIN		20
+#define	LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE		21
+#define	LS_LCK_MTX_EXT_UNLOCK_RELEASE		22
+/*
+ * Our reader-writer locks support a blocking upgrade primitive, as
+ * well as the possibility of spinning on the interlock.
+ */
+#define	LS_LCK_RW_LOCK_SHARED_ACQUIRE		23
+#define	LS_LCK_RW_LOCK_SHARED_BLOCK		24
+#define	LS_LCK_RW_LOCK_SHARED_SPIN		25
+
+#define	LS_LCK_RW_LOCK_EXCL_ACQUIRE		26
+#define	LS_LCK_RW_LOCK_EXCL_BLOCK		27
+#define	LS_LCK_RW_LOCK_EXCL_SPIN		28
+
+#define	LS_LCK_RW_DONE_RELEASE			29
+
+#define	LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE	30
+#define	LS_LCK_RW_TRY_LOCK_SHARED_SPIN		31
+
+#define	LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE		32
+#define	LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN	33
+
+#define	LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE	34
+#define	LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN	35
+#define	LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK	36
+
+#define	LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE	37
+#define	LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN	38
+
+#define	LS_NPROBES			39
+
+/*
+ * Name the various locking functions...
+ */
+#define	LS_LCK_MTX_LOCK			"lck_mtx_lock"
+#define	LS_LCK_MTX_SPIN_LOCK		"lck_mtx_spin_lock"
+#define	LS_LCK_MTX_UNLOCK		"lck_mtx_unlock"
+#define	LS_LCK_MTX_TRY_LOCK		"lck_mtx_try_lock"
+#define	LS_LCK_MTX_TRY_SPIN_LOCK	"lck_mtx_try_spin_lock"
+#define	LS_LCK_MTX_EXT_LOCK		"lck_mtx_ext_lock"
+#define	LS_LCK_MTX_EXT_UNLOCK		"lck_mtx_ext_unlock"
+#define	LS_LCK_MTX_EXT_TRY_LOCK		"lck_mtx_ext_try_lock"
+#define	LS_MUTEX_CONVERT_SPIN		"mutex_convert_spin"
+#define	LS_MUTEX_LOCK			"mutex_lock"
+#define	LS_MUTEX_UNLOCK			"mutex_unlock"
+#define	LS_MUTEX_TRY_LOCK		"mutex_try_lock"
+#define	LS_MUTEX_TRY_SPIN		"mutex_try_spin"
+#define	LS_MUTEX_LOCK_SPIN		"mutex_lock_spin"
+#define	LS_LCK_SPIN_LOCK		"lck_spin_lock"
+#define	LS_LCK_SPIN_TRY_LOCK		"lck_spin_try_lock"
+#define	LS_LCK_SPIN_UNLOCK		"lck_spin_unlock"
+#define	LS_LCK_RW_LOCK_SHARED		"lck_rw_lock_shared"
+#define	LS_LCK_RW_LOCK_EXCL		"lck_rw_lock_exclusive"
+#define	LS_LCK_RW_DONE			"lck_rw_done"
+#define	LS_LCK_RW_TRY_LOCK_EXCL		"lck_rw_try_lock_exclusive"
+#define	LS_LCK_RW_TRY_LOCK_SHARED	"lck_rw_try_lock_shared"
+#define	LS_LCK_RW_LOCK_SHARED_TO_EXCL	"lck_rw_shared_to_exclusive"
+#define	LS_LCK_RW_LOCK_EXCL_TO_SHARED	"lck_rw_exclusive_to_shared"
+
+#define	LS_ACQUIRE			"acquire"
+#define	LS_RELEASE			"release"
+#define	LS_SPIN				"spin"
+#define	LS_BLOCK			"block"
+#define	LS_UPGRADE			"upgrade"
+#define	LS_DOWNGRADE			"downgrade"
+
+#define	LS_TYPE_ADAPTIVE		"adaptive"
+#define	LS_TYPE_SPIN			"spin"
+#define	LS_TYPE_ILK			"interlock"	/* OS X only */
+#define	LS_TYPE_THREAD			"thread"	/* Solaris only */
+#define	LS_TYPE_RW			"rw"
+#define	LS_TYPE_RWUPGRADE		"rwupgrade"	/* OS X only */
+
+#define	LSA_ACQUIRE			(LS_TYPE_ADAPTIVE "-" LS_ACQUIRE)
+#define	LSA_RELEASE			(LS_TYPE_ADAPTIVE "-" LS_RELEASE)
+#define	LSA_SPIN			(LS_TYPE_ADAPTIVE "-" LS_SPIN)
+#define	LSA_BLOCK			(LS_TYPE_ADAPTIVE "-" LS_BLOCK)
+#define	LSA_ILK_SPIN			(LS_TYPE_ILK "-" LS_SPIN)
+#define	LSS_ACQUIRE			(LS_TYPE_SPIN "-" LS_ACQUIRE)
+#define	LSS_RELEASE			(LS_TYPE_SPIN "-" LS_RELEASE)
+#define	LSS_SPIN			(LS_TYPE_SPIN "-" LS_SPIN)
+#define	LSR_ACQUIRE			(LS_TYPE_RW "-" LS_ACQUIRE)
+#define	LSR_RELEASE			(LS_TYPE_RW "-" LS_RELEASE)
+#define	LSR_BLOCK			(LS_TYPE_RW "-" LS_BLOCK)
+#define	LSR_SPIN			(LS_TYPE_RW "-" LS_SPIN)
+#define	LSR_UPGRADE			(LS_TYPE_RW "-" LS_UPGRADE)
+#define	LSR_UPGRADE_BLOCK		(LS_TYPE_RWUPGRADE "-" LS_BLOCK)
+#define	LSR_DOWNGRADE			(LS_TYPE_RW "-" LS_DOWNGRADE)
+#define	LST_SPIN			(LS_TYPE_THREAD "-" LS_SPIN)
+
+#ifndef _ASM
+
+#include <stdint.h>
+#ifdef KERNEL
+
+#ifndef _KERNEL
+#define _KERNEL /* Solaris vs. Darwin */
+#endif
+
+/*
+ * Platform-independent kernel support for the lockstat driver.
+ */
+#if defined(NEED_DTRACE_DEFS)
+typedef uint32_t dtrace_id_t;		/* probe identifier - also in dtrace.h! */
+typedef uint64_t	u_longlong_t;	/* also in dtrace.h! */
+#endif
+
+extern dtrace_id_t lockstat_probemap[LS_NPROBES];
+extern void (*lockstat_probe)(dtrace_id_t, uint64_t, uint64_t,
+    uint64_t, uint64_t, uint64_t);
+
+
+#ifdef _KERNEL
+
+#if	CONFIG_DTRACE
+extern int lockstat_depth(void);
+extern void lockstat_hot_patch(boolean_t);
+extern void dtrace_membar_producer(void);
+
+/*
+ * Macros to record lockstat probes.
+ */
+#define	LOCKSTAT_RECORD4(probe, lp, arg0, arg1, arg2, arg3)		\
+	{								\
+		dtrace_id_t id;						\
+		if ((id = lockstat_probemap[(probe)])) {		\
+			(*lockstat_probe)(id, (uintptr_t)(lp), (arg0),	\
+			    (arg1), (arg2), (arg3));			\
+		}							\
+	}
+
+#define	LOCKSTAT_RECORD2(probe, lp, arg1, arg2)	\
+	LOCKSTAT_RECORD4(probe, lp, arg1, arg2, 0, 0)
+
+#define	LOCKSTAT_RECORD(probe, lp, arg)	\
+	LOCKSTAT_RECORD4(probe, lp, arg, 0, 0, 0)
+
+#define	LOCKSTAT_RECORD0(probe, lp)	\
+	LOCKSTAT_RECORD4(probe, lp, 0, 0, 0, 0)
+#else
+	/* No Lockstat provider */
+
+#define LOCKSTAT_RECORD()
+#define	LOCKSTAT_RECORD0()
+#define	LOCKSTAT_RECORD2()
+#define	LOCKSTAT_RECORD4()
+
+#endif /* !CONFIG_DTRACE */
+
+#endif /* _KERNEL */
+
+#endif /* _ASM */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SYS_LOCKSTAT_H */
+
diff --git a/bsd/sys/mach_swapon.h b/bsd/sys/mach_swapon.h
index 152a9265b..f0330c413 100644
--- a/bsd/sys/mach_swapon.h
+++ b/bsd/sys/mach_swapon.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (c) 1989,1995 NeXT, Inc.
diff --git a/bsd/sys/malloc.h b/bsd/sys/malloc.h
index 1bb501e9c..ea59c270b 100644
--- a/bsd/sys/malloc.h
+++ b/bsd/sys/malloc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
@@ -55,6 +61,12 @@
  *
  *	@(#)malloc.h	8.5 (Berkeley) 5/3/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_MALLOC_H_
 #define	_SYS_MALLOC_H_
@@ -73,7 +85,7 @@
 
 #ifdef BSD_KERNEL_PRIVATE
 
-#define KMEMSTATS
+#define KMEMSTATS 0
 
 /*
  * Types of memory to be allocated (not all are used by us)
@@ -108,8 +120,8 @@
 #define	M_DQUOT		27	/* UFS quota entries */
 #define	M_UFSMNT	28	/* UFS mount structure */
 #define	M_SHM		29	/* SVID compatible shared memory segments */
-#define	M_VMMAP		30	/* VM map structures */
-#define	M_VMMAPENT	31	/* VM map entry structures */
+#define	M_PLIMIT	30	/* plimit  structures */
+#define	M_SIGACTS	31	/* sigacts structures */
 #define	M_VMOBJ		32	/* VM object structure */
 #define	M_VMOBJHASH	33	/* VM object hash structure */
 #define	M_VMPMAP	34	/* VM pmap */
@@ -120,7 +132,7 @@
 #define	M_FILEDESC	39	/* Open file descriptor table */
 #define	M_LOCKF		40	/* Byte-range locking structures */
 #define	M_PROC		41	/* Proc structures */
-#define	M_SUBPROC	42	/* Proc sub-structures */
+#define	M_PSTATS	42	/* pstats  proc sub-structures */
 #define	M_SEGMENT	43	/* Segment for LFS */
 #define	M_LFSNODE	44	/* LFS vnode private part */
 #define	M_FFSNODE	45	/* FFS vnode private part */
@@ -128,9 +140,9 @@
 #define	M_NQLEASE	47	/* XXX: Nqnfs lease */
 #define	M_NQMHOST	48	/* XXX: Nqnfs host address table */
 #define	M_NETADDR	49	/* Export host address structure */
-#define	M_NFSSVC	50	/* Nfs server structure */
-#define	M_NFSUID	51	/* Nfs uid mapping structure */
-#define	M_NFSD		52	/* Nfs server daemon structure */
+#define	M_NFSSVC	50	/* NFS server structure */
+#define	M_NFSUID	51	/* XXX: NFS uid mapping structure */
+#define	M_NFSD		52	/* NFS server daemon structure */
 #define	M_IPMOPTS	53	/* internet multicast options */
 #define	M_IPMADDR	54	/* internet multicast address */
 #define	M_IFMADDR	55	/* link-level multicast address */
@@ -156,10 +168,9 @@
 #define	M_HFSMNT	75	/* HFS mount structure */
 #define	M_HFSNODE	76	/* HFS catalog node */
 #define	M_HFSFORK	77	/* HFS file fork */
-#define M_VOLFSMNT	78  /* VOLFS mount structure */
-#define	M_VOLFSNODE	79	/* VOLFS private node part */
+#define M_ZFSMNT	78	/* ZFS mount data */
+#define	M_ZFSNODE	79	/* ZFS inode */
 #define	M_TEMP		80	/* misc temporary data buffers */
-#define	M_KTRACE	M_TEMP	/* ktrace buffers */
 #define	M_SECA		81	/* security associations, key management */
 #define M_DEVFS		82
 #define M_IPFW		83	/* IP Forwarding/NAT */
@@ -182,6 +193,13 @@
 #define M_KAUTH		100	/* kauth subsystem */
 #define M_DUMMYNET	101	/* dummynet */
 #define M_UNSAFEFS	102	/* storage for vnode lock state for unsafe FS */
+#define M_MACPIPELABEL	103	/* MAC pipe labels */
+#define M_MACTEMP	104	/* MAC framework */
+#define M_SBUF		105	/* string buffers */
+#define M_EXTATTR	106	/* extended attribute */
+#define M_LCTX		107	/* process login context */
+
+#define	M_LAST		109	/* Must be last type + 1 */
 
 #else /* BSD_KERNEL_PRIVATE */
 
@@ -200,117 +218,7 @@
 
 #ifdef BSD_KERNEL_PRIVATE
 
-
-#define	M_LAST		103	/* Must be last type + 1 */
-
-/* Strings corresponding to types of memory */
-/* Must be in synch with the #defines above */
-#define	INITKMEMNAMES { \
-	"free",		/* 0 M_FREE */ \
-	"mbuf",		/* 1 M_MBUF */ \
-	"devbuf",	/* 2 M_DEVBUF */ \
-	"socket",	/* 3 M_SOCKET */ \
-	"pcb",		/* 4 M_PCB */ \
-	"routetbl",	/* 5 M_RTABLE */ \
-	"hosttbl",	/* 6 M_HTABLE */ \
-	"fragtbl",	/* 7 M_FTABLE */ \
-	"zombie",	/* 8 M_ZOMBIE */ \
-	"ifaddr",	/* 9 M_IFADDR */ \
-	"soopts",	/* 10 M_SOOPTS */ \
-	"soname",	/* 11 M_SONAME */ \
-	"namei",	/* 12 M_NAMEI */ \
-	"gprof",	/* 13 M_GPROF */ \
-	"ioctlops",	/* 14 M_IOCTLOPS */ \
-	"mapmem",	/* 15 M_MAPMEM */ \
-	"cred",		/* 16 M_CRED */ \
-	"pgrp",		/* 17 M_PGRP */ \
-	"session",	/* 18 M_SESSION */ \
-	"iov32",	/* 19 M_IOV32 */ \
-	"mount",	/* 20 M_MOUNT */ \
-	"fhandle",	/* 21 M_FHANDLE */ \
-	"NFS req",	/* 22 M_NFSREQ */ \
-	"NFS mount",	/* 23 M_NFSMNT */ \
-	"NFS node",	/* 24 M_NFSNODE */ \
-	"vnodes",	/* 25 M_VNODE */ \
-	"namecache",	/* 26 M_CACHE */ \
-	"UFS quota",	/* 27 M_DQUOT */ \
-	"UFS mount",	/* 28 M_UFSMNT */ \
-	"shm",		/* 29 M_SHM */ \
-	"VM map",	/* 30 M_VMMAP */ \
-	"VM mapent",	/* 31 M_VMMAPENT */ \
-	"VM object",	/* 32 M_VMOBJ */ \
-	"VM objhash",	/* 33 M_VMOBJHASH */ \
-	"VM pmap",	/* 34 M_VMPMAP */ \
-	"VM pvmap",	/* 35 M_VMPVENT */ \
-	"VM pager",	/* 36 M_VMPAGER */ \
-	"VM pgdata",	/* 37 M_VMPGDATA */ \
-	"fileproc",	/* 38 M_FILEPROC */ \
-	"file desc",	/* 39 M_FILEDESC */ \
-	"lockf",	/* 40 M_LOCKF */ \
-	"proc",		/* 41 M_PROC */ \
-	"subproc",	/* 42 M_SUBPROC */ \
-	"LFS segment",	/* 43 M_SEGMENT */ \
-	"LFS node",	/* 44 M_LFSNODE */ \
-	"FFS node",	/* 45 M_FFSNODE */ \
-	"MFS node",	/* 46 M_MFSNODE */ \
-	"NQNFS Lease",	/* 47 M_NQLEASE */ \
-	"NQNFS Host",	/* 48 M_NQMHOST */ \
-	"Export Host",	/* 49 M_NETADDR */ \
-	"NFS srvsock",	/* 50 M_NFSSVC */ \
-	"NFS uid",	/* 51 M_NFSUID */ \
-	"NFS daemon",	/* 52 M_NFSD */ \
-	"ip_moptions",	/* 53 M_IPMOPTS */ \
-	"in_multi",	/* 54 M_IPMADDR */ \
-	"ether_multi",	/* 55 M_IFMADDR */ \
-	"mrt",		/* 56 M_MRTABLE */ \
-	"ISOFS mount",	/* 57 M_ISOFSMNT */ \
-	"ISOFS node",	/* 58 M_ISOFSNODE */ \
-	"NFSV3 srvdesc",/* 59 M_NFSRVDESC */ \
-	"NFSV3 diroff",	/* 60 M_NFSDIROFF */ \
-	"NFSV3 bigfh",	/* 61 M_NFSBIGFH */ \
-	"MSDOSFS mount",/* 62 M_MSDOSFSMNT */ \
-	"MSDOSFS fat",	/* 63 M_MSDOSFSFAT */ \
-	"MSDOSFS node",	/* 64 M_MSDOSFSNODE */ \
-	"ttys",		/* 65 M_TTYS */ \
-	"exec",		/* 66 M_EXEC */ \
-	"miscfs mount",	/* 67 M_MISCFSMNT */ \
-	"miscfs node",	/* 68 M_MISCFSNODE */ \
-	"adosfs mount",	/* 69 M_ADOSFSMNT */ \
-	"adosfs node",	/* 70 M_ADOSFSNODE */ \
-	"adosfs anode",	/* 71 M_ANODE */ \
-	"buf hdrs",	/* 72 M_BUFHDR */ \
-	"ofile tabl",	/* 73 M_OFILETABL */ \
-	"mbuf clust",	/* 74 M_MCLUST */ \
-	"HFS mount",	/* 75 M_HFSMNT */ \
-	"HFS node",	/* 76 M_HFSNODE */ \
-	"HFS fork",	/* 77 M_HFSFORK */ \
-	"VOLFS mount", 	/* 78 M_VOLFSMNT */ \
-	"VOLFS node", 	/* 79 M_VOLFSNODE */ \
-	"temp",		/* 80 M_TEMP */ \
-	"key mgmt",	/* 81 M_SECA */ \
-	"DEVFS",	/* 82 M_DEVFS */ \
-	"IpFw/IpAcct",	/* 83 M_IPFW */ \
-	"UDF node",	/* 84 M_UDFNODE */ \
-	"UDF mount",	/* 85 M_UDFMNT */ \
-	"IPv6 NDP",	/* 86 M_IP6NDP */ \
-	"IPv6 options",	/* 87 M_IP6OPT */ \
-	"IPv6 Misc",	/* 88 M_IP6MISC */\
-	"TCP Segment Q",/* 89 M_TSEGQ */\
-	"IGMP state",	/* 90 M_IGMP */\
-	"Journal",    /* 91 M_JNL_JNL */\
-	"Transaction",    /* 92 M_JNL_TR */\
-	"specinfo",		/* 93 M_SPECINFO */\
-	"kqueue",	/* 94 M_KQUEUE */\
-	"HFS dirhint",	/* 95 M_HFSDIRHINT */ \
-        "cluster_read",	/* 96 M_CLRDAHEAD */ \
-        "cluster_write",/* 97 M_CLWRBEHIND */ \
-	"iov64",	/* 98 M_IOV64 */ \
-	"fileglob",	/* 99 M_FILEGLOB */ \
-	"kauth",		/* 100 M_KAUTH */ \
-	"dummynet",		/* 101 M_DUMMYNET */ \
-        "unsafe_fsnode"	/* 102 M_UNSAFEFS */ \
-}
-
+#if KMEMSTATS
 struct kmemstats {
 	long	ks_inuse;	/* # of packets of this type currently
 				 * in use */
@@ -325,6 +233,7 @@ struct kmemstats {
 };
 
 extern struct kmemstats kmemstats[];
+#endif /* KMEMSTATS */
 
 #endif /* BSD_KERNEL_PRIVATE */
 
diff --git a/bsd/sys/mbuf.h b/bsd/sys/mbuf.h
index 23c79b866..ec7c2733e 100644
--- a/bsd/sys/mbuf.h
+++ b/bsd/sys/mbuf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -68,6 +74,12 @@
  *	New version based on 4.4
  *	Purged old history
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef	_SYS_MBUF_H_
 #define	_SYS_MBUF_H_
@@ -100,16 +112,9 @@
  * Macros for type conversion
  * mtod(m,t) -	convert mbuf pointer to data pointer of correct type
  * dtom(x) -	convert data pointer within mbuf to mbuf pointer (XXX)
- * mtocl(x) -	convert pointer within cluster to cluster index #
- * cltom(x) -	convert cluster # to ptr to beginning of cluster
  */
 #define mtod(m,t)       ((t)m_mtod(m))
 #define dtom(x)         m_dtom(x)
-#define mtocl(x)        m_mtocl(x)
-#define cltom(x)        m_cltom(x)
-
-#define MCLREF(p)       m_mclref(p)
-#define MCLUNREF(p)     m_mclunref(p)
 
 /* header at beginning of each mbuf: */
 struct m_hdr {
@@ -144,7 +149,7 @@ struct	pkthdr {
 #endif KERNEL_PRIVATE
         int     csum_flags;             /* flags regarding checksum */       
         int     csum_data;              /* data field used by csum routines */
-	struct mbuf *aux;		/* extra data buffer; ipsec/others */
+	void	*reserved0;		/* unused, for future use */
 #ifdef KERNEL_PRIVATE
 	u_short	vlan_tag;		/* VLAN tag, host byte order */
 	u_short socket_id;		/* socket id */
@@ -164,6 +169,10 @@ struct m_ext {
 	struct	ext_refsq {		/* references held */
 		struct ext_refsq *forward, *backward;
 	} ext_refs;
+	struct ext_ref {
+		u_int32_t refcnt;
+		u_int32_t flags;
+	} *ext_refflags;
 };
 
 struct mbuf {
@@ -269,23 +278,6 @@ struct mbuf {
 #define	M_DONTWAIT	M_NOWAIT
 #define	M_WAIT		M_WAITOK
 
-/*
- * mbuf utility macros:
- *
- *	MBUFLOCK(code)
- * prevents a section of code from from being interrupted by network
- * drivers.
- */
-
-#ifdef _KERN_LOCKS_H_
-extern lck_mtx_t * mbuf_mlock;
-#else
-extern void * mbuf_mlock;
-#endif
-
-#define MBUF_LOCK()	lck_mtx_lock(mbuf_mlock);
-#define MBUF_UNLOCK()	lck_mtx_unlock(mbuf_mlock);
-
 /*
  * mbuf allocation/deallocation macros:
  *
@@ -303,8 +295,6 @@ extern void * mbuf_mlock;
 #define MCHECK(m)
 #endif
 
-extern struct mbuf *mfree;				/* mbuf free list */
-
 #define	MGET(m, how, type) ((m) = m_get((how), (type)))
 
 #define	MGETHDR(m, how, type)	((m) = m_gethdr((how), (type)))
@@ -341,6 +331,12 @@ union mbigcluster {
 	char 			mbc_buf[NBPG];
 };
 
+#define	M16KCLBYTES	(16 * 1024)
+
+union m16kcluster {
+	union m16kcluster	*m16kcl_next;
+	char			m16kcl_buf[M16KCLBYTES];
+};
 
 #define MCLHASREFERENCE(m) m_mclhasreference(m)
 
@@ -406,72 +402,114 @@ union mbigcluster {
 #endif /* KERNEL_PRIVATE */
 
 /*
- * Mbuf statistics.
+ * Mbuf statistics (legacy).
  */
-/* LP64todo - not 64-bit safe */
 struct mbstat {
-	u_long  m_mbufs;        /* mbufs obtained from page pool */
-	u_long  m_clusters;     /* clusters obtained from page pool */
-	u_long  m_spare;        /* spare field */
-	u_long  m_clfree;       /* free clusters */
-	u_long  m_drops;        /* times failed to find space */
-	u_long  m_wait;         /* times waited for space */
-	u_long  m_drain;        /* times drained protocols for space */
-	u_short m_mtypes[256];  /* type specific mbuf allocations */
-	u_long  m_mcfail;       /* times m_copym failed */
-	u_long  m_mpfail;       /* times m_pullup failed */
-	u_long  m_msize;        /* length of an mbuf */
-	u_long  m_mclbytes;     /* length of an mbuf cluster */
-	u_long  m_minclsize;    /* min length of data to allocate a cluster */
-	u_long  m_mlen;         /* length of data in an mbuf */
-	u_long  m_mhlen;        /* length of data in a header mbuf */
-	u_long  m_bigclusters;     /* clusters obtained from page pool */
-	u_long  m_bigclfree;       /* free clusters */
-	u_long  m_bigmclbytes;     /* length of an mbuf cluster */
+	u_int32_t	m_mbufs;	/* mbufs obtained from page pool */
+	u_int32_t	m_clusters;	/* clusters obtained from page pool */
+	u_int32_t	m_spare;	/* spare field */
+	u_int32_t	m_clfree;	/* free clusters */
+	u_int32_t	m_drops;	/* times failed to find space */
+	u_int32_t	m_wait;		/* times waited for space */
+	u_int32_t	m_drain;	/* times drained protocols for space */
+	u_short		m_mtypes[256];	/* type specific mbuf allocations */
+	u_int32_t	m_mcfail;	/* times m_copym failed */
+	u_int32_t	m_mpfail;	/* times m_pullup failed */
+	u_int32_t	m_msize;	/* length of an mbuf */
+	u_int32_t	m_mclbytes;	/* length of an mbuf cluster */
+	u_int32_t	m_minclsize;	/* min length of data to allocate a cluster */
+	u_int32_t	m_mlen;		/* length of data in an mbuf */
+	u_int32_t	m_mhlen;	/* length of data in a header mbuf */
+	u_int32_t	m_bigclusters;	/* clusters obtained from page pool */
+	u_int32_t	m_bigclfree;	/* free clusters */
+	u_int32_t	m_bigmclbytes;	/* length of an mbuf cluster */
 };
 
 /* Compatibillity with 10.3 */
 struct ombstat {
-	u_long	m_mbufs;	/* mbufs obtained from page pool */
-	u_long	m_clusters;	/* clusters obtained from page pool */
-	u_long	m_spare;	/* spare field */
-	u_long	m_clfree;	/* free clusters */
-	u_long	m_drops;	/* times failed to find space */
-	u_long	m_wait;		/* times waited for space */
-	u_long	m_drain;	/* times drained protocols for space */
-	u_short	m_mtypes[256];	/* type specific mbuf allocations */
-	u_long	m_mcfail;	/* times m_copym failed */
-	u_long	m_mpfail;	/* times m_pullup failed */
-	u_long	m_msize;	/* length of an mbuf */
-	u_long	m_mclbytes;	/* length of an mbuf cluster */
-	u_long	m_minclsize;	/* min length of data to allocate a cluster */
-	u_long	m_mlen;		/* length of data in an mbuf */
-	u_long	m_mhlen;	/* length of data in a header mbuf */
+	u_int32_t	m_mbufs;	/* mbufs obtained from page pool */
+	u_int32_t	m_clusters;	/* clusters obtained from page pool */
+	u_int32_t	m_spare;	/* spare field */
+	u_int32_t	m_clfree;	/* free clusters */
+	u_int32_t	m_drops;	/* times failed to find space */
+	u_int32_t	m_wait;		/* times waited for space */
+	u_int32_t	m_drain;	/* times drained protocols for space */
+	u_short		m_mtypes[256];	/* type specific mbuf allocations */
+	u_int32_t	m_mcfail;	/* times m_copym failed */
+	u_int32_t	m_mpfail;	/* times m_pullup failed */
+	u_int32_t	m_msize;	/* length of an mbuf */
+	u_int32_t	m_mclbytes;	/* length of an mbuf cluster */
+	u_int32_t	m_minclsize;	/* min length of data to allocate a cluster */
+	u_int32_t	m_mlen;		/* length of data in an mbuf */
+	u_int32_t	m_mhlen;	/* length of data in a header mbuf */
 };
-#ifdef KERNEL_PRIVATE
 
 /*
- * pkthdr.aux type tags.
+ * mbuf class statistics.
  */
-struct mauxtag {
-	int af;
-	int type;
-};
+#define	MAX_MBUF_CNAME	15
+
+typedef struct mb_class_stat {
+	char		mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */
+	u_int32_t	mbcl_size;	/* buffer size */
+	u_int32_t	mbcl_total;	/* # of buffers created */
+	u_int32_t	mbcl_active;	/* # of active buffers */
+	u_int32_t	mbcl_infree;	/* # of available buffers */
+	u_int32_t	mbcl_slab_cnt;	/* # of available slabs */
+	u_int64_t	mbcl_alloc_cnt;	/* # of times alloc is called */
+	u_int64_t	mbcl_free_cnt;	/* # of times free is called */
+	u_int64_t	mbcl_notified;	/* # of notified wakeups */
+	u_int64_t	mbcl_purge_cnt;	/* # of purges so far */
+	u_int64_t	mbcl_fail_cnt;	/* # of allocation failures */
+	u_int32_t	mbcl_ctotal;	/* total only for this class */
+	/*
+	 * Cache layer statistics
+	 */
+	u_int32_t	mbcl_mc_state;	/* cache state (see below) */
+	u_int32_t	mbcl_mc_cached;	/* # of cached buffers */
+	u_int32_t	mbcl_mc_waiter_cnt;  /* # waiters on the cache */
+	u_int32_t	mbcl_mc_wretry_cnt;  /* # of wait retries */
+	u_int32_t	mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */
+	u_int64_t	mbcl_reserved[4];    /* for future use */
+} mb_class_stat_t;
+
+#define	MCS_DISABLED	0	/* cache is permanently disabled */
+#define	MCS_ONLINE	1	/* cache is online */
+#define	MCS_PURGING	2	/* cache is being purged */
+#define	MCS_OFFLINE	3	/* cache is offline (resizing) */
+
+typedef struct mb_stat {
+	u_int32_t	mbs_cnt;	/* number of classes */
+	mb_class_stat_t	mbs_class[1];	/* class array */
+} mb_stat_t;
+
+#ifdef KERNEL_PRIVATE
 
 #ifdef	KERNEL
 extern union 	mcluster *mbutl;	/* virtual address of mclusters */
 extern union 	mcluster *embutl;	/* ending virtual address of mclusters */
-extern short 	*mclrefcnt;		/* cluster reference counts */
-extern int 	*mcl_paddr;		/* physical addresses of clusters */
 extern struct 	mbstat mbstat;		/* statistics */
 extern int 	nmbclusters;		/* number of mapped clusters */
-extern union 	mcluster *mclfree;	/* free mapped cluster list */
+extern int	njcl;			/* # of clusters for jumbo sizes */
+extern int	njclbytes;		/* size of a jumbo cluster */
 extern int	max_linkhdr;		/* largest link-level header */
 extern int	max_protohdr;		/* largest protocol header */
 extern int	max_hdr;		/* largest link+protocol header */
 extern int	max_datalen;		/* MHLEN - max_hdr */
 
 __BEGIN_DECLS
+/* Not exported */
+__private_extern__ void mbinit(void);
+__private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t,
+    void (*)(caddr_t , u_int, caddr_t), u_int, caddr_t, int);
+__private_extern__ caddr_t m_bigalloc(int);
+__private_extern__ void m_bigfree(caddr_t, u_int, caddr_t);
+__private_extern__ struct mbuf *m_mbigget(struct mbuf *, int);
+__private_extern__ caddr_t m_16kalloc(int);
+__private_extern__ void m_16kfree(caddr_t, u_int, caddr_t);
+__private_extern__ struct mbuf *m_m16kget(struct mbuf *, int);
+
+/* Exported */
 struct	mbuf *m_copym(struct mbuf *, int, int, int);
 struct	mbuf *m_split(struct mbuf *, int, int);
 struct	mbuf *m_free(struct mbuf *);
@@ -490,10 +528,6 @@ int m_freem_list(struct mbuf *);
 struct	mbuf *m_devget(char *, int, int, struct ifnet *, void (*)(const void *, void *, size_t));
 char   *mcl_to_paddr(char *);
 struct mbuf *m_pulldown(struct mbuf*, int, int, int*);
-struct mbuf *m_aux_add(struct mbuf *, int, int);
-struct mbuf *m_aux_find(struct mbuf *, int, int);
-void m_aux_delete(struct mbuf *, struct mbuf *);
-struct mbuf *m_aux_copy(struct mbuf *, struct mbuf *);
 
 struct mbuf *m_mclget(struct mbuf *, int);
 caddr_t m_mclalloc(int);
@@ -512,6 +546,7 @@ union mcluster *m_cltom(int );
 int m_trailingspace(struct mbuf *);
 int m_leadingspace(struct mbuf *);
 
+struct mbuf *m_normalize(struct mbuf *m);
 void m_mchtype(struct mbuf *m, int t);
 void m_mcheck(struct mbuf*);
 
@@ -562,7 +597,12 @@ enum {
 	KERNEL_TAG_TYPE_DUMMYNET		= 1,
 	KERNEL_TAG_TYPE_DIVERT			= 2,
 	KERNEL_TAG_TYPE_IPFORWARD		= 3,
-	KERNEL_TAG_TYPE_IPFILT			= 4
+	KERNEL_TAG_TYPE_IPFILT			= 4,
+	KERNEL_TAG_TYPE_MACLABEL		= 5,
+	KERNEL_TAG_TYPE_MAC_POLICY_LABEL	= 6,
+	KERNEL_TAG_TYPE_ENCAP			= 8,
+	KERNEL_TAG_TYPE_INET6			= 9,
+	KERNEL_TAG_TYPE_IPSEC			= 10
 };
 
 /*
diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h
new file mode 100644
index 000000000..caa625031
--- /dev/null
+++ b/bsd/sys/mcache.h
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _SYS_MCACHE_H
+#define	_SYS_MCACHE_H
+
+#ifdef KERNEL_PRIVATE
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <mach/boolean.h>
+#include <kern/locks.h>
+
+#ifdef ASSERT
+#undef ASSERT
+#endif
+
+#ifdef VERIFY
+#undef VERIFY
+#endif
+
+/*
+ * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build.
+ */
+#define	VERIFY(EX)	((void)((EX) || assfail(#EX, __FILE__, __LINE__)))
+#if DEBUG
+#define	ASSERT(EX)	VERIFY(EX)
+#else
+#define	ASSERT(EX)	((void)0)
+#endif
+
+#if defined(__ppc__)
+#define	CPU_CACHE_SIZE	128
+#elif defined(__arm__)
+#define	CPU_CACHE_SIZE	32
+#else
+#define	CPU_CACHE_SIZE	64
+#endif
+
+#ifndef IS_P2ALIGNED
+#define	IS_P2ALIGNED(v, a) \
+	((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
+#endif /* IS_P2ALIGNED */
+
+#ifndef P2ROUNDUP
+#define	P2ROUNDUP(x, align) \
+	(-(-((uintptr_t)(x)) & -(align)))
+#endif /* P2ROUNDUP */
+
+#ifndef P2ROUNDDOWN
+#define	P2ROUNDDOWN(x, align) \
+	(((uintptr_t)(x)) & ~((uintptr_t)(align) - 1))
+#endif /* P2ROUNDDOWN */
+
+#define	MCACHE_FREE_PATTERN		0xdeadbeefdeadbeefULL
+#define	MCACHE_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
+
+/*
+ * mcache allocation request flags.
+ *
+ * MCR_NOSLEEP and MCR_FAILOK are mutually exclusive.  The latter is used
+ * by the mbuf allocator to handle the implementation of several caches that
+ * involve multiple layers of mcache.  It implies a best effort blocking
+ * allocation request; if the request cannot be satisfied, the caller will
+ * be blocked until further notice, similar to MCR_SLEEP, except that upon
+ * a wake up it will return immediately to the caller regardless of whether
+ * the request can been fulfilled.
+ *
+ * MCR_TRYHARD implies a non-blocking allocation request, regardless of
+ * whether MCR_NOSLEEP is set.  It informs the allocator that the request
+ * should not cause the calling thread to block, and that it must have
+ * exhausted all possible schemes to fulfill the request, including doing
+ * reclaims and/or purges, before returning to the caller.
+ *
+ * Regular mcache clients should only use MCR_SLEEP or MCR_NOSLEEP.
+ */
+#define	MCR_SLEEP	0x0000		/* same as M_WAITOK */
+#define	MCR_NOSLEEP	0x0001		/* same as M_NOWAIT */
+#define	MCR_FAILOK	0x0100		/* private, for internal use only */
+#define	MCR_TRYHARD	0x0200		/* private, for internal use only */
+#define	MCR_USR1	0x1000		/* private, for internal use only */
+
+#define	MCR_NONBLOCKING	(MCR_NOSLEEP | MCR_FAILOK | MCR_TRYHARD)
+
+/*
+ * Generic one-way linked list element structure.  This is used to handle
+ * mcache_alloc_ext() requests in order to chain the allocated objects
+ * together before returning them to the caller.
+ */
+typedef struct mcache_obj {
+	struct mcache_obj	*obj_next;
+} mcache_obj_t;
+
+typedef struct mcache_bkt {
+	void		*bkt_next;	/* next bucket in list */
+	void		*bkt_obj[1];	/* one or more objects */
+} mcache_bkt_t;
+
+typedef struct mcache_bktlist {
+	mcache_bkt_t	*bl_list;	/* bucket list */
+	u_int32_t	bl_total;	/* number of buckets */
+	u_int32_t	bl_min;		/* min since last update */
+	u_int32_t	bl_reaplimit;	/* max reapable buckets */
+	u_int64_t	bl_alloc;	/* allocations from this list */
+} mcache_bktlist_t;
+
+typedef struct mcache_bkttype {
+	int		bt_bktsize;	/* bucket size (number of elements) */
+	size_t		bt_minbuf;	/* all smaller buffers qualify */
+	size_t		bt_maxbuf;	/* no larger bfufers qualify */
+	struct mcache	*bt_cache;	/* bucket cache */
+} mcache_bkttype_t;
+
+typedef struct mcache_cpu {
+	decl_lck_mtx_data(, cc_lock);
+	mcache_bkt_t	*cc_filled;	/* the currently filled bucket */
+	mcache_bkt_t	*cc_pfilled;	/* the previously filled bucket */
+	u_int64_t	cc_alloc;	/* allocations from this cpu */
+	u_int64_t	cc_free;	/* frees to this cpu */
+	int		cc_objs;	/* number of objects in filled bkt */
+	int		cc_pobjs;	/* number of objects in previous bkt */
+	int		cc_bktsize;	/* number of elements in a full bkt */
+} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mcache_cpu_t;
+
+typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***,
+    unsigned int, int);
+typedef void (*mcache_freefn_t)(void *, mcache_obj_t *, boolean_t);
+typedef void (*mcache_auditfn_t)(void *, mcache_obj_t *, boolean_t);
+typedef void (*mcache_notifyfn_t)(void *, u_int32_t);
+
+typedef struct mcache {
+	/*
+	 * Cache properties
+	 */
+	LIST_ENTRY(mcache) mc_list;	/* cache linkage */
+	char		mc_name[32];	/* cache name */
+	struct zone	*mc_slab_zone;	/* backend zone allocator */
+	mcache_allocfn_t mc_slab_alloc;	/* slab layer allocate callback */
+	mcache_freefn_t	mc_slab_free;	/* slab layer free callback */
+	mcache_auditfn_t mc_slab_audit;	/* slab layer audit callback */
+	mcache_notifyfn_t mc_slab_notify; /* slab layer notify callback */
+	void		*mc_private;	/* opaque arg to callbacks */
+	size_t		mc_bufsize;	/* object size */
+	size_t		mc_align;	/* object alignment */
+	u_int32_t	mc_flags;	/* cache creation flags */
+	u_int32_t	mc_purge_cnt;	/* # of purges requested by slab */
+	u_int32_t	mc_enable_cnt;	/* # of reenables due to purges */
+	u_int32_t	mc_waiter_cnt;	/* # of slab layer waiters */
+	u_int32_t	mc_wretry_cnt;	/* # of wait retries */
+	u_int32_t	mc_nwretry_cnt;	/* # of no-wait retry attempts */
+	u_int32_t	mc_nwfail_cnt;	/* # of no-wait retries that failed */
+	decl_lck_mtx_data(, mc_sync_lock); /* protects purges and reenables */
+	lck_attr_t	*mc_sync_lock_attr;
+	lck_grp_t	*mc_sync_lock_grp;
+	lck_grp_attr_t	*mc_sync_lock_grp_attr;
+	/*
+	 * Keep CPU and buckets layers lock statistics separate.
+	 */
+	lck_attr_t	*mc_cpu_lock_attr;
+	lck_grp_t	*mc_cpu_lock_grp;
+	lck_grp_attr_t	*mc_cpu_lock_grp_attr;
+
+	/*
+	 * Bucket layer common to all CPUs
+	 */
+	decl_lck_mtx_data(, mc_bkt_lock);
+	lck_attr_t	*mc_bkt_lock_attr;
+	lck_grp_t	*mc_bkt_lock_grp;
+	lck_grp_attr_t  *mc_bkt_lock_grp_attr;
+	mcache_bkttype_t *cache_bkttype;	/* bucket type */
+	mcache_bktlist_t mc_full;		/* full buckets */
+	mcache_bktlist_t mc_empty;		/* empty buckets */
+	size_t		mc_chunksize;		/* bufsize + alignment */
+	u_int32_t	mc_bkt_contention;	/* lock contention count */
+	u_int32_t	mc_bkt_contention_prev;	/* previous snapshot */
+
+	/*
+	 * Per-CPU layer, aligned at cache line boundary
+	 */
+	mcache_cpu_t	mc_cpu[1];
+} mcache_t;
+
+#define	MCACHE_ALIGN	8	/* default guaranteed alignment */
+
+/* Valid values for mc_flags */
+#define	MCF_VERIFY	0x00000001	/* enable verification */
+#define	MCF_AUDIT	0x00000002	/* enable transaction auditing */
+#define	MCF_NOCPUCACHE	0x00000010	/* disable CPU layer caching */
+
+#define	MCF_DEBUG	(MCF_VERIFY | MCF_AUDIT)
+#define	MCF_FLAGS_MASK	(MCF_DEBUG | MCF_NOCPUCACHE)
+
+/* Valid values for notify callback */
+#define	MCN_RETRYALLOC	0x00000001	/* Allocation should be retried */
+
+#define	MCACHE_STACK_DEPTH 16
+
+typedef struct mcache_audit {
+	struct mcache_audit *mca_next;	/* next audit struct */
+	void		*mca_addr;	/* address of buffer */
+	mcache_t	*mca_cache;	/* parent cache of the buffer */
+	struct thread	*mca_thread;	/* thread doing transaction */
+	struct thread	*mca_pthread;	/* previous transaction thread */
+	size_t		mca_contents_size; /* size of contents */
+	void		*mca_contents;	/* contents at last free */
+	uint16_t	mca_depth;	/* pc stack depth */
+	uint16_t	mca_pdepth;	/* previous transaction pc stack */
+	void		*mca_stack[MCACHE_STACK_DEPTH];
+	void		*mca_pstack[MCACHE_STACK_DEPTH];
+	void		*mca_uptr;	/* user-specific pointer */
+	uint32_t	mca_uflags;	/* user-specific flags */
+} mcache_audit_t;
+
+__private_extern__ int assfail(const char *, const char *, int);
+__private_extern__ void mcache_init(void);
+__private_extern__ unsigned int mcache_getflags(void);
+__private_extern__ mcache_t *mcache_create(const char *, size_t,
+    size_t, u_int32_t, int);
+__private_extern__ void *mcache_alloc(mcache_t *, int);
+__private_extern__ void mcache_free(mcache_t *, void *);
+__private_extern__ mcache_t *mcache_create_ext(const char *, size_t,
+    mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t,
+    void *, u_int32_t, int);
+__private_extern__ void mcache_destroy(mcache_t *);
+__private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **,
+    unsigned int, int);
+__private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *);
+__private_extern__ void mcache_reap(void);
+__private_extern__ boolean_t mcache_purge_cache(mcache_t *);
+__private_extern__ void mcache_waiter_inc(mcache_t *);
+__private_extern__ void mcache_waiter_dec(mcache_t *);
+__private_extern__ boolean_t mcache_bkt_isempty(mcache_t *);
+
+__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *);
+__private_extern__ void mcache_set_pattern(u_int64_t, void *, size_t);
+__private_extern__ void *mcache_verify_pattern(u_int64_t, void *, size_t);
+__private_extern__ void *mcache_verify_set_pattern(u_int64_t, u_int64_t,
+    void *, size_t);
+__private_extern__ void mcache_audit_free_verify(mcache_audit_t *,
+    void *, size_t, size_t);
+__private_extern__ void mcache_audit_free_verify_set(mcache_audit_t *,
+    void *, size_t, size_t);
+__private_extern__ char *mcache_dump_mca(mcache_audit_t *);
+__private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t,
+    int64_t, int64_t);
+
+__private_extern__ mcache_t *mcache_audit_cache;
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif /* KERNEL_PRIVATE */
+
+#endif /* _SYS_MCACHE_H */
diff --git a/bsd/sys/md5.h b/bsd/sys/md5.h
index f825f0aaa..8d581add4 100644
--- a/bsd/sys/md5.h
+++ b/bsd/sys/md5.h
@@ -1,54 +1,41 @@
-/* MD5.H - header file for MD5C.C
- */
-
-/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
-rights reserved.
-
-License to copy and use this software is granted provided that it
-is identified as the "RSA Data Security, Inc. MD5 Message-Digest
-Algorithm" in all material mentioning or referencing this software
-or this function.
-
-License is also granted to make and use derivative works provided
-that such works are identified as "derived from the RSA Data
-Security, Inc. MD5 Message-Digest Algorithm" in all material
-mentioning or referencing the derived work.
-
-RSA Data Security, Inc. makes no representations concerning either
-the merchantability of this software or the suitability of this
-software for any particular purpose. It is provided "as is"
-without express or implied warranty of any kind.
-
-These notices must be retained in any copies of any part of this
-documentation and/or software.
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_MD5_H_
 #define _SYS_MD5_H_
 
-#include <sys/appleapiopts.h>
-
-#if !defined(KERNEL) || defined(__APPLE_API_PRIVATE)
-/* MD5 context. */
-typedef struct MD5Context {
-  u_int32_t state[4];	/* state (ABCD) */
-  u_int32_t count[2];	/* number of bits, modulo 2^64 (lsb first) */
-  unsigned char buffer[64];	/* input buffer */
-} MD5_CTX;
+/*
+ * Warn developers that this header file is kept for legacy reasons.
+ * The supported interface resides in <libkern/crypto/md5.h>.
+ */
+#warning \
+    "<sys/md5.h> is a legacy header file; use <libkern/crypto/md5.h> instead."
 
-#include <sys/cdefs.h>
+#include <libkern/crypto/md5.h>
 
-__BEGIN_DECLS
-void   MD5Init (MD5_CTX *);
-void   MD5Update (MD5_CTX *, const unsigned char *, unsigned int);
-void   MD5Pad (MD5_CTX *);
-void   MD5Final (unsigned char [16], MD5_CTX *);
-char * MD5End(MD5_CTX *, char *);
-char * MD5File(const char *, char *);
-char * MD5Data(const unsigned char *, unsigned int, char *);
-#ifdef KERNEL
-void MD5Transform(u_int32_t [4], const unsigned char [64]);
-#endif
-__END_DECLS
-#endif /* !KERNEL || __APPLE_API_PRIVATE */
 #endif /* _SYS_MD5_H_ */
diff --git a/bsd/sys/mman.h b/bsd/sys/mman.h
index aeaab5a7f..f3eb0578e 100644
--- a/bsd/sys/mman.h
+++ b/bsd/sys/mman.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -109,21 +115,22 @@ typedef __darwin_size_t	size_t;
  */
 #define	MAP_SHARED	0x0001		/* [MF|SHM] share changes */
 #define	MAP_PRIVATE	0x0002		/* [MF|SHM] changes are private */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	MAP_COPY	MAP_PRIVATE	/* Obsolete */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Other flags
  */
 #define	MAP_FIXED	 0x0010	/* [MF|SHM] interpret addr exactly */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	MAP_RENAME	 0x0020	/* Sun: rename private pages to file */
 #define	MAP_NORESERVE	 0x0040	/* Sun: don't reserve needed swap area */
 #define	MAP_RESERVED0080 0x0080	/* previously unimplemented MAP_INHERIT */
 #define	MAP_NOEXTEND	 0x0100	/* for MAP_FILE, don't change file size */
 #define	MAP_HASSEMAPHORE 0x0200	/* region may contain semaphores */
-#endif	/* !_POSIX_C_SOURCE */
+#define MAP_NOCACHE	 0x0400 /* don't cache pages for this mapping */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Process memory locking
@@ -143,7 +150,7 @@ typedef __darwin_size_t	size_t;
 #define MS_INVALIDATE	0x0002	/* [MF|SIO] invalidate all cached data */
 #define	MS_SYNC		0x0010	/* [MF|SIO] msync synchronously */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define MS_KILLPAGES    0x0004  /* invalidate pages, leave mapped */
 #define MS_DEACTIVATE   0x0008  /* deactivate pages, leave mapped */
 
@@ -152,7 +159,7 @@ typedef __darwin_size_t	size_t;
  */
 #define	MAP_FILE	0x0000	/* map from file (default) */
 #define	MAP_ANON	0x1000	/* allocated from memory, swap space */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 /*
@@ -164,7 +171,7 @@ typedef __darwin_size_t	size_t;
 #define	POSIX_MADV_WILLNEED	3	/* [MC1] will need these pages */
 #define	POSIX_MADV_DONTNEED	4	/* [MC1] dont need these pages */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	MADV_NORMAL		POSIX_MADV_NORMAL
 #define	MADV_RANDOM		POSIX_MADV_RANDOM
 #define	MADV_SEQUENTIAL		POSIX_MADV_SEQUENTIAL
@@ -180,7 +187,7 @@ typedef __darwin_size_t	size_t;
 #define	MINCORE_MODIFIED	 0x4	 /* Page has been modified by us */
 #define	MINCORE_REFERENCED_OTHER 0x8	 /* Page has been referenced */
 #define	MINCORE_MODIFIED_OTHER	0x10	 /* Page has been modified */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 #ifndef KERNEL
@@ -199,7 +206,7 @@ void *	mmap(void *, size_t, int, int, int, off_t) __DARWIN_ALIAS(mmap);
 /* [MPR] */
 int	mprotect(void *, size_t, int) __DARWIN_ALIAS(mprotect);
 /* [MF|SIO] */
-int	msync(void *, size_t, int) __DARWIN_ALIAS(msync);
+int	msync(void *, size_t, int) __DARWIN_ALIAS_C(msync);
 /* [MR] */
 int	munlock(const void *, size_t);
 /* [MC3]*/
@@ -210,7 +217,7 @@ int	shm_unlink(const char *);
 /* [ADV] */
 int	posix_madvise(void *, size_t, int);
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int	madvise(void *, size_t, int);
 int	mincore(const void *, size_t, char *);
 int	minherit(void *, size_t, int);
@@ -219,7 +226,8 @@ __END_DECLS
 
 #else	/* KERNEL */
 
-void pshm_cache_init(void);	/* for bsd_init() */
+void pshm_cache_init(void) __attribute__((section("__TEXT, initcode")));	/* for bsd_init() */
+void pshm_lock_init(void);
 
 /*
  * XXX routine exported by posix_shm.c, but never used there, only used in
@@ -231,7 +239,8 @@ int pshm_mmap(struct proc *p, struct mmap_args *uap, user_addr_t *retval,
 		struct fileproc *fp, off_t pageoff);
 /* Really need to overhaul struct fileops to avoid this... */
 struct pshmnode;
-int pshm_stat(struct pshmnode *pnode, struct stat *sb);
+struct stat;
+int pshm_stat(struct pshmnode *pnode, void *ub, int isstat64);
 struct fileproc;
 int pshm_truncate(struct proc *p, struct fileproc *fp, int fd, off_t length, register_t *retval);
 
diff --git a/bsd/sys/mount.h b/bsd/sys/mount.h
index e2078d904..80f813839 100644
--- a/bsd/sys/mount.h
+++ b/bsd/sys/mount.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,13 @@
  *
  *	@(#)mount.h	8.21 (Berkeley) 5/20/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+
 
 #ifndef _SYS_MOUNT_H_
 #define	_SYS_MOUNT_H_
@@ -77,8 +90,36 @@ typedef struct fsid { int32_t val[2]; } fsid_t;	/* file system id type */
  */
 
 #define	MFSNAMELEN	15	/* length of fs type name, not inc. null */
+#define	MFSTYPENAMELEN	16	/* length of fs type name including null */
 #define	MNAMELEN	90	/* length of buffer for returned name */
 
+#define __DARWIN_STRUCT_STATFS64 { \
+	uint32_t	f_bsize;	/* fundamental file system block size */ \
+	int32_t		f_iosize;	/* optimal transfer block size */ \
+	uint64_t	f_blocks;	/* total data blocks in file system */ \
+	uint64_t	f_bfree;	/* free blocks in fs */ \
+	uint64_t	f_bavail;	/* free blocks avail to non-superuser */ \
+	uint64_t	f_files;	/* total file nodes in file system */ \
+	uint64_t	f_ffree;	/* free file nodes in fs */ \
+	fsid_t		f_fsid;		/* file system id */ \
+	uid_t		f_owner;	/* user that mounted the filesystem */ \
+	uint32_t	f_type;		/* type of filesystem */ \
+	uint32_t	f_flags;	/* copy of mount exported flags */ \
+	uint32_t	f_fssubtype;	/* fs sub-type (flavor) */ \
+	char		f_fstypename[MFSTYPENAMELEN];	/* fs type name */ \
+	char		f_mntonname[MAXPATHLEN];	/* directory on which mounted */ \
+	char		f_mntfromname[MAXPATHLEN];	/* mounted filesystem */ \
+	uint32_t	f_reserved[8];	/* For future use */ \
+}
+
+struct statfs64 __DARWIN_STRUCT_STATFS64;
+
+#if __DARWIN_64_BIT_INO_T
+
+struct statfs __DARWIN_STRUCT_STATFS64;
+
+#else /* !__DARWIN_64_BIT_INO_T */
+
 /*
  * LP64 - WARNING - must be kept in sync with struct user_statfs in mount_internal.h.
  */
@@ -96,7 +137,7 @@ struct statfs {
 	uid_t	f_owner;		/* user that mounted the filesystem */
 	short	f_reserved1;	/* spare for later */
 	short	f_type;			/* type of filesystem */
-    long	f_flags;		/* copy of mount exported flags */
+	long	f_flags;		/* copy of mount exported flags */
 	long    f_reserved2[2];	/* reserved for future use */
 	char	f_fstypename[MFSNAMELEN]; /* fs type name */
 	char	f_mntonname[MNAMELEN];	/* directory on which mounted */
@@ -110,8 +151,7 @@ struct statfs {
 #endif
 };
 
-
-#define	MFSTYPENAMELEN	16	/* length of fs type name including null */
+#endif /* __DARWIN_64_BIT_INO_T */
 
 #pragma pack(4)
 
@@ -136,6 +176,13 @@ struct vfsstatfs {
 
 #pragma pack()
 
+#ifdef KERNEL
+/*
+ * Kernel level support for the VFS_GETATTR(), VFS_SETATTR() for use in
+ * implementation of filesystem KEXTs, and by the vfs_getattr() and
+ * vfs_setattr() KPIs.
+ */
+
 #define VFSATTR_INIT(s)			((s)->f_supported = (s)->f_active = 0LL)
 #define VFSATTR_SET_SUPPORTED(s, a)	((s)->f_supported |= VFSATTR_ ## a)
 #define VFSATTR_IS_SUPPORTED(s, a)	((s)->f_supported & VFSATTR_ ## a)
@@ -171,7 +218,7 @@ struct vfsstatfs {
 #define VFSATTR_f_carbon_fsid		(1LL<< 23)
 
 /*
- * New VFS_STAT argument structure.
+ * Argument structure.
  */
 #pragma pack(4)
 
@@ -213,6 +260,8 @@ struct vfs_attr {
 
 #pragma pack()
 
+#endif	/* KERNEL */
+
 /*
  * User specifiable flags.
  *
@@ -225,29 +274,38 @@ struct vfs_attr {
 #define	MNT_NODEV	0x00000010	/* don't interpret special files */
 #define	MNT_UNION	0x00000020	/* union with underlying filesystem */
 #define	MNT_ASYNC	0x00000040	/* file system written asynchronously */
-#define MNT_DONTBROWSE	0x00100000	/* file system is not appropriate path to user data */
-#define MNT_IGNORE_OWNERSHIP 0x00200000 /* VFS will ignore ownership information on filesystem
-					 * objects */
-#define MNT_AUTOMOUNTED 0x00400000	/* filesystem was mounted by automounter */
-#define MNT_JOURNALED   0x00800000  /* filesystem is journaled */
-#define MNT_NOUSERXATTR	0x01000000	/* Don't allow user extended attributes */
-#define MNT_DEFWRITE	0x02000000	/* filesystem should defer writes */
-
-/* backwards compatibility only */
-#define MNT_UNKNOWNPERMISSIONS MNT_IGNORE_OWNERSHIP
 
 /*
  * NFS export related mount flags.
  */
 #define	MNT_EXPORTED	0x00000100	/* file system is exported */
 
+/*
+ * MAC labeled / "quarantined" flag
+ */
+#define MNT_QUARANTINE	0x00000400	/* file system is quarantined */
+
 /*
  * Flags set by internal operations.
  */
 #define	MNT_LOCAL	0x00001000	/* filesystem is stored locally */
 #define	MNT_QUOTA	0x00002000	/* quotas are enabled on filesystem */
 #define	MNT_ROOTFS	0x00004000	/* identifies the root filesystem */
-#define	MNT_DOVOLFS	0x00008000	/* FS supports volfs */
+#define	MNT_DOVOLFS	0x00008000	/* FS supports volfs (deprecated flag in Mac OS X 10.5) */
+
+
+#define MNT_DONTBROWSE	0x00100000	/* file system is not appropriate path to user data */
+#define MNT_IGNORE_OWNERSHIP 0x00200000 /* VFS will ignore ownership information on filesystem objects */
+#define MNT_AUTOMOUNTED 0x00400000	/* filesystem was mounted by automounter */
+#define MNT_JOURNALED   0x00800000  	/* filesystem is journaled */
+#define MNT_NOUSERXATTR	0x01000000	/* Don't allow user extended attributes */
+#define MNT_DEFWRITE	0x02000000	/* filesystem should defer writes */
+#define MNT_MULTILABEL	0x04000000	/* MAC support for individual labels */
+#define MNT_NOATIME	0x10000000	/* disable update of file access time */
+
+/* backwards compatibility only */
+#define MNT_UNKNOWNPERMISSIONS MNT_IGNORE_OWNERSHIP
+
 
 /*
  * XXX I think that this could now become (~(MNT_CMDFLAGS))
@@ -255,11 +313,11 @@ struct vfs_attr {
  */
 #define	MNT_VISFLAGMASK	(MNT_RDONLY	| MNT_SYNCHRONOUS | MNT_NOEXEC	| \
 			MNT_NOSUID	| MNT_NODEV	| MNT_UNION	| \
-			MNT_ASYNC	| MNT_EXPORTED	| \
-			MNT_LOCAL	|		MNT_QUOTA	| \
+			MNT_ASYNC	| MNT_EXPORTED	| MNT_QUARANTINE | \
+			MNT_LOCAL	| MNT_QUOTA | \
 			MNT_ROOTFS	| MNT_DOVOLFS	| MNT_DONTBROWSE | \
-			MNT_UNKNOWNPERMISSIONS | MNT_AUTOMOUNTED | MNT_JOURNALED | \
-			MNT_DEFWRITE)
+			MNT_IGNORE_OWNERSHIP | MNT_AUTOMOUNTED | MNT_JOURNALED | \
+			MNT_NOUSERXATTR | MNT_DEFWRITE	| MNT_MULTILABEL | MNT_NOATIME)
 /*
  * External filesystem command modifier flags.
  * Unmount can use the MNT_FORCE flag.
@@ -330,7 +388,6 @@ struct vfsidctl {
 #define VFS_CTL_VERS1	0x01
 
 #ifdef KERNEL
-// LP64todo - should this move?
 
 /* LP64 version of vfsconf.  all pointers 
  * grow when we're dealing with a 64-bit process.
@@ -402,19 +459,26 @@ struct vfsioattr {
 	u_int32_t	io_maxsegreadsize;	/* Max. segment read size  */
 	u_int32_t	io_maxsegwritesize;	/* Max. segment write size */
 	u_int32_t	io_devblocksize;	/* the underlying device block size */
-	void *		io_reserved[3];		/* extended attribute information */
+	u_int32_t	io_flags;			/* flags for underlying device */
+	void *		io_reserved[2];		/* extended attribute information */
 };
 
+#define VFS_IOATTR_FLAGS_FUA	0x01	/* Write-through cache supported */
 
 /*
  * Filesystem Registration information
  */
-
-#define VFS_TBLTHREADSAFE	0x01
-#define VFS_TBLFSNODELOCK	0x02
-#define VFS_TBLNOTYPENUM	0x08
-#define VFS_TBLLOCALVOL		0x10
-#define VFS_TBL64BITREADY	0x20
+#define VFS_TBLTHREADSAFE		0x0001
+#define VFS_TBLFSNODELOCK		0x0002
+#define VFS_TBLNOTYPENUM		0x0008
+#define VFS_TBLLOCALVOL			0x0010
+#define VFS_TBL64BITREADY		0x0020
+#define VFS_TBLNATIVEXATTR		0x0040
+#define VFS_TBLDIRLINKS			0x0080
+#define VFS_TBLUNMOUNT_PREFLIGHT	0x0100	/* does a preflight check before unmounting */
+#define VFS_TBLGENERICMNTARGS		0x0200  /* force generic mount args for local fs */
+#define VFS_TBLREADDIR_EXTENDED		0x0400  /* fs supports VNODE_READDIR_EXTENDED */
+#define	VFS_TBLNOMACLABEL		0x1000
 
 struct vfs_fsentry {
 	struct vfsops * vfe_vfsops;	/* vfs operations */
@@ -470,6 +534,8 @@ extern int VFS_START(mount_t, int, vfs_context_t);
 extern int VFS_UNMOUNT(mount_t, int, vfs_context_t);
 extern int VFS_ROOT(mount_t, vnode_t *, vfs_context_t);
 extern int VFS_QUOTACTL(mount_t, int, uid_t, caddr_t, vfs_context_t);
+extern int VFS_GETATTR(mount_t, struct vfs_attr *, vfs_context_t);
+extern int VFS_SETATTR(mount_t, struct vfs_attr *, vfs_context_t);
 extern int VFS_SYNC(mount_t, int, vfs_context_t);
 extern int VFS_VGET(mount_t, ino64_t, vnode_t *, vfs_context_t);
 extern int VFS_FHTOVP(mount_t, int, unsigned char *, vnode_t *, vfs_context_t);
@@ -501,7 +567,18 @@ int	vfs_extendedsecurity(mount_t);
 void	vfs_setextendedsecurity(mount_t);
 void	vfs_clearextendedsecurity(mount_t);
 void	vfs_setlocklocal(mount_t);
+int	vfs_authcache_ttl(mount_t);
+void	vfs_setauthcache_ttl(mount_t, int);
+void	vfs_clearauthcache_ttl(mount_t);
+void    vfs_markdependency(mount_t);
 
+/*
+ * return value from vfs_cachedrights_ttl if
+ * neither MNTK_AUTH_OPAQUE | MNTK_AUTH_CACHE_TTL
+ * is set in mnt_kern_flag.. it indicates
+ * that no TTL is being applied to the vnode rights cache
+ */
+#define CACHED_RIGHT_INFINITE_TTL	~0
 
 
 uint32_t vfs_maxsymlen(mount_t);
@@ -510,7 +587,9 @@ void *	vfs_fsprivate(mount_t);
 void	vfs_setfsprivate(mount_t, void *mntdata);
 
 struct vfsstatfs *	vfs_statfs(mount_t);
-int	vfs_update_vfsstat(mount_t, vfs_context_t);
+#define	VFS_USER_EVENT		0
+#define	VFS_KERNEL_EVENT	1
+int	vfs_update_vfsstat(mount_t, vfs_context_t, int eventtype);
 int	vfs_getattr(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx);
 int	vfs_setattr(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx);
 
@@ -522,13 +601,15 @@ void	vfs_setioattr(mount_t, struct vfsioattr *);
 int 	vfs_64bitready(mount_t);
 
 
+#define LK_NOWAIT 1
 int	vfs_busy(mount_t, int);
 void	vfs_unbusy(mount_t);
 
 void	vfs_getnewfsid(struct mount *);
 mount_t	vfs_getvfs(fsid_t *);
-mount_t	vfs_getvfs_by_mntonname(u_char *);
+mount_t	vfs_getvfs_by_mntonname(char *);
 int	vfs_mountedon(struct vnode *);
+int	vfs_unmountbyfsid(fsid_t *, int, vfs_context_t);
 
 void	vfs_event_signal(fsid_t *, u_int32_t, intptr_t);
 void	vfs_event_init(void);
@@ -541,7 +622,9 @@ __END_DECLS
 /*
  * Generic file handle
  */
-#define	NFS_MAX_FH_SIZE		64
+#define	NFS_MAX_FH_SIZE		NFSV4_MAX_FH_SIZE
+#define	NFSV4_MAX_FH_SIZE	128
+#define	NFSV3_MAX_FH_SIZE	64
 #define	NFSV2_MAX_FH_SIZE	32
 struct fhandle {
 	int		fh_len;				/* length of file handle */
@@ -552,12 +635,16 @@ typedef struct fhandle	fhandle_t;
 
 __BEGIN_DECLS
 int	fhopen(const struct fhandle *, int);
-int	fstatfs(int, struct statfs *);
+int	fstatfs(int, struct statfs *) __DARWIN_INODE64(fstatfs);
+int	fstatfs64(int, struct statfs64 *);
 int	getfh(const char *, fhandle_t *);
-int	getfsstat(struct statfs *, int, int);
-int	getmntinfo(struct statfs **, int);
+int	getfsstat(struct statfs *, int, int) __DARWIN_INODE64(getfsstat);
+int	getfsstat64(struct statfs64 *, int, int);
+int	getmntinfo(struct statfs **, int) __DARWIN_INODE64(getmntinfo);
+int	getmntinfo64(struct statfs64 **, int);
 int	mount(const char *, const char *, int, void *);
-int	statfs(const char *, struct statfs *);
+int	statfs(const char *, struct statfs *) __DARWIN_INODE64(statfs);
+int	statfs64(const char *, struct statfs64 *);
 int	unmount(const char *, int);
 int	getvfsbyname(const char *, struct vfsconf *);
 __END_DECLS
diff --git a/bsd/sys/mount_internal.h b/bsd/sys/mount_internal.h
index ca490eb6e..1ff74ca3e 100644
--- a/bsd/sys/mount_internal.h
+++ b/bsd/sys/mount_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)mount.h	8.21 (Berkeley) 5/20/95
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_MOUNT_INTERNAL_H_
 #define	_SYS_MOUNT_INTERNAL_H_
@@ -73,6 +85,8 @@
 #include <sys/mount.h>
 #include <sys/cdefs.h>
 
+struct label;
+
 /*
  * Structure per mounted file system.  Each mounted file system has an
  * array of operations and an instance record.  The file systems are
@@ -103,7 +117,9 @@ struct mount {
 	u_int32_t	mnt_segwritecnt;	/* Max. segment count for write */
 	u_int32_t	mnt_maxsegreadsize;	/* Max. segment read size  */
 	u_int32_t	mnt_maxsegwritesize;	/* Max. segment write size */
+        u_int32_t	mnt_alignmentmask;	/* Mask of bits that aren't addressable via DMA */
 	u_int32_t	mnt_devblocksize;	/* the underlying device block size */
+	u_int32_t	mnt_ioflags;		/* flags for  underlying device */
 	lck_rw_t	mnt_rwlock;		/* mutex readwrite lock */
         lck_mtx_t	mnt_renamelock;		/* mutex that serializes renames that change shape of tree */
 	vnode_t		mnt_devvp;		/* the device mounted on for local file systems */
@@ -113,7 +129,66 @@ struct mount {
  	/* XXX 3762912 hack to support HFS filesystem 'owner' */
  	uid_t		mnt_fsowner;
  	gid_t		mnt_fsgroup;
+
+	struct label	*mnt_mntlabel;		/* MAC mount label */
+	struct label	*mnt_fslabel;		/* MAC default fs label */
+
+	/*
+	 * cache the rootvp of the last mount point
+	 * in the chain in the mount struct pointed
+	 * to by the vnode sitting in '/'
+	 * this cache is used to shortcircuit the
+	 * mount chain traversal and allows us
+	 * to traverse to the true underlying rootvp
+	 * in 1 easy step inside of 'cache_lookup_path'
+	 *
+	 * make sure to validate against the cached vid
+	 * in case the rootvp gets stolen away since
+	 * we don't take an explicit long term reference
+	 * on it when we mount it
+	 */
+	vnode_t		mnt_realrootvp;
+        int		mnt_realrootvp_vid;
+	/*
+	 * bumped each time a mount or unmount
+	 * occurs... its used to invalidate
+	 * 'mnt_realrootvp' from the cache
+	 */
+        int             mnt_generation;
+        /*
+	 * if 'MNTK_AUTH_CACHE_TIMEOUT' is 
+	 * set, then 'mnt_authcache_ttl' is
+	 * the time-to-live for the per-vnode authentication cache
+	 * on this mount... if zero, no cache is maintained...
+	 * if 'MNTK_AUTH_CACHE_TIMEOUT' isn't set, its the
+	 * time-to-live for the cached lookup right for
+	 * volumes marked 'MNTK_AUTH_OPAQUE'.
+	 */
+	int		mnt_authcache_ttl;
+	/*
+	 * The proc structure pointer and process ID form a
+	 * sufficiently unique duple identifying the process
+	 * hosting this mount point. Set by vfs_markdependency()
+	 * and utilized in new_vnode() to avoid reclaiming vnodes
+	 * with this dependency (radar 5192010).
+	 */
+	pid_t		mnt_dependent_pid;
+	void		*mnt_dependent_process;
+
+	struct timeval	last_normal_IO_timestamp;
 };
+
+/*
+ * default number of seconds to keep cached lookup
+ * rights valid on mounts marked MNTK_AUTH_OPAQUE
+ */
+#define CACHED_LOOKUP_RIGHT_TTL		2
+
+/*
+ * ioflags
+ */
+#define MNT_IOFLAGS_FUA_SUPPORTED	0x00000001
+
   
 /* XXX 3762912 hack to support HFS filesystem 'owner' */
 #define vfs_setowner(_mp, _uid, _gid)	do {(_mp)->mnt_fsowner = (_uid); (_mp)->mnt_fsgroup = (_gid); } while (0)
@@ -133,6 +208,11 @@ extern struct mount * dead_mountp;
  *		because the bits here were broken out from the high bits
  *		of the mount flags.
  */
+#define MNTK_AUTH_CACHE_TTL	0x00008000      /* rights cache has TTL - TTL of 0 disables cache */
+#define	MNTK_PATH_FROM_ID	0x00010000	/* mounted file system supports id-to-path lookups */
+#define	MNTK_UNMOUNT_PREFLIGHT	0x00020000	/* mounted file system wants preflight check during unmount */
+#define	MNTK_NAMED_STREAMS	0x00040000	/* mounted file system supports Named Streams VNOPs */
+#define	MNTK_EXTENDED_ATTRS	0x00080000	/* mounted file system supports Extended Attributes VNOPs */
 #define	MNTK_LOCK_LOCAL		0x00100000	/* advisory locking is done above the VFS itself */
 #define MNTK_VIRTUALDEV 	0x00200000      /* mounted on a virtual device i.e. a disk image */
 #define MNTK_ROOTDEV    	0x00400000      /* this filesystem resides on the same device as the root */
@@ -161,7 +241,9 @@ extern struct mount * dead_mountp;
 /*
  * Generic file handle
  */
-#define	NFS_MAX_FH_SIZE		64
+#define	NFS_MAX_FH_SIZE		NFSV4_MAX_FH_SIZE
+#define	NFSV4_MAX_FH_SIZE	128
+#define	NFSV3_MAX_FH_SIZE	64
 #define	NFSV2_MAX_FH_SIZE	32
 struct fhandle {
 	int		fh_len;				/* length of file handle */
@@ -194,10 +276,14 @@ struct vfstable {
 	int 		vfc_64bitready;	/* The file system is ready for 64bit */
 };
 
+/* vfc_vfsflags: */
 #define VFC_VFSLOCALARGS	0x02
 #define	VFC_VFSGENERICARGS	0x04
 #define	VFC_VFSNATIVEXATTR	0x10
-
+#define	VFC_VFSDIRLINKS		0x20
+#define	VFC_VFSPREFLIGHT	0x40
+#define	VFC_VFSREADDIR_EXTENDED	0x80
+#define	VFC_VFSNOMACLABEL	0x1000
 
 extern int maxvfsconf;		/* highest defined filesystem type */
 extern struct vfstable  *vfsconf;	/* head of list of filesystem types */
@@ -256,6 +342,7 @@ struct user_statfs {
 
 __BEGIN_DECLS
 
+extern int mount_generation;
 extern TAILQ_HEAD(mntlist, mount) mountlist;
 void mount_list_lock(void);
 void mount_list_unlock(void);
@@ -267,6 +354,7 @@ void mount_lock_renames(mount_t);
 void mount_unlock_renames(mount_t);
 void mount_ref(mount_t, int);
 void mount_drop(mount_t, int);
+int  mount_refdrain(mount_t);
 
 /* vfs_rootmountalloc should be kept as a private api */
 errno_t vfs_rootmountalloc(const char *, const char *, mount_t *mpp);
@@ -274,14 +362,15 @@ errno_t	vfs_init_io_attributes(vnode_t, mount_t);
 
 int	vfs_mountroot(void);
 void	vfs_unmountall(void);
-int	safedounmount(struct mount *, int, struct proc *);
-int	dounmount(struct mount *, int, int *, int, struct proc *);
+int	safedounmount(struct mount *, int, vfs_context_t);
+int	dounmount(struct mount *, int, int, vfs_context_t);
 
 /* xnuy internal api */
 void  mount_dropcrossref(mount_t, vnode_t, int);
-int validfsnode(mount_t);
 mount_t mount_lookupby_volfsid(int, int);
 mount_t mount_list_lookupby_fsid(fsid_t *, int, int);
+void mount_list_add(mount_t);
+void mount_list_remove(mount_t);
 int  mount_iterref(mount_t, int);
 int  mount_isdrained(mount_t, int);
 void mount_iterdrop(mount_t);
diff --git a/bsd/sys/msg.h b/bsd/sys/msg.h
index d53ebd98c..7286e0473 100644
--- a/bsd/sys/msg.h
+++ b/bsd/sys/msg.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: msg.h,v 1.4 1994/06/29 06:44:43 cgd Exp $	*/
 
@@ -39,6 +45,12 @@
  *
  * This software is provided ``AS IS'' without any warranties of any kind.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_MSG_H_
 #define _SYS_MSG_H_
@@ -84,9 +96,11 @@ typedef	__darwin_ssize_t	ssize_t;
 
 /* [XSI] Used for the number of messages in the message queue */
 typedef unsigned long		msgqnum_t;
+typedef unsigned long long	user_msgqnum_t;
 
 /* [XSI] Used for the number of bytes allowed in a message queue */
 typedef unsigned long		msglen_t;
+typedef unsigned long long	user_msglen_t;
 
 
 /*
@@ -102,7 +116,8 @@ typedef unsigned long		msglen_t;
  * legacy interface there for binary compatibility only.  Currently, we
  * are only forcing this for programs requesting standards conformance.
  */
-#if defined(__POSIX_C_SOURCE) || defined(kernel) || defined(__LP64__)
+#if __DARWIN_UNIX03 || defined(KERNEL)
+#pragma pack(4)
 /*
  * Structure used internally.
  * 
@@ -115,7 +130,13 @@ typedef unsigned long		msglen_t;
  *
  * NOTES:	Reserved fields are not preserved across IPC_SET/IPC_STAT.
  */
-struct __msqid_ds_new {
+#if (defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE))
+struct msqid_ds
+#else
+#define	msqid_ds	__msqid_ds_new
+struct __msqid_ds_new
+#endif
+{
 	struct __ipc_perm_new	msg_perm; /* [XSI] msg queue permissions */
 	__int32_t	msg_first;	/* RESERVED: kernel use only */
 	__int32_t	msg_last;	/* RESERVED: kernel use only */
@@ -132,12 +153,12 @@ struct __msqid_ds_new {
 	__int32_t	msg_pad3;	/* RESERVED: DO NOT USE */
 	__int32_t	msg_pad4[4];	/* RESERVED: DO NOT USE */
 };
-#define	msqid_ds	__msqid_ds_new
-#else	/* !_POSIX_C_SOURCE */
+#pragma pack()
+#else	/* !__DARWIN_UNIX03 */
 #define	msqid_ds	__msqid_ds_old
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
-#if !defined(__POSIX_C_SOURCE) && !defined(__LP64__)
+#if !__DARWIN_UNIX03
 struct __msqid_ds_old {
 	struct __ipc_perm_old	msg_perm; /* [XSI] msg queue permissions */
 	__int32_t	msg_first;	/* RESERVED: kernel use only */
@@ -155,14 +176,12 @@ struct __msqid_ds_old {
 	__int32_t	msg_pad3;	/* RESERVED: DO NOT USE */
 	__int32_t	msg_pad4[4];	/* RESERVED: DO NOT USE */
 };
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
 #ifdef KERNEL
 #ifdef __APPLE_API_PRIVATE
 #include <machine/types.h>
 
-// LP64todo - should this move?
-
 #if __DARWIN_ALIGN_NATURAL
 #pragma options align=natural
 #endif
@@ -171,9 +190,9 @@ struct user_msqid_ds {
 	struct ipc_perm	msg_perm;	/* [XSI] msg queue permissions */
 	struct msg	*msg_first;	/* first message in the queue */
 	struct msg	*msg_last;	/* last message in the queue */
-	msglen_t	msg_cbytes;	/* # of bytes on the queue */
-	msgqnum_t	msg_qnum;	/* [XSI] number of msgs on the queue */
-	msglen_t	msg_qbytes;	/* [XSI] max bytes on the queue */
+	user_msglen_t	msg_cbytes;	/* # of bytes on the queue */
+	user_msgqnum_t	msg_qnum;	/* [XSI] number of msgs on the queue */
+	user_msglen_t	msg_qbytes;	/* [XSI] max bytes on the queue */
 	pid_t		msg_lspid;	/* [XSI] pid of last msgsnd() */
 	pid_t		msg_lrpid;	/* [XSI] pid of last msgrcv() */
 	user_time_t	msg_stime;	/* [XSI] time of last msgsnd() */
@@ -189,11 +208,21 @@ struct user_msqid_ds {
 #pragma options align=reset
 #endif
 
+struct label;
+
+/*
+ * Kernel wrapper for the user-level structure
+ */
+struct msqid_kernel {
+	struct	user_msqid_ds u;
+	struct	label *label;	/* MAC framework label */
+};
+
 #endif	/* __APPLE_API_PRIVATE */
 #endif	/* KERNEL */
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifdef __APPLE_API_UNSTABLE
 /* XXX kernel only; protect with macro later */
 
@@ -204,6 +233,7 @@ struct msg {
     					/* 0 -> free header */
 	unsigned short	msg_ts;		/* size of this message */
 	short		msg_spot;	/* location of msg start in buffer */
+	struct label	*label;		/* MAC label */
 };
 
 /*
@@ -283,26 +313,28 @@ struct msgmap {
 extern char *msgpool;		/* MSGMAX byte long msg buffer pool */
 extern struct msgmap *msgmaps;	/* MSGSEG msgmap structures */
 extern struct msg *msghdrs;	/* MSGTQL msg headers */
-extern struct user_msqid_ds	*msqids; /* MSGMNI user_msqid_ds struct's */
+extern struct msqid_kernel *msqids;	/* MSGMNI user_msqid_ds struct's */
 
 #define MSG_LOCKED	01000	/* Is this msqid_ds locked? */
 
 #endif	/* KERNEL */
 #endif	/* __APPLE_API_UNSTABLE */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #ifndef KERNEL
 
 __BEGIN_DECLS
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int msgsys(int, ...);
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 int msgctl(int, int, struct msqid_ds *) __DARWIN_ALIAS(msgctl);
 int msgget(key_t, int);
-ssize_t msgrcv(int, void *, size_t, long, int);
-int msgsnd(int, const void *, size_t, int);
+ssize_t msgrcv(int, void *, size_t, long, int) __DARWIN_ALIAS_C(msgrcv);
+int msgsnd(int, const void *, size_t, int) __DARWIN_ALIAS_C(msgsnd);
 __END_DECLS
 
 #endif /* !KERNEL */
 
 #endif /* !_SYS_MSG_H_ */
+
+
diff --git a/bsd/sys/msgbuf.h b/bsd/sys/msgbuf.h
index 15e2dcb86..5b2a5fd2d 100644
--- a/bsd/sys/msgbuf.h
+++ b/bsd/sys/msgbuf.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -57,16 +63,24 @@
 #ifndef	_SYS_MSGBUF_H_
 #define _SYS_MSGBUF_H_
 
-#define	MSG_BSIZE	(4096 - 3 * sizeof(long))
+#include <sys/cdefs.h>
+
+#define	MSG_BSIZE	4096
 struct	msgbuf {
 #define	MSG_MAGIC	0x063061
 	long	msg_magic;
+	long	msg_size;
 	long	msg_bufx;		/* write pointer */
 	long	msg_bufr;		/* read pointer */
-	char	msg_bufc[MSG_BSIZE];	/* buffer */
+	char	*msg_bufc;	/* buffer */
 };
 #ifdef KERNEL
+__BEGIN_DECLS
 extern struct	msgbuf *msgbufp;
 extern void log_putc(char);
+extern void log_putc_locked(char);
+extern void log_setsize(long size);
+extern int log_dmesg(user_addr_t, uint32_t, register_t *);
+__END_DECLS
 #endif
 #endif	/* !_SYS_MSGBUF_H_ */
diff --git a/bsd/sys/mtio.h b/bsd/sys/mtio.h
index e6516a033..31ae0dad5 100644
--- a/bsd/sys/mtio.h
+++ b/bsd/sys/mtio.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/sys/namei.h b/bsd/sys/namei.h
index f588c1b05..c0a8368ba 100644
--- a/bsd/sys/namei.h
+++ b/bsd/sys/namei.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -69,6 +75,10 @@
 
 #ifdef BSD_KERNEL_PRIVATE
 
+/* VFS Supports "/..namedfork/rsrc" access. */
+#define NAMEDRSRCFORK		NAMEDSTREAMS
+
+
 #include <sys/queue.h>
 #include <sys/uio.h>
 #include <sys/vnode.h>
@@ -111,11 +121,14 @@ struct nameidata {
 #ifdef KERNEL
 /*
  * namei operational modifier flags, stored in ni_cnd.flags
+ * Also includes LOCKLEAF, LOCKPARENT, and WANTPARENT flags, defined above.
  */
-#define	NOCACHE		0x0020	/* name must not be left in cache */
-#define	NOFOLLOW	0x0000	/* do not follow symbolic links (pseudo) */
-#define	SHAREDLEAF	0x0080	/* OK to have shared leaf lock */
-#define	MODMASK		0x00fc	/* mask of operational modifiers */
+#define	NOCACHE		0x00000020 /* name must not be left in cache */
+#define	NOFOLLOW	0x00000000 /* do not follow symbolic links (pseudo) */
+/* public FOLLOW	0x00000040    see vnode.h */
+#define	SHAREDLEAF	0x00000080 /* OK to have shared leaf lock */
+/* public NOTRIGGER	0x10000000    see vnode.h */
+#define	MODMASK		0x100000fc /* mask of operational modifiers */
 /*
  * Namei parameter descriptors.
  *
@@ -125,19 +138,31 @@ struct nameidata {
  * name being sought. The caller is responsible for releasing the
  * buffer and for vrele'ing ni_startdir.
  */
+#define	SAVENAME	0          /* save pathanme buffer ***obsolete */
 #define	NOCROSSMOUNT	0x00000100 /* do not cross mount points */
 #define	RDONLY		0x00000200 /* lookup with read-only semantics */
 #define	HASBUF		0x00000400 /* has allocated pathname buffer */
-#define	SAVENAME	0x00000800 /* save pathanme buffer */
+#define DONOTAUTH	0x00000800 /* do not authorize during lookup */
 #define	SAVESTART	0x00001000 /* save starting directory */
+/* public ISDOTDOT	0x00002000    see vnode.h */
+/* public MAKEENTRY	0x00004000    see vnode.h */
+/* public ISLASTCN	0x00008000    see vnode.h */
 #define	ISSYMLINK	0x00010000 /* symlink needs interpretation */
-#define DONOTAUTH	0x00020000 /* do not authorize during lookup */
+/* public ISWHITEOUT	0x00020000    see vnode.h */
+/* public DOWHITEOUT	0x00040000    see vnode.h */
 #define	WILLBEDIR	0x00080000 /* new files will be dirs; allow trailing / */
 #define	AUDITVNPATH1	0x00100000 /* audit the path/vnode info */
 #define	AUDITVNPATH2	0x00200000 /* audit the path/vnode info */
 #define	USEDVP		0x00400000 /* start the lookup at ndp.ni_dvp */
-#define	PARAMASK	0x003fff00 /* mask of parameter descriptors */
+#define	CN_VOLFSPATH	0x00800000 /* user path was a volfs style path */
 #define FSNODELOCKHELD	0x01000000
+#define UNIONCREATED	0x02000000 /* union fs creation of vnode */
+#if NAMEDRSRCFORK
+#define CN_WANTSRSRCFORK 0x04000000
+#define CN_ALLOWRSRCFORK 0x08000000
+#endif
+/* public NOTRIGGER		0x10000000    see vnode.h */
+#define CN_NBMOUNTLOOK	0x20000000 /* do not block for cross mount lookups */
 
 /*
  * Initialization of an nameidata structure.
@@ -164,7 +189,6 @@ struct nameidata {
  * names looked up by namei.
  */
 
-#define	NCHNAMLEN	31	/* maximum name segment length we bother with */
 #define NCHASHMASK	0x7fffffff
 
 struct	namecache {
@@ -179,7 +203,7 @@ struct	namecache {
 	vnode_t			nc_vp;		/* vnode the name refers to */
         unsigned int		nc_whiteout:1,	/* name has whiteout applied */
 	                        nc_hashval:31;	/* hashval of stringname */
-	char		*	nc_name;	/* pointer to segment name in string cache */
+	const char		*nc_name;	/* pointer to segment name in string cache */
 };
 
 
@@ -198,10 +222,10 @@ void    cache_purgevfs(mount_t mp);
 int		cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 			  vfs_context_t context, int *trailing_slash, int *dp_authorized);
 
-void	vnode_cache_credentials(vnode_t vp, vfs_context_t context);
-void	vnode_uncache_credentials(vnode_t vp);
-int		reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp, 
-				struct filedesc *fdp, vfs_context_t context, int *dp_authorized);
+void		vnode_cache_authorized_action(vnode_t vp, vfs_context_t context, kauth_action_t action);
+void		vnode_uncache_authorized_action(vnode_t vp, kauth_action_t action);
+boolean_t	vnode_cache_is_stale(vnode_t vp);
+boolean_t	vnode_cache_is_authorized(vnode_t vp, vfs_context_t context, kauth_action_t action);
 
 #endif /* KERNEL */
 
diff --git a/bsd/sys/netport.h b/bsd/sys/netport.h
index 177eeb37a..a56f0ce10 100644
--- a/bsd/sys/netport.h
+++ b/bsd/sys/netport.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /* 
diff --git a/bsd/sys/param.h b/bsd/sys/param.h
index f9b6aafc2..f655358eb 100644
--- a/bsd/sys/param.h
+++ b/bsd/sys/param.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -141,8 +147,6 @@
 #define PTTYBLOCK 0x200		/* for tty SIGTTOU and SIGTTIN blocking */
 #define PDROP	0x400		/* OR'd with pri to stop re-entry of interlock mutex */
 
-#define	NZERO	0		/* default "nice" */
-
 #define	NBPW	sizeof(int)	/* number of bytes per word (integer) */
 
 #define	CMASK	022		/* default file mask: S_IWGRP|S_IWOTH */
diff --git a/bsd/sys/paths.h b/bsd/sys/paths.h
index 6078c7d7c..992148345 100644
--- a/bsd/sys/paths.h
+++ b/bsd/sys/paths.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)paths.h	1.0	11/13/00	*/
 
diff --git a/bsd/sys/pipe.h b/bsd/sys/pipe.h
index c999c7dbf..a13a27598 100644
--- a/bsd/sys/pipe.h
+++ b/bsd/sys/pipe.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1996 John S. Dyson
@@ -41,6 +47,12 @@
  *
  * $FreeBSD: src/sys/sys/pipe.h,v 1.24 2003/08/13 20:01:38 alc Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_PIPE_H_
 #define _SYS_PIPE_H_
@@ -121,6 +133,9 @@ struct pipemapping {
 #define PIPE_KNOTE     0x1000   /* Pipe has kernel events activated */
 
 #ifdef	KERNEL
+
+struct label;
+
 /*
  * Per-pipe data structure.
  * Two of these are linked together to produce bi-directional pipes.
@@ -135,11 +150,12 @@ struct pipe {
 	struct	pipe *pipe_peer;	/* link with other direction */
 	u_int	pipe_state;		/* pipe status info */
 	int	pipe_busy;		/* busy flag, mostly to handle rundown sanely */
-#ifdef MAC
-	struct	label *pipe_label;	/* pipe MAC label - shared */
-#endif
 	TAILQ_HEAD(,eventqelt) pipe_evlist;
 	lck_mtx_t *pipe_mtxp;		/* shared mutex between both pipes */
+	struct	timespec st_atimespec;	/* time of last access */
+	struct	timespec st_mtimespec;	/* time of last data modification */
+	struct	timespec st_ctimespec;	/* time of last status change */
+	struct	label *pipe_label;	/* pipe MAC label - shared */
 };
 
 #define PIPE_MTX(pipe)		((pipe)->pipe_mtxp)
@@ -149,7 +165,8 @@ struct pipe {
 #define PIPE_LOCK_ASSERT(pipe, type)  lck_mtx_assert(PIPE_MTX(pipe), (type))
 
 __BEGIN_DECLS
-extern int pipe_stat(struct pipe *, struct stat *);
+void pipeinit(void);
+extern int pipe_stat(struct pipe *, void *, int);
 __END_DECLS
 
 #endif	/* KERNEL */
diff --git a/bsd/sys/poll.h b/bsd/sys/poll.h
index 2e8700a32..03e44923a 100644
--- a/bsd/sys/poll.h
+++ b/bsd/sys/poll.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1997 Peter Wemm <peter@freebsd.org>
@@ -106,7 +112,7 @@ __BEGIN_DECLS
  * This is defined here (instead of <poll.h>) because this is where
  * traditional SVR4 code will look to find it.
  */
-extern int poll (struct pollfd *, nfds_t, int);
+extern int poll (struct pollfd *, nfds_t, int) __DARWIN_ALIAS_C(poll);
 
 __END_DECLS
 
diff --git a/bsd/sys/posix_sem.h b/bsd/sys/posix_sem.h
new file mode 100644
index 000000000..4b1e405f6
--- /dev/null
+++ b/bsd/sys/posix_sem.h
@@ -0,0 +1,79 @@
+
+/*
+ * Copyright (c) 1999-2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
+ *	All Rights Reserved.
+ */
+/*
+ * posix_shm.c : Support for POSIX semaphore APIs
+ *
+ *	File:	posix_sem.c
+ *	Author:	Ananthakrishna Ramesh
+ *
+ * HISTORY
+ * 2-Sep-1999	A.Ramesh
+ *	Created for MacOSX
+ *
+ */
+
+#ifndef	_SYS_POSIX_SEM_H_
+#define _SYS_POSIX_SEM_H_
+
+#include <sys/appleapiopts.h>
+#include <sys/types.h>
+#include <sys/proc.h>
+
+struct label;
+
+#define	PSEMNAMLEN	31	/* maximum name segment length we bother with */
+
+struct pseminfo {
+	unsigned int	psem_flags;
+	unsigned int	psem_usecount;
+	mode_t		psem_mode;
+	uid_t		psem_uid;
+	gid_t		psem_gid;
+	char		psem_name[PSEMNAMLEN + 1];	/* segment name */
+	void *		psem_semobject;
+	struct proc *	sem_proc;
+	struct label *  psem_label;
+};
+
+#define PSEMINFO_NULL (struct pseminfo *)0
+
+#define	PSEM_NONE	1
+#define	PSEM_DEFINED	2
+#define	PSEM_ALLOCATED	4
+#define	PSEM_MAPPED	8
+#define	PSEM_INUSE	0x10
+#define	PSEM_REMOVED	0x20
+#define	PSEM_INCREATE	0x40
+#define	PSEM_INDELETE	0x80
+
+#endif
diff --git a/bsd/sys/posix_shm.h b/bsd/sys/posix_shm.h
new file mode 100644
index 000000000..068fdee73
--- /dev/null
+++ b/bsd/sys/posix_shm.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 1999-2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *	Copyright (c) 1990, 1996-1998 Apple Computer, Inc.
+ *	All Rights Reserved.
+ */
+/*
+ * posix_shm.c : Support for POSIX shared memory APIs
+ *
+ *	File:	posix_shm.c
+ *	Author:	Ananthakrishna Ramesh
+ *
+ * HISTORY
+ * 2-Sep-1999	A.Ramesh
+ *	Created for MacOSX
+ *
+ */
+
+#ifndef	_SYS_POSIX_SHM_H_
+#define _SYS_POSIX_SHM_H_
+
+#include <sys/appleapiopts.h>
+#include <sys/types.h>
+#include <sys/proc.h>
+
+struct label;
+
+#define	PSHMNAMLEN	31	/* maximum name segment length we bother with */
+
+struct pshminfo {
+	unsigned int	pshm_flags;
+	unsigned int	pshm_usecount;
+	off_t		pshm_length;
+	mode_t		pshm_mode;
+	uid_t		pshm_uid;
+	gid_t		pshm_gid;
+	char		pshm_name[PSHMNAMLEN + 1];	/* segment name */
+	void *		pshm_memobject;
+#if DIAGNOSTIC
+	unsigned int 	pshm_readcount;
+	unsigned int 	pshm_writecount;
+	struct proc *	pshm_proc;
+#endif /* DIAGNOSTIC */
+	struct label *	pshm_label;
+};
+#define PSHMINFO_NULL (struct pshminfo *)0
+
+#define	PSHM_NONE	1
+#define	PSHM_DEFINED	2
+#define	PSHM_ALLOCATED	4
+#define	PSHM_MAPPED	8
+#define	PSHM_INUSE	0x10
+#define	PSHM_REMOVED	0x20
+#define	PSHM_INCREATE	0x40
+#define	PSHM_INDELETE	0x80
+
+#endif
diff --git a/bsd/sys/proc.h b/bsd/sys/proc.h
index 161b7b1f3..9d41f7ca7 100644
--- a/bsd/sys/proc.h
+++ b/bsd/sys/proc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -153,7 +159,7 @@ struct extern_proc {
 #define	SSTOP	4		/* Process debugging or suspension. */
 #define	SZOMB	5		/* Awaiting collection by parent. */
 
-/* These flags are kept in p_flags. */
+/* These flags are kept in extern_proc.p_flag. */
 #define	P_ADVLOCK	0x00000001	/* Process may hold POSIX adv. lock */
 #define	P_CONTROLT	0x00000002	/* Has a controlling terminal */
 #define	P_LP64		0x00000004	/* Process is LP64 */
@@ -169,8 +175,8 @@ struct extern_proc {
 #define	P_TIMEOUT	0x00000400	/* Timing out during sleep */
 #define	P_TRACED	0x00000800	/* Debugged process being traced */
 
-#define	P_WAITED	0x00001000	/* Debugging prc has waited for child */
-#define	P_WEXIT		0x00002000	/* Working on exiting. */
+#define	P_RESV3 	0x00001000	/* (P_WAITED)Debugging prc has waited for child */
+#define	P_WEXIT		0x00002000	/* Working on exiting */
 #define	P_EXEC		0x00004000	/* Process called exec. */
 
 /* Should be moved to machine-dependent areas. */
@@ -184,18 +190,18 @@ struct extern_proc {
 #define	P_SSTEP		0x20000	/ * process needs single-step fixup ??? * /
 */
 
-#define	P_WAITING	0x00040000	/* process has a wait() in progress */
-#define	P_KDEBUG	0x00080000	/* kdebug tracing on for this process */
+#define	P_RESV5 	0x00040000	/* (P_WAITING) process has a wait() in progress */
+#define	P_CHECKOPENEVT 	0x00080000	/* check if a vnode has the OPENEVT flag set on open */
 
-#define	P_TTYSLEEP	0x00100000	/* blocked due to SIGTTOU or SIGTTIN */
+#define	P_DEPENDENCY_CAPABLE	0x00100000	/* process is ok to call vfs_markdependency() */
 #define	P_REBOOT	0x00200000	/* Process called reboot() */
 #define	P_TBE		0x00400000	/* Process is TBE */
-#define	P_SIGEXC	0x00800000	/* signal exceptions */
+#define	P_RESV7		0x00800000	/* (P_SIGEXC)signal exceptions */
 
-#define	P_BTRACE	0x01000000	/* process is being branch traced */
-#define	P_VFORK		0x02000000	/* process has vfork children */
-#define	P_NOATTACH	0x04000000
-#define	P_INVFORK	0x08000000	/* proc in vfork */
+#define	P_THCWD		0x01000000	/* process has thread cwd  */
+#define	P_RESV9		0x02000000	/* (P_VFORK)process has vfork children */
+#define	P_RESV10 	0x04000000	/* used to be P_NOATTACH */
+#define	P_RESV11	0x08000000	/* (P_INVFORK) proc in vfork */
 
 #define	P_NOSHLIB	0x10000000	/* no shared libs are in use for proc */
 					/* flag set on exec */
@@ -216,11 +222,11 @@ __BEGIN_DECLS
 
 extern proc_t kernproc;
 
-extern int proc_is_classic(struct proc *p);
-struct proc *current_proc_EXTERNAL(void);
+extern int proc_is_classic(proc_t p);
+proc_t current_proc_EXTERNAL(void);
 
 extern int	msleep(void *chan, lck_mtx_t *mtx, int pri, const char *wmesg, struct timespec * ts );
-extern void	unsleep(struct proc *);
+extern void	unsleep(proc_t);
 extern void	wakeup(void *chan);
 extern void wakeup_one(caddr_t chan);
 
@@ -262,17 +268,24 @@ extern int proc_forcequota(proc_t);
 extern int proc_is64bit(proc_t);
 /* is this process exiting? */
 extern int proc_exiting(proc_t);
-/* this routine returns error is the process is not one with super user privileges */
-int proc_suser(struct proc *p);
-/* returns the ucred assicaited with the process; temporary api */
-struct ucred * proc_ucred(struct proc *p);
+/* this routine returns error if the process is not one with super user privileges */
+int proc_suser(proc_t p);
+/* returns the cred assicaited with the process; temporary api */
+kauth_cred_t proc_ucred(proc_t p);
+#ifdef __APPLE_API_UNSTABLE
+/* returns the first thread_t in the process, or NULL XXX for NFS, DO NOT USE */
+thread_t proc_thread(proc_t);
+#endif
+// mark a process as being allowed to call vfs_markdependency()
+void bsd_set_dependency_capable(task_t task);
+
+extern int proc_pendingsignals(proc_t, sigset_t);
+extern int proc_tbe(proc_t);
 
+#ifdef KERNEL_PRIVATE
 /* LP64todo - figure out how to identify 64-bit processes if NULL procp */
 extern int IS_64BIT_PROCESS(proc_t);
-extern int proc_pendingsignals(struct proc *, sigset_t);
-extern int proc_tbe(struct proc *);
 
-#ifdef KERNEL_PRIVATE
 extern int	tsleep(void *chan, int pri, const char *wmesg, int timo);
 extern int	msleep1(void *chan, lck_mtx_t *mtx, int pri, const char *wmesg, u_int64_t timo);
 #endif
diff --git a/bsd/sys/proc_info.h b/bsd/sys/proc_info.h
index 956a0684b..98bf6d972 100644
--- a/bsd/sys/proc_info.h
+++ b/bsd/sys/proc_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_PROC_INFO_H
@@ -79,6 +85,7 @@ struct proc_bsdinfo {
 #define PROC_FLAG_SLEADER	0x20
 #define PROC_FLAG_CTTY		0x40
 #define PROC_FLAG_CONTROLT	0x80
+#define PROC_FLAG_THCWD		0x100
 
 
 struct proc_taskinfo {
@@ -119,6 +126,7 @@ struct proc_threadinfo {
 	int32_t			pth_curpri;		/* cur priority*/
 	int32_t			pth_priority;		/*  priority*/
 	int32_t			pth_maxpriority;		/* max priority*/
+	char *			pth_name[64];		/* thread name, if any */
 };
 
 struct proc_regioninfo {
@@ -140,9 +148,9 @@ struct proc_regioninfo {
 	uint32_t		pri_private_pages_resident;
 	uint32_t		pri_shared_pages_resident;
 	uint32_t		pri_obj_id;
+	uint32_t		pri_depth;
 	uint64_t		pri_address;
 	uint64_t		pri_size;
-	uint32_t		pri_depth;
 };
 
 #define PROC_REGION_SUBMAP	1
@@ -184,11 +192,42 @@ struct proc_fileinfo {
 	int32_t			fi_type;
 };
 
+/* stats flags in proc_fileinfo */
+#define PROC_FP_SHARED	1	/* shared by more than one fd */
+#define PROC_FP_CLEXEC	2	/* close on exec */
+
+/*
+ * A copy of stat64 with static sized fields.
+ */
+struct vinfo_stat {
+	uint32_t	vst_dev;	/* [XSI] ID of device containing file */
+	uint16_t	vst_mode;	/* [XSI] Mode of file (see below) */
+	uint16_t	vst_nlink;	/* [XSI] Number of hard links */
+	uint64_t	vst_ino;	/* [XSI] File serial number */
+	uid_t		vst_uid;	/* [XSI] User ID of the file */
+	gid_t		vst_gid;	/* [XSI] Group ID of the file */
+	int64_t		vst_atime;	/* [XSI] Time of last access */
+	int64_t		vst_atimensec;	/* nsec of last access */
+	int64_t		vst_mtime;	/* [XSI] Last data modification time */
+	int64_t		vst_mtimensec;	/* last data modification nsec */
+	int64_t		vst_ctime;	/* [XSI] Time of last status change */
+	int64_t		vst_ctimensec;	/* nsec of last status change */
+	int64_t		vst_birthtime;	/*  File creation time(birth)  */
+	int64_t		vst_birthtimensec;	/* nsec of File creation time */
+	off_t		vst_size;	/* [XSI] file size, in bytes */
+	int64_t		vst_blocks;	/* [XSI] blocks allocated for file */
+	int32_t		vst_blksize;	/* [XSI] optimal blocksize for I/O */
+	uint32_t	vst_flags;	/* user defined flags for file */
+	uint32_t	vst_gen;	/* file generation number */
+	uint32_t	vst_rdev;	/* [XSI] Device ID */
+	int64_t		vst_qspare[2];	/* RESERVED: DO NOT USE! */
+};
 
 struct vnode_info {
-	struct stat 	vi_stat;
-	int		vi_type;
-	fsid_t		vi_fsid;
+	struct vinfo_stat	vi_stat;
+	int			vi_type;
+	fsid_t			vi_fsid;
+	int			vi_pad;
 };
 
 struct vnode_info_path {
@@ -197,17 +236,15 @@ struct vnode_info_path {
 };
 
 struct vnode_fdinfo {
-	struct proc_fileinfo pfi;
+	struct proc_fileinfo	pfi;
 	struct vnode_info	pvi;
 };
 
 struct vnode_fdinfowithpath {
 	struct proc_fileinfo pfi;
 	struct vnode_info_path pvip;
-
 };
 
-
 struct proc_regionwithpathinfo {
 	struct proc_regioninfo prp_prinfo;
 	struct vnode_info_path prp_vip;
@@ -218,6 +255,10 @@ struct proc_vnodepathinfo {
 	struct vnode_info_path pvi_rdir;
 };
 
+struct proc_threadwithpathinfo {
+	struct proc_threadinfo pt;
+	struct vnode_info_path pvip;
+};
 
 /*
  *  Socket 
@@ -423,7 +464,7 @@ struct socket_fdinfo {
 
 
 struct psem_info {
-	struct stat 	psem_stat;
+	struct vinfo_stat 	psem_stat;
 	char			psem_name[MAXPATHLEN];
 };
 
@@ -435,7 +476,7 @@ struct psem_fdinfo {
 
 
 struct pshm_info  {
-	struct stat		pshm_stat;
+	struct vinfo_stat		pshm_stat;
 	uint64_t		pshm_mappaddr;
 	char			pshm_name[MAXPATHLEN];
 };
@@ -447,7 +488,7 @@ struct pshm_fdinfo {
 
 
 struct pipe_info {
-	struct stat pipe_stat;
+	struct vinfo_stat pipe_stat;
 	uint64_t	pipe_handle;
 	uint64_t	pipe_peerhandle;
 	int			pipe_status;
@@ -460,7 +501,7 @@ struct pipe_fdinfo {
 
 
 struct kqueue_info {
-	struct stat 	kq_stat;
+	struct vinfo_stat 	kq_stat;
 	uint32_t	kq_state;
 };
 #define PROC_KQUEUE_SELECT	1
@@ -472,7 +513,7 @@ struct kqueue_fdinfo {
 };
 
 struct appletalk_info {
-	struct stat 	atalk_stat;
+	struct vinfo_stat 	atalk_stat;
 };
 
 struct appletalk_fdinfo {
@@ -526,6 +567,13 @@ struct proc_fdinfo {
 #define PROC_PIDVNODEPATHINFO 9
 #define PROC_PIDVNODEPATHINFO_SIZE  (sizeof(struct proc_vnodepathinfo))
 
+#define PROC_PIDTHREADPATHINFO 10
+#define PROC_PIDTHREADPATHINFO_SIZE  (sizeof(struct proc_threadwithpathinfo))
+
+#define PROC_PIDPATHINFO 11
+#define PROC_PIDPATHINFO_SIZE  (MAXPATHLEN)
+#define PROC_PIDPATHINFO_MAXSIZE  (4*MAXPATHLEN)
+
 /* Flavors for proc_pidfdinfo */
 
 #define PROC_PIDFDVNODEINFO 1
diff --git a/bsd/sys/proc_internal.h b/bsd/sys/proc_internal.h
index e0e150146..08f5fe12c 100644
--- a/bsd/sys/proc_internal.h
+++ b/bsd/sys/proc_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -59,45 +65,108 @@
  *
  *	@(#)proc_internal.h	8.15 (Berkeley) 5/19/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_PROC_INTERNAL_H_
 #define	_SYS_PROC_INTERNAL_H_
 
+#include <libkern/OSAtomic.h>
 #include <sys/proc.h>
 __BEGIN_DECLS
 #include <kern/locks.h>
 __END_DECLS
 
+#define	__PROC_INTERNAL_DEBUG 1
+
+/* 
+ * The short form for various locks that protect fields in the data structures.
+ * PL = Process Lock
+ * PGL = Process Group Lock
+ * PFDL = Process File Desc Lock
+ * PSL = Process Spin Lock
+ * PPL = Parent Process Lock (planed for later usage)
+ * LL = List Lock
+ * SL = Session Lock
+*/
+struct label;
+
+/*
+ * Added by SPARTA, Inc.
+ */
+/*
+ * Login context.
+ */
+struct lctx {
+	LIST_ENTRY(lctx) lc_list;	/* List of all login contexts. */
+	LIST_HEAD(, proc) lc_members;	/* Pointer to lc members. */
+	int		lc_mc;		/* Member Count. */
+	pid_t		lc_id;		/* Login context ID. */
+	lck_mtx_t	lc_mtx;		/* Mutex to protect members */
+
+	struct label	*lc_label;	/* Login context MAC label. */
+};
+
 /*
  * One structure allocated per session.
  */
 struct	session {
-	int	s_count;		/* Ref cnt; pgrps in session. */
-	struct	proc *s_leader;		/* Session leader. */
-	struct	vnode *s_ttyvp;		/* Vnode of controlling terminal. */
-	struct	tty *s_ttyp;		/* Controlling terminal. */
-	pid_t	s_sid;		/* Session ID */
-	char	s_login[MAXLOGNAME];	/* Setlogin() name. */
+	int			s_count;		/* Ref cnt; pgrps in session. (LL) */
+	struct	proc *		s_leader;		/* Session leader.(static) */
+	struct	vnode *		s_ttyvp;		/* Vnode of controlling terminal.(SL) */
+	int			s_ttyvid;		/* Vnode id of the controlling terminal (SL) */
+	struct	tty *		s_ttyp;			/* Controlling terminal. (SL) */
+	pid_t			s_ttypgrpid;		/* tty's pgrp id */
+	pid_t			s_sid;			/* Session ID (static) */
+	char			s_login[MAXLOGNAME];	/* Setlogin() name.(SL) */
+	int			s_flags;		/* Session flags (s_mlock)  */
+	LIST_ENTRY(session)	s_hash;			/* Hash chain.(LL) */
+	lck_mtx_t		s_mlock;		/* mutex lock to protect session */
+	int			s_listflags;
 };
 
+#define SESSION_NULL (struct session *)0
+
+/*
+ * Session flags; used to tunnel information to lower layers and line
+ * disciplines, etc.
+ */
+#define	S_DEFAULT	0x00000000	/* No flags set */
+#define	S_NOCTTY	0x00000001	/* Do not associate controlling tty */
+
+
+#define S_LIST_TERM	1		/* marked for termination */
+#define	S_LIST_DEAD	2		/* already dead */
 /*
  * One structure allocated per process group.
  */
 struct	pgrp {
-	LIST_ENTRY(pgrp) pg_hash;	/* Hash chain. */
-	LIST_HEAD(, proc) pg_members;	/* Pointer to pgrp members. */
-	struct	session *pg_session;	/* Pointer to session. */
-	pid_t	pg_id;			/* Pgrp id. */
-	int	pg_jobc;	/* # procs qualifying pgrp for job control */
+	LIST_ENTRY(pgrp)	pg_hash;	/* Hash chain. (LL) */
+	LIST_HEAD(, proc) 	pg_members;	/* Pointer to pgrp members. (PGL) */
+	struct	session *	pg_session;	/* Pointer to session. (LL ) */
+	pid_t			pg_id;		/* Pgrp id. (static) */
+	int			pg_jobc;	/* # procs qualifying pgrp for job control (PGL) */
+	int			pg_membercnt;	/* Number of processes in the pgrocess group (PGL) */
+	int			pg_refcount;	/* number of current iterators (LL) */
+	unsigned int		pg_listflags;	/* (LL) */
+	lck_mtx_t		pg_mlock;	/* mutex lock to protect pgrp */
 };
 
+#define PGRP_FLAG_TERMINATE 	1
+#define PGRP_FLAG_WAITTERMINATE 2
+#define PGRP_FLAG_DEAD 		4
+#define PGRP_FLAG_ITERABEGIN 	8
+#define PGRP_FLAG_ITERWAIT 	0x10
+
+#define PGRP_NULL (struct pgrp *)0
 struct proc;
 
 #define PROC_NULL (struct proc *)0
 
-#define	p_session	p_pgrp->pg_session
-#define	p_pgid		p_pgrp->pg_id
-
 /*
  * Description of a process.
  *
@@ -110,123 +179,157 @@ struct proc;
  * is running.
  */
 struct	proc {
-	LIST_ENTRY(proc) p_list;	/* List of all processes. */
+	LIST_ENTRY(proc) p_list;		/* List of all processes. */
 
-	/* substructures: */
-	struct	ucred *p_ucred;		/* Process owner's identity. */
-	struct	filedesc *p_fd;		/* Ptr to open files structure. */
-	struct	pstats *p_stats;	/* Accounting/statistics (PROC ONLY). */
-	struct	plimit *p_limit;	/* Process limits. */
-	struct	sigacts *p_sigacts;	/* Signal actions, state (PROC ONLY). */
+	pid_t		p_pid;			/* Process identifier. (static)*/
+	void * 		task;			/* corresponding task (static)*/
+	struct	proc *	p_pptr;		 	/* Pointer to parent process.(LL) */
+	pid_t		p_ppid;			/* process's parent pid number */
+	pid_t		p_pgrpid;		/* process group id of the process (LL)*/
 
-#define	p_rlimit	p_limit->pl_rlimit
+	lck_mtx_t 	p_mlock;		/* mutex lock for proc */
 
-	int	p_flag;			/* P_* flags. */
-	char	p_stat;			/* S* process status. */
-	char	p_shutdownstate;
-	char	p_pad1[2];
+	char		p_stat;			/* S* process status. (PL)*/
+	char		p_shutdownstate;
+	char		p_kdebug;		/* P_KDEBUG eq (CC)*/ 
+	char		p_btrace;		/* P_BTRACE eq (CC)*/
 
-	pid_t	p_pid;			/* Process identifier. */
-	LIST_ENTRY(proc) p_pglist;	/* List of processes in pgrp. */
-	struct	proc *p_pptr;	 	/* Pointer to parent process. */
-	LIST_ENTRY(proc) p_sibling;	/* List of sibling processes. */
-	LIST_HEAD(, proc) p_children;	/* Pointer to list of children. */
+	LIST_ENTRY(proc) p_pglist;		/* List of processes in pgrp.(PGL) */
+	LIST_ENTRY(proc) p_sibling;		/* List of sibling processes. (LL)*/
+	LIST_HEAD(, proc) p_children;		/* Pointer to list of children. (LL)*/
+	TAILQ_HEAD( , uthread) p_uthlist; 	/* List of uthreads  (PL) */
 
-/* The following fields are all zeroed upon creation in fork. */
-#define	p_startzero	p_oppid
+	LIST_ENTRY(proc) p_hash;		/* Hash chain. (LL)*/
+	TAILQ_HEAD( ,eventqelt) p_evlist;	/* (PL) */
 
-	pid_t	p_oppid;	 /* Save parent pid during ptrace. XXX */
-	int	p_dupfd;	 /* Sideways return value from fdopen. XXX */
+	lck_mtx_t	p_fdmlock;		/* proc lock to protect fdesc */
 
-	/* scheduling */
-	u_int	p_estcpu;	 /* Time averaged value of p_cpticks. */
-	int	p_cpticks;	 /* Ticks of cpu time. */
-	fixpt_t	p_pctcpu;	 /* %cpu for this process during p_swtime */
-	void	*p_wchan;	 /* Sleep address. */
-	char	*p_wmesg;	 /* Reason for sleep. */
-	u_int	p_swtime;	 /* DEPRECATED (Time swapped in or out.) */
-#define	p_argslen p_swtime	 /* Length of process arguments. */
-	u_int	p_slptime;	 /* Time since last blocked. */
+	/* substructures: */
+	kauth_cred_t	p_ucred;		/* Process owner's identity. (PL) */
+	struct	filedesc *p_fd;			/* Ptr to open files structure. (PFDL) */
+	struct	pstats *p_stats;		/* Accounting/statistics (PL). */
+	struct	plimit *p_limit;		/* Process limits.(PL) */
 
-	struct	itimerval p_realtimer;	/* Alarm timer. */
-	struct	timeval p_rtime;	/* Real time. */
-	u_quad_t p_uticks;		/* Statclock hits in user mode. */
-	u_quad_t p_sticks;		/* Statclock hits in system mode. */
-	u_quad_t p_iticks;		/* Statclock hits processing intr. */
+	struct	sigacts *p_sigacts;		/* Signal actions, state (PL) */
+	lck_spin_t	p_slock;		/* spin lock for itimer/profil protection */
 
-	int	p_traceflag;		/* Kernel trace points. */
-	struct	vnode *p_tracep;	/* Trace to vnode. */
+#define	p_rlimit	p_limit->pl_rlimit
 
-	sigset_t p_siglist;		/* DEPRECATED. */
+	struct	plimit *p_olimit;		/* old process limits  - not inherited by child  (PL) */
+	unsigned int	p_flag;			/* P_* flags. (atomic bit ops) */
+	unsigned int	p_lflag;		/* local flags  (PL) */
+	unsigned int	p_listflag;		/* list flags (LL) */
+	unsigned int	p_ladvflag;		/* local adv flags (atomic) */
+	int		p_refcount;		/* number of outstanding users(LL) */
+	int		p_childrencnt;		/* children holding ref on parent (LL) */
+	int		p_parentref;		/* children lookup ref on parent (LL) */
+
+	pid_t		p_oppid;	 	/* Save parent pid during ptrace. XXX */
+	u_int		p_xstat;		/* Exit status for wait; also stop signal. */
+
+#ifdef _PROC_HAS_SCHEDINFO_
+	/* may need cleanup, not used */
+	u_int		p_estcpu;	 	/* Time averaged value of p_cpticks.(used by aio and proc_comapre) */
+	fixpt_t		p_pctcpu;	 	/* %cpu for this process during p_swtime (used by aio)*/
+	u_int		p_slptime;		/* used by proc_compare */
+#endif /* _PROC_HAS_SCHEDINFO_ */
+
+	struct	itimerval p_realtimer;		/* Alarm timer. (PSL) */
+	struct	timeval p_rtime;		/* Real time.(PSL)  */
+	struct	itimerval p_vtimer_user;	/* Virtual timers.(PSL)  */
+	struct	itimerval p_vtimer_prof;	/* (PSL) */
+
+	struct	timeval	p_rlim_cpu;		/* Remaining rlim cpu value.(PSL) */
+	int		p_debugger;		/*  NU 1: can exec set-bit programs if suser */
+	boolean_t	sigwait;	/* indication to suspend (PL) */
+	void	*sigwait_thread;	/* 'thread' holding sigwait(PL)  */
+	void	*exit_thread;		/* Which thread is exiting(PL)  */
+	int	p_vforkcnt;		/* number of outstanding vforks(PL)  */
+    	void *  p_vforkact;     	/* activation running this vfork proc)(static)  */
+	int	p_fpdrainwait;		/* (PFDL) */
+	pid_t	p_contproc;	/* last PID to send us a SIGCONT (PL) */
+
+	/* Following fields are info from SIGCHLD (PL) */
+	pid_t	si_pid;			/* (PL) */
+	u_int   si_status;		/* (PL) */
+	u_int	si_code;		/* (PL) */
+	uid_t	si_uid;			/* (PL) */
+
+	void * vm_shm;			/* (SYSV SHM Lock) for sysV shared memory */
+
+#if CONFIG_DTRACE
+	user_addr_t			p_dtrace_argv;			/* (write once, read only after that) */
+	user_addr_t			p_dtrace_envp;			/* (write once, read only after that) */
+	lck_mtx_t			p_dtrace_sprlock;		/* sun proc lock emulation */
+	int				p_dtrace_probes;		/* (PL) are there probes for this proc? */
+	u_int				p_dtrace_count;			/* (sprlock) number of DTrace tracepoints */
+	struct dtrace_ptss_page*	p_dtrace_ptss_pages;		/* (sprlock) list of user ptss pages */
+	struct dtrace_ptss_page_entry*	p_dtrace_ptss_free_list;	/* (atomic) list of individual ptss entries */
+	struct dtrace_helpers*		p_dtrace_helpers;		/* (dtrace_lock) DTrace per-proc private */
+	struct dof_ioctl_data*		p_dtrace_lazy_dofs;		/* (sprlock) unloaded dof_helper_t's */
+#endif /* CONFIG_DTRACE */
+
+/* XXXXXXXXXXXXX BCOPY'ed on fork XXXXXXXXXXXXXXXX */
+/* The following fields are all copied upon creation in fork. */
+#define	p_startcopy	p_argslen
 
+	u_int	p_argslen;	 /* Length of process arguments. */
+	int  	p_argc;			/* saved argc for sysctl_procargs() */
+	user_addr_t user_stack;		/* where user stack was allocated */
 	struct	vnode *p_textvp;	/* Vnode of executable. */
+	off_t	p_textoff;		/* offset in executable vnode */
 
-/* End area that is zeroed on creation. */
-#define	p_endzero	p_hash.le_next
-
-	/*
-	 * Not copied, not zero'ed.
-	 * Belongs after p_pid, but here to avoid shifting proc elements.
-	 */
-	LIST_ENTRY(proc) p_hash;	/* Hash chain. */
-	TAILQ_HEAD( ,eventqelt) p_evlist;
+	sigset_t p_sigmask;		/* DEPRECATED */
+	sigset_t p_sigignore;	/* Signals being ignored. (PL) */
+	sigset_t p_sigcatch;	/* Signals being caught by user.(PL)  */
 
-/* The following fields are all copied upon creation in fork. */
-#define	p_startcopy	p_sigmask
+	u_char	p_priority;	/* (NU) Process priority. */
+	u_char	p_resv0;	/* (NU) User-priority based on p_cpu and p_nice. */
+	char	p_nice;		/* Process "nice" value.(PL) */
+	u_char	p_resv1;	/* (NU) User-priority based on p_cpu and p_nice. */
 
-	sigset_t p_sigmask;		/* DEPRECATED */
-	sigset_t p_sigignore;	/* Signals being ignored. */
-	sigset_t p_sigcatch;	/* Signals being caught by user. */
+#if CONFIG_MACF
+	int	p_mac_enforce;			/* MAC policy enforcement control */
+#endif
 
-	u_char	p_priority;	/* Process priority. */
-	u_char	p_usrpri;	/* User-priority based on p_cpu and p_nice. */
-	char	p_nice;		/* Process "nice" value. */
 	char	p_comm[MAXCOMLEN+1];
-	char	p_name[(2*MAXCOMLEN)+1];
+	char	p_name[(2*MAXCOMLEN)+1];	/* PL */
 
-	struct 	pgrp *p_pgrp;	/* Pointer to process group. */
+	struct 	pgrp *p_pgrp;	/* Pointer to process group. (LL) */
+	int		p_iopol_disk;	/* disk I/O policy (PL) */
+	uint32_t	p_csflags;	/* flags for codesign (PL) */
 
 /* End area that is copied on creation. */
-#define	p_endcopy	p_xstat
-	
-	u_short	p_xstat;	/* Exit status for wait; also stop signal. */
-	u_short	p_acflag;	/* Accounting flags. */
-	struct	rusage *p_ru;	/* Exit information. XXX */
-
-	int	p_debugger;	/* 1: can exec set-bit programs if suser */
-
-	void	*task;			/* corresponding task */
-	void	*sigwait_thread;	/* 'thread' holding sigwait */
-	char 	signal_lock[72];
-	boolean_t	 sigwait;	/* indication to suspend */
-	void	*exit_thread;		/* Which thread is exiting? */
-	user_addr_t user_stack;		/* where user stack was allocated */
-	void * exitarg;			/* exit arg for proc terminate */
-	void * vm_shm;			/* for sysV shared memory */
-	int  p_argc;			/* saved argc for sysctl_procargs() */
-	int		p_vforkcnt;		/* number of outstanding vforks */
-    void *  p_vforkact;     /* activation running this vfork proc */
-	TAILQ_HEAD( , uthread) p_uthlist; /* List of uthreads  */
-	/* Following fields are info from SIGCHLD */
-	pid_t	si_pid;
-	u_short si_status;
-	u_short	si_code;
-	uid_t	si_uid;
-	TAILQ_HEAD( , aio_workq_entry ) aio_activeq; /* active async IO requests */
+/* XXXXXXXXXXXXX End of BCOPY'ed on fork (AIOLOCK)XXXXXXXXXXXXXXXX */
+#define	p_endcopy	aio_active_count
 	int		aio_active_count;	/* entries on aio_activeq */
-	TAILQ_HEAD( , aio_workq_entry ) aio_doneq;	 /* completed async IO requests */
 	int		aio_done_count;		/* entries on aio_doneq */
+	TAILQ_HEAD( , aio_workq_entry ) aio_activeq; /* active async IO requests */
+	TAILQ_HEAD( , aio_workq_entry ) aio_doneq;	 /* completed async IO requests */
+
+	struct klist p_klist;  /* knote list (PL ?)*/
 
-	struct klist p_klist;  /* knote list */
-	lck_mtx_t	p_mlock;	/* proc lock to protect evques */
-	lck_mtx_t	p_fdmlock;	/* proc lock to protect evques */
+	struct	rusage *p_ru;	/* Exit information. (PL) */
+	thread_t 	p_signalholder;
+	thread_t 	p_transholder;
+
+	/* DEPRECATE following field  */
+	u_short	p_acflag;	/* Accounting flags. */
+
+	struct lctx *p_lctx;		/* Pointer to login context. */
+	LIST_ENTRY(proc) p_lclist;	/* List of processes in lctx. */
+	user_addr_t 	p_threadstart;		/* pthread start fn */
+	user_addr_t 	p_wqthread;		/* pthread workqueue fn */
+	int 	p_pthsize;			/* pthread size */
+	void * 	p_wqptr;			/* workq ptr */
+	int 	p_wqsize;			/* allocated size */
+	lck_mtx_t	p_wqlock;		/* lock to protect work queue */
+	struct  timeval p_start;        	/* starting time */
+	void *	p_rcall;
+	int		p_ractive;
+#if DIAGNOSTIC
 	unsigned int p_fdlock_pc[4];
 	unsigned int p_fdunlock_pc[4];
-	int		p_fpdrainwait;
-	unsigned int		p_lflag;		/* local flags */
-	unsigned int		p_ladvflag;		/* local adv flags*/
-	unsigned int		p_internalref;	/* temp refcount field */
-#if DIAGNOSTIC
 #if SIGNAL_DEBUG
 	unsigned int lockpc[8];
 	unsigned int unlockpc[8];
@@ -234,23 +337,73 @@ struct	proc {
 #endif /* DIAGNOSTIC */
 };
 
+#define PGRPID_DEAD 0xdeaddead
+
+/* p_listflag */ 
+#define P_LIST_DRAIN   			0x00000001
+#define P_LIST_DRAINWAIT 		0x00000002
+#define P_LIST_DRAINED   		0x00000004
+#define P_LIST_DEAD				0x00000008
+#define P_LIST_WAITING   		0x00000010
+#define P_LIST_EXITED    		0x00000040 
+#define P_LIST_CHILDDRSTART    	0x00000080 
+#define P_LIST_CHILDDRAINED    	0x00000100 
+#define P_LIST_CHILDDRWAIT    	0x00000200 
+#define P_LIST_CHILDLKWAIT    	0x00000400 
+#define P_LIST_DEADPARENT    	0x00000800 
+#define P_LIST_PARENTREFWAIT   	0x00001000 
+#define P_LIST_INCREATE   		0x00002000 
+/* 0x4000 &  0x8000 Not used */
+#define P_LIST_INHASH	 		0x00010000	/* process is in hash */
+#define P_LIST_INPGRP	 		0x00020000	/* process is in pgrp */
+#define P_LIST_PGRPTRANS 		0x00040000	/* pgrp is getting replaced */
+#define P_LIST_PGRPTRWAIT 		0x00080000	/* wait for pgrp replacement */
+ 
 
 /* local flags */
-#define	P_LDELAYTERM		0x1	/* */
-#define	P_LNOZOMB		0x2	/* */
-#define P_LLOW_PRI_IO		0x4
-#define P_LPEXIT		0x8
-#define P_LBACKGROUND_IO	0x10
-#define P_LWAITING		0x20
-#define P_LREFDRAIN		0x40
-#define P_LREFDRAINWAIT		0x80
-#define P_LREFDEAD		0x100
-#define P_LTHSIGSTACK	0x200
+#define	P_LDELAYTERM	0x00000001	/* */
+#define	P_LNOZOMB	0x00000002 	/* */
+#define P_LTERM		0x00000004      /* */ 
+#define P_LEXIT		0x00000008      /* */
+#define P_LPEXIT	0x00000010
+#define	P_LTHSIGSTACK	0x00000020
+#define	P_LINTRANSIT	0x00000040	/* process in exec or in creation */
+#define	P_LTRANSWAIT	0x00000080	/* waiting for trans to complete */
+#define P_LVFORK        0x00000100      /* */
+#define P_LINVFORK      0x00000200      /* */
+#define P_LTRACED       0x00000400      /* */
+#define P_LSIGEXC       0x00000800      /* */
+#define P_LNOATTACH     0x00001000      /* */
+#define P_LPPWAIT       0x00002000      /* */
+#define	P_LKQWDRAIN	0x00004000
+#define	P_LKQWDRAINWAIT	0x00008000
+#define	P_LKQWDEAD	0x00010000
+#define	P_LLIMCHANGE	0x00020000
+#define	P_LLIMWAIT	0x00040000
+#define P_LWAITED   	0x00080000 
+#define P_LINSIGNAL    	0x00100000 
+#define P_LSIGNALWAIT  	0x00200000 
+#define P_LRAGE_VNODES	0x00400000
 
 /* advisory flags in the proc */
 #define P_LADVLOCK		0x01
 
-// LP64todo - should this move?
+/* defns for proc_iterate */
+#define PROC_ALLPROCLIST        1		/* walk the allproc list (procs not exited yet) */
+#define PROC_ZOMBPROCLIST       2		/*  walk the zombie list */
+#define PROC_NOWAITTRANS       4		/* do not wait for transitions (checkdirs only)  */
+ 
+/* defns for pgrp_iterate */ 
+#define PGRP_DROPREF    	1
+#define	PGRP_BLOCKITERATE 	2
+ 
+/* return values of the proc iteration callback routine */ 
+#define PROC_RETURNED           0
+#define PROC_RETURNED_DONE      1
+#define PROC_CLAIMED            2
+#define PROC_CLAIMED_DONE       3
+
+
 /* LP64 version of extern_proc.  all pointers 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with extern_proc
@@ -321,12 +474,24 @@ extern int nprocs, maxproc;		/* Current and max number of procs. */
 extern int maxprocperuid;		/* Current number of procs per uid */
 __private_extern__ int hard_maxproc;	/* hard limit */
 
-#define	PID_MAX		30000
-#define	NO_PID		30001
-
-#define SESS_LEADER(p)	((p)->p_session->s_leader == (p))
-#define	SESSHOLD(s)	((s)->s_count++)
-#define	SESSRELE(s)	sessrele(s)
+#define	PID_MAX		99999
+#define	NO_PID		100000
+extern lck_mtx_t * proc_list_mlock;
+extern lck_mtx_t * proc_klist_mlock;
+
+#define SESS_LEADER(p, sessp)	((sessp)->s_leader == (p))
+
+/* Lock and unlock a login context. */
+#define LCTX_LOCK(lc)	lck_mtx_lock(&(lc)->lc_mtx)
+#define LCTX_UNLOCK(lc)	lck_mtx_unlock(&(lc)->lc_mtx)
+#define LCTX_LOCKED(lc)
+#define LCTX_LOCK_ASSERT(lc, type)
+#define ALLLCTX_LOCK	lck_mtx_lock(&alllctx_lock)
+#define ALLLCTX_UNLOCK	lck_mtx_unlock(&alllctx_lock)
+extern lck_mtx_t alllctx_lock;
+extern lck_grp_t * lctx_lck_grp;
+extern lck_grp_attr_t * lctx_lck_grp_attr;
+extern lck_attr_t * lctx_lck_attr;
 
 #define	PIDHASH(pid)	(&pidhashtbl[(pid) & pidhash])
 extern LIST_HEAD(pidhashhead, proc) *pidhashtbl;
@@ -335,6 +500,10 @@ extern u_long pidhash;
 #define	PGRPHASH(pgid)	(&pgrphashtbl[(pgid) & pgrphash])
 extern LIST_HEAD(pgrphashhead, pgrp) *pgrphashtbl;
 extern u_long pgrphash;
+#define	SESSHASH(sessid) (&sesshashtbl[(sessid) & sesshash])
+extern LIST_HEAD(sesshashhead, session) *sesshashtbl;
+extern u_long sesshash;
+
 extern lck_grp_t * proc_lck_grp;
 extern lck_grp_attr_t * proc_lck_grp_attr;
 extern lck_attr_t * proc_lck_attr;
@@ -343,24 +512,35 @@ LIST_HEAD(proclist, proc);
 extern struct proclist allproc;		/* List of all processes. */
 extern struct proclist zombproc;	/* List of zombie processes. */
 extern struct proc *initproc;
-extern void	pgdelete(struct pgrp *pgrp);
-extern void	sessrele(struct session *sess);
-extern void	procinit(void);
+extern void	procinit(void) __attribute__((section("__TEXT, initcode")));
 extern void proc_lock(struct proc *);
 extern void proc_unlock(struct proc *);
+extern void proc_spinlock(struct proc *);
+extern void proc_spinunlock(struct proc *);
+extern void proc_list_lock(void);
+extern void proc_list_unlock(void);
+extern void proc_klist_lock(void);
+extern void proc_klist_unlock(void);
 extern void proc_fdlock(struct proc *);
+extern void proc_fdlock_spin(struct proc *);
 extern void proc_fdunlock(struct proc *);
-__private_extern__ char *proc_core_name(const char *name, uid_t uid, pid_t pid);
+extern void proc_fdlock_assert(proc_t p, int assertflags);
+__private_extern__ int proc_core_name(const char *name, uid_t uid, pid_t pid,
+		char *cr_name, size_t cr_name_len);
 extern int isinferior(struct proc *, struct proc *);
-extern struct	proc *pfind(pid_t);	/* Find process by id. */
 __private_extern__ struct proc *pzfind(pid_t);	/* Find zombie by id. */
-extern struct	pgrp *pgfind(pid_t);	/* Find process group by id. */
+
+extern struct	lctx *lcfind(pid_t);		/* Find a login context by id */
+extern struct	lctx *lccreate(void);		/* Create a new login context */
 
 extern int	chgproccnt(uid_t uid, int diff);
+extern void	enterlctx(struct proc *p, struct lctx *l, int create);
+extern void	pinsertchild(struct proc *parent, struct proc *child);
 extern int	enterpgrp(struct proc *p, pid_t pgid, int mksess);
 extern void	fixjobc(struct proc *p, struct pgrp *pgrp, int entering);
 extern int	inferior(struct proc *p);
 extern int	leavepgrp(struct proc *p);
+extern void	leavelctx(struct proc *p);
 extern void	resetpriority(struct proc *);
 extern void	setrunnable(struct proc *);
 extern void	setrunqueue(struct proc *);
@@ -368,10 +548,47 @@ extern int	sleep(void *chan, int pri);
 extern int	tsleep0(void *chan, int pri, const char *wmesg, int timo, int (*continuation)(int));
 extern int	tsleep1(void *chan, int pri, const char *wmesg, u_int64_t abstime, int (*continuation)(int));
 extern int	msleep0(void *chan, lck_mtx_t *mtx, int pri, const char *wmesg, int timo, int (*continuation)(int));
-extern void	vfork_return(thread_t th_act, struct proc *p, struct proc *p2, register_t *retval);
-extern struct proc * proc_findref(pid_t pid);
-extern void  proc_dropref(struct proc *  p);
-extern struct proc * proc_refinternal(proc_t  p, int funneled);
-extern void  proc_dropinternal(struct proc *  p, int funneled);
-
+extern void	vfork_return(struct proc *child, register_t *retval, int rval);
+extern int	exit1(struct proc *, int, int *);
+extern void vfork_exit_internal(struct proc *p, int rv, int forced);
+extern void proc_reparentlocked(struct proc *child, struct proc * newparent, int cansignal, int locked);
+extern int pgrp_iterate(struct pgrp * pgrp, int flags, int (*callout)(proc_t , void *), void *arg, int (*filterfn)(proc_t , void *), void *filterarg);
+extern int proc_iterate(int flags, int (*callout)(proc_t , void *), void *arg, int (*filterfn)(proc_t , void *), void *filterarg);
+extern int proc_rebootscan(int (*callout)(proc_t , void *), void *arg, int (*filterfn)(proc_t , void *), void *filterarg);
+extern int proc_childrenwalk(proc_t p, int (*callout)(proc_t , void *), void *arg);
+extern proc_t proc_findinternal(int pid, int funneled);
+extern void proc_refdrain(proc_t);
+extern void proc_childdrainlocked(proc_t);
+extern void proc_childdrainstart(proc_t);
+extern void proc_childdrainend(proc_t);
+extern void  proc_checkdeadrefs(proc_t);
+struct proc *pfind_locked(pid_t);
+extern struct pgrp *pgfind(pid_t);
+extern void pg_rele(struct pgrp * pgrp);
+extern struct session * session_find_internal(pid_t sessid);
+extern struct pgrp * proc_pgrp(proc_t);
+extern struct pgrp * tty_pgrp(struct tty * tp);
+extern struct pgrp * pgfind_internal(pid_t);
+extern struct session * proc_session(proc_t);
+extern void pgrp_lock(struct pgrp * pgrp);
+extern void pgrp_unlock(struct pgrp * pgrp);
+extern void session_lock(struct session * sess);
+extern void session_unlock(struct session * sess);
+extern struct session * pgrp_session(struct pgrp * pgrp);
+extern void	session_rele(struct session *sess);
+extern int isbackground(proc_t p, struct tty  *tp);
+extern proc_t proc_parent(proc_t);
+extern proc_t proc_parentholdref(proc_t);
+extern int proc_parentdropref(proc_t, int);
+int     itimerfix(struct timeval *tv);
+int     itimerdecr(struct proc * p, struct itimerval *itp, int usec);
+void proc_signalstart(struct proc *, int locked);
+void proc_signalend(struct proc *, int locked);
+void proc_transstart(struct proc *, int locked);
+void proc_transend(struct proc *, int locked);
+void proc_transwait(struct proc *, int locked);
+void  proc_rele_locked(struct proc *  p);
+void proc_knote(struct proc * p, long hint);
+void workqueue_init_lock(proc_t p);
+void workqueue_destroy_lock(proc_t p);
 #endif	/* !_SYS_PROC_INTERNAL_H_ */
diff --git a/bsd/sys/protosw.h b/bsd/sys/protosw.h
index 0609553ac..aaa56fe74 100644
--- a/bsd/sys/protosw.h
+++ b/bsd/sys/protosw.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -263,7 +269,7 @@ struct pr_usrreqs {
 #define	PRUS_OOB	0x1
 #define	PRUS_EOF	0x2
 #define	PRUS_MORETOCOME	0x4
-	int	(*pru_sense)(struct socket *so, struct stat *sb);
+	int	(*pru_sense)(struct socket *so, void  *sb, int isstat64);
 	int	(*pru_shutdown)(struct socket *so);
 	int	(*pru_sockaddr)(struct socket *so, struct sockaddr **nam);
 	 
@@ -310,7 +316,7 @@ extern int	pru_rcvoob_notsupp(struct socket *so, struct mbuf *m,
 extern int	pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, 
 				 struct sockaddr *addr, struct mbuf *control,
 				 struct proc *p);
-extern int	pru_sense_null(struct socket *so, struct stat *sb);
+extern int	pru_sense_null(struct socket *so, void * sb, int isstat64);
 extern int	pru_shutdown_notsupp(struct socket *so);
 extern int	pru_sockaddr_notsupp(struct socket *so, 
 				     struct sockaddr **nam);
@@ -400,6 +406,7 @@ char	*prcorequests[] = {
 #ifdef KERNEL
 
 __BEGIN_DECLS
+void	domaininit(void);
 
 void	pfctlinput(int, struct sockaddr *);
 void	pfctlinput2(int, struct sockaddr *, void *);
diff --git a/bsd/sys/pthread_internal.h b/bsd/sys/pthread_internal.h
new file mode 100644
index 000000000..2f37bf1a8
--- /dev/null
+++ b/bsd/sys/pthread_internal.h
@@ -0,0 +1,230 @@
+/*
+ * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _SYS_PTHREAD_INTERNAL_H_
+#define _SYS_PTHREAD_INTERNAL_H_
+
+#undef pthread_mutexattr_t;
+
+#include <kern/thread_call.h>
+
+/*
+ * Mutex attributes
+ */
+typedef struct 
+{
+	long sig;		     /* Unique signature for this structure */
+	int prioceiling;
+	u_int32_t protocol:2,		/* protocol attribute */
+		type:2,			/* mutex type */
+		pshared:2,
+		rfu:26;
+} pthread_mutexattr_t;
+
+#undef pthread_mutex_t
+/*
+ * Mutex variables
+ */
+typedef struct _pthread_mutex
+{
+	long	       sig;	      /* Unique signature for this structure */
+	lck_mtx_t *	mutex;		/* the kernel internal mutex */
+	lck_mtx_t *	lock;
+	thread_t      owner;	      /* Which thread has this mutex locked */
+	proc_t      owner_proc;	      /* Which thread has this mutex locked */
+	u_int32_t	 protocol:2,		/* protocol */
+		type:2,			/* mutex type */
+		pshared:2,			/* mutex type */
+		refcount:10,
+		lock_count:16;
+	int16_t       prioceiling;
+	int16_t	       priority;      /* Priority to restore when mutex unlocked */
+} pthread_mutex_t;
+
+#define MTX_LOCK lck_mtx_lock
+#define MTX_UNLOCK lck_mtx_unlock
+
+/*
+ * Condition variable attributes
+ */
+#undef pthread_condattr_t
+typedef struct 
+{
+	long	       sig;	     /* Unique signature for this structure */
+	u_int32_t	 pshared:2,		/* pshared */
+		unsupported:30;
+} pthread_condattr_t;
+
+/*
+ * Condition variables
+ */
+#undef pthread_cond_t
+typedef struct _pthread_cond
+{
+	long	       sig;	     /* Unique signature for this structure */
+	lck_mtx_t *	 lock;	     /* Used for internal mutex on structure */
+	u_int32_t	waiters:15,	/* Number of threads waiting */
+		   sigpending:15,	/* Number of outstanding signals */
+			pshared:2;
+	int 	refcount;
+	pthread_mutex_t * mutex;
+	proc_t      owner_proc;	      /* Which thread has this mutex locked */
+	semaphore_t	sem;
+} pthread_cond_t;
+#define COND_LOCK lck_mtx_lock
+#define COND_UNLOCK lck_mtx_unlock
+
+#undef pthread_rwlockattr_t
+typedef struct {
+	long	       sig;	      /* Unique signature for this structure */
+	int             pshared;
+	int		rfu[2];		/* reserved for future use */
+} pthread_rwlockattr_t;
+
+#undef pthread_rwlock_t
+typedef struct {
+	long 		sig;
+	lck_rw_t * rwlock;
+	int             pshared;
+	thread_t		owner;
+	int	rfu[2];
+} pthread_rwlock_t;
+
+#define _PTHREAD_NO_SIG			0x00000000
+#define _PTHREAD_MUTEX_ATTR_SIG		0x4D545841  /* 'MTXA' */
+#define _PTHREAD_MUTEX_SIG		0x4D555458  /* 'MUTX' */
+#define _PTHREAD_MUTEX_SIG_init		0x32AAABA7  /* [almost] ~'MUTX' */
+#define _PTHREAD_COND_ATTR_SIG		0x434E4441  /* 'CNDA' */
+#define _PTHREAD_COND_SIG		0x434F4E44  /* 'COND' */
+#define _PTHREAD_COND_SIG_init		0x3CB0B1BB  /* [almost] ~'COND' */
+#define _PTHREAD_ATTR_SIG		0x54484441  /* 'THDA' */
+#define _PTHREAD_ONCE_SIG		0x4F4E4345  /* 'ONCE' */
+#define _PTHREAD_ONCE_SIG_init		0x30B1BCBA  /* [almost] ~'ONCE' */
+#define _PTHREAD_SIG			0x54485244  /* 'THRD' */
+#define _PTHREAD_RWLOCK_ATTR_SIG	0x52574C41  /* 'RWLA' */
+#define _PTHREAD_RWLOCK_SIG		0x52574C4B  /* 'RWLK' */
+#define _PTHREAD_RWLOCK_SIG_init	0x2DA8B3B4  /* [almost] ~'RWLK' */
+
+#define _PTHREAD_KERN_COND_SIG		0x12345678  /*  */
+#define _PTHREAD_KERN_MUTEX_SIG		0x34567812  /*  */
+#define _PTHREAD_KERN_RWLOCK_SIG	0x56781234  /*  */
+
+
+#define PTHREAD_PROCESS_SHARED 1
+#define PTHREAD_PROCESS_PRIVATE 2
+
+#define WORKQUEUE_MAXTHREADS 64
+#define WORKITEM_SIZE 64
+#define WORKQUEUE_NUMPRIOS 5
+
+struct threadlist {
+	TAILQ_ENTRY(threadlist) th_entry;
+	thread_t th_thread;
+	int	 th_flags;
+	uint32_t th_unparked;
+	uint32_t th_affinity_tag;
+	struct workqueue *th_workq;
+	mach_vm_size_t th_stacksize;
+	mach_vm_size_t th_allocsize;
+	mach_vm_offset_t th_stackaddr;
+	mach_port_t th_thport;
+};
+#define TH_LIST_INITED 		0x01
+#define TH_LIST_RUNNING 	0x02
+#define TH_LIST_BLOCKED 	0x04
+#define TH_LIST_SUSPENDED 	0x08
+
+struct workitem {
+	TAILQ_ENTRY(workitem) wi_entry;
+	user_addr_t wi_item;
+};
+
+struct workitemlist {
+	TAILQ_HEAD(, workitem) wl_itemlist;
+	TAILQ_HEAD(, workitem) wl_freelist;
+};
+
+
+struct workqueue {
+	struct workitem wq_array[WORKITEM_SIZE * WORKQUEUE_NUMPRIOS];
+        proc_t		wq_proc;
+        vm_map_t	wq_map;
+        task_t		wq_task;
+        thread_call_t	wq_timer_call;
+	int 		wq_flags;
+        int		wq_itemcount;
+        struct timeval	wq_lastran_ts;
+        struct timeval	wq_reduce_ts;
+        uint32_t	wq_stalled_count;
+        uint32_t	wq_max_threads_scheduled;
+        uint32_t	wq_affinity_max;
+        uint32_t	wq_threads_scheduled;
+	uint32_t	wq_nthreads;
+        uint32_t	wq_nextaffinitytag;
+	struct workitemlist  wq_list[WORKQUEUE_NUMPRIOS]; /* prio based item list */
+	TAILQ_HEAD(, threadlist) wq_thrunlist;
+	TAILQ_HEAD(wq_thidlelist, threadlist) * wq_thidlelist;
+        uint32_t      *	wq_thactivecount;
+        uint32_t      *	wq_thcount;
+};
+#define WQ_LIST_INITED		0x01
+#define WQ_BUSY			0x02
+#define WQ_TIMER_RUNNING	0x04
+#define WQ_TIMER_WATCH		0x08
+#define WQ_ADD_TO_POOL		0x10
+
+#define WQ_STALLED_WINDOW_USECS		20000
+#define WQ_REDUCE_POOL_WINDOW_USECS	3000000
+#define WQ_MAX_RUN_LATENCY_USECS	500
+#define	WQ_TIMER_INTERVAL_MSECS		40
+
+/* workq_ops commands */
+#define WQOPS_QUEUE_ADD 1
+#define WQOPS_QUEUE_REMOVE 2
+#define WQOPS_THREAD_RETURN 4
+
+#define PTH_DEFAULT_STACKSIZE 512*1024
+#define PTH_DEFAULT_GUARDSIZE 4*1024
+#define MAX_PTHREAD_SIZE 64*1024
+
+void workqueue_exit(struct proc *);
+
+pthread_mutex_t * pthread_id_to_mutex(int mutexid);
+int	pthread_id_mutex_add(pthread_mutex_t *);
+void	pthread_id_mutex_remove(int);
+void pthread_mutex_release(pthread_mutex_t *);
+pthread_cond_t * pthread_id_to_cond(int condid);
+int	pthread_id_cond_add(pthread_cond_t *);
+void	pthread_id_cond_remove(int);
+void pthread_cond_release(pthread_cond_t *);
+
+void pthread_list_lock(void);
+void pthread_list_unlock(void);
+
+#endif /* _SYS_PTHREAD_INTERNAL_H_ */
+
diff --git a/bsd/sys/ptrace.h b/bsd/sys/ptrace.h
index 61ae0448a..fda2515a5 100644
--- a/bsd/sys/ptrace.h
+++ b/bsd/sys/ptrace.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -84,13 +90,7 @@
 
 __BEGIN_DECLS
 
-#ifdef KERNEL
-#ifdef __APPLE_API_PRIVATE
-
-void	proc_reparent(struct proc *child, struct proc *newparent);
-
-#endif /* __APPLE_API_PRIVATE */
-#else /* !KERNEL */
+#ifndef KERNEL
 
 int	ptrace(int _request, pid_t _pid, caddr_t _addr, int _data);
 
diff --git a/bsd/sys/ptrace_internal.h b/bsd/sys/ptrace_internal.h
deleted file mode 100644
index 6d2f11907..000000000
--- a/bsd/sys/ptrace_internal.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
-/*
- * Copyright (c) 1982, 1986, 1993, 1994
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)uio.h	8.5 (Berkeley) 2/22/94
- */
-
-#ifndef _SYS_PTRACE_INTERNAL_H_
-#define	_SYS_PTRACE_INTERNAL_H_
-
-#include <sys/appleapiopts.h>
-
-#ifdef KERNEL_PRIVATE
-#include <sys/ptrace.h>
-
-/*
- * Additional request flags used by shipping 3rd party products that have been
- * patching ptrace.  We should be able to remove these additional requests once
- * the 3rd party products move to the KPis introduced in Tiger.
- */
-
-#define	PT_VENDOR_REQUEST1	5561	/* reserved for 3rd party vendor */
-
-
-__BEGIN_DECLS
-
-/*
- * WARNING - these are temporary KPI that allow binary compatibility with
- * shipping product that must patch ptrace.  These KPI will be removed in the
- * next system release that follows Tiger.  radar - 3928003
- *
- * temp_patch_ptrace - patch ptrace using new_ptrace as current implementation.  
- *		Returns the address of the original ptrace implementation.
- *
- * temp_unpatch_ptrace - restore ptrace to the original implementation.  Caller
- * 		must insure all inflight ptrace requests have drained before their kext
- *		is unloaded.
- */
-uintptr_t temp_patch_ptrace(uintptr_t new_ptrace);
-void temp_unpatch_ptrace(void);
-
-__END_DECLS
-
-
-
-#endif /* KERNEL */
-#endif /* !_SYS_PTRACE_INTERNAL_H_ */
diff --git a/bsd/sys/queue.h b/bsd/sys/queue.h
index feb16f76e..a8e96cd4c 100644
--- a/bsd/sys/queue.h
+++ b/bsd/sys/queue.h
@@ -1,25 +1,31 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
+/*-
  * Copyright (c) 1991, 1993
  *	The Regents of the University of California.  All rights reserved.
  *
@@ -31,10 +37,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
  * 4. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
@@ -59,7 +61,7 @@
 
 /*
  * This file defines five types of data structures: singly-linked lists,
- * slingly-linked tail queues, lists, tail queues, and circular queues.
+ * singly-linked tail queues, lists, tail queues, and circular queues.
  *
  * A singly-linked list is headed by a single forward pointer. The elements
  * are singly linked for minimum space and pointer manipulation overhead at
@@ -94,7 +96,7 @@
  * linked so that an arbitrary element can be removed without a need to
  * traverse the list. New elements can be added to the list before or
  * after an existing element, at the head of the list, or at the end of
- * the list. A tail queue may only be traversed in the forward direction.
+ * the list. A tail queue may be traversed in either direction.
  *
  * A circle queue is headed by a pair of pointers, one to the head of the
  * list and the other to the tail of the list. The elements are doubly
@@ -103,42 +105,87 @@
  * an existing element, at the head of the list, or at the end of the list.
  * A circle queue may be traversed in either direction, but has a more
  * complex end of list detection.
+ * Note that circle queues are deprecated, because, as the removal log
+ * in FreeBSD states, "CIRCLEQs are a disgrace to everything Knuth taught
+ * us in Volume 1 Chapter 2. [...] Use TAILQ instead, it provides the same
+ * functionality." Code using them will continue to compile, but they
+ * are no longer documented on the man page.
  *
  * For details on the use of these macros, see the queue(3) manual page.
  *
  *
- *			SLIST	LIST	STAILQ	TAILQ	CIRCLEQ
- * _HEAD		+	+	+	+	+
- * _ENTRY		+	+	+	+	+
- * _INIT		+	+	+	+	+
- * _EMPTY		+	+	+	+	+
- * _FIRST		+	+	+	+	+
- * _NEXT		+	+	+	+	+
- * _PREV		-	-	-	+	+
- * _LAST		-	-	+	+	+
- * _FOREACH		+	+	-	+	+
- * _INSERT_HEAD		+	+	+	+	+
- * _INSERT_BEFORE	-	+	-	+	+
- * _INSERT_AFTER	+	+	+	+	+
- * _INSERT_TAIL		-	-	+	+	+
- * _REMOVE_HEAD		+	-	+	-	-
- * _REMOVE		+	+	+	+	+
+ *				SLIST	LIST	STAILQ	TAILQ	CIRCLEQ
+ * _HEAD			+	+	+	+	+
+ * _HEAD_INITIALIZER		+	+	+	+	-
+ * _ENTRY			+	+	+	+	+
+ * _INIT			+	+	+	+	+
+ * _EMPTY			+	+	+	+	+
+ * _FIRST			+	+	+	+	+
+ * _NEXT			+	+	+	+	+
+ * _PREV			-	-	-	+	+
+ * _LAST			-	-	+	+	+
+ * _FOREACH			+	+	+	+	+
+ * _FOREACH_SAFE		+	+	+	+	-
+ * _FOREACH_REVERSE		-	-	-	+	-
+ * _FOREACH_REVERSE_SAFE	-	-	-	+	-
+ * _INSERT_HEAD			+	+	+	+	+
+ * _INSERT_BEFORE		-	+	-	+	+
+ * _INSERT_AFTER		+	+	+	+	+
+ * _INSERT_TAIL			-	-	+	+	+
+ * _CONCAT			-	-	+	+	-
+ * _REMOVE_HEAD			+	-	+	-	-
+ * _REMOVE			+	+	+	+	+
  *
  */
+#ifdef QUEUE_MACRO_DEBUG
+/* Store the last 2 places the queue element or head was altered */
+struct qm_trace {
+	char * lastfile;
+	int lastline;
+	char * prevfile;
+	int prevline;
+};
+
+#define	TRACEBUF	struct qm_trace trace;
+#define	TRASHIT(x)	do {(x) = (void *)-1;} while (0)
+
+#define	QMD_TRACE_HEAD(head) do {					\
+	(head)->trace.prevline = (head)->trace.lastline;		\
+	(head)->trace.prevfile = (head)->trace.lastfile;		\
+	(head)->trace.lastline = __LINE__;				\
+	(head)->trace.lastfile = __FILE__;				\
+} while (0)
+
+#define	QMD_TRACE_ELEM(elem) do {					\
+	(elem)->trace.prevline = (elem)->trace.lastline;		\
+	(elem)->trace.prevfile = (elem)->trace.lastfile;		\
+	(elem)->trace.lastline = __LINE__;				\
+	(elem)->trace.lastfile = __FILE__;				\
+} while (0)
+
+#else
+#define	QMD_TRACE_ELEM(elem)
+#define	QMD_TRACE_HEAD(head)
+#define	TRACEBUF
+#define	TRASHIT(x)
+#endif	/* QUEUE_MACRO_DEBUG */
 
 /*
- * Singly-linked List definitions.
+ * Singly-linked List declarations.
  */
-#define SLIST_HEAD(name, type)						\
+#define	SLIST_HEAD(name, type)						\
 struct name {								\
 	struct type *slh_first;	/* first element */			\
 }
- 
-#define SLIST_ENTRY(type)						\
+
+#define	SLIST_HEAD_INITIALIZER(head)					\
+	{ NULL }
+
+#define	SLIST_ENTRY(type)						\
 struct {								\
 	struct type *sle_next;	/* next element */			\
 }
- 
+
 /*
  * Singly-linked List functions.
  */
@@ -146,55 +193,68 @@ struct {								\
 
 #define	SLIST_FIRST(head)	((head)->slh_first)
 
-#define SLIST_FOREACH(var, head, field)					\
-	for((var) = (head)->slh_first; (var); (var) = (var)->field.sle_next)
+#define	SLIST_FOREACH(var, head, field)					\
+	for ((var) = SLIST_FIRST((head));				\
+	    (var);							\
+	    (var) = SLIST_NEXT((var), field))
 
-#define SLIST_INIT(head) {						\
-	(head)->slh_first = NULL;					\
-}
+#define	SLIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = SLIST_FIRST((head));				\
+	    (var) && ((tvar) = SLIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
 
-#define SLIST_INSERT_AFTER(slistelm, elm, field) do {			\
-	(elm)->field.sle_next = (slistelm)->field.sle_next;		\
-	(slistelm)->field.sle_next = (elm);				\
-} while (0)
+#define	SLIST_FOREACH_PREVPTR(var, varp, head, field)			\
+	for ((varp) = &SLIST_FIRST((head));				\
+	    ((var) = *(varp)) != NULL;					\
+	    (varp) = &SLIST_NEXT((var), field))
 
-#define SLIST_INSERT_HEAD(head, elm, field) do {			\
-	(elm)->field.sle_next = (head)->slh_first;			\
-	(head)->slh_first = (elm);					\
+#define	SLIST_INIT(head) do {						\
+	SLIST_FIRST((head)) = NULL;					\
 } while (0)
 
-#define SLIST_NEXT(elm, field)	((elm)->field.sle_next)
+#define	SLIST_INSERT_AFTER(slistelm, elm, field) do {			\
+	SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field);	\
+	SLIST_NEXT((slistelm), field) = (elm);				\
+} while (0)
 
-#define SLIST_REMOVE_HEAD(head, field) do {				\
-	(head)->slh_first = (head)->slh_first->field.sle_next;		\
+#define	SLIST_INSERT_HEAD(head, elm, field) do {			\
+	SLIST_NEXT((elm), field) = SLIST_FIRST((head));			\
+	SLIST_FIRST((head)) = (elm);					\
 } while (0)
 
-#define SLIST_REMOVE(head, elm, type, field) do {			\
-	if ((head)->slh_first == (elm)) {				\
+#define	SLIST_NEXT(elm, field)	((elm)->field.sle_next)
+
+#define	SLIST_REMOVE(head, elm, type, field) do {			\
+	if (SLIST_FIRST((head)) == (elm)) {				\
 		SLIST_REMOVE_HEAD((head), field);			\
 	}								\
 	else {								\
-		struct type *curelm = (head)->slh_first;		\
-		while( curelm->field.sle_next != (elm) )		\
-			curelm = curelm->field.sle_next;		\
-		curelm->field.sle_next =				\
-		    curelm->field.sle_next->field.sle_next;		\
+		struct type *curelm = SLIST_FIRST((head));		\
+		while (SLIST_NEXT(curelm, field) != (elm))		\
+			curelm = SLIST_NEXT(curelm, field);		\
+		SLIST_NEXT(curelm, field) =				\
+		    SLIST_NEXT(SLIST_NEXT(curelm, field), field);	\
 	}								\
+	TRASHIT((elm)->field.sle_next);					\
+} while (0)
+
+#define	SLIST_REMOVE_HEAD(head, field) do {				\
+	SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field);	\
 } while (0)
 
 /*
- * Singly-linked Tail queue definitions.
+ * Singly-linked Tail queue declarations.
  */
-#define STAILQ_HEAD(name, type)						\
+#define	STAILQ_HEAD(name, type)						\
 struct name {								\
 	struct type *stqh_first;/* first element */			\
 	struct type **stqh_last;/* addr of last next element */		\
 }
 
-#define STAILQ_HEAD_INITIALIZER(head)					\
+#define	STAILQ_HEAD_INITIALIZER(head)					\
 	{ NULL, &(head).stqh_first }
 
-#define STAILQ_ENTRY(type)						\
+#define	STAILQ_ENTRY(type)						\
 struct {								\
 	struct type *stqe_next;	/* next element */			\
 }
@@ -202,74 +262,98 @@ struct {								\
 /*
  * Singly-linked Tail queue functions.
  */
-#define STAILQ_EMPTY(head) ((head)->stqh_first == NULL)
-
-#define	STAILQ_INIT(head) do {						\
-	(head)->stqh_first = NULL;					\
-	(head)->stqh_last = &(head)->stqh_first;			\
+#define	STAILQ_CONCAT(head1, head2) do {				\
+	if (!STAILQ_EMPTY((head2))) {					\
+		*(head1)->stqh_last = (head2)->stqh_first;		\
+		(head1)->stqh_last = (head2)->stqh_last;		\
+		STAILQ_INIT((head2));					\
+	}								\
 } while (0)
 
-#define STAILQ_FIRST(head)	((head)->stqh_first)
-#define STAILQ_LAST(head)	(*(head)->stqh_last)
+#define	STAILQ_EMPTY(head)	((head)->stqh_first == NULL)
 
-#define STAILQ_INSERT_HEAD(head, elm, field) do {			\
-	if (((elm)->field.stqe_next = (head)->stqh_first) == NULL)	\
-		(head)->stqh_last = &(elm)->field.stqe_next;		\
-	(head)->stqh_first = (elm);					\
-} while (0)
+#define	STAILQ_FIRST(head)	((head)->stqh_first)
+
+#define	STAILQ_FOREACH(var, head, field)				\
+	for((var) = STAILQ_FIRST((head));				\
+	   (var);							\
+	   (var) = STAILQ_NEXT((var), field))
 
-#define STAILQ_INSERT_TAIL(head, elm, field) do {			\
-	(elm)->field.stqe_next = NULL;					\
-	*(head)->stqh_last = (elm);					\
-	(head)->stqh_last = &(elm)->field.stqe_next;			\
-} while (0)
 
-#define STAILQ_INSERT_AFTER(head, tqelm, elm, field) do {		\
-	if (((elm)->field.stqe_next = (tqelm)->field.stqe_next) == NULL)\
-		(head)->stqh_last = &(elm)->field.stqe_next;		\
-	(tqelm)->field.stqe_next = (elm);				\
+#define	STAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = STAILQ_FIRST((head));				\
+	    (var) && ((tvar) = STAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	STAILQ_INIT(head) do {						\
+	STAILQ_FIRST((head)) = NULL;					\
+	(head)->stqh_last = &STAILQ_FIRST((head));			\
 } while (0)
 
-#define STAILQ_NEXT(elm, field)	((elm)->field.stqe_next)
+#define	STAILQ_INSERT_AFTER(head, tqelm, elm, field) do {		\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_NEXT((tqelm), field) = (elm);				\
+} while (0)
 
-#define STAILQ_REMOVE_HEAD(head, field) do {				\
-	if (((head)->stqh_first =					\
-	     (head)->stqh_first->field.stqe_next) == NULL)		\
-		(head)->stqh_last = &(head)->stqh_first;		\
+#define	STAILQ_INSERT_HEAD(head, elm, field) do {			\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_FIRST((head)) = (elm);					\
 } while (0)
 
-#define STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do {			\
-	if (((head)->stqh_first = (elm)->field.stqe_next) == NULL)	\
-		(head)->stqh_last = &(head)->stqh_first;		\
+#define	STAILQ_INSERT_TAIL(head, elm, field) do {			\
+	STAILQ_NEXT((elm), field) = NULL;				\
+	*(head)->stqh_last = (elm);					\
+	(head)->stqh_last = &STAILQ_NEXT((elm), field);			\
 } while (0)
 
+#define	STAILQ_LAST(head, type, field)					\
+	(STAILQ_EMPTY((head)) ?						\
+		NULL :							\
+	        ((struct type *)(void *)				\
+		((char *)((head)->stqh_last) - __offsetof(struct type, field))))
 
-#define STAILQ_REMOVE(head, elm, type, field) do {			\
-	if ((head)->stqh_first == (elm)) {				\
-		STAILQ_REMOVE_HEAD(head, field);			\
+#define	STAILQ_NEXT(elm, field)	((elm)->field.stqe_next)
+
+#define	STAILQ_REMOVE(head, elm, type, field) do {			\
+	if (STAILQ_FIRST((head)) == (elm)) {				\
+		STAILQ_REMOVE_HEAD((head), field);			\
 	}								\
 	else {								\
-		struct type *curelm = (head)->stqh_first;		\
-		while( curelm->field.stqe_next != (elm) )		\
-			curelm = curelm->field.stqe_next;		\
-		if((curelm->field.stqe_next =				\
-		    curelm->field.stqe_next->field.stqe_next) == NULL)	\
-			(head)->stqh_last = &(curelm)->field.stqe_next;	\
+		struct type *curelm = STAILQ_FIRST((head));		\
+		while (STAILQ_NEXT(curelm, field) != (elm))		\
+			curelm = STAILQ_NEXT(curelm, field);		\
+		if ((STAILQ_NEXT(curelm, field) =			\
+		     STAILQ_NEXT(STAILQ_NEXT(curelm, field), field)) == NULL)\
+			(head)->stqh_last = &STAILQ_NEXT((curelm), field);\
 	}								\
+	TRASHIT((elm)->field.stqe_next);				\
+} while (0)
+
+#define	STAILQ_REMOVE_HEAD(head, field) do {				\
+	if ((STAILQ_FIRST((head)) =					\
+	     STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL)		\
+		(head)->stqh_last = &STAILQ_FIRST((head));		\
+} while (0)
+
+#define	STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do {			\
+	if ((STAILQ_FIRST((head)) = STAILQ_NEXT((elm), field)) == NULL)	\
+		(head)->stqh_last = &STAILQ_FIRST((head));		\
 } while (0)
 
 /*
- * List definitions.
+ * List declarations.
  */
-#define LIST_HEAD(name, type)						\
+#define	LIST_HEAD(name, type)						\
 struct name {								\
 	struct type *lh_first;	/* first element */			\
 }
 
-#define LIST_HEAD_INITIALIZER(head)					\
+#define	LIST_HEAD_INITIALIZER(head)					\
 	{ NULL }
 
-#define LIST_ENTRY(type)						\
+#define	LIST_ENTRY(type)						\
 struct {								\
 	struct type *le_next;	/* next element */			\
 	struct type **le_prev;	/* address of previous next element */	\
@@ -279,134 +363,215 @@ struct {								\
  * List functions.
  */
 
-#define	LIST_EMPTY(head) ((head)->lh_first == NULL)
+#if (defined(_KERNEL) && defined(INVARIANTS)) || defined(QUEUE_MACRO_DEBUG)
+#define	QMD_LIST_CHECK_HEAD(head, field) do {				\
+	if (LIST_FIRST((head)) != NULL &&				\
+	    LIST_FIRST((head))->field.le_prev !=			\
+	     &LIST_FIRST((head)))					\
+		panic("Bad list head %p first->prev != head", (head));	\
+} while (0)
 
-#define LIST_FIRST(head)	((head)->lh_first)
+#define	QMD_LIST_CHECK_NEXT(elm, field) do {				\
+	if (LIST_NEXT((elm), field) != NULL &&				\
+	    LIST_NEXT((elm), field)->field.le_prev !=			\
+	     &((elm)->field.le_next))					\
+	     	panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_PREV(elm, field) do {				\
+	if (*(elm)->field.le_prev != (elm))				\
+		panic("Bad link elm %p prev->next != elm", (elm));	\
+} while (0)
+#else
+#define	QMD_LIST_CHECK_HEAD(head, field)
+#define	QMD_LIST_CHECK_NEXT(elm, field)
+#define	QMD_LIST_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) || QUEUE_MACRO_DEBUG */
 
-#define LIST_FOREACH(var, head, field)					\
-	for((var) = (head)->lh_first; (var); (var) = (var)->field.le_next)
+#define	LIST_EMPTY(head)	((head)->lh_first == NULL)
+
+#define	LIST_FIRST(head)	((head)->lh_first)
+
+#define	LIST_FOREACH(var, head, field)					\
+	for ((var) = LIST_FIRST((head));				\
+	    (var);							\
+	    (var) = LIST_NEXT((var), field))
+
+#define	LIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = LIST_FIRST((head));				\
+	    (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
 
 #define	LIST_INIT(head) do {						\
-	(head)->lh_first = NULL;					\
+	LIST_FIRST((head)) = NULL;					\
 } while (0)
 
-#define LIST_INSERT_AFTER(listelm, elm, field) do {			\
-	if (((elm)->field.le_next = (listelm)->field.le_next) != NULL)	\
-		(listelm)->field.le_next->field.le_prev =		\
-		    &(elm)->field.le_next;				\
-	(listelm)->field.le_next = (elm);				\
-	(elm)->field.le_prev = &(listelm)->field.le_next;		\
+#define	LIST_INSERT_AFTER(listelm, elm, field) do {			\
+	QMD_LIST_CHECK_NEXT(listelm, field);				\
+	if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\
+		LIST_NEXT((listelm), field)->field.le_prev =		\
+		    &LIST_NEXT((elm), field);				\
+	LIST_NEXT((listelm), field) = (elm);				\
+	(elm)->field.le_prev = &LIST_NEXT((listelm), field);		\
 } while (0)
 
-#define LIST_INSERT_BEFORE(listelm, elm, field) do {			\
+#define	LIST_INSERT_BEFORE(listelm, elm, field) do {			\
+	QMD_LIST_CHECK_PREV(listelm, field);				\
 	(elm)->field.le_prev = (listelm)->field.le_prev;		\
-	(elm)->field.le_next = (listelm);				\
+	LIST_NEXT((elm), field) = (listelm);				\
 	*(listelm)->field.le_prev = (elm);				\
-	(listelm)->field.le_prev = &(elm)->field.le_next;		\
+	(listelm)->field.le_prev = &LIST_NEXT((elm), field);		\
 } while (0)
 
-#define LIST_INSERT_HEAD(head, elm, field) do {				\
-	if (((elm)->field.le_next = (head)->lh_first) != NULL)		\
-		(head)->lh_first->field.le_prev = &(elm)->field.le_next;\
-	(head)->lh_first = (elm);					\
-	(elm)->field.le_prev = &(head)->lh_first;			\
+#define	LIST_INSERT_HEAD(head, elm, field) do {				\
+	QMD_LIST_CHECK_HEAD((head), field);				\
+	if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL)	\
+		LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\
+	LIST_FIRST((head)) = (elm);					\
+	(elm)->field.le_prev = &LIST_FIRST((head));			\
 } while (0)
 
-#define LIST_NEXT(elm, field)	((elm)->field.le_next)
+#define	LIST_NEXT(elm, field)	((elm)->field.le_next)
 
-#define LIST_REMOVE(elm, field) do {					\
-	if ((elm)->field.le_next != NULL)				\
-		(elm)->field.le_next->field.le_prev = 			\
+#define	LIST_REMOVE(elm, field) do {					\
+	QMD_LIST_CHECK_NEXT(elm, field);				\
+	QMD_LIST_CHECK_PREV(elm, field);				\
+	if (LIST_NEXT((elm), field) != NULL)				\
+		LIST_NEXT((elm), field)->field.le_prev = 		\
 		    (elm)->field.le_prev;				\
-	*(elm)->field.le_prev = (elm)->field.le_next;			\
+	*(elm)->field.le_prev = LIST_NEXT((elm), field);		\
+	TRASHIT((elm)->field.le_next);					\
+	TRASHIT((elm)->field.le_prev);					\
 } while (0)
 
 /*
- * Tail queue definitions.
+ * Tail queue declarations.
  */
-#define TAILQ_HEAD(name, type)						\
+#define	TAILQ_HEAD(name, type)						\
 struct name {								\
 	struct type *tqh_first;	/* first element */			\
 	struct type **tqh_last;	/* addr of last next element */		\
+	TRACEBUF							\
 }
 
-#define TAILQ_HEAD_INITIALIZER(head)					\
+#define	TAILQ_HEAD_INITIALIZER(head)					\
 	{ NULL, &(head).tqh_first }
 
-#define TAILQ_ENTRY(type)						\
+#define	TAILQ_ENTRY(type)						\
 struct {								\
 	struct type *tqe_next;	/* next element */			\
 	struct type **tqe_prev;	/* address of previous next element */	\
+	TRACEBUF							\
 }
 
 /*
  * Tail queue functions.
  */
-#define	TAILQ_EMPTY(head) ((head)->tqh_first == NULL)
+#define	TAILQ_CONCAT(head1, head2, field) do {				\
+	if (!TAILQ_EMPTY(head2)) {					\
+		*(head1)->tqh_last = (head2)->tqh_first;		\
+		(head2)->tqh_first->field.tqe_prev = (head1)->tqh_last;	\
+		(head1)->tqh_last = (head2)->tqh_last;			\
+		TAILQ_INIT((head2));					\
+		QMD_TRACE_HEAD(head1);					\
+		QMD_TRACE_HEAD(head2);					\
+	}								\
+} while (0)
 
-#define TAILQ_FOREACH(var, head, field)					\
-	for (var = TAILQ_FIRST(head); var; var = TAILQ_NEXT(var, field))
+#define	TAILQ_EMPTY(head)	((head)->tqh_first == NULL)
 
-#define TAILQ_FOREACH_REVERSE(var, head, field, headname)					\
-	for (var = TAILQ_LAST(head, headname);		       \
-	       var; var = TAILQ_PREV(var, headname, field))
+#define	TAILQ_FIRST(head)	((head)->tqh_first)
 
-#define	TAILQ_FIRST(head) ((head)->tqh_first)
+#define	TAILQ_FOREACH(var, head, field)					\
+	for ((var) = TAILQ_FIRST((head));				\
+	    (var);							\
+	    (var) = TAILQ_NEXT((var), field))
 
-#define	TAILQ_LAST(head, headname) \
-	(*(((struct headname *)((head)->tqh_last))->tqh_last))
+#define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = TAILQ_FIRST((head));				\
+	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
 
-#define	TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
+#define	TAILQ_FOREACH_REVERSE(var, head, headname, field)		\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var);							\
+	    (var) = TAILQ_PREV((var), headname, field))
 
-#define TAILQ_PREV(elm, headname, field) \
-	(*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+#define	TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar)	\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1);	\
+	    (var) = (tvar))
 
 #define	TAILQ_INIT(head) do {						\
-	(head)->tqh_first = NULL;					\
-	(head)->tqh_last = &(head)->tqh_first;				\
+	TAILQ_FIRST((head)) = NULL;					\
+	(head)->tqh_last = &TAILQ_FIRST((head));			\
+	QMD_TRACE_HEAD(head);						\
 } while (0)
 
-#define TAILQ_INSERT_HEAD(head, elm, field) do {			\
-	if (((elm)->field.tqe_next = (head)->tqh_first) != NULL)	\
-		(head)->tqh_first->field.tqe_prev =			\
-		    &(elm)->field.tqe_next;				\
-	else								\
-		(head)->tqh_last = &(elm)->field.tqe_next;		\
-	(head)->tqh_first = (elm);					\
-	(elm)->field.tqe_prev = &(head)->tqh_first;			\
+#define	TAILQ_INSERT_AFTER(head, listelm, elm, field) do {		\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
+		    &TAILQ_NEXT((elm), field);				\
+	else {								\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);		\
+		QMD_TRACE_HEAD(head);					\
+	}								\
+	TAILQ_NEXT((listelm), field) = (elm);				\
+	(elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field);		\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+	QMD_TRACE_ELEM(&listelm->field);				\
 } while (0)
 
-#define TAILQ_INSERT_TAIL(head, elm, field) do {			\
-	(elm)->field.tqe_next = NULL;					\
-	(elm)->field.tqe_prev = (head)->tqh_last;			\
-	*(head)->tqh_last = (elm);					\
-	(head)->tqh_last = &(elm)->field.tqe_next;			\
+#define	TAILQ_INSERT_BEFORE(listelm, elm, field) do {			\
+	(elm)->field.tqe_prev = (listelm)->field.tqe_prev;		\
+	TAILQ_NEXT((elm), field) = (listelm);				\
+	*(listelm)->field.tqe_prev = (elm);				\
+	(listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field);		\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+	QMD_TRACE_ELEM(&listelm->field);				\
 } while (0)
 
-#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do {		\
-	if (((elm)->field.tqe_next = (listelm)->field.tqe_next) != NULL)\
-		(elm)->field.tqe_next->field.tqe_prev = 		\
-		    &(elm)->field.tqe_next;				\
+#define	TAILQ_INSERT_HEAD(head, elm, field) do {			\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL)	\
+		TAILQ_FIRST((head))->field.tqe_prev =			\
+		    &TAILQ_NEXT((elm), field);				\
 	else								\
-		(head)->tqh_last = &(elm)->field.tqe_next;		\
-	(listelm)->field.tqe_next = (elm);				\
-	(elm)->field.tqe_prev = &(listelm)->field.tqe_next;		\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);		\
+	TAILQ_FIRST((head)) = (elm);					\
+	(elm)->field.tqe_prev = &TAILQ_FIRST((head));			\
+	QMD_TRACE_HEAD(head);						\
+	QMD_TRACE_ELEM(&(elm)->field);					\
 } while (0)
 
-#define TAILQ_INSERT_BEFORE(listelm, elm, field) do {			\
-	(elm)->field.tqe_prev = (listelm)->field.tqe_prev;		\
-	(elm)->field.tqe_next = (listelm);				\
-	*(listelm)->field.tqe_prev = (elm);				\
-	(listelm)->field.tqe_prev = &(elm)->field.tqe_next;		\
+#define	TAILQ_INSERT_TAIL(head, elm, field) do {			\
+	TAILQ_NEXT((elm), field) = NULL;				\
+	(elm)->field.tqe_prev = (head)->tqh_last;			\
+	*(head)->tqh_last = (elm);					\
+	(head)->tqh_last = &TAILQ_NEXT((elm), field);			\
+	QMD_TRACE_HEAD(head);						\
+	QMD_TRACE_ELEM(&(elm)->field);					\
 } while (0)
 
-#define TAILQ_REMOVE(head, elm, field) do {				\
-	if (((elm)->field.tqe_next) != NULL)				\
-		(elm)->field.tqe_next->field.tqe_prev = 		\
+#define	TAILQ_LAST(head, headname)					\
+	(*(((struct headname *)((head)->tqh_last))->tqh_last))
+
+#define	TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
+
+#define	TAILQ_PREV(elm, headname, field)				\
+	(*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+
+#define	TAILQ_REMOVE(head, elm, field) do {				\
+	if ((TAILQ_NEXT((elm), field)) != NULL)				\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
 		    (elm)->field.tqe_prev;				\
-	else								\
+	else {								\
 		(head)->tqh_last = (elm)->field.tqe_prev;		\
-	*(elm)->field.tqe_prev = (elm)->field.tqe_next;			\
+		QMD_TRACE_HEAD(head);					\
+	}								\
+	*(elm)->field.tqe_prev = TAILQ_NEXT((elm), field);		\
+	TRASHIT((elm)->field.tqe_next);					\
+	TRASHIT((elm)->field.tqe_prev);					\
+	QMD_TRACE_ELEM(&(elm)->field);					\
 } while (0)
 
 /*
@@ -500,7 +665,7 @@ struct {								\
 		    (elm)->field.cqe_next;				\
 } while (0)
 
-#ifdef KERNEL
+#ifdef _KERNEL
 
 #if NOTFB31
 
@@ -514,12 +679,13 @@ struct quehead {
 	struct quehead *qh_rlink;
 };
 
-#ifdef	__GNUC__
+#ifdef __GNUC__
 
 static __inline void
 insque(void *a, void *b)
 {
-	struct quehead *element = a, *head = b;
+	struct quehead *element = (struct quehead *)a,
+		 *head = (struct quehead *)b;
 
 	element->qh_link = head->qh_link;
 	element->qh_rlink = head;
@@ -530,7 +696,7 @@ insque(void *a, void *b)
 static __inline void
 remque(void *a)
 {
-	struct quehead *element = a;
+	struct quehead *element = (struct quehead *)a;
 
 	element->qh_link->qh_rlink = element->qh_rlink;
 	element->qh_rlink->qh_link = element->qh_link;
@@ -545,6 +711,6 @@ void	remque(void *a);
 #endif /* __GNUC__ */
 
 #endif
-#endif /* KERNEL */
+#endif /* _KERNEL */
 
 #endif /* !_SYS_QUEUE_H_ */
diff --git a/bsd/sys/quota.h b/bsd/sys/quota.h
index b2b8ebc89..b1a2db3aa 100644
--- a/bsd/sys/quota.h
+++ b/bsd/sys/quota.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -238,7 +244,7 @@ dqhashshift(u_long size)
 
 #ifndef KERNEL
 __BEGIN_DECLS
-int quotactl(char *, int, int, caddr_t);
+int quotactl(const char *, int, int, caddr_t);
 __END_DECLS
 #endif /* !KERNEL */
 
@@ -252,7 +258,7 @@ __END_DECLS
 struct quotafile {
   	lck_mtx_t     qf_lock;	     /* quota file mutex */
 	struct vnode *qf_vp;         /* quota file vnode */
-	struct ucred *qf_cred;       /* quota file access cred */
+	kauth_cred_t  qf_cred;       /* quota file access cred */
 	int           qf_shift;      /* primary hash shift */
 	int           qf_maxentries; /* size of hash table (power of 2) */
 	int           qf_entrycnt;   /* count of active entries */
@@ -283,14 +289,15 @@ struct quotafile {
 struct dquot {
 	LIST_ENTRY(dquot) dq_hash;	/* hash list */
 	TAILQ_ENTRY(dquot) dq_freelist;	/* free list */
-	uint32_t  dq_cnt;		/* count of active references */
 	u_int16_t dq_flags;		/* flags, see below */
+	u_int16_t dq_cnt_unused;	/* Replaced by dq_cnt below */
         u_int16_t dq_lflags;		/* protected by the quota list lock */
 	u_int16_t dq_type;		/* quota type of this dquot */
 	u_int32_t dq_id;		/* identifier this applies to */
 	u_int32_t dq_index;		/* index into quota file */
 	struct	quotafile *dq_qfile;	/* quota file that this is taken from */
 	struct	dqblk dq_dqb;		/* actual usage & quotas */
+	uint32_t  dq_cnt;		/* count of active references */
 };
 
 /*
@@ -343,7 +350,9 @@ int	dqfileopen(struct quotafile *, int);
 void	dqfileclose(struct quotafile *, int);
 void	dqflush(struct vnode *);
 int	dqget(u_long, struct quotafile *, int, struct dquot **);
+void	dqhashinit(void);
 void	dqinit(void);
+int	dqisinitialized(void);
 void	dqref(struct dquot *);
 void	dqrele(struct dquot *);
 void	dqreclaim(struct dquot *);
diff --git a/bsd/sys/random.h b/bsd/sys/random.h
index db6e0f701..2df36fc21 100644
--- a/bsd/sys/random.h
+++ b/bsd/sys/random.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999, 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __SYS_RANDOM_H__
diff --git a/bsd/sys/reboot.h b/bsd/sys/reboot.h
index 4a2384d9e..83d48fbc4 100644
--- a/bsd/sys/reboot.h
+++ b/bsd/sys/reboot.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -80,6 +86,7 @@
 #define RB_DFLTROOT	0x20	/* use compiled-in rootdev */
 #define RB_ALTBOOT	0x40	/* use /boot.old vs /boot */
 #define RB_UNIPROC	0x80	/* don't start slaves */
+#define RB_SAFEBOOT	0x100	/* booting safe */
 #define RB_UPSDELAY 0x200   /* Delays restart by 5 minutes */
 #define RB_PANIC	0	/* reboot due to panic */
 #define RB_BOOT		1	/* reboot due to boot() */
@@ -126,6 +133,11 @@
 
 #ifdef BSD_KERNEL_PRIVATE
 #include <machine/reboot.h>
+
+__BEGIN_DECLS
+void	boot(int, int, char *);
+__END_DECLS
+
 #endif /* BSD_KERNEL_PRIVATE */
 
 #endif	/* _SYS_REBOOT_H_ */
diff --git a/bsd/sys/resource.h b/bsd/sys/resource.h
index 45b65fa5d..747b77b68 100644
--- a/bsd/sys/resource.h
+++ b/bsd/sys/resource.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -65,17 +71,9 @@
 
 /* [XSI] The timeval structure shall be defined as described in
  * <sys/time.h>
- *
- * NB: We use __darwin_time_t and __darwin_suseconds_t here to avoid
- * improperly exposing time_t and suseconds_t into the namespace.
  */
-#ifndef _TIMEVAL
-#define _TIMEVAL
-struct timeval {
-	__darwin_time_t		tv_sec;		/* seconds */
-	__darwin_suseconds_t	tv_usec;	/* and microseconds */
-};
-#endif
+#define __need_struct_timeval
+#include <sys/_structs.h>
 
 /* The id_t type shall be defined as described in <sys/types.h> */
 #ifndef _ID_T
@@ -87,7 +85,7 @@ typedef __darwin_id_t	id_t;		/* can hold pid_t, gid_t, or uid_t */
 /*
  * Resource limit type (low 63 bits, excluding the sign bit)
  */
-typedef __int64_t	rlim_t;
+typedef __uint64_t	rlim_t;
 
 
 /*****
@@ -102,13 +100,22 @@ typedef __int64_t	rlim_t;
 #define	PRIO_PGRP	1		/* Second argument is a GID */
 #define	PRIO_USER	2		/* Second argument is a UID */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define	PRIO_DARWIN_THREAD	3		/* Second argument is always 0 (current thread) */
+
 /*
  * Range limitations for the value of the third parameter to setpriority().
  */
 #define	PRIO_MIN	-20
 #define	PRIO_MAX	20
-#endif	/* !_POSIX_C_SOURCE */
+
+/* use PRIO_DARWIN_BG to set the current thread into "background" state
+ * which lowers CPU, disk IO, and networking priorites until thread terminates
+ * or "background" state is revoked
+ */
+#define PRIO_DARWIN_BG 0x1000
+
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 
@@ -133,38 +140,36 @@ typedef __int64_t	rlim_t;
  *       is discouraged for standards compliant programs.
  */
 struct	rusage {
-	struct timeval ru_utime;	/* user time used */
-	struct timeval ru_stime;	/* system time used */
-#ifdef _POSIX_C_SOURCE
+	struct timeval ru_utime;	/* user time used (PL) */
+	struct timeval ru_stime;	/* system time used (PL) */
+#if defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE)
 	long	ru_opaque[14];		/* implementation defined */
-#else	/* !_POSIX_C_SOURCE */
+#else	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 	/*
 	 * Informational aliases for source compatibility with programs
 	 * that need more information than that provided by standards,
 	 * and which do not mind being OS-dependent.
 	 */
-	long	ru_maxrss;		/* max resident set size */
+	long	ru_maxrss;		/* max resident set size (PL) */
 #define	ru_first	ru_ixrss	/* internal: ruadd() range start */
-	long	ru_ixrss;		/* integral shared memory size */
-	long	ru_idrss;		/* integral unshared data " */
-	long	ru_isrss;		/* integral unshared stack " */
-	long	ru_minflt;		/* page reclaims */
-	long	ru_majflt;		/* page faults */
-	long	ru_nswap;		/* swaps */
-	long	ru_inblock;		/* block input operations */
-	long	ru_oublock;		/* block output operations */
-	long	ru_msgsnd;		/* messages sent */
-	long	ru_msgrcv;		/* messages received */
-	long	ru_nsignals;		/* signals received */
-	long	ru_nvcsw;		/* voluntary context switches */
+	long	ru_ixrss;		/* integral shared memory size (NU) */
+	long	ru_idrss;		/* integral unshared data (NU)  */
+	long	ru_isrss;		/* integral unshared stack (NU) */
+	long	ru_minflt;		/* page reclaims (NU) */
+	long	ru_majflt;		/* page faults (NU) */
+	long	ru_nswap;		/* swaps (NU) */
+	long	ru_inblock;		/* block input operations (atomic) */
+	long	ru_oublock;		/* block output operations (atomic) */
+	long	ru_msgsnd;		/* messages sent (atomic) */
+	long	ru_msgrcv;		/* messages received (atomic) */
+	long	ru_nsignals;		/* signals received (atomic) */
+	long	ru_nvcsw;		/* voluntary context switches (atomic) */
 	long	ru_nivcsw;		/* involuntary " */
 #define	ru_last		ru_nivcsw	/* internal: ruadd() range end */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 };
 
 
-
-// LP64todo - should this move?
 #ifdef KERNEL
 #include <machine/types.h>	/* user_time_t */
 
@@ -216,21 +221,22 @@ struct	user_rusage {
  * Possible values of the first parameter to getrlimit()/setrlimit(), to
  * indicate for which resource the operation is being performed.
  */
-#define	RLIMIT_CPU	0		/* cpu time per process, in ms */
+#define	RLIMIT_CPU	0		/* cpu time per process */
 #define	RLIMIT_FSIZE	1		/* file size */
 #define	RLIMIT_DATA	2		/* data segment size */
 #define	RLIMIT_STACK	3		/* stack size */
 #define	RLIMIT_CORE	4		/* core file size */
 #define	RLIMIT_AS	5		/* address space (resident set size) */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	RLIMIT_RSS	RLIMIT_AS	/* source compatibility alias */
 #define	RLIMIT_MEMLOCK	6		/* locked-in-memory address space */
 #define	RLIMIT_NPROC	7		/* number of processes */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	RLIMIT_NOFILE	8		/* number of open files */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	RLIM_NLIMITS	9		/* total number of resource limits */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+#define _RLIMIT_POSIX_FLAG	0x1000	/* Set bit for strict POSIX */
 
 /*
  * A structure representing a resource limit.  The address of an instance
@@ -241,15 +247,58 @@ struct rlimit {
 	rlim_t	rlim_max;		/* maximum value for rlim_cur */
 };
 
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+/* I/O type */
+#define IOPOL_TYPE_DISK	0
+
+/* scope */
+#define IOPOL_SCOPE_PROCESS   0
+#define IOPOL_SCOPE_THREAD    1
+
+/* I/O Priority */
+#define IOPOL_DEFAULT	0
+#define IOPOL_NORMAL	1
+#define IOPOL_PASSIVE	2
+#define IOPOL_THROTTLE	3
+
+#ifdef PRIVATE
+/*
+ * Structures for use in communicating via iopolicysys() between Lic and the
+ * kernel.  Not to be used by uesr programs directly.
+ */
+
+/*
+ * the command to iopolicysys()
+ */
+#define	IOPOL_CMD_GET		0x00000001	/* Get I/O policy */
+#define	IOPOL_CMD_SET		0x00000002	/* Set I/O policy */
+
+/*
+ * Second parameter to iopolicysys()
+ */
+struct _iopol_param_t {
+	int iop_scope;	/* current process or a thread */
+	int iop_iotype;
+	int iop_policy;
+};
+
+#endif	/* PRIVATE */
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
 
 #ifndef KERNEL
 
 __BEGIN_DECLS
 int	getpriority(int, id_t);
-int	getrlimit(int, struct rlimit *);
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+int	getiopolicy_np(int, int);
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
+int	getrlimit(int, struct rlimit *) __DARWIN_ALIAS(getrlimit);
 int	getrusage(int, struct rusage *);
 int	setpriority(int, id_t, int);
-int	setrlimit(int, const struct rlimit *);
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+int	setiopolicy_np(int, int, int);
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
+int	setrlimit(int, const struct rlimit *) __DARWIN_ALIAS(setrlimit);
 __END_DECLS
 
 #endif	/* !KERNEL */
diff --git a/bsd/sys/resourcevar.h b/bsd/sys/resourcevar.h
index 8593ed6ef..588847dc8 100644
--- a/bsd/sys/resourcevar.h
+++ b/bsd/sys/resourcevar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -60,7 +66,6 @@
 
 #include <sys/appleapiopts.h>
 
-#ifdef __APPLE_API_PRIVATE
 /*
  * Kernel per-process accounting / statistics
  * (not necessarily resident except when running).
@@ -68,7 +73,7 @@
 struct pstats {
 #define	pstat_startzero	p_ru
 	struct	rusage p_ru;		/* stats for this proc */
-	struct	rusage p_cru;		/* sum of stats for reaped children */
+	struct	rusage p_cru;		/* (PL) sum of stats for reaped children */
 
 	struct uprof {			/* profile arguments */
 		struct uprof *pr_next;  /* multiple prof buffers allowed */
@@ -79,12 +84,9 @@ struct pstats {
 		u_long	pr_addr;	/* temp storage for addr until AST */
 		u_long	pr_ticks;	/* temp storage for ticks until AST */
 	} p_prof;
-#define	pstat_endzero	pstat_startcopy
+#define	pstat_endzero	p_start
 
-#define	pstat_startcopy	p_timer
-	struct	itimerval p_timer[3];	/* virtual-time timers */
-#define	pstat_endcopy	p_start
-	struct	timeval p_start;	/* starting time */
+	struct  timeval p_start;	/* starting time ; compat only */
 #ifdef KERNEL
 	struct user_uprof {			    /* profile arguments */
 		struct user_uprof *pr_next;  /* multiple prof buffers allowed */
@@ -106,11 +108,14 @@ struct pstats {
  * and a copy must be made for the child of a new fork that isn't
  * sharing modifications to the limits.
  */
+/* 
+ * Modifications are done with the list lock held (p_limit as well)and access indv 
+ * limits can be done without limit as we keep the old copy in p_olimit. Which is 
+ * dropped in proc_exit. This way all access will have a valid kernel address
+ */
 struct plimit {
 	struct	rlimit pl_rlimit[RLIM_NLIMITS];
-#define	PL_SHAREMOD	0x01		/* modifications are shared */
-	int	p_lflags;
-	int	p_refcnt;		/* number of references */
+	int	pl_refcnt;		/* number of references */
 };
 
 #ifdef KERNEL
@@ -127,9 +132,13 @@ void	 addupc_task(struct proc *p, user_addr_t pc, u_int ticks);
 void	 calcru(struct proc *p, struct timeval *up, struct timeval *sp,
 	    struct timeval *ip);
 void	 ruadd(struct rusage *ru, struct rusage *ru2);
-struct plimit *limcopy(struct plimit *lim);
+void proc_limitget(proc_t p, int whichi, struct rlimit * limp);
+void proc_limitdrop(proc_t p, int exiting);
+void proc_limitfork(proc_t parent, proc_t child);
+int proc_limitreplace(proc_t p);
+void proc_limitblock(proc_t);
+void proc_limitunblock(proc_t);
 #endif /* KERNEL */
 
-#endif /* __APPLE_API_PRIVATE */
 
 #endif	/* !_SYS_RESOURCEVAR_H_ */
diff --git a/bsd/sys/sbuf.h b/bsd/sys/sbuf.h
new file mode 100644
index 000000000..86ff6eebd
--- /dev/null
+++ b/bsd/sys/sbuf.h
@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2000 Poul-Henning Kamp and Dag-Erling Codan Sm¿rgrav
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer
+ *    in this position and unchanged.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *      $FreeBSD: /repoman/r/ncvs/src/sys/sys/sbuf.h,v 1.14 2004/07/09 11:35:30 des Exp $
+ */
+
+#ifndef _SYS_SBUF_H_
+#define	_SYS_SBUF_H_
+
+#include <sys/_types.h>
+#include <stdarg.h>
+
+/*
+ * Structure definition
+ */
+struct sbuf {
+	char		*s_buf;		/* storage buffer */
+	void		*s_unused;	/* binary compatibility. */
+	int		 s_size;	/* size of storage buffer */
+	int		 s_len;		/* current length of string */
+#define	SBUF_FIXEDLEN	0x00000000	/* fixed length buffer (default) */
+#define	SBUF_AUTOEXTEND	0x00000001	/* automatically extend buffer */
+#define	SBUF_USRFLAGMSK 0x0000ffff	/* mask of flags the user may specify */
+#define	SBUF_DYNAMIC	0x00010000	/* s_buf must be freed */
+#define	SBUF_FINISHED	0x00020000	/* set by sbuf_finish() */
+#define	SBUF_OVERFLOWED	0x00040000	/* sbuf overflowed */
+#define	SBUF_DYNSTRUCT	0x00080000	/* sbuf must be freed */
+	int		 s_flags;	/* flags */
+};
+
+__BEGIN_DECLS
+/*
+ * API functions
+ */
+struct sbuf	*sbuf_new(struct sbuf *, char *, int, int);
+void		 sbuf_clear(struct sbuf *);
+int		 sbuf_setpos(struct sbuf *, int);
+int		 sbuf_bcat(struct sbuf *, const void *, size_t);
+int		 sbuf_bcpy(struct sbuf *, const void *, size_t);
+int		 sbuf_cat(struct sbuf *, const char *);
+int		 sbuf_cpy(struct sbuf *, const char *);
+int		 sbuf_printf(struct sbuf *, const char *, ...) __printflike(2, 3);
+int		 sbuf_vprintf(struct sbuf *, const char *, va_list) __printflike(2, 0);
+int		 sbuf_putc(struct sbuf *, int);
+int		 sbuf_trim(struct sbuf *);
+int		 sbuf_overflowed(struct sbuf *);
+void		 sbuf_finish(struct sbuf *);
+char		*sbuf_data(struct sbuf *);
+int		 sbuf_len(struct sbuf *);
+int		 sbuf_done(struct sbuf *);
+void		 sbuf_delete(struct sbuf *);
+
+#ifdef KERNEL
+struct uio;
+struct sbuf	*sbuf_uionew(struct sbuf *, struct uio *, int *);
+int		 sbuf_bcopyin(struct sbuf *, const void *, size_t);
+int		 sbuf_copyin(struct sbuf *, const void *, size_t);
+#endif
+__END_DECLS
+
+#endif
diff --git a/bsd/sys/sdt.h b/bsd/sys/sdt.h
new file mode 100644
index 000000000..420956bdb
--- /dev/null
+++ b/bsd/sys/sdt.h
@@ -0,0 +1,41 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SYS_SDT_H
+#define	_SYS_SDT_H
+
+/*
+ * This is a wrapper header that wraps the mach visible sdt.h header so that
+ * the header file ends up vidible where software expects it to be.  We also
+ * do the C/C++ symbol wrapping here, since Mach headers are technically C
+ * interfaces.
+ */
+#include <sys/cdefs.h>
+__BEGIN_DECLS
+#include <mach/sdt.h>
+__END_DECLS
+
+#endif	/* _SYS_SDT_H */
diff --git a/bsd/sys/sdt_impl.h b/bsd/sys/sdt_impl.h
new file mode 100644
index 000000000..cbd117b61
--- /dev/null
+++ b/bsd/sys/sdt_impl.h
@@ -0,0 +1,134 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _SDT_IMPL_H
+#define	_SDT_IMPL_H
+
+/*
+ * This file has been created by splitting up the original DTrace sdt.h
+ * header. Keep the pragma notice here to allow version tracking.
+ */
+
+/* #pragma ident	"@(#)sdt.h	1.7	05/06/08 SMI" */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+extern const char *sdt_prefix;
+
+typedef struct sdt_probedesc {
+	char			*sdpd_name;	/* name of this probe */
+#if defined(__APPLE__)
+	char			*sdpd_func;	
+#endif /* __APPLE__ */
+	unsigned long		sdpd_offset;	/* offset of call in text */
+	struct sdt_probedesc	*sdpd_next;	/* next static probe */
+} sdt_probedesc_t;
+
+#ifdef	__cplusplus
+}
+#endif
+
+/* #pragma ident	"@(#)sdt_impl.h	1.3	05/06/08 SMI" */
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+#include <sys/dtrace.h>
+
+#if !defined(__APPLE__)
+#if defined(__i386) || defined(__amd64)
+typedef uint8_t sdt_instr_t;
+#else
+typedef uint32_t sdt_instr_t;
+#endif
+#else
+struct module {
+	int sdt_nprobes;
+	sdt_probedesc_t *sdt_probes;
+};
+
+extern int sdt_invop(uintptr_t, uintptr_t *, uintptr_t);
+
+void sdt_provide_module(void *, struct modctl *);
+void sdt_init(void);
+
+extern int          sdt_probetab_size;
+extern int          sdt_probetab_mask;
+#define SDT_ADDR2NDX(addr)  ((((uintptr_t)(addr)) >> 4) & sdt_probetab_mask)
+
+
+#if defined(__i386__) || defined(__x86_64__)
+typedef uint8_t sdt_instr_t;
+#elif defined(__ppc__) || defined(__ppc64__)
+typedef uint32_t sdt_instr_t;
+#else
+#error Unknown implementation
+#endif
+#endif /* __APPLE__ */
+
+typedef struct sdt_provider {
+	const char			*sdtp_name;	/* name of provider */
+	const char			*sdtp_prefix;	/* prefix for probe names */
+	dtrace_pattr_t		*sdtp_attr;	/* stability attributes */
+	dtrace_provider_id_t	sdtp_id;	/* provider ID */
+} sdt_provider_t;
+
+extern sdt_provider_t sdt_providers[];		/* array of providers */
+
+typedef struct sdt_probe {
+	sdt_provider_t	*sdp_provider;		/* provider */
+	char		*sdp_name;		/* name of probe */
+	int		sdp_namelen;		/* length of allocated name */
+	dtrace_id_t	sdp_id;			/* probe ID */
+	struct modctl	*sdp_ctl;		/* modctl for module */
+	int		sdp_loadcnt;		/* load count for module */
+	int		sdp_primary;		/* non-zero if primary mod */
+	sdt_instr_t	*sdp_patchpoint;	/* patch point */
+	sdt_instr_t	sdp_patchval;		/* instruction to patch */
+	sdt_instr_t	sdp_savedval;		/* saved instruction value */
+	struct sdt_probe *sdp_next;		/* next probe */
+	struct sdt_probe *sdp_hashnext;		/* next on hash */
+} sdt_probe_t;
+
+typedef struct sdt_argdesc {
+	const char *sda_provider;		/* provider for arg */
+	const char *sda_name;			/* name of probe */
+	const int sda_ndx;			/* argument index */
+	const int sda_mapping;			/* mapping of argument */
+	const char *sda_native;			/* native type of argument */
+	const char *sda_xlate;			/* translated type of arg */
+} sdt_argdesc_t;
+
+extern void sdt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _SDT_IMPL_H */
diff --git a/bsd/sys/select.h b/bsd/sys/select.h
index 1bcd93484..0bb8d0594 100644
--- a/bsd/sys/select.h
+++ b/bsd/sys/select.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992, 1993
@@ -61,11 +67,20 @@
 #include <sys/cdefs.h>
 #include <sys/_types.h>
 
+/*
+ * [XSI] The <sys/select.h> header shall define the fd_set type as a structure.
+ * The timespec structure shall be defined as described in <time.h>
+ * The <sys/select.h> header shall define the timeval structure.
+ */
+#define __need_fd_set
+#define __need_struct_timespec
+#define __need_struct_timeval
+#include <sys/_structs.h>
+
 /*
  * The time_t and suseconds_t types shall be defined as described in
  * <sys/types.h>
  * The sigset_t type shall be defined as described in <signal.h>
- * The timespec structure shall be defined as described in <time.h>
  */
 #ifndef	_TIME_T
 #define	_TIME_T
@@ -82,25 +97,11 @@ typedef __darwin_suseconds_t	suseconds_t;
 typedef __darwin_sigset_t	sigset_t;
 #endif
 
-#ifndef _TIMESPEC
-#define _TIMESPEC
-struct timespec {
-	time_t	tv_sec;
-	long	tv_nsec;
-};
-#endif
-
 /*
- * [XSI] The <sys/select.h> header shall define the fd_set type as a structure.
  * [XSI] FD_CLR, FD_ISSET, FD_SET, FD_ZERO may be declared as a function, or
  *	 defined as a macro, or both
  * [XSI] FD_SETSIZE shall be defined as a macro
- *
- * Note:	We use _FD_SET to protect all select related
- *		types and macros
  */
-#ifndef _FD_SET
-#define	_FD_SET
 
 /*
  * Select uses bit masks of file descriptors in longs.  These macros
@@ -109,34 +110,25 @@ struct timespec {
  * the default size.
  */
 #ifndef	FD_SETSIZE
-#define	FD_SETSIZE	1024
-#endif
-
-#define	__DARWIN_NBBY	8				/* bits in a byte */
-#define __DARWIN_NFDBITS	(sizeof(__int32_t) * __DARWIN_NBBY) /* bits per mask */
-#define	__DARWIN_howmany(x, y) (((x) + ((y) - 1)) / (y))	/* # y's == x bits? */
-
-typedef	struct fd_set {
-	__int32_t	fds_bits[__DARWIN_howmany(FD_SETSIZE, __DARWIN_NFDBITS)];
-} fd_set;
-
-#define	FD_SET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] |= (1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_CLR(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] &= ~(1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_ISSET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] & (1<<((n) % __DARWIN_NFDBITS)))
-#if __GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3
-/*
- * Use the built-in bzero function instead of the library version so that
- * we do not pollute the namespace or introduce prototype warnings.
- */
-#define	FD_ZERO(p)	__builtin_bzero(p, sizeof(*(p)))
-#else
-#define	FD_ZERO(p)	bzero(p, sizeof(*(p)))
-#endif
-#ifndef _POSIX_C_SOURCE
-#define	FD_COPY(f, t)	bcopy(f, t, sizeof(*(f)))
-#endif	/* !_POSIX_C_SOURCE */
-
-#endif	/* !_FD_SET */
+#define	FD_SETSIZE	__DARWIN_FD_SETSIZE
+#endif /* FD_SETSIZE */
+#ifndef FD_SET
+#define	FD_SET(n, p)	__DARWIN_FD_SET(n, p)
+#endif /* FD_SET */
+#ifndef FD_CLR
+#define	FD_CLR(n, p)	__DARWIN_FD_CLR(n, p)
+#endif /* FD_CLR */
+#ifndef FD_ISSET
+#define	FD_ISSET(n, p)	__DARWIN_FD_ISSET(n, p)
+#endif /* FD_ISSET */
+#ifndef FD_ZERO
+#define	FD_ZERO(p)	__DARWIN_FD_ZERO(p)
+#endif /* FD_ZERO */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef FD_COPY
+#define	FD_COPY(f, t)	__DARWIN_FD_COPY(f, t)
+#endif /* FD_COPY */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #ifdef KERNEL
 #ifdef KERNEL_PRIVATE
@@ -174,27 +166,30 @@ void	selthreadclear(struct selinfo *);
 
 __END_DECLS
 
-#endif /* KERNEL */
-
-
-#ifndef KERNEL
-#ifndef _POSIX_C_SOURCE
-#include <sys/types.h>
-#ifndef  __MWERKS__
-#include <signal.h>
-#endif /* __MWERKS__ */
-#include <sys/time.h>
-#endif	/* !_POSIX_C_SOURCE */
+#else /* !KERNEL */
 
 __BEGIN_DECLS
+
 #ifndef  __MWERKS__
 int	 pselect(int, fd_set * __restrict, fd_set * __restrict,
 		fd_set * __restrict, const struct timespec * __restrict,
-		const sigset_t * __restrict);
+		const sigset_t * __restrict)
+#if defined(_DARWIN_C_SOURCE) || defined(_DARWIN_UNLIMITED_SELECT)
+		__DARWIN_EXTSN_C(pselect)
+#else /* !_DARWIN_C_SOURCE && !_DARWIN_UNLIMITED_SELECT */
+#  if defined(__LP64__) && !__DARWIN_NON_CANCELABLE
+		__DARWIN_1050(pselect)
+#  else /* !__LP64__ || __DARWIN_NON_CANCELABLE */
+		__DARWIN_ALIAS_C(pselect)
+#  endif /* __LP64__ && !__DARWIN_NON_CANCELABLE */
+#endif /* _DARWIN_C_SOURCE || _DARWIN_UNLIMITED_SELECT */
+		;
 #endif /* __MWERKS__ */
-int	 select(int, fd_set * __restrict, fd_set * __restrict,
-		fd_set * __restrict, struct timeval * __restrict);
+
+#include <sys/_select.h>	/* select() prototype */
+
 __END_DECLS
+
 #endif /* ! KERNEL */
 
 #endif /* !_SYS_SELECT_H_ */
diff --git a/bsd/sys/sem.h b/bsd/sys/sem.h
index c31acf9ac..18eeb2061 100644
--- a/bsd/sys/sem.h
+++ b/bsd/sys/sem.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: sem.h,v 1.5 1994/06/29 06:45:15 cgd Exp $	*/
 
@@ -70,7 +76,8 @@ typedef __darwin_size_t	size_t;
  * legacy interface there for binary compatibility only.  Currently, we
  * are only forcing this for programs requesting standards conformance.
  */
-#if defined(__POSIX_C_SOURCE) || defined(kernel) || defined(__LP64__)
+#if __DARWIN_UNIX03 || defined(KERNEL)
+#pragma pack(4)
 /*
  * Structure used internally.
  * 
@@ -81,7 +88,13 @@ typedef __darwin_size_t	size_t;
  * Note: only the fields sem_perm, sem_nsems, sem_otime, and sem_ctime
  * are meaningful in user space.
  */
-struct __semid_ds_new {
+#if (defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE))
+struct semid_ds
+#else
+#define	semid_ds	__semid_ds_new
+struct __semid_ds_new
+#endif
+{
 	struct __ipc_perm_new sem_perm;	/* [XSI] operation permission struct */
 	__int32_t	sem_base;	/* 32 bit base ptr for semaphore set */
 	unsigned short	sem_nsems;	/* [XSI] number of sems in set */
@@ -93,12 +106,12 @@ struct __semid_ds_new {
 	__int32_t	sem_pad2;	/* RESERVED: DO NOT USE! */
 	__int32_t	sem_pad3[4];	/* RESERVED: DO NOT USE! */
 };
-#define	semid_ds	__semid_ds_new
-#else	/* !_POSIX_C_SOURCE */
+#pragma pack()
+#else	/* !__DARWIN_UNIX03 */
 #define	semid_ds	__semid_ds_old
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* __DARWIN_UNIX03 */
 
-#if !defined(__POSIX_C_SOURCE) && !defined(__LP64__)
+#if !__DARWIN_UNIX03
 struct __semid_ds_old {
 	struct __ipc_perm_old sem_perm;	/* [XSI] operation permission struct */
 	__int32_t	sem_base;	/* 32 bit base ptr for semaphore set */
@@ -111,7 +124,7 @@ struct __semid_ds_old {
 	__int32_t	sem_pad2;	/* RESERVED: DO NOT USE! */
 	__int32_t	sem_pad3[4];	/* RESERVED: DO NOT USE! */
 };
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
 /*
  * Possible values for the third argument to semctl()
@@ -149,7 +162,7 @@ struct sembuf {
 #define SEM_UNDO	010000		/* [XSI] Set up adjust on exit entry */
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 
 /*
  * System imposed limit on the value of the third parameter to semop().
@@ -195,16 +208,16 @@ typedef union semun semun_t;
 #define SEM_A		0200	/* alter permission */
 #define SEM_R		0400	/* read permission */
 
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 
 #ifndef KERNEL
 
 __BEGIN_DECLS
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int	semsys(int, ...);
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 int	semctl(int, int, int, ...) __DARWIN_ALIAS(semctl);
 int	semget(key_t, int, int);
 int	semop(int, struct sembuf *, size_t);
diff --git a/bsd/sys/sem_internal.h b/bsd/sys/sem_internal.h
index bcaa2b54a..e59aca2aa 100644
--- a/bsd/sys/sem_internal.h
+++ b/bsd/sys/sem_internal.h
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 /*	$NetBSD: sem.h,v 1.5 1994/06/29 06:45:15 cgd Exp $	*/
 
@@ -114,7 +126,7 @@ typedef union user_semun user_semun_t;
  * Undo structure (internal: one per process)
  */
 struct sem_undo {
-	struct	sem_undo *un_next;	/* ptr to next active undo structure */
+	int 	un_next_idx;	/* index of next active undo structure */
 	struct	proc *un_proc;		/* owner of this structure */
 	short	un_cnt;			/* # of active entries */
 	struct undo {
@@ -142,6 +154,15 @@ struct seminfo {
 };
 extern struct seminfo	seminfo;
 
+/*
+ * Kernel wrapper for the user-level structure
+ */
+struct semid_kernel {
+	struct	user_semid_ds u;
+	struct	label *label;	/* MAC framework label */
+};
+
+
 /* internal "mode" bits */
 #define	SEM_ALLOC	01000	/* semaphore is allocated */
 #define	SEM_DEST	02000	/* semaphore will be destroyed on last detach */
@@ -168,7 +189,7 @@ extern struct seminfo	seminfo;
  */
 #define SEMUSZ		sizeof(struct sem_undo)
 
-extern struct user_semid_ds	*sema;		/* semaphore id pool */
+extern struct semid_kernel	*sema;		/* semaphore id pool */
 extern struct sem		*sem_pool;	/* semaphore pool */
 /* This is now a struct sem_undo with the new memory allocation 
  * extern int	*semu;		// undo structure pool
diff --git a/bsd/sys/semaphore.h b/bsd/sys/semaphore.h
index 10ba6378b..977655231 100644
--- a/bsd/sys/semaphore.h
+++ b/bsd/sys/semaphore.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	@(#)semaphore.h	1.0	2/29/00		*/
 
@@ -35,9 +41,10 @@
 #define _SYS_SEMAPHORE_H_
 
 typedef int sem_t;
+
 /* this should go in limits.h> */
 #define SEM_VALUE_MAX 32767
-#define SEM_FAILED -1
+#define SEM_FAILED ((sem_t *)-1)
 
 #ifndef KERNEL
 #include <sys/cdefs.h>
@@ -45,17 +52,18 @@ typedef int sem_t;
 __BEGIN_DECLS
 int sem_close(sem_t *);
 int sem_destroy(sem_t *);
-int sem_getvalue(sem_t *, int *);
+int sem_getvalue(sem_t * __restrict, int * __restrict);
 int sem_init(sem_t *, int, unsigned int);
 sem_t * sem_open(const char *, int, ...);
 int sem_post(sem_t *);
 int sem_trywait(sem_t *);
 int sem_unlink(const char *);
-int sem_wait(sem_t *);
+int sem_wait(sem_t *) __DARWIN_ALIAS_C(sem_wait);
 __END_DECLS
 
 #else	/* KERNEL */
-void psem_cache_init(void);
+void psem_lock_init(void);
+void psem_cache_init(void) __attribute__((section("__TEXT, initcode")));
 #endif	/* KERNEL */
 
 #endif	/* _SYS_SEMAPHORE_H_ */
diff --git a/bsd/sys/shm.h b/bsd/sys/shm.h
index dc3fc2b56..40ac2ea53 100644
--- a/bsd/sys/shm.h
+++ b/bsd/sys/shm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: shm.h,v 1.15 1994/06/29 06:45:17 cgd Exp $	*/
 
@@ -127,7 +133,7 @@ typedef unsigned short	shmatt_t;
  * legacy interface there for binary compatibility only.  Currently, we
  * are only forcing this for programs requesting standards conformance.
  */
-#if defined(__POSIX_C_SOURCE) || defined(kernel) || defined(__LP64__)
+#if __DARWIN_UNIX03 || defined(KERNEL)
 /*
  * Structure used internally.
  * 
@@ -135,44 +141,49 @@ typedef unsigned short	shmatt_t;
  * the third argment to shmctl().
  *
  * NOTE:	The field shm_internal is not meaningful in user space,
- *		and mst not be used there.
+ *		and must not be used there.
  */
-struct __shmid_ds_new {
+#if (defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE))
+struct shmid_ds
+#else
+#define	shmid_ds	__shmid_ds_new
+struct __shmid_ds_new
+#endif
+{
 	struct __ipc_perm_new shm_perm;	/* [XSI] Operation permission value */
 	size_t		shm_segsz;	/* [XSI] Size of segment in bytes */
 	pid_t		shm_lpid;	/* [XSI] PID of last shared memory op */
 	pid_t		shm_cpid;	/* [XSI] PID of creator */
-	short		shm_nattch;	/* [XSI] Number of current attaches */
+	shmatt_t	shm_nattch;	/* [XSI] Number of current attaches */
 	time_t		shm_atime;	/* [XSI] Time of last shmat() */
 	time_t		shm_dtime;	/* [XSI] Time of last shmdt() */
 	time_t		shm_ctime;	/* [XSI] Time of last shmctl() change */
 	void		*shm_internal;	/* reserved for kernel use */
 };
-#define	shmid_ds	__shmid_ds_new
-#else	/* !_POSIX_C_SOURCE */
+#else	/* !__DARWIN_UNIX03 */
 #define	shmid_ds	__shmid_ds_old
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
-#if !defined(__POSIX_C_SOURCE) && !defined(__LP64__)
+#if !__DARWIN_UNIX03
 struct __shmid_ds_old {
 	struct __ipc_perm_old shm_perm;	/* [XSI] Operation permission value */
 	size_t		shm_segsz;	/* [XSI] Size of segment in bytes */
 	pid_t		shm_lpid;	/* [XSI] PID of last shared memory op */
 	pid_t		shm_cpid;	/* [XSI] PID of creator */
-	short		shm_nattch;	/* [XSI] Number of current attaches */
+	shmatt_t	shm_nattch;	/* [XSI] Number of current attaches */
 	time_t		shm_atime;	/* [XSI] Time of last shmat() */
 	time_t		shm_dtime;	/* [XSI] Time of last shmdt() */
 	time_t		shm_ctime;	/* [XSI] Time of last shmctl() change */
 	void		*shm_internal;	/* reserved for kernel use */
 };
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* !__DARWIN_UNIX03 */
 
 #ifndef KERNEL
 
 __BEGIN_DECLS
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int	shmsys(int, ...);
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 void	*shmat (int, const void *, int);
 int	shmctl(int, int, struct shmid_ds *) __DARWIN_ALIAS(shmctl);
 int	shmdt(const void *);
diff --git a/bsd/sys/shm_internal.h b/bsd/sys/shm_internal.h
index a13cb9cd0..67b42d87a 100644
--- a/bsd/sys/shm_internal.h
+++ b/bsd/sys/shm_internal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: shm.h,v 1.15 1994/06/29 06:45:17 cgd Exp $	*/
 
@@ -50,6 +56,12 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * As defined+described in "X/Open System Interfaces and Headers"
@@ -101,8 +113,20 @@ struct shminfo {
 };
 
 #ifdef KERNEL
+
+struct label;
+
+/* 
+ * Add a kernel wrapper to the shmid_ds struct so that private info (like the
+ * MAC label) can be added to it, without changing the user interface.
+ */
+struct shmid_kernel {
+	struct user_shmid_ds u;
+	struct label *label;	/* MAC label */
+};
+
 extern struct shminfo shminfo;
-extern struct user_shmid_ds *shmsegs;
+extern struct shmid_kernel *shmsegs;
 
 struct proc;
 
diff --git a/bsd/sys/signal.h b/bsd/sys/signal.h
index 27914af93..6899927f7 100644
--- a/bsd/sys/signal.h
+++ b/bsd/sys/signal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -66,8 +72,10 @@
 #include <sys/cdefs.h>
 #include <sys/appleapiopts.h>
 
-#if !defined(_ANSI_SOURCE) && !defined(_POSIX_C_SOURCE)
-#define NSIG	32		/* counting 0; could be 33 (mask is 1-32) */
+#define __DARWIN_NSIG	32	/* counting 0; could be 33 (mask is 1-32) */
+
+#if !defined(_ANSI_SOURCE) && (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
+#define NSIG	__DARWIN_NSIG
 #endif
 
 #include <machine/signal.h>	/* sigcontext; codes for SIGILL, SIGFPE */
@@ -78,12 +86,12 @@
 #define	SIGILL	4	/* illegal instruction (not reset when caught) */
 #define	SIGTRAP	5	/* trace trap (not reset when caught) */
 #define	SIGABRT	6	/* abort() */
-#if  defined(_POSIX_C_SOURCE)
+#if  (defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE))
 #define	SIGPOLL	7	/* pollable event ([XSR] generated, not supported) */
-#else	/* !_POSIX_C_SOURCE */
+#else	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	SIGIOT	SIGABRT	/* compatibility */
 #define	SIGEMT	7	/* EMT instruction */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	SIGFPE	8	/* floating point exception */
 #define	SIGKILL	9	/* kill (cannot be caught or ignored) */
 #define	SIGBUS	10	/* bus error */
@@ -99,21 +107,21 @@
 #define	SIGCHLD	20	/* to parent on child stop or exit */
 #define	SIGTTIN	21	/* to readers pgrp upon background tty read */
 #define	SIGTTOU	22	/* like TTIN for output if (tp->t_local&LTOSTOP) */
-#if  !defined(_POSIX_C_SOURCE)
+#if  (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 #define	SIGIO	23	/* input/output possible signal */
 #endif
 #define	SIGXCPU	24	/* exceeded CPU time limit */
 #define	SIGXFSZ	25	/* exceeded file size limit */
 #define	SIGVTALRM 26	/* virtual time alarm */
 #define	SIGPROF	27	/* profiling time alarm */
-#if  !defined(_POSIX_C_SOURCE)
+#if  (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 #define SIGWINCH 28	/* window size changes */
 #define SIGINFO	29	/* information request */
 #endif
 #define SIGUSR1 30	/* user defined signal 1 */
 #define SIGUSR2 31	/* user defined signal 2 */
 
-#if defined(_ANSI_SOURCE) || defined(_POSIX_C_SOURCE) || defined(__cplusplus)
+#if defined(_ANSI_SOURCE) || __DARWIN_UNIX03 || defined(__cplusplus)
 /*
  * Language spec sez we must list exactly one parameter, even though we
  * actually supply three.  Ugh!
@@ -134,17 +142,16 @@
 #ifndef _ANSI_SOURCE
 #include <sys/_types.h>
 
-#ifndef _MCONTEXT_T
-#define _MCONTEXT_T
-typedef __darwin_mcontext_t		mcontext_t;
-#endif
-
-#ifndef _POSIX_C_SOURCE
-#ifndef _MCONTEXT64_T
-#define _MCONTEXT64_T
-typedef __darwin_mcontext64_t		mcontext64_t;
-#endif
-#endif /* _POSIX_C_SOURCE */
+#define __need_mcontext_t
+#define __need_stack_t
+#define __need_ucontext_t
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#if defined(__ppc__) || defined(__ppc64__)
+#define __need_mcontext64_t
+#define __need_ucontext64_t
+#endif /* __ppc__  || __ppc64__ */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+#include <sys/_structs.h>
 
 #ifndef _PID_T
 #define _PID_T
@@ -166,18 +173,6 @@ typedef __darwin_sigset_t		sigset_t;
 typedef	__darwin_size_t			size_t;
 #endif
 
-#ifndef _UCONTEXT_T
-#define _UCONTEXT_T
-typedef __darwin_ucontext_t		ucontext_t;
-#endif
-
-#ifndef _POSIX_C_SOURCE
-#ifndef _UCONTEXT64_T
-#define _UCONTEXT64_T
-typedef __darwin_ucontext64_t		ucontext64_t;
-#endif
-#endif /* _POSIX_C_SOURCE */
-
 #ifndef _UID_T
 #define _UID_T
 typedef __darwin_uid_t			uid_t;
@@ -201,7 +196,6 @@ struct sigevent {
 	pthread_attr_t	*sigev_notify_attributes;	/* Notification attributes */
 };
 
-// LP64todo - should this move?
 #ifdef BSD_KERNEL_PRIVATE
 
 union user_sigval {
@@ -232,7 +226,7 @@ typedef struct __siginfo {
 	void	*si_addr;		/* faulting instruction */
 	union sigval si_value;		/* signal value */
 	long	si_band;		/* band event for SIGPOLL */
-	unsigned long	pad[7];		/* Reserved for Future Use */
+	unsigned long	__pad[7];	/* Reserved for Future Use */
 } siginfo_t;
 
 #ifdef BSD_KERNEL_PRIVATE
@@ -244,7 +238,7 @@ typedef struct __user_siginfo {
 	pid_t		si_pid;		/* sending process */
 	uid_t		si_uid;		/* sender's ruid */
 	int		si_status;	/* exit value */
-	user_addr_t	si_addr;	/* faulting instruction */
+	user_addr_t	si_addr;	/* faulting instruction (see below) */
 	union user_sigval si_value;	/* signal value */
 	user_long_t	si_band;	/* band event for SIGPOLL */
 	user_ulong_t	pad[7];		/* Reserved for Future Use */
@@ -253,19 +247,20 @@ typedef struct __user_siginfo {
 #endif	/* BSD_KERNEL_PRIVATE */
 
 /* 
- * Incase of SIGILL and SIGFPE, si_addr contains the address of 
- *  faulting instruction.
- * Incase of SIGSEGV and SIGBUS, si_addr contains address of 
- *  faulting memory reference.
- * Incase of SIGCHLD, si_pid willhave child process ID,
- *  si_status will contain exit value or signal.
- *  si_uid contains real user ID of the process that sent the signal
+ * When the signal is SIGILL or SIGFPE, si_addr contains the address of 
+ * the faulting instruction.
+ * When the signal is SIGSEGV or SIGBUS, si_addr contains the address of 
+ * the faulting memory reference. Although for x86 there are cases of SIGSEGV
+ * for which si_addr cannot be determined and is NULL. 
+ * If the signal is SIGCHLD, the si_pid field will contain the child process ID,
+ *  si_status contains the exit value or signal and
+ *  si_uid contains the real user ID of the process that sent the signal.
  */
 
 /* Values for si_code */
 
 /* Codes for SIGILL */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ILL_NOOP	0	/* if only I knew... */
 #endif
 #define	ILL_ILLOPC	1	/* [XSI] illegal opcode */
@@ -278,7 +273,7 @@ typedef struct __user_siginfo {
 #define	ILL_BADSTK	8	/* [XSI] internal stack error -NOTIMP */
 
 /* Codes for SIGFPE */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	FPE_NOOP	0	/* if only I knew... */
 #endif
 #define FPE_FLTDIV	1	/* [XSI] floating point divide by zero */
@@ -291,14 +286,14 @@ typedef struct __user_siginfo {
 #define	FPE_INTOVF	8	/* [XSI] integer overflow -NOTIMP */
 
 /* Codes for SIGSEGV */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SEGV_NOOP	0	/* if only I knew... */
 #endif
 #define	SEGV_MAPERR	1	/* [XSI] address not mapped to object */
 #define	SEGV_ACCERR	2	/* [XSI] invalid permission for mapped object */
 
 /* Codes for SIGBUS */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	BUS_NOOP	0	/* if only I knew... */
 #endif
 #define	BUS_ADRALN	1	/* [XSI] Invalid address alignment */
@@ -310,7 +305,7 @@ typedef struct __user_siginfo {
 #define	TRAP_TRACE	2	/* [XSI] Process trace trap -NOTIMP */
 
 /* Codes for SIGCHLD */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	CLD_NOOP	0	/* if only I knew... */
 #endif
 #define	CLD_EXITED	1	/* [XSI] child has exited */
@@ -389,17 +384,24 @@ struct	__user_sigaction {
 
 #define SA_ONSTACK	0x0001	/* take signal on signal stack */
 #define SA_RESTART	0x0002	/* restart system on signal return */
-#define	SA_DISABLE	0x0004	/* disable taking signals on alternate stack */
+#ifdef	BSD_KERNEL_PRIVATE
+#define	SA_DISABLE	0x0004	/* disable taking signals on alternate stack - for user_sigaltstack.ss_flags only */
+#endif	/* BSD_KERNEL_PRIVATE */
 #define	SA_RESETHAND	0x0004	/* reset to SIG_DFL when taking signal */
 #define SA_NOCLDSTOP	0x0008	/* do not generate SIGCHLD on child stop */
 #define	SA_NODEFER	0x0010	/* don't mask the signal we're delivering */
 #define	SA_NOCLDWAIT	0x0020	/* don't keep zombies around */
 #define	SA_SIGINFO	0x0040	/* signal handler with SA_SIGINFO args */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SA_USERTRAMP	0x0100	/* do not bounce off kernel's sigtramp */
 /* This will provide 64bit register set in a 32bit user address space */
 #define	SA_64REGSET	0x0200	/* signal handler with SA_SIGINFO args with 64bit regs information */
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
+/* the following are the only bits we support from user space, the 
+ * rest are for kernel use only.
+ */
+#define SA_USERSPACE_MASK (SA_ONSTACK | SA_RESTART | SA_RESETHAND | SA_NOCLDSTOP | SA_NODEFER | SA_NOCLDWAIT | SA_SIGINFO)
 
 /*
  * Flags for sigprocmask:
@@ -415,7 +417,7 @@ struct	__user_sigaction {
 #define SI_ASYNCIO	0x10004	/* [CX] aio request completion */
 #define SI_MESGQ	0x10005	/* [CX]	from message arrival on empty queue */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 typedef	void (*sig_t)(int);	/* type of signal function */
 #endif
 
@@ -432,17 +434,12 @@ struct  user_sigaltstack {
 
 #endif	/* BSD_KERNEL_PRIVATE */
 
-#ifndef _STACK_T
-#define _STACK_T
-typedef __darwin_stack_t stack_t;
-#endif
-
 #define SS_ONSTACK	0x0001	/* take signal on signal stack */
 #define	SS_DISABLE	0x0004	/* disable taking signals on alternate stack */
 #define	MINSIGSTKSZ	32768	/* (32K)minimum allowable stack */
 #define	SIGSTKSZ	131072	/* (128K)recommended stack size */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * 4.3 compatibility:
  * Signal vector "template" used in sigvec call.
@@ -461,7 +458,7 @@ struct	sigvec {
 #define SV_SIGINFO	SA_SIGINFO
 
 #define sv_onstack sv_flags	/* isn't compatibility wonderful! */
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Structure used in sigstack call.
@@ -471,7 +468,7 @@ struct	sigstack {
 	int	ss_onstack;		/* current status */
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * Macro for converting signal number to a mask suitable for
  * sigblock().
@@ -491,7 +488,7 @@ struct	sigstack {
 
 #define	BADSIG		SIG_ERR
 
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #endif	/* !_ANSI_SOURCE */
 
 /*
diff --git a/bsd/sys/signalvar.h b/bsd/sys/signalvar.h
index cd0f01880..1597c85ff 100644
--- a/bsd/sys/signalvar.h
+++ b/bsd/sys/signalvar.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -102,15 +108,6 @@ struct	sigacts {
 #define	KERN_SIG_HOLD	(void (*)(int))3
 #define	KERN_SIG_WAIT	(void (*)(int))4
 
-#define pgsigio(pgid, sig, notused) \
-	{ \
-	struct proc *p; \
-	if (pgid < 0) \
-		gsignal(-(pgid), sig);\
-	else if (pgid > 0 && (p = pfind(pgid)) != 0) \
-		psignal(p, sig); \
-}
-
 /*
  * get signal action for process and signal; currently only for current process
  */
@@ -122,7 +119,7 @@ struct	sigacts {
 #define SHOULDissignal(p,uthreadp) \
 	 (((uthreadp)->uu_siglist)	\
 	  & ~((((uthreadp)->uu_sigmask) \
-	       | (((p)->p_flag & P_TRACED) ? 0 : (p)->p_sigignore)) \
+	       | (((p)->p_lflag & P_LTRACED) ? 0 : (p)->p_sigignore)) \
 	      & ~sigcantmask))
 
 /*
@@ -194,8 +191,6 @@ int sigprop[NSIG + 1] = {
 /*
  * Machine-independent functions:
  */
-int	signal_lock(struct proc *);
-int	signal_unlock(struct proc *);
 int	coredump(struct proc *p);
 void	execsigs(struct proc *p, thread_t thread);
 void	gsignal(int pgid, int sig);
@@ -204,11 +199,11 @@ int	CURSIG(struct proc *p);
 int clear_procsiglist(struct proc *p, int bit);
 int clear_procsigmask(struct proc *p, int bit);
 int set_procsigmask(struct proc *p, int bit);
-void	tty_pgsignal(struct pgrp *pgrp, int sig);
 void	postsig(int sig);
-void	siginit(struct proc *p);
+void	siginit(struct proc *p) __attribute__((section("__TEXT, initcode")));
 void	trapsignal(struct proc *p, int sig, unsigned code);
 void	pt_setrunnable(struct proc *p);
+int	hassigprop(int sig, int prop);
 
 /*
  * Machine-dependent functions:
@@ -217,20 +212,22 @@ void	sendsig(struct proc *, /*sig_t*/ user_addr_t  action, int sig,
 	int returnmask, u_long code);
 
 void	psignal(struct proc *p, int sig);
+void	psignal_locked(struct proc *, int);
 void	pgsignal(struct pgrp *pgrp, int sig, int checkctty);
-void	threadsignal(thread_t sig_actthread, int signum, u_long code);
+void	tty_pgsignal(struct tty * tp, int sig, int checkctty);
+void	threadsignal(thread_t sig_actthread, int signum, 
+		mach_exception_code_t code);
 int	thread_issignal(proc_t p, thread_t th, sigset_t mask);
-void	psignal_vfork(struct proc *p, task_t new_task, thread_t thr_act,
+void	psignal_vfork(struct proc *p, task_t new_task, thread_t thread,
 		int signum);
 void	psignal_vtalarm(struct proc *);
 void	psignal_xcpu(struct proc *);
 void	psignal_sigprof(struct proc *);
-void	psignal_lock(struct proc *, int, int);
 void	signal_setast(thread_t sig_actthread);
+void	pgsigio(pid_t pgid, int signalnum);
 
-/* XXX not really very "inline"... */
-__inline__ void sig_lock_to_exit(struct proc *p);
-__inline__ int sig_try_locked(struct proc *p);
+void sig_lock_to_exit(struct proc *p);
+int sig_try_locked(struct proc *p);
 
 #endif	/* BSD_KERNEL_PRIVATE */
 
diff --git a/bsd/sys/socket.h b/bsd/sys/socket.h
index 36392eaab..6d7d93465 100644
--- a/bsd/sys/socket.h
+++ b/bsd/sys/socket.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -56,16 +62,19 @@
  *	@(#)socket.h	8.4 (Berkeley) 2/21/94
  * $FreeBSD: src/sys/sys/socket.h,v 1.39.2.7 2001/07/03 11:02:01 ume Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_SOCKET_H_
 #define	_SYS_SOCKET_H_
 
 #include <sys/_types.h>
 #include <sys/cdefs.h>
-
-#define _NO_NAMESPACE_POLLUTION
-#include <machine/param.h>
-#undef _NO_NAMESPACE_POLLUTION
+#include <machine/_param.h>
 
 /*
  * Definitions related to sockets: types, address families, options.
@@ -128,9 +137,9 @@ struct iovec {
 #define	SOCK_STREAM	1		/* stream socket */
 #define	SOCK_DGRAM	2		/* datagram socket */
 #define	SOCK_RAW	3		/* raw-protocol interface */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SOCK_RDM	4		/* reliably-delivered message */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	SOCK_SEQPACKET	5		/* sequenced packet stream */
 
 /*
@@ -142,14 +151,14 @@ struct iovec {
 #define	SO_KEEPALIVE	0x0008		/* keep connections alive */
 #define	SO_DONTROUTE	0x0010		/* just use interface addresses */
 #define	SO_BROADCAST	0x0020		/* permit sending of broadcast msgs */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SO_USELOOPBACK	0x0040		/* bypass hardware when possible */
 #define SO_LINGER	0x0080          /* linger on close if data present (in ticks) */
 #else
 #define SO_LINGER	0x1080          /* linger on close if data present (in seconds) */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	SO_OOBINLINE	0x0100		/* leave received OOB data in line */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SO_REUSEPORT	0x0200		/* allow local address & port reuse */
 #define	SO_TIMESTAMP	0x0400		/* timestamp received dgram traffic */
 #ifndef __APPLE__
@@ -160,7 +169,7 @@ struct iovec {
 #define SO_WANTMORE		0x4000		/* APPLE: Give hint when more data ready */
 #define SO_WANTOOBFLAG	0x8000		/* APPLE: Want OOB in MSG_FLAG on receive */
 #endif
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Additional options, not kept in so_options.
@@ -173,7 +182,7 @@ struct iovec {
 #define SO_RCVTIMEO	0x1006		/* receive timeout */
 #define	SO_ERROR	0x1007		/* get error status and clear */
 #define	SO_TYPE		0x1008		/* get socket type */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*efine	SO_PRIVSTATE	0x1009		   get/deny privileged state */
 #ifdef __APPLE__
 #define SO_NREAD	0x1020		/* APPLE: get 1st-packet byte count */
@@ -181,9 +190,22 @@ struct iovec {
 #define SO_NOSIGPIPE	0x1022		/* APPLE: No SIGPIPE on EPIPE */
 #define SO_NOADDRERR	0x1023		/* APPLE: Returns EADDRNOTAVAIL when src is not available anymore */
 #define SO_NWRITE	0x1024		/* APPLE: Get number of bytes currently in send socket buffer */
+#define SO_REUSESHAREUID	0x1025		/* APPLE: Allow reuse of port/socket by different userids */
+#ifdef __APPLE_API_PRIVATE
+#define SO_NOTIFYCONFLICT	0x1026	/* APPLE: send notification if there is a bind on a port which is already in use */
+#endif
 #define SO_LINGER_SEC	0x1080          /* linger on close if data present (in seconds) */
+#define SO_RESTRICTIONS	0x1081	/* APPLE: deny inbound/outbound/both/flag set */
+#define SO_RESTRICT_DENYIN		0x00000001	/* flag for SO_RESTRICTIONS - deny inbound */
+#define SO_RESTRICT_DENYOUT		0x00000002	/* flag for SO_RESTRICTIONS - deny outbound */
+#define SO_RESTRICT_DENYSET		0x80000000	/* flag for SO_RESTRICTIONS - deny has been set */
 #endif
-#endif	/* !_POSIX_C_SOURCE */
+#ifdef PRIVATE
+#define	SO_EXECPATH	0x1085 		/* Application Firewall Socket option */
+#endif
+#define	SO_LABEL	0x1010		/* socket's MAC label */
+#define	SO_PEERLABEL	0x1011		/* socket's peer MAC label */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Structure used for manipulating linger option.
@@ -205,16 +227,17 @@ struct	accept_filter_arg {
  */
 #define	SOL_SOCKET	0xffff		/* options for socket level */
 
+
 /*
  * Address families.
  */
 #define	AF_UNSPEC	0		/* unspecified */
 #define	AF_UNIX		1		/* local to host (pipes) */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	AF_LOCAL	AF_UNIX		/* backward compatibility */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	AF_INET		2		/* internetwork: UDP, TCP, etc. */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	AF_IMPLINK	3		/* arpanet imp addresses */
 #define	AF_PUP		4		/* pup protocols: e.g. BSP */
 #define	AF_CHAOS	5		/* mit CHAOS protocols */
@@ -246,9 +269,9 @@ struct	accept_filter_arg {
 #define	AF_ISDN		28		/* Integrated Services Digital Network*/
 #define	AF_E164		AF_ISDN		/* CCITT E.164 recommendation */
 #define	pseudo_AF_KEY	29		/* Internal key-management function */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #define	AF_INET6	30		/* IPv6 */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	AF_NATM		31		/* native ATM access */
 #ifdef __APPLE__
 #define AF_SYSTEM	32		/* Kernel event messages */
@@ -270,7 +293,7 @@ struct	accept_filter_arg {
 #define	AF_NETGRAPH	32		/* Netgraph sockets */
 #endif
 #define	AF_MAX		37
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * [XSI] Structure used by kernel to store most addresses.
@@ -281,7 +304,7 @@ struct sockaddr {
 	char		sa_data[14];	/* [XSI] addr value (actually larger) */
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SOCK_MAXADDRLEN	255		/* longest possible addresses */
 
 /*
@@ -292,7 +315,7 @@ struct sockproto {
 	__uint16_t	sp_family;		/* address family */
 	__uint16_t	sp_protocol;		/* protocol */
 };
-#endif	/* !_POSIX_C_SOURCE*/
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE)*/
 
 /*
  * RFC 2553: protocol-independent placeholder for socket addresses
@@ -385,8 +408,9 @@ struct sockaddr_storage {
  *
  * Further levels are defined by the individual families below.
  */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define NET_MAXID	AF_MAX
-
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifdef KERNEL_PRIVATE
 #define CTL_NET_NAMES { \
@@ -427,8 +451,9 @@ struct sockaddr_storage {
 	{ "ppp", CTLTYPE_NODE }, \
 	{ "hdrcomplete", CTLTYPE_NODE }, \
 }
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * PF_ROUTE - Routing table
  *
@@ -445,6 +470,7 @@ struct sockaddr_storage {
 #define NET_RT_IFLIST2	6		/* interface list with addresses */
 #define NET_RT_DUMP2                     7               /* dump; may limit to a.f. */
 #define	NET_RT_MAXID		8
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #ifdef KERNEL_PRIVATE
 #define CTL_NET_RT_NAMES { \
@@ -458,7 +484,7 @@ struct sockaddr_storage {
         { "dump2", CTLTYPE_STRUCT }, \
 }
 
-#endif KERNEL_PRIVATE
+#endif /* KERNEL_PRIVATE */
 
 /*
  * Maximum queue length specifiable by listen.
@@ -479,7 +505,6 @@ struct msghdr {
 	int		msg_flags;	/* [XSI] flags on received message */
 };
 
-// LP64todo - should this move?
 #ifdef KERNEL
 /* LP64 version of struct msghdr.  all pointers 
  * grow when we're dealing with a 64-bit process.
@@ -505,7 +530,7 @@ struct user_msghdr {
 #define	MSG_TRUNC	0x10		/* data discarded before delivery */
 #define	MSG_CTRUNC	0x20		/* control data lost before delivery */
 #define	MSG_WAITALL	0x40		/* wait for full request or error */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	MSG_DONTWAIT	0x80		/* this message should be nonblocking */
 #define	MSG_EOF		0x100		/* data completes connection */
 #ifdef __APPLE__
@@ -516,7 +541,9 @@ struct user_msghdr {
 #define MSG_HAVEMORE	0x2000		/* Data ready to be read */
 #define MSG_RCVMORE	0x4000		/* Data remains in current pkt */
 #endif
-#define MSG_COMPAT      0x8000		/* used in sendit() */
+#ifdef KERNEL_PRIVATE
+#define MSG_COMPAT      0x8000		/* deprecated */
+#endif /* KERNEL_PRIVATE */
 #define MSG_NEEDSA	0x10000		/* Fail receive if socket address cannot be allocated */
 #ifdef KERNEL_PRIVATE
 #define MSG_NBIO	0x20000		/* FIONBIO mode, used by fifofs */
@@ -524,7 +551,7 @@ struct user_msghdr {
 #ifdef	KERNEL
 #define MSG_USEUPCALL	0x80000000 /* Inherit upcall in sock_accept */
 #endif
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * Header for ancillary data objects in msg_control buffer.
@@ -539,7 +566,7 @@ struct cmsghdr {
 /* followed by	unsigned char  cmsg_data[]; */
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifndef __APPLE__
 /*
  * While we may have more groups than this, the cmsgcred struct must
@@ -564,40 +591,41 @@ struct cmsgcred {
 	gid_t	cmcred_groups[CMGROUP_MAX];	/* groups */
 };
 #endif
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /* given pointer to struct cmsghdr, return pointer to data */
 #define	CMSG_DATA(cmsg)		((unsigned char *)(cmsg) + \
-				 ALIGN(sizeof(struct cmsghdr)))
+				 __DARWIN_ALIGN(sizeof(struct cmsghdr)))
 
 /* given pointer to struct cmsghdr, return pointer to next cmsghdr */
 #define	CMSG_NXTHDR(mhdr, cmsg)	\
-	(((unsigned char *)(cmsg) + ALIGN((cmsg)->cmsg_len) + \
-	  ALIGN(sizeof(struct cmsghdr)) > \
+	(((unsigned char *)(cmsg) + __DARWIN_ALIGN((__darwin_intptr_t)(cmsg)->cmsg_len) + \
+	  __DARWIN_ALIGN(sizeof(struct cmsghdr)) > \
 	    (unsigned char *)(mhdr)->msg_control + (mhdr)->msg_controllen) ? \
-	    (struct cmsghdr *)0 /* NULL */ : \
-	    (struct cmsghdr *)((unsigned char *)(cmsg) + ALIGN((cmsg)->cmsg_len)))
+	    (struct cmsghdr *)0L /* NULL */ : \
+	    (struct cmsghdr *)((unsigned char *)(cmsg) + __DARWIN_ALIGN((__darwin_intptr_t)(cmsg)->cmsg_len)))
 
 #define	CMSG_FIRSTHDR(mhdr)	((struct cmsghdr *)(mhdr)->msg_control)
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* RFC 2292 additions */
-#define	CMSG_SPACE(l)		(ALIGN(sizeof(struct cmsghdr)) + ALIGN(l))
-#define	CMSG_LEN(l)		(ALIGN(sizeof(struct cmsghdr)) + (l))
+#define	CMSG_SPACE(l)		(__DARWIN_ALIGN(sizeof(struct cmsghdr)) + __DARWIN_ALIGN(l))
+#define	CMSG_LEN(l)		(__DARWIN_ALIGN(sizeof(struct cmsghdr)) + (l))
 
 #ifdef KERNEL
-#define	CMSG_ALIGN(n)	ALIGN(n)
+#define	CMSG_ALIGN(n)	__DARWIN_ALIGN(n)
 #endif
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /* "Socket"-level control message types: */
 #define	SCM_RIGHTS	0x01		/* access rights (array of int) */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	SCM_TIMESTAMP	0x02		/* timestamp (struct timeval) */
 #define	SCM_CREDS	0x03		/* process creds (struct cmsgcred) */
 
+#ifdef KERNEL_PRIVATE
 /*
- * 4.3 compat sockaddr, move to compat file later
+ * 4.3 compat sockaddr (deprecated)
  */
 struct osockaddr {
 	__uint16_t	sa_family;	/* address family */
@@ -605,9 +633,8 @@ struct osockaddr {
 };
 
 /*
- * 4.3-compat message header (move to compat file later).
+ * 4.3-compat message header (deprecated)
  */
-	// LP64todo - fix this.  should msg_iov be * iovec_64?
 struct omsghdr {
 	void		*msg_name;		/* optional address */
 	socklen_t	msg_namelen;		/* size of address */
@@ -616,7 +643,8 @@ struct omsghdr {
 	void		*msg_accrights;		/* access rights sent/rcvd */
 	int		msg_accrightslen;
 };
-#endif	/* !_POSIX_C_SOURCE */
+#endif /* KERNEL_PRIVATE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * howto arguments for shutdown(2), specified by Posix.1g.
@@ -625,8 +653,7 @@ struct omsghdr {
 #define	SHUT_WR		1		/* shut down the writing side */
 #define	SHUT_RDWR	2		/* shut down both sides */
 
-#ifndef _POSIX_C_SOURCE
-#if SENDFILE
+#if !defined(_POSIX_C_SOURCE)
 /*
  * sendfile(2) header/trailer struct
  */
@@ -636,44 +663,54 @@ struct sf_hdtr {
 	struct iovec *trailers;	/* pointer to an array of trailer struct iovec's */
 	int trl_cnt;		/* number of trailer iovec's */
 };
-#endif
+
+#ifdef KERNEL
+
+struct user_sf_hdtr {
+	user_addr_t headers __attribute((aligned(8)));	/* pointer to an array of header struct iovec's */
+	int hdr_cnt;		/* number of header iovec's */
+	user_addr_t trailers __attribute((aligned(8)));	/* pointer to an array of trailer struct iovec's */
+	int trl_cnt;		/* number of trailer iovec's */
+};
+
+
+#endif /* KERNEL */
+
 #endif	/* !_POSIX_C_SOURCE */
 
 #ifndef	KERNEL
 __BEGIN_DECLS
 int	accept(int, struct sockaddr * __restrict, socklen_t * __restrict)
-		__DARWIN_ALIAS(accept);
+		__DARWIN_ALIAS_C(accept);
 int	bind(int, const struct sockaddr *, socklen_t) __DARWIN_ALIAS(bind);
-int	connect(int, const struct sockaddr *, socklen_t) __DARWIN_ALIAS( connect);
+int	connect(int, const struct sockaddr *, socklen_t) __DARWIN_ALIAS_C( connect);
 int	getpeername(int, struct sockaddr * __restrict, socklen_t * __restrict)
 		__DARWIN_ALIAS(getpeername);
 int	getsockname(int, struct sockaddr * __restrict, socklen_t * __restrict)
 		__DARWIN_ALIAS(getsockname);
 int	getsockopt(int, int, int, void * __restrict, socklen_t * __restrict);
 int	listen(int, int) __DARWIN_ALIAS(listen);
-ssize_t	recv(int, void *, size_t, int) __DARWIN_ALIAS(recv);
+ssize_t	recv(int, void *, size_t, int) __DARWIN_ALIAS_C(recv);
 ssize_t	recvfrom(int, void *, size_t, int, struct sockaddr * __restrict,
-		socklen_t * __restrict) __DARWIN_ALIAS(recvfrom);
-ssize_t	recvmsg(int, struct msghdr *, int) __DARWIN_ALIAS(recvmsg);
-ssize_t	send(int, const void *, size_t, int) __DARWIN_ALIAS(send);
-ssize_t	sendmsg(int, const struct msghdr *, int) __DARWIN_ALIAS(sendmsg);
+		socklen_t * __restrict) __DARWIN_ALIAS_C(recvfrom);
+ssize_t	recvmsg(int, struct msghdr *, int) __DARWIN_ALIAS_C(recvmsg);
+ssize_t	send(int, const void *, size_t, int) __DARWIN_ALIAS_C(send);
+ssize_t	sendmsg(int, const struct msghdr *, int) __DARWIN_ALIAS_C(sendmsg);
 ssize_t	sendto(int, const void *, size_t,
-		int, const struct sockaddr *, socklen_t) __DARWIN_ALIAS(sendto);
+		int, const struct sockaddr *, socklen_t) __DARWIN_ALIAS_C(sendto);
 int	setsockopt(int, int, int, const void *, socklen_t);
 int	shutdown(int, int);
+int	sockatmark(int);
 int	socket(int, int, int);
 int	socketpair(int, int, int, int *) __DARWIN_ALIAS(socketpair);
-/*
- * NOTIMP:
- * int sockatmark(int s);
- */
 
-#ifndef _POSIX_C_SOURCE
-#if SENDFILE
-int	sendfile(int, int, off_t, size_t, struct sf_hdtr *, off_t *, int);
-#endif
-void	pfctlinput(int, struct sockaddr *);
+#if !defined(_POSIX_C_SOURCE)
+int	sendfile(int, int, off_t, off_t *, struct sf_hdtr *, int);
 #endif	/* !_POSIX_C_SOURCE */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+void	pfctlinput(int, struct sockaddr *);
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 __END_DECLS
 
 #endif /* !KERNEL */
diff --git a/bsd/sys/socketvar.h b/bsd/sys/socketvar.h
index e71e3b8b7..988ec8d82 100644
--- a/bsd/sys/socketvar.h
+++ b/bsd/sys/socketvar.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
@@ -56,9 +62,15 @@
  *	@(#)socketvar.h	8.3 (Berkeley) 2/19/95
  * $FreeBSD: src/sys/sys/socketvar.h,v 1.46.2.6 2001/08/31 13:45:49 jlemon Exp $
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_SOCKETVAR_H_
-#define _SYS_SOCKETVAR_H_
+#define	_SYS_SOCKETVAR_H_
 
 #include <sys/appleapiopts.h>
 #include <sys/queue.h>			/* for TAILQ macros */
@@ -79,12 +91,12 @@ struct sockutil;
 #ifdef KERNEL_PRIVATE
 /* strings for sleep message: */
 extern	char netio[], netcon[], netcls[];
-#define SOCKET_CACHE_ON	
-#define SO_CACHE_FLUSH_INTERVAL 1	/* Seconds */
-#define SO_CACHE_TIME_LIMIT	(120/SO_CACHE_FLUSH_INTERVAL) /* Seconds */
-#define SO_CACHE_MAX_FREE_BATCH	50
-#define MAX_CACHED_SOCKETS	60000
-#define TEMPDEBUG		0
+#define	SOCKET_CACHE_ON
+#define	SO_CACHE_FLUSH_INTERVAL 1	/* Seconds */
+#define	SO_CACHE_TIME_LIMIT	(120/SO_CACHE_FLUSH_INTERVAL) /* Seconds */
+#define	SO_CACHE_MAX_FREE_BATCH	50
+#define	MAX_CACHED_SOCKETS	60000
+#define	TEMPDEBUG		0
 
 /*
  * Kernel structure per socket.
@@ -103,27 +115,27 @@ struct accept_filter;
 #endif
 
 struct socket {
-	int		so_zone;			/* zone we were allocated from */
-	short	so_type;			/* generic type, see socket.h */
+	int	so_zone;		/* zone we were allocated from */
+	short	so_type;		/* generic type, see socket.h */
 	short	so_options;		/* from socket call, see socket.h */
 	short	so_linger;		/* time to linger while closing */
-	short	so_state;			/* internal state flags SS_*, below */
+	short	so_state;		/* internal state flags SS_*, below */
 	caddr_t	so_pcb;			/* protocol control block */
 	struct	protosw *so_proto;	/* protocol handle */
-/*
- * Variables for connection queuing.
- * Socket where accepts occur is so_head in all subsidiary sockets.
- * If so_head is 0, socket is not related to an accept.
- * For head socket so_incomp queues partially completed connections,
- * while so_comp is a queue of connections ready to be accepted.
- * If a connection is aborted and it has so_head set, then
- * it has to be pulled out of either so_incomp or so_comp.
- * We allow connections to queue up based on current queue lengths
- * and limit on number of queued connections for this socket.
- */
+	/*
+	 * Variables for connection queueing.
+	 * Socket where accepts occur is so_head in all subsidiary sockets.
+	 * If so_head is 0, socket is not related to an accept.
+	 * For head socket so_incomp queues partially completed connections,
+	 * while so_comp is a queue of connections ready to be accepted.
+	 * If a connection is aborted and it has so_head set, then
+	 * it has to be pulled out of either so_incomp or so_comp.
+	 * We allow connections to queue up based on current queue lengths
+	 * and limit on number of queued connections for this socket.
+	 */
 	struct	socket *so_head;	/* back pointer to accept socket */
-	TAILQ_HEAD(, socket) so_incomp;	/* queue of partial unaccepted connections */
-	TAILQ_HEAD(, socket) so_comp;	/* queue of complete unaccepted connections */
+	TAILQ_HEAD(, socket) so_incomp;	/* q of partially unaccepted conns */
+	TAILQ_HEAD(, socket) so_comp;	/* q of complete unaccepted conns */
 	TAILQ_ENTRY(socket) so_list;	/* list of unaccepted connections */
 	short	so_qlen;		/* number of unaccepted connections */
 	short	so_incqlen;		/* number of unaccepted incomplete
@@ -137,41 +149,47 @@ struct socket {
 	/* We don't support AIO ops */
 	TAILQ_HEAD(, aiocblist) so_aiojobq; /* AIO ops waiting on socket */
 #endif
-/*
- * Variables for socket buffering.
- */
+	/*
+	 * Variables for socket buffering.
+	 */
 	struct	sockbuf {
 		u_long	sb_cc;		/* actual chars in buffer */
 		u_long	sb_hiwat;	/* max actual char count */
 		u_long	sb_mbcnt;	/* chars of mbufs used */
 		u_long	sb_mbmax;	/* max chars of mbufs to use */
-		long	sb_lowat;	/* low water mark */
+		u_long	sb_ctl;		/* non-data chars in buffer */
+		u_long	sb_lowat;	/* low water mark */
 		struct	mbuf *sb_mb;	/* the mbuf chain */
+		struct	mbuf *sb_mbtail; /* the last mbuf in the chain */
+		struct	mbuf *sb_lastrecord; /* first mbuf of last record */
 #if __APPLE__
-        struct  socket *sb_so;  /* socket back ptr for kexts */
+		struct  socket *sb_so;  /* socket back ptr for kexts */
 #endif
 		struct	selinfo sb_sel;	/* process selecting read/write */
 		short	sb_flags;	/* flags, see below */
-		struct timeval	sb_timeo;	/* timeout for read/write */
-		void    *reserved1;     /* for future use if needed */
-		void    *reserved2;
+		struct timeval sb_timeo; /* timeout for read/write */
+		u_int	sb_maxused;	/* max char count ever used in sockbuf */
+		void	*reserved1[4];	/* for future use */
 	} so_rcv, so_snd;
-#define	SB_MAX		(256*1024)	/* default for max chars in sockbuf */
+#define	SB_MAX		(8192*1024)	/* default for max chars in sockbuf */
 #define	SB_LOCK		0x01		/* lock on data queue */
 #define	SB_WANT		0x02		/* someone is waiting to lock */
 #define	SB_WAIT		0x04		/* someone is waiting for data/space */
 #define	SB_SEL		0x08		/* someone is selecting */
-#define	SB_ASYNC		0x10		/* ASYNC I/O, need signals */
-#define	SB_UPCALL		0x20		/* someone wants an upcall */
-#define	SB_NOINTR		0x40		/* operations not interruptible */
-#define SB_KNOTE		0x100	/* kernel note attached */
+#define	SB_ASYNC	0x10		/* ASYNC I/O, need signals */
+#define	SB_UPCALL	0x20		/* someone wants an upcall */
+#define	SB_NOINTR	0x40		/* operations not interruptible */
+#define	SB_KNOTE	0x100		/* kernel note attached */
+#define SB_USRSIZE	0x200		/* user specified sbreserve */
 #ifndef __APPLE__
-#define SB_AIO		0x80		/* AIO operations queued */
+#define	SB_AIO		0x80		/* AIO operations queued */
 #else
 #define	SB_NOTIFY	(SB_WAIT|SB_SEL|SB_ASYNC)
-#define SB_RECV		0x8000		/* this is rcv sb */
+#define	SB_DROP		0x400		/* does not accept any more data */
+#define	SB_UNIX		0x800		/* UNIX domain socket buffer */
+#define	SB_RECV		0x8000		/* this is rcv sb */
 
-	caddr_t	so_tpcb;		/* Wisc. protocol control block - XXX unused? */
+	caddr_t	so_tpcb;	/* Wisc. protocol control block - XXX unused? */
 #endif
 
 	void	(*so_upcall)(struct socket *so, caddr_t arg, int waitf);
@@ -181,14 +199,14 @@ struct socket {
 	so_gen_t so_gencnt;		/* generation count */
 #ifndef __APPLE__
 	void	*so_emuldata;		/* private data for emulators */
-	struct	so_accf { 
+	struct so_accf {
 		struct	accept_filter *so_accept_filter;
 		void	*so_accept_filter_arg;	/* saved filter args */
 		char	*so_accept_filter_str;	/* saved user args */
 	} *so_accf;
 #else
-	TAILQ_HEAD(,eventqelt) so_evlist;
-	int	cached_in_sock_layer;	/* Is socket bundled with pcb/pcb.inp_ppcb? */
+	TAILQ_HEAD(, eventqelt) so_evlist;
+	int	cached_in_sock_layer;	/* bundled with pcb/pcb.inp_ppcb? */
 	struct	socket	*cache_next;
 	struct	socket	*cache_prev;
 	u_long		cache_timestamp;
@@ -196,87 +214,112 @@ struct socket {
 	struct	mbuf *so_temp;		/* Holding area for outbound frags */
 	/* Plug-in support - make the socket interface overridable */
 	struct	mbuf *so_tail;
-	struct socket_filter_entry *so_filt;		/* NKE hook */
-	u_long	so_flags;		/* Flags */
-#define SOF_NOSIGPIPE		0x00000001
-#define SOF_NOADDRAVAIL		0x00000002	/* returns EADDRNOTAVAIL if src address is gone */
-#define SOF_PCBCLEARING		0x00000004	/* pru_disconnect done, no need to call pru_detach */
-	int	so_usecount;		/* refcounting of socket use */;
+	struct socket_filter_entry *so_filt;	/* NKE hook */
+	u_long	so_flags;	/* Flags */
+#define	SOF_NOSIGPIPE	0x1
+#define	SOF_NOADDRAVAIL	0x2	/* EADDRNOTAVAIL if src addr is gone */
+#define	SOF_PCBCLEARING	0x4	/* pru_disconnect done; don't call pru_detach */
+#define	SOF_DEFUNCT	0x8	/* accepted socket marked as inactive */
+#define	SOF_CLOSEWAIT	0x10	/* blocked in close awaiting some events */
+#define	SOF_UPCALLINUSE	0x20	/* socket upcall is currently in progress */
+#define SOF_REUSESHAREUID	0x40	/* Allows SO_REUSEADDR/SO_REUSEPORT for multiple so_uid */
+#define	SOF_MULTIPAGES	0x80	/* jumbo clusters may be used for sosend */
+#define SOF_ABORTED	0x100	/* soabort was already called once on the socket */
+#define SOF_OVERFLOW 0x200	/* socket was dropped as overflow of listen queue */
+#ifdef __APPLE_API_PRIVATE
+#define SOF_NOTIFYCONFLICT 0x400	/* notify that a bind was done on a port already in use */
+#endif
+	int	so_usecount;	/* refcounting of socket use */;
 	int	so_retaincnt;
-	u_int32_t	so_filteruse; /* usecount for the socket filters */
+	u_int32_t so_filteruse;	/* usecount for the socket filters */
+	u_int32_t	so_traffic_mgt_flags;	/* traffic_mgt socket config */
+	thread_t	so_send_filt_thread;
+	u_int32_t	so_restrictions;
+
 /* for debug pruposes */
-#define SO_LCKDBG_MAX		4	/* number of debug locking Link Registers recorded */
-	u_int32_t lock_lr[SO_LCKDBG_MAX];	/* socket locking calling history */ 
+#define	SO_LCKDBG_MAX 4	/* number of debug locking Link Registers recorded */
+	u_int32_t lock_lr[SO_LCKDBG_MAX];	/* locking calling history */
 	int	next_lock_lr;
-	u_int32_t unlock_lr[SO_LCKDBG_MAX];	/* socket unlocking caller history */
+	u_int32_t unlock_lr[SO_LCKDBG_MAX];	/* unlocking caller history */
 	int	next_unlock_lr;
-	
+	void *reserved; /* reserved for future use */
+#endif /* __APPLE__ */
 
-	void	*reserved3;		/* Temporarily in use/debug: last socket lock LR */
-	void	*reserved4;		/* Temporarily in use/debug: last socket unlock LR */
-	thread_t	so_send_filt_thread;
-#endif
+	struct	label *so_label;	/* MAC label for socket */
+	struct	label *so_peerlabel;	/* cached MAC label for socket peer */
+	thread_t	so_background_thread;	/* thread that marked this socket background */
 };
 #endif /* KERNEL_PRIVATE */
 
 /*
  * Socket state bits.
  */
-#define	SS_NOFDREF		0x001	/* no file table ref any more */
-#define	SS_ISCONNECTED		0x002	/* socket connected to a peer */
-#define	SS_ISCONNECTING		0x004	/* in process of connecting to peer */
-#define	SS_ISDISCONNECTING	0x008	/* in process of disconnecting */
-#define	SS_CANTSENDMORE		0x010	/* can't send more data to peer */
-#define	SS_CANTRCVMORE		0x020	/* can't receive more data from peer */
-#define	SS_RCVATMARK		0x040	/* at mark on input */
-
-#define	SS_PRIV			0x080	/* privileged for broadcast, raw... */
-#define	SS_NBIO			0x100	/* non-blocking ops */
-#define	SS_ASYNC		0x200	/* async i/o notify */
-#define	SS_ISCONFIRMING		0x400	/* deciding to accept connection req */
-#define	SS_INCOMP		0x800	/* Unaccepted, incomplete connection */
+#define	SS_NOFDREF		0x0001	/* no file table ref any more */
+#define	SS_ISCONNECTED		0x0002	/* socket connected to a peer */
+#define	SS_ISCONNECTING		0x0004	/* in process of connecting to peer */
+#define	SS_ISDISCONNECTING	0x0008	/* in process of disconnecting */
+#define	SS_CANTSENDMORE		0x0010	/* can't send more data to peer */
+#define	SS_CANTRCVMORE		0x0020	/* can't receive more data from peer */
+#define	SS_RCVATMARK		0x0040	/* at mark on input */
+
+#define	SS_PRIV			0x0080	/* privileged for broadcast, raw... */
+#define	SS_NBIO			0x0100	/* non-blocking ops */
+#define	SS_ASYNC		0x0200	/* async i/o notify */
+#define	SS_ISCONFIRMING		0x0400	/* deciding to accept connection req */
+#define	SS_INCOMP		0x0800	/* Unaccepted, incomplete connection */
 #define	SS_COMP			0x1000	/* unaccepted, complete connection */
 #define	SS_ISDISCONNECTED	0x2000	/* socket disconnected from peer */
-#define	SS_DRAINING		0x4000	/* close waiting for blocked system calls to drain */
+#define	SS_DRAINING		0x4000	/* close waiting for blocked system
+					   calls to drain */
+
+#if defined(__LP64__)
+#define	_XSOCKET_PTR(x)		u_int32_t
+#else
+#define	_XSOCKET_PTR(x)		x
+#endif
+
+#pragma pack(4)
 
 /*
  * Externalized form of struct socket used by the sysctl(3) interface.
  */
 struct	xsocket {
-	size_t	xso_len;	/* length of this structure */
-	struct	socket *xso_so;	/* makes a convenient handle sometimes */
-	short	so_type;
-	short	so_options;
-	short	so_linger;
-	short	so_state;
-	caddr_t	so_pcb;		/* another convenient handle */
-	int	xso_protocol;
-	int	xso_family;
-	short	so_qlen;
-	short	so_incqlen;
-	short	so_qlimit;
-	short	so_timeo;
-	u_short	so_error;
-	pid_t	so_pgid;
-	u_long	so_oobmark;
+	u_int32_t	xso_len;		/* length of this structure */
+	_XSOCKET_PTR(struct socket *) xso_so;	/* makes a convenient handle */
+	short		so_type;
+	short		so_options;
+	short		so_linger;
+	short		so_state;
+	_XSOCKET_PTR(caddr_t)	so_pcb;		/* another convenient handle */
+	int		xso_protocol;
+	int		xso_family;
+	short		so_qlen;
+	short		so_incqlen;
+	short		so_qlimit;
+	short		so_timeo;
+	u_short		so_error;
+	pid_t		so_pgid;
+	u_int32_t	so_oobmark;
 	struct	xsockbuf {
-		u_long	sb_cc;
-		u_long	sb_hiwat;
-		u_long	sb_mbcnt;
-		u_long	sb_mbmax;
-		long	sb_lowat;
-		short	sb_flags;
-		short	sb_timeo;
+		u_int32_t	sb_cc;
+		u_int32_t	sb_hiwat;
+		u_int32_t	sb_mbcnt;
+		u_int32_t	sb_mbmax;
+		int32_t		sb_lowat;
+		short		sb_flags;
+		short		sb_timeo;
 	} so_rcv, so_snd;
-	uid_t	so_uid;		/* XXX */
+	uid_t		so_uid;		/* XXX */
 };
 
+#pragma pack()
+
 #ifdef KERNEL_PRIVATE
 /*
  * Macros for sockets and socket buffering.
  */
 
-#define sbtoso(sb) (sb->sb_so)
+#define	sbtoso(sb) (sb->sb_so)
 
 /*
  * Functions for sockets and socket buffering.
@@ -285,17 +328,17 @@ struct	xsocket {
  */
 
 __BEGIN_DECLS
-int		sb_notify(struct sockbuf *sb);
-long		sbspace(struct sockbuf *sb);
-int		sosendallatonce(struct socket *so);
-int		soreadable(struct socket *so);
-int		sowriteable(struct socket *so);
-void		sballoc(struct sockbuf *sb, struct mbuf *m);
-void		sbfree(struct sockbuf *sb, struct mbuf *m);
-int		sblock(struct sockbuf *sb, int wf);
-void		sbunlock(struct sockbuf *sb, int locked);
-void		sorwakeup(struct socket * so);
-void		sowwakeup(struct socket * so);
+int	sb_notify(struct sockbuf *sb);
+long	sbspace(struct sockbuf *sb);
+int	sosendallatonce(struct socket *so);
+int	soreadable(struct socket *so);
+int	sowriteable(struct socket *so);
+void	sballoc(struct sockbuf *sb, struct mbuf *m);
+void	sbfree(struct sockbuf *sb, struct mbuf *m);
+int	sblock(struct sockbuf *sb, int wf);
+void	sbunlock(struct sockbuf *sb, int locked);
+void	sorwakeup(struct socket *so);
+void	sowwakeup(struct socket *so);
 __END_DECLS
 
 /*
@@ -305,20 +348,20 @@ __END_DECLS
  *  the NFDescriptor to permit selective replacement during
  *  operation, e.g., to disable some functions.
  */
-struct kextcb
-{	struct kextcb *e_next;		/* Next kext control block */
+struct kextcb {
+	struct kextcb *e_next;		/* Next kext control block */
 	void *e_fcb;			/* Real filter control block */
 	struct NFDescriptor *e_nfd;	/* NKE Descriptor */
 	/* Plug-in support - intercept functions */
 	struct sockif *e_soif;		/* Socket functions */
 	struct sockutil *e_sout;	/* Sockbuf utility functions */
 };
-#define EXT_NULL	0x0		/* STATE: Not in use */
-#define sotokextcb(so) (so ? so->so_ext : 0)
+#define	EXT_NULL	0x0		/* STATE: Not in use */
+#define	sotokextcb(so) (so ? so->so_ext : 0)
 
 #ifdef KERNEL
 
-#define SO_FILT_HINT_LOCKED 0x1
+#define	SO_FILT_HINT_LOCKED 0x1
 
 /*
  * Argument structure for sosetopt et seq.  This is in the KERNEL
@@ -352,6 +395,10 @@ extern int	maxsockets;
 extern u_long	sb_max;
 extern int socket_zone;
 extern so_gen_t so_gencnt;
+extern int	socket_debug;
+extern int sosendjcl;
+extern int sosendjcl_ignore_capab;
+extern int somaxconn;
 
 struct file;
 struct filedesc;
@@ -362,93 +409,120 @@ struct ucred;
 struct uio;
 struct knote;
 
+#define	SBLASTRECORDCHK(sb, s)	\
+	if (socket_debug) sblastrecordchk(sb, s);
+
+#define	SBLASTMBUFCHK(sb, s)	\
+	if (socket_debug) sblastmbufchk(sb, s);
+
+#define	SB_EMPTY_FIXUP(sb) {			\
+	if ((sb)->sb_mb == NULL) {		\
+		(sb)->sb_mbtail = NULL;		\
+		(sb)->sb_lastrecord = NULL;	\
+	}					\
+}
+
 /*
  * From uipc_socket and friends
  */
 __BEGIN_DECLS
-struct	sockaddr *dup_sockaddr(struct sockaddr *sa, int canwait);
-int	getsock(struct filedesc *fdp, int fd, struct file **fpp);
-int	sockargs(struct mbuf **mp, user_addr_t data, int buflen, int type);
-int	getsockaddr(struct sockaddr **namp, user_addr_t uaddr, size_t len);
-int	sbappend(struct sockbuf *sb, struct mbuf *m);
-int	sbappendaddr(struct sockbuf *sb, struct sockaddr *asa,
-	    struct mbuf *m0, struct mbuf *control, int *error_out);
-int	sbappendcontrol(struct sockbuf *sb, struct mbuf *m0,
-	    struct mbuf *control, int *error_out);
-int	sbappendrecord(struct sockbuf *sb, struct mbuf *m0);
-void	sbcheck(struct sockbuf *sb);
-struct mbuf *
-	sbcreatecontrol(caddr_t p, int size, int type, int level);
-void	sbdrop(struct sockbuf *sb, int len);
-void	sbdroprecord(struct sockbuf *sb);
-void	sbflush(struct sockbuf *sb);
-int		sbinsertoob(struct sockbuf *sb, struct mbuf *m0);
-void	sbrelease(struct sockbuf *sb);
-int	sbreserve(struct sockbuf *sb, u_long cc);
-void	sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb);
-int	sbwait(struct sockbuf *sb);
-int	sb_lock(struct sockbuf *sb);
-int	soabort(struct socket *so);
-int	soaccept(struct socket *so, struct sockaddr **nam);
-int	soacceptlock (struct socket *so, struct sockaddr **nam, int dolock);
-struct	socket *soalloc(int waitok, int dom, int type);
-int	sobind(struct socket *so, struct sockaddr *nam);
-void	socantrcvmore(struct socket *so);
-void	socantsendmore(struct socket *so);
-int	soclose(struct socket *so);
-int	soconnect(struct socket *so, struct sockaddr *nam);
-int	soconnectlock (struct socket *so, struct sockaddr *nam, int dolock);
-int	soconnect2(struct socket *so1, struct socket *so2);
-int	socreate(int dom, struct socket **aso, int type, int proto);
-void	sodealloc(struct socket *so);
-int	sodisconnect(struct socket *so);
-void	sofree(struct socket *so);
-int	sogetopt(struct socket *so, struct sockopt *sopt);
-void	sohasoutofband(struct socket *so);
-void	soisconnected(struct socket *so);
-void	soisconnecting(struct socket *so);
-void	soisdisconnected(struct socket *so);
-void	soisdisconnecting(struct socket *so);
-int	solisten(struct socket *so, int backlog);
-struct socket *
-	sodropablereq(struct socket *head);
-struct socket *
-	sonewconn(struct socket *head, int connstatus, const struct sockaddr* from);
-int	sooptcopyin(struct sockopt *sopt, void *data, size_t len, size_t minlen);
-int	sooptcopyout(struct sockopt *sopt, void *data, size_t len);
-int 	socket_lock(struct socket *so, int refcount);
-int 	socket_unlock(struct socket *so, int refcount);
+/* Not exported */
+extern void socketinit(void) __attribute__((section("__TEXT, initcode")));
+
+/* Exported */
+extern struct sockaddr *dup_sockaddr(struct sockaddr *sa, int canwait);
+extern int getsock(struct filedesc *fdp, int fd, struct file **fpp);
+extern int sockargs(struct mbuf **mp, user_addr_t data, int buflen, int type);
+
+extern int sbappend(struct sockbuf *sb, struct mbuf *m);
+extern int sbappendstream(struct sockbuf *sb, struct mbuf *m);
+extern int sbappendaddr(struct sockbuf *sb, struct sockaddr *asa,
+    struct mbuf *m0, struct mbuf *control, int *error_out);
+extern int sbappendcontrol(struct sockbuf *sb, struct mbuf *m0,
+    struct mbuf *control, int *error_out);
+extern int sbappendrecord(struct sockbuf *sb, struct mbuf *m0);
+extern void sbcheck(struct sockbuf *sb);
+extern void sblastmbufchk(struct sockbuf *, const char *);
+extern void sblastrecordchk(struct sockbuf *, const char *);
+extern struct mbuf *sbcreatecontrol(caddr_t p, int size, int type, int level);
+extern void sbdrop(struct sockbuf *sb, int len);
+extern void sbdroprecord(struct sockbuf *sb);
+extern void sbflush(struct sockbuf *sb);
+extern int sbinsertoob(struct sockbuf *sb, struct mbuf *m0);
+extern void sbrelease(struct sockbuf *sb);
+extern int sbreserve(struct sockbuf *sb, u_long cc);
+extern void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb);
+extern int sbwait(struct sockbuf *sb);
+extern int sb_lock(struct sockbuf *sb);
+extern void sb_empty_assert(struct sockbuf *, const char *);
+
+extern int soabort(struct socket *so);
+extern int soaccept(struct socket *so, struct sockaddr **nam);
+extern int soacceptlock(struct socket *so, struct sockaddr **nam, int dolock);
+extern int soacceptfilter(struct socket *so);
+extern struct socket *soalloc(int waitok, int dom, int type);
+extern int sobind(struct socket *so, struct sockaddr *nam);
+extern void socantrcvmore(struct socket *so);
+extern void socantsendmore(struct socket *so);
+extern int soclose(struct socket *so);
+extern int soconnect(struct socket *so, struct sockaddr *nam);
+extern int soconnectlock(struct socket *so, struct sockaddr *nam, int dolock);
+extern int soconnect2(struct socket *so1, struct socket *so2);
+extern int socreate(int dom, struct socket **aso, int type, int proto);
+extern void sodealloc(struct socket *so);
+extern int sodisconnect(struct socket *so);
+extern int sodisconnectlocked(struct socket *so);
+extern void sofree(struct socket *so);
+extern void soreference(struct socket *so);
+extern void sodereference(struct socket *so);
+extern void somultipages(struct socket *, boolean_t);
+extern int sogetopt(struct socket *so, struct sockopt *sopt);
+extern void sohasoutofband(struct socket *so);
+extern void soisconnected(struct socket *so);
+extern void soisconnecting(struct socket *so);
+extern void soisdisconnected(struct socket *so);
+extern void soisdisconnecting(struct socket *so);
+extern int solisten(struct socket *so, int backlog);
+extern struct socket *sodropablereq(struct socket *head);
+extern struct socket *sonewconn(struct socket *head, int connstatus,
+    const struct sockaddr *from);
+extern int sooptcopyin(struct sockopt *sopt, void *data, size_t len,
+    size_t minlen);
+extern int sooptcopyout(struct sockopt *sopt, void *data, size_t len);
+extern int socket_lock(struct socket *so, int refcount);
+extern int socket_unlock(struct socket *so, int refcount);
+extern void sofreelastref(struct socket *, int);
 
 /*
  * XXX; prepare mbuf for (__FreeBSD__ < 3) routines.
  * Used primarily in IPSec and IPv6 code.
  */
-int	soopt_getm(struct sockopt *sopt, struct mbuf **mp);
-int	soopt_mcopyin(struct sockopt *sopt, struct mbuf *m);
-int	soopt_mcopyout(struct sockopt *sopt, struct mbuf *m);
-
-int	sopoll(struct socket *so, int events, struct ucred *cred, void *wql);
-int	soreceive(struct socket *so, struct sockaddr **paddr,
-		       struct uio *uio, struct mbuf **mp0,
-		       struct mbuf **controlp, int *flagsp);
-int	soreserve(struct socket *so, u_long sndcc, u_long rcvcc);
-void	sorflush(struct socket *so);
-int	sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
-		    struct mbuf *top, struct mbuf *control, int flags);
-
-int	sosetopt(struct socket *so, struct sockopt *sopt);
-int	soshutdown(struct socket *so, int how);
-void	sotoxsocket(struct socket *so, struct xsocket *xso);
-void	sowakeup(struct socket *so, struct sockbuf *sb);
-int	soioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p);
+extern int soopt_getm(struct sockopt *sopt, struct mbuf **mp);
+extern int soopt_mcopyin(struct sockopt *sopt, struct mbuf *m);
+extern int soopt_mcopyout(struct sockopt *sopt, struct mbuf *m);
+
+extern int sopoll(struct socket *so, int events, struct ucred *cred, void *wql);
+extern int soreceive(struct socket *so, struct sockaddr **paddr,
+    struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp);
+extern int soreserve(struct socket *so, u_long sndcc, u_long rcvcc);
+extern void sorflush(struct socket *so);
+extern int sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
+    struct mbuf *top, struct mbuf *control, int flags);
+
+extern int sosetopt(struct socket *so, struct sockopt *sopt);
+extern int soshutdown(struct socket *so, int how);
+extern int soshutdownlock(struct socket *so, int how);
+extern void sotoxsocket(struct socket *so, struct xsocket *xso);
+extern void sowakeup(struct socket *so, struct sockbuf *sb);
+extern int soioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p);
 
 #ifndef __APPLE__
 /* accept filter functions */
-int	accept_filt_add(struct accept_filter *filt);
-int	accept_filt_del(char *name);
-struct accept_filter *	accept_filt_get(char *name);
+extern int accept_filt_add(struct accept_filter *filt);
+extern int accept_filt_del(char *name);
+extern struct accept_filter *accept_filt_get(char *name);
 #ifdef ACCEPT_FILTER_MOD
-int accept_filt_generic_mod_event(module_t mod, int event, void *data);
+extern int accept_filt_generic_mod_event(module_t mod, int event, void *data);
 SYSCTL_DECL(_net_inet_accf);
 #endif /* ACCEPT_FILTER_MOD */
 #endif /* !defined(__APPLE__) */
diff --git a/bsd/sys/sockio.h b/bsd/sys/sockio.h
index 701b94c1b..405c8088b 100644
--- a/bsd/sys/sockio.h
+++ b/bsd/sys/sockio.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -76,16 +82,27 @@
 #define        SIOCDELRT        _IOW('r', 11, struct ortentry) /* delete route */
 #endif
 
+/*
+ * OSIOCGIF* ioctls are deprecated; they are kept for binary compatibility.
+ */
 #define	SIOCSIFADDR	_IOW('i', 12, struct ifreq)	/* set ifnet address */
-#define	OSIOCGIFADDR	_IOWR('i', 13, struct ifreq)	/* get ifnet address */
+#ifdef KERNEL_PRIVATE
+#define	OSIOCGIFADDR	_IOWR('i', 13, struct ifreq)	/* deprecated */
+#endif /* KERNEL_PRIVATE */
 #define	SIOCSIFDSTADDR	 _IOW('i', 14, struct ifreq)	/* set p-p address */
-#define	OSIOCGIFDSTADDR	_IOWR('i', 15, struct ifreq)	/* get p-p address */
+#ifdef KERNEL_PRIVATE
+#define	OSIOCGIFDSTADDR	_IOWR('i', 15, struct ifreq)	/* deprecated */
+#endif /* KERNEL_PRIVATE */
 #define	SIOCSIFFLAGS	 _IOW('i', 16, struct ifreq)	/* set ifnet flags */
 #define	SIOCGIFFLAGS	_IOWR('i', 17, struct ifreq)	/* get ifnet flags */
-#define	OSIOCGIFBRDADDR	_IOWR('i', 18, struct ifreq)	/* get broadcast addr */
+#ifdef KERNEL_PRIVATE
+#define	OSIOCGIFBRDADDR	_IOWR('i', 18, struct ifreq)	/* deprecated */
+#endif /* KERNEL_PRIVATE */
 #define	SIOCSIFBRDADDR	 _IOW('i', 19, struct ifreq)	/* set broadcast addr */
-#define	OSIOCGIFCONF	_IOWR('i', 20, struct ifconf)	/* get ifnet list */
-#define	OSIOCGIFNETMASK	_IOWR('i', 21, struct ifreq)	/* get net addr mask */
+#ifdef KERNEL_PRIVATE
+#define	OSIOCGIFCONF	_IOWR('i', 20, struct ifconf)	/* deprecated */
+#define	OSIOCGIFNETMASK	_IOWR('i', 21, struct ifreq)	/* deprecated */
+#endif /* KERNEL_PRIVATE */
 #define	SIOCSIFNETMASK	 _IOW('i', 22, struct ifreq)	/* set net addr mask */
 #define	SIOCGIFMETRIC	_IOWR('i', 23, struct ifreq)	/* get IF metric */
 #define	SIOCSIFMETRIC	_IOW('i', 24, struct ifreq)	/* set IF metric */
@@ -171,4 +188,9 @@
 #define SIOCSETOT     _IOW('s', 128, int)             /* set socket for LibOT */
 #endif /* PRIVATE */
 
+#define SIOCGIFMAC	_IOWR('i', 130, struct ifreq)	/* get IF MAC label */
+#define SIOCSIFMAC	_IOW('i', 131, struct ifreq)	/* set IF MAC label */
+#define	SIOCSIFKPI	_IOW('i', 134, struct ifreq) /* set interface kext param - root only */
+#define	SIOCGIFKPI	_IOWR('i', 135, struct ifreq) /* get interface kext param */
+
 #endif /* !_SYS_SOCKIO_H_ */
diff --git a/bsd/sys/spawn.h b/bsd/sys/spawn.h
new file mode 100644
index 000000000..c9f9e018f
--- /dev/null
+++ b/bsd/sys/spawn.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * [SPN] Support for _POSIX_SPAWN
+ *
+ * This header contains information that is shared between the user space
+ * and kernel versions of the posix_spawn() code.  Shared elements are all
+ * manifest constants, at the current time.
+ */
+
+#ifndef	_SYS_SPAWN_H_
+#define	_SYS_SPAWN_H_
+
+/*
+ * Possible bit values which may be OR'ed together and provided as the second
+ * parameter to posix_spawnattr_setflags() or implicit returned in the value of
+ * the second parameter to posix_spawnattr_getflags().
+ */
+#define	POSIX_SPAWN_RESETIDS		0x0001	/* [SPN] R[UG]ID not E[UG]ID */
+#define	POSIX_SPAWN_SETPGROUP		0x0002	/* [SPN] set non-parent PGID */
+#define	POSIX_SPAWN_SETSIGDEF		0x0004	/* [SPN] reset sigset default */
+#define	POSIX_SPAWN_SETSIGMASK		0x0008	/* [SPN] set signal mask */
+
+#if 0	/* _POSIX_PRIORITY_SCHEDULING [PS] : not supported */
+#define	POSIX_SPAWN_SETSCHEDPARAM	0x0010
+#define	POSIX_SPAWN_SETSCHEDULER	0x0020
+#endif	/* 0 */
+
+#if	!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+/*
+ * Darwin-specific flags
+ */
+#define	POSIX_SPAWN_SETEXEC		0x0040
+#define	POSIX_SPAWN_START_SUSPENDED	0x0080
+
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
+#endif	/* _SYS_SPAWN_H_ */
diff --git a/bsd/sys/spawn_internal.h b/bsd/sys/spawn_internal.h
new file mode 100644
index 000000000..fe3524b84
--- /dev/null
+++ b/bsd/sys/spawn_internal.h
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+
+/*
+ * [SPN] Support for _POSIX_SPAWN
+ *
+ * This file contains intern datastructures which are externally represented
+ * as opaque void pointers to prevent introspection.  This permits us to
+ * change the underlying implementation of the code to maintain it or to
+ * support new features, as needed, without the consumer needing to recompile
+ * their code because of structure size changes or data reorganization.
+ */
+
+#ifndef	_SYS_SPAWN_INTERNAL_H_
+#define	_SYS_SPAWN__INTERNALH_
+
+#include <sys/_types.h>		/* __offsetof(), __darwin_size_t */
+#include <sys/syslimits.h>	/* PATH_MAX */
+#include <sys/spawn.h>
+#include <mach/machine.h>
+#include <mach/port.h>
+#include <mach/exception_types.h>
+
+/*
+ * Allowable posix_spawn() port action types
+ */
+typedef enum {
+	PSPA_SPECIAL = 0,
+	PSPA_EXCEPTION = 1,
+} pspa_t;
+
+/*
+ * Internal representation of one port to be set on posix_spawn().
+ * Currently this is limited to setting special and exception ports,
+ * but could be extended to other inheritable port types.
+ */
+typedef struct _ps_port_action {
+	pspa_t			port_type;
+	exception_mask_t	mask;
+	mach_port_t		new_port;
+	exception_behavior_t	behavior;
+	thread_state_flavor_t	flavor;
+	int			which;
+} _ps_port_action_t;
+
+/*
+ * A collection of port actions to take on the newly spawned process.
+ */
+typedef struct _posix_spawn_port_actions {
+	int			pspa_alloc;
+	int			pspa_count;
+	_ps_port_action_t 	pspa_actions[];
+} *_posix_spawn_port_actions_t;
+
+/*
+ * Returns size in bytes of a _posix_spawn_port_actions holding x elements.
+ */
+#define	PS_PORT_ACTIONS_SIZE(x)	\
+	__offsetof(struct _posix_spawn_port_actions, pspa_actions[(x)])
+
+#define NBINPREFS	4
+
+/*
+ * A posix_spawnattr structure contains all of the attribute elements that
+ * can be set, as well as any metadata whose validity is signalled by the
+ * presence of a bit in the flags field.  All fields are initialized to the
+ * appropriate default values by posix_spawnattr_init().
+ */
+typedef struct _posix_spawnattr {
+	short		psa_flags;		/* spawn attribute flags */
+	sigset_t	psa_sigdefault;		/* signal set to default */
+	sigset_t	psa_sigmask;		/* signal set to mask */
+	pid_t		psa_pgroup;		/* pgroup to spawn into */
+	cpu_type_t	psa_binprefs[NBINPREFS];   /* cpu affinity prefs*/
+	_posix_spawn_port_actions_t	psa_ports; /* special/exception ports */
+} *_posix_spawnattr_t;
+
+
+/*
+ * Allowable posix_spawn() file actions
+ */
+typedef enum {
+	PSFA_OPEN = 0,
+	PSFA_CLOSE = 1,
+	PSFA_DUP2 = 2
+} psfa_t;
+
+
+/*
+ * A posix_spawn() file action record for a single action
+ *
+ * Notes:	We carry around the full open arguments for both the open
+ *		and the close to permit the use of a single array of action
+ *		elements to be associated with a file actions object.
+ *
+ *		A possible future optimization would be to break this into
+ *		a variable sized vector list to save space (i.e. a separate
+ *		string area, allocation of least amount of path buffer per
+ *		open action, etc.).
+ *
+ * XXX:		Currently overloading psfao_oflag for PSFA_DUP2
+ */
+typedef struct _psfa_action {
+	psfa_t	psfaa_type;			/* file action type */
+	int	psfaa_filedes;			/* fd to operate on */
+	struct _psfaa_open {
+		int	psfao_oflag;		/* open flags to use */
+		mode_t	psfao_mode;		/* mode for open */
+		char	psfao_path[PATH_MAX];	/* path to open */
+	} psfaa_openargs;
+} _psfa_action_t;
+
+
+/*
+ * Internal representation of posix_spawn() file actions structure
+ *
+ * Notes:	This is implemented as a structure followed by an array of
+ *		file action records.  The psfa_act_alloc value is the number
+ *		of elements allocated in this array, and the psfa_act_count is
+ *		the number of elements currently in use (to permit some form
+ *		of preallocation, e.g. a power of 2 growth for reallocation,
+ *		etc.).
+ *
+ *		A possible future optimization would keep a size value and
+ *		a structure base reference pointer to permit copyin to the
+ *		kernel directly as a single blob, without damaging relative
+ *		internal pointer math.  It's probably better that this be a
+ *		long long rather than a true pointer, to make it invariant
+ *		for 32 vs. 64 bt programming SPIs.
+ */
+typedef struct _posix_spawn_file_actions {
+	int		psfa_act_alloc;		/* available actions space */
+	int		psfa_act_count;		/* count of defined actions */
+	_psfa_action_t	psfa_act_acts[];	/* actions array (uses c99) */
+} *_posix_spawn_file_actions_t;
+
+/*
+ * Calculate the size of a structure, given the number of elements that it is
+ * capable of containing.
+ */
+#define	PSF_ACTIONS_SIZE(x)	\
+	__offsetof(struct _posix_spawn_file_actions, psfa_act_acts[(x)])
+
+/*
+ * Initial count of actions in a struct _posix_spawn_file_actions after it is
+ * first allocated; this should be non-zero, since we expect that one would not
+ * have been allocated unless there was an intent to use it.
+ */
+#define	PSF_ACTIONS_INIT_COUNT	2
+
+/*
+ * Structure defining the true third argument to the posix_spawn() system call
+ * entry point; we wrap it and pass a descriptor so that we can know the
+ * copyin size ahead of time, and deal with copying in variant lists of things
+ * as single monolithic units, instead of many individual elements.  This is a
+ * performance optimization.
+ */
+struct _posix_spawn_args_desc {
+	__darwin_size_t		attr_size;	/* size of attributes block */
+	_posix_spawnattr_t	attrp;		/* pointer to block */
+	__darwin_size_t	file_actions_size;	/* size of file actions block */
+	_posix_spawn_file_actions_t
+				file_actions;	/* pointer to block */
+	__darwin_size_t	port_actions_size; /* size of port actions block */
+	_posix_spawn_port_actions_t
+				port_actions; 	/* pointer to port block */
+};
+
+#ifdef KERNEL
+#include <sys/appleapiopts.h>
+#ifdef __APPLE_API_PRIVATE
+
+#if __DARWIN_ALIGN_NATURAL
+#pragma options align=natural
+#endif
+
+struct user__posix_spawn_args_desc {
+	user_size_t		attr_size;	/* size of attributes block */
+	user_addr_t		attrp;		/* pointer to block */
+	user_size_t	file_actions_size;	/* size of file actions block */
+	user_addr_t		file_actions;	/* pointer to block */
+	user_size_t	port_actions_size;	/* size of port actions block */
+	user_addr_t		port_actions;	/* pointer to block */
+};
+
+
+#if __DARWIN_ALIGN_NATURAL
+#pragma options align=reset
+#endif
+
+#endif	/* __APPLE_API_PRIVATE */
+#endif	/* KERNEL */
+	
+#endif	/* _SYS_SPAWN_INTERNAL_H_ */
diff --git a/bsd/sys/stat.h b/bsd/sys/stat.h
index 2ac214111..f60c65821 100644
--- a/bsd/sys/stat.h
+++ b/bsd/sys/stat.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -66,6 +72,16 @@
 
 #include <sys/_types.h>
 #include <sys/cdefs.h>
+#ifdef KERNEL
+#include <machine/types.h>
+#endif /* KERNEL */
+
+/* [XSI] The timespec structure may be defined as described in <time.h> */
+#define __need_struct_timespec
+#ifdef KERNEL
+#define __need_struct_user_timespec
+#endif /* KERNEL */
+#include <sys/_structs.h>
 
 /*
  * [XSI] The blkcnt_t, blksize_t, dev_t, ino_t, mode_t, nlink_t, uid_t,
@@ -92,6 +108,13 @@ typedef	__darwin_ino_t		ino_t;		/* inode number */
 #define _INO_T
 #endif
 
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef	_INO64_T
+typedef	__darwin_ino64_t	ino64_t;	/* 64bit inode number */
+#define _INO64_T
+#endif
+#endif /* !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) */
+
 #ifndef	_MODE_T
 typedef	__darwin_mode_t		mode_t;
 #define _MODE_T
@@ -123,29 +146,7 @@ typedef	__darwin_time_t		time_t;
 #endif
 
 
-/* [XSI] The timespec structure may be defined as described in <time.h> */
-#ifndef _TIMESPEC
-#define _TIMESPEC
-struct timespec {
-	time_t	tv_sec;		/* seconds */
-	long	tv_nsec;	/* and nanoseconds */
-};
-
-// LP64todo - should this move?
-#ifdef KERNEL
-/* LP64 version of struct timespec.  time_t is a long and must grow when 
- * we're dealing with a 64-bit process.
- * WARNING - keep in sync with struct timespec
- */
-struct user_timespec {
-	user_time_t	tv_sec;		/* seconds */
-	int32_t	tv_nsec __attribute((aligned(8)));	/* and nanoseconds */
-};
-#endif // KERNEL
-#endif	/* _TIMESPEC */
-
-
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * XXX So deprecated, it would make your head spin
  *
@@ -171,12 +172,65 @@ struct ostat {
 	__uint32_t	st_flags;	/* user defined flags for file */
 	__uint32_t	st_gen;		/* file generation number */
 };
-#endif /* !_POSIX_C_SOURCE */
+
+#define __DARWIN_STRUCT_STAT64_TIMES \
+	struct timespec st_atimespec;		/* time of last access */ \
+	struct timespec st_mtimespec;		/* time of last data modification */ \
+	struct timespec st_ctimespec;		/* time of last status change */ \
+	struct timespec st_birthtimespec;	/* time of file creation(birth) */
+
+#else /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+
+#define __DARWIN_STRUCT_STAT64_TIMES \
+	time_t		st_atime;		/* [XSI] Time of last access */ \
+	long		st_atimensec;		/* nsec of last access */ \
+	time_t		st_mtime;		/* [XSI] Last data modification time */ \
+	long		st_mtimensec;		/* last data modification nsec */ \
+	time_t		st_ctime;		/* [XSI] Time of last status change */ \
+	long		st_ctimensec;		/* nsec of last status change */ \
+	time_t		st_birthtime;		/*  File creation time(birth)  */ \
+	long		st_birthtimensec;	/* nsec of File creation time */
+
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
+/*
+ * This structure is used as the second parameter to the fstat64(),
+ * lstat64(), and stat64() functions, and for struct stat when
+ * __DARWIN_64_BIT_INO_T is set. __DARWIN_STRUCT_STAT64 is defined
+ * above, depending on whether we use struct timespec or the direct
+ * components.
+ *
+ * This is simillar to stat except for 64bit inode number
+ * number instead of 32bit ino_t and the addition of create(birth) time.
+ */
+#define __DARWIN_STRUCT_STAT64 { \
+	dev_t		st_dev;			/* [XSI] ID of device containing file */ \
+	mode_t		st_mode;		/* [XSI] Mode of file (see below) */ \
+	nlink_t		st_nlink;		/* [XSI] Number of hard links */ \
+	__darwin_ino64_t st_ino;		/* [XSI] File serial number */ \
+	uid_t		st_uid;			/* [XSI] User ID of the file */ \
+	gid_t		st_gid;			/* [XSI] Group ID of the file */ \
+	dev_t		st_rdev;		/* [XSI] Device ID */ \
+	__DARWIN_STRUCT_STAT64_TIMES \
+	off_t		st_size;		/* [XSI] file size, in bytes */ \
+	blkcnt_t	st_blocks;		/* [XSI] blocks allocated for file */ \
+	blksize_t	st_blksize;		/* [XSI] optimal blocksize for I/O */ \
+	__uint32_t	st_flags;		/* user defined flags for file */ \
+	__uint32_t	st_gen;			/* file generation number */ \
+	__int32_t	st_lspare;		/* RESERVED: DO NOT USE! */ \
+	__int64_t	st_qspare[2];		/* RESERVED: DO NOT USE! */ \
+}
 
 /*
  * [XSI] This structure is used as the second parameter to the fstat(),
  * lstat(), and stat() functions.
  */
+#if __DARWIN_64_BIT_INO_T
+
+struct stat __DARWIN_STRUCT_STAT64;
+
+#else /* !__DARWIN_64_BIT_INO_T */
+
 struct stat {
 	dev_t	 	st_dev;		/* [XSI] ID of device containing file */
 	ino_t	  	st_ino;		/* [XSI] File serial number */
@@ -185,7 +239,7 @@ struct stat {
 	uid_t		st_uid;		/* [XSI] User ID of the file */
 	gid_t		st_gid;		/* [XSI] Group ID of the file */
 	dev_t		st_rdev;	/* [XSI] Device ID */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 	struct	timespec st_atimespec;	/* time of last access */
 	struct	timespec st_mtimespec;	/* time of last data modification */
 	struct	timespec st_ctimespec;	/* time of last status change */
@@ -206,10 +260,16 @@ struct stat {
 	__int64_t	st_qspare[2];	/* RESERVED: DO NOT USE! */
 };
 
-// LP64todo - should this move?
-#ifdef KERNEL
-#include <machine/types.h>
+#endif /* __DARWIN_64_BIT_INO_T */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+
+struct stat64 __DARWIN_STRUCT_STAT64;
+
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
 
+#ifdef KERNEL
 /* LP64 version of struct stat.  time_t (see timespec) is a long and must 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with struct stat
@@ -223,7 +283,7 @@ struct user_stat {
 	uid_t		st_uid;		/* [XSI] User ID of the file */
 	gid_t		st_gid;		/* [XSI] Group ID of the file */
 	dev_t		st_rdev;	/* [XSI] Device ID */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 	struct	user_timespec st_atimespec; /* time of last access */
 	struct	user_timespec st_mtimespec; /* time of last data modification */
 	struct	user_timespec st_ctimespec; /* time of last status change */
@@ -246,14 +306,50 @@ struct user_stat {
 
 extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 
+
+struct user_stat64 {
+	dev_t		st_dev;					/* [XSI] ID of device containing file */
+	mode_t		st_mode;				/* [XSI] Mode of file (see below) */
+	nlink_t		st_nlink;				/* [XSI] Number of hard links */
+	ino64_t		st_ino;					/* [XSI] File serial number */
+	uid_t		st_uid;					/* [XSI] User ID of the file */
+	gid_t		st_gid;					/* [XSI] Group ID of the file */
+	dev_t		st_rdev;				/* [XSI] Device ID */
+#ifndef _POSIX_C_SOURCE
+	struct user_timespec st_atimespec;		/* time of last access */
+	struct user_timespec st_mtimespec;		/* time of last data modification */
+	struct user_timespec st_ctimespec;		/* time of last status change */
+	struct user_timespec st_birthtimespec;	/* time of file creation(birth) */
+#else
+	user_time_t	st_atime;				/* [XSI] Time of last access */
+	__int64_t	st_atimensec;			/* nsec of last access */
+	user_time_t	st_mtime;				/* [XSI] Last data modification time */
+	__int64_t	st_mtimensec;			/* last data modification nsec */
+	user_time_t	st_ctime;				/* [XSI] Time of last status change */
+	__int64_t	st_ctimensec;			/* nsec of last status change */
+	user_time_t	st_birthtime;			/*  File creation time(birth)  */
+	__int64_t	st_birthtimensec;		/* nsec of File creation time */
+#endif
+	off_t		st_size;				/* [XSI] file size, in bytes */
+	blkcnt_t	st_blocks;				/* [XSI] blocks allocated for file */
+	blksize_t	st_blksize;				/* [XSI] optimal blocksize for I/O */
+	__uint32_t	st_flags;				/* user defined flags for file */
+	__uint32_t	st_gen;					/* file generation number */
+	__uint32_t	st_lspare;				/* RESERVED: DO NOT USE! */
+	__int64_t	st_qspare[2];			/* RESERVED: DO NOT USE! */
+};
+
+extern void munge_stat64(struct stat64 *sbp, struct user_stat64 *usbp);
+
 #endif // KERNEL
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define st_atime st_atimespec.tv_sec
 #define st_mtime st_mtimespec.tv_sec
 #define st_ctime st_ctimespec.tv_sec
-#endif
+#define st_birthtime st_birthtimespec.tv_sec
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 /*
  * [XSI] The following are symbolic names for the values of type mode_t.  They
@@ -269,7 +365,7 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	S_IFREG		0100000		/* [XSI] regular */
 #define	S_IFLNK		0120000		/* [XSI] symbolic link */
 #define	S_IFSOCK	0140000		/* [XSI] socket */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	S_IFWHT		0160000		/* whiteout */
 #define S_IFXATTR	0200000		/* extended attribute */
 #endif
@@ -295,7 +391,7 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	S_ISGID		0002000		/* [XSI] set group id on execution */
 #define	S_ISVTX		0001000		/* [XSI] directory restrcted delete */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	S_ISTXT		S_ISVTX		/* sticky bit: not supported */
 #define	S_IREAD		S_IRUSR		/* backward compatability */
 #define	S_IWRITE	S_IWUSR		/* backward compatability */
@@ -316,7 +412,7 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	S_ISREG(m)	(((m) & 0170000) == 0100000)	/* regular file */
 #define	S_ISLNK(m)	(((m) & 0170000) == 0120000)	/* symbolic link */
 #define	S_ISSOCK(m)	(((m) & 0170000) == 0140000)	/* socket */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	S_ISWHT(m)	(((m) & 0170000) == 0160000)	/* whiteout */
 #define S_ISXATTR(m)	(((m) & 0200000) == 0200000)	/* extended attribute */
 #endif
@@ -357,7 +453,7 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	S_TYPEISTMO(buf)	(0)	/* Test for a typed memory object */
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ACCESSPERMS	(S_IRWXU|S_IRWXG|S_IRWXO)	/* 0777 */
 							/* 7777 */
 #define	ALLPERMS	(S_ISUID|S_ISGID|S_ISTXT|S_IRWXU|S_IRWXG|S_IRWXO)
@@ -376,6 +472,14 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	UF_IMMUTABLE	0x00000002	/* file may not be changed */
 #define	UF_APPEND	0x00000004	/* writes to file may only append */
 #define UF_OPAQUE	0x00000008	/* directory is opaque wrt. union */
+/*
+ * The following bit is reserved for FreeBSD.  It is not implemented
+ * in Mac OS X.
+ */
+/* #define UF_NOUNLINK	0x00000010 */	/* file may not be removed or renamed */
+/* Bits 0x0020 through 0x4000 are currently undefined. */
+#define UF_HIDDEN	0x00008000	/* hint that this item should not be */
+					/* displayed in a GUI */
 /*
  * Super-user changeable flags.
  */
@@ -383,6 +487,13 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 #define	SF_ARCHIVED	0x00010000	/* file is archived */
 #define	SF_IMMUTABLE	0x00020000	/* file may not be changed */
 #define	SF_APPEND	0x00040000	/* writes to file may only append */
+/*
+ * The following two bits are reserved for FreeBSD.  They are not
+ * implemented in Mac OS X.
+ */
+/* #define SF_NOUNLINK	0x00100000 */	/* file may not be removed or renamed */
+/* #define SF_SNAPSHOT	0x00200000 */	/* snapshot inode */
+/* NOTE: There is no SF_HIDDEN bit. */
 
 #ifdef KERNEL
 /*
@@ -398,17 +509,17 @@ extern void munge_stat(struct stat *sbp, struct user_stat *usbp);
 
 __BEGIN_DECLS
 /* [XSI] */
-int	chmod(const char *, mode_t);
-int	fchmod(int, mode_t);
-int	fstat(int, struct stat *);
-int	lstat(const char *, struct stat *);
+int	chmod(const char *, mode_t) __DARWIN_ALIAS(chmod);
+int	fchmod(int, mode_t) __DARWIN_ALIAS(fchmod);
+int	fstat(int, struct stat *) __DARWIN_INODE64(fstat);
+int	lstat(const char *, struct stat *) __DARWIN_INODE64(lstat);
 int	mkdir(const char *, mode_t);
 int	mkfifo(const char *, mode_t);
-int	stat(const char *, struct stat *);
+int	stat(const char *, struct stat *) __DARWIN_INODE64(stat);
 int	mknod(const char *, mode_t, dev_t);
 mode_t	umask(mode_t);
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #ifndef _FILESEC_T
 struct _filesec;
 typedef struct _filesec	*filesec_t;
@@ -418,13 +529,23 @@ int	chflags(const char *, __uint32_t);
 int	chmodx_np(const char *, filesec_t);
 int	fchflags(int, __uint32_t);
 int	fchmodx_np(int, filesec_t);
-int	fstatx_np(int, struct stat *, filesec_t);
-int	lstatx_np(const char *, struct stat *, filesec_t);
+int	fstatx_np(int, struct stat *, filesec_t) __DARWIN_INODE64(fstatx_np);
+int	lchflags(const char *, __uint32_t);
+int	lchmod(const char *, mode_t);
+int	lstatx_np(const char *, struct stat *, filesec_t) __DARWIN_INODE64(lstatx_np);
 int	mkdirx_np(const char *, filesec_t);
 int	mkfifox_np(const char *, filesec_t);
-int	statx_np(const char *, struct stat *, filesec_t);
+int	statx_np(const char *, struct stat *, filesec_t) __DARWIN_INODE64(statx_np);
 int	umaskx_np(filesec_t);
-#endif	/* POSIX_C_SOURCE */
+/* The following are simillar  to stat and friends except provide struct stat64 instead of struct stat  */
+int	fstatx64_np(int, struct stat64 *, filesec_t);
+int	lstatx64_np(const char *, struct stat64 *, filesec_t);
+int	statx64_np(const char *, struct stat64 *, filesec_t);
+int	fstat64(int, struct stat64 *);
+int	lstat64(const char *, struct stat64 *);
+int	stat64(const char *, struct stat64 *);
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
 __END_DECLS
 #endif
 #endif /* !_SYS_STAT_H_ */
diff --git a/bsd/sys/subr_prf.h b/bsd/sys/subr_prf.h
index 830ea6a74..c3408ef9f 100644
--- a/bsd/sys/subr_prf.h
+++ b/bsd/sys/subr_prf.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -69,11 +75,13 @@
 
 /*
  * "flags" argument to prf().
+ * NB: Used in integer flags field, private to bsd/kern/subr_prf.c
  */
-#define TOCONS	0x01
-#define TOTTY	0x02
-#define TOLOG	0x04
-#define	TOSTR	0x8
+#define TOCONS		0x00000001	/* output to console */
+#define TOTTY		0x00000002	/* output to tty */
+#define TOLOG		0x00000004	/* output to log (log lock not held) */
+#define TOSTR		0x00000008	/* output to string */
+#define TOLOGLOCKED	0x00000010	/* output to log (log lock held) */
 
 extern int prf(const char *fmt, va_list ap, int flags, struct tty *ttyp);
 
diff --git a/bsd/sys/sys_domain.h b/bsd/sys/sys_domain.h
index 166637902..013959c8a 100644
--- a/bsd/sys/sys_domain.h
+++ b/bsd/sys/sys_domain.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -99,8 +105,8 @@ extern struct domain systemdomain;
 
 /* built in system domain protocols init function */
 __BEGIN_DECLS
-int kern_event_init(void);
-int kern_control_init(void);
+int kern_event_init(void) __attribute__((section("__TEXT, initcode")));
+int kern_control_init(void) __attribute__((section("__TEXT, initcode")));
 __END_DECLS
 
 #endif /* KERNEL_PRIVATE */
diff --git a/bsd/sys/syscall.h b/bsd/sys/syscall.h
index 5d20a40f4..8dd2a6b20 100644
--- a/bsd/sys/syscall.h
+++ b/bsd/sys/syscall.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  * 
- * @APPLE_LICENSE_HEADER_START@ 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and 
- * are subject to the Apple Public Source License Version 1.1 (the 
- * "License").  You may not use this file except in compliance with the 
- * License.  Please obtain a copy of the License at 
- * http://www.apple.com/publicsource and read it before using this file. 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are 
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the 
- * License for the specific language governing rights and limitations 
- * under the License. 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * @APPLE_LICENSE_HEADER_END@ 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  * 
  * 
  * System call switch table.
@@ -49,11 +55,8 @@
 #define	SYS_chmod          15
 #define	SYS_chown          16
 #define	SYS_obreak         17
-#if COMPAT_GETFSSTAT
 #define	SYS_ogetfsstat     18
-#else
 #define	SYS_getfsstat      18
-#endif
 			/* 19  old lseek */
 #define	SYS_getpid         20
 			/* 21  old mount */
@@ -80,7 +83,7 @@
 #define	SYS_pipe           42
 #define	SYS_getegid        43
 #define	SYS_profil         44
-#define	SYS_ktrace         45
+			/* 45  old ktrace */
 #define	SYS_sigaction      46
 #define	SYS_getgid         47
 #define	SYS_sigprocmask    48
@@ -161,8 +164,8 @@
 #define	SYS_fchown         123
 #define	SYS_fchmod         124
 			/* 125  old recvfrom */
-			/* 126  old setreuid */
-			/* 127  old setregid */
+#define	SYS_setreuid       126
+#define	SYS_setregid       127
 #define	SYS_rename         128
 			/* 129  old truncate */
 			/* 130  old ftruncate */
@@ -177,7 +180,7 @@
 #define	SYS_futimes        139
 #define	SYS_adjtime        140
 			/* 141  old getpeername */
-			/* 142  old gethostid */
+#define	SYS_gethostuuid    142
 			/* 143  old sethostid */
 			/* 144  old getrlimit */
 			/* 145  old setrlimit */
@@ -190,21 +193,13 @@
 #define	SYS_setprivexec    152
 #define	SYS_pread          153
 #define	SYS_pwrite         154
-#if NFSSERVER
 #define	SYS_nfssvc         155
-#else
-			/* 155  */
-#endif
 			/* 156  old getdirentries */
 #define	SYS_statfs         157
 #define	SYS_fstatfs        158
 #define	SYS_unmount        159
 			/* 160  old async_daemon */
-#if NFSCLIENT
 #define	SYS_getfh          161
-#else
-			/* 161  */
-#endif
 			/* 162  old getdomainname */
 			/* 163  old setdomainname */
 			/* 164  */
@@ -212,7 +207,7 @@
 			/* 166  old exportfs */
 #define	SYS_mount          167
 			/* 168  old ustat */
-			/* 169  */
+#define	SYS_csops          169
 #define	SYS_table          170
 			/* 171  old wait3 */
 			/* 172  old rpause */
@@ -236,11 +231,7 @@
 #define	SYS_lstat          190
 #define	SYS_pathconf       191
 #define	SYS_fpathconf      192
-#if COMPAT_GETFSSTAT
-#define	SYS_getfsstat      193
-#else
 			/* 193  */
-#endif
 #define	SYS_getrlimit      194
 #define	SYS_setrlimit      195
 #define	SYS_getdirentries  196
@@ -271,11 +262,7 @@
 #define	SYS_setattrlist    221
 #define	SYS_getdirentriesattr 222
 #define	SYS_exchangedata   223
-#ifdef __APPLE_API_OBSOLETE
-#define	SYS_checkuseraccess 224
-#else
-			/* 224  HFS checkuseraccess check access to a file */
-#endif /* __APPLE_API_OBSOLETE */
+			/* 224  was checkuseraccess */
 #define	SYS_searchfs       225
 #define	SYS_delete         226
 #define	SYS_copyfile       227
@@ -295,16 +282,11 @@
 #define	SYS_flistxattr     241
 #define	SYS_fsctl          242
 #define	SYS_initgroups     243
-			/* 244  */
+#define	SYS_posix_spawn    244
 			/* 245  */
 			/* 246  */
-#if NFSCLIENT
 #define	SYS_nfsclnt        247
 #define	SYS_fhopen         248
-#else
-			/* 247  */
-			/* 248  */
-#endif
 			/* 249  */
 #define	SYS_minherit       250
 #define	SYS_semsys         251
@@ -313,7 +295,7 @@
 #define	SYS_semctl         254
 #define	SYS_semget         255
 #define	SYS_semop          256
-			/* 257 */
+			/* 257  */
 #define	SYS_msgctl         258
 #define	SYS_msgget         259
 #define	SYS_msgsnd         260
@@ -350,25 +332,25 @@
 #define	SYS_mkfifo_extended 291
 #define	SYS_mkdir_extended 292
 #define	SYS_identitysvc    293
-			/* 294  */
-			/* 295  */
-#define	SYS_load_shared_file 296
-#define	SYS_reset_shared_file 297
-#define	SYS_new_system_shared_regions 298
-#define	SYS_shared_region_map_file_np 299
-#define	SYS_shared_region_make_private_np 300
-			/* 301  */
-			/* 302  */
-			/* 303  */
-			/* 304  */
-			/* 305  */
-			/* 306  */
-			/* 307  */
-			/* 308  */
-			/* 309  */
+#define	SYS_shared_region_check_np 294
+#define	SYS_shared_region_map_np 295
+			/* 296  old load_shared_file */
+			/* 297  old reset_shared_file */
+			/* 298  old new_system_shared_regions */
+			/* 299  old shared_region_map_file_np */
+			/* 300  old shared_region_make_private_np */
+#define	SYS___pthread_mutex_destroy 301
+#define	SYS___pthread_mutex_init 302
+#define	SYS___pthread_mutex_lock 303
+#define	SYS___pthread_mutex_trylock 304
+#define	SYS___pthread_mutex_unlock 305
+#define	SYS___pthread_cond_init 306
+#define	SYS___pthread_cond_destroy 307
+#define	SYS___pthread_cond_broadcast 308
+#define	SYS___pthread_cond_signal 309
 #define	SYS_getsid         310
 #define	SYS_settid_with_pid 311
-			/* 312  */
+#define	SYS___pthread_cond_timedwait 312
 #define	SYS_aio_fsync      313
 #define	SYS_aio_return     314
 #define	SYS_aio_suspend    315
@@ -377,35 +359,35 @@
 #define	SYS_aio_read       318
 #define	SYS_aio_write      319
 #define	SYS_lio_listio     320
-			/* 321  */
-			/* 322  */
+#define	SYS___pthread_cond_wait 321
+#define	SYS_iopolicysys    322
 			/* 323  */
 #define	SYS_mlockall       324
 #define	SYS_munlockall     325
 			/* 326  */
 #define	SYS_issetugid      327
 #define	SYS___pthread_kill 328
-#define	SYS_pthread_sigmask 329
-#define	SYS_sigwait        330
+#define	SYS___pthread_sigmask 329
+#define	SYS___sigwait      330
 #define	SYS___disable_threadsignal 331
 #define	SYS___pthread_markcancel 332
 #define	SYS___pthread_canceled 333
 #define	SYS___semwait_signal 334
-#define	SYS_utrace         335
+			/* 335  old utrace */
 #define	SYS_proc_info      336
-			/* 337  */
-			/* 338  */
-			/* 339  */
-			/* 340  */
-			/* 341  */
-			/* 342  */
-			/* 343  */
-			/* 344  */
-			/* 345  */
-			/* 346  */
-			/* 347  */
-			/* 348  */
-			/* 349  */
+#define	SYS_sendfile       337
+#define	SYS_stat64         338
+#define	SYS_fstat64        339
+#define	SYS_lstat64        340
+#define	SYS_stat64_extended 341
+#define	SYS_lstat64_extended 342
+#define	SYS_fstat64_extended 343
+#define	SYS_getdirentries64 344
+#define	SYS_statfs64       345
+#define	SYS_fstatfs64      346
+#define	SYS_getfsstat64    347
+#define	SYS___pthread_chdir 348
+#define	SYS___pthread_fchdir 349
 #define	SYS_audit          350
 #define	SYS_auditon        351
 			/* 352  */
@@ -416,17 +398,74 @@
 #define	SYS_getaudit_addr  357
 #define	SYS_setaudit_addr  358
 #define	SYS_auditctl       359
-			/* 360  */
-			/* 361  */
+#define	SYS_bsdthread_create 360
+#define	SYS_bsdthread_terminate 361
 #define	SYS_kqueue         362
 #define	SYS_kevent         363
 #define	SYS_lchown         364
 #define	SYS_stack_snapshot 365
-			/* 366  */
-			/* 367  */
-			/* 368  */
+#define	SYS_bsdthread_register 366
+#define	SYS_workq_open     367
+#define	SYS_workq_ops      368
 			/* 369  */
-#define	SYS_MAXSYSCALL	370
+			/* 370  */
+			/* 371  */
+			/* 372  */
+			/* 373  */
+			/* 374  */
+			/* 375  */
+			/* 376  */
+			/* 377  */
+			/* 378  */
+			/* 379  */
+#define	SYS___mac_execve   380
+#define	SYS___mac_syscall  381
+#define	SYS___mac_get_file 382
+#define	SYS___mac_set_file 383
+#define	SYS___mac_get_link 384
+#define	SYS___mac_set_link 385
+#define	SYS___mac_get_proc 386
+#define	SYS___mac_set_proc 387
+#define	SYS___mac_get_fd   388
+#define	SYS___mac_set_fd   389
+#define	SYS___mac_get_pid  390
+#define	SYS___mac_get_lcid 391
+#define	SYS___mac_get_lctx 392
+#define	SYS___mac_set_lctx 393
+#define	SYS_setlcid        394
+#define	SYS_getlcid        395
+#define	SYS_read_nocancel  396
+#define	SYS_write_nocancel 397
+#define	SYS_open_nocancel  398
+#define	SYS_close_nocancel 399
+#define	SYS_wait4_nocancel 400
+#define	SYS_recvmsg_nocancel 401
+#define	SYS_sendmsg_nocancel 402
+#define	SYS_recvfrom_nocancel 403
+#define	SYS_accept_nocancel 404
+#define	SYS_msync_nocancel 405
+#define	SYS_fcntl_nocancel 406
+#define	SYS_select_nocancel 407
+#define	SYS_fsync_nocancel 408
+#define	SYS_connect_nocancel 409
+#define	SYS_sigsuspend_nocancel 410
+#define	SYS_readv_nocancel 411
+#define	SYS_writev_nocancel 412
+#define	SYS_sendto_nocancel 413
+#define	SYS_pread_nocancel 414
+#define	SYS_pwrite_nocancel 415
+#define	SYS_waitid_nocancel 416
+#define	SYS_poll_nocancel  417
+#define	SYS_msgsnd_nocancel 418
+#define	SYS_msgrcv_nocancel 419
+#define	SYS_sem_wait_nocancel 420
+#define	SYS_aio_suspend_nocancel 421
+#define	SYS___sigwait_nocancel 422
+#define	SYS___semwait_signal_nocancel 423
+#define	SYS___mac_mount    424
+#define	SYS___mac_get_mount 425
+#define	SYS___mac_getfsstat 426
+#define	SYS_MAXSYSCALL	427
 
 #endif /* __APPLE_API_PRIVATE */
 #endif /* !_SYS_SYSCALL_H_ */
diff --git a/bsd/sys/sysctl.h b/bsd/sys/sysctl.h
index 57b88b8db..6b72d6bd5 100644
--- a/bsd/sys/sysctl.h
+++ b/bsd/sys/sysctl.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -57,6 +63,12 @@
  *
  *	@(#)sysctl.h	8.1 (Berkeley) 6/2/93
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_SYSCTL_H_
 #define	_SYS_SYSCTL_H_
@@ -76,7 +88,7 @@
 #include <sys/proc.h>
 #include <sys/vm.h>
 
-#ifdef BSD_KERNEL_PRIVATE
+#ifdef XNU_KERNEL_PRIVATE
 #include <sys/linker_set.h>
 #endif
 
@@ -97,6 +109,13 @@
  * levels defined below it, or it is a leaf of some particular
  * type given below. Each sysctl level defines a set of name/type
  * pairs to be used by sysctl(1) in manipulating the subsystem.
+ *
+ * When declaring new sysctl names, please use the CTLFLAG_LOCKED
+ * flag in the type to indicate that all necessary locking will
+ * be handled within the sysctl. Any sysctl defined without
+ * CTLFLAG_LOCKED is considered legacy and will be protected by
+ * both the kernel funnel and the sysctl memlock. This is not
+ * optimal, so it is best to handle locking yourself.
  */
 struct ctlname {
 	char	*ctl_name;	/* subsystem name */
@@ -120,20 +139,56 @@ struct ctlname {
 #define CTLFLAG_MASKED	0x04000000	/* deprecated variable, do not display */
 #define CTLFLAG_NOAUTO	0x02000000	/* do not auto-register */
 #define CTLFLAG_KERN	0x01000000	/* valid inside the kernel */
+#define CTLFLAG_LOCKED	0x00800000	/* node will handle locking itself (highly encouraged) */
 
 /*
  * USE THIS instead of a hardwired number from the categories below
  * to get dynamically assigned sysctl entries using the linker-set
  * technology. This is the way nearly all new sysctl variables should
  * be implemented.
+ *
  * e.g. SYSCTL_INT(_parent, OID_AUTO, name, CTLFLAG_RW, &variable, 0, "");
- */ 
+ *
+ * Note that linker set technology will automatically register all nodes
+ * declared like this on kernel initialization, UNLESS they are defined
+ * in I/O-Kit. In this case, you have to call sysctl_register_oid()
+ * manually - just like in a KEXT.
+ */
 #define OID_AUTO	(-1)
+#define OID_AUTO_START 100 /* conventional */
 
 #ifdef KERNEL
 #define SYSCTL_HANDLER_ARGS (struct sysctl_oid *oidp, void *arg1, int arg2, \
 	struct sysctl_req *req)
 
+/*
+ * Locking and stats
+ */
+struct sysctl_lock {
+	int	sl_lock;
+	int	sl_want;
+	int	sl_locked;
+};
+
+#define MEMLOCK_LOCK() \
+	do { \
+		while (memlock.sl_lock) { \
+			memlock.sl_want = 1; \
+			(void) tsleep((caddr_t)&memlock, PRIBIO+1, "sysctl", 0); \
+			memlock.sl_locked++; \
+		} \
+		memlock.sl_lock = 1; \
+	} while(0)
+
+#define MEMLOCK_UNLOCK() \
+	do { \
+		memlock.sl_lock = 0; \
+		if (memlock.sl_want) { \
+			memlock.sl_want = 0; \
+			wakeup((caddr_t)&memlock); \
+		} \
+	}while(0)
+
 /*
  * This describes the access space for a sysctl request.  This is needed
  * so that we can use the interface from the kernel or from user-space.
@@ -172,15 +227,21 @@ struct sysctl_oid {
 #define SYSCTL_IN(r, p, l) (r->newfunc)(r, p, l)
 #define SYSCTL_OUT(r, p, l) (r->oldfunc)(r, p, l)
 
+typedef int (* sysctl_handler_t) SYSCTL_HANDLER_ARGS;
+
 __BEGIN_DECLS
 
+/* old interface */
 int sysctl_handle_int SYSCTL_HANDLER_ARGS;
 int sysctl_handle_long SYSCTL_HANDLER_ARGS;
 int sysctl_handle_quad SYSCTL_HANDLER_ARGS;
 int sysctl_handle_int2quad SYSCTL_HANDLER_ARGS;
-/*int sysctl_handle_intptr SYSCTL_HANDLER_ARGS; XXX not implemented */
 int sysctl_handle_string SYSCTL_HANDLER_ARGS;
 int sysctl_handle_opaque SYSCTL_HANDLER_ARGS;
+/* new interface */
+int sysctl_io_number(struct sysctl_req *req, long long bigValue, size_t valueSize, void *pValue, int *changed);
+int sysctl_io_string(struct sysctl_req *req, char *pValue, size_t valueSize, int trunc, int *changed);
+int sysctl_io_opaque(struct sysctl_req *req, void *pValue, size_t valueSize, int *changed);
 
 /*
  * These functions are used to add/remove an oid from the mib.
@@ -188,18 +249,26 @@ int sysctl_handle_opaque SYSCTL_HANDLER_ARGS;
 void sysctl_register_oid(struct sysctl_oid *oidp);
 void sysctl_unregister_oid(struct sysctl_oid *oidp);
 
+/* Not exported */
+void sysctl_register_fixed(void);
+
 __END_DECLS
 
 /* Declare an oid to allow child oids to be added to it. */
 #define SYSCTL_DECL(name)					\
 	extern struct sysctl_oid_list sysctl_##name##_children
 
+#ifdef XNU_KERNEL_PRIVATE
+#define SYSCTL_LINKER_SET_ENTRY LINKER_SET_ENTRY
+#else
+#define SYSCTL_LINKER_SET_ENTRY(a, b)
+#endif
 /* This constructs a "raw" MIB oid. */
 #define SYSCTL_OID(parent, nbr, name, kind, a1, a2, handler, fmt, descr) \
-	struct sysctl_oid sysctl_##parent##_##name = {		 \
+	struct sysctl_oid sysctl_##parent##_##name = {			 \
 		&sysctl_##parent##_children, { 0 },			 \
-		nbr, kind, a1, a2, #name, handler, fmt };
-
+		nbr, kind, a1, a2, #name, handler, fmt };		 \
+	SYSCTL_LINKER_SET_ENTRY(__sysctl_set, sysctl_##parent##_##name)
 
 /* This constructs a node from which other oids can hang. */
 #define SYSCTL_NODE(parent, nbr, name, access, handler, descr)		    \
@@ -233,11 +302,6 @@ __END_DECLS
 	SYSCTL_OID(parent, nbr, name, CTLTYPE_QUAD|access, \
 		ptr, 0, sysctl_handle_quad, "Q", descr)
 
-/* Oid for a int returned as quad.  The pointer must be non NULL. */
-#define SYSCTL_INT2QUAD(parent, nbr, name, access, ptr, descr) \
-	SYSCTL_OID(parent, nbr, name, CTLTYPE_QUAD|access, \
-		ptr, 0, sysctl_handle_int2quad, "Q", descr)
-
 /* Oid for an opaque object.  Specified by a pointer and a length. */
 #define SYSCTL_OPAQUE(parent, nbr, name, access, ptr, len, fmt, descr) \
 	SYSCTL_OID(parent, nbr, name, CTLTYPE_OPAQUE|access, \
@@ -345,7 +409,7 @@ SYSCTL_DECL(_user);
 #define	KERN_LOGSIGEXIT	36	/* int: do we log sigexit procs? */
 #define KERN_SYMFILE		37	/* string: kernel symbol filename */
 #define KERN_PROCARGS		38
-#define KERN_PCSAMPLES		39	/* node: pc sampling */
+                             /* 39 was KERN_PCSAMPLES... now deprecated */
 #define KERN_NETBOOT		40	/* int: are we netbooted? 1=yes,0=no */
 #define	KERN_PANICINFO		41	/* node: panic UI information */
 #define	KERN_SYSV		42	/* node: System V IPC information */
@@ -373,8 +437,24 @@ SYSCTL_DECL(_user);
 #define KERN_NX_PROTECTION	60	/* int: whether no-execute protection is enabled */
 #define	KERN_TFP 		61	/* Task for pid settings */
 #define	KERN_PROCNAME 		62	/* setup process program  name(2*MAXCOMLEN) */
-#define	KERN_THALTSTACK 	63	/* setup process to have per thread sigaltstack */
-#define	KERN_MAXID		64	/* number of valid kern ids */
+#define	KERN_THALTSTACK		63	/* for compat with older x86 and does nothing */
+#define	KERN_SPECULATIVE_READS	64	/* int: whether speculative reads are disabled */
+#define	KERN_OSVERSION		65	/* for build number i.e. 9A127 */
+#define	KERN_SAFEBOOT		66	/* are we booted safe? */
+#define	KERN_LCTX		67	/* node: login context */
+#define KERN_RAGEVNODE		68
+#define KERN_TTY		69	/* node: tty settings */
+#define KERN_CHECKOPENEVT       70      /* spi: check the VOPENEVT flag on vnodes at open time */
+#define	KERN_MAXID		71	/* number of valid kern ids */
+/*
+ * Don't add any more sysctls like this.  Instead, use the SYSCTL_*() macros
+ * and OID_AUTO. This will have the added benefit of not having to recompile
+ * sysctl(8) to pick up your changes.
+ */
+
+#if COUNT_SYSCALLS && defined(KERNEL) 
+#define	KERN_COUNT_SYSCALLS (KERN_OSTYPE + 1000)	/* keep called count for each bsd syscall */
+#endif
 
 #if defined(__LP64__)
 #define	KERN_USRSTACK KERN_USRSTACK64
@@ -382,15 +462,23 @@ SYSCTL_DECL(_user);
 #define	KERN_USRSTACK KERN_USRSTACK32
 #endif
 
+
+/* KERN_RAGEVNODE types */
+#define KERN_RAGE_PROC		1
+#define KERN_RAGE_THREAD	2
+#define KERN_UNRAGE_PROC	3
+#define KERN_UNRAGE_THREAD	4
+
+/* KERN_OPENEVT types */
+#define KERN_OPENEVT_PROC     1
+#define KERN_UNOPENEVT_PROC   2
+
 /* KERN_TFP types */
 #define KERN_TFP_POLICY 		1
-#define KERN_TFP_READ_GROUP 	2
-#define KERN_TFP_RW_GROUP 		3
 
 /* KERN_TFP_POLICY values . All policies allow task port for self */
 #define KERN_TFP_POLICY_DENY 		0 	/* Deny Mode: None allowed except privileged */
-#define KERN_TFP_POLICY_PERMISSIVE 	1	/* Permissive Mode: related ones allowed or privileged */
-#define KERN_TFP_POLICY_RESTRICTED 	2	/* Restricted Mode: privileged or setgid and realted */
+#define KERN_TFP_POLICY_DEFAULT 	2	/* Default  Mode: related ones allowed and upcall authentication */
 
 /* KERN_KDEBUG types */
 #define KERN_KDEFLAGS		1
@@ -410,35 +498,10 @@ SYSCTL_DECL(_user);
 #define KERN_KDSETRTCDEC        15
 #define KERN_KDGETENTROPY       16
 
-/* KERN_PCSAMPLES types */
-#define KERN_PCDISABLE		1
-#define KERN_PCSETBUF	        2
-#define KERN_PCGETBUF		3
-#define KERN_PCSETUP		4
-#define KERN_PCREMOVE		5
-#define KERN_PCREADBUF		6
-#define KERN_PCSETREG           7
-#define KERN_PCCOMM             8
-
 /* KERN_PANICINFO types */
 #define	KERN_PANICINFO_MAXSIZE	1	/* quad: panic UI image size limit */
 #define	KERN_PANICINFO_IMAGE	2	/* panic UI in 8-bit kraw format */
 
-/*
- * KERN_SYSV identifiers
- */
-#define KSYSV_SHMMAX		1	/* int: max shared memory segment size (bytes) */
-#define	KSYSV_SHMMIN		2	/* int: min shared memory segment size (bytes) */
-#define	KSYSV_SHMMNI		3	/* int: max number of shared memory identifiers */
-#define	KSYSV_SHMSEG		4	/* int: max shared memory segments per process */
-#define	KSYSV_SHMALL		5	/* int: max amount of shared memory (pages) */
-#define KSYSV_SEMMNI		6	/* int: max num of semaphore identifiers  */
-#define KSYSV_SEMMNS		7	/* int: max num of semaphores in system */
-#define KSYSV_SEMMNU		8	/* int: max num of undo structures in system  */
-#define KSYSV_SEMMSL		9	/* int: max num of semaphores per id  */
-#define KSYSV_SEMUNE		10	/* int: max num of undo entries per process */
-
-
 #define CTL_KERN_NAMES { \
 	{ 0, 0 }, \
 	{ "ostype", CTLTYPE_STRING }, \
@@ -479,7 +542,7 @@ SYSCTL_DECL(_user);
 	{ "logsigexit", CTLTYPE_INT }, \
 	{ "symfile",CTLTYPE_STRING },\
 	{ "procargs",CTLTYPE_STRUCT },\
-	{ "pcsamples",CTLTYPE_STRUCT },\
+        { "dummy", CTLTYPE_INT },		/* deprecated pcsamples */ \
 	{ "netboot", CTLTYPE_INT }, \
 	{ "panicinfo", CTLTYPE_NODE }, \
 	{ "sysv", CTLTYPE_NODE }, \
@@ -503,7 +566,14 @@ SYSCTL_DECL(_user);
 	{ "nx", CTLTYPE_INT }, \
 	{ "tfp", CTLTYPE_NODE }, \
 	{ "procname", CTLTYPE_STRING }, \
-	{ "threadsigaltstack", CTLTYPE_INT } \
+	{ "threadsigaltstack", CTLTYPE_INT }, \
+	{ "speculative_reads_disabled", CTLTYPE_INT }, \
+	{ "osversion", CTLTYPE_STRING }, \
+	{ "safeboot", CTLTYPE_INT }, \
+	{ "lctx", CTLTYPE_NODE }, \
+	{ "rage_vnode", CTLTYPE_INT }, \
+	{ "tty", CTLTYPE_NODE },	\
+	{ "check_openevt", CTLTYPE_INT } \
 }
 
 /*
@@ -523,6 +593,13 @@ SYSCTL_DECL(_user);
 #define	KERN_PROC_TTY		4	/* by controlling tty */
 #define	KERN_PROC_UID		5	/* by effective uid */
 #define	KERN_PROC_RUID		6	/* by real uid */
+#define	KERN_PROC_LCID		7	/* by login context id */
+
+/*
+ * KERN_LCTX subtypes
+ */
+#define	KERN_LCTX_ALL		0	/* everything */
+#define	KERN_LCTX_LCID		1	/* by login context id */
 
 /* 
  * KERN_PROC subtype ops return arrays of augmented proc structures:
@@ -571,15 +648,23 @@ struct kinfo_proc {
 #define	EPROC_SLEADER	0x02	/* session leader */
 #define	COMAPT_MAXLOGNAME	12
 		char	e_login[COMAPT_MAXLOGNAME];	/* short setlogin() name */
+#if CONFIG_LCTX
+		pid_t	e_lcid;
+		int32_t	e_spare[3];
+#else
 		int32_t	e_spare[4];
+#endif
 	} kp_eproc;
 };
 
+struct kinfo_lctx {
+	pid_t	id;	/* Login Context ID */
+	int	mc;	/* Member Count */
+};
+
 #ifdef BSD_KERNEL_PRIVATE
 #include <sys/proc_internal.h>
 
-// LP64todo - should this move?
-
 /* LP64 version of _pcred.  all pointers 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with _pcred
@@ -620,7 +705,12 @@ struct user_kinfo_proc {
 		short	e_xswrss;
 		int32_t	e_flag;
 		char	e_login[COMAPT_MAXLOGNAME];	/* short setlogin() name */
+#if CONFIG_LCTX
+		pid_t	e_lcid;
+		int32_t	e_spare[3];
+#else
 		int32_t	e_spare[4];
+#endif
 	} kp_eproc;
 };
 
@@ -640,6 +730,7 @@ struct user_kinfo_proc {
 #define	KIPC_MAX_DATALEN	7	/* int: max length of data? */
 #define	KIPC_MBSTAT		8	/* struct: mbuf usage statistics */
 #define	KIPC_NMBCLUSTERS	9	/* int: maximum mbuf clusters */
+#define KIPC_SOQLIMITCOMPAT	10	/* int: socket queue limit */
 
 /*
  * CTL_VM identifiers
@@ -659,7 +750,7 @@ struct user_kinfo_proc {
 	{ "vmmeter", CTLTYPE_STRUCT }, \
 	{ "loadavg", CTLTYPE_STRUCT }, \
 	{ 0, 0 }, /* placeholder for "3" (see comment above) */ \
-	{ "machfactor", CTLTYPE_STRUCT }, \
+	{ "dummy", CTLTYPE_INT }, \
 	{ "swapusage", CTLTYPE_STRUCT } \
 }
 
@@ -681,7 +772,6 @@ struct loadavg {
 extern struct loadavg averunnable;
 #define LSCALE	1000		/* scaling for "fixed point" arithmetic */
 
-// LP64todo - should this move?
 #ifdef BSD_KERNEL_PRIVATE
 
 struct user_loadavg {
@@ -753,6 +843,8 @@ struct user_loadavg {
 }
 
 /*
+ * XXX This information should be moved to the man page.
+ *
  * These are the support HW selectors for sysctlbyname.  Parameters that are byte counts or frequencies are 64 bit numbers.
  * All other parameters are 32 bit numbers.
  *
@@ -804,6 +896,7 @@ struct user_loadavg {
  *   hw.l2cachesize            -
  *   hw.l3cachesize            -
  *
+ *   hw.packages               - Gives the number of processor packages.
  *
  * These are the selectors for optional processor features for specific processors.  Selectors that return errors are not support 
  * on the system.  Supported features will return 1 if they are recommended or 0 if they are supported but are not expected to help .
@@ -823,12 +916,13 @@ struct user_loadavg {
  *
  * For x86 Architecture:
  * 
- *   hw.optional.floatingpoint - Floating Point Instructions
- *   hw.optional.mmx           - Original MMX vector instructions
- *   hw.optional.sse           - Streaming SIMD Extensions
- *   hw.optional.sse2          - Streaming SIMD Extensions 2
- *   hw.optional.sse3          - Streaming SIMD Extensions 3
- *   hw.optional.x86_64        - 64-bit support
+ *   hw.optional.floatingpoint     - Floating Point Instructions
+ *   hw.optional.mmx               - Original MMX vector instructions
+ *   hw.optional.sse               - Streaming SIMD Extensions
+ *   hw.optional.sse2              - Streaming SIMD Extensions 2
+ *   hw.optional.sse3              - Streaming SIMD Extensions 3
+ *   hw.optional.supplementalsse3  - Supplemental Streaming SIMD Extensions 3
+ *   hw.optional.x86_64            - 64-bit support
  */
 
 
@@ -893,6 +987,12 @@ struct user_loadavg {
 #define	CTL_DEBUG_VALUE		1	/* int: variable value */
 #define	CTL_DEBUG_MAXID		20
 
+
+#if (CTL_MAXID != 9) || (KERN_MAXID != 71) || (VM_MAXID != 6) || (HW_MAXID != 26) || (USER_MAXID != 21) || (CTL_DEBUG_MAXID != 20)
+#error Use the SYSCTL_*() macros and OID_AUTO instead!
+#endif
+
+
 #ifdef	KERNEL
 #ifdef DEBUG
 /*
@@ -921,12 +1021,13 @@ extern struct ctldebug debug15, debug16, debug17, debug18, debug19;
 extern char	machine[];
 extern char	osrelease[];
 extern char	ostype[];
+extern char	osversion[];
 
 struct linker_set;
 
-void	sysctl_register_set(struct linker_set *lsp);
-void	sysctl_unregister_set(struct linker_set *lsp);
-void	sysctl_mib_init(void);
+void	sysctl_register_set(const char *set);
+void	sysctl_unregister_set(const char *set);
+void	sysctl_mib_init(void) __attribute__((section("__TEXT, initcode")));
 int	kernel_sysctl(struct proc *p, int *name, u_int namelen, void *old,
 		      size_t *oldlenp, void *newp, size_t newlen);
 int	userland_sysctl(struct proc *p, int *name, u_int namelen, user_addr_t old,
@@ -948,12 +1049,18 @@ typedef int (sysctlfn)
 int sysctl_int(user_addr_t, size_t *, user_addr_t, size_t, int *);
 int sysctl_rdint(user_addr_t, size_t *, user_addr_t, int);
 int sysctl_quad(user_addr_t, size_t *, user_addr_t, size_t, quad_t *);
-int sysctl_rdquad(void *, size_t *, void *, quad_t);
+int sysctl_rdquad(user_addr_t, size_t *, user_addr_t, quad_t);
 int sysctl_string(user_addr_t, size_t *, user_addr_t, size_t, char *, int);
 int sysctl_trstring(user_addr_t, size_t *, user_addr_t, size_t, char *, int);
 int sysctl_rdstring(user_addr_t, size_t *, user_addr_t, char *);
 int sysctl_rdstruct(user_addr_t, size_t *, user_addr_t, void *, int);
 
+/* XXX should be in <sys/sysproto.h>, but not a real system call */
+struct sysctl_args;
+int new_sysctl(struct proc *, struct sysctl_args *);
+
+void sysctl_register_all(void);
+
 #endif /* BSD_KERNEL_PRIVATE */
 #else	/* !KERNEL */
 
diff --git a/bsd/sys/sysent.h b/bsd/sys/sysent.h
index b8d73190d..59aec4ffc 100644
--- a/bsd/sys/sysent.h
+++ b/bsd/sys/sysent.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_SYSENT_H_
@@ -35,19 +41,29 @@
 typedef	int32_t	sy_call_t(struct proc *, void *, int *);
 typedef	void	sy_munge_t(const void *, void *);
 
-extern struct sysent {		/* system call table */
+struct sysent {		/* system call table */
 	int16_t		sy_narg;	/* number of args */
-	int8_t		sy_cancel;	/* funnel type */
-	int8_t		sy_funnel;	/* funnel type */
+	int8_t		sy_resv;	/* reserved  */
+	int8_t		sy_flags;	/* flags */
 	sy_call_t	*sy_call;	/* implementing function */
-	sy_munge_t	*sy_arg_munge32; /* system call aguments munger for 32-bit process */
-	sy_munge_t	*sy_arg_munge64; /* system call aguments munger for 64-bit process */
+	sy_munge_t	*sy_arg_munge32; /* system call arguments munger for 32-bit process */
+	sy_munge_t	*sy_arg_munge64; /* system call arguments munger for 64-bit process */
 	int32_t		sy_return_type; /* system call return types */
-} sysent[];
+	uint16_t	sy_arg_bytes;	/* Total size of arguments in bytes for
+					 * 32-bit system calls
+					 */
+};
+
+#ifndef __INIT_SYSENT_C__
+extern struct sysent sysent[];
+#endif	/* __INIT_SYSENT_C__ */
+
+extern int nsysent;
+#define NUM_SYSENT	427	/* Current number of defined syscalls */
 
 /* sy_funnel flags bits */
-#define FUNNEL_MASK	0x00ff
-#define	UNSAFE_64BIT	0x0100
+#define FUNNEL_MASK	0x07f
+#define	UNSAFE_64BIT	0x080
 
 /* 
  * Valid values for sy_cancel
@@ -67,8 +83,6 @@ extern struct sysent {		/* system call table */
 #define _SYSCALL_RET_SIZE_T		5	
 #define _SYSCALL_RET_SSIZE_T	6	
 
-extern int nsysent;
-
 #endif /* __APPLE_API_PRIVATE */
 #endif /* KERNEL_PRIVATE */
 
diff --git a/bsd/sys/syslimits.h b/bsd/sys/syslimits.h
index 703c82433..a020b3919 100644
--- a/bsd/sys/syslimits.h
+++ b/bsd/sys/syslimits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*	$NetBSD: syslimits.h,v 1.15 1997/06/25 00:48:09 lukem Exp $	*/
 
@@ -59,6 +65,8 @@
 #ifndef _SYS_SYSLIMITS_H_
 #define _SYS_SYSLIMITS_H_
 
+#include <sys/cdefs.h>
+
 #if !defined(_ANSI_SOURCE)
 /*
  * Note: CHILD_MAX *must* be less than hard_maxproc, which is set at
@@ -66,14 +74,20 @@
  */
 #define	ARG_MAX		   (256 * 1024)	/* max bytes for an exec function */
 #define	CHILD_MAX		   266	/* max simultaneous processes */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	GID_MAX		   2147483647U	/* max value for a gid_t (2^31-2) */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	LINK_MAX		32767	/* max file link count */
-#define	MAX_CANON		  255	/* max bytes in term canon input line */
-#define	MAX_INPUT		  255	/* max bytes in terminal input */
+#define	MAX_CANON		 1024	/* max bytes in term canon input line */
+#define	MAX_INPUT		 1024	/* max bytes in terminal input */
 #define	NAME_MAX		  255	/* max bytes in a file name */
 #define	NGROUPS_MAX		   16	/* max supplemental group id's */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	UID_MAX		   2147483647U	/* max value for a uid_t (2^31-2) */
-#define	OPEN_MAX		10240	/* max open files per process */
+
+#define	OPEN_MAX		10240	/* max open files per process - todo, make a config option? */
+
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	PATH_MAX		 1024	/* max bytes in pathname */
 #define	PIPE_BUF		  512	/* max bytes for atomic pipe writes */
 
@@ -81,11 +95,23 @@
 #define	BC_DIM_MAX		 2048	/* max array elements in bc(1) */
 #define	BC_SCALE_MAX		   99	/* max scale value in bc(1) */
 #define	BC_STRING_MAX		 1000	/* max const string length in bc(1) */
+#define	CHARCLASS_NAME_MAX	   14	/* max character class name size */
 #define	COLL_WEIGHTS_MAX	    2	/* max weights for order keyword */
 #define	EQUIV_CLASS_MAX		    2
 #define	EXPR_NEST_MAX		   32	/* max expressions nested in expr(1) */
 #define	LINE_MAX		 2048	/* max bytes in an input line */
 #define	RE_DUP_MAX		  255	/* max RE's in interval notation */
+
+#if __DARWIN_UNIX03
+#define	NZERO			   20	/* default priority [XSI] */
+					/* = ((PRIO_MAX - PRIO_MIN) / 2) + 1 */
+					/* range: 0 - 39 [(2 * NZERO) - 1] */
+					/* 0 is not actually used */
+#else /* !__DARWIN_UNIX03 */
+#define	NZERO			    0	/* default priority */
+					/* range: -20 - 20 */
+					/* (PRIO_MIN - PRIO_MAX) */
+#endif /* __DARWIN_UNIX03 */
 #endif /* !_ANSI_SOURCE */
 
 #endif /* !_SYS_SYSLIMITS_H_ */
diff --git a/bsd/sys/syslog.h b/bsd/sys/syslog.h
index 4b8618692..a681d394f 100644
--- a/bsd/sys/syslog.h
+++ b/bsd/sys/syslog.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -205,18 +211,18 @@ CODE facilitynames[] = {
 #define	LOG_PERROR	0x20	/* log to stderr as well */
 
 #ifndef KERNEL
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #include <sys/_types.h>		/* for __darwin_va_list */
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 __BEGIN_DECLS
 void	closelog(void);
 void	openlog(const char *, int, int);
 int	setlogmask(int);
 void	syslog(int, const char *, ...) __DARWIN_LDBL_COMPAT(syslog);
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 void	vsyslog(int, const char *, __darwin_va_list) __DARWIN_LDBL_COMPAT(vsyslog);
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 __END_DECLS
 
 #else /* !KERNEL */
@@ -302,10 +308,12 @@ struct reg_desc {
 
 #endif /* __APPLE_API_OBSOLETE */
 
+#include <stdarg.h>
 __BEGIN_DECLS
 void	logpri(int);
 void	log(int, const char *, ...);
-void	addlog(const char *, ...);
+int	vaddlog(const char *, va_list);
+void	logtime(time_t);
 __END_DECLS
 
 #endif /* !KERNEL */
diff --git a/bsd/sys/sysproto.h b/bsd/sys/sysproto.h
index c42fb5d72..a918a8a16 100644
--- a/bsd/sys/sysproto.h
+++ b/bsd/sys/sysproto.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  * 
- * @APPLE_LICENSE_HEADER_START@ 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and 
- * are subject to the Apple Public Source License Version 1.1 (the 
- * "License").  You may not use this file except in compliance with the 
- * License.  Please obtain a copy of the License at 
- * http://www.apple.com/publicsource and read it before using this file. 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are 
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the 
- * License for the specific language governing rights and limitations 
- * under the License. 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * @APPLE_LICENSE_HEADER_END@ 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  * 
  * 
  * System call switch table.
@@ -36,7 +42,7 @@
 #include <sys/sem_internal.h>
 #include <sys/semaphore.h>
 #include <sys/wait.h>
-#include <mach/shared_memory_server.h>
+#include <mach/shared_region.h>
 
 #ifdef KERNEL
 #ifdef __APPLE_API_PRIVATE
@@ -64,7 +70,8 @@ void munge_wwwwwwww(const void *, void *);
 void munge_wl(const void *, void *);  
 void munge_wlw(const void *, void *);  
 void munge_wwwl(const void *, void *);  
-void munge_wwwlww(const void *, void *); 
+void munge_wwwlww(const void *, void *);  
+void munge_wwlwww(const void *, void *);  
 void munge_wwwwl(const void *, void *);  
 void munge_wwwwwl(const void *, void *);  
 void munge_wsw(const void *, void *);  
@@ -185,6 +192,7 @@ struct ptrace_args {
 	char addr_l_[PADL_(user_addr_t)]; user_addr_t addr; char addr_r_[PADR_(user_addr_t)];
 	char data_l_[PADL_(int)]; int data; char data_r_[PADR_(int)];
 };
+#if SOCKETS
 struct recvmsg_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char msg_l_[PADL_(user_addr_t)]; user_addr_t msg; char msg_r_[PADR_(user_addr_t)];
@@ -218,6 +226,8 @@ struct getsockname_args {
 	char asa_l_[PADL_(user_addr_t)]; user_addr_t asa; char asa_r_[PADR_(user_addr_t)];
 	char alen_l_[PADL_(user_addr_t)]; user_addr_t alen; char alen_r_[PADR_(user_addr_t)];
 };
+#else
+#endif /* SOCKETS */
 struct access_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
@@ -236,6 +246,7 @@ struct sync_args {
 struct kill_args {
 	char pid_l_[PADL_(int)]; int pid; char pid_r_[PADR_(int)];
 	char signum_l_[PADL_(int)]; int signum; char signum_r_[PADR_(int)];
+	char posix_l_[PADL_(int)]; int posix; char posix_r_[PADR_(int)];
 };
 struct getppid_args {
 	register_t dummy;
@@ -255,12 +266,6 @@ struct profil_args {
 	char pcoffset_l_[PADL_(user_ulong_t)]; user_ulong_t pcoffset; char pcoffset_r_[PADR_(user_ulong_t)];
 	char pcscale_l_[PADL_(u_int)]; u_int pcscale; char pcscale_r_[PADR_(u_int)];
 };
-struct ktrace_args {
-	char fname_l_[PADL_(user_addr_t)]; user_addr_t fname; char fname_r_[PADR_(user_addr_t)];
-	char ops_l_[PADL_(int)]; int ops; char ops_r_[PADR_(int)];
-	char facs_l_[PADL_(int)]; int facs; char facs_r_[PADR_(int)];
-	char pid_l_[PADL_(int)]; int pid; char pid_r_[PADR_(int)];
-};
 struct sigaction_args {
 	char signum_l_[PADL_(int)]; int signum; char signum_r_[PADR_(int)];
 	char nsa_l_[PADL_(user_addr_t)]; user_addr_t nsa; char nsa_r_[PADR_(user_addr_t)];
@@ -410,9 +415,10 @@ struct fsync_args {
 };
 struct setpriority_args {
 	char which_l_[PADL_(int)]; int which; char which_r_[PADR_(int)];
-	char who_l_[PADL_(int)]; int who; char who_r_[PADR_(int)];
+	char who_l_[PADL_(id_t)]; id_t who; char who_r_[PADR_(id_t)];
 	char prio_l_[PADL_(int)]; int prio; char prio_r_[PADR_(int)];
 };
+#if SOCKETS
 struct socket_args {
 	char domain_l_[PADL_(int)]; int domain; char domain_r_[PADR_(int)];
 	char type_l_[PADL_(int)]; int type; char type_r_[PADR_(int)];
@@ -423,10 +429,13 @@ struct connect_args {
 	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
 	char namelen_l_[PADL_(socklen_t)]; socklen_t namelen; char namelen_r_[PADR_(socklen_t)];
 };
+#else
+#endif /* SOCKETS */
 struct getpriority_args {
 	char which_l_[PADL_(int)]; int which; char which_r_[PADR_(int)];
-	char who_l_[PADL_(int)]; int who; char who_r_[PADR_(int)];
+	char who_l_[PADL_(id_t)]; id_t who; char who_r_[PADR_(id_t)];
 };
+#if SOCKETS
 struct bind_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
@@ -443,9 +452,14 @@ struct listen_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char backlog_l_[PADL_(int)]; int backlog; char backlog_r_[PADR_(int)];
 };
+#else
+#endif /* SOCKETS */
 struct sigsuspend_args {
 	char mask_l_[PADL_(sigset_t)]; sigset_t mask; char mask_r_[PADR_(sigset_t)];
 };
+#if SOCKETS
+#else
+#endif /* SOCKETS */
 struct gettimeofday_args {
 	char tp_l_[PADL_(user_addr_t)]; user_addr_t tp; char tp_r_[PADR_(user_addr_t)];
 	char tzp_l_[PADL_(user_addr_t)]; user_addr_t tzp; char tzp_r_[PADR_(user_addr_t)];
@@ -454,6 +468,7 @@ struct getrusage_args {
 	char who_l_[PADL_(int)]; int who; char who_r_[PADR_(int)];
 	char rusage_l_[PADL_(user_addr_t)]; user_addr_t rusage; char rusage_r_[PADR_(user_addr_t)];
 };
+#if SOCKETS
 struct getsockopt_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char level_l_[PADL_(int)]; int level; char level_r_[PADR_(int)];
@@ -461,6 +476,8 @@ struct getsockopt_args {
 	char val_l_[PADL_(user_addr_t)]; user_addr_t val; char val_r_[PADR_(user_addr_t)];
 	char avalsize_l_[PADL_(user_addr_t)]; user_addr_t avalsize; char avalsize_r_[PADR_(user_addr_t)];
 };
+#else
+#endif /* SOCKETS */
 struct readv_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char iovp_l_[PADL_(user_addr_t)]; user_addr_t iovp; char iovp_r_[PADR_(user_addr_t)];
@@ -484,6 +501,14 @@ struct fchmod_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)];
 };
+struct setreuid_args {
+	char ruid_l_[PADL_(uid_t)]; uid_t ruid; char ruid_r_[PADR_(uid_t)];
+	char euid_l_[PADL_(uid_t)]; uid_t euid; char euid_r_[PADR_(uid_t)];
+};
+struct setregid_args {
+	char rgid_l_[PADL_(gid_t)]; gid_t rgid; char rgid_r_[PADR_(gid_t)];
+	char egid_l_[PADL_(gid_t)]; gid_t egid; char egid_r_[PADR_(gid_t)];
+};
 struct rename_args {
 	char from_l_[PADL_(user_addr_t)]; user_addr_t from; char from_r_[PADR_(user_addr_t)];
 	char to_l_[PADL_(user_addr_t)]; user_addr_t to; char to_r_[PADR_(user_addr_t)];
@@ -496,6 +521,7 @@ struct mkfifo_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)];
 };
+#if SOCKETS
 struct sendto_args {
 	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
 	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
@@ -514,6 +540,8 @@ struct socketpair_args {
 	char protocol_l_[PADL_(int)]; int protocol; char protocol_r_[PADR_(int)];
 	char rsv_l_[PADL_(user_addr_t)]; user_addr_t rsv; char rsv_r_[PADR_(user_addr_t)];
 };
+#else
+#endif /* SOCKETS */
 struct mkdir_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)];
@@ -533,6 +561,10 @@ struct adjtime_args {
 	char delta_l_[PADL_(user_addr_t)]; user_addr_t delta; char delta_r_[PADR_(user_addr_t)];
 	char olddelta_l_[PADL_(user_addr_t)]; user_addr_t olddelta; char olddelta_r_[PADR_(user_addr_t)];
 };
+struct gethostuuid_args {
+	char uuid_buf_l_[PADL_(user_addr_t)]; user_addr_t uuid_buf; char uuid_buf_r_[PADR_(user_addr_t)];
+	char timeoutp_l_[PADL_(user_addr_t)]; user_addr_t timeoutp; char timeoutp_r_[PADR_(user_addr_t)];
+};
 struct setsid_args {
 	register_t dummy;
 };
@@ -573,7 +605,7 @@ struct unmount_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
 };
-#if NFSCLIENT
+#if NFSSERVER
 struct getfh_args {
 	char fname_l_[PADL_(user_addr_t)]; user_addr_t fname; char fname_r_[PADR_(user_addr_t)];
 	char fhp_l_[PADL_(user_addr_t)]; user_addr_t fhp; char fhp_r_[PADR_(user_addr_t)];
@@ -592,6 +624,12 @@ struct mount_args {
 	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
 	char data_l_[PADL_(user_addr_t)]; user_addr_t data; char data_r_[PADR_(user_addr_t)];
 };
+struct csops_args {
+	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
+	char ops_l_[PADL_(uint32_t)]; uint32_t ops; char ops_r_[PADR_(uint32_t)];
+	char useraddr_l_[PADL_(user_addr_t)]; user_addr_t useraddr; char useraddr_r_[PADR_(user_addr_t)];
+	char usersize_l_[PADL_(user_size_t)]; user_size_t usersize; char usersize_r_[PADR_(user_size_t)];
+};
 struct waitid_args {
 	char idtype_l_[PADL_(idtype_t)]; idtype_t idtype; char idtype_r_[PADR_(idtype_t)];
 	char id_l_[PADL_(id_t)]; id_t id; char id_r_[PADR_(id_t)];
@@ -653,14 +691,6 @@ struct fpathconf_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char name_l_[PADL_(int)]; int name; char name_r_[PADR_(int)];
 };
-#if COMPAT_GETFSSTAT
-struct getfsstat_args {
-	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
-	char bufsize_l_[PADL_(user_long_t)]; user_long_t bufsize; char bufsize_r_[PADR_(user_long_t)];
-	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
-};
-#else
-#endif
 struct getrlimit_args {
 	char which_l_[PADL_(u_int)]; u_int which; char which_r_[PADR_(u_int)];
 	char rlp_l_[PADL_(user_addr_t)]; user_addr_t rlp; char rlp_r_[PADR_(user_addr_t)];
@@ -715,6 +745,7 @@ struct munlock_args {
 struct undelete_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 };
+#if NETAT
 struct ATsocket_args {
 	char proto_l_[PADL_(int)]; int proto; char proto_r_[PADR_(int)];
 };
@@ -751,6 +782,8 @@ struct ATPgetrsp_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
 	char bdsp_l_[PADL_(unsigned char *)]; unsigned char * bdsp; char bdsp_r_[PADR_(unsigned char *)];
 };
+#else
+#endif /* NETAT */
 struct kqueue_from_portset_np_args {
 	char portset_l_[PADL_(int)]; int portset; char portset_r_[PADR_(int)];
 };
@@ -786,17 +819,6 @@ struct exchangedata_args {
 	char path2_l_[PADL_(user_addr_t)]; user_addr_t path2; char path2_r_[PADR_(user_addr_t)];
 	char options_l_[PADL_(user_ulong_t)]; user_ulong_t options; char options_r_[PADR_(user_ulong_t)];
 };
-#ifdef __APPLE_API_OBSOLETE
-struct checkuseraccess_args {
-	char path_l_[PADL_(const char *)]; const char * path; char path_r_[PADR_(const char *)];
-	char userid_l_[PADL_(uid_t)]; uid_t userid; char userid_r_[PADR_(uid_t)];
-	char groups_l_[PADL_(gid_t *)]; gid_t * groups; char groups_r_[PADR_(gid_t *)];
-	char ngroups_l_[PADL_(int)]; int ngroups; char ngroups_r_[PADR_(int)];
-	char accessrequired_l_[PADL_(int)]; int accessrequired; char accessrequired_r_[PADR_(int)];
-	char options_l_[PADL_(u_long)]; u_long options; char options_r_[PADR_(u_long)];
-};
-#else
-#endif /* __APPLE_API_OBSOLETE */
 struct searchfs_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 	char searchblock_l_[PADL_(user_addr_t)]; user_addr_t searchblock; char searchblock_r_[PADR_(user_addr_t)];
@@ -820,15 +842,15 @@ struct poll_args {
 	char timeout_l_[PADL_(int)]; int timeout; char timeout_r_[PADR_(int)];
 };
 struct watchevent_args {
-	char u_req_l_[PADL_(struct eventreq *)]; struct eventreq * u_req; char u_req_r_[PADR_(struct eventreq *)];
+	char u_req_l_[PADL_(user_addr_t)]; user_addr_t u_req; char u_req_r_[PADR_(user_addr_t)];
 	char u_eventmask_l_[PADL_(int)]; int u_eventmask; char u_eventmask_r_[PADR_(int)];
 };
 struct waitevent_args {
-	char u_req_l_[PADL_(struct eventreq *)]; struct eventreq * u_req; char u_req_r_[PADR_(struct eventreq *)];
-	char tv_l_[PADL_(struct timeval *)]; struct timeval * tv; char tv_r_[PADR_(struct timeval *)];
+	char u_req_l_[PADL_(user_addr_t)]; user_addr_t u_req; char u_req_r_[PADR_(user_addr_t)];
+	char tv_l_[PADL_(user_addr_t)]; user_addr_t tv; char tv_r_[PADR_(user_addr_t)];
 };
 struct modwatch_args {
-	char u_req_l_[PADL_(struct eventreq *)]; struct eventreq * u_req; char u_req_r_[PADR_(struct eventreq *)];
+	char u_req_l_[PADL_(user_addr_t)]; user_addr_t u_req; char u_req_r_[PADR_(user_addr_t)];
 	char u_eventmask_l_[PADL_(int)]; int u_eventmask; char u_eventmask_r_[PADR_(int)];
 };
 struct getxattr_args {
@@ -896,11 +918,21 @@ struct initgroups_args {
 	char gidset_l_[PADL_(user_addr_t)]; user_addr_t gidset; char gidset_r_[PADR_(user_addr_t)];
 	char gmuid_l_[PADL_(int)]; int gmuid; char gmuid_r_[PADR_(int)];
 };
+struct posix_spawn_args {
+	char pid_l_[PADL_(user_addr_t)]; user_addr_t pid; char pid_r_[PADR_(user_addr_t)];
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char adesc_l_[PADL_(user_addr_t)]; user_addr_t adesc; char adesc_r_[PADR_(user_addr_t)];
+	char argv_l_[PADL_(user_addr_t)]; user_addr_t argv; char argv_r_[PADR_(user_addr_t)];
+	char envp_l_[PADL_(user_addr_t)]; user_addr_t envp; char envp_r_[PADR_(user_addr_t)];
+};
 #if NFSCLIENT
 struct nfsclnt_args {
 	char flag_l_[PADL_(int)]; int flag; char flag_r_[PADR_(int)];
 	char argp_l_[PADL_(user_addr_t)]; user_addr_t argp; char argp_r_[PADR_(user_addr_t)];
 };
+#else
+#endif
+#if NFSSERVER
 struct fhopen_args {
 	char u_fhp_l_[PADL_(user_addr_t)]; user_addr_t u_fhp; char u_fhp_r_[PADR_(user_addr_t)];
 	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
@@ -912,6 +944,7 @@ struct minherit_args {
 	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
 	char inherit_l_[PADL_(int)]; int inherit; char inherit_r_[PADR_(int)];
 };
+#if SYSV_SEM
 struct semsys_args {
 	char which_l_[PADL_(u_int)]; u_int which; char which_r_[PADR_(u_int)];
 	char a2_l_[PADL_(int)]; int a2; char a2_r_[PADR_(int)];
@@ -919,6 +952,9 @@ struct semsys_args {
 	char a4_l_[PADL_(int)]; int a4; char a4_r_[PADR_(int)];
 	char a5_l_[PADL_(int)]; int a5; char a5_r_[PADR_(int)];
 };
+#else
+#endif
+#if SYSV_MSG
 struct msgsys_args {
 	char which_l_[PADL_(u_int)]; u_int which; char which_r_[PADR_(u_int)];
 	char a2_l_[PADL_(int)]; int a2; char a2_r_[PADR_(int)];
@@ -926,12 +962,18 @@ struct msgsys_args {
 	char a4_l_[PADL_(int)]; int a4; char a4_r_[PADR_(int)];
 	char a5_l_[PADL_(int)]; int a5; char a5_r_[PADR_(int)];
 };
+#else
+#endif
+#if SYSV_SHM
 struct shmsys_args {
 	char which_l_[PADL_(u_int)]; u_int which; char which_r_[PADR_(u_int)];
 	char a2_l_[PADL_(int)]; int a2; char a2_r_[PADR_(int)];
 	char a3_l_[PADL_(int)]; int a3; char a3_r_[PADR_(int)];
 	char a4_l_[PADL_(int)]; int a4; char a4_r_[PADR_(int)];
 };
+#else
+#endif
+#if SYSV_SEM
 struct semctl_args {
 	char semid_l_[PADL_(int)]; int semid; char semid_r_[PADR_(int)];
 	char semnum_l_[PADL_(int)]; int semnum; char semnum_r_[PADR_(int)];
@@ -948,6 +990,9 @@ struct semop_args {
 	char sops_l_[PADL_(user_addr_t)]; user_addr_t sops; char sops_r_[PADR_(user_addr_t)];
 	char nsops_l_[PADL_(int)]; int nsops; char nsops_r_[PADR_(int)];
 };
+#else
+#endif
+#if SYSV_MSG
 struct msgctl_args {
 	char msqid_l_[PADL_(int)]; int msqid; char msqid_r_[PADR_(int)];
 	char cmd_l_[PADL_(int)]; int cmd; char cmd_r_[PADR_(int)];
@@ -970,6 +1015,9 @@ struct msgrcv_args {
 	char msgtyp_l_[PADL_(user_long_t)]; user_long_t msgtyp; char msgtyp_r_[PADR_(user_long_t)];
 	char msgflg_l_[PADL_(int)]; int msgflg; char msgflg_r_[PADR_(int)];
 };
+#else
+#endif
+#if SYSV_SHM
 struct shmat_args {
 	char shmid_l_[PADL_(int)]; int shmid; char shmid_r_[PADR_(int)];
 	char shmaddr_l_[PADL_(user_addr_t)]; user_addr_t shmaddr; char shmaddr_r_[PADR_(user_addr_t)];
@@ -988,6 +1036,8 @@ struct shmget_args {
 	char size_l_[PADL_(user_size_t)]; user_size_t size; char size_r_[PADR_(user_size_t)];
 	char shmflg_l_[PADL_(int)]; int shmflg; char shmflg_r_[PADR_(int)];
 };
+#else
+#endif
 struct shm_open_args {
 	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
 	char oflag_l_[PADL_(int)]; int oflag; char oflag_r_[PADR_(int)];
@@ -1121,32 +1171,42 @@ struct identitysvc_args {
 	char opcode_l_[PADL_(int)]; int opcode; char opcode_r_[PADR_(int)];
 	char message_l_[PADL_(user_addr_t)]; user_addr_t message; char message_r_[PADR_(user_addr_t)];
 };
-struct load_shared_file_args {
-	char filename_l_[PADL_(char *)]; char * filename; char filename_r_[PADR_(char *)];
-	char mfa_l_[PADL_(caddr_t)]; caddr_t mfa; char mfa_r_[PADR_(caddr_t)];
-	char mfs_l_[PADL_(u_long)]; u_long mfs; char mfs_r_[PADR_(u_long)];
-	char ba_l_[PADL_(caddr_t *)]; caddr_t * ba; char ba_r_[PADR_(caddr_t *)];
-	char map_cnt_l_[PADL_(int)]; int map_cnt; char map_cnt_r_[PADR_(int)];
-	char mappings_l_[PADL_(sf_mapping_t *)]; sf_mapping_t * mappings; char mappings_r_[PADR_(sf_mapping_t *)];
-	char flags_l_[PADL_(int *)]; int * flags; char flags_r_[PADR_(int *)];
+struct shared_region_check_np_args {
+	char start_address_l_[PADL_(user_addr_t)]; user_addr_t start_address; char start_address_r_[PADR_(user_addr_t)];
 };
-struct reset_shared_file_args {
-	char ba_l_[PADL_(caddr_t *)]; caddr_t * ba; char ba_r_[PADR_(caddr_t *)];
-	char map_cnt_l_[PADL_(int)]; int map_cnt; char map_cnt_r_[PADR_(int)];
-	char mappings_l_[PADL_(sf_mapping_t *)]; sf_mapping_t * mappings; char mappings_r_[PADR_(sf_mapping_t *)];
-};
-struct new_system_shared_regions_args {
-	register_t dummy;
-};
-struct shared_region_map_file_np_args {
+struct shared_region_map_np_args {
 	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
-	char mappingCount_l_[PADL_(uint32_t)]; uint32_t mappingCount; char mappingCount_r_[PADR_(uint32_t)];
+	char count_l_[PADL_(uint32_t)]; uint32_t count; char count_r_[PADR_(uint32_t)];
 	char mappings_l_[PADL_(user_addr_t)]; user_addr_t mappings; char mappings_r_[PADR_(user_addr_t)];
-	char slide_p_l_[PADL_(user_addr_t)]; user_addr_t slide_p; char slide_p_r_[PADR_(user_addr_t)];
 };
-struct shared_region_make_private_np_args {
-	char rangeCount_l_[PADL_(uint32_t)]; uint32_t rangeCount; char rangeCount_r_[PADR_(uint32_t)];
-	char ranges_l_[PADL_(user_addr_t)]; user_addr_t ranges; char ranges_r_[PADR_(user_addr_t)];
+struct __pthread_mutex_destroy_args {
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+};
+struct __pthread_mutex_init_args {
+	char mutex_l_[PADL_(user_addr_t)]; user_addr_t mutex; char mutex_r_[PADR_(user_addr_t)];
+	char attr_l_[PADL_(user_addr_t)]; user_addr_t attr; char attr_r_[PADR_(user_addr_t)];
+};
+struct __pthread_mutex_lock_args {
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+};
+struct __pthread_mutex_trylock_args {
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+};
+struct __pthread_mutex_unlock_args {
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+};
+struct __pthread_cond_init_args {
+	char cond_l_[PADL_(user_addr_t)]; user_addr_t cond; char cond_r_[PADR_(user_addr_t)];
+	char attr_l_[PADL_(user_addr_t)]; user_addr_t attr; char attr_r_[PADR_(user_addr_t)];
+};
+struct __pthread_cond_destroy_args {
+	char condid_l_[PADL_(int)]; int condid; char condid_r_[PADR_(int)];
+};
+struct __pthread_cond_broadcast_args {
+	char condid_l_[PADL_(int)]; int condid; char condid_r_[PADR_(int)];
+};
+struct __pthread_cond_signal_args {
+	char condid_l_[PADL_(int)]; int condid; char condid_r_[PADR_(int)];
 };
 struct getsid_args {
 	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
@@ -1155,6 +1215,11 @@ struct settid_with_pid_args {
 	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
 	char assume_l_[PADL_(int)]; int assume; char assume_r_[PADR_(int)];
 };
+struct __pthread_cond_timedwait_args {
+	char condid_l_[PADL_(int)]; int condid; char condid_r_[PADR_(int)];
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+	char abstime_l_[PADL_(user_addr_t)]; user_addr_t abstime; char abstime_r_[PADR_(user_addr_t)];
+};
 struct aio_fsync_args {
 	char op_l_[PADL_(int)]; int op; char op_r_[PADR_(int)];
 	char aiocbp_l_[PADL_(user_addr_t)]; user_addr_t aiocbp; char aiocbp_r_[PADR_(user_addr_t)];
@@ -1186,6 +1251,14 @@ struct lio_listio_args {
 	char nent_l_[PADL_(int)]; int nent; char nent_r_[PADR_(int)];
 	char sigp_l_[PADL_(user_addr_t)]; user_addr_t sigp; char sigp_r_[PADR_(user_addr_t)];
 };
+struct __pthread_cond_wait_args {
+	char condid_l_[PADL_(int)]; int condid; char condid_r_[PADR_(int)];
+	char mutexid_l_[PADL_(int)]; int mutexid; char mutexid_r_[PADR_(int)];
+};
+struct iopolicysys_args {
+	char cmd_l_[PADL_(int)]; int cmd; char cmd_r_[PADR_(int)];
+	char arg_l_[PADL_(user_addr_t)]; user_addr_t arg; char arg_r_[PADR_(user_addr_t)];
+};
 struct mlockall_args {
 	char how_l_[PADL_(int)]; int how; char how_r_[PADR_(int)];
 };
@@ -1199,12 +1272,12 @@ struct __pthread_kill_args {
 	char thread_port_l_[PADL_(int)]; int thread_port; char thread_port_r_[PADR_(int)];
 	char sig_l_[PADL_(int)]; int sig; char sig_r_[PADR_(int)];
 };
-struct pthread_sigmask_args {
+struct __pthread_sigmask_args {
 	char how_l_[PADL_(int)]; int how; char how_r_[PADR_(int)];
 	char set_l_[PADL_(user_addr_t)]; user_addr_t set; char set_r_[PADR_(user_addr_t)];
 	char oset_l_[PADL_(user_addr_t)]; user_addr_t oset; char oset_r_[PADR_(user_addr_t)];
 };
-struct sigwait_args {
+struct __sigwait_args {
 	char set_l_[PADL_(user_addr_t)]; user_addr_t set; char set_r_[PADR_(user_addr_t)];
 	char sig_l_[PADL_(user_addr_t)]; user_addr_t sig; char sig_r_[PADR_(user_addr_t)];
 };
@@ -1225,10 +1298,6 @@ struct __semwait_signal_args {
 	char tv_sec_l_[PADL_(time_t)]; time_t tv_sec; char tv_sec_r_[PADR_(time_t)];
 	char tv_nsec_l_[PADL_(int32_t)]; int32_t tv_nsec; char tv_nsec_r_[PADR_(int32_t)];
 };
-struct utrace_args {
-	char addr_l_[PADL_(user_addr_t)]; user_addr_t addr; char addr_r_[PADR_(user_addr_t)];
-	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
-};
 struct proc_info_args {
 	char callnum_l_[PADL_(int32_t)]; int32_t callnum; char callnum_r_[PADR_(int32_t)];
 	char pid_l_[PADL_(int32_t)]; int32_t pid; char pid_r_[PADR_(int32_t)];
@@ -1237,6 +1306,73 @@ struct proc_info_args {
 	char buffer_l_[PADL_(user_addr_t)]; user_addr_t buffer; char buffer_r_[PADR_(user_addr_t)];
 	char buffersize_l_[PADL_(int32_t)]; int32_t buffersize; char buffersize_r_[PADR_(int32_t)];
 };
+#if SENDFILE
+struct sendfile_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char offset_l_[PADL_(off_t)]; off_t offset; char offset_r_[PADR_(off_t)];
+	char nbytes_l_[PADL_(user_addr_t)]; user_addr_t nbytes; char nbytes_r_[PADR_(user_addr_t)];
+	char hdtr_l_[PADL_(user_addr_t)]; user_addr_t hdtr; char hdtr_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
+#else /* !SENDFILE */
+#endif /* SENDFILE */
+struct stat64_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+};
+struct fstat64_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+};
+struct lstat64_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+};
+struct stat64_extended_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+	char xsecurity_l_[PADL_(user_addr_t)]; user_addr_t xsecurity; char xsecurity_r_[PADR_(user_addr_t)];
+	char xsecurity_size_l_[PADL_(user_addr_t)]; user_addr_t xsecurity_size; char xsecurity_size_r_[PADR_(user_addr_t)];
+};
+struct lstat64_extended_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+	char xsecurity_l_[PADL_(user_addr_t)]; user_addr_t xsecurity; char xsecurity_r_[PADR_(user_addr_t)];
+	char xsecurity_size_l_[PADL_(user_addr_t)]; user_addr_t xsecurity_size; char xsecurity_size_r_[PADR_(user_addr_t)];
+};
+struct fstat64_extended_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char ub_l_[PADL_(user_addr_t)]; user_addr_t ub; char ub_r_[PADR_(user_addr_t)];
+	char xsecurity_l_[PADL_(user_addr_t)]; user_addr_t xsecurity; char xsecurity_r_[PADR_(user_addr_t)];
+	char xsecurity_size_l_[PADL_(user_addr_t)]; user_addr_t xsecurity_size; char xsecurity_size_r_[PADR_(user_addr_t)];
+};
+struct getdirentries64_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char bufsize_l_[PADL_(user_size_t)]; user_size_t bufsize; char bufsize_r_[PADR_(user_size_t)];
+	char position_l_[PADL_(user_addr_t)]; user_addr_t position; char position_r_[PADR_(user_addr_t)];
+};
+struct statfs64_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+};
+struct fstatfs64_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+};
+struct getfsstat64_args {
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char bufsize_l_[PADL_(int)]; int bufsize; char bufsize_r_[PADR_(int)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
+struct __pthread_chdir_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+};
+struct __pthread_fchdir_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+};
+#if AUDIT
 struct audit_args {
 	char record_l_[PADL_(user_addr_t)]; user_addr_t record; char record_r_[PADR_(user_addr_t)];
 	char length_l_[PADL_(int)]; int length; char length_r_[PADR_(int)];
@@ -1269,6 +1405,21 @@ struct setaudit_addr_args {
 struct auditctl_args {
 	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
 };
+#else
+#endif
+struct bsdthread_create_args {
+	char func_l_[PADL_(user_addr_t)]; user_addr_t func; char func_r_[PADR_(user_addr_t)];
+	char func_arg_l_[PADL_(user_addr_t)]; user_addr_t func_arg; char func_arg_r_[PADR_(user_addr_t)];
+	char stack_l_[PADL_(user_addr_t)]; user_addr_t stack; char stack_r_[PADR_(user_addr_t)];
+	char pthread_l_[PADL_(user_addr_t)]; user_addr_t pthread; char pthread_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(uint32_t)]; uint32_t flags; char flags_r_[PADR_(uint32_t)];
+};
+struct bsdthread_terminate_args {
+	char stackaddr_l_[PADL_(user_addr_t)]; user_addr_t stackaddr; char stackaddr_r_[PADR_(user_addr_t)];
+	char freesize_l_[PADL_(user_size_t)]; user_size_t freesize; char freesize_r_[PADR_(user_size_t)];
+	char port_l_[PADL_(uint32_t)]; uint32_t port; char port_r_[PADR_(uint32_t)];
+	char sem_l_[PADL_(uint32_t)]; uint32_t sem; char sem_r_[PADR_(uint32_t)];
+};
 struct kqueue_args {
 	register_t dummy;
 };
@@ -1291,6 +1442,260 @@ struct stack_snapshot_args {
 	char tracebuf_size_l_[PADL_(uint32_t)]; uint32_t tracebuf_size; char tracebuf_size_r_[PADR_(uint32_t)];
 	char options_l_[PADL_(uint32_t)]; uint32_t options; char options_r_[PADR_(uint32_t)];
 };
+struct bsdthread_register_args {
+	char threadstart_l_[PADL_(user_addr_t)]; user_addr_t threadstart; char threadstart_r_[PADR_(user_addr_t)];
+	char wqthread_l_[PADL_(user_addr_t)]; user_addr_t wqthread; char wqthread_r_[PADR_(user_addr_t)];
+	char pthsize_l_[PADL_(int)]; int pthsize; char pthsize_r_[PADR_(int)];
+};
+struct workq_open_args {
+	register_t dummy;
+};
+struct workq_ops_args {
+	char options_l_[PADL_(int)]; int options; char options_r_[PADR_(int)];
+	char item_l_[PADL_(user_addr_t)]; user_addr_t item; char item_r_[PADR_(user_addr_t)];
+	char prio_l_[PADL_(int)]; int prio; char prio_r_[PADR_(int)];
+};
+struct __mac_execve_args {
+	char fname_l_[PADL_(user_addr_t)]; user_addr_t fname; char fname_r_[PADR_(user_addr_t)];
+	char argp_l_[PADL_(user_addr_t)]; user_addr_t argp; char argp_r_[PADR_(user_addr_t)];
+	char envp_l_[PADL_(user_addr_t)]; user_addr_t envp; char envp_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_syscall_args {
+	char policy_l_[PADL_(user_addr_t)]; user_addr_t policy; char policy_r_[PADR_(user_addr_t)];
+	char call_l_[PADL_(int)]; int call; char call_r_[PADR_(int)];
+	char arg_l_[PADL_(user_addr_t)]; user_addr_t arg; char arg_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_file_args {
+	char path_p_l_[PADL_(user_addr_t)]; user_addr_t path_p; char path_p_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_set_file_args {
+	char path_p_l_[PADL_(user_addr_t)]; user_addr_t path_p; char path_p_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_link_args {
+	char path_p_l_[PADL_(user_addr_t)]; user_addr_t path_p; char path_p_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_set_link_args {
+	char path_p_l_[PADL_(user_addr_t)]; user_addr_t path_p; char path_p_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_proc_args {
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_set_proc_args {
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_fd_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_set_fd_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_pid_args {
+	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_lcid_args {
+	char lcid_l_[PADL_(pid_t)]; pid_t lcid; char lcid_r_[PADR_(pid_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_lctx_args {
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_set_lctx_args {
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct setlcid_args {
+	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
+	char lcid_l_[PADL_(pid_t)]; pid_t lcid; char lcid_r_[PADR_(pid_t)];
+};
+struct getlcid_args {
+	char pid_l_[PADL_(pid_t)]; pid_t pid; char pid_r_[PADR_(pid_t)];
+};
+struct read_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char cbuf_l_[PADL_(user_addr_t)]; user_addr_t cbuf; char cbuf_r_[PADR_(user_addr_t)];
+	char nbyte_l_[PADL_(user_size_t)]; user_size_t nbyte; char nbyte_r_[PADR_(user_size_t)];
+};
+struct write_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char cbuf_l_[PADL_(user_addr_t)]; user_addr_t cbuf; char cbuf_r_[PADR_(user_addr_t)];
+	char nbyte_l_[PADL_(user_size_t)]; user_size_t nbyte; char nbyte_r_[PADR_(user_size_t)];
+};
+struct open_nocancel_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char mode_l_[PADL_(int)]; int mode; char mode_r_[PADR_(int)];
+};
+struct close_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+};
+struct wait4_nocancel_args {
+	char pid_l_[PADL_(int)]; int pid; char pid_r_[PADR_(int)];
+	char status_l_[PADL_(user_addr_t)]; user_addr_t status; char status_r_[PADR_(user_addr_t)];
+	char options_l_[PADL_(int)]; int options; char options_r_[PADR_(int)];
+	char rusage_l_[PADL_(user_addr_t)]; user_addr_t rusage; char rusage_r_[PADR_(user_addr_t)];
+};
+#if SOCKETS
+struct recvmsg_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char msg_l_[PADL_(user_addr_t)]; user_addr_t msg; char msg_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
+struct sendmsg_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char msg_l_[PADL_(user_addr_t)]; user_addr_t msg; char msg_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
+struct recvfrom_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char from_l_[PADL_(user_addr_t)]; user_addr_t from; char from_r_[PADR_(user_addr_t)];
+	char fromlenaddr_l_[PADL_(user_addr_t)]; user_addr_t fromlenaddr; char fromlenaddr_r_[PADR_(user_addr_t)];
+};
+struct accept_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
+	char anamelen_l_[PADL_(user_addr_t)]; user_addr_t anamelen; char anamelen_r_[PADR_(user_addr_t)];
+};
+#else
+#endif /* SOCKETS */
+struct msync_nocancel_args {
+	char addr_l_[PADL_(user_addr_t)]; user_addr_t addr; char addr_r_[PADR_(user_addr_t)];
+	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
+struct fcntl_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char cmd_l_[PADL_(int)]; int cmd; char cmd_r_[PADR_(int)];
+	char arg_l_[PADL_(user_long_t)]; user_long_t arg; char arg_r_[PADR_(user_long_t)];
+};
+struct select_nocancel_args {
+	char nd_l_[PADL_(int)]; int nd; char nd_r_[PADR_(int)];
+	char in_l_[PADL_(user_addr_t)]; user_addr_t in; char in_r_[PADR_(user_addr_t)];
+	char ou_l_[PADL_(user_addr_t)]; user_addr_t ou; char ou_r_[PADR_(user_addr_t)];
+	char ex_l_[PADL_(user_addr_t)]; user_addr_t ex; char ex_r_[PADR_(user_addr_t)];
+	char tv_l_[PADL_(user_addr_t)]; user_addr_t tv; char tv_r_[PADR_(user_addr_t)];
+};
+struct fsync_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+};
+#if SOCKETS
+struct connect_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char name_l_[PADL_(user_addr_t)]; user_addr_t name; char name_r_[PADR_(user_addr_t)];
+	char namelen_l_[PADL_(socklen_t)]; socklen_t namelen; char namelen_r_[PADR_(socklen_t)];
+};
+#else
+#endif /* SOCKETS */
+struct sigsuspend_nocancel_args {
+	char mask_l_[PADL_(sigset_t)]; sigset_t mask; char mask_r_[PADR_(sigset_t)];
+};
+struct readv_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char iovp_l_[PADL_(user_addr_t)]; user_addr_t iovp; char iovp_r_[PADR_(user_addr_t)];
+	char iovcnt_l_[PADL_(u_int)]; u_int iovcnt; char iovcnt_r_[PADR_(u_int)];
+};
+struct writev_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char iovp_l_[PADL_(user_addr_t)]; user_addr_t iovp; char iovp_r_[PADR_(user_addr_t)];
+	char iovcnt_l_[PADL_(u_int)]; u_int iovcnt; char iovcnt_r_[PADR_(u_int)];
+};
+#if SOCKETS
+struct sendto_nocancel_args {
+	char s_l_[PADL_(int)]; int s; char s_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char len_l_[PADL_(user_size_t)]; user_size_t len; char len_r_[PADR_(user_size_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char to_l_[PADL_(user_addr_t)]; user_addr_t to; char to_r_[PADR_(user_addr_t)];
+	char tolen_l_[PADL_(socklen_t)]; socklen_t tolen; char tolen_r_[PADR_(socklen_t)];
+};
+#else
+#endif /* SOCKETS */
+struct pread_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char nbyte_l_[PADL_(user_size_t)]; user_size_t nbyte; char nbyte_r_[PADR_(user_size_t)];
+	char offset_l_[PADL_(off_t)]; off_t offset; char offset_r_[PADR_(off_t)];
+};
+struct pwrite_nocancel_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char nbyte_l_[PADL_(user_size_t)]; user_size_t nbyte; char nbyte_r_[PADR_(user_size_t)];
+	char offset_l_[PADL_(off_t)]; off_t offset; char offset_r_[PADR_(off_t)];
+};
+struct waitid_nocancel_args {
+	char idtype_l_[PADL_(idtype_t)]; idtype_t idtype; char idtype_r_[PADR_(idtype_t)];
+	char id_l_[PADL_(id_t)]; id_t id; char id_r_[PADR_(id_t)];
+	char infop_l_[PADL_(user_addr_t)]; user_addr_t infop; char infop_r_[PADR_(user_addr_t)];
+	char options_l_[PADL_(int)]; int options; char options_r_[PADR_(int)];
+};
+struct poll_nocancel_args {
+	char fds_l_[PADL_(user_addr_t)]; user_addr_t fds; char fds_r_[PADR_(user_addr_t)];
+	char nfds_l_[PADL_(u_int)]; u_int nfds; char nfds_r_[PADR_(u_int)];
+	char timeout_l_[PADL_(int)]; int timeout; char timeout_r_[PADR_(int)];
+};
+#if SYSV_MSG
+struct msgsnd_nocancel_args {
+	char msqid_l_[PADL_(int)]; int msqid; char msqid_r_[PADR_(int)];
+	char msgp_l_[PADL_(user_addr_t)]; user_addr_t msgp; char msgp_r_[PADR_(user_addr_t)];
+	char msgsz_l_[PADL_(user_size_t)]; user_size_t msgsz; char msgsz_r_[PADR_(user_size_t)];
+	char msgflg_l_[PADL_(int)]; int msgflg; char msgflg_r_[PADR_(int)];
+};
+struct msgrcv_nocancel_args {
+	char msqid_l_[PADL_(int)]; int msqid; char msqid_r_[PADR_(int)];
+	char msgp_l_[PADL_(user_addr_t)]; user_addr_t msgp; char msgp_r_[PADR_(user_addr_t)];
+	char msgsz_l_[PADL_(user_size_t)]; user_size_t msgsz; char msgsz_r_[PADR_(user_size_t)];
+	char msgtyp_l_[PADL_(user_long_t)]; user_long_t msgtyp; char msgtyp_r_[PADR_(user_long_t)];
+	char msgflg_l_[PADL_(int)]; int msgflg; char msgflg_r_[PADR_(int)];
+};
+#else
+#endif
+struct sem_wait_nocancel_args {
+	char sem_l_[PADL_(user_addr_t)]; user_addr_t sem; char sem_r_[PADR_(user_addr_t)];
+};
+struct aio_suspend_nocancel_args {
+	char aiocblist_l_[PADL_(user_addr_t)]; user_addr_t aiocblist; char aiocblist_r_[PADR_(user_addr_t)];
+	char nent_l_[PADL_(int)]; int nent; char nent_r_[PADR_(int)];
+	char timeoutp_l_[PADL_(user_addr_t)]; user_addr_t timeoutp; char timeoutp_r_[PADR_(user_addr_t)];
+};
+struct __sigwait_nocancel_args {
+	char set_l_[PADL_(user_addr_t)]; user_addr_t set; char set_r_[PADR_(user_addr_t)];
+	char sig_l_[PADL_(user_addr_t)]; user_addr_t sig; char sig_r_[PADR_(user_addr_t)];
+};
+struct __semwait_signal_nocancel_args {
+	char cond_sem_l_[PADL_(int)]; int cond_sem; char cond_sem_r_[PADR_(int)];
+	char mutex_sem_l_[PADL_(int)]; int mutex_sem; char mutex_sem_r_[PADR_(int)];
+	char timeout_l_[PADL_(int)]; int timeout; char timeout_r_[PADR_(int)];
+	char relative_l_[PADL_(int)]; int relative; char relative_r_[PADR_(int)];
+	char tv_sec_l_[PADL_(time_t)]; time_t tv_sec; char tv_sec_r_[PADR_(time_t)];
+	char tv_nsec_l_[PADL_(int32_t)]; int32_t tv_nsec; char tv_nsec_r_[PADR_(int32_t)];
+};
+struct __mac_mount_args {
+	char type_l_[PADL_(user_addr_t)]; user_addr_t type; char type_r_[PADR_(user_addr_t)];
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+	char data_l_[PADL_(user_addr_t)]; user_addr_t data; char data_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_get_mount_args {
+	char path_l_[PADL_(user_addr_t)]; user_addr_t path; char path_r_[PADR_(user_addr_t)];
+	char mac_p_l_[PADL_(user_addr_t)]; user_addr_t mac_p; char mac_p_r_[PADR_(user_addr_t)];
+};
+struct __mac_getfsstat_args {
+	char buf_l_[PADL_(user_addr_t)]; user_addr_t buf; char buf_r_[PADR_(user_addr_t)];
+	char bufsize_l_[PADL_(int)]; int bufsize; char bufsize_r_[PADR_(int)];
+	char mac_l_[PADL_(user_addr_t)]; user_addr_t mac; char mac_r_[PADR_(user_addr_t)];
+	char macsize_l_[PADL_(int)]; int macsize; char macsize_r_[PADR_(int)];
+	char flags_l_[PADL_(int)]; int flags; char flags_r_[PADR_(int)];
+};
 int nosys(struct proc *, struct nosys_args *, int *);
 void exit(struct proc *, struct exit_args *, int *);
 int fork(struct proc *, struct fork_args *, int *);
@@ -1317,12 +1722,15 @@ int setuid(struct proc *, struct setuid_args *, int *);
 int getuid(struct proc *, struct getuid_args *, int *);
 int geteuid(struct proc *, struct geteuid_args *, int *);
 int ptrace(struct proc *, struct ptrace_args *, int *);
+#if SOCKETS
 int recvmsg(struct proc *, struct recvmsg_args *, int *);
 int sendmsg(struct proc *, struct sendmsg_args *, int *);
 int recvfrom(struct proc *, struct recvfrom_args *, int *);
 int accept(struct proc *, struct accept_args *, int *);
 int getpeername(struct proc *, struct getpeername_args *, int *);
 int getsockname(struct proc *, struct getsockname_args *, int *);
+#else
+#endif /* SOCKETS */
 int access(struct proc *, struct access_args *, int *);
 int chflags(struct proc *, struct chflags_args *, int *);
 int fchflags(struct proc *, struct fchflags_args *, int *);
@@ -1333,7 +1741,6 @@ int dup(struct proc *, struct dup_args *, int *);
 int pipe(struct proc *, struct pipe_args *, int *);
 int getegid(struct proc *, struct getegid_args *, int *);
 int profil(struct proc *, struct profil_args *, int *);
-int ktrace(struct proc *, struct ktrace_args *, int *);
 int sigaction(struct proc *, struct sigaction_args *, int *);
 int getgid(struct proc *, struct getgid_args *, int *);
 int sigprocmask(struct proc *, struct sigprocmask_args *, int *);
@@ -1372,32 +1779,50 @@ int fcntl(struct proc *, struct fcntl_args *, int *);
 int select(struct proc *, struct select_args *, int *);
 int fsync(struct proc *, struct fsync_args *, int *);
 int setpriority(struct proc *, struct setpriority_args *, int *);
+#if SOCKETS
 int socket(struct proc *, struct socket_args *, int *);
 int connect(struct proc *, struct connect_args *, int *);
+#else
+#endif /* SOCKETS */
 int getpriority(struct proc *, struct getpriority_args *, int *);
+#if SOCKETS
 int bind(struct proc *, struct bind_args *, int *);
 int setsockopt(struct proc *, struct setsockopt_args *, int *);
 int listen(struct proc *, struct listen_args *, int *);
+#else
+#endif /* SOCKETS */
 int sigsuspend(struct proc *, struct sigsuspend_args *, int *);
+#if SOCKETS
+#else
+#endif /* SOCKETS */
 int gettimeofday(struct proc *, struct gettimeofday_args *, int *);
 int getrusage(struct proc *, struct getrusage_args *, int *);
+#if SOCKETS
 int getsockopt(struct proc *, struct getsockopt_args *, int *);
+#else
+#endif /* SOCKETS */
 int readv(struct proc *, struct readv_args *, user_ssize_t *);
 int writev(struct proc *, struct writev_args *, user_ssize_t *);
 int settimeofday(struct proc *, struct settimeofday_args *, int *);
 int fchown(struct proc *, struct fchown_args *, int *);
 int fchmod(struct proc *, struct fchmod_args *, int *);
+int setreuid(struct proc *, struct setreuid_args *, int *);
+int setregid(struct proc *, struct setregid_args *, int *);
 int rename(struct proc *, struct rename_args *, int *);
 int flock(struct proc *, struct flock_args *, int *);
 int mkfifo(struct proc *, struct mkfifo_args *, int *);
+#if SOCKETS
 int sendto(struct proc *, struct sendto_args *, int *);
 int shutdown(struct proc *, struct shutdown_args *, int *);
 int socketpair(struct proc *, struct socketpair_args *, int *);
+#else
+#endif /* SOCKETS */
 int mkdir(struct proc *, struct mkdir_args *, int *);
 int rmdir(struct proc *, struct rmdir_args *, int *);
 int utimes(struct proc *, struct utimes_args *, int *);
 int futimes(struct proc *, struct futimes_args *, int *);
 int adjtime(struct proc *, struct adjtime_args *, int *);
+int gethostuuid(struct proc *, struct gethostuuid_args *, int *);
 int setsid(struct proc *, struct setsid_args *, int *);
 int getpgid(struct proc *, struct getpgid_args *, int *);
 int setprivexec(struct proc *, struct setprivexec_args *, int *);
@@ -1410,12 +1835,13 @@ int nfssvc(struct proc *, struct nfssvc_args *, int *);
 int statfs(struct proc *, struct statfs_args *, int *);
 int fstatfs(struct proc *, struct fstatfs_args *, int *);
 int unmount(struct proc *, struct unmount_args *, int *);
-#if NFSCLIENT
+#if NFSSERVER
 int getfh(struct proc *, struct getfh_args *, int *);
 #else
 #endif
 int quotactl(struct proc *, struct quotactl_args *, int *);
 int mount(struct proc *, struct mount_args *, int *);
+int csops(struct proc *, struct csops_args *, int *);
 int waitid(struct proc *, struct waitid_args *, int *);
 int add_profil(struct proc *, struct add_profil_args *, int *);
 int kdebug_trace(struct proc *, struct kdebug_trace_args *, int *);
@@ -1429,10 +1855,6 @@ int fstat(struct proc *, struct fstat_args *, int *);
 int lstat(struct proc *, struct lstat_args *, int *);
 int pathconf(struct proc *, struct pathconf_args *, int *);
 int fpathconf(struct proc *, struct fpathconf_args *, int *);
-#if COMPAT_GETFSSTAT
-int getfsstat(struct proc *, struct getfsstat_args *, int *);
-#else
-#endif
 int getrlimit(struct proc *, struct getrlimit_args *, int *);
 int setrlimit(struct proc *, struct setrlimit_args *, int *);
 int getdirentries(struct proc *, struct getdirentries_args *, int *);
@@ -1444,6 +1866,7 @@ int __sysctl(struct proc *, struct __sysctl_args *, int *);
 int mlock(struct proc *, struct mlock_args *, int *);
 int munlock(struct proc *, struct munlock_args *, int *);
 int undelete(struct proc *, struct undelete_args *, int *);
+#if NETAT
 int ATsocket(struct proc *, struct ATsocket_args *, int *);
 int ATgetmsg(struct proc *, struct ATgetmsg_args *, int *);
 int ATputmsg(struct proc *, struct ATputmsg_args *, int *);
@@ -1451,16 +1874,14 @@ int ATPsndreq(struct proc *, struct ATPsndreq_args *, int *);
 int ATPsndrsp(struct proc *, struct ATPsndrsp_args *, int *);
 int ATPgetreq(struct proc *, struct ATPgetreq_args *, int *);
 int ATPgetrsp(struct proc *, struct ATPgetrsp_args *, int *);
+#else
+#endif /* NETAT */
 int kqueue_from_portset_np(struct proc *, struct kqueue_from_portset_np_args *, int *);
 int kqueue_portset_np(struct proc *, struct kqueue_portset_np_args *, int *);
 int getattrlist(struct proc *, struct getattrlist_args *, int *);
 int setattrlist(struct proc *, struct setattrlist_args *, int *);
 int getdirentriesattr(struct proc *, struct getdirentriesattr_args *, int *);
 int exchangedata(struct proc *, struct exchangedata_args *, int *);
-#ifdef __APPLE_API_OBSOLETE
-int checkuseraccess(struct proc *, struct checkuseraccess_args *, int *);
-#else
-#endif /* __APPLE_API_OBSOLETE */
 int searchfs(struct proc *, struct searchfs_args *, int *);
 int delete(struct proc *, struct delete_args *, int *);
 int copyfile(struct proc *, struct copyfile_args *, int *);
@@ -1478,26 +1899,48 @@ int listxattr(struct proc *, struct listxattr_args *, user_ssize_t *);
 int flistxattr(struct proc *, struct flistxattr_args *, user_ssize_t *);
 int fsctl(struct proc *, struct fsctl_args *, int *);
 int initgroups(struct proc *, struct initgroups_args *, int *);
+int posix_spawn(struct proc *, struct posix_spawn_args *, int *);
 #if NFSCLIENT
 int nfsclnt(struct proc *, struct nfsclnt_args *, int *);
+#else
+#endif
+#if NFSSERVER
 int fhopen(struct proc *, struct fhopen_args *, int *);
 #else
 #endif
 int minherit(struct proc *, struct minherit_args *, int *);
+#if SYSV_SEM
 int semsys(struct proc *, struct semsys_args *, int *);
+#else
+#endif
+#if SYSV_MSG
 int msgsys(struct proc *, struct msgsys_args *, int *);
+#else
+#endif
+#if SYSV_SHM
 int shmsys(struct proc *, struct shmsys_args *, int *);
+#else
+#endif
+#if SYSV_SEM
 int semctl(struct proc *, struct semctl_args *, int *);
 int semget(struct proc *, struct semget_args *, int *);
 int semop(struct proc *, struct semop_args *, int *);
+#else
+#endif
+#if SYSV_MSG
 int msgctl(struct proc *, struct msgctl_args *, int *);
 int msgget(struct proc *, struct msgget_args *, int *);
 int msgsnd(struct proc *, struct msgsnd_args *, int *);
 int msgrcv(struct proc *, struct msgrcv_args *, user_ssize_t *);
-int shmat(struct proc *, struct shmat_args *, int *);
+#else
+#endif
+#if SYSV_SHM
+int shmat(struct proc *, struct shmat_args *, user_addr_t *);
 int shmctl(struct proc *, struct shmctl_args *, int *);
 int shmdt(struct proc *, struct shmdt_args *, int *);
 int shmget(struct proc *, struct shmget_args *, int *);
+#else
+#endif
 int shm_open(struct proc *, struct shm_open_args *, int *);
 int shm_unlink(struct proc *, struct shm_unlink_args *, int *);
 int sem_open(struct proc *, struct sem_open_args *, user_addr_t *);
@@ -1526,13 +1969,20 @@ int getwgroups(struct proc *, struct getwgroups_args *, int *);
 int mkfifo_extended(struct proc *, struct mkfifo_extended_args *, int *);
 int mkdir_extended(struct proc *, struct mkdir_extended_args *, int *);
 int identitysvc(struct proc *, struct identitysvc_args *, int *);
-int load_shared_file(struct proc *, struct load_shared_file_args *, int *);
-int reset_shared_file(struct proc *, struct reset_shared_file_args *, int *);
-int new_system_shared_regions(struct proc *, struct new_system_shared_regions_args *, int *);
-int shared_region_map_file_np(struct proc *, struct shared_region_map_file_np_args *, int *);
-int shared_region_make_private_np(struct proc *, struct shared_region_make_private_np_args *, int *);
+int shared_region_check_np(struct proc *, struct shared_region_check_np_args *, int *);
+int shared_region_map_np(struct proc *, struct shared_region_map_np_args *, int *);
+int __pthread_mutex_destroy(struct proc *, struct __pthread_mutex_destroy_args *, int *);
+int __pthread_mutex_init(struct proc *, struct __pthread_mutex_init_args *, int *);
+int __pthread_mutex_lock(struct proc *, struct __pthread_mutex_lock_args *, int *);
+int __pthread_mutex_trylock(struct proc *, struct __pthread_mutex_trylock_args *, int *);
+int __pthread_mutex_unlock(struct proc *, struct __pthread_mutex_unlock_args *, int *);
+int __pthread_cond_init(struct proc *, struct __pthread_cond_init_args *, int *);
+int __pthread_cond_destroy(struct proc *, struct __pthread_cond_destroy_args *, int *);
+int __pthread_cond_broadcast(struct proc *, struct __pthread_cond_broadcast_args *, int *);
+int __pthread_cond_signal(struct proc *, struct __pthread_cond_signal_args *, int *);
 int getsid(struct proc *, struct getsid_args *, int *);
 int settid_with_pid(struct proc *, struct settid_with_pid_args *, int *);
+int __pthread_cond_timedwait(struct proc *, struct __pthread_cond_timedwait_args *, int *);
 int aio_fsync(struct proc *, struct aio_fsync_args *, int *);
 int aio_return(struct proc *, struct aio_return_args *, user_ssize_t *);
 int aio_suspend(struct proc *, struct aio_suspend_args *, int *);
@@ -1541,18 +1991,36 @@ int aio_error(struct proc *, struct aio_error_args *, int *);
 int aio_read(struct proc *, struct aio_read_args *, int *);
 int aio_write(struct proc *, struct aio_write_args *, int *);
 int lio_listio(struct proc *, struct lio_listio_args *, int *);
+int __pthread_cond_wait(struct proc *, struct __pthread_cond_wait_args *, int *);
+int iopolicysys(struct proc *, struct iopolicysys_args *, int *);
 int mlockall(struct proc *, struct mlockall_args *, int *);
 int munlockall(struct proc *, struct munlockall_args *, int *);
 int issetugid(struct proc *, struct issetugid_args *, int *);
 int __pthread_kill(struct proc *, struct __pthread_kill_args *, int *);
-int pthread_sigmask(struct proc *, struct pthread_sigmask_args *, int *);
-int sigwait(struct proc *, struct sigwait_args *, int *);
+int __pthread_sigmask(struct proc *, struct __pthread_sigmask_args *, int *);
+int __sigwait(struct proc *, struct __sigwait_args *, int *);
 int __disable_threadsignal(struct proc *, struct __disable_threadsignal_args *, int *);
 int __pthread_markcancel(struct proc *, struct __pthread_markcancel_args *, int *);
 int __pthread_canceled(struct proc *, struct __pthread_canceled_args *, int *);
 int __semwait_signal(struct proc *, struct __semwait_signal_args *, int *);
-int utrace(struct proc *, struct utrace_args *, int *);
 int proc_info(struct proc *, struct proc_info_args *, int *);
+#if SENDFILE
+int sendfile(struct proc *, struct sendfile_args *, int *);
+#else /* !SENDFILE */
+#endif /* SENDFILE */
+int stat64(struct proc *, struct stat64_args *, int *);
+int fstat64(struct proc *, struct fstat64_args *, int *);
+int lstat64(struct proc *, struct lstat64_args *, int *);
+int stat64_extended(struct proc *, struct stat64_extended_args *, int *);
+int lstat64_extended(struct proc *, struct lstat64_extended_args *, int *);
+int fstat64_extended(struct proc *, struct fstat64_extended_args *, int *);
+int getdirentries64(struct proc *, struct getdirentries64_args *, user_ssize_t *);
+int statfs64(struct proc *, struct statfs64_args *, int *);
+int fstatfs64(struct proc *, struct fstatfs64_args *, int *);
+int getfsstat64(struct proc *, struct getfsstat64_args *, int *);
+int __pthread_chdir(struct proc *, struct __pthread_chdir_args *, int *);
+int __pthread_fchdir(struct proc *, struct __pthread_fchdir_args *, int *);
+#if AUDIT
 int audit(struct proc *, struct audit_args *, int *);
 int auditon(struct proc *, struct auditon_args *, int *);
 int getauid(struct proc *, struct getauid_args *, int *);
@@ -1562,10 +2030,76 @@ int setaudit(struct proc *, struct setaudit_args *, int *);
 int getaudit_addr(struct proc *, struct getaudit_addr_args *, int *);
 int setaudit_addr(struct proc *, struct setaudit_addr_args *, int *);
 int auditctl(struct proc *, struct auditctl_args *, int *);
+#else
+#endif
+int bsdthread_create(struct proc *, struct bsdthread_create_args *, user_addr_t *);
+int bsdthread_terminate(struct proc *, struct bsdthread_terminate_args *, int *);
 int kqueue(struct proc *, struct kqueue_args *, int *);
 int kevent(struct proc *, struct kevent_args *, int *);
 int lchown(struct proc *, struct lchown_args *, int *);
 int stack_snapshot(struct proc *, struct stack_snapshot_args *, int *);
+int bsdthread_register(struct proc *, struct bsdthread_register_args *, int *);
+int workq_open(struct proc *, struct workq_open_args *, int *);
+int workq_ops(struct proc *, struct workq_ops_args *, int *);
+int __mac_execve(struct proc *, struct __mac_execve_args *, int *);
+int __mac_syscall(struct proc *, struct __mac_syscall_args *, int *);
+int __mac_get_file(struct proc *, struct __mac_get_file_args *, int *);
+int __mac_set_file(struct proc *, struct __mac_set_file_args *, int *);
+int __mac_get_link(struct proc *, struct __mac_get_link_args *, int *);
+int __mac_set_link(struct proc *, struct __mac_set_link_args *, int *);
+int __mac_get_proc(struct proc *, struct __mac_get_proc_args *, int *);
+int __mac_set_proc(struct proc *, struct __mac_set_proc_args *, int *);
+int __mac_get_fd(struct proc *, struct __mac_get_fd_args *, int *);
+int __mac_set_fd(struct proc *, struct __mac_set_fd_args *, int *);
+int __mac_get_pid(struct proc *, struct __mac_get_pid_args *, int *);
+int __mac_get_lcid(struct proc *, struct __mac_get_lcid_args *, int *);
+int __mac_get_lctx(struct proc *, struct __mac_get_lctx_args *, int *);
+int __mac_set_lctx(struct proc *, struct __mac_set_lctx_args *, int *);
+int setlcid(struct proc *, struct setlcid_args *, int *);
+int getlcid(struct proc *, struct getlcid_args *, int *);
+int read_nocancel(struct proc *, struct read_nocancel_args *, user_ssize_t *);
+int write_nocancel(struct proc *, struct write_nocancel_args *, user_ssize_t *);
+int open_nocancel(struct proc *, struct open_nocancel_args *, int *);
+int close_nocancel(struct proc *, struct close_nocancel_args *, int *);
+int wait4_nocancel(struct proc *, struct wait4_nocancel_args *, int *);
+#if SOCKETS
+int recvmsg_nocancel(struct proc *, struct recvmsg_nocancel_args *, int *);
+int sendmsg_nocancel(struct proc *, struct sendmsg_nocancel_args *, int *);
+int recvfrom_nocancel(struct proc *, struct recvfrom_nocancel_args *, int *);
+int accept_nocancel(struct proc *, struct accept_nocancel_args *, int *);
+#else
+#endif /* SOCKETS */
+int msync_nocancel(struct proc *, struct msync_nocancel_args *, int *);
+int fcntl_nocancel(struct proc *, struct fcntl_nocancel_args *, int *);
+int select_nocancel(struct proc *, struct select_nocancel_args *, int *);
+int fsync_nocancel(struct proc *, struct fsync_nocancel_args *, int *);
+#if SOCKETS
+int connect_nocancel(struct proc *, struct connect_nocancel_args *, int *);
+#else
+#endif /* SOCKETS */
+int sigsuspend_nocancel(struct proc *, struct sigsuspend_nocancel_args *, int *);
+int readv_nocancel(struct proc *, struct readv_nocancel_args *, user_ssize_t *);
+int writev_nocancel(struct proc *, struct writev_nocancel_args *, user_ssize_t *);
+#if SOCKETS
+int sendto_nocancel(struct proc *, struct sendto_nocancel_args *, int *);
+#else
+#endif /* SOCKETS */
+int pread_nocancel(struct proc *, struct pread_nocancel_args *, user_ssize_t *);
+int pwrite_nocancel(struct proc *, struct pwrite_nocancel_args *, user_ssize_t *);
+int waitid_nocancel(struct proc *, struct waitid_nocancel_args *, int *);
+int poll_nocancel(struct proc *, struct poll_nocancel_args *, int *);
+#if SYSV_MSG
+int msgsnd_nocancel(struct proc *, struct msgsnd_nocancel_args *, int *);
+int msgrcv_nocancel(struct proc *, struct msgrcv_nocancel_args *, user_ssize_t *);
+#else
+#endif
+int sem_wait_nocancel(struct proc *, struct sem_wait_nocancel_args *, int *);
+int aio_suspend_nocancel(struct proc *, struct aio_suspend_nocancel_args *, int *);
+int __sigwait_nocancel(struct proc *, struct __sigwait_nocancel_args *, int *);
+int __semwait_signal_nocancel(struct proc *, struct __semwait_signal_nocancel_args *, int *);
+int __mac_mount(struct proc *, struct __mac_mount_args *, int *);
+int __mac_get_mount(struct proc *, struct __mac_get_mount_args *, int *);
+int __mac_getfsstat(struct proc *, struct __mac_getfsstat_args *, int *);
 
 __END_DECLS
 #undef PAD_
diff --git a/bsd/sys/systm.h b/bsd/sys/systm.h
index e81bed439..368f4f3e6 100644
--- a/bsd/sys/systm.h
+++ b/bsd/sys/systm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -100,7 +106,6 @@
 #ifdef BSD_KERNEL_PRIVATE
 #include <sys/tty.h>
 #include <sys/vm.h>
-#include <sys/linker_set.h>
 #endif
 #include <sys/proc.h>
 __BEGIN_DECLS
@@ -147,6 +152,9 @@ __BEGIN_DECLS
 int	nullop(void);
 int	nulldev(void);
 int	enoioctl(void);
+int	enosys(void);
+int	errsys(void);
+void	nullsys(void);
 int	enxio(void);
 int	eopnotsupp(void);
 int	einval(void);
@@ -154,7 +162,7 @@ int	einval(void);
 #ifdef BSD_KERNEL_PRIVATE
 int	seltrue(dev_t dev, int which, struct proc *p);
 void	ttyprintf(struct tty *, const char *, ...);
-void	realitexpire(void *);
+void	realitexpire(struct proc *);
 int	hzto(struct timeval *tv);
 #endif /* __APPLE_API_UNSTABLE */
 
@@ -165,7 +173,7 @@ void	tablefull(const char *);
 int	kvprintf(char const *, void (*)(int, void*), void *, int,
 		      __darwin_va_list);
 
-void	uprintf(const char *, ...);
+void	uprintf(const char *, ...) __printflike(1,2);
 
 
 void	ovbcopy(const void *from, void *to, size_t len);
@@ -206,10 +214,9 @@ int	clone_system_shared_regions(int shared_regions_active,
 				    int chain_regions,
 				    int base_vnode);
 
-extern kern_return_t bsd_exception(int, exception_data_type_t codes[], int);
+extern kern_return_t bsd_exception(int, mach_exception_data_t codes, int);
 extern void	bsdinit_task(void);
-void	bsd_hardclock(boolean_t usermode, caddr_t pc, int numticks);
-void	gatherstats(boolean_t usermode, caddr_t pc);
+extern void unix_syscall_return(int) __dead2;
 
 void	initclocks(void);
 
@@ -221,10 +228,11 @@ struct time_value;
 void	get_procrustime(struct time_value *tv);
 
 void	load_init_program(struct proc *p);
+void __pthread_testcancel(int presyscall);
+void syscall_exit_funnelcheck(void);
 #endif /* BSD_KERNEL_PRIVATE */
 
 
 __END_DECLS
 
 #endif /* !_SYS_SYSTM_H_ */
-
diff --git a/bsd/sys/termios.h b/bsd/sys/termios.h
index d6d13dae3..e6bd8c950 100644
--- a/bsd/sys/termios.h
+++ b/bsd/sys/termios.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -69,33 +75,33 @@
  */
 #define	VEOF		0	/* ICANON */
 #define	VEOL		1	/* ICANON */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	VEOL2		2	/* ICANON together with IEXTEN */
 #endif
 #define	VERASE		3	/* ICANON */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define VWERASE 	4	/* ICANON together with IEXTEN */
 #endif
 #define VKILL		5	/* ICANON */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	VREPRINT 	6	/* ICANON together with IEXTEN */
 #endif
 /*			7	   spare 1 */
 #define VINTR		8	/* ISIG */
 #define VQUIT		9	/* ISIG */
 #define VSUSP		10	/* ISIG */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define VDSUSP		11	/* ISIG together with IEXTEN */
 #endif
 #define VSTART		12	/* IXON, IXOFF */
 #define VSTOP		13	/* IXON, IXOFF */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	VLNEXT		14	/* IEXTEN */
 #define	VDISCARD	15	/* IEXTEN */
 #endif
 #define VMIN		16	/* !ICANON */
 #define VTIME		17	/* !ICANON */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define VSTATUS		18	/* ICANON together with IEXTEN */
 /*			19	   spare 2 */
 #endif
@@ -105,7 +111,7 @@
 #define	_POSIX_VDISABLE	0xff
 #endif
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	CCEQ(val, c)	((c) == (val) ? (val) != _POSIX_VDISABLE : 0)
 #endif
 
@@ -124,19 +130,20 @@
 #define	IXON		0x00000200	/* enable output flow control */
 #define	IXOFF		0x00000400	/* enable input flow control */
 #define	IXANY		0x00000800	/* any char will restart after stop */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define IMAXBEL		0x00002000	/* ring bell on input queue full */
-#endif  /*_POSIX_C_SOURCE */
+#define	IUTF8		0x00004000	/* maintain state for UTF-8 VERASE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Output flags - software output processing
  */
 #define	OPOST		0x00000001	/* enable following output processing */
 #define ONLCR		0x00000002	/* map NL to CR-NL (ala CRMOD) */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define OXTABS		0x00000004	/* expand tabs to spaces */
 #define ONOEOT		0x00000008	/* discard EOT's (^D) on output) */
-#endif  /*_POSIX_C_SOURCE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 /*
  * The following block of features is unimplemented.  Use of these flags in
  * programs will currently result in unexpected behaviour.
@@ -148,31 +155,34 @@
 #define	ONLRET		0x00000040	/* NL performs CR function */
 #define	OFILL		0x00000080	/* use fill characters for delay */
 #define	NLDLY		0x00000300	/* \n delay */
-#define	TABDLY		0x00000c00	/* horizontal tab delay */
+#define	TABDLY		0x00000c04	/* horizontal tab delay */
 #define	CRDLY		0x00003000	/* \r delay */
 #define	FFDLY		0x00004000	/* form feed delay */
 #define	BSDLY		0x00008000	/* \b delay */
 #define	VTDLY		0x00010000	/* vertical tab delay */
 #define	OFDEL		0x00020000	/* fill is DEL, else NUL */
-#if !defined(_SYS_IOCTL_COMPAT_H_) || defined(_POSIX_C_SOURCE)
+#if !defined(_SYS_IOCTL_COMPAT_H_) || __DARWIN_UNIX03
 /*
  * These manifest constants have the same names as those in the header
  * <sys/ioctl_compat.h>, so you are not permitted to have both definitions
  * in scope simultaneously in the same compilation unit.  Nevertheless,
  * they are required to be in scope when _POSIX_C_SOURCE is requested;
  * this means that including the <sys/ioctl_compat.h> header before this
- * one whien _POSIX_C_SOURCE is in scope will result in redefintions.  We
+ * one when _POSIX_C_SOURCE is in scope will result in redefintions.  We
  * attempt to maintain these as the same values so as to avoid this being
  * an outright error in most compilers.
  */
 #define		NL0	0x00000000
 #define		NL1	0x00000100
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define		NL2	0x00000200
 #define		NL3	0x00000300
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define		TAB0	0x00000000
 #define		TAB1	0x00000400
 #define		TAB2	0x00000800
-#define		TAB3	0x00000c00
+/* not in sys/ioctl_compat.h, use OXTABS value */
+#define		TAB3	0x00000004
 #define		CR0	0x00000000
 #define		CR1	0x00001000
 #define		CR2	0x00002000
@@ -191,7 +201,7 @@
 /*
  * Control flags - hardware control of terminal
  */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	CIGNORE		0x00000001	/* ignore control flags */
 #endif
 #define CSIZE		0x00000300	/* character size mask */
@@ -205,7 +215,7 @@
 #define PARODD		0x00002000	/* odd parity, else even */
 #define HUPCL		0x00004000	/* hang up on last close */
 #define CLOCAL		0x00008000	/* ignore modem status lines */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define CCTS_OFLOW	0x00010000	/* CTS flow control of output */
 #define CRTSCTS		(CCTS_OFLOW | CRTS_IFLOW)
 #define CRTS_IFLOW	0x00020000	/* RTS flow control of input */
@@ -224,35 +234,37 @@
  * input flag.
  */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ECHOKE		0x00000001	/* visual erase for line kill */
-#endif  /*_POSIX_C_SOURCE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	ECHOE		0x00000002	/* visually erase chars */
 #define	ECHOK		0x00000004	/* echo NL after line kill */
 #define ECHO		0x00000008	/* enable echoing */
 #define	ECHONL		0x00000010	/* echo NL even if ECHO is off */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	ECHOPRT		0x00000020	/* visual erase mode for hardcopy */
 #define ECHOCTL  	0x00000040	/* echo control chars as ^(Char) */
-#endif  /*_POSIX_C_SOURCE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	ISIG		0x00000080	/* enable signals INTR, QUIT, [D]SUSP */
 #define	ICANON		0x00000100	/* canonicalize input lines */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define ALTWERASE	0x00000200	/* use alternate WERASE algorithm */
-#endif  /*_POSIX_C_SOURCE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	IEXTEN		0x00000400	/* enable DISCARD and LNEXT */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define EXTPROC         0x00000800      /* external processing */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define TOSTOP		0x00400000	/* stop background jobs from output */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define FLUSHO		0x00800000	/* output being flushed (state) */
 #define	NOKERNINFO	0x02000000	/* no kernel output from VSTATUS */
 #define PENDIN		0x20000000	/* XXX retype pending input (state) */
-#endif  /*_POSIX_C_SOURCE */
+#endif  /*(_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 #define	NOFLSH		0x80000000	/* don't flush after interrupt */
 
 typedef unsigned long	tcflag_t;
 typedef unsigned char	cc_t;
-typedef long		speed_t;	/* XXX should be unsigned long */
+typedef unsigned long	speed_t;
 
 struct termios {
 	tcflag_t	c_iflag;	/* input flags */
@@ -292,7 +304,7 @@ struct user_termios {
 #define	TCSANOW		0		/* make change immediate */
 #define	TCSADRAIN	1		/* drain output, then change */
 #define	TCSAFLUSH	2		/* drain output, flush input */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define TCSASOFT	0x10		/* flag - don't alter h.w. state */
 #endif
 
@@ -315,7 +327,7 @@ struct user_termios {
 #define B9600	9600
 #define B19200	19200
 #define B38400	38400
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define B7200	7200
 #define B14400	14400
 #define B28800	28800
@@ -325,7 +337,7 @@ struct user_termios {
 #define B230400	230400
 #define EXTA	19200
 #define EXTB	38400
-#endif  /* !_POSIX_C_SOURCE */
+#endif  /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #ifndef KERNEL
 
@@ -346,20 +358,20 @@ int	cfsetispeed(struct termios *, speed_t);
 int	cfsetospeed(struct termios *, speed_t);
 int	tcgetattr(int, struct termios *);
 int	tcsetattr(int, int, const struct termios *);
-int	tcdrain(int);
+int	tcdrain(int) __DARWIN_ALIAS_C(tcdrain);
 int	tcflow(int, int);
 int	tcflush(int, int);
 int	tcsendbreak(int, int);
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 void	cfmakeraw(struct termios *);
 int	cfsetspeed(struct termios *, speed_t);
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 __END_DECLS
 
 #endif /* !KERNEL */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 
 /*
  * Include tty ioctl's that aren't just for backwards compatibility
@@ -374,6 +386,6 @@ __END_DECLS
  */
 #endif /* !_SYS_TERMIOS_H_ */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #include <sys/ttydefaults.h>
 #endif
diff --git a/bsd/sys/time.h b/bsd/sys/time.h
index 2801abed4..6f3d264cb 100644
--- a/bsd/sys/time.h
+++ b/bsd/sys/time.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -60,6 +66,21 @@
 
 #include <sys/cdefs.h>
 #include <sys/_types.h>
+#ifdef KERNEL
+#include <machine/types.h>	/* user_time_t */
+#endif /* KERNEL */
+
+/*
+ * [XSI] The fd_set type shall be defined as described in <sys/select.h>.
+ * The timespec structure shall be defined as described in <time.h>
+ */
+#define __need_fd_set
+#define __need_struct_timespec
+#define __need_struct_timeval
+#ifdef KERNEL
+#define __need_struct_user_timespec
+#endif /* KERNEL */
+#include <sys/_structs.h>
 
 #ifndef	_TIME_T
 #define	_TIME_T
@@ -71,19 +92,6 @@ typedef	__darwin_time_t	time_t;
 typedef __darwin_suseconds_t	suseconds_t;
 #endif
 
-
-/*
- * Structure returned by gettimeofday(2) system call,
- * and used in other calls.
- */
-#ifndef _TIMEVAL
-#define _TIMEVAL
-struct timeval {
-	time_t		tv_sec;		/* seconds */
-	suseconds_t	tv_usec;	/* and microseconds */
-};
-#endif	/* _TIMEVAL */
-
 /*
  * Structure used as a parameter by getitimer(2) and setitimer(2) system
  * calls.
@@ -101,16 +109,6 @@ struct	itimerval {
 #define	ITIMER_VIRTUAL	1
 #define	ITIMER_PROF	2
 
-
-/*
- * [XSI] The fd_set type shall be defined as described in <sys/select.h>.
- *
- * Note:	We use _FD_SET to protect all select related
- *		types and macros
- */
-#ifndef _FD_SET
-#define	_FD_SET
-
 /*
  * Select uses bit masks of file descriptors in longs.  These macros
  * manipulate such bit fields (the filesystem macros use chars).  The
@@ -118,65 +116,26 @@ struct	itimerval {
  * the default size.
  */
 #ifndef	FD_SETSIZE
-#define	FD_SETSIZE	1024
-#endif
-
-#define	__DARWIN_NBBY	8				/* bits in a byte */
-#define __DARWIN_NFDBITS	(sizeof(__int32_t) * __DARWIN_NBBY) /* bits per mask */
-#define	__DARWIN_howmany(x, y) (((x) + ((y) - 1)) / (y))	/* # y's == x bits? */
-
-__BEGIN_DECLS
-typedef	struct fd_set {
-	__int32_t	fds_bits[__DARWIN_howmany(FD_SETSIZE, __DARWIN_NFDBITS)];
-} fd_set;
-__END_DECLS
-
-#define	FD_SET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] |= (1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_CLR(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] &= ~(1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_ISSET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] & (1<<((n) % __DARWIN_NFDBITS)))
-#if __GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3
-/*
- * Use the built-in bzero function instead of the library version so that
- * we do not pollute the namespace or introduce prototype warnings.
- */
-#define	FD_ZERO(p)	__builtin_bzero(p, sizeof(*(p)))
-#else
-#define	FD_ZERO(p)	bzero(p, sizeof(*(p)))
-#endif
-#ifndef _POSIX_C_SOURCE
-#define	FD_COPY(f, t)	bcopy(f, t, sizeof(*(f)))
-#endif	/* !_POSIX_C_SOURCE */
-
-#endif	/* !_FD_SET */
-
-
-#ifndef _POSIX_C_SOURCE
-/*
- * Structure defined by POSIX.4 to be like a timeval.
- */
-#ifndef _TIMESPEC
-#define _TIMESPEC
-struct timespec {
-	time_t	tv_sec;		/* seconds */
-	long	tv_nsec;	/* and nanoseconds */
-};
-
-#ifdef KERNEL
-// LP64todo - should this move?
-#include <machine/types.h>	/* user_time_t */
-
-/* LP64 version of struct timespec.  time_t is a long and must grow when 
- * we're dealing with a 64-bit process.
- * WARNING - keep in sync with struct timespec
- */
-struct user_timespec {
-	user_time_t	tv_sec;		/* seconds */
-	int32_t	tv_nsec __attribute((aligned(8)));	/* and nanoseconds */
-};
-
-#endif
-#endif
-
+#define	FD_SETSIZE	__DARWIN_FD_SETSIZE
+#endif	/* FD_SETSIZE */
+#ifndef FD_SET
+#define	FD_SET(n, p)	__DARWIN_FD_SET(n, p)
+#endif	/* FD_SET */
+#ifndef	FD_CLR
+#define	FD_CLR(n,p)	__DARWIN_FD_CLR(n, p)
+#endif	/* FD_CLR */
+#ifndef FD_ISSET
+#define	FD_ISSET(n, p)	__DARWIN_FD_ISSET(n, p)
+#endif	/* FD_ISSET */
+#ifndef FD_ZERO
+#define	FD_ZERO(p)	__DARWIN_FD_ZERO(p)
+#endif	/* FD_ZERO */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+
+#ifndef FD_COPY
+#define	FD_COPY(f, t)	__DARWIN_FD_COPY(f, t)
+#endif	/* FD_COPY */
 
 #ifdef KERNEL
 #ifndef _USERTIMEVAL
@@ -199,9 +158,8 @@ struct	user_itimerval {
 	struct	user_timeval it_value;		/* current value */
 };
 
-#endif
-#endif // KERNEL
-
+#endif /* _USERTIMEVAL */
+#endif /* KERNEL */
 
 #define	TIMEVAL_TO_TIMESPEC(tv, ts) {					\
 	(ts)->tv_sec = (tv)->tv_sec;					\
@@ -262,12 +220,11 @@ struct clockinfo {
 	int	stathz;		/* statistics clock frequency */
 	int	profhz;		/* profiling clock frequency */
 };
-#endif /* ! _POSIX_C_SOURCE */
-
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #ifdef KERNEL
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 __BEGIN_DECLS
 void	microtime(struct timeval *tv);
 void	microuptime(struct timeval *tv);
@@ -282,31 +239,33 @@ void	timevalsub(struct timeval *t1, struct timeval *t2);
 void	timevalfix(struct timeval *t1);
 #ifdef	BSD_KERNEL_PRIVATE
 time_t	boottime_sec(void);
-void	inittodr(time_t base);
-int	itimerfix(struct timeval *tv);
-int	itimerdecr(struct itimerval *itp, int usec);
+void	inittodr(time_t base) __attribute__((section("__TEXT, initcode")));
 #endif /* BSD_KERNEL_PRIVATE */
 
 __END_DECLS
 
-#endif /* ! _POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #else /* !KERNEL */
 
-__BEGIN_DECLS
-
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #include <time.h>
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
+
+__BEGIN_DECLS
 
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 int	adjtime(const struct timeval *, struct timeval *);
 int	futimes(int, const struct timeval *);
+int	lutimes(const char *, const struct timeval *);
 int	settimeofday(const struct timeval *, const struct timezone *);
-#endif /* ! _POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 int	getitimer(int, struct itimerval *);
-int	gettimeofday(struct timeval * __restrict, struct timezone * __restrict);
-int	select(int, fd_set * __restrict, fd_set * __restrict,
-		fd_set * __restrict, struct timeval * __restrict);
+int	gettimeofday(struct timeval * __restrict, void * __restrict);
+
+#include <sys/_select.h>	/* select() prototype */
+
 int	setitimer(int, const struct itimerval * __restrict,
 		struct itimerval * __restrict);
 int	utimes(const char *, const struct timeval *);
diff --git a/bsd/sys/timeb.h b/bsd/sys/timeb.h
index 604a4fabd..a5c92708a 100644
--- a/bsd/sys/timeb.h
+++ b/bsd/sys/timeb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/times.h b/bsd/sys/times.h
index 0cd072e49..d1b014321 100644
--- a/bsd/sys/times.h
+++ b/bsd/sys/times.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/tprintf.h b/bsd/sys/tprintf.h
index 16f35aa38..cf066876d 100644
--- a/bsd/sys/tprintf.h
+++ b/bsd/sys/tprintf.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/trace.h b/bsd/sys/trace.h
index 04d279aa5..75ec17900 100644
--- a/bsd/sys/trace.h
+++ b/bsd/sys/trace.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/tty.h b/bsd/sys/tty.h
index b74c3f510..97178a419 100644
--- a/bsd/sys/tty.h
+++ b/bsd/sys/tty.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -132,6 +138,8 @@ struct tty {
 	int	t_gen;			/* Generation number. */
 };
 
+#define TTY_NULL (struct tty *)0
+
 #define	t_cc		t_termios.c_cc
 #define	t_cflag		t_termios.c_cflag
 #define	t_iflag		t_termios.c_iflag
@@ -235,13 +243,6 @@ struct speedtab {
 #define	TTY_BI		0x08000000	/* Break condition */
 
 #ifdef KERNEL
-/* Is tp controlling terminal for p? */
-#define	isctty(p, tp)							\
-	((p)->p_session == (tp)->t_session && (p)->p_flag & P_CONTROLT)
-
-/* Is p in background of tp? */
-#define	isbackground(p, tp)						\
-	(isctty((p), (tp)) && (p)->p_pgrp != (tp)->t_pgrp)
 
 /* Unique sleep addresses. */
 #define	TSA_CARR_ON(tp)		((void *)&(tp)->t_rawq)
diff --git a/bsd/sys/ttychars.h b/bsd/sys/ttychars.h
index 15a12df1a..e536c1a2b 100644
--- a/bsd/sys/ttychars.h
+++ b/bsd/sys/ttychars.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/ttycom.h b/bsd/sys/ttycom.h
index b7c4d8b3c..eb6a60830 100644
--- a/bsd/sys/ttycom.h
+++ b/bsd/sys/ttycom.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -112,6 +118,8 @@ struct winsize {
 #endif	/* KERNEL */
 #define	TIOCGETD	_IOR('t', 26, int)	/* get line discipline */
 #define	TIOCSETD	_IOW('t', 27, int)	/* set line discipline */
+#define	TIOCIXON	 _IO('t', 129)		/* internal input VSTART */
+#define	TIOCIXOFF	 _IO('t', 128)		/* internal input VSTOP */
 						/* 127-124 compat */
 #define	TIOCSBRK	 _IO('t', 123)		/* set break bit */
 #define	TIOCCBRK	 _IO('t', 122)		/* clear break bit */
@@ -156,10 +164,17 @@ struct winsize {
 						 * of last input event */
 #define	TIOCDCDTIMESTAMP _IOR('t', 88, struct timeval)	/* enable/get timestamp
 						 * of last DCd rise */
+#ifdef KERNEL
+#define	TIOCTIMESTAMP_64	_IOR('t', 89, struct user_timeval)
+#define	TIOCDCDTIMESTAMP_64	_IOR('t', 88, struct user_timeval)
+#endif
 #define	TIOCSDRAINWAIT	_IOW('t', 87, int)	/* set ttywait timeout */
 #define	TIOCGDRAINWAIT	_IOR('t', 86, int)	/* get ttywait timeout */
 #define	TIOCDSIMICROCODE _IO('t', 85)		/* download microcode to
 						 * DSI Softmodem */
+#define TIOCPTYGRANT	_IO('t', 84)		/* grantpt(3) */
+#define TIOCPTYGNAME	_IOC(IOC_OUT, 't', 83, 128)	/* ptsname(3) */
+#define TIOCPTYUNLK	_IO('t', 82)		/* unlockpt(3) */
 
 #define	TTYDISC		0		/* termios tty line discipline */
 #define	TABLDISC	3		/* tablet discipline */
diff --git a/bsd/sys/ttydefaults.h b/bsd/sys/ttydefaults.h
index 078882c79..115d7dac4 100644
--- a/bsd/sys/ttydefaults.h
+++ b/bsd/sys/ttydefaults.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/ttydev.h b/bsd/sys/ttydev.h
index 82a1ef4b9..00b1fe0db 100644
--- a/bsd/sys/ttydev.h
+++ b/bsd/sys/ttydev.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/types.h b/bsd/sys/types.h
index d9f9d810a..45ff20e24 100644
--- a/bsd/sys/types.h
+++ b/bsd/sys/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -74,7 +80,7 @@
 
 #include <machine/endian.h>
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 typedef	unsigned char		u_char;
 typedef	unsigned short		u_short;
 typedef	unsigned int		u_int;
@@ -130,6 +136,13 @@ typedef	__darwin_ino_t		ino_t;		/* inode number */
 #define _INO_T
 #endif
 
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef	_INO64_T
+typedef	__darwin_ino64_t	ino64_t;	/* 64bit inode number */
+#define _INO64_T
+#endif
+#endif /* !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) */
+
 #ifndef _KEY_T
 #define _KEY_T
 typedef	__int32_t		key_t;		/* IPC key (for Sys V IPC) */
@@ -173,12 +186,37 @@ typedef __darwin_id_t		id_t;
 #define _ID_T
 #endif
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* Major, minor numbers, dev_t's. */
+#if defined(__cplusplus)
+/*
+ * These lowercase macros tend to match member functions in some C++ code,
+ * so for C++, we must use inline functions instead.
+ */
+
+static inline __int32_t  major(__uint32_t _x)
+{
+	return (__int32_t)(((__uint32_t)_x >> 24) & 0xff);
+}
+
+static inline __int32_t  minor(__uint32_t _x)
+{
+	return (__int32_t)((_x) & 0xffffff);
+}
+
+static inline dev_t  makedev(__uint32_t _major, __uint32_t _minor)
+{
+	return (dev_t)(((_major) << 24) | (_minor));
+}
+
+#else	/* !__cplusplus */
+
 #define	major(x)	((int32_t)(((u_int32_t)(x) >> 24) & 0xff))
 #define	minor(x)	((int32_t)((x) & 0xffffff))
 #define	makedev(x,y)	((dev_t)(((x) << 24) | (y)))
-#endif
+
+#endif	/* !__cplusplus */
+#endif	/* !_POSIX_C_SOURCE */
 
 #ifndef	_CLOCK_T
 #define	_CLOCK_T
@@ -212,24 +250,19 @@ typedef __darwin_useconds_t	useconds_t;
 typedef __darwin_suseconds_t	suseconds_t;
 #endif
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * This code is present here in order to maintain historical backward
  * compatability, and is intended to be removed at some point in the
  * future; please include <sys/select.h> instead.
  */
-#define	NBBY		8				/* bits in a byte */
-#define NFDBITS	(sizeof(__int32_t) * NBBY)		/* bits per mask */
-#define	howmany(x, y)	(((x) + ((y) - 1)) / (y))	/* # y's == x bits? */
-typedef __int32_t	fd_mask;
+#define __need_fd_set
+#include <sys/_structs.h>
 
-
-/*
- * Note:	We use _FD_SET to protect all select related
- *		types and macros
- */
-#ifndef _FD_SET
-#define	_FD_SET
+#define	NBBY		__DARWIN_NBBY		/* bits in a byte */
+#define NFDBITS		__DARWIN_NFDBITS	/* bits per mask */
+#define	howmany(x, y)	__DARWIN_howmany(x, y)	/* # y's == x bits? */
+typedef __int32_t	fd_mask;
 
 /*
  * Select uses bit masks of file descriptors in longs.  These macros
@@ -238,36 +271,25 @@ typedef __int32_t	fd_mask;
  * the default size.
  */
 #ifndef	FD_SETSIZE
-#define	FD_SETSIZE	1024
-#endif
-
-#define	__DARWIN_NBBY	8				/* bits in a byte */
-#define __DARWIN_NFDBITS	(sizeof(__int32_t) * __DARWIN_NBBY) /* bits per mask */
-#define	__DARWIN_howmany(x, y) (((x) + ((y) - 1)) / (y))	/* # y's == x bits? */
-
-__BEGIN_DECLS
-typedef	struct fd_set {
-	__int32_t	fds_bits[__DARWIN_howmany(FD_SETSIZE, __DARWIN_NFDBITS)];
-} fd_set;
-__END_DECLS
-
-#define	FD_SET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] |= (1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_CLR(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] &= ~(1<<((n) % __DARWIN_NFDBITS)))
-#define	FD_ISSET(n, p)	((p)->fds_bits[(n)/__DARWIN_NFDBITS] & (1<<((n) % __DARWIN_NFDBITS)))
-#if __GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 3
-/*
- * Use the built-in bzero function instead of the library version so that
- * we do not pollute the namespace or introduce prototype warnings.
- */
-#define	FD_ZERO(p)	__builtin_bzero(p, sizeof(*(p)))
-#else
-#define	FD_ZERO(p)	bzero(p, sizeof(*(p)))
-#endif
-#ifndef _POSIX_C_SOURCE
-#define	FD_COPY(f, t)	bcopy(f, t, sizeof(*(f)))
-#endif	/* !_POSIX_C_SOURCE */
-
-#endif	/* !_FD_SET */
+#define	FD_SETSIZE	__DARWIN_FD_SETSIZE
+#endif /* FD_SETSIZE */
+#ifndef FD_SET
+#define	FD_SET(n, p)	__DARWIN_FD_SET(n, p)
+#endif /* FD_SET */
+#ifndef FD_CLR
+#define	FD_CLR(n, p)	__DARWIN_FD_CLR(n, p)
+#endif /* FD_CLR */
+#ifndef FD_ISSET
+#define	FD_ISSET(n, p)	__DARWIN_FD_ISSET(n, p)
+#endif /* FD_ISSET */
+#ifndef FD_ZERO
+#define	FD_ZERO(p)	__DARWIN_FD_ZERO(p)
+#endif /* FD_ZERO */
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#ifndef FD_COPY
+#define	FD_COPY(f, t)	__DARWIN_FD_COPY(f, t)
+#endif /* FD_COPY */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 #if defined(__STDC__) && defined(KERNEL)
@@ -286,7 +308,7 @@ struct	tty;
 struct	uio;
 #endif
 
-#endif /* !_POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #endif /* __ASSEMBLER__ */
 
 #ifndef __POSIX_LIB__
diff --git a/bsd/sys/ubc.h b/bsd/sys/ubc.h
index 999cfb054..d22dfcaab 100644
--- a/bsd/sys/ubc.h
+++ b/bsd/sys/ubc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  *	Header file for Unified Buffer Cache.
@@ -48,8 +54,11 @@ daddr64_t	ubc_offtoblk(struct vnode *, off_t);
 off_t	ubc_getsize(struct vnode *);
 int	ubc_setsize(struct vnode *, off_t);
 
-struct ucred *ubc_getcred(struct vnode *);
-int	ubc_setcred(struct vnode *, struct proc *);
+kauth_cred_t ubc_getcred(struct vnode *);
+#ifdef __APPLE_API_OBSOLETE
+/* This API continues to exist only until <rdar://4714366> is resolved */
+int	ubc_setcred(struct vnode *, struct proc *) __deprecated;
+#endif
 struct thread;
 int	ubc_setthreadcred(struct vnode *, struct proc *, struct thread *);
 
@@ -57,17 +66,40 @@ int     ubc_sync_range(vnode_t, off_t, off_t, int);
 errno_t ubc_msync(vnode_t, off_t, off_t, off_t *, int);
 int	ubc_pages_resident(vnode_t);
 
+/* code signing */
+struct cs_blob;
+int	ubc_cs_blob_add(vnode_t, cpu_type_t, off_t, vm_address_t, vm_size_t);
+struct cs_blob *ubc_cs_blob_get(vnode_t, cpu_type_t, off_t);
+struct cs_blob *ubc_get_cs_blobs(vnode_t);
+int	ubc_cs_getcdhash(vnode_t, off_t, unsigned char *);
+
 
 /* cluster IO routines */
 int	advisory_read(vnode_t, off_t, off_t, int);
+int	advisory_read_ext(vnode_t, off_t, off_t, int, int (*)(buf_t, void *), void *, int);
 
 int	cluster_read(vnode_t, struct uio *, off_t, int);
+int	cluster_read_ext(vnode_t, struct uio *, off_t, int, int (*)(buf_t, void *), void *);
+
 int	cluster_write(vnode_t, struct uio *, off_t, off_t, off_t, off_t, int);
+int	cluster_write_ext(vnode_t, struct uio *, off_t, off_t, off_t, off_t, int, int (*)(buf_t, void *), void *);
+
 int	cluster_pageout(vnode_t, upl_t, vm_offset_t, off_t, int, off_t, int);
+int	cluster_pageout_ext(vnode_t, upl_t, vm_offset_t, off_t, int, off_t, int, int (*)(buf_t, void *), void *);
+
 int	cluster_pagein(vnode_t, upl_t, vm_offset_t, off_t, int, off_t, int);
+int	cluster_pagein_ext(vnode_t, upl_t, vm_offset_t, off_t, int, off_t, int, int (*)(buf_t, void *), void *);
+
 int	cluster_push(vnode_t, int);
-int cluster_bp(buf_t);
-void cluster_zero(upl_t, vm_offset_t, int, buf_t);
+int	cluster_push_ext(vnode_t, int, int (*)(buf_t, void *), void *);
+
+int	cluster_bp(buf_t);
+int	cluster_bp_ext(buf_t, int (*)(buf_t, void *), void *);
+
+void	cluster_zero(upl_t, vm_offset_t, int, buf_t);
+
+int	cluster_copy_upl_data(uio_t, upl_t, int, int *);
+int	cluster_copy_ubc_data(vnode_t, uio_t, int *, int);
 
 
 /* UPL routines */
@@ -80,6 +112,7 @@ int	ubc_upl_abort(upl_t, int);
 int	ubc_upl_abort_range(upl_t, upl_offset_t, upl_size_t, int);
 
 upl_page_info_t *ubc_upl_pageinfo(upl_t);
+upl_size_t ubc_upl_maxbufsize(void);
 
 __END_DECLS
 
diff --git a/bsd/sys/ubc_internal.h b/bsd/sys/ubc_internal.h
index 1362f30ee..99b9008cc 100644
--- a/bsd/sys/ubc_internal.h
+++ b/bsd/sys/ubc_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  *	File:	ubc.h
@@ -44,6 +50,8 @@
 #include <kern/locks.h>
 #include <mach/memory_object_types.h>
 
+#include <libkern/crypto/sha1.h>
+
 
 #define UBC_INFO_NULL	((struct ubc_info *) 0)
 
@@ -61,7 +69,7 @@ struct cl_extent {
 struct cl_wextent {
 	daddr64_t	b_addr;
 	daddr64_t	e_addr;
-        int		io_nocache;
+        int		io_flags;
 };
 
 struct cl_readahead {
@@ -73,7 +81,6 @@ struct cl_readahead {
 
 struct cl_writebehind {
 	lck_mtx_t	cl_lockw;
-        int		cl_hasbeenpaged;		/* if set, indicates pager has cleaned pages associated with this file */
         void	*	cl_scmap;			/* pointer to sparse cluster map */
         int		cl_scdirty;			/* number of dirty pages in the sparse cluster map */
 	int		cl_number;			/* number of packed write behind clusters currently valid */
@@ -81,40 +88,59 @@ struct cl_writebehind {
 };
 
 
+struct cs_blob {
+	struct cs_blob	*csb_next;
+	cpu_type_t	csb_cpu_type;
+	unsigned int	csb_flags;
+	off_t		csb_base_offset;
+	off_t		csb_start_offset;
+	off_t		csb_end_offset;
+	ipc_port_t	csb_mem_handle;
+	vm_size_t	csb_mem_size;
+	vm_offset_t	csb_mem_offset;
+	vm_address_t	csb_mem_kaddr;
+	unsigned char	csb_sha1[SHA1_RESULTLEN];
+};
+
 /*
  *	The following data structure keeps the information to associate
  *	a vnode to the correspondig VM objects.
  */
 struct ubc_info {
-	memory_object_t			ui_pager;	/* pager */
-	memory_object_control_t	ui_control;		/* VM control for the pager */
-	long				ui_flags;	/* flags */
-	vnode_t 			*ui_vnode;	/* The vnode for this ubc_info */
-	ucred_t	 			*ui_ucred;	/* holds credentials for NFS paging */
-	off_t				ui_size;	/* file size for the vnode */
-
-        struct	cl_readahead   *cl_rahead;		/* cluster read ahead context */
-        struct	cl_writebehind *cl_wbehind;		/* cluster write behind context */
+	memory_object_t		ui_pager;	/* pager */
+	memory_object_control_t	ui_control;	/* VM control for the pager */
+	long			ui_flags;	/* flags */
+	vnode_t 		ui_vnode;	/* vnode for this ubc_info */
+	kauth_cred_t	 	ui_ucred;	/* holds credentials for NFS paging */
+	off_t			ui_size;	/* file size for the vnode */
+
+        struct	cl_readahead   *cl_rahead;	/* cluster read ahead context */
+        struct	cl_writebehind *cl_wbehind;	/* cluster write behind context */
+
+	struct	cs_blob		*cs_blobs; 	/* for CODE SIGNING */
 };
 
 /* Defines for ui_flags */
-#define	UI_NONE			0x00000000		/* none */
-#define	UI_HASPAGER		0x00000001		/* has a pager associated */
-#define	UI_INITED		0x00000002		/* newly initialized vnode */
-#define UI_HASOBJREF	0x00000004		/* hold a reference on object */
-#define UI_WASMAPPED	0x00000008		/* vnode was mapped */
-#define	UI_ISMAPPED	0x00000010		/* vnode is currently mapped */
+#define	UI_NONE		0x00000000	/* none */
+#define	UI_HASPAGER	0x00000001	/* has a pager associated */
+#define	UI_INITED	0x00000002	/* newly initialized vnode */
+#define UI_HASOBJREF	0x00000004	/* hold a reference on object */
+#define UI_WASMAPPED	0x00000008	/* vnode was mapped */
+#define	UI_ISMAPPED	0x00000010	/* vnode is currently mapped */
+#define UI_MAPBUSY	0x00000020	/* vnode is being mapped or unmapped */
+#define UI_MAPWAITING	0x00000040	/* someone waiting for UI_MAPBUSY */
 
 /*
  * exported primitives for loadable file systems.
  */
 
 __BEGIN_DECLS
-__private_extern__ int	ubc_umount(struct mount *mp);
+__private_extern__ void ubc_init(void);
+__private_extern__ int	ubc_umount(mount_t mp);
 __private_extern__ void	ubc_unmountall(void);
-__private_extern__ memory_object_t ubc_getpager(struct vnode *);
-__private_extern__ int  ubc_map(struct vnode *, int);
-__private_extern__ int	ubc_destroy_named(struct vnode *);
+__private_extern__ memory_object_t ubc_getpager(vnode_t);
+__private_extern__ int  ubc_map(vnode_t, int);
+__private_extern__ void	ubc_destroy_named(vnode_t);
 
 /* internal only */
 __private_extern__ void	cluster_release(struct ubc_info *);
@@ -125,27 +151,21 @@ __private_extern__ void	cluster_release(struct ubc_info *);
 #define UBC_HOLDOBJECT		0x0001
 #define UBC_FOR_PAGEOUT         0x0002
 
-memory_object_control_t ubc_getobject(struct vnode *, int);
+memory_object_control_t ubc_getobject(vnode_t, int);
 
-int	ubc_info_init(struct vnode *);
-void	ubc_info_deallocate (struct ubc_info *);
+int	ubc_info_init(vnode_t);
+int	ubc_info_init_withsize(vnode_t, off_t);
+void	ubc_info_deallocate(struct ubc_info *);
 
-int	ubc_isinuse(struct vnode *, int);
+int	ubc_isinuse(vnode_t, int);
+int	ubc_isinuse_locked(vnode_t, int, int);
 
 int	ubc_page_op(vnode_t, off_t, int, ppnum_t *, int *);
 int	ubc_range_op(vnode_t, off_t, off_t, int, int *);
 
+int	ubc_getcdhash(vnode_t, off_t, unsigned char *);
 
-int	cluster_copy_upl_data(struct uio *, upl_t, int, int);
-int	cluster_copy_ubc_data(vnode_t, struct uio *, int *, int);
-
-
-int UBCINFOMISSING(vnode_t);
-int UBCINFORECLAIMED(vnode_t);
 int UBCINFOEXISTS(vnode_t);
-int UBCISVALID(vnode_t);
-int UBCINVALID(vnode_t);
-int UBCINFOCHECK(const char *, vnode_t);
 
 __END_DECLS
 
diff --git a/bsd/sys/ucontext.h b/bsd/sys/ucontext.h
index edb6499db..7ddf51a15 100644
--- a/bsd/sys/ucontext.h
+++ b/bsd/sys/ucontext.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_UCONTEXT_H_
@@ -25,34 +31,23 @@
 
 #include <sys/cdefs.h>
 #include <sys/_types.h>
-#include <machine/ucontext.h>
+
+#define __need_mcontext_t
+#define __need_stack_t
+#define __need_ucontext_t
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#if defined(__ppc__) || defined(__ppc64__)
+#define __need_mcontext64_t
+#define __need_ucontext64_t
+#endif /* __ppc__|| __ppc64__  */ 
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+#include <sys/_structs.h>
 
 #ifndef _SIGSET_T
 #define _SIGSET_T
 typedef __darwin_sigset_t	sigset_t;
 #endif
 
-#ifndef _STACK_T
-#define _STACK_T
-typedef __darwin_stack_t	stack_t;
-#endif
-
-#ifndef _UCONTEXT_T
-#define _UCONTEXT_T
-#ifndef _POSIX_C_SOURCE
-typedef struct ucontext 	ucontext_t;
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ucontext ucontext_t;
-#endif /* _POSIX_C_SOURCE */
-#endif
-
-#ifndef _POSIX_C_SOURCE
-#ifndef _UCONTEXT64_T
-#define _UCONTEXT64_T
-typedef struct ucontext64	ucontext64_t;
-#endif
-#endif /* _POSIX_C_SOURCE */
-
 #ifdef KERNEL
 #include <machine/types.h>	/* user_addr_t, user_size_t */
 
diff --git a/bsd/sys/ucred.h b/bsd/sys/ucred.h
index b43cb3f5c..786f23aee 100644
--- a/bsd/sys/ucred.h
+++ b/bsd/sys/ucred.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)ucred.h	8.4 (Berkeley) 1/9/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _SYS_UCRED_H_
 #define	_SYS_UCRED_H_
@@ -63,19 +75,24 @@
 #include <sys/param.h>
 #include <bsm/audit.h>
 
+struct label;
+
 #ifdef __APPLE_API_UNSTABLE
 
 /*
  * In-kernel credential structure.
  *
  * Note that this structure should not be used outside the kernel, nor should
- * it or copies of it be exported outside.  
+ * it or copies of it be exported outside.
  */
 struct ucred {
 	TAILQ_ENTRY(ucred)	cr_link; /* never modify this without KAUTH_CRED_HASH_LOCK */
 	u_long	cr_ref;			/* reference count */
 	
-	/* credential hash depends on everything from this point on (see kauth_cred_get_hashkey) */
+	/*
+	 * The credential hash depends on everything from this point on
+	 * (see kauth_cred_get_hashkey)
+	 */
 	uid_t	cr_uid;			/* effective user id */
 	uid_t	cr_ruid;		/* real user id */
 	uid_t	cr_svuid;		/* saved user id */
@@ -83,10 +100,28 @@ struct ucred {
 	gid_t	cr_groups[NGROUPS];	/* advisory group list */
 	gid_t	cr_rgid;		/* real group id */
 	gid_t	cr_svgid;		/* saved group id */
-	uid_t	cr_gmuid;		/* user id for group membership purposes */
+	uid_t	cr_gmuid;		/* UID for group membership purposes */
 	struct auditinfo cr_au;		/* user auditing data */
+	struct label	*cr_label;	/* MAC label */
+
+	int	cr_flags;		/* flags on credential */
+	/* 
+	 * NOTE: If anything else (besides the flags)
+	 * added after the label, you must change
+	 * kauth_cred_find().
+	 */
 };
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
 typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
+
+/*
+ * Credential flags that can be set on a credential
+ */
+#define CRF_NOMEMBERD	0x00000001	/* memberd opt out by setgroups() */
+#define CRF_MAC_ENFORCE 0x00000002	/* force entry through MAC Framework */
+                                  	/* also forces credential cache miss */
 
 /*
  * This is the external representation of struct ucred.
@@ -110,6 +145,8 @@ struct xucred {
 __BEGIN_DECLS
 int		crcmp(kauth_cred_t cr1, kauth_cred_t cr2);
 int		suser(kauth_cred_t cred, u_short *acflag);
+int		is_suser(void);
+int		is_suser1(void);
 int		set_security_token(struct proc * p);
 void		cru2x(kauth_cred_t cr, struct xucred *xcr);
 __END_DECLS
diff --git a/bsd/sys/uio.h b/bsd/sys/uio.h
index b133e43a0..059c5fa5a 100644
--- a/bsd/sys/uio.h
+++ b/bsd/sys/uio.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -88,7 +94,7 @@ struct iovec {
 #endif
 
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * IO direction for uio_t.
  *	UIO_READ - data moves into iovec(s) associated with uio_t
@@ -251,8 +257,8 @@ user_size_t uio_curriovlen( uio_t a_uio );
 #define UIO_MAXIOV	1024		/* max 1K of iov's */
 #define UIO_SMALLIOV	8		/* 8 on stack, else malloc */
 
-extern int uiomove(caddr_t cp, int n, struct uio *uio);
-extern int uiomove64(unsigned long long cp, int n, struct uio *uio);
+extern int uiomove(const char * cp, int n, struct uio *uio);
+extern int uiomove64(const unsigned long long cp, int n, struct uio *uio);
 extern int ureadc(int c, struct uio *uio);
 extern int uwritec(struct uio *uio);
 __END_DECLS
@@ -262,8 +268,8 @@ __END_DECLS
 #ifndef	KERNEL
 
 __BEGIN_DECLS
-ssize_t	readv(int, const struct iovec *, int);
-ssize_t	writev(int, const struct iovec *, int);
+ssize_t	readv(int, const struct iovec *, int) __DARWIN_ALIAS_C(readv);
+ssize_t	writev(int, const struct iovec *, int) __DARWIN_ALIAS_C(writev);
 __END_DECLS
 #endif /* !KERNEL */
 
diff --git a/bsd/sys/uio_internal.h b/bsd/sys/uio_internal.h
index a2a7cc0f5..db7c5790f 100644
--- a/bsd/sys/uio_internal.h
+++ b/bsd/sys/uio_internal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/sys/un.h b/bsd/sys/un.h
index d200eff87..92017608d 100644
--- a/bsd/sys/un.h
+++ b/bsd/sys/un.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -76,10 +82,10 @@ struct	sockaddr_un {
 	char		sun_path[104];	/* [XSI] path name (gag) */
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* Socket options. */
 #define LOCAL_PEERCRED          0x001           /* retrieve peer credentails */
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 
 #ifdef KERNEL
@@ -87,6 +93,7 @@ struct	sockaddr_un {
 __BEGIN_DECLS
 struct mbuf;
 struct socket;
+struct sockopt;
 
 int	uipc_usrreq(struct socket *so, int req, struct mbuf *m,
 		struct mbuf *nam, struct mbuf *control);
@@ -94,17 +101,17 @@ int	uipc_ctloutput (struct socket *so, struct sockopt *sopt);
 int	unp_connect2(struct socket *so, struct socket *so2);
 void	unp_dispose(struct mbuf *m);
 int	unp_externalize(struct mbuf *rights);
-void	unp_init(void);
+void	unp_init(void) __attribute__((section("__TEXT, initcode")));
 extern	struct pr_usrreqs uipc_usrreqs;
 __END_DECLS
 #endif /* PRIVATE */
 #else /* !KERNEL */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* actual length of an initialized sockaddr_un */
 #define SUN_LEN(su) \
 	(sizeof(*(su)) - sizeof((su)->sun_path) + strlen((su)->sun_path))
-#endif	/* !_POSIX_C_SOURCE */
+#endif	/* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 
 #endif /* KERNEL */
 
diff --git a/bsd/sys/unistd.h b/bsd/sys/unistd.h
index 74fcc4411..d80b3bbd3 100644
--- a/bsd/sys/unistd.h
+++ b/bsd/sys/unistd.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,6 +64,8 @@
 #ifndef _SYS_UNISTD_H_
 #define	_SYS_UNISTD_H_
 
+#include <sys/cdefs.h>
+
 /*
  * Although we have saved user/group IDs, we do not use them in setuid
  * as described in POSIX 1003.1, because the feature does not work for
@@ -85,7 +93,7 @@
 #define	W_OK		(1<<1)	/* test for write permission */
 #define	R_OK		(1<<2)	/* test for read permission */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /*
  * Extended access functions.
  * Note that we depend on these matching the definitions in sys/kauth.h,
@@ -119,16 +127,16 @@
 #define	SEEK_END	2	/* set file offset to EOF plus offset */
 #endif	/* !SEEK_SET */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 /* whence values for lseek(2); renamed by POSIX 1003.1 */
 #define	L_SET		SEEK_SET
 #define	L_INCR		SEEK_CUR
 #define	L_XTND		SEEK_END
 #endif
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 struct accessx_descriptor {
-	unsigned ad_name_offset;
+	unsigned int ad_name_offset;
 	int ad_flags;
 	int ad_pad[2];
 };
@@ -147,7 +155,7 @@ struct accessx_descriptor {
 #define	_PC_NO_TRUNC		 8
 #define	_PC_VDISABLE		 9
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #define	_PC_NAME_CHARS_MAX	 10
 #define	_PC_CASE_SENSITIVE		 11
 #define	_PC_CASE_PRESERVING		 12
@@ -155,6 +163,18 @@ struct accessx_descriptor {
 #define _PC_AUTH_OPAQUE_NP      14
 #endif
 
+#define	_PC_2_SYMLINKS		15	/* Symlink supported in directory */
+#define	_PC_ALLOC_SIZE_MIN	16	/* Minimum storage actually allocated */
+#define	_PC_ASYNC_IO		17	/* Async I/O [AIO] supported? */
+#define	_PC_FILESIZEBITS	18	/* # of bits to represent file size */
+#define	_PC_PRIO_IO		19	/* Priority I/O [PIO] supported? */
+#define	_PC_REC_INCR_XFER_SIZE	20	/* Recommended increment for next two */
+#define	_PC_REC_MAX_XFER_SIZE	21	/* Recommended max file transfer size */
+#define	_PC_REC_MIN_XFER_SIZE	22	/* Recommended min file transfer size */
+#define	_PC_REC_XFER_ALIGN	23	/* Recommended buffer alignment */
+#define	_PC_SYMLINK_MAX		24	/* Max # of bytes in symlink name */
+#define	_PC_SYNC_IO		25	/* Sync I/O [SIO] supported? */
+
 /* configurable system strings */
 #define	_CS_PATH		 1
 
diff --git a/bsd/sys/unpcb.h b/bsd/sys/unpcb.h
index 2a5f8af33..bc2039358 100644
--- a/bsd/sys/unpcb.h
+++ b/bsd/sys/unpcb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1989, 1993
@@ -90,25 +96,42 @@
 
 typedef u_quad_t        unp_gen_t;
 
+#if defined(__LP64__)
+struct _unpcb_list_entry {
+    u_int32_t   le_next; 
+    u_int32_t   le_prev;
+}; 
+#define _UCPCB_LIST_HEAD(name, type)	\
+struct name {				\
+	u_int32_t	lh_first;	\
+};
+#define _UNPCB_LIST_ENTRY(x)		struct _unpcb_list_entry
+#define _UNPCB_PTR(x)			u_int32_t
+#else 
+#define _UCPCB_LIST_HEAD(name, type)	LIST_HEAD(name, type)
+#define _UNPCB_LIST_ENTRY(x)		LIST_ENTRY(x)
+#define _UNPCB_PTR(x)			x
+#endif
+
 #ifdef PRIVATE
-LIST_HEAD(unp_head, unpcb);
+_UCPCB_LIST_HEAD(unp_head, unpcb);
 #ifdef KERNEL
 #define sotounpcb(so)   ((struct unpcb *)((so)->so_pcb))
 
 struct	unpcb {
-	LIST_ENTRY(unpcb) unp_link; 	/* glue on list of all PCBs */
-	struct	socket *unp_socket;	/* pointer back to socket */
-	struct	vnode *unp_vnode;	/* if associated with file */
-	ino_t	unp_ino;		/* fake inode number */
-	struct	unpcb *unp_conn;	/* control block of connected socket */
-	struct	unp_head unp_refs;	/* referencing socket linked list */
+	LIST_ENTRY(unpcb) unp_link;	/* glue on list of all PCBs */
+	struct socket	*unp_socket;	/* pointer back to socket */
+	struct vnode	*unp_vnode;	/* if associated with file */
+	ino_t			unp_ino;	/* fake inode number */
+	struct unpcb	*unp_conn;	/* control block of connected socket */
+	struct unp_head	unp_refs;	/* referencing socket linked list */
 	LIST_ENTRY(unpcb) unp_reflink;	/* link in unp_refs list */
-	struct	sockaddr_un *unp_addr;	/* bound address of socket */
-	int	unp_cc;			/* copy of rcv.sb_cc */
-	int	unp_mbcnt;		/* copy of rcv.sb_mbcnt */
-	unp_gen_t unp_gencnt;		/* generation count of this instance */
-	int	unp_flags;		/* flags */
-	struct	xucred unp_peercred;	/* peer credentials, if applicable */
+	struct sockaddr_un *unp_addr;	/* bound address of socket */
+	int		unp_cc;		/* copy of rcv.sb_cc */
+	int		unp_mbcnt;	/* copy of rcv.sb_mbcnt */
+	unp_gen_t	unp_gencnt;	/* generation count of this instance */
+	int		unp_flags;	/* flags */
+	struct xucred	unp_peercred;	/* peer credentials, if applicable */
 };
 #endif /* KERNEL */
 
@@ -134,46 +157,52 @@ struct  unpcb_compat {
 #define unpcb_compat unpcb
 struct	unpcb {
 #endif /* KERNEL */
-        LIST_ENTRY(unpcb_compat) unp_link;     /* glue on list of all PCBs */
-        struct  socket *unp_socket;     /* pointer back to socket */
-        struct  vnode *unp_vnode;       /* if associated with file */
-        ino_t   unp_ino;                /* fake inode number */
-        struct  unpcb_compat *unp_conn;        /* control block of connected socket */
-        struct  unp_head unp_refs;      /* referencing socket linked list */
-        LIST_ENTRY(unpcb_compat) unp_reflink;  /* link in unp_refs list */
-        struct  sockaddr_un *unp_addr;  /* bound address of socket */
-        int     unp_cc;                 /* copy of rcv.sb_cc */
-        int     unp_mbcnt;              /* copy of rcv.sb_mbcnt */
-        unp_gen_t unp_gencnt;           /* generation count of this instance */
+	_UNPCB_LIST_ENTRY(unpcb_compat)	unp_link;	/* glue on list of all PCBs */
+	_UNPCB_PTR(struct socket *)	unp_socket;	/* pointer back to socket */
+	_UNPCB_PTR(struct vnode *)	unp_vnode;	/* if associated with file */
+	ino_t				unp_ino;	/* fake inode number */
+	_UNPCB_PTR(struct unpcb_compat *) unp_conn;	/* control block of connected socket */
+	struct unp_head			unp_refs;	/* referencing socket linked list */
+	_UNPCB_LIST_ENTRY(unpcb_compat) unp_reflink;	/* link in unp_refs list */
+	_UNPCB_PTR(struct sockaddr_un *) unp_addr;	/* bound address of socket */
+	int				unp_cc;		/* copy of rcv.sb_cc */
+	int				unp_mbcnt;	/* copy of rcv.sb_mbcnt */
+	unp_gen_t			unp_gencnt;	/* generation count of this instance */
 };
 
 /* Hack alert -- this structure depends on <sys/socketvar.h>. */
 #ifdef  _SYS_SOCKETVAR_H_
+
+#pragma pack(4)
+
 struct  xunpcb {
-        size_t  xu_len;                 /* length of this structure */
-        struct  unpcb_compat *xu_unpp;         /* to help netstat, fstat */
-        struct  unpcb_compat xu_unp;           /* our information */
-        union {
-                struct  sockaddr_un xuu_addr;   /* our bound address */
-                char    xu_dummy1[256];
-        } xu_au;
+	u_int32_t			xu_len;		/* length of this structure */
+	_UNPCB_PTR(struct unpcb_compat *) xu_unpp;	/* to help netstat, fstat */
+	struct unpcb_compat		xu_unp;		/* our information */
+	union {
+		struct sockaddr_un	xuu_addr;	/* our bound address */
+		char			xu_dummy1[256];
+	} xu_au;
 #define xu_addr xu_au.xuu_addr
-        union {
-                struct  sockaddr_un xuu_caddr; /* their bound address */
-                char    xu_dummy2[256];
-        } xu_cau;
+	union {
+		struct sockaddr_un	xuu_caddr;	/* their bound address */
+		char			xu_dummy2[256];
+	} xu_cau;
 #define xu_caddr xu_cau.xuu_caddr
-        struct  xsocket xu_socket;
-        u_quad_t        xu_alignment_hack;
+	struct xsocket			xu_socket;
+	u_quad_t			xu_alignment_hack;
 };
+
+#pragma pack()
+
 #endif /* _SYS_SOCKETVAR_H_ */
 #endif /* PRIVATE */
 
 struct	xunpgen {
-	size_t	xug_len;
-	u_int	xug_count;
-	unp_gen_t xug_gen;
-	so_gen_t xug_sogen;
+	u_int32_t	xug_len;
+	u_int		xug_count;
+	unp_gen_t	xug_gen;
+	so_gen_t	xug_sogen;
 };
 
 #endif /* _SYS_UNPCB_H_ */
diff --git a/bsd/sys/user.h b/bsd/sys/user.h
index a304b3d0d..89dd1cd4f 100644
--- a/bsd/sys/user.h
+++ b/bsd/sys/user.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995, 1997 Apple Computer, Inc. All Rights Reserved */
 /*
@@ -78,35 +84,46 @@
 #ifdef __APPLE_API_PRIVATE
 #include <sys/eventvar.h>
 
+/*
+ * VFS context structure (part of uthread)
+ */
+struct vfs_context {
+	thread_t	vc_thread;		/* pointer to Mach thread */
+	kauth_cred_t	vc_ucred;		/* per thread credential */
+};
+
+/* XXX Deprecated: xnu source compatability */
+#define uu_ucred	uu_context.vc_ucred
+
+#ifdef BSD_KERNEL_PRIVATE
 /*
  *	Per-thread U area.
  */
+ 
 struct uthread {
 	/* syscall parameters, results and catches */
-	u_int64_t uu_arg[8]; /* arguments to current system call */
+	user_addr_t uu_arg[8]; /* arguments to current system call */
 	int	*uu_ap;			/* pointer to arglist */
-        int uu_rval[2];
+    int uu_rval[2];
 
 	/* thread exception handling */
 	int	uu_exception;
-	int	uu_code;			/* ``code'' to trap */
-	int	uu_subcode;
+	mach_exception_code_t uu_code;	/* ``code'' to trap */
+	mach_exception_subcode_t uu_subcode;
 	char uu_cursig;			/* p_cursig for exc. */
-	/* support for select - across system calls */
+	/* support for syscalls which use continuations */
 	struct _select {
-		u_int32_t	*ibits, *obits; /* bits to select on */
-		uint	nbytes;	/* number of bytes in ibits and obits */
-		wait_queue_set_t wqset;	 /* cached across select calls */
-		size_t allocsize;		 /* ...size of select cache */
-		u_int64_t abstime;
-		int poll;
-		int error;
-		int count;
-		char * wql;
+			u_int32_t	*ibits, *obits; /* bits to select on */
+			uint	nbytes;	/* number of bytes in ibits and obits */
+			u_int64_t abstime;
+			int poll;
+			int error;
+			int count;
+			int kfcount;
+			char * wql;
 	} uu_select;			/* saved state for select() */
-	/* to support continuations */
+	/* to support kevent continuations */
 	union {
-		int uu_nfs_myiod;    /* saved state for nfsd */
 		struct _kevent_scan {
 			kevent_callback_t call; /* per-event callback */
 			kevent_continue_t cont; /* whole call continuation */
@@ -119,44 +136,95 @@ struct uthread {
 			int fd;			 /* filedescriptor for kq */
 			register_t *retval;	 /* place to store return val */
 			user_addr_t eventlist;	 /* user-level event list address */
+			size_t eventsize;	/* user-level event size (LP64) */
 			int eventcount;	 	/* user-level event count */
 			int eventout;		 /* number of events output */
 		} ss_kevent;			 /* saved state for kevent() */
-	} uu_state;
+	} uu_kevent;
+	struct _kauth {
+		user_addr_t message;	/* message in progress */
+	} uu_kauth;
   /* internal support for continuation framework */
     int (*uu_continuation)(int);
     int uu_pri;
     int uu_timo;
+	caddr_t uu_wchan;			/* sleeping thread wait channel */
+	const char *uu_wmesg;			/* ... wait message */
 	int uu_flag;
+	int uu_iopol_disk;			/* disk I/O policy */
 	struct proc * uu_proc;
 	void * uu_userstate;
+	wait_queue_set_t uu_wqset;			/* cached across select calls */
+	size_t uu_allocsize;				/* ...size of select cache */
 	sigset_t uu_siglist;				/* signals pending for the thread */
 	sigset_t  uu_sigwait;				/*  sigwait on this thread*/
 	sigset_t  uu_sigmask;				/* signal mask for the thread */
 	sigset_t  uu_oldmask;				/* signal mask saved before sigpause */
-	thread_t uu_act;
+	struct vfs_context uu_context;			/* thread + cred */
 	sigset_t  uu_vforkmask;				/* saved signal mask during vfork */
 
 	TAILQ_ENTRY(uthread) uu_list;		/* List of uthreads in proc */
 
 	struct kaudit_record 		*uu_ar;		/* audit record */
 	struct task*	uu_aio_task;			/* target task for async io */
-
+     
   /* network support for dlil layer locking */
 	u_int32_t	dlil_incremented_read;
 	lck_mtx_t	*uu_mtx;
 
-        int		uu_lowpri_delay;
+        int		uu_lowpri_window;
 
-	struct ucred	*uu_ucred;		/* per thread credential */
+	struct user_sigaltstack uu_sigstk;
         int		uu_defer_reclaims;
         vnode_t		uu_vreclaims;
-		struct user_sigaltstack uu_sigstk;
+	int		uu_notrigger;		/* XXX - flag for autofs */
+	vnode_t		uu_cdir;		/* per thread CWD */
+	int		uu_dupfd;		/* fd in fdesc_open/dupfdopen */
+
 #ifdef JOE_DEBUG
         int		uu_iocount;
         int		uu_vpindex;
         void 	*	uu_vps[32];
 #endif
+#if CONFIG_DTRACE
+	uint32_t	t_dtrace_errno; /* Most recent errno */
+        uint8_t         t_dtrace_stop;  /* indicates a DTrace-desired stop */
+        uint8_t         t_dtrace_sig;   /* signal sent via DTrace's raise() */
+
+        union __tdu {
+                struct __tds {
+                        uint8_t _t_dtrace_on;   /* hit a fasttrap tracepoint */
+                        uint8_t _t_dtrace_step; /* about to return to kernel */
+                        uint8_t _t_dtrace_ret;  /* handling a return probe */
+                        uint8_t _t_dtrace_ast;  /* saved ast flag */
+#if __sol64 || defined(__APPLE__)
+                        uint8_t _t_dtrace_reg;  /* modified register */
+#endif
+                } _tds;
+                unsigned long _t_dtrace_ft;           /* bitwise or of these flags */
+        } _tdu;
+#define t_dtrace_ft     _tdu._t_dtrace_ft
+#define t_dtrace_on     _tdu._tds._t_dtrace_on
+#define t_dtrace_step   _tdu._tds._t_dtrace_step
+#define t_dtrace_ret    _tdu._tds._t_dtrace_ret
+#define t_dtrace_ast    _tdu._tds._t_dtrace_ast
+#if __sol64 || defined(__APPLE__)
+#define t_dtrace_reg    _tdu._tds._t_dtrace_reg
+#endif
+
+        user_addr_t	t_dtrace_pc;    /* DTrace saved pc from fasttrap */
+        user_addr_t	t_dtrace_npc;   /* DTrace next pc from fasttrap */
+        user_addr_t	t_dtrace_scrpc; /* DTrace per-thread scratch location */
+        user_addr_t	t_dtrace_astpc; /* DTrace return sequence location */
+
+	struct dtrace_ptss_page_entry*	t_dtrace_scratch; /* scratch space entry */
+
+#if __sol64 || defined(__APPLE__)
+        uint64_t        t_dtrace_regv;  /* DTrace saved reg from fasttrap */
+#endif
+#endif /* CONFIG_DTRACE */
+	void *		uu_threadlist;
+	mount_t		v_mount;
 };
 
 typedef struct uthread * uthread_t;
@@ -168,12 +236,19 @@ typedef struct uthread * uthread_t;
 #define UT_CANCEL	0x00000008             /* thread marked for cancel */
 #define UT_CANCELED	0x00000010            /* thread cancelled */
 #define UT_CANCELDISABLE 0x00000020            /* thread cancel disabled */
-#define UT_ALTSTACK 0x00000040  /* this thread has alt stack for signals */
+#define	UT_ALTSTACK	0x00000040	/* this thread has alt stack for signals */
+#define UT_THROTTLE_IO	0x00000080	/* this thread issues throttle I/O */
+#define UT_PASSIVE_IO	0x00000100	/* this thread issues passive I/O */
+#define UT_PROCEXIT	0x00000200	/* this thread completed the  proc exit */
+#define UT_RAGE_VNODES	0x00000400	/* rapid age any vnodes created by this thread */	
+#define UT_BACKGROUND	0x00000800	/* this thread is in background state */	
 
 #define	UT_VFORK	0x02000000	/* thread has vfork children */
 #define	UT_SETUID	0x04000000	/* thread is settugid() */
 #define UT_WASSETUID	0x08000000	/* thread was settugid() (in vfork) */
 
+#endif /* BSD_KERNEL_PRIVATE */
+
 #endif /* __APPLE_API_PRIVATE */
 
 #endif	/* KERNEL */
diff --git a/bsd/sys/utfconv.h b/bsd/sys/utfconv.h
index 1feafbf5b..bff9d0667 100644
--- a/bsd/sys/utfconv.h
+++ b/bsd/sys/utfconv.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_UTFCONV_H_
@@ -28,24 +34,159 @@
 
 #ifdef KERNEL
 #ifdef __APPLE_API_UNSTABLE
+
 /*
  * UTF-8 encode/decode flags
  */
-#define	UTF_REVERSE_ENDIAN	0x01	/* reverse UCS-2 byte order */
-#define UTF_NO_NULL_TERM	0x02	/* do not add null termination */
-#define	UTF_DECOMPOSED		0x04	/* generate fully decomposed UCS-2 */
-#define	UTF_PRECOMPOSED		0x08	/* generate precomposed UCS-2 */
+#define	UTF_REVERSE_ENDIAN   0x0001   /* reverse UCS-2 byte order */
+#define UTF_NO_NULL_TERM     0x0002   /* do not add null termination */
+#define	UTF_DECOMPOSED       0x0004   /* generate fully decomposed UCS-2 */
+#define	UTF_PRECOMPOSED      0x0008   /* generate precomposed UCS-2 */
+#define UTF_ESCAPE_ILLEGAL   0x0010   /* escape illegal UTF-8 */
+#define UTF_SFM_CONVERSIONS  0x0020   /* Use SFM mappings for illegal NTFS chars */
+
+#define UTF_BIG_ENDIAN       \
+        ((BYTE_ORDER == BIG_ENDIAN) ? 0 : UTF_REVERSE_ENDIAN)
+
+#define UTF_LITTLE_ENDIAN    \
+        ((BYTE_ORDER == LITTLE_ENDIAN) ? 0 : UTF_REVERSE_ENDIAN)
 
 __BEGIN_DECLS
-size_t	utf8_encodelen(const u_int16_t *, size_t, u_int16_t, int);
 
-int	utf8_encodestr(const u_int16_t *, size_t, u_int8_t *, size_t *,
-		size_t, u_int16_t, int);
+/*
+ * utf8_encodelen - Calculate the UTF-8 encoding length
+ *
+ * This function takes an Unicode input string, ucsp, of ucslen bytes
+ * and calculates the size of the UTF-8 output in bytes (not including
+ * a NULL termination byte). The string must reside in kernel memory.
+ *
+ * FLAGS
+ *    UTF_REVERSE_ENDIAN:  Unicode byte order is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_DECOMPOSED:  assume fully decomposed output
+ *
+ * ERRORS
+ *    None
+ */
+size_t
+utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
+               int flags);
+
+
+/*
+ * utf8_encodestr - Encodes a Unicode string into UTF-8
+ *
+ * This function takes an Unicode input string, ucsp, of ucslen bytes
+ * and produces the UTF-8 output into a buffer of buflen bytes pointed
+ * to by utf8p. The size of the output in bytes (not including a NULL
+ * termination byte) is returned in utf8len. The UTF-8 string output
+ * is NULL terminated. Both buffers must reside in kernel memory.
+ *
+ * If '/' chars are possible in the Unicode input then an alternate
+ * (replacement) char must be provided in altslash.
+ *
+ * FLAGS
+ *    UTF_REVERSE_ENDIAN:  Unicode byte order is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_NO_NULL_TERM:  do not add null termination to output string
+ *
+ *    UTF_DECOMPOSED:  generate fully decomposed output
+ *
+ * ERRORS
+ *    ENAMETOOLONG:  output did not fit; only utf8len bytes were encoded
+ *
+ *    EINVAL:  illegal Unicode char encountered
+ */
+int
+utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
+               size_t * utf8len, size_t buflen, u_int16_t altslash, int flags);
 
-int	utf8_decodestr(const u_int8_t *, size_t, u_int16_t *,size_t *,
-		size_t, u_int16_t, int);
 
-int	utf8_validatestr(const u_int8_t*, size_t);
+/*
+ * utf8_decodestr - Decodes a UTF-8 string into Unicode
+ *
+ * This function takes an UTF-8 input string, utf8p, of utf8len bytes
+ * and produces the Unicode output into a buffer of buflen bytes pointed
+ * to by ucsp. The size of the output in bytes (not including a NULL
+ * termination byte) is returned in ucslen. Both buffers must reside
+ * in kernel memory.
+ *
+ * If '/' chars are allowed in the Unicode output then an alternate
+ * (replacement) char must be provided in altslash.
+ *
+ * FLAGS
+ *    UTF_REV_ENDIAN:  Unicode byte order is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_DECOMPOSED:  generate fully decomposed output (NFD)
+ *
+ *    UTF_PRECOMPOSED:  generate precomposed output (NFC)
+ *
+ *    UTF_ESCAPE_ILLEGAL:  percent escape any illegal UTF-8 input
+ *
+ * ERRORS
+ *    ENAMETOOLONG:  output did not fit; only ucslen bytes were decoded.
+ *
+ *    EINVAL:  illegal UTF-8 sequence encountered.
+ */
+int
+utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
+               size_t *ucslen, size_t buflen, u_int16_t altslash, int flags);
+
+
+/*
+ * utf8_normalizestr - Normalize a UTF-8 string (NFC or NFD)
+ *
+ * This function takes an UTF-8 input string, instr, of inlen bytes
+ * and produces normalized UTF-8 output into a buffer of buflen bytes
+ * pointed to by outstr. The size of the output in bytes (not including
+ * a NULL termination byte) is returned in outlen. In-place conversions
+ * are not supported (i.e. instr != outstr).  Both buffers must reside
+ * in kernel memory.
+ *
+ * FLAGS
+ *    UTF_DECOMPOSED:  output string will be fully decomposed (NFD)
+ *
+ *    UTF_PRECOMPOSED:  output string will be precomposed (NFC)
+ *
+ *    UTF_NO_NULL_TERM:  do not add null termination to output string
+ *
+ *    UTF_ESCAPE_ILLEGAL:  percent escape any illegal UTF-8 input
+ *
+ * ERRORS
+ *    ENAMETOOLONG:  output did not fit or input exceeded MAXPATHLEN bytes
+ *
+ *    EINVAL:  illegal UTF-8 sequence encountered or invalid flags
+ */
+int
+utf8_normalizestr(const u_int8_t* instr, size_t inlen, u_int8_t* outstr,
+                  size_t *outlen, size_t buflen, int flags);
+
+
+/*
+ * utf8_validatestr - validates a UTF-8 string
+ *
+ * This function takes an UTF-8 input string, utf8p, of utf8len bytes
+ * and determines if its valid UTF-8.  The string must reside in kernel
+ * memory.
+ *
+ * ERRORS
+ *    EINVAL:  illegal UTF-8 sequence encountered.
+ */
+int
+utf8_validatestr(const u_int8_t* utf8p, size_t utf8len);
+
 
 __END_DECLS
 
diff --git a/bsd/sys/utsname.h b/bsd/sys/utsname.h
index 8f3d2e2f8..5143e5085 100644
--- a/bsd/sys/utsname.h
+++ b/bsd/sys/utsname.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright 1993,1995 NeXT Computer Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/ux_exception.h b/bsd/sys/ux_exception.h
index 2f4372642..d7c3ac3bc 100644
--- a/bsd/sys/ux_exception.h
+++ b/bsd/sys/ux_exception.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -57,8 +63,9 @@
 
 extern mach_port_name_t			ux_exception_port;
 
-boolean_t	machine_exception(int exception, int code, int subcode,
-	        int *unix_signal, int *unix_code);
+boolean_t	machine_exception(int exception, mach_exception_code_t code, 
+			mach_exception_subcode_t subcode,
+			int *unix_signal, mach_exception_code_t *unix_code);
 void	ux_handler_init(void);
 
 #endif /* __APPLE_API_PRIVATE */
diff --git a/bsd/sys/vadvise.h b/bsd/sys/vadvise.h
index 09ee127ca..7148b55b7 100644
--- a/bsd/sys/vadvise.h
+++ b/bsd/sys/vadvise.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/vcmd.h b/bsd/sys/vcmd.h
index 9e5b596cd..435299f22 100644
--- a/bsd/sys/vcmd.h
+++ b/bsd/sys/vcmd.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/version.h b/bsd/sys/version.h
deleted file mode 100644
index a11538fb6..000000000
--- a/bsd/sys/version.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1987 Carnegie-Mellon University
- * All rights reserved.  The CMU software License Agreement specifies
- * the terms and conditions for use and redistribution.
- */
-/*
- *	File:	sys/version.h
- *
- * HISTORY
- * 29-Oct-86  Avadis Tevanian (avie) at Carnegie-Mellon University
- *	Created.
- */ 
-
-/*
- *	Each kernel has a major and minor version number.  Changes in
- *	the major number in general indicate a change in exported features.
- *	Changes in minor number usually correspond to internal-only
- *	changes that the user need not be aware of (in general).  These
- *	values are stored at boot time in the machine_info strucuture and
- *	can be obtained by user programs with the host_info kernel call.
- *	This mechanism is intended to be the formal way for Mach programs
- *	to provide for backward compatibility in future releases.
- *
- *	Following is an informal history of the numbers:
- *
- *	20-Mar-1998	Umesh Vaishampayan
- *		MacOSX DR2
- *
- *	28-Sep-94 ?
- *		NEXTSTEP Release 4.0.
- *
- *	03-Sep-91 Doug Mitchell
- *		Major 3 for NeXT release 3.0.
- *
- *	04-Mar-90 Avadis Tevanian, Jr.
- *		Major 2, minor 0 for NeXT release 2.0.
- *
- *	11-May-89 Avadis Tevanian, Jr.
- *		Advance version to major 1, minor 0 to conform to NeXT
- *		release 1.0.
- *
- *	05-December-88 Avadis Tevanian, Jr.
- *		Aborted previous numbering, set major to 0, minor to 9
- *		to conform to NeXT's 0.9 release.
- *
- *	25-March-87  Avadis Tevanian, Jr.
- *		Created version numbering scheme.  Started with major 1,
- *		minor 0.
- */
-#ifndef _SYS_VERSION_H_
-#define _SYS_VERSION_H_
-
-#define KERNEL_MAJOR_VERSION	10
-#define KERNEL_MINOR_VERSION	0
-
-#endif /* ! _SYS_VERSION_H_ */
diff --git a/bsd/sys/vfs_context.h b/bsd/sys/vfs_context.h
index 931271de2..16453bb7a 100644
--- a/bsd/sys/vfs_context.h
+++ b/bsd/sys/vfs_context.h
@@ -4,11 +4,13 @@
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <sys/kernel_types.h>
+#include <kern/thread.h>
+#include <sys/user.h>
 #include <stdint.h>
 
-struct vfs_context {
-	proc_t   vc_proc;
-	ucred_t  vc_ucred;
-};
+/*
+ * XXX should go away; pulls in vfs_context structure definition from
+ * XXX <sys/user.h>
+ */
 
 #endif /* !_BSD_SYS_VFS_CONTEXT_H_ */
diff --git a/bsd/sys/vlimit.h b/bsd/sys/vlimit.h
index 5c0f0b617..48348249c 100644
--- a/bsd/sys/vlimit.h
+++ b/bsd/sys/vlimit.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
diff --git a/bsd/sys/vm.h b/bsd/sys/vm.h
index 1e171d9e1..77a6b7b57 100644
--- a/bsd/sys/vm.h
+++ b/bsd/sys/vm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -95,7 +101,6 @@ struct vmspace {
 };
 
 #ifdef KERNEL
-// LP64todo - should this move?
 /* LP64 version of vmspace.  all pointers 
  * grow when we're dealing with a 64-bit process.
  * WARNING - keep in sync with vmspace
diff --git a/bsd/sys/vmmeter.h b/bsd/sys/vmmeter.h
index caa2fbc15..f1041c842 100644
--- a/bsd/sys/vmmeter.h
+++ b/bsd/sys/vmmeter.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -68,47 +74,47 @@ struct vmmeter {
 	/*
 	 * General system activity.
 	 */
-	u_int v_swtch;		/* context switches */
-	u_int v_trap;		/* calls to trap */
-	u_int v_syscall;	/* calls to syscall() */
-	u_int v_intr;		/* device interrupts */
-	u_int v_soft;		/* software interrupts */
-	u_int v_faults;		/* total faults taken */
+	unsigned int v_swtch;		/* context switches */
+	unsigned int v_trap;		/* calls to trap */
+	unsigned int v_syscall;	/* calls to syscall() */
+	unsigned int v_intr;		/* device interrupts */
+	unsigned int v_soft;		/* software interrupts */
+	unsigned int v_faults;		/* total faults taken */
 	/*
 	 * Virtual memory activity.
 	 */
-	u_int v_lookups;	/* object cache lookups */
-	u_int v_hits;		/* object cache hits */
-	u_int v_vm_faults;	/* number of address memory faults */
-	u_int v_cow_faults;	/* number of copy-on-writes */
-	u_int v_swpin;		/* swapins */
-	u_int v_swpout;		/* swapouts */
-	u_int v_pswpin;		/* pages swapped in */
-	u_int v_pswpout;	/* pages swapped out */
-	u_int v_pageins;	/* number of pageins */
-	u_int v_pageouts;	/* number of pageouts */
-	u_int v_pgpgin;		/* pages paged in */
-	u_int v_pgpgout;	/* pages paged out */
-	u_int v_intrans;	/* intransit blocking page faults */
-	u_int v_reactivated;	/* number of pages reactivated from free list */
-	u_int v_rev;		/* revolutions of the hand */
-	u_int v_scan;		/* scans in page out daemon */
-	u_int v_dfree;		/* pages freed by daemon */
-	u_int v_pfree;		/* pages freed by exiting processes */
-	u_int v_zfod;		/* pages zero filled on demand */
-	u_int v_nzfod;		/* number of zfod's created */
+	unsigned int v_lookups;	/* object cache lookups */
+	unsigned int v_hits;		/* object cache hits */
+	unsigned int v_vm_faults;	/* number of address memory faults */
+	unsigned int v_cow_faults;	/* number of copy-on-writes */
+	unsigned int v_swpin;		/* swapins */
+	unsigned int v_swpout;		/* swapouts */
+	unsigned int v_pswpin;		/* pages swapped in */
+	unsigned int v_pswpout;	/* pages swapped out */
+	unsigned int v_pageins;	/* number of pageins */
+	unsigned int v_pageouts;	/* number of pageouts */
+	unsigned int v_pgpgin;		/* pages paged in */
+	unsigned int v_pgpgout;	/* pages paged out */
+	unsigned int v_intrans;	/* intransit blocking page faults */
+	unsigned int v_reactivated;	/* number of pages reactivated from free list */
+	unsigned int v_rev;		/* revolutions of the hand */
+	unsigned int v_scan;		/* scans in page out daemon */
+	unsigned int v_dfree;		/* pages freed by daemon */
+	unsigned int v_pfree;		/* pages freed by exiting processes */
+	unsigned int v_zfod;		/* pages zero filled on demand */
+	unsigned int v_nzfod;		/* number of zfod's created */
 	/*
 	 * Distribution of page usages.
 	 */
-	u_int v_page_size;	/* page size in bytes */
-	u_int v_kernel_pages;	/* number of pages in use by kernel */
-	u_int v_free_target;	/* number of pages desired free */
-	u_int v_free_min;	/* minimum number of pages desired free */
-	u_int v_free_count;	/* number of pages free */
-	u_int v_wire_count;	/* number of pages wired down */
-	u_int v_active_count;	/* number of pages active */
-	u_int v_inactive_target; /* number of pages desired inactive */
-	u_int v_inactive_count;  /* number of pages inactive */
+	unsigned int v_page_size;	/* page size in bytes */
+	unsigned int v_kernel_pages;	/* number of pages in use by kernel */
+	unsigned int v_free_target;	/* number of pages desired free */
+	unsigned int v_free_min;	/* minimum number of pages desired free */
+	unsigned int v_free_count;	/* number of pages free */
+	unsigned int v_wire_count;	/* number of pages wired down */
+	unsigned int v_active_count;	/* number of pages active */
+	unsigned int v_inactive_target; /* number of pages desired inactive */
+	unsigned int v_inactive_count;  /* number of pages inactive */
 };
 
 /* systemwide totals computed every five seconds */
diff --git a/bsd/sys/vmparam.h b/bsd/sys/vmparam.h
index d3ada1c8d..19f781a9a 100644
--- a/bsd/sys/vmparam.h
+++ b/bsd/sys/vmparam.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
diff --git a/bsd/sys/vnioctl.h b/bsd/sys/vnioctl.h
index 93812dcac..5841be4b2 100644
--- a/bsd/sys/vnioctl.h
+++ b/bsd/sys/vnioctl.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988 University of Utah.
diff --git a/bsd/sys/vnode.h b/bsd/sys/vnode.h
index 125d020d2..681ad05f0 100644
--- a/bsd/sys/vnode.h
+++ b/bsd/sys/vnode.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -85,8 +91,8 @@ enum vtype	{ VNON, VREG, VDIR, VBLK, VCHR, VLNK, VSOCK, VFIFO, VBAD, VSTR,
 enum vtagtype	{
 	VT_NON, VT_UFS, VT_NFS, VT_MFS, VT_MSDOSFS, VT_LFS, VT_LOFS, VT_FDESC,
 	VT_PORTAL, VT_NULL, VT_UMAP, VT_KERNFS, VT_PROCFS, VT_AFS, VT_ISOFS,
-	VT_UNION, VT_HFS, VT_VOLFS, VT_DEVFS, VT_WEBDAV, VT_UDF, VT_AFP,
-	VT_CDDA, VT_CIFS,VT_OTHER};
+	VT_UNION, VT_HFS, VT_ZFS, VT_DEVFS, VT_WEBDAV, VT_UDF, VT_AFP,
+	VT_CDDA, VT_CIFS, VT_OTHER};
 
 
 /*
@@ -123,6 +129,9 @@ enum vtagtype	{
 #define	IO_NODELOCKED	0x0008		/* underlying node already locked */
 #define	IO_NDELAY	0x0010		/* FNDELAY flag set in file table */
 #define	IO_NOZEROFILL	0x0020		/* F_SETSIZE fcntl uses to prevent zero filling */
+#ifdef XNU_KERNEL_PRIVATE
+#define IO_REVOKE	IO_NOZEROFILL	/* revoked close for tty, will Not be used in conjunction */
+#endif /* XNU_KERNEL_PRIVATE */
 #define	IO_TAILZEROFILL	0x0040		/* zero fills at the tail of write */
 #define	IO_HEADZEROFILL	0x0080		/* zero fills at the head of write */
 #define	IO_NOZEROVALID	0x0100		/* do not zero fill if valid page */
@@ -131,6 +140,11 @@ enum vtagtype	{
 #define IO_NOCACHE	0x0800		/* same effect as VNOCACHE_DATA, but only for this 1 I/O */
 #define IO_RAOFF	0x1000		/* same effect as VRAOFF, but only for this 1 I/O */
 #define IO_DEFWRITE	0x2000		/* defer write if vfs.defwrite is set */
+#define IO_PASSIVE	0x4000		/* this I/O is marked as background I/O so it won't throttle Throttleable I/O */
+#define IO_BACKGROUND IO_PASSIVE /* used for backward compatibility.  to be removed after IO_BACKGROUND is no longer
+								  * used by DiskImages in-kernel mode */
+#define	IO_NOAUTH	0x8000		/* No authorization checks. */
+
 
 /*
  * Component Name: this structure describes the pathname
@@ -177,16 +191,17 @@ struct componentname {
 /*
  * component name operational modifier flags
  */
-#define	FOLLOW		0x0040	/* follow symbolic links */
+#define	FOLLOW		0x00000040 /* follow symbolic links */
+#define NOTRIGGER	0x10000000 /* don't trigger automounts */
 
 /*
  * component name parameter descriptors.
  */
-#define	ISDOTDOT	0x002000 /* current component name is .. */
-#define	MAKEENTRY	0x004000 /* entry is to be added to name cache */
-#define	ISLASTCN	0x008000 /* this is last component of pathname */
-#define	ISWHITEOUT	0x020000 /* found whiteout */
-#define	DOWHITEOUT	0x040000 /* do whiteouts */
+#define	ISDOTDOT	0x00002000 /* current component name is .. */
+#define	MAKEENTRY	0x00004000 /* entry is to be added to name cache */
+#define	ISLASTCN	0x00008000 /* this is last component of pathname */
+#define	ISWHITEOUT	0x00020000 /* found whiteout */
+#define	DOWHITEOUT	0x00040000 /* do whiteouts */
 
 
 
@@ -208,6 +223,7 @@ struct vnode_fsparam {
 
 #define	VNFS_NOCACHE	0x01	/* do not add to name cache at this time */
 #define	VNFS_CANTCACHE	0x02	/* never add this instance to the name cache */
+#define	VNFS_ADDFSREF	0x04	/* take fs (named) reference */
 
 #define VNCREATE_FLAVOR	0
 #define VCREATESIZE sizeof(struct vnode_fsparam)
@@ -270,6 +286,7 @@ struct vnode_fsparam {
 #define VNODE_ATTR_va_uuuid		(1LL<<26)	/* 04000000 */
 #define VNODE_ATTR_va_guuid		(1LL<<27)	/* 08000000 */
 #define VNODE_ATTR_va_nchildren		(1LL<<28)       /* 10000000 */
+#define VNODE_ATTR_va_dirlinkcount	(1LL<<29)       /* 20000000 */
 
 #define VNODE_ATTR_BIT(n)	(VNODE_ATTR_ ## n)
 /*
@@ -289,7 +306,8 @@ struct vnode_fsparam {
 				VNODE_ATTR_BIT(va_gen) |		\
 				VNODE_ATTR_BIT(va_name) |		\
 				VNODE_ATTR_BIT(va_type) |		\
-				VNODE_ATTR_BIT(va_nchildren))
+				VNODE_ATTR_BIT(va_nchildren) |		\
+				VNODE_ATTR_BIT(va_dirlinkcount))
 /*
  * Attributes that can be applied to a new file object.
  */
@@ -357,8 +375,9 @@ struct vnode_attr {
 	guid_t		va_uuuid;	/* file owner UUID */
 	guid_t		va_guuid;	/* file group UUID */
 	
-	uint64_t	va_nchildren;	/* Number of items in a directory */
-					/* Meaningful for directories only */
+	/* Meaningful for directories only */
+	uint64_t	va_nchildren;     /* Number of items in a directory */
+	uint64_t	va_dirlinkcount;  /* Real references to dir (i.e. excluding "." and ".." refs) */
 
 	/* add new fields here only */
 };
@@ -476,9 +495,9 @@ int	vnode_removefsref(vnode_t);
 
 int	vnode_hasdirtyblks(vnode_t);
 int	vnode_hascleanblks(vnode_t);
-#define	VNODE_ASYNC_THROTTLE	18
+#define	VNODE_ASYNC_THROTTLE	15
 /* timeout is in 10 msecs and not hz tick based */
-int	vnode_waitforwrites(vnode_t, int, int, int, char *);
+int	vnode_waitforwrites(vnode_t, int, int, int, const char *);
 void	vnode_startwrite(vnode_t);
 void	vnode_writedone(vnode_t);
 
@@ -486,12 +505,15 @@ enum vtype	vnode_vtype(vnode_t);
 uint32_t	vnode_vid(vnode_t);
 mount_t	vnode_mountedhere(vnode_t vp);
 mount_t	vnode_mount(vnode_t);
+errno_t vfs_mountref(mount_t);
+errno_t vfs_mountrele(mount_t);
 dev_t	vnode_specrdev(vnode_t);
 void *	vnode_fsnode(vnode_t);
 void	vnode_clearfsnode(vnode_t);
 
 int	vnode_isvroot(vnode_t);
 int	vnode_issystem(vnode_t);
+int	vnode_isswap(vnode_t vp);
 int	vnode_ismount(vnode_t);
 int	vnode_isreg(vnode_t);
 int	vnode_isdir(vnode_t);
@@ -500,22 +522,48 @@ int	vnode_isfifo(vnode_t);
 int	vnode_isblk(vnode_t);
 int	vnode_ischr(vnode_t);
 
+#ifdef __APPLE_API_UNSTABLE
+int	vnode_isnamedstream(vnode_t);
+#endif
+
+enum vtype	vnode_iftovt(int);
+int	vnode_vttoif(enum vtype);
+int	vnode_makeimode(int, int);
+
 int	vnode_ismountedon(vnode_t);
 void	vnode_setmountedon(vnode_t);
 void	vnode_clearmountedon(vnode_t);
 
+int	vnode_isrecycled(vnode_t);
 int	vnode_isnocache(vnode_t);
 void	vnode_setnocache(vnode_t);
 void	vnode_clearnocache(vnode_t);
 int	vnode_isnoreadahead(vnode_t);
+int	vnode_isstandard(vnode_t);
 void	vnode_setnoreadahead(vnode_t);
 void	vnode_clearnoreadahead(vnode_t);
+
+int	vnode_is_openevt(vnode_t);
+void	vnode_set_openevt(vnode_t);
+void	vnode_clear_openevt(vnode_t);
+
 /* left only for compat reasons as User code depends on this from getattrlist, for ex */
 void	vnode_settag(vnode_t, int);
 int	vnode_tag(vnode_t);
 int	vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx);
 int	vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx);
 
+vnode_t vfs_rootvnode(void);
+uint32_t vnode_vfsvisflags(vnode_t);
+uint32_t vnode_vfscmdflags(vnode_t);
+struct vfsstatfs *vnode_vfsstatfs(vnode_t);
+void	*vnode_vfsfsprivate(vnode_t);
+vnode_t	current_workingdir(void);
+vnode_t	current_rootdir(void);
+
+void	vnode_uncache_credentials(vnode_t vp);
+void	vnode_setmultipath(vnode_t vp);
+
 #ifdef BSD_KERNEL_PRIVATE
 
 /*
@@ -523,15 +571,16 @@ int	vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx);
  * VNOP_LOOKUP on this vnode.  Volfs will always ask for it's parent
  * object ID (instead of using the v_parent pointer).
  */
-void	vnode_set_hard_link(vnode_t vp);
 
 vnode_t vnode_parent(vnode_t);
 void vnode_setparent(vnode_t, vnode_t);
-char * vnode_name(vnode_t);
+const char * vnode_name(vnode_t);
 void vnode_setname(vnode_t, char *);
 int vnode_isnoflush(vnode_t);
 void vnode_setnoflush(vnode_t);
 void vnode_clearnoflush(vnode_t);
+/* XXX temporary until we can arrive at a KPI for NFS, Seatbelt */
+thread_t vfs_context_thread(vfs_context_t);
 #endif
 
 uint32_t  vnode_vfsmaxsymlen(vnode_t);
@@ -540,8 +589,11 @@ int	vnode_vfstypenum(vnode_t);
 void	vnode_vfsname(vnode_t, char *);
 int 	vnode_vfs64bitready(vnode_t);
 
+int	vfs_context_get_special_port(vfs_context_t, int, ipc_port_t *);
+int	vfs_context_set_special_port(vfs_context_t, int, ipc_port_t);
 proc_t	vfs_context_proc(vfs_context_t);
-ucred_t	vfs_context_ucred(vfs_context_t);
+vnode_t vfs_context_cwd(vfs_context_t);
+kauth_cred_t	vfs_context_ucred(vfs_context_t);
 int	vfs_context_issuser(vfs_context_t);
 int	vfs_context_pid(vfs_context_t);
 int	vfs_context_issignal(vfs_context_t, sigset_t);
@@ -549,6 +601,10 @@ int	vfs_context_suser(vfs_context_t);
 int	vfs_context_is64bit(vfs_context_t);
 vfs_context_t vfs_context_create(vfs_context_t);
 int vfs_context_rele(vfs_context_t);
+vfs_context_t vfs_context_current(void);	/* get from current uthread */
+#ifdef __APPLE_API_UNSTABLE
+vfs_context_t vfs_context_kernel(void);		/* get from 1st kernel thread */
+#endif
 
 
 int	vflush(struct mount *mp, struct vnode *skipvp, int flags);
@@ -566,7 +622,7 @@ void	vnode_reclaim(vnode_t);
 #define	VNODE_UPDATE_PARENT	0x01
 #define	VNODE_UPDATE_NAME	0x02
 #define	VNODE_UPDATE_CACHE	0x04
-void	vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int name_hashval, int flags);
+void	vnode_update_identity(vnode_t vp, vnode_t dvp, const char *name, int name_len, int name_hashval, int flags);
 
 int	vn_bwrite(struct vnop_bwrite_args *ap);
 
@@ -576,6 +632,7 @@ int	vnode_authattr_new(vnode_t /*dvp*/, struct vnode_attr *, int /*noauth*/, vfs
 errno_t vnode_close(vnode_t, int, vfs_context_t);
 
 int vn_getpath(struct vnode *vp, char *pathbuf, int *len);
+int vn_getcdhash(struct vnode *vp, off_t offset, unsigned char *cdhash);
 
 /*
  * Flags for the vnode_lookup and vnode_open
@@ -615,13 +672,16 @@ int	vnode_iterate(struct mount *, int, int (*)(struct vnode *, void *), void *);
 #define VNODE_CLAIMED_DONE	3	/* don't drop reference, terminate iteration */
 
 
+#ifdef BSD_KERNEL_PRIVATE
+/* Not in export list so can be private */
 struct stat;
-int	vn_stat(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_context_t ctx);
-int	vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_context_t ctx);
-int	vn_revoke(vnode_t vp, int flags, vfs_context_t);
-/* XXX BOGUS */
+int	vn_stat(struct vnode *vp, void * sb, kauth_filesec_t *xsec, int isstat64, vfs_context_t ctx);
+int	vn_stat_noauth(struct vnode *vp, void * sb, kauth_filesec_t *xsec, int isstat64, vfs_context_t ctx);
 int	vaccess(mode_t file_mode, uid_t uid, gid_t gid,
-	  		mode_t acc_mode, struct ucred *cred);
+	  		mode_t acc_mode, kauth_cred_t cred);
+#endif /* BSD_KERNEL_PRIVATE */
+
+int	vn_revoke(vnode_t vp, int flags, vfs_context_t);
 
 
 /* namecache function prototypes */
@@ -635,9 +695,12 @@ void	cache_purge_negatives(vnode_t vp);
  * if you don't know it (add_name() will then compute the hash).
  * There are no flags for now but maybe someday.
  */
-char *vfs_addname(const char *name, size_t len, u_int nc_hash, u_int flags);
+const char *vfs_addname(const char *name, size_t len, u_int nc_hash, u_int flags);
 int   vfs_removename(const char *name);
 
+int	check_mountedon(dev_t dev, enum vtype type, int  *errorp);
+int	vcount(vnode_t vp);
+
 __END_DECLS
 
 #endif /* KERNEL */
diff --git a/bsd/sys/vnode_if.h b/bsd/sys/vnode_if.h
index 062cc0fee..87fa29333 100644
--- a/bsd/sys/vnode_if.h
+++ b/bsd/sys/vnode_if.h
@@ -2,23 +2,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved
@@ -53,7 +59,12 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
-
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * Warning: This file is generated automatically.
@@ -89,8 +100,6 @@ extern struct vnodeop_desc vnop_close_desc;
 extern struct vnodeop_desc vnop_access_desc;
 extern struct vnodeop_desc vnop_getattr_desc;
 extern struct vnodeop_desc vnop_setattr_desc;
-extern struct vnodeop_desc vnop_getattrlist_desc;
-extern struct vnodeop_desc vnop_setattrlist_desc;
 extern struct vnodeop_desc vnop_read_desc;
 extern struct vnodeop_desc vnop_write_desc;
 extern struct vnodeop_desc vnop_ioctl_desc;
@@ -126,6 +135,16 @@ extern struct vnodeop_desc vnop_blockmap_desc;
 extern struct vnodeop_desc vnop_strategy_desc;
 extern struct vnodeop_desc vnop_bwrite_desc;
 
+#ifdef __APPLE_API_UNSTABLE
+
+#if NAMEDSTREAMS
+extern struct vnodeop_desc vnop_getnamedstream_desc;
+extern struct vnodeop_desc vnop_makenamedstream_desc;
+extern struct vnodeop_desc vnop_removenamedstream_desc;
+#endif
+
+#endif
+
 __BEGIN_DECLS
 /*
  *# 
@@ -257,37 +276,6 @@ struct vnop_setattr_args {
 };
 extern errno_t VNOP_SETATTR(vnode_t, struct vnode_attr *, vfs_context_t);
 
-/*
- *#
- *#% getattrlist  vp      = = =
- *#
- */
-struct vnop_getattrlist_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context_t a_context;
-};
-extern errno_t VNOP_GETATTRLIST(vnode_t, struct attrlist *, struct uio *, int, vfs_context_t);
-
-
-/*
- *#
- *#% setattrlist  vp      L L L
- *#
- */
-struct vnop_setattrlist_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context_t a_context;
-};
-extern errno_t VNOP_SETATTRLIST(vnode_t, struct attrlist *, struct uio *, int, vfs_context_t);
-
 
 /*
  *#
@@ -750,7 +738,7 @@ extern errno_t VNOP_COPYFILE(vnode_t, vnode_t, vnode_t, struct componentname *,
 struct vnop_getxattr_args {
 	struct vnodeop_desc *a_desc;
 	vnode_t a_vp;
-	char * a_name;
+	const char * a_name;
 	uio_t a_uio;
 	size_t *a_size;
 	int a_options;
@@ -762,7 +750,7 @@ extern errno_t VNOP_GETXATTR(vnode_t, const char *, uio_t, size_t *, int, vfs_co
 struct vnop_setxattr_args {
 	struct vnodeop_desc *a_desc;
 	vnode_t a_vp;
-	char * a_name;
+	const char * a_name;
 	uio_t a_uio;
 	int a_options;
 	vfs_context_t a_context;
@@ -773,7 +761,7 @@ extern errno_t VNOP_SETXATTR(vnode_t, const char *, uio_t, int, vfs_context_t);
 struct vnop_removexattr_args {
 	struct vnodeop_desc *a_desc;
 	vnode_t a_vp;
-	char * a_name;
+	const char * a_name;
 	int a_options;
 	vfs_context_t a_context;
 };
@@ -870,6 +858,56 @@ struct vnop_kqfilt_remove_args {
 extern struct vnodeop_desc vnop_kqfilt_remove_desc;
 errno_t VNOP_KQFILT_REMOVE(vnode_t , uintptr_t , vfs_context_t);
 
+struct label;
+struct vnop_setlabel_args {
+	struct vnodeop_desc *a_desc;
+	struct vnode *a_vp;
+	struct label *a_vl;
+	vfs_context_t a_context;
+};
+extern struct vnodeop_desc vnop_setlabel_desc;
+errno_t VNOP_SETLABEL(vnode_t, struct label *, vfs_context_t);
+
+#ifdef __APPLE_API_UNSTABLE
+
+#if NAMEDSTREAMS
+
+enum nsoperation	{ NS_OPEN, NS_CREATE, NS_DELETE };
+
+struct vnop_getnamedstream_args {
+	struct vnodeop_desc *a_desc;
+	vnode_t a_vp;
+	vnode_t *a_svpp;
+	const char *a_name;
+	enum nsoperation a_operation;
+	int a_flags;
+	vfs_context_t a_context;
+};
+extern errno_t VNOP_GETNAMEDSTREAM(vnode_t, vnode_t *, const char *, enum nsoperation, int flags, vfs_context_t);
+
+struct vnop_makenamedstream_args {
+	struct vnodeop_desc *a_desc;
+	vnode_t *a_svpp;
+	vnode_t a_vp;
+	const char *a_name;
+	int a_flags;
+	vfs_context_t a_context;
+};
+extern errno_t VNOP_MAKENAMEDSTREAM(vnode_t, vnode_t *, const char *, int flags, vfs_context_t);
+
+struct vnop_removenamedstream_args {
+	struct vnodeop_desc *a_desc;
+	vnode_t a_vp;
+	vnode_t a_svp;
+	const char *a_name;
+	int a_flags;
+	vfs_context_t a_context;
+};
+extern errno_t VNOP_REMOVENAMEDSTREAM(vnode_t, vnode_t, const char *, int flags, vfs_context_t);
+#endif
+
+#endif
+
 __END_DECLS
 
 #endif /* KERNEL */
diff --git a/bsd/sys/vnode_internal.h b/bsd/sys/vnode_internal.h
index aa296e4a0..ae4be39a9 100644
--- a/bsd/sys/vnode_internal.h
+++ b/bsd/sys/vnode_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,7 +60,13 @@
  *
  *	@(#)vnode.h	8.17 (Berkeley) 5/20/95
  */
- 
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+
 #ifndef _SYS_VNODE_INTERNAL_H_
 #define _SYS_VNODE_INTERNAL_H_
 
@@ -75,9 +87,11 @@
 #include <sys/vnode.h>
 #include <sys/namei.h>
 #include <sys/vfs_context.h>
+#include <sys/sysctl.h>
 
 
 struct lockf;
+struct label;
 
 LIST_HEAD(buflists, buf);
 
@@ -103,6 +117,7 @@ struct vnode {
         LIST_HEAD(, namecache) v_nclinks;	/* name cache entries that name this vnode */
         LIST_HEAD(, namecache) v_ncchildren;	/* name cache entries that regard us as there parent */
         vnode_t	 v_defer_reclaimlist;		/* in case we have to defer the reclaim to avoid recursion */
+        u_long	 v_listflag;			/* flags protected by the vnode_list_lock (see below) */
 	u_long	 v_flag;			/* vnode flags (see below) */
 	u_short	 v_lflag;			/* vnode local and named ref flags */
 	u_char	 v_iterblkflags;		/* buf iterator flags */
@@ -111,8 +126,9 @@ struct vnode {
 	int32_t	 v_usecount;			/* reference count of users */
 	int32_t	 v_iocount;			/* iocounters */
 	void *   v_owner;			/* act that owns the vnode */
-	enum vtype v_type;			/* vnode type */
-	u_long	 v_id;				/* identity of vnode contents */
+	u_short	 v_type;			/* vnode type */
+	u_short	 v_tag;				/* type of underlying data */
+	int	 v_id;				/* identity of vnode contents */
 	union {
 		struct mount	*vu_mountedhere;/* ptr to mounted vfs (VDIR) */
 		struct socket	*vu_socket;	/* unix ipc (VSOCK) */
@@ -122,20 +138,32 @@ struct vnode {
 	} v_un;
 	struct	buflists v_cleanblkhd;		/* clean blocklist head */
 	struct	buflists v_dirtyblkhd;		/* dirty blocklist head */
-        kauth_cred_t v_cred;
-        int	v_cred_timestamp;
+        /*
+	 * the following 4 fields are protected
+	 * by the name_cache_lock held in 
+	 * excluive mode
+	 */
+        kauth_cred_t	v_cred;			/* last authorized credential */
+        kauth_action_t	v_authorized_actions;	/* current authorized actions for v_cred */
+        int		v_cred_timestamp;	/* determine if entry is stale for MNTK_AUTH_OPAQUE */
+        int		v_nc_generation;	/* changes when nodes are removed from the name cache */
+        /*
+	 * back to the vnode lock for protection
+	 */
 	long	v_numoutput;			/* num of writes in progress */
 	long	v_writecount;			/* reference count of writers */
-	char *	v_name;				/* name component of the vnode */
+	const char *v_name;			/* name component of the vnode */
 	vnode_t v_parent;			/* pointer to parent vnode */
 #ifdef INTERIM_FSNODE_LOCK
 	struct lockf	*v_lockf;		/* advisory lock list head */
         struct unsafe_fsnode *v_unsafefs;	/* pointer to struct used to lock */
 #endif						/* vnodes on unsafe filesystems */
 	int 	(**v_op)(void *);		/* vnode operations vector */
-	enum vtagtype v_tag;			/* type of underlying data */
 	mount_t v_mount;			/* ptr to vfs we are in */
 	void *	v_data;				/* private data for fs */
+#if CONFIG_MACF
+	struct label *v_label;			/* MAC security label */
+#endif
 };
 
 #define	v_mountedhere	v_un.vu_mountedhere
@@ -154,6 +182,11 @@ struct vnode {
 #define VBI_DIRTY		0x8
 #define VBI_NEWBUF		0x10
 
+/*
+ * v_listflag
+ */
+#define VLIST_RAGE    0x01            /* vnode is currently in the rapid age list */
+#define VLIST_DEAD    0x02            /* vnode is currently in the dead list */
 
 /*
  * v_lflags
@@ -167,11 +200,15 @@ struct vnode {
 #define	VL_MOUNTDEAD	0x0040		/* v_moutnedhere is dead   */
 #define VL_NEEDINACTIVE	0x0080		/* delay VNOP_INACTIVE until iocount goes to 0 */
 
+#define	VL_LABEL	0x0100		/* vnode is marked for labeling */
+#define	VL_LABELWAIT	0x0200		/* vnode is marked for labeling */
+#define	VL_LABELED	0x0400		/* vnode is labeled */
+#define	VL_LWARNED	0x0800
+
 #define	VNAMED_UBC	0x2000		/* ubc named reference */
 #define VNAMED_MOUNT	0x4000		/* mount point named reference */
 #define	VNAMED_FSHASH	0x8000		/* FS hash named reference */
 
-
 /*
  * v_flags
  */
@@ -179,9 +216,9 @@ struct vnode {
 #define	VTEXT		0x000002	/* vnode is a pure text prototype */
 #define	VSYSTEM		0x000004	/* vnode being used by kernel */
 #define	VISTTY		0x000008	/* vnode represents a tty */
-#define	VWASMAPPED	0x000010	/* vnode was mapped before */
-#define	VTERMINATE	0x000020	/* terminating memory object */
-#define	VTERMWANT	0x000040	/* wating for memory object death */
+#define	VRAGE		0x000010	/* vnode is in rapid age state */
+#define VBDEVVP		0x000020        /* vnode created by bdevvp */
+#define VDEVFLUSH	0x000040        /* device vnode after vflush */
 #define	VMOUNT		0x000080	/* mount operation in progress */
 #define	VBWAIT		0x000100	/* waiting for output to complete */
 #define	VALIASED	0x000200	/* vnode has an alias */
@@ -197,9 +234,11 @@ struct vnode {
 #define	VNOFLUSH	0x040000	/* don't vflush() if SKIPSYSTEM */
 #define	VLOCKLOCAL	0x080000	/* this vnode does adv locking in vfs */
 #define	VISHARDLINK	0x100000	/* hard link needs special processing on lookup and in volfs */
-
-#define VCRED_EXPIRED	2		/* number of seconds to keep cached credential valid */
-
+#define	VISUNION	0x200000	/* union special processing */
+#if NAMEDSTREAMS
+#define	VISNAMEDSTREAM	0x400000	/* vnode is a named stream (eg HFS resource fork) */
+#endif
+#define VOPENEVT        0x800000        /* if process is P_CHECKOPENEVT, then or in the O_EVTONLY flag on open */
 
 /*
  * Global vnode data.
@@ -234,7 +273,7 @@ extern	struct vnode *rootvnode;	/* root (i.e. "/") vnode */
  */
 struct vnodeop_desc {
 	int	vdesc_offset;		/* offset in vector--first for speed */
-	char    *vdesc_name;		/* a readable name for debugging */
+	const char *vdesc_name;		/* a readable name for debugging */
 	int	vdesc_flags;		/* VDESC_* flags */
 
 	/*
@@ -262,10 +301,6 @@ struct vnodeop_desc {
  */
 extern struct vnodeop_desc *vnodeop_descs[];
 
-/*
- * Interlock for scanning list of vnodes attached to a mountpoint
- */
-extern void * mntvnode_slock;
 
 /*
  * This macro is very helpful in defining those offsets in the vdesc struct.
@@ -299,30 +334,38 @@ extern void * mntvnode_slock;
 
 struct ostat;
 
-int	build_path(vnode_t first_vp, char *buff, int buflen, int *outlen);
+#define BUILDPATH_NO_FS_ENTER 0x1 /* Use cache values, do not enter file system */
+int	build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx);
+
 int 	bdevvp(dev_t dev, struct vnode **vpp);
 void	cvtstat(struct stat *st, struct ostat *ost);
 void	vprint(const char *label, struct vnode *vp);
 
 
-__private_extern__ int is_package_name(char *name, int len);
+__private_extern__ int is_package_name(const char *name, int len);
 __private_extern__ int set_package_extensions_table(void *data, int nentries, int maxwidth);
 int 	vn_rdwr(enum uio_rw rw, struct vnode *vp, caddr_t base,
 	    		int len, off_t offset, enum uio_seg segflg, int ioflg,
-	    		struct ucred *cred, int *aresid, struct proc *p);
+	    		kauth_cred_t cred, int *aresid, struct proc *p);
 int 	vn_rdwr_64(enum uio_rw rw, struct vnode *vp, uint64_t base,
 	    		int64_t len, off_t offset, enum uio_seg segflg,
-			int ioflg, struct ucred *cred, int *aresid,
+			int ioflg, kauth_cred_t cred, int *aresid,
 			struct proc *p);
+#if CONFIG_MACF
+int	vn_setlabel (struct vnode *vp, struct label *intlabel,
+	    vfs_context_t context);
+#endif
 void	fifo_printinfo(struct vnode *vp);
 int	vn_lock(struct vnode *vp, int flags, struct proc *p);
 int 	vn_open(struct nameidata *ndp, int fmode, int cmode);
 int	vn_open_modflags(struct nameidata *ndp, int *fmode, int cmode);
 int	vn_open_auth(struct nameidata *ndp, int *fmode, struct vnode_attr *);
-int 	vn_close(vnode_t, int flags, struct ucred *cred, struct proc *p);
+int 	vn_close(vnode_t, int flags, vfs_context_t ctx);
 
 #define VN_CREATE_NOAUTH		(1<<0)
 #define VN_CREATE_NOINHERIT		(1<<1)
+#define VN_CREATE_UNION			(1<<2)
+#define	VN_CREATE_NOLABEL		(1<<3)
 errno_t vn_create(vnode_t, vnode_t *, struct componentname *, struct vnode_attr *, int flags, vfs_context_t);
 
 
@@ -331,23 +374,46 @@ int	vn_setxattr(vnode_t, const char *, uio_t, int, vfs_context_t);
 int	vn_removexattr(vnode_t, const char *, int, vfs_context_t);
 int	vn_listxattr(vnode_t, uio_t, size_t *, int, vfs_context_t);
 
+int	default_getxattr(vnode_t, const char *, uio_t, size_t *, int, vfs_context_t);
+int	default_setxattr(vnode_t, const char *, uio_t, int, vfs_context_t);
+int	default_removexattr(vnode_t, const char *, int, vfs_context_t);
+
+#if NAMEDSTREAMS
+errno_t  vnode_getnamedstream(vnode_t, vnode_t *, const char *, enum nsoperation, int, vfs_context_t);
+errno_t  vnode_makenamedstream(vnode_t, vnode_t *, const char *, int, vfs_context_t);
+errno_t  vnode_removenamedstream(vnode_t, vnode_t, const char *, int, vfs_context_t);
+errno_t  vnode_flushnamedstream(vnode_t vp, vnode_t svp, vfs_context_t context);
+errno_t  vnode_relenamedstream(vnode_t vp, vnode_t svp, vfs_context_t context);
+#endif
+
+int vn_path_package_check(vnode_t vp, char *path, int pathlen, int *component);
+
+void	nchinit(void) __attribute__((section("__TEXT, initcode")));
+int	resize_namecache(u_int newsize);
 void	name_cache_lock_shared(void);
 void	name_cache_lock(void);
 void	name_cache_unlock(void);
+void	cache_enter_with_gen(vnode_t dvp, vnode_t vp, struct componentname *cnp, int gen);
 
-char *	vnode_getname(vnode_t vp);
-void	vnode_putname(char *name);
+const char	*vnode_getname(vnode_t vp);
+void	vnode_putname(const char *name);
 
 vnode_t	vnode_getparent(vnode_t vp);
 
 int vn_pathconf(vnode_t, int, register_t *, vfs_context_t);
 
+#define	vnode_lock_convert(v)	lck_mtx_convert_spin(&(v)->v_lock)
+
+void	vnode_lock_spin(vnode_t);
+
+
 void	vnode_list_lock(void);
 void	vnode_list_unlock(void);
 int	vnode_ref_ext(vnode_t, int);
 void	vnode_rele_ext(vnode_t, int, int);
 void	vnode_rele_internal(vnode_t, int, int, int);
 int	vnode_getwithref(vnode_t);
+int	vnode_get_locked(vnode_t);
 int	vnode_put_locked(vnode_t);
 
 int	vnode_issock(vnode_t);
@@ -356,16 +422,33 @@ void	unlock_fsnode(vnode_t, int *);
 int	lock_fsnode(vnode_t, int *);
 
 errno_t	vnode_resume(vnode_t);
+errno_t	vnode_suspend(vnode_t);
+
 
 errno_t	vnode_size(vnode_t, off_t *, vfs_context_t);
 errno_t	vnode_setsize(vnode_t, off_t, int ioflag, vfs_context_t);
 int	vnode_setattr_fallback(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx);
 
+void vn_setunionwait(vnode_t);
+void vn_checkunionwait(vnode_t);
+void vn_clearunionwait(vnode_t, int);
+
 void SPECHASH_LOCK(void);
 void SPECHASH_UNLOCK(void);
 
 int check_cdevmounted(dev_t, enum vtype, int *);
 
-void	vnode_authorize_init(void);
+void	vnode_authorize_init(void) __attribute__((section("__TEXT, initcode")));
+
+void	vfsinit(void);
+
+/*
+ * XXX exported symbols; should be static
+ */
+void	vfs_op_init(void) __attribute__((section("__TEXT, initcode")));
+void	vfs_opv_init(void) __attribute__((section("__TEXT, initcode")));
+int	vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
+		user_addr_t newp, size_t newlen, struct proc *p);
+int	sysctl_vnode(struct sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req);
 
 #endif /* !_SYS_VNODE_INTERNAL_H_ */
diff --git a/bsd/sys/vstat.h b/bsd/sys/vstat.h
index 4a8817c8d..2685b741c 100644
--- a/bsd/sys/vstat.h
+++ b/bsd/sys/vstat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1998 Apple Computer, Inc. All Rights Reserved */
 /*-
@@ -28,6 +34,7 @@
 #define	_SYS_VSTAT_H_
 
 #include <sys/appleapiopts.h>
+#include <sys/cdefs.h>
 
 #warning obsolete header! delete the include from your sources
 
@@ -36,7 +43,7 @@
 #include <sys/time.h>
 #include <sys/attr.h>
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 
 struct vstat {
 	fsid_t			vst_volid;		/* volume identifier */
@@ -49,7 +56,7 @@ struct vstat {
 	gid_t	  		vst_gid;		/* group ID of the file's group */
 	dev_t			vst_dev;		/* inode's device */
 	dev_t	  		vst_rdev;		/* device type */
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 	struct	timespec vst_atimespec;	/* time of last access */
 	struct	timespec vst_mtimespec;	/* time of last data modification */
 	struct	timespec vst_ctimespec;	/* time of last file status change */
@@ -67,7 +74,7 @@ struct vstat {
 	u_int32_t 		vst_flags;		/* user defined flags for file */
 };
 
-#endif /* ! _POSIX_C_SOURCE */
+#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */
 #endif /* __APPLE_API_OBSOLETE */
 
 #endif /* !_SYS_VSTAT_H_ */
diff --git a/bsd/sys/wait.h b/bsd/sys/wait.h
index 34aba1bb8..1de5ce97a 100644
--- a/bsd/sys/wait.h
+++ b/bsd/sys/wait.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -119,19 +125,19 @@ typedef __darwin_id_t	id_t;
  * this option is done, it is as though they were still running... nothing
  * about them is returned.
  */
-#define WNOHANG		0x01	/* [XSI] don't hang in wait/no child to reap */
-#define WUNTRACED	0x02	/* [XSI] notify on stopped, untraced children */
+#define WNOHANG		0x00000001  /* [XSI] no hang in wait/no child to reap */
+#define WUNTRACED	0x00000002  /* [XSI] notify on stop, untraced child */
 
 /*
  * Macros to test the exit status returned by wait
  * and extract the relevant values.
  */
-#ifdef _POSIX_C_SOURCE
+#if defined(_POSIX_C_SOURCE) && !defined(_DARWIN_C_SOURCE)
 #define	_W_INT(i)	(i)
 #else
 #define	_W_INT(w)	(*(int *)&(w))	/* convert union wait to int */
 #define	WCOREFLAG	0200
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /* These macros are permited, as they are in the implementation namespace */
 #define	_WSTATUS(x)	(_W_INT(x) & 0177)
@@ -141,19 +147,24 @@ typedef __darwin_id_t	id_t;
  * [XSI] The <sys/wait.h> header shall define the following macros for
  * analysis of process status values
  */
+#if __DARWIN_UNIX03
+#define WEXITSTATUS(x)	((_W_INT(x) >> 8) & 0x000000ff)
+#else /* !__DARWIN_UNIX03 */
 #define WEXITSTATUS(x)	(_W_INT(x) >> 8)
-#define WIFCONTINUED(x) (x == 0x13)	/* 0x13 == SIGCONT */
+#endif /* !__DARWIN_UNIX03 */
+/* 0x13 == SIGCONT */
+#define WSTOPSIG(x)	(_W_INT(x) >> 8)
+#define WIFCONTINUED(x) (_WSTATUS(x) == _WSTOPPED && WSTOPSIG(x) == 0x13)
+#define WIFSTOPPED(x)	(_WSTATUS(x) == _WSTOPPED && WSTOPSIG(x) != 0x13)
 #define WIFEXITED(x)	(_WSTATUS(x) == 0)
 #define WIFSIGNALED(x)	(_WSTATUS(x) != _WSTOPPED && _WSTATUS(x) != 0)
-#define WIFSTOPPED(x)	(_WSTATUS(x) == _WSTOPPED)
-#define WSTOPSIG(x)	(_W_INT(x) >> 8)
 #define WTERMSIG(x)	(_WSTATUS(x))
-#if !defined(_POSIX_C_SOURCE)
+#if (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 #define WCOREDUMP(x)	(_W_INT(x) & WCOREFLAG)
 
 #define	W_EXITCODE(ret, sig)	((ret) << 8 | (sig))
 #define	W_STOPCODE(sig)		((sig) << 8 | _WSTOPPED)
-#endif /* !defined(_POSIX_C_SOURCE) */
+#endif /* (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
 
 /*
  * [XSI] The following symbolic constants shall be defined as possible
@@ -161,16 +172,16 @@ typedef __darwin_id_t	id_t;
  */
 /* WNOHANG already defined for wait4() */
 /* WUNTRACED defined for wait4() but not for waitid() */
-#define	WEXITED		0x04	/* [XSI] Processes which have exitted */
-#ifdef _POSIX_C_SOURCE
+#define	WEXITED		0x00000004  /* [XSI] Processes which have exitted */
+#if __DARWIN_UNIX03
 /* waitid() parameter */
-#define	WSTOPPED	0x08	/* [XSI] Any child stopped by signal receipt */
+#define	WSTOPPED	0x00000008  /* [XSI] Any child stopped by signal */
 #endif
-#define	WCONTINUED	0x10	/* [XSI] Any child stopped then continued */
-#define	WNOWAIT		0x20	/* [XSI] Leave process returned waitable */
+#define	WCONTINUED	0x00000010  /* [XSI] Any child stopped then continued */
+#define	WNOWAIT		0x00000020  /* [XSI] Leave process returned waitable */
 
 
-#if !defined(_POSIX_C_SOURCE)
+#if (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 /* POSIX extensions and 4.2/4.3 compatability: */
 
 /*
@@ -184,7 +195,7 @@ typedef __darwin_id_t	id_t;
 /*
  * Deprecated:
  * Structure of the information in the status word returned by wait4.
- * If w_stopval==WSTOPPED, then the second structure describes
+ * If w_stopval==_WSTOPPED, then the second structure describes
  * the information returned, else the first.
  */
 union wait {
@@ -230,24 +241,27 @@ union wait {
 #define w_stopval	w_S.w_Stopval
 #define w_stopsig	w_S.w_Stopsig
 
+#endif /* (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
+
+#if !(__DARWIN_UNIX03 - 0)
 /*
  * Stopped state value; cannot use waitid() parameter of the same name
  * in the same scope
  */
 #define	WSTOPPED	_WSTOPPED
-#endif /* !defined(_POSIX_C_SOURCE) */
+#endif /* !__DARWIN_UNIX03 */
 
 #ifndef KERNEL
 __BEGIN_DECLS
-pid_t	wait(int *);
-pid_t	waitpid(pid_t, int *, int);
+pid_t	wait(int *) __DARWIN_ALIAS_C(wait);
+pid_t	waitpid(pid_t, int *, int) __DARWIN_ALIAS_C(waitpid);
 #ifndef _ANSI_SOURCE
-int	waitid(idtype_t, id_t, siginfo_t *, int);
+int	waitid(idtype_t, id_t, siginfo_t *, int) __DARWIN_ALIAS_C(waitid);
 #endif /* !_ANSI_SOURCE */
-#if  !defined(_POSIX_C_SOURCE)
+#if  (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE))
 pid_t	wait3(int *, int, struct rusage *);
 pid_t	wait4(pid_t, int *, int, struct rusage *);
-#endif /* !defined(_POSIX_C_SOURCE) */
+#endif /* (!defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)) */
 __END_DECLS
 #endif
 #endif /* !_SYS_WAIT_H_ */
diff --git a/bsd/sys/xattr.h b/bsd/sys/xattr.h
index 6628bbeee..01f9c71e0 100644
--- a/bsd/sys/xattr.h
+++ b/bsd/sys/xattr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _SYS_XATTR_H_
@@ -35,6 +41,9 @@
 /* Set this to bypass authorization checking (eg. if doing auth-related work) */
 #define XATTR_NOSECURITY 0x0008
 
+/* Set this to bypass the default extended attribute file (dot-underscore file) */
+#define XATTR_NODEFAULT  0x0010
+
 #define	XATTR_MAXNAMELEN   127
 
 #define	XATTR_FINDERINFO_NAME	  "com.apple.FinderInfo"
diff --git a/bsd/ufs/ffs/ffs_alloc.c b/bsd/ufs/ffs/ffs_alloc.c
index e2177c801..0c127dcfd 100644
--- a/bsd/ufs/ffs/ffs_alloc.c
+++ b/bsd/ufs/ffs/ffs_alloc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ffs/ffs_balloc.c b/bsd/ufs/ffs/ffs_balloc.c
index e8d879344..cf998b4e5 100644
--- a/bsd/ufs/ffs/ffs_balloc.c
+++ b/bsd/ufs/ffs/ffs_balloc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ffs/ffs_extern.h b/bsd/ufs/ffs/ffs_extern.h
index c833abc20..840263068 100644
--- a/bsd/ufs/ffs/ffs_extern.h
+++ b/bsd/ufs/ffs/ffs_extern.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 
@@ -87,7 +93,11 @@ struct nameidata;
 struct proc;
 struct vfsstatfs;
 struct timeval;
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
 struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
 struct uio;
 struct vnode;
 struct mbuf;
@@ -100,9 +110,9 @@ int	ffs_fsync_internal(vnode_t, int);
 int	ffs_blkatoff(vnode_t, off_t, char **, buf_t *);
 
 int	ffs_alloc(struct inode *,
-	    ufs_daddr_t, ufs_daddr_t, int, struct ucred *, ufs_daddr_t *);
+	    ufs_daddr_t, ufs_daddr_t, int, kauth_cred_t, ufs_daddr_t *);
 int	ffs_balloc(struct inode *,
-	    ufs_daddr_t, int, struct ucred *, struct buf **, int, int *);
+	    ufs_daddr_t, int, kauth_cred_t, struct buf **, int, int *);
 void	ffs_blkfree(struct inode *, ufs_daddr_t, long);
 ufs_daddr_t ffs_blkpref(struct inode *, ufs_daddr_t, int, ufs_daddr_t *);
 void	ffs_clrblock(struct fs *, u_char *, ufs_daddr_t);
@@ -116,7 +126,7 @@ int	ffs_mountfs(struct vnode *, struct mount *, vfs_context_t);
 int	ffs_mountroot(mount_t, vnode_t, vfs_context_t);
 int	ffs_read(struct vnop_read_args *);
 int	ffs_realloccg(struct inode *,
-	    ufs_daddr_t, ufs_daddr_t, int, int, struct ucred *, struct buf **);
+	    ufs_daddr_t, ufs_daddr_t, int, int, kauth_cred_t, struct buf **);
 int	ffs_reclaim(struct vnop_reclaim_args *);
 void	ffs_setblock(struct fs *, u_char *, ufs_daddr_t);
 int	ffs_vfs_getattr(struct mount *, struct vfs_attr *, vfs_context_t);
diff --git a/bsd/ufs/ffs/ffs_inode.c b/bsd/ufs/ffs/ffs_inode.c
index 2887b305c..19342d55b 100644
--- a/bsd/ufs/ffs/ffs_inode.c
+++ b/bsd/ufs/ffs/ffs_inode.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -86,6 +92,7 @@
 #include <ufs/ufs/ufs_byte_order.h>
 #include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
+#include <libkern/OSAtomic.h>
 
 static int ffs_indirtrunc(struct inode *, ufs_daddr_t, ufs_daddr_t,
 	    ufs_daddr_t, int, long *);
@@ -505,7 +512,7 @@ ffs_indirtrunc(ip, lbn, dbn, lastbn, level, countp)
 		trace(TR_BREADHIT, pack(vp, fs->fs_bsize), lbn);
 	} else {
 		trace(TR_BREADMISS, pack(vp, fs->fs_bsize), lbn);
-		current_proc()->p_stats->p_ru.ru_inblock++;	/* pay for read */
+		OSIncrementAtomic(&current_proc()->p_stats->p_ru.ru_inblock);	/* pay for read */
 		buf_setflags(bp,  B_READ);
 		if (buf_count(bp) > buf_size(bp))
 			panic("ffs_indirtrunc: bad buffer size");
diff --git a/bsd/ufs/ffs/ffs_subr.c b/bsd/ufs/ffs/ffs_subr.c
index 6795bba72..d226a5680 100644
--- a/bsd/ufs/ffs/ffs_subr.c
+++ b/bsd/ufs/ffs/ffs_subr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ffs/ffs_tables.c b/bsd/ufs/ffs/ffs_tables.c
index b0bec64c9..6dd00a64a 100644
--- a/bsd/ufs/ffs/ffs_tables.c
+++ b/bsd/ufs/ffs/ffs_tables.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ffs/ffs_vfsops.c b/bsd/ufs/ffs/ffs_vfsops.c
index b462ca7e0..2214f3753 100644
--- a/bsd/ufs/ffs/ffs_vfsops.c
+++ b/bsd/ufs/ffs/ffs_vfsops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -229,9 +235,10 @@ ffs_mount(struct mount *mp, vnode_t devvp, __unused user_addr_t data,  vfs_conte
 			return(0);
 		}
 	}
-	if ((mp->mnt_flag & MNT_UPDATE) == 0)
+	if ((mp->mnt_flag & MNT_UPDATE) == 0) {
+		ufs_ihashinit();
 		error = ffs_mountfs(devvp, mp, context);
-	else {
+	} else {
 		if (devvp != ump->um_devvp)
 			error = EINVAL;	/* needs translation */
 	}
@@ -532,9 +539,6 @@ ffs_mountfs(devvp, mp, context)
 		error = EINVAL;
 		goto out;
 	}
- 
-        /*
-         * Buffer cache does not handle multiple pages in a buf when
 
 	/*
 	 * Buffer cache does not handle multiple pages in a buf when
@@ -997,7 +1001,8 @@ ffs_vfs_getattr(mp, fsap, context)
 		    VOL_CAP_FMT_SPARSE_FILES |
 		    VOL_CAP_FMT_CASE_SENSITIVE |
 		    VOL_CAP_FMT_CASE_PRESERVING |
-		    VOL_CAP_FMT_FAST_STATFS ;
+		    VOL_CAP_FMT_FAST_STATFS |
+		    VOL_CAP_FMT_HIDDEN_FILES ;
 		fsap->f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES]
 		    = VOL_CAP_INT_NFSEXPORT |
 		    VOL_CAP_INT_VOL_RENAME |
@@ -1021,7 +1026,9 @@ ffs_vfs_getattr(mp, fsap, context)
 		    VOL_CAP_FMT_CASE_SENSITIVE |
 		    VOL_CAP_FMT_CASE_PRESERVING |
 		    VOL_CAP_FMT_FAST_STATFS |
-		    VOL_CAP_FMT_2TB_FILESIZE;
+		    VOL_CAP_FMT_2TB_FILESIZE |
+		    VOL_CAP_FMT_OPENDENYMODES |
+		    VOL_CAP_FMT_HIDDEN_FILES ;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] =
 		    VOL_CAP_INT_SEARCHFS |
 		    VOL_CAP_INT_ATTRLIST |
@@ -1032,7 +1039,8 @@ ffs_vfs_getattr(mp, fsap, context)
 		    VOL_CAP_INT_ALLOCATE |
 		    VOL_CAP_INT_VOL_RENAME |
 		    VOL_CAP_INT_ADVLOCK |
-		    VOL_CAP_INT_FLOCK ;
+		    VOL_CAP_INT_FLOCK |
+		    VOL_CAP_INT_MANLOCK;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_RESERVED1] = 0;
 		fsap->f_capabilities.valid[VOL_CAPABILITIES_RESERVED2] = 0;
 
diff --git a/bsd/ufs/ffs/ffs_vnops.c b/bsd/ufs/ffs/ffs_vnops.c
index d33003352..9120f2831 100644
--- a/bsd/ufs/ffs/ffs_vnops.c
+++ b/bsd/ufs/ffs/ffs_vnops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -260,7 +266,7 @@ ffs_fsync_internal(vnode_t vp, int waitfor)
 	/*
 	 * Flush all dirty buffers associated with a vnode.
 	 */
-	buf_flushdirtyblks(vp, wait, 0, (char *)"ffs_fsync");
+	buf_flushdirtyblks(vp, wait, 0, "ffs_fsync");
 	microtime(&tv);
 
 	return (ffs_update(vp, &tv, &tv, wait));
diff --git a/bsd/ufs/ffs/fs.h b/bsd/ufs/ffs/fs.h
index 9b9e18c21..3b9afc7b9 100644
--- a/bsd/ufs/ffs/fs.h
+++ b/bsd/ufs/ffs/fs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ufs/dinode.h b/bsd/ufs/ufs/dinode.h
index 09ff1dfc9..1e8abccec 100644
--- a/bsd/ufs/ufs/dinode.h
+++ b/bsd/ufs/ufs/dinode.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ufs/dir.h b/bsd/ufs/ufs/dir.h
index 9a86ec824..fb78b431d 100644
--- a/bsd/ufs/ufs/dir.h
+++ b/bsd/ufs/ufs/dir.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -85,7 +91,7 @@
  * the length of the entry, and the length of the name contained in
  * the entry.  These are followed by the name padded to a 4 byte boundary
  * with null bytes.  All names are guaranteed null terminated.
- * The maximum length of a name in a directory is MAXNAMLEN.
+ * The maximum length of a name in a directory is UFSMAXNAMLEN.
  *
  * The macro DIRSIZ(fmt, dp) gives the amount of space required to represent
  * a directory entry.  Free space in a directory is represented by
@@ -104,14 +110,14 @@
 #else
 #define DIRBLKSIZ	DEV_BSIZE
 #endif
-#define	MAXNAMLEN	255
+#define	UFSMAXNAMLEN	255
 
 struct	direct {
 	u_int32_t d_ino;		/* inode number of entry */
 	u_int16_t d_reclen;		/* length of this record */
 	u_int8_t  d_type; 		/* file type, see below */
 	u_int8_t  d_namlen;		/* length of string in d_name */
-	char	  d_name[MAXNAMLEN + 1];/* name with length <= MAXNAMLEN */
+	char	  d_name[UFSMAXNAMLEN + 1];/* name with length <= UFSMAXNAMLEN */
 };
 
 /*
@@ -142,18 +148,18 @@ struct	direct {
 #if (BYTE_ORDER == LITTLE_ENDIAN)
 #define DIRSIZ(oldfmt, dp) \
     ((oldfmt) ? \
-    ((sizeof(struct direct) - (MAXNAMLEN+1)) + (((dp)->d_type+1 + 3) &~ 3)) : \
-    ((sizeof(struct direct) - (MAXNAMLEN+1)) + (((dp)->d_namlen+1 + 3) &~ 3)))
+    ((sizeof(struct direct) - (UFSMAXNAMLEN+1)) + (((dp)->d_type+1 + 3) &~ 3)) : \
+    ((sizeof(struct direct) - (UFSMAXNAMLEN+1)) + (((dp)->d_namlen+1 + 3) &~ 3)))
 #else
 #define DIRSIZ(oldfmt, dp) \
-    ((sizeof(struct direct) - (MAXNAMLEN+1)) + (((dp)->d_namlen+1 + 3) &~ 3))
+    ((sizeof(struct direct) - (UFSMAXNAMLEN+1)) + (((dp)->d_namlen+1 + 3) &~ 3))
 #endif
 #define OLDDIRFMT	1
 #define NEWDIRFMT	0
 
 /*
  * Template for manipulating directories.  Should use struct direct's,
- * but the name field is MAXNAMLEN - 1, and this just won't do.
+ * but the name field is UFSMAXNAMLEN - 1, and this just won't do.
  */
 struct dirtemplate {
 	u_int32_t	dot_ino;
diff --git a/bsd/ufs/ufs/inode.h b/bsd/ufs/ufs/inode.h
index 37a5fc619..3a6f212ed 100644
--- a/bsd/ufs/ufs/inode.h
+++ b/bsd/ufs/ufs/inode.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ufs/quota.h b/bsd/ufs/ufs/quota.h
index f48b7f8f5..e8d532b76 100644
--- a/bsd/ufs/ufs/quota.h
+++ b/bsd/ufs/ufs/quota.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
@@ -74,12 +80,18 @@
 struct inode;
 struct mount;
 struct proc;
+
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
 struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
+
 __BEGIN_DECLS
-int	chkdq(struct inode *, int64_t, struct ucred *, int);
-int	chkdqchg(struct inode *, int64_t, struct ucred *, int);
-int	chkiq(struct inode *, long, struct ucred *, int);
-int	chkiqchg(struct inode *, long, struct ucred *, int);
+int	chkdq(struct inode *, int64_t, kauth_cred_t, int);
+int	chkdqchg(struct inode *, int64_t, kauth_cred_t, int);
+int	chkiq(struct inode *, long, kauth_cred_t, int);
+int	chkiqchg(struct inode *, long, kauth_cred_t, int);
 int	getinoquota(struct inode *);
 int	getquota(struct mount *, u_long, int, caddr_t);
 int	qsync(struct mount *mp);
diff --git a/bsd/ufs/ufs/ufs_attrlist.c b/bsd/ufs/ufs/ufs_attrlist.c
index f676c7828..99a0940ee 100644
--- a/bsd/ufs/ufs/ufs_attrlist.c
+++ b/bsd/ufs/ufs/ufs_attrlist.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/bsd/ufs/ufs/ufs_bmap.c b/bsd/ufs/ufs/ufs_bmap.c
index e1652c564..ea9811544 100644
--- a/bsd/ufs/ufs/ufs_bmap.c
+++ b/bsd/ufs/ufs/ufs_bmap.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -84,6 +90,7 @@
 #include <ufs/ufs/ufs_byte_order.h>
 #include <libkern/OSByteOrder.h>
 #endif /* REV_ENDIAN_FS */
+#include <libkern/OSAtomic.h>
 
 
 /*
@@ -210,7 +217,7 @@ ufs_bmaparray(vp, bn, bnp, ap, nump, runp)
 			buf_setblkno(bp, blkptrtodb(ump, (daddr64_t)((unsigned)daddr)));
 			buf_setflags(bp, B_READ);
 			VNOP_STRATEGY(bp);
-			current_proc()->p_stats->p_ru.ru_inblock++;	/* XXX */
+			OSIncrementAtomic(&current_proc()->p_stats->p_ru.ru_inblock);
 			if (error = (int)buf_biowait(bp)) {
 				buf_brelse(bp);
 				return (error);
@@ -363,63 +370,81 @@ ufs_blockmap(ap)
 	int	retsize = 0;
 	int	error = 0;
 	int	nblks;
+	int	lblk_offset;
 
 	ip = VTOI(vp);
 	fs = ip->i_fs;
 	
-	lbn = (ufs_daddr_t)lblkno(fs, ap->a_foffset);
 	devBlockSize = vfs_devblocksize(vnode_mount(vp));
 
-	if (blkoff(fs, ap->a_foffset))
-		panic("ufs_blockmap; allocation requested inside a block");
+	if (ap->a_foffset % devBlockSize)
+		panic("ufs_blockmap; allocation requested inside a device block");
 
 	if (size % devBlockSize)
 		panic("ufs_blockmap: size is not multiple of device block size\n");
 
+	/*
+	 * round down to the beginning of a filesystem block
+	 */
+	lbn = (ufs_daddr_t)lblkno(fs, ap->a_foffset);
+
+	lblk_offset = (int)(ap->a_foffset - lblktosize(fs, lbn));
+
 	if ((error = ufs_bmaparray(vp, lbn, &daddr, NULL, NULL, &nblks)))
 	        return (error);
 
-	if (bnp)
-		*bnp = (daddr64_t)daddr;
-
 	if (ap->a_poff) 
 		*(int *)ap->a_poff = 0;
 
-	if (runp) {
-	        if (lbn < 0) {
-		        /*
-			 * we're dealing with the indirect blocks
-			 * which are always fs_bsize in size
-			 */
-		        retsize = (nblks + 1) * fs->fs_bsize;
-		} else if (daddr == -1 || nblks == 0) {
-		        /*
-			 * we're dealing with a 'hole'... UFS doesn't
-			 * have a clean way to determine it's size
-			 * or
-			 * there's are no physically contiguous blocks
-			 * so
-			 * just return the size of the lbn we started with
-			 */
-		        retsize = blksize(fs, ip, lbn);
-		} else {
-		        /*
-			 * we have 1 or more blocks that are physically contiguous
-			 * to our starting block number... the orignal block + (nblks - 1)
-			 * blocks must be full sized since only the last block can be 
-			 * composed of fragments...
-			 */
-			 retsize = nblks * fs->fs_bsize;
+        if (lbn < 0) {
+	        /*
+		 * we're dealing with the indirect blocks
+		 * which are always fs_bsize in size
+		 */
+	        retsize = (nblks + 1) * fs->fs_bsize;
+	} else if (daddr == -1 || nblks == 0) {
+	        /*
+		 * we're dealing with a 'hole'... UFS doesn't
+		 * have a clean way to determine it's size
+		 * or
+		 * there's are no physically contiguous blocks
+		 * so
+		 * just return the size of the lbn we started with
+		 */
+	        retsize = blksize(fs, ip, lbn);
+	} else {
+	        /*
+		 * we have 1 or more blocks that are physically contiguous
+		 * to our starting block number... the orignal block + (nblks - 1)
+		 * blocks must be full sized since only the last block can be 
+		 * composed of fragments...
+		 */
+		 retsize = nblks * fs->fs_bsize;
+
+		 /*
+		  * now compute the size of the last block and add it in
+		  */
+		 retsize += blksize(fs, ip, (lbn + nblks));
+	}
+	if (lblk_offset) {
+	        if (daddr != -1)
+	                daddr += (lblk_offset / devBlockSize);
 
-			 /*
-			  * now compute the size of the last block and add it in
-			  */
-			 retsize += blksize(fs, ip, (lbn + nblks));
+		if (retsize > lblk_offset)
+		        retsize -= lblk_offset;
+		else {
+		        retsize = 0;
+			daddr = -1;
 		}
+	}
+	if (runp) {
 		if (retsize < size)
 		        *runp = retsize;
 		else
 		        *runp = size;
 	}
+	if (bnp)
+		*bnp = (daddr64_t)daddr;
+
 	return (0);
 }
diff --git a/bsd/ufs/ufs/ufs_byte_order.c b/bsd/ufs/ufs/ufs_byte_order.c
index 59bbf8303..6562d1376 100644
--- a/bsd/ufs/ufs/ufs_byte_order.c
+++ b/bsd/ufs/ufs/ufs_byte_order.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright 1998 Apple Computer, Inc.
  *
diff --git a/bsd/ufs/ufs/ufs_byte_order.h b/bsd/ufs/ufs/ufs_byte_order.h
index 13f909415..009a51de5 100644
--- a/bsd/ufs/ufs/ufs_byte_order.h
+++ b/bsd/ufs/ufs/ufs_byte_order.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright 1998 Apple Computer, Inc.
  *
diff --git a/bsd/ufs/ufs/ufs_extern.h b/bsd/ufs/ufs/ufs_extern.h
index d7e9815c9..3d1cfd1bb 100644
--- a/bsd/ufs/ufs/ufs_extern.h
+++ b/bsd/ufs/ufs/ufs_extern.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -69,7 +75,11 @@ struct mbuf;
 struct mount;
 struct nameidata;
 struct proc;
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
 struct ucred;
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
 struct ufs_args;
 struct uio;
 struct vnode_attr;
@@ -78,11 +88,11 @@ struct vnode;
 
 __BEGIN_DECLS
 int	ufs_remove_internal(vnode_t, vnode_t, struct componentname *, int);
-int	ufs_access_internal(vnode_t, mode_t, ucred_t);
+int	ufs_access_internal(vnode_t, mode_t, kauth_cred_t);
 
 int	ffs_read_internal(vnode_t, struct uio *, int);
-int	ffs_write_internal(vnode_t, struct uio *, int, ucred_t);
-int	ffs_truncate_internal(vnode_t, off_t, int, ucred_t);
+int	ffs_write_internal(vnode_t, struct uio *, int, kauth_cred_t);
+int	ffs_truncate_internal(vnode_t, off_t, int, kauth_cred_t);
 
 void	 diskerr
 	   (struct buf *, char *, char *, int, int, struct disklabel *);
@@ -93,12 +103,12 @@ int	 setdisklabel(struct disklabel *, struct disklabel *, u_long);
 int	 writedisklabel(dev_t, int (*)(), struct disklabel *);
 
 int	 ufs_access(struct vnop_access_args *);
-int	 ufs_checkpath(struct inode *, struct inode *, struct ucred *);
+int	 ufs_checkpath(struct inode *, struct inode *, kauth_cred_t);
 int	 ufs_close(struct vnop_close_args *);
 int	 ufs_create(struct vnop_create_args *);
 void	 ufs_dirbad(struct inode *, doff_t, const char *);
 int	 ufs_dirbadentry(struct vnode *, struct direct *, int);
-int	 ufs_dirempty(struct inode *, ino_t, struct ucred *);
+int	 ufs_dirempty(struct inode *, ino_t, kauth_cred_t);
 int	 ufs_direnter(struct inode *, struct vnode *,struct componentname *);
 int	 ufs_dirremove(struct vnode *, struct componentname*);
 int	 ufs_dirrewrite
diff --git a/bsd/ufs/ufs/ufs_ihash.c b/bsd/ufs/ufs/ufs_ihash.c
index 140f8564a..18909bae7 100644
--- a/bsd/ufs/ufs/ufs_ihash.c
+++ b/bsd/ufs/ufs/ufs_ihash.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -79,7 +85,12 @@ u_long	ihash;		/* size of hash table - 1 */
 void
 ufs_ihashinit()
 {
+	static int done;
+
+	if (done)
+		return;
 
+	done = 1;
 	ihashtbl = hashinit(desiredvnodes, M_UFSMNT, &ihash);
 }
 
diff --git a/bsd/ufs/ufs/ufs_inode.c b/bsd/ufs/ufs/ufs_inode.c
index 41ae10abe..dcd42d9d3 100644
--- a/bsd/ufs/ufs/ufs_inode.c
+++ b/bsd/ufs/ufs/ufs_inode.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/ufs/ufs/ufs_lookup.c b/bsd/ufs/ufs/ufs_lookup.c
index a3ab94a59..77b3857aa 100644
--- a/bsd/ufs/ufs/ufs_lookup.c
+++ b/bsd/ufs/ufs/ufs_lookup.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -160,7 +166,7 @@ ufs_lookup(ap)
 #endif /* REV_ENDIAN_FS */
 
 
-	if (cnp->cn_namelen > MAXNAMLEN)
+	if (cnp->cn_namelen > UFSMAXNAMLEN)
 		return (ENAMETOOLONG);
 
 	cred = vfs_context_ucred(context);
@@ -205,7 +211,7 @@ ufs_lookup(ap)
 	if ((nameiop == CREATE || nameiop == RENAME) &&
 	    (flags & ISLASTCN)) {
 		slotstatus = NONE;
-		slotneeded = (sizeof(struct direct) - MAXNAMLEN +
+		slotneeded = (sizeof(struct direct) - UFSMAXNAMLEN +
 			cnp->cn_namelen + 3) &~ 3;
 	}
 	/*
@@ -265,11 +271,14 @@ searchloop:
 		}
 		/*
 		 * Get pointer to next entry.
-		 * Full validation checks are slow, but necessary.
+		 * Full validation checks are slow, so we only check
+		 * enough to insure forward progress through the
+		 * directory. Complete checks can be run by patching
+		 * "dirchk" to be true.
 		 */
 		ep = (struct direct *)((char *)buf_dataptr(bp) + entryoffsetinblock);
 		if (ep->d_reclen == 0 ||
-		    ufs_dirbadentry(vdp, ep, entryoffsetinblock)) {
+		    dirchk && ufs_dirbadentry(vdp, ep, entryoffsetinblock)) {
 			int i;
 
 			ufs_dirbad(dp, dp->i_offset, "mangled entry");
@@ -566,7 +575,7 @@ ufs_dirbad(ip, offset, how)
  *	record length must be multiple of 4
  *	entry must fit in rest of its DIRBLKSIZ block
  *	record must be large enough to contain entry
- *	name is not longer than MAXNAMLEN
+ *	name is not longer than UFSMAXNAMLEN
  *	name must be as long as advertised, and null terminated
  */
 int
@@ -591,7 +600,7 @@ ufs_dirbadentry(dp, ep, entryoffsetinblock)
 #	endif
 	if ((ep->d_reclen & 0x3) != 0 ||
 	    ep->d_reclen > DIRBLKSIZ - (entryoffsetinblock & (DIRBLKSIZ - 1)) ||
-	    ep->d_reclen < DIRSIZ(FSFMT(dp), ep) || namlen > MAXNAMLEN) {
+	    ep->d_reclen < DIRSIZ(FSFMT(dp), ep) || namlen > UFSMAXNAMLEN) {
 		/*return (1); */
 		printf("First bad\n");
 		goto bad;
diff --git a/bsd/ufs/ufs/ufs_quota.c b/bsd/ufs/ufs/ufs_quota.c
index 8f012e8b2..df236996f 100644
--- a/bsd/ufs/ufs/ufs_quota.c
+++ b/bsd/ufs/ufs/ufs_quota.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993, 1995
@@ -453,6 +459,9 @@ quotaon(context, mp, type, fnamep)
 	int error = 0;
 	struct ufs_quotaon_cargs args;
 
+	/* Finish setting up quota structures. */
+	dqhashinit();
+
 	qfp = &ump->um_qfiles[type];
 
 	if ( (qf_get(qfp, QTF_OPENING)) )
@@ -550,6 +559,12 @@ quotaoff(struct mount *mp, register int type)
 	int error = 0;
 	struct ufs_quotaoff_cargs args;
 	
+	/*
+	 * If quotas haven't been initialized, there's no work to be done.
+	 */
+	if (!dqisinitialized())
+		return (0);
+
 	qfp = &ump->um_qfiles[type];
 
 	if ( (qf_get(qfp, QTF_CLOSING)) )
@@ -762,6 +777,9 @@ qsync(mp)
 	struct ufsmount *ump = VFSTOUFS(mp);
 	int i;
 
+	if (!dqisinitialized())
+		return (0);
+
 	/*
 	 * Check if the mount point has any quotas.
 	 * If not, simply return.
diff --git a/bsd/ufs/ufs/ufs_readwrite.c b/bsd/ufs/ufs/ufs_readwrite.c
index 9aa3b92cd..50fdce582 100644
--- a/bsd/ufs/ufs/ufs_readwrite.c
+++ b/bsd/ufs/ufs/ufs_readwrite.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -120,7 +126,7 @@ ffs_read_internal(vnode_t vp, struct uio *uio, int ioflag)
 		return (EFBIG);
 
 	if (UBCINFOEXISTS(vp)) {
-		error = cluster_read(vp, uio, (off_t)ip->i_size, 0);
+		error = cluster_read(vp, uio, (off_t)ip->i_size, ioflag);
 	} else {
 	for (error = 0, bp = NULL; uio_resid(uio) > 0; 
 	    bp = NULL) {
@@ -193,7 +199,8 @@ ffs_read_internal(vnode_t vp, struct uio *uio, int ioflag)
 	}
 	if (bp != NULL)
 		buf_brelse(bp);
-	ip->i_flag |= IN_ACCESS;
+	if ((vnode_vfsvisflags(vp) & MNT_NOATIME) == 0)
+		ip->i_flag |= IN_ACCESS;
 	return (error);
 }
 
@@ -212,7 +219,7 @@ ffs_write(ap)
 }
 
 
-ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
+ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, kauth_cred_t cred)
 {
 	buf_t	bp;
 	proc_t	p;
@@ -226,6 +233,9 @@ ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
 	int error = 0;
 	int file_extended = 0;
 	int doingdirectory = 0;
+	user_ssize_t clippedsize = 0;	/* Truncate writes near fs->fs_maxfilesize */
+	user_ssize_t residcount, oldcount;
+	int partialwrite=0;
 
 #if REV_ENDIAN_FS
 	int rev_endian=0;
@@ -260,9 +270,18 @@ ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
 	}
 
 	fs = ip->I_FS;
-	if (uio->uio_offset < 0 ||
-	    (u_int64_t)uio->uio_offset + uio_resid(uio) > fs->fs_maxfilesize)
+	if (uio->uio_offset < 0)
+		return (EFBIG);
+	if ( uio_resid(uio) > fs->fs_maxfilesize - uio->uio_offset ) {
+		residcount = uio_resid(uio);
+		clippedsize = residcount - (fs->fs_maxfilesize - uio->uio_offset);
+		if (clippedsize >= residcount) {
 		return (EFBIG);
+		} else {
+			uio_setresid(uio, residcount - clippedsize);
+			partialwrite = 1;
+		}
+	}
 	if (uio_resid(uio) == 0)
 	        return (0);
 
@@ -349,8 +368,7 @@ ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
 			filesize -= rsd;
 	}
 
-	flags = ioflag & IO_SYNC ? IO_SYNC : 0;
-	/* flags |= IO_NOZEROVALID; */
+	flags = ioflag & ~(IO_TAILZEROFILL | IO_HEADZEROFILL | IO_NOZEROVALID | IO_NOZERODIRTY);
 
 	if((error == 0) && fblk && fboff) {
 		if( fblk > fs->fs_bsize) 
@@ -358,7 +376,6 @@ ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
 		/* We need to zero out the head */
 		head_offset = uio->uio_offset - (off_t)fboff ;
 		flags |= IO_HEADZEROFILL;
-		/* flags &= ~IO_NOZEROVALID; */
 	}
 
 	if((error == 0) && blkalloc && ((blkalloc - xfersize) > 0)) {
@@ -470,6 +487,10 @@ ffs_write_internal(vnode_t vp, struct uio *uio, int ioflag, ucred_t cred)
 		microtime(&tv);		
 		error = ffs_update(vp, &tv, &tv, 1);
 	}
+	if (partialwrite) {
+		oldcount = uio_resid(uio);
+		uio_setresid(uio, oldcount + clippedsize);
+	}
 	return (error);
 }
 
@@ -500,12 +521,6 @@ ffs_pagein(ap)
 
 	ip = VTOI(vp);
 
-	/* check pageins for reg file only  and ubc info is present*/
-	if  (UBCINVALID(vp))
-		panic("ffs_pagein: Not a  VREG: vp=%x", vp);
-	if (UBCINFOMISSING(vp))
-		panic("ffs_pagein: No mapping: vp=%x", vp);
-
 #if DIAGNOSTIC
 	if (vp->v_type == VLNK) {
 		if ((int)ip->i_size < vp->v_mount->mnt_maxsymlinklen)
@@ -559,12 +574,6 @@ ffs_pageout(ap)
 
 	ip = VTOI(vp);
 
-	/* check pageouts for reg file only  and ubc info is present*/
-	if  (UBCINVALID(vp))
-		panic("ffs_pageout: Not a  VREG: vp=%x", vp);
-	if (UBCINFOMISSING(vp))
-		panic("ffs_pageout: No mapping: vp=%x", vp);
-
         if (vp->v_mount->mnt_flag & MNT_RDONLY) {
 		if (!nocommit)
   			ubc_upl_abort_range(pl, pl_offset, size, 
diff --git a/bsd/ufs/ufs/ufs_vfsops.c b/bsd/ufs/ufs/ufs_vfsops.c
index 2285303a6..26e59bafd 100644
--- a/bsd/ufs/ufs/ufs_vfsops.c
+++ b/bsd/ufs/ufs/ufs_vfsops.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -195,15 +201,6 @@ int
 ufs_init(vfsp)
 	struct vfsconf *vfsp;
 {
-	static int done;
-
-	if (done)
-		return (0);
-	done = 1;
-	ufs_ihashinit();
-#if QUOTA
-	dqinit();
-#endif
 	return (0);
 }
 
diff --git a/bsd/ufs/ufs/ufs_vnops.c b/bsd/ufs/ufs/ufs_vnops.c
index 846a83253..c3f07dd6d 100644
--- a/bsd/ufs/ufs/ufs_vnops.c
+++ b/bsd/ufs/ufs/ufs_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -564,8 +570,15 @@ ufs_remove_internal(vnode_t dvp, vnode_t vp, struct componentname *cnp, int flag
 
 	if (tvp == NULL)
 	        return (ENOENT);
-	if (tvp != vp)
-	        panic("ufs_remove_internal: relookup returned a different vp");
+	if (tvp != vp) {
+		/*
+		 * The file has already gone away.  Somewhat annoying, but that's
+		 * life in a threaded world.  We need to release the reference we
+		 * got, and then return ENOENT.
+		 */
+		vnode_put(tvp);
+		return ENOENT;
+	}
 	/*
 	 * get rid of reference relookup returned
 	 */
@@ -1848,8 +1861,16 @@ filt_ufsread(struct knote *kn, long hint)
 		kn->kn_data = 1;
 		result = 1;
 	} else {
+		off_t amount;
+
 		ip = VTOI(vp);
-		kn->kn_data = ip->i_size - kn->kn_fp->f_fglob->fg_offset;
+		amount = ip->i_size - kn->kn_fp->f_fglob->fg_offset;
+		if (amount > (off_t)INTPTR_MAX)
+			kn->kn_data = INTPTR_MAX;
+		else if (amount < (off_t)INTPTR_MIN)
+			kn->kn_data = INTPTR_MIN;
+		else
+			kn->kn_data = (intptr_t)amount;
 		result = (kn->kn_data != 0);
 	}
 
@@ -1864,6 +1885,7 @@ filt_ufswrite(struct knote *kn, long hint)
 {
 
 	int dropvp = 0;
+	int result;
 	
 	if (hint == 0)  {
 		if ((vnode_getwithvid(kn->kn_hook, kn->kn_hookid) != 0)) {
@@ -1931,11 +1953,15 @@ ufs_pathconf(ap)
 		*ap->a_retval = PIPE_BUF;
 		return (0);
 	case _PC_CHOWN_RESTRICTED:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_CHOWN_RESTRICTED */
 		return (0);
 	case _PC_NO_TRUNC:
-		*ap->a_retval = 1;
+		*ap->a_retval = 200112;		/* _POSIX_NO_TRUNC */
 		return (0);
+	case _PC_FILESIZEBITS:
+		*ap->a_retval = 34;
+		return (0);
+
 	default:
 		return (EINVAL);
 	}
diff --git a/bsd/ufs/ufs/ufsmount.h b/bsd/ufs/ufs/ufsmount.h
index 79a073abd..6a25572e5 100644
--- a/bsd/ufs/ufs/ufsmount.h
+++ b/bsd/ufs/ufs/ufsmount.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
diff --git a/bsd/uxkern/ux_exception.c b/bsd/uxkern/ux_exception.c
index 678666e39..6eb09940f 100644
--- a/bsd/uxkern/ux_exception.c
+++ b/bsd/uxkern/ux_exception.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -42,6 +48,7 @@
 #include <mach/mach_port.h>
 #include <mach/mig_errors.h>
 #include <mach/exc_server.h>
+#include <mach/mach_exc_server.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/sched_prim.h>
@@ -51,9 +58,9 @@
 #include <sys/user.h>
 #include <sys/systm.h>
 #include <sys/ux_exception.h>
+#include <sys/vmparam.h>	/* MAXSSIZ */
 
 #include <vm/vm_protos.h>	/* get_task_ipcspace() */
-
 /*
  * XXX Things that should be retrieved from Mach headers, but aren't
  */
@@ -74,12 +81,14 @@ extern void ipc_port_release(ipc_port_t);
 
 
 
+
 /*
  *	Unix exception handler.
  */
 
-static void	ux_exception(int exception, int code, int subcode,
-				int *ux_signal, int *ux_code);
+static void	ux_exception(int exception, mach_exception_code_t code, 
+				mach_exception_subcode_t subcode,
+				int *ux_signal, mach_exception_code_t *ux_code);
 
 mach_port_name_t		ux_exception_port;
 static task_t			ux_handler_self;
@@ -92,8 +101,6 @@ ux_handler(void)
     mach_port_name_t	exc_port_name;
     mach_port_name_t	exc_set_name;
 
-    (void) thread_funnel_set(kernel_flock, TRUE);
-
     /* self->kernel_vm_space = TRUE; */
     ux_handler_self = self;
 
@@ -139,7 +146,7 @@ ux_handler(void)
 		NDR_record_t NDR;
 		exception_type_t exception;
 		mach_msg_type_number_t codeCnt;
-		exception_data_t code;
+		mach_exception_data_t code;
 		/* some times RCV_TO_LARGE probs */
 		char pad[512];
 	} exc_msg;
@@ -159,7 +166,7 @@ ux_handler(void)
 	if (result == MACH_MSG_SUCCESS) {
 	    reply_port = (mach_port_name_t)exc_msg.Head.msgh_remote_port;
 
-	    if (exc_server(&exc_msg.Head, &rep_msg.Head))
+	    if (mach_exc_server(&exc_msg.Head, &rep_msg.Head))
 		(void) mach_msg_send(&rep_msg.Head, MACH_SEND_MSG,
 			sizeof (rep_msg),MACH_MSG_TIMEOUT_NONE,MACH_PORT_NULL);
 
@@ -171,7 +178,6 @@ ux_handler(void)
 	else
 		panic("exception_handler");
     }
-	thread_funnel_set(kernel_flock, FALSE);
 }
 
 void
@@ -182,7 +188,7 @@ ux_handler_init(void)
 	if (ux_exception_port == MACH_PORT_NULL)  {
 		assert_wait(&ux_exception_port, THREAD_UNINT);
 		thread_block(THREAD_CONTINUE_NULL);
-		}
+	}
 }
 
 kern_return_t
@@ -195,19 +201,44 @@ catch_exception_raise(
         __unused mach_msg_type_number_t codeCnt
 )
 {
-	task_t		self = current_task();
-	thread_t	th_act;
-	ipc_port_t 	thread_port;
-	kern_return_t	result = MACH_MSG_SUCCESS;
-	int		ux_signal = 0;
-	u_long		ucode = 0;
-	struct uthread *ut;
+	mach_exception_data_type_t big_code[EXCEPTION_CODE_MAX];
+	big_code[0] = code[0];
+	big_code[1] = code[1];
+
+	return catch_mach_exception_raise(exception_port,
+			thread,
+			task,
+			exception,
+			big_code,
+			codeCnt);
+
+}
+
+kern_return_t
+catch_mach_exception_raise(
+        __unused mach_port_t exception_port,
+        mach_port_t thread,
+        mach_port_t task,
+        exception_type_t exception,
+        mach_exception_data_t code,
+        __unused mach_msg_type_number_t codeCnt
+)
+{
+	task_t			self = current_task();
+	thread_t		th_act;
+	ipc_port_t 		thread_port;
+	struct task		*sig_task;
+	struct proc		*p;
+	kern_return_t		result = MACH_MSG_SUCCESS;
+	int			ux_signal = 0;
+	mach_exception_code_t 	ucode = 0;
+	struct uthread 		*ut;
 	mach_port_name_t thread_name = (mach_port_name_t)thread; /* XXX */
 	mach_port_name_t task_name = (mach_port_name_t)task;	/* XXX */
 
-   /*
-     *	Convert local thread name to global port.
-     */
+	/*
+	 *	Convert local thread name to global port.
+	 */
    if (MACH_PORT_VALID(thread_name) &&
        (ipc_object_copyin(get_task_ipcspace(self), thread_name,
 		       MACH_MSG_TYPE_PORT_SEND,
@@ -223,14 +254,75 @@ catch_exception_raise(
 	 *	Catch bogus ports
 	 */
 	if (th_act != THREAD_NULL) {
-  
+
 	    /*
 	     *	Convert exception to unix signal and code.
 	     */
-		ut = get_bsdthread_info(th_act);
-	    ux_exception(exception, code[0], code[1],
-	    			&ux_signal, (int *)&ucode);
+	    ux_exception(exception, code[0], code[1], &ux_signal, &ucode);
+
+	    ut = get_bsdthread_info(th_act);
+	    sig_task = get_threadtask(th_act);
+	    p = (struct proc *) get_bsdtask_info(sig_task);
+
+	    /* Can't deliver a signal without a bsd process */
+	    if (p == NULL) {
+		    ux_signal = 0;
+		    result = KERN_FAILURE;
+	    }
 
+	    /*
+	     * Stack overflow should result in a SIGSEGV signal
+	     * on the alternate stack.
+	     * but we have one or more guard pages after the
+	     * stack top, so we would get a KERN_PROTECTION_FAILURE
+	     * exception instead of KERN_INVALID_ADDRESS, resulting in
+	     * a SIGBUS signal.
+	     * Detect that situation and select the correct signal.
+	     */
+	    if (code[0] == KERN_PROTECTION_FAILURE &&
+		ux_signal == SIGBUS) {
+		    user_addr_t		sp, stack_min, stack_max;
+		    int			mask;
+		    struct sigacts	*ps;
+
+		    sp = code[1];
+		    if (ut && (ut->uu_flag & UT_VFORK))
+			    p = ut->uu_proc;
+#if STACK_GROWTH_UP
+		    stack_min = p->user_stack;
+		    stack_max = p->user_stack + MAXSSIZ;
+#else /* STACK_GROWTH_UP */
+		    stack_max = p->user_stack;
+		    stack_min = p->user_stack - MAXSSIZ;
+#endif /* STACK_GROWTH_UP */
+		    if (sp >= stack_min &&
+			sp < stack_max) {
+			    /*
+			     * This is indeed a stack overflow.  Deliver a
+			     * SIGSEGV signal.
+			     */
+			    ux_signal = SIGSEGV;
+
+			    /*
+			     * If the thread/process is not ready to handle
+			     * SIGSEGV on an alternate stack, force-deliver
+			     * SIGSEGV with a SIG_DFL handler.
+			     */
+			    mask = sigmask(ux_signal);
+			    ps = p->p_sigacts;
+			    if ((p->p_sigignore & mask) ||
+				(ut->uu_sigwait & mask) ||
+				(ut->uu_sigmask & mask) ||
+				(ps->ps_sigact[SIGSEGV] == SIG_IGN) ||
+				(! (ps->ps_sigonstack & mask))) {
+				    p->p_sigignore &= ~mask;
+				    p->p_sigcatch &= ~mask;
+				    ps->ps_sigact[SIGSEGV] = SIG_DFL;
+				    ut->uu_sigwait &= ~mask;
+				    ut->uu_sigmask &= ~mask;
+			    }
+		    }
+	    }
 	    /*
 	     *	Send signal.
 	     */
@@ -239,7 +331,7 @@ catch_exception_raise(
 			//ut->uu_code = code[0]; // filled in by threadsignal
 			ut->uu_subcode = code[1];			
 			threadsignal(th_act, ux_signal, code[0]);
-		}
+	    }
 
 	    thread_deallocate(th_act);
 	}
@@ -273,6 +365,21 @@ catch_exception_raise_state(
 	return(KERN_INVALID_ARGUMENT);
 }
 
+kern_return_t
+catch_mach_exception_raise_state(
+        __unused mach_port_t exception_port,
+        __unused exception_type_t exception,
+        __unused const mach_exception_data_t code,
+        __unused mach_msg_type_number_t codeCnt,
+        __unused int *flavor,
+        __unused const thread_state_t old_state,
+        __unused mach_msg_type_number_t old_stateCnt,
+        __unused thread_state_t new_state,
+        __unused mach_msg_type_number_t *new_stateCnt)
+{
+	return(KERN_INVALID_ARGUMENT);
+}
+
 kern_return_t
 catch_exception_raise_state_identity(
         __unused mach_port_t exception_port,
@@ -290,6 +397,24 @@ catch_exception_raise_state_identity(
 	return(KERN_INVALID_ARGUMENT);
 }
 
+kern_return_t
+catch_mach_exception_raise_state_identity(
+        __unused mach_port_t exception_port,
+        __unused mach_port_t thread,
+        __unused mach_port_t task,
+        __unused exception_type_t exception,
+        __unused mach_exception_data_t code,
+        __unused mach_msg_type_number_t codeCnt,
+        __unused int *flavor,
+        __unused thread_state_t old_state,
+        __unused mach_msg_type_number_t old_stateCnt,
+        __unused thread_state_t new_state,
+        __unused mach_msg_type_number_t *new_stateCnt)
+{
+	return(KERN_INVALID_ARGUMENT);
+}
+
+
 /*
  *	ux_exception translates a mach exception, code and subcode to
  *	a signal and u.u_code.  Calls machine_exception (machine dependent)
@@ -298,12 +423,11 @@ catch_exception_raise_state_identity(
 
 static
 void ux_exception(
-    int			exception,
-    int			code,
-    int			subcode,
-    int			*ux_signal,
-    int			*ux_code
-)
+		int			exception,
+		mach_exception_code_t 	code,
+		mach_exception_subcode_t subcode,
+		int			*ux_signal,
+		mach_exception_code_t 	*ux_code)
 {
     /*
      *	Try machine-dependent translation first.
diff --git a/bsd/vfs/kpi_vfs.c b/bsd/vfs/kpi_vfs.c
index 709bc0467..a1832c532 100644
--- a/bsd/vfs/kpi_vfs.c
+++ b/bsd/vfs/kpi_vfs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,13 +65,17 @@
  *
  *	@(#)kpi_vfs.c
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * External virtual filesystem routines
  */
 
-#undef	DIAGNOSTIC
-#define DIAGNOSTIC 1
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -95,6 +105,7 @@
 
 #include <kern/assert.h>
 #include <kern/kalloc.h>
+#include <kern/task.h>
 
 #include <libkern/OSByteOrder.h>
 
@@ -102,6 +113,11 @@
 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
+#include <mach/task.h>
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 #define ESUCCESS 0
 #undef mount_t
@@ -114,12 +130,13 @@
 	((VP)->v_unsafefs ? 0 : 1)
 
 #define NATIVE_XATTR(VP)  \
-	((VP)->v_mount ? (VP)->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSNATIVEXATTR : 0)
+	((VP)->v_mount ? (VP)->v_mount->mnt_kern_flag & MNTK_EXTENDED_ATTRS : 0)
 
-static void xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context,
-                             int thread_safe, int force);
-static void xattrfile_setattr(vnode_t dvp, const char * basename, struct vnode_attr * vap,
-                              vfs_context_t context, int thread_safe);
+static void xattrfile_remove(vnode_t dvp, const char *basename,
+				vfs_context_t ctx, int thread_safe, int force);
+static void xattrfile_setattr(vnode_t dvp, const char * basename,
+				struct vnode_attr * vap, vfs_context_t ctx,
+				int thread_safe);
 
 
 static void
@@ -127,7 +144,7 @@ vnode_setneedinactive(vnode_t vp)
 {
         cache_purge(vp);
 
-        vnode_lock(vp);
+        vnode_lock_spin(vp);
 	vp->v_lflag |= VL_NEEDINACTIVE;
 	vnode_unlock(vp);
 }
@@ -183,7 +200,7 @@ unlock_fsnode(vnode_t vp, int *funnel_state)
  * prototypes for exported VFS operations
  */
 int 
-VFS_MOUNT(struct mount * mp, vnode_t devvp, user_addr_t data, vfs_context_t context)
+VFS_MOUNT(mount_t mp, vnode_t devvp, user_addr_t data, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
@@ -199,16 +216,16 @@ VFS_MOUNT(struct mount * mp, vnode_t devvp, user_addr_t data, vfs_context_t cont
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
 	
-	if (vfs_context_is64bit(context)) {
+	if (vfs_context_is64bit(ctx)) {
 		if (vfs_64bitready(mp)) {
-			error = (*mp->mnt_op->vfs_mount)(mp, devvp, data, context);
+			error = (*mp->mnt_op->vfs_mount)(mp, devvp, data, ctx);
 		}
 		else {
 			error = ENOTSUP;
 		}
 	}
 	else {
-		error = (*mp->mnt_op->vfs_mount)(mp, devvp, data, context);
+		error = (*mp->mnt_op->vfs_mount)(mp, devvp, data, ctx);
 	}
 	
 	if (!thread_safe) {
@@ -218,7 +235,7 @@ VFS_MOUNT(struct mount * mp, vnode_t devvp, user_addr_t data, vfs_context_t cont
 }
 
 int 
-VFS_START(struct mount * mp, int flags, vfs_context_t context)
+VFS_START(mount_t mp, int flags, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
@@ -232,7 +249,7 @@ VFS_START(struct mount * mp, int flags, vfs_context_t context)
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_start)(mp, flags, context);
+	error = (*mp->mnt_op->vfs_start)(mp, flags, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -240,7 +257,7 @@ VFS_START(struct mount * mp, int flags, vfs_context_t context)
 }
 
 int 
-VFS_UNMOUNT(struct mount *mp, int flags, vfs_context_t context)
+VFS_UNMOUNT(mount_t mp, int flags, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
@@ -254,35 +271,49 @@ VFS_UNMOUNT(struct mount *mp, int flags, vfs_context_t context)
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_unmount)(mp, flags, context);
+	error = (*mp->mnt_op->vfs_unmount)(mp, flags, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOTSUP			Not supported
+ *		<vfs_root>:ENOENT
+ *		<vfs_root>:???
+ *
+ * Note:	The return codes from the underlying VFS's root routine can't
+ *		be fully enumerated here, since third party VFS authors may not
+ *		limit their error returns to the ones documented here, even
+ *		though this may result in some programs functioning incorrectly.
+ *
+ *		The return codes documented above are those which may currently
+ *		be returned by HFS from hfs_vfs_root, which is a simple wrapper
+ *		for a call to hfs_vget on the volume mount poit, not including
+ *		additional error codes which may be propagated from underlying
+ *		routines called by hfs_vget.
+ */
 int 
-VFS_ROOT(struct mount * mp, struct vnode  ** vpp, vfs_context_t context)
+VFS_ROOT(mount_t mp, struct vnode  ** vpp, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_root == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_root)(mp, vpp, context);
+	error = (*mp->mnt_op->vfs_root)(mp, vpp, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -290,7 +321,7 @@ VFS_ROOT(struct mount * mp, struct vnode  ** vpp, vfs_context_t context)
 }
 
 int 
-VFS_QUOTACTL(struct mount *mp, int cmd, uid_t uid, caddr_t datap, vfs_context_t context)
+VFS_QUOTACTL(mount_t mp, int cmd, uid_t uid, caddr_t datap, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
@@ -304,7 +335,7 @@ VFS_QUOTACTL(struct mount *mp, int cmd, uid_t uid, caddr_t datap, vfs_context_t
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_quotactl)(mp, cmd, uid, datap, context);
+	error = (*mp->mnt_op->vfs_quotactl)(mp, cmd, uid, datap, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -312,27 +343,25 @@ VFS_QUOTACTL(struct mount *mp, int cmd, uid_t uid, caddr_t datap, vfs_context_t
 }
 
 int 
-VFS_GETATTR(struct mount *mp, struct vfs_attr *vfa, vfs_context_t context)
+VFS_GETATTR(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_getattr == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
+
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 	
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_getattr)(mp, vfa, context);
+	error = (*mp->mnt_op->vfs_getattr)(mp, vfa, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -340,27 +369,25 @@ VFS_GETATTR(struct mount *mp, struct vfs_attr *vfa, vfs_context_t context)
 }
 
 int 
-VFS_SETATTR(struct mount *mp, struct vfs_attr *vfa, vfs_context_t context)
+VFS_SETATTR(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_setattr == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
+
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 	
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_setattr)(mp, vfa, context);
+	error = (*mp->mnt_op->vfs_setattr)(mp, vfa, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -368,27 +395,24 @@ VFS_SETATTR(struct mount *mp, struct vfs_attr *vfa, vfs_context_t context)
 }
 
 int 
-VFS_SYNC(struct mount *mp, int flags, vfs_context_t context)
+VFS_SYNC(mount_t mp, int flags, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_sync == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_sync)(mp, flags, context);
+	error = (*mp->mnt_op->vfs_sync)(mp, flags, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -396,27 +420,24 @@ VFS_SYNC(struct mount *mp, int flags, vfs_context_t context)
 }
 
 int 
-VFS_VGET(struct mount * mp, ino64_t ino, struct vnode **vpp, vfs_context_t context)
+VFS_VGET(mount_t mp, ino64_t ino, struct vnode **vpp, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_vget == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_vget)(mp, ino, vpp, context);
+	error = (*mp->mnt_op->vfs_vget)(mp, ino, vpp, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -424,27 +445,24 @@ VFS_VGET(struct mount * mp, ino64_t ino, struct vnode **vpp, vfs_context_t conte
 }
 
 int 
-VFS_FHTOVP(struct mount * mp, int fhlen, unsigned char * fhp, vnode_t * vpp, vfs_context_t context) 
+VFS_FHTOVP(mount_t mp, int fhlen, unsigned char * fhp, vnode_t * vpp, vfs_context_t ctx) 
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((mp == dead_mountp) || (mp->mnt_op->vfs_fhtovp == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	thread_safe = mp->mnt_vtable->vfc_threadsafe;
 
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*mp->mnt_op->vfs_fhtovp)(mp, fhlen, fhp, vpp, context);
+	error = (*mp->mnt_op->vfs_fhtovp)(mp, fhlen, fhp, vpp, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -452,27 +470,24 @@ VFS_FHTOVP(struct mount * mp, int fhlen, unsigned char * fhp, vnode_t * vpp, vfs
 }
 
 int 
-VFS_VPTOFH(struct vnode * vp, int *fhlenp, unsigned char * fhp, vfs_context_t context)
+VFS_VPTOFH(struct vnode * vp, int *fhlenp, unsigned char * fhp, vfs_context_t ctx)
 {
 	int error;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
 	if ((vp->v_mount == dead_mountp) || (vp->v_mount->mnt_op->vfs_vptofh == 0))
 		return(ENOTSUP);
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	}
-	error = (*vp->v_mount->mnt_op->vfs_vptofh)(vp, fhlenp, fhp, context);
+	error = (*vp->v_mount->mnt_op->vfs_vptofh)(vp, fhlenp, fhp, ctx);
 	if (!thread_safe) {
 		(void) thread_funnel_set(kernel_flock, funnel_state);
 	}
@@ -508,7 +523,9 @@ vfs_setflags(mount_t mp, uint64_t flags)
 {
 	uint32_t lflags = (uint32_t)(flags & (MNT_CMDFLAGS | MNT_VISFLAGMASK)); 
 
+	mount_lock(mp);
 	mp->mnt_flag |= lflags;
+	mount_unlock(mp);
 }
 
 /* clear any of the command modifier flags(MNT_CMDFLAGS) in mount_t */
@@ -517,7 +534,9 @@ vfs_clearflags(mount_t mp , uint64_t flags)
 {
 	uint32_t lflags = (uint32_t)(flags & (MNT_CMDFLAGS | MNT_VISFLAGMASK)); 
 
+	mount_lock(mp);
 	mp->mnt_flag &= ~lflags;
+	mount_unlock(mp);
 }
 
 /* Is the mount_t ronly and upgrade read/write requested? */
@@ -569,7 +588,7 @@ vfs_isreload(mount_t mp)
 int 
 vfs_isforce(mount_t mp)
 {
-	if ((mp->mnt_flag & MNT_FORCE) || (mp->mnt_kern_flag & MNTK_FRCUNMOUNT))
+	if ((mp->mnt_lflag & MNT_LFORCE) || (mp->mnt_kern_flag & MNTK_FRCUNMOUNT))
 		return(1);
 	else
 		return(0);
@@ -584,6 +603,49 @@ vfs_64bitready(mount_t mp)
 		return(0);
 }
 
+
+int
+vfs_authcache_ttl(mount_t mp)
+{
+        if ( (mp->mnt_kern_flag & (MNTK_AUTH_OPAQUE | MNTK_AUTH_CACHE_TTL)) )
+	        return (mp->mnt_authcache_ttl);
+	else
+	        return (CACHED_RIGHT_INFINITE_TTL);
+}
+
+void
+vfs_setauthcache_ttl(mount_t mp, int ttl)
+{
+	mount_lock(mp);
+	mp->mnt_kern_flag |= MNTK_AUTH_CACHE_TTL;
+	mp->mnt_authcache_ttl = ttl;
+	mount_unlock(mp);
+}
+
+void
+vfs_clearauthcache_ttl(mount_t mp)
+{
+	mount_lock(mp);
+	mp->mnt_kern_flag &= ~MNTK_AUTH_CACHE_TTL;
+	/*
+	 * back to the default TTL value in case
+	 * MNTK_AUTH_OPAQUE is set on this mount
+	 */
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
+	mount_unlock(mp);
+}
+
+void
+vfs_markdependency(mount_t mp)
+{
+	proc_t p = current_proc();
+	mount_lock(mp);
+	mp->mnt_dependent_process = p;
+	mp->mnt_dependent_pid = proc_pid(p);
+	mount_unlock(mp);
+}
+
+
 int
 vfs_authopaque(mount_t mp)
 {
@@ -681,7 +743,6 @@ int
 vfs_getattr(mount_t mp, struct vfs_attr *vfa, vfs_context_t ctx)
 {
 	int		error;
-	char		*vname;
 
 	if ((error = VFS_GETATTR(mp, vfa, ctx)) != 0)
 		return(error);
@@ -726,7 +787,9 @@ vfs_fsprivate(mount_t mp)
 void 
 vfs_setfsprivate(mount_t mp, void *mntdata)
 {
+	mount_lock(mp);
 	mp->mnt_data = mntdata;
+	mount_unlock(mp);
 }
 
 
@@ -755,6 +818,7 @@ vfs_ioattr(mount_t mp, struct vfsioattr *ioattrp)
 		ioattrp->io_maxsegreadsize  = MAXPHYS;
 		ioattrp->io_maxsegwritesize = MAXPHYS;
 		ioattrp->io_devblocksize = DEV_BSIZE;
+		ioattrp->io_flags = 0;
 	} else {
 	        ioattrp->io_maxreadcnt  = mp->mnt_maxreadcnt;
 		ioattrp->io_maxwritecnt = mp->mnt_maxwritecnt;
@@ -763,10 +827,10 @@ vfs_ioattr(mount_t mp, struct vfsioattr *ioattrp)
 		ioattrp->io_maxsegreadsize  = mp->mnt_maxsegreadsize;
 		ioattrp->io_maxsegwritesize = mp->mnt_maxsegwritesize;
 		ioattrp->io_devblocksize = mp->mnt_devblocksize;
+		ioattrp->io_flags = mp->mnt_ioflags;
 	}
-	ioattrp->io_reserved[0] = 0;
-	ioattrp->io_reserved[1] = 0;
-	ioattrp->io_reserved[2] = 0;
+	ioattrp->io_reserved[0] = NULL;
+	ioattrp->io_reserved[1] = NULL;
 }
 
 
@@ -785,6 +849,7 @@ vfs_setioattr(mount_t mp, struct vfsioattr * ioattrp)
 	mp->mnt_maxsegreadsize = ioattrp->io_maxsegreadsize;
 	mp->mnt_maxsegwritesize = ioattrp->io_maxsegwritesize;
 	mp->mnt_devblocksize = ioattrp->io_devblocksize;
+	mp->mnt_ioflags = ioattrp->io_flags;
 }
  
 /*
@@ -846,11 +911,19 @@ vfs_fsadd(struct vfs_fsentry *vfe, vfstable_t * handle)
 		newvfstbl->vfc_threadsafe= 1;
 	if ((vfe->vfe_flags & VFS_TBLLOCALVOL) == VFS_TBLLOCALVOL)
 		newvfstbl->vfc_flags |= MNT_LOCAL;
-	if (vfe->vfe_flags & VFS_TBLLOCALVOL)
+	if ((vfe->vfe_flags & VFS_TBLLOCALVOL) && (vfe->vfe_flags & VFS_TBLGENERICMNTARGS) == 0)
 		newvfstbl->vfc_vfsflags |= VFC_VFSLOCALARGS;
 	else
 		newvfstbl->vfc_vfsflags |= VFC_VFSGENERICARGS;
-		
+
+	if (vfe->vfe_flags &  VFS_TBLNATIVEXATTR)
+		newvfstbl->vfc_vfsflags |= VFC_VFSNATIVEXATTR;
+	if (vfe->vfe_flags &  VFS_TBLUNMOUNT_PREFLIGHT)
+		newvfstbl->vfc_vfsflags |= VFC_VFSPREFLIGHT;
+	if (vfe->vfe_flags &  VFS_TBLREADDIR_EXTENDED)
+		newvfstbl->vfc_vfsflags |= VFC_VFSREADDIR_EXTENDED;
+	if (vfe->vfe_flags & VFS_TBLNOMACLABEL)
+		newvfstbl->vfc_vfsflags |= VFC_VFSNOMACLABEL;
 
 	/*
 	 * Allocate and init the vectors.
@@ -1007,52 +1080,200 @@ vfs_mountrele(__unused mount_t mp ) /* drops reference */
 }
 
 int
-vfs_context_pid(vfs_context_t context)
+vfs_context_pid(vfs_context_t ctx)
 {
-	return (context->vc_proc->p_pid);
+	return (proc_pid(vfs_context_proc(ctx)));
 }
 
 int
-vfs_context_suser(vfs_context_t context)
+vfs_context_suser(vfs_context_t ctx)
 {
-	return (suser(context->vc_ucred, 0));
+	return (suser(ctx->vc_ucred, NULL));
 }
+
+/*
+ * XXX Signals should be tied to threads, not processes, for most uses of this
+ * XXX call.
+ */
 int
-vfs_context_issignal(vfs_context_t context, sigset_t mask)
+vfs_context_issignal(vfs_context_t ctx, sigset_t mask)
 {
-	if (context->vc_proc)
-		return(proc_pendingsignals(context->vc_proc, mask));
+	proc_t p = vfs_context_proc(ctx);
+	if (p)
+		return(proc_pendingsignals(p, mask));
 	return(0);
 }
 
 int
-vfs_context_is64bit(vfs_context_t context)
+vfs_context_is64bit(vfs_context_t ctx)
 {
-	if (context->vc_proc)
-		return(proc_is64bit(context->vc_proc));
+	proc_t proc = vfs_context_proc(ctx);
+
+	if (proc)
+		return(proc_is64bit(proc));
 	return(0);
 }
 
+
+/*
+ * vfs_context_proc
+ *
+ * Description:	Given a vfs_context_t, return the proc_t associated with it.
+ *
+ * Parameters:	vfs_context_t			The context to use
+ *
+ * Returns:	proc_t				The process for this context
+ *
+ * Notes:	This function will return the current_proc() if any of the
+ *		following conditions are true:
+ *
+ *		o	The supplied context pointer is NULL
+ *		o	There is no Mach thread associated with the context
+ *		o	There is no Mach task associated with the Mach thread
+ *		o	There is no proc_t associated with the Mach task
+ *		o	The proc_t has no per process open file table
+ *		o	The proc_t is post-vfork()
+ *
+ *		This causes this function to return a value matching as
+ *		closely as possible the previous behaviour, while at the
+ *		same time avoiding the task lending that results from vfork()
+ */
 proc_t
-vfs_context_proc(vfs_context_t context)
+vfs_context_proc(vfs_context_t ctx)
+{
+	proc_t	proc = NULL;
+
+	if (ctx != NULL && ctx->vc_thread != NULL)
+		proc = (proc_t)get_bsdthreadtask_info(ctx->vc_thread);
+	if (proc != NULL && (proc->p_fd == NULL || (proc->p_lflag & P_LVFORK)))
+		proc = NULL;
+
+	return(proc == NULL ? current_proc() : proc);
+}
+
+/*
+ * vfs_context_get_special_port
+ *
+ * Description: Return the requested special port from the task associated
+ * 		with the given context.
+ *
+ * Parameters:	vfs_context_t			The context to use
+ * 		int				Index of special port
+ * 		ipc_port_t *			Pointer to returned port
+ *
+ * Returns:	kern_return_t			see task_get_special_port()
+ */
+kern_return_t
+vfs_context_get_special_port(vfs_context_t ctx, int which, ipc_port_t *portp)
+{
+	task_t			task = NULL;
+
+	if (ctx != NULL && ctx->vc_thread != NULL)
+		task = get_threadtask(ctx->vc_thread);
+
+	return task_get_special_port(task, which, portp);
+}
+
+/*
+ * vfs_context_set_special_port
+ *
+ * Description: Set the requested special port in the task associated
+ * 		with the given context.
+ *
+ * Parameters:	vfs_context_t			The context to use
+ * 		int				Index of special port
+ * 		ipc_port_t			New special port
+ *
+ * Returns:	kern_return_t			see task_set_special_port()
+ */
+kern_return_t
+vfs_context_set_special_port(vfs_context_t ctx, int which, ipc_port_t port)
+{
+	task_t			task = NULL;
+
+	if (ctx != NULL && ctx->vc_thread != NULL)
+		task = get_threadtask(ctx->vc_thread);
+
+	return task_set_special_port(task, which, port);
+}
+
+/*
+ * vfs_context_thread
+ *
+ * Description:	Return the Mach thread associated with a vfs_context_t
+ *
+ * Parameters:	vfs_context_t			The context to use
+ *
+ * Returns:	thread_t			The thread for this context, or
+ *						NULL, if there is not one.
+ *
+ * Notes:	NULL thread_t's are legal, but discouraged.  They occur only
+ *		as a result of a static vfs_context_t declaration in a function
+ *		and will result in this function returning NULL.
+ *
+ *		This is intentional; this function should NOT return the
+ *		current_thread() in this case.
+ */
+thread_t
+vfs_context_thread(vfs_context_t ctx)
 {
-	return (context->vc_proc);
+	return(ctx->vc_thread);
+}
+
+
+/*
+ * vfs_context_cwd
+ *
+ * Description:	Returns a reference on the vnode for the current working
+ *		directory for the supplied context
+ *
+ * Parameters:	vfs_context_t			The context to use
+ *
+ * Returns:	vnode_t				The current working directory
+ *						for this context
+ *
+ * Notes:	The function first attempts to obtain the current directory
+ *		from the thread, and if it is not present there, falls back
+ *		to obtaining it from the process instead.  If it can't be
+ *		obtained from either place, we return NULLVP.
+ */
+vnode_t
+vfs_context_cwd(vfs_context_t ctx)
+{
+	vnode_t cwd = NULLVP;
+
+	if(ctx != NULL && ctx->vc_thread != NULL) {
+		uthread_t uth = get_bsdthread_info(ctx->vc_thread);
+		proc_t proc;
+
+		/*
+		 * Get the cwd from the thread; if there isn't one, get it
+		 * from the process, instead.
+		 */
+		if ((cwd = uth->uu_cdir) == NULLVP &&
+		    (proc = (proc_t)get_bsdthreadtask_info(ctx->vc_thread)) != NULL &&
+		    proc->p_fd != NULL)
+			cwd = proc->p_fd->fd_cdir;
+	}
+
+	return(cwd);
 }
 
+
 vfs_context_t
-vfs_context_create(vfs_context_t context)
+vfs_context_create(vfs_context_t ctx)
 {
-	struct vfs_context *  newcontext;
+	vfs_context_t newcontext;
 
-	newcontext = (struct vfs_context *)kalloc(sizeof(struct vfs_context));
+	newcontext = (vfs_context_t)kalloc(sizeof(struct vfs_context));
 
 	if (newcontext) {
 		kauth_cred_t safecred;
-		if (context) {
-			newcontext->vc_proc = context->vc_proc;
-			safecred = context->vc_ucred;
+		if (ctx) {
+			newcontext->vc_thread = ctx->vc_thread;
+			safecred = ctx->vc_ucred;
 		} else {
-			newcontext->vc_proc = proc_self();
+			newcontext->vc_thread = current_thread();
 			safecred = kauth_cred_get();
 		}
 		if (IS_VALID_CRED(safecred))
@@ -1060,34 +1281,81 @@ vfs_context_create(vfs_context_t context)
 		newcontext->vc_ucred = safecred;
 		return(newcontext);
 	}
-	return((vfs_context_t)0);	
+	return(NULL);	
+}
+
+
+vfs_context_t
+vfs_context_current(void)
+{
+	vfs_context_t ctx = NULL;
+	volatile uthread_t ut = (uthread_t)get_bsdthread_info(current_thread());
+
+	if (ut != NULL ) {
+		if (ut->uu_context.vc_ucred != NULL) {
+			ctx = &ut->uu_context;
+		}
+	}
+
+	return(ctx == NULL ? vfs_context_kernel() : ctx);
+}
+
+
+/*
+ * XXX Do not ask
+ *
+ * Dangerous hack - adopt the first kernel thread as the current thread, to
+ * get to the vfs_context_t in the uthread associated with a kernel thread.
+ * This is used by UDF to make the call into IOCDMediaBSDClient,
+ * IOBDMediaBSDClient, and IODVDMediaBSDClient to determine whether the
+ * ioctl() is being called from kernel or user space (and all this because
+ * we do not pass threads into our ioctl()'s, instead of processes).
+ *
+ * This is also used by imageboot_setup(), called early from bsd_init() after
+ * kernproc has been given a credential.
+ *
+ * Note: The use of proc_thread() here is a convenience to avoid inclusion
+ * of many Mach headers to do the reference directly rather than indirectly;
+ * we will need to forego this convenience when we reture proc_thread().
+ */
+static struct vfs_context kerncontext;
+vfs_context_t
+vfs_context_kernel(void)
+{
+	if (kerncontext.vc_ucred == NOCRED)
+		kerncontext.vc_ucred = kernproc->p_ucred;
+	if (kerncontext.vc_thread == NULL)
+		kerncontext.vc_thread = proc_thread(kernproc);
+
+	return(&kerncontext);
 }
 
+
 int
-vfs_context_rele(vfs_context_t context)
+vfs_context_rele(vfs_context_t ctx)
 {
-	if (context) {
-		if (IS_VALID_CRED(context->vc_ucred))
-			kauth_cred_unref(&context->vc_ucred);
-		kfree(context, sizeof(struct vfs_context));
+	if (ctx) {
+		if (IS_VALID_CRED(ctx->vc_ucred))
+			kauth_cred_unref(&ctx->vc_ucred);
+		kfree(ctx, sizeof(struct vfs_context));
 	}
 	return(0);
 }
 
 
 ucred_t
-vfs_context_ucred(vfs_context_t context)
+vfs_context_ucred(vfs_context_t ctx)
 {
-	return (context->vc_ucred);
+	return (ctx->vc_ucred);
 }
 
 /*
  * Return true if the context is owned by the superuser.
  */
 int
-vfs_context_issuser(vfs_context_t context)
+vfs_context_issuser(vfs_context_t ctx)
 {
-	return(context->vc_ucred->cr_uid == 0);
+	return(kauth_cred_issuser(vfs_context_ucred(ctx)));
 }
 
 
@@ -1178,7 +1446,7 @@ vnode_fsnode(vnode_t vp)
 void 
 vnode_clearfsnode(vnode_t vp)
 {
-	vp->v_data = 0;
+	vp->v_data = NULL;
 }
 
 dev_t 
@@ -1203,6 +1471,13 @@ vnode_issystem(vnode_t vp)
 	return ((vp->v_flag & VSYSTEM)? 1 : 0);
 }
 
+/* is vnode_t a swap file vnode */
+int 
+vnode_isswap(vnode_t vp)
+{
+	return ((vp->v_flag & VSWAP)? 1 : 0);
+}
+
 /* if vnode_t mount operation in progress */
 int 
 vnode_ismount(vnode_t vp)
@@ -1216,7 +1491,7 @@ vnode_isrecycled(vnode_t vp)
 {
 	int ret;
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	ret =  (vp->v_lflag & (VL_TERMINATE|VL_DEAD))? 1 : 0;
 	vnode_unlock(vp);
 	return(ret);
@@ -1238,6 +1513,12 @@ vnode_isnoreadahead(vnode_t vp)
 	return ((vp->v_flag & VRAOFF)? 1 : 0);
 }
 
+int
+vnode_is_openevt(vnode_t vp)
+{
+	return ((vp->v_flag & VOPENEVT)? 1 : 0);
+}
+
 /* is vnode_t a standard one? */
 int 
 vnode_isstandard(vnode_t vp)
@@ -1301,12 +1582,28 @@ vnode_issock(vnode_t vp)
 	return ((vp->v_type == VSOCK)? 1 : 0);
 }
 
+/* is vnode_t a named stream? */
+int 
+vnode_isnamedstream(
+#if NAMEDSTREAMS
+		vnode_t vp
+#else
+		__unused vnode_t vp
+#endif
+		)
+{
+#if NAMEDSTREAMS
+	return ((vp->v_flag & VISNAMEDSTREAM) ? 1 : 0);
+#else
+	return (0);
+#endif
+}
 
 /* TBD:  set vnode_t to not cache data after it is consumed once; used for quota */
 void 
 vnode_setnocache(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag |= VNOCACHE_DATA;
 	vnode_unlock(vp);
 }
@@ -1314,15 +1611,32 @@ vnode_setnocache(vnode_t vp)
 void 
 vnode_clearnocache(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag &= ~VNOCACHE_DATA;
 	vnode_unlock(vp);
 }
 
+void
+vnode_set_openevt(vnode_t vp)
+{
+	vnode_lock_spin(vp);
+	vp->v_flag |= VOPENEVT;
+	vnode_unlock(vp);
+}
+
+void
+vnode_clear_openevt(vnode_t vp)
+{
+	vnode_lock_spin(vp);
+	vp->v_flag &= ~VOPENEVT;
+	vnode_unlock(vp);
+}
+
+
 void 
 vnode_setnoreadahead(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag |= VRAOFF;
 	vnode_unlock(vp);
 }
@@ -1330,7 +1644,7 @@ vnode_setnoreadahead(vnode_t vp)
 void 
 vnode_clearnoreadahead(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag &= ~VRAOFF;
 	vnode_unlock(vp);
 }
@@ -1340,7 +1654,7 @@ vnode_clearnoreadahead(vnode_t vp)
 void 
 vnode_setnoflush(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag |= VNOFLUSH;
 	vnode_unlock(vp);
 }
@@ -1348,7 +1662,7 @@ vnode_setnoflush(vnode_t vp)
 void 
 vnode_clearnoflush(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_flag &= ~VNOFLUSH;
 	vnode_unlock(vp);
 }
@@ -1364,7 +1678,7 @@ vnode_ismountedon(vnode_t vp)
 void 
 vnode_setmountedon(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_specflags |= SI_MOUNTEDON;
 	vnode_unlock(vp);
 }
@@ -1372,7 +1686,7 @@ vnode_setmountedon(vnode_t vp)
 void 
 vnode_clearmountedon(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_specflags &= ~SI_MOUNTEDON;
 	vnode_unlock(vp);
 }
@@ -1404,7 +1718,7 @@ vnode_setparent(vnode_t vp, vnode_t dvp)
 	vp->v_parent = dvp;
 }
 
-char *
+const char *
 vnode_name(vnode_t vp)
 {
 	/* we try to keep v_name a reasonable name for the node */    
@@ -1486,28 +1800,26 @@ vnode_vfsisrdonly(vnode_t vp)
 }
 
 
-/* returns vnode ref to current working directory */
+/*
+ * Returns vnode ref to current working directory; if a per-thread current
+ * working directory is in effect, return that instead of the per process one.
+ *
+ * XXX Published, but not used.
+ */
 vnode_t 
 current_workingdir(void)
 {
-	struct proc *p = current_proc();
-	struct vnode * vp ;
-
-	if ( (vp = p->p_fd->fd_cdir) ) {
-	        if ( (vnode_getwithref(vp)) )
-		        return (NULL);
-	}
-	return vp;
+	return vfs_context_cwd(vfs_context_current());
 }
 
 /* returns vnode ref to current root(chroot) directory */
 vnode_t 
 current_rootdir(void)
 {
-	struct proc *p = current_proc();
+	proc_t proc = current_proc();
 	struct vnode * vp ;
 
-	if ( (vp = p->p_fd->fd_rdir) ) {
+	if ( (vp = proc->p_fd->fd_rdir) ) {
 	        if ( (vnode_getwithref(vp)) )
 		        return (NULL);
 	}
@@ -1555,7 +1867,6 @@ vnode_get_filesec(vnode_t vp, kauth_filesec_t *fsecp, vfs_context_t ctx)
 	size_t	fsec_size;
 	size_t	xsize, rsize;
 	int	error;
-	int	i;
 	uint32_t	host_fsec_magic;
 	uint32_t	host_acl_entrycount;
 
@@ -1694,7 +2005,6 @@ vnode_set_filesec(vnode_t vp, kauth_filesec_t fsec, kauth_acl_t acl, vfs_context
 {
 	uio_t		fsec_uio;
 	int		error;
-	int		i;
 	uint32_t	saved_acl_copysize;
 
 	fsec_uio = NULL;
@@ -1730,6 +2040,15 @@ out:
 }
 
 
+/*
+ * Returns:	0			Success
+ *		ENOMEM			Not enough space [only if has filesec]
+ *		VNOP_GETATTR:		???
+ *		vnode_get_filesec:	???
+ *		kauth_cred_guid2uid:	???
+ *		kauth_cred_guid2gid:	???
+ *		vfs_update_vfsstat:	???
+ */
 int
 vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 {
@@ -1840,7 +2159,9 @@ vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	 * Handle uid/gid == 99 and MNT_IGNORE_OWNERSHIP here.
 	 */
 	if (VATTR_IS_ACTIVE(vap, va_uid)) {
-		if (vp->v_mount->mnt_flag & MNT_IGNORE_OWNERSHIP) {
+		if (vfs_context_issuser(ctx) && VATTR_IS_SUPPORTED(vap, va_uid)) {
+			nuid = vap->va_uid;
+		} else if (vp->v_mount->mnt_flag & MNT_IGNORE_OWNERSHIP) {
 			nuid = vp->v_mount->mnt_fsowner;
 			if (nuid == KAUTH_UID_NONE)
 				nuid = 99;
@@ -1855,7 +2176,9 @@ vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		VATTR_RETURN(vap, va_uid, nuid);
 	}
 	if (VATTR_IS_ACTIVE(vap, va_gid)) {
-		if (vp->v_mount->mnt_flag & MNT_IGNORE_OWNERSHIP) {
+		if (vfs_context_issuser(ctx) && VATTR_IS_SUPPORTED(vap, va_gid)) {
+			ngid = vap->va_gid;
+		} else if (vp->v_mount->mnt_flag & MNT_IGNORE_OWNERSHIP) {
 			ngid = vp->v_mount->mnt_fsgroup;
 			if (ngid == KAUTH_GID_NONE)
 				ngid = 99;
@@ -1897,7 +2220,7 @@ vnode_getattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	    VATTR_IS_ACTIVE(vap, va_total_alloc)) {
                 /* make sure f_bsize is valid */
                 if (vp->v_mount->mnt_vfsstat.f_bsize == 0) {
-                    if ((error = vfs_update_vfsstat(vp->v_mount, ctx)) != 0)
+                    if ((error = vfs_update_vfsstat(vp->v_mount, ctx, VFS_KERNEL_EVENT)) != 0)
                         goto out;
                 }
 
@@ -1961,7 +2284,7 @@ out:
 int
 vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 {
-	int	error, is_ownership_change=0;
+	int	error, is_perm_change=0;
 
 	/*
 	 * Make sure the filesystem is mounted R/W.
@@ -1971,6 +2294,13 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		error = EROFS;
 		goto out;
 	}
+#if NAMEDSTREAMS
+	/* For streams, va_data_size is the only setable attribute. */
+	if ((vp->v_flag & VISNAMEDSTREAM) && (vap->va_active != VNODE_ATTR_va_data_size)) {
+		error = EPERM;
+		goto out;
+	}
+#endif
 	
 	/*
 	 * If ownership is being ignored on this volume, we silently discard
@@ -1981,8 +2311,9 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 		VATTR_CLEAR_ACTIVE(vap, va_gid);
 	}
 
-	if (VATTR_IS_ACTIVE(vap, va_uid) || VATTR_IS_ACTIVE(vap, va_gid)) {
-	    is_ownership_change = 1;
+	if (   VATTR_IS_ACTIVE(vap, va_uid)  || VATTR_IS_ACTIVE(vap, va_gid)
+	    || VATTR_IS_ACTIVE(vap, va_mode) || VATTR_IS_ACTIVE(vap, va_acl)) {
+	    is_perm_change = 1;
 	}
 	
 	/*
@@ -2001,29 +2332,19 @@ vnode_setattr(vnode_t vp, struct vnode_attr *vap, vfs_context_t ctx)
 	if ((error == 0) && !VATTR_ALL_SUPPORTED(vap))
 		error = vnode_setattr_fallback(vp, vap, ctx);
 
-	/*
-	 * If we have changed any of the things about the file that are likely
-	 * to result in changes to authorisation results, blow the vnode auth
-	 * cache
-	 */
-	if (VATTR_IS_SUPPORTED(vap, va_mode) ||
-	    VATTR_IS_SUPPORTED(vap, va_uid) ||
-	    VATTR_IS_SUPPORTED(vap, va_gid) ||
-	    VATTR_IS_SUPPORTED(vap, va_flags) ||
-	    VATTR_IS_SUPPORTED(vap, va_acl) ||
-	    VATTR_IS_SUPPORTED(vap, va_uuuid) ||
-	    VATTR_IS_SUPPORTED(vap, va_guuid))
-		vnode_uncache_credentials(vp);
+#if CONFIG_FSE
 	// only send a stat_changed event if this is more than
 	// just an access time update
 	if (error == 0 && (vap->va_active != VNODE_ATTR_BIT(va_access_time))) {
-	    if (need_fsevent(FSE_STAT_CHANGED, vp) || (is_ownership_change && need_fsevent(FSE_CHOWN, vp))) {
-		if (is_ownership_change == 0)
-		        add_fsevent(FSE_STAT_CHANGED, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
-		else
-		        add_fsevent(FSE_CHOWN, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
+	    if (is_perm_change) {
+		if (need_fsevent(FSE_CHOWN, vp)) {
+		    add_fsevent(FSE_CHOWN, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
+		}
+	    } else if(need_fsevent(FSE_STAT_CHANGED, vp)) {
+		add_fsevent(FSE_STAT_CHANGED, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
 	    }
 	}
+#endif
 
 out:
 	return(error);
@@ -2187,8 +2508,30 @@ struct vnop_lookup_args {
 };
 #endif /* 0*/
 
+/*
+ * Returns:	0			Success
+ *	lock_fsnode:ENOENT		No such file or directory [only for VFS
+ *					 that is not thread safe & vnode is
+ *					 currently being/has been terminated]
+ *	<vfs_lookup>:ENAMETOOLONG
+ *	<vfs_lookup>:ENOENT
+ *	<vfs_lookup>:EJUSTRETURN
+ *	<vfs_lookup>:EPERM
+ *	<vfs_lookup>:EISDIR
+ *	<vfs_lookup>:ENOTDIR
+ *	<vfs_lookup>:???
+ *
+ * Note:	The return codes from the underlying VFS's lookup routine can't
+ *		be fully enumerated here, since third party VFS authors may not
+ *		limit their error returns to the ones documented here, even
+ *		though this may result in some programs functioning incorrectly.
+ *
+ *		The return codes documented above are those which may currently
+ *		be returned by HFS from hfs_lookup, not including additional
+ *		error code which may be propagated from underlying routines.
+ */
 errno_t 
-VNOP_LOOKUP(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, vfs_context_t context)
+VNOP_LOOKUP(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_lookup_args a;
@@ -2200,11 +2543,9 @@ VNOP_LOOKUP(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, vfs_context_t
 	a.a_dvp = dvp;
 	a.a_vpp = vpp;
 	a.a_cnp = cnp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(dvp);
 
-	vnode_cache_credentials(dvp, context);
-
 	if (!thread_safe) {
 		if ( (_err = lock_fsnode(dvp, &funnel_state)) ) {
 			return (_err);
@@ -2255,7 +2596,7 @@ struct vnop_create_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_CREATE(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode_attr * vap, vfs_context_t context)
+VNOP_CREATE(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode_attr * vap, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_create_args a;
@@ -2267,7 +2608,7 @@ VNOP_CREATE(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode
 	a.a_vpp = vpp;
 	a.a_cnp = cnp;
 	a.a_vap = vap;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(dvp);
 
 	if (!thread_safe) {
@@ -2280,7 +2621,7 @@ VNOP_CREATE(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode
 		/* 
 		 * Remove stale Apple Double file (if any).
 		 */
-		xattrfile_remove(dvp, cnp->cn_nameptr, context, thread_safe, 0);
+		xattrfile_remove(dvp, cnp->cn_nameptr, ctx, thread_safe, 0);
 	}
 	if (!thread_safe) {
 		unlock_fsnode(dvp, &funnel_state);
@@ -2305,7 +2646,7 @@ struct vnop_whiteout_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_WHITEOUT(vnode_t dvp, struct componentname * cnp, int flags, vfs_context_t context)
+VNOP_WHITEOUT(vnode_t dvp, struct componentname * cnp, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_whiteout_args a;
@@ -2316,7 +2657,7 @@ VNOP_WHITEOUT(vnode_t dvp, struct componentname * cnp, int flags, vfs_context_t
 	a.a_dvp = dvp;
 	a.a_cnp = cnp;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(dvp);
 
 	if (!thread_safe) {
@@ -2348,7 +2689,7 @@ struct vnop_mknod_args {
 };
 #endif /* 0*/
 errno_t
-VNOP_MKNOD(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode_attr * vap, vfs_context_t context)
+VNOP_MKNOD(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode_attr * vap, vfs_context_t ctx)
 {
 
        int _err;
@@ -2361,7 +2702,7 @@ VNOP_MKNOD(vnode_t dvp, vnode_t * vpp, struct componentname * cnp, struct vnode_
        a.a_vpp = vpp;
        a.a_cnp = cnp;
        a.a_vap = vap;
-       a.a_context = context;
+       a.a_context = ctx;
        thread_safe = THREAD_SAFE_FS(dvp);
 
        if (!thread_safe) {
@@ -2390,23 +2731,20 @@ struct vnop_open_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_OPEN(vnode_t vp, int mode, vfs_context_t context)
+VNOP_OPEN(vnode_t vp, int mode, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_open_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	a.a_desc = &vnop_open_desc;
 	a.a_vp = vp;
 	a.a_mode = mode;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2442,23 +2780,20 @@ struct vnop_close_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_CLOSE(vnode_t vp, int fflag, vfs_context_t context)
+VNOP_CLOSE(vnode_t vp, int fflag, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_close_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	a.a_desc = &vnop_close_desc;
 	a.a_vp = vp;
 	a.a_fflag = fflag;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2494,23 +2829,20 @@ struct vnop_access_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_ACCESS(vnode_t vp, int action, vfs_context_t context)
+VNOP_ACCESS(vnode_t vp, int action, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_access_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	a.a_desc = &vnop_access_desc;
 	a.a_vp = vp;
 	a.a_action = action;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2539,17 +2871,17 @@ struct vnop_getattr_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_GETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t context)
+VNOP_GETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_getattr_args a;
 	int thread_safe;
-	int funnel_state;
+	int funnel_state = 0;	/* protected by thread_safe */
 
 	a.a_desc = &vnop_getattr_desc;
 	a.a_vp = vp;
 	a.a_vap = vap;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2578,17 +2910,17 @@ struct vnop_setattr_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_SETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t context)
+VNOP_SETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_setattr_args a;
 	int thread_safe;
-	int funnel_state;
+	int funnel_state = 0;	/* protected by thread_safe */
 
 	a.a_desc = &vnop_setattr_desc;
 	a.a_vp = vp;
 	a.a_vap = vap;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2599,7 +2931,7 @@ VNOP_SETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t context)
 	_err = (*vp->v_op[vnop_setattr_desc.vdesc_offset])(&a);
 
 	/* 
-	 * Shadow uid/gid/mod change to extended attibute file.
+	 * Shadow uid/gid/mod change to extended attribute file.
 	 */
 	if (_err == 0 && !NATIVE_XATTR(vp)) {
 		struct vnode_attr va;
@@ -2620,12 +2952,12 @@ VNOP_SETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t context)
 		}
 		if (change) {
 		        vnode_t dvp;
-			char   *vname;
+			const char   *vname;
 
 			dvp = vnode_getparent(vp);
 			vname = vnode_getname(vp);
 
-			xattrfile_setattr(dvp, vname, &va, context, thread_safe);
+			xattrfile_setattr(dvp, vname, &va, ctx, thread_safe);
 			if (dvp != NULLVP)
 			        vnode_put(dvp);
 			if (vname != NULL)
@@ -2635,95 +2967,21 @@ VNOP_SETATTR(vnode_t vp, struct vnode_attr * vap, vfs_context_t context)
 	if (!thread_safe) {
 		unlock_fsnode(vp, &funnel_state);
 	}
-	return (_err);
-}
-
-#if 0
-/*
- *#
- *#% getattrlist  vp      = = =
- *#
- */
-struct vnop_getattrlist_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context_t a_context;
-};
-#endif /* 0*/
-errno_t
-VNOP_GETATTRLIST(vnode_t vp, struct attrlist * alist, struct uio * uio, int options, vfs_context_t context)
-{
-	int _err;
-	struct vnop_getattrlist_args a;
-	int thread_safe;
-	int funnel_state = 0;
-
-	a.a_desc = &vnop_getattrlist_desc;
-	a.a_vp = vp;
-	a.a_alist = alist;
-	a.a_uio = uio;
-	a.a_options = options;
-	a.a_context = context;
-	thread_safe = THREAD_SAFE_FS(vp);
-
-	if (!thread_safe) {
-		if ( (_err = lock_fsnode(vp, &funnel_state)) ) {
-			return (_err);
-		}
-	}
-	_err = (*vp->v_op[vnop_getattrlist_desc.vdesc_offset])(&a);
-	if (!thread_safe) {
-		unlock_fsnode(vp, &funnel_state);
-	}
-	return (_err);
-}
-
-#if 0
-/*
- *#
- *#% setattrlist  vp      L L L
- *#
- */
-struct vnop_setattrlist_args {
-	struct vnodeop_desc *a_desc;
-	vnode_t a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context_t a_context;
-};
-#endif /* 0*/
-errno_t 
-VNOP_SETATTRLIST(vnode_t vp, struct attrlist * alist, struct uio * uio, int options, vfs_context_t context)
-{
-	int _err;
-	struct vnop_setattrlist_args a;
-	int thread_safe;
-	int funnel_state = 0;
-
-	a.a_desc = &vnop_setattrlist_desc;
-	a.a_vp = vp;
-	a.a_alist = alist;
-	a.a_uio = uio;
-	a.a_options = options;
-	a.a_context = context;
-	thread_safe = THREAD_SAFE_FS(vp);
-
-	if (!thread_safe) {
-		if ( (_err = lock_fsnode(vp, &funnel_state)) ) {
-			return (_err);
-		}
-	}
-	_err = (*vp->v_op[vnop_setattrlist_desc.vdesc_offset])(&a);
-
-	vnode_uncache_credentials(vp);
+	/*
+	 * If we have changed any of the things about the file that are likely
+	 * to result in changes to authorization results, blow the vnode auth
+	 * cache
+	 */
+	if (_err == 0 && (
+			  VATTR_IS_SUPPORTED(vap, va_mode) ||
+			  VATTR_IS_SUPPORTED(vap, va_uid) ||
+			  VATTR_IS_SUPPORTED(vap, va_gid) ||
+			  VATTR_IS_SUPPORTED(vap, va_flags) ||
+			  VATTR_IS_SUPPORTED(vap, va_acl) ||
+			  VATTR_IS_SUPPORTED(vap, va_uuuid) ||
+			  VATTR_IS_SUPPORTED(vap, va_guuid)))
+	        vnode_uncache_authorized_action(vp, KAUTH_INVALIDATE_CACHED_RIGHTS);
 
-	if (!thread_safe) {
-		unlock_fsnode(vp, &funnel_state);
-	}
 	return (_err);
 }
 
@@ -2743,25 +3001,22 @@ struct vnop_read_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_READ(vnode_t vp, struct uio * uio, int ioflag, vfs_context_t context)
+VNOP_READ(vnode_t vp, struct uio * uio, int ioflag, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_read_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 
 	a.a_desc = &vnop_read_desc;
 	a.a_vp = vp;
 	a.a_uio = uio;
 	a.a_ioflag = ioflag;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2800,25 +3055,22 @@ struct vnop_write_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_WRITE(vnode_t vp, struct uio * uio, int ioflag, vfs_context_t context)
+VNOP_WRITE(vnode_t vp, struct uio * uio, int ioflag, vfs_context_t ctx)
 {
 	struct vnop_write_args a;
 	int _err;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 
 	a.a_desc = &vnop_write_desc;
 	a.a_vp = vp;
 	a.a_uio = uio;
 	a.a_ioflag = ioflag;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2858,21 +3110,18 @@ struct vnop_ioctl_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_IOCTL(vnode_t vp, u_long command, caddr_t data, int fflag, vfs_context_t context)
+VNOP_IOCTL(vnode_t vp, u_long command, caddr_t data, int fflag, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_ioctl_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 
-	if (vfs_context_is64bit(context)) {
+	if (vfs_context_is64bit(ctx)) {
 		if (!vnode_vfs64bitready(vp)) {
 			return(ENOTTY);
 		}
@@ -2883,7 +3132,7 @@ VNOP_IOCTL(vnode_t vp, u_long command, caddr_t data, int fflag, vfs_context_t co
 	a.a_command = command;
 	a.a_data = data;
 	a.a_fflag = fflag;
-	a.a_context= context;
+	a.a_context= ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -2922,24 +3171,21 @@ struct vnop_select_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_SELECT(vnode_t vp, int which , int fflags, void * wql, vfs_context_t context)
+VNOP_SELECT(vnode_t vp, int which , int fflags, void * wql, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_select_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	a.a_desc = &vnop_select_desc;
 	a.a_vp = vp;
 	a.a_which = which;
 	a.a_fflags = fflags;
-	a.a_context = context;
+	a.a_context = ctx;
 	a.a_wql = wql;
 	thread_safe = THREAD_SAFE_FS(vp);
 
@@ -2979,7 +3225,7 @@ struct vnop_exchange_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_EXCHANGE(vnode_t fvp, vnode_t tvp, int options, vfs_context_t context)
+VNOP_EXCHANGE(vnode_t fvp, vnode_t tvp, int options, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_exchange_args a;
@@ -2991,7 +3237,7 @@ VNOP_EXCHANGE(vnode_t fvp, vnode_t tvp, int options, vfs_context_t context)
 	a.a_fvp = fvp;
 	a.a_tvp = tvp;
 	a.a_options = options;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(fvp);
 
 	if (!thread_safe) {
@@ -3036,7 +3282,7 @@ struct vnop_revoke_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_REVOKE(vnode_t vp, int flags, vfs_context_t context)
+VNOP_REVOKE(vnode_t vp, int flags, vfs_context_t ctx)
 {
 	struct vnop_revoke_args a;
 	int _err;
@@ -3046,7 +3292,7 @@ VNOP_REVOKE(vnode_t vp, int flags, vfs_context_t context)
 	a.a_desc = &vnop_revoke_desc;
 	a.a_vp = vp;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3074,7 +3320,7 @@ struct vnop_mmap_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_MMAP(vnode_t vp, int fflags, vfs_context_t context)
+VNOP_MMAP(vnode_t vp, int fflags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_mmap_args a;
@@ -3084,7 +3330,7 @@ VNOP_MMAP(vnode_t vp, int fflags, vfs_context_t context)
 	a.a_desc = &vnop_mmap_desc;
 	a.a_vp = vp;
 	a.a_fflags = fflags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3113,7 +3359,7 @@ struct vnop_mnomap_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_MNOMAP(vnode_t vp, vfs_context_t context)
+VNOP_MNOMAP(vnode_t vp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_mnomap_args a;
@@ -3122,7 +3368,7 @@ VNOP_MNOMAP(vnode_t vp, vfs_context_t context)
 
 	a.a_desc = &vnop_mnomap_desc;
 	a.a_vp = vp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3152,7 +3398,7 @@ struct vnop_fsync_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_FSYNC(vnode_t vp, int waitfor, vfs_context_t context)
+VNOP_FSYNC(vnode_t vp, int waitfor, vfs_context_t ctx)
 {
 	struct vnop_fsync_args a;
 	int _err;
@@ -3162,7 +3408,7 @@ VNOP_FSYNC(vnode_t vp, int waitfor, vfs_context_t context)
 	a.a_desc = &vnop_fsync_desc;
 	a.a_vp = vp;
 	a.a_waitfor = waitfor;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3195,7 +3441,7 @@ struct vnop_remove_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_REMOVE(vnode_t dvp, vnode_t vp, struct componentname * cnp, int flags, vfs_context_t context)
+VNOP_REMOVE(vnode_t dvp, vnode_t vp, struct componentname * cnp, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_remove_args a;
@@ -3207,7 +3453,7 @@ VNOP_REMOVE(vnode_t dvp, vnode_t vp, struct componentname * cnp, int flags, vfs_
 	a.a_vp = vp;
 	a.a_cnp = cnp;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(dvp);
 
 	if (!thread_safe) {
@@ -3222,9 +3468,9 @@ VNOP_REMOVE(vnode_t dvp, vnode_t vp, struct componentname * cnp, int flags, vfs_
 
 		if ( !(NATIVE_XATTR(dvp)) ) {
 		        /* 
-			 * Remove any associated extended attibute file (._ AppleDouble file).
+			 * Remove any associated extended attribute file (._ AppleDouble file).
 			 */
-		        xattrfile_remove(dvp, cnp->cn_nameptr, context, thread_safe, 1);
+		        xattrfile_remove(dvp, cnp->cn_nameptr, ctx, thread_safe, 1);
 		}
 	}
 	if (!thread_safe) {
@@ -3250,7 +3496,7 @@ struct vnop_link_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_LINK(vnode_t vp, vnode_t tdvp, struct componentname * cnp, vfs_context_t context)
+VNOP_LINK(vnode_t vp, vnode_t tdvp, struct componentname * cnp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_link_args a;
@@ -3262,7 +3508,7 @@ VNOP_LINK(vnode_t vp, vnode_t tdvp, struct componentname * cnp, vfs_context_t co
 	 * disallow linking to an existing "._" Apple Double file.
 	 */
 	if ( !NATIVE_XATTR(tdvp) && (vp->v_type == VREG)) {
-		char   *vname;
+		const char   *vname;
 
 		vname = vnode_getname(vp);
 		if (vname != NULL) {
@@ -3279,7 +3525,7 @@ VNOP_LINK(vnode_t vp, vnode_t tdvp, struct componentname * cnp, vfs_context_t co
 	a.a_vp = vp;
 	a.a_tdvp = tdvp;
 	a.a_cnp = cnp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3318,7 +3564,7 @@ struct vnop_rename_args {
 errno_t
 VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
             struct vnode *tdvp, struct vnode *tvp, struct componentname *tcnp,
-            vfs_context_t context)
+            vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_rename_args a;
@@ -3338,7 +3584,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 	a.a_tdvp = tdvp;
 	a.a_tvp = tvp;
 	a.a_tcnp = tcnp;
-	a.a_context = context;
+	a.a_context = ctx;
 
 	if (!THREAD_SAFE_FS(fdvp))
 	        fdvp_unsafe = fdvp;
@@ -3407,7 +3653,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 		} else {
 			xfromname = &smallname1[0];
 		}
-		strcpy(xfromname, "._");
+		strlcpy(xfromname, "._", min(sizeof smallname1, len));
 		strncat(xfromname, fcnp->cn_nameptr, fcnp->cn_namelen);
 		xfromname[len-1] = '\0';
 
@@ -3418,7 +3664,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 		} else {
 			xtoname = &smallname2[0];
 		}
-		strcpy(xtoname, "._");
+		strlcpy(xtoname, "._", min(sizeof smallname2, len));
 		strncat(xtoname, tcnp->cn_nameptr, tcnp->cn_namelen);
 		xtoname[len-1] = '\0';
 	}
@@ -3436,7 +3682,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 	}
 
 	/* 
-	 * Rename any associated extended attibute file (._ AppleDouble file).
+	 * Rename any associated extended attribute file (._ AppleDouble file).
 	 */
 	if (_err == 0 && !NATIVE_XATTR(fdvp) && xfromname != NULL) {
 		struct nameidata fromnd, tond;
@@ -3448,8 +3694,8 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 		 * Note that fdvp already has an iocount reference and
 		 * using DELETE will take an additional reference.
 		 */
-		NDINIT(&fromnd, DELETE, NOFOLLOW | USEDVP, UIO_SYSSPACE,
-		       CAST_USER_ADDR_T(xfromname), context);
+		NDINIT(&fromnd, DELETE, NOFOLLOW | USEDVP | CN_NBMOUNTLOOK, UIO_SYSSPACE,
+		       CAST_USER_ADDR_T(xfromname), ctx);
 		fromnd.ni_dvp = fdvp;
 		error = namei(&fromnd);
 
@@ -3472,8 +3718,8 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 			 * Get destination attribute file vnode.
 			 * Note that tdvp already has an iocount reference.
 			 */
-			NDINIT(&tond, DELETE, NOFOLLOW | USEDVP, UIO_SYSSPACE,
-			       CAST_USER_ADDR_T(xtoname), context);
+			NDINIT(&tond, DELETE, NOFOLLOW | USEDVP | CN_NBMOUNTLOOK, UIO_SYSSPACE,
+			       CAST_USER_ADDR_T(xtoname), ctx);
 			tond.ni_dvp = tdvp;
 			error = namei(&tond);
 			if (error) {
@@ -3488,7 +3734,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 			args.a_dvp     = tdvp;
 			args.a_vp      = tond.ni_vp;
 			args.a_cnp     = &tond.ni_cnd;
-			args.a_context = context;
+			args.a_context = ctx;
 
 			if (fdvp_unsafe != NULLVP)
 			        error = lock_fsnode(tond.ni_vp, NULL);
@@ -3510,8 +3756,8 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 		 * Get destination attribute file vnode.
 		 */
 		NDINIT(&tond, RENAME,
-		       NOCACHE | NOFOLLOW | USEDVP, UIO_SYSSPACE,
-		       CAST_USER_ADDR_T(xtoname), context);
+		       NOCACHE | NOFOLLOW | USEDVP | CN_NBMOUNTLOOK, UIO_SYSSPACE,
+		       CAST_USER_ADDR_T(xtoname), ctx);
 		tond.ni_dvp = tdvp;
 		error = namei(&tond);
 
@@ -3527,7 +3773,7 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 		a.a_tdvp = tdvp;
 		a.a_tvp = tond.ni_vp;
 		a.a_tcnp = &tond.ni_cnd;
-		a.a_context = context;
+		a.a_context = ctx;
 
 		if (fdvp_unsafe != NULLVP) {
 		        /*
@@ -3549,6 +3795,13 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 			}
 		}
 		if (error == 0) {
+			const char *oname;
+			vnode_t oparent;
+
+			/* Save these off so we can later verify them (fix up below) */
+			oname   = fromnd.ni_vp->v_name;
+			oparent = fromnd.ni_vp->v_parent;
+
 		        error = (*fdvp->v_op[vnop_rename_desc.vdesc_offset])(&a);
 
 			if (fdvp_unsafe != NULLVP) {
@@ -3561,6 +3814,24 @@ VNOP_RENAME(struct vnode *fdvp, struct vnode *fvp, struct componentname *fcnp,
 				
 				if (tond.ni_vp && tond.ni_vp != fromnd.ni_vp)
 				        vnode_setneedinactive(tond.ni_vp);
+				/*
+				 * Fix up name & parent pointers on ._ file
+				 */
+				if (oname == fromnd.ni_vp->v_name &&
+				    oparent == fromnd.ni_vp->v_parent) {
+					int update_flags;
+			
+					update_flags = VNODE_UPDATE_NAME;
+			
+					if (fdvp != tdvp)
+						update_flags |= VNODE_UPDATE_PARENT;
+			
+					vnode_update_identity(fromnd.ni_vp, tdvp,
+					                      tond.ni_cnd.cn_nameptr,
+					                      tond.ni_cnd.cn_namelen,
+					                      tond.ni_cnd.cn_hash,
+					                      update_flags);
+				}
 			}
 		}
 		vnode_put(fromnd.ni_vp);
@@ -3604,7 +3875,7 @@ struct vnop_mkdir_args {
 #endif /* 0*/
 errno_t
 VNOP_MKDIR(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
-           struct vnode_attr *vap, vfs_context_t context)
+           struct vnode_attr *vap, vfs_context_t ctx)
 {
        int _err;
        struct vnop_mkdir_args a;
@@ -3616,7 +3887,7 @@ VNOP_MKDIR(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
        a.a_vpp = vpp;
        a.a_cnp = cnp;
        a.a_vap = vap;
-       a.a_context = context;
+       a.a_context = ctx;
        thread_safe = THREAD_SAFE_FS(dvp);
 
        if (!thread_safe) {
@@ -3629,7 +3900,7 @@ VNOP_MKDIR(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 		/* 
 		 * Remove stale Apple Double file (if any).
 		 */
-		xattrfile_remove(dvp, cnp->cn_nameptr, context, thread_safe, 0);
+		xattrfile_remove(dvp, cnp->cn_nameptr, ctx, thread_safe, 0);
 	}
        if (!thread_safe) {
                unlock_fsnode(dvp, &funnel_state);
@@ -3655,7 +3926,7 @@ struct vnop_rmdir_args {
 
 #endif /* 0*/
 errno_t
-VNOP_RMDIR(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, vfs_context_t context)
+VNOP_RMDIR(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_rmdir_args a;
@@ -3666,7 +3937,7 @@ VNOP_RMDIR(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, vfs_c
 	a.a_dvp = dvp;
 	a.a_vp = vp;
 	a.a_cnp = cnp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(dvp);
 
 	if (!thread_safe) {
@@ -3681,9 +3952,9 @@ VNOP_RMDIR(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, vfs_c
 
 		if ( !(NATIVE_XATTR(dvp)) ) {
 		        /* 
-			 * Remove any associated extended attibute file (._ AppleDouble file).
+			 * Remove any associated extended attribute file (._ AppleDouble file).
 			 */
-		        xattrfile_remove(dvp, cnp->cn_nameptr, context, thread_safe, 1);
+		        xattrfile_remove(dvp, cnp->cn_nameptr, ctx, thread_safe, 1);
 		}
 	}
 	if (!thread_safe) {
@@ -3697,7 +3968,7 @@ VNOP_RMDIR(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, vfs_c
  */
 #define AD_STALE_SECS  (180)
 static void
-xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context, int thread_safe, int force) {
+xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t ctx, int thread_safe, int force) {
 	vnode_t xvp;
 	struct nameidata nd;
 	char smallname[64];
@@ -3715,8 +3986,8 @@ xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context, int
 		MALLOC(filename, char *, len, M_TEMP, M_WAITOK);
 		len = snprintf(filename, len, "._%s", basename);
 	}
-	NDINIT(&nd, DELETE, LOCKLEAF | NOFOLLOW | USEDVP, UIO_SYSSPACE,
-	       CAST_USER_ADDR_T(filename), context);
+	NDINIT(&nd, DELETE, WANTPARENT | LOCKLEAF | NOFOLLOW | USEDVP, UIO_SYSSPACE,
+	       CAST_USER_ADDR_T(filename), ctx);
 	nd.ni_dvp = dvp;
 	if (namei(&nd) != 0)
 		goto out2;
@@ -3737,7 +4008,7 @@ xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context, int
 		VATTR_INIT(&va);
 		VATTR_WANTED(&va, va_data_size);
 		VATTR_WANTED(&va, va_modify_time);
-		if (VNOP_GETATTR(xvp, &va, context) == 0  &&
+		if (VNOP_GETATTR(xvp, &va, ctx) == 0  &&
 		    VATTR_IS_SUPPORTED(&va, va_data_size)  &&
 		    VATTR_IS_SUPPORTED(&va, va_modify_time)  &&
 		    va.va_data_size != 0) {
@@ -3758,7 +4029,7 @@ xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context, int
 		a.a_dvp     = nd.ni_dvp;
 		a.a_vp      = xvp;
 		a.a_cnp     = &nd.ni_cnd;
-		a.a_context = context;
+		a.a_context = ctx;
 
 		if (!thread_safe) {
 			if ( (lock_fsnode(xvp, NULL)) )
@@ -3773,7 +4044,7 @@ xattrfile_remove(vnode_t dvp, const char * basename, vfs_context_t context, int
 			vnode_setneedinactive(xvp);
 	}
 out1:		
-	/* Note: nd.ni_dvp's iocount is dropped by caller of VNOP_XXXX */
+	vnode_put(dvp);
 	vnode_put(xvp);
 out2:
 	if (filename && filename != &smallname[0]) {
@@ -3786,7 +4057,7 @@ out2:
  */
 static void
 xattrfile_setattr(vnode_t dvp, const char * basename, struct vnode_attr * vap,
-                  vfs_context_t context, int thread_safe) {
+                  vfs_context_t ctx, int thread_safe) {
 	vnode_t xvp;
 	struct nameidata nd;
 	char smallname[64];
@@ -3806,7 +4077,7 @@ xattrfile_setattr(vnode_t dvp, const char * basename, struct vnode_attr * vap,
 		len = snprintf(filename, len, "._%s", basename);
 	}
 	NDINIT(&nd, LOOKUP, NOFOLLOW | USEDVP, UIO_SYSSPACE,
-	       CAST_USER_ADDR_T(filename), context);
+	       CAST_USER_ADDR_T(filename), ctx);
 	nd.ni_dvp = dvp;
 	if (namei(&nd) != 0)
 		goto out2;
@@ -3820,7 +4091,7 @@ xattrfile_setattr(vnode_t dvp, const char * basename, struct vnode_attr * vap,
 		a.a_desc = &vnop_setattr_desc;
 		a.a_vp = xvp;
 		a.a_vap = vap;
-		a.a_context = context;
+		a.a_context = ctx;
 
 		if (!thread_safe) {
 			if ( (lock_fsnode(xvp, NULL)) )
@@ -3859,7 +4130,7 @@ struct vnop_symlink_args {
 #endif /* 0*/
 errno_t
 VNOP_SYMLINK(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
-             struct vnode_attr *vap, char *target, vfs_context_t context)
+             struct vnode_attr *vap, char *target, vfs_context_t ctx)
 {
        int _err;
        struct vnop_symlink_args a;
@@ -3872,7 +4143,7 @@ VNOP_SYMLINK(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
        a.a_cnp = cnp;
        a.a_vap = vap;
        a.a_target = target;
-       a.a_context = context;
+       a.a_context = ctx;
        thread_safe = THREAD_SAFE_FS(dvp);
 
        if (!thread_safe) {
@@ -3885,7 +4156,7 @@ VNOP_SYMLINK(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp,
 		/* 
 		 * Remove stale Apple Double file (if any).
 		 */
-		xattrfile_remove(dvp, cnp->cn_nameptr, context, thread_safe, 0);
+		xattrfile_remove(dvp, cnp->cn_nameptr, ctx, thread_safe, 0);
 	}
        if (!thread_safe) {
                unlock_fsnode(dvp, &funnel_state);
@@ -3912,7 +4183,7 @@ struct vnop_readdir_args {
 #endif /* 0*/
 errno_t 
 VNOP_READDIR(struct vnode *vp, struct uio *uio, int flags, int *eofflag,
-             int *numdirent, vfs_context_t context)
+             int *numdirent, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_readdir_args a;
@@ -3925,7 +4196,7 @@ VNOP_READDIR(struct vnode *vp, struct uio *uio, int flags, int *eofflag,
 	a.a_flags = flags;
 	a.a_eofflag = eofflag;
 	a.a_numdirent = numdirent;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -3962,7 +4233,7 @@ struct vnop_readdirattr_args {
 #endif /* 0*/
 errno_t 
 VNOP_READDIRATTR(struct vnode *vp, struct attrlist *alist, struct uio *uio, u_long maxcount,
-                 u_long options, u_long *newstate, int *eofflag, u_long *actualcount, vfs_context_t context)
+                 u_long options, u_long *newstate, int *eofflag, u_long *actualcount, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_readdirattr_args a;
@@ -3978,7 +4249,7 @@ VNOP_READDIRATTR(struct vnode *vp, struct attrlist *alist, struct uio *uio, u_lo
 	a.a_newstate = newstate;
 	a.a_eofflag = eofflag;
 	a.a_actualcount = actualcount;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4007,8 +4278,27 @@ struct vnop_readlink_args {
 };
 #endif /* 0 */
 
+/*
+ * Returns:	0			Success
+ *		lock_fsnode:ENOENT	No such file or directory [only for VFS
+ *					 that is not thread safe & vnode is
+ *					 currently being/has been terminated]
+ *		<vfs_readlink>:EINVAL
+ *		<vfs_readlink>:???
+ *
+ * Note:	The return codes from the underlying VFS's readlink routine
+ *		can't be fully enumerated here, since third party VFS authors
+ *		may not limit their error returns to the ones documented here,
+ *		even though this may result in some programs functioning
+ *		incorrectly.
+ *
+ *		The return codes documented above are those which may currently
+ *		be returned by HFS from hfs_vnop_readlink, not including
+ *		additional error code which may be propagated from underlying
+ *		routines.
+ */
 errno_t 
-VNOP_READLINK(struct vnode *vp, struct uio *uio, vfs_context_t context)
+VNOP_READLINK(struct vnode *vp, struct uio *uio, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_readlink_args a;
@@ -4018,7 +4308,7 @@ VNOP_READLINK(struct vnode *vp, struct uio *uio, vfs_context_t context)
 	a.a_desc = &vnop_readlink_desc;
 	a.a_vp = vp;
 	a.a_uio = uio;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4046,7 +4336,7 @@ struct vnop_inactive_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_INACTIVE(struct vnode *vp, vfs_context_t context)
+VNOP_INACTIVE(struct vnode *vp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_inactive_args a;
@@ -4055,7 +4345,7 @@ VNOP_INACTIVE(struct vnode *vp, vfs_context_t context)
 
 	a.a_desc = &vnop_inactive_desc;
 	a.a_vp = vp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4084,7 +4374,7 @@ struct vnop_reclaim_args {
 };
 #endif /* 0*/
 errno_t
-VNOP_RECLAIM(struct vnode *vp, vfs_context_t context)
+VNOP_RECLAIM(struct vnode *vp, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_reclaim_args a;
@@ -4093,7 +4383,7 @@ VNOP_RECLAIM(struct vnode *vp, vfs_context_t context)
 
 	a.a_desc = &vnop_reclaim_desc;
 	a.a_vp = vp;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4107,6 +4397,13 @@ VNOP_RECLAIM(struct vnode *vp, vfs_context_t context)
 }
 
 
+/*
+ * Returns:	0			Success
+ *	lock_fsnode:ENOENT		No such file or directory [only for VFS
+ *					 that is not thread safe & vnode is
+ *					 currently being/has been terminated]
+ *	<vnop_pathconf_desc>:???	[per FS implementation specific]
+ */
 #if 0
 /*
  *#
@@ -4122,7 +4419,7 @@ struct vnop_pathconf_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_PATHCONF(struct vnode *vp, int name, register_t *retval, vfs_context_t context)
+VNOP_PATHCONF(struct vnode *vp, int name, register_t *retval, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_pathconf_args a;
@@ -4133,7 +4430,7 @@ VNOP_PATHCONF(struct vnode *vp, int name, register_t *retval, vfs_context_t cont
 	a.a_vp = vp;
 	a.a_name = name;
 	a.a_retval = retval;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4148,6 +4445,17 @@ VNOP_PATHCONF(struct vnode *vp, int name, register_t *retval, vfs_context_t cont
 	return (_err);
 }
 
+/*
+ * Returns:	0			Success
+ *	err_advlock:ENOTSUP
+ *	lf_advlock:???
+ *	<vnop_advlock_desc>:???
+ *
+ * Notes:	VFS implementations of advisory locking using calls through
+ *		<vnop_advlock_desc> because lock enforcement does not occur
+ *		locally should try to limit themselves to the return codes
+ *		documented above for lf_advlock and err_advlock.
+ */
 #if 0
 /*
  *#
@@ -4165,7 +4473,7 @@ struct vnop_advlock_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_ADVLOCK(struct vnode *vp, caddr_t id, int op, struct flock *fl, int flags, vfs_context_t context)
+VNOP_ADVLOCK(struct vnode *vp, caddr_t id, int op, struct flock *fl, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_advlock_args a;
@@ -4179,7 +4487,7 @@ VNOP_ADVLOCK(struct vnode *vp, caddr_t id, int op, struct flock *fl, int flags,
 	a.a_op = op;
 	a.a_fl = fl;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	uth = get_bsdthread_info(current_thread());
@@ -4224,7 +4532,7 @@ struct vnop_allocate_args {
 
 #endif /* 0*/
 errno_t 
-VNOP_ALLOCATE(struct vnode *vp, off_t length, u_int32_t flags, off_t *bytesallocated, off_t offset, vfs_context_t context)
+VNOP_ALLOCATE(struct vnode *vp, off_t length, u_int32_t flags, off_t *bytesallocated, off_t offset, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_allocate_args a;
@@ -4237,7 +4545,7 @@ VNOP_ALLOCATE(struct vnode *vp, off_t length, u_int32_t flags, off_t *bytesalloc
 	a.a_flags = flags;
 	a.a_bytesallocated = bytesallocated;
 	a.a_offset = offset;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4270,7 +4578,7 @@ struct vnop_pagein_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_PAGEIN(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset, size_t size, int flags, vfs_context_t context)
+VNOP_PAGEIN(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset, size_t size, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_pagein_args a;
@@ -4284,7 +4592,7 @@ VNOP_PAGEIN(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset, s
 	a.a_f_offset = f_offset;
 	a.a_size = size;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4316,7 +4624,7 @@ struct vnop_pageout_args {
 
 #endif /* 0*/
 errno_t 
-VNOP_PAGEOUT(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset, size_t size, int flags, vfs_context_t context)
+VNOP_PAGEOUT(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset, size_t size, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_pageout_args a;
@@ -4330,7 +4638,7 @@ VNOP_PAGEOUT(struct vnode *vp, upl_t pl, vm_offset_t pl_offset, off_t f_offset,
 	a.a_f_offset = f_offset;
 	a.a_size = size;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4369,7 +4677,7 @@ struct vnop_searchfs_args {
 
 #endif /* 0*/
 errno_t 
-VNOP_SEARCHFS(struct vnode *vp, void *searchparams1, void *searchparams2, struct attrlist *searchattrs, u_long maxmatches, struct timeval *timelimit, struct attrlist *returnattrs, u_long *nummatches, u_long scriptcode, u_long options, struct uio *uio, struct searchstate *searchstate, vfs_context_t context)
+VNOP_SEARCHFS(struct vnode *vp, void *searchparams1, void *searchparams2, struct attrlist *searchattrs, u_long maxmatches, struct timeval *timelimit, struct attrlist *returnattrs, u_long *nummatches, u_long scriptcode, u_long options, struct uio *uio, struct searchstate *searchstate, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_searchfs_args a;
@@ -4389,7 +4697,7 @@ VNOP_SEARCHFS(struct vnode *vp, void *searchparams1, void *searchparams2, struct
 	a.a_options = options;
 	a.a_uio = uio;
 	a.a_searchstate = searchstate;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4425,7 +4733,7 @@ struct vnop_copyfile_args {
 #endif /* 0*/
 errno_t 
 VNOP_COPYFILE(struct vnode *fvp, struct vnode *tdvp, struct vnode *tvp, struct componentname *tcnp,
-              int mode, int flags, vfs_context_t context)
+              int mode, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_copyfile_args a;
@@ -4436,14 +4744,13 @@ VNOP_COPYFILE(struct vnode *fvp, struct vnode *tdvp, struct vnode *tvp, struct c
 	a.a_tcnp = tcnp;
 	a.a_mode = mode;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	_err = (*fvp->v_op[vnop_copyfile_desc.vdesc_offset])(&a);
 	return (_err);
 }
 
-
 errno_t
-VNOP_GETXATTR(vnode_t vp, const char *name, uio_t uio, size_t *size, int options, vfs_context_t context)
+VNOP_GETXATTR(vnode_t vp, const char *name, uio_t uio, size_t *size, int options, vfs_context_t ctx)
 {
 	struct vnop_getxattr_args a;
 	int error;
@@ -4456,7 +4763,7 @@ VNOP_GETXATTR(vnode_t vp, const char *name, uio_t uio, size_t *size, int options
 	a.a_uio = uio;
 	a.a_size = size;
 	a.a_options = options;
-	a.a_context = context;
+	a.a_context = ctx;
 
 	thread_safe = THREAD_SAFE_FS(vp);
 	if (!thread_safe) {
@@ -4472,7 +4779,7 @@ VNOP_GETXATTR(vnode_t vp, const char *name, uio_t uio, size_t *size, int options
 }
 
 errno_t
-VNOP_SETXATTR(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_t context)
+VNOP_SETXATTR(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_t ctx)
 {
 	struct vnop_setxattr_args a;
 	int error;
@@ -4484,7 +4791,7 @@ VNOP_SETXATTR(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_
 	a.a_name = name;
 	a.a_uio = uio;
 	a.a_options = options;
-	a.a_context = context;
+	a.a_context = ctx;
 
 	thread_safe = THREAD_SAFE_FS(vp);
 	if (!thread_safe) {
@@ -4496,11 +4803,13 @@ VNOP_SETXATTR(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_
 	if (!thread_safe) {
 		unlock_fsnode(vp, &funnel_state);
 	}
+	if (error == 0)
+	        vnode_uncache_authorized_action(vp, KAUTH_INVALIDATE_CACHED_RIGHTS);
 	return (error);
 }
 
 errno_t
-VNOP_REMOVEXATTR(vnode_t vp, const char *name, int options, vfs_context_t context)
+VNOP_REMOVEXATTR(vnode_t vp, const char *name, int options, vfs_context_t ctx)
 {
 	struct vnop_removexattr_args a;
 	int error;
@@ -4511,7 +4820,7 @@ VNOP_REMOVEXATTR(vnode_t vp, const char *name, int options, vfs_context_t contex
 	a.a_vp = vp;
 	a.a_name = name;
 	a.a_options = options;
-	a.a_context = context;
+	a.a_context = ctx;
 
 	thread_safe = THREAD_SAFE_FS(vp);
 	if (!thread_safe) {
@@ -4527,7 +4836,7 @@ VNOP_REMOVEXATTR(vnode_t vp, const char *name, int options, vfs_context_t contex
 }
 
 errno_t
-VNOP_LISTXATTR(vnode_t vp, uio_t uio, size_t *size, int options, vfs_context_t context)
+VNOP_LISTXATTR(vnode_t vp, uio_t uio, size_t *size, int options, vfs_context_t ctx)
 {
 	struct vnop_listxattr_args a;
 	int error;
@@ -4539,7 +4848,7 @@ VNOP_LISTXATTR(vnode_t vp, uio_t uio, size_t *size, int options, vfs_context_t c
 	a.a_uio = uio;
 	a.a_size = size;
 	a.a_options = options;
-	a.a_context = context;
+	a.a_context = ctx;
 
 	thread_safe = THREAD_SAFE_FS(vp);
 	if (!thread_safe) {
@@ -4648,18 +4957,15 @@ struct vnop_blockmap_args {
 };
 #endif /* 0*/
 errno_t 
-VNOP_BLOCKMAP(struct vnode *vp, off_t foffset, size_t size, daddr64_t *bpn, size_t *run, void *poff, int flags, vfs_context_t context)
+VNOP_BLOCKMAP(struct vnode *vp, off_t foffset, size_t size, daddr64_t *bpn, size_t *run, void *poff, int flags, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_blockmap_args a;
 	int thread_safe;
 	int funnel_state = 0;
-	struct vfs_context acontext;
 
-	if (context == NULL) {
-		acontext.vc_proc = current_proc();
-		acontext.vc_ucred = kauth_cred_get();
-		context = &acontext;
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	a.a_desc = &vnop_blockmap_desc;
 	a.a_vp = vp;
@@ -4669,7 +4975,7 @@ VNOP_BLOCKMAP(struct vnode *vp, off_t foffset, size_t size, daddr64_t *bpn, size
 	a.a_run = run;
 	a.a_poff = poff;
 	a.a_flags = flags;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4726,7 +5032,7 @@ struct vnop_kqfilt_add_args {
 };
 #endif
 errno_t
-VNOP_KQFILT_ADD(struct vnode *vp, struct knote *kn, vfs_context_t context)
+VNOP_KQFILT_ADD(struct vnode *vp, struct knote *kn, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_kqfilt_add_args a;
@@ -4736,7 +5042,7 @@ VNOP_KQFILT_ADD(struct vnode *vp, struct knote *kn, vfs_context_t context)
 	a.a_desc = VDESC(vnop_kqfilt_add);
 	a.a_vp = vp;
 	a.a_kn = kn;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4760,7 +5066,7 @@ struct vnop_kqfilt_remove_args {
 };
 #endif
 errno_t
-VNOP_KQFILT_REMOVE(struct vnode *vp, uintptr_t ident, vfs_context_t context)
+VNOP_KQFILT_REMOVE(struct vnode *vp, uintptr_t ident, vfs_context_t ctx)
 {
 	int _err;
 	struct vnop_kqfilt_remove_args a;
@@ -4770,7 +5076,7 @@ VNOP_KQFILT_REMOVE(struct vnode *vp, uintptr_t ident, vfs_context_t context)
 	a.a_desc = VDESC(vnop_kqfilt_remove);
 	a.a_vp = vp;
 	a.a_ident = ident;
-	a.a_context = context;
+	a.a_context = ctx;
 	thread_safe = THREAD_SAFE_FS(vp);
 
 	if (!thread_safe) {
@@ -4785,3 +5091,101 @@ VNOP_KQFILT_REMOVE(struct vnode *vp, uintptr_t ident, vfs_context_t context)
 	return(_err);
 }
 
+#if 0
+struct vnop_setlabel_args {
+	struct vnodeop_desc *a_desc;
+	struct vnode *a_vp;
+	struct label *a_vl;
+	vfs_context_t a_context;
+};
+#endif
+errno_t
+VNOP_SETLABEL(struct vnode *vp, struct label *label, vfs_context_t ctx)
+{
+	int _err;
+	struct vnop_setlabel_args a;
+	int thread_safe;
+	int funnel_state = 0;
+
+	a.a_desc = VDESC(vnop_setlabel);
+	a.a_vp = vp;
+	a.a_vl = label;
+	a.a_context = ctx;
+	thread_safe = THREAD_SAFE_FS(vp);
+
+	if (!thread_safe) {
+		if ( (_err = lock_fsnode(vp, &funnel_state)) ) {
+			return (_err);
+		}
+	}
+	_err = (*vp->v_op[vnop_setlabel_desc.vdesc_offset])(&a);
+	if (!thread_safe) {
+		unlock_fsnode(vp, &funnel_state);
+	}
+	return(_err);
+}
+
+
+#if NAMEDSTREAMS
+/*
+ * Get a named streamed
+ */
+errno_t 
+VNOP_GETNAMEDSTREAM(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperation operation, int flags, vfs_context_t ctx)
+{
+	struct vnop_getnamedstream_args a;
+
+	if (!THREAD_SAFE_FS(vp))
+		return (ENOTSUP);
+	a.a_desc = &vnop_getnamedstream_desc;
+	a.a_vp = vp;
+	a.a_svpp = svpp;
+	a.a_name = name;
+	a.a_operation = operation;
+	a.a_flags = flags;
+	a.a_context = ctx;
+
+	return (*vp->v_op[vnop_getnamedstream_desc.vdesc_offset])(&a);
+}
+
+/*
+ * Create a named streamed
+ */
+errno_t 
+VNOP_MAKENAMEDSTREAM(vnode_t vp, vnode_t *svpp, const char *name, int flags, vfs_context_t ctx)
+{
+	struct vnop_makenamedstream_args a;
+
+	if (!THREAD_SAFE_FS(vp))
+		return (ENOTSUP);
+	a.a_desc = &vnop_makenamedstream_desc;
+	a.a_vp = vp;
+	a.a_svpp = svpp;
+	a.a_name = name;
+	a.a_flags = flags;
+	a.a_context = ctx;
+
+	return (*vp->v_op[vnop_makenamedstream_desc.vdesc_offset])(&a);
+}
+
+
+/*
+ * Remove a named streamed
+ */
+errno_t 
+VNOP_REMOVENAMEDSTREAM(vnode_t vp, vnode_t svp, const char *name, int flags, vfs_context_t ctx)
+{
+	struct vnop_removenamedstream_args a;
+
+	if (!THREAD_SAFE_FS(vp))
+		return (ENOTSUP);
+	a.a_desc = &vnop_removenamedstream_desc;
+	a.a_vp = vp;
+	a.a_svp = svp;
+	a.a_name = name;
+	a.a_flags = flags;
+	a.a_context = ctx;
+
+	return (*vp->v_op[vnop_removenamedstream_desc.vdesc_offset])(&a);
+}
+#endif
diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c
index f8fdcd653..dcc9e1b71 100644
--- a/bsd/vfs/vfs_attrlist.c
+++ b/bsd/vfs/vfs_attrlist.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 1995-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #include <sys/param.h>
@@ -39,6 +51,10 @@
 #include <miscfs/specfs/specdev.h>
 #include <hfs/hfs.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 #define ATTR_TIME_SIZE	-1
 
 /*
@@ -140,6 +156,22 @@ attrlist_pack_string(struct _attrlist_buf *ab, const char *source, ssize_t count
 	ab->varcursor += roundup(count + 1, 4);
 }
 
+#define ATTR_PACK4(AB, V)                                                 \
+	do {                                                              \
+		if ((AB.allocated - (AB.fixedcursor - AB.base)) >= 4) {   \
+			*(uint32_t *)AB.fixedcursor = V;                  \
+			AB.fixedcursor += 4;                              \
+		}                                                         \
+	} while (0)
+
+#define ATTR_PACK8(AB, V)                                                 \
+	do {                                                              \
+		if ((AB.allocated - (AB.fixedcursor - AB.base)) >= 8) {   \
+			*(uint64_t *)AB.fixedcursor = *(uint64_t *)&V;    \
+			AB.fixedcursor += 8;                              \
+		}                                                         \
+	} while (0)
+
 #define ATTR_PACK(b, v)	attrlist_pack_fixed(b, &v, sizeof(v))
 #define ATTR_PACK_CAST(b, t, v)						\
 	do {								\
@@ -151,9 +183,9 @@ attrlist_pack_string(struct _attrlist_buf *ab, const char *source, ssize_t count
 	do {										\
 		if (is64) {								\
 			struct user_timespec us = {v.tv_sec, v.tv_nsec};		\
-			ATTR_PACK(b, us);						\
+			ATTR_PACK(&b, us);						\
 		} else {								\
-			ATTR_PACK(b, v);						\
+			ATTR_PACK8(b, v);						\
 		}									\
 	} while(0)
 
@@ -302,10 +334,12 @@ static struct getattrlist_attrtab getattrlist_common_tab[] = {
 	{ATTR_CMN_EXTENDED_SECURITY, VATTR_BIT(va_acl),	sizeof(struct attrreference),		KAUTH_VNODE_READ_SECURITY},
 	{ATTR_CMN_UUID,		VATTR_BIT(va_uuuid),		sizeof(guid_t),			KAUTH_VNODE_READ_ATTRIBUTES},
 	{ATTR_CMN_GRPUUID,	VATTR_BIT(va_guuid),		sizeof(guid_t),			KAUTH_VNODE_READ_ATTRIBUTES},
+	{ATTR_CMN_FILEID,	VATTR_BIT(va_fileid), 		sizeof(uint64_t),		KAUTH_VNODE_READ_ATTRIBUTES},
+	{ATTR_CMN_PARENTID,	VATTR_BIT(va_parentid),		sizeof(uint64_t),		KAUTH_VNODE_READ_ATTRIBUTES},
 	{0, 0, 0, 0}
 };
 static struct getattrlist_attrtab getattrlist_dir_tab[] = {
-	{ATTR_DIR_LINKCOUNT,	VATTR_BIT(va_nlink),		sizeof(uint32_t),		KAUTH_VNODE_READ_ATTRIBUTES},
+	{ATTR_DIR_LINKCOUNT,	VATTR_BIT(va_dirlinkcount),	sizeof(uint32_t),		KAUTH_VNODE_READ_ATTRIBUTES},
 	{ATTR_DIR_ENTRYCOUNT,	VATTR_BIT(va_nchildren),	sizeof(uint32_t),		KAUTH_VNODE_READ_ATTRIBUTES},
 	/* ATTR_DIR_ENTRYCOUNT falls back to va_nlink-2 if va_nchildren isn't supported, so request va_nlink just in case */
 	{ATTR_DIR_ENTRYCOUNT,	VATTR_BIT(va_nlink),		0,				KAUTH_VNODE_READ_ATTRIBUTES},
@@ -325,6 +359,38 @@ static struct getattrlist_attrtab getattrlist_file_tab[] = {
 	{0, 0, 0, 0}
 };	
 
+/*
+ * The following are attributes that VFS can derive.
+ *
+ * A majority of them are the same attributes that are required for stat(2) and statfs(2).
+ */
+#define VFS_DFLT_ATTR_VOL	(ATTR_VOL_FSTYPE | ATTR_VOL_SIGNATURE |  \
+				 ATTR_VOL_SIZE | ATTR_VOL_SPACEFREE |  \
+				 ATTR_VOL_SPACEAVAIL | ATTR_VOL_MINALLOCATION |  \
+				 ATTR_VOL_ALLOCATIONCLUMP |  ATTR_VOL_IOBLOCKSIZE |  \
+				 ATTR_VOL_MOUNTPOINT | ATTR_VOL_MOUNTFLAGS |  \
+				 ATTR_VOL_MOUNTEDDEVICE | ATTR_VOL_CAPABILITIES |  \
+				 ATTR_VOL_ATTRIBUTES)
+
+#define VFS_DFLT_ATTR_CMN	(ATTR_CMN_NAME | ATTR_CMN_DEVID |  \
+				 ATTR_CMN_FSID | ATTR_CMN_OBJTYPE |  \
+				 ATTR_CMN_OBJTAG | ATTR_CMN_OBJID |  \
+				 ATTR_CMN_PAROBJID | ATTR_CMN_SCRIPT |  \
+				 ATTR_CMN_MODTIME | ATTR_CMN_CHGTIME |  \
+				 ATTR_CMN_ACCTIME | ATTR_CMN_FNDRINFO |  \
+				 ATTR_CMN_OWNERID  | ATTR_CMN_GRPID |  \
+				 ATTR_CMN_ACCESSMASK | ATTR_CMN_FLAGS |  \
+				 ATTR_CMN_USERACCESS | ATTR_CMN_FILEID | \
+				 ATTR_CMN_PARENTID)
+
+#define VFS_DFLT_ATTR_DIR	(ATTR_DIR_LINKCOUNT | ATTR_DIR_MOUNTSTATUS)
+
+#define VFS_DFLT_ATTR_FILE	(ATTR_FILE_LINKCOUNT | ATTR_FILE_TOTALSIZE |  \
+				 ATTR_FILE_ALLOCSIZE  | ATTR_FILE_IOBLOCKSIZE |  \
+				 ATTR_FILE_DEVTYPE | ATTR_FILE_DATALENGTH |  \
+				 ATTR_FILE_DATAALLOCSIZE | ATTR_FILE_RSRCLENGTH |  \
+				 ATTR_FILE_RSRCALLOCSIZE)
+
 static int
 getattrlist_parsetab(struct getattrlist_attrtab *tab, attrgroup_t attrs, struct vnode_attr *vap,
     ssize_t *sizep, kauth_action_t *actionp, int is_64bit)
@@ -398,10 +464,11 @@ setattrlist_setfinderinfo(vnode_t vp, char *fndrinfo, struct vfs_context *ctx)
 		uio_free(auio);
 	}
 
+#if CONFIG_FSE
 	if (error == 0 && need_fsevent(FSE_FINDER_INFO_CHANGED, vp)) {
 	    add_fsevent(FSE_FINDER_INFO_CHANGED, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
 	}
-
+#endif
 	return (error);
 }
 
@@ -452,8 +519,8 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 	struct _attrlist_buf ab;
 	int		error;
 	ssize_t		fixedsize, varsize;
-	const char	*cnp;
-	ssize_t		cnl;
+	const char	*cnp = NULL;	/* protected by ATTR_CMN_NAME */
+	ssize_t		cnl = 0;	/* protected by ATTR_CMN_NAME */
 	mount_t		mnt;
 
 	ab.base = NULL;
@@ -492,6 +559,11 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 			}
 		}
 
+#if CONFIG_MACF
+		error = mac_mount_check_getattr(ctx, mnt, &vs);
+		if (error != 0)
+			goto out;
+#endif
 		VFS_DEBUG(ctx, vp, "ATTRLIST -       calling to get %016llx with supported %016llx", vs.f_active, vs.f_supported);
 		if ((error = vfs_getattr(mnt, &vs, ctx)) != 0) {
 			VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: filesystem returned %d", error);
@@ -521,6 +593,47 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 			    && !VFSATTR_IS_SUPPORTED(&vs, f_bsize))
 				VFSATTR_RETURN(&vs, f_bsize, mnt->mnt_devblocksize);
 
+			/* default value for volume f_attributes */
+			if (VFSATTR_IS_ACTIVE(&vs, f_attributes)
+			    && !VFSATTR_IS_SUPPORTED(&vs, f_attributes)) {
+				vol_attributes_attr_t *attrp = &vs.f_attributes;
+		
+				attrp->validattr.commonattr = VFS_DFLT_ATTR_CMN;
+				attrp->validattr.volattr = VFS_DFLT_ATTR_VOL;
+				attrp->validattr.dirattr = VFS_DFLT_ATTR_DIR;
+				attrp->validattr.fileattr = VFS_DFLT_ATTR_FILE;
+				attrp->validattr.forkattr = 0;
+		
+				attrp->nativeattr.commonattr =  0;
+				attrp->nativeattr.volattr = 0;
+				attrp->nativeattr.dirattr = 0;
+				attrp->nativeattr.fileattr = 0;
+				attrp->nativeattr.forkattr = 0;
+				VFSATTR_SET_SUPPORTED(&vs, f_attributes);
+			}
+
+			/* default value for volume f_capabilities */
+			if (VFSATTR_IS_ACTIVE(&vs, f_capabilities)) {
+				/* getattrlist is always supported now. */
+				if (!VFSATTR_IS_SUPPORTED(&vs, f_capabilities)) {
+					vs.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] = 0;
+					vs.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] = VOL_CAP_INT_ATTRLIST;
+					vs.f_capabilities.capabilities[VOL_CAPABILITIES_RESERVED1] = 0;
+					vs.f_capabilities.capabilities[VOL_CAPABILITIES_RESERVED2] = 0;
+	
+					vs.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] = 0;
+					vs.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] = VOL_CAP_INT_ATTRLIST;
+					vs.f_capabilities.valid[VOL_CAPABILITIES_RESERVED1] = 0;
+					vs.f_capabilities.valid[VOL_CAPABILITIES_RESERVED2] = 0;
+					VFSATTR_SET_SUPPORTED(&vs, f_capabilities);
+				}
+				else {
+					/* OR in VOL_CAP_INT_ATTRLIST if f_capabilities is supported */
+					vs.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] |= VOL_CAP_INT_ATTRLIST;
+					vs.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] |= VOL_CAP_INT_ATTRLIST;
+				}
+			}
+
 			/* check to see if our fixups were enough */
 			if (!VFSATTR_ALL_SUPPORTED(&vs)) {
 				error = EINVAL;
@@ -615,38 +728,38 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 	if (alp->commonattr & ATTR_CMN_NAME)
 		attrlist_pack_string(&ab, cnp, cnl);
 	if (alp->commonattr & ATTR_CMN_DEVID)
-		ATTR_PACK_CAST(&ab, dev_t, mnt->mnt_vfsstat.f_fsid.val[0]);
+		ATTR_PACK4(ab, mnt->mnt_vfsstat.f_fsid.val[0]);
 	if (alp->commonattr & ATTR_CMN_FSID)
-		ATTR_PACK(&ab, mnt->mnt_vfsstat.f_fsid);
+		ATTR_PACK8(ab, mnt->mnt_vfsstat.f_fsid);
 	if (alp->commonattr & ATTR_CMN_OBJTYPE)
-		ATTR_PACK_CAST(&ab, fsobj_type_t, 0);
+		ATTR_PACK4(ab, 0);
 	if (alp->commonattr & ATTR_CMN_OBJTAG)
-		ATTR_PACK_CAST(&ab, fsobj_tag_t, vp->v_tag);
+		ATTR_PACK4(ab, vp->v_tag);
 	if (alp->commonattr & ATTR_CMN_OBJID) {
 		fsobj_id_t f = {0, 0};
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	if (alp->commonattr & ATTR_CMN_OBJPERMANENTID) {
 		fsobj_id_t f = {0, 0};
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	if (alp->commonattr & ATTR_CMN_PAROBJID) {
 		fsobj_id_t f = {0, 0};
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	/* note that this returns the encoding for the volume name, not the node name */
 	if (alp->commonattr & ATTR_CMN_SCRIPT)
-		ATTR_PACK_CAST(&ab, text_encoding_t, va.va_encoding);
+		ATTR_PACK4(ab, va.va_encoding);
 	if (alp->commonattr & ATTR_CMN_CRTIME)
-		ATTR_PACK_TIME(&ab, vs.f_create_time, is_64bit);
+		ATTR_PACK_TIME(ab, vs.f_create_time, is_64bit);
 	if (alp->commonattr & ATTR_CMN_MODTIME)
-		ATTR_PACK_TIME(&ab, vs.f_modify_time, is_64bit);
+		ATTR_PACK_TIME(ab, vs.f_modify_time, is_64bit);
 	if (alp->commonattr & ATTR_CMN_CHGTIME)
-		ATTR_PACK_TIME(&ab, vs.f_modify_time, is_64bit);
+		ATTR_PACK_TIME(ab, vs.f_modify_time, is_64bit);
 	if (alp->commonattr & ATTR_CMN_ACCTIME)
-		ATTR_PACK_TIME(&ab, vs.f_access_time, is_64bit);
+		ATTR_PACK_TIME(ab, vs.f_access_time, is_64bit);
 	if (alp->commonattr & ATTR_CMN_BKUPTIME)
-		ATTR_PACK_TIME(&ab, vs.f_backup_time, is_64bit);
+		ATTR_PACK_TIME(ab, vs.f_backup_time, is_64bit);
 	if (alp->commonattr & ATTR_CMN_FNDRINFO) {
 		char f[32];
 		/*
@@ -662,13 +775,13 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 		attrlist_pack_fixed(&ab, f, sizeof(f));
 	}
 	if (alp->commonattr & ATTR_CMN_OWNERID)
-		ATTR_PACK(&ab, va.va_uid);
+		ATTR_PACK4(ab, va.va_uid);
 	if (alp->commonattr & ATTR_CMN_GRPID)
-		ATTR_PACK(&ab, va.va_gid);
+		ATTR_PACK4(ab, va.va_gid);
 	if (alp->commonattr & ATTR_CMN_ACCESSMASK)
 		ATTR_PACK_CAST(&ab, uint32_t, va.va_mode);
 	if (alp->commonattr & ATTR_CMN_FLAGS)
-		ATTR_PACK(&ab, va.va_flags);
+		ATTR_PACK4(ab, va.va_flags);
 	if (alp->commonattr & ATTR_CMN_USERACCESS) {	/* XXX this is expensive and also duplicate work */
 		uint32_t	perms = 0;
 		if (vnode_isdir(vp)) {
@@ -687,8 +800,24 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_EXECUTE, ctx) == 0)
 				perms |= X_OK;
 		}
+#if CONFIG_MACF
+		/* 
+		 * Rather than MAC preceding DAC, in this case we want
+		 * the smallest set of permissions granted by both MAC & DAC
+		 * checks.  We won't add back any permissions.
+		 */
+		if (perms & W_OK)
+			if (mac_vnode_check_access(ctx, vp, W_OK) != 0)
+				perms &= ~W_OK;
+		if (perms & R_OK)
+			if (mac_vnode_check_access(ctx, vp, R_OK) != 0)
+				perms &= ~R_OK;
+		if (perms & X_OK)
+			if (mac_vnode_check_access(ctx, vp, X_OK) != 0)
+				perms &= ~X_OK;
+#endif /* MAC */
 		KAUTH_DEBUG("ATTRLIST - returning user access %x", perms);
-		ATTR_PACK(&ab, perms);
+		ATTR_PACK4(ab, perms);
 	}
 
 	/* volume attributes **************************************************/
@@ -739,6 +868,12 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 	}
 	if (alp->volattr & ATTR_VOL_ATTRIBUTES) {
 		/* fix up volume attribute information */
+
+		vs.f_attributes.validattr.commonattr |= VFS_DFLT_ATTR_CMN;
+		vs.f_attributes.validattr.volattr |= VFS_DFLT_ATTR_VOL;
+		vs.f_attributes.validattr.dirattr |= VFS_DFLT_ATTR_DIR;
+		vs.f_attributes.validattr.fileattr |= VFS_DFLT_ATTR_FILE;
+
 		if (vfs_extendedsecurity(mnt)) {
 			vs.f_attributes.validattr.commonattr |= (ATTR_CMN_EXTENDED_SECURITY | ATTR_CMN_UUID | ATTR_CMN_GRPUUID);
 		} else {
@@ -750,10 +885,10 @@ getvolattrlist(vnode_t vp, struct getattrlist_args *uap, struct attrlist *alp,
 	
 	/* diagnostic */
 	if ((ab.fixedcursor - ab.base) != fixedsize)
-		panic("packed field size mismatch; allocated %d but packed %d for common %08x vol %08x",
+		panic("packed field size mismatch; allocated %ld but packed %d for common %08x vol %08x",
 		    fixedsize, ab.fixedcursor - ab.base, alp->commonattr, alp->volattr);
 	if (ab.varcursor != (ab.base + ab.needed))
-		panic("packed variable field size mismatch; used %d but expected %d", ab.varcursor - ab.base, ab.needed);
+		panic("packed variable field size mismatch; used %d but expected %ld", ab.varcursor - ab.base, ab.needed);
 
 	/*
 	 * In the compatible case, we report the smaller of the required and returned sizes.
@@ -778,11 +913,11 @@ out:
  * Obtain attribute information about a filesystem object.
  */
 int
-getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *retval)
+getattrlist(proc_t p, struct getattrlist_args *uap, __unused register_t *retval)
 {
 	struct attrlist	al;
 	struct vnode_attr va;
-	struct vfs_context context, *ctx;
+	struct vfs_context *ctx;
 	struct nameidata nd;
 	struct _attrlist_buf ab;
 	vnode_t		vp;
@@ -790,15 +925,15 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	kauth_action_t	action;
 	ssize_t		fixedsize, varsize;
 	const char	*cnp;
-	char		*vname = NULL;
+	const char	*vname = NULL;
 	ssize_t		cnl;
+	int		proc_is64;
 	int		error;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	ctx = &context;
+	ctx = vfs_context_current();
 	vp = NULL;
 	error = 0;
+	proc_is64 = proc_is64bit(p);
 	VATTR_INIT(&va);
 	va.va_name = NULL;
 	ab.base = NULL;
@@ -808,10 +943,10 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	/*
 	 * Look up the file.
 	 */
-	nameiflags = AUDITVNPATH1;
+	nameiflags = NOTRIGGER | AUDITVNPATH1;
 	if (!(uap->options & FSOPT_NOFOLLOW))
 		nameiflags |= FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, UIO_USERSPACE, uap->path, &context);
+	NDINIT(&nd, LOOKUP, nameiflags, UIO_USERSPACE, uap->path, ctx);
 
 	if ((error = namei(&nd)) != 0)
 		goto out;
@@ -832,6 +967,12 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	    vp, p->p_comm, al.commonattr, al.volattr, al.fileattr, al.dirattr, al.forkattr,
 	    (uap->options & FSOPT_NOFOLLOW) ? "no":"", vp->v_name);
 	
+#if CONFIG_MACF
+	error = mac_vnode_check_getattrlist(ctx, vp, &al);
+	if (error)
+		goto out;
+#endif /* MAC */
+
 	/*
 	 * It is legal to request volume or file attributes,
 	 * but not both.
@@ -843,18 +984,18 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 			goto out;
 		}
 		/* handle volume attribute request */
-		error = getvolattrlist(vp, uap, &al, &context, proc_is64bit(p));
+		error = getvolattrlist(vp, uap, &al, ctx, proc_is64);
 		goto out;
 	}
 
 	/*
 	 * Set up the vnode_attr structure and authorise.
 	 */
-	if ((error = getattrlist_setupvattr(&al, &va, &fixedsize, &action, proc_is64bit(p), vnode_isdir(vp))) != 0) {
+	if ((error = getattrlist_setupvattr(&al, &va, &fixedsize, &action, proc_is64, vnode_isdir(vp))) != 0) {
 		VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: setup for request failed");
 		goto out;
 	}
-	if ((error = vnode_authorize(vp, NULL, action, &context)) != 0) {
+	if ((error = vnode_authorize(vp, NULL, action, ctx)) != 0) {
 		VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: authorisation failed/denied");
 		goto out;
 	}
@@ -875,7 +1016,7 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 		/*
 		 * Call the filesystem.
 		 */
-		if ((error = vnode_getattr(vp, &va, &context)) != 0) {
+		if ((error = vnode_getattr(vp, &va, ctx)) != 0) {
 			VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: filesystem returned %d", error);
 			goto out;
 		}
@@ -887,13 +1028,13 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 			 * There are a couple of special cases.  If we are after object IDs,
 			 * we can make do with va_fileid.
 			 */
-			if ((al.commonattr & (ATTR_CMN_OBJID | ATTR_CMN_OBJPERMANENTID)) && !VATTR_IS_SUPPORTED(&va, va_linkid))
+			if ((al.commonattr & (ATTR_CMN_OBJID | ATTR_CMN_OBJPERMANENTID | ATTR_CMN_FILEID)) && !VATTR_IS_SUPPORTED(&va, va_linkid))
 				VATTR_CLEAR_ACTIVE(&va, va_linkid);	/* forget we wanted this */
 			/*
 			 * Many (most?) filesystems don't know their parent object id.  We can get it the
 			 * hard way.
 			 */
-			if ((al.commonattr & ATTR_CMN_PAROBJID) && !VATTR_IS_SUPPORTED(&va, va_parentid))
+			if ((al.commonattr & (ATTR_CMN_PAROBJID | ATTR_CMN_PARENTID)) && !VATTR_IS_SUPPORTED(&va, va_parentid))
 				VATTR_CLEAR_ACTIVE(&va, va_parentid);
 			/*
 			 * And we can report datasize/alloc from total.
@@ -916,21 +1057,14 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 				VATTR_CLEAR_ACTIVE(&va, va_name);
 			}
 
-			/*
-			 * We used to return va_nlink-2 for ATTR_DIR_ENTRYCOUNT.  The va_nchildren
-			 * field is preferred, but we'll fall back to va_nlink-2 for compatibility
-			 * with file systems which haven't adopted va_nchildren.  Note: the "- 2"
-			 * reflects the "." and ".." entries which are reported via POSIX APIs, but
-			 * not via Carbon (since they don't in fact exist in HFS).
-			 */
-			if ((al.dirattr & ATTR_DIR_ENTRYCOUNT) && !VATTR_IS_SUPPORTED(&va, va_nchildren) &&
-			    VATTR_IS_SUPPORTED(&va, va_nlink)) {
-				VATTR_RETURN(&va, va_nchildren, va.va_nlink - 2);
+			/* If va_dirlinkcount isn't supported use a default of 1. */
+			if ((al.dirattr & ATTR_DIR_LINKCOUNT) && !VATTR_IS_SUPPORTED(&va, va_dirlinkcount)) {
+				VATTR_RETURN(&va, va_dirlinkcount, 1);
 			}
 			
 			/* check again */
 			if (!VATTR_ALL_SUPPORTED(&va)) {
-				error = ENOTSUP;
+				error = EINVAL;
 				VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: could not get all requested file attributes");
 				VFS_DEBUG(ctx, vp, "ATTRLIST -        have %016llx wanted %016llx missing %016llx",
 				    va.va_supported, va.va_active, va.va_active & ~va.va_supported);
@@ -994,7 +1128,7 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	 * Allocate a target buffer for attribute results.
 	 *
 	 * Note that we won't ever copy out more than the caller requested, even though
-	 * we might have to allocate more than they offer do that the diagnostic checks
+	 * we might have to allocate more than they offer so that the diagnostic checks
 	 * don't result in a panic if the caller's buffer is too small..
 	 */
 	ab.allocated = fixedsize + varsize;
@@ -1010,6 +1144,36 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 		goto out;
 	}
 
+	/* set the S_IFMT bits for the mode */
+	if (al.commonattr & ATTR_CMN_ACCESSMASK) {
+		switch (vp->v_type) {
+		case VREG:
+			va.va_mode |= S_IFREG;
+			break;
+		case VDIR:
+			va.va_mode |= S_IFDIR;
+			break;
+		case VBLK:
+			va.va_mode |= S_IFBLK;
+			break;
+		case VCHR:
+			va.va_mode |= S_IFCHR;
+			break;
+		case VLNK:
+			va.va_mode |= S_IFLNK;
+			break;
+		case VSOCK:
+			va.va_mode |= S_IFSOCK;
+			break;
+		case VFIFO:
+			va.va_mode |= S_IFIFO;
+			break;
+		default:
+			error = EBADF;
+			goto out;
+		}
+	}
+
 	/*
 	 * Pack results into the destination buffer.
 	 */
@@ -1021,13 +1185,13 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	if (al.commonattr & ATTR_CMN_NAME)
 		attrlist_pack_string(&ab, cnp, cnl);
 	if (al.commonattr & ATTR_CMN_DEVID)
-		ATTR_PACK_CAST(&ab, dev_t, vp->v_mount->mnt_vfsstat.f_fsid.val[0]);
+		ATTR_PACK4(ab, vp->v_mount->mnt_vfsstat.f_fsid.val[0]);
 	if (al.commonattr & ATTR_CMN_FSID)
-		ATTR_PACK(&ab, vp->v_mount->mnt_vfsstat.f_fsid);
+		ATTR_PACK8(ab, vp->v_mount->mnt_vfsstat.f_fsid);
 	if (al.commonattr & ATTR_CMN_OBJTYPE)
-		ATTR_PACK_CAST(&ab, fsobj_type_t, vp->v_type);
+		ATTR_PACK4(ab, vp->v_type);
 	if (al.commonattr & ATTR_CMN_OBJTAG)
-		ATTR_PACK_CAST(&ab, fsobj_tag_t, vp->v_tag);
+		ATTR_PACK4(ab, vp->v_tag);
 	if (al.commonattr & ATTR_CMN_OBJID) {
 		fsobj_id_t f;
 		/*
@@ -1042,7 +1206,7 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 			f.fid_objno = va.va_fileid;
 		}
 		f.fid_generation = 0;
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	if (al.commonattr & ATTR_CMN_OBJPERMANENTID) {
 		fsobj_id_t f;
@@ -1058,7 +1222,7 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 			f.fid_objno = va.va_fileid;
 		}
 		f.fid_generation = 0;
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	if (al.commonattr & ATTR_CMN_PAROBJID) {
 		fsobj_id_t f;
@@ -1077,33 +1241,33 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 			pvp = vnode_getparent(vp);
 
 			if (pvp == NULLVP) {
-				error = ENOTSUP;
+				error = EINVAL;
 				goto out;
 			}
 			VATTR_INIT(&lva);
 			VATTR_WANTED(&lva, va_fileid);
-			error = vnode_getattr(pvp, &lva, &context);
+			error = vnode_getattr(pvp, &lva, ctx);
 			vnode_put(pvp);
-			
+
 			if (error != 0)
 				goto out;
 			f.fid_objno = lva.va_fileid;
 		}
 		f.fid_generation = 0;
-		ATTR_PACK(&ab, f);
+		ATTR_PACK8(ab, f);
 	}
 	if (al.commonattr & ATTR_CMN_SCRIPT)
-		ATTR_PACK_CAST(&ab, text_encoding_t, va.va_encoding);
+		ATTR_PACK4(ab, va.va_encoding);
 	if (al.commonattr & ATTR_CMN_CRTIME)
-		ATTR_PACK_TIME(&ab, va.va_create_time, proc_is64bit(p));
+		ATTR_PACK_TIME(ab, va.va_create_time, proc_is64);
 	if (al.commonattr & ATTR_CMN_MODTIME)
-		ATTR_PACK_TIME(&ab, va.va_modify_time, proc_is64bit(p));
+		ATTR_PACK_TIME(ab, va.va_modify_time, proc_is64);
 	if (al.commonattr & ATTR_CMN_CHGTIME)
-		ATTR_PACK_TIME(&ab, va.va_change_time, proc_is64bit(p));
+		ATTR_PACK_TIME(ab, va.va_change_time, proc_is64);
 	if (al.commonattr & ATTR_CMN_ACCTIME)
-		ATTR_PACK_TIME(&ab, va.va_access_time, proc_is64bit(p));
+		ATTR_PACK_TIME(ab, va.va_access_time, proc_is64);
 	if (al.commonattr & ATTR_CMN_BKUPTIME)
-		ATTR_PACK_TIME(&ab, va.va_backup_time, proc_is64bit(p));
+		ATTR_PACK_TIME(ab, va.va_backup_time, proc_is64);
 	if (al.commonattr & ATTR_CMN_FNDRINFO) {
 		uio_t	auio;
 		size_t	fisize;
@@ -1116,11 +1280,11 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 				goto out;
 			} else {
 				uio_addiov(auio, CAST_USER_ADDR_T(ab.fixedcursor), fisize);
-				error = vn_getxattr(vp, XATTR_FINDERINFO_NAME, auio, &fisize, XATTR_NOSECURITY, &context);
+				error = vn_getxattr(vp, XATTR_FINDERINFO_NAME, auio, &fisize, XATTR_NOSECURITY, ctx);
 				uio_free(auio);
 			}
 			if (error != 0) {
-				if ((error == ENOENT) || (error == ENOATTR) || (error == ENOTSUP) || (error == EPERM)) {
+				if ((error == ENOATTR) || (error == ENOENT) || (error == ENOTSUP) || (error == EPERM)) {
 					VFS_DEBUG(ctx, vp, "ATTRLIST - No system.finderinfo attribute, returning zeroes");
 					bzero(ab.fixedcursor, 32);
 					error = 0;
@@ -1135,33 +1299,50 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 		ab.fixedcursor += 32;
 	}
 	if (al.commonattr & ATTR_CMN_OWNERID)
-		ATTR_PACK(&ab, va.va_uid);
+		ATTR_PACK4(ab, va.va_uid);
 	if (al.commonattr & ATTR_CMN_GRPID)
-		ATTR_PACK(&ab, va.va_gid);
+		ATTR_PACK4(ab, va.va_gid);
 	if (al.commonattr & ATTR_CMN_ACCESSMASK)
-		ATTR_PACK_CAST(&ab, uint32_t, va.va_mode);
+		ATTR_PACK4(ab, va.va_mode);
 	if (al.commonattr & ATTR_CMN_FLAGS)
-		ATTR_PACK(&ab, va.va_flags);
+		ATTR_PACK4(ab, va.va_flags);
 	if (al.commonattr & ATTR_CMN_USERACCESS) {	/* this is expensive */
 		uint32_t	perms = 0;
 		if (vnode_isdir(vp)) {
 			if (vnode_authorize(vp, NULL,
-				KAUTH_VNODE_ACCESS | KAUTH_VNODE_ADD_FILE | KAUTH_VNODE_ADD_SUBDIRECTORY | KAUTH_VNODE_DELETE_CHILD, &context) == 0)
+				KAUTH_VNODE_ACCESS | KAUTH_VNODE_ADD_FILE | KAUTH_VNODE_ADD_SUBDIRECTORY | KAUTH_VNODE_DELETE_CHILD, ctx) == 0)
 				perms |= W_OK;
-			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_LIST_DIRECTORY, &context) == 0)
+			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_LIST_DIRECTORY, ctx) == 0)
 				perms |= R_OK;
-			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_SEARCH, &context) == 0)
+			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_SEARCH, ctx) == 0)
 				perms |= X_OK;
 		} else {
-			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_WRITE_DATA, &context) == 0)
+			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_WRITE_DATA, ctx) == 0)
 				perms |= W_OK;
-			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_READ_DATA, &context) == 0)
+			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_READ_DATA, ctx) == 0)
 				perms |= R_OK;
-			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_EXECUTE, &context) == 0)
+			if (vnode_authorize(vp, NULL, KAUTH_VNODE_ACCESS | KAUTH_VNODE_EXECUTE, ctx) == 0)
 				perms |= X_OK;
 		}
+
+#if CONFIG_MACF
+		/* 
+		 * Rather than MAC preceding DAC, in this case we want
+		 * the smallest set of permissions granted by both MAC & DAC
+		 * checks.  We won't add back any permissions.
+		 */
+		if (perms & W_OK)
+			if (mac_vnode_check_access(ctx, vp, W_OK) != 0)
+				perms &= ~W_OK;
+		if (perms & R_OK)
+			if (mac_vnode_check_access(ctx, vp, R_OK) != 0)
+				perms &= ~R_OK;
+		if (perms & X_OK)
+			if (mac_vnode_check_access(ctx, vp, X_OK) != 0)
+				perms &= ~X_OK;
+#endif /* MAC */
 		VFS_DEBUG(ctx, vp, "ATTRLIST - granting perms %d", perms);
-		ATTR_PACK(&ab, perms);
+		ATTR_PACK4(ab, perms);
 	}
 	if (al.commonattr & ATTR_CMN_EXTENDED_SECURITY) {
 		if (VATTR_IS_SUPPORTED(&va, va_acl) && (va.va_acl != NULL)) {
@@ -1191,13 +1372,45 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
  			ATTR_PACK(&ab, va.va_guuid);
   		}
   	}
+	if (al.commonattr & ATTR_CMN_FILEID) {
+		ATTR_PACK8(ab, va.va_fileid);
+	}
+	if (al.commonattr & ATTR_CMN_PARENTID) {
+		uint64_t fileid;
+		/*
+		 * If the filesystem doesn't know the parent ID, we can
+		 * try to get it via v->v_parent.
+		 */
+		if (VATTR_IS_SUPPORTED(&va, va_parentid)) {
+			fileid = va.va_parentid;
+		} else {
+			struct vnode_attr lva;
+			vnode_t	pvp;
+
+			pvp = vnode_getparent(vp);
+
+			if (pvp == NULLVP) {
+				error = EINVAL;
+				goto out;
+			}
+			VATTR_INIT(&lva);
+			VATTR_WANTED(&lva, va_fileid);
+			error = vnode_getattr(pvp, &lva, ctx);
+			vnode_put(pvp);
+			
+			if (error != 0)
+				goto out;
+			fileid = lva.va_fileid;
+		}
+		ATTR_PACK8(ab, fileid);
+	}
 
 	/* directory attributes **************************************************/
 	if (vnode_isdir(vp)) {
 		if (al.dirattr & ATTR_DIR_LINKCOUNT)			/* full count of entries */
-			ATTR_PACK_CAST(&ab, uint32_t, va.va_nlink);
+			ATTR_PACK4(ab, (uint32_t)va.va_dirlinkcount);
 		if (al.dirattr & ATTR_DIR_ENTRYCOUNT)
-			ATTR_PACK_CAST(&ab, uint32_t, va.va_nchildren);
+			ATTR_PACK4(ab, (uint32_t)va.va_nchildren);
 		if (al.dirattr & ATTR_DIR_MOUNTSTATUS)
 			ATTR_PACK_CAST(&ab, uint32_t, (vp->v_flag & VROOT) ? DIR_MNTSTATUS_MNTPOINT : 0);
 	}
@@ -1205,15 +1418,15 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 	/* file attributes **************************************************/
 	if (!vnode_isdir(vp)) {
 		if (al.fileattr & ATTR_FILE_LINKCOUNT)
-			ATTR_PACK_CAST(&ab, uint32_t, va.va_nlink);
+			ATTR_PACK4(ab, (uint32_t)va.va_nlink);
 		if (al.fileattr & ATTR_FILE_TOTALSIZE)
-			ATTR_PACK(&ab, va.va_total_size);
+			ATTR_PACK8(ab, va.va_total_size);
 		if (al.fileattr & ATTR_FILE_ALLOCSIZE)
-			ATTR_PACK(&ab, va.va_total_alloc);
+			ATTR_PACK8(ab, va.va_total_alloc);
 		if (al.fileattr & ATTR_FILE_IOBLOCKSIZE)
-			ATTR_PACK(&ab, va.va_iosize);
+			ATTR_PACK4(ab, va.va_iosize);
 		if (al.fileattr & ATTR_FILE_CLUMPSIZE)
-			ATTR_PACK_CAST(&ab, uint32_t, 0);		/* XXX value is deprecated */
+			ATTR_PACK4(ab, 0);     /* XXX value is deprecated */
 		if (al.fileattr & ATTR_FILE_DEVTYPE) {
 			if ((vp->v_type == VCHR) || (vp->v_type == VBLK)) {
 				ATTR_PACK(&ab, vp->v_specinfo->si_rdev);
@@ -1223,22 +1436,24 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 		}
 		if (al.fileattr & ATTR_FILE_DATALENGTH) {
 			if (VATTR_IS_SUPPORTED(&va, va_data_size)) {
-				ATTR_PACK(&ab, va.va_data_size);
+				ATTR_PACK8(ab, va.va_data_size);
 			} else {
-				ATTR_PACK(&ab, va.va_total_size);
+				ATTR_PACK8(ab, va.va_total_size);
 			}
 		}
 		if (al.fileattr & ATTR_FILE_DATAALLOCSIZE) {
 			if (VATTR_IS_SUPPORTED(&va, va_data_alloc)) {
-				ATTR_PACK(&ab, va.va_data_alloc);
+				ATTR_PACK8(ab, va.va_data_alloc);
 			} else {
-				ATTR_PACK(&ab, va.va_total_alloc);
+				ATTR_PACK8(ab, va.va_total_alloc);
 			}
 		}
 		/* fetch resource fork size/allocation via xattr interface */
 		if (al.fileattr & (ATTR_FILE_RSRCLENGTH | ATTR_FILE_RSRCALLOCSIZE)) {
 			size_t	rsize;
-			if ((error = vn_getxattr(vp, XATTR_RESOURCEFORK_NAME, NULL, &rsize, XATTR_NOSECURITY, &context)) != 0) {
+			uint64_t rlength;
+
+			if ((error = vn_getxattr(vp, XATTR_RESOURCEFORK_NAME, NULL, &rsize, XATTR_NOSECURITY, ctx)) != 0) {
 				if ((error == ENOENT) || (error == ENOATTR) || (error == ENOTSUP) || (error == EPERM)) {
 					rsize = 0;
 					error = 0;
@@ -1246,23 +1461,26 @@ getattrlist(struct proc *p, struct getattrlist_args *uap, __unused register_t *r
 					goto out;
 				}
 			}
-			if (al.fileattr & ATTR_FILE_RSRCLENGTH)
-				ATTR_PACK_CAST(&ab, off_t, rsize);
+			if (al.fileattr & ATTR_FILE_RSRCLENGTH) {
+				rlength = rsize;
+				ATTR_PACK8(ab, rlength);
+			}
 			if (al.fileattr & ATTR_FILE_RSRCALLOCSIZE) {
 				uint32_t  blksize = vp->v_mount->mnt_vfsstat.f_bsize;
 				if (blksize == 0)
 					blksize = 512;
-				ATTR_PACK_CAST(&ab, off_t, (roundup(rsize, blksize)));
+				rlength = roundup(rsize, blksize);
+				ATTR_PACK8(ab, rlength);
 			}
 		}
 	}
 	
 	/* diagnostic */
 	if ((ab.fixedcursor - ab.base) != fixedsize)
-		panic("packed field size mismatch; allocated %d but packed %d for common %08x vol %08x",
+		panic("packed field size mismatch; allocated %ld but packed %d for common %08x vol %08x",
 		    fixedsize, ab.fixedcursor - ab.base, al.commonattr, al.volattr);
 	if (ab.varcursor != (ab.base + ab.needed))
-		panic("packed variable field size mismatch; used %d but expected %d", ab.varcursor - ab.base, ab.needed);
+		panic("packed variable field size mismatch; used %d but expected %ld", ab.varcursor - ab.base, ab.needed);
 
 	/*
 	 * In the compatible case, we report the smaller of the required and returned sizes.
@@ -1322,7 +1540,7 @@ attrlist_unpack_fixed(char **cursor, char *end, void *buf, ssize_t size)
  * Write attributes.
  */
 int
-setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused register_t *retval)
+setattrlist(proc_t p, struct setattrlist_args *uap, __unused register_t *retval)
 {
 	struct attrlist al;
 	struct vfs_context context, *ctx;
@@ -1337,7 +1555,7 @@ setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused regi
 	uint32_t	nace;
 	kauth_filesec_t rfsec;
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = kauth_cred_get();
 	ctx = &context;
 	vp = NULL;
@@ -1421,6 +1639,12 @@ setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused regi
 	}
 	VFS_DEBUG(ctx, vp, "ATTRLIST - copied in %d bytes of user attributes to %p", uap->bufferSize, user_buf);
 
+#if CONFIG_MACF
+	error = mac_vnode_check_setattrlist(&context, vp, &al);
+	if (error)
+		goto out;
+#endif /* MAC */
+
 	/*
 	 * Unpack the argument buffer.
 	 */
@@ -1590,7 +1814,7 @@ setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused regi
 	 */
 	if ((fndrinfo != NULL) && !(al.volattr & ATTR_VOL_INFO) &&
 	    (al.commonattr & ATTR_CMN_ACCESSMASK) && !(va.va_mode & S_IWUSR)) {
-		if ((error = setattrlist_setfinderinfo(vp, fndrinfo, &context)) != 0) {
+		if ((error = setattrlist_setfinderinfo(vp, fndrinfo, ctx)) != 0) {
 			goto out;
 		}
 		fndrinfo = NULL;  /* it was set here so skip setting below */
@@ -1616,7 +1840,7 @@ setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused regi
 			} else {
 				/* XXX should never get here */
 			}
-		} else if ((error = setattrlist_setfinderinfo(vp, fndrinfo, &context)) != 0) {
+		} else if ((error = setattrlist_setfinderinfo(vp, fndrinfo, ctx)) != 0) {
 			goto out;
 		}
 	}
@@ -1633,6 +1857,12 @@ setattrlist(struct proc *p, register struct setattrlist_args *uap, __unused regi
 		vs.f_vol_name = volname;	/* References the setattrlist buffer directly */
 		VFSATTR_WANTED(&vs, f_vol_name);
 		
+#if CONFIG_MACF
+		error = mac_mount_check_setattr(ctx, vp->v_mount, &vs);
+		if (error != 0)
+			goto out;
+#endif
+
 		if ((error = vfs_setattr(vp->v_mount, &vs, ctx)) != 0) {
 			VFS_DEBUG(ctx, vp, "ATTRLIST - ERROR: setting volume name failed");
 			goto out;
diff --git a/bsd/vfs/vfs_bio.c b/bsd/vfs/vfs_bio.c
index e3a310624..3871c2a3b 100644
--- a/bsd/vfs/vfs_bio.c
+++ b/bsd/vfs/vfs_bio.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*-
@@ -87,10 +93,20 @@
 #include <kern/zalloc.h>
 #include <kern/lock.h>
 
+#include <sys/fslog.h>		/* fslog_io_error() */
+
+#include <mach/mach_types.h>
+#include <mach/memory_object_types.h>
+#include <kern/sched_prim.h>	/* thread_block() */
+
 #include <vm/vm_kern.h>
 
 #include <sys/kdebug.h>
-#include <machine/spl.h>
+
+#include <libkern/OSAtomic.h>
+#include <sys/ubc_internal.h>
+
+#include <sys/sdt.h>
 
 #if BALANCE_QUEUES
 static __inline__ void bufqinc(int q);
@@ -100,7 +116,6 @@ static __inline__ void bufqdec(int q);
 static int	bcleanbuf(buf_t bp);
 static int	brecover_data(buf_t bp);
 static boolean_t incore(vnode_t vp, daddr64_t blkno);
-static buf_t	incore_locked(vnode_t vp, daddr64_t blkno);
 /* timeout is in msecs */
 static buf_t	getnewbuf(int slpflag, int slptimeo, int *queue);
 static void	bremfree_locked(buf_t bp);
@@ -112,8 +127,8 @@ static void	buf_itercomplete(vnode_t vp, struct buflists *, int flags);
 __private_extern__ int  bdwrite_internal(buf_t, int);
 
 /* zone allocated buffer headers */
-static void	bufzoneinit(void);
-static void	bcleanbuf_thread_init(void);
+static void	bufzoneinit(void) __attribute__((section("__TEXT, initcode")));
+static void	bcleanbuf_thread_init(void) __attribute__((section("__TEXT, initcode")));
 static void	bcleanbuf_thread(void);
 
 static zone_t	buf_hdr_zone;
@@ -128,12 +143,15 @@ static int	buf_hdr_count;
 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
 u_long	bufhash;
 
+static buf_t	incore_locked(vnode_t vp, daddr64_t blkno, struct bufhashhdr *dp);
+
 /* Definitions for the buffer stats. */
 struct bufstats bufstats;
 
 /* Number of delayed write buffers */
-int nbdwrite = 0;
+long nbdwrite = 0;
 int blaundrycnt = 0;
+static int boot_nbuf_headers = 0;
 
 
 static TAILQ_HEAD(ioqueue, buf) iobufqueue;
@@ -162,34 +180,26 @@ buf_timestamp(void)
 #define	binsheadfree(bp, dp, whichq)	do { \
 				    TAILQ_INSERT_HEAD(dp, bp, b_freelist); \
 					bufqinc((whichq));	\
-					(bp)->b_whichq = whichq; \
-				    (bp)->b_timestamp = buf_timestamp(); \
 				} while (0)
 
 #define	binstailfree(bp, dp, whichq)	do { \
 				    TAILQ_INSERT_TAIL(dp, bp, b_freelist); \
 					bufqinc((whichq));	\
-					(bp)->b_whichq = whichq; \
-				    (bp)->b_timestamp = buf_timestamp(); \
 				} while (0)
 #else
 #define	binsheadfree(bp, dp, whichq)	do { \
 				    TAILQ_INSERT_HEAD(dp, bp, b_freelist); \
-					(bp)->b_whichq = whichq; \
-				    (bp)->b_timestamp = buf_timestamp(); \
 				} while (0)
 
 #define	binstailfree(bp, dp, whichq)	do { \
 				    TAILQ_INSERT_TAIL(dp, bp, b_freelist); \
-					(bp)->b_whichq = whichq; \
-				    (bp)->b_timestamp = buf_timestamp(); \
 				} while (0)
 #endif
 
 
 #define BHASHENTCHECK(bp)	\
 	if ((bp)->b_hash.le_prev != (struct buf **)0xdeadbeef)	\
-		panic("%x: b_hash.le_prev is not deadbeef", (bp));
+		panic("%p: b_hash.le_prev is not deadbeef", (bp));
 
 #define BLISTNONE(bp)	\
 	(bp)->b_hash.le_next = (struct buf *)0;	\
@@ -215,7 +225,7 @@ buf_timestamp(void)
 int lru_is_stale = LRU_IS_STALE;
 int age_is_stale = AGE_IS_STALE;
 int meta_is_stale = META_IS_STALE;
-static int boot_nbuf = 0;
+
 
 
 /* LIST_INSERT_HEAD() with assertions */
@@ -291,8 +301,13 @@ buf_markinvalid(buf_t bp) {
 void
 buf_markdelayed(buf_t bp) {
   
-        SET(bp->b_flags, B_DELWRI);
-	buf_reassign(bp, bp->b_vp);
+	if (!ISSET(bp->b_flags, B_DELWRI)) {
+		SET(bp->b_flags, B_DELWRI);
+
+		OSAddAtomic(1, &nbdwrite);
+		buf_reassign(bp, bp->b_vp);
+	}
+        SET(bp->b_flags, B_DONE);
 }
 
 void
@@ -301,12 +316,27 @@ buf_markeintr(buf_t bp) {
         SET(bp->b_flags, B_EINTR);
 }
 
+
 void
 buf_markaged(buf_t bp) {
   
         SET(bp->b_flags, B_AGE);
 }
 
+int
+buf_fua(buf_t bp) {
+
+        if ((bp->b_flags & B_FUA) == B_FUA)
+	        return 1;
+	return 0;
+}
+
+void 
+buf_markfua(buf_t bp) {
+
+        SET(bp->b_flags, B_FUA);
+}
+
 errno_t
 buf_error(buf_t bp) {
         
@@ -343,7 +373,7 @@ buf_flags(buf_t bp) {
 void
 buf_reset(buf_t bp, int32_t io_flags) {
         
-        CLR(bp->b_flags, (B_READ | B_WRITE | B_ERROR | B_DONE | B_INVAL | B_ASYNC | B_NOCACHE));
+        CLR(bp->b_flags, (B_READ | B_WRITE | B_ERROR | B_DONE | B_INVAL | B_ASYNC | B_NOCACHE | B_FUA));
 	SET(bp->b_flags, (io_flags & (B_ASYNC | B_READ | B_WRITE | B_NOCACHE)));
 
 	bp->b_error = 0;
@@ -437,8 +467,6 @@ buf_setvnode(buf_t bp, vnode_t vp) {
 void *
 buf_callback(buf_t bp)
 {
-        if ( !(bp->b_lflags & BL_IOBUF) )
-	        return ((void *) NULL);
         if ( !(bp->b_flags & B_CALL) )
 	        return ((void *) NULL);
 
@@ -449,10 +477,6 @@ buf_callback(buf_t bp)
 errno_t
 buf_setcallback(buf_t bp, void (*callback)(buf_t, void *), void *transaction)
 {
-
-        if ( !(bp->b_lflags & BL_IOBUF) )
-	        return (EINVAL);
-	
 	if (callback)
 	        bp->b_flags |= (B_CALL | B_ASYNC);
 	else
@@ -500,7 +524,7 @@ buf_clone(buf_t bp, int io_offset, int io_size, void (*iodone)(buf_t, void *), v
 	}
 	io_bp = alloc_io_buf(bp->b_vp, 0);
 
-	io_bp->b_flags = bp->b_flags & (B_COMMIT_UPL | B_META | B_PAGEIO | B_CLUSTER | B_PHYS | B_ASYNC | B_READ);
+	io_bp->b_flags = bp->b_flags & (B_COMMIT_UPL | B_META | B_PAGEIO | B_CLUSTER | B_PHYS | B_RAW | B_ASYNC | B_READ | B_FUA);
 
 	if (iodone) {
 	        io_bp->b_transaction = arg;
@@ -531,7 +555,10 @@ buf_setfilter(buf_t bp, void (*filter)(buf_t, void *), void *transaction,
 
 	bp->b_transaction = transaction;
 	bp->b_iodone = filter;
-	bp->b_flags |= B_FILTER;
+	if (filter)
+	        bp->b_flags |= B_FILTER;
+	else
+	        bp->b_flags &= ~B_FILTER;
 }
 
 
@@ -658,7 +685,7 @@ buf_map(buf_t bp, caddr_t *io_addr)
 	kret = ubc_upl_map(bp->b_upl, &vaddr);    /* Map it in */
 
 	if (kret != KERN_SUCCESS) {
-	        *io_addr = 0;
+	        *io_addr = NULL;
 
 	        return(ENOMEM);
 	}
@@ -685,8 +712,13 @@ buf_unmap(buf_t bp)
 	if (real_bp && real_bp->b_datap)
 	        return (0);
 
-	if (bp->b_lflags & BL_IOBUF) {
+	if ((bp->b_lflags & BL_IOBUF) &&
+	    ((bp->b_flags & (B_PAGEIO | B_READ)) != (B_PAGEIO | B_READ))) {
 	        /*
+		 * ignore pageins... the 'right' thing will
+		 * happen due to the way we handle speculative
+		 * clusters...
+		 *
 		 * when we commit these pages, we'll hit
 		 * it with UPL_COMMIT_INACTIVE which
 		 * will clear the reference bit that got
@@ -772,9 +804,14 @@ buf_strategy_fragmented(vnode_t devvp, buf_t bp, off_t f_offset, size_t contig_b
 			io_bp->b_blkno   = io_blkno;
 
 			buf_reset(io_bp, io_direction);
+
 			/*
-			 * Call the device to do the I/O and wait for it
+			 * Call the device to do the I/O and wait for it.  Make sure the appropriate party is charged for write
 			 */
+
+			if (!ISSET(bp->b_flags, B_READ))
+			        OSAddAtomic(1, &devvp->v_numoutput);
+
 			if ((error = VNOP_STRATEGY(io_bp)))
 			        break;
 			if ((error = (int)buf_biowait(io_bp)))
@@ -830,6 +867,7 @@ buf_strategy(vnode_t devvp, void *ap)
 	 * end up issuing the I/O...
 	 */
 	bp->b_dev = devvp->v_rdev;
+	DTRACE_IO1(start, buf_t, bp);
 
 	if (bp->b_flags & B_READ)
 	        bmap_flags = VNODE_READ;
@@ -899,58 +937,98 @@ buf_free(buf_t bp) {
 }
 
 
+/*
+ * iterate buffers for the specified vp. 
+ *   if BUF_SCAN_DIRTY is set, do the dirty list
+ *   if BUF_SCAN_CLEAN is set, do the clean list
+ *   if neither flag is set, default to BUF_SCAN_DIRTY
+ *   if BUF_NOTIFY_BUSY is set, call the callout function using a NULL bp for busy pages
+ */
+
+struct buf_iterate_info_t {
+	int flag;
+	struct buflists *listhead;
+};
 
 void
-buf_iterate(vnode_t vp, int (*callout)(buf_t, void *), int flags, void *arg) {
+buf_iterate(vnode_t vp, int (*callout)(buf_t, void *), int flags, void *arg)
+{
 	buf_t 	bp;
 	int	retval;
 	struct	buflists local_iterblkhd;
 	int	lock_flags = BAC_NOWAIT | BAC_REMOVE;
+	int	notify_busy = flags & BUF_NOTIFY_BUSY;
+	struct buf_iterate_info_t list[2];
+	int	num_lists, i;
 
 	if (flags & BUF_SKIP_LOCKED)
 	        lock_flags |= BAC_SKIP_LOCKED;
 	if (flags & BUF_SKIP_NONLOCKED)
 	        lock_flags |= BAC_SKIP_NONLOCKED;
 
-	lck_mtx_lock(buf_mtxp);
-	
-	if (buf_iterprepare(vp, &local_iterblkhd, VBI_DIRTY))  {
-		lck_mtx_unlock(buf_mtxp);
-		return;
+	if ( !(flags & (BUF_SCAN_DIRTY | BUF_SCAN_CLEAN)))
+	        flags |= BUF_SCAN_DIRTY;
+
+	num_lists = 0;
+
+	if (flags & BUF_SCAN_DIRTY) {
+	        list[num_lists].flag = VBI_DIRTY;
+		list[num_lists].listhead = &vp->v_dirtyblkhd;
+		num_lists++;
+	}
+	if (flags & BUF_SCAN_CLEAN) {
+		list[num_lists].flag = VBI_CLEAN;
+		list[num_lists].listhead = &vp->v_cleanblkhd;
+		num_lists++;
 	}
-	while (!LIST_EMPTY(&local_iterblkhd)) {
-		bp = LIST_FIRST(&local_iterblkhd);
-		LIST_REMOVE(bp, b_vnbufs);
-		LIST_INSERT_HEAD(&vp->v_dirtyblkhd, bp, b_vnbufs);
 
-		if (buf_acquire_locked(bp, lock_flags, 0, 0))
-		        continue;
+	for (i = 0; i < num_lists; i++) {
+		lck_mtx_lock(buf_mtxp);
+	
+		if (buf_iterprepare(vp, &local_iterblkhd, list[i].flag))  {
+			lck_mtx_unlock(buf_mtxp);
+			continue;
+		}
+		while (!LIST_EMPTY(&local_iterblkhd)) {
+			bp = LIST_FIRST(&local_iterblkhd);
+			LIST_REMOVE(bp, b_vnbufs);
+			LIST_INSERT_HEAD(list[i].listhead, bp, b_vnbufs);
 
-		lck_mtx_unlock(buf_mtxp);
+			if (buf_acquire_locked(bp, lock_flags, 0, 0)) {
+				if (notify_busy) {
+					bp = NULL;
+				} else {
+					continue;
+				}
+			}
 
-		retval = callout(bp, arg);
+			lck_mtx_unlock(buf_mtxp);
 
-		switch (retval) {
-		case BUF_RETURNED:
-			buf_brelse(bp);
-			break;
-		case BUF_CLAIMED:
-			break;
-		case BUF_RETURNED_DONE:
-		        buf_brelse(bp);
-			lck_mtx_lock(buf_mtxp);
-			goto out;
-		case BUF_CLAIMED_DONE:
-		        lck_mtx_lock(buf_mtxp);
-			goto out;
-		}
-		lck_mtx_lock(buf_mtxp);
-	}
-out:
-	buf_itercomplete(vp, &local_iterblkhd, VBI_DIRTY);
+			retval = callout(bp, arg);
 
-	lck_mtx_unlock(buf_mtxp);
-}
+			switch (retval) {
+			case BUF_RETURNED:
+				if (bp)
+					buf_brelse(bp);
+				break;
+			case BUF_CLAIMED:
+				break;
+			case BUF_RETURNED_DONE:
+				if (bp)
+					buf_brelse(bp);
+				lck_mtx_lock(buf_mtxp);
+				goto out;
+			case BUF_CLAIMED_DONE:
+				lck_mtx_lock(buf_mtxp);
+				goto out;
+			}
+			lck_mtx_lock(buf_mtxp);
+		} /* while list has more nodes */
+	  out:
+		buf_itercomplete(vp, &local_iterblkhd, list[i].flag);
+		lck_mtx_unlock(buf_mtxp);
+	} /* for each list */
+} /* buf_iterate */
 
 
 /*
@@ -1111,7 +1189,7 @@ try_dirty_list:
 }
 
 void
-buf_flushdirtyblks(vnode_t vp, int wait, int flags, char *msg) {
+buf_flushdirtyblks(vnode_t vp, int wait, int flags, const char *msg) {
 	buf_t	bp;
 	int	writes_issued = 0;
 	errno_t	error;
@@ -1200,7 +1278,7 @@ buf_iterprepare(vnode_t vp, struct buflists *iterheadp, int flags)
 		
 	while (vp->v_iterblkflags & VBI_ITER) 	{
 	        vp->v_iterblkflags |= VBI_ITERWANT;
-		msleep(&vp->v_iterblkflags, buf_mtxp, 0, "buf_iterprepare", 0);	
+		msleep(&vp->v_iterblkflags, buf_mtxp, 0, "buf_iterprepare", NULL);	
 	}
 	if (LIST_EMPTY(listheadp)) {
 	        LIST_INIT(iterheadp);
@@ -1248,8 +1326,7 @@ static void
 bremfree_locked(buf_t bp)
 {
 	struct bqueues *dp = NULL;
-	int whichq = -1;
-
+	int whichq;
 	/*
 	 * We only calculate the head of the freelist when removing
 	 * the last element of the list as that is the only time that
@@ -1257,31 +1334,36 @@ bremfree_locked(buf_t bp)
 	 *
 	 * NB: This makes an assumption about how tailq's are implemented.
 	 */
+	whichq = bp->b_whichq;
+
 	if (bp->b_freelist.tqe_next == NULL) {
-		for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
-			if (dp->tqh_last == &bp->b_freelist.tqe_next)
-				break;
-		if (dp == &bufqueues[BQUEUES])
+	        dp = &bufqueues[whichq];
+
+		if (dp->tqh_last != &bp->b_freelist.tqe_next)
 			panic("bremfree: lost tail");
 	}
 	TAILQ_REMOVE(dp, bp, b_freelist);
-	whichq = bp->b_whichq;
+
 #if BALANCE_QUEUES
 	bufqdec(whichq);
 #endif
+	if (whichq == BQ_LAUNDRY)
+	        blaundrycnt--;
+
 	bp->b_whichq = -1;
 	bp->b_timestamp = 0; 
 }
 
 /*
  * Associate a buffer with a vnode.
+ * buf_mtxp must be locked on entry
  */
 static void
-bgetvp(vnode_t vp, buf_t bp)
+bgetvp_locked(vnode_t vp, buf_t bp)
 {
 
 	if (bp->b_vp != vp)
-		panic("bgetvp: not free");
+		panic("bgetvp_locked: not free");
 
 	if (vp->v_type == VBLK || vp->v_type == VCHR)
 		bp->b_dev = vp->v_rdev;
@@ -1290,28 +1372,21 @@ bgetvp(vnode_t vp, buf_t bp)
 	/*
 	 * Insert onto list for new vnode.
 	 */
-	lck_mtx_lock(buf_mtxp);
 	bufinsvn(bp, &vp->v_cleanblkhd);
-	lck_mtx_unlock(buf_mtxp);
 }
 
 /*
  * Disassociate a buffer from a vnode.
+ * buf_mtxp must be locked on entry
  */
 static void
-brelvp(buf_t bp)
+brelvp_locked(buf_t bp)
 {
-	vnode_t vp;
-
-	if ((vp = bp->b_vp) == (vnode_t)NULL)
-		panic("brelvp: NULL vp");
 	/*
 	 * Delete from old vnode list, if on one.
 	 */
-	lck_mtx_lock(buf_mtxp);
 	if (bp->b_vnbufs.le_next != NOLIST)
 		bufremvn(bp);
-	lck_mtx_unlock(buf_mtxp);
 
 	bp->b_vp = (vnode_t)NULL;
 }
@@ -1330,7 +1405,7 @@ buf_reassign(buf_t bp, vnode_t newvp)
 		printf("buf_reassign: NULL");
 		return;
 	}
-	lck_mtx_lock(buf_mtxp);
+	lck_mtx_lock_spin(buf_mtxp);
 
 	/*
 	 * Delete from old vnode list, if on one.
@@ -1367,55 +1442,44 @@ bufhdrinit(buf_t bp)
  * Initialize buffers and hash links for buffers.
  */
 __private_extern__ void
-bufinit()
+bufinit(void)
 {
 	buf_t	bp;
 	struct bqueues *dp;
 	int	i;
-	int	metabuf;
-	long	whichq;
 
-	nbuf = 0;
+	nbuf_headers = 0;
 	/* Initialize the buffer queues ('freelists') and the hash table */
 	for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
 		TAILQ_INIT(dp);
 	bufhashtbl = hashinit(nbuf_hashelements, M_CACHE, &bufhash);
 
-	metabuf = max_nbuf_headers/8; /* reserved for meta buf */
-
 	/* Initialize the buffer headers */
 	for (i = 0; i < max_nbuf_headers; i++) {
-		nbuf++;
-		bp = &buf[i];
+		nbuf_headers++;
+		bp = &buf_headers[i];
 		bufhdrinit(bp);
 
-		/*
-		 * metabuf buffer headers on the meta-data list and
-		 * rest of the buffer headers on the empty list
-		 */
-		if (--metabuf) 
-			whichq = BQ_META;
-		else 
-			whichq = BQ_EMPTY;
-
 		BLISTNONE(bp);
-		dp = &bufqueues[whichq];
-		binsheadfree(bp, dp, whichq);
+		dp = &bufqueues[BQ_EMPTY];
+		bp->b_whichq = BQ_EMPTY;
+		bp->b_timestamp = buf_timestamp();
+		binsheadfree(bp, dp, BQ_EMPTY);
 		binshash(bp, &invalhash);
 	}
 
-	boot_nbuf = nbuf;
-
-	for (; i < nbuf + niobuf; i++) {
-		bp = &buf[i];
+	boot_nbuf_headers = nbuf_headers;
+	for (; i < nbuf_headers + niobuf_headers; i++) {
+		bp = &buf_headers[i];
 		bufhdrinit(bp);
+		bp->b_whichq = -1;
 		binsheadfree(bp, &iobufqueue, -1);
 	}
 
-    /*
+	/*
 	 * allocate lock group attribute and group
 	 */
-    buf_mtx_grp_attr = lck_grp_attr_alloc_init();
+	buf_mtx_grp_attr = lck_grp_attr_alloc_init();
 	buf_mtx_grp = lck_grp_alloc_init("buffer cache", buf_mtx_grp_attr);
 		
 	/*
@@ -1441,7 +1505,7 @@ bufinit()
 	cluster_init();
 
 	printf("using %d buffer headers and %d cluster IO buffer headers\n",
-		nbuf, niobuf);
+		nbuf_headers, niobuf_headers);
 
 	/* Set up zones used by the buffer cache */
 	bufzoneinit();
@@ -1451,13 +1515,74 @@ bufinit()
 
 #if BALANCE_QUEUES
 	{
-	static void bufq_balance_thread_init();
+	static void bufq_balance_thread_init(void) __attribute__((section("__TEXT, initcode")));
 	/* create a thread to do dynamic buffer queue balancing */
 	bufq_balance_thread_init();
 	}
 #endif /* notyet */
 }
 
+
+
+/*
+ * Zones for the meta data buffers
+ */
+
+#define MINMETA 512
+#define MAXMETA 8192
+
+struct meta_zone_entry {
+	zone_t mz_zone;
+	vm_size_t mz_size;
+	vm_size_t mz_max;
+	const char *mz_name;
+};
+
+struct meta_zone_entry meta_zones[] = {
+	{NULL, (MINMETA * 1), 128 * (MINMETA * 1), "buf.512" },
+	{NULL, (MINMETA * 2),  64 * (MINMETA * 2), "buf.1024" },
+	{NULL, (MINMETA * 4),  16 * (MINMETA * 4), "buf.2048" },
+	{NULL, (MINMETA * 8), 512 * (MINMETA * 8), "buf.4096" },
+	{NULL, (MINMETA * 16), 512 * (MINMETA * 16), "buf.8192" },
+	{NULL, 0, 0, "" } /* End */
+};
+
+/*
+ * Initialize the meta data zones
+ */
+static void
+bufzoneinit(void)
+{
+	int i;
+
+	for (i = 0; meta_zones[i].mz_size != 0; i++) {
+		meta_zones[i].mz_zone = 
+				zinit(meta_zones[i].mz_size,
+					meta_zones[i].mz_max,
+					PAGE_SIZE,
+					meta_zones[i].mz_name);
+	}
+	buf_hdr_zone = zinit(sizeof(struct buf), 32, PAGE_SIZE, "buf headers");
+}
+
+static __inline__ zone_t
+getbufzone(size_t size)
+{
+	int i;
+
+	if ((size % 512) || (size < MINMETA) || (size > MAXMETA))
+		panic("getbufzone: incorect size = %lu", size);
+
+	for (i = 0; meta_zones[i].mz_size != 0; i++) {
+		if (meta_zones[i].mz_size >= size)
+			break;
+	}
+
+	return (meta_zones[i].mz_zone);
+}
+
+
+
 static struct buf *
 bio_doread(vnode_t vp, daddr64_t blkno, int size, ucred_t cred, int async, int queuetype)
 {
@@ -1488,7 +1613,7 @@ bio_doread(vnode_t vp, daddr64_t blkno, int size, ucred_t cred, int async, int q
 
 		/* Pay for the read. */
 		if (p && p->p_stats) 
-			p->p_stats->p_ru.ru_inblock++;		/* XXX */
+			OSIncrementAtomic(&p->p_stats->p_ru.ru_inblock);		/* XXX */
 
 		if (async) {
 		        /*
@@ -1624,7 +1749,7 @@ buf_bwrite(buf_t bp)
 			buf_reassign(bp, vp);
 		else
 		if (p && p->p_stats) 
-			p->p_stats->p_ru.ru_oublock++;		/* XXX */
+			OSIncrementAtomic(&p->p_stats->p_ru.ru_oublock);	/* XXX */
 	}
 	trace(TR_BUFWRITE, pack(vp, bp->b_bcount), bp->b_lblkno);
 
@@ -1649,7 +1774,7 @@ buf_bwrite(buf_t bp)
 			buf_reassign(bp, vp);
 		else
 		if (p && p->p_stats) 
-			p->p_stats->p_ru.ru_oublock++;		/* XXX */
+			OSIncrementAtomic(&p->p_stats->p_ru.ru_oublock);	/* XXX */
 
 		/* Release the buffer. */
 		// XXXdbg - only if the unused bit is set
@@ -1666,8 +1791,7 @@ buf_bwrite(buf_t bp)
 }
 
 int
-vn_bwrite(ap)
-	struct vnop_bwrite_args *ap;
+vn_bwrite(struct vnop_bwrite_args *ap)
 {
 	return (buf_bwrite(ap->a_bp));
 }
@@ -1706,7 +1830,7 @@ bdwrite_internal(buf_t bp, int return_error)
 	if (!ISSET(bp->b_flags, B_DELWRI)) {
 		SET(bp->b_flags, B_DELWRI);
 		if (p && p->p_stats) 
-			p->p_stats->p_ru.ru_oublock++;		/* XXX */
+			OSIncrementAtomic(&p->p_stats->p_ru.ru_oublock);	/* XXX */
 		OSAddAtomic(1, &nbdwrite);
 		buf_reassign(bp, vp);
 	}
@@ -1731,14 +1855,14 @@ bdwrite_internal(buf_t bp, int return_error)
 	 * buffer is part of a transaction and can't go to disk until
 	 * the LOCKED bit is cleared.
 	 */
-	if (!ISSET(bp->b_flags, B_LOCKED) && nbdwrite > ((nbuf/4)*3)) {
+	if (!ISSET(bp->b_flags, B_LOCKED) && nbdwrite > ((nbuf_headers/4)*3)) {
 		if (return_error)
 			return (EAGAIN);
 		/*
 		 * If the vnode has "too many" write operations in progress
 		 * wait for them to finish the IO
 		 */
-		(void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, (char *)"buf_bdwrite");
+		(void)vnode_waitforwrites(vp, VNODE_ASYNC_THROTTLE, 0, 0, "buf_bdwrite");
 
 		return (buf_bawrite(bp));
 	}
@@ -1812,15 +1936,15 @@ buf_brelse(buf_t bp)
 
 
 	if (bp->b_whichq != -1 || !(bp->b_lflags & BL_BUSY))
-	        panic("buf_brelse: bad buffer = %x\n", bp);
+	        panic("buf_brelse: bad buffer = %p\n", bp);
 
 #ifdef JOE_DEBUG
-	bp->b_stackbrelse[0] = __builtin_return_address(0);
-	bp->b_stackbrelse[1] = __builtin_return_address(1);
-	bp->b_stackbrelse[2] = __builtin_return_address(2);
-	bp->b_stackbrelse[3] = __builtin_return_address(3);
-	bp->b_stackbrelse[4] = __builtin_return_address(4);
-	bp->b_stackbrelse[5] = __builtin_return_address(5);
+	bp->b_stackbrelse[0] = (int)__builtin_return_address(0);
+	bp->b_stackbrelse[1] = (int)__builtin_return_address(1);
+	bp->b_stackbrelse[2] = (int)__builtin_return_address(2);
+	bp->b_stackbrelse[3] = (int)__builtin_return_address(3);
+	bp->b_stackbrelse[4] = (int)__builtin_return_address(4);
+	bp->b_stackbrelse[5] = (int)__builtin_return_address(5);
 
 	bp->b_lastbrelse = current_thread();
 	bp->b_tag = 0;
@@ -1853,7 +1977,7 @@ buf_brelse(buf_t bp)
 			bp->b_transaction = NULL;
 
 			if (iodone_func == NULL) {
-				panic("brelse: bp @ 0x%x has NULL b_iodone!\n", bp);
+				panic("brelse: bp @ %p has NULL b_iodone!\n", bp);
 			}
 			(*iodone_func)(bp, arg);
 		}
@@ -1912,7 +2036,7 @@ buf_brelse(buf_t bp)
 		}
 	} else {
 		if ( (upl) )
-			panic("brelse: UPL set for non VREG; vp=%x", bp->b_vp);
+			panic("brelse: UPL set for non VREG; vp=%p", bp->b_vp);
 	}	
 
 	/*
@@ -1928,30 +2052,51 @@ buf_brelse(buf_t bp)
 	
 	if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) {
 		/*
-		 * If it's invalid or empty, dissociate it from its vnode
-		 * and put on the head of the appropriate queue.
+		 * If it's invalid or empty, dissociate it from its vnode,
+		 * release its storage if B_META, and
+		 * clean it up a bit and put it on the EMPTY queue
 		 */
-		if (bp->b_vp)
-			brelvp(bp);
-
 		if (ISSET(bp->b_flags, B_DELWRI))
 			OSAddAtomic(-1, &nbdwrite);
 
-		CLR(bp->b_flags, (B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE));
+		if (ISSET(bp->b_flags, B_META)) {
+		        if (bp->b_bufsize) {
+			        if (ISSET(bp->b_flags, B_ZALLOC)) {
+				        zone_t z;
+
+					z = getbufzone(bp->b_bufsize);
+					zfree(z, (void *)bp->b_datap);
+				} else
+				        kmem_free(kernel_map, bp->b_datap, bp->b_bufsize); 
+
+				 bp->b_datap = (uintptr_t)NULL;
+				 bp->b_bufsize = 0;
+			}
+		}
 		/*
-		 * Determine which queue the buffer should be on, then put it there.
+		 * nuke any credentials we were holding
 		 */
-		if (bp->b_bufsize <= 0)
-			whichq = BQ_EMPTY;	/* no data */
-		else if (ISSET(bp->b_flags, B_META))
-			whichq = BQ_META;		/* meta-data */
-		else
-			whichq = BQ_AGE;	/* invalid data */
-		bufq = &bufqueues[whichq];
+		if (IS_VALID_CRED(bp->b_rcred)) {
+		        kauth_cred_unref(&bp->b_rcred);
+		}
+		if (IS_VALID_CRED(bp->b_wcred)) {
+		        kauth_cred_unref(&bp->b_wcred);
+		}
+		CLR(bp->b_flags, (B_META | B_ZALLOC | B_DELWRI | B_LOCKED | B_AGE | B_ASYNC | B_NOCACHE | B_FUA));
 
-		lck_mtx_lock(buf_mtxp);
+		bufq = &bufqueues[BQ_EMPTY];
+		bp->b_whichq = BQ_EMPTY;
 
-		binsheadfree(bp, bufq, whichq);
+		lck_mtx_lock_spin(buf_mtxp);
+
+		if (bp->b_vp)
+			brelvp_locked(bp);
+
+		bremhash(bp);
+		BLISTNONE(bp);
+		binshash(bp, &invalhash);
+
+		binsheadfree(bp, bufq, BQ_EMPTY);
 	} else {
 		/*
 		 * It has valid data.  Put it on the end of the appropriate
@@ -1968,8 +2113,10 @@ buf_brelse(buf_t bp)
 		bufq = &bufqueues[whichq];
 
 		CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE));
+		bp->b_whichq = whichq;
+		bp->b_timestamp = buf_timestamp();
 
-	        lck_mtx_lock(buf_mtxp);
+	        lck_mtx_lock_spin(buf_mtxp);
 
 		binstailfree(bp, bufq, whichq);
 	}
@@ -2024,10 +2171,13 @@ static boolean_t
 incore(vnode_t vp, daddr64_t blkno)
 {
         boolean_t retval;
+	struct	bufhashhdr *dp;
 
-	lck_mtx_lock(buf_mtxp);
+	dp = BUFHASH(vp, blkno);
 
-	if (incore_locked(vp, blkno))
+	lck_mtx_lock_spin(buf_mtxp);
+
+	if (incore_locked(vp, blkno, dp))
 	        retval = TRUE;
 	else
 	        retval = FALSE;
@@ -2038,20 +2188,18 @@ incore(vnode_t vp, daddr64_t blkno)
 
 
 static buf_t
-incore_locked(vnode_t vp, daddr64_t blkno)
+incore_locked(vnode_t vp, daddr64_t blkno, struct bufhashhdr *dp)
 {
 	struct buf *bp;
 
-	bp = BUFHASH(vp, blkno)->lh_first;
-
 	/* Search hash chain */
-	for (; bp != NULL; bp = bp->b_hash.le_next) {
+	for (bp = dp->lh_first; bp != NULL; bp = bp->b_hash.le_next) {
 		if (bp->b_lblkno == blkno && bp->b_vp == vp &&
 		    !ISSET(bp->b_flags, B_INVAL)) {
 			return (bp);
 		}
 	}
-	return (0);
+	return (NULL);
 }
 
 
@@ -2075,16 +2223,18 @@ buf_getblk(vnode_t vp, daddr64_t blkno, int size, int slpflag, int slptimeo, int
 	int ret_only_valid;
 	struct timespec ts;
 	int upl_flags;
+	struct	bufhashhdr *dp;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 386)) | DBG_FUNC_START,
 		     (int)(blkno * PAGE_SIZE), size, operation, 0, 0);
 
 	ret_only_valid = operation & BLK_ONLYVALID;
 	operation &= ~BLK_ONLYVALID;
+	dp = BUFHASH(vp, blkno);
 start:
-	lck_mtx_lock(buf_mtxp);
+	lck_mtx_lock_spin(buf_mtxp);
 start_locked:
-	if ((bp = incore_locked(vp, blkno))) {
+	if ((bp = incore_locked(vp, blkno, dp))) {
 		/*
 		 * Found in the Buffer Cache
 		 */
@@ -2099,6 +2249,7 @@ start_locked:
 				SET(bp->b_lflags, BL_WANTED);
 				bufstats.bufs_busyincore++;
 
+				lck_mtx_convert_spin(buf_mtxp);
 				/*
 				 * don't retake the mutex after being awakened...
 				 * the time out is in msecs 
@@ -2130,20 +2281,21 @@ start_locked:
 			/*
 			 * buffer in core and not busy
 			 */
-			if ( (bp->b_upl) )
-			        panic("buffer has UPL, but not marked BUSY: %x", bp);
 			SET(bp->b_lflags, BL_BUSY);
 			SET(bp->b_flags, B_CACHE);
-#ifdef JOE_DEBUG
-			bp->b_owner = current_thread();
-			bp->b_tag   = 1;
-#endif
+
 			bremfree_locked(bp);
 			bufstats.bufs_incore++;
 			
 			lck_mtx_unlock(buf_mtxp);
+#ifdef JOE_DEBUG
+			bp->b_owner = current_thread();
+			bp->b_tag   = 1;
+#endif
+			if ( (bp->b_upl) )
+			        panic("buffer has UPL, but not marked BUSY: %p", bp);
 
-			if ( !ret_only_valid)
+			if ( !ret_only_valid && bp->b_bufsize != size)
 			        allocbuf(bp, size);
 
 			upl_flags = 0;
@@ -2204,8 +2356,9 @@ start_locked:
 			lck_mtx_unlock(buf_mtxp);
 			return (NULL);
 		}
+		lck_mtx_convert_spin(buf_mtxp);
 
-		if ((UBCINVALID(vp)) || !(UBCINFOEXISTS(vp)))
+		if ((vnode_isreg(vp) == 0) || (UBCINFOEXISTS(vp) == 0) /*|| (vnode_issystem(vp) == 1)*/)
 			operation = BLK_META;
 
 		if ((bp = getnewbuf(slpflag, slptimeo, &queue)) == NULL)
@@ -2218,7 +2371,7 @@ start_locked:
 		 * the hash... if we see it incore at this point we dump
 		 * the buffer we were working on and start over
 		 */
-		if (incore_locked(vp, blkno)) {
+		if (incore_locked(vp, blkno, dp)) {
 			SET(bp->b_flags, B_INVAL);
 			binshash(bp, &invalhash);
 
@@ -2247,9 +2400,9 @@ start_locked:
 		 */
 		binshash(bp, BUFHASH(vp, blkno)); 
 
-		lck_mtx_unlock(buf_mtxp);
+		bgetvp_locked(vp, bp);
 
-		bgetvp(vp, bp);
+		lck_mtx_unlock(buf_mtxp);
 
 		allocbuf(bp, size);
 
@@ -2281,7 +2434,7 @@ start_locked:
 			int	bmap_flags;
 
 			if ( (bp->b_upl) )
-				panic("bp already has UPL: %x",bp);
+				panic("bp already has UPL: %p",bp);
 
 			f_offset = ubc_blktooff(vp, blkno);
 
@@ -2358,12 +2511,12 @@ start_locked:
 		     (int)bp, (int)bp->b_datap, bp->b_flags, 3, 0);
 
 #ifdef JOE_DEBUG
-	bp->b_stackgetblk[0] = __builtin_return_address(0);
-	bp->b_stackgetblk[1] = __builtin_return_address(1);
-	bp->b_stackgetblk[2] = __builtin_return_address(2);
-	bp->b_stackgetblk[3] = __builtin_return_address(3);
-	bp->b_stackgetblk[4] = __builtin_return_address(4);
-	bp->b_stackgetblk[5] = __builtin_return_address(5);
+	bp->b_stackgetblk[0] = (int)__builtin_return_address(0);
+	bp->b_stackgetblk[1] = (int)__builtin_return_address(1);
+	bp->b_stackgetblk[2] = (int)__builtin_return_address(2);
+	bp->b_stackgetblk[3] = (int)__builtin_return_address(3);
+	bp->b_stackgetblk[4] = (int)__builtin_return_address(4);
+	bp->b_stackgetblk[5] = (int)__builtin_return_address(5);
 #endif
 	return (bp);
 }
@@ -2372,8 +2525,7 @@ start_locked:
  * Get an empty, disassociated buffer of given size.
  */
 buf_t
-buf_geteblk(size)
-	int size;
+buf_geteblk(int size)
 {
 	buf_t	bp;
 	int queue = BQ_EMPTY;
@@ -2399,61 +2551,6 @@ buf_geteblk(size)
 	return (bp);
 }
 
-/*
- * Zones for the meta data buffers
- */
-
-#define MINMETA 512
-#define MAXMETA 4096
-
-struct meta_zone_entry {
-	zone_t mz_zone;
-	vm_size_t mz_size;
-	vm_size_t mz_max;
-	char *mz_name;
-};
-
-struct meta_zone_entry meta_zones[] = {
-	{NULL, (MINMETA * 1), 128 * (MINMETA * 1), "buf.512" },
-	{NULL, (MINMETA * 2),  64 * (MINMETA * 2), "buf.1024" },
-	{NULL, (MINMETA * 4),  16 * (MINMETA * 4), "buf.2048" },
-	{NULL, (MINMETA * 8), 512 * (MINMETA * 8), "buf.4096" },
-	{NULL, 0, 0, "" } /* End */
-};
-
-/*
- * Initialize the meta data zones
- */
-static void
-bufzoneinit(void)
-{
-	int i;
-
-	for (i = 0; meta_zones[i].mz_size != 0; i++) {
-		meta_zones[i].mz_zone = 
-				zinit(meta_zones[i].mz_size,
-					meta_zones[i].mz_max,
-					PAGE_SIZE,
-					meta_zones[i].mz_name);
-	}
-	buf_hdr_zone = zinit(sizeof(struct buf), 32, PAGE_SIZE, "buf headers");
-}
-
-static __inline__ zone_t
-getbufzone(size_t size)
-{
-	int i;
-
-	if ((size % 512) || (size < MINMETA) || (size > MAXMETA))
-		panic("getbufzone: incorect size = %d", size);
-
-	for (i = 0; meta_zones[i].mz_size != 0; i++) {
-		if (meta_zones[i].mz_size >= size)
-			break;
-	}
-
-	return (meta_zones[i].mz_zone);
-}
 
 /*
  * With UBC, there is no need to expand / shrink the file data 
@@ -2494,7 +2591,8 @@ allocbuf(buf_t bp, int size)
 					if (nsize <= MAXMETA) {
 					        desired_size = nsize;
 						z = getbufzone(nsize);
-						bp->b_datap = (uintptr_t)zalloc(z);
+						/* b_datap not really a ptr */
+						*(void **)(&bp->b_datap) = zalloc(z);
 					} else {
 					        bp->b_datap = (uintptr_t)NULL;
 					        kmem_alloc_wired(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
@@ -2522,11 +2620,15 @@ allocbuf(buf_t bp, int size)
 			if (nsize <= MAXMETA) {
 				desired_size = nsize;
 				z = getbufzone(nsize);
-				bp->b_datap = (uintptr_t)zalloc(z);
+				/* b_datap not really a ptr */
+				*(void **)(&bp->b_datap) = zalloc(z);
 				SET(bp->b_flags, B_ZALLOC);
 			} else
 				kmem_alloc_wired(kernel_map, (vm_offset_t *)&bp->b_datap, desired_size);
 		}
+
+		if (bp->b_datap == 0)
+		        panic("allocbuf: NULL b_datap");
 	}
 	bp->b_bufsize = desired_size;
 	bp->b_bcount = size;
@@ -2570,29 +2672,30 @@ start:
 	/*
 	 * invalid request gets empty queue
 	 */
-	if ((*queue > BQUEUES) || (*queue < 0)
+	if ((*queue >= BQUEUES) || (*queue < 0)
 		|| (*queue == BQ_LAUNDRY) || (*queue == BQ_LOCKED))
 		*queue = BQ_EMPTY;
-	/* need to grow number of bufs, add another one rather than recycling */
-	if (nbuf < max_nbuf_headers) {
+
+
+	if (*queue == BQ_EMPTY && (bp = bufqueues[*queue].tqh_first))
+	        goto found;
+
+	/*
+	 * need to grow number of bufs, add another one rather than recycling
+	 */
+	if (nbuf_headers < max_nbuf_headers) {
 		/*
 		 * Increment  count now as lock 
 		 * is dropped for allocation.
 		 * That avoids over commits
 		 */
-		nbuf++;
+		nbuf_headers++;
 		goto add_newbufs;
 	}
-
-	/*
-	 * (*queue == BQUEUES) means no preference
-	 */
-	if (*queue != BQUEUES) {
-		/* Try for the requested queue first */
-		bp = bufqueues[*queue].tqh_first;
-		if (bp)
-			goto found;
-	}
+	/* Try for the requested queue first */
+	bp = bufqueues[*queue].tqh_first;
+	if (bp)
+	        goto found;
 
 	/* Unable to use requested queue */
 	age_bp = bufqueues[BQ_AGE].tqh_first;
@@ -2620,21 +2723,25 @@ add_newbufs:
 
 		/* Create a new temporary buffer header */
 		bp = (struct buf *)zalloc(buf_hdr_zone);
-	
-		lck_mtx_lock(buf_mtxp);
-
+		
 		if (bp) {
 			bufhdrinit(bp);
+			bp->b_whichq = BQ_EMPTY;
+			bp->b_timestamp = buf_timestamp();
 			BLISTNONE(bp);
-			binshash(bp, &invalhash);
 			SET(bp->b_flags, B_HDRALLOC);
 			*queue = BQ_EMPTY;
+		}
+		lck_mtx_lock(buf_mtxp);
+
+		if (bp) {
+			binshash(bp, &invalhash);
 			binsheadfree(bp, &bufqueues[BQ_EMPTY], BQ_EMPTY);
 			buf_hdr_count++;
 			goto found;
 		}
 		/* subtract already accounted bufcount */
-		nbuf--;
+		nbuf_headers--;
 
 		bufstats.bufs_sleeps++;
 
@@ -2644,8 +2751,8 @@ add_newbufs:
 		ts.tv_sec = (slptimeo/1000);
 		/* the hz value is 100; which leads to 10ms */
 		ts.tv_nsec = (slptimeo % 1000) * NSEC_PER_USEC * 1000 * 10;
-		msleep(&needbuffer, buf_mtxp, slpflag|(PRIBIO+1), (char *)"getnewbuf", &ts);
-		return (0);
+		msleep(&needbuffer, buf_mtxp, slpflag|(PRIBIO+1), "getnewbuf", &ts);
+		return (NULL);
 	}
 
 	/* Buffer available either on AGE or LRU or META */
@@ -2706,7 +2813,7 @@ add_newbufs:
 	}
 found:
 	if (ISSET(bp->b_flags, B_LOCKED) || ISSET(bp->b_lflags, BL_BUSY))
-	        panic("getnewbuf: bp @ 0x%x is LOCKED or BUSY! (flags 0x%x)\n", bp, bp->b_flags);
+	        panic("getnewbuf: bp @ %p is LOCKED or BUSY! (flags 0x%lx)\n", bp, bp->b_flags);
 
 	/* Clean it */
 	if (bcleanbuf(bp)) {
@@ -2735,8 +2842,6 @@ bcleanbuf(buf_t bp)
 	/* Remove from the queue */
 	bremfree_locked(bp);
 
-	/* Buffer is no longer on free lists. */
-	SET(bp->b_lflags, BL_BUSY);
 #ifdef JOE_DEBUG
 	bp->b_owner = current_thread();
 	bp->b_tag   = 2;
@@ -2746,28 +2851,43 @@ bcleanbuf(buf_t bp)
 	 * it on the LAUNDRY queue, and return 1
 	 */
 	if (ISSET(bp->b_flags, B_DELWRI)) {
+		bp->b_whichq = BQ_LAUNDRY;
+		bp->b_timestamp = buf_timestamp();
 		binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
 		blaundrycnt++;
 
 		lck_mtx_unlock(buf_mtxp);
 
-		wakeup(&blaundrycnt);
-		/* and give it a chance to run */
+		wakeup(&bufqueues[BQ_LAUNDRY]);
+		/*
+		 * and give it a chance to run
+		 */
 		(void)thread_block(THREAD_CONTINUE_NULL);
 
 		lck_mtx_lock(buf_mtxp);
+
 		return (1);
 	}
-	bremhash(bp);
+#ifdef JOE_DEBUG
+	bp->b_owner = current_thread();
+	bp->b_tag   = 8;
+#endif
+	/*
+	 * Buffer is no longer on any free list... we own it
+	 */
+	SET(bp->b_lflags, BL_BUSY);
 
-	lck_mtx_unlock(buf_mtxp);
+	bremhash(bp);
 
-	BLISTNONE(bp);
 	/*
 	 * disassociate us from our vnode, if we had one...
 	 */
 	if (bp->b_vp)
-		brelvp(bp);
+		brelvp_locked(bp);
+
+	lck_mtx_unlock(buf_mtxp);
+
+	BLISTNONE(bp);
 
 	if (ISSET(bp->b_flags, B_META)) {
 	        vm_offset_t elem;
@@ -2830,10 +2950,13 @@ buf_invalblkno(vnode_t vp, daddr64_t lblkno, int flags)
 {
         buf_t	bp;
 	errno_t	error;
+	struct bufhashhdr *dp;
+
+	dp = BUFHASH(vp, lblkno);
 
 	lck_mtx_lock(buf_mtxp);
 relook:	
-	if ((bp = incore_locked(vp, lblkno)) == (struct buf *)0) {
+	if ((bp = incore_locked(vp, lblkno, dp)) == (struct buf *)0) {
 	        lck_mtx_unlock(buf_mtxp);
 		return (0);
 	}
@@ -2844,10 +2967,12 @@ relook:
 		}
 	        SET(bp->b_lflags, BL_WANTED);
 
-		error = msleep((caddr_t)bp, buf_mtxp, (PRIBIO + 1), (char *)"buf_invalblkno", 0);
+		error = msleep((caddr_t)bp, buf_mtxp, (PRIBIO + 1), "buf_invalblkno", NULL);
 
-		if (error)
+		if (error) {
+		        lck_mtx_unlock(buf_mtxp);
 			return (error);
+		}
 		goto relook;
 	}
 	bremfree_locked(bp);
@@ -2869,7 +2994,7 @@ buf_drop(buf_t bp)
 {
         int need_wakeup = 0;
 
-	lck_mtx_lock(buf_mtxp);
+	lck_mtx_lock_spin(buf_mtxp);
 
 	if (ISSET(bp->b_lflags, BL_WANTED)) {
 	        /*	
@@ -2878,6 +3003,10 @@ buf_drop(buf_t bp)
 		 */
 		need_wakeup = 1;
 	}
+#ifdef JOE_DEBUG
+	bp->b_owner = current_thread();
+	bp->b_tag   = 9;
+#endif
 	/*
 	 * Unlock the buffer.
 	 */
@@ -2934,7 +3063,7 @@ buf_acquire_locked(buf_t bp, int flags, int slpflag, int slptimeo)
 		/* the hz value is 100; which leads to 10ms */
 		ts.tv_sec = (slptimeo/100);
 		ts.tv_nsec = (slptimeo % 100) * 10  * NSEC_PER_USEC * 1000;
-		error = msleep((caddr_t)bp, buf_mtxp, slpflag | (PRIBIO + 1), (char *)"buf_acquire", &ts);
+		error = msleep((caddr_t)bp, buf_mtxp, slpflag | (PRIBIO + 1), "buf_acquire", &ts);
 
 		if (error)
 			return (error);
@@ -2960,8 +3089,10 @@ buf_biowait(buf_t bp)
 {
 	lck_mtx_lock(buf_mtxp);
 
+	DTRACE_IO1(wait__start, buf_t, bp);
 	while (!ISSET(bp->b_flags, B_DONE))
-		(void) msleep(bp, buf_mtxp, (PRIBIO+1), (char *)"buf_biowait", 0);
+		(void) msleep(bp, buf_mtxp, (PRIBIO+1), "buf_biowait", NULL);
+	DTRACE_IO1(wait__done, buf_t, bp);
 
 	lck_mtx_unlock(buf_mtxp);
 	
@@ -2975,6 +3106,22 @@ buf_biowait(buf_t bp)
 		return (0);
 }
 
+/*
+ * Wait for the callback operation on a B_CALL buffer to complete.
+ */
+void
+buf_biowait_callback(buf_t bp)
+{
+	lck_mtx_lock(buf_mtxp);
+
+	DTRACE_IO1(wait__start, buf_t, bp);
+	while (!ISSET(bp->b_lflags, BL_CALLDONE))
+		(void) msleep(bp, buf_mtxp, (PRIBIO+1), "buf_biowait", NULL);
+	DTRACE_IO1(wait__done, buf_t, bp);
+
+	lck_mtx_unlock(buf_mtxp);
+}
+
 /*
  * Mark I/O complete on a buffer.
  *
@@ -3003,6 +3150,10 @@ buf_biodone(buf_t bp)
 	if (ISSET(bp->b_flags, B_DONE))
 		panic("biodone already");
 
+	if (ISSET(bp->b_flags, B_ERROR)) {
+		fslog_io_error(bp);
+	}
+
         if (kdebug_enable) {
 	        int    code = DKIO_DONE;
 
@@ -3031,6 +3182,7 @@ buf_biodone(buf_t bp)
 	 * the DIRTY state from VM anymore
 	 */
 	CLR(bp->b_flags, B_WASDIRTY);
+	DTRACE_IO1(done, buf_t, bp);
 
 	if (!ISSET(bp->b_flags, B_READ) && !ISSET(bp->b_flags, B_RAW))
 	        /*
@@ -3050,18 +3202,40 @@ buf_biodone(buf_t bp)
 		bp->b_transaction = NULL;
 
 		if (iodone_func == NULL) {
-			panic("biodone: bp @ 0x%x has NULL b_iodone!\n", bp);			
+			panic("biodone: bp @ %p has NULL b_iodone!\n", bp);			
 		} else { 
 		        if (callout)
 			        SET(bp->b_flags, B_DONE);	/* note that it's done */
 			(*iodone_func)(bp, arg);
 		}
-		if (callout)
+		if (callout) {
+		        int need_wakeup = 0;
+
 		        /*
-			 * assumes that the call back function takes
+			 * assumes that the callback function takes
 			 * ownership of the bp and deals with releasing it if necessary
+			 * BL_WANTED indicates that we've decided to wait on the
+			 * completion of this I/O in a synchronous manner... we
+			 * still call the callback function, but in addition we
+			 * will do a wakeup... BL_CALLDONE indicates that the callback
+			 * routine has completed and its ok for the waiter to take
+			 * 'ownership' of this bp back
 			 */
-		        goto biodone_done;
+		        lck_mtx_lock_spin(buf_mtxp);
+
+			if (bp->b_lflags & BL_WANTED) {
+			        CLR(bp->b_lflags, BL_WANTED);
+				need_wakeup = 1;
+			}
+			SET(bp->b_lflags, BL_CALLDONE);
+
+			lck_mtx_unlock(buf_mtxp);
+			
+			if (need_wakeup)
+			        wakeup(bp);
+
+			goto biodone_done;
+		}
 		/*
 		 * in this case the call back function is acting
 		 * strictly as a filter... it does not take
@@ -3089,7 +3263,7 @@ buf_biodone(buf_t bp)
 		 * they do get to run, their going to re-set
 		 * BL_WANTED and go back to sleep
 		 */
-	        lck_mtx_lock(buf_mtxp);
+	        lck_mtx_lock_spin(buf_mtxp);
 
 		CLR(bp->b_lflags, BL_WANTED);
 		SET(bp->b_flags, B_DONE);		/* note that it's done */
@@ -3132,7 +3306,7 @@ count_busy_buffers(void)
 	int	nbusy = 0;
 
 	lck_mtx_lock(buf_mtxp);
-	for (bp = &buf[boot_nbuf]; --bp >= buf; )
+	for (bp = &buf_headers[boot_nbuf_headers]; --bp >= buf_headers; )
 	        if (!ISSET(bp->b_flags, B_INVAL) && ISSET(bp->b_lflags, BL_BUSY))
 			nbusy++;
 	lck_mtx_unlock(buf_mtxp);
@@ -3188,12 +3362,12 @@ alloc_io_buf(vnode_t vp, int priv)
 
 	lck_mtx_lock(iobuffer_mtxp);
 
-	while (((niobuf - NRESERVEDIOBUFS < bufstats.bufs_iobufinuse) && !priv) || 
+	while (((niobuf_headers - NRESERVEDIOBUFS < bufstats.bufs_iobufinuse) && !priv) || 
 	       (bp = iobufqueue.tqh_first) == NULL) {
 		bufstats.bufs_iobufsleeps++;
 
 		need_iobuffer = 1;
-		(void) msleep(&need_iobuffer, iobuffer_mtxp, (PRIBIO+1), (const char *)"alloc_io_buf", 0);
+		(void) msleep(&need_iobuffer, iobuffer_mtxp, (PRIBIO+1), (const char *)"alloc_io_buf", NULL);
 	}
 	TAILQ_REMOVE(&iobufqueue, bp, b_freelist);
 
@@ -3248,7 +3422,7 @@ free_io_buf(buf_t bp)
 	bp->b_vp = NULL;
 	bp->b_flags = B_INVAL;
 
-	lck_mtx_lock(iobuffer_mtxp);
+	lck_mtx_lock_spin(iobuffer_mtxp);
 
 	binsheadfree(bp, &iobufqueue, -1);
 
@@ -3274,15 +3448,23 @@ free_io_buf(buf_t bp)
 }
 
 
+void
+buf_list_lock(void)
+{
+        lck_mtx_lock(buf_mtxp);
+}
+
+void
+buf_list_unlock(void)
+{
+        lck_mtx_unlock(buf_mtxp);
+}
 
 /*
  * If getnewbuf() calls bcleanbuf() on the same thread
  * there is a potential for stack overrun and deadlocks.
  * So we always handoff the work to a worker thread for completion
  */
-#include <mach/mach_types.h>
-#include <mach/memory_object_types.h>
-#include <kern/sched_prim.h>
 
 
 static void
@@ -3302,15 +3484,23 @@ bcleanbuf_thread(void)
 	for (;;) {
 	        lck_mtx_lock(buf_mtxp);
 
-		while (blaundrycnt == 0)
-		        (void)msleep((void *)&blaundrycnt, buf_mtxp, PRIBIO, "blaundry", 0);
+		while ( (bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY])) == NULL)
+		        (void)msleep((void *)&bufqueues[BQ_LAUNDRY], buf_mtxp, PRIBIO, "blaundry", NULL);
 
-		bp = TAILQ_FIRST(&bufqueues[BQ_LAUNDRY]);
 		/*
 		 * Remove from the queue
 		 */
 		bremfree_locked(bp);
-		blaundrycnt--;
+
+		/*
+		 * Buffer is no longer on any free list
+		 */
+		SET(bp->b_lflags, BL_BUSY);
+
+#ifdef JOE_DEBUG
+		bp->b_owner = current_thread();
+		bp->b_tag   = 10;
+#endif
 
 		lck_mtx_unlock(buf_mtxp);
 		/*
@@ -3319,15 +3509,25 @@ bcleanbuf_thread(void)
 		error = bawrite_internal(bp, 0);
 
 		if (error) {
-		        lck_mtx_lock(buf_mtxp);
+		        bp->b_whichq = BQ_LAUNDRY;
+			bp->b_timestamp = buf_timestamp();
+
+		        lck_mtx_lock_spin(buf_mtxp);
 
 			binstailfree(bp, &bufqueues[BQ_LAUNDRY], BQ_LAUNDRY);
 			blaundrycnt++;
 
+			/* we never leave a busy page on the laundary queue */
+			CLR(bp->b_lflags, BL_BUSY);
+#ifdef JOE_DEBUG
+			bp->b_owner = current_thread();
+			bp->b_tag   = 11;
+#endif
+
 			lck_mtx_unlock(buf_mtxp);
 
 			if (loopcnt > 10) {
-			        (void)tsleep((void *)&blaundrycnt, PRIBIO, "blaundry", 1);
+			        (void)tsleep((void *)&bufqueues[BQ_LAUNDRY], PRIBIO, "blaundry", 1);
 				loopcnt = 0;
 			} else {
 			        (void)thread_block(THREAD_CONTINUE_NULL);
@@ -3610,14 +3810,14 @@ bufqdec(int q)
 }
 
 static void
-bufq_balance_thread_init()
+bufq_balance_thread_init(void)
 {
 
 	if (bufqscanwait++ == 0) {
 
 		/* Initalize globals */
 		MAXNBUF = (sane_size / PAGE_SIZE);
-		nbufh = nbuf;
+		nbufh = nbuf_headers;
 		nbuflow = min(nbufh, 100);
 		nbufhigh = min(MAXNBUF, max(nbufh, 2048));
 		nbuftarget = (sane_size >> 5) / PAGE_SIZE;
@@ -3721,7 +3921,6 @@ static int
 balancebufq(int q)
 {
 	int moretodo = 0;
-	int s = splbio();
 	int n, t;
 	
 	/* reject invalid q */
@@ -3771,7 +3970,6 @@ balancebufq(int q)
 	}
 
 out:
-	splx(s);
 	return (moretodo);		
 }
 
diff --git a/bsd/vfs/vfs_cache.c b/bsd/vfs/vfs_cache.c
index dbe7d1385..ed6ea3203 100644
--- a/bsd/vfs/vfs_cache.c
+++ b/bsd/vfs/vfs_cache.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -58,6 +64,12 @@
  *
  *	@(#)vfs_cache.c	8.5 (Berkeley) 3/22/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/time.h>
@@ -68,6 +80,11 @@
 #include <sys/malloc.h>
 #include <sys/kauth.h>
 #include <sys/user.h>
+#include <sys/paths.h>
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 /*
  * Name caching works as follows:
@@ -130,12 +147,19 @@ lck_rw_t * namecache_rw_lock;
 
 static vnode_t cache_lookup_locked(vnode_t dvp, struct componentname *cnp);
 static int  remove_name_locked(const char *);
-static char *add_name_locked(const char *, size_t, u_int, u_int);
-static void init_string_table(void);
+static const char *add_name_locked(const char *, size_t, u_int, u_int);
+static void init_string_table(void) __attribute__((section("__TEXT, initcode")));
 static void cache_delete(struct namecache *, int);
-static void dump_string_table(void);
+static void cache_enter_locked(vnode_t dvp, vnode_t vp, struct componentname *cnp);
+
+#ifdef DUMP_STRING_TABLE
+/*
+ * Internal dump function used for debugging
+ */
+void dump_string_table(void);
+#endif	/* DUMP_STRING_TABLE */
 
-static void init_crc32(void);
+static void init_crc32(void) __attribute__((section("__TEXT, initcode")));
 static unsigned int crc32tab[256];
 
 
@@ -151,105 +175,232 @@ static unsigned int crc32tab[256];
 // zero byte and thus the length is one greater than what strlen
 // would return.  This is important and lots of code elsewhere
 // in the kernel assumes this behavior.
+// 
+// This function can call vnop in file system if the parent vnode 
+// does not exist or when called for hardlinks via volfs path.  
+// If BUILDPATH_NO_FS_ENTER is set in flags, it only uses values present
+// in the name cache and does not enter the file system.
 //
 int
-build_path(vnode_t first_vp, char *buff, int buflen, int *outlen)
+build_path(vnode_t first_vp, char *buff, int buflen, int *outlen, int flags, vfs_context_t ctx)
 {
-        vnode_t vp = first_vp;
-	char *end, *str;
-	int   len, ret=0, counter=0;
-
+        vnode_t vp;
+        vnode_t proc_root_dir_vp;
+	char *end;
+	const char *str;
+	int  len;
+	int  ret = 0;
+	int  fixhardlink;
+
+	if (first_vp == NULLVP) {
+		return (EINVAL);
+	}
+	/* Grab the process fd so we can evaluate fd_rdir. */
+	if (vfs_context_proc(ctx)->p_fd) {
+	    proc_root_dir_vp = vfs_context_proc(ctx)->p_fd->fd_rdir;
+	} else {
+	    proc_root_dir_vp = NULL;
+	}
+again:
+	vp = first_vp;
 	end = &buff[buflen-1];
 	*end = '\0';
 
-	/*
-	 * if this is the root dir of a file system...
-	 */
-	if (vp && (vp->v_flag & VROOT) && vp->v_mount) {
-	        /*
-		 * then if it's the root fs, just put in a '/' and get out of here
-		 */
-	        if (vp->v_mount->mnt_flag & MNT_ROOTFS) {
+	/* Check if this is the root of a file system. */
+	while (vp && vp->v_flag & VROOT) {
+		if (vp->v_mount == NULL) {
+			return (EINVAL);
+		}
+	        if ((vp->v_mount->mnt_flag & MNT_ROOTFS) || (vp == proc_root_dir_vp)) {
+			/*
+			 * It's the root of the root file system, so it's
+			 * just "/".
+			 */
 		        *--end = '/';
 			goto out;
 		} else {
-		        /*
-			 * else just use the covered vnode to get the mount path
-			 */
 		        vp = vp->v_mount->mnt_vnodecovered;
 		}
 	}
 	NAME_CACHE_LOCK_SHARED();
 
-	while (vp && vp->v_parent != vp) {
-	        /*
-		 * the maximum depth of a file system hierarchy is MAXPATHLEN/2
-		 * (with single-char names separated by slashes).  we panic if
-		 * we've ever looped more than that.
-		 */
-	        if (counter++ > MAXPATHLEN/2) {
-		        panic("build_path: vnode parent chain is too long! vp 0x%x\n", vp);
-		}
-		str = vp->v_name;
-
-		if (str == NULL) {
-		        if (vp->v_parent != NULL) {
-			        ret = EINVAL;
-			}
-			break;
-		}
-		len = strlen(str);
-
+	while ((vp != NULLVP) && (vp->v_parent != vp)) {
 		/*
-		 * check that there's enough space (make sure to include space for the '/')
+		 * For hardlinks the v_name may be stale, so if its OK
+		 * to enter a file system, ask the file system for the
+		 * name and parent (below).
 		 */
-		if ((end - buff) < (len + 1)) {
-			ret = ENOSPC;
-			break;
+		fixhardlink = (vp->v_flag & VISHARDLINK) &&
+		              (vp->v_mount->mnt_kern_flag & MNTK_PATH_FROM_ID) &&
+		              !(flags & BUILDPATH_NO_FS_ENTER);
+		if (!fixhardlink) {
+			str = vp->v_name;
+			if (str == NULL || *str == '\0') {
+				if (vp->v_parent != NULL) {
+					ret = EINVAL;
+				} else {
+					ret = ENOENT;
+				}
+				break;
+			}
+			len = strlen(str);
+			/*
+			 * Check that there's enough space (including space for the '/')
+			 */
+			if ((end - buff) < (len + 1)) {
+				ret = ENOSPC;
+				break;
+			}
+			/* Copy the name backwards. */
+			str += len;
+	
+			for (; len > 0; len--) {
+			       *--end = *--str;
+			}
+			/* Add a path separator. */
+			*--end = '/';
 		}
-		/*
-		 * copy it backwards
-		 */
-		str += len;
 
-		for (; len > 0; len--) {
-		       *--end = *--str;
-		}
 		/*
-		 * put in the path separator
+		 * Walk up the parent chain.
 		 */
-		*--end = '/';
+		if (((vp->v_parent != NULLVP) && !fixhardlink) ||
+		    (flags & BUILDPATH_NO_FS_ENTER)) {
+			vp = vp->v_parent;
+
+			// if the vnode we have in hand isn't a directory and it
+			// has a v_parent, then we started with the resource fork
+			// so skip up to avoid getting a duplicate copy of the
+			// file name in the path.
+			if (vp && !vnode_isdir(vp) && vp->v_parent) {
+			    vp = vp->v_parent;
+			}
+		} else /* No parent, go get it if supported. */ {
+			struct vnode_attr  va;
+			vnode_t  dvp;
+			int  vid;
+
+			/* Make sure file system supports obtaining a path from id. */
+			if (!(vp->v_mount->mnt_kern_flag & MNTK_PATH_FROM_ID)) {
+				ret = ENOENT;
+				break;
+			}
+		        vid = vp->v_id;
+			NAME_CACHE_UNLOCK();
 
-		/*
-		 * walk up the chain (as long as we're not the root)  
-		 */
-		if (vp == first_vp && (vp->v_flag & VROOT)) {
-		        if (vp->v_mount && vp->v_mount->mnt_vnodecovered) {
-			        vp = vp->v_mount->mnt_vnodecovered->v_parent;
+			if (vnode_getwithvid(vp, vid) != 0) {
+				/* vnode was recycled, so start over. */
+				goto again;
+		        }
+			
+			VATTR_INIT(&va);
+			VATTR_WANTED(&va, va_parentid);
+			if (fixhardlink) {
+				VATTR_WANTED(&va, va_name);
+				MALLOC_ZONE(va.va_name, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
 			} else {
-			        vp = NULLVP;
+				va.va_name = NULL;
+			}
+			/* Ask the file system for its parent id and for its name (optional). */
+			ret = vnode_getattr(vp, &va, ctx);
+			if (fixhardlink) {
+				if (vp->v_name || VATTR_IS_SUPPORTED(&va, va_name)) {
+					if (ret == 0) {
+						str = va.va_name;
+					} else if (vp->v_name) {
+						str = vp->v_name;
+						ret = 0;
+					} else {
+						ret = ENOENT;
+						goto bad_news;
+					}
+
+					len = strlen(str);
+
+					/* Check that there's enough space. */
+					if ((end - buff) < (len + 1)) {
+						ret = ENOSPC;
+					} else {
+						/* Copy the name backwards. */
+						str += len;
+				
+						for (; len > 0; len--) {
+						       *--end = *--str;
+						}
+						/* Add a path separator. */
+						*--end = '/';
+					}
+				}
+			  bad_news:
+				FREE_ZONE(va.va_name, MAXPATHLEN, M_NAMEI);
+			}
+			if (ret || !VATTR_IS_SUPPORTED(&va, va_parentid)) {
+				vnode_put(vp);
+				ret = ENOENT;
+				goto out;
+			}
+			/* Ask the file system for the parent vnode. */
+			ret = VFS_VGET(vp->v_mount, (ino64_t)va.va_parentid, &dvp, ctx);
+			if (ret) {
+				vnode_put(vp);
+				goto out;
+			}
+			if (!fixhardlink && (vp->v_parent != dvp)) {
+				vnode_update_identity(vp, dvp, NULL, 0, 0, VNODE_UPDATE_PARENT);
+			}
+			vnode_put(vp);
+			vp = dvp;
+			/*
+			 * We are no longer under the name cache lock here.
+			 * So to avoid a race for vnode termination, take a
+			 * reference on the vnode and drop that reference
+			 * after reacquiring the name cache lock. We use the
+			 * vnode_rele_ext call with the dont_reenter flag
+			 * set to avoid re-entering the file system which
+			 * could possibly re-enter the name cache.
+			 */
+			if (vnode_ref(dvp) != 0) {
+				dvp = NULLVP;
+			}
+			vnode_put(vp);
+			NAME_CACHE_LOCK_SHARED();
+
+			if (dvp) {
+				vnode_rele_ext(dvp, 0, 1);
+			}
+
+			// if the vnode we have in hand isn't a directory and it
+			// has a v_parent, then we started with the resource fork
+			// so skip up to avoid getting a duplicate copy of the
+			// file name in the path.
+			if (vp && !vnode_isdir(vp) && vp->v_parent) {
+			    vp = vp->v_parent;
 			}
-		} else {
-		        vp = vp->v_parent;
 		}
 		/*
-		 * check if we're crossing a mount point and
-		 * switch the vp if we are.
+		 * When a mount point is crossed switch the vp.
+		 * Continue until we find the root or we find
+		 * a vnode that's not the root of a mounted
+		 * file system.
 		 */
-		if (vp && (vp->v_flag & VROOT) && vp->v_mount) {
-		        vp = vp->v_mount->mnt_vnodecovered;
+		while (vp) {
+			if (vp == proc_root_dir_vp) {
+				NAME_CACHE_UNLOCK();
+				goto out;  /* encountered the root */
+			}
+			if (!(vp->v_flag & VROOT) || !vp->v_mount)
+				break;	/* not the root of a mounted FS */
+	        	vp = vp->v_mount->mnt_vnodecovered;
 		}
 	}
 	NAME_CACHE_UNLOCK();
 out:
-	/*
-	 * slide it down to the beginning of the buffer
-	 */
+	/* Slide the name down to the beginning of the buffer. */
 	memmove(buff, end, &buff[buflen] - end);
     
-	*outlen = &buff[buflen] - end;  // length includes the trailing zero byte
+	*outlen = &buff[buflen] - end;  /* length includes the trailing zero byte */
  
-	return ret;
+	return (ret);
 }
 
 
@@ -284,10 +435,10 @@ vnode_getparent(vnode_t vp)
 	return (pvp);
 }
 
-char *
+const char *
 vnode_getname(vnode_t vp)
 {
-        char *name = NULL;
+        const char *name = NULL;
 
 	NAME_CACHE_LOCK();
 	
@@ -299,7 +450,7 @@ vnode_getname(vnode_t vp)
 }
 
 void
-vnode_putname(char *name)
+vnode_putname(const char *name)
 {
         NAME_CACHE_LOCK();
 
@@ -323,17 +474,31 @@ vnode_putname(char *name)
  * if VNODE_UPDATE_CACHE, flush the name cache entries associated with vp
  */
 void
-vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int name_hashval, int flags)
+vnode_update_identity(vnode_t vp, vnode_t dvp, const char *name, int name_len, int name_hashval, int flags)
 {
 	struct	namecache *ncp;
         vnode_t	old_parentvp = NULLVP;
-
+#if NAMEDSTREAMS
+	int isstream = (vp->v_flag & VISNAMEDSTREAM);
+	int kusecountbumped = 0;
+#endif
 
 	if (flags & VNODE_UPDATE_PARENT) {
-	        if (dvp && vnode_ref(dvp) != 0)
-		        dvp = NULLVP;
-	} else
+	        if (dvp && vnode_ref(dvp) != 0) {
+			dvp = NULLVP;
+		}
+#if NAMEDSTREAMS
+		/* Don't count a stream's parent ref during unmounts */
+		if (isstream && dvp && (dvp != vp) && (dvp != vp->v_parent) && (dvp->v_type == VREG)) {
+			vnode_lock_spin(dvp);
+			++dvp->v_kusecount;
+			kusecountbumped = 1;
+			vnode_unlock(dvp);
+		}
+#endif
+	} else {
 	        dvp = NULLVP;
+	}
 	NAME_CACHE_LOCK();
 
 	if ( (flags & VNODE_UPDATE_NAME) && (name != vp->v_name) ) {
@@ -363,12 +528,29 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 	}
 	NAME_CACHE_UNLOCK();
 	
-	if (dvp != NULLVP)
+	if (dvp != NULLVP) {
+#if NAMEDSTREAMS
+		/* Back-out the ref we took if we lost a race for vp->v_parent. */
+		if (kusecountbumped) {
+			vnode_lock_spin(dvp);
+			if (dvp->v_kusecount > 0)
+				--dvp->v_kusecount;  
+			vnode_unlock(dvp);
+		}
+#endif
 	        vnode_rele(dvp);
-	
+	}
 	if (old_parentvp) {
 	        struct  uthread *ut;
 
+#if NAMEDSTREAMS
+		if (isstream) {
+		        vnode_lock_spin(old_parentvp);
+			if ((old_parentvp->v_type != VDIR) && (old_parentvp->v_kusecount > 0))
+				--old_parentvp->v_kusecount;
+			vnode_unlock(old_parentvp);
+		}
+#endif
 	        ut = get_bsdthread_info(current_thread());
 
 		/*
@@ -384,7 +566,6 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 	        while ( (vp = old_parentvp) != NULLVP ) {
 	  
 		        vnode_lock(vp);
-
 			vnode_rele_internal(vp, 0, 0, 1);
 
 			/*
@@ -413,7 +594,13 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
 				 * pull the parent pointer now so that when we do the
 				 * vnode_reclaim for each of the vnodes in the uu_vreclaims
 				 * list, we won't recurse back through here
+				 *
+				 * need to do a convert here in case vnode_rele_internal
+				 * returns with the lock held in the spin mode... it 
+				 * can drop and retake the lock under certain circumstances
 				 */
+			        vnode_lock_convert(vp);
+
 			        NAME_CACHE_LOCK();
 				old_parentvp = vp->v_parent;
 				vp->v_parent = NULLVP;
@@ -451,9 +638,9 @@ vnode_update_identity(vnode_t vp, vnode_t dvp, char *name, int name_len, int nam
  * so that HFS can post-process the lookup.  Also, volfs will call
  * VNOP_GETATTR2 to determine the parent, instead of using v_parent.
  */
-void vnode_set_hard_link(vnode_t vp)
+void vnode_setmultipath(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	/*
 	 * In theory, we're changing the vnode's identity as far as the
@@ -472,135 +659,200 @@ void vnode_set_hard_link(vnode_t vp)
 }
 
 
+
+/*
+ * backwards compatibility
+ */
 void vnode_uncache_credentials(vnode_t vp)
 {
-        kauth_cred_t ucred = NOCRED;
-
-	vnode_lock(vp);
-        if (IS_VALID_CRED(vp->v_cred)) {
-		ucred = vp->v_cred;
-		vp->v_cred = NOCRED;
-	}
-	vnode_unlock(vp);
-
-	if (ucred != NOCRED)
-		kauth_cred_unref(&ucred);
+        vnode_uncache_authorized_action(vp, KAUTH_INVALIDATE_CACHED_RIGHTS);
 }
 
 
-void vnode_cache_credentials(vnode_t vp, vfs_context_t context)
-{
-	kauth_cred_t ucred;
-	kauth_cred_t tcred = NOCRED;
-	struct timeval tv;
+/*
+ * use the exclusive form of NAME_CACHE_LOCK to protect the update of the
+ * following fields in the vnode: v_cred_timestamp, v_cred, v_authorized_actions
+ * we use this lock so that we can look at the v_cred and v_authorized_actions
+ * atomically while behind the NAME_CACHE_LOCK in shared mode in 'cache_lookup_path',
+ * which is the super-hot path... if we are updating the authorized actions for this
+ * vnode, we are already in the super-slow and far less frequented path so its not
+ * that bad that we take the lock exclusive for this case... of course we strive
+ * to hold it for the minimum amount of time possible
+ */
 
-	ucred = vfs_context_ucred(context);
+void vnode_uncache_authorized_action(vnode_t vp, kauth_action_t action)
+{
+        kauth_cred_t tcred = NOCRED;
 
-	vnode_lock(vp);
-	if (IS_VALID_CRED(ucred) && (vp->v_cred != ucred || (vp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE))) {
+	NAME_CACHE_LOCK();
 
-		microuptime(&tv);
-		vp->v_cred_timestamp = tv.tv_sec;
+	vp->v_authorized_actions &= ~action;
 
-		if (vp->v_cred != ucred) {
-			kauth_cred_ref(ucred);
-	
-			tcred = vp->v_cred;
-			vp->v_cred = ucred;
-		}
+	if (action == KAUTH_INVALIDATE_CACHED_RIGHTS &&
+	    IS_VALID_CRED(vp->v_cred)) {
+	        /*
+		 * Use a temp variable to avoid kauth_cred_unref() while NAME_CACHE_LOCK is held
+		 */
+	        tcred = vp->v_cred;
+		vp->v_cred = NOCRED;
 	}
-	vnode_unlock(vp);
+	NAME_CACHE_UNLOCK();
 
-	if (IS_VALID_CRED(tcred))
+	if (tcred != NOCRED)
 		kauth_cred_unref(&tcred);
 }
 
-/*	reverse_lookup - lookup by walking back up the parent chain while leveraging
- *	use of the name cache lock in order to protect our starting vnode.
- *	NOTE - assumes you already have search access to starting point.
- *  returns 0 when we have reached the root, current working dir, or chroot root 
- *
- */
-int
-reverse_lookup(vnode_t start_vp, vnode_t *lookup_vpp, struct filedesc *fdp, vfs_context_t context, int *dp_authorized)
+
+boolean_t vnode_cache_is_authorized(vnode_t vp, vfs_context_t ctx, kauth_action_t action)
 {
-	int				vid, done = 0;
-	int				auth_opaque = 0;
-	vnode_t			dp = start_vp;
-	vnode_t			vp = NULLVP;
 	kauth_cred_t	ucred;
-	struct timeval 	tv;
+	boolean_t	retval = FALSE;
 
-	ucred = vfs_context_ucred(context);
-	*lookup_vpp = start_vp;
+	if ( (vp->v_mount->mnt_kern_flag & (MNTK_AUTH_OPAQUE | MNTK_AUTH_CACHE_TTL)) ) {
+	        /*
+		 * a TTL is enabled on the rights cache... handle it here
+		 * a TTL of 0 indicates that no rights should be cached
+		 */
+	        if (vp->v_mount->mnt_authcache_ttl) {
+		        if ( !(vp->v_mount->mnt_kern_flag & MNTK_AUTH_CACHE_TTL) ) {
+			        /*
+				 * For filesystems marked only MNTK_AUTH_OPAQUE (generally network ones),
+				 * we will only allow a SEARCH right on a directory to be cached...
+				 * that cached right always has a default TTL associated with it
+				 */
+			        if (action != KAUTH_VNODE_SEARCH || vp->v_type != VDIR)
+				        vp = NULLVP;
+			}
+			if (vp != NULLVP && vnode_cache_is_stale(vp) == TRUE) {
+			        vnode_uncache_authorized_action(vp, vp->v_authorized_actions);
+				vp = NULLVP;
+			}
+		} else
+		        vp = NULLVP;
+	}
+	if (vp != NULLVP) {
+	        ucred = vfs_context_ucred(ctx);
 
-	NAME_CACHE_LOCK_SHARED();
+		NAME_CACHE_LOCK_SHARED();
 
-	if ( dp->v_mount && (dp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE) ) {
-		auth_opaque = 1;
-		microuptime(&tv);
+		if (vp->v_cred == ucred && (vp->v_authorized_actions & action) == action)
+		        retval = TRUE;
+		
+		NAME_CACHE_UNLOCK();
 	}
-	for (;;) {
-		*dp_authorized = 0;
+	return retval;
+}
 
-		if (auth_opaque && ((tv.tv_sec - dp->v_cred_timestamp) > VCRED_EXPIRED))
-			break;
-		/* XXX should be safe without vnode_lock() */
-		if (dp->v_cred != ucred)
-			break;
-		/*
-		 * indicate that we're allowed to traverse this directory...
-		 * even if we bail for some reason, this information is valid and is used
-		 * to avoid doing a vnode_authorize
+
+void vnode_cache_authorized_action(vnode_t vp, vfs_context_t ctx, kauth_action_t action)
+{
+	kauth_cred_t tcred = NOCRED;
+	kauth_cred_t ucred;
+	struct timeval tv;
+	boolean_t ttl_active = FALSE;
+
+	ucred = vfs_context_ucred(ctx);
+
+	if (!IS_VALID_CRED(ucred) || action == 0)
+	        return;
+
+	if ( (vp->v_mount->mnt_kern_flag & (MNTK_AUTH_OPAQUE | MNTK_AUTH_CACHE_TTL)) ) {
+	        /*
+		 * a TTL is enabled on the rights cache... handle it here
+		 * a TTL of 0 indicates that no rights should be cached
 		 */
-		*dp_authorized = 1;
+	        if (vp->v_mount->mnt_authcache_ttl == 0) 
+		        return;
 
-		if ((dp->v_flag & VROOT) != 0 	||		/* Hit "/" */
-		    (dp == fdp->fd_cdir) 	||		/* Hit process's working directory */
-		    (dp == fdp->fd_rdir)) {			/* Hit process chroot()-ed root */
-	 		done = 1;
-	 		break;
+		if ( !(vp->v_mount->mnt_kern_flag & MNTK_AUTH_CACHE_TTL) ) {
+		        /*
+			 * only cache SEARCH action for filesystems marked
+			 * MNTK_AUTH_OPAQUE on VDIRs...
+			 * the lookup_path code will time these out
+			 */
+		        if ( (action & ~KAUTH_VNODE_SEARCH) || vp->v_type != VDIR )
+			        return;
 		}
+		ttl_active = TRUE;
 
-		if ( (vp = dp->v_parent) == NULLVP)
-			break;
+		microuptime(&tv);
+	}
+	NAME_CACHE_LOCK();
 
-		dp = vp;
-		*lookup_vpp = dp;
-	} /* for (;;) */
+	if (vp->v_cred != ucred) {
+	        kauth_cred_ref(ucred);
+	        /*
+		 * Use a temp variable to avoid kauth_cred_unref() while NAME_CACHE_LOCK is held
+		 */
+		tcred = vp->v_cred;
+		vp->v_cred = ucred;
+		vp->v_authorized_actions = 0;
+	}
+	if (ttl_active == TRUE && vp->v_authorized_actions == 0) {
+	        /*
+		 * only reset the timestamnp on the
+		 * first authorization cached after the previous
+		 * timer has expired or we're switching creds...
+		 * 'vnode_cache_is_authorized' will clear the 
+		 * authorized actions if the TTL is active and
+		 * it has expired
+		 */
+	        vp->v_cred_timestamp = tv.tv_sec;
+	}
+	vp->v_authorized_actions |= action;
 
-	vid = dp->v_id;
-	
 	NAME_CACHE_UNLOCK();
-	
-	if (done == 0 && dp != start_vp) {
-		if (vnode_getwithvid(dp, vid) != 0) {
-			*lookup_vpp = start_vp;
-		}
-	}
 
-	return((done == 1) ? 0 : -1);
+	if (IS_VALID_CRED(tcred))
+		kauth_cred_unref(&tcred);
 }
 
+
+boolean_t vnode_cache_is_stale(vnode_t vp)
+{
+	struct timeval	tv;
+	boolean_t	retval;
+
+	microuptime(&tv);
+
+	if ((tv.tv_sec - vp->v_cred_timestamp) > vp->v_mount->mnt_authcache_ttl)
+	        retval = TRUE;
+	else
+	        retval = FALSE;
+
+	return retval;
+}
+
+
+
+/*
+ * Returns:	0			Success
+ *		ENOENT			No such file or directory
+ */
 int 
-cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp, vfs_context_t context, int *trailing_slash, int *dp_authorized)
+cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp, vfs_context_t ctx, int *trailing_slash, int *dp_authorized)
 {
 	char		*cp;		/* pointer into pathname argument */
-	int		vid, vvid;
-	int		auth_opaque = 0;
+	int		vid;
+	int		vvid = 0;	/* protected by vp != NULLVP */
 	vnode_t		vp = NULLVP;
 	vnode_t		tdp = NULLVP;
 	kauth_cred_t	ucred;
-	struct timeval tv;
+	boolean_t	ttl_enabled = FALSE;
+	struct timeval	tv;
+        mount_t		mp;
 	unsigned int	hash;
+#if CONFIG_MACF
+	int		error;
+#endif
 
-	ucred = vfs_context_ucred(context);
+	ucred = vfs_context_ucred(ctx);
 	*trailing_slash = 0;
 
 	NAME_CACHE_LOCK_SHARED();
 
-	if ( dp->v_mount && (dp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE) ) {
-		auth_opaque = 1;
+	if ( dp->v_mount && (dp->v_mount->mnt_kern_flag & (MNTK_AUTH_OPAQUE | MNTK_AUTH_CACHE_TTL)) ) {
+		ttl_enabled = TRUE;
 		microuptime(&tv);
 	}
 	for (;;) {
@@ -657,13 +909,56 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 		        cnp->cn_flags |= ISDOTDOT;
 
 		*dp_authorized = 0;
+#if NAMEDRSRCFORK
+		/*
+		 * Process a request for a file's resource fork.
+		 *
+		 * Consume the _PATH_RSRCFORKSPEC suffix and tag the path.
+		 */
+		if ((ndp->ni_pathlen == sizeof(_PATH_RSRCFORKSPEC)) &&
+		    (cp[1] == '.' && cp[2] == '.') &&
+		    bcmp(cp, _PATH_RSRCFORKSPEC, sizeof(_PATH_RSRCFORKSPEC)) == 0) {
+		    	/* Skip volfs file systems that don't support native streams. */
+			if ((dp->v_mount != NULL) &&
+			    (dp->v_mount->mnt_flag & MNT_DOVOLFS) &&
+			    (dp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) == 0) {
+				goto skiprsrcfork;
+			}
+			cnp->cn_flags |= CN_WANTSRSRCFORK;
+			cnp->cn_flags |= ISLASTCN;
+			ndp->ni_next[0] = '\0';
+			ndp->ni_pathlen = 1;
+		}
+skiprsrcfork:
+#endif
 
-		if (auth_opaque && ((tv.tv_sec - dp->v_cred_timestamp) > VCRED_EXPIRED))
+#if CONFIG_MACF
+
+		/* 
+		 * Name cache provides authorization caching (see below)
+		 * that will short circuit MAC checks in lookup().
+		 * We must perform MAC check here.  On denial
+		 * dp_authorized will remain 0 and second check will
+		 * be perfomed in lookup().
+		 */
+		if (!(cnp->cn_flags & DONOTAUTH)) {
+			error = mac_vnode_check_lookup(ctx, dp, cnp);
+			if (error) {
+				name_cache_unlock();
+				return (error);
+			}
+		}
+#endif /* MAC */
+		if (ttl_enabled && ((tv.tv_sec - dp->v_cred_timestamp) > dp->v_mount->mnt_authcache_ttl))
 		        break;
 
-		/* XXX should be safe without vnode_lock() */
-		if (dp->v_cred != ucred)
+		/*
+		 * NAME_CACHE_LOCK holds these fields stable
+		 */
+		if ((dp->v_cred != ucred || !(dp->v_authorized_actions & KAUTH_VNODE_SEARCH)) &&
+		    !(dp->v_authorized_actions & KAUTH_VNODE_SEARCHBYANYONE))
 		        break;
+
 		/*
 		 * indicate that we're allowed to traverse this directory...
 		 * even if we fail the cache lookup or decide to bail for
@@ -678,6 +973,13 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 		        if (cnp->cn_flags & (LOCKPARENT | NOCACHE))
 			        break;
 			if (cnp->cn_flags & ISDOTDOT) {
+				/*
+				 * Force directory hardlinks to go to
+				 * file system for ".." requests.
+				 */
+				if (dp && (dp->v_flag & VISHARDLINK)) {
+					break;
+				}
 				/*
 				 * Quit here only if we can't use
 				 * the parent directory pointer or
@@ -697,7 +999,7 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 		 */
 		if (cnp->cn_namelen == 1 && cnp->cn_nameptr[0] == '.')
 			vp = dp;
-		else if (cnp->cn_flags & ISDOTDOT)
+		else if ((cnp->cn_flags & ISDOTDOT) && dp->v_parent)
 			vp = dp->v_parent;
 		else {
 			if ( (vp = cache_lookup_locked(dp, cnp)) == NULLVP)
@@ -712,9 +1014,13 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 			        vp = NULL;
 		        break;
 		}
-		if (vp->v_mountedhere && ((cnp->cn_flags & NOCROSSMOUNT) == 0))
-		        break;
+		if ( (mp = vp->v_mountedhere) && ((cnp->cn_flags & NOCROSSMOUNT) == 0)) {
 
+		        if (mp->mnt_realrootvp == NULLVP || mp->mnt_generation != mount_generation ||
+				mp->mnt_realrootvp_vid != mp->mnt_realrootvp->v_id)
+			        break;
+			vp = mp->mnt_realrootvp;
+		}
 		dp = vp;
 		vp = NULLVP;
 
@@ -731,7 +1037,6 @@ cache_lookup_path(struct nameidata *ndp, struct componentname *cnp, vnode_t dp,
 	
 	NAME_CACHE_UNLOCK();
 
-
 	if ((vp != NULLVP) && (vp->v_type != VLNK) &&
 	    ((cnp->cn_flags & (ISLASTCN | LOCKPARENT | WANTPARENT | SAVESTART)) == ISLASTCN)) {
 	        /*
@@ -768,9 +1073,14 @@ need_dp:
 			 * failure indicates the vnode
 			 * changed identity or is being
 			 * TERMINATED... in either case
-			 * punt this lookup
+			 * punt this lookup.
+			 * 
+			 * don't necessarily return ENOENT, though, because
+			 * we really want to go back to disk and make sure it's
+			 * there or not if someone else is changing this
+			 * vnode.
 			 */
-		        return (ENOENT);
+		        return (ERESTART);
 		}
 	}
 	if (vp != NULLVP) {
@@ -801,9 +1111,9 @@ need_dp:
 static vnode_t
 cache_lookup_locked(vnode_t dvp, struct componentname *cnp)
 {
-	register struct namecache *ncp;
-	register struct nchashhead *ncpp;
-	register long namelen = cnp->cn_namelen;
+	struct namecache *ncp;
+	struct nchashhead *ncpp;
+	long namelen = cnp->cn_namelen;
 	char *nameptr = cnp->cn_nameptr;
 	unsigned int hashval = (cnp->cn_hash & NCHASHMASK);
 	vnode_t vp;
@@ -881,14 +1191,11 @@ hash_string(const char *cp, int len)
  */
 
 int
-cache_lookup(dvp, vpp, cnp)
-	struct vnode *dvp;
-	struct vnode **vpp;
-	struct componentname *cnp;
+cache_lookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp)
 {
-	register struct namecache *ncp;
-	register struct nchashhead *ncpp;
-	register long namelen = cnp->cn_namelen;
+	struct namecache *ncp;
+	struct nchashhead *ncpp;
+	long namelen = cnp->cn_namelen;
 	char *nameptr = cnp->cn_nameptr;
 	unsigned int hashval = (cnp->cn_hash & NCHASHMASK);
 	boolean_t	have_exclusive = FALSE;
@@ -972,30 +1279,62 @@ relook:
 	return (ENOENT);
 }
 
+
 /*
- * Add an entry to the cache.
+ * Add an entry to the cache...
+ * but first check to see if the directory
+ * that this entry is to be associated with has
+ * had any cache_purges applied since we took
+ * our identity snapshot... this check needs to
+ * be done behind the name cache lock
  */
 void
-cache_enter(dvp, vp, cnp)
-	struct vnode *dvp;
-	struct vnode *vp;
-	struct componentname *cnp;
+cache_enter_with_gen(struct vnode *dvp, struct vnode *vp, struct componentname *cnp, int gen)
 {
-        register struct namecache *ncp, *negp;
-	register struct nchashhead *ncpp;
 
         if (cnp->cn_hash == 0)
 	        cnp->cn_hash = hash_string(cnp->cn_nameptr, cnp->cn_namelen);
 
 	NAME_CACHE_LOCK();
 
-	/* if the entry is for -ve caching vp is null */
+	if (dvp->v_nc_generation == gen)
+	        cache_enter_locked(dvp, vp, cnp);
+
+	NAME_CACHE_UNLOCK();
+}
+
+
+/*
+ * Add an entry to the cache.
+ */
+void
+cache_enter(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
+{
+        if (cnp->cn_hash == 0)
+	        cnp->cn_hash = hash_string(cnp->cn_nameptr, cnp->cn_namelen);
+
+	NAME_CACHE_LOCK();
+
+	cache_enter_locked(dvp, vp, cnp);
+
+	NAME_CACHE_UNLOCK();
+}
+
+
+static void
+cache_enter_locked(struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
+{
+        struct namecache *ncp, *negp;
+	struct nchashhead *ncpp;
+
+	/*
+	 * if the entry is for -ve caching vp is null
+	 */
 	if ((vp != NULLVP) && (LIST_FIRST(&vp->v_nclinks))) {
 	        /*
 		 * someone beat us to the punch..
 		 * this vnode is already in the cache
 		 */
-	        NAME_CACHE_UNLOCK();
 		return;
 	}
 	/*
@@ -1047,7 +1386,7 @@ cache_enter(dvp, vp, cnp)
 	ncpp = NCHHASH(dvp, cnp->cn_hash);
 #if DIAGNOSTIC
 	{
-		register struct namecache *p;
+		struct namecache *p;
 
 		for (p = ncpp->lh_first; p != 0; p = p->nc_hash.le_next)
 			if (p == ncp)
@@ -1094,8 +1433,6 @@ cache_enter(dvp, vp, cnp)
 	 * are children of dvp
 	 */
 	LIST_INSERT_HEAD(&dvp->v_ncchildren, ncp, nc_child);
-
-	NAME_CACHE_UNLOCK();
 }
 
 
@@ -1143,7 +1480,7 @@ nchinit(void)
 
 	init_crc32();
 
-	nchashtbl = hashinit(MAX(4096, (2 *desiredNodes)), M_CACHE, &nchash);
+	nchashtbl = hashinit(MAX(CONFIG_NC_HASH, (2 *desiredNodes)), M_CACHE, &nchash);
 	nchashmask = nchash;
 	nchash++;
 
@@ -1284,19 +1621,33 @@ void
 cache_purge(vnode_t vp)
 {
         struct namecache *ncp;
+	kauth_cred_t tcred = NULL;
 
 	if ((LIST_FIRST(&vp->v_nclinks) == NULL) && (LIST_FIRST(&vp->v_ncchildren) == NULL))
 	        return;
 
 	NAME_CACHE_LOCK();
 
+	if (vp->v_parent)
+	        vp->v_parent->v_nc_generation++;
+
 	while ( (ncp = LIST_FIRST(&vp->v_nclinks)) )
 	        cache_delete(ncp, 1);
 
 	while ( (ncp = LIST_FIRST(&vp->v_ncchildren)) )
 	        cache_delete(ncp, 1);
 
+	/*
+	 * Use a temp variable to avoid kauth_cred_unref() while NAME_CACHE_LOCK is held
+	 */
+	tcred = vp->v_cred;
+	vp->v_cred = NOCRED;
+	vp->v_authorized_actions = 0;
+
 	NAME_CACHE_UNLOCK();
+
+	if (IS_VALID_CRED(tcred))
+	        kauth_cred_unref(&tcred);
 }
 
 /*
@@ -1327,8 +1678,7 @@ cache_purge_negatives(vnode_t vp)
  * entries at the same time.
  */
 void
-cache_purgevfs(mp)
-	struct mount *mp;
+cache_purgevfs(struct mount *mp)
 {
 	struct nchashhead *ncpp;
 	struct namecache *ncp;
@@ -1362,7 +1712,7 @@ static uint32_t nstrings=0;
 
 typedef struct string_t {
     LIST_ENTRY(string_t)  hash_chain;
-    unsigned char        *str;
+    const char *str;
     uint32_t              refcount;
 } string_t;
 
@@ -1389,7 +1739,7 @@ resize_string_ref_table(void)
     old_mask          = string_table_mask;
     string_table_mask = new_mask;
 
-    printf("resize: max chain len %d, new table size %d\n",
+    printf("resize: max chain len %d, new table size %lu\n",
 	   max_chain_len, new_mask + 1);
     max_chain_len   = 0;
     long_chain_head = NULL;
@@ -1401,7 +1751,7 @@ resize_string_ref_table(void)
     for(i=0; i <= old_mask; i++) {
 	old_head = &old_table[i];
 	for (entry=old_head->lh_first; entry != NULL; entry=next) {
-	    hashval = hash_string(entry->str, 0);
+	    hashval = hash_string((const char *)entry->str, 0);
 	    head = &string_ref_table[hashval & string_table_mask];
 	    if (head->lh_first == NULL) {
 		filled_buckets++;
@@ -1421,14 +1771,14 @@ resize_string_ref_table(void)
 static void
 init_string_table(void)
 {
-    string_ref_table = hashinit(4096, M_CACHE, &string_table_mask);
+	string_ref_table = hashinit(CONFIG_VFS_NAMES, M_CACHE, &string_table_mask);
 }
 
 
-char *
+const char *
 vfs_addname(const char *name, size_t len, u_int hashval, u_int flags)
 {
-        char * ptr;
+        const char * ptr;
 
 	NAME_CACHE_LOCK();
 	ptr = add_name_locked(name, len, hashval, flags);
@@ -1437,12 +1787,13 @@ vfs_addname(const char *name, size_t len, u_int hashval, u_int flags)
 	return(ptr);
 }
 
-static char *
+static const char *
 add_name_locked(const char *name, size_t len, u_int hashval, __unused u_int flags)
 {
     struct stringhead *head;
     string_t          *entry;
     uint32_t          chain_len = 0;
+    char              *ptr;
     
     //
     // If the table gets more than 3/4 full, resize it
@@ -1456,6 +1807,15 @@ add_name_locked(const char *name, size_t len, u_int hashval, __unused u_int flag
 	hashval = hash_string(name, 0);
     }
 
+    //
+    // if the length already accounts for the null-byte, then
+    // subtract one so later on we don't index past the end
+    // of the string.
+    //
+    if (len > 0 && name[len-1] == '\0') {
+	len--;
+    }
+
     head = &string_ref_table[hashval & string_table_mask];
     for (entry=head->lh_first; entry != NULL; chain_len++, entry=entry->hash_chain.le_next) {
 	if (memcmp(entry->str, name, len) == 0 && entry->str[len] == '\0') {
@@ -1477,9 +1837,10 @@ add_name_locked(const char *name, size_t len, u_int hashval, __unused u_int flag
 	    filled_buckets++;
 	}
 
-	entry->str = (char *)((char *)entry + sizeof(string_t));
-	strncpy(entry->str, name, len);
-	entry->str[len] = '\0';
+	ptr = (char *)((char *)entry + sizeof(string_t));
+	strncpy(ptr, name, len);
+	ptr[len] = '\0';
+	entry->str = ptr;
 	entry->refcount = 1;
 	LIST_INSERT_HEAD(head, entry, hash_chain);
 
@@ -1491,7 +1852,7 @@ add_name_locked(const char *name, size_t len, u_int hashval, __unused u_int flag
 	nstrings++;
     }
     
-    return entry->str;
+    return (const char *)entry->str;
 }
 
 int
@@ -1514,12 +1875,12 @@ remove_name_locked(const char *nameref)
     struct stringhead *head;
     string_t          *entry;
     uint32_t           hashval;
-    char * ptr;
+    const char        *ptr;
 
     hashval = hash_string(nameref, 0);
     head = &string_ref_table[hashval & string_table_mask];
     for (entry=head->lh_first; entry != NULL; entry=entry->hash_chain.le_next) {
-	if (entry->str == (unsigned char *)nameref) {
+	if (entry->str == nameref) {
 	    entry->refcount--;
 	    if (entry->refcount == 0) {
 		LIST_REMOVE(entry, hash_chain);
@@ -1543,6 +1904,7 @@ remove_name_locked(const char *nameref)
 }
 
 
+#ifdef DUMP_STRING_TABLE
 void
 dump_string_table(void)
 {
@@ -1560,3 +1922,4 @@ dump_string_table(void)
     }
     NAME_CACHE_UNLOCK();
 }
+#endif	/* DUMP_STRING_TABLE */
diff --git a/bsd/vfs/vfs_cluster.c b/bsd/vfs/vfs_cluster.c
index e2b4b14d0..8d3657909 100644
--- a/bsd/vfs/vfs_cluster.c
+++ b/bsd/vfs/vfs_cluster.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -70,6 +76,7 @@
 #include <machine/machine_routines.h>
 
 #include <sys/ubc_internal.h>
+#include <vm/vnode_pager.h>
 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
@@ -82,19 +89,19 @@
 
 #include <sys/kdebug.h>
 
-
-#define CL_READ      0x01
-#define CL_ASYNC     0x02
-#define CL_COMMIT    0x04
-#define CL_PAGEOUT   0x10
-#define CL_AGE       0x20
-#define CL_DUMP      0x40
-#define CL_NOZERO    0x80
-#define CL_PAGEIN    0x100
-#define CL_DEV_MEMORY 0x200
-#define CL_PRESERVE   0x400
-#define CL_THROTTLE   0x800
-#define CL_KEEPCACHED 0x1000
+#define CL_READ		0x01
+#define CL_ASYNC	0x02
+#define CL_COMMIT	0x04
+#define CL_PAGEOUT	0x10
+#define CL_AGE		0x20
+#define CL_NOZERO	0x40
+#define CL_PAGEIN	0x80
+#define CL_DEV_MEMORY	0x100
+#define CL_PRESERVE	0x200
+#define CL_THROTTLE	0x400
+#define CL_KEEPCACHED	0x800
+#define CL_DIRECT_IO	0x1000
+#define CL_PASSIVE	0x2000
 
 
 struct clios {
@@ -110,38 +117,84 @@ static lck_grp_attr_t   *cl_mtx_grp_attr;
 static lck_mtx_t	*cl_mtxp;
 
 
+#define	IO_UNKNOWN	0
+#define	IO_DIRECT	1
+#define IO_CONTIG	2
+#define IO_COPY		3
+
+#define	PUSH_DELAY	0x01
+#define PUSH_ALL	0x02
+#define	PUSH_SYNC	0x04
+
+
+static void cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset);
+static void cluster_wait_IO(buf_t cbp_head, int async);
+static void cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait);
+
+static int cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length);
+
 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
-		      int flags, buf_t real_bp, struct clios *iostate);
-static int cluster_iodone(buf_t bp, void *dummy);
-static int cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize);
+		      int flags, buf_t real_bp, struct clios *iostate, int (*)(buf_t, void *), void *callback_arg);
+static int cluster_iodone(buf_t bp, void *callback_arg);
+static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags);
 static int cluster_hard_throttle_on(vnode_t vp);
 
-static int cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags);
-static int cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
-			   off_t headOff, off_t tailOff, int flags);
-static int cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize);
-static int cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF);
-static int cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize);
-static int cluster_phys_write(vnode_t vp, struct uio *uio, off_t newEOF);
-static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize, int flags);
+static void cluster_syncup(vnode_t vp, off_t newEOF, int (*)(buf_t, void *), void *callback_arg);
+
+static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags);
+static int cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference);
+
+static int cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size,  off_t filesize, int flags,
+			     int (*)(buf_t, void *), void *callback_arg);
+static int cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
+			       int flags, int (*)(buf_t, void *), void *callback_arg);
+static int cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
+			       int (*)(buf_t, void *), void *callback_arg, int flags);
 
-static void	cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra);
+static int cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF,
+			      off_t headOff, off_t tailOff, int flags, int (*)(buf_t, void *), void *callback_arg);
+static int cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
+				int *write_type, u_int32_t *write_length, int flags, int (*)(buf_t, void *), void *callback_arg);
+static int cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF,
+				int *write_type, u_int32_t *write_length, int (*)(buf_t, void *), void *callback_arg, int bflag);
 
-static int	cluster_push_x(vnode_t vp, struct cl_extent *, off_t EOF, int flags);
-static void	cluster_push_EOF(vnode_t vp, off_t EOF);
+static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*)(buf_t, void *), void *callback_arg);
 
-static int	cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int can_delay, int push_all);
+static int 	cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
+static void	cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
 
-static void	sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF);
-static void	sparse_cluster_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_all);
-static void	sparse_cluster_add(struct cl_writebehind *, vnode_t vp, struct cl_extent *, off_t EOF);
+static int	cluster_push_now(vnode_t vp, struct cl_extent *, off_t EOF, int flags, int (*)(buf_t, void *), void *callback_arg);
 
-static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, int *setcountp);
+static int	cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
+
+static void	sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
+static void	sparse_cluster_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
+static void	sparse_cluster_add(struct cl_writebehind *, vnode_t vp, struct cl_extent *, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
+
+static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp);
 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
 
 int	is_file_clean(vnode_t, off_t);
 
+/*
+ * limit the internal I/O size so that we
+ * can represent it in a 32 bit int
+ */
+#define MAX_IO_REQUEST_SIZE	(1024 * 1024 * 256)
+#define MAX_IO_CONTIG_SIZE	(1024 * 1024 * 8)
+#define MAX_VECTS	16
+
+/*
+ * note:  MAX_CLUSTER_SIZE CANNOT be larger than MAX_UPL_TRANSFER
+ */
+#define MAX_CLUSTER_SIZE	(MAX_UPL_TRANSFER)
+#define MAX_PREFETCH		(MAX_CLUSTER_SIZE * PAGE_SIZE * 2)
+#define MIN_DIRECT_WRITE_SIZE	(4 * PAGE_SIZE)
+
+
+int speculative_reads_disabled = 0;
+
 /*
  * throttle the number of async writes that
  * can be outstanding on a single vnode
@@ -156,10 +209,10 @@ struct timeval priority_IO_timestamp_for_root;
 
 void
 cluster_init(void) {
-    /*
+        /*
 	 * allocate lock group attribute and group
 	 */
-    cl_mtx_grp_attr = lck_grp_attr_alloc_init();
+        cl_mtx_grp_attr = lck_grp_attr_alloc_init();
 	cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
 		
 	/*
@@ -181,6 +234,9 @@ cluster_init(void) {
 
 #define CLW_ALLOCATE		0x01
 #define CLW_RETURNLOCKED	0x02
+#define CLW_IONOCACHE		0x04
+#define CLW_IOPASSIVE	0x08
+
 /*
  * if the read ahead context doesn't yet exist,
  * allocate and initialize it...
@@ -220,7 +276,7 @@ cluster_get_rap(vnode_t vp)
 		else {
 		        lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
 		        FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
-				rap = ubc->cl_rahead;
+			rap = ubc->cl_rahead;
 		}
 		vnode_unlock(vp);
 	}
@@ -269,7 +325,7 @@ cluster_get_wbp(vnode_t vp, int flags)
 		else {
 		        lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
 		        FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
-				wbp = ubc->cl_wbehind;
+			wbp = ubc->cl_wbehind;
 		}
 		vnode_unlock(vp);
 	}
@@ -280,6 +336,24 @@ cluster_get_wbp(vnode_t vp, int flags)
 }
 
 
+static void
+cluster_syncup(vnode_t vp, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg)
+{
+	struct cl_writebehind *wbp;
+
+	if ((wbp = cluster_get_wbp(vp, 0)) != NULL) {
+	  
+	        if (wbp->cl_number) {
+		        lck_mtx_lock(&wbp->cl_lockw);
+
+			cluster_try_push(wbp, vp, newEOF, PUSH_ALL | PUSH_SYNC, callback, callback_arg);
+
+			lck_mtx_unlock(&wbp->cl_lockw);
+		}
+	}
+}
+
+
 static int 
 cluster_hard_throttle_on(vnode_t vp)
 {
@@ -302,7 +376,47 @@ cluster_hard_throttle_on(vnode_t vp)
 
 
 static int
-cluster_iodone(buf_t bp, __unused void *dummy)
+cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags)
+{
+        int upl_abort_code = 0;
+	int page_in  = 0;
+	int page_out = 0;
+
+	if (io_flags & B_PHYS)
+	        /*
+		 * direct write of any flavor, or a direct read that wasn't aligned
+		 */
+	        ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
+	else {
+	        if (io_flags & B_PAGEIO) {
+		        if (io_flags & B_READ)
+			        page_in  = 1;
+			else
+			        page_out = 1;
+		}
+		if (io_flags & B_CACHE)
+		        /*
+			 * leave pages in the cache unchanged on error
+			 */
+		        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
+		else if (page_out && (error != ENXIO))
+		        /*
+			 * transient error... leave pages unchanged
+			 */
+		        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
+		else if (page_in)
+		        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
+		else
+		        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
+
+		ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
+	}
+	return (upl_abort_code);
+}
+
+
+static int
+cluster_iodone(buf_t bp, void *callback_arg)
 {
         int	b_flags;
         int	error;
@@ -310,14 +424,17 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 	int	total_resid;
 	int	upl_offset;
 	int	zero_offset;
+	int	pg_offset = 0;
+        int	commit_size = 0;
+        int	upl_flags = 0;
+	int	transaction_size = 0;
 	upl_t	upl;
 	buf_t	cbp;
 	buf_t	cbp_head;
 	buf_t	cbp_next;
 	buf_t	real_bp;
 	struct	clios *iostate;
-	int	commit_size;
-	int	pg_offset;
+	boolean_t	transaction_complete = FALSE;
 
 	cbp_head = (buf_t)(bp->b_trans_head);
 
@@ -334,8 +451,16 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
 				     (int)cbp_head, (int)cbp, cbp->b_bcount, cbp->b_flags, 0);
 
-		        return 0;
+			return 0;
 		}
+		if (cbp->b_flags & B_EOT)
+		        transaction_complete = TRUE;
+	}
+	if (transaction_complete == FALSE) {
+	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
+			     (int)cbp_head, 0, 0, 0, 0);
+
+		return 0;
 	}
 	error       = 0;
 	total_size  = 0;
@@ -361,13 +486,37 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 
 		cbp_next = cbp->b_trans_next;
 
-		free_io_buf(cbp);
+		if (cbp_next == NULL)
+		        /*
+			 * compute the overall size of the transaction
+			 * in case we created one that has 'holes' in it
+			 * 'total_size' represents the amount of I/O we
+			 * did, not the span of the transaction w/r to the UPL
+			 */
+			transaction_size = cbp->b_uploffset + cbp->b_bcount - upl_offset;
+
+		if (cbp != cbp_head)
+		        free_io_buf(cbp);
 
 		cbp = cbp_next;
 	}
+	if (error == 0 && total_resid)
+		error = EIO;
+
+	if (error == 0) {
+	        int	(*cliodone_func)(buf_t, void *) = (int (*)(buf_t, void *))(cbp_head->b_cliodone);
+
+		if (cliodone_func != NULL) {
+		        cbp_head->b_bcount = transaction_size;
+
+		        error = (*cliodone_func)(cbp_head, callback_arg);
+		}
+	}
 	if (zero_offset)
 	        cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
 
+        free_io_buf(cbp_head);
+
 	if (iostate) {
 	        int need_wakeup = 0;
 
@@ -375,7 +524,7 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 		 * someone has issued multiple I/Os asynchrounsly
 		 * and is waiting for them to complete (streaming)
 		 */
-		lck_mtx_lock(cl_mtxp);
+		lck_mtx_lock_spin(cl_mtxp);
 
 	        if (error && iostate->io_error == 0)
 		        iostate->io_error = error;
@@ -395,69 +544,37 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 		if (need_wakeup)
 		        wakeup((caddr_t)&iostate->io_wanted);
 	}
-	if ((b_flags & B_NEED_IODONE) && real_bp) {
-		if (error) {
-		        real_bp->b_flags |= B_ERROR;
-			real_bp->b_error = error;
-		}
-		real_bp->b_resid = total_resid;
-
-		buf_biodone(real_bp);
-	}
-	if (error == 0 && total_resid)
-	        error = EIO;
 
 	if (b_flags & B_COMMIT_UPL) {
-	        pg_offset   = upl_offset & PAGE_MASK;
-		commit_size = (pg_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
-
-		if (error || (b_flags & B_NOCACHE)) {
-		        int upl_abort_code;
-			int page_in  = 0;
-			int page_out = 0;
-
-		        if (b_flags & B_PAGEIO) {
-			        if (b_flags & B_READ)
-				        page_in  = 1;
-				else
-				        page_out = 1;
-			}
-			if (b_flags & B_CACHE)		/* leave pages in the cache unchanged on error */
-			        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
-			else if (page_out && (error != ENXIO)) /* transient error */
-			        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
-			else if (page_in)
-			        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
-			else
-			        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
-
-			ubc_upl_abort_range(upl, upl_offset - pg_offset, commit_size,
-						    upl_abort_code);
 
-		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
-				     (int)upl, upl_offset - pg_offset, commit_size,
-				     0x80000000|upl_abort_code, 0);
+	        pg_offset   = upl_offset & PAGE_MASK;
+		commit_size = (pg_offset + transaction_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
 
-		} else {
-		        int upl_commit_flags = UPL_COMMIT_FREE_ON_EMPTY;
+		if (error)
+			upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags);
+		else {
+		        upl_flags = UPL_COMMIT_FREE_ON_EMPTY;
 
 			if ((b_flags & B_PHYS) && (b_flags & B_READ)) 
-			        upl_commit_flags |= UPL_COMMIT_SET_DIRTY;
+			        upl_flags |= UPL_COMMIT_SET_DIRTY;
 
 			if (b_flags & B_AGE)
-			        upl_commit_flags |= UPL_COMMIT_INACTIVATE;
-
-			ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size,
-					upl_commit_flags);
+			        upl_flags |= UPL_COMMIT_INACTIVATE;
 
-			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
-				     (int)upl, upl_offset - pg_offset, commit_size,
-				     upl_commit_flags, 0);
+			ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
 		}
-	} else {
-	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
-			     (int)upl, upl_offset, 0, error, 0);
 	}
+	if ((b_flags & B_NEED_IODONE) && real_bp) {
+		if (error) {
+			real_bp->b_flags |= B_ERROR;
+			real_bp->b_error = error;
+		}
+		real_bp->b_resid = total_resid;
+
+		buf_biodone(real_bp);
+	}
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
+		     (int)upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
 
 	return (error);
 }
@@ -466,31 +583,37 @@ cluster_iodone(buf_t bp, __unused void *dummy)
 void
 cluster_zero(upl_t upl, vm_offset_t upl_offset, int size, buf_t bp)
 {
-	upl_page_info_t *pl;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
 		     upl_offset, size, (int)bp, 0, 0);
 
 	if (bp == NULL || bp->b_datap == 0) {
+	        upl_page_info_t *pl;
+	        addr64_t	zero_addr;
 
 	        pl = ubc_upl_pageinfo(upl);
 
-	        while (size) {
-		        int           page_offset;
-			int           page_index;
-			addr64_t      zero_addr;
-			int           zero_cnt;
+		if (upl_device_page(pl) == TRUE) {
+		        zero_addr = ((addr64_t)upl_phys_page(pl, 0) << 12) + upl_offset;
+
+			bzero_phys_nc(zero_addr, size);
+		} else {
+		        while (size) {
+			        int	page_offset;
+				int	page_index;
+				int	zero_cnt;
 
-			page_index  = upl_offset / PAGE_SIZE;
-			page_offset = upl_offset & PAGE_MASK;
+				page_index  = upl_offset / PAGE_SIZE;
+				page_offset = upl_offset & PAGE_MASK;
 
-			zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
-			zero_cnt  = min(PAGE_SIZE - page_offset, size);
+				zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
+				zero_cnt  = min(PAGE_SIZE - page_offset, size);
 
-			bzero_phys(zero_addr, zero_cnt);
+				bzero_phys(zero_addr, zero_cnt);
 
-			size       -= zero_cnt;
-			upl_offset += zero_cnt;
+				size       -= zero_cnt;
+				upl_offset += zero_cnt;
+			}
 		}
 	} else
 		bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
@@ -500,9 +623,75 @@ cluster_zero(upl_t upl, vm_offset_t upl_offset, int size, buf_t bp)
 }
 
 
+static void
+cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset)
+{
+        cbp_head->b_validend = zero_offset;
+        cbp_tail->b_flags |= B_EOT;
+}
+
+static void
+cluster_wait_IO(buf_t cbp_head, int async)
+{
+        buf_t	cbp;
+
+	if (async) {
+	        /*
+		 * async callback completion will not normally
+		 * generate a wakeup upon I/O completion...
+		 * by setting BL_WANTED, we will force a wakeup
+		 * to occur as any outstanding I/Os complete... 
+		 * I/Os already completed will have BL_CALLDONE already
+		 * set and we won't block in buf_biowait_callback..
+		 * note that we're actually waiting for the bp to have
+		 * completed the callback function... only then
+		 * can we safely take back ownership of the bp
+		 * need the main buf mutex in order to safely
+		 * update b_lflags
+		 */
+	        buf_list_lock();
+
+		for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
+		      cbp->b_lflags |= BL_WANTED;
+
+		buf_list_unlock();
+	}
+	for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
+	        if (async)
+		        buf_biowait_callback(cbp);
+		else
+		        buf_biowait(cbp);
+	}
+}
+
+static void
+cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait)
+{
+        buf_t	cbp;
+	int	error;
+
+	/*
+	 * cluster_complete_transaction will
+	 * only be called if we've issued a complete chain in synchronous mode
+	 * or, we've already done a cluster_wait_IO on an incomplete chain
+	 */
+        if (needwait) {
+	        for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next)
+		        buf_biowait(cbp);
+	}
+	error = cluster_iodone(*cbp_head, callback_arg);
+
+	if ( !(flags & CL_ASYNC) && error && *retval == 0) {
+	        if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) != CL_PAGEOUT) || (error != ENXIO))
+		        *retval = error;
+	}
+	*cbp_head = (buf_t)NULL;
+}
+
+
 static int
 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
-	   int flags, buf_t real_bp, struct clios *iostate)
+	   int flags, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	buf_t	cbp;
 	u_int	size;
@@ -514,6 +703,7 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 	buf_t	cbp_head = NULL;
 	buf_t	cbp_tail = NULL;
 	int	trans_count = 0;
+	int	max_trans_count;
 	u_int	pg_count;
 	int	pg_offset;
 	u_int	max_iosize;
@@ -522,10 +712,27 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 	int	zero_offset = 0;
 	int	async_throttle = 0;
 	mount_t	mp;
+	vm_offset_t upl_end_offset;
+	boolean_t   need_EOT = FALSE;
+
+	/*
+	 * we currently don't support buffers larger than a page
+	 */
+	if (real_bp && non_rounded_size > PAGE_SIZE)
+		panic("%s(): Called with real buffer of size %d bytes which "
+				"is greater than the maximum allowed size of "
+				"%d bytes (the system PAGE_SIZE).\n",
+				__FUNCTION__, non_rounded_size, PAGE_SIZE);
 
 	mp = vp->v_mount;
 
-	if (mp->mnt_devblocksize > 1) {
+	/*
+	 * we don't want to do any funny rounding of the size for IO requests
+	 * coming through the DIRECT or CONTIGUOUS paths...  those pages don't
+	 * belong to us... we can't extend (nor do we need to) the I/O to fill
+	 * out a page
+	 */
+	if (mp->mnt_devblocksize > 1 && !(flags & (CL_DEV_MEMORY | CL_DIRECT_IO))) {
 	        /*
 		 * round the requested size up so that this I/O ends on a
 		 * page boundary in case this is a 'write'... if the filesystem
@@ -549,17 +756,26 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 		 */
 	        size = non_rounded_size;
 	}
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START,
-		     (int)f_offset, size, upl_offset, flags, 0);
+	upl_end_offset = upl_offset + size;
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START, (int)f_offset, size, upl_offset, flags, 0);
+
+	/*
+	 * Set the maximum transaction size to the maximum desired number of
+	 * buffers.
+	 */
+	max_trans_count = 8;
+	if (flags & CL_DEV_MEMORY)
+		max_trans_count = 16;
 
 	if (flags & CL_READ) {
-	        io_flags = (B_READ);
+	        io_flags = B_READ;
 		bmap_flags = VNODE_READ;
 
 		max_iosize  = mp->mnt_maxreadcnt;
 		max_vectors = mp->mnt_segreadcnt;
 	} else {
-	        io_flags = 0;
+	        io_flags = B_WRITE;
 		bmap_flags = VNODE_WRITE;
 
 		max_iosize  = mp->mnt_maxwritecnt;
@@ -573,18 +789,37 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 	 */
 	max_iosize  &= ~PAGE_MASK;
 
+	/*
+	 * Ensure the maximum iosize is sensible.
+	 */
+	if (!max_iosize)
+		max_iosize = PAGE_SIZE;
+
 	if (flags & CL_THROTTLE) {
 	        if ( !(flags & CL_PAGEOUT) && cluster_hard_throttle_on(vp)) {
 		        if (max_iosize > HARD_THROTTLE_MAXSIZE)
 			        max_iosize = HARD_THROTTLE_MAXSIZE;
 			async_throttle = HARD_THROTTLE_MAXCNT;
-		} else
-		        async_throttle = VNODE_ASYNC_THROTTLE;
+		} else {
+		        if ( (flags & CL_DEV_MEMORY) )
+			        async_throttle = VNODE_ASYNC_THROTTLE;
+			else {
+			        u_int max_cluster;
+				
+				if (max_iosize > (MAX_CLUSTER_SIZE * PAGE_SIZE))
+				        max_cluster = (MAX_CLUSTER_SIZE * PAGE_SIZE);
+				else
+				        max_cluster = max_iosize;
+
+				if (size < max_cluster)
+				        max_cluster = size;
+
+			        async_throttle = min(VNODE_ASYNC_THROTTLE, (MAX_PREFETCH / max_cluster) - 1);
+			}
+		}
 	}
 	if (flags & CL_AGE)
 	        io_flags |= B_AGE;
-	if (flags & CL_DUMP)
-	        io_flags |= B_NOCACHE;
 	if (flags & (CL_PAGEIN | CL_PAGEOUT))
 		io_flags |= B_PAGEIO;
 	if (flags & CL_COMMIT)
@@ -593,6 +828,10 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 	        io_flags |= B_PHYS;
 	if (flags & CL_KEEPCACHED)
 	        io_flags |= B_CACHE;
+	if (flags & CL_PASSIVE)
+	        io_flags |= B_PASSIVE;
+	if (vp->v_flag & VSYSTEM)
+	        io_flags |= B_META;
 
 	if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
 	        /*
@@ -605,23 +844,28 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 	        zero_offset = upl_offset + non_rounded_size;
 	}
 	while (size) {
-		int	pg_resid;
 		daddr64_t blkno;
 		daddr64_t lblkno;
+		u_int	io_size_wanted;
 
 		if (size > max_iosize)
 		        io_size = max_iosize;
 		else
 		        io_size = size;
+
+		io_size_wanted = io_size;
 		
-		if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, bmap_flags, NULL))) {
+		if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, bmap_flags, NULL)))
 			break;
-		}
+
+		if (io_size > io_size_wanted)
+		        io_size = io_size_wanted;
+
 		if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno))
 		        real_bp->b_blkno = blkno;
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
-			     (int)f_offset, (int)blkno, io_size, zero_offset, 0);
+			     (int)f_offset, (int)(blkno>>32), (int)blkno, io_size, 0);
 
 		if (io_size == 0) {
 		        /*
@@ -635,7 +879,8 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 			break;
 		}
 		if ( !(flags & CL_READ) && blkno == -1) {
-		        off_t e_offset;
+		        off_t	e_offset;
+			int	pageout_flags;
 
 		        /*
 			 * we're writing into a 'hole'
@@ -649,17 +894,6 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 		        	error = EINVAL;
 				break;
 			}
-			if ( !(flags & CL_COMMIT)) {
-			        /*
-				 * currently writes always request the commit to happen
-				 * as part of the io completion... however, if the CL_COMMIT
-				 * flag isn't specified, than we can't issue the abort_range
-				 * since the call site is going to abort or commit the same upl..
-				 * in this case we can only return an error
-				 */
-		        	error = EINVAL;
-				break;
-			}
 			/*
 			 * we can get here if the cluster code happens to 
 			 * pick up a page that was dirtied via mmap vs
@@ -678,15 +912,84 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 			 * giving up entirely and dumping it (the pageout
 			 * path will insure that the zero extent accounting
 			 * has been taken care of before we get back into cluster_io)
+			 *
+			 * go direct to vnode_pageout so that we don't have to
+			 * unbusy the page from the UPL... we used to do this
+			 * so that we could call ubc_sync_range, but that results
+			 * in a potential deadlock if someone else races us to acquire
+			 * that page and wins and in addition needs one of the pages
+			 * we're continuing to hold in the UPL
 			 */
-			ubc_upl_abort_range(upl, trunc_page(upl_offset), PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
+			pageout_flags = UPL_MSYNC | UPL_VNODE_PAGER | UPL_NESTED_PAGEOUT;
 
-			e_offset = round_page_64(f_offset + 1);
+			if ( !(flags & CL_ASYNC))
+			        pageout_flags |= UPL_IOSYNC;
+			if ( !(flags & CL_COMMIT))
+			        pageout_flags |= UPL_NOCOMMIT;
+
+			if (cbp_head) {
+			        buf_t last_cbp;
+
+				/*
+				 * first we have to wait for the the current outstanding I/Os
+				 * to complete... EOT hasn't been set yet on this transaction
+				 * so the pages won't be released just because all of the current
+				 * I/O linked to this transaction has completed...
+				 */
+				cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
+
+			        /*
+				 * we've got a transcation that
+				 * includes the page we're about to push out through vnode_pageout...
+				 * find the last bp in the list which will be the one that
+				 * includes the head of this page and round it's iosize down
+				 * to a page boundary...
+				 */
+                                for (last_cbp = cbp = cbp_head; cbp->b_trans_next; cbp = cbp->b_trans_next)
+				        last_cbp = cbp;
+
+				cbp->b_bcount &= ~PAGE_MASK;
+
+				if (cbp->b_bcount == 0) {
+				        /*
+					 * this buf no longer has any I/O associated with it
+					 */
+				        free_io_buf(cbp);
+
+					if (cbp == cbp_head) {
+					        /*
+						 * the buf we just freed was the only buf in
+						 * this transaction... so there's no I/O to do
+						 */
+					        cbp_head = NULL;
+					} else {
+					        /*
+						 * remove the buf we just freed from
+						 * the transaction list
+						 */
+					        last_cbp->b_trans_next = NULL;
+						cbp_tail = last_cbp;
+					}
+				}
+				if (cbp_head) {
+				        /*
+					 * there was more to the current transaction
+					 * than just the page we are pushing out via vnode_pageout...
+					 * mark it as finished and complete it... we've already
+					 * waited for the I/Os to complete above in the call to cluster_wait_IO
+					 */
+				        cluster_EOT(cbp_head, cbp_tail, 0);
+
+					cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
 
-			if (ubc_sync_range(vp, f_offset, e_offset, UBC_PUSHDIRTY) == 0) {
+					trans_count = 0;
+				}
+			}
+			if (vnode_pageout(vp, upl, trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
 			        error = EINVAL;
 			 	break;
 			}
+			e_offset = round_page_64(f_offset + 1);
 			io_size = e_offset - f_offset;
 
 			f_offset   += io_size;
@@ -724,13 +1027,6 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 		pg_offset = upl_offset & PAGE_MASK;
 
 		if (flags & CL_DEV_MEMORY) {
-		        /*
-			 * currently, can't deal with reading 'holes' in file
-			 */
-		        if (blkno == -1) {
-			        error = EINVAL;
-				break;
-			}
 			/*
 			 * treat physical requests as one 'giant' page
 			 */
@@ -739,14 +1035,16 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 		        pg_count  = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
 
 		if ((flags & CL_READ) && blkno == -1) {
+			vm_offset_t  commit_offset;
 		        int bytes_to_zero;
+			int complete_transaction_now = 0;
 
 		        /*
 			 * if we're reading and blkno == -1, then we've got a
 			 * 'hole' in the file that we need to deal with by zeroing
 			 * out the affected area in the upl
 			 */
-		        if (zero_offset && io_size == size) {
+			if (io_size >= (u_int)non_rounded_size) {
 			        /*
 				 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
 				 * than 'zero_offset' will be non-zero
@@ -756,56 +1054,73 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 				 * last page in this upl... we need to zero both the hole and the tail
 				 * of the page beyond the EOF, since the delayed zero-fill won't kick in 
 				 */
-			        bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
+				bytes_to_zero = non_rounded_size;
+				if (!(flags & CL_NOZERO))
+					bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
 
 				zero_offset = 0;
 			} else
 			        bytes_to_zero = io_size;
 
-		        cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
+			pg_count = 0;
+
+			cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
 			  
-			if (cbp_head)
+			if (cbp_head) {
+			        int	pg_resid;
+
 			        /*
 				 * if there is a current I/O chain pending
 				 * then the first page of the group we just zero'd
 				 * will be handled by the I/O completion if the zero
 				 * fill started in the middle of the page
 				 */
-			        pg_count = (io_size - pg_offset) / PAGE_SIZE;
-			else {
-			        /*
-				 * no pending I/O to pick up that first page
-				 * so, we have to make sure it gets committed
-				 * here.
-				 * set the pg_offset to 0 so that the upl_commit_range
-				 * starts with this page
-				 */
-			        pg_count = (io_size + pg_offset) / PAGE_SIZE;
-				pg_offset = 0;
-			}
-			if (io_size == size && ((upl_offset + io_size) & PAGE_MASK))
+			        commit_offset = (upl_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
+
+				pg_resid = commit_offset - upl_offset;
+					
+				if (bytes_to_zero >= pg_resid) {
+				        /*
+					 * the last page of the current I/O 
+					 * has been completed...
+					 * compute the number of fully zero'd 
+					 * pages that are beyond it
+					 * plus the last page if its partial
+					 * and we have no more I/O to issue...
+					 * otherwise a partial page is left
+					 * to begin the next I/O
+					 */
+				        if ((int)io_size >= non_rounded_size)
+					        pg_count = (bytes_to_zero - pg_resid + (PAGE_SIZE - 1)) / PAGE_SIZE;
+					else
+					        pg_count = (bytes_to_zero - pg_resid) / PAGE_SIZE;
+					
+					complete_transaction_now = 1;
+				}
+			} else {
 			        /*
-				 * if we're done with the request for this UPL
-				 * then we have to make sure to commit the last page
-				 * even if we only partially zero-filled it
+				 * no pending I/O to deal with
+				 * so, commit all of the fully zero'd pages
+				 * plus the last page if its partial
+				 * and we have no more I/O to issue...
+				 * otherwise a partial page is left
+				 * to begin the next I/O
 				 */
-			        pg_count++;
-
-			if (pg_count) {
-			        if (pg_offset)
-				        pg_resid = PAGE_SIZE - pg_offset;
+			        if ((int)io_size >= non_rounded_size)
+				        pg_count = (pg_offset + bytes_to_zero + (PAGE_SIZE - 1)) / PAGE_SIZE;
 				else
-				        pg_resid = 0;
+				        pg_count = (pg_offset + bytes_to_zero) / PAGE_SIZE;
 
-				if (flags & CL_COMMIT)
-				        ubc_upl_commit_range(upl,
-							(upl_offset + pg_resid) & ~PAGE_MASK, 
-							pg_count * PAGE_SIZE,
-							UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
+				commit_offset = upl_offset & ~PAGE_MASK;
+			}
+			if ( (flags & CL_COMMIT) && pg_count) {
+			        ubc_upl_commit_range(upl, commit_offset, pg_count * PAGE_SIZE,
+						     UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
 			}
 			upl_offset += io_size;
 			f_offset   += io_size;
 			size       -= io_size;
+
 			/*
 			 * keep track of how much of the original request
 			 * that we've actually completed... non_rounded_size
@@ -823,10 +1138,16 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 				 */
 			        size = 0;
 			}
-			if (cbp_head && pg_count) 
-			        goto start_io;
-			continue;
+			if (cbp_head && (complete_transaction_now || size == 0))  {
+			        cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
+
+				cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
+
+				cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
 
+				trans_count = 0;
+			}
+			continue;
 		}
 		if (pg_count > max_vectors) {
 		        if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
@@ -837,6 +1158,35 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 			        pg_count = max_vectors;
 			}
 		}
+		/*
+		 * If the transaction is going to reach the maximum number of
+		 * desired elements, truncate the i/o to the nearest page so
+		 * that the actual i/o is initiated after this buffer is
+		 * created and added to the i/o chain.
+		 *
+		 * I/O directed to physically contiguous memory 
+		 * doesn't have a requirement to make sure we 'fill' a page
+		 */
+		if ( !(flags & CL_DEV_MEMORY) && trans_count >= max_trans_count &&
+				((upl_offset + io_size) & PAGE_MASK)) {
+			vm_offset_t aligned_ofs;
+
+			aligned_ofs = (upl_offset + io_size) & ~PAGE_MASK;
+			/*
+			 * If the io_size does not actually finish off even a
+			 * single page we have to keep adding buffers to the
+			 * transaction despite having reached the desired limit.
+			 *
+			 * Eventually we get here with the page being finished
+			 * off (and exceeded) and then we truncate the size of
+			 * this i/o request so that it is page aligned so that
+			 * we can finally issue the i/o on the transaction.
+			 */
+			if (aligned_ofs > upl_offset) {
+				io_size = aligned_ofs - upl_offset;
+				pg_count--;
+			}
+		}
 
 		if ( !(mp->mnt_kern_flag & MNTK_VIRTUALDEV))
 		        /*
@@ -865,9 +1215,10 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 			}
 		}
 		if (flags & CL_ASYNC) {
-		        if (buf_setcallback(cbp, (void *)cluster_iodone, NULL))
+		        if (buf_setcallback(cbp, (void *)cluster_iodone, callback_arg))
 			        panic("buf_setcallback failed\n");
 		}
+		cbp->b_cliodone = (void *)callback;
 		cbp->b_flags |= io_flags;
 
 		cbp->b_lblkno = lblkno;
@@ -901,8 +1252,14 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 		} else {
 		        cbp_head = cbp;
 			cbp_tail = cbp;
+
+			if ( (cbp_head->b_real_bp = real_bp) ) {
+			        cbp_head->b_flags |= B_NEED_IODONE;
+				real_bp = (buf_t)NULL;
+			}
 		}
-		(buf_t)(cbp->b_trans_head) = cbp_head;
+		*(buf_t *)(&cbp->b_trans_head) = cbp_head;
+
 		trans_count++;
 
 		upl_offset += io_size;
@@ -925,86 +1282,77 @@ cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int no
 			 */
 		        size = 0;
 		}
-		if ( (!(upl_offset & PAGE_MASK) && !(flags & CL_DEV_MEMORY) && ((flags & CL_ASYNC) || trans_count > 8)) || size == 0) {
+		if (size == 0) {
+		        /*
+			 * we have no more I/O to issue, so go
+			 * finish the final transaction
+			 */
+		        need_EOT = TRUE;
+		} else if ( ((flags & CL_DEV_MEMORY) || (upl_offset & PAGE_MASK) == 0) &&
+			    ((flags & CL_ASYNC) || trans_count > max_trans_count) ) {
 		        /*
-			 * if we have no more I/O to issue or
+			 * I/O directed to physically contiguous memory...
+			 * which doesn't have a requirement to make sure we 'fill' a page
+			 * or... 
 			 * the current I/O we've prepared fully
 			 * completes the last page in this request
-			 * and it's either an ASYNC request or 
+			 * and ...
+			 * it's either an ASYNC request or 
 			 * we've already accumulated more than 8 I/O's into
-			 * this transaction and it's not an I/O directed to 
-			 * special DEVICE memory
-			 * then go ahead and issue the I/O
+			 * this transaction so mark it as complete so that
+			 * it can finish asynchronously or via the cluster_complete_transaction
+			 * below if the request is synchronous
 			 */
-start_io:		
-			if (real_bp) {
-			        cbp_head->b_flags |= B_NEED_IODONE;
-				cbp_head->b_real_bp = real_bp;
-			} else
-			        cbp_head->b_real_bp = (buf_t)NULL;
-
-			if (size == 0) {
-			        /*
-				 * we're about to issue the last I/O for this upl
-				 * if this was a read to the eof and the eof doesn't
-				 * finish on a page boundary, than we need to zero-fill
-				 * the rest of the page....
-				 */
-			        cbp_head->b_validend = zero_offset;
-			} else
-			        cbp_head->b_validend = 0;
-			  
-			if (flags & CL_THROTTLE)
-			        (void)vnode_waitforwrites(vp, async_throttle, 0, 0, (char *)"cluster_io");
-			  
-		        for (cbp = cbp_head; cbp;) {
-				buf_t	cbp_next;
+		        need_EOT = TRUE;
+		}
+		if (need_EOT == TRUE)
+		        cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
 
-			        if ( !(io_flags & B_READ))
-				        vnode_startwrite(vp);
+		if (flags & CL_THROTTLE)
+		        (void)vnode_waitforwrites(vp, async_throttle, 0, 0, "cluster_io");
 
-				cbp_next = cbp->b_trans_next;
+		if ( !(io_flags & B_READ))
+		        vnode_startwrite(vp);
 				
-				(void) VNOP_STRATEGY(cbp);
-				cbp = cbp_next;
-			}
-			if ( !(flags & CL_ASYNC)) {
-			        int dummy;
-
-			        for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
-				        buf_biowait(cbp);
-
-				if ((error = cluster_iodone(cbp_head, (void *)&dummy))) {
-				        if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) == CL_PAGEOUT) && (error == ENXIO))
-						error = 0;	/* drop the error */
-					else {
-					        if (retval == 0)
-						        retval = error;
-						error = 0;
-					}
-				}
-			}
-			cbp_head = (buf_t)NULL;
-			cbp_tail = (buf_t)NULL;
+		(void) VNOP_STRATEGY(cbp);
+
+		if (need_EOT == TRUE) {
+		        if ( !(flags & CL_ASYNC))
+			        cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 1);
 
+			need_EOT = FALSE;
 			trans_count = 0;
+			cbp_head = NULL;
 		}
-	}
+        }
 	if (error) {
 	        int abort_size;
 
 		io_size = 0;
 		
-	        for (cbp = cbp_head; cbp;) {
-			buf_t	cbp_next;
- 
-			upl_offset -= cbp->b_bcount;
-			size       += cbp->b_bcount;
-			io_size    += cbp->b_bcount;
+		if (cbp_head) {
+		         /*
+			  * first wait until all of the outstanding I/O
+			  * for this partial transaction has completed
+			  */
+		        cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
 
-			cbp_next = cbp->b_trans_next;
-			free_io_buf(cbp);
-			cbp = cbp_next;
+			/*
+			 * Rewind the upl offset to the beginning of the
+			 * transaction.
+			 */
+			upl_offset = cbp_head->b_uploffset;
+
+			for (cbp = cbp_head; cbp;) {
+				buf_t	cbp_next;
+	 
+				size       += cbp->b_bcount;
+				io_size    += cbp->b_bcount;
+
+				cbp_next = cbp->b_trans_next;
+				free_io_buf(cbp);
+				cbp = cbp_next;
+			}
 		}
 		if (iostate) {
 		        int need_wakeup = 0;
@@ -1014,7 +1362,7 @@ start_io:
 			 * since we never really issued the io
 			 * just go ahead and adjust it back
 			 */
-		        lck_mtx_lock(cl_mtxp);
+		        lck_mtx_lock_spin(cl_mtxp);
 
 		        if (iostate->io_error == 0)
 			        iostate->io_error = error;
@@ -1026,54 +1374,49 @@ start_io:
 				 * this io stream to change
 				 */
 			        iostate->io_wanted = 0;
-				need_wakeup = 0;
+				need_wakeup = 1;
 			}
 		        lck_mtx_unlock(cl_mtxp);
 
 			if (need_wakeup)
 			        wakeup((caddr_t)&iostate->io_wanted);
 		}
-		pg_offset  = upl_offset & PAGE_MASK;
-		abort_size = (size + pg_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
-
 		if (flags & CL_COMMIT) {
-		        int upl_abort_code;
-
-			if (flags & CL_PRESERVE) {
-			        ubc_upl_commit_range(upl, upl_offset - pg_offset, abort_size,
-						     UPL_COMMIT_FREE_ON_EMPTY);
-			} else {
-			        if ((flags & CL_PAGEOUT) && (error != ENXIO)) /* transient error */
-				        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
-				else if (flags & CL_PAGEIN)
-				        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
-				else
-				        upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
+		        int	upl_flags;
 
-				ubc_upl_abort_range(upl, upl_offset - pg_offset, abort_size,
-						upl_abort_code);
-			}
+		        pg_offset  = upl_offset & PAGE_MASK;
+			abort_size = (upl_end_offset - upl_offset + PAGE_MASK) & ~PAGE_MASK;
+			
+			upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, abort_size, error, io_flags);
+			
 			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
-				     (int)upl, upl_offset - pg_offset, abort_size, error, 0);
-		}
-		if (real_bp) {
-		        real_bp->b_flags |= B_ERROR;
-			real_bp->b_error  = error;
-
-			buf_biodone(real_bp);
+				     (int)upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
 		}
 		if (retval == 0)
 		        retval = error;
+	} else if (cbp_head)
+			panic("%s(): cbp_head is not NULL.\n", __FUNCTION__);
+
+	if (real_bp) {
+	        /*
+		 * can get here if we either encountered an error
+		 * or we completely zero-filled the request and
+		 * no I/O was issued
+		 */
+		if (error) {
+			real_bp->b_flags |= B_ERROR;
+			real_bp->b_error = error;
+		}
+		buf_biodone(real_bp);
 	}
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END,
-		     (int)f_offset, size, upl_offset, retval, 0);
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END, (int)f_offset, size, upl_offset, retval, 0);
 
 	return (retval);
 }
 
 
 static int
-cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize)
+cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
 {
 	int           pages_in_prefetch;
 
@@ -1085,16 +1428,11 @@ cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize)
 			     (int)f_offset, 0, 0, 0, 0);
 	        return(0);
 	}
-	if (size > (MAX_UPL_TRANSFER * PAGE_SIZE))
-	        size = (MAX_UPL_TRANSFER * PAGE_SIZE);
-	else
-	        size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
-
         if ((off_t)size > (filesize - f_offset))
                 size = filesize - f_offset;
 	pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
 
-	advisory_read(vp, filesize, f_offset, size);
+	advisory_read_ext(vp, filesize, f_offset, size, callback, callback_arg, bflag);
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
 		     (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
@@ -1105,7 +1443,8 @@ cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize)
 
 
 static void
-cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap)
+cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap, int (*callback)(buf_t, void *), void *callback_arg,
+		   int bflag)
 {
 	daddr64_t	r_addr;
 	off_t		f_offset;
@@ -1120,8 +1459,7 @@ cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl
 			     rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
 		return;
 	}
-	if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1) &&
-				   (extent->b_addr != (rap->cl_maxra + 1) || rap->cl_ralen == 0))) {
+	if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1))) {
 	        rap->cl_ralen = 0;
 		rap->cl_maxra = 0;
 
@@ -1131,7 +1469,7 @@ cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl
 		return;
 	}
 	if (extent->e_addr < rap->cl_maxra) {
-	        if ((rap->cl_maxra - extent->e_addr) > (MAX_UPL_TRANSFER / 4)) {
+	        if ((rap->cl_maxra - extent->e_addr) > ((MAX_PREFETCH / PAGE_SIZE) / 4)) {
 
 		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
 				     rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
@@ -1153,17 +1491,17 @@ cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl
 	if (f_offset < filesize) {
 	        daddr64_t read_size;
 
-	        rap->cl_ralen = rap->cl_ralen ? min(MAX_UPL_TRANSFER, rap->cl_ralen << 1) : 1;
+	        rap->cl_ralen = rap->cl_ralen ? min(MAX_PREFETCH / PAGE_SIZE, rap->cl_ralen << 1) : 1;
 
 		read_size = (extent->e_addr + 1) - extent->b_addr;
 
 		if (read_size > rap->cl_ralen) {
-		        if (read_size > MAX_UPL_TRANSFER)
-			        rap->cl_ralen = MAX_UPL_TRANSFER;
+		        if (read_size > MAX_PREFETCH / PAGE_SIZE)
+			        rap->cl_ralen = MAX_PREFETCH / PAGE_SIZE;
 			else
 			        rap->cl_ralen = read_size;
 		}
-		size_of_prefetch = cluster_rd_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize);
+		size_of_prefetch = cluster_read_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize, callback, callback_arg, bflag);
 
 		if (size_of_prefetch)
 		        rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
@@ -1172,15 +1510,24 @@ cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl
 		     rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
 }
 
+
 int
 cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 		int size, off_t filesize, int flags)
+{
+        return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
+
+}
+
+
+int
+cluster_pageout_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
+		int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	int           io_size;
 	int           rounded_size;
         off_t         max_size;
 	int           local_flags;
-	struct cl_writebehind *wbp;
 
 	if (vp->v_mount->mnt_kern_flag & MNTK_VIRTUALDEV)
 	        /*
@@ -1201,6 +1548,8 @@ cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 		local_flags |= CL_COMMIT;
 	if ((flags & UPL_KEEPCACHED))
 	        local_flags |= CL_KEEPCACHED;
+	if (flags & IO_PASSIVE) 
+		local_flags |= CL_PASSIVE;
 
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
@@ -1245,16 +1594,22 @@ cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 			ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
 					UPL_ABORT_FREE_ON_EMPTY);
 	}
-	if ((wbp = cluster_get_wbp(vp, 0)) != NULL)
-	        wbp->cl_hasbeenpaged = 1;
-
 	return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
-			   local_flags, (buf_t)NULL, (struct clios *)NULL));
+			   local_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg));
 }
 
+
 int
 cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 	       int size, off_t filesize, int flags)
+{
+        return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
+}
+
+
+int
+cluster_pagein_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
+	       int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	u_int         io_size;
 	int           rounded_size;
@@ -1269,6 +1624,8 @@ cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 	        local_flags |= CL_ASYNC;
 	if ((flags & UPL_NOCOMMIT) == 0) 
 		local_flags |= CL_COMMIT;
+	if (flags & IO_PASSIVE) 
+		local_flags |= CL_PASSIVE;
 
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
@@ -1300,37 +1657,21 @@ cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
 				    size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
 	
 	retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
-			   local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL);
-
-	if (retval == 0 && !(flags & UPL_NORDAHEAD) && !(vp->v_flag & VRAOFF)) {
-		struct cl_readahead *rap;
-
-		rap = cluster_get_rap(vp);
-
-		if (rap != NULL) {
-		        struct cl_extent extent;
-
-		        extent.b_addr = (daddr64_t)(f_offset / PAGE_SIZE_64);
-			extent.e_addr = (daddr64_t)((f_offset + ((off_t)io_size - 1)) / PAGE_SIZE_64);
+			    local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 
-			if (rounded_size == PAGE_SIZE) {
-			        /*
-				 * we haven't read the last page in of the file yet
-				 * so let's try to read ahead if we're in 
-				 * a sequential access pattern
-				 */
-			        cluster_rd_ahead(vp, &extent, filesize, rap);
-			}
-			rap->cl_lastr = extent.e_addr;
-
-			lck_mtx_unlock(&rap->cl_lockr);
-		}
-	}
 	return (retval);
 }
 
+
 int
 cluster_bp(buf_t bp)
+{
+       return cluster_bp_ext(bp, NULL, NULL);
+}
+
+
+int
+cluster_bp_ext(buf_t bp, int (*callback)(buf_t, void *), void *callback_arg)
 {
         off_t  f_offset;
 	int    flags;
@@ -1342,244 +1683,245 @@ cluster_bp(buf_t bp)
 	        flags = CL_ASYNC | CL_READ;
 	else
 	        flags = CL_ASYNC;
+	if (bp->b_flags & B_PASSIVE) 
+		flags |= CL_PASSIVE;
 
 	f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
 
-        return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL));
+        return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL, callback, callback_arg));
 }
 
+
+
 int
 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
 {
-	int           prev_resid;
-	u_int         clip_size;
-	off_t         max_io_size;
-	int           upl_size;
-	int           upl_flags;
-	upl_t         upl;
-	int           retval = 0;
-	int	      flags;
+        return cluster_write_ext(vp, uio, oldEOF, newEOF, headOff, tailOff, xflags, NULL, NULL);
+}
+
+
+int
+cluster_write_ext(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff,
+		  int xflags, int (*callback)(buf_t, void *), void *callback_arg)
+{
+        user_ssize_t	cur_resid;
+	int		retval = 0;
+	int		flags;
+        int		zflags;
+	int             bflag;
+	int		write_type = IO_COPY;
+	u_int32_t	write_length;
 
 	flags = xflags;
 
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
+
 	if (vp->v_flag & VNOCACHE_DATA)
 	        flags |= IO_NOCACHE;
 
-	if ( (!(flags & IO_NOCACHE)) || (!uio) || (!UIO_SEG_IS_USER_SPACE(uio->uio_segflg))) {
+        if (uio == NULL) {
 	        /*
-		 * go do a write through the cache if one of the following is true....
-		 *   NOCACHE is not true
-		 *   there is no uio structure or it doesn't target USERSPACE
+		 * no user data...
+		 * this call is being made to zero-fill some range in the file
 		 */
-	        return (cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags));
-	}
+	        retval = cluster_write_copy(vp, NULL, (u_int32_t)0, oldEOF, newEOF, headOff, tailOff, flags, callback, callback_arg);
 
-#if LP64_DEBUG
-	if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
-		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); 
+		return(retval);
 	}
-#endif /* LP64_DEBUG */
-	
-	while (uio_resid(uio) && uio->uio_offset < newEOF && retval == 0) {
-	  	user_size_t	iov_len;
-	  	user_addr_t	iov_base;
-
-		/*
-		 * we know we have a resid, so this is safe
-		 * skip over any emtpy vectors
-		 */
-		uio_update(uio, (user_size_t)0);
-
-		iov_len  = uio_curriovlen(uio);
-		iov_base = uio_curriovbase(uio);
-
-		upl_size  = PAGE_SIZE;
-		upl_flags = UPL_QUERY_OBJECT_TYPE;
-
-		// LP64todo - fix this!
-		if ((vm_map_get_upl(current_map(),
-				    (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
-				    &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
-		        /*
-			 * the user app must have passed in an invalid address
-			 */
-		        return (EFAULT);
-		}	      
-
-		/*
-		 * We check every vector target but if it is physically
-		 * contiguous space, we skip the sanity checks.
+        /*
+         * do a write through the cache if one of the following is true....
+         *   NOCACHE is not true and
+         *   the uio request doesn't target USERSPACE
+         * otherwise, find out if we want the direct or contig variant for
+         * the first vector in the uio request
+         */
+        if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
+	        retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
+
+        if ( (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL)) && write_type == IO_DIRECT)
+	        /*
+		 * must go through the cached variant in this case
 		 */
-		if (upl_flags & UPL_PHYS_CONTIG) {
-		        int zflags;
+	        write_type = IO_COPY;
 
-			zflags = flags & ~IO_TAILZEROFILL;
-			zflags |= IO_HEADZEROFILL;
-
-		        if (flags & IO_HEADZEROFILL) {
-			        /*
-				 * in case we have additional vectors, we don't want to do this again
-				 */
-			        flags &= ~IO_HEADZEROFILL;
-
-				if ((retval = cluster_write_x(vp, (struct uio *)0, 0, uio->uio_offset, headOff, 0, zflags)))
-				        return(retval);
-			}
-			retval = cluster_phys_write(vp, uio, newEOF);
+	while ((cur_resid = uio_resid(uio)) && uio->uio_offset < newEOF && retval == 0) {
+	  
+	        switch (write_type) {
 
-			if (uio_resid(uio) == 0 && (flags & IO_TAILZEROFILL)) {
-			        return (cluster_write_x(vp, (struct uio *)0, 0, tailOff, uio->uio_offset, 0, zflags));
-			}
-		}
-		else if ((uio_resid(uio) < PAGE_SIZE) || (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL))) {
+		case IO_COPY:
 		        /*
-			 * we're here because we're don't have a physically contiguous target buffer
-			 * go do a write through the cache if one of the following is true....
-			 *   the total xfer size is less than a page...
-			 *   we're being asked to ZEROFILL either the head or the tail of the I/O...
+			 * make sure the uio_resid isn't too big...
+			 * internally, we want to handle all of the I/O in
+			 * chunk sizes that fit in a 32 bit int
 			 */
-		        return (cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags));
-		}
-		// LP64todo - fix this!
-		else if (((int)uio->uio_offset & PAGE_MASK) || (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
-		        if (((int)uio->uio_offset & PAGE_MASK) == (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
+		        if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
 			        /*
-				 * Bring the file offset write up to a pagesize boundary
-				 * this will also bring the base address to a page boundary
-				 * since they both are currently on the same offset within a page
-				 * note: if we get here, uio->uio_resid is greater than PAGE_SIZE
-				 * so the computed clip_size must always be less than the current uio_resid
+				 * we're going to have to call cluster_write_copy
+				 * more than once...
+				 *
+				 * only want the last call to cluster_write_copy to
+				 * have the IO_TAILZEROFILL flag set and only the
+				 * first call should have IO_HEADZEROFILL
 				 */
-			        clip_size = (PAGE_SIZE - (uio->uio_offset & PAGE_MASK_64));
+			        zflags = flags & ~IO_TAILZEROFILL;
+				flags &= ~IO_HEADZEROFILL;
 
-				/* 
-				 * Fake the resid going into the cluster_write_x call
-				 * and restore it on the way out.
+				write_length = MAX_IO_REQUEST_SIZE;
+			} else {
+		                /*
+				 * last call to cluster_write_copy
 				 */
-				// LP64todo - fix this
-				prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
+			        zflags = flags;
+			  
+				write_length = (u_int32_t)cur_resid;
+			}
+			retval = cluster_write_copy(vp, uio, write_length, oldEOF, newEOF, headOff, tailOff, zflags, callback, callback_arg);
+			break;
 
-				retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
+		case IO_CONTIG:
+		        zflags = flags & ~(IO_TAILZEROFILL | IO_HEADZEROFILL);
 
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-			} else {
-			        /*
-				 * can't get both the file offset and the buffer offset aligned to a page boundary
-				 * so fire an I/O through the cache for this entire vector
+			if (flags & IO_HEADZEROFILL) {
+		                /*
+				 * only do this once per request
 				 */
-			        // LP64todo - fix this
-			        clip_size = iov_len;
-				// LP64todo - fix this
-				prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
+		                flags &= ~IO_HEADZEROFILL;
 
-				retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
-
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
+				retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, uio->uio_offset,
+							    headOff, (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
+				if (retval)
+			                break;
 			}
-		} else {
-		        /* 
-			 * If we come in here, we know the offset into
-			 * the file is on a pagesize boundary and the
-			 * target buffer address is also on a page boundary
-			 */
-		        max_io_size = newEOF - uio->uio_offset;
-			// LP64todo - fix this
-			clip_size = uio_resid(uio);
-			if (iov_len < clip_size)
-			        // LP64todo - fix this!
-			        clip_size = iov_len;
-			if (max_io_size < clip_size)
-			        clip_size = max_io_size;
-
-			if (clip_size < PAGE_SIZE) {
-			        /*
-				 * Take care of tail end of write in this vector
+			retval = cluster_write_contig(vp, uio, newEOF, &write_type, &write_length, callback, callback_arg, bflag);
+
+			if (retval == 0 && (flags & IO_TAILZEROFILL) && uio_resid(uio) == 0) {
+		                /*
+				 * we're done with the data from the user specified buffer(s)
+				 * and we've been requested to zero fill at the tail
+				 * treat this as an IO_HEADZEROFILL which doesn't require a uio
+				 * by rearranging the args and passing in IO_HEADZEROFILL
 				 */
-			        // LP64todo - fix this
-			        prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
-
-				retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
-
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-			} else {
-			        /* round clip_size down to a multiple of pagesize */
-			        clip_size = clip_size & ~(PAGE_MASK);
-				// LP64todo - fix this
-				prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
-
-				retval = cluster_nocopy_write(vp, uio, newEOF);
-
-				if ((retval == 0) && uio_resid(uio))
-				        retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
-
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
+		                retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, tailOff, uio->uio_offset,
+							    (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
 			}
-		} /* end else */
-	} /* end while */
+			break;
 
-	return(retval);
+		case IO_DIRECT:
+			/*
+			 * cluster_write_direct is never called with IO_TAILZEROFILL || IO_HEADZEROFILL
+			 */
+			retval = cluster_write_direct(vp, uio, oldEOF, newEOF, &write_type, &write_length, flags, callback, callback_arg);
+			break;
+
+		case IO_UNKNOWN:
+		        retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
+			break;
+		}
+	}
+	return (retval);
 }
 
 
 static int
-cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
+cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
+		     int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	upl_t            upl;
 	upl_page_info_t  *pl;
 	vm_offset_t      upl_offset;
+	u_int32_t	 io_req_size;
+	u_int32_t	 offset_in_file;
+	u_int32_t	 offset_in_iovbase;
 	int              io_size;
 	int              io_flag;
-	int              upl_size;
-	int              upl_needed_size;
-	int              pages_in_pl;
+	int              bflag;
+	vm_size_t	 upl_size;
+	vm_size_t	 upl_needed_size;
+	mach_msg_type_number_t	pages_in_pl;
 	int              upl_flags;
 	kern_return_t    kret;
-	int              i;
+	mach_msg_type_number_t	i;
 	int              force_data_sync;
-	int              error  = 0;
+	int              retval = 0;
+	int		 first_IO = 1;
 	struct clios     iostate;
-	struct cl_writebehind *wbp;
-
+	user_addr_t	 iov_base;
+	u_int32_t	 mem_alignment_mask;
+	u_int32_t	 devblocksize;
 
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
-		     (int)uio->uio_offset, (int)uio_resid(uio), 
-		     (int)newEOF, 0, 0);
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
 	/*
 	 * When we enter this routine, we know
-	 *  -- the offset into the file is on a pagesize boundary
-	 *  -- the resid is a page multiple
 	 *  -- the resid will not exceed iov_len
 	 */
-	
-	if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) != NULL) {
-
-		cluster_try_push(wbp, vp, newEOF, 0, 1);
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
+		     (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
 
-	        lck_mtx_unlock(&wbp->cl_lockw);
-	}
 	iostate.io_completed = 0;
 	iostate.io_issued = 0;
 	iostate.io_error = 0;
 	iostate.io_wanted = 0;
 
-	while (uio_resid(uio) && uio->uio_offset < newEOF && error == 0) {
-	  	user_addr_t	iov_base;
+	mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
+	devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
+
+	if (devblocksize == 1) {
+               /*
+                * the AFP client advertises a devblocksize of 1
+                * however, its BLOCKMAP routine maps to physical
+                * blocks that are PAGE_SIZE in size...
+                * therefore we can't ask for I/Os that aren't page aligned
+                * or aren't multiples of PAGE_SIZE in size
+                * by setting devblocksize to PAGE_SIZE, we re-instate
+                * the old behavior we had before the mem_alignment_mask
+                * changes went in...
+                */
+               devblocksize = PAGE_SIZE;
+	}
 
-	        io_size = uio_resid(uio);
+next_dwrite:
+	io_req_size = *write_length;
+	iov_base = uio_curriovbase(uio);
 
-		if (io_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
-		        io_size = MAX_UPL_TRANSFER * PAGE_SIZE;
+	offset_in_file = (u_int32_t)uio->uio_offset & PAGE_MASK;
+	offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
+
+	if (offset_in_file || offset_in_iovbase) {
+	        /*
+		 * one of the 2 important offsets is misaligned
+		 * so fire an I/O through the cache for this entire vector
+		 */
+	        goto wait_for_dwrites;
+	}
+	if (iov_base & (devblocksize - 1)) {
+	        /*
+		 * the offset in memory must be on a device block boundary
+		 * so that we can guarantee that we can generate an
+		 * I/O that ends on a page boundary in cluster_io
+		 */
+	        goto wait_for_dwrites;
+        }
+
+	while (io_req_size >= PAGE_SIZE && uio->uio_offset < newEOF && retval == 0) {
 
+	        if (first_IO) {
+		        cluster_syncup(vp, newEOF, callback, callback_arg);
+			first_IO = 0;
+		}
+	        io_size  = io_req_size & ~PAGE_MASK;
 		iov_base = uio_curriovbase(uio);
 
-		// LP64todo - fix this!
-		upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
-		
+		if (io_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
+		        io_size = MAX_UPL_TRANSFER * PAGE_SIZE;
+
+		upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
 		upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
@@ -1591,7 +1933,6 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 			upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
 		                    UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
 
-			// LP64todo - fix this!
 			kret = vm_map_get_upl(current_map(),
 					      (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
 					      &upl_size,
@@ -1605,13 +1946,13 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 			        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
 					     0, 0, 0, kret, 0);
 				/*
-				 * cluster_nocopy_write: failed to get pagelist
+				 * failed to get pagelist
 				 *
 				 * we may have already spun some portion of this request
 				 * off as async requests... we need to wait for the I/O
 				 * to complete before returning
 				 */
-				goto wait_for_writes;
+				goto wait_for_dwrites;
 			}
 			pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
 			pages_in_pl = upl_size / PAGE_SIZE;
@@ -1627,8 +1968,7 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 			 * didn't get all the pages back that we
 			 * needed... release this upl and try again
 			 */
-			ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, 
-					    UPL_ABORT_FREE_ON_EMPTY);
+			ubc_upl_abort(upl, 0);
 		}
 		if (force_data_sync >= 3) {
 		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
@@ -1641,28 +1981,31 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 			 * off as async requests... we need to wait for the I/O
 			 * to complete before returning
 			 */
-			goto wait_for_writes;
+			goto wait_for_dwrites;
 		}
 
 		/*
 		 * Consider the possibility that upl_size wasn't satisfied.
 		 */
-		if (upl_size != upl_needed_size)
-		        io_size = (upl_size - (int)upl_offset) & ~PAGE_MASK;
-
+		if (upl_size < upl_needed_size) {
+		        if (upl_size && upl_offset == 0)
+			        io_size = upl_size;
+			else
+			        io_size = 0;
+		}
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
 			     (int)upl_offset, upl_size, (int)iov_base, io_size, 0);		       
 
 		if (io_size == 0) {
-		        ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, 
-					    UPL_ABORT_FREE_ON_EMPTY);
+		        ubc_upl_abort(upl, 0);
 			/*
 			 * we may have already spun some portion of this request
 			 * off as async requests... we need to wait for the I/O
 			 * to complete before returning
 			 */
-			goto wait_for_writes;
+			goto wait_for_dwrites;
 		}
+
 		/*
 		 * Now look for pages already in the cache
 		 * and throw them away.
@@ -1681,7 +2024,7 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 
 		while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_UPL_TRANSFER * PAGE_SIZE)) {
 	                iostate.io_wanted = 1;
-			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_write", 0);
+			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
 		}	
 		lck_mtx_unlock(cl_mtxp);
 
@@ -1693,91 +2036,126 @@ cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
 			 * go wait for all writes that are part of this stream
 			 * to complete before returning the error to the caller
 			 */
-		        ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, 
-					    UPL_ABORT_FREE_ON_EMPTY);
+		        ubc_upl_abort(upl, 0);
 
-		        goto wait_for_writes;
+		        goto wait_for_dwrites;
 	        }
-		io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE;
+		io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE | CL_DIRECT_IO | bflag;
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
 			     (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
 
-		error = cluster_io(vp, upl, upl_offset, uio->uio_offset,
-				   io_size, io_flag, (buf_t)NULL, &iostate);
+		retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
+				   io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
 
+		/*
+		 * update the uio structure to
+		 * reflect the I/O that we just issued
+		 */
 		uio_update(uio, (user_size_t)io_size);
 
+		io_req_size -= io_size;
+
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
-			     (int)upl_offset, (int)uio->uio_offset, (int)uio_resid(uio), error, 0);
+			     (int)upl_offset, (int)uio->uio_offset, io_req_size, retval, 0);
 
 	} /* end while */
 
-wait_for_writes:
-	/*
-	 * make sure all async writes issued as part of this stream
-	 * have completed before we return
-	 */
-	lck_mtx_lock(cl_mtxp);
+        if (retval == 0 && iostate.io_error == 0 && io_req_size == 0) {
 
-	while (iostate.io_issued != iostate.io_completed) {
-	        iostate.io_wanted = 1;
-		msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_write", 0);
-	}	
-	lck_mtx_unlock(cl_mtxp);
+	        retval = cluster_io_type(uio, write_type, write_length, MIN_DIRECT_WRITE_SIZE);
+
+		if (retval == 0 && *write_type == IO_DIRECT) {
+
+		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_NONE,
+				     (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
+
+		        goto next_dwrite;
+		}
+        }
+
+wait_for_dwrites:
+	if (iostate.io_issued) {
+	        /*
+		 * make sure all async writes issued as part of this stream
+		 * have completed before we return
+		 */
+	        lck_mtx_lock(cl_mtxp);
 
+		while (iostate.io_issued != iostate.io_completed) {
+		        iostate.io_wanted = 1;
+			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
+		}	
+		lck_mtx_unlock(cl_mtxp);
+	}
 	if (iostate.io_error)
-		error = iostate.io_error;
+	        retval = iostate.io_error;
+
+	if (io_req_size && retval == 0) {
+	        /*
+		 * we couldn't handle the tail of this request in DIRECT mode
+		 * so fire it through the copy path
+		 *
+		 * note that flags will never have IO_HEADZEROFILL or IO_TAILZEROFILL set
+		 * so we can just pass 0 in for the headOff and tailOff
+		 */
+	        retval = cluster_write_copy(vp, uio, io_req_size, oldEOF, newEOF, (off_t)0, (off_t)0, flags, callback, callback_arg);
 
+		*write_type = IO_UNKNOWN;
+	}
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
-		     (int)uio->uio_offset, (int)uio->uio_resid, error, 4, 0);
+		     (int)uio->uio_offset, io_req_size, retval, 4, 0);
 
-	return (error);
+	return (retval);
 }
 
 
 static int
-cluster_phys_write(vnode_t vp, struct uio *uio, off_t newEOF)
+cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF, int *write_type, u_int32_t *write_length,
+		     int (*callback)(buf_t, void *), void *callback_arg, int bflag)
 {
 	upl_page_info_t *pl;
-	addr64_t	 src_paddr;
- 	upl_t            upl;
+	addr64_t	 src_paddr = 0;
+ 	upl_t            upl[MAX_VECTS];
 	vm_offset_t      upl_offset;
-	int              tail_size;
-	int              io_size;
-	int              upl_size;
-	int              upl_needed_size;
-	int              pages_in_pl;
+	u_int32_t        tail_size = 0;
+	u_int32_t	 io_size;
+	u_int32_t	 xsize;
+	vm_size_t	 upl_size;
+	vm_size_t	 upl_needed_size;
+	mach_msg_type_number_t	pages_in_pl;
 	int              upl_flags;
 	kern_return_t    kret;
+        struct clios     iostate;
 	int              error  = 0;
+	int		 cur_upl = 0;
+	int		 num_upl = 0;
+	int		 n;
 	user_addr_t	 iov_base;
-	int		 devblocksize;
-	struct cl_writebehind *wbp;
+	u_int32_t	 devblocksize;
+	u_int32_t	 mem_alignment_mask;
 
-	devblocksize = vp->v_mount->mnt_devblocksize;
 	/*
 	 * When we enter this routine, we know
-	 *  -- the resid will not exceed iov_len
-	 *  -- the vector target address is physcially contiguous
+	 *  -- the io_req_size will not exceed iov_len
+	 *  -- the target address is physically contiguous
 	 */
-	if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) != NULL) {
+	cluster_syncup(vp, newEOF, callback, callback_arg);
 
-		cluster_try_push(wbp, vp, newEOF, 0, 1);
+	devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
+	mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
 
-	        lck_mtx_unlock(&wbp->cl_lockw);
-	}
-#if LP64_DEBUG
-	if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
-		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); 
-	}
-#endif /* LP64_DEBUG */
+        iostate.io_completed = 0;
+        iostate.io_issued = 0;
+        iostate.io_error = 0;
+        iostate.io_wanted = 0;
+
+next_cwrite:
+	io_size = *write_length;
 
-	// LP64todo - fix this!
-	io_size = (int)uio_curriovlen(uio);
 	iov_base = uio_curriovbase(uio);
 
-	upl_offset = CAST_DOWN(upl_offset_t, iov_base) & PAGE_MASK;
+	upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
 	upl_needed_size = upl_offset + io_size;
 
 	pages_in_pl = 0;
@@ -1785,88 +2163,166 @@ cluster_phys_write(vnode_t vp, struct uio *uio, off_t newEOF)
 	upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC | 
 	            UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
 
-	// LP64todo - fix this!
 	kret = vm_map_get_upl(current_map(),
 			      (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
-			      &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, 0);
+			      &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
 
 	if (kret != KERN_SUCCESS) {
 	        /*
-		 * cluster_phys_write: failed to get pagelist
-		 * note: return kret here
+		 * failed to get pagelist
 		 */
-	      return(EINVAL);
+	        error = EINVAL;
+		goto wait_for_cwrites;
 	}
+	num_upl++;
+
 	/*
 	 * Consider the possibility that upl_size wasn't satisfied.
-	 * This is a failure in the physical memory case.
 	 */
 	if (upl_size < upl_needed_size) {
-	        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
-		return(EINVAL);
+		/*
+		 * This is a failure in the physical memory case.
+		 */
+		error = EINVAL;
+		goto wait_for_cwrites;
 	}
-	pl = ubc_upl_pageinfo(upl);
+	pl = ubc_upl_pageinfo(upl[cur_upl]);
 
 	src_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
 
 	while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
-	        int   head_size;
+	        u_int32_t   head_size;
 
-		head_size = devblocksize - (int)(uio->uio_offset & (devblocksize - 1));
+		head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
 
 		if (head_size > io_size)
 		        head_size = io_size;
 
-		error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0);
+		error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0, callback, callback_arg);
 
-		if (error) {
-		        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
+		if (error)
+		        goto wait_for_cwrites;
 
-			return(EINVAL);
-		}
 		upl_offset += head_size;
 		src_paddr  += head_size;
 		io_size    -= head_size;
+
+		iov_base   += head_size;
 	}
+	if ((u_int32_t)iov_base & mem_alignment_mask) {
+	        /*
+		 * request doesn't set up on a memory boundary
+		 * the underlying DMA engine can handle...
+		 * return an error instead of going through
+		 * the slow copy path since the intent of this
+		 * path is direct I/O from device memory
+		 */
+	        error = EINVAL;
+		goto wait_for_cwrites;
+	}
+
 	tail_size = io_size & (devblocksize - 1);
 	io_size  -= tail_size;
 
-	if (io_size) {
-	        /*
-		 * issue a synchronous write to cluster_io
+	while (io_size && error == 0) {
+
+	        if (io_size > MAX_IO_CONTIG_SIZE)
+		        xsize = MAX_IO_CONTIG_SIZE;
+		else
+		        xsize = io_size;
+		/*
+		 * request asynchronously so that we can overlap
+		 * the preparation of the next I/O... we'll do
+		 * the commit after all the I/O has completed
+		 * since its all issued against the same UPL
+		 * if there are already too many outstanding writes
+		 * wait until some have completed before issuing the next
 		 */
-	        error = cluster_io(vp, upl, upl_offset, uio->uio_offset,
-				   io_size, CL_DEV_MEMORY, (buf_t)NULL, (struct clios *)NULL);
-	}
-	if (error == 0) {
+		if (iostate.io_issued) {
+		        lck_mtx_lock(cl_mtxp);
+
+			while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_IO_CONTIG_SIZE)) {
+			        iostate.io_wanted = 1;
+				msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
+			}
+			lck_mtx_unlock(cl_mtxp);
+		}
+                if (iostate.io_error) {
+                        /*
+                         * one of the earlier writes we issued ran into a hard error
+                         * don't issue any more writes...
+                         * go wait for all writes that are part of this stream
+                         * to complete before returning the error to the caller
+                         */
+		        goto wait_for_cwrites;
+		}
 	        /*
-		 * The cluster_io write completed successfully,
-		 * update the uio structure
+		 * issue an asynchronous write to cluster_io
 		 */
-	        uio_update(uio, (user_size_t)io_size);
+	        error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
+				   xsize, CL_DEV_MEMORY | CL_ASYNC | bflag, (buf_t)NULL, (struct clios *)&iostate, callback, callback_arg);
 
-		src_paddr += io_size;
+		if (error == 0) {
+		        /*
+			 * The cluster_io write completed successfully,
+			 * update the uio structure
+			 */
+		        uio_update(uio, (user_size_t)xsize);
 
-		if (tail_size)
-		        error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0);
+			upl_offset += xsize;
+			src_paddr  += xsize;
+			io_size    -= xsize;
+		}
 	}
+        if (error == 0 && iostate.io_error == 0 && tail_size == 0) {
+
+	        error = cluster_io_type(uio, write_type, write_length, 0);
+
+		if (error == 0 && *write_type == IO_CONTIG) {
+		        cur_upl++;
+                        goto next_cwrite;
+		}
+	} else
+	        *write_type = IO_UNKNOWN;
+
+wait_for_cwrites:
 	/*
-	 * just release our hold on the physically contiguous
-	 * region without changing any state
-	 */
-	ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
+         * make sure all async writes that are part of this stream
+         * have completed before we proceed
+         */
+        lck_mtx_lock(cl_mtxp);
+
+        while (iostate.io_issued != iostate.io_completed) {
+	        iostate.io_wanted = 1;
+		msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
+        }
+        lck_mtx_unlock(cl_mtxp);
+
+        if (iostate.io_error)
+	        error = iostate.io_error;
+
+	if (error == 0 && tail_size)
+	        error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0, callback, callback_arg);
+
+        for (n = 0; n < num_upl; n++)
+	        /*
+		 * just release our hold on each physically contiguous
+		 * region without changing any state
+		 */
+	        ubc_upl_abort(upl[n], 0);
 
 	return (error);
 }
 
 
 static int
-cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int flags)
+cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF, off_t headOff,
+		   off_t tailOff, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	upl_page_info_t *pl;
 	upl_t            upl;
 	vm_offset_t      upl_offset = 0;
-	int              upl_size;
+	vm_size_t	 upl_size;
 	off_t 	         upl_f_offset;
 	int              pages_in_upl;
 	int		 start_offset;
@@ -1886,36 +2342,18 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 	struct cl_extent cl;
         int              intersection;
 	struct cl_writebehind *wbp;
+	int              bflag;
 
-	if ((wbp = cluster_get_wbp(vp, 0)) != NULL)
-	{
-	        if (wbp->cl_hasbeenpaged) {
-		        /*
-			 * this vnode had pages cleaned to it by
-			 * the pager which indicates that either
-			 * it's not very 'hot', or the system is
-			 * being overwhelmed by a lot of dirty 
-			 * data being delayed in the VM cache...
-			 * in either event, we'll push our remaining
-			 * delayed data at this point...  this will
-			 * be more efficient than paging out 1 page at 
-			 * a time, and will also act as a throttle
-			 * by delaying this client from writing any
-			 * more data until all his delayed data has
-			 * at least been queued to the uderlying driver.
-			 */
-		        if (wbp->cl_number || wbp->cl_scmap)
-			        cluster_push_EOF(vp, newEOF);
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
-			wbp->cl_hasbeenpaged = 0;
-		}
-	}
 	if (uio) {
 	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
-			     (int)uio->uio_offset, uio_resid(uio), (int)oldEOF, (int)newEOF, 0);
+			     (int)uio->uio_offset, io_req_size, (int)oldEOF, (int)newEOF, 0);
 
-		// LP64todo - fix this
-	        io_resid = uio_resid(uio);
+	        io_resid = io_req_size;
 	} else {
 	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
 			     0, 0, (int)oldEOF, (int)newEOF, 0);
@@ -1947,8 +2385,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 	}
 	if (flags & IO_TAILZEROFILL) {
 	        if (uio) {
-		        // LP64todo - fix this
-		        zero_off1 = uio->uio_offset + uio_resid(uio);
+		        zero_off1 = uio->uio_offset + io_req_size;
 
 			if (zero_off1 < tailOff)
 			        zero_cnt1 = tailOff - zero_off1;
@@ -1982,7 +2419,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 
 		cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
 		
-		if (uio && ((flags & (IO_NOCACHE | IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
+		if (uio && ((flags & (IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
 		        /*
 			 * assumption... total_size <= io_resid
 			 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
@@ -1991,12 +2428,12 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 			        total_size -= start_offset;
 		        xfer_resid = total_size;
 
-		        retval = cluster_copy_ubc_data(vp, uio, &xfer_resid, 1);
+		        retval = cluster_copy_ubc_data_internal(vp, uio, &xfer_resid, 1, 1);
 			
 			if (retval)
 			        break;
 
-			io_resid   -= (total_size - xfer_resid);
+			io_resid    -= (total_size - xfer_resid);
 			total_size   = xfer_resid;
 			start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
 			upl_f_offset = uio->uio_offset - start_offset;
@@ -2052,7 +2489,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 				      &pl,
 				      UPL_SET_LITE | UPL_WILL_MODIFY);
 		if (kret != KERN_SUCCESS)
-			panic("cluster_write: failed to get pagelist");
+			panic("cluster_write_copy: failed to get pagelist");
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
 			(int)upl, (int)upl_f_offset, start_offset, 0, 0);
@@ -2071,7 +2508,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 			        read_size = newEOF - upl_f_offset;
 
 		        retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
-					    CL_READ, (buf_t)NULL, (struct clios *)NULL);
+					    CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 			if (retval) {
 				/*
 				 * we had an error during the read which causes us to abort
@@ -2107,7 +2544,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 				        read_size = newEOF - (upl_f_offset + upl_offset);
 
 			        retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
-						    CL_READ, (buf_t)NULL, (struct clios *)NULL);
+						    CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 				if (retval) {
 					/*
 					 * we had an error during the read which causes us to abort
@@ -2158,9 +2595,12 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 			io_offset  += bytes_to_zero;
 		}
 		if (xfer_resid && io_resid) {
+		        u_int32_t  io_requested;
+
 			bytes_to_move = min(io_resid, xfer_resid);
+			io_requested = bytes_to_move;
 
-			retval = cluster_copy_upl_data(uio, upl, io_offset, bytes_to_move);
+			retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
 
 			if (retval) {
 
@@ -2169,7 +2609,7 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
 					     (int)upl, 0, 0, retval, 0);
 			} else {
-			        io_resid  -= bytes_to_move;
+			        io_resid   -= bytes_to_move;
 				xfer_resid -= bytes_to_move;
 				io_offset  += bytes_to_move;
 			}
@@ -2204,11 +2644,11 @@ cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t h
 
 		if (retval == 0) {
 			int cl_index;
-			int can_delay;
+			int ret_cluster_try_push;
 
 		        io_size += start_offset;
 
-			if ((upl_f_offset + io_size) >= newEOF && io_size < upl_size) {
+			if ((upl_f_offset + io_size) >= newEOF && (u_int)io_size < upl_size) {
 			        /*
 				 * if we're extending the file with this write
 				 * we'll zero fill the rest of the page so that
@@ -2253,7 +2693,7 @@ check_cluster:
 					        ubc_upl_commit_range(upl, 0, upl_size,
 								     UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
 
-					sparse_cluster_add(wbp, vp, &cl, newEOF);
+					sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
 
 					lck_mtx_unlock(&wbp->cl_lockw);
 
@@ -2277,7 +2717,7 @@ check_cluster:
 					 */
 					upl_size = 0;
 				}
-				sparse_cluster_push(wbp, vp, newEOF, 1);
+				sparse_cluster_push(wbp, vp, newEOF, PUSH_ALL, callback, callback_arg);
 
 				wbp->cl_number = 0;
 				/*
@@ -2309,7 +2749,7 @@ check_cluster:
 				        /*
 					 * the current write starts at or after the current cluster
 					 */
-				        if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER)) {
+				        if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + MAX_CLUSTER_SIZE)) {
 					        /*
 						 * we have a write that fits entirely
 						 * within the existing cluster limits
@@ -2321,7 +2761,7 @@ check_cluster:
 						        wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
 						break;
 					}
-					if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER)) {
+					if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + MAX_CLUSTER_SIZE)) {
 					        /*
 						 * we have a write that starts in the middle of the current cluster
 						 * but extends beyond the cluster's limit... we know this because
@@ -2332,7 +2772,7 @@ check_cluster:
 						 * note that we'll always have a leftover tail in this case since
 						 * full absorbtion would have occurred in the clause above
 						 */
-					        wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER;
+					        wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + MAX_CLUSTER_SIZE;
 
 						if (upl_size) {
 						        daddr64_t start_pg_in_upl;
@@ -2363,7 +2803,7 @@ check_cluster:
 				        /*
 					 * the current write starts in front of the cluster we're currently considering
 					 */
-				        if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= MAX_UPL_TRANSFER) {
+				        if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= MAX_CLUSTER_SIZE) {
 					        /*
 						 * we can just merge the new request into
 						 * this cluster and leave it in the cache
@@ -2376,7 +2816,7 @@ check_cluster:
 						        /*
 							 * the current write completely
 							 * envelops the existing cluster and since
-							 * each write is limited to at most MAX_UPL_TRANSFER bytes
+							 * each write is limited to at most MAX_CLUSTER_SIZE pages
 							 * we can just use the start and last blocknos of the write
 							 * to generate the cluster limits
 							 */
@@ -2394,18 +2834,18 @@ check_cluster:
 					 * get an intersection with the current write
 					 * 
 					 */
-					if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - MAX_UPL_TRANSFER) {
+					if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - MAX_CLUSTER_SIZE) {
 					        /*
 						 * the current write extends into the proposed cluster
 						 * clip the length of the current write after first combining it's
 						 * tail with the newly shaped cluster
 						 */
-					        wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - MAX_UPL_TRANSFER;
+					        wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - MAX_CLUSTER_SIZE;
 
 						if (upl_size) {
 						        intersection = (int)((cl.e_addr - wbp->cl_clusters[cl_index].b_addr) * PAGE_SIZE);
 
-							if (intersection > upl_size)
+							if ((u_int)intersection > upl_size)
 							        /*
 								 * because the current write may consist of a number of pages found in the cache
 								 * which are not part of the UPL, we may have an intersection that exceeds
@@ -2435,7 +2875,7 @@ check_cluster:
 				 */
 			        goto delay_io;
 
-			if (wbp->cl_number < MAX_CLUSTERS && !(flags & IO_NOCACHE))
+			if (wbp->cl_number < MAX_CLUSTERS)
 			        /*
 				 * we didn't find an existing cluster to
 				 * merge into, but there's room to start
@@ -2453,18 +2893,19 @@ check_cluster:
 			 * number of remaining clusters... and
 			 * returns the number of currently unused clusters
 			 */
-			int ret_cluster_try_push = 0;
-			/* if writes are not deferred, call cluster push immediately */
+			ret_cluster_try_push = 0;
+
+			/*
+			 * if writes are not deferred, call cluster push immediately
+			 */
 			if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
-				if (flags & IO_NOCACHE)
-			        	can_delay = 0;
-				else
-			        	can_delay = 1;
 				
-				ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, can_delay, 0);
+				ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, (flags & IO_NOCACHE) ? 0 : PUSH_DELAY, callback, callback_arg);
 			}
 
-			/* execute following regardless writes are deferred or not */
+			/*
+			 * execute following regardless of writes being deferred or not
+			 */
 			if (ret_cluster_try_push == 0) {
 			        /*
 				 * no more room in the normal cluster mechanism
@@ -2480,8 +2921,8 @@ check_cluster:
 				        ubc_upl_commit_range(upl, upl_offset, upl_size,
 							     UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
 
-			        sparse_cluster_switch(wbp, vp, newEOF);
-				sparse_cluster_add(wbp, vp, &cl, newEOF);
+			        sparse_cluster_switch(wbp, vp, newEOF, callback, callback_arg);
+				sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
 
 				lck_mtx_unlock(&wbp->cl_lockw);
 
@@ -2490,25 +2931,27 @@ check_cluster:
 			/*
 			 * we pushed one cluster successfully, so we must be sequentially writing this file
 			 * otherwise, we would have failed and fallen into the sparse cluster support
-			 * so let's take the opportunity to push out additional clusters as long as we
-			 * remain below the throttle... this will give us better I/O locality if we're
-			 * in a copy loop (i.e.  we won't jump back and forth between the read and write points
-			 * however, we don't want to push so much out that the write throttle kicks in and
-			 * hangs this thread up until some of the I/O completes...
+			 * so let's take the opportunity to push out additional clusters...
+			 * this will give us better I/O locality if we're in a copy loop
+			 * (i.e.  we won't jump back and forth between the read and write points
 			 */
 			if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
-				while (wbp->cl_number && (vp->v_numoutput <= (VNODE_ASYNC_THROTTLE / 2)))
-			        	cluster_try_push(wbp, vp, newEOF, 0, 0);
+			        while (wbp->cl_number)
+				        cluster_try_push(wbp, vp, newEOF, 0, callback, callback_arg);
 			}
 
 start_new_cluster:
 			wbp->cl_clusters[wbp->cl_number].b_addr = cl.b_addr;
 			wbp->cl_clusters[wbp->cl_number].e_addr = cl.e_addr;
 
+			wbp->cl_clusters[wbp->cl_number].io_flags = 0;
+
 			if (flags & IO_NOCACHE)
-			        wbp->cl_clusters[wbp->cl_number].io_nocache = 1;
-			else
-			        wbp->cl_clusters[wbp->cl_number].io_nocache = 0;
+			        wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IONOCACHE;
+
+			if (bflag & CL_PASSIVE)
+			        wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IOPASSIVE;
+
 			wbp->cl_number++;
 delay_io:
 			if (upl_size)
@@ -2529,182 +2972,112 @@ issue_io:
 			 * in order to maintain some semblance of coherency with mapped writes
 			 * we need to drop the current upl and pick it back up with COPYOUT_FROM set
 			 * so that we correctly deal with a change in state of the hardware modify bit...
-			 * we do this via cluster_push_x... by passing along the IO_SYNC flag, we force
-			 * cluster_push_x to wait until all the I/Os have completed... cluster_push_x is also
+			 * we do this via cluster_push_now... by passing along the IO_SYNC flag, we force
+			 * cluster_push_now to wait until all the I/Os have completed... cluster_push_now is also
 			 * responsible for generating the correct sized I/O(s)
 			 */
 		        ubc_upl_commit_range(upl, 0, upl_size,
-						     UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
+					     UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
 
 			cl.e_addr = (upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64;
 
-			retval = cluster_push_x(vp, &cl, newEOF, flags);
+			retval = cluster_push_now(vp, &cl, newEOF, flags, callback, callback_arg);
 		}
 	}
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
-		     retval, 0, io_resid, 0, 0);
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END, retval, 0, io_resid, 0, 0);
 
 	return (retval);
 }
 
+
+
 int
 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
 {
-	int           prev_resid;
-	u_int         clip_size;
-	off_t         max_io_size;
-	int           upl_size;
-	int           upl_flags;
-	upl_t         upl;
-	int           retval = 0;
-	int	      flags;
+        return cluster_read_ext(vp, uio, filesize, xflags, NULL, NULL);
+}
+
+
+int
+cluster_read_ext(vnode_t vp, struct uio *uio, off_t filesize, int xflags, int (*callback)(buf_t, void *), void *callback_arg)
+{
+	int		retval = 0;
+	int		flags;
+	user_ssize_t	cur_resid;
+        u_int32_t	io_size;
+	u_int32_t	read_length = 0;
+	int		read_type = IO_COPY;
 
 	flags = xflags;
 
 	if (vp->v_flag & VNOCACHE_DATA)
 	        flags |= IO_NOCACHE;
-	if (vp->v_flag & VRAOFF)
+	if ((vp->v_flag & VRAOFF) || speculative_reads_disabled)
 	        flags |= IO_RAOFF;
 
-	if (!((flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg))) {
-	        /*
-		 * go do a read through the cache if one of the following is true....
-		 *   NOCACHE is not true
-		 *   the uio request doesn't target USERSPACE
-		 */
-	        return (cluster_read_x(vp, uio, filesize, flags));
-	}
-
-#if LP64_DEBUG
-	if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
-		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); 
-	}
-#endif /* LP64_DEBUG */
-
-	while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
-	  	user_size_t	iov_len;
-	  	user_addr_t	iov_base;
-
-		/*
-		 * we know we have a resid, so this is safe
-		 * skip over any emtpy vectors
-		 */
-		uio_update(uio, (user_size_t)0);
+        /*
+	 * do a read through the cache if one of the following is true....
+	 *   NOCACHE is not true
+	 *   the uio request doesn't target USERSPACE
+	 * otherwise, find out if we want the direct or contig variant for
+	 * the first vector in the uio request
+	 */
+	if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
+	        retval = cluster_io_type(uio, &read_type, &read_length, 0);
 
-		iov_len  = uio_curriovlen(uio);
-		iov_base = uio_curriovbase(uio);
+	while ((cur_resid = uio_resid(uio)) && uio->uio_offset < filesize && retval == 0) {
 
-		upl_size  = PAGE_SIZE;
-		upl_flags = UPL_QUERY_OBJECT_TYPE;
-  
-		// LP64todo - fix this!
-		if ((vm_map_get_upl(current_map(),
-				    (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
-				    &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
+		switch (read_type) {
+		
+		case IO_COPY:
 		        /*
-			 * the user app must have passed in an invalid address
+			 * make sure the uio_resid isn't too big...
+			 * internally, we want to handle all of the I/O in
+			 * chunk sizes that fit in a 32 bit int
 			 */
-		        return (EFAULT);
-		}
+		        if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE))
+		                io_size = MAX_IO_REQUEST_SIZE;
+			else
+		                io_size = (u_int32_t)cur_resid;
 
-		/*
-		 * We check every vector target but if it is physically 
-		 * contiguous space, we skip the sanity checks.
-		 */
-		if (upl_flags & UPL_PHYS_CONTIG) {
-		        retval = cluster_phys_read(vp, uio, filesize);
-		}
-		else if (uio_resid(uio) < PAGE_SIZE) {
-		        /*
-			 * we're here because we're don't have a physically contiguous target buffer
-			 * go do a read through the cache if
-			 *   the total xfer size is less than a page...
-			 */
-		        return (cluster_read_x(vp, uio, filesize, flags));
-		}
-		// LP64todo - fix this!
-		else if (((int)uio->uio_offset & PAGE_MASK) || (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
-		       if (((int)uio->uio_offset & PAGE_MASK) == (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
-			       /*
-				* Bring the file offset read up to a pagesize boundary
-				* this will also bring the base address to a page boundary
-				* since they both are currently on the same offset within a page
-				* note: if we get here, uio->uio_resid is greater than PAGE_SIZE
-				* so the computed clip_size must always be less than the current uio_resid
-				*/
-			       clip_size = (PAGE_SIZE - (int)(uio->uio_offset & PAGE_MASK_64));
-
-			       /* 
-				* Fake the resid going into the cluster_read_x call
-				* and restore it on the way out.
-				*/
-			       prev_resid = uio_resid(uio);
-			       // LP64todo - fix this
-			       uio_setresid(uio, clip_size);
-
-			       retval = cluster_read_x(vp, uio, filesize, flags);
-
-			       uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-		       } else {
-			       /*
-				* can't get both the file offset and the buffer offset aligned to a page boundary
-				* so fire an I/O through the cache for this entire vector
-				*/
-			       // LP64todo - fix this!
-			       clip_size = iov_len;
-			       prev_resid = uio_resid(uio);
-			       uio_setresid(uio, clip_size);
-
-			       retval = cluster_read_x(vp, uio, filesize, flags);
-
-			       uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-		       }
-		} else {
-		        /* 
-			 * If we come in here, we know the offset into
-			 * the file is on a pagesize boundary
-			 */
-		        max_io_size = filesize - uio->uio_offset;
-			// LP64todo - fix this
-			clip_size = uio_resid(uio);
-			if (iov_len < clip_size)
-			        clip_size = iov_len;
-			if (max_io_size < clip_size)
-			        clip_size = (int)max_io_size;
-
-			if (clip_size < PAGE_SIZE) {
-			        /*
-				 * Take care of the tail end of the read in this vector.
-				 */
-			        // LP64todo - fix this
-			        prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
+			retval = cluster_read_copy(vp, uio, io_size, filesize, flags, callback, callback_arg);
+			break;
 
-				retval = cluster_read_x(vp, uio, filesize, flags);
+		case IO_DIRECT:
+		        retval = cluster_read_direct(vp, uio, filesize, &read_type, &read_length, flags, callback, callback_arg);
+			break;
 
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-			} else {
-			        /* round clip_size down to a multiple of pagesize */
-			        clip_size = clip_size & ~(PAGE_MASK);
-				// LP64todo - fix this
-				prev_resid = uio_resid(uio);
-				uio_setresid(uio, clip_size);
+		case IO_CONTIG:
+		        retval = cluster_read_contig(vp, uio, filesize, &read_type, &read_length, callback, callback_arg, flags);
+			break;
+		  
+		case IO_UNKNOWN:
+		        retval = cluster_io_type(uio, &read_type, &read_length, 0);
+			break;
+		}
+	}
+	return (retval);
+}
 
-				retval = cluster_nocopy_read(vp, uio, filesize);
 
-				if ((retval==0) && uio_resid(uio))
-				        retval = cluster_read_x(vp, uio, filesize, flags);
 
-				uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
-			}
-		} /* end else */
-	} /* end while */
+static void
+cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags)
+{
+	int range;
+	int abort_flags = UPL_ABORT_FREE_ON_EMPTY;
 
-	return(retval);
+	if ((range = last_pg - start_pg)) {
+		if ( !(flags & IO_NOCACHE))
+			abort_flags |= UPL_ABORT_REFERENCE;
+
+		ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
+	}
 }
 
+
 static int
-cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
+cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	upl_page_info_t *pl;
 	upl_t            upl;
@@ -2719,26 +3092,44 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 	off_t            max_size;
 	off_t            last_ioread_offset;
 	off_t            last_request_offset;
-	u_int            size_of_prefetch;
-	u_int            io_size;
 	kern_return_t    kret;
 	int              error  = 0;
 	int              retval = 0;
-	u_int            max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
+	u_int32_t        size_of_prefetch;
+	u_int32_t        xsize;
+	u_int32_t        io_size;
+	u_int32_t        max_rd_size = MAX_PREFETCH;
 	u_int            rd_ahead_enabled = 1;
 	u_int            prefetch_enabled = 1;
 	struct cl_readahead *	rap;
 	struct clios		iostate;
 	struct cl_extent	extent;
+	int              bflag;
+	int		 take_reference = 1;
+	struct uthread  *ut;
+	int		 policy = IOPOL_DEFAULT;
+
+	policy = current_proc()->p_iopol_disk;
+
+	ut = get_bsdthread_info(current_thread());
+
+	if (ut->uu_iopol_disk != IOPOL_DEFAULT)
+		policy = ut->uu_iopol_disk;
+
+	if (policy == IOPOL_THROTTLE)
+		take_reference = 0;
+
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
-		     (int)uio->uio_offset, uio_resid(uio), (int)filesize, 0, 0);
+		     (int)uio->uio_offset, io_req_size, (int)filesize, flags, 0);
 			 
-	// LP64todo - fix this
-	last_request_offset = uio->uio_offset + uio_resid(uio);
+	last_request_offset = uio->uio_offset + io_req_size;
 
-	if ((flags & (IO_RAOFF|IO_NOCACHE)) ||
-		((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
+	if ((flags & (IO_RAOFF|IO_NOCACHE)) || ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
 	        rd_ahead_enabled = 0;
 		rap = NULL;
 	} else {
@@ -2773,7 +3164,7 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 	} else
 	        last_ioread_offset = (off_t)0;
 
-	while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
+	while (io_req_size && uio->uio_offset < filesize && retval == 0) {
 		/*
 		 * compute the size of the upl needed to encompass
 		 * the requested read... limit each call to cluster_io
@@ -2786,21 +3177,20 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 		upl_f_offset = uio->uio_offset - (off_t)start_offset;
 		max_size     = filesize - uio->uio_offset;
 
-	// LP64todo - fix this!
-		if ((off_t)((unsigned int)uio_resid(uio)) < max_size)
-		        io_size = uio_resid(uio);
+		if ((off_t)(io_req_size) < max_size)
+		        io_size = io_req_size;
 		else
 		        io_size = max_size;
 
 		if (!(flags & IO_NOCACHE)) {
 
 		        while (io_size) {
-			        u_int io_resid;
-				u_int io_requested;
+			        u_int32_t io_resid;
+				u_int32_t io_requested;
 
 				/*
 				 * if we keep finding the pages we need already in the cache, then
-				 * don't bother to call cluster_rd_prefetch since it costs CPU cycles
+				 * don't bother to call cluster_read_prefetch since it costs CPU cycles
 				 * to determine that we have all the pages we need... once we miss in
 				 * the cache and have issued an I/O, than we'll assume that we're likely
 				 * to continue to miss in the cache and it's to our advantage to try and prefetch
@@ -2817,7 +3207,7 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 					        if (size_of_prefetch > max_rd_size)
 						        size_of_prefetch = max_rd_size;
 
-					        size_of_prefetch = cluster_rd_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize);
+					        size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
 
 						last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
 				
@@ -2837,9 +3227,12 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 
 				io_requested = io_resid;
 
-			        retval = cluster_copy_ubc_data(vp, uio, &io_resid, 0);
+			        retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_resid, 0, take_reference);
 
-				io_size -= (io_requested - io_resid);
+				xsize = io_requested - io_resid;
+
+				io_size -= xsize;
+				io_req_size -= xsize;
 
 				if (retval || io_resid)
 				        /*
@@ -2854,7 +3247,7 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 					 * we're already finished the I/O for this read request
 					 * let's see if we should do a read-ahead
 					 */
-				        cluster_rd_ahead(vp, &extent, filesize, rap);
+				        cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
 				}
 			}
 			if (retval)
@@ -2876,8 +3269,13 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 
 		upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
 
-	        if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE) / 4)
-		        upl_size = (MAX_UPL_TRANSFER * PAGE_SIZE) / 4;
+		if (flags & IO_NOCACHE) {
+		        if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
+			        upl_size = (MAX_UPL_TRANSFER * PAGE_SIZE);
+		} else {
+		        if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE) / 4)
+			        upl_size = (MAX_UPL_TRANSFER * PAGE_SIZE) / 4;
+		}
 		pages_in_upl = upl_size / PAGE_SIZE;
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
@@ -2888,9 +3286,9 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 				      upl_size,
 				      &upl,
 				      &pl,
-				      UPL_SET_LITE);
+				      UPL_FILE_IO | UPL_SET_LITE);
 		if (kret != KERN_SUCCESS)
-			panic("cluster_read: failed to get pagelist");
+			panic("cluster_read_copy: failed to get pagelist");
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
 			     (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
@@ -2939,7 +3337,7 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 			 */
 
 			error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
-					   io_size, CL_READ | CL_ASYNC, (buf_t)NULL, &iostate);
+					   io_size, CL_READ | CL_ASYNC | bflag, (buf_t)NULL, &iostate, callback, callback_arg);
 		}
 		if (error == 0) {
 		        /*
@@ -2954,8 +3352,19 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 			        if (!upl_valid_page(pl, uio_last))
 				        break;
 			}
+			if (uio_last < pages_in_upl) {
+			        /*
+				 * there were some invalid pages beyond the valid pages
+				 * that we didn't issue an I/O for, just release them
+				 * unchanged now, so that any prefetch/readahed can
+				 * include them
+				 */
+			        ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
+						    (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
+			}
+
 			/*
-			 * compute size to transfer this round,  if uio->uio_resid is
+			 * compute size to transfer this round,  if io_req_size is
 			 * still non-zero after this attempt, we'll loop around and
 			 * set up for another I/O.
 			 */
@@ -2964,26 +3373,32 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 			if (val_size > max_size)
 			        val_size = max_size;
 
-			if (val_size > uio_resid(uio))
-	// LP64todo - fix this
-			        val_size = uio_resid(uio);
+			if (val_size > io_req_size)
+			        val_size = io_req_size;
 
-			if (last_ioread_offset == 0)
+			if ((uio->uio_offset + val_size) > last_ioread_offset)
 			        last_ioread_offset = uio->uio_offset + val_size;
 
 			if ((size_of_prefetch = (last_request_offset - last_ioread_offset)) && prefetch_enabled) {
-			        /*
-				 * if there's still I/O left to do for this request, and...
-				 * we're not in hard throttle mode, then issue a
-				 * pre-fetch I/O... the I/O latency will overlap
-				 * with the copying of the data
-				 */
-     				size_of_prefetch = cluster_rd_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize);
 
-				last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
+			        if ((last_ioread_offset - (uio->uio_offset + val_size)) <= upl_size) {
+				        /*
+					 * if there's still I/O left to do for this request, and...
+					 * we're not in hard throttle mode, and...
+					 * we're close to using up the previous prefetch, then issue a
+					 * new pre-fetch I/O... the I/O latency will overlap
+					 * with the copying of the data
+					 */
+				        if (size_of_prefetch > max_rd_size)
+					        size_of_prefetch = max_rd_size;
+
+					size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
+
+					last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
 				
-				if (last_ioread_offset > last_request_offset)
-				        last_ioread_offset = last_request_offset;
+					if (last_ioread_offset > last_request_offset)
+					        last_ioread_offset = last_request_offset;
+				}
 
 			} else if ((uio->uio_offset + val_size) == last_request_offset) {
 			        /*
@@ -2993,7 +3408,7 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 				 * explicitly disabled it
 				 */
 			        if (rd_ahead_enabled)
-				        cluster_rd_ahead(vp, &extent, filesize, rap);
+					cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
 					
 				if (rap != NULL) {
 				        if (extent.e_addr < rap->cl_lastr)
@@ -3005,42 +3420,43 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 
 			while (iostate.io_issued != iostate.io_completed) {
 			        iostate.io_wanted = 1;
-				msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_x", 0);
+				msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_copy", NULL);
 			}	
 			lck_mtx_unlock(cl_mtxp);
 
 			if (iostate.io_error)
 			        error = iostate.io_error;
-			else
-			        retval = cluster_copy_upl_data(uio, upl, start_offset, val_size);
+			else {
+			        u_int32_t io_requested;
+
+			        io_requested = val_size;
+
+			        retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
+				
+				io_req_size -= (val_size - io_requested);
+			}
 		}
 		if (start_pg < last_pg) {
 		        /*
 			 * compute the range of pages that we actually issued an I/O for
 			 * and either commit them as valid if the I/O succeeded
-			 * or abort them if the I/O failed
+			 * or abort them if the I/O failed or we're not supposed to 
+			 * keep them in the cache
 			 */
 		        io_size = (last_pg - start_pg) * PAGE_SIZE;
 
-			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
-				     (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
+			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
 
 			if (error || (flags & IO_NOCACHE))
 			        ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
-						UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
+						    UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
 			else
 			        ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size, 
-						     UPL_COMMIT_CLEAR_DIRTY |
-						     UPL_COMMIT_FREE_ON_EMPTY | 
-						     UPL_COMMIT_INACTIVATE);
+						     UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE);
 
-			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END,
-				     (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
+			KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
 		}
 		if ((last_pg - start_pg) < pages_in_upl) {
-		        int cur_pg;
-			int commit_flags;
-
 		        /*
 			 * the set of pages that we issued an I/O for did not encompass
 			 * the entire upl... so just release these without modifying
@@ -3049,91 +3465,57 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 			if (error)
 				ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
 			else {
-				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
-					     (int)upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
 
-				if (start_pg) {
-					/*
-					 * we found some already valid pages at the beginning of
-					 * the upl commit these back to the inactive list with
-					 * reference cleared
-					 */
-					for (cur_pg = 0; cur_pg < start_pg; cur_pg++) {
-						commit_flags = UPL_COMMIT_FREE_ON_EMPTY 
-							           | UPL_COMMIT_INACTIVATE;
-						
-						if (upl_dirty_page(pl, cur_pg))
-							commit_flags |= UPL_COMMIT_SET_DIRTY;
-						
-						if ( !(commit_flags & UPL_COMMIT_SET_DIRTY) && (flags & IO_NOCACHE))
-							ubc_upl_abort_range(upl, cur_pg * PAGE_SIZE, PAGE_SIZE,
-								UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
-						else
-							ubc_upl_commit_range(upl, cur_pg * PAGE_SIZE, 
-								PAGE_SIZE, commit_flags);
-					}
-				}
-				if (last_pg < uio_last) {
-					/*
-					 * we found some already valid pages immediately after the
-					 * pages we issued I/O for, commit these back to the
-					 * inactive list with reference cleared
-					 */
-					for (cur_pg = last_pg; cur_pg < uio_last; cur_pg++) {
-						commit_flags =  UPL_COMMIT_FREE_ON_EMPTY 
-										| UPL_COMMIT_INACTIVATE;
-
-						if (upl_dirty_page(pl, cur_pg))
-							commit_flags |= UPL_COMMIT_SET_DIRTY;
-						
-						if ( !(commit_flags & UPL_COMMIT_SET_DIRTY) && (flags & IO_NOCACHE))
-							ubc_upl_abort_range(upl, cur_pg * PAGE_SIZE, PAGE_SIZE,
-								UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
-						else
-							ubc_upl_commit_range(upl, cur_pg * PAGE_SIZE, 
-								PAGE_SIZE, commit_flags);
-					}
-				}
-				if (uio_last < pages_in_upl) {
-					/*
-					 * there were some invalid pages beyond the valid pages
-					 * that we didn't issue an I/O for, just release them
-					 * unchanged
-					 */
-				        ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
-							    (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
-				}
+				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
+					     (int)upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
+
+				/*
+				 * handle any valid pages at the beginning of
+				 * the upl... release these appropriately
+				 */
+				cluster_read_upl_release(upl, 0, start_pg, flags);
+
+				/*
+				 * handle any valid pages immediately after the
+				 * pages we issued I/O for... ... release these appropriately
+				 */
+				cluster_read_upl_release(upl, last_pg, uio_last, flags);
 
-				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END,
-					(int)upl, -1, -1, 0, 0);
+				KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, -1, -1, 0, 0);
 			}
 		}
 		if (retval == 0)
 		        retval = error;
 
-		if ( uio_resid(uio) ) {
+		if (io_req_size) {
 		        if (cluster_hard_throttle_on(vp)) {
 			        rd_ahead_enabled = 0;
 				prefetch_enabled = 0;
 
 				max_rd_size = HARD_THROTTLE_MAXSIZE;
 			} else {
-				if (rap != NULL)
-				        rd_ahead_enabled = 1;
-				prefetch_enabled = 1;
+			        if (max_rd_size == HARD_THROTTLE_MAXSIZE) {
+				        /*
+					 * coming out of throttled state
+					 */
+				        if (rap != NULL)
+					        rd_ahead_enabled = 1;
+					prefetch_enabled = 1;
 
-			        max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
+					max_rd_size = MAX_PREFETCH;
+					last_ioread_offset = 0;
+				}
 			}
 		}
 	}
 	if (rap != NULL) {
 	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
-			     (int)uio->uio_offset, uio_resid(uio), rap->cl_lastr, retval, 0);
+			     (int)uio->uio_offset, io_req_size, rap->cl_lastr, retval, 0);
 
 	        lck_mtx_unlock(&rap->cl_lockr);
 	} else {
 	        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
-			     (int)uio->uio_offset, uio_resid(uio), 0, retval, 0);
+			     (int)uio->uio_offset, io_req_size, 0, retval, 0);
 	}
 
 	return (retval);
@@ -3141,107 +3523,197 @@ cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
 
 
 static int
-cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
+cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
+		    int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	upl_t            upl;
 	upl_page_info_t  *pl;
+        off_t		 max_io_size;
 	vm_offset_t      upl_offset;
-	off_t            max_io_size;
-	int              io_size;
-	int              upl_size;
-	int              upl_needed_size;
-	int              pages_in_pl;
+	vm_size_t	 upl_size;
+	vm_size_t	 upl_needed_size;
+	unsigned int	 pages_in_pl;
 	int              upl_flags;
+	int              bflag;
 	kern_return_t    kret;
-	int              i;
+	unsigned int     i;
 	int              force_data_sync;
 	int              retval = 0;
 	int		 no_zero_fill = 0;
 	int		 abort_flag = 0;
+	int              io_flag = 0;
+	int		 misaligned = 0;
 	struct clios     iostate;
-	u_int            max_rd_size  = MAX_UPL_TRANSFER * PAGE_SIZE;
-	u_int            max_rd_ahead = MAX_UPL_TRANSFER * PAGE_SIZE * 2;
-
+	user_addr_t	 iov_base;
+	u_int32_t	 io_req_size;
+	u_int32_t	 offset_in_file;
+	u_int32_t	 offset_in_iovbase;
+	u_int32_t	 io_size;
+	u_int32_t	 io_min;
+        u_int32_t	 xsize;
+	u_int32_t	 devblocksize;
+	u_int32_t	 mem_alignment_mask;
+	u_int32_t        max_rd_size  = MAX_UPL_TRANSFER * PAGE_SIZE;
+	u_int32_t        max_rd_ahead = MAX_PREFETCH;
+
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
-		     (int)uio->uio_offset, uio_resid(uio), (int)filesize, 0, 0);
-
-	/*
-	 * When we enter this routine, we know
-	 *  -- the offset into the file is on a pagesize boundary
-	 *  -- the resid is a page multiple
-	 *  -- the resid will not exceed iov_len
-	 */
+		     (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
 
 	iostate.io_completed = 0;
 	iostate.io_issued = 0;
 	iostate.io_error = 0;
 	iostate.io_wanted = 0;
 
-	while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
-	  	user_addr_t	iov_base;
+	devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
+	mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
+		     (int)devblocksize, (int)mem_alignment_mask, 0, 0, 0);
+
+	if (devblocksize == 1) {
+               /*
+                * the AFP client advertises a devblocksize of 1
+                * however, its BLOCKMAP routine maps to physical
+                * blocks that are PAGE_SIZE in size...
+                * therefore we can't ask for I/Os that aren't page aligned
+                * or aren't multiples of PAGE_SIZE in size
+                * by setting devblocksize to PAGE_SIZE, we re-instate
+                * the old behavior we had before the mem_alignment_mask
+                * changes went in...
+                */
+               devblocksize = PAGE_SIZE;
+	}
+next_dread:
+	io_req_size = *read_length;
+	iov_base = uio_curriovbase(uio);
+
+        max_io_size = filesize - uio->uio_offset;
+
+	if ((off_t)io_req_size > max_io_size)
+	        io_req_size = max_io_size;
+
+	offset_in_file = (u_int32_t)uio->uio_offset & (devblocksize - 1);
+	offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
+
+	if (offset_in_file || offset_in_iovbase) {
+	        /*
+		 * one of the 2 important offsets is misaligned
+		 * so fire an I/O through the cache for this entire vector
+		 */
+		misaligned = 1;
+	}
+	if (iov_base & (devblocksize - 1)) {
+	        /*
+		 * the offset in memory must be on a device block boundary
+		 * so that we can guarantee that we can generate an
+		 * I/O that ends on a page boundary in cluster_io
+		 */
+		misaligned = 1;
+        }
+	/*
+	 * When we get to this point, we know...
+	 *  -- the offset into the file is on a devblocksize boundary
+	 */
+
+	while (io_req_size && retval == 0) {
+	        u_int32_t io_start;
 
 	        if (cluster_hard_throttle_on(vp)) {
 		        max_rd_size  = HARD_THROTTLE_MAXSIZE;
 			max_rd_ahead = HARD_THROTTLE_MAXSIZE - 1;
 		} else {
 		        max_rd_size  = MAX_UPL_TRANSFER * PAGE_SIZE;
-			max_rd_ahead = MAX_UPL_TRANSFER * PAGE_SIZE * 8;
+			max_rd_ahead = MAX_PREFETCH;
 		}
-	        max_io_size = filesize - uio->uio_offset;
-
-	// LP64todo - fix this
-		if (max_io_size < (off_t)((unsigned int)uio_resid(uio)))
-		        io_size = max_io_size;
-		else
-		        io_size = uio_resid(uio);
+		io_start = io_size = io_req_size;
 
 		/*
 		 * First look for pages already in the cache
 		 * and move them to user space.
+		 *
+		 * cluster_copy_ubc_data returns the resid
+		 * in io_size
 		 */
-		retval = cluster_copy_ubc_data(vp, uio, &io_size, 0);
+		retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_size, 0, 0);
 			
-		if (retval) {
-			/*
-			 * we may have already spun some portion of this request
-			 * off as async requests... we need to wait for the I/O
-			 * to complete before returning
+		/*
+		 * calculate the number of bytes actually copied
+		 * starting size - residual
+		 */
+		xsize = io_start - io_size;
+
+		io_req_size -= xsize;
+
+		/*
+		 * check to see if we are finished with this request...
+		 */
+		if (io_req_size == 0 || misaligned) {
+		        /*
+			 * see if there's another uio vector to
+			 * process that's of type IO_DIRECT
+			 *
+			 * break out of while loop to get there
 			 */
-			goto wait_for_reads;
+		        break;
 		}
 		/*
-		 * If we are already finished with this read, then return
+		 * assume the request ends on a device block boundary
 		 */
-		if (io_size == 0) {
-			/*
+		io_min = devblocksize;
+
+		/*
+		 * we can handle I/O's in multiples of the device block size
+		 * however, if io_size isn't a multiple of devblocksize we
+		 * want to clip it back to the nearest page boundary since
+		 * we are going to have to go through cluster_read_copy to
+		 * deal with the 'overhang'... by clipping it to a PAGE_SIZE
+		 * multiple, we avoid asking the drive for the same physical
+		 * blocks twice.. once for the partial page at the end of the
+		 * request and a 2nd time for the page we read into the cache
+		 * (which overlaps the end of the direct read) in order to 
+		 * get at the overhang bytes
+		 */
+		if (io_size & (devblocksize - 1)) {
+		        /*
+			 * request does NOT end on a device block boundary
+			 * so clip it back to a PAGE_SIZE boundary
+			 */
+		        io_size &= ~PAGE_MASK;
+			io_min = PAGE_SIZE;
+		}
+		if (retval || io_size < io_min) {
+		        /*
+			 * either an error or we only have the tail left to
+			 * complete via the copy path...
 			 * we may have already spun some portion of this request
 			 * off as async requests... we need to wait for the I/O
 			 * to complete before returning
 			 */
-			goto wait_for_reads;
+		        goto wait_for_dreads;
 		}
-		max_io_size = io_size;
-
-		if (max_io_size > max_rd_size)
-		        max_io_size = max_rd_size;
+		if ((xsize = io_size) > max_rd_size)
+		        xsize = max_rd_size;
 
 		io_size = 0;
 
-		ubc_range_op(vp, uio->uio_offset, uio->uio_offset + max_io_size, UPL_ROP_ABSENT, &io_size);
+		ubc_range_op(vp, uio->uio_offset, uio->uio_offset + xsize, UPL_ROP_ABSENT, (int *)&io_size);
 
-		if (io_size == 0)
+		if (io_size == 0) {
 			/*
-			 * we may have already spun some portion of this request
-			 * off as async requests... we need to wait for the I/O
-			 * to complete before returning
+			 * a page must have just come into the cache
+			 * since the first page in this range is no
+			 * longer absent, go back and re-evaluate
 			 */
-			goto wait_for_reads;
-		
+		        continue;
+		}
 		iov_base = uio_curriovbase(uio);
 
-		// LP64todo - fix this!
-		upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
+		upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
 		upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
 
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
@@ -3264,7 +3736,6 @@ cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
 			if (force_data_sync)
 			        upl_flags |= UPL_FORCE_DATA_SYNC;
 
-			// LP64todo - fix this!
 			kret = vm_map_create_upl(current_map(),
 						 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
 						 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
@@ -3273,13 +3744,13 @@ cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
 			        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
 					     (int)upl_offset, upl_size, io_size, kret, 0);
 				/*
-				 * cluster_nocopy_read: failed to get pagelist
+				 * failed to get pagelist
 				 *
 				 * we may have already spun some portion of this request
 				 * off as async requests... we need to wait for the I/O
 				 * to complete before returning
 				 */
-				goto wait_for_reads;
+				goto wait_for_dreads;
 			}
 			pages_in_pl = upl_size / PAGE_SIZE;
 			pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
@@ -3291,23 +3762,26 @@ cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
 			if (i == pages_in_pl)
 			        break;
 
-			ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
+			ubc_upl_abort(upl, abort_flag);
 		}
 		if (force_data_sync >= 3) {
 		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
 				     (int)upl_offset, upl_size, io_size, kret, 0);
 		  
-			goto wait_for_reads;
+			goto wait_for_dreads;
 		}
 		/*
 		 * Consider the possibility that upl_size wasn't satisfied.
 		 */
-		if (upl_size != upl_needed_size)
-		        io_size = (upl_size - (int)upl_offset) & ~PAGE_MASK;
-
+		if (upl_size < upl_needed_size) {
+		        if (upl_size && upl_offset == 0)
+			        io_size = upl_size;
+			else
+			        io_size = 0;
+		}
 		if (io_size == 0) {
-		        ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
-			goto wait_for_reads;
+			ubc_upl_abort(upl, abort_flag);
+			goto wait_for_dreads;
 		}
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
 			     (int)upl_offset, upl_size, io_size, kret, 0);
@@ -3322,7 +3796,7 @@ cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
 
 		while ((iostate.io_issued - iostate.io_completed) > max_rd_ahead) {
 	                iostate.io_wanted = 1;
-			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_read", 0);
+			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
 		}	
 		lck_mtx_unlock(cl_mtxp);
 			
@@ -3334,158 +3808,213 @@ cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
 			 * go wait for any other reads to complete before
 			 * returning the error to the caller
 			 */
-		        ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
+			ubc_upl_abort(upl, abort_flag);
 
-		        goto wait_for_reads;
+		        goto wait_for_dreads;
 	        }
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
 			     (int)upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
 
-		retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size,
-				   CL_PRESERVE | CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO,
-				   (buf_t)NULL, &iostate);
+		if (no_zero_fill)
+		        io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | bflag;
+		else
+		        io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | CL_PRESERVE | bflag;
+
+		retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
 
 		/*
 		 * update the uio structure
 		 */
 		uio_update(uio, (user_size_t)io_size);
 
+		io_req_size -= io_size;
+
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
-			     (int)upl, (int)uio->uio_offset, (int)uio_resid(uio), retval, 0);
+			     (int)upl, (int)uio->uio_offset, io_req_size, retval, 0);
 
 	} /* end while */
 
-wait_for_reads:
-	/*
-	 * make sure all async reads that are part of this stream
-	 * have completed before we return
-	 */
-	lck_mtx_lock(cl_mtxp);
+	if (retval == 0 && iostate.io_error == 0 && io_req_size == 0 && uio->uio_offset < filesize) {
 
-	while (iostate.io_issued != iostate.io_completed) {
-	        iostate.io_wanted = 1;
-		msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_read", 0);
-	}	
-	lck_mtx_unlock(cl_mtxp);
+	        retval = cluster_io_type(uio, read_type, read_length, 0);
+	  
+		if (retval == 0 && *read_type == IO_DIRECT) {
+
+		        KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
+				     (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
+
+			goto next_dread;
+		}
+	}
+
+wait_for_dreads:
+	if (iostate.io_issued) {
+	        /*
+		 * make sure all async reads that are part of this stream
+		 * have completed before we return
+		 */
+	        lck_mtx_lock(cl_mtxp);
+
+		while (iostate.io_issued != iostate.io_completed) {
+		        iostate.io_wanted = 1;
+			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
+		}	
+		lck_mtx_unlock(cl_mtxp);
+	}
 
 	if (iostate.io_error)
-		retval = iostate.io_error;
+	        retval = iostate.io_error;
+
+	if (io_req_size && retval == 0) {
+	        /*
+		 * we couldn't handle the tail of this request in DIRECT mode
+		 * so fire it through the copy path
+		 */
+	        retval = cluster_read_copy(vp, uio, io_req_size, filesize, flags, callback, callback_arg);
 
+		*read_type = IO_UNKNOWN;
+	}
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
-		     (int)uio->uio_offset, (int)uio_resid(uio), 6, retval, 0);
+		     (int)uio->uio_offset, (int)uio_resid(uio), io_req_size, retval, 0);
 
 	return (retval);
 }
 
 
 static int
-cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
+cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
+		    int (*callback)(buf_t, void *), void *callback_arg, int flags)
 {
 	upl_page_info_t *pl;
-	upl_t            upl;
+	upl_t            upl[MAX_VECTS];
 	vm_offset_t      upl_offset;
-	addr64_t	 dst_paddr;
-	off_t            max_size;
-	int		 io_size;
-	user_size_t	 iov_len;
+	addr64_t	 dst_paddr = 0;
 	user_addr_t	 iov_base;
-	int              tail_size;
-	int              upl_size;
-	int              upl_needed_size;
-	int              pages_in_pl;
+	off_t            max_size;
+	vm_size_t	 upl_size;
+	vm_size_t	 upl_needed_size;
+	mach_msg_type_number_t	pages_in_pl;
 	int              upl_flags;
 	kern_return_t    kret;
 	struct clios     iostate;
-	int              error;
-	int		 devblocksize;
+	int              error= 0;
+	int		 cur_upl = 0;
+	int		 num_upl = 0;
+	int		 n;
+        u_int32_t	 xsize;
+	u_int32_t	 io_size;
+	u_int32_t	 devblocksize;
+	u_int32_t	 mem_alignment_mask;
+	u_int32_t	 tail_size = 0;
+	int              bflag;
+
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
-	devblocksize = vp->v_mount->mnt_devblocksize;
 	/*
 	 * When we enter this routine, we know
-	 *  -- the resid will not exceed iov_len
-	 *  -- the target address is physically contiguous
+	 *  -- the read_length will not exceed the current iov_len
+	 *  -- the target address is physically contiguous for read_length
 	 */
+	cluster_syncup(vp, filesize, callback, callback_arg);
 
-#if LP64_DEBUG
-	if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
-		panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__); 
-	}
-#endif /* LP64_DEBUG */
+	devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
+	mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
 
-	iov_len = uio_curriovlen(uio);
-	iov_base = uio_curriovbase(uio);
+	iostate.io_completed = 0;
+	iostate.io_issued = 0;
+	iostate.io_error = 0;
+	iostate.io_wanted = 0;
+
+next_cread:
+	io_size = *read_length;
 
 	max_size = filesize - uio->uio_offset;
 
-	// LP64todo - fix this!
-	if (max_size < 0 || (u_int64_t)max_size > iov_len)
-	        io_size = iov_len;
-	else
+	if (io_size > max_size)
 	        io_size = max_size;
 
-	// LP64todo - fix this!
-	upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
+	iov_base = uio_curriovbase(uio);
+
+	upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
 	upl_needed_size = upl_offset + io_size;
 
-	error       = 0;
 	pages_in_pl = 0;
 	upl_size = upl_needed_size;
 	upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
 
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START,
+		     (int)upl_offset, (int)upl_size, (int)iov_base, io_size, 0);
+
 	kret = vm_map_get_upl(current_map(),
 			      (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
-			      &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, 0);
+			      &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END,
+		     (int)upl_offset, upl_size, io_size, kret, 0);
 
 	if (kret != KERN_SUCCESS) {
 	        /*
-		 * cluster_phys_read: failed to get pagelist
+		 * failed to get pagelist
 		 */
-	        return(EINVAL);
+	        error = EINVAL;
+		goto wait_for_creads;
 	}
+	num_upl++;
+
 	if (upl_size < upl_needed_size) {
 	        /*
 		 * The upl_size wasn't satisfied.
 		 */
-	        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
-
-		return(EINVAL);
+	        error = EINVAL;
+		goto wait_for_creads;
 	}
-	pl = ubc_upl_pageinfo(upl);
+	pl = ubc_upl_pageinfo(upl[cur_upl]);
 
 	dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
 
 	while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
-	        int   head_size;
+	        u_int32_t   head_size;
 
-		head_size = devblocksize - (int)(uio->uio_offset & (devblocksize - 1));
+		head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
 
 		if (head_size > io_size)
 		        head_size = io_size;
 
-		error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ);
+		error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ, callback, callback_arg);
 
-		if (error) {
-		        ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
+		if (error)
+			goto wait_for_creads;
 
-			return(EINVAL);
-		}
 		upl_offset += head_size;
 		dst_paddr  += head_size;
 		io_size    -= head_size;
+
+		iov_base   += head_size;
+	}
+	if ((u_int32_t)iov_base & mem_alignment_mask) {
+	        /*
+		 * request doesn't set up on a memory boundary
+		 * the underlying DMA engine can handle...
+		 * return an error instead of going through
+		 * the slow copy path since the intent of this
+		 * path is direct I/O to device memory
+		 */
+	        error = EINVAL;
+		goto wait_for_creads;
 	}
+
 	tail_size = io_size & (devblocksize - 1);
-	io_size  -= tail_size;
 
-	iostate.io_completed = 0;
-	iostate.io_issued = 0;
-	iostate.io_error = 0;
-	iostate.io_wanted = 0;
+	io_size  -= tail_size;
 
 	while (io_size && error == 0) {
-	        int  xsize;
 
-		if (io_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
-		        xsize = MAX_UPL_TRANSFER * PAGE_SIZE;
+		if (io_size > MAX_IO_CONTIG_SIZE)
+		        xsize = MAX_IO_CONTIG_SIZE;
 		else
 		        xsize = io_size;
 		/*
@@ -3496,17 +4025,27 @@ cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
 		 * if there are already too many outstanding reads
 		 * wait until some have completed before issuing the next
 		 */
-		lck_mtx_lock(cl_mtxp);
-
-		while ((iostate.io_issued - iostate.io_completed) > (8 * MAX_UPL_TRANSFER * PAGE_SIZE)) {
-	                iostate.io_wanted = 1;
-			msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_phys_read", 0);
-		}	
-		lck_mtx_unlock(cl_mtxp);
+		if (iostate.io_issued) {
+		        lck_mtx_lock(cl_mtxp);
 
-	        error = cluster_io(vp, upl, upl_offset, uio->uio_offset, xsize, 
-				   CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC,
-				   (buf_t)NULL, &iostate);
+			while ((iostate.io_issued - iostate.io_completed) > (3 * MAX_IO_CONTIG_SIZE)) {
+			        iostate.io_wanted = 1;
+				msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
+			}	
+			lck_mtx_unlock(cl_mtxp);
+		}
+		if (iostate.io_error) {
+		        /*
+			 * one of the earlier reads we issued ran into a hard error
+			 * don't issue any more reads...
+			 * go wait for any other reads to complete before
+			 * returning the error to the caller
+			 */
+		        goto wait_for_creads;
+		}
+	        error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize, 
+				   CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC | bflag,
+				   (buf_t)NULL, &iostate, callback, callback_arg);
 	        /*
 		 * The cluster_io read was issued successfully,
 		 * update the uio structure
@@ -3519,6 +4058,18 @@ cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
 			io_size    -= xsize;
 		}
 	}
+	if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS && uio->uio_offset < filesize) {
+
+	        error = cluster_io_type(uio, read_type, read_length, 0);
+	  
+		if (error == 0 && *read_type == IO_CONTIG) {
+		        cur_upl++;
+			goto next_cread;
+		}
+	} else
+	        *read_type = IO_UNKNOWN;
+
+wait_for_creads:
 	/*
 	 * make sure all async reads that are part of this stream
 	 * have completed before we proceed
@@ -3527,7 +4078,7 @@ cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
 
 	while (iostate.io_issued != iostate.io_completed) {
 	        iostate.io_wanted = 1;
-		msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_phys_read", 0);
+		msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
 	}	
 	lck_mtx_unlock(cl_mtxp);
 
@@ -3535,24 +4086,96 @@ cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
 	        error = iostate.io_error;
 
 	if (error == 0 && tail_size)
-	        error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ);
+	        error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ, callback, callback_arg);
 
-	/*
-	 * just release our hold on the physically contiguous
-	 * region without changing any state
-	 */
-	ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
+	for (n = 0; n < num_upl; n++)
+	        /*
+		 * just release our hold on each physically contiguous
+		 * region without changing any state
+		 */
+	        ubc_upl_abort(upl[n], 0);
 	
 	return (error);
 }
 
 
+static int
+cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length)
+{
+        user_size_t	 iov_len;
+  	user_addr_t	 iov_base = 0;
+	upl_t            upl;
+	vm_size_t        upl_size;
+	int              upl_flags;
+	int		 retval = 0;
+
+        /*
+	 * skip over any emtpy vectors
+	 */
+        uio_update(uio, (user_size_t)0);
+
+	iov_len = uio_curriovlen(uio);
+
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_START, (int)uio, (int)iov_len, 0, 0, 0);
+
+	if (iov_len) {
+	        iov_base = uio_curriovbase(uio);
+	        /*
+		 * make sure the size of the vector isn't too big...
+		 * internally, we want to handle all of the I/O in
+		 * chunk sizes that fit in a 32 bit int
+		 */
+	        if (iov_len > (user_size_t)MAX_IO_REQUEST_SIZE)
+		        upl_size = MAX_IO_REQUEST_SIZE;
+		else
+		        upl_size = (u_int32_t)iov_len;
+
+		upl_flags = UPL_QUERY_OBJECT_TYPE;
+  
+		if ((vm_map_get_upl(current_map(),
+				    (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
+				    &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
+		        /*
+			 * the user app must have passed in an invalid address
+			 */
+		        retval = EFAULT;
+		}
+		if (upl_size == 0)
+		        retval = EFAULT;
+
+		*io_length = upl_size;
+
+		if (upl_flags & UPL_PHYS_CONTIG)
+		        *io_type = IO_CONTIG;
+		else if (iov_len >= min_length)
+		        *io_type = IO_DIRECT;
+		else
+		        *io_type = IO_COPY;
+	} else {
+	        /*
+		 * nothing left to do for this uio
+		 */
+	        *io_length = 0;
+		*io_type   = IO_UNKNOWN;
+	}
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_END, (int)iov_base, *io_type, *io_length, retval, 0);
+
+	return (retval);
+}
+
+
 /*
  * generate advisory I/O's in the largest chunks possible
  * the completed pages will be released into the VM cache
  */
 int
 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
+{
+        return advisory_read_ext(vp, filesize, f_offset, resid, NULL, NULL, CL_PASSIVE);
+}
+
+int
+advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
 {
 	upl_page_info_t *pl;
 	upl_t            upl;
@@ -3696,7 +4319,7 @@ advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
 				 * issue an asynchronous read to cluster_io
 				 */
 				retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
-						    CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE, (buf_t)NULL, (struct clios *)NULL);
+						    CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 
 				issued_io = 1;
 			}
@@ -3721,6 +4344,13 @@ advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
 
 int
 cluster_push(vnode_t vp, int flags)
+{
+        return cluster_push_ext(vp, flags, NULL, NULL);
+}
+
+
+int
+cluster_push_ext(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
         int	retval;
 	struct	cl_writebehind *wbp;
@@ -3747,16 +4377,16 @@ cluster_push(vnode_t vp, int flags)
 		     (int)wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
 
 	if (wbp->cl_scmap) {
-	        sparse_cluster_push(wbp, vp, ubc_getsize(vp), 1);
+		sparse_cluster_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
 
 		retval = 1;
 	} else 
-	        retval = cluster_try_push(wbp, vp, ubc_getsize(vp), 0, 1);
+		retval = cluster_try_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
 
 	lck_mtx_unlock(&wbp->cl_lockw);
 
 	if (flags & IO_SYNC)
-	        (void)vnode_waitforwrites(vp, 0, 0, 0, (char *)"cluster_push");
+	        (void)vnode_waitforwrites(vp, 0, 0, 0, "cluster_push");
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
 		     (int)wbp->cl_scmap, wbp->cl_number, retval, 0, 0);
@@ -3798,30 +4428,8 @@ cluster_release(struct ubc_info *ubc)
 }
 
 
-static void
-cluster_push_EOF(vnode_t vp, off_t EOF)
-{
-        struct cl_writebehind *wbp;
-
-	wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
-
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
-		     (int)wbp->cl_scmap, wbp->cl_number, (int)EOF, 0, 0);
-
-	if (wbp->cl_scmap)
-	        sparse_cluster_push(wbp, vp, EOF, 1);
-	else 
-	        cluster_try_push(wbp, vp, EOF, 0, 1);
-
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
-		     (int)wbp->cl_scmap, wbp->cl_number, 0, 0, 0);
-
-	lck_mtx_unlock(&wbp->cl_lockw);
-}
-
-
 static int
-cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_delay, int push_all)
+cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
 {
         int cl_index;
 	int cl_index1;
@@ -3833,7 +4441,15 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 	/*
 	 * the write behind context exists and has
 	 * already been locked...
-	 *
+	 */
+	if (wbp->cl_number == 0)
+	        /*
+		 * no clusters to push
+		 * return number of empty slots
+		 */
+	        return (MAX_CLUSTERS);
+	 
+	/*
 	 * make a local 'sorted' copy of the clusters
 	 * and clear wbp->cl_number so that new clusters can
 	 * be developed
@@ -3851,7 +4467,7 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 		        break;
 	        l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
 		l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
-		l_clusters[cl_index].io_nocache = wbp->cl_clusters[min_index].io_nocache;
+		l_clusters[cl_index].io_flags = wbp->cl_clusters[min_index].io_flags;
 
 	        wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
 	}
@@ -3859,7 +4475,7 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 
 	cl_len = cl_index;
 
-	if (can_delay && cl_len == MAX_CLUSTERS) {
+	if ( (push_flag & PUSH_DELAY) && cl_len == MAX_CLUSTERS ) {
 		int   i;
 		
 		/*
@@ -3869,7 +4485,7 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 		 * used for managing more random I/O patterns
 		 *
 		 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
-		 * that's why we're in try_push with can_delay true...
+		 * that's why we're in try_push with PUSH_DELAY...
 		 *
 		 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
 		 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
@@ -3882,7 +4498,7 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 		 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
 		 */
 		for (i = 0; i < MAX_CLUSTERS - 1; i++) {
-		        if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != MAX_UPL_TRANSFER)
+		        if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != MAX_CLUSTER_SIZE)
 			        goto dont_try;
 			if (l_clusters[i].e_addr != l_clusters[i+1].b_addr)
 		                goto dont_try;
@@ -3893,31 +4509,43 @@ cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_dela
 	 * this is safe since I'm working off of a private sorted copy
 	 * of the clusters, and I'm going to re-evaluate the public
 	 * state after I retake the lock
+	 *
+	 * we need to drop it to avoid a lock inversion when trying to
+	 * grab pages into the UPL... another thread in 'write' may
+	 * have these pages in its UPL and be blocked trying to
+	 * gain the write-behind lock for this vnode
 	 */
 	lck_mtx_unlock(&wbp->cl_lockw);
 
 	for (cl_index = 0; cl_index < cl_len; cl_index++) {
-	        int flags;
-		struct cl_extent cl;
+	        int	flags;
+		struct	cl_extent cl;
 
 	        /*
 		 * try to push each cluster in turn...
 		 */
-		if (l_clusters[cl_index].io_nocache)
+		if (l_clusters[cl_index].io_flags & CLW_IONOCACHE)
 		        flags = IO_NOCACHE;
 		else
 		        flags = 0;
+
+		if ((l_clusters[cl_index].io_flags & CLW_IOPASSIVE) || (push_flag & IO_PASSIVE))
+		        flags |= IO_PASSIVE;
+
+		if (push_flag & PUSH_SYNC)
+		        flags |= IO_SYNC;
+
 		cl.b_addr = l_clusters[cl_index].b_addr;
 		cl.e_addr = l_clusters[cl_index].e_addr;
 
-	        cluster_push_x(vp, &cl, EOF, flags);
+	        cluster_push_now(vp, &cl, EOF, flags, callback, callback_arg);
 
 		l_clusters[cl_index].b_addr = 0;
 		l_clusters[cl_index].e_addr = 0;
 
 		cl_pushed++;
 
-		if (push_all == 0)
+		if ( !(push_flag & PUSH_ALL) )
 		        break;
 	}
 	lck_mtx_lock(&wbp->cl_lockw);
@@ -3938,14 +4566,14 @@ dont_try:
 			 *
 			 * collect the active public clusters...
 			 */
-		        sparse_cluster_switch(wbp, vp, EOF);
+		        sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
 
 		        for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
 			        if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
 				        continue;
 			        wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
 				wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
-				wbp->cl_clusters[cl_index1].io_nocache = l_clusters[cl_index].io_nocache;
+				wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
 
 				cl_index1++;
 			}
@@ -3958,7 +4586,7 @@ dont_try:
 			 * and collect the original clusters that were moved into the 
 			 * local storage for sorting purposes
 			 */
-		        sparse_cluster_switch(wbp, vp, EOF);
+		        sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
 
 		} else {
 		        /*
@@ -3972,7 +4600,7 @@ dont_try:
 
 			        wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
 				wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
-				wbp->cl_clusters[cl_index1].io_nocache = l_clusters[cl_index].io_nocache;
+				wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
 
 				cl_index1++;
 			}
@@ -3982,13 +4610,13 @@ dont_try:
 			wbp->cl_number = cl_index1;
 		}
 	}
-	return(MAX_CLUSTERS - wbp->cl_number);
+	return (MAX_CLUSTERS - wbp->cl_number);
 }
 
 
 
 static int
-cluster_push_x(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags)
+cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
 	upl_page_info_t *pl;
 	upl_t            upl;
@@ -4001,11 +4629,16 @@ cluster_push_x(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags)
 	int              io_size;
 	int              io_flags;
 	int              upl_flags;
+	int              bflag;
 	int              size;
 	int              error = 0;
 	int              retval;
 	kern_return_t    kret;
 
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
 		     (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
@@ -4127,13 +4760,13 @@ cluster_push_x(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags)
 
 		io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
 
-		io_flags = CL_THROTTLE | CL_COMMIT;
+		io_flags = CL_THROTTLE | CL_COMMIT | CL_AGE | bflag;
 
 		if ( !(flags & IO_SYNC))
 		        io_flags |= CL_ASYNC;
 
 		retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
-				    io_flags, (buf_t)NULL, (struct clios *)NULL);
+				    io_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 
 		if (error == 0 && retval)
 		        error = retval;
@@ -4150,7 +4783,7 @@ cluster_push_x(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags)
  * sparse_cluster_switch is called with the write behind lock held
  */
 static void
-sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF)
+sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
 {
         int	cl_index;
 
@@ -4165,11 +4798,11 @@ sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF)
 
 	        for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
 
-		        if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, 0, &flags) == KERN_SUCCESS) {
+		        if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, NULL, &flags) == KERN_SUCCESS) {
 			        if (flags & UPL_POP_DIRTY) {
 				        cl.e_addr = cl.b_addr + 1;
 
-				        sparse_cluster_add(wbp, vp, &cl, EOF);
+				        sparse_cluster_add(wbp, vp, &cl, EOF, callback, callback_arg);
 				}
 			}
 		}
@@ -4184,15 +4817,15 @@ sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF)
  * sparse_cluster_push is called with the write behind lock held
  */
 static void
-sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_all)
+sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
 {
         struct cl_extent cl;
         off_t		offset;
 	u_int		length;
 
-	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, push_all, 0);
+	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, push_flag, 0);
 
-	if (push_all)
+	if (push_flag & PUSH_ALL)
 	        vfs_drt_control(&(wbp->cl_scmap), 1);
 
 	for (;;) {
@@ -4204,9 +4837,24 @@ sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_
 
 		wbp->cl_scdirty -= (int)(cl.e_addr - cl.b_addr);
 
-		cluster_push_x(vp, &cl, EOF, 0);
+		/*
+		 * drop the lock while we're firing off the I/Os...
+		 * this is safe since I've already updated the state
+		 * this lock is protecting and I'm going to re-evaluate
+		 * the public state after I retake the lock
+		 *
+		 * we need to drop it to avoid a lock inversion when trying to
+		 * grab pages into the UPL... another thread in 'write' may
+		 * have these pages in its UPL and be blocked trying to
+		 * gain the write-behind lock for this vnode
+		 */
+		lck_mtx_unlock(&wbp->cl_lockw);
+
+		cluster_push_now(vp, &cl, EOF, push_flag & IO_PASSIVE, callback, callback_arg);
+
+		lck_mtx_lock(&wbp->cl_lockw);
 
-		if (push_all == 0)
+		if ( !(push_flag & PUSH_ALL) )
 		        break;
 	}
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
@@ -4217,7 +4865,7 @@ sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_
  * sparse_cluster_add is called with the write behind lock held
  */
 static void
-sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl, off_t EOF)
+sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
 {
         u_int	new_dirty;
 	u_int	length;
@@ -4236,7 +4884,7 @@ sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl,
 		 */
 	        wbp->cl_scdirty += new_dirty;
 
-	        sparse_cluster_push(wbp, vp, EOF, 0);
+	        sparse_cluster_push(wbp, vp, EOF, 0, callback, callback_arg);
 
 		offset += (new_dirty * PAGE_SIZE_64);
 		length -= (new_dirty * PAGE_SIZE);
@@ -4248,7 +4896,7 @@ sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl,
 
 
 static int
-cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize, int flags)
+cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
 {
         upl_page_info_t  *pl;
         upl_t            upl;
@@ -4258,17 +4906,30 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize
 	int		 did_read = 0;
 	int		 abort_flags;
 	int		 upl_flags;
+	int              bflag;
+
+	if (flags & IO_PASSIVE)
+	    bflag = CL_PASSIVE;
+	else
+	    bflag = 0;
 
 	upl_flags = UPL_SET_LITE;
-	if (! (flags & CL_READ)) {
+
+	if ( !(flags & CL_READ) ) {
 		/*
 		 * "write" operation:  let the UPL subsystem know
 		 * that we intend to modify the buffer cache pages
 		 * we're gathering.
 		 */
 		upl_flags |= UPL_WILL_MODIFY;
+	} else {
+	        /*
+		 * indicate that there is no need to pull the
+		 * mapping for this page... we're only going
+		 * to read from it, not modify it.
+		 */
+		upl_flags |= UPL_FILE_IO;
 	}
-
         kret = ubc_create_upl(vp,
                               uio->uio_offset & ~PAGE_MASK_64,
                               PAGE_SIZE,
@@ -4284,7 +4945,7 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize
                  * issue a synchronous read to cluster_io
                  */
                 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
-				   CL_READ, (buf_t)NULL, (struct clios *)NULL);
+				   CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
                 if (error) {
                           ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
 
@@ -4312,9 +4973,9 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize
 		 * issue a synchronous write to cluster_io
 		 */
 		error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
-					0, (buf_t)NULL, (struct clios *)NULL);
+				   bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
 	}
-	if (error == 0)
+	if (error == 0) 
 	        uio_update(uio, (user_size_t)xsize);
 
 	if (did_read)
@@ -4330,17 +4991,20 @@ cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize
 
 
 int
-cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int xsize)
+cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
 {
         int       pg_offset;
 	int       pg_index;
         int   	  csize;
 	int       segflg;
 	int       retval = 0;
+	int	  xsize;
 	upl_page_info_t *pl;
 
+	xsize = *io_resid;
+
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
-		     (int)uio->uio_offset, uio_resid(uio), upl_offset, xsize, 0);
+		     (int)uio->uio_offset, upl_offset, xsize, 0, 0);
 
 	segflg = uio->uio_segflg;
 
@@ -4391,10 +5055,12 @@ cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int xsize)
 		xsize    -= csize;
 		csize     = min(PAGE_SIZE, xsize);
 	}
+	*io_resid = xsize;
+
 	uio->uio_segflg = segflg;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
-		     (int)uio->uio_offset, uio_resid(uio), retval, segflg, 0);
+		     (int)uio->uio_offset, xsize, retval, segflg, 0);
 
 	return (retval);
 }
@@ -4402,6 +5068,14 @@ cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int xsize)
 
 int
 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
+{
+
+	return (cluster_copy_ubc_data_internal(vp, uio, io_resid, mark_dirty, 1));
+}
+
+
+static int
+cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference)
 {
 	int       segflg;
 	int       io_size;
@@ -4410,14 +5084,16 @@ cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty
 	int       retval = 0;
 	memory_object_control_t	 control;
 
+	io_size = *io_resid;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
-		     (int)uio->uio_offset, uio_resid(uio), 0, *io_resid, 0);
+		     (int)uio->uio_offset, 0, io_size, 0, 0);
 
 	control = ubc_getobject(vp, UBC_FLAGS_NONE);
+
 	if (control == MEMORY_OBJECT_CONTROL_NULL) {
 		KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
-			     (int)uio->uio_offset, uio_resid(uio), retval, 3, 0);
+			     (int)uio->uio_offset, io_size, retval, 3, 0);
 
 		return(0);
 	}
@@ -4457,8 +5133,8 @@ cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty
 	        start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
 		xsize = uio_resid(uio);
 
-		retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset,
-						       uio, start_offset, io_size, mark_dirty);
+		retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset, uio,
+						       start_offset, io_size, mark_dirty, take_reference);
 		xsize -= uio_resid(uio);
 		io_size -= xsize;
 	}
@@ -4466,7 +5142,7 @@ cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty
 	*io_resid       = io_size;
 
 	KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
-		     (int)uio->uio_offset, uio_resid(uio), retval, 0x80000000 | segflg, 0);
+		     (int)uio->uio_offset, io_size, retval, 0x80000000 | segflg, 0);
 
 	return(retval);
 }
@@ -4480,7 +5156,7 @@ is_file_clean(vnode_t vp, off_t filesize)
 	int   total_dirty = 0;
 
 	for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
-	        if (ubc_page_op(vp, f_offset, 0, 0, &flags) == KERN_SUCCESS) {
+	        if (ubc_page_op(vp, f_offset, 0, NULL, &flags) == KERN_SUCCESS) {
 		        if (flags & UPL_POP_DIRTY) {
 			        total_dirty++;
 			}
@@ -4667,7 +5343,7 @@ static kern_return_t	vfs_drt_do_mark_pages(
 	void		**cmapp,
 	u_int64_t	offset,
 	u_int    	length,
-	int		*setcountp,
+	u_int		*setcountp,
 	int		dirty);
 static void		vfs_drt_trace(
 	struct vfs_drt_clustermap *cmap,
@@ -4692,7 +5368,8 @@ vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
 	struct vfs_drt_clustermap *cmap, *ocmap;
 	kern_return_t	kret;
 	u_int64_t	offset;
-	int		nsize, i, active_buckets, index, copycount;
+	u_int32_t	i;
+	int		nsize, active_buckets, index, copycount;
 
 	ocmap = NULL;
 	if (cmapp != NULL)
@@ -4771,6 +5448,7 @@ vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
 			if (kret != KERN_SUCCESS) {
 				/* XXX need to bail out gracefully here */
 				panic("vfs_drt: new cluster map mysteriously too small");
+				index = 0;
 			}
 			/* copy */
 			DRT_HASH_COPY(ocmap, i, cmap, index);
@@ -4819,7 +5497,8 @@ vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
 static kern_return_t
 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
 {
-	int		index, i;
+	int		index;
+	u_int32_t	i;
 
 	offset = DRT_ALIGN_ADDRESS(offset);
 	index = DRT_HASH(cmap, offset);
@@ -4863,7 +5542,8 @@ vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *inde
 {
 	struct vfs_drt_clustermap *cmap;
 	kern_return_t	kret;
-	int		index, i;
+	u_int32_t	index;
+	u_int32_t	i;
 
 	cmap = *cmapp;
 
@@ -4918,7 +5598,7 @@ vfs_drt_do_mark_pages(
 	void		**private,
 	u_int64_t	offset,
 	u_int    	length,
-	int		*setcountp,
+	u_int		*setcountp,
 	int		dirty)
 {
 	struct vfs_drt_clustermap *cmap, **cmapp;
@@ -5033,7 +5713,7 @@ vfs_drt_do_mark_pages(
  * Returns KERN_SUCCESS if all the pages were successfully marked.
  */
 static kern_return_t
-vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, int *setcountp)
+vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp)
 {
 	/* XXX size unused, drop from interface */
 	return(vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1));
@@ -5073,7 +5753,8 @@ vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
 	struct vfs_drt_clustermap *cmap;
 	u_int64_t	offset;
 	u_int		length;
-	int		index, i, j, fs, ls;
+	u_int32_t	j;
+	int		index, i, fs, ls;
 
 	/* sanity */
 	if ((cmapp == NULL) || (*cmapp == NULL))
diff --git a/bsd/vfs/vfs_conf.c b/bsd/vfs/vfs_conf.c
index 722e6d1f6..2727afe16 100644
--- a/bsd/vfs/vfs_conf.c
+++ b/bsd/vfs/vfs_conf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -54,6 +60,12 @@
  *
  *	@(#)vfs_conf.c	8.11 (Berkeley) 5/10/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -65,103 +77,109 @@
  */
 struct mount *rootfs;
 struct vnode *rootvnode;
-int (*mountroot)() = NULL;
+int (*mountroot)(void) = NULL;
 
 /*
  * Set up the initial array of known filesystem types.
  */
 extern	struct vfsops ufs_vfsops;
 #if FFS
-extern	int ffs_mountroot();
+extern	int ffs_mountroot(mount_t, vnode_t, vfs_context_t);
 #endif
 extern	struct vfsops mfs_vfsops;
-extern	int mfs_mountroot();
+extern	int mfs_mountroot(mount_t, vnode_t, vfs_context_t);	/* dead */
 extern  struct vfsops hfs_vfsops;
-extern	int hfs_mountroot();
-extern  struct vfsops volfs_vfsops;
+extern	int hfs_mountroot(mount_t, vnode_t, vfs_context_t);
 extern	struct vfsops cd9660_vfsops;
-extern	int cd9660_mountroot();
+extern	int cd9660_mountroot(mount_t, vnode_t, vfs_context_t);
 extern	struct vfsops nfs_vfsops;
-extern	int nfs_mountroot();
+extern	int nfs_mountroot(void);
 extern	struct vfsops afs_vfsops;
 extern	struct vfsops null_vfsops;
 extern	struct vfsops union_vfsops;
 extern	struct vfsops fdesc_vfsops;
 extern	struct vfsops devfs_vfsops;
 
+/*
+ * For nfs_mountroot(void) cast.  nfs_mountroot ignores its parameters, if
+ * invoked through this table.
+ */
+typedef int (*mountroot_t)(mount_t, vnode_t, vfs_context_t);
+
 /*
  * Set up the filesystem operations for vnodes.
  */
 static struct vfstable vfstbllist[] = {
 	/* HFS/HFS+ Filesystem */
 #if HFS
-	{ &hfs_vfsops, "hfs", 17, 0, MNT_LOCAL | MNT_DOVOLFS, hfs_mountroot, NULL, 0, {0}, VFC_VFSLOCALARGS, 0, 0 },
+	{ &hfs_vfsops, "hfs", 17, 0, (MNT_LOCAL | MNT_DOVOLFS), hfs_mountroot, NULL, 1, {{0}}, VFC_VFSLOCALARGS | VFC_VFSREADDIR_EXTENDED, NULL, 0, 1},
 #endif
 
 	/* Fast Filesystem */
 #if FFS
-	{ &ufs_vfsops, "ufs", 1, 0, MNT_LOCAL, ffs_mountroot, NULL, 0, {0}, VFC_VFSLOCALARGS, 0, 0  },
+	{ &ufs_vfsops, "ufs", 1, 0, MNT_LOCAL, ffs_mountroot, NULL, 0, {{0}}, VFC_VFSLOCALARGS | VFC_VFSREADDIR_EXTENDED, NULL, 0, 0},
 #endif
 
 	/* ISO9660 (aka CDROM) Filesystem */
 #if CD9660
-	{ &cd9660_vfsops, "cd9660", 14, 0, MNT_LOCAL, cd9660_mountroot, NULL, 0, {0}, VFC_VFSLOCALARGS, 0, 0 },
+	{ &cd9660_vfsops, "cd9660", 14, 0, MNT_LOCAL, cd9660_mountroot, NULL, 0, {{0}}, VFC_VFSLOCALARGS, NULL, 0, 0},
 #endif
 
 	/* Memory-based Filesystem */
 #if MFS
-	{ &mfs_vfsops, "mfs", 3, 0, MNT_LOCAL, mfs_mountroot, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &mfs_vfsops, "mfs", 3, 0, MNT_LOCAL, mfs_mountroot, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
 #endif
 
 	/* Sun-compatible Network Filesystem */
 #if NFSCLIENT
-	{ &nfs_vfsops, "nfs", 2, 0, 0, nfs_mountroot, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &nfs_vfsops, "nfs", 2, 0, 0, (mountroot_t)nfs_mountroot, NULL, 1, {{0}}, VFC_VFSGENERICARGS|VFC_VFSPREFLIGHT, NULL, 0, 1},
 #endif
 
 	/* Andrew Filesystem */
 #if AFS
-	{ &afs_vfsops, "andrewfs", 13, 0, 0, afs_mountroot, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &afs_vfsops, "andrewfs", 13, 0, 0, afs_mountroot, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
 #endif
 
 	/* Loopback (Minimal) Filesystem Layer */
 #if NULLFS
-	{ &null_vfsops, "loopback", 9, 0, 0, NULL, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &null_vfsops, "loopback", 9, 0, 0, NULL, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
 #endif
 
 	/* Union (translucent) Filesystem */
 #if UNION
-	{ &union_vfsops, "union", 15, 0, 0, NULL, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &union_vfsops, "unionfs", 15, 0, 0, NULL, NULL, 1, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
 #endif
 
 	/* File Descriptor Filesystem */
 #if FDESC
-	{ &fdesc_vfsops, "fdesc", 7, 0, 0, NULL, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
-#endif
-
-	/* Volume ID Filesystem */
-#if VOLFS
-	{ &volfs_vfsops, "volfs", 18, 0, 0, NULL, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+	{ &fdesc_vfsops, "fdesc", 7, 0, 0, NULL, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
 #endif
 
 	/* Device Filesystem */
 #if DEVFS
-	{ &devfs_vfsops, "devfs", 19, 0, 0, NULL, NULL, 0, {0}, VFC_VFSGENERICARGS , 0, 0},
+#if CONFIG_MACF
+	{ &devfs_vfsops, "devfs", 19, 0, MNT_MULTILABEL, NULL, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
+#else
+	{ &devfs_vfsops, "devfs", 19, 0, 0, NULL, NULL, 0, {{0}}, VFC_VFSGENERICARGS , NULL, 0, 0},
+#endif /* MAC */
 #endif
 
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0},
-	{0}
+
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0},
+	{NULL, "<unassigned>", 0, 0, 0, NULL, NULL, 0, {{0}}, 0, NULL, 0, 0}
 };
 
 /*
@@ -187,17 +205,21 @@ extern struct vnodeopv_desc ffs_fifoop_opv_desc;
 #endif
 extern struct vnodeopv_desc mfs_vnodeop_opv_desc;
 extern struct vnodeopv_desc dead_vnodeop_opv_desc;
+#if FIFO && SOCKETS
 extern struct vnodeopv_desc fifo_vnodeop_opv_desc;
+#endif /* SOCKETS */
 extern struct vnodeopv_desc spec_vnodeop_opv_desc;
 extern struct vnodeopv_desc nfsv2_vnodeop_opv_desc;
 extern struct vnodeopv_desc spec_nfsv2nodeop_opv_desc;
 extern struct vnodeopv_desc fifo_nfsv2nodeop_opv_desc;
+extern struct vnodeopv_desc nfsv4_vnodeop_opv_desc;
+extern struct vnodeopv_desc spec_nfsv4nodeop_opv_desc;
+extern struct vnodeopv_desc fifo_nfsv4nodeop_opv_desc;
 extern struct vnodeopv_desc fdesc_vnodeop_opv_desc;
 extern struct vnodeopv_desc null_vnodeop_opv_desc;
 extern struct vnodeopv_desc hfs_vnodeop_opv_desc;
 extern struct vnodeopv_desc hfs_specop_opv_desc;
 extern struct vnodeopv_desc hfs_fifoop_opv_desc;
-extern struct vnodeopv_desc volfs_vnodeop_opv_desc;
 extern struct vnodeopv_desc cd9660_vnodeop_opv_desc;
 extern struct vnodeopv_desc cd9660_cdxaop_opv_desc;
 extern struct vnodeopv_desc cd9660_specop_opv_desc;
@@ -215,7 +237,7 @@ struct vnodeopv_desc *vfs_opv_descs[] = {
 #endif
 #endif
 	&dead_vnodeop_opv_desc,
-#if FIFO
+#if FIFO && SOCKETS
 	&fifo_vnodeop_opv_desc,
 #endif
 	&spec_vnodeop_opv_desc,
@@ -225,8 +247,11 @@ struct vnodeopv_desc *vfs_opv_descs[] = {
 #if NFSCLIENT
 	&nfsv2_vnodeop_opv_desc,
 	&spec_nfsv2nodeop_opv_desc,
+	&nfsv4_vnodeop_opv_desc,
+	&spec_nfsv4nodeop_opv_desc,
 #if FIFO
 	&fifo_nfsv2nodeop_opv_desc,
+	&fifo_nfsv4nodeop_opv_desc,
 #endif
 #endif
 #if FDESC
@@ -253,9 +278,6 @@ struct vnodeopv_desc *vfs_opv_descs[] = {
 #if UNION
 	&union_vnodeop_opv_desc,
 #endif
-#if VOLFS
-	&volfs_vnodeop_opv_desc,
-#endif
 #if DEVFS
 	&devfs_vnodeop_opv_desc,
 	&devfs_spec_vnodeop_opv_desc,
diff --git a/bsd/vfs/vfs_fsevents.c b/bsd/vfs/vfs_fsevents.c
index 801faa3c1..e955664a0 100644
--- a/bsd/vfs/vfs_fsevents.c
+++ b/bsd/vfs/vfs_fsevents.c
@@ -1,27 +1,37 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <stdarg.h>
+#include <libsa/stdlib.h>
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/fsevents.h>
+
+#if CONFIG_FSE
 #include <sys/namei.h>
 #include <sys/filedesc.h>
 #include <sys/kernel.h>
@@ -43,41 +53,50 @@
 #include <sys/filio.h>
 #include <kern/locks.h>
 #include <libkern/OSAtomic.h>
+#include <kern/zalloc.h>
+#include <mach/mach_time.h>
+#include <kern/thread_call.h>
+#include <kern/clock.h>
 
 #include <bsm/audit_kernel.h>
 #include <bsm/audit_kevents.h>
 
-// where all our structs and defines come from
-#include <sys/fsevents.h>
 
 
-typedef struct kfs_event_arg {
-    u_int16_t  type;
-    u_int16_t  len;
-    union {
-	struct vnode *vp;
-	char         *str;
-	void         *ptr;
-	int32_t       int32;
-	dev_t         dev;
-	ino_t         ino;
-	int32_t       mode;
-	uid_t         uid;
-	gid_t         gid;
-    } data;
-}kfs_event_arg;
-
-#define KFS_NUM_ARGS  FSE_MAX_ARGS
 typedef struct kfs_event {
-    int32_t        type;          // type code of this event
-    u_int32_t      refcount;      // number of clients referencing this
-    pid_t          pid;           // pid of the process that did the op
-    kfs_event_arg  args[KFS_NUM_ARGS];
+    LIST_ENTRY(kfs_event) kevent_list;
+    int16_t        type;           // type code of this event
+    u_int16_t      flags,          // per-event flags
+                   len;            // the length of the path in "str"
+    int32_t        refcount;       // number of clients referencing this
+    pid_t          pid;            // pid of the process that did the op
+
+    uint64_t       abstime;        // when this event happened (mach_absolute_time())
+    ino64_t        ino;
+    dev_t          dev;
+    int32_t        mode;
+    uid_t          uid;
+    gid_t          gid;
+
+    const char    *str;
+
+    struct kfs_event *dest;    // if this is a two-file op
 } kfs_event;
 
+// flags for the flags field
+#define KFSE_COMBINED_EVENTS          0x0001
+#define KFSE_CONTAINS_DROPPED_EVENTS  0x0002
+#define KFSE_RECYCLED_EVENT           0x0004
+#define KFSE_BEING_CREATED            0x0008
+
+LIST_HEAD(kfse_list, kfs_event) kfse_list_head = LIST_HEAD_INITIALIZER(x);
+int num_events_outstanding = 0;
+int num_pending_rename = 0;
+
+
+struct fsevent_handle;
 
 typedef struct fs_event_watcher {
-    SLIST_ENTRY(fs_event_watcher) link;
     int8_t      *event_list;             // the events we're interested in
     int32_t      num_events;
     dev_t       *devices_to_watch;       // only report events from these devices
@@ -85,23 +104,30 @@ typedef struct fs_event_watcher {
     int32_t      flags;
     kfs_event  **event_queue;
     int32_t      eventq_size;            // number of event pointers in queue
-    int32_t      rd, wr;                 // indices to the event_queue
-    int32_t      blockers;
     int32_t      num_readers;
+    int32_t      rd;                     // read index into the event_queue
+    int32_t      wr;                     // write index into the event_queue
+    int32_t      blockers;
+    int32_t      my_id;
+    uint32_t     num_dropped;
+    struct fsevent_handle *fseh;
 } fs_event_watcher;
 
 // fs_event_watcher flags
-#define WATCHER_DROPPED_EVENTS  0x0001
-#define WATCHER_CLOSING         0x0002
+#define WATCHER_DROPPED_EVENTS         0x0001
+#define WATCHER_CLOSING                0x0002
+#define WATCHER_WANTS_COMPACT_EVENTS   0x0004
+#define WATCHER_WANTS_EXTENDED_INFO    0x0008
+
 
-static SLIST_HEAD(watch_list, fs_event_watcher) watch_list_head = { NULL };
+#define MAX_WATCHERS  8
+static fs_event_watcher *watcher_table[MAX_WATCHERS];
 
 
-#define MAX_KFS_EVENTS   2048
+#define MAX_KFS_EVENTS   4096
 
-// this array holds each pending event 
-static kfs_event  fs_event_buf[MAX_KFS_EVENTS];
-static int        free_event_idx = 0;
+// we allocate kfs_event structures out of this zone
+static zone_t     event_zone;
 static int        fs_event_init = 0;
 
 //
@@ -112,6 +138,7 @@ static int        fs_event_init = 0;
 static int16_t     fs_event_type_watchers[FSE_MAX_EVENTS];
 
 static int  watcher_add_event(fs_event_watcher *watcher, kfs_event *kfse);
+static void fsevents_wakeup(fs_event_watcher *watcher);
 
 //
 // Locks
@@ -122,10 +149,10 @@ static lck_grp_t *       fsevent_mutex_group;
 
 static lck_grp_t *       fsevent_rw_group;
 
-static lck_rw_t  fsevent_big_lock;    // always grab this first
-static lck_mtx_t watch_list_lock;
+static lck_rw_t  event_handling_lock; // handles locking for event manipulation and recycling
+static lck_mtx_t watch_table_lock;
 static lck_mtx_t event_buf_lock;
-
+static lck_mtx_t event_writer_lock;
 
 static void init_pathbuff(void);
 
@@ -143,52 +170,65 @@ fsevents_internal_init(void)
 	fs_event_type_watchers[i] = 0;
     }
 
-    for(i=0; i < MAX_KFS_EVENTS; i++) {
-	fs_event_buf[i].type      = FSE_INVALID;
-	fs_event_buf[i].refcount  = 0;
-    }
-
-    SLIST_INIT(&watch_list_head);
+    memset(watcher_table, 0, sizeof(watcher_table));
 
     fsevent_lock_attr    = lck_attr_alloc_init();
     fsevent_group_attr   = lck_grp_attr_alloc_init();
     fsevent_mutex_group  = lck_grp_alloc_init("fsevent-mutex", fsevent_group_attr);
     fsevent_rw_group     = lck_grp_alloc_init("fsevent-rw", fsevent_group_attr);
 
-    lck_mtx_init(&watch_list_lock, fsevent_mutex_group, fsevent_lock_attr);
+    lck_mtx_init(&watch_table_lock, fsevent_mutex_group, fsevent_lock_attr);
     lck_mtx_init(&event_buf_lock, fsevent_mutex_group, fsevent_lock_attr);
+    lck_mtx_init(&event_writer_lock, fsevent_mutex_group, fsevent_lock_attr);
+
+    lck_rw_init(&event_handling_lock, fsevent_rw_group, fsevent_lock_attr);
 
-    lck_rw_init(&fsevent_big_lock, fsevent_rw_group, fsevent_lock_attr);
+    event_zone = zinit(sizeof(kfs_event),
+	               MAX_KFS_EVENTS * sizeof(kfs_event),
+	               MAX_KFS_EVENTS * sizeof(kfs_event),
+	               "fs-event-buf");
+    if (event_zone == NULL) {
+	printf("fsevents: failed to initialize the event zone.\n");
+    }
+
+    if (zfill(event_zone, MAX_KFS_EVENTS) != MAX_KFS_EVENTS) {
+	printf("fsevents: failed to pre-fill the event zone.\n");	
+    }
+    
+    // mark the zone as exhaustible so that it will not
+    // ever grow beyond what we initially filled it with
+    zone_change(event_zone, Z_EXHAUST, TRUE);
+    zone_change(event_zone, Z_COLLECT, FALSE);
 
     init_pathbuff();
 }
 
 static void
-lock_watch_list(void)
+lock_watch_table(void)
 {
-    lck_mtx_lock(&watch_list_lock);
+    lck_mtx_lock(&watch_table_lock);
 }
 
 static void
-unlock_watch_list(void)
+unlock_watch_table(void)
 {
-    lck_mtx_unlock(&watch_list_lock);
+    lck_mtx_unlock(&watch_table_lock);
 }
 
 static void
-lock_fs_event_buf(void)
+lock_fs_event_list(void)
 {
     lck_mtx_lock(&event_buf_lock);
 }
 
 static void
-unlock_fs_event_buf(void)
+unlock_fs_event_list(void)
 {
     lck_mtx_unlock(&event_buf_lock);
 }
 
 // forward prototype
-static void do_free_event(kfs_event *kfse);
+static void release_event_ref(kfs_event *kfse);
 
 static int
 watcher_cares_about_dev(fs_event_watcher *watcher, dev_t dev)
@@ -219,153 +259,757 @@ watcher_cares_about_dev(fs_event_watcher *watcher, dev_t dev)
 int
 need_fsevent(int type, vnode_t vp)
 {
-        fs_event_watcher *watcher;
-        dev_t dev;
+    if (type >= 0 && type < FSE_MAX_EVENTS && fs_event_type_watchers[type] == 0)
+	return (0);
+
+    // events in /dev aren't really interesting...
+    if (vp->v_tag == VT_DEVFS) {
+	return (0);
+    }
+
+    return 1;
+}
+
+static int
+prefix_match_len(const char *str1, const char *str2)
+{
+    int len=0;
+
+    while(*str1 && *str2 && *str1 == *str2) {
+	len++;
+	str1++;
+	str2++;
+    }
+
+    if (*str1 == '\0' && *str2 == '\0') {
+	len++;
+    }
+
+    return len;
+}
+
+
+struct history_item {
+    kfs_event *kfse;
+    kfs_event *oldest_kfse;
+    int        counter;
+};
+
+static int
+compare_history_items(const void *_a, const void *_b)
+{
+    const struct history_item *a = (const struct history_item *)_a;
+    const struct history_item *b = (const struct history_item *)_b;
+
+    // we want a descending order
+    return (b->counter - a->counter);
+}
+
+#define is_throw_away(x)  ((x) == FSE_STAT_CHANGED || (x) == FSE_CONTENT_MODIFIED)
 
-	if (fs_event_type_watchers[type] == 0)
-	        return (0);
-	dev = (dev_t)(vp->v_mount->mnt_vfsstat.f_fsid.val[0]);
 
-	lock_watch_list();
+// Ways that an event can be reused:
+//
+// "combined" events mean that there were two events for 
+// the same vnode or path and we're combining both events 
+// into a single event.  The primary event gets a bit that
+// marks it as having been combined.  The secondary event 
+// is essentially dropped and the kfse structure reused.
+//
+// "collapsed" means that multiple events below a given
+// directory are collapsed into a single event.  in this
+// case, the directory that we collapse into and all of
+// its children must be re-scanned.
+//
+// "recycled" means that we're completely blowing away 
+// the event since there are other events that have info 
+// about the same vnode or path (and one of those other 
+// events will be marked as combined or collapsed as
+// appropriate).
+//
+#define KFSE_COMBINED   0x0001
+#define KFSE_COLLAPSED  0x0002
+#define KFSE_RECYCLED   0x0004
+
+int num_dropped         = 0;
+int num_combined_events = 0;
+int num_added_to_parent = 0;
+int num_parent_switch   = 0;
+int num_recycled_rename = 0;
+
+//
+// NOTE: you must call lock_fs_event_list() before calling
+//       this function.
+//
+static kfs_event *
+find_an_event(const char *str, int len, kfs_event *do_not_reuse, int *reuse_type, int *longest_match_len)
+{
+    kfs_event *kfse, *best_kfse=NULL;
+
+// this seems to be enough to find most duplicate events for the same vnode
+#define MAX_HISTORY  12 
+    struct history_item history[MAX_HISTORY];
+    int           i;
+
+    *longest_match_len = 0;
+    *reuse_type = 0;
     
-	SLIST_FOREACH(watcher, &watch_list_head, link) {
-	        if (watcher->event_list[type] == FSE_REPORT && watcher_cares_about_dev(watcher, dev)) {
-		        unlock_watch_list();
-		        return (1);
+    memset(history, 0, sizeof(history));
+
+    //
+    // now walk the list of events and try to find the best match
+    // for this event.  if we have a vnode, we look for an event
+    // that already references the vnode.  if we don't find one
+    // we'll also take the parent of this vnode (in which case it
+    // will be marked as having dropped events within it).
+    //
+    // if we have a string we look for the longest match on the
+    // path we have.
+    //
+
+    LIST_FOREACH(kfse, &kfse_list_head, kevent_list) {
+	int match_len;
+
+	//
+	// don't look at events that are still in the process of being
+	// created, have a null vnode ptr or rename/exchange events.
+	//
+	if (   (kfse->flags & KFSE_BEING_CREATED) || kfse->type == FSE_RENAME || kfse->type == FSE_EXCHANGE) {
+
+	    continue;
+	}
+	
+	if (str != NULL) {
+	    if (kfse->len != 0 && kfse->str != NULL) {
+		match_len = prefix_match_len(str, kfse->str);
+		if (match_len > *longest_match_len) {
+		    best_kfse = kfse;
+		    *longest_match_len = match_len;
 		}
+	    }
+	}
+
+	if (kfse == do_not_reuse) {
+	    continue;
+	}
+
+	for(i=0; i < MAX_HISTORY; i++) {
+	    if (history[i].kfse == NULL) {
+		break;
+	    }
+
+	    //
+	    // do a quick check to see if we've got two simple events
+	    // that we can cheaply combine.  if the event we're looking
+	    // at and one of the events in the history table are for the
+	    // same path then we'll just mark the newer event as combined
+	    // and recyle the older event.
+	    //
+	    if (history[i].kfse->str == kfse->str) {
+
+		OSBitOrAtomic16(KFSE_COMBINED_EVENTS, &kfse->flags);
+		*reuse_type = KFSE_RECYCLED;
+		history[i].kfse->flags |= KFSE_RECYCLED_EVENT;
+		return history[i].kfse;
+	    }
+	}
+
+	if (i < MAX_HISTORY && history[i].kfse == NULL) {
+	    history[i].kfse = kfse;
+	    history[i].counter = 1;
+	} else if (i >= MAX_HISTORY) {
+	    qsort(history, MAX_HISTORY, sizeof(struct history_item), compare_history_items);
+
+	    // pluck off the lowest guy if he's only got a count of 1
+	    if (history[MAX_HISTORY-1].counter == 1) {
+		history[MAX_HISTORY-1].kfse = kfse;
+	    }
 	}
-	unlock_watch_list();
+    }
+
     
-	return (0);
+    if (str != NULL && best_kfse) {
+	if (*longest_match_len <= 1) {
+	    // if the best match we had was "/" then basically we're toast...
+	    *longest_match_len = 0;
+	    best_kfse = NULL;
+	} else if (*longest_match_len != len) {
+	    OSBitOrAtomic16(KFSE_CONTAINS_DROPPED_EVENTS, &best_kfse->flags);
+	    *reuse_type = KFSE_COLLAPSED;
+	} else {
+	    OSBitOrAtomic16(KFSE_COMBINED_EVENTS, &best_kfse->flags);
+	    *reuse_type = KFSE_COMBINED;
+	}
+    }
+
+    return best_kfse;
 }
 
 
+static struct timeval last_print;
+
+//
+// These variables are used to track coalescing multiple identical
+// events for the same vnode/pathname.  If we get the same event
+// type and same vnode/pathname as the previous event, we just drop
+// the event since it's superfluous.  This improves some micro-
+// benchmarks considerably and actually has a real-world impact on
+// tests like a Finder copy where multiple stat-changed events can
+// get coalesced.
+//
+static int     last_event_type=-1;
+static void   *last_ptr=NULL;
+static char    last_str[MAXPATHLEN];
+static int     last_nlen=0;
+static int     last_vid=-1;
+static uint64_t last_coalesced_time=0;
+int            last_coalesced = 0;
+static mach_timebase_info_data_t    sTimebaseInfo = { 0, 0 };
+
+
 int
 add_fsevent(int type, vfs_context_t ctx, ...) 
 {
     struct proc	     *p = vfs_context_proc(ctx);
-    int               i, arg_idx, num_deliveries=0;
-    kfs_event_arg    *kea;
-    kfs_event        *kfse;
+    int               i, arg_type, skip_init=0, longest_match_len, ret;
+    kfs_event        *kfse, *kfse_dest=NULL, *cur;
     fs_event_watcher *watcher;
     va_list           ap;
-    int 	      error = 0, base;
+    int 	      error = 0, did_alloc=0, need_event_unlock = 0;
     dev_t             dev = 0;
+    uint64_t          now, elapsed;
+    int               reuse_type = 0;
+    char             *pathbuff=NULL;
+    int               pathbuff_len;
+
 
     va_start(ap, ctx);
 
+    // ignore bogus event types..
+    if (type < 0 || type >= FSE_MAX_EVENTS) {
+	return EINVAL;
+    }
+
     // if no one cares about this type of event, bail out
     if (fs_event_type_watchers[type] == 0) {
 	va_end(ap);
 	return 0;
     }
 
-    lck_rw_lock_shared(&fsevent_big_lock);
+    now = mach_absolute_time();
 
     // find a free event and snag it for our use
     // NOTE: do not do anything that would block until
     //       the lock is dropped.
-    lock_fs_event_buf();
+    lock_fs_event_list();
     
-    base = free_event_idx;
-    for(i=0; i < MAX_KFS_EVENTS; i++) {
-	if (fs_event_buf[(base + i) % MAX_KFS_EVENTS].type == FSE_INVALID) {
-	    break;
+    //
+    // check if this event is identical to the previous one...
+    // (as long as it's not an event type that can never be the
+    // same as a previous event)
+    //
+    if (type != FSE_CREATE_FILE && type != FSE_DELETE && type != FSE_RENAME && type != FSE_EXCHANGE) {
+	void *ptr=NULL;
+	int   vid=0, was_str=0, nlen=0;
+
+	for(arg_type=va_arg(ap, int32_t); arg_type != FSE_ARG_DONE; arg_type=va_arg(ap, int32_t)) {
+	    switch(arg_type) {
+		case FSE_ARG_VNODE: {
+		    ptr = va_arg(ap, void *);
+		    vid = vnode_vid((struct vnode *)ptr);
+		    last_str[0] = '\0';
+		    break;
+		}
+		case FSE_ARG_STRING: {
+		    nlen = va_arg(ap, int32_t);
+		    ptr = va_arg(ap, void *);
+		    was_str = 1;
+		    break;
+		}
+	    }
+	    if (ptr != NULL) {
+		break;
+	    }
+	}
+
+	if ( sTimebaseInfo.denom == 0 ) {
+	    (void) clock_timebase_info(&sTimebaseInfo);
+	}
+	
+	elapsed = (now - last_coalesced_time);
+	if (sTimebaseInfo.denom != sTimebaseInfo.numer) {
+	    if (sTimebaseInfo.denom == 1) {
+		elapsed *= sTimebaseInfo.numer;
+	    } else {
+		// this could overflow... the worst that will happen is that we'll
+		// send (or not send) an extra event so I'm not going to worry about
+		// doing the math right like dtrace_abs_to_nano() does.
+		elapsed = (elapsed * sTimebaseInfo.numer) / (uint64_t)sTimebaseInfo.denom;
+	    }
+	}
+	
+	if (type == last_event_type
+            && (elapsed < 1000000000)
+	    && 
+	    ((vid && vid == last_vid && last_ptr == ptr)
+	      ||
+	     (last_str[0] && last_nlen == nlen && ptr && strcmp(last_str, ptr) == 0))
+	   ) {
+	    
+	    last_coalesced++;
+	    unlock_fs_event_list();
+	    va_end(ap);
+	    return 0;
+	} else {
+	    last_ptr = ptr;
+	    if (was_str) {
+		strlcpy(last_str, ptr, sizeof(last_str));
+	    }
+	    last_nlen = nlen;
+	    last_vid = vid;
+	    last_event_type = type;
+	    last_coalesced_time = now;
 	}
     }
+    va_start(ap, ctx);
+
+
+    kfse = zalloc_noblock(event_zone);
+    if (kfse && (type == FSE_RENAME || type == FSE_EXCHANGE)) {
+	kfse_dest = zalloc_noblock(event_zone);
+	if (kfse_dest == NULL) {
+	    did_alloc = 1;
+	    zfree(event_zone, kfse);
+	    kfse = NULL;
+	}
+    }
+
+
+    if (kfse == NULL) {        // yikes! no free events
+	int len=0;
+	char *str;
+	
+	//
+	// Figure out what kind of reference we have to the
+	// file in this event.  This helps us find an event
+	// to combine/collapse into to make room.
+	//
+	// If we have a rename or exchange event then we
+	// don't want to go through the normal path, we
+	// want to "steal" an event instead (which is what
+	// find_an_event() will do if str is null).
+	//
+	arg_type = va_arg(ap, int32_t);
+	if (type == FSE_RENAME || type == FSE_EXCHANGE) {
+	    str = NULL;
+	} else if (arg_type == FSE_ARG_STRING) {
+	    len = va_arg(ap, int32_t);
+	    str = va_arg(ap, char *);
+	} else if (arg_type == FSE_ARG_VNODE) {
+	    struct vnode *vp;
+
+	    vp  = va_arg(ap, struct vnode *);
+	    pathbuff = get_pathbuff();
+	    pathbuff_len = MAXPATHLEN;
+	    if (vn_getpath(vp, pathbuff, &pathbuff_len) != 0 || pathbuff[0] == '\0') {
+		release_pathbuff(pathbuff);
+		pathbuff = NULL;
+	    }
+	    str = pathbuff;
+	} else {
+	    str = NULL;
+	}
+
+	//
+	// This will go through all events and find one that we
+        // can combine with (hopefully), or "collapse" into (i.e
+	// it has the same parent) or in the worst case we have
+	// to "recycle" an event which means that it will combine
+	// two other events and return us the now unused event.
+	// failing all that, find_an_event() could still return
+	// null and if it does then we have a catastrophic dropped
+	// events scenario.
+	//
+	kfse = find_an_event(str, len, NULL, &reuse_type, &longest_match_len);
+
+	if (kfse == NULL) {
+	  bail_early:
+	    
+	    unlock_fs_event_list();
+	    lock_watch_table();
+
+	    for(i=0; i < MAX_WATCHERS; i++) {
+		watcher = watcher_table[i];
+		if (watcher == NULL) {
+		    continue;
+		}
+
+		watcher->flags |= WATCHER_DROPPED_EVENTS;
+		fsevents_wakeup(watcher);
+	    }
+	    unlock_watch_table();
+
+	    {	    
+		struct timeval current_tv;
+
+		num_dropped++;
+
+		// only print a message at most once every 5 seconds
+		microuptime(&current_tv);
+		if ((current_tv.tv_sec - last_print.tv_sec) > 10) {
+		    int ii;
+		    void *junkptr=zalloc_noblock(event_zone), *listhead=kfse_list_head.lh_first;
+		    
+		    printf("add_fsevent: event queue is full! dropping events (num dropped events: %d; num events outstanding: %d).\n", num_dropped, num_events_outstanding);
+		    printf("add_fsevent: kfse_list head %p ; num_pending_rename %d\n", listhead, num_pending_rename);
+		    printf("add_fsevent: zalloc sez: %p\n", junkptr);
+		    printf("add_fsevent: event_zone info: %d %p\n", ((int *)event_zone)[0], (void *)((int *)event_zone)[1]);
+		    for(ii=0; ii < MAX_WATCHERS; ii++) {
+			if (watcher_table[ii] == NULL) {
+			    continue;
+			}
+			
+			printf("add_fsevent: watcher %p: num dropped %d rd %4d wr %4d q_size %4d flags 0x%x\n",
+			    watcher_table[ii], watcher_table[ii]->num_dropped,
+			    watcher_table[ii]->rd, watcher_table[ii]->wr,
+			    watcher_table[ii]->eventq_size, watcher_table[ii]->flags);
+		    }
+
+		    last_print = current_tv;
+		    if (junkptr) {
+			zfree(event_zone, junkptr);
+		    }
+		}
+	    }
+
+	    if (pathbuff) {
+		release_pathbuff(pathbuff);
+		pathbuff = NULL;
+	    }
+
+	    return ENOSPC;
+	}
+
+	if ((type == FSE_RENAME || type == FSE_EXCHANGE) && reuse_type != KFSE_RECYCLED) {
+	    panic("add_fsevent: type == %d but reuse type == %d!\n", type, reuse_type);
+	} else if ((kfse->type == FSE_RENAME || kfse->type == FSE_EXCHANGE) && kfse->dest == NULL) {
+	    panic("add_fsevent: bogus kfse %p (type %d, but dest is NULL)\n", kfse, kfse->type);
+	} else if (kfse->type == FSE_RENAME || kfse->type == FSE_EXCHANGE) {
+	    panic("add_fsevent: we should never re-use rename events (kfse %p reuse type %d)!\n", kfse, reuse_type);
+	}
+
+	if (reuse_type == KFSE_COLLAPSED) {
+	    if (str) {
+		const char *tmp_ptr, *new_str;
+		
+		//
+		// if we collapsed and have a string we have to chop off the
+		// tail component of the pathname to get the parent.
+		//
+		// NOTE: it is VERY IMPORTANT that we leave the trailing slash
+		//       on the pathname.  user-level code depends on this.
+		//
+		if (str[0] == '\0' || longest_match_len <= 1) {
+		    printf("add_fsevent: strange state (str %s / longest_match_len %d)\n", str, longest_match_len);
+		    if (longest_match_len < 0) {
+			panic("add_fsevent: longest_match_len %d\n", longest_match_len);
+		    }
+		}
+		// chop off the tail component if it's not the
+		// first character...
+		if (longest_match_len > 1) {
+		    str[longest_match_len] = '\0';
+		} else if (longest_match_len == 0) {
+		    longest_match_len = 1;
+		}
+
+		new_str = vfs_addname(str, longest_match_len, 0, 0);
+		if (new_str == NULL || new_str[0] == '\0') {
+		    panic("add_fsevent: longest match is strange (new_str %p).\n", new_str);
+		}
+		
+		lck_rw_lock_exclusive(&event_handling_lock);
+
+		kfse->len      = longest_match_len;
+		tmp_ptr        = kfse->str;
+		kfse->str = new_str;
+		kfse->ino      = 0;
+		kfse->mode     = 0;
+		kfse->uid      = 0;
+		kfse->gid      = 0;
+		
+		lck_rw_unlock_exclusive(&event_handling_lock);
+		
+		vfs_removename(tmp_ptr);
+	    } else {
+		panic("add_fsevent: don't have a vnode or a string pointer (kfse %p)\n", kfse);
+	    }
+	}
+
+	if (reuse_type == KFSE_RECYCLED && (type == FSE_RENAME || type == FSE_EXCHANGE)) {
+	    
+	    // if we're recycling this kfse and we have a rename or
+	    // exchange event then we need to also get an event for
+	    // kfse_dest. 
+	    //
+	    if (did_alloc) {
+		// only happens if we allocated one but then failed
+		// for kfse_dest (and thus free'd the first one we
+		// allocated)
+		kfse_dest = zalloc_noblock(event_zone);
+		if (kfse_dest != NULL) {
+		    memset(kfse_dest, 0, sizeof(kfs_event));
+		    kfse_dest->refcount = 1;
+		    OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse_dest->flags);
+		} else {
+		    did_alloc = 0;
+		}
+	    }
+
+	    if (kfse_dest == NULL) {
+		int dest_reuse_type, dest_match_len;
+		
+		kfse_dest = find_an_event(NULL, 0, kfse, &dest_reuse_type, &dest_match_len);
+		
+		if (kfse_dest == NULL) {
+		    // nothing we can do... gotta bail out
+		    goto bail_early;
+		}
+
+		if (dest_reuse_type != KFSE_RECYCLED) {
+		    panic("add_fsevent: type == %d but dest_reuse type == %d!\n", type, dest_reuse_type);
+		}
+	    }
+	}
+
+
+	//
+	// Here we check for some fast-path cases so that we can
+	// jump over the normal initialization and just get on
+	// with delivering the event.  These cases are when we're
+	// combining/collapsing an event and so basically there is
+	// no more work to do (aside from a little book-keeping)
+	//
+	if (str && kfse->len != 0) {
+	    kfse->abstime = now;
+	    OSAddAtomic(1, (SInt32 *)&kfse->refcount);
+	    skip_init = 1;
+
+	    if (reuse_type == KFSE_COMBINED) {
+		num_combined_events++;
+	    } else if (reuse_type == KFSE_COLLAPSED) {
+		num_added_to_parent++;
+	    }
+	} else if (reuse_type != KFSE_RECYCLED) {
+	    panic("add_fsevent: I'm so confused! (reuse_type %d str %p kfse->len %d)\n",
+		  reuse_type, str, kfse->len);
+	}
 
-    if (i >= MAX_KFS_EVENTS) {
-	// yikes! no free slots
-	unlock_fs_event_buf();
 	va_end(ap);
 
-	lock_watch_list();
-	SLIST_FOREACH(watcher, &watch_list_head, link) {
-	    watcher->flags |= WATCHER_DROPPED_EVENTS;
-	    wakeup((caddr_t)watcher);
+
+	if (skip_init) {
+	    if (kfse->refcount < 1) {
+		panic("add_fsevent: line %d: kfse recount %d but should be at least 1\n", __LINE__, kfse->refcount);
+	    }
+
+	    unlock_fs_event_list();
+	    goto normal_delivery;
+	    
+	} else if (reuse_type == KFSE_RECYCLED || reuse_type == KFSE_COMBINED) {
+
+	    //
+	    // If we're here we have to clear out the kfs_event(s)
+	    // that we were given by find_an_event() and set it
+	    // up to be re-filled in by the normal code path.
+	    //
+	    va_start(ap, ctx);
+
+	    need_event_unlock = 1;
+	    lck_rw_lock_exclusive(&event_handling_lock);
+
+	    OSAddAtomic(1, (SInt32 *)&kfse->refcount);
+
+	    if (kfse->refcount < 1) {
+		panic("add_fsevent: line %d: kfse recount %d but should be at least 1\n", __LINE__, kfse->refcount);
+	    }
+
+	    if (kfse->len == 0) {
+		panic("%s:%d: no more fref.vp\n", __FILE__, __LINE__);
+		// vnode_rele_ext(kfse->fref.vp, O_EVTONLY, 0);
+	    } else {
+		vfs_removename(kfse->str);
+		kfse->len = 0;
+	    }
+	    kfse->str = NULL;
+
+	    if (kfse->kevent_list.le_prev != NULL) {
+		num_events_outstanding--;
+		if (kfse->type == FSE_RENAME) {
+		    num_pending_rename--;
+		}
+		LIST_REMOVE(kfse, kevent_list);
+		memset(&kfse->kevent_list, 0, sizeof(kfse->kevent_list));
+	    }
+
+	    kfse->flags = 0 | KFSE_RECYCLED_EVENT;
+	    
+	    if (kfse_dest) {
+		OSAddAtomic(1, (SInt32 *)&kfse_dest->refcount);
+		kfse_dest->flags = 0 | KFSE_RECYCLED_EVENT;
+
+		if (did_alloc == 0) {
+		    if (kfse_dest->len == 0) {
+			panic("%s:%d: no more fref.vp\n", __FILE__, __LINE__);
+			// vnode_rele_ext(kfse_dest->fref.vp, O_EVTONLY, 0);
+		    } else {
+			vfs_removename(kfse_dest->str);
+			kfse_dest->len = 0;
+		    }
+		    kfse_dest->str = NULL;
+
+		    if (kfse_dest->kevent_list.le_prev != NULL) {
+			num_events_outstanding--;
+			LIST_REMOVE(kfse_dest, kevent_list);
+			memset(&kfse_dest->kevent_list, 0, sizeof(kfse_dest->kevent_list));
+		    }
+
+		    if (kfse_dest->dest) {
+			panic("add_fsevent: should never recycle a rename event! kfse %p\n", kfse);
+		    }
+		}
+	    }
+
+	    OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse->flags);
+	    if (kfse_dest) {
+		OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse_dest->flags);
+	    }
+
+	    goto process_normally;
 	}
-	unlock_watch_list();
-	lck_rw_done(&fsevent_big_lock);
+    }
 
-	printf("fs_events: add_event: event queue is full! dropping events.\n");
-	return ENOSPC;
+    if (reuse_type != 0) {
+	panic("fsevents: we have a reuse_type (%d) but are about to clear out kfse %p\n", reuse_type, kfse);
     }
 
-    kfse = &fs_event_buf[(base + i) % MAX_KFS_EVENTS];
+    //
+    // we only want to do this for brand new events, not
+    // events which have been recycled.
+    //
+    memset(kfse, 0, sizeof(kfs_event));
+    kfse->refcount = 1;
+    OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse->flags);
 
-    free_event_idx = ((base + i) % MAX_KFS_EVENTS) + 1;
-    
+  process_normally:
     kfse->type     = type;
-    kfse->refcount = 1;
+    kfse->abstime  = now;
     kfse->pid      = p->p_pid;
+    if (type == FSE_RENAME || type == FSE_EXCHANGE) {
+	if (need_event_unlock == 0) {
+	    memset(kfse_dest, 0, sizeof(kfs_event));
+	    kfse_dest->refcount = 1;
+	    OSBitOrAtomic16(KFSE_BEING_CREATED, &kfse_dest->flags);
+	}
+	kfse_dest->type     = type;
+	kfse_dest->pid      = p->p_pid;
+	kfse_dest->abstime  = now;
+	
+	kfse->dest = kfse_dest;
+    }
+    
+    num_events_outstanding++;
+    if (kfse->type == FSE_RENAME) {
+	num_pending_rename++;
+    }
+    LIST_INSERT_HEAD(&kfse_list_head, kfse, kevent_list);
 
-    unlock_fs_event_buf();  // at this point it's safe to unlock
+    if (kfse->refcount < 1) {
+	panic("add_fsevent: line %d: kfse recount %d but should be at least 1\n", __LINE__, kfse->refcount);
+    }
+
+    unlock_fs_event_list();  // at this point it's safe to unlock
 
     //
     // now process the arguments passed in and copy them into
     // the kfse
     //
-    arg_idx = 0;
-    while(arg_idx < KFS_NUM_ARGS) {
-	kea = &kfse->args[arg_idx++];
-	kea->type = va_arg(ap, int32_t);
-
-	if (kea->type == FSE_ARG_DONE) {
-	    break;
-	}
+    if (need_event_unlock == 0) {
+	lck_rw_lock_shared(&event_handling_lock);
+    }
+    
+    cur = kfse;
+    for(arg_type=va_arg(ap, int32_t); arg_type != FSE_ARG_DONE; arg_type=va_arg(ap, int32_t))
 
-	switch(kea->type) {
+	switch(arg_type) {
 	    case FSE_ARG_VNODE: {
 		// this expands out into multiple arguments to the client
 		struct vnode *vp;
 		struct vnode_attr va;
 
-		kea->data.vp = vp = va_arg(ap, struct vnode *);
-		if (kea->data.vp == NULL) {
-		    panic("add_fsevent: you can't pass me a NULL vnode ptr (type %d)!\n",
-			  kfse->type);
+		if (kfse->str != NULL) {
+		    cur = kfse_dest;
 		}
 
-		if (vnode_ref_ext(kea->data.vp, O_EVTONLY) != 0) {
-		    kea->type = FSE_ARG_DONE;
-		    
-		    error = EINVAL;
-		    goto clean_up;
+		vp = va_arg(ap, struct vnode *);
+		if (vp == NULL) {
+		    panic("add_fsevent: you can't pass me a NULL vnode ptr (type %d)!\n",
+			  cur->type);
 		}
+
 		VATTR_INIT(&va);
 		VATTR_WANTED(&va, va_fsid);
 		VATTR_WANTED(&va, va_fileid);
 		VATTR_WANTED(&va, va_mode);
 		VATTR_WANTED(&va, va_uid);
 		VATTR_WANTED(&va, va_gid);
-		if (vnode_getattr(kea->data.vp, &va, ctx) != 0) {
-		    vnode_rele_ext(kea->data.vp, O_EVTONLY, 0);
-		    kea->type = FSE_ARG_DONE;
-
+		if ((ret = vnode_getattr(vp, &va, ctx)) != 0) {
+		    // printf("add_fsevent: failed to getattr on vp %p (%d)\n", cur->fref.vp, ret);
+		    cur->str = NULL;
 		    error = EINVAL;
+		    if (need_event_unlock == 0) {
+			// then we only grabbed it shared 
+			lck_rw_unlock_shared(&event_handling_lock);
+		    }
 		    goto clean_up;
 		}
 
-		kea++;
-		kea->type = FSE_ARG_DEV;
-		kea->data.dev = dev = (dev_t)va.va_fsid;
+		cur->dev  = dev = (dev_t)va.va_fsid;
+		cur->ino  = (ino_t)va.va_fileid;
+		cur->mode = (int32_t)vnode_vttoif(vnode_vtype(vp)) | va.va_mode;
+		cur->uid  = va.va_uid;
+		cur->gid  = va.va_gid;
 
-		kea++;
-		kea->type = FSE_ARG_INO;
-		kea->data.ino = (ino_t)va.va_fileid;
-
-		kea++;
-		kea->type = FSE_ARG_MODE;
-		kea->data.mode = (int32_t)vnode_vttoif(vnode_vtype(vp)) | va.va_mode;
+		// if we haven't gotten the path yet, get it.
+		if (pathbuff == NULL) {
+		    pathbuff = get_pathbuff();
+		    pathbuff_len = MAXPATHLEN;
+		    
+		    pathbuff[0] = '\0';
+		    if (vn_getpath(vp, pathbuff, &pathbuff_len) != 0 || pathbuff[0] == '\0') {
+			printf("add_fsevent: no name hard-link! dropping the event. (event %d vp == %p (%s)). \n",
+			       type, vp, vp->v_name ? vp->v_name : "-UNKNOWN-FILE");
+			error = ENOENT;
+			release_pathbuff(pathbuff);
+			pathbuff = NULL;
+			if (need_event_unlock == 0) {
+			    // then we only grabbed it shared 
+			    lck_rw_unlock_shared(&event_handling_lock);
+			}
+			goto clean_up;
+		    }
+		}
 
-		kea++;
-		kea->type = FSE_ARG_UID;
-		kea->data.uid = va.va_uid;
+		// store the path by adding it to the global string table
+		cur->len = pathbuff_len;
+		cur->str = vfs_addname(pathbuff, pathbuff_len, 0, 0);
+		if (cur->str == NULL || cur->str[0] == '\0') {
+		    panic("add_fsevent: was not able to add path %s to event %p.\n", pathbuff, cur);
+		}
+		
+		release_pathbuff(pathbuff);
+		pathbuff = NULL;
 
-		kea++;
-		kea->type = FSE_ARG_GID;
-		kea->data.gid = va.va_gid;
-		arg_idx += 5;
 		break;
 	    }
 
@@ -374,147 +1018,211 @@ add_fsevent(int type, vfs_context_t ctx, ...)
 		
 		fse = va_arg(ap, fse_info *);
 		
-		kea->type = FSE_ARG_DEV;
-		kea->data.dev = dev = (dev_t)fse->dev;
-
-		kea++;
-		kea->type = FSE_ARG_INO;
-		kea->data.ino = (ino_t)fse->ino;
-
-		kea++;
-		kea->type = FSE_ARG_MODE;
-		kea->data.mode = (int32_t)fse->mode;
-
-		kea++;
-		kea->type = FSE_ARG_UID;
-		kea->data.uid = (uid_t)fse->uid;
-
-		kea++;
-		kea->type = FSE_ARG_GID;
-		kea->data.gid = (uid_t)fse->gid;
-		arg_idx += 4;
+		cur->dev  = dev = (dev_t)fse->dev;
+		cur->ino  = (ino_t)fse->ino;
+		cur->mode = (int32_t)fse->mode;
+		cur->uid  = (uid_t)fse->uid;
+		cur->gid  = (uid_t)fse->gid;
+		// if it's a hard-link and this is the last link, flag it
+		if ((fse->mode & FSE_MODE_HLINK) && fse->nlink == 0) {
+		    cur->mode |= FSE_MODE_LAST_HLINK;
+		}
 		break;
 	    }
 
 	    case FSE_ARG_STRING:
-		kea->len      = (int16_t)(va_arg(ap, int32_t) & 0xffff);
-		kea->data.str = vfs_addname(va_arg(ap, char *), kea->len, 0, 0);
-		break;
-
-	    case FSE_ARG_INT32:
-		kea->data.int32 = va_arg(ap, int32_t);
-		break;
-
-	    case FSE_ARG_INT64:
-		printf("fs_events: 64-bit args not implemented.\n");
-//		kea->data.int64 = va_arg(ap, int64_t);
-		break;
+		if (kfse->str != NULL) {
+		    cur = kfse_dest;
+		}
 
-	    case FSE_ARG_RAW:
-		kea->len = (int16_t)(va_arg(ap, int32_t) & 0xffff);
-		MALLOC(kea->data.ptr, void *, kea->len, M_TEMP, M_WAITOK);
-		memcpy(kea->data.ptr, va_arg(ap, void *), kea->len);
+		cur->len = (int16_t)(va_arg(ap, int32_t) & 0x7fff);
+		if (cur->len >= 1) {
+		    cur->str = vfs_addname(va_arg(ap, char *), cur->len, 0, 0);
+		} else {
+		    printf("add_fsevent: funny looking string length: %d\n", (int)cur->len);
+		    cur->len = 2;
+		    cur->str = vfs_addname("/", cur->len, 0, 0);
+		}
+		if (cur->str[0] == 0) {
+		    printf("add_fsevent: bogus looking string (len %d)\n", cur->len);
+		}
 		break;
 
-	    case FSE_ARG_DEV:
-		kea->data.dev = dev = va_arg(ap, dev_t);
-		break;
-		
-	    case FSE_ARG_MODE:
-		kea->data.mode = va_arg(ap, int32_t);
-		break;
-		
-	    case FSE_ARG_INO:
-		kea->data.ino = va_arg(ap, ino_t);
-		break;
-		
-	    case FSE_ARG_UID:
-		kea->data.uid = va_arg(ap, uid_t);
-		break;
-		
-	    case FSE_ARG_GID:
-		kea->data.gid = va_arg(ap, gid_t);
-		break;
-		
 	    default:
-		printf("add_fsevent: unknown type %d\n", kea->type);
+		printf("add_fsevent: unknown type %d\n", arg_type);
 		// just skip one 32-bit word and hope we sync up...
 		(void)va_arg(ap, int32_t);
 	}
-    }
 
     va_end(ap);
 
+    OSBitAndAtomic16(~KFSE_BEING_CREATED, &kfse->flags);
+    if (kfse_dest) {
+	OSBitAndAtomic16(~KFSE_BEING_CREATED, &kfse_dest->flags);
+    }
+
+    if (need_event_unlock == 0) {
+	// then we only grabbed it shared 
+	lck_rw_unlock_shared(&event_handling_lock);
+    }
+    
+  normal_delivery:
+    // unlock this here so we don't hold it across the
+    // event delivery loop.
+    if (need_event_unlock) {
+	lck_rw_unlock_exclusive(&event_handling_lock);
+	need_event_unlock = 0;
+    }
+
     //
     // now we have to go and let everyone know that
-    // is interested in this type of event...
+    // is interested in this type of event
     //
-    lock_watch_list();
+    lock_watch_table();
     
-    SLIST_FOREACH(watcher, &watch_list_head, link) {
-	if (watcher->event_list[type] == FSE_REPORT && watcher_cares_about_dev(watcher, dev)) {
-	    if (watcher_add_event(watcher, kfse) == 0) {
-		num_deliveries++;
+    for(i=0; i < MAX_WATCHERS; i++) {
+	watcher = watcher_table[i];
+	if (watcher == NULL) {
+	    continue;
+	}
+	
+	if (   watcher->event_list[type] == FSE_REPORT
+	    && watcher_cares_about_dev(watcher, dev)) {
+	    
+	    if (watcher_add_event(watcher, kfse) != 0) {
+		watcher->num_dropped++;
 	    }
 	}
+
+	if (kfse->refcount < 1) {
+	    panic("add_fsevent: line %d: kfse recount %d but should be at least 1\n", __LINE__, kfse->refcount);
+	}
     }
 
-    unlock_watch_list();
-    
+    unlock_watch_table();
+
   clean_up:
-    // just in case no one was interested after all...
-    if (OSAddAtomic(-1, (SInt32 *)&kfse->refcount) == 1) {
-	do_free_event(kfse);
-    }	
+    // have to check if this needs to be unlocked (in
+    // case we came here from an error handling path)
+    if (need_event_unlock) {
+	lck_rw_unlock_exclusive(&event_handling_lock);
+	need_event_unlock = 0;
+    }
+
+    if (pathbuff) {
+	release_pathbuff(pathbuff);
+	pathbuff = NULL;
+    }
+
+    release_event_ref(kfse);
 
-    lck_rw_done(&fsevent_big_lock);
     return error;
 }
 
+
 static void
-do_free_event(kfs_event *kfse)
+release_event_ref(kfs_event *kfse)
 {
-    int i;
-    kfs_event_arg *kea, all_args[KFS_NUM_ARGS];
+    int old_refcount;
+    kfs_event copy, dest_copy;
     
-    lock_fs_event_buf();
     
-    if (kfse->refcount > 0) {
-	panic("do_free_event: free'ing a kfsevent w/refcount == %d (kfse %p)\n",
-	    kfse->refcount, kfse);
+    old_refcount = OSAddAtomic(-1, (SInt32 *)&kfse->refcount);
+    if (old_refcount > 1) {
+	return;
+    }
+
+    lock_fs_event_list();
+    if (kfse->refcount < 0) {
+	panic("release_event_ref: bogus kfse refcount %d\n", kfse->refcount);
     }
 
+    if (kfse->refcount > 0 || kfse->type == FSE_INVALID) {
+	// This is very subtle.  Either of these conditions can
+	// be true if an event got recycled while we were waiting
+	// on the fs_event_list lock or the event got recycled,
+	// delivered, _and_ free'd by someone else while we were
+	// waiting on the fs event list lock.  In either case
+	// we need to just unlock the list and return without
+	// doing anything because if the refcount is > 0 then
+	// someone else will take care of free'ing it and when
+	// the kfse->type is invalid then someone else already
+	// has handled free'ing the event (while we were blocked
+	// on the event list lock).
+	//
+	unlock_fs_event_list();
+	return;
+    }
+
+    //
     // make a copy of this so we can free things without
     // holding the fs_event_buf lock
     //
-    memcpy(&all_args[0], &kfse->args[0], sizeof(all_args));
+    copy = *kfse;
+    if (kfse->dest && OSAddAtomic(-1, (SInt32 *)&kfse->dest->refcount) == 1) {
+	dest_copy = *kfse->dest;
+    } else {
+	dest_copy.str  = NULL;
+	dest_copy.len  = 0;
+	dest_copy.type = FSE_INVALID;
+    }
+
+    kfse->pid = kfse->type;             // save this off for debugging...
+    kfse->uid = (uid_t)kfse->str;       // save this off for debugging...
+    kfse->gid = (gid_t)current_thread();
+
+    kfse->str = (char *)0xdeadbeef;             // XXXdbg - catch any cheaters...
+
+    if (dest_copy.type != FSE_INVALID) {
+	kfse->dest->str = (char *)0xbadc0de;   // XXXdbg - catch any cheaters...
+	kfse->dest->type = FSE_INVALID;
+
+	if (kfse->dest->kevent_list.le_prev != NULL) {
+	    num_events_outstanding--;
+	    LIST_REMOVE(kfse->dest, kevent_list);
+	    memset(&kfse->dest->kevent_list, 0xa5, sizeof(kfse->dest->kevent_list));
+	}
+
+	zfree(event_zone, kfse->dest);
+    }
 
-    // and just to be anal, set this so that there are no args
-    kfse->args[0].type = FSE_ARG_DONE;
-    
     // mark this fsevent as invalid
+    {
+	int otype;
+	
+	otype = kfse->type;
     kfse->type = FSE_INVALID;
 
-    free_event_idx = (kfse - fs_event_buf);
-
-    unlock_fs_event_buf();
+    if (kfse->kevent_list.le_prev != NULL) {
+	num_events_outstanding--;
+	if (otype == FSE_RENAME) {
+	    num_pending_rename--;
+	}
+	LIST_REMOVE(kfse, kevent_list);
+	memset(&kfse->kevent_list, 0, sizeof(kfse->kevent_list));
+    }
+    }
     
-    for(i=0; i < KFS_NUM_ARGS; i++) {
-	kea = &all_args[i];
-	if (kea->type == FSE_ARG_DONE) {
-	    break;
+    zfree(event_zone, kfse);
+    
+    unlock_fs_event_list();
+    
+    // if we have a pointer in the union
+    if (copy.str) {
+	if (copy.len == 0) {    // and it's not a string
+	    panic("%s:%d: no more fref.vp!\n", __FILE__, __LINE__);
+	    // vnode_rele_ext(copy.fref.vp, O_EVTONLY, 0);
+	} else {                // else it's a string
+	    vfs_removename(copy.str);
 	}
+    }
 
-	switch(kea->type) {
-	    case FSE_ARG_VNODE:
-		vnode_rele_ext(kea->data.vp, O_EVTONLY, 0);
-		break;
-	    case FSE_ARG_STRING:
-		vfs_removename(kea->data.str);
-		break;
-	    case FSE_ARG_RAW:
-		FREE(kea->data.ptr, M_TEMP);
-		break;
+    if (dest_copy.type != FSE_INVALID && dest_copy.str) {
+	if (dest_copy.len == 0) {
+	    panic("%s:%d: no more fref.vp!\n", __FILE__, __LINE__);
+	    // vnode_rele_ext(dest_copy.fref.vp, O_EVTONLY, 0);
+	} else {
+	    vfs_removename(dest_copy.str);
 	}
     }
 }
@@ -526,7 +1234,7 @@ add_watcher(int8_t *event_list, int32_t num_events, int32_t eventq_size, fs_even
     int               i;
     fs_event_watcher *watcher;
 
-    if (eventq_size < 0 || eventq_size > MAX_KFS_EVENTS) {
+    if (eventq_size <= 0 || eventq_size > 100*MAX_KFS_EVENTS) {
 	eventq_size = MAX_KFS_EVENTS;
     }
 
@@ -536,6 +1244,9 @@ add_watcher(int8_t *event_list, int32_t num_events, int32_t eventq_size, fs_even
 	   fs_event_watcher *,
 	   sizeof(fs_event_watcher) + eventq_size * sizeof(kfs_event *),
 	   M_TEMP, M_WAITOK);
+    if (watcher == NULL) {
+	return ENOMEM;
+    }
 
     watcher->event_list   = event_list;
     watcher->num_events   = num_events;
@@ -548,8 +1259,11 @@ add_watcher(int8_t *event_list, int32_t num_events, int32_t eventq_size, fs_even
     watcher->wr           = 0;
     watcher->blockers     = 0;
     watcher->num_readers  = 0;
+    watcher->fseh         = NULL;
+
+    watcher->num_dropped  = 0;      // XXXdbg - debugging
 
-    lock_watch_list();
+    lock_watch_table();
 
     // now update the global list of who's interested in
     // events of a particular type...
@@ -559,93 +1273,435 @@ add_watcher(int8_t *event_list, int32_t num_events, int32_t eventq_size, fs_even
 	}
     }
 
-    SLIST_INSERT_HEAD(&watch_list_head, watcher, link);
+    for(i=0; i < MAX_WATCHERS; i++) {
+	if (watcher_table[i] == NULL) {
+	    watcher->my_id   = i;
+	    watcher_table[i] = watcher;
+	    break;
+	}
+    }
+
+    if (i > MAX_WATCHERS) {
+	printf("fsevents: too many watchers!\n");
+	unlock_watch_table();
+	return ENOSPC;
+    }
 
-    unlock_watch_list();
+    unlock_watch_table();
 
     *watcher_out = watcher;
 
     return 0;
 }
 
+
+
 static void
 remove_watcher(fs_event_watcher *target)
 {
-    int i;
+    int i, j, counter=0;
     fs_event_watcher *watcher;
     kfs_event *kfse;
     
-    lck_rw_lock_shared(&fsevent_big_lock);
-
-    lock_watch_list();
+    lock_watch_table();
     
-    SLIST_FOREACH(watcher, &watch_list_head, link) {
-	if (watcher == target) {
-	    SLIST_REMOVE(&watch_list_head, watcher, fs_event_watcher, link);
+    for(j=0; j < MAX_WATCHERS; j++) {
+	watcher = watcher_table[j];
+	if (watcher != target) {
+	    continue;
+	}
 
-	    for(i=0; i < watcher->num_events; i++) {
-		if (watcher->event_list[i] != FSE_IGNORE && i < FSE_MAX_EVENTS) {
-		    fs_event_type_watchers[i]--;
-		}
+	watcher_table[j] = NULL;
+
+	for(i=0; i < watcher->num_events; i++) {
+	    if (watcher->event_list[i] != FSE_IGNORE && i < FSE_MAX_EVENTS) {
+		fs_event_type_watchers[i]--;
 	    }
+	}
 
-	    unlock_watch_list();
+	if (watcher->flags & WATCHER_CLOSING) {
+	    unlock_watch_table();
+	    return;
+	}
+
+	// printf("fsevents: removing watcher %p (rd %d wr %d num_readers %d flags 0x%x)\n", watcher, watcher->rd, watcher->wr, watcher->num_readers, watcher->flags);
+	watcher->flags |= WATCHER_CLOSING;
+	OSAddAtomic(1, (SInt32 *)&watcher->num_readers);
+	
+	unlock_watch_table();
 	    
-	    // drain the event_queue 
-	    for(i=watcher->rd; i != watcher->wr; i=(i+1) % watcher->eventq_size) {
-		kfse = watcher->event_queue[i];
-		
-		if (OSAddAtomic(-1, (SInt32 *)&kfse->refcount) == 1) {
-		    do_free_event(kfse);
-		}
+	while (watcher->num_readers > 1 && counter++ < 5000) {
+	    fsevents_wakeup(watcher);      // in case they're asleep
+	    
+	    tsleep(watcher, PRIBIO, "fsevents-close", 1);
+	}
+	if (counter++ >= 5000) {
+	    // printf("fsevents: close: still have readers! (%d)\n", watcher->num_readers);
+	    panic("fsevents: close: still have readers! (%d)\n", watcher->num_readers);
+	}
+
+	// drain the event_queue 
+	while(watcher->rd != watcher->wr) {
+	    lck_rw_lock_shared(&event_handling_lock);
+
+	    kfse = watcher->event_queue[watcher->rd];
+	    if (kfse->type == FSE_INVALID || kfse->refcount < 1) {
+		panic("remove_watcher: bogus kfse %p during cleanup (type %d refcount %d rd %d wr %d)\n", kfse, kfse->type, kfse->refcount, watcher->rd, watcher->wr);
+	    }
+
+	    lck_rw_unlock_shared(&event_handling_lock);
+	    
+	    watcher->rd = (watcher->rd+1) % watcher->eventq_size;
+
+	    if (kfse != NULL) {
+		release_event_ref(kfse);
 	    }
+	}
 	    
-	    if (watcher->event_list) {
-		FREE(watcher->event_list, M_TEMP);
-		watcher->event_list = NULL;
+	if (watcher->event_list) {
+	    FREE(watcher->event_list, M_TEMP);
+	    watcher->event_list = NULL;
+	}
+	if (watcher->devices_to_watch) {
+	    FREE(watcher->devices_to_watch, M_TEMP);
+	    watcher->devices_to_watch = NULL;
+	}
+	FREE(watcher, M_TEMP);
+
+	return;
+    }
+
+    unlock_watch_table();
+}
+
+
+#define EVENT_DELAY_IN_MS   10
+static thread_call_t event_delivery_timer = NULL;
+static int timer_set = 0;
+
+
+static void
+delayed_event_delivery(__unused void *param0, __unused void *param1)
+{
+    int i;
+    
+    lock_watch_table();
+
+    for(i=0; i < MAX_WATCHERS; i++) {
+	if (watcher_table[i] != NULL && watcher_table[i]->rd != watcher_table[i]->wr) {
+	    fsevents_wakeup(watcher_table[i]);
+	}
+    }
+
+    timer_set = 0;
+
+    unlock_watch_table();
+}
+
+
+//
+// The watch table must be locked before calling this function.
+//
+static void
+schedule_event_wakeup(void)
+{
+    uint64_t deadline;
+    
+    if (event_delivery_timer == NULL) {
+	event_delivery_timer = thread_call_allocate((thread_call_func_t)delayed_event_delivery, NULL);
+    }
+
+    clock_interval_to_deadline(EVENT_DELAY_IN_MS, 1000 * 1000, &deadline);
+    
+    thread_call_enter_delayed(event_delivery_timer, deadline);
+    timer_set = 1;
+}
+
+
+
+#define MAX_NUM_PENDING  16
+
+//
+// NOTE: the watch table must be locked before calling
+//       this routine.
+//
+static int
+watcher_add_event(fs_event_watcher *watcher, kfs_event *kfse)
+{
+    if (((watcher->wr + 1) % watcher->eventq_size) == watcher->rd) {
+	watcher->flags |= WATCHER_DROPPED_EVENTS;
+	fsevents_wakeup(watcher);
+	return ENOSPC;
+    }
+
+    OSAddAtomic(1, (SInt32 *)&kfse->refcount);
+    watcher->event_queue[watcher->wr] = kfse;
+    OSSynchronizeIO();
+    watcher->wr = (watcher->wr + 1) % watcher->eventq_size;
+    
+    //
+    // wake up the watcher if there are more than MAX_NUM_PENDING events.
+    // otherwise schedule a timer (if one isn't already set) which will 
+    // send any pending events if no more are received in the next 
+    // EVENT_DELAY_IN_MS milli-seconds.
+    //
+    if (   (watcher->rd < watcher->wr && (watcher->wr - watcher->rd) > MAX_NUM_PENDING)
+	|| (watcher->rd > watcher->wr && (watcher->wr + watcher->eventq_size - watcher->rd) > MAX_NUM_PENDING)) {
+
+	fsevents_wakeup(watcher);
+
+    } else if (timer_set == 0) {
+
+	schedule_event_wakeup();
+    }
+
+    return 0;
+}
+
+
+// check if the next chunk of data will fit in the user's
+// buffer.  if not, just goto get_out which will return
+// the number of bytes worth of events that we did read.
+// this leaves the event that didn't fit in the queue.
+//    
+	// LP64todo - fix this
+#define CHECK_UPTR(size) if (size > (unsigned)uio_resid(uio)) { \
+                            uio_setresid(uio, last_full_event_resid); \
+                            goto get_out; \
+                         }
+
+static int
+fill_buff(uint16_t type, int32_t size, const void *data,
+          char *buff, int32_t *_buff_idx, int32_t buff_sz,
+          struct uio *uio)
+{
+    int32_t amt, error = 0, buff_idx = *_buff_idx;
+    uint16_t tmp;
+    
+    //
+    // the +1 on the size is to guarantee that the main data
+    // copy loop will always copy at least 1 byte
+    //
+    if ((buff_sz - buff_idx) <= (int)(2*sizeof(uint16_t) + 1)) {
+	if (buff_idx > uio_resid(uio)) {
+	    error = ENOSPC;
+	    goto get_out;
+	}
+
+	error = uiomove(buff, buff_idx, uio);
+	if (error) {
+	    goto get_out;
+	}
+	buff_idx = 0;
+    }
+
+    // copy out the header (type & size)
+    memcpy(&buff[buff_idx], &type, sizeof(uint16_t));
+    buff_idx += sizeof(uint16_t);
+    
+    tmp = size & 0xffff;
+    memcpy(&buff[buff_idx], &tmp, sizeof(uint16_t));
+    buff_idx += sizeof(uint16_t);
+    
+    // now copy the body of the data, flushing along the way 
+    // if the buffer fills up.
+    //
+    while(size > 0) {
+	amt = (size < (buff_sz - buff_idx)) ? size : (buff_sz - buff_idx);
+	memcpy(&buff[buff_idx], data, amt);
+
+	size -= amt;
+	buff_idx += amt;
+	data = (const char *)data + amt;
+	if (size > (buff_sz - buff_idx)) {
+	    if (buff_idx > uio_resid(uio)) {
+		error = ENOSPC;
+		goto get_out;
 	    }
-	    if (watcher->devices_to_watch) {
-		FREE(watcher->devices_to_watch, M_TEMP);
-		watcher->devices_to_watch = NULL;
+	    error = uiomove(buff, buff_idx, uio);
+	    if (error) {
+		goto get_out;
 	    }
-	    FREE(watcher, M_TEMP);
+	    buff_idx = 0;
+	}
+
+	if (amt == 0) {   // just in case...
+	    break;
+	}
+    }
+
+  get_out:
+    *_buff_idx = buff_idx;
+    
+    return error;
+}
+
+
+static int copy_out_kfse(fs_event_watcher *watcher, kfs_event *kfse, struct uio *uio)  __attribute__((noinline));
+
+static int
+copy_out_kfse(fs_event_watcher *watcher, kfs_event *kfse, struct uio *uio)
+{
+    int      error;
+    uint16_t tmp16;
+    int32_t  type;
+    kfs_event *cur;
+    char     evbuff[512];
+    int      evbuff_idx = 0;
+
+    if (kfse->type == FSE_INVALID) {
+	panic("fsevents: copy_out_kfse: asked to copy out an invalid event (kfse %p, refcount %d fref ptr %p)\n", kfse, kfse->refcount, kfse->str);
+    }
+
+    if (kfse->flags & KFSE_BEING_CREATED) {
+	return 0;
+    }
+
+    if (kfse->type == FSE_RENAME && kfse->dest == NULL) {
+	//
+	// This can happen if an event gets recycled but we had a
+	// pointer to it in our event queue.  The event is the
+	// destination of a rename which we'll process separately
+	// (that is, another kfse points to this one so it's ok
+	// to skip this guy because we'll process it when we process
+	// the other one)
+	error = 0;
+	goto get_out;
+    }
+
+    if (watcher->flags & WATCHER_WANTS_EXTENDED_INFO) {
+
+	type = (kfse->type & 0xfff);
+
+	if (kfse->flags & KFSE_CONTAINS_DROPPED_EVENTS) {
+	    type |= (FSE_CONTAINS_DROPPED_EVENTS << FSE_FLAG_SHIFT);
+	} else if (kfse->flags & KFSE_COMBINED_EVENTS) {
+	    type |= (FSE_COMBINED_EVENTS << FSE_FLAG_SHIFT);
+	}
+
+    } else {
+	type = (int32_t)kfse->type;
+    }
+
+    // copy out the type of the event
+    memcpy(evbuff, &type, sizeof(int32_t));
+    evbuff_idx += sizeof(int32_t);
+
+    // copy out the pid of the person that generated the event
+    memcpy(&evbuff[evbuff_idx], &kfse->pid, sizeof(pid_t));
+    evbuff_idx += sizeof(pid_t);
+
+    cur = kfse;
+
+  copy_again:
+
+    if (cur->str == NULL || cur->str[0] == '\0') {
+	printf("copy_out_kfse:2: empty/short path (%s)\n", cur->str);
+	error = fill_buff(FSE_ARG_STRING, 2, "/", evbuff, &evbuff_idx, sizeof(evbuff), uio);
+    } else {
+	error = fill_buff(FSE_ARG_STRING, cur->len, cur->str, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+    }
+    if (error != 0) {
+	goto get_out;
+    }
+
+    if (cur->dev == 0 && cur->ino == 0) {
+	// this happens when a rename event happens and the
+	// destination of the rename did not previously exist.
+	// it thus has no other file info so skip copying out
+	// the stuff below since it isn't initialized
+	goto done;
+    }
+
+    
+    if (watcher->flags & WATCHER_WANTS_COMPACT_EVENTS) {
+	int32_t finfo_size;
+	
+	finfo_size = sizeof(dev_t) + sizeof(ino64_t) + sizeof(int32_t) + sizeof(uid_t) + sizeof(gid_t);
+	error = fill_buff(FSE_ARG_FINFO, finfo_size, &cur->ino, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
+	}
+    } else {
+	ino_t ino;
+	
+	error = fill_buff(FSE_ARG_DEV, sizeof(dev_t), &cur->dev, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
+	}
+
+	ino = (ino_t)cur->ino;
+	error = fill_buff(FSE_ARG_INO, sizeof(ino_t), &ino, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
+	}
+
+	error = fill_buff(FSE_ARG_MODE, sizeof(int32_t), &cur->mode, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
+	}
+
+	error = fill_buff(FSE_ARG_UID, sizeof(uid_t), &cur->uid, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
+	}
 
-	    lck_rw_done(&fsevent_big_lock);
-	    return;
+	error = fill_buff(FSE_ARG_GID, sizeof(gid_t), &cur->gid, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+	if (error != 0) {
+	    goto get_out;
 	}
     }
 
-    unlock_watch_list();
-    lck_rw_done(&fsevent_big_lock);
-}
 
+    if (cur->dest) {
+	cur = cur->dest;
+	goto copy_again;
+    }
 
-static int
-watcher_add_event(fs_event_watcher *watcher, kfs_event *kfse)
-{
-    if (((watcher->wr + 1) % watcher->eventq_size) == watcher->rd) {
-	watcher->flags |= WATCHER_DROPPED_EVENTS;
-	wakeup((caddr_t)watcher);
-	return ENOSPC;
+  done:
+    // very last thing: the time stamp
+    error = fill_buff(FSE_ARG_INT64, sizeof(uint64_t), &cur->abstime, evbuff, &evbuff_idx, sizeof(evbuff), uio);
+    if (error != 0) {
+	goto get_out;
     }
 
-    watcher->event_queue[watcher->wr] = kfse;
-    OSAddAtomic(1, (SInt32 *)&kfse->refcount);
-    watcher->wr = (watcher->wr + 1) % watcher->eventq_size;
-    
-    // wake up the watcher if he's waiting!
-    wakeup((caddr_t)watcher);
+    // check if the FSE_ARG_DONE will fit
+    if (sizeof(uint16_t) > sizeof(evbuff) - evbuff_idx) {
+	if (evbuff_idx > uio_resid(uio)) {
+	    error = ENOSPC;
+	    goto get_out;
+	}
+	error = uiomove(evbuff, evbuff_idx, uio);
+	if (error) {
+	    goto get_out;
+	}
+	evbuff_idx = 0;
+    }
 
-    return 0;
+    tmp16 = FSE_ARG_DONE;
+    memcpy(&evbuff[evbuff_idx], &tmp16, sizeof(uint16_t));
+    evbuff_idx += sizeof(uint16_t);
+
+    // flush any remaining data in the buffer (and hopefully
+    // in most cases this is the only uiomove we'll do)
+    if (evbuff_idx > uio_resid(uio)) {
+	error = ENOSPC;
+    } else {
+	error = uiomove(evbuff, evbuff_idx, uio);
+    }
+
+  get_out:
+
+    return error;
 }
 
 
+
 static int
 fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 {
-    int               i, error=0, last_full_event_resid;
+    int               error=0, last_full_event_resid;
     kfs_event        *kfse;
-    kfs_event_arg    *kea;
     uint16_t          tmp16;
 
     // LP64todo - fix this
@@ -656,6 +1712,10 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	return EINVAL;
     }
 
+    if (watcher->flags & WATCHER_CLOSING) {
+	return 0;
+    }
+
     if (OSAddAtomic(1, (SInt32 *)&watcher->num_readers) != 0) {
 	// don't allow multiple threads to read from the fd at the same time
 	OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
@@ -691,6 +1751,9 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	    
 	    tmp16 = FSE_ARG_DONE;  // makes it a consistent msg
 	    error = uiomove((caddr_t)&tmp16, sizeof(int16_t), uio);
+
+	    // LP64todo - fix this
+	    last_full_event_resid = uio_resid(uio);
 	} 
 
 	if (error) {
@@ -701,170 +1764,59 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 	watcher->flags &= ~WATCHER_DROPPED_EVENTS;
     }
 
-// check if the next chunk of data will fit in the user's
-// buffer.  if not, just goto get_out which will return
-// the number of bytes worth of events that we did read.
-// this leaves the event that didn't fit in the queue.
-//    
-	// LP64todo - fix this
-#define CHECK_UPTR(size) if (size > (unsigned)uio_resid(uio)) { \
-                            uio_setresid(uio, last_full_event_resid); \
-                            goto get_out; \
-                         }
-
-    for (; uio_resid(uio) > 0 && watcher->rd != watcher->wr; ) {
-	kfse = watcher->event_queue[watcher->rd];
-
-	// copy out the type of the event
-	CHECK_UPTR(sizeof(int32_t));
-	if ((error = uiomove((caddr_t)&kfse->type, sizeof(int32_t), uio)) != 0) {
-	    goto get_out;
+    while (uio_resid(uio) > 0 && watcher->rd != watcher->wr) {
+	if (watcher->flags & WATCHER_CLOSING) {
+	    break;
 	}
+	
+	//
+	// check if the event is something of interest to us
+	// (since it may have been recycled/reused and changed
+	// its type or which device it is for)
+	//
+	lck_rw_lock_shared(&event_handling_lock);
 
-	// now copy out the pid of the person that changed the file
-	CHECK_UPTR(sizeof(pid_t));
-	if ((error = uiomove((caddr_t)&kfse->pid, sizeof(pid_t), uio)) != 0) {
-	    goto get_out;
+	kfse = watcher->event_queue[watcher->rd];
+	if (kfse->type == FSE_INVALID || kfse->refcount < 1) {
+	    panic("fmod_watch: someone left me a bogus kfse %p (type %d refcount %d rd %d wr %d)\n", kfse, kfse->type, kfse->refcount, watcher->rd, watcher->wr);
 	}
 
-	error = 0;
-	for(i=0; i < KFS_NUM_ARGS && error == 0; i++) {
-	    char    *pathbuff;
-	    int      pathbuff_len;
-
-	    kea = &kfse->args[i];
-
-	    tmp16 = (uint16_t)kea->type;
-	    CHECK_UPTR(sizeof(uint16_t));
-	    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-	    if (error || kea->type == FSE_ARG_DONE) {
-		break;
+	if (watcher->event_list[kfse->type] == FSE_REPORT && watcher_cares_about_dev(watcher, kfse->dev)) {
+
+	    error = copy_out_kfse(watcher, kfse, uio);
+	    if (error != 0) {
+		// if an event won't fit or encountered an error while
+		// we were copying it out, then backup to the last full
+		// event and just bail out.  if the error was ENOENT
+		// then we can continue regular processing, otherwise
+		// we should unlock things and return.
+		uio_setresid(uio, last_full_event_resid);
+		if (error != ENOENT) {
+		    lck_rw_unlock_shared(&event_handling_lock);
+		    error = 0;
+		    goto get_out;
+		}
 	    }
 
-	    switch(kea->type) {
-		case FSE_ARG_VNODE:
-		    pathbuff = get_pathbuff();
-		    pathbuff_len = MAXPATHLEN;
-		    if (kea->data.vp == NULL) {
-			printf("fmod_watch: whoa... vp == NULL (%d)!\n", kfse->type);
-			i--;
-			release_pathbuff(pathbuff);
-			continue;
-		    }
-		    
-		    if (vn_getpath(kea->data.vp, pathbuff, &pathbuff_len) != 0 || pathbuff[0] == '\0') {
-//			printf("fmod_watch: vn_getpath failed! vp 0x%x vname 0x%x (%s) vparent 0x%x\n",
-//			       kea->data.vp,
-//			       VNAME(kea->data.vp),
-//			       VNAME(kea->data.vp) ? VNAME(kea->data.vp) : "<null>",
-//			       VPARENT(kea->data.vp));
-		    }
-		    CHECK_UPTR(sizeof(uint16_t));
-		    tmp16 = (uint16_t)pathbuff_len;
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-
-		    CHECK_UPTR((unsigned)pathbuff_len);
-		    error = uiomove((caddr_t)pathbuff, pathbuff_len, uio);
-		    release_pathbuff(pathbuff);
-		    break;
-		    
-
-		case FSE_ARG_STRING:
-		    tmp16 = (int32_t)kea->len;
-		    CHECK_UPTR(sizeof(uint16_t));
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-
-		    CHECK_UPTR(kea->len);
-		    error = uiomove((caddr_t)kea->data.str, kea->len, uio);
-		    break;
-
-		case FSE_ARG_INT32:
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(int32_t));
-		    tmp16 = sizeof(int32_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.int32, sizeof(int32_t), uio);
-		    break;
-
-		case FSE_ARG_INT64:
-		    printf("fs_events: 64-bit args not implemented on copyout.\n");
-//		    CHECK_UPTR(sizeof(uint16_t) + sizeof(int64_t));
-//		    tmp16 = sizeof(int64_t);
-//		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-//		    error = uiomove((caddr_t)&kea->data.int64, sizeof(int64_t), uio);
-		    break;
-
-		case FSE_ARG_RAW:
-		    tmp16 = (uint16_t)kea->len;
-		    CHECK_UPTR(sizeof(uint16_t));
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-
-		    CHECK_UPTR(kea->len);
-		    error = uiomove((caddr_t)kea->data.ptr, kea->len, uio);
-		    break;
-
-		case FSE_ARG_DEV:
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(dev_t));
-		    tmp16 = sizeof(dev_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.dev, sizeof(dev_t), uio);
-		    break;
-
-		case FSE_ARG_INO:
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(ino_t));
-		    tmp16 = sizeof(ino_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.ino, sizeof(ino_t), uio);
-		    break;
-
-		case FSE_ARG_MODE:
-		    // XXXdbg - NOTE: we use 32-bits for the mode, not
-		    //                16-bits like a real mode_t
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(int32_t));
-		    tmp16 = sizeof(int32_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.mode, sizeof(int32_t), uio);
-		    break;
-
-		case FSE_ARG_UID:
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(uid_t));
-		    tmp16 = sizeof(uid_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.uid, sizeof(uid_t), uio);
-		    break;
-
-		case FSE_ARG_GID:
-		    CHECK_UPTR(sizeof(uint16_t) + sizeof(gid_t));
-		    tmp16 = sizeof(gid_t);
-		    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-		    error = uiomove((caddr_t)&kea->data.gid, sizeof(gid_t), uio);
-		    break;
-
-		default:
-		    printf("fmod_watch: unknown arg type %d.\n", kea->type);
-		    break;
-	    }
+	    // LP64todo - fix this
+	    last_full_event_resid = uio_resid(uio);
 	}
 
-	// make sure that we always end with a FSE_ARG_DONE
-	if (i >= KFS_NUM_ARGS) {
-	    tmp16 = FSE_ARG_DONE;
-	    CHECK_UPTR(sizeof(uint16_t));
-	    error = uiomove((caddr_t)&tmp16, sizeof(uint16_t), uio);
-	}
-	
+	lck_rw_unlock_shared(&event_handling_lock);
 
-	// LP64todo - fix this
-	last_full_event_resid = uio_resid(uio);
-	
 	watcher->rd = (watcher->rd + 1) % watcher->eventq_size;
+	OSSynchronizeIO();
 	    
-	if (OSAddAtomic(-1, (SInt32 *)&kfse->refcount) == 1) {
-	    do_free_event(kfse);
+	if (kfse->type == FSE_INVALID || kfse->refcount < 1) {
+	    panic("fmod_watch:2: my kfse became bogus! kfse %p (type %d refcount %d rd %d wr %d)\n", kfse, kfse->type, kfse->refcount, watcher->rd, watcher->wr);
 	}
+
+	release_event_ref(kfse);
     }
 
   get_out:
     OSAddAtomic(-1, (SInt32 *)&watcher->num_readers);
+
     return error;
 }
 
@@ -874,68 +1826,13 @@ fmod_watch(fs_event_watcher *watcher, struct uio *uio)
 // unmounted).
 //
 // since we don't want to lose the events we'll convert the
-// vnode refs to the full path, inode #, and uid.
+// vnode refs to full paths.
 //
 void
-fsevent_unmount(struct mount *mp)
+fsevent_unmount(__unused struct mount *mp)
 {
-    int            i, j;
-    kfs_event     *kfse;
-    kfs_event_arg *kea;
-    
-    lck_rw_lock_exclusive(&fsevent_big_lock);
-    lock_fs_event_buf();
-    
-    for(i=0; i < MAX_KFS_EVENTS; i++) {
-	if (fs_event_buf[i].type == FSE_INVALID) {
-	    continue;
-	}
-
-	kfse = &fs_event_buf[i];
-	for(j=0; j < KFS_NUM_ARGS; j++) {
-	    kea = &kfse->args[j];
-	    if (kea->type == FSE_ARG_DONE) {
-		break;
-	    }
-
-	    if (kea->type == FSE_ARG_VNODE && kea->data.vp->v_mount == mp) {
-		struct vnode *vp;
-		char         *pathbuff;
-		int           pathbuff_len;
-
-		vp = kea->data.vp;
-		pathbuff = get_pathbuff();
-		pathbuff_len = MAXPATHLEN;
-		    
-		if (vn_getpath(vp, pathbuff, &pathbuff_len) != 0 || pathbuff[0] == '\0') {
-		        char *vname;
-
-			vname = vnode_getname(vp);
-
-			printf("fsevent_unmount: vn_getpath failed! vp 0x%x vname 0x%x (%s) vparent 0x%x\n",
-			       vp, vname, vname ? vname : "<null>", vp->v_parent);
-
-			if (vname)
-			        vnode_putname(vname);
-
-			strcpy(pathbuff, "UNKNOWN-FILE");
-			pathbuff_len = strlen(pathbuff) + 1;
-		}
-
-		// switch the type of the string
-		kea->type     = FSE_ARG_STRING;
-		kea->data.str = vfs_addname(pathbuff, pathbuff_len, 0, 0);
-		kea->len      = pathbuff_len;
-		release_pathbuff(pathbuff);
-
-		// and finally let go of the reference on the vnode
-		vnode_rele_ext(vp, O_EVTONLY, 0);
-	    }
-	}
-    }
-
-    unlock_fs_event_buf();
-    lck_rw_done(&fsevent_big_lock);
+    // we no longer maintain pointers to vnodes so
+    // there is nothing to do... 
 }
 
 
@@ -943,7 +1840,6 @@ fsevent_unmount(struct mount *mp)
 // /dev/fsevents device code
 //
 static int fsevents_installed = 0;
-static struct lock__bsd__ fsevents_lck;
 
 typedef struct fsevent_handle {
     UInt32            flags;
@@ -956,8 +1852,7 @@ typedef struct fsevent_handle {
 
 static int
 fseventsf_read(struct fileproc *fp, struct uio *uio,
-		__unused kauth_cred_t *cred, __unused int flags,
-		__unused struct proc *p)
+	       __unused int flags, __unused vfs_context_t ctx)
 {
     fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data;
     int error;
@@ -967,22 +1862,54 @@ fseventsf_read(struct fileproc *fp, struct uio *uio,
     return error;
 }
 
+
 static int
 fseventsf_write(__unused struct fileproc *fp, __unused struct uio *uio,
-		__unused kauth_cred_t *cred, __unused int flags,
-		__unused struct proc *p)
+		__unused int flags, __unused vfs_context_t ctx)
 {
     return EIO;
 }
 
+typedef struct ext_fsevent_dev_filter_args {
+    uint32_t    num_devices;
+    user_addr_t devices;
+} ext_fsevent_dev_filter_args;
+
+typedef struct old_fsevent_dev_filter_args {
+    uint32_t  num_devices;
+    int32_t   devices;
+} old_fsevent_dev_filter_args;
+
+#define	OLD_FSEVENTS_DEVICE_FILTER	_IOW('s', 100, old_fsevent_dev_filter_args)
+#define	NEW_FSEVENTS_DEVICE_FILTER	_IOW('s', 100, ext_fsevent_dev_filter_args)
+
 
 static int
-fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
+fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, vfs_context_t ctx)
 {
     fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data;
     int ret = 0;
-    pid_t pid = 0;
-    fsevent_dev_filter_args *devfilt_args=(fsevent_dev_filter_args *)data;
+    ext_fsevent_dev_filter_args *devfilt_args, _devfilt_args;
+
+    if (proc_is64bit(vfs_context_proc(ctx))) {
+	devfilt_args = (ext_fsevent_dev_filter_args *)data;
+    } else if (cmd == OLD_FSEVENTS_DEVICE_FILTER) {
+	old_fsevent_dev_filter_args *udev_filt_args = (old_fsevent_dev_filter_args *)data;
+	
+	devfilt_args = &_devfilt_args;
+	memset(devfilt_args, 0, sizeof(ext_fsevent_dev_filter_args));
+
+	devfilt_args->num_devices = udev_filt_args->num_devices;
+	devfilt_args->devices     = CAST_USER_ADDR_T(udev_filt_args->devices);
+    } else {
+	fsevent_dev_filter_args *udev_filt_args = (fsevent_dev_filter_args *)data;
+	
+	devfilt_args = &_devfilt_args;
+	memset(devfilt_args, 0, sizeof(ext_fsevent_dev_filter_args));
+
+	devfilt_args->num_devices = udev_filt_args->num_devices;
+	devfilt_args->devices     = CAST_USER_ADDR_T(udev_filt_args->devices);
+    }
 
     OSAddAtomic(1, &fseh->active);
     if (fseh->flags & FSEH_CLOSING) {
@@ -993,18 +1920,23 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
     switch (cmd) {
 	case FIONBIO:
 	case FIOASYNC:
-	    ret = 0;
 	    break;
 
-	case FSEVENTS_DEVICE_FILTER: {
+	case FSEVENTS_WANT_COMPACT_EVENTS: {
+	    fseh->watcher->flags |= WATCHER_WANTS_COMPACT_EVENTS;
+	    break;
+	}
+
+	case FSEVENTS_WANT_EXTENDED_INFO: {
+	    fseh->watcher->flags |= WATCHER_WANTS_EXTENDED_INFO;
+	    break;
+	}
+
+	case OLD_FSEVENTS_DEVICE_FILTER:
+	case NEW_FSEVENTS_DEVICE_FILTER: {
 	    int new_num_devices;
 	    dev_t *devices_to_watch, *tmp=NULL;
 	    
-	    if (fseh->flags & FSEH_CLOSING) {
-		ret = 0;
-		break;
-	    }
-
 	    if (devfilt_args->num_devices > 256) {
 		ret = EINVAL;
 		break;
@@ -1014,10 +1946,10 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
 	    if (new_num_devices == 0) {
 		tmp = fseh->watcher->devices_to_watch;
 
-		lock_watch_list();
+		lock_watch_table();
 		fseh->watcher->devices_to_watch = NULL;
 		fseh->watcher->num_devices = new_num_devices;
-		unlock_watch_list();
+		unlock_watch_table();
 
 		if (tmp) {
 		    FREE(tmp, M_TEMP);
@@ -1033,7 +1965,7 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
 		break;
 	    }
     
-	    ret = copyin(CAST_USER_ADDR_T(devfilt_args->devices),
+	    ret = copyin(devfilt_args->devices,
 			 (void *)devices_to_watch,
 			 new_num_devices * sizeof(dev_t));
 	    if (ret) {
@@ -1041,11 +1973,11 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
 		break;
 	    }
 
-	    lock_watch_list();
+	    lock_watch_table();
 	    fseh->watcher->num_devices = new_num_devices;
 	    tmp = fseh->watcher->devices_to_watch;
 	    fseh->watcher->devices_to_watch = devices_to_watch;
-	    unlock_watch_list();
+	    unlock_watch_table();
 
 	    if (tmp) {
 		FREE(tmp, M_TEMP);
@@ -1065,7 +1997,7 @@ fseventsf_ioctl(struct fileproc *fp, u_long cmd, caddr_t data, struct proc *p)
 
 
 static int
-fseventsf_select(struct fileproc *fp, int which, void *wql, struct proc *p)
+fseventsf_select(struct fileproc *fp, int which, __unused void *wql, vfs_context_t ctx)
 {
     fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data;
     int ready = 0;
@@ -1076,41 +2008,40 @@ fseventsf_select(struct fileproc *fp, int which, void *wql, struct proc *p)
 
 
     // if there's nothing in the queue, we're not ready
-    if (fseh->watcher->rd == fseh->watcher->wr) {
-	ready = 0;
-    } else {
+    if (fseh->watcher->rd != fseh->watcher->wr) {
 	ready = 1;
     }
 
     if (!ready) {
-	selrecord(p, &fseh->si, wql);
+	selrecord(vfs_context_proc(ctx), &fseh->si, wql);
     }
 
     return ready;
 }
 
 
+#if NOTUSED
 static int
-fseventsf_stat(struct fileproc *fp, struct stat *sb, struct proc *p)
+fseventsf_stat(__unused struct fileproc *fp, __unused struct stat *sb, __unused vfs_context_t ctx)
 {
     return ENOTSUP;
 }
-
+#endif
 
 static int
-fseventsf_close(struct fileglob *fg, struct proc *p)
+fseventsf_close(struct fileglob *fg, __unused vfs_context_t ctx)
 {
     fsevent_handle *fseh = (struct fsevent_handle *)fg->fg_data;
     fs_event_watcher *watcher;
-    
+
     OSBitOrAtomic(FSEH_CLOSING, &fseh->flags);
     while (OSAddAtomic(0, &fseh->active) > 0) {
 	tsleep((caddr_t)fseh->watcher, PRIBIO, "fsevents-close", 1);
     }
 
     watcher = fseh->watcher;
-    fseh->watcher = NULL;
     fg->fg_data = NULL;
+    fseh->watcher = NULL;
 
     remove_watcher(watcher);
     FREE(fseh, M_TEMP);
@@ -1118,8 +2049,8 @@ fseventsf_close(struct fileglob *fg, struct proc *p)
     return 0;
 }
 
-int
-fseventsf_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p)
+static int
+fseventsf_kqfilter(__unused struct fileproc *fp, __unused struct knote *kn, __unused vfs_context_t ctx)
 {
     // XXXdbg
     return 0;
@@ -1127,7 +2058,7 @@ fseventsf_kqfilter(struct fileproc *fp, struct knote *kn, struct proc *p)
 
 
 static int
-fseventsf_drain(struct fileproc *fp, struct proc *p)
+fseventsf_drain(struct fileproc *fp, __unused vfs_context_t ctx)
 {
     int counter = 0;
     fsevent_handle *fseh = (struct fsevent_handle *)fp->f_fglob->fg_data;
@@ -1145,7 +2076,7 @@ fseventsf_drain(struct fileproc *fp, struct proc *p)
         // and decision to tsleep in fmod_watch... this bit of 
         // latency is a decent tradeoff against not having to
         // take and drop a lock in fmod_watch
-	wakeup((caddr_t)fseh->watcher);
+	fsevents_wakeup(fseh->watcher);
 
 	tsleep((caddr_t)fseh->watcher, PRIBIO, "watcher-close", 1);
     }
@@ -1155,7 +2086,7 @@ fseventsf_drain(struct fileproc *fp, struct proc *p)
 
 
 static int
-fseventsopen(dev_t dev, int flag, int mode, struct proc *p)
+fseventsopen(__unused dev_t dev, __unused int flag, __unused int mode, __unused struct proc *p)
 {
     if (!is_suser()) {
 	return EPERM;
@@ -1165,21 +2096,173 @@ fseventsopen(dev_t dev, int flag, int mode, struct proc *p)
 }
 
 static int
-fseventsclose(dev_t dev, int flag, int mode, struct proc *p)
+fseventsclose(__unused dev_t dev, __unused int flag, __unused int mode, __unused struct proc *p)
 {
     return 0;
 }
 
 static int
-fseventsread(dev_t dev, struct uio *uio, int ioflag)
+fseventsread(__unused dev_t dev, __unused struct uio *uio, __unused int ioflag)
 {
     return EIO;
 }
 
+
 static int
-fseventswrite(dev_t dev, struct uio *uio, int ioflag)
+parse_buffer_and_add_events(const char *buffer, int bufsize, vfs_context_t ctx, long *remainder)
 {
-    return EIO;
+    const fse_info *finfo, *dest_finfo;
+    const char *path, *ptr, *dest_path, *event_start=buffer;
+    int path_len, type, dest_path_len, err = 0;
+
+
+    ptr = buffer;
+    while ((ptr+sizeof(int)+sizeof(fse_info)+1) < buffer+bufsize) {
+	type = *(const int *)ptr;
+	if (type < 0 || type >= FSE_MAX_EVENTS) {
+	    err = EINVAL;
+	    break;
+	}
+
+	ptr += sizeof(int);
+	
+	finfo = (const fse_info *)ptr;
+	ptr += sizeof(fse_info);
+
+	path = ptr;
+	while(ptr < buffer+bufsize && *ptr != '\0') {
+	    ptr++;
+	}
+
+	if (ptr >= buffer+bufsize) {
+	    break;
+	}
+
+	ptr++;   // advance over the trailing '\0'
+
+	path_len = ptr - path;
+
+	if (type != FSE_RENAME && type != FSE_EXCHANGE) {
+	    event_start = ptr;   // record where the next event starts
+
+	    err = add_fsevent(type, ctx, FSE_ARG_STRING, path_len, path, FSE_ARG_FINFO, finfo, FSE_ARG_DONE);
+	    if (err) {
+		break;
+	    }
+	    continue;
+	}
+
+	//
+	// if we're here we have to slurp up the destination finfo
+	// and path so that we can pass them to the add_fsevent()
+	// call.  basically it's a copy of the above code.
+	//
+	dest_finfo = (const fse_info *)ptr;
+	ptr += sizeof(fse_info);
+
+	dest_path = ptr;
+	while(ptr < buffer+bufsize && *ptr != '\0') {
+	    ptr++;
+	}
+
+	if (ptr >= buffer+bufsize) {
+	    break;
+	}
+
+	ptr++;               // advance over the trailing '\0'
+	event_start = ptr;   // record where the next event starts
+
+	dest_path_len = ptr - dest_path;
+	err = add_fsevent(type, ctx,
+	                  FSE_ARG_STRING, path_len,      path,      FSE_ARG_FINFO, finfo,
+	                  FSE_ARG_STRING, dest_path_len, dest_path, FSE_ARG_FINFO, dest_finfo,
+	                  FSE_ARG_DONE);
+	if (err) {
+	    break;
+	}
+
+    }
+
+    // if the last event wasn't complete, set the remainder
+    // to be the last event start boundary.
+    //
+    *remainder = (long)((buffer+bufsize) - event_start);
+
+    return err;
+}
+
+
+//
+// Note: this buffer size can not ever be less than
+//       2*MAXPATHLEN + 2*sizeof(fse_info) + sizeof(int)
+//       because that is the max size for a single event.
+//       I made it 4k to be a "nice" size.  making it
+//       smaller is not a good idea.
+//
+#define WRITE_BUFFER_SIZE  4096
+char *write_buffer=NULL;
+
+static int
+fseventswrite(__unused dev_t dev, struct uio *uio, __unused int ioflag)
+{
+    int error=0, count;
+    vfs_context_t ctx = vfs_context_current();
+    long offset=0, remainder;
+
+    lck_mtx_lock(&event_writer_lock);
+
+    if (write_buffer == NULL) {
+	if (kmem_alloc(kernel_map, (vm_offset_t *)&write_buffer, WRITE_BUFFER_SIZE)) {
+	    lck_mtx_unlock(&event_writer_lock);
+	    return ENOMEM;
+	}
+    }
+
+    //
+    // this loop copies in and processes the events written.
+    // it takes care to copy in reasonable size chunks and
+    // process them.  if there is an event that spans a chunk
+    // boundary we're careful to copy those bytes down to the
+    // beginning of the buffer and read the next chunk in just
+    // after it.
+    //
+    while(uio_resid(uio)) {
+	if (uio_resid(uio) > (WRITE_BUFFER_SIZE-offset)) {
+	    count = WRITE_BUFFER_SIZE - offset;
+	} else {
+	    count = uio_resid(uio);
+	}
+
+	error = uiomove(write_buffer+offset, count, uio);
+	if (error) {
+	    break;
+	}
+
+	// printf("fsevents: write: copied in %d bytes (offset: %ld)\n", count, offset);
+	error = parse_buffer_and_add_events(write_buffer, offset+count, ctx, &remainder);
+	if (error) {
+	    break;
+	}	    
+
+	//
+	// if there's any remainder, copy it down to the beginning
+	// of the buffer so that it will get processed the next time
+	// through the loop.  note that the remainder always starts
+	// at an event boundary.
+	//
+	if (remainder != 0) {
+	    // printf("fsevents: write: an event spanned a %d byte boundary.  remainder: %ld\n",
+	    //	WRITE_BUFFER_SIZE, remainder);
+	    memmove(write_buffer, (write_buffer+count+offset) - remainder, remainder);
+	    offset = remainder;
+	} else {
+	    offset = 0;
+	}
+    }
+
+    lck_mtx_unlock(&event_writer_lock);
+
+    return error;
 }
 
 
@@ -1193,32 +2276,82 @@ static struct fileops fsevents_fops = {
     fseventsf_drain
 };
 
+typedef struct ext_fsevent_clone_args {
+    user_addr_t  event_list;
+    int32_t      num_events;
+    int32_t      event_queue_depth;
+    user_addr_t  fd;
+} ext_fsevent_clone_args;
+
+typedef struct old_fsevent_clone_args {
+    int32_t  event_list;
+    int32_t  num_events;
+    int32_t  event_queue_depth;
+    int32_t  fd;
+} old_fsevent_clone_args;
 
+#define	OLD_FSEVENTS_CLONE	_IOW('s', 1, old_fsevent_clone_args)
 
 static int
-fseventsioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
+fseventsioctl(__unused dev_t dev, u_long cmd, caddr_t data, __unused int flag, struct proc *p)
 {
     struct fileproc *f;
     int fd, error;
     fsevent_handle *fseh = NULL;
-    fsevent_clone_args *fse_clone_args=(fsevent_clone_args *)data;
+    ext_fsevent_clone_args *fse_clone_args, _fse_clone;
     int8_t *event_list;
+    int is64bit = proc_is64bit(p);
 
     switch (cmd) {
+	case OLD_FSEVENTS_CLONE: {
+	    old_fsevent_clone_args *old_args = (old_fsevent_clone_args *)data;
+
+	    fse_clone_args = &_fse_clone;
+	    memset(fse_clone_args, 0, sizeof(ext_fsevent_clone_args));
+
+	    fse_clone_args->event_list        = CAST_USER_ADDR_T(old_args->event_list);
+	    fse_clone_args->num_events        = old_args->num_events;
+	    fse_clone_args->event_queue_depth = old_args->event_queue_depth;
+	    fse_clone_args->fd                = CAST_USER_ADDR_T(old_args->fd);
+	    goto handle_clone;
+	}
+	    
 	case FSEVENTS_CLONE:
+	    if (is64bit) {
+		fse_clone_args = (ext_fsevent_clone_args *)data;
+	    } else {
+		fsevent_clone_args *ufse_clone = (fsevent_clone_args *)data;
+		
+		fse_clone_args = &_fse_clone;
+		memset(fse_clone_args, 0, sizeof(ext_fsevent_clone_args));
+
+		fse_clone_args->event_list        = CAST_USER_ADDR_T(ufse_clone->event_list);
+		fse_clone_args->num_events        = ufse_clone->num_events;
+		fse_clone_args->event_queue_depth = ufse_clone->event_queue_depth;
+		fse_clone_args->fd                = CAST_USER_ADDR_T(ufse_clone->fd);
+	    }
+
+	handle_clone:
 	    if (fse_clone_args->num_events < 0 || fse_clone_args->num_events > 4096) {
 		return EINVAL;
 	    }
 
 	    MALLOC(fseh, fsevent_handle *, sizeof(fsevent_handle),
 		   M_TEMP, M_WAITOK);
+	    if (fseh == NULL) {
+		return ENOMEM;
+	    }
 	    memset(fseh, 0, sizeof(fsevent_handle));
 
 	    MALLOC(event_list, int8_t *,
 		   fse_clone_args->num_events * sizeof(int8_t),
 		   M_TEMP, M_WAITOK);
+	    if (event_list == NULL) {
+		FREE(fseh, M_TEMP);
+		return ENOMEM;
+	    }
     
-	    error = copyin(CAST_USER_ADDR_T(fse_clone_args->event_list),
+	    error = copyin(fse_clone_args->event_list,
 			   (void *)event_list,
 			   fse_clone_args->num_events * sizeof(int8_t));
 	    if (error) {
@@ -1237,7 +2370,10 @@ fseventsioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
 		return error;
 	    }
 
-	    error = falloc(p, &f, &fd);
+	    // connect up the watcher with this fsevent_handle
+	    fseh->watcher->fseh = fseh;
+
+	    error = falloc(p, &f, &fd, vfs_context_current());
 	    if (error) {
 		FREE(event_list, M_TEMP);
 		FREE(fseh, M_TEMP);
@@ -1248,12 +2384,16 @@ fseventsioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
 	    f->f_fglob->fg_type = DTYPE_FSEVENTS;
 	    f->f_fglob->fg_ops = &fsevents_fops;
 	    f->f_fglob->fg_data = (caddr_t) fseh;
-		proc_fdunlock(p);
-	    copyout((void *)&fd, CAST_USER_ADDR_T(fse_clone_args->fd), sizeof(int32_t));
+	    proc_fdunlock(p);
+	    error = copyout((void *)&fd, fse_clone_args->fd, sizeof(int32_t));
+	    if (error != 0) {
+		fp_free(p, fd, f);
+	    } else {
 		proc_fdlock(p);
 		procfdtbl_releasefd(p, fd, NULL);
 		fp_drop(p, fd, f, 1);
 		proc_fdunlock(p);
+	    }
 	    break;
 
 	default:
@@ -1264,17 +2404,11 @@ fseventsioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
     return error;
 }
 
-static int
-fseventsselect(dev_t dev, int rw, struct proc *p)
-{
-    return 0;
-}
-
 static void
-fsevents_wakeup(fsevent_handle *fseh)
+fsevents_wakeup(fs_event_watcher *watcher)
 {
-    wakeup((caddr_t)fseh);
-    selwakeup(&fseh->si);
+    wakeup((caddr_t)watcher);
+    selwakeup(&watcher->fseh->si);
 }
 
 
@@ -1289,8 +2423,8 @@ static struct cdevsw fsevents_cdevsw =
     fseventsread,		/* read */
     fseventswrite,		/* write */
     fseventsioctl,		/* ioctl */
-    nulldev,			/* stop */
-    nulldev,			/* reset */
+    (stop_fcn_t *)&nulldev,	/* stop */
+    (reset_fcn_t *)&nulldev,	/* reset */
     NULL,			/* tty's */
     eno_select,			/* select */
     eno_mmap,			/* mmap */
@@ -1317,8 +2451,6 @@ fsevents_init(void)
 
     fsevents_installed = 1;
 
-    lockinit(&fsevents_lck, PLOCK, "fsevents", 0, 0);
-
     ret = cdevsw_add(-1, &fsevents_cdevsw);
     if (ret < 0) {
 	fsevents_installed = 0;
@@ -1429,15 +2561,55 @@ get_fse_info(struct vnode *vp, fse_info *fse, vfs_context_t ctx)
     VATTR_WANTED(&va, va_mode);
     VATTR_WANTED(&va, va_uid);
     VATTR_WANTED(&va, va_gid);
+    if (vp->v_flag & VISHARDLINK) {
+	if (vp->v_type == VDIR) {
+	    VATTR_WANTED(&va, va_dirlinkcount);
+	} else {
+	    VATTR_WANTED(&va, va_nlink);
+	}
+    }
+    
     if (vnode_getattr(vp, &va, ctx) != 0) {
+	memset(fse, 0, sizeof(fse_info));
 	return -1;
     }
     
+    fse->ino  = (ino64_t)va.va_fileid;
     fse->dev  = (dev_t)va.va_fsid;
-    fse->ino  = (ino_t)va.va_fileid;
     fse->mode = (int32_t)vnode_vttoif(vnode_vtype(vp)) | va.va_mode;
     fse->uid  = (uid_t)va.va_uid;
     fse->gid  = (gid_t)va.va_gid;
-    
+    if (vp->v_flag & VISHARDLINK) {
+	fse->mode |= FSE_MODE_HLINK;
+	if (vp->v_type == VDIR) {
+	    fse->nlink = (uint64_t)va.va_dirlinkcount;
+	} else {
+	    fse->nlink = (uint64_t)va.va_nlink;
+	}
+    }    
+
     return 0;
 }
+
+#else /* CONFIG_FSE */
+/*
+ * The get_pathbuff and release_pathbuff routines are used in places not
+ * related to fsevents, and it's a handy abstraction, so define trivial
+ * versions that don't cache a pool of buffers.  This way, we don't have
+ * to conditionalize the callers, and they still get the advantage of the
+ * pool of buffers if CONFIG_FSE is turned on.
+ */
+char *
+get_pathbuff(void)
+{
+	char *path;
+	MALLOC_ZONE(path, char *, MAXPATHLEN, M_NAMEI, M_WAITOK);
+	return path;
+}
+
+void
+release_pathbuff(char *path)
+{
+	FREE_ZONE(path, MAXPATHLEN, M_NAMEI);
+}
+#endif /* CONFIG_FSE */
diff --git a/bsd/vfs/vfs_fslog.c b/bsd/vfs/vfs_fslog.c
new file mode 100644
index 000000000..742390951
--- /dev/null
+++ b/bsd/vfs/vfs_fslog.c
@@ -0,0 +1,460 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <stdarg.h>
+#include <sys/errno.h>
+#include <sys/types.h>
+#include <sys/malloc.h>
+#include <sys/buf.h>
+#include <sys/time.h>
+#include <sys/kauth.h>
+#include <sys/mount.h>
+#include <sys/vnode.h>
+#include <sys/syslog.h>		/* for vaddlog() */
+#include <sys/vnode_internal.h>
+#include <dev/random/randomdev.h>
+
+#include <sys/fslog.h>
+#include <sys/mount_internal.h>
+
+/* String to append as format modifier for each key-value pair */
+#define FSLOG_KEYVAL_FMT	"[%s %s] " 
+#define FSLOG_KEYVAL_FMT_LEN	(sizeof(FSLOG_KEYVAL_FMT) - 1)
+
+#define FSLOG_NEWLINE_CHAR	"\n"
+#define FSLOG_NEWLINE_CHAR_LEN	(sizeof(FSLOG_NEWLINE_CHAR) - 1)
+
+/* Length of entire ASL message in 10 characters.  Kernel defaults to zero */
+#define FSLOG_ASL_MSG_LEN	"         0"
+
+/* Length of default format string to be used by printf */
+#define MAX_FMT_LEN		256
+
+/* Internal function to print input values as key-value pairs in format 
+ * identifiable by Apple system log (ASL) facility.   All key-value pairs
+ * are assumed to be pointer to strings and are provided using two ways - 
+ * (a) va_list argument which is a list of varying number of arguments 
+ *     created by the caller of this function.
+ * (b) variable number of arguments passed to this function. 
+ * 
+ * Parameters -
+ * 	level 	  - Priority level for this ASL message 
+ *	facility  - Facility for this ASL message.
+ *	num_pairs - Number of key-value pairs provided by vargs argument.
+ *	vargs 	  - List of key-value pairs.  
+ *	... 	  - Additional key-value pairs (apart from vargs) as variable 
+ *	      	    argument list.  A NULL value indicates the end of the 
+ *	      	    variable argument list.
+ * 
+ * Returns - 
+ *	zero	- On success, when it prints all key-values pairs provided.
+ *	E2BIG	- When it cannot print all key-value pairs provided and had
+ *		  to truncate the output.
+ */
+static int fslog_asl_msg(int level, const char *facility, int num_pairs, va_list vargs, ...)
+{
+	int err = 0;
+	char fmt[MAX_FMT_LEN];	/* Format string to use with vaddlog */
+	int calc_pairs = 0;
+	size_t len;
+	int i;
+	va_list ap;
+	char *ptr;
+
+	/* Mask extra bits, if any, from priority level */
+	level = LOG_PRI(level);
+
+	/* Create the first part of format string consisting of ASL 
+	 * message length, level, and facility.
+	 */
+	if (facility) {
+		snprintf(fmt, MAX_FMT_LEN, "%s [%s %d] [%s %d] [%s %s] ",
+			FSLOG_ASL_MSG_LEN,
+			FSLOG_KEY_LEVEL, level,
+			FSLOG_KEY_READ_UID, FSLOG_VAL_READ_UID, 
+			FSLOG_KEY_FACILITY, facility);
+	} else {
+		snprintf(fmt, MAX_FMT_LEN, "%s [%s %d] [%s %d] ", 
+			FSLOG_ASL_MSG_LEN,
+			FSLOG_KEY_LEVEL, level,
+			FSLOG_KEY_READ_UID, FSLOG_VAL_READ_UID);
+	}
+
+	/* Determine the number of key-value format string [%s %s] that 
+	 * should be added in format string for every key-value pair provided
+	 * in va_list.  Calculate maximum number of format string that can be 
+	 * accommodated in the remaining format buffer (after saving space
+	 * for newline character).  If the caller provided pairs in va_list 
+	 * is more than calculated pairs, truncate extra pairs.
+	 */
+	len = MAX_FMT_LEN - strlen(fmt) - FSLOG_NEWLINE_CHAR_LEN - 1;
+	calc_pairs = len / FSLOG_KEYVAL_FMT_LEN;
+	if (num_pairs <= calc_pairs) {
+		calc_pairs = num_pairs;
+	} else {
+		err = E2BIG;
+	}
+
+	/* Append format strings [%s %s] for the key-value pairs in vargs */
+	len = MAX_FMT_LEN - FSLOG_NEWLINE_CHAR_LEN;
+	for (i = 0; i < calc_pairs; i++) {
+		(void) strlcat(fmt, FSLOG_KEYVAL_FMT, len);
+	}
+
+	/* Count number of variable arguments provided to this function 
+	 * and determine total number of key-value pairs.
+	 */
+	calc_pairs = 0;
+	va_start(ap, vargs);
+	ptr = va_arg(ap, char *);
+	while (ptr) {
+		calc_pairs++;
+		ptr = va_arg(ap, char *);
+	}
+	calc_pairs /= 2;
+	va_end(ap);
+
+	/* If user provided variable number of arguments, append them as
+	 * as real key-value "[k v]" into the format string.  If the format 
+	 * string is too small, ignore the key-value pair completely.
+	 */
+	if (calc_pairs) {
+		char *key, *val;
+		size_t pairlen;
+		int offset;
+
+		/* Calculate bytes available for key-value pairs after reserving 
+		 * bytes for newline character and NULL terminator
+		 */
+		len = MAX_FMT_LEN - strlen(fmt) - FSLOG_NEWLINE_CHAR_LEN - 1;
+		offset = strlen(fmt);
+
+		va_start(ap, vargs);
+		for (i = 0; i < calc_pairs; i++) {
+			key = va_arg(ap, char *);
+			val = va_arg(ap, char *);
+
+			/* Calculate bytes required to store next key-value pair as
+			 * "[key val] " including space for '[', ']', and two spaces.
+			 */
+			pairlen = strlen(key) + strlen(val) + 4;
+			if (pairlen > len) {
+				err = E2BIG;
+				break;
+			} 
+
+			/* len + 1 because one byte has been set aside for NULL 
+			 * terminator in calculation of 'len' above
+			 */
+			snprintf((fmt + offset), len + 1, FSLOG_KEYVAL_FMT, key, val);
+			offset += pairlen;
+			len -= pairlen;
+		}
+		va_end(ap);
+	}
+
+	/* Append newline */
+	(void) strlcat(fmt, FSLOG_NEWLINE_CHAR, MAX_FMT_LEN);
+
+	/* Print the key-value pairs in ASL format */
+	vaddlog(fmt, vargs);
+
+	return err;
+}
+
+/* Log file system related error in key-value format identified by Apple 
+ * system log (ASL) facility.  The key-value pairs are string pointers 
+ * (char *) and are provided as variable arguments list.  A NULL value 
+ * indicates end of the list.
+ *
+ * Keys can not contain '[', ']', space, and newline.  Values can not 
+ * contain '[', ']', and newline.  If any key-value contains any of the 
+ * reserved characters, the behavior is undefined.  The caller of the 
+ * function should escape any occurrences of '[' and ']' by prefixing
+ * it with '\'.
+ * 
+ * The function takes a message ID which can be used to logically group
+ * different ASL messages.  Messages in same logical group have same message
+ * ID and have information to describe order of the message --- first, 
+ * middle, or last.
+ *
+ * The following message IDs have special meaning - 
+ * FSLOG_MSG_FIRST - This message is the first message in its logical
+ *	group.  This generates a unique message ID, creates two key-value 
+ *	pairs with message ID and order of the message as "First".
+ * FSLOG_MSG_LAST - This is really a MASK which should be logically OR'ed 
+ *	with message ID to indicate the last message for a logical group.  
+ *	This also creates two key-value pairs with message ID and order of 
+ *	message as "Last".
+ * FSLOG_MSG_SINGLE - This signifies that the message is the only message
+ * 	in its logical group.  Therefore no extra key-values are generated 
+ *	for this option.
+ * For all other values of message IDs, it regards them as intermediate 
+ * message and generates two key-value pairs with message ID and order of
+ * message as "Middle".
+ * 
+ * Returns - 
+ *	Message ID of the ASL message printed.  The caller should use
+ * 	this value to print intermediate messages or end the logical message
+ *	group.
+ *	For FSLOG_MSG_SINGLE option, it returns FSLOG_MSG_SINGLE. 
+ */
+unsigned long fslog_err(unsigned long msg_id, ... )
+{
+	va_list ap;
+	int num_pairs;
+	char msg_id_str[21]; /* To convert 64-bit number to string with NULL char */
+	char *arg;
+	const char *msg_order_ptr;
+
+	/* Count number of arguments and key-value pairs provided by user */
+	num_pairs = 0;
+	va_start(ap, msg_id);
+	arg = va_arg(ap, char *);
+	while (arg) {
+		num_pairs++;
+		arg = va_arg(ap, char *);
+	}
+	num_pairs /= 2;
+	va_end(ap);
+
+	va_start(ap, msg_id);
+	if (msg_id == FSLOG_MSG_SINGLE) {
+		/* Single message, do not print message ID and message order */
+		(void) fslog_asl_msg(FSLOG_VAL_LEVEL, FSLOG_VAL_FACILITY, 
+				num_pairs, ap, NULL);
+	} else {
+		if (msg_id == FSLOG_MSG_FIRST) {
+			/* First message, generate random message ID */
+			while ((msg_id == FSLOG_MSG_FIRST) ||
+			       (msg_id == FSLOG_MSG_LAST) ||
+			       (msg_id == FSLOG_MSG_SINGLE)) {
+				msg_id = RandomULong();
+				/* MSB is reserved for indicating last message 
+				 * in sequence.  Clear the MSB while generating
+				 * new message ID.
+				 */
+				msg_id = msg_id >> 1;
+			}
+			msg_order_ptr = FSLOG_VAL_ORDER_FIRST;
+		} else if (msg_id & FSLOG_MSG_LAST) { 
+			/* MSB set to indicate last message for this ID */
+			msg_order_ptr = FSLOG_VAL_ORDER_LAST;
+			/* MSB of message ID is set to indicate last message
+			 * in sequence.  Clear the bit to get real message ID.
+			 */
+			msg_id = msg_id & ~FSLOG_MSG_LAST;
+		} else {
+			/* Intermediate message for this ID */
+			msg_order_ptr = FSLOG_VAL_ORDER_MIDDLE;
+		}
+
+		snprintf(msg_id_str, sizeof(msg_id_str), "%lu", msg_id);
+		(void) fslog_asl_msg(FSLOG_VAL_LEVEL, FSLOG_VAL_FACILITY, 
+			 num_pairs, ap, 
+			 FSLOG_KEY_MSG_ID, msg_id_str, 
+			 FSLOG_KEY_MSG_ORDER, msg_order_ptr, NULL);
+	}
+	va_end(ap);
+	return msg_id;
+}
+
+/* Search if given string contains '[' and ']'.  If any, escape it by 
+ * prefixing with a '\'.  If the length of the string is not big enough, 
+ * no changes are done and error is returned.
+ *
+ * Parameters -
+ * 	str - string that can contain '[' or ']', should be NULL terminated
+ *	len - length, in bytes, of valid data, including NULL character.
+ *	buflen - size of buffer that contains the string
+ */
+static int escape_str(char *str, int len, int buflen)
+{
+	int count;
+	char *src, *dst;
+
+	/* Count number of characters to escape */
+	src = str;
+	count = 0;
+	do {
+		if ((*src == '[') || (*src == ']')) {
+			count++;
+		}
+	} while (*src++);
+
+	if (count) {
+		/* Check if the buffer has enough space to escape all characters */
+		if ((buflen - len) < count) {
+			return ENOSPC;	
+		}
+
+		src = str + len;
+		dst = src + count;
+		while (count) {
+			*dst-- = *src;
+			if ((*src == '[') || (*src == ']')) {
+				/* Last char copied needs to be escaped */
+				*dst-- = '\\';
+				count--;
+			}
+			*src--;
+		}
+	}
+
+	return 0;
+}
+
+/* Log information about runtime file system corruption detected by
+ * the file system.  It takes the VFS mount structure as 
+ * parameter which is used to access the mount point of the 
+ * corrupt volume.  If no mount structure or mount point string 
+ * string exists, nothing is logged to ASL database.
+ * 
+ * Currently prints following information - 
+ * 	1. Mount Point
+ */
+void fslog_fs_corrupt(struct mount *mnt)
+{
+	if (mnt != NULL) {
+		if (mnt->mnt_vfsstat.f_mntonname != NULL) {
+			fslog_err(FSLOG_MSG_SINGLE,
+				  FSLOG_KEY_ERR_TYPE, FSLOG_VAL_ERR_TYPE_FS,
+				  FSLOG_KEY_MNTPT, mnt->mnt_vfsstat.f_mntonname,
+				  NULL);
+		}
+	}
+		
+	return;
+} 
+
+/* Log information about IO error detected in buf_biodone()
+ * Currently prints following information - 
+ * 	1. Physical block number
+ *	2. Logical block number
+ *	3. Device node 
+ *	4. Mount point
+ *	5. Path for file, if any
+ *	6. Error number
+ *	7. Type of IO (read/write)
+ */
+void fslog_io_error(const buf_t bp)
+{
+	int err;
+	unsigned long msg_id;
+	char blknum_str[21];
+	char lblknum_str[21];
+	char errno_str[6];
+	const char *iotype;
+	unsigned char print_last = 0;
+	vnode_t	vp;
+
+	if (buf_error(bp) == 0) {
+		return;
+	}
+
+	/* Convert error number to string */
+	snprintf (errno_str, sizeof(errno_str), "%d", buf_error(bp));
+
+	/* Determine type of IO operation */
+	if (buf_flags(bp) & B_READ) {
+		iotype = FSLOG_VAL_IOTYPE_READ;	
+	} else {
+		iotype = FSLOG_VAL_IOTYPE_WRITE;	
+	}
+
+	/* Convert physical block number to string */
+	snprintf (blknum_str, sizeof(blknum_str), "%lld", buf_blkno(bp));
+
+	/* Convert logical block number to string */
+	snprintf (lblknum_str, sizeof(lblknum_str), "%lld", buf_lblkno(bp));
+
+	msg_id = fslog_err(FSLOG_MSG_FIRST,
+				FSLOG_KEY_ERR_TYPE, FSLOG_VAL_ERR_TYPE_IO,
+				FSLOG_KEY_ERRNO, errno_str,
+				FSLOG_KEY_IOTYPE, iotype,
+				FSLOG_KEY_PHYS_BLKNUM, blknum_str,
+				FSLOG_KEY_LOG_BLKNUM, lblknum_str,
+				NULL);
+	
+	/* Access the vnode for this buffer */
+	vp = buf_vnode(bp);
+	if (vp) {
+		struct vfsstatfs *sp;
+		mount_t	mp;
+		char *path;
+		int len;
+		struct vfs_context context;
+
+		mp = vnode_mount(vp);
+		/* mp should be NULL only for bdevvp during boot */
+		if (mp == NULL) {
+			goto out;
+		}
+		sp = vfs_statfs(mp);
+
+		/* Access the file path */
+		MALLOC(path, char *, MAXPATHLEN, M_TEMP, M_WAITOK);
+		if (path) {
+			len = MAXPATHLEN;
+			context.vc_thread = current_thread();
+			context.vc_ucred = kauth_cred_get();
+			/* Find path without entering file system */
+			err = build_path(vp, path, len, &len, BUILDPATH_NO_FS_ENTER,
+					 &context);	
+			if (!err) {
+				err = escape_str(path, len, MAXPATHLEN);
+				if (!err) {
+					/* Print device node, mount point, path */
+					msg_id = fslog_err(msg_id | FSLOG_MSG_LAST, 
+						FSLOG_KEY_DEVNODE, sp->f_mntfromname,
+						FSLOG_KEY_MNTPT, sp->f_mntonname, 
+						FSLOG_KEY_PATH, path, 
+						NULL);
+					print_last = 1;
+				}
+			}
+			FREE(path, M_TEMP);
+		} 
+
+		if (print_last == 0) {
+			/* Print device node and mount point */
+			msg_id = fslog_err(msg_id | FSLOG_MSG_LAST, 
+					FSLOG_KEY_DEVNODE, sp->f_mntfromname,
+					FSLOG_KEY_MNTPT, sp->f_mntonname, 
+					NULL);
+			print_last = 1;
+		} 
+	}
+
+out:
+	if (print_last == 0) {
+		msg_id = fslog_err(msg_id | FSLOG_MSG_LAST, NULL);
+	}
+
+	return;
+}
diff --git a/bsd/vfs/vfs_init.c b/bsd/vfs/vfs_init.c
index 22e7a8d80..fe9c904c5 100644
--- a/bsd/vfs/vfs_init.c
+++ b/bsd/vfs/vfs_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,12 @@
  *
  *	@(#)vfs_init.c	8.5 (Berkeley) 5/11/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 
 #include <sys/param.h>
@@ -72,6 +84,14 @@
 #include <sys/errno.h>
 #include <sys/malloc.h>
 
+#include <vfs/vfs_journal.h>	/* journal_init() */
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#include <sys/kauth.h>
+#endif
+#if QUOTA
+#include <sys/quota.h>
+#endif
 
 /*
  * Sigh, such primitive tools are these...
@@ -82,7 +102,8 @@
 #define DODEBUG(A)
 #endif
 
-extern uid_t console_user;
+__private_extern__ void vntblinit(void) __attribute__((section("__TEXT, initcode")));
+
 extern struct vnodeopv_desc *vfs_opv_descs[];
 				/* a list of lists of vnodeops defns */
 extern struct vnodeop_desc *vfs_op_descs[];
@@ -95,14 +116,14 @@ extern struct vnodeop_desc *vfs_op_descs[];
  */
 int vfs_opv_numops;
 
-typedef (*PFI)();   /* the standard Pointer to a Function returning an Int */
+typedef int (*PFIvp)(void *); 
 
 /*
  * A miscellaneous routine.
  * A generic "default" routine that just returns an error.
  */
 int
-vn_default_error()
+vn_default_error(void)
 {
 
 	return (ENOTSUP);
@@ -125,7 +146,7 @@ vn_default_error()
  * that is a(whole)nother story.) This is a feature.
  */
 void
-vfs_opv_init()
+vfs_opv_init(void)
 {
 	int i, j, k;
 	int (***opv_desc_vector_p)(void *);
@@ -142,9 +163,9 @@ vfs_opv_init()
 		 * Also handle backwards compatibility.
 		 */
 		if (*opv_desc_vector_p == NULL) {
-			MALLOC(*opv_desc_vector_p, PFI*,
-			       vfs_opv_numops*sizeof(PFI), M_TEMP, M_WAITOK);
-			bzero (*opv_desc_vector_p, vfs_opv_numops*sizeof(PFI));
+			MALLOC(*opv_desc_vector_p, PFIvp*,
+			       vfs_opv_numops*sizeof(PFIvp), M_TEMP, M_WAITOK);
+			bzero (*opv_desc_vector_p, vfs_opv_numops*sizeof(PFIvp));
 			DODEBUG(printf("vector at %x allocated\n",
 			    opv_desc_vector_p));
 		}
@@ -208,7 +229,7 @@ vfs_opv_init()
  * Initialize known vnode operations vectors.
  */
 void
-vfs_op_init()
+vfs_op_init(void)
 {
 	int i;
 
@@ -245,11 +266,8 @@ lck_attr_t * vnode_lck_attr;
 lck_grp_t * vnode_list_lck_grp;
 lck_grp_attr_t * vnode_list_lck_grp_attr;
 lck_attr_t * vnode_list_lck_attr;
-lck_mtx_t * vnode_list_mtx_lock;
+lck_spin_t * vnode_list_spin_lock;
 lck_mtx_t * spechash_mtx_lock;
-/* Routine to lock and unlock the  vnode lists */
-void vnode_list_lock(void);
-void vnode_list_unlock(void);
 
 /* vars for vfsconf lock */
 lck_grp_t * fsconf_lck_grp;
@@ -268,14 +286,12 @@ lck_grp_attr_t * mnt_list_lck_grp_attr;
 lck_attr_t * mnt_list_lck_attr;
 lck_mtx_t * mnt_list_mtx_lock;
 
-extern void journal_init();
-
 struct mount * dead_mountp;
 /*
  * Initialize the vnode structures and initialize each file system type.
  */
 void
-vfsinit()
+vfsinit(void)
 {
 	struct vfstable *vfsp;
 	int i, maxtypenum;
@@ -290,7 +306,7 @@ vfsinit()
 	vnode_list_lck_attr = lck_attr_alloc_init();
 
 	/* Allocate vnode list lock */
-	vnode_list_mtx_lock = lck_mtx_alloc_init(vnode_list_lck_grp, vnode_list_lck_attr);
+	vnode_list_spin_lock = lck_spin_alloc_init(vnode_list_lck_grp, vnode_list_lck_attr);
 
 	/* Allocate spec hash list lock */
 	spechash_mtx_lock = lck_mtx_alloc_init(vnode_list_lck_grp, vnode_list_lck_attr);
@@ -333,11 +349,6 @@ vfsinit()
 	/* Allocate mount lock attribute */
 	mnt_lck_attr = lck_attr_alloc_init();
 
-	/*
-	 * Initialize the "console user" for access purposes:
-	 */
-	console_user = (uid_t)0;
-	
 	/*
 	 * Initialize the vnode table
 	 */
@@ -350,10 +361,14 @@ vfsinit()
 	 * Initialize the vnode name cache
 	 */
 	nchinit();
+
+#if JOURNALING
 	/*
 	 * Initialize the journaling locks
 	 */
 	journal_init();
+#endif 
+
 	/*
 	 * Build vnode operation vectors.
 	 */
@@ -370,7 +385,8 @@ vfsinit()
 		if (i) vfsconf[i-1].vfc_next = vfsp;
 		if (maxtypenum <= vfsp->vfc_typenum)
 			maxtypenum = vfsp->vfc_typenum + 1;
-		(*vfsp->vfc_vfsops->vfs_init)(vfsp);
+		/* a vfsconf is a prefix subset of a vfstable... */
+		(*vfsp->vfc_vfsops->vfs_init)((struct vfsconf *)vfsp);
 		
 		lck_mtx_init(&vfsp->vfc_lock, fsconf_lck_grp, fsconf_lck_attr);
 		
@@ -383,6 +399,13 @@ vfsinit()
 	 * Initialize the vnop authorization scope.
 	 */
 	vnode_authorize_init();
+
+	/*
+	 * Initialiize the quota system.
+	 */
+#if QUOTA
+	dqinit();
+#endif
 	
 	/* 
 	 * create a mount point for dead vnodes
@@ -396,6 +419,10 @@ vfsinit()
 	mp->mnt_maxsegreadsize = mp->mnt_maxreadcnt;
 	mp->mnt_maxsegwritesize = mp->mnt_maxwritecnt;
 	mp->mnt_devblocksize = DEV_BSIZE;
+	mp->mnt_alignmentmask = PAGE_MASK;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
     
 	TAILQ_INIT(&mp->mnt_vnodelist);
 	TAILQ_INIT(&mp->mnt_workerqueue);
@@ -403,29 +430,34 @@ vfsinit()
 	mp->mnt_flag = MNT_LOCAL;
 	mp->mnt_lflag = MNT_LDEAD;
 	mount_lock_init(mp);
+
+#if CONFIG_MACF
+	mac_mount_label_init(mp);
+	mac_mount_label_associate(vfs_context_kernel(), mp);
+#endif
 	dead_mountp = mp;
 }
 
 void
-vnode_list_lock()
+vnode_list_lock(void)
 {
-	lck_mtx_lock(vnode_list_mtx_lock);
+	lck_spin_lock(vnode_list_spin_lock);
 }
 
 void
-vnode_list_unlock()
+vnode_list_unlock(void)
 {
-	lck_mtx_unlock(vnode_list_mtx_lock);
+	lck_spin_unlock(vnode_list_spin_lock);
 }
 
 void
-mount_list_lock()
+mount_list_lock(void)
 {
 	lck_mtx_lock(mnt_list_mtx_lock);
 }
 
 void
-mount_list_unlock()
+mount_list_unlock(void)
 {
 	lck_mtx_unlock(mnt_list_mtx_lock);
 }
diff --git a/bsd/vfs/vfs_journal.c b/bsd/vfs/vfs_journal.c
index d2c369223..f1d48861c 100644
--- a/bsd/vfs/vfs_journal.c
+++ b/bsd/vfs/vfs_journal.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1995-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 //
 // This file implements a simple write-ahead journaling layer.  
@@ -47,10 +53,14 @@
 #include <sys/malloc.h>
 #include <kern/thread.h>
 #include <sys/disk.h>
+#include <sys/kdebug.h>
 #include <miscfs/specfs/specdev.h>
+#include <libkern/OSAtomic.h>	/* OSAddAtomic */
 
 extern task_t kernel_task;
 
+#define DBG_JOURNAL_FLUSH 1
+
 #else
 
 #include <stdio.h>
@@ -68,6 +78,14 @@ extern task_t kernel_task;
 
 #include "vfs_journal.h"
 
+/* XXX next prototytype should be from libsa/stdlib.h> but conflicts libkern */
+__private_extern__ void qsort(
+    void * array,
+    size_t nmembers,
+    size_t member_size,
+    int (*)(const void *, const void *));
+
+
 
 // number of bytes to checksum in a block_list_header
 // NOTE: this should be enough to clear out the header
@@ -75,8 +93,7 @@ extern task_t kernel_task;
 #define BLHDR_CHECKSUM_SIZE 32
 
 
-
-static int  end_transaction(transaction *tr, int force_it);
+static int end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg);
 static void abort_transaction(journal *jnl, transaction *tr);
 static void dump_journal(journal *jnl);
 
@@ -93,18 +110,19 @@ static __inline__ void  unlock_oldstart(journal *jnl);
 //
 
 typedef struct bucket {
-    off_t block_num;
-    size_t jnl_offset;
-    size_t block_size;
+    off_t   block_num;
+    size_t  jnl_offset;
+    size_t  block_size;
+    int32_t cksum;
 } bucket;
 
 #define STARTING_BUCKETS 256
 
-static int add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr);
+static int add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr);
 static int grow_table(struct bucket **buf_ptr, int num_buckets, int new_size);
 static int lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full);
-static int do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr);
-static int insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr, int overwriting);
+static int do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr);
+static int insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr, int overwriting);
 
 #define CHECK_JOURNAL(jnl) \
     do { \
@@ -148,7 +166,7 @@ static int insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, of
 	panic("%s:%d: null tr->jnl ptr?\n", __FILE__, __LINE__);\
     }\
     if (tr->blhdr != (block_list_header *)tr->tbuffer) {\
-	panic("%s:%d: blhdr (0x%x) != tbuffer (0x%x)\n", __FILE__, __LINE__, tr->blhdr, tr->tbuffer);\
+	panic("%s:%d: blhdr (%p) != tbuffer (%p)\n", __FILE__, __LINE__, tr->blhdr, tr->tbuffer);\
     }\
     if (tr->total_bytes < 0) {\
 	panic("%s:%d: tr total_bytes looks bad: %d\n", __FILE__, __LINE__, tr->total_bytes);\
@@ -192,7 +210,7 @@ lck_attr_t *      jnl_lock_attr;
 lck_grp_t *       jnl_mutex_group;
 
 void
-journal_init()
+journal_init(void)
 {
 	jnl_lock_attr    = lck_attr_alloc_init();
 	jnl_group_attr   = lck_grp_attr_alloc_init();
@@ -241,20 +259,21 @@ unlock_oldstart(journal *jnl)
 static size_t
 do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction)
 {
-    int         err, io_sz=0, curlen=len;
+    int         err, curlen=len;
+    size_t      io_sz = 0;
     buf_t	bp;
-    int 	max_iosize = 128 * 1024;
-    struct vfsioattr ioattr;
+    off_t 	max_iosize;
 
     if (*offset < 0 || *offset > jnl->jhdr->size) {
 		panic("jnl: do_jnl_io: bad offset 0x%llx (max 0x%llx)\n", *offset, jnl->jhdr->size);
     }
-    vfs_ioattr(vnode_mount(jnl->jdev), &ioattr);
 
     if (direction & JNL_WRITE)
-            max_iosize = ioattr.io_maxwritecnt;
+	max_iosize = jnl->max_write_size;
     else if (direction & JNL_READ)
-            max_iosize = ioattr.io_maxreadcnt;
+	max_iosize = jnl->max_read_size;
+    else
+	max_iosize = 128 * 1024;
 
   again:
     bp = alloc_io_buf(jnl->jdev, 1);
@@ -272,7 +291,7 @@ do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction
 	}
 
     if (curlen <= 0) {
-		panic("jnl: do_jnl_io: curlen == %d, offset 0x%llx len %d\n", curlen, *offset, len);
+		panic("jnl: do_jnl_io: curlen == %d, offset 0x%llx len %lu\n", curlen, *offset, len);
     }
 
 	if (*offset == 0 && (direction & JNL_HEADER) == 0) {
@@ -292,6 +311,9 @@ do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction
     buf_setdataptr(bp, (uintptr_t)data);
     buf_setblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size));
     buf_setlblkno(bp, (daddr64_t) ((jnl->jdev_offset + *offset) / (off_t)jnl->jhdr->jhdr_size));
+    if ((direction & JNL_WRITE) && (jnl->flags & JOURNAL_DO_FUA_WRITES)) {
+	buf_markfua(bp);
+    }
 
     err = VNOP_STRATEGY(bp);
     if (!err) {
@@ -300,8 +322,8 @@ do_journal_io(journal *jnl, off_t *offset, void *data, size_t len, int direction
     free_io_buf(bp);
 
     if (err) {
-		printf("jnl: do_jnl_io: strategy err 0x%x\n", err);
-		return 0;
+	printf("jnl: %s: do_jnl_io: strategy err 0x%x\n", jnl->jdev_name, err);
+	return 0;
     }
 
     *offset += curlen;
@@ -332,7 +354,7 @@ write_journal_data(journal *jnl, off_t *offset, void *data, size_t len)
 }
 
 
-static int
+static size_t
 read_journal_header(journal *jnl, void *data, size_t len)
 {
 	off_t hdr_offset = 0;
@@ -343,17 +365,20 @@ read_journal_header(journal *jnl, void *data, size_t len)
 static int
 write_journal_header(journal *jnl)
 {
-	static int num_err_prints = 0;
-    int ret;
+    static int num_err_prints = 0;
+    int ret=0;
     off_t jhdr_offset = 0;
     struct vfs_context context;
 
-    context.vc_proc = current_proc();
+    context.vc_thread = current_thread();
     context.vc_ucred = NOCRED;
     // 
-    // XXXdbg note: this ioctl doesn't seem to do anything on firewire disks.
+    // Flush the track cache if we're not doing force-unit-access
+    // writes.
     //
-    ret = VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context);
+    if ((jnl->flags & JOURNAL_DO_FUA_WRITES) == 0) {
+	ret = VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context);
+    }
     if (ret != 0) {
 	//
 	// Only print this error if it's a different error than the
@@ -368,30 +393,32 @@ write_journal_header(journal *jnl)
 	    || (jnl->flags & JOURNAL_FLUSHCACHE_ERR) == 0
 	    || num_err_prints++ < 25) {
 	    
-	    printf("jnl: flushing fs disk buffer returned 0x%x\n", ret);
+	    printf("jnl: %s: flushing fs disk buffer returned 0x%x\n", jnl->jdev_name, ret);
 	    
 	    jnl->flags |= JOURNAL_FLUSHCACHE_ERR;
 	    jnl->last_flush_err = ret;
 	}
     }
 
-    
     jnl->jhdr->checksum = 0;
-    jnl->jhdr->checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header));
-    if (do_journal_io(jnl, &jhdr_offset, jnl->header_buf, jnl->jhdr->jhdr_size, JNL_WRITE|JNL_HEADER) != jnl->jhdr->jhdr_size) {
-	printf("jnl: write_journal_header: error writing the journal header!\n");
+    jnl->jhdr->checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
+    if (do_journal_io(jnl, &jhdr_offset, jnl->header_buf, jnl->jhdr->jhdr_size, JNL_WRITE|JNL_HEADER) != (size_t)jnl->jhdr->jhdr_size) {
+	printf("jnl: %s: write_journal_header: error writing the journal header!\n", jnl->jdev_name);
 	jnl->flags |= JOURNAL_INVALID;
 	return -1;
     }	
 
-    // Have to flush after writing the journal header so that
+    // If we're not doing force-unit-access writes, then we
+    // have to flush after writing the journal header so that
     // a future transaction doesn't sneak out to disk before
     // the header does and thus overwrite data that the old
     // journal header refers to.  Saw this exact case happen
     // on an IDE bus analyzer with Larry Barras so while it
     // may seem obscure, it's not.
     //
-    VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context);
+    if ((jnl->flags & JOURNAL_DO_FUA_WRITES) == 0) {
+	VNOP_IOCTL(jnl->jdev, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, &context);
+    }
 
     return 0;
 }
@@ -437,7 +464,8 @@ buffer_flushed_callback(struct buf *bp, void *arg)
     transaction  *tr;
     journal      *jnl;
     transaction  *ctr, *prev=NULL, *next;
-    int           i, bufsize;
+    size_t        i;
+    int           bufsize, amt_flushed, total_bytes;
 
 
     //printf("jnl: buf flush: bp @ 0x%x l/blkno %qd/%qd vp 0x%x tr @ 0x%x\n",
@@ -461,119 +489,139 @@ buffer_flushed_callback(struct buf *bp, void *arg)
 
     CHECK_JOURNAL(jnl);
 
+    amt_flushed = tr->num_killed;
+    total_bytes = tr->total_bytes;
+    
     // update the number of blocks that have been flushed.
     // this buf may represent more than one block so take
     // that into account.
-    OSAddAtomic(bufsize, &tr->num_flushed);
+    //
+    // OSAddAtomic() returns the value of tr->num_flushed before the add
+    //
+    amt_flushed += OSAddAtomic(bufsize, (SInt32 *)&tr->num_flushed);
 
 
     // if this transaction isn't done yet, just return as
     // there is nothing to do.
-    if ((tr->num_flushed + tr->num_killed) < tr->total_bytes) {
+    //
+    // NOTE: we are careful to not reference anything through
+    //       the tr pointer after doing the OSAddAtomic().  if
+    //       this if statement fails then we are the last one
+    //       and then it's ok to dereference "tr".
+    //
+    if ((amt_flushed + bufsize) < total_bytes) {
 		return;
     }
 
-	// this will single thread checking the transaction
-	lock_oldstart(jnl);
+    // this will single thread checking the transaction
+    lock_oldstart(jnl);
 
-	if (tr->total_bytes == 0xfbadc0de) {
-		// then someone beat us to it...
-		unlock_oldstart(jnl);
-		return;
-	}
+    if (tr->total_bytes == (int)0xfbadc0de) {
+	// then someone beat us to it...
+	unlock_oldstart(jnl);
+	return;
+    }
 
     // mark this so that we're the owner of dealing with the
-	// cleanup for this transaction
+    // cleanup for this transaction
     tr->total_bytes = 0xfbadc0de;
 
     //printf("jnl: tr 0x%x (0x%llx 0x%llx) in jnl 0x%x completed.\n",
     //   tr, tr->journal_start, tr->journal_end, jnl);
 
-	// find this entry in the old_start[] index and mark it completed
-	for(i=0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) {
-
-		if ((jnl->old_start[i] & ~(0x8000000000000000LL)) == tr->journal_start) {
-			jnl->old_start[i] &= ~(0x8000000000000000LL);
-			break;
-		}
-	}
-	if (i >= sizeof(jnl->old_start)/sizeof(jnl->old_start[0])) {
-		panic("jnl: buffer_flushed: did not find tr w/start @ %lld (tr 0x%x, jnl 0x%x)\n",
-			  tr->journal_start, tr, jnl);
+    // find this entry in the old_start[] index and mark it completed
+    for(i=0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) {
+	
+	if ((off_t)(jnl->old_start[i] & ~(0x8000000000000000ULL)) == tr->journal_start) {
+	    jnl->old_start[i] &= ~(0x8000000000000000ULL);
+	    break;
 	}
-	unlock_oldstart(jnl);
+    }
+
+    if (i >= sizeof(jnl->old_start)/sizeof(jnl->old_start[0])) {
+	panic("jnl: buffer_flushed: did not find tr w/start @ %lld (tr %p, jnl %p)\n",
+	    tr->journal_start, tr, jnl);
+    }
 
 
     // if we are here then we need to update the journal header
     // to reflect that this transaction is complete
     if (tr->journal_start == jnl->active_start) {
-		jnl->active_start = tr->journal_end;
-		tr->journal_start = tr->journal_end = (off_t)0;
+	jnl->active_start = tr->journal_end;
+	tr->journal_start = tr->journal_end = (off_t)0;
     }
 
     // go through the completed_trs list and try to coalesce
     // entries, restarting back at the beginning if we have to.
     for(ctr=jnl->completed_trs; ctr; prev=ctr, ctr=next) {
-		if (ctr->journal_start == jnl->active_start) {
-			jnl->active_start = ctr->journal_end;
-			if (prev) {
-				prev->next = ctr->next;
-			}
-			if (ctr == jnl->completed_trs) {
-				jnl->completed_trs = ctr->next;
-			}
+	if (ctr->journal_start == jnl->active_start) {
+	    jnl->active_start = ctr->journal_end;
+	    if (prev) {
+		prev->next = ctr->next;
+	    }
+	    if (ctr == jnl->completed_trs) {
+		jnl->completed_trs = ctr->next;
+	    }
 	    
-			lock_oldstart(jnl);
-			next           = jnl->completed_trs;   // this starts us over again
-			ctr->next      = jnl->tr_freeme;
-			jnl->tr_freeme = ctr;
-			ctr            = NULL;
-			unlock_oldstart(jnl);
-		} else if (tr->journal_end == ctr->journal_start) {
-			ctr->journal_start = tr->journal_start;
-			next               = jnl->completed_trs;  // this starts us over again
-			ctr                = NULL;
-			tr->journal_start  = tr->journal_end = (off_t)0;
-		} else if (tr->journal_start == ctr->journal_end) {
-			ctr->journal_end  = tr->journal_end;
-			next              = ctr->next;
-			tr->journal_start = tr->journal_end = (off_t)0;
-		} else {
-			next = ctr->next;
-		}
+	    next           = jnl->completed_trs;   // this starts us over again
+	    ctr->next      = jnl->tr_freeme;
+	    jnl->tr_freeme = ctr;
+	    ctr            = NULL;
+	} else if (tr->journal_end == ctr->journal_start) {
+	    ctr->journal_start = tr->journal_start;
+	    next               = jnl->completed_trs;  // this starts us over again
+	    ctr                = NULL;
+	    tr->journal_start  = tr->journal_end = (off_t)0;
+	} else if (tr->journal_start == ctr->journal_end) {
+	    ctr->journal_end  = tr->journal_end;
+	    next              = ctr->next;
+	    tr->journal_start = tr->journal_end = (off_t)0;
+	} else if (ctr->next && ctr->journal_end == ctr->next->journal_start) {
+	    // coalesce the next entry with this one and link the next
+	    // entry in at the head of the tr_freeme list
+	    next              = ctr->next;           // temporarily use the "next" variable
+	    ctr->journal_end  = next->journal_end;
+	    ctr->next         = next->next;
+	    next->next        = jnl->tr_freeme;      // link in the next guy at the head of the tr_freeme list
+	    jnl->tr_freeme    = next;
+
+	    next              = jnl->completed_trs;  // this starts us over again
+	    ctr               = NULL;
+	} else {
+	    next = ctr->next;
+	}
     }
     
     // if this is true then we didn't merge with anyone
     // so link ourselves in at the head of the completed
     // transaction list.
     if (tr->journal_start != 0) {
-		// put this entry into the correct sorted place
-		// in the list instead of just at the head.
-		//
+	// put this entry into the correct sorted place
+	// in the list instead of just at the head.
+	//
 	
-		prev = NULL;
-		for(ctr=jnl->completed_trs; ctr && tr->journal_start > ctr->journal_start; prev=ctr, ctr=ctr->next) {
-			// just keep looping
-		}
+	prev = NULL;
+	for(ctr=jnl->completed_trs; ctr && tr->journal_start > ctr->journal_start; prev=ctr, ctr=ctr->next) {
+	    // just keep looping
+	}
 
-		if (ctr == NULL && prev == NULL) {
-			jnl->completed_trs = tr;
-			tr->next = NULL;
-		} else if (ctr == jnl->completed_trs) {
-			tr->next = jnl->completed_trs;
-			jnl->completed_trs = tr;
-		} else {
-			tr->next = prev->next;
-			prev->next = tr;
-		}
+	if (ctr == NULL && prev == NULL) {
+	    jnl->completed_trs = tr;
+	    tr->next = NULL;
+	} else if (ctr == jnl->completed_trs) {
+	    tr->next = jnl->completed_trs;
+	    jnl->completed_trs = tr;
+	} else {
+	    tr->next = prev->next;
+	    prev->next = tr;
+	}
     } else {
-		// if we're here this tr got merged with someone else so
-		// put it on the list to be free'd
-		lock_oldstart(jnl);
-		tr->next       = jnl->tr_freeme;
-		jnl->tr_freeme = tr;
-		unlock_oldstart(jnl);
+	// if we're here this tr got merged with someone else so
+	// put it on the list to be free'd
+	tr->next       = jnl->tr_freeme;
+	jnl->tr_freeme = tr;
     }
+    unlock_oldstart(jnl);
 }
 
 
@@ -595,6 +643,7 @@ swap_journal_header(journal *jnl)
     jnl->jhdr->blhdr_size = SWAP32(jnl->jhdr->blhdr_size);
     jnl->jhdr->checksum   = SWAP32(jnl->jhdr->checksum);
     jnl->jhdr->jhdr_size  = SWAP32(jnl->jhdr->jhdr_size);
+    jnl->jhdr->sequence_num  = SWAP32(jnl->jhdr->sequence_num);
 }
 
 static void
@@ -606,17 +655,17 @@ swap_block_list_header(journal *jnl, block_list_header *blhdr)
     blhdr->num_blocks = SWAP16(blhdr->num_blocks);
     blhdr->bytes_used = SWAP32(blhdr->bytes_used);
     blhdr->checksum   = SWAP32(blhdr->checksum);
-    blhdr->pad        = SWAP32(blhdr->pad);
+    blhdr->flags      = SWAP32(blhdr->flags);
 
-	if (blhdr->num_blocks * sizeof(blhdr->binfo[0]) > jnl->jhdr->blhdr_size) {
-		printf("jnl: blhdr num blocks looks suspicious (%d).  not swapping.\n", blhdr->num_blocks);
-		return;
-	}
+    if (blhdr->num_blocks >= ((jnl->jhdr->blhdr_size / sizeof(block_info)) - 1)) {
+	printf("jnl: %s: blhdr num blocks looks suspicious (%d / blhdr size %d).  not swapping.\n", jnl->jdev_name, blhdr->num_blocks, jnl->jhdr->blhdr_size);
+	return;
+    }
 
     for(i=0; i < blhdr->num_blocks; i++) {
-		blhdr->binfo[i].bnum  = SWAP64(blhdr->binfo[i].bnum);
-		blhdr->binfo[i].bsize = SWAP32(blhdr->binfo[i].bsize);
-		blhdr->binfo[i].bp    = (void *)SWAP32((int)blhdr->binfo[i].bp);
+		blhdr->binfo[i].bnum    = SWAP64(blhdr->binfo[i].bnum);
+		blhdr->binfo[i].bsize   = SWAP32(blhdr->binfo[i].bsize);
+		blhdr->binfo[i].b.cksum = SWAP32(blhdr->binfo[i].b.cksum);
     }
 }
 
@@ -630,7 +679,7 @@ update_fs_block(journal *jnl, void *block_ptr, off_t fs_block, size_t bsize)
     // first read the block we want.
     ret = buf_meta_bread(jnl->fsdev, (daddr64_t)fs_block, bsize, NOCRED, &oblock_bp);
     if (ret != 0) {
-		printf("jnl: update_fs_block: error reading fs block # %lld! (ret %d)\n", fs_block, ret);
+	printf("jnl: %s: update_fs_block: error reading fs block # %lld! (ret %d)\n", jnl->jdev_name, fs_block, ret);
 
 		if (oblock_bp) {
 			buf_brelse(oblock_bp);
@@ -640,8 +689,8 @@ update_fs_block(journal *jnl, void *block_ptr, off_t fs_block, size_t bsize)
 		// let's try to be aggressive here and just re-write the block
 		oblock_bp = buf_getblk(jnl->fsdev, (daddr64_t)fs_block, bsize, 0, 0, BLK_META);
 		if (oblock_bp == NULL) {
-			printf("jnl: update_fs_block: buf_getblk() for %lld failed! failing update.\n", fs_block);
-			return -1;
+		    printf("jnl: %s: update_fs_block: buf_getblk() for %lld failed! failing update.\n", jnl->jdev_name, fs_block);
+		    return -1;
 		}
     }
 	    
@@ -652,11 +701,11 @@ update_fs_block(journal *jnl, void *block_ptr, off_t fs_block, size_t bsize)
     }
 
     // copy the journal data over top of it
-    memcpy((void *)buf_dataptr(oblock_bp), block_ptr, bsize);
+    memcpy((char *)0 + buf_dataptr(oblock_bp), block_ptr, bsize);
 
     if ((ret = VNOP_BWRITE(oblock_bp)) != 0) {
-		printf("jnl: update_fs_block: failed to update block %lld (ret %d)\n", fs_block,ret);
-		return ret;
+	printf("jnl: %s: update_fs_block: failed to update block %lld (ret %d)\n", jnl->jdev_name, fs_block,ret);
+	return ret;
     }
 
     // and now invalidate it so that if someone else wants to read
@@ -763,7 +812,7 @@ lookup_bucket(struct bucket **buf_ptr, off_t block_num, int num_full)
 }
 
 static int
-insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr, int overwriting)
+insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, size_t size, size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr, int overwriting)
 {
     if (!overwriting) {
 	// grow the table if we're out of space
@@ -772,7 +821,7 @@ insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, si
 	    int grow_size = grow_table(buf_ptr, *num_buckets_ptr, new_size);
 	    
 	    if (grow_size < new_size) {
-		printf("jnl: add_block: grow_table returned an error!\n");
+		printf("jnl: %s: add_block: grow_table returned an error!\n", jnl->jdev_name);
 		return -1;
 	    }
 	    
@@ -792,18 +841,19 @@ insert_block(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t num, si
 	offset = jnl->jhdr->jhdr_size + (offset - jnl->jhdr->size);
     }
     if (size <= 0) {
-	panic("jnl: insert_block: bad size in insert_block (%d)\n", size);
+	panic("jnl: insert_block: bad size in insert_block (%lu)\n", size);
     }	 
 
     (*buf_ptr)[blk_index].block_num = num;
     (*buf_ptr)[blk_index].block_size = size;
     (*buf_ptr)[blk_index].jnl_offset = offset;
+    (*buf_ptr)[blk_index].cksum = cksum;
     
     return blk_index;
 }
 
 static int
-do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr)
+do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num, size_t size, __unused size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr)
 {
     int num_to_remove, index, i, overwrite, err;
     size_t jhdr_size = jnl->jhdr->jhdr_size, new_offset;
@@ -820,7 +870,7 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 	overlap = prev_block_end - block_start;
 	if (overlap > 0) {
 	    if (overlap % jhdr_size != 0) {
-		panic("jnl: do_overlap: overlap with previous entry not a multiple of %d\n", jhdr_size);
+		panic("jnl: do_overlap: overlap with previous entry not a multiple of %lu\n", jhdr_size);
 	    }
 
 	    // if the previous entry completely overlaps this one, we need to break it into two pieces.
@@ -830,7 +880,7 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 
 		new_offset = (*buf_ptr)[blk_index-1].jnl_offset + (block_end - prev_block_start);
 		
-		err = insert_block(jnl, buf_ptr, blk_index, new_num, new_size, new_offset, num_buckets_ptr, num_full_ptr, 0);
+		err = insert_block(jnl, buf_ptr, blk_index, new_num, new_size, new_offset, cksum, num_buckets_ptr, num_full_ptr, 0);
 		if (err < 0) {
 		    panic("jnl: do_overlap: error inserting during pre-overlap\n");
 		}
@@ -838,6 +888,7 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 	    
 	    // Regardless, we need to truncate the previous entry to the beginning of the overlap
 	    (*buf_ptr)[blk_index-1].block_size = block_start - prev_block_start;
+	    (*buf_ptr)[blk_index-1].cksum = 0;   // have to blow it away because there's no way to check it
 	}
     }
 
@@ -845,6 +896,8 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
     if (!overwrite && block_end <= (*buf_ptr)[blk_index].block_num*jhdr_size) {
 	return 0; // no overlap, no overwrite
     } else if (overwrite && (blk_index + 1 >= *num_full_ptr || block_end <= (*buf_ptr)[blk_index+1].block_num*jhdr_size)) {
+
+	(*buf_ptr)[blk_index].cksum = cksum;   // update this
 	return 1; // simple overwrite
     }
     
@@ -862,11 +915,12 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 	    overlap = block_end - (*buf_ptr)[index].block_num*jhdr_size;
 	    if (overlap > 0) {
 		if (overlap % jhdr_size != 0) {
-		    panic("jnl: do_overlap: overlap of %lld is not multiple of %d\n", overlap, jhdr_size);
+		    panic("jnl: do_overlap: overlap of %lld is not multiple of %lu\n", overlap, jhdr_size);
 		}
 		
 		// if we partially overlap this entry, adjust its block number, jnl offset, and size
 		(*buf_ptr)[index].block_num += (overlap / jhdr_size); // make sure overlap is multiple of jhdr_size, or round up
+		(*buf_ptr)[index].cksum = 0;
 		
 		new_offset = (*buf_ptr)[index].jnl_offset + overlap; // check for wrap-around
 		if (new_offset >= jnl->jhdr->size) {
@@ -876,7 +930,7 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 		
 		(*buf_ptr)[index].block_size -= overlap; // sanity check for negative value
 		if ((*buf_ptr)[index].block_size <= 0) {
-		    panic("jnl: do_overlap: after overlap, new block size is invalid (%d)\n", (*buf_ptr)[index].block_size);
+		    panic("jnl: do_overlap: after overlap, new block size is invalid (%lu)\n", (*buf_ptr)[index].block_size);
 		    // return -1; // if above panic is removed, return -1 for error
 		}
 	    }
@@ -917,7 +971,7 @@ do_overlap(journal *jnl, struct bucket **buf_ptr, int blk_index, off_t block_num
 // disrupt the existing time-ordering of blocks in the journal playback, care
 // is taken to catch any overlaps and keep the array consistent. 
 static int
-add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, size_t offset, int *num_buckets_ptr, int *num_full_ptr)
+add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, __unused size_t offset, int32_t cksum, int *num_buckets_ptr, int *num_full_ptr)
 {
     int blk_index, overwriting;
     
@@ -933,13 +987,13 @@ add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, s
     } // else printf("jnl: add_block: adding block 0x%llx at i=%d\n", block_num, blk_index);
     
     // Determine whether we're overwriting an existing entry by checking for overlap
-    overwriting = do_overlap(jnl, buf_ptr, blk_index, block_num, size, offset, num_buckets_ptr, num_full_ptr);
+    overwriting = do_overlap(jnl, buf_ptr, blk_index, block_num, size, offset, cksum, num_buckets_ptr, num_full_ptr);
     if (overwriting < 0) {
 	return -1; // if we got an error, pass it along
     }
         
     // returns the index, or -1 on error
-    blk_index = insert_block(jnl, buf_ptr, blk_index, block_num, size, offset, num_buckets_ptr, num_full_ptr, overwriting);
+    blk_index = insert_block(jnl, buf_ptr, blk_index, block_num, size, offset, cksum, num_buckets_ptr, num_full_ptr, overwriting);
     
     return blk_index;
 }
@@ -947,13 +1001,16 @@ add_block(journal *jnl, struct bucket **buf_ptr, off_t block_num, size_t size, s
 static int
 replay_journal(journal *jnl)
 {
-    int i, ret, orig_checksum, checksum, max_bsize;
+    int i, orig_checksum, checksum, check_block_checksums=0, bad_blocks=0;
+    size_t ret;
+    size_t  max_bsize = 0;		/* protected by block_ptr */
     block_list_header *blhdr;
-    off_t offset;
+    off_t offset, txn_start_offset=0, blhdr_offset, orig_jnl_start;
     char *buff, *block_ptr=NULL;
     struct bucket *co_buf;
-    int num_buckets = STARTING_BUCKETS, num_full;
-
+    int num_buckets = STARTING_BUCKETS, num_full, check_past_jnl_end = 1, in_uncharted_territory=0;
+    uint32_t last_sequence_num = 0;
+    
     // wrap the start ptr if it points to the very end of the journal
     if (jnl->jhdr->start == jnl->jhdr->size) {
 		jnl->jhdr->start = jnl->jhdr->jhdr_size;
@@ -966,19 +1023,23 @@ replay_journal(journal *jnl)
 		return 0;
     }
 
+    orig_jnl_start = jnl->jhdr->start;
+
     // allocate memory for the header_block.  we'll read each blhdr into this
     if (kmem_alloc(kernel_map, (vm_offset_t *)&buff, jnl->jhdr->blhdr_size)) {
-		printf("jnl: replay_journal: no memory for block buffer! (%d bytes)\n",
-			   jnl->jhdr->blhdr_size);
+		printf("jnl: %s: replay_journal: no memory for block buffer! (%d bytes)\n",
+		    jnl->jdev_name, jnl->jhdr->blhdr_size);
 		return -1;
     }
 
     // allocate memory for the coalesce buffer
     if ((MALLOC(co_buf, struct bucket *, num_buckets*sizeof(struct bucket), M_TEMP, M_WAITOK)) == NULL) {
-        printf("jnl: replay_journal: no memory for coalesce buffer!\n");
+        printf("jnl: %s: replay_journal: no memory for coalesce buffer!\n", jnl->jdev_name);
 	return -1;
     }
 
+  restart_replay:
+
     // initialize entries
     for(i=0; i < num_buckets; i++) {
         co_buf[i].block_num = -1;
@@ -986,15 +1047,16 @@ replay_journal(journal *jnl)
     num_full = 0; // empty at first
 
 
-    printf("jnl: replay_journal: from: %lld to: %lld (joffset 0x%llx)\n",
-		   jnl->jhdr->start, jnl->jhdr->end, jnl->jdev_offset);
+    printf("jnl: %s: replay_journal: from: %lld to: %lld (joffset 0x%llx)\n",
+	jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end, jnl->jdev_offset);
 
-    while(jnl->jhdr->start != jnl->jhdr->end) {
-		offset = jnl->jhdr->start;
+    while(check_past_jnl_end || jnl->jhdr->start != jnl->jhdr->end) {
+		offset = blhdr_offset = jnl->jhdr->start;
 		ret = read_journal_data(jnl, &offset, buff, jnl->jhdr->blhdr_size);
-		if (ret != jnl->jhdr->blhdr_size) {
-			printf("jnl: replay_journal: Could not read block list header block @ 0x%llx!\n", offset);
-			goto bad_replay;
+		if (ret != (size_t)jnl->jhdr->blhdr_size) {
+		    printf("jnl: %s: replay_journal: Could not read block list header block @ 0x%llx!\n", jnl->jdev_name, offset);
+		    bad_blocks = 1;
+		    goto bad_txn_handling;
 		}
 
 		blhdr = (block_list_header *)buff;
@@ -1010,27 +1072,96 @@ replay_journal(journal *jnl)
 		} else {
 			checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE);
 		}
+
+
+		//
+		// XXXdbg - if these checks fail, we should replay as much
+		//          we can in the hopes that it will still leave the
+		//          drive in a better state than if we didn't replay
+		//          anything
+		//
 		if (checksum != orig_checksum) {
-		    printf("jnl: replay_journal: bad block list header @ 0x%llx (checksum 0x%x != 0x%x)\n",
-			   offset, orig_checksum, checksum);
-		    goto bad_replay;
+		    if (check_past_jnl_end && in_uncharted_territory) {
+
+			if (blhdr_offset != jnl->jhdr->end) {
+			    printf("jnl: %s: Extra txn replay stopped @ %lld / 0x%llx\n", jnl->jdev_name, blhdr_offset, blhdr_offset);
+			}
+
+			check_past_jnl_end = 0;
+			jnl->jhdr->end = blhdr_offset;
+			continue;
+		    }
+
+		    printf("jnl: %s: replay_journal: bad block list header @ 0x%llx (checksum 0x%x != 0x%x)\n",
+			jnl->jdev_name, blhdr_offset, orig_checksum, checksum);
+
+		    if (blhdr_offset == orig_jnl_start) {
+			// if there's nothing in the journal at all, just bail out altogether.
+			goto bad_replay;
+		    }
+
+		    bad_blocks = 1;
+		    goto bad_txn_handling;
+		}
+
+		if (blhdr->binfo[0].b.sequence_num < last_sequence_num) {
+		    txn_start_offset = jnl->jhdr->end = blhdr_offset;
+
+		    if (check_past_jnl_end) {
+			check_past_jnl_end = 0;
+			printf("jnl: %s: 2: extra replay stopped @ %lld / 0x%llx (seq %d < %d)\n",
+			    jnl->jdev_name, blhdr_offset, blhdr_offset, blhdr->binfo[0].b.sequence_num, last_sequence_num);
+			continue;
+		    }
+
+		    printf("jnl: %s: txn sequence numbers out of order in txn @ %lld / %llx! (%d < %d)\n",
+			jnl->jdev_name, blhdr_offset, blhdr_offset, blhdr->binfo[0].b.sequence_num, last_sequence_num);
+		    bad_blocks = 1;
+		    goto bad_txn_handling;
 		}
+		last_sequence_num = blhdr->binfo[0].b.sequence_num;
+
+		if (blhdr_offset >= jnl->jhdr->end && jnl->jhdr->start <= jnl->jhdr->end) {
+		    printf("jnl: %s: examining extra transactions starting @ %lld / 0x%llx\n", jnl->jdev_name, blhdr_offset, blhdr_offset);
+		}
+
 		if (   blhdr->max_blocks <= 0 || blhdr->max_blocks > 2048
 			   || blhdr->num_blocks <= 0 || blhdr->num_blocks > blhdr->max_blocks) {
-		    printf("jnl: replay_journal: bad looking journal entry: max: %d num: %d\n",
-			   blhdr->max_blocks, blhdr->num_blocks);
-		    goto bad_replay;
+		    printf("jnl: %s: replay_journal: bad looking journal entry: max: %d num: %d\n",
+			jnl->jdev_name, blhdr->max_blocks, blhdr->num_blocks);
+		    bad_blocks = 1;
+		    goto bad_txn_handling;
 		}
 	
+		max_bsize = 0;
 		for(i=1; i < blhdr->num_blocks; i++) {
 			if (blhdr->binfo[i].bnum < 0 && blhdr->binfo[i].bnum != (off_t)-1) {
-				printf("jnl: replay_journal: bogus block number 0x%llx\n", blhdr->binfo[i].bnum);
-				goto bad_replay;
+			    printf("jnl: %s: replay_journal: bogus block number 0x%llx\n", jnl->jdev_name, blhdr->binfo[i].bnum);
+			    bad_blocks = 1;
+			    goto bad_txn_handling;
+			}
+			
+			if (blhdr->binfo[i].bsize > max_bsize) {
+			    max_bsize = blhdr->binfo[i].bsize;
 			}
 		}
 
+		if (blhdr->flags & BLHDR_CHECK_CHECKSUMS) {
+		    check_block_checksums = 1;
+		    if (kmem_alloc(kernel_map, (vm_offset_t *)&block_ptr, max_bsize)) {
+			goto bad_replay;
+		    }
+		} else {
+		    block_ptr = NULL;
+		}
+
+		if (blhdr->flags & BLHDR_FIRST_HEADER) {
+		    txn_start_offset = blhdr_offset;
+		}
+
 		//printf("jnl: replay_journal: adding %d blocks in journal entry @ 0x%llx to co_buf\n", 
 		//       blhdr->num_blocks-1, jnl->jhdr->start);
+		bad_blocks = 0;
 		for(i=1; i < blhdr->num_blocks; i++) {
 			int size, ret_val;
 			off_t number;
@@ -1042,12 +1173,44 @@ replay_journal(journal *jnl)
 			if (number == (off_t)-1) {
 			    //printf("jnl: replay_journal: skipping killed fs block (index %d)\n", i);
 			} else {
+
+			    if (check_block_checksums) {
+				int32_t disk_cksum;
+				off_t block_offset;
+
+				block_offset = offset;
+
+				// read the block so we can check the checksum
+				ret = read_journal_data(jnl, &block_offset, block_ptr, size);
+				if (ret != (size_t)size) {
+				    printf("jnl: %s: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", jnl->jdev_name, offset);
+				    bad_blocks = 1;
+				    goto bad_txn_handling;
+				}
+				
+				disk_cksum = calc_checksum(block_ptr, size);
+
+				// there is no need to swap the checksum from disk because
+				// it got swapped when the blhdr was read in.
+				if (blhdr->binfo[i].b.cksum != 0 && disk_cksum != blhdr->binfo[i].b.cksum) {
+				    printf("jnl: %s: txn starting at %lld (%lld) @ index %3d bnum %lld (%d) with disk cksum != blhdr cksum (0x%.8x 0x%.8x)\n",
+					jnl->jdev_name, txn_start_offset, blhdr_offset, i, number, size, disk_cksum, blhdr->binfo[i].b.cksum);
+				    printf("jnl: 0x%.8x 0x%.8x 0x%.8x 0x%.8x  0x%.8x 0x%.8x 0x%.8x 0x%.8x\n",
+					*(int *)&block_ptr[0*sizeof(int)], *(int *)&block_ptr[1*sizeof(int)], *(int *)&block_ptr[2*sizeof(int)], *(int *)&block_ptr[3*sizeof(int)],
+					*(int *)&block_ptr[4*sizeof(int)], *(int *)&block_ptr[5*sizeof(int)], *(int *)&block_ptr[6*sizeof(int)], *(int *)&block_ptr[7*sizeof(int)]);
+
+				    bad_blocks = 1;
+				    goto bad_txn_handling;
+				}
+			    }
+
+
 			    // add this bucket to co_buf, coalescing where possible
 			    // printf("jnl: replay_journal: adding block 0x%llx\n", number);
-			    ret_val = add_block(jnl, &co_buf, number, size, (size_t) offset, &num_buckets, &num_full);
+			    ret_val = add_block(jnl, &co_buf, number, size, (size_t) offset, blhdr->binfo[i].b.cksum, &num_buckets, &num_full);
 			    
 			    if (ret_val == -1) {
-				printf("jnl: replay_journal: trouble adding block to co_buf\n");
+				printf("jnl: %s: replay_journal: trouble adding block to co_buf\n", jnl->jdev_name);
 				goto bad_replay;
 			    } // else printf("jnl: replay_journal: added block 0x%llx at i=%d\n", number);
 			}
@@ -1064,14 +1227,41 @@ replay_journal(journal *jnl)
 			}
 		}
 
+		if (block_ptr) {
+		    kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize);
+		    block_ptr = NULL;
+		}
 		
+      bad_txn_handling:
+		if (bad_blocks) {
+		    if (txn_start_offset == 0) {
+			printf("jnl: %s: no known good txn start offset! aborting journal replay.\n", jnl->jdev_name);
+			goto bad_replay;
+		    }
+
+		    jnl->jhdr->start = orig_jnl_start;
+		    jnl->jhdr->end = txn_start_offset;
+		    check_past_jnl_end = 0;
+		    last_sequence_num = 0;
+		    printf("jnl: %s: restarting journal replay (%lld - %lld)!\n", jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end);
+		    goto restart_replay;
+		}
+
 		jnl->jhdr->start += blhdr->bytes_used;
 		if (jnl->jhdr->start >= jnl->jhdr->size) {
 			// wrap around and skip the journal header block
 			jnl->jhdr->start = (jnl->jhdr->start % jnl->jhdr->size) + jnl->jhdr->jhdr_size;
 		}
+
+		if (jnl->jhdr->start == jnl->jhdr->end) {
+		    in_uncharted_territory = 1;
+		}
     }
 
+    if (jnl->jhdr->start != jnl->jhdr->end) {
+	printf("jnl: %s: start %lld != end %lld.  resetting end.\n", jnl->jdev_name, jnl->jhdr->start, jnl->jhdr->end);
+	jnl->jhdr->end = jnl->jhdr->start;
+    }
 
     //printf("jnl: replay_journal: replaying %d blocks\n", num_full);
     
@@ -1115,7 +1305,7 @@ replay_journal(journal *jnl)
 	    // do journal read, and set the phys. block 
 	    ret = read_journal_data(jnl, &jnl_offset, block_ptr, size);
 	    if (ret != size) {
-		printf("jnl: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", offset);
+		printf("jnl: %s: replay_journal: Could not read journal entry data @ offset 0x%llx!\n", jnl->jdev_name, offset);
 		goto bad_replay;
 	    }
 	    	    
@@ -1130,10 +1320,14 @@ replay_journal(journal *jnl)
     if (write_journal_header(jnl) != 0) {
 	goto bad_replay;
     }
+
+    printf("jnl: %s: journal replay done.\n", jnl->jdev_name);
     
     // free block_ptr
-    kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize);
-    block_ptr = NULL;
+    if (block_ptr) {
+	kmem_free(kernel_map, (vm_offset_t)block_ptr, max_bsize);
+	block_ptr = NULL;
+    }
     
     // free the coalesce buffer
     FREE(co_buf, M_TEMP);
@@ -1223,6 +1417,76 @@ size_up_tbuffer(journal *jnl, int tbuffer_size, int phys_blksz)
 
 
 
+static void
+get_io_info(struct vnode *devvp, size_t phys_blksz, journal *jnl, struct vfs_context *context)
+{
+    off_t	readblockcnt;
+    off_t	writeblockcnt;
+    off_t	readmaxcnt;
+    off_t	writemaxcnt;
+    int32_t     features;
+
+    if (VNOP_IOCTL(devvp, DKIOCGETFEATURES, (caddr_t)&features, 0, context) == 0) {
+	if (features & DK_FEATURE_FORCE_UNIT_ACCESS) {
+	    const char *name = vnode_name(devvp);
+	    jnl->flags |= JOURNAL_DO_FUA_WRITES;
+	    printf("jnl: %s: enabling FUA writes (features 0x%x)\n", name ? name : "no-name-dev", features);
+	}
+    }
+
+    if (VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTREAD, (caddr_t)&readmaxcnt, 0, context)) {
+	readmaxcnt = 0;
+    } 
+
+    if (readmaxcnt == 0) {
+	if (VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t)&readblockcnt, 0, context)) {
+	    readmaxcnt = 128 * 1024;
+	} else {
+	    readmaxcnt = readblockcnt * phys_blksz;
+	}
+    }
+
+
+    if (VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t)&writemaxcnt, 0, context)) {
+	writemaxcnt = 0;
+    }
+
+    if (writemaxcnt == 0) {
+	if (VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t)&writeblockcnt, 0, context)) {
+	    writemaxcnt = 128 * 1024;
+	} else {
+	    writemaxcnt = writeblockcnt * phys_blksz;
+	}
+    }
+
+    jnl->max_read_size  = readmaxcnt;
+    jnl->max_write_size = writemaxcnt;
+
+    // just in case it's still zero...
+    if (jnl->max_read_size == 0) {
+	jnl->max_read_size = 128 * 1024;
+	jnl->max_write_size = 128 * 1024;
+    }
+}
+
+
+static const char *
+get_jdev_name(struct vnode *jvp)
+{
+    const char *jdev_name;
+    
+    jdev_name = vnode_name(jvp);
+    if (jdev_name == NULL) {
+	jdev_name = vfs_addname("unknown-dev", strlen("unknown-dev"), 0, 0);
+    } else {
+	// this just bumps the refcount on the name so we have our own copy
+	jdev_name = vfs_addname(jdev_name, strlen(jdev_name), 0, 0);
+    }
+
+    return jdev_name;
+}
+
+
 journal *
 journal_create(struct vnode *jvp,
 			   off_t         offset,
@@ -1235,29 +1499,33 @@ journal_create(struct vnode *jvp,
 			   void         *arg)
 {
     journal *jnl;
-    int      phys_blksz;
+    size_t      phys_blksz;
     struct vfs_context context;
+    const char *jdev_name;
 
-    context.vc_proc = current_proc();
+    context.vc_thread = current_thread();
     context.vc_ucred = FSCRED;
 
+    jdev_name = get_jdev_name(jvp);
+
     /* Get the real physical block size. */
     if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, &context)) {
-		return NULL;
+	return NULL;
     }
 
     if (phys_blksz > min_fs_blksz) {
-		printf("jnl: create: error: phys blksize %d bigger than min fs blksize %d\n",
-			   phys_blksz, min_fs_blksz);
+		printf("jnl: %s: create: error: phys blksize %lu bigger than min fs blksize %lu\n",
+		    jdev_name, phys_blksz, min_fs_blksz);
 		return NULL;
     }
 
     if ((journal_size % phys_blksz) != 0) {
-		printf("jnl: create: journal size 0x%llx is not an even multiple of block size 0x%x\n",
-			   journal_size, phys_blksz);
+		printf("jnl: %s: create: journal size 0x%llx is not an even multiple of block size 0x%lx\n",
+		    jdev_name, journal_size, phys_blksz);
 		return NULL;
     }
 
+
     MALLOC_ZONE(jnl, struct journal *, sizeof(struct journal), M_JNL_JNL, M_WAITOK);
     memset(jnl, 0, sizeof(*jnl));
 
@@ -1267,11 +1535,14 @@ journal_create(struct vnode *jvp,
     jnl->flush        = flush;
     jnl->flush_arg    = arg;
     jnl->flags        = (flags & JOURNAL_OPTION_FLAGS_MASK);
+    jnl->jdev_name    = jdev_name;
     lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr);
+
+    get_io_info(jvp, phys_blksz, jnl, &context);
 	
     if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl->header_buf, phys_blksz)) {
-		printf("jnl: create: could not allocate space for header buffer (%d bytes)\n", phys_blksz);
-		goto bad_kmem_alloc;
+	printf("jnl: %s: create: could not allocate space for header buffer (%lu bytes)\n", jdev_name, phys_blksz);
+	goto bad_kmem_alloc;
     }
 
     memset(jnl->header_buf, 0, phys_blksz);
@@ -1291,11 +1562,13 @@ journal_create(struct vnode *jvp,
     // jnl->jhdr->start = jnl->jhdr->size - (phys_blksz*3);
     // jnl->jhdr->end   = jnl->jhdr->size - (phys_blksz*3);
     
+    jnl->jhdr->sequence_num = random() & 0x00ffffff;
+
 	lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr);
 
     if (write_journal_header(jnl) != 0) {
-		printf("jnl: journal_create: failed to write journal header.\n");
-		goto bad_write;
+	printf("jnl: %s: journal_create: failed to write journal header.\n", jdev_name);
+	goto bad_write;
     }
 
     return jnl;
@@ -1304,8 +1577,11 @@ journal_create(struct vnode *jvp,
   bad_write:
     kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, phys_blksz);
   bad_kmem_alloc:
+    if (jdev_name) {
+	vfs_removename(jdev_name);
+    }
     jnl->jhdr = NULL;
-	FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
+    FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
     return NULL;
 }
 
@@ -1322,11 +1598,13 @@ journal_open(struct vnode *jvp,
 			 void         *arg)
 {
     journal *jnl;
-    int      orig_blksz=0, phys_blksz;
-	int      orig_checksum, checksum;
+    int      orig_blksz=0;
+    size_t   phys_blksz;
+    int      orig_checksum, checksum;
     struct vfs_context context;
+    const char *jdev_name = get_jdev_name(jvp);
 
-    context.vc_proc = current_proc();
+    context.vc_thread = current_thread();
     context.vc_ucred = FSCRED;
 
     /* Get the real physical block size. */
@@ -1335,14 +1613,14 @@ journal_open(struct vnode *jvp,
     }
 
     if (phys_blksz > min_fs_blksz) {
-		printf("jnl: create: error: phys blksize %d bigger than min fs blksize %d\n",
-			   phys_blksz, min_fs_blksz);
+		printf("jnl: %s: open: error: phys blksize %lu bigger than min fs blksize %lu\n",
+		    jdev_name, phys_blksz, min_fs_blksz);
 		return NULL;
     }
 
     if ((journal_size % phys_blksz) != 0) {
-		printf("jnl: open: journal size 0x%llx is not an even multiple of block size 0x%x\n",
-			   journal_size, phys_blksz);
+		printf("jnl: %s: open: journal size 0x%llx is not an even multiple of block size 0x%lx\n",
+		    jdev_name, journal_size, phys_blksz);
 		return NULL;
     }
 
@@ -1355,22 +1633,25 @@ journal_open(struct vnode *jvp,
     jnl->flush        = flush;
     jnl->flush_arg    = arg;
     jnl->flags        = (flags & JOURNAL_OPTION_FLAGS_MASK);
+    jnl->jdev_name    = jdev_name;
     lck_mtx_init(&jnl->old_start_lock, jnl_mutex_group, jnl_lock_attr);
 
+    get_io_info(jvp, phys_blksz, jnl, &context);
+
     if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl->header_buf, phys_blksz)) {
-		printf("jnl: create: could not allocate space for header buffer (%d bytes)\n", phys_blksz);
-		goto bad_kmem_alloc;
+	printf("jnl: %s: create: could not allocate space for header buffer (%lu bytes)\n", jdev_name, phys_blksz);
+	goto bad_kmem_alloc;
     }
 
     jnl->jhdr = (journal_header *)jnl->header_buf;
-    memset(jnl->jhdr, 0, sizeof(journal_header)+4);
+    memset(jnl->jhdr, 0, sizeof(journal_header));
 
     // we have to set this up here so that do_journal_io() will work
     jnl->jhdr->jhdr_size = phys_blksz;
 
     if (read_journal_header(jnl, jnl->jhdr, phys_blksz) != phys_blksz) {
-		printf("jnl: open: could not read %d bytes for the journal header.\n",
-			   phys_blksz);
+		printf("jnl: %s: open: could not read %lu bytes for the journal header.\n",
+		    jdev_name, phys_blksz);
 		goto bad_journal;
     }
 
@@ -1380,16 +1661,16 @@ journal_open(struct vnode *jvp,
 	if (jnl->jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) {
 		// do this before the swap since it's done byte-at-a-time
 		orig_checksum = SWAP32(orig_checksum);
-		checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header));
+		checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
 		swap_journal_header(jnl);
 		jnl->flags |= JOURNAL_NEED_SWAP;
 	} else {
-		checksum = calc_checksum((char *)jnl->jhdr, sizeof(struct journal_header));
+		checksum = calc_checksum((char *)jnl->jhdr, JOURNAL_HEADER_CKSUM_SIZE);
 	}
 
     if (jnl->jhdr->magic != JOURNAL_HEADER_MAGIC && jnl->jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) {
-		printf("jnl: open: journal magic is bad (0x%x != 0x%x)\n",
-			   jnl->jhdr->magic, JOURNAL_HEADER_MAGIC);
+		printf("jnl: %s: open: journal magic is bad (0x%x != 0x%x)\n",
+		    jnl->jdev_name, jnl->jhdr->magic, JOURNAL_HEADER_MAGIC);
 		goto bad_journal;
     }
 
@@ -1397,8 +1678,8 @@ journal_open(struct vnode *jvp,
 	if (jnl->jhdr->magic == JOURNAL_HEADER_MAGIC) {
 
 		if (orig_checksum != checksum) {
-			printf("jnl: open: journal checksum is bad (0x%x != 0x%x)\n",
-				   orig_checksum, checksum);
+			printf("jnl: %s: open: journal checksum is bad (0x%x != 0x%x)\n",
+			    jdev_name, orig_checksum, checksum);
 				   
 			//goto bad_journal;
 		}
@@ -1409,15 +1690,15 @@ journal_open(struct vnode *jvp,
 		jnl->jhdr->magic = JOURNAL_HEADER_MAGIC;
 	}
 
-    if (phys_blksz != jnl->jhdr->jhdr_size && jnl->jhdr->jhdr_size != 0) {
-		printf("jnl: open: phys_blksz %d does not match journal header size %d\n",
-			   phys_blksz, jnl->jhdr->jhdr_size);
+    if (phys_blksz != (size_t)jnl->jhdr->jhdr_size && jnl->jhdr->jhdr_size != 0) {
+		printf("jnl: %s: open: phys_blksz %lu does not match journal header size %d\n",
+		    jdev_name, phys_blksz, jnl->jhdr->jhdr_size);
 
 		orig_blksz = phys_blksz;
 		phys_blksz = jnl->jhdr->jhdr_size;
 		if (VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&phys_blksz, FWRITE, &context)) {
-			printf("jnl: could not set block size to %d bytes.\n", phys_blksz);
-			goto bad_journal;
+		    printf("jnl: %s: could not set block size to %lu bytes.\n", jdev_name, phys_blksz);
+		    goto bad_journal;
 		}
 //		goto bad_journal;
     }
@@ -1425,22 +1706,22 @@ journal_open(struct vnode *jvp,
     if (   jnl->jhdr->start <= 0
 		   || jnl->jhdr->start > jnl->jhdr->size
 		   || jnl->jhdr->start > 1024*1024*1024) {
-		printf("jnl: open: jhdr start looks bad (0x%llx max size 0x%llx)\n",
-			   jnl->jhdr->start, jnl->jhdr->size);
+		printf("jnl: %s: open: jhdr start looks bad (0x%llx max size 0x%llx)\n",
+		    jdev_name, jnl->jhdr->start, jnl->jhdr->size);
 		goto bad_journal;
     }
 
     if (   jnl->jhdr->end <= 0
 		   || jnl->jhdr->end > jnl->jhdr->size
 		   || jnl->jhdr->end > 1024*1024*1024) {
-		printf("jnl: open: jhdr end looks bad (0x%llx max size 0x%llx)\n",
-			   jnl->jhdr->end, jnl->jhdr->size);
+		printf("jnl: %s: open: jhdr end looks bad (0x%llx max size 0x%llx)\n",
+		    jdev_name, jnl->jhdr->end, jnl->jhdr->size);
 		goto bad_journal;
     }
 
     if (jnl->jhdr->size > 1024*1024*1024) {
-		printf("jnl: open: jhdr size looks bad (0x%llx)\n", jnl->jhdr->size);
-		goto bad_journal;
+	printf("jnl: %s: open: jhdr size looks bad (0x%llx)\n", jdev_name, jnl->jhdr->size);
+	goto bad_journal;
     }
 
 // XXXdbg - can't do these checks because hfs writes all kinds of
@@ -1450,57 +1731,60 @@ journal_open(struct vnode *jvp,
 //          do more relaxed checking...
 // XXXdbg    if ((jnl->jhdr->start % jnl->jhdr->jhdr_size) != 0) {
     if ((jnl->jhdr->start % 512) != 0) {
-		printf("jnl: open: journal start (0x%llx) not a multiple of 512?\n",
-			   jnl->jhdr->start);
+		printf("jnl: %s: open: journal start (0x%llx) not a multiple of 512?\n",
+		    jdev_name, jnl->jhdr->start);
 		goto bad_journal;
     }
 
 //XXXdbg    if ((jnl->jhdr->end % jnl->jhdr->jhdr_size) != 0) {
     if ((jnl->jhdr->end % 512) != 0) {
-		printf("jnl: open: journal end (0x%llx) not a multiple of block size (0x%x)?\n",
-			   jnl->jhdr->end, jnl->jhdr->jhdr_size);
+		printf("jnl: %s: open: journal end (0x%llx) not a multiple of block size (0x%x)?\n",
+		    jdev_name, jnl->jhdr->end, jnl->jhdr->jhdr_size);
 		goto bad_journal;
     }
 
     // take care of replaying the journal if necessary
-	if (flags & JOURNAL_RESET) {
-		printf("jnl: journal start/end pointers reset! (jnl 0x%x; s 0x%llx e 0x%llx)\n",
-			   jnl, jnl->jhdr->start, jnl->jhdr->end);
-		jnl->jhdr->start = jnl->jhdr->end;
-	} else if (replay_journal(jnl) != 0) {
-		printf("jnl: journal_open: Error replaying the journal!\n");
-		goto bad_journal;
-    }
-
-	if (orig_blksz != 0) {
-		VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context);
-		phys_blksz = orig_blksz;
-		if (orig_blksz < jnl->jhdr->jhdr_size) {
-			printf("jnl: open: jhdr_size is %d but orig phys blk size is %d.  switching.\n",
-				   jnl->jhdr->jhdr_size, orig_blksz);
+    if (flags & JOURNAL_RESET) {
+	printf("jnl: %s: journal start/end pointers reset! (jnl %p; s 0x%llx e 0x%llx)\n",
+	    jdev_name, jnl, jnl->jhdr->start, jnl->jhdr->end);
+	jnl->jhdr->start = jnl->jhdr->end;
+    } else if (replay_journal(jnl) != 0) {
+	printf("jnl: %s: journal_open: Error replaying the journal!\n", jdev_name);
+	goto bad_journal;
+    }
+
+    if (orig_blksz != 0) {
+	VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context);
+	phys_blksz = orig_blksz;
+	if (orig_blksz < jnl->jhdr->jhdr_size) {
+	    printf("jnl: %s: open: jhdr_size is %d but orig phys blk size is %d.  switching.\n",
+		jdev_name, jnl->jhdr->jhdr_size, orig_blksz);
 				   
-			jnl->jhdr->jhdr_size = orig_blksz;
-		}
+	    jnl->jhdr->jhdr_size = orig_blksz;
 	}
+    }
 
-	// make sure this is in sync!
-	jnl->active_start = jnl->jhdr->start;
+    // make sure this is in sync!
+    jnl->active_start = jnl->jhdr->start;
 
     // set this now, after we've replayed the journal
     size_up_tbuffer(jnl, tbuffer_size, phys_blksz);
 
-	lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr);
+    lck_mtx_init(&jnl->jlock, jnl_mutex_group, jnl_lock_attr);
 
     return jnl;
 
   bad_journal:
-	if (orig_blksz != 0) {
-		phys_blksz = orig_blksz;
-		VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context);
-	}
+    if (orig_blksz != 0) {
+	phys_blksz = orig_blksz;
+	VNOP_IOCTL(jvp, DKIOCSETBLOCKSIZE, (caddr_t)&orig_blksz, FWRITE, &context);
+    }
     kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, phys_blksz);
   bad_kmem_alloc:
-	FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
+    if (jdev_name) {
+	vfs_removename(jdev_name);
+    }
+    FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
     return NULL;    
 }
 
@@ -1516,37 +1800,40 @@ journal_is_clean(struct vnode *jvp,
     int     phys_blksz, ret;
     int     orig_checksum, checksum;
     struct vfs_context context;
+    const char *jdev_name = get_jdev_name(jvp);
 
-    context.vc_proc = current_proc();
+    context.vc_thread = current_thread();
     context.vc_ucred = FSCRED;
 
     /* Get the real physical block size. */
     if (VNOP_IOCTL(jvp, DKIOCGETBLOCKSIZE, (caddr_t)&phys_blksz, 0, &context)) {
-	printf("jnl: is_clean: failed to get device block size.\n");
+	printf("jnl: %s: is_clean: failed to get device block size.\n", jdev_name);
 	return EINVAL;
     }
 
-    if (phys_blksz > min_fs_block_size) {
-	printf("jnl: is_clean: error: phys blksize %d bigger than min fs blksize %d\n",
-	       phys_blksz, min_fs_block_size);
+    if (phys_blksz > (int)min_fs_block_size) {
+	printf("jnl: %s: is_clean: error: phys blksize %d bigger than min fs blksize %lu\n",
+	    jdev_name, phys_blksz, min_fs_block_size);
 	return EINVAL;
     }
 
     if ((journal_size % phys_blksz) != 0) {
-	printf("jnl: is_clean: journal size 0x%llx is not an even multiple of block size 0x%x\n",
-	       journal_size, phys_blksz);
+	printf("jnl: %s: is_clean: journal size 0x%llx is not an even multiple of block size 0x%x\n",
+	    jdev_name, journal_size, phys_blksz);
 	return EINVAL;
     }
 
     memset(&jnl, 0, sizeof(jnl));
 
     if (kmem_alloc(kernel_map, (vm_offset_t *)&jnl.header_buf, phys_blksz)) {
-	printf("jnl: is_clean: could not allocate space for header buffer (%d bytes)\n", phys_blksz);
+	printf("jnl: %s: is_clean: could not allocate space for header buffer (%d bytes)\n", jdev_name, phys_blksz);
 	return ENOMEM;
     }
 
+    get_io_info(jvp, phys_blksz, &jnl, &context);
+    
     jnl.jhdr = (journal_header *)jnl.header_buf;
-    memset(jnl.jhdr, 0, sizeof(journal_header)+4);
+    memset(jnl.jhdr, 0, sizeof(journal_header));
 
     jnl.jdev        = jvp;
     jnl.jdev_offset = offset;
@@ -1555,9 +1842,9 @@ journal_is_clean(struct vnode *jvp,
     // we have to set this up here so that do_journal_io() will work
     jnl.jhdr->jhdr_size = phys_blksz;
 
-    if (read_journal_header(&jnl, jnl.jhdr, phys_blksz) != phys_blksz) {
-	printf("jnl: is_clean: could not read %d bytes for the journal header.\n",
-	       phys_blksz);
+    if (read_journal_header(&jnl, jnl.jhdr, phys_blksz) != (unsigned)phys_blksz) {
+	printf("jnl: %s: is_clean: could not read %d bytes for the journal header.\n",
+	    jdev_name, phys_blksz);
 	ret = EINVAL;
 	goto get_out;
     }
@@ -1568,22 +1855,22 @@ journal_is_clean(struct vnode *jvp,
     if (jnl.jhdr->magic == SWAP32(JOURNAL_HEADER_MAGIC)) {
 	// do this before the swap since it's done byte-at-a-time
 	orig_checksum = SWAP32(orig_checksum);
-	checksum = calc_checksum((char *)jnl.jhdr, sizeof(struct journal_header));
+	checksum = calc_checksum((char *)jnl.jhdr, JOURNAL_HEADER_CKSUM_SIZE);
 	swap_journal_header(&jnl);
 	jnl.flags |= JOURNAL_NEED_SWAP;
     } else {
-	checksum = calc_checksum((char *)jnl.jhdr, sizeof(struct journal_header));
+	checksum = calc_checksum((char *)jnl.jhdr, JOURNAL_HEADER_CKSUM_SIZE);
     }
 
     if (jnl.jhdr->magic != JOURNAL_HEADER_MAGIC && jnl.jhdr->magic != OLD_JOURNAL_HEADER_MAGIC) {
-	printf("jnl: is_clean: journal magic is bad (0x%x != 0x%x)\n",
-	       jnl.jhdr->magic, JOURNAL_HEADER_MAGIC);
+	printf("jnl: %s: is_clean: journal magic is bad (0x%x != 0x%x)\n",
+	    jdev_name, jnl.jhdr->magic, JOURNAL_HEADER_MAGIC);
 	ret = EINVAL;
 	goto get_out;
     }
 
     if (orig_checksum != checksum) {
-	printf("jnl: is_clean: journal checksum is bad (0x%x != 0x%x)\n", orig_checksum, checksum);
+	printf("jnl: %s: is_clean: journal checksum is bad (0x%x != 0x%x)\n", jdev_name, orig_checksum, checksum);
 	ret = EINVAL;
 	goto get_out;
     }
@@ -1600,6 +1887,9 @@ journal_is_clean(struct vnode *jvp,
 
   get_out:
     kmem_free(kernel_map, (vm_offset_t)jnl.header_buf, phys_blksz);
+    if (jdev_name) {
+	vfs_removename(jdev_name);
+    }
     
     return ret;    
 
@@ -1607,7 +1897,6 @@ journal_is_clean(struct vnode *jvp,
 }
 
 
-
 void
 journal_close(journal *jnl)
 {
@@ -1639,24 +1928,24 @@ journal_close(journal *jnl)
 			transaction *tr = jnl->cur_tr;
 
 			jnl->cur_tr = NULL;
-			end_transaction(tr, 1);   // force it to get flushed
+			end_transaction(tr, 1, NULL, NULL);   // force it to get flushed
 		}
     
 		//start = &jnl->jhdr->start;
 		start = &jnl->active_start;
 		end   = &jnl->jhdr->end;
     
-		while (*start != *end && counter++ < 500) {
-			printf("jnl: close: flushing the buffer cache (start 0x%llx end 0x%llx)\n", *start, *end);
+		while (*start != *end && counter++ < 5000) {
+			//printf("jnl: close: flushing the buffer cache (start 0x%llx end 0x%llx)\n", *start, *end);
 			if (jnl->flush) {
 				jnl->flush(jnl->flush_arg);
 			}
-			tsleep((caddr_t)jnl, PRIBIO, "jnl_close", 1);
+			tsleep((caddr_t)jnl, PRIBIO, "jnl_close", 2);
 		}
 
 		if (*start != *end) {
-			printf("jnl: close: buffer flushing didn't seem to flush out all the transactions! (0x%llx - 0x%llx)\n",
-				   *start, *end);
+			printf("jnl: %s: close: buffer flushing didn't seem to flush out all the transactions! (0x%llx - 0x%llx)\n",
+			    jnl->jdev_name, *start, *end);
 		}
 
 		// make sure this is in sync when we close the journal
@@ -1667,7 +1956,7 @@ journal_close(journal *jnl)
     } else {
 		// if we're here the journal isn't valid any more.
 		// so make sure we don't leave any locked blocks lying around
-		printf("jnl: close: journal 0x%x, is invalid.  aborting outstanding transactions\n", jnl);
+		printf("jnl: %s: close: journal %p, is invalid.  aborting outstanding transactions\n", jnl->jdev_name, jnl);
 		if (jnl->active_tr || jnl->cur_tr) {
 			transaction *tr;
 			if (jnl->active_tr) {
@@ -1680,7 +1969,7 @@ journal_close(journal *jnl)
 
 			abort_transaction(jnl, tr);
 			if (jnl->active_tr || jnl->cur_tr) {
-				panic("jnl: close: jnl @ 0x%x had both an active and cur tr\n", jnl);
+			    panic("jnl: %s: close: jnl @ %p had both an active and cur tr\n", jnl->jdev_name, jnl);
 			}
 		}
     }
@@ -1690,7 +1979,11 @@ journal_close(journal *jnl)
     kmem_free(kernel_map, (vm_offset_t)jnl->header_buf, jnl->jhdr->jhdr_size);
     jnl->jhdr = (void *)0xbeefbabe;
 
-	FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
+    if (jnl->jdev_name) {
+	vfs_removename(jnl->jdev_name);
+    }
+
+    FREE_ZONE(jnl, sizeof(struct journal), M_JNL_JNL);
 }
 
 static void
@@ -1698,7 +1991,7 @@ dump_journal(journal *jnl)
 {
     transaction *ctr;
 
-    printf("journal:");
+    printf("journal for dev %s:", jnl->jdev_name);
     printf("  jdev_offset %.8llx\n", jnl->jdev_offset);
     printf("  magic: 0x%.8x\n", jnl->jhdr->magic);
     printf("  start: 0x%.8llx\n", jnl->jhdr->start);
@@ -1719,18 +2012,18 @@ dump_journal(journal *jnl)
 static off_t
 free_space(journal *jnl)
 {
-    off_t free_space;
+    off_t free_space_offset;
 	
     if (jnl->jhdr->start < jnl->jhdr->end) {
-		free_space = jnl->jhdr->size - (jnl->jhdr->end - jnl->jhdr->start) - jnl->jhdr->jhdr_size;
+		free_space_offset = jnl->jhdr->size - (jnl->jhdr->end - jnl->jhdr->start) - jnl->jhdr->jhdr_size;
     } else if (jnl->jhdr->start > jnl->jhdr->end) {
-		free_space = jnl->jhdr->start - jnl->jhdr->end;
+		free_space_offset = jnl->jhdr->start - jnl->jhdr->end;
     } else {
 		// journal is completely empty
-		free_space = jnl->jhdr->size - jnl->jhdr->jhdr_size;
+		free_space_offset = jnl->jhdr->size - jnl->jhdr->jhdr_size;
     }
 
-    return free_space;
+    return free_space_offset;
 }
 
 
@@ -1741,7 +2034,8 @@ free_space(journal *jnl)
 static int
 check_free_space(journal *jnl, int desired_size)
 {
-    int    i, counter=0;
+    size_t i;
+    int    counter=0;
 
     //printf("jnl: check free space (desired 0x%x, avail 0x%Lx)\n",
 //	   desired_size, free_space(jnl));
@@ -1752,19 +2046,18 @@ check_free_space(journal *jnl, int desired_size)
 		if (counter++ == 5000) {
 			dump_journal(jnl);
 			panic("jnl: check_free_space: buffer flushing isn't working "
-				  "(jnl @ 0x%x s %lld e %lld f %lld [active start %lld]).\n", jnl,
+				  "(jnl @ %p s %lld e %lld f %lld [active start %lld]).\n", jnl,
 				  jnl->jhdr->start, jnl->jhdr->end, free_space(jnl), jnl->active_start);
 		}
 		if (counter > 7500) {
-			printf("jnl: check_free_space: giving up waiting for free space.\n");
-			return ENOSPC;
+		    printf("jnl: %s: check_free_space: giving up waiting for free space.\n", jnl->jdev_name);
+		    return ENOSPC;
 		}
 
 		// make sure there's space in the journal to hold this transaction
-		if (free_space(jnl) > desired_size) {
+		if (free_space(jnl) > desired_size && jnl->old_start[0] == 0) {
 			break;
 		}
-
 		//
 		// here's where we lazily bump up jnl->jhdr->start.  we'll consume
 		// entries until there is enough space for the next transaction.
@@ -1772,12 +2065,12 @@ check_free_space(journal *jnl, int desired_size)
 		old_start_empty = 1;
 		lock_oldstart(jnl);
 		for(i=0; i < sizeof(jnl->old_start)/sizeof(jnl->old_start[0]); i++) {
-			int   counter;
+			int   lcl_counter;
 
-			counter = 0;
+			lcl_counter = 0;
 			while (jnl->old_start[i] & 0x8000000000000000LL) {
-				if (counter++ > 100) {
-					panic("jnl: check_free_space: tr starting @ 0x%llx not flushing (jnl 0x%x).\n",
+				if (lcl_counter++ > 100) {
+					panic("jnl: check_free_space: tr starting @ 0x%llx not flushing (jnl %p).\n",
 						  jnl->old_start[i], jnl);
 				}
 				
@@ -1839,11 +2132,50 @@ check_free_space(journal *jnl, int desired_size)
     return 0;
 }
 
+/*
+ * Allocate a new active transaction.
+ */
+static errno_t
+journal_allocate_transaction(journal *jnl)
+{
+	transaction *tr;
+	
+	MALLOC_ZONE(tr, transaction *, sizeof(transaction), M_JNL_TR, M_WAITOK);
+    memset(tr, 0, sizeof(transaction));
+
+    tr->tbuffer_size = jnl->tbuffer_size;
+
+    if (kmem_alloc(kernel_map, (vm_offset_t *)&tr->tbuffer, tr->tbuffer_size)) {
+		FREE_ZONE(tr, sizeof(transaction), M_JNL_TR);
+		jnl->active_tr = NULL;
+		return ENOMEM;
+    }
+
+    // journal replay code checksum check depends on this.
+    memset(tr->tbuffer, 0, BLHDR_CHECKSUM_SIZE);
+    // Fill up the rest of the block with unimportant bytes (0x5a 'Z' chosen for visibility)
+    memset(tr->tbuffer + BLHDR_CHECKSUM_SIZE, 0x5a, jnl->jhdr->blhdr_size - BLHDR_CHECKSUM_SIZE);
+
+    tr->blhdr = (block_list_header *)tr->tbuffer;
+    tr->blhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
+    tr->blhdr->num_blocks = 1;      // accounts for this header block
+    tr->blhdr->bytes_used = jnl->jhdr->blhdr_size;
+    tr->blhdr->flags = BLHDR_CHECK_CHECKSUMS | BLHDR_FIRST_HEADER;
+
+    tr->sequence_num = ++jnl->jhdr->sequence_num;
+    tr->num_blhdrs  = 1;
+    tr->total_bytes = jnl->jhdr->blhdr_size;
+    tr->jnl         = jnl;
+
+	jnl->active_tr  = tr;
+	
+	return 0;
+}
+
 int
 journal_start_transaction(journal *jnl)
 {
     int ret;
-    transaction *tr;
 
     CHECK_JOURNAL(jnl);
     
@@ -1853,7 +2185,7 @@ journal_start_transaction(journal *jnl)
 
     if (jnl->owner == current_thread()) {
 		if (jnl->active_tr == NULL) {
-			panic("jnl: start_tr: active_tr is NULL (jnl @ 0x%x, owner 0x%x, current_thread 0x%x\n",
+			panic("jnl: start_tr: active_tr is NULL (jnl @ %p, owner %p, current_thread %p\n",
 				  jnl, jnl->owner, current_thread());
 		}
 		jnl->nested_count++;
@@ -1863,7 +2195,7 @@ journal_start_transaction(journal *jnl)
     lock_journal(jnl);
 
     if (jnl->owner != NULL || jnl->nested_count != 0 || jnl->active_tr != NULL) {
-		panic("jnl: start_tr: owner 0x%x, nested count 0x%x, active_tr 0x%x jnl @ 0x%x\n",
+		panic("jnl: start_tr: owner %p, nested count %d, active_tr %p jnl @ %p\n",
 			  jnl->owner, jnl->nested_count, jnl->active_tr, jnl);
     }
 
@@ -1873,10 +2205,14 @@ journal_start_transaction(journal *jnl)
     free_old_stuff(jnl);
 
     // make sure there's room in the journal
-    if (check_free_space(jnl, jnl->tbuffer_size) != 0) {
-		printf("jnl: start transaction failed: no space\n");
+    if (free_space(jnl) < jnl->tbuffer_size) {
+	// this is the call that really waits for space to free up
+	// as well as updating jnl->jhdr->start
+	if (check_free_space(jnl, jnl->tbuffer_size) != 0) {
+		printf("jnl: %s: start transaction failed: no space\n", jnl->jdev_name);
 		ret = ENOSPC;
 		goto bad_start;
+	}
     }
 
     // if there's a buffered transaction, use it.
@@ -1887,35 +2223,12 @@ journal_start_transaction(journal *jnl)
 		return 0;
     }
 
-	MALLOC_ZONE(tr, transaction *, sizeof(transaction), M_JNL_TR, M_WAITOK);
-    memset(tr, 0, sizeof(transaction));
-
-    tr->tbuffer_size = jnl->tbuffer_size;
-
-    if (kmem_alloc(kernel_map, (vm_offset_t *)&tr->tbuffer, tr->tbuffer_size)) {
-		FREE_ZONE(tr, sizeof(transaction), M_JNL_TR);
-		printf("jnl: start transaction failed: no tbuffer mem\n");
-		ret = ENOMEM;
+	ret = journal_allocate_transaction(jnl);
+	if (ret) {
 		goto bad_start;
-    }
-
-    // journal replay code checksum check depends on this.
-    memset(tr->tbuffer, 0, BLHDR_CHECKSUM_SIZE);
-    // Fill up the rest of the block with unimportant bytes (0x5a 'Z' chosen for visibility)
-    memset(tr->tbuffer + BLHDR_CHECKSUM_SIZE, 0x5a, jnl->jhdr->blhdr_size - BLHDR_CHECKSUM_SIZE);
-
-    tr->blhdr = (block_list_header *)tr->tbuffer;
-    tr->blhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
-    tr->blhdr->num_blocks = 1;      // accounts for this header block
-    tr->blhdr->bytes_used = jnl->jhdr->blhdr_size;
-
-    tr->num_blhdrs  = 1;
-    tr->total_bytes = jnl->jhdr->blhdr_size;
-    tr->jnl         = jnl;
-
-    jnl->active_tr    = tr;
+	}
 
-    // printf("jnl: start_tr: owner 0x%x new tr @ 0x%x\n", jnl->owner, tr);
+    // printf("jnl: start_tr: owner 0x%x new tr @ 0x%x\n", jnl->owner, jnl->active_tr);
 
     return 0;
 
@@ -1941,14 +2254,14 @@ journal_modify_block_start(journal *jnl, struct buf *bp)
     // XXXdbg - for debugging I want this to be true.  later it may
     //          not be necessary.
     if ((buf_flags(bp) & B_META) == 0) {
-		panic("jnl: modify_block_start: bp @ 0x%x is not a meta-data block! (jnl 0x%x)\n", bp, jnl);
+		panic("jnl: modify_block_start: bp @ %p is not a meta-data block! (jnl %p)\n", bp, jnl);
     }
 
     tr = jnl->active_tr;
     CHECK_TRANSACTION(tr);
 
     if (jnl->owner != current_thread()) {
-		panic("jnl: modify_block_start: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n",
+		panic("jnl: modify_block_start: called w/out a transaction! jnl %p, owner %p, curact %p\n",
 			  jnl, jnl->owner, current_thread());
     }
 
@@ -1967,7 +2280,7 @@ journal_modify_block_start(journal *jnl, struct buf *bp)
 
     // make sure that this transaction isn't bigger than the whole journal
     if (tr->total_bytes+buf_size(bp) >= (jnl->jhdr->size - jnl->jhdr->jhdr_size)) {
-		panic("jnl: transaction too big (%d >= %lld bytes, bufsize %d, tr 0x%x bp 0x%x)\n",
+		panic("jnl: transaction too big (%d >= %lld bytes, bufsize %d, tr %p bp %p)\n",
 			  tr->total_bytes, (tr->jnl->jhdr->size - jnl->jhdr->jhdr_size), buf_size(bp), tr, bp);
 		return -1;
     }
@@ -1979,7 +2292,7 @@ journal_modify_block_start(journal *jnl, struct buf *bp)
     if ((buf_flags(bp) & (B_DELWRI | B_LOCKED)) == B_DELWRI) {
 
 		if (buf_flags(bp) & B_ASYNC) {
-			panic("modify_block_start: bp @ 0x% has async flag set!\n", bp);
+			panic("modify_block_start: bp @ %p has async flag set!\n", bp);
 		}
 
 		// this will cause it to not be buf_brelse()'d
@@ -1996,7 +2309,7 @@ journal_modify_block_abort(journal *jnl, struct buf *bp)
 {
     transaction *tr;
 	block_list_header *blhdr;
-	int i, j;
+	int i;
     
     CHECK_JOURNAL(jnl);
 
@@ -2019,7 +2332,7 @@ journal_modify_block_abort(journal *jnl, struct buf *bp)
     CHECK_TRANSACTION(tr);
     
     if (jnl->owner != current_thread()) {
-		panic("jnl: modify_block_abort: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n",
+		panic("jnl: modify_block_abort: called w/out a transaction! jnl %p, owner %p, curact %p\n",
 			  jnl, jnl->owner, current_thread());
     }
 
@@ -2030,9 +2343,9 @@ journal_modify_block_abort(journal *jnl, struct buf *bp)
     // first check if it's already part of this transaction
     for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) {
 		for(i=1; i < blhdr->num_blocks; i++) {
-			if (bp == blhdr->binfo[i].bp) {
+			if (bp == blhdr->binfo[i].b.bp) {
 				if (buf_size(bp) != blhdr->binfo[i].bsize) {
-					panic("jnl: bp @ 0x%x changed size on me! (%d vs. %d, jnl 0x%x)\n",
+					panic("jnl: bp @ %p changed size on me! (%d vs. %lu, jnl %p)\n",
 						  bp, buf_size(bp), blhdr->binfo[i].bsize, jnl);
 				}
 				break;
@@ -2061,9 +2374,10 @@ journal_modify_block_abort(journal *jnl, struct buf *bp)
 
 
 int
-journal_modify_block_end(journal *jnl, struct buf *bp)
+journal_modify_block_end(journal *jnl, struct buf *bp, void (*func)(struct buf *bp, void *arg), void *arg)
 {
-    int                i, j, tbuffer_offset;
+    int                i = 1;
+    int	               tbuffer_offset=0;
     char              *blkptr;
     block_list_header *blhdr, *prev=NULL;
     transaction       *tr;
@@ -2078,7 +2392,7 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
     CHECK_TRANSACTION(tr);
 
     if (jnl->owner != current_thread()) {
-		panic("jnl: modify_block_end: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n",
+		panic("jnl: modify_block_end: called w/out a transaction! jnl %p, owner %p, curact %p\n",
 			  jnl, jnl->owner, current_thread());
     }
 
@@ -2088,7 +2402,7 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
     //   bp, buf_vnode(bp), buf_lblkno(bp), buf_blkno(bp), buf_size(bp), tr->total_bytes);
 
     if ((buf_flags(bp) & B_LOCKED) == 0) {
-		panic("jnl: modify_block_end: bp 0x%x not locked! jnl @ 0x%x\n", bp, jnl);
+		panic("jnl: modify_block_end: bp %p not locked! jnl @ %p\n", bp, jnl);
     }
 	 
     // first check if it's already part of this transaction
@@ -2096,9 +2410,9 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
 		tbuffer_offset = jnl->jhdr->blhdr_size;
 
 		for(i=1; i < blhdr->num_blocks; i++) {
-			if (bp == blhdr->binfo[i].bp) {
+			if (bp == blhdr->binfo[i].b.bp) {
 				if (buf_size(bp) != blhdr->binfo[i].bsize) {
-					panic("jnl: bp @ 0x%x changed size on me! (%d vs. %d, jnl 0x%x)\n",
+					panic("jnl: bp @ %p changed size on me! (%d vs. %lu, jnl %p)\n",
 						  bp, buf_size(bp), blhdr->binfo[i].bsize, jnl);
 				}
 				break;
@@ -2114,13 +2428,13 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
     if (blhdr == NULL
 		&& prev
 		&& (prev->num_blocks+1) <= prev->max_blocks
-		&& (prev->bytes_used+buf_size(bp)) <= tr->tbuffer_size) {
+		&& (prev->bytes_used+buf_size(bp)) <= (uint32_t)tr->tbuffer_size) {
 		blhdr = prev;
     } else if (blhdr == NULL) {
 		block_list_header *nblhdr;
 
 		if (prev == NULL) {
-			panic("jnl: modify block end: no way man, prev == NULL?!?, jnl 0x%x, bp 0x%x\n", jnl, bp);
+			panic("jnl: modify block end: no way man, prev == NULL?!?, jnl %p, bp %p\n", jnl, bp);
 		}
 
 		// we got to the end of the list, didn't find the block and there's
@@ -2131,7 +2445,7 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
 		// avoids having yet another linked list of small data structures to manage.
 
 		if (kmem_alloc(kernel_map, (vm_offset_t *)&nblhdr, tr->tbuffer_size)) {
-			panic("jnl: end_tr: no space for new block tr @ 0x%x (total bytes: %d)!\n",
+			panic("jnl: end_tr: no space for new block tr @ %p (total bytes: %d)!\n",
 				  tr, tr->total_bytes);
 		}
 
@@ -2144,6 +2458,7 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
 		nblhdr->max_blocks = (jnl->jhdr->blhdr_size / sizeof(block_info)) - 1;
 		nblhdr->num_blocks = 1;      // accounts for this header block
 		nblhdr->bytes_used = jnl->jhdr->blhdr_size;
+		nblhdr->flags = BLHDR_CHECK_CHECKSUMS;
 	    
 		tr->num_blhdrs++;
 		tr->total_bytes += jnl->jhdr->blhdr_size;
@@ -2162,9 +2477,15 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
 		panic("jnl: modify_block_end: i = %d, max_blocks %d\n", i, blhdr->max_blocks);
     }
 
-    // copy the data into the in-memory transaction buffer
-    blkptr = (char *)&((char *)blhdr)[tbuffer_offset];
-    memcpy(blkptr, buf_dataptr(bp), buf_size(bp));
+	// if the function pointer is not set then copy the
+	// block of data now.  if the function pointer is set 
+	// the copy will happen after calling the callback in
+	// end_transaction() just before it goes to disk.
+	//
+	if (func == NULL) {
+		blkptr = (char *)&((char *)blhdr)[tbuffer_offset];
+		memcpy(blkptr, (char *)0 + buf_dataptr(bp), buf_size(bp));
+	}
 
     // if this is true then this is a new block we haven't seen
     if (i >= blhdr->num_blocks) {
@@ -2177,8 +2498,16 @@ journal_modify_block_end(journal *jnl, struct buf *bp)
 
 		blhdr->binfo[i].bnum  = (off_t)(buf_blkno(bp));
 		blhdr->binfo[i].bsize = bsize;
-		blhdr->binfo[i].bp    = bp;
-
+		blhdr->binfo[i].b.bp    = bp;
+		if (func) {
+			void *old_func=NULL, *old_arg=NULL;
+			
+			buf_setfilter(bp, func, arg, &old_func, &old_arg);
+			if (old_func != NULL) {
+				panic("jnl: modify_block_end: old func %p / arg %p", old_func, old_arg);
+			}
+		}
+		
 		blhdr->bytes_used += bsize;
 		tr->total_bytes   += bsize;
 
@@ -2207,7 +2536,7 @@ journal_kill_block(journal *jnl, struct buf *bp)
     CHECK_TRANSACTION(tr);
 
     if (jnl->owner != current_thread()) {
-		panic("jnl: modify_block_end: called w/out a transaction! jnl 0x%x, owner 0x%x, curact 0x%x\n",
+		panic("jnl: modify_block_end: called w/out a transaction! jnl %p, owner %p, curact %p\n",
 			  jnl, jnl->owner, current_thread());
     }
 
@@ -2225,7 +2554,7 @@ journal_kill_block(journal *jnl, struct buf *bp)
     for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) {
 
 		for(i=1; i < blhdr->num_blocks; i++) {
-			if (bp == blhdr->binfo[i].bp) {
+			if (bp == blhdr->binfo[i].b.bp) {
 			        vnode_t vp;
 
 				buf_clearflags(bp, B_LOCKED);
@@ -2244,9 +2573,10 @@ journal_kill_block(journal *jnl, struct buf *bp)
 				//} else {
 					tr->num_killed += buf_size(bp);
 				//}
-				blhdr->binfo[i].bp   = NULL;
+				blhdr->binfo[i].b.bp   = NULL;
 				blhdr->binfo[i].bnum = (off_t)-1;
 
+				buf_markinvalid(bp);
 				buf_brelse(bp);
 
 				break;
@@ -2263,40 +2593,71 @@ journal_kill_block(journal *jnl, struct buf *bp)
 
 
 static int
-journal_binfo_cmp(void *a, void *b)
+journal_binfo_cmp(const void *a, const void *b)
 {
-    block_info *bi_a = (struct block_info *)a;
-    block_info *bi_b = (struct block_info *)b;
+    const block_info *bi_a = (const struct block_info *)a;
+    const block_info *bi_b = (const struct block_info *)b;
     daddr64_t res;
 
-    if (bi_a->bp == NULL) {
+    if (bi_a->b.bp == NULL) {
 		return 1;
     }
-    if (bi_b->bp == NULL) {
+    if (bi_b->b.bp == NULL) {
 		return -1;
     }
 
     // don't have to worry about negative block
     // numbers so this is ok to do.
     //
-    res = (buf_blkno(bi_a->bp) - buf_blkno(bi_b->bp));
+    res = (buf_blkno(bi_a->b.bp) - buf_blkno(bi_b->b.bp));
 
     return (int)res;
 }
 
 
+/*
+ * End a transaction.  If the transaction is small enough, and we're not forcing
+ * a write to disk, the "active" transaction becomes the "current" transaction,
+ * and will be reused for the next transaction that is started (group commit).
+ *
+ * If the transaction gets written to disk (because force_it is true, or no
+ * group commit, or the transaction is sufficiently full), the blocks get
+ * written into the journal first, then the are written asynchronously.  When
+ * those async writes complete, the transaction can be freed and removed from
+ * the journal.
+ *
+ * An optional callback can be supplied.  If given, it is called after the
+ * the blocks have been written to the journal, but before the async writes
+ * of those blocks to their normal on-disk locations.  This is used by
+ * journal_relocate so that the location of the journal can be changed and
+ * flushed to disk before the blocks get written to their normal locations.
+ * Note that the callback is only called if the transaction gets written to
+ * the journal during this end_transaction call; you probably want to set the
+ * force_it flag.
+ *
+ * Inputs:
+ *	tr			 Transaction to add to the journal
+ *	force_it	 If true, force this transaction to the on-disk journal immediately.
+ *	callback	 See description above.  Pass NULL for no callback.
+ *	callback_arg Argument passed to callback routine.
+ *
+ * Result
+ *		 0		No errors
+ *		-1		An error occurred.  The journal is marked invalid.
+ */
 static int
-end_transaction(transaction *tr, int force_it)
+end_transaction(transaction *tr, int force_it, errno_t (*callback)(void*), void *callback_arg)
 {
-    int                 i, j, ret, amt;
+    int                 i, ret, amt;
     errno_t		errno;
     off_t               end;
     journal            *jnl = tr->jnl;
-    struct buf         *bp;
+    struct buf         *bp, **bparray;
     block_list_header  *blhdr=NULL, *next=NULL;
+    size_t              tbuffer_offset;
 
 	if (jnl->cur_tr) {
-		panic("jnl: jnl @ 0x%x already has cur_tr 0x%x, new tr: 0x%x\n",
+		panic("jnl: jnl @ %p already has cur_tr %p, new tr: %p\n",
 			  jnl, jnl->cur_tr, tr);
 	}
 
@@ -2357,7 +2718,7 @@ end_transaction(transaction *tr, int force_it)
 			lock_oldstart(jnl);
 		}
 		if (i++ >= 500) {
-			panic("jnl: transaction that started at 0x%llx is not completing! jnl 0x%x\n",
+			panic("jnl: transaction that started at 0x%llx is not completing! jnl %p\n",
 				  jnl->old_start[0] & (~0x8000000000000000LL), jnl);
 		}
 	}
@@ -2374,27 +2735,68 @@ end_transaction(transaction *tr, int force_it)
 
     // for each block, make sure that the physical block # is set
     for(blhdr=tr->blhdr; blhdr; blhdr=next) {
-
+		char *blkptr;
+		
+		tbuffer_offset = jnl->jhdr->blhdr_size;
 		for(i=1; i < blhdr->num_blocks; i++) {
-		        daddr64_t blkno;
-		        daddr64_t lblkno;
+			daddr64_t blkno;
+			daddr64_t lblkno;
 			struct vnode *vp;
 
-			bp = blhdr->binfo[i].bp;
-			if (bp == NULL) {   // only true if a block was "killed" 
+			bp = blhdr->binfo[i].b.bp;
+
+			// if this block has a callback function set, call
+			// it now and then copy the data from the bp into
+			// the journal. 
+			if (bp) {
+				void (*func)(struct buf *, void *);
+				void  *arg;
+
+				buf_setfilter(bp, NULL, NULL, (void **)&func, &arg);
+
+				if (func) {
+					// acquire the bp here so that we can safely
+					// mess around with its data.  buf_acquire()
+					// will return EAGAIN if the buffer was busy,
+					// so loop trying again.
+					do {
+						errno = buf_acquire(bp, 0, 0, 0);
+					} while (errno == EAGAIN);
+					
+					if (errno == 0) {
+					
+						// call the hook function and then copy the
+						// data into the transaction buffer...
+						func(bp, arg);
+
+						blkptr = (char *)&((char *)blhdr)[tbuffer_offset];
+						memcpy(blkptr, (char *)buf_dataptr(bp), buf_size(bp));
+
+						buf_drop(bp);
+					} else {
+						panic("could not acquire bp %p (err %d)\n", bp, errno);
+					}
+				}
+
+			} else {   // bp == NULL, only true if a block was "killed" 
 				if (blhdr->binfo[i].bnum != (off_t)-1) {
-					panic("jnl: inconsistent binfo (NULL bp w/bnum %lld; jnl @ 0x%x, tr 0x%x)\n",
-						  blhdr->binfo[i].bnum, jnl, tr);
+					panic("jnl: inconsistent binfo (NULL bp w/bnum %lld; jnl @ %p, tr %p)\n",
+						blhdr->binfo[i].bnum, jnl, tr);
 				}
+
+				tbuffer_offset += blhdr->binfo[i].bsize;
 				continue;
 			}
+
+			tbuffer_offset += blhdr->binfo[i].bsize;
+
 			vp = buf_vnode(bp);
 			blkno = buf_blkno(bp);
 			lblkno = buf_lblkno(bp);
 
 			if (vp == NULL && lblkno == blkno) {
-			    printf("jnl: end_tr: bad news! bp @ 0x%x w/null vp and l/blkno = %qd/%qd.  aborting the transaction (tr 0x%x jnl 0x%x).\n",
-					  bp, lblkno, blkno, tr, jnl);
+			    printf("jnl: %s: end_tr: bad news! bp @ %p w/null vp and l/blkno = %qd/%qd.  aborting the transaction (tr %p jnl %p).\n",
+				jnl->jdev_name, bp, lblkno, blkno, tr, jnl);
 			    goto bad_journal;
 			}
 	    
@@ -2407,15 +2809,15 @@ end_transaction(transaction *tr, int force_it)
 				size_t 	contig_bytes;
 
 				if (VNOP_BLKTOOFF(vp, lblkno, &f_offset)) {
-					printf("jnl: end_tr: vnop_blktooff failed @ 0x%x, jnl 0x%x\n", bp, jnl);
+					printf("jnl: %s: end_tr: vnop_blktooff failed @ %p, jnl %p\n", jnl->jdev_name, bp, jnl);
 					goto bad_journal;
 				}
 				if (VNOP_BLOCKMAP(vp, f_offset, buf_count(bp), &blkno, &contig_bytes, NULL, 0, NULL)) {
-					printf("jnl: end_tr: can't blockmap the bp @ 0x%x, jnl 0x%x\n", bp, jnl);
+					printf("jnl: %s: end_tr: can't blockmap the bp @ %p, jnl %p\n", jnl->jdev_name, bp, jnl);
 					goto bad_journal;
 				}
 				if ((uint32_t)contig_bytes < buf_count(bp)) {
-					printf("jnl: end_tr: blk not physically contiguous on disk@ 0x%x, jnl 0x%x\n", bp, jnl);
+					printf("jnl: %s: end_tr: blk not physically contiguous on disk@ %p, jnl %p\n", jnl->jdev_name, bp, jnl);
 					goto bad_journal;
 				}
 				buf_setblkno(bp, blkno);
@@ -2427,17 +2829,45 @@ end_transaction(transaction *tr, int force_it)
 		next = (block_list_header *)((long)blhdr->binfo[0].bnum);
     }
     
-    for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) {
 
+
+    for(blhdr=tr->blhdr; blhdr; blhdr=(block_list_header *)((long)blhdr->binfo[0].bnum)) {
 		amt = blhdr->bytes_used;
 
+		blhdr->binfo[0].b.sequence_num = tr->sequence_num;
+
 		blhdr->checksum = 0;
 		blhdr->checksum = calc_checksum((char *)blhdr, BLHDR_CHECKSUM_SIZE);
-	
+
+		if (kmem_alloc(kernel_map, (vm_offset_t *)&bparray, tr->blhdr->num_blocks * sizeof(struct buf *))) {
+		    panic("can't allocate %lu bytes for bparray\n", tr->blhdr->num_blocks * sizeof(struct buf *));
+		}
+
+		// calculate individual block checksums
+		tbuffer_offset = jnl->jhdr->blhdr_size;
+		for(i=1; i < blhdr->num_blocks; i++) {
+		    bparray[i] = blhdr->binfo[i].b.bp;
+		    if (bparray[i]) {
+			blhdr->binfo[i].b.cksum = calc_checksum(&((char *)blhdr)[tbuffer_offset], blhdr->binfo[i].bsize);
+		    } else {
+			blhdr->binfo[i].b.cksum = 0;
+		    }
+
+		    tbuffer_offset += blhdr->binfo[i].bsize;
+		}
+
 		ret = write_journal_data(jnl, &end, blhdr, amt);
+
+		// always put the bp pointers back
+		for(i=1; i < blhdr->num_blocks; i++) {
+		    blhdr->binfo[i].b.bp = bparray[i];
+		}
+
+		kmem_free(kernel_map, (vm_offset_t)bparray, tr->blhdr->num_blocks * sizeof(struct buf *));
+
 		if (ret != amt) {
-			printf("jnl: end_transaction: only wrote %d of %d bytes to the journal!\n",
-				   ret, amt);
+			printf("jnl: %s: end_transaction: only wrote %d of %d bytes to the journal!\n",
+			    jnl->jdev_name, ret, amt);
 
 			goto bad_journal;
 		}
@@ -2454,6 +2884,15 @@ end_transaction(transaction *tr, int force_it)
 		goto bad_journal;
     }
 
+	/*
+	 * If the caller supplied a callback, call it now that the blocks have been
+	 * written to the journal.  This is used by journal_relocate so, for example,
+	 * the file system can change its pointer to the new journal.
+	 */
+	if (callback != NULL && callback(callback_arg) != 0) {
+		goto bad_journal;
+	}
+	
     //
     // setup for looping through all the blhdr's.  we null out the
     // tbuffer and blhdr fields so that they're not used any more.
@@ -2475,31 +2914,29 @@ end_transaction(transaction *tr, int force_it)
 		qsort(&blhdr->binfo[1], blhdr->num_blocks-1, sizeof(block_info), journal_binfo_cmp);
 
 		for(i=1; i < blhdr->num_blocks; i++) {
-			if (blhdr->binfo[i].bp == NULL) {
+			if (blhdr->binfo[i].b.bp == NULL) {
 				continue;
 			}
 
-			errno = buf_meta_bread(buf_vnode(blhdr->binfo[i].bp),
-							 buf_lblkno(blhdr->binfo[i].bp),
-							 buf_size(blhdr->binfo[i].bp),
-							 NOCRED,
-							 &bp);
-			if (errno == 0 && bp != NULL) {
+			bp = blhdr->binfo[i].b.bp;
+
+			// have to pass BAC_REMOVE here because we're going to bawrite()
+			// the buffer when we're done
+			do {
+				errno = buf_acquire(bp, BAC_REMOVE, 0, 0);
+			} while (errno == EAGAIN);
+			
+			if (errno == 0) {
 				struct vnode *save_vp;
 				void *cur_filter;
 
-				if (bp != blhdr->binfo[i].bp) {
-					panic("jnl: end_tr: got back a different bp! (bp 0x%x should be 0x%x, jnl 0x%x\n",
-						  bp, blhdr->binfo[i].bp, jnl);
-				}
-
 				if ((buf_flags(bp) & (B_LOCKED|B_DELWRI)) != (B_LOCKED|B_DELWRI)) {
 					if (jnl->flags & JOURNAL_CLOSE_PENDING) {
 					    buf_clearflags(bp, B_LOCKED);
 					    buf_brelse(bp);
 						continue;
 					} else {
-						panic("jnl: end_tr: !!!DANGER!!! bp 0x%x flags (0x%x) not LOCKED & DELWRI\n", bp, buf_flags(bp));
+						panic("jnl: end_tr: !!!DANGER!!! bp %p flags (0x%x) not LOCKED & DELWRI\n", bp, buf_flags(bp));
 					}
 				}
 				save_vp = buf_vnode(bp);
@@ -2507,7 +2944,7 @@ end_transaction(transaction *tr, int force_it)
 				buf_setfilter(bp, buffer_flushed_callback, tr, &cur_filter, NULL);
 
 				if (cur_filter) {
-					panic("jnl: bp @ 0x%x (blkno %qd, vp 0x%x) has non-null iodone (0x%x) buffflushcb 0x%x\n",
+					panic("jnl: bp @ %p (blkno %qd, vp %p) has non-null iodone (%p) buffflushcb %p\n",
 						  bp, buf_blkno(bp), save_vp, cur_filter, buffer_flushed_callback);
 				}
 				buf_clearflags(bp, B_LOCKED);
@@ -2520,12 +2957,8 @@ end_transaction(transaction *tr, int force_it)
 				// this undoes the vnode_ref() in journal_modify_block_end()
 				vnode_rele_ext(save_vp, 0, 1);
 			} else {
-				printf("jnl: end_transaction: could not find block %Ld vp 0x%x!\n",
-					   blhdr->binfo[i].bnum, blhdr->binfo[i].bp);
-				if (bp) {
-					buf_clearflags(bp, B_LOCKED);
-					buf_brelse(bp);
-				}
+				printf("jnl: %s: end_transaction: could not acquire block %p (errno %d)!\n",
+				    jnl->jdev_name,bp, errno);
 			}
 		}
 
@@ -2565,23 +2998,23 @@ abort_transaction(journal *jnl, transaction *tr)
     for(blhdr=tr->blhdr; blhdr; blhdr=next) {
 
 		for(i=1; i < blhdr->num_blocks; i++) {
-			if (blhdr->binfo[i].bp == NULL) {
+			if (blhdr->binfo[i].b.bp == NULL) {
 				continue;
 			}
-			if ( (buf_vnode(blhdr->binfo[i].bp) == NULL) ||
-			     !(buf_flags(blhdr->binfo[i].bp) & B_LOCKED) ) {
+			if ( (buf_vnode(blhdr->binfo[i].b.bp) == NULL) ||
+			     !(buf_flags(blhdr->binfo[i].b.bp) & B_LOCKED) ) {
 			        continue;
 			}
 
-			errno = buf_meta_bread(buf_vnode(blhdr->binfo[i].bp),
-							 buf_lblkno(blhdr->binfo[i].bp),
-							 buf_size(blhdr->binfo[i].bp),
+			errno = buf_meta_bread(buf_vnode(blhdr->binfo[i].b.bp),
+							 buf_lblkno(blhdr->binfo[i].b.bp),
+							 buf_size(blhdr->binfo[i].b.bp),
 							 NOCRED,
 							 &bp);
 			if (errno == 0) {
-				if (bp != blhdr->binfo[i].bp) {
-					panic("jnl: abort_tr: got back a different bp! (bp 0x%x should be 0x%x, jnl 0x%x\n",
-						  bp, blhdr->binfo[i].bp, jnl);
+				if (bp != blhdr->binfo[i].b.bp) {
+					panic("jnl: abort_tr: got back a different bp! (bp %p should be %p, jnl %p\n",
+						  bp, blhdr->binfo[i].b.bp, jnl);
 				}
 
 				// releasing a bp marked invalid
@@ -2593,8 +3026,8 @@ abort_transaction(journal *jnl, transaction *tr)
 
 				vnode_rele_ext(save_vp, 0, 1);
 			} else {
-				printf("jnl: abort_tr: could not find block %Ld vp 0x%x!\n",
-					   blhdr->binfo[i].bnum, blhdr->binfo[i].bp);
+				printf("jnl: %s: abort_tr: could not find block %Ld vp %p!\n",
+				    jnl->jdev_name, blhdr->binfo[i].bnum, blhdr->binfo[i].b.bp);
 				if (bp) {
 					buf_brelse(bp);
 				}
@@ -2628,7 +3061,7 @@ journal_end_transaction(journal *jnl)
 	}
 
     if (jnl->owner != current_thread()) {
-		panic("jnl: end_tr: I'm not the owner! jnl 0x%x, owner 0x%x, curact 0x%x\n",
+		panic("jnl: end_tr: I'm not the owner! jnl %p, owner %p, curact %p\n",
 			  jnl, jnl->owner, current_thread());
     }
 
@@ -2638,13 +3071,13 @@ journal_end_transaction(journal *jnl)
     if (jnl->nested_count > 0) {
 		return 0;
     } else if (jnl->nested_count < 0) {
-		panic("jnl: jnl @ 0x%x has negative nested count (%d). bad boy.\n", jnl, jnl->nested_count);
+		panic("jnl: jnl @ %p has negative nested count (%d). bad boy.\n", jnl, jnl->nested_count);
     }
     
     if (jnl->flags & JOURNAL_INVALID) {
 		if (jnl->active_tr) {
 			if (jnl->cur_tr != NULL) {
-				panic("jnl: journal @ 0x%x has active tr (0x%x) and cur tr (0x%x)\n",
+				panic("jnl: journal @ %p has active tr (%p) and cur tr (%p)\n",
 					  jnl, jnl->active_tr, jnl->cur_tr);
 			}
 	    
@@ -2668,7 +3101,7 @@ journal_end_transaction(journal *jnl)
     // called from end_transaction().
     // 
     jnl->active_tr = NULL;
-    ret = end_transaction(tr, 0);
+    ret = end_transaction(tr, 0, NULL, NULL);
 
     jnl->owner = NULL;
     unlock_journal(jnl);
@@ -2688,9 +3121,10 @@ journal_flush(journal *jnl)
 		return -1;
     }
 
-    if (jnl->owner != current_thread()) {
-		int ret;
+    KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_JOURNAL, DBG_JOURNAL_FLUSH)) 
+    	| DBG_FUNC_START, 0, 0, 0, 0, 0);
 
+    if (jnl->owner != current_thread()) {
 		lock_journal(jnl);
 		need_signal = 1;
     }
@@ -2702,13 +3136,16 @@ journal_flush(journal *jnl)
 		transaction *tr = jnl->cur_tr;
 
 		jnl->cur_tr = NULL;
-		end_transaction(tr, 1);   // force it to get flushed
+		end_transaction(tr, 1, NULL, NULL);   // force it to get flushed
     }
 
     if (need_signal) {
 		unlock_journal(jnl);
     }
 
+    KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_JOURNAL, DBG_JOURNAL_FLUSH)) 
+    	| DBG_FUNC_END, 0, 0, 0, 0, 0);
+
     return 0;
 }
 
@@ -2727,3 +3164,151 @@ journal_owner(journal *jnl)
 {
     return jnl->owner;
 }
+
+int journal_uses_fua(journal *jnl)
+{
+	if (jnl->flags & JOURNAL_DO_FUA_WRITES)
+		return 1;
+	return 0;
+}
+
+/*
+ * Relocate the journal.
+ * 
+ * You provide the new starting offset and size for the journal. You may
+ * optionally provide a new tbuffer_size; passing zero defaults to not
+ * changing the tbuffer size except as needed to fit within the new journal
+ * size.
+ * 
+ * You must have already started a transaction. The transaction may contain
+ * modified blocks (such as those needed to deallocate the old journal,
+ * allocate the new journal, and update the location and size of the journal
+ * in filesystem-private structures). Any transactions prior to the active
+ * transaction will be flushed to the old journal. The new journal will be
+ * initialized, and the blocks from the active transaction will be written to
+ * the new journal.
+ *
+ * The caller will need to update the structures that identify the location
+ * and size of the journal.  These updates should be made in the supplied
+ * callback routine.  These updates must NOT go into a transaction.  You should
+ * force these updates to the media before returning from the callback.  In the
+ * even of a crash, either the old journal will be found, with an empty journal,
+ * or the new journal will be found with the contents of the active transaction.
+ *
+ * Upon return from the callback, the blocks from the active transaction are
+ * written to their normal locations on disk.
+ *
+ * (Remember that we have to ensure that blocks get committed to the journal
+ * before being committed to their normal locations.  But the blocks don't count
+ * as committed until the new journal is pointed at.)
+ *
+ * Upon return, there is still an active transaction: newly allocated, and
+ * with no modified blocks.  Call journal_end_transaction as normal.  You may
+ * modifiy additional blocks before calling journal_end_transaction, and those
+ * blocks will (eventually) go to the relocated journal.
+ *
+ * Inputs:
+ *	jnl				The (opened) journal to relocate.
+ *	offset			The new journal byte offset (from start of the journal device).
+ *	journal_size	The size, in bytes, of the new journal.
+ *	tbuffer_size	The new desired transaction buffer size.  Pass zero to keep
+ *					the same size as the current journal.  The size will be
+ *					modified as needed to fit the new journal.
+ *	callback		Routine called after the new journal has been initialized,
+ *					and the active transaction written to the new journal, but
+ *					before the blocks are written to their normal locations.
+ *					Pass NULL for no callback.
+ *	callback_arg	An argument passed to the callback routine.
+ *
+ * Result:
+ *	0				No errors
+ *	EINVAL			The offset is not block aligned
+ *	EINVAL			The journal_size is not a multiple of the block size
+ *	EINVAL			The journal is invalid
+ *	(any)			An error returned by journal_flush.
+ *
+ */
+int journal_relocate(journal *jnl, off_t offset, off_t journal_size, int32_t tbuffer_size,
+	errno_t (*callback)(void *), void *callback_arg)
+{
+	int ret;
+	transaction *tr;
+	
+	/*
+	 * Sanity check inputs, and adjust the size of the transaction buffer.
+	 */
+    if ((offset % jnl->jhdr->jhdr_size) != 0) {
+		printf("jnl: %s: relocate: offset 0x%llx is not an even multiple of block size 0x%x\n",
+		    jnl->jdev_name, offset, jnl->jhdr->jhdr_size);
+		return EINVAL;
+    }
+    if ((journal_size % jnl->jhdr->jhdr_size) != 0) {
+		printf("jnl: %s: relocate: journal size 0x%llx is not an even multiple of block size 0x%x\n",
+		    jnl->jdev_name, journal_size, jnl->jhdr->jhdr_size);
+		return EINVAL;
+    }
+
+    CHECK_JOURNAL(jnl);
+
+	/* Guarantee we own the active transaction. */
+    if (jnl->flags & JOURNAL_INVALID) {
+		return EINVAL;
+    }
+    if (jnl->owner != current_thread()) {
+    	panic("jnl: relocate: Not the owner! jnl %p, owner %p, curact %p\n",
+    		jnl, jnl->owner, current_thread());
+	}
+	
+    if (tbuffer_size == 0)
+    	tbuffer_size = jnl->tbuffer_size;
+    size_up_tbuffer(jnl, tbuffer_size, jnl->jhdr->jhdr_size);
+	
+	/*
+	 * Flush any non-active transactions.  We have to temporarily hide the
+	 * active transaction to make journal_flush flush out non-active but
+	 * current (unwritten) transactions.
+	 */
+	tr = jnl->active_tr;
+	CHECK_TRANSACTION(tr);
+	jnl->active_tr = NULL;
+	ret = journal_flush(jnl);
+	jnl->active_tr = tr;
+	if (ret) {
+		return ret;
+	}
+	
+	/* Update the journal's offset and size in memory. */
+	jnl->jdev_offset = offset;
+	jnl->jhdr->start = jnl->jhdr->end = jnl->jhdr->jhdr_size;
+	jnl->jhdr->size = journal_size;
+	jnl->active_start = jnl->jhdr->start;
+	
+	/*
+	 * Force the active transaction to be written to the new journal.  Call the
+	 * supplied callback after the blocks have been written to the journal, but
+	 * before they get written to their normal on-disk locations.
+	 */
+	jnl->active_tr = NULL;
+	ret = end_transaction(tr, 1, callback, callback_arg);
+	if (ret) {
+		printf("jnl: %s: relocate: end_transaction failed (%d)\n", jnl->jdev_name, ret);
+		goto bad_journal;
+	}
+	
+	/*
+	 * Create a new, empty transaction to be the active transaction.  This way
+	 * our caller can use journal_end_transaction as usual.
+	 */
+	ret = journal_allocate_transaction(jnl);
+	if (ret) {
+		printf("jnl: %s: relocate: could not allocate new transaction (%d)\n", jnl->jdev_name, ret);
+		goto bad_journal;
+	}
+	
+	return 0;
+
+bad_journal:
+    jnl->flags |= JOURNAL_INVALID;
+    abort_transaction(jnl, tr);
+    return ret;
+}
diff --git a/bsd/vfs/vfs_journal.h b/bsd/vfs/vfs_journal.h
index b03209b9a..e6461cc1a 100644
--- a/bsd/vfs/vfs_journal.h
+++ b/bsd/vfs/vfs_journal.h
@@ -1,24 +1,30 @@
 
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * This header contains the structures and function prototypes
@@ -40,7 +46,11 @@
 typedef struct block_info {
     off_t       bnum;                // block # on the file system device
     size_t      bsize;               // in bytes
-    struct buf *bp;
+    union {
+	int32_t     cksum;
+	uint32_t    sequence_num;    // only used in block_list_header->binfo[0]
+	struct buf *bp;
+    } b;
 } block_info;
 
 typedef struct block_list_header {
@@ -48,10 +58,13 @@ typedef struct block_list_header {
     u_int16_t   num_blocks;          // number of valid block numbers in block_nums
     int32_t     bytes_used;          // how many bytes of this tbuffer are used
     int32_t     checksum;            // on-disk: checksum of this header and binfo[0]
-    int32_t     pad;                 // pad out to 16 bytes
+    int32_t     flags;               // check-checksums, initial blhdr, etc
     block_info  binfo[1];            // so we can reference them by name
 } block_list_header;
 
+#define BLHDR_CHECK_CHECKSUMS   0x0001
+#define BLHDR_FIRST_HEADER      0x0002
+
 
 struct journal;
 
@@ -67,6 +80,7 @@ typedef struct transaction {
     off_t               journal_end;   // where in the journal this transaction ends
     struct journal     *jnl;           // ptr back to the journal structure
     struct transaction *next;          // list of tr's (either completed or to be free'd)
+    uint32_t            sequence_num;
 } transaction;
 
 
@@ -83,11 +97,20 @@ typedef struct journal_header {
     int32_t        blhdr_size;    // size in bytes of each block_list_header in the journal
     int32_t        checksum;
     int32_t        jhdr_size;     // block size (in bytes) of the journal header
+    uint32_t       sequence_num;  // NEW FIELD: a monotonically increasing value assigned to all txn's
 } journal_header;
 
 #define JOURNAL_HEADER_MAGIC  0x4a4e4c78   // 'JNLx'
 #define ENDIAN_MAGIC          0x12345678
 
+//
+// we only checksum the original size of the journal_header to remain
+// backwards compatible.  the size of the original journal_heade is
+// everything up to the the sequence_num field, hence we use the
+// offsetof macro to calculate the size.
+//
+#define JOURNAL_HEADER_CKSUM_SIZE  (offsetof(struct journal_header, sequence_num))
+
 #define OLD_JOURNAL_HEADER_MAGIC  0x4a484452   // 'JHDR'
 
 
@@ -99,6 +122,7 @@ typedef struct journal {
 
     struct vnode       *jdev;              // vnode of the device where the journal lives
     off_t               jdev_offset;       // byte offset to the start of the journal
+    const char         *jdev_name;
 
     struct vnode       *fsdev;             // vnode of the file system device
     
@@ -111,6 +135,9 @@ typedef struct journal {
     char               *header_buf;        // in-memory copy of the journal header
     journal_header     *jhdr;              // points to the first byte of header_buf
 
+    off_t               max_read_size;
+    off_t               max_write_size;
+
     transaction        *cur_tr;            // for group-commit
     transaction        *completed_trs;     // out-of-order transactions that completed
     transaction        *active_tr;         // for nested transactions
@@ -131,6 +158,7 @@ typedef struct journal {
 #define JOURNAL_INVALID           0x00020000
 #define JOURNAL_FLUSHCACHE_ERR    0x00040000   // means we already printed this err
 #define JOURNAL_NEED_SWAP         0x00080000   // swap any data read from disk
+#define JOURNAL_DO_FUA_WRITES     0x00100000   // do force-unit-access writes
 
 /* journal_open/create options are always in the low-16 bits */
 #define JOURNAL_OPTION_FLAGS_MASK 0x0000ffff
@@ -143,7 +171,7 @@ __BEGIN_DECLS
 /*
  * Call journal_init() to initialize the journaling code (sets up lock attributes)
  */
-void      journal_init(void);
+void      journal_init(void) __attribute__((section("__TEXT, initcode")));
 
 /*
  * Call journal_create() to create a new journal.  You only
@@ -252,13 +280,35 @@ void      journal_close(journal *journalp);
 int   journal_start_transaction(journal *jnl);
 int   journal_modify_block_start(journal *jnl, struct buf *bp);
 int   journal_modify_block_abort(journal *jnl, struct buf *bp);
-int   journal_modify_block_end(journal *jnl, struct buf *bp);
+int   journal_modify_block_end(journal *jnl, struct buf *bp, void (*func)(struct buf *bp, void *arg), void *arg);
 int   journal_kill_block(journal *jnl, struct buf *bp);
 int   journal_end_transaction(journal *jnl);
 
 int   journal_active(journal *jnl);
 int   journal_flush(journal *jnl);
 void *journal_owner(journal *jnl);    // compare against current_thread()
+int   journal_uses_fua(journal *jnl);
+
+
+/*
+ * Relocate the journal.
+ * 
+ * You provide the new starting offset and size for the journal. You may
+ * optionally provide a new tbuffer_size; passing zero defaults to not
+ * changing the tbuffer size except as needed to fit within the new journal
+ * size.
+ * 
+ * You must have already started a transaction. The transaction may contain
+ * modified blocks (such as those needed to deallocate the old journal,
+ * allocate the new journal, and update the location and size of the journal
+ * in filesystem-private structures). Any transactions prior to the active
+ * transaction will be flushed to the old journal. The new journal will be
+ * initialized, and the blocks from the active transaction will be written to
+ * the new journal. The caller will need to update the structures that
+ * identify the location and size of the journal from the callback routine.
+ */
+int journal_relocate(journal *jnl, off_t offset, off_t journal_size, int32_t tbuffer_size,
+	errno_t (*callback)(void *), void *callback_arg);
 
 __END_DECLS
 
diff --git a/bsd/vfs/vfs_lookup.c b/bsd/vfs/vfs_lookup.c
index f0aadcc4e..aaabf7bc1 100644
--- a/bsd/vfs/vfs_lookup.c
+++ b/bsd/vfs/vfs_lookup.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,6 +65,12 @@
  *
  *	@(#)vfs_lookup.c	8.10 (Berkeley) 5/27/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -79,13 +91,26 @@
 
 #include <bsm/audit_kernel.h>
 
-#if KTRACE
-#include <sys/ktrace.h>
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
+#if NAMEDRSRCFORK
+#include <sys/xattr.h>
 #endif
+/*
+ * The minimum volfs-style pathname is 9.
+ * Example:  "/.vol/1/2"
+ */
+#define VOLFS_MIN_PATH_LEN  9
 
 
 static	void kdebug_lookup(struct vnode *dp, struct componentname *cnp);
 
+#if CONFIG_VOLFS
+static int vfs_getrealpath(const char * path, char * realpath, size_t bufsize, vfs_context_t ctx);
+#endif
+
 /*
  * Convert a pathname into a pointer to a locked inode.
  *
@@ -105,20 +130,33 @@ static	void kdebug_lookup(struct vnode *dp, struct componentname *cnp);
  *		call lookup to search path.
  *		if symbolic link, massage name in buffer and continue
  *	}
+ *
+ * Returns:	0			Success
+ *		ENOENT			No such file or directory
+ *		ELOOP			Too many levels of symbolic links
+ *		ENAMETOOLONG		Filename too long
+ *		copyinstr:EFAULT	Bad address
+ *		copyinstr:ENAMETOOLONG	Filename too long
+ *		lookup:EBADF		Bad file descriptor
+ *		lookup:EROFS
+ *		lookup:EACCES
+ *		lookup:EPERM
+ *		lookup:???
+ *		VNOP_READLINK:???
  */
-
 int
-namei(ndp)
-	register struct nameidata *ndp;
+namei(struct nameidata *ndp)
 {
-	register struct filedesc *fdp;	/* pointer to file descriptor state */
-	register char *cp;		/* pointer into pathname argument */
-	register struct vnode *dp;	/* the directory we are searching */
+	struct filedesc *fdp;	/* pointer to file descriptor state */
+	char *cp;		/* pointer into pathname argument */
+	struct vnode *dp;	/* the directory we are searching */
 	uio_t auio;
 	int error;
 	struct componentname *cnp = &ndp->ni_cnd;
 	vfs_context_t ctx = cnp->cn_context;
-	struct proc *p = vfs_context_proc(ctx);
+	proc_t p = vfs_context_proc(ctx);
+/* XXX ut should be from context */
+	uthread_t ut = (struct uthread *)get_bsdthread_info(current_thread());
 	char *tmppn;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
@@ -137,7 +175,7 @@ namei(ndp)
 	 * name into the buffer.
 	 */
 	if ((cnp->cn_flags & HASBUF) == 0) {
-		cnp->cn_pnbuf = &ndp->ni_pathbuf;
+		cnp->cn_pnbuf = ndp->ni_pathbuf;
 		cnp->cn_pnlen = PATHBUFLEN;
 	}
 #if LP64_DEBUG
@@ -147,16 +185,19 @@ namei(ndp)
 #endif /* LP64_DEBUG */
 
 retry_copy:
-	if (UIO_SEG_IS_USER_SPACE(ndp->ni_segflg))
+	if (UIO_SEG_IS_USER_SPACE(ndp->ni_segflg)) {
 		error = copyinstr(ndp->ni_dirp, cnp->cn_pnbuf,
 			    cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen);
-	else
+	} else {
 		error = copystr(CAST_DOWN(void *, ndp->ni_dirp), cnp->cn_pnbuf,
 			    cnp->cn_pnlen, (size_t *)&ndp->ni_pathlen);
-
+	}
 	if (error == ENAMETOOLONG && !(cnp->cn_flags & HASBUF)) {
-	        MALLOC_ZONE(cnp->cn_pnbuf, caddr_t,
-			    MAXPATHLEN, M_NAMEI, M_WAITOK);
+		MALLOC_ZONE(cnp->cn_pnbuf, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
+		if (cnp->cn_pnbuf == NULL) {
+			error = ENOMEM;
+			goto error_out;
+		}
 
 		cnp->cn_flags |= HASBUF;
 		cnp->cn_pnlen = MAXPATHLEN;
@@ -166,24 +207,58 @@ retry_copy:
 	if (error)
 	        goto error_out;
 
+#if CONFIG_VOLFS
+ 	/*
+	 * Check for legacy volfs style pathnames.
+	 *
+	 * For compatibility reasons we currently allow these paths,
+	 * but future versions of the OS may not support them.
+	 */
+	if (ndp->ni_pathlen >= VOLFS_MIN_PATH_LEN &&
+	    cnp->cn_pnbuf[0] == '/' &&
+	    cnp->cn_pnbuf[1] == '.' &&
+	    cnp->cn_pnbuf[2] == 'v' &&
+	    cnp->cn_pnbuf[3] == 'o' &&
+	    cnp->cn_pnbuf[4] == 'l' &&
+	    cnp->cn_pnbuf[5] == '/' ) {
+		char * realpath;
+		int realpath_err;
+		/* Attempt to resolve a legacy volfs style pathname. */
+		MALLOC_ZONE(realpath, caddr_t, MAXPATHLEN, M_NAMEI, M_WAITOK);
+		if (realpath) {
+			if ((realpath_err= vfs_getrealpath(&cnp->cn_pnbuf[6], realpath, MAXPATHLEN, ctx))) {
+				FREE_ZONE(realpath, MAXPATHLEN, M_NAMEI);
+				if (realpath_err == ENOSPC){
+					error = ENAMETOOLONG;
+					goto error_out;
+				}
+			} else {
+				if (cnp->cn_flags & HASBUF) {
+					FREE_ZONE(cnp->cn_pnbuf, cnp->cn_pnlen, M_NAMEI);
+				}
+				cnp->cn_pnbuf = realpath;
+				cnp->cn_pnlen = MAXPATHLEN;
+				ndp->ni_pathlen = strlen(realpath) + 1;
+				cnp->cn_flags |= HASBUF | CN_VOLFSPATH;
+			}
+		}
+	}
+ #endif /* CONFIG_VOLFS */
+
 	/* If we are auditing the kernel pathname, save the user pathname */
 	if (cnp->cn_flags & AUDITVNPATH1)
-		AUDIT_ARG(upath, p, cnp->cn_pnbuf, ARG_UPATH1); 
+		AUDIT_ARG(upath, ut->uu_cdir, cnp->cn_pnbuf, ARG_UPATH1); 
 	if (cnp->cn_flags & AUDITVNPATH2)
-		AUDIT_ARG(upath, p, cnp->cn_pnbuf, ARG_UPATH2); 
+		AUDIT_ARG(upath, ut->uu_cdir, cnp->cn_pnbuf, ARG_UPATH2); 
 
 	/*
 	 * Do not allow empty pathnames
 	 */
 	if (*cnp->cn_pnbuf == '\0') {
 		error = ENOENT;
-	        goto error_out;
+		goto error_out;
 	}
 	ndp->ni_loopcnt = 0;
-#if KTRACE
-	if (KTRPOINT(p, KTR_NAMEI))
-		ktrnamei(p->p_tracep, cnp->cn_pnbuf);
-#endif
 
 	/*
 	 * determine the starting point for the translation.
@@ -206,9 +281,9 @@ retry_copy:
 	        dp = ndp->ni_dvp;
 		ndp->ni_usedvp = dp;
 	} else
-	        dp = fdp->fd_cdir;
+	        dp = vfs_context_cwd(ctx);
 
-	if (dp == NULLVP) {
+	if (dp == NULLVP || (dp->v_lflag & VL_DEAD)) {
 	        error = ENOENT;
 		goto error_out;
 	}
@@ -238,6 +313,10 @@ retry_copy:
 			error = ELOOP;
 			break;
 		}
+#if CONFIG_MACF
+		if ((error = mac_vnode_check_readlink(ctx, ndp->ni_vp)) != 0)
+			break;
+#endif /* MAC */
 		if (ndp->ni_pathlen > 1 || !(cnp->cn_flags & HASBUF))
 		        need_newpathbuf = 1;
 		else
@@ -245,6 +324,10 @@ retry_copy:
 
 		if (need_newpathbuf) {
 			MALLOC_ZONE(cp, char *, MAXPATHLEN, M_NAMEI, M_WAITOK);
+			if (cp == NULL) {
+				error = ENOMEM;
+				break;
+			}
 		} else {
 			cp = cnp->cn_pnbuf;
 		}
@@ -294,7 +377,7 @@ retry_copy:
 		 * get rid of references returned via 'lookup'
 		 */
 		vnode_put(ndp->ni_vp);
-	        vnode_put(ndp->ni_dvp);
+		vnode_put(ndp->ni_dvp);
 
 		ndp->ni_vp = NULLVP;
 		ndp->ni_dvp = NULLVP;
@@ -325,7 +408,7 @@ retry_copy:
 	        vnode_put(ndp->ni_vp);
  error_out:
 	if ( (cnp->cn_flags & HASBUF) ) {
-	        cnp->cn_flags &= ~HASBUF;
+		cnp->cn_flags &= ~HASBUF;
 		FREE_ZONE(cnp->cn_pnbuf, cnp->cn_pnlen, M_NAMEI);
 	}
 	cnp->cn_pnbuf = NULL;
@@ -372,12 +455,29 @@ retry_copy:
  *	return the answer in ni_vp, locked if LOCKLEAF set
  *	    if LOCKPARENT set, return locked parent in ni_dvp
  *	    if WANTPARENT set, return unlocked parent in ni_dvp
+ *
+ * Returns:	0			Success
+ *		ENOENT			No such file or directory
+ *		EBADF			Bad file descriptor
+ *		ENOTDIR			Not a directory
+ *		EROFS			Read-only file system [CREATE]
+ *		EISDIR			Is a directory [CREATE]
+ *		cache_lookup_path:ENOENT
+ *		vnode_authorize:EROFS
+ *		vnode_authorize:EACCES
+ *		vnode_authorize:EPERM
+ *		vnode_authorize:???
+ *		VNOP_LOOKUP:ENOENT	No such file or directory
+ *		VNOP_LOOKUP:EJUSTRETURN	Restart system call (INTERNAL)
+ *		VNOP_LOOKUP:???
+ *		VFS_ROOT:ENOTSUP
+ *		VFS_ROOT:ENOENT
+ *		VFS_ROOT:???
  */
 int
-lookup(ndp)
-	register struct nameidata *ndp;
+lookup(struct nameidata *ndp)
 {
-	register char	*cp;		/* pointer into pathname argument */
+	char	*cp;		/* pointer into pathname argument */
 	vnode_t		tdp;		/* saved dp */
 	vnode_t		dp;		/* the directory we are searching */
 	mount_t		mp;		/* mount table entry */
@@ -389,6 +489,12 @@ lookup(ndp)
 	int error = 0;
 	struct componentname *cnp = &ndp->ni_cnd;
 	vfs_context_t ctx = cnp->cn_context;
+	int mounted_on_depth = 0;
+	int dont_cache_mp = 0;
+	vnode_t	mounted_on_dp = NULLVP;
+	int current_mount_generation = 0;
+	int vbusyflags = 0;
+	int nc_generation = 0;
 
 	/*
 	 * Setup: break out flag bits into variables.
@@ -405,6 +511,8 @@ lookup(ndp)
 	dp = ndp->ni_startdir;
 	ndp->ni_startdir = NULLVP;
 
+	if ((cnp->cn_flags & CN_NBMOUNTLOOK) != 0)
+			vbusyflags = LK_NOWAIT;
 	cp = cnp->cn_nameptr;
 
 	if (*cp == '\0') {
@@ -435,7 +543,15 @@ dirloop:
 		 * cache_lookup_path returned a non-NULL ni_vp then,
 		 * we're guaranteed that the dp is a VDIR, it's 
 		 * been authorized, and vp is not ".."
+		 *
+		 * make sure we don't try to enter the name back into
+		 * the cache if this vp is purged before we get to that
+		 * check since we won't have serialized behind whatever
+		 * activity is occurring in the FS that caused the purge
 		 */
+	        if (dp != NULLVP)
+		        nc_generation = dp->v_nc_generation - 1;
+
 	        goto returned_from_lookup_path;
 	}
 
@@ -500,10 +616,21 @@ unionlookup:
 	        error = ENOTDIR;
 	        goto lookup_error;
 	}
-	if ( !(dp_authorized || (cnp->cn_flags & DONOTAUTH)) ) {
-	        if ( (error = vnode_authorize(dp, NULL, KAUTH_VNODE_SEARCH, ctx)) )
-		        goto lookup_error;
+	if ( (cnp->cn_flags & DONOTAUTH) != DONOTAUTH ) {
+		if (!dp_authorized) {
+			error = vnode_authorize(dp, NULL, KAUTH_VNODE_SEARCH, ctx);
+			if (error)
+				goto lookup_error;
+		}
+#if CONFIG_MACF
+		error = mac_vnode_check_lookup(ctx, dp, cnp);
+		if (error)
+			goto lookup_error;
+#endif /* CONFIG_MACF */
 	}
+
+        nc_generation = dp->v_nc_generation;
+
 	if ( (error = VNOP_LOOKUP(dp, &ndp->ni_vp, cnp, ctx)) ) {
 lookup_error:
 		if ((error == ENOENT) &&
@@ -594,7 +721,6 @@ returned_from_lookup_path:
 				        vnode_update_identity(dp, ndp->ni_dvp, cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_hash, update_flags);
 			}
 		}
-
 		if ( (cnp->cn_flags & MAKEENTRY) && (dp->v_flag & VNCACHEABLE) && LIST_FIRST(&dp->v_nclinks) == NULL) {
 		        /*
 			 * missing from name cache, but should
@@ -604,12 +730,20 @@ returned_from_lookup_path:
 			 * vnode didn't...
 			 * check to make sure that ni_dvp is valid
 			 * cache_lookup_path may return a NULL
+			 * do a quick check to see if the generation of the
+			 * directory matches our snapshot... this will get
+			 * rechecked behind the name cache lock, but if it
+			 * already fails to match, no need to go any further
 			 */
-		        if (ndp->ni_dvp != NULL)
-			        cache_enter(ndp->ni_dvp, dp, cnp);
+		        if (ndp->ni_dvp != NULLVP && (nc_generation == ndp->ni_dvp->v_nc_generation))
+			        cache_enter_with_gen(ndp->ni_dvp, dp, cnp, nc_generation);
 		}
 	}
 
+	mounted_on_dp = dp;
+	mounted_on_depth = 0;
+	dont_cache_mp = 0;
+	current_mount_generation = mount_generation;
 	/*
 	 * Check to see if the vnode has been mounted on...
 	 * if so find the root of the mounted file system.
@@ -621,15 +755,37 @@ check_mounted_on:
 	        vnode_lock(dp);
 
 		if ((dp->v_type == VDIR) && (mp = dp->v_mountedhere)) {
+			struct uthread *uth = (struct uthread *)get_bsdthread_info(current_thread());
 
 			mp->mnt_crossref++;
 			vnode_unlock(dp);
 
-			if (vfs_busy(mp, 0)) {
+				
+			if (vfs_busy(mp, vbusyflags)) {
 				mount_dropcrossref(mp, dp, 0);
+				if (vbusyflags == LK_NOWAIT) {
+					error = ENOENT;
+					goto bad2;	
+				}
 				goto check_mounted_on;
 			}
+
+			/*
+			 * XXX - if this is the last component of the
+			 * pathname, and it's either not a lookup operation
+			 * or the NOTRIGGER flag is set for the operation,
+			 * set a uthread flag to let VFS_ROOT() for autofs
+			 * know it shouldn't trigger a mount.
+			 */
+			if ((cnp->cn_flags & ISLASTCN) &&
+			    (cnp->cn_nameiop != LOOKUP ||
+			     (cnp->cn_flags & NOTRIGGER))) {
+				uth->uu_notrigger = 1;
+				dont_cache_mp = 1;
+			}
 			error = VFS_ROOT(mp, &tdp, ctx);
+			/* XXX - clear the uthread flag */
+			uth->uu_notrigger = 0;
 			/*
 			 * mount_dropcrossref does a vnode_put
 			 * on dp if the 3rd arg is non-zero
@@ -642,12 +798,34 @@ check_mounted_on:
 				goto bad2;
 			}
 			ndp->ni_vp = dp = tdp;
+			mounted_on_depth++;
 			
 			goto check_mounted_on;
 		} 
 		vnode_unlock(dp);
 	}
 
+#if CONFIG_MACF
+	if (vfs_flags(vnode_mount(dp)) & MNT_MULTILABEL) {
+		error = vnode_label(vnode_mount(dp), NULL, dp, NULL,
+		    VNODE_LABEL_NEEDREF, ctx);
+		if (error)
+		        goto bad2;
+	}
+#endif
+
+	if (mounted_on_depth && !dont_cache_mp) {
+	        mp = mounted_on_dp->v_mountedhere;
+
+		if (mp) {
+		        mount_lock(mp);
+			mp->mnt_realrootvp_vid = dp->v_id;
+			mp->mnt_realrootvp = dp;
+			mp->mnt_generation = current_mount_generation;
+			mount_unlock(mp);
+		}
+	}
+
 	/*
 	 * Check for symbolic link
 	 */
@@ -712,14 +890,80 @@ nextname:
 		}
 		ndp->ni_startdir = ndp->ni_dvp;
 	}
-	if (!wantparent && ndp->ni_dvp)
+	if (!wantparent && ndp->ni_dvp) {
 		vnode_put(ndp->ni_dvp);
+		ndp->ni_dvp = NULLVP;
+	}
 
 	if (cnp->cn_flags & AUDITVNPATH1)
 		AUDIT_ARG(vnpath, dp, ARG_VNODE1);
 	else if (cnp->cn_flags & AUDITVNPATH2)
 		AUDIT_ARG(vnpath, dp, ARG_VNODE2);
 
+#if NAMEDRSRCFORK
+	/*
+	 * Caller wants the resource fork.
+	 */
+	if ((cnp->cn_flags & CN_WANTSRSRCFORK) && (dp != NULLVP)) {
+		vnode_t svp = NULLVP;
+		enum nsoperation nsop;
+
+		if (dp->v_type != VREG) {
+			error = ENOENT;
+			goto bad2;
+		}
+		switch (cnp->cn_nameiop) {
+		case DELETE:
+			nsop = NS_DELETE;
+			break;
+		case CREATE:
+			nsop = NS_CREATE;
+			break;
+		case LOOKUP:
+			/* Make sure our lookup of "/..namedfork/rsrc" is allowed. */
+			if (cnp->cn_flags & CN_ALLOWRSRCFORK) {
+				nsop = NS_OPEN;
+			} else {
+				error = EPERM;
+				goto bad2;
+			}
+			break;
+		default:
+			error = EPERM;
+			goto bad2;
+		}
+		/* Ask the file system for the resource fork. */
+		error = vnode_getnamedstream(dp, &svp, XATTR_RESOURCEFORK_NAME, nsop, 0, ctx);
+
+		/* During a create, it OK for stream vnode to be missing. */
+		if (error == ENOATTR || error == ENOENT) {
+			error = (nsop == NS_CREATE) ? 0 : ENOENT;
+		}		
+		if (error) {
+			goto bad2;
+		}
+		/* The "parent" of the stream is the file. */
+		if (wantparent) {
+			if (ndp->ni_dvp) {
+				if (ndp->ni_cnd.cn_flags & FSNODELOCKHELD) {
+					ndp->ni_cnd.cn_flags &= ~FSNODELOCKHELD;
+					unlock_fsnode(ndp->ni_dvp, NULL);
+				}	
+				vnode_put(ndp->ni_dvp);
+			}
+			ndp->ni_dvp = dp;
+		} else {
+			vnode_put(dp);
+		}
+		ndp->ni_vp = dp = svp;  /* on create this may be null */
+
+		/* Restore the truncated pathname buffer (for audits). */
+		if (ndp->ni_pathlen == 1 && ndp->ni_next[0] == '\0') {
+			ndp->ni_next[0] = '/';
+		}
+		cnp->cn_flags  &= ~MAKEENTRY;
+	}
+#endif
 	if (kdebug_enable)
 	        kdebug_lookup(dp, cnp);
 	return (0);
@@ -798,11 +1042,9 @@ bad:
  *    Used by lookup to re-aquire things.
  */
 int
-relookup(dvp, vpp, cnp)
-	struct vnode *dvp, **vpp;
-	struct componentname *cnp;
+relookup(struct vnode *dvp, struct vnode **vpp, struct componentname *cnp)
 {
-	struct vnode *dp = 0;		/* the directory we are searching */
+	struct vnode *dp = NULL;		/* the directory we are searching */
 	int wantparent;			/* 1 => wantparent or lockparent flag */
 	int rdonly;			/* lookup read-only flag bit */
 	int error = 0;
@@ -927,76 +1169,158 @@ nameidone(struct nameidata *ndp)
 
 #define NUMPARMS 23
 
+/*
+ * Log (part of) a pathname using the KERNEL_DEBUG_CONSTANT mechanism, as used
+ * by fs_usage.  The path up to and including the current component name are
+ * logged.  Up to NUMPARMS*4 bytes of pathname will be logged.  If the path
+ * to be logged is longer than that, then the last NUMPARMS*4 bytes are logged.
+ * That is, the truncation removes the leading portion of the path.
+ *
+ * The logging is done via multiple KERNEL_DEBUG_CONSTANT calls.  The first one
+ * is marked with DBG_FUNC_START.  The last one is marked with DBG_FUNC_END
+ * (in addition to DBG_FUNC_START if it is also the first).  There may be
+ * intermediate ones with neither DBG_FUNC_START nor DBG_FUNC_END.
+ *
+ * The first KERNEL_DEBUG_CONSTANT passes the vnode pointer and 12 bytes of
+ * pathname.  The remaining KERNEL_DEBUG_CONSTANT calls add 16 bytes of pathname
+ * each.  The minimum number of KERNEL_DEBUG_CONSTANT calls required to pass
+ * the path are used.  Any excess padding in the final KERNEL_DEBUG_CONSTANT
+ * (because not all of the 12 or 16 bytes are needed for the remainder of the
+ * path) is set to zero bytes, or '>' if there is more path beyond the
+ * current component name (usually because an intermediate component was not
+ * found).
+ *
+ * NOTE: If the path length is greater than NUMPARMS*4, or is not of the form
+ * 12+N*16, there will be no padding.
+ *
+ * TODO: If there is more path beyond the current component name, should we
+ * force some padding?  For example, a lookup for /foo_bar_baz/spam that
+ * fails because /foo_bar_baz is not found will only log "/foo_bar_baz", with
+ * no '>' padding.  But /foo_bar/spam would log "/foo_bar>>>>".
+ */
 static void
-kdebug_lookup(dp, cnp)
-	struct vnode *dp;
-	struct componentname *cnp;
+kdebug_lookup(struct vnode *dp, struct componentname *cnp)
 {
-	register unsigned int i, n, code;
-	register int dbg_namelen;
-	register int save_dbg_namelen;
-	register char *dbg_nameptr;
+	unsigned int i;
+	int code;
+	int dbg_namelen;
+	char *dbg_nameptr;
 	long dbg_parms[NUMPARMS];
-	char dbg_buf[4];
-	static char *dbg_filler = ">>>>";
 
 	/* Collect the pathname for tracing */
 	dbg_namelen = (cnp->cn_nameptr - cnp->cn_pnbuf) + cnp->cn_namelen;
 	dbg_nameptr = cnp->cn_nameptr + cnp->cn_namelen;
 
-	if (dbg_namelen > sizeof(dbg_parms))
-	    dbg_namelen = sizeof(dbg_parms);
+	if (dbg_namelen > (int)sizeof(dbg_parms))
+		dbg_namelen = sizeof(dbg_parms);
 	dbg_nameptr -= dbg_namelen;
-	save_dbg_namelen = dbg_namelen;
-
-	i = 0;
-
-	while (dbg_namelen > 0) {
-	    if (dbg_namelen >= 4) {
-	        dbg_parms[i++] = *(long *)dbg_nameptr;
-		dbg_nameptr += sizeof(long);
-		dbg_namelen -= sizeof(long);
-	    } else {
-	        for (n = 0; n < dbg_namelen; n++)
-		    dbg_buf[n] = *dbg_nameptr++;
-		while (n <= 3) {
-		    if (*dbg_nameptr)
-		        dbg_buf[n++] = '>';
-		    else
-		        dbg_buf[n++] = 0;
-		}
-		dbg_parms[i++] = *(long *)&dbg_buf[0];
-
-		break;
-	    }
-	}
-	while (i < NUMPARMS) {
-	    if (*dbg_nameptr)
-	        dbg_parms[i++] = *(long *)dbg_filler;
-	    else
-	        dbg_parms[i++] = 0;
+	
+	/* Copy the (possibly truncated) path itself */
+	memcpy(dbg_parms, dbg_nameptr, dbg_namelen);
+	
+	/* Pad with '\0' or '>' */
+	if (dbg_namelen < (int)sizeof(dbg_parms)) {
+		memset((char *)dbg_parms + dbg_namelen,
+		       *(cnp->cn_nameptr + cnp->cn_namelen) ? '>' : 0,
+		       sizeof(dbg_parms) - dbg_namelen);
 	}
-	dbg_namelen = save_dbg_namelen - 12;
-
+	
 	/*
 	 * In the event that we collect multiple, consecutive pathname
-	 * entries, we must mark the start of the path's string and the end
+	 * entries, we must mark the start of the path's string and the end.
 	 */
 	code = (FSDBG_CODE(DBG_FSRW,36)) | DBG_FUNC_START;
 
-	if (dbg_namelen <= 0)
-	        code |= DBG_FUNC_END;
+	if (dbg_namelen <= 12)
+		code |= DBG_FUNC_END;
 
 	KERNEL_DEBUG_CONSTANT(code, (unsigned int)dp, dbg_parms[0], dbg_parms[1], dbg_parms[2], 0);
 
 	code &= ~DBG_FUNC_START;
 
-	for (i = 3; dbg_namelen > 0; i += 4) {
+	for (i=3, dbg_namelen -= 12; dbg_namelen > 0; i+=4, dbg_namelen -= 16) {
+		if (dbg_namelen <= 16)
+			code |= DBG_FUNC_END;
 
-	        dbg_namelen -= (4 * sizeof(long));
-	        if (dbg_namelen <= 0)
-		        code |= DBG_FUNC_END;
+		KERNEL_DEBUG_CONSTANT(code, dbg_parms[i], dbg_parms[i+1], dbg_parms[i+2], dbg_parms[i+3], 0);
+	}
+}
 
-	        KERNEL_DEBUG_CONSTANT(code, dbg_parms[i], dbg_parms[i+1], dbg_parms[i+2], dbg_parms[i+3], 0);
+/*
+ * Obtain the real path from a legacy volfs style path.
+ *
+ * Valid formats of input path:
+ *
+ *	"555/@"
+ *	"555/2"
+ *	"555/123456"
+ *	"555/123456/foobar"
+ *
+ * Where:
+ *	555 represents the volfs file system id
+ *	'@' and '2' are aliases to the root of a file system
+ *	123456 represents a file id
+ *	"foobar" represents a file name
+ */
+#if CONFIG_VOLFS
+static int
+vfs_getrealpath(const char * path, char * realpath, size_t bufsize, vfs_context_t ctx)
+{
+	vnode_t vp;
+	struct mount *mp = NULL;
+	char  *str;
+	char ch;
+	unsigned long  id;
+	ino64_t ino;
+	int error;
+	int length;
+
+	/* Get file system id and move str to next component. */
+	id = strtoul(path, &str, 10);
+	if (id == 0 || str[0] != '/') {
+		return (EINVAL);
+	}
+	while (*str == '/') {
+		str++;
 	}
+	ch = *str;
+
+	mp = mount_lookupby_volfsid(id, 1);
+	if (mp == NULL) {
+		return (EINVAL);  /* unexpected failure */
+	}
+	/* Check for an alias to a file system root. */
+	if (ch == '@' && str[1] == '\0') {
+		ino = 2;
+		str++;
+	} else {
+		/* Get file id and move str to next component. */
+	    ino = strtouq(str, &str, 10);
+	}
+
+	/* Get the target vnode. */
+	if (ino == 2) {
+		error = VFS_ROOT(mp, &vp, ctx);
+	} else {
+		error = VFS_VGET(mp, ino, &vp, ctx);
+	}
+	vfs_unbusy(mp);
+	if (error) {
+		goto out;
+	}
+	realpath[0] = '\0';
+
+	/* Get the absolute path to this vnode. */
+	error = build_path(vp, realpath, bufsize, &length, 0, ctx);
+	vnode_put(vp);
+
+	if (error == 0 && *str != '\0') {
+		int attempt = strlcat(realpath, str, MAXPATHLEN);
+		if (attempt > MAXPATHLEN){
+			error = ENAMETOOLONG;
+		}
+	}
+out:
+	return (error);
 }
+#endif
diff --git a/bsd/vfs/vfs_quota.c b/bsd/vfs/vfs_quota.c
index 93c5d052b..91ffe8bce 100644
--- a/bsd/vfs/vfs_quota.c
+++ b/bsd/vfs/vfs_quota.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1990, 1993, 1995
@@ -124,16 +130,11 @@ static void qf_rele(struct quotafile *);
 
 
 /*
- * Initialize the quota system.
+ * Initialize locks for the quota system.
  */
 void
-dqinit()
+dqinit(void)
 {
-
-	dqhashtbl = hashinit(desiredvnodes, M_DQUOT, &dqhash);
-	TAILQ_INIT(&dqfreelist);
-	TAILQ_INIT(&dqdirtylist);
-
 	/*
 	 * Allocate quota list lock group attribute and group
 	 */
@@ -163,6 +164,32 @@ dqinit()
 	qf_lck_attr = lck_attr_alloc_init();
 }
 
+/*
+ * Report whether dqhashinit has been run.
+ */
+int
+dqisinitialized(void)
+{
+	return (dqhashtbl != NULL);
+}
+
+/*
+ * Initialize hash table for dquot structures.
+ */
+void
+dqhashinit(void)
+{
+	dq_list_lock();
+	if (dqisinitialized())
+		goto out;
+
+	TAILQ_INIT(&dqfreelist);
+	TAILQ_INIT(&dqdirtylist);
+	dqhashtbl = hashinit(desiredvnodes, M_DQUOT, &dqhash);
+out:
+	dq_list_unlock();
+}
+
 
 static volatile int dq_list_lock_cnt = 0;
 
@@ -198,7 +225,7 @@ dq_lock_internal(struct dquot *dq)
 {
         while (dq->dq_lflags & DQ_LLOCK) {
 	        dq->dq_lflags |= DQ_LWANT;
-	        msleep(&dq->dq_lflags, quota_list_mtx_lock, PVFS, "dq_lock_internal", 0);
+	        msleep(&dq->dq_lflags, quota_list_mtx_lock, PVFS, "dq_lock_internal", NULL);
 	}
 	dq->dq_lflags |= DQ_LLOCK;
 }
@@ -256,7 +283,7 @@ qf_get(struct quotafile *qfp, int type)
 			}
 			if ( (qfp->qf_qflags & QTF_CLOSING) ) {
 		                qfp->qf_qflags |= QTF_WANTED;
-				msleep(&qfp->qf_qflags, quota_list_mtx_lock, PVFS, "qf_get", 0);
+				msleep(&qfp->qf_qflags, quota_list_mtx_lock, PVFS, "qf_get", NULL);
 			}
 		}
 		if (qfp->qf_vp != NULLVP)
@@ -274,7 +301,7 @@ qf_get(struct quotafile *qfp, int type)
 
 		while ( (qfp->qf_qflags & QTF_OPENING) || qfp->qf_refcnt ) {
 		        qfp->qf_qflags |= QTF_WANTED;
-			msleep(&qfp->qf_qflags, quota_list_mtx_lock, PVFS, "qf_get", 0);
+			msleep(&qfp->qf_qflags, quota_list_mtx_lock, PVFS, "qf_get", NULL);
 		}
 		if (qfp->qf_vp == NULLVP) {
 		        qfp->qf_qflags &= ~QTF_CLOSING;
@@ -382,9 +409,7 @@ dqfileinit(struct quotafile *qfp)
  * must be called with the quota file lock held
  */
 int
-dqfileopen(qfp, type)
-	struct quotafile *qfp;
-	int type;
+dqfileopen(struct quotafile *qfp, int type)
 {
 	struct dqfilehdr header;
 	struct vfs_context context;
@@ -393,7 +418,7 @@ dqfileopen(qfp, type)
 	int error = 0;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 	context.vc_ucred = qfp->qf_cred;
 	
 	/* Obtain the file size */
@@ -452,7 +477,7 @@ dqfileclose(struct quotafile *qfp, __unused int type)
 								  &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, CAST_USER_ADDR_T(&header), sizeof (header));
 
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 	context.vc_ucred = qfp->qf_cred;
 	
 	if (VNOP_READ(qfp->qf_vp, auio, 0, &context) == 0) {
@@ -469,11 +494,7 @@ dqfileclose(struct quotafile *qfp, __unused int type)
  * reading the information from the file if necessary.
  */
 int
-dqget(id, qfp, type, dqp)
-	u_long id;
-	struct quotafile *qfp;
-	register int type;
-	struct dquot **dqp;
+dqget(u_long id, struct quotafile *qfp, int type, struct dquot **dqp)
 {
 	struct dquot *dq;
 	struct dquot *ndq = NULL;
@@ -483,6 +504,11 @@ dqget(id, qfp, type, dqp)
 	int error = 0;
 	int listlockval = 0;
 
+	if (!dqisinitialized()) {
+		*dqp = NODQUOT;
+		return (EINVAL);
+	}
+
 	if ( id == 0 || qfp->qf_vp == NULLVP ) {
 		*dqp = NODQUOT;
 		return (EINVAL);
@@ -539,13 +565,6 @@ relookup:
 				TAILQ_REMOVE(&dqdirtylist, dq, dq_freelist);
 			else
 				TAILQ_REMOVE(&dqfreelist, dq, dq_freelist);
-		} else if (dq->dq_cnt == 0) {
-			/* We've overflowed */
-			--dq->dq_cnt;
-			dq_unlock_internal(dq);
-			dq_list_unlock();
-			*dqp = NODQUOT;
-			return (EINVAL);
 		}
 		dq_unlock_internal(dq);
 
@@ -765,11 +784,7 @@ relookup:
  * one is inserted.  The actual hash table index is returned.
  */
 static int
-dqlookup(qfp, id, dqb, index)
-	struct quotafile *qfp;
-	u_long id;
-	struct	dqblk *dqb;
-	u_int32_t *index;
+dqlookup(struct quotafile *qfp, u_long id, struct dqblk *dqb, uint32_t *index)
 {
 	struct vnode *dqvp;
 	struct vfs_context context;
@@ -784,7 +799,7 @@ dqlookup(qfp, id, dqb, index)
 
 	dqvp = qfp->qf_vp;
 
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 	context.vc_ucred = qfp->qf_cred;
 
 	mask = qfp->qf_maxentries - 1;
@@ -799,11 +814,11 @@ dqlookup(qfp, id, dqb, index)
 		uio_addiov(auio, CAST_USER_ADDR_T(dqb), sizeof (struct dqblk));
 		error = VNOP_READ(dqvp, auio, 0, &context);
 		if (error) {
-			printf("dqlookup: error %d looking up id %d at index %d\n", error, id, i);
+			printf("dqlookup: error %d looking up id %lu at index %d\n", error, id, i);
 			break;
 		} else if (uio_resid(auio)) {
 			error = EIO;
-			printf("dqlookup: error looking up id %d at index %d\n", id, i);
+			printf("dqlookup: error looking up id %lu at index %d\n", id, i);
 			break;
 		}
 		/*
@@ -880,7 +895,7 @@ dqrele(struct dquot *dq)
  * Release a reference to a dquot but don't do any I/O.
  */
 void
-dqreclaim(register struct dquot *dq)
+dqreclaim(struct dquot *dq)
 {
 
 	if (dq == NODQUOT)
@@ -907,25 +922,17 @@ dqreclaim(register struct dquot *dq)
  * Update a quota file's orphaned disk quotas.
  */
 void
-dqsync_orphans(qfp)
-	struct quotafile *qfp;
+dqsync_orphans(struct quotafile *qfp)
 {
 	struct dquot *dq;
-	int listlockval = 0;
-
+	
 	dq_list_lock();
   loop:
-	listlockval = dq_list_lock_val();
-
 	TAILQ_FOREACH(dq, &dqdirtylist, dq_freelist) {
 		if (dq->dq_qfile != qfp)
 		        continue;
 
 		dq_lock_internal(dq);
-		if (dq_list_lock_changed(listlockval)) {
-			dq_unlock_internal(dq);
-			goto loop;
-		}
 
 		if (dq->dq_qfile != qfp) {
 		        /*
@@ -992,7 +999,6 @@ dqsync(struct dquot *dq)
 int
 dqsync_locked(struct dquot *dq)
 {
-	struct proc *p = current_proc();		/* XXX */
 	struct vfs_context context;
 	struct vnode *dqvp;
 	struct dqblk dqb, *dqblkp;
@@ -1013,7 +1019,7 @@ dqsync_locked(struct dquot *dq)
 								  UIO_WRITE, &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, CAST_USER_ADDR_T(&dqb), sizeof (struct dqblk));
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();	/* XXX */
 	context.vc_ucred = dq->dq_qfile->qf_cred;
 
 	dqblkp = &dq->dq_dqb;
@@ -1043,12 +1049,14 @@ dqsync_locked(struct dquot *dq)
  * Flush all entries from the cache for a particular vnode.
  */
 void
-dqflush(vp)
-	register struct vnode *vp;
+dqflush(struct vnode *vp)
 {
-	register struct dquot *dq, *nextdq;
+	struct dquot *dq, *nextdq;
 	struct dqhash *dqh;
 
+	if (!dqisinitialized())
+		return;
+
 	/*
 	 * Move all dquot's that used to refer to this quota
 	 * file off their hash chains (they will eventually
diff --git a/bsd/vfs/vfs_subr.c b/bsd/vfs/vfs_subr.c
index 825f8de7c..be1ba3291 100644
--- a/bsd/vfs/vfs_subr.c
+++ b/bsd/vfs/vfs_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -59,13 +65,17 @@
  *
  *	@(#)vfs_subr.c	8.31 (Berkeley) 5/26/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 /*
  * External virtual filesystem routines
  */
 
-#undef	DIAGNOSTIC
-#define DIAGNOSTIC 1
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -74,6 +84,7 @@
 #include <sys/mount_internal.h>
 #include <sys/time.h>
 #include <sys/lock.h>
+#include <sys/vnode.h>
 #include <sys/vnode_internal.h>
 #include <sys/stat.h>
 #include <sys/namei.h>
@@ -81,6 +92,8 @@
 #include <sys/buf_internal.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
+#include <sys/uio_internal.h>
+#include <sys/uio.h>
 #include <sys/domain.h>
 #include <sys/mbuf.h>
 #include <sys/syslog.h>
@@ -105,6 +118,17 @@
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
 
+#include <kern/kalloc.h>	/* kalloc()/kfree() */
+#include <kern/clock.h>		/* delay_for_interval() */
+#include <libkern/OSAtomic.h>	/* OSAddAtomic() */
+
+
+#include <vm/vm_protos.h>	/* vnode_pager_vrele() */
+
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
 extern lck_grp_t *vnode_lck_grp;
 extern lck_attr_t *vnode_lck_attr;
 
@@ -120,19 +144,32 @@ int	vttoif_tab[9] = {
 	S_IFSOCK, S_IFIFO, S_IFMT,
 };
 
-extern int ubc_isinuse_locked(vnode_t, int, int);
+/* XXX next protptype should be from <nfs/nfs.h> */
+extern int       nfs_vinvalbuf(vnode_t, int, vfs_context_t, int);
+
+/* XXX next prototytype should be from libsa/stdlib.h> but conflicts libkern */
+__private_extern__ void qsort(
+    void * array,
+    size_t nmembers,
+    size_t member_size,
+    int (*)(const void *, const void *));
+
 extern kern_return_t adjust_vm_object_cache(vm_size_t oval, vm_size_t nval);
+__private_extern__ void vntblinit(void);
+__private_extern__ kern_return_t reset_vmobjectcache(unsigned int val1,
+			unsigned int val2);
+__private_extern__ int unlink1(vfs_context_t, struct nameidata *, int);
 
 static void vnode_list_add(vnode_t);
 static void vnode_list_remove(vnode_t);
 
 static errno_t vnode_drain(vnode_t);
-static void vgone(vnode_t);
-static void vclean(vnode_t vp, int flag, proc_t p);
-static void vnode_reclaim_internal(vnode_t, int, int);
+static void vgone(vnode_t, int flags);
+static void vclean(vnode_t vp, int flag);
+static void vnode_reclaim_internal(vnode_t, int, int, int);
 
-static void vnode_dropiocount (vnode_t, int);
-static errno_t vnode_getiocount(vnode_t vp, int locked, int vid, int vflags);
+static void vnode_dropiocount (vnode_t);
+static errno_t vnode_getiocount(vnode_t vp, int vid, int vflags);
 static int vget_internal(vnode_t, int, int);
 
 static vnode_t checkalias(vnode_t vp, dev_t nvp_rdev);
@@ -140,7 +177,6 @@ static int  vnode_reload(vnode_t);
 static int  vnode_isinuse_locked(vnode_t, int, int);
 
 static void insmntque(vnode_t vp, mount_t mp);
-mount_t mount_list_lookupby_fsid(fsid_t *, int, int);
 static int mount_getvfscnt(void);
 static int mount_fillfsids(fsid_t *, int );
 static void vnode_iterate_setup(mount_t);
@@ -149,8 +185,19 @@ static int vnode_iterate_prepare(mount_t);
 static int vnode_iterate_reloadq(mount_t);
 static void vnode_iterate_clear(mount_t);
 
+errno_t rmdir_remove_orphaned_appleDouble(vnode_t, vfs_context_t, int *); 
+
 TAILQ_HEAD(freelst, vnode) vnode_free_list;	/* vnode free list */
-TAILQ_HEAD(inactivelst, vnode) vnode_inactive_list;	/* vnode inactive list */
+TAILQ_HEAD(deadlst, vnode) vnode_dead_list;	/* vnode dead list */
+
+TAILQ_HEAD(ragelst, vnode) vnode_rage_list;	/* vnode rapid age list */
+struct timeval rage_tv;
+int	rage_limit = 0;
+int	ragevnodes = 0;			   
+
+#define RAGE_LIMIT_MIN	100
+#define RAGE_TIME_LIMIT	5
+
 struct mntlist mountlist;			/* mounted filesystem list */
 static int nummounts = 0;
 
@@ -158,18 +205,8 @@ static int nummounts = 0;
 #define VLISTCHECK(fun, vp, list)	\
 	if ((vp)->v_freelist.tqe_prev == (struct vnode **)0xdeadb) \
 		panic("%s: %s vnode not on %slist", (fun), (list), (list));
-
-#define VINACTIVECHECK(fun, vp, expected)	\
-	do {	\
-		int __is_inactive = ISSET((vp)->v_flag, VUINACTIVE);	\
-		if (__is_inactive ^ expected)	\
-			panic("%s: %sinactive vnode, expected %s", (fun),	\
-				__is_inactive? "" : "not ",	\
-				expected? "inactive": "not inactive"); \
-	} while(0)
 #else
 #define VLISTCHECK(fun, vp, list)
-#define VINACTIVECHECK(fun, vp, expected)
 #endif /* DIAGNOSTIC */
 
 #define VLISTNONE(vp)	\
@@ -190,32 +227,37 @@ static int nummounts = 0;
 		freevnodes--;	\
 	} while(0)
 
-/* remove a vnode from inactive vnode list */
-#define VREMINACTIVE(fun, vp)	\
+
+
+/* remove a vnode from dead vnode list */
+#define VREMDEAD(fun, vp)	\
 	do {	\
-		VLISTCHECK((fun), (vp), "inactive"); \
-		VINACTIVECHECK((fun), (vp), VUINACTIVE); \
-		TAILQ_REMOVE(&vnode_inactive_list, (vp), v_freelist); \
-		CLR((vp)->v_flag, VUINACTIVE); \
+		VLISTCHECK((fun), (vp), "dead");	\
+		TAILQ_REMOVE(&vnode_dead_list, (vp), v_freelist);	\
 		VLISTNONE((vp));	\
-		inactivevnodes--;	\
+		vp->v_listflag &= ~VLIST_DEAD;	\
+		deadvnodes--;	\
+	} while(0)
+
+
+/* remove a vnode from rage vnode list */
+#define VREMRAGE(fun, vp)	\
+	do {	\
+	        if ( !(vp->v_listflag & VLIST_RAGE))			\
+		        panic("VREMRAGE: vp not on rage list");		\
+		VLISTCHECK((fun), (vp), "rage");			\
+		TAILQ_REMOVE(&vnode_rage_list, (vp), v_freelist);	\
+		VLISTNONE((vp));		\
+		vp->v_listflag &= ~VLIST_RAGE;	\
+		ragevnodes--;			\
 	} while(0)
 
-/*
- * Have to declare first two locks as actual data even if !MACH_SLOCKS, since
- * a pointers to them get passed around.
- */
-void * mntvnode_slock;
-void * mntid_slock;
-void * spechash_slock;
 
 /*
- * vnodetarget is the amount of vnodes we expect to get back 
- * from the the inactive vnode list and VM object cache.
- * As vnreclaim() is a mainly cpu bound operation for faster 
- * processers this number could be higher.
- * Having this number too high introduces longer delays in 
- * the execution of new_vnode().
+ * vnodetarget hasn't been used in a long time, but
+ * it was exported for some reason... I'm leaving in
+ * place for now...  it should be deprecated out of the
+ * exports and removed eventually.
  */
 unsigned long vnodetarget;		/* target for vnreclaim() */
 #define VNODE_FREE_TARGET	20	/* Default value for vnodetarget */
@@ -226,21 +268,7 @@ unsigned long vnodetarget;		/* target for vnreclaim() */
  * cache. Having too few vnodes on the free list causes serious disk
  * thrashing as we cycle through them.
  */
-#define VNODE_FREE_MIN		300	/* freelist should have at least these many */
-
-/*
- * We need to get vnodes back from the VM object cache when a certain #
- * of vnodes are reused from the freelist. This is essential for the
- * caching to be effective in the namecache and the buffer cache [for the
- * metadata].
- */
-#define	VNODE_TOOMANY_REUSED	(VNODE_FREE_MIN/4)
-
-/*
- * If we have enough vnodes on the freelist we do not want to reclaim
- * the vnodes from the VM object cache.
- */
-#define VNODE_FREE_ENOUGH	(VNODE_FREE_MIN + (VNODE_FREE_MIN/2))
+#define VNODE_FREE_MIN		CONFIG_VNODE_FREE_MIN	/* freelist should have at least this many */
 
 /*
  * Initialize the vnode management data structures.
@@ -249,12 +277,19 @@ __private_extern__ void
 vntblinit(void)
 {
 	TAILQ_INIT(&vnode_free_list);
-	TAILQ_INIT(&vnode_inactive_list);
+	TAILQ_INIT(&vnode_rage_list);
+	TAILQ_INIT(&vnode_dead_list);
 	TAILQ_INIT(&mountlist);
 
 	if (!vnodetarget)
 		vnodetarget = VNODE_FREE_TARGET;
 
+	microuptime(&rage_tv);
+	rage_limit = desiredvnodes / 100;
+
+	if (rage_limit < RAGE_LIMIT_MIN)
+	        rage_limit = RAGE_LIMIT_MIN;
+	
 	/*
 	 * Scale the vm_object_cache to accomodate the vnodes 
 	 * we want to cache
@@ -280,7 +315,7 @@ reset_vmobjectcache(unsigned int val1, unsigned int val2)
 
 /* the timeout is in 10 msecs */
 int
-vnode_waitforwrites(vnode_t vp, int output_target, int slpflag, int slptimeout, char *msg) {
+vnode_waitforwrites(vnode_t vp, int output_target, int slpflag, int slptimeout, const char *msg) {
         int error = 0;
 	struct timespec ts;
 
@@ -297,6 +332,7 @@ vnode_waitforwrites(vnode_t vp, int output_target, int slpflag, int slptimeout,
 			        vp->v_flag |= VTHROTTLED;
 			else
 			        vp->v_flag |= VBWAIT;
+
 			ts.tv_sec = (slptimeout/100);
 			ts.tv_nsec = (slptimeout % 1000)  * 10 * NSEC_PER_USEC * 1000 ;
 			error = msleep((caddr_t)&vp->v_numoutput, &vp->v_lock, (slpflag | (PRIBIO + 1)), msg, &ts);
@@ -320,27 +356,29 @@ void
 vnode_writedone(vnode_t vp)
 {
 	if (vp) {
-	        int need_wakeup = 0;
-	  
 	        OSAddAtomic(-1, &vp->v_numoutput);
 
-		vnode_lock(vp);
+		if (vp->v_numoutput <= 1) {
+		        int need_wakeup = 0;
 
-		if (vp->v_numoutput < 0)
-			panic("vnode_writedone: numoutput < 0");
+		        vnode_lock_spin(vp);
 
-		if ((vp->v_flag & VTHROTTLED) && (vp->v_numoutput < (VNODE_ASYNC_THROTTLE / 3))) {
-			vp->v_flag &= ~VTHROTTLED;
-			need_wakeup = 1;
-		}
-		if ((vp->v_flag & VBWAIT) && (vp->v_numoutput == 0)) {
-		        vp->v_flag &= ~VBWAIT;
-			need_wakeup = 1;
-		}
-		vnode_unlock(vp);
+			if (vp->v_numoutput < 0)
+			        panic("vnode_writedone: numoutput < 0");
+
+			if ((vp->v_flag & VTHROTTLED) && (vp->v_numoutput <= 1)) {
+			        vp->v_flag &= ~VTHROTTLED;
+				need_wakeup = 1;
+			}
+			if ((vp->v_flag & VBWAIT) && (vp->v_numoutput == 0)) {
+			        vp->v_flag &= ~VBWAIT;
+				need_wakeup = 1;
+			}
+			vnode_unlock(vp);
 		
-		if (need_wakeup)
-		        wakeup((caddr_t)&vp->v_numoutput);
+			if (need_wakeup)
+			        wakeup((caddr_t)&vp->v_numoutput);
+		}
 	}
 }
 
@@ -392,7 +430,7 @@ vnode_iterate_setup(mount_t mp)
 {
 	while (mp->mnt_lflag & MNT_LITER) {
 		mp->mnt_lflag |= MNT_LITERWAIT;
-		msleep((caddr_t)mp, &mp->mnt_mlock, PVFS, "vnode_iterate_setup", 0);	
+		msleep((caddr_t)mp, &mp->mnt_mlock, PVFS, "vnode_iterate_setup", NULL);	
 	}
 
 	mp->mnt_lflag |= MNT_LITER;
@@ -405,8 +443,11 @@ vnode_umount_preflight(mount_t mp, vnode_t skipvp, int flags)
 	vnode_t vp;
 
 	TAILQ_FOREACH(vp, &mp->mnt_vnodelist, v_mntvnodes) {
-	        if (vp->v_type == VDIR)
-		        continue;
+		/* disable preflight only for udf, a hack to be removed after 4073176 is fixed */
+		if (vp->v_tag == VT_UDF)
+			return 0;
+		if (vp->v_type == VDIR)
+			continue;
 		if (vp == skipvp)
 			continue;
 		if ((flags & SKIPSYSTEM) && ((vp->v_flag & VSYSTEM) ||
@@ -507,11 +548,8 @@ vnode_iterate_clear(mount_t mp)
 
 
 int
-vnode_iterate(mp, flags, callout, arg)
-	mount_t mp;
-	int flags;
-	int (*callout)(struct vnode *, void *);
-	void * arg;
+vnode_iterate(mount_t mp, int flags, int (*callout)(struct vnode *, void *),
+	      void *arg)
 {
 	struct vnode *vp;
 	int vid, retval;
@@ -690,7 +728,7 @@ mount_iterdrain(mount_t mp)
 {
 	mount_list_lock();
 	while (mp->mnt_iterref)
-		msleep((caddr_t)&mp->mnt_iterref, mnt_list_mtx_lock, PVFS, "mount_iterdrain", 0 );
+		msleep((caddr_t)&mp->mnt_iterref, mnt_list_mtx_lock, PVFS, "mount_iterdrain", NULL);
 	/* mount iterations drained */
 	mp->mnt_iterref = -1;
 	mount_list_unlock();
@@ -713,7 +751,7 @@ mount_refdrain(mount_t mp)
 	mp->mnt_lflag |= MNT_LDRAIN;
 
 	while (mp->mnt_count)
-		msleep((caddr_t)&mp->mnt_lflag, &mp->mnt_mlock, PVFS, "mount_drain", 0 );
+		msleep((caddr_t)&mp->mnt_lflag, &mp->mnt_mlock, PVFS, "mount_drain", NULL);
 
 	if (mp->mnt_vnodelist.tqh_first != NULL)
 		 panic("mount_refdrain: dangling vnode"); 
@@ -754,7 +792,7 @@ restart:
 			 * wakeup needs to be done is at the release of the
 			 * exclusive lock at the end of dounmount.
 			 */
-			msleep((caddr_t)mp, &mp->mnt_mlock, (PVFS | PDROP), "vfsbusy", 0 );
+			msleep((caddr_t)mp, &mp->mnt_mlock, (PVFS | PDROP), "vfsbusy", NULL);
 			return (ENOENT);
 		}
 		mount_unlock(mp);
@@ -796,6 +834,10 @@ vfs_rootmountfailed(mount_t mp) {
 
 	mount_lock_destroy(mp);
 
+#if CONFIG_MACF
+	mac_mount_label_destroy(mp);
+#endif
+
 	FREE_ZONE(mp, sizeof(struct mount), M_MOUNT);
 }
 
@@ -819,6 +861,10 @@ vfs_rootmountalloc_internal(struct vfstable *vfsp, const char *devname)
 	mp->mnt_maxsegreadsize = mp->mnt_maxreadcnt;
 	mp->mnt_maxsegwritesize = mp->mnt_maxwritecnt;
 	mp->mnt_devblocksize = DEV_BSIZE;
+	mp->mnt_alignmentmask = PAGE_MASK;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
 
 	mount_lock_init(mp);
 	(void)vfs_busy(mp, LK_NOWAIT);
@@ -840,8 +886,13 @@ vfs_rootmountalloc_internal(struct vfstable *vfsp, const char *devname)
 
 	strncpy(mp->mnt_vfsstat.f_fstypename, vfsp->vfc_name, MFSTYPENAMELEN);
 	mp->mnt_vfsstat.f_mntonname[0] = '/';
-	(void) copystr((char *)devname, mp->mnt_vfsstat.f_mntfromname, MAXPATHLEN - 1, 0);
+	/* XXX const poisoning layering violation */
+	(void) copystr((const void *)devname, mp->mnt_vfsstat.f_mntfromname, MAXPATHLEN - 1, NULL);
 
+#if CONFIG_MACF
+	mac_mount_label_init(mp);
+	mac_mount_label_associate(vfs_context_kernel(), mp);
+#endif
 	return (mp);
 }
 
@@ -851,7 +902,8 @@ vfs_rootmountalloc(const char *fstypename, const char *devname, mount_t *mpp)
         struct vfstable *vfsp;
 
 	for (vfsp = vfsconf; vfsp; vfsp = vfsp->vfc_next)
-	        if (!strcmp(vfsp->vfc_name, fstypename))
+	        if (!strncmp(vfsp->vfc_name, fstypename,
+			     sizeof(vfsp->vfc_name)))
 		        break;
         if (vfsp == NULL)
 	        return (ENODEV);
@@ -874,26 +926,34 @@ vfs_rootmountalloc(const char *fstypename, const char *devname, mount_t *mpp)
 extern int (*mountroot)(void);
 
 int
-vfs_mountroot()
+vfs_mountroot(void)
 {
+#if CONFIG_MACF
+	struct vnode *vp;
+#endif
 	struct vfstable *vfsp;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_kernel();
+	struct vfs_attr	vfsattr;
 	int	error;
 	mount_t mp;
+	vnode_t	bdevvp_rootvp;
 
 	if (mountroot != NULL) {
-	        /*
+		/*
 		 * used for netboot which follows a different set of rules
 		 */
-	        error = (*mountroot)();
+		error = (*mountroot)();
 		return (error);
 	}
 	if ((error = bdevvp(rootdev, &rootvp))) {
-	        printf("vfs_mountroot: can't setup bdevvp\n");
+		printf("vfs_mountroot: can't setup bdevvp\n");
 		return (error);
 	}
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
+	/*
+	 * 4951998 - code we call in vfc_mountroot may replace rootvp 
+	 * so keep a local copy for some house keeping.
+	 */
+	bdevvp_rootvp = rootvp;
 
 	for (vfsp = vfsconf; vfsp; vfsp = vfsp->vfc_next) {
 		if (vfsp->vfc_mountroot == NULL)
@@ -902,10 +962,24 @@ vfs_mountroot()
 		mp = vfs_rootmountalloc_internal(vfsp, "root_device");
 		mp->mnt_devvp = rootvp;
 
-		if ((error = (*vfsp->vfc_mountroot)(mp, rootvp, &context)) == 0) {
-		        mp->mnt_devvp->v_specflags |= SI_MOUNTEDON;
+		if ((error = (*vfsp->vfc_mountroot)(mp, rootvp, ctx)) == 0) {
+			if ( bdevvp_rootvp != rootvp ) {
+				/*
+				 * rootvp changed...
+				 *   bump the iocount and fix up mnt_devvp for the
+				 *   new rootvp (it will already have a usecount taken)...
+				 *   drop the iocount and the usecount on the orignal
+				 *   since we are no longer going to use it...
+				 */
+				vnode_getwithref(rootvp);
+				mp->mnt_devvp = rootvp;
+
+			        vnode_rele(bdevvp_rootvp);
+			        vnode_put(bdevvp_rootvp);
+			}
+			mp->mnt_devvp->v_specflags |= SI_MOUNTEDON;
 
-		        vfs_unbusy(mp);
+			vfs_unbusy(mp);
 
 			mount_list_add(mp);
 
@@ -916,15 +990,76 @@ vfs_mountroot()
 			 *   defaults will have been set, so no reason to bail or care
 			 */
 			vfs_init_io_attributes(rootvp, mp);
+
+			/*
+			 * Shadow the VFC_VFSNATIVEXATTR flag to MNTK_EXTENDED_ATTRS.
+			 */
+			if (mp->mnt_vtable->vfc_vfsflags & VFC_VFSNATIVEXATTR) {
+				mp->mnt_kern_flag |= MNTK_EXTENDED_ATTRS;
+			}
+			if (mp->mnt_vtable->vfc_vfsflags & VFC_VFSPREFLIGHT) {
+				mp->mnt_kern_flag |= MNTK_UNMOUNT_PREFLIGHT;
+			}
+
+			/*
+			 * Probe root file system for additional features.
+			 */
+			(void)VFS_START(mp, 0, ctx);
+
+			VFSATTR_INIT(&vfsattr);
+			VFSATTR_WANTED(&vfsattr, f_capabilities);
+			if (vfs_getattr(mp, &vfsattr, ctx) == 0 && 
+			    VFSATTR_IS_SUPPORTED(&vfsattr, f_capabilities)) {
+				if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR) &&
+				    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR)) {
+					mp->mnt_kern_flag |= MNTK_EXTENDED_ATTRS;
+				}
+#if NAMEDSTREAMS
+				if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_NAMEDSTREAMS) &&
+				    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_NAMEDSTREAMS)) {
+					mp->mnt_kern_flag |= MNTK_NAMED_STREAMS;
+				}
+#endif
+				if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_PATH_FROM_ID) &&
+				    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_PATH_FROM_ID)) {
+					mp->mnt_kern_flag |= MNTK_PATH_FROM_ID;
+				}
+			}
+
 			/*
 			 * get rid of iocount reference returned
-			 * by bdevvp... it will have also taken
+			 * by bdevvp (or picked up by us on the substitued
+			 * rootvp)... it (or we) will have also taken
 			 * a usecount reference which we want to keep
 			 */
 			vnode_put(rootvp);
 
+#if CONFIG_MACF
+			if ((vfs_flags(mp) & MNT_MULTILABEL) == 0)
+				return (0);
+
+			error = VFS_ROOT(mp, &vp, ctx);
+			if (error) {
+				printf("%s() VFS_ROOT() returned %d\n",
+				    __func__, error);
+				dounmount(mp, MNT_FORCE, 0, ctx);
+				goto fail;
+			}
+
+			/* VFS_ROOT provides reference so flags = 0 */
+			error = vnode_label(mp, NULL, vp, NULL, 0, ctx);
+			if (error) {
+				printf("%s() vnode_label() returned %d\n",
+				    __func__, error);
+				dounmount(mp, MNT_FORCE, 0, ctx);
+				goto fail;
+			}
+#endif
 			return (0);
 		}
+#if CONFIG_MACF
+fail:
+#endif
 		vfs_rootmountfailed(mp);
 		
 		if (error != EINVAL)
@@ -939,28 +1074,27 @@ vfs_mountroot()
 extern mount_t vfs_getvfs_locked(fsid_t *);
 
 struct mount *
-vfs_getvfs(fsid)
-	fsid_t *fsid;
+vfs_getvfs(fsid_t *fsid)
 {
 	return (mount_list_lookupby_fsid(fsid, 0, 0));
 }
 
 struct mount *
-vfs_getvfs_locked(fsid)
-	fsid_t *fsid;
+vfs_getvfs_locked(fsid_t *fsid)
 {
 	return(mount_list_lookupby_fsid(fsid, 1, 0));
 }
 
 struct mount *
-vfs_getvfs_by_mntonname(u_char *path)
+vfs_getvfs_by_mntonname(char *path)
 {
 	mount_t retmp = (mount_t)0;
 	mount_t mp;
 
 	mount_list_lock();
 	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
-		if (!strcmp(mp->mnt_vfsstat.f_mntonname, path)) {
+		if (!strncmp(mp->mnt_vfsstat.f_mntonname, path,
+					sizeof(mp->mnt_vfsstat.f_mntonname))) {
 			retmp = mp;
 			goto out;
 		}
@@ -976,8 +1110,7 @@ u_short mntid_gen = 0;
  * Get a new unique fsid
  */
 void
-vfs_getnewfsid(mp)
-	struct mount *mp;
+vfs_getnewfsid(struct mount *mp)
 {
 
 	fsid_t tfsid;
@@ -1009,8 +1142,7 @@ vfs_getnewfsid(mp)
  * Routines having to do with the management of the vnode table.
  */
 extern int (**dead_vnodeop_p)(void *);
-long numvnodes, freevnodes;
-long inactivevnodes;
+long numvnodes, freevnodes, deadvnodes;
 
 
 /*
@@ -1043,8 +1175,8 @@ insmntque(vnode_t vp, mount_t mp)
 			vp->v_mntvnodes.tqe_next->v_mntvnodes.tqe_prev = vp->v_mntvnodes.tqe_prev;
 			*vp->v_mntvnodes.tqe_prev = vp->v_mntvnodes.tqe_next;
 		}	
-		vp->v_mntvnodes.tqe_next = 0;
-		vp->v_mntvnodes.tqe_prev = 0;
+		vp->v_mntvnodes.tqe_next = NULL;
+		vp->v_mntvnodes.tqe_prev = NULL;
 		mount_unlock(lmp);
 		return;
 	}
@@ -1089,15 +1221,15 @@ bdevvp(dev_t dev, vnode_t *vpp)
 		return (ENODEV);
 	}
 
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 	context.vc_ucred = FSCRED;
 
 	vfsp.vnfs_mp = (struct mount *)0;
 	vfsp.vnfs_vtype = VBLK;
 	vfsp.vnfs_str = "bdevvp";
-	vfsp.vnfs_dvp = 0;
-	vfsp.vnfs_fsnode = 0;
-	vfsp.vnfs_cnp = 0;
+	vfsp.vnfs_dvp = NULL;
+	vfsp.vnfs_fsnode = NULL;
+	vfsp.vnfs_cnp = NULL;
 	vfsp.vnfs_vops = spec_vnodeop_p;
 	vfsp.vnfs_rdev = dev;
 	vfsp.vnfs_filesize = 0;
@@ -1111,6 +1243,10 @@ bdevvp(dev_t dev, vnode_t *vpp)
 		*vpp = NULLVP;
 		return (error);
 	}
+	vnode_lock_spin(nvp);
+	nvp->v_flag |= VBDEVVP;
+	nvp->v_tag = VT_NON;	/* set this to VT_NON so during aliasing it can be replaced */
+	vnode_unlock(nvp);
 	if ( (error = vnode_ref(nvp)) ) {
 		panic("bdevvp failed: vnode_ref");
 		return (error);
@@ -1123,6 +1259,14 @@ bdevvp(dev_t dev, vnode_t *vpp)
 		panic("bdevvp failed: invalidateblks");
 		return (error);
 	}
+
+#if CONFIG_MACF
+	/* 
+	 * XXXMAC: We can't put a MAC check here, the system will
+	 * panic without this vnode.
+	 */
+#endif /* MAC */	
+
 	if ( (error = VNOP_OPEN(nvp, FREAD, &context)) ) {
 		panic("bdevvp failed: open");
 		return (error);
@@ -1141,9 +1285,7 @@ bdevvp(dev_t dev, vnode_t *vpp)
  * caller is responsible for filling it with its new contents.
  */
 static vnode_t
-checkalias(nvp, nvp_rdev)
-	register struct vnode *nvp;
-	dev_t nvp_rdev;
+checkalias(struct vnode *nvp, dev_t nvp_rdev)
 {
 	struct vnode *vp;
 	struct vnode **vpp;
@@ -1174,13 +1316,14 @@ loop:
 		 * Alias, but not in use, so flush it out.
 		 */
 		if ((vp->v_iocount == 1) && (vp->v_usecount == 0)) {
-		        vnode_reclaim_internal(vp, 1, 0);
+		        vnode_reclaim_internal(vp, 1, 0, 0);
 			vnode_unlock(vp);
 			vnode_put(vp);
 			goto loop;
 		}
 	}
 	if (vp == NULL || vp->v_tag != VT_NON) {
+retnullvp:
 	        MALLOC_ZONE(nvp->v_specinfo, struct specinfo *, sizeof(struct specinfo),
 			    M_SPECINFO, M_WAITOK);
 		bzero(nvp->v_specinfo, sizeof(struct specinfo));
@@ -1202,6 +1345,12 @@ loop:
 		}
 		return (NULLVP);
 	}
+	if ((vp->v_flag & (VBDEVVP | VDEVFLUSH)) != 0)
+		return(vp);
+	else  {
+		panic("checkalias with VT_NON vp that shouldn't: %x", (unsigned int)vp);
+		goto retnullvp;
+	}
 	return (vp);
 }
 
@@ -1220,9 +1369,9 @@ static int
 vget_internal(vnode_t vp, int vid, int vflags)
 {
 	int error = 0;
-	u_long vpid;
+	int vpid;
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	if (vflags & VNODE_WITHID)
 		vpid = vid;
@@ -1235,13 +1384,17 @@ vget_internal(vnode_t vp, int vid, int vflags)
 		 */
 	        error = EINVAL;
 	else
-	        error = vnode_getiocount(vp, 1, vpid, vflags);
+	        error = vnode_getiocount(vp, vpid, vflags);
 
 	vnode_unlock(vp);
 
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOENT			No such file or directory [terminating]
+ */
 int
 vnode_ref(vnode_t vp)
 {
@@ -1249,12 +1402,16 @@ vnode_ref(vnode_t vp)
         return (vnode_ref_ext(vp, 0));
 }
 
+/*
+ * Returns:	0			Success
+ *		ENOENT			No such file or directory [terminating]
+ */
 int
 vnode_ref_ext(vnode_t vp, int fmode)
 {
 	int	error = 0;
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	/*
 	 * once all the current call sites have been fixed to insure they have
@@ -1262,7 +1419,7 @@ vnode_ref_ext(vnode_t vp, int fmode)
 	 * iocount is non-zero... a non-zero usecount doesn't insure correctness
 	 */
 	if (vp->v_iocount <= 0 && vp->v_usecount <= 0) 
-		panic("vnode_ref_ext: vp %x has no valid reference %d, %d", vp, vp->v_iocount, vp->v_usecount);
+		panic("vnode_ref_ext: vp %p has no valid reference %d, %d", vp, vp->v_iocount, vp->v_usecount);
 
 	/*
 	 * if you are the owner of drain/termination, can acquire usecount
@@ -1283,6 +1440,26 @@ vnode_ref_ext(vnode_t vp, int fmode)
 	        if (++vp->v_kusecount <= 0)
 		        panic("vnode_ref_ext: v_kusecount");
 	}
+	if (vp->v_flag & VRAGE) {
+	        struct  uthread *ut;
+
+	        ut = get_bsdthread_info(current_thread());
+		
+	        if ( !(current_proc()->p_lflag & P_LRAGE_VNODES) &&
+		     !(ut->uu_flag & UT_RAGE_VNODES)) {
+		        /*
+			 * a 'normal' process accessed this vnode
+			 * so make sure its no longer marked
+			 * for rapid aging...  also, make sure
+			 * it gets removed from the rage list...
+			 * when v_usecount drops back to 0, it
+			 * will be put back on the real free list
+			 */
+			vp->v_flag &= ~VRAGE;
+			vp->v_references = 0;
+			vnode_list_remove(vp);
+		}
+	}
 out:
 	vnode_unlock(vp);
 
@@ -1297,7 +1474,6 @@ out:
 static void
 vnode_list_add(vnode_t vp)
 {
-
 	/*
 	 * if it is already on a list or non zero references return 
 	 */
@@ -1305,17 +1481,45 @@ vnode_list_add(vnode_t vp)
 		return;
 	vnode_list_lock();
 
-	/*
-	 * insert at tail of LRU list or at head if VAGE or VL_DEAD is set
-	 */
-	if ((vp->v_flag & VAGE) || (vp->v_lflag & VL_DEAD)) {
-	        TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
-		vp->v_flag &= ~VAGE;
+	if ((vp->v_flag & VRAGE) && !(vp->v_lflag & VL_DEAD)) {
+		/*
+		 * add the new guy to the appropriate end of the RAGE list
+		 */
+		if ((vp->v_flag & VAGE))
+		        TAILQ_INSERT_HEAD(&vnode_rage_list, vp, v_freelist);
+		else
+		        TAILQ_INSERT_TAIL(&vnode_rage_list, vp, v_freelist);
+
+		vp->v_listflag |= VLIST_RAGE;
+		ragevnodes++;
+
+		/*
+		 * reset the timestamp for the last inserted vp on the RAGE
+		 * queue to let new_vnode know that its not ok to start stealing
+		 * from this list... as long as we're actively adding to this list
+		 * we'll push out the vnodes we want to donate to the real free list
+		 * once we stop pushing, we'll let some time elapse before we start
+		 * stealing them in the new_vnode routine
+		 */
+		microuptime(&rage_tv);
 	} else {
-	        TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
+	        /*
+		 * if VL_DEAD, insert it at head of the dead list
+		 * else insert at tail of LRU list or at head if VAGE is set
+		 */
+	        if ( (vp->v_lflag & VL_DEAD)) {
+		        TAILQ_INSERT_HEAD(&vnode_dead_list, vp, v_freelist);
+			vp->v_listflag |= VLIST_DEAD;
+			deadvnodes++;
+		} else if ((vp->v_flag & VAGE)) {
+		        TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
+			vp->v_flag &= ~VAGE;
+			freevnodes++;
+		} else {
+		        TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
+			freevnodes++;
+		}
 	}
-	freevnodes++;
-
 	vnode_list_unlock();
 }
 
@@ -1353,7 +1557,13 @@ vnode_list_remove(vnode_t vp)
 		 * on the free list
 		 */
 		if (VONLIST(vp)) {
-		        VREMFREE("vnode_list_remove", vp);
+		        if (vp->v_listflag & VLIST_RAGE)
+			        VREMRAGE("vnode_list_remove", vp);
+			else if (vp->v_listflag & VLIST_DEAD)
+			        VREMDEAD("vnode_list_remove", vp);
+			else
+			        VREMFREE("vnode_list_remove", vp);
+
 			VLISTNONE(vp);
 		}
 		vnode_list_unlock();
@@ -1378,22 +1588,22 @@ vnode_rele_ext(vnode_t vp, int fmode, int dont_reenter)
 void
 vnode_rele_internal(vnode_t vp, int fmode, int dont_reenter, int locked)
 {
-	struct vfs_context context;
-
 	if ( !locked)
-	        vnode_lock(vp);
+	        vnode_lock_spin(vp);
 
 	if (--vp->v_usecount < 0)
-		panic("vnode_rele_ext: vp %x usecount -ve : %d", vp,  vp->v_usecount);
+		panic("vnode_rele_ext: vp %p usecount -ve : %d", vp,  vp->v_usecount);
 
 	if (fmode & FWRITE) {
 	        if (--vp->v_writecount < 0)
-		        panic("vnode_rele_ext: vp %x writecount -ve : %d", vp,  vp->v_writecount);
+		        panic("vnode_rele_ext: vp %p writecount -ve : %ld", vp,  vp->v_writecount);
 	}
 	if (fmode & O_EVTONLY) {
 	        if (--vp->v_kusecount < 0)
-		        panic("vnode_rele_ext: vp %x kusecount -ve : %d", vp,  vp->v_kusecount);
+		        panic("vnode_rele_ext: vp %p kusecount -ve : %d", vp,  vp->v_kusecount);
 	}
+	if (vp->v_kusecount > vp->v_usecount)
+		panic("vnode_rele_ext: vp %p kusecount(%d) out of balance with usecount(%d)\n",vp, vp->v_kusecount, vp->v_usecount);
 	if ((vp->v_iocount > 0) || (vp->v_usecount > 0)) {
 	        /*
 		 * vnode is still busy... if we're the last
@@ -1402,13 +1612,13 @@ vnode_rele_internal(vnode_t vp, int fmode, int dont_reenter, int locked)
 		 */
 	        if (vp->v_usecount == 0) {
 	                vp->v_lflag |= VL_NEEDINACTIVE;
-			vp->v_flag  &= ~(VNOCACHE_DATA | VRAOFF);
+			vp->v_flag  &= ~(VNOCACHE_DATA | VRAOFF | VOPENEVT);
 		}
 		if ( !locked)
 		        vnode_unlock(vp);
 		return;
 	}
-	vp->v_flag  &= ~(VNOCACHE_DATA | VRAOFF);
+	vp->v_flag  &= ~(VNOCACHE_DATA | VRAOFF | VOPENEVT);
 
 	if ( (vp->v_lflag & (VL_TERMINATE | VL_DEAD)) || dont_reenter) {
 	        /*
@@ -1441,11 +1651,9 @@ vnode_rele_internal(vnode_t vp, int fmode, int dont_reenter, int locked)
         vp->v_lflag &= ~VL_NEEDINACTIVE;
 	vnode_unlock(vp);
 
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
-	VNOP_INACTIVE(vp, &context);
+	VNOP_INACTIVE(vp, vfs_context_current());
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	/*
 	 * because we dropped the vnode lock to call VNOP_INACTIVE
 	 * the state of the vnode may have changed... we may have
@@ -1466,9 +1674,10 @@ vnode_rele_internal(vnode_t vp, int fmode, int dont_reenter, int locked)
 				ut->uu_vreclaims = vp;
 		        goto defer_reclaim;
 		}
-	        vnode_reclaim_internal(vp, 1, 0);
+		vnode_lock_convert(vp);
+	        vnode_reclaim_internal(vp, 1, 0, 0);
 	}
-	vnode_dropiocount(vp, 1);
+	vnode_dropiocount(vp);
 	vnode_list_add(vp);
 defer_reclaim:
 	if ( !locked)
@@ -1492,16 +1701,13 @@ struct ctldebug debug1 = { "busyprt", &busyprt };
 #endif
 
 int
-vflush(mp, skipvp, flags)
-	struct mount *mp;
-	struct vnode *skipvp;
-	int flags;
+vflush(struct mount *mp, struct vnode *skipvp, int flags)
 {
-	struct proc *p = current_proc();
 	struct vnode *vp;
 	int busy = 0;
 	int reclaimed = 0;
-	int vid, retval;
+	int retval;
+	int vid;
 
 	mount_lock(mp);
 	vnode_iterate_setup(mp);
@@ -1512,7 +1718,7 @@ vflush(mp, skipvp, flags)
 	 * tries unmounting every so often to see whether
 	 * it is still busy or not.
 	 */
-	if ((flags & FORCECLOSE)==0) {
+	if (((flags & FORCECLOSE)==0)  && ((mp->mnt_kern_flag & MNTK_UNMOUNT_PREFLIGHT) != 0)) {
 		if (vnode_umount_preflight(mp, skipvp, flags)) {
 			vnode_iterate_clear(mp);
 			mount_unlock(mp);
@@ -1593,8 +1799,8 @@ loop:
 #ifdef JOE_DEBUG
 			record_vp(vp, 1);
 #endif
-			vnode_reclaim_internal(vp, 1, 0);
-			vnode_dropiocount(vp, 1);
+			vnode_reclaim_internal(vp, 1, 0, 0);
+			vnode_dropiocount(vp);
 			vnode_list_add(vp);
 
 			vnode_unlock(vp);
@@ -1613,14 +1819,15 @@ loop:
 #ifdef JOE_DEBUG
 				record_vp(vp, 1);
 #endif
-				vnode_reclaim_internal(vp, 1, 0);
-				vnode_dropiocount(vp, 1);
+				vnode_reclaim_internal(vp, 1, 0, 0);
+				vnode_dropiocount(vp);
 				vnode_list_add(vp);
 				vnode_unlock(vp);
 			} else {
-				vclean(vp, 0, p);
+				vclean(vp, 0);
 				vp->v_lflag &= ~VL_DEAD;
 				vp->v_op = spec_vnodeop_p;
+				vp->v_flag |= VDEVFLUSH;
 				vnode_unlock(vp);
 			}
 			mount_lock(mp);
@@ -1657,22 +1864,19 @@ loop:
 	return (0);
 }
 
-int num_recycledvnodes=0;
+long num_recycledvnodes = 0;	/* long for OSAddAtomic */
 /*
  * Disassociate the underlying file system from a vnode.
  * The vnode lock is held on entry.
  */
 static void
-vclean(vnode_t vp, int flags, proc_t p)
+vclean(vnode_t vp, int flags)
 {
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int active;
 	int need_inactive;
 	int already_terminating;
-	kauth_cred_t ucred = NULL;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	int clflags = 0;
 
 	/*
 	 * Check to see if the vnode is in use.
@@ -1704,22 +1908,21 @@ vclean(vnode_t vp, int flags, proc_t p)
 	 */
 	insmntque(vp, (struct mount *)0);
 
-	ucred = vp->v_cred;
-	vp->v_cred = NOCRED;
-
 	vnode_unlock(vp);
 
-	if (IS_VALID_CRED(ucred))
-	        kauth_cred_unref(&ucred);
-
 	OSAddAtomic(1, &num_recycledvnodes);
 	/*
 	 * purge from the name cache as early as possible...
 	 */
 	cache_purge(vp);
 
+	if (flags & DOCLOSE)
+		clflags |= IO_NDELAY;
+	if (flags & REVOKEALL)
+		clflags |= IO_REVOKE;
+	
 	if (active && (flags & DOCLOSE))
-		VNOP_CLOSE(vp, IO_NDELAY, &context);
+		VNOP_CLOSE(vp, clflags, ctx);
 
 	/*
 	 * Clean out any buffers associated with the vnode.
@@ -1727,11 +1930,11 @@ vclean(vnode_t vp, int flags, proc_t p)
 	if (flags & DOCLOSE) {
 #if NFSCLIENT
 		if (vp->v_tag == VT_NFS)
-			nfs_vinvalbuf(vp, V_SAVE, NOCRED, p, 0);
+			nfs_vinvalbuf(vp, V_SAVE, ctx, 0);
 		else
 #endif
 		{
-		        VNOP_FSYNC(vp, MNT_WAIT, &context);
+		        VNOP_FSYNC(vp, MNT_WAIT, ctx);
 			buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
 		}
 		if (UBCINFOEXISTS(vp))
@@ -1740,19 +1943,20 @@ vclean(vnode_t vp, int flags, proc_t p)
 			 */
 		        (void)ubc_sync_range(vp, (off_t)0, ubc_getsize(vp), UBC_PUSHALL);
 	}
-	if (UBCINFOEXISTS(vp))
-	        cluster_release(vp->v_ubcinfo);
-
 	if (active || need_inactive) 
-		VNOP_INACTIVE(vp, &context);
+		VNOP_INACTIVE(vp, ctx);
 
-	/* Destroy ubc named reference */
+	/*
+	 * Destroy ubc named reference
+	 * cluster_release is done on this path
+	 * along with dropping the reference on the ucred
+	 */
 	ubc_destroy_named(vp);
 
 	/*
 	 * Reclaim the vnode.
 	 */
-	if (VNOP_RECLAIM(vp, &context))
+	if (VNOP_RECLAIM(vp, ctx))
 		panic("vclean: cannot reclaim");
 	
 	// make sure the name & parent ptrs get cleaned out!
@@ -1784,7 +1988,11 @@ vclean(vnode_t vp, int flags, proc_t p)
  * and with all vnodes aliased to the requested vnode.
  */
 int
+#if DIAGNOSTIC
 vn_revoke(vnode_t vp, int flags, __unused vfs_context_t a_context)
+#else
+vn_revoke(vnode_t vp, __unused int flags, __unused vfs_context_t a_context)
+#endif
 {
 	struct vnode *vq;
 	int vid;
@@ -1821,7 +2029,7 @@ vn_revoke(vnode_t vp, int flags, __unused vfs_context_t a_context)
 					SPECHASH_LOCK();	
 					break;
 				}
-				vnode_reclaim_internal(vq, 0, 0);
+				vnode_reclaim_internal(vq, 0, 0, 0);
 				vnode_put(vq);
 				SPECHASH_LOCK();
 				break;
@@ -1829,7 +2037,7 @@ vn_revoke(vnode_t vp, int flags, __unused vfs_context_t a_context)
 		}
 		SPECHASH_UNLOCK();
 	}
-	vnode_reclaim_internal(vp, 0, 0);
+	vnode_reclaim_internal(vp, 0, 0, REVOKEALL);
 
 	return (0);
 }
@@ -1839,8 +2047,7 @@ vn_revoke(vnode_t vp, int flags, __unused vfs_context_t a_context)
  * Release the passed interlock if the vnode will be recycled.
  */
 int
-vnode_recycle(vp)
-	struct vnode *vp;
+vnode_recycle(struct vnode *vp)
 {
 	vnode_lock(vp);
 
@@ -1849,7 +2056,7 @@ vnode_recycle(vp)
 		vnode_unlock(vp);
 		return(0);
 	} 
-	vnode_reclaim_internal(vp, 1, 0);
+	vnode_reclaim_internal(vp, 1, 0, 0);
 	vnode_unlock(vp);
 
 	return (1);
@@ -1858,7 +2065,7 @@ vnode_recycle(vp)
 static int
 vnode_reload(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	if ((vp->v_iocount > 1) || vp->v_usecount) {
 		vnode_unlock(vp);
@@ -1876,7 +2083,7 @@ vnode_reload(vnode_t vp)
 
 
 static void
-vgone(vnode_t vp)
+vgone(vnode_t vp, int flags)
 {
 	struct vnode *vq;
 	struct vnode *vx;
@@ -1886,7 +2093,7 @@ vgone(vnode_t vp)
 	 * vclean also takes care of removing the
 	 * vnode from any mount list it might be on
 	 */
-	vclean(vp, DOCLOSE, current_proc());
+	vclean(vp, flags | DOCLOSE);
 
 	/*
 	 * If special device, remove it from special device alias list
@@ -1950,7 +2157,7 @@ loop:
 		SPECHASH_UNLOCK();
 		if (vnode_getwithvid(vp,vid))
 			goto loop;
-		vnode_lock(vp);
+		vnode_lock_spin(vp);
 		if ((vp->v_usecount > 0) || (vp->v_iocount > 1)) {
 			vnode_unlock(vp);
 			if ((*errorp = vfs_mountedon(vp)) != 0)
@@ -1977,35 +2184,55 @@ vcount(vnode_t vp)
 loop:
 	if ((vp->v_flag & VALIASED) == 0)
 	        return (vp->v_usecount - vp->v_kusecount);
+	count = 0;
 
 	SPECHASH_LOCK();
-	for (count = 0, vq = *vp->v_hashchain; vq; vq = vnext) {
-		vnext = vq->v_specnext;
-		if (vq->v_rdev != vp->v_rdev || vq->v_type != vp->v_type)
-			continue;
-		vid = vq->v_id;
-		SPECHASH_UNLOCK();
+	/*
+	 * Grab first vnode and its vid.
+	 */
+	vq = *vp->v_hashchain;
+	vid = vq ? vq->v_id : 0;
+
+	SPECHASH_UNLOCK();
 
+	while (vq) {
+		/*
+		 * Attempt to get the vnode outside the SPECHASH lock.
+		 */
 		if (vnode_getwithvid(vq, vid)) {
 			goto loop;
 		}
-		/*
-		 * Alias, but not in use, so flush it out.
-		 */
 		vnode_lock(vq);
-		if ((vq->v_usecount == 0) && (vq->v_iocount == 1)  && vq != vp) {
-			vnode_reclaim_internal(vq, 1, 0);
-			vnode_unlock(vq);
-			vnode_put(vq);
-			goto loop;
+
+		if (vq->v_rdev == vp->v_rdev && vq->v_type == vp->v_type) {
+			if ((vq->v_usecount == 0) && (vq->v_iocount == 1)  && vq != vp) {
+				/*
+				 * Alias, but not in use, so flush it out.
+				 */
+				vnode_reclaim_internal(vq, 1, 0, 0);
+				vnode_unlock(vq);
+				vnode_put(vq);
+				goto loop;
+			}
+			count += (vq->v_usecount - vq->v_kusecount);
 		}
-		count += (vq->v_usecount - vq->v_kusecount);
 		vnode_unlock(vq);
-		vnode_put(vq);	
 
 		SPECHASH_LOCK();
+		/*
+		 * must do this with the reference still held on 'vq'
+		 * so that it can't be destroyed while we're poking
+		 * through v_specnext
+		 */
+		vnext = vq->v_specnext;
+		vid = vnext ? vnext->v_id : 0;
+
+		SPECHASH_UNLOCK();
+
+		vnode_put(vq);
+
+		vq = vnext;
 	}
-	SPECHASH_UNLOCK();
 
 	return (count);
 }
@@ -2015,8 +2242,10 @@ int	prtactive = 0;		/* 1 => print out reclaim of active vnodes */
 /*
  * Print out a description of a vnode.
  */
-static char *typename[] =
+#if !CONFIG_NO_PRINTF_STRINGS
+static const char *typename[] =
    { "VNON", "VREG", "VDIR", "VBLK", "VCHR", "VLNK", "VSOCK", "VFIFO", "VBAD" };
+#endif
 
 void
 vprint(const char *label, struct vnode *vp)
@@ -2025,21 +2254,21 @@ vprint(const char *label, struct vnode *vp)
 
 	if (label != NULL)
 		printf("%s: ", label);
-	printf("type %s, usecount %d, writecount %d",
+	printf("type %s, usecount %d, writecount %ld",
 	       typename[vp->v_type], vp->v_usecount, vp->v_writecount);
 	sbuf[0] = '\0';
 	if (vp->v_flag & VROOT)
-		strcat(sbuf, "|VROOT");
+		strlcat(sbuf, "|VROOT", sizeof(sbuf));
 	if (vp->v_flag & VTEXT)
-		strcat(sbuf, "|VTEXT");
+		strlcat(sbuf, "|VTEXT", sizeof(sbuf));
 	if (vp->v_flag & VSYSTEM)
-		strcat(sbuf, "|VSYSTEM");
+		strlcat(sbuf, "|VSYSTEM", sizeof(sbuf));
 	if (vp->v_flag & VNOFLUSH)
-		strcat(sbuf, "|VNOFLUSH");
+		strlcat(sbuf, "|VNOFLUSH", sizeof(sbuf));
 	if (vp->v_flag & VBWAIT)
-		strcat(sbuf, "|VBWAIT");
+		strlcat(sbuf, "|VBWAIT", sizeof(sbuf));
 	if (vp->v_flag & VALIASED)
-		strcat(sbuf, "|VALIASED");
+		strlcat(sbuf, "|VALIASED", sizeof(sbuf));
 	if (sbuf[0] != '\0')
 		printf(" flags (%s)", &sbuf[1]);
 }
@@ -2048,7 +2277,14 @@ vprint(const char *label, struct vnode *vp)
 int
 vn_getpath(struct vnode *vp, char *pathbuf, int *len)
 {
-    return build_path(vp, pathbuf, *len, len);
+	return build_path(vp, pathbuf, *len, len, BUILDPATH_NO_FS_ENTER, vfs_context_current());
+}
+
+
+int
+vn_getcdhash(struct vnode *vp, off_t offset, unsigned char *cdhash)
+{
+	return ubc_cs_getcdhash(vp, offset, cdhash);
 }
 
 
@@ -2057,9 +2293,9 @@ static int   nexts;
 static int   max_ext_width;
 
 static int
-extension_cmp(void *a, void *b)
+extension_cmp(const void *a, const void *b)
 {
-    return (strlen((char *)a) - strlen((char *)b));
+    return (strlen((const char *)a) - strlen((const char *)b));
 }
 
 
@@ -2078,8 +2314,8 @@ extension_cmp(void *a, void *b)
 __private_extern__ int
 set_package_extensions_table(void *data, int nentries, int maxwidth)
 {
-    char *new_exts, *ptr;
-    int error, i, len;
+    char *new_exts;
+    int error;
     
     if (nentries <= 0 || nentries > 1024 || maxwidth <= 0 || maxwidth > 255) {
 	return EINVAL;
@@ -2107,10 +2343,10 @@ set_package_extensions_table(void *data, int nentries, int maxwidth)
 
 
 __private_extern__ int
-is_package_name(char *name, int len)
+is_package_name(const char *name, int len)
 {
     int i, extlen;
-    char *ptr, *name_ext;
+    const char *ptr, *name_ext;
     
     if (len <= 3) {
 	return 0;
@@ -2194,7 +2430,7 @@ extern unsigned int vfs_nummntops;
 
 int
 vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp, 
-           user_addr_t newp, size_t newlen, struct proc *p)
+           user_addr_t newp, size_t newlen, proc_t p)
 {
 	struct vfstable *vfsp;
 	int *username;
@@ -2202,6 +2438,18 @@ vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	int error;
 	struct vfsconf *vfsc;
 
+	/* All non VFS_GENERIC and in VFS_GENERIC, 
+	 * VFS_MAXTYPENUM, VFS_CONF, VFS_SET_PACKAGE_EXTS
+	 * needs to have root priv to have modifiers. 
+	 * For rest the userland_sysctl(CTLFLAG_ANYBODY) would cover.
+	 */
+	if ((newp != USER_ADDR_NULL) && ((name[0] != VFS_GENERIC) || 
+			((name[1] == VFS_MAXTYPENUM) ||
+			 (name[1] == VFS_CONF) ||
+			 (name[1] == VFS_SET_PACKAGE_EXTS)))
+	     && (error = suser(kauth_cred_get(), &p->p_acflag))) {
+			return(error);
+	}
 	/*
 	 * The VFS_NUMMNTOPS shouldn't be at name[0] since
 	 * is a VFS generic variable. So now we must check
@@ -2220,18 +2468,16 @@ vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	if (namelen < 2)
 		return (EISDIR);		/* overloaded */
 	if (name[0] != VFS_GENERIC) {
-	        struct vfs_context context;
-
 		for (vfsp = vfsconf; vfsp; vfsp = vfsp->vfc_next)
 			if (vfsp->vfc_typenum == name[0])
 				break;
 		if (vfsp == NULL)
 			return (ENOTSUP);
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
 
+		/* XXX current context proxy for proc p? */
 		return ((*vfsp->vfc_vfsops->vfs_sysctl)(&name[1], namelen - 1,
-		            oldp, oldlenp, newp, newlen, &context));
+		            oldp, oldlenp, newp, newlen,
+			    vfs_context_current()));
 	}
 	switch (name[1]) {
 	case VFS_MAXTYPENUM:
@@ -2280,79 +2526,28 @@ vfs_sysctl(int *name, u_int namelen, user_addr_t oldp, size_t *oldlenp,
 	return (error);
 }
 
-int kinfo_vdebug = 1;
-#define KINFO_VNODESLOP	10
 /*
- * Dump vnode list (via sysctl).
- * Copyout address of vnode followed by vnode.
+ * Dump vnode list (via sysctl) - defunct
+ * use "pstat" instead
  */
 /* ARGSUSED */
 int
-sysctl_vnode(__unused user_addr_t where, __unused size_t *sizep)
+sysctl_vnode
+(__unused struct sysctl_oid *oidp, __unused void *arg1, __unused int arg2, __unused struct sysctl_req *req)
 {
-#if 0
-	struct mount *mp, *nmp;
-	struct vnode *nvp, *vp;
-	char *bp = where, *savebp;
-	char *ewhere;
-	int error;
-
-#define VPTRSZ	sizeof (struct vnode *)
-#define VNODESZ	sizeof (struct vnode)
-	if (where == NULL) {
-		*sizep = (numvnodes + KINFO_VNODESLOP) * (VPTRSZ + VNODESZ);
-		return (0);
-	}
-	ewhere = where + *sizep;
-		
-	for (mp = mountlist.cqh_first; mp != (void *)&mountlist; mp = nmp) {
-		if (vfs_busy(mp, LK_NOWAIT)) {
-			nmp = mp->mnt_list.cqe_next;
-			continue;
-		}
-		savebp = bp;
-again:
-		TAILQ_FOREACH(vp, &mp->mnt_vnodelist, v_mntvnodes) {
-			/*
-			 * Check that the vp is still associated with
-			 * this filesystem.  RACE: could have been
-			 * recycled onto the same filesystem.
-			 */
-			if (vp->v_mount != mp) {
-				if (kinfo_vdebug)
-					printf("kinfo: vp changed\n");
-				bp = savebp;
-				goto again;
-			}
-			if (bp + VPTRSZ + VNODESZ > ewhere) {
-				vfs_unbusy(mp);
-				*sizep = bp - where;
-				return (ENOMEM);
-			}
-			if ((error = copyout((caddr_t)&vp, bp, VPTRSZ)) ||
-			    (error = copyout((caddr_t)vp, bp + VPTRSZ, VNODESZ))) {
-				vfs_unbusy(mp);
-				return (error);
-			}
-			bp += VPTRSZ + VNODESZ;
-		}
-		nmp = mp->mnt_list.cqe_next;
-		vfs_unbusy(mp);
-	}
-
-	*sizep = bp - where;
-	return (0);
-#else
 	return(EINVAL);
-#endif
 }
 
+SYSCTL_PROC(_kern, KERN_VNODE, vnode,
+		CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_MASKED,
+		0, 0, sysctl_vnode, "S,", "");
+
+
 /*
  * Check to see if a filesystem is mounted on a block device.
  */
 int
-vfs_mountedon(vp)
-	struct vnode *vp;
+vfs_mountedon(struct vnode *vp)
 {
 	struct vnode *vq;
 	int error = 0;
@@ -2383,12 +2578,10 @@ out:
  * of mounting to avoid dependencies.
  */
 __private_extern__ void
-vfs_unmountall()
+vfs_unmountall(void)
 {
 	struct mount *mp;
-	struct proc *p = current_proc();
 	int error;
-	int skip_listremove;
 
 	/*
 	 * Since this only runs when rebooting, it is not interlocked.
@@ -2397,21 +2590,17 @@ vfs_unmountall()
 	while(!TAILQ_EMPTY(&mountlist)) {
 		mp = TAILQ_LAST(&mountlist, mntlist);
 		mount_list_unlock();
-		skip_listremove = 0;
-		error = dounmount(mp, MNT_FORCE, &skip_listremove, 0, p);
+		error = dounmount(mp, MNT_FORCE, 0, vfs_context_current());
 		if ((error != 0) && (error != EBUSY)) {
 			printf("unmount of %s failed (", mp->mnt_vfsstat.f_mntonname);
 			printf("%d)\n", error);
-
 			mount_list_lock();
-			if (skip_listremove == 0) {
-				TAILQ_REMOVE(&mountlist, mp, mnt_list);
-			}
+			TAILQ_REMOVE(&mountlist, mp, mnt_list);
 			continue;
 		} else if (error == EBUSY) {
 			/* If EBUSY is returned,  the unmount was already in progress */
 			printf("unmount of %x failed (", (unsigned int)mp);
-			printf("BUSY)\n");
+			printf("BUSY)\n");	
 		} 
 		mount_list_lock();
 	}
@@ -2420,35 +2609,25 @@ vfs_unmountall()
 
 
 /*  
- * This routine is called from vnode_pager_no_senders()
- * which in turn can be called with vnode locked by vnode_uncache()
- * But it could also get called as a result of vm_object_cache_trim().
- * In that case lock state is unknown.
- * AGE the vnode so that it gets recycled quickly.
+ * This routine is called from vnode_pager_deallocate out of the VM
+ * The path to vnode_pager_deallocate can only be initiated by ubc_destroy_named
+ * on a vnode that has a UBCINFO
  */
 __private_extern__ void
-vnode_pager_vrele(struct vnode *vp)
+vnode_pager_vrele(vnode_t vp)
 {
+        struct ubc_info *uip;
+
 	vnode_lock(vp);
 
-	if (!ISSET(vp->v_lflag, VL_TERMINATE))
-		panic("vnode_pager_vrele: vp not in termination");
 	vp->v_lflag &= ~VNAMED_UBC;
 
-	if (UBCINFOEXISTS(vp)) {
-		struct ubc_info *uip = vp->v_ubcinfo;
+	uip = vp->v_ubcinfo;
+	vp->v_ubcinfo = UBC_INFO_NULL;
 
-		if (ISSET(uip->ui_flags, UI_WASMAPPED))
-			SET(vp->v_flag, VWASMAPPED);
-		vp->v_ubcinfo = UBC_INFO_NULL;
+	ubc_info_deallocate(uip);
 
-		ubc_info_deallocate(uip);
-	} else {
-		panic("NO ubcinfo in vnode_pager_vrele");
-	}
 	vnode_unlock(vp);
-
-	wakeup(&vp->v_lflag);
 }
 
 
@@ -2466,14 +2645,11 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp)
 	off_t	writesegcnt;
 	off_t	readsegsize;
 	off_t	writesegsize;
+	off_t	alignment;
 	u_long 	blksize;
 	u_int64_t temp;
-	struct vfs_context context;
-
-	proc_t	p = current_proc();
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	u_int32_t features;
+	vfs_context_t ctx = vfs_context_current();
 
 	int isvirtual = 0;
 	/*
@@ -2484,13 +2660,13 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp)
 	static int rootunit = -1;
 
 	if (rootunit == -1) {
-	        if (VNOP_IOCTL(rootvp, DKIOCGETBSDUNIT, (caddr_t)&rootunit, 0, &context))
+	        if (VNOP_IOCTL(rootvp, DKIOCGETBSDUNIT, (caddr_t)&rootunit, 0, ctx))
 		        rootunit = -1; 
 		else if (rootvp == devvp)
 		        mp->mnt_kern_flag |= MNTK_ROOTDEV;
 	}
 	if (devvp != rootvp && rootunit != -1) {
-	        if (VNOP_IOCTL(devvp, DKIOCGETBSDUNIT, (caddr_t)&thisunit, 0, &context) == 0) {
+	        if (VNOP_IOCTL(devvp, DKIOCGETBSDUNIT, (caddr_t)&thisunit, 0, ctx) == 0) {
 		        if (thisunit == rootunit)
 			        mp->mnt_kern_flag |= MNTK_ROOTDEV;
 		}
@@ -2504,46 +2680,54 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp)
 
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETBLOCKSIZE,
-				(caddr_t)&blksize, 0, &context)))
+				(caddr_t)&blksize, 0, ctx)))
 		return (error);
 
 	mp->mnt_devblocksize = blksize;
 
-	if (VNOP_IOCTL(devvp, DKIOCISVIRTUAL, (caddr_t)&isvirtual, 0, &context) == 0) {
+	if (VNOP_IOCTL(devvp, DKIOCISVIRTUAL, (caddr_t)&isvirtual, 0, ctx) == 0) {
 	        if (isvirtual)
 		        mp->mnt_kern_flag |= MNTK_VIRTUALDEV;
 	}
 
+	if ((error = VNOP_IOCTL(devvp, DKIOCGETFEATURES,
+				(caddr_t)&features, 0, ctx)))
+		return (error);
+
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTREAD,
-				(caddr_t)&readblockcnt, 0, &context)))
+				(caddr_t)&readblockcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXBLOCKCOUNTWRITE,
-				(caddr_t)&writeblockcnt, 0, &context)))
+				(caddr_t)&writeblockcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTREAD,
-				(caddr_t)&readmaxcnt, 0, &context)))
+				(caddr_t)&readmaxcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXBYTECOUNTWRITE,
-				(caddr_t)&writemaxcnt, 0, &context)))
+				(caddr_t)&writemaxcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTCOUNTREAD,
-				(caddr_t)&readsegcnt, 0, &context)))
+				(caddr_t)&readsegcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTCOUNTWRITE,
-				(caddr_t)&writesegcnt, 0, &context)))
+				(caddr_t)&writesegcnt, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTBYTECOUNTREAD,
-				(caddr_t)&readsegsize, 0, &context)))
+				(caddr_t)&readsegsize, 0, ctx)))
 		return (error);
 
 	if ((error = VNOP_IOCTL(devvp, DKIOCGETMAXSEGMENTBYTECOUNTWRITE,
-				(caddr_t)&writesegsize, 0, &context)))
+				(caddr_t)&writesegsize, 0, ctx)))
+		return (error);
+
+	if ((error = VNOP_IOCTL(devvp, DKIOCGETMINSEGMENTALIGNMENTBYTECOUNT,
+				(caddr_t)&alignment, 0, ctx)))
 		return (error);
 
 	if (readmaxcnt)
@@ -2588,6 +2772,15 @@ vfs_init_io_attributes(vnode_t devvp, mount_t mp)
 	        temp = mp->mnt_maxwritecnt;
 	mp->mnt_maxsegwritesize = (u_int32_t)temp;
 
+	if (alignment)
+	        temp = (alignment > PAGE_SIZE) ? PAGE_MASK : alignment - 1;
+	else
+	        temp = 0;
+	mp->mnt_alignmentmask = temp;
+
+	if (features & DK_FEATURE_FORCE_UNIT_ACCESS)
+	        mp->mnt_ioflags |= MNT_IOFLAGS_FUA_SUPPORTED;
+	
 	return (error);
 }
 
@@ -2674,7 +2867,8 @@ sysctl_vfs_getvfslist(fsid_t *fsidlst, int count, int *actual)
 }
 
 static int
-sysctl_vfs_vfslist SYSCTL_HANDLER_ARGS
+sysctl_vfs_vfslist(__unused struct sysctl_oid *oidp, __unused void *arg1,
+		__unused int arg2, struct sysctl_req *req)
 {
 	int actual, error;
 	size_t space;
@@ -2724,43 +2918,48 @@ again:
  * Do a sysctl by fsid.
  */
 static int
-sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
+sysctl_vfs_ctlbyfsid(__unused struct sysctl_oid *oidp, void *arg1, int arg2,
+		struct sysctl_req *req)
 {
 	struct vfsidctl vc;
 	struct user_vfsidctl user_vc;
 	struct mount *mp;
 	struct vfsstatfs *sp;
-	struct proc *p;
-	int *name;
-	int error, flags, namelen;
-	struct vfs_context context;
+	int *name, flags, namelen;
+	int error=0, gotref=0;
+	vfs_context_t ctx = vfs_context_current();
+	proc_t p = req->p;	/* XXX req->p != current_proc()? */
 	boolean_t is_64_bit;
 
 	name = arg1;
 	namelen = arg2;
-	p = req->p;
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 	is_64_bit = proc_is64bit(p);
 
 	if (is_64_bit) {
 		error = SYSCTL_IN(req, &user_vc, sizeof(user_vc));
 		if (error)
-			return (error);
-		if (user_vc.vc_vers != VFS_CTL_VERS1)
-			return (EINVAL);
-		mp = mount_list_lookupby_fsid(&user_vc.vc_fsid, 0, 0);
+			goto out;
+		if (user_vc.vc_vers != VFS_CTL_VERS1) {
+			error = EINVAL;
+			goto out;
+		}
+		mp = mount_list_lookupby_fsid(&user_vc.vc_fsid, 0, 1);
 	} 
 	else {
 		error = SYSCTL_IN(req, &vc, sizeof(vc));
 		if (error)
-			return (error);
-		if (vc.vc_vers != VFS_CTL_VERS1)
-			return (EINVAL);
-		mp = mount_list_lookupby_fsid(&vc.vc_fsid, 0, 0);
+			goto out;
+		if (vc.vc_vers != VFS_CTL_VERS1) {
+			error = EINVAL;
+			goto out;
+		}
+		mp = mount_list_lookupby_fsid(&vc.vc_fsid, 0, 1);
+	}
+	if (mp == NULL) {
+		error = ENOENT;
+		goto out;
 	}
-	if (mp == NULL)
-		return (ENOENT);
+	gotref = 1;
 	/* reset so that the fs specific code can fetch it. */
 	req->newidx = 0;
 	/*
@@ -2774,7 +2973,7 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 				error = mp->mnt_op->vfs_sysctl(name, namelen,
 				    CAST_USER_ADDR_T(req),
 				    NULL, USER_ADDR_NULL, 0, 
-				    &context);
+				    ctx);
 			}
 			else {
 				error = ENOTSUP;
@@ -2784,10 +2983,11 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 			error = mp->mnt_op->vfs_sysctl(name, namelen,
 			    CAST_USER_ADDR_T(req),
 			    NULL, USER_ADDR_NULL, 0, 
-			    &context);
+			    ctx);
+		}
+		if (error != ENOTSUP) {
+			goto out;
 		}
-		if (error != ENOTSUP)
-			return (error);
 	}
 	switch (name[0]) {
 	case VFS_CTL_UMOUNT:
@@ -2803,9 +3003,12 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 		error = SYSCTL_IN(req, &flags, sizeof(flags));
 		if (error)
 			break;
+
 		mount_ref(mp, 0);
+		mount_iterdrop(mp);
+		gotref = 0;
 		/* safedounmount consumes a ref */
-		error = safedounmount(mp, flags, p);
+		error = safedounmount(mp, flags, ctx);
 		break;
 	case VFS_CTL_STATFS:
 		req->newidx = 0;
@@ -2822,8 +3025,8 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 			break;
 		sp = &mp->mnt_vfsstat;
 		if (((flags & MNT_NOWAIT) == 0 || (flags & MNT_WAIT)) &&
-		    (error = vfs_update_vfsstat(mp, &context)))
-			return (error);
+		    (error = vfs_update_vfsstat(mp, ctx, VFS_USER_EVENT)))
+			goto out;
 		if (is_64_bit) {
 			struct user_statfs sfs;
 			bzero(&sfs, sizeof(sfs));
@@ -2839,9 +3042,9 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 			sfs.f_fsid = sp->f_fsid;
 			sfs.f_owner = sp->f_owner;
     
-			strncpy(&sfs.f_fstypename, &sp->f_fstypename, MFSNAMELEN-1);
-			strncpy(&sfs.f_mntonname, &sp->f_mntonname, MNAMELEN-1);
-			strncpy(&sfs.f_mntfromname, &sp->f_mntfromname, MNAMELEN-1);
+			strlcpy(sfs.f_fstypename, sp->f_fstypename, MFSNAMELEN);
+			strlcpy(sfs.f_mntonname, sp->f_mntonname, MNAMELEN);
+			strlcpy(sfs.f_mntfromname, sp->f_mntfromname, MNAMELEN);
             
 			error = SYSCTL_OUT(req, &sfs, sizeof(sfs));
 		}
@@ -2893,16 +3096,20 @@ sysctl_vfs_ctlbyfsid SYSCTL_HANDLER_ARGS
 			sfs.f_fsid = sp->f_fsid;
 			sfs.f_owner = sp->f_owner;
     
-			strncpy(&sfs.f_fstypename, &sp->f_fstypename, MFSNAMELEN-1);
-			strncpy(&sfs.f_mntonname, &sp->f_mntonname, MNAMELEN-1);
-			strncpy(&sfs.f_mntfromname, &sp->f_mntfromname, MNAMELEN-1);
+			strlcpy(sfs.f_fstypename, sp->f_fstypename, MFSNAMELEN);
+			strlcpy(sfs.f_mntonname, sp->f_mntonname, MNAMELEN);
+			strlcpy(sfs.f_mntfromname, sp->f_mntfromname, MNAMELEN);
             
 			error = SYSCTL_OUT(req, &sfs, sizeof(sfs));
 		}
 		break;
 	default:
-		return (ENOTSUP);
+		error = ENOTSUP;
+		goto out;
 	}
+out:
+	if(gotref != 0)
+		mount_iterdrop(mp);
 	return (error);
 }
 
@@ -2932,18 +3139,25 @@ filt_fsdetach(struct knote *kn)
 static int
 filt_fsevent(struct knote *kn, long hint)
 {
+	/*
+	 * Backwards compatibility:
+	 * Other filters would do nothing if kn->kn_sfflags == 0
+	 */
+
+	if ((kn->kn_sfflags == 0) || (kn->kn_sfflags & hint)) {
+		kn->kn_fflags |= hint;
+	}
 
-	kn->kn_fflags |= hint;
 	return (kn->kn_fflags != 0);
 }
 
 static int
-sysctl_vfs_noremotehang SYSCTL_HANDLER_ARGS
+sysctl_vfs_noremotehang(__unused struct sysctl_oid *oidp,
+		__unused void *arg1, __unused int arg2, struct sysctl_req *req)
 {
 	int out, error;
 	pid_t pid;
-	size_t space;
-	struct proc *p;
+	proc_t p;
 
 	/* We need a pid. */
 	if (req->newptr == USER_ADDR_NULL)
@@ -2953,7 +3167,7 @@ sysctl_vfs_noremotehang SYSCTL_HANDLER_ARGS
 	if (error)
 		return (error);
 
-	p = pfind(pid < 0 ? -pid : pid);
+	p = proc_find(pid < 0 ? -pid : pid);
 	if (p == NULL)
 		return (ESRCH);
 
@@ -2963,68 +3177,76 @@ sysctl_vfs_noremotehang SYSCTL_HANDLER_ARGS
 	 */
 	if (req->oldptr != USER_ADDR_NULL) {
 		out = !((p->p_flag & P_NOREMOTEHANG) == 0);
+		proc_rele(p);
 		error = SYSCTL_OUT(req, &out, sizeof(out));
 		return (error);
 	}
 
-	/* XXX req->p->p_ucred -> kauth_cred_get() - current unsafe ??? */
 	/* cansignal offers us enough security. */
-	if (p != req->p && suser(req->p->p_ucred, &req->p->p_acflag) != 0)
+	if (p != req->p && proc_suser(req->p) != 0) {
+		proc_rele(p);
 		return (EPERM);
+	}
 
 	if (pid < 0)
-		p->p_flag &= ~P_NOREMOTEHANG;
+		OSBitAndAtomic(~((uint32_t)P_NOREMOTEHANG), (UInt32 *)&p->p_flag);
 	else
-		p->p_flag |= P_NOREMOTEHANG;
+		OSBitOrAtomic(P_NOREMOTEHANG, (UInt32 *)&p->p_flag);
+	proc_rele(p);
 
 	return (0);
 }
+
 /* the vfs.generic. branch. */
-SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RW, 0, "vfs generic hinge");
+SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RW|CTLFLAG_LOCKED, NULL, "vfs generic hinge");
 /* retreive a list of mounted filesystem fsid_t */
 SYSCTL_PROC(_vfs_generic, OID_AUTO, vfsidlist, CTLFLAG_RD,
-    0, 0, sysctl_vfs_vfslist, "S,fsid", "List of mounted filesystem ids");
+    NULL, 0, sysctl_vfs_vfslist, "S,fsid", "List of mounted filesystem ids");
 /* perform operations on filesystem via fsid_t */
-SYSCTL_NODE(_vfs_generic, OID_AUTO, ctlbyfsid, CTLFLAG_RW,
+SYSCTL_NODE(_vfs_generic, OID_AUTO, ctlbyfsid, CTLFLAG_RW|CTLFLAG_LOCKED,
     sysctl_vfs_ctlbyfsid, "ctlbyfsid");
-SYSCTL_PROC(_vfs_generic, OID_AUTO, noremotehang, CTLFLAG_RW,
-    0, 0, sysctl_vfs_noremotehang, "I", "noremotehang");
+SYSCTL_PROC(_vfs_generic, OID_AUTO, noremotehang, CTLFLAG_RW|CTLFLAG_ANYBODY,
+    NULL, 0, sysctl_vfs_noremotehang, "I", "noremotehang");
 	
 	
-int num_reusedvnodes=0;
+long num_reusedvnodes = 0;	/* long for OSAddAtomic */
 
 static int
 new_vnode(vnode_t *vpp)
 {
 	vnode_t	vp;
 	int retries = 0;				/* retry incase of tablefull */
+	int force_alloc = 0, walk_count = 0;
 	int vpid;
 	struct timespec ts;
+        struct timeval current_tv;
+        struct unsafe_fsnode *l_unsafefs = 0;
+	proc_t  curproc = current_proc();
+	pid_t current_pid = proc_pid(curproc);
 
 retry:
+	microuptime(&current_tv);
+
+	vp = NULLVP;
+
 	vnode_list_lock();
 
-	if ( !TAILQ_EMPTY(&vnode_free_list)) {
+	if ( !TAILQ_EMPTY(&vnode_dead_list)) {
 	        /*
-		 * Pick the first vp for possible reuse
+		 * Can always reuse a dead one
 		 */
-	        vp = TAILQ_FIRST(&vnode_free_list);
-
-		if (vp->v_lflag & VL_DEAD)
-		        goto steal_this_vp;
-	} else
-	        vp = NULL;
-
+	        vp = TAILQ_FIRST(&vnode_dead_list);
+		goto steal_this_vp;
+	}
 	/*
-	 * we're either empty, or the next guy on the
-	 * list is a valid vnode... if we're under the
-	 * limit, we'll create a new vnode
+	 * no dead vnodes available... if we're under
+	 * the limit, we'll create a new vnode
 	 */
-	if (numvnodes < desiredvnodes) {
+	if (numvnodes < desiredvnodes || force_alloc) {
 		numvnodes++;
 		vnode_list_unlock();
-		MALLOC_ZONE(vp, struct vnode *, sizeof *vp, M_VNODE, M_WAITOK);
-		bzero((char *)vp, sizeof *vp);
+		MALLOC_ZONE(vp, struct vnode *, sizeof(*vp), M_VNODE, M_WAITOK);
+		bzero((char *)vp, sizeof(*vp));
 		VLISTNONE(vp);		/* avoid double queue removal */
 		lck_mtx_init(&vp->v_lock, vnode_lck_grp, vnode_lck_attr);
 
@@ -3032,10 +3254,82 @@ retry:
 		vp->v_id = ts.tv_nsec;
 		vp->v_flag = VSTANDARD;
 
+#if CONFIG_MACF
+		mac_vnode_label_init(vp);
+#endif /* MAC */
+
 		goto done;
 	}
-	if (vp == NULL) {
-	        /*
+
+#define MAX_WALK_COUNT 1000
+
+	if ( !TAILQ_EMPTY(&vnode_rage_list) &&
+	     (ragevnodes >= rage_limit ||
+	      (current_tv.tv_sec - rage_tv.tv_sec) >= RAGE_TIME_LIMIT)) {
+
+		TAILQ_FOREACH(vp, &vnode_rage_list, v_freelist) {
+		    if ( !(vp->v_listflag & VLIST_RAGE) || !(vp->v_flag & VRAGE))
+		        panic("new_vnode: vp on RAGE list not marked both VLIST_RAGE and VRAGE");
+
+		    // skip vnodes which have a dependency on this process
+		    // (i.e. they're vnodes in a disk image and this process
+		    // is diskimages-helper)
+		    //
+		    if (vp->v_mount && vp->v_mount->mnt_dependent_pid != current_pid && vp->v_mount->mnt_dependent_process != curproc) {
+			break;
+		    }
+
+		    // don't iterate more than MAX_WALK_COUNT vnodes to
+		    // avoid keeping the vnode list lock held for too long.
+		    if (walk_count++ > MAX_WALK_COUNT) {
+			vp = NULL;
+			break;
+		    }
+		}
+
+	}
+
+	if (vp == NULL && !TAILQ_EMPTY(&vnode_free_list)) {
+	        /*
+		 * Pick the first vp for possible reuse
+		 */
+		walk_count = 0;
+		TAILQ_FOREACH(vp, &vnode_free_list, v_freelist) {
+		    // skip vnodes which have a dependency on this process
+		    // (i.e. they're vnodes in a disk image and this process
+		    // is diskimages-helper)
+		    //
+		    if (vp->v_mount && vp->v_mount->mnt_dependent_pid != current_pid && vp->v_mount->mnt_dependent_process != curproc) {
+			break;
+		    }
+
+		    // don't iterate more than MAX_WALK_COUNT vnodes to
+		    // avoid keeping the vnode list lock held for too long.
+		    if (walk_count++ > MAX_WALK_COUNT) {
+			vp = NULL;
+			break;
+		    }
+		}
+
+	}
+
+	//
+	// if we don't have a vnode and the walk_count is >= MAX_WALK_COUNT
+	// then we're trying to create a vnode on behalf of a
+	// process like diskimages-helper that has file systems
+	// mounted on top of itself (and thus we can't reclaim
+	// vnodes in the file systems on top of us).  if we can't
+	// find a vnode to reclaim then we'll just have to force
+	// the allocation.
+	//
+	if (vp == NULL && walk_count >= MAX_WALK_COUNT) {
+	    force_alloc = 1;
+	    vnode_list_unlock();
+	    goto retry;
+	}
+
+	if (vp == NULL) {
+	        /*
 		 * we've reached the system imposed maximum number of vnodes
 		 * but there isn't a single one available
 		 * wait a bit and then retry... if we can't get a vnode
@@ -3043,26 +3337,35 @@ retry:
 		 */
 		if (++retries <= 100) {
 			vnode_list_unlock();
-			IOSleep(1);
+			delay_for_interval(1, 1000 * 1000);
 			goto retry;
 		}
 			
 		vnode_list_unlock();
 		tablefull("vnode");
 		log(LOG_EMERG, "%d desired, %d numvnodes, "
-			"%d free, %d inactive\n",
-			desiredvnodes, numvnodes, freevnodes, inactivevnodes);
-		*vpp = 0;
+			"%d free, %d dead, %d rage\n",
+		        desiredvnodes, numvnodes, freevnodes, deadvnodes, ragevnodes);
+		*vpp = NULL;
 		return (ENFILE);
 	}
 steal_this_vp:
 	vpid = vp->v_id;
 
-	VREMFREE("new_vnode", vp);
+	/*
+	 * the v_listflag field is
+	 * protected by the vnode_list_lock
+	 */
+	if (vp->v_listflag & VLIST_DEAD)
+	        VREMDEAD("new_vnode", vp);
+	else if (vp->v_listflag & VLIST_RAGE)
+	        VREMRAGE("new_vnode", vp);
+	else
+	        VREMFREE("new_vnode", vp);
 	VLISTNONE(vp);
 
 	vnode_list_unlock();
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	/* 
 	 * We could wait for the vnode_lock after removing the vp from the freelist
@@ -3110,8 +3413,9 @@ steal_this_vp:
 	if (vp->v_type != VBAD) {
 		if (vp->v_lflag & VL_DEAD)
 			panic("new_vnode: the vnode is VL_DEAD but not VBAD");
+		vnode_lock_convert(vp);
 
-		(void)vnode_reclaim_internal(vp, 1, 1);
+		(void)vnode_reclaim_internal(vp, 1, 1, 0);
 
 		if ((VONLIST(vp)))
 		        panic("new_vnode: vp on list ");
@@ -3124,20 +3428,38 @@ steal_this_vp:
 		        panic("new_vnode: vnode still hooked into the name cache");
 	}
 	if (vp->v_unsafefs) {
-	        lck_mtx_destroy(&vp->v_unsafefs->fsnodelock, vnode_lck_grp);
-		FREE_ZONE((void *)vp->v_unsafefs, sizeof(struct unsafe_fsnode), M_UNSAFEFS);
+	        l_unsafefs = vp->v_unsafefs;
 		vp->v_unsafefs = (struct unsafe_fsnode *)NULL;
 	}
+
+#if CONFIG_MACF
+	/*
+	 * We should never see VL_LABELWAIT or VL_LABEL here.
+	 * as those operations hold a reference.
+	 */
+	assert ((vp->v_lflag & VL_LABELWAIT) != VL_LABELWAIT);
+	assert ((vp->v_lflag & VL_LABEL) != VL_LABEL);
+	if (vp->v_lflag & VL_LABELED) {
+	        vnode_lock_convert(vp);
+		mac_vnode_label_recycle(vp);
+	}
+#endif /* MAC */
+
 	vp->v_lflag = 0;
 	vp->v_writecount = 0;
         vp->v_references = 0;
 	vp->v_iterblkflags = 0;
 	vp->v_flag = VSTANDARD;
 	/* vbad vnodes can point to dead_mountp */
-	vp->v_mount = 0;
+	vp->v_mount = NULL;
 	vp->v_defer_reclaimlist = (vnode_t)0;
 
 	vnode_unlock(vp);
+
+	if (l_unsafefs) {
+	        lck_mtx_destroy(&l_unsafefs->fsnodelock, vnode_lck_grp);
+		FREE_ZONE((void *)l_unsafefs, sizeof(struct unsafe_fsnode), M_UNSAFEFS);
+	}
 done:
 	*vpp = vp;
 
@@ -3150,6 +3472,12 @@ vnode_lock(vnode_t vp)
 	lck_mtx_lock(&vp->v_lock);
 }
 
+void
+vnode_lock_spin(vnode_t vp)
+{
+	lck_mtx_lock_spin(&vp->v_lock);
+}
+
 void
 vnode_unlock(vnode_t vp)
 {
@@ -3161,19 +3489,27 @@ vnode_unlock(vnode_t vp)
 int
 vnode_get(struct vnode *vp)
 {
-        vnode_lock(vp);
+        int retval;
 
-	if ( (vp->v_iocount == 0) && (vp->v_lflag & (VL_TERMINATE | VL_DEAD)) ) {
-	        vnode_unlock(vp);
+        vnode_lock_spin(vp);
+	retval = vnode_get_locked(vp);
+	vnode_unlock(vp);
+
+	return(retval);	
+}
+
+int
+vnode_get_locked(struct vnode *vp)
+{
+
+	if ((vp->v_iocount == 0) && (vp->v_lflag & (VL_TERMINATE | VL_DEAD))) {
 		return(ENOENT);	
 	}
 	vp->v_iocount++;
 #ifdef JOE_DEBUG
 	record_vp(vp, 1);
 #endif
-	vnode_unlock(vp);
-
-	return(0);	
+	return (0);
 }
 
 int
@@ -3194,7 +3530,7 @@ vnode_put(vnode_t vp)
 {
         int retval;
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	retval = vnode_put_locked(vp);
 	vnode_unlock(vp);
 
@@ -3204,14 +3540,14 @@ vnode_put(vnode_t vp)
 int
 vnode_put_locked(vnode_t vp)
 {
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();	/* hoist outside loop */
 
 retry:
 	if (vp->v_iocount < 1) 
-		panic("vnode_put(%x): iocount < 1", vp);
+		panic("vnode_put(%p): iocount < 1", vp);
 
 	if ((vp->v_usecount > 0) || (vp->v_iocount > 1))  {
-		vnode_dropiocount(vp, 1);
+		vnode_dropiocount(vp);
 		return(0);
 	}
 	if ((vp->v_lflag & (VL_MARKTERM | VL_TERMINATE | VL_DEAD | VL_NEEDINACTIVE)) == VL_NEEDINACTIVE) {
@@ -3219,11 +3555,9 @@ retry:
 	        vp->v_lflag &= ~VL_NEEDINACTIVE;
 	        vnode_unlock(vp);
 
-		context.vc_proc = current_proc();
-		context.vc_ucred = kauth_cred_get();
-		VNOP_INACTIVE(vp, &context);
+		VNOP_INACTIVE(vp, ctx);
 
-		vnode_lock(vp);
+		vnode_lock_spin(vp);
 		/*
 		 * because we had to drop the vnode lock before calling
 		 * VNOP_INACTIVE, the state of this vnode may have changed...
@@ -3237,10 +3571,11 @@ retry:
 	}
         vp->v_lflag &= ~VL_NEEDINACTIVE;
 
-	if ((vp->v_lflag & (VL_MARKTERM | VL_TERMINATE | VL_DEAD)) == VL_MARKTERM)
-	        vnode_reclaim_internal(vp, 1, 0);
-
-	vnode_dropiocount(vp, 1);
+	if ((vp->v_lflag & (VL_MARKTERM | VL_TERMINATE | VL_DEAD)) == VL_MARKTERM) {
+	        vnode_lock_convert(vp);
+	        vnode_reclaim_internal(vp, 1, 0, 0);
+	}
+	vnode_dropiocount(vp);
 	vnode_list_add(vp);
 
 	return(0);
@@ -3260,8 +3595,8 @@ vnode_isinuse_locked(vnode_t vp, int refcnt, int locked)
 	int retval = 0;
 
 	if (!locked)
-		vnode_lock(vp);
-	if ((vp->v_type != VREG) && (vp->v_usecount >  refcnt)) {
+		vnode_lock_spin(vp);
+	if ((vp->v_type != VREG) && ((vp->v_usecount - vp->v_kusecount) >  refcnt)) {
 		retval = 1;
 		goto out;
 	}
@@ -3281,11 +3616,11 @@ errno_t
 vnode_resume(vnode_t vp)
 {
 
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 
 	if (vp->v_owner == current_thread()) {
 	        vp->v_lflag &= ~VL_SUSPENDED;
-		vp->v_owner = 0;
+		vp->v_owner = NULL;
 		vnode_unlock(vp);
 		wakeup(&vp->v_iocount);
 	} else
@@ -3294,6 +3629,37 @@ vnode_resume(vnode_t vp)
 	return(0);
 }
 
+/* suspend vnode_t
+ * Please do not use on more than one vnode at a time as it may
+ * cause deadlocks.
+ * xxx should we explicity prevent this from happening?
+ */
+
+errno_t
+vnode_suspend(vnode_t vp)
+{
+	if (vp->v_lflag & VL_SUSPENDED) {
+		return(EBUSY);
+	}
+
+	vnode_lock_spin(vp);
+
+	/* 
+	 * xxx is this sufficient to check if a vnode_drain is 
+	 * progress?
+	 */
+
+	if (vp->v_owner == NULL) {
+		vp->v_lflag |= VL_SUSPENDED;
+		vp->v_owner = current_thread();
+	}
+	vnode_unlock(vp);
+
+	return(0);
+}
+					
+					
+
 static errno_t 
 vnode_drain(vnode_t vp)
 {
@@ -3306,7 +3672,7 @@ vnode_drain(vnode_t vp)
 	vp->v_owner = current_thread();
 
 	while (vp->v_iocount > 1)
-		msleep(&vp->v_iocount, &vp->v_lock, PVFS, "vnode_drain", 0);
+		msleep(&vp->v_iocount, &vp->v_lock, PVFS, "vnode_drain", NULL);
 	return(0);
 }
 
@@ -3320,24 +3686,19 @@ vnode_drain(vnode_t vp)
  * to constantly remove and add to the list each time a vnode w/o a usecount is
  * referenced which costs us taking and dropping a global lock twice.
  */
-#define UNAGE_THRESHHOLD	10
+#define UNAGE_THRESHHOLD	25
 
-errno_t
-vnode_getiocount(vnode_t vp, int locked, int vid, int vflags)
+static errno_t
+vnode_getiocount(vnode_t vp, int vid, int vflags)
 {
 	int nodead = vflags & VNODE_NODEAD;
 	int nosusp = vflags & VNODE_NOSUSPEND;
 
-	if (!locked)
-		vnode_lock(vp);
-
 	for (;;) {
 		/*
 		 * if it is a dead vnode with deadfs
 		 */
-		if (nodead && (vp->v_lflag & VL_DEAD) && ((vp->v_type == VBAD) || (vp->v_data == 0))) {
-			if (!locked)
-				vnode_unlock(vp);
+	        if (nodead && (vp->v_lflag & VL_DEAD) && ((vp->v_type == VBAD) || (vp->v_data == 0))) {
 			return(ENOENT);
 		}
 		/*
@@ -3350,8 +3711,6 @@ vnode_getiocount(vnode_t vp, int locked, int vid, int vflags)
 		 * if suspended vnodes are to be failed
 		 */
 		if (nosusp && (vp->v_lflag & VL_SUSPENDED)) {
-			if (!locked)
-				vnode_unlock(vp);
 			return(ENOENT);
 		}
 		/*
@@ -3362,16 +3721,16 @@ vnode_getiocount(vnode_t vp, int locked, int vid, int vflags)
 		    (vp->v_owner == current_thread())) {
 		        break;
 		}
+		vnode_lock_convert(vp);
+
 		if (vp->v_lflag & VL_TERMINATE) {
 			vp->v_lflag |= VL_TERMWANT;
 
-			msleep(&vp->v_lflag,   &vp->v_lock, PVFS, "vnode getiocount", 0);
+			msleep(&vp->v_lflag,   &vp->v_lock, PVFS, "vnode getiocount", NULL);
 		} else
-			msleep(&vp->v_iocount, &vp->v_lock, PVFS, "vnode_getiocount", 0);
+			msleep(&vp->v_iocount, &vp->v_lock, PVFS, "vnode_getiocount", NULL);
 	}
 	if (vid != vp->v_id) {
-		if (!locked)
-			vnode_unlock(vp);
 		return(ENOENT);
 	}
 	if (++vp->v_references >= UNAGE_THRESHHOLD) {
@@ -3382,40 +3741,35 @@ vnode_getiocount(vnode_t vp, int locked, int vid, int vflags)
 #ifdef JOE_DEBUG
 	record_vp(vp, 1);
 #endif
-	if (!locked)
-	        vnode_unlock(vp);
 	return(0);	
 }
 
 static void
-vnode_dropiocount (vnode_t vp, int locked)
+vnode_dropiocount (vnode_t vp)
 {
-	if (!locked)
-		vnode_lock(vp);
 	if (vp->v_iocount < 1)
-		panic("vnode_dropiocount(%x): v_iocount < 1", vp);
+		panic("vnode_dropiocount(%p): v_iocount < 1", vp);
 
 	vp->v_iocount--;
 #ifdef JOE_DEBUG
 	record_vp(vp, -1);
 #endif
-	if ((vp->v_lflag & (VL_DRAIN | VL_SUSPENDED)) && (vp->v_iocount <= 1))
+	if ((vp->v_lflag & (VL_DRAIN | VL_SUSPENDED)) && (vp->v_iocount <= 1)) {
+	        vnode_lock_convert(vp);
 		wakeup(&vp->v_iocount);
-
-	if (!locked)
-		vnode_unlock(vp);
+	}
 }
 
 
 void
 vnode_reclaim(struct vnode * vp)
 {
-	vnode_reclaim_internal(vp, 0, 0);
+	vnode_reclaim_internal(vp, 0, 0, 0);
 }
 
 __private_extern__
 void
-vnode_reclaim_internal(struct vnode * vp, int locked, int reuse)
+vnode_reclaim_internal(struct vnode * vp, int locked, int reuse, int flags)
 {
 	int isfifo = 0;
 
@@ -3427,6 +3781,8 @@ vnode_reclaim_internal(struct vnode * vp, int locked, int reuse)
 	}
 	vp->v_lflag |= VL_TERMINATE;
 
+	vn_clearunionwait(vp, 1);
+
 	if (vnode_drain(vp)) {
 		panic("vnode drain failed");
 		vnode_unlock(vp);
@@ -3435,7 +3791,7 @@ vnode_reclaim_internal(struct vnode * vp, int locked, int reuse)
 	isfifo = (vp->v_type == VFIFO);
 
 	if (vp->v_type != VBAD)
-		vgone(vp);		/* clean and reclaim the vnode */
+		vgone(vp, flags);		/* clean and reclaim the vnode */
 
 	/*
 	 * give the vnode a new identity so
@@ -3464,18 +3820,24 @@ vnode_reclaim_internal(struct vnode * vp, int locked, int reuse)
 	if (vp->v_name)
 	        panic("vnode_reclaim_internal: vname not removed");
 
-	vp->v_socket = 0;
+	vp->v_socket = NULL;
 
 	vp->v_lflag &= ~VL_TERMINATE;
 	vp->v_lflag &= ~VL_DRAIN;
-	vp->v_owner = 0;
+	vp->v_owner = NULL;
 
 	if (vp->v_lflag & VL_TERMWANT) {
 		vp->v_lflag &= ~VL_TERMWANT;
 		wakeup(&vp->v_lflag);
 	}
-	if (!reuse && vp->v_usecount == 0)
+	if (!reuse && vp->v_usecount == 0) {
+	        /*
+		 * make sure we get on the
+		 * dead list
+		 */
+	        vnode_list_remove(vp);
 	        vnode_list_add(vp);
+	}
 	if (!locked)
 	        vnode_unlock(vp);
 }
@@ -3494,6 +3856,7 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 	vnode_t vp;
 	vnode_t nvp;
 	vnode_t dvp;
+        struct  uthread *ut;
 	struct componentname *cnp;
 	struct vnode_fsparam *param = (struct vnode_fsparam *)data;
 	
@@ -3513,16 +3876,13 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 				vp->v_flag |= VROOT;
 			if (param->vnfs_marksystem)
 				vp->v_flag |= VSYSTEM;
-			else if (vp->v_type == VREG) {
-				/*
-				 * only non SYSTEM vp
-				 */
+			if (vp->v_type == VREG) {
 				error = ubc_info_init_withsize(vp, param->vnfs_filesize);
 				if (error) {
 #ifdef JOE_DEBUG
 				        record_vp(vp, 1);
 #endif
-					vp->v_mount = 0;
+					vp->v_mount = NULL;
 					vp->v_op = dead_vnodeop_p;
 					vp->v_tag = VT_NON;
 					vp->v_data = NULL;
@@ -3538,6 +3898,8 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 #endif
 			if (vp->v_type == VCHR || vp->v_type == VBLK) {
                 
+				vp->v_tag = VT_DEVFS;		/* callers will reset if needed (bdevvp) */
+
 				if ( (nvp = checkalias(vp, param->vnfs_rdev)) ) {
 				        /*
 					 * if checkalias returns a vnode, it will be locked
@@ -3558,7 +3920,7 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 					 */
 					vp = nvp;
 
-					vclean(vp, 0, current_proc());
+					vclean(vp, 0);
 					vp->v_op = param->vnfs_vops;
 					vp->v_type = param->vnfs_vtype;
 					vp->v_data = param->vnfs_fsnode;
@@ -3585,6 +3947,10 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 			 */
 			*vpp = vp;
 
+			/* Add fs named reference. */
+			if (param->vnfs_flags & VNFS_ADDFSREF) {
+				vp->v_lflag |= VNAMED_FSHASH;
+			}
 			if (param->vnfs_mp) {
 					if (param->vnfs_mp->mnt_kern_flag & MNTK_LOCK_LOCAL)
 						vp->v_flag |= VLOCKLOCAL;
@@ -3616,6 +3982,8 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 					cache_enter(dvp, vp, cnp);
 				}
 				vp->v_name = vfs_addname(cnp->cn_nameptr, cnp->cn_namelen, cnp->cn_hash, 0);
+				if ((cnp->cn_flags & UNIONCREATED) == UNIONCREATED)
+					vp->v_flag |= VISUNION;
 			}
 			if ((param->vnfs_flags & VNFS_CANTCACHE) == 0) {
 			        /*
@@ -3624,9 +3992,18 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 				 */
 			        vp->v_flag |= VNCACHEABLE;
 			}
-			if ((vp->v_flag & VSYSTEM) && (vp->v_type != VREG))
-			        panic("incorrect vnode setup");
-
+			ut = get_bsdthread_info(current_thread());
+		
+			if ((current_proc()->p_lflag & P_LRAGE_VNODES) ||
+			    (ut->uu_flag & UT_RAGE_VNODES)) {
+			        /*
+				 * process has indicated that it wants any
+				 * vnodes created on its behalf to be rapidly
+				 * aged to reduce the impact on the cached set
+				 * of vnodes
+				 */
+			        vp->v_flag |= VRAGE;
+			}
 			return(0);
 		}
 	}
@@ -3636,7 +4013,7 @@ vnode_create(int flavor, size_t size, void *data, vnode_t *vpp)
 int
 vnode_addfsref(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	if (vp->v_lflag & VNAMED_FSHASH)
 		panic("add_fsref: vp already has named reference");
 	if ((vp->v_freelist.tqe_prev != (struct vnode **)0xdeadb))
@@ -3649,7 +4026,7 @@ vnode_addfsref(vnode_t vp)
 int
 vnode_removefsref(vnode_t vp)
 {
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	if ((vp->v_lflag & VNAMED_FSHASH) == 0)
 		panic("remove_fsref: no named reference");
 	vp->v_lflag &= ~VNAMED_FSHASH;
@@ -3723,9 +4100,12 @@ out:
 
 /*
  * Update the vfsstatfs structure in the mountpoint.
+ * MAC: Parameter eventtype added, indicating whether the event that
+ * triggered this update came from user space, via a system call
+ * (VFS_USER_EVENT) or an internal kernel call (VFS_KERNEL_EVENT).
  */
 int
-vfs_update_vfsstat(mount_t mp, vfs_context_t ctx)
+vfs_update_vfsstat(mount_t mp, vfs_context_t ctx, __unused int eventtype)
 {
 	struct vfs_attr	va;
 	int		error;
@@ -3744,6 +4124,14 @@ vfs_update_vfsstat(mount_t mp, vfs_context_t ctx)
 	VFSATTR_WANTED(&va, f_ffree);
 	VFSATTR_WANTED(&va, f_bsize);
 	VFSATTR_WANTED(&va, f_fssubtype);
+#if CONFIG_MACF
+	if (eventtype == VFS_USER_EVENT) {
+		error = mac_mount_check_getattr(ctx, mp, &va);
+		if (error != 0)
+			return (error);
+	}
+#endif
+
 	if ((error = vfs_getattr(mp, &va, ctx)) != 0) {
 		KAUTH_DEBUG("STAT - filesystem returned error %d", error);
 		return(error);
@@ -3768,7 +4156,8 @@ vfs_update_vfsstat(mount_t mp, vfs_context_t ctx)
 	 *
 	 */
 	if (VFSATTR_IS_SUPPORTED(&va, f_bsize)) {
-		mp->mnt_vfsstat.f_bsize = va.f_bsize;
+		/* 4822056 - protect against malformed server mount */
+		mp->mnt_vfsstat.f_bsize = (va.f_bsize > 0 ? va.f_bsize : 512);
 	} else {
 		mp->mnt_vfsstat.f_bsize = mp->mnt_devblocksize;	/* default from the device block size */
 	}
@@ -3812,21 +4201,24 @@ mount_list_remove(mount_t mp)
 	mount_list_lock();
 	TAILQ_REMOVE(&mountlist, mp, mnt_list);
 	nummounts--;
-	mp->mnt_list.tqe_next = 0;
-	mp->mnt_list.tqe_prev = 0;
+	mp->mnt_list.tqe_next = NULL;
+	mp->mnt_list.tqe_prev = NULL;
 	mount_list_unlock();
 }
 
+#if CONFIG_VOLFS
 mount_t
 mount_lookupby_volfsid(int volfs_id, int withref)
 {
 	mount_t cur_mount = (mount_t)0;
-	mount_t mp ;
+	mount_t mp;
 
 	mount_list_lock();
-	TAILQ_FOREACH(mp, &mountlist, mnt_list) { 
-		if (validfsnode(mp) && mp->mnt_vfsstat.f_fsid.val[0] == volfs_id) {
-            cur_mount = mp;
+	TAILQ_FOREACH(mp, &mountlist, mnt_list) {
+		if (!(mp->mnt_kern_flag & MNTK_UNMOUNT) &&
+		    (mp->mnt_kern_flag & MNTK_PATH_FROM_ID) &&
+		    (mp->mnt_vfsstat.f_fsid.val[0] == volfs_id)) {
+			cur_mount = mp;
 			if (withref) {
 				if (mount_iterref(cur_mount, 1))  {
 					cur_mount = (mount_t)0;
@@ -3834,27 +4226,25 @@ mount_lookupby_volfsid(int volfs_id, int withref)
 					goto out;
 				}
 			}
-            break;
-          }
+			break;
+		}
 	}
 	mount_list_unlock();
 	if (withref && (cur_mount != (mount_t)0)) {
 		mp = cur_mount;
 		if (vfs_busy(mp, LK_NOWAIT) != 0) {
 			cur_mount = (mount_t)0;
-        } 
+		}
 		mount_iterdrop(mp);
 	}
 out:
 	return(cur_mount);
 }
+#endif
 
 
 mount_t 
-mount_list_lookupby_fsid(fsid, locked, withref)
-	fsid_t *fsid;
-	int locked;
-	int withref;
+mount_list_lookupby_fsid(fsid_t *fsid, int locked, int withref)
 {
 	mount_t retmp = (mount_t)0;
 	mount_t mp;
@@ -3878,18 +4268,14 @@ out:
 }
 
 errno_t
-vnode_lookup(const char *path, int flags, vnode_t *vpp, vfs_context_t context)
+vnode_lookup(const char *path, int flags, vnode_t *vpp, vfs_context_t ctx)
 {
 	struct nameidata nd;
 	int error;
-	struct vfs_context context2;
-	vfs_context_t ctx = context;
 	u_long ndflags = 0;
 
-	if (context == NULL) {		/* XXX technically an error */
-		context2.vc_proc = current_proc();
-		context2.vc_ucred = kauth_cred_get();
-		ctx = &context2;
+	if (ctx == NULL) {		/* XXX technically an error */
+		ctx = vfs_context_current();
 	}
 
 	if (flags & VNODE_LOOKUP_NOFOLLOW)
@@ -3914,19 +4300,15 @@ vnode_lookup(const char *path, int flags, vnode_t *vpp, vfs_context_t context)
 }
 
 errno_t
-vnode_open(const char *path, int fmode, int cmode, int flags, vnode_t *vpp, vfs_context_t context)
+vnode_open(const char *path, int fmode, int cmode, int flags, vnode_t *vpp, vfs_context_t ctx)
 {
 	struct nameidata nd;
 	int error;
-	struct vfs_context context2;
-	vfs_context_t ctx = context;
 	u_long ndflags = 0;
 	int lflags = flags;
 
-	if (context == NULL) {		/* XXX technically an error */
-		context2.vc_proc = current_proc();
-		context2.vc_ucred = kauth_cred_get();
-		ctx = &context2;
+	if (ctx == NULL) {		/* XXX technically an error */
+		ctx = vfs_context_current();
 	}
 
 	if (fmode & O_NOFOLLOW)
@@ -3954,25 +4336,23 @@ vnode_open(const char *path, int fmode, int cmode, int flags, vnode_t *vpp, vfs_
 }
 
 errno_t
-vnode_close(vnode_t vp, int flags, vfs_context_t context)
+vnode_close(vnode_t vp, int flags, vfs_context_t ctx)
 {
-	kauth_cred_t cred;
-	struct proc *p;
 	int error;
 
-	if (context) {
-		p = context->vc_proc;
-		cred = context->vc_ucred;
-	} else {
-		p = current_proc();
-		cred = kauth_cred_get();
+	if (ctx == NULL) {
+		ctx = vfs_context_current();
 	}
 	
-	error = vn_close(vp, flags, cred, p);
+	error = vn_close(vp, flags, ctx);
 	vnode_put(vp);
 	return (error);
 }
 
+/*
+ * Returns:	0			Success
+ *	vnode_getattr:???
+ */
 errno_t
 vnode_size(vnode_t vp, off_t *sizep, vfs_context_t ctx)
 {
@@ -4120,6 +4500,15 @@ vn_create(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, struct vnode_att
 	}
 
 	vp = *vpp;
+#if CONFIG_MACF
+	if (!(flags & VN_CREATE_NOLABEL)) {
+		error = vnode_label(vnode_mount(vp), dvp, vp, cnp,
+		    VNODE_LABEL_CREATE|VNODE_LABEL_NEEDREF, ctx);
+		if (error)
+			goto error;
+	}
+#endif
+
 	/*
 	 * If some of the requested attributes weren't handled by the VNOP,
 	 * use our fallback code.
@@ -4128,6 +4517,9 @@ vn_create(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, struct vnode_att
 		KAUTH_DEBUG("     CREATE - doing fallback with ACL %p", vap->va_acl);
 		error = vnode_setattr_fallback(*vpp, vap, ctx);
 	}
+#if CONFIG_MACF
+error:
+#endif
 	if ((error != 0 ) && (vp != (vnode_t)0)) {
 		*vpp = (vnode_t) 0;
 		vnode_put(vp);
@@ -4151,7 +4543,9 @@ out:
 }
 
 static kauth_scope_t	vnode_scope;
-static int	vnode_authorize_callback(kauth_cred_t credential, __unused void *idata, kauth_action_t action,
+static int	vnode_authorize_callback(kauth_cred_t credential, void *idata, kauth_action_t action,
+    uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
+static int	vnode_authorize_callback_int(__unused kauth_cred_t credential, __unused void *idata, kauth_action_t action,
     uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3);
 
 typedef struct _vnode_authorize_context {
@@ -4178,9 +4572,22 @@ vnode_authorize_init(void)
  * Authorize an operation on a vnode.
  *
  * This is KPI, but here because it needs vnode_scope.
+ *
+ * Returns:	0			Success
+ *	kauth_authorize_action:EPERM	...
+ *	xlate => EACCES			Permission denied
+ *	kauth_authorize_action:0	Success
+ *	kauth_authorize_action:		Depends on callback return; this is
+ *					usually only vnode_authorize_callback(),
+ *					but may include other listerners, if any
+ *					exist.
+ *		EROFS
+ *		EACCES
+ *		EPERM
+ *		???
  */
 int
-vnode_authorize(vnode_t vp, vnode_t dvp, kauth_action_t action, vfs_context_t context)
+vnode_authorize(vnode_t vp, vnode_t dvp, kauth_action_t action, vfs_context_t ctx)
 {
 	int	error, result;
 
@@ -4192,13 +4599,13 @@ vnode_authorize(vnode_t vp, vnode_t dvp, kauth_action_t action, vfs_context_t co
 		return(0);
 	
 	error = 0;
-	result = kauth_authorize_action(vnode_scope, vfs_context_ucred(context), action,
-		   (uintptr_t)context, (uintptr_t)vp, (uintptr_t)dvp, (uintptr_t)&error);
+	result = kauth_authorize_action(vnode_scope, vfs_context_ucred(ctx), action,
+		   (uintptr_t)ctx, (uintptr_t)vp, (uintptr_t)dvp, (uintptr_t)&error);
 	if (result == EPERM)		/* traditional behaviour */
 		result = EACCES;
 	/* did the lower layers give a better error return? */
 	if ((result != 0) && (error != 0))
-		return(error);
+	        return(error);
 	return(result);
 }
 
@@ -4380,7 +4787,7 @@ vnode_authorize_posix(vauth_ctx vcp, int action, int on_dir)
 	struct vnode_attr *vap;
 	int needed, error, owner_ok, group_ok, world_ok, ismember;
 #ifdef KAUTH_DEBUG_ENABLE
-	const char *where;
+	const char *where = "uninitialized";
 # define _SETWHERE(c)	where = c;
 #else
 # define _SETWHERE(c)
@@ -4620,7 +5027,7 @@ vnode_authorize_delete(vauth_ctx vcp)
  * Authorize an operation based on the node's attributes.
  */
 static int
-vnode_authorize_simple(vauth_ctx vcp, kauth_ace_rights_t acl_rights, kauth_ace_rights_t preauth_rights)
+vnode_authorize_simple(vauth_ctx vcp, kauth_ace_rights_t acl_rights, kauth_ace_rights_t preauth_rights, boolean_t *found_deny)
 {
 	struct vnode_attr	*vap = vcp->vap;
 	kauth_cred_t		cred = vcp->ctx->vc_ucred;
@@ -4683,6 +5090,8 @@ vnode_authorize_simple(vauth_ctx vcp, kauth_ace_rights_t acl_rights, kauth_ace_r
 			KAUTH_DEBUG("%p    ALLOWED - all rights granted by ACL", vcp->vp);
 			return(0);
 		}
+		*found_deny = eval.ae_found_deny;
+
 		/* fall through and evaluate residual rights */
 	} else {
 		/* no ACL, everything is residual */
@@ -4820,7 +5229,7 @@ vnode_authorize_checkimmutable(vnode_t vp, struct vnode_attr *vap, int rights, i
 	if (rights & KAUTH_VNODE_WRITE_RIGHTS) {
 		
 		/* check per-filesystem options if possible */
-		mp = vnode_mount(vp);
+		mp = vp->v_mount;
 		if (mp != NULL) {
 	
 			/* check for no-EA filesystems */
@@ -4851,8 +5260,19 @@ out:
 }
 
 /*
- * Handle authorization actions for filesystems that advertise that the server will
- * be enforcing.
+ * Handle authorization actions for filesystems that advertise that the
+ * server will be enforcing.
+ *
+ * Returns:	0			Authorization should be handled locally
+ *		1			Authorization was handled by the FS
+ *
+ * Note:	Imputed returns will only occur if the authorization request
+ *		was handled by the FS.
+ *
+ * Imputed:	*resultp, modified	Return code from FS when the request is
+ *					handled by the FS.
+ *		VNOP_ACCESS:???
+ *		VNOP_OPEN:???
  */
 static int
 vnode_authorize_opaque(vnode_t vp, int *resultp, kauth_action_t action, vfs_context_t ctx)
@@ -4877,7 +5297,7 @@ vnode_authorize_opaque(vnode_t vp, int *resultp, kauth_action_t action, vfs_cont
 	 * In the advisory request case, if the filesystem doesn't think it's reliable
 	 * we will attempt to formulate a result ourselves based on VNOP_GETATTR data.
 	 */
-	if ((action & KAUTH_VNODE_ACCESS) && !vfs_authopaqueaccess(vnode_mount(vp)))
+	if ((action & KAUTH_VNODE_ACCESS) && !vfs_authopaqueaccess(vp->v_mount))
 		return(0);
 
 	/*
@@ -4894,7 +5314,7 @@ vnode_authorize_opaque(vnode_t vp, int *resultp, kauth_action_t action, vfs_cont
 	 * Typically opaque filesystems do authorisation in-line, but exec is a special case.  In
 	 * order to be reasonably sure that exec will be permitted, we try a bit harder here.
 	 */
-	if ((action & KAUTH_VNODE_EXECUTE) && vnode_isreg(vp)) {
+	if ((action & KAUTH_VNODE_EXECUTE) && (vp->v_type == VREG)) {
 		/* try a VNOP_OPEN for readonly access */
 		if ((error = VNOP_OPEN(vp, FREAD, ctx)) != 0) {
 			*resultp = error;
@@ -4913,8 +5333,72 @@ vnode_authorize_opaque(vnode_t vp, int *resultp, kauth_action_t action, vfs_cont
 	return(1);
 }
 
+
+
+
+/*
+ * Returns:	KAUTH_RESULT_ALLOW
+ *		KAUTH_RESULT_DENY
+ *
+ * Imputed:	*arg3, modified		Error code in the deny case
+ *		EROFS			Read-only file system
+ *		EACCES			Permission denied
+ *		EPERM			Operation not permitted [no execute]
+ *	vnode_getattr:ENOMEM		Not enough space [only if has filesec]
+ *	vnode_getattr:???
+ *	vnode_authorize_opaque:*arg2	???
+ *	vnode_authorize_checkimmutable:???
+ *	vnode_authorize_delete:???
+ *	vnode_authorize_simple:???
+ */
+
+
+static int
+vnode_authorize_callback(kauth_cred_t cred, void *idata, kauth_action_t action,
+			 uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3)
+{
+	vfs_context_t	ctx;
+	vnode_t		cvp = NULLVP;
+	vnode_t		vp, dvp;
+	int		result;
+
+	ctx = (vfs_context_t)arg0;
+	vp = (vnode_t)arg1;
+	dvp = (vnode_t)arg2;
+
+	/*
+	 * if there are 2 vnodes passed in, we don't know at
+	 * this point which rights to look at based on the 
+	 * combined action being passed in... defer until later...
+	 * otherwise check the kauth 'rights' cache hung
+	 * off of the vnode we're interested in... if we've already
+	 * been granted the right we're currently interested in,
+	 * we can just return success... otherwise we'll go through
+	 * the process of authorizing the requested right(s)... if that
+	 * succeeds, we'll add the right(s) to the cache.
+	 * VNOP_SETATTR and VNOP_SETXATTR will invalidate this cache
+	 */
+        if (dvp && vp)
+	        goto defer;
+	if (dvp)
+	        cvp = dvp;
+	else
+	        cvp = vp;
+
+	if (vnode_cache_is_authorized(cvp, ctx, action) == TRUE)
+	        return KAUTH_RESULT_ALLOW;
+defer:
+        result = vnode_authorize_callback_int(cred, idata, action, arg0, arg1, arg2, arg3);
+
+	if (result == KAUTH_RESULT_ALLOW && cvp != NULLVP)
+	        vnode_cache_authorized_action(cvp, ctx, action);
+
+	return result;
+}
+
+
 static int
-vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata, kauth_action_t action,
+vnode_authorize_callback_int(__unused kauth_cred_t unused_cred, __unused void *idata, kauth_action_t action,
     uintptr_t arg0, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3)
 {
 	struct _vnode_authorize_context auth_context;
@@ -4927,13 +5411,19 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	int			result;
 	int			*errorp;
 	int			noimmutable;
+	boolean_t		parent_authorized_for_delete = FALSE;
+	boolean_t		found_deny = FALSE;
+	boolean_t		parent_ref= FALSE;
 
 	vcp = &auth_context;
 	ctx = vcp->ctx = (vfs_context_t)arg0;
 	vp = vcp->vp = (vnode_t)arg1;
 	dvp = vcp->dvp = (vnode_t)arg2;
 	errorp = (int *)arg3;
-	/* note that we authorize against the context, not the passed cred (the same thing anyway) */
+	/*
+	 * Note that we authorize against the context, not the passed cred
+	 * (the same thing anyway)
+	 */
 	cred = ctx->vc_ucred;
 
 	VATTR_INIT(&va);
@@ -4980,6 +5470,14 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 		if (dvp == NULL)
 			panic("vnode_authorize: KAUTH_VNODE_DELETE test requires a directory");
 #endif
+		/*
+		 * check to see if we've already authorized the parent
+		 * directory for deletion of its children... if so, we
+		 * can skip a whole bunch of work... we will still have to
+		 * authorize that this specific child can be removed
+		 */
+		if (vnode_cache_is_authorized(dvp, ctx, KAUTH_VNODE_DELETE) == TRUE)
+		        parent_authorized_for_delete = TRUE;
 	} else {
 		dvp = NULL;
 	}
@@ -4999,7 +5497,7 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	/*
 	 * Check for noexec filesystems.
 	 */
-	if ((rights & KAUTH_VNODE_EXECUTE) && vnode_isreg(vp) && (vp->v_mount->mnt_flag & MNT_NOEXEC)) {
+	if ((rights & KAUTH_VNODE_EXECUTE) && (vp->v_type == VREG) && (vp->v_mount->mnt_flag & MNT_NOEXEC)) {
 		result = EACCES;
 		goto out;
 	}
@@ -5010,7 +5508,7 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	 * check based on VNOP_GETATTR data.  Otherwise it returns 1 and sets
 	 * an appropriate result, at which point we can return immediately.
 	 */
-	if (vfs_authopaque(vp->v_mount) && vnode_authorize_opaque(vp, &result, action, ctx))
+	if ((vp->v_mount->mnt_kern_flag & MNTK_AUTH_OPAQUE) && vnode_authorize_opaque(vp, &result, action, ctx))
 		goto out;
 
 	/*
@@ -5026,7 +5524,7 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 		KAUTH_DEBUG("%p    ERROR - failed to get vnode attributes - %d", vp, result);
 		goto out;
 	}
-	if (dvp) {
+	if (dvp && parent_authorized_for_delete == FALSE) {
 		VATTR_WANTED(&dva, va_mode);
 		VATTR_WANTED(&dva, va_uid);
 		VATTR_WANTED(&dva, va_gid);
@@ -5042,7 +5540,7 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	 * If the vnode is an extended attribute data vnode (eg. a resource fork), *_DATA becomes
 	 * *_EXTATTRIBUTES.
 	 */
-	if (S_ISXATTR(va.va_mode)) {
+	if (S_ISXATTR(va.va_mode) || vnode_isnamedstream(vp)) {
 		if (rights & KAUTH_VNODE_READ_DATA) {
 			rights &= ~KAUTH_VNODE_READ_DATA;
 			rights |= KAUTH_VNODE_READ_EXTATTRIBUTES;
@@ -5052,7 +5550,27 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 			rights |= KAUTH_VNODE_WRITE_EXTATTRIBUTES;
 		}
 	}
-	
+
+	/*
+	 * Point 'vp' to the resource fork's parent for ACL checking
+	 */
+	if (vnode_isnamedstream(vp) &&
+	    (vp->v_parent != NULL) &&
+	    (vget_internal(vp->v_parent, 0, VNODE_NODEAD) == 0)) {
+		parent_ref = TRUE;
+		vcp->vp = vp = vp->v_parent;
+		if (VATTR_IS_SUPPORTED(&va, va_acl) && (va.va_acl != NULL))
+			kauth_acl_free(va.va_acl);
+		VATTR_INIT(&va);
+		VATTR_WANTED(&va, va_mode);
+		VATTR_WANTED(&va, va_uid);
+		VATTR_WANTED(&va, va_gid);
+		VATTR_WANTED(&va, va_flags);
+		VATTR_WANTED(&va, va_acl);
+		if ((result = vnode_getattr(vp, &va, ctx)) != 0)
+			goto out;
+	}
+
 	/*
 	 * Check for immutability.
 	 *
@@ -5062,6 +5580,7 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	if ((result = vnode_authorize_checkimmutable(vp, &va, rights, noimmutable)) != 0)
 		goto out;
 	if ((rights & KAUTH_VNODE_DELETE) &&
+	    parent_authorized_for_delete == FALSE &&
 	    ((result = vnode_authorize_checkimmutable(dvp, &dva, KAUTH_VNODE_DELETE_CHILD, 0)) != 0))
 		goto out;
 
@@ -5079,12 +5598,13 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 	if (!vfs_context_issuser(ctx)) {
 		/* process delete rights */
 		if ((rights & KAUTH_VNODE_DELETE) &&
+		    parent_authorized_for_delete == FALSE &&
 		    ((result = vnode_authorize_delete(vcp)) != 0))
 		    goto out;
 
 		/* process remaining rights */
 		if ((rights & ~KAUTH_VNODE_DELETE) &&
-		    ((result = vnode_authorize_simple(vcp, rights, rights & KAUTH_VNODE_DELETE)) != 0))
+		    (result = vnode_authorize_simple(vcp, rights, rights & KAUTH_VNODE_DELETE, &found_deny)) != 0)
 			goto out;
 	} else {
 
@@ -5103,18 +5623,42 @@ vnode_authorize_callback(__unused kauth_cred_t unused_cred, __unused void *idata
 		
 		KAUTH_DEBUG("%p    ALLOWED - caller is superuser", vp);
 	}
-
 out:
 	if (VATTR_IS_SUPPORTED(&va, va_acl) && (va.va_acl != NULL))
 		kauth_acl_free(va.va_acl);
 	if (VATTR_IS_SUPPORTED(&dva, va_acl) && (dva.va_acl != NULL))
 		kauth_acl_free(dva.va_acl);
+
 	if (result) {
+		if (parent_ref)
+			vnode_put(vp);
 		*errorp = result;
 		KAUTH_DEBUG("%p    DENIED - auth denied", vp);
 		return(KAUTH_RESULT_DENY);
 	}
-
+	if ((rights & KAUTH_VNODE_SEARCH) && found_deny == FALSE && vp->v_type == VDIR) {
+	        /*
+		 * if we were successfully granted the right to search this directory
+		 * and there were NO ACL DENYs for search and the posix permissions also don't
+		 * deny execute, we can synthesize a global right that allows anyone to 
+		 * traverse this directory during a pathname lookup without having to
+		 * match the credential associated with this cache of rights.
+		 */
+	        if (!VATTR_IS_SUPPORTED(&va, va_mode) ||
+		    ((va.va_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) ==
+		     (S_IXUSR | S_IXGRP | S_IXOTH))) {
+		        vnode_cache_authorized_action(vp, ctx, KAUTH_VNODE_SEARCHBYANYONE);
+		}
+	}
+	if ((rights & KAUTH_VNODE_DELETE) && parent_authorized_for_delete == FALSE) {
+	        /*
+		 * parent was successfully and newly authorized for deletions
+		 * add it to the cache
+		 */
+	        vnode_cache_authorized_action(dvp, ctx, KAUTH_VNODE_DELETE);
+	}
+	if (parent_ref)
+		vnode_put(vp);
 	/*
 	 * Note that this implies that we will allow requests for no rights, as well as
 	 * for rights that we do not recognise.  There should be none of these.
@@ -5131,7 +5675,7 @@ int
 vnode_authattr_new(vnode_t dvp, struct vnode_attr *vap, int noauth, vfs_context_t ctx)
 {
 	int		error;
-	int		is_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode;
+	int		has_priv_suser, ismember, defaulted_owner, defaulted_group, defaulted_mode;
 	kauth_cred_t	cred;
 	guid_t		changer;
 	mount_t		dmp;
@@ -5221,7 +5765,7 @@ vnode_authattr_new(vnode_t dvp, struct vnode_attr *vap, int noauth, vfs_context_
 	 * Quickly check for the applicability of any enforcement here.
 	 * Tests below maintain the integrity of the local security model.
 	 */
-	if (vfs_authopaque(vnode_mount(dvp)))
+	if (vfs_authopaque(dvp->v_mount))
 	    goto out;
 
 	/*
@@ -5231,14 +5775,14 @@ vnode_authattr_new(vnode_t dvp, struct vnode_attr *vap, int noauth, vfs_context_
 	cred = vfs_context_ucred(ctx);
 	if (noauth) {
 		/* doing work for the kernel */
-		is_suser = 1;
+		has_priv_suser = 1;
 	} else {
-		is_suser = vfs_context_issuser(ctx);
+		has_priv_suser = vfs_context_issuser(ctx);
 	}
 
 
 	if (VATTR_IS_ACTIVE(vap, va_flags)) {
-		if (is_suser) {
+		if (has_priv_suser) {
 			if ((vap->va_flags & (UF_SETTABLE | SF_SETTABLE)) != vap->va_flags) {
 				error = EPERM;
 				KAUTH_DEBUG("  DENIED - superuser attempt to set illegal flag(s)");
@@ -5254,7 +5798,7 @@ vnode_authattr_new(vnode_t dvp, struct vnode_attr *vap, int noauth, vfs_context_
 	}
 
 	/* if not superuser, validate legality of new-item attributes */
-	if (!is_suser) {
+	if (!has_priv_suser) {
 		if (!defaulted_mode && VATTR_IS_ACTIVE(vap, va_mode)) {
 			/* setgid? */
 			if (vap->va_mode & S_ISGID) {
@@ -5323,10 +5867,11 @@ out:
 }
 
 /*
- * Check that the attribute information in vap can be legally written by the context.
+ * Check that the attribute information in vap can be legally written by the
+ * context.
  *
- * Call this when you're not sure about the vnode_attr; either its contents have come
- * from an unknown source, or when they are variable.
+ * Call this when you're not sure about the vnode_attr; either its contents
+ * have come from an unknown source, or when they are variable.
  *
  * Returns errno, or zero and sets *actionp to the KAUTH_VNODE_* actions that
  * must be authorized to be permitted to write the vattr.
@@ -5336,7 +5881,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 {
 	struct vnode_attr ova;
 	kauth_action_t	required_action;
-	int		error, is_suser, ismember, chowner, chgroup;
+	int		error, has_priv_suser, ismember, chowner, chgroup, clear_suid, clear_sgid;
 	guid_t		changer;
 	gid_t		group;
 	uid_t		owner;
@@ -5351,7 +5896,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 	/*
 	 * Quickly check for enforcement applicability.
 	 */
-	if (vfs_authopaque(vnode_mount(vp)))
+	if (vfs_authopaque(vp->v_mount))
 		goto out;
 	
 	/*
@@ -5367,7 +5912,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 	 * We need to know if the caller is the superuser.
 	 */
 	cred = vfs_context_ucred(ctx);
-	is_suser = kauth_cred_issuser(cred);
+	has_priv_suser = kauth_cred_issuser(cred);
 	
 	/*
 	 * If any of the following are changing, we need information from the old file:
@@ -5465,7 +6010,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		 * provided that the file is not immutable.  The owner still needs
 		 * WRITE_ATTRIBUTES (implied by ownership but still deniable).
 		 */
-		if (is_suser || vauth_node_owner(&ova, cred)) {
+		if (has_priv_suser || vauth_node_owner(&ova, cred)) {
 			KAUTH_DEBUG("ATTR - root or owner changing timestamps");
 			required_action |= KAUTH_VNODE_CHECKIMMUTABLE | KAUTH_VNODE_WRITE_ATTRIBUTES;
 		} else {
@@ -5490,7 +6035,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		/*
 		 * Mode changes always have the same basic auth requirements.
 		 */
-		if (is_suser) {
+		if (has_priv_suser) {
 			KAUTH_DEBUG("ATTR - superuser mode change, requiring immutability check");
 			required_action |= KAUTH_VNODE_CHECKIMMUTABLE;
 		} else {
@@ -5505,7 +6050,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		 */
 		if (vap->va_mode & S_ISGID) {
 			required_action |= KAUTH_VNODE_CHECKIMMUTABLE;	/* always required */
-			if (!is_suser) {
+			if (!has_priv_suser) {
 				if (VATTR_IS_ACTIVE(vap, va_gid)) {
 					group = vap->va_gid;
 				} else if (VATTR_IS_SUPPORTED(&ova, va_gid)) {
@@ -5536,7 +6081,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		 */
 		if (vap->va_mode & S_ISUID) {
 			required_action |= KAUTH_VNODE_CHECKIMMUTABLE;	/* always required */
-			if (!is_suser) {
+			if (!has_priv_suser) {
 				if (VATTR_IS_ACTIVE(vap, va_uid)) {
 					owner = vap->va_uid;
 				} else if (VATTR_IS_SUPPORTED(&ova, va_uid)) {
@@ -5581,7 +6126,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 			required_action |= KAUTH_VNODE_WRITE_SECURITY;
 
 			/* check that changing bits are legal */
-			if (is_suser) {
+			if (has_priv_suser) {
 				/*
 				 * The immutability check will prevent us from clearing the SF_*
 				 * flags unless the system securelevel permits it, so just check
@@ -5619,6 +6164,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 	 */
 	chowner = 0;
 	chgroup = 0;
+	clear_suid = 0;
+	clear_sgid = 0;
 
 	/*
 	 * uid changing
@@ -5626,14 +6173,17 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 	 * support them in general, and will ignore it if/when we try to set it.
 	 * We might want to clear the uid out of vap completely here.
 	 */
-	if (VATTR_IS_ACTIVE(vap, va_uid) && VATTR_IS_SUPPORTED(&ova, va_uid) && (vap->va_uid != ova.va_uid)) {
-		if (!is_suser && (kauth_cred_getuid(cred) != vap->va_uid)) {
+	if (VATTR_IS_ACTIVE(vap, va_uid)) {
+		if (VATTR_IS_SUPPORTED(&ova, va_uid) && (vap->va_uid != ova.va_uid)) {
+		if (!has_priv_suser && (kauth_cred_getuid(cred) != vap->va_uid)) {
 			KAUTH_DEBUG("  DENIED - non-superuser cannot change ownershipt to a third party");
 			error = EPERM;
 			goto out;
 		}
 		chowner = 1;
 	}
+		clear_suid = 1;
+	}
 	
 	/*
 	 * gid changing
@@ -5641,8 +6191,9 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 	 * support them in general, and will ignore it if/when we try to set it.
 	 * We might want to clear the gid out of vap completely here.
 	 */
-	if (VATTR_IS_ACTIVE(vap, va_gid) && VATTR_IS_SUPPORTED(&ova, va_gid) && (vap->va_gid != ova.va_gid)) {
-		if (!is_suser) {
+	if (VATTR_IS_ACTIVE(vap, va_gid)) {
+		if (VATTR_IS_SUPPORTED(&ova, va_gid) && (vap->va_gid != ova.va_gid)) {
+		if (!has_priv_suser) {
 			if ((error = kauth_cred_ismember_gid(cred, vap->va_gid, &ismember)) != 0) {
 				KAUTH_DEBUG("  ERROR - got %d checking for membership in %d", error, vap->va_gid);
 				goto out;
@@ -5656,6 +6207,8 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		}
 		chgroup = 1;
 	}
+		clear_sgid = 1;
+	}
 
 	/*
 	 * Owner UUID being set or changed.
@@ -5669,7 +6222,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 		 * The owner UUID cannot be set by a non-superuser to anything other than
 		 * their own.
 		 */
-		if (!is_suser) {
+		if (!has_priv_suser) {
 			if ((error = kauth_cred_getguid(cred, &changer)) != 0) {
 				KAUTH_DEBUG("  ERROR - got %d trying to get caller UUID", error);
 				/* XXX ENOENT here - no UUID - should perhaps become EPERM */
@@ -5682,6 +6235,7 @@ vnode_authattr(vnode_t vp, struct vnode_attr *vap, kauth_action_t *actionp, vfs_
 			}
 		}
 		chowner = 1;
+		clear_suid = 1;
 	}
 no_uuuid_change:
 	/*
@@ -5696,7 +6250,7 @@ no_uuuid_change:
 		 * The group UUID cannot be set by a non-superuser to anything other than
 		 * one of which they are a member.
 		 */
-		if (!is_suser) {
+		if (!has_priv_suser) {
 			if ((error = kauth_cred_ismember_guid(cred, &vap->va_guuid, &ismember)) != 0) {
 				KAUTH_DEBUG("  ERROR - got %d trying to check group membership", error);
 				goto out;
@@ -5714,8 +6268,8 @@ no_guuid_change:
 	/*
 	 * Compute authorisation for group/ownership changes.
 	 */
-	if (chowner || chgroup) {
-		if (is_suser) {
+	if (chowner || chgroup || clear_suid || clear_sgid) {
+		if (has_priv_suser) {
 			KAUTH_DEBUG("ATTR - superuser changing file owner/group, requiring immutability check");
 			required_action |= KAUTH_VNODE_CHECKIMMUTABLE;
 		} else {
@@ -5819,6 +6373,238 @@ vfs_setlocklocal(mount_t mp)
 	mount_unlock(mp);
 }
 
+void
+vn_setunionwait(vnode_t vp)
+{
+	vnode_lock_spin(vp);
+	vp->v_flag |= VISUNION;
+	vnode_unlock(vp);
+}
+
+
+void
+vn_checkunionwait(vnode_t vp)
+{
+	vnode_lock(vp);
+	while ((vp->v_flag & VISUNION) == VISUNION)
+		msleep((caddr_t)&vp->v_flag, &vp->v_lock, 0, 0, 0);
+	vnode_unlock(vp);
+}
+
+void
+vn_clearunionwait(vnode_t vp, int locked)
+{
+	if (!locked)
+		vnode_lock(vp);
+	if((vp->v_flag & VISUNION) == VISUNION) {
+		vp->v_flag &= ~VISUNION;
+		wakeup((caddr_t)&vp->v_flag);
+	}
+	if (!locked)
+		vnode_unlock(vp);
+}
+
+/*
+ * XXX - get "don't trigger mounts" flag for thread; used by autofs.
+ */
+extern int thread_notrigger(void);
+
+int
+thread_notrigger(void)
+{
+	struct uthread *uth = (struct uthread *)get_bsdthread_info(current_thread());
+	return (uth->uu_notrigger);
+}
+
+/* 
+ * Removes orphaned apple double files during a rmdir
+ * Works by:
+ * 1. vnode_suspend().
+ * 2. Call VNOP_READDIR() till the end of directory is reached.  
+ * 3. Check if the directory entries returned are regular files with name starting with "._".  If not, return ENOTEMPTY.  
+ * 4. Continue (2) and (3) till end of directory is reached.
+ * 5. If all the entries in the directory were files with "._" name, delete all the files.
+ * 6. vnode_resume()
+ * 7. If deletion of all files succeeded, call VNOP_RMDIR() again.
+ */
+
+errno_t rmdir_remove_orphaned_appleDouble(vnode_t vp , vfs_context_t ctx, int * restart_flag) 
+{
+
+#define UIO_BUFF_SIZE 2048
+	uio_t auio = NULL;
+	int eofflag, siz = UIO_BUFF_SIZE, nentries = 0;
+	int open_flag = 0, full_erase_flag = 0;
+	char uio_buf[ UIO_SIZEOF(1) ];
+	char *rbuf = NULL, *cpos, *cend;
+	struct nameidata nd_temp;
+	struct dirent *dp;
+	errno_t error;
+
+	error = vnode_suspend(vp);
+
+	/*
+	 * restart_flag is set so that the calling rmdir sleeps and resets
+	 */
+	if (error == EBUSY)
+		*restart_flag = 1;
+	if (error != 0)
+		goto outsc;
+
+	/*
+	 * set up UIO
+	 */
+	MALLOC(rbuf, caddr_t, siz, M_TEMP, M_WAITOK);
+	if (rbuf)
+		auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
+				&uio_buf[0], sizeof(uio_buf));
+	if (!rbuf || !auio) {
+		error = ENOMEM;
+		goto outsc;
+	}
+
+	uio_setoffset(auio,0);
+
+	eofflag = 0;
+
+	if ((error = VNOP_OPEN(vp, FREAD, ctx))) 
+		goto outsc; 	
+	else
+		open_flag = 1;
+
+	/*
+	 * First pass checks if all files are appleDouble files.
+	 */
+
+	do {
+		siz = UIO_BUFF_SIZE;
+		uio_reset(auio, uio_offset(auio), UIO_SYSSPACE, UIO_READ);
+		uio_addiov(auio, CAST_USER_ADDR_T(rbuf), UIO_BUFF_SIZE);
+
+		if((error = VNOP_READDIR(vp, auio, 0, &eofflag, &nentries, ctx)))
+			goto outsc;
+
+		if (uio_resid(auio) != 0) 
+			siz -= uio_resid(auio);
+
+		/*
+		 * Iterate through directory
+		 */
+		cpos = rbuf;
+		cend = rbuf + siz;
+		dp = (struct dirent*) cpos;
+
+		if (cpos == cend)
+			eofflag = 1;
+
+		while ((cpos < cend)) {
+			/*
+			 * Check for . and .. as well as directories
+			 */
+			if (dp->d_ino != 0 && 
+					!((dp->d_namlen == 1 && dp->d_name[0] == '.') ||
+					    (dp->d_namlen == 2 && dp->d_name[0] == '.' && dp->d_name[1] == '.'))) {
+				/*
+				 * Check for irregular files and ._ files
+				 * If there is a ._._ file abort the op
+				 */
+				if ( dp->d_namlen < 2 ||
+						strncmp(dp->d_name,"._",2) ||
+						(dp->d_namlen >= 4 && !strncmp(&(dp->d_name[2]), "._",2))) {
+					error = ENOTEMPTY;
+					goto outsc;
+				}
+			}
+			cpos += dp->d_reclen;
+			dp = (struct dirent*)cpos;
+		}
+
+	} while (!eofflag);
+	/*
+	 * If we've made it here all the files in the dir are AppleDouble 
+	 * We can delete the files even though the node is suspended
+	 * because we are the owner of the file.
+	 */
+
+	uio_reset(auio, 0, UIO_SYSSPACE, UIO_READ);
+	eofflag = 0;
+
+	do {
+		siz = UIO_BUFF_SIZE;
+		uio_reset(auio, uio_offset(auio), UIO_SYSSPACE, UIO_READ);
+		uio_addiov(auio, CAST_USER_ADDR_T(rbuf), UIO_BUFF_SIZE);
+
+		error = VNOP_READDIR(vp, auio, 0, &eofflag, &nentries, ctx);
+
+		if (error != 0) 
+			goto outsc;
+
+		if (uio_resid(auio) != 0) 
+			siz -= uio_resid(auio);
+
+		/*
+		 * Iterate through directory
+		 */
+		cpos = rbuf;
+		cend = rbuf + siz;
+		dp = (struct dirent*) cpos;
+		
+		if (cpos == cend)
+			eofflag = 1;
+	
+		while ((cpos < cend)) {
+			/*
+			 * Check for . and .. as well as directories
+			 */
+			if (dp->d_ino != 0 && 
+					!((dp->d_namlen == 1 && dp->d_name[0] == '.') ||
+					    (dp->d_namlen == 2 && dp->d_name[0] == '.' && dp->d_name[1] == '.'))
+					  ) {
+				NDINIT(&nd_temp, DELETE, USEDVP, UIO_SYSSPACE, CAST_USER_ADDR_T(dp->d_name), ctx);
+				nd_temp.ni_dvp = vp;
+				error = unlink1(ctx, &nd_temp, 0);
+				if(error && error != ENOENT)
+					goto outsc;
+			}
+			cpos += dp->d_reclen;
+			dp = (struct dirent*)cpos;
+		}
+		
+		/*
+		 * workaround for HFS/NFS setting eofflag before end of file 
+		 */
+		if (vp->v_tag == VT_HFS && nentries > 2)
+			eofflag=0;
+
+		if (vp->v_tag == VT_NFS) {
+			if (eofflag && !full_erase_flag) {
+				full_erase_flag = 1;
+				eofflag = 0;
+				uio_reset(auio, 0, UIO_SYSSPACE, UIO_READ);
+			}
+			else if (!eofflag && full_erase_flag)
+				full_erase_flag = 0;
+		}
+
+	} while (!eofflag);
+
+
+	error = 0;
+
+outsc:
+	if (open_flag)
+		VNOP_CLOSE(vp, FREAD, ctx);
+
+	uio_free(auio);
+	FREE(rbuf, M_TEMP);
+
+	vnode_resume(vp);
+
+
+	return(error);
+
+}
+
 
 #ifdef JOE_DEBUG
 
diff --git a/bsd/vfs/vfs_support.c b/bsd/vfs/vfs_support.c
index 79e071e46..3d20ffa7d 100644
--- a/bsd/vfs/vfs_support.c
+++ b/bsd/vfs/vfs_support.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-1999 Apple Computer, Inc.  All rights reserved.
@@ -44,6 +50,7 @@
 
 #include <vfs/vfs_support.h>
 #include <sys/kauth.h>
+#include <sys/ubc.h>	/* ubc_upl_abort_range() */
 
 
 struct vnop_create_args /* {
@@ -55,7 +62,7 @@ struct vnop_create_args /* {
 } */;
 
 int
-nop_create(struct vnop_create_args *ap)
+nop_create(__unused struct vnop_create_args *ap)
 {
 #if DIAGNOSTIC
 	if ((ap->a_cnp->cn_flags & HASBUF) == 0)
@@ -80,13 +87,13 @@ struct vnop_whiteout_args /* {
 } */;
 
 int
-nop_whiteout(struct vnop_whiteout_args *ap)
+nop_whiteout(__unused struct vnop_whiteout_args *ap)
 {
 	return (0);
 }
 
 int
-err_whiteout(struct vnop_whiteout_args *ap)
+err_whiteout(__unused struct vnop_whiteout_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -101,7 +108,7 @@ struct vnop_mknod_args /* {
 } */;
 
 int
-nop_mknod(struct vnop_mknod_args *ap)
+nop_mknod(__unused struct vnop_mknod_args *ap)
 {
 #if DIAGNOSTIC
 	if ((ap->a_cnp->cn_flags & HASBUF) == 0)
@@ -124,13 +131,13 @@ struct vnop_open_args /* {
 } */;
 
 int
-nop_open(struct vnop_open_args *ap)
+nop_open(__unused struct vnop_open_args *ap)
 {
 	return (0);
 }
 
 int
-err_open(struct vnop_open_args *ap)
+err_open(__unused struct vnop_open_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -143,13 +150,13 @@ struct vnop_close_args /* {
 } */;
 
 int
-nop_close(struct vnop_close_args *ap)
+nop_close(__unused struct vnop_close_args *ap)
 {
 	return (0);
 }
 
 int
-err_close(struct vnop_close_args *ap)
+err_close(__unused struct vnop_close_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -162,13 +169,13 @@ struct vnop_access_args /* {
 } */;
 
 int
-nop_access(struct vnop_access_args *ap)
+nop_access(__unused struct vnop_access_args *ap)
 {
 	return (0);
 }
 
 int
-err_access(struct vnop_access_args *ap)
+err_access(__unused struct vnop_access_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -181,13 +188,13 @@ struct vnop_getattr_args /* {
 } */;
 
 int
-nop_getattr(struct vnop_getattr_args *ap)
+nop_getattr(__unused struct vnop_getattr_args *ap)
 {
 	return (0);
 }
 
 int
-err_getattr(struct vnop_getattr_args *ap)
+err_getattr(__unused struct vnop_getattr_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -200,60 +207,17 @@ struct vnop_setattr_args /* {
 } */;
 
 int
-nop_setattr(struct vnop_setattr_args *ap)
+nop_setattr(__unused struct vnop_setattr_args *ap)
 {
 	return (0);
 }
 
 int
-err_setattr(struct vnop_setattr_args *ap)
+err_setattr(__unused struct vnop_setattr_args *ap)
 {
 	return (ENOTSUP);
 }
 
-
-struct vnop_getattrlist_args /* {
-	struct vnode *a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context a_context;
-} */;
-
-int
-nop_getattrlist(struct vnop_getattrlist_args *ap)
-{
-	return (0);
-}
-
-int
-err_getattrlist(struct vnop_getattrlist_args *ap)
-{
-	return (ENOTSUP);
-}
-
-
-struct vnop_setattrlist_args /* {
-	struct vnode *a_vp;
-	struct attrlist *a_alist;
-	struct uio *a_uio;
-	int a_options;
-	vfs_context_t a_context;
-} */;
-
-int
-nop_setattrlist(struct vnop_setattrlist_args *ap)
-{
-	return (0);
-}
-
-int
-err_setattrlist(struct vnop_setattrlist_args *ap)
-{
-	return (ENOTSUP);
-}
-
-
 struct vnop_read_args /*  {
 	struct vnode *a_vp;
 	struct uio *a_uio;
@@ -262,13 +226,13 @@ struct vnop_read_args /*  {
 } */;
 
 int
-nop_read(struct vnop_read_args *ap)
+nop_read(__unused struct vnop_read_args *ap)
 {
 	return (0);
 }
 
 int
-err_read(struct vnop_read_args *ap)
+err_read(__unused struct vnop_read_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -282,13 +246,13 @@ struct vnop_write_args /*  {
 } */;
 
 int
-nop_write(struct vnop_write_args *ap)
+nop_write(__unused struct vnop_write_args *ap)
 {
 	return (0);
 }
 
 int
-err_write(struct vnop_write_args *ap)
+err_write(__unused struct vnop_write_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -310,7 +274,7 @@ nop_ioctl(__unused struct vnop_ioctl_args *ap)
 }
 
 int
-err_ioctl(struct vnop_ioctl_args *ap)
+err_ioctl(__unused struct vnop_ioctl_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -332,7 +296,7 @@ nop_select(__unused struct vnop_select_args *ap)
 }
 
 int
-err_select(struct vnop_select_args *ap)
+err_select(__unused struct vnop_select_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -346,13 +310,13 @@ struct vnop_exchange_args /* {
 } */;
 
 int
-nop_exchange(struct vnop_exchange_args *ap)
+nop_exchange(__unused struct vnop_exchange_args *ap)
 {
 	return (0);
 }
 
 int
-err_exchange(struct vnop_exchange_args *ap)
+err_exchange(__unused struct vnop_exchange_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -392,7 +356,7 @@ nop_mmap(__unused struct vnop_mmap_args *ap)
 }
 
 int
-err_mmap(struct vnop_mmap_args *ap)
+err_mmap(__unused struct vnop_mmap_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -405,13 +369,13 @@ struct vnop_fsync_args /* {
 } */;
 
 int
-nop_fsync(struct vnop_fsync_args *ap)
+nop_fsync(__unused struct vnop_fsync_args *ap)
 {
 	return (0);
 }
 
 int
-err_fsync(struct vnop_fsync_args *ap)
+err_fsync(__unused struct vnop_fsync_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -426,7 +390,7 @@ struct vnop_remove_args /* {
 } */;
 
 int
-nop_remove(struct vnop_remove_args *ap)
+nop_remove(__unused struct vnop_remove_args *ap)
 {
 	return (0);
 }
@@ -447,7 +411,7 @@ struct vnop_link_args /* {
 } */;
 
 int
-nop_link(struct vnop_link_args *ap)
+nop_link(__unused struct vnop_link_args *ap)
 {
 	return (0);
 }
@@ -471,7 +435,7 @@ struct vnop_rename_args /* {
 } */;
 
 int
-nop_rename(struct vnop_rename_args *ap)
+nop_rename(__unused struct vnop_rename_args *ap)
 {
 	return (0);
 }
@@ -493,13 +457,13 @@ struct vnop_mkdir_args /* {
 } */;
 
 int
-nop_mkdir(struct vnop_mkdir_args *ap)
+nop_mkdir(__unused struct vnop_mkdir_args *ap)
 {
 	return (0);
 }
 
 int
-err_mkdir(struct vnop_mkdir_args *ap)
+err_mkdir(__unused struct vnop_mkdir_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -513,7 +477,7 @@ struct vnop_rmdir_args /* {
 } */;
 
 int
-nop_rmdir(struct vnop_rmdir_args *ap)
+nop_rmdir(__unused struct vnop_rmdir_args *ap)
 {
 	return (0);
 }
@@ -536,7 +500,7 @@ struct vnop_symlink_args /* {
 } */;
 
 int
-nop_symlink(struct vnop_symlink_args *ap)
+nop_symlink(__unused struct vnop_symlink_args *ap)
 {
 #if DIAGNOSTIC
 	if ((ap->a_cnp->cn_flags & HASBUF) == 0)
@@ -563,13 +527,13 @@ struct vnop_readdir_args /* {
 } */;
 
 int
-nop_readdir(struct vnop_readdir_args *ap)
+nop_readdir(__unused struct vnop_readdir_args *ap)
 {
 	return (0);
 }
 
 int
-err_readdir(struct vnop_readdir_args *ap)
+err_readdir(__unused struct vnop_readdir_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -610,13 +574,13 @@ struct vnop_readlink_args /* {
 } */;
 
 int
-nop_readlink(struct vnop_readlink_args *ap)
+nop_readlink(__unused struct vnop_readlink_args *ap)
 {
 	return (0);
 }
 
 int
-err_readlink(struct vnop_readlink_args *ap)
+err_readlink(__unused struct vnop_readlink_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -628,7 +592,7 @@ struct vnop_inactive_args /* {
 } */;
 
 int
-nop_inactive(struct vnop_inactive_args *ap)
+nop_inactive(__unused struct vnop_inactive_args *ap)
 {
 	return (0);
 }
@@ -647,13 +611,13 @@ struct vnop_reclaim_args /* {
 } */;
 
 int
-nop_reclaim(struct vnop_reclaim_args *ap)
+nop_reclaim(__unused struct vnop_reclaim_args *ap)
 {
 	return (0);
 }
 
 int
-err_reclaim(struct vnop_reclaim_args *ap)
+err_reclaim(__unused struct vnop_reclaim_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -664,13 +628,13 @@ struct vnop_strategy_args /* {
 } */;
 
 int
-nop_strategy(struct vnop_strategy_args *ap)
+nop_strategy(__unused struct vnop_strategy_args *ap)
 {
 	return (0);
 }
 
 int
-err_strategy(struct vnop_strategy_args *ap)
+err_strategy(__unused struct vnop_strategy_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -684,13 +648,13 @@ struct vnop_pathconf_args /* {
 } */;
 
 int
-nop_pathconf(struct vnop_pathconf_args *ap)
+nop_pathconf(__unused struct vnop_pathconf_args *ap)
 {
 	return (0);
 }
 
 int
-err_pathconf(struct vnop_pathconf_args *ap)
+err_pathconf(__unused struct vnop_pathconf_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -706,13 +670,13 @@ struct vnop_advlock_args /* {
 } */;
 
 int
-nop_advlock(struct vnop_advlock_args *ap)
+nop_advlock(__unused struct vnop_advlock_args *ap)
 {
 	return (0);
 }
 
 int
-err_advlock(struct vnop_advlock_args *ap)
+err_advlock(__unused struct vnop_advlock_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -753,7 +717,7 @@ nop_bwrite(struct vnop_bwrite_args *ap)
 }
 
 int
-err_bwrite(struct vnop_bwrite_args *ap)
+err_bwrite(__unused struct vnop_bwrite_args *ap)
 {
 	return (ENOTSUP);
 }
@@ -853,7 +817,7 @@ struct vnop_copyfile_args /*{
 }*/; 
 
 int
-nop_copyfile(struct vnop_copyfile_args *ap)
+nop_copyfile(__unused struct vnop_copyfile_args *ap)
 {
 	return (0);
 }
@@ -917,12 +881,12 @@ struct vnop_blockmap_args /* {
 	int a_flags;
 } */;
 
-int nop_blockmap(struct vnop_blockmap_args *ap)
+int nop_blockmap(__unused struct vnop_blockmap_args *ap)
 {
 	return (0);
 }
 
-int err_blockmap(struct vnop_blockmap_args *ap)
+int err_blockmap(__unused struct vnop_blockmap_args *ap)
 {
 	return (ENOTSUP);
 }
diff --git a/bsd/vfs/vfs_support.h b/bsd/vfs/vfs_support.h
index 21c0b21e1..ae94ff643 100644
--- a/bsd/vfs/vfs_support.h
+++ b/bsd/vfs/vfs_support.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
@@ -76,12 +82,6 @@ extern int err_getattr(struct vnop_getattr_args *ap);
 extern int nop_setattr(struct vnop_setattr_args *ap);
 extern int err_setattr(struct vnop_setattr_args *ap);
 
-extern int nop_getattrlist(struct vnop_getattrlist_args *ap);
-extern int err_getattrlist(struct vnop_getattrlist_args *ap);
-
-extern int nop_setattrlist(struct vnop_setattrlist_args *ap);
-extern int err_setattrlist(struct vnop_setattrlist_args *ap);
-
 extern int nop_read(struct vnop_read_args *ap);
 extern int err_read(struct vnop_read_args *ap);
 
diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c
index 29ecef5cc..c266535dc 100644
--- a/bsd/vfs/vfs_syscalls.c
+++ b/bsd/vfs/vfs_syscalls.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1995-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1995-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1989, 1993
@@ -58,6 +64,12 @@
  *
  *	@(#)vfs_syscalls.c	8.41 (Berkeley) 6/15/95
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -86,6 +98,7 @@
 #include <machine/cons.h>
 #include <machine/limits.h>
 #include <miscfs/specfs/specdev.h>
+#include <miscfs/union/union.h>
 
 #include <bsm/audit_kernel.h>
 #include <bsm/audit_kevents.h>
@@ -98,25 +111,53 @@
 
 #include <libkern/OSAtomic.h>
 
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_framework.h>
+#endif
 
-/*
- * The currently logged-in user, for ownership of files/directories whose on-disk
- * permissions are ignored:
- */
-uid_t console_user;
+#if CONFIG_FSE 
+#define GET_PATH(x) \
+	(x) = get_pathbuff(); 
+#define RELEASE_PATH(x) \
+	release_pathbuff(x);
+#else 
+#define GET_PATH(x)	\
+	MALLOC_ZONE((x), char *, MAXPATHLEN, M_NAMEI, M_WAITOK); 
+#define RELEASE_PATH(x) \
+	FREE_ZONE((x), MAXPATHLEN, M_NAMEI);
+#endif /* CONFIG_FSE */
+
+/* struct for checkdirs iteration */
+struct cdirargs {
+	vnode_t olddp;
+	vnode_t newdp;
+};
+/* callback  for checkdirs iteration */
+static int checkdirs_callback(proc_t p, void * arg);
 
 static int change_dir(struct nameidata *ndp, vfs_context_t ctx);
 static int checkdirs(vnode_t olddp, vfs_context_t ctx);
 void enablequotas(struct mount *mp, vfs_context_t ctx);
 static int getfsstat_callback(mount_t mp, void * arg);
 static int getutimes(user_addr_t usrtvp, struct timespec *tsp);
-static int setutimes(vfs_context_t ctx, struct vnode *vp, const struct timespec *ts, int nullflag);
+static int setutimes(vfs_context_t ctx, vnode_t vp, const struct timespec *ts, int nullflag);
 static int sync_callback(mount_t, void *);
 static int munge_statfs(struct mount *mp, struct vfsstatfs *sfsp, 
 			user_addr_t bufp, int *sizep, boolean_t is_64_bit, 
 						boolean_t partial_copy);
+static int statfs64_common(struct mount *mp, struct vfsstatfs *sfsp, user_addr_t bufp);
+int (*union_dircheckp)(struct vnode **, struct fileproc *, vfs_context_t);
+
+__private_extern__
+int sync_internal(void);
+
+__private_extern__
+int open1(vfs_context_t, struct nameidata *, int, struct vnode_attr *, register_t *);
+
+__private_extern__
+int unlink1(vfs_context_t, struct nameidata *, int);
 
-__private_extern__ int sync_internal(void);
 
 #ifdef __APPLE_API_OBSOLETE
 struct fstatv_args {
@@ -137,27 +178,25 @@ struct statv_args {
         struct vstat *vsb;	/* vstat structure for returned info */
 };
 
-int fstatv(struct proc *p, struct fstatv_args *uap, register_t *retval);
-int lstatv(struct proc *p, struct lstatv_args *uap, register_t *retval);
-int mkcomplex(struct proc *p, struct mkcomplex_args *uap, register_t *retval);
-int statv(struct proc *p, struct statv_args *uap, register_t *retval);
+int fstatv(proc_t p, struct fstatv_args *uap, register_t *retval);
+int lstatv(proc_t p, struct lstatv_args *uap, register_t *retval);
+int mkcomplex(proc_t p, struct mkcomplex_args *uap, register_t *retval);
+int statv(proc_t p, struct statv_args *uap, register_t *retval);
 
 #endif /* __APPLE_API_OBSOLETE */
 
-#if UNION
-extern int (**union_vnodeop_p)(void *);
-extern struct vnode *union_dircache(struct vnode*, struct proc*);
-#endif /* UNION */
+/*
+ * incremented each time a mount or unmount operation occurs
+ * used to invalidate the cached value of the rootvp in the
+ * mount structure utilized by cache_lookup_path
+ */
+int mount_generation = 0;
 
 /* counts number of mount and unmount operations */
 unsigned int vfs_nummntops=0;
 
 extern struct fileops vnops;
-
-extern void mount_list_add(mount_t mp);
-extern void mount_list_remove(mount_t mp);
-extern int mount_refdrain(mount_t mp);
-extern int vcount(struct vnode *vp);
+extern errno_t rmdir_remove_orphaned_appleDouble(vnode_t, vfs_context_t, int *); 
 
 
 /*
@@ -169,16 +208,32 @@ extern int vcount(struct vnode *vp);
  */
 /* ARGSUSED */
 int
-mount(struct proc *p, register struct mount_args *uap, __unused register_t *retval)
+mount(proc_t p, struct mount_args *uap, __unused register_t *retval)
+{
+	struct __mac_mount_args muap;
+
+	muap.type = uap->type;
+	muap.path = uap->path;
+	muap.flags = uap->flags;
+	muap.data = uap->data;
+	muap.mac_p = USER_ADDR_NULL;
+	return (__mac_mount(p, &muap, retval));
+}
+
+int
+__mac_mount(struct proc *p, register struct __mac_mount_args *uap, __unused register_t *retval)
 {
 	struct vnode *vp;
 	struct vnode *devvp = NULLVP;
 	struct vnode *device_vnode = NULLVP;
+#if CONFIG_MACF
+	struct vnode *rvp;
+#endif
 	struct mount *mp;
 	struct vfstable *vfsp = (struct vfstable *)0;
 	int error, flag = 0;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct nameidata nd;
 	struct nameidata nd1;
 	char fstypename[MFSNAMELEN];
@@ -193,15 +248,13 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 
 	AUDIT_ARG(fflags, uap->flags);
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 	is_64bit = proc_is64bit(p);
 
 	/*
 	 * Get vnode to be covered
 	 */
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		   UIO_USERSPACE, uap->path, &context);
+	NDINIT(&nd, LOOKUP, NOTRIGGER | FOLLOW | AUDITVNPATH1, 
+		   UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -210,6 +263,10 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 	if ((vp->v_flag & VROOT) &&
 		(vp->v_mount->mnt_flag & MNT_ROOTFS)) 
 			uap->flags |= MNT_UPDATE;
+
+	error = copyinstr(uap->type, fstypename, MFSNAMELEN, &dummy);
+	if (error)
+		goto out1;
 	
 	if (uap->flags & MNT_UPDATE) {
 		if ((vp->v_flag & VROOT) == 0) {
@@ -241,15 +298,22 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 		 * Only root, or the user that did the original mount is
 		 * permitted to update it.
 		 */
-		if (mp->mnt_vfsstat.f_owner != kauth_cred_getuid(context.vc_ucred) &&
-		    (error = suser(context.vc_ucred, &p->p_acflag))) {
+		if (mp->mnt_vfsstat.f_owner != kauth_cred_getuid(vfs_context_ucred(ctx)) &&
+		    (error = suser(vfs_context_ucred(ctx), &p->p_acflag))) {
+			goto out1;
+		}
+#if CONFIG_MACF
+		error = mac_mount_check_remount(ctx, mp);
+		if (error != 0) {
+			lck_rw_done(&mp->mnt_rwlock);
 			goto out1;
 		}
+#endif
 		/*
 		 * For non-root users, silently enforce MNT_NOSUID and MNT_NODEV,
 		 * and MNT_NOEXEC if mount point is already MNT_NOEXEC.
 		 */
-		if (suser(context.vc_ucred, NULL)) {
+		if (suser(vfs_context_ucred(ctx), NULL)) {
 			uap->flags |= MNT_NOSUID | MNT_NODEV;
 			if (mp->mnt_flag & MNT_NOEXEC)
 				uap->flags |= MNT_NOEXEC;
@@ -268,21 +332,21 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 	 */
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_uid);
-	if ((error = vnode_getattr(vp, &va, &context)) ||
-	    (va.va_uid != kauth_cred_getuid(context.vc_ucred) &&
-	     (error = suser(context.vc_ucred, &p->p_acflag)))) {
+	if ((error = vnode_getattr(vp, &va, ctx)) ||
+	    (va.va_uid != kauth_cred_getuid(vfs_context_ucred(ctx)) &&
+	     (error = suser(vfs_context_ucred(ctx), &p->p_acflag)))) {
 		goto out1;
 	}
 	/*
 	 * For non-root users, silently enforce MNT_NOSUID and MNT_NODEV, and
 	 * MNT_NOEXEC if mount point is already MNT_NOEXEC.
 	 */
-	if (suser(context.vc_ucred, NULL)) {
+	if (suser(vfs_context_ucred(ctx), NULL)) {
 		uap->flags |= MNT_NOSUID | MNT_NODEV;
 		if (vp->v_mount->mnt_flag & MNT_NOEXEC)
 			uap->flags |= MNT_NOEXEC;
 	}
-	if ( (error = VNOP_FSYNC(vp, MNT_WAIT, &context)) )
+	if ( (error = VNOP_FSYNC(vp, MNT_WAIT, ctx)) )
 		goto out1;
 
 	if ( (error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0)) )
@@ -292,26 +356,30 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 		error = ENOTDIR;
 		goto out1;
 	}
-	if ( (error = copyinstr(uap->type, fstypename, MFSNAMELEN, &dummy)) )
-		goto out1;
 
 	/* XXXAUDIT: Should we capture the type on the error path as well? */
 	AUDIT_ARG(text, fstypename);
 	mount_list_lock();
 	for (vfsp = vfsconf; vfsp; vfsp = vfsp->vfc_next)
-		if (!strcmp(vfsp->vfc_name, fstypename))
+		if (!strncmp(vfsp->vfc_name, fstypename, MFSNAMELEN))
 			break;
 	mount_list_unlock();
 	if (vfsp == NULL) {
 		error = ENODEV;
 		goto out1;
 	}
+#if CONFIG_MACF
+	error = mac_mount_check_mount(ctx, vp,
+	    &nd.ni_cnd, vfsp->vfc_name);
+	if (error != 0)
+		goto out1;
+#endif
 	if (ISSET(vp->v_flag, VMOUNT) && (vp->v_mountedhere != NULL)) {
 		error = EBUSY;
 		goto out1;
 	}
-	vnode_lock(vp);
- 	SET(vp->v_flag, VMOUNT);
+	vnode_lock_spin(vp);
+	SET(vp->v_flag, VMOUNT);
 	vnode_unlock(vp);
 
 	/*
@@ -328,6 +396,10 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 	mp->mnt_maxsegreadsize = mp->mnt_maxreadcnt;
 	mp->mnt_maxsegwritesize = mp->mnt_maxwritecnt;
 	mp->mnt_devblocksize = DEV_BSIZE;
+	mp->mnt_alignmentmask = PAGE_MASK;
+	mp->mnt_ioflags = 0;
+	mp->mnt_realrootvp = NULLVP;
+	mp->mnt_authcache_ttl = CACHED_LOOKUP_RIGHT_TTL;
 
 	TAILQ_INIT(&mp->mnt_vnodelist);
 	TAILQ_INIT(&mp->mnt_workerqueue);
@@ -345,7 +417,7 @@ mount(struct proc *p, register struct mount_args *uap, __unused register_t *retv
 	strncpy(mp->mnt_vfsstat.f_fstypename, vfsp->vfc_name, MFSTYPENAMELEN);
 	strncpy(mp->mnt_vfsstat.f_mntonname, nd.ni_cnd.cn_pnbuf, MAXPATHLEN);
 	mp->mnt_vnodecovered = vp;
-	mp->mnt_vfsstat.f_owner = kauth_cred_getuid(context.vc_ucred);
+	mp->mnt_vfsstat.f_owner = kauth_cred_getuid(vfs_context_ucred(ctx));
 
 	/* XXX 3762912 hack to support HFS filesystem 'owner' - filesystem may update later */
 	vfs_setowner(mp, KAUTH_UID_NONE, KAUTH_GID_NONE);
@@ -360,11 +432,22 @@ update:
 		mp->mnt_kern_flag |= MNTK_WANTRDWR;
 	mp->mnt_flag &= ~(MNT_NOSUID | MNT_NOEXEC | MNT_NODEV |
 			  MNT_SYNCHRONOUS | MNT_UNION | MNT_ASYNC |
-			  MNT_UNKNOWNPERMISSIONS | MNT_DONTBROWSE | MNT_AUTOMOUNTED);
+			  MNT_UNKNOWNPERMISSIONS | MNT_DONTBROWSE | MNT_AUTOMOUNTED |
+			  MNT_DEFWRITE | MNT_NOATIME | MNT_QUARANTINE);
 	mp->mnt_flag |= uap->flags & (MNT_NOSUID | MNT_NOEXEC |	MNT_NODEV |
 				      MNT_SYNCHRONOUS | MNT_UNION | MNT_ASYNC |
 				      MNT_UNKNOWNPERMISSIONS | MNT_DONTBROWSE | MNT_AUTOMOUNTED | 
-					  MNT_DEFWRITE);
+					  MNT_DEFWRITE | MNT_NOATIME | MNT_QUARANTINE);
+
+#if CONFIG_MACF
+	if (uap->flags & MNT_MULTILABEL) {
+		if (vfsp->vfc_vfsflags & VFC_VFSNOMACLABEL) {
+			error = EINVAL;
+			goto out1;
+		}
+		mp->mnt_flag |= MNT_MULTILABEL;
+	}
+#endif
 
 	if (vfsp->vfc_vfsflags & VFC_VFSLOCALARGS) {
 		if (is_64bit) {
@@ -382,7 +465,7 @@ update:
 
 		/* if it is not update and device name needs to be parsed */
 		if ((devpath)) {
-			NDINIT(&nd1, LOOKUP, FOLLOW, UIO_USERSPACE, devpath, &context);
+			NDINIT(&nd1, LOOKUP, FOLLOW, UIO_USERSPACE, devpath, ctx);
 			if ( (error = namei(&nd1)) )
 				goto out1;
 
@@ -403,11 +486,11 @@ update:
 			* If mount by non-root, then verify that user has necessary
 			* permissions on the device.
 			*/
-			if (suser(context.vc_ucred, NULL) != 0) {
+			if (suser(vfs_context_ucred(ctx), NULL) != 0) {
 				accessmode = KAUTH_VNODE_READ_DATA;
 				if ((mp->mnt_flag & MNT_RDONLY) == 0)
 					accessmode |= KAUTH_VNODE_WRITE_DATA;
-				if ((error = vnode_authorize(devvp, NULL, accessmode, &context)) != 0)
+				if ((error = vnode_authorize(devvp, NULL, accessmode, ctx)) != 0)
 					goto out2;
 			}
 		}
@@ -427,7 +510,7 @@ update:
 				error = EBUSY;
 				goto out3;
 			}
-			if ( (error = VNOP_FSYNC(devvp, MNT_WAIT, &context)) ) {
+			if ( (error = VNOP_FSYNC(devvp, MNT_WAIT, ctx)) ) {
 				error = ENOTBLK;
 				goto out3;
 			}
@@ -435,7 +518,14 @@ update:
 				goto out3;
 
 			ronly = (mp->mnt_flag & MNT_RDONLY) != 0;
-			if ( (error = VNOP_OPEN(devvp, ronly ? FREAD : FREAD|FWRITE, &context)) )
+#if CONFIG_MACF
+			error = mac_vnode_check_open(ctx,
+			    devvp,
+			    ronly ? FREAD : FREAD|FWRITE);
+			if (error)
+				goto out3;
+#endif /* MAC */
+			if ( (error = VNOP_OPEN(devvp, ronly ? FREAD : FREAD|FWRITE, ctx)) )
 				goto out3;
 
 			mp->mnt_devvp = devvp;
@@ -447,21 +537,63 @@ update:
 				 * that user has necessary permissions on the device.
 				 */
 				device_vnode = mp->mnt_devvp;
-				if (device_vnode && suser(context.vc_ucred, NULL)) {
+				if (device_vnode && suser(vfs_context_ucred(ctx), NULL)) {
 					if ((error = vnode_authorize(device_vnode, NULL,
-						 KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, &context)) != 0)
+						 KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, ctx)) != 0)
 						goto out2;
 				}
 			}
 			device_vnode = NULLVP;
 		}
 	}
+#if CONFIG_MACF
+	if ((uap->flags & MNT_UPDATE) == 0) {
+		mac_mount_label_init(mp);
+		mac_mount_label_associate(ctx, mp);
+	}
+	if (uap->mac_p != USER_ADDR_NULL) {
+		struct user_mac mac;
+		char *labelstr = NULL;
+		size_t ulen = 0;
 
-
+		if ((uap->flags & MNT_UPDATE) != 0) {
+			error = mac_mount_check_label_update(
+			    ctx, mp);
+			if (error != 0)
+				goto out3;
+		}
+		if (is_64bit) {
+			error = copyin(uap->mac_p, &mac, sizeof(mac));
+		} else {
+			struct mac mac32;
+			error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+			mac.m_buflen = mac32.m_buflen;
+			mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+		}
+		if (error != 0)
+			goto out3;
+		if ((mac.m_buflen > MAC_MAX_LABEL_BUF_LEN) ||
+		    (mac.m_buflen < 2)) {
+			error = EINVAL;
+			goto out3;
+		}
+		MALLOC(labelstr, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+		error = copyinstr(mac.m_string, labelstr, mac.m_buflen, &ulen);
+		if (error != 0) {
+			FREE(labelstr, M_MACTEMP);
+			goto out3;
+		}
+		AUDIT_ARG(mac_string, labelstr);
+		error = mac_mount_label_internalize(mp->mnt_mntlabel, labelstr);
+		FREE(labelstr, M_MACTEMP);
+		if (error != 0)
+			goto out3;
+	}
+#endif
 	/*
 	 * Mount the filesystem.
 	 */
-	error = VFS_MOUNT(mp, device_vnode, fsmountargs, &context);
+	error = VFS_MOUNT(mp, device_vnode, fsmountargs, ctx);
 
 	if (uap->flags & MNT_UPDATE) {
 		if (mp->mnt_kern_flag & MNTK_WANTRDWR)
@@ -475,22 +607,49 @@ update:
 		lck_rw_done(&mp->mnt_rwlock);
 		is_rwlock_locked = FALSE;
 		if (!error)
-			enablequotas(mp,&context);
+			enablequotas(mp, ctx);
 		goto out2;
 	}
 	/*
 	 * Put the new filesystem on the mount list after root.
 	 */
 	if (error == 0) {
-		vnode_lock(vp);
-		CLR(vp->v_flag, VMOUNT);
+		struct vfs_attr	vfsattr;
+#if CONFIG_MACF
+		if (vfs_flags(mp) & MNT_MULTILABEL) {
+			error = VFS_ROOT(mp, &rvp, ctx);
+			if (error) {
+				printf("%s() VFS_ROOT returned %d\n", __func__, error);
+				goto out3;
+			}
 
+			/* VFS_ROOT provides reference so needref = 0 */
+			error = vnode_label(mp, NULL, rvp, NULL, 0, ctx);
+			if (error)
+				goto out3;
+		}
+#endif	/* MAC */
+
+		vnode_lock_spin(vp);
+		CLR(vp->v_flag, VMOUNT);
 		vp->v_mountedhere = mp;
 		vnode_unlock(vp);
 
+		/*
+		 * taking the name_cache_lock exclusively will
+		 * insure that everyone is out of the fast path who
+		 * might be trying to use a now stale copy of
+		 * vp->v_mountedhere->mnt_realrootvp
+		 * bumping mount_generation causes the cached values
+		 * to be invalidated
+		 */
+		name_cache_lock();
+		mount_generation++;
+		name_cache_unlock();
+
 		vnode_ref(vp);
 
-		error = checkdirs(vp, &context);
+		error = checkdirs(vp, ctx);
 		if (error != 0)  {
 			/* Unmount the filesystem as cdir/rdirs cannot be updated */
 			goto out4;
@@ -499,15 +658,47 @@ update:
 		 * there is no cleanup code here so I have made it void 
 		 * we need to revisit this
 		 */
-		(void)VFS_START(mp, 0, &context);
+		(void)VFS_START(mp, 0, ctx);
 
 		mount_list_add(mp);
 		lck_rw_done(&mp->mnt_rwlock);
 		is_rwlock_locked = FALSE;
 
+		/* Check if this mounted file system supports EAs or named streams. */
+		/* Skip WebDAV file systems for now since they hang in VFS_GETATTR here. */
+		VFSATTR_INIT(&vfsattr);
+		VFSATTR_WANTED(&vfsattr, f_capabilities);
+		if (strncmp(mp->mnt_vfsstat.f_fstypename, "webdav", sizeof("webdav")) != 0 &&
+		    vfs_getattr(mp, &vfsattr, ctx) == 0 && 
+		    VFSATTR_IS_SUPPORTED(&vfsattr, f_capabilities)) {
+			if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR) &&
+			    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_EXTENDED_ATTR)) {
+				mp->mnt_kern_flag |= MNTK_EXTENDED_ATTRS;
+			}
+#if NAMEDSTREAMS
+			if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_NAMEDSTREAMS) &&
+			    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_NAMEDSTREAMS)) {
+				mp->mnt_kern_flag |= MNTK_NAMED_STREAMS;
+			}
+#endif
+			/* Check if this file system supports path from id lookups. */
+			if ((vfsattr.f_capabilities.capabilities[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_PATH_FROM_ID) &&
+			    (vfsattr.f_capabilities.valid[VOL_CAPABILITIES_FORMAT] & VOL_CAP_FMT_PATH_FROM_ID)) {
+				mp->mnt_kern_flag |= MNTK_PATH_FROM_ID;
+			} else if (mp->mnt_flag & MNT_DOVOLFS) {
+				/* Legacy MNT_DOVOLFS flag also implies path from id lookups. */
+				mp->mnt_kern_flag |= MNTK_PATH_FROM_ID;
+			}
+		}
+		if (mp->mnt_vtable->vfc_vfsflags & VFC_VFSNATIVEXATTR) {
+			mp->mnt_kern_flag |= MNTK_EXTENDED_ATTRS;
+		}
+		if (mp->mnt_vtable->vfc_vfsflags & VFC_VFSPREFLIGHT) {
+			mp->mnt_kern_flag |= MNTK_UNMOUNT_PREFLIGHT;
+		}
 		/* increment the operations count */
 		OSAddAtomic(1, (SInt32 *)&vfs_nummntops);
-		enablequotas(mp,&context);
+		enablequotas(mp, ctx);
 
 		if (device_vnode) {
 			device_vnode->v_specflags |= SI_MOUNTEDON;
@@ -524,7 +715,7 @@ update:
 		/* Now that mount is setup, notify the listeners */
 		vfs_event_signal(NULL, VQ_MOUNT, (intptr_t)NULL);
 	} else {
-		vnode_lock(vp);
+		vnode_lock_spin(vp);
 		CLR(vp->v_flag, VMOUNT);
 		vnode_unlock(vp);
 		mount_list_lock();
@@ -532,12 +723,15 @@ update:
 		mount_list_unlock();
 
 		if (device_vnode ) {
-			VNOP_CLOSE(device_vnode, ronly ? FREAD : FREAD|FWRITE, &context);
+			VNOP_CLOSE(device_vnode, ronly ? FREAD : FREAD|FWRITE, ctx);
 			vnode_rele(device_vnode);
 		}
 		lck_rw_done(&mp->mnt_rwlock);
 		is_rwlock_locked = FALSE;
 		mount_lock_destroy(mp);
+#if CONFIG_MACF
+		mac_mount_label_destroy(mp);
+#endif
 		FREE_ZONE((caddr_t)mp, sizeof (struct mount), M_MOUNT);
 	}
 	nameidone(&nd);
@@ -551,18 +745,20 @@ update:
 	vnode_put(vp);
 
 	return(error);
-
 out4:
-	(void)VFS_UNMOUNT(mp, MNT_FORCE, &context);
+	(void)VFS_UNMOUNT(mp, MNT_FORCE, ctx);
 	if (device_vnode != NULLVP) {
-		VNOP_CLOSE(device_vnode, mp->mnt_flag & MNT_RDONLY ? FREAD : FREAD|FWRITE, &context);
+		VNOP_CLOSE(device_vnode, mp->mnt_flag & MNT_RDONLY ? FREAD : FREAD|FWRITE,
+                       ctx);
+
 	}
-	vnode_lock(vp);
+	vnode_lock_spin(vp);
 	vp->v_mountedhere = (mount_t) 0;
 	vnode_unlock(vp);
 	vnode_rele(vp);
 out3:
-	vnode_rele(devvp);
+	if (devpath && ((uap->flags & MNT_UPDATE) == 0))
+		vnode_rele(devvp);
 out2:
 	if (devpath && devvp)
 	        vnode_put(devvp);
@@ -572,6 +768,9 @@ out1:
 		lck_rw_done(&mp->mnt_rwlock);
 	}
 	if (mntalloc) {
+#if CONFIG_MACF
+		mac_mount_label_destroy(mp);
+#endif
 		mount_list_lock();
 		vfsp->vfc_refcount--;
 		mount_list_unlock();
@@ -581,11 +780,10 @@ out1:
 	nameidone(&nd);
 
 	return(error);
-
 }
 
 void
-enablequotas(struct mount *mp, vfs_context_t context)
+enablequotas(struct mount *mp, vfs_context_t ctx)
 {
 	struct nameidata qnd;
 	int type;
@@ -594,8 +792,9 @@ enablequotas(struct mount *mp, vfs_context_t context)
 	const char *qfopsname = QUOTAOPSNAME;
 	const char *qfextension[] = INITQFNAMES;
 
-	if ((strcmp(mp->mnt_vfsstat.f_fstypename, "hfs") != 0 )
-                && (strcmp( mp->mnt_vfsstat.f_fstypename, "ufs") != 0))
+	/* XXX Shoulkd be an MNTK_ flag, instead of strncmp()'s */
+	if ((strncmp(mp->mnt_vfsstat.f_fstypename, "hfs", sizeof("hfs")) != 0 )
+                && (strncmp( mp->mnt_vfsstat.f_fstypename, "ufs", sizeof("ufs")) != 0))
 	  return;
 
 	/* 
@@ -603,90 +802,114 @@ enablequotas(struct mount *mp, vfs_context_t context)
 	 * We ignore errors as this should not interfere with final mount
 	 */
 	for (type=0; type < MAXQUOTAS; type++) {
-		sprintf(qfpath, "%s/%s.%s", mp->mnt_vfsstat.f_mntonname, qfopsname, qfextension[type]);
-		NDINIT(&qnd, LOOKUP, FOLLOW, UIO_SYSSPACE32, CAST_USER_ADDR_T(qfpath), context);
+		snprintf(qfpath, sizeof(qfpath), "%s/%s.%s", mp->mnt_vfsstat.f_mntonname, qfopsname, qfextension[type]);
+		NDINIT(&qnd, LOOKUP, FOLLOW, UIO_SYSSPACE32, CAST_USER_ADDR_T(qfpath), ctx);
 		if (namei(&qnd) != 0)
 			continue; 	    /* option file to trigger quotas is not present */
 		vnode_put(qnd.ni_vp);
 		nameidone(&qnd);
-		sprintf(qfpath, "%s/%s.%s", mp->mnt_vfsstat.f_mntonname, qfname, qfextension[type]);
+		snprintf(qfpath, sizeof(qfpath),  "%s/%s.%s", mp->mnt_vfsstat.f_mntonname, qfname, qfextension[type]);
 
-		(void) VFS_QUOTACTL(mp, QCMD(Q_QUOTAON, type), 0, qfpath, context);
+		(void) VFS_QUOTACTL(mp, QCMD(Q_QUOTAON, type), 0, qfpath, ctx);
 	}
 	return;
 }
 
+
+static int
+checkdirs_callback(proc_t p, void * arg) 
+{
+	struct cdirargs * cdrp = (struct cdirargs * )arg;
+	vnode_t olddp = cdrp->olddp;
+	vnode_t newdp = cdrp->newdp;
+	struct filedesc *fdp;
+	vnode_t tvp;
+	vnode_t fdp_cvp;
+	vnode_t fdp_rvp;
+	int cdir_changed = 0;
+	int rdir_changed = 0;
+
+	/*
+	 * XXX Also needs to iterate each thread in the process to see if it
+	 * XXX is using a per-thread current working directory, and, if so,
+	 * XXX update that as well.
+	 */
+
+	proc_fdlock(p);
+	fdp = p->p_fd;
+	if (fdp == (struct filedesc *)0) {
+		proc_fdunlock(p);
+		return(PROC_RETURNED);
+	}
+	fdp_cvp = fdp->fd_cdir;
+	fdp_rvp = fdp->fd_rdir;
+	proc_fdunlock(p);
+
+	if (fdp_cvp == olddp) {
+		vnode_ref(newdp);
+		tvp = fdp->fd_cdir;
+		fdp_cvp = newdp;
+		cdir_changed = 1;
+		vnode_rele(tvp);
+	}
+	if (fdp_rvp == olddp) {
+		vnode_ref(newdp);
+		tvp = fdp->fd_rdir;
+		fdp_rvp = newdp;
+		rdir_changed = 1;
+		vnode_rele(tvp);
+	}
+	if (cdir_changed || rdir_changed) {
+		proc_fdlock(p);
+		fdp->fd_cdir = fdp_cvp;
+		fdp->fd_rdir = fdp_rvp;
+		proc_fdunlock(p);
+	}
+	return(PROC_RETURNED);
+}
+
+
+
 /*
  * Scan all active processes to see if any of them have a current
  * or root directory onto which the new filesystem has just been
  * mounted. If so, replace them with the new mount point.
  */
 static int
-checkdirs(olddp, context)
-	struct vnode *olddp;
-	vfs_context_t context;
+checkdirs(vnode_t olddp, vfs_context_t ctx)
 {
-	struct filedesc *fdp;
-	struct vnode *newdp;
-	struct proc *p;
-	struct vnode *tvp;
-	struct vnode *fdp_cvp;
-	struct vnode *fdp_rvp;
-	int cdir_changed = 0;
-	int rdir_changed = 0;
-	boolean_t funnel_state;
+	vnode_t newdp;
+	vnode_t tvp;
 	int err;
+	struct cdirargs cdr;
+	struct uthread * uth = get_bsdthread_info(current_thread());
 
 	if (olddp->v_usecount == 1)
 		return(0);
-	err = VFS_ROOT(olddp->v_mountedhere, &newdp, context);
-	if (err) {
+	if (uth != (struct uthread *)0)
+		uth->uu_notrigger = 1;
+	err = VFS_ROOT(olddp->v_mountedhere, &newdp, ctx);
+	if (uth != (struct uthread *)0)
+		uth->uu_notrigger = 0;
+
+	if (err != 0) {
 #if DIAGNOSTIC
-		panic("mount: lost mount");
+		panic("mount: lost mount: error %d", err);
 #endif
 		return(err);
 	}
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 
-	for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
-		proc_fdlock(p);
-		fdp = p->p_fd;
-		if (fdp == (struct filedesc *)0) {
-			proc_fdunlock(p);
-			continue;
-		}
-		fdp_cvp = fdp->fd_cdir;
-		fdp_rvp = fdp->fd_rdir;
-		proc_fdunlock(p);
+	cdr.olddp = olddp;
+	cdr.newdp = newdp;
+	/* do not block for exec/fork trans as the vp in cwd & rootdir are not changing */
+	proc_iterate(PROC_ALLPROCLIST | PROC_NOWAITTRANS, checkdirs_callback, (void *)&cdr, NULL, NULL);
 
-		if (fdp_cvp == olddp) {
-			vnode_ref(newdp);
-			tvp = fdp->fd_cdir;
-			fdp_cvp = newdp;
-			cdir_changed = 1;
-			vnode_rele(tvp);
-		}
-		if (fdp_rvp == olddp) {
-			vnode_ref(newdp);
-			tvp = fdp->fd_rdir;
-			fdp_rvp = newdp;
-			rdir_changed = 1;
-			vnode_rele(tvp);
-		}
-		if (cdir_changed || rdir_changed) {
-			proc_fdlock(p);
-			fdp->fd_cdir = fdp_cvp;
-			fdp->fd_rdir = fdp_rvp;
-			proc_fdunlock(p);
-		}
-	}
 	if (rootvnode == olddp) {
 		vnode_ref(newdp);
 		tvp = rootvnode;
 		rootvnode = newdp;
 		vnode_rele(tvp);
 	}
-	thread_funnel_set(kernel_flock, funnel_state);
 
 	vnode_put(newdp);
 	return(0);
@@ -700,19 +923,16 @@ checkdirs(olddp, context)
  */
 /* ARGSUSED */
 int
-unmount(struct proc *p, register struct unmount_args *uap, __unused register_t *retval)
+unmount(__unused proc_t p, struct unmount_args *uap, __unused register_t *retval)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	struct mount *mp;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+	NDINIT(&nd, LOOKUP, NOTRIGGER | FOLLOW | AUDITVNPATH1, 
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -720,6 +940,13 @@ unmount(struct proc *p, register struct unmount_args *uap, __unused register_t *
 	mp = vp->v_mount;
 	nameidone(&nd);
 
+#if CONFIG_MACF
+	error = mac_mount_check_umount(ctx, mp);
+	if (error != 0) {
+		vnode_put(vp);
+		return (error);
+	}
+#endif
 	/*
 	 * Must be the root of the filesystem
 	 */
@@ -730,41 +957,53 @@ unmount(struct proc *p, register struct unmount_args *uap, __unused register_t *
 	mount_ref(mp, 0);
 	vnode_put(vp);
 	/* safedounmount consumes the mount ref */
-	return (safedounmount(mp, uap->flags, p));
+	return (safedounmount(mp, uap->flags, ctx));
+}
+
+int
+vfs_unmountbyfsid(fsid_t * fsid, int flags, vfs_context_t ctx)
+{
+	mount_t mp;
+
+	mp = mount_list_lookupby_fsid(fsid, 0, 1);
+	if (mp == (mount_t)0) {
+		return(ENOENT);
+	}
+	mount_ref(mp, 0);
+	mount_iterdrop(mp);
+	/* safedounmount consumes the mount ref */
+	return(safedounmount(mp, flags, ctx));
 }
 
+
 /*
  * The mount struct comes with a mount ref which will be consumed.
  * Do the actual file system unmount, prevent some common foot shooting.
- *
- * XXX Should take a "vfs_context_t" instead of a "struct proc *"
  */
 int
-safedounmount(mp, flags, p)
-	struct mount *mp;
-	int flags;
-	struct proc *p;
+safedounmount(struct mount *mp, int flags, vfs_context_t ctx)
 {
 	int error;
+	proc_t p = vfs_context_proc(ctx);
 
 	/*
 	 * Only root, or the user that did the original mount is
 	 * permitted to unmount this filesystem.
 	 */
 	if ((mp->mnt_vfsstat.f_owner != kauth_cred_getuid(kauth_cred_get())) &&
-	    (error = suser(kauth_cred_get(), &p->p_acflag))) {
+	    (error = suser(kauth_cred_get(), &p->p_acflag)))
 		goto out;
-	}
 
 	/*
 	 * Don't allow unmounting the root file system.
 	 */
 	if (mp->mnt_flag & MNT_ROOTFS) {
-		error  = EBUSY;
+		error = EBUSY; /* the root is always busy */
 		goto out;
 	}
 
-	return (dounmount(mp, flags, NULL, 1, p));
+	return (dounmount(mp, flags, 1, ctx));
+
 out:
 	mount_drop(mp, 0);
 	return(error);
@@ -774,23 +1013,14 @@ out:
  * Do the actual file system unmount.
  */
 int
-dounmount(mp, flags, skiplistrmp, withref, p)
-	register struct mount *mp;
-	int flags;
-	int * skiplistrmp;
-	int withref;
-	struct proc *p;
+dounmount(struct mount *mp, int flags, int withref, vfs_context_t ctx)
 {
-	struct vnode *coveredvp = (vnode_t)0;
+	vnode_t coveredvp = (vnode_t)0;
 	int error;
 	int needwakeup = 0;
-	struct vfs_context context;
 	int forcedunmount = 0;
 	int lflags = 0;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	if (flags & MNT_FORCE)
 		forcedunmount = 1;
 	mount_lock(mp);
@@ -801,30 +1031,51 @@ dounmount(mp, flags, skiplistrmp, withref, p)
 	}
 	if (mp->mnt_lflag & MNT_LUNMOUNT) {
 		mp->mnt_lflag |= MNT_LWAIT;
-		if (withref != 0)
+		if(withref != 0)
 			mount_drop(mp, 1);
-		msleep((caddr_t)mp, &mp->mnt_mlock, (PVFS | PDROP), "dounmount", 0 );
+		msleep((caddr_t)mp, &mp->mnt_mlock, (PVFS | PDROP), "dounmount", NULL);
 		/*
 		 * The prior unmount attempt has probably succeeded.
 		 * Do not dereference mp here - returning EBUSY is safest.
 		 */
-		if (skiplistrmp != NULL)
-			*skiplistrmp = 1;
 		return (EBUSY);
 	}
 	mp->mnt_kern_flag |= MNTK_UNMOUNT;
 	mp->mnt_lflag |= MNT_LUNMOUNT;
 	mp->mnt_flag &=~ MNT_ASYNC;
+	/*
+	 * anyone currently in the fast path that
+	 * trips over the cached rootvp will be
+	 * dumped out and forced into the slow path
+	 * to regenerate a new cached value
+	 */
+	mp->mnt_realrootvp = NULLVP;
 	mount_unlock(mp);
+ 
+	/*
+	 * taking the name_cache_lock exclusively will
+	 * insure that everyone is out of the fast path who
+	 * might be trying to use a now stale copy of
+	 * vp->v_mountedhere->mnt_realrootvp
+	 * bumping mount_generation causes the cached values
+	 * to be invalidated
+	 */
+	name_cache_lock();
+	mount_generation++;
+	name_cache_unlock();
+
+
 	lck_rw_lock_exclusive(&mp->mnt_rwlock);
 	if (withref != 0)
 		mount_drop(mp, 0);
+#if CONFIG_FSE
 	fsevent_unmount(mp);  /* has to come first! */
+#endif
 	error = 0;
 	if (forcedunmount == 0) {
 		ubc_umount(mp);	/* release cached vnodes */
 		if ((mp->mnt_flag & MNT_RDONLY) == 0) {
-			error = VFS_SYNC(mp, MNT_WAIT, &context);
+			error = VFS_SYNC(mp, MNT_WAIT, ctx);
 			if (error) {
 				mount_lock(mp);
 				mp->mnt_kern_flag &= ~MNTK_UNMOUNT;
@@ -849,7 +1100,7 @@ dounmount(mp, flags, skiplistrmp, withref, p)
 	/* make sure there are no one in the mount iterations or lookup */
 	mount_iterdrain(mp);
 
-	error = VFS_UNMOUNT(mp, flags, &context);
+	error = VFS_UNMOUNT(mp, flags, ctx);
 	if (error) {
 		mount_iterreset(mp);
 		mount_lock(mp);
@@ -866,7 +1117,7 @@ dounmount(mp, flags, skiplistrmp, withref, p)
 	if ( mp->mnt_devvp && mp->mnt_vtable->vfc_vfsflags & VFC_VFSLOCALARGS) {
 		mp->mnt_devvp->v_specflags &= ~SI_MOUNTEDON;
 		VNOP_CLOSE(mp->mnt_devvp, mp->mnt_flag & MNT_RDONLY ? FREAD : FREAD|FWRITE,
-                       &context);
+                       ctx);
 		vnode_rele(mp->mnt_devvp);
 	}
 	lck_rw_done(&mp->mnt_rwlock);
@@ -876,7 +1127,7 @@ dounmount(mp, flags, skiplistrmp, withref, p)
 	/* mark the mount point hook in the vp but not drop the ref yet */
 	if ((coveredvp = mp->mnt_vnodecovered) != NULLVP) {
 			vnode_getwithref(coveredvp);
-			vnode_lock(coveredvp);
+			vnode_lock_spin(coveredvp);
 			coveredvp->v_mountedhere = (struct mount *)0;
 			vnode_unlock(coveredvp);
 			vnode_put(coveredvp);
@@ -917,10 +1168,13 @@ out:
 		if ((coveredvp != NULLVP)) {
 			vnode_getwithref(coveredvp);
 			vnode_rele(coveredvp);
-			vnode_lock(coveredvp);
+			vnode_lock_spin(coveredvp);
 			if(mp->mnt_crossref == 0) {
 				vnode_unlock(coveredvp);
 				mount_lock_destroy(mp);
+#if CONFIG_MACF
+				mac_mount_label_destroy(mp);
+#endif
 				FREE_ZONE((caddr_t)mp, sizeof (struct mount), M_MOUNT);
 			}  else {
 				coveredvp->v_lflag |= VL_MOUNTDEAD;
@@ -929,6 +1183,9 @@ out:
 			vnode_put(coveredvp);
 		} else if (mp->mnt_flag & MNT_ROOTFS) {
 				mount_lock_destroy(mp);
+#if CONFIG_MACF
+				mac_mount_label_destroy(mp);
+#endif
 				FREE_ZONE((caddr_t)mp, sizeof (struct mount), M_MOUNT);
 		} else
 			panic("dounmount: no coveredvp");
@@ -949,6 +1206,9 @@ mount_dropcrossref(mount_t mp, vnode_t dp, int need_put)
 			        vnode_put_locked(dp);
 			vnode_unlock(dp);
 			mount_lock_destroy(mp);
+#if CONFIG_MACF
+			mac_mount_label_destroy(mp);
+#endif
 			FREE_ZONE((caddr_t)mp, sizeof (struct mount), M_MOUNT);
 			return;
 		}
@@ -971,17 +1231,12 @@ int print_vmpage_stat=0;
 static int 
 sync_callback(mount_t mp, __unused void * arg)
 {
-	struct proc * p = current_proc();
 	int asyncflag;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
 	if ((mp->mnt_flag & MNT_RDONLY) == 0) {
 			asyncflag = mp->mnt_flag & MNT_ASYNC;
 			mp->mnt_flag &= ~MNT_ASYNC;
-			VFS_SYNC(mp, MNT_NOWAIT, &context);
+			VFS_SYNC(mp, MNT_NOWAIT, vfs_context_current());
 			if (asyncflag)
 				mp->mnt_flag |= MNT_ASYNC;
 	}
@@ -994,7 +1249,7 @@ extern unsigned int dp_pgins, dp_pgouts;
 
 /* ARGSUSED */
 int
-sync(__unused struct proc *p, __unused struct sync_args *uap, __unused register_t *retval)
+sync(__unused proc_t p, __unused struct sync_args *uap, __unused register_t *retval)
 {
 
 	vfs_iterate(LK_NOWAIT, sync_callback, (void *)0);
@@ -1015,25 +1270,36 @@ sync(__unused struct proc *p, __unused struct sync_args *uap, __unused register_
 /*
  * Change filesystem quotas.
  */
-/* ARGSUSED */
+#if QUOTA
+static int quotactl_funneled(proc_t p, struct quotactl_args *uap, register_t *retval);
+
 int
-quotactl(struct proc *p, register struct quotactl_args *uap, __unused register_t *retval)
+quotactl(proc_t p, struct quotactl_args *uap, register_t *retval)
+{
+	boolean_t funnel_state;
+	int error;
+	
+	funnel_state = thread_funnel_set(kernel_flock, TRUE);
+	error = quotactl_funneled(p, uap, retval);
+	thread_funnel_set(kernel_flock, funnel_state);
+	return(error);
+}
+
+static int
+quotactl_funneled(proc_t p, struct quotactl_args *uap, __unused register_t *retval)
 {
-	register struct mount *mp;
+	struct mount *mp;
 	int error, quota_cmd, quota_status;
 	caddr_t datap;
 	size_t fnamelen;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct dqblk my_dqblk;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	AUDIT_ARG(uid, uap->uid, 0, 0, 0);
 	AUDIT_ARG(cmd, uap->cmd);
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -1080,7 +1346,7 @@ quotactl(struct proc *p, register struct quotactl_args *uap, __unused register_t
 	} /* switch */
 
 	if (error == 0) {
-		error = VFS_QUOTACTL(mp, uap->cmd, uap->uid, datap, &context);
+		error = VFS_QUOTACTL(mp, uap->cmd, uap->uid, datap, ctx);
 	}
 
 	switch (quota_cmd) {
@@ -1113,26 +1379,35 @@ quotactl(struct proc *p, register struct quotactl_args *uap, __unused register_t
 
 	return (error);
 }
+#else
+int
+quotactl(__unused proc_t p, __unused struct quotactl_args *uap, __unused register_t *retval)
+{
+	return (EOPNOTSUPP);
+}
+#endif /* QUOTA */
 
 /*
  * Get filesystem statistics.
+ *
+ * Returns:	0			Success
+ *	namei:???
+ *	vfs_update_vfsstat:???
+ *	munge_statfs:EFAULT
  */
 /* ARGSUSED */
 int
-statfs(struct proc *p, register struct statfs_args *uap, __unused register_t *retval)
+statfs(__unused proc_t p, struct statfs_args *uap, __unused register_t *retval)
 {
 	struct mount *mp;
 	struct vfsstatfs *sp;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	vnode_t vp;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+	NDINIT(&nd, LOOKUP, NOTRIGGER | FOLLOW | AUDITVNPATH1, 
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -1141,7 +1416,7 @@ statfs(struct proc *p, register struct statfs_args *uap, __unused register_t *re
 	sp = &mp->mnt_vfsstat;
 	nameidone(&nd);
 
-	error = vfs_update_vfsstat(mp, &context);
+	error = vfs_update_vfsstat(mp, ctx, VFS_USER_EVENT);
 	vnode_put(vp);
 	if (error != 0) 
 		return (error);
@@ -1155,16 +1430,12 @@ statfs(struct proc *p, register struct statfs_args *uap, __unused register_t *re
  */
 /* ARGSUSED */
 int
-fstatfs(struct proc *p, register struct fstatfs_args *uap, __unused register_t *retval)
+fstatfs(__unused proc_t p, struct fstatfs_args *uap, __unused register_t *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct mount *mp;
 	struct vfsstatfs *sp;
 	int error;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
 	AUDIT_ARG(fd, uap->fd);
 
@@ -1179,7 +1450,7 @@ fstatfs(struct proc *p, register struct fstatfs_args *uap, __unused register_t *
 		return (EBADF);
 	}
 	sp = &mp->mnt_vfsstat;
-	if ((error = vfs_update_vfsstat(mp, &context)) != 0) {
+	if ((error = vfs_update_vfsstat(mp,vfs_context_current(),VFS_USER_EVENT)) != 0) {
 		file_drop(uap->fd);
 		return (error);
 	}
@@ -1190,9 +1461,109 @@ fstatfs(struct proc *p, register struct fstatfs_args *uap, __unused register_t *
 	return (error);
 }
 
+/* 
+ * Common routine to handle copying of statfs64 data to user space 
+ */
+static int 
+statfs64_common(struct mount *mp, struct vfsstatfs *sfsp, user_addr_t bufp)
+{
+	int error;
+	struct statfs64 sfs;
+	
+	bzero(&sfs, sizeof(sfs));
+
+	sfs.f_bsize = sfsp->f_bsize;
+	sfs.f_iosize = (int32_t)sfsp->f_iosize;
+	sfs.f_blocks = sfsp->f_blocks;
+	sfs.f_bfree = sfsp->f_bfree;
+	sfs.f_bavail = sfsp->f_bavail;
+	sfs.f_files = sfsp->f_files;
+	sfs.f_ffree = sfsp->f_ffree;
+	sfs.f_fsid = sfsp->f_fsid;
+	sfs.f_owner = sfsp->f_owner;
+	sfs.f_type = mp->mnt_vtable->vfc_typenum;
+	sfs.f_flags = mp->mnt_flag & MNT_VISFLAGMASK;
+	sfs.f_fssubtype = sfsp->f_fssubtype;
+	strlcpy(&sfs.f_fstypename[0], &sfsp->f_fstypename[0], MFSTYPENAMELEN);
+	strlcpy(&sfs.f_mntonname[0], &sfsp->f_mntonname[0], MAXPATHLEN);
+	strlcpy(&sfs.f_mntfromname[0], &sfsp->f_mntfromname[0], MAXPATHLEN);
+
+	error = copyout((caddr_t)&sfs, bufp, sizeof(sfs));
+
+	return(error);
+}
+
+/* 
+ * Get file system statistics in 64-bit mode 
+ */
+int
+statfs64(__unused struct proc *p, struct statfs64_args *uap, __unused register_t *retval)
+{
+	struct mount *mp;
+	struct vfsstatfs *sp;
+	int error;
+	struct nameidata nd;
+	vfs_context_t ctxp = vfs_context_current();
+	vnode_t vp;
+
+	NDINIT(&nd, LOOKUP, NOTRIGGER | FOLLOW | AUDITVNPATH1, 
+		UIO_USERSPACE, uap->path, ctxp);
+	error = namei(&nd);
+	if (error)
+		return (error);
+	vp = nd.ni_vp;
+	mp = vp->v_mount;
+	sp = &mp->mnt_vfsstat;
+	nameidone(&nd);
+
+	error = vfs_update_vfsstat(mp, ctxp, VFS_USER_EVENT);
+	vnode_put(vp);
+	if (error != 0) 
+		return (error);
+
+	error = statfs64_common(mp, sp, uap->buf);
+
+	return (error);
+}
+
+/* 
+ * Get file system statistics in 64-bit mode 
+ */
+int
+fstatfs64(__unused struct proc *p, struct fstatfs64_args *uap, __unused register_t *retval)
+{
+	struct vnode *vp;
+	struct mount *mp;
+	struct vfsstatfs *sp;
+	int error;
+
+	AUDIT_ARG(fd, uap->fd);
+
+	if ( (error = file_vnode(uap->fd, &vp)) )
+		return (error);
+
+	AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
+
+	mp = vp->v_mount;
+	if (!mp) {
+		file_drop(uap->fd);
+		return (EBADF);
+	}
+	sp = &mp->mnt_vfsstat;
+	if ((error = vfs_update_vfsstat(mp, vfs_context_current(), VFS_USER_EVENT)) != 0) {
+		file_drop(uap->fd);
+		return (error);
+	}
+	file_drop(uap->fd);
+
+	error = statfs64_common(mp, sp, uap->buf);
+
+	return (error);
+}
 
 struct getfsstat_struct {
 	user_addr_t	sfsp;
+	user_addr_t	*mp;
 	int		count;
 	int		maxcount;
 	int		flags;
@@ -1206,12 +1577,8 @@ getfsstat_callback(mount_t mp, void * arg)
 	
 	struct getfsstat_struct *fstp = (struct getfsstat_struct *)arg;
 	struct vfsstatfs *sp;
-	struct proc * p = current_proc();
 	int error, my_size;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	if (fstp->sfsp && fstp->count < fstp->maxcount) {
 		sp = &mp->mnt_vfsstat;
@@ -1220,7 +1587,8 @@ getfsstat_callback(mount_t mp, void * arg)
 		 * fsstat cache. MNT_WAIT overrides MNT_NOWAIT.
 		 */
 		if (((fstp->flags & MNT_NOWAIT) == 0 || (fstp->flags & MNT_WAIT)) &&
-			(error = vfs_update_vfsstat(mp, &context))) {
+			(error = vfs_update_vfsstat(mp, ctx,
+			    VFS_USER_EVENT))) {
 			KAUTH_DEBUG("vfs_update_vfsstat returned %d", error);
 			return(VFS_RETURNED);
 		}
@@ -1228,13 +1596,22 @@ getfsstat_callback(mount_t mp, void * arg)
 		/*
 		 * Need to handle LP64 version of struct statfs
 		 */
-		error = munge_statfs(mp, sp, fstp->sfsp, &my_size, IS_64BIT_PROCESS(p), FALSE);
+		error = munge_statfs(mp, sp, fstp->sfsp, &my_size, IS_64BIT_PROCESS(vfs_context_proc(ctx)), FALSE);
 		if (error) {
 			fstp->error = error;
 			return(VFS_RETURNED_DONE);
 		}
 		fstp->sfsp += my_size;
-	}
+
+		if (fstp->mp) {
+			error = mac_mount_label_get(mp, *fstp->mp);
+			if (error) {
+				fstp->error = error;
+				return(VFS_RETURNED_DONE);
+			}
+			fstp->mp++;
+		}
+	}
 	fstp->count++;
 	return(VFS_RETURNED);
 }
@@ -1244,8 +1621,23 @@ getfsstat_callback(mount_t mp, void * arg)
  */
 int
 getfsstat(__unused proc_t p, struct getfsstat_args *uap, int *retval)
+{
+	struct __mac_getfsstat_args muap;
+
+	muap.buf = uap->buf;
+	muap.bufsize = uap->bufsize;
+	muap.mac = USER_ADDR_NULL;
+	muap.macsize = 0;
+	muap.flags = uap->flags;
+
+	return (__mac_getfsstat(p, &muap, retval));
+}
+
+int
+__mac_getfsstat(__unused proc_t p, struct __mac_getfsstat_args *uap, int *retval)
 {
 	user_addr_t sfsp;
+	user_addr_t *mp;
 	int count, maxcount;
 	struct getfsstat_struct fst;
 
@@ -1258,7 +1650,39 @@ getfsstat(__unused proc_t p, struct getfsstat_args *uap, int *retval)
 	sfsp = uap->buf;
 	count = 0;
 
+	mp = NULL;
+
+#if CONFIG_MACF
+	if (uap->mac != USER_ADDR_NULL) {
+		u_int32_t *mp0;
+		int error;
+		int i;
+
+		count = (int)(uap->macsize / (IS_64BIT_PROCESS(p) ? 8 : 4));
+		if (count != maxcount)
+			return (EINVAL);
+
+		/* Copy in the array */
+		MALLOC(mp0, u_int32_t *, uap->macsize, M_MACTEMP, M_WAITOK);
+		error = copyin(uap->mac, mp0, uap->macsize);
+		if (error)
+			return (error);
+
+		/* Normalize to an array of user_addr_t */
+		MALLOC(mp, user_addr_t *, count * sizeof(user_addr_t), M_MACTEMP, M_WAITOK);
+		for (i = 0; i < count; i++) {
+			if (IS_64BIT_PROCESS(p))
+				mp[i] = ((user_addr_t *)mp0)[i];
+			else
+				mp[i] = (user_addr_t)mp0[i];
+		}
+		FREE(mp0, M_MACTEMP);
+	}
+#endif
+
+
 	fst.sfsp = sfsp;
+	fst.mp = mp;
 	fst.flags = uap->flags;
 	fst.count = 0;
 	fst.error = 0;
@@ -1267,6 +1691,9 @@ getfsstat(__unused proc_t p, struct getfsstat_args *uap, int *retval)
 	
 	vfs_iterate(0, getfsstat_callback, &fst);
 
+	if (mp)
+		FREE(mp, M_MACTEMP);
+
 	if (fst.error ) {
 		KAUTH_DEBUG("ERROR - %s gets %d", p->p_comm, fst.error);
 		return(fst.error);
@@ -1279,11 +1706,74 @@ getfsstat(__unused proc_t p, struct getfsstat_args *uap, int *retval)
 	return (0);
 }
 
+static int
+getfsstat64_callback(mount_t mp, void * arg)
+{
+	struct getfsstat_struct *fstp = (struct getfsstat_struct *)arg;
+	struct vfsstatfs *sp;
+	int error;
+
+	if (fstp->sfsp && fstp->count < fstp->maxcount) {
+		sp = &mp->mnt_vfsstat;
+		/*
+		 * If MNT_NOWAIT is specified, do not refresh the
+		 * fsstat cache. MNT_WAIT overrides MNT_NOWAIT.
+		 */
+		if (((fstp->flags & MNT_NOWAIT) == 0 || (fstp->flags & MNT_WAIT)) &&
+		    (error = vfs_update_vfsstat(mp, vfs_context_current(), VFS_USER_EVENT))) {
+			KAUTH_DEBUG("vfs_update_vfsstat returned %d", error);
+			return(VFS_RETURNED);
+		}
+
+		error = statfs64_common(mp, sp, fstp->sfsp);
+		if (error) {
+			fstp->error = error;
+			return(VFS_RETURNED_DONE);
+		}
+		fstp->sfsp += sizeof(struct statfs64);
+	}
+	fstp->count++;
+	return(VFS_RETURNED);
+}
+
+/*
+ * Get statistics on all file systems in 64 bit mode.
+ */
+int
+getfsstat64(__unused proc_t p, struct getfsstat64_args *uap, int *retval)
+{
+	user_addr_t sfsp;
+	int count, maxcount;
+	struct getfsstat_struct fst;
+
+	maxcount = uap->bufsize / sizeof(struct statfs64);
+
+	sfsp = uap->buf;
+	count = 0;
+
+	fst.sfsp = sfsp;
+	fst.flags = uap->flags;
+	fst.count = 0;
+	fst.error = 0;
+	fst.maxcount = maxcount;
+
+	vfs_iterate(0, getfsstat64_callback, &fst);
+
+	if (fst.error ) {
+		KAUTH_DEBUG("ERROR - %s gets %d", p->p_comm, fst.error);
+		return(fst.error);
+	}
+
+	if (fst.sfsp && fst.count > fst.maxcount)
+		*retval = fst.maxcount;
+	else
+		*retval = fst.count;
+
+	return (0);
+}
+
 #if COMPAT_GETFSSTAT
-ogetfsstat(p, uap, retval)
-	struct proc *p;
-	register struct getfsstat_args *uap;
-	register_t *retval;
+ogetfsstat(proc_t p, struct getfsstat_args *uap, register_t *retval)
 {
 	return (ENOTSUP);
 }
@@ -1293,17 +1783,36 @@ ogetfsstat(p, uap, retval)
  * Change current working directory to a given file descriptor.
  */
 /* ARGSUSED */
-int
-fchdir(struct proc *p, struct fchdir_args *uap, __unused register_t *retval)
+static int
+common_fchdir(proc_t p, struct fchdir_args *uap, int per_thread)
 {
-	register struct filedesc *fdp = p->p_fd;
-	struct vnode *vp, *tdp, *tvp;
+	struct filedesc *fdp = p->p_fd;
+	vnode_t vp;
+	vnode_t tdp;
+	vnode_t tvp;
 	struct mount *mp;
 	int error;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	if (per_thread && uap->fd == -1) {
+		/*
+		 * Switching back from per-thread to per process CWD; verify we
+		 * in fact have one before proceeding.  The only success case
+		 * for this code path is to return 0 preemptively after zapping
+		 * the thread structure contents.
+		 */
+		thread_t th = vfs_context_thread(ctx);
+		if (th) {
+			uthread_t uth = get_bsdthread_info(th);
+			tvp = uth->uu_cdir;
+			uth->uu_cdir = NULLVP;
+			if (tvp != NULLVP) {
+				vnode_rele(tvp);
+				return (0);
+			}
+		}
+		return (EBADF);
+	}
 
 	if ( (error = file_vnode(uap->fd, &vp)) )
 		return(error);
@@ -1314,16 +1823,26 @@ fchdir(struct proc *p, struct fchdir_args *uap, __unused register_t *retval)
 
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-	if (vp->v_type != VDIR)
+	if (vp->v_type != VDIR) {
 		error = ENOTDIR;
-	else
-		error = vnode_authorize(vp, NULL, KAUTH_VNODE_SEARCH, &context);
+		goto out;
+	}
+
+#if CONFIG_MACF
+	error = mac_vnode_check_chdir(ctx, vp);
+	if (error)
+		goto out;
+#endif
+	error = vnode_authorize(vp, NULL, KAUTH_VNODE_SEARCH, ctx);
+	if (error)
+		goto out;
+
 	while (!error && (mp = vp->v_mountedhere) != NULL) {
 		if (vfs_busy(mp, LK_NOWAIT)) {
 			error = EACCES;
 			goto out;
 		}
-		error = VFS_ROOT(mp, &tdp, &context);
+		error = VFS_ROOT(mp, &tdp, ctx);
 		vfs_unbusy(mp);
 		if (error)
 			break;
@@ -1336,10 +1855,23 @@ fchdir(struct proc *p, struct fchdir_args *uap, __unused register_t *retval)
 	        goto out;
 	vnode_put(vp);
 
-	proc_fdlock(p);
-	tvp = fdp->fd_cdir;
-	fdp->fd_cdir = vp;
-	proc_fdunlock(p);
+	if (per_thread) {
+		thread_t th = vfs_context_thread(ctx);
+		if (th) {
+			uthread_t uth = get_bsdthread_info(th);
+			tvp = uth->uu_cdir;
+			uth->uu_cdir = vp;
+			OSBitOrAtomic(P_THCWD, (UInt32 *)&p->p_flag);
+		} else {
+			vnode_rele(vp);
+			return (ENOENT);
+		}
+	} else {
+		proc_fdlock(p);
+		tvp = fdp->fd_cdir;
+		fdp->fd_cdir = vp;
+		proc_fdunlock(p);
+	}
 
 	if (tvp)
 	        vnode_rele(tvp);
@@ -1353,25 +1885,39 @@ out:
 	return(error);
 }
 
+int
+fchdir(proc_t p, struct fchdir_args *uap, __unused register_t *retval)
+{
+	return common_fchdir(p, uap, 0);
+}
+
+int
+__pthread_fchdir(proc_t p, struct __pthread_fchdir_args *uap, __unused register_t *retval)
+{
+	return common_fchdir(p, (void *)uap, 1);
+}
+
 /*
  * Change current working directory (``.'').
+ *
+ * Returns:	0			Success
+ *	change_dir:ENOTDIR
+ *	change_dir:???
+ *	vnode_ref:ENOENT		No such file or directory
  */
 /* ARGSUSED */
-int
-chdir(struct proc *p, struct chdir_args *uap, __unused register_t *retval)
+static int
+common_chdir(proc_t p, struct chdir_args *uap, int per_thread)
 {
-	register struct filedesc *fdp = p->p_fd;
+	struct filedesc *fdp = p->p_fd;
 	int error;
 	struct nameidata nd;
-	struct vnode *tvp;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vnode_t tvp;
+	vfs_context_t ctx = vfs_context_current();
 
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
-	error = change_dir(&nd, &context);
+		UIO_USERSPACE, uap->path, ctx);
+	error = change_dir(&nd, ctx);
 	if (error)
 		return (error);
 	if ( (error = vnode_ref(nd.ni_vp)) ) {
@@ -1383,10 +1929,23 @@ chdir(struct proc *p, struct chdir_args *uap, __unused register_t *retval)
 	 */
 	vnode_put(nd.ni_vp);
 
-	proc_fdlock(p);
-	tvp = fdp->fd_cdir;
-	fdp->fd_cdir = nd.ni_vp;
-	proc_fdunlock(p);
+	if (per_thread) {
+		thread_t th = vfs_context_thread(ctx);
+		if (th) {
+			uthread_t uth = get_bsdthread_info(th);
+			tvp = uth->uu_cdir;
+			uth->uu_cdir = nd.ni_vp;
+			OSBitOrAtomic(P_THCWD, (UInt32 *)&p->p_flag);
+		} else {
+			vnode_rele(nd.ni_vp);
+			return (ENOENT);
+		}
+	} else {
+		proc_fdlock(p);
+		tvp = fdp->fd_cdir;
+		fdp->fd_cdir = nd.ni_vp;
+		proc_fdunlock(p);
+	}
 
 	if (tvp)
 	        vnode_rele(tvp);
@@ -1394,43 +1953,50 @@ chdir(struct proc *p, struct chdir_args *uap, __unused register_t *retval)
 	return (0);
 }
 
+int
+chdir(proc_t p, struct chdir_args *uap, __unused register_t *retval)
+{
+	return common_chdir(p, (void *)uap, 0);
+}
+
+int
+__pthread_chdir(proc_t p, struct __pthread_chdir_args *uap, __unused register_t *retval)
+{
+	return common_chdir(p, (void *)uap, 1);
+}
+
+
 /*
  * Change notion of root (``/'') directory.
  */
 /* ARGSUSED */
 int
-chroot(struct proc *p, struct chroot_args *uap, __unused register_t *retval)
+chroot(proc_t p, struct chroot_args *uap, __unused register_t *retval)
 {
-	register struct filedesc *fdp = p->p_fd;
+	struct filedesc *fdp = p->p_fd;
 	int error;
 	struct nameidata nd;
-	boolean_t	shared_regions_active;
-	struct vnode *tvp;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vnode_t tvp;
+	vfs_context_t ctx = vfs_context_current();
 
 	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
 		return (error);
 
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
-	error = change_dir(&nd, &context);
+		UIO_USERSPACE, uap->path, ctx);
+	error = change_dir(&nd, ctx);
 	if (error)
 		return (error);
 
-	if(p->p_flag & P_NOSHLIB) {
-		shared_regions_active = FALSE;
-	} else {
-		shared_regions_active = TRUE;
-	}
-	if ((error = clone_system_shared_regions(shared_regions_active,
-						 TRUE, /* chain_regions */
-						 (int)nd.ni_vp))) {
+#if CONFIG_MACF
+	error = mac_vnode_check_chroot(ctx, nd.ni_vp,
+	    &nd.ni_cnd);
+	if (error) {
 		vnode_put(nd.ni_vp);
 		return (error);
 	}
+#endif
+
 	if ( (error = vnode_ref(nd.ni_vp)) ) {
 	        vnode_put(nd.ni_vp);
 		return (error);
@@ -1451,23 +2017,41 @@ chroot(struct proc *p, struct chroot_args *uap, __unused register_t *retval)
 
 /*
  * Common routine for chroot and chdir.
+ *
+ * Returns:	0			Success
+ *		ENOTDIR			Not a directory
+ *		namei:???		[anything namei can return]
+ *		vnode_authorize:???	[anything vnode_authorize can return]
  */
 static int
 change_dir(struct nameidata *ndp, vfs_context_t ctx)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	int error;
 
 	if ((error = namei(ndp)))
 		return (error);
 	nameidone(ndp);
 	vp = ndp->ni_vp;
-	if (vp->v_type != VDIR)
-		error = ENOTDIR;
-	else
-		error = vnode_authorize(vp, NULL, KAUTH_VNODE_SEARCH, ctx);
-	if (error)
+
+	if (vp->v_type != VDIR) {
 		vnode_put(vp);
+		return (ENOTDIR);
+	}
+
+#if CONFIG_MACF
+	error = mac_vnode_check_chdir(ctx, vp);
+	if (error) {
+		vnode_put(vp);
+		return (error);
+	}
+#endif
+
+	error = vnode_authorize(vp, NULL, KAUTH_VNODE_SEARCH, ctx);
+	if (error) {
+		vnode_put(vp);
+		return (error);
+	}
 
 	return (error);
 }
@@ -1475,21 +2059,35 @@ change_dir(struct nameidata *ndp, vfs_context_t ctx)
 /*
  * Check permissions, allocate an open file structure,
  * and call the device open routine if any.
+ *
+ * Returns:	0			Success
+ *		EINVAL
+ *		EINTR
+ *	falloc:ENFILE
+ *	falloc:EMFILE
+ *	falloc:ENOMEM
+ *	vn_open_auth:???
+ *	dupfdopen:???
+ *	VNOP_ADVLOCK:???
+ *	vnode_setsize:???
  */
-
 #warning XXX implement uid, gid
-static int
-open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap, register_t *retval)
+int
+open1(vfs_context_t ctx, struct nameidata *ndp, int uflags, struct vnode_attr *vap, register_t *retval)
 {
-	struct proc *p = vfs_context_proc(ctx);
-	register struct filedesc *fdp = p->p_fd;
-	register struct fileproc *fp;
-	register struct vnode *vp;
+	proc_t p = vfs_context_proc(ctx);
+	uthread_t uu = get_bsdthread_info(vfs_context_thread(ctx));
+	struct filedesc *fdp = p->p_fd;
+	struct fileproc *fp;
+	vnode_t vp;
 	int flags, oflags;
 	struct fileproc *nfp;
 	int type, indx, error;
 	struct flock lf;
-	struct nameidata nd;
+	int no_controlling_tty = 0;
+	int deny_controlling_tty = 0;
+	struct session *sessp = SESSION_NULL;
+	struct vfs_context context = *vfs_context_current();	/* local copy */
 
 	oflags = uflags;
 
@@ -1500,19 +2098,43 @@ open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap,
 	AUDIT_ARG(fflags, oflags);
 	AUDIT_ARG(mode, vap->va_mode);
 
-	if ( (error = falloc(p, &nfp, &indx)) ) {
+	if ( (error = falloc(p, &nfp, &indx, ctx)) ) {
 		return (error);
 	}
 	fp = nfp;
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, upath, ctx);
-	p->p_dupfd = -indx - 1;			/* XXX check for fdopen */
+	uu->uu_dupfd = -indx - 1;
 
-	if ((error = vn_open_auth(&nd, &flags, vap))) {
-		if ((error == ENODEV || error == ENXIO) && (p->p_dupfd >= 0)) {	/* XXX from fdopen */
-			if ((error = dupfdopen(fdp, indx, p->p_dupfd, flags, error)) == 0) {
-				fp_drop(p, indx, 0, 0);
+	if (!(p->p_flag & P_CONTROLT)) {
+		sessp = proc_session(p);
+		no_controlling_tty = 1;
+		/*
+		 * If conditions would warrant getting a controlling tty if
+		 * the device being opened is a tty (see ttyopen in tty.c),
+		 * but the open flags deny it, set a flag in the session to
+		 * prevent it.
+		 */
+		if (SESS_LEADER(p, sessp) &&
+		    sessp->s_ttyvp == NULL &&
+		    (flags & O_NOCTTY)) {
+			session_lock(sessp);
+		    	sessp->s_flags |= S_NOCTTY;
+			session_unlock(sessp);
+			deny_controlling_tty = 1;
+		}
+	}
+
+	if ((error = vn_open_auth(ndp, &flags, vap))) {
+		if ((error == ENODEV || error == ENXIO) && (uu->uu_dupfd >= 0)){	/* XXX from fdopen */
+			if ((error = dupfdopen(fdp, indx, uu->uu_dupfd, flags, error)) == 0) {
+				fp_drop(p, indx, NULL, 0);
 			        *retval = indx;
+				if (deny_controlling_tty) {
+					session_lock(sessp);
+					sessp->s_flags &= ~S_NOCTTY;
+					session_unlock(sessp);
+				}
+				if (sessp != SESSION_NULL)
+					session_rele(sessp);
 				return (0);
 			}
 		}
@@ -1520,10 +2142,17 @@ open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap,
 		        error = EINTR;
 		fp_free(p, indx, fp);
 
+		if (deny_controlling_tty) {
+			session_lock(sessp);
+			sessp->s_flags &= ~S_NOCTTY;
+			session_unlock(sessp);
+		}
+		if (sessp != SESSION_NULL)
+			session_rele(sessp);
 		return (error);
 	}
-	p->p_dupfd = 0;
-	vp = nd.ni_vp;
+	uu->uu_dupfd = 0;
+	vp = ndp->ni_vp;
 
 	fp->f_fglob->fg_flag = flags & (FMASK | O_EVTONLY);
 	fp->f_fglob->fg_type = DTYPE_VNODE;
@@ -1541,6 +2170,12 @@ open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap,
 		type = F_FLOCK;
 		if ((flags & FNONBLOCK) == 0)
 			type |= F_WAIT;
+#if CONFIG_MACF
+		error = mac_file_check_lock(vfs_context_ucred(ctx), fp->f_fglob,
+		    F_SETLK, &lf);
+		if (error)
+			goto bad;
+#endif
 		if ((error = VNOP_ADVLOCK(vp, (caddr_t)fp->f_fglob, F_SETLK, &lf, type, ctx)))
 			goto bad;
 		fp->f_fglob->fg_flag |= FHASLOCK;
@@ -1550,6 +2185,35 @@ open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap,
 	if ((flags & O_TRUNC) && ((error = vnode_setsize(vp, (off_t)0, 0, ctx)) != 0))
 		goto bad;
 
+	/*
+	 * If the open flags denied the acquisition of a controlling tty,
+	 * clear the flag in the session structure that prevented the lower
+	 * level code from assigning one.
+	 */
+	if (deny_controlling_tty) {
+		session_lock(sessp);
+		sessp->s_flags &= ~S_NOCTTY;
+		session_unlock(sessp);
+	}
+
+	/*
+	 * If a controlling tty was set by the tty line discipline, then we
+	 * want to set the vp of the tty into the session structure.  We have
+	 * a race here because we can't get to the vp for the tp in ttyopen,
+	 * because it's not passed as a parameter in the open path.
+	 */
+	if (no_controlling_tty && (p->p_flag & P_CONTROLT)) {
+		vnode_t ttyvp;
+		vnode_ref(vp);
+		session_lock(sessp);
+		ttyvp = sessp->s_ttyvp;
+		sessp->s_ttyvp = vp;
+		sessp->s_ttyvid = vnode_vid(vp);
+		session_unlock(sessp);
+		if (ttyvp != NULLVP)
+			vnode_rele(ttyvp);
+	}
+
 	vnode_put(vp);
 
 	proc_fdlock(p);
@@ -1559,9 +2223,22 @@ open1(vfs_context_t ctx, user_addr_t upath, int uflags, struct vnode_attr *vap,
 
 	*retval = indx;
 
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
 	return (0);
 bad:
-	vn_close(vp, fp->f_fglob->fg_flag, fp->f_fglob->fg_cred, p);
+	if (deny_controlling_tty) {
+		session_lock(sessp);
+		sessp->s_flags &= ~S_NOCTTY;
+		session_unlock(sessp);
+	}
+	if (sessp != SESSION_NULL)
+		session_rele(sessp);
+
+	/* Modify local copy (to not damage thread copy) */
+	context.vc_ucred = fp->f_fglob->fg_cred;
+
+	vn_close(vp, fp->f_fglob->fg_flag, &context);
 	vnode_put(vp);
 	fp_free(p, indx, fp);
 
@@ -1594,13 +2271,13 @@ bad:
  *		in the code they originated.
  */
 int
-open_extended(struct proc *p, struct open_extended_args *uap, register_t *retval)
+open_extended(proc_t p, struct open_extended_args *uap, register_t *retval)
 {
-	struct vfs_context context;
-	register struct filedesc *fdp = p->p_fd;
+	struct filedesc *fdp = p->p_fd;
 	int ciferror;
 	kauth_filesec_t xsecdst;
 	struct vnode_attr va;
+	struct nameidata nd;
 	int cmode;
 
 	xsecdst = NULL;
@@ -1608,9 +2285,6 @@ open_extended(struct proc *p, struct open_extended_args *uap, register_t *retval
 	    ((ciferror = kauth_copyinfilesec(uap->xsecurity, &xsecdst)) != 0))
 		return ciferror;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
 	cmode = ((uap->mode &~ fdp->fd_cmask) & ALLPERMS) & ~S_ISTXT;
 	VATTR_SET(&va, va_mode, cmode);
@@ -1621,7 +2295,9 @@ open_extended(struct proc *p, struct open_extended_args *uap, register_t *retval
 	if (xsecdst != NULL)
 		VATTR_SET(&va, va_acl, &xsecdst->fsec_acl);
 
-	ciferror = open1(&context, uap->path, uap->flags, &va, retval);
+	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, UIO_USERSPACE, uap->path, vfs_context_current());
+
+	ciferror = open1(vfs_context_current(), &nd, uap->flags, &va, retval);
 	if (xsecdst != NULL)
 		kauth_filesec_free(xsecdst);
 
@@ -1629,22 +2305,29 @@ open_extended(struct proc *p, struct open_extended_args *uap, register_t *retval
 }
 
 int
-open(struct proc *p, struct open_args *uap, register_t *retval)
+open(proc_t p, struct open_args *uap, register_t *retval)
 {
-	struct vfs_context context;
-	register struct filedesc *fdp = p->p_fd;
+	__pthread_testcancel(1);
+	return(open_nocancel(p, (struct open_nocancel_args *)uap, retval));
+}
+
+
+int
+open_nocancel(proc_t p, struct open_nocancel_args *uap, register_t *retval)
+{
+	struct filedesc *fdp = p->p_fd;
 	struct vnode_attr va;
+	struct nameidata nd;
 	int cmode;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
 	/* Mask off all but regular access permissions */
 	cmode = ((uap->mode &~ fdp->fd_cmask) & ALLPERMS) & ~S_ISTXT;
 	VATTR_SET(&va, va_mode, cmode & ACCESSPERMS);
 
-	return(open1(&context, uap->path, uap->flags, &va, retval));
+	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, UIO_USERSPACE, uap->path, vfs_context_current());
+
+	return(open1(vfs_context_current(), &nd, uap->flags, &va, retval));
 }
 
 
@@ -1654,33 +2337,30 @@ open(struct proc *p, struct open_args *uap, register_t *retval)
 static int mkfifo1(vfs_context_t ctx, user_addr_t upath, struct vnode_attr *vap);
 
 int
-mknod(struct proc *p, register struct mknod_args *uap, __unused register_t *retval)
+mknod(proc_t p, struct mknod_args *uap, __unused register_t *retval)
 {
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	int whiteout = 0;
 	struct nameidata nd;
 	vnode_t	vp, dvp;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
  	VATTR_INIT(&va);
  	VATTR_SET(&va, va_mode, (uap->mode & ALLPERMS) & ~p->p_fd->fd_cmask);
  	VATTR_SET(&va, va_rdev, uap->dev);
 
 	/* If it's a mknod() of a FIFO, call mkfifo1() instead */
 	if ((uap->mode & S_IFMT) == S_IFIFO)
- 		return(mkfifo1(&context, uap->path, &va));
+ 		return(mkfifo1(ctx, uap->path, &va));
 
 	AUDIT_ARG(mode, uap->mode);
 	AUDIT_ARG(dev, uap->dev);
 
-	if ((error = suser(context.vc_ucred, &p->p_acflag)))
+	if ((error = suser(vfs_context_ucred(ctx), &p->p_acflag)))
 		return (error);
 	NDINIT(&nd, CREATE, LOCKPARENT | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -1692,9 +2372,6 @@ mknod(struct proc *p, register struct mknod_args *uap, __unused register_t *retv
 		goto out;
 	}
 
- 	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context)) != 0)
- 		goto out;
-  
 	switch (uap->mode & S_IFMT) {
 	case S_IFMT:	/* used by badsect to flag bad sectors */
 		VATTR_SET(&va, va_type, VBAD);
@@ -1712,10 +2389,23 @@ mknod(struct proc *p, register struct mknod_args *uap, __unused register_t *retv
 		error = EINVAL;
 		goto out;
 	}
+
+#if CONFIG_MACF
+	if (!whiteout) {
+		error = mac_vnode_check_create(ctx,
+		    nd.ni_dvp, &nd.ni_cnd, &va);
+		if (error)
+			goto out;
+	}
+#endif
+
+ 	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx)) != 0)
+ 		goto out;
+
 	if (whiteout) {
-		error = VNOP_WHITEOUT(dvp, &nd.ni_cnd, CREATE, &context);
+		error = VNOP_WHITEOUT(dvp, &nd.ni_cnd, CREATE, ctx);
 	} else {
-		error = vn_create(dvp, &vp, &nd.ni_cnd, &va, 0, &context);
+		error = vn_create(dvp, &vp, &nd.ni_cnd, &va, 0, ctx);
 	}
 	if (error)
 		goto out;
@@ -1732,9 +2422,11 @@ mknod(struct proc *p, register struct mknod_args *uap, __unused register_t *retv
 		if (update_flags)
 		        vnode_update_identity(vp, dvp, nd.ni_cnd.cn_nameptr, nd.ni_cnd.cn_namelen, nd.ni_cnd.cn_hash, update_flags);
 
-		add_fsevent(FSE_CREATE_FILE, &context,
+#if CONFIG_FSE
+		add_fsevent(FSE_CREATE_FILE, ctx,
 		    FSE_ARG_VNODE, vp,
 		    FSE_ARG_DONE);
+#endif
 	}
 
 out:
@@ -1753,6 +2445,12 @@ out:
 
 /*
  * Create a named pipe.
+ *
+ * Returns:	0			Success
+ *		EEXIST
+ *	namei:???
+ *	vnode_authorize:???
+ *	vn_create:???
  */
 static int
 mkfifo1(vfs_context_t ctx, user_addr_t upath, struct vnode_attr *vap)
@@ -1774,10 +2472,19 @@ mkfifo1(vfs_context_t ctx, user_addr_t upath, struct vnode_attr *vap)
    		error = EEXIST;
    		goto out;
    	}
+   	VATTR_SET(vap, va_type, VFIFO);
+
+#if CONFIG_MACF
+	error = mac_vnode_check_create(ctx, nd.ni_dvp,
+	    &nd.ni_cnd, vap);
+	if (error)
+		goto out;
+#endif
+
+
    	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx)) != 0)
    		goto out;
 
-   	VATTR_SET(vap, va_type, VFIFO);
  	
   	error = vn_create(dvp, &vp, &nd.ni_cnd, vap, 0, ctx);
 out:
@@ -1818,11 +2525,10 @@ out:
  *		in the code they originated.
  */
 int
-mkfifo_extended(struct proc *p, struct mkfifo_extended_args *uap, __unused register_t *retval)
+mkfifo_extended(proc_t p, struct mkfifo_extended_args *uap, __unused register_t *retval)
 {
 	int ciferror;
 	kauth_filesec_t xsecdst;
-	struct vfs_context context;
 	struct vnode_attr va;
 
 	xsecdst = KAUTH_FILESEC_NONE;
@@ -1831,9 +2537,6 @@ mkfifo_extended(struct proc *p, struct mkfifo_extended_args *uap, __unused regis
 			return ciferror;
 	}
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
    	VATTR_SET(&va, va_mode, (uap->mode & ALLPERMS) & ~p->p_fd->fd_cmask);
 	if (uap->uid != KAUTH_UID_NONE)
@@ -1843,7 +2546,7 @@ mkfifo_extended(struct proc *p, struct mkfifo_extended_args *uap, __unused regis
 	if (xsecdst != KAUTH_FILESEC_NONE)
 		VATTR_SET(&va, va_acl, &xsecdst->fsec_acl);
 
-	ciferror = mkfifo1(&context, uap->path, &va);
+	ciferror = mkfifo1(vfs_context_current(), uap->path, &va);
 
 	if (xsecdst != KAUTH_FILESEC_NONE)
 		kauth_filesec_free(xsecdst);
@@ -1852,41 +2555,44 @@ mkfifo_extended(struct proc *p, struct mkfifo_extended_args *uap, __unused regis
 
 /* ARGSUSED */
 int
-mkfifo(struct proc *p, register struct mkfifo_args *uap, __unused register_t *retval)
+mkfifo(proc_t p, struct mkfifo_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
 	struct vnode_attr va;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
    	VATTR_INIT(&va);
    	VATTR_SET(&va, va_mode, (uap->mode & ALLPERMS) & ~p->p_fd->fd_cmask);
 
-	return(mkfifo1(&context, uap->path, &va));
+	return(mkfifo1(vfs_context_current(), uap->path, &va));
 }
 
 /*
  * Make a hard file link.
+ *
+ * Returns:	0			Success
+ *		EPERM
+ *		EEXIST
+ *		EXDEV
+ *	namei:???
+ *	vnode_authorize:???
+ *	VNOP_LINK:???
  */
 /* ARGSUSED */
 int
-link(struct proc *p, register struct link_args *uap, __unused register_t *retval)
+link(__unused proc_t p, struct link_args *uap, __unused register_t *retval)
 {
 	vnode_t	vp, dvp, lvp;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	fse_info finfo;
 	int need_event, has_listeners;
+	char *target_path = NULL;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 	vp = dvp = lvp = NULLVP;
 
 	/* look up the object we are linking to */
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -1894,25 +2600,49 @@ link(struct proc *p, register struct link_args *uap, __unused register_t *retval
 
 	nameidone(&nd);
 
-	/* we're not allowed to link to directories */
+	/*
+	 * Normally, linking to directories is not supported.
+	 * However, some file systems may have limited support.
+	 */
 	if (vp->v_type == VDIR) {
-		error = EPERM;   /* POSIX */
-		goto out;
+		if (!(vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSDIRLINKS)) {
+			error = EPERM;   /* POSIX */
+			goto out;
+		}
+		/* Linking to a directory requires ownership. */
+		if (!kauth_cred_issuser(vfs_context_ucred(ctx))) {
+			struct vnode_attr dva;
+
+			VATTR_INIT(&dva);
+			VATTR_WANTED(&dva, va_uid);
+			if (vnode_getattr(vp, &dva, ctx) != 0 ||
+			    !VATTR_IS_SUPPORTED(&dva, va_uid) ||
+			    (dva.va_uid != kauth_cred_getuid(vfs_context_ucred(ctx)))) {
+				error = EACCES;
+				goto out;
+			}
+		}
 	}
 
-  	/* or to anything that kauth doesn't want us to (eg. immutable items) */
-  	if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_LINKTARGET, &context)) != 0)
- 		goto out;
-
 	/* lookup the target node */
 	nd.ni_cnd.cn_nameiop = CREATE;
-	nd.ni_cnd.cn_flags = LOCKPARENT | AUDITVNPATH2;
+	nd.ni_cnd.cn_flags = LOCKPARENT | AUDITVNPATH2 | CN_NBMOUNTLOOK;
 	nd.ni_dirp = uap->link;
 	error = namei(&nd);
 	if (error != 0)
 		goto out;
 	dvp = nd.ni_dvp;
 	lvp = nd.ni_vp;
+
+#if CONFIG_MACF
+	if ((error = mac_vnode_check_link(ctx, dvp, vp, &nd.ni_cnd)) != 0)
+		goto out2;
+#endif
+
+  	/* or to anything that kauth doesn't want us to (eg. immutable items) */
+  	if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_LINKTARGET, ctx)) != 0)
+ 		goto out2;
+
 	/* target node must not exist */
 	if (lvp != NULLVP) {
 		error = EEXIST;
@@ -1925,55 +2655,73 @@ link(struct proc *p, register struct link_args *uap, __unused register_t *retval
   	}
 		
   	/* authorize creation of the target note */
-  	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context)) != 0)
+  	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx)) != 0)
   		goto out2;
 
 	/* and finally make the link */
-	error = VNOP_LINK(vp, dvp, &nd.ni_cnd, &context);
+	error = VNOP_LINK(vp, dvp, &nd.ni_cnd, ctx);
 	if (error)
 		goto out2;
 
+#if CONFIG_FSE
 	need_event = need_fsevent(FSE_CREATE_FILE, dvp);
+#else
+	need_event = 0;
+#endif
 	has_listeners = kauth_authorize_fileop_has_listeners();
 
 	if (need_event || has_listeners) {
-		char *target_path = NULL;
 		char *link_to_path = NULL;
 		int len, link_name_len;
 
 		/* build the path to the new link file */
-		target_path = get_pathbuff();
+		GET_PATH(target_path);
+		if (target_path == NULL) {
+			error = ENOMEM;
+			goto out2;
+		}
+
 		len = MAXPATHLEN;
 		vn_getpath(dvp, target_path, &len);
-		target_path[len-1] = '/';
-		strcpy(&target_path[len], nd.ni_cnd.cn_nameptr);
-		len += nd.ni_cnd.cn_namelen;
+		if ((len + 1 + nd.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+		    target_path[len-1] = '/';
+		    strlcpy(&target_path[len], nd.ni_cnd.cn_nameptr, MAXPATHLEN-len);
+		    len += nd.ni_cnd.cn_namelen;
+		}
 
 		if (has_listeners) {
 		        /* build the path to file we are linking to */
-		        link_to_path = get_pathbuff();
+			GET_PATH(link_to_path);
+			if (link_to_path == NULL) {
+				error = ENOMEM;
+				goto out2;
+			}
+
 			link_name_len = MAXPATHLEN;
 			vn_getpath(vp, link_to_path, &link_name_len);
 
-			/* call out to allow 3rd party notification of rename. 
+			/*
+			 * Call out to allow 3rd party notification of rename. 
 			 * Ignore result of kauth_authorize_fileop call.
 			 */
-			kauth_authorize_fileop(vfs_context_ucred(&context), KAUTH_FILEOP_LINK, 
+			kauth_authorize_fileop(vfs_context_ucred(ctx), KAUTH_FILEOP_LINK, 
 					       (uintptr_t)link_to_path, (uintptr_t)target_path);
-			if (link_to_path != NULL)
-			        release_pathbuff(link_to_path);
+			if (link_to_path != NULL) {
+				RELEASE_PATH(link_to_path);
+			}
 		}
+#if CONFIG_FSE
 		if (need_event) {
 		        /* construct fsevent */
-		        if (get_fse_info(vp, &finfo, &context) == 0) {
+		        if (get_fse_info(vp, &finfo, ctx) == 0) {
 			        // build the path to the destination of the link
-			        add_fsevent(FSE_CREATE_FILE, &context,
+			        add_fsevent(FSE_CREATE_FILE, ctx,
 					    FSE_ARG_STRING, len, target_path,
 					    FSE_ARG_FINFO, &finfo,
 					    FSE_ARG_DONE);
 			}
 		}
-		release_pathbuff(target_path);
+#endif
 	}
 out2:
 	/*
@@ -1981,6 +2729,9 @@ out2:
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
 	nameidone(&nd);
+	if (target_path != NULL) {
+		RELEASE_PATH(target_path);
+	}
 out:
 	if (lvp)
 		vnode_put(lvp);
@@ -1997,19 +2748,16 @@ out:
  */
 /* ARGSUSED */
 int
-symlink(struct proc *p, register struct symlink_args *uap, __unused register_t *retval)
+symlink(proc_t p, struct symlink_args *uap, __unused register_t *retval)
 {
 	struct vnode_attr va;
 	char *path;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	vnode_t	vp, dvp;
 	size_t dummy=0;
 	
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	MALLOC_ZONE(path, char *, MAXPATHLEN, M_NAMEI, M_WAITOK);
 	error = copyinstr(uap->path, path, MAXPATHLEN, &dummy);
 	if (error)
@@ -2017,80 +2765,94 @@ symlink(struct proc *p, register struct symlink_args *uap, __unused register_t *
 	AUDIT_ARG(text, path);	/* This is the link string */
 
 	NDINIT(&nd, CREATE, LOCKPARENT | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->link, &context);
+		UIO_USERSPACE, uap->link, ctx);
 	error = namei(&nd);
 	if (error)
 		goto out;
 	dvp = nd.ni_dvp;
 	vp = nd.ni_vp;
 
-	if (vp == NULL) {
-		VATTR_INIT(&va);
-		VATTR_SET(&va, va_type, VLNK);
-		VATTR_SET(&va, va_mode, ACCESSPERMS & ~p->p_fd->fd_cmask);
-
- 		/* authorize */
-		error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, &context);
-		/* get default ownership, etc. */
-		if (error == 0)
-			error = vnode_authattr_new(dvp, &va, 0, &context);
- 		if (error == 0)
-			error = VNOP_SYMLINK(dvp, &vp, &nd.ni_cnd, &va, path, &context);
+	VATTR_INIT(&va);
+	VATTR_SET(&va, va_type, VLNK);
+	VATTR_SET(&va, va_mode, ACCESSPERMS & ~p->p_fd->fd_cmask);
+#if CONFIG_MACF
+	error = mac_vnode_check_create(ctx,
+			dvp, &nd.ni_cnd, &va);
+#endif
+	if (error != 0) {
+	    goto skipit;
+	}
 
-		/* do fallback attribute handling */
-		if (error == 0)
-			error = vnode_setattr_fallback(vp, &va, &context);
+	if (vp != NULL) {
+	    error = EEXIST;
+	    goto skipit;
+	}
+
+	/* authorize */
+	if (error == 0)
+		error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx);
+	/* get default ownership, etc. */
+	if (error == 0)
+		error = vnode_authattr_new(dvp, &va, 0, ctx);
+	if (error == 0)
+		error = VNOP_SYMLINK(dvp, &vp, &nd.ni_cnd, &va, path, ctx);
+
+	/* do fallback attribute handling */
+	if (error == 0)
+		error = vnode_setattr_fallback(vp, &va, ctx);
 		
-		if (error == 0) {
-			int	update_flags = 0;
+	if (error == 0) {
+		int	update_flags = 0;
 
-			if (vp == NULL) {
-				nd.ni_cnd.cn_nameiop = LOOKUP;
-				nd.ni_cnd.cn_flags = 0;
-				error = namei(&nd);
-				vp = nd.ni_vp;
+		if (vp == NULL) {
+			nd.ni_cnd.cn_nameiop = LOOKUP;
+			nd.ni_cnd.cn_flags = 0;
+			error = namei(&nd);
+			vp = nd.ni_vp;
 
-				if (vp == NULL)
-				        goto skipit;
-			}
+			if (vp == NULL)
+				goto skipit;
+		}
 			
 #if 0  /* XXX - kauth_todo - is KAUTH_FILEOP_SYMLINK needed? */
-			/* call out to allow 3rd party notification of rename. 
-			 * Ignore result of kauth_authorize_fileop call.
-			 */
-			if (kauth_authorize_fileop_has_listeners() &&
-				namei(&nd) == 0) {
-				char *new_link_path = NULL;
-				int		len;
+		/* call out to allow 3rd party notification of rename. 
+		 * Ignore result of kauth_authorize_fileop call.
+		 */
+		if (kauth_authorize_fileop_has_listeners() &&
+		    namei(&nd) == 0) {
+			char *new_link_path = NULL;
+			int		len;
 				
-				/* build the path to the new link file */
-				new_link_path = get_pathbuff();
-				len = MAXPATHLEN;
-				vn_getpath(dvp, new_link_path, &len);
+			/* build the path to the new link file */
+			new_link_path = get_pathbuff();
+			len = MAXPATHLEN;
+			vn_getpath(dvp, new_link_path, &len);
+			if ((len + 1 + nd.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
 				new_link_path[len - 1] = '/';
-				strcpy(&new_link_path[len], nd.ni_cnd.cn_nameptr);
-
-				kauth_authorize_fileop(vfs_context_ucred(&context), KAUTH_FILEOP_SYMLINK, 
-								   (uintptr_t)path, (uintptr_t)new_link_path);
-				if (new_link_path != NULL)
-					release_pathbuff(new_link_path);
+				strlcpy(&new_link_path[len], nd.ni_cnd.cn_nameptr, MAXPATHLEN-len);
 			}
+				
+			kauth_authorize_fileop(vfs_context_ucred(ctx), KAUTH_FILEOP_SYMLINK, 
+					   (uintptr_t)path, (uintptr_t)new_link_path);
+			if (new_link_path != NULL)
+				release_pathbuff(new_link_path);
+		}
 #endif 
-			// Make sure the name & parent pointers are hooked up
-			if (vp->v_name == NULL)
-			        update_flags |= VNODE_UPDATE_NAME;
-			if (vp->v_parent == NULLVP)
-			        update_flags |= VNODE_UPDATE_PARENT;
-
-			if (update_flags)
-			        vnode_update_identity(vp, dvp, nd.ni_cnd.cn_nameptr, nd.ni_cnd.cn_namelen, nd.ni_cnd.cn_hash, update_flags);
+		// Make sure the name & parent pointers are hooked up
+		if (vp->v_name == NULL)
+			update_flags |= VNODE_UPDATE_NAME;
+		if (vp->v_parent == NULLVP)
+			update_flags |= VNODE_UPDATE_PARENT;
+		
+		if (update_flags)
+			vnode_update_identity(vp, dvp, nd.ni_cnd.cn_nameptr, nd.ni_cnd.cn_namelen, nd.ni_cnd.cn_hash, update_flags);
 
-			add_fsevent(FSE_CREATE_FILE, &context,
-				    FSE_ARG_VNODE, vp,
-				    FSE_ARG_DONE);
-		}
-	} else
-		error = EEXIST;
+#if CONFIG_FSE
+		add_fsevent(FSE_CREATE_FILE, ctx,
+			    FSE_ARG_VNODE, vp,
+			    FSE_ARG_DONE);
+#endif
+	}
 
 skipit:
 	/*
@@ -2114,18 +2876,15 @@ out:
 /* ARGSUSED */
 #warning XXX authorization not implmented for whiteouts
 int
-undelete(struct proc *p, register struct undelete_args *uap, __unused register_t *retval)
+undelete(__unused proc_t p, struct undelete_args *uap, __unused register_t *retval)
 {
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	vnode_t	vp, dvp;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	NDINIT(&nd, DELETE, LOCKPARENT|DOWHITEOUT|AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -2133,7 +2892,7 @@ undelete(struct proc *p, register struct undelete_args *uap, __unused register_t
 	vp = nd.ni_vp;
 
 	if (vp == NULLVP && (nd.ni_cnd.cn_flags & ISWHITEOUT)) {
-		error = VNOP_WHITEOUT(dvp, &nd.ni_cnd, DELETE, &context);
+		error = VNOP_WHITEOUT(dvp, &nd.ni_cnd, DELETE, ctx);
 	} else
 	        error = EEXIST;
 
@@ -2154,74 +2913,133 @@ undelete(struct proc *p, register struct undelete_args *uap, __unused register_t
  * Delete a name from the filesystem.
  */
 /* ARGSUSED */
-static int
-_unlink(struct proc *p, struct unlink_args *uap, __unused register_t *retval, int nodelbusy)
+int
+unlink1(vfs_context_t ctx, struct nameidata *ndp, int nodelbusy)
 {
 	vnode_t	vp, dvp;
 	int error;
-	struct nameidata nd;
-	struct vfs_context context;
 	struct componentname *cnp;
+	char  *path = NULL;
+	int  len;
+	fse_info  finfo;
 	int flags = 0;
+	int need_event = 0;
+	int has_listeners = 0;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	ndp->ni_cnd.cn_flags |= LOCKPARENT;
+	cnp = &ndp->ni_cnd;
 
-	NDINIT(&nd, DELETE, LOCKPARENT | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
-	cnp = &nd.ni_cnd;
+	error = namei(ndp);
+	if (error)
+		return (error);
+	dvp = ndp->ni_dvp;
+	vp = ndp->ni_vp;
 
 	/* With Carbon delete semantics, busy files cannot be deleted */
-	if (nodelbusy)
+	if (nodelbusy) {
 		flags |= VNODE_REMOVE_NODELETEBUSY;
+	}
 
-	error = namei(&nd);
-	if (error)
-		return (error);
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
-
-	if (vp->v_type == VDIR) {
+	/*
+	 * Normally, unlinking of directories is not supported. 
+	 * However, some file systems may have limited support.
+	 */
+	if ((vp->v_type == VDIR) &&
+	    !(vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSDIRLINKS)) {
 		error = EPERM;	/* POSIX */
-	} else {
-		/*
-		 * The root of a mounted filesystem cannot be deleted.
-		 */
-		if (vp->v_flag & VROOT) {
-			error = EBUSY;
-		}
 	}
+
+	/*
+	 * The root of a mounted filesystem cannot be deleted.
+	 */
+	if (vp->v_flag & VROOT) {
+		error = EBUSY;
+	}
+	if (error)
+		goto out;
+
+
 	/* authorize the delete operation */
+#if CONFIG_MACF
 	if (!error)
-		error = vnode_authorize(vp, nd.ni_dvp, KAUTH_VNODE_DELETE, &context);
+		error = mac_vnode_check_unlink(ctx,
+		    dvp, vp, cnp);
+#endif /* MAC */
+	if (!error)
+		error = vnode_authorize(vp, ndp->ni_dvp, KAUTH_VNODE_DELETE, ctx);
+	if (error)
+		goto out;
 	
+#if CONFIG_FSE
+	need_event = need_fsevent(FSE_DELETE, dvp);
+	if (need_event) {
+		if ((vp->v_flag & VISHARDLINK) == 0) {
+			get_fse_info(vp, &finfo, ctx);
+		}
+	}
+#endif
+	has_listeners = kauth_authorize_fileop_has_listeners();
+	if (need_event || has_listeners) {
+		GET_PATH(path);
+		if (path == NULL) {
+			error = ENOMEM;
+			goto out;
+		}
+		len = MAXPATHLEN;
+		vn_getpath(vp, path, &len);
+	}
+
+#if NAMEDRSRCFORK
+	if (ndp->ni_cnd.cn_flags & CN_WANTSRSRCFORK)
+		error = vnode_removenamedstream(dvp, vp, XATTR_RESOURCEFORK_NAME, 0, ctx);
+	else
+#endif
+		error = VNOP_REMOVE(dvp, vp, &ndp->ni_cnd, flags, ctx);
+
+	/*
+	 * Call out to allow 3rd party notification of delete. 
+	 * Ignore result of kauth_authorize_fileop call.
+	 */
 	if (!error) {
-		char     *path = NULL;
-		int       len;
-		fse_info  finfo;
+		if (has_listeners) {
+			kauth_authorize_fileop(vfs_context_ucred(ctx), 
+				KAUTH_FILEOP_DELETE, 
+				(uintptr_t)vp,
+				(uintptr_t)path);
+		}
 
-		if (need_fsevent(FSE_DELETE, dvp)) {
-		        path = get_pathbuff();
-			len = MAXPATHLEN;
-			vn_getpath(vp, path, &len);
-			get_fse_info(vp, &finfo, &context);
+		if (vp->v_flag & VISHARDLINK) {
+		    //
+		    // if a hardlink gets deleted we want to blow away the
+		    // v_parent link because the path that got us to this
+		    // instance of the link is no longer valid.  this will
+		    // force the next call to get the path to ask the file
+		    // system instead of just following the v_parent link.
+		    //
+		    vnode_update_identity(vp, NULL, NULL, 0, 0, VNODE_UPDATE_PARENT);
 		}
-	    	error = VNOP_REMOVE(dvp, vp, &nd.ni_cnd, flags, &context);
 
-	    	if ( !error && path != NULL) {
-			add_fsevent(FSE_DELETE, &context,
-				    FSE_ARG_STRING, len, path,
-				    FSE_ARG_FINFO, &finfo,
-				    FSE_ARG_DONE);
-	    	}
-		if (path != NULL)
-		        release_pathbuff(path);
+#if CONFIG_FSE
+		if (need_event) {
+			if (vp->v_flag & VISHARDLINK) {
+				get_fse_info(vp, &finfo, ctx);
+			}
+			add_fsevent(FSE_DELETE, ctx,
+						FSE_ARG_STRING, len, path,
+						FSE_ARG_FINFO, &finfo,
+						FSE_ARG_DONE);
+		}
+#endif
 	}
+	if (path != NULL)
+		RELEASE_PATH(path);
+
 	/*
 	 * nameidone has to happen before we vnode_put(dvp)
 	 * since it may need to release the fs_nodelock on the dvp
 	 */
-	nameidone(&nd);
+out:
+	nameidone(ndp);
 	vnode_put(dvp);
 	vnode_put(vp);
 	return (error);
@@ -2231,38 +3049,37 @@ _unlink(struct proc *p, struct unlink_args *uap, __unused register_t *retval, in
  * Delete a name from the filesystem using POSIX semantics.
  */
 int
-unlink(p, uap, retval)
-	struct proc *p;
-	struct unlink_args *uap;
-	register_t *retval;
+unlink(__unused proc_t p, struct unlink_args *uap, __unused register_t *retval)
 {
-	return _unlink(p, uap, retval, 0);
+	struct nameidata nd;
+	vfs_context_t ctx = vfs_context_current();
+
+	NDINIT(&nd, DELETE, AUDITVNPATH1, UIO_USERSPACE, uap->path, ctx);
+	return unlink1(ctx, &nd, 0);
 }
 
 /*
  * Delete a name from the filesystem using Carbon semantics.
  */
 int
-delete(p, uap, retval)
-	struct proc *p;
-	struct delete_args *uap;
-	register_t *retval;
+delete(__unused proc_t p, struct delete_args *uap, __unused register_t *retval)
 {
-	return _unlink(p, (struct unlink_args *)uap, retval, 1);
+	struct nameidata nd;
+	vfs_context_t ctx = vfs_context_current();
+
+	NDINIT(&nd, DELETE, AUDITVNPATH1, UIO_USERSPACE, uap->path, ctx);
+	return unlink1(ctx, &nd, 1);
 }
 
 /*
  * Reposition read/write file offset.
  */
 int
-lseek(p, uap, retval)
-	struct proc *p;
-	register struct lseek_args *uap;
-	off_t *retval;
+lseek(proc_t p, struct lseek_args *uap, off_t *retval)
 {
 	struct fileproc *fp;
-	struct vnode *vp;
-	struct vfs_context context;
+	vnode_t vp;
+	struct vfs_context *ctx;
 	off_t offset = uap->offset, file_size;
 	int error;
 
@@ -2275,6 +3092,21 @@ lseek(p, uap, retval)
 		file_drop(uap->fd);
 		return(ESPIPE);
 	}
+
+
+	ctx = vfs_context_current();
+#if CONFIG_MACF
+	if (uap->whence == L_INCR && uap->offset == 0)
+		error = mac_file_check_get_offset(vfs_context_ucred(ctx),
+		    fp->f_fglob);
+	else
+		error = mac_file_check_change_offset(vfs_context_ucred(ctx),
+		    fp->f_fglob);
+	if (error) {
+		file_drop(uap->fd);
+		return (error);
+	}
+#endif
 	if ( (error = vnode_getwithref(vp)) ) {
 		file_drop(uap->fd);
 		return(error);
@@ -2285,9 +3117,7 @@ lseek(p, uap, retval)
 		offset += fp->f_fglob->fg_offset;
 		break;
 	case L_XTND:
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
-		if ((error = vnode_size(vp, &file_size, &context)) != 0)
+		if ((error = vnode_size(vp, &file_size, ctx)) != 0)
 			break;
 		offset += file_size;
 		break;
@@ -2324,6 +3154,9 @@ lseek(p, uap, retval)
 
 /*
  * Check access permissions.
+ *
+ * Returns:	0			Success
+ *		vnode_authorize:???
  */
 static int
 access1(vnode_t vp, vnode_t dvp, int uflags, vfs_context_t ctx)
@@ -2360,6 +3193,12 @@ access1(vnode_t vp, vnode_t dvp, int uflags, vfs_context_t ctx)
 		action = uflags >> 8;
 	}
 	
+#if CONFIG_MACF
+	error = mac_vnode_check_access(ctx, vp, uflags);
+	if (error)
+		return (error);
+#endif /* MAC */
+
  	/* action == 0 means only check for existence */
  	if (action != 0) {
  		error = vnode_authorize(vp, dvp, action | KAUTH_VNODE_ACCESS, ctx);
@@ -2372,95 +3211,189 @@ access1(vnode_t vp, vnode_t dvp, int uflags, vfs_context_t ctx)
 
 
 
-/* XXX need to support the check-as uid argument */
+/*
+ * access_extended
+ *
+ * Description:	uap->entries			Pointer to argument descriptor
+ *		uap->size			Size of the area pointed to by
+ *						the descriptor
+ *		uap->results			Pointer to the results array
+ *
+ * Returns:	0			Success
+ *		ENOMEM			Insufficient memory
+ *		EINVAL			Invalid arguments
+ *		namei:EFAULT		Bad address
+ *		namei:ENAMETOOLONG	Filename too long
+ *		namei:ENOENT		No such file or directory
+ *		namei:ELOOP		Too many levels of symbolic links
+ *		namei:EBADF		Bad file descriptor
+ *		namei:ENOTDIR		Not a directory
+ *		namei:???
+ *		access1:
+ *
+ * Implicit returns:
+ *		uap->results		Array contents modified
+ *
+ * Notes:	The uap->entries are structured as an arbitrary length array
+ *		of accessx descriptors, followed by one or more NULL terniated
+ *		strings
+ *
+ *			struct accessx_descriptor[0]
+ *			...
+ *			struct accessx_descriptor[n]
+ *			char name_data[0];
+ *
+ *		We determine the entry count by walking the buffer containing
+ *		the uap->entries argument descriptor.  For each descrptor we
+ *		see, the valid values for the offset ad_name_offset will be
+ *		in the byte range:
+ *
+ *			[ uap->entries + sizeof(struct accessx_descriptor) ]
+ *						to
+ *				[ uap->entries + uap->size - 2 ]
+ *
+ *		since we must have at least one string, and the string must
+ *		be at least one character plus the NUL terminator in length.
+ *		
+ * XXX:		Need to support the check-as uid argument
+ */
 int
-access_extended(__unused struct proc *p, struct access_extended_args *uap, __unused register_t *retval)
+access_extended(__unused proc_t p, struct access_extended_args *uap, __unused register_t *retval)
 {
-	struct accessx_descriptor *input;
-	errno_t *result;
-	int error, limit, nent, i, j, wantdelete;
+	struct accessx_descriptor *input = NULL;
+	errno_t *result = NULL;
+	errno_t error = 0;
+	int wantdelete = 0;
+	unsigned int desc_max, desc_actual, i, j;
 	struct vfs_context context;
 	struct nameidata nd;
  	int niopts;
-	vnode_t vp, dvp;
+	vnode_t vp = NULL;
+	vnode_t dvp = NULL;
+#define ACCESSX_MAX_DESCR_ON_STACK 10
+	struct accessx_descriptor stack_input[ACCESSX_MAX_DESCR_ON_STACK];
 
-	input = NULL;
-	result = NULL;
-	error = 0;
-	vp = NULL;
-	dvp = NULL;
 	context.vc_ucred = NULL;
 
-	/* check input size and fetch descriptor array into allocated storage */
+	/*
+	 * Validate parameters; if valid, copy the descriptor array and string
+	 * arguments into local memory.  Before proceeding, the following
+	 * conditions must have been met:
+	 *
+	 * o	The total size is not permitted to exceed ACCESSX_MAX_TABLESIZE
+	 * o	There must be sufficient room in the request for at least one
+	 *	descriptor and a one yte NUL terminated string.
+	 * o	The allocation of local storage must not fail.
+	 */
 	if (uap->size > ACCESSX_MAX_TABLESIZE)
 		return(ENOMEM);
-	if (uap->size < sizeof(struct accessx_descriptor))
+	if (uap->size < (sizeof(struct accessx_descriptor) + 2))
 		return(EINVAL);
+	if (uap->size <= sizeof (stack_input)) {
+		input = stack_input;
+	} else {
 	MALLOC(input, struct accessx_descriptor *, uap->size, M_TEMP, M_WAITOK);
 	if (input == NULL) {
 		error = ENOMEM;
 		goto out;
 	}
+	}
 	error = copyin(uap->entries, input, uap->size);
 	if (error)
 		goto out;
 
 	/*
-	 * Access is defined as checking against the process'
- 	 * real identity, even if operations are checking the
- 	 * effective identity.  So we need to tweak the credential
- 	 * in the context.
+	 * Force NUL termination of the copyin buffer to avoid nami() running
+	 * off the end.  If the caller passes us bogus data, they may get a
+	 * bogus result.
+	 */
+	((char *)input)[uap->size - 1] = 0;
+
+	/*
+	 * Access is defined as checking against the process' real identity,
+ 	 * even if operations are checking the effective identity.  This
+	 * requires that we use a local vfs context.
  	 */
 	context.vc_ucred = kauth_cred_copy_real(kauth_cred_get());
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 
 	/*
-	 * Find out how many entries we have, so we can allocate the result array.
+	 * Find out how many entries we have, so we can allocate the result
+	 * array by walking the list and adjusting the count downward by the
+	 * earliest string offset we see.
 	 */
-	limit = uap->size / sizeof(struct accessx_descriptor);
-	nent = limit;
-	wantdelete = 0;
-	for (i = 0; i < nent; i++) {
+	desc_max = (uap->size - 2) / sizeof(struct accessx_descriptor);
+	desc_actual = desc_max;
+	for (i = 0; i < desc_actual; i++) {
 		/*
-		 * Take the offset to the name string for this entry and convert to an
-		 * input array index, which would be one off the end of the array if this
-		 * was the lowest-addressed name string.
+		 * Take the offset to the name string for this entry and
+		 * convert to an input array index, which would be one off
+		 * the end of the array if this entry was the lowest-addressed
+		 * name string.
 		 */
 		j = input[i].ad_name_offset / sizeof(struct accessx_descriptor);
-		/* bad input */
-		if (j > limit) {
+
+		/*
+		 * An offset greater than the max allowable offset is an error.
+		 * It is also an error for any valid entry to point
+		 * to a location prior to the end of the current entry, if
+		 * it's not a reference to the string of the previous entry.
+		 */
+		if (j > desc_max || (j != 0 && j <= i)) {
 			error = EINVAL;
 			goto out;
 		}
-		/* implicit reference to previous name, not a real offset */
+
+		/*
+		 * An offset of 0 means use the previous descriptor's offset;
+		 * this is used to chain multiple requests for the same file
+		 * to avoid multiple lookups.
+		 */
 		if (j == 0) {
-			/* first entry must have a name string */
+			/* This is not valid for the first entry */
 			if (i == 0) {
 				error = EINVAL;
 				goto out;
 			}
 			continue;
 		}
-		if (j < nent)
-			nent = j;
+
+		/*
+		 * If the offset of the string for this descriptor is before
+		 * what we believe is the current actual last descriptor,
+		 * then we need to adjust our estimate downward; this permits
+		 * the string table following the last descriptor to be out
+		 * of order relative to the descriptor list.
+		 */
+		if (j < desc_actual)
+			desc_actual = j;
 	}
-	if (nent > ACCESSX_MAX_DESCRIPTORS) {
+
+	/*
+	 * We limit the actual number of descriptors we are willing to process
+	 * to a hard maximum of ACCESSX_MAX_DESCRIPTORS.  If the number being
+	 * requested does not exceed this limit,
+	 */
+	if (desc_actual > ACCESSX_MAX_DESCRIPTORS) {
 		error = ENOMEM;
 		goto out;
 	}
-	MALLOC(result, errno_t *, nent * sizeof(errno_t), M_TEMP, M_WAITOK);
+	MALLOC(result, errno_t *, desc_actual * sizeof(errno_t), M_TEMP, M_WAITOK);
 	if (result == NULL) {
 		error = ENOMEM;
 		goto out;
 	}
 
 	/*
-	 * Do the work.
+	 * Do the work by iterating over the descriptor entries we know to
+	 * at least appear to contain valid data.
 	 */
 	error = 0;
-	for (i = 0; i < nent; i++) {
+	for (i = 0; i < desc_actual; i++) {
 		/*
-		 * Looking up a new name?
+		 * If the ad_name_offset is 0, then we use the previous
+		 * results to make the check; otherwise, we are looking up
+		 * a new file name.
 		 */
 		if (input[i].ad_name_offset != 0) {
 			/* discard old vnodes */
@@ -2473,19 +3406,26 @@ access_extended(__unused struct proc *p, struct access_extended_args *uap, __unu
 				dvp = NULL;
 			}
 			
-			/* scan forwards to see if we need the parent this time */
+			/*
+			 * Scan forward in the descriptor list to see if we
+			 * need the parent vnode.  We will need it if we are
+			 * deleting, since we must have rights  to remove
+			 * entries in the parent directory, as well as the
+			 * rights to delete the object itself.
+			 */
 			wantdelete = input[i].ad_flags & _DELETE_OK;
-			for (j = i + 1; (j < nent) && (input[j].ad_name_offset == 0); j++)
+			for (j = i + 1; (j < desc_actual) && (input[j].ad_name_offset == 0); j++)
 				if (input[j].ad_flags & _DELETE_OK)
 					wantdelete = 1;
 			
 			niopts = FOLLOW | AUDITVNPATH1;
+
 			/* need parent for vnode_authorize for deletion test */
 			if (wantdelete)
 				niopts |= WANTPARENT;
 
 			/* do the lookup */
-			NDINIT(&nd, LOOKUP, niopts, UIO_SYSSPACE, CAST_USER_ADDR_T((const char *)input + input[i].ad_name_offset), &context);
+			NDINIT(&nd, LOOKUP, niopts, UIO_SYSSPACE, CAST_USER_ADDR_T(((const char *)input) + input[i].ad_name_offset), &context);
 			error = namei(&nd);
 			if (!error) {
 				vp = nd.ni_vp;
@@ -2517,10 +3457,10 @@ access_extended(__unused struct proc *p, struct access_extended_args *uap, __unu
 	}
 
 	/* copy out results */
-	error = copyout(result, uap->results, nent * sizeof(errno_t));
+	error = copyout(result, uap->results, desc_actual * sizeof(errno_t));
 	
 out:
-	if (input)
+	if (input && input != stack_input)
 		FREE(input, M_TEMP);
 	if (result)
 		FREE(result, M_TEMP);
@@ -2533,8 +3473,20 @@ out:
 	return(error);
 }
 
+
+/*
+ * Returns:	0			Success
+ *		namei:EFAULT		Bad address
+ *		namei:ENAMETOOLONG	Filename too long
+ *		namei:ENOENT		No such file or directory
+ *		namei:ELOOP		Too many levels of symbolic links
+ *		namei:EBADF		Bad file descriptor
+ *		namei:ENOTDIR		Not a directory
+ *		namei:???
+ *		access1:
+ */
 int
-access(__unused struct proc *p, register struct access_args *uap, __unused register_t *retval)
+access(__unused proc_t p, struct access_args *uap, __unused register_t *retval)
 {
 	int error;
 	struct nameidata nd;
@@ -2548,13 +3500,19 @@ access(__unused struct proc *p, register struct access_args *uap, __unused regis
  	 * in the context.
  	 */
 	context.vc_ucred = kauth_cred_copy_real(kauth_cred_get());
-	context.vc_proc = current_proc();
+	context.vc_thread = current_thread();
 
 	niopts = FOLLOW | AUDITVNPATH1;
  	/* need parent for vnode_authorize for deletion test */
  	if (uap->flags & _DELETE_OK)
  		niopts |= WANTPARENT;
  	NDINIT(&nd, LOOKUP, niopts, UIO_USERSPACE, uap->path, &context);
+
+#if NAMEDRSRCFORK
+	/* access(F_OK) calls are allowed for resource forks. */
+	if (uap->flags == F_OK)
+		nd.ni_cnd.cn_flags |= CN_ALLOWRSRCFORK;
+#endif
  	error = namei(&nd);
  	if (error)
  		goto out;
@@ -2572,38 +3530,91 @@ out:
 }
 
 
+/*
+ * Returns:	0			Success
+ *		EFAULT
+ *	copyout:EFAULT
+ *	namei:???
+ *	vn_stat:???
+ */
 static int
-stat2(vfs_context_t ctx, struct nameidata *ndp, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size)
+stat2(vfs_context_t ctx, struct nameidata *ndp, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size, int isstat64)
 {
 	struct stat sb;
+	struct stat64 sb64;
 	struct user_stat user_sb;
+	struct user_stat64 user_sb64;
 	caddr_t sbp;
 	int error, my_size;
 	kauth_filesec_t fsec;
 	size_t xsecurity_bufsize;
+	void * statptr;
 
+#if NAMEDRSRCFORK
+	/* stat calls are allowed for resource forks. */
+	ndp->ni_cnd.cn_flags |= CN_ALLOWRSRCFORK;
+#endif
 	error = namei(ndp);
 	if (error)
 		return (error);
 	fsec = KAUTH_FILESEC_NONE;
-	error = vn_stat(ndp->ni_vp, &sb, (xsecurity != USER_ADDR_NULL ? &fsec : NULL), ctx);
+	if (isstat64 != 0) 
+		statptr	 = (void *)&sb64;
+	else
+		statptr	 = (void *)&sb;
+	error = vn_stat(ndp->ni_vp, statptr, (xsecurity != USER_ADDR_NULL ? &fsec : NULL), isstat64, ctx);
+
+#if NAMEDRSRCFORK
+	/* Clean up resource fork shadow file if needed. */
+	if ((ndp->ni_vp->v_flag & VISNAMEDSTREAM) && 
+	    (ndp->ni_vp->v_parent != NULLVP) &&
+	    !(ndp->ni_vp->v_parent->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)) {
+		(void) vnode_relenamedstream(ndp->ni_vp->v_parent, ndp->ni_vp, ctx);
+	}
+#endif
 	vnode_put(ndp->ni_vp);
 	nameidone(ndp);
 
 	if (error)
 		return (error);
 	/* Zap spare fields */
-	sb.st_lspare = 0;
-	sb.st_qspare[0] = 0LL;
-	sb.st_qspare[1] = 0LL;
-	if (IS_64BIT_PROCESS(vfs_context_proc(ctx))) {
-		munge_stat(&sb, &user_sb); 
-		my_size = sizeof(user_sb);
-		sbp = (caddr_t)&user_sb;
-	}
-	else {
-		my_size = sizeof(sb);
-		sbp = (caddr_t)&sb;
+	if (isstat64 != 0) {
+		sb64.st_lspare = 0;
+		sb64.st_qspare[0] = 0LL;
+		sb64.st_qspare[1] = 0LL;
+		if (IS_64BIT_PROCESS(vfs_context_proc(ctx))) {
+			munge_stat64(&sb64, &user_sb64); 
+			my_size = sizeof(user_sb64);
+			sbp = (caddr_t)&user_sb64;
+		} else {
+			my_size = sizeof(sb64);
+			sbp = (caddr_t)&sb64;
+		}
+		/*
+		 * Check if we raced (post lookup) against the last unlink of a file.
+		 */
+		if ((sb64.st_nlink == 0) && S_ISREG(sb64.st_mode)) {
+			sb64.st_nlink = 1;
+		}
+	} else {
+		sb.st_lspare = 0;
+		sb.st_qspare[0] = 0LL;
+		sb.st_qspare[1] = 0LL;
+		if (IS_64BIT_PROCESS(vfs_context_proc(ctx))) {
+			munge_stat(&sb, &user_sb); 
+			my_size = sizeof(user_sb);
+			sbp = (caddr_t)&user_sb;
+		} else {
+			my_size = sizeof(sb);
+			sbp = (caddr_t)&sb;
+		}
+
+		/*
+		 * Check if we raced (post lookup) against the last unlink of a file.
+		 */
+		if ((sb.st_nlink == 0) && S_ISREG(sb.st_mode)) {
+			sb.st_nlink = 1;
+		}
 	}
 	if ((error = copyout(sbp, ub, my_size)) != 0)
 		goto out;
@@ -2640,87 +3651,116 @@ out:
 
 /*
  * Get file status; this version follows links.
+ *
+ * Returns:	0			Success
+ *	stat2:???			[see stat2() in this file]
  */
 static int
-stat1(struct proc *p, user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size)
+stat1(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size, int isstat64)
 {
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	NDINIT(&nd, LOOKUP, NOTRIGGER | FOLLOW | AUDITVNPATH1, 
+	    UIO_USERSPACE, path, ctx);
+	return(stat2(ctx, &nd, ub, xsecurity, xsecurity_size, isstat64));
+}
 
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-	    UIO_USERSPACE, path, &context);
-	return(stat2(&context, &nd, ub, xsecurity, xsecurity_size));
+int
+stat_extended(__unused proc_t p, struct stat_extended_args *uap, __unused register_t *retval)
+{
+	return (stat1(uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size, 0));
 }
 
+/*
+ * Returns:	0			Success
+ *	stat1:???			[see stat1() in this file]
+ */
 int
-stat_extended(struct proc *p, struct stat_extended_args *uap, __unused register_t *retval)
+stat(__unused proc_t p, struct stat_args *uap, __unused register_t *retval)
 {
-	return (stat1(p, uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size));
+	return(stat1(uap->path, uap->ub, 0, 0, 0));
 }
 
 int
-stat(struct proc *p, struct stat_args *uap, __unused register_t *retval)
+stat64(__unused proc_t p, struct stat64_args *uap, __unused register_t *retval)
 {
-	return(stat1(p, uap->path, uap->ub, 0, 0));
+	return(stat1(uap->path, uap->ub, 0, 0, 1));
 }
 
+int
+stat64_extended(__unused proc_t p, struct stat64_extended_args *uap, __unused register_t *retval)
+{
+	return (stat1(uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size, 1));
+}
 /*
  * Get file status; this version does not follow links.
  */
 static int
-lstat1(struct proc *p, user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size)
+lstat1(user_addr_t path, user_addr_t ub, user_addr_t xsecurity, user_addr_t xsecurity_size, int isstat64)
 {
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	NDINIT(&nd, LOOKUP, NOTRIGGER | NOFOLLOW | AUDITVNPATH1, 
+	    UIO_USERSPACE, path, ctx);
 
-	NDINIT(&nd, LOOKUP, NOFOLLOW | AUDITVNPATH1, 
-	    UIO_USERSPACE, path, &context);
+	return(stat2(ctx, &nd, ub, xsecurity, xsecurity_size, isstat64));
+}
 
-	return(stat2(&context, &nd, ub, xsecurity, xsecurity_size));
+int
+lstat_extended(__unused proc_t p, struct lstat_extended_args *uap, __unused register_t *retval)
+{
+	return (lstat1(uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size, 0));
 }
 
 int
-lstat_extended(struct proc *p, struct lstat_extended_args *uap, __unused register_t *retval)
+lstat(__unused proc_t p, struct lstat_args *uap, __unused register_t *retval)
 {
-	return (lstat1(p, uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size));
+	return(lstat1(uap->path, uap->ub, 0, 0, 0));
+}
+int
+lstat64(__unused proc_t p, struct lstat64_args *uap, __unused register_t *retval)
+{
+	return(lstat1(uap->path, uap->ub, 0, 0, 1));
 }
 
 int
-lstat(struct proc *p, struct lstat_args *uap, __unused register_t *retval)
+lstat64_extended(__unused proc_t p, struct lstat64_extended_args *uap, __unused register_t *retval)
 {
-	return(lstat1(p, uap->path, uap->ub, 0, 0));
+	return (lstat1(uap->path, uap->ub, uap->xsecurity, uap->xsecurity_size, 1));
 }
 
 /*
  * Get configurable pathname variables.
+ *
+ * Returns:	0			Success
+ *	namei:???
+ *	vn_pathconf:???
+ *
+ * Notes:	Global implementation  constants are intended to be
+ *		implemented in this function directly; all other constants
+ *		are per-FS implementation, and therefore must be handled in
+ *		each respective FS, instead.
+ *
+ * XXX We implement some things globally right now that should actually be
+ * XXX per-FS; we will need to deal with this at some point.
  */
 /* ARGSUSED */
 int
-pathconf(p, uap, retval)
-	struct proc *p;
-	register struct pathconf_args *uap;
-	register_t *retval;
+pathconf(__unused proc_t p, struct pathconf_args *uap, register_t *retval)
 {
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
 
-	error = vn_pathconf(nd.ni_vp, uap->name, retval, &context);
+	error = vn_pathconf(nd.ni_vp, uap->name, retval, ctx);
 
 	vnode_put(nd.ni_vp);
 	nameidone(&nd);
@@ -2732,24 +3772,18 @@ pathconf(p, uap, retval)
  */
 /* ARGSUSED */
 int
-readlink(p, uap, retval)
-	struct proc *p;
-	register struct readlink_args *uap;
-	register_t *retval;
+readlink(proc_t p, struct readlink_args *uap, register_t *retval)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	uio_t auio;
 	int spacetype = proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	NDINIT(&nd, LOOKUP, NOFOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -2763,9 +3797,14 @@ readlink(p, uap, retval)
 	if (vp->v_type != VLNK)
 		error = EINVAL;
 	else {
-		error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, &context);
+#if CONFIG_MACF
+		error = mac_vnode_check_readlink(ctx,
+		    vp);
+#endif
+		if (error == 0)
+			error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_DATA, ctx);
 		if (error == 0)
-			error = VNOP_READLINK(vp, auio, &context);
+			error = VNOP_READLINK(vp, auio, ctx);
 	}
 	vnode_put(vp);
 	// LP64todo - fix this
@@ -2786,6 +3825,12 @@ chflags1(vnode_t vp, int flags, vfs_context_t ctx)
 	VATTR_INIT(&va);
 	VATTR_SET(&va, va_flags, flags);
 
+#if CONFIG_MACF
+	error = mac_vnode_check_setflags(ctx, vp, flags);
+	if (error)
+		goto out;
+#endif
+
 	/* request authorisation, disregard immutability */
  	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0)
 		goto out;
@@ -2798,6 +3843,9 @@ chflags1(vnode_t vp, int flags, vfs_context_t ctx)
 		goto out;
 	error = vnode_setattr(vp, &va, ctx);
 
+	if ((error == 0) && !VATTR_IS_SUPPORTED(&va, va_flags)) {
+		error = ENOTSUP;
+	}
 out:
 	vnode_put(vp);
 	return(error);
@@ -2808,26 +3856,23 @@ out:
  */
 /* ARGSUSED */
 int
-chflags(struct proc *p, register struct chflags_args *uap, __unused register_t *retval)
+chflags(__unused proc_t p, struct chflags_args *uap, __unused register_t *retval)
 {
-	register struct vnode *vp;
-	struct vfs_context context;
+	vnode_t vp;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	struct nameidata nd;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	AUDIT_ARG(fflags, uap->flags);
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
 	vp = nd.ni_vp;
 	nameidone(&nd);
 
-	error = chflags1(vp, uap->flags, &context);
+	error = chflags1(vp, uap->flags, ctx);
 
 	return(error);
 }
@@ -2837,10 +3882,9 @@ chflags(struct proc *p, register struct chflags_args *uap, __unused register_t *
  */
 /* ARGSUSED */
 int
-fchflags(struct proc *p, register struct fchflags_args *uap, __unused register_t *retval)
+fchflags(__unused proc_t p, struct fchflags_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
-	struct vnode *vp;
+	vnode_t vp;
 	int error;
 
 	AUDIT_ARG(fd, uap->fd);
@@ -2855,10 +3899,7 @@ fchflags(struct proc *p, register struct fchflags_args *uap, __unused register_t
 
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	error = chflags1(vp, uap->flags, &context);
+	error = chflags1(vp, uap->flags, vfs_context_current());
 
 	file_drop(uap->fd);
 	return (error);
@@ -2866,9 +3907,18 @@ fchflags(struct proc *p, register struct fchflags_args *uap, __unused register_t
 
 /*
  * Change security information on a filesystem object.
+ *
+ * Returns:	0			Success
+ *		EPERM			Operation not permitted
+ *		vnode_authattr:???	[anything vnode_authattr can return]
+ *		vnode_authorize:???	[anything vnode_authorize can return]
+ *		vnode_setattr:???	[anything vnode_setattr can return]
+ *
+ * Notes:	If vnode_authattr or vnode_authorize return EACCES, it will be
+ *		translated to EPERM before being returned.
  */
 static int
-chmod2(vfs_context_t ctx, struct vnode *vp, struct vnode_attr *vap)
+chmod2(vfs_context_t ctx, vnode_t vp, struct vnode_attr *vap)
 {
 	kauth_action_t action;
 	int error;
@@ -2876,6 +3926,19 @@ chmod2(vfs_context_t ctx, struct vnode *vp, struct vnode_attr *vap)
 	AUDIT_ARG(mode, (mode_t)vap->va_mode);
 #warning XXX audit new args
 
+#if NAMEDSTREAMS
+	/* chmod calls are not allowed for resource forks. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		return (EPERM);
+	}
+#endif
+
+#if CONFIG_MACF
+	error = mac_vnode_check_setmode(ctx, vp, (mode_t)vap->va_mode);
+	if (error)
+		return (error);
+#endif
+
  	/* make sure that the caller is allowed to set this security information */
 	if (((error = vnode_authattr(vp, vap, &action, ctx)) != 0) ||
 	    ((error = vnode_authorize(vp, NULL, action, ctx)) != 0)) {
@@ -2892,6 +3955,10 @@ chmod2(vfs_context_t ctx, struct vnode *vp, struct vnode_attr *vap)
 
 /*
  * Change mode of a file given path name.
+ *
+ * Returns:	0			Success
+ *		namei:???		[anything namei can return]
+ *		chmod2:???		[anything chmod2 can return]
  */
 static int
 chmod1(vfs_context_t ctx, user_addr_t path, struct vnode_attr *vap)
@@ -2932,9 +3999,8 @@ chmod1(vfs_context_t ctx, user_addr_t path, struct vnode_attr *vap)
  *		in the code they originated.
  */
 int
-chmod_extended(struct proc *p, struct chmod_extended_args *uap, __unused register_t *retval)
+chmod_extended(__unused proc_t p, struct chmod_extended_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
 	int error;
 	struct vnode_attr va;
 	kauth_filesec_t xsecdst;
@@ -2962,43 +4028,37 @@ chmod_extended(struct proc *p, struct chmod_extended_args *uap, __unused registe
 		VATTR_SET(&va, va_acl, &xsecdst->fsec_acl);
 		KAUTH_DEBUG("CHMOD - setting ACL with %d entries", va.va_acl->acl_entrycount);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
-	error = chmod1(&context, uap->path, &va);
+	error = chmod1(vfs_context_current(), uap->path, &va);
 
 	if (xsecdst != NULL)
 		kauth_filesec_free(xsecdst);
 	return(error);
 }
 
+/*
+ * Returns:	0			Success
+ *		chmod1:???		[anything chmod1 can return]
+ */
 int
-chmod(struct proc *p, register struct chmod_args *uap, __unused register_t *retval)
+chmod(__unused proc_t p, struct chmod_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
 	struct vnode_attr va;
 
 	VATTR_INIT(&va);
 	VATTR_SET(&va, va_mode, uap->mode & ALLPERMS);
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	return(chmod1(&context, uap->path, &va));
+	return(chmod1(vfs_context_current(), uap->path, &va));
 }
 
 /*
  * Change mode of a file given a file descriptor.
  */
 static int
-fchmod1(struct proc *p, int fd, struct vnode_attr *vap)
+fchmod1(__unused proc_t p, int fd, struct vnode_attr *vap)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	int error;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
 	AUDIT_ARG(fd, fd);
 
@@ -3010,7 +4070,7 @@ fchmod1(struct proc *p, int fd, struct vnode_attr *vap)
 	}
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
-	error = chmod2(&context, vp, vap);
+	error = chmod2(vfs_context_current(), vp, vap);
 	(void)vnode_put(vp);
 	file_drop(fd);
 
@@ -3018,7 +4078,7 @@ fchmod1(struct proc *p, int fd, struct vnode_attr *vap)
 }
 
 int
-fchmod_extended(struct proc *p, struct fchmod_extended_args *uap, __unused register_t *retval)
+fchmod_extended(proc_t p, struct fchmod_extended_args *uap, __unused register_t *retval)
 {
 	int error;
 	struct vnode_attr va;
@@ -3060,7 +4120,7 @@ fchmod_extended(struct proc *p, struct fchmod_extended_args *uap, __unused regis
 }
 
 int
-fchmod(struct proc *p, register struct fchmod_args *uap, __unused register_t *retval)
+fchmod(proc_t p, struct fchmod_args *uap, __unused register_t *retval)
 {
 	struct vnode_attr va;
 
@@ -3076,9 +4136,9 @@ fchmod(struct proc *p, register struct fchmod_args *uap, __unused register_t *re
  */
 /* ARGSUSED */
 static int
-chown1(vfs_context_t ctx, register struct chown_args *uap, __unused register_t *retval, int follow)
+chown1(vfs_context_t ctx, struct chown_args *uap, __unused register_t *retval, int follow)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	struct vnode_attr va;
 	int error;
 	struct nameidata nd;
@@ -3086,7 +4146,7 @@ chown1(vfs_context_t ctx, register struct chown_args *uap, __unused register_t *
 
 	AUDIT_ARG(owner, uap->uid, uap->gid);
 
-	NDINIT(&nd, LOOKUP, (follow ? FOLLOW : 0) | AUDITVNPATH1, 
+	NDINIT(&nd, LOOKUP, (follow ? FOLLOW : 0) | NOTRIGGER | AUDITVNPATH1, 
 		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
@@ -3095,21 +4155,18 @@ chown1(vfs_context_t ctx, register struct chown_args *uap, __unused register_t *
 
 	nameidone(&nd);
 
-	/*
-	 * XXX A TEMPORARY HACK FOR NOW: Try to track console_user
-	 * by looking for chown() calls on /dev/console from a console process.
-	 */
-	if ((vp) && (vp->v_type == VBLK || vp->v_type == VCHR) && (vp->v_specinfo) &&
-		(major(vp->v_specinfo->si_rdev) == CONSMAJOR) &&
-		(minor(vp->v_specinfo->si_rdev) == 0)) {
-		console_user = uap->uid;
-	};
 	VATTR_INIT(&va);
 	if (uap->uid != VNOVAL)
 		VATTR_SET(&va, va_uid, uap->uid);
 	if (uap->gid != VNOVAL)
 		VATTR_SET(&va, va_gid, uap->gid);
 
+#if CONFIG_MACF
+	error = mac_vnode_check_setowner(ctx, vp, uap->uid, uap->gid);
+	if (error)
+		goto out;
+#endif
+
 	/* preflight and authorize attribute changes */
 	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0)
 		goto out;
@@ -3130,26 +4187,16 @@ out:
 }
 
 int
-chown(struct proc *p, register struct chown_args *uap, register_t *retval)
+chown(__unused proc_t p, struct chown_args *uap, register_t *retval)
 {
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	return chown1(&context, uap, retval, 1);
+	return chown1(vfs_context_current(), uap, retval, 1);
 }
 
 int
-lchown(struct proc *p, register struct lchown_args *uap, register_t *retval)
+lchown(__unused proc_t p, struct lchown_args *uap, register_t *retval)
 {
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	/* Argument list identical, but machine generated; cast for chown1() */
-	return chown1(&context, (struct chown_args *)uap, retval, 0);
+	return chown1(vfs_context_current(), (struct chown_args *)uap, retval, 0);
 }
 
 /*
@@ -3157,11 +4204,11 @@ lchown(struct proc *p, register struct lchown_args *uap, register_t *retval)
  */
 /* ARGSUSED */
 int
-fchown(struct proc *p, register struct fchown_args *uap, __unused register_t *retval)
+fchown(__unused proc_t p, struct fchown_args *uap, __unused register_t *retval)
 {
 	struct vnode_attr va;
-	struct vfs_context context;
-	struct vnode *vp;
+	vfs_context_t ctx = vfs_context_current();
+	vnode_t vp;
 	int error;
 	kauth_action_t action;
 
@@ -3183,18 +4230,29 @@ fchown(struct proc *p, register struct fchown_args *uap, __unused register_t *re
 	if (uap->gid != VNOVAL)
 		VATTR_SET(&va, va_gid, uap->gid);
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+#if NAMEDSTREAMS
+	/* chown calls are not allowed for resource forks. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		error = EPERM;
+		goto out;
+	}
+#endif
+
+#if CONFIG_MACF
+	error = mac_vnode_check_setowner(ctx, vp, uap->uid, uap->gid);
+	if (error)
+		goto out;
+#endif
 
  	/* preflight and authorize attribute changes */
-	if ((error = vnode_authattr(vp, &va, &action, &context)) != 0)
+	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0)
 		goto out;
-	if (action && ((error = vnode_authorize(vp, NULL, action, &context)) != 0)) {
+	if (action && ((error = vnode_authorize(vp, NULL, action, ctx)) != 0)) {
 		if (error == EACCES)
 			error = EPERM;
 		goto out;
 	}
-	error = vnode_setattr(vp, &va, &context);
+	error = vnode_setattr(vp, &va, ctx);
 
 out:
 	(void)vnode_put(vp);
@@ -3203,9 +4261,7 @@ out:
 }
 
 static int
-getutimes(usrtvp, tsp)
-	user_addr_t usrtvp;
-	struct timespec *tsp;
+getutimes(user_addr_t usrtvp, struct timespec *tsp)
 {
 	struct user_timeval tv[2];
 	int error;
@@ -3236,7 +4292,7 @@ getutimes(usrtvp, tsp)
 }
 
 static int
-setutimes(vfs_context_t ctx, struct vnode *vp, const struct timespec *ts,
+setutimes(vfs_context_t ctx, vnode_t vp, const struct timespec *ts,
 	int nullflag)
 {
 	int error;
@@ -3251,11 +4307,31 @@ setutimes(vfs_context_t ctx, struct vnode *vp, const struct timespec *ts,
 	if (nullflag)
 		va.va_vaflags |= VA_UTIMES_NULL;
 
-	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0)
+#if NAMEDSTREAMS
+	/* utimes calls are not allowed for resource forks. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		error = EPERM;
+		goto out;
+	}
+#endif
+
+#if CONFIG_MACF
+	error = mac_vnode_check_setutimes(ctx, vp, ts[0], ts[1]);
+	if (error)
+		goto out;
+#endif
+	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0) {
+		if (!nullflag && error == EACCES)
+			error = EPERM;
 		goto out;
+	}
+
 	/* since we may not need to auth anything, check here */
-	if ((action != 0) && ((error = vnode_authorize(vp, NULL, action, ctx)) != 0))
+	if ((action != 0) && ((error = vnode_authorize(vp, NULL, action, ctx)) != 0)) {
+		if (!nullflag && error == EACCES)
+			error = EPERM;
 		goto out;
+	}
 	error = vnode_setattr(vp, &va, ctx);
 
 out:
@@ -3267,22 +4343,20 @@ out:
  */
 /* ARGSUSED */
 int
-utimes(struct proc *p, register struct utimes_args *uap, __unused register_t *retval)
+utimes(__unused proc_t p, struct utimes_args *uap, __unused register_t *retval)
 {
 	struct timespec ts[2];
 	user_addr_t usrtvp;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
-	/* AUDIT: Needed to change the order of operations to do the 
+	/*
+	 * AUDIT: Needed to change the order of operations to do the 
 	 * name lookup first because auditing wants the path.
 	 */
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -3296,7 +4370,7 @@ utimes(struct proc *p, register struct utimes_args *uap, __unused register_t *re
 	if ((error = getutimes(usrtvp, ts)) != 0)
 		goto out;
 
-	error = setutimes(&context, nd.ni_vp, ts, usrtvp == USER_ADDR_NULL);
+	error = setutimes(ctx, nd.ni_vp, ts, usrtvp == USER_ADDR_NULL);
 
 out:
 	vnode_put(nd.ni_vp);
@@ -3308,16 +4382,12 @@ out:
  */
 /* ARGSUSED */
 int
-futimes(struct proc *p, register struct futimes_args *uap, __unused register_t *retval)
+futimes(__unused proc_t p, struct futimes_args *uap, __unused register_t *retval)
 {
 	struct timespec ts[2];
-	struct vnode *vp;
+	vnode_t vp;
 	user_addr_t usrtvp;
 	int error;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
 	AUDIT_ARG(fd, uap->fd);
 	usrtvp = uap->tptr;
@@ -3330,7 +4400,7 @@ futimes(struct proc *p, register struct futimes_args *uap, __unused register_t *
 		return(error);
 	}
 
-	error =  setutimes(&context, vp, ts, usrtvp == 0);
+	error =  setutimes(vfs_context_current(), vp, ts, usrtvp == 0);
 	vnode_put(vp);
 	file_drop(uap->fd);
 	return(error);
@@ -3341,22 +4411,19 @@ futimes(struct proc *p, register struct futimes_args *uap, __unused register_t *
  */
 /* ARGSUSED */
 int
-truncate(struct proc *p, register struct truncate_args *uap, __unused register_t *retval)
+truncate(__unused proc_t p, struct truncate_args *uap, __unused register_t *retval)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	struct nameidata nd;
 	kauth_action_t action;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	if (uap->length < 0)
 		return(EINVAL);
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	if ((error = namei(&nd)))
 		return (error);
 	vp = nd.ni_vp;
@@ -3365,11 +4432,18 @@ truncate(struct proc *p, register struct truncate_args *uap, __unused register_t
 
 	VATTR_INIT(&va);
 	VATTR_SET(&va, va_data_size, uap->length);
-	if ((error = vnode_authattr(vp, &va, &action, &context)) != 0)
+
+#if CONFIG_MACF
+	error = mac_vnode_check_truncate(ctx, NOCRED, vp);
+	if (error)
+		goto out;
+#endif
+
+	if ((error = vnode_authattr(vp, &va, &action, ctx)) != 0)
 		goto out;
-	if ((action != 0) && ((error = vnode_authorize(vp, NULL, action, &context)) != 0))
+	if ((action != 0) && ((error = vnode_authorize(vp, NULL, action, ctx)) != 0))
 		goto out;
-	error = vnode_setattr(vp, &va, &context);
+	error = vnode_setattr(vp, &va, ctx);
 out:
 	vnode_put(vp);
 	return (error);
@@ -3380,21 +4454,15 @@ out:
  */
 /* ARGSUSED */
 int
-ftruncate(p, uap, retval)
-	struct proc *p;
-	register struct ftruncate_args *uap;
-	register_t *retval;
+ftruncate(proc_t p, struct ftruncate_args *uap, register_t *retval)
 {
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	struct vnode_attr va;
-	struct vnode *vp;
+	vnode_t vp;
 	struct fileproc *fp;
 	int error ;
 	int fd = uap->fd;
 
-	context.vc_proc = current_proc();
-	context.vc_ucred = kauth_cred_get();
-	
 	AUDIT_ARG(fd, uap->fd);
 	if (uap->length < 0)
 		return(EINVAL);
@@ -3412,7 +4480,7 @@ ftruncate(p, uap, retval)
 		goto out;
 	}
 
-	vp = (struct vnode *)fp->f_fglob->fg_data;
+	vp = (vnode_t)fp->f_fglob->fg_data;
 
 	if ((fp->f_fglob->fg_flag & FWRITE) == 0) {
 		AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
@@ -3426,9 +4494,17 @@ ftruncate(p, uap, retval)
 
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
+#if CONFIG_MACF
+	error = mac_vnode_check_truncate(ctx,
+	    fp->f_fglob->fg_cred, vp);
+	if (error) {
+		(void)vnode_put(vp);
+		goto out;
+	}
+#endif
 	VATTR_INIT(&va);
 	VATTR_SET(&va, va_data_size, uap->length);
-	error = vnode_setattr(vp, &va, &context);
+	error = vnode_setattr(vp, &va, ctx);
 	(void)vnode_put(vp);
 out:
 	file_drop(fd);
@@ -3441,11 +4517,18 @@ out:
  */
 /* ARGSUSED */
 int
-fsync(struct proc *p, struct fsync_args *uap, __unused register_t *retval)
+fsync(proc_t p, struct fsync_args *uap, register_t *retval)
 {
-	struct vnode *vp;
+	__pthread_testcancel(1);
+	return(fsync_nocancel(p, (struct fsync_nocancel_args *)uap, retval));
+}
+
+int
+fsync_nocancel(proc_t p, struct fsync_nocancel_args *uap, __unused register_t *retval)
+{
+	vnode_t vp;
 	struct fileproc *fp;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 
 	if ( (error = fp_getfvp(p, uap->fd, &fp, &vp)) )
@@ -3454,10 +4537,19 @@ fsync(struct proc *p, struct fsync_args *uap, __unused register_t *retval)
 		file_drop(uap->fd);
 		return(error);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = fp->f_fglob->fg_cred;
 
-	error = VNOP_FSYNC(vp, MNT_WAIT, &context);
+	error = VNOP_FSYNC(vp, MNT_WAIT, ctx);
+
+#if NAMEDRSRCFORK
+	/* Sync resource fork shadow file if necessary. */
+	if ((error == 0) &&
+	    (vp->v_flag & VISNAMEDSTREAM) && 
+	    (vp->v_parent != NULLVP) &&
+	    !(vp->v_parent->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS) &&
+	    (fp->f_flags & FP_WRITTEN)) {
+		(void) vnode_flushnamedstream(vp->v_parent, vp, ctx);
+	}
+#endif
 
 	(void)vnode_put(vp);
 	file_drop(uap->fd);
@@ -3473,15 +4565,12 @@ fsync(struct proc *p, struct fsync_args *uap, __unused register_t *retval)
  */
 /* ARGSUSED */
 int
-copyfile(struct proc *p, register struct copyfile_args *uap, __unused register_t *retval)
+copyfile(__unused proc_t p, struct copyfile_args *uap, __unused register_t *retval)
 {
 	vnode_t tvp, fvp, tdvp, sdvp;
 	struct nameidata fromnd, tond;
 	int error;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	/* Check that the flags are valid. */
 
@@ -3490,13 +4579,13 @@ copyfile(struct proc *p, register struct copyfile_args *uap, __unused register_t
 	}
 
 	NDINIT(&fromnd, LOOKUP, SAVESTART | AUDITVNPATH1,
-		UIO_USERSPACE, uap->from, &context);
+		UIO_USERSPACE, uap->from, ctx);
 	if ((error = namei(&fromnd)))
 		return (error);
 	fvp = fromnd.ni_vp;
 
-	NDINIT(&tond, CREATE,  LOCKPARENT | LOCKLEAF | NOCACHE | SAVESTART | AUDITVNPATH2,
-	    UIO_USERSPACE, uap->to, &context);
+	NDINIT(&tond, CREATE,  LOCKPARENT | LOCKLEAF | NOCACHE | SAVESTART | AUDITVNPATH2 | CN_NBMOUNTLOOK,
+	    UIO_USERSPACE, uap->to, ctx);
 	if ((error = namei(&tond))) {
 		goto out1;
 	}
@@ -3514,7 +4603,7 @@ copyfile(struct proc *p, register struct copyfile_args *uap, __unused register_t
 		goto out;
 	}
 
-	if ((error = vnode_authorize(tdvp, NULL, KAUTH_VNODE_ADD_FILE, &context)) != 0)
+	if ((error = vnode_authorize(tdvp, NULL, KAUTH_VNODE_ADD_FILE, ctx)) != 0)
 		goto out;
 
 	if (fvp == tdvp)
@@ -3527,7 +4616,7 @@ copyfile(struct proc *p, register struct copyfile_args *uap, __unused register_t
 	if (fvp == tvp)
 		error = -1;
 	if (!error)
-	        error = VNOP_COPYFILE(fvp,tdvp,tvp,&tond.ni_cnd,uap->mode,uap->flags,&context);
+	        error = VNOP_COPYFILE(fvp, tdvp, tvp, &tond.ni_cnd, uap->mode, uap->flags, ctx);
 out:
 	sdvp = tond.ni_startdir;
 	/*
@@ -3559,37 +4648,43 @@ out1:
  */
 /* ARGSUSED */
 int
-rename(proc_t p, register struct rename_args *uap, __unused register_t *retval)
+rename(__unused proc_t p, struct rename_args *uap, __unused register_t *retval)
 {
 	vnode_t tvp, tdvp;
 	vnode_t fvp, fdvp;
 	struct nameidata fromnd, tond;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	int mntrename;
-	char *oname, *from_name, *to_name;
+	int need_event;
+	const char *oname;
+	char *from_name = NULL, *to_name = NULL;
 	int from_len, to_len;
 	int holding_mntlock;
 	mount_t locked_mp = NULL;
 	vnode_t oparent;
 	fse_info from_finfo, to_finfo;
 	
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 	holding_mntlock = 0;
 retry:
 	fvp = tvp = NULL;
 	fdvp = tdvp = NULL;
 	mntrename = FALSE;
 
-	NDINIT(&fromnd, DELETE, WANTPARENT | AUDITVNPATH1, UIO_USERSPACE, uap->from, &context);
+	NDINIT(&fromnd, DELETE, WANTPARENT | AUDITVNPATH1, UIO_USERSPACE, uap->from, ctx);
 	
 	if ( (error = namei(&fromnd)) )
 	        goto out1;
 	fdvp = fromnd.ni_dvp;
 	fvp  = fromnd.ni_vp;
 
-	NDINIT(&tond, RENAME, WANTPARENT | AUDITVNPATH2, UIO_USERSPACE, uap->to, &context);
+#if CONFIG_MACF
+	error = mac_vnode_check_rename_from(ctx, fdvp, fvp, &fromnd.ni_cnd);
+	if (error)
+		goto out1;
+#endif
+
+	NDINIT(&tond, RENAME, WANTPARENT | AUDITVNPATH2 | CN_NBMOUNTLOOK , UIO_USERSPACE, uap->to, ctx);
 	if (fvp->v_type == VDIR)
 		tond.ni_cnd.cn_flags |= WILLBEDIR;
 
@@ -3604,6 +4699,13 @@ retry:
 	tdvp = tond.ni_dvp;
 	tvp  = tond.ni_vp;
 
+#if CONFIG_MACF
+	error = mac_vnode_check_rename_to(ctx,
+	    tdvp, tvp, fdvp == tdvp, &tond.ni_cnd);
+	if (error)
+		goto out1;
+#endif
+
 	if (tvp != NULL) {
 		if (fvp->v_type == VDIR && tvp->v_type != VDIR) {
 			error = ENOTDIR;
@@ -3617,18 +4719,37 @@ retry:
 		error = EINVAL;
 		goto out1;
 	}
+        /*
+         * If the source and destination are the same (i.e. they're
+         * links to the same vnode) and the target file system is
+         * case sensitive, then there is nothing to do.
+         */
+	if (fvp == tvp) {
+		int pathconf_val;
+		
+		/*
+		 * Note: if _PC_CASE_SENSITIVE selector isn't supported,
+		 * then assume that this file system is case sensitive.
+		 */
+		if (VNOP_PATHCONF(fvp, _PC_CASE_SENSITIVE, &pathconf_val, ctx) != 0 ||
+		    pathconf_val != 0) {
+			goto out1;
+		}	
+	}
 
 	/*
 	 * Authorization.
 	 *
-	 * If tvp is a directory and not the same as fdvp, or tdvp is not the same as fdvp,
-	 * the node is moving between directories and we need rights to remove from the
-	 * old and add to the new.
+	 * If tvp is a directory and not the same as fdvp, or tdvp is not
+	 * the same as fdvp, the node is moving between directories and we
+	 * need rights to remove from the old and add to the new.
+	 *
+	 * If tvp already exists and is not a directory, we need to be
+	 * allowed to delete it.
 	 *
-	 * If tvp already exists and is not a directory, we need to be allowed to delete it.
+	 * Note that we do not inherit when renaming.
 	 *
-	 * Note that we do not inherit when renaming.  XXX this needs to be revisited to
-	 * implement the deferred-inherit bit.
+	 * XXX This needs to be revisited to implement the deferred-inherit bit
 	 */
 	{
 		int moving = 0;
@@ -3641,27 +4762,35 @@ retry:
 			moving = 1;
 		}
 		/*
-		 * must have delete rights to remove the old name even in the simple case of
-		 * fdvp == tdvp
+		 * must have delete rights to remove the old name even in
+		 * the simple case of fdvp == tdvp.
+		 *
+		 * If fvp is a directory, and we are changing it's parent,
+		 * then we also need rights to rewrite its ".." entry as well.
 		 */
-		if ((error = vnode_authorize(fvp, fdvp, KAUTH_VNODE_DELETE, &context)) != 0)
+		if (vnode_isdir(fvp)) {
+			if ((error = vnode_authorize(fvp, fdvp, KAUTH_VNODE_DELETE | KAUTH_VNODE_ADD_SUBDIRECTORY, ctx)) != 0)
+				goto auth_exit;
+		} else {
+		if ((error = vnode_authorize(fvp, fdvp, KAUTH_VNODE_DELETE, ctx)) != 0)
 			goto auth_exit;
+		}
 		if (moving) {
 			/* moving into tdvp or tvp, must have rights to add */
 			if ((error = vnode_authorize(((tvp != NULL) && vnode_isdir(tvp)) ? tvp : tdvp,
 				 NULL, 
 				 vnode_isdir(fvp) ? KAUTH_VNODE_ADD_SUBDIRECTORY : KAUTH_VNODE_ADD_FILE,
-				 &context)) != 0)
+				 ctx)) != 0)
 				goto auth_exit;
 		} else {
 			/* node staying in same directory, must be allowed to add new name */
 			if ((error = vnode_authorize(fdvp, NULL,
-				 vnode_isdir(fvp) ? KAUTH_VNODE_ADD_SUBDIRECTORY : KAUTH_VNODE_ADD_FILE, &context)) != 0)
+				 vnode_isdir(fvp) ? KAUTH_VNODE_ADD_SUBDIRECTORY : KAUTH_VNODE_ADD_FILE, ctx)) != 0)
 				goto auth_exit;
 		}
 		/* overwriting tvp */
 		if ((tvp != NULL) && !vnode_isdir(tvp) &&
-		    ((error = vnode_authorize(tvp, tdvp, KAUTH_VNODE_DELETE, &context)) != 0))
+		    ((error = vnode_authorize(tvp, tdvp, KAUTH_VNODE_DELETE, ctx)) != 0))
 			goto auth_exit;
  		    
 		/* XXX more checks? */
@@ -3685,7 +4814,7 @@ auth_exit:
 	    (fdvp == tdvp)  &&
 	    ((fvp->v_mount->mnt_flag & (MNT_UNION | MNT_ROOTFS)) == 0)  &&
 	    (fvp->v_mount->mnt_vnodecovered != NULLVP)) {
-		struct vnode *coveredvp;
+		vnode_t coveredvp;
 	
 		/* switch fvp to the covered vnode */
 		coveredvp = fvp->v_mount->mnt_vnodecovered;
@@ -3839,40 +4968,62 @@ auth_exit:
 	oname   = fvp->v_name;
 	oparent = fvp->v_parent;
 
-	if (need_fsevent(FSE_RENAME, fvp)) {
-	        get_fse_info(fvp, &from_finfo, &context);
+#if CONFIG_FSE
+	need_event = need_fsevent(FSE_RENAME, fvp);
+	if (need_event) { 
+	        get_fse_info(fvp, &from_finfo, ctx);
 
 		if (tvp) {
-		        get_fse_info(tvp, &to_finfo, &context);
+		        get_fse_info(tvp, &to_finfo, ctx);
+		}
+	}
+#else
+	need_event = 0;
+#endif /* CONFIG_FSE */
+
+	if (need_event || kauth_authorize_fileop_has_listeners()) {
+		GET_PATH(from_name);
+		if (from_name == NULL) {
+			error = ENOMEM;
+			goto out1;
 		}
-	        from_name = get_pathbuff();
 		from_len = MAXPATHLEN;
-		vn_getpath(fvp, from_name, &from_len);
+		vn_getpath(fdvp, from_name, &from_len);
+		if ((from_len + 1 + fromnd.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+		    if (from_len > 2) {
+			from_name[from_len-1] = '/';
+		    } else {
+			from_len--;
+		    }
+		    strlcpy(&from_name[from_len], fromnd.ni_cnd.cn_nameptr, MAXPATHLEN-from_len);
+		    from_len += fromnd.ni_cnd.cn_namelen + 1;
+		    from_name[from_len] = '\0';
+		}
 
-		to_name = get_pathbuff();
-		to_len = MAXPATHLEN;
+		GET_PATH(to_name);
+		if (to_name == NULL) {
+			error = ENOMEM;
+			goto out1;
+		}
 
-		if (tvp && tvp->v_type != VDIR) {
-		        vn_getpath(tvp, to_name, &to_len);
-		} else {
-		        vn_getpath(tdvp, to_name, &to_len);
-			// if the path is not just "/", then append a "/"
-			if (to_len > 2) {
-			        to_name[to_len-1] = '/';
-			} else {
-			        to_len--;
-			}
-			strcpy(&to_name[to_len], tond.ni_cnd.cn_nameptr);
-			to_len += tond.ni_cnd.cn_namelen + 1;
-			to_name[to_len] = '\0';
+		to_len = MAXPATHLEN;
+		vn_getpath(tdvp, to_name, &to_len);
+		// if the path is not just "/", then append a "/"
+		if ((to_len + 1 + tond.ni_cnd.cn_namelen + 1) < MAXPATHLEN) {
+		    if (to_len > 2) {
+			to_name[to_len-1] = '/';
+		    } else {
+			to_len--;
+		    }
+		    strlcpy(&to_name[to_len], tond.ni_cnd.cn_nameptr, MAXPATHLEN-to_len);
+		    to_len += tond.ni_cnd.cn_namelen + 1;
+		    to_name[to_len] = '\0';
 		}
-	} else {
-	        from_name = NULL;
-	        to_name   = NULL;
-	}
+	} 
+	
 	error = VNOP_RENAME(fdvp, fvp, &fromnd.ni_cnd,
 			    tdvp, tvp, &tond.ni_cnd,
-			    &context);
+			    ctx);
 
 	if (holding_mntlock) {
 		/*
@@ -3884,11 +5035,6 @@ auth_exit:
 		holding_mntlock = 0;
 	}
 	if (error) {
-		if (to_name != NULL)
-		        release_pathbuff(to_name);
-		if (from_name != NULL)
-		        release_pathbuff(from_name);
-		from_name = to_name = NULL;
 
 		goto out1;
 	} 
@@ -3896,30 +5042,28 @@ auth_exit:
 	/* call out to allow 3rd party notification of rename. 
 	 * Ignore result of kauth_authorize_fileop call.
 	 */
-	kauth_authorize_fileop(vfs_context_ucred(&context), KAUTH_FILEOP_RENAME, 
-						   (uintptr_t)from_name, (uintptr_t)to_name);
+	kauth_authorize_fileop(vfs_context_ucred(ctx), 
+			KAUTH_FILEOP_RENAME, 
+			(uintptr_t)from_name, (uintptr_t)to_name);
 
+#if CONFIG_FSE
 	if (from_name != NULL && to_name != NULL) {
 	        if (tvp) {
-		        add_fsevent(FSE_RENAME, &context,
+		        add_fsevent(FSE_RENAME, ctx,
 				    FSE_ARG_STRING, from_len, from_name,
 				    FSE_ARG_FINFO, &from_finfo,
 				    FSE_ARG_STRING, to_len, to_name,
 				    FSE_ARG_FINFO, &to_finfo,
 				    FSE_ARG_DONE);
 		} else {
-		        add_fsevent(FSE_RENAME, &context,
+		        add_fsevent(FSE_RENAME, ctx,
 				    FSE_ARG_STRING, from_len, from_name,
 				    FSE_ARG_FINFO, &from_finfo,
 				    FSE_ARG_STRING, to_len, to_name,
 				    FSE_ARG_DONE);
 		}
 	}
-	if (to_name != NULL)
-	        release_pathbuff(to_name);
-	if (from_name != NULL)
-	        release_pathbuff(from_name);
-	from_name = to_name = NULL;
+#endif /* CONFIG_FSE */
 		
 	/*
 	 * update filesystem's mount point data
@@ -3955,7 +5099,7 @@ auth_exit:
 			/* append name to prefix */
 			maxlen = MAXPATHLEN - (pathend - mp->mnt_vfsstat.f_mntonname);
 			bzero(pathend, maxlen);
-			strncpy(pathend, mpname, maxlen - 1);
+			strlcpy(pathend, mpname, maxlen);
 		}
 		FREE_ZONE(tobuf, MAXPATHLEN, M_NAMEI);
 
@@ -3978,6 +5122,11 @@ auth_exit:
 	        vnode_update_identity(fvp, tdvp, tond.ni_cnd.cn_nameptr, tond.ni_cnd.cn_namelen, tond.ni_cnd.cn_hash, update_flags);
 	}
 out1:
+	if (to_name != NULL)
+	        RELEASE_PATH(to_name);
+	if (from_name != NULL)
+	        RELEASE_PATH(from_name);
+
 	if (holding_mntlock) {
 	        mount_unlock_renames(locked_mp);
 		mount_drop(locked_mp, 0);
@@ -4009,6 +5158,12 @@ out1:
 
 /*
  * Make a directory file.
+ *
+ * Returns:	0			Success
+ *		EEXIST
+ *	namei:???
+ *	vnode_authorize:???
+ *	vn_create:???
  */
 /* ARGSUSED */
 static int
@@ -4033,12 +5188,20 @@ mkdir1(vfs_context_t ctx, user_addr_t path, struct vnode_attr *vap)
   		error = EEXIST;
   		goto out;
   	}
+
+	VATTR_SET(vap, va_type, VDIR);
    
+#if CONFIG_MACF
+	error = mac_vnode_check_create(ctx,
+	    nd.ni_dvp, &nd.ni_cnd, vap);
+	if (error)
+		goto out;
+#endif
+
   	/* authorize addition of a directory to the parent */
   	if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_SUBDIRECTORY, ctx)) != 0)
   		goto out;
  	
-	VATTR_SET(vap, va_type, VDIR);
    
 	/* make the directory */
   	if ((error = vn_create(dvp, &vp, &nd.ni_cnd, vap, 0, ctx)) != 0)
@@ -4053,7 +5216,9 @@ mkdir1(vfs_context_t ctx, user_addr_t path, struct vnode_attr *vap)
 	if (update_flags)
 	        vnode_update_identity(vp, dvp, nd.ni_cnd.cn_nameptr, nd.ni_cnd.cn_namelen, nd.ni_cnd.cn_hash, update_flags);
 
+#if CONFIG_FSE
 	add_fsevent(FSE_CREATE_DIR, ctx, FSE_ARG_VNODE, vp, FSE_ARG_DONE);
+#endif
 
 out:
 	/*
@@ -4071,9 +5236,8 @@ out:
 
 
 int
-mkdir_extended(struct proc *p, register struct mkdir_extended_args *uap, __unused register_t *retval)
+mkdir_extended(proc_t p, struct mkdir_extended_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
 	int ciferror;
 	kauth_filesec_t xsecdst;
 	struct vnode_attr va;
@@ -4083,33 +5247,26 @@ mkdir_extended(struct proc *p, register struct mkdir_extended_args *uap, __unuse
 	    ((ciferror = kauth_copyinfilesec(uap->xsecurity, &xsecdst)) != 0))
 		return ciferror;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
   	VATTR_SET(&va, va_mode, (uap->mode & ACCESSPERMS) & ~p->p_fd->fd_cmask);
 	if (xsecdst != NULL)
 		VATTR_SET(&va, va_acl, &xsecdst->fsec_acl);
 
-	ciferror = mkdir1(&context, uap->path, &va);
+	ciferror = mkdir1(vfs_context_current(), uap->path, &va);
 	if (xsecdst != NULL)
 		kauth_filesec_free(xsecdst);
 	return ciferror;
 }
 
 int
-mkdir(struct proc *p, register struct mkdir_args *uap, __unused register_t *retval)
+mkdir(proc_t p, struct mkdir_args *uap, __unused register_t *retval)
 {
-	struct vfs_context context;
 	struct vnode_attr va;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	VATTR_INIT(&va);
   	VATTR_SET(&va, va_mode, (uap->mode & ACCESSPERMS) & ~p->p_fd->fd_cmask);
 
-	return(mkdir1(&context, uap->path, &va));
+	return(mkdir1(vfs_context_current(), uap->path, &va));
 }
 
 /*
@@ -4117,110 +5274,273 @@ mkdir(struct proc *p, register struct mkdir_args *uap, __unused register_t *retv
  */
 /* ARGSUSED */
 int
-rmdir(struct proc *p, struct rmdir_args *uap, __unused register_t *retval)
+rmdir(__unused proc_t p, struct rmdir_args *uap, __unused register_t *retval)
 {
-        vnode_t vp, dvp;
+	vnode_t vp, dvp;
 	int error;
 	struct nameidata nd;
-	struct vfs_context context;
-	
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
-	NDINIT(&nd, DELETE, LOCKPARENT | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
-	error = namei(&nd);
-	if (error)
-		return (error);
-	dvp = nd.ni_dvp;
-	vp = nd.ni_vp;
+	int restart_flag, oldvp_id = -1;
 
-	if (vp->v_type != VDIR) {
-	        /*
-		 * rmdir only deals with directories
-		 */
-		error = ENOTDIR;
-	} else if (dvp == vp) {
-	        /*
-		 * No rmdir "." please.
-		 */
-		error = EINVAL;
-	} else if (vp->v_flag & VROOT) {
-	        /*
-		 * The root of a mounted filesystem cannot be deleted.
+	/* 
+	 * This loop exists to restart rmdir in the unlikely case that two
+	 * processes are simultaneously trying to remove the same directory
+	 * containing orphaned appleDouble files.
+	 */
+	do {
+		restart_flag = 0;
+
+		NDINIT(&nd, DELETE, LOCKPARENT | AUDITVNPATH1, 
+				UIO_USERSPACE, uap->path, ctx);
+		error = namei(&nd);
+		if (error)
+			return (error);
+
+		dvp = nd.ni_dvp;
+		vp = nd.ni_vp;
+
+
+		/*
+		 * If being restarted check if the new vp
+		 * still has the same v_id.
 		 */
-		error = EBUSY;
-	} else {
-		error = vnode_authorize(vp, nd.ni_dvp, KAUTH_VNODE_DELETE, &context);
-	}
-	if (!error) {
-		char     *path = NULL;
-		int       len;
-		fse_info  finfo;
-		
-		if (need_fsevent(FSE_DELETE, dvp)) {
-		        path = get_pathbuff();
-			len = MAXPATHLEN;
-			vn_getpath(vp, path, &len);
-			get_fse_info(vp, &finfo, &context);
+		if (oldvp_id != -1 && oldvp_id != vp->v_id) {
+			error = ENOENT;
+			goto out;
 		}
-		error = VNOP_RMDIR(dvp, vp, &nd.ni_cnd, &context);
 
-		if (!error && path != NULL) {
-			add_fsevent(FSE_DELETE, &context,
-				FSE_ARG_STRING, len, path,
-				FSE_ARG_FINFO, &finfo,
-				FSE_ARG_DONE);
+		if (vp->v_type != VDIR) {
+			/*
+			 * rmdir only deals with directories
+			 */
+			error = ENOTDIR;
+		} else if (dvp == vp) {
+			/*
+			 * No rmdir "." please.
+			 */
+			error = EINVAL;
+		} else if (vp->v_flag & VROOT) {
+			/*
+			 * The root of a mounted filesystem cannot be deleted.
+			 */
+			error = EBUSY;
+		} else {
+#if CONFIG_MACF
+			error = mac_vnode_check_unlink(ctx, dvp,
+					vp, &nd.ni_cnd);
+			if (!error)
+#endif
+				error = vnode_authorize(vp, nd.ni_dvp, KAUTH_VNODE_DELETE, ctx);
 		}
-		if (path != NULL)
-		        release_pathbuff(path);
-	}
-	/*
-	 * nameidone has to happen before we vnode_put(dvp)
-	 * since it may need to release the fs_nodelock on the dvp
-	 */
-	nameidone(&nd);
+		if (!error) {
+			char     *path = NULL;
+			int       len;
+			fse_info  finfo;
+			int has_listeners = 0;
+			int need_event = 0;
+
+#if CONFIG_FSE
+			need_event = need_fsevent(FSE_DELETE, dvp);
+			if (need_event) {
+				get_fse_info(vp, &finfo, ctx);
+			}
+#endif
+			has_listeners = kauth_authorize_fileop_has_listeners();
+			if (need_event || has_listeners) {
+				GET_PATH(path);
+				if (path == NULL) {
+					error = ENOMEM;
+					goto out;
+				}
+				len = MAXPATHLEN;
+				vn_getpath(vp, path, &len);
+			}
 
-	vnode_put(dvp);
-	vnode_put(vp);
+			error = VNOP_RMDIR(dvp, vp, &nd.ni_cnd, ctx);
+
+			/*
+			 * Special case to remove orphaned AppleDouble
+			 * files. I don't like putting this in the kernel,
+			 * but carbon does not like putting this in carbon either,
+			 * so here we are.
+			 */
+			if (error == ENOTEMPTY) {
+				error = rmdir_remove_orphaned_appleDouble(vp, ctx, &restart_flag);
+				if (error == EBUSY) {
+					oldvp_id = vp->v_id;
+					goto out;
+				}
+
+
+				/*
+				 * Assuming everything went well, we will try the RMDIR again 
+				 */
+				if (!error)
+					error = VNOP_RMDIR(dvp, vp, &nd.ni_cnd, ctx);
+			}
+
+			/*
+			 * Call out to allow 3rd party notification of delete. 
+			 * Ignore result of kauth_authorize_fileop call.
+			 */
+			if (!error) {
+				if (has_listeners) {
+					kauth_authorize_fileop(vfs_context_ucred(ctx), 
+							KAUTH_FILEOP_DELETE, 
+							(uintptr_t)vp,
+							(uintptr_t)path);
+				}
+
+				if (vp->v_flag & VISHARDLINK) {
+				    // see the comment in unlink1() about why we update
+				    // the parent of a hard link when it is removed
+				    vnode_update_identity(vp, NULL, NULL, 0, 0, VNODE_UPDATE_PARENT);
+				}
+
+#if CONFIG_FSE
+				if (need_event) {
+					add_fsevent(FSE_DELETE, ctx,
+							FSE_ARG_STRING, len, path,
+							FSE_ARG_FINFO, &finfo,
+							FSE_ARG_DONE);
+				}
+#endif
+			}
+			if (path != NULL)
+				RELEASE_PATH(path);
+		}
+
+out:
+		/*
+		 * nameidone has to happen before we vnode_put(dvp)
+		 * since it may need to release the fs_nodelock on the dvp
+		 */
+		nameidone(&nd);
+
+		vnode_put(dvp);
+		vnode_put(vp);
+
+		if (restart_flag == 0) {
+			wakeup_one((caddr_t)vp);
+			return (error);
+		}
+		tsleep(vp, PVFS, "rm AD", 1);
+
+	} while (restart_flag != 0);
 
 	return (error);
+
 }
 
+/* Get direntry length padded to 8 byte alignment */
+#define DIRENT64_LEN(namlen) \
+	((sizeof(struct direntry) + (namlen) - (MAXPATHLEN-1) + 7) & ~7)
+
+static errno_t 
+vnode_readdir64(struct vnode *vp, struct uio *uio, int flags, int *eofflag,
+                int *numdirent, vfs_context_t ctxp)
+{
+	/* Check if fs natively supports VNODE_READDIR_EXTENDED */
+	if (vp->v_mount->mnt_vtable->vfc_vfsflags & VFC_VFSREADDIR_EXTENDED) {
+		return VNOP_READDIR(vp, uio, flags, eofflag, numdirent, ctxp);
+	} else {
+		size_t bufsize;
+		void * bufptr;
+		uio_t auio;
+		struct direntry entry64;
+		struct dirent *dep;
+		int bytesread;
+		int error;
+
+		/*
+		 * Our kernel buffer needs to be smaller since re-packing
+		 * will expand each dirent.  The worse case (when the name
+		 * length is 3) corresponds to a struct direntry size of 32
+		 * bytes (8-byte aligned) and a struct dirent size of 12 bytes
+		 * (4-byte aligned).  So having a buffer that is 3/8 the size
+		 * will prevent us from reading more than we can pack.
+                 *
+		 * Since this buffer is wired memory, we will limit the
+		 * buffer size to a maximum of 32K. We would really like to 
+		 * use 32K in the MIN(), but we use magic number 87371 to
+		 * prevent uio_resid() * 3 / 8 from overflowing. 
+		 */
+		bufsize = 3 * MIN(uio_resid(uio), 87371) / 8;
+		MALLOC(bufptr, void *, bufsize, M_TEMP, M_WAITOK);
+
+		auio = uio_create(1, 0, UIO_SYSSPACE32, UIO_READ);
+		uio_addiov(auio, (uintptr_t)bufptr, bufsize);
+		auio->uio_offset = uio->uio_offset;
+
+		error = VNOP_READDIR(vp, auio, 0, eofflag, numdirent, ctxp);
+
+		dep = (struct dirent *)bufptr;
+		bytesread = bufsize - uio_resid(auio);
+
+		/*
+		 * Convert all the entries and copy them out to user's buffer.
+		 */
+		while (error == 0 && (char *)dep < ((char *)bufptr + bytesread)) {
+			/* Convert a dirent to a dirent64. */
+			entry64.d_ino = dep->d_ino;
+			entry64.d_seekoff = 0;
+			entry64.d_reclen = DIRENT64_LEN(dep->d_namlen);
+			entry64.d_namlen = dep->d_namlen;
+			entry64.d_type = dep->d_type;
+			bcopy(dep->d_name, entry64.d_name, dep->d_namlen + 1);
+
+			/* Move to next entry. */
+			dep = (struct dirent *)((char *)dep + dep->d_reclen);
+
+			/* Copy entry64 to user's buffer. */
+			error = uiomove((caddr_t)&entry64, entry64.d_reclen, uio);
+		}
+
+		/* Update the real offset using the offset we got from VNOP_READDIR. */
+		if (error == 0) {
+			uio->uio_offset = auio->uio_offset;
+		}
+		uio_free(auio);
+		FREE(bufptr, M_TEMP);
+		return (error);
+	}
+}
 
 /*
  * Read a block of directory entries in a file system independent format.
  */
-int
-getdirentries(p, uap, retval)
-	struct proc *p;
-	register struct getdirentries_args *uap;
-	register_t *retval;
+static int
+getdirentries_common(int fd, user_addr_t bufp, user_size_t bufsize, ssize_t *bytesread,
+                     off_t *offset, int flags)
 {
-	struct vnode *vp;
-	struct vfs_context context;
+	vnode_t vp;
+	struct vfs_context context = *vfs_context_current();	/* local copy */
 	struct fileproc *fp;
 	uio_t auio;
-	int spacetype = proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
-	long loff;
-	int error, eofflag;
-	int fd = uap->fd;
+	int spacetype = proc_is64bit(vfs_context_proc(&context)) ? UIO_USERSPACE64 : UIO_USERSPACE32;
+	off_t loff;
+	int error, eofflag, numdirent;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	AUDIT_ARG(fd, uap->fd);
-	error = fp_getfvp(p, fd, &fp, &vp);
-	if (error)
+	error = fp_getfvp(vfs_context_proc(&context), fd, &fp, &vp);
+	if (error) {
 		return (error);
-
+	}
 	if ((fp->f_fglob->fg_flag & FREAD) == 0) {
 		AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
 		error = EBADF;
 		goto out;
 	}
+
+#if CONFIG_MACF
+	error = mac_file_check_change_offset(vfs_context_ucred(&context), fp->f_fglob);
+	if (error)
+		goto out;
+#endif
 	if ( (error = vnode_getwithref(vp)) ) {
 		goto out;
 	}
-
 	AUDIT_ARG(vnpath, vp, ARG_VNODE1);
 
 unionread:
@@ -4229,86 +5549,102 @@ unionread:
 		error = EINVAL;
 		goto out;
 	}
-	context.vc_proc = p;
-	context.vc_ucred = fp->f_fglob->fg_cred;
+
+#if CONFIG_MACF
+	error = mac_vnode_check_readdir(&context, vp);
+	if (error != 0) {
+		(void)vnode_put(vp);
+		goto out;
+	}
+#endif /* MAC */
 
 	loff = fp->f_fglob->fg_offset;
-	auio = uio_createwithbuffer(1, loff, spacetype, UIO_READ, 
-								  &uio_buf[0], sizeof(uio_buf));
-	uio_addiov(auio, uap->buf, uap->count);
+	auio = uio_createwithbuffer(1, loff, spacetype, UIO_READ, &uio_buf[0], sizeof(uio_buf));
+	uio_addiov(auio, bufp, bufsize);
 
-	error = VNOP_READDIR(vp, auio, 0, &eofflag, (int *)NULL, &context);
-	fp->f_fglob->fg_offset = uio_offset(auio);
+	if (flags & VNODE_READDIR_EXTENDED) {
+		error = vnode_readdir64(vp, auio, flags, &eofflag, &numdirent, &context);
+		fp->f_fglob->fg_offset = uio_offset(auio);
+	} else {
+		error = VNOP_READDIR(vp, auio, 0, &eofflag, &numdirent, &context);
+		fp->f_fglob->fg_offset = uio_offset(auio);
+	}
 	if (error) {
 		(void)vnode_put(vp);
 		goto out;
 	}
 
-#if UNION
-{
-	if ((uap->count == uio_resid(auio)) &&
-	    (vp->v_op == union_vnodeop_p)) {
-		struct vnode *lvp;
-
-		lvp = union_dircache(vp, p);
-		if (lvp != NULLVP) {
-			struct vnode_attr va;
-			/*
-			 * If the directory is opaque,
-			 * then don't show lower entries
-			 */
-			VATTR_INIT(&va);
-			VATTR_WANTED(&va, va_flags);
-			error = vnode_getattr(vp, &va, &context);
-			if (va.va_flags & OPAQUE) {
-				vnode_put(lvp);
-				lvp = NULL;
-			}
+	if ((user_ssize_t)bufsize == uio_resid(auio)){
+		if (union_dircheckp) {
+			error = union_dircheckp(&vp, fp, &context);
+			if (error == -1)
+				goto unionread;
+			if (error)
+				goto out;
 		}
 
-		if (lvp != NULLVP) {
-			error = VNOP_OPEN(lvp, FREAD, &context);
-			if (error) {
-				vnode_put(lvp);
-				goto out;
-			}
-			vnode_ref(lvp);
-			fp->f_fglob->fg_data = (caddr_t) lvp;
+		if ((vp->v_flag & VROOT) && (vp->v_mount->mnt_flag & MNT_UNION)) {
+			struct vnode *tvp = vp;
+			vp = vp->v_mount->mnt_vnodecovered;
+			vnode_getwithref(vp);
+			vnode_ref(vp);
+			fp->f_fglob->fg_data = (caddr_t) vp;
 			fp->f_fglob->fg_offset = 0;
-			error = VNOP_CLOSE(vp, FREAD, &context);
-			vnode_rele(vp);
-			vnode_put(vp);
-			if (error)
-				goto out;
-			vp = lvp;
+			vnode_rele(tvp);
+			vnode_put(tvp);
 			goto unionread;
 		}
 	}
-}
-#endif /* UNION */
-
-	if (((user_ssize_t)uap->count == uio_resid(auio)) &&
-	    (vp->v_flag & VROOT) &&
-	    (vp->v_mount->mnt_flag & MNT_UNION)) {
-		struct vnode *tvp = vp;
-		vp = vp->v_mount->mnt_vnodecovered;
-		vnode_getwithref(vp);
-		vnode_ref(vp);
-		fp->f_fglob->fg_data = (caddr_t) vp;
-		fp->f_fglob->fg_offset = 0;
-		vnode_rele(tvp);
-		vnode_put(tvp);
-		goto unionread;
-	}
+
 	vnode_put(vp);
-	error = copyout((caddr_t)&loff, uap->basep, sizeof(long));
+	if (offset) {
+		*offset = loff;
+	}
 	// LP64todo - fix this
-	*retval = uap->count - uio_resid(auio);
+	*bytesread = bufsize - uio_resid(auio);
 out:
 	file_drop(fd);
 	return (error);
 }
 
+
+int
+getdirentries(__unused struct proc *p, struct getdirentries_args *uap, register_t *retval)
+{
+	off_t offset;
+	long loff;
+	ssize_t bytesread;
+	int error;
+
+	AUDIT_ARG(fd, uap->fd);
+	error = getdirentries_common(uap->fd, uap->buf, uap->count, &bytesread, &offset, 0);
+
+	if (error == 0) {
+		loff = (long)offset;
+		error = copyout((caddr_t)&loff, uap->basep, sizeof(long));
+		*retval = bytesread;
+	}
+	return (error);
+}
+
+int
+getdirentries64(__unused struct proc *p, struct getdirentries64_args *uap, user_ssize_t *retval)
+{
+	off_t offset;
+	ssize_t bytesread;
+	int error;
+
+	AUDIT_ARG(fd, uap->fd);
+	error = getdirentries_common(uap->fd, uap->buf, uap->bufsize, &bytesread, &offset, VNODE_READDIR_EXTENDED);
+
+	if (error == 0) {
+		*retval = bytesread;
+		error = copyout((caddr_t)&offset, uap->position, sizeof(off_t));
+	}
+	return (error);
+}
+
+
 /*
  * Set the mode mask for creation of filesystem nodes.
  */
@@ -4316,20 +5652,22 @@ out:
 
 #define UMASK_NOXSECURITY	 (void *)1	/* leave existing xsecurity alone */
 static int
-umask1(struct proc *p, int newmask, __unused kauth_filesec_t fsec, register_t *retval)
+umask1(proc_t p, int newmask, __unused kauth_filesec_t fsec, register_t *retval)
 {
-	register struct filedesc *fdp;
+	struct filedesc *fdp;
 
 	AUDIT_ARG(mask, newmask);
+	proc_fdlock(p);
 	fdp = p->p_fd;
 	*retval = fdp->fd_cmask;
 	fdp->fd_cmask = newmask & ALLPERMS;
+	proc_fdunlock(p);
 	return (0);
 }
 
 
 int
-umask_extended(struct proc *p, struct umask_extended_args *uap, register_t *retval)
+umask_extended(proc_t p, struct umask_extended_args *uap, register_t *retval)
 {
 	int ciferror;
 	kauth_filesec_t xsecdst;
@@ -4350,7 +5688,7 @@ umask_extended(struct proc *p, struct umask_extended_args *uap, register_t *retv
 }
 
 int
-umask(struct proc *p, struct umask_args *uap, register_t *retval)
+umask(proc_t p, struct umask_args *uap, register_t *retval)
 {
 	return(umask1(p, uap->newmask, UMASK_NOXSECURITY, retval));
 }
@@ -4361,19 +5699,16 @@ umask(struct proc *p, struct umask_args *uap, register_t *retval)
  */
 /* ARGSUSED */
 int
-revoke(struct proc *p, register struct revoke_args *uap, __unused register_t *retval)
+revoke(proc_t p, struct revoke_args *uap, __unused register_t *retval)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	struct vnode_attr va;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int error;
 	struct nameidata nd;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -4381,15 +5716,21 @@ revoke(struct proc *p, register struct revoke_args *uap, __unused register_t *re
 
 	nameidone(&nd);
 
+#if CONFIG_MACF
+	error = mac_vnode_check_revoke(ctx, vp);
+	if (error)
+		goto out;
+#endif
+
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_uid);
-	if ((error = vnode_getattr(vp, &va, &context)))
+	if ((error = vnode_getattr(vp, &va, ctx)))
 		goto out;
-	if (kauth_cred_getuid(context.vc_ucred) != va.va_uid &&
-	    (error = suser(context.vc_ucred, &p->p_acflag)))
+	if (kauth_cred_getuid(vfs_context_ucred(ctx)) != va.va_uid &&
+	    (error = suser(vfs_context_ucred(ctx), &p->p_acflag)))
 		goto out;
 	if (vp->v_usecount > 1 || (vp->v_flag & VALIASED))
-		VNOP_REVOKE(vp, REVOKEALL, &context);
+		VNOP_REVOKE(vp, REVOKEALL, ctx);
 out:
 	vnode_put(vp);
 	return (error);
@@ -4413,7 +5754,7 @@ out:
  */
 /* ARGSUSED */
 int
-mkcomplex(__unused struct proc *p, __unused struct mkcomplex_args *uap, __unused register_t *retval)
+mkcomplex(__unused proc_t p, __unused struct mkcomplex_args *uap, __unused register_t *retval)
 {
 	return (ENOTSUP);
 }
@@ -4423,7 +5764,7 @@ mkcomplex(__unused struct proc *p, __unused struct mkcomplex_args *uap, __unused
  */
 /* ARGSUSED */
 int
-statv(__unused struct proc *p,
+statv(__unused proc_t p,
 	  __unused struct statv_args *uap,
 	  __unused register_t *retval)
 {
@@ -4436,7 +5777,7 @@ statv(__unused struct proc *p,
 */
 /* ARGSUSED */
 int
-lstatv(__unused struct proc *p,
+lstatv(__unused proc_t p,
 	   __unused struct lstatv_args *uap,
 	   __unused register_t *retval)
 {
@@ -4448,7 +5789,7 @@ lstatv(__unused struct proc *p,
 */
 /* ARGSUSED */
 int
-fstatv(__unused struct proc *p, 
+fstatv(__unused proc_t p, 
 	   __unused struct fstatv_args *uap, 
 	   __unused register_t *retval)
 {
@@ -4471,19 +5812,18 @@ fstatv(__unused struct proc *p,
 
 /* ARGSUSED */
 int
-getdirentriesattr (struct proc *p, struct getdirentriesattr_args *uap, register_t *retval)
+getdirentriesattr (proc_t p, struct getdirentriesattr_args *uap, register_t *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct fileproc *fp;
 	uio_t auio = NULL;
 	int spacetype = proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
-	uint64_t actualcount;
-	u_long tmpcount;
-	u_long newstate;
+	uint32_t count;
+	uint32_t newstate;
 	int error, eofflag;
-	u_long loff;
+	uint32_t loff;
 	struct attrlist attributelist; 
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	int fd = uap->fd;
 	char uio_buf[ UIO_SIZEOF(1) ];
 	kauth_action_t action;
@@ -4491,20 +5831,30 @@ getdirentriesattr (struct proc *p, struct getdirentriesattr_args *uap, register_
 	AUDIT_ARG(fd, fd);
     
 	/* Get the attributes into kernel space */
-	if ((error = copyin(uap->alist, (caddr_t) &attributelist, sizeof (attributelist))))
+	if ((error = copyin(uap->alist, (caddr_t)&attributelist, sizeof(attributelist)))) {
 		return(error);
-	actualcount = fuulong(uap->count);
-	if (actualcount == -1ULL)
-		return(-1);
-
-	if ( (error = fp_getfvp(p, fd, &fp, &vp)) )
+	}
+	if ((error = copyin(uap->count, (caddr_t)&count, sizeof(count)))) {
+		return(error);
+	}
+	if ( (error = fp_getfvp(p, fd, &fp, &vp)) ) {
 		return (error);
-
+	}
 	if ((fp->f_fglob->fg_flag & FREAD) == 0) {
 		AUDIT_ARG(vnpath_withref, vp, ARG_VNODE1);
 		error = EBADF;
 		goto out;
 	}
+
+
+#if CONFIG_MACF
+	error = mac_file_check_change_offset(vfs_context_ucred(ctx),
+	    fp->f_fglob);
+	if (error)
+		goto out;
+#endif
+
+
 	if ( (error = vnode_getwithref(vp)) )
 		goto out;
 
@@ -4516,16 +5866,20 @@ getdirentriesattr (struct proc *p, struct getdirentriesattr_args *uap, register_
 		goto out;
 	}
 
+#if CONFIG_MACF
+	error = mac_vnode_check_readdir(ctx, vp);
+	if (error != 0) {
+		(void)vnode_put(vp);
+		goto out;
+	}
+#endif /* MAC */
+
 	/* set up the uio structure which will contain the users return buffer */
 	loff = fp->f_fglob->fg_offset;
 	auio = uio_createwithbuffer(1, loff, spacetype, UIO_READ, 
 	    &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, uap->buffer, uap->buffersize);
        
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-	tmpcount = (u_long) actualcount;
-
 	/*
 	 * If the only item requested is file names, we can let that past with
 	 * just LIST_DIRECTORY.  If they want any other attributes, that means
@@ -4536,27 +5890,32 @@ getdirentriesattr (struct proc *p, struct getdirentriesattr_args *uap, register_
 	    attributelist.fileattr || attributelist.dirattr)
 		action |= KAUTH_VNODE_SEARCH;
 	
-	if ((error = vnode_authorize(vp, NULL, action, &context)) == 0)
+	if ((error = vnode_authorize(vp, NULL, action, ctx)) == 0) {
+		u_long ulcount = count;
+
 		error = VNOP_READDIRATTR(vp, &attributelist, auio,
-		    tmpcount, uap->options, &newstate, &eofflag,
-		    &tmpcount, &context);
+					 count,
+		                         uap->options, (unsigned long *)&newstate, &eofflag,
+		                         &ulcount, ctx);
+		if (!error)
+			count = ulcount;
+	}
 	(void)vnode_put(vp);
-	actualcount = tmpcount;
 
 	if (error) 
 		goto out;
 	fp->f_fglob->fg_offset = uio_offset(auio); /* should be multiple of dirent, not variable */
 
-	if ((error = suulong(uap->count, actualcount)) != 0)
+	if ((error = copyout((caddr_t) &count, uap->count, sizeof(count))))
 		goto out;
-	if ((error = suulong(uap->newstate, (uint64_t)newstate)) != 0)
+	if ((error = copyout((caddr_t) &newstate, uap->newstate, sizeof(newstate))))
 		goto out;
-	if ((error = suulong(uap->basep, (uint64_t)loff)) != 0)
+	if ((error = copyout((caddr_t) &loff, uap->basep, sizeof(loff))))
 		goto out;
 
 	*retval = eofflag;  /* similar to getdirentries */
 	error = 0;
-	out:
+out:
 	file_drop(fd);
 	return (error); /* return error earlier, an retval of 0 or 1 now */
 
@@ -4568,27 +5927,25 @@ getdirentriesattr (struct proc *p, struct getdirentriesattr_args *uap, register_
 
 /* ARGSUSED */
 int
-exchangedata (struct proc *p, register struct exchangedata_args *uap, __unused register_t *retval)
+exchangedata (__unused proc_t p, struct exchangedata_args *uap, __unused register_t *retval)
 {
 
 	struct nameidata fnd, snd;
-	struct vfs_context context;
-	struct vnode *fvp, *svp;
-    int error;
+	vfs_context_t ctx = vfs_context_current();
+	vnode_t fvp;
+	vnode_t svp;
+	int error;
 	u_long nameiflags;
 	char *fpath = NULL;
 	char *spath = NULL;
 	int   flen, slen;
 	fse_info f_finfo, s_finfo;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	nameiflags = 0;
 	if ((uap->options & FSOPT_NOFOLLOW) == 0) nameiflags |= FOLLOW;
 
     NDINIT(&fnd, LOOKUP, nameiflags | AUDITVNPATH1, 
-        	UIO_USERSPACE, uap->path1, &context);
+        	UIO_USERSPACE, uap->path1, ctx);
 
     error = namei(&fnd);
     if (error)
@@ -4597,8 +5954,8 @@ exchangedata (struct proc *p, register struct exchangedata_args *uap, __unused r
 	nameidone(&fnd);
 	fvp = fnd.ni_vp;
 
-    NDINIT(&snd, LOOKUP, nameiflags | AUDITVNPATH2, 
-        	UIO_USERSPACE, uap->path2, &context);
+    NDINIT(&snd, LOOKUP | CN_NBMOUNTLOOK, nameiflags | AUDITVNPATH2, 
+        	UIO_USERSPACE, uap->path2, ctx);
 
     error = namei(&snd);
     if (error) {
@@ -4623,37 +5980,54 @@ exchangedata (struct proc *p, register struct exchangedata_args *uap, __unused r
 	        error = EXDEV;
 		goto out;
 	}
-	if (((error = vnode_authorize(fvp, NULL, KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, &context)) != 0) ||
-	    ((error = vnode_authorize(svp, NULL, KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, &context)) != 0))
+
+#if CONFIG_MACF
+	error = mac_vnode_check_exchangedata(ctx,
+	    fvp, svp);
+	if (error)
+		goto out;
+#endif
+	if (((error = vnode_authorize(fvp, NULL, KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, ctx)) != 0) ||
+	    ((error = vnode_authorize(svp, NULL, KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA, ctx)) != 0))
 		goto out;
 
-	if (need_fsevent(FSE_EXCHANGE, fvp) || kauth_authorize_fileop_has_listeners()) {
-	        fpath = get_pathbuff();
-		spath = get_pathbuff();
+	if (
+#if CONFIG_FSE
+	need_fsevent(FSE_EXCHANGE, fvp) || 
+#endif
+	kauth_authorize_fileop_has_listeners()) {
+		GET_PATH(fpath);
+		GET_PATH(spath);
+		if (fpath == NULL || spath == NULL) {
+			error = ENOMEM;
+			goto out;
+		}
 		flen = MAXPATHLEN;
 		slen = MAXPATHLEN;
 		if (vn_getpath(fvp, fpath, &flen) != 0 || fpath[0] == '\0') {
-		        printf("exchange: vn_getpath(fvp=0x%x) failed <<%s>>\n",
+		        printf("exchange: vn_getpath(fvp=%p) failed <<%s>>\n",
 			       fvp, fpath);
 		}
 		if (vn_getpath(svp, spath, &slen) != 0 || spath[0] == '\0') {
-		        printf("exchange: vn_getpath(svp=0x%x) failed <<%s>>\n",
+		        printf("exchange: vn_getpath(svp=%p) failed <<%s>>\n",
 			       svp, spath);
 		}
-		get_fse_info(fvp, &f_finfo, &context);
-		get_fse_info(svp, &s_finfo, &context);
+#if CONFIG_FSE
+		get_fse_info(fvp, &f_finfo, ctx);
+		get_fse_info(svp, &s_finfo, ctx);
+#endif
 	}
 	/* Ok, make the call */
-	error = VNOP_EXCHANGE(fvp, svp, 0, &context);
+	error = VNOP_EXCHANGE(fvp, svp, 0, ctx);
 
 	if (error == 0) {
-	    char *tmpname;
+	    const char *tmpname;
 
 	    if (fpath != NULL && spath != NULL) {
 	            /* call out to allow 3rd party notification of exchangedata. 
 		     * Ignore result of kauth_authorize_fileop call.
 		     */
-	            kauth_authorize_fileop(vfs_context_ucred(&context), KAUTH_FILEOP_EXCHANGE, 
+	            kauth_authorize_fileop(vfs_context_ucred(ctx), KAUTH_FILEOP_EXCHANGE, 
 					   (uintptr_t)fpath, (uintptr_t)spath);
 	    }
 	    name_cache_lock();
@@ -4663,7 +6037,7 @@ exchangedata (struct proc *p, register struct exchangedata_args *uap, __unused r
 	    svp->v_name = tmpname;
 	    
 	    if (fvp->v_parent != svp->v_parent) {
-		struct vnode *tmp;
+		vnode_t tmp;
 
 		tmp           = fvp->v_parent;
 		fvp->v_parent = svp->v_parent;
@@ -4671,21 +6045,23 @@ exchangedata (struct proc *p, register struct exchangedata_args *uap, __unused r
 	    }
 	    name_cache_unlock();
 
+#if CONFIG_FSE
 	    if (fpath != NULL && spath != NULL) {
-	            add_fsevent(FSE_EXCHANGE, &context,
+	            add_fsevent(FSE_EXCHANGE, ctx,
 				FSE_ARG_STRING, flen, fpath,
 				FSE_ARG_FINFO, &f_finfo,
 				FSE_ARG_STRING, slen, spath,
 				FSE_ARG_FINFO, &s_finfo,
 				FSE_ARG_DONE);
 	    }
+#endif
 	}
-	if (spath != NULL)
-	        release_pathbuff(spath);
-	if (fpath != NULL)
-	        release_pathbuff(fpath);
 
 out:
+	if (fpath != NULL)
+	        RELEASE_PATH(fpath);
+	if (spath != NULL)
+	        RELEASE_PATH(spath);
 	vnode_put(svp);
 	vnode_put(fvp);
 out2:
@@ -4693,102 +6069,12 @@ out2:
 }
 
 
-#ifdef __APPLE_API_OBSOLETE
-
-/************************************************/
-/* *** Following calls will be deleted soon *** */
-/************************************************/
-
-/*
-* Check users access to a file 
-*/
-
-/* ARGSUSED */
-#warning "checkuseraccess copies a cred in from user space but"
-#warning "user space has no way of knowing what one looks like"
-#warning "this code should use the access_extended spoof-as functionality"
-int
-checkuseraccess (struct proc *p, register struct checkuseraccess_args *uap, __unused register_t *retval)
-{
-	register struct vnode *vp;
-	int error;
-	struct nameidata nd;
-	struct ucred template_cred;
-	int flags;		/*what will actually get passed to access*/
-	u_long nameiflags;
-	struct vfs_context context;
-	kauth_cred_t my_cred;
-
-	/* Make sure that the number of groups is correct before we do anything */
-
-	if ((uap->ngroups <=  0) || (uap->ngroups > NGROUPS))
-		return (EINVAL);
-
-	/* Verify that the caller is root */
-	
-	if ((error = suser(kauth_cred_get(), &p->p_acflag)))
-		return(error);
-
-	/*
-	 * Fill in the template credential structure; we use a template because
-	 * lookup can attempt to take a persistent reference.
-	 */
-	template_cred.cr_uid = uap->userid;
-	template_cred.cr_ngroups = uap->ngroups;
-	if ((error = copyin(CAST_USER_ADDR_T(uap->groups), (caddr_t) &(template_cred.cr_groups), (sizeof(gid_t))*uap->ngroups)))
-		return (error);
-
-	my_cred = kauth_cred_create(&template_cred);
-	context.vc_proc = p;
-	context.vc_ucred = my_cred;
-
-	/* Get our hands on the file */
-	nameiflags = 0;
-	if ((uap->options & FSOPT_NOFOLLOW) == 0) nameiflags |= FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags | AUDITVNPATH1, 
-		UIO_USERSPACE, CAST_USER_ADDR_T(uap->path), &context);
-
-	if ((error = namei(&nd))) {
-		kauth_cred_unref(&context.vc_ucred);
-       		 return (error);
-	}
-	nameidone(&nd);
-	vp = nd.ni_vp;
-	
-	/* Flags == 0 means only check for existence. */
-
-	flags = 0;
-
-	if (uap->accessrequired) {
-		if (uap->accessrequired & R_OK)
-			flags |= KAUTH_VNODE_READ_DATA;
-		if (uap->accessrequired & W_OK)
-			flags |= KAUTH_VNODE_WRITE_DATA;
-		if (uap->accessrequired & X_OK)
-			flags |= KAUTH_VNODE_EXECUTE;
-		}
-	error = vnode_authorize(vp, NULL, flags, &context);
-		    	
-	vnode_put(vp);
-
-	kauth_cred_unref(&context.vc_ucred);
-	return (error);
-} /* end of checkuseraccess system call */
-
-/************************************************/
-/* *** Preceding calls will be deleted soon *** */
-/************************************************/
-
-#endif /* __APPLE_API_OBSOLETE */
-
-
-
 /* ARGSUSED */
 
 int
-searchfs (struct proc *p, register struct searchfs_args *uap, __unused register_t *retval)
+searchfs(proc_t p, struct searchfs_args *uap, __unused register_t *retval)
 {
-	register struct vnode *vp;
+	vnode_t vp;
 	int error=0;
 	int fserror = 0;
 	struct nameidata nd;
@@ -4801,12 +6087,9 @@ searchfs (struct proc *p, register struct searchfs_args *uap, __unused register_
 	u_long nummatches;
 	int mallocsize;
 	u_long nameiflags;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
 	/* Start by copying in fsearchblock paramater list */
     if (IS_64BIT_PROCESS(p)) {
        error = copyin(uap->searchblock, (caddr_t) &searchblock, sizeof(searchblock));
@@ -4875,7 +6158,7 @@ searchfs (struct proc *p, register struct searchfs_args *uap, __unused register_
 	nameiflags = 0;
 	if ((uap->options & FSOPT_NOFOLLOW) == 0) nameiflags |= FOLLOW;
 	NDINIT(&nd, LOOKUP, nameiflags | AUDITVNPATH1, 
-		UIO_USERSPACE, uap->path, &context);
+		UIO_USERSPACE, uap->path, ctx);
 
 	error = namei(&nd);
 	if (error)
@@ -4914,7 +6197,7 @@ searchfs (struct proc *p, register struct searchfs_args *uap, __unused register_
 							uap->options,
 							auio,
 							state,
-							&context);
+							ctx);
 		
 saveandexit:
 
@@ -4946,21 +6229,18 @@ freeandexit:
  */
 /* ARGSUSED */
 int
-fsctl (struct proc *p, struct fsctl_args *uap, __unused register_t *retval)
+fsctl (proc_t p, struct fsctl_args *uap, __unused register_t *retval)
 {
 	int error;
 	boolean_t is64bit;
 	struct nameidata nd;	
 	u_long nameiflags;
 	u_long cmd = uap->cmd;
-	register u_int size;
+	u_int size;
 #define STK_PARAMS 128
 	char stkbuf[STK_PARAMS];
 	caddr_t data, memp;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	size = IOCPARM_LEN(cmd);
 	if (size > IOCPARM_MAX) return (EINVAL);
@@ -5005,11 +6285,20 @@ fsctl (struct proc *p, struct fsctl_args *uap, __unused register_t *retval)
 	/* Get the vnode for the file we are getting info on:  */
 	nameiflags = 0;
 	if ((uap->options & FSOPT_NOFOLLOW) == 0) nameiflags |= FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, UIO_USERSPACE, uap->path, &context);
+	NDINIT(&nd, LOOKUP, nameiflags, UIO_USERSPACE, uap->path, ctx);
 	if ((error = namei(&nd))) goto FSCtl_Exit;
 
+#if CONFIG_MACF
+	error = mac_mount_check_fsctl(ctx, vnode_mount(nd.ni_vp), cmd);
+	if (error) {
+		vnode_put(nd.ni_vp);
+		nameidone(&nd);
+		goto FSCtl_Exit;
+	}
+#endif
+
 	/* Invoke the filesystem-specific code */
-	error = VNOP_IOCTL(nd.ni_vp, IOCBASECMD(cmd), data, uap->options, &context);
+	error = VNOP_IOCTL(nd.ni_vp, IOCBASECMD(cmd), data, uap->options, ctx);
 	
 	vnode_put(nd.ni_vp);
 	nameidone(&nd);
@@ -5052,12 +6341,12 @@ sync_internal(void)
  *  Retrieve the data of an extended attribute.
  */
 int
-getxattr(struct proc *p, struct getxattr_args *uap, user_ssize_t *retval)
+getxattr(proc_t p, struct getxattr_args *uap, user_ssize_t *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct nameidata nd;
 	char attrname[XATTR_MAXNAMELEN+1];
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	uio_t auio = NULL;
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t attrsize = 0;
@@ -5066,14 +6355,11 @@ getxattr(struct proc *p, struct getxattr_args *uap, user_ssize_t *retval)
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	if (uap->options & XATTR_NOSECURITY)
+	if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	nameiflags = (uap->options & XATTR_NOFOLLOW) ? 0 : FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, &context);
+	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, ctx);
 	if ((error = namei(&nd))) {
 		return (error);
 	}
@@ -5093,7 +6379,7 @@ getxattr(struct proc *p, struct getxattr_args *uap, user_ssize_t *retval)
 		uio_addiov(auio, uap->value, uap->size);
 	}
 
-	error = vn_getxattr(vp, attrname, auio, &attrsize, uap->options, &context);
+	error = vn_getxattr(vp, attrname, auio, &attrsize, uap->options, ctx);
 out:
 	vnode_put(vp);
 
@@ -5110,11 +6396,10 @@ out:
  * Retrieve the data of an extended attribute.
  */
 int
-fgetxattr(struct proc *p, struct fgetxattr_args *uap, user_ssize_t *retval)
+fgetxattr(proc_t p, struct fgetxattr_args *uap, user_ssize_t *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	char attrname[XATTR_MAXNAMELEN+1];
-	struct vfs_context context;
 	uio_t auio = NULL;
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t attrsize = 0;
@@ -5122,7 +6407,7 @@ fgetxattr(struct proc *p, struct fgetxattr_args *uap, user_ssize_t *retval)
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY))
+	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	if ( (error = file_vnode(uap->fd, &vp)) ) {
@@ -5144,10 +6429,8 @@ fgetxattr(struct proc *p, struct fgetxattr_args *uap, user_ssize_t *retval)
 		                            &uio_buf[0], sizeof(uio_buf));
 		uio_addiov(auio, uap->value, uap->size);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
-	error = vn_getxattr(vp, attrname, auio, &attrsize, uap->options, &context);
+	error = vn_getxattr(vp, attrname, auio, &attrsize, uap->options, vfs_context_current());
 out:
 	(void)vnode_put(vp);
 	file_drop(uap->fd);
@@ -5164,12 +6447,12 @@ out:
  * Set the data of an extended attribute.
  */
 int
-setxattr(struct proc *p, struct setxattr_args *uap, int *retval)
+setxattr(proc_t p, struct setxattr_args *uap, int *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct nameidata nd;
 	char attrname[XATTR_MAXNAMELEN+1];
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	uio_t auio = NULL;
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t namelen;
@@ -5177,10 +6460,7 @@ setxattr(struct proc *p, struct setxattr_args *uap, int *retval)
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	if (uap->options & XATTR_NOSECURITY)
+	if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	if ((error = copyinstr(uap->attrname, attrname, sizeof(attrname), &namelen) != 0)) {
@@ -5188,12 +6468,12 @@ setxattr(struct proc *p, struct setxattr_args *uap, int *retval)
 	}
 	if (xattr_protected(attrname))
 		return(EPERM);
-	if (uap->value == 0 || uap->size == 0) {
+	if (uap->size != 0 && uap->value == 0) {
 		return (EINVAL);
 	}
 
 	nameiflags = (uap->options & XATTR_NOFOLLOW) ? 0 : FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, &context);
+	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, ctx);
 	if ((error = namei(&nd))) {
 		return (error);
 	}
@@ -5204,7 +6484,14 @@ setxattr(struct proc *p, struct setxattr_args *uap, int *retval)
 	                            &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, uap->value, uap->size);
 
-	error = vn_setxattr(vp, attrname, auio, uap->options, &context);
+	error = vn_setxattr(vp, attrname, auio, uap->options, ctx);
+#if CONFIG_FSE
+	if (error == 0) {
+		add_fsevent(FSE_XATTR_MODIFIED, ctx,
+		    FSE_ARG_VNODE, vp,
+		    FSE_ARG_DONE);
+	}
+#endif
 	vnode_put(vp);
 	*retval = 0;
 	return (error);
@@ -5214,18 +6501,18 @@ setxattr(struct proc *p, struct setxattr_args *uap, int *retval)
  * Set the data of an extended attribute.
  */
 int
-fsetxattr(struct proc *p, struct fsetxattr_args *uap, int *retval)
+fsetxattr(proc_t p, struct fsetxattr_args *uap, int *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	char attrname[XATTR_MAXNAMELEN+1];
-	struct vfs_context context;
 	uio_t auio = NULL;
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t namelen;
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
+	vfs_context_t ctx = vfs_context_current();
 
-	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY))
+	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	if ((error = copyinstr(uap->attrname, attrname, sizeof(attrname), &namelen) != 0)) {
@@ -5233,7 +6520,7 @@ fsetxattr(struct proc *p, struct fsetxattr_args *uap, int *retval)
 	}
 	if (xattr_protected(attrname))
 		return(EPERM);
-	if (uap->value == 0 || uap->size == 0) {
+	if (uap->size != 0 && uap->value == 0) {
 		return (EINVAL);
 	}
 	if ( (error = file_vnode(uap->fd, &vp)) ) {
@@ -5246,10 +6533,15 @@ fsetxattr(struct proc *p, struct fsetxattr_args *uap, int *retval)
 	auio = uio_createwithbuffer(1, uap->position, spacetype, UIO_WRITE,
 	                            &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, uap->value, uap->size);
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
-	error = vn_setxattr(vp, attrname, auio, uap->options, &context);
+	error = vn_setxattr(vp, attrname, auio, uap->options, vfs_context_current());
+#if CONFIG_FSE
+	if (error == 0) {
+		add_fsevent(FSE_XATTR_MODIFIED, ctx,
+		    FSE_ARG_VNODE, vp,
+		    FSE_ARG_DONE);
+	}
+#endif
 	vnode_put(vp);
 	file_drop(uap->fd);
 	*retval = 0;
@@ -5261,21 +6553,18 @@ fsetxattr(struct proc *p, struct fsetxattr_args *uap, int *retval)
  */
 #warning "code duplication"
 int
-removexattr(struct proc *p, struct removexattr_args *uap, int *retval)
+removexattr(proc_t p, struct removexattr_args *uap, int *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct nameidata nd;
 	char attrname[XATTR_MAXNAMELEN+1];
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	size_t namelen;
 	u_long nameiflags;
 	int error;
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	if (uap->options & XATTR_NOSECURITY)
+	if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	error = copyinstr(uap->attrname, attrname, sizeof(attrname), &namelen);
@@ -5285,14 +6574,21 @@ removexattr(struct proc *p, struct removexattr_args *uap, int *retval)
 	if (xattr_protected(attrname))
 		return(EPERM);
 	nameiflags = (uap->options & XATTR_NOFOLLOW) ? 0 : FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, &context);
+	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, ctx);
 	if ((error = namei(&nd))) {
 		return (error);
 	}
 	vp = nd.ni_vp;
 	nameidone(&nd);
 
-	error = vn_removexattr(vp, attrname, uap->options, &context);
+	error = vn_removexattr(vp, attrname, uap->options, ctx);
+#if CONFIG_FSE
+	if (error == 0) {
+		add_fsevent(FSE_XATTR_REMOVED, ctx,
+		    FSE_ARG_VNODE, vp,
+		    FSE_ARG_DONE);
+	}
+#endif
 	vnode_put(vp);
 	*retval = 0;
 	return (error);
@@ -5303,15 +6599,15 @@ removexattr(struct proc *p, struct removexattr_args *uap, int *retval)
  */
 #warning "code duplication"
 int
-fremovexattr(struct proc *p, struct fremovexattr_args *uap, int *retval)
+fremovexattr(__unused proc_t p, struct fremovexattr_args *uap, int *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	char attrname[XATTR_MAXNAMELEN+1];
-	struct vfs_context context;
 	size_t namelen;
 	int error;
+	vfs_context_t ctx = vfs_context_current();
 
-	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY))
+	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	error = copyinstr(uap->attrname, attrname, sizeof(attrname), &namelen);
@@ -5327,10 +6623,15 @@ fremovexattr(struct proc *p, struct fremovexattr_args *uap, int *retval)
 		file_drop(uap->fd);
 		return(error);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
-	error = vn_removexattr(vp, attrname, uap->options, &context);
+	error = vn_removexattr(vp, attrname, uap->options, vfs_context_current());
+#if CONFIG_FSE
+	if (error == 0) {
+		add_fsevent(FSE_XATTR_REMOVED, ctx,
+		    FSE_ARG_VNODE, vp,
+		    FSE_ARG_DONE);
+	}
+#endif
 	vnode_put(vp);
 	file_drop(uap->fd);
 	*retval = 0;
@@ -5342,11 +6643,11 @@ fremovexattr(struct proc *p, struct fremovexattr_args *uap, int *retval)
  */
 #warning "code duplication"
 int
-listxattr(struct proc *p, struct listxattr_args *uap, user_ssize_t *retval)
+listxattr(proc_t p, struct listxattr_args *uap, user_ssize_t *retval)
 {
-	struct vnode *vp;
+	vnode_t vp;
 	struct nameidata nd;
-	struct vfs_context context;
+	vfs_context_t ctx = vfs_context_current();
 	uio_t auio = NULL;
 	int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t attrsize = 0;
@@ -5354,14 +6655,11 @@ listxattr(struct proc *p, struct listxattr_args *uap, user_ssize_t *retval)
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	if (uap->options & XATTR_NOSECURITY)
+	if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
-	nameiflags = (uap->options & XATTR_NOFOLLOW) ? 0 : FOLLOW;
-	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, &context);
+	nameiflags = ((uap->options & XATTR_NOFOLLOW) ? 0 : FOLLOW) | NOTRIGGER;
+	NDINIT(&nd, LOOKUP, nameiflags, spacetype, uap->path, ctx);
 	if ((error = namei(&nd))) {
 		return (error);
 	}
@@ -5374,7 +6672,7 @@ listxattr(struct proc *p, struct listxattr_args *uap, user_ssize_t *retval)
 		uio_addiov(auio, uap->namebuf, uap->bufsize);
 	}
 
-	error = vn_listxattr(vp, auio, &attrsize, uap->options, &context);
+	error = vn_listxattr(vp, auio, &attrsize, uap->options, ctx);
 
 	vnode_put(vp);
 	if (auio) {
@@ -5390,17 +6688,16 @@ listxattr(struct proc *p, struct listxattr_args *uap, user_ssize_t *retval)
  */
 #warning "code duplication"
 int
-flistxattr(struct proc *p, struct flistxattr_args *uap, user_ssize_t *retval)
+flistxattr(proc_t p, struct flistxattr_args *uap, user_ssize_t *retval)
 {
-	struct vnode *vp;
-	struct vfs_context context;
+	vnode_t vp;
 	uio_t auio = NULL;
 	int spacetype = proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
 	size_t attrsize = 0;
 	int error;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY))
+	if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT))
 		return (EINVAL);
 
 	if ( (error = file_vnode(uap->fd, &vp)) ) {
@@ -5416,10 +6713,8 @@ flistxattr(struct proc *p, struct flistxattr_args *uap, user_ssize_t *retval)
 								  	  UIO_READ, &uio_buf[0], sizeof(uio_buf));
 		uio_addiov(auio, uap->namebuf, uap->bufsize);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
 
-	error = vn_listxattr(vp, auio, &attrsize, uap->options, &context);
+	error = vn_listxattr(vp, auio, &attrsize, uap->options, vfs_context_current());
 
 	vnode_put(vp);
 	file_drop(uap->fd);
@@ -5434,6 +6729,9 @@ flistxattr(struct proc *p, struct flistxattr_args *uap, user_ssize_t *retval)
 /*
  * Common routine to handle various flavors of statfs data heading out
  *	to user space.
+ *
+ * Returns:	0			Success
+ *		EFAULT
  */
 static int
 munge_statfs(struct mount *mp, struct vfsstatfs *sfsp, 
@@ -5459,9 +6757,9 @@ munge_statfs(struct mount *mp, struct vfsstatfs *sfsp,
 		sfs.f_ffree = (user_long_t)sfsp->f_ffree;
 		sfs.f_fsid = sfsp->f_fsid;
 		sfs.f_owner = sfsp->f_owner;
-		strncpy(&sfs.f_fstypename[0], &sfsp->f_fstypename[0], MFSNAMELEN-1);
-		strncpy(&sfs.f_mntonname[0], &sfsp->f_mntonname[0], MNAMELEN-1);
-		strncpy(&sfs.f_mntfromname[0], &sfsp->f_mntfromname[0], MNAMELEN-1);
+		strlcpy(&sfs.f_fstypename[0], &sfsp->f_fstypename[0], MFSNAMELEN);
+		strlcpy(&sfs.f_mntonname[0], &sfsp->f_mntonname[0], MNAMELEN);
+		strlcpy(&sfs.f_mntfromname[0], &sfsp->f_mntfromname[0], MNAMELEN);
 
 		if (partial_copy) {
 			copy_size -= (sizeof(sfs.f_reserved3) + sizeof(sfs.f_reserved4));
@@ -5492,9 +6790,9 @@ munge_statfs(struct mount *mp, struct vfsstatfs *sfsp,
 			 * disk as they look huge. This change should not affect
 			 * XSAN as they should not setting these to -1..
 			 */
-			 && (sfsp->f_blocks != 0xffffffffffffffff)
-			 && (sfsp->f_bfree != 0xffffffffffffffff)
-			 && (sfsp->f_bavail != 0xffffffffffffffff)) {
+			 && (sfsp->f_blocks != 0xffffffffffffffffULL)
+			 && (sfsp->f_bfree != 0xffffffffffffffffULL)
+			 && (sfsp->f_bavail != 0xffffffffffffffffULL)) {
 			int		shift;
 
 			/*
@@ -5531,9 +6829,9 @@ munge_statfs(struct mount *mp, struct vfsstatfs *sfsp,
 		sfs.f_ffree = (long)sfsp->f_ffree;
 		sfs.f_fsid = sfsp->f_fsid;
 		sfs.f_owner = sfsp->f_owner;
-		strncpy(&sfs.f_fstypename[0], &sfsp->f_fstypename[0], MFSNAMELEN-1);
-		strncpy(&sfs.f_mntonname[0], &sfsp->f_mntonname[0], MNAMELEN-1);
-		strncpy(&sfs.f_mntfromname[0], &sfsp->f_mntfromname[0], MNAMELEN-1);
+		strlcpy(&sfs.f_fstypename[0], &sfsp->f_fstypename[0], MFSNAMELEN);
+		strlcpy(&sfs.f_mntonname[0], &sfsp->f_mntonname[0], MNAMELEN);
+		strlcpy(&sfs.f_mntfromname[0], &sfsp->f_mntfromname[0], MNAMELEN);
 
 		if (partial_copy) {
 			copy_size -= (sizeof(sfs.f_reserved3) + sizeof(sfs.f_reserved4));
@@ -5561,13 +6859,54 @@ void munge_stat(struct stat *sbp, struct user_stat *usbp)
 	usbp->st_uid = sbp->st_uid;
 	usbp->st_gid = sbp->st_gid;
 	usbp->st_rdev = sbp->st_rdev;
-#ifndef _POSIX_SOURCE
+#ifndef _POSIX_C_SOURCE
+	usbp->st_atimespec.tv_sec = sbp->st_atimespec.tv_sec;
+	usbp->st_atimespec.tv_nsec = sbp->st_atimespec.tv_nsec;
+	usbp->st_mtimespec.tv_sec = sbp->st_mtimespec.tv_sec;
+	usbp->st_mtimespec.tv_nsec = sbp->st_mtimespec.tv_nsec;
+	usbp->st_ctimespec.tv_sec = sbp->st_ctimespec.tv_sec;
+	usbp->st_ctimespec.tv_nsec = sbp->st_ctimespec.tv_nsec;
+#else
+	usbp->st_atime = sbp->st_atime;
+	usbp->st_atimensec = sbp->st_atimensec;
+	usbp->st_mtime = sbp->st_mtime;
+	usbp->st_mtimensec = sbp->st_mtimensec;
+	usbp->st_ctime = sbp->st_ctime;
+	usbp->st_ctimensec = sbp->st_ctimensec;
+#endif
+	usbp->st_size = sbp->st_size;
+	usbp->st_blocks = sbp->st_blocks;
+	usbp->st_blksize = sbp->st_blksize;
+	usbp->st_flags = sbp->st_flags;
+	usbp->st_gen = sbp->st_gen;
+	usbp->st_lspare = sbp->st_lspare;
+	usbp->st_qspare[0] = sbp->st_qspare[0];
+	usbp->st_qspare[1] = sbp->st_qspare[1];
+}
+
+/*
+ * copy stat64 structure into user_stat64 structure.
+ */
+void munge_stat64(struct stat64 *sbp, struct user_stat64 *usbp)
+{
+        bzero(usbp, sizeof(struct user_stat));
+
+	usbp->st_dev = sbp->st_dev;
+	usbp->st_ino = sbp->st_ino;
+	usbp->st_mode = sbp->st_mode;
+	usbp->st_nlink = sbp->st_nlink;
+	usbp->st_uid = sbp->st_uid;
+	usbp->st_gid = sbp->st_gid;
+	usbp->st_rdev = sbp->st_rdev;
+#ifndef _POSIX_C_SOURCE
 	usbp->st_atimespec.tv_sec = sbp->st_atimespec.tv_sec;
 	usbp->st_atimespec.tv_nsec = sbp->st_atimespec.tv_nsec;
 	usbp->st_mtimespec.tv_sec = sbp->st_mtimespec.tv_sec;
 	usbp->st_mtimespec.tv_nsec = sbp->st_mtimespec.tv_nsec;
 	usbp->st_ctimespec.tv_sec = sbp->st_ctimespec.tv_sec;
 	usbp->st_ctimespec.tv_nsec = sbp->st_ctimespec.tv_nsec;
+	usbp->st_birthtimespec.tv_sec = sbp->st_birthtimespec.tv_sec;
+	usbp->st_birthtimespec.tv_nsec = sbp->st_birthtimespec.tv_nsec;
 #else
 	usbp->st_atime = sbp->st_atime;
 	usbp->st_atimensec = sbp->st_atimensec;
@@ -5575,6 +6914,8 @@ void munge_stat(struct stat *sbp, struct user_stat *usbp)
 	usbp->st_mtimensec = sbp->st_mtimensec;
 	usbp->st_ctime = sbp->st_ctime;
 	usbp->st_ctimensec = sbp->st_ctimensec;
+	usbp->st_birthtime = sbp->st_birthtime;
+	usbp->st_birthtimensec = sbp->st_birthtimensec;
 #endif
 	usbp->st_size = sbp->st_size;
 	usbp->st_blocks = sbp->st_blocks;
diff --git a/bsd/vfs/vfs_utfconv.c b/bsd/vfs/vfs_utfconv.c
index 5cef2a453..948076da9 100644
--- a/bsd/vfs/vfs_utfconv.c
+++ b/bsd/vfs/vfs_utfconv.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
  /*
@@ -27,6 +33,7 @@
 #include <sys/param.h>
 #include <sys/utfconv.h>
 #include <sys/errno.h>
+#include <sys/malloc.h>
 #include <libkern/OSByteOrder.h>
 
 /*
@@ -143,44 +150,80 @@ static u_int16_t unicode_combine(u_int16_t base, u_int16_t combining);
 
 static void priortysort(u_int16_t* characters, int count);
 
+static u_int16_t  ucs_to_sfm(u_int16_t ucs_ch, int lastchar);
+
+static u_int16_t  sfm_to_ucs(u_int16_t ucs_ch);
+
+
 char utf_extrabytes[32] = {
 	 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
 	-1, -1, -1, -1, -1, -1, -1, -1,  1,  1,  1,  1,  2,  2,  3, -1
 };
 
+const char hexdigits[16] = {
+	 '0',  '1',  '2',  '3',  '4',  '5',  '6', '7',
+	 '8',  '9',  'A',  'B',  'C',  'D',  'E', 'F'
+};
 
 /*
- * utf8_encodelen - Calculates the UTF-8 encoding length for a Unicode filename
+ * utf8_encodelen - Calculate the UTF-8 encoding length
  *
- * NOTES:
- *    If '/' chars are allowed on disk then an alternate
- *    (replacement) char must be provided in altslash.
+ * This function takes a Unicode input string, ucsp, of ucslen bytes
+ * and calculates the size of the UTF-8 output in bytes (not including
+ * a NULL termination byte). The string must reside in kernel memory.
  *
- * input flags:
- *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
+ * If '/' chars are possible in the Unicode input then an alternate
+ * (replacement) char should be provided in altslash.
+ *
+ * FLAGS
+ *    UTF_REVERSE_ENDIAN:  Unicode byte order is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_DECOMPOSED:  generate fully decomposed output
+ *
+ *    UTF_PRECOMPOSED is ignored since utf8_encodestr doesn't support it
+ *
+ * ERRORS
+ *    None
  */
 size_t
-utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
-               int flags)
+utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash, int flags)
 {
 	u_int16_t ucs_ch;
+	u_int16_t * chp = NULL;
+	u_int16_t sequence[8];
+	int extra = 0;
 	int charcnt;
 	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
+	int decompose = (flags & UTF_DECOMPOSED);
 	size_t len;
-	
+
 	charcnt = ucslen / 2;
 	len = 0;
 
 	while (charcnt-- > 0) {
-		ucs_ch = *ucsp++;
-
-		if (swapbytes)
-			ucs_ch = OSSwapInt16(ucs_ch);
-		if (ucs_ch == '/')
-			ucs_ch = altslash ? altslash : '_';
-		else if (ucs_ch == '\0')
-			ucs_ch = UCS_ALT_NULL;
-		
+		if (extra > 0) {
+			--extra;
+			ucs_ch = *chp++;
+		} else {
+			ucs_ch = *ucsp++;
+			if (swapbytes) {
+				ucs_ch = OSSwapInt16(ucs_ch);
+			}
+			if (ucs_ch == '/') {
+				ucs_ch = altslash ? altslash : '_';
+			} else if (ucs_ch == '\0') {
+				ucs_ch = UCS_ALT_NULL;
+			} else if (decompose && unicode_decomposeable(ucs_ch)) {
+				extra = unicode_decompose(ucs_ch, sequence) - 1;
+				charcnt += extra;
+				ucs_ch = sequence[0];
+				chp = &sequence[1];
+			}
+		}
 		len += UNICODE_TO_UTF8_LEN(ucs_ch);
 	}
 
@@ -199,10 +242,18 @@ utf8_encodelen(const u_int16_t * ucsp, size_t ucslen, u_int16_t altslash,
  *
  * input flags:
  *    UTF_REVERSE_ENDIAN: Unicode byteorder is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_DECOMPOSED:  generate fully decomposed output
+ *
  *    UTF_NO_NULL_TERM:  don't add NULL termination to UTF-8 output
  *
  * result:
  *    ENAMETOOLONG: Name didn't fit; only buflen bytes were encoded
+ *
  *    EINVAL: Illegal char found; char was replaced by an '_'.
  */
 int
@@ -219,8 +270,9 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
 	int swapbytes = (flags & UTF_REVERSE_ENDIAN);
 	int nullterm  = ((flags & UTF_NO_NULL_TERM) == 0);
 	int decompose = (flags & UTF_DECOMPOSED);
+	int sfmconv = (flags & UTF_SFM_CONVERSIONS);
 	int result = 0;
-	
+
 	bufstart = utf8p;
 	bufend = bufstart + buflen;
 	if (nullterm)
@@ -270,6 +322,12 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
 			*utf8p++ = 0x80 | (0x3f & ucs_ch);
 
 		} else {
+			/* These chars never valid Unicode. */
+			if (ucs_ch == 0xFFFE || ucs_ch == 0xFFFF) {
+				result = EINVAL;
+				break;
+			}
+
 			/* Combine valid surrogate pairs */
 			if (ucs_ch >= SP_HIGH_FIRST && ucs_ch <= SP_HIGH_LAST
 				&& charcnt > 0) {
@@ -292,6 +350,16 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
 					*utf8p++ = 0x80 | (0x3f & pair);
 					continue;
 				}
+			} else if (sfmconv) {
+				ucs_ch = sfm_to_ucs(ucs_ch);
+				if (ucs_ch < 0x0080) {
+					if (utf8p >= bufend) {
+						result = ENAMETOOLONG;
+						break;
+					}			
+					*utf8p++ = ucs_ch;
+					continue;
+				}
 			}
 			if ((utf8p + 2) >= bufend) {
 				result = ENAMETOOLONG;
@@ -322,11 +390,21 @@ utf8_encodestr(const u_int16_t * ucsp, size_t ucslen, u_int8_t * utf8p,
  *    (replacement) char must be provided in altslash.
  *
  * input flags:
- *    UTF_REV_ENDIAN:   Unicode byteorder is oposite current runtime
- *    UTF_DECOMPOSED:   Unicode output string must be fully decompsed
+ *    UTF_REV_ENDIAN:  Unicode byte order is opposite current runtime
+ *
+ *    UTF_BIG_ENDIAN:  Unicode byte order is always big endian
+ *
+ *    UTF_LITTLE_ENDIAN:  Unicode byte order is always little endian
+ *
+ *    UTF_DECOMPOSED:  generate fully decomposed output (NFD)
+ *
+ *    UTF_PRECOMPOSED:  generate precomposed output (NFC)
+ *
+ *    UTF_ESCAPE_ILLEGAL:  percent escape any illegal UTF-8 input
  *
  * result:
  *    ENAMETOOLONG: Name didn't fit; only ucslen chars were decoded.
+ *
  *    EINVAL: Illegal UTF-8 sequence found.
  */
 int
@@ -339,11 +417,15 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 	unsigned int byte;
 	int combcharcnt = 0;
 	int result = 0;
-	int decompose, precompose, swapbytes;
+	int decompose, precompose, swapbytes, escaping;
+	int sfmconv;
+	int extrabytes;
 
-	decompose =  (flags & UTF_DECOMPOSED);
+	decompose  = (flags & UTF_DECOMPOSED);
 	precompose = (flags & UTF_PRECOMPOSED);
-	swapbytes =  (flags & UTF_REVERSE_ENDIAN);
+	swapbytes  = (flags & UTF_REVERSE_ENDIAN);
+	escaping   = (flags & UTF_ESCAPE_ILLEGAL);
+	sfmconv    = (flags & UTF_SFM_CONVERSIONS);
 
 	bufstart = ucsp;
 	bufend = (u_int16_t *)((u_int8_t *)ucsp + buflen);
@@ -354,13 +436,14 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 
 		/* check for ascii */
 		if (byte < 0x80) {
-			ucs_ch = byte;                 /* 1st byte */
+			ucs_ch = sfmconv ? ucs_to_sfm(byte, utf8len == 0) : byte;
 		} else {
 			u_int32_t ch;
-			int extrabytes = utf_extrabytes[byte >> 3];
 
-			if (utf8len < extrabytes)
-				goto invalid;
+			extrabytes = utf_extrabytes[byte >> 3];
+			if ((extrabytes < 0) || ((int)utf8len < extrabytes)) {
+				goto escape;
+			}
 			utf8len -= extrabytes;
 
 			switch (extrabytes) {
@@ -368,31 +451,31 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 				ch = byte; ch <<= 6;   /* 1st byte */
 				byte = *utf8p++;       /* 2nd byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape2;
 				ch += byte;
 				ch -= 0x00003080UL;
 				if (ch < 0x0080)
-					goto invalid;
+					goto escape2;
 				ucs_ch = ch;
 			        break;
 			case 2:
 				ch = byte; ch <<= 6;   /* 1st byte */
 				byte = *utf8p++;       /* 2nd byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape2;
 				ch += byte; ch <<= 6;
 				byte = *utf8p++;       /* 3rd byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape3;
 				ch += byte;
 				ch -= 0x000E2080UL;
 				if (ch < 0x0800)
-					goto invalid;
+					goto escape3;
 				if (ch >= 0xD800) {
 					if (ch <= 0xDFFF)
-						goto invalid;
+						goto escape3;
 					if (ch == 0xFFFE || ch == 0xFFFF)
-						goto invalid;
+						goto escape3;
 				}
 				ucs_ch = ch;
 				break;
@@ -400,49 +483,52 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 				ch = byte; ch <<= 6;   /* 1st byte */
 				byte = *utf8p++;       /* 2nd byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape2;
 				ch += byte; ch <<= 6;
 				byte = *utf8p++;       /* 3rd byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape3;
 				ch += byte; ch <<= 6;
 				byte = *utf8p++;       /* 4th byte */
 				if ((byte >> 6) != 2)
-					goto invalid;
+					goto escape4;
 			        ch += byte;
 				ch -= 0x03C82080UL + SP_HALF_BASE;
 				ucs_ch = (ch >> SP_HALF_SHIFT) + SP_HIGH_FIRST;
 				if (ucs_ch < SP_HIGH_FIRST || ucs_ch > SP_HIGH_LAST)
-					goto invalid;
-				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
+					goto escape4;
+				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
 				if (ucsp >= bufend)
 					goto toolong;
 				ucs_ch = (ch & SP_HALF_MASK) + SP_LOW_FIRST;
-				if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST)
-					goto invalid;
-				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
+				if (ucs_ch < SP_LOW_FIRST || ucs_ch > SP_LOW_LAST) {
+					--ucsp;
+					goto escape4;
+				}
+				*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
 			        continue;
 			default:
-				goto invalid;
+				result = EINVAL;
+				goto exit;
 			}
 			if (decompose) {
 				if (unicode_decomposeable(ucs_ch)) {
 					u_int16_t sequence[8];
 					int count, i;
-					
-					/* Before decomposing a new unicode character, sort
-					 * previous combining characters, if any, and reset 
-					 * the counter
+
+					/* Before decomposing a new unicode character, sort 
+					 * previous combining characters, if any, and reset
+					 * the counter.
 					 */
-					if (combcharcnt > 1){
+					if (combcharcnt > 1) {
 						priortysort(ucsp - combcharcnt, combcharcnt);
 					}
 					combcharcnt = 0;
-					count = unicode_decompose(ucs_ch, sequence);
 
+					count = unicode_decompose(ucs_ch, sequence);
 					for (i = 0; i < count; ++i) {
 						ucs_ch = sequence[i];
-						*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
+						*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
 						if (ucsp >= bufend)
 							goto toolong;
 					}
@@ -478,7 +564,39 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 			}
 			combcharcnt = 0;  /* start over */
 		}
-		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : ucs_ch;
+
+		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
+		continue;
+
+		/* 
+		 * Escape illegal UTF-8 into something legal.
+		 */
+escape4:
+		utf8p -= 3;
+		goto escape;
+escape3:
+		utf8p -= 2;
+		goto escape;
+escape2:
+		utf8p -= 1;
+escape:
+		if (!escaping) {
+			result = EINVAL;
+			goto exit;
+		}
+		if (extrabytes > 0)
+			utf8len += extrabytes;
+		byte = *(utf8p - 1);
+
+		if ((ucsp + 2) >= bufend)
+			goto toolong;
+
+		ucs_ch = '%';
+		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
+		ucs_ch =  hexdigits[byte >> 4];
+		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
+		ucs_ch =  hexdigits[byte & 0x0F];
+		*ucsp++ = swapbytes ? OSSwapInt16(ucs_ch) : (u_int16_t)ucs_ch;
 	}
 	/*
 	 * Make a previous combining sequence canonical
@@ -486,16 +604,11 @@ utf8_decodestr(const u_int8_t* utf8p, size_t utf8len, u_int16_t* ucsp,
 	if (combcharcnt > 1) {
 		priortysort(ucsp - combcharcnt, combcharcnt);
 	}
-
 exit:
 	*ucslen = (u_int8_t*)ucsp - (u_int8_t*)bufstart;
 
 	return (result);
 
-invalid:
-	result = EINVAL;
-	goto exit;
-
 toolong:
 	result = ENAMETOOLONG;
 	goto exit;
@@ -586,6 +699,122 @@ invalid:
 	return (EINVAL);
 }
 
+/*
+ * utf8_normalizestr - Normalize a UTF-8 string (NFC or NFD)
+ *
+ * This function takes an UTF-8 input string, instr, of inlen bytes
+ * and produces normalized UTF-8 output into a buffer of buflen bytes
+ * pointed to by outstr. The size of the output in bytes (not including
+ * a NULL termination byte) is returned in outlen. In-place conversions
+ * are not supported (i.e. instr != outstr).]
+ 
+ * FLAGS
+ *    UTF_DECOMPOSED:  output string will be fully decomposed (NFD)
+ *
+ *    UTF_PRECOMPOSED:  output string will be precomposed (NFC)
+ *
+ *    UTF_NO_NULL_TERM:  do not add null termination to output string
+ *
+ *    UTF_ESCAPE_ILLEGAL:  percent escape any illegal UTF-8 input
+ *
+ * ERRORS
+ *    ENAMETOOLONG:  output did not fit or input exceeded MAXPATHLEN bytes
+ *
+ *    EINVAL:  illegal UTF-8 sequence encountered or invalid flags
+ */
+int
+utf8_normalizestr(const u_int8_t* instr, size_t inlen, u_int8_t* outstr,
+                  size_t *outlen, size_t buflen, int flags)
+{
+	u_int16_t unicodebuf[32];
+	u_int16_t* unistr = NULL;
+	size_t unicode_bytes;
+	size_t uft8_bytes;
+	size_t inbuflen;
+	u_int8_t *outbufstart, *outbufend;
+	const u_int8_t *inbufstart;
+	unsigned int byte;
+	int decompose, precompose;
+	int result = 0;
+
+	if (flags & ~(UTF_DECOMPOSED | UTF_PRECOMPOSED | UTF_NO_NULL_TERM | UTF_ESCAPE_ILLEGAL)) {
+		return (EINVAL);
+	}
+	decompose = (flags & UTF_DECOMPOSED);
+	precompose = (flags & UTF_PRECOMPOSED);
+	if ((decompose && precompose) || (!decompose && !precompose)) {
+		return (EINVAL);
+	}
+	outbufstart = outstr;
+	outbufend = outbufstart + buflen;
+	inbufstart = instr;
+	inbuflen = inlen;
+
+	while (inlen-- > 0 && (byte = *instr++) != '\0') {
+		if (outstr >= outbufend) {
+			result = ENAMETOOLONG;
+			goto exit;
+		}
+		if (byte >= 0x80) {
+			goto nonASCII;
+		}
+		/* ASCII is already normalized. */
+		*outstr++ = byte;
+	}
+exit:
+	*outlen = outstr - outbufstart;
+	if (((flags & UTF_NO_NULL_TERM) == 0)) {
+		if (outstr < outbufend)
+			*outstr++ = '\0';
+		else
+			result = ENAMETOOLONG;
+	}
+	return (result);
+
+
+	/* 
+	 * Non-ASCII uses the existing utf8_encodestr/utf8_decodestr
+	 * functions to perform the normalization.  Since this will
+	 * presumably be used to normalize filenames in the back-end
+	 * (on disk or over-the-wire), it should be fast enough.
+	 */
+nonASCII:
+
+	/* Make sure the input size is reasonable. */
+	if (inbuflen > MAXPATHLEN) {
+		result = ENAMETOOLONG;
+		goto exit;
+	}
+	/*
+	 * Compute worst case Unicode buffer size.
+	 *
+	 * For pre-composed output, every UTF-8 input byte will be at
+	 * most 2 Unicode bytes.  For decomposed output, 2 UTF-8 bytes
+	 * (smallest composite char sequence) may yield 6 Unicode bytes
+	 * (1 base char + 2 combining chars).
+	 */
+	unicode_bytes = precompose ? (inbuflen * 2) : (inbuflen * 3);
+
+	if (unicode_bytes <= sizeof(unicodebuf))
+		unistr = &unicodebuf[0];
+	else
+		MALLOC(unistr, u_int16_t *, unicode_bytes, M_TEMP, M_WAITOK);
+
+	/* Normalize the string. */
+	result = utf8_decodestr(inbufstart, inbuflen, unistr, &unicode_bytes,
+	                        unicode_bytes, 0, flags & ~UTF_NO_NULL_TERM);
+	if (result == 0) {
+		/* Put results back into UTF-8. */
+		result = utf8_encodestr(unistr, unicode_bytes, outbufstart,
+		                        &uft8_bytes, buflen, 0, UTF_NO_NULL_TERM);
+		outstr = outbufstart + uft8_bytes;
+	}
+	if (unistr && unistr != &unicodebuf[0]) {
+		FREE(unistr, M_TEMP);
+	}
+	goto exit;
+}
+
 
  /*
   * Unicode 3.2 decomposition code (derived from Core Foundation)
@@ -771,7 +1000,7 @@ unicode_combine(u_int16_t base, u_int16_t combining)
 	if (value) {
 		value = getmappedvalue16(
 			(const unicode_mappings16 *)
-			((u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)),
+			((const u_int32_t *)__CFUniCharBMPPrecompDestinationTable + (value & 0xFFFF)),
 			(value >> 16), base);
 	}
 	return (value);
@@ -813,3 +1042,108 @@ priortysort(u_int16_t* characters, int count)
 		}
 	} while (changes);
 }
+
+
+/*
+ * Invalid NTFS filename characters are encodeded using the
+ * SFM (Services for Macintosh) private use Unicode characters.
+ *
+ * These should only be used for SMB, MSDOS or NTFS.
+ *
+ *    Illegal NTFS Char   SFM Unicode Char
+ *  ----------------------------------------
+ *    0x01-0x1f           0xf001-0xf01f
+ *    '"'                 0xf020
+ *    '*'                 0xf021
+ *    '/'                 0xf022
+ *    '<'                 0xf023
+ *    '>'                 0xf024
+ *    '?'                 0xf025
+ *    '\'                 0xf026
+ *    '|'                 0xf027
+ *    ' '                 0xf028  (Only if last char of the name)
+ *    '.'                 0xf029  (Only if last char of the name)
+ *  ----------------------------------------
+ *
+ *  Reference: http://support.microsoft.com/kb/q117258/
+ */
+
+#define MAX_SFM2MAC           0x29
+#define SFMCODE_PREFIX_MASK   0xf000 
+
+/*
+ * In the Mac OS 9 days the colon was illegal in a file name. For that reason
+ * SFM had no conversion for the colon. There is a conversion for the
+ * slash. In Mac OS X the slash is illegal in a file name. So for us the colon
+ * is a slash and a slash is a colon. So we can just replace the slash with the
+ * colon in our tables and everything will just work. 
+ */
+static u_int8_t
+sfm2mac[42] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,   /* 00 - 07 */
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,   /* 08 - 0F */
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,   /* 10 - 17 */
+	0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,   /* 18 - 1F */
+	0x22, 0x2a, 0x3a, 0x3c, 0x3e, 0x3f, 0x5c, 0x7c,   /* 20 - 27 */
+	0x20, 0x2e                                        /* 28 - 29 */
+};
+
+static u_int8_t
+mac2sfm[112] = {
+	0x20, 0x21, 0x20, 0x23, 0x24, 0x25, 0x26, 0x27,	  /* 20 - 27 */
+	0x28, 0x29, 0x21, 0x2b, 0x2c, 0x2d, 0x2e, 0x22,   /* 28 - 2f */
+	0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,   /* 30 - 37 */
+	0x38, 0x39, 0x22, 0x3b, 0x23, 0x3d, 0x24, 0x25,   /* 38 - 3f */
+	0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,   /* 40 - 47 */
+	0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,   /* 48 - 4f */
+	0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,   /* 50 - 57 */
+	0x58, 0x59, 0x5a, 0x5b, 0x26, 0x5d, 0x5e, 0x5f,   /* 58 - 5f */
+	0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,   /* 60 - 67 */
+	0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,   /* 68 - 6f */
+	0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,   /* 70 - 77 */
+	0x78, 0x79, 0x7a, 0x7b, 0x27, 0x7d, 0x7e, 0x7f    /* 78 - 7f */
+};
+
+
+/*
+ * Encode illegal NTFS filename characters into SFM Private Unicode characters
+ *
+ * Assumes non-zero ASCII input.
+ */
+static u_int16_t
+ucs_to_sfm(u_int16_t ucs_ch, int lastchar)
+{
+	/* The last character of filename cannot be a space or period. */
+	if (lastchar) {
+		if (ucs_ch == 0x20)
+			return (0xf028);
+		else if (ucs_ch == 0x2e)
+			return (0xf029);
+	}
+	/* 0x01 - 0x1f is simple transformation. */
+	if (ucs_ch <= 0x1f) {
+		return (ucs_ch | 0xf000);
+	} else /* 0x20 - 0x7f */ {
+		u_int16_t lsb;
+
+		lsb = mac2sfm[ucs_ch - 0x0020];
+		if (lsb != ucs_ch)
+			return(0xf000 | lsb); 
+	}
+	return (ucs_ch);
+}
+
+/*
+ * Decode any SFM Private Unicode characters
+ */
+static u_int16_t
+sfm_to_ucs(u_int16_t ucs_ch)
+{
+	if (((ucs_ch & 0xffC0) == SFMCODE_PREFIX_MASK) && 
+	    ((ucs_ch & 0x003f) <= MAX_SFM2MAC)) {
+		ucs_ch = sfm2mac[ucs_ch & 0x003f];
+	}
+	return (ucs_ch);
+}
+
+
diff --git a/bsd/vfs/vfs_utfconvdata.h b/bsd/vfs/vfs_utfconvdata.h
index ec64223d9..589bd650e 100644
--- a/bsd/vfs/vfs_utfconvdata.h
+++ b/bsd/vfs/vfs_utfconvdata.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 /*
diff --git a/bsd/vfs/vfs_vnops.c b/bsd/vfs/vfs_vnops.c
index d9d2b947e..2d9ced958 100644
--- a/bsd/vfs/vfs_vnops.c
+++ b/bsd/vfs/vfs_vnops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
 /*
@@ -60,6 +66,12 @@
  *	@(#)vfs_vnops.c	8.14 (Berkeley) 6/15/95
  *
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -74,7 +86,11 @@
 #include <sys/vnode_internal.h>
 #include <sys/ioctl.h>
 #include <sys/tty.h>
+/* Temporary workaround for ubc.h until <rdar://4714366 is resolved */
+#define ubc_setcred ubc_setcred_deprecated
 #include <sys/ubc.h>
+#undef ubc_setcred
+int	ubc_setcred(struct vnode *, struct proc *);
 #include <sys/conf.h>
 #include <sys/disk.h>
 #include <sys/fsevents.h>
@@ -83,28 +99,36 @@
 #include <sys/ubc_internal.h>
 #include <sys/uio_internal.h>
 #include <sys/resourcevar.h>
+#include <sys/signalvar.h>
 
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
 
 #include <miscfs/specfs/specdev.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 
-static int vn_closefile(struct fileglob *fp, struct proc *p);
-static int vn_ioctl(struct fileproc *fp, u_long com, caddr_t data, struct proc *p);
-static int vn_read(struct fileproc *fp, struct uio *uio,
-		kauth_cred_t cred, int flags, struct proc *p);
-static int vn_write(struct fileproc *fp, struct uio *uio,
-		kauth_cred_t cred, int flags, struct proc *p);
-static int vn_select( struct fileproc *fp, int which, void * wql, struct proc *p);
-static int vn_kqfilt_add(struct fileproc *fp, struct knote *kn, struct proc *p);
+static int vn_closefile(struct fileglob *fp, vfs_context_t ctx);
+static int vn_ioctl(struct fileproc *fp, u_long com, caddr_t data,
+			vfs_context_t ctx);
+static int vn_read(struct fileproc *fp, struct uio *uio, int flags,
+			vfs_context_t ctx);
+static int vn_write(struct fileproc *fp, struct uio *uio, int flags,
+			vfs_context_t ctx);
+static int vn_select( struct fileproc *fp, int which, void * wql,
+			vfs_context_t ctx);
+static int vn_kqfilt_add(struct fileproc *fp, struct knote *kn,
+			vfs_context_t ctx);
 #if 0
-static int vn_kqfilt_remove(struct vnode *vp, uintptr_t ident, struct proc *p);
+static int vn_kqfilt_remove(struct vnode *vp, uintptr_t ident,
+			vfs_context_t ctx);
 #endif
 
 struct 	fileops vnops =
-	{ vn_read, vn_write, vn_ioctl, vn_select, vn_closefile, vn_kqfilt_add, 0 };
+	{ vn_read, vn_write, vn_ioctl, vn_select, vn_closefile, vn_kqfilt_add, NULL };
 
 /*
  * Common code for vnode open operations.
@@ -194,9 +218,14 @@ again:
 			goto out;
 		}
 		ndp->ni_cnd.cn_nameiop = CREATE;
-		ndp->ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF | AUDITVNPATH1;
-
-		if ((fmode & O_EXCL) == 0)
+		/* Inherit USEDVP flag only */
+		ndp->ni_cnd.cn_flags &= USEDVP;
+		ndp->ni_cnd.cn_flags |= LOCKPARENT | LOCKLEAF | AUDITVNPATH1;
+#if NAMEDRSRCFORK
+		/* open calls are allowed for resource forks. */
+		ndp->ni_cnd.cn_flags |= CN_ALLOWRSRCFORK;
+#endif
+		if ((fmode & O_EXCL) == 0 && (fmode & O_NOFOLLOW) == 0)
 			ndp->ni_cnd.cn_flags |= FOLLOW;
 		if ( (error = namei(ndp)) )
 			goto out;
@@ -211,14 +240,26 @@ again:
 				goto badcreate;
  			}
 
+			VATTR_SET(vap, va_type, VREG);
+#if CONFIG_MACF
+			error = mac_vnode_check_create(ctx,
+			    dvp, &ndp->ni_cnd, vap);
+			if (error)
+				goto badcreate;
+#endif /* MAC */
+
 			/* authorize before creating */
  			if ((error = vnode_authorize(dvp, NULL, KAUTH_VNODE_ADD_FILE, ctx)) != 0)
 				goto badcreate;
 
-			VATTR_SET(vap, va_type, VREG);
 			if (fmode & O_EXCL)
 				vap->va_vaflags |= VA_EXCLUSIVE;
-
+#if NAMEDRSRCFORK
+			if (ndp->ni_cnd.cn_flags & CN_WANTSRSRCFORK) {
+				if ((error = vnode_makenamedstream(dvp, &ndp->ni_vp, XATTR_RESOURCEFORK_NAME, 0, ctx)) != 0)
+					goto badcreate;
+			} else
+#endif
 			if ((error = vn_create(dvp, &ndp->ni_vp, &ndp->ni_cnd, vap, 0, ctx)) != 0)
 				goto badcreate;
 			
@@ -236,11 +277,14 @@ again:
 				if (update_flags)
 				        vnode_update_identity(vp, dvp, ndp->ni_cnd.cn_nameptr, ndp->ni_cnd.cn_namelen, ndp->ni_cnd.cn_hash, update_flags);
 
+#if CONFIG_FSE
 				if (need_fsevent(FSE_CREATE_FILE, vp)) {
 				        add_fsevent(FSE_CREATE_FILE, ctx,
 						    FSE_ARG_VNODE, vp,
 						    FSE_ARG_DONE);
 				}
+#endif
+
 			}
 			/*
 			 * nameidone has to happen before we vnode_put(dvp)
@@ -275,7 +319,17 @@ badcreate:
 		}
 	} else {
 		ndp->ni_cnd.cn_nameiop = LOOKUP;
-		ndp->ni_cnd.cn_flags = FOLLOW | LOCKLEAF | AUDITVNPATH1;
+		/* Inherit USEDVP flag only */
+		ndp->ni_cnd.cn_flags &= USEDVP;
+		ndp->ni_cnd.cn_flags |= FOLLOW | LOCKLEAF | AUDITVNPATH1;
+#if NAMEDRSRCFORK
+		/* open calls are allowed for resource forks. */
+		ndp->ni_cnd.cn_flags |= CN_ALLOWRSRCFORK;
+#endif
+		if (fmode & O_NOFOLLOW || fmode & O_SYMLINK) {
+		    ndp->ni_cnd.cn_flags &= ~FOLLOW;
+		}
+
 		if ( (error = namei(ndp)) )
 			goto out;
 		vp = ndp->ni_vp;
@@ -287,15 +341,16 @@ badcreate:
 			goto bad;
 		}
 	}
+
 	if (vp->v_type == VSOCK && vp->v_tag != VT_FDESC) {
 		error = EOPNOTSUPP;	/* Operation not supported on socket */
 		goto bad;
 	}
 
-#if DIAGNOSTIC
-	if (UBCINFOMISSING(vp))
-		panic("vn_open: ubc_info_init");
-#endif /* DIAGNOSTIC */
+	if (vp->v_type == VLNK && (fmode & O_NOFOLLOW) != 0) {
+		error = ELOOP;	/* O_NOFOLLOW was specified and the target is a symbolic link */
+		goto bad;
+	}
 
 	/* authorize open of an existing file */
 	if ((fmode & O_CREAT) == 0) {
@@ -306,22 +361,55 @@ badcreate:
 			goto bad;
 		}
 
+#if CONFIG_MACF
+		error = mac_vnode_check_open(ctx, vp, fmode);
+		if (error)
+			goto bad;
+#endif
+
 		/* compute action to be authorized */
 		action = 0;
-		if (fmode & FREAD)
+		if (fmode & FREAD) {
 			action |= KAUTH_VNODE_READ_DATA;
-		if (fmode & (FWRITE | O_TRUNC))
+		}
+		if (fmode & (FWRITE | O_TRUNC)) {
+			/*
+			 * If we are writing, appending, and not truncating,
+			 * indicate that we are appending so that if the
+			 * UF_APPEND or SF_APPEND bits are set, we do not deny
+			 * the open.
+			 */
+			if ((fmode & O_APPEND) && !(fmode & O_TRUNC)) {
+				action |= KAUTH_VNODE_APPEND_DATA;
+			} else {
 			action |= KAUTH_VNODE_WRITE_DATA;
+			}
+		}
 		if ((error = vnode_authorize(vp, NULL, action, ctx)) != 0)
 			goto bad;
 		
+
+		//
+		// if the vnode is tagged VOPENEVT and the current process
+		// has the P_CHECKOPENEVT flag set, then we or in the O_EVTONLY
+		// flag to the open mode so that this open won't count against
+		// the vnode when carbon delete() does a vnode_isinuse() to see
+		// if a file is currently in use.  this allows spotlight
+		// importers to not interfere with carbon apps that depend on
+		// the no-delete-if-busy semantics of carbon delete().
+		//
+		if ((vp->v_flag & VOPENEVT) && (current_proc()->p_flag & P_CHECKOPENEVT)) {
+		    fmode |= O_EVTONLY;
+		}
+
 	}
 
 	if ( (error = VNOP_OPEN(vp, fmode, ctx)) ) {
 		goto bad;
 	}
-	if ( (error = vnode_ref_ext(vp, fmode)) )
+	if ( (error = vnode_ref_ext(vp, fmode)) ) {
 		goto bad;
+	}
 
 	/* call out to allow 3rd party notification of open. 
 	 * Ignore result of kauth_authorize_fileop call.
@@ -348,6 +436,7 @@ out:
 	return (error);
 }
 
+#if vn_access_DEPRECATED
 /*
  * Authorize an action against a vnode.  This has been the canonical way to
  * ensure that the credential/process/etc. referenced by a vfs_context
@@ -357,7 +446,6 @@ out:
  * to add support for more rights.  As such, this interface will be deprecated
  * and callers will use vnode_authorize instead.
  */
-#warning vn_access is deprecated
 int
 vn_access(vnode_t vp, int mode, vfs_context_t context)
 {
@@ -373,28 +461,39 @@ vn_access(vnode_t vp, int mode, vfs_context_t context)
   
  	return(vnode_authorize(vp, NULL, action, context));
 }
+#endif	/* vn_access_DEPRECATED */
 
 /*
  * Vnode close call
  */
 int
-vn_close(struct vnode *vp, int flags, kauth_cred_t cred, struct proc *p)
+vn_close(struct vnode *vp, int flags, vfs_context_t ctx)
 {
-	struct vfs_context context;
 	int error;
 
-	context.vc_proc = p;
-	context.vc_ucred = cred;
-
+#if CONFIG_FSE
 	if (flags & FWASWRITTEN) {
 	        if (need_fsevent(FSE_CONTENT_MODIFIED, vp)) {
-		        add_fsevent(FSE_CONTENT_MODIFIED, &context,
+		        add_fsevent(FSE_CONTENT_MODIFIED, ctx,
 				    FSE_ARG_VNODE, vp,
 				    FSE_ARG_DONE);
 		}
 	}
+#endif
 
-	error = VNOP_CLOSE(vp, flags, &context);
+#if NAMEDRSRCFORK
+	/* Clean up resource fork shadow file if needed. */
+	if ((vp->v_flag & VISNAMEDSTREAM) && 
+	    (vp->v_parent != NULLVP) &&
+	    !(vp->v_parent->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)) {
+		if (flags & FWASWRITTEN) {
+			(void) vnode_flushnamedstream(vp->v_parent, vp, ctx);
+		}
+		/* XXX failure ignored */
+		vnode_relenamedstream(vp->v_parent, vp, ctx);
+	}
+#endif
+	error = VNOP_CLOSE(vp, flags, ctx);
 	(void)vnode_rele_ext(vp, flags, 0);
 
 	return (error);
@@ -495,7 +594,7 @@ vn_rdwr(
 	int ioflg,
 	kauth_cred_t cred,
 	int *aresid,
-	struct proc *p)
+	proc_t p)
 {
 	return vn_rdwr_64(rw,
 			vp,
@@ -521,7 +620,7 @@ vn_rdwr_64(
 	int ioflg,
 	kauth_cred_t cred,
 	int *aresid,
-	struct proc *p)
+	proc_t p)
 {
 	uio_t auio;
 	int spacetype;
@@ -529,7 +628,7 @@ vn_rdwr_64(
 	int error=0;
 	char uio_buf[ UIO_SIZEOF(1) ];
 
-	context.vc_proc = p;
+	context.vc_thread = current_thread();
 	context.vc_ucred = cred;
 
 	if (UIO_SEG_IS_USER_SPACE(segflg)) {
@@ -542,14 +641,30 @@ vn_rdwr_64(
 								  &uio_buf[0], sizeof(uio_buf));
 	uio_addiov(auio, base, len);
 
-	if (rw == UIO_READ) {
-		if (vp->v_flag & VSWAP) {
-			error = vn_read_swapfile(vp, auio);
+#if CONFIG_MACF
+	/* XXXMAC
+	 * 	IO_NOAUTH should be re-examined.
+ 	 *	Likely that mediation should be performed in caller.
+	 */
+	if ((ioflg & IO_NOAUTH) == 0) {
+	/* passed cred is fp->f_cred */
+		if (rw == UIO_READ)
+			error = mac_vnode_check_read(&context, cred, vp);
+		else
+			error = mac_vnode_check_write(&context, cred, vp);
+	}
+#endif
+
+	if (error == 0) {
+		if (rw == UIO_READ) {
+			if (vp->v_flag & VSWAP) {
+				error = vn_read_swapfile(vp, auio);
+			} else {
+				error = VNOP_READ(vp, auio, ioflg, &context);
+			}
 		} else {
-			error = VNOP_READ(vp, auio, ioflg, &context);
+			error = VNOP_WRITE(vp, auio, ioflg, &context);
 		}
-	} else {
-		error = VNOP_WRITE(vp, auio, ioflg, &context);
 	}
 
 	if (aresid)
@@ -565,24 +680,32 @@ vn_rdwr_64(
  * File table vnode read routine.
  */
 static int
-vn_read(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
-	int flags, struct proc *p)
+vn_read(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx)
 {
 	struct vnode *vp;
 	int error, ioflag;
 	off_t count;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = cred;
 
 	vp = (struct vnode *)fp->f_fglob->fg_data;
 	if ( (error = vnode_getwithref(vp)) ) {
 		return(error);
 	}
+
+#if CONFIG_MACF
+	error = mac_vnode_check_read(ctx, vfs_context_ucred(ctx), vp);
+	if (error) {
+		(void)vnode_put(vp);
+		return (error);
+	}
+#endif
+
 	ioflag = 0;
 	if (fp->f_fglob->fg_flag & FNONBLOCK)
 		ioflag |= IO_NDELAY;
+	if ((fp->f_fglob->fg_flag & FNOCACHE) || vnode_isnocache(vp))
+	        ioflag |= IO_NOCACHE;
+	if (fp->f_fglob->fg_flag & FNORDAHEAD)
+	        ioflag |= IO_RAOFF;
 
 	if ((flags & FOF_OFFSET) == 0)
 		uio->uio_offset = fp->f_fglob->fg_offset;
@@ -592,7 +715,7 @@ vn_read(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
 		/* special case for swap files */
 		error = vn_read_swapfile(vp, uio);
 	} else {
-		error = VNOP_READ(vp, uio, ioflag, &context);
+		error = VNOP_READ(vp, uio, ioflag, ctx);
 	}
 	if ((flags & FOF_OFFSET) == 0)
 		fp->f_fglob->fg_offset += count - uio_resid(uio);
@@ -606,26 +729,37 @@ vn_read(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
  * File table vnode write routine.
  */
 static int
-vn_write(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
-	int flags, struct proc *p)
+vn_write(struct fileproc *fp, struct uio *uio, int flags, vfs_context_t ctx)
 {
 	struct vnode *vp;
 	int error, ioflag;
 	off_t count;
-	struct vfs_context context;
+	int clippedsize = 0;
+	int partialwrite=0;
+	int residcount, oldcount;
+	proc_t p = vfs_context_proc(ctx);
 
-	context.vc_proc = p;
-	context.vc_ucred = cred;
 	count = 0;
 	vp = (struct vnode *)fp->f_fglob->fg_data;
 	if ( (error = vnode_getwithref(vp)) ) {
 		return(error);
 	}
+
+#if CONFIG_MACF
+	error = mac_vnode_check_write(ctx, vfs_context_ucred(ctx), vp);
+	if (error) {
+		(void)vnode_put(vp);
+		return (error);
+	}
+#endif
+
 	ioflag = IO_UNIT;
 	if (vp->v_type == VREG && (fp->f_fglob->fg_flag & O_APPEND))
 		ioflag |= IO_APPEND;
 	if (fp->f_fglob->fg_flag & FNONBLOCK)
 		ioflag |= IO_NDELAY;
+	if ((fp->f_fglob->fg_flag & FNOCACHE) || vnode_isnocache(vp))
+	        ioflag |= IO_NOCACHE;
 	if ((fp->f_fglob->fg_flag & O_FSYNC) ||
 		(vp->v_mount && (vp->v_mount->mnt_flag & MNT_SYNCHRONOUS)))
 		ioflag |= IO_SYNC;
@@ -634,15 +768,56 @@ vn_write(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
 		uio->uio_offset = fp->f_fglob->fg_offset;
 		count = uio_resid(uio);
 	}
-	if (p && (vp->v_type == VREG) &&
-            (((uio->uio_offset + uio_uio_resid(uio)) > p->p_rlimit[RLIMIT_FSIZE].rlim_cur) ||
-             (uio_uio_resid(uio) > (p->p_rlimit[RLIMIT_FSIZE].rlim_cur - uio->uio_offset)))) {
+	if (((flags & FOF_OFFSET) == 0) &&
+	 	vfs_context_proc(ctx) && (vp->v_type == VREG) &&
+            (((rlim_t)(uio->uio_offset + uio_uio_resid(uio)) > p->p_rlimit[RLIMIT_FSIZE].rlim_cur) ||
+             ((rlim_t)uio_uio_resid(uio) > (p->p_rlimit[RLIMIT_FSIZE].rlim_cur - uio->uio_offset)))) {
+	     	/*
+		 * If the requested residual would cause us to go past the
+		 * administrative limit, then we need to adjust the residual
+		 * down to cause fewer bytes than requested to be written.  If
+		 * we can't do that (e.g. the residual is already 1 byte),
+		 * then we fail the write with EFBIG.
+		 */
+		residcount = uio_uio_resid(uio);
+            	if ((rlim_t)(uio->uio_offset + uio_uio_resid(uio)) > p->p_rlimit[RLIMIT_FSIZE].rlim_cur) {
+			clippedsize =  (uio->uio_offset + uio_uio_resid(uio)) - p->p_rlimit[RLIMIT_FSIZE].rlim_cur;
+		} else if ((rlim_t)uio_uio_resid(uio) > (p->p_rlimit[RLIMIT_FSIZE].rlim_cur - uio->uio_offset)) {
+			clippedsize = (p->p_rlimit[RLIMIT_FSIZE].rlim_cur - uio->uio_offset);
+		}
+		if (clippedsize >= residcount) {
+			psignal(p, SIGXFSZ);
+			vnode_put(vp);
+			return (EFBIG);
+		}
+		partialwrite = 1;
+		uio_setresid(uio, residcount-clippedsize);
+	}
+	if ((flags & FOF_OFFSET) != 0) {
+		/* for pwrite, append should  be ignored */
+		ioflag &= ~IO_APPEND;
+		if (p && (vp->v_type == VREG) &&
+            	((rlim_t)uio->uio_offset  >= p->p_rlimit[RLIMIT_FSIZE].rlim_cur)) {
 		psignal(p, SIGXFSZ);
 		vnode_put(vp);
 		return (EFBIG);
 	}
+		if (p && (vp->v_type == VREG) &&
+			((rlim_t)(uio->uio_offset + uio_uio_resid(uio)) > p->p_rlimit[RLIMIT_FSIZE].rlim_cur)) {
+			//Debugger("vn_bwrite:overstepping the bounds");
+			residcount = uio_uio_resid(uio);
+			clippedsize =  (uio->uio_offset + uio_uio_resid(uio)) - p->p_rlimit[RLIMIT_FSIZE].rlim_cur;
+			partialwrite = 1;
+			uio_setresid(uio, residcount-clippedsize);
+		}
+	}
+
+	error = VNOP_WRITE(vp, uio, ioflag, ctx);
 
-	error = VNOP_WRITE(vp, uio, ioflag, &context);
+	if (partialwrite) {
+		oldcount = uio_resid(uio);
+		uio_setresid(uio, oldcount + clippedsize);
+	}
 
 	if ((flags & FOF_OFFSET) == 0) {
 		if (ioflag & IO_APPEND)
@@ -675,14 +850,26 @@ vn_write(struct fileproc *fp, struct uio *uio, kauth_cred_t cred,
 
 /*
  * File table vnode stat routine.
+ *
+ * Returns:	0			Success
+ *		EBADF
+ *		ENOMEM
+ *	vnode_getattr:???
  */
 int
-vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_context_t ctx)
+vn_stat_noauth(struct vnode *vp, void *sbptr, kauth_filesec_t *xsec, int isstat64, vfs_context_t ctx)
 {
 	struct vnode_attr va;
 	int error;
 	u_short mode;
 	kauth_filesec_t fsec;
+	struct stat *sb = (struct stat *)0;	/* warning avoidance ; protected by isstat64 */
+	struct stat64 * sb64 = (struct stat64 *)0;  /* warning avoidance ; protected by isstat64 */
+
+	if (isstat64 != 0)
+		sb64 = (struct stat64 *)sbptr;
+	else
+		sb = (struct stat *)sbptr;
 
 	VATTR_INIT(&va);
 	VATTR_WANTED(&va, va_fsid);
@@ -697,6 +884,7 @@ vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_con
 	VATTR_WANTED(&va, va_access_time);
 	VATTR_WANTED(&va, va_modify_time);
 	VATTR_WANTED(&va, va_change_time);
+	VATTR_WANTED(&va, va_create_time);
 	VATTR_WANTED(&va, va_flags);
 	VATTR_WANTED(&va, va_gen);
 	VATTR_WANTED(&va, va_iosize);
@@ -713,8 +901,14 @@ vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_con
 	/*
 	 * Copy from vattr table
 	 */
-	sb->st_dev = va.va_fsid;
-	sb->st_ino = (ino_t)va.va_fileid;
+	if (isstat64 != 0) {
+		sb64->st_dev = va.va_fsid;
+		sb64->st_ino = (ino64_t)va.va_fileid;
+
+	} else {
+		sb->st_dev = va.va_fsid;
+		sb->st_ino = (ino_t)va.va_fileid;
+	}
 	mode = va.va_mode;
 	switch (vp->v_type) {
 	case VREG:
@@ -742,18 +936,35 @@ vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_con
 		error = EBADF;
 		goto out;
 	};
-	sb->st_mode = mode;
-	sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1;
-	sb->st_uid = va.va_uid;
-	sb->st_gid = va.va_gid;
-	sb->st_rdev = va.va_rdev;
-	sb->st_size = va.va_data_size;
-	sb->st_atimespec = va.va_access_time;
-	sb->st_mtimespec = va.va_modify_time;
-	sb->st_ctimespec = va.va_change_time;
-	sb->st_blksize = va.va_iosize;
-	sb->st_flags = va.va_flags;
-	sb->st_blocks = roundup(va.va_total_alloc, 512) / 512;
+	if (isstat64 != 0) {
+		sb64->st_mode = mode;
+		sb64->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1;
+		sb64->st_uid = va.va_uid;
+		sb64->st_gid = va.va_gid;
+		sb64->st_rdev = va.va_rdev;
+		sb64->st_size = va.va_data_size;
+		sb64->st_atimespec = va.va_access_time;
+		sb64->st_mtimespec = va.va_modify_time;
+		sb64->st_ctimespec = va.va_change_time;
+		sb64->st_birthtimespec = 
+				VATTR_IS_SUPPORTED(&va, va_create_time) ? va.va_create_time : va.va_change_time;
+		sb64->st_blksize = va.va_iosize;
+		sb64->st_flags = va.va_flags;
+		sb64->st_blocks = roundup(va.va_total_alloc, 512) / 512;
+	} else {
+		sb->st_mode = mode;
+		sb->st_nlink = VATTR_IS_SUPPORTED(&va, va_nlink) ? (u_int16_t)va.va_nlink : 1;
+		sb->st_uid = va.va_uid;
+		sb->st_gid = va.va_gid;
+		sb->st_rdev = va.va_rdev;
+		sb->st_size = va.va_data_size;
+		sb->st_atimespec = va.va_access_time;
+		sb->st_mtimespec = va.va_modify_time;
+		sb->st_ctimespec = va.va_change_time;
+		sb->st_blksize = va.va_iosize;
+		sb->st_flags = va.va_flags;
+		sb->st_blocks = roundup(va.va_total_alloc, 512) / 512;
+	}
 
 	/* if we're interested in exended security data and we got an ACL */
 	if (xsec != NULL) {
@@ -793,10 +1004,17 @@ vn_stat_noauth(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_con
 	}
 	
 	/* Do not give the generation number out to unpriviledged users */
-	if (va.va_gen && !vfs_context_issuser(ctx))
-		sb->st_gen = 0; 
-	else
-		sb->st_gen = va.va_gen;
+	if (va.va_gen && !vfs_context_issuser(ctx)) {
+		if (isstat64 != 0)
+			sb64->st_gen = 0; 
+		else
+			sb->st_gen = 0; 
+	} else {
+		if (isstat64 != 0)
+			sb64->st_gen = va.va_gen; 
+		else
+			sb->st_gen = va.va_gen;
+	}
 
 	error = 0;
 out:
@@ -806,16 +1024,22 @@ out:
 }
 
 int
-vn_stat(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_context_t ctx)
+vn_stat(struct vnode *vp, void *sb, kauth_filesec_t *xsec, int isstat64, vfs_context_t ctx)
 {
 	int error;
 
+#if CONFIG_MACF
+	error = mac_vnode_check_stat(ctx, NOCRED, vp);
+	if (error)
+		return (error);
+#endif
+
 	/* authorize */
 	if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_READ_SECURITY, ctx)) != 0)
 		return(error);
 
 	/* actual stat */
-	return(vn_stat_noauth(vp, sb, xsec, ctx));
+	return(vn_stat_noauth(vp, sb, xsec, isstat64, ctx));
 }
 
 
@@ -823,31 +1047,30 @@ vn_stat(struct vnode *vp, struct stat *sb, kauth_filesec_t *xsec, vfs_context_t
  * File table vnode ioctl routine.
  */
 static int
-vn_ioctl(fp, com, data, p)
-	struct fileproc *fp;
-	u_long com;
-	caddr_t data;
-	struct proc *p;
+vn_ioctl(struct fileproc *fp, u_long com, caddr_t data, vfs_context_t ctx)
 {
-	register struct vnode *vp = ((struct vnode *)fp->f_fglob->fg_data);
-	struct vfs_context context;
+	struct vnode *vp = ((struct vnode *)fp->f_fglob->fg_data);
 	off_t file_size;
 	int error;
 	struct vnode *ttyvp;
 	int funnel_state;
+	struct session * sessp;
 	
 	if ( (error = vnode_getwithref(vp)) ) {
 		return(error);
 	}
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
 
-	switch (vp->v_type) {
+#if CONFIG_MACF
+	error = mac_vnode_check_ioctl(ctx, vp, com);
+	if (error)
+		goto out;
+#endif
 
+	switch (vp->v_type) {
 	case VREG:
 	case VDIR:
 		if (com == FIONREAD) {
-			if ((error = vnode_size(vp, &file_size, &context)) != 0)
+			if ((error = vnode_size(vp, &file_size, ctx)) != 0)
 				goto out;
 			*(int *)data = file_size - fp->f_fglob->fg_offset;
 			goto out;
@@ -891,14 +1114,20 @@ vn_ioctl(fp, com, data, p)
 			}
 			goto out;
 		}
-		error = VNOP_IOCTL(vp, com, data, fp->f_fglob->fg_flag, &context);
+		error = VNOP_IOCTL(vp, com, data, fp->f_fglob->fg_flag, ctx);
 
 		if (error == 0 && com == TIOCSCTTY) {
 			vnode_ref(vp);
 
 			funnel_state = thread_funnel_set(kernel_flock, TRUE);
-			ttyvp = p->p_session->s_ttyvp;
-			p->p_session->s_ttyvp = vp;
+			sessp = proc_session(vfs_context_proc(ctx));
+
+			session_lock(sessp);
+			ttyvp = sessp->s_ttyvp;
+			sessp->s_ttyvp = vp;
+			sessp->s_ttyvid = vnode_vid(vp);
+			session_unlock(sessp);
+			session_rele(sessp);
 			thread_funnel_set(kernel_flock, funnel_state);
 
 			if (ttyvp)
@@ -907,7 +1136,6 @@ vn_ioctl(fp, com, data, p)
 	}
 out:
 	(void)vnode_put(vp);
-	kauth_cred_unref(&context.vc_ucred);
 	return(error);
 }
 
@@ -915,21 +1143,26 @@ out:
  * File table vnode select routine.
  */
 static int
-vn_select(fp, which, wql, p)
-	struct fileproc *fp;
-	int which;
-	void * wql;
-	struct proc *p;
+vn_select(struct fileproc *fp, int which, void *wql, __unused vfs_context_t ctx)
 {
 	int error;
 	struct vnode * vp = (struct vnode *)fp->f_fglob->fg_data;
 	struct vfs_context context;
 
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
-	        context.vc_proc = p;
+		context.vc_thread = current_thread();
 		context.vc_ucred = fp->f_fglob->fg_cred;
 
-	        error = VNOP_SELECT(vp, which, fp->f_fglob->fg_flag, wql, &context);
+#if CONFIG_MACF
+		/*
+		 * XXX We should use a per thread credential here; minimally,
+		 * XXX the process credential should have a persistent
+		 * XXX reference on it before being passed in here.
+		 */
+		error = mac_vnode_check_select(ctx, vp, which);
+		if (error == 0)
+#endif
+	        error = VNOP_SELECT(vp, which, fp->f_fglob->fg_flag, wql, ctx);
 
 		(void)vnode_put(vp);
 	}
@@ -951,21 +1184,33 @@ vn_lock(__unused vnode_t vp, __unused int flags, __unused proc_t p)
  * File table vnode close routine.
  */
 static int
-vn_closefile(fg, p)
-	struct fileglob *fg;
-	struct proc *p;
+vn_closefile(struct fileglob *fg, vfs_context_t ctx)
 {
 	struct vnode *vp = (struct vnode *)fg->fg_data;
 	int error;
+	struct flock lf;
 
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
-	        error = vn_close(vp, fg->fg_flag, fg->fg_cred, p);
+
+		if ((fg->fg_flag & FHASLOCK) && fg->fg_type == DTYPE_VNODE) {
+			lf.l_whence = SEEK_SET;
+			lf.l_start = 0;
+			lf.l_len = 0;
+			lf.l_type = F_UNLCK;
+
+			(void)VNOP_ADVLOCK(vp, (caddr_t)fg, F_UNLCK, &lf, F_FLOCK, ctx);
+		}
+	        error = vn_close(vp, fg->fg_flag, ctx);
 
 		(void)vnode_put(vp);
 	}
 	return(error);
 }
 
+/*
+ * Returns:	0			Success
+ *	VNOP_PATHCONF:???
+ */
 int
 vn_pathconf(vnode_t vp, int name, register_t *retval, vfs_context_t ctx)
 {
@@ -973,11 +1218,41 @@ vn_pathconf(vnode_t vp, int name, register_t *retval, vfs_context_t ctx)
 
 	switch(name) {
 	case _PC_EXTENDED_SECURITY_NP:
-		*retval = vfs_extendedsecurity(vnode_mount(vp));
+		*retval = vfs_extendedsecurity(vnode_mount(vp)) ? 1 : 0;
 		break;
 	case _PC_AUTH_OPAQUE_NP:
 		*retval = vfs_authopaque(vnode_mount(vp));
 		break;
+	case _PC_2_SYMLINKS:
+		*retval = 1;	/* XXX NOTSUP on MSDOS, etc. */
+		break;
+	case _PC_ALLOC_SIZE_MIN:
+		*retval = 1;	/* XXX lie: 1 byte */
+		break;
+	case _PC_ASYNC_IO:	/* unistd.h: _POSIX_ASYNCHRONUS_IO */
+		*retval = 1;	/* [AIO] option is supported */
+		break;
+	case _PC_PRIO_IO:	/* unistd.h: _POSIX_PRIORITIZED_IO */
+		*retval = 0;	/* [PIO] option is not supported */
+		break;
+	case _PC_REC_INCR_XFER_SIZE:
+		*retval = 4096;	/* XXX go from MIN to MAX 4K at a time */
+		break;
+	case _PC_REC_MIN_XFER_SIZE:
+		*retval = 4096;	/* XXX recommend 4K minimum reads/writes */
+		break;
+	case _PC_REC_MAX_XFER_SIZE:
+		*retval = 65536; /* XXX recommend 64K maximum reads/writes */
+		break;
+	case _PC_REC_XFER_ALIGN:
+		*retval = 4096;	/* XXX recommend page aligned buffers */
+		break;
+	case _PC_SYMLINK_MAX:
+		*retval = 255;	/* Minimum acceptable POSIX value */
+		break;
+	case _PC_SYNC_IO:	/* unistd.h: _POSIX_SYNCHRONIZED_IO */
+		*retval = 0;	/* [SIO] option is not supported */
+		break;
 	default:
 		error = VNOP_PATHCONF(vp, name, retval, ctx);
 		break;
@@ -987,26 +1262,27 @@ vn_pathconf(vnode_t vp, int name, register_t *retval, vfs_context_t ctx)
 }
 
 static int
-vn_kqfilt_add(fp, kn, p)
-	struct fileproc *fp;
-	struct knote *kn;
-	struct proc *p;
+vn_kqfilt_add(struct fileproc *fp, struct knote *kn, vfs_context_t ctx)
 {
 	struct vnode *vp = (struct vnode *)fp->f_fglob->fg_data;
-	struct vfs_context context;
 	int error;
 	int funnel_state;
 	
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
+
+#if CONFIG_MACF
+		error = mac_vnode_check_kqfilter(ctx, fp->f_fglob->fg_cred, kn, vp);
+		if (error) {
+			(void)vnode_put(vp);
+			return (error);
+		}
+#endif
 
 	        funnel_state = thread_funnel_set(kernel_flock, TRUE);
-		error = VNOP_KQFILT_ADD(vp, kn, &context);
+		error = VNOP_KQFILT_ADD(vp, kn, ctx);
 		thread_funnel_set(kernel_flock, funnel_state);
 
 		(void)vnode_put(vp);
-		kauth_cred_unref(&context.vc_ucred);
 	}
 	return (error);
 }
@@ -1014,25 +1290,21 @@ vn_kqfilt_add(fp, kn, p)
 #if 0
 /* No one calls this yet. */
 static int
-vn_kqfilt_remove(vp, ident, p)
+vn_kqfilt_remove(vp, ident, ctx)
 	struct vnode *vp;
 	uintptr_t ident;
-	struct proc *p;
+	vfs_context_t ctx;
 {
-	struct vfs_context context;
 	int error;
 	int funnel_state;
 	
 	if ( (error = vnode_getwithref(vp)) == 0 ) {
-	        context.vc_proc = p;
-		context.vc_ucred = kauth_cred_proc_ref(p);	/* XXX kauth_cred_get() ??? */
 
 		funnel_state = thread_funnel_set(kernel_flock, TRUE);
-		error = VNOP_KQFILT_REMOVE(vp, ident, &context);
+		error = VNOP_KQFILT_REMOVE(vp, ident, ctx);
 		thread_funnel_set(kernel_flock, funnel_state);
 
 		(void)vnode_put(vp);
-		kauth_cred_unref(&context.vc_ucred);
 	}
 	return (error);
 }
diff --git a/bsd/vfs/vfs_xattr.c b/bsd/vfs/vfs_xattr.c
index 4d225635f..e9ef92850 100644
--- a/bsd/vfs/vfs_xattr.c
+++ b/bsd/vfs/vfs_xattr.c
@@ -1,25 +1,37 @@
 /*
- * Copyright (c) 2004-2007 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
- 
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+
 #include <sys/param.h>
 
 #include <sys/fcntl.h>
@@ -27,6 +39,7 @@
 #include <sys/kernel.h>
 #include <sys/kauth.h>
 #include <sys/malloc.h>
+#include <sys/mount_internal.h>
 #include <sys/namei.h>
 #include <sys/proc_internal.h>
 #include <sys/stat.h>
@@ -34,22 +47,45 @@
 #include <sys/utfconv.h>
 #include <sys/vnode.h>
 #include <sys/vnode_internal.h>
-
 #include <sys/xattr.h>
 
 #include <libkern/OSByteOrder.h>
 #include <vm/vm_kern.h>
 
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
+
+
+#if NAMEDSTREAMS
+
 /*
- * Default xattr support routines.
+ * Cast to 'unsigned int' loses precision - hope that's OK...
  */
-static int default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size,
-                            int options, vfs_context_t context);
+#define MAKE_SHADOW_NAME(VP, NAME)  \
+	snprintf((NAME), sizeof((NAME)), ".vfs_rsrc_stream_%x%08x%x", (unsigned int)(VP), (VP)->v_id, (unsigned int)(VP)->v_data);
+
+static vnode_t shadow_dvp;  /* tmp directory to hold stream shadow files */
+static int shadow_vid;
+static int shadow_sequence;
+
+
+static int  default_getnamedstream(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperation op, vfs_context_t context);
+
+static int  default_makenamedstream(vnode_t vp, vnode_t *svpp, const char *name, vfs_context_t context);
 
-static int default_setxattr(vnode_t vp, const char *name, uio_t uio,
-                            int options, vfs_context_t context);
+static int  default_removenamedstream(vnode_t vp, const char *name, vfs_context_t context);
 
-static int default_removexattr(vnode_t vp, const char *name, int options, vfs_context_t context);
+static int  getshadowfile(vnode_t vp, vnode_t *svpp, int makestream, size_t *rsrcsize, int *creator, vfs_context_t context);
+
+static int  get_shadow_dir(vnode_t *sdvpp, vfs_context_t context);
+
+#endif
+
+
+/*
+ * Default xattr support routines.
+ */
 
 static int default_listxattr(vnode_t vp, uio_t uio, size_t *size, int options,
                              vfs_context_t context);
@@ -68,11 +104,37 @@ vn_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size,
 	if (!(vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VLNK)) {
 		return (EPERM);
 	}
-	if ((error = xattr_validatename(name))) {
-		return (error);
-	}
-	if (!(options & XATTR_NOSECURITY) && (error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_EXTATTRIBUTES, context)))
+#if NAMEDSTREAMS
+	/* getxattr calls are not allowed for streams. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		error = EPERM;
 		goto out;
+	}
+#endif
+	/*
+	 * Non-kernel request need extra checks performed.
+	 *
+	 * The XATTR_NOSECURITY flag implies a kernel request.
+	 */
+	if (!(options & XATTR_NOSECURITY)) {
+#if CONFIG_MACF
+		error = mac_vnode_check_getextattr(context, vp, name, uio);
+		if (error)
+			goto out;
+#endif /* MAC */
+		if ((error = xattr_validatename(name))) {
+			goto out;
+		}
+		if ((error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_EXTATTRIBUTES, context))) {
+			goto out;
+		}
+		/* The offset can only be non-zero for resource forks. */
+		if (uio != NULL && uio_offset(uio) != 0 && 
+		    bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+			error = EINVAL;
+			goto out;
+		}
+	}
 
 	/* The offset can only be non-zero for resource forks. */
 	if (uio != NULL && uio_offset(uio) != 0 && 
@@ -82,7 +144,7 @@ vn_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size,
 	}
 
 	error = VNOP_GETXATTR(vp, name, uio, size, options, context);
-	if (error == ENOTSUP) {
+	if (error == ENOTSUP && !(options & XATTR_NODEFAULT)) {
 		/*
 		 * A filesystem may keep some EAs natively and return ENOTSUP for others.
 		 * SMB returns ENOTSUP for finderinfo and resource forks.
@@ -104,15 +166,29 @@ vn_setxattr(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_t
 	if (!(vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VLNK)) {
 		return (EPERM);
 	}
+#if NAMEDSTREAMS
+	/* setxattr calls are not allowed for streams. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		error = EPERM;
+		goto out;
+	}
+#endif
 	if ((options & (XATTR_REPLACE|XATTR_CREATE)) == (XATTR_REPLACE|XATTR_CREATE)) {
 		return (EINVAL);
 	}
 	if ((error = xattr_validatename(name))) {
 		return (error);
 	}
- 	if (!(options & XATTR_NOSECURITY) && (error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_EXTATTRIBUTES, context)))
-		goto out;
-
+ 	if (!(options & XATTR_NOSECURITY)) {
+#if CONFIG_MACF
+		error = mac_vnode_check_setextattr(context, vp, name, uio);
+		if (error)
+			goto out;
+#endif /* MAC */
+		error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_EXTATTRIBUTES, context);
+		if (error)
+			goto out;
+	}
 	/* The offset can only be non-zero for resource forks. */
 	if (uio_offset(uio) != 0 && 
 	    bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0 ) {
@@ -154,13 +230,18 @@ vn_setxattr(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_t
 		/* the mainline path here is to have error==ENOTSUP ... */
 	}
 #endif /* DUAL_EAS */
-	if (error == ENOTSUP) {
+	if (error == ENOTSUP && !(options & XATTR_NODEFAULT)) {
 		/*
 		 * A filesystem may keep some EAs natively and return ENOTSUP for others.
 		 * SMB returns ENOTSUP for finderinfo and resource forks.
 		 */
 		error = default_setxattr(vp, name, uio, options, context);
 	}
+#if CONFIG_MACF
+	if ((error == 0) && !(options & XATTR_NOSECURITY) &&
+	    (vfs_flags(vnode_mount(vp)) & MNT_MULTILABEL))
+		mac_vnode_label_update_extattr(vnode_mount(vp), vp, name);
+#endif
 out:
 	return (error);
 }
@@ -176,13 +257,28 @@ vn_removexattr(vnode_t vp, const char * name, int options, vfs_context_t context
 	if (!(vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VLNK)) {
 		return (EPERM);
 	}
+#if NAMEDSTREAMS
+	/* removexattr calls are not allowed for streams. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		error = EPERM;
+		goto out;
+	}
+#endif
 	if ((error = xattr_validatename(name))) {
 		return (error);
 	}
-	if (!(options & XATTR_NOSECURITY) && (error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_EXTATTRIBUTES, context)))
-		goto out;
+	if (!(options & XATTR_NOSECURITY)) {
+#if CONFIG_MACF
+		error = mac_vnode_check_deleteextattr(context, vp, name);
+		if (error)
+			goto out;
+#endif /* MAC */
+		error = vnode_authorize(vp, NULL, KAUTH_VNODE_WRITE_EXTATTRIBUTES, context);
+		if (error)
+			goto out;
+	}
 	error = VNOP_REMOVEXATTR(vp, name, options, context);
-	if (error == ENOTSUP) {
+	if (error == ENOTSUP && !(options & XATTR_NODEFAULT)) {
 		/*
 		 * A filesystem may keep some EAs natively and return ENOTSUP for others.
 		 * SMB returns ENOTSUP for finderinfo and resource forks.
@@ -201,6 +297,11 @@ vn_removexattr(vnode_t vp, const char * name, int options, vfs_context_t context
 			error = 0;
 #endif /* DUAL_EAS */
 	}
+#if CONFIG_MACF
+	if ((error == 0) && !(options & XATTR_NOSECURITY) &&
+	    (vfs_flags(vnode_mount(vp)) & MNT_MULTILABEL))
+		mac_vnode_label_update_extattr(vnode_mount(vp), vp, name);
+#endif
 out:
 	return (error);
 }
@@ -216,11 +317,27 @@ vn_listxattr(vnode_t vp, uio_t uio, size_t *size, int options, vfs_context_t con
 	if (!(vp->v_type == VREG || vp->v_type == VDIR || vp->v_type == VLNK)) {
 		return (EPERM);
 	}
-	if (!(options & XATTR_NOSECURITY) && (error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_EXTATTRIBUTES, context)))
-		goto out;
+#if NAMEDSTREAMS
+	/* listxattr calls are not allowed for streams. */
+	if (vp->v_flag & VISNAMEDSTREAM) {
+		return (EPERM);
+	}
+#endif
+
+	if (!(options & XATTR_NOSECURITY)) {
+#if CONFIG_MACF
+		error = mac_vnode_check_listextattr(context, vp);
+		if (error)
+			goto out;
+#endif /* MAC */
+
+		error = vnode_authorize(vp, NULL, KAUTH_VNODE_READ_EXTATTRIBUTES, context);
+		if (error)
+			goto out;
+	}
 
 	error = VNOP_LISTXATTR(vp, uio, size, options, context);
-	if (error == ENOTSUP) {
+	if (error == ENOTSUP && !(options & XATTR_NODEFAULT)) {
 		/*
 		 * A filesystem may keep some but not all EAs natively, in which case
 		 * the native EA names will have been uiomove-d out (or *size updated)
@@ -244,8 +361,8 @@ xattr_validatename(const char *name)
 	namelen = strnlen(name, XATTR_MAXNAMELEN);
 	if (name[namelen] != '\0') 
 		return (ENAMETOOLONG);
-
-	if (utf8_validatestr(name, namelen) != 0) 
+	
+	if (utf8_validatestr((const unsigned char *)name, namelen) != 0) 
 		return (EINVAL);
 	
 	return (0);
@@ -262,6 +379,620 @@ xattr_protected(const char *attrname)
 }
 
 
+#if NAMEDSTREAMS
+/*
+ * Obtain a named stream from vnode vp.
+ */
+errno_t
+vnode_getnamedstream(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperation op, int flags, vfs_context_t context)
+{
+	int error;
+
+	if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)
+		error = VNOP_GETNAMEDSTREAM(vp, svpp, name, op, flags, context);
+	else
+		error = default_getnamedstream(vp, svpp, name, op, context);
+
+	if (error == 0) {
+		vnode_t svp = *svpp;
+		
+		/* Tag the vnode. */
+		vnode_lock(svp);
+		svp->v_flag |= VISNAMEDSTREAM;
+		vnode_unlock(svp);
+		/* Make the file it's parent. */
+		vnode_update_identity(svp, vp, NULL, 0, 0, VNODE_UPDATE_PARENT);
+	}		
+
+	return (error);
+}
+
+/*
+ * Make a named stream for vnode vp.
+ */
+errno_t 
+vnode_makenamedstream(vnode_t vp, vnode_t *svpp, const char *name, int flags, vfs_context_t context)
+{
+	int error;
+
+	if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)
+		error = VNOP_MAKENAMEDSTREAM(vp, svpp, name, flags, context);
+	else
+		error = default_makenamedstream(vp, svpp, name, context);
+
+	if (error == 0) {
+		vnode_t svp = *svpp;
+
+		/* Tag the vnode. */
+		vnode_lock(svp);
+		svp->v_flag |= VISNAMEDSTREAM;
+		vnode_unlock(svp);
+		/* Make the file it's parent. */
+		vnode_update_identity(svp, vp, NULL, 0, 0, VNODE_UPDATE_PARENT);
+	}
+	return (error);
+}
+
+/*
+ * Remove a named stream from vnode vp.
+ */
+errno_t 
+vnode_removenamedstream(vnode_t vp, vnode_t svp, const char *name, int flags, vfs_context_t context)
+{
+	int error;
+
+	if (vp->v_mount->mnt_kern_flag & MNTK_NAMED_STREAMS)
+		error = VNOP_REMOVENAMEDSTREAM(vp, svp, name, flags, context);
+	else
+		error = default_removenamedstream(vp, name, context);
+
+	return (error);
+}
+
+#define NS_IOBUFSIZE  (128 * 1024)
+
+/*
+ * Release a named stream shadow file.
+ */
+errno_t
+vnode_relenamedstream(vnode_t vp, vnode_t svp, vfs_context_t context)
+{
+	vnode_t dvp;
+	struct componentname cn;
+	char tmpname[48];
+	errno_t err;
+
+	if (vnode_isinuse(svp, 1)) {
+		return (EBUSY);
+	}
+	cache_purge(svp);
+
+	vnode_lock(svp);
+	MAKE_SHADOW_NAME(vp, tmpname);
+	vnode_unlock(svp);
+
+	cn.cn_nameiop = DELETE;
+	cn.cn_flags = ISLASTCN;
+	cn.cn_context = context;
+	cn.cn_pnbuf = tmpname;
+	cn.cn_pnlen = sizeof(tmpname);
+	cn.cn_nameptr = cn.cn_pnbuf;
+	cn.cn_namelen = strlen(tmpname);
+
+	/* Obtain the vnode for the shadow files directory. */
+	err = get_shadow_dir(&dvp, context);
+	if (err != 0) {
+		return err;
+	}
+	/* Check for busy svp one last time. */
+	if (vnode_isinuse(svp, 1) == 0) {
+		(void) VNOP_REMOVE(dvp, svp, &cn, 0, context);
+		(void) vnode_recycle(svp);
+	}
+	vnode_put(dvp);
+
+	return (0);
+}
+
+/*
+ * Flush a named stream shadow file.
+ */
+errno_t 
+vnode_flushnamedstream(vnode_t vp, vnode_t svp, vfs_context_t context)
+{
+	struct vnode_attr va;
+	uio_t auio = NULL;
+	caddr_t  bufptr = NULL;
+	size_t  bufsize = 0;
+	size_t  offset;
+	size_t  iosize;
+	size_t datasize;
+	int error;
+
+	VATTR_INIT(&va);
+	VATTR_WANTED(&va, va_data_size);
+	if (VNOP_GETATTR(svp, &va, context) != 0  ||
+		!VATTR_IS_SUPPORTED(&va, va_data_size)) {
+		return (0);
+	}
+	datasize = va.va_data_size;
+	if ((datasize == 0)) {
+		(void) default_removexattr(vp, XATTR_RESOURCEFORK_NAME, 0, context);
+		return (0);
+	}
+
+	iosize = bufsize = MIN(datasize, NS_IOBUFSIZE);
+	if (kmem_alloc(kernel_map, (vm_offset_t *)&bufptr, bufsize)) {
+		return (ENOMEM);
+	}
+	auio = uio_create(1, 0, UIO_SYSSPACE32, UIO_READ);
+	offset = 0;
+
+	/*
+	 * Copy the shadow stream file data into the resource fork.
+	 */
+	error = VNOP_OPEN(svp, 0, context);
+	if (error) {
+		printf("vnode_flushnamedstream: err %d opening file\n", error);
+		goto out;
+	}
+	while (offset < datasize) {
+		iosize = MIN(datasize - offset, iosize);
+
+		uio_reset(auio, offset, UIO_SYSSPACE32, UIO_READ);
+		uio_addiov(auio, (uintptr_t)bufptr, iosize);
+		error = VNOP_READ(svp, auio, 0, context);
+		if (error) {
+			break;
+		}
+		/* Since there's no truncate xattr we must remove the resource fork. */
+		if (offset == 0) {
+			error = default_removexattr(vp, XATTR_RESOURCEFORK_NAME, 0, context);
+			if ((error != 0) && (error != ENOATTR)) {
+				break;
+			}
+		}
+		uio_reset(auio, offset, UIO_SYSSPACE32, UIO_WRITE);
+		uio_addiov(auio, (uintptr_t)bufptr, iosize);
+		error = vn_setxattr(vp, XATTR_RESOURCEFORK_NAME, auio, XATTR_NOSECURITY, context);
+		if (error) {
+			break;
+		}
+		offset += iosize;
+	}
+	(void) VNOP_CLOSE(svp, 0, context);
+out:
+	if (bufptr) {
+		kmem_free(kernel_map, (vm_offset_t)bufptr, bufsize);
+	}
+	if (auio) {
+		uio_free(auio);
+	}
+	return (error);
+}
+
+
+static int
+getshadowfile(vnode_t vp, vnode_t *svpp, int makestream, size_t *rsrcsize,
+              int *creator, vfs_context_t context)
+{
+	vnode_t  dvp = NULLVP;
+	vnode_t  svp = NULLVP;
+	struct componentname cn;
+	struct vnode_attr va;
+	char tmpname[48];
+	size_t datasize = 0;
+	int  error = 0;
+
+	*creator = 0;
+
+	/* Establish a unique file name. */
+	MAKE_SHADOW_NAME(vp, tmpname);
+	bzero(&cn, sizeof(cn));
+	cn.cn_nameiop = LOOKUP;
+	cn.cn_flags = ISLASTCN;
+	cn.cn_context = context;
+	cn.cn_pnbuf = tmpname;
+	cn.cn_pnlen = sizeof(tmpname);
+	cn.cn_nameptr = cn.cn_pnbuf;
+	cn.cn_namelen = strlen(tmpname);
+
+	/* Pick up uid, gid, mode and date from original file. */
+	VATTR_INIT(&va);
+	VATTR_WANTED(&va, va_uid);
+	VATTR_WANTED(&va, va_gid);
+	VATTR_WANTED(&va, va_mode);
+	VATTR_WANTED(&va, va_create_time);
+	VATTR_WANTED(&va, va_modify_time);
+	if (VNOP_GETATTR(vp, &va, context) != 0  ||
+		!VATTR_IS_SUPPORTED(&va, va_uid)  ||
+		!VATTR_IS_SUPPORTED(&va, va_gid)  ||
+		!VATTR_IS_SUPPORTED(&va, va_mode)) {
+		va.va_uid = KAUTH_UID_NONE;
+		va.va_gid = KAUTH_GID_NONE;
+		va.va_mode = S_IRUSR | S_IWUSR;
+	}
+	va.va_vaflags = VA_EXCLUSIVE;
+	VATTR_SET(&va, va_type, VREG);
+	/* We no longer change the access, but we still hide it. */
+	VATTR_SET(&va, va_flags, UF_HIDDEN);
+
+	/* Obtain the vnode for the shadow files directory. */
+	if (get_shadow_dir(&dvp, context) != 0) {
+		error = ENOTDIR;
+		goto out;
+	}
+	if (!makestream) {
+		/* See if someone else already has it open. */
+		if (VNOP_LOOKUP(dvp, &svp, &cn, context) == 0) {
+			/* Double check existence by asking for size. */
+			VATTR_INIT(&va);
+			VATTR_WANTED(&va, va_data_size);
+			if (VNOP_GETATTR(svp, &va, context) == 0  &&
+			    VATTR_IS_SUPPORTED(&va, va_data_size)) {
+				goto out;  /* OK to use. */
+			}
+		}
+		
+		/* Otherwise make sure the resource fork data exists. */
+		error = vn_getxattr(vp, XATTR_RESOURCEFORK_NAME, NULL, &datasize,
+		                    XATTR_NOSECURITY, context);
+		/*
+		 * To maintain binary compatibility with legacy Carbon
+		 * emulated resource fork support, if the resource fork
+		 * doesn't exist but the Finder Info does,  then act as
+		 * if an empty resource fork is present (see 4724359).
+		 */
+		if ((error == ENOATTR) &&
+		    (vn_getxattr(vp, XATTR_FINDERINFO_NAME, NULL, &datasize,
+		                 XATTR_NOSECURITY, context) == 0)) {
+			datasize = 0;
+			error = 0;
+		} else {
+			if (error) {
+				goto out;
+			}
+	
+			/* If the resource fork exists, its size is expected to be non-zero. */
+			if (datasize == 0) {
+				error = ENOATTR;
+				goto out;
+			}
+		}
+	}
+	/* Create the shadow stream file. */
+	error = VNOP_CREATE(dvp, &svp, &cn, &va, context);
+	if (error == 0) {
+		*creator = 1;
+	} else if ((error == EEXIST) && !makestream) {
+		error = VNOP_LOOKUP(dvp, &svp, &cn, context);
+	}
+out:
+	if (dvp) {
+		vnode_put(dvp);
+	}
+	if (error) {
+		/* On errors, clean up shadow stream file. */
+		if (svp) {
+			vnode_put(svp);
+			svp = NULLVP;
+		}
+	}
+	*svpp = svp;
+	if (rsrcsize) {
+		*rsrcsize = datasize;
+	}
+	return (error);
+}
+
+
+static int
+default_getnamedstream(vnode_t vp, vnode_t *svpp, const char *name, enum nsoperation op, vfs_context_t context)
+{
+	vnode_t  svp = NULLVP;
+	uio_t auio = NULL;
+	caddr_t  bufptr = NULL;
+	size_t  bufsize = 0;
+	size_t  datasize = 0;
+	int  creator;
+	int  error;
+
+	/*
+	 * Only the "com.apple.ResourceFork" stream is supported here.
+	 */
+	if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		*svpp = NULLVP;
+		return (ENOATTR);
+	}
+retry:
+	/*
+	 * Obtain a shadow file for the resource fork I/O.
+	 */
+	error = getshadowfile(vp, &svp, 0, &datasize, &creator, context);
+	if (error) {
+		*svpp = NULLVP;
+		return (error);
+	}
+
+	/*
+	 * The creator of the shadow file provides its file data,
+	 * all other threads should wait until its ready.
+	 */
+	if (!creator) {
+		vnode_lock(svp);
+		if (svp->v_flag & VISNAMEDSTREAM) {
+			/* data is ready, go use it */
+			vnode_unlock(svp);
+			goto out;
+		} else {
+			/* its not ready, wait for it (sleep using v_parent as channel) */
+			msleep((caddr_t)&svp->v_parent, &svp->v_lock, PINOD | PDROP,
+				"getnamedstream", NULL);
+			vnode_put(svp);
+			svp = NULLVP;
+			goto retry;
+		}
+	}
+
+	/*
+	 * Copy the real resource fork data into shadow stream file.
+	 */
+	if (op == NS_OPEN && datasize != 0) {
+		size_t  offset;
+        	size_t  iosize;
+
+		iosize = bufsize = MIN(datasize, NS_IOBUFSIZE);
+		if (kmem_alloc(kernel_map, (vm_offset_t *)&bufptr, bufsize)) {
+			error = ENOMEM;
+			goto out;
+		}
+
+		auio = uio_create(1, 0, UIO_SYSSPACE32, UIO_READ);
+		offset = 0;
+
+		error = VNOP_OPEN(svp, 0, context);
+		if (error) {
+			goto out;
+		}
+		while (offset < datasize) {
+			size_t	tmpsize;
+
+			iosize = MIN(datasize - offset, iosize);
+
+			uio_reset(auio, offset, UIO_SYSSPACE32, UIO_READ);
+			uio_addiov(auio, (uintptr_t)bufptr, iosize);
+			error = vn_getxattr(vp, XATTR_RESOURCEFORK_NAME, auio, &tmpsize,
+			                    XATTR_NOSECURITY, context);
+			if (error) {
+				break;
+			}
+		
+			uio_reset(auio, offset, UIO_SYSSPACE32, UIO_WRITE);
+			uio_addiov(auio, (uintptr_t)bufptr, iosize);
+			error = VNOP_WRITE(svp, auio, 0, context);
+			if (error) {
+				break;
+			}
+			offset += iosize;
+		}
+		(void) VNOP_CLOSE(svp, 0, context);
+	}
+out:
+	/* Wake up anyone waiting for svp file content */
+	if (creator) {
+		if (error == 0) {
+			vnode_lock(svp);
+			svp->v_flag |= VISNAMEDSTREAM;
+			wakeup((caddr_t)&svp->v_parent);
+			vnode_unlock(svp);
+		} else {
+			/* On post create errors, get rid of shadow file. */
+			(void)vnode_relenamedstream(vp, svp, context);
+
+			wakeup((caddr_t)&svp->v_parent);
+		}
+	}
+
+	if (bufptr) {
+		kmem_free(kernel_map, (vm_offset_t)bufptr, bufsize);
+	}
+	if (auio) {
+		uio_free(auio);
+	}
+	if (error) {
+		/* On errors, clean up shadow stream file. */
+		if (svp) {
+			vnode_put(svp);
+			svp = NULLVP;
+		}
+	}
+	*svpp = svp;
+	return (error);
+}
+
+static int
+default_makenamedstream(vnode_t vp, vnode_t *svpp, const char *name, vfs_context_t context)
+{
+	int creator;
+	int error;
+
+	/*
+	 * Only the "com.apple.ResourceFork" stream is supported here.
+	 */
+	if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		*svpp = NULLVP;
+		return (ENOATTR);
+	}
+	error = getshadowfile(vp, svpp, 1, NULL, &creator, context);
+
+	/*
+	 * Wake up any waiters over in default_getnamedstream().
+	 */
+	if ((error == 0) && (*svpp != NULL) && creator) {
+		vnode_t svp = *svpp;
+
+		vnode_lock(svp);
+		svp->v_flag |= VISNAMEDSTREAM;
+		/* Wakeup any waiters on the v_parent channel */
+		wakeup((caddr_t)&svp->v_parent);
+		vnode_unlock(svp);
+	}
+	return (error);
+}
+
+static int 
+default_removenamedstream(vnode_t vp, const char *name, vfs_context_t context)
+{
+	/*
+	 * Only the "com.apple.ResourceFork" stream is supported here.
+	 */
+	if (bcmp(name, XATTR_RESOURCEFORK_NAME, sizeof(XATTR_RESOURCEFORK_NAME)) != 0) {
+		return (ENOATTR);
+	}
+	/*
+	 * XXX - what about other opened instances?
+	 */
+	return default_removexattr(vp, XATTR_RESOURCEFORK_NAME, 0, context);
+}
+
+static int
+get_shadow_dir(vnode_t *sdvpp, vfs_context_t context)
+{
+	vnode_t  dvp = NULLVP;
+	vnode_t  sdvp = NULLVP;
+	struct componentname  cn;
+	struct vnode_attr  va;
+	char tmpname[48];
+	uint32_t  tmp_fsid;
+	int  error;
+
+	/* Check if we've already created it. */
+	if (shadow_dvp != NULLVP) {
+		if ((error = vnode_getwithvid(shadow_dvp, shadow_vid))) {
+			shadow_dvp = NULLVP;
+		} else {
+			*sdvpp = shadow_dvp;
+			return (0);
+		}
+	}
+
+	/* Obtain the vnode for "/tmp" directory. */
+	if (vnode_lookup("/tmp", 0, &dvp, context) != 0) {
+		error = ENOTSUP;
+		goto out;
+	}
+
+	/* Create the shadow stream directory. */
+	snprintf(tmpname, sizeof(tmpname), ".vfs_rsrc_streams_%x%x",
+	         (unsigned int)rootvnode, shadow_sequence);
+	bzero(&cn, sizeof(cn));
+	cn.cn_nameiop = LOOKUP;
+	cn.cn_flags = ISLASTCN;
+	cn.cn_context = context;
+	cn.cn_pnbuf = tmpname;
+	cn.cn_pnlen = sizeof(tmpname);
+	cn.cn_nameptr = cn.cn_pnbuf;
+	cn.cn_namelen = strlen(tmpname);
+
+	/*
+	 * owned by root, only readable by root, hidden
+	 */
+	VATTR_INIT(&va);
+	VATTR_SET(&va, va_uid, 0);
+	VATTR_SET(&va, va_gid, 0);
+	VATTR_SET(&va, va_mode, S_IRUSR | S_IXUSR);
+	VATTR_SET(&va, va_type, VDIR);
+	VATTR_SET(&va, va_flags, UF_HIDDEN);
+	va.va_vaflags = VA_EXCLUSIVE;
+
+	error = VNOP_MKDIR(dvp, &sdvp, &cn, &va, context);
+	
+	/*
+	 * There can be only one winner for an exclusive create.
+	 */
+	if (error == 0) {
+		/* Take a long term ref to keep this dir around. */
+		error = vnode_ref(sdvp);
+		if (error == 0) {
+			shadow_dvp = sdvp;
+			shadow_vid = sdvp->v_id;
+		}
+	} else if (error == EEXIST) {
+		/* loser has to look up directory */
+		error = VNOP_LOOKUP(dvp, &sdvp, &cn, context);
+		if (error == 0) {
+			/* Make sure its in fact a directory */
+			if (sdvp->v_type != VDIR) {
+				goto baddir;
+			}
+			/* Obtain the fsid for /tmp directory */
+			VATTR_INIT(&va);
+			VATTR_WANTED(&va, va_fsid);
+			if (VNOP_GETATTR(dvp, &va, context) != 0  ||
+			    !VATTR_IS_SUPPORTED(&va, va_fsid)) {
+				goto baddir;
+			}
+			tmp_fsid = va.va_fsid;
+
+			VATTR_INIT(&va);
+			VATTR_WANTED(&va, va_uid);
+			VATTR_WANTED(&va, va_gid);
+			VATTR_WANTED(&va, va_mode);
+			VATTR_WANTED(&va, va_fsid);
+			VATTR_WANTED(&va, va_dirlinkcount);
+			VATTR_WANTED(&va, va_acl);
+			/* Provide defaults for attrs that may not be supported */
+			va.va_dirlinkcount = 1;
+			va.va_acl = (kauth_acl_t) KAUTH_FILESEC_NONE;
+
+			if (VNOP_GETATTR(sdvp, &va, context) != 0  ||
+			    !VATTR_IS_SUPPORTED(&va, va_uid)  ||
+			    !VATTR_IS_SUPPORTED(&va, va_gid)  ||
+			    !VATTR_IS_SUPPORTED(&va, va_mode)  ||
+			    !VATTR_IS_SUPPORTED(&va, va_fsid)) {
+				goto baddir;
+			}
+			/*
+			 * Make sure its what we want: 
+			 * 	- owned by root
+			 *	- not writable by anyone
+			 *	- on same file system as /tmp
+			 *	- not a hard-linked directory
+			 *	- no ACLs (they might grant write access)
+			 */
+			if ((va.va_uid != 0) || (va.va_gid != 0) ||
+			    (va.va_mode & (S_IWUSR | S_IRWXG | S_IRWXO)) ||
+			    (va.va_fsid != tmp_fsid) ||
+			    (va.va_dirlinkcount != 1) ||
+			     (va.va_acl != (kauth_acl_t) KAUTH_FILESEC_NONE)) {
+				goto baddir;
+			}
+		}
+	}
+out:
+	if (dvp) {
+		vnode_put(dvp);
+	}
+	if (error) {
+		/* On errors, clean up shadow stream directory. */
+		if (sdvp) {
+			vnode_put(sdvp);
+			sdvp = NULLVP;
+		}
+	}
+	*sdvpp = sdvp;
+	return (error);
+
+baddir:
+	/* This is not the dir we're looking for, move along */
+	++shadow_sequence;  /* try something else next time */
+	error = ENOTDIR;
+	goto out;
+}
+#endif
+
+
+
 /*
  * Default Implementation (Non-native EA) 
  */
@@ -408,7 +1139,7 @@ typedef struct attr_entry {
 } attr_entry_t;
 
 
-/* Header + entries must fit into 64K. Data may extend beyond 64k. */
+/* Header + entries must fit into 64K.  Data may extend beyond 64K. */
 typedef struct attr_header {
 	apple_double_header_t  appledouble;
 	u_int32_t   magic;        /* == ATTR_HDR_MAGIC */
@@ -456,7 +1187,7 @@ typedef struct attr_info {
 	size_t                 filesize;
 	size_t                 iosize;
 	u_int8_t               *rawdata;
-	size_t                 rawsize;  /* minimum of filesize or ATTR_MAX_HDR_SIZE */ 
+	size_t                 rawsize;  /* minimum of filesize or ATTR_MAX_HDR_SIZE */
 	apple_double_header_t  *filehdr;
 	apple_double_entry_t   *finderinfo;
 	apple_double_entry_t   *rsrcfork;
@@ -518,13 +1249,14 @@ static int  unlock_xattrfile(vnode_t xvp, vfs_context_t context);
 
 #else
 #define swap_adhdr(x)
-#define swap_attrhdr(x,y)
+#define swap_attrhdr(x, y)
 #endif
 
-static int  validate_attrhdr(attr_header_t *ah, size_t bufsize);
 static int  check_and_swap_attrhdr(attr_header_t *ah, attr_info_t* ainfop);
 static int  shift_data_down(vnode_t xvp, off_t start, size_t len, off_t delta, vfs_context_t context);
 static int  shift_data_up(vnode_t xvp, off_t start, size_t len, off_t delta, vfs_context_t context);
+
+
 /*
  * Sanity check and swap the header of an AppleDouble file.  Assumes the buffer
  * is in big endian (as it would exist on disk).  Verifies the following:
@@ -538,7 +1270,6 @@ static int  shift_data_up(vnode_t xvp, off_t start, size_t len, off_t delta, vfs
  * NOTE: Does not attempt to validate the extended attributes header that
  * may be embedded in the Finder Info entry.
  */
-
 static int check_and_swap_apple_double_header(attr_info_t *ainfop)
 {
 	int i, j;
@@ -546,56 +1277,56 @@ static int check_and_swap_apple_double_header(attr_info_t *ainfop)
 	u_int32_t entry_end;
 	size_t rawsize;
 	apple_double_header_t *header;
-
+	
 	rawsize = ainfop->rawsize;
 	header = (apple_double_header_t *) ainfop->rawdata;
-
+	
 	/* Is the file big enough to contain an AppleDouble header? */
 	if (rawsize < offsetof(apple_double_header_t, entries))
 		return ENOATTR;
-
+	
 	/* Swap the AppleDouble header fields to native order */
 	header->magic = SWAP32(header->magic);
 	header->version = SWAP32(header->version);
 	header->numEntries = SWAP16(header->numEntries);
-
+	
 	/* Sanity check the AppleDouble header fields */
 	if (header->magic != ADH_MAGIC ||
-			header->version != ADH_VERSION ||
-			header->numEntries < 1 ||
-			header->numEntries > 15) {
+	    header->version != ADH_VERSION ||
+	    header->numEntries < 1 ||
+	    header->numEntries > 15) {
 		return ENOATTR;
 	}
-
+	
 	/* Calculate where the entries[] array ends */
 	header_end = offsetof(apple_double_header_t, entries) +
 		header->numEntries * sizeof(apple_double_entry_t);
-
+	
 	/* Is the file big enough to contain the AppleDouble entries? */
 	if (rawsize < header_end) {
-		return ENOATTR;
+	    	return ENOATTR;
 	}
-
+	
 	/* Swap and sanity check each AppleDouble entry */
 	for (i=0; i<header->numEntries; i++) {
 		/* Swap the per-entry fields to native order */
 		header->entries[i].type   = SWAP32(header->entries[i].type);
 		header->entries[i].offset = SWAP32(header->entries[i].offset);
 		header->entries[i].length = SWAP32(header->entries[i].length);
-
+		
 		entry_end = header->entries[i].offset + header->entries[i].length;
-
+		
 		/*
 		 * Does the entry's content start within the header itself,
 		 * did the addition overflow, or does the entry's content
 		 * extend past the end of the file?
 		 */
 		if (header->entries[i].offset < header_end ||
-				entry_end < header->entries[i].offset  ||
-				entry_end > ainfop->filesize) {
+		    entry_end < header->entries[i].offset  ||
+		    entry_end > ainfop->filesize) {
 			return ENOATTR;
 		}
-
+		
 		/*
 		 * Does the current entry's content overlap with a previous
 		 * entry's content?
@@ -605,22 +1336,24 @@ static int check_and_swap_apple_double_header(attr_info_t *ainfop)
 		 * But we have already ensured N < 16, and N is almost always 2.
 		 * So there's no point in using a more complex algorithm.
 		 */
-
+		
 		for (j=0; j<i; j++) {
 			if (entry_end > header->entries[j].offset &&
-					header->entries[j].offset + header->entries[j].length > header->entries[i].offset) {
+			    header->entries[j].offset + header->entries[j].length > header->entries[i].offset) {
 				return ENOATTR;
 			}
 		}
 	}
-
+	
 	return 0;
 }
 
+
+
 /*
  * Retrieve the data of an extended attribute.
  */
-static int
+int
 default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size,
                  __unused int options, vfs_context_t context)
 {
@@ -709,14 +1442,14 @@ default_getxattr(vnode_t vp, const char *name, uio_t uio, size_t *size,
 	 * Search for attribute name in the header.
 	 */
 	for (i = 0; i < header->num_attrs && ATTR_VALID(entry, ainfo); i++) {
-		if (strncmp(entry->name, name, namelen) == 0) {
+		if (strncmp((const char *)entry->name, name, namelen) == 0) {
 			datalen = (size_t)entry->length;
 			if (uio == NULL) {
 				*size = datalen;
 				error = 0;
 				break;
 			}
-			if (uio_resid(uio) < datalen) {
+			if (uio_resid(uio) < (user_ssize_t)datalen) {
 				error = ERANGE;
 				break;
 			}
@@ -742,7 +1475,7 @@ out:
 /*
  * Set the data of an extended attribute.
  */
-static int
+int
 default_setxattr(vnode_t vp, const char *name, uio_t uio, int options, vfs_context_t context)
 {
 	vnode_t xvp = NULL;
@@ -760,14 +1493,47 @@ default_setxattr(vnode_t vp, const char *name, uio_t uio, int options, vfs_conte
 	int splitdata;
 	int fileflags;
 	int error;
-
+	char finfo[FINDERINFOSIZE];
+	
 	datalen = uio_resid(uio);
 	namelen = strlen(name) + 1;
 	entrylen = ATTR_ENTRY_LENGTH(namelen);
 
-	if (datalen > ATTR_MAX_SIZE) {
-		return (E2BIG);  /* EINVAL instead ? */
+	/*
+	 * By convention, Finder Info that is all zeroes is equivalent to not
+	 * having a Finder Info EA.  So if we're trying to set the Finder Info
+	 * to all zeroes, then delete it instead.  If a file didn't have an
+	 * AppleDouble file before, this prevents creating an AppleDouble file
+	 * with no useful content.
+	 *
+	 * If neither XATTR_CREATE nor XATTR_REPLACE were specified, we check
+	 * for all zeroes Finder Info before opening the AppleDouble file.
+	 * But if either of those options were specified, we need to open the
+	 * AppleDouble file to see whether there was already Finder Info (so we
+	 * can return an error if needed); this case is handled further below.
+	 *
+	 * NOTE: this copies the Finder Info data into the "finfo" local.
+	 */
+	if (bcmp(name, XATTR_FINDERINFO_NAME, sizeof(XATTR_FINDERINFO_NAME)) == 0) {
+		/*
+		 * TODO: check the XATTR_CREATE and XATTR_REPLACE flags.
+		 * That means we probably have to open_xattrfile and get_xattrinfo.
+		 */
+		if (uio_offset(uio) != 0 || datalen != FINDERINFOSIZE) {
+			return EINVAL;
+		}
+		error = uiomove(finfo, datalen, uio);
+		if (error)
+			return error;
+		if ((options & (XATTR_CREATE|XATTR_REPLACE)) == 0 &&
+		    bcmp(finfo, emptyfinfo, FINDERINFOSIZE) == 0) {
+			error = default_removexattr(vp, name, 0, context);
+			if (error == ENOATTR)
+				error = 0;
+			return error;
+		}
 	}
+	
 start:
 	/*
 	 * Open the file locked since setting an attribute
@@ -797,15 +1563,29 @@ start:
 				goto out;
 			}
 		}
-		if (uio_offset(uio) != 0 || datalen != FINDERINFOSIZE) {
-			error = EINVAL;
-			goto out;
+		if (options != 0 && bcmp(finfo, emptyfinfo, FINDERINFOSIZE) == 0) {
+			/*
+			 * Setting the Finder Info to all zeroes is equivalent to
+			 * removing it.  Close the xattr file and let
+			 * default_removexattr do the work (including deleting
+			 * the xattr file if there are no other xattrs).
+			 *
+			 * Note that we have to handle the case where the
+			 * Finder Info was already all zeroes, and we ignore
+			 * ENOATTR.
+			 *
+			 * The common case where options == 0 was handled above.
+			 */
+			rel_xattrinfo(&ainfo);
+			close_xattrfile(xvp, fileflags, context);
+			error = default_removexattr(vp, name, 0, context);
+			if (error == ENOATTR)
+				error = 0;
+			return error;
 		}
 		if (ainfo.finderinfo) {
 			attrdata = (u_int8_t *)ainfo.filehdr + ainfo.finderinfo->offset;
-			error = uiomove((caddr_t)attrdata, datalen, uio);
-			if (error)
-				goto out;
+			bcopy(finfo, attrdata, datalen);
 			ainfo.iosize = sizeof(attr_header_t);
 			error = write_xattrinfo(&ainfo);
 			goto out;
@@ -850,6 +1630,10 @@ start:
 		goto out;
 	}
 
+	if (datalen > ATTR_MAX_SIZE) {
+		return (E2BIG);  /* EINVAL instead ? */
+	}
+
 	if (ainfo.attrhdr == NULL) {
 		error = ENOATTR;
 		goto out;
@@ -867,7 +1651,7 @@ start:
 	 * See if attribute already exists.
 	 */
 	for (i = 0; i < header->num_attrs && ATTR_VALID(entry, ainfo); i++) {
-		if (strncmp(entry->name, name, namelen) == 0) {
+		if (strncmp((const char *)entry->name, name, namelen) == 0) {
 			found = 1;
 			break;
 		}
@@ -908,8 +1692,10 @@ start:
 			close_xattrfile(xvp, fileflags, context);
 			error = default_removexattr(vp, name, options, context);
 			if (error) {
-				goto out;
+				return (error);
 			}
+			/* Clear XATTR_REPLACE option since we just removed the attribute. */
+			options &= ~XATTR_REPLACE;
 			goto start; /* start over */
 		}
 
@@ -1055,7 +1841,7 @@ out:
 /*
  * Remove an extended attribute.
  */
-static int
+int
 default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_context_t context)
 {
 	vnode_t xvp = NULL;
@@ -1162,7 +1948,7 @@ default_removexattr(vnode_t vp, const char *name, __unused int options, vfs_cont
 	 * See if this attribute exists.
 	 */
 	for (i = 0; i < header->num_attrs && ATTR_VALID(entry, ainfo); i++) {
-		if (strncmp(entry->name, name, namelen) == 0) {
+		if (strncmp((const char *)entry->name, name, namelen) == 0) {
 			found = 1;
 			if ((i+1) == header->num_attrs)
 				lastone = 1;
@@ -1304,11 +2090,11 @@ default_listxattr(vnode_t vp, uio_t uio, size_t *size, __unused int options, vfs
 	if (ainfo.finderinfo && !ainfo.emptyfinderinfo) {
 		if (uio == NULL) {
 			*size += sizeof(XATTR_FINDERINFO_NAME);
-		} else if (uio_resid(uio) < sizeof(XATTR_FINDERINFO_NAME)) {
+		} else if (uio_resid(uio) < (user_ssize_t)sizeof(XATTR_FINDERINFO_NAME)) {
 			error = ERANGE;
 			goto out;
 		} else {
-			error = uiomove((caddr_t)XATTR_FINDERINFO_NAME,
+			error = uiomove(XATTR_FINDERINFO_NAME,
 			                sizeof(XATTR_FINDERINFO_NAME), uio);
 			if (error) {
 				error = ERANGE;
@@ -1321,11 +2107,11 @@ default_listxattr(vnode_t vp, uio_t uio, size_t *size, __unused int options, vfs
 	if (vnode_isreg(vp) && ainfo.rsrcfork) {
 		if (uio == NULL) {
 			*size += sizeof(XATTR_RESOURCEFORK_NAME);
-		} else if (uio_resid(uio) < sizeof(XATTR_RESOURCEFORK_NAME)) {
+		} else if (uio_resid(uio) < (user_ssize_t)sizeof(XATTR_RESOURCEFORK_NAME)) {
 			error = ERANGE;
 			goto out;
 		} else {
-			error = uiomove((caddr_t)XATTR_RESOURCEFORK_NAME,
+			error = uiomove(XATTR_RESOURCEFORK_NAME,
 			                sizeof(XATTR_RESOURCEFORK_NAME), uio);
 			if (error) {
 				error = ERANGE;
@@ -1338,8 +2124,8 @@ default_listxattr(vnode_t vp, uio_t uio, size_t *size, __unused int options, vfs
 	if (ainfo.attrhdr) {
 		count = ainfo.attrhdr->num_attrs;
 		for (i = 0, entry = ainfo.attr_entry; i < count && ATTR_VALID(entry, ainfo); i++) {
-			if (xattr_protected(entry->name) ||
-			    xattr_validatename(entry->name) != 0) {
+			if (xattr_protected((const char *)entry->name) ||
+			    xattr_validatename((const char *)entry->name) != 0) {
 				entry = ATTR_NEXT(entry);
 				continue;
 			}
@@ -1377,7 +2163,7 @@ open_xattrfile(vnode_t vp, int fileflags, vnode_t *xvpp, vfs_context_t context)
 	struct nameidata nd;
 	char smallname[64];
 	char *filename = NULL;
-	char *basename = NULL;
+	const char *basename = NULL;
 	size_t len;
 	errno_t error;
 	int opened = 0;
@@ -1388,7 +2174,7 @@ open_xattrfile(vnode_t vp, int fileflags, vnode_t *xvpp, vfs_context_t context)
 		 * For the root directory use "._." to hold the attributes.
 		 */
 		filename = &smallname[0];
-		sprintf(filename, "%s%s", ATTR_FILE_PREFIX, ".");
+		snprintf(filename, sizeof(smallname), "%s%s", ATTR_FILE_PREFIX, ".");
 		dvp = vp;  /* the "._." file resides in the root dir */
 		goto lookup;
 	}
@@ -1471,7 +2257,7 @@ lookup:
 				VATTR_SET(&va, va_gid, gid);
 
 			error = vn_create(dvp, &nd.ni_vp, &nd.ni_cnd, &va,
-			                  VN_CREATE_NOAUTH | VN_CREATE_NOINHERIT,
+			                  VN_CREATE_NOAUTH | VN_CREATE_NOINHERIT | VN_CREATE_NOLABEL,
 			                  context);
 			if (error == 0)
 			        xvp = nd.ni_vp;
@@ -1513,7 +2299,7 @@ lookup:
 		}
 	}
 	
-	if ( (error = VNOP_OPEN(xvp, fileflags, context))) {
+	if ( (error = VNOP_OPEN(xvp, fileflags & ~(O_EXLOCK | O_SHLOCK), context))) {
 		error = ENOATTR;
 		goto out;
 	}
@@ -1601,18 +2387,25 @@ remove_xattrfile(vnode_t xvp, vfs_context_t context)
 {
 	vnode_t dvp;
 	struct nameidata nd;
-	char *path;
+	char *path = NULL;
 	int pathlen;
 	int error = 0;
 
-	path = get_pathbuff();
+	MALLOC_ZONE(path, char *, MAXPATHLEN, M_NAMEI, M_WAITOK);
+	if (path == NULL)
+		return ENOMEM;
+
 	pathlen = MAXPATHLEN;
-	vn_getpath(xvp, path, &pathlen);
+	error = vn_getpath(xvp, path, &pathlen);
+	if (error) {
+		FREE_ZONE(path, MAXPATHLEN, M_NAMEI);
+		return (error);
+	}
 
 	NDINIT(&nd, DELETE, LOCKPARENT | NOFOLLOW | DONOTAUTH,
 	       UIO_SYSSPACE, CAST_USER_ADDR_T(path), context);
 	error = namei(&nd);
-	release_pathbuff(path);
+	FREE_ZONE(path, MAXPATHLEN, M_NAMEI);
 	if (error) {
 		return (error);
 	}
@@ -1626,6 +2419,7 @@ remove_xattrfile(vnode_t xvp, vfs_context_t context)
 
 	return (error);
 }
+
 /*
  * Read in and parse the AppleDouble header and entries, and the extended
  * attribute header and entries if any.  Populates the fields of ainfop
@@ -1694,13 +2488,13 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 	}
 	ainfop->rawsize = iosize - uio_resid(auio);
 	ainfop->rawdata = (u_int8_t *)buffer;
-
+	
 	filehdr = (apple_double_header_t *)buffer;
 
 	error = check_and_swap_apple_double_header(ainfop);
 	if (error)
 		goto bail;
-
+	
 	ainfop->filehdr = filehdr;  /* valid AppleDouble header */
 
 	/* rel_xattrinfo is responsible for freeing the header buffer */
@@ -1709,10 +2503,10 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 	/* Find the Finder Info and Resource Fork entries, if any */
 	for (i = 0; i < filehdr->numEntries; ++i) {
 		if (filehdr->entries[i].type == AD_FINDERINFO &&
-				filehdr->entries[i].length >= FINDERINFOSIZE) {
+		    filehdr->entries[i].length >= FINDERINFOSIZE) {
 			/* We found the Finder Info entry. */
 			ainfop->finderinfo = &filehdr->entries[i];
-
+			
 			/*
 			 * Is the Finder Info "empty" (all zeroes)?  If so,
 			 * we'll pretend like the Finder Info extended attribute
@@ -1725,12 +2519,10 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			 * Finder Info is non-empty.
 			 */
 			if (ainfop->finderinfo->offset + FINDERINFOSIZE <= ainfop->rawsize &&
-					bcmp((u_int8_t*)ainfop->filehdr + ainfop->finderinfo->offset, emptyfinfo, sizeof(emptyfinfo)) == 0) {
+			    bcmp((u_int8_t*)ainfop->filehdr + ainfop->finderinfo->offset, emptyfinfo, sizeof(emptyfinfo)) == 0) {
 				ainfop->emptyfinderinfo = 1;
 			}
-
 		}
-
 		if (filehdr->entries[i].type == AD_RESOURCE) {
 			/*
 			 * Ignore zero-length resource forks when getting.  If setting,
@@ -1739,7 +2531,7 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			 */
 			if (filehdr->entries[i].length == 0 && !setting)
 				continue;
-
+			
 			/*
 			 * Check to see if any "empty" resource fork is ours (i.e. is ignorable).
 			 *
@@ -1751,6 +2543,7 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 				u_int8_t  systemData[64];
 				int  rf_err;
 
+
 				/* Read the system data which starts at byte 16 */
 				rf_uio = uio_create(1, 0, UIO_SYSSPACE32, UIO_READ);
 				uio_addiov(rf_uio, (uintptr_t)systemData, sizeof(systemData));
@@ -1759,7 +2552,7 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 				uio_free(rf_uio);
 
 				if (rf_err != 0 ||
-						bcmp(systemData, RF_EMPTY_TAG, sizeof(RF_EMPTY_TAG)) == 0) {
+				    bcmp(systemData, RF_EMPTY_TAG, sizeof(RF_EMPTY_TAG)) == 0) {
 					continue;  /* skip this resource fork */
 				}
 			}
@@ -1771,7 +2564,7 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			continue;
 		}
 	}
-
+	
 	/*
 	 * See if this file looks like it is laid out correctly to contain
 	 * extended attributes.  If so, then do the following:
@@ -1784,9 +2577,9 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 	 *   (if any).
 	 */
 	if (filehdr->numEntries == 2 &&
-			ainfop->finderinfo == &filehdr->entries[0] &&
-			ainfop->rsrcfork == &filehdr->entries[1] &&
-			ainfop->finderinfo->offset == offsetof(apple_double_header_t, finfo)) {
+	    ainfop->finderinfo == &filehdr->entries[0] &&
+	    ainfop->rsrcfork == &filehdr->entries[1] &&
+	    ainfop->finderinfo->offset == offsetof(apple_double_header_t, finfo)) {
 		attr_header_t *attrhdr;
 		attrhdr = (attr_header_t *)filehdr;
 		/*
@@ -1797,7 +2590,7 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 		if (setting && ainfop->finderinfo->length == FINDERINFOSIZE) {
 			size_t delta;
 			size_t writesize;
-
+	
 			delta = ATTR_BUF_SIZE - (filehdr->entries[0].offset + FINDERINFOSIZE);
 			if (ainfop->rsrcfork && filehdr->entries[1].length) {
 				/* Make some room before existing resource fork. */
@@ -1809,24 +2602,24 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			} else {
 				/* Create a new, empty resource fork. */
 				rsrcfork_header_t *rsrcforkhdr;
-
+	
 				vnode_setsize(xvp, filehdr->entries[1].offset + delta, 0, context);
-
+	
 				/* Steal some space for an empty RF header. */
 				delta -= sizeof(rsrcfork_header_t);
-
+	
 				bzero(&attrhdr->appledouble.pad[0], delta);
 				rsrcforkhdr = (rsrcfork_header_t *)((char *)filehdr + filehdr->entries[1].offset + delta);
-
+	
 				/* Fill in Empty Resource Fork Header. */
 				init_empty_resource_fork(rsrcforkhdr);
-
+				
 				filehdr->entries[1].length = sizeof(rsrcfork_header_t);
 				writesize = ATTR_BUF_SIZE;
 			}
 			filehdr->entries[0].length += delta;
 			filehdr->entries[1].offset += delta;
-
+	
 			/* Fill in Attribute Header. */
 			attrhdr->magic       = ATTR_HDR_MAGIC;
 			attrhdr->debug_tag   = (u_int32_t)va.va_fileid;
@@ -1838,11 +2631,11 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			attrhdr->reserved[2] = 0;
 			attrhdr->flags       = 0;
 			attrhdr->num_attrs   = 0;
-
+	
 			/* Push out new header */
 			uio_reset(auio, 0, UIO_SYSSPACE32, UIO_WRITE);
 			uio_addiov(auio, (uintptr_t)filehdr, writesize);
-
+	
 			swap_adhdr(filehdr);	/* to big endian */
 			swap_attrhdr(attrhdr, ainfop);	/* to big endian */
 			error = VNOP_WRITE(xvp, auio, 0, context);
@@ -1850,7 +2643,6 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 			/* The attribute header gets swapped below. */
 		}
 	}
-
 	/*
 	 * Swap and sanity check the extended attribute header and
 	 * entries (if any).  The Finder Info content must be big enough
@@ -1866,9 +2658,10 @@ get_xattrinfo(vnode_t xvp, int setting, attr_info_t *ainfop, vfs_context_t conte
 	 * header was found.
 	 */
 	if (ainfop->finderinfo &&
-			ainfop->finderinfo == &filehdr->entries[0] &&
-			ainfop->finderinfo->length >= (sizeof(attr_header_t) - sizeof(apple_double_header_t))) {
+		ainfop->finderinfo == &filehdr->entries[0] &&
+		ainfop->finderinfo->length >= (sizeof(attr_header_t) - sizeof(apple_double_header_t))) {
 		attr_header_t *attrhdr = (attr_header_t*)filehdr;
+
 		if ((error = check_and_swap_attrhdr(attrhdr, ainfop)) == 0) {
 			ainfop->attrhdr = attrhdr;  /* valid attribute header */
 			/* First attr_entry starts immediately following attribute header */
@@ -1968,16 +2761,18 @@ write_xattrinfo(attr_info_t *ainfop)
 	uio_addiov(auio, (uintptr_t)ainfop->filehdr, ainfop->iosize);
 
 	swap_adhdr(ainfop->filehdr);
-	if (ainfop->attrhdr != NULL)
-	swap_attrhdr(ainfop->attrhdr, ainfop);
+	if (ainfop->attrhdr != NULL) {
+		swap_attrhdr(ainfop->attrhdr, ainfop);
+	}
 
 	error = VNOP_WRITE(ainfop->filevp, auio, 0, ainfop->context);
 
 	swap_adhdr(ainfop->filehdr);
-	if (ainfop->attrhdr != NULL)
-	swap_attrhdr(ainfop->attrhdr, ainfop);
+	if (ainfop->attrhdr != NULL) {
+		swap_attrhdr(ainfop->attrhdr, ainfop);
+	}
+	uio_free(auio);	
 
-	uio_free(auio);
 	return (error);
 }
 
@@ -2041,7 +2836,7 @@ swap_attrhdr(attr_header_t *ah, attr_info_t* info)
  * enough to contain the attr_header structure itself.  Therefore, we can
  * swap the header fields before sanity checking them.
  */
-	static int
+static int
 check_and_swap_attrhdr(attr_header_t *ah, attr_info_t *ainfop)
 {
 	attr_entry_t *ae;
@@ -2055,7 +2850,7 @@ check_and_swap_attrhdr(attr_header_t *ah, attr_info_t *ainfop)
 
 	if (SWAP32(ah->magic) != ATTR_HDR_MAGIC)
 		return EINVAL;
-
+	
 	/* Swap the basic header fields */
 	ah->magic	= SWAP32(ah->magic);
 	ah->debug_tag   = SWAP32 (ah->debug_tag);
@@ -2071,41 +2866,41 @@ check_and_swap_attrhdr(attr_header_t *ah, attr_info_t *ainfop)
 	 */
 	end = ah->data_start + ah->data_length;
 	if (ah->total_size > ainfop->finderinfo->offset + ainfop->finderinfo->length ||
-			end < ah->data_start ||
-			end > ah->total_size) {
+	    end < ah->data_start ||
+	    end > ah->total_size) {
 		return EINVAL;
 	}
-
+	
 	/*
 	 * Make sure each of the attr_entry_t's fits within total_size.
 	 */
 	buf_end = ainfop->rawdata + ah->total_size;
 	count = ah->num_attrs;
 	ae = (attr_entry_t *)(&ah[1]);
-
+	
 	for (i=0; i<count; i++) {
 		/* Make sure the fixed-size part of this attr_entry_t fits. */
 		if ((u_int8_t *) &ae[1] > buf_end)
 			return EINVAL;
-
+		
 		/* Make sure the variable-length name fits (+1 is for NUL terminator) */
 		/* TODO: Make sure namelen matches strnlen(name,namelen+1)? */
 		if (&ae->name[ae->namelen+1] > buf_end)
 			return EINVAL;
-
+		
 		/* Swap the attribute entry fields */
 		ae->offset	= SWAP32(ae->offset);
 		ae->length	= SWAP32(ae->length);
 		ae->flags	= SWAP16(ae->flags);
-
+		
 		/* Make sure the attribute content fits. */
 		end = ae->offset + ae->length;
 		if (end < ae->offset || end > ah->total_size)
 			return EINVAL;
-
+		
 		ae = ATTR_NEXT(ae);
 	}
-
+	
 	/*
 	 * TODO: Make sure the contents of attributes don't overlap the header
 	 * and don't overlap each other.  The hard part is that we don't know
@@ -2120,7 +2915,6 @@ check_and_swap_attrhdr(attr_header_t *ah, attr_info_t *ainfop)
 	return 0;
 }
 
-
 //
 // "start" & "end" are byte offsets in the file.
 // "to" is the byte offset we want to move the
@@ -2154,16 +2948,16 @@ shift_data_down(vnode_t xvp, off_t start, size_t len, off_t delta, vfs_context_t
 	}
 
 	for(pos=start+len-chunk; pos >= start; pos-=chunk) {
-		ret = vn_rdwr(UIO_READ, xvp, buff, chunk, pos, UIO_SYSSPACE, IO_NODELOCKED, ucred, &iolen, p);
+		ret = vn_rdwr(UIO_READ, xvp, buff, chunk, pos, UIO_SYSSPACE, IO_NODELOCKED|IO_NOAUTH, ucred, &iolen, p);
 		if (iolen != 0) {
-			printf("xattr:shift_data: error reading data @ %lld (read %d of %d) (%d)\n",
+			printf("xattr:shift_data: error reading data @ %lld (read %d of %lu) (%d)\n",
 				pos, ret, chunk, ret);
 			break;
 		}
 		
-		ret = vn_rdwr(UIO_WRITE, xvp, buff, chunk, pos + delta, UIO_SYSSPACE, IO_NODELOCKED, ucred, &iolen, p);
+		ret = vn_rdwr(UIO_WRITE, xvp, buff, chunk, pos + delta, UIO_SYSSPACE, IO_NODELOCKED|IO_NOAUTH, ucred, &iolen, p);
 		if (iolen != 0) {
-			printf("xattr:shift_data: error writing data @ %lld (wrote %d of %d) (%d)\n",
+			printf("xattr:shift_data: error writing data @ %lld (wrote %d of %lu) (%d)\n",
 				pos+delta, ret, chunk, ret);
 			break;
 		}
@@ -2209,16 +3003,16 @@ shift_data_up(vnode_t xvp, off_t start, size_t len, off_t delta, vfs_context_t c
 	}
 
 	for(pos = start; pos < end; pos += chunk) {
-		ret = vn_rdwr(UIO_READ, xvp, buff, chunk, pos, UIO_SYSSPACE, IO_NODELOCKED, ucred, &iolen, p);
+		ret = vn_rdwr(UIO_READ, xvp, buff, chunk, pos, UIO_SYSSPACE, IO_NODELOCKED|IO_NOAUTH, ucred, &iolen, p);
 		if (iolen != 0) {
-			printf("xattr:shift_data: error reading data @ %lld (read %d of %d) (%d)\n",
+			printf("xattr:shift_data: error reading data @ %lld (read %d of %lu) (%d)\n",
 				pos, ret, chunk, ret);
 			break;
 		}
 		
-		ret = vn_rdwr(UIO_WRITE, xvp, buff, chunk, pos - delta, UIO_SYSSPACE, IO_NODELOCKED, ucred, &iolen, p);
+		ret = vn_rdwr(UIO_WRITE, xvp, buff, chunk, pos - delta, UIO_SYSSPACE, IO_NODELOCKED|IO_NOAUTH, ucred, &iolen, p);
 		if (iolen != 0) {
-			printf("xattr:shift_data: error writing data @ %lld (wrote %d of %d) (%d)\n",
+			printf("xattr:shift_data: error writing data @ %lld (wrote %d of %lu) (%d)\n",
 				pos+delta, ret, chunk, ret);
 			break;
 		}
@@ -2240,25 +3034,29 @@ static int
 lock_xattrfile(vnode_t xvp, short locktype, vfs_context_t context)
 {
 	struct flock lf;
+	int error;
 
 	lf.l_whence = SEEK_SET;
 	lf.l_start = 0;
 	lf.l_len = 0;
 	lf.l_type = locktype; /* F_WRLCK or F_RDLCK */
 	/* Note: id is just a kernel address that's not a proc */
-	return  VNOP_ADVLOCK(xvp, (caddr_t)xvp, F_SETLK, &lf, F_FLOCK, context);
+	error = VNOP_ADVLOCK(xvp, (caddr_t)xvp, F_SETLK, &lf, F_FLOCK, context);
+	return (error == ENOTSUP ? 0 : error);
 }
 
 static int
 unlock_xattrfile(vnode_t xvp, vfs_context_t context)
 {
 	struct flock lf;
+	int error;
 
 	lf.l_whence = SEEK_SET;
 	lf.l_start = 0;
 	lf.l_len = 0;
 	lf.l_type = F_UNLCK;
 	/* Note: id is just a kernel address that's not a proc */
-	return  VNOP_ADVLOCK(xvp, (caddr_t)xvp, F_UNLCK, &lf, F_FLOCK, context);
+	error = VNOP_ADVLOCK(xvp, (caddr_t)xvp, F_UNLCK, &lf, F_FLOCK, context);
+	return (error == ENOTSUP ? 0 : error);
 }
 
diff --git a/bsd/vfs/vnode_if.c b/bsd/vfs/vnode_if.c
index 066760dda..df30827ae 100644
--- a/bsd/vfs/vnode_if.c
+++ b/bsd/vfs/vnode_if.c
@@ -2,23 +2,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved
@@ -78,7 +84,8 @@ struct vnodeop_desc vnop_default_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 
@@ -96,7 +103,7 @@ struct vnodeop_desc vnop_lookup_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_lookup_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_lookup_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_create_vp_offsets[] = {
@@ -113,7 +120,7 @@ struct vnodeop_desc vnop_create_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_create_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_create_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_whiteout_vp_offsets[] = {
@@ -130,7 +137,7 @@ struct vnodeop_desc vnop_whiteout_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_whiteout_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_whiteout_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_mknod_vp_offsets[] = {
@@ -147,7 +154,7 @@ struct vnodeop_desc vnop_mknod_desc = {
        VDESC_NO_OFFSET,
        VOPARG_OFFSETOF(struct vnop_mknod_args, a_cnp),
        VOPARG_OFFSETOF(struct vnop_mknod_args, a_context),
-       NULL,
+       NULL
 };
 
 int vnop_open_vp_offsets[] = {
@@ -164,7 +171,7 @@ struct vnodeop_desc vnop_open_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_open_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_close_vp_offsets[] = {
@@ -181,7 +188,7 @@ struct vnodeop_desc vnop_close_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_close_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_access_vp_offsets[] = {
@@ -198,7 +205,7 @@ struct vnodeop_desc vnop_access_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_close_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_getattr_vp_offsets[] = {
@@ -215,7 +222,7 @@ struct vnodeop_desc vnop_getattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_getattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_setattr_vp_offsets[] = {
@@ -232,41 +239,7 @@ struct vnodeop_desc vnop_setattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_setattr_args, a_context),
-	NULL,
-};
-
-int vnop_getattrlist_vp_offsets[] = {
-	VOPARG_OFFSETOF(struct vnop_getattrlist_args,a_vp),
-	VDESC_NO_OFFSET
-};
-struct vnodeop_desc vnop_getattrlist_desc = {
-	0,
-	"vnop_getattrlist",
-	0,
-	vnop_getattrlist_vp_offsets,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VOPARG_OFFSETOF(struct vnop_getattrlist_args, a_context),
-	NULL,
-};
-
-int vnop_setattrlist_vp_offsets[] = {
-	VOPARG_OFFSETOF(struct vnop_setattrlist_args,a_vp),
-	VDESC_NO_OFFSET
-};
-struct vnodeop_desc vnop_setattrlist_desc = {
-	0,
-	"vnop_setattrlist",
-	0,
-	vnop_setattrlist_vp_offsets,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VDESC_NO_OFFSET,
-	VOPARG_OFFSETOF(struct vnop_setattrlist_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_read_vp_offsets[] = {
@@ -283,7 +256,7 @@ struct vnodeop_desc vnop_read_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_read_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_write_vp_offsets[] = {
@@ -300,7 +273,7 @@ struct vnodeop_desc vnop_write_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_write_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_ioctl_vp_offsets[] = {
@@ -317,7 +290,7 @@ struct vnodeop_desc vnop_ioctl_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_ioctl_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_select_vp_offsets[] = {
@@ -334,8 +307,7 @@ struct vnodeop_desc vnop_select_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_select_args, a_context),
-	VDESC_NO_OFFSET,
-	NULL,
+	NULL
 };
 
 int vnop_exchange_vp_offsets[] = {
@@ -353,7 +325,7 @@ struct vnodeop_desc vnop_exchange_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_exchange_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_kqfilt_add_vp_offsets[] = {
@@ -370,7 +342,7 @@ struct vnodeop_desc vnop_kqfilt_add_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_kqfilt_add_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_kqfilt_remove_vp_offsets[] = {
@@ -387,6 +359,23 @@ struct vnodeop_desc vnop_kqfilt_remove_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_kqfilt_remove_args, a_context),
+	NULL
+};
+
+int vnop_setlabel_vp_offsets[] = {
+	VOPARG_OFFSETOF(struct vnop_setlabel_args,a_vp),
+	VDESC_NO_OFFSET
+};
+struct vnodeop_desc vnop_setlabel_desc = {
+	0,
+	"vnop_setlabel",
+	0,
+	vnop_setlabel_vp_offsets,
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VOPARG_OFFSETOF(struct vnop_setlabel_args, a_context),
 	NULL,
 };
 
@@ -403,7 +392,8 @@ struct vnodeop_desc vnop_revoke_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 
@@ -420,7 +410,8 @@ struct vnodeop_desc vnop_mmap_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 
@@ -437,7 +428,8 @@ struct vnodeop_desc vnop_mnomap_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 
@@ -455,7 +447,7 @@ struct vnodeop_desc vnop_fsync_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_fsync_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_remove_vp_offsets[] = {
@@ -473,7 +465,7 @@ struct vnodeop_desc vnop_remove_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_remove_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_remove_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_link_vp_offsets[] = {
@@ -491,7 +483,7 @@ struct vnodeop_desc vnop_link_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_link_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_link_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_rename_vp_offsets[] = {
@@ -511,7 +503,7 @@ struct vnodeop_desc vnop_rename_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_rename_args, a_fcnp),
 	VOPARG_OFFSETOF(struct vnop_rename_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_mkdir_vp_offsets[] = {
@@ -528,7 +520,7 @@ struct vnodeop_desc vnop_mkdir_desc = {
        VDESC_NO_OFFSET,
        VOPARG_OFFSETOF(struct vnop_mkdir_args, a_cnp),
        VOPARG_OFFSETOF(struct vnop_mkdir_args, a_context),
-       NULL,
+       NULL
 };
 
 int vnop_rmdir_vp_offsets[] = {
@@ -546,7 +538,7 @@ struct vnodeop_desc vnop_rmdir_desc = {
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_rmdir_args, a_cnp),
 	VOPARG_OFFSETOF(struct vnop_rmdir_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_symlink_vp_offsets[] = {
@@ -563,7 +555,7 @@ struct vnodeop_desc vnop_symlink_desc = {
        VDESC_NO_OFFSET,
        VOPARG_OFFSETOF(struct vnop_symlink_args, a_cnp),
        VOPARG_OFFSETOF(struct vnop_symlink_args, a_context),
-       NULL,
+       NULL
 };
 
 int vnop_readdir_vp_offsets[] = {
@@ -580,7 +572,7 @@ struct vnodeop_desc vnop_readdir_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_symlink_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_readdirattr_vp_offsets[] = {
@@ -597,7 +589,7 @@ struct vnodeop_desc vnop_readdirattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_readdirattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_readlink_vp_offsets[] = {
@@ -614,7 +606,7 @@ struct vnodeop_desc vnop_readlink_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_readlink_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_inactive_vp_offsets[] = {
@@ -631,7 +623,7 @@ struct vnodeop_desc vnop_inactive_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_inactive_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_reclaim_vp_offsets[] = {
@@ -648,7 +640,7 @@ struct vnodeop_desc vnop_reclaim_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_reclaim_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_pathconf_vp_offsets[] = {
@@ -665,7 +657,7 @@ struct vnodeop_desc vnop_pathconf_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_pathconf_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_advlock_vp_offsets[] = {
@@ -682,7 +674,7 @@ struct vnodeop_desc vnop_advlock_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_advlock_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_allocate_vp_offsets[] = {
@@ -699,7 +691,7 @@ struct vnodeop_desc vnop_allocate_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_allocate_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_pagein_vp_offsets[] = {
@@ -716,7 +708,7 @@ struct vnodeop_desc vnop_pagein_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_pagein_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_pageout_vp_offsets[] = {
@@ -733,7 +725,7 @@ struct vnodeop_desc vnop_pageout_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_pageout_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_searchfs_vp_offsets[] = {
@@ -749,7 +741,8 @@ struct vnodeop_desc vnop_searchfs_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 int vnop_copyfile_vp_offsets[] = {
@@ -767,7 +760,8 @@ struct vnodeop_desc vnop_copyfile_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_copyfile_args, a_tcnp),
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 int vop_getxattr_vp_offsets[] = {
@@ -784,7 +778,7 @@ struct vnodeop_desc vnop_getxattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_getxattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vop_setxattr_vp_offsets[] = {
@@ -801,7 +795,7 @@ struct vnodeop_desc vnop_setxattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_setxattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vop_removexattr_vp_offsets[] = {
@@ -818,7 +812,7 @@ struct vnodeop_desc vnop_removexattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_removexattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vop_listxattr_vp_offsets[] = {
@@ -835,7 +829,7 @@ struct vnodeop_desc vnop_listxattr_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VOPARG_OFFSETOF(struct vnop_listxattr_args, a_context),
-	NULL,
+	NULL
 };
 
 int vnop_blktooff_vp_offsets[] = {
@@ -851,7 +845,8 @@ struct vnodeop_desc vnop_blktooff_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 int vnop_offtoblk_vp_offsets[] = {
@@ -867,7 +862,8 @@ struct vnodeop_desc vnop_offtoblk_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 int vnop_blockmap_vp_offsets[] = {
@@ -883,8 +879,67 @@ struct vnodeop_desc vnop_blockmap_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
+};
+
+#if NAMEDSTREAMS
+int vnop_getnamedstream_vp_offsets[] = {
+	VOPARG_OFFSETOF(struct vnop_getnamedstream_args, a_vp),
+	VDESC_NO_OFFSET
+};
+struct vnodeop_desc vnop_getnamedstream_desc = {
+	0,
+	"vnop_getnamedstream",
+	0,
+	vnop_getnamedstream_vp_offsets,
+	VOPARG_OFFSETOF(struct vnop_getnamedstream_args, a_svpp),
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VOPARG_OFFSETOF(struct vnop_getnamedstream_args, a_name),
+	VOPARG_OFFSETOF(struct vnop_getnamedstream_args, a_context),
+	NULL
+};
+
+int vnop_makenamedstream_vp_offsets[] = {
+	VOPARG_OFFSETOF(struct vnop_makenamedstream_args, a_vp),
+	VDESC_NO_OFFSET
+};
+struct vnodeop_desc vnop_makenamedstream_desc = {
+	0,
+	"vnop_makenamedstream",
+	0, /* flags */
+	vnop_makenamedstream_vp_offsets,
+	VOPARG_OFFSETOF(struct vnop_makenamedstream_args, a_svpp),
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VOPARG_OFFSETOF(struct vnop_makenamedstream_args, a_name),
+	VOPARG_OFFSETOF(struct vnop_makenamedstream_args, a_context),
+	NULL
+};
+
+int vnop_removenamedstream_vp_offsets[] = {
+	VOPARG_OFFSETOF(struct vnop_removenamedstream_args, a_vp),
+	VDESC_NO_OFFSET
+};
+struct vnodeop_desc vnop_removenamedstream_desc = {
+	0,
+	"vnop_removenamedstream",
+	0,
+	vnop_removenamedstream_vp_offsets,
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VDESC_NO_OFFSET,
+	VOPARG_OFFSETOF(struct vnop_removenamedstream_args, a_name),
+	VOPARG_OFFSETOF(struct vnop_removenamedstream_args, a_context),
+	NULL
 };
+#else
+/* These symbols are in the exports list so they need to always be defined. */
+int vnop_getnamedstream_desc;
+int vnop_makenamedstream_desc;
+int vnop_removenamedstream_desc;
+#endif
 
 /* Special cases: */
 
@@ -900,7 +955,8 @@ struct vnodeop_desc vnop_strategy_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 int vnop_bwrite_vp_offsets[] = {
@@ -915,7 +971,8 @@ struct vnodeop_desc vnop_bwrite_desc = {
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
 	VDESC_NO_OFFSET,
-	NULL,
+	VDESC_NO_OFFSET,
+	NULL
 };
 
 /* End of special cases. */
@@ -934,8 +991,6 @@ struct vnodeop_desc *vfs_op_descs[] = {
 	&vnop_access_desc,
 	&vnop_getattr_desc,
 	&vnop_setattr_desc,
-	&vnop_getattrlist_desc,
-	&vnop_setattrlist_desc,
 	&vnop_read_desc,
 	&vnop_write_desc,
 	&vnop_ioctl_desc,
@@ -943,6 +998,7 @@ struct vnodeop_desc *vfs_op_descs[] = {
 	&vnop_exchange_desc,
 	&vnop_kqfilt_add_desc,
 	&vnop_kqfilt_remove_desc,
+	&vnop_setlabel_desc,
 	&vnop_revoke_desc,
 	&vnop_mmap_desc,
 	&vnop_mnomap_desc,
@@ -972,6 +1028,11 @@ struct vnodeop_desc *vfs_op_descs[] = {
 	&vnop_blktooff_desc,
 	&vnop_offtoblk_desc,
 	&vnop_blockmap_desc,
+#if NAMEDSTREAMS
+	&vnop_getnamedstream_desc,
+	&vnop_makenamedstream_desc,
+	&vnop_removenamedstream_desc,
+#endif
 	NULL
 };
 
diff --git a/bsd/vfs/vnode_if.sh b/bsd/vfs/vnode_if.sh
index 610af8127..ff699a78e 100644
--- a/bsd/vfs/vnode_if.sh
+++ b/bsd/vfs/vnode_if.sh
@@ -1,9 +1,9 @@
 #!/bin/sh -
 copyright='
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * The contents of this file constitute Original Code as defined in and
  * are subject to the Apple Public Source License Version 1.1 (the
@@ -19,7 +19,7 @@ copyright='
  * License for the specific language governing rights and limitations
  * under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved
diff --git a/bsd/vm/dp_backing_file.c b/bsd/vm/dp_backing_file.c
index 92eb65eb9..6b8467198 100644
--- a/bsd/vm/dp_backing_file.c
+++ b/bsd/vm/dp_backing_file.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #include <sys/param.h>
@@ -57,8 +69,9 @@
 #include <vm/vm_kern.h>
 #include <vm/vnode_pager.h>
 #include <vm/vm_protos.h>
-
-extern thread_t current_act(void);
+#if CONFIG_MACF
+#include <security/mac_framework.h>
+#endif
 
 /*
  * temporary support for delayed instantiation
@@ -153,7 +166,6 @@ macx_swapon(
 	int			size = args->size;
 	vnode_t			vp = (vnode_t)NULL; 
 	struct nameidata 	nd, *ndp;
-	struct proc		*p =  current_proc();
 	register int		error;
 	kern_return_t		kr;
 	mach_port_t		backing_store;
@@ -161,10 +173,8 @@ macx_swapon(
 	int			i;
 	boolean_t		funnel_state;
 	off_t			file_size;
-	struct vfs_context	context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t		ctx = vfs_context_current();
+	struct proc		*p =  current_proc();
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPON);
 	AUDIT_ARG(value, args->priority);
@@ -185,7 +195,7 @@ macx_swapon(
 	 */
 	NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
 	       ((IS_64BIT_PROCESS(p)) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-	       CAST_USER_ADDR_T(args->filename), &context);
+	       CAST_USER_ADDR_T(args->filename), ctx);
 
 	if ((error = namei(ndp)))
 		goto swapon_bailout;
@@ -196,14 +206,20 @@ macx_swapon(
 		error = EINVAL;
 		goto swapon_bailout;
 	}
-	UBCINFOCHECK("macx_swapon", vp);
 
 	/* get file size */
-	if ((error = vnode_size(vp, &file_size, &context)) != 0)
+	if ((error = vnode_size(vp, &file_size, ctx)) != 0)
+		goto swapon_bailout;
+#if CONFIG_MACF
+	vnode_lock(vp);
+	error = mac_system_check_swapon(vfs_context_ucred(ctx), vp);
+	vnode_unlock(vp);
+	if (error)
 		goto swapon_bailout;
+#endif
 
 	/* resize to desired size if it's too small */
-	if ((file_size < (off_t)size) && ((error = vnode_setsize(vp, (off_t)size, 0, &context)) != 0))
+	if ((file_size < (off_t)size) && ((error = vnode_setsize(vp, (off_t)size, 0, ctx)) != 0))
 		goto swapon_bailout;
 
 	/* add new backing store to list */
@@ -305,10 +321,7 @@ macx_swapoff(
 	int			i;
 	int			error;
 	boolean_t		funnel_state;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_SWAPOFF);
 
@@ -324,7 +337,7 @@ macx_swapoff(
 	 */
 	NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNPATH1,
 	       ((IS_64BIT_PROCESS(p)) ? UIO_USERSPACE64 : UIO_USERSPACE32),
-	       CAST_USER_ADDR_T(args->filename), &context);
+	       CAST_USER_ADDR_T(args->filename), ctx);
 
 	if ((error = namei(ndp)))
 		goto swapoff_bailout;
@@ -335,6 +348,13 @@ macx_swapoff(
 		error = EINVAL;
 		goto swapoff_bailout;
 	}
+#if CONFIG_MACF
+	vnode_lock(vp);
+	error = mac_system_check_swapoff(vfs_context_ucred(ctx), vp);
+	vnode_unlock(vp);
+	if (error)
+		goto swapoff_bailout;
+#endif
 
 	for(i = 0; i < MAX_BACKING_STORE; i++) {
 		if(bs_port_table[i].vp == vp) {
diff --git a/bsd/vm/vm_pager.h b/bsd/vm/vm_pager.h
index 633318cda..2a146275d 100644
--- a/bsd/vm/vm_pager.h
+++ b/bsd/vm/vm_pager.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
diff --git a/bsd/vm/vm_unix.c b/bsd/vm/vm_unix.c
index b1bcfddd5..56c2201dc 100644
--- a/bsd/vm/vm_unix.c
+++ b/bsd/vm/vm_unix.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -25,11 +31,13 @@
  * All rights reserved.  The CMU software License Agreement specifies
  * the terms and conditions for use and redistribution.
  */
-
 /*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
-
 #include <meta_features.h>
 
 #include <kern/task.h>
@@ -37,11 +45,14 @@
 #include <kern/debug.h>
 #include <kern/lock.h>
 #include <mach/mach_traps.h>
+#include <mach/port.h>
+#include <mach/task.h>
+#include <mach/task_access.h>
+#include <mach/task_special_ports.h>
 #include <mach/time_value.h>
 #include <mach/vm_map.h>
 #include <mach/vm_param.h>
 #include <mach/vm_prot.h>
-#include <mach/port.h>
 
 #include <sys/file_internal.h>
 #include <sys/param.h>
@@ -73,16 +84,44 @@
 
 #include <machine/spl.h>
 
-#include <mach/shared_memory_server.h>
-#include <vm/vm_shared_memory_server.h>
+#include <mach/shared_region.h>
+#include <vm/vm_shared_region.h>
 
 #include <vm/vm_protos.h>
 
+/*
+ * Sysctl's related to data/stack execution.  See osfmk/vm/vm_map.c
+ */
+
+extern int allow_stack_exec, allow_data_exec;
+
+SYSCTL_INT(_vm, OID_AUTO, allow_stack_exec, CTLFLAG_RW, &allow_stack_exec, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, allow_data_exec, CTLFLAG_RW, &allow_data_exec, 0, "");
+
+#if CONFIG_NO_PRINTF_STRINGS
+void
+log_stack_execution_failure(__unused addr64_t a, __unused vm_prot_t b)
+{
+}
+#else
+static const char *prot_values[] = {
+	"none",
+	"read-only",
+	"write-only",
+	"read-write",
+	"execute-only",
+	"read-execute",
+	"write-execute",
+	"read-write-execute"
+};
+
 void
-log_nx_failure(addr64_t vaddr, vm_prot_t prot)
+log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot)
 {
-        printf("NX failure: %s  -  vaddr=%qx,  prot=%x\n", current_proc()->p_comm, vaddr, prot);
+	printf("Data/Stack execution not permitted: %s[pid %d] at virtual address 0x%qx, protections were %s\n", 
+		current_proc()->p_comm, current_proc()->p_pid, vaddr, prot_values[prot & VM_PROT_ALL]);
 }
+#endif
 
 
 int
@@ -295,7 +334,7 @@ fuulong(user_addr_t addr)
 }
 
 int
-swapon(__unused struct proc *procp, __unused struct swapon_args *uap, __unused int *retval)
+swapon(__unused proc_t procp, __unused struct swapon_args *uap, __unused int *retval)
 {
 	return(ENOTSUP);
 }
@@ -307,16 +346,14 @@ pid_for_task(
 {
 	mach_port_name_t	t = args->t;
 	user_addr_t		pid_addr  = args->pid;  
-	struct proc * p;
+	proc_t p;
 	task_t		t1;
 	int	pid = -1;
 	kern_return_t	err = KERN_SUCCESS;
-	boolean_t funnel_state;
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_PIDFORTASK);
 	AUDIT_ARG(mach_port1, t);
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
 	t1 = port_name_to_task(t);
 
 	if (t1 == TASK_NULL) {
@@ -335,11 +372,95 @@ pid_for_task(
 pftout:
 	AUDIT_ARG(pid, pid);
 	(void) copyout((char *) &pid, pid_addr, sizeof(int));
-	thread_funnel_set(kernel_flock, funnel_state);
 	AUDIT_MACH_SYSCALL_EXIT(err);
 	return(err);
 }
 
+/* 
+ *
+ * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
+ * tfp_policy = KERN_TFP_POLICY_DEFAULT; default mode: all posix checks and upcall via task port for authentication
+ *
+ */
+static  int tfp_policy = KERN_TFP_POLICY_DEFAULT;
+
+/*
+ *	Routine:	task_for_pid_posix_check
+ *	Purpose:
+ *			Verify that the current process should be allowed to
+ *			get the target process's task port. This is only 
+ *			permitted if:
+ *			- The current process is root
+ *			OR all of the following are true:
+ *			- The target process's real, effective, and saved uids
+ *			  are the same as the current proc's euid,
+ *			- The target process's group set is a subset of the
+ *			  calling process's group set, and
+ *			- The target process hasn't switched credentials.
+ *
+ *	Returns:	TRUE: permitted
+ *			FALSE: denied
+ */
+static int
+task_for_pid_posix_check(proc_t target)
+{
+	kauth_cred_t targetcred, mycred;
+	uid_t myuid;
+	int allowed; 
+
+	/* No task_for_pid on bad targets */
+	if (target == PROC_NULL || target->p_stat == SZOMB) {
+		return FALSE;
+	}
+
+	mycred = kauth_cred_get();
+	myuid = kauth_cred_getuid(mycred);
+
+	/* If we're running as root, the check passes */
+	if (kauth_cred_issuser(mycred))
+		return TRUE;
+
+	/* We're allowed to get our own task port */
+	if (target == current_proc())
+		return TRUE;
+
+	/* 
+	 * Under DENY, only root can get another proc's task port,
+	 * so no more checks are needed.
+	 */
+	if (tfp_policy == KERN_TFP_POLICY_DENY) { 
+		return FALSE;
+	}
+
+	targetcred = kauth_cred_proc_ref(target);
+	allowed = TRUE;
+
+	/* Do target's ruid, euid, and saved uid match my euid? */
+	if ((kauth_cred_getuid(targetcred) != myuid) || 
+			(targetcred->cr_ruid != myuid) ||
+			(targetcred->cr_svuid != myuid)) {
+		allowed = FALSE;
+		goto out;
+	}
+
+	/* Are target's groups a subset of my groups? */
+	if (kauth_cred_gid_subset(targetcred, mycred, &allowed) ||
+			allowed == 0) {
+		allowed = FALSE;
+		goto out;
+	}
+
+	/* Has target switched credentials? */
+	if (target->p_flag & P_SUGID) {
+		allowed = FALSE;
+		goto out;
+	}
+	
+out:
+	kauth_cred_unref(&targetcred);
+	return allowed;
+}
+
 /*
  *	Routine:	task_for_pid
  *	Purpose:
@@ -351,19 +472,6 @@ pftout:
  *
  * XXX This should be a BSD system call, not a Mach trap!!!
  */
-/* 
- *
- * tfp_policy = KERN_TFP_POLICY_DENY; Deny Mode: None allowed except for self
- * tfp_policy = KERN_TFP_POLICY_PERMISSIVE; Permissive Mode: all permissive; related ones allowed or privileged
- * tfp_policy = KERN_TFP_POLICY_RESTRICTED; Restricted Mode: self access allowed; setgid (to tfp_group) are allowed for other tasks
- *
- */
-static  int tfp_policy = KERN_TFP_POLICY_RESTRICTED;
-/* the groutp is inited to kmem group and is modifiable by sysctl */
-static int tfp_group_inited = 0; /* policy groups are loaded ... */
-static  gid_t tfp_group_ronly = 0; /* procview group */
-static  gid_t tfp_group_rw = 0; /* procmod group */
-
 kern_return_t
 task_for_pid(
 	struct task_for_pid_args *args)
@@ -372,151 +480,93 @@ task_for_pid(
 	int			pid = args->pid;
 	user_addr_t		task_addr = args->t;
 	struct uthread		*uthread;
-	struct proc	*p;
-	struct proc *p1;
-	task_t		t1;
-	mach_port_name_t	tret;
+	proc_t 			p = PROC_NULL;
+	task_t			t1 = TASK_NULL;
+	mach_port_name_t	tret = MACH_PORT_NULL;
+ 	ipc_port_t 		tfpport;
 	void * sright;
 	int error = 0;
-	int is_member = 0;
-	boolean_t funnel_state;
-	boolean_t ispermitted = FALSE;
-#if DIAGNOSTIC
-	char procname[MAXCOMLEN+1];
-#endif /* DIAGNOSTIC */
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_TASKFORPID);
 	AUDIT_ARG(pid, pid);
 	AUDIT_ARG(mach_port1, target_tport);
 
+#if defined(SECURE_KERNEL)
+	if (0 == pid) {
+		(void ) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
+		AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
+		return(KERN_FAILURE);
+	}
+#endif
+
 	t1 = port_name_to_task(target_tport);
 	if (t1 == TASK_NULL) {
-		(void ) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
+		(void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
 		AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
 		return(KERN_FAILURE);
 	} 
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
-	p1 = current_proc();
 
 	/*
 	 * Delayed binding of thread credential to process credential, if we
 	 * are not running with an explicitly set thread credential.
 	 */
 	uthread = get_bsdthread_info(current_thread());
-	if (uthread->uu_ucred != p1->p_ucred &&
-	    (uthread->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = uthread->uu_ucred;
-		proc_lock(p1);
-		kauth_cred_ref(p1->p_ucred);
-		uthread->uu_ucred = p1->p_ucred;
-		proc_unlock(p1);
-		if (IS_VALID_CRED(old))
-			kauth_cred_unref(&old);
-	}
+	kauth_cred_uthread_update(uthread, current_proc());
 
-	p = pfind(pid);
+	p = proc_find(pid);
 	AUDIT_ARG(process, p);
 
-	/*
-	 * XXX p_ucred check can be bogus in multithreaded processes,
-	 * XXX unless the funnel is held.
-	 */
-	switch (tfp_policy) {
-
-		case KERN_TFP_POLICY_PERMISSIVE:
-			/* self or suser or related ones */
-			if ((p != (struct proc *) 0)
-				&& (p->p_stat != SZOMB)
-				&& (p1 != (struct proc *) 0)
-				&& (
-					(p1 == p)
-					|| !(suser(kauth_cred_get(), 0))
-					|| ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
-						((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid))
-						&& ((p->p_flag & P_SUGID) == 0))
-					)
-				)
-					ispermitted = TRUE;
-			break;
-
-		case KERN_TFP_POLICY_RESTRICTED:
-			/* self or suser or  setgid and related ones only */
-			if ((p != (struct proc *) 0)
-				&& (p1 != (struct proc *) 0)
-				&& (p->p_stat != SZOMB)
-				&& (
-					(p1 == p)
-					|| !(suser(kauth_cred_get(), 0))
-					|| (((tfp_group_inited != 0) && 
-							(
-							((kauth_cred_ismember_gid(kauth_cred_get(), 
-									tfp_group_ronly, &is_member) == 0) && is_member)
-							||((kauth_cred_ismember_gid(kauth_cred_get(), 
-									tfp_group_rw, &is_member) == 0) && is_member)
-							)
-					   )
-					   && ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
-							((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid))
-							&& ((p->p_flag & P_SUGID) == 0))
-					  )
-					)
-				)
-					ispermitted = TRUE;
+	if (!(task_for_pid_posix_check(p))) {
+		error = KERN_FAILURE;
+		goto tfpout;
+	}
 
-			break;
+	if (p->task != TASK_NULL) {
+		/* If we aren't root and target's task access port is set... */
+		if (!kauth_cred_issuser(kauth_cred_get()) &&
+			(task_get_task_access_port(p->task, &tfpport) == 0) &&
+			(tfpport != IPC_PORT_NULL)) {
 
-		case KERN_TFP_POLICY_DENY:
-			/* self or suser only */
-		default:
-			/* do not return task port of other task at all */
-			if ((p1 != (struct proc *) 0) && (p != (struct proc *) 0) && (p->p_stat != SZOMB)
-					&& ((p1 == p)  || !(suser(kauth_cred_get(), 0))))
-				ispermitted = TRUE;
-			else
-				ispermitted = FALSE;
-			break;
-	};
+			if (tfpport == IPC_PORT_DEAD) {
+				error = KERN_PROTECTION_FAILURE;
+				goto tfpout;
+			}
 
+			/* Call up to the task access server */
+			error = check_task_access(tfpport, proc_selfpid(), kauth_getgid(), pid);
 
-	if (ispermitted == TRUE) {
-		if (p->task != TASK_NULL) {
-			task_reference(p->task);
-			sright = (void *)convert_task_to_port(p->task);
-			tret = ipc_port_copyout_send(
-						sright, 
-						get_task_ipcspace(current_task()));
-			} else
-				tret  = MACH_PORT_NULL;
-			AUDIT_ARG(mach_port2, tret);
-			(void ) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
-	        task_deallocate(t1);
-			error = KERN_SUCCESS;
+			if (error != MACH_MSG_SUCCESS) {
+				if (error == MACH_RCV_INTERRUPTED)
+					error = KERN_ABORTED;
+				else
+					error = KERN_FAILURE;
+				goto tfpout;
+			}
+		}
+#if CONFIG_MACF
+		error = mac_proc_check_get_task(kauth_cred_get(), p);
+		if (error) {
+			error = KERN_FAILURE;
 			goto tfpout;
-	}
-#if DIAGNOSTIC
-	else {
-		/* 
-		 * There is no guarantee that p_comm is null terminated and
-		 * kernel implementation of string functions are complete. So 
-		 * ensure stale info is not leaked out, bzero the  buffer
-		 */
-		bzero(&procname[0], MAXCOMLEN+1);
-		strncpy(&procname[0], &p1->p_comm[0], MAXCOMLEN);
-		if (tfp_policy != KERN_TFP_POLICY_PERMISSIVE)
-			log(LOG_NOTICE, "(%d: %s)tfp: failed on %d:\n",
-				((p1 != PROC_NULL)?(p1->p_pid):0), &procname[0],
-				((p != PROC_NULL)?(p->p_pid):0));
-	}
-#endif /* DIAGNOSTIC */
+		}
+#endif
+
+		/* Grant task port access */
+		task_reference(p->task);
+		sright = (void *) convert_task_to_port(p->task);
+		tret = ipc_port_copyout_send(
+				sright, 
+				get_task_ipcspace(current_task()));
+	} 
+	error = KERN_SUCCESS;
 
-    task_deallocate(t1);
-	tret = MACH_PORT_NULL;
-	(void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
-	error = KERN_FAILURE;
 tfpout:
-	thread_funnel_set(kernel_flock, funnel_state);
+	task_deallocate(t1);
+	AUDIT_ARG(mach_port2, tret);
+	(void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
+	if (p != PROC_NULL)
+		proc_rele(p);
 	AUDIT_MACH_SYSCALL_EXIT(error);
 	return(error);
 }
@@ -541,13 +591,12 @@ task_name_for_pid(
 	int			pid = args->pid;
 	user_addr_t		task_addr = args->t;
 	struct uthread		*uthread;
-	struct proc	*p;
-	struct proc *p1;
+	proc_t		p = PROC_NULL;
 	task_t		t1;
 	mach_port_name_t	tret;
 	void * sright;
-	int error = 0;
-	boolean_t funnel_state;
+	int error = 0, refheld = 0;
+	kauth_cred_t target_cred;
 
 	AUDIT_MACH_SYSCALL_ENTER(AUE_TASKNAMEFORPID);
 	AUDIT_ARG(pid, pid);
@@ -555,68 +604,66 @@ task_name_for_pid(
 
 	t1 = port_name_to_task(target_tport);
 	if (t1 == TASK_NULL) {
-		(void ) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
+		(void) copyout((char *)&t1, task_addr, sizeof(mach_port_name_t));
 		AUDIT_MACH_SYSCALL_EXIT(KERN_FAILURE);
 		return(KERN_FAILURE);
 	} 
 
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
-	p1 = current_proc();
 
 	/*
 	 * Delayed binding of thread credential to process credential, if we
 	 * are not running with an explicitly set thread credential.
 	 */
-	/*
-	 * XXX p_ucred check can be bogus in multithreaded processes,
-	 * XXX unless the funnel is held.
-	 */
 	uthread = get_bsdthread_info(current_thread());
-	if (uthread->uu_ucred != p1->p_ucred &&
-	    (uthread->uu_flag & UT_SETUID) == 0) {
-		kauth_cred_t old = uthread->uu_ucred;
-		proc_lock(p1);
-		kauth_cred_ref(p1->p_ucred);
-		uthread->uu_ucred = p1->p_ucred;
-		proc_unlock(p1);
-		if (IS_VALID_CRED(old))
-			kauth_cred_unref(&old);
-	}
+	kauth_cred_uthread_update(uthread, current_proc());
 
-	p = pfind(pid);
+	p = proc_find(pid);
 	AUDIT_ARG(process, p);
-
-	if ((p != (struct proc *) 0)
-	    && (p->p_stat != SZOMB)
-	    && (p1 != (struct proc *) 0)
-	    && ((p1 == p)
-		|| !(suser(kauth_cred_get(), 0))
-		|| ((kauth_cred_getuid(p->p_ucred) == kauth_cred_getuid(kauth_cred_get())) && 
-		    ((p->p_ucred->cr_ruid == kauth_cred_get()->cr_ruid)))))
-	{
-		if (p->task != TASK_NULL)
-		{
-			task_reference(p->task);
-			sright = (void *)convert_task_name_to_port(p->task);
-			tret = ipc_port_copyout_send(
-						sright, 
+	if (p != PROC_NULL) {
+		target_cred = kauth_cred_proc_ref(p);
+		refheld = 1;
+
+		if ((p->p_stat != SZOMB)
+		    && ((current_proc() == p)
+			|| kauth_cred_issuser(kauth_cred_get()) 
+			|| ((kauth_cred_getuid(target_cred) == kauth_cred_getuid(kauth_cred_get())) && 
+			    ((target_cred->cr_ruid == kauth_cred_get()->cr_ruid))))) {
+
+			if (p->task != TASK_NULL) {
+				task_reference(p->task);
+#if CONFIG_MACF
+				error = mac_proc_check_get_task_name(kauth_cred_get(),  p);
+				if (error) {
+					task_deallocate(p->task);
+					goto noperm;
+				}
+#endif
+				sright = (void *)convert_task_name_to_port(p->task);
+				tret = ipc_port_copyout_send(sright, 
 						get_task_ipcspace(current_task()));
-		} else
-			tret  = MACH_PORT_NULL;
-		AUDIT_ARG(mach_port2, tret);
-		(void ) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
-	        task_deallocate(t1);
-		error = KERN_SUCCESS;
-		goto tnfpout;
+			} else
+				tret  = MACH_PORT_NULL;
+
+			AUDIT_ARG(mach_port2, tret);
+			(void) copyout((char *)&tret, task_addr, sizeof(mach_port_name_t));
+			task_deallocate(t1);
+			error = KERN_SUCCESS;
+			goto tnfpout;
+		}
 	}
 
-	task_deallocate(t1);
+#if CONFIG_MACF
+noperm:
+#endif
+    task_deallocate(t1);
 	tret = MACH_PORT_NULL;
 	(void) copyout((char *) &tret, task_addr, sizeof(mach_port_name_t));
 	error = KERN_FAILURE;
 tnfpout:
-	thread_funnel_set(kernel_flock, funnel_state);
+	if (refheld != 0)
+		kauth_cred_unref(&target_cred);
+	if (p != PROC_NULL)
+		proc_rele(p);
 	AUDIT_MACH_SYSCALL_EXIT(error);
 	return(error);
 }
@@ -639,8 +686,7 @@ sysctl_settfp_policy(__unused struct sysctl_oid *oidp, void *arg1,
 		goto out;
 	}
 	if ((new_value == KERN_TFP_POLICY_DENY) 
-		|| (new_value == KERN_TFP_POLICY_PERMISSIVE)
-		|| (new_value == KERN_TFP_POLICY_RESTRICTED))
+		|| (new_value == KERN_TFP_POLICY_DEFAULT))
 			tfp_policy = new_value;
 	else
 			error = EINVAL;		
@@ -649,280 +695,156 @@ out:
 
 }
 
-static int
-sysctl_settfp_groups(__unused struct sysctl_oid *oidp, void *arg1,
-    __unused int arg2, struct sysctl_req *req)
-{
-    int error = 0;
-	int new_value;
-
-    error = SYSCTL_OUT(req, arg1, sizeof(int));
-    if (error || req->newptr == USER_ADDR_NULL)
-        return(error);
-
-	if (!is_suser())
-		return(EPERM);
-
-	/* 
-	 * Once set; cannot be reset till next boot. Launchd will set this
-	 * in its pid 1 init and no one can set after that.
-	 */
-	if (tfp_group_inited != 0)
-		return(EPERM);
-		
-	if ((error = SYSCTL_IN(req, &new_value, sizeof(int)))) {
-		goto out;
-	}
-
-	if (new_value >= 100) 
-			error = EINVAL;		
-	else {
-		if (arg1 == &tfp_group_ronly) 
-			tfp_group_ronly = new_value;
-		else if (arg1 == &tfp_group_rw)
-			tfp_group_rw = new_value;
-		else
-			error = EINVAL;
-		if ((tfp_group_ronly != 0 ) && (tfp_group_rw != 0 ))
-			tfp_group_inited = 1;
-	}
+#if defined(SECURE_KERNEL)
+static int kern_secure_kernel = 1;
+#else
+static int kern_secure_kernel = 0;
+#endif
 
-out:
-    return(error);
-}
+SYSCTL_INT(_kern, OID_AUTO, secure_kernel, CTLFLAG_RD, &kern_secure_kernel, 0, "");
 
-SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW, 0, "tfp");
+SYSCTL_NODE(_kern, KERN_TFP, tfp, CTLFLAG_RW|CTLFLAG_LOCKED, 0, "tfp");
 SYSCTL_PROC(_kern_tfp, KERN_TFP_POLICY, policy, CTLTYPE_INT | CTLFLAG_RW,
     &tfp_policy, sizeof(uint32_t), &sysctl_settfp_policy ,"I","policy");
-SYSCTL_PROC(_kern_tfp, KERN_TFP_READ_GROUP, read_group, CTLTYPE_INT | CTLFLAG_RW,
-    &tfp_group_ronly, sizeof(uint32_t), &sysctl_settfp_groups ,"I","read_group");
-SYSCTL_PROC(_kern_tfp, KERN_TFP_RW_GROUP, rw_group, CTLTYPE_INT | CTLFLAG_RW,
-    &tfp_group_rw, sizeof(uint32_t), &sysctl_settfp_groups ,"I","rw_group");
-
 
-SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW, &shared_region_trace_level, 0, "");
-
-/*
- * Try and cap the number of mappings the user might be trying to deal with,
- * so that we don't end up allocating insane amounts of wired memory in the
- * kernel based on bogus user arguments.
- * There are 2 shared regions (TEXT and DATA). The size of each submap
- * is SHARED_TEXT_REGION_SIZE and we can have at most 1 VM map entry per page,
- * so the maximum number of mappings we could ever have to deal with is...
- */
-#define SHARED_REGION_MAX_MAPPINGS ((2 *SHARED_TEXT_REGION_SIZE) >> PAGE_SHIFT)
+SYSCTL_INT(_vm, OID_AUTO, shared_region_trace_level, CTLFLAG_RW,
+	   &shared_region_trace_level, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, shared_region_version, CTLFLAG_RD,
+	   &shared_region_version, 0, "");
+SYSCTL_INT(_vm, OID_AUTO, shared_region_persistence, CTLFLAG_RW,
+	   &shared_region_persistence, 0, "");
 
 /*
- * shared_region_make_private_np:
+ * shared_region_check_np:
  *
- * This system call is for "dyld" only.
- * 
- * It creates a private copy of the current process's "shared region" for
- * split libraries.  "dyld" uses this when the shared region is full or
- * it needs to load a split library that conflicts with an already loaded one
- * that this process doesn't need.  "dyld" specifies a set of address ranges
- * that it wants to keep in the now-private "shared region".  These cover
- * the set of split libraries that the process needs so far.  The kernel needs
- * to deallocate the rest of the shared region, so that it's available for 
- * more libraries for this process.
+ * This system call is intended for dyld.
+ *
+ * dyld calls this when any process starts to see if the process's shared
+ * region is already set up and ready to use.
+ * This call returns the base address of the first mapping in the
+ * process's shared region's first mapping.
+ * dyld will then check what's mapped at that address.
+ *
+ * If the shared region is empty, dyld will then attempt to map the shared
+ * cache file in the shared region via the shared_region_map_np() system call.
+ *
+ * If something's already mapped in the shared region, dyld will check if it
+ * matches the shared cache it would like to use for that process.
+ * If it matches, evrything's ready and the process can proceed and use the
+ * shared region.
+ * If it doesn't match, dyld will unmap the shared region and map the shared
+ * cache into the process's address space via mmap().
+ *
+ * ERROR VALUES
+ * EINVAL	no shared region
+ * ENOMEM	shared region is empty
+ * EFAULT	bad address for "start_address"
  */
 int
-shared_region_make_private_np(
-	struct proc					*p,
-	struct shared_region_make_private_np_args	*uap,
-	__unused int					*retvalp)
+shared_region_check_np(
+	__unused struct proc			*p,
+	struct shared_region_check_np_args	*uap,
+	__unused int				*retvalp)
 {
-	int				error;
-	kern_return_t			kr;
-	boolean_t			using_shared_regions;
-	user_addr_t			user_ranges;
-	unsigned int			range_count;
-	vm_size_t			ranges_size;
-	struct shared_region_range_np	*ranges;
-	shared_region_mapping_t 	shared_region;
-	struct shared_region_task_mappings	task_mapping_info;
-	shared_region_mapping_t		next;
-
-	ranges = NULL;
-
-	range_count = uap->rangeCount;
-	user_ranges = uap->ranges;
-	ranges_size = (vm_size_t) (range_count * sizeof (ranges[0]));
-
-	SHARED_REGION_TRACE(
-		SHARED_REGION_TRACE_INFO,
-		("shared_region: %p [%d(%s)] "
-		 "make_private(rangecount=%d)\n",
-		 current_thread(), p->p_pid, p->p_comm, range_count));
-
-	/* allocate kernel space for the "ranges" */
-	if (range_count != 0) {
-		if (range_count > SHARED_REGION_MAX_MAPPINGS) {
-			error = EINVAL;
-			goto done;
-		}
-		if ((mach_vm_size_t) ranges_size !=
-		    (mach_vm_size_t) range_count * sizeof (ranges[0])) {
-			/* 32-bit integer overflow */
-			error = EINVAL;
-			goto done;
-		}
-		kr = kmem_alloc(kernel_map,
-				(vm_offset_t *) &ranges,
-				ranges_size);
+	vm_shared_region_t	shared_region;
+	mach_vm_offset_t	start_address;
+	int			error;
+	kern_return_t		kr;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: %p [%d(%s)] -> check_np(0x%llx)\n",
+		 current_thread(), p->p_pid, p->p_comm,
+		 (uint64_t)uap->start_address));
+
+	/* retrieve the current tasks's shared region */
+	shared_region = vm_shared_region_get(current_task());
+	if (shared_region != NULL) {
+		/* retrieve address of its first mapping... */
+		kr = vm_shared_region_start_address(shared_region,
+						    &start_address);
 		if (kr != KERN_SUCCESS) {
 			error = ENOMEM;
-			goto done;
-		}
-
-		/* copy "ranges" from user-space */
-		error = copyin(user_ranges,
-			       ranges,
-			       ranges_size);
-		if (error) {
-			goto done;
+		} else {
+			/* ... and give it to the caller */
+			error = copyout(&start_address,
+					(user_addr_t) uap->start_address,
+					sizeof (start_address));
+			if (error) {
+				SHARED_REGION_TRACE_ERROR(
+					("shared_region: %p [%d(%s)] "
+					 "check_np(0x%llx) "
+					 "copyout(0x%llx) error %d\n",
+					 current_thread(), p->p_pid, p->p_comm,
+					 (uint64_t)uap->start_address, (uint64_t)start_address,
+					 error));
+			}
 		}
-	}
-
-	if (p->p_flag & P_NOSHLIB) {
-		/* no split library has been mapped for this process so far */
-		using_shared_regions = FALSE;
+		vm_shared_region_deallocate(shared_region);
 	} else {
-		/* this process has already mapped some split libraries */
-		using_shared_regions = TRUE;
-	}
-
-	/*
-	 * Get a private copy of the current shared region.
-	 * Do not chain it to the system-wide shared region, as we'll want
-	 * to map other split libraries in place of the old ones.  We want
-	 * to completely detach from the system-wide shared region and go our
-	 * own way after this point, not sharing anything with other processes.
-	 */
-	error = clone_system_shared_regions(using_shared_regions,
-					    FALSE, /* chain_regions */
-					    ENV_DEFAULT_ROOT);
-	if (error) {
-		goto done;
-	}
-
-	/* get info on the newly allocated shared region */
-	vm_get_shared_region(current_task(), &shared_region);
-	task_mapping_info.self = (vm_offset_t) shared_region;
-	shared_region_mapping_info(shared_region,
-				   &(task_mapping_info.text_region),
-				   &(task_mapping_info.text_size),
-				   &(task_mapping_info.data_region),
-				   &(task_mapping_info.data_size),
-				   &(task_mapping_info.region_mappings),
-				   &(task_mapping_info.client_base),
-				   &(task_mapping_info.alternate_base),
-				   &(task_mapping_info.alternate_next),
-				   &(task_mapping_info.fs_base),
-				   &(task_mapping_info.system),
-				   &(task_mapping_info.flags),
-				   &next);
-
-	/*
-	 * We now have our private copy of the shared region, as it was before
-	 * the call to clone_system_shared_regions().  We now need to clean it
-	 * up and keep only the memory areas described by the "ranges" array.
-	 */
-	kr = shared_region_cleanup(range_count, ranges, &task_mapping_info);
-	switch (kr) {
-	case KERN_SUCCESS:
-		error = 0;
-		break;
-	default:
+		/* no shared region ! */
 		error = EINVAL;
-		goto done;
-	}
-
-done:
-	if (ranges != NULL) {
-		kmem_free(kernel_map,
-			  (vm_offset_t) ranges,
-			  ranges_size);
-		ranges = NULL;
 	}
 
-	SHARED_REGION_TRACE(
-		SHARED_REGION_TRACE_INFO,
-		("shared_region: %p [%d(%s)] "
-		 "make_private(rangecount=%d) -> %d "
-		 "shared_region=%p[%x,%x,%x]\n",
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: %p [%d(%s)] check_np(0x%llx) <- 0x%llx %d\n",
 		 current_thread(), p->p_pid, p->p_comm,
-		 range_count, error, shared_region,
-		 task_mapping_info.fs_base,
-		 task_mapping_info.system,
-		 task_mapping_info.flags));
+		 (uint64_t)uap->start_address, (uint64_t)start_address, error));
 
 	return error;
 }
 
-
 /*
- * shared_region_map_file_np:
- *
- * This system call is for "dyld" only.
+ * shared_region_map_np()
  *
- * "dyld" wants to map parts of a split library in the shared region.
- * We get a file descriptor on the split library to be mapped and a set
- * of mapping instructions, describing which parts of the file to map in\
- * which areas of the shared segment and with what protection.
- * The "shared region" is split in 2 areas:
- * 0x90000000 - 0xa0000000 : read-only area (for TEXT and LINKEDIT sections), 
- * 0xa0000000 - 0xb0000000 : writable area (for DATA sections).
+ * This system call is intended for dyld.
  *
+ * dyld uses this to map a shared cache file into a shared region.
+ * This is usually done only the first time a shared cache is needed.
+ * Subsequent processes will just use the populated shared region without
+ * requiring any further setup.
  */
 int
-shared_region_map_file_np(
+shared_region_map_np(
 	struct proc				*p,
-	struct shared_region_map_file_np_args	*uap,
+	struct shared_region_map_np_args	*uap,
 	__unused int				*retvalp)
 {
-	int					error;
-	kern_return_t				kr;
-	int					fd;
-	unsigned int				mapping_count;
-	user_addr_t				user_mappings; /* 64-bit */
-	user_addr_t				user_slide_p;  /* 64-bit */
-	struct shared_file_mapping_np 		*mappings;
-	vm_size_t				mappings_size;
-	struct fileproc				*fp;
-	mach_vm_offset_t 			slide;
-	struct vnode				*vp;
-	struct vfs_context 			context;
-	memory_object_control_t 		file_control;
-	memory_object_size_t			file_size;
-	shared_region_mapping_t 		shared_region;
-	struct shared_region_task_mappings	task_mapping_info;
-	shared_region_mapping_t			next;
-	shared_region_mapping_t			default_shared_region;
-	boolean_t				using_default_region;
-	unsigned int				j;
-	vm_prot_t				max_prot;
-	mach_vm_offset_t			base_offset, end_offset;
-	mach_vm_offset_t			original_base_offset;
-	boolean_t				mappings_in_segment;
-#define SFM_MAX_STACK	6
-	struct shared_file_mapping_np		stack_mappings[SFM_MAX_STACK];
-
+	int				error;
+	kern_return_t			kr;
+	int				fd;
+	struct fileproc			*fp;
+	struct vnode			*vp, *root_vp;
+	struct vnode_attr		va;
+	off_t				fs;
+	memory_object_size_t		file_size;
+	user_addr_t			user_mappings;
+	struct shared_file_mapping_np	*mappings;
+#define SFM_MAX_STACK	4
+	struct shared_file_mapping_np	stack_mappings[SFM_MAX_STACK];
+	unsigned int			mappings_count;
+	vm_size_t			mappings_size;
+	memory_object_control_t		file_control;
+	struct vm_shared_region		*shared_region;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: %p [%d(%s)] -> map\n",
+		 current_thread(), p->p_pid, p->p_comm));
+
+	shared_region = NULL;
+	mappings_count = 0;
 	mappings_size = 0;
 	mappings = NULL;
-	mapping_count = 0;
 	fp = NULL;
 	vp = NULL;
 
-	/* get file descriptor for split library from arguments */
+	/* get file descriptor for shared region cache file */
 	fd = uap->fd;
 
 	/* get file structure from file descriptor */
 	error = fp_lookup(p, fd, &fp, 0);
 	if (error) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file: "
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map: "
 			 "fd=%d lookup failed (error=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm, fd, error));
 		goto done;
@@ -930,9 +852,8 @@ shared_region_map_file_np(
 
 	/* make sure we're attempting to map a vnode */
 	if (fp->f_fglob->fg_type != DTYPE_VNODE) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file: "
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map: "
 			 "fd=%d not a vnode (type=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm,
 			 fd, fp->f_fglob->fg_type));
@@ -942,9 +863,8 @@ shared_region_map_file_np(
 
 	/* we need at least read permission on the file */
 	if (! (fp->f_fglob->fg_flag & FREAD)) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file: "
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map: "
 			 "fd=%d not readable\n",
 			 current_thread(), p->p_pid, p->p_comm, fd));
 		error = EPERM;
@@ -952,11 +872,10 @@ shared_region_map_file_np(
 	}
 
 	/* get vnode from file structure */
-	error = vnode_getwithref((vnode_t)fp->f_fglob->fg_data);
+	error = vnode_getwithref((vnode_t) fp->f_fglob->fg_data);
 	if (error) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file: "
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map: "
 			 "fd=%d getwithref failed (error=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm, fd, error));
 		goto done;
@@ -965,9 +884,8 @@ shared_region_map_file_np(
 
 	/* make sure the vnode is a regular file */
 	if (vp->v_type != VREG) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
 			 "not a file (type=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm,
 			 vp, vp->v_name, vp->v_type));
@@ -975,342 +893,165 @@ shared_region_map_file_np(
 		goto done;
 	}
 
-	/* get vnode size */
-	{
-		off_t	fs;
-		
-		context.vc_proc = p;
-		context.vc_ucred = kauth_cred_get();
-		if ((error = vnode_size(vp, &fs, &context)) != 0) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: %p [%d(%s)] "
-				 "map_file(%p:'%s'): "
-				 "vnode_size(%p) failed (error=%d)\n",
-				 current_thread(), p->p_pid, p->p_comm,
-				 vp, vp->v_name, vp));
-			goto done;
-		}
-		file_size = fs;
+	/* make sure vnode is on the process's root volume */
+	root_vp = p->p_fd->fd_rdir;
+	if (root_vp == NULL) {
+		root_vp = rootvnode;
 	}
-
-	/*
-	 * Get the list of mappings the caller wants us to establish.
-	 */
-	mapping_count = uap->mappingCount; /* the number of mappings */
-	mappings_size = (vm_size_t) (mapping_count * sizeof (mappings[0]));
-	if (mapping_count == 0) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_INFO,
-			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
-			 "no mappings\n",
+	if (vp->v_mount != root_vp->v_mount) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "not on process's root volume\n",
 			 current_thread(), p->p_pid, p->p_comm,
 			 vp, vp->v_name));
-		error = 0;	/* no mappings: we're done ! */
+		error = EPERM;
 		goto done;
-	} else if (mapping_count <= SFM_MAX_STACK) {
-		mappings = &stack_mappings[0];
-	} else {
-		if (mapping_count > SHARED_REGION_MAX_MAPPINGS) {
-			error = EINVAL;
-			goto done;
-		}
-		if ((mach_vm_size_t) mappings_size !=
-		    (mach_vm_size_t) mapping_count * sizeof (mappings[0])) {
-			/* 32-bit integer overflow */
-			error = EINVAL;
-			goto done;
-		}
-		kr = kmem_alloc(kernel_map,
-				(vm_offset_t *) &mappings,
-				mappings_size);
-		if (kr != KERN_SUCCESS) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: %p [%d(%s)] "
-				 "map_file(%p:'%s'): "
-				 "failed to allocate %d mappings (kr=0x%x)\n",
-				 current_thread(), p->p_pid, p->p_comm,
-				 vp, vp->v_name, mapping_count, kr));
-			error = ENOMEM;
-			goto done;
-		}
 	}
 
-	user_mappings = uap->mappings;	   /* the mappings, in user space */
-	error = copyin(user_mappings,
-		       mappings,
-		       mappings_size);
-	if (error != 0) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
-			 "failed to copyin %d mappings (error=%d)\n",
+	/* make sure vnode is owned by "root" */
+	VATTR_INIT(&va);
+	VATTR_WANTED(&va, va_uid);
+	error = vnode_getattr(vp, &va, vfs_context_current());
+	if (error) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "vnode_getattr(%p) failed (error=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm,
-			 vp, vp->v_name, mapping_count, error));
+			 vp, vp->v_name, vp, error));
 		goto done;
 	}
-
-	/*
-	 * If the caller provides a "slide" pointer, it means they're OK
-	 * with us moving the mappings around to make them fit.
-	 */
-	user_slide_p = uap->slide_p;
-
-	/*
-	 * Make each mapping address relative to the beginning of the
-	 * shared region.  Check that all mappings are in the shared region.
-	 * Compute the maximum set of protections required to tell the
-	 * buffer cache how we mapped the file (see call to ubc_map() below).
-	 */
-	max_prot = VM_PROT_NONE;
-	base_offset = -1LL;
-	end_offset = 0;
-	mappings_in_segment = TRUE;
-	for (j = 0; j < mapping_count; j++) {
-		mach_vm_offset_t segment;
-		segment = (mappings[j].sfm_address &
-			   GLOBAL_SHARED_SEGMENT_MASK);
-		if (segment != GLOBAL_SHARED_TEXT_SEGMENT &&
-		    segment != GLOBAL_SHARED_DATA_SEGMENT) {
-			/* this mapping is not in the shared region... */
-			if (user_slide_p == NULL) {
-				/* ... and we can't slide it in: fail */
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p [%d(%s)] "
-					 "map_file(%p:'%s'): "
-					 "mapping %p not in shared segment & "
-					 "no sliding\n",
-					 current_thread(), p->p_pid, p->p_comm,
-					 vp, vp->v_name,
-					 mappings[j].sfm_address));
-				error = EINVAL;
-				goto done;
-			}
-			if (j == 0) {
-				/* expect all mappings to be outside */
-				mappings_in_segment = FALSE;
-			} else if (mappings_in_segment != FALSE) {
-				/* other mappings were not outside: fail */
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p [%d(%s)] "
-					 "map_file(%p:'%s'): "
-					 "mapping %p not in shared segment & "
-					 "other mappings in shared segment\n",
-					 current_thread(), p->p_pid, p->p_comm,
-					 vp, vp->v_name,
-					 mappings[j].sfm_address));
-				error = EINVAL;
-				goto done;
-			}
-			/* we'll try and slide that mapping in the segments */
-		} else {
-			if (j == 0) {
-				/* expect all mappings to be inside */
-				mappings_in_segment = TRUE;
-			} else if (mappings_in_segment != TRUE) {
-				/* other mappings were not inside: fail */
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p [%d(%s)] "
-					 "map_file(%p:'%s'): "
-					 "mapping %p in shared segment & "
-					 "others in shared segment\n",
-					 current_thread(), p->p_pid, p->p_comm,
-					 vp, vp->v_name,
-					 mappings[j].sfm_address));
-				error = EINVAL;
-				goto done;
-			}
-			/* get a relative offset inside the shared segments */
-			mappings[j].sfm_address -= GLOBAL_SHARED_TEXT_SEGMENT;
-		}
-		if ((mappings[j].sfm_address & SHARED_TEXT_REGION_MASK)
-		    < base_offset) {
-			base_offset = (mappings[j].sfm_address &
-				       SHARED_TEXT_REGION_MASK);
-		}
-		if ((mappings[j].sfm_address & SHARED_TEXT_REGION_MASK) +
-		    mappings[j].sfm_size > end_offset) {
-			end_offset =
-				(mappings[j].sfm_address &
-				 SHARED_TEXT_REGION_MASK) +
-				mappings[j].sfm_size;
-		}
-		max_prot |= mappings[j].sfm_max_prot;
-	}
-	/* Make all mappings relative to the base_offset */
-	base_offset = vm_map_trunc_page(base_offset);
-	end_offset = vm_map_round_page(end_offset);
-	for (j = 0; j < mapping_count; j++) {
-		mappings[j].sfm_address -= base_offset;
+	if (va.va_uid != 0) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "owned by uid=%d instead of 0\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, va.va_uid));
+		error = EPERM;
+		goto done;
 	}
-	original_base_offset = base_offset;
-	if (mappings_in_segment == FALSE) {
-		/*
-		 * We're trying to map a library that was not pre-bound to
-		 * be in the shared segments.  We want to try and slide it
-		 * back into the shared segments but as far back as possible,
-		 * so that it doesn't clash with pre-bound libraries.  Set
-		 * the base_offset to the end of the region, so that it can't
-		 * possibly fit there and will have to be slid.
-		 */
-		base_offset = SHARED_TEXT_REGION_SIZE - end_offset;
+
+	/* get vnode size */
+	error = vnode_size(vp, &fs, vfs_context_current());
+	if (error) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "vnode_size(%p) failed (error=%d)\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, vp, error));
+		goto done;
 	}
+	file_size = fs;
 
 	/* get the file's memory object handle */
-	UBCINFOCHECK("shared_region_map_file_np", vp);
 	file_control = ubc_getobject(vp, UBC_HOLDOBJECT);
 	if (file_control == MEMORY_OBJECT_CONTROL_NULL) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p [%d(%s)] map_file(%p:'%s'): "
-			 "ubc_getobject() failed\n",
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "no memory object\n",
 			 current_thread(), p->p_pid, p->p_comm,
 			 vp, vp->v_name));
 		error = EINVAL;
 		goto done;
 	}
-
-	/*
-	 * Get info about the current process's shared region.
-	 * This might change if we decide we need to clone the shared region.
-	 */
-	vm_get_shared_region(current_task(), &shared_region);
-	task_mapping_info.self = (vm_offset_t) shared_region;
-	shared_region_mapping_info(shared_region,
-				   &(task_mapping_info.text_region),
-				   &(task_mapping_info.text_size),
-				   &(task_mapping_info.data_region),
-				   &(task_mapping_info.data_size),
-				   &(task_mapping_info.region_mappings),
-				   &(task_mapping_info.client_base),
-				   &(task_mapping_info.alternate_base),
-				   &(task_mapping_info.alternate_next),
-				   &(task_mapping_info.fs_base),
-				   &(task_mapping_info.system),
-				   &(task_mapping_info.flags),
-				   &next);
-
-	/*
-	 * Are we using the system's current shared region
-	 * for this environment ?
-	 */
-	default_shared_region =
-		lookup_default_shared_region(ENV_DEFAULT_ROOT,
-					     task_mapping_info.system);
-	if (shared_region == default_shared_region) {
-		using_default_region = TRUE;
+			 
+	/* get the list of mappings the caller wants us to establish */
+	mappings_count = uap->count;	/* number of mappings */
+	mappings_size = (vm_size_t) (mappings_count * sizeof (mappings[0]));
+	if (mappings_count == 0) {
+		SHARED_REGION_TRACE_INFO(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "no mappings\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name));
+		error = 0;	/* no mappings: we're done ! */
+		goto done;
+	} else if (mappings_count <= SFM_MAX_STACK) {
+		mappings = &stack_mappings[0];
 	} else {
-		using_default_region = FALSE;
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "too many mappings (%d)\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name, mappings_count));
+		error = EINVAL;
+		goto done;
 	}
-	shared_region_mapping_dealloc(default_shared_region);
 
-	if (vp->v_mount != rootvnode->v_mount &&
-	    using_default_region) {
-		/*
-		 * The split library is not on the root filesystem.  We don't
-		 * want to polute the system-wide ("default") shared region
-		 * with it.
-		 * Reject the mapping.  The caller (dyld) should "privatize"
-		 * (via shared_region_make_private()) the shared region and
-		 * try to establish the mapping privately for this process.
-		 */
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_CONFLICT,
-			("shared_region: %p [%d(%s)] "
-			 "map_file(%p:'%s'): "
-			 "not on root volume\n",
+	user_mappings = uap->mappings;	/* the mappings, in user space */
+	error = copyin(user_mappings,
+		       mappings,
+		       mappings_size);
+	if (error) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "copyin(0x%llx, %d) failed (error=%d)\n",
 			 current_thread(), p->p_pid, p->p_comm,
-			 vp->v_name));
-		error = EXDEV;
+			 vp, vp->v_name, (uint64_t)user_mappings, mappings_count, error));
 		goto done;
 	}
 
+	/* get the process's shared region (setup in vm_map_exec()) */
+	shared_region = vm_shared_region_get(current_task());
+	if (shared_region == NULL) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "no shared region\n",
+			 current_thread(), p->p_pid, p->p_comm,
+			 vp, vp->v_name));
+		goto done;
+	}
 
-	/*
-	 * Map the split library.
-	 */
-	kr = map_shared_file(mapping_count,
-			     mappings,
-			     file_control,
-			     file_size,
-			     &task_mapping_info,
-			     base_offset,
-			     (user_slide_p) ? &slide : NULL);
-
-	if (kr == KERN_SUCCESS) {
-		/*
-		 * The mapping was successful.  Let the buffer cache know
-		 * that we've mapped that file with these protections.  This
-		 * prevents the vnode from getting recycled while it's mapped.
-		 */
-		(void) ubc_map(vp, max_prot);
-		error = 0;
-	} else {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_CONFLICT,
-			("shared_region: %p [%d(%s)] "
-			 "map_file(%p:'%s'): "
-			 "map_shared_file failed, kr=0x%x\n",
+	/* map the file into that shared region's submap */
+	kr = vm_shared_region_map_file(shared_region,
+				       mappings_count,
+				       mappings,
+				       file_control,
+				       file_size,
+				       (void *) p->p_fd->fd_rdir);
+	if (kr != KERN_SUCCESS) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: %p [%d(%s)] map(%p:'%s'): "
+			 "vm_shared_region_map_file() failed kr=0x%x\n",
 			 current_thread(), p->p_pid, p->p_comm,
 			 vp, vp->v_name, kr));
 		switch (kr) {
 		case KERN_INVALID_ADDRESS:
 			error = EFAULT;
-			goto done;
+			break;
 		case KERN_PROTECTION_FAILURE:
 			error = EPERM;
-			goto done;
+			break;
 		case KERN_NO_SPACE:
 			error = ENOMEM;
-			goto done;
+			break;
 		case KERN_FAILURE:
 		case KERN_INVALID_ARGUMENT:
 		default:
 			error = EINVAL;
-			goto done;
+			break;
 		}
+		goto done;
 	}
 
-	if (p->p_flag & P_NOSHLIB) {
-		/* signal that this process is now using split libraries */
-		p->p_flag &= ~P_NOSHLIB;
+	/*
+	 * The mapping was successful.  Let the buffer cache know
+	 * that we've mapped that file with these protections.  This
+	 * prevents the vnode from getting recycled while it's mapped.
+	 */
+	(void) ubc_map(vp, VM_PROT_READ);
+	error = 0;
+
+	/* update the vnode's access time */
+	if (! (vnode_vfsvisflags(vp) & MNT_NOATIME)) {
+		VATTR_INIT(&va);
+		nanotime(&va.va_access_time);
+		VATTR_SET_ACTIVE(&va, va_access_time);
+		vnode_setattr(vp, &va, vfs_context_current());
 	}
 
-	if (user_slide_p) {
-		/*
-		 * The caller provided a pointer to a "slide" offset.  Let
-		 * them know by how much we slid the mappings.
-		 */
-		if (mappings_in_segment == FALSE) {
-			/*
-			 * We faked the base_offset earlier, so undo that
-			 * and take into account the real base_offset.
-			 */
-			slide += SHARED_TEXT_REGION_SIZE - end_offset;
-			slide -= original_base_offset;
-			/*
-			 * The mappings were slid into the shared segments
-			 * and "slide" is relative to the beginning of the
-			 * shared segments.  Adjust it to be absolute.
-			 */
-			slide += GLOBAL_SHARED_TEXT_SEGMENT;
-		}
-		error = copyout(&slide,
-				user_slide_p,
-				sizeof (slide));
-		if (slide != 0) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_CONFLICT,
-				("shared_region: %p [%d(%s)] "
-				 "map_file(%p:'%s'): "
-				 "slid by 0x%llx\n",
-				 current_thread(), p->p_pid, p->p_comm,
-				 vp, vp->v_name, slide));
-		}
+	if (p->p_flag & P_NOSHLIB) {
+		/* signal that this process is now using split libraries */
+		OSBitAndAtomic(~((uint32_t)P_NOSHLIB), (UInt32 *)&p->p_flag);
 	}
 
 done:
@@ -1327,1175 +1068,25 @@ done:
 		fp_drop(p, fd, fp, 0);
 		fp = NULL;
 	}
-	if (mappings != NULL &&
-	    mappings != &stack_mappings[0]) {
-		kmem_free(kernel_map,
-			  (vm_offset_t) mappings,
-			  mappings_size);
-	}
-	mappings = NULL;
-
-	return error;
-}
-
-int
-load_shared_file(
-	__unused struct proc *p,
-	__unused struct load_shared_file_args *uap,
-	__unused int *retval)
-{
-	return ENOSYS;
-}
-
-int
-reset_shared_file(
-	__unused struct proc *p,
-	__unused struct reset_shared_file_args *uap,
-	__unused int *retval)
-{
-	return ENOSYS;
-}
-
-int
-new_system_shared_regions(
-	__unused struct proc *p,
-	__unused struct new_system_shared_regions_args *uap,
-	__unused int *retval)
-{
-	return ENOSYS;
-}
-
-
-
-int
-clone_system_shared_regions(
-	int		shared_regions_active,
-	int		chain_regions,
-	int		base_vnode)
-{
-	shared_region_mapping_t	new_shared_region;
-	shared_region_mapping_t	next;
-	shared_region_mapping_t	old_shared_region;
-	struct shared_region_task_mappings old_info;
-	struct shared_region_task_mappings new_info;
-
-	vm_get_shared_region(current_task(), &old_shared_region);
-	old_info.self = (vm_offset_t)old_shared_region;
-	shared_region_mapping_info(old_shared_region,
-		&(old_info.text_region),   
-		&(old_info.text_size),
-		&(old_info.data_region),
-		&(old_info.data_size),
-		&(old_info.region_mappings),
-		&(old_info.client_base),
-		&(old_info.alternate_base),
-		&(old_info.alternate_next), 
-		&(old_info.fs_base),
-		&(old_info.system),
-		&(old_info.flags), &next);
-
-	if (shared_regions_active ||
-	    base_vnode == ENV_DEFAULT_ROOT) {
-		if (shared_file_create_system_region(&new_shared_region,
-						     old_info.fs_base,
-						     old_info.system))
-			return ENOMEM;
-	} else {
-		if (old_shared_region &&
-		    base_vnode == ENV_DEFAULT_ROOT) {
-			base_vnode = old_info.fs_base;
-		}
-		new_shared_region =
-			lookup_default_shared_region(base_vnode,
-						     old_info.system);
-		if (new_shared_region == NULL) {
-			shared_file_boot_time_init(base_vnode,
-						   old_info.system);
-			vm_get_shared_region(current_task(),
-					     &new_shared_region);
-		} else {
-			vm_set_shared_region(current_task(), new_shared_region);
-		}
-		if (old_shared_region)
-			shared_region_mapping_dealloc(old_shared_region);
-	}
-	new_info.self = (vm_offset_t)new_shared_region;
-	shared_region_mapping_info(new_shared_region,
-		&(new_info.text_region),   
-		&(new_info.text_size),
-		&(new_info.data_region),
-		&(new_info.data_size),
-		&(new_info.region_mappings),
-		&(new_info.client_base),
-		&(new_info.alternate_base),
-		&(new_info.alternate_next), 
-		&(new_info.fs_base),
-		&(new_info.system),
-		&(new_info.flags), &next);
-	if(shared_regions_active) {
-	   if(vm_region_clone(old_info.text_region, new_info.text_region)) {
-	   panic("clone_system_shared_regions: shared region mis-alignment 1");
-		shared_region_mapping_dealloc(new_shared_region);
-		return(EINVAL);
-	   }
-	   if (vm_region_clone(old_info.data_region, new_info.data_region)) {
-	   panic("clone_system_shared_regions: shared region mis-alignment 2");
-		shared_region_mapping_dealloc(new_shared_region);
-		return(EINVAL);
-	   }
-	   if (chain_regions) {
-		   /*
-		    * We want a "shadowed" clone, a private superset of the old
-		    * shared region.  The info about the old mappings is still
-		    * valid for us.
-		    */
-		   shared_region_object_chain_attach(
-			   new_shared_region, old_shared_region);
-	   }
-	}
-	if (!chain_regions) {
-		/*
-		 * We want a completely detached clone with no link to
-		 * the old shared region.  We'll be removing some mappings
-		 * in our private, cloned, shared region, so the old mappings
-		 * will become irrelevant to us.  Since we have a private
-		 * "shared region" now, it isn't going to be shared with
-		 * anyone else and we won't need to maintain mappings info.
-		 */
-		shared_region_object_chain_detached(new_shared_region);
-	}
-	if (vm_map_region_replace(current_map(), old_info.text_region, 
-			new_info.text_region, old_info.client_base, 
-			old_info.client_base+old_info.text_size)) {
-	panic("clone_system_shared_regions: shared region mis-alignment 3");
-		shared_region_mapping_dealloc(new_shared_region);
-		return(EINVAL);
-	}
-	if(vm_map_region_replace(current_map(), old_info.data_region, 
-			new_info.data_region, 
-			old_info.client_base + old_info.text_size, 
-			old_info.client_base
-				+ old_info.text_size + old_info.data_size)) {
-	panic("clone_system_shared_regions: shared region mis-alignment 4");
-		shared_region_mapping_dealloc(new_shared_region);
-		return(EINVAL);
-	}
-	vm_set_shared_region(current_task(), new_shared_region);
-
-	/* consume the reference which wasn't accounted for in object */
-	/* chain attach */
-	if (!shared_regions_active || !chain_regions)
-		shared_region_mapping_dealloc(old_shared_region);
-
-	SHARED_REGION_TRACE(
-		SHARED_REGION_TRACE_INFO,
-		("shared_region: %p task=%p "
-		 "clone(active=%d, base=0x%x,chain=%d) "
-		 "old=%p[%x,%x,%x] new=%p[%x,%x,%x]\n",
-		 current_thread(), current_task(), 
-		 shared_regions_active, base_vnode, chain_regions,
-		 old_shared_region,
-		 old_info.fs_base,
-		 old_info.system,
-		 old_info.flags,
-		 new_shared_region,
-		 new_info.fs_base,
-		 new_info.system,
-		 new_info.flags));
-
-	return(0);
-
-}
-
-/* header for the profile name file.  The profiled app info is held */
-/* in the data file and pointed to by elements in the name file     */
-
-struct profile_names_header {
-	unsigned int	number_of_profiles;
-	unsigned int	user_id;
-	unsigned int	version;
-	off_t		element_array;
-	unsigned int	spare1;
-	unsigned int	spare2;
-	unsigned int	spare3;
-};
-
-struct profile_element {
-	off_t		addr;
-	vm_size_t	size;
-	unsigned int	mod_date;
-	unsigned int	inode;
-	char name[12];
-};
-
-struct global_profile {
-	struct vnode	*names_vp;
-	struct vnode	*data_vp;
-	vm_offset_t	buf_ptr;
-	unsigned int	user;
-	unsigned int	age;
-	unsigned int	busy;
-};
-
-struct global_profile_cache {
-	int			max_ele;
-	unsigned int		age;
-	struct global_profile	profiles[3];
-};
-
-/* forward declarations */
-int bsd_open_page_cache_files(unsigned int user,
-			      struct global_profile **profile);
-void bsd_close_page_cache_files(struct global_profile *profile);
-int bsd_search_page_cache_data_base(
-	struct	vnode			*vp,
-	struct profile_names_header	*database,
-	char				*app_name,
-	unsigned int			mod_date,
-	unsigned int			inode,
-	off_t				*profile,
-	unsigned int			*profile_size);
-
-struct global_profile_cache global_user_profile_cache =
-	{3, 0, {{NULL, NULL, 0, 0, 0, 0},
-		    {NULL, NULL, 0, 0, 0, 0},
-		    {NULL, NULL, 0, 0, 0, 0}} };
-
-/* BSD_OPEN_PAGE_CACHE_FILES:                                 */
-/* Caller provides a user id.  This id was used in            */
-/* prepare_profile_database to create two unique absolute     */
-/* file paths to the associated profile files.  These files   */
-/* are either opened or bsd_open_page_cache_files returns an  */
-/* error.  The header of the names file is then consulted.    */
-/* The header and the vnodes for the names and data files are */
-/* returned. */
-
-int
-bsd_open_page_cache_files(
-	unsigned int	user,
-	struct global_profile **profile)
-{
-	const char *cache_path = "/var/vm/app_profile/";
-	struct proc	*p;
-	int		error;
-	vm_size_t	resid;
-	off_t		resid_off;
-	unsigned int	lru;
-	vm_size_t	size;
-
-	struct	vnode	*names_vp;
-	struct  vnode	*data_vp;
-	vm_offset_t	names_buf;
-	vm_offset_t	buf_ptr;
-
-	int		profile_names_length;
-	int		profile_data_length;
-	char		*profile_data_string;
-	char		*profile_names_string;
-	char		*substring;
-
-	off_t		file_size;
-	struct vfs_context  context;
-
-	kern_return_t	ret;
-
-	struct nameidata nd_names;
-	struct nameidata nd_data;
-	int		i;
-
-
-	p = current_proc();
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-restart:
-	for(i = 0; i<global_user_profile_cache.max_ele; i++) {
-		if((global_user_profile_cache.profiles[i].user == user) 
-			&&  (global_user_profile_cache.profiles[i].data_vp 
-								!= NULL)) {
-			*profile = &global_user_profile_cache.profiles[i];
-			/* already in cache, we're done */
-			if ((*profile)->busy) {
-       				/*
-       				* drop funnel and wait 
-       				*/
-				(void)tsleep((void *)
-					*profile, 
-					PRIBIO, "app_profile", 0);
-				goto restart;
-			}
-			(*profile)->busy = 1;
-			(*profile)->age = global_user_profile_cache.age;
-
-			/*
-			 * entries in cache are held with a valid
-			 * usecount... take an iocount which will
-			 * be dropped in "bsd_close_page_cache_files"
-			 * which is called after the read or writes to
-			 * these files are done
-			 */
-			if ( (vnode_getwithref((*profile)->data_vp)) ) {
-			  
-			        vnode_rele((*profile)->data_vp);
-			        vnode_rele((*profile)->names_vp);
-
-				(*profile)->data_vp = NULL;
-				(*profile)->busy = 0;
-				wakeup(*profile);
-
-				goto restart;
-			}
-			if ( (vnode_getwithref((*profile)->names_vp)) ) {
-
-			        vnode_put((*profile)->data_vp);
-			        vnode_rele((*profile)->data_vp);
-			        vnode_rele((*profile)->names_vp);
-
-				(*profile)->data_vp = NULL;
-				(*profile)->busy = 0;
-				wakeup(*profile);
-
-				goto restart;
-			}
-			global_user_profile_cache.age+=1;
-			return 0;
-		}
-	}
-
-	lru = global_user_profile_cache.age;
-	*profile = NULL;
-	for(i = 0; i<global_user_profile_cache.max_ele; i++) {
-		/* Skip entry if it is in the process of being reused */
-		if(global_user_profile_cache.profiles[i].data_vp ==
-						(struct vnode *)0xFFFFFFFF)
-			continue;
-		/* Otherwise grab the first empty entry */
-		if(global_user_profile_cache.profiles[i].data_vp == NULL) {
-			*profile = &global_user_profile_cache.profiles[i];
-			(*profile)->age = global_user_profile_cache.age;
-			break;
-		}
-		/* Otherwise grab the oldest entry */
-		if(global_user_profile_cache.profiles[i].age < lru) {
-			lru = global_user_profile_cache.profiles[i].age;
-			*profile = &global_user_profile_cache.profiles[i];
-		}
-	}
-
-	/* Did we set it? */
-	if (*profile == NULL) {
-		/*
-		 * No entries are available; this can only happen if all
-		 * of them are currently in the process of being reused;
-		 * if this happens, we sleep on the address of the first
-		 * element, and restart.  This is less than ideal, but we
-		 * know it will work because we know that there will be a
-		 * wakeup on any entry currently in the process of being
-		 * reused.
-		 *
-		 * XXX Reccomend a two handed clock and more than 3 total
-		 * XXX cache entries at some point in the future.
-		 */
-       		/*
-       		* drop funnel and wait 
-       		*/
-		(void)tsleep((void *)
-		 &global_user_profile_cache.profiles[0],
-			PRIBIO, "app_profile", 0);
-		goto restart;
-	}
-
-	/*
-	 * If it's currently busy, we've picked the one at the end of the
-	 * LRU list, but it's currently being actively used.  We sleep on
-	 * its address and restart.
-	 */
-	if ((*profile)->busy) {
-       		/*
-       		* drop funnel and wait 
-       		*/
-		(void)tsleep((void *)
-			*profile, 
-			PRIBIO, "app_profile", 0);
-		goto restart;
-	}
-	(*profile)->busy = 1;
-	(*profile)->user = user;
-
-	/*
-	 * put dummy value in for now to get competing request to wait
-	 * above until we are finished
-	 *
-	 * Save the data_vp before setting it, so we can set it before
-	 * we kmem_free() or vrele().  If we don't do this, then we
-	 * have a potential funnel race condition we have to deal with.
-	 */
-	data_vp = (*profile)->data_vp;
-	(*profile)->data_vp = (struct vnode *)0xFFFFFFFF;
-
-	/*
-	 * Age the cache here in all cases; this guarantees that we won't
-	 * be reusing only one entry over and over, once the system reaches
-	 * steady-state.
-	 */
-	global_user_profile_cache.age+=1;
-
-	if(data_vp != NULL) {
-		kmem_free(kernel_map, 
-				(*profile)->buf_ptr, 4 * PAGE_SIZE);
-		if ((*profile)->names_vp) {
-			vnode_rele((*profile)->names_vp);
-			(*profile)->names_vp = NULL;
-		}
-		vnode_rele(data_vp);
-	}
-	
-	/* Try to open the appropriate users profile files */
-	/* If neither file is present, try to create them  */
-	/* If one file is present and the other not, fail. */
-	/* If the files do exist, check them for the app_file */
-	/* requested and read it in if present */
-
-	ret = kmem_alloc(kernel_map,
-		(vm_offset_t *)&profile_data_string, PATH_MAX);
-
-	if(ret) {
-		(*profile)->data_vp = NULL;
-		(*profile)->busy = 0;
-		wakeup(*profile);
-		return ENOMEM;
-	}
-
-	/* Split the buffer in half since we know the size of */
-	/* our file path and our allocation is adequate for   */
-	/* both file path names */
-	profile_names_string = profile_data_string + (PATH_MAX/2);
-
-
-	strcpy(profile_data_string, cache_path);
-	strcpy(profile_names_string, cache_path);
-	profile_names_length = profile_data_length 
-			= strlen(profile_data_string);
-	substring = profile_data_string + profile_data_length;
-	sprintf(substring, "%x_data", user);
-	substring = profile_names_string + profile_names_length;
-	sprintf(substring, "%x_names", user);
-
-	/* We now have the absolute file names */
-
-	ret = kmem_alloc(kernel_map,
-       			(vm_offset_t *)&names_buf, 4 * PAGE_SIZE);
-	if(ret) {
-		kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, PATH_MAX);
-		(*profile)->data_vp = NULL;
-		(*profile)->busy = 0;
-		wakeup(*profile);
-		return ENOMEM;
-	}
-
-	NDINIT(&nd_names, LOOKUP, FOLLOW | LOCKLEAF, 
-			UIO_SYSSPACE32, CAST_USER_ADDR_T(profile_names_string), &context);
-	NDINIT(&nd_data, LOOKUP, FOLLOW | LOCKLEAF, 
-			UIO_SYSSPACE32, CAST_USER_ADDR_T(profile_data_string), &context);
-
-	if ( (error = vn_open(&nd_data, FREAD | FWRITE, 0)) ) {
-#ifdef notdef
-		printf("bsd_open_page_cache_files: CacheData file not found %s\n",
-			profile_data_string);
-#endif
-		kmem_free(kernel_map, 
-				(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-		kmem_free(kernel_map, 
-			(vm_offset_t)profile_data_string, PATH_MAX);
-		(*profile)->data_vp = NULL;
-		(*profile)->busy = 0;
-		wakeup(*profile);
-		return error;
-	}
-	data_vp = nd_data.ni_vp;
-
-	if ( (error = vn_open(&nd_names, FREAD | FWRITE, 0)) ) {
-		printf("bsd_open_page_cache_files: NamesData file not found %s\n",
-			profile_data_string);
-		kmem_free(kernel_map, 
-				(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-		kmem_free(kernel_map, 
-			(vm_offset_t)profile_data_string, PATH_MAX);
-
-		vnode_rele(data_vp);
-		vnode_put(data_vp);
-
-		(*profile)->data_vp = NULL;
-		(*profile)->busy = 0;
-		wakeup(*profile);
-		return error;
-	}
-	names_vp = nd_names.ni_vp;
-
-	if ((error = vnode_size(names_vp, &file_size, &context)) != 0) {
-		printf("bsd_open_page_cache_files: Can't stat name file %s\n", profile_names_string);
-		kmem_free(kernel_map, 
-			(vm_offset_t)profile_data_string, PATH_MAX);
-		kmem_free(kernel_map, 
-			(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-
-		vnode_rele(names_vp);
-		vnode_put(names_vp);
-		vnode_rele(data_vp);
-		vnode_put(data_vp);
-
-		(*profile)->data_vp = NULL;
-		(*profile)->busy = 0;
-		wakeup(*profile);
-		return error;
-	}
 
-	size = file_size;
-	if(size > 4 * PAGE_SIZE) 
-		size = 4 * PAGE_SIZE;
-	buf_ptr = names_buf;
-	resid_off = 0;
-
-	while(size) {
-		int resid_int;
-		error = vn_rdwr(UIO_READ, names_vp, (caddr_t)buf_ptr, 
-			size, resid_off,
-			UIO_SYSSPACE32, IO_NODELOCKED, kauth_cred_get(),
-			&resid_int, p);
-		resid = (vm_size_t) resid_int;
-		if((error) || (size == resid)) {
-			if(!error) {
-				error = EINVAL;
-			}
-			kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, PATH_MAX);
-			kmem_free(kernel_map, 
-				(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-
-			vnode_rele(names_vp);
-			vnode_put(names_vp);
-			vnode_rele(data_vp);
-			vnode_put(data_vp);
-
-			(*profile)->data_vp = NULL;
-			(*profile)->busy = 0;
-			wakeup(*profile);
-			return error;
-		}
-		buf_ptr += size-resid;
-		resid_off += size-resid;
-		size = resid;
-	}
-	kmem_free(kernel_map, (vm_offset_t)profile_data_string, PATH_MAX);
-
-	(*profile)->names_vp = names_vp;
-	(*profile)->data_vp = data_vp;
-	(*profile)->buf_ptr = names_buf;
-
-	/*
-	 * at this point, the both the names_vp and the data_vp have
-	 * both a valid usecount and an iocount held
-	 */
-	return 0;
-
-}
-
-void
-bsd_close_page_cache_files(
-	struct global_profile *profile)
-{
-        vnode_put(profile->data_vp);
-	vnode_put(profile->names_vp);
-
-	profile->busy = 0;
-	wakeup(profile);
-}
-
-int
-bsd_read_page_cache_file(
-	unsigned int	user,
-	int		*fid,
-	int		*mod,
-	char		*app_name,
-	struct vnode	*app_vp,
-	vm_offset_t	*buffer,
-	vm_offset_t	*bufsize)
-{
-
-	boolean_t	funnel_state;
-
-	struct proc	*p;
-	int		error;
-	unsigned int	resid;
-
-	off_t		profile;
-	unsigned int	profile_size;
-
-	vm_offset_t	names_buf;
-	struct vnode_attr	va;
-	struct vfs_context  context;
-
-	kern_return_t	ret;
-
-	struct	vnode	*names_vp;
-	struct	vnode	*data_vp;
-
-	struct global_profile *uid_files;
-
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
-	/* Try to open the appropriate users profile files */
-	/* If neither file is present, try to create them  */
-	/* If one file is present and the other not, fail. */
-	/* If the files do exist, check them for the app_file */
-	/* requested and read it in if present */
-
-
-	error = bsd_open_page_cache_files(user, &uid_files);
-	if(error) {
-		thread_funnel_set(kernel_flock, funnel_state);
-		return EINVAL;
-	}
-
-	p = current_proc();
-
-	names_vp = uid_files->names_vp;
-	data_vp = uid_files->data_vp;
-	names_buf = uid_files->buf_ptr;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	VATTR_INIT(&va);
-	VATTR_WANTED(&va, va_fileid);
-	VATTR_WANTED(&va, va_modify_time);
-	
-	if ((error = vnode_getattr(app_vp, &va, &context))) {
-		printf("bsd_read_cache_file: Can't stat app file %s\n", app_name);
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return error;
-	}
-
-	*fid = (u_long)va.va_fileid;
-	*mod = va.va_modify_time.tv_sec;
-		
-	if (bsd_search_page_cache_data_base(
-		    names_vp,
-		    (struct profile_names_header *)names_buf,
-		    app_name, 
-		    (unsigned int) va.va_modify_time.tv_sec,  
-		    (u_long)va.va_fileid, &profile, &profile_size) == 0) {
-		/* profile is an offset in the profile data base */
-		/* It is zero if no profile data was found */
-		
-		if(profile_size == 0) {
-			*buffer = 0;
-			*bufsize = 0;
-			bsd_close_page_cache_files(uid_files);
-			thread_funnel_set(kernel_flock, funnel_state);
-			return 0;
-		}
-		ret = (vm_offset_t)(kmem_alloc(kernel_map, buffer, profile_size));
-		if(ret) {
-			bsd_close_page_cache_files(uid_files);
-			thread_funnel_set(kernel_flock, funnel_state);
-			return ENOMEM;
-		}
-		*bufsize = profile_size;
-		while(profile_size) {
-			int resid_int;
-			error = vn_rdwr(UIO_READ, data_vp, 
-				(caddr_t) *buffer, profile_size, 
-				profile, UIO_SYSSPACE32, IO_NODELOCKED, 
-				kauth_cred_get(), &resid_int, p);
-			resid = (vm_size_t) resid_int;
-			if((error) || (profile_size == resid)) {
-				bsd_close_page_cache_files(uid_files);
-				kmem_free(kernel_map, (vm_offset_t)*buffer, profile_size);
-				thread_funnel_set(kernel_flock, funnel_state);
-				return EINVAL;
-			}
-		        profile += profile_size - resid;
-			profile_size = resid;
-		}
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return 0;
-	} else {
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return EINVAL;
-	}
-	
-}
-
-int
-bsd_search_page_cache_data_base(
-	struct	vnode			*vp,
-	struct profile_names_header	*database,
-	char				*app_name,
-	unsigned int			mod_date,
-	unsigned int			inode,
-	off_t				*profile,
-	unsigned int			*profile_size)
-{
-
-	struct proc		*p;
-
-	unsigned int 		i;
-	struct profile_element	*element;
-	unsigned int		ele_total;
-	unsigned int		extended_list = 0;
-	off_t			file_off = 0;
-	unsigned int		size;
-	off_t			resid_off;
-	unsigned int		resid;
-	vm_offset_t		local_buf = 0;
-
-	int			error;
-	kern_return_t		ret;
-
-	p = current_proc();
-
-	if(((vm_offset_t)database->element_array) !=
-				sizeof(struct profile_names_header)) {
-		return EINVAL;
-	}
-	element = (struct profile_element *)(
-			(vm_offset_t)database->element_array + 
-						(vm_offset_t)database);
-
-	ele_total = database->number_of_profiles;
-	
-	*profile = 0;
-	*profile_size = 0;
-	while(ele_total) {
-		/* note: code assumes header + n*ele comes out on a page boundary */
-		if(((local_buf == 0) && (sizeof(struct profile_names_header) + 
-			(ele_total * sizeof(struct profile_element))) 
-					> (PAGE_SIZE * 4)) ||
-			((local_buf != 0) && 
-				(ele_total * sizeof(struct profile_element))
-					 > (PAGE_SIZE * 4))) {
-			extended_list = ele_total;
-			if(element == (struct profile_element *)
-				((vm_offset_t)database->element_array + 
-						(vm_offset_t)database)) {
-				ele_total = ((PAGE_SIZE * 4)/sizeof(struct profile_element)) - 1;
-			} else {
-				ele_total = (PAGE_SIZE * 4)/sizeof(struct profile_element);
-			}
-			extended_list -= ele_total;
-		}
-		for (i=0; i<ele_total; i++) {
-			if((mod_date == element[i].mod_date) 
-					&& (inode == element[i].inode)) {
-				if(strncmp(element[i].name, app_name, 12) == 0) {
-					*profile = element[i].addr;
-					*profile_size = element[i].size;
-					if(local_buf != 0) {
-						kmem_free(kernel_map, local_buf, 4 * PAGE_SIZE);
-					}
-					return 0;
-				}
-			}
-		}
-		if(extended_list == 0)
-			break;
-		if(local_buf == 0) {
-			ret = kmem_alloc(kernel_map, &local_buf, 4 * PAGE_SIZE);
-			if(ret != KERN_SUCCESS) {
-				return ENOMEM;
-			}
-		}
-		element = (struct profile_element *)local_buf;
-		ele_total = extended_list;
-		extended_list = 0;
-		file_off +=  4 * PAGE_SIZE;
-		if((ele_total * sizeof(struct profile_element)) > 
-							(PAGE_SIZE * 4)) {
-			size = PAGE_SIZE * 4;
-		} else {
-			size = ele_total * sizeof(struct profile_element);
-		}
-		resid_off = 0;
-		while(size) {
-			int resid_int;
-			error = vn_rdwr(UIO_READ, vp, 
-				CAST_DOWN(caddr_t, (local_buf + resid_off)),
-				size, file_off + resid_off, UIO_SYSSPACE32, 
-				IO_NODELOCKED, kauth_cred_get(), &resid_int, p);
-			resid = (vm_size_t) resid_int;
-			if((error) || (size == resid)) {
-				if(local_buf != 0) {
-					kmem_free(kernel_map, local_buf, 4 * PAGE_SIZE);
-				}
-				return EINVAL;
-			}
-			resid_off += size-resid;
-			size = resid;
-		}
-	}
-	if(local_buf != 0) {
-		kmem_free(kernel_map, local_buf, 4 * PAGE_SIZE);
-	}
-	return 0;
-}
-
-int
-bsd_write_page_cache_file(
-	unsigned int	user,
-	char	 	*file_name,
-	caddr_t		buffer,
-	vm_size_t	size,
-	int		mod,
-	int		fid)
-{
-	struct proc		*p;
-	int				resid;
-	off_t			resid_off;
-	int				error;
-	boolean_t		funnel_state;
-	off_t			file_size;
-	struct vfs_context	context;
-	off_t			profile;
-	unsigned int	profile_size;
-
-	vm_offset_t	names_buf;
-	struct	vnode	*names_vp;
-	struct	vnode	*data_vp;
-	struct	profile_names_header *profile_header;
-	off_t			name_offset;
-	struct global_profile *uid_files;
-
-
-	funnel_state = thread_funnel_set(kernel_flock, TRUE);
-
-
-	error = bsd_open_page_cache_files(user, &uid_files);
-	if(error) {
-		thread_funnel_set(kernel_flock, funnel_state);
-		return EINVAL;
+	if (shared_region != NULL) {
+		vm_shared_region_deallocate(shared_region);
 	}
 
-	p = current_proc();
-
-	names_vp = uid_files->names_vp;
-	data_vp = uid_files->data_vp;
-	names_buf = uid_files->buf_ptr;
-
-	/* Stat data file for size */
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	if ((error = vnode_size(data_vp, &file_size, &context)) != 0) {
-		printf("bsd_write_page_cache_file: Can't stat profile data %s\n", file_name);
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return error;
-	}
-		
-	if (bsd_search_page_cache_data_base(names_vp, 
-			(struct profile_names_header *)names_buf, 
-			file_name, (unsigned int) mod,  
-			fid, &profile, &profile_size) == 0) {
-		/* profile is an offset in the profile data base */
-		/* It is zero if no profile data was found */
-		
-		if(profile_size == 0) {
-			unsigned int	header_size;
-			vm_offset_t	buf_ptr;
-
-			/* Our Write case */
-
-			/* read header for last entry */
-			profile_header = 
-				(struct profile_names_header *)names_buf;
-			name_offset = sizeof(struct profile_names_header) + 
-				(sizeof(struct profile_element) 
-					* profile_header->number_of_profiles);
-			profile_header->number_of_profiles += 1;
-
-			if(name_offset < PAGE_SIZE * 4) {
-				struct profile_element	*name;
-				/* write new entry */
-				name = (struct profile_element *)
-					(names_buf + (vm_offset_t)name_offset);
-				name->addr =  file_size;
-				name->size = size;
-				name->mod_date = mod;
-				name->inode = fid;
-				strncpy (name->name, file_name, 12);
-			} else {
-				unsigned int	ele_size;
-				struct profile_element	name;
-				/* write new entry */
-				name.addr = file_size;
-				name.size = size;
-				name.mod_date = mod;
-				name.inode = fid;
-				strncpy (name.name, file_name, 12);
-				/* write element out separately */
-				ele_size = sizeof(struct profile_element);
-				buf_ptr = (vm_offset_t)&name;
-				resid_off = name_offset;
-
-				while(ele_size) {
-					error = vn_rdwr(UIO_WRITE, names_vp, 
-						(caddr_t)buf_ptr, 
-						ele_size, resid_off, 
-						UIO_SYSSPACE32, IO_NODELOCKED, 
-						kauth_cred_get(), &resid, p);
-					if(error) {
-						printf("bsd_write_page_cache_file: Can't write name_element %x\n", user);
-						bsd_close_page_cache_files(
-							uid_files);
-						thread_funnel_set(
-							kernel_flock, 
-							funnel_state);
-						return error;
-					}
-					buf_ptr += (vm_offset_t)
-							ele_size-resid;
-					resid_off += ele_size-resid;
-					ele_size = resid;
-				}
-			}
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: %p [%d(%s)] <- map\n",
+		 current_thread(), p->p_pid, p->p_comm));
 
-			if(name_offset < PAGE_SIZE * 4) {
-				header_size = name_offset + 
-					sizeof(struct profile_element);
-				
-			} else {
-				header_size = 
-					sizeof(struct profile_names_header);
-			}
-			buf_ptr = (vm_offset_t)profile_header;
-			resid_off = 0;
-
-			/* write names file header */
-			while(header_size) {
-				error = vn_rdwr(UIO_WRITE, names_vp, 
-					(caddr_t)buf_ptr, 
-					header_size, resid_off, 
-					UIO_SYSSPACE32, IO_NODELOCKED, 
-					kauth_cred_get(), &resid, p);
-				if(error) {
-					printf("bsd_write_page_cache_file: Can't write header %x\n", user);
-					bsd_close_page_cache_files(
-						uid_files);
-					thread_funnel_set(
-						kernel_flock, funnel_state);
-					return error;
-				}
-				buf_ptr += (vm_offset_t)header_size-resid;
-				resid_off += header_size-resid;
-				header_size = resid;
-			}
-			/* write profile to data file */
-			resid_off = file_size;
-			while(size) {
-				error = vn_rdwr(UIO_WRITE, data_vp, 
-					(caddr_t)buffer, size, resid_off, 
-					UIO_SYSSPACE32, IO_NODELOCKED, 
-					kauth_cred_get(), &resid, p);
-				if(error) {
-					printf("bsd_write_page_cache_file: Can't write header %x\n", user);
-					bsd_close_page_cache_files(
-						uid_files);
-					thread_funnel_set(
-						kernel_flock, funnel_state);
-					return error;
-				}
-				buffer += size-resid;
-				resid_off += size-resid;
-				size = resid;
-			}
-			bsd_close_page_cache_files(uid_files);
-			thread_funnel_set(kernel_flock, funnel_state);
-			return 0;
-		}
-		/* Someone else wrote a twin profile before us */
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return 0;
-	} else {		
-		bsd_close_page_cache_files(uid_files);
-		thread_funnel_set(kernel_flock, funnel_state);
-		return EINVAL;
-	}
-	
+	return error;
 }
 
-int
-prepare_profile_database(int	user)
-{
-	const char *cache_path = "/var/vm/app_profile/";
-	struct proc	*p;
-	int		error;
-	int		resid;
-	off_t		resid_off;
-	vm_size_t	size;
-
-	struct	vnode	*names_vp;
-	struct  vnode	*data_vp;
-	vm_offset_t	names_buf;
-	vm_offset_t	buf_ptr;
-
-	int		profile_names_length;
-	int		profile_data_length;
-	char		*profile_data_string;
-	char		*profile_names_string;
-	char		*substring;
-
-	struct vnode_attr va;
-	struct vfs_context context;
 
-	struct	profile_names_header *profile_header;
-	kern_return_t	ret;
+/* sysctl overflow room */
 
-	struct nameidata nd_names;
-	struct nameidata nd_data;
+/* vm_page_free_target is provided as a makeshift solution for applications that want to
+	allocate buffer space, possibly purgeable memory, but not cause inactive pages to be
+	reclaimed. It allows the app to calculate how much memory is free outside the free target. */
+extern unsigned int	vm_page_free_target;
+SYSCTL_INT(_vm, OID_AUTO, vm_page_free_target, CTLFLAG_RD, 
+		   &vm_page_free_target, 0, "Pageout daemon free target");
 
-	p = current_proc();
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
-
-	ret = kmem_alloc(kernel_map,
-		(vm_offset_t *)&profile_data_string, PATH_MAX);
-
-	if(ret) {
-		return ENOMEM;
-	}
-
-	/* Split the buffer in half since we know the size of */
-	/* our file path and our allocation is adequate for   */
-	/* both file path names */
-	profile_names_string = profile_data_string + (PATH_MAX/2);
-
-
-	strcpy(profile_data_string, cache_path);
-	strcpy(profile_names_string, cache_path);
-	profile_names_length = profile_data_length 
-			= strlen(profile_data_string);
-	substring = profile_data_string + profile_data_length;
-	sprintf(substring, "%x_data", user);
-	substring = profile_names_string + profile_names_length;
-	sprintf(substring, "%x_names", user);
-
-	/* We now have the absolute file names */
-
-	ret = kmem_alloc(kernel_map,
-       			(vm_offset_t *)&names_buf, 4 * PAGE_SIZE);
-	if(ret) {
-		kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, PATH_MAX);
-		return ENOMEM;
-	}
-
-	NDINIT(&nd_names, LOOKUP, FOLLOW, 
-			UIO_SYSSPACE32, CAST_USER_ADDR_T(profile_names_string), &context);
-	NDINIT(&nd_data, LOOKUP, FOLLOW,
-			UIO_SYSSPACE32, CAST_USER_ADDR_T(profile_data_string), &context);
-
-	if ( (error = vn_open(&nd_data, 
-							O_CREAT | O_EXCL | FWRITE, S_IRUSR|S_IWUSR)) ) {
-			kmem_free(kernel_map, 
-					(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-			kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, PATH_MAX);
-			
-			return 0;
-	}
-	data_vp = nd_data.ni_vp;
-
-	if ( (error = vn_open(&nd_names, 
-							O_CREAT | O_EXCL | FWRITE, S_IRUSR|S_IWUSR)) ) {
-			printf("prepare_profile_database: Can't create CacheNames %s\n",
-				profile_data_string);
-			kmem_free(kernel_map, 
-					(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-			kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, PATH_MAX);
-
-			vnode_rele(data_vp);
-			vnode_put(data_vp);
-
-			return error;
-	}
-	names_vp = nd_names.ni_vp;
-
-	/* Write Header for new names file */
-
-	profile_header = (struct profile_names_header *)names_buf;
-
-	profile_header->number_of_profiles = 0;
-	profile_header->user_id =  user;
-	profile_header->version = 1;
-	profile_header->element_array = 
-				sizeof(struct profile_names_header);
-	profile_header->spare1 = 0;
-	profile_header->spare2 = 0;
-	profile_header->spare3 = 0;
-
-	size = sizeof(struct profile_names_header);
-	buf_ptr = (vm_offset_t)profile_header;
-	resid_off = 0;
-
-	while(size) {
-		error = vn_rdwr(UIO_WRITE, names_vp, 
-				(caddr_t)buf_ptr, size, resid_off,
-				UIO_SYSSPACE32, IO_NODELOCKED, 
-				kauth_cred_get(), &resid, p);
-		if(error) {
-			printf("prepare_profile_database: Can't write header %s\n", profile_names_string);
-			kmem_free(kernel_map, 
-				(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-			kmem_free(kernel_map, 
-				(vm_offset_t)profile_data_string, 
-				PATH_MAX);
-
-			vnode_rele(names_vp);
-			vnode_put(names_vp);
-			vnode_rele(data_vp);
-			vnode_put(data_vp);
-
-			return error;
-		}
-		buf_ptr += size-resid;
-		resid_off += size-resid;
-		size = resid;
-	}
-	VATTR_INIT(&va);
-	VATTR_SET(&va, va_uid, user);
-
-       	error = vnode_setattr(names_vp, &va, &context);
-	if(error) {
-		printf("prepare_profile_database: "
-			"Can't set user %s\n", profile_names_string);
-	}
-	vnode_rele(names_vp);
-	vnode_put(names_vp);
-	
-	VATTR_INIT(&va);
-	VATTR_SET(&va, va_uid, user);
-       	error = vnode_setattr(data_vp, &va, &context);
-	if(error) {
-		printf("prepare_profile_database: "
-			"Can't set user %s\n", profile_data_string);
-	}
-	vnode_rele(data_vp);
-	vnode_put(data_vp);
-
-	kmem_free(kernel_map, 
-			(vm_offset_t)profile_data_string, PATH_MAX);
-	kmem_free(kernel_map, 
-			(vm_offset_t)names_buf, 4 * PAGE_SIZE);
-	return 0;
-
-}
diff --git a/bsd/vm/vnode_pager.c b/bsd/vm/vnode_pager.c
index c994f396b..e53ae4e15 100644
--- a/bsd/vm/vnode_pager.c
+++ b/bsd/vm/vnode_pager.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -49,6 +55,7 @@
 
 #include <mach/mach_types.h>
 #include <mach/memory_object_types.h>
+#include <mach/sdt.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
@@ -102,12 +109,21 @@ vnode_pager_get_pathname(
 kern_return_t
 vnode_pager_get_filename(
 	struct vnode	*vp,
-	char		**filename)
+	const char	**filename)
 {
 	*filename = vp->v_name;
 	return KERN_SUCCESS;
 }
 
+kern_return_t
+vnode_pager_get_cs_blobs(
+	struct vnode	*vp,
+	void		**blobs)
+{
+	*blobs = ubc_get_cs_blobs(vp);
+	return KERN_SUCCESS;
+}
+
 pager_return_t
 vnode_pageout(struct vnode *vp,
 	upl_t			upl,
@@ -117,7 +133,6 @@ vnode_pageout(struct vnode *vp,
 	int			flags,
 	int			*errorp)
 {
-	struct proc 	*p = current_proc();
 	int		result = PAGER_SUCCESS;
 	int		error = 0;
 	int		error_ret = 0;
@@ -127,10 +142,7 @@ vnode_pageout(struct vnode *vp,
 	int base_index;
 	int offset;
 	upl_page_info_t *pl;
-	struct vfs_context context;
-
-	context.vc_proc = p;
-	context.vc_ucred = kauth_cred_get();
+	vfs_context_t ctx = vfs_context_current();	/* pager context */
 
 	isize = (int)size;
 
@@ -139,9 +151,8 @@ vnode_pageout(struct vnode *vp,
 		error_ret = EINVAL;
 		goto out;
 	}
-	UBCINFOCHECK("vnode_pageout", vp);
 
-	if (UBCINVALID(vp)) {
+	if (UBCINFOEXISTS(vp) == 0) {
 		result    = PAGER_ERROR;
 		error_ret = EINVAL;
 
@@ -161,7 +172,7 @@ vnode_pageout(struct vnode *vp,
 				      size, 1, 0, 0, 0);
 
 		if ( (error_ret = VNOP_PAGEOUT(vp, upl, upl_offset, (off_t)f_offset,
-					       (size_t)size, flags, &context)) )
+					       (size_t)size, flags, ctx)) )
 			result = PAGER_ERROR;
 
 		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
@@ -232,7 +243,7 @@ vnode_pageout(struct vnode *vp,
 			goto out;
 		}
 	}
-	isize = (pg_index + 1) * PAGE_SIZE;
+	isize = ((pg_index + 1) - base_index) * PAGE_SIZE;
 
 	offset = upl_offset;
 	pg_index = base_index;
@@ -247,8 +258,9 @@ vnode_pageout(struct vnode *vp,
 			 * to get back empty slots in the UPL
 			 * just skip over them
 			 */
-			offset += PAGE_SIZE;
-			isize  -= PAGE_SIZE;
+		        f_offset += PAGE_SIZE;
+			offset   += PAGE_SIZE;
+			isize    -= PAGE_SIZE;
 			pg_index++;
 
 			continue;
@@ -269,11 +281,11 @@ vnode_pageout(struct vnode *vp,
 #if NFSCLIENT
 			if (vp->v_tag == VT_NFS)
 				/* check with nfs if page is OK to drop */
-				error = nfs_buf_page_inval(vp, (off_t)(f_offset + offset));
+				error = nfs_buf_page_inval(vp, (off_t)f_offset);
 			else
 #endif
 			{
-			        blkno = ubc_offtoblk(vp, (off_t)(f_offset + offset));
+			        blkno = ubc_offtoblk(vp, (off_t)f_offset);
 			        error = buf_invalblkno(vp, blkno, 0);
 			}
 			if (error) {
@@ -286,8 +298,9 @@ vnode_pageout(struct vnode *vp,
 			} else if ( !(flags & UPL_NOCOMMIT)) {
 			        ubc_upl_commit_range(upl, offset, PAGE_SIZE, UPL_COMMIT_FREE_ON_EMPTY);
 			}
-			offset += PAGE_SIZE;
-			isize  -= PAGE_SIZE;
+		        f_offset += PAGE_SIZE;
+			offset   += PAGE_SIZE;
+			isize    -= PAGE_SIZE;
 			pg_index++;
 
 			continue;
@@ -306,11 +319,10 @@ vnode_pageout(struct vnode *vp,
 		xsize = num_of_pages * PAGE_SIZE;
 
 		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_START, 
-				      xsize, (int)(f_offset + offset), 0, 0, 0);
+				      xsize, (int)f_offset, 0, 0, 0);
 
-		if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset,
-					(off_t)(f_offset + offset), xsize,
-					   flags, &context)) ) {
+		if ( (error = VNOP_PAGEOUT(vp, upl, (vm_offset_t)offset, (off_t)f_offset,
+					   xsize, flags, ctx)) ) {
 		        if (error_ret == 0)
 		                error_ret = error;
 			result = PAGER_ERROR;
@@ -318,8 +330,9 @@ vnode_pageout(struct vnode *vp,
 		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 1)) | DBG_FUNC_END, 
 				      xsize, 0, 0, 0, 0);
 
-		offset += xsize;
-		isize  -= xsize;
+	        f_offset += xsize;
+		offset   += xsize;
+		isize    -= xsize;
 		pg_index += num_of_pages;
 	}
 out:
@@ -330,7 +343,7 @@ out:
 }
 
 
-void IOSleep(int);
+extern void throttle_lowpri_io(int *lowpri_window,mount_t v_mount);
 
 pager_return_t
 vnode_pagein(
@@ -342,7 +355,6 @@ vnode_pagein(
 	int           		flags,
 	int 			*errorp)
 {
-        struct proc     *p = current_proc();
         struct uthread	*ut;
         upl_page_info_t *pl;
 	int	        result = PAGER_SUCCESS;
@@ -352,48 +364,59 @@ vnode_pagein(
         int             last_pg;
 	int             first_pg;
         int             xsize;
-	int             abort_needed = 1;
-
+	int		must_commit = 1;
 
-	UBCINFOCHECK("vnode_pagein", vp);
+	if (flags & UPL_NOCOMMIT)
+	        must_commit = 0;
 
-	if (UBCINVALID(vp)) {
+	if (UBCINFOEXISTS(vp) == 0) {
 		result = PAGER_ERROR;
 		error  = PAGER_ERROR;
-		if (upl && !(flags & UPL_NOCOMMIT)) {
+
+		if (upl && must_commit)
 			ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
-		}
+
 		goto out;
 	}
 	if (upl == (upl_t)NULL) {
 	        if (size > (MAX_UPL_TRANSFER * PAGE_SIZE)) {
+
+		  panic("vnode_pagein: size = %x\n", size);
+
 		        result = PAGER_ERROR;
 			error  = PAGER_ERROR;
 			goto out;
 		}
-	        ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
+	        ubc_create_upl(vp, f_offset, size, &upl, &pl, UPL_NOBLOCK | UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
 
 		if (upl == (upl_t)NULL) {
+
+		  panic("vnode_pagein: ubc_create_upl failed\n");
+
 		        result =  PAGER_ABSENT;
 			error = PAGER_ABSENT;
 			goto out;
 		}
 		upl_offset = 0;
+		first_pg = 0;
+		
 		/*
 		 * if we get here, we've created the upl and
 		 * are responsible for commiting/aborting it
 		 * regardless of what the caller has passed in
 		 */
 		flags &= ~UPL_NOCOMMIT;
+		must_commit = 1;
 		
 		vp_pagein++;
 	} else {
 	        pl = ubc_upl_pageinfo(upl);
+		first_pg = upl_offset / PAGE_SIZE;
 
 		dp_pgins++;
 	}
 	pages_in_upl = size / PAGE_SIZE;
-	first_pg     = upl_offset / PAGE_SIZE;
+	DTRACE_VM2(pgpgin, int, pages_in_upl, (uint64_t *), NULL);
 
 	/*
 	 * before we start marching forward, we must make sure we end on 
@@ -403,30 +426,31 @@ vnode_pagein(
 	for (last_pg = pages_in_upl - 1; last_pg >= first_pg; last_pg--) {
 		if (upl_page_present(pl, last_pg))
 			break;
+		if (last_pg == first_pg) {
+		        /*
+			 * empty UPL, no pages are present
+			 */
+		        if (must_commit)
+			        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
+			goto out;
+		}
 	}
 	pages_in_upl = last_pg + 1;
+	last_pg = first_pg;
 
-	for (last_pg = first_pg; last_pg < pages_in_upl;) {
+	while (last_pg < pages_in_upl) {
 	        /*
-		 * scan the upl looking for the next
-		 * page that is present.... if all of the 
-		 * pages are absent, we're done
+		 * skip over missing pages...
 		 */
-	        for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
+	        for ( ; last_pg < pages_in_upl; last_pg++) {
 		        if (upl_page_present(pl, last_pg))
 			        break;
 		}
-		if (last_pg == pages_in_upl)
-		        break;
-
 	        /*
-		 * if we get here, we've sitting on a page 
-		 * that is present... we want to skip over
-		 * any range of 'valid' pages... if this takes
-		 * us to the end of the request, than we're done
+		 * skip over 'valid' pages... we don't want to issue I/O for these
 		 */
 	        for (start_pg = last_pg; last_pg < pages_in_upl; last_pg++) {
-		        if (!upl_valid_page(pl, last_pg) || !upl_page_present(pl, last_pg))
+		        if (!upl_valid_page(pl, last_pg))
 			        break;
 		}
 		if (last_pg > start_pg) {
@@ -438,22 +462,26 @@ vnode_pagein(
 			 */
 		        xsize = (last_pg - start_pg) * PAGE_SIZE;
 
-			if (!(flags & UPL_NOCOMMIT))
+			if (must_commit)
 			        ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, xsize, UPL_ABORT_FREE_ON_EMPTY);
-
-			abort_needed = 0;
 		}
 		if (last_pg == pages_in_upl)
+		        /*
+			 * we're done... all pages that were present
+			 * have either had I/O issued on them or 
+			 * were aborted unchanged...
+			 */
 		        break;
 
-		if (!upl_page_present(pl, last_pg))
+		if (!upl_page_present(pl, last_pg)) {
 		        /*
-			 * if we found a range of valid pages 
-			 * terminated by a non-present page
-			 * than start over
+			 * we found a range of valid pages 
+			 * terminated by a missing page...
+			 * bump index to the next page and continue on
 			 */
+		        last_pg++;
 		        continue;
-
+		}
 		/*
 		 * scan from the found invalid page looking for a valid
 		 * or non-present page before the end of the upl is reached, if we
@@ -466,32 +494,25 @@ vnode_pagein(
 		}
 		if (last_pg > start_pg) {
 		        int xoff;
-			struct vfs_context context;
-
-			context.vc_proc = p;
-			context.vc_ucred = kauth_cred_get();
 		        xsize = (last_pg - start_pg) * PAGE_SIZE;
 			xoff  = start_pg * PAGE_SIZE;
 
 			if ( (error = VNOP_PAGEIN(vp, upl, (vm_offset_t) xoff,
 					       (off_t)f_offset + xoff,
-					       xsize, flags, &context)) ) {
+					       xsize, flags, vfs_context_current())) ) {
 				result = PAGER_ERROR;
 				error  = PAGER_ERROR;
 
 			}
-			abort_needed = 0;
 		}
         }
-	if (!(flags & UPL_NOCOMMIT) && abort_needed)
-	        ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
 out:
 	if (errorp)
 		*errorp = result;
 
 	ut = get_bsdthread_info(current_thread());
 
-	if (ut->uu_lowpri_delay) {
+	if (ut->uu_lowpri_window && ut->v_mount) {
 	        /*
 		 * task is marked as a low priority I/O type
 		 * and the I/O we issued while in this system call
@@ -499,8 +520,7 @@ out:
 		 * delay in order to mitigate the impact of this
 		 * task on the normal operation of the system
 		 */
-		IOSleep(ut->uu_lowpri_delay);
-	        ut->uu_lowpri_delay = 0;
+		throttle_lowpri_io(&ut->uu_lowpri_window,ut->v_mount);
 	}
 	return (error);
 }
diff --git a/bsd/vm/vnode_pager.h b/bsd/vm/vnode_pager.h
index 6db5fcb71..89962ac72 100644
--- a/bsd/vm/vnode_pager.h
+++ b/bsd/vm/vnode_pager.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -140,7 +146,11 @@ extern kern_return_t vnode_pager_get_pathname(
 
 extern kern_return_t vnode_pager_get_filename(
 	struct vnode	*vp,
-	char		**filename);
+	const char	**filename);
+
+extern kern_return_t vnode_pager_get_cs_blobs(
+	struct vnode	*vp,
+	void		**blobs);
 
 #endif	/* KERNEL */
 
diff --git a/config/BSDKernel.exports b/config/BSDKernel.exports
index fbbf9ac5e..678fe7d7a 100644
--- a/config/BSDKernel.exports
+++ b/config/BSDKernel.exports
@@ -8,11 +8,14 @@ __FREE
 __FREE_ZONE
 __MALLOC
 __MALLOC_ZONE
-_bpfattach
 _bcd2bin_data
 _bdevsw_add
 _bdevsw_isfree
 _bdevsw_remove
+_bpf_attach
+_bpf_tap_in
+_bpf_tap_out
+_bpfattach
 _bsd_timeout
 _bsd_untimeout
 _buf_alloc
@@ -38,6 +41,7 @@ _buf_drvdata
 _buf_error
 _buf_flags
 _buf_flushdirtyblks
+_buf_fua
 _buf_free
 _buf_fsprivate
 _buf_fromcache
@@ -51,6 +55,7 @@ _buf_map
 _buf_markaged
 _buf_markdelayed
 _buf_markeintr
+_buf_markfua
 _buf_markinvalid
 _buf_meta_bread
 _buf_meta_breadn
@@ -93,11 +98,19 @@ _cdevsw_remove
 _cluster_bp
 _cluster_copy_ubc_data
 _cluster_copy_upl_data
+_advisory_read
 _cluster_pagein
 _cluster_pageout
 _cluster_push
 _cluster_read
 _cluster_write
+_cluster_bp_ext
+_advisory_read_ext
+_cluster_pagein_ext
+_cluster_pageout_ext
+_cluster_push_ext
+_cluster_read_ext
+_cluster_write_ext
 _cluster_zero
 _copystr
 _ctl_deregister
@@ -108,6 +121,7 @@ _ctl_register
 _current_proc
 _desiredvnodes
 _devfs_make_node
+_devfs_make_node_clone
 _devfs_remove
 _enodev
 _enodev_strat
@@ -127,7 +141,6 @@ _err_create
 _err_exchange
 _err_fsync
 _err_getattr
-_err_getattrlist
 _err_inactive
 _err_ioctl
 _err_link
@@ -151,7 +164,6 @@ _err_rmdir
 _err_searchfs
 _err_select
 _err_setattr
-_err_setattrlist
 _err_strategy
 _err_symlink
 _err_whiteout
@@ -242,6 +254,7 @@ _ifnet_output
 _ifnet_output_raw
 _ifnet_reference
 _ifnet_release
+_ifnet_resolve_multicast:_dlil_resolve_multi
 _ifnet_remove_multicast
 _ifnet_set_addrlen
 _ifnet_set_baudrate
@@ -264,6 +277,7 @@ _ifnet_touch_lastchange
 _ifnet_type
 _ifnet_unit
 _iftovt_tab
+_in_cksum:_inet_cksum_simple
 _inet_arp_handle_input:_arp_ip_handle_input
 _inet_arp_init_ifaddr:_arp_ifinit
 _inet_arp_lookup:_arp_lookup_ip
@@ -328,11 +342,12 @@ _ldisc_deregister
 _ldisc_register
 _lightning_bolt
 _mbuf_adj
-_mbuf_allocpacket
+_mbuf_adjustlen
 _mbuf_align_32
-_mbuf_aux_add
-_mbuf_aux_delete
-_mbuf_aux_find
+_mbuf_alloccluster
+_mbuf_allocpacket
+_mbuf_allocpacket_list
+_mbuf_attachcluster
 _mbuf_clear_csum_performed
 _mbuf_clear_csum_requested
 _mbuf_clear_vlan_tag
@@ -340,34 +355,29 @@ _mbuf_copy_pkthdr
 _mbuf_copyback
 _mbuf_copydata
 _mbuf_copym
-_mbuf_data
 _mbuf_data_to_physical
 _mbuf_datastart
 _mbuf_dup
 _mbuf_flags
 _mbuf_free
+_mbuf_freecluster
 _mbuf_freem
 _mbuf_freem_list
 _mbuf_get
-_mbuf_getcluster
 _mbuf_get_csum_performed
 _mbuf_get_csum_requested
 _mbuf_get_vlan_tag
+_mbuf_getcluster
 _mbuf_gethdr
 _mbuf_getpacket
 _mbuf_inbound_modified
+_mbuf_inet_cksum
 _mbuf_leadingspace
-_mbuf_len
 _mbuf_maxlen
 _mbuf_mclget
 _mbuf_mclhasreference
-_mbuf_next
-_mbuf_nextpkt
 _mbuf_outbound_finalize
-_mbuf_pkthdr_header
-_mbuf_pkthdr_len
-_mbuf_pkthdr_rcvif
-_mbuf_pkthdr_setheader
+_mbuf_pkthdr_adjustlen
 _mbuf_pkthdr_setlen
 _mbuf_pkthdr_setrcvif
 _mbuf_prepend
@@ -379,25 +389,21 @@ _mbuf_set_vlan_tag
 _mbuf_setdata
 _mbuf_setflags
 _mbuf_setflags_mask
-_mbuf_setlen
 _mbuf_setnext
-_mbuf_setnextpkt
 _mbuf_settype
 _mbuf_split
 _mbuf_stats
 _mbuf_tag_allocate
 _mbuf_tag_find
-_mbuf_tag_id_find
 _mbuf_tag_free
+_mbuf_tag_id_find
 _mbuf_trailingspace
-_mbuf_type
 _microtime
 _microuptime
 _minphys
 _msleep
 _nanotime
 _nanouptime
-_nd6_lookup_ipv6
 _net_init_add
 _nop_access
 _nop_advlock
@@ -411,7 +417,6 @@ _nop_create
 _nop_exchange
 _nop_fsync
 _nop_getattr
-_nop_getattrlist
 _nop_inactive
 _nop_ioctl
 _nop_link
@@ -435,7 +440,6 @@ _nop_rmdir
 _nop_searchfs
 _nop_select
 _nop_setattr
-_nop_setattrlist
 _nop_strategy
 _nop_symlink
 _nop_whiteout
@@ -444,6 +448,7 @@ _nulldev
 _nullop
 _physio
 _postevent
+_proc_exiting
 _proc_find
 _proc_forcequota
 _proc_is64bit
@@ -484,7 +489,6 @@ _sflt_attach
 _sflt_detach
 _sflt_register
 _sflt_unregister
-_snprintf
 _sock_accept
 _sock_bind
 _sock_close
@@ -590,6 +594,7 @@ _ubc_info_deallocate
 _ubc_info_init
 _ubc_info_zone
 _ubc_isinuse
+_ubc_msync
 _ubc_offtoblk
 _ubc_page_op
 _ubc_range_op
@@ -602,6 +607,7 @@ _ubc_upl_abort_range
 _ubc_upl_commit
 _ubc_upl_commit_range
 _ubc_upl_map
+_ubc_upl_maxbufsize
 _ubc_upl_unmap
 _ubc_upl_pageinfo
 _upl_page_present
@@ -628,15 +634,19 @@ _uiomove
 _uiomove64
 _useracc
 _utf8_decodestr
+_utf8_encodelen
 _utf8_encodestr
+_utf8_normalizestr
 _utf8_validatestr
 _vcount
 _vflush
 _vfs_64bitready
 _vfs_addname
+_vfs_authcache_ttl
 _vfs_authopaque
 _vfs_authopaqueaccess
 _vfs_busy
+_vfs_clearauthcache_ttl
 _vfs_clearauthopaque
 _vfs_clearauthopaqueaccess
 _vfs_clearextendedsecurity
@@ -679,6 +689,7 @@ _vfs_opv_descs
 _vfs_opv_init
 _vfs_opv_numops
 _vfs_removename
+_vfs_setauthcache_ttl
 _vfs_setauthopaque
 _vfs_setauthopaqueaccess
 _vfs_setextendedsecurity
@@ -690,10 +701,11 @@ _vfs_statfs
 _vfs_sysctl
 _vfs_typenum
 _vfs_unbusy
-_vn_access
+_vfs_unmountbyfsid
 _vn_bwrite
 _vn_default_error
 _vn_getpath
+_vn_path_package_check
 _vn_rdwr
 _vn_revoke
 _vnode_addfsref
@@ -738,11 +750,13 @@ _vnode_rele
 _vnode_removefsref
 _vnode_setattr
 _vnode_setmountedon
+_vnode_setmultipath
 _vnode_setnocache
 _vnode_setnoreadahead
 _vnode_settag
 _vnode_specrdev
 _vnode_startwrite
+_vnode_uncache_credentials
 _vnode_vfs64bitready
 _vnode_vfsisrdonly
 _vnode_vfsmaxsymlen
@@ -766,7 +780,6 @@ _vnop_default_desc
 _vnop_exchange_desc
 _vnop_fsync_desc
 _vnop_getattr_desc
-_vnop_getattrlist_desc
 _vnop_getxattr_desc
 _vnop_inactive_desc
 _vnop_ioctl_desc
@@ -795,7 +808,6 @@ _vnop_rmdir_desc
 _vnop_searchfs_desc
 _vnop_select_desc
 _vnop_setattr_desc
-_vnop_setattrlist_desc
 _vnop_setxattr_desc
 _vnop_strategy_desc
 _vnop_symlink_desc
diff --git a/config/BSDKernel.i386.exports b/config/BSDKernel.i386.exports
index e69de29bb..4fb38fe48 100644
--- a/config/BSDKernel.i386.exports
+++ b/config/BSDKernel.i386.exports
@@ -0,0 +1,14 @@
+_in6_cksum:_inet6_cksum
+_mbuf_data
+_mbuf_inet6_cksum
+_mbuf_len
+_mbuf_next
+_mbuf_nextpkt
+_mbuf_pkthdr_header
+_mbuf_pkthdr_len
+_mbuf_pkthdr_rcvif
+_mbuf_pkthdr_setheader
+_mbuf_setlen
+_mbuf_setnextpkt
+_mbuf_type
+_nd6_lookup_ipv6
diff --git a/config/BSDKernel.ppc.exports b/config/BSDKernel.ppc.exports
index e69de29bb..4fb38fe48 100644
--- a/config/BSDKernel.ppc.exports
+++ b/config/BSDKernel.ppc.exports
@@ -0,0 +1,14 @@
+_in6_cksum:_inet6_cksum
+_mbuf_data
+_mbuf_inet6_cksum
+_mbuf_len
+_mbuf_next
+_mbuf_nextpkt
+_mbuf_pkthdr_header
+_mbuf_pkthdr_len
+_mbuf_pkthdr_rcvif
+_mbuf_pkthdr_setheader
+_mbuf_setlen
+_mbuf_setnextpkt
+_mbuf_type
+_nd6_lookup_ipv6
diff --git a/config/DtraceIgnored.symbols b/config/DtraceIgnored.symbols
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/config/DtraceIgnored.symbols
@@ -0,0 +1 @@
+
diff --git a/config/IOKit.exports b/config/IOKit.exports
index 6cfdf9312..fd0f8de59 100644
--- a/config/IOKit.exports
+++ b/config/IOKit.exports
@@ -58,6 +58,7 @@ _IONetworkNamePrefixMatching
 _IOOFPathMatching
 _IOPageableMapForAddress
 _IOPanic
+_IOPause
 _IOPrintPlane
 _IORWLockAlloc
 _IORWLockFree
@@ -66,6 +67,7 @@ _IORWLockRead:_lck_rw_lock_shared
 _IORWLockUnlock:_lck_rw_done
 _IORWLockWrite:_lck_rw_lock_exclusive
 _IORecursiveLockAlloc
+_IORecursiveLockAllocWithLockGroup
 _IORecursiveLockFree
 _IORecursiveLockGetMachLock
 _IORecursiveLockHaveLock
@@ -106,6 +108,7 @@ _PE_cpu_start
 _PE_enter_debugger
 _PE_halt_restart
 _PE_parse_boot_arg
+_PE_parse_boot_argn
 _PE_poll_input
 _StartIOKit
 __Z10tellClientP8OSObjectPv
@@ -167,6 +170,7 @@ __ZN10IOWorkLoop14addEventSourceEP13IOEventSource
 __ZN10IOWorkLoop15runEventSourcesEv
 __ZN10IOWorkLoop17removeEventSourceEP13IOEventSource
 __ZN10IOWorkLoop19signalWorkAvailableEv
+__ZN10IOWorkLoop19workLoopWithOptionsEm
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop2Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop3Ev
 __ZN10IOWorkLoop20_RESERVEDIOWorkLoop4Ev
@@ -263,6 +267,7 @@ __ZN11IOResourcesD0Ev
 __ZN11IOResourcesD2Ev
 __ZN12IODMACommand10gMetaClassE
 __ZN12IODMACommand10superClassE
+__ZN12IODMACommand10writeBytesEyPKvy
 __ZN12IODMACommand11OutputBig32EPS_NS_9Segment64EPvm
 __ZN12IODMACommand11OutputBig64EPS_NS_9Segment64EPvm
 __ZN12IODMACommand11synchronizeEm
@@ -276,8 +281,6 @@ __ZN12IODMACommand17withSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOpti
 __ZN12IODMACommand19setMemoryDescriptorEPK18IOMemoryDescriptorb
 __ZN12IODMACommand21clearMemoryDescriptorEb
 __ZN12IODMACommand21initWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperS2_
-__ZN12IODMACommand22_RESERVEDIODMACommand0Ev
-__ZN12IODMACommand22_RESERVEDIODMACommand1Ev
 __ZN12IODMACommand22_RESERVEDIODMACommand2Ev
 __ZN12IODMACommand22_RESERVEDIODMACommand3Ev
 __ZN12IODMACommand22_RESERVEDIODMACommand4Ev
@@ -292,12 +295,15 @@ __ZN12IODMACommand23_RESERVEDIODMACommand12Ev
 __ZN12IODMACommand23_RESERVEDIODMACommand13Ev
 __ZN12IODMACommand23_RESERVEDIODMACommand14Ev
 __ZN12IODMACommand23_RESERVEDIODMACommand15Ev
+__ZN12IODMACommand24prepareWithSpecificationEPFbPS_NS_9Segment64EPvmEhyNS_14MappingOptionsEymP8IOMapperyybb
 __ZN12IODMACommand4freeEv
 __ZN12IODMACommand7prepareEyybb
 __ZN12IODMACommand8completeEbb
+__ZN12IODMACommand8transferEmyPvy
 __ZN12IODMACommand9MetaClassC1Ev
 __ZN12IODMACommand9MetaClassC2Ev
 __ZN12IODMACommand9metaClassE
+__ZN12IODMACommand9readBytesEyPvy
 __ZN12IODMACommandC1EPK11OSMetaClass
 __ZN12IODMACommandC1Ev
 __ZN12IODMACommandC2EPK11OSMetaClass
@@ -344,14 +350,15 @@ __ZN12IOUserClient11clientCloseEv
 __ZN12IOUserClient12initWithTaskEP4taskPvm
 __ZN12IOUserClient12initWithTaskEP4taskPvmP12OSDictionary
 __ZN12IOUserClient13connectClientEPS_
+__ZN12IOUserClient14externalMethodEjP25IOExternalMethodArgumentsP24IOExternalMethodDispatchP8OSObjectPv
 __ZN12IOUserClient15mapClientMemoryEmP4taskmj
 __ZN12IOUserClient15sendAsyncResultEPjiPPvm
+__ZN12IOUserClient17sendAsyncResult64EPyiS0_m
 __ZN12IOUserClient17setAsyncReferenceEPjP8ipc_portPvS3_
 __ZN12IOUserClient18clientHasPrivilegeEPvPKc
 __ZN12IOUserClient19clientMemoryForTypeEmPmPP18IOMemoryDescriptor
 __ZN12IOUserClient20exportObjectToClientEP4taskP8OSObjectPS3_
 __ZN12IOUserClient21destroyUserReferencesEP8OSObject
-__ZN12IOUserClient22_RESERVEDIOUserClient0Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient1Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient2Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient3Ev
@@ -386,30 +393,6 @@ __ZN12IOUserClientC1EPK11OSMetaClass
 __ZN12IOUserClientC2EPK11OSMetaClass
 __ZN12IOUserClientD0Ev
 __ZN12IOUserClientD2Ev
-__ZN12_IOMemoryMap10gMetaClassE
-__ZN12_IOMemoryMap10superClassE
-__ZN12_IOMemoryMap13getMapOptionsEv
-__ZN12_IOMemoryMap14copyCompatibleEP18IOMemoryDescriptorP4taskjmmm
-__ZN12_IOMemoryMap14getAddressTaskEv
-__ZN12_IOMemoryMap14initCompatibleEP18IOMemoryDescriptorP11IOMemoryMapmm
-__ZN12_IOMemoryMap17getVirtualAddressEv
-__ZN12_IOMemoryMap18getPhysicalSegmentEmPm
-__ZN12_IOMemoryMap18initWithDescriptorEP18IOMemoryDescriptorP4taskjmmm
-__ZN12_IOMemoryMap19getMemoryDescriptorEv
-__ZN12_IOMemoryMap4freeEv
-__ZN12_IOMemoryMap5unmapEv
-__ZN12_IOMemoryMap8redirectEP4taskb
-__ZN12_IOMemoryMap8taskDiedEv
-__ZN12_IOMemoryMap9MetaClassC1Ev
-__ZN12_IOMemoryMap9MetaClassC2Ev
-__ZN12_IOMemoryMap9getLengthEv
-__ZN12_IOMemoryMap9metaClassE
-__ZN12_IOMemoryMapC1EPK11OSMetaClass
-__ZN12_IOMemoryMapC1Ev
-__ZN12_IOMemoryMapC2EPK11OSMetaClass
-__ZN12_IOMemoryMapC2Ev
-__ZN12_IOMemoryMapD0Ev
-__ZN12_IOMemoryMapD2Ev
 __ZN13IOCommandGate10gMetaClassE
 __ZN13IOCommandGate10runCommandEPvS0_S0_S0_
 __ZN13IOCommandGate10superClassE
@@ -616,6 +599,21 @@ __ZN15IOConditionLockC2EPK11OSMetaClass
 __ZN15IOConditionLockC2Ev
 __ZN15IOConditionLockD0Ev
 __ZN15IOConditionLockD2Ev
+__ZN15IODMAController10gMetaClassE
+__ZN15IODMAController10superClassE
+__ZN15IODMAController13getControllerEP9IOServicem
+__ZN15IODMAController16notifyDMACommandEP16IODMAEventSourceP12IODMACommandim
+__ZN15IODMAController18completeDMACommandEP16IODMAEventSourceP12IODMACommand
+__ZN15IODMAController20createControllerNameEm
+__ZN15IODMAController21registerDMAControllerEm
+__ZN15IODMAController5startEP9IOService
+__ZN15IODMAController9MetaClassC1Ev
+__ZN15IODMAController9MetaClassC2Ev
+__ZN15IODMAController9metaClassE
+__ZN15IODMAControllerC1EPK11OSMetaClass
+__ZN15IODMAControllerC2EPK11OSMetaClass
+__ZN15IODMAControllerD0Ev
+__ZN15IODMAControllerD2Ev
 __ZN15IOPMPowerSource10cycleCountEv
 __ZN15IOPMPowerSource10gMetaClassE
 __ZN15IOPMPowerSource10isChargingEv
@@ -629,8 +627,10 @@ __ZN15IOPMPowerSource11setAmperageEi
 __ZN15IOPMPowerSource11setLocationEi
 __ZN15IOPMPowerSource12manufacturerEv
 __ZN15IOPMPowerSource12updateStatusEv
+__ZN15IOPMPowerSource13getPSPropertyEPK8OSSymbol
 __ZN15IOPMPowerSource13setCycleCountEj
 __ZN15IOPMPowerSource13setIsChargingEb
+__ZN15IOPMPowerSource13setPSPropertyEPK8OSSymbolP8OSObject
 __ZN15IOPMPowerSource13timeRemainingEv
 __ZN15IOPMPowerSource14errorConditionEv
 __ZN15IOPMPowerSource14setAdapterInfoEi
@@ -650,8 +650,8 @@ __ZN15IOPMPowerSource19setBatteryInstalledEb
 __ZN15IOPMPowerSource20setExternalConnectedEb
 __ZN15IOPMPowerSource21externalChargeCapableEv
 __ZN15IOPMPowerSource22setLegacyIOBatteryInfoEP12OSDictionary
-__ZN15IOPMPowerSource24capacityPercentRemainingEv
 __ZN15IOPMPowerSource24setExternalChargeCapableEb
+__ZN15IOPMPowerSource24capacityPercentRemainingEv
 __ZN15IOPMPowerSource4freeEv
 __ZN15IOPMPowerSource4initEv
 __ZN15IOPMPowerSource5modelEv
@@ -768,6 +768,24 @@ __ZN15_IOConfigThreadC2EPK11OSMetaClass
 __ZN15_IOConfigThreadC2Ev
 __ZN15_IOConfigThreadD0Ev
 __ZN15_IOConfigThreadD2Ev
+__ZN16IODMAEventSource10gMetaClassE
+__ZN16IODMAEventSource10superClassE
+__ZN16IODMAEventSource12checkForWorkEv
+__ZN16IODMAEventSource14dmaEventSourceEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimES8_m
+__ZN16IODMAEventSource14stopDMACommandEbP13mach_timespec
+__ZN16IODMAEventSource15startDMACommandEP12IODMACommand11IODirectionmm
+__ZN16IODMAEventSource16notifyDMACommandEP12IODMACommandim
+__ZN16IODMAEventSource18completeDMACommandEP12IODMACommand
+__ZN16IODMAEventSource4initEP8OSObjectP9IOServicePFvS1_PS_P12IODMACommandimES8_m
+__ZN16IODMAEventSource9MetaClassC1Ev
+__ZN16IODMAEventSource9MetaClassC2Ev
+__ZN16IODMAEventSource9metaClassE
+__ZN16IODMAEventSourceC1EPK11OSMetaClass
+__ZN16IODMAEventSourceC1Ev
+__ZN16IODMAEventSourceC2EPK11OSMetaClass
+__ZN16IODMAEventSourceC2Ev
+__ZN16IODMAEventSourceD0Ev
+__ZN16IODMAEventSourceD2Ev
 __ZN16IOKitDiagnostics10gMetaClassE
 __ZN16IOKitDiagnostics10superClassE
 __ZN16IOKitDiagnostics11diagnosticsEv
@@ -882,6 +900,32 @@ __ZN17IOPowerConnectionC2EPK11OSMetaClass
 __ZN17IOPowerConnectionC2Ev
 __ZN17IOPowerConnectionD0Ev
 __ZN17IOPowerConnectionD2Ev
+__ZN17IOSharedDataQueue10gMetaClassE
+__ZN17IOSharedDataQueue10superClassE
+__ZN17IOSharedDataQueue16initWithCapacityEm
+__ZN17IOSharedDataQueue11withEntriesEmm
+__ZN17IOSharedDataQueue12withCapacityEm
+__ZN17IOSharedDataQueue19getMemoryDescriptorEv
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue0Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue1Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue2Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue3Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue4Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue5Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue6Ev
+__ZN17IOSharedDataQueue27_RESERVEDIOSharedDataQueue7Ev
+__ZN17IOSharedDataQueue4freeEv
+__ZN17IOSharedDataQueue4peekEv
+__ZN17IOSharedDataQueue7dequeueEPvPm
+__ZN17IOSharedDataQueue9MetaClassC1Ev
+__ZN17IOSharedDataQueue9MetaClassC2Ev
+__ZN17IOSharedDataQueue9metaClassE
+__ZN17IOSharedDataQueueC1EPK11OSMetaClass
+__ZN17IOSharedDataQueueC1Ev
+__ZN17IOSharedDataQueueC2EPK11OSMetaClass
+__ZN17IOSharedDataQueueC2Ev
+__ZN17IOSharedDataQueueD0Ev
+__ZN17IOSharedDataQueueD2Ev
 __ZN18IOMemoryDescriptor10addMappingEP11IOMemoryMap
 __ZN18IOMemoryDescriptor10gMetaClassE
 __ZN18IOMemoryDescriptor10initializeEv
@@ -889,7 +933,6 @@ __ZN18IOMemoryDescriptor10setMappingEP4taskjm
 __ZN18IOMemoryDescriptor10superClassE
 __ZN18IOMemoryDescriptor10withRangesEP14IOVirtualRangem11IODirectionP4taskb
 __ZN18IOMemoryDescriptor10writeBytesEmPKvm
-__ZN18IOMemoryDescriptor11handleFaultEPvP6vm_mapjmmm
 __ZN18IOMemoryDescriptor11makeMappingEPS_P4taskjmmm
 __ZN18IOMemoryDescriptor11withAddressEPvm11IODirection
 __ZN18IOMemoryDescriptor11withAddressEjm11IODirectionP4task
@@ -900,7 +943,6 @@ __ZN18IOMemoryDescriptor13removeMappingEP11IOMemoryMap
 __ZN18IOMemoryDescriptor15initWithOptionsEPvmmP4taskmP8IOMapper
 __ZN18IOMemoryDescriptor16getSourceSegmentEmPm
 __ZN18IOMemoryDescriptor16performOperationEmmm
-__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task
 __ZN18IOMemoryDescriptor17withAddressRangesEP14IOAddressRangemmP4task
 __ZN18IOMemoryDescriptor18getPhysicalAddressEv
 __ZN18IOMemoryDescriptor18withPhysicalRangesEP15IOPhysicalRangem11IODirectionb
@@ -920,10 +962,10 @@ __ZN18IOMemoryDescriptor30withPersistentMemoryDescriptorEPS_
 __ZN18IOMemoryDescriptor3mapEP4taskjmmm
 __ZN18IOMemoryDescriptor3mapEm
 __ZN18IOMemoryDescriptor4freeEv
-__ZN18IOMemoryDescriptor5doMapEP6vm_mapPjmmm
+__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPjmmm
 __ZN18IOMemoryDescriptor6getTagEv
 __ZN18IOMemoryDescriptor6setTagEm
-__ZN18IOMemoryDescriptor7doUnmapEP6vm_mapjm
+__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapjm
 __ZN18IOMemoryDescriptor8redirectEP4taskb
 __ZN18IOMemoryDescriptor9MetaClassC1Ev
 __ZN18IOMemoryDescriptor9MetaClassC2Ev
@@ -933,29 +975,6 @@ __ZN18IOMemoryDescriptorC1EPK11OSMetaClass
 __ZN18IOMemoryDescriptorC2EPK11OSMetaClass
 __ZN18IOMemoryDescriptorD0Ev
 __ZN18IOMemoryDescriptorD2Ev
-__ZN18IOPMchangeNoteList10gMetaClassE
-__ZN18IOPMchangeNoteList10initializeEv
-__ZN18IOPMchangeNoteList10superClassE
-__ZN18IOPMchangeNoteList12latestChangeEv
-__ZN18IOPMchangeNoteList13currentChangeEv
-__ZN18IOPMchangeNoteList14nextChangeNoteEm
-__ZN18IOPMchangeNoteList15changeNoteInUseEm
-__ZN18IOPMchangeNoteList16createChangeNoteEv
-__ZN18IOPMchangeNoteList18previousChangeNoteEm
-__ZN18IOPMchangeNoteList21releaseHeadChangeNoteEv
-__ZN18IOPMchangeNoteList21releaseTailChangeNoteEv
-__ZN18IOPMchangeNoteList9MetaClassC1Ev
-__ZN18IOPMchangeNoteList9MetaClassC2Ev
-__ZN18IOPMchangeNoteList9decrementEm
-__ZN18IOPMchangeNoteList9incrementEm
-__ZN18IOPMchangeNoteList9listEmptyEv
-__ZN18IOPMchangeNoteList9metaClassE
-__ZN18IOPMchangeNoteListC1EPK11OSMetaClass
-__ZN18IOPMchangeNoteListC1Ev
-__ZN18IOPMchangeNoteListC2EPK11OSMetaClass
-__ZN18IOPMchangeNoteListC2Ev
-__ZN18IOPMchangeNoteListD0Ev
-__ZN18IOPMchangeNoteListD2Ev
 __ZN18IORegistryIterator10enterEntryEPK15IORegistryPlane
 __ZN18IORegistryIterator10enterEntryEv
 __ZN18IORegistryIterator10gMetaClassE
@@ -1173,7 +1192,7 @@ __ZN21IOSubMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN21IOSubMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN21IOSubMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN21IOSubMemoryDescriptor4freeEv
-__ZN21IOSubMemoryDescriptor5doMapEP6vm_mapPjmmm
+__ZN21IOSubMemoryDescriptor5doMapEP7_vm_mapPjmmm
 __ZN21IOSubMemoryDescriptor7prepareE11IODirection
 __ZN21IOSubMemoryDescriptor8completeE11IODirection
 __ZN21IOSubMemoryDescriptor8redirectEP4taskb
@@ -1272,8 +1291,6 @@ __ZN24IOBufferMemoryDescriptor15initWithAddressEjm11IODirectionP4task
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task
 __ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj
-__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
-__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
 __ZN24IOBufferMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN24IOBufferMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor2Ev
@@ -1319,8 +1336,8 @@ __ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN25IOGeneralMemoryDescriptor4freeEv
-__ZN25IOGeneralMemoryDescriptor5doMapEP6vm_mapPjmmm
-__ZN25IOGeneralMemoryDescriptor7doUnmapEP6vm_mapjm
+__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPjmmm
+__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapjm
 __ZN25IOGeneralMemoryDescriptor7prepareE11IODirection
 __ZN25IOGeneralMemoryDescriptor8completeE11IODirection
 __ZN25IOGeneralMemoryDescriptor9MetaClassC1Ev
@@ -1332,22 +1349,6 @@ __ZN25IOGeneralMemoryDescriptorC2EPK11OSMetaClass
 __ZN25IOGeneralMemoryDescriptorC2Ev
 __ZN25IOGeneralMemoryDescriptorD0Ev
 __ZN25IOGeneralMemoryDescriptorD2Ev
-__ZN25IOServiceUserNotification10gMetaClassE
-__ZN25IOServiceUserNotification10superClassE
-__ZN25IOServiceUserNotification13getNextObjectEv
-__ZN25IOServiceUserNotification4freeEv
-__ZN25IOServiceUserNotification4initEP8ipc_portjPj
-__ZN25IOServiceUserNotification7handlerEPvP9IOService
-__ZN25IOServiceUserNotification8_handlerEPvS0_P9IOService
-__ZN25IOServiceUserNotification9MetaClassC1Ev
-__ZN25IOServiceUserNotification9MetaClassC2Ev
-__ZN25IOServiceUserNotification9metaClassE
-__ZN25IOServiceUserNotificationC1EPK11OSMetaClass
-__ZN25IOServiceUserNotificationC1Ev
-__ZN25IOServiceUserNotificationC2EPK11OSMetaClass
-__ZN25IOServiceUserNotificationC2Ev
-__ZN25IOServiceUserNotificationD0Ev
-__ZN25IOServiceUserNotificationD2Ev
 __ZN26_IOServiceInterestNotifier10gMetaClassE
 __ZN26_IOServiceInterestNotifier10superClassE
 __ZN26_IOServiceInterestNotifier4freeEv
@@ -1413,22 +1414,33 @@ __ZN28IOFilterInterruptEventSourceC2EPK11OSMetaClass
 __ZN28IOFilterInterruptEventSourceC2Ev
 __ZN28IOFilterInterruptEventSourceD0Ev
 __ZN28IOFilterInterruptEventSourceD2Ev
-__ZN32IOServiceMessageUserNotification10gMetaClassE
-__ZN32IOServiceMessageUserNotification10superClassE
-__ZN32IOServiceMessageUserNotification13getNextObjectEv
-__ZN32IOServiceMessageUserNotification4freeEv
-__ZN32IOServiceMessageUserNotification4initEP8ipc_portjPjj
-__ZN32IOServiceMessageUserNotification7handlerEPvmP9IOServiceS0_j
-__ZN32IOServiceMessageUserNotification8_handlerEPvS0_mP9IOServiceS0_j
-__ZN32IOServiceMessageUserNotification9MetaClassC1Ev
-__ZN32IOServiceMessageUserNotification9MetaClassC2Ev
-__ZN32IOServiceMessageUserNotification9metaClassE
-__ZN32IOServiceMessageUserNotificationC1EPK11OSMetaClass
-__ZN32IOServiceMessageUserNotificationC1Ev
-__ZN32IOServiceMessageUserNotificationC2EPK11OSMetaClass
-__ZN32IOServiceMessageUserNotificationC2Ev
-__ZN32IOServiceMessageUserNotificationD0Ev
-__ZN32IOServiceMessageUserNotificationD2Ev
+__ZN29IOInterleavedMemoryDescriptor10gMetaClassE
+__ZN29IOInterleavedMemoryDescriptor10superClassE
+__ZN29IOInterleavedMemoryDescriptor12withCapacityEm11IODirection
+__ZN29IOInterleavedMemoryDescriptor14initWithRangesEP14IOVirtualRangem11IODirectionP4taskb
+__ZN29IOInterleavedMemoryDescriptor15initWithAddressEPvm11IODirection
+__ZN29IOInterleavedMemoryDescriptor15initWithAddressEjm11IODirectionP4task
+__ZN29IOInterleavedMemoryDescriptor16getSourceSegmentEmPm
+__ZN29IOInterleavedMemoryDescriptor16initWithCapacityEm11IODirection
+__ZN29IOInterleavedMemoryDescriptor17getVirtualSegmentEmPm
+__ZN29IOInterleavedMemoryDescriptor18getPhysicalSegmentEmPm
+__ZN29IOInterleavedMemoryDescriptor19setMemoryDescriptorEP18IOMemoryDescriptormm
+__ZN29IOInterleavedMemoryDescriptor20getPhysicalSegment64EmPm
+__ZN29IOInterleavedMemoryDescriptor22clearMemoryDescriptorsE11IODirection
+__ZN29IOInterleavedMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
+__ZN29IOInterleavedMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
+__ZN29IOInterleavedMemoryDescriptor4freeEv
+__ZN29IOInterleavedMemoryDescriptor7prepareE11IODirection
+__ZN29IOInterleavedMemoryDescriptor8completeE11IODirection
+__ZN29IOInterleavedMemoryDescriptor9MetaClassC1Ev
+__ZN29IOInterleavedMemoryDescriptor9MetaClassC2Ev
+__ZN29IOInterleavedMemoryDescriptor9metaClassE
+__ZN29IOInterleavedMemoryDescriptorC1EPK11OSMetaClass
+__ZN29IOInterleavedMemoryDescriptorC1Ev
+__ZN29IOInterleavedMemoryDescriptorC2EPK11OSMetaClass
+__ZN29IOInterleavedMemoryDescriptorC2Ev
+__ZN29IOInterleavedMemoryDescriptorD0Ev
+__ZN29IOInterleavedMemoryDescriptorD2Ev
 __ZN8IOMapper10allocTableEm
 __ZN8IOMapper10gMetaClassE
 __ZN8IOMapper10iovmInsertEjmP13upl_page_infom
@@ -1463,17 +1475,6 @@ __ZN8IOMapperC1EPK11OSMetaClass
 __ZN8IOMapperC2EPK11OSMetaClass
 __ZN8IOMapperD0Ev
 __ZN8IOMapperD2Ev
-__ZN8IOPMpriv10gMetaClassE
-__ZN8IOPMpriv10superClassE
-__ZN8IOPMpriv9MetaClassC1Ev
-__ZN8IOPMpriv9MetaClassC2Ev
-__ZN8IOPMpriv9metaClassE
-__ZN8IOPMprivC1EPK11OSMetaClass
-__ZN8IOPMprivC1Ev
-__ZN8IOPMprivC2EPK11OSMetaClass
-__ZN8IOPMprivC2Ev
-__ZN8IOPMprivD0Ev
-__ZN8IOPMprivD2Ev
 __ZN8IOPMprot10gMetaClassE
 __ZN8IOPMprot10superClassE
 __ZN8IOPMprot9MetaClassC1Ev
@@ -1515,7 +1516,6 @@ __ZN9IOCommandD0Ev
 __ZN9IOCommandD2Ev
 __ZN9IOService10actionStopEPS_S0_
 __ZN9IOService10adjustBusyEl
-__ZN9IOService10ask_parentEm
 __ZN9IOService10gMetaClassE
 __ZN9IOService10handleOpenEPS_mPv
 __ZN9IOService10initializeEv
@@ -1526,17 +1526,12 @@ __ZN9IOService10systemWakeEv
 __ZN9IOService10youAreRootEv
 __ZN9IOService11_adjustBusyEl
 __ZN9IOService11addLocationEP12OSDictionary
-__ZN9IOService11changeStateEv
 __ZN9IOService11getPlatformEv
 __ZN9IOService11handleCloseEPS_m
-__ZN9IOService11notifyChildEP17IOPowerConnectionb
 __ZN9IOService11setPlatformEP16IOPlatformExpert
 __ZN9IOService11tellClientsEi
-__ZN9IOService12acquire_lockEv
-__ZN9IOService12checkForDoneEv
 __ZN9IOService12clampPowerOnEm
 __ZN9IOService12didTerminateEPS_mPb
-__ZN9IOService12driver_ackedEv
 __ZN9IOService12getBusyStateEv
 __ZN9IOService12getResourcesEv
 __ZN9IOService12nameMatchingEPK8OSStringP12OSDictionary
@@ -1555,8 +1550,6 @@ __ZN9IOService13matchLocationEPS_
 __ZN9IOService13messageClientEmP8OSObjectPvj
 __ZN9IOService13newUserClientEP4taskPvmP12OSDictionaryPP12IOUserClient
 __ZN9IOService13newUserClientEP4taskPvmPP12IOUserClient
-__ZN9IOService13responseValidEm
-__ZN9IOService13setParentInfoEmP17IOPowerConnection
 __ZN9IOService13setPowerStateEmPS_
 __ZN9IOService13startMatchingEm
 __ZN9IOService13waitMatchIdleEm
@@ -1572,10 +1565,8 @@ __ZN9IOService14messageClientsEmPvj
 __ZN9IOService14newTemperatureElPS_
 __ZN9IOService14setPowerParentEP17IOPowerConnectionbm
 __ZN9IOService14startCandidateEPS_
-__ZN9IOService14stop_ack_timerEv
 __ZN9IOService14tellChangeDownEm
 __ZN9IOService14waitForServiceEP12OSDictionaryP13mach_timespec
-__ZN9IOService15OurChangeFinishEv
 __ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l
 __ZN9IOService15comparePropertyEP12OSDictionaryPK8OSString
 __ZN9IOService15comparePropertyEP12OSDictionaryPKc
@@ -1583,11 +1574,11 @@ __ZN9IOService15enableInterruptEi
 __ZN9IOService15errnoFromReturnEi
 __ZN9IOService15getDeviceMemoryEv
 __ZN9IOService15getPMRootDomainEv
-__ZN9IOService15instruct_driverEm
 __ZN9IOService15lookupInterruptEibPP21IOInterruptController
 __ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService15powerChangeDoneEm
 __ZN9IOService15probeCandidatesEP12OSOrderedSet
+__ZN9IOService16propertyMatchingEPK8OSSymbolPK8OSObjectP12OSDictionary
 __ZN9IOService15publishResourceEPK8OSSymbolP8OSObject
 __ZN9IOService15publishResourceEPKcP8OSObject
 __ZN9IOService15registerServiceEm
@@ -1596,7 +1587,6 @@ __ZN9IOService15serviceMatchingEPKcP12OSDictionary
 __ZN9IOService15setDeviceMemoryEP7OSArray
 __ZN9IOService15setNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l
 __ZN9IOService15setPMRootDomainEP14IOPMrootDomain
-__ZN9IOService15start_ack_timerEv
 __ZN9IOService15tellChangeDown1Em
 __ZN9IOService15tellChangeDown2Em
 __ZN9IOService15terminateClientEPS_m
@@ -1619,24 +1609,21 @@ __ZN9IOService16resourceMatchingEPK8OSStringP12OSDictionary
 __ZN9IOService16resourceMatchingEPKcP12OSDictionary
 __ZN9IOService16scheduleFinalizeEv
 __ZN9IOService16setCPUSnoopDelayEm
-__ZN9IOService16startSettleTimerEm
-__ZN9IOService16start_our_changeEm
 __ZN9IOService16stringFromReturnEi
 __ZN9IOService16tellNoChangeDownEm
 __ZN9IOService17addNeededResourceEPKc
-__ZN9IOService17allowCancelCommonEv
 __ZN9IOService17applyToInterestedEPK8OSSymbolPFvP8OSObjectPvES5_
 __ZN9IOService17cancelPowerChangeEm
 __ZN9IOService17catalogNewDriversEP12OSOrderedSet
 __ZN9IOService17comparePropertiesEP12OSDictionaryP12OSCollection
 __ZN9IOService17currentCapabilityEv
 __ZN9IOService17getAggressivenessEmPm
+__ZN9IOService13getPowerStateEv
 __ZN9IOService17registerInterruptEiP8OSObjectPFvS1_PvPS_iES2_
 __ZN9IOService17setAggressivenessEmm
 __ZN9IOService18actionDidTerminateEPS_m
 __ZN9IOService18changePowerStateToEm
 __ZN9IOService18doServiceTerminateEm
-__ZN9IOService18enqueuePowerChangeEmmmP17IOPowerConnectionm
 __ZN9IOService18getResourceServiceEv
 __ZN9IOService18lockForArbitrationEb
 __ZN9IOService18matchPropertyTableEP12OSDictionary
@@ -1644,15 +1631,13 @@ __ZN9IOService18matchPropertyTableEP12OSDictionaryPl
 __ZN9IOService18requireMaxBusStallEm
 __ZN9IOService18setIdleTimerPeriodEm
 __ZN9IOService18settleTimerExpiredEv
-__ZN9IOService19_RESERVEDIOService4Ev
+__ZN9IOService18systemWillShutdownEm
 __ZN9IOService19_RESERVEDIOService5Ev
 __ZN9IOService19_RESERVEDIOService6Ev
 __ZN9IOService19_RESERVEDIOService7Ev
 __ZN9IOService19_RESERVEDIOService8Ev
 __ZN9IOService19_RESERVEDIOService9Ev
 __ZN9IOService19actionWillTerminateEPS_mP7OSArray
-__ZN9IOService19computeDesiredStateEv
-__ZN9IOService19compute_settle_timeEv
 __ZN9IOService19deliverNotificationEPK8OSSymbolmm
 __ZN9IOService19getExistingServicesEP12OSDictionarymm
 __ZN9IOService19getMatchingServicesEP12OSDictionary
@@ -1660,7 +1645,6 @@ __ZN9IOService19installNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_
 __ZN9IOService19powerOverrideOnPrivEv
 __ZN9IOService19registerPowerDriverEPS_P14IOPMPowerStatem
 __ZN9IOService19start_PM_idle_timerEv
-__ZN9IOService19start_parent_changeEm
 __ZN9IOService19unregisterInterruptEi
 __ZN9IOService20_RESERVEDIOService10Ev
 __ZN9IOService20_RESERVEDIOService11Ev
@@ -1708,10 +1692,8 @@ __ZN9IOService20unlockForArbitrationEv
 __ZN9IOService21doInstallNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_lPP10OSIterator
 __ZN9IOService21getClientWithCategoryEPK8OSSymbol
 __ZN9IOService21powerStateDidChangeToEmmPS_
-__ZN9IOService21rebuildChildClampBitsEv
 __ZN9IOService21temporaryPowerClampOnEv
 __ZN9IOService21unregisterAllInterestEv
-__ZN9IOService22OurChangeSetPowerStateEv
 __ZN9IOService22PM_Clamp_Timer_ExpiredEv
 __ZN9IOService22acknowledgePowerChangeEPS_
 __ZN9IOService22changePowerStateToPrivEm
@@ -1730,39 +1712,15 @@ __ZN9IOService24getDeviceMemoryWithIndexEj
 __ZN9IOService24mapDeviceMemoryWithIndexEjm
 __ZN9IOService24powerStateForDomainStateEm
 __ZN9IOService24registerInterestedDriverEPS_
-__ZN9IOService26add_child_to_active_changeEP17IOPowerConnection
 __ZN9IOService26deRegisterInterestedDriverEPS_
 __ZN9IOService26temperatureCriticalForZoneEPS_
-__ZN9IOService27OurChangeWaitForPowerSettleEv
-__ZN9IOService27add_driver_to_active_changeEP12IOPMinformee
 __ZN9IOService27maxCapabilityForDomainStateEm
 __ZN9IOService27serializedAllowPowerChange2Em
 __ZN9IOService28serializedCancelPowerChange2Em
-__ZN9IOService29OurChangeTellClientsPowerDownEv
-__ZN9IOService29ParentUpSetPowerState_DelayedEv
-__ZN9IOService31ParentDownSetPowerState_DelayedEv
-__ZN9IOService31ParentUpSetPowerState_ImmediateEv
 __ZN9IOService31initialPowerStateForDomainStateEm
-__ZN9IOService33ParentDownSetPowerState_ImmediateEv
-__ZN9IOService33ParentUpWaitForSettleTime_DelayedEv
-__ZN9IOService35ParentDownAcknowledgeChange_DelayedEv
-__ZN9IOService35ParentUpWaitForSettleTime_ImmediateEv
-__ZN9IOService36ParentDownWaitForPowerSettle_DelayedEv
-__ZN9IOService37OurChangeTellPriorityClientsPowerDownEv
-__ZN9IOService38ParentUpAcknowledgePowerChange_DelayedEv
-__ZN9IOService41OurChangeNotifyInterestedDriversDidChangeEv
-__ZN9IOService42OurChangeNotifyInterestedDriversWillChangeEv
-__ZN9IOService46ParentDownTellPriorityClientsPowerDown_DelayedEv
-__ZN9IOService48ParentDownTellPriorityClientsPowerDown_ImmediateEv
-__ZN9IOService48ParentUpNotifyInterestedDriversDidChange_DelayedEv
 __ZN9IOService4freeEv
 __ZN9IOService4openEPS_mPv
 __ZN9IOService4stopEPS_
-__ZN9IOService50ParentUpNotifyInterestedDriversDidChange_ImmediateEv
-__ZN9IOService51ParentDownNotifyInterestedDriversWillChange_DelayedEv
-__ZN9IOService53ParentDownNotifyDidChangeAndAcknowledgeChange_DelayedEv
-__ZN9IOService53ParentDownNotifyInterestedDriversWillChange_ImmediateEv
-__ZN9IOService56ParentDownWaitForPowerSettleAndNotifyDidChange_ImmediateEv
 __ZN9IOService5closeEPS_m
 __ZN9IOService5probeEPS_Pl
 __ZN9IOService5startEPS_
@@ -1771,15 +1729,11 @@ __ZN9IOService6PMinitEv
 __ZN9IOService6PMstopEv
 __ZN9IOService6attachEPS_
 __ZN9IOService6detachEPS_
-__ZN9IOService6informEP12IOPMinformeeb
 __ZN9IOService7messageEmPS_Pv
-__ZN9IOService8all_doneEv
 __ZN9IOService8finalizeEm
 __ZN9IOService9MetaClassC1Ev
 __ZN9IOService9MetaClassC2Ev
-__ZN9IOService9all_ackedEv
 __ZN9IOService9metaClassE
-__ZN9IOService9notifyAllEb
 __ZN9IOService9resourcesEv
 __ZN9IOService9terminateEm
 __ZN9IOService9waitQuietEP13mach_timespec
@@ -1827,9 +1781,6 @@ __ZNK12IORootParent12getMetaClassEv
 __ZNK12IORootParent9MetaClass5allocEv
 __ZNK12IOUserClient12getMetaClassEv
 __ZNK12IOUserClient9MetaClass5allocEv
-__ZNK12_IOMemoryMap12getMetaClassEv
-__ZNK12_IOMemoryMap13taggedReleaseEPKv
-__ZNK12_IOMemoryMap9MetaClass5allocEv
 __ZNK13IOCommandGate12getMetaClassEv
 __ZNK13IOCommandGate9MetaClass5allocEv
 __ZNK13IOCommandPool12getMetaClassEv
@@ -1853,6 +1804,8 @@ __ZNK15IOConditionLock12getConditionEv
 __ZNK15IOConditionLock12getMetaClassEv
 __ZNK15IOConditionLock16getInterruptibleEv
 __ZNK15IOConditionLock9MetaClass5allocEv
+__ZNK15IODMAController12getMetaClassEv
+__ZNK15IODMAController9MetaClass5allocEv
 __ZNK15IOPMPowerSource12getMetaClassEv
 __ZNK15IOPMPowerSource9MetaClass5allocEv
 __ZNK15IORegistryEntry11compareNameEP8OSStringPS1_
@@ -1902,6 +1855,8 @@ __ZNK15IORegistryPlane9MetaClass5allocEv
 __ZNK15IORegistryPlane9serializeEP11OSSerialize
 __ZNK15_IOConfigThread12getMetaClassEv
 __ZNK15_IOConfigThread9MetaClass5allocEv
+__ZNK16IODMAEventSource12getMetaClassEv
+__ZNK16IODMAEventSource9MetaClass5allocEv
 __ZNK16IOKitDiagnostics12getMetaClassEv
 __ZNK16IOKitDiagnostics9MetaClass5allocEv
 __ZNK16IOKitDiagnostics9serializeEP11OSSerialize
@@ -1914,13 +1869,13 @@ __ZNK17IOBigMemoryCursor12getMetaClassEv
 __ZNK17IOBigMemoryCursor9MetaClass5allocEv
 __ZNK17IOPowerConnection12getMetaClassEv
 __ZNK17IOPowerConnection9MetaClass5allocEv
+__ZNK17IOSharedDataQueue12getMetaClassEv
+__ZNK17IOSharedDataQueue9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor12getDirectionEv
 __ZNK18IOMemoryDescriptor12getMetaClassEv
 __ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK18IOMemoryDescriptor9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor9getLengthEv
-__ZNK18IOPMchangeNoteList12getMetaClassEv
-__ZNK18IOPMchangeNoteList9MetaClass5allocEv
 __ZNK18IORegistryIterator12getMetaClassEv
 __ZNK18IORegistryIterator9MetaClass5allocEv
 __ZNK18IOTimerEventSource12getMetaClassEv
@@ -1961,8 +1916,6 @@ __ZNK25IOGeneralMemoryDescriptor12getMetaClassEv
 __ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK25IOGeneralMemoryDescriptor9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor9serializeEP11OSSerialize
-__ZNK25IOServiceUserNotification12getMetaClassEv
-__ZNK25IOServiceUserNotification9MetaClass5allocEv
 __ZNK26_IOServiceInterestNotifier12getMetaClassEv
 __ZNK26_IOServiceInterestNotifier9MetaClass5allocEv
 __ZNK27IOSharedInterruptController12getMetaClassEv
@@ -1970,17 +1923,13 @@ __ZNK27IOSharedInterruptController9MetaClass5allocEv
 __ZNK28IOFilterInterruptEventSource12getMetaClassEv
 __ZNK28IOFilterInterruptEventSource15getFilterActionEv
 __ZNK28IOFilterInterruptEventSource9MetaClass5allocEv
-__ZNK32IOServiceMessageUserNotification12getMetaClassEv
-__ZNK32IOServiceMessageUserNotification9MetaClass5allocEv
+__ZNK29IOInterleavedMemoryDescriptor12getMetaClassEv
+__ZNK29IOInterleavedMemoryDescriptor9MetaClass5allocEv
 __ZNK8IOMapper12getMetaClassEv
 __ZNK8IOMapper13getBypassMaskEPy
 __ZNK8IOMapper9MetaClass5allocEv
-__ZNK8IOPMpriv12getMetaClassEv
-__ZNK8IOPMpriv9MetaClass5allocEv
-__ZNK8IOPMpriv9serializeEP11OSSerialize
 __ZNK8IOPMprot12getMetaClassEv
 __ZNK8IOPMprot9MetaClass5allocEv
-__ZNK8IOPMprot9serializeEP11OSSerialize
 __ZNK8IOSyncer12getMetaClassEv
 __ZNK8IOSyncer9MetaClass5allocEv
 __ZNK9IOCommand12getMetaClassEv
@@ -2010,7 +1959,6 @@ __ZTV12IODMACommand
 __ZTV12IOPMinformee
 __ZTV12IORootParent
 __ZTV12IOUserClient
-__ZTV12_IOMemoryMap
 __ZTV13IOCommandGate
 __ZTV13IOCommandPool
 __ZTV13IOEventSource
@@ -2019,18 +1967,20 @@ __ZTV14IOCommandQueue
 __ZTV14IOMemoryCursor
 __ZTV14IOPMrootDomain
 __ZTV15IOConditionLock
+__ZTV15IODMAController
 __ZTV15IOPMPowerSource
 __ZTV15IORegistryEntry
 __ZTV15IORegistryPlane
 __ZTV15_IOConfigThread
+__ZTV16IODMAEventSource
 __ZTV16IOKitDiagnostics
 __ZTV16IOPMinformeeList
 __ZTV16IORangeAllocator
 __ZTV17IOBigMemoryCursor
 __ZTV17IOPolledInterface
 __ZTV17IOPowerConnection
+__ZTV17IOSharedDataQueue
 __ZTV18IOMemoryDescriptor
-__ZTV18IOPMchangeNoteList
 __ZTV18IORegistryIterator
 __ZTV18IOTimerEventSource
 __ZTV18IOUserNotification
@@ -2047,13 +1997,11 @@ __ZTV22_IOOpenServiceIterator
 __ZTV23IOMultiMemoryDescriptor
 __ZTV24IOBufferMemoryDescriptor
 __ZTV25IOGeneralMemoryDescriptor
-__ZTV25IOServiceUserNotification
 __ZTV26_IOServiceInterestNotifier
 __ZTV27IOSharedInterruptController
 __ZTV28IOFilterInterruptEventSource
-__ZTV32IOServiceMessageUserNotification
+__ZTV29IOInterleavedMemoryDescriptor
 __ZTV8IOMapper
-__ZTV8IOPMpriv
 __ZTV8IOPMprot
 __ZTV8IOSyncer
 __ZTV9IOCommand
@@ -2069,7 +2017,6 @@ __ZTVN12IODMACommand9MetaClassE
 __ZTVN12IOPMinformee9MetaClassE
 __ZTVN12IORootParent9MetaClassE
 __ZTVN12IOUserClient9MetaClassE
-__ZTVN12_IOMemoryMap9MetaClassE
 __ZTVN13IOCommandGate9MetaClassE
 __ZTVN13IOCommandPool9MetaClassE
 __ZTVN13IOEventSource9MetaClassE
@@ -2078,17 +2025,19 @@ __ZTVN14IOCommandQueue9MetaClassE
 __ZTVN14IOMemoryCursor9MetaClassE
 __ZTVN14IOPMrootDomain9MetaClassE
 __ZTVN15IOConditionLock9MetaClassE
+__ZTVN15IODMAController9MetaClassE
 __ZTVN15IOPMPowerSource9MetaClassE
 __ZTVN15IORegistryEntry9MetaClassE
 __ZTVN15IORegistryPlane9MetaClassE
 __ZTVN15_IOConfigThread9MetaClassE
+__ZTVN16IODMAEventSource9MetaClassE
 __ZTVN16IOKitDiagnostics9MetaClassE
 __ZTVN16IOPMinformeeList9MetaClassE
 __ZTVN16IORangeAllocator9MetaClassE
 __ZTVN17IOBigMemoryCursor9MetaClassE
 __ZTVN17IOPowerConnection9MetaClassE
+__ZTVN17IOSharedDataQueue9MetaClassE
 __ZTVN18IOMemoryDescriptor9MetaClassE
-__ZTVN18IOPMchangeNoteList9MetaClassE
 __ZTVN18IORegistryIterator9MetaClassE
 __ZTVN18IOTimerEventSource9MetaClassE
 __ZTVN18IOUserNotification9MetaClassE
@@ -2105,13 +2054,11 @@ __ZTVN22_IOOpenServiceIterator9MetaClassE
 __ZTVN23IOMultiMemoryDescriptor9MetaClassE
 __ZTVN24IOBufferMemoryDescriptor9MetaClassE
 __ZTVN25IOGeneralMemoryDescriptor9MetaClassE
-__ZTVN25IOServiceUserNotification9MetaClassE
 __ZTVN26_IOServiceInterestNotifier9MetaClassE
 __ZTVN27IOSharedInterruptController9MetaClassE
 __ZTVN28IOFilterInterruptEventSource9MetaClassE
-__ZTVN32IOServiceMessageUserNotification9MetaClassE
+__ZTVN29IOInterleavedMemoryDescriptor9MetaClassE
 __ZTVN8IOMapper9MetaClassE
-__ZTVN8IOPMpriv9MetaClassE
 __ZTVN8IOPMprot9MetaClassE
 __ZTVN8IOSyncer9MetaClassE
 __ZTVN9IOCommand9MetaClassE
@@ -2134,6 +2081,7 @@ _ev_try_lock
 _ev_unlock
 _gIOAppPowerStateInterest
 _gIOBusyInterest
+_gIOCatalogue
 _gIOClassKey
 _gIOCommandPoolSizeKey
 _gIODTAAPLInterruptsKey
@@ -2175,6 +2123,10 @@ _gIONameMatchKey
 _gIONameMatchedKey
 _gIOParentMatchKey
 _gIOPathMatchKey
+_gIOPlatformActiveActionKey
+_gIOPlatformQuiesceActionKey
+_gIOPlatformSleepActionKey
+_gIOPlatformWakeActionKey
 _gIOPowerPlane
 _gIOPrelinkedModules
 _gIOPriorityPowerStateInterest
diff --git a/config/IOKit.i386.exports b/config/IOKit.i386.exports
index e69de29bb..a483c4173 100644
--- a/config/IOKit.i386.exports
+++ b/config/IOKit.i386.exports
@@ -0,0 +1,6 @@
+__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
+__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task
+__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskymyy
+__ZN12IOUserClient17mapClientMemory64EmP4taskmy
+__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy
diff --git a/config/IOKit.ppc.exports b/config/IOKit.ppc.exports
index 42d90d845..b08a79c40 100644
--- a/config/IOKit.ppc.exports
+++ b/config/IOKit.ppc.exports
@@ -1,3 +1,5 @@
+__ZN10AppleMacIO9metaClassE
+__ZN16AppleMacIODevice9metaClassE
 __Z11IODBDMAStopPV23IODBDMAChannelRegisters
 __Z12IODBDMAFlushPV23IODBDMAChannelRegisters
 __Z12IODBDMAPausePV23IODBDMAChannelRegisters
@@ -37,3 +39,9 @@ __ZNK19IODBDMAMemoryCursor12getMetaClassEv
 __ZNK19IODBDMAMemoryCursor9MetaClass5allocEv
 __ZTV19IODBDMAMemoryCursor
 __ZTVN19IODBDMAMemoryCursor9MetaClassE
+__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
+__ZN18IOMemoryDescriptor16withAddressRangeEyymP4task
+__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskymyy
+__ZN12IOUserClient17mapClientMemory64EmP4taskmy
+__ZN12IOUserClient19setAsyncReference64EPyP8ipc_portyy
diff --git a/config/Libkern.exports b/config/Libkern.exports
index 261f08b79..cb7928a5a 100644
--- a/config/Libkern.exports
+++ b/config/Libkern.exports
@@ -1,4 +1,7 @@
 _Assert
+_MD5Final
+_MD5Init
+_MD5Update
 _OSAddAtomic
 _OSAddAtomic16
 _OSAddAtomic8
@@ -36,6 +39,9 @@ _OSUnserializechar
 _OSUnserializelval
 _OSUnserializenerrs
 _OSlibkernInit
+_SHA1Final
+_SHA1Init
+_SHA1Update
 __Z13OSUnserializePKcPP8OSString
 __Z16OSUnserializeXMLPKcPP8OSString
 __ZN10OSIterator10gMetaClassE
@@ -707,6 +713,7 @@ __ZTVN9OSBoolean9MetaClassE
 __ZdlPv
 __Znwm
 ___cxa_pure_virtual
+_adler32
 _atoi
 _bcmp
 _bcopy
@@ -719,10 +726,26 @@ _copyout
 _copyoutstr
 _crc32
 _debug_ivars_size
+_deflate
+_deflateCopy
+_deflateEnd
+_deflateInit2_
+_deflateInit_
+_deflateParams
+_deflateReset
+_deflateSetDictionary
 _ffs
 _flush_dcache
 _flush_dcache64
 _inet_ntop
+_inflate
+_inflateEnd
+_inflateInit2_
+_inflateInit_
+_inflateReset
+_inflateSetDictionary
+_inflateSync
+_inflateSyncPoint
 _invalidate_icache
 _invalidate_icache64
 _itoa
@@ -749,13 +772,17 @@ _lck_mtx_free
 _lck_mtx_init
 _lck_mtx_lock
 _lck_mtx_unlock
+_lck_mtx_try_lock
 _lck_rw_alloc_init
 _lck_rw_destroy
 _lck_rw_free
 _lck_rw_init
 _lck_rw_lock
 _lck_rw_lock_exclusive
+_lck_rw_lock_exclusive_to_shared
 _lck_rw_lock_shared
+_lck_rw_lock_shared_to_exclusive
+_lck_rw_try_lock
 _lck_rw_unlock
 _lck_rw_unlock_exclusive
 _lck_rw_unlock_shared
@@ -778,6 +805,10 @@ _page_shift
 _page_size
 _panic
 _printf
+_sha1_init:_SHA1Init
+_sha1_loop:_SHA1Update
+_sha1_result:_SHA1Final_r
+_snprintf
 _sprintf
 _sscanf
 _strcasecmp
@@ -785,12 +816,15 @@ _strcat
 _strchr
 _strcmp
 _strcpy
+_STRDUP
 _strlen
 _strncasecmp
 _strncat
 _strncmp
 _strncpy
 _strnlen
+_strlcat
+_strlcpy
 _strprefix
 _strtol
 _strtoq
@@ -815,6 +849,9 @@ _version_prerelease_level
 _version_revision
 _version_stage
 _version_variant
+_vprintf
 _vsnprintf
 _vsprintf
 _vsscanf
+_zError
+_zlibVersion
diff --git a/config/Libkern.i386.exports b/config/Libkern.i386.exports
index e69de29bb..362488593 100644
--- a/config/Libkern.i386.exports
+++ b/config/Libkern.i386.exports
@@ -0,0 +1,2 @@
+_OSCompareAndSwap64
+_OSAddAtomic64
diff --git a/config/MACFramework.exports b/config/MACFramework.exports
new file mode 100644
index 000000000..1d551bebf
--- /dev/null
+++ b/config/MACFramework.exports
@@ -0,0 +1,19 @@
+_mac_policy_register
+_mac_policy_unregister
+_mac_vnop_getxattr
+_mac_vnop_setxattr
+_mac_vnop_removexattr
+
+_mac_audit_text
+
+_sbuf_cat
+_sbuf_data
+_sbuf_delete
+_sbuf_finish
+_sbuf_new
+_sbuf_printf
+_sbuf_putc
+_strsep
+
+_VNOP_SETXATTR
+_VNOP_GETXATTR
diff --git a/config/MACFramework.i386.exports b/config/MACFramework.i386.exports
new file mode 100644
index 000000000..6006136b4
--- /dev/null
+++ b/config/MACFramework.i386.exports
@@ -0,0 +1,9 @@
+_kau_will_audit
+_mac_kalloc
+_mac_kalloc_noblock
+_mac_kfree
+_mac_mbuf_alloc
+_mac_mbuf_free
+_mac_unwire
+_mac_wire
+_sysctl__security_mac_children
diff --git a/config/MACFramework.ppc.exports b/config/MACFramework.ppc.exports
new file mode 100644
index 000000000..6006136b4
--- /dev/null
+++ b/config/MACFramework.ppc.exports
@@ -0,0 +1,9 @@
+_kau_will_audit
+_mac_kalloc
+_mac_kalloc_noblock
+_mac_kfree
+_mac_mbuf_alloc
+_mac_mbuf_free
+_mac_unwire
+_mac_wire
+_sysctl__security_mac_children
diff --git a/config/Mach.i386.exports b/config/Mach.i386.exports
index e69de29bb..09d50d0f1 100644
--- a/config/Mach.i386.exports
+++ b/config/Mach.i386.exports
@@ -0,0 +1,2 @@
+_host_vmxon
+_host_vmxoff
diff --git a/config/Makefile b/config/Makefile
index a0de5228d..78fb43d85 100644
--- a/config/Makefile
+++ b/config/Makefile
@@ -1,3 +1,5 @@
+MAC = defined
+
 export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
 export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
@@ -15,12 +17,16 @@ INSTINC_SUBDIRS_PPC =
 
 INSTINC_SUBDIRS_I386 = 
 
+INSTINC_SUBDIRS_ARM = 
+
 EXPINC_SUBDIRS = 
 
 EXPINC_SUBDIRS_PPC =  
 
 EXPINC_SUBDIRS_I386 = 
 
+EXPINC_SUBDIRS_ARM = 
+
 COMP_SUBDIRS = 
 
 INST_SUBDIRS =	
@@ -32,6 +38,7 @@ INSTALL_DATA_LIST= \
 	System.kext/PlugIns/Libkern.kext/Info.plist \
 	System.kext/PlugIns/Mach.kext/Info.plist \
 	System.kext/PlugIns/BSDKernel.kext/Info.plist \
+	System.kext/PlugIns/MACFramework.kext/Info.plist \
 	System.kext/PlugIns/IOKit.kext/Info.plist \
 	System.kext/PlugIns/AppleNMI.kext/Info.plist \
 	System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist \
@@ -55,6 +62,12 @@ INSTMAN_SUBDIRS =
 KEXT_CREATE_SYMBOL_SET = /usr/local/bin/kextsymboltool
 NEWVERS = $(SRCROOT)/config/newvers.pl
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+OBJPATH = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)
+else
+OBJPATH = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)
+endif
+
 SYMBOL_COMPONENT_LIST =	\
         System6.0	\
         BSDKernel	\
@@ -63,30 +76,41 @@ SYMBOL_COMPONENT_LIST =	\
         Mach	\
 	Unsupported
 
-SYMBOL_SET_BUILD = $(foreach set, $(SYMBOL_COMPONENT_LIST), $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(set).symbolset)
+ifdef MAC
+SYMBOL_COMPONENT_LIST += MACFramework
+MACFRAMEWORKEXPORTS = \
+	-export $(SRCROOT)/$(COMPONENT)/MACFramework.exports \
+	-export $(SRCROOT)/$(COMPONENT)/MACFramework.$(ARCH_CONFIG_LC).exports
+endif
+
+SYMBOL_SET_BUILD = $(foreach set, $(SYMBOL_COMPONENT_LIST), $(OBJPATH)/$(set).symbolset)
 SYMBOL_SET_FAT = $(foreach set, $(SYMBOL_COMPONENT_LIST), $(OBJROOT)/$(set).symbolset)
 
 ## .SUFFIXES: .symbolset .symbollist
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/allsymbols: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/mach_kernel
-	nm -gj $< > $@
+$(OBJPATH)/allsymbols: $(OBJPATH)/mach_kernel
+	$(_v)nm -gj $< > $@
 
-$(SYMBOL_SET_BUILD): $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/%.symbolset :  %.exports %.$(ARCH_CONFIG_LC).exports $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/allsymbols
-	$(KEXT_CREATE_SYMBOL_SET) \
-		-arch $(ARCH_CONFIG_LC) \
-		-import $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/allsymbols \
+$(SYMBOL_SET_BUILD): $(OBJPATH)/%.symbolset :  %.exports %.$(ARCH_CONFIG_LC).exports $(OBJPATH)/allsymbols
+	$(_v)$(KEXT_CREATE_SYMBOL_SET) \
+		$($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
+		-import $(OBJPATH)/allsymbols \
 		-export $*.exports \
 		-export $*.$(ARCH_CONFIG_LC).exports \
 		-output $@;
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+$(SYMBOL_SET_FAT): $(OBJROOT)/%.symbolset : $(foreach arch, $(INSTALL_ARCHS), $(OBJROOT)/$(KERNEL_CONFIG)_$(arch)_$(MACHINE_CONFIG)/%.symbolset)
+	$(LIPO) $(foreach arch, $(INSTALL_ARCHS), $(OBJROOT)/$(KERNEL_CONFIG)_$(arch)_$(MACHINE_CONFIG)/$*.symbolset) -create -output $@;
+else
 $(SYMBOL_SET_FAT): $(OBJROOT)/%.symbolset : $(foreach arch, $(INSTALL_ARCHS), $(OBJROOT)/$(KERNEL_CONFIG)_$(arch)/%.symbolset)
-	$(LIPO) $(foreach arch, $(INSTALL_ARCHS), $(OBJROOT)/$(KERNEL_CONFIG)_$(arch)/$*.symbolset) -create -output $@;
-
+	$(_v)$(LIPO) $(foreach arch, $(INSTALL_ARCHS), $(OBJROOT)/$(KERNEL_CONFIG)_$(arch)/$*.symbolset) -create -output $@;
+endif
 
 build_symbol_sets:	$(SYMBOL_SET_BUILD)
-	$(KEXT_CREATE_SYMBOL_SET) \
-		-arch $(ARCH_CONFIG_LC) \
-		-import $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/allsymbols \
+	$(_v)$(KEXT_CREATE_SYMBOL_SET) \
+		$($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
+		-import $(OBJPATH)/allsymbols \
 		-export $(SRCROOT)/$(COMPONENT)/Libkern.exports \
 		-export $(SRCROOT)/$(COMPONENT)/Libkern.$(ARCH_CONFIG_LC).exports \
 		-export $(SRCROOT)/$(COMPONENT)/Mach.exports \
@@ -95,21 +119,26 @@ build_symbol_sets:	$(SYMBOL_SET_BUILD)
 		-export $(SRCROOT)/$(COMPONENT)/IOKit.$(ARCH_CONFIG_LC).exports \
 		-export $(SRCROOT)/$(COMPONENT)/BSDKernel.exports \
 		-export $(SRCROOT)/$(COMPONENT)/BSDKernel.$(ARCH_CONFIG_LC).exports \
+		$(MACFRAMEWORKEXPORTS) \
 		-export $(SRCROOT)/$(COMPONENT)/Unsupported.exports \
 		-export $(SRCROOT)/$(COMPONENT)/Unsupported.$(ARCH_CONFIG_LC).exports \
 		-output /dev/null;
 
 install_symbol_sets:	$(SYMBOL_SET_FAT) $(SRCROOT)/config/MasterVersion
-	install $(INSTALL_FLAGS) $(OBJROOT)/System6.0.symbolset $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/System6.0.kext/kernel.6.0;
-	install $(INSTALL_FLAGS) $(OBJROOT)/BSDKernel.symbolset $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/BSDKernel.kext/BSDKernel;
-	install $(INSTALL_FLAGS) $(OBJROOT)/IOKit.symbolset     $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/IOKit.kext/IOKit;
-	install $(INSTALL_FLAGS) $(OBJROOT)/Libkern.symbolset   $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Libkern.kext/Libkern;
-	install $(INSTALL_FLAGS) $(OBJROOT)/Mach.symbolset      $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Mach.kext/Mach;
-	install $(INSTALL_FLAGS) $(OBJROOT)/Unsupported.symbolset      $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Unsupported.kext/Unsupported;
-	$(NEWVERS) $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/Info.plist \
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/System6.0.symbolset $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/System6.0.kext/kernel.6.0;
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/BSDKernel.symbolset $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/BSDKernel.kext/BSDKernel;
+ifdef MAC
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/MACFramework.symbolset $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/MACFramework.kext/MACFramework;
+endif
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/IOKit.symbolset     $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/IOKit.kext/IOKit;
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/Libkern.symbolset   $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Libkern.kext/Libkern;
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/Mach.symbolset      $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Mach.kext/Mach;
+	$(_v)install $(INSTALL_FLAGS) $(OBJROOT)/Unsupported.symbolset      $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/Unsupported.kext/Unsupported;
+	$(_v)$(NEWVERS) $(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/AppleNMI.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/ApplePlatformFamily.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/BSDKernel.kext/Info.plist \
+		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/MACFramework.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/IOKit.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/IONVRAMFamily.kext/Info.plist \
 		$(DSTROOT)/$(INSTALL_DATA_DIR)/System.kext/PlugIns/IOSystemManagement.kext/Info.plist \
diff --git a/config/MasterVersion b/config/MasterVersion
index 62386907b..d3efc95f2 100644
--- a/config/MasterVersion
+++ b/config/MasterVersion
@@ -1,4 +1,4 @@
-8.11.1
+9.0.0
 
 # The first line of this file contains the master version number for the kernel.
 # All other instances of the kernel version in xnu are derived from this file.
diff --git a/config/System.kext/PlugIns/MACFramework.kext/Info.plist b/config/System.kext/PlugIns/MACFramework.kext/Info.plist
new file mode 100644
index 000000000..93ab38088
--- /dev/null
+++ b/config/System.kext/PlugIns/MACFramework.kext/Info.plist
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>English</string>
+	<key>CFBundleExecutable</key>
+	<string>MACFramework</string>
+	<key>CFBundleGetInfoString</key>
+	<string>MAC Framework Pseudoextension, SPARTA Inc, ###KERNEL_VERSION_LONG###</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.apple.kpi.dsep</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>MAC Framework Pseudoextension</string>
+	<key>CFBundlePackageType</key>
+	<string>KEXT</string>
+	<key>CFBundleShortVersionString</key>
+	<string>###KERNEL_VERSION_SHORT###</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>###KERNEL_VERSION_LONG###</string>
+	<key>OSBundleCompatibleVersion</key>
+	<string>8.0.0b1</string>
+	<key>OSBundleRequired</key>
+	<string>Root</string>
+	<key>OSKernelResource</key>
+	<true/>
+</dict>
+</plist>
diff --git a/config/System6.0.exports b/config/System6.0.exports
index a3ad829d3..71c1ae0fd 100644
--- a/config/System6.0.exports
+++ b/config/System6.0.exports
@@ -177,14 +177,10 @@ _PE_state
 _StartIOKit
 __Z10tellClientP8OSObjectPv
 __Z13OSUnserializePKcPP8OSString
-__Z13readExtensionP12OSDictionaryPKc
-__Z14readExtensionsP12OSDictionaryPKcS0_
 __Z16IOCPUSleepKernelv
 __Z16IODTFindSlotNameP15IORegistryEntrym
 __Z16IODTSetResolvingP15IORegistryEntryPFlmPmS1_EPFvS0_PhS4_S4_E
 __Z16OSUnserializeXMLPKcPP8OSString
-__Z16addPersonalitiesP12OSDictionary
-__Z16uncompressModuleP6OSDataPS0_
 __Z17IODTGetCellCountsP15IORegistryEntryPmS1_
 __Z17IODTMapInterruptsP15IORegistryEntry
 __Z17IODeviceTreeAllocPv
@@ -195,25 +191,15 @@ __Z19printDictionaryKeysP12OSDictionaryPc
 __Z19tellAppWithResponseP8OSObjectPv
 __Z20IODTMakeNVDescriptorP15IORegistryEntryP17IONVRAMDescriptor
 __Z20IODTMatchNubWithKeysP15IORegistryEntryPKc
-__Z20getBootLoaderObjectsv
-__Z20getStartupExtensionsv
 __Z21IODTResolveAddressingP15IORegistryEntryPKcP14IODeviceMemory
-__Z21validateExtensionDictP12OSDictionaryi
 __Z22IODTResolveAddressCellP15IORegistryEntryPmS1_S1_
-__Z22removeStartupExtensionPKc
 __Z22tellClientWithResponseP8OSObjectPv
 __Z23IODTFindInterruptParentP15IORegistryEntry
 __Z23IODTFindMatchingEntriesP15IORegistryEntrymPKc
-__Z23recordStartupExtensionsv
-__Z24addExtensionsFromArchiveP6OSData
 __Z24broadcast_aggressivenessP8OSObjectPvS1_S1_S1_
-__Z24compareExtensionVersionsP12OSDictionaryS0_
-__Z26mergeExtensionDictionariesP12OSDictionaryS0_
 __Z26serializedAllowPowerChangeP8OSObjectPvS1_S1_S1_
 __Z27IODTInterruptControllerNameP15IORegistryEntry
 __Z27serializedCancelPowerChangeP8OSObjectPvS1_S1_S1_
-__Z28extractExtensionsFromArchiveP17MemoryMapFileInfoP12OSDictionary
-__Z35clearStartupExtensionsAndLoaderInfov
 __ZN10IOMachPort10gMetaClassE
 __ZN10IOMachPort10superClassE
 __ZN10IOMachPort11dictForTypeEj
@@ -460,6 +446,7 @@ __ZN12IOUserClient11clientCloseEv
 __ZN12IOUserClient12initWithTaskEP4taskPvm
 __ZN12IOUserClient12initWithTaskEP4taskPvmP12OSDictionary
 __ZN12IOUserClient13connectClientEPS_
+__ZN12IOUserClient14externalMethodEjP25IOExternalMethodArgumentsP24IOExternalMethodDispatchP8OSObjectPv
 __ZN12IOUserClient15mapClientMemoryEmP4taskmj
 __ZN12IOUserClient15sendAsyncResultEPjiPPvm
 __ZN12IOUserClient17setAsyncReferenceEPjP8ipc_portPvS3_
@@ -467,7 +454,6 @@ __ZN12IOUserClient18clientHasPrivilegeEPvPKc
 __ZN12IOUserClient19clientMemoryForTypeEmPmPP18IOMemoryDescriptor
 __ZN12IOUserClient20exportObjectToClientEP4taskP8OSObjectPS3_
 __ZN12IOUserClient21destroyUserReferencesEP8OSObject
-__ZN12IOUserClient22_RESERVEDIOUserClient0Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient1Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient2Ev
 __ZN12IOUserClient22_RESERVEDIOUserClient3Ev
@@ -502,10 +488,6 @@ __ZN12IOUserClientC1EPK11OSMetaClass
 __ZN12IOUserClientC2EPK11OSMetaClass
 __ZN12IOUserClientD0Ev
 __ZN12IOUserClientD2Ev
-__ZN12KLDBootstrapC1Ev
-__ZN12KLDBootstrapC2Ev
-__ZN12KLDBootstrapD1Ev
-__ZN12KLDBootstrapD2Ev
 __ZN12OSCollection10gMetaClassE
 __ZN12OSCollection10setOptionsEjjPv
 __ZN12OSCollection10superClassE
@@ -626,30 +608,6 @@ __ZN12OSSymbolPoolD1Ev
 __ZN12OSSymbolPoolD2Ev
 __ZN12OSSymbolPooldlEPvm
 __ZN12OSSymbolPoolnwEm
-__ZN12_IOMemoryMap10gMetaClassE
-__ZN12_IOMemoryMap10superClassE
-__ZN12_IOMemoryMap13getMapOptionsEv
-__ZN12_IOMemoryMap14copyCompatibleEP18IOMemoryDescriptorP4taskjmmm
-__ZN12_IOMemoryMap14getAddressTaskEv
-__ZN12_IOMemoryMap14initCompatibleEP18IOMemoryDescriptorP11IOMemoryMapmm
-__ZN12_IOMemoryMap17getVirtualAddressEv
-__ZN12_IOMemoryMap18getPhysicalSegmentEmPm
-__ZN12_IOMemoryMap18initWithDescriptorEP18IOMemoryDescriptorP4taskjmmm
-__ZN12_IOMemoryMap19getMemoryDescriptorEv
-__ZN12_IOMemoryMap4freeEv
-__ZN12_IOMemoryMap5unmapEv
-__ZN12_IOMemoryMap8redirectEP4taskb
-__ZN12_IOMemoryMap8taskDiedEv
-__ZN12_IOMemoryMap9MetaClassC1Ev
-__ZN12_IOMemoryMap9MetaClassC2Ev
-__ZN12_IOMemoryMap9getLengthEv
-__ZN12_IOMemoryMap9metaClassE
-__ZN12_IOMemoryMapC1EPK11OSMetaClass
-__ZN12_IOMemoryMapC1Ev
-__ZN12_IOMemoryMapC2EPK11OSMetaClass
-__ZN12_IOMemoryMapC2Ev
-__ZN12_IOMemoryMapD0Ev
-__ZN12_IOMemoryMapD2Ev
 __ZN13IOCommandGate10gMetaClassE
 __ZN13IOCommandGate10runCommandEPvS0_S0_S0_
 __ZN13IOCommandGate10superClassE
@@ -1222,7 +1180,6 @@ __ZN18IOMemoryDescriptor10setMappingEP4taskjm
 __ZN18IOMemoryDescriptor10superClassE
 __ZN18IOMemoryDescriptor10withRangesEP14IOVirtualRangem11IODirectionP4taskb
 __ZN18IOMemoryDescriptor10writeBytesEmPKvm
-__ZN18IOMemoryDescriptor11handleFaultEPvP6vm_mapjmmm
 __ZN18IOMemoryDescriptor11makeMappingEPS_P4taskjmmm
 __ZN18IOMemoryDescriptor11withAddressEPvm11IODirection
 __ZN18IOMemoryDescriptor11withAddressEjm11IODirectionP4task
@@ -1251,10 +1208,10 @@ __ZN18IOMemoryDescriptor30withPersistentMemoryDescriptorEPS_
 __ZN18IOMemoryDescriptor3mapEP4taskjmmm
 __ZN18IOMemoryDescriptor3mapEm
 __ZN18IOMemoryDescriptor4freeEv
-__ZN18IOMemoryDescriptor5doMapEP6vm_mapPjmmm
+__ZN18IOMemoryDescriptor5doMapEP7_vm_mapPjmmm
 __ZN18IOMemoryDescriptor6getTagEv
 __ZN18IOMemoryDescriptor6setTagEm
-__ZN18IOMemoryDescriptor7doUnmapEP6vm_mapjm
+__ZN18IOMemoryDescriptor7doUnmapEP7_vm_mapjm
 __ZN18IOMemoryDescriptor8redirectEP4taskb
 __ZN18IOMemoryDescriptor9MetaClassC1Ev
 __ZN18IOMemoryDescriptor9MetaClassC2Ev
@@ -1264,29 +1221,6 @@ __ZN18IOMemoryDescriptorC1EPK11OSMetaClass
 __ZN18IOMemoryDescriptorC2EPK11OSMetaClass
 __ZN18IOMemoryDescriptorD0Ev
 __ZN18IOMemoryDescriptorD2Ev
-__ZN18IOPMchangeNoteList10gMetaClassE
-__ZN18IOPMchangeNoteList10initializeEv
-__ZN18IOPMchangeNoteList10superClassE
-__ZN18IOPMchangeNoteList12latestChangeEv
-__ZN18IOPMchangeNoteList13currentChangeEv
-__ZN18IOPMchangeNoteList14nextChangeNoteEm
-__ZN18IOPMchangeNoteList15changeNoteInUseEm
-__ZN18IOPMchangeNoteList16createChangeNoteEv
-__ZN18IOPMchangeNoteList18previousChangeNoteEm
-__ZN18IOPMchangeNoteList21releaseHeadChangeNoteEv
-__ZN18IOPMchangeNoteList21releaseTailChangeNoteEv
-__ZN18IOPMchangeNoteList9MetaClassC1Ev
-__ZN18IOPMchangeNoteList9MetaClassC2Ev
-__ZN18IOPMchangeNoteList9decrementEm
-__ZN18IOPMchangeNoteList9incrementEm
-__ZN18IOPMchangeNoteList9listEmptyEv
-__ZN18IOPMchangeNoteList9metaClassE
-__ZN18IOPMchangeNoteListC1EPK11OSMetaClass
-__ZN18IOPMchangeNoteListC1Ev
-__ZN18IOPMchangeNoteListC2EPK11OSMetaClass
-__ZN18IOPMchangeNoteListC2Ev
-__ZN18IOPMchangeNoteListD0Ev
-__ZN18IOPMchangeNoteListD2Ev
 __ZN18IORegistryIterator10enterEntryEPK15IORegistryPlane
 __ZN18IORegistryIterator10enterEntryEv
 __ZN18IORegistryIterator10gMetaClassE
@@ -1521,7 +1455,7 @@ __ZN21IOSubMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN21IOSubMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN21IOSubMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN21IOSubMemoryDescriptor4freeEv
-__ZN21IOSubMemoryDescriptor5doMapEP6vm_mapPjmmm
+__ZN21IOSubMemoryDescriptor5doMapEP7_vm_mapPjmmm
 __ZN21IOSubMemoryDescriptor7prepareE11IODirection
 __ZN21IOSubMemoryDescriptor8completeE11IODirection
 __ZN21IOSubMemoryDescriptor8redirectEP4taskb
@@ -1566,6 +1500,7 @@ __ZN22IOInterruptEventSourceD2Ev
 __ZN22IOPlatformExpertDevice10gMetaClassE
 __ZN22IOPlatformExpertDevice10superClassE
 __ZN22IOPlatformExpertDevice12initWithArgsEPvS0_S0_S0_
+__ZN22IOPlatformExpertDevice13setPropertiesEP8OSObject
 __ZN22IOPlatformExpertDevice32_RESERVEDIOPlatformExpertDevice0Ev
 __ZN22IOPlatformExpertDevice32_RESERVEDIOPlatformExpertDevice1Ev
 __ZN22IOPlatformExpertDevice32_RESERVEDIOPlatformExpertDevice2Ev
@@ -1637,8 +1572,7 @@ __ZN24IOBufferMemoryDescriptor15initWithAddressEjm11IODirectionP4task
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjj
 __ZN24IOBufferMemoryDescriptor15initWithOptionsEmjjP4task
 __ZN24IOBufferMemoryDescriptor17inTaskWithOptionsEP4taskmjj
-__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
-__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
+__ZN24IOBufferMemoryDescriptor17getVirtualSegmentEmPm
 __ZN24IOBufferMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN24IOBufferMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN24IOBufferMemoryDescriptor34_RESERVEDIOBufferMemoryDescriptor2Ev
@@ -1711,8 +1645,8 @@ __ZN25IOGeneralMemoryDescriptor20getPhysicalSegment64EmPm
 __ZN25IOGeneralMemoryDescriptor22initWithPhysicalRangesEP15IOPhysicalRangem11IODirectionb
 __ZN25IOGeneralMemoryDescriptor23initWithPhysicalAddressEmm11IODirection
 __ZN25IOGeneralMemoryDescriptor4freeEv
-__ZN25IOGeneralMemoryDescriptor5doMapEP6vm_mapPjmmm
-__ZN25IOGeneralMemoryDescriptor7doUnmapEP6vm_mapjm
+__ZN25IOGeneralMemoryDescriptor5doMapEP7_vm_mapPjmmm
+__ZN25IOGeneralMemoryDescriptor7doUnmapEP7_vm_mapjm
 __ZN25IOGeneralMemoryDescriptor7prepareE11IODirection
 __ZN25IOGeneralMemoryDescriptor8completeE11IODirection
 __ZN25IOGeneralMemoryDescriptor9MetaClassC1Ev
@@ -1724,22 +1658,6 @@ __ZN25IOGeneralMemoryDescriptorC2EPK11OSMetaClass
 __ZN25IOGeneralMemoryDescriptorC2Ev
 __ZN25IOGeneralMemoryDescriptorD0Ev
 __ZN25IOGeneralMemoryDescriptorD2Ev
-__ZN25IOServiceUserNotification10gMetaClassE
-__ZN25IOServiceUserNotification10superClassE
-__ZN25IOServiceUserNotification13getNextObjectEv
-__ZN25IOServiceUserNotification4freeEv
-__ZN25IOServiceUserNotification4initEP8ipc_portjPj
-__ZN25IOServiceUserNotification7handlerEPvP9IOService
-__ZN25IOServiceUserNotification8_handlerEPvS0_P9IOService
-__ZN25IOServiceUserNotification9MetaClassC1Ev
-__ZN25IOServiceUserNotification9MetaClassC2Ev
-__ZN25IOServiceUserNotification9metaClassE
-__ZN25IOServiceUserNotificationC1EPK11OSMetaClass
-__ZN25IOServiceUserNotificationC1Ev
-__ZN25IOServiceUserNotificationC2EPK11OSMetaClass
-__ZN25IOServiceUserNotificationC2Ev
-__ZN25IOServiceUserNotificationD0Ev
-__ZN25IOServiceUserNotificationD2Ev
 __ZN26_IOServiceInterestNotifier10gMetaClassE
 __ZN26_IOServiceInterestNotifier10superClassE
 __ZN26_IOServiceInterestNotifier4freeEv
@@ -1805,22 +1723,6 @@ __ZN28IOFilterInterruptEventSourceC2EPK11OSMetaClass
 __ZN28IOFilterInterruptEventSourceC2Ev
 __ZN28IOFilterInterruptEventSourceD0Ev
 __ZN28IOFilterInterruptEventSourceD2Ev
-__ZN32IOServiceMessageUserNotification10gMetaClassE
-__ZN32IOServiceMessageUserNotification10superClassE
-__ZN32IOServiceMessageUserNotification13getNextObjectEv
-__ZN32IOServiceMessageUserNotification4freeEv
-__ZN32IOServiceMessageUserNotification4initEP8ipc_portjPjj
-__ZN32IOServiceMessageUserNotification7handlerEPvmP9IOServiceS0_j
-__ZN32IOServiceMessageUserNotification8_handlerEPvS0_mP9IOServiceS0_j
-__ZN32IOServiceMessageUserNotification9MetaClassC1Ev
-__ZN32IOServiceMessageUserNotification9MetaClassC2Ev
-__ZN32IOServiceMessageUserNotification9metaClassE
-__ZN32IOServiceMessageUserNotificationC1EPK11OSMetaClass
-__ZN32IOServiceMessageUserNotificationC1Ev
-__ZN32IOServiceMessageUserNotificationC2EPK11OSMetaClass
-__ZN32IOServiceMessageUserNotificationC2Ev
-__ZN32IOServiceMessageUserNotificationD0Ev
-__ZN32IOServiceMessageUserNotificationD2Ev
 __ZN5IOCPU10gMetaClassE
 __ZN5IOCPU10superClassE
 __ZN5IOCPU11getCPUGroupEv
@@ -1993,17 +1895,6 @@ __ZN8IOMapperC1EPK11OSMetaClass
 __ZN8IOMapperC2EPK11OSMetaClass
 __ZN8IOMapperD0Ev
 __ZN8IOMapperD2Ev
-__ZN8IOPMpriv10gMetaClassE
-__ZN8IOPMpriv10superClassE
-__ZN8IOPMpriv9MetaClassC1Ev
-__ZN8IOPMpriv9MetaClassC2Ev
-__ZN8IOPMpriv9metaClassE
-__ZN8IOPMprivC1EPK11OSMetaClass
-__ZN8IOPMprivC1Ev
-__ZN8IOPMprivC2EPK11OSMetaClass
-__ZN8IOPMprivC2Ev
-__ZN8IOPMprivD0Ev
-__ZN8IOPMprivD2Ev
 __ZN8IOPMprot10gMetaClassE
 __ZN8IOPMprot10superClassE
 __ZN8IOPMprot9MetaClassC1Ev
@@ -2205,7 +2096,6 @@ __ZN9IODTNVRAMD0Ev
 __ZN9IODTNVRAMD2Ev
 __ZN9IOService10actionStopEPS_S0_
 __ZN9IOService10adjustBusyEl
-__ZN9IOService10ask_parentEm
 __ZN9IOService10gMetaClassE
 __ZN9IOService10handleOpenEPS_mPv
 __ZN9IOService10initializeEv
@@ -2216,17 +2106,12 @@ __ZN9IOService10systemWakeEv
 __ZN9IOService10youAreRootEv
 __ZN9IOService11_adjustBusyEl
 __ZN9IOService11addLocationEP12OSDictionary
-__ZN9IOService11changeStateEv
 __ZN9IOService11getPlatformEv
 __ZN9IOService11handleCloseEPS_m
-__ZN9IOService11notifyChildEP17IOPowerConnectionb
 __ZN9IOService11setPlatformEP16IOPlatformExpert
 __ZN9IOService11tellClientsEi
-__ZN9IOService12acquire_lockEv
-__ZN9IOService12checkForDoneEv
 __ZN9IOService12clampPowerOnEm
 __ZN9IOService12didTerminateEPS_mPb
-__ZN9IOService12driver_ackedEv
 __ZN9IOService12getBusyStateEv
 __ZN9IOService12getResourcesEv
 __ZN9IOService12nameMatchingEPK8OSStringP12OSDictionary
@@ -2245,8 +2130,6 @@ __ZN9IOService13matchLocationEPS_
 __ZN9IOService13messageClientEmP8OSObjectPvj
 __ZN9IOService13newUserClientEP4taskPvmP12OSDictionaryPP12IOUserClient
 __ZN9IOService13newUserClientEP4taskPvmPP12IOUserClient
-__ZN9IOService13responseValidEm
-__ZN9IOService13setParentInfoEmP17IOPowerConnection
 __ZN9IOService13setPowerStateEmPS_
 __ZN9IOService13startMatchingEm
 __ZN9IOService13waitMatchIdleEm
@@ -2262,10 +2145,8 @@ __ZN9IOService14messageClientsEmPvj
 __ZN9IOService14newTemperatureElPS_
 __ZN9IOService14setPowerParentEP17IOPowerConnectionbm
 __ZN9IOService14startCandidateEPS_
-__ZN9IOService14stop_ack_timerEv
 __ZN9IOService14tellChangeDownEm
 __ZN9IOService14waitForServiceEP12OSDictionaryP13mach_timespec
-__ZN9IOService15OurChangeFinishEv
 __ZN9IOService15addNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l
 __ZN9IOService15comparePropertyEP12OSDictionaryPK8OSString
 __ZN9IOService15comparePropertyEP12OSDictionaryPKc
@@ -2273,11 +2154,11 @@ __ZN9IOService15enableInterruptEi
 __ZN9IOService15errnoFromReturnEi
 __ZN9IOService15getDeviceMemoryEv
 __ZN9IOService15getPMRootDomainEv
-__ZN9IOService15instruct_driverEm
 __ZN9IOService15lookupInterruptEibPP21IOInterruptController
 __ZN9IOService15nextIdleTimeoutE12UnsignedWideS0_j
 __ZN9IOService15powerChangeDoneEm
 __ZN9IOService15probeCandidatesEP12OSOrderedSet
+__ZN9IOService16propertyMatchingEPK8OSSymbolPK8OSObjectP12OSDictionary
 __ZN9IOService15publishResourceEPK8OSSymbolP8OSObject
 __ZN9IOService15publishResourceEPKcP8OSObject
 __ZN9IOService15registerServiceEm
@@ -2286,7 +2167,6 @@ __ZN9IOService15serviceMatchingEPKcP12OSDictionary
 __ZN9IOService15setDeviceMemoryEP7OSArray
 __ZN9IOService15setNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_l
 __ZN9IOService15setPMRootDomainEP14IOPMrootDomain
-__ZN9IOService15start_ack_timerEv
 __ZN9IOService15tellChangeDown1Em
 __ZN9IOService15tellChangeDown2Em
 __ZN9IOService15terminateClientEPS_m
@@ -2307,12 +2187,9 @@ __ZN9IOService16resolveInterruptEPS_i
 __ZN9IOService16resourceMatchingEPK8OSStringP12OSDictionary
 __ZN9IOService16resourceMatchingEPKcP12OSDictionary
 __ZN9IOService16scheduleFinalizeEv
-__ZN9IOService16startSettleTimerEm
-__ZN9IOService16start_our_changeEm
 __ZN9IOService16stringFromReturnEi
 __ZN9IOService16tellNoChangeDownEm
 __ZN9IOService17addNeededResourceEPKc
-__ZN9IOService17allowCancelCommonEv
 __ZN9IOService17applyToInterestedEPK8OSSymbolPFvP8OSObjectPvES5_
 __ZN9IOService17cancelPowerChangeEm
 __ZN9IOService17catalogNewDriversEP12OSOrderedSet
@@ -2324,22 +2201,19 @@ __ZN9IOService17setAggressivenessEmm
 __ZN9IOService18actionDidTerminateEPS_m
 __ZN9IOService18changePowerStateToEm
 __ZN9IOService18doServiceTerminateEm
-__ZN9IOService18enqueuePowerChangeEmmmP17IOPowerConnectionm
 __ZN9IOService18getResourceServiceEv
 __ZN9IOService18lockForArbitrationEb
 __ZN9IOService18matchPropertyTableEP12OSDictionary
 __ZN9IOService18matchPropertyTableEP12OSDictionaryPl
 __ZN9IOService18setIdleTimerPeriodEm
 __ZN9IOService18settleTimerExpiredEv
-__ZN9IOService19_RESERVEDIOService4Ev
+__ZN9IOService18systemWillShutdownEm
 __ZN9IOService19_RESERVEDIOService5Ev
 __ZN9IOService19_RESERVEDIOService6Ev
 __ZN9IOService19_RESERVEDIOService7Ev
 __ZN9IOService19_RESERVEDIOService8Ev
 __ZN9IOService19_RESERVEDIOService9Ev
 __ZN9IOService19actionWillTerminateEPS_mP7OSArray
-__ZN9IOService19computeDesiredStateEv
-__ZN9IOService19compute_settle_timeEv
 __ZN9IOService19deliverNotificationEPK8OSSymbolmm
 __ZN9IOService19getExistingServicesEP12OSDictionarymm
 __ZN9IOService19getMatchingServicesEP12OSDictionary
@@ -2347,7 +2221,6 @@ __ZN9IOService19installNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_
 __ZN9IOService19powerOverrideOnPrivEv
 __ZN9IOService19registerPowerDriverEPS_P14IOPMPowerStatem
 __ZN9IOService19start_PM_idle_timerEv
-__ZN9IOService19start_parent_changeEm
 __ZN9IOService19unregisterInterruptEi
 __ZN9IOService20_RESERVEDIOService10Ev
 __ZN9IOService20_RESERVEDIOService11Ev
@@ -2395,10 +2268,8 @@ __ZN9IOService20unlockForArbitrationEv
 __ZN9IOService21doInstallNotificationEPK8OSSymbolP12OSDictionaryPFbPvS5_PS_ES5_S5_lPP10OSIterator
 __ZN9IOService21getClientWithCategoryEPK8OSSymbol
 __ZN9IOService21powerStateDidChangeToEmmPS_
-__ZN9IOService21rebuildChildClampBitsEv
 __ZN9IOService21temporaryPowerClampOnEv
 __ZN9IOService21unregisterAllInterestEv
-__ZN9IOService22OurChangeSetPowerStateEv
 __ZN9IOService22PM_Clamp_Timer_ExpiredEv
 __ZN9IOService22acknowledgePowerChangeEPS_
 __ZN9IOService22changePowerStateToPrivEm
@@ -2417,39 +2288,15 @@ __ZN9IOService24getDeviceMemoryWithIndexEj
 __ZN9IOService24mapDeviceMemoryWithIndexEjm
 __ZN9IOService24powerStateForDomainStateEm
 __ZN9IOService24registerInterestedDriverEPS_
-__ZN9IOService26add_child_to_active_changeEP17IOPowerConnection
 __ZN9IOService26deRegisterInterestedDriverEPS_
 __ZN9IOService26temperatureCriticalForZoneEPS_
-__ZN9IOService27OurChangeWaitForPowerSettleEv
-__ZN9IOService27add_driver_to_active_changeEP12IOPMinformee
 __ZN9IOService27maxCapabilityForDomainStateEm
 __ZN9IOService27serializedAllowPowerChange2Em
 __ZN9IOService28serializedCancelPowerChange2Em
-__ZN9IOService29OurChangeTellClientsPowerDownEv
-__ZN9IOService29ParentUpSetPowerState_DelayedEv
-__ZN9IOService31ParentDownSetPowerState_DelayedEv
-__ZN9IOService31ParentUpSetPowerState_ImmediateEv
 __ZN9IOService31initialPowerStateForDomainStateEm
-__ZN9IOService33ParentDownSetPowerState_ImmediateEv
-__ZN9IOService33ParentUpWaitForSettleTime_DelayedEv
-__ZN9IOService35ParentDownAcknowledgeChange_DelayedEv
-__ZN9IOService35ParentUpWaitForSettleTime_ImmediateEv
-__ZN9IOService36ParentDownWaitForPowerSettle_DelayedEv
-__ZN9IOService37OurChangeTellPriorityClientsPowerDownEv
-__ZN9IOService38ParentUpAcknowledgePowerChange_DelayedEv
-__ZN9IOService41OurChangeNotifyInterestedDriversDidChangeEv
-__ZN9IOService42OurChangeNotifyInterestedDriversWillChangeEv
-__ZN9IOService46ParentDownTellPriorityClientsPowerDown_DelayedEv
-__ZN9IOService48ParentDownTellPriorityClientsPowerDown_ImmediateEv
-__ZN9IOService48ParentUpNotifyInterestedDriversDidChange_DelayedEv
 __ZN9IOService4freeEv
 __ZN9IOService4openEPS_mPv
 __ZN9IOService4stopEPS_
-__ZN9IOService50ParentUpNotifyInterestedDriversDidChange_ImmediateEv
-__ZN9IOService51ParentDownNotifyInterestedDriversWillChange_DelayedEv
-__ZN9IOService53ParentDownNotifyDidChangeAndAcknowledgeChange_DelayedEv
-__ZN9IOService53ParentDownNotifyInterestedDriversWillChange_ImmediateEv
-__ZN9IOService56ParentDownWaitForPowerSettleAndNotifyDidChange_ImmediateEv
 __ZN9IOService5closeEPS_m
 __ZN9IOService5probeEPS_Pl
 __ZN9IOService5startEPS_
@@ -2458,15 +2305,11 @@ __ZN9IOService6PMinitEv
 __ZN9IOService6PMstopEv
 __ZN9IOService6attachEPS_
 __ZN9IOService6detachEPS_
-__ZN9IOService6informEP12IOPMinformeeb
 __ZN9IOService7messageEmPS_Pv
-__ZN9IOService8all_doneEv
 __ZN9IOService8finalizeEm
 __ZN9IOService9MetaClassC1Ev
 __ZN9IOService9MetaClassC2Ev
-__ZN9IOService9all_ackedEv
 __ZN9IOService9metaClassE
-__ZN9IOService9notifyAllEb
 __ZN9IOService9resourcesEv
 __ZN9IOService9terminateEm
 __ZN9IOService9waitQuietEP13mach_timespec
@@ -2594,9 +2437,6 @@ __ZNK12OSSerializer12getMetaClassEv
 __ZNK12OSSerializer9MetaClass5allocEv
 __ZNK12OSSerializer9serializeEP11OSSerialize
 __ZNK12OSSymbolPool10findSymbolEPKc
-__ZNK12_IOMemoryMap12getMetaClassEv
-__ZNK12_IOMemoryMap13taggedReleaseEPKv
-__ZNK12_IOMemoryMap9MetaClass5allocEv
 __ZNK13IOCommandGate12getMetaClassEv
 __ZNK13IOCommandGate9MetaClass5allocEv
 __ZNK13IOCommandPool12getMetaClassEv
@@ -2705,8 +2545,6 @@ __ZNK18IOMemoryDescriptor12getMetaClassEv
 __ZNK18IOMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK18IOMemoryDescriptor9MetaClass5allocEv
 __ZNK18IOMemoryDescriptor9getLengthEv
-__ZNK18IOPMchangeNoteList12getMetaClassEv
-__ZNK18IOPMchangeNoteList9MetaClass5allocEv
 __ZNK18IORegistryIterator12getMetaClassEv
 __ZNK18IORegistryIterator9MetaClass5allocEv
 __ZNK18IOTimerEventSource12getMetaClassEv
@@ -2755,8 +2593,6 @@ __ZNK25IOGeneralMemoryDescriptor12getMetaClassEv
 __ZNK25IOGeneralMemoryDescriptor19dmaCommandOperationEmPvj
 __ZNK25IOGeneralMemoryDescriptor9MetaClass5allocEv
 __ZNK25IOGeneralMemoryDescriptor9serializeEP11OSSerialize
-__ZNK25IOServiceUserNotification12getMetaClassEv
-__ZNK25IOServiceUserNotification9MetaClass5allocEv
 __ZNK26_IOServiceInterestNotifier12getMetaClassEv
 __ZNK26_IOServiceInterestNotifier9MetaClass5allocEv
 __ZNK27IOSharedInterruptController12getMetaClassEv
@@ -2764,8 +2600,6 @@ __ZNK27IOSharedInterruptController9MetaClass5allocEv
 __ZNK28IOFilterInterruptEventSource12getMetaClassEv
 __ZNK28IOFilterInterruptEventSource15getFilterActionEv
 __ZNK28IOFilterInterruptEventSource9MetaClass5allocEv
-__ZNK32IOServiceMessageUserNotification12getMetaClassEv
-__ZNK32IOServiceMessageUserNotification9MetaClass5allocEv
 __ZNK5IOCPU11getPropertyEPK8OSSymbol
 __ZNK5IOCPU12getMetaClassEv
 __ZNK5IOCPU19serializePropertiesEP11OSSerialize
@@ -2813,12 +2647,8 @@ __ZNK7OSArray9serializeEP11OSSerialize
 __ZNK8IOMapper12getMetaClassEv
 __ZNK8IOMapper13getBypassMaskEPy
 __ZNK8IOMapper9MetaClass5allocEv
-__ZNK8IOPMpriv12getMetaClassEv
-__ZNK8IOPMpriv9MetaClass5allocEv
-__ZNK8IOPMpriv9serializeEP11OSSerialize
 __ZNK8IOPMprot12getMetaClassEv
 __ZNK8IOPMprot9MetaClass5allocEv
-__ZNK8IOPMprot9serializeEP11OSSerialize
 __ZNK8IOSyncer12getMetaClassEv
 __ZNK8IOSyncer9MetaClass5allocEv
 __ZNK8OSNumber12getMetaClassEv
@@ -2910,7 +2740,6 @@ __ZTV12OSDictionary
 __ZTV12OSOrderedSet
 __ZTV12OSSerializer
 __ZTV12OSSymbolPool
-__ZTV12_IOMemoryMap
 __ZTV13IOCommandGate
 __ZTV13IOCommandPool
 __ZTV13IOEventSource
@@ -2937,7 +2766,6 @@ __ZTV17IOPolledInterface
 __ZTV17IOPowerConnection
 __ZTV18IODTPlatformExpert
 __ZTV18IOMemoryDescriptor
-__ZTV18IOPMchangeNoteList
 __ZTV18IORegistryIterator
 __ZTV18IOTimerEventSource
 __ZTV18IOUserNotification
@@ -2957,17 +2785,14 @@ __ZTV23IOMultiMemoryDescriptor
 __ZTV24IOBufferMemoryDescriptor
 __ZTV24IOCPUInterruptController
 __ZTV25IOGeneralMemoryDescriptor
-__ZTV25IOServiceUserNotification
 __ZTV26_IOServiceInterestNotifier
 __ZTV27IOSharedInterruptController
 __ZTV28IOFilterInterruptEventSource
-__ZTV32IOServiceMessageUserNotification
 __ZTV5IOCPU
 __ZTV5OSSet
 __ZTV6OSData
 __ZTV7OSArray
 __ZTV8IOMapper
-__ZTV8IOPMpriv
 __ZTV8IOPMprot
 __ZTV8IOSyncer
 __ZTV8OSNumber
@@ -2994,7 +2819,6 @@ __ZTVN12OSCollection9MetaClassE
 __ZTVN12OSDictionary9MetaClassE
 __ZTVN12OSOrderedSet9MetaClassE
 __ZTVN12OSSerializer9MetaClassE
-__ZTVN12_IOMemoryMap9MetaClassE
 __ZTVN13IOCommandGate9MetaClassE
 __ZTVN13IOCommandPool9MetaClassE
 __ZTVN13IOEventSource9MetaClassE
@@ -3018,7 +2842,6 @@ __ZTVN17IOBigMemoryCursor9MetaClassE
 __ZTVN17IOPowerConnection9MetaClassE
 __ZTVN18IODTPlatformExpert9MetaClassE
 __ZTVN18IOMemoryDescriptor9MetaClassE
-__ZTVN18IOPMchangeNoteList9MetaClassE
 __ZTVN18IORegistryIterator9MetaClassE
 __ZTVN18IOTimerEventSource9MetaClassE
 __ZTVN18IOUserNotification9MetaClassE
@@ -3038,17 +2861,14 @@ __ZTVN23IOMultiMemoryDescriptor9MetaClassE
 __ZTVN24IOBufferMemoryDescriptor9MetaClassE
 __ZTVN24IOCPUInterruptController9MetaClassE
 __ZTVN25IOGeneralMemoryDescriptor9MetaClassE
-__ZTVN25IOServiceUserNotification9MetaClassE
 __ZTVN26_IOServiceInterestNotifier9MetaClassE
 __ZTVN27IOSharedInterruptController9MetaClassE
 __ZTVN28IOFilterInterruptEventSource9MetaClassE
-__ZTVN32IOServiceMessageUserNotification9MetaClassE
 __ZTVN5IOCPU9MetaClassE
 __ZTVN5OSSet9MetaClassE
 __ZTVN6OSData9MetaClassE
 __ZTVN7OSArray9MetaClassE
 __ZTVN8IOMapper9MetaClassE
-__ZTVN8IOPMpriv9MetaClassE
 __ZTVN8IOPMprot9MetaClassE
 __ZTVN8IOSyncer9MetaClassE
 __ZTVN8OSNumber9MetaClassE
@@ -3114,7 +2934,6 @@ _clock_interval_to_deadline
 _clock_timebase_info
 _cnputc
 _conslog_putc
-_console_user
 _convert_port_entry_to_map
 _convert_port_entry_to_object
 _current_act
@@ -3135,21 +2954,12 @@ _devfs_remove
 _device_close
 _device_data_action
 _devnode_free
-_dgraph_add_dependency
-_dgraph_add_dependent
-_dgraph_establish_load_order
-_dgraph_find_dependent
-_dgraph_find_root
-_dgraph_free
-_dgraph_init
-_dgraph_log
-_disableSerialOuput
+_disable_serial_output
 _ether_check_multi
 _ev_try_lock
 _ev_unlock
 _fatfile_getarch
 _fatfile_getarch_affinity
-_fill_backward_load_order
 _find_entry
 _flush_dcache
 _flush_dcache64
@@ -3227,8 +3037,6 @@ _get_kernel_symfile
 _get_procrustime
 _get_task_map
 _getval
-_hfs_addconverter
-_hfs_remconverter
 _invalidate_icache
 _invalidate_icache64
 _iokit_add_reference
@@ -3327,13 +3135,6 @@ _kernel_pmap
 _kernel_task
 _kernel_thread
 _kfree
-_kld_file_cleanup_all_resources
-_kld_file_getaddr
-_kld_file_lookupsymbol
-_kld_file_map
-_kld_file_merge_OSObjects
-_kld_file_patch_OSObjects
-_kld_file_prepare_for_link
 _kmem_alloc
 _kmem_free
 _kmod
@@ -3359,7 +3160,6 @@ _libsa_version:_version
 _libsa_version_major:_version_major
 _libsa_version_minor:_version_minor
 _libsa_version_variant:_version_variant
-_load_kernel_extension
 _lock_alloc:_lock_alloc_EXT
 _lock_done:_lock_done_EXT
 _lock_free:_lock_free_EXT
@@ -3368,16 +3168,13 @@ _lock_read:_lock_read_EXT
 _lock_read_to_write:_lock_read_to_write_EXT
 _lock_write:_lock_write_EXT
 _lock_write_to_read:_lock_write_to_read_EXT
-_log_level
 _m_mclfree
-_m_mtod
 _mach_absolute_time
 _mach_make_memory_entry_64
 _mach_msg_send_from_kernel
 _machine_idle
 _max_mem
 _mbstat
-_mcl_to_paddr
 _mem_size
 _memcmp
 _memcpy
@@ -3429,7 +3226,6 @@ _pmap_find_phys
 _pmsBuild
 _pmsPark
 _pmsRun
-_pmsRunLocal
 _pmsStart
 _print_vmpage_stat
 _printf
@@ -3437,13 +3233,10 @@ _processor_exit
 _processor_info
 _processor_start
 _random
-_rc4_crypt
-_rc4_init
 _read_random
 _record_startup_extensions_function
 _registerPrioritySleepWakeInterest
 _registerSleepWakeInterest
-_rem3_remangle_name
 _remove_startup_extension_function
 _rootDomainRestart
 _rootDomainShutdown
@@ -3455,10 +3248,9 @@ _semaphore_signal
 _semaphore_signal_all
 _semaphore_timedwait
 _semaphore_wait
-_serial_putc
-_sha1_init
-_sha1_loop
-_sha1_result
+_sha1_init:_SHA1Init
+_sha1_loop:_SHA1Update
+_sha1_result:_SHA1Final_r
 _snprintf
 _spl0
 _splbio
@@ -3483,7 +3275,6 @@ _strcat
 _strchr
 _strcmp
 _strcpy
-_strdup
 _strlen
 _strncat
 _strncmp
diff --git a/config/System6.0.i386.exports b/config/System6.0.i386.exports
index b6968b6d1..10c23908d 100644
--- a/config/System6.0.i386.exports
+++ b/config/System6.0.i386.exports
@@ -7,10 +7,14 @@ _cpu_number
 _cpu_to_lapic
 _cpuid_features
 _cpuid_info
+_hfs_addconverter
+_hfs_remconverter
 _kdreboot
 _lapic_end_of_interrupt
 _lapic_smm_restore
 _ml_get_max_cpus
+_mp_broadcast
+_mp_cpus_call 
 _mp_rendezvous_no_intrs
 _mtrr_range_add
 _mtrr_range_remove
@@ -19,3 +23,6 @@ _pmsCPULoadVIDTable
 _rtc_clock_stepped
 _rtc_clock_stepping
 _smp_initialized
+_thread_bind
+__ZN24IOBufferMemoryDescriptor20initWithPhysicalMaskEP4taskmyyy
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
diff --git a/config/System6.0.ppc.exports b/config/System6.0.ppc.exports
index 3b367efcc..66b5ad2e0 100644
--- a/config/System6.0.ppc.exports
+++ b/config/System6.0.ppc.exports
@@ -115,6 +115,7 @@ __ZN19IODBDMAMemoryCursorC2EPK11OSMetaClass
 __ZN19IODBDMAMemoryCursorC2Ev
 __ZN19IODBDMAMemoryCursorD0Ev
 __ZN19IODBDMAMemoryCursorD2Ev
+__ZN24IOBufferMemoryDescriptor22inTaskWithPhysicalMaskEP4taskmyy
 __ZN8AppleCPU10gMetaClassE
 __ZN8AppleCPU10getCPUNameEv
 __ZN8AppleCPU10quiesceCPUEv
@@ -227,6 +228,8 @@ _delay_for_interval
 _gGetDefaultBusSpeedsKey
 _get_io_base_addr
 _get_preemption_level
+_hfs_addconverter
+_hfs_remconverter
 _ignore_zero_fault
 _killprint
 _kprintf_lock
@@ -241,4 +244,7 @@ _ml_throttle
 _ml_mem_backoff
 _pe_do_clock_test
 _pe_run_clock_test
+_pmsRunLocal
 _scc
+_rc4_crypt
+_rc4_init
diff --git a/config/Unsupported.exports b/config/Unsupported.exports
index 06b4312eb..ae2f63239 100644
--- a/config/Unsupported.exports
+++ b/config/Unsupported.exports
@@ -5,9 +5,10 @@ _KUNCGetNotificationID
 _KUNCUserNotificationDisplayAlert
 _KUNCUserNotificationDisplayFromBundle
 _KUNCUserNotificationDisplayNotice
-_MD5Final
-_MD5Init
-_MD5Update
+_NDR_record
+_PE_kputc
+_cons_ops
+__Z22OSFlushObjectTrackListv
 __ZN15IOWatchDogTimer10gMetaClassE
 __ZN15IOWatchDogTimer10superClassE
 __ZN15IOWatchDogTimer13setPropertiesEP8OSObject
@@ -33,12 +34,35 @@ __ZN16IOPlatformDevice9metaClassE
 __ZN16IOPlatformDeviceC2EPK11OSMetaClass
 __ZN16IOPlatformDeviceD2Ev
 __ZN18IODTPlatformExpert9metaClassE
+__ZN9IODTNVRAM10gMetaClassE
+__ZN9IODTNVRAM15initOFVariablesEv
+__ZN9IODTNVRAM15syncOFVariablesEv
+__ZN9IODTNVRAM16escapeDataToDataEP6OSData
+__ZN9IODTNVRAM16updateOWBootArgsEPK8OSSymbolP8OSObject
+__ZN9IODTNVRAM17getOWVariableInfoEmPPK8OSSymbolPmS4_
+__ZN9IODTNVRAM18generateOWChecksumEPh
+__ZN9IODTNVRAM18validateOWChecksumEPh
+__ZN9IODTNVRAM19convertObjectToPropEPhPmPK8OSSymbolP8OSObject
+__ZN9IODTNVRAM19convertPropToObjectEPhmS0_mPPK8OSSymbolPP8OSObject
+__ZN9IODTNVRAM19searchNVRAMPropertyEP17IONVRAMDescriptorPm
+__ZN9IODTNVRAM19unescapeBytesToDataEPKhm
+__ZN9IODTNVRAM22readNVRAMPropertyType0EP15IORegistryEntryPPK8OSSymbolPP6OSData
+__ZN9IODTNVRAM22readNVRAMPropertyType1EP15IORegistryEntryPPK8OSSymbolPP6OSData
+__ZN9IODTNVRAM23writeNVRAMPropertyType0EP15IORegistryEntryPK8OSSymbolP6OSData
+__ZN9IODTNVRAM23writeNVRAMPropertyType1EP15IORegistryEntryPK8OSSymbolP6OSData
+__ZN9IODTNVRAM26calculatePartitionChecksumEPh
+__ZN9IODTNVRAMC2EPK11OSMetaClass
+__ZN9IODTNVRAMD2Ev
 __ZNK15IOWatchDogTimer12getMetaClassEv
 __ZNK15IOWatchDogTimer9MetaClass5allocEv
+__ZNK9IODTNVRAM17getOFVariablePermEPK8OSSymbol
+__ZNK9IODTNVRAM17getOFVariableTypeEPK8OSSymbol
 __ZTV15IOWatchDogTimer
 __ZTV16IOPlatformDevice
+__ZTV9IODTNVRAM
 __ZTVN15IOWatchDogTimer9MetaClassE
 __doprnt
+__dtrace_register_anon_DOF
 _aes_decrypt_cbc
 _aes_decrypt_key
 _aes_decrypt_key128
@@ -48,14 +72,13 @@ _appleClut8
 _b_to_q
 _bdevsw
 _boot
-_bpf_mtap
+_bsd_set_dependency_capable
 _cdevsw
 _clalloc
 _clfree
 _clock_get_system_value
 _cons_cinput
 _conslog_putc
-_console_user
 _convert_port_entry_to_map
 _convert_port_entry_to_object
 _current_act
@@ -63,20 +86,11 @@ _delay
 _delay_for_interval
 _des_ecb_encrypt
 _des_set_key
-_dlil_attach_protocol
-_dlil_dereg_if_modules
-_dlil_dereg_proto_module
-_dlil_detach_protocol
-_dlil_if_acquire
-_dlil_if_attach
-_dlil_if_detach
-_dlil_if_release
-_dlil_input
-_dlil_output
-_dlil_reg_if_modules
-_dlil_reg_proto_module
 _domains
 _gIODTSharedInterrupts
+_gOSObjectTrackList
+_gOSObjectTrackThread
+_gPEClockFrequencyInfo
 _gPESerialBaud
 _get_aiotask
 _get_bsdtask_info
@@ -85,29 +99,30 @@ _getsectdatafromheader
 _hfs_getconverter
 _hfs_pickencoding
 _hfs_relconverter
+_host_get_special_port
+_host_priv_self
 _hz
-_ifbyfamily
 _ifunit
-_in6_cksum
 _in_broadcast
-_in_cksum
 _in_ifaddrhead
 _in_pcb_get_owner
 _in_pcb_grab_port
 _in_pcb_letgo_port
 _in_pcb_new_share_client
 _in_pcb_rem_share_client
+_inaddr_local
 _inet_domain_mutex
-_inflate
-_inflateEnd
-_inflateInit_
 _ip_mutex
 _ip_output
 _ip_protox
+_ipc_kernel_map
 _ipc_port_release_send
 _ipflow_fastforward
 _kalloc
+_kauth_cred_issuser
+_kauth_cred_label_update
 _kauth_guid_equal
+_kauth_proc_label_update
 _kdp_register_send_receive
 _kdp_set_interface
 _kdp_unregister_send_receive
@@ -127,36 +142,43 @@ _kmod_lookupbyname
 _kmputc
 _lbolt
 _lck_mtx_assert
-_lck_mtx_try_lock
 _lck_rw_done
 _linesw
 _lo_ifp
 _log
 _logwakeup
-_loif
-_m_adj
+_m_adj:_mbuf_adj
 _m_cat
 _m_copydata
 _m_copym
-_m_free
-_m_freem
+_m_free:_mbuf_free
+_m_freem:_mbuf_freem
 _m_get
 _m_gethdr
 _m_getpacket
 _m_getpackets
 _m_mclget
-_m_mtod
 _m_prepend_2
 _m_pullup
 _m_split
-_m_trailingspace
+_m_trailingspace:_mbuf_trailingspace
+_mach_gss_accept_sec_context
+_mach_gss_init_sec_context
 _mach_make_memory_entry_64
 _mach_memory_entry_page_op
 _mach_memory_entry_range_op
+_mach_msg_rpc_from_kernel
+_mach_msg_send_from_kernel_with_options
 _max_mem
 _mcl_to_paddr
 _mem_size
 _memory_object_page_op
+_mig_dealloc_reply_port
+_mig_get_reply_port
+_mig_put_reply_port
+_mig_strncpy
+_mig_user_allocate
+_mig_user_deallocate
 _ml_io_map
 _ml_phys_read
 _ml_phys_write
@@ -165,7 +187,6 @@ _ml_processor_register
 _ml_thread_policy
 _mountroot_post_hook
 _msleep1
-_nd6_storelladdr
 _net_add_domain
 _net_add_proto
 _net_del_domain
@@ -175,6 +196,7 @@ _ovbcopy
 _pffinddomain
 _pffindproto
 _pmap_find_phys
+_populate_model_name
 _prf
 _processor_exit
 _processor_info
@@ -211,14 +233,12 @@ _sbappendrecord
 _sbflush
 _sbspace
 _securelevel
-_sha1_init
-_sha1_loop
-_sha1_result
 _sleep
 _soabort
 _sobind
 _socantrcvmore
 _socantsendmore
+_sock_getlistener
 _sock_release
 _sock_retain
 _soclose
@@ -241,11 +261,10 @@ _sorwakeup
 _sosend
 _sosetopt
 _stack_privilege
+_task_get_special_port
 _task_resume
 _task_suspend
 _tcbinfo
-_temp_patch_ptrace
-_temp_unpatch_ptrace
 _termioschars
 _thread_call_func
 _thread_call_func_cancel
@@ -253,6 +272,7 @@ _thread_call_func_delayed
 _thread_call_is_delayed
 _thread_cancel_timer
 _thread_funnel_set
+_thread_notrigger
 _thread_set_timer
 _thread_set_timer_deadline
 _timeout
@@ -274,11 +294,17 @@ _uio_iovsaddr
 _uio_spacetype
 _unputc
 _untimeout
-_utf8_encodelen
+_vfs_context_current
+_vfs_context_get_special_port
+_vfs_context_set_special_port
+_vfs_setlocklocal
 _vfs_update_vfsstat
 _vm_allocate
 _vm_deallocate
 _vm_map
+_vm_map_copyin
+_vm_map_copyin_common
+_vm_map_copyout
 _vm_map_deallocate
 _vm_map_unwire
 _vm_map_wire
@@ -287,8 +313,12 @@ _vm_region
 _vm_region_object_create
 _vnode_getname
 _vnode_getparent
+_vnode_isnamedstream
 _vnode_putname
 _vnode_tag
 _vnode_update_identity
+_vnop_getnamedstream_desc
 _vnop_kqfilt_add_desc
 _vnop_kqfilt_remove_desc
+_vnop_makenamedstream_desc
+_vnop_removenamedstream_desc
diff --git a/config/Unsupported.i386.exports b/config/Unsupported.i386.exports
index a34b29222..1664fbf31 100644
--- a/config/Unsupported.i386.exports
+++ b/config/Unsupported.i386.exports
@@ -1,19 +1,32 @@
-_cpu_data_ptr
+_cpu_number
 _dsmos_page_transform_hook
 _gPEEFISystemTable
 _hpet_get_info
+_hpet_register_callback
+_hpet_request
+_in6addr_local
 _io_map_spec
+_kdp_register_callout
+_kdp_set_ip_and_mac_addresses
 _lapic_start
+_m_mtod
 _ml_get_apicid
 _ml_get_maxbusdelay
 _ml_get_maxsnoop
 _ml_hpet_cfg
+_ml_cpu_int_event_time
 _mp_rendezvous
-_pmRegister
+_mp_rendezvous_no_intrs
+_nd6_storelladdr
+_pmCPUControl
+_pmKextRegister
 _pm_init_lock
 _rdHPET
 _real_ncpus
 _rtc_clock_napped
+_serial_getc
+_serial_init
+_serial_putc
 _tmrCvt
 _tsc_get_info
 _hibernate_vm_lock
diff --git a/config/Unsupported.ppc.exports b/config/Unsupported.ppc.exports
index 350464074..24fffc53e 100644
--- a/config/Unsupported.ppc.exports
+++ b/config/Unsupported.ppc.exports
@@ -12,15 +12,18 @@ _gref_alloc
 _gref_close
 _gref_wput
 _ignore_zero_fault
+_in6addr_local
 _killprint
 _mapping_prealloc
 _mapping_relpre
+_m_mtod
 _ml_enable_cache_level
 _ml_enable_nap
 _ml_ppc_sleep
 _ml_set_processor_speed
 _ml_set_processor_voltage
 _ml_throttle
+_nd6_storelladdr
 _pmsStart
 _pmsPark
 _pmsRun
diff --git a/config/compress-man-pages.pl b/config/compress-man-pages.pl
new file mode 100755
index 000000000..7711919e0
--- /dev/null
+++ b/config/compress-man-pages.pl
@@ -0,0 +1,95 @@
+#!/usr/bin/perl
+# Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+#
+# @APPLE_LICENSE_HEADER_START@
+# 
+# This file contains Original Code and/or Modifications of Original Code
+# as defined in and that are subject to the Apple Public Source License
+# Version 2.0 (the 'License'). You may not use this file except in
+# compliance with the License. Please obtain a copy of the License at
+# http://www.opensource.apple.com/apsl/ and read it before using this
+# file.
+# 
+# The Original Code and all software distributed under the License are
+# distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+# EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+# INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+# Please see the License for the specific language governing rights and
+# limitations under the License.
+# 
+# @APPLE_LICENSE_HEADER_END@
+
+use strict;
+use File::Basename ();
+use File::Find ();
+use Getopt::Std ();
+
+my $MyName = File::Basename::basename($0);
+my $N = 100;
+my $MinSize = 64;
+my %inodes;
+my @symlinks;
+our $opt_d = '';
+
+sub wanted {
+    return unless /\.[\dn][a-z]*$/;
+    if(-l $_) {
+	push(@symlinks, $_);
+    } elsif(-f _) {
+	return if -s _ < $MinSize;
+	my($dev, $ino) = stat(_);
+	my $list = $inodes{$ino};
+	$list = $inodes{$ino} = [] unless defined($list);
+	push(@$list, $_);
+    }
+}
+
+sub usage {
+    die "Usage: $MyName [-d prefix] dir ...\n";
+}
+
+Getopt::Std::getopts('d:');
+usage() unless scalar(@ARGV) > 0;
+
+for my $dir (@ARGV) {
+    $dir = $opt_d . $dir if $opt_d ne '';
+    next unless -e $dir;
+    die "$dir: no such directory\n" unless -d _;
+
+    %inodes = ();
+    @symlinks = ();
+    File::Find::find({
+	wanted => \&wanted,
+	no_chdir => 1,
+    }, $dir);
+
+    my(@compress, @links);
+    for(values(%inodes)) {
+	push(@compress, $_->[0]);
+	push(@links, $_) if scalar(@$_) > 1;
+    }
+
+    my $count;
+    while(($count = scalar(@compress)) > 0) {
+	$_ = $count > $N ? $N : $count;
+	my @args = splice(@compress, 0, $_);
+	print "gzip -f @args\n";
+	system('gzip', '-f', @args) == 0 or die "gzip failed\n";;
+    }
+    foreach my $list (@links) {
+	my $main = shift(@$list);
+	for(@$list) {
+	    printf "rm $_; ln $main.gz $_.gz\n";
+	    unlink $_ or die "Can't unlink: $!\n";
+	    unlink "$_.gz";
+	    link("$main.gz", "$_.gz") or die "Can't link: $!\n";;
+	}
+    }
+    for(@symlinks) {
+	my $link = readlink($_);
+	printf "rm $_; ln -s $link.gz $_.gz\n";
+	unlink $_ or die "Can't unlink: $!\n";
+	symlink("$link.gz", "$_.gz") or die "Can't symlink: $!\n";
+    }
+}
diff --git a/config/newvers.pl b/config/newvers.pl
index 661737ee4..31deccace 100755
--- a/config/newvers.pl
+++ b/config/newvers.pl
@@ -42,6 +42,9 @@ sub WriteFile {
 my $versfile = "MasterVersion";
 $versfile = "$ENV{'SRCROOT'}/config/$versfile" if ($ENV{'SRCROOT'});
 my $BUILD_OBJROOT=$ENV{'OBJROOT'} . "/" . $ENV{'KERNEL_CONFIG'} . '_' . $ENV{'ARCH_CONFIG'};
+if($ENV{'MACHINE_CONFIG'} ne "DEFAULT") {
+    $BUILD_OBJROOT .= '_' . $ENV{'MACHINE_CONFIG'};
+}
 my $BUILD_DATE = `date`;
 $BUILD_DATE =~ s/[\n\t]//g;
 my $BUILDER=`whoami`;
diff --git a/config/version.c b/config/version.c
index 0811a8a29..d916cbee8 100644
--- a/config/version.c
+++ b/config/version.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* version.c
@@ -39,4 +45,4 @@ const char version_variant[] = VERSION_VARIANT;
 const char osbuilder[] = "###KERNEL_BUILDER###";
 const char osrelease[] = OSRELEASE;
 const char ostype[] = OSTYPE;
-
+char osversion[OSVERSIZE];
diff --git a/iokit/Drivers/platform/drvAppleMacIO/AppleMacIO.cpp b/iokit/Drivers/platform/drvAppleMacIO/AppleMacIO.cpp
index 693c43dbf..7231090c3 100644
--- a/iokit/Drivers/platform/drvAppleMacIO/AppleMacIO.cpp
+++ b/iokit/Drivers/platform/drvAppleMacIO/AppleMacIO.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp b/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
index c3e51b319..07da66d56 100644
--- a/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
+++ b/iokit/Drivers/platform/drvAppleNMI/AppleNMI.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-2003 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.cpp b/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.cpp
index 27b69da0e..805d034bd 100644
--- a/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.cpp
+++ b/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IOLib.h>
diff --git a/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.h b/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.h
index 63ee60261..2bfae06b7 100644
--- a/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.h
+++ b/iokit/Drivers/platform/drvAppleNVRAM/AppleNVRAM.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/nvram/IONVRAMController.h>
diff --git a/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.cpp b/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.cpp
index f76e6e8e9..3b00616fb 100644
--- a/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.cpp
+++ b/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.h b/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.h
index 7338ade5b..ee029d636 100644
--- a/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.h
+++ b/iokit/Drivers/platform/drvApplePlatformExpert/AppleCPU.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Drivers/platform/drvApplePlatformExpert/ApplePlatformExpert.cpp b/iokit/Drivers/platform/drvApplePlatformExpert/ApplePlatformExpert.cpp
index d197eba3c..029ef3056 100644
--- a/iokit/Drivers/platform/drvApplePlatformExpert/ApplePlatformExpert.cpp
+++ b/iokit/Drivers/platform/drvApplePlatformExpert/ApplePlatformExpert.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * HISTORY
diff --git a/iokit/Examples/drvGenericInterruptController/GenericInterruptController.cpp b/iokit/Examples/drvGenericInterruptController/GenericInterruptController.cpp
index 2f2271ca4..ff4950a87 100644
--- a/iokit/Examples/drvGenericInterruptController/GenericInterruptController.cpp
+++ b/iokit/Examples/drvGenericInterruptController/GenericInterruptController.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Examples/drvGenericInterruptController/GenericInterruptController.h b/iokit/Examples/drvGenericInterruptController/GenericInterruptController.h
index 7b9fce561..52ded55ec 100644
--- a/iokit/Examples/drvGenericInterruptController/GenericInterruptController.h
+++ b/iokit/Examples/drvGenericInterruptController/GenericInterruptController.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Families/IONVRAM/IONVRAMController.cpp b/iokit/Families/IONVRAM/IONVRAMController.cpp
index 78d7ff9f1..7a866c5af 100644
--- a/iokit/Families/IONVRAM/IONVRAMController.cpp
+++ b/iokit/Families/IONVRAM/IONVRAMController.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/nvram/IONVRAMController.h>
diff --git a/iokit/Families/IOSystemManagement/IOWatchDogTimer.cpp b/iokit/Families/IOSystemManagement/IOWatchDogTimer.cpp
index a7ea145a1..f0a274d9f 100644
--- a/iokit/Families/IOSystemManagement/IOWatchDogTimer.cpp
+++ b/iokit/Families/IOSystemManagement/IOWatchDogTimer.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IOUserClient.h>
diff --git a/iokit/IOKit/IOBSD.h b/iokit/IOKit/IOBSD.h
index 1961186b9..8938656ca 100644
--- a/iokit/IOKit/IOBSD.h
+++ b/iokit/IOKit/IOBSD.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOBSD_H
 #define _IOBSD_H
diff --git a/iokit/IOKit/IOBufferMemoryDescriptor.h b/iokit/IOKit/IOBufferMemoryDescriptor.h
index f976759aa..f5922d5a5 100644
--- a/iokit/IOKit/IOBufferMemoryDescriptor.h
+++ b/iokit/IOKit/IOBufferMemoryDescriptor.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOBUFFERMEMORYDESCRIPTOR_H
 #define _IOBUFFERMEMORYDESCRIPTOR_H
@@ -43,12 +49,12 @@ class IOBufferMemoryDescriptor : public IOGeneralMemoryDescriptor
 {
     OSDeclareDefaultStructors(IOBufferMemoryDescriptor);
 
-protected:
+private:
 /*! @struct ExpansionData
     @discussion This structure will be used to expand the capablilties of this class in the future.
     */    
     struct ExpansionData {
-	vm_map_t	map;
+	IOMemoryMap *   map;
     };
 
 /*! @var reserved
@@ -69,19 +75,16 @@ private:
                                vm_size_t    capacity,
                                vm_offset_t  alignment,
 			       task_t	    inTask);
-    OSMetaClassDeclareReservedUsed(IOBufferMemoryDescriptor, 0);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual bool initWithPhysicalMask(
 				task_t		  inTask,
 				IOOptionBits      options,
 				mach_vm_size_t    capacity,
 				mach_vm_address_t alignment,
 				mach_vm_address_t physicalMask);
+
+    OSMetaClassDeclareReservedUsed(IOBufferMemoryDescriptor, 0);
     OSMetaClassDeclareReservedUsed(IOBufferMemoryDescriptor, 1);
-#else
-    OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 1);
-#endif
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 2);
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 3);
     OSMetaClassDeclareReservedUnused(IOBufferMemoryDescriptor, 4);
@@ -169,7 +172,6 @@ public:
                                             vm_size_t    capacity,
                                             vm_offset_t  alignment = 1);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
 /*! @function inTaskWithPhysicalMask
     @abstract Creates a memory buffer with memory descriptor for that buffer. 
     @discussion Added in Mac OS X 10.5, this method allocates a memory buffer with a given size and alignment in the task's address space specified, and returns a memory descriptor instance representing the memory. It is recommended that memory allocated for I/O or sharing via mapping be created via IOBufferMemoryDescriptor. Options passed with the request specify the kind of memory to be allocated - pageablity and sharing are specified with option bits. This function may block and so should not be called from interrupt level or while a simple lock is held.
@@ -187,7 +189,6 @@ public:
                                             IOOptionBits      options,
                                             mach_vm_size_t    capacity,
                                             mach_vm_address_t physicalMask);
-#endif
 
     /*
      * withCapacity:
@@ -274,10 +275,8 @@ public:
      */
     virtual bool appendBytes(const void *bytes, vm_size_t withLength);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     /* DEPRECATED */ virtual void * getVirtualSegment(IOByteCount offset,
     /* DEPRECATED */					IOByteCount * length);
-#endif
 };
 
 #endif /* !_IOBUFFERMEMORYDESCRIPTOR_H */
diff --git a/iokit/IOKit/IOCPU.h b/iokit/IOKit/IOCPU.h
index d16a3e5c1..1140be62c 100644
--- a/iokit/IOKit/IOCPU.h
+++ b/iokit/IOKit/IOCPU.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
@@ -104,7 +110,8 @@ public:
 };
 
 void IOCPUSleepKernel(void);
-
+extern "C" kern_return_t IOCPURunPlatformQuiesceActions(void);
+extern "C" kern_return_t IOCPURunPlatformActiveActions(void);
 
 class IOCPUInterruptController : public IOInterruptController
 {
diff --git a/iokit/IOKit/IOCatalogue.h b/iokit/IOKit/IOCatalogue.h
index 4f1bfa07c..f666bf426 100644
--- a/iokit/IOKit/IOCatalogue.h
+++ b/iokit/IOKit/IOCatalogue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IOCommand.h b/iokit/IOKit/IOCommand.h
index 860c44f03..136c41bd5 100644
--- a/iokit/IOKit/IOCommand.h
+++ b/iokit/IOKit/IOCommand.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/iokit/IOKit/IOCommandGate.h b/iokit/IOKit/IOCommandGate.h
index 5ac54a891..21a0dceb3 100644
--- a/iokit/IOKit/IOCommandGate.h
+++ b/iokit/IOKit/IOCommandGate.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*[
     1999-8-10	Godfrey van der Linden(gvdl)
@@ -110,11 +116,9 @@ compiler warning.  Defaults to zero, see $link IOEventSource::setAction.
     @result True if inherited classes initialise successfully. */
     virtual bool init(OSObject *owner, Action action = 0);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     // Superclass overrides
     virtual void free();
     virtual void setWorkLoop(IOWorkLoop *inWorkLoop);
-#endif
 
 /*! @function runCommand
     @abstract Single thread a command with the target work-loop.
@@ -198,7 +202,6 @@ client's thread attemptCommand will fail if the work-loop's gate is closed.
     @param onlyOneThread true to only wake up at most one thread, false otherwise. */
     virtual void commandWakeup(void *event, bool oneThread = false);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
 /*! @function disable
     @abstract Disable the command gate
     @discussion When a command gate is disabled all future calls to runAction and runCommand will stall until the gate is enable()d later.  This can be used to block client threads when a system sleep is requested.  The IOWorkLoop thread itself will never stall, even when making runAction/runCommand calls.  This call must be made from a gated context, to clear potential race conditions.  */
@@ -208,7 +211,6 @@ client's thread attemptCommand will fail if the work-loop's gate is closed.
     @abstract Enable command gate, this will unblock any blocked Commands and Actions.
     @discussion  Enable the command gate.  The attemptAction/attemptCommand calls will now be enabled and can succeeed.  Stalled runCommand/runAction calls will be woken up. */
     virtual void enable();
-#endif
 
 private:
     OSMetaClassDeclareReservedUnused(IOCommandGate, 0);
diff --git a/iokit/IOKit/IOCommandPool.h b/iokit/IOKit/IOCommandPool.h
index 811664751..91069f3d3 100644
--- a/iokit/IOKit/IOCommandPool.h
+++ b/iokit/IOKit/IOCommandPool.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/iokit/IOKit/IOCommandQueue.h b/iokit/IOKit/IOCommandQueue.h
index 8b7f3dfdf..5337cdfa5 100644
--- a/iokit/IOKit/IOCommandQueue.h
+++ b/iokit/IOKit/IOCommandQueue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
@@ -36,13 +42,7 @@ HISTORY
 #include <IOKit/IOEventSource.h>
 
 
-
-
-#warning IOCommandQueue has been deprecated in favour of IOCommandGate and will be going away before too long.
-
-
-
-
+#define DEPRECATED __attribute__((deprecated))
 
 class IOCommandQueue;
 
@@ -69,14 +69,17 @@ protected:
 public:
     static IOCommandQueue *commandQueue(OSObject *inOwner,
                                         IOCommandQueueAction inAction = 0,
-                                        int inSize = kIOCQDefaultSize);
+                                        int inSize = kIOCQDefaultSize) 
+	DEPRECATED;
     virtual bool init(OSObject *inOwner,
                       IOCommandQueueAction inAction = 0,
-                      int inSize = kIOCQDefaultSize);
+                      int inSize = kIOCQDefaultSize)
+	DEPRECATED;
 
     virtual kern_return_t enqueueCommand(bool gotoSleep = true,
                                          void *field0 = 0, void *field1 = 0,
-                                         void *field2 = 0, void *field3 = 0);
+                                         void *field2 = 0, void *field3 = 0)
+	DEPRECATED;
 
     // WARNING:  This function can only be safely called from the appropriate
     // work loop context.  You should check IOWorkLoop::onThread is true.
@@ -85,7 +88,8 @@ public:
     // Lockout all new entries to the queue while iterating.
     // If the input fields are zero then the queue's owner/action will be used.
     virtual int performAndFlush(OSObject *target = 0,
-                                IOCommandQueueAction inAction = 0);
+                                IOCommandQueueAction inAction = 0)
+	DEPRECATED;
 };
 
 #endif /* !_IOKIT_IOCOMMANDQUEUE_H */
diff --git a/iokit/IOKit/IOConditionLock.h b/iokit/IOKit/IOConditionLock.h
index 2bfef6f18..f628feebc 100644
--- a/iokit/IOKit/IOConditionLock.h
+++ b/iokit/IOKit/IOConditionLock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
  * Copyright (c) 1994-1996 NeXT Software, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IODMACommand.h b/iokit/IOKit/IODMACommand.h
index 02eed40ed..0365cc0ce 100644
--- a/iokit/IOKit/IODMACommand.h
+++ b/iokit/IOKit/IODMACommand.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IODMACOMMAND_H
 #define _IODMACOMMAND_H
@@ -44,6 +50,8 @@ class IODMACommand : public IOCommand
 {
     OSDeclareDefaultStructors(IODMACommand);
 
+friend class IODMAEventSource;
+
 public:
 
 /*!
@@ -313,6 +321,31 @@ public:
 				     void   *segments,
 				     UInt32 *numSegments);
 
+private:
+    virtual UInt64 transfer( IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length );
+
+public:
+
+/*! @function writeBytes
+    @abstract Copy data to the IODMACommand's buffer from the specified buffer.
+    @discussion This method copies data to the IODMACommand's memory at the given offset, from the caller's buffer. The IODMACommand must be prepared, and the offset is relative to the prepared offset.
+    @param offset A byte offset into the IODMACommand's memory, relative to the prepared offset.
+    @param bytes The caller supplied buffer to copy the data from.
+    @param length The length of the data to copy.
+    @result The number of bytes copied, zero will be returned if the specified offset is beyond the prepared length of the IODMACommand. */
+
+    UInt64 writeBytes(UInt64 offset, const void *bytes, UInt64 length);
+
+/*! @function readBytes
+    @abstract Copy data from the IODMACommand's buffer to the specified buffer.
+    @discussion This method copies data from the IODMACommand's memory at the given offset, to the caller's buffer. The IODMACommand must be prepared, and the offset is relative to the prepared offset.
+    @param offset A byte offset into the IODMACommand's memory, relative to the prepared offset.
+    @param bytes The caller supplied buffer to copy the data to.
+    @param length The length of the data to copy.
+    @result The number of bytes copied, zero will be returned if the specified offset is beyond the prepared length of the IODMACommand. */
+
+    UInt64 readBytes(UInt64 offset, void *bytes, UInt64 length);
+
 /*! @function gen32IOVMSegments
     @abstract Helper function for a type checked call to genIOVMSegments(qv), for use with an IODMACommand set up with the output function kIODMACommandOutputHost32, kIODMACommandOutputBig32, or kIODMACommandOutputLittle32. If the output function of the IODMACommand is not a 32 bit function, results will be incorrect.
 */
@@ -332,7 +365,6 @@ public:
     virtual void free();
 
 private:
-
     typedef IOReturn (*InternalSegmentFunction)(
 				    void         *reference,
 				    IODMACommand *target,
@@ -358,9 +390,47 @@ private:
 			UInt32        segmentIndex);
     IOReturn IODMACommand::walkAll(UInt8 op);
 
+public:
+
+/*! @function prepareWithSpecification
+    @abstract Prepare the memory for an I/O transfer with a new specification.
+    @discussion Allocate the mapping resources neccessary for this transfer, specifying a sub range of the IOMemoryDescriptor that will be the target of the I/O.  The complete() method frees these resources.  Data may be copied to buffers for kIODirectionOut memory descriptors, depending on hardware mapping resource availabilty or alignment restrictions.  It should be noted that the this function may block and should only be called on the clients context, i.e never call this routine while gated; also the call itself is not thread safe though this should be an issue as each IODMACommand is independant.
+    @param outSegFunc SegmentFunction to call to output one physical segment. A set of nine commonly required segment functions are provided.
+    @param numAddressBits Number of bits that the hardware uses on its internal address bus. Typically 32 but may be more on modern hardware.  A 0 implies no-restriction other than that implied by the output segment function.
+    @param maxSegmentSize Maximum allowable size for one segment.  Defaults to 0 which means any size.
+    @param mappingOptions is the type of mapping that is required to translate an IOMemoryDescriptor into the desired number of bits.  For instance if your hardware only supports 32 bits but must run on machines with > 4G of RAM some mapping will be required.  Number of bits will be specified in numAddressBits, see below.This parameter can take 3 values:- kNonCoherent - used for non-coherent hardware transfers, Mapped - Validate that all I/O bus generated addresses are within the number of addressing bits specified, Bypassed indicates that bypassed addressing is required, this is used when the hardware transferes are into coherent memory but no mapping is required.  See also prepare() for failure cases.
+    @param maxTransferSize Maximum size of an entire transfer.	Defaults to 0 indicating no maximum.
+    @param alignment Alignment restriction, in bytes, on I/O bus addresses.  Defaults to single byte alignment.
+    @param mapper For mapping types kMapped & kBypassed mapper is used to define the hardware that will perform the mapping, defaults to the system mapper.
+    @param offset defines the starting offset in the memory descriptor the DMA command will operate on. genIOVMSegments will produce its results based on the offset and length passed to the prepare method.
+    @param length defines the ending position in the memory descriptor the DMA command will operate on. genIOVMSegments will produce its results based on the offset and length passed to the prepare method.
+    @param flushCache Flush the caches for the memory descriptor and make certain that the memory cycles are complete.  Defaults to true for kNonCoherent and is ignored by the other types.
+    @param synchronize Copy any buffered data back from the target IOMemoryDescriptor.  Defaults to true, if synchronize() is being used to explicitly copy data, passing false may avoid an unneeded copy.
+    @result An IOReturn code. Can fail if the mapping type is not recognised, if one of the 3 mandatory parameters are set to 0, if a 32 bit output function is selected when more than 32 bits of address is required or, if kBypassed is requested on a machine that doesn't support bypassing.
+*/
+
+    virtual IOReturn prepareWithSpecification(SegmentFunction	outSegFunc,
+					      UInt8		numAddressBits,
+					      UInt64		maxSegmentSize,
+					      MappingOptions	mappingOptions = kMapped,
+					      UInt64		maxTransferSize = 0,
+					      UInt32		alignment = 1,
+					      IOMapper		*mapper = 0,
+					      UInt64		offset = 0,
+					      UInt64		length = 0,
+					      bool		flushCache = true,
+					      bool		synchronize = true);
+
+    static IOReturn transferSegment(
+			void         *reference,
+			IODMACommand *target,
+			Segment64     segment,
+			void         *segments,
+			UInt32        segmentIndex);
+
 private:
-    OSMetaClassDeclareReservedUnused(IODMACommand,  0);
-    OSMetaClassDeclareReservedUnused(IODMACommand,  1);
+    OSMetaClassDeclareReservedUsed(IODMACommand,  0);
+    OSMetaClassDeclareReservedUsed(IODMACommand,  1);
     OSMetaClassDeclareReservedUnused(IODMACommand,  2);
     OSMetaClassDeclareReservedUnused(IODMACommand,  3);
     OSMetaClassDeclareReservedUnused(IODMACommand,  4);
@@ -426,7 +496,7 @@ protected:
 
 /*! @var reserved
     Reserved for future use.  (Internal use only)  */
-    struct ExpansionData * reserved;
+    struct IODMACommandInternal * reserved;
 };
 
 IOReturn IODMACommand::
diff --git a/iokit/IOKit/IODMAController.h b/iokit/IOKit/IODMAController.h
new file mode 100644
index 000000000..1e6632fbb
--- /dev/null
+++ b/iokit/IOKit/IODMAController.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOKIT_IODMACONTROLLER_H
+#define _IOKIT_IODMACONTROLLER_H
+
+#include <IOKit/IODMACommand.h>
+#include <IOKit/IODMAEventSource.h>
+#include <IOKit/IOService.h>
+
+class IODMAController : public IOService
+{
+  OSDeclareAbstractStructors(IODMAController);
+  
+  friend class IODMAEventSource;
+  
+ private:
+  IOService       *_provider;
+  const OSSymbol  *_dmaControllerName;
+  
+ protected:
+  virtual void registerDMAController(IOOptionBits options = 0);  
+  virtual IOReturn initDMAChannel(IOService *provider, IODMAEventSource *dmaES, UInt32 *dmaIndex, UInt32 reqIndex) = 0;
+  virtual IOReturn startDMACommand(UInt32 dmaIndex, IODMACommand *dmaCommand, IODirection direction,
+				   IOByteCount byteCount = 0, IOByteCount byteOffset = 0) = 0;
+  virtual IOReturn stopDMACommand(UInt32 dmaIndex, bool flush = false, mach_timespec_t * timeout = 0) = 0;
+  virtual void completeDMACommand(IODMAEventSource *dmaES, IODMACommand *dmaCommand);
+  virtual void notifyDMACommand(IODMAEventSource *dmaES, IODMACommand *dmaCommand, IOReturn status, IOByteCount actualByteCount);
+  virtual IOReturn queryDMACommand(UInt32 dmaIndex, IODMACommand **dmaCommand, IOByteCount *transferCount, bool waitForIdle = false) = 0;
+  
+ public:
+  static const OSSymbol *createControllerName(UInt32 phandle);
+  static IODMAController *getController(IOService *provider, UInt32 dmaIndex);
+  
+  virtual bool start(IOService *provider);
+};
+
+
+#endif /* _IOKIT_IODMACONTROLLER_H */
diff --git a/iokit/IOKit/IODMAEventSource.h b/iokit/IOKit/IODMAEventSource.h
new file mode 100644
index 000000000..ce68f6f86
--- /dev/null
+++ b/iokit/IOKit/IODMAEventSource.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOKIT_IODMAEVENTSOURCE_H
+#define _IOKIT_IODMAEVENTSOURCE_H
+
+#include <IOKit/IOService.h>
+#include <IOKit/IODMACommand.h>
+#include <IOKit/IODMAController.h>
+#include <IOKit/IOEventSource.h>
+
+class IODMAEventSource : public IOEventSource
+{
+  OSDeclareDefaultStructors(IODMAEventSource);
+  
+  friend class IODMAController;
+  
+ public:
+  typedef void (*Action)(OSObject *owner, IODMAEventSource *dmaES, IODMACommand *dmaCommand, IOReturn status, IOByteCount actualByteCount);
+#define IODMAEventAction IODMAEventSource::Action
+  
+ protected:
+  virtual void completeDMACommand(IODMACommand *dmaCommand);
+  virtual void notifyDMACommand(IODMACommand *dmaCommand, IOReturn status, IOByteCount actualByteCount);
+  
+ public:
+  static IODMAEventSource *dmaEventSource(OSObject *owner,
+					  IOService *provider,
+					  Action completion = 0,
+					  Action notification = 0,
+					  UInt32 dmaIndex = 0);
+  
+  virtual IOReturn startDMACommand(IODMACommand *dmaCommand, IODirection direction, IOByteCount byteCount = 0, IOByteCount byteOffset = 0);
+  virtual IOReturn stopDMACommand(bool flush = false, mach_timespec_t *timeout = 0);
+  
+  virtual IOReturn queryDMACommand(IODMACommand **dmaCommand, IOByteCount *transferCount, bool waitForIdle = false);
+  
+ private:
+  IOService       *dmaProvider;
+  IODMAController *dmaController;
+  UInt32          dmaIndex;
+  queue_head_t    dmaCommandsCompleted;
+  IOSimpleLock	  *dmaCommandsCompletedLock;
+  Action          dmaCompletionAction;
+  Action          dmaNotificationAction;
+  bool            dmaSynchBusy;
+  
+  virtual bool init(OSObject *owner,
+		    IOService *provider,
+		    Action completion = 0,
+		    Action notification = 0,
+		    UInt32 dmaIndex = 0);
+  virtual bool checkForWork(void);
+};
+
+#endif /* _IOKIT_IODMAEVENTSOURCE_H */
diff --git a/iokit/IOKit/IODataQueue.h b/iokit/IOKit/IODataQueue.h
index 6445ba5e8..458cb478a 100644
--- a/iokit/IOKit/IODataQueue.h
+++ b/iokit/IOKit/IODataQueue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IODATAQUEUE_H
diff --git a/iokit/IOKit/IODataQueueShared.h b/iokit/IOKit/IODataQueueShared.h
index 6e853785e..2fa0e9a45 100644
--- a/iokit/IOKit/IODataQueueShared.h
+++ b/iokit/IOKit/IODataQueueShared.h
@@ -1,29 +1,37 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IODATAQUEUESHARED_H
 #define _IOKIT_IODATAQUEUESHARED_H
 
 #include <libkern/OSTypes.h>
+#include <mach/port.h>
+#include <mach/message.h>
 
 /*!
  * @typedef IODataQueueEntry
@@ -34,7 +42,7 @@
  */
 typedef struct _IODataQueueEntry{
     UInt32  size;
-    void *  data;
+    UInt8   data[4];
 } IODataQueueEntry;
 
 /*!
@@ -53,15 +61,32 @@ typedef struct _IODataQueueMemory {
     IODataQueueEntry  queue[1];
 } IODataQueueMemory;
 
+/*!
+ * @typedef IODataQueueAppendix
+ * @abstract A struct mapping to the appendix region of a data queue.
+ * @discussion This struct is variable sized dependent on the version.  The struct represents the data queue appendix information.
+ * @field version The version of the queue appendix.
+ * @field port The notification port associated with this queue.
+ */
+typedef struct _IODataQueueAppendix {
+    UInt32            version;
+    mach_msg_header_t msgh;
+} IODataQueueAppendix;
+
 /*!
  * @defined DATA_QUEUE_ENTRY_HEADER_SIZE Represents the size of the data queue entry header independent of the actual size of the data in the entry.  This is the overhead of each entry in the queue.  The total size of an entry is equal to this value plus the size stored in the entry's size field (in IODataQueueEntry).
  */
-#define DATA_QUEUE_ENTRY_HEADER_SIZE (sizeof(IODataQueueEntry) - sizeof(void *))
+#define DATA_QUEUE_ENTRY_HEADER_SIZE (sizeof(IODataQueueEntry) - 4)
 
 /*!
- * @defined DATA_QUEUE_MEMORY_HEADER_SIZE Represents the size of the data queue memory header independent of the actual size of the queue data itself.  The total size of the queue memory is equal to this value plus the size of the queue data region which is stored in the queueSize field of IODataQueueMeory.
+ * @defined DATA_QUEUE_MEMORY_HEADER_SIZE Represents the size of the data queue memory header independent of the actual size of the queue data itself.  The total size of the queue memory is equal to this value plus the size of the queue appendix and the size of the queue data region which is stored in the queueSize field of IODataQueueMeory.
  */
 #define DATA_QUEUE_MEMORY_HEADER_SIZE (sizeof(IODataQueueMemory) - sizeof(IODataQueueEntry))
 
+/*!
+ * @defined DATA_QUEUE_MEMORY_APPENDIX_SIZE Represents the size of the data queue memory appendix independent of the actual size of the queue data itself.  The total size of the queue memory is equal to this value plus the size of queue header and size of the queue data region which is stored in the queueSize field of IODataQueueMeory.
+ */
+#define DATA_QUEUE_MEMORY_APPENDIX_SIZE (sizeof(IODataQueueAppendix))
+
 #endif /* _IOKIT_IODATAQUEUESHARED_H */
 
diff --git a/iokit/IOKit/IODeviceMemory.h b/iokit/IOKit/IODeviceMemory.h
index 94b50cb00..0665efc3c 100644
--- a/iokit/IOKit/IODeviceMemory.h
+++ b/iokit/IOKit/IODeviceMemory.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IODeviceTreeSupport.h b/iokit/IOKit/IODeviceTreeSupport.h
index 46e3da7b6..eacf339bc 100644
--- a/iokit/IOKit/IODeviceTreeSupport.h
+++ b/iokit/IOKit/IODeviceTreeSupport.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -37,6 +43,8 @@ class IOService;
 
 extern const IORegistryPlane *	gIODTPlane;
 
+extern const OSSymbol *		gIODTPHandleKey;
+
 extern const OSSymbol *		gIODTCompatibleKey;
 extern const OSSymbol * 	gIODTTypeKey;
 extern const OSSymbol * 	gIODTModelKey;
diff --git a/iokit/IOKit/IOEventSource.h b/iokit/IOKit/IOEventSource.h
index f9868a69f..ae6060b95 100644
--- a/iokit/IOKit/IOEventSource.h
+++ b/iokit/IOKit/IOEventSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.	 All rights reserved.
diff --git a/iokit/IOKit/IOFilterInterruptEventSource.h b/iokit/IOKit/IOFilterInterruptEventSource.h
index 0dad3b6a4..de05c90d5 100644
--- a/iokit/IOKit/IOFilterInterruptEventSource.h
+++ b/iokit/IOKit/IOFilterInterruptEventSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1999 Apple Computer, Inc.	 All rights reserved.
diff --git a/iokit/IOKit/IOHibernatePrivate.h b/iokit/IOKit/IOHibernatePrivate.h
index e6046f5c1..ab897a850 100644
--- a/iokit/IOKit/IOHibernatePrivate.h
+++ b/iokit/IOKit/IOHibernatePrivate.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -110,27 +116,23 @@ struct hibernate_page_list_t
 };
 typedef struct hibernate_page_list_t hibernate_page_list_t;
 
+#if defined(_AES_H)
+
 struct hibernate_cryptwakevars_t
 {
-#ifdef _AES_H
     uint8_t aes_iv[AES_BLOCK_SIZE];
-#else
-#warning undef _AES_H
-#endif
 };
 typedef struct hibernate_cryptwakevars_t hibernate_cryptwakevars_t;
 
 struct hibernate_cryptvars_t
 {
-#ifdef _AES_H
     uint8_t aes_iv[AES_BLOCK_SIZE];
     aes_ctx ctx;
-#else
-#warning undef _AES_H
-#endif
 };
 typedef struct hibernate_cryptvars_t hibernate_cryptvars_t;
 
+#endif /* defined(_AES_H) */
+
 
 enum 
 {
@@ -147,6 +149,15 @@ enum
     kIOHibernateProgressDarkGray      = 92
 };
 
+enum
+{
+    kIOHibernatePostWriteSleep   = 0,
+    kIOHibernatePostWriteWake    = 1,
+    kIOHibernatePostWriteHalt    = 2,
+    kIOHibernatePostWriteRestart = 3
+};
+
+
 struct hibernate_graphics_t
 {
     uint32_t physicalAddress;		// Base address of video memory
@@ -209,7 +220,7 @@ void
 kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref);
 int
 kern_write_file(struct kern_direct_file_io_ref_t * ref, off_t offset, caddr_t addr, vm_size_t len);
-int get_kernel_symfile(struct proc *p, char **symfile);
+int get_kernel_symfile(struct proc *p, char const **symfile);
 #endif /* _SYS_CONF_H_ */
 
 hibernate_page_list_t *
@@ -277,7 +288,8 @@ hibernate_restore_phys_page(uint64_t src, uint64_t dst, uint32_t len, uint32_t p
 
 void
 hibernate_machine_init(void);
-boolean_t
+
+uint32_t
 hibernate_write_image(void);
 
 long
@@ -320,9 +332,10 @@ enum
     kIOHibernateModeOn      = 0x00000001,
     kIOHibernateModeSleep   = 0x00000002,
     kIOHibernateModeEncrypt = 0x00000004,
-
     kIOHibernateModeDiscardCleanInactive = 0x00000008,
-    kIOHibernateModeDiscardCleanActive   = 0x00000010
+    kIOHibernateModeDiscardCleanActive   = 0x00000010,
+    kIOHibernateModeSwitch	= 0x00000020,
+    kIOHibernateModeRestart	= 0x00000040
 };
 
 // IOHibernateImageHeader.signature
@@ -349,6 +362,13 @@ enum
 #define kIOHibernateFeatureKey		"Hibernation"
 #define kIOHibernatePreviewBufferKey	"IOPreviewBuffer"
 
+#define kIOHibernatePreviewActiveKey	"IOHibernatePreviewActive"
+// values for kIOHibernatePreviewActiveKey
+enum {
+    kIOHibernatePreviewActive  = 0x00000001,
+    kIOHibernatePreviewUpdates = 0x00000002
+};
+
 #define kIOHibernateBootImageKey	"boot-image"
 #define kIOHibernateBootImageKeyKey	"boot-image-key"
 #define kIOHibernateBootSignatureKey	"boot-signature"
@@ -359,6 +379,9 @@ enum
 
 #define kIOHibernateRTCVariablesKey	"IOHibernateRTCVariables"
 
+#define kIOHibernateBootSwitchVarsKey			"boot-switch-vars"
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/iokit/IOKit/IOInterleavedMemoryDescriptor.h b/iokit/IOKit/IOInterleavedMemoryDescriptor.h
new file mode 100644
index 000000000..960dded40
--- /dev/null
+++ b/iokit/IOKit/IOInterleavedMemoryDescriptor.h
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOINTERLEAVEDMEMORYDESCRIPTOR_H
+#define _IOINTERLEAVEDMEMORYDESCRIPTOR_H
+
+#include <IOKit/IOMemoryDescriptor.h>
+
+/*! @class IOInterleavedMemoryDescriptor : public IOMemoryDescriptor
+    @abstract The IOInterleavedMemoryDescriptor object describes a memory area made up of portions of several other IOMemoryDescriptors.
+    @discussion The IOInterleavedMemoryDescriptor object represents interleaved ranges of memory, specified as an ordered list of portions of individual IOMemoryDescriptors.  The portions are chained end-to-end to make up a single contiguous buffer. */
+
+class IOInterleavedMemoryDescriptor : public IOMemoryDescriptor
+{
+    OSDeclareDefaultStructors(IOInterleavedMemoryDescriptor);
+
+protected:
+
+    UInt32                _descriptorCapacity;
+    UInt32                _descriptorCount;
+    IOMemoryDescriptor ** _descriptors;
+    IOByteCount         * _descriptorOffsets;
+    IOByteCount         * _descriptorLengths;
+    bool                  _descriptorPrepared;
+
+    virtual void free();
+
+    /*
+     * These methods are not supported under this subclass.
+     */
+
+    virtual bool initWithAddress( void *      address,       /* not supported */
+                                  IOByteCount withLength,
+                                  IODirection withDirection );
+
+    virtual bool initWithAddress( vm_address_t address,      /* not supported */
+                                  IOByteCount  withLength,
+                                  IODirection  withDirection,
+                                  task_t       withTask );
+
+    virtual bool initWithPhysicalAddress( 
+                                  IOPhysicalAddress address, /* not supported */
+                                  IOByteCount       withLength,
+                                  IODirection       withDirection );
+
+    virtual bool initWithPhysicalRanges( 
+                                  IOPhysicalRange * ranges,  /* not supported */
+                                  UInt32            withCount,
+                                  IODirection       withDirection,
+                                  bool              asReference = false );
+
+    virtual bool initWithRanges(  IOVirtualRange * ranges,   /* not supported */
+                                  UInt32           withCount,
+                                  IODirection      withDirection,
+                                  task_t           withTask,
+                                  bool             asReference = false );
+
+    virtual void * getVirtualSegment( IOByteCount   offset,  /* not supported */
+                                      IOByteCount * length );
+
+    IOMemoryDescriptor::withAddress;                         /* not supported */
+    IOMemoryDescriptor::withPhysicalAddress;                 /* not supported */
+    IOMemoryDescriptor::withPhysicalRanges;                  /* not supported */
+    IOMemoryDescriptor::withRanges;                          /* not supported */
+    IOMemoryDescriptor::withSubRange;                        /* not supported */
+
+public:
+
+/*! @function withCapacity
+    @abstract Create an IOInterleavedMemoryDescriptor to describe a memory area made up of several other IOMemoryDescriptors.
+    @discussion This method creates and initializes an IOInterleavedMemoryDescriptor for memory consisting of portions of a number of other IOMemoryDescriptors, chained end-to-end (in the order they appear in the array) to represent a single contiguous memory buffer.
+    @param capacity The maximum number of IOMemoryDescriptors that may be subsequently added to this IOInterleavedMemoryDescriptor.
+    @param direction An I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures.
+    @result The created IOInterleavedMemoryDescriptor on success, to be released by the caller, or zero on failure. */
+
+    static IOInterleavedMemoryDescriptor * withCapacity( UInt32		       capacity,
+                                                         IODirection           direction);
+
+/*! @function initWithCapacity
+    @abstract Initialize an IOInterleavedMemoryDescriptor to describe a memory area made up of several other IOMemoryDescriptors.
+    @discussion This method initializes an IOInterleavedMemoryDescriptor for memory consisting of portions of a number of other IOMemoryDescriptors, chained end-to-end (in the order they appear in the array) to represent a single contiguous memory buffer.
+    @param capacity The maximum number of IOMemoryDescriptors that may be subsequently added to this IOInterleavedMemoryDescriptor.
+    @param direction An I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures.
+    @result The created IOInterleavedMemoryDescriptor on success, to be released by the caller, or zero on failure. */
+
+    virtual bool initWithCapacity( UInt32                capacity,
+                                   IODirection           direction );
+
+/*! @function clearMemoryDescriptors
+    @abstract Clear all of the IOMemoryDescriptors currently contained in and reset the IOInterleavedMemoryDescriptor.
+    @discussion Clears each IOMemoryDescriptor by completing (if needed) and releasing.  The IOInterleavedMemoryDescriptor is then reset and may accept new descriptors up to the capacity specified when it was created.
+    @param direction An I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. */
+
+    virtual void clearMemoryDescriptors( IODirection direction = kIODirectionNone );
+
+/*! @function setMemoryDescriptor
+    @abstract Add a portion of an IOMemoryDescriptor to the IOInterleavedMemoryDescriptor.
+    @discussion This method adds the portion of an IOMemoryDescriptor described by the offset and length parameters to the end of the IOInterleavedMemoryDescriptor.  A single IOMemoryDescriptor may be added as many times as there is room for it.  The offset and length must describe a portion entirely within the IOMemoryDescriptor.
+    @param descriptor An IOMemoryDescriptor to be added to the IOInterleavedMemoryDescriptor.  Its direction must be compatible with that of the IOInterleavedMemoryDescriptor.
+    @param offset The offset into the IOMemoryDescriptor of the portion that will be added to the virtualized buffer.
+    @param length The length of the portion of the IOMemoryDescriptor to be added to the virtualized buffer. 
+    @result Returns true the portion was successfully added. */
+
+    virtual bool setMemoryDescriptor( IOMemoryDescriptor * descriptor,
+                                      IOByteCount offset,
+                                      IOByteCount length );
+
+/*! @function getPhysicalSegment
+    @abstract Break a memory descriptor into its physically contiguous segments.
+    @discussion This method returns the physical address of the byte at the given offset into the memory, and optionally the length of the physically contiguous segment from that offset.
+    @param offset A byte offset into the memory whose physical address to return.
+    @param length If non-zero, getPhysicalSegment will store here the length of the physically contiguous segement at the given offset.
+    @result A physical address, or zero if the offset is beyond the length of the memory. */
+
+    virtual IOPhysicalAddress getPhysicalSegment( IOByteCount   offset,
+                                                  IOByteCount * length );
+
+    virtual addr64_t getPhysicalSegment64(        IOByteCount   offset,
+                                                  IOByteCount * length );
+
+/*! @function prepare
+    @abstract Prepare the memory for an I/O transfer.
+    @discussion This involves paging in the memory, if necessary, and wiring it down for the duration of the transfer.  The complete() method completes the processing of the memory after the I/O transfer finishes.  This method need not called for non-pageable memory.
+    @param forDirection The direction of the I/O to be performed, or kIODirectionNone for the direction specified by the memory descriptor.
+    @result An IOReturn code. */
+
+    virtual IOReturn prepare(IODirection forDirection = kIODirectionNone);
+
+/*! @function complete
+    @abstract Complete processing of the memory after an I/O transfer finishes.
+    @discussion This method should not be called unless a prepare was previously issued; the prepare() and complete() must occur in pairs, before and after an I/O transfer involving pageable memory.
+    @param forDirection The direction of the I/O just completed, or kIODirectionNone for the direction specified by the memory descriptor.
+    @result An IOReturn code. */
+
+    virtual IOReturn complete(IODirection forDirection = kIODirectionNone);
+
+    virtual IOPhysicalAddress getSourceSegment(IOByteCount offset,
+                                               IOByteCount * length);
+};
+
+#endif /* !_IOINTERLEAVEDMEMORYDESCRIPTOR_H */
diff --git a/iokit/IOKit/IOInterruptController.h b/iokit/IOKit/IOInterruptController.h
index 4b4216b9b..d99523594 100644
--- a/iokit/IOKit/IOInterruptController.h
+++ b/iokit/IOKit/IOInterruptController.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/IOKit/IOInterruptEventSource.h b/iokit/IOKit/IOInterruptEventSource.h
index 7a4930d5c..0be7caf45 100644
--- a/iokit/IOKit/IOInterruptEventSource.h
+++ b/iokit/IOKit/IOInterruptEventSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.	 All rights reserved.
diff --git a/iokit/IOKit/IOInterrupts.h b/iokit/IOKit/IOInterrupts.h
index 547f190ad..fa8aa7b33 100644
--- a/iokit/IOKit/IOInterrupts.h
+++ b/iokit/IOKit/IOInterrupts.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IOKitDebug.h b/iokit/IOKit/IOKitDebug.h
index 80aa6c4d8..954f1bd5f 100644
--- a/iokit/IOKit/IOKitDebug.h
+++ b/iokit/IOKit/IOKitDebug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -66,16 +72,19 @@ enum {
     kIOLogMapping 	= 0x00000100ULL,
     kIOLogCatalogue 	= 0x00000200ULL,
     kIOLogTracePower 	= 0x00000400ULL,
-
+    kIOLogDebugPower 	= 0x00000800ULL,
     kIOLogServiceTree 	= 0x00001000ULL,
     kIOLogDTree 	= 0x00002000ULL,
     kIOLogMemory 	= 0x00004000ULL,
-    // available	= 0x00008000ULL,
+    kIOLogKextMemory 	= 0x00008000ULL,
     kOSLogRegistryMods  = 0x00010000ULL,	// Log attempts to modify registry collections
 
 // debug aids - change behaviour
     kIONoFreeObjects 	= 0x00100000ULL,
     kIOLogSynchronous   = 0x00200000ULL,	// IOLog completes synchrounsly
+    kOSTraceObjectAlloc	= 0x00400000ULL,
+
+    _kIODebugTopFlag    = 0x8000000000000000ULL // force enum to be 64 bits
 };
 
 extern SInt64	gIOKitDebug;
diff --git a/iokit/IOKit/IOKitKeys.h b/iokit/IOKit/IOKitKeys.h
index 11a17a23b..ed5843340 100644
--- a/iokit/IOKit/IOKitKeys.h
+++ b/iokit/IOKit/IOKitKeys.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -132,6 +138,9 @@
 // property of root that describes the machine's serial number as a string
 #define kIOPlatformSerialNumberKey	"IOPlatformSerialNumber"	// (OSString)
 
+// property of root that describes the machine's UUID as a string
+#define kIOPlatformUUIDKey	"IOPlatformUUID"	// (OSString)
+
 // IODTNVRAM property keys
 #define kIONVRAMDeletePropertyKey	"IONVRAM-DELETE-PROPERTY"
 #define kIODTNVRAMPanicInfoKey		"aapl,panic-info"
@@ -141,4 +150,7 @@
 #define kIOBootDevicePathKey      "IOBootDevicePath"	// arch-neutral OSString
 #define kIOBootDeviceSizeKey      "IOBootDeviceSize"	// OSNumber of bytes
 
+// keys for OS Version information
+#define kOSBuildVersionKey		"OS Build Version"
+
 #endif /* ! _IOKIT_IOKITKEYS_H */
diff --git a/iokit/IOKit/IOKitKeysPrivate.h b/iokit/IOKit/IOKitKeysPrivate.h
index 9e1a3249b..b9a6f5b6c 100644
--- a/iokit/IOKit/IOKitKeysPrivate.h
+++ b/iokit/IOKit/IOKitKeysPrivate.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IOKITKEYSPRIVATE_H
@@ -59,4 +65,9 @@ enum { kIOPrepareToPhys32 = 0x04 };
 
 #define kIODirectionPrepareToPhys32 ((IODirection) kIOPrepareToPhys32)
 
+#define kIOPlatformSleepActionKey	"IOPlatformSleepAction"		/* value is OSNumber (priority) */
+#define kIOPlatformWakeActionKey	"IOPlatformWakeAction"		/* value is OSNumber (priority) */
+#define kIOPlatformQuiesceActionKey	"IOPlatformQuiesceAction"	/* value is OSNumber (priority) */
+#define kIOPlatformActiveActionKey	"IOPlatformActiveAction"	/* value is OSNumber (priority) */
+
 #endif /* ! _IOKIT_IOKITKEYSPRIVATE_H */
diff --git a/iokit/IOKit/IOKitServer.h b/iokit/IOKit/IOKitServer.h
index 460248bf4..5bfc7862a 100644
--- a/iokit/IOKit/IOKitServer.h
+++ b/iokit/IOKit/IOKitServer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -74,7 +80,8 @@ enum {
     kIOCatalogRemoveDrivers,
     kIOCatalogRemoveDriversNoMatch,
     kIOCatalogStartMatching,
-    kIOCatalogRemoveKernelLinker
+    kIOCatalogRemoveKernelLinker,
+    kIOCatalogKextdFinishedLaunching
 };
 
 // IOCatalogueGetData
diff --git a/iokit/IOKit/IOLib.h b/iokit/IOKit/IOLib.h
index ccbf5554c..dc85274ae 100644
--- a/iokit/IOKit/IOLib.h
+++ b/iokit/IOKit/IOLib.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -248,7 +254,7 @@ IOReturn IOFlushProcessorCache( task_t task, IOVirtualAddress address,
 
 /*! @function IOCreateThread
     @abstract Create a kernel thread.
-    @discussion This function creates a kernel thread, and passes the caller supplied argument to the new thread.
+    @discussion This function creates a kernel thread, and passes the caller supplied argument to the new thread.  Warning: the value returned by this function is not 100% reliable.  There is a race condition where it is possible that the new thread has already terminated before this call returns.  Under that circumstance the IOThread returned will be invalid.  In general there is little that can be done with this value except compare it against 0.  The thread itself can call IOThreadSelf() 100% reliably and that is the prefered mechanism to manipulate the IOThreads state.
     @param function A C-function pointer where the thread will begin execution.
     @param argument Caller specified data to be passed to the new thread.
     @result An IOThread identifier for the new thread, equivalent to an osfmk thread_t. */
@@ -275,6 +281,13 @@ void IOSleep(unsigned milliseconds);
 
 void IODelay(unsigned microseconds);
 
+/*! @function IOPause
+    @abstract Spin delay for a number of nanoseconds.
+    @discussion This function spins to delay for at least the number of specified nanoseconds. Since the CPU is busy spinning no time is made available to other processes; this method of delay should be used only for short periods.
+    @param microseconds The integer number of nanoseconds to spin wait. */
+
+void IOPause(unsigned nanoseconds);
+
 /*! @function IOLog
     @abstract Log a message to console in text mode, and /var/log/system.log.
     @discussion This function allows a driver to log diagnostic information to the screen during verbose boots, and to a log file found at /var/log/system.log. IOLog should not be called from interrupt context.
diff --git a/iokit/IOKit/IOLocks.h b/iokit/IOKit/IOLocks.h
index 734230164..8b3e7c12c 100644
--- a/iokit/IOKit/IOLocks.h
+++ b/iokit/IOKit/IOLocks.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -176,7 +182,7 @@ typedef struct _IORecursiveLock IORecursiveLock;
 
 /*! @function IORecursiveLockAlloc
     @abstract Allocates and initializes an recursive lock.
-    @discussion Allocates a recursive lock in general purpose memory, and initilizes it. Recursive locks function identically to mutexes but allow one thread to lock more than once, with balanced unlocks.
+    @discussion Allocates a recursive lock in general purpose memory, and initializes it. Recursive locks function identically to mutexes but allow one thread to lock more than once, with balanced unlocks.
     @result Pointer to the allocated lock, or zero on failure. */
 
 IORecursiveLock * IORecursiveLockAlloc( void );
diff --git a/iokit/IOKit/IOLocksPrivate.h b/iokit/IOKit/IOLocksPrivate.h
new file mode 100644
index 000000000..16eed6a4d
--- /dev/null
+++ b/iokit/IOKit/IOLocksPrivate.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *
+ */
+
+#ifndef __IOKIT_IOLOCKS_PRIVATE_H
+#define __IOKIT_IOLOCKS_PRIVATE_H
+
+#ifndef KERNEL
+#error IOLocksPrivate.h is for kernel use only
+#endif
+
+#include <sys/appleapiopts.h>
+
+#include <IOKit/system.h>
+
+#include <IOKit/IOReturn.h>
+#include <IOKit/IOTypes.h>
+#include <IOKit/IOLocks.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+IORecursiveLock *
+IORecursiveLockAllocWithLockGroup ( lck_grp_t * lockGroup );
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* !__IOKIT_IOLOCKS_PRIVATE_H */
+
diff --git a/iokit/IOKit/IOMapper.h b/iokit/IOKit/IOMapper.h
index 6507f5d7e..9fa09459a 100644
--- a/iokit/IOKit/IOMapper.h
+++ b/iokit/IOKit/IOMapper.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __IOKIT_IOMAPPER_H
@@ -82,10 +88,8 @@ protected:
     virtual bool allocTable(IOByteCount size);
 
 public:
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual bool start(IOService *provider);
     virtual void free();
-#endif
 
     // Static routines capable of allocating tables that are physically
     // contiguous in real memory space.
@@ -112,16 +116,11 @@ public:
     // iovm mapping.
     virtual addr64_t mapAddr(IOPhysicalAddress addr) = 0;
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     // Get the address mask to or into an address to bypass this mapper
     virtual bool getBypassMask(addr64_t *maskP) const
     OSMetaClassDeclareReservedUsed(IOMapper, 0);
-#endif
 
 private:
-#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
-    OSMetaClassDeclareReservedUnused(IOMapper, 0);
-#endif
     OSMetaClassDeclareReservedUnused(IOMapper, 1);
     OSMetaClassDeclareReservedUnused(IOMapper, 2);
     OSMetaClassDeclareReservedUnused(IOMapper, 3);
diff --git a/iokit/IOKit/IOMemoryCursor.h b/iokit/IOKit/IOMemoryCursor.h
index 28a0bbcd4..dfe9eed8c 100644
--- a/iokit/IOKit/IOMemoryCursor.h
+++ b/iokit/IOKit/IOMemoryCursor.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOMEMORYCURSOR_H
 #define _IOMEMORYCURSOR_H
diff --git a/iokit/IOKit/IOMemoryDescriptor.h b/iokit/IOKit/IOMemoryDescriptor.h
index 9d37372fd..080f692f5 100644
--- a/iokit/IOKit/IOMemoryDescriptor.h
+++ b/iokit/IOKit/IOMemoryDescriptor.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOMEMORYDESCRIPTOR_H
 #define _IOMEMORYDESCRIPTOR_H
@@ -25,6 +31,7 @@
 #include <sys/cdefs.h>
 
 #include <IOKit/IOTypes.h>
+#include <IOKit/IOLocks.h>
 #include <libkern/c++/OSContainers.h>
 
 __BEGIN_DECLS
@@ -71,7 +78,8 @@ enum {
     kIOMemoryAsReference	= 0x00000100,
     kIOMemoryBufferPageable	= 0x00000400,
     kIOMemoryDontMap		= 0x00000800,
-    kIOMemoryPersistent		= 0x00010000
+    kIOMemoryPersistent		= 0x00010000,
+    kIOMemoryThreadSafe		= 0x00020000
 };
 
 #define kIOMapperNone	((IOMapper *) -1)
@@ -183,16 +191,11 @@ typedef IOOptionBits DMACommandOps;
                                         IOByteCount offset, IOByteCount length );
     OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 4);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     // Used for dedicated communications for IODMACommand
     virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
     OSMetaClassDeclareReservedUsed(IOMemoryDescriptor, 5);
-#endif
 
 private:
-#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
-    OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 5);
-#endif
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 6);
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 7);
     OSMetaClassDeclareReservedUnused(IOMemoryDescriptor, 8);
@@ -265,42 +268,38 @@ public:
                                             task_t           withTask,
                                             bool             asReference = false);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
 /*! @function withAddressRange
     @abstract Create an IOMemoryDescriptor to describe one virtual range of the specified map.
-    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of a single virtual memory range mapped into the specified map.
+    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of a single virtual memory range mapped into the specified map. Note that unlike IOMemoryDescriptor::withAddress(), kernel_task memory must be explicitly prepared when passed to this api.
     @param address The virtual address of the first byte in the memory.
     @param withLength The length of memory.
     @param options
-        kIOMemoryDirectionMask (options:direction)     This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
-        kIOMemoryNoAutoPrepare Indicates that the temporary AutoPrepare of kernel_task memory should not be performed.
-    @param task The task the virtual ranges are mapped into.
+        kIOMemoryDirectionMask (options:direction)	This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
+    @param task The task the virtual ranges are mapped into. Note that unlike IOMemoryDescriptor::withAddress(), kernel_task memory must be explicitly prepared when passed to this api.
     @result The created IOMemoryDescriptor on success, to be released by the caller, or zero on failure. */
 
     static IOMemoryDescriptor * withAddressRange(
-                                       mach_vm_address_t address,
-                                       mach_vm_size_t    length,
-                                       IOOptionBits      options,
-                               task_t            task);
+					mach_vm_address_t address,
+					mach_vm_size_t	  length,
+					IOOptionBits      options,
+					task_t            task);
 
 /*! @function withAddressRanges
     @abstract Create an IOMemoryDescriptor to describe one or more virtual ranges.
-    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of an array of virtual memory ranges each mapped into a specified source task.
+    @discussion This method creates and initializes an IOMemoryDescriptor for memory consisting of an array of virtual memory ranges each mapped into a specified source task. Note that unlike IOMemoryDescriptor::withAddress(), kernel_task memory must be explicitly prepared when passed to this api.
     @param ranges An array of IOAddressRange structures which specify the virtual ranges in the specified map which make up the memory to be described. IOAddressRange is the 64bit version of IOVirtualRange.
     @param rangeCount The member count of the ranges array.
     @param options
-        kIOMemoryDirectionMask (options:direction)     This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
-        kIOMemoryAsReference   For options:type = Virtual or Physical this indicate that the memory descriptor need not copy the ranges array into local memory.  This is an optimisation to try to minimise unnecessary allocations.
-        kIOMemoryNoAutoPrepare Indicates that the temporary AutoPrepare of kernel_task memory should not be performed.
-    @param task The task each of the virtual ranges are mapped into.
+        kIOMemoryDirectionMask (options:direction)	This nibble indicates the I/O direction to be associated with the descriptor, which may affect the operation of the prepare and complete methods on some architectures. 
+        kIOMemoryAsReference	For options:type = Virtual or Physical this indicate that the memory descriptor need not copy the ranges array into local memory.  This is an optimisation to try to minimise unnecessary allocations.
+    @param task The task each of the virtual ranges are mapped into. Note that unlike IOMemoryDescriptor::withAddress(), kernel_task memory must be explicitly prepared when passed to this api.
     @result The created IOMemoryDescriptor on success, to be released by the caller, or zero on failure. */
 
     static IOMemoryDescriptor * withAddressRanges(
-                                       IOAddressRange * ranges,
-                                       UInt32           rangeCount,
-                                       IOOptionBits     options,
-                                       task_t           withTask);
-#endif
+					IOAddressRange * ranges,
+					UInt32           rangeCount,
+					IOOptionBits     options,
+					task_t           withTask);
 
 /*! @function withOptions
     @abstract Master initialiser for all variants of memory descriptors.
@@ -469,7 +468,7 @@ public:
 
 /*! @function readBytes
     @abstract Copy data from the memory descriptor's buffer to the specified buffer.
-    @discussion This method copies data from the memory descriptor's memory at the given offset, to the caller's buffer.
+    @discussion This method copies data from the memory descriptor's memory at the given offset, to the caller's buffer.  The memory descriptor MUST have the kIODirectionOut direcction bit set  and be prepared.  kIODirectionOut means that this memory descriptor will be output to an external device, so readBytes is used to get memory into a local buffer for a PIO transfer to the device.
     @param offset A byte offset into the memory descriptor's memory.
     @param bytes The caller supplied buffer to copy the data to.
     @param withLength The length of the data to copy.
@@ -480,7 +479,7 @@ public:
 
 /*! @function writeBytes
     @abstract Copy data to the memory descriptor's buffer from the specified buffer.
-    @discussion This method copies data to the memory descriptor's memory at the given offset, from the caller's buffer.
+    @discussion This method copies data to the memory descriptor's memory at the given offset, from the caller's buffer.  The memory descriptor MUST have the kIODirectionIn direcction bit set  and be prepared.  kIODirectionIn means that this memory descriptor will be input from an external device, so writeBytes is used to write memory into the descriptor for PIO drivers.
     @param offset A byte offset into the memory descriptor's memory.
     @param bytes The caller supplied buffer to copy the data from.
     @param withLength The length of the data to copy.
@@ -532,7 +531,7 @@ public:
      * Mapping functions.
      */
 
-/*! @function map
+/*! @function createMappingInTask
     @abstract Maps a IOMemoryDescriptor into a task.
     @discussion This is the general purpose method to map all or part of the memory described by a memory descriptor into a task at any available address, or at a fixed address if possible. Caching & read-only options may be set for the mapping. The mapping is represented as a returned reference to a IOMemoryMap object, which may be shared if the mapping is compatible with an existing mapping of the IOMemoryDescriptor. The IOMemoryMap object returned should be released only when the caller has finished accessing the mapping, as freeing the object destroys the mapping. 
     @param intoTask Sets the target task for the mapping. Pass kernel_task for the kernel address space.
@@ -548,6 +547,29 @@ public:
     @param length Is the length of the mapping requested for a subset of the IOMemoryDescriptor. Zero is the default to map all the memory.
     @result A reference to an IOMemoryMap object representing the mapping, which can supply the virtual address of the mapping and other information. The mapping may be shared with multiple callers - multiple maps are avoided if a compatible one exists. The IOMemoryMap object returned should be released only when the caller has finished accessing the mapping, as freeing the object destroys the mapping. The IOMemoryMap instance also retains the IOMemoryDescriptor it maps while it exists. */
 
+    IOMemoryMap * 	createMappingInTask(
+	task_t			intoTask,
+	mach_vm_address_t	atAddress,
+	IOOptionBits		options,
+	mach_vm_size_t		offset = 0,
+	mach_vm_size_t		length = 0 );
+
+/*! @function map
+    @abstract Maps a IOMemoryDescriptor into a task - deprecated, only safe for 32 bit tasks. Use createMappingInTask instead.
+    @discussion This is the general purpose method to map all or part of the memory described by a memory descriptor into a task at any available address, or at a fixed address if possible. Caching & read-only options may be set for the mapping. The mapping is represented as a returned reference to a IOMemoryMap object, which may be shared if the mapping is compatible with an existing mapping of the IOMemoryDescriptor. The IOMemoryMap object returned should be released only when the caller has finished accessing the mapping, as freeing the object destroys the mapping. 
+    @param intoTask Sets the target task for the mapping. Pass kernel_task for the kernel address space.
+    @param atAddress If a placed mapping is requested, atAddress specifies its address, and the kIOMapAnywhere should not be set. Otherwise, atAddress is ignored.
+    @param options Mapping options are defined in IOTypes.h,<br>
+	kIOMapAnywhere should be passed if the mapping can be created anywhere. If not set, the atAddress parameter sets the location of the mapping, if it is available in the target map.<br>
+	kIOMapDefaultCache to inhibit the cache in I/O areas, kIOMapCopybackCache in general purpose RAM.<br>
+	kIOMapInhibitCache, kIOMapWriteThruCache, kIOMapCopybackCache to set the appropriate caching.<br>
+	kIOMapReadOnly to allow only read only accesses to the memory - writes will cause and access fault.<br>
+	kIOMapReference will only succeed if the mapping already exists, and the IOMemoryMap object is just an extra reference, ie. no new mapping will be created.<br>
+	kIOMapUnique allows a special kind of mapping to be created that may be used with the IOMemoryMap::redirect() API. These mappings will not be shared as is the default - there will always be a unique mapping created for the caller, not an existing mapping with an extra reference.<br>
+    @param offset Is a beginning offset into the IOMemoryDescriptor's memory where the mapping starts. Zero is the default to map all the memory.
+    @param length Is the length of the mapping requested for a subset of the IOMemoryDescriptor. Zero is the default to map all the memory.
+    @result A reference to an IOMemoryMap object representing the mapping, which can supply the virtual address of the mapping and other information. The mapping may be shared with multiple callers - multiple maps are avoided if a compatible one exists. The IOMemoryMap object returned should be released only when the caller has finished accessing the mapping, as freeing the object destroys the mapping. The IOMemoryMap instance also retains the IOMemoryDescriptor it maps while it exists. */
+
     virtual IOMemoryMap * 	map(
 	task_t		intoTask,
 	IOVirtualAddress	atAddress,
@@ -555,11 +577,12 @@ public:
 	IOByteCount		offset = 0,
 	IOByteCount		length = 0 );
 
+
 /*! @function map
     @abstract Maps a IOMemoryDescriptor into the kernel map.
-    @discussion This is a shortcut method to map all the memory described by a memory descriptor into the kernel map at any available address. See the full version of the map method for further details.
-    @param options Mapping options as in the full version of the map method, with kIOMapAnywhere assumed.
-    @result See the full version of the map method. */
+    @discussion This is a shortcut method to map all the memory described by a memory descriptor into the kernel map at any available address. See the full version of the createMappingInTask method for further details.
+    @param options Mapping options as in the full version of the createMappingInTask method, with kIOMapAnywhere assumed.
+    @result See the full version of the createMappingInTask method. */
 
     virtual IOMemoryMap * 	map(
 	IOOptionBits		options = 0 );
@@ -585,15 +608,15 @@ public:
     IOReturn handleFault(
         void *			pager,
 	vm_map_t		addressMap,
-	IOVirtualAddress	address,
-	IOByteCount		sourceOffset,
-	IOByteCount		length,
+	mach_vm_address_t	address,
+	mach_vm_size_t		sourceOffset,
+	mach_vm_size_t		length,
         IOOptionBits		options );
 
 protected:
     virtual IOMemoryMap * 	makeMapping(
 	IOMemoryDescriptor *	owner,
-	task_t		intoTask,
+	task_t			intoTask,
 	IOVirtualAddress	atAddress,
 	IOOptionBits		options,
 	IOByteCount		offset,
@@ -702,6 +725,13 @@ public:
     virtual IOReturn		redirect(IOMemoryDescriptor * newBackingMemory,
 					 IOOptionBits         options,
 					 IOByteCount          offset = 0) = 0;
+
+    virtual IOReturn		redirect(IOMemoryDescriptor * newBackingMemory,
+					 IOOptionBits         options,
+					 mach_vm_size_t       offset = 0) = 0;
+
+    virtual mach_vm_address_t 	getAddress() = 0;
+    virtual mach_vm_size_t 	getSize() = 0;
 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -752,9 +782,7 @@ protected:
 
     virtual void free();
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
-#endif
 
 private:
 
@@ -772,7 +800,8 @@ private:
     ppnum_t	    _highestPage;
     uint32_t	    __iomd_reservedA;
     uint32_t	    __iomd_reservedB;
-    uint32_t	    __iomd_reservedC;
+
+    IOLock *	    _prepareLock;
 
 public:
     /*
@@ -813,10 +842,8 @@ public:
                                         IODirection      withDirection,
                                         bool             asReference = false);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual addr64_t getPhysicalSegment64( IOByteCount offset,
                                             IOByteCount * length );
-#endif
 
     virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
 						 IOByteCount * length);
@@ -842,11 +869,13 @@ public:
 	vm_map_t		addressMap,
 	IOVirtualAddress	logical,
 	IOByteCount		length );
+
     virtual bool serialize(OSSerialize *s) const;
 
     // Factory method for cloning a persistent IOMD, see IOMemoryDescriptor
     static IOMemoryDescriptor *
 	withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD);
+
 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -894,10 +923,8 @@ protected:
     IOMemoryDescriptor::withRanges;
     IOMemoryDescriptor::withSubRange;
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     // used by IODMACommand
     virtual IOReturn dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const;
-#endif
 
 public:
     /*
@@ -916,10 +943,8 @@ public:
      * IOMemoryDescriptor required methods
      */
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual addr64_t getPhysicalSegment64( IOByteCount offset,
                                             IOByteCount * length );
-#endif
 
     virtual IOPhysicalAddress getPhysicalSegment(IOByteCount offset,
 						 IOByteCount * length);
@@ -953,18 +978,18 @@ public:
 protected:
     virtual IOMemoryMap * 	makeMapping(
 	IOMemoryDescriptor *	owner,
-	task_t		intoTask,
+	task_t			intoTask,
 	IOVirtualAddress	atAddress,
 	IOOptionBits		options,
 	IOByteCount		offset,
 	IOByteCount		length );
 
     virtual IOReturn doMap(
-	vm_map_t		addressMap,
-	IOVirtualAddress *	atAddress,
-	IOOptionBits		options,
-	IOByteCount		sourceOffset = 0,
-	IOByteCount		length = 0 );
+       vm_map_t                addressMap,
+       IOVirtualAddress *      atAddress,
+       IOOptionBits            options,
+       IOByteCount             sourceOffset = 0,
+       IOByteCount             length = 0 );
 };
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
diff --git a/iokit/IOKit/IOMessage.h b/iokit/IOKit/IOMessage.h
index 40655f323..3ca9e1e79 100644
--- a/iokit/IOKit/IOMessage.h
+++ b/iokit/IOKit/IOMessage.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __IOKIT_IOMESSAGE_H
@@ -64,4 +70,6 @@ typedef UInt32 IOMessage;
 #define kIOMessageSystemWillRestart        iokit_common_msg(0x310)
 #define kIOMessageSystemWillPowerOn        iokit_common_msg(0x320)
 
+#define kIOMessageCopyClientID		   iokit_common_msg(0x330)
+
 #endif /* ! __IOKIT_IOMESSAGE_H */
diff --git a/iokit/IOKit/IOMultiMemoryDescriptor.h b/iokit/IOKit/IOMultiMemoryDescriptor.h
index 190dca1da..c4efe6086 100644
--- a/iokit/IOKit/IOMultiMemoryDescriptor.h
+++ b/iokit/IOKit/IOMultiMemoryDescriptor.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOMULTIMEMORYDESCRIPTOR_H
@@ -120,10 +126,8 @@ public:
     virtual IOPhysicalAddress getPhysicalSegment( IOByteCount   offset,
                                                   IOByteCount * length );
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual addr64_t getPhysicalSegment64(        IOByteCount   offset,
                                                   IOByteCount * length );
-#endif
 
 /*! @function prepare
     @abstract Prepare the memory for an I/O transfer.
diff --git a/iokit/IOKit/IONVRAM.h b/iokit/IOKit/IONVRAM.h
index 6114b0112..3322945d5 100644
--- a/iokit/IOKit/IONVRAM.h
+++ b/iokit/IOKit/IONVRAM.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IONVRAM_H
diff --git a/iokit/IOKit/IONotifier.h b/iokit/IOKit/IONotifier.h
index 24e3d2560..8f4378ab7 100644
--- a/iokit/IOKit/IONotifier.h
+++ b/iokit/IOKit/IONotifier.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IOPMEventSource.h b/iokit/IOKit/IOPMEventSource.h
index e82cbd157..370b9e14e 100644
--- a/iokit/IOKit/IOPMEventSource.h
+++ b/iokit/IOKit/IOPMEventSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
  #ifndef _IOPMEVENTSOURCE_H_
diff --git a/iokit/IOKit/IOPlatformExpert.h b/iokit/IOKit/IOPlatformExpert.h
index ff98488f7..1826c256e 100644
--- a/iokit/IOKit/IOPlatformExpert.h
+++ b/iokit/IOKit/IOPlatformExpert.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -47,7 +53,9 @@ enum {
   kPEHaltCPU,
   kPERestartCPU,
   kPEHangCPU,
-  kPEUPSDelayHaltCPU
+  kPEUPSDelayHaltCPU,
+  kPEPanicRestartCPU,
+  kPEPanicSync
 };
 extern int (*PE_halt_restart)(unsigned int type);
 extern int PEHaltRestart(unsigned int type);
@@ -63,7 +71,13 @@ extern void PESetGMTTimeOfDay( long secs );
 
 #define kIOPlatformMapperPresentKey "IOPlatformMapperPresent"
 
-extern OSSymbol * gPlatformInterruptControllerName;
+
+extern OSSymbol *               gPlatformInterruptControllerName;
+
+extern const OSSymbol *		gIOPlatformSleepActionKey;
+extern const OSSymbol *		gIOPlatformWakeActionKey;
+extern const OSSymbol *		gIOPlatformQuiesceActionKey;
+extern const OSSymbol *		gIOPlatformActiveActionKey;
 
 class IORangeAllocator;
 class IONVRAMController;
@@ -265,12 +279,10 @@ public:
     virtual bool compareName( OSString * name, OSString ** matched = 0 ) const;
 
     virtual IOWorkLoop *getWorkLoop() const;
+    virtual IOReturn setProperties( OSObject * properties );
 
     virtual void free();
 
-    virtual bool attachToChild( IORegistryEntry * child,
-                                const IORegistryPlane * plane );
-
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  0);
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  1);
     OSMetaClassDeclareReservedUnused(IOPlatformExpertDevice,  2);
diff --git a/iokit/IOKit/IOPolledInterface.h b/iokit/IOKit/IOPolledInterface.h
index 6be5edd0f..09ef735d9 100644
--- a/iokit/IOKit/IOPolledInterface.h
+++ b/iokit/IOKit/IOPolledInterface.h
@@ -1,25 +1,34 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#ifndef _IOPOLLEDINTERFACE_H_
+#define _IOPOLLEDINTERFACE_H_
+
 #include <libkern/c++/OSObject.h>
 
 #define kIOPolledInterfaceSupportKey "IOPolledInterface"
@@ -91,3 +100,4 @@ public:
     OSMetaClassDeclareReservedUnused(IOPolledInterface, 15);
 };
 
+#endif /* _IOPOLLEDINTERFACE_H_ */
diff --git a/iokit/IOKit/IORangeAllocator.h b/iokit/IOKit/IORangeAllocator.h
index f222cacaf..b6f7ee17b 100644
--- a/iokit/IOKit/IORangeAllocator.h
+++ b/iokit/IOKit/IORangeAllocator.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.
diff --git a/iokit/IOKit/IORegistryEntry.h b/iokit/IOKit/IORegistryEntry.h
index ffb9d6844..de924f9c1 100644
--- a/iokit/IOKit/IORegistryEntry.h
+++ b/iokit/IOKit/IORegistryEntry.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -314,7 +320,7 @@ public:
 
 /*! @function setProperty
     @abstract Synchronized method to construct and add an OSData property to a registry entry's property table.
-    @discussion This method will add or replace a property in a registry entry's property table, using the OSDictionary::setObject semantics. This method is synchronized with other IORegistryEntry accesses to the property table. The property is created as an OSData copied from the supplied date and length, set in the property table with the given name, and released.
+    @discussion This method will add or replace a property in a registry entry's property table, using the OSDictionary::setObject semantics. This method is synchronized with other IORegistryEntry accesses to the property table. The property is created as an OSData copied from the supplied data and length, set in the property table with the given name, and released.
     @param aKey The property's name as a C-string.
     @param bytes The property's value as a pointer. OSData will copy this data.
     @param length The property's size in bytes, for OSData.
diff --git a/iokit/IOKit/IOReturn.h b/iokit/IOKit/IOReturn.h
index 32f8ad690..ae24a19d6 100644
--- a/iokit/IOKit/IOReturn.h
+++ b/iokit/IOKit/IOReturn.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * HISTORY
diff --git a/iokit/IOKit/IOService.h b/iokit/IOKit/IOService.h
index b40a47543..45baa9348 100644
--- a/iokit/IOKit/IOService.h
+++ b/iokit/IOKit/IOService.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998,1999 Apple Computer, Inc.  All rights reserved. 
@@ -25,7 +31,12 @@
  * HISTORY
  *
  */
+/*!
+	@header
+	This header contains the definition of the IOService class.  IOService is the sole direct subclass of IORegistryEntry and is the base class of almost all I/O Kit family superclasses.  IOService defines methods that support the life cycle of I/O Kit drivers.  For more information on IOService, see {@linkdoc //apple_ref/doc/uid/TP0000011 I/O Kit Fundamentals}.
 
+	@seealso //apple_ref/doc/header/IORegistryEntry.h IORegistryEntry
+*/
 
 #ifndef _IOKIT_IOSERVICE_H
 #define _IOKIT_IOSERVICE_H
@@ -39,9 +50,6 @@
 #include <IOKit/IOKitDebug.h>
 #include <IOKit/IOInterrupts.h>
 
-class IOPMinformee;
-class IOPowerConnection;
-
 #include <IOKit/pwr_mgt/IOPMpowerState.h>
 #include <IOKit/IOServicePM.h>
 
@@ -158,99 +166,112 @@ typedef void (*OSObjectApplierFunction)(OSObject * object, void * context);
 class IOUserClient;
 class IOPlatformExpert;
 
-/*! @class IOService : public IORegistryEntry
-    @abstract The base class for most families, devices and drivers.
+/*! @class IOService
+    @abstract The base class for most I/O Kit families, devices, and drivers.
     @discussion The IOService base class defines APIs used to publish services, instantiate other services based on the existance of a providing service (ie. driver stacking), destroy a service and its dependent stack, notify interested parties of service state changes, and general utility functions useful across all families. 
 
 Types of service are specified with a matching dictionary that describes properties of the service. For example, a matching dictionary might describe any IOUSBDevice (or subclass), an IOUSBDevice with a certain class code, or a IOPCIDevice with a set of OpenFirmware matching names or device & vendor IDs. Since the matching dictionary is interpreted by the family which created the service, as well as generically by IOService, the list of properties considered for matching depends on the familiy.
 
 Matching dictionaries are associated with IOService classes by the catalogue, as driver property tables, and also supplied by clients of the notification APIs.
 
-IOService provides matching based on c++ class (via OSMetaClass dynamic casting), registry entry name, a registry path to the service (which includes OpenFirmware paths), a name assigned by BSD, or by its location (its point of attachment).
+IOService provides matching based on C++ class (via OSMetaClass dynamic casting), registry entry name, a registry path to the service (which includes OpenFirmware paths), a name assigned by BSD, or by its location (its point of attachment).
 
 <br><br>Driver Instantiation by IOService<br><br>
 
 Drivers are subclasses of IOService, and their availability is managed through the catalogue. They are instantiated based on the publication of an IOService they use (for example, an IOPCIDevice or IOUSBDevice), or when they are added to  the catalogue and the IOService(s) they use are already available.
 
-When an IOService (the "provider") is published with the registerService() method, the matching and probing process begins, which is always single threaded per provider. A list of matching dictionaries from the catalog and installed publish notification requests, that successfully match the IOService, is constructed, with ordering supplied by kIOProbeScoreKey ("IOProbeScore") property in the dictionary, or supplied with the notification. 
+When an IOService (the "provider") is published with the @link registerService registerService@/link method, the matching and probing process begins, which is always single threaded per provider. A list of matching dictionaries from the catalog and installed publish notification requests, that successfully match the IOService, is constructed, with ordering supplied by <code>kIOProbeScoreKey</code> ("IOProbeScore") property in the dictionary, or supplied with the notification. 
 
 Each entry in the list is then processed in order - for notifications, the notification is delivered, for driver property tables a lot more happens.
 
-The driver class is instantiated and init() called with its property table. The new driver instance is then attached to the provider, and has its probe() method called with the provider as an argument. The default probe method does nothing but return success, but a driver may implement this method to interrogate the provider to make sure it can work with it. It may also modify its probe score at this time. After probe, the driver is detached and the next in the list is considered (ie. attached, probed, and detached).
+The driver class is instantiated and <code>init()</code> called with its property table. The new driver instance is then attached to the provider, and has its @link probe probe@/link method called with the provider as an argument. The default <code>probe</code> method does nothing but return success, but a driver may implement this method to interrogate the provider to make sure it can work with it. It may also modify its probe score at this time. After probe, the driver is detached and the next in the list is considered (ie. attached, probed, and detached).
 
-When the probing phase is complete, the list consists of successfully probed drivers, in order of their probe score (after adjustment during the probe() call). The list is then divided into categories based on the kIOMatchCategoryKey property ("IOMatchCategory"); drivers without a match category are all considered in one default category. Match categories allow multiple clients of a provider to be attached and started, though the provider may also enforce open/close semantics to gain active access to it.
+When the probing phase is complete, the list consists of successfully probed drivers, in order of their probe score (after adjustment during the @link probe probe@/link call). The list is then divided into categories based on the <code>kIOMatchCategoryKey</code> property ("IOMatchCategory"); drivers without a match category are all considered in one default category. Match categories allow multiple clients of a provider to be attached and started, though the provider may also enforce open/close semantics to gain active access to it.
 
-For each category, the highest scoring driver in that category is attached to the provider, and its start() method called. If start() is successful, the rest of the drivers in the same match category are discarded, otherwise the next highest scoring driver is started, and so one.
+For each category, the highest scoring driver in that category is attached to the provider, and its @link start start@/link method called. If <code>start</code> is successful, the rest of the drivers in the same match category are discarded, otherwise the next highest scoring driver is started, and so on.
 
-The driver should only consider itself in action when the start method is called, meaning it has been selected for use on the provider, and consuming that particular match category. It should also be prepared to be allocated, probed and freed even if the probe was sucessful.
+The driver should only consider itself in action when the start method is called, meaning it has been selected for use on the provider, and consuming that particular match category. It should also be prepared to be allocated, probed and freed even if the probe was successful.
 
 After the drivers have all synchronously been started, the installed "matched" notifications that match the registered IOService are delivered.
 
 <br><br>Properties used by IOService<br><br>
 
-	kIOClassKey, extern const OSSymbol * gIOClassKey, "IOClass"
+	<code>kIOClassKey, extern const OSSymbol * gIOClassKey, "IOClass"</code>
+<br>
 <br>
 Class of the driver to instantiate on matching providers.
 <br>
 <br>
-	kIOProviderClassKey, extern const OSSymbol * gIOProviderClassKey, "IOProviderClass"
+	<code>kIOProviderClassKey, extern const OSSymbol * gIOProviderClassKey, "IOProviderClass"</code>
+<br>
 <br>
 Class of the provider(s) to be considered for matching, checked with OSDynamicCast so subclasses will also match.
 <br>
 <br>
-	kIOProbeScoreKey, extern const OSSymbol * gIOProbeScoreKey, "IOProbeScore"
+	<code>kIOProbeScoreKey, extern const OSSymbol * gIOProbeScoreKey, "IOProbeScore"</code>
 <br>
-The probe score initially used to order multiple matching drivers.
 <br>
+The probe score initially used to order multiple matching drivers.
 <br>
-	kIOMatchCategoryKey, extern const OSSymbol * gIOMatchCategoryKey, "IOMatchCategory"
 <br>
-A string defining the driver category for matching purposes. All drivers with no IOMatchCategory property are considered to be in the same default category. Only one driver in a category can be started on each provider.
+	<code>kIOMatchCategoryKey, extern const OSSymbol * gIOMatchCategoryKey, "IOMatchCategory"</code>
 <br>
 <br>
-	kIONameMatchKey, extern const OSSymbol * gIONameMatchKey, "IONameMatch"
+A string defining the driver category for matching purposes. All drivers with no <code>IOMatchCategory</code> property are considered to be in the same default category. Only one driver in a category can be started on each provider.
 <br>
-A string or collection of strings that match the provider's name. The comparison is implemented with the IORegistryEntry::compareNames method, which supports a single string, or any collection (OSArray, OSSet, OSDictionary etc.) of strings. IOService objects with OpenFirmware device tree properties (eg. IOPCIDevice) will also be matched based on that standard's "compatible", "name", "device_type" properties. The matching name will be left in the driver's property table in the kIONameMatchedKey property.
 <br>
-Examples
+	<code>kIONameMatchKey, extern const OSSymbol * gIONameMatchKey, "IONameMatch"</code>
 <br>
-      &ltkey&gtIONameMatch&lt/key&gt		<br>
-	&ltstring&gtpci106b,7&ltstring&gt
+A string or collection of strings that match the provider's name. The comparison is implemented with the @link //apple_ref/cpp/instm/IORegistryEntry/compareNames/virtualbool/(OSObject*,OSString**) IORegistryEntry::compareNames@/link method, which supports a single string, or any collection (OSArray, OSSet, OSDictionary etc.) of strings. IOService objects with OpenFirmware device tree properties (eg. IOPCIDevice) will also be matched based on that standard's "compatible", "name", "device_type" properties. The matching name will be left in the driver's property table in the <code>kIONameMatchedKey</code> property.
 <br>
+Examples
+<pre>
+@textblock
+	<key>IONameMatch</key>
+	<string>pci106b,7</string>
+@/textblock
+</pre>
+
 For a list of possible matching names, a serialized array of strings should used, eg.
+<pre>
+@textblock
+	<key>IONameMatch</key>
+	<array>
+		<string>APPL,happy16</string>
+		<string>pci106b,7</string>
+	</array>
+@/textblock
+</pre>
+
 <br>
-      &ltkey&gtIONameMatch&lt/key&gt		<br>
-	&ltarray&gt				<br>
-		&ltstring&gtAPPL,happy16&lt/string&gt	<br>
-		&ltstring&gtpci106b,7&lt/string&gt	<br>
-	&lt/array&gt
-<br>
-<br>
-	kIONameMatchedKey, extern const OSSymbol * gIONameMatchedKey, "IONameMatched"
-<br>
-The name successfully matched name from the kIONameMatchKey property will be left in the driver's property table as the kIONameMatchedKey property.
+	<code>kIONameMatchedKey, extern const OSSymbol * gIONameMatchedKey, "IONameMatched"</code>
 <br>
+The name successfully matched name from the <code>kIONameMatchKey</code> property will be left in the driver's property table as the <code>kIONameMatchedKey</code> property.
 <br>
-	kIOPropertyMatchKey, extern const OSSymbol * gIOPropertyMatchKey, "IOPropertyMatch"
 <br>
-A dictionary of properties that each must exist in the matching IOService and compare sucessfully with the isEqualTo method.
-      &ltkey&gtIOPropertyMatch&lt/key&gt		<br>
-	&ltdictionary&gt				<br>
-		&ltkey&gtname&lt/key&gt	<br>
-		&ltstring&gtAPPL,meek8&lt/string&gt	<br>
-	&lt/dictionary&gt
+	<code>kIOPropertyMatchKey, extern const OSSymbol * gIOPropertyMatchKey, "IOPropertyMatch"</code>
 <br>
+A dictionary of properties that each must exist in the matching IOService and compare successfully with the <code>isEqualTo</code> method.
+
+<pre>
+@textblock
+	<key>IOPropertyMatch</key>
+	<dictionary>
+		<key>APPL,happy16</key>
+		<string>APPL,meek8</string>
+	</dictionary>
+@/textblock
+</pre>
+
 <br>
-	kIOUserClientClassKey, extern const OSSymbol * gIOUserClientClassKey, "IOUserClientClass"
+	<code>kIOUserClientClassKey, extern const OSSymbol * gIOUserClientClassKey, "IOUserClientClass"</code>
 <br>
 The class name that the service will attempt to allocate when a user client connection is requested.  First the device nub is queried, then the nub's provider is queried by default.
 <br>
 <br>
-	kIOKitDebugKey, extern const OSSymbol * gIOKitDebugKey, "IOKitDebug"
+	<code>kIOKitDebugKey, extern const OSSymbol * gIOKitDebugKey, "IOKitDebug"</code>
 <br>
-Set some debug flags for logging the driver loading process. Flags are defined in IOKit/IOKitDebug.h, but 65535 works well.
-
-*/
+Set some debug flags for logging the driver loading process. Flags are defined in <code>IOKit/IOKitDebug.h</code>, but <code>65535</code> works well.*/
     
 class IOService : public IORegistryEntry
 {
@@ -272,26 +293,24 @@ private:
     IOService *		__owner;
     IOOptionBits	__state[2];
     IOOptionBits	__reserved[4];
-
-    // pointer to private instance variables for power management
-    IOPMpriv *		priv;
+    IOServicePM *	pwrMgt;
 
 protected:
     // TRUE once PMinit has been called
-        bool		initialized;
-        
+    bool			initialized;
+
 public:
-    // pointer to protected instance variables for power management
+    // DEPRECATED
     IOPMprot * 	 	pm_vars;
 
 public:
     /* methods available in Mac OS X 10.1 or later */
 /*! @function requestTerminate
     @abstract Passes a termination up the stack.
-    @discussion When an IOService is made inactive the default behaviour is to also make any of its clients that have it as their only provider also inactive, in this way recursing the termination up the driver stack. This method allows an IOService object to override this behaviour. Returning true from this method when passed a just terminated provider will cause the client to also be terminated.
+    @discussion When an IOService is made inactive the default behavior is to also make any of its clients that have it as their only provider also inactive, in this way recursing the termination up the driver stack. This method allows an IOService object to override this behavior. Returning <code>true</code> from this method when passed a just terminated provider will cause the client to also be terminated.
     @param provider The terminated provider of this object.
-    @param options Options originally passed to terminate, plus kIOServiceRecursing.
-    @result true if this object should be terminated now that its provider as been. */
+    @param options Options originally passed to terminate, plus <code>kIOServiceRecursing</code>.
+    @result <code>true</code> if this object should be terminated now that its provider has been. */
 
     virtual bool requestTerminate( IOService * provider, IOOptionBits options );
 
@@ -300,7 +319,7 @@ public:
     @discussion Notification that a provider has been terminated, sent before recursing up the stack, in root-to-leaf order.
     @param provider The terminated provider of this object.
     @param options Options originally passed to terminate.
-    @result true return true. */
+    @result <code>true</code>. */
 
     virtual bool willTerminate( IOService * provider, IOOptionBits options );
 
@@ -309,15 +328,17 @@ public:
     @discussion Notification that a provider has been terminated, sent after recursing up the stack, in leaf-to-root order.
     @param provider The terminated provider of this object.
     @param options Options originally passed to terminate.
-    @param defer If there is pending I/O that requires this object to persist, and the provider is not opened by this object set defer to true and call the IOService::didTerminate() implementation when the I/O completes. Otherwise, leave defer set to its default value of false.
-    @result true return true. */
+    @param defer If there is pending I/O that requires this object to persist, and the provider is not opened by this object set <code>defer</code> to <code>true</code> and call the <code>IOService::didTerminate()</code> implementation when the I/O completes. Otherwise, leave <code>defer</code> set to its default value of <code>false</code>.
+    @result <code>true</code>. */
 
     virtual bool didTerminate( IOService * provider, IOOptionBits options, bool * defer );
 
-    /* method available in Mac OS X 10.4 or later */
 /*! @function nextIdleTimeout
+    @availability Mac OS X v10.4 and later
     @abstract Allows subclasses to customize idle power management behavior.
     @discussion Returns the next time that the device should idle into its next lower power state. Subclasses may override for custom idle behavior.
+    
+    A power managed driver might override this method to provide a more sophisticated idle power off algorithm than the one defined by power management.
     @param currentTime The current time
     @param lastActivity The time of last activity on this device
     @param powerState The device's current power state.
@@ -326,13 +347,23 @@ public:
     virtual SInt32 nextIdleTimeout(AbsoluteTime currentTime, 
         AbsoluteTime lastActivity, unsigned int powerState);
 
+/*! @function systemWillShutdown
+    @availability Mac OS X v10.5 and later
+    @abstract Notifies members of the power plane of system shutdown and restart.
+    @discussion This function is called for all members of the power plane in leaf-to-root order. If a subclass needs to wait for a pending I/O, then the call to <code>systemWillShutdown</code> should be postponed until the I/O completes.
+    
+    Any power managed driver (which has called @link joinPMtree joinPMtree@/link to join the power plane) interested in taking action at system shutdown or restart should override this method.
+    @param specifier <code>kIOMessageSystemWillPowerOff</code> or <code>kIOMessageSystemWillRestart</code>. */
+
+    virtual void systemWillShutdown( IOOptionBits specifier );
+
 private:
     OSMetaClassDeclareReservedUsed(IOService, 0);
     OSMetaClassDeclareReservedUsed(IOService, 1);
     OSMetaClassDeclareReservedUsed(IOService, 2);
     OSMetaClassDeclareReservedUsed(IOService, 3);
+    OSMetaClassDeclareReservedUsed(IOService, 4);
 
-    OSMetaClassDeclareReservedUnused(IOService, 4);
     OSMetaClassDeclareReservedUnused(IOService, 5);
     OSMetaClassDeclareReservedUnused(IOService, 6);
     OSMetaClassDeclareReservedUnused(IOService, 7);
@@ -399,40 +430,40 @@ private:
 public:
 /*! @function getState
     @abstract Accessor for IOService state bits, not normally needed or used outside IOService.
-    @result State bits for the IOService, eg. kIOServiceInactiveState, kIOServiceRegisteredState. */
+    @result State bits for the IOService, eg. <code>kIOServiceInactiveState</code>, <code>kIOServiceRegisteredState</code>. */
 
     virtual IOOptionBits getState( void ) const;
 
 /*! @function isInactive
-    @abstract Check the IOService has been terminated, and is in the process of being destroyed.
-    @discussion When an IOService is successfully terminated, it is immediately made inactive, which blocks further attach()es, matching or notifications occuring on the object. It remains inactive until the last client closes, and is then finalized and destroyed.
-    @result Returns true if the IOService has been terminated. */
+    @abstract Checks if the IOService object has been terminated, and is in the process of being destroyed.
+    @discussion When an IOService object is successfully terminated, it is immediately made inactive, which blocks further attach()es, matching or notifications occuring on the object. It remains inactive until the last client closes, and is then finalized and destroyed.
+    @result <code>true</code> if the IOService object has been terminated. */
 
     bool isInactive( void ) const;
 
     /* Stack creation */
 
 /*! @function registerService
-    @abstract Start the registration process for a newly discovered IOService.
-    @discussion This function allows an IOService subclass to be published and made available to possible clients, by starting the registration process and delivering notifications to registered clients. The object should be completely setup and ready to field requests from clients before registerService is called.
-    @param options The default zero options mask is recommended & should be used in most cases. The registration process is usually asynchronous, with possible driver probing & notification occurring some time later. kIOServiceSynchronous may be passed to carry out the matching and notification process for currently registered clients before returning to the caller. */
+    @abstract Starts the registration process for a newly discovered IOService object.
+    @discussion This function allows an IOService subclass to be published and made available to possible clients, by starting the registration process and delivering notifications to registered clients. The object should be completely setup and ready to field requests from clients before <code>registerService</code> is called.
+    @param options The default zero options mask is recommended and should be used in most cases. The registration process is usually asynchronous, with possible driver probing and notification occurring some time later. <code>kIOServiceSynchronous</code> may be passed to carry out the matching and notification process for currently registered clients before returning to the caller. */
 
     virtual void registerService( IOOptionBits options = 0 );
 
 /*! @function probe
-    @abstract During an IOService instantiation probe a matched service to see if it can be used.
-    @discussion The registration process for an IOService (the provider) includes instantiating possible driver clients. The probe method is called in the client instance to check the matched service can be used before the driver is considered to be started. Since matching screens many possible providers, in many cases the probe method can be left unimplemented by IOService subclasses. The client is already attached to the provider when probe is called.
-    @param provider The registered IOService which matches a driver personality's matching dictionary.
-    @param score Pointer to the current driver's probe score, which is used to order multiple matching drivers in the same match category. It defaults to the value of the IOProbeScore property in the drivers property table, or kIODefaultProbeScore if none is specified. The probe method may alter the score to affect start order.
-    @result Returns an IOService instance or zero when the probe is unsuccessful. In almost all cases the value of this is returned on success. If another IOService object is returned, the probed instance is detached and freed, and the returned instance is used in its stead for start. */
+    @abstract During an IOService object's instantiation, probes a matched service to see if it can be used.
+    @discussion The registration process for an IOService object (the provider) includes instantiating possible driver clients. The <code>probe</code> method is called in the client instance to check the matched service can be used before the driver is considered to be started. Since matching screens many possible providers, in many cases the <code>probe</code> method can be left unimplemented by IOService subclasses. The client is already attached to the provider when <code>probe</code> is called.
+    @param provider The registered IOService object that matches a driver personality's matching dictionary.
+    @param score Pointer to the current driver's probe score, which is used to order multiple matching drivers in the same match category. It defaults to the value of the <code>IOProbeScore</code> property in the drivers property table, or <code>kIODefaultProbeScore</code> if none is specified. The <code>probe</code> method may alter the score to affect start order.
+    @result An IOService instance or zero when the probe is unsuccessful. In almost all cases the value of <code>this</code> is returned on success. If another IOService object is returned, the probed instance is detached and freed, and the returned instance is used in its stead for <code>start</code>. */
     
     virtual IOService * probe(	IOService * 	provider,
 				SInt32 	  *	score );
 
 /*! @function start
-    @abstract During an IOService instantiation, the start method is called when the IOService has been selected to run on the provider.
-    @discussion The registration process for an IOService (the provider) includes instantiating possible driver clients. The start method is called in the client instance when it has been selected (by its probe score and match category) to be the winning client. The client is already attached to the provider when start is called.
-    @result Return true if the start was successful, false otherwise (which will cause the instance to be detached and usually freed). */
+    @abstract During an IOService object's instantiation, starts the IOService object that has been selected to run on the provider.
+    @discussion The <code>start</code> method of an IOService instance is called by its provider when it has been selected (due to its probe score and match category) as the winning client. The client is already attached to the provider when <code>start</code> is called.<br>Implementations of <code>start</code> must call <code>start</code> on their superclass at an appropriate point. If an implementation of <code>start</code> has already called <code>super::start</code> but subsequently determines that it will fail, it must call <code>super::stop</code> to balance the prior call to <code>super::start</code> and prevent reference leaks.
+    @result <code>true</code> if the start was successful; <code>false</code> otherwise (which will cause the instance to be detached and usually freed). */
     
     virtual bool start( IOService * provider );
     
@@ -445,102 +476,102 @@ public:
     /* Open / Close */
 
 /*! @function open
-    @abstract Request active access to a provider.
-    @discussion IOService provides generic open and close semantics to track clients of a provider that have established an active datapath. The use of open & close, and rules regarding ownership are family defined, and defined by the handleOpen / handleClose methods in the provider. Some families will limit access to a provider based on its open state.
+    @abstract Requests active access to a provider.
+    @discussion IOService provides generic open and close semantics to track clients of a provider that have established an active datapath. The use of <code>open</code> and @link close close@/link, and rules regarding ownership are family defined, and defined by the @link handleOpen handleOpen@/link and @link handleClose handleClose@/link methods in the provider. Some families will limit access to a provider based on its open state.
     @param forClient Designates the client of the provider requesting the open.
-    @param options Options for the open. The provider family may implement options for open; IOService defines only kIOServiceSeize to request the device be withdrawn from its current owner.
-    @result Return true if the open was successful, false otherwise. */
+    @param options Options for the open. The provider family may implement options for open; IOService defines only <code>kIOServiceSeize</code> to request the device be withdrawn from its current owner.
+    @result <code>true</code> if the open was successful; <code>false</code> otherwise. */
 
     virtual bool open( 	 IOService *	   forClient,
                          IOOptionBits	   options = 0,
                          void *		   arg = 0 );
 
 /*! @function close
-    @abstract Release active access to a provider.
-    @discussion IOService provides generic open and close semantics to track clients of a provider that have established an active datapath. The use of open & close, and rules regarding ownership are family defined, and defined by the handleOpen / handleClose methods in the provider.
+    @abstract Releases active access to a provider.
+    @discussion IOService provides generic open and close semantics to track clients of a provider that have established an active datapath. The use of @link open open@/link and <code>close</code>, and rules regarding ownership are family defined, and defined by the @link handleOpen handleOpen@/link and @link handleClose handleClose@/link methods in the provider.
     @param forClient Designates the client of the provider requesting the close.
     @param options Options available for the close. The provider family may implement options for close; IOService defines none.
-    @param arg Family specific arguments, ignored by IOService. */
+    @param arg Family specific arguments which are ignored by IOService. */
     
     virtual void close(  IOService *	   forClient,
                          IOOptionBits	   options = 0 );
                          
 /*! @function isOpen
-    @abstract Determine whether a specific, or any, client has an IOService open.
-    @discussion Returns the open state of an IOService with respect to the specified client, or when it is open by any client.
-    @param forClient If non-zero, isOpen returns the open state for that client. If zero is passed, isOpen returns the open state for all clients.
-    @result Returns true if the specific, or any, client has the IOService open. */
+    @abstract Determines whether a specific, or any, client has an IOService object open.
+    @discussion Returns the open state of an IOService object with respect to the specified client, or when it is open by any client.
+    @param forClient If non-zero, <codeisOpen</code returns the open state for that client. If zero is passed, <codeisOpen</code returns the open state for all clients.
+    @result <codetrue</code if the specific, or any, client has the IOService object open. */
 
     virtual bool isOpen( const IOService * forClient = 0 ) const;
 
 /*! @function handleOpen
-    @abstract Overrideable method to control the open / close behaviour of an IOService.
-    @discussion IOService calls this method in its subclasses in response to the open method, so the subclass may implement the request. The default implementation provides single owner access to an IOService via open. The object is locked via lockForArbitration before handleOpen is called.
+    @abstract Controls the open / close behavior of an IOService object (overrideable by subclasses).
+    @discussion IOService calls this method in its subclasses in response to the @link open open@/link method, so the subclass may implement the request. The default implementation provides single owner access to an IOService object via <code>open</code>. The object is locked via @link lockForArbitration lockForArbitration@/link before <code>handleOpen</code> is called.
     @param forClient Designates the client of the provider requesting the open.
-    @param options Options for the open, may be interpreted by the implementor of handleOpen.
-    @result Return true if the open was successful, false otherwise. */
+    @param options Options for the open, may be interpreted by the implementor of <code>handleOpen</code>.
+    @result <code>true</code>if the open was successful; <code>false</code> otherwise. */
 
     virtual bool handleOpen( 	IOService *	  forClient,
                                 IOOptionBits	  options,
                                 void *		  arg );
                                 
 /*! @function handleClose
-    @abstract Overrideable method to control the open / close behaviour of an IOService.
-    @discussion IOService calls this method in its subclasses in response to the close method, so the subclass may implement the request. The default implementation provides single owner access to an IOService via open. The object is locked via lockForArbitration before handleClose is called.
+    @abstract Controls the open / close behavior of an IOService object (overrideable by subclasses).
+    @discussion IOService calls this method in its subclasses in response to the @link close close@/link method, so the subclass may implement the request. The default implementation provides single owner access to an IOService object via @link open open@/link. The object is locked via @link lockForArbitration lockForArbitration@/link before <code>handleClose</code> is called.
     @param forClient Designates the client of the provider requesting the close.
-    @param options Options for the close, may be interpreted by the implementor of handleOpen. */
+    @param options Options for the close, may be interpreted by the implementor of @link handleOpen handleOpen@/link. */
 
     virtual void handleClose(   IOService *	  forClient,
                                 IOOptionBits	  options );
                                 
 /*! @function handleIsOpen
-    @abstract Overrideable method to control the open / close behaviour of an IOService.
-    @discussion IOService calls this method in its subclasses in response to the open method, so the subclass may implement the request. The default implementation provides single owner access to an IOService via open. The object is locked via lockForArbitration before handleIsOpen is called.
-    @param forClient If non-zero, isOpen returns the open state for that client. If zero is passed, isOpen returns the open state for all clients.
-    @result Returns true if the specific, or any, client has the IOService open. */
+    @abstract Controls the open / close behavior of an IOService object (overrideable by subclasses).
+    @discussion IOService calls this method in its subclasses in response to the @link open open@/link method, so the subclass may implement the request. The default implementation provides single owner access to an IOService object via @link open open@/link. The object is locked via @link lockForArbitration lockForArbitration@/link before <code>handleIsOpen</code> is called.
+    @param forClient If non-zero, <code>isOpen</code> returns the open state for that client. If zero is passed, <code>isOpen</code> returns the open state for all clients.
+    @result <code>true</code> if the specific, or any, client has the IOService object open. */
 
     virtual bool handleIsOpen(  const IOService * forClient ) const;
 
     /* Stacking change */
 
 /*! @function terminate
-    @abstract Make an IOService inactive and begin its destruction.
-    @discussion Registering an IOService informs possible clients of its existance and instantiates drivers that may be used with it; terminate involves the opposite process of informing clients that an IOService is no longer able to be used and will be destroyed. By default, if any client has the service open, terminate fails. If the kIOServiceRequired flag is passed however, terminate will be sucessful though further progress in the destruction of the IOService will not proceed until the last client has closed it. The service will be made inactive immediately upon successful termination, and all its clients will be notified via their message method with a message of type kIOMessageServiceIsTerminated. Both these actions take place on the callers thread. After the IOService is made inactive, further matching or attach calls will fail on it. Each client has its stop method called upon their close of an inactive IOService, or on its termination if they do not have it open. After stop, detach is called in each client. When all clients have been detached, the finalize method is called in the inactive service. The terminate process is inherently asynchronous since it will be deferred until all clients have chosen to close.
-    @param options In most cases no options are needed. kIOServiceSynchronous may be passed to cause terminate to not return until the service is finalized. */
+    @abstract Makes an IOService object inactive and begins its destruction.
+    @discussion Registering an IOService object informs possible clients of its existance and instantiates drivers that may be used with it; <code>terminate</code> involves the opposite process of informing clients that an IOService object is no longer able to be used and will be destroyed. By default, if any client has the service open, <code>terminate</code> fails. If the <code>kIOServiceRequired</code> flag is passed however, <code>terminate</code> will be successful though further progress in the destruction of the IOService object will not proceed until the last client has closed it. The service will be made inactive immediately upon successful termination, and all its clients will be notified via their @link message message@/link method with a message of type <code>kIOMessageServiceIsTerminated</code>. Both these actions take place on the caller's thread. After the IOService object is made inactive, further matching or attach calls will fail on it. Each client has its @link stop stop@/link method called upon their close of an inactive IOService object , or on its termination if they do not have it open. After <code>stop</code>, @link detach detach@/link is called in each client. When all clients have been detached, the @link finalize finalize@/link method is called in the inactive service. The termination process is inherently asynchronous because it will be deferred until all clients have chosen to close.
+    @param options In most cases no options are needed. <code>kIOServiceSynchronous</code> may be passed to cause <code>terminate</code> to not return until the service is finalized. */
 
     virtual bool terminate( IOOptionBits options = 0 );
 
 /*! @function finalize
-    @abstract The last stage in an IOService destruction.
-    @discussion The finalize method is called in an inactive (ie. terminated) IOService after the last client has detached. IOService's implementation will call stop, close, and detach on each provider. When finalize returns, the object's retain count will have no references generated by IOService's registration process.
-    @param options The options passed to the terminate method of the IOService are passed on to finalize.
-    @result Returns true. */
+    @abstract Finalizes the destruction of an IOService object.
+    @discussion The <code>finalize</code> method is called in an inactive (ie. terminated) IOService object after the last client has detached. IOService's implementation will call @link stop stop@/link, @link close close@/link, and @link detach detach@/link on each provider. When <code>finalize</code> returns, the object's retain count will have no references generated by IOService's registration process.
+    @param options The options passed to the @link terminate terminate@/link method of the IOService object are passed on to <code>finalize</code>.
+    @result <code>true</code>. */
     
     virtual bool finalize( IOOptionBits options );
 
 /*! @function free
-    @discussion Free data structures that were allocated when power management was initialized on this service. */
+    @abstract Frees data structures that were allocated when power management was initialized on this service. */
     
     virtual void free( void );
 
 /*! @function lockForArbitration
-    @abstract Locks an IOService against changes in state or ownership.
-    @discussion The registration, termination and open / close functions of IOService use lockForArbtration to single thread access to an IOService. lockForArbitration will grant recursive access to the same thread.
-    @param isSuccessRequired If a request for access to an IOService should be denied if it is terminated, isSuccessRequired should passed as false, otherwise pass true. */
+    @abstract Locks an IOService object against changes in state or ownership.
+    @discussion The registration, termination and open / close functions of IOService use <code>lockForArbtration</code> to single-thread access to an IOService object. <code>lockForArbitration</code> grants recursive access to the same thread.
+    @param isSuccessRequired If a request for access to an IOService object should be denied if it is terminated, pass <code>false</code>, otherwise pass <code>true</code>. */
     
     virtual bool lockForArbitration( bool isSuccessRequired = true );
     
 /*! @function unlockForArbitration
-    @abstract Unlocks an IOService after a successful lockForArbitration.
-    @discussion A thread granted exclusive access to an IOService should release it with unlockForArbitration. */
+    @abstract Unlocks an IOService obkect after a successful @link lockForArbitration lockForArbitration@/link.
+    @discussion A thread granted exclusive access to an IOService object should release it with <code>unlockForArbitration</code>. */
     
     virtual void unlockForArbitration( void );
 
 /*! @function terminateClient
     @abstract Passes a termination up the stack.
-    @discussion When an IOService is made inactive the default behaviour is to also make any of its clients that have it as their only provider also inactive, in this way recursing the termination up the driver stack. This method allows a terminated  IOService to override this behaviour. Note the client may also override this behaviour by overriding its terminate method.
-    @param client The client of the of the terminated provider.
-    @param options Options originally passed to terminate, plus kIOServiceRecursing.
+    @discussion When an IOService object is made inactive the default behavior is to also make any of its clients that have it as their only provider inactive, in this way recursing the termination up the driver stack. This method allows a terminated  IOService object to override this behavior. Note the client may also override this behavior by overriding its @link terminate terminate@/link method.
+    @param client The client of the terminated provider.
+    @param options Options originally passed to @link terminate terminate@/link, plus <code>kIOServiceRecursing</code>.
     @result result of the terminate request on the client. */
 
     virtual bool terminateClient( IOService * client, IOOptionBits options );
@@ -548,35 +579,35 @@ public:
     /* Busy state indicates discovery, matching or termination is in progress */
 
 /*! @function getBusyState
-    @abstract Returns the busyState of an IOService.
-    @discussion Many activities in IOService are asynchronous. When registration, matching, or termination is in progress on an IOService, its busyState is increased by one. Change in busyState to or from zero also changes the IOService's provider's busyState by one, which means that an IOService is marked busy when any of the above activities is ocurring on it or any of its clients.
-    @result The busyState. */
+    @abstract Returns the <code>busyState</code> of an IOService object.
+    @discussion Many activities in IOService are asynchronous. When registration, matching, or termination is in progress on an IOService object, its <code>busyState</code> is increased by one. Change in <code>busyState</code> to or from zero also changes the IOService object's provider's <code>busyState</code> by one, which means that an IOService object is marked busy when any of the above activities is ocurring on it or any of its clients.
+    @result The <code>busyState</code> value. */
 
     virtual UInt32 getBusyState( void );
     
 /*! @function adjustBusy
-    @abstract Adjusts the busyState of an IOService.
-    @discussion Applies a delta to an IOService's busyState. A change in the busyState to or from zero will change the IOService's provider's busyState by one (in the same direction). 
-    @param delta The delta to be applied to the IOService busy state. */
+    @abstract Adjusts the <code>busyState</code> of an IOService object.
+    @discussion Applies a delta to an IOService object's <code>busyState</code>. A change in the <code>busyState</code> to or from zero will change the IOService object's provider's <code>busyState</code> by one (in the same direction). 
+    @param delta The delta to be applied to the IOService object's <code>busyState</code>. */
 
     virtual void adjustBusy( SInt32 delta );
 
 /*! @function waitQuiet
-    @abstract Wait for an IOService's busyState to be zero.
-    @discussion Blocks the caller until an IOService is non busy.
+    @abstract Waits for an IOService object's <code>busyState</code> to be zero.
+    @discussion Blocks the caller until an IOService object is non busy.
     @param timeout Specifies a maximum time to wait.
-    @result Returns an error code if mach synchronization primitives fail, kIOReturnTimeout, or kIOReturnSuccess. */
+    @result Returns an error code if Mach synchronization primitives fail, <code>kIOReturnTimeout</code>, or <code>kIOReturnSuccess</code>. */
     
     virtual IOReturn waitQuiet( mach_timespec_t * timeout = 0 );
 
     /* Matching */
 
 /*! @function matchPropertyTable
-    @abstract Allows a registered IOService to implement family specific matching.
-    @discussion All matching on an IOService will call this method to allow a family writer to implement matching in addition to the generic methods provided by IOService. The implementer should examine the matching dictionary passed to see if it contains properties the family understands for matching, and use them to match with the IOService if so. Note that since matching is also carried out by other parts of IOKit, the matching dictionary may contain properties the family does not understand - these should not be considered matching failures.
+    @abstract Allows a registered IOService object to implement family specific matching.
+    @discussion All matching on an IOService object will call this method to allow a family writer to implement matching in addition to the generic methods provided by IOService. The implementer should examine the matching dictionary passed to see if it contains properties the family understands for matching, and use them to match with the IOService object if so. Note that since matching is also carried out by other parts of the I/O Kit, the matching dictionary may contain properties the family does not understand - these should not be considered matching failures.
     @param table The dictionary of properties to be matched against.
-    @param score Pointer to the current driver's probe score, which is used to order multiple matching drivers in the same match category. It defaults to the value of the IOProbeScore property in the drivers property table, or kIODefaultProbeScore if none is specified.
-    @result Returns false if the family considers the matching dictionary does not match in properties it understands, true otherwise. */
+    @param score Pointer to the current driver's probe score, which is used to order multiple matching drivers in the same match category. It defaults to the value of the <code>IOProbeScore</code> property in the drivers property table, or <code>kIODefaultProbeScore</code> if none is specified.
+    @result <code>false</code> if the family considers the matching dictionary does not match in properties it understands; <code>true</code> otherwise. */
 
     virtual bool matchPropertyTable( OSDictionary *	table,
                                      SInt32       *	score );
@@ -584,9 +615,9 @@ public:
     virtual bool matchPropertyTable( OSDictionary * table );
 
 /*! @function matchLocation
-    @abstract Allows a registered IOService to direct location matching.
-    @discussion By default, a location matching property will be applied to an IOService's provider. This method allows that behaviour to be overridden by families.
-    @param client The IOService at which matching is taking place.
+    @abstract Allows a registered IOService object to direct location matching.
+    @discussion By default, a location matching property will be applied to an IOService object's provider. This method allows that behavior to be overridden by families.
+    @param client The IOService object at which matching is taking place.
     @result Returns the IOService instance to be used for location matching. */
 
     virtual IOService * matchLocation( IOService * client );
@@ -594,17 +625,17 @@ public:
     /* Resource service */
 
 /*! @function publishResource
-    @abstract Use the resource service to publish a property.
-    @discussion The resource service uses IOService's matching and notification to allow objects to be published and found by any IOKit client by a global name. publishResource makes an object available to anyone waiting for it or looking for it in the future.
+    @abstract Uses the resource service to publish a property.
+    @discussion The resource service uses IOService's matching and notification to allow objects to be published and found by any I/O Kit client by a global name. <code>publishResource</code> makes an object available to anyone waiting for it or looking for it in the future.
     @param key An OSSymbol key that globally identifies the object.
     @param The object to be published. */
 
     static void publishResource( const OSSymbol * key, OSObject * value = 0 );
 
 /*! @function publishResource
-    @abstract Use the resource service to publish a property.
-    @discussion The resource service uses IOService's matching and notification to allow objects to be published and found by any IOKit client by a global name. publishResource makes an object available to anyone waiting for it or looking for it in the future.
-    @param key A C-string key that globally identifies the object.
+    @abstract Uses the resource service to publish a property.
+    @discussion The resource service uses IOService object's matching and notification to allow objects to be published and found by any I/O Kit client by a global name. <code>publishResource</code> makes an object available to anyone waiting for it or looking for it in the future.
+    @param key A C string key that globally identifies the object.
     @param The object to be published. */
 
     static void publishResource( const char * key, OSObject * value = 0 );
@@ -613,20 +644,20 @@ public:
     /* Notifications */
 
 /*! @function addNotification
-    @abstract Add a persistant notification handler to be notified of IOService events.
-    @discussion IOService will deliver notifications of changes in state of an IOService to registered clients. The type of notification is specified by a symbol, for example gIOMatchedNotification or gIOTerminatedNotification, and notifications will only include IOService's that match the supplied matching dictionary. Notifications are ordered by a priority set with addNotification. When the notification is installed, its handler will be called with each of any currently existing IOService's that are in the correct state (eg. registered) and match the supplied matching dictionary, avoiding races between finding preexisting and new IOService events. The notification request is identified by an instance of an IONotifier object, through which it can be enabled, disabled or removed. addNotification will consume a retain count on the matching dictionary when the notification is removed.
+    @abstract Adds a persistant notification handler to be notified of IOService events.
+    @discussion IOService will deliver notifications of changes in state of an IOService object to registered clients. The type of notification is specified by a symbol, for example <code>gIOMatchedNotification</code> or <code>gIOTerminatedNotification</code>, and notifications will only include IOService objects that match the supplied matching dictionary. Notifications are ordered by a priority set with <code>addNotification</code>. When the notification is installed, its handler will be called with each of any currently existing IOService objects that are in the correct state (eg. registered) and match the supplied matching dictionary, avoiding races between finding preexisting and new IOService events. The notification request is identified by an instance of an IONotifier object, through which it can be enabled, disabled, or removed. <code>addNotification</code> consumes a retain count on the matching dictionary when the notification is removed.
     @param type An OSSymbol identifying the type of notification and IOService state:
-<br>	gIOPublishNotification Delivered when an IOService is registered.
-<br>	gIOFirstPublishNotification Delivered when an IOService is registered, but only once per IOService instance. Some IOService's may be reregistered when their state is changed.
-<br>	gIOMatchedNotification Delivered when an IOService has been matched with all client drivers, and they have been probed and started.
-<br>	gIOFirstMatchNotification Delivered when an IOService has been matched with all client drivers, but only once per IOService instance. Some IOService's may be reregistered when their state is changed.
-<br>	gIOTerminatedNotification Delivered after an IOService has been terminated, during its finalize stage.
-    @param matching A matching dictionary to restrict notifications to only matching IOServices. The dictionary will be released when the notification is removed - consuming the passed in reference.
-    @param handler A C-function callback to deliver notifications.
-    @param target An instance reference for the callbacks use.
-    @param ref A reference constant for the callbacks use
+<br>	<code>gIOPublishNotification</code> Delivered when an IOService object is registered.
+<br>	<code>gIOFirstPublishNotification</code> Delivered when an IOService object is registered, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed.
+<br>	<code>gIOMatchedNotification</code> Delivered when an IOService object has been matched with all client drivers, and they have been probed and started.
+<br>	<code>gIOFirstMatchNotification</code> Delivered when an IOService object has been matched with all client drivers, but only once per IOService instance. Some IOService objects may be reregistered when their state is changed.
+<br>	<code>gIOTerminatedNotification</code> Delivered after an IOService object has been terminated, during its finalize stage.
+    @param matching A matching dictionary to restrict notifications to only matching IOService objects. The dictionary will be released when the notification is removed, consuming the passed-in reference.
+    @param handler A C function callback to deliver notifications.
+    @param target An instance reference for the callback's use.
+    @param ref A reference constant for the callback's use.
     @param priority A constant ordering all notifications of a each type.
-    @result Returns an instance of an IONotifier object that can be used to control or destroy the notification request. */
+    @result An instance of an IONotifier object that can be used to control or destroy the notification request. */
 
     static IONotifier * addNotification(
                             const OSSymbol * type, OSDictionary * matching,
@@ -635,34 +666,34 @@ public:
                             SInt32 priority = 0 );
 
 /*! @function waitForService
-    @abstract Wait for a matching to service to be published.
-    @discussion Provides a method of waiting for an IOService matching the supplied matching dictionary to be registered and fully matched. 
-    @param matching The matching dictionary describing the desired IOService. waitForService will consume one reference of the matching dictionary.
+    @abstract Waits for a matching to service to be published.
+    @discussion Provides a method of waiting for an IOService object matching the supplied matching dictionary to be registered and fully matched. 
+    @param matching The matching dictionary describing the desired IOService object. <code>waitForService</code> consumes one reference of the matching dictionary.
     @param timeout The maximum time to wait.
-    @result A published IOService matching the supplied dictionary. */
+    @result A published IOService object matching the supplied dictionary. */
 
     static IOService * waitForService( OSDictionary * matching,
                             mach_timespec_t * timeout = 0);
 
 /*! @function getMatchingServices
-    @abstract Finds the set of current published IOServices matching a matching dictionary.
-    @discussion Provides a method of finding the current set of published IOServices matching the supplied matching dictionary.   
-    @param matching The matching dictionary describing the desired IOServices.
-    @result An instance of an iterator over a set of IOServices. To be released by the caller. */
+    @abstract Finds the set of current published IOService objects matching a matching dictionary.
+    @discussion Provides a method of finding the current set of published IOService objects matching the supplied matching dictionary.   
+    @param matching The matching dictionary describing the desired IOService objects.
+    @result An instance of an iterator over a set of IOService objects. To be released by the caller. */
 
     static OSIterator * getMatchingServices( OSDictionary * matching );
 
 /*! @function installNotification
-    @abstract Add a persistant notification handler to be notified of IOService events.
-    @discussion A lower level interface to addNotification that will install a handler and return the current set of IOServices that are in the specified state and match the matching dictionary.
-    @param type See addNotification.
-    @param matching See addNotification.
-    @param handler See addNotification.
-    @param self See addNotification.
-    @param ref See addNotification.
-    @param priority See addNotification.
-    @param existing Returns an iterator over the set of IOServices that are currently in the specified state and match the matching dictionary.
-    @result See addNotification.  */
+    @abstract Adds a persistant notification handler to be notified of IOService events.
+    @discussion A lower level interface to @link addNotification addNotification@/link that installs a handler and returns the current set of IOService objects that are in the specified state and match the matching dictionary.
+    @param type See <code>addNotification</code>.
+    @param matching See <code>addNotification</code>.
+    @param handler See <code>addNotification</code>.
+    @param self See <code>addNotification</code>.
+    @param ref See <code>addNotification</code>.
+    @param priority See <code>addNotification</code>.
+    @param existing Returns an iterator over the set of IOService objects that are currently in the specified state and match the matching dictionary.
+    @result See <code>addNotification</code>.  */
 
     static IONotifier * installNotification(
 			const OSSymbol * type, OSDictionary * matching,
@@ -674,68 +705,80 @@ public:
      * they add keys to an existing dictionary, or create one. */
 
 /*! @function serviceMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify an IOService class match.
-    @discussion A very common matching criteria for IOService is based on its class. serviceMatching will create a matching dictionary that specifies any IOService of a class, or its subclasses. The class is specified by name, and an existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
-    @param className The class name, as a const C-string. Class matching is successful on IOService's of this class or any subclass.
-    @param table If zero, serviceMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify an IOService class match.
+    @discussion A very common matching criteria for IOService object is based on its class. <code>serviceMatching</code> creates a matching dictionary that specifies any IOService object of a class, or its subclasses. The class is specified by name, and an existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
+    @param className The class name, as a const C string. Class matching is successful on IOService objects of this class or any subclass.
+    @param table If zero, <code>serviceMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * serviceMatching( const char * className,
 			OSDictionary * table = 0 );
 
 /*! @function serviceMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify an IOService class match.
-    @discussion A very common matching criteria for IOService is based on its class. serviceMatching will create a matching dictionary that specifies any IOService of a class, or its subclasses. The class is specified by name, and an existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
-    @param className The class name, as an OSString (which includes OSSymbol). Class matching is successful on IOService's of this class or any subclass.
-    @param table If zero, serviceMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify an IOService class match.
+    @discussion A very common matching criteria for IOService object is based on its class. <code>serviceMatching</code> creates a matching dictionary that specifies any IOService of a class, or its subclasses. The class is specified by name, and an existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
+    @param className The class name, as an OSString (which includes OSSymbol). Class matching is successful on IOService objects of this class or any subclass.
+    @param table If zero, <code>serviceMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * serviceMatching( const OSString * className,
 			OSDictionary * table = 0 );
 
 /*! @function nameMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify an IOService name match.
-    @discussion A very common matching criteria for IOService is based on its name. nameMatching will create a matching dictionary that specifies any IOService which respond sucessfully to the IORegistryEntry method compareName. An existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
-    @param name The service's name, as a const C-string. Name matching is successful on IOService's which respond sucessfully to the IORegistryEntry method compareName.
-    @param table If zero, nameMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify an IOService name match.
+    @discussion A very common matching criteria for IOService object is based on its name. <code>nameMatching</code> creates a matching dictionary that specifies any IOService object which responds successfully to the @link //apple_ref/cpp/instm/IORegistryEntry/compareName/virtualbool/(OSString*,OSString**) IORegistryEntry::compareName@/link method. An existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
+    @param name The service's name, as a const C string. Name matching is successful on IOService objects that respond successfully to the <code>IORegistryEntry::compareName</code> method.
+    @param table If zero, <code>nameMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * nameMatching( const char * name,
 			OSDictionary * table = 0 );
 
 /*! @function nameMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify an IOService name match.
-    @discussion A very common matching criteria for IOService is based on its name. nameMatching will create a matching dictionary that specifies any IOService which respond sucessfully to the IORegistryEntry method compareName. An existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
-    @param name The service's name, as an OSString (which includes OSSymbol). Name matching is successful on IOService's which respond sucessfully to the IORegistryEntry method compareName.
-    @param table If zero, nameMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify an IOService name match.
+    @discussion A very common matching criteria for IOService object is based on its name. <code>nameMatching</code> creates a matching dictionary that specifies any IOService object which responds successfully to the @link //apple_ref/cpp/instm/IORegistryEntry/compareName/virtualbool/(OSString*,OSString**) IORegistryEntry::compareName@/link method. An existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
+    @param name The service's name, as an OSString (which includes OSSymbol). Name matching is successful on IOService objects that respond successfully to the <code>IORegistryEntry::compareName</code> method.
+    @param table If zero, <code>nameMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * nameMatching( const OSString* name,
 			OSDictionary * table = 0 );
 
 /*! @function resourceMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify a resource service match.
-    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in IOKit based on a name, using the standard IOService matching and notification calls.
-    @param name The resource name, as a const C-string. Resource matching is successful when an object by that name has been published with the publishResource method.
-    @param table If zero, resourceMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify a resource service match.
+    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in the I/O Kit based on a name, using the standard IOService matching and notification calls.
+    @param name The resource name, as a const C string. Resource matching is successful when an object by that name has been published with the <code>publishResource</code> method.
+    @param table If zero, <code>resourceMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * resourceMatching( const char * name,
 			OSDictionary * table = 0 );
 
 /*! @function resourceMatching
-    @abstract Create a matching dictionary, or add matching properties to an existing dictionary, that specify a resource service match.
-    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in IOKit based on a name, using the standard IOService matching and notification calls.
-    @param name The resource name, as an OSString (which includes OSSymbol). Resource matching is successful when an object by that name has been published with the publishResource method.
-    @param table If zero, resourceMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify a resource service match.
+    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in the I/O Kit based on a name, using the standard IOService matching and notification calls.
+    @param name The resource name, as an OSString (which includes OSSymbol). Resource matching is successful when an object by that name has been published with the <code>publishResource</code> method.
+    @param table If zero, <code>resourceMatching</code> creates a matching dictionary and returns a reference to it, otherwise the matching properties are added to the specified dictionary.
     @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
 
     static OSDictionary * resourceMatching( const OSString * name,
 			OSDictionary * table = 0 );
 
+
+/*! @function propertyMatching
+    @abstract Creates a matching dictionary, or adds matching properties to an existing dictionary, that specify an IOService phandle match.
+    @discussion TODO A very common matching criteria for IOService is based on its name. nameMatching will create a matching dictionary that specifies any IOService which respond successfully to the IORegistryEntry method compareName. An existing dictionary may be passed in, in which case the matching properties will be added to that dictionary rather than creating a new one.
+    @param key The service's phandle, as a const UInt32. PHandle matching is successful on IOService objects that respond successfully to the IORegistryEntry method compareName.
+    @param value The service's phandle, as a const UInt32. PHandle matching is successful on IOService's which respond successfully to the IORegistryEntry method compareName.
+    @param table If zero, nameMatching will create a matching dictionary and return a reference to it, otherwise the matching properties are added to the specified dictionary.
+    @result The matching dictionary created, or passed in, is returned on success, or zero on failure. */
+
+    static OSDictionary * propertyMatching( const OSSymbol * key, const OSObject * value,
+			OSDictionary * table = 0 );
+
 /*! @function addLocation
-    @abstract Add a location matching property to an existing dictionary.
-    @discussion This function creates matching properties that specify the location of a IOService, as an embedded matching dictionary. This matching will be successful on an IOService which attached to an IOService which matches this location matching dictionary.
+    @abstract Adds a location matching property to an existing dictionary.
+    @discussion This function creates matching properties that specify the location of a IOService object, as an embedded matching dictionary. This matching will be successful on an IOService object that attached to an IOService object which matches this location matching dictionary.
     @param table The matching properties are added to the specified dictionary, which must be non-zero.
     @result The location matching dictionary created is returned on success, or zero on failure. */
 
@@ -744,30 +787,30 @@ public:
     /* Helpers for matching dictionaries. */
 
 /*! @function compareProperty
-    @abstract Utility to compare a property in a matching dictionary with an IOService's property table.
-    @discussion This is a helper function to aid in implementing matchPropertyTable. If the property specified by key exists in the matching dictionary, it is compared with a property of the same name in the IOService's property table. The comparison is performed with the isEqualTo method. If the property does not exist in the matching table, success is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned.
+    @abstract Compares a property in a matching dictionary with an IOService object's property table.
+    @discussion This is a helper function to aid in implementing @link matchPropertyTable matchPropertyTable@/link. If the property specified by <code>key</code> exists in the matching dictionary, it is compared with a property of the same name in the IOService object's property table. The comparison is performed with the <code>isEqualTo</code> method. If the property does not exist in the matching table, success is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned.
     @param matching The matching dictionary, which must be non-zero.
-    @param key The dictionary key specifying the property to be compared, as a C-string.
-    @result If the property does not exist in the matching table, true is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned. Otherwise the result of calling the property from the matching dictionary's isEqualTo method with the IOService property as an argument is returned. */
+    @param key The dictionary key specifying the property to be compared, as a C string.
+    @result <code>true</code> if the property does not exist in the matching table. If the property exists in the matching dictionary but not the IOService property table, failure is returned. Otherwise the result of calling the property from the matching dictionary's <code>isEqualTo</code> method with the IOService property as an argument is returned. */
 
     virtual bool compareProperty(   OSDictionary   * matching,
                                     const char     * key );
 /*! @function compareProperty
-    @abstract Utility to compare a property in a matching dictionary with an IOService's property table.
-    @discussion This is a helper function to aid in implementing matchPropertyTable. If the property specified by key exists in the matching dictionary, it is compared with a property of the same name in the IOService's property table. The comparison is performed with the isEqualTo method. If the property does not exist in the matching table, success is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned.
+    @abstract Compares a property in a matching dictionary with an IOService object's property table.
+    @discussion This is a helper function to aid in implementing @link matchPropertyTable matchPropertyTable@/link. If the property specified by <code>key</code> exists in the matching dictionary, it is compared with a property of the same name in the IOService object's property table. The comparison is performed with the <code>isEqualTo</code> method. If the property does not exist in the matching table, success is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned.
     @param matching The matching dictionary, which must be non-zero.
     @param key The dictionary key specifying the property to be compared, as an OSString (which includes OSSymbol).
-    @result If the property does not exist in the matching table, true is returned. If the property exists in the matching dictionary but not the IOService property table, failure is returned. Otherwise the result of calling the property from the matching dictionary's isEqualTo method with the IOService property as an argument is returned. */
+    @result <code>true</code> if the property does not exist in the matching table. If the property exists in the matching dictionary but not the IOService property table, failure is returned. Otherwise the result of calling the property from the matching dictionary's <code>isEqualTo</code> method with the IOService property as an argument is returned. */
 
     virtual bool compareProperty(   OSDictionary   * matching,
                                     const OSString * key );
 
 /*! @function compareProperties
-    @abstract Utility to compare a set of properties in a matching dictionary with an IOService's property table.
-    @discussion This is a helper function to aid in implementing matchPropertyTable. A collection of dictionary keys specifies properties in a matching dictionary to be compared, with compareProperty, with an IOService property table, if compareProperty returns true for each key, success is return else failure.
+    @abstract Compares a set of properties in a matching dictionary with an IOService object's property table.
+    @discussion This is a helper function to aid in implementing @link matchPropertyTable matchPropertyTable@/link. A collection of dictionary keys specifies properties in a matching dictionary to be compared, with <code>compareProperty</code>, with an IOService object's property table, if <code>compareProperty</code> returns <code>true</code> for each key, success is returned; otherwise failure.
     @param matching The matching dictionary, which must be non-zero.
     @param keys A collection (eg. OSSet, OSArray, OSDictionary) which should contain OSStrings (or OSSymbols) that specify the property keys to be compared.
-    @result if compareProperty returns true for each key in the collection, success is return else failure. */
+    @result Success if <code>compareProperty</code> returns <code>true</code> for each key in the collection; otherwise failure. */
 
     virtual bool compareProperties( OSDictionary   * matching,
                                     OSCollection   * keys );
@@ -775,75 +818,75 @@ public:
     /* Client / provider accessors */
 
 /*! @function attach
-    @abstract Attaches an IOService client to a provider in the registry.
-    @discussion This function called in an IOService client enters the client into the registry as a child of the provider in the service plane. The provider must be active or the attach will fail. Multiple attach calls to the same provider are no-ops and return success. A client may be attached to multiple providers. Entering an object into the registry will retain both the client and provider until they are detached.
-    @param provider The IOService object which will serve as this objects provider.
-    @result false if the provider is inactive or on a resource failure, otherwise true. */
+    @abstract Attaches an IOService client to a provider in the I/O Registry.
+    @discussion This function called in an IOService client enters the client into the I/O Registry as a child of the provider in the service plane. The provider must be active or the attach will fail. Multiple attach calls to the same provider are no-ops and return success. A client may be attached to multiple providers. Entering an object into the I/O Registry retains both the client and provider until they are detached.
+    @param provider The IOService object which will serve as this object's provider.
+    @result <code>false</code> if the provider is inactive or on a resource failure; otherwise <code>true</code>. */
 
     virtual bool attach( IOService * provider );
     
 /*! @function detach
-    @abstract Detaches an IOService client from a provider in the registry.
-    @discussion This function called in an IOService client removes the client as a child of the provider in the service plane of the registry. If the provider is not a parent of the client this is a no-op, otherwise the registry will release both the client and provider.
+    @abstract Detaches an IOService client from a provider in the I/O Registry.
+    @discussion This function called in an IOService client removes the client as a child of the provider in the service plane of the I/O Registry. If the provider is not a parent of the client this is a no-op, otherwise the I/O Registry releases both the client and provider.
     @param provider The IOService object to detach from. */
 
     virtual void detach( IOService * provider );
 
 /*! @function getProvider
-    @abstract Returns an IOService's primary provider.
-    @discussion This function called in an IOService client will return the provider to which it was first attached. Since the majority of IOService objects have only one provider, this is a useful simplification and also supports caching of the provider when the registry is unchanged.
-    @result Returns the first provider of the client, or zero if the IOService is not attached into the registry. The provider is retained while the client is attached, and should not be released by the caller. */
+    @abstract Returns an IOService object's primary provider.
+    @discussion This function called in an IOService client will return the provider to which it was first attached. Because the majority of IOService objects have only one provider, this is a useful simplification and also supports caching of the provider when the I/O Registry is unchanged.
+    @result The first provider of the client, or zero if the IOService object is not attached into the I/O Registry. The provider is retained while the client is attached, and should not be released by the caller. */
 
     virtual IOService * getProvider( void ) const;
 
 /*! @function getWorkLoop
-    @abstract Returns the current work loop or provider->getWorkLoop().
-    @discussion This function returns a valid work loop that a client can use to add an IOCommandGate to.  The intention is that an IOService client has data that needs to be protected but doesn't want to pay the cost of an entire dedicated thread.  This data has to be accessed from a providers call out context as well.  So to achieve both of these goals the client creates an IOCommandGate to lock access to his data but he registers it with the providers work loop, i.e. the work loop which will make the completion call outs.  In one fell swoop we avoid a potentially nasty deadlock 'cause a work loop's gate is recursive.
-    @result Always returns a work loop, either the current work loop or it walks up the $link getProvider() chain calling getWorkLoop.  Eventually it will reach a valid work loop based driver or the root of the io tree where it will return a system wide work loop.  Returns 0 if it fails to find (or create) */
+    @abstract Returns the current work loop or <code>provider->getWorkLoop</code>.
+    @discussion This function returns a valid work loop that a client can use to add an IOCommandGate to. The intention is that an IOService client has data that needs to be protected but doesn't want to pay the cost of a dedicated thread. This data has to be accessed from a provider's call-out context as well. So to achieve both of these goals the client creates an IOCommandGate to lock access to his data but he registers it with the provider's work loop, i.e. the work loop which will make the completion call-outs. This avoids a potential deadlock because the work loop gate uses a recursive lock, which allows the same lock to be held multiple times by a single thread.
+    @result A work loop, either the current work loop or it walks up the @link getProvider getProvider@/link chain calling <code>getWorkLoop</code>. Eventually it will reach a valid work loop-based driver or the root of the I/O tree, where it will return a system-wide work loop. Returns 0 if it fails to find (or create) a work loop.*/
 
     virtual IOWorkLoop * getWorkLoop() const;
 
 /*! @function getProviderIterator
-    @abstract Returns an iterator over an IOService's providers.
+    @abstract Returns an iterator over an IOService object's providers.
     @discussion For those few IOService objects that obtain service from multiple providers, this method supplies an iterator over a client's providers. 
-    @result Returns an iterator over the providers of the client, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, though they may no longer be attached during the iteration. */
+    @result An iterator over the providers of the client, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, though they may no longer be attached during the iteration. */
 
     virtual OSIterator * getProviderIterator( void ) const;
 
 /*! @function getOpenProviderIterator
     @abstract Returns an iterator over an client's providers that are currently opened by the client.
-    @discussion For those few IOService objects that obtain service from multiple providers, this method supplies an iterator over a client's providers, locking each in turn with lockForArbitration and returning those that have been opened by the client. 
-    @result Returns an iterator over the providers the client has open, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, and the current entry in the iteration is locked with lockForArbitration, protecting it from state changes. */
+    @discussion For those few IOService objects that obtain service from multiple providers, this method supplies an iterator over a client's providers, locking each in turn with @link lockForArbitration lockForArbitration@/link and returning those that have been opened by the client. 
+    @result An iterator over the providers the client has open, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, and the current entry in the iteration is locked with <code>lockForArbitration</code>, protecting it from state changes. */
 
     virtual OSIterator * getOpenProviderIterator( void ) const;
 
 /*! @function getClient
-    @abstract Returns an IOService's primary client.
+    @abstract Returns an IOService object's primary client.
     @discussion This function called in an IOService provider will return the first client to attach to it. For IOService objects which have only only one client, this may be a useful simplification.
-    @result Returns the first client of the provider, or zero if the IOService is not attached into the registry. The client is retained while it is attached, and should not be released by the caller. */
+    @result The first client of the provider, or zero if the IOService object is not attached into the I/O Registry. The client is retained while it is attached, and should not be released by the caller. */
 
     virtual IOService * getClient( void ) const;
 
 /*! @function getClientIterator
-    @abstract Returns an iterator over an IOService's clients.
+    @abstract Returns an iterator over an IOService object's clients.
     @discussion For IOService objects that may have multiple clients, this method supplies an iterator over a provider's clients. 
-    @result Returns an iterator over the clients of the provider, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, though they may no longer be attached during the iteration. */
+    @result An iterator over the clients of the provider, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, though they may no longer be attached during the iteration. */
 
     virtual OSIterator * getClientIterator( void ) const;
 
 /*! @function getOpenClientIterator
-    @abstract Returns an iterator over an provider's clients that currently have opened the provider.
-    @discussion For IOService objects that may have multiple clients, this method supplies an iterator over a provider's clients, locking each in turn with lockForArbitration and returning those that have opened the provider. 
-    @result Returns an iterator over the clients which the provider open, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, and the current entry in the iteration is locked with lockForArbitration, protecting it from state changes. */
+    @abstract Returns an iterator over a provider's clients that currently have opened the provider.
+    @discussion For IOService objects that may have multiple clients, this method supplies an iterator over a provider's clients, locking each in turn with @link lockForArbitration lockForArbitration@/link and returning those that have opened the provider. 
+    @result An iterator over the clients that have opened the provider, or zero if there is a resource failure. The iterator must be released when the iteration is finished. All objects returned by the iteration are retained while the iterator is valid, and the current entry in the iteration is locked with <code>lockForArbitration</code>, protecting it from state changes. */
 
     virtual OSIterator * getOpenClientIterator( void ) const;
 
 /*! @function callPlatformFunction
     @abstract Calls the platform function with the given name.
-    @discussion The platform expert or other drivers may implement various functions to control hardware features.  callPlatformFunction allows any IOService object to access these functions.  Normally callPlatformFunction will be called on a service's provider.  The provider will service the request or pass it to it's provider.  The systems IOPlatformExpert subclass will catch functions it knows about and redirect them into other parts of the IOService plane.  If the IOPlatformExpert subclass can not execute the function, the base class will be called.  The IOPlatformExpert base class will attempt to find a service to execute the function by looking up the function name in a IOResources name space.  A service may publish a service using publishResource(functionName, this).  If no service can be found to execute the function an error will be returned.
-    @param functionName name of the function to be called.  When functionName is a c-string, callPlatformFunction will convert the c-string to a OSSymbol and call other OSSymbol version of callPlatformFunction.  This process can block and should not be used from an interrupt context.
-    @param waitForFunction if true callPlatformFunction will not return until the function has been called.
-    @result Return an IOReturn code, kIOReturnSuccess if the function was successfully executed, kIOReturnUnsupported if a service to execute the function could not be found.  Other return codes may be returned by the function.*/
+    @discussion The platform expert or other drivers may implement various functions to control hardware features.  <code>callPlatformFunction</code> allows any IOService object to access these functions. Normally <code>callPlatformFunction</code> is called on a service's provider. The provider services the request or passes it to its provider. The system's IOPlatformExpert subclass catches functions it knows about and redirects them into other parts of the service plane. If the IOPlatformExpert subclass cannot execute the function, the base class is called. The IOPlatformExpert base class attempts to find a service to execute the function by looking up the function name in an IOResources name space. A service may publish a service using <code>publishResource(functionName, this)</code>. If no service can be found to execute the function an error is returned.
+    @param functionName Name of the function to be called. When <code>functionName</code> is a C string, <code>callPlatformFunction</code> converts the C string to an OSSymbol and calls the OSSymbol version of <code>callPlatformFunction</code>. This process can block and should not be used from an interrupt context.
+    @param waitForFunction If <code>true</code>, <code>callPlatformFunction</code> will not return until the function has been called.
+    @result An IOReturn code; <code>kIOReturnSuccess</code> if the function was successfully executed, <code>kIOReturnUnsupported</code> if a service to execute the function could not be found. Other return codes may be returned by the function.*/
 
     virtual IOReturn callPlatformFunction( const OSSymbol * functionName,
 					   bool waitForFunction,
@@ -859,29 +902,29 @@ public:
     /* Some accessors */
 
 /*! @function getPlatform
-    @abstract Returns a pointer to the platform expert instance for the machine.
-    @discussion This method provides an accessor to the platform expert instance for the machine. 
+    @abstract Returns a pointer to the platform expert instance for the computer.
+    @discussion This method provides an accessor to the platform expert instance for the computer. 
     @result A pointer to the IOPlatformExport instance. It should not be released by the caller. */
 
     static IOPlatformExpert * getPlatform( void );
 
 /*! @function getPMRootDomain
-    @abstract Returns a pointer to the power management root domain instance for the machine.
-    @discussion This method provides an accessor to the power management root domain instance for the machine. 
+    @abstract Returns a pointer to the power management root domain instance for the computer.
+    @discussion This method provides an accessor to the power management root domain instance for the computer. 
     @result A pointer to the power management root domain instance. It should not be released by the caller. */
 
     static class IOPMrootDomain * getPMRootDomain( void );
 
 /*! @function getServiceRoot
     @abstract Returns a pointer to the root of the service plane.
-    @discussion This method provides an accessor to the root of the service plane for the machine. 
+    @discussion This method provides an accessor to the root of the service plane for the computer. 
     @result A pointer to the IOService instance at the root of the service plane. It should not be released by the caller. */
 
     static IOService * getServiceRoot( void );
 
 /*! @function getResourceService
     @abstract Returns a pointer to the IOResources service.
-    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in IOKit based on a name, using the standard IOService matching and notification calls.
+    @discussion IOService maintains a resource service IOResources that allows objects to be published and found globally in the I/O Kit based on a name, using the standard IOService matching and notification calls.
     @result A pointer to the IOResources instance. It should not be released by the caller. */
 
     static IOService * getResourceService( void );
@@ -889,9 +932,9 @@ public:
     /* Allocate resources for a matched service */
 
 /*! @function getResources
-    @abstract Allocate any needed resources for a published IOService before clients attach.
-    @discussion This method is called during the registration process for an IOService object if there are success driver matches, before any clients attach. It allows for lazy allocation of resources to an IOService when a matching driver is found.
-    @result Return an IOReturn code, kIOReturnSuccess is necessary for the IOService to be successfully used, otherwise the registration process for the object is halted. */
+    @abstract Allocates any needed resources for a published IOService object before clients attach.
+    @discussion This method is called during the registration process for an IOService object if there are successful driver matches, before any clients attach. It allows for lazy allocation of resources to an IOService object when a matching driver is found.
+    @result An IOReturn code; <code>kIOReturnSuccess</code> is necessary for the IOService object to be successfully used, otherwise the registration process for the object is halted. */
     
     virtual IOReturn getResources( void );
 
@@ -899,22 +942,22 @@ public:
 
 /*! @function getDeviceMemoryCount
     @abstract Returns a count of the physical memory ranges available for a device.
-    @discussion This method will return the count of physical memory ranges, each represented by an IODeviceMemory instance, that have been allocated for a memory mapped device.
+    @discussion This method returns the count of physical memory ranges, each represented by an IODeviceMemory instance, that have been allocated for a memory mapped device.
     @result An integer count of the number of ranges available. */
 
     virtual IOItemCount getDeviceMemoryCount( void );
 
 /*! @function getDeviceMemoryWithIndex
-    @abstract Returns an instance of IODeviceMemory representing one of a device's memory mapped  ranges.
-    @discussion This method will return a pointer to an instance of IODeviceMemory for the physical memory range at the given index for a memory mapped device.
+    @abstract Returns an instance of IODeviceMemory representing one of a device's memory mapped ranges.
+    @discussion This method returns a pointer to an instance of IODeviceMemory for the physical memory range at the given index for a memory mapped device.
     @param index An index into the array of ranges assigned to the device.
-    @result A pointer to an instance of IODeviceMemory, or zero if the index is beyond the count available. The IODeviceMemory is retained by the provider, so is valid while attached, or while any mappings to it exist. It should not be released by the caller. See also mapDeviceMemory() which will create a device memory mapping. */
+    @result A pointer to an instance of IODeviceMemory, or zero if the index is beyond the count available. The IODeviceMemory is retained by the provider, so is valid while attached, or while any mappings to it exist. It should not be released by the caller. See also @link mapDeviceMemoryWithIndex mapDeviceMemoryWithIndex@/link, which creates a device memory mapping. */
 
     virtual IODeviceMemory * getDeviceMemoryWithIndex( unsigned int index );
 
 /*! @function mapDeviceMemoryWithIndex
     @abstract Maps a physical range of a device.
-    @discussion This method will create a mapping for the IODeviceMemory at the given index, with IODeviceMemory::map(options). The mapping is represented by the returned instance of IOMemoryMap, which should not be released until the mapping is no longer required.
+    @discussion This method creates a mapping for the IODeviceMemory at the given index, with <code>IODeviceMemory::map(options)</code>. The mapping is represented by the returned instance of IOMemoryMap, which should not be released until the mapping is no longer required.
     @param index An index into the array of ranges assigned to the device.
     @result An instance of IOMemoryMap, or zero if the index is beyond the count available. The mapping should be released only when access to it is no longer required. */
 
@@ -923,14 +966,14 @@ public:
 
 /*! @function getDeviceMemory
     @abstract Returns the array of IODeviceMemory objects representing a device's memory mapped ranges.
-    @discussion This method will return an array of IODeviceMemory objects representing the physical memory ranges allocated to a memory mapped device.
+    @discussion This method returns an array of IODeviceMemory objects representing the physical memory ranges allocated to a memory mapped device.
     @result An OSArray of IODeviceMemory objects, or zero if none are available. The array is retained by the provider, so is valid while attached. */
 
     virtual OSArray * getDeviceMemory( void );
 
 /*! @function setDeviceMemory
     @abstract Sets the array of IODeviceMemory objects representing a device's memory mapped ranges.
-    @discussion This method will set an array of IODeviceMemory objects representing the physical memory ranges allocated to a memory mapped device.
+    @discussion This method sets an array of IODeviceMemory objects representing the physical memory ranges allocated to a memory mapped device.
     @param array An OSArray of IODeviceMemory objects, or zero if none are available. The array will be retained by the object. */
 
     virtual void setDeviceMemory( OSArray * array );
@@ -938,61 +981,60 @@ public:
     /* Interrupt accessors */
 
 /*! @function registerInterrupt
-    @abstract Register a C-function interrupt handler for a device supplying interrupts.
-    @discussion This method will install a C-function interrupt handler to be called at primary interrupt time for a device's interrupt. Only one handler may be installed per interrupt source. IOInterruptEventSource provides an IOWorkLoop based abstraction for interrupt delivery that may be more appropriate for work loop based drivers.
+    @abstract Registers a C function interrupt handler for a device supplying interrupts.
+    @discussion This method installs a C function interrupt handler to be called at primary interrupt time for a device's interrupt. Only one handler may be installed per interrupt source. IOInterruptEventSource provides a work loop based abstraction for interrupt delivery that may be more appropriate for work loop based drivers.
     @param source The index of the interrupt source in the device.
     @param target An object instance to be passed to the interrupt handler.
-    @param handler The C-function to be to be called at primary interrupt time when the interrupt occurs. The handler should process the interrupt by clearing the interrupt, or by disabling the source.
+    @param handler The C function to be called at primary interrupt time when the interrupt occurs. The handler should process the interrupt by clearing the interrupt, or by disabling the source.
     @param refCon A reference constant for the handler's use.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid.<br>kIOReturnNoResources is returned if the interrupt already has an installed handler. */
+    @result An IOReturn code.<br><code>kIOReturnNoInterrupt</code> is returned if the source is not valid; <code>kIOReturnNoResources</code> is returned if the interrupt already has an installed handler. */
 
     virtual IOReturn registerInterrupt(int source, OSObject *target,
 				       IOInterruptAction handler,
 				       void *refCon = 0);
                                        
 /*! @function unregisterInterrupt
-    @abstract Remove a C-function interrupt handler for a device supplying hardware interrupts.
-    @discussion This method will remove a C-function interrupt handler previously installed with registerInterrupt.
+    @abstract Removes a C function interrupt handler for a device supplying hardware interrupts.
+    @discussion This method removes a C function interrupt handler previously installed with @link registerInterrupt registerInterrupt@/link.
     @param source The index of the interrupt source in the device.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
+    @result An IOReturn code (<code>kIOReturnNoInterrupt</code> is returned if the source is not valid). */
 
     virtual IOReturn unregisterInterrupt(int source);
 
 /*! @function getInterruptType
-    @abstract Return the type of interrupt used for a device supplying hardware interrupts.
-    @discussion This method will return the type of interrupt used by the device.
+    @abstract Returns the type of interrupt used for a device supplying hardware interrupts.
     @param source The index of the interrupt source in the device.
-    @param interruptType The interrupt type for the interrupt source will be stored here by getInterruptType.<br> kIOInterruptTypeEdge will be returned for edge trigggered sources.<br> kIOInterruptTypeLevel will be returned for level trigggered sources.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
+    @param interruptType The interrupt type for the interrupt source will be stored here by <code>getInterruptType</code>.<br> <code>kIOInterruptTypeEdge</code> will be returned for edge-trigggered sources.<br><code>kIOInterruptTypeLevel</code> will be returned for level-trigggered sources.
+    @result An IOReturn code (<code>kIOReturnNoInterrupt</code> is returned if the source is not valid). */
 
     virtual IOReturn getInterruptType(int source, int *interruptType);
 
 /*! @function enableInterrupt
-    @abstract Enable a device interrupt.
-    @discussion Enable a device interrupt. It is the callers responsiblity to keep track of the enable state of the interrupt source.
+    @abstract Enables a device interrupt.
+    @discussion It is the caller's responsiblity to keep track of the enable state of the interrupt source.
     @param source The index of the interrupt source in the device.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
+    @result An IOReturn code (<code>kIOReturnNoInterrupt</code> is returned if the source is not valid). */
 
     virtual IOReturn enableInterrupt(int source);
 
 /*! @function disableInterrupt
-    @abstract Disable a device interrupt.
-    @discussion Synchronously disable a device interrupt.  If the interrupt routine is running, the call will block until the routine completes.  It is the callers responsiblity to keep track of the enable state of the interrupt source.
+    @abstract Synchronously disables a device interrupt.
+    @discussion If the interrupt routine is running, the call will block until the routine completes. It is the caller's responsiblity to keep track of the enable state of the interrupt source.
     @param source The index of the interrupt source in the device.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
+    @result An IOReturn code (<code>kIOReturnNoInterrupt</code> is returned if the source is not valid). */
 
     virtual IOReturn disableInterrupt(int source);
 
 /*! @function causeInterrupt
-    @abstract Cause a device interrupt to occur.
-    @discussion Emulate a hardware interrupt, to be called from task level.
+    @abstract Causes a device interrupt to occur.
+    @discussion Emulates a hardware interrupt, to be called from task level.
     @param source The index of the interrupt source in the device.
-    @result An IOReturn code.<br>kIOReturnNoInterrupt is returned if the source is not valid. */
+    @result An IOReturn code (<code>kIOReturnNoInterrupt</code> is returned if the source is not valid). */
 
     virtual IOReturn causeInterrupt(int source);
 
 /*! @function requestProbe
-    @abstract An external request that hardware be re-scanned for devices.
+    @abstract Requests that hardware be re-scanned for devices.
     @discussion For bus families that do not usually detect device addition or removal, this method represents an external request (eg. from a utility application) to rescan and publish or remove found devices.
     @param options Family defined options, not interpreted by IOService.
     @result An IOReturn code. */
@@ -1002,9 +1044,9 @@ public:
     /* Generic API for non-data-path upstream calls */
 
 /*! @function message
-    @abstract Receive a generic message delivered from an attached provider.
-    @discussion A provider may deliver messages via the message method to its clients informing them of state changes, for example kIOMessageServiceIsTerminated or kIOMessageServiceIsSuspended. Certain messages are defined by IOKit in IOMessage.h while others may family dependent. This method is implemented in the client to receive messages.
-    @param type A type defined in IOMessage.h or defined by the provider family.
+    @abstract Receives a generic message delivered from an attached provider.
+    @discussion A provider may deliver messages via the <code>message</code> method to its clients informing them of state changes, such as <code>kIOMessageServiceIsTerminated</code> or <code>kIOMessageServiceIsSuspended</code>. Certain messages are defined by the I/O Kit in <code>IOMessage.h</code> while others may be family dependent. This method is implemented in the client to receive messages.
+    @param type A type defined in <code>IOMessage.h</code> or defined by the provider family.
     @param provider The provider from which the message originates.
     @param argument An argument defined by the provider family, not used by IOService.
     @result An IOReturn code defined by the message type. */
@@ -1013,9 +1055,9 @@ public:
                               void * argument = 0 );
                                 
 /*! @function messageClient
-    @abstract Send a generic message to an attached client.
-    @discussion A provider may deliver messages via the message method to its clients informing them of state changes, for example kIOMessageServiceIsTerminated or kIOMessageServiceIsSuspended. Certain messages are defined by IOKit in IOMessage.h while others may family dependent. This method may be called in the provider to send a message to the specified client, which may be useful for overrides.
-    @param type A type defined in IOMessage.h or defined by the provider family.
+    @abstract Sends a generic message to an attached client.
+    @discussion A provider may deliver messages via the @link message message@/link method to its clients informing them of state changes, such as <code>kIOMessageServiceIsTerminated</code> or <code>kIOMessageServiceIsSuspended</code>. Certain messages are defined by the I/O Kit in <code>IOMessage.h</code> while others may be family dependent. This method may be called in the provider to send a message to the specified client, which may be useful for overrides.
+    @param type A type defined in <code>IOMessage.h</code> or defined by the provider family.
     @param client A client of the IOService to send the message.
     @param argument An argument defined by the provider family, not used by IOService.
     @result The return code from the client message call. */
@@ -1024,11 +1066,11 @@ public:
                                     void * messageArgument = 0, vm_size_t argSize = 0 );
 
 /*! @function messageClients
-    @abstract Send a generic message to all attached clients.
-    @discussion A provider may deliver messages via the message method to its clients informing them of state changes, for example kIOMessageServiceIsTerminated or kIOMessageServiceIsSuspended. Certain messages are defined by IOKit in IOMessage.h while others may family dependent. This method may be called in the provider to send a message to all the attached clients, via the messageClient method.
-    @param type A type defined in IOMessage.h or defined by the provider family.
+    @abstract Sends a generic message to all attached clients.
+    @discussion A provider may deliver messages via the @link message message@/link method to its clients informing them of state changes, such as <code>kIOMessageServiceIsTerminated</code> or <code>kIOMessageServiceIsSuspended</code>. Certain messages are defined by the I/O Kit in <code>IOMessage.h</code> while others may be family dependent. This method may be called in the provider to send a message to all the attached clients, via the @link messageClient messageClient@/link method.
+    @param type A type defined in <code>IOMessage.h</code> or defined by the provider family.
     @param argument An argument defined by the provider family, not used by IOService.
-    @result Any non-kIOReturnSuccess return codes returned by the clients, or kIOReturnSuccess if all return kIOReturnSuccess. */
+    @result Any non-<code>kIOReturnSuccess</code> return codes returned by the clients, or <code>kIOReturnSuccess</code> if all return <code>kIOReturnSuccess</code>. */
 
     virtual IOReturn messageClients( UInt32 type,
                                      void * argument = 0, vm_size_t argSize = 0 );
@@ -1053,14 +1095,14 @@ public:
     /* User client create */
 
 /*! @function newUserClient
-    @abstract A request to create a connection for a non kernel client.
-    @discussion A non kernel client may request a connection be opened via the IOServiceOpen() library function, which will call this method in an IOService. The rules & capabilities of user level clients are family dependent, and use the functions of the IOUserClient class for support. IOService's implementation returns kIOReturnUnsupported, so any family supporting user clients must implement this method.
-    @param owningTask The mach task requesting the connection.
+    @abstract Creates a connection for a non kernel client.
+    @discussion A non kernel client may request a connection be opened via the @link //apple_ref/c/func/IOServiceOpen IOServiceOpen@/link library function, which will call this method in an IOService object. The rules and capabilities of user level clients are family dependent, and use the functions of the IOUserClient class for support. IOService's implementation returns <code>kIOReturnUnsupported</code>, so any family supporting user clients must implement this method.
+    @param owningTask The Mach task of the client thread in the process of opening the user client. Note that in Mac OS X, each process is based on a Mach task and one or more Mach threads. For more information on the composition of a Mach task and its relationship with Mach threads, see {@linkdoc //apple_ref/doc/uid/TP30000905-CH209-TPXREF103 "Tasks and Threads"}.
     @param securityID A token representing the access level for the task.
-    @param type A constant specifying the type of connection to be created, specified by the caller of IOServiceOpen and interpreted only by the family.
+    @param type A constant specifying the type of connection to be created, specified by the caller of @link //apple_ref/c/func/IOServiceOpen IOServiceOpen@/link and interpreted only by the family.
     @param handler An instance of an IOUserClient object to represent the connection, which will be released when the connection is closed, or zero if the connection was not opened.    
     @param properties A dictionary of additional properties for the connection.
-    @result A return code to be passed back to the caller of IOServiceOpen. */
+    @result A return code to be passed back to the caller of <code>IOServiceOpen</code>. */
 
     virtual IOReturn newUserClient( task_t owningTask, void * securityID,
                                     UInt32 type, OSDictionary * properties,
@@ -1072,18 +1114,18 @@ public:
     /* Return code utilities */
 
 /*! @function stringFromReturn
-    @abstract A utility to supply a programmer friendly string from an IOReturn code.
-    @discussion Strings are available for the standard return codes in IOReturn.h in IOService, while subclasses may implement this method to interpret family dependent return codes.
+    @abstract Supplies a programmer-friendly string from an IOReturn code.
+    @discussion Strings are available for the standard return codes in <code>IOReturn.h</code> in IOService, while subclasses may implement this method to interpret family dependent return codes.
     @param rtn The IOReturn code.
     @result A pointer to a constant string, or zero if the return code is unknown. */
     
     virtual const char * stringFromReturn( IOReturn rtn );
 
 /*! @function errnoFromReturn
-    @abstract A utility to translate an IOReturn code to a BSD errno.
-    @discussion BSD defines its own return codes for its functions in sys/errno.h, and IOKit families may need to supply compliant results in BSD shims. Results are available for the standard return codes in IOReturn.h in IOService, while subclasses may implement this method to interpret family dependent return codes.
+    @abstract Translates an IOReturn code to a BSD <code>errno</code>.
+    @discussion BSD defines its own return codes for its functions in <code>sys/errno.h</code>, and I/O Kit families may need to supply compliant results in BSD shims. Results are available for the standard return codes in <code>IOReturn.h</code> in IOService, while subclasses may implement this method to interpret family dependent return codes.
     @param rtn The IOReturn code.
-    @result The BSD errno or EIO if unknown. */
+    @result The BSD <code>errno</code> or <code>EIO</code> if unknown. */
     
     virtual int errnoFromReturn( IOReturn rtn );
 
@@ -1174,570 +1216,432 @@ public:
     /* power management */
     
 /*! @function PMinit
-        A power managment policy-maker for a device calls itself here to initialize its power management.
-        PMinit allocates and initializes the power management instance variables, and it should be called before any
-        access to those variables or the power management methods. */
+    @abstract Initializes power management for a driver.
+    @discussion <code>PMinit</code> allocates and initializes the power management instance variables, and it should be called before accessing those variables or calling the power management methods. This method should be called inside the driver's <code>start</code> routine and must be paired with a call to @link PMstop PMstop@/link.
+    
+    Most calls to <code>PMinit</code> are followed by calls to @link joinPMtree joinPMtree@/link and @link registerPowerDriver registerPowerDriver@/link.
+*/
     virtual void PMinit (void );
 
 /*! @function PMstop
-        A power managment policy-maker for a device calls itself here when it resigns its responsibilities as
-        policy-maker.  This typically happens when it is handing off the responsibility to another policy-maker,
-        or when the device is removed from the system.  The power managment variables don't exist after
-        this call,  and the power managment methods in the caller shouldn't be accessed.  */
+    @abstract Frees and removes the driver from power management.
+    @discussion The power managment variables don't exist after this call and the power managment methods in the caller shouldn't be called. 
+    
+    Calling <code>PMstop</code> cleans up for the three power management initialization calls: @link PMinit PMinit@/link, @link joinPMtree joinPMtree@/link, and @link registerPowerDriver registerPowerDriver@/link.
+*/
     virtual void PMstop ( void );
 
 /*! @function joinPMtree
-        A policy-maker calls its nub here when initializing, to be attached into
-        the power management hierarchy.  The default function is to call the
-        platform expert, which knows how to do it.  This method is overridden
-        by a nub subclass which may either know how to do it, or may need
-        to take other action.
-
-        This may be the only "power management" method used in a nub, meaning
-        it may be called even if the nub is not initialized for power management.
-
-        Before the nub returns from this method, the caller will probably be called
-        at "setPowerParent" and "setAggressiveness" and possibly at "addPowerChild" as it is
-        added to me hierarchy. */
+    @abstract Joins the driver into the power plane of the I/O Registry .
+    @discussion A driver uses this method to call its nub when initializing (usually in its <code>start</code> routine after calling @link PMinit PMinit@/link), to be attached into the power management hierarchy (i.e., the power plane). A driver usually calls this method on the driver for the device that provides it power (this is frequently the nub).
+    
+    Before this call returns, the caller will probably be called at @link setPowerParent setPowerParent@/link and @link setAggressiveness setAggressiveness@/link and possibly at @link addPowerChild addPowerChild@/link as it is added to the hierarchy. 
+
+    This method may be overridden by a nub subclass.
+    @param driver The driver to be added to the power plane, usually <code>this</code>.
+*/
     virtual void joinPMtree ( IOService * driver );
 
 /*! @function registerPowerDriver
-        A driver calls a policy-maker here to volunteer to control power to the device.
-        If the policy-maker accepts the volunteer, it adds the volunteer to its list of
-        interested drivers, and it will call the volunteer at appropriate times to switch
-        the power state of the device.
-        @param controllingDriver
-        This points to the calling driver.
-        @param powerStates
-        This is an array of power states which the driver can deal with.  If this array
-        is no less rich than one supplied by an earlier volunteer, then the policy-maker
-        uses the calling driver as its power-controlling driver.
-        @param numberOfStates
-        The number of power states in the array.  Power states are defined in
-        pwr_mgt/IOPMpowerState.h.
-        @result
-        IOPMNoErr is returned.  There are various error conditions possible which prevent
-        the policy-maker from accepting the new power state array.  These conditions
-        are logged in the power managment event log, but not returned to the caller. */
-    virtual IOReturn registerPowerDriver ( IOService* controllingDriver, IOPMPowerState* powerStates, unsigned long numberOfStates );
-
-/*! @function registerInterestedDriver
-        Some IOService calls a policy-maker here to register interest in the changing
-        power state of its device.
-        @param theDriver
-        The policy-maker adds this pointer to the calling IOService to its list of
-        interested drivers.  It informs drivers on this list pre- and post-power change.
-        @result
-        The policy-maker returns flags describing the capability of the device in its
-        current power state.  The policy-maker does not interpret these flags or
-        understand them; they come from the power state array, and are understood
-        only by interested drivers and perhaps the power-controlling driver.  If the
-        current power state is not yet defined, zero is returned.  This is the case when
-        the policy-maker is not yet in the power domain hierarchy or when it doesn't
-        have a power-controlling driver yet. */
+    @abstract Registers a set of power states that the driver supports.
+    
+    @discussion A driver defines its array of supported power states with power management in its power management initialization (its <code>start</code> routine). If successful, power management will call the driver to instruct it to change its power state through @link setPowerState setPowerState@/link.
+    
+    Most drivers do not need to override <code>registerPowerDriver</code>. A nub may override <code>registerPowerDriver</code> if it needs to arrange its children in the power plane differently than the default placement, but this is uncommon.
+    
+    @param controllingDriver A pointer to the calling driver, usually <code>this</code>.
+    @param powerStates A driver-defined array of power states that the driver and device support. Power states are defined in <code>pwr_mgt/IOPMpowerState.h</code>.
+    @param numberOfStates The number of power states in the array. 
+    @result </code>IOPMNoErr</code>. All errors are logged via <code>kprintf</code>.
+*/
+    virtual IOReturn registerPowerDriver ( 
+                    IOService* controllingDriver, 
+                    IOPMPowerState* powerStates, 
+                    unsigned long numberOfStates );
+
+/*! 
+    @function registerInterestedDriver
+    @abstract Allows an IOService object to register interest in the changing power state of a power-managed IOService object.
+    @discussion Call <code>registerInterestedDriver</code> on the IOService object you are interested in receiving power state messages from, and pass a pointer to the interested driver (<code>this</code>) as an argument.
+    
+    The interested driver should override @link powerStateWillChangeTo powerStateWillChangeTo@/link and @link powerStateDidChangeTo powerStateDidChangeTo@/link to receive these power change messages.
+    
+    Interested drivers must acknowledge power changes in <code>powerStateWillChangeTo</code> or <code>powerStateDidChangeTo</code>, either via return value or later calls to @link acknowledgePowerChange acknowledgePowerChange@/link.
+    
+    Most drivers do not need to override <code>registerInterestedDriver</code>.
+    @param theDriver The driver of interest adds this pointer to the list of interested drivers. It informs drivers on this list before and after the power change. 
+    @result Flags describing the capability of the device in its current power state. If the current power state is not yet defined, zero is returned (this is the case when the driver is not yet in the power domain hierarchy or hasn't fully registered with power management yet). 
+*/
     virtual IOPMPowerFlags registerInterestedDriver ( IOService* theDriver );
 
 /*! @function deRegisterInterestedDriver
-        An IOService which has previously registered with a policy-maker as an interested
-        driver calls the policy-maker here to withdraw its interest.  The policy-maker removes
-        it from its list of interested drivers.
-        @result
-        These bits describe the capability of the device in its current power state.  They are
-        not understood by the policy-maker; they come from the capabilityFlags field of the
-        current power state in the power state array. */
+    @abstract De-registers power state interest from a previous call to <code>registerInterestedDriver</code>.
+    @discussion Most drivers do not need to override <code>deRegisterInterestedDriver</code>.
+    @param theDriver The interested driver previously passed into @link registerInterestedDriver registerInterestedDriver@/link.
+    @result A return code that can be ignored by the caller.
+*/
     virtual IOReturn deRegisterInterestedDriver ( IOService * theDriver );
 
 /*! @function acknowledgePowerChange
-        When a device is changing power state, its policy-maker informs interested
-        parties before and after the change.  Interested parties are those which
-        have registered as interested drivers and also children of the policy-maker
-        in the case that it is a power domain.
-        When an object is so informed, it can return an indication that it is prepared
-        for the change, or it can return an indication that it needs some time to
-        prepare.  In this case it will call this method in the policy-maker when it has
-        prepared.
-        @param theDriver
-        This points to the calling driver.  The policy-maker uses it to know if all
-        interested parties have acknowledged the power state change.
-        @result
-        IOPMNoErr is returned. */
+    @abstract Acknowledges an in-progress power state change.
+    @discussion When power management informs an interested object (via @link powerStateWillChangeTo powerStateWillChangeTo@/link or @link powerStateDidChangeTo powerStateDidChangeTo@/link), the object can return an immediate acknowledgement via a return code, or it may return an indication that it will acknowledge later by calling <code>acknowledgePowerChange</code>. 
+
+    Interested objects are those that have registered as interested drivers, as well as power plane children of the power changing driver. 
+    
+    A driver that calls @link registerInterestedDriver registerInterestedDriver@/link must call <code>acknowledgePowerChange</code>, or use an immediate acknowledgement return from <code>powerStateWillChangeTo</code> or <code>powerStateDidChangeTo</code>. 
+    
+    Most drivers do not need to override <code>acknowledgePowerChange</code>.
+
+    @param whichDriver A pointer to the calling driver. The called object tracks all interested parties to ensure that all have acknowledged the power state change.
+    @result <code>IOPMNoErr</code>. 
+*/
     virtual IOReturn acknowledgePowerChange ( IOService * whichDriver );
 
+
+
 /*! @function acknowledgeSetPowerState
-        When a policy-maker instructs its controlling driver to switch the state of
-        the device, the driver can return an indication that the change is complete,
-        or it can return an indication that it needs some time to make the change.
-        In this case it will call this method in the policy-maker when it has made the
-        power state change.
-        @result
-        IOPMNoErr is returned. */
+    @abstract Acknowledges the belated completion of a driver's <code>setPowerState</code> power state change.
+    @discussion After power management instructs a driver to change its state via @link setPowerState setPowerState@/link, that driver must acknowledge the change when its device has completed its transition. The acknowledgement may be immediate, via a return code from <code>setPowerState</code>, or delayed, via this call to <code>acknowledgeSetPowerState</code>.
+    
+    Any driver that does not return <code>kIOPMAckImplied</code> from its <code>setPowerState</code> implementation must later call <code>acknowledgeSetPowerState</code>. 
+    
+    Most drivers do not need to override <code>acknowledgeSetPowerState</code>.
+    @result <code>IOPMNoErr</code>.
+*/
     virtual IOReturn acknowledgeSetPowerState ( void );
 
 /*! @function powerDomainWillChangeTo
-        When a power domain changes state, it notifies its children, which
-        are policy-makers, by calling them at this method.  It calls here
-        before it makes the change, and a called policy-maker can return
-        IOPMAckImplied to indicate that it is prepared for the change,
-        or it can return a non-zero number to indicate that it is not prepared
-        but will prepare and then call the parent at acknowledgePowerChange.
-
-        To prepare for a lowering of the power domain, the policy-maker
-        informs all its interested parties of any resulting change in its device,
-        and when they have all acknowledged, it calls its controlling driver
-        to switch the device to an appropriate power state for the imminent
-        domain state.  If any interested driver or the controlling driver does
-        not acknowledge immediately, then the policy-maker also will not.
-
-        To prepare for a raising of the power domain, the policy-maker
-        informs all its interested parties of any resulting change in its device.
-        If any do not acknowledge immediately, then the policy-maker also will not.
-        @param newPowerStateFlags
-        These flags describe the character of power in the imminent domain state.
-        They are not understood by the policy-maker.  It asks the controlling
-        driver to translate them into a state number within the power state array.
-        (The policy-maker for the domain also doesn't understand the bits; they
-         come from a outputPowerCharacter field of the power state array for
-         the power domain.)
-        @param whichParent
-        This pointer identifies the calling parent. */
-    IOReturn powerDomainWillChangeTo ( IOPMPowerFlags newPowerStateFlags, IOPowerConnection * whichParent );
+    @abstract Notifies a driver that its power domain is about to change state.
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
+    IOReturn powerDomainWillChangeTo ( 
+                    IOPMPowerFlags newPowerStateFlags, 
+                    IOPowerConnection * whichParent );
 
 /*! @function powerDomainDidChangeTo
-        When a power domain changes state, it notifies its children, which
-        are policy-makers, by calling them at this method.  It calls here
-        after the changed power of the power domain has settled at the
-        new level.  A called policy-maker can return
-        IOPMAckImplied to indicate that it is prepared for the change,
-        or it can return a non-zero number to indicate that it is not prepared
-        but will prepare and then call the parent at acknowledgePowerChange.
-
-        To prepare for a lowered power domain, the policy-maker
-        informs all its interested parties of the new power state of its device.
-        If any do not acknowledge immediately, then the policy-maker also will not.
-
-        To prepare for a raised power domain, the policy-maker calls its controlling
-        driver to switch the device to the appropriate power state for the new
-        domain state.  When that is accomplished, the policy-maker informs
-        all its interested parties of the new power state.  If any interested driver
-        or the controlling driver does not acknowledge immediately, then the
-        policy-maker also will not.
-        
-        @param newPowerStateFlags
-        These flags describe the character of power in the new domain state.
-        They are not understood by the policy-maker.  It asks the controlling
-        driver to translate them into a state number within the power state array.
-        (The policy-maker for the domain also doesn't understand the bits; they
-         come from a outputPowerCharacter field of the power state array for
-         the power domain.)
-        @param whichParent
-        This pointer identifies the calling parent. */
-    IOReturn powerDomainDidChangeTo ( IOPMPowerFlags newPowerStateFlags, IOPowerConnection * whichParent );
+    @abstract Notifies a driver that its power domain is about to change state.
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
+    IOReturn powerDomainDidChangeTo ( 
+                    IOPMPowerFlags newPowerStateFlags, 
+                    IOPowerConnection * whichParent );
 
 /*! @function requestPowerDomainState
-        The child of a power domain calls it parent here to request power of a certain
-        character.  It does this after lowering power in its own device which allows
-        it to tolerate lower power in the domain, and it does this if it needs more
-        power for its device than is currently available in the domain.
-        @param desiredState
-        These flags describe the power required for some state of the caller's device.
-        They are not understood by either the child or the parent.  They come from
-        the power state array of the child (in the inputPowerRequirement field), and
-        the parent compares them to bits in the outputPowerCharacter fields of its
-        power state array.
-        @param whichChild
-        This points to the caller, so the power domain can know which child is requesting.
-        @param specificationFlags
-        This value modifies the parent's choice of power state.
-        If the parameter is IOPMNextHigherState, the parent will choose the lowest state
-        which matches desiredState and which is higher than the current state.
-        If the parameter is IOPMHighestState	, the parent will choose the highest state
-        which matches desiredState.
-        If the parameter is IOPMNextLowerState, the parent will choose the highest state
-        which matches desiredState and which is lower than the current state.
-        If the parameter is IOPMLowestState, the parent will choose the lowest state
-        which matches desiredState.
-        A state matches desiredState if all the bits set in desiredState are also set in the
-        outputPowerCharacter field of that state in the parent's power state array.
-        @result
-        The power domain parent returns IOPMBadSpecification if specificationFlags
-        not wellformed.  It returns IOPMNoSuchState if no state in its array satisfies
-        the callers specification.  It returns IOPMNotYetInitialized if it has not power
-        state array yet to compare with.  Otherwise it returns IOPMNoErr.  In the last
-        case it will initiate its change to the new state if it has a parent in the hierarchy
-        (or is the root power domain.)  */
-    virtual IOReturn requestPowerDomainState ( IOPMPowerFlags desiredState, IOPowerConnection * whichChild, unsigned long specificationFlags );
+    @abstract Tells a driver to adjust its power state.
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
+    virtual IOReturn requestPowerDomainState ( 
+                    IOPMPowerFlags desiredState, 
+                    IOPowerConnection * whichChild, 
+                    unsigned long specificationFlags );
 
 /*! @function makeUsable
-        Some client of a device is asking that it become usable.  Although
-        this has not come from the policy-maker for the device, treat it exactly
-        as if it had.  In this way, subsequent requests for lower power from
-        the policy-maker will pre-empt this request.
-        We treat this as policy-maker request to switch to the highest power state.
-        @result
-        The return code reflects the state of the policy-maker's internal queue of power
-        changes and can be ignored by the caller.  */
+    @abstract Requests that a device become usable.
+    @discussion This method is called when some client of a device (or the device's own driver) is asking for the device to become usable. Power management responds by telling the object upon which this method is called to change to its highest power state.
+    
+    <code>makeUsable</code> is implemented using @link changePowerStateToPriv changePowerStateToPriv@/link.
+    
+    Subsequent requests for lower power, such as from <code>changePowerStateToPriv</code>, will pre-empt this request.
+    @result A return code that can be ignored by the caller.
+*/
     virtual IOReturn makeUsable ( void );
 
-    /*! @function temporaryPowerClampOn
-        A power domain calls this method to hold itself in the highest power state until it
-        has children, and at that point the domain state is controlled by the childrens'
-        requirements.
-        @result
-        The return code reflects the state of the policy-maker's internal queue of power
-        changes and can be ignored by the caller.  */
+/*! @function temporaryPowerClampOn
+    @abstract A driver calls this method to hold itself in the highest power state until it has children.
+    @discussion Use <code>temporaryPowerClampOn</code> to hold your driver in its highest power state while waiting for child devices to attach. After children have attached, the clamp is released and the device's power state is controlled by the children's requirements. 
+    @result A return code that can be ignored by the caller.
+*/
     virtual IOReturn temporaryPowerClampOn ( void );
     
 /*! @function changePowerStateTo
-        The power-controlling driver calls the policy-maker here when it wants the device
-        switched to a different power state.  This is mildly ironic in that it is the controlling
-        driver which does the switching, but it must do it this way so that the policy-maker
-        can make sure the power domain is correct and to notify interested parties
-        pre-change.  When appropriate, the policy-maker will call the controlling driver and
-        have it switch the device to the requested state in the usual way.
-        This request by the controlling driver is sticky in that the policy-maker will not
-        switch the device lower than this request, so if the driver needs power raised for
-        some reason and then gets it and does what it needs, it should then rescind the
-        request by requesting state zero.  This will allow the policy-maker to control the
-        device as usual.
-        @param ordinal
-        This is the number, in the power state array, of the desired power state.
-        @result
-        The return code reflects the state of the policy-maker's internal queue of power
-        changes and can be ignored by the caller. */
+    @abstract Sets a driver's power state.
+    @discussion This function is one of several that are used to set a driver's power state. In most circumstances, however, you should call @link changePowerStateToPriv changePowerStateToPriv@/link instead. 
+	
+	Calls to <code>changePowerStateTo</code>, <code>changePowerStateToPriv</code>, and a driver's power children all affect the power state of a driver. For legacy design reasons, they have overlapping functionality. Although you should call <code>changePowerStateToPriv</code> to change your device's power state, you might need to call <code>changePowerStateTo</code> in the following circumstances:
+    
+    <ul><li>If a driver will be using <code>changePowerStateToPriv</code> to change its power state, it should call <code>changePowerStateTo(0)</code> in its <code>start</code> routine to eliminate the influence <code>changePowerStateTo</code> has on power state calculations.
+    
+    <li>Call <code>changePowerStateTo</code> in conjunction with @link setIdleTimerPeriod setIdleTimerPeriod@/link and @link activityTickle activityTickle@/link to idle a driver into a low power state. For a driver with 3 power states, for example, <code>changePowerStateTo(1)</code> sets a minimum level of power state 1, such that the idle timer period may not set your device's power any lower than state 1.</ul>
+
+    @param ordinal The number of the desired power state in the power state array. 
+    @result A return code that can be ignored by the caller. */
     virtual IOReturn changePowerStateTo ( unsigned long ordinal );
 
 /*! @function currentCapability
-        Some object calls a policy-maker here to find out the current capability of a device.
-        The policy-maker returns a copy of the capabilityFlags field for the current power
-        state in the power state array. */
+    @abstract Finds out the capability of a device's current power state.
+    @result A copy of the <code>capabilityFlags</code> field for the current power state in the power state array. 
+    */
    virtual  IOPMPowerFlags currentCapability ( void );
 
 /*! @function currentPowerConsumption
-        Some object calls a policy-maker here to find out the current power consumption of a device.
-        The policy-maker returns a copy of the staticPower field for the current power state in the
-        power state array. */
+    @abstract Finds out the current power consumption of a device.
+    @discussion Most Mac OS X power managed drivers do not report their power consumption via the <code>staticPower</code> field. Thus this call will not accurately reflect power consumption for most drivers.
+    @result A copy of the <code>staticPower</code> field for the current power state in the power state array. 
+*/
     virtual  unsigned long currentPowerConsumption ( void );
 
 /*! @function activityTickle
-        A principal function of a policy-maker is deciding when the device is idle and can be
-        powered down.  To do this it needs to know when the device is being used.  In some
-        cases it is in the data path to the device so it knows when it is being used.  In others
-        it is not and must be told.  The activityTickle method is provided for objects in the
-        system to tell a policy-maker that its device is being used.
-
-        If the policy-maker is managing the idleness determination totally on its own, the
-        paramter should be kIOPMSubclassPolicy, and the policy-maker should intercept
-        the activityTickle call, because the superclass will do nothing with it.
-
-        The IOService superclass can manage idleness determination, too, with the simple
-        mechanism of an idle timer and this activityTickle call.  To start this up, the policy-
-        maker calls its superclass at setIdleTimerPeriod.  This starts a timer for the time
-        interval specified in the call.  When the timer expires, the superclass checks to see
-        if there has been any activity since the last timer expiration. (It checks to see if
-        activityTickle has been called).  If there has been activity, it restarts the timer, and
-        this process continues.  When the timer expires, and there has been no device
-        activity, the superclass lowers the device power state to the next lower state.
-        This can continue until the device is in state zero.
-
-        After the device has been powered down by at least one power state,
-        a call to activityTickle will cause the device to be switched to a higher state
-        required for the activity.
-
-        activityTickle in the IOService superclass is meant to be called by sub-classed
-        policy-makers, because only they understand the paramters.  They may implement
-        an activityTickle for their clients and then call this activityTickle in the superclass.
-        @param type
-        activityTickle with parameter kIOPMSubclassPolicy is not handled in IOService
-        and should be intercepted by the subclass policy-maker.
-        activityTickle with parameter kIOPMSuperclassPolicy1 causes an activity flag to be set,
-        and the device state checked.  If the device has been powered down, it is powered up again.
-        @param stateNumber
-        When the type parameter is kIOPMSuperclassPolicy1, the stateNumber contains
-        the desired power state ordinal for the activity.  If the device is in a lower state,
-        the superclass will switch it to this state.  This is for devices which can handle
-        some accesses in lower power states than others; the device is powered up only
-        as far as it needs to be for the activity.
-        @result
-        When the type parameter is kIOPMSuperclassPolicy1, the superclass returns true
-        if the device is currently in the state specified by stateNumber.  If it is in a lower
-        state and must be brought up, it returns false.  In this case the superclass will
-        cause the device to be brought up. */
-    virtual bool activityTickle ( unsigned long type, unsigned long stateNumber=0 );
+    @abstract Informs power management when a power-managed device is in use, so that power management can track when it is idle and adjust its power state accordingly.
+    @discussion The <code>activityTickle</code> method is provided for objects in the system (or for the driver itself) to tell a driver that its device is being used.
+
+    The IOService superclass can manage idleness determination with a simple idle timer mechanism and this <code>activityTickle</code> call. To start this up, the driver calls its superclass's <code>setIdleTimerPeriod</code>. This starts a timer for the time interval specified in the call. When the timer expires, the superclass checks to see if there has been any activity since the last timer expiration. (It checks to see if <code>activityTickle</code> has been called). If there has been activity, it restarts the timer, and this process continues. When the timer expires, and there has been no device activity, the superclass lowers the device power state to the next lower state. This can continue until the device is in state zero.
+
+    After the device has been powered down by at least one power state, a subsequent call to <code>activityTickle</code> causes the device to be switched to a higher state required for the activity.
+    
+    If the driver is managing the idleness determination totally on its own, the value of the <code>type</code> parameter should be <code>kIOPMSubclassPolicy</code>, and the driver should override the <code>activityTickle</code> method. The superclass IOService implementation of <code>activityTickle</code> does nothing with the <code>kIOPMSubclassPolicy</code> argument.
+
+    @param type When <code>type</code> is <code>kIOPMSubclassPolicy</code>, <code>activityTickle</code> is not handled in IOService and should be intercepted by the subclass. When <code>type</code> is <code>kIOPMSuperclassPolicy1</code>, an activity flag is set and the device state is checked. If the device has been powered down, it is powered up again. 
+    @param stateNumber When <code>type</code> is <code>kIOPMSuperclassPolicy1</code>, <code>stateNumber</code> contains the desired power state ordinal for the activity. If the device is in a lower state, the superclass will switch it to this state. This is for devices that can handle some accesses in lower power states; the device is powered up only as far as it needs to be for the activity. 
+    @result When <code>type</code> is <code>kIOPMSuperclassPolicy1</code>, the superclass returns <code>true</code> if the device is currently in the state specified by <code>stateNumber</code>. If the device is in a lower state and must be powered up, the superclass returns <code>false</code>; in this case the superclass will initiate a power change to power the device up. 
+*/
+    virtual bool activityTickle ( 
+                unsigned long type, 
+                unsigned long stateNumber=0 );
 
 /*! @function setAggressiveness
-        The parent of a policy-maker calls it here while broadcasting an aggressiveness factor
-        around the power management hierarchy.
-
-        A policy-maker may want to intercept this call if it needs to do something with the
-        new factor, like change its idle timeout, for example.  A policy-maker which does
-        intercept should call setAggressiveness in its superclass, though.
-        @param type
-        There are several aggressiveness factors which can be broadcast.  One is a general
-        aggressiveness factor, and the others are specific to parts of the system, like the
-        hard drive or the display.  A policy-maker takes action only on a factor that applies
-        to its policy.  These factor types (e.g. kPMSetGeneralAggressiveness) are defined
-        in pwr_mgt/IOPM.h.
-        @param newLevel
-        This is the aggressiveness factor's new value.
-        @result
-        setAggressiveness returns IOPMNoErr. */
-    virtual  IOReturn setAggressiveness ( unsigned long, unsigned long newLevel );
-
-    /*! @function getAggressiveness
-        Return the current aggressiveness value for the given type. 
-     */
-    virtual IOReturn getAggressiveness ( unsigned long, unsigned long * );
-
-    /*! @function systemWake
-        The parent of a policy-maker calls it here while broadcasting a system wake event.
-
-        A policy-maker must intercept this call if its device can wake the system from sleep.
-        It should check to see if its device did in fact wake the system, and if so, treat the
-        waking action as activity:  it should request power from its parent to keep the system
-        up until it idles again.
-
-        A policy-maker which does  intercept should call systemWake in its superclass.
-        @result
-        systemWake returns IOPMNoErr. */
+    @abstract Broadcasts an aggressiveness factor from the parent of a driver to the driver.
+
+    @discussion Implement <code>setAggressiveness</code> to receive a notification when an "aggressiveness Aggressiveness factors are a loose set of power management variables that contain values for system sleep timeout, display sleep timeout, whether the system is on battery or AC, and other power management features. There are several aggressiveness factors that can be broadcast and a driver may take action on whichever factors apply to it.
+
+    A driver that has joined the power plane via @link joinPMtree joinPMtree@/link will receive <code>setAgressiveness</code> calls when aggressiveness factors change.
+    
+    A driver may override this call if it needs to do something with the new factor (such as change its idle timeout). If overridden, the driver must  call its superclass's <code>setAgressiveness</code> method in its own <code>setAgressiveness</code> implementation.
+    
+    Most drivers do not need to implement <code>setAgressiveness</code>.
+    
+    @param type The aggressiveness factor type, such as <code>kPMMinutesToDim</code>, <code>kPMMinutesToSpinDown</code>, <code>kPMMinutesToSleep</code>, and <code>kPMPowerSource</code>. (Aggressiveness factors are defined in <code>pwr_mgt/IOPM.h</code>.)
+    
+    @param newLevel The aggressiveness factor's new value. 
+    @result <code>IOPMNoErr</code>. 
+*/
+    virtual  IOReturn setAggressiveness ( 
+                unsigned long type, 
+                unsigned long newLevel );
+
+/*! @function getAggressiveness
+    @abstract Returns the current aggressiveness value for the given type. 
+    @param type The aggressiveness factor to query.
+    @param currentLevel Upon successful return, contains the value of aggressiveness factor <code>type</code>.
+    @result <code>kIOReturnSuccess</code> upon success; an I/O Kit error code otherwise.
+ */
+    virtual IOReturn getAggressiveness ( 
+                unsigned long type, 
+                unsigned long *currentLevel );
+
+/*! @function systemWake
+    @abstract Tells every driver in the power plane that the system is waking up.  
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual  IOReturn systemWake ( void );
 
-    /*! @function temperatureCriticalForZone
-        A policy-maker calls its parent power domain to alert it to critical temperature in
-        some thermal zone.
-        @param whichZone
-        This is a pointer to the IOService policy-maker for the thermal zone which has
-        reported critical temperature.
-        @result
-        temperatureCriticalForZone returns IOPMNoErr. */
+/*! @function temperatureCriticalForZone
+    @abstract Alerts a driver to a critical temperature in some thermal zone.  
+    @discussion This call is unused by power management. It is not intended to be called or overridden.
+*/
     virtual  IOReturn temperatureCriticalForZone ( IOService * whichZone );
 
 /*! @function youAreRoot
-        The Platform Expert instantiates the root power domain IOService and
-        calls it here to inform it that it is the root power domain.
-        (The only difference between the root domain and any other power domain
-         is that the root has no parent and therefore never calls it. */
+    @abstract Informs the root power domain IOService object that is is the root power domain.
+    @discussion The Platform Expert instantiates the root power domain IOService object and calls it with this method to inform it that it is the root power domain.
+	
+	This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual IOReturn youAreRoot ( void );
 
 /*! @function setPowerParent
-        The Platform Expert or some other IOService calls a policy-maker here to
-        inform it who its parent is in the power management hierarchy.  This is
-        part of the process of attaching a policy-maker into the hierarchy.
-        @param theParent
-        This is a pointer to the parent IOService power domain.
-        @param stateKnown
-        This is true if the parent knows its power state. (It would not if it doesn't yet
-        have a parent or a controlling driver)
-        @param currentState
-        If the stateKnown parameter is true, these flags describe the character of
-        power in the power domain.  If the policy-maker has a controlling driver,
-        the policy-maker asks the driver, given this power domain state,
-        what state it would be in, and then it  tells the driver to assume that state. */
-    virtual IOReturn setPowerParent ( IOPowerConnection * theParent, bool stateKnown, IOPMPowerFlags currentState );
+    @abstract For internal use only; deprecated; not intended to be called or overridden.
+*/
+    virtual IOReturn setPowerParent ( 
+                IOPowerConnection * theParent, 
+                bool stateKnown, 
+                IOPMPowerFlags currentState );
 
 /*! @function addPowerChild
-        The Platform Expert or some other IOService calls a power domain policy-maker
-        here to introduce it to a child of it, a member of the domain.
-        @param theChild
-        This is a pointer to the child IOService, which is another power domain policy-maker
-        or a device policy-maker. */
+    @abstract Informs a driver that it has a new child. 
+    @discussion The Platform Expert uses this method to call a driver and introduce it to a new child.
+	
+	This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+    @param theChild A pointer to the child IOService object.
+*/
     virtual IOReturn addPowerChild ( IOService * theChild );
 
 /*! @function removePowerChild
-        A power domain policy-maker is called here to tell it that one of its enclosed members
-        is disappearing.  This happens when a device policy-maker hands off its responsibility
-        to another policy-maker or when its device disappears. */
+    @abstract Informs a power managed driver that one of its power plane childen is disappearing. 
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+
+*/
     virtual IOReturn removePowerChild ( IOPowerConnection * theChild );
 
 /* @function command_received
-            */
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual void command_received ( void *, void * , void * , void *);
 
 /* @function start_PM_idle_timer
-            */
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual void start_PM_idle_timer ( void );
 
 /* @function PM_idle_timer_expiration
-            */
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual void PM_idle_timer_expiration ( void );
 
 /* @function PM_Clamp_Timer_Expired
-            */
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     virtual void PM_Clamp_Timer_Expired (void);
 
 /*! @function setIdleTimerPeriod
-        A policy-maker which uses the type 1 idleness determination provided by IOService
-        calls its superclass here to set or change the idle timer period.
-
-        See activityTickle for a description of this idleness determination.
-        @param period
-        This is the desired idle timer period in seconds.
-        @result
-        The normal return is IOPMNoErr, but it is possible to return kIOReturnError if there
-        was difficulty creating the timer event or the command queue, for example (which is
-        done only on the first call.) */
+    @abstract Sets or changes the idle timer period.
+    @discussion A driver using the idleness determination provided by IOService calls its superclass with this method to set or change the idle timer period. See @link activityTickle activityTickle@/link for a description of this type of idleness determination. 
+    @param period The desired idle timer period in seconds. 
+    @result <code>kIOReturnSuccess</code> if successful. May return <code>kIOReturnError</code> if there was difficulty creating the timer event or the command queue. 
+*/
     virtual IOReturn  setIdleTimerPeriod ( unsigned long );
 
 /*! @function getPMworkloop
-            */
+    @abstract Returns a pointer to the system-wide power management work loop.
+    @discussion Most drivers should create their own work loops to synchronize their code; drivers should not run arbitrary code on the power management work loop.
+*/
     virtual IOWorkLoop *getPMworkloop ( void );
 
+/*! @function getPowerState
+    @abstract Determines a device's power state.
+    @discussion A device's "current power state" is updated at the end of each power state transition (e.g. transition from state 1 to state 0, or state 0 to state 2). This transition includes the time spent powering on or off any power plane children. Thus, if a child calls <code>getPowerState</code> on its power parent during system wake from sleep, the call will return the index to the device's off state rather than its on state.
+    @result The current power state's index into the device's power state array.
+*/
+    UInt32 getPowerState();
+
+
 /* @function ack_timer_ticked
-            */
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     void ack_timer_ticked ( void );
 
 /* @function settleTimerExpired
-            */
+   @abstract For internal use only.
+*/
     void settleTimerExpired ( void );
     
+/* @function serializedAllowPowerChange2
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     IOReturn serializedAllowPowerChange2 ( unsigned long );
+
+/* @function serializedCancelPowerChange2
+    @discussion This call is handled internally by power management. It is not intended to be overridden or called by drivers.
+*/
     IOReturn serializedCancelPowerChange2 ( unsigned long );
 
-// implemented by power-controlling driver...
 
 /*! @function setPowerState
-        A policy-maker (usually its superclass) calls its controlling driver here to change
-        the power state of its device.
-        @param powerStateOrdinal
-        This is the number in the power state array of the state the driver is being
-        instructed to switch to.
-        @param whatDevice
-        This is a pointer to the policy-maker.  It is useful when a single power-controlling
-        driver controls multiple devices and needs to know for which device it is being
-        called.
-        @result
-        The driver returns IOPMAckImplied if it has complied with the request when it
-        returns.  If it has started the process of changing power state but not finished
-        it, it should return a number of microseconds which is an upper limit of the time
-        it will need to finish.  Then, when it has completed the power switch, it should
-        call acknowledgeSetPowerState in the policy-maker. */
-virtual IOReturn setPowerState ( unsigned long powerStateOrdinal, IOService* whatDevice );
+    @abstract Requests a power managed driver to change the power state of its device.
+
+    @discussion A power managed driver must override <code>setPowerState</code> to take part in system power management. After a driver is registered with power management, the system uses <code>setPowerState</code> to power the device off and on for system sleep and wake.
+    
+    Calls to @link PMinit PMinit@/link and @link registerPowerDriver registerPowerDriver@/link enable power management to change a device's power state using <code>setPowerState</code>.
+    
+    <code>setPowerState</code> is called in a clean and separate thread context.
+
+    @param powerStateOrdinal The number in the power state array of the state the driver is being instructed to switch to. 
+
+    @param whatDevice A pointer to the power management object which registered to manage power for this device. In most cases, <code>whatDevice</code> will be equal to your driver's own <code>this</code> pointer.
+    @result The driver must return <code>IOPMAckImplied</code> if it has complied with the request when it returns. Otherwise if it has started the process of changing power state but not finished it, the driver should return a number of microseconds which is an upper limit of the time it will need to finish. Then, when it has completed the power switch, it should call @link acknowledgeSetPowerState acknowledgeSetPowerState@/link. 
+*/
+virtual IOReturn setPowerState ( 
+                unsigned long powerStateOrdinal, 
+                IOService* whatDevice );
 
 /*! @function clampPowerOn
-    This method sets the device to the highest power state and ensures it stays there
-    until a timer of duration length expires.
+    @abstract Deprecated. Do not use.
  */
 virtual void clampPowerOn (unsigned long duration);
 
 /*! @function maxCapabilityForDomainState
-    A policy-maker (usually its superclass) calls its controlling driver here to find out
-    the highest power state possible for a given power domain state.  This happens
-    when the power domain is changing state and the policy-maker wants to find
-    out what states the device is capable of in the new domain state.
-    @param domainState
-    These flags describe the character of domain power in some domain power state.
-    The flags are not understood by the calling policy-maker; they were passed to it
-    by its power domain parent.  They come from the outputPowerCharacter field
-    of a state in the power domain's power state array.
-
-    This method is implemented in a simple way in IOService.  It scans the power state
-    array looking for the highest state whose inputPowerRequirement field exactly
-    matches the parameter.  If more intelligent determination is required, the
-    power-controlling driver should implement the method and override the superclass.
-    @result
-    A state number is returned. */
-virtual unsigned long maxCapabilityForDomainState ( IOPMPowerFlags domainState );
+    @abstract Determines a driver's highest power state possible for a given power domain state. 
+    @discussion This happens when the power domain is changing state and power management needs to determine which state the device is capable of in the new domain state. 
+
+    Most drivers do not need to implement this method, and can rely upon the default IOService implementation. The IOService implementation scans the power state array looking for the highest state whose <code>inputPowerRequirement</code> field exactly matches the value of the <code>domainState</code> parameter. If more intelligent determination is required, the driver itself should implement the method and override the superclass's implementation.
+    
+    @param domainState Flags that describe the character of "domain power"; they represent the <code>outputPowerCharacter</code> field of a state in the power domain's power state array. 
+
+    @result A state number. 
+ */
+virtual unsigned long maxCapabilityForDomainState ( 
+                IOPMPowerFlags domainState );
 
 /*! @function initialPowerStateForDomainState
-    A policy-maker (usually its superclass) calls its controlling driver here to find out
-    which power state the device is in, given the current power domain state.  This
-    happens once, when the policy-maker is initializing, and the controlling driver
-    can use this to know what state the device is in initially.
-    @param domainState
-    These flags describe the character of domain power in the current state of the
-    power domain.  The flags are not understood by the calling policy-maker; they
-    were passed to it by its power domain parent.  They come from the
-    outputPowerCharacter field of the current power state in the power domain's
-    power state array.
-
-    This method is implemented in a simple way in IOService.  It scans the power state
-    array looking for the highest state whose inputPowerRequirement field exactly
-    matches the parameter.  If more intelligent determination is required, the
-    power-controlling driver should implement the method and override the superclass.
-    @result
-    A state number is returned. */
-virtual unsigned long initialPowerStateForDomainState ( IOPMPowerFlags );
+    @abstract Determines which power state a device is in, given the current power domain state.
+    @discussion Power management calls this method once, when the driver is initializing power management.
+
+    Most drivers do not need to implement this method, and can rely upon the default IOService implementation. The IOService implementation scans the power state array looking for the highest state whose <code>inputPowerRequirement</code> field exactly matches the value of the <code>domainState</code> parameter. If more intelligent determination is required, the power managed driver should implement the method and override the superclass's implementation. 
+
+    @param domainState Flags that describe the character of "domain power"; they represent the <code>outputPowerCharacter</code> field of a state in the power domain's power state array. 
+    @result A state number. 
+*/
+virtual unsigned long initialPowerStateForDomainState ( 
+                IOPMPowerFlags domainState);
 
 /*! @function powerStateForDomainState
-    A policy-maker (usually its superclass) calls its controlling driver here to find out
-    what power state the device would be in for a given power domain state.  This
-    happens when the power domain is changing state and the policy-maker wants
-    to find out the effect of the change.
-    @param domainState
-    These flags describe the character of domain power in some domain power state.
-    The flags are not understood by the calling policy-maker; they were passed to it
-    by its power domain parent.  They come from the outputPowerCharacter field
-    of a state in the power domain's power state array.
-
-    This method is implemented in a simple way in IOService.  It scans the power state
-    array looking for the highest state whose inputPowerRequirement field exactly
-    matches the parameter.  If more intelligent determination is required, the
-    power-controlling driver should implement the method and override the superclass.
-    @result
-    A state number is returned. */
+    @abstract Determines what power state the device would be in for a given power domain state. 
+    @discussion Power management calls a driver with this method to find out what power state the device would be in for a given power domain state. This happens when the power domain is changing state and power management needs to determine the effect of the change. 
+
+    Most drivers do not need to implement this method, and can rely upon the default IOService implementation. The IOService implementation scans the power state array looking for the highest state whose <code>inputPowerRequirement</code> field exactly matches the value of the <code>domainState</code> parameter. If more intelligent determination is required, the power managed driver should implement the method and override the superclass's implementation.
+
+    @param domainState Flags that describe the character of "domain power"; they represent the <code>outputPowerCharacter</code> field of a state in the power domain's power state array. 
+
+    @result A state number. 
+*/
 virtual unsigned long powerStateForDomainState ( IOPMPowerFlags domainState );
 
 /*! @function powerStateWillChangeTo
-    A policy-maker informs interested parties that its device is about to change to
-    a different power state.  Interested parties are those that have registered for
-    this notification via registerInterestedDriver and also the power-controlling
-    driver which is registered as an interested driver automatically when it registers
-    as the controlling driver.
-    @param capabilities
-    These flags describe the capability of the device in the new power state.  They
-    are not understood by the policy-maker; they come from the capabilityFlags field
-    of the new state in the power state array.
-    @param stateNumber
-    This is the number of the state in the state array that the device is switching to.
-    @param whatDevice
-    This points to the policy-maker, and it is used by a driver which is receiving power
-    state change notifications for multiple devices.
-    @result
-    The driver returns IOPMAckImplied if it has prepared for the power change when it
-     returns.  If it has started preparing but not finished, it should return a number of
-    microseconds which is an upper limit of the time  it will need to finish preparing.
-    Then, when it has completed its preparations, it should call acknowledgePowerChange
-    in the policy-maker. */
-virtual IOReturn powerStateWillChangeTo ( IOPMPowerFlags, unsigned long, IOService* );
+    @abstract Informs interested parties that a device is about to change its power state.
+    @discussion Power management informs interested parties that a device is about to change to a different power state. Interested parties are those that have registered for this notification via @link registerInterestedDriver registerInterestedDriver@/link. If you have called <code>registerInterestedDriver</code> on a power managed driver, you must implement <code>powerStateWillChangeTo</code> and @link powerStateDidChangeTo powerStateDidChangeTo@/link to receive the notifications.
+
+    <code>powerStateWillChangeTo</code> is called in a clean and separate thread context.
+    
+    <code>powerStateWillChangeTo</code> is called before a power state transition takes place; <code>powerStateDidChangeTo</code> is called after the transition has completed.
+    
+    @param capabilities Flags that describe the capability of the device in the new power state (they come from the <code>capabilityFlags</code> field of the new state in the power state array). 
+	@param stateNumber The number of the state in the state array that the device is switching to. 
+    @param whatDevice A pointer to the driver that is changing. It can be used by a driver that is receiving power state change notifications for multiple devices to distinguish between them. 
+    @result The driver returns <code>IOPMAckImplied</code> if it has prepared for the power change when it returns. If it has started preparing but not finished, it should return a number of microseconds which is an upper limit of the time it will need to finish preparing. Then, when it has completed its preparations, it should call @link acknowledgePowerChange acknowledgePowerChange@/link. 
+*/
+virtual IOReturn powerStateWillChangeTo ( 
+                    IOPMPowerFlags  capabilities, 
+                    unsigned long   stateNumber, 
+                    IOService*      whatDevice);
 
 /*! @function powerStateDidChangeTo
-    A policy-maker informs interested parties that its device has changed to
-    a different power state.  Interested parties are those that have registered for
-    this notification via registerInterestedDriver and also the power-controlling
-    driver which is registered as an interested driver automatically when it registers
-    as the controlling driver.
-    @param capabilities
-    These flags describe the capability of the device in the new power state.  They
-    are not understood by the policy-maker; they come from the capabilityFlags field
-    of the new state in the power state array.
-    @param stateNumber
-    This is the number of the state in the state array that the device has switched to.
-    @param whatDevice
-    This points to the policy-maker, and it is used by a driver which is receiving power
-    state change notifications for multiple devices.
-    @result
-    The driver returns IOPMAckImplied if it has prepared for the power change when it
-     returns.  If it has started preparing but not finished, it should return a number of
-    microseconds which is an upper limit of the time  it will need to finish preparing.
-    Then, when it has completed its preparations, it should call acknowledgePowerChange
-    in the policy-maker. */
-virtual IOReturn powerStateDidChangeTo ( IOPMPowerFlags, unsigned long, IOService* );
+    @abstract Informs interested parties that a device has changed to a different power state.
+    @discussion Power management informs interested parties that a device has changed to a different power state. Interested parties are those that have registered for this notification via @link registerInterestedDriver registerInterestedDriver@/link. If you have called <code>registerInterestedDriver</code> on a power managed driver, you must implemnt @link powerStateWillChangeTo powerStateWillChangeTo@/link and <code>powerStateDidChangeTo</code> to receive the notifications.
+    
+    <code>powerStateDidChangeTo</code> is called in a clean and separate thread context.
+    
+    <code>powerStateWillChangeTo</code> is called before a power state transition takes place; <code>powerStateDidChangeTo</code> is called after the transition has completed.
+    
+    @param capabilities Flags that describe the capability of the device in the new power state (they come from the <code>capabilityFlags</code> field of the new state in the power state array). 
+	@param stateNumber The number of the state in the state array that the device is switching to. 
+    @param whatDevice A pointer to the driver that is changing. It can be used by a driver that is receiving power state change notifications for multiple devices to distinguish between them. 
+    @result The driver returns <code>IOPMAckImplied</code> if it has prepared for the power change when it returns. If it has started preparing but not finished, it should return a number of microseconds which is an upper limit of the time it will need to finish preparing. Then, when it has completed its preparations, it should call @link acknowledgePowerChange acknowledgePowerChange@/link. 
+*/
+virtual IOReturn powerStateDidChangeTo ( 
+                    IOPMPowerFlags  capabilities, 
+                    unsigned long   stateNumber, 
+                    IOService*      whatDevice);
 
 /*! @function didYouWakeSystem
-    A policy-maker calls its power driver here to ask if its device is the one
-    which just woke the system from sleep.
-    @result
-    The driver returns true if it did wake the system and false if it didn't. */
+    @abstract Asks a driver if its device is the one that just woke the system from sleep. 
+    @discussion Power management calls a power managed driver with this method to ask if its device is the one that just woke the system from sleep. If a device is capable of waking the system from sleep, its driver should implement <code>didYouWakeSystem</code> and return <code>true</code> if its device was responsible for waking the system.
+    @result <code>true</code> if the driver's device did wake the system and <code>false</code> if it didn't. 
+*/
 virtual bool didYouWakeSystem  ( void );
 
 /*! @function newTemperature
-    A thermal-zone driver calls its policy-maker here to tell it that the temperature in
-    the zone has changed.  The thermal-zone policy-maker uses this information to
-    manage its thermal zone.
-    @param currentTemp
-    This is the new temperature in the thermal zone.
-    @param whichZone
-    This is a pointer to the controlling driver.
-        */
+    @abstract (Deprecated. Do not use.) Tells a power managed driver that the temperature in the thermal zone has changed.
+    @discussion A thermal-zone driver calls a power managed driver with this method to tell it that the temperature in the zone has changed. This method is not intended to be overridden or called by drivers. This method is deprecated.
+*/
 virtual IOReturn newTemperature  ( long currentTemp, IOService * whichZone );
 
     virtual bool askChangeDown ( unsigned long );
@@ -1749,71 +1653,63 @@ virtual IOReturn newTemperature  ( long currentTemp, IOService * whichZone );
     virtual IOReturn allowPowerChange ( unsigned long refcon );
     virtual IOReturn cancelPowerChange ( unsigned long refcon );
 
-// ...implemented by power-controlling driver
 
     protected:
-/*! @function changePowerStateToPriv
-    A policy-maker calls its superclass here to change the power state of the device.
-    The superclass takes care of making sure the power domain state is appropriate
-    and informing interested parties.  It calls the controlling driver to make the change.
-    @param ordinal
-    This is the number, in the power state array, of the desired power state.
-    @result
-    The return code reflects the state of the policy-maker's internal queue of power
-    changes and can be ignored by the caller.          
-            */
+/*! @function changePowerStateToPriv 
+    @abstract Tells a driver's superclass to change the power state of its device.
+    @discussion A driver uses this method to tell its superclass to change the power state of the device. This is the recommended way to change the power state of a device. 
+    
+    Three things affect driver power state: @link changePowerStateTo changePowerStateTo@/link, <code>changePowerStateToPriv</code>, and the desires of the driver's power plane children. Power management puts the device into the maximum state governed by those three entities. 
+    
+    Drivers may eliminate the influence of the <code>changePowerStateTo</code> method on power state one of two ways. See @link powerOverrideOnPriv powerOverrideOnPriv@/link to ignore the method's influence, or call <code>changePowerStateTo(0)</code> in the driver's <code>start</code> routine to remove the <code>changePowerStateTo</code> method's power request.
+    
+    @param ordinal The number of the desired power state in the power state array. 
+    @result A return code that can be ignored by the caller. 
+*/
     IOReturn changePowerStateToPriv ( unsigned long ordinal );
 
 /*! @function powerOverrideOnPriv
-    A policy-maker normally keeps its device at the highest state required by itself,
-    its power-controlling driver, and its children (when the power domain state
-    allows).  There may be times, however, when a policy-maker needs the power
-    state lower than its driver or its children desire, and when this is the case, it
-    calls powerOverrideOnPriv in its superclass to enable this override.  When the override
-    is on, the superclass keeps the device in the state desired by the policy-maker
-    (requested via changePowerStateToPriv), regardless of the children's or driver's desire.
-    Turning on the override will initiate a power change if the policy-maker's desired
-    power state is different from the maximum of the controlling driver's desire and
-    the children's desires.
-    @result
-    The return code reflects the state of the policy-maker's internal queue of power
-    changes and can be ignored by the caller.  */
+    @abstract Allows a driver to ignore its children's power management requests and only use changePowerStateToPriv to define its own power state.
+
+    @discussion Power management normally keeps a device at the highest state required by its requests via @link changePowerStateTo changePowerStateTo@/link, @link changePowerStateToPriv changePowerStateToPriv@/link, and its children. However, a driver may ensure a lower power state than otherwise required by itself and its children using <code>powerOverrideOnPriv</code>.
+    
+    When the override is on, power management keeps the device's power state in the state specified by <code>changePowerStateToPriv</code>.
+    
+    Turning on the override will initiate a power change if the driver's <code>changePowerStateToPriv</code> desired power state is different from the maximum of the <code>changePowerStateTo</code> desired power state and the children's desires. 
+    
+    @result A return code that can be ignored by the caller.
+
+*/
     IOReturn powerOverrideOnPriv ( void );
 
 /*! @function powerOverrideOffPriv
-        When a policy-maker has enabled the override, it can disable it again by calling
-        this method in its superclass.  This will allow the superclass to keep the device
-         at the highest state required by itself, its power-controlling driver, and its
-        children (when the power domain state allows).  Turning off the override
-        will initiate a power change if the policy-maker's desired power state is different
-        from the maximum of the controlling driver's desire and the children's desires.
-        @result
-        The return code reflects the state of the policy-maker's internal queue of power
-        changes and can be ignored by the caller.  */
+    @abstract Allows a driver to disable a power override.
+
+    @discussion When a driver has enabled an override via @link powerOverrideOnPriv powerOverrideOnPriv@/link, it can disable it again by calling this method in its superclass. Disabling the override reverts to the default algorithm for determining a device's power state. The superclass will now keep the device at the highest state required by <code>changePowerStateTo</code>, <code>changePowerStateToPriv</code>, and its children.
+    
+    Turning off the override will initiate a power change if the driver's desired power state is different from the maximum of the power managed driver's desire and the children's desires. 
+    
+    @result A return code that can be ignored by the caller.
+*/
     IOReturn powerOverrideOffPriv ( void );
 
         /*! @function powerChangeDone
-            A policy-maker calls itself here when a power change is completely done, when
-        all interested parties have acknowledged the powerStateDidChangeTo call.
-        The implementation here is null; the method is meant to be overridden by
-        subclassed policy-makers, and that is how one finds out that a power change
-        it initiated is complete
-            @param stateNumber
-            This is the number of the state in the state array that the device has switched from.  */
-    virtual void powerChangeDone ( unsigned long );
+    @abstract Tells a driver when a power change is complete.
+    
+    @discussion Power management uses this method to call into a driver when a power change is completely done, when all interested parties have acknowledged the @link powerStateDidChangeTo powerStateDidChangeTo@/link call. The default implementation of this method is null; the method is meant to be overridden by subclassed power managed drivers. A driver should use this method to find out if a power change it initiated is complete. 
+    @param stateNumber The number of the state in the state array that the device has switched from.
+*/
+    virtual void powerChangeDone ( unsigned long stateNumber);
     
     bool tellClientsWithResponse ( int messageType );
     void tellClients ( int messageType );
 
 private:
-
     IOReturn enqueuePowerChange ( unsigned long, unsigned long, unsigned long, IOPowerConnection *, unsigned long );
-    void setParentInfo ( IOPMPowerFlags, IOPowerConnection * );
+	void setParentInfo ( IOPMPowerFlags, IOPowerConnection *, bool );
     IOReturn notifyAll ( bool is_prechange );
     bool notifyChild ( IOPowerConnection * nextObject, bool is_prechange );
-    bool inform ( IOPMinformee * nextObject, bool is_prechange );
-    
-    // Power Management state machine
+
     // power change initiated by driver
     void OurChangeTellClientsPowerDown ( void );
     void OurChangeTellPriorityClientsPowerDown ( void );
@@ -1824,48 +1720,75 @@ private:
     void OurChangeFinish ( void );
     
     // downward power change initiated by a power parent
-    IOReturn ParentDownTellPriorityClientsPowerDown_Immediate ( void );
-    IOReturn ParentDownNotifyInterestedDriversWillChange_Immediate ( void );
-    void ParentDownTellPriorityClientsPowerDown_Delayed ( void );
-    void ParentDownNotifyInterestedDriversWillChange_Delayed ( void );
-    IOReturn ParentDownSetPowerState_Immediate ( void );
-    IOReturn ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate ( void );
-    void ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed ( void );
-    void ParentDownSetPowerState_Delayed ( void );
-    void ParentDownWaitForPowerSettle_Delayed ( void );
-    void ParentDownAcknowledgeChange_Delayed ( void );
+    void ParentDownTellPriorityClientsPowerDown ( void );
+    void ParentDownNotifyInterestedDriversWillChange ( void );
+    void ParentDownNotifyDidChangeAndAcknowledgeChange ( void );
+    void ParentDownSetPowerState ( void );
+    void ParentDownWaitForPowerSettle ( void );
+    void ParentDownAcknowledgeChange ( void );
     
     // upward power change initiated by a power parent
-    void ParentUpSetPowerState_Delayed ( void );
-    IOReturn ParentUpSetPowerState_Immediate ( void );
-    IOReturn ParentUpWaitForSettleTime_Immediate ( void );
-    IOReturn ParentUpNotifyInterestedDriversDidChange_Immediate ( void );
-    void ParentUpWaitForSettleTime_Delayed ( void );
-    void ParentUpNotifyInterestedDriversDidChange_Delayed ( void );
-    void ParentUpAcknowledgePowerChange_Delayed ( void );
+    void ParentUpSetPowerState ( void );
+    void ParentUpWaitForSettleTime ( void );
+    void ParentUpNotifyInterestedDriversDidChange ( void );
+    void ParentUpAcknowledgePowerChange ( void );
     
     void all_done ( void );
-    void all_acked ( void );
-    void driver_acked ( void );
     void start_ack_timer ( void );
     void stop_ack_timer ( void );
-
     unsigned long compute_settle_time ( void );
     IOReturn startSettleTimer ( unsigned long delay );
     IOReturn changeState ( void );
-    IOReturn add_child_to_active_change ( IOPowerConnection * );
-    IOReturn add_driver_to_active_change ( IOPMinformee * );
-    IOReturn instruct_driver ( unsigned long newState );
-    bool acquire_lock ( void );
-    IOReturn start_parent_change ( unsigned long queue_head );
-    void start_our_change ( unsigned long queue_head );
     IOReturn ask_parent ( unsigned long requestedState );
     bool checkForDone ( void );
-    bool responseValid ( unsigned long x );
-    IOReturn allowCancelCommon ( void );
-    void computeDesiredState ( void );
+    bool responseValid ( unsigned long x, int pid );
+    void computeDesiredState ( unsigned long tempDesire = 0 );
     void rebuildChildClampBits ( void );
-    IOReturn temporaryMakeUsable ( void );
+
+	static void ack_timer_expired( thread_call_param_t, thread_call_param_t );
+	static IOReturn actionAckTimerExpired(OSObject *, void *, void *, void *, void * );
+ 	static IOReturn actionDriverCalloutDone(OSObject *, void *, void *, void *, void * );
+	static IOPMRequest * acquirePMRequest( IOService * target, UInt32 type );
+	static void releasePMRequest( IOPMRequest * request );
+	static void pmDriverCallout( IOService * from );
+	static void pmTellClientWithResponse( OSObject * object, void * context );
+	static void pmTellAppWithResponse( OSObject * object, void * context );
+	bool ackTimerTick( void );
+	void addPowerChild1( IOPMRequest * request );
+	void addPowerChild2( IOPMRequest * request );
+	void addPowerChild3( IOPMRequest * request );
+    void adjustPowerState( void );
+	void start_ack_timer( UInt32 value, UInt32 scale );
+	void handlePMstop( IOPMRequest * request );
+	void handleRegisterPowerDriver( IOPMRequest * request );
+	bool handleAcknowledgePowerChange( IOPMRequest * request );
+	void handlePowerDomainWillChangeTo( IOPMRequest * request );
+	void handlePowerDomainDidChangeTo( IOPMRequest * request );
+	void handleMakeUsable( IOPMRequest * request );
+	void handleChangePowerStateTo( IOPMRequest * request );
+	void handleChangePowerStateToPriv( IOPMRequest * request );
+	void handlePowerOverrideChanged( IOPMRequest * request );
+	void handleInterestChanged( IOPMRequest * request );
+	void submitPMRequest( IOPMRequest * request );
+	void submitPMRequest( IOPMRequest ** request, IOItemCount count );
+	void executePMRequest( IOPMRequest * request );
+	bool servicePMRequest( IOPMRequest * request, IOPMWorkQueue * queue  );
+	bool retirePMRequest(  IOPMRequest * request, IOPMWorkQueue * queue );
+	bool servicePMRequestQueue( IOPMRequest * request, IOPMRequestQueue * queue );
+	bool servicePMReplyQueue( IOPMRequest * request, IOPMRequestQueue * queue );
+	bool servicePMFreeQueue( IOPMRequest * request, IOPMRequestQueue * queue );
+	bool notifyInterestedDrivers( void );
+	void notifyInterestedDriversDone( void );
+	bool notifyControllingDriver( void );
+	void notifyControllingDriverDone( void );
+	void driverSetPowerState( void );
+	void driverInformPowerChange( void );
+	bool isPMBlocked( IOPMRequest * request, int count );
+	void start_our_change( const changeNoteItem * changeNote );
+	IOReturn start_parent_change( const changeNoteItem * changeNote );
+	void notifyChildren( void );
+	void notifyChildrenDone( void );
+    void cleanClientResponses ( bool logErrors );
 };
 
 #endif /* ! _IOKIT_IOSERVICE_H */
diff --git a/iokit/IOKit/IOServicePM.h b/iokit/IOKit/IOServicePM.h
index 55e34fc96..940a75160 100644
--- a/iokit/IOKit/IOServicePM.h
+++ b/iokit/IOKit/IOServicePM.h
@@ -1,415 +1,115 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#ifndef _IOKIT_IOSERVICEPM_H
+#define _IOKIT_IOSERVICEPM_H
+
+#include <libkern/c++/OSObject.h>
+#include <IOKit/IOLocks.h>
+#include <IOKit/pwr_mgt/IOPM.h>
+
 extern "C" {
 #include <kern/thread_call.h>
 }
 
-#include <libkern/c++/OSObject.h>
-#include <IOKit/IOLocks.h>
+class IOService;
+class IOServicePM;
+class IOPowerConnection;
 class IOPMinformee;
 class IOPMinformeeList;
-class IOPMchangeNoteList;
-class IOPMpmChild;
 class IOWorkLoop;
-class IOCommandQueue;
 class IOCommandGate;
 class IOTimerEventSource;
 class IOPlatformExpert;
-
-#include <IOKit/pwr_mgt/IOPM.h>
-
-
-/*!
-@defined ACK_TIMER_PERIOD
-@discussion When an IOService is waiting for acknowledgement to a power state change
-notification from an interested driver or the controlling driver its ack timer is ticking every tenth of a second.
-(100000000 nanoseconds are one tenth of a second).
-*/
- #define ACK_TIMER_PERIOD 100000000
-
-
-
-/*!
-@class IOPMpriv
-@abstract Private power management private instance variables for IOService objects.
-*/
-class IOPMpriv : public OSObject
-{
-    friend class IOService;
-
-    OSDeclareDefaultStructors(IOPMpriv)
-
-    public:
-
-/*! @var we_are_root 
-    TRUE if this device is the root power domain.
-*/
-    bool			we_are_root;
-    
-    /*! @var interestedDrivers 
-        List of interested drivers.
-    */
-    IOPMinformeeList *	interestedDrivers;
-    
-    /*! @var children
-        List of power domain children.
-    */
-    IOPMinformeeList *	children;
+class IOPMWorkQueue;
+class IOPMRequest;
+class IOPMRequestQueue;
+struct changeNoteItem;
+
+/* DEPRECATED */
+/*! @class IOPMprot
+    @abstract Protected power management instance variables for IOService objects.
+    @availability Mac OS X version 10.0. Deprecated in version 10.5.
+    @discussion IOPMprot is deprecated. Do not use it in any new code.
     
-    /*! @var changeList
-        List of pending power state changes.
-    */
-    IOPMchangeNoteList *	changeList;
-    
-    /*! @var driver_timer
-        Timeout on waiting for controlling driver to acknowledgeSetPowerState.
-    */
-    IOReturn			driver_timer;
-    
-    /*! @var ackTimer									*/
-    thread_call_t		ackTimer;
-
-    /*! @var settleTimer								*/
-    thread_call_t		settleTimer;
-
-    /*! @var machine_state
-        State number of state machine processing current change note.
-    */
-    unsigned long		machine_state;
-    
-    /*! @var settle_time
-        Settle timer after changing power state.
-    */
-    unsigned long		settle_time;
-    
-    /*! @var head_note
-        Ordinal of change note currently being processed.
-    */
-    long			head_note;
-    
-    /*! @var head_note_flags
-        Copy of flags field in change note currently being processed.
-    */
-    unsigned long		head_note_flags;
-
-    /*! @var head_note_state
-        Copy of newStateNumberfield in change note currently being  processed.
-    */
-    unsigned long		head_note_state;
-
-    /*! @var head_note_outputFlags
-    OutputPowerCharacter field from change note currently being processed.
-    */
-    unsigned long		head_note_outputFlags;
-
-    /*! @var head_note_domainState
-    Power domain flags from parent... (only on parent change).
-    */
-    unsigned long		head_note_domainState;
-
-    /*! @var head_note_parent
-        Pointer to initiating parent... (only on parent change).
-    */
-    IOPowerConnection * 	head_note_parent;
-    
-    /*! @var head_note_capabilityFlags
-        Copy of capabilityFlags field in change note currently being processed.
-    */
-    unsigned long		head_note_capabilityFlags;
-
-    /*! @var head_note_pendingAcks
-        Number of acks we are waiting for during notification.
-    */
-    unsigned long		head_note_pendingAcks;
-
-    /*! @var our_lock
-        Used to control access to head_note_pendingAcks and driver_timer.
-    */
-    IOLock	*		our_lock;
-
-    /*! @var flags_lock
-        Used to control access to response flags array.
-    */
-    IOLock	*		flags_lock;
-
-    /*! @var queue_lock
-        Used to control access to change note queue.
-    */
-    IOLock	*		queue_lock;
-
-    /*! @var initial_change
-        True forces first state to be broadcast even if it isn't a change.
-    */
-    bool			initial_change;
-
-    /*! @var need_to_become_usable
-        Someone called makeUsable before we had a controlling driver.
-    */
-    bool			need_to_become_usable;
-
-    /*! @var device_overrides
-        State changes are made based only on subclass's desire.
-    */
-    bool			device_overrides;
-
-    /*! @var clampOn
-        Domain is clamped on till first child registers.
-    */
-    bool			clampOn;
-
-    /*! @var owner
-        Points to object which made this struct.  Used for debug output only.
-    */
-    IOService * 		owner;
-
-    /*! @var activityLock
-        Used to protect activity flag.
-    */
-    IOLock *		activityLock;
-
-    /*! @var timerEventSrc
-        An idle timer.
-    */
-    IOTimerEventSource * 	timerEventSrc;
-
-    /*! @var idle_timer_period
-        Timer's period in seconds.
-    */
-    unsigned long		idle_timer_period;
-   
-    /*! @var clampTimerEventSrc
-        Timer for clamping power on.
-    */
-    IOTimerEventSource *        clampTimerEventSrc;
-
-    /*! @var device_active
-        True: there has been device activity since last idle timer expiration.
-    */
-    bool			device_active;
-
-    /*! @var device_active_timestamp
-        Time in ticks of last activity.
-    */
-    AbsoluteTime                device_active_timestamp;
-
-    /*! @var driverDesire
-    This is the power state desired by our controlling driver.  It is initialized to myCurrentState and is changed
-    when the controlling driver calls changePowerStateTo.   A change in driverDesire may cause a change in ourDesiredPowerState.
-*/
-    unsigned long		driverDesire;
-
-
-
-    /*! @var deviceDesire
-    This is the power state desired by a subclassed device object.  It is initialized to myCurrentState and is changed when the subclassed object calls changePowerStateToPriv.  A change in deviceDesire may cause a change in ourDesiredPowerState.
-*/
-    unsigned long		deviceDesire;
-
-
-
-    /*! @var ourDesiredPowerState
-This is the power state we desire currently.  If equal to myCurrentState, we're happy.
-Otherwise, we're waiting for the parent to raise the power domain to at least this level.
-    
-If this is a power domain, this is the maximum of all our children's desires, driverDesire, and deviceDesire.
-It increases when:
-a child asks for more power via requestDomainState,
-the controlling driver asks for more power via changePowerStateTo
-
-It decreases when:
-we lose a child and the child had the highest power need of all our children,
-the child with the highest power need suggests a lower power domain state,
-the controlling driver asks for lower power for some reason via changePowerStateTo
-
-If this is not a power domain, ourDesiredPowerState represents the greater of driverDesire and deviceDesire.
-It increases when:
-the controlling driver asks for more power via changePowerStateTo
-some driver calls makeUsable
-a subclassed object asks for more power via changePowerStateToPriv
-
-It decreases when:
-the controlling driver asks for lower power for some reason via changePowerStateTo
-a subclassed object asks for lower power for some reason via changePowerStateToPriv
-*/
-    unsigned long		ourDesiredPowerState;
-
-
-    /*! @var previousRequest
-This is a reminder of what our parent thinks our need is.  Whenever it changes,
-we call requestDomainState in the parent to keep it current.  It is usually equal to ourDesiredPowerState
-except while a power change is in progress.
-*/
-    unsigned long		previousRequest;
-
-
-    /*! @var askingFor
-        Not used.
-*/
-    unsigned long		askingFor;		 
-
-
-    /*! @var imminentState
-        Usually the same as myCurrentState, except right after calling powerStateWillChangeTo.
-*/
-    unsigned long		imminentState;
-
-    /*! @function serialize
-        Serialize private instance variables for debug output (IORegistryDumper).
-*/
-    virtual bool serialize(OSSerialize *s) const;
-
-};
-
-
-
-
-/*!
-@class IOPMprott
-@abstract Protected power management instance variables for IOService objects.
+    Call IOService::getPowerState to query the current power state rather than access myCurrentState.
 */
-class IOPMprot : public OSObject //management
+class IOPMprot : public OSObject
 {
     friend class IOService;
     
     OSDeclareDefaultStructors(IOPMprot)
 
-    public:
-
-        /*! @var ourName
-            From getName(), used in logging.
-        */
-    const char *		ourName;
+public:
+    /*! @var ourName
+        From getName(), used in logging.
+    */
+    const char *            ourName;
 
     /*! @var thePlatform
         From getPlatform, used in logging and registering.
     */
-    IOPlatformExpert * 	thePlatform;
+    IOPlatformExpert *      thePlatform;
 
     /*! @var theNumberOfPowerStates
         The number of states in the array.
     */
-    unsigned long		theNumberOfPowerStates;			// the number of states in the array
+    unsigned long           theNumberOfPowerStates;
 
     /*! @var thePowerStates
         The array.
     */
-    IOPMPowerState	thePowerStates[IOPMMaxPowerStates];
+    IOPMPowerState          thePowerStates[IOPMMaxPowerStates];
 
     /*! @var theControllingDriver
         Points to the controlling driver.
     */
-    IOService * 		theControllingDriver;
+    IOService *             theControllingDriver;
 
     /*! @var aggressiveness
         Current value of power management aggressiveness.
     */
-    unsigned long		aggressiveness;
+    unsigned long           aggressiveness;
 
     /*! @var current_aggressiveness_values
         Array of aggressiveness values.
     */
-    unsigned long               current_aggressiveness_values [kMaxType+1];
+    unsigned long           current_aggressiveness_values [kMaxType+1];
 
     /*! @var current_aggressiveness_validity
         True for values that are currently valid.
     */
-    bool	               current_aggressiveness_valid [kMaxType+1];
+    bool                    current_aggressiveness_valid [kMaxType+1];
 
     /*! @var myCurrentState
         The ordinal of our current power state.
     */
-    unsigned long		myCurrentState;
-
-    /*! @var parentsKnowState
-        True if all our parents know the state of their power domain.
-    */
-    bool		parentsKnowState;
-
-    /*! @var parentsCurrentPowerFlags
-        Logical OR of power flags for the current state of each power domainparent.
-    */
-    IOPMPowerFlags	parentsCurrentPowerFlags;
-
-    /*! @var maxCapability
-        Ordinal of highest state we can achieve in current power domain state.
-    */
-    unsigned long		maxCapability;
-
-    /*! @var PMworkloop
-        Points to the single power management workloop.
-    */
-    IOWorkLoop *		PMworkloop;
-
-    /*! @var commandQueue
-        Used to serialize idle-power-down and busy-power-up.
-    */
-    IOCommandQueue * 	commandQueue;
-
-    /*! @var PMcommandGate
-        Used to serialize timer expirations and incoming acknowledgements.
-    */
-    IOCommandGate *	PMcommandGate;
-
-    /*! @var myCharacterFlags
-        Logical OR of all output power character flags in the array.
-    */
-    IOPMPowerFlags  		myCharacterFlags;
-
-    /*! @var serialNumber
-        Used to uniquely identify power management notification to apps and clients.
-    */
-    UInt16   serialNumber;
-
-    /*! @var responseFlags
-        Points to an OSArray which manages responses from notified apps and clients.
-    */
-    OSArray* responseFlags;
-    
-    /*! @var doNotPowerDown
-        Keeps track of any negative responses from notified apps and clients.
-    */
-    bool	doNotPowerDown;
-    
-    /*! @var childLock
-        Used to serialize scanning the children.
-    */
-    IOLock	*		childLock;
-
-    /*! @var parentLock
-        Used to serialize scanning the parents.
-    */
-    IOLock	*		parentLock;
-
-    /*! @var outofbandparameter
-        Used to communicate desired function to tellClientsWithResponse().
-        This is used because it avoids changing the signatures of the affected virtual methods. 
-    */
-    int				outofbandparameter;
-
-    /*! @function serialize
-Serialize protected instance variables for debug output (IORegistryDumper).
-*/
-    virtual bool serialize(OSSerialize *s) const;
-
+    unsigned long           myCurrentState;
 };
 
+#endif /* !_IOKIT_IOSERVICEPM_H */
diff --git a/iokit/IOKit/IOSharedDataQueue.h b/iokit/IOKit/IOSharedDataQueue.h
new file mode 100644
index 000000000..fdfa1d673
--- /dev/null
+++ b/iokit/IOKit/IOSharedDataQueue.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOKIT_IOSHAREDDATAQUEUE_H
+#define _IOKIT_IOSHAREDDATAQUEUE_H
+
+#ifdef dequeue
+#undef dequeue
+#endif
+
+#include <IOKit/IODataQueue.h>
+
+typedef struct _IODataQueueEntry IODataQueueEntry;
+
+/*!
+ * @class IOSharedDataQueue : public IODataQueue
+ * @abstract A generic queue designed to pass data both from the kernel to a user process and from a user process to the kernel.
+ * @discussion The IOSharedDataQueue class is designed to also allow a user process to queue data to kernel code.  IOSharedDataQueue objects are designed to be used in a single producer / single consumer situation.  As such, there are no locks on the data itself.  Because the kernel enqueue and user-space dequeue methods follow a strict set of guidelines, no locks are necessary to maintain the integrity of the data struct.
+ *
+ * <br>Each data entry can be variable sized, but the entire size of the queue data region (including overhead for each entry) must be specified up front.
+ *
+ * <br>In order for the IODataQueue instance to notify the user process that data is available, a notification mach port must be set.  When the queue is empty and a new entry is added, a message is sent to the specified port.
+ *
+ * <br>In order to make the data queue memory available to a user process, the method getMemoryDescriptor() must be used to get an IOMemoryDescriptor instance that can be mapped into a user process.  Typically, the clientMemoryForType() method on an IOUserClient instance will be used to request the IOMemoryDescriptor and then return it to be mapped into the user process.
+ */
+class IOSharedDataQueue : public IODataQueue
+{
+    OSDeclareDefaultStructors(IOSharedDataQueue)
+
+    struct ExpansionData { 
+    };
+    /*! @var reserved
+        Reserved for future use.  (Internal use only)  */
+    ExpansionData * _reserved;
+
+protected:
+    virtual void free();
+
+public:
+    /*!
+     * @function withCapacity
+     * @abstract Static method that creates a new IOSharedDataQueue instance with the capacity specified in the size parameter.
+     * @discussion The actual size of the entire data queue memory region (to be shared into a user process) is equal to the capacity plus the IODataQueueMemory overhead.  This overhead value can be determined from the DATA_QUEUE_MEMORY_HEADER_SIZE macro in <IOKit/IODataQueueShared.h>.  The size of the data queue memory region must include space for the overhead of each IODataQueueEntry.  This entry overhead can be determined from the DATA_QUEUE_ENTRY_HEADER_SIZE macro in <IOKit/IODataQueueShared.h>.<br>  This method allocates a new IODataQueue instance and then calls initWithCapacity() with the given size parameter.  If the initWithCapacity() fails, the new instance is released and zero is returned.
+     * @param size The size of the data queue memory region.
+     * @result Returns the newly allocated IOSharedDataQueue instance.  Zero is returned on failure.
+     */
+    static IOSharedDataQueue *withCapacity(UInt32 size);
+
+    /*!
+     * @function withEntries
+     * @abstract Static method that creates a new IOSharedDataQueue instance with the specified number of entries of the given size.
+     * @discussion This method will create a new IOSharedDataQueue instance with enough capacity for numEntries of entrySize.  It does account for the IODataQueueEntry overhead for each entry.  Note that the numEntries and entrySize are simply used to determine the data region size.  They do not actually restrict the size of number of entries that can be added to the queue.<br>  This method allocates a new IODataQueue instance and then calls initWithEntries() with the given numEntries and entrySize parameters.  If the initWithEntries() fails, the new instance is released and zero is returned.
+     * @param numEntries Number of entries to allocate space for.
+     * @param entrySize Size of each entry.
+     * @result Reeturns the newly allocated IOSharedDataQueue instance.  Zero is returned on failure.
+     */
+    static IOSharedDataQueue *withEntries(UInt32 numEntries, UInt32 entrySize);
+
+    /*!
+     * @function initWithCapacity
+     * @abstract Initializes an IOSharedDataQueue instance with the capacity specified in the size parameter.
+     * @discussion The actual size of the entire data queue memory region (to be shared into a user process) is equal to the capacity plus the IODataQueueMemory overhead.  This overhead value can be determined from the DATA_QUEUE_MEMORY_HEADER_SIZE and DATA_QUEUE_MEMORY_APPENDIX_SIZE macro in <IOKit/IODataQueueShared.h>.  The size of the data queue memory region must include space for the overhead of each IODataQueueEntry.  This entry overhead can be determined from the DATA_QUEUE_ENTRY_HEADER_SIZE macro in <IOKit/IODataQueueShared.h>.
+     * @param size The size of the data queue memory region.
+     * @result Returns true on success and false on failure.
+     */
+    virtual Boolean initWithCapacity(UInt32 size);
+
+    /*!
+     * @function getMemoryDescriptor
+     * @abstract Returns a memory descriptor covering the IODataQueueMemory region.
+     * @discussion The IOMemoryDescriptor instance returned by this method is intended to be mapped into a user process.  This is the memory region that the IODataQueueClient code operates on.
+     * @result Returns a newly allocated IOMemoryDescriptor for the IODataQueueMemory region.  Returns zero on failure.
+     */
+    virtual IOMemoryDescriptor *getMemoryDescriptor();
+
+    /*!
+     * @function peek
+     * @abstract Used to peek at the next entry on the queue.
+     * @discussion This function can be used to look at the next entry which allows the entry to be received without having to copy it with dequeue.  In order to do this, call peek to get the entry.  Then call dequeue with a NULL data pointer.  That will cause the head to be moved to the next entry, but no memory to be copied.
+     * @result Returns a pointer to the next IODataQueueEntry if one is available.  0 (NULL) is returned if the queue is empty.
+     */
+    virtual IODataQueueEntry * peek();
+
+    /*!
+     * @function dequeue
+     * @abstract Dequeues the next available entry on the queue and copies it into the given data pointer.
+     * @discussion This function will dequeue the next available entry on the queue.  If a data pointer is provided, it will copy the data into the memory region if there is enough space available as specified in the dataSize parameter.  If no data pointer is provided, it will simply move the head value past the current entry.
+     * @param data A pointer to the data memory region in which to copy the next entry data on the queue.  If this parameter is 0 (NULL), it will simply move to the next entry.
+     * @param dataSize A pointer to the size of the data parameter.  On return, this contains the size of the actual entry data - even if the original size was not large enough.
+     * @result Returns true on success and false on failure.  Typically failure means that the queue is empty.
+     */
+    virtual Boolean dequeue(void *data, UInt32 *dataSize);
+
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 0);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 1);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 2);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 3);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 4);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 5);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 6);
+    OSMetaClassDeclareReservedUnused(IOSharedDataQueue, 7);
+};
+
+#endif /* _IOKIT_IOSHAREDDATAQUEUE_H */
diff --git a/iokit/IOKit/IOSharedLock.h b/iokit/IOKit/IOSharedLock.h
index 6d08fc561..eadfc407d 100644
--- a/iokit/IOKit/IOSharedLock.h
+++ b/iokit/IOKit/IOSharedLock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IOSyncer.h b/iokit/IOKit/IOSyncer.h
index 3bb2df5ac..f2d09e267 100644
--- a/iokit/IOKit/IOSyncer.h
+++ b/iokit/IOKit/IOSyncer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOSYNCER_H
 #define _IOSYNCER_H
@@ -26,6 +32,8 @@
 #include <IOKit/IOTypes.h>
 #include <IOKit/IOLocks.h>
 
+#define DEPRECATED __attribute__((deprecated))
+
 class IOSyncer : public OSObject
 {
     OSDeclareDefaultStructors(IOSyncer)
@@ -40,13 +48,18 @@ private:
 
 public:
 
-    static IOSyncer * create(bool twoRetains = true);
+    static IOSyncer * create(bool twoRetains = true)
+	DEPRECATED;
 
-    virtual bool init(bool twoRetains);
-    virtual void reinit();
-    virtual IOReturn wait(bool autoRelease = true);
+    virtual bool init(bool twoRetains)
+	DEPRECATED;
+    virtual void reinit()
+	DEPRECATED;
+    virtual IOReturn wait(bool autoRelease = true)
+	DEPRECATED;
     virtual void signal(IOReturn res = kIOReturnSuccess,
-					bool autoRelease = true);
+					bool autoRelease = true)
+	DEPRECATED;
 };
 
 #endif /* !_IOSYNCER */
diff --git a/iokit/IOKit/IOTimeStamp.h b/iokit/IOKit/IOTimeStamp.h
index 8e01e7e66..b900cdfdd 100644
--- a/iokit/IOKit/IOTimeStamp.h
+++ b/iokit/IOKit/IOTimeStamp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef IOKIT_IOTIMESTAMP_H
 #define IOKIT_IOTIMESTAMP_H
diff --git a/iokit/IOKit/IOTimerEventSource.h b/iokit/IOKit/IOTimerEventSource.h
index 80fb27309..cedfaa40d 100644
--- a/iokit/IOKit/IOTimerEventSource.h
+++ b/iokit/IOKit/IOTimerEventSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/IOTypes.h b/iokit/IOKit/IOTypes.h
index 4564ad3eb..0dd2dc1ce 100644
--- a/iokit/IOKit/IOTypes.h
+++ b/iokit/IOKit/IOTypes.h
@@ -1,31 +1,30 @@
 /*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
- *
- * HISTORY
- *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-
 #ifndef	__IOKIT_IOTYPES_H
 #define __IOKIT_IOTYPES_H
 
@@ -47,7 +46,11 @@ extern "C" {
 #endif
 
 #ifndef	NULL
+#if defined (__cplusplus)
 #define	NULL	0
+#else
+#define NULL ((void *)0)
+#endif
 #endif
 	
 /*
@@ -62,6 +65,10 @@ extern "C" {
 #endif /* __TYPES__ */
 #endif /* __MACTYPES__ */
 
+#if KERNEL
+#include <libkern/OSBase.h>
+#endif
+
 typedef UInt32		IOOptionBits;
 typedef SInt32		IOFixed;
 typedef UInt32		IOVersion;
@@ -90,6 +97,7 @@ typedef UInt32	IOPhysicalLength;
 
 #endif
 
+
 #if __cplusplus
 struct IOVirtualRange
 {
@@ -108,11 +116,11 @@ typedef struct
     IOByteCount		length;
 } IOVirtualRange;
 
-struct IOAddressRange
+typedef struct 
 {
     mach_vm_address_t	address;
     mach_vm_size_t	length;
-};
+} IOAddressRange;
 #endif
 
 /*
@@ -192,7 +200,8 @@ enum {
 
     kIOMapStatic		= 0x01000000,
     kIOMapReference		= 0x02000000,
-    kIOMapUnique		= 0x04000000
+    kIOMapUnique		= 0x04000000,
+    kIOMap64Bit			= 0x08000000
 };
 
 /*! @enum Scale Factors
diff --git a/iokit/IOKit/IOUserClient.h b/iokit/IOKit/IOUserClient.h
index cbf9d4969..1a2782e50 100644
--- a/iokit/IOKit/IOUserClient.h
+++ b/iokit/IOKit/IOUserClient.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -40,6 +46,15 @@ enum {
     kIOUCScalarIStructI = 4
 };
 
+/*! @enum
+    @abstract Constant to denote a variable length structure argument to IOUserClient.
+    @constant kIOUCVariableStructureSize Use in the structures IOExternalMethod, IOExternalAsyncMethod, IOExternalMethodDispatch to specify the size of the structure is variable.
+*/
+enum {
+    kIOUCVariableStructureSize = 0xffffffff
+};
+
+
 typedef IOReturn (IOService::*IOMethod)(void * p1, void * p2, void * p3,
                                         void * p4, void * p5, void * p6 );
 
@@ -79,6 +94,64 @@ enum {
 #define kIOClientPrivilegeAdministrator	"root"
 #define kIOClientPrivilegeLocalUser	"local"
 
+/*! @enum
+    @abstract Constants to specify the maximum number of scalar arguments in the IOExternalMethodArguments structure. These constants are documentary since the scalarInputCount, scalarOutputCount fields reflect the actual number passed.
+    @constant kIOExternalMethodScalarInputCountMax The maximum number of scalars able to passed on input.
+    @constant kIOExternalMethodScalarOutputCountMax The maximum number of scalars able to passed on output.
+*/
+enum {
+    kIOExternalMethodScalarInputCountMax  = 16,
+    kIOExternalMethodScalarOutputCountMax = 16,
+};
+
+
+struct IOExternalMethodArguments
+{
+    uint32_t		version;
+
+    uint32_t		selector;
+
+    mach_port_t           asyncWakePort;
+    io_user_reference_t * asyncReference;
+    uint32_t              asyncReferenceCount;
+
+    const uint64_t *    scalarInput;
+    uint32_t		scalarInputCount;
+
+    const void *	structureInput;
+    uint32_t		structureInputSize;
+
+    IOMemoryDescriptor * structureInputDescriptor;
+   
+    uint64_t *		scalarOutput;
+    uint32_t		scalarOutputCount;
+
+    void *		structureOutput;
+    uint32_t		structureOutputSize;
+
+    IOMemoryDescriptor * structureOutputDescriptor;
+    uint32_t		 structureOutputDescriptorSize;
+
+    uint32_t		__reserved[32];
+};
+
+typedef IOReturn (*IOExternalMethodAction)(OSObject * target, void * reference, 
+					    IOExternalMethodArguments * arguments);
+struct IOExternalMethodDispatch
+{
+    IOExternalMethodAction function;
+    uint32_t		   checkScalarInputCount;
+    uint32_t		   checkStructureInputSize;
+    uint32_t		   checkScalarOutputCount;
+    uint32_t		   checkStructureOutputSize;
+};
+
+enum {
+#define IO_EXTERNAL_METHOD_ARGUMENTS_CURRENT_VERSION	1
+    kIOExternalMethodArgumentsCurrentVersion = IO_EXTERNAL_METHOD_ARGUMENTS_CURRENT_VERSION
+};
+
+
 /*!
     @class IOUserClient
     @abstract   Provides a basis for communication between client applications and I/O Kit objects.
@@ -103,12 +176,15 @@ protected:
 public:
     OSSet * mappings;
     UInt8   sharedInstance;
-
     UInt8   __reservedA[3];
     void  * __reserved[7];
 
+    virtual IOReturn externalMethod( uint32_t selector, IOExternalMethodArguments * arguments,
+					IOExternalMethodDispatch * dispatch = 0, OSObject * target = 0, void * reference = 0 );
+    OSMetaClassDeclareReservedUsed(IOUserClient, 0);
+
 private:
-    OSMetaClassDeclareReservedUnused(IOUserClient, 0);
+
     OSMetaClassDeclareReservedUnused(IOUserClient, 1);
     OSMetaClassDeclareReservedUnused(IOUserClient, 2);
     OSMetaClassDeclareReservedUnused(IOUserClient, 3);
@@ -131,6 +207,12 @@ protected:
     static void setAsyncReference(OSAsyncReference asyncRef,
                                   mach_port_t wakePort,
                                   void *callback, void *refcon);
+
+    static IOReturn sendAsyncResult64(OSAsyncReference64 reference,
+                                        IOReturn result, io_user_reference_t args[], UInt32 numArgs);
+    static void setAsyncReference64(OSAsyncReference64 asyncRef,
+					mach_port_t wakePort,
+					mach_vm_address_t callback, io_user_reference_t refcon);
 public:
 
     static void initialize( void );
@@ -140,10 +222,8 @@ public:
     static IOReturn clientHasPrivilege( void * securityToken,
                                         const char * privilegeName );
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
     virtual bool init();
     virtual bool init( OSDictionary * dictionary );
-#endif
     // Currently ignores the all args, just passes up to IOService::init()
     virtual bool initWithTask(
                     task_t owningTask, void * securityToken, UInt32 type,
@@ -177,6 +257,11 @@ public:
                                     IOOptionBits mapFlags = kIOMapAnywhere,
 				    IOVirtualAddress atAddress = 0 );
 
+    IOMemoryMap * mapClientMemory64( IOOptionBits type,
+                                    task_t task,
+                                    IOOptionBits mapFlags = kIOMapAnywhere,
+				    mach_vm_address_t atAddress = 0 );
+
     /*!
         @function removeMappingForDescriptor
         Remove the first mapping created from the memory descriptor returned by clientMemoryForType() from IOUserClient's list of mappings. If such a mapping exists, it is retained and the reference currently held by IOUserClient is returned to the caller.
diff --git a/iokit/IOKit/IOWorkLoop.h b/iokit/IOKit/IOWorkLoop.h
index 0fa9251aa..c278c9537 100644
--- a/iokit/IOKit/IOWorkLoop.h
+++ b/iokit/IOKit/IOWorkLoop.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.	 All rights reserved.
@@ -72,6 +78,9 @@ member function's parameter list.
     typedef IOReturn (*Action)(OSObject *target,
 			       void *arg0, void *arg1,
 			       void *arg2, void *arg3);
+    enum {
+	kPreciousStack = 0x00000001
+    };
 
 private:
 /*! @function threadMainContinuation
@@ -90,6 +99,7 @@ protected:
 
 /*! @var gateLock
     Mutual exclusion lock that is used by close and open Gate functions.  
+    This is a recursive lock, which allows multiple layers of code to share a single IOWorkLoop without deadlock.  This is common in IOKit since threads of execution tend to follow the service plane in the IORegistry, and multiple objects along the call path may acquire the gate for the same (shared) workloop.
 */
     IORecursiveLock *gateLock;
 
@@ -103,7 +113,7 @@ protected:
 */
     IOCommandGate *controlG;
 
-/*! @var workSpinLock
+/*! @var workToDoLock
     The spin lock that is used to guard the 'workToDo' variable. 
 */
     IOSimpleLock *workToDoLock;
@@ -126,7 +136,9 @@ protected:
 /*! @struct ExpansionData
     @discussion This structure will be used to expand the capablilties of the IOWorkLoop in the future.
 */    
-    struct ExpansionData { };
+    struct ExpansionData {
+	IOOptionBits options;
+    };
 
 /*! @var reserved
     Reserved for future use.  (Internal use only) 
@@ -161,13 +173,20 @@ protected:
 public:
 
 /*! @function workLoop
-    @abstract Factory member function to constuct and intialize a work loop.
+    @abstract Factory member function to construct and intialize a work loop.
     @result Returns a workLoop instance if constructed successfully, 0 otherwise. 
 */
     static IOWorkLoop *workLoop();
 
+/*! @function workLoopWithOptions(IOOptionBits options)
+    @abstract Factory member function to constuct and intialize a work loop.
+    @param options Options - kPreciousStack to avoid stack deallocation on paging path.
+    @result Returns a workLoop instance if constructed successfully, 0 otherwise. 
+*/
+    static IOWorkLoop *workLoopWithOptions(IOOptionBits options);
+
 /*! @function init
-    @discussion Initializes an instance of the workloop.  This method creates and initialses the signaling semaphore and forks the thread that will continue executing.
+    @discussion Initializes an instance of the workloop.  This method creates and initializes the signaling semaphore, the controller gate lock, and spawns the thread that will continue executing.
     @result Returns true if initialized successfully, false otherwise. 
 */
     virtual bool init();
@@ -258,7 +277,6 @@ public:
 			       void *arg0 = 0, void *arg1 = 0,
 			       void *arg2 = 0, void *arg3 = 0);
 
-#if !(defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
 /*! @function runEventSources
     @discussion Consists of the inner 2 loops of the threadMain function(qv).
     The outer loop terminates when there is no more work, and the inside loop
@@ -280,13 +298,9 @@ public:
 */
     OSMetaClassDeclareReservedUsed(IOWorkLoop, 1);
     virtual bool runEventSources();
-#endif
 
 protected:
 
-#if (defined(__ppc__) && defined(KPI_10_4_0_PPC_COMPAT))
-    OSMetaClassDeclareReservedUnused(IOWorkLoop, 1);
-#endif
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 2);
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 3);
     OSMetaClassDeclareReservedUnused(IOWorkLoop, 4);
diff --git a/iokit/IOKit/Makefile b/iokit/IOKit/Makefile
index 963483625..90246e45d 100644
--- a/iokit/IOKit/Makefile
+++ b/iokit/IOKit/Makefile
@@ -24,13 +24,20 @@ INSTINC_SUBDIRS_PPC = \
 INSTINC_SUBDIRS_I386 = \
 	i386
 
+INSTINC_SUBDIRS_ARM = \
+	arm
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 NOT_EXPORT_HEADERS = 
 
-NOT_KF_MI_HEADERS  = $(NOT_EXPORT_HEADERS) IOKitKeysPrivate.h IOCPU.h IOHibernatePrivate.h IOPolledInterface.h
+NOT_KF_MI_HEADERS  = $(NOT_EXPORT_HEADERS)			\
+		     IOKitKeysPrivate.h IOCPU.h			\
+		     IOHibernatePrivate.h IOPolledInterface.h	\
+		     IOCommandQueue.h IOLocksPrivate.h
 
 NOT_LOCAL_HEADERS = 
 
@@ -40,7 +47,7 @@ INSTALL_MI_LIST	= IOBSD.h IOKitKeys.h IOKitServer.h IOReturn.h\
 		  IOSharedLock.h IOTypes.h OSMessageNotification.h\
 		  IODataQueueShared.h IOMessage.h
                   
-INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h
+INSTALL_MI_LCL_LIST = IOKitKeysPrivate.h IOHibernatePrivate.h IOLocksPrivate.h
 
 INSTALL_MI_DIR = .
 
diff --git a/iokit/IOKit/OSMessageNotification.h b/iokit/IOKit/OSMessageNotification.h
index 75f3f95ec..1c62d7b0c 100644
--- a/iokit/IOKit/OSMessageNotification.h
+++ b/iokit/IOKit/OSMessageNotification.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -34,6 +40,7 @@ extern "C" {
 #endif
 
 #include <mach/mach_types.h>
+#include <device/device_types.h>
 #include <IOKit/IOReturn.h>
 
 enum {
@@ -70,6 +77,35 @@ enum {
     kIOInterestCalloutCount
 };
 
+
+
+// --------------
+enum {
+    kOSAsyncRef64Count	= 8,
+    kOSAsyncRef64Size 	= kOSAsyncRef64Count * sizeof(io_user_reference_t)
+};
+typedef io_user_reference_t OSAsyncReference64[kOSAsyncRef64Count];
+
+struct OSNotificationHeader64 {
+    mach_msg_size_t	size;		/* content size */
+    natural_t		type;
+    OSAsyncReference64	reference;
+
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+    unsigned char	content[];
+#else
+    unsigned char	content[0];
+#endif
+};
+
+struct IOServiceInterestContent64 {
+    natural_t		messageType;
+    io_user_reference_t	messageArgument[1];
+};
+// --------------
+
+#if !KERNEL_USER32
+
 enum {
     kOSAsyncRefCount	= 8,
     kOSAsyncRefSize 	= 32
@@ -77,7 +113,7 @@ enum {
 typedef natural_t OSAsyncReference[kOSAsyncRefCount];
 
 struct OSNotificationHeader {
-    vm_size_t		size;		/* content size */
+    mach_msg_size_t	size;		/* content size */
     natural_t		type;
     OSAsyncReference	reference;
 
@@ -88,10 +124,14 @@ struct OSNotificationHeader {
 #endif
 };
 
+#pragma pack(4)
 struct IOServiceInterestContent {
     natural_t	messageType;
     void *	messageArgument[1];
 };
+#pragma pack()
+
+#endif /* KERNEL_USER32  */
 
 struct IOAsyncCompletionContent {
     IOReturn result;
diff --git a/iokit/IOKit/assert.h b/iokit/IOKit/assert.h
index 70c7361d3..f7212675f 100644
--- a/iokit/IOKit/assert.h
+++ b/iokit/IOKit/assert.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_IO_ASSERT_H_
diff --git a/iokit/IOKit/i386/IOSharedLockImp.h b/iokit/IOKit/i386/IOSharedLockImp.h
index 421f8d9eb..ee2bd16d3 100644
--- a/iokit/IOKit/i386/IOSharedLockImp.h
+++ b/iokit/IOKit/i386/IOSharedLockImp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/machine/IOSharedLockImp.h b/iokit/IOKit/machine/IOSharedLockImp.h
index 543a83ed4..7746639ba 100644
--- a/iokit/IOKit/machine/IOSharedLockImp.h
+++ b/iokit/IOKit/machine/IOSharedLockImp.h
@@ -1,29 +1,37 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #if defined (__ppc__)
 #include "IOKit/ppc/IOSharedLockImp.h"
 #elif defined (__i386__)
 #include "IOKit/i386/IOSharedLockImp.h"
+#elif defined (__arm__)
+#include "IOKit/arm/IOSharedLockImp.h"
 #else
 #error architecture not supported
 #endif
diff --git a/iokit/IOKit/nvram/IONVRAMController.h b/iokit/IOKit/nvram/IONVRAMController.h
index 691d4a1d3..307d666a6 100644
--- a/iokit/IOKit/nvram/IONVRAMController.h
+++ b/iokit/IOKit/nvram/IONVRAMController.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IONVRAMCONTROLLER_H
diff --git a/iokit/IOKit/nvram/Makefile b/iokit/IOKit/nvram/Makefile
index 9ae63c0b5..3a5bd5cda 100644
--- a/iokit/IOKit/nvram/Makefile
+++ b/iokit/IOKit/nvram/Makefile
@@ -16,10 +16,12 @@ NOT_EXPORT_HEADERS =
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
diff --git a/iokit/IOKit/pci/IOPCIDevice.h b/iokit/IOKit/pci/IOPCIDevice.h
index d39d45c03..92069bbe7 100644
--- a/iokit/IOKit/pci/IOPCIDevice.h
+++ b/iokit/IOKit/pci/IOPCIDevice.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOKIT_IOPCIDEVICE_H
@@ -55,6 +61,9 @@ union IOPCIAddressSpace {
     } s;
 };
 
+class IOPCIBridge;
+class IOPCI2PCIBridge;
+
 class IOPCIDevice : public IOService
 {
     OSDeclareDefaultStructors(IOPCIDevice)
diff --git a/iokit/IOKit/platform/AppleMacIO.h b/iokit/IOKit/platform/AppleMacIO.h
index 553246eab..ae12eca13 100644
--- a/iokit/IOKit/platform/AppleMacIO.h
+++ b/iokit/IOKit/platform/AppleMacIO.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/platform/AppleMacIODevice.h b/iokit/IOKit/platform/AppleMacIODevice.h
index 51f9abd39..8d033fd25 100644
--- a/iokit/IOKit/platform/AppleMacIODevice.h
+++ b/iokit/IOKit/platform/AppleMacIODevice.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/platform/AppleNMI.h b/iokit/IOKit/platform/AppleNMI.h
index 713259f6a..14cd1ed10 100644
--- a/iokit/IOKit/platform/AppleNMI.h
+++ b/iokit/IOKit/platform/AppleNMI.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-9 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/IOKit/platform/ApplePlatformExpert.h b/iokit/IOKit/platform/ApplePlatformExpert.h
index a684a5b73..59728d23d 100644
--- a/iokit/IOKit/platform/ApplePlatformExpert.h
+++ b/iokit/IOKit/platform/ApplePlatformExpert.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/platform/Makefile b/iokit/IOKit/platform/Makefile
index 5399c0501..17669c894 100644
--- a/iokit/IOKit/platform/Makefile
+++ b/iokit/IOKit/platform/Makefile
@@ -12,23 +12,33 @@ include $(MakeInc_def)
 
 MI_DIR = platform
 NOT_EXPORT_HEADERS = 
+NOT_KF_MI_HEADERS  = AppleARMCPU.h AppleARMFunction.h AppleARMIICController.h \
+			AppleARMIICDevice.h AppleARMIISController.h \
+			AppleARMIISDevice.h AppleARMIO.h AppleARMIODevice.h \
+			AppleARMNORFlashController.h AppleARMNORFlashDevice.h \
+			AppleARMPE.h AppleARMRTC.h AppleARMSPIController.h \
+			AppleARMSPIDevice.h
 
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
 INSTALL_MI_LIST	= 
-INSTALL_MI_LCL_LIST = ""
+INSTALL_MI_LCL_LIST = 
 INSTALL_MI_DIR = $(MI_DIR)
 
 EXPORT_MI_LIST	= $(filter-out $(NOT_EXPORT_HEADERS), $(ALL_HEADERS))
 EXPORT_MI_DIR = IOKit/$(MI_DIR)
 
+INSTALL_KF_MI_LIST = $(filter-out $(NOT_KF_MI_HEADERS), $(ALL_HEADERS))
+
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/iokit/IOKit/power/IOPwrController.h b/iokit/IOKit/power/IOPwrController.h
index 82eb90e1e..709a0fb1c 100644
--- a/iokit/IOKit/power/IOPwrController.h
+++ b/iokit/IOKit/power/IOPwrController.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * 24 Nov  1998 suurballe  Created.
diff --git a/iokit/IOKit/power/Makefile b/iokit/IOKit/power/Makefile
index f7ee70086..6a9beac4b 100644
--- a/iokit/IOKit/power/Makefile
+++ b/iokit/IOKit/power/Makefile
@@ -16,10 +16,12 @@ NOT_EXPORT_HEADERS =
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
diff --git a/iokit/IOKit/ppc/IODBDMA.h b/iokit/IOKit/ppc/IODBDMA.h
index 956f35ccf..afe1337bb 100644
--- a/iokit/IOKit/ppc/IODBDMA.h
+++ b/iokit/IOKit/ppc/IODBDMA.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 Apple Computer, Inc.
diff --git a/iokit/IOKit/ppc/IOSharedLockImp.h b/iokit/IOKit/ppc/IOSharedLockImp.h
index 3644a328c..8c685b223 100644
--- a/iokit/IOKit/ppc/IOSharedLockImp.h
+++ b/iokit/IOKit/ppc/IOSharedLockImp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/IOKit/pwr_mgt/IOPM.h b/iokit/IOKit/pwr_mgt/IOPM.h
index 90b8e19c3..362c5ed31 100644
--- a/iokit/IOKit/pwr_mgt/IOPM.h
+++ b/iokit/IOKit/pwr_mgt/IOPM.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOKIT_IOPM_H
 #define _IOKIT_IOPM_H
@@ -30,63 +36,115 @@
 #include <IOKit/pwr_mgt/IOPMDeprecated.h>
 #endif
 
+/*! @header IOPM.h
+    @abstract Defines power management constants and keys used by both in-kernel and user space power management.
+    @discussion IOPM.h defines a range of power management constants used in several in-kernel and user space APIs. Most significantly, the IOPMPowerFlags used to specify the fields of an IOPMPowerState struct are defined here.
 
+    Most of the constants defined in IOPM.h are deprecated or for Apple internal use only, and are not elaborated on in headerdoc.
+*/
 
 enum {
     kIOPMMaxPowerStates = 10,
     IOPMMaxPowerStates = kIOPMMaxPowerStates
 };
 
+/*! @enum IOPMPowerFlags
+    @abstract Bits are used in defining capabilityFlags, inputPowerRequirements, and outputPowerCharacter in the IOPMPowerState structure.
+    @discussion These bits may be bitwise-OR'd together in the IOPMPowerState capabilityFlags field, the outputPowerCharacter field, and/or the inputPowerRequirement field. 
+    
+    The comments clearly mark whether each flag should be used in the capabilityFlags field, outputPowerCharacter field, and inputPowerRequirement field, or all three.
+    
+    The value of capabilityFlags, inputPowerRequirement or outputPowerCharacter may be 0. Most drivers implement their 'OFF' state, used when asleep, by defininf each of the 3 fields as 0.
+    
+    The bits listed below are only the most common bits used to define a device's power states. Your device's IO family may require that your device specify other input or output power flags to interact properly. Consult family-specific documentation to determine if your IOPower plane parents or children require other power flags; they probably don't.
+
+    @constant kIOPMPowerOn Indicates the device is on, requires power, and provides power. Useful as a: Capability, InputPowerRequirement, OutputPowerCharacter
+
+    @constant kIOPMDeviceUsable Indicates the device is usable in this state. Useful only as a Capability
+
+    @constant kIOPMLowPower 
+    Indicates device is in a low power state. May be bitwis-OR'd together
+    with kIOPMDeviceUsable flag, to indicate the device is still usable.
+    
+    A device with a capability of kIOPMLowPower may:
+       Require either 0 or kIOPMPowerOn from its power parent
+       Offer either kIOPMLowPower, kIOPMPowerOn, or 0 (no power at all)
+         to its power plane children.
+
+    Useful only as a Capability, although USB drivers should consult USB family documentation for other valid circumstances to use the kIOPMLowPower bit.
+
+    @constant kIOPMPreventIdleSleep
+    In the capability field of a power state, disallows idle system sleep while the device is in that state.
+    
+    For example, displays and disks set this capability for their ON power state; since the system may not idle sleep while the display (and thus keyboard or mouse) or the disk is active.
+
+    Useful only as a Capability.
+    
+    @constant kIOPMSleepCapability 
+    Used only by certain IOKit Families (USB). Not defined or used by generic Power Management. Read your family documentation to see if you should define a powerstate using these capabilities.
+    
+    @constant kIOPMRestartCapability
+    Used only by certain IOKit Families (USB). Not defined or used by generic Power Management. Read your family documentation to see if you should define a powerstate using these capabilities.
+    
+    @constant kIOPMSleep
+    Used only by certain IOKit Families (USB). Not defined or used by generic Power Management. Read your family documentation to see if you should define a powerstate using these capabilities.
+
+    @constant kIOPMRestart
+    Used only by certain IOKit Families (USB). Not defined or used by generic Power Management. Read your family documentation to see if you should define a powerstate using these capabilities.
+*/
 typedef unsigned long IOPMPowerFlags;
 enum {
-    // The following  bits are used in the input and output power fields.
+    kIOPMPowerOn                    = 0x00000002,
+    kIOPMDeviceUsable               = 0x00008000,
+    kIOPMLowPower                   = 0x00010000,
+    kIOPMPreventIdleSleep           = 0x00000040,
+    kIOPMSleepCapability            = 0x00000004,
+    kIOPMRestartCapability          = 0x00000080,
+    kIOPMSleep                      = 0x00000001,
+    kIOPMRestart                    = 0x00000080
+};
+
+/*
+ * Private IOPMPowerFlags
+ *
+ * For Apple use only
+ * Not for use with non-Apple drivers
+ * Their behavior is undefined
+ */
+enum {
     kIOPMClockNormal                = 0x0004,
     kIOPMClockRunning               = 0x0008,
-    // Reserved - Used only between root and root parent.
-    kIOPMAuxPowerOn                 = 0x0020,
-    // Reserved - kIOPMPagingAvailable used only by now-defunct paging plexus
-    kIOPMPagingAvailable            = 0x0020,
-    kIOPMPassThrough                = 0x0100,
-    kIOPMDoze                       = 0x0400,
-    // Obsolete - use kIOPMDoze instead of kIOPMSoftSleep
-    kIOPMSoftSleep                  = 0x0400,
-    kIOPMSleep                      = 0x0001,
-    kIOPMRestart                    = 0x0080,
-
-    // The following bits are used in the capabilites field and the power fields
-    kIOPMPowerOn                    = 0x0002,
     kIOPMPreventSystemSleep         = 0x0010,
-    kIOPMPreventIdleSleep           = 0x0040,
-
-    // The following  bits are used in the capabilites field only.
-    // Used between a driver and its policy-maker
-    kIOPMNotAttainable              = 0x0001,
-    // Used internally in a power domain parent
+    kIOPMDoze                       = 0x0400,
     kIOPMChildClamp                 = 0x0080,
-    // Used internally in a power domain parent
     kIOPMChildClamp2                = 0x0200,
-    // Marks device as usable in this state
-    kIOPMDeviceUsable               = 0x8000,
-    // Device runs at max performance in this state
+    kIOPMNotPowerManaged            = 0x0800
+};
+
+
+/*
+ * Deprecated IOPMPowerFlags
+ * Their behavior is undefined when used in IOPMPowerState
+ * Capability, InputPowerRequirement, or OutputPowerCharacter fields.
+ */
+enum {
     kIOPMMaxPerformance             = 0x4000,
+    kIOPMPassThrough                = 0x0100,
+    kIOPMAuxPowerOn                 = 0x0020,
+    kIOPMNotAttainable              = 0x0001,
     kIOPMContextRetained            = 0x2000,
     kIOPMConfigRetained             = 0x1000,
-    // Device is capable of system sleep in this state
-    kIOPMSleepCapability            = 0x0004,
-    kIOPMRestartCapability          = 0x0080,
-
-    // Reserved - Error code. (this is an error return rather than a bit)
-    kIOPMNotPowerManaged            = 0x0800,
-    // Therefore this bit safely overloads it
     kIOPMStaticPowerValid           = 0x0800,
-    
+    kIOPMSoftSleep                  = 0x0400,
     kIOPMCapabilitiesMask =     kIOPMPowerOn | kIOPMDeviceUsable | 
                                 kIOPMMaxPerformance | kIOPMContextRetained | 
                                 kIOPMConfigRetained | kIOPMSleepCapability |
                                 kIOPMRestartCapability
 };
 
-
+/*
+ * Support for old names of IOPMPowerFlag constants
+ */
 enum {
     IOPMNotAttainable           = kIOPMNotAttainable,
     IOPMPowerOn                 = kIOPMPowerOn,
@@ -98,7 +156,6 @@ enum {
     IOPMContextRetained         = kIOPMContextRetained,
     IOPMConfigRetained          = kIOPMConfigRetained,
     IOPMNotPowerManaged         = kIOPMNotPowerManaged,
-    IOPMPagingAvailable         = kIOPMPagingAvailable,
     IOPMSoftSleep               = kIOPMSoftSleep
 };
 
@@ -290,6 +347,7 @@ enum {
 #define kIOPMPSLegacyBatteryInfoKey                 "LegacyBatteryInfo"
 #define kIOPMPSBatteryHealthKey                     "BatteryHealth"
 #define kIOPMPSHealthConfidenceKey                  "HealthConfidence"
+#define kIOPMPSCapacityEstimatedKey		    "CapacityEstimated"
 
 // Definitions for battery location, in case of multiple batteries.
 // A location of 0 is unspecified
diff --git a/iokit/IOKit/pwr_mgt/IOPMDeprecated.h b/iokit/IOKit/pwr_mgt/IOPMDeprecated.h
index 67932d8a9..3bee01a3b 100644
--- a/iokit/IOKit/pwr_mgt/IOPMDeprecated.h
+++ b/iokit/IOKit/pwr_mgt/IOPMDeprecated.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOPMDeprecated_h_
diff --git a/iokit/IOKit/pwr_mgt/IOPMLibDefs.h b/iokit/IOKit/pwr_mgt/IOPMLibDefs.h
index a1546528c..10da68d8e 100644
--- a/iokit/IOKit/pwr_mgt/IOPMLibDefs.h
+++ b/iokit/IOKit/pwr_mgt/IOPMLibDefs.h
@@ -1,32 +1,39 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
-#define	kPMSetAggressiveness		0
-#define	kPMGetAggressiveness		1
-#define	kPMSleepSystem			2
-#define	kPMAllowPowerChange		3
-#define	kPMCancelPowerChange		4
-#define	kPMShutdownSystem		5
-#define	kPMRestartSystem		6
+#define	kPMSetAggressiveness        0
+#define	kPMGetAggressiveness        1
+#define	kPMSleepSystem              2
+#define	kPMAllowPowerChange         3
+#define	kPMCancelPowerChange        4
+#define	kPMShutdownSystem           5
+#define	kPMRestartSystem            6
+#define kPMSleepSystemOptions       7
 
-#define kNumPMMethods 7
+#define kNumPMMethods               8
diff --git a/iokit/IOKit/pwr_mgt/IOPMPagingPlexus.h b/iokit/IOKit/pwr_mgt/IOPMPagingPlexus.h
index 0840c4c3a..74fb00731 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPagingPlexus.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPagingPlexus.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -32,13 +38,10 @@
 
 #include <IOKit/IOService.h>
 
+/* Deprecated in Mac OS X version 10.0
 
-
-/*! @class IOPMPagingPlexus : public IOService
-    @abstract 
-    @discussion  
-*/
-
+   Under no circumstances should any new software use or reference IOPMPagingPlexus.
+ */
 class IOPMPagingPlexus : public IOService
 {
     OSDeclareDefaultStructors(IOPMPagingPlexus)
@@ -62,5 +65,4 @@ protected:
 
 };
 
-#endif /* ! _IOKIT_IOPMPAGINGPLEXUS_H */
-
+#endif
diff --git a/iokit/IOKit/pwr_mgt/IOPMPowerSource.h b/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
index b8e4889e6..ed6891819 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPowerSource.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #ifndef _IOPMPowerSource_h_
@@ -131,6 +137,15 @@ enum {
  * IORegistry Key: kIOPMPSManufacturerKey
  * String describing battery manufacturer
  *
+ * Manufactured Date
+ * Type: unsigned 16-bit bitfield
+ * IORegistry Key: kIOPMPSManufactureDateKey
+ * Date is published in a bitfield per the Smart Battery Data spec rev 1.1 
+ * in section 5.1.26
+ *   Bits 0...4 => day (value 1-31; 5 bits)
+ *   Bits 5...8 => month (value 1-12; 4 bits)
+ *   Bits 9...15 => years since 1980 (value 0-127; 7 bits)
+ *
  * Model
  * Type: OSSymbol
  * IORegistry Key: kIOPMPSModelKey
@@ -139,7 +154,10 @@ enum {
  * Serial
  * Type: OSSymbol
  * IORegistry Key: kIOPMPSSerialKey
- * String describing serial number  or unique info
+ * String describing serial number or unique info
+ * The serial number published hear bears no correspondence to the Apple serial
+ * number printed on each battery. This is a manufacturer serial number with 
+ * no correlation to the printed serial number.
  *
  * LegacyIOBatteryInfo
  * Type: OSDictionary
@@ -154,9 +172,17 @@ class IOPMPowerSource : public IOService
     friend class IOPMPowerSourceList;
 
  protected:
-    // Tracking for IOPMPowerSourceList
-    IOPMPowerSource         *nextInList;
+
+/* bool settingsChangedSinceLastUpdate
+ * Used by subclasses to determine if any settings have been modified via the
+ * accessors below since last call to update(). true is settings have changed;
+ * false otherwise.
+ */
+    bool settingsChangedSinceUpdate;
     
+/* OSDictionary properties
+ * Stores power source state
+ */
     OSDictionary            *properties;
 
     const OSSymbol *externalConnectedKey;
@@ -179,6 +205,9 @@ class IOPMPowerSource : public IOService
     const OSSymbol *serialKey;
     const OSSymbol *batteryInfoKey;
 
+    // Tracking for IOPMPowerSourceList
+    IOPMPowerSource         *nextInList;
+
  public:
 
 /*! @function powerSource
@@ -227,7 +256,10 @@ class IOPMPowerSource : public IOService
     OSSymbol *serial(void);
     OSDictionary *legacyIOBatteryInfo(void);
     
+    OSObject *getPSProperty(const OSSymbol *);
+    
 protected:
+
 /* Protected "setter" methods for subclasses
  * Subclasses should use these setters to modify all battery properties.
  * 
@@ -256,7 +288,13 @@ protected:
     void setModel(OSSymbol *);
     void setSerial(OSSymbol *);
     void setLegacyIOBatteryInfo(OSDictionary *);
-    
+
+/* All of these methods funnel through the generic accessor method
+   setPSProperty. Caller can pass in any arbitrary OSSymbol key, and
+   that value will be stored in the PM settings dictionary, and relayed
+   onto the IORegistry at update time.
+ */
+    void setPSProperty(const OSSymbol *, OSObject *);
 };
 
 #endif
diff --git a/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h b/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
index eda4983d0..cb1c8ea37 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPowerSourceList.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/c++/OSObject.h>
 #include <IOKit/IOReturn.h>
diff --git a/iokit/IOKit/pwr_mgt/IOPMPrivate.h b/iokit/IOKit/pwr_mgt/IOPMPrivate.h
index 3c35bd5d0..8161a3b99 100644
--- a/iokit/IOKit/pwr_mgt/IOPMPrivate.h
+++ b/iokit/IOKit/pwr_mgt/IOPMPrivate.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOKIT_IOPMPRIVATE_H
 #define _IOKIT_IOPMPRIVATE_H
diff --git a/iokit/IOKit/pwr_mgt/IOPMchangeNoteList.h b/iokit/IOKit/pwr_mgt/IOPMchangeNoteList.h
deleted file mode 100644
index ce8b9090e..000000000
--- a/iokit/IOKit/pwr_mgt/IOPMchangeNoteList.h
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <libkern/c++/OSObject.h>
-#include <IOKit/IOReturn.h>
-
-class IOPowerConnection;
-
-// This is a list of State Changes which are in progress.  Its purpose is to keep track of the
-// notifications and acknowledgements caused by state change.  A change is added to the list
-// when either our parent notifies us that the power domain is changing state or when we decide
-// to change power to our device or domain.
-//
-// A change is removed from the list when all interested parties have been informed of the upcoming
-// change, all of them have acknowledged the notification, the change has been made, all interested
-// parties have been informed that the change was made, and all of them have acknowledged.
-//
-// The list is strictly first-in, first-out.  It is implemented as a circular list in a linear
-// array.  There are two pointers into the array.  The circular list is empty when these two
-// pointers are equal.
-
-// More specifically, a change note is put into the array when one of these things happens:
-//   the device decides it is idle and needs to reduce power. (changePowerStateTo)
-//   the device decides it is not idle and needs to increase power. (changePowerStateTo)
-//   the controlling driver requests a state change. (changePowerStateTo)
-//   some client needs to use the device but it is powered down. (makeUsable)
-//   the parent says the domain power is changing. (powerStateWillChangeTo)
-//   a child says it no longer needs power, and all other children are similarly idle. (requestDomainState)
-//.  a child wants more power in the domain so it can raise its power state. (requestDomainState)
-//
-// A change note is removed from the array when all interested drivers and power domain
-// children have acknowledged the change.
-
-// Each change note contains:
-// A flag field which describes the change.
-// Which power state the device will be in after the change.
-// The power flags which describe the result of this change.
-
-struct changeNoteItem{
-unsigned long		flags;
-unsigned long		newStateNumber;
-IOPMPowerFlags		outputPowerCharacter;
-IOPMPowerFlags		inputPowerRequirement;
-IOPMPowerFlags		domainState;
-IOPowerConnection *	parent;
-IOPMPowerFlags		singleParentState;
-IOPMPowerFlags		capabilityFlags;
-};
-
-typedef struct changeNoteItem changeNoteItem;
-
-
-					// flags field
-	
-#define IOPMParentInitiated		1		// this power change initiated by our  parent
-#define IOPMWeInitiated		2		// this power change initiated by this device (not parent)
-#define IOPMNotDone		4		// we couldn't make this change
-#define IOPMNotInUse		8		// this list element not currently in use
-#define IOPMDomainWillChange	16		// parent change started by PowerDomainWillChangeTo
-#define IOPMDomainDidChange	32		// parent change started by PowerDomainDidChangeTo
-
-
-// Length of change note list is maximum 5.  There cannot be two adjacent device-initiated change notes unless
-// one is currently being actioned, because two adjacent in-active device-initiated changes are always collapsed
-// into one, and there cannot be more than two parent-initiated change notes in the queue (from one parent),
-// because the parent does not
-// initiate a change (by calling domainStateWillChange) until everybody has acknowledged the previous one
-// (by calling domainStateDidChange), although if we are the last to send that acknowledgement, the change we
-// are acknowledging will still be in the queue as we acknowledge, and at that point the parent can give us another
-// (by callingdomainStateWillChange).  So we need room for two parent changes, two non-adjacent device changes,
-// one more per additional parent, say two more,
-// and room for one more device change to get into the queue before collapsing it with its neighbor.  In this case, seven
-// entryies suffices.  Or, we need
-// room for two adjacent device changes (one in progress), a parent change, another device change, another parent change,
-// another device change, another parent change, another device change, plus
-// one more device change to get into the queue before collapsing it with its neighbor.  Nine entries in this case. 
-// I'm not sure what irrationallity causes me to code for twenty entries in the queue.
-#define IOPMMaxChangeNotes 20
-
-class IOPMchangeNoteList :public OSObject
-{
-OSDeclareDefaultStructors(IOPMchangeNoteList)
-    
-private:
-    unsigned long		firstInList;		// points to oldest active change note in list
-    unsigned long		firstUnused;		// points just beyond newest change note in list
-
-public:
-
-        changeNoteItem		changeNote[IOPMMaxChangeNotes];
-    
-
-void initialize ( void );
-
-long createChangeNote ( void );
-
-long currentChange ( void );
-
-long latestChange ( void );
-
-IOReturn releaseHeadChangeNote ( void );
-
-IOReturn releaseTailChangeNote ( void );
-
-bool changeNoteInUse ( unsigned long ordinal );
-
-long nextChangeNote ( unsigned long ordinal );
-
-unsigned long increment (unsigned long ordinal );
-
-unsigned long decrement (unsigned long ordinal );
-
-long previousChangeNote (unsigned long ordinal );
-
-bool listEmpty ( void );
-
-};
diff --git a/iokit/IOKit/pwr_mgt/IOPMinformee.h b/iokit/IOKit/pwr_mgt/IOPMinformee.h
index bab21e8a3..a06eb788c 100644
--- a/iokit/IOKit/pwr_mgt/IOPMinformee.h
+++ b/iokit/IOKit/pwr_mgt/IOPMinformee.h
@@ -1,44 +1,54 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+#ifndef _IOKIT_IOPMINFORMEE_H
+#define _IOKIT_IOPMINFORMEE_H
+
 #include <IOKit/IOService.h>
 #include <IOKit/IOReturn.h>
 
 class IOPMinformee : public OSObject
 {
-OSDeclareDefaultStructors(IOPMinformee)
+    OSDeclareDefaultStructors(IOPMinformee)
+    friend class IOPMinformeeList;
     
 public:
+    static IOPMinformee * withObject( IOService * theObject );
 
-// points to interested driver or power domain child
-IOService *	whatObject;
-
-// -1, 0, or positive number decrementing with each tick
- IOReturn	timer;
+    void initialize( IOService * theObject );
 
-// next informee in the list
- IOPMinformee *	nextInList;
+    void free( void );
 
-
-void initialize ( IOService * theObject );
-void free ( void );
+public:
+    IOService *     whatObject;     // interested driver  
+    int32_t         timer;          // -1, 0, or positive number of ticks
+    IOPMinformee *  nextInList;     // linkage pointer
+    AbsoluteTime    startTime;      // start time of last inform
+    bool            active;         // enable flag
 };
 
+#endif /* !_IOKIT_IOPMINFORMEE_H */
diff --git a/iokit/IOKit/pwr_mgt/IOPMinformeeList.h b/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
index 54c3f8f88..8efd4a654 100644
--- a/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
+++ b/iokit/IOKit/pwr_mgt/IOPMinformeeList.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/c++/OSObject.h>
 #include <IOKit/IOLocks.h>
@@ -29,10 +35,13 @@ class IOService;
 class IOPMinformeeList : public OSObject
 {
 OSDeclareDefaultStructors(IOPMinformeeList)
+    friend class IOPMinformee;
 
 private:
-    IOPMinformee       *firstItem;      // pointer to first informee in the list
-    unsigned long       length;         // how many informees are in the list
+    // pointer to first informee in the list
+    IOPMinformee       *firstItem;
+    // how many informees are in the list
+    unsigned long       length;
 
 public:
     void initialize ( void );
@@ -40,6 +49,10 @@ public:
 
     unsigned long numberOfItems ( void );
 
+    IOPMinformee *appendNewInformee( IOService * newObject );
+    
+    // OBSOLETE
+    // do not use addToList(); Use appendNewInformee() instead
     IOReturn addToList ( IOPMinformee *   newInformee );
     IOReturn removeFromList ( IOService * theItem );
     
diff --git a/iokit/IOKit/pwr_mgt/IOPMlog.h b/iokit/IOKit/pwr_mgt/IOPMlog.h
index 1ad911a5a..187638627 100644
--- a/iokit/IOKit/pwr_mgt/IOPMlog.h
+++ b/iokit/IOKit/pwr_mgt/IOPMlog.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 enum PMLogEnum {
     kPMLogSetParent = 1,		//  1	0x05100004
@@ -25,9 +31,9 @@ enum PMLogEnum {
     kPMLogRemoveChild,			//  3	0x0510000c
     kPMLogControllingDriver,		//  4	0x05100010
     kPMLogControllingDriverErr1,	//  5	0x05100014 - bad power state array version
-    kPMLogControllingDriverErr2,	//  6	0x05100018 - too many power states
-    kPMLogControllingDriverErr3,	//  7	0x0510001c - not a real IOPMDriver
-    kPMLogControllingDriverErr4,	//  8	0x05100020 - power state change in progress
+    kPMLogControllingDriverErr2,	//  6	0x05100018 - power states already registered
+    kPMLogControllingDriverErr3,	//  7	0x0510001c
+    kPMLogControllingDriverErr4,	//  8	0x05100020 - power driver is invalid
     kPMLogInterestedDriver,		//  9	0x05100024
     kPMLogAcknowledgeErr1,		// 10	0x05100028 - unknown entity called acknowledgePowerChange
     kPMLogChildAcknowledge,		// 11	0x0510002c
@@ -56,7 +62,7 @@ enum PMLogEnum {
     kPMLogAmendParentChange,		// 34	0x05100088
     kPMLogStartDeviceChange,		// 35	0x0510008c
     kPMLogRequestDenied,		// 36	0x05100090 - parent denied domain state change request
-    kPMLogControllingDriverErr5,	// 37	0x05100094 - zero power states or we already have a driver with more power states
+    kPMLogControllingDriverErr5,	// 37	0x05100094 - too few power states
     kPMLogProgramHardware,		// 38	0x05100098
     kPMLogInformDriverPreChange,	// 39	0x0510009c
     kPMLogInformDriverPostChange,	// 40	0x051000a0
@@ -69,6 +75,9 @@ enum PMLogEnum {
     kPMLogClientCancel,			// 47	0x051000bc
     kPMLogClientNotify,			// 48	0x051000c0 - client sent a notification
     kPMLogAppNotify,			// 49	0x051000c4 - application sent a notification
+    kPMLogSetClockGating,		// 50	0x051000c8 - platform device specific clock control
+    kPMLogSetPowerGating,		// 51	0x051000cc - platform device specific power control
+    kPMLogSetPinGroup,			// 52	0x051000d0 - platform device specific gpio control
     kIOPMlogLastEvent
 };
 
diff --git a/iokit/IOKit/pwr_mgt/IOPMpmChild.h b/iokit/IOKit/pwr_mgt/IOPMpmChild.h
deleted file mode 100644
index 6df79af9b..000000000
--- a/iokit/IOKit/pwr_mgt/IOPMpmChild.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <IOKit/pwr_mgt/IOPMinformee.h>
-
-class IOPMpmChild: public IOPMinformee
-{
-OSDeclareDefaultStructors(IOPMpmChild)
-    
-public:
-unsigned long	desiredDomainState;	// which power domain state the child desires
-
-
-void initialize ( IOService * theObject );
-
-};
-
diff --git a/iokit/IOKit/pwr_mgt/IOPMpowerState.h b/iokit/IOKit/pwr_mgt/IOPMpowerState.h
index ff0c89e97..f7f4c8a60 100644
--- a/iokit/IOKit/pwr_mgt/IOPMpowerState.h
+++ b/iokit/IOKit/pwr_mgt/IOPMpowerState.h
@@ -1,40 +1,68 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/pwr_mgt/IOPM.h>
 
+/*! @header IOPMpowerState.h
+    @abstract Defines the struct IOPMPowerState that power managed drivers should use to describe their power states.
+*/
+
+/*! @struct IOPMPowerState
+    @abstract Describes a device's power state.
+    @discussion To take part in system power management, drivers should define an array of 2 or more power states and pass it to kernel power management through IOService::registerPowerDriver.
+    @field version Defines version number of this struct. Just use the value "1" when defining an IOPMPowerState.
+    @field	capabilityFlags Describes the capability of the device in this state.
+    @field	outputPowerCharacter Describes the power provided in this state.
+    @field	inputPowerRequirement Describes the input power required in this state.
+    @field	staticPower Describes average consumption in milliwatts. Unused; drivers may specify 0.
+    @field	unbudgetedPower Describes additional consumption from separate power supply (milliWatts). Unused; drivers may specify 0.
+    @field	powerToAttain Describes dditional power to attain this state from next lower state (in milliWatts). Unused; drivers may specify 0.
+    @field	timeToAttain Describes time required to enter this state from next lower state (in microseconds). Unused; drivers may specify 0.
+    @field	settleUpTime Describes settle time required after entering this state from next lower state (microseconds). Unused; drivers may specify 0.
+    @field timeToLower Describes time required to enter next lower state from this one (microseconds). Unused; drivers may specify 0.
+    @field	settleDownTime Settle time required after entering next lower state from this state (microseconds). Unused; drivers may specify 0.
+    @field	powerDomainBudget Describes power in milliWatts a domain in this state can deliver to its children. Unused; drivers may specify 0.
+}
+*/
+
 struct IOPMPowerState
 {
- unsigned long	version;		// version number of this struct
-IOPMPowerFlags	capabilityFlags;	// bits that describe (to interested drivers) the capability of the device in this state 
-IOPMPowerFlags	outputPowerCharacter;	// description (to power domain children) of the power provided in this state 
-IOPMPowerFlags	inputPowerRequirement;	// description (to power domain parent) of input power required in this state
-unsigned long	staticPower;	// average consumption in milliwatts
-unsigned long	unbudgetedPower;	// additional consumption from separate power supply (mw)
-unsigned long	powerToAttain;	// additional power to attain this state from next lower state (in mw)
-unsigned long	timeToAttain;	// time required to enter this state from next lower state (in microseconds)
-unsigned long	settleUpTime;	// settle time required after entering this state from next lower state (microseconds)
-unsigned long	timeToLower;	// time required to enter next lower state from this one (in microseconds)
-unsigned long	settleDownTime;	// settle time required after entering next lower state from this state (microseconds)
-unsigned long	powerDomainBudget;	// power in mw a domain in this state can deliver to its children
+    unsigned long	version;
+    IOPMPowerFlags	capabilityFlags;
+    IOPMPowerFlags	outputPowerCharacter;
+    IOPMPowerFlags	inputPowerRequirement;
+    unsigned long	staticPower;
+    unsigned long	unbudgetedPower;
+    unsigned long	powerToAttain;
+    unsigned long	timeToAttain;
+    unsigned long	settleUpTime;
+    unsigned long	timeToLower;
+    unsigned long	settleDownTime;
+    unsigned long	powerDomainBudget;
 };
 
 typedef struct IOPMPowerState IOPMPowerState;
diff --git a/iokit/IOKit/pwr_mgt/IOPowerConnection.h b/iokit/IOKit/pwr_mgt/IOPowerConnection.h
index c24469971..179035b2f 100644
--- a/iokit/IOKit/pwr_mgt/IOPowerConnection.h
+++ b/iokit/IOKit/pwr_mgt/IOPowerConnection.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
@@ -63,6 +69,10 @@ protected:
 					used by parent */
     bool		awaitingAck;
 
+    /*! @field readyFlag		true if the child has been added as a power child
+					used by parent */
+	bool		readyFlag;
+
 public:
         /*! @function setParentKnowsState
             @abstract Sets the stateKnown variable.
@@ -133,6 +143,16 @@ public:
         @abstract Returns the awaitingAck variable.
         @discussion Called by the parent. */
     bool getAwaitingAck ( void );
+
+    /*! @function setReadyFlag
+        @abstract Sets the readyFlag variable.
+        @discussion Called by the parent. */
+	void setReadyFlag( bool flag );
+
+    /*! @function getReadyFlag
+        @abstract Returns the readyFlag variable.
+        @discussion Called by the parent. */
+	bool getReadyFlag( void ) const;
 };
 
 #endif /* ! _IOKIT_IOPOWERCONNECTION_H */
diff --git a/iokit/IOKit/pwr_mgt/Makefile b/iokit/IOKit/pwr_mgt/Makefile
index 5b6f80277..dd2bff563 100644
--- a/iokit/IOKit/pwr_mgt/Makefile
+++ b/iokit/IOKit/pwr_mgt/Makefile
@@ -12,21 +12,21 @@ include $(MakeInc_def)
 
 MI_DIR = pwr_mgt
 NOT_EXPORT_HEADERS = \
-	IOPMchangeNoteList.h	\
 	IOPMinformee.h		\
 	IOPMinformeeList.h	\
 	IOPMlog.h		\
-	IOPMpmChild.h		\
 	IOPMPagingPlexus.h	\
 	IOPMPrivate.h
 
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
diff --git a/iokit/IOKit/pwr_mgt/RootDomain.h b/iokit/IOKit/pwr_mgt/RootDomain.h
index 2da2b1dc2..2605169c3 100644
--- a/iokit/IOKit/pwr_mgt/RootDomain.h
+++ b/iokit/IOKit/pwr_mgt/RootDomain.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _IOKIT_ROOTDOMAIN_H
 #define _IOKIT_ROOTDOMAIN_H
@@ -25,7 +31,6 @@
 #include <IOKit/IOService.h>
 #include <IOKit/pwr_mgt/IOPM.h>
 
-class IOPMWorkArbiter;
 class IOPMPowerStateQueue;
 class RootDomainUserClient;
 
@@ -36,7 +41,29 @@ enum {
     kPCICantSleep			= 0x00000004
 };
 
-#define kRootDomainSupportedFeatures "Supported Features"
+/* 
+ *IOPMrootDomain registry property keys
+ */
+#define kRootDomainSupportedFeatures        "Supported Features"
+#define kRootDomainSleepReasonKey           "Last Sleep Reason"
+#define kRootDomainSleepOptionsKey          "Last Sleep Options"
+
+/*
+ * Possible sleep reasons found under kRootDomainSleepReasonsKey
+ */
+#define kIOPMClamshellSleepKey              "Clamshell Sleep"
+#define kIOPMPowerButtonSleepKey            "Power Button Sleep"
+#define kIOPMSoftwareSleepKey               "Software Sleep"
+#define kIOPMOSSwitchHibernationKey         "OS Switch Sleep"
+#define kIOPMIdleSleepKey                   "Idle Sleep"
+#define kIOPMLowPowerSleepKey               "Low Power Sleep"
+#define kIOPMThermalEmergencySleepKey       "Thermal Emergency Sleep"
+
+/*
+ * String constants for communication with PM CPU
+ */
+#define kIOPMRootDomainLidCloseCString      "LidClose"
+#define kIOPMRootDomainBatPowerCString      "BatPower"
 
 // Supported Feature bitfields for IOPMrootDomain::publishFeature()
 enum {
@@ -76,14 +103,15 @@ OSDeclareDefaultStructors(IOPMrootDomain)
 public:
 
     class IOService * wrangler;			// we tickle the wrangler on button presses, etc
-    
-    IOPMWorkArbiter * getPMArbiter(void);
 
     static IOPMrootDomain * construct( void );
     virtual bool start( IOService * provider );
     virtual IOReturn setAggressiveness ( unsigned long, unsigned long );
     virtual IOReturn youAreRoot ( void );
+
     virtual IOReturn sleepSystem ( void );
+    IOReturn sleepSystemOptions ( OSDictionary *options );
+
     virtual IOReturn setProperties ( OSObject * );
     IOReturn shutdownSystem ( void );
     IOReturn restartSystem ( void );
@@ -165,16 +193,29 @@ public:
                                  uintptr_t  refcon,
                                  OSObject   **handle);    // out param
 
+/*! @function acknowledgeSystemWillShutdown
+    @abstract Handle callbacks from IOService::systemWillShutdown().
+    @param The IOService sender of the callback. */
+	void acknowledgeSystemWillShutdown( IOService * from );
+
+/*! @function handlePlatformHaltRestart
+    @abstract Handle platform halt and restart notifications.
+    @param kPEHaltCPU or kPERestartCPU. */
+	void handlePlatformHaltRestart( UInt32 pe_type );
+
 private:
 
     // Points to our parent
     class IORootParent * patriarch;
 
     // Pref: idle time before idle sleep
-    long		sleepSlider;			
+    long		sleepSlider;		
+    long		idleSeconds;
+    uint64_t		autoWakeStart;
+    uint64_t		autoWakeEnd;
 
     // Pref: longest of other idle times (disk and display)
-    long		longestNonSleepSlider;		
+    long		longestNonSleepSlider;
 
     // Difference between sleepSlider and longestNonSleepSlider
     long		extraSleepDelay;		
@@ -185,6 +226,10 @@ private:
     // Used to ignore clamshell close events while we're waking from sleep
     thread_call_t   clamshellWakeupIgnore;   
     
+	// IOPMrootDomain internal sleep call
+    IOReturn 	privateSleepSystem ( const char *sleepReason );
+
+    
     virtual void powerChangeDone ( unsigned long );
     virtual void command_received ( void *, void * , void * , void *);
     virtual bool tellChangeDown ( unsigned long stateNum);
@@ -213,12 +258,11 @@ private:
     bool shouldSleepOnClamshellClosed (void );
     void sendClientClamshellNotification ( void );
     
+    // Inform PMCPU of changes to state like lid, AC vs. battery
+    void informCPUStateChange( uint32_t type, uint32_t value );
+        
     IOLock                  *featuresDictLock;  // guards supportedFeatures
     IOPMPowerStateQueue     *pmPowerStateQueue;
-    
-    IOWorkLoop              *arbiterWorkLoop;
-    IOPMWorkArbiter         *pmArbiter;
-    
     unsigned int user_spindown;       // User's selected disk spindown value
 
     unsigned int systemBooting:1;
@@ -230,13 +274,12 @@ private:
     unsigned int idleSleepPending:1;
     unsigned int sleepASAP:1;
     unsigned int desktopMode:1;
+    unsigned int userDisabledAllSleep:1;
 
     unsigned int acAdaptorConnect:1;
     unsigned int ignoringClamshellDuringWakeup:1;
     unsigned int clamshellIsClosed:1;
     unsigned int clamshellExists:1;
-    unsigned int reservedA:4;
-    unsigned char reservedB[3];
     
     OSArray         *allowedPMSettings;
     
@@ -250,6 +293,10 @@ private:
     IONotifier              *_batteryPublishNotifier;
     IONotifier              *_displayWranglerNotifier;
 
+    // Info for communicating system state changes to PMCPU
+    int32_t                idxPMCPUClamshell;
+    int32_t                idxPMCPULimitedPower;
+
     struct ExpansionData {    
     };
     ExpansionData   *_reserved;
diff --git a/iokit/IOKit/rtc/IORTCController.h b/iokit/IOKit/rtc/IORTCController.h
index 6cbf02ef9..eaf681d12 100644
--- a/iokit/IOKit/rtc/IORTCController.h
+++ b/iokit/IOKit/rtc/IORTCController.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * 24 Nov  1998 suurballe  Created.
diff --git a/iokit/IOKit/rtc/Makefile b/iokit/IOKit/rtc/Makefile
index 4e074c0e4..0095f4b1a 100644
--- a/iokit/IOKit/rtc/Makefile
+++ b/iokit/IOKit/rtc/Makefile
@@ -16,10 +16,12 @@ NOT_EXPORT_HEADERS =
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
diff --git a/iokit/IOKit/system.h b/iokit/IOKit/system.h
index f6427aa27..eee4d8132 100644
--- a/iokit/IOKit/system.h
+++ b/iokit/IOKit/system.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __IOKIT_SYSTEM_H
 #define __IOKIT_SYSTEM_H
@@ -59,10 +65,10 @@ __BEGIN_DECLS
 #include <vm/vm_kern.h>
 #endif /* KERNEL_PRIVATE */
 
-
+#ifndef	_MISC_PROTOS_H_
 extern void	_doprnt( const char *format, va_list *arg,
-			void (*putc)(char), int radix );
-
+			void (*lputc)(char), int radix );
+#endif
 
 __END_DECLS
 
diff --git a/iokit/IOKit/system_management/IOWatchDogTimer.h b/iokit/IOKit/system_management/IOWatchDogTimer.h
index 9fc09fe98..a89b95bd1 100644
--- a/iokit/IOKit/system_management/IOWatchDogTimer.h
+++ b/iokit/IOKit/system_management/IOWatchDogTimer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _IOWATCHDOGTIMER_H
diff --git a/iokit/IOKit/system_management/Makefile b/iokit/IOKit/system_management/Makefile
index 8b2463850..a9ad1604c 100644
--- a/iokit/IOKit/system_management/Makefile
+++ b/iokit/IOKit/system_management/Makefile
@@ -16,10 +16,12 @@ NOT_EXPORT_HEADERS =
 INSTINC_SUBDIRS =
 INSTINC_SUBDIRS_PPC =
 INSTINC_SUBDIRS_I386 =
+INSTINC_SUBDIRS_ARM =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 ALL_HEADERS = $(shell (cd $(SOURCE); echo *.h))
 
diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
index 0d934cdb8..efe64454d 100644
--- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/assert.h>
 #include <IOKit/system.h>
@@ -108,9 +114,11 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 				mach_vm_address_t physicalMask)
 {
     kern_return_t 	kr;
+    task_t		mapTask = NULL;
+    vm_map_t 		vmmap = NULL;
     addr64_t            lastIOAddr;
-    vm_map_t 		vmmap = 0;
-    IOOptionBits	iomdOptions = kIOMemoryAsReference | kIOMemoryTypeVirtual;
+    IOAddressRange	range;
+    IOOptionBits	iomdOptions = kIOMemoryTypeVirtual64;
 
     if (!capacity)
         return false;
@@ -120,100 +128,72 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
     _physAddrs    = 0;
     _physSegCount = 0;
     _buffer	  = 0;
+    range.address = 0;
+    range.length  = 0;
+    _ranges.v64   = &range;
 
     // Grab the direction and the Auto Prepare bits from the Buffer MD options
     iomdOptions  |= options & (kIOMemoryDirectionMask | kIOMemoryAutoPrepare);
 
-    if ((options & kIOMemorySharingTypeMask) && (alignment < page_size))
-        alignment = page_size;
-
-    if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
-        return false;
+    if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask)) && (alignment < page_size))
+	alignment = page_size;
 
     if (physicalMask && (alignment <= 1))
 	alignment = ((physicalMask ^ PAGE_MASK) & PAGE_MASK) + 1;
 
-    if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask)
-	physicalMask = 0xFFFFFFFF;
-
     _alignment = alignment;
-    if (options & kIOMemoryPageable)
-    {
-        iomdOptions |= kIOMemoryBufferPageable;
 
-        ipc_port_t sharedMem;
-        vm_size_t size = round_page_32(capacity);
+    if (((inTask != kernel_task) && !(options & kIOMemoryPageable)) ||
+	(physicalMask && (options & kIOMapCacheMask)))
+	return false;
 
-        // must create the entry before any pages are allocated
+    if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask)
+	physicalMask = 0xFFFFFFFF;
 
-	// set flags for entry + object create
-	vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE
-				    | MAP_MEM_NAMED_CREATE;
+    // set flags for entry + object create
+    vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
 
-        if (options & kIOMemoryPurgeable)
-            memEntryCacheMode |= MAP_MEM_PURGABLE;
+    // set memory entry cache mode
+    switch (options & kIOMapCacheMask)
+    {
+	case kIOMapInhibitCache:
+	    SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
+	    break;
+
+	case kIOMapWriteThruCache:
+	    SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
+	    break;
+
+	case kIOMapWriteCombineCache:
+	    SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
+	    break;
+
+	case kIOMapCopybackCache:
+	    SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
+	    break;
+
+	case kIOMapDefaultCache:
+	default:
+	    SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
+	    break;
+    }
 
-	// set memory entry cache mode
-	switch (options & kIOMapCacheMask)
-	{
-	    case kIOMapInhibitCache:
-		SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
-		break;
-    
-	    case kIOMapWriteThruCache:
-		SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
-		break;
-
-	    case kIOMapWriteCombineCache:
-		SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
-		break;
-
-	    case kIOMapCopybackCache:
-		SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
-		break;
-
-	    case kIOMapDefaultCache:
-	    default:
-		SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
-		break;
-	}
+    if (options & kIOMemoryPageable)
+    {
+	iomdOptions |= kIOMemoryBufferPageable;
 
-	kr = mach_make_memory_entry( vmmap,
-		    &size, 0,
-		    memEntryCacheMode, &sharedMem,
-		    NULL );
+	// must create the entry before any pages are allocated
 
-	if( (KERN_SUCCESS == kr) && (size != round_page_32(capacity))) {
-	    ipc_port_release_send( sharedMem );
-	    kr = kIOReturnVMError;
-	}
-	if( KERN_SUCCESS != kr)
-	    return( false );
+	// set flags for entry + object create
+	memEntryCacheMode |= MAP_MEM_NAMED_CREATE;
 
-	_memEntry = (void *) sharedMem;
-#if IOALLOCDEBUG
-       debug_iomallocpageable_size += size;
-#endif
-	if (NULL == inTask)
-	    inTask = kernel_task;
-	else if (inTask == kernel_task)
-	{
-	    vmmap = kernel_map;
-	}
-	else
-	{
-	    if( !reserved) {
-		reserved = IONew( ExpansionData, 1 );
-		if( !reserved)
-		    return( false );
-	    }
-	    vmmap = get_task_map(inTask);
-	    vm_map_reference(vmmap);
-	    reserved->map = vmmap;
-	}
+	if (options & kIOMemoryPurgeable)
+	    memEntryCacheMode |= MAP_MEM_PURGABLE;
     }
     else
     {
+	memEntryCacheMode |= MAP_MEM_NAMED_REUSE;
+
 	if (IOMapper::gSystem)
 	    // assuming mapped space is 2G
 	    lastIOAddr = (1UL << 31) - PAGE_SIZE;
@@ -223,33 +203,21 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 	if (physicalMask && (lastIOAddr != (lastIOAddr & physicalMask)))
 	{
 	    mach_vm_address_t address;
-	    iomdOptions &= ~kIOMemoryTypeVirtual;
-	    iomdOptions |= kIOMemoryTypePhysical;
+	    iomdOptions &= ~kIOMemoryTypeVirtual64;
+	    iomdOptions |= kIOMemoryTypePhysical64;
 
 	    address = IOMallocPhysical(capacity, physicalMask);
 	    _buffer = (void *) address;
 	    if (!_buffer)
 		return false;
 
-	    if (inTask == kernel_task)
-	    {
-		vmmap = kernel_map;
-	    }
-	    else if (NULL != inTask)
-	    {
-		if( !reserved) {
-		    reserved = IONew( ExpansionData, 1 );
-		    if( !reserved)
-			return( false );
-		}
-		vmmap = get_task_map(inTask);
-		vm_map_reference(vmmap);
-		reserved->map = vmmap;
-	    }
+	    mapTask = inTask;
 	    inTask = 0;
 	}
 	else
 	{
+	    vmmap = kernel_map;
+
 	    // Buffer shouldn't auto prepare they should be prepared explicitly
 	    // But it never was enforced so what are you going to do?
 	    iomdOptions |= kIOMemoryAutoPrepare;
@@ -266,11 +234,52 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 	}
     }
 
-    _singleRange.v.address = (vm_address_t) _buffer;
-    _singleRange.v.length  = capacity;
+    if( (kIOMemoryTypePhysical64 != (kIOMemoryTypeMask & iomdOptions)) 
+	&& (options & (kIOMemoryPageable | kIOMapCacheMask))) {
+	ipc_port_t	sharedMem;
+	vm_size_t	size = round_page_32(capacity);
+
+	kr = mach_make_memory_entry(vmmap,
+				    &size, (vm_offset_t)_buffer,
+				    memEntryCacheMode, &sharedMem,
+				    NULL );
+
+	if( (KERN_SUCCESS == kr) && (size != round_page_32(capacity))) {
+	    ipc_port_release_send( sharedMem );
+	    kr = kIOReturnVMError;
+	}
+	if( KERN_SUCCESS != kr)
+	    return( false );
+
+	_memEntry = (void *) sharedMem;
 
-    if (!super::initWithOptions(&_singleRange.v, 1, 0,
-                               inTask, iomdOptions, /* System mapper */ 0))
+	if( options & kIOMemoryPageable) {
+#if IOALLOCDEBUG
+	    debug_iomallocpageable_size += size;
+#endif
+	    mapTask = inTask;
+	    if (NULL == inTask)
+		inTask = kernel_task;
+	}
+	else if (options & kIOMapCacheMask)
+	{
+	    // Prefetch each page to put entries into the pmap
+	    volatile UInt8 *	startAddr = (UInt8 *)_buffer;
+	    volatile UInt8 *	endAddr   = (UInt8 *)_buffer + capacity;
+
+	    while (startAddr < endAddr)
+	    {
+		*startAddr;
+		startAddr += page_size;
+	    }
+	}
+    }
+
+    range.address = (mach_vm_address_t) _buffer;
+    range.length  = capacity;
+
+    if (!super::initWithOptions(&range, 1, 0,
+				inTask, iomdOptions, /* System mapper */ 0))
 	return false;
 
     if (physicalMask && !IOMapper::gSystem)
@@ -300,7 +309,7 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 
 	if (lastIOAddr != (lastIOAddr & physicalMask))
 	{
-	    if (kIOMemoryTypePhysical != (_flags & kIOMemoryTypeMask))
+	    if (kIOMemoryTypePhysical64 != (_flags & kIOMemoryTypeMask))
 	    {
 		// flag a retry
 		_physSegCount = 1;
@@ -309,21 +318,28 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 	}
     }
 
-    if (vmmap)
+    if (mapTask)
     {
-	kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, capacity);
-	if (KERN_SUCCESS != kr)
+	if (!reserved) {
+	    reserved = IONew( ExpansionData, 1 );
+	    if( !reserved)
+		return( false );
+	}
+	reserved->map = map(mapTask, 0, kIOMapAnywhere, 0, 0);
+	if (!reserved->map)
 	{
 	    _buffer = 0;
 	    return( false );
 	}
-
-	if (kIOMemoryTypeVirtual & iomdOptions)
-	    _singleRange.v.address = (vm_address_t) _buffer;
+	release();	    // map took a retain on this
+	mach_vm_address_t buffer = reserved->map->getAddress();
+	_buffer = (void *) buffer;
+	if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions))
+	    _ranges.v64->address = buffer;
     }
 
     setLength(capacity);
-    
+
     return true;
 }
 
@@ -484,14 +500,16 @@ void IOBufferMemoryDescriptor::free()
     IOOptionBits     options   = _options;
     vm_size_t        size      = _capacity;
     void *           buffer    = _buffer;
-    IOVirtualAddress source    = _singleRange.v.address;
-    vm_map_t	     vmmap     = 0;
+    IOVirtualAddress source    = _ranges.v64->address;
+    IOMemoryMap *    map       = 0;
     vm_offset_t      alignment = _alignment;
 
     if (reserved)
     {
-	vmmap = reserved->map;
+	map = reserved->map;
         IODelete( reserved, ExpansionData, 1 );
+	if (map)
+	    map->release();
     }
 
     /* super::free may unwire - deallocate buffer afterwards */
@@ -500,25 +518,13 @@ void IOBufferMemoryDescriptor::free()
     if (options & kIOMemoryPageable)
     {
 #if IOALLOCDEBUG
-        if (!buffer || vmmap)
-            debug_iomallocpageable_size -= round_page_32(size);
+	debug_iomallocpageable_size -= round_page_32(size);
 #endif
-        if (buffer)
-        {
-            if (vmmap)
-                vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size));
-            else
-                IOFreePageable(buffer, size);
-        }
     }
     else if (buffer)
     {
-	if (kIOMemoryTypePhysical == (flags & kIOMemoryTypeMask))
-	{
-            if (vmmap)
-                vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size));
+	if (kIOMemoryTypePhysical64 == (flags & kIOMemoryTypeMask))
 	    IOFreePhysical((mach_vm_address_t) source, size);
-	}
         else if (options & kIOMemoryPhysicallyContiguous)
             IOKernelFreeContiguous((mach_vm_address_t) buffer, size);
         else if (alignment > 1)
@@ -526,8 +532,6 @@ void IOBufferMemoryDescriptor::free()
         else
             IOFree(buffer, size);
     }
-    if (vmmap)
-	vm_map_deallocate(vmmap);
 }
 
 /*
@@ -555,7 +559,7 @@ void IOBufferMemoryDescriptor::setLength(vm_size_t length)
     assert(length <= _capacity);
 
     _length = length;
-    _singleRange.v.length = length;
+    _ranges.v64->length = length;
 }
 
 /*
@@ -587,10 +591,10 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 
     offset = _length;
     _length += actualBytesToCopy;
-    _singleRange.v.length += actualBytesToCopy;
+    _ranges.v64->length += actualBytesToCopy;
 
     if (_task == kernel_task)
-	bcopy(/* from */ bytes, (void *)(_singleRange.v.address + offset),
+	bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
 	      actualBytesToCopy);
     else
 	writeBytes(offset, bytes, actualBytesToCopy);
@@ -605,10 +609,10 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
  */
 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 {
-    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+    if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 	return _buffer;
     else
-	return (void *)_singleRange.v.address;
+	return (void *)_ranges.v64->address;
 }
 
 
@@ -621,10 +625,10 @@ void *
 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 {
     IOVirtualAddress address;
-    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+    if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 	address = (IOVirtualAddress) _buffer;
     else
-	address = _singleRange.v.address;
+	address = _ranges.v64->address;
 
    if (start < _length && (start + withLength) <= _length)
         return (void *)(address + start);
diff --git a/iokit/Kernel/IOCPU.cpp b/iokit/Kernel/IOCPU.cpp
index c86d5a231..610229ce3 100644
--- a/iokit/Kernel/IOCPU.cpp
+++ b/iokit/Kernel/IOCPU.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1999-2000 Apple Computer, Inc.  All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999-2000 Apple Computer, Inc.  All rights reserved.
@@ -31,11 +37,197 @@ extern "C" {
 #include <pexpert/pexpert.h>
 }
 
+#include <machine/machine_routines.h>
+
 #include <IOKit/IOLib.h>
 #include <IOKit/IOPlatformExpert.h>
+#include <IOKit/pwr_mgt/RootDomain.h>
 #include <IOKit/IOUserClient.h>
+#include <IOKit/IOKitKeysPrivate.h>
 #include <IOKit/IOCPU.h>
 
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#include <kern/queue.h>
+
+typedef kern_return_t (*iocpu_platform_action_t)(void * refcon0, void * refcon1, uint32_t priority,
+						 void * param1, void * param2, void * param3);
+
+struct iocpu_platform_action_entry
+{
+    queue_chain_t                     link;
+    iocpu_platform_action_t           action;
+    int32_t	                      priority;
+    void *	                      refcon0;
+    void *			      refcon1;
+    struct iocpu_platform_action_entry * alloc_list;
+};
+typedef struct iocpu_platform_action_entry iocpu_platform_action_entry_t;
+
+queue_head_t * 
+iocpu_get_platform_quiesce_queue(void);
+
+queue_head_t * 
+iocpu_get_platform_active_queue(void);
+
+void
+iocpu_platform_cpu_action_init(queue_head_t * quiesce_queue, queue_head_t * init_queue);
+
+void
+iocpu_add_platform_action(queue_head_t * queue, iocpu_platform_action_entry_t * entry);
+
+void
+iocpu_remove_platform_action(iocpu_platform_action_entry_t * entry);
+
+kern_return_t
+iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32_t last_priority,
+					void * param1, void * param2, void * param3);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+static iocpu_platform_action_entry_t * gIOAllActionsQueue;
+static queue_head_t gIOSleepActionQueue;
+static queue_head_t gIOWakeActionQueue;
+
+static queue_head_t iocpu_quiesce_queue;
+static queue_head_t iocpu_active_queue;
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void 
+iocpu_platform_cpu_action_init(queue_head_t * quiesce_queue, __unused queue_head_t * init_queue)
+{
+#if 0
+    enum { kNumQuiesceActions = 2 };
+    static iocpu_platform_action_entry_t quiesce_actions[kNumQuiesceActions] = 
+    {
+	{ { NULL, NULL }, (iocpu_platform_action_t) &clean_mmu_dcache, 97000, 0, 0, NULL },    
+	{ { NULL, NULL }, (iocpu_platform_action_t) &arm_sleep, 99000, 0, 0, NULL },    
+    };
+    unsigned int idx;
+
+    for (idx = 0; idx < kNumQuiesceActions; idx++)
+	iocpu_add_platform_action(quiesce_queue, &quiesce_actions[idx]);
+#endif
+}
+
+queue_head_t * iocpu_get_platform_quiesce_queue(void)
+{
+    if (!iocpu_quiesce_queue.next)
+    {
+	queue_init(&iocpu_quiesce_queue);
+	queue_init(&iocpu_active_queue);
+	iocpu_platform_cpu_action_init(&iocpu_quiesce_queue, &iocpu_active_queue);
+    }
+    return (&iocpu_quiesce_queue);
+}
+
+queue_head_t * iocpu_get_platform_active_queue(void)
+{
+    return (&iocpu_active_queue);
+}
+
+void iocpu_add_platform_action(queue_head_t * queue, iocpu_platform_action_entry_t * entry)
+{
+    iocpu_platform_action_entry_t * next;
+
+    queue_iterate(queue, next, iocpu_platform_action_entry_t *, link)
+    {
+	if (next->priority > entry->priority)
+	{
+	    queue_insert_before(queue, entry, next, iocpu_platform_action_entry_t *, link);
+	    return;
+	}
+    }
+    queue_enter(queue, entry, iocpu_platform_action_entry_t *, link);	// at tail
+}
+
+void iocpu_remove_platform_action(iocpu_platform_action_entry_t * entry)
+{
+    remque(&entry->link);
+}
+
+kern_return_t
+iocpu_run_platform_actions(queue_head_t * queue, uint32_t first_priority, uint32_t last_priority,
+					void * param1, void * param2, void * param3)
+{
+    kern_return_t                ret = KERN_SUCCESS;
+    kern_return_t                result = KERN_SUCCESS;
+    iocpu_platform_action_entry_t * next;
+
+    queue_iterate(queue, next, iocpu_platform_action_entry_t *, link)
+    {
+	uint32_t pri = (next->priority < 0) ? -next->priority : next->priority;
+	if ((pri >= first_priority) && (pri <= last_priority))
+	{
+	    //kprintf("[%p]", next->action);
+	    ret = (*next->action)(next->refcon0, next->refcon1, pri, param1, param2, param3);
+	}
+	if (KERN_SUCCESS == result)
+	    result = ret;
+    }
+    return (result);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+extern "C" kern_return_t 
+IOCPURunPlatformQuiesceActions(void)
+{
+    return (iocpu_run_platform_actions(iocpu_get_platform_quiesce_queue(), 0, 0UL-1,
+				    NULL, NULL, NULL));
+}
+
+extern "C" kern_return_t 
+IOCPURunPlatformActiveActions(void)
+{
+    return (iocpu_run_platform_actions(iocpu_get_platform_active_queue(), 0, 0UL-1,
+				    NULL, NULL, NULL));
+}
+
+static kern_return_t 
+IOServicePlatformAction(void * refcon0, void * refcon1, uint32_t priority,
+			  void * param1, void * param2, void * param3)
+{
+    IOReturn	     ret;
+    IOService *      service  = (IOService *)      refcon0;
+    const OSSymbol * function = (const OSSymbol *) refcon1;
+
+    kprintf("%s -> %s\n", function->getCStringNoCopy(), service->getName());
+
+    ret = service->callPlatformFunction(function, false, 
+					 (void *) priority, param1, param2, param3);
+
+    return (ret);
+}
+
+static void
+IOInstallServicePlatformAction(IOService * service, 
+				const OSSymbol * key, queue_head_t * queue,
+				bool reverse)
+{
+    OSNumber * num;
+    iocpu_platform_action_entry_t * entry;
+    uint32_t priority;
+
+    num = OSDynamicCast(OSNumber, service->getProperty(key));
+    if (!num)
+	return;
+
+    entry = IONew(iocpu_platform_action_entry_t, 1);
+    entry->action = &IOServicePlatformAction;
+    priority = num->unsigned32BitValue();
+    if (reverse)
+	entry->priority = -priority;
+    else
+	entry->priority = priority;
+    entry->refcon0 = service;
+    entry->refcon1 = (void *) key;
+
+    iocpu_add_platform_action(queue, entry);
+    entry->alloc_list = gIOAllActionsQueue;
+    gIOAllActionsQueue = entry;
+}
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -63,17 +255,17 @@ void PE_cpu_signal(cpu_id_t source, cpu_id_t target)
   if (sourceCPU && targetCPU) sourceCPU->signalCPU(targetCPU);
 }
 
-void PE_cpu_machine_init(cpu_id_t target, boolean_t boot)
+void PE_cpu_machine_init(cpu_id_t target, boolean_t bootb)
 {
   IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target);
   
-  if (targetCPU) targetCPU->initCPU(boot);
+  if (targetCPU) targetCPU->initCPU(bootb);
 }
 
 void PE_cpu_machine_quiesce(cpu_id_t target)
 {
   IOCPU *targetCPU = OSDynamicCast(IOCPU, (OSObject *)target);
-  
+
   if (targetCPU) targetCPU->quiesceCPU();
 }
 
@@ -99,27 +291,72 @@ static OSString *gIOCPUStateNames[kIOCPUStateCount];
 
 void IOCPUSleepKernel(void)
 {
-  long cnt, numCPUs;
-  IOCPU *target;
-  
-  numCPUs = gIOCPUs->getCount();
-  
-  // Sleep the CPUs.
-  cnt = numCPUs;
-  while (cnt--) {
-    target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt));
-    if (target->getCPUState() == kIOCPUStateRunning) {
-      target->haltCPU();
+    long cnt, numCPUs;
+    IOCPU *target;
+  
+    kprintf("IOCPUSleepKernel\n");
+
+    OSIterator * iter;
+    IOService *  service;
+
+    queue_init(&gIOSleepActionQueue);
+    queue_init(&gIOWakeActionQueue);
+
+    iter = IORegistryIterator::iterateOver( gIOServicePlane,
+					    kIORegistryIterateRecursively );
+    if( iter)
+    {
+	do
+	{
+	    iter->reset();
+	    while((service = (IOService *) iter->getNextObject()))
+	    {
+		IOInstallServicePlatformAction(service, gIOPlatformSleepActionKey,   &gIOSleepActionQueue,		 false);
+		IOInstallServicePlatformAction(service, gIOPlatformWakeActionKey,    &gIOWakeActionQueue,		 true);
+		IOInstallServicePlatformAction(service, gIOPlatformQuiesceActionKey, iocpu_get_platform_quiesce_queue(), false);
+		IOInstallServicePlatformAction(service, gIOPlatformActiveActionKey,  iocpu_get_platform_active_queue(),  true);
+	    }
+	}
+	while( !service && !iter->isValid());
+	iter->release();
     }
-  }
-  
-  // Wake the other CPUs.
-  for (cnt = 1; cnt < numCPUs; cnt++) {
-    target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt));
-    if (target->getCPUState() == kIOCPUStateStopped) {
-      processor_start(target->getMachProcessor());
+
+    iocpu_run_platform_actions(&gIOSleepActionQueue, 0, 0UL-1,
+				NULL, NULL, NULL);
+
+    numCPUs = gIOCPUs->getCount();
+    // Sleep the CPUs.
+    cnt = numCPUs;
+    while (cnt--) {
+	target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt));
+	if (target->getCPUState() == kIOCPUStateRunning) {
+	    target->haltCPU();
+	}
+    }
+
+    iocpu_run_platform_actions(&gIOWakeActionQueue, 0, 0UL-1,
+				    NULL, NULL, NULL);
+
+    iocpu_platform_action_entry_t * entry;
+    while ((entry = gIOAllActionsQueue))
+    {
+	gIOAllActionsQueue = entry->alloc_list;
+	iocpu_remove_platform_action(entry);
+	IODelete(entry, iocpu_platform_action_entry_t, 1);
+    }
+
+    if (!queue_empty(&gIOSleepActionQueue))
+	IOPanic("gIOSleepActionQueue");
+    if (!queue_empty(&gIOWakeActionQueue))
+	IOPanic("gIOWakeActionQueue");
+  
+    // Wake the other CPUs.
+    for (cnt = 1; cnt < numCPUs; cnt++) {
+	target = OSDynamicCast(IOCPU, gIOCPUs->getObject(cnt));
+	if (target->getCPUState() == kIOCPUStateStopped) {
+	    processor_start(target->getMachProcessor());
+	}
     }
-  }
 }
 
 void IOCPU::initCPUs(void)
@@ -365,6 +602,10 @@ void IOCPUInterruptController::setCPUInterruptProperties(IOService *service)
   OSData       *tmpData;
   long         tmpLong;
   
+  if ((service->getProperty(gIOInterruptControllersKey) != 0) &&
+      (service->getProperty(gIOInterruptSpecifiersKey) != 0))
+    return;
+  
   // Create the interrupt specifer array.
   specifier = OSArray::withCapacity(numCPUs);
   for (cnt = 0; cnt < numCPUs; cnt++) {
@@ -394,7 +635,7 @@ void IOCPUInterruptController::enableCPUInterrupt(IOCPU *cpu)
 
 	ml_install_interrupt_handler(cpu, cpu->getCPUNumber(), this, handler, 0);
   
-  enabledCPUs++;
+	enabledCPUs++;
   
   if (enabledCPUs == numCPUs) thread_wakeup(this);
 }
diff --git a/iokit/Kernel/IOCatalogue.cpp b/iokit/Kernel/IOCatalogue.cpp
index 2d05a9b4a..939c78a3c 100644
--- a/iokit/Kernel/IOCatalogue.cpp
+++ b/iokit/Kernel/IOCatalogue.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -25,6 +31,12 @@
  * HISTORY
  *
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <IOKit/IODeviceTreeSupport.h>
 #include <IOKit/IOService.h>
@@ -36,6 +48,7 @@ extern "C" {
 #include <mach/kmod.h>
 #include <mach-o/mach_header.h>
 #include <kern/host.h>
+#include <security/mac_data.h>
 };
 
 #include <IOKit/IOLib.h>
@@ -115,6 +128,410 @@ kmod_start_or_stop(
 extern "C" kern_return_t kmod_retain(kmod_t id);
 extern "C" kern_return_t kmod_release(kmod_t id);
 
+#if CONFIG_MACF_KEXT
+/* MAC Framework support */
+
+/* 
+ * define IOC_DEBUG to display run-time debugging information
+ * #define IOC_DEBUG 1
+ */
+
+#ifdef IOC_DEBUG
+#define DPRINTF(x)	printf x
+#else
+#define IOC_DEBUG
+#define DPRINTF(x)
+#endif
+
+static bool
+primitive_type(OSObject *obj)
+{
+    const OSMetaClass *typeID;
+
+    typeID = OSTypeIDInst(obj);
+    if (typeID == OSTypeID(OSString) || typeID == OSTypeID(OSNumber) ||
+        typeID == OSTypeID(OSBoolean) || typeID == OSTypeID(OSData))
+	return(true);
+    else
+	return(false);
+}
+
+static int
+primitive_type_length(OSObject *obj)
+{
+    const OSMetaClass *typeID;
+    int len;
+
+    typeID = OSTypeIDInst(obj);
+    if (typeID == OSTypeID(OSString)) {
+        OSString * stringObj = OSDynamicCast(OSString, obj);
+        len = stringObj->getLength() + 1;
+    }
+    else if (typeID == OSTypeID(OSNumber)) {
+        len = sizeof("4294967295");	/* UINT32_MAX */
+    }
+    else if (typeID == OSTypeID(OSBoolean)) {
+        OSBoolean * boolObj = OSDynamicCast(OSBoolean, obj);
+        len = boolObj->isTrue() ? sizeof("true") : sizeof("false");
+    }
+    else if (typeID == OSTypeID(OSData)) {
+        OSData * dataObj = OSDynamicCast(OSData, obj);
+        len = dataObj->getLength();
+    }
+    else {
+	len = 0;
+    }
+    return(len);
+}
+
+static void
+primitive_type_collect(struct mac_module_data_element *element, OSObject *value)
+{
+    const OSMetaClass *typeID;
+
+    typeID = OSTypeIDInst(value);
+    if (typeID == OSTypeID(OSString)) {
+        OSString *stringObj = OSDynamicCast(OSString, value);
+        element->value_type = MAC_DATA_TYPE_PRIMITIVE;
+        element->value_size = stringObj->getLength() + 1;
+	DPRINTF(("osdict: string %s size %d\n", 
+	    stringObj->getCStringNoCopy(), element->value_size));
+        memcpy(element->value, stringObj->getCStringNoCopy(),
+            element->value_size);
+    } else if (typeID == OSTypeID(OSNumber)) {
+        OSNumber *numberObj = OSDynamicCast(OSNumber, value);
+        element->value_type = MAC_DATA_TYPE_PRIMITIVE;
+        element->value_size = sprintf(element->value, "%u",
+	    numberObj->unsigned32BitValue()) + 1;
+    } else if (typeID == OSTypeID(OSBoolean)) {
+        OSBoolean *boolObj = OSDynamicCast(OSBoolean, value);
+        element->value_type = MAC_DATA_TYPE_PRIMITIVE;
+        if (boolObj->isTrue()) {
+            strcpy(element->value, "true");
+            element->value_size = 5;
+        } else {
+            strcpy(element->value, "false");
+            element->value_size = 6;
+        }
+    } else if (typeID == OSTypeID(OSData)) {
+        OSData *dataObj = OSDynamicCast(OSData, value);
+        element->value_type = MAC_DATA_TYPE_PRIMITIVE;
+        element->value_size = dataObj->getLength();
+ 	DPRINTF(("osdict: data size %d\n", dataObj->getLength()));
+        memcpy(element->value, dataObj->getBytesNoCopy(),
+            element->value_size);
+    }
+}
+
+/*********************************************************************
+* This function takes an OSDictionary and returns a struct mac_module_data
+* list.
+*********************************************************************/
+struct mac_module_data *
+osdict_encode(OSDictionary *dict)
+{
+    const OSMetaClass * typeID;	            // don't release
+    OSString * key = NULL;                  // don't release
+    OSCollectionIterator * keyIterator = 0; // must release
+    struct mac_module_data * module_data = 0;
+    struct mac_module_data_element * element;
+    unsigned int strtabsize = 0;
+    unsigned int listtabsize = 0;
+    unsigned int dicttabsize = 0;
+    unsigned int nkeys = 0;
+    unsigned int datalen;
+    char *strtab = NULL;
+    char *listtab = NULL;
+    char *dicttab = NULL;
+    vm_offset_t data_addr;
+
+    keyIterator = OSCollectionIterator::withCollection(dict);
+    if (!keyIterator)
+        goto finish;
+
+    /* Iterate over OSModuleData to figure out total size */
+    while ( (key = OSDynamicCast(OSString, keyIterator->getNextObject())) ) {
+
+	// Get the key's value and determine its type
+        OSObject * value = dict->getObject(key);
+        if (!value)
+            continue;
+
+	typeID = OSTypeIDInst(value);
+	if (primitive_type(value)) {
+	    strtabsize += primitive_type_length(value);
+	}
+	else if (typeID == OSTypeID(OSArray)) {
+	    unsigned int k, cnt, nents;
+	    OSArray *arrayObj = OSDynamicCast(OSArray, value);
+
+	    nents = 0;
+	    cnt = arrayObj->getCount();
+	    for (k = 0; k < cnt; k++) {
+		value = arrayObj->getObject(k);
+		typeID = OSTypeIDInst(value);
+		if (primitive_type(value)) {
+		    listtabsize += primitive_type_length(value);
+		    nents++;
+		}
+		else if (typeID == OSTypeID(OSDictionary)) {
+		    unsigned int dents;
+		    OSDictionary *dictObj;
+		    OSString *dictkey;
+		    OSCollectionIterator *dictIterator;
+
+		    dents = 0;
+		    dictObj = OSDynamicCast(OSDictionary, value);
+		    dictIterator = OSCollectionIterator::withCollection(dictObj);
+		    if (!dictIterator)
+			goto finish;
+		    while ((dictkey = OSDynamicCast(OSString,
+			    		      dictIterator->getNextObject()))) {
+			OSObject *dictvalue;
+
+			dictvalue = dictObj->getObject(dictkey);
+			if (!dictvalue)
+			    continue;
+			if (primitive_type(dictvalue)) {
+			    strtabsize += primitive_type_length(dictvalue);
+			}
+			else {
+			    continue;	/* Only handle primitive types here.  */
+			}
+			/*
+			 * Allow for the "arraynnn/" prefix in the key length.
+			 */
+			strtabsize += dictkey->getLength() + 1;
+			dents++;
+		    }
+		    dictIterator->release();
+		    if (dents-- > 0) {
+			dicttabsize += sizeof(struct mac_module_data_list) +
+			    dents * sizeof(struct mac_module_data_element);
+			nents++;
+		    }
+		}
+		else {
+		    continue;		/* Skip everything else.              */
+		}
+	    }
+	    if (nents == 0)
+		continue;
+	    listtabsize += sizeof(struct mac_module_data_list) +
+		(nents - 1) * sizeof(struct mac_module_data_element);
+	}
+	else {
+	    continue;		/* skip anything else */
+	}
+	strtabsize += key->getLength() + 1;
+	nkeys++;
+    }
+    if (nkeys == 0)
+    	goto finish;
+
+    /*
+     * Allocate and fill in the module data structures.
+     */
+    datalen = sizeof(struct mac_module_data) +
+	sizeof(mac_module_data_element) * (nkeys - 1) +
+        strtabsize + listtabsize + dicttabsize;
+    DPRINTF(("osdict: datalen %d strtabsize %d listtabsize %d dicttabsize %d\n", 
+	    datalen, strtabsize, listtabsize, dicttabsize));
+    if (kmem_alloc(kernel_map, &data_addr, datalen) != KERN_SUCCESS)
+	goto finish;
+    module_data = (mac_module_data *)data_addr;
+    module_data->base_addr = data_addr;
+    module_data->size = datalen;
+    module_data->count = nkeys;
+    strtab = (char *)&module_data->data[nkeys];
+    listtab = strtab + strtabsize;
+    dicttab = listtab + listtabsize;
+    DPRINTF(("osdict: data_addr %p strtab %p listtab %p dicttab %p end %p\n", 
+	    data_addr, strtab, listtab, dicttab, data_addr + datalen));
+
+    keyIterator->reset();
+    nkeys = 0;
+    element = &module_data->data[0];
+    DPRINTF(("osdict: element %p\n", element));
+    while ( (key = OSDynamicCast(OSString, keyIterator->getNextObject())) ) {
+
+	// Get the key's value and determine its type
+        OSObject * value = dict->getObject(key);
+        if (!value)
+            continue;
+
+	/* Store key */
+	DPRINTF(("osdict: element @%p\n", element));
+	element->key = strtab;
+	element->key_size = key->getLength() + 1;
+	DPRINTF(("osdict: key %s size %d @%p\n", key->getCStringNoCopy(), element->key_size, strtab));
+	memcpy(element->key, key->getCStringNoCopy(), element->key_size);
+
+	typeID = OSTypeIDInst(value);
+	if (primitive_type(value)) {
+	    /* Store value */
+	    element->value = element->key + element->key_size;
+	    DPRINTF(("osdict: primitive element value %p\n", element->value));
+	    primitive_type_collect(element, value);
+	    strtab += element->key_size + element->value_size;
+	    DPRINTF(("osdict: new strtab %p\n", strtab));
+	}
+	else if (typeID == OSTypeID(OSArray)) {
+	    unsigned int k, cnt, nents;
+	    char *astrtab;
+	    struct mac_module_data_list *arrayhd;
+	    struct mac_module_data_element *ele;
+	    OSArray *arrayObj = OSDynamicCast(OSArray, value);
+
+	    element->value = listtab;
+	    DPRINTF(("osdict: array element value %p\n", element->value));
+	    element->value_type = MAC_DATA_TYPE_ARRAY;
+	    arrayhd = (struct mac_module_data_list *)element->value;
+	    arrayhd->type = 0;
+	    DPRINTF(("osdict: arrayhd %p\n", arrayhd));
+	    nents = 0;
+	    astrtab = strtab + element->key_size;
+	    ele = &(arrayhd->list[0]);
+	    cnt = arrayObj->getCount();
+	    for (k = 0; k < cnt; k++) {
+		value = arrayObj->getObject(k);
+		DPRINTF(("osdict: array ele %d @%p\n", nents, ele));
+		ele->key = NULL;
+		ele->key_size = 0;
+		typeID = OSTypeIDInst(value);
+		if (primitive_type(value)) {
+		    if (arrayhd->type != 0 &&
+			arrayhd->type != MAC_DATA_TYPE_PRIMITIVE)
+			continue;
+		    arrayhd->type = MAC_DATA_TYPE_PRIMITIVE;
+		    ele->value = astrtab;
+		    primitive_type_collect(ele, value);
+		    astrtab += ele->value_size;
+		    DPRINTF(("osdict: array new astrtab %p\n", astrtab));
+		}
+		else if (typeID == OSTypeID(OSDictionary)) {
+		    unsigned int dents;
+		    char *dstrtab;
+		    OSDictionary *dictObj;
+		    OSString *dictkey;
+		    OSCollectionIterator *dictIterator;
+		    struct mac_module_data_list *dicthd;
+		    struct mac_module_data_element *dele;
+
+		    if (arrayhd->type != 0 &&
+			arrayhd->type != MAC_DATA_TYPE_DICT)
+			continue;
+		    dictObj = OSDynamicCast(OSDictionary, value);
+		    dictIterator = OSCollectionIterator::withCollection(dictObj);
+		    if (!dictIterator)
+			goto finish;
+		    DPRINTF(("osdict: dict\n"));
+		    ele->value = dicttab;
+		    ele->value_type = MAC_DATA_TYPE_DICT;
+		    dicthd = (struct mac_module_data_list *)ele->value;
+		    DPRINTF(("osdict: dicthd %p\n", dicthd));
+		    dstrtab = astrtab;
+		    dents = 0;
+		    while ((dictkey = OSDynamicCast(OSString,
+			    		      dictIterator->getNextObject()))) {
+			OSObject *dictvalue;
+
+			dictvalue = dictObj->getObject(dictkey);
+			if (!dictvalue)
+			    continue;
+			dele = &(dicthd->list[dents]);
+			DPRINTF(("osdict: dict ele %d @%p\n", dents, dele));
+			if (primitive_type(dictvalue)) {
+			    dele->key = dstrtab;
+			    dele->key_size = dictkey->getLength() + 1;
+			    DPRINTF(("osdict: dictkey %s size %d @%p\n",
+				dictkey->getCStringNoCopy(), dictkey->getLength(), dstrtab));
+			    memcpy(dele->key, dictkey->getCStringNoCopy(),
+				dele->key_size);
+			    dele->value = dele->key + dele->key_size;
+			    primitive_type_collect(dele, dictvalue);
+			    dstrtab += dele->key_size + dele->value_size;
+			    DPRINTF(("osdict: dict new dstrtab %p\n", dstrtab));
+			}
+			else {
+			    continue;	/* Only handle primitive types here.  */
+			}
+			dents++;
+		    }
+		    dictIterator->release();
+		    if (dents == 0)
+			continue;
+		    arrayhd->type = MAC_DATA_TYPE_DICT;
+		    ele->value_size = sizeof(struct mac_module_data_list) +
+			(dents - 1) * sizeof(struct mac_module_data_element);
+		    DPRINTF(("osdict: dict ele size %d ents %d\n", ele->value_size, dents));
+		    dicttab += ele->value_size;
+		    DPRINTF(("osdict: new dicttab %p\n", dicttab));
+		    dicthd->count = dents;
+		    astrtab = dstrtab;
+		}
+		else {
+		    continue;		/* Skip everything else.              */
+		}
+		nents++;
+		ele++;
+	    }
+	    if (nents == 0)
+		continue;
+	    element->value_size = sizeof(struct mac_module_data_list) +
+		(nents - 1) * sizeof(struct mac_module_data_element);
+	    listtab += element->value_size;
+	    DPRINTF(("osdict: new listtab %p\n", listtab));
+	    arrayhd->count = nents;
+	    strtab = astrtab;
+	    DPRINTF(("osdict: new strtab %p\n", strtab));
+	}
+	else {
+	    continue;		/* skip anything else */
+	}
+	element++;
+    }
+    DPRINTF(("module_data list @%p, key %p value %p\n",
+	module_data, module_data->data[0].key, module_data->data[0].value));
+finish:
+    if (keyIterator)
+	keyIterator->release();
+    return(module_data);
+}
+
+/*********************************************************************
+* This function takes a plist and looks for an OSModuleData dictionary.
+* If it is found, an encoded copy is returned.
+*********************************************************************/
+kmod_args_t
+get_module_data(OSDictionary * kextPlist, mach_msg_type_number_t * datalen)
+{
+
+    OSDictionary * kextModuleData = 0;      // don't release
+    struct mac_module_data * module_data = 0;
+    vm_map_copy_t copy = 0;
+
+    kextModuleData = OSDynamicCast(OSDictionary,
+	kextPlist->getObject("OSModuleData"));
+    if (!kextModuleData)
+        goto finish;
+
+    module_data = osdict_encode(kextModuleData);
+    if (!module_data)
+        goto finish;
+    *datalen = module_data->size;
+    /*
+     * Make a CoW copy of data and free the original.  The copy is
+     * consumed by a call to vm_map_copyout() in kmod_start_or_stop().
+     */
+    vm_map_copyin(kernel_map, (vm_offset_t)module_data, *datalen, FALSE, &copy);
+    kmem_free(kernel_map, (vm_offset_t)module_data, *datalen);
+    DPRINTF(("get_module_data: copy @ %p\n", copy));
+finish:
+    return (kmod_args_t)copy;
+}
+#endif /* MAC */
+
 static 
 kern_return_t start_prelink_module(UInt32 moduleIndex)
 {
@@ -1056,7 +1473,7 @@ bool IOCatalogue::serializeData(IOOptionBits kind, OSSerialize * s) const
     switch ( kind )
     {
         case kIOCatalogGetContents:
-            if (!serialize(s))
+            if (!array->serialize(s))
                 kr = kIOReturnNoMemory;
             break;
 
@@ -1173,7 +1590,7 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
     kern_return_t result = KERN_SUCCESS;
     struct segment_command * segmentLE, *segmentKLD;
     boolean_t	keepsyms = FALSE;
-#if __ppc__
+#if __ppc__ || __arm__
     char * dt_segment_name;
     void * segment_paddress;
     int    segment_size;
@@ -1215,22 +1632,13 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
     */
     segmentKLD = getsegbyname("__KLD");
     if (!segmentKLD) {
-        IOLog("error removing kernel linker: can't find %s segment\n",
-            "__KLD");
+        IOLog("error removing kernel linker: can't find __KLD segment\n");
         result = KERN_FAILURE;
         goto finish;
     }
     OSRuntimeUnloadCPPForSegment(segmentKLD);
 
-    segmentLE = getsegbyname("__LINKEDIT");
-    if (!segmentLE) {
-        IOLog("error removing kernel linker: can't find %s segment\n",
-            "__LINKEDIT");
-        result = KERN_FAILURE;
-        goto finish;
-    }
-    OSRuntimeUnloadCPPForSegment(segmentLE);
-#if __ppc__
+#if __ppc__ || __arm__
    /* Free the memory that was set up by bootx.
     */
     dt_segment_name = "Kernel-__KLD";
@@ -1240,37 +1648,20 @@ kern_return_t IOCatalogue::removeKernelLinker(void) {
     }
 #elif __i386__
     /* On x86, use the mapping data from the segment load command to
-     * unload KLD and LINKEDIT directly, unless the keepsyms boot-arg
-     * was enabled. This may invalidate any assumptions about 
-     * "avail_start" defining the lower bound for valid physical addresses.
+     * unload KLD directly, unless the keepsyms boot-arg was enabled.
+     * This may invalidate any assumptions about  "avail_start"
+     * defining the lower bound for valid physical addresses.
      */
     if (!keepsyms && segmentKLD->vmaddr && segmentKLD->vmsize)
 	    ml_static_mfree(segmentKLD->vmaddr, segmentKLD->vmsize);
 #else
 #error arch
-#endif
-#if __ppc__
-    dt_segment_name = "Kernel-__LINKEDIT";
-    if (0 == IODTGetLoaderInfo(dt_segment_name, &segment_paddress, &segment_size)) {
-        IODTFreeLoaderInfo(dt_segment_name, (void *)segment_paddress,
-            (int)segment_size);
-    }
-#elif __i386__
-    if (!keepsyms && segmentLE->vmaddr && segmentLE->vmsize)
-	    ml_static_mfree(segmentLE->vmaddr, segmentLE->vmsize);
-#else
-#error arch
 #endif
 
     struct section * sect;
     sect = getsectbyname("__PRELINK", "__symtab");
-    if (sect && sect->addr)
-    {
-	vm_offset_t
-	virt = ml_static_ptovirt(sect->addr);
-	if( virt) {
-	    ml_static_mfree(virt, sect->size);
-	}
+    if (sect && sect->addr) {
+	ml_static_mfree(sect->addr, sect->size);
     }
 
 finish:
@@ -1298,3 +1689,9 @@ void IOCatalogue::disableExternalLinker(void) {
     IOLockUnlock(gIOKLDLock);
 }
 
+extern "C"
+void jettison_kernel_linker(void)
+{
+    if (gIOCatalogue != NULL)
+	gIOCatalogue->removeKernelLinker();
+}
diff --git a/iokit/Kernel/IOCommand.cpp b/iokit/Kernel/IOCommand.cpp
index 8d0c557d2..ac80d9da0 100644
--- a/iokit/Kernel/IOCommand.cpp
+++ b/iokit/Kernel/IOCommand.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/iokit/Kernel/IOCommandGate.cpp b/iokit/Kernel/IOCommandGate.cpp
index 37a5304c2..f276ebc22 100644
--- a/iokit/Kernel/IOCommandGate.cpp
+++ b/iokit/Kernel/IOCommandGate.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/OSDebug.h>
 
diff --git a/iokit/Kernel/IOCommandPool.cpp b/iokit/Kernel/IOCommandPool.cpp
index 145cac0a6..2a3b6e71b 100644
--- a/iokit/Kernel/IOCommandPool.cpp
+++ b/iokit/Kernel/IOCommandPool.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/iokit/Kernel/IOCommandQueue.cpp b/iokit/Kernel/IOCommandQueue.cpp
index 328c96190..88249acf6 100644
--- a/iokit/Kernel/IOCommandQueue.cpp
+++ b/iokit/Kernel/IOCommandQueue.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Kernel/IOConditionLock.cpp b/iokit/Kernel/IOConditionLock.cpp
index 7be2d21ba..9e2bd04cb 100644
--- a/iokit/Kernel/IOConditionLock.cpp
+++ b/iokit/Kernel/IOConditionLock.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* Copyright (c) 1997 Apple Computer, Inc.  All rights reserved.
  * Copyright (c) 1994-1996 NeXT Software, Inc.  All rights reserved. 
diff --git a/iokit/Kernel/IOCopyMapper.cpp b/iokit/Kernel/IOCopyMapper.cpp
index a808e04c4..4755e46cd 100644
--- a/iokit/Kernel/IOCopyMapper.cpp
+++ b/iokit/Kernel/IOCopyMapper.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 // 45678901234567890123456789012345678901234567890123456789012345678901234567890
 
@@ -208,7 +214,7 @@ ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages)
     // Can't alloc anything bigger than 1/2 table
     if (pages >= fMapperRegionSize/2)
     {
-	panic("iovmAlloc 0x%x", pages);
+	panic("iovmAlloc 0x%lx", pages);
 	return 0;
     }
 
@@ -242,7 +248,7 @@ ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages)
 
 	freeDART[ret].fInUse = true;	// Mark entry as In Use
 	next = freeDART[ret].fNext;
-	DEBG("va:  0x%x, %d, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next);
+	DEBG("va:  0x%lx, %ld, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next);
 
 	fFreeLists[z] = next;
 	if (next)
@@ -329,7 +335,7 @@ void IOCopyMapper::iovmFree(ppnum_t addr, IOItemCount pages)
 	    addr = pair;
     }
 
-    DEBG("vf:  0x%x, %d, z %d, head %x, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr);
+    DEBG("vf:  0x%lx, %ld, z %d, head %lx, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr);
 
     // Add the allocation entry into it's free list and re-init it
     freeDART[addr].fSize = z;
diff --git a/iokit/Kernel/IOCopyMapper.h b/iokit/Kernel/IOCopyMapper.h
index e2a5f5df7..5371afa04 100644
--- a/iokit/Kernel/IOCopyMapper.h
+++ b/iokit/Kernel/IOCopyMapper.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 // 45678901234567890123456789012345678901234567890123456789012345678901234567890
 
diff --git a/iokit/Kernel/IODMACommand.cpp b/iokit/Kernel/IODMACommand.cpp
index 2b79fb980..9aece35ff 100644
--- a/iokit/Kernel/IODMACommand.cpp
+++ b/iokit/Kernel/IODMACommand.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/assert.h>
@@ -55,30 +61,6 @@ enum
     kWalkClient       = 0x80
 };
 
-struct ExpansionData
-{
-    IOMDDMAWalkSegmentState fState;
-    IOMDDMACharacteristics  fMDSummary;
-
-    UInt64 fPreparedOffset;
-    UInt64 fPreparedLength;
-
-    UInt8  fCursor;
-    UInt8  fCheckAddressing;
-    UInt8  fIterateOnly;
-    UInt8  fMisaligned;
-    UInt8  fCopyContig;
-    UInt8  fPrepared;
-    UInt8  fDoubleBuffer;
-    UInt8  __pad[1];
-
-    ppnum_t  fCopyPageAlloc;
-    ppnum_t  fCopyPageCount;
-    addr64_t fCopyNext;
-
-    class IOBufferMemoryDescriptor * fCopyMD;
-};
-typedef ExpansionData IODMACommandInternal;
 
 #define fInternalState reserved
 #define fState         reserved->fState
@@ -110,8 +92,8 @@ typedef ExpansionData IODMACommandInternal;
 #define super OSObject
 OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
 
-OSMetaClassDefineReservedUnused(IODMACommand,  0);
-OSMetaClassDefineReservedUnused(IODMACommand,  1);
+OSMetaClassDefineReservedUsed(IODMACommand,  0);
+OSMetaClassDefineReservedUsed(IODMACommand,  1);
 OSMetaClassDefineReservedUnused(IODMACommand,  2);
 OSMetaClassDefineReservedUnused(IODMACommand,  3);
 OSMetaClassDefineReservedUnused(IODMACommand,  4);
@@ -222,10 +204,10 @@ IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
 	return false;
     };
 
-    reserved = IONew(ExpansionData, 1);
+    reserved = IONew(IODMACommandInternal, 1);
     if (!reserved)
 	return false;
-    bzero(reserved, sizeof(ExpansionData));
+    bzero(reserved, sizeof(IODMACommandInternal));
 
     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
     
@@ -236,7 +218,7 @@ void
 IODMACommand::free()
 {
     if (reserved)
-	IODelete(reserved, ExpansionData, 1);
+	IODelete(reserved, IODMACommandInternal, 1);
 
     super::free();
 }
@@ -430,6 +412,7 @@ IODMACommand::walkAll(UInt8 op)
 	state->fCopyNext       = 0;
 	state->fCopyPageAlloc  = 0;
 	state->fCopyPageCount  = 0;
+	state->fNextRemapIndex = 0;
 	state->fCopyMD         = 0;
 
 	if (!(kWalkDoubleBuffer & op))
@@ -550,11 +533,85 @@ IODMACommand::walkAll(UInt8 op)
     return (ret);
 }
 
+IOReturn
+IODMACommand::prepareWithSpecification(SegmentFunction	outSegFunc,
+				       UInt8		numAddressBits,
+				       UInt64		maxSegmentSize,
+				       MappingOptions	mappingOptions,
+				       UInt64		maxTransferSize,
+				       UInt32		alignment,
+				       IOMapper		*mapper,
+				       UInt64		offset,
+				       UInt64		length,
+				       bool		flushCache,
+				       bool		synchronize)
+{
+    if (fActive)
+        return kIOReturnNotPermitted;
+
+    if (!outSegFunc || !numAddressBits)
+        return kIOReturnBadArgument;
+
+    bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
+                 || OutputLittle32 == outSegFunc);
+    if (is32Bit)
+    {
+	if (!numAddressBits)
+	    numAddressBits = 32;
+	else if (numAddressBits > 32)
+	    return kIOReturnBadArgument;		// Wrong output function for bits
+    }
+
+    if (numAddressBits && (numAddressBits < PAGE_SHIFT))
+	return kIOReturnBadArgument;
+
+    if (!maxSegmentSize)
+	maxSegmentSize--;	// Set Max segment to -1
+    if (!maxTransferSize)
+	maxTransferSize--;	// Set Max transfer to -1
+
+    if (!mapper)
+    {
+        IOMapper::checkForSystemMapper();
+	mapper = IOMapper::gSystem;
+    }
+
+    switch (MAPTYPE(mappingOptions))
+    {
+    case kMapped:                   break;
+    case kNonCoherent: fMapper = 0; break;
+    case kBypassed:
+	if (mapper && !mapper->getBypassMask(&fBypassMask))
+	    return kIOReturnBadArgument;
+	break;
+    default:
+	return kIOReturnBadArgument;
+    };
+
+    fNumSegments     = 0;
+    fBypassMask      = 0;
+    fOutSeg	     = outSegFunc;
+    fNumAddressBits  = numAddressBits;
+    fMaxSegmentSize  = maxSegmentSize;
+    fMappingOptions  = mappingOptions;
+    fMaxTransferSize = maxTransferSize;
+    if (!alignment)
+	alignment = 1;
+    fAlignMask	     = alignment - 1;
+    fMapper          = mapper;
+
+    fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
+
+    return prepare(offset, length, flushCache, synchronize);
+}
+
+
 IOReturn 
 IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
 {
     IODMACommandInternal * state = fInternalState;
     IOReturn               ret   = kIOReturnSuccess;
+    MappingOptions mappingOptions    = fMappingOptions;
 
     if (!length)
 	length = fMDSummary.fLength;
@@ -562,13 +619,11 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr
     if (length > fMaxTransferSize)
 	return kIOReturnNoSpace;
 
-#if 0
     if (IS_NONCOHERENT(mappingOptions) && flushCache) {
 	IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 
 	poMD->performOperation(kIOMemoryIncoherentIOStore, 0, fMDSummary.fLength);
     }
-#endif
     if (fActive++)
     {
 	if ((state->fPreparedOffset != offset)
@@ -587,6 +642,7 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr
 	state->fCopyNext       = 0;
 	state->fCopyPageAlloc  = 0;
 	state->fCopyPageCount  = 0;
+	state->fNextRemapIndex = 0;
 	state->fCopyMD         = 0;
 
 	state->fCursor = state->fIterateOnly
@@ -619,22 +675,19 @@ IODMACommand::complete(bool invalidateCache, bool synchronize)
     {
 	if (!state->fCursor)
 	{
-	    IOOptionBits op = kWalkComplete;
-	    if (synchronize)
-		op |= kWalkSyncIn;
-	    ret = walkAll(op);
+		IOOptionBits op = kWalkComplete;
+		if (synchronize)
+			op |= kWalkSyncIn;
+		ret = walkAll(op);
 	}
 	state->fPrepared = false;
 
-#if 0
 	if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
 	{ 
-	    // XXX gvdl: need invalidate before Chardonnay ships
 	    IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 
-	    poMD->performOperation(kIOMemoryIncoherentIOInvalidate, 0, fMDSummary.fLength);
+	    poMD->performOperation(kIOMemoryIncoherentIOFlush, 0, fMDSummary.fLength);
 	}
-#endif
     }
 
     return ret;
@@ -678,6 +731,97 @@ IODMACommand::synchronize(IOOptionBits options)
     return ret;
 }
 
+struct IODMACommandTransferContext
+{
+    void *   buffer;
+    UInt64   bufferOffset;
+    UInt64   remaining;
+    UInt32   op;
+};
+enum
+{
+    kIODMACommandTransferOpReadBytes  = 1,
+    kIODMACommandTransferOpWriteBytes = 2
+};
+
+IOReturn
+IODMACommand::transferSegment(void   *reference,
+			IODMACommand *target,
+			Segment64     segment,
+			void         *segments,
+			UInt32        segmentIndex)
+{
+    IODMACommandTransferContext * context = (IODMACommandTransferContext *) segments;
+    UInt64   length  = min(segment.fLength, context->remaining);
+    addr64_t ioAddr  = segment.fIOVMAddr;
+    addr64_t cpuAddr = ioAddr;
+
+    context->remaining -= length;
+
+    while (length)
+    {
+	UInt64 copyLen = length;
+	if ((kMapped == MAPTYPE(target->fMappingOptions))
+	    && target->fMapper)
+	{
+	    cpuAddr = target->fMapper->mapAddr(ioAddr);
+	    copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
+	    ioAddr += copyLen;
+	}
+
+	switch (context->op)
+	{
+	    case kIODMACommandTransferOpReadBytes:
+		copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen,
+				    cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
+		break;
+	    case kIODMACommandTransferOpWriteBytes:
+		copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen,
+				cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+		break;
+	}
+	length                -= copyLen;
+	context->bufferOffset += copyLen;
+    }
+    
+    return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun);
+}
+
+UInt64
+IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
+{
+    IODMACommandInternal *      state = fInternalState;
+    IODMACommandTransferContext context;
+    UInt32                      numSegments = 0-1;
+
+    if (fActive < 1)
+        return (0);
+
+    if (offset >= state->fPreparedLength)
+        return (0);
+    length = min(length, state->fPreparedLength - offset);
+
+    context.buffer       = buffer;
+    context.bufferOffset = 0;
+    context.remaining    = length;
+    context.op           = transferOp;
+    (void) genIOVMSegments(transferSegment, (void *) kWalkClient, &offset, &context, &numSegments);
+
+    return (length - context.remaining);
+}
+
+UInt64
+IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
+{
+    return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length));
+}
+
+UInt64
+IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
+{
+    return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length));
+}
+
 IOReturn
 IODMACommand::genIOVMSegments(UInt64 *offsetP,
 			      void   *segmentsP,
@@ -707,17 +851,18 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc,
     IOMDDMAWalkSegmentArgs *state =
 	(IOMDDMAWalkSegmentArgs *) fState;
 
-    UInt64 offset = *offsetP + internalState->fPreparedOffset;
+    UInt64 offset    = *offsetP + internalState->fPreparedOffset;
     UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
 
     if (offset >= memLength)
 	return kIOReturnOverrun;
 
-    if (!offset || offset != state->fOffset) {
-	state->fOffset   = 0;
-	state->fIOVMAddr = 0;
-	state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper);
-	mdOp = kIOMDFirstSegment;
+    if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset)) {
+	state->fOffset                 = 0;
+	state->fIOVMAddr               = 0;
+	internalState->fNextRemapIndex = 0;
+	state->fMapped                 = (IS_MAPPED(fMappingOptions) && fMapper);
+	mdOp                           = kIOMDFirstSegment;
     };
 	
     UInt64    bypassMask = fBypassMask;
@@ -808,13 +953,22 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc,
 		    else if (gIOCopyMapper)
 		    {
 			DEBG("sparse switch %qx, %qx ", curSeg.fIOVMAddr, curSeg.fLength);
-			// Cache this!
-			for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++)
+			if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
+							    ptoa_64(internalState->fCopyPageAlloc + internalState->fNextRemapIndex)))
+			{
+
+			    curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + internalState->fNextRemapIndex)
+						+ (curSeg.fIOVMAddr & PAGE_MASK);
+			    internalState->fNextRemapIndex += atop_64(round_page(curSeg.fLength));
+			}
+			else for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++)
 			{
 			    if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
 							    ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex)))
 			    {
-				curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex) + (curSeg.fIOVMAddr & PAGE_MASK);
+				curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex) 
+						    + (curSeg.fIOVMAddr & PAGE_MASK);
+				internalState->fNextRemapIndex = checkRemapIndex + atop_64(round_page(curSeg.fLength));
 				break;
 			    }
 			}
diff --git a/iokit/Kernel/IODMAController.cpp b/iokit/Kernel/IODMAController.cpp
new file mode 100644
index 000000000..4f81342d4
--- /dev/null
+++ b/iokit/Kernel/IODMAController.cpp
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/IODMAController.h>
+
+
+#define super IOService
+OSDefineMetaClassAndAbstractStructors(IODMAController, IOService);
+
+const OSSymbol *IODMAController::createControllerName(UInt32 phandle)
+{
+#define CREATE_BUF_LEN 48
+  char           buf[CREATE_BUF_LEN];
+  
+  snprintf(buf, CREATE_BUF_LEN, "IODMAController%08lX", phandle);
+  
+  return OSSymbol::withCString(buf);
+}
+
+IODMAController *IODMAController::getController(IOService *provider, UInt32 dmaIndex)
+{
+  OSData          *dmaParentData;
+  const OSSymbol  *dmaParentName;
+  IODMAController *dmaController;
+  
+  // Find the name of the parent dma controller
+  dmaParentData = OSDynamicCast(OSData, provider->getProperty("dma-parent"));
+  if (dmaParentData == 0) return false;
+  dmaParentName = createControllerName(*(UInt32 *)dmaParentData->getBytesNoCopy());
+  if (dmaParentName == 0) return false;
+  
+  // Wait for the parent dma controller
+  dmaController = OSDynamicCast(IODMAController, IOService::waitForService(IOService::nameMatching(dmaParentName)));
+  
+  return dmaController;
+}
+
+
+bool IODMAController::start(IOService *provider)
+{
+  if (!super::start(provider)) return false;
+  
+  _provider = provider;
+  
+  return true;
+}
+
+
+// protected
+
+void IODMAController::registerDMAController(IOOptionBits options)
+{
+  OSData *phandleData;
+  
+  phandleData = OSDynamicCast(OSData, _provider->getProperty("AAPL,phandle"));
+  
+  _dmaControllerName = createControllerName(*(UInt32 *)phandleData->getBytesNoCopy());
+  
+  setName(_dmaControllerName);
+  
+  registerService(options | ((options & kIOServiceAsynchronous) ? 0 : kIOServiceSynchronous));
+}
+
+void IODMAController::completeDMACommand(IODMAEventSource *dmaES, IODMACommand *dmaCommand)
+{
+  dmaES->completeDMACommand(dmaCommand);
+}
+
+void IODMAController::notifyDMACommand(IODMAEventSource *dmaES, IODMACommand *dmaCommand, IOReturn status, IOByteCount actualByteCount)
+{
+  dmaES->notifyDMACommand(dmaCommand, status, actualByteCount);
+}
+
+
+// private
diff --git a/iokit/Kernel/IODMAEventSource.cpp b/iokit/Kernel/IODMAEventSource.cpp
new file mode 100644
index 000000000..eeaf70c8a
--- /dev/null
+++ b/iokit/Kernel/IODMAEventSource.cpp
@@ -0,0 +1,166 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/IODMAEventSource.h>
+#include <IOKit/IOService.h>
+
+#include "IOKitKernelInternal.h"
+
+
+#define super IOEventSource
+OSDefineMetaClassAndStructors(IODMAEventSource, IOEventSource);
+
+bool IODMAEventSource::init(OSObject *inOwner,
+			    IOService *inProvider,
+			    Action inCompletion,
+			    Action inNotification,
+			    UInt32 inDMAIndex)
+{
+  IOReturn result;
+  
+  if (!super::init(inOwner)) return false;
+  
+  if (inProvider == 0) return false;
+  
+  dmaProvider = inProvider;
+  dmaIndex = 0xFFFFFFFF;
+  dmaCompletionAction = inCompletion;
+  dmaNotificationAction = inNotification;
+  
+  dmaController = IODMAController::getController(dmaProvider, inDMAIndex);
+  if (dmaController == 0) return false;
+  dmaController->retain();
+  
+  result = dmaController->initDMAChannel(dmaProvider, this, &dmaIndex, inDMAIndex);
+  if (result != kIOReturnSuccess) return false;
+  
+  queue_init(&dmaCommandsCompleted);
+  dmaCommandsCompletedLock = IOSimpleLockAlloc();
+  
+  return true;
+}
+
+IODMAEventSource *IODMAEventSource::dmaEventSource(OSObject *inOwner,
+						   IOService *inProvider,
+						   Action inCompletion,
+						   Action inNotification,
+						   UInt32 inDMAIndex)
+{
+  IODMAEventSource *dmaES = new IODMAEventSource;
+  
+  if (dmaES && !dmaES->init(inOwner, inProvider, inCompletion, inNotification, inDMAIndex)) {
+    dmaES->release();
+    return 0;
+  }
+  
+  return dmaES;
+}
+
+IOReturn IODMAEventSource::startDMACommand(IODMACommand *dmaCommand, IODirection direction, IOByteCount byteCount, IOByteCount byteOffset)
+{
+  IOReturn result;
+  
+  if ((dmaController == 0) || (dmaIndex == 0xFFFFFFFF)) return kIOReturnError;
+  
+  if (dmaSynchBusy) return kIOReturnBusy;
+  
+  if (dmaCompletionAction == 0) dmaSynchBusy = true;
+  
+  result = dmaController->startDMACommand(dmaIndex, dmaCommand, direction, byteCount, byteOffset);
+  
+  if (result != kIOReturnSuccess) {
+    dmaSynchBusy = false;
+    return result;
+  }
+  
+  while (dmaSynchBusy) sleepGate(&dmaSynchBusy, THREAD_UNINT);
+  
+  return kIOReturnSuccess;
+}
+
+IOReturn IODMAEventSource::stopDMACommand(bool flush, mach_timespec_t *timeout)
+{
+  if ((dmaController == 0) || (dmaIndex == 0xFFFFFFFF)) return kIOReturnError;
+  
+  return dmaController->stopDMACommand(dmaIndex, flush, timeout);
+}
+
+
+IOReturn IODMAEventSource::queryDMACommand(IODMACommand **dmaCommand, IOByteCount *transferCount, bool waitForIdle)
+{
+  if ((dmaController == 0) || (dmaIndex == 0xFFFFFFFF)) return kIOReturnError;
+  
+  return dmaController->queryDMACommand(dmaIndex, dmaCommand, transferCount, waitForIdle);
+}
+
+
+// protected
+
+bool IODMAEventSource::checkForWork(void)
+{
+  IODMACommand     *dmaCommand = NULL;
+  bool work, again;
+  
+  IOSimpleLockLock(dmaCommandsCompletedLock);
+  work = !queue_empty(&dmaCommandsCompleted);
+  if (work) {
+    queue_remove_first(&dmaCommandsCompleted, dmaCommand, IODMACommand *, fCommandChain);
+    again = !queue_empty(&dmaCommandsCompleted);
+  } else {
+    again = false;
+  }
+  IOSimpleLockUnlock(dmaCommandsCompletedLock);
+
+  if (work) {
+    (*dmaCompletionAction)(owner, this, dmaCommand, dmaCommand->reserved->fStatus, dmaCommand->reserved->fActualByteCount);
+  }
+  
+  return again;
+}
+
+void IODMAEventSource::completeDMACommand(IODMACommand *dmaCommand)
+{
+  if (dmaCompletionAction != 0) {
+    IOSimpleLockLock(dmaCommandsCompletedLock);
+    queue_enter(&dmaCommandsCompleted, dmaCommand, IODMACommand *, fCommandChain);
+    IOSimpleLockUnlock(dmaCommandsCompletedLock);
+    
+    signalWorkAvailable();
+  } else {
+    dmaSynchBusy = false;
+    wakeupGate(&dmaSynchBusy, true);
+  }
+}
+
+void IODMAEventSource::notifyDMACommand(IODMACommand *dmaCommand, IOReturn status, IOByteCount actualByteCount)
+{
+  dmaCommand->reserved->fStatus = status;
+  dmaCommand->reserved->fActualByteCount = actualByteCount;  
+  
+  if (dmaNotificationAction != 0) (*dmaNotificationAction)(owner, this, dmaCommand, status, actualByteCount);
+}
diff --git a/iokit/Kernel/IODataQueue.cpp b/iokit/Kernel/IODataQueue.cpp
index c0ea4b9e6..3a7f4c832 100644
--- a/iokit/Kernel/IODataQueue.cpp
+++ b/iokit/Kernel/IODataQueue.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IODataQueue.h>
@@ -76,9 +82,9 @@ Boolean IODataQueue::initWithCapacity(UInt32 size)
         return false;
     }
 
-    dataQueue->queueSize = size;
-    dataQueue->head = 0;
-    dataQueue->tail = 0;
+    dataQueue->queueSize    = size;
+    dataQueue->head         = 0;
+    dataQueue->tail         = 0;
 
     return true;
 }
@@ -202,7 +208,7 @@ void IODataQueue::sendDataAvailableNotification()
     mach_msg_header_t *	msgh;
 
     msgh = (mach_msg_header_t *)notifyMsg;
-    if (msgh) {
+    if (msgh && msgh->msgh_remote_port) {
         kr = mach_msg_send_from_kernel(msgh, msgh->msgh_size);
         switch(kr) {
             case MACH_SEND_TIMED_OUT:	// Notification already sent
diff --git a/iokit/Kernel/IODeviceMemory.cpp b/iokit/Kernel/IODeviceMemory.cpp
index a1357a49a..d379e1755 100644
--- a/iokit/Kernel/IODeviceMemory.cpp
+++ b/iokit/Kernel/IODeviceMemory.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IODeviceMemory.h>
diff --git a/iokit/Kernel/IODeviceTreeSupport.cpp b/iokit/Kernel/IODeviceTreeSupport.cpp
index 5c3afed44..4b6a1fdf2 100644
--- a/iokit/Kernel/IODeviceTreeSupport.cpp
+++ b/iokit/Kernel/IODeviceTreeSupport.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IODeviceTreeSupport.h>
@@ -88,7 +94,7 @@ IODeviceTreeAlloc( void * dtTop )
     OSObject *			obj;
     OSDictionary *		allInts;
     vm_offset_t *		dtMap;
-    int				propSize;
+    unsigned int		propSize;
     bool			intMap;
     bool			freeDT;
 
@@ -138,7 +144,7 @@ IODeviceTreeAlloc( void * dtTop )
     freeDT = (kSuccess == DTLookupEntry( 0, "/chosen/memory-map", &mapEntry ))
 	  && (kSuccess == DTGetProperty( mapEntry,
                 "DeviceTree", (void **) &dtMap, &propSize ))
-	  && ((2 * sizeof( vm_offset_t)) == propSize);
+	  && ((2 * sizeof(vm_offset_t)) == propSize);
 
     parent = MakeReferenceTable( (DTEntry)dtTop, freeDT );
 
@@ -216,7 +222,7 @@ IODeviceTreeAlloc( void * dtTop )
             if( (obj = child->getProperty( "driver,AAPL,MacOS,PowerPC"))) {
 
                 if( (0 == (prop = (OSData *)child->getProperty( gIODTTypeKey )))
-                  || (strcmp( "display", (char *) prop->getBytesNoCopy())) ) {
+                  || (strncmp("display", (char *)prop->getBytesNoCopy(), sizeof("display"))) ) {
                     child->removeProperty( "driver,AAPL,MacOS,PowerPC");
                 }
             }
@@ -322,7 +328,7 @@ MakeReferenceTable( DTEntry dtEntry, bool copy )
     const OSSymbol		*sym;
     DTPropertyIterator	dtIter;
     void				*prop;
-    int					propSize;
+    unsigned int		propSize;
     char				*name;
     char				location[ 32 ];
     bool				noLocation = true;
@@ -367,17 +373,17 @@ MakeReferenceTable( DTEntry dtEntry, bool copy )
 
             } else if( nameKey == gIODTUnitKey ) {
                 // all OF strings are null terminated... except this one
-                if( propSize >= (int) sizeof( location))
-                    propSize = sizeof( location) - 1;
+                if( propSize >= (int) sizeof(location))
+                    propSize = sizeof(location) - 1;
                 strncpy( location, (const char *) prop, propSize );
                 location[ propSize ] = 0;
                 regEntry->setLocation( location );
                 propTable->removeObject( gIODTUnitKey );
                 noLocation = false;
     
-            } else if( noLocation && (0 == strcmp( name, "reg"))) {
+            } else if(noLocation && (!strncmp(name, "reg", sizeof("reg")))) {
                 // default location - override later
-                sprintf( location, "%lX", *((UInt32 *) prop) );
+                snprintf(location, sizeof(location), "%lX", *((UInt32 *) prop));
                 regEntry->setLocation( location );
             }
         }
@@ -456,7 +462,7 @@ const OSSymbol * IODTInterruptControllerName( IORegistryEntry * regEntry )
     assert( ok );
 
     if( ok) {
-        sprintf( buf, "IOInterruptController%08lX", phandle);
+        snprintf(buf, sizeof(buf), "IOInterruptController%08lX", phandle);
         sym = OSSymbol::withCString( buf );
     } else
         sym = 0;
@@ -493,7 +499,7 @@ UInt32 IODTMapOneInterrupt( IORegistryEntry * regEntry, UInt32 * intSpec,
     addrCmp = 0;
     if( acells) {
         data = OSDynamicCast( OSData, regEntry->getProperty( "reg" ));
-        if( data && (data->getLength() >= (acells * sizeof( UInt32))))
+        if( data && (data->getLength() >= (acells * sizeof(UInt32))))
             addrCmp = (UInt32 *) data->getBytesNoCopy();
     }
     original_icells = icells;
@@ -514,7 +520,7 @@ UInt32 IODTMapOneInterrupt( IORegistryEntry * regEntry, UInt32 * intSpec,
             // found a controller - don't want to follow cascaded controllers
             parent = 0;
             *spec = OSData::withBytesNoCopy( (void *) intSpec,
-                                            icells * sizeof( UInt32));
+                                            icells * sizeof(UInt32));
             *controller = IODTInterruptControllerName( regEntry );
             ok = (*spec && *controller);
         } else if( parent && (data = OSDynamicCast( OSData,
@@ -523,7 +529,7 @@ UInt32 IODTMapOneInterrupt( IORegistryEntry * regEntry, UInt32 * intSpec,
             map = (UInt32 *) data->getBytesNoCopy();
             endMap = map + (data->getLength() / sizeof(UInt32));
             data = OSDynamicCast( OSData, regEntry->getProperty( "interrupt-map-mask" ));
-            if( data && (data->getLength() >= ((acells + icells) * sizeof( UInt32))))
+            if( data && (data->getLength() >= ((acells + icells) * sizeof(UInt32))))
                 maskCmp = (UInt32 *) data->getBytesNoCopy();
             else
                 maskCmp = 0;
@@ -659,7 +665,7 @@ static bool IODTMapInterruptsSharing( IORegistryEntry * regEntry, OSDictionary *
     }
 
     localBits = (UInt32 *) local->getBytesNoCopy();
-    localEnd = localBits + (local->getLength() / sizeof( UInt32));
+    localEnd = localBits + (local->getLength() / sizeof(UInt32));
     mapped = OSArray::withCapacity( 1 );
     controllers = OSArray::withCapacity( 1 );
 
@@ -674,8 +680,8 @@ static bool IODTMapInterruptsSharing( IORegistryEntry * regEntry, OSDictionary *
                 break;
             }
         } else {
-            map = OSData::withData( local, mapped->getCount() * sizeof( UInt32),
-				sizeof( UInt32));
+            map = OSData::withData( local, mapped->getCount() * sizeof(UInt32),
+				sizeof(UInt32));
             controller = gIODTDefaultInterruptController;
             controller->retain();
         }
@@ -901,7 +907,7 @@ void IODTSetResolving( IORegistryEntry * 	regEntry,
 
     persist.compareFunc = compareFunc;
     persist.locationFunc = locationFunc;
-    prop = OSData::withBytes( &persist, sizeof( persist));
+    prop = OSData::withBytes( &persist, sizeof(persist));
     if( !prop)
         return;
 
@@ -950,7 +956,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry,
     UInt32		*startRange;
     UInt32		*endRanges;
     bool		ok = true;
-    SInt32		diff, endDiff;
+    SInt32		diff, diff2, endDiff;
 
     IODTPersistent	*persist;
     IODTCompareAddressCellFunc	compare;
@@ -958,7 +964,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry,
     IODTGetCellCounts( regEntry, &childSizeCells, &childAddressCells );
     childCells = childAddressCells + childSizeCells;
 
-    bcopy( cellsIn, cell, 4 * childCells );
+    bcopy( cellsIn, cell, sizeof(UInt32) * childCells );
     if( childSizeCells > 1)
         *len = IOPhysical32( cellsIn[ childAddressCells ],
                              cellsIn[ childAddressCells + 1 ] );
@@ -981,7 +987,7 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry,
 	    // search
 	    startRange = (UInt32 *) prop->getBytesNoCopy();
 	    range = startRange;
-	    endRanges = range + (length / 4);
+	    endRanges = range + (length / sizeof(UInt32));
 
 	    prop = (OSData *) regEntry->getProperty( gIODTPersistKey );
 	    if( prop) {
@@ -994,9 +1000,14 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry,
 		 range < endRanges;
 		 range += (childCells + addressCells) ) {
 
-		// is cell start >= range start?
+		// is cell start within range?
 		diff = (*compare)( childAddressCells, cell, range );
-		if( diff < 0)
+
+		bcopy(range, endCell, childAddressCells * sizeof(UInt32));
+		endCell[childAddressCells - 1] += range[childCells + addressCells - 1];
+		diff2 = (*compare)( childAddressCells, cell, endCell );
+
+		if ((diff < 0) || (diff2 >= 0))
 		    continue;
 
 		ok = (0 == cell[childCells - 1]);
@@ -1029,8 +1040,8 @@ bool IODTResolveAddressCell( IORegistryEntry * regEntry,
 	    }
 
 	    // Get the physical start of the range from our parent
-	    bcopy( range + childAddressCells, cell, 4 * addressCells );
-	    bzero( cell + addressCells, 4 * sizeCells );
+	    bcopy( range + childAddressCells, cell, sizeof(UInt32) * addressCells );
+	    bzero( cell + addressCells, sizeof(UInt32) * sizeCells );
 
 	} /* else zero length range => pass thru to parent */
 
diff --git a/iokit/Kernel/IOEventSource.cpp b/iokit/Kernel/IOEventSource.cpp
index 8c469db09..0e0dbc8f8 100644
--- a/iokit/Kernel/IOEventSource.cpp
+++ b/iokit/Kernel/IOEventSource.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Kernel/IOFilterInterruptEventSource.cpp b/iokit/Kernel/IOFilterInterruptEventSource.cpp
index eb818ccec..9e268f6e7 100644
--- a/iokit/Kernel/IOFilterInterruptEventSource.cpp
+++ b/iokit/Kernel/IOFilterInterruptEventSource.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/iokit/Kernel/IOHibernateIO.cpp b/iokit/Kernel/IOHibernateIO.cpp
index e98692e32..58d4836c4 100644
--- a/iokit/Kernel/IOHibernateIO.cpp
+++ b/iokit/Kernel/IOHibernateIO.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -462,28 +468,39 @@ IOHibernatePollerIO(IOPolledFileIOVars * vars,
 }
 
 static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
+IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
 {
-    IOReturn            err = kIOReturnError;
-    int32_t		idx;
+    IOReturn            err = kIOReturnSuccess;
+    int32_t		idx = 0;
     IOPolledInterface * poller;
 
     while (-1 == vars->ioStatus)
     {
-        for (idx = 0;
-             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+        for (idx = 0; 
+	    (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
              idx++)
         {
-            err = poller->checkForWork();
-            if (err)
-                HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+	    IOReturn newErr;
+            newErr = poller->checkForWork();
+	    if ((newErr == kIOReturnAborted) && !abortable)
+		newErr = kIOReturnSuccess;
+	    if (kIOReturnSuccess == err)
+		err = newErr;
         }
     }
 
-    if (kIOReturnSuccess != vars->ioStatus)
-        HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
+    if (err)
+    {
+	HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+    }
+    else 
+    {
+	err = vars->ioStatus;
+	if (kIOReturnSuccess != err)
+	    HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
+    }
 
-    return (vars->ioStatus);
+    return (err);
 }
 
 IOReturn
@@ -602,7 +619,9 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 	    part->retain();
 	    iter->release();
 	}
-    
+    	if (!part)
+	    break;
+
 	int minor, major;
 	IORegistryEntry * next;
 	IORegistryEntry * child;
@@ -671,7 +690,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 	    if (!gIOCreateEFIDevicePathSymbol)
 		gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
 
-	    sprintf(str2, "%qx", vars->extentMap[0]);
+	    snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 
 	    err = IOService::getPlatform()->callPlatformFunction(
 						gIOCreateEFIDevicePathSymbol, false,
@@ -685,7 +704,7 @@ IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
 		err = kIOReturnNotFound;
 	    else
 	    {
-		sprintf(str2, ",%qx", vars->extentMap[0]);
+		snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
 		// (strip the plane name)
 		char * tail = strchr(str1, ':');
 		if (!tail)
@@ -804,6 +823,7 @@ IOPolledFileWrite(IOPolledFileIOVars * vars,
 				- vars->extentPosition + vars->currentExtent->start);
 	    uint32_t length = (vars->bufferOffset);
 
+#if CRYPTO
             if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
             {
                 uint32_t encryptLen, encryptStart;
@@ -823,10 +843,11 @@ IOPolledFileWrite(IOPolledFileIOVars * vars,
                         &cryptvars->aes_iv[0],
                         AES_BLOCK_SIZE);
             }
+#endif /* CRYPTO */
 
 	    if (vars->io)
             {
-		err = IOHibernatePollerIODone(vars);
+		err = IOHibernatePollerIODone(vars, true);
                 if (kIOReturnSuccess != err)
                     break;
             }
@@ -896,7 +917,7 @@ IOPolledFileRead(IOPolledFileIOVars * vars,
 	{
 	    if (vars->io)
             {
-		err = IOHibernatePollerIODone(vars);
+		err = IOHibernatePollerIODone(vars, false);
                 if (kIOReturnSuccess != err)
                     break;
             }
@@ -940,6 +961,7 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
 	    vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 	    vars->bufferOffset = 0;
 
+#if CRYPTO
             if (cryptvars)
             {
                 uint8_t thisVector[AES_BLOCK_SIZE];
@@ -954,6 +976,7 @@ if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n",
                                 vars->buffer + vars->bufferHalf,
                                 &cryptvars->ctx.decrypt);
             }
+#endif CRYPTO
 	}
     }
     while (size);
@@ -971,6 +994,7 @@ IOHibernateSystemSleep(void)
     OSObject * obj;
     OSString * str;
     OSNumber * num;
+    OSDictionary *sleepOverrideOptions;
 
     IOHibernateVars * vars  = &gIOHibernateVars;
 
@@ -980,23 +1004,53 @@ IOHibernateSystemSleep(void)
 
     gIOHibernateState = kIOHibernateStateInactive;
 
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
+    /* The invocation of IOPMSleepSystemWithOptions() may override
+     * existing hibernation settings.
+     */
+    sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
+        IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
+
+
+    /* Hibernate mode overriden by sleep otions ? */
+    obj = NULL;
+    
+    if (sleepOverrideOptions) {
+        obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
+        if (obj) obj->retain();
+    }
+
+    if(!obj) {
+        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
+    }
+
+    if (obj && (num = OSDynamicCast(OSNumber, obj)) )
     {
-	if ((num = OSDynamicCast(OSNumber, obj)))
 	    gIOHibernateMode = num->unsigned32BitValue();
         if (kIOHibernateModeSleep & gIOHibernateMode)
             // default to discard clean for safe sleep
             gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive 
                                 | kIOHibernateModeDiscardCleanActive);
+    }
 
-	obj->release();
+    if (obj) obj->release();
+    
+    /* Hibernate free rotio overriden by sleep options ? */
+    obj = NULL;
+    
+    if (sleepOverrideOptions) {
+        obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
+        if (obj) obj->retain();
     }
-    if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
+    
+    if(!obj) {
+        obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
+    }
+    if (obj && (num = OSDynamicCast(OSNumber, obj)))
     {
-	if ((num = OSDynamicCast(OSNumber, obj)))
 	    gIOHibernateFreeRatio = num->unsigned32BitValue();
-	obj->release();
     }
+    if (obj) obj->release();
+    
     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
     {
 	if ((num = OSDynamicCast(OSNumber, obj)))
@@ -1006,15 +1060,20 @@ IOHibernateSystemSleep(void)
     if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
     {
 	if ((str = OSDynamicCast(OSString, obj)))
-	    strcpy(gIOHibernateFilename, str->getCStringNoCopy());
+	    strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+			    sizeof(gIOHibernateFilename));
 	obj->release();
     }
 
+    if (sleepOverrideOptions)
+        sleepOverrideOptions->release();
+
     if (!gIOHibernateMode || !gIOHibernateFilename[0])
 	return (kIOReturnUnsupported);
 
     HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
 
+
     do
     {
         vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
@@ -1173,9 +1232,24 @@ IOHibernateSystemSleep(void)
 	    }
 	    data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
 	    if (data)
-	    {
-		IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
-		data->release();
+	    { 
+			IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
+		
+			if( gIOOptionsEntry )
+			{
+				if( gIOHibernateMode & kIOHibernateModeSwitch )
+				{
+					const OSSymbol *sym;
+					sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
+					if( sym )
+					{
+						gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
+						sym->release();
+					}
+				}	
+			}
+
+			data->release();
 	    }
             if (gIOChosenEntry)
             {
@@ -1194,7 +1268,7 @@ IOHibernateSystemSleep(void)
                     sym->release();
                 if (data)
                     data->release();
-                if (gIOHibernateBootSignature[0])
+                if (false && gIOHibernateBootSignature[0])
                 {
                     data = OSData::withCapacity(16);
                     sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
@@ -1251,27 +1325,173 @@ IOHibernateSystemSleep(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+DECLARE_IOHIBERNATEPROGRESSALPHA
+
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
+{
+    uint32_t	rowBytes, pixelShift;
+    uint32_t	x, y;
+    int32_t	blob;
+    uint32_t	alpha, in, color, result;
+    uint8_t *	out;
+    uint32_t	saveindex[kIOHibernateProgressCount] = { 0 };
+
+    rowBytes = display->rowBytes;
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1) return;
+    
+    screen += ((display->width
+                - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+        + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+    
+    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    {
+        out = screen + y * rowBytes;
+        for (blob = 0; blob < kIOHibernateProgressCount; blob++)
+        {
+            color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)
+            {
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        if (1 == pixelShift)
+                        {
+                            in = *((uint16_t *)out) & 0x1f;	// 16
+                            in = (in << 3) | (in >> 2);
+                        }
+                        else
+                            in = *((uint32_t *)out) & 0xff;	// 32
+                        saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+                        result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;	// 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;	// 32
+                }
+                out += (1 << pixelShift);
+            }
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }
+    }
+}
+
+
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
+{
+    uint32_t  rowBytes, pixelShift;
+    uint32_t  x, y;
+    int32_t   blob, lastBlob;
+    uint32_t  alpha, in, color, result;
+    uint8_t * out;
+    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
+
+    pixelShift = display->depth >> 4;
+    if (pixelShift < 1)
+        return;
+
+    rowBytes = display->rowBytes;
+
+    screen += ((display->width 
+            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
+
+    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+
+    for (y = 0; y < kIOHibernateProgressHeight; y++)
+    {
+        out = screen + y * rowBytes;
+        for (blob = firstBlob; blob <= lastBlob; blob++)
+        {
+            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+            for (x = 0; x < kIOHibernateProgressWidth; x++)
+            {
+                alpha  = gIOHibernateProgressAlpha[y][x];
+                result = color;
+                if (alpha)
+                {
+                    if (0xff != alpha)
+                    {
+                        in = display->progressSaveUnder[blob][saveindex[blob]++];
+                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+                    }
+                    if (1 == pixelShift)
+                    {
+                        result >>= 3;
+                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;	// 16
+                    }
+                    else
+                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;	// 32
+                }
+                out += (1 << pixelShift);
+            }
+            out += (kIOHibernateProgressSpacing << pixelShift);
+        }
+    }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 IOReturn
 IOHibernateSystemHasSlept(void)
 {
     IOHibernateVars * vars  = &gIOHibernateVars;
+    OSObject        * obj;
+    OSData          * data;
+
+    obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+    vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+    if (obj && !vars->previewBuffer)
+	obj->release();
 
-    if ((vars->previewData = OSDynamicCast(OSData, 
-            IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
+    vars->consoleMapping = NULL;
+    if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
     {
-        vars->previewBuffer = IOMemoryDescriptor::withAddress(
-                                    (void *) vars->previewData->getBytesNoCopy(), 
-                                    vars->previewData->getLength(), 
-                                    kIODirectionInOut);
+	vars->previewBuffer->release();
+	vars->previewBuffer = 0;
+    }
 
-        if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
-        {
-            vars->previewBuffer->release();
-            vars->previewBuffer = 0;
-        }
-        if (!vars->previewBuffer)
-            vars->previewData = 0;
+    if (vars->previewBuffer && (data = OSDynamicCast(OSData, 
+	IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
+    {
+	UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+	HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
+
+	IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
+
+	if (kIOHibernatePreviewUpdates & flags)
+	{
+	    PE_Video	       consoleInfo;
+	    hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+	    IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+	    graphicsInfo->width    = consoleInfo.v_width;
+	    graphicsInfo->height   = consoleInfo.v_height;
+	    graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
+	    graphicsInfo->depth    = consoleInfo.v_depth;
+	    vars->consoleMapping   = (uint8_t *) consoleInfo.v_baseAddr;
+
+	    HIBPRINT("video %p %d %d %d\n",
+			vars->consoleMapping, gIOHibernateGraphicsInfo->depth, 
+			gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
+	    if (vars->consoleMapping)
+			ProgressInit(graphicsInfo, vars->consoleMapping,
+					&graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
+	}
     }
+
     if (gIOOptionsEntry)
         gIOOptionsEntry->sync();
 
@@ -1367,20 +1587,20 @@ IOHibernateSystemWake(void)
 #endif
 
 #ifdef __i386__
-    IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
-	
-    /*
-     * Hibernate variable is written to NVRAM on platforms in which RtcRam
-     * is not backed by coin cell.  Remove Hibernate data from NVRAM.
-     */
-    if (gIOOptionsEntry) {
-        const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
-		
-        if (sym) {
-            gIOOptionsEntry->removeProperty(sym);
-            sym->release();
-        }
-    }
+	IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
+
+	/*
+	 * Hibernate variable is written to NVRAM on platforms in which RtcRam
+	 * is not backed by coin cell.  Remove Hibernate data from NVRAM.
+	 */
+	if (gIOOptionsEntry) {
+		const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+
+		if (sym) {
+			gIOOptionsEntry->removeProperty(sym);
+			sym->release();
+		}
+	}
 #endif
 
     if (vars->srcBuffer)
@@ -1417,6 +1637,16 @@ IOHibernateSystemPostWake(void)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
+		CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+		gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
+		CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+		gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
+		CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+		&gIOHibernateMode, 0, "");
+
 void
 IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 {
@@ -1427,27 +1657,17 @@ IOHibernateSystemInit(IOPMrootDomain * rootDomain)
 	data->release();
     }
 
-    if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
+    if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
 	gIOHibernateMode = kIOHibernateModeOn;
     else
 	gIOHibernateFilename[0] = 0;
 
-    static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile, 
-				CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-				gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
     sysctl_register_oid(&sysctl__kern_hibernatefile);
-
-    static SYSCTL_STRING(_kern, OID_AUTO, bootsignature, 
-				CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-				gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
     sysctl_register_oid(&sysctl__kern_bootsignature);
-
-    static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode, 
-				CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
-				&gIOHibernateMode, 0, "");
     sysctl_register_oid(&sysctl__kern_hibernatemode);
 }
 
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 static void
@@ -1463,7 +1683,7 @@ hibernate_setup_for_wake(void)
 
 #define C_ASSERT(e) typedef char    __C_ASSERT__[(e) ? 1 : -1]
 
-extern "C" boolean_t
+extern "C" uint32_t
 hibernate_write_image(void)
 {
     IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
@@ -1493,6 +1713,7 @@ hibernate_write_image(void)
     uint64_t     nsec;
     uint32_t     lastProgressStamp = 0;
     uint32_t     progressStamp;
+    uint32_t	 blob, lastBlob = (uint32_t) -1L;
 
     hibernate_cryptvars_t _cryptvars;
     hibernate_cryptvars_t * cryptvars = 0;
@@ -1502,6 +1723,7 @@ hibernate_write_image(void)
 
     restore1Sum = sum1 = sum2 = 0;
 
+#if CRYPTO
     // encryption data. "iv" is the "initial vector".
     if (kIOHibernateModeEncrypt & gIOHibernateMode)
     {
@@ -1529,6 +1751,7 @@ hibernate_write_image(void)
         bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
         bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
     }
+#endif /* CRYPTO */
 
     hibernate_setup_for_wake();
 
@@ -1637,7 +1860,7 @@ hibernate_write_image(void)
         addr64_t phys64;
         IOByteCount segLen;
 
-        if (vars->previewData)
+        if (vars->previewBuffer)
         {
             ppnum = 0;
             count = 0;
@@ -1658,8 +1881,8 @@ hibernate_write_image(void)
             if (kIOReturnSuccess != err)
                 break;
 
-            src = (uint8_t *) vars->previewData->getBytesNoCopy();
-            count = vars->previewData->getLength();
+            src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
+            count = vars->previewBuffer->getLength();
 
             header->previewPageListSize = ppnum;
             header->previewSize = count + ppnum;
@@ -1728,6 +1951,7 @@ hibernate_write_image(void)
     
         ppnum     = 0;
         pagesDone = 0;
+        lastBlob = 0;
     
         HIBLOG("writing %d pages\n", pageCount);
 
@@ -1808,7 +2032,16 @@ hibernate_write_image(void)
                     uncompressedSize += page_size;
                 ppnum++;
                 pagesDone++;
-    
+
+		if (vars->consoleMapping && (0 == (1023 & pagesDone)))
+		{
+		    blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+		    if (blob != lastBlob)
+		    {
+			ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+			lastBlob = blob;
+		    }
+		}
                 if (0 == (8191 & pagesDone))
                 {
                     clock_get_uptime(&endTime);
@@ -1882,7 +2115,7 @@ hibernate_write_image(void)
         err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
         if (kIOReturnSuccess != err)
             break;
-        err = IOHibernatePollerIODone(vars->fileVars);
+        err = IOHibernatePollerIODone(vars->fileVars, true);
         if (kIOReturnSuccess != err)
             break;
     }
@@ -1911,74 +2144,39 @@ hibernate_write_image(void)
     if (pollerOpen)
         IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
 
+    if (vars->consoleMapping)
+        ProgressUpdate(gIOHibernateGraphicsInfo, 
+                        vars->consoleMapping, 0, kIOHibernateProgressCount);
+
     HIBLOG("hibernate_write_image done(%x)\n", err);
 
     // should we come back via regular wake, set the state in memory.
     gIOHibernateState = kIOHibernateStateInactive;
 
-    if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
-        return (true  /* power down */ );
+    if (kIOReturnSuccess == err)
+    {
+	if (kIOHibernateModeSleep & gIOHibernateMode)
+	{
+	    return (kIOHibernatePostWriteSleep);
+	}
+	else if(kIOHibernateModeRestart & gIOHibernateMode)
+	{
+	    return (kIOHibernatePostWriteRestart);
+	}
+	else
+	{
+	    /* by default, power down */
+	    return (kIOHibernatePostWriteHalt);
+	}
+    }
+    else if (kIOReturnAborted == err)
+    {
+	return (kIOHibernatePostWriteWake);
+    }
     else
-        return (false /* sleep */ );
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-DECLARE_IOHIBERNATEPROGRESSALPHA
-
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
-    uint32_t  rowBytes, pixelShift;
-    uint32_t  x, y;
-    int32_t   blob, lastBlob;
-    uint32_t  alpha, in, color, result;
-    uint8_t * out;
-    uint32_t  saveindex[kIOHibernateProgressCount] = { 0 };
-
-    pixelShift = display->depth >> 4;
-    if (pixelShift < 1)
-        return;
-
-    rowBytes = display->rowBytes;
-
-    screen += ((display->width 
-            - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
-                + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
-    lastBlob  = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
-    screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
-
-    for (y = 0; y < kIOHibernateProgressHeight; y++)
     {
-        out = screen + y * rowBytes;
-        for (blob = firstBlob; blob <= lastBlob; blob++)
-        {
-            color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
-            for (x = 0; x < kIOHibernateProgressWidth; x++)
-            {
-                alpha  = gIOHibernateProgressAlpha[y][x];
-                result = color;
-                if (alpha)
-                {
-                    if (0xff != alpha)
-                    {
-                        in = display->progressSaveUnder[blob][saveindex[blob]++];
-                        result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
-                    }
-                    if (1 == pixelShift)
-                    {
-                        result >>= 3;
-                        *((uint16_t *)out) = (result << 10) | (result << 5) | result;	// 16
-                    }
-                    else
-                        *((uint32_t *)out) = (result << 16) | (result << 8) | result;	// 32
-                }
-                out += (1 << pixelShift);
-            }
-            out += (kIOHibernateProgressSpacing << pixelShift);
-        }
+	/* on error, sleep */
+	return (kIOHibernatePostWriteSleep);
     }
 }
 
@@ -2069,7 +2267,7 @@ hibernate_machine_init(void)
 
     IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
 
-    if (vars->videoMapping)
+    if (vars->videoMapSize)
     {
         lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
@@ -2149,7 +2347,7 @@ hibernate_machine_init(void)
 	    ppnum++;
 	    pagesDone++;
 
-            if (vars->videoMapping && (0 == (255 & pagesDone)))
+            if (vars->videoMapSize && (0 == (1023 & pagesDone)))
             {
                 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
                         / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
@@ -2180,11 +2378,11 @@ hibernate_machine_init(void)
     gIOHibernateCurrentHeader->actualImage2Sum = sum;
 
     if (vars->fileVars->io)
-        (void) IOHibernatePollerIODone(vars->fileVars);
+        (void) IOHibernatePollerIODone(vars->fileVars, false);
 
     err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
 
-    if (vars->videoMapping)
+    if (vars->videoMapSize)
         ProgressUpdate(gIOHibernateGraphicsInfo, 
                         (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
 
diff --git a/iokit/Kernel/IOHibernateInternal.h b/iokit/Kernel/IOHibernateInternal.h
index 04b5fd3eb..1e0adbc5c 100644
--- a/iokit/Kernel/IOHibernateInternal.h
+++ b/iokit/Kernel/IOHibernateInternal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -41,6 +47,7 @@ struct IOHibernateVars
     vm_offset_t				videoMapping;
     vm_size_t				videoAllocSize;
     vm_size_t				videoMapSize;
+    uint8_t *				consoleMapping;
     uint8_t				haveFastBoot;
     uint8_t				saveBootAudioVolume;
     uint8_t				wiredCryptKey[kIOHibernateAESKeySize / 8];
diff --git a/iokit/Kernel/IOHibernateRestoreKernel.c b/iokit/Kernel/IOHibernateRestoreKernel.c
index a61421190..255f93221 100644
--- a/iokit/Kernel/IOHibernateRestoreKernel.c
+++ b/iokit/Kernel/IOHibernateRestoreKernel.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -27,6 +33,7 @@
 #include <IOKit/IOLib.h>
 #include <pexpert/boot.h>
 #include <crypto/aes.h>
+#include <libkern/libkern.h>
 
 #include "WKdm.h"
 #include "IOHibernateInternal.h"
@@ -95,6 +102,11 @@ hibernate_sum(uint8_t *buf, int32_t len)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+static __inline__ unsigned int cntlzw(unsigned int num)
+{
+	return clz(num);
+}
+
 static hibernate_bitmap_t *
 hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 {
@@ -108,7 +120,7 @@ hibernate_page_bitmap(hibernate_page_list_t * list, uint32_t page)
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
     if (bank == list->bank_count)
-	bitmap = 0;
+	bitmap = NULL;
 	
     return (bitmap);
 }
@@ -131,7 +143,7 @@ hibernate_page_bitmap_pin(hibernate_page_list_t * list, uint32_t * pPage)
 	bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
     }
     if (bank == list->bank_count)
-	bitmap = 0;
+	bitmap = NULL;
 	
     return (bitmap);
 }
@@ -225,9 +237,6 @@ hibernate_page_list_grab(hibernate_page_list_t * list, uint32_t * pNextFree)
 	}
     }
 
-    if (!bitmap)
-	IOPanic(__FUNCTION__);
-
     return (nextFree);
 }
 
@@ -298,6 +307,7 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     uint32_t lastImagePage;
     uint32_t lastMapPage;
     uint32_t lastPageIndexPage;
+    ResetProc proc;
 
     C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
 
@@ -501,18 +511,18 @@ hibernate_kernel_entrypoint(IOHibernateImageHeader * header,
     gIOHibernateState = kIOHibernateStateWakingFromHibernate;
 
 #if __ppc__
-    ResetProc proc;
     proc = (ResetProc) 0x100;
     __asm__ volatile("ori 0, 0, 0" : : );
     proc();
 #elif __i386__
-    ResetProc proc;
     proc = (ResetProc) acpi_wake_prot_entry;
     // flush caches
     __asm__("wbinvd");
     proc();
+#elif __arm__
+    proc = (ResetProc)0x00000000;
+    proc();
 #endif
   
     return -1;
 }
-
diff --git a/iokit/Kernel/IOInterleavedMemoryDescriptor.cpp b/iokit/Kernel/IOInterleavedMemoryDescriptor.cpp
new file mode 100644
index 000000000..0d7713a81
--- /dev/null
+++ b/iokit/Kernel/IOInterleavedMemoryDescriptor.cpp
@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/IOLib.h>
+#include <IOKit/IOInterleavedMemoryDescriptor.h>
+
+#define super IOMemoryDescriptor
+OSDefineMetaClassAndStructors(IOInterleavedMemoryDescriptor, IOMemoryDescriptor)
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithAddress(
+                                  void *      /* address       */ ,
+                                  IOByteCount /* withLength    */ ,
+                                  IODirection /* withDirection */ )
+{
+    return false;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithAddress(
+                                  vm_address_t /* address       */ ,
+                                  IOByteCount  /* withLength    */ ,
+                                  IODirection  /* withDirection */ ,
+                                  task_t       /* withTask      */ )
+{
+    return false;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithPhysicalAddress(
+                                  IOPhysicalAddress /* address       */ ,
+                                  IOByteCount       /* withLength    */ ,
+                                  IODirection       /* withDirection */ )
+{
+    return false;
+}
+
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithPhysicalRanges(
+                                  IOPhysicalRange * /* ranges        */ ,
+                                  UInt32            /* withCount     */ ,
+                                  IODirection       /* withDirection */ ,
+                                  bool              /* asReference   */ )
+{
+    return false;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithRanges(
+                                  IOVirtualRange * /* ranges        */ ,
+                                  UInt32           /* withCount     */ ,
+                                  IODirection      /* withDirection */ ,
+                                  task_t           /* withTask      */ ,
+                                  bool             /* asReference   */ )
+{
+    return false;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOInterleavedMemoryDescriptor * IOInterleavedMemoryDescriptor::withCapacity(
+                                  UInt32                capacity,
+                                  IODirection           direction )
+{
+    //
+    // Create a new IOInterleavedMemoryDescriptor.  The "buffer" will be made up
+    // of several memory descriptors, that are to be chained end-to-end to make up
+    // a single memory descriptor.
+    //
+
+    IOInterleavedMemoryDescriptor * me = new IOInterleavedMemoryDescriptor;
+
+    if ( me && !me->initWithCapacity(
+                                  /* capacity  */ capacity,
+                                  /* direction */ direction ))
+    {
+	    me->release();
+	    me = 0;
+    }
+
+    return me;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::initWithCapacity(
+                                  UInt32                capacity,
+                                  IODirection           direction )
+{
+    //
+    // Initialize an IOInterleavedMemoryDescriptor. The "buffer" will be made up
+    // of several memory descriptors, that are to be chained end-to-end to make up
+    // a single memory descriptor.
+    //
+
+    assert(capacity);
+
+    // Ask our superclass' opinion.
+    if ( super::init() == false )  return false;
+    
+    // Initialize our minimal state.
+
+    _direction              = direction;
+    _length                 = 0;
+    _mappings               = 0;
+    _tag                    = 0;
+    _descriptorCount        = 0;
+    _descriptors            = IONew(IOMemoryDescriptor *, capacity);
+    _descriptorOffsets      = IONew(IOByteCount, capacity);
+    _descriptorLengths      = IONew(IOByteCount, capacity);
+
+    if ( (_descriptors == 0) || (_descriptorOffsets == 0) || (_descriptorLengths == 0) )
+        return false;
+
+    _descriptorCapacity     = capacity;
+
+    return true;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+void IOInterleavedMemoryDescriptor::clearMemoryDescriptors( IODirection direction )
+{
+    UInt32 index;
+
+    for ( index = 0; index < _descriptorCount; index++ )
+    {
+        if ( _descriptorPrepared )
+	    _descriptors[index]->complete(_direction);
+
+	_descriptors[index]->release();
+	_descriptors[index] = 0;
+
+	_descriptorOffsets[index] = 0;
+	_descriptorLengths[index] = 0;
+    }
+
+    if ( direction != kIODirectionNone )
+        _direction = direction;
+
+    _descriptorCount = 0;
+    _length = 0;
+    _mappings = 0;
+    _tag = 0;
+
+};
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+bool IOInterleavedMemoryDescriptor::setMemoryDescriptor(
+                                             IOMemoryDescriptor * descriptor,
+					     IOByteCount offset,
+					     IOByteCount length )
+{
+    if ( _descriptorPrepared || (_descriptorCount == _descriptorCapacity) )
+        return false;
+
+    if ( (offset + length) > descriptor->getLength() )
+        return false;
+
+//    if ( descriptor->getDirection() != _direction )
+//        return false;
+
+    descriptor->retain();
+    _descriptors[_descriptorCount] = descriptor;
+    _descriptorOffsets[_descriptorCount] = offset;
+    _descriptorLengths[_descriptorCount] = length;
+
+    _descriptorCount++;
+
+    _length += length;
+
+    return true;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+void IOInterleavedMemoryDescriptor::free()
+{
+    //
+    // Free all of this object's outstanding resources.
+    //
+
+    if ( _descriptors )
+    {
+        for ( unsigned index = 0; index < _descriptorCount; index++ ) 
+            _descriptors[index]->release();
+
+        if ( _descriptors != 0 )
+            IODelete(_descriptors, IOMemoryDescriptor *, _descriptorCapacity);
+
+        if ( _descriptorOffsets != 0 )
+            IODelete(_descriptorOffsets, IOMemoryDescriptor *, _descriptorCapacity);
+
+        if ( _descriptorLengths != 0 )
+            IODelete(_descriptorLengths, IOMemoryDescriptor *, _descriptorCapacity);
+    }
+
+    super::free();
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOReturn IOInterleavedMemoryDescriptor::prepare(IODirection forDirection)
+{
+    //
+    // Prepare the memory for an I/O transfer.
+    //
+    // This involves paging in the memory and wiring it down for the duration
+    // of the transfer.  The complete() method finishes the processing of the
+    // memory after the I/O transfer finishes.
+    //
+
+    unsigned index;
+    IOReturn status = kIOReturnSuccess;
+    IOReturn statusUndo;
+
+    if ( forDirection == kIODirectionNone )
+    {
+        forDirection = _direction;
+    }
+
+    for ( index = 0; index < _descriptorCount; index++ ) 
+    {
+        status = _descriptors[index]->prepare(forDirection);
+        if ( status != kIOReturnSuccess )  break;
+    }
+
+    if ( status != kIOReturnSuccess )
+    {
+        for ( unsigned indexUndo = 0; indexUndo < index; indexUndo++ )
+        {
+            statusUndo = _descriptors[index]->complete(forDirection);
+            assert(statusUndo == kIOReturnSuccess);
+        }
+    }
+
+    if ( status == kIOReturnSuccess ) _descriptorPrepared = true;
+
+    return status;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOReturn IOInterleavedMemoryDescriptor::complete(IODirection forDirection)
+{
+    //
+    // Complete processing of the memory after an I/O transfer finishes.
+    //
+    // This method shouldn't be called unless a prepare() was previously issued;
+    // the prepare() and complete() must occur in pairs, before and after an I/O
+    // transfer.
+    //
+
+    IOReturn status;
+    IOReturn statusFinal = kIOReturnSuccess;
+
+    if ( forDirection == kIODirectionNone )
+    {
+        forDirection = _direction;
+    }
+
+    for ( unsigned index = 0; index < _descriptorCount; index++ ) 
+    {
+        status = _descriptors[index]->complete(forDirection);
+        if ( status != kIOReturnSuccess )  statusFinal = status;
+        assert(status == kIOReturnSuccess);
+    }
+
+    _descriptorPrepared = false;
+
+    return statusFinal;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+addr64_t IOInterleavedMemoryDescriptor::getPhysicalSegment64( 
+				    IOByteCount   offset, IOByteCount * length )
+{
+    //
+    // This method returns the physical address of the byte at the given offset
+    // into the memory,  and optionally the length of the physically contiguous
+    // segment from that offset.
+    //
+
+    addr64_t pa;
+
+    assert(offset <= _length);
+
+    for ( unsigned index = 0; index < _descriptorCount; index++ ) 
+    {
+        if ( offset < _descriptorLengths[index] )
+        {
+            pa = _descriptors[index]->getPhysicalSegment64(_descriptorOffsets[index] + offset, length);
+	    if ((_descriptorLengths[index] - offset) < *length) *length = _descriptorLengths[index] - offset;
+            return pa;
+        }
+        offset -= _descriptorLengths[index];
+    }
+
+    if ( length )  *length = 0;
+
+    return 0;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOPhysicalAddress IOInterleavedMemoryDescriptor::getPhysicalSegment( 
+				    IOByteCount   offset, IOByteCount * length )
+{
+    //
+    // This method returns the physical address of the byte at the given offset
+    // into the memory,  and optionally the length of the physically contiguous
+    // segment from that offset.
+    //
+
+    IOPhysicalAddress pa;
+
+    assert(offset <= _length);
+
+    for ( unsigned index = 0; index < _descriptorCount; index++ ) 
+    {
+        if ( offset < _descriptorLengths[index] )
+        {
+            pa = _descriptors[index]->getPhysicalSegment(_descriptorOffsets[index] + offset, length);
+	    if ((_descriptorLengths[index] - offset) < *length) *length = _descriptorLengths[index] - offset;
+            return pa;
+        }
+        offset -= _descriptorLengths[index];
+    }
+
+    if ( length )  *length = 0;
+
+    return 0;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+IOPhysicalAddress IOInterleavedMemoryDescriptor::getSourceSegment(
+                                                       IOByteCount   offset,
+                                                       IOByteCount * length )
+{
+    //
+    // This method returns the physical address of the byte at the given offset
+    // into the memory,  and optionally the length of the physically contiguous
+    // segment from that offset.
+    //
+
+    IOPhysicalAddress pa;
+
+    assert(offset <= _length);
+
+    for ( unsigned index = 0; index < _descriptorCount; index++ ) 
+    {
+        if ( offset < _descriptorLengths[index] )
+        {
+            pa = _descriptors[index]->getSourceSegment(_descriptorOffsets[index] + offset, length);
+	    if ((_descriptorLengths[index] - offset) < *length) *length = _descriptorLengths[index] - offset;
+            return pa;
+        }
+        offset -= _descriptorLengths[index];
+    }
+
+    if ( length )  *length = 0;
+
+    return 0;
+}
+
+// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+void * IOInterleavedMemoryDescriptor::getVirtualSegment( IOByteCount   /* offset */ ,
+                                                   IOByteCount * /* length */ )
+{
+    return 0;
+}
diff --git a/iokit/Kernel/IOInterruptController.cpp b/iokit/Kernel/IOInterruptController.cpp
index c44d47b89..f65dbe3b6 100644
--- a/iokit/Kernel/IOInterruptController.cpp
+++ b/iokit/Kernel/IOInterruptController.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/iokit/Kernel/IOInterruptEventSource.cpp b/iokit/Kernel/IOInterruptEventSource.cpp
index bda259692..4bfd27786 100644
--- a/iokit/Kernel/IOInterruptEventSource.cpp
+++ b/iokit/Kernel/IOInterruptEventSource.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
@@ -143,6 +149,7 @@ void IOInterruptEventSource::enable()
     if (provider && intIndex != -1) {
         provider->enableInterrupt(intIndex);
         explicitDisable = false;
+        enabled = true;
     }
 }
 
@@ -151,6 +158,7 @@ void IOInterruptEventSource::disable()
     if (provider && intIndex != -1) {
         provider->disableInterrupt(intIndex);
         explicitDisable = true;
+        enabled = false;
     }
 }
 
diff --git a/iokit/Kernel/IOKitDebug.cpp b/iokit/Kernel/IOKitDebug.cpp
index 06c6e1030..3a2415196 100644
--- a/iokit/Kernel/IOKitDebug.cpp
+++ b/iokit/Kernel/IOKitDebug.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -49,6 +55,7 @@ SYSCTL_QUAD(_debug, OID_AUTO, iokit, CTLFLAG_RW, &gIOKitDebug, "boot_arg io");
 
 int 		debug_malloc_size;
 int		debug_iomalloc_size;
+
 vm_size_t	debug_iomallocpageable_size;
 int 		debug_container_malloc_size;
 // int 		debug_ivars_size; // in OSObject.cpp
@@ -75,7 +82,7 @@ void IOPrintPlane( const IORegistryPlane * plane )
 
     iter->reset();
     while( (next = iter->getNextObjectRecursive())) {
-	sprintf( format + 1, "%ds", 2 * next->getDepth( plane ));
+	snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane ));
 	IOLog( format, "");
 	IOLog( "\033[33m%s", next->getName( plane ));
 	if( (next->getLocation( plane )))
@@ -131,7 +138,7 @@ void db_dumpiojunk( const IORegistryPlane * plane )
 
     iter->reset();
     while( (next = iter->getNextObjectRecursive())) {
-		sprintf( format + 1, "%ds", 2 * next->getDepth( plane ));
+		snprintf(format + 1, sizeof(format) - 1, "%ds", 2 * next->getDepth( plane ));
 		dbugprintf( format, "");
 		dbugprintf( "%s", next->getName( plane ));
 		if( (next->getLocation( plane )))
@@ -197,7 +204,6 @@ void IOKitDiagnostics::updateOffset( OSDictionary * dict,
     off->release();
 }
 
-
 bool IOKitDiagnostics::serialize(OSSerialize *s) const
 {
     OSDictionary * 	dict;
diff --git a/iokit/Kernel/IOKitKernelInternal.h b/iokit/Kernel/IOKitKernelInternal.h
index 373783f4f..afa64a600 100644
--- a/iokit/Kernel/IOKitKernelInternal.h
+++ b/iokit/Kernel/IOKitKernelInternal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -37,7 +43,12 @@ typedef kern_return_t (*IOIteratePageableMapsCallback)(vm_map_t map, void * ref)
 void IOLibInit(void);
 kern_return_t IOIteratePageableMaps(vm_size_t size,
                     IOIteratePageableMapsCallback callback, void * ref);
-vm_map_t IOPageableMapForAddress( vm_address_t address );
+vm_map_t IOPageableMapForAddress(vm_address_t address);
+
+kern_return_t 
+IOMemoryDescriptorMapMemEntry(vm_map_t map, ipc_port_t entry, IOOptionBits options, bool pageable,
+				mach_vm_size_t offset, mach_vm_address_t * address, mach_vm_size_t length);
+
 SInt32 OSKernelStackRemaining( void );
 
 mach_vm_address_t
@@ -48,9 +59,8 @@ IOKernelFreeContiguous(mach_vm_address_t address, mach_vm_size_t size);
 extern vm_size_t debug_iomallocpageable_size;
 
 // osfmk/device/iokit_rpc.c
-// LP64todo - these need to expand 
 extern kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t pa,
-			mach_vm_size_t length, unsigned int options);
+                                 mach_vm_size_t length, unsigned int mapFlags);
 extern kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t length);
 
 extern kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
@@ -88,4 +98,41 @@ struct IOMDDMAWalkSegmentArgs {
 };
 typedef UInt8 IOMDDMAWalkSegmentState[128];
 
+struct IODMACommandInternal
+{
+    IOMDDMAWalkSegmentState fState;
+    IOMDDMACharacteristics  fMDSummary;
+
+    UInt64 fPreparedOffset;
+    UInt64 fPreparedLength;
+
+    UInt8  fCursor;
+    UInt8  fCheckAddressing;
+    UInt8  fIterateOnly;
+    UInt8  fMisaligned;
+    UInt8  fCopyContig;
+    UInt8  fPrepared;
+    UInt8  fDoubleBuffer;
+    UInt8  __pad[1];
+
+    ppnum_t  fCopyPageAlloc;
+    ppnum_t  fCopyPageCount;
+    ppnum_t  fNextRemapIndex;
+    addr64_t fCopyNext;
+
+    class IOBufferMemoryDescriptor * fCopyMD;
+
+    // IODMAEventSource use
+    IOReturn fStatus;
+    UInt64   fActualByteCount;
+};
+
+extern "C" struct timeval gIOLastSleepTime;
+extern "C" struct timeval gIOLastWakeTime;
+
+extern "C" void IOKitResetTime( void );
+extern "C" void IOKitInitializeTime( void );
+
+extern "C" OSString * IOCopyLogNameForPID(int pid);
+
 #endif /* ! _IOKIT_KERNELINTERNAL_H */
diff --git a/iokit/Kernel/IOLib.cpp b/iokit/Kernel/IOLib.cpp
index 5f2b9f77c..5fca33623 100644
--- a/iokit/Kernel/IOLib.cpp
+++ b/iokit/Kernel/IOLib.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * HISTORY
@@ -43,6 +49,11 @@
 
 #include "IOKitKernelInternal.h"
 
+#ifdef IOALLOCDEBUG
+#include <libkern/OSDebug.h>
+#include <sys/sysctl.h>
+#endif
+
 extern "C"
 {
 
@@ -165,8 +176,9 @@ void * IOMalloc(vm_size_t size)
 
     address = (void *)kalloc(size);
 #if IOALLOCDEBUG
-    if (address)
-	debug_iomalloc_size += size;
+    if (address) {
+		debug_iomalloc_size += size;
+	}
 #endif
     return address;
 }
@@ -174,9 +186,9 @@ void * IOMalloc(vm_size_t size)
 void IOFree(void * address, vm_size_t size)
 {
     if (address) {
-	kfree(address, size);
+		kfree(address, size);
 #if IOALLOCDEBUG
-	debug_iomalloc_size -= size;
+		debug_iomalloc_size -= size;
 #endif
     }
 }
@@ -236,8 +248,9 @@ void * IOMallocAligned(vm_size_t size, vm_size_t alignment)
     assert(0 == (address & alignMask));
 
 #if IOALLOCDEBUG
-    if( address)
-	debug_iomalloc_size += size;
+    if( address) {
+		debug_iomalloc_size += size;
+	}
 #endif
 
     return (void *) address;
@@ -331,7 +344,7 @@ IOKernelAllocateContiguous(mach_vm_size_t size, mach_vm_size_t alignment)
 	if (adjustedSize > page_size)
 	{
 	    kr = kmem_alloc_contig(kernel_map, &virt, size,
-				    alignMask, 0);
+				   alignMask, 0, 0);
 	}
 	else
 	{
@@ -366,8 +379,9 @@ IOKernelAllocateContiguous(mach_vm_size_t size, mach_vm_size_t alignment)
     }
 
 #if IOALLOCDEBUG
-    if (address)
-	debug_iomalloc_size += size;
+    if (address) {
+		debug_iomalloc_size += size;
+	}
 #endif
 
     return (address);
@@ -650,9 +664,7 @@ IOReturn IOFlushProcessorCache( task_t task, IOVirtualAddress address,
     if( task != kernel_task)
 	return( kIOReturnUnsupported );
 
-#if __ppc__
     flush_dcache64( (addr64_t) address, (unsigned) length, false );
-#endif
 
     return( kIOReturnSuccess );
 }
@@ -670,6 +682,9 @@ SInt32 OSKernelStackRemaining( void )
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
+/*
+ * Spin for indicated number of milliseconds.
+ */
 void IOSleep(unsigned milliseconds)
 {
     delay_for_interval(milliseconds, kMillisecondScale);
@@ -683,6 +698,14 @@ void IODelay(unsigned microseconds)
     delay_for_interval(microseconds, kMicrosecondScale);
 }
 
+/*
+ * Spin for indicated number of nanoseconds.
+ */
+void IOPause(unsigned nanoseconds)
+{
+    delay_for_interval(nanoseconds, kNanosecondScale);
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 void IOLog(const char *format, ...)
@@ -698,7 +721,7 @@ void IOLog(const char *format, ...)
 
 void IOPanic(const char *reason)
 {
-	panic(reason);
+	panic("%s", reason);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -714,7 +737,7 @@ const char *IOFindNameForValue(int value, const IONamedValue *regValueArray)
 		if(regValueArray->value == value)
 			return(regValueArray->name);
 	}
-	sprintf(noValue, "0x%x (UNDEFINED)", value);
+	snprintf(noValue, sizeof(noValue), "0x%x (UNDEFINED)", value);
 	return((const char *)noValue);
 }
 
@@ -731,6 +754,16 @@ IOReturn IOFindValueForName(const char *string,
 	return kIOReturnBadArgument;
 }
 
+OSString * IOCopyLogNameForPID(int pid)
+{
+    char   buf[128];
+    size_t len;
+    snprintf(buf, sizeof(buf), "pid %d, ", pid);
+    len = strlen(buf);
+    proc_name(pid, buf + len, sizeof(buf) - len);
+    return (OSString::withCString(buf));
+}
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 IOAlignment IOSizeToAlignment(unsigned int size)
@@ -757,3 +790,6 @@ unsigned int IOAlignmentToSize(IOAlignment align)
 }
 
 } /* extern "C" */
+
+
+
diff --git a/iokit/Kernel/IOLocks.cpp b/iokit/Kernel/IOLocks.cpp
index 064a89de0..988e01529 100644
--- a/iokit/Kernel/IOLocks.cpp
+++ b/iokit/Kernel/IOLocks.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -35,6 +41,8 @@
 #include <IOKit/IOLib.h> 
 #include <IOKit/assert.h>
 
+#include <IOKit/IOLocksPrivate.h>
+
 extern "C" {
 #include <kern/locks.h>
 
@@ -78,42 +86,53 @@ void	IOLockWakeup(IOLock * lock, void *event, bool oneThread)
 
 
 struct _IORecursiveLock {
-    lck_mtx_t   *mutex;
-    thread_t	thread;
-    UInt32	count;
+	lck_mtx_t	*mutex;
+	lck_grp_t	*group;
+	thread_t	thread;
+	UInt32		count;
 };
 
-IORecursiveLock * IORecursiveLockAlloc( void )
+IORecursiveLock * IORecursiveLockAllocWithLockGroup( lck_grp_t * lockGroup )
 {
     _IORecursiveLock * lock;
 
-    lock = IONew( _IORecursiveLock, 1);
-    if( !lock)
+    if( lockGroup == 0 )
         return( 0 );
 
-    lock->mutex = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
-    if( lock->mutex) {
+    lock = IONew( _IORecursiveLock, 1 );
+    if( !lock )
+        return( 0 );
+
+    lock->mutex = lck_mtx_alloc_init( lockGroup, LCK_ATTR_NULL );
+    if( lock->mutex ) {
+		lock->group = lockGroup;
         lock->thread = 0;
         lock->count  = 0;
     } else {
-        IODelete( lock, _IORecursiveLock, 1);
+        IODelete( lock, _IORecursiveLock, 1 );
         lock = 0;
     }
 
     return( (IORecursiveLock *) lock );
 }
 
+
+IORecursiveLock * IORecursiveLockAlloc( void )
+{
+    return IORecursiveLockAllocWithLockGroup( IOLockGroup );
+}
+
 void IORecursiveLockFree( IORecursiveLock * _lock )
 {
     _IORecursiveLock * lock = (_IORecursiveLock *)_lock;
-
-    lck_mtx_free( lock->mutex , IOLockGroup);
-    IODelete( lock, _IORecursiveLock, 1);
+	
+    lck_mtx_free( lock->mutex, lock->group );
+    IODelete( lock, _IORecursiveLock, 1 );
 }
 
-lck_mtx_t * IORecursiveLockGetMachLock( IORecursiveLock * lock)
+lck_mtx_t * IORecursiveLockGetMachLock( IORecursiveLock * lock )
 {
-    return( lock->mutex);
+    return( lock->mutex );
 }
 
 void IORecursiveLockLock( IORecursiveLock * _lock)
diff --git a/iokit/Kernel/IOMapper.cpp b/iokit/Kernel/IOMapper.cpp
index f2f0633c3..43f7bbe54 100644
--- a/iokit/Kernel/IOMapper.cpp
+++ b/iokit/Kernel/IOMapper.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMapper.h>
@@ -146,7 +152,7 @@ NewARTTable(IOByteCount size, void ** virtAddrP, ppnum_t *physAddrP)
     vm_address_t address;
 
     size = round_page_32(size);
-    kr = kmem_alloc_contig(kernel_map, &address, size, PAGE_MASK, 0);
+    kr = kmem_alloc_contig(kernel_map, &address, size, PAGE_MASK, 0, 0);
     if (kr)
         return 0;
 
@@ -344,7 +350,6 @@ mach_vm_address_t IOMallocPhysical(mach_vm_size_t size, mach_vm_address_t mask)
     {
 	address = ptoa_64(gIOCopyMapper->iovmAlloc(atop_64(round_page(size))));
     }
-
     return (address);
 }
 
diff --git a/iokit/Kernel/IOMemoryCursor.cpp b/iokit/Kernel/IOMemoryCursor.cpp
index ccc953957..2487209f8 100644
--- a/iokit/Kernel/IOMemoryCursor.cpp
+++ b/iokit/Kernel/IOMemoryCursor.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOMemoryCursor.cpp created by wgulland on 1999-3-02 */
 
diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp
index 5177b197b..3c0b8f7e1 100644
--- a/iokit/Kernel/IOMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -36,6 +42,7 @@
 #include <IOKit/IOKitKeysPrivate.h>
 
 #include <IOKit/IOKitDebug.h>
+#include <libkern/OSDebug.h>
 
 #include "IOKitKernelInternal.h"
 #include "IOCopyMapper.h"
@@ -51,12 +58,13 @@
 __BEGIN_DECLS
 #include <vm/pmap.h>
 #include <vm/vm_pageout.h>
-#include <vm/vm_shared_memory_server.h>
 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 
 #include <mach/vm_prot.h>
+#include <mach/mach_vm.h>
 #include <vm/vm_fault.h>
+#include <vm/vm_protos.h>
 
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 void ipc_port_release_send(ipc_port_t port);
@@ -133,17 +141,17 @@ class _IOMemoryMap : public IOMemoryMap
 {
     OSDeclareDefaultStructors(_IOMemoryMap)
 public:
-    IOMemoryDescriptor * memory;
-    IOMemoryMap *	superMap;
-    IOByteCount		offset;
-    IOByteCount		length;
-    IOVirtualAddress	logical;
-    task_t		addressTask;
-    vm_map_t		addressMap;
-    IOOptionBits	options;
-    upl_t		redirUPL;
-    ipc_port_t		redirEntry;
-    IOMemoryDescriptor * owner;
+    IOMemoryDescriptor * fMemory;
+    IOMemoryMap *	 fSuperMap;
+    mach_vm_size_t	 fOffset;
+    mach_vm_address_t	 fAddress;
+    mach_vm_size_t	 fLength;
+    task_t		 fAddressTask;
+    vm_map_t		 fAddressMap;
+    IOOptionBits	 fOptions;
+    upl_t		 fRedirUPL;
+    ipc_port_t		 fRedirEntry;
+    IOMemoryDescriptor * fOwner;
 
 protected:
     virtual void taggedRelease(const void *tag = 0) const;
@@ -155,6 +163,8 @@ public:
     virtual IOVirtualAddress 	getVirtualAddress();
     virtual IOByteCount 	getLength();
     virtual task_t		getAddressTask();
+    virtual mach_vm_address_t 	getAddress();
+    virtual mach_vm_size_t 	getSize();
     virtual IOMemoryDescriptor * getMemoryDescriptor();
     virtual IOOptionBits 	getMapOptions();
 
@@ -165,31 +175,24 @@ public:
 					 IOOptionBits         options,
 					 IOByteCount          offset = 0);
 
+    virtual IOReturn		redirect(IOMemoryDescriptor * newBackingMemory,
+					 IOOptionBits         options,
+					 mach_vm_size_t       offset = 0);
+
     virtual IOPhysicalAddress 	getPhysicalSegment(IOByteCount offset,
 	       					   IOByteCount * length);
 
     // for IOMemoryDescriptor use
-    _IOMemoryMap * copyCompatible(
-		IOMemoryDescriptor *	owner,
-                task_t			intoTask,
-                IOVirtualAddress	toAddress,
-                IOOptionBits		options,
-                IOByteCount		offset,
-                IOByteCount		length );
-
-    bool initCompatible(
-	IOMemoryDescriptor *	memory,
-	IOMemoryMap *		superMap,
-        IOByteCount		offset,
-        IOByteCount		length );
-
-    bool initWithDescriptor(
-	IOMemoryDescriptor *	memory,
+    _IOMemoryMap * copyCompatible( _IOMemoryMap * newMapping );
+
+    bool init(
 	task_t			intoTask,
-	IOVirtualAddress	toAddress,
+	mach_vm_address_t	toAddress,
 	IOOptionBits		options,
-        IOByteCount		offset,
-        IOByteCount		length );
+        mach_vm_size_t		offset,
+        mach_vm_size_t		length );
+
+    bool    setMemoryDescriptor(IOMemoryDescriptor * _memory, mach_vm_size_t _offset);
 
     IOReturn redirect(
 	task_t			intoTask, bool redirect );
@@ -297,7 +300,7 @@ kern_return_t device_close(
 // This means that pointers are not passed and NULLs don't have to be
 // checked for as a NULL reference is illegal.
 static inline void
-getAddrLenForInd(addr64_t &addr, IOPhysicalLength &len, // Output variables
+getAddrLenForInd(user_addr_t &addr, IOPhysicalLength &len, // Output variables
      UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
 {
     assert(kIOMemoryTypeUIO       == type
@@ -402,9 +405,9 @@ IOMemoryDescriptor::withRanges(	IOVirtualRange * ranges,
 
 IOMemoryDescriptor *
 IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
-                                       mach_vm_size_t length,
-                                       IOOptionBits   options,
-                                       task_t         task)
+					mach_vm_size_t length,
+					IOOptionBits   options,
+					task_t         task)
 {
     IOAddressRange range = { address, length };
     return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
@@ -412,9 +415,9 @@ IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
 
 IOMemoryDescriptor *
 IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
-                                       UInt32           rangeCount,
-                                       IOOptionBits     options,
-                                       task_t           task)
+					UInt32           rangeCount,
+					IOOptionBits     options,
+					task_t           task)
 {
     IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
     if (that)
@@ -424,10 +427,10 @@ IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
 	else
 	    options |= kIOMemoryTypePhysical64;
 
-       if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
-           return that;
+	if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
+	    return that;
 
-       that->release();
+	that->release();
     }
 
     return 0;
@@ -566,7 +569,12 @@ void *IOGeneralMemoryDescriptor::createNamedEntry()
 		    : get_task_map(_task);
 
     memory_object_size_t  actualSize = size;
-    vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE;
+    vm_prot_t             prot       = VM_PROT_READ;
+#if CONFIG_EMBEDDED
+    if (kIODirectionOut != (kIODirectionOutIn & _flags))
+#endif
+	prot |= VM_PROT_WRITE;
+
     if (_memEntry)
 	prot |= MAP_MEM_NAMED_REUSE;
 
@@ -668,7 +676,7 @@ IOGeneralMemoryDescriptor::initWithRanges(
 
 	// Auto-prepare if this is a kernel memory descriptor as very few
 	// clients bother to prepare() kernel memory.
-	// But it was not  enforced so what are you going to do?
+	// But it was not enforced so what are you going to do?
         if (task == kernel_task)
             mdOpts |= kIOMemoryAutoPrepare;
     }
@@ -735,8 +743,15 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
         assert(task);
         if (!task)
             return false;
-        else
-            break;
+
+	if (vm_map_is_64bit(get_task_map(task)) 
+	    && (kIOMemoryTypeVirtual == type) 
+	    && ((IOVirtualRange *) buffers)->address)
+	{
+	    OSReportWithBacktrace("IOMemoryDescriptor: attempt to create 32b virtual in 64b task, use ::withAddressRange()");
+            return false;
+	}
+	break;
 
     case kIOMemoryTypePhysical:		// Neither Physical nor UPL should have a task
     case kIOMemoryTypePhysical64:
@@ -763,9 +778,12 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
          * An existing memory descriptor is being retargeted to point to
          * somewhere else.  Clean up our present state.
          */
-
-        while (_wireCount)
-            complete();
+	IOOptionBits type = _flags & kIOMemoryTypeMask;
+	if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
+	{
+	    while (_wireCount)
+		complete();
+	}
         if (_ranges.v && _rangesIsAllocated)
 	{
 	    if (kIOMemoryTypeUIO == type)
@@ -775,8 +793,11 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 	    else
 		IODelete(_ranges.v, IOVirtualRange, _rangesCount);
 	}
+
 	if (_memEntry)
 	    { ipc_port_release_send((ipc_port_t) _memEntry); _memEntry = 0; }
+	if (_mappings)
+	    _mappings->flushCollection();
     }
     else {
         if (!super::init())
@@ -802,10 +823,19 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 
     __iomd_reservedA = 0;
     __iomd_reservedB = 0;
-    __iomd_reservedC = 0;
-
     _highestPage = 0;
 
+    if (kIOMemoryThreadSafe & options)
+    {
+	if (!_prepareLock)
+	    _prepareLock = IOLockAlloc();
+    }
+    else if (_prepareLock)
+    {
+	IOLockFree(_prepareLock);
+	_prepareLock = NULL;
+    }
+	
     if (kIOMemoryTypeUPL == type) {
 
         ioGMDData *dataP;
@@ -887,6 +917,7 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
 		bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
 		break;
 	      case kIOMemoryTypeVirtual:
+	      case kIOMemoryTypePhysical:
 		_ranges.v = IONew(IOVirtualRange, count);
 		if (!_ranges.v)
 		    return false;
@@ -961,19 +992,25 @@ IOGeneralMemoryDescriptor::initWithOptions(void *	buffers,
  */
 void IOGeneralMemoryDescriptor::free()
 {
-    LOCK;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+
     if( reserved)
+    {
+	LOCK;
 	reserved->memory = 0;
-    UNLOCK;
+	UNLOCK;
+    }
 
-    while (_wireCount)
-        complete();
+    if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
+    {
+	while (_wireCount)
+	    complete();
+    }
     if (_memoryEntries)
         _memoryEntries->release();
 
     if (_ranges.v && _rangesIsAllocated)
     {
-	IOOptionBits type = _flags & kIOMemoryTypeMask;
 	if (kIOMemoryTypeUIO == type)
 	    uio_free((uio_t) _ranges.v);
 	else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
@@ -990,6 +1027,9 @@ void IOGeneralMemoryDescriptor::free()
     if (_memEntry)
         ipc_port_release_send( (ipc_port_t) _memEntry );
 
+    if (_prepareLock)
+	IOLockFree(_prepareLock);
+
     super::free();
 }
 
@@ -1057,7 +1097,6 @@ IOByteCount IOMemoryDescriptor::readBytes
     assert(offset < _length);
     assert(offset + length <= _length);
     if (offset >= _length) {
-IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);	// @@@ gvdl
         return 0;
     }
 
@@ -1100,7 +1139,6 @@ IOByteCount IOMemoryDescriptor::writeBytes
     assert( !(kIOMemoryPreparedReadOnly & _flags) );
 
     if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
-IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);	// @@@ gvdl
         return 0;
     }
 
@@ -1359,6 +1397,9 @@ IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount
     IOByteCount length  = 0;
     addr64_t    address = 0;
 
+    if (gIOSystemMapper && (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask)))
+	return (super::getPhysicalSegment64(offset, lengthOfSegment));
+
     if (offset < _length) // (within bounds?)
     {
 	IOMDDMAWalkSegmentState _state;
@@ -1426,7 +1467,7 @@ IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *l
 
     if ((address + length) > 0x100000000ULL)
     {
-	panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%x, class %s",
+	panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%lx, class %s",
 		    address, length, (getMetaClass())->getClassName());
     }
 
@@ -1690,7 +1731,7 @@ IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
     return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);
 }
 
-#ifdef __ppc__
+#if	defined(__ppc__) || defined(__arm__)
 extern vm_offset_t		static_memory_end;
 #define io_kernel_static_end	static_memory_end
 #else
@@ -1741,7 +1782,7 @@ io_get_kernel_static_upl(
 IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
 {
     IOOptionBits type = _flags & kIOMemoryTypeMask;
-    IOReturn error = kIOReturnNoMemory;
+    IOReturn error = kIOReturnCannotWire;
     ioGMDData *dataP;
     ppnum_t mapBase = 0;
     IOMapper *mapper;
@@ -1875,7 +1916,7 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
 	    if (highPage > highestPage)
 		highestPage = highPage;
 
-            error = kIOReturnNoMemory;
+            error = kIOReturnCannotWire;
 
             if (baseInfo->device) {
                 numPageInfo = 1;
@@ -1954,6 +1995,9 @@ abortExit:
             mapper->iovmFree(mapBase, _pages);
     }
 
+    if (error == KERN_FAILURE)
+        error = kIOReturnCannotWire;
+
     return error;
 }
 
@@ -1971,16 +2015,24 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
     IOReturn error    = kIOReturnSuccess;
     IOOptionBits type = _flags & kIOMemoryTypeMask;
 
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+	return kIOReturnSuccess;
+
+    if (_prepareLock)
+	IOLockLock(_prepareLock);
+
     if (!_wireCount
     && (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) ) {
         error = wireVirtual(forDirection);
-        if (error)
-            return error;
     }
 
-    _wireCount++;
+    if (kIOReturnSuccess == error)
+	_wireCount++;
 
-    return kIOReturnSuccess;
+    if (_prepareLock)
+	IOLockUnlock(_prepareLock);
+
+    return error;
 }
 
 /*
@@ -1994,49 +2046,61 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
  
 IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
 {
-    assert(_wireCount);
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
 
-    if (!_wireCount)
-        return kIOReturnSuccess;
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+	return kIOReturnSuccess;
 
-    _wireCount--;
-    if (!_wireCount) {
-	IOOptionBits type = _flags & kIOMemoryTypeMask;
+    if (_prepareLock)
+	IOLockLock(_prepareLock);
 
-        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) {
-            /* kIOMemoryTypePhysical */
-            // DO NOTHING
-        }
-        else {
-            ioGMDData * dataP = getDataP(_memoryEntries);
-            ioPLBlock *ioplList = getIOPLList(dataP);
+    assert(_wireCount);
+
+    if (_wireCount)
+    {
+	_wireCount--;
+	if (!_wireCount)
+	{
+	    IOOptionBits type = _flags & kIOMemoryTypeMask;
+	    ioGMDData * dataP = getDataP(_memoryEntries);
+	    ioPLBlock *ioplList = getIOPLList(dataP);
 	    UInt count = getNumIOPL(_memoryEntries, dataP);
 
-            if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
-                dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
+	    if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
+		dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
 
-            // Only complete iopls that we created which are for TypeVirtual
-            if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
-                for (UInt ind = 0; ind < count; ind++)
+	    // Only complete iopls that we created which are for TypeVirtual
+	    if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
+		for (UInt ind = 0; ind < count; ind++)
 		    if (ioplList[ind].fIOPL) {
 			 upl_commit(ioplList[ind].fIOPL, 0, 0);
 			 upl_deallocate(ioplList[ind].fIOPL);
 		    }
-            }
-
-            (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
-        }
+	    }
+	    (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
+	}
     }
+
+    if (_prepareLock)
+	IOLockUnlock(_prepareLock);
+
     return kIOReturnSuccess;
 }
 
 IOReturn IOGeneralMemoryDescriptor::doMap(
-	vm_map_t		addressMap,
-	IOVirtualAddress *	atAddress,
+	vm_map_t		__addressMap,
+	IOVirtualAddress *	__address,
 	IOOptionBits		options,
-	IOByteCount		sourceOffset,
-	IOByteCount		length )
+	IOByteCount		__offset,
+	IOByteCount		__length )
+
 {
+    if (!(kIOMap64Bit & options)) panic("IOGeneralMemoryDescriptor::doMap !64bit");
+
+    _IOMemoryMap * mapping = (_IOMemoryMap *) *__address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;
+
     kern_return_t kr;
     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
 
@@ -2051,15 +2115,14 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
 
     // mapping source == dest? (could be much better)
     if( _task
-    && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
-    && (1 == _rangesCount) && (0 == sourceOffset)
-    && range0Addr && (length <= range0Len) ) {
-	if (sizeof(user_addr_t) > 4 && ((UInt64) range0Addr) >> 32)
-	    return kIOReturnOverrun;	// Doesn't fit in 32bit return field
-	else {
-	    *atAddress = range0Addr;
-	    return( kIOReturnSuccess );
-	}
+     && (mapping->fAddressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
+     && (1 == _rangesCount) && (0 == offset)
+     && range0Addr && (length <= range0Len) )
+    {
+	mapping->fAddress = range0Addr;
+	mapping->fOptions |= kIOMapStatic;
+
+	return( kIOReturnSuccess );
     }
 
     if( 0 == sharedMem) {
@@ -2069,9 +2132,15 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
         if( _task) {
 
             memory_object_size_t actualSize = size;
+	    vm_prot_t            prot       = VM_PROT_READ;
+	    if (!(kIOMapReadOnly & options))
+		prot |= VM_PROT_WRITE;
+	    else if (kIOMapDefaultCache != (options & kIOMapCacheMask))
+		prot |= VM_PROT_WRITE;
+
             kr = mach_make_memory_entry_64(get_task_map(_task),
                         &actualSize, range0Addr,
-                        VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
+                        prot, &sharedMem,
                         NULL );
 
             if( (KERN_SUCCESS == kr) && (actualSize != round_page_32(size))) {
@@ -2093,7 +2162,7 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
     	    addr64_t		pa;
     	    IOPhysicalLength	segLen;
 
-	    pa = getPhysicalSegment64( sourceOffset, &segLen );
+	    pa = getPhysicalSegment64( offset, &segLen );
 
             if( !reserved) {
                 reserved = IONew( ExpansionData, 1 );
@@ -2109,6 +2178,17 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
 		case kIOMapDefaultCache:
 		default:
 		    flags = IODefaultCacheBits(pa);
+		    if (DEVICE_PAGER_CACHE_INHIB & flags)
+		    {
+			if (DEVICE_PAGER_GUARDED & flags)
+			    mapping->fOptions |= kIOMapInhibitCache;
+			else
+			    mapping->fOptions |= kIOMapWriteCombineCache;
+		    }
+		    else if (DEVICE_PAGER_WRITE_THROUGH & flags)
+			mapping->fOptions |= kIOMapWriteThruCache;
+		    else
+			mapping->fOptions |= kIOMapCopybackCache;
 		    break;
 	
 		case kIOMapInhibitCache:
@@ -2142,7 +2222,8 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
                             size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
 
                 assert( KERN_SUCCESS == kr );
-                if( KERN_SUCCESS != kr) {
+                if( KERN_SUCCESS != kr)
+		{
 		    device_pager_deallocate( pager );
                     pager = MACH_PORT_NULL;
                     sharedMem = MACH_PORT_NULL;
@@ -2160,34 +2241,22 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
         _memEntry = (void *) sharedMem;
     }
 
-
-    if( 0 == sharedMem)
-      kr = kIOReturnVMError;
+    IOReturn result;
+    if (0 == sharedMem)
+      result = kIOReturnVMError;
     else
-      kr = super::doMap( addressMap, atAddress,
-                           options, sourceOffset, length );
+      result = super::doMap( __addressMap, __address,
+					options, __offset, __length );
 
-    return( kr );
+    return( result );
 }
 
 IOReturn IOGeneralMemoryDescriptor::doUnmap(
 	vm_map_t		addressMap,
-	IOVirtualAddress	logical,
-	IOByteCount		length )
+	IOVirtualAddress	__address,
+	IOByteCount		__length )
 {
-    // could be much better
-    if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)) {
-
-	IOOptionBits type = _flags & kIOMemoryTypeMask;
-	user_addr_t range0Addr;
-	IOByteCount range0Len;
-
-	getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
-	if (logical == range0Addr && length <= range0Len)
-	    return( kIOReturnSuccess );
-    }
-
-    return( super::doUnmap( addressMap, logical, length ));
+    return (super::doUnmap(addressMap, __address, __length));
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -2206,108 +2275,63 @@ OSDefineMetaClassAndStructors(_IOMemoryMap, IOMemoryMap)
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
-bool _IOMemoryMap::initCompatible(
-	IOMemoryDescriptor *	_memory,
-	IOMemoryMap *		_superMap,
-        IOByteCount		_offset,
-        IOByteCount		_length )
+bool _IOMemoryMap::init(
+        task_t			intoTask,
+        mach_vm_address_t	toAddress,
+        IOOptionBits		_options,
+        mach_vm_size_t		_offset,
+        mach_vm_size_t		_length )
 {
-
-    if( !super::init())
+    if (!intoTask)
 	return( false);
 
-    if( (_offset + _length) > _superMap->getLength())
-	return( false);
+    if (!super::init())
+	return(false);
 
-    _memory->retain();
-    memory	= _memory;
-    _superMap->retain();
-    superMap 	= _superMap;
+    fAddressMap  = get_task_map(intoTask);
+    if (!fAddressMap)
+	return(false);
+    vm_map_reference(fAddressMap);
 
-    offset	= _offset;
-    if( _length)
-        length	= _length;
-    else
-        length	= _memory->getLength();
-
-    options	= superMap->getMapOptions();
-    logical	= superMap->getVirtualAddress() + offset;
+    fAddressTask = intoTask;
+    fOptions     = _options;
+    fLength      = _length;
+    fOffset	 = _offset;
+    fAddress     = toAddress;
 
-    return( true );
+    return (true);
 }
 
-bool _IOMemoryMap::initWithDescriptor(
-        IOMemoryDescriptor *	_memory,
-        task_t			intoTask,
-        IOVirtualAddress	toAddress,
-        IOOptionBits		_options,
-        IOByteCount		_offset,
-        IOByteCount		_length )
+bool _IOMemoryMap::setMemoryDescriptor(IOMemoryDescriptor * _memory, mach_vm_size_t _offset)
 {
-    bool ok;
-    bool redir = ((kIOMapUnique|kIOMapReference) == ((kIOMapUnique|kIOMapReference) & _options));
+    if (!_memory)
+	return(false);
 
-    if ((!_memory) || (!intoTask))
-	return( false);
-
-    if( (_offset + _length) > _memory->getLength())
-	return( false);
-
-    if (!redir)
+    if (!fSuperMap)
     {
-	if (!super::init())
-	    return(false);
-	addressMap  = get_task_map(intoTask);
-	if( !addressMap)
+	if( (_offset + fLength) > _memory->getLength())
 	    return( false);
-	vm_map_reference(addressMap);
-	addressTask = intoTask;
-	logical	    = toAddress;
-	options     = _options;
+	fOffset = _offset;
     }
 
     _memory->retain();
-
-    offset	= _offset;
-    if( _length)
-        length	= _length;
-    else
-        length	= _memory->getLength();
-
-    if( options & kIOMapStatic)
-	ok = true;
-    else
-	ok = (kIOReturnSuccess == _memory->doMap( addressMap, &toAddress,
-						  _options, offset, length ));
-    if (ok || redir)
-    {
-	if (memory)
-	    memory->release();
-	memory	= _memory;
-	logical = toAddress;
-    }
-    else
+    if (fMemory)
     {
-        _memory->release();
-	if (!redir)
-	{
-	    logical = 0;
-	    memory = 0;
-	    vm_map_deallocate(addressMap);
-	    addressMap = 0;
-	}
+	if (fMemory != _memory)
+	    fMemory->removeMapping(this);
+	fMemory->release();
     }
+    fMemory = _memory;
 
-    return( ok );
+    return( true );
 }
 
-/* LP64todo - these need to expand */
 struct IOMemoryDescriptorMapAllocRef
 {
     ipc_port_t		sharedMem;
-    vm_size_t		size;
-    vm_offset_t		mapped;
-    IOByteCount		sourceOffset;
+    mach_vm_address_t	mapped;
+    mach_vm_size_t	size;
+    mach_vm_size_t	sourceOffset;
     IOOptionBits	options;
 };
 
@@ -2317,10 +2341,15 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
     IOReturn			    err;
 
     do {
-        if( ref->sharedMem) {
+        if( ref->sharedMem)
+	{
             vm_prot_t prot = VM_PROT_READ
                             | ((ref->options & kIOMapReadOnly) ? 0 : VM_PROT_WRITE);
 
+	    // VM system requires write access to change cache mode
+	    if (kIOMapDefaultCache != (ref->options & kIOMapCacheMask))
+		prot |= VM_PROT_WRITE;
+
             // set memory entry cache
             vm_prot_t memEntryCacheMode = prot | MAP_MEM_ONLY;
             switch (ref->options & kIOMapCacheMask)
@@ -2354,7 +2383,7 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
             if (KERN_SUCCESS != err)
                 IOLog("MAP_MEM_ONLY failed %d\n", err);
 
-            err = vm_map( map,
+            err = mach_vm_map( map,
                             &ref->mapped,
                             ref->size, 0 /* mask */, 
                             (( ref->options & kIOMapAnywhere ) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
@@ -2370,72 +2399,97 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
                 continue;
             }
     
-        } else {
-    
-            err = vm_allocate( map, &ref->mapped, ref->size,
+        }
+	else
+	{
+            err = mach_vm_allocate( map, &ref->mapped, ref->size,
                             ((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
                             | VM_MAKE_TAG(VM_MEMORY_IOKIT) );
-    
             if( KERN_SUCCESS != err) {
                 ref->mapped = 0;
                 continue;
             }
-    
             // we have to make sure that these guys don't get copied if we fork.
             err = vm_inherit( map, ref->mapped, ref->size, VM_INHERIT_NONE);
             assert( KERN_SUCCESS == err );
         }
-
-    } while( false );
+    }
+    while( false );
 
     return( err );
 }
 
+kern_return_t 
+IOMemoryDescriptorMapMemEntry(vm_map_t map, ipc_port_t entry, IOOptionBits options, bool pageable,
+				mach_vm_size_t offset, 
+				mach_vm_address_t * address, mach_vm_size_t length)
+{
+    IOReturn err;
+    IOMemoryDescriptorMapAllocRef ref;
+
+    ref.sharedMem	= entry;
+    ref.sourceOffset   	= offset;
+    ref.options		= options;
+
+    ref.size = length;
+
+    if (options & kIOMapAnywhere)
+	// vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
+	ref.mapped = 0;
+    else
+	ref.mapped = *address;
+
+    if( ref.sharedMem && (map == kernel_map) && pageable)
+	err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
+    else
+	err = IOMemoryDescriptorMapAlloc( map, &ref );
+
+    *address = ref.mapped;
+    return (err);
+}
+
 
 IOReturn IOMemoryDescriptor::doMap(
-	vm_map_t		addressMap,
-	IOVirtualAddress *	atAddress,
+	vm_map_t		__addressMap,
+	IOVirtualAddress *	__address,
 	IOOptionBits		options,
-	IOByteCount		sourceOffset,
-	IOByteCount		length )
+	IOByteCount		__offset,
+	IOByteCount		__length )
 {
-    IOReturn		err = kIOReturnSuccess;
-    memory_object_t	pager;
-    vm_address_t	logical;
-    IOByteCount		pageOffset;
-    IOPhysicalAddress	sourceAddr;
-    IOMemoryDescriptorMapAllocRef	ref;
-
-    ref.sharedMem	= (ipc_port_t) _memEntry;
-    ref.sourceOffset	= sourceOffset;
-    ref.options		= options;
+    if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::doMap !64bit");
 
-    do {
+    _IOMemoryMap * mapping = (_IOMemoryMap *) *__address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;
 
-        if( 0 == length)
-            length = getLength();
+    IOReturn	      err = kIOReturnSuccess;
+    memory_object_t   pager;
+    mach_vm_size_t    pageOffset;
+    IOPhysicalAddress sourceAddr;
 
-	sourceAddr = getSourceSegment( sourceOffset, NULL );
+    do
+    {
+	sourceAddr = getSourceSegment( offset, NULL );
 	pageOffset = sourceAddr - trunc_page_32( sourceAddr );
 
-	ref.size = round_page_32( length + pageOffset );
+	if( reserved)
+	    pager = (memory_object_t) reserved->devicePager;
+	else
+	    pager = MACH_PORT_NULL;
 
 	if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))
 	{
-	    upl_t	  redirUPL2;
-	    vm_size_t	  size;
-	    int		  flags;
+	    upl_t	   redirUPL2;
+	    vm_size_t      size;
+	    int		   flags;
 
-	    _IOMemoryMap * mapping = (_IOMemoryMap *) *atAddress;
-	    ref.mapped = mapping->getVirtualAddress();
-    
 	    if (!_memEntry)
 	    {
 		err = kIOReturnNotReadable;
 		continue;
 	    }
 
-	    size = length;
+	    size = mapping->fLength + pageOffset;
 	    flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
 			| UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
 
@@ -2444,7 +2498,7 @@ IOReturn IOMemoryDescriptor::doMap(
 					    &flags))
 		redirUPL2 = NULL;
 
-	    err = upl_transpose(redirUPL2, mapping->redirUPL);
+	    err = upl_transpose(redirUPL2, mapping->fRedirUPL);
 	    if (kIOReturnSuccess != err)
 	    {
 		IOLog("upl_transpose(%x)\n", err);
@@ -2460,52 +2514,54 @@ IOReturn IOMemoryDescriptor::doMap(
 	    {
 		// swap the memEntries since they now refer to different vm_objects
 		void * me = _memEntry;
-		_memEntry = mapping->memory->_memEntry;
-		mapping->memory->_memEntry = me;
+		_memEntry = mapping->fMemory->_memEntry;
+		mapping->fMemory->_memEntry = me;
 	    }
+	    if (pager)
+		err = handleFault( reserved->devicePager, mapping->fAddressMap, mapping->fAddress, offset, length, options );
 	}
 	else
 	{
-    
-	    logical = *atAddress;
-	    if( options & kIOMapAnywhere) 
-		// vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
-		ref.mapped = 0;
-	    else {
-		ref.mapped = trunc_page_32( logical );
-		if( (logical - ref.mapped) != pageOffset) {
+	    mach_vm_address_t address;
+
+	    if (!(options & kIOMapAnywhere))
+	    {
+		address = trunc_page_64(mapping->fAddress);
+		if( (mapping->fAddress - address) != pageOffset)
+		{
 		    err = kIOReturnVMError;
 		    continue;
 		}
 	    }
-    
-	    if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-		err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
-	    else
-		err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
-	}
-
-        if( err != KERN_SUCCESS)
-            continue;
 
-        if( reserved)
-            pager = (memory_object_t) reserved->devicePager;
-        else
-            pager = MACH_PORT_NULL;
+	    err = IOMemoryDescriptorMapMemEntry(mapping->fAddressMap, (ipc_port_t) _memEntry,
+						    options, (kIOMemoryBufferPageable & _flags),
+						    offset, &address, round_page_64(length + pageOffset));
+	    if( err != KERN_SUCCESS)
+		continue;
 
-        if( !ref.sharedMem || pager )
-            err = handleFault( pager, addressMap, ref.mapped, sourceOffset, length, options );
+	    if (!_memEntry || pager)
+	    {
+		err = handleFault( pager, mapping->fAddressMap, address, offset, length, options );
+		if (err != KERN_SUCCESS)
+		    doUnmap( mapping->fAddressMap, (IOVirtualAddress) mapping, 0 );
+	    }
 
-    } while( false );
+#ifdef DEBUG
+	if (kIOLogMapping & gIOKitDebug)
+	    IOLog("mapping(%x) desc %p @ %lx, map %p, address %qx, offset %qx, length %qx\n", 
+		    err, this, sourceAddr, mapping, address, offset, length);
+#endif
 
-    if( err != KERN_SUCCESS) {
-        if( ref.mapped)
-            doUnmap( addressMap, ref.mapped, ref.size );
-        *atAddress = NULL;
-    } else
-        *atAddress = ref.mapped + pageOffset;
+	    if (err == KERN_SUCCESS)
+		mapping->fAddress = address + pageOffset;
+	    else
+		mapping->fAddress = NULL;
+	}
+    }
+    while( false );
 
-    return( err );
+    return (err);
 }
 
 enum {
@@ -2515,26 +2571,27 @@ enum {
 IOReturn IOMemoryDescriptor::handleFault(
         void *			_pager,
 	vm_map_t		addressMap,
-	IOVirtualAddress	address,
-	IOByteCount		sourceOffset,
-	IOByteCount		length,
+	mach_vm_address_t	address,
+	mach_vm_size_t		sourceOffset,
+	mach_vm_size_t		length,
         IOOptionBits		options )
 {
     IOReturn		err = kIOReturnSuccess;
     memory_object_t	pager = (memory_object_t) _pager;
-    vm_size_t		size;
-    vm_size_t		bytes;
-    vm_size_t		page;
-    IOByteCount		pageOffset;
-    IOByteCount		pagerOffset;
+    mach_vm_size_t	size;
+    mach_vm_size_t	bytes;
+    mach_vm_size_t	page;
+    mach_vm_size_t	pageOffset;
+    mach_vm_size_t	pagerOffset;
     IOPhysicalLength	segLen;
     addr64_t		physAddr;
 
-    if( !addressMap) {
-
-        if( kIOMemoryRedirected & _flags) {
+    if( !addressMap)
+    {
+        if( kIOMemoryRedirected & _flags)
+	{
 #ifdef DEBUG
-            IOLog("sleep mem redirect %p, %lx\n", this, sourceOffset);
+            IOLog("sleep mem redirect %p, %qx\n", this, sourceOffset);
 #endif
             do {
 	    	SLEEP;
@@ -2554,7 +2611,8 @@ IOReturn IOMemoryDescriptor::handleFault(
 
     segLen += pageOffset;
     bytes = size;
-    do {
+    do
+    {
 	// in the middle of the loop only map whole pages
 	if( segLen >= bytes)
 	    segLen = bytes;
@@ -2567,11 +2625,12 @@ IOReturn IOMemoryDescriptor::handleFault(
 
 #ifdef DEBUG
 	if( kIOLogMapping & gIOKitDebug)
-	    IOLog("_IOMemoryMap::map(%p) %08lx->%08qx:%08lx\n",
+	    IOLog("_IOMemoryMap::map(%p) 0x%qx->0x%qx:0x%qx\n",
                 addressMap, address + pageOffset, physAddr + pageOffset,
 		segLen - pageOffset);
 #endif
 
+
         if( pager) {
             if( reserved && reserved->pagerContig) {
                 IOPhysicalLength	allLen;
@@ -2579,16 +2638,18 @@ IOReturn IOMemoryDescriptor::handleFault(
 
                 allPhys = getPhysicalSegment64( 0, &allLen );
                 assert( allPhys );
-                err = device_pager_populate_object( pager, 0, allPhys >> PAGE_SHIFT, round_page_32(allLen) );
-
-            } else {
+		err = device_pager_populate_object( pager, 0, atop_64(allPhys), round_page_32(allLen) );
+            }
+	    else
+	    {
 
-	    for( page = 0;
+		for( page = 0;
                      (page < segLen) && (KERN_SUCCESS == err);
-                     page += page_size) {
-                        err = device_pager_populate_object(pager, pagerOffset,
-                        	(ppnum_t)((physAddr + page) >> PAGE_SHIFT), page_size);
-			pagerOffset += page_size;
+                     page += page_size)
+		{
+		    err = device_pager_populate_object(pager, pagerOffset,
+			    (ppnum_t)(atop_64(physAddr + page)), page_size);
+		    pagerOffset += page_size;
                 }
             }
             assert( KERN_SUCCESS == err );
@@ -2596,24 +2657,14 @@ IOReturn IOMemoryDescriptor::handleFault(
                 break;
         }
 
+	// This call to vm_fault causes an early pmap level resolution
+	// of the mappings created above for kernel mappings, since
+	// faulting in later can't take place from interrupt level.
 	/*  *** ALERT *** */
 	/*  *** Temporary Workaround *** */
 
-	/* This call to vm_fault causes an early pmap level resolution	*/
-	/* of the mappings created above.  Need for this is in absolute	*/
-	/* violation of the basic tenet that the pmap layer is a cache.	*/
-	/* Further, it implies a serious I/O architectural violation on	*/
-	/* the part of some user of the mapping.  As of this writing, 	*/
-	/* the call to vm_fault is needed because the NVIDIA driver 	*/
-	/* makes a call to pmap_extract.  The NVIDIA driver needs to be	*/
-	/* fixed as soon as possible.  The NVIDIA driver should not 	*/
-	/* need to query for this info as it should know from the doMap	*/
-	/* call where the physical memory is mapped.  When a query is 	*/
-	/* necessary to find a physical mapping, it should be done 	*/
-	/* through an iokit call which includes the mapped memory 	*/
-	/* handle.  This is required for machine architecture independence.*/
-
-	if(!(kIOMemoryRedirected & _flags)) {
+	if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags))
+	{
 		vm_fault(addressMap, 
 			 (vm_map_offset_t)address, 
 			 VM_PROT_READ|VM_PROT_WRITE, 
@@ -2629,39 +2680,48 @@ IOReturn IOMemoryDescriptor::handleFault(
 	bytes -= segLen;
 	pageOffset = 0;
 
-    } while( bytes
-	&& (physAddr = getPhysicalSegment64( sourceOffset, &segLen )));
+    } 
+    while (bytes && (physAddr = getPhysicalSegment64( sourceOffset, &segLen )));
 
-    if( bytes)
+    if (bytes)
         err = kIOReturnBadArgument;
 
-    return( err );
+    return (err);
 }
 
 IOReturn IOMemoryDescriptor::doUnmap(
 	vm_map_t		addressMap,
-	IOVirtualAddress	logical,
-	IOByteCount		length )
+	IOVirtualAddress	__address,
+	IOByteCount		__length )
 {
-    IOReturn	err;
+    IOReturn	      err;
+    mach_vm_address_t address;
+    mach_vm_size_t    length;
+
+    if (__length)
+    {
+	address = __address;
+	length  = __length;
+    }
+    else
+    {
+	addressMap = ((_IOMemoryMap *) __address)->fAddressMap;
+	address    = ((_IOMemoryMap *) __address)->fAddress;
+	length     = ((_IOMemoryMap *) __address)->fLength;
+    }
+
+    if( _memEntry && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
+	addressMap = IOPageableMapForAddress( address );
 
 #ifdef DEBUG
     if( kIOLogMapping & gIOKitDebug)
-	kprintf("IOMemoryDescriptor::doUnmap(%x) %08x:%08x\n",
-                addressMap, logical, length );
+	IOLog("IOMemoryDescriptor::doUnmap map %p, 0x%qx:0x%qx\n",
+		addressMap, address, length );
 #endif
 
-    if( true /* && (addressMap == kernel_map) || (addressMap == get_task_map(current_task()))*/) {
-
-        if( _memEntry && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-            addressMap = IOPageableMapForAddress( logical );
-
-        err = vm_deallocate( addressMap, logical, length );
-
-    } else
-        err = kIOReturnSuccess;
+    err = mach_vm_deallocate( addressMap, address, length );
 
-    return( err );
+    return (err);
 }
 
 IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
@@ -2712,7 +2772,7 @@ IOReturn _IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
 {
     IOReturn err = kIOReturnSuccess;
 
-    if( superMap) {
+    if( fSuperMap) {
 //        err = ((_IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );
     } else {
 
@@ -2720,46 +2780,46 @@ IOReturn _IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
 
 	do
 	{
-	    if (!logical)
+	    if (!fAddress)
 		break;
-	    if (!addressMap)
+	    if (!fAddressMap)
 		break;
 
-	    if ((!safeTask || (get_task_map(safeTask) != addressMap))
-	      && (0 == (options & kIOMapStatic)))
+	    if ((!safeTask || (get_task_map(safeTask) != fAddressMap))
+	      && (0 == (fOptions & kIOMapStatic)))
 	    {
-		IOUnmapPages( addressMap, logical, length );
+		IOUnmapPages( fAddressMap, fAddress, fLength );
 		if(!doRedirect && safeTask
-		 && (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) 
-		    || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64)))
+		 && (((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) 
+		    || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64)))
 		 {
-		    err = vm_deallocate( addressMap, logical, length );
-		    err = memory->doMap( addressMap, &logical,
-					 (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
-					 offset, length );
+		    IOVirtualAddress iova = (IOVirtualAddress) this;
+		    err = mach_vm_deallocate( fAddressMap, fAddress, fLength );
+		    err = fMemory->doMap( fAddressMap, &iova,
+					 (fOptions & ~kIOMapAnywhere) | kIOMap64Bit/*| kIOMapReserve*/,
+					 0, 0 );
 		} else
 		    err = kIOReturnSuccess;
 #ifdef DEBUG
-		IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", doRedirect, this, logical, length, addressMap);
+		IOLog("IOMemoryMap::redirect(%d, %p) 0x%qx:0x%qx from %p\n", doRedirect, this, fAddress, fLength, fAddressMap);
 #endif
 	    }
-	    else if (kIOMapWriteCombineCache == (options & kIOMapCacheMask))
+	    else if (kIOMapWriteCombineCache == (fOptions & kIOMapCacheMask))
 	    {
 		IOOptionBits newMode;
-		newMode = (options & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
-		IOProtectCacheMode(addressMap, logical, length, newMode);
+		newMode = (fOptions & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
+		IOProtectCacheMode(fAddressMap, fAddress, fLength, newMode);
 	    }
 	}
 	while (false);
-
 	UNLOCK;
     }
 
-    if ((((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-	 || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+    if ((((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	 || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
      && safeTask
-     && (doRedirect != (0 != (memory->_flags & kIOMemoryRedirected))))
-	memory->redirect(safeTask, doRedirect);
+     && (doRedirect != (0 != (fMemory->_flags & kIOMemoryRedirected))))
+	fMemory->redirect(safeTask, doRedirect);
 
     return( err );
 }
@@ -2770,17 +2830,21 @@ IOReturn _IOMemoryMap::unmap( void )
 
     LOCK;
 
-    if( logical && addressMap && (0 == superMap)
-	&& (0 == (options & kIOMapStatic))) {
+    if( fAddress && fAddressMap && (0 == fSuperMap) && fMemory
+	&& (0 == (fOptions & kIOMapStatic))) {
 
-        err = memory->doUnmap( addressMap, logical, length );
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+        err = fMemory->doUnmap(fAddressMap, (IOVirtualAddress) this, 0);
 
     } else
 	err = kIOReturnSuccess;
 
-    logical = 0;
+    if (fAddressMap)
+    {
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;
+    }
+
+    fAddress = 0;
 
     UNLOCK;
 
@@ -2790,12 +2854,12 @@ IOReturn _IOMemoryMap::unmap( void )
 void _IOMemoryMap::taskDied( void )
 {
     LOCK;
-    if( addressMap) {
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+    if( fAddressMap) {
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;
     }
-    addressTask	= 0;
-    logical	= 0;
+    fAddressTask = 0;
+    fAddress	 = 0;
     UNLOCK;
 }
 
@@ -2814,26 +2878,27 @@ void _IOMemoryMap::free()
 {
     unmap();
 
-    if( memory) {
+    if (fMemory)
+    {
         LOCK;
-	memory->removeMapping( this);
+	fMemory->removeMapping(this);
 	UNLOCK;
-	memory->release();
+	fMemory->release();
     }
 
-    if (owner && (owner != memory))
+    if (fOwner && (fOwner != fMemory))
     {
         LOCK;
-	owner->removeMapping(this);
+	fOwner->removeMapping(this);
 	UNLOCK;
     }
 
-    if( superMap)
-	superMap->release();
+    if (fSuperMap)
+	fSuperMap->release();
 
-    if (redirUPL) {
-	upl_commit(redirUPL, NULL, 0);
-	upl_deallocate(redirUPL);
+    if (fRedirUPL) {
+	upl_commit(fRedirUPL, NULL, 0);
+	upl_deallocate(fRedirUPL);
     }
 
     super::free();
@@ -2841,77 +2906,92 @@ void _IOMemoryMap::free()
 
 IOByteCount _IOMemoryMap::getLength()
 {
-    return( length );
+    return( fLength );
 }
 
 IOVirtualAddress _IOMemoryMap::getVirtualAddress()
 {
-    return( logical);
+    if (fSuperMap)
+	fSuperMap->getVirtualAddress();
+    else if (fAddressMap && vm_map_is_64bit(fAddressMap))
+    {
+	OSReportWithBacktrace("IOMemoryMap::getVirtualAddress(0x%qx) called on 64b map; use ::getAddress()", fAddress);
+    }
+
+    return (fAddress);
+}
+
+mach_vm_address_t 	_IOMemoryMap::getAddress()
+{
+    return( fAddress);
+}
+
+mach_vm_size_t 	_IOMemoryMap::getSize()
+{
+    return( fLength );
 }
 
+
 task_t _IOMemoryMap::getAddressTask()
 {
-    if( superMap)
-	return( superMap->getAddressTask());
+    if( fSuperMap)
+	return( fSuperMap->getAddressTask());
     else
-        return( addressTask);
+        return( fAddressTask);
 }
 
 IOOptionBits _IOMemoryMap::getMapOptions()
 {
-    return( options);
+    return( fOptions);
 }
 
 IOMemoryDescriptor * _IOMemoryMap::getMemoryDescriptor()
 {
-    return( memory );
+    return( fMemory );
 }
 
 _IOMemoryMap * _IOMemoryMap::copyCompatible(
-		IOMemoryDescriptor *	owner,
-                task_t			task,
-                IOVirtualAddress	toAddress,
-                IOOptionBits		_options,
-                IOByteCount		_offset,
-                IOByteCount		_length )
+		_IOMemoryMap * newMapping )
 {
-    _IOMemoryMap * mapping;
+    task_t		task      = newMapping->getAddressTask();
+    mach_vm_address_t	toAddress = newMapping->fAddress;
+    IOOptionBits	_options  = newMapping->fOptions;
+    mach_vm_size_t	_offset   = newMapping->fOffset;
+    mach_vm_size_t	_length   = newMapping->fLength;
 
-    if( (!task) || (!addressMap) || (addressMap != get_task_map(task)))
+    if( (!task) || (!fAddressMap) || (fAddressMap != get_task_map(task)))
 	return( 0 );
-    if( options & kIOMapUnique)
-	return( 0 );
-    if( (options ^ _options) & kIOMapReadOnly)
+    if( (fOptions ^ _options) & kIOMapReadOnly)
 	return( 0 );
     if( (kIOMapDefaultCache != (_options & kIOMapCacheMask)) 
-     && ((options ^ _options) & kIOMapCacheMask))
+     && ((fOptions ^ _options) & kIOMapCacheMask))
 	return( 0 );
 
-    if( (0 == (_options & kIOMapAnywhere)) && (logical != toAddress))
+    if( (0 == (_options & kIOMapAnywhere)) && (fAddress != toAddress))
 	return( 0 );
 
-    if( _offset < offset)
+    if( _offset < fOffset)
 	return( 0 );
 
-    _offset -= offset;
+    _offset -= fOffset;
 
-    if( (_offset + _length) > length)
+    if( (_offset + _length) > fLength)
 	return( 0 );
 
-    if( (length == _length) && (!_offset)) {
-        retain();
-	mapping = this;
-
-    } else {
-        mapping = new _IOMemoryMap;
-        if( mapping
-        && !mapping->initCompatible( owner, this, _offset, _length )) {
-            mapping->release();
-            mapping = 0;
-        }
+    retain();
+    if( (fLength == _length) && (!_offset))
+    {
+	newMapping->release();
+	newMapping = this;
+    }
+    else
+    {
+	newMapping->fSuperMap = this;
+	newMapping->fOffset   = _offset;
+	newMapping->fAddress  = fAddress + _offset;
     }
 
-    return( mapping );
+    return( newMapping );
 }
 
 IOPhysicalAddress 
@@ -2920,7 +3000,7 @@ _IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _lengt
     IOPhysicalAddress	address;
 
     LOCK;
-    address = memory->getPhysicalSegment( offset + _offset, _length );
+    address = fMemory->getPhysicalSegment( fOffset + _offset, _length );
     UNLOCK;
 
     return( address );
@@ -2969,97 +3049,129 @@ IOMemoryMap * IOMemoryDescriptor::setMapping(
 	IOVirtualAddress	mapAddress,
 	IOOptionBits		options )
 {
-    _IOMemoryMap *		newMap;
-
-    newMap = new _IOMemoryMap;
-
-    LOCK;
-
-    if( newMap
-     && !newMap->initWithDescriptor( this, intoTask, mapAddress,
-                    options | kIOMapStatic, 0, getLength() )) {
-	newMap->release();
-	newMap = 0;
-    }
-
-    addMapping( newMap);
-
-    UNLOCK;
-
-    return( newMap);
+    return (createMappingInTask( intoTask, mapAddress,
+				    options | kIOMapStatic,
+				    0, getLength() ));
 }
 
 IOMemoryMap * IOMemoryDescriptor::map( 
 	IOOptionBits		options )
 {
-
-    return( makeMapping( this, kernel_task, 0,
-			options | kIOMapAnywhere,
-			0, getLength() ));
+    return (createMappingInTask( kernel_task, 0,
+				options | kIOMapAnywhere,
+				0, getLength() ));
 }
 
-IOMemoryMap * IOMemoryDescriptor::map(
-	task_t			intoTask,
-	IOVirtualAddress	toAddress,
+IOMemoryMap * IOMemoryDescriptor::map( 
+	task_t		        intoTask,
+	IOVirtualAddress	atAddress,
 	IOOptionBits		options,
 	IOByteCount		offset,
 	IOByteCount		length )
 {
-    if( 0 == length)
+    if ((!(kIOMapAnywhere & options)) && vm_map_is_64bit(get_task_map(intoTask)))
+    {
+	OSReportWithBacktrace("IOMemoryDescriptor::map() in 64b task, use ::createMappingInTask()");
+	return (0);
+    }
+
+    return (createMappingInTask(intoTask, atAddress,
+				options, offset, length));
+}
+
+IOMemoryMap * IOMemoryDescriptor::createMappingInTask(
+	task_t			intoTask,
+	mach_vm_address_t	atAddress,
+	IOOptionBits		options,
+	mach_vm_size_t		offset,
+	mach_vm_size_t		length)
+{
+    IOMemoryMap  * result;
+    _IOMemoryMap * mapping;
+
+    if (0 == length)
 	length = getLength();
 
-    return( makeMapping( this, intoTask, toAddress, options, offset, length ));
+    mapping = new _IOMemoryMap;
+
+    if( mapping
+     && !mapping->init( intoTask, atAddress,
+			options, offset, length )) {
+	mapping->release();
+	mapping = 0;
+    }
+
+    if (mapping)
+	result = makeMapping(this, intoTask, (IOVirtualAddress) mapping, options | kIOMap64Bit, 0, 0);
+    else
+	result = 0;
+
+#ifdef DEBUG
+    if (!result)
+	IOLog("createMappingInTask failed desc %p, addr %qx, options %lx, offset %qx, length %qx\n",
+		    this, atAddress, options, offset, length);
+#endif
+
+    return (result);
 }
 
 IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
 			        IOOptionBits         options,
 			        IOByteCount          offset)
+{
+    return (redirect(newBackingMemory, options, (mach_vm_size_t)offset));
+}
+
+IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
+			        IOOptionBits         options,
+			        mach_vm_size_t       offset)
 {
     IOReturn err = kIOReturnSuccess;
     IOMemoryDescriptor * physMem = 0;
 
     LOCK;
 
-    if (logical && addressMap) do 
+    if (fAddress && fAddressMap) do 
     {
-	if (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-	    || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+	if (((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+	    || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
 	{
-	    physMem = memory;
+	    physMem = fMemory;
 	    physMem->retain();
 	}
 
-	if (!redirUPL)
+	if (!fRedirUPL)
 	{
-	    vm_size_t size = length;
+	    vm_size_t size = fLength;
 	    int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
 			| UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
-	    if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) memory->_memEntry, 0, &size, &redirUPL,
+	    if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) fMemory->_memEntry, 0, &size, &fRedirUPL,
 					    NULL, NULL,
 					    &flags))
-		redirUPL = 0;
+		fRedirUPL = 0;
 
 	    if (physMem)
 	    {
-		IOUnmapPages( addressMap, logical, length );
+		IOUnmapPages( fAddressMap, fAddress, fLength );
 		physMem->redirect(0, true);
 	    }
 	}
 
 	if (newBackingMemory)
 	{
-	    if (newBackingMemory != memory)
+	    if (newBackingMemory != fMemory)
 	    {
-		if (this != newBackingMemory->makeMapping(newBackingMemory, addressTask, (IOVirtualAddress) this, 
-							    options | kIOMapUnique | kIOMapReference,
-							    offset, length))
+		fOffset = 0;
+		if (this != newBackingMemory->makeMapping(newBackingMemory, fAddressTask, (IOVirtualAddress) this, 
+							    options | kIOMapUnique | kIOMapReference | kIOMap64Bit,
+							    offset, fLength))
 		    err = kIOReturnError;
 	    }
-	    if (redirUPL)
+	    if (fRedirUPL)
 	    {
-		upl_commit(redirUPL, NULL, 0);
-		upl_deallocate(redirUPL);
-		redirUPL = 0;
+		upl_commit(fRedirUPL, NULL, 0);
+		upl_deallocate(fRedirUPL);
+		fRedirUPL = 0;
 	    }
 	    if (physMem)
 		physMem->redirect(0, false);
@@ -3077,27 +3189,42 @@ IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
 
 IOMemoryMap * IOMemoryDescriptor::makeMapping(
 	IOMemoryDescriptor *	owner,
-	task_t			intoTask,
-	IOVirtualAddress	toAddress,
+	task_t			__intoTask,
+	IOVirtualAddress	__address,
 	IOOptionBits		options,
-	IOByteCount		offset,
-	IOByteCount		length )
+	IOByteCount		__offset,
+	IOByteCount		__length )
 {
+    if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::makeMapping !64bit");
+
     IOMemoryDescriptor * mapDesc = 0;
-    _IOMemoryMap *	mapping = 0;
-    OSIterator *	iter;
+    _IOMemoryMap *	 result = 0;
+    OSIterator *	 iter;
+
+    _IOMemoryMap * mapping = (_IOMemoryMap *) __address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;
+
+    mapping->fOffset = offset;
 
     LOCK;
 
     do
     {
+	if (kIOMapStatic & options)
+	{
+	    result = mapping;
+	    addMapping(mapping);
+	    mapping->setMemoryDescriptor(this, 0);
+	    continue;
+	}
+
 	if (kIOMapUnique & options)
 	{
 	    IOPhysicalAddress phys;
 	    IOByteCount       physLen;
 
-	    if (owner != this)
-		continue;
+//	    if (owner != this)		continue;
 
 	    if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
 		|| ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
@@ -3111,107 +3238,64 @@ IOMemoryMap * IOMemoryDescriptor::makeMapping(
 		if (!mapDesc)
 		    continue;
 		offset = 0;
-	    }
-	    else
-	    {
-		mapDesc = this;
-		mapDesc->retain();
-	    }
-
-	    if (kIOMapReference & options)
-	    {
-		mapping = (_IOMemoryMap *) toAddress;
-		mapping->retain();
-
-#if 1
-		uint32_t pageOffset1 = mapDesc->getSourceSegment( offset, NULL );
-		pageOffset1 -= trunc_page_32( pageOffset1 );
-
-		uint32_t pageOffset2 = mapping->getVirtualAddress();
-		pageOffset2 -= trunc_page_32( pageOffset2 );
-		
-		if (pageOffset1 != pageOffset2)
-		    IOLog("::redirect can't map offset %x to addr %x\n", 
-			    pageOffset1, mapping->getVirtualAddress());
-#endif
-
-
-		if (!mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
-						    offset, length ))
-		{
-#ifdef DEBUG
-		    IOLog("Didn't redirect map %08lx : %08lx\n", offset, length );
-#endif
-		}
-
-		if (mapping->owner)
-		    mapping->owner->removeMapping(mapping);
-		continue;
+		mapping->fOffset = offset;
 	    }
 	}
 	else
 	{
-	    // look for an existing mapping
-	    if( (iter = OSCollectionIterator::withCollection( _mappings))) {
-    
-		while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
-    
-		    if( (mapping = mapping->copyCompatible( 
-					    owner, intoTask, toAddress,
-					    options | kIOMapReference,
-					    offset, length )))
+	    // look for a compatible existing mapping
+	    if( (iter = OSCollectionIterator::withCollection(_mappings)))
+	    {
+		_IOMemoryMap * lookMapping;
+		while ((lookMapping = (_IOMemoryMap *) iter->getNextObject()))
+		{
+		    if ((result = lookMapping->copyCompatible(mapping)))
+		    {
+			addMapping(result);
+			result->setMemoryDescriptor(this, offset);
 			break;
+		    }
 		}
 		iter->release();
 	    }
-
-
-	if (mapping)
-	    mapping->retain();
-
-	    if( mapping || (options & kIOMapReference))
+	    if (result || (options & kIOMapReference))
 		continue;
+	}
 
-	    mapDesc = owner;
+	if (!mapDesc)
+	{
+	    mapDesc = this;
 	    mapDesc->retain();
 	}
-	owner = this;
-
-        mapping = new _IOMemoryMap;
-	if( mapping
-	&& !mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
-			   offset, length )) {
-#ifdef DEBUG
-	    IOLog("Didn't make map %08lx : %08lx\n", offset, length );
-#endif
+	IOReturn
+	kr = mapDesc->doMap( 0, (IOVirtualAddress *) &mapping, options, 0, 0 );
+	if (kIOReturnSuccess == kr)
+	{
+	    result = mapping;
+	    mapDesc->addMapping(result);
+	    result->setMemoryDescriptor(mapDesc, offset);
+	}
+	else
+	{
 	    mapping->release();
-            mapping = 0;
+	    mapping = NULL;
 	}
-
-	if (mapping)
-	    mapping->retain();
-
-    } while( false );
-
-    if (mapping)
-    {
-	mapping->owner = owner;
-	owner->addMapping( mapping);
-	mapping->release();
     }
+    while( false );
 
     UNLOCK;
 
     if (mapDesc)
 	mapDesc->release();
 
-    return( mapping);
+    return (result);
 }
 
 void IOMemoryDescriptor::addMapping(
 	IOMemoryMap * mapping )
 {
-    if( mapping) {
+    if( mapping)
+    {
         if( 0 == _mappings)
             _mappings = OSSet::withCapacity(1);
 	if( _mappings )
@@ -3368,19 +3452,6 @@ IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset, IOByteCount * len
     return( address );
 }
 
-
-IOReturn IOSubMemoryDescriptor::doMap(
-	vm_map_t		addressMap,
-	IOVirtualAddress *	atAddress,
-	IOOptionBits		options,
-	IOByteCount		sourceOffset,
-	IOByteCount		length )
-{
-    if( sourceOffset >= _length)
-        return( kIOReturnOverrun );
-    return (_parent->doMap(addressMap, atAddress, options, sourceOffset + _start, length));
-}
-
 IOPhysicalAddress 
 IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset, IOByteCount * length )
 {
@@ -3409,6 +3480,17 @@ void * IOSubMemoryDescriptor::getVirtualSegment(IOByteCount offset,
     return( 0 );
 }
 
+IOReturn IOSubMemoryDescriptor::doMap(
+	vm_map_t		addressMap,
+	IOVirtualAddress *	atAddress,
+	IOOptionBits		options,
+	IOByteCount		sourceOffset,
+	IOByteCount		length )
+{
+    panic("IOSubMemoryDescriptor::doMap");
+    return (IOMemoryDescriptor::doMap(addressMap, atAddress, options, sourceOffset, length));
+}
+
 IOByteCount IOSubMemoryDescriptor::readBytes(IOByteCount offset,
 					void * bytes, IOByteCount length)
 {
@@ -3502,29 +3584,23 @@ IOReturn IOSubMemoryDescriptor::complete(
 IOMemoryMap * IOSubMemoryDescriptor::makeMapping(
 	IOMemoryDescriptor *	owner,
 	task_t			intoTask,
-	IOVirtualAddress	toAddress,
+	IOVirtualAddress	address,
 	IOOptionBits		options,
 	IOByteCount		offset,
 	IOByteCount		length )
 {
     IOMemoryMap * mapping = 0;
 
-    if (!(kIOMapUnique & options))
-	mapping = (IOMemoryMap *) _parent->makeMapping(
-					_parent, intoTask,
-					toAddress - (_start + offset),
-					options | kIOMapReference,
-					_start + offset, length );
-
-    if( !mapping)
-        mapping = (IOMemoryMap *) _parent->makeMapping(
-					_parent, intoTask,
-					toAddress,
-					options, _start + offset, length );
+    if (!(kIOMap64Bit & options))
+    {
+	panic("IOSubMemoryDescriptor::makeMapping !64bit");
+    }
 
-    if( !mapping)
-	mapping = super::makeMapping( owner, intoTask, toAddress, options,
-					offset, length );
+    mapping = (IOMemoryMap *) _parent->makeMapping(
+					owner,
+					intoTask,
+					address,
+					options, _start + offset, length );
 
     return( mapping );
 }
@@ -3671,7 +3747,7 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     if (keys[1])
       keys[1]->release();
     if (vcopy)
-        IOFree(vcopy, sizeof(IOVirtualRange) * nRanges);
+        IOFree(vcopy, sizeof(SerData) * nRanges);
     return result;
 }
 
diff --git a/iokit/Kernel/IOMultiMemoryDescriptor.cpp b/iokit/Kernel/IOMultiMemoryDescriptor.cpp
index 257a3aadb..fb1a38f22 100644
--- a/iokit/Kernel/IOMultiMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMultiMemoryDescriptor.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IOLib.h>
diff --git a/iokit/Kernel/IONVRAM.cpp b/iokit/Kernel/IONVRAM.cpp
index 315aefe11..9905c6028 100644
--- a/iokit/Kernel/IONVRAM.cpp
+++ b/iokit/Kernel/IONVRAM.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/IOLib.h>
@@ -108,7 +114,7 @@ void IODTNVRAM::registerNVRAMController(IONVRAMController *nvram)
 	freePartitionSize = currentLength;
       } else {
 	// Construct the partition ID from the signature and name.
-	sprintf(partitionID, "0x%02x,",
+	snprintf(partitionID, sizeof(partitionID), "0x%02x,",
 		*(UInt8 *)(_nvramImage + currentOffset));
 	strncpy(partitionID + 5,
 		(const char *)(_nvramImage + currentOffset + 4), 12);
@@ -556,9 +562,15 @@ UInt32 IODTNVRAM::savePanicInfo(UInt8 *buffer, IOByteCount length)
   *(UInt32 *)_piImage = length;
   
   _nvramImageDirty = true;
-  
+  /* 
+   * This prevents OF variables from being committed if the system has panicked
+   */
   _systemPaniced = true;
-  
+  /* The call to sync() forces the NVRAM controller to write the panic info
+   * partition to NVRAM.
+   */
+  sync();
+
   return length;
 }
 
@@ -656,7 +668,7 @@ IOReturn IODTNVRAM::initOFVariables(void)
     // Create the 'aapl,panic-info' property if needed.
     if (_piImage != 0) {
       propDataLength = *(UInt32 *)_piImage;
-      if ((propDataLength != 0) && (propDataLength < (_piPartitionSize - 4))) {
+      if ((propDataLength != 0) && (propDataLength <= (_piPartitionSize - 4))) {
 	propObject = OSData::withBytes(_piImage + 4, propDataLength);
 	_ofDict->setObject(kIODTNVRAMPanicInfoKey, propObject);
 	propObject->release();
@@ -1082,31 +1094,30 @@ bool IODTNVRAM::convertObjectToProp(UInt8 *buffer, UInt32 *length,
   if ((propNameLength + propDataLength + 2) > *length) return false;
   
   // Copy the property name equal sign.
-  sprintf((char *)buffer, "%s=", propName);
-  buffer += propNameLength + 1;
+  buffer += snprintf((char *)buffer, *length, "%s=", propName);
   
   switch (propType) {
   case kOFVariableTypeBoolean :
     if (tmpBoolean->getValue()) {
-      strcpy((char *)buffer, "true");
+      strlcpy((char *)buffer, "true", *length - propNameLength);
     } else {
-      strcpy((char *)buffer, "false");
+      strlcpy((char *)buffer, "false", *length - propNameLength);
     }
     break;
     
   case kOFVariableTypeNumber :
     tmpValue = tmpNumber->unsigned32BitValue();
     if (tmpValue == 0xFFFFFFFF) {
-      strcpy((char *)buffer, "-1");
+      strlcpy((char *)buffer, "-1", *length - propNameLength);
     } else if (tmpValue < 1000) {
-      sprintf((char *)buffer, "%ld", tmpValue);
+      snprintf((char *)buffer, *length - propNameLength, "%ld", tmpValue);
     } else {
-      sprintf((char *)buffer, "0x%lx", tmpValue);
+      snprintf((char *)buffer, *length - propNameLength, "0x%lx", tmpValue);
     }
     break;
     
   case kOFVariableTypeString :
-    strcpy((char *)buffer, tmpString->getCStringNoCopy());
+    strlcpy((char *)buffer, tmpString->getCStringNoCopy(), *length - propNameLength);
     break;
     
   case kOFVariableTypeData :
@@ -1199,9 +1210,8 @@ void IODTNVRAM::updateOWBootArgs(const OSSymbol *key, OSObject *value)
     tmpData = IONew(UInt8, tmpDataLength + 1);
     if (tmpData == 0) return;
     
-    strncpy((char *)tmpData, (const char *)bootCommandData, cnt);
-    tmpData[cnt] = '\0';
-    strcat((char *)tmpData, (const char *)bootArgsData);
+    cnt -= strlcpy((char *)tmpData, (const char *)bootCommandData, cnt);
+    strlcat((char *)tmpData, (const char *)bootArgsData, cnt);
     
     bootCommand = OSString::withCString((const char *)tmpData);
     if (bootCommand != 0) {
@@ -1604,8 +1614,8 @@ IOReturn IODTNVRAM::writeNVRAMPropertyType1(IORegistryEntry *entry,
 
 			name = entry->getName(gIODTPlane);
 			comp = entry->getLocation(gIODTPlane);
-			if( comp && (0 == strcmp("pci", name))
-			 && (0 == strcmp("80000000", comp))) {
+			if( comp && (0 == strncmp("pci", name, sizeof("pci")))
+			 && (0 == strncmp("80000000", comp, sizeof("80000000")))) {
 				// yosemite hack
 				comp = "/pci@80000000";
 			} else {
diff --git a/iokit/Kernel/IOPMPowerSource.cpp b/iokit/Kernel/IOPMPowerSource.cpp
index d71aa2da1..e6a11fc07 100644
--- a/iokit/Kernel/IOPMPowerSource.cpp
+++ b/iokit/Kernel/IOPMPowerSource.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/pwr_mgt/IOPMPowerSource.h>
@@ -126,6 +132,9 @@ void IOPMPowerSource::updateStatus (void)
     OSObject                        *iteratorKey;
     OSObject                        *obj;
 
+    // do nothing if settings haven't changed
+    if(!settingsChangedSinceUpdate) return;
+
     iterator = OSCollectionIterator::withCollection(properties);
     if(!iterator) return;
 
@@ -140,6 +149,8 @@ void IOPMPowerSource::updateStatus (void)
     }
     iterator->release();
 
+    settingsChangedSinceUpdate = false;
+
     // And up goes the flare
     messageClients(kIOPMMessageBatteryStatusHasChanged);
 }
@@ -151,125 +162,140 @@ void IOPMPowerSource::updateStatus (void)
  *
  ******************************************************************************/
  
+void IOPMPowerSource::setPSProperty(const OSSymbol *key, OSObject *val)
+{
+    OSObject    *lastVal;
+    OSNumber    *newNumVal;
+
+    if(!key || !val) return;
+
+    // Compare new setting with existing setting; update 
+    // 'settingsChangedSinceUpdate' if the setting has changed.
+    // If values are OSNumbers, do equality comparison.
+    // Otherwise, just compare pointers.
+    
+    if( (lastVal = properties->getObject(key)) ) {
+        newNumVal = OSDynamicCast(OSNumber, val);
+        if(newNumVal) {
+            if(newNumVal->isEqualTo(lastVal)) {
+                // settings didn't change
+            } else {
+                // num val is not equal to last val
+                settingsChangedSinceUpdate = true;
+            }
+        } else {
+            // pointer compare as last resort
+            if(lastVal != val)
+                settingsChangedSinceUpdate = true;        
+        }
+    } else {
+        // new setting; no last value
+        settingsChangedSinceUpdate = true;
+    }
+    
+    // here's the part where we go crazy.
+    properties->setObject(key, val);
+}
+
+
+ 
 void IOPMPowerSource::setExternalConnected(bool b) {
-    properties->setObject(
-                externalConnectedKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(externalConnectedKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 void IOPMPowerSource::setExternalChargeCapable(bool b) {
-    properties->setObject(
-                externalChargeCapableKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(externalChargeCapableKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 void IOPMPowerSource::setBatteryInstalled(bool b) {
-    properties->setObject(
-                batteryInstalledKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(batteryInstalledKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 void IOPMPowerSource::setIsCharging(bool b) {
-    properties->setObject(
-                chargingKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(chargingKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 void IOPMPowerSource::setAtWarnLevel(bool b) {
-    properties->setObject(
-                warnLevelKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(warnLevelKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 void IOPMPowerSource::setAtCriticalLevel(bool b) {
-    properties->setObject(
-                criticalLevelKey,
-                b ? kOSBooleanTrue:kOSBooleanFalse);    
+    setPSProperty(criticalLevelKey,
+            b ? kOSBooleanTrue : kOSBooleanFalse);    
 }
 
 
 void IOPMPowerSource::setCurrentCapacity(unsigned int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                currentCapacityKey,
-                n);
+    setPSProperty(currentCapacityKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setMaxCapacity(unsigned int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                maxCapacityKey,
-                n);
+    setPSProperty(maxCapacityKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setTimeRemaining(int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                timeRemainingKey,
-                n);
+    setPSProperty(timeRemainingKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setAmperage(int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                amperageKey,
-                n);
+    setPSProperty(amperageKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setVoltage(unsigned int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                voltageKey,
-                n);
+    setPSProperty(voltageKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setCycleCount(unsigned int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                cycleCountKey,
-                n);
+    setPSProperty(cycleCountKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setAdapterInfo(int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                adapterInfoKey,
-                n);
+    setPSProperty(adapterInfoKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setLocation(int val) {
     OSNumber *n = OSNumber::withNumber(val, 32);
-    properties->setObject(
-                locationKey,
-                n);
+    setPSProperty(locationKey, n);
     n->release();
 }
 
 void IOPMPowerSource::setErrorCondition(OSSymbol *s) {
-    properties->setObject(errorConditionKey, s);
+    setPSProperty(errorConditionKey, s);
 }
 
 void IOPMPowerSource::setManufacturer(OSSymbol *s) {
-    properties->setObject(manufacturerKey, s);
+    setPSProperty(manufacturerKey, s);
 }
 
 void IOPMPowerSource::setModel(OSSymbol *s) {
-    properties->setObject(modelKey, s);
+    setPSProperty(modelKey, s);
 }
 
 void IOPMPowerSource::setSerial(OSSymbol *s) {
-    properties->setObject(serialKey, s);
+    setPSProperty(serialKey, s);
 }
 
 void IOPMPowerSource::setLegacyIOBatteryInfo(OSDictionary *d) {
-    properties->setObject(batteryInfoKey, d);
+    setPSProperty(batteryInfoKey, d);
 }
 
 
@@ -281,6 +307,11 @@ void IOPMPowerSource::setLegacyIOBatteryInfo(OSDictionary *d) {
  *
  ******************************************************************************/
 
+OSObject *IOPMPowerSource::getPSProperty(const OSSymbol *symmie) {
+    if(!symmie) return NULL;
+    return properties->getObject(symmie);
+}
+
 bool IOPMPowerSource::externalConnected(void) {
     return (kOSBooleanTrue == properties->getObject(externalConnectedKey));
 }
diff --git a/iokit/Kernel/IOPMPowerSourceList.cpp b/iokit/Kernel/IOPMPowerSourceList.cpp
index cace37fa4..a933ebf76 100644
--- a/iokit/Kernel/IOPMPowerSourceList.cpp
+++ b/iokit/Kernel/IOPMPowerSourceList.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/pwr_mgt/IOPM.h>
 #include <IOKit/pwr_mgt/IOPMPowerSourceList.h>
diff --git a/iokit/Kernel/IOPMPowerStateQueue.cpp b/iokit/Kernel/IOPMPowerStateQueue.cpp
index 54fc10387..dee1aff49 100644
--- a/iokit/Kernel/IOPMPowerStateQueue.cpp
+++ b/iokit/Kernel/IOPMPowerStateQueue.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include "IOPMPowerStateQueue.h"
@@ -26,12 +32,13 @@
 #define super IOEventSource
 OSDefineMetaClassAndStructors(IOPMPowerStateQueue, IOEventSource);
 
-#ifdef __i386__ /* ppc does this right and doesn't need these routines */
+#ifndef __ppc__ /* ppc does this right and doesn't need these routines */
 static
-void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
-{
-	void *	oldListHead;
-	void *	newListHead;
+void *	OSDequeueAtomic(void * volatile * inList, SInt32 inOffset)
+{	
+	/* The _pointer_ is volatile, not the listhead itself */
+	void * volatile	oldListHead;
+	void * volatile	newListHead;
 
 	do {
 		oldListHead = *inList;
@@ -39,26 +46,27 @@ void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
 			break;
 		}
 		
-		newListHead = *(void **) (((char *) oldListHead) + inOffset);
+		newListHead = *(void * volatile *) (((char *) oldListHead) + inOffset);
 	} while (! OSCompareAndSwap((UInt32)oldListHead,
-					(UInt32)newListHead, (UInt32 *)inList));
+					(UInt32)newListHead, (volatile UInt32 *)inList));
 	return oldListHead;
 }
 
 static
-void	OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
+void	OSEnqueueAtomic(void * volatile * inList, void * inNewLink, SInt32 inOffset)
 {
-	void *	oldListHead;
-	void *	newListHead = inNewLink;
-	void **	newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
+	/* The _pointer_ is volatile, not the listhead itself */
+	void *	volatile oldListHead;
+	void *	volatile newListHead = inNewLink;
+	void * volatile *	newLinkNextPtr = (void * volatile *) (((char *) inNewLink) + inOffset);
 
 	do {
 		oldListHead = *inList;
 		*newLinkNextPtr = oldListHead;
 	} while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
-					(UInt32 *)inList));
+					(volatile UInt32 *)inList));
 }
-#endif /* __i386__ */
+#endif /* ! __ppc__ */
 
 
 IOPMPowerStateQueue *IOPMPowerStateQueue::PMPowerStateQueue(OSObject *inOwner)
@@ -80,7 +88,7 @@ bool IOPMPowerStateQueue::init(OSObject *owner, Action action)
 
     // Queue of powerstate changes
     changes = NULL;
-#ifdef __i386__
+#ifndef __ppc__
     if (!(tmpLock = IOLockAlloc()))  panic("IOPMPowerStateQueue::init can't alloc lock");
 #endif
     return true;
@@ -98,6 +106,32 @@ bool IOPMPowerStateQueue::unIdleOccurred(IOService *inTarget, unsigned long inSt
     new_one->state = inState;
     new_one->target = inTarget;
     
+    // Change to queue
+#ifndef __ppc__
+    IOLockLock(tmpLock);
+#endif
+    OSEnqueueAtomic((void **)&changes, (void *)new_one, 0);
+#ifndef __ppc__
+    IOLockUnlock(tmpLock);
+#endif
+    signalWorkAvailable();
+
+    return true;
+}
+
+bool IOPMPowerStateQueue::featureChangeOccurred(
+    uint32_t inState, 
+    IOService *inTarget)
+{
+    PowerChangeEntry             *new_one = NULL;
+
+    new_one = (PowerChangeEntry *)IOMalloc(sizeof(PowerChangeEntry));
+    if(!new_one) return false;
+    
+    new_one->actionType = IOPMPowerStateQueue::kPMFeatureChange;
+    new_one->state = inState;
+    new_one->target = inTarget;
+    
     // Change to queue
 #ifdef __i386__
     IOLockLock(tmpLock);
@@ -111,21 +145,22 @@ bool IOPMPowerStateQueue::unIdleOccurred(IOService *inTarget, unsigned long inSt
     return true;
 }
 
+
 // checkForWork() is called in a gated context
 bool IOPMPowerStateQueue::checkForWork()
 {
     PowerChangeEntry            *theNode;
-    int                         theState;
+    uint32_t                    theState;
     IOService                   *theTarget;
-    UInt16                      theAction;
+    uint16_t                    theAction;
     
     // Dequeue and process the state change request
-#ifdef __i386__
+#ifndef __ppc__
     IOLockLock(tmpLock);
 #endif
     if((theNode = (PowerChangeEntry *)OSDequeueAtomic((void **)&changes, 0)))
     {
-#ifdef __i386__
+#ifndef __ppc__
       IOLockUnlock(tmpLock);
 #endif
         theState = theNode->state;
@@ -138,9 +173,13 @@ bool IOPMPowerStateQueue::checkForWork()
             case kUnIdle:
                 theTarget->command_received((void *)theState, 0, 0, 0);
                 break;
+                
+            case kPMFeatureChange:
+                theTarget->messageClients(theState, theTarget);
+                break;
         }
     }
-#ifdef __i386__
+#ifndef __ppc__
     else {
       IOLockUnlock(tmpLock);
     }
diff --git a/iokit/Kernel/IOPMPowerStateQueue.h b/iokit/Kernel/IOPMPowerStateQueue.h
index 4ec35004a..a2862c500 100644
--- a/iokit/Kernel/IOPMPowerStateQueue.h
+++ b/iokit/Kernel/IOPMPowerStateQueue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #ifndef _IOPMPOWERSTATEQUEUE_H_
@@ -36,20 +42,21 @@ class IOPMPowerStateQueue : public IOEventSource
 
 private:
     enum {
-        kUnIdle = 0
+        kUnIdle = 0,
+        kPMFeatureChange = 1
     };
 
     // Queue of requested states
     struct PowerChangeEntry 
     {
         void                    *next;
-        UInt16                  actionType;
-        UInt16                  state;
+        uint16_t                actionType;
+        uint32_t                state;
         IOService               *target;
     };
 
     void                        *changes;
-#ifdef __i386__
+#ifndef __ppc__
     IOLock                      *tmpLock;
 #endif
 
@@ -66,6 +73,9 @@ public:
          
     // Enqueues an activityTickle request to be executed on the workloop
     virtual bool unIdleOccurred(IOService *, unsigned long);
+    
+    // Enqueues a feature changed notify request to be executed on the workloop
+    virtual bool featureChangeOccurred(uint32_t, IOService *);
  };
  
  #endif /* _IOPMPOWERSTATEQUEUE_H_ */
diff --git a/iokit/Kernel/IOPMWorkArbiter.cpp b/iokit/Kernel/IOPMWorkArbiter.cpp
deleted file mode 100644
index 060c457f0..000000000
--- a/iokit/Kernel/IOPMWorkArbiter.cpp
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
- 
-#include "IOPMWorkArbiter.h"
-#include "IOKit/IOLocks.h"
-#undef super
-#define super IOEventSource
-OSDefineMetaClassAndStructors(IOPMWorkArbiter, IOEventSource);
-
-/*************************************************************************/
-static
-void *    _dequeue(void ** inList, SInt32 inOffset)
-{
-    void *    oldListHead;
-    void *    newListHead;
-
-    do {
-        oldListHead = *inList;
-        if (oldListHead == NULL) {
-            break;
-        }
-        
-        newListHead = *(void **) (((char *) oldListHead) + inOffset);
-    } while (! OSCompareAndSwap((UInt32)oldListHead,
-                    (UInt32)newListHead, (UInt32 *)inList));
-    return oldListHead;
-}
-
-/*************************************************************************/
-static
-void    _enqueue(void ** inList, void * inNewLink, SInt32 inOffset)
-{
-    void *    oldListHead;
-    void *    newListHead = inNewLink;
-    void **    newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
-
-    do {
-        oldListHead = *inList;
-        *newLinkNextPtr = oldListHead;
-    } while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
-                    (UInt32 *)inList));
-}
-
-/*************************************************************************/
-IOPMWorkArbiter *IOPMWorkArbiter::pmWorkArbiter(
-    IOPMrootDomain *inOwner)
-{
-    IOPMWorkArbiter     *me = new IOPMWorkArbiter;
-
-    if(me && !me->init((OSObject *)inOwner, 0) )
-    {
-        me->release();
-        return NULL;
-    }
-
-    return me;
-}
-
-/*************************************************************************/
-bool IOPMWorkArbiter::init(OSObject *owner, Action action)
-{
-    if(!(super::init(owner, (IOEventSource::Action) action))) 
-        return false;
-        
-    events = NULL;
-    fRootDomain = (IOPMrootDomain *)owner;
-    
-    if (!(tmpLock = IOLockAlloc())) {
-        panic("IOPMWorkArbiter::init can't alloc lock");
-    }
-    return true;
-}
-
-
-/*************************************************************************/
-bool IOPMWorkArbiter::clamshellStateChangeOccurred(uint32_t messageValue)
-{
-    PMEventEntry                 *new_one = NULL;
-
-    new_one = (PMEventEntry *)IOMalloc(sizeof(PMEventEntry));
-    if(!new_one) return false;
-    
-    new_one->actionType = IOPMWorkArbiter::kRootDomainClamshellChanged;
-    new_one->target = (IOService *)fRootDomain;
-    new_one->intArgument = messageValue;
-       new_one->target->retain();
-    
-    IOLockLock(tmpLock);
-    _enqueue((void **)&events, (void *)new_one, 0);
-    IOLockUnlock(tmpLock);
-    signalWorkAvailable();
-
-    return true;
-}
-
-
-/*************************************************************************/
-// checkForWork() is called in a gated context
-bool IOPMWorkArbiter::checkForWork()
-{
-    PMEventEntry                *theNode;
-    IOService                   *theTarget;
-    UInt16                      theAction;
-    
-    // Dequeue and process the state change request
-    IOLockLock(tmpLock);
-    if((theNode = (PMEventEntry *)_dequeue((void **)&events, 0)))
-    {
-      IOLockUnlock(tmpLock);
-        theTarget = theNode->target;
-        theAction = theNode->actionType;
-
-        switch (theAction)
-        {
-            case kRootDomainClamshellChanged:
-                theTarget->messageClients(
-                                kIOPMMessageClamshellStateChange, 
-                                (void *)theNode->intArgument);
-                break;
-        }
-
-               if (theTarget)
-                       theTarget->release();
-        IOFree((void *)theNode, sizeof(PMEventEntry));
-    }
-    else {
-      IOLockUnlock(tmpLock);
-    }
-    // Return true if there's more work to be done
-    if(events) return true;
-    else return false;
-}
-
-
diff --git a/iokit/Kernel/IOPMWorkArbiter.h b/iokit/Kernel/IOPMWorkArbiter.h
deleted file mode 100644
index 5bfe5254a..000000000
--- a/iokit/Kernel/IOPMWorkArbiter.h
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright (c) 2001-2002 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
- 
-#ifndef _IOPMWorkArbiter_H_
-#define _IOPMWorkArbiter_H_
- 
-#include <IOKit/IOWorkLoop.h>
-#include <IOKit/IOEventSource.h>
-#include <IOKit/IOService.h>
-extern "C" {
-    #include <kern/queue.h>
-}
-
-class IOPMWorkArbiter : public IOEventSource
-{
-    OSDeclareDefaultStructors(IOPMWorkArbiter);
-
-private:
-    enum {
-        kAllAcked = 0,
-        kDriverAcked,
-        kRootDomainClamshellChanged
-    };
-
-    // Queue of requested states
-    struct PMEventEntry 
-    {
-        void                    *next;
-        IOService               *target;
-        uint16_t                actionType;
-        uint32_t                intArgument;
-    };
-
-    IOPMrootDomain              *fRootDomain;
-
-    void                        *events;
-    IOLock                      *tmpLock;
-
-protected:
-    virtual bool checkForWork(void);
-
-public:
-    IOLock              *arbiterLock;
-
-    //typedef void (*Action)(IOService *target, unsigned long state);
-
-    virtual bool init(OSObject *owner, Action action = 0);
-
-    // static initialiser
-    static IOPMWorkArbiter *pmWorkArbiter(IOPMrootDomain *owner);
-      
-    /* IOPMrootDomain work */
-    bool clamshellStateChangeOccurred(uint32_t messageValue);
- };
- 
- 
- #endif /* _IOPMWorkArbiter_H_ */
- 
diff --git a/iokit/Kernel/IOPMchangeNoteList.cpp b/iokit/Kernel/IOPMchangeNoteList.cpp
deleted file mode 100644
index 427ff5085..000000000
--- a/iokit/Kernel/IOPMchangeNoteList.cpp
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <IOKit/pwr_mgt/IOPM.h>
-#include <IOKit/pwr_mgt/IOPMchangeNoteList.h>
-#include <IOKit/pwr_mgt/IOPowerConnection.h>
-
-#define super OSObject
-OSDefineMetaClassAndStructors(IOPMchangeNoteList,OSObject)
-
-//*********************************************************************************
-// init
-//
-//*********************************************************************************
-void IOPMchangeNoteList::initialize ( void )
-{
-    long i;
-
-    firstInList = 0;
-    firstUnused = 0;
-    for ( i = 0; i < IOPMMaxChangeNotes; i++ ) {
-        changeNote[i].flags = IOPMNotInUse;
-    }
-}
-
-//*********************************************************************************
-// createChangeNote
-//
-//*********************************************************************************
-
-long IOPMchangeNoteList::createChangeNote ( void )
-{
-    unsigned long i, j;
-
-    i = increment(firstUnused);
-    if ( firstInList == i ) {
-        return -1;
-    }
-    j = firstUnused;
-    firstUnused = i;
-
-    return j;
-}
-
-//*********************************************************************************
-// currentChange
-//
-// Return the ordinal of the first change note in the list.
-// If the list is empty, return -1.
-//*********************************************************************************
-
-long IOPMchangeNoteList::currentChange ( void )
-{
-    if ( firstUnused == firstInList ) {
-        return -1;
-    }
-    else {
-        return firstInList;
-    }
-}
-
-//*********************************************************************************
-// latestChange
-//
-// Return the ordinal of the last change note in the list.
-// If the list is empty, return -1.
-//*********************************************************************************
-
-long IOPMchangeNoteList::latestChange ( void )
-{
-    if ( firstUnused == firstInList ) {
-        return -1;
-    }
-    else {
-        return decrement(firstUnused);
-    }
-}
-
-//*********************************************************************************
-// releaseHeadChangeNote
-//
-// Mark the head node unused.
-// This happens when the first change in the list is completely processed.
-// That is, all interested parties have acknowledged it, and power is settled
-// at the new level.
-//*********************************************************************************
-
-IOReturn IOPMchangeNoteList::releaseHeadChangeNote ( void )
-{
-    IOPowerConnection *tmp;
-
-    if((tmp = changeNote[firstInList].parent)) {
-       changeNote[firstInList].parent = 0;
-       tmp->release();
-    }
-
-    changeNote[firstInList].flags = IOPMNotInUse;
-    firstInList = increment(firstInList);
-    return IOPMNoErr;
-}
-
-//*********************************************************************************
-// releaseTailChangeNote
-//
-// Mark the tail node unused.
-// This happens when a power change is queued up after another which has
-// not yet been started, and the second one supercedes the first.  The data in
-// the second is copied into the first and the the second is released.  This
-// collapses the first change out of the list.
-//*********************************************************************************
-
-IOReturn IOPMchangeNoteList::releaseTailChangeNote ( void )
-{
-    IOPowerConnection *tmp;
-    
-    if((tmp = changeNote[firstInList].parent)) {
-       changeNote[firstInList].parent = 0;
-       tmp->release();
-    }
-
-    firstUnused = decrement(firstUnused);
-    changeNote[firstUnused].flags = IOPMNotInUse;
-    return IOPMNoErr;
-}
-
-//*********************************************************************************
-// changeNoteInUse
-//
-//*********************************************************************************
-
-bool IOPMchangeNoteList::changeNoteInUse ( unsigned long ordinal )
-{
-    if ( changeNote[ordinal].flags == IOPMNotInUse ) {
-        return false;
-    }
-    else {
-        return true;
-    }
-}
-
-//*********************************************************************************
-// nextChangeNote
-//
-// If the parameter corresponds to the most recent power change notification
-// passed to drivers and children, return -1.  Otherwise, return the array
-// position of the next notification in the circular list.
-//*********************************************************************************
-
-long IOPMchangeNoteList::nextChangeNote ( unsigned long ordinal )
-{
-    unsigned long i;
-
-    i = increment(ordinal);
-    if ( i == firstUnused)  {
-        return -1;
-    }
-    return ( i );
-}
-
-//*********************************************************************************
-// increment
-//
-// Increment the parameter mod the circular list size and return it.
-//*********************************************************************************
-
-unsigned long IOPMchangeNoteList::increment ( unsigned long ordinal )
-{
-    if ( ordinal == (IOPMMaxChangeNotes - 1) ) {
-        return 0;
-    }
-    else {
-        return ordinal + 1;
-    }
-}
-
-//*********************************************************************************
-// decrement
-//
-// Decrement the parameter mod the circular list size and return it.
-//*********************************************************************************
-
-unsigned long IOPMchangeNoteList::decrement ( unsigned long  ordinal )
-{
-    if ( ordinal == 0 ) {
-        return IOPMMaxChangeNotes - 1;
-    }
-    else {
-        return ordinal - 1;
-    }
-}
-
-//*********************************************************************************
-// previousChangeNote
-//
-// If the parameter corresponds to the oldest power change notification
-// passed to drivers and children, return -1.  Otherwise, return the array
-// position of the previous notification in the circular list.
-//*********************************************************************************
-
-long IOPMchangeNoteList::previousChangeNote ( unsigned long ordinal )
-{
-    if ( ordinal == firstInList )  {
-        return -1;
-    }
-    return decrement(ordinal);
-}
-
-//*********************************************************************************
-// listEmpty
-//
-//*********************************************************************************
-
-bool IOPMchangeNoteList::listEmpty ( void )
-{
-    return ( firstInList == firstUnused ) ;
-}
diff --git a/iokit/Kernel/IOPMinformee.cpp b/iokit/Kernel/IOPMinformee.cpp
index 8f7452cf7..3889ffe44 100644
--- a/iokit/Kernel/IOPMinformee.cpp
+++ b/iokit/Kernel/IOPMinformee.cpp
@@ -1,40 +1,63 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/pwr_mgt/IOPMinformee.h>
 
 #define super OSObject
-OSDefineMetaClassAndStructors(IOPMinformee,OSObject)
+
+OSDefineMetaClassAndStructors(IOPMinformee, OSObject)
+
 
 //*********************************************************************************
-// constructor
+// static constructor
 //
 //*********************************************************************************
+IOPMinformee *IOPMinformee::withObject( IOService *theObject )
+{
+    IOPMinformee        *newInformee = new IOPMinformee;
+    
+    if (!newInformee) return NULL;
+    newInformee->init();
+    newInformee->initialize( theObject );
+ 
+    return newInformee;
+}
 
+
+//*********************************************************************************
+// constructor
+//
+//*********************************************************************************
 void IOPMinformee::initialize ( IOService * theObject )
 {
    whatObject = theObject;
    timer = 0;
+   active = true;
    whatObject->retain();
-    
 }
 
 
@@ -42,7 +65,6 @@ void IOPMinformee::initialize ( IOService * theObject )
 // free
 //
 //*********************************************************************************
-
 void IOPMinformee::free (void )
 {
     whatObject->release();
diff --git a/iokit/Kernel/IOPMinformeeList.cpp b/iokit/Kernel/IOPMinformeeList.cpp
index 79ef784fe..5857bb6b4 100644
--- a/iokit/Kernel/IOPMinformeeList.cpp
+++ b/iokit/Kernel/IOPMinformeeList.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/pwr_mgt/IOPM.h>
 #include <IOKit/pwr_mgt/IOPMinformeeList.h>
@@ -62,11 +68,34 @@ IORecursiveLock *IOPMinformeeList::getSharedRecursiveLock( void )
     return sharedListLock;
 }
 
+ //*********************************************************************************
+// appendNewInformee
+ //
+ //*********************************************************************************
+IOPMinformee *IOPMinformeeList::appendNewInformee ( IOService * newObject )
+{
+    IOPMinformee * newInformee;
+    
+    if (!newObject)
+        return NULL;
+ 
+    newInformee = IOPMinformee::withObject (newObject);    
+
+    if (!newInformee)
+        return NULL;
+
+    if( IOPMNoErr == addToList (newInformee))
+        return newInformee;
+    else
+        return NULL;
+}
+
+
 //*********************************************************************************
 // addToList
-//
+// *OBSOLETE* do not call from outside of this file.
+// Try appendNewInformee() instead
 //*********************************************************************************
-
 IOReturn IOPMinformeeList::addToList ( IOPMinformee * newInformee )
 {
     IOPMinformee * nextInformee;
diff --git a/iokit/Kernel/IOPMpmChild.cpp b/iokit/Kernel/IOPMpmChild.cpp
deleted file mode 100644
index 0b34a9e45..000000000
--- a/iokit/Kernel/IOPMpmChild.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <IOKit/pwr_mgt/IOPMpmChild.h>
-
-#define super IOPMinformee
-OSDefineMetaClassAndStructors(IOPMpmChild,IOPMinformee)
-
-//*********************************************************************************
-// constructor
-//
-//*********************************************************************************
-
-void IOPMpmChild::initialize ( IOService * theObject )
-{
-    desiredDomainState = 0;
-    super::initialize(theObject);
-}
diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp
index 1e61a1b87..9af5919f4 100644
--- a/iokit/Kernel/IOPMrootDomain.cpp
+++ b/iokit/Kernel/IOPMrootDomain.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/IOWorkLoop.h>
 #include <IOKit/IOCommandGate.h>
@@ -34,12 +40,29 @@
 #include "IOKit/pwr_mgt/IOPowerConnection.h"
 #include "IOPMPowerStateQueue.h"
 #include <IOKit/IOCatalogue.h>
+#if HIBERNATION
 #include <IOKit/IOHibernatePrivate.h>
-#include "IOPMWorkArbiter.h"
+#endif
+#include <sys/syslog.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+#include "IOServicePrivate.h"	// _IOServiceInterestNotifier
+
+
+#if __i386__
+__BEGIN_DECLS
+#include "IOPMrootDomainInternal.h"
+__END_DECLS
+#endif
+
 
-#ifdef __ppc__
-#include <ppc/pms.h>
+//#define DEBUG   1
+#if DEBUG
+#define DEBUG_LOG(x...) do { kprintf(x); } while (0)
+#else
+#define DEBUG_LOG(x...)
 #endif
+#define HaltRestartLog(x...)  do { kprintf(x); } while (0)
 
 extern "C" {
 IOReturn OSMetaClassSystemSleepOrWake( UInt32 );
@@ -50,6 +73,7 @@ extern const IORegistryPlane * gIOPowerPlane;
 IOReturn broadcast_aggressiveness ( OSObject *, void *, void *, void *, void * );
 static void sleepTimerExpired(thread_call_param_t);
 static void wakeupClamshellTimerExpired ( thread_call_param_t us);
+static void notifySystemShutdown( IOService * root, unsigned long event );
 
 // "IOPMSetSleepSupported"  callPlatformFunction name
 static const OSSymbol *sleepSupportedPEFunction = NULL;
@@ -72,6 +96,14 @@ static const OSSymbol *sleepSupportedPEFunction = NULL;
 #define SLEEP_POWER kIOPMAuxPowerOn
 #define DOZE_POWER kIOPMDoze
 
+enum 
+{
+    // not idle around autowake time, secs
+    kAutoWakePreWindow  = 45,
+    kAutoWakePostWindow = 15
+};
+
+
 #define kLocalEvalClamshellCommand        (1 << 15)
 
 static IOPMPowerState ourPowerStates[number_of_power_states] = {
@@ -90,6 +122,16 @@ static IOPMPowerState ourPowerStates[number_of_power_states] = {
 static IOPMrootDomain * gRootDomain;
 static UInt32           gSleepOrShutdownPending = 0;
 
+struct timeval gIOLastSleepTime;
+struct timeval gIOLastWakeTime;
+
+// Constants used as arguments to IOPMrootDomain::informCPUStateChange
+#define kCPUUnknownIndex    9999999
+enum {
+    kInformAC = 0,
+    kInformLid = 1,
+    kInformableCount = 2
+};
 
 class PMSettingObject : public OSObject
 {
@@ -116,6 +158,32 @@ public:
     void free(void);
 };
 
+/*
+ * Internal helper object for Shutdown/Restart notifications.
+ */
+#define kPMHaltMaxWorkers   8
+#define kPMHaltTimeoutMS    100
+
+class PMHaltWorker : public OSObject
+{
+    OSDeclareDefaultStructors( PMHaltWorker )
+
+public:
+    IOService *  service;    // service being worked on
+    AbsoluteTime startTime;  // time when work started
+    int          depth;      // work on nubs at this PM-tree depth
+    int          visits;     // number of nodes visited (debug)
+    IOLock *     lock;
+    bool         timeout;    // service took too long
+
+    static  PMHaltWorker * worker( void );
+    static  void main( void * arg );
+    static  void work( PMHaltWorker * me );
+    static  void checkTimeout( PMHaltWorker * me, AbsoluteTime * now );
+    virtual void free( void );
+};
+
+OSDefineMetaClassAndStructors( PMHaltWorker, OSObject )
 
 
 #define super IOService
@@ -206,7 +274,7 @@ one power state from Sleep to Doze, and any objects in the tree for which this i
 ADB will turn on again so that they can wake the system out of Doze (keyboard/mouse activity will cause the Display Wrangler
 to be tickled)).
 */
-    
+
 // **********************************************************************************
 
 IOPMrootDomain * IOPMrootDomain::construct( void )
@@ -224,20 +292,35 @@ IOPMrootDomain * IOPMrootDomain::construct( void )
 
 static void disk_sync_callout(thread_call_param_t p0, thread_call_param_t p1)
 {
-    IOService                               *rootDomain = (IOService *) p0;
-    unsigned long                           pmRef = (unsigned long) p1;
+    IOService       *rootDomain = (IOService *) p0;
+    unsigned long   pmRef = (unsigned long) p1;
+
+    DEBUG_LOG("disk_sync_callout: start\n");
 
+#if	HIBERNATION
     IOHibernateSystemSleep();
+#endif
     sync_internal();
     rootDomain->allowPowerChange(pmRef);
+    DEBUG_LOG("disk_sync_callout: finish\n");
 }
 
 // **********************************************************************************
-IOPMWorkArbiter *IOPMrootDomain::getPMArbiter(void)
+
+static UInt32 computeDeltaTimeMS( const AbsoluteTime * startTime )
 {
-    return pmArbiter;
-}
+	AbsoluteTime	endTime;
+	UInt64			nano = 0;
+
+	clock_get_uptime(&endTime);
+	if (CMP_ABSOLUTETIME(&endTime, startTime) > 0)
+	{
+		SUB_ABSOLUTETIME(&endTime, startTime);
+		absolutetime_to_nanoseconds(endTime, &nano);
+	}
 
+	return (UInt32)(nano / 1000000ULL);
+}
 
 // **********************************************************************************
 // start
@@ -246,16 +329,29 @@ IOPMWorkArbiter *IOPMrootDomain::getPMArbiter(void)
 // expert informs us we are the root.
 // **********************************************************************************
 
-#define kRootDomainSettingsCount        13
+#define kRootDomainSettingsCount        14
+
+static SYSCTL_STRUCT(_kern, OID_AUTO, sleeptime, 
+		     CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+		     &gIOLastSleepTime, timeval, "");
+
+static SYSCTL_STRUCT(_kern, OID_AUTO, waketime, 
+		     CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, 
+		     &gIOLastWakeTime, timeval, "");
+
+static const OSSymbol * gIOPMSettingAutoWakeSecondsKey;
 
 bool IOPMrootDomain::start ( IOService * nub )
 {
     OSIterator      *psIterator;
     OSDictionary    *tmpDict;
+
+    gIOPMSettingAutoWakeSecondsKey = OSSymbol::withCString(kIOPMSettingAutoWakeSecondsKey);
+
     const OSSymbol  *settingsArr[kRootDomainSettingsCount] = 
         {
             OSSymbol::withCString(kIOPMSettingSleepOnPowerButtonKey),
-            OSSymbol::withCString(kIOPMSettingAutoWakeSecondsKey),
+            gIOPMSettingAutoWakeSecondsKey,
             OSSymbol::withCString(kIOPMSettingAutoPowerSecondsKey),
             OSSymbol::withCString(kIOPMSettingAutoWakeCalendarKey),
             OSSymbol::withCString(kIOPMSettingAutoPowerCalendarKey),
@@ -266,6 +362,7 @@ bool IOPMrootDomain::start ( IOService * nub )
             OSSymbol::withCString(kIOPMSettingWakeOnClamshellKey),
             OSSymbol::withCString(kIOPMSettingWakeOnACChangeKey),
             OSSymbol::withCString(kIOPMSettingTimeZoneOffsetKey),
+            OSSymbol::withCString(kIOPMSettingDisplaySleepUsesDimKey),
             OSSymbol::withCString(kIOPMSettingMobileMotionModuleKey)
         };
     
@@ -285,6 +382,7 @@ bool IOPMrootDomain::start ( IOService * nub )
     canSleep = true;
     setProperty(kIOSleepSupportedKey,true);
 
+    userDisabledAllSleep = false;
     allowSleep = true;
     sleepIsSupported = true;
     systemBooting = true;
@@ -297,7 +395,10 @@ bool IOPMrootDomain::start ( IOService * nub )
     ignoringClamshell = true;
     ignoringClamshellDuringWakeup = false;
     acAdaptorConnect = true;
-    
+
+    idxPMCPUClamshell = kCPUUnknownIndex;
+    idxPMCPULimitedPower = kCPUUnknownIndex;
+        
     tmpDict = OSDictionary::withCapacity(1);
     setProperty(kRootDomainSupportedFeatures, tmpDict);
     tmpDict->release();
@@ -313,14 +414,8 @@ bool IOPMrootDomain::start ( IOService * nub )
                     
     fPMSettingsDict = OSDictionary::withCapacity(5);
             
-    pm_vars->PMworkloop = IOWorkLoop::workLoop();				
     pmPowerStateQueue = IOPMPowerStateQueue::PMPowerStateQueue(this);
-    pm_vars->PMworkloop->addEventSource(pmPowerStateQueue);
-    
-    /* Initialize PM arbiter */
-    pmArbiter = IOPMWorkArbiter::pmWorkArbiter(this);
-    arbiterWorkLoop = IOWorkLoop::workLoop();
-    arbiterWorkLoop->addEventSource(pmArbiter);
+    getPMworkloop()->addEventSource(pmPowerStateQueue);
     
     featuresDictLock = IOLockAlloc();
     settingsCtrlLock = IORecursiveLockAlloc();
@@ -340,8 +435,6 @@ bool IOPMrootDomain::start ( IOService * nub )
     patriarch->init();
     patriarch->attach(this);
     patriarch->start(this);
-    patriarch->youAreRoot();
-    patriarch->wakeSystem();
     patriarch->addPowerChild(this);
         
     registerPowerDriver(this,ourPowerStates,number_of_power_states);
@@ -353,10 +446,12 @@ bool IOPMrootDomain::start ( IOService * nub )
     // install power change handler
     registerPrioritySleepWakeInterest( &sysPowerDownHandler, this, 0);
 
+#if !NO_KERNEL_HID
     // Register for a notification when IODisplayWrangler is published
     _displayWranglerNotifier = addNotification( 
                 gIOPublishNotification, serviceMatching("IODisplayWrangler"), 
                 &displayWranglerPublished, this, 0);
+#endif
 
     // Battery location published - ApplePMU support only
     _batteryPublishNotifier = addNotification( 
@@ -382,7 +477,13 @@ bool IOPMrootDomain::start ( IOService * nub )
         psIterator->release();
     }
 
+
+    sysctl_register_oid(&sysctl__kern_sleeptime);
+    sysctl_register_oid(&sysctl__kern_waketime);
+
+#if	HIBERNATION
     IOHibernateSystemInit(this);
+#endif
 
     registerService();						// let clients find us
 
@@ -412,6 +513,11 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
                 OSSymbol::withCString("System Shutdown");
     const OSSymbol *stall_halt_string = 
                 OSSymbol::withCString("StallSystemAtHalt");
+    const OSSymbol *battery_warning_disabled_string = 
+                OSSymbol::withCString("BatteryWarningsDisabled");
+    const OSSymbol *idle_seconds_string = 
+                OSSymbol::withCString("System Idle Seconds");
+#if	HIBERNATION
     const OSSymbol *hibernatemode_string = 
                 OSSymbol::withCString(kIOHibernateModeKey);
     const OSSymbol *hibernatefile_string = 
@@ -420,6 +526,9 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
                 OSSymbol::withCString(kIOHibernateFreeRatioKey);
     const OSSymbol *hibernatefreetime_string = 
                 OSSymbol::withCString(kIOHibernateFreeTimeKey);
+#endif
+    const OSSymbol *sleepdisabled_string =
+                OSSymbol::withCString("SleepDisabled");
     
     if(!dict) 
     {
@@ -427,6 +536,12 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
         goto exit;
     }
 
+    if ((n = OSDynamicCast(OSNumber, dict->getObject(idle_seconds_string))))
+    {
+        setProperty(idle_seconds_string, n);
+	idleSeconds = n->unsigned32BitValue();
+    }
+
     if( systemBooting 
         && boot_complete_string 
         && dict->getObject(boot_complete_string)) 
@@ -442,6 +557,13 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
         }
     }
     
+    if( battery_warning_disabled_string
+        && dict->getObject(battery_warning_disabled_string))
+    {
+        setProperty( battery_warning_disabled_string, 
+                        dict->getObject(battery_warning_disabled_string));
+    }
+    
     if( sys_shutdown_string 
         && (b = OSDynamicCast(OSBoolean, dict->getObject(sys_shutdown_string)))) 
     {
@@ -475,6 +597,7 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
         setProperty(stall_halt_string, b);
     }
 
+#if	HIBERNATION
     if ( hibernatemode_string
     && (n = OSDynamicCast(OSNumber, dict->getObject(hibernatemode_string))))
     {
@@ -495,6 +618,15 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
     {
         setProperty(hibernatefile_string, str);
     }
+#endif
+    
+    if( sleepdisabled_string
+        && (b = OSDynamicCast(OSBoolean, dict->getObject(sleepdisabled_string))) )
+    {
+        setProperty(sleepdisabled_string, b);
+        
+        userDisabledAllSleep = (kOSBooleanTrue == b);
+    }
 
     // Relay our allowed PM settings onto our registered PM clients
     for(i = 0; i < allowedPMSettings->getCount(); i++) {
@@ -504,15 +636,36 @@ IOReturn IOPMrootDomain::setProperties ( OSObject *props_obj)
 
         obj = dict->getObject(type);
         if(!obj) continue;
+
+	if ((gIOPMSettingAutoWakeSecondsKey == type) && ((n = OSDynamicCast(OSNumber, obj))))
+	{
+	    UInt32 rsecs = n->unsigned32BitValue();
+	    if (!rsecs)
+		autoWakeStart = autoWakeEnd = 0;
+	    else
+	    {
+		AbsoluteTime deadline;
+		clock_interval_to_deadline(rsecs + kAutoWakePostWindow, kSecondScale, &deadline);
+		autoWakeEnd = AbsoluteTime_to_scalar(&deadline);
+		if (rsecs > kAutoWakePreWindow)
+		    rsecs -= kAutoWakePreWindow;
+		else
+		    rsecs = 0;
+		clock_interval_to_deadline(rsecs, kSecondScale, &deadline);
+		autoWakeStart = AbsoluteTime_to_scalar(&deadline);
+	    }
+	}
         
         return_value = setPMSetting(type, obj);
         
         if(kIOReturnSuccess != return_value) goto exit;
     }
 
-    exit:
+exit:
+    if(sleepdisabled_string) sleepdisabled_string->release();
     if(boot_complete_string) boot_complete_string->release();
     if(stall_halt_string) stall_halt_string->release();
+    if(idle_seconds_string) idle_seconds_string->release();
     return return_value;
 }
 
@@ -566,14 +719,50 @@ void IOPMrootDomain::broadcast_it (unsigned long type, unsigned long value)
     if( type == kPMMinutesToSpinDown ) user_spindown = value;
 
     // Use longestNonSleepSlider to calculate dimming adjust idle sleep timer
-    longestNonSleepSlider = pm_vars->current_aggressiveness_values[kPMMinutesToDim];
-
+	if (getAggressiveness(kPMMinutesToDim, (unsigned long *)&longestNonSleepSlider)
+		!= kIOReturnSuccess)
+		longestNonSleepSlider = 0;
 
     if ( type == kPMMinutesToSleep ) {
+        DEBUG_LOG("PM idle time -> %ld secs (ena %d)\n", idleSeconds, (value != 0));
+        if (0x7fffffff == value)
+            value = idleSeconds;
+
         if ( (sleepSlider == 0) && (value != 0) ) {
-            sleepSlider = value;
-            // idle sleep is now enabled, maybe sleep now
-            adjustPowerState();
+            if (!wrangler)
+            {
+                sleepASAP = false;
+                changePowerStateToPriv(ON_STATE);
+                if (idleSeconds)
+                {
+                    AbsoluteTime deadline;
+                    // stay awake for at least idleSeconds
+                    clock_interval_to_deadline(idleSeconds, kSecondScale, &deadline);	
+                    thread_call_enter_delayed(extraSleepTimer, deadline);
+                    // this gets turned off when we sleep again
+                    idleSleepPending = true;
+                }
+            }
+            else
+            {
+                // If sleepASAP is already set, then calling adjustPowerState() here
+                // will put the system to sleep immediately which is bad.  Note that
+                // this aggressiveness change can occur without waking up the display
+                // by (dis)connecting the AC adapter. To get around this, the power
+                // clamp is restore to ON state then dropped after waiting for the
+                // sleep timer to expire.
+
+                if (sleepASAP)
+                {
+                    AbsoluteTime deadline;
+                    // stay awake for at least sleepSlider minutes
+                    clock_interval_to_deadline(value * 60, kSecondScale, &deadline);	
+                    thread_call_enter_delayed(extraSleepTimer, deadline);
+                    // this gets turned off when we sleep again
+                    idleSleepPending = true;
+                    sleepASAP = false;
+                }
+            }
         }
         sleepSlider = value;
         if ( sleepSlider == 0 ) {			
@@ -616,6 +805,17 @@ static void wakeupClamshellTimerExpired ( thread_call_param_t us)
 // **********************************************************************************
 void IOPMrootDomain::handleSleepTimerExpiration ( void )
 {
+    DEBUG_LOG("SleepTimerExpired\n");
+
+    AbsoluteTime time;
+
+    clock_get_uptime(&time);
+    if ((AbsoluteTime_to_scalar(&time) > autoWakeStart) && (AbsoluteTime_to_scalar(&time) < autoWakeEnd))
+    {
+	thread_call_enter_delayed(extraSleepTimer, *((AbsoluteTime *) &autoWakeEnd));
+	return;
+    }
+
     // accelerate disk spin down if spin down timer is non-zero (zero = never spin down)  
     if(0 != user_spindown)
         setQuickSpinDownTimeout();
@@ -644,28 +844,12 @@ void IOPMrootDomain::stopIgnoringClamshellEventsDuringWakeup(void)
 // same thread.
 //*********************************************************************************
 
-static int pmsallsetup = 0;
-
 IOReturn IOPMrootDomain::setAggressiveness ( unsigned long type, unsigned long newLevel )
 {
-#ifdef __ppc__
-	if(pmsExperimental & 3) kprintf("setAggressiveness: type = %08X, newlevel = %08X\n", type, newLevel);
-	if(pmsExperimental & 1) {						/* Is experimental mode enabled? */
-		if(pmsInstalled && (type == kPMSetProcessorSpeed)) {	/* We want to look at all processor speed changes if stepper is installed */
-			if(pmsallsetup) return kIOReturnSuccess;	/* If already running, just eat this */
-			kprintf("setAggressiveness: starting stepper...\n");
-			pmsallsetup = 1;						/* Remember we did this */
-			pmsPark();
-			pmsStart();								/* Get it all started up... */
-			return kIOReturnSuccess;				/* Leave now... */
-		}
-	}
-#endif
+    IOWorkLoop * pmWorkLoop = getPMworkloop();
+    if (pmWorkLoop)
+        pmWorkLoop->runAction(broadcast_aggressiveness,this,(void *)type,(void *)newLevel);
 
-    if ( pm_vars->PMcommandGate ) {
-        pm_vars->PMcommandGate->runAction(broadcast_aggressiveness,(void *)type,(void *)newLevel);
-    }
-    
     return kIOReturnSuccess;
 }
 
@@ -674,20 +858,69 @@ IOReturn IOPMrootDomain::setAggressiveness ( unsigned long type, unsigned long n
 // sleepSystem
 //
 // **********************************************************************************
+/* public */
 IOReturn IOPMrootDomain::sleepSystem ( void )
 {
+    return sleepSystemOptions (NULL);
+}
+
+/* private */
+IOReturn IOPMrootDomain::sleepSystemOptions ( OSDictionary *options )
+{
+	/* sleepSystem is a public function, and may be called by any kernel driver.
+     * And that's bad - drivers should sleep the system by calling 
+     * receivePowerNotification() instead. Drivers should not use sleepSystem.
+     *
+     * Note that user space app calls to IOPMSleepSystem() will also travel
+     * this code path and thus be correctly identified as software sleeps.
+     */
+          
+    if (options && options->getObject("OSSwitch")) 
+    {
+
+        // Log specific sleep cause for OS Switch hibernation
+        return privateSleepSystem( kIOPMOSSwitchHibernationKey) ;
+
+    } else {
+
+        return privateSleepSystem( kIOPMSoftwareSleepKey);
+
+    }
+}
+
+/* private */
+IOReturn IOPMrootDomain::privateSleepSystem ( const char *sleepReason )
+{
+    // Record sleep cause in IORegistry
+    if (sleepReason) {
+        setProperty(kRootDomainSleepReasonKey, sleepReason);
+    }
+
     if(systemShutdown) {
         kprintf("Preventing system sleep on grounds of systemShutdown.\n");
     }
+    
+    if( userDisabledAllSleep )
+    {
+        /* Prevent sleep of all kinds if directed to by user space */
+        return kIOReturnNotPermitted;
+    }
 
-    if ( !systemBooting && !systemShutdown && allowSleep ) {
-        if ( !sleepIsSupported )
+    if ( !systemBooting 
+      && !systemShutdown 
+      && allowSleep)
+    {
+        if ( !sleepIsSupported ) {
             setSleepSupported( kPCICantSleep );
-
+            kprintf("Sleep prevented by kIOPMPreventSystemSleep flag\n");
+        }
         patriarch->sleepSystem();
         return kIOReturnSuccess;
+    } else {
+        // Unable to sleep because system is in the process of booting or shutting down,
+        // or sleep has otherwise been disallowed.
+        return kIOReturnError;
     }
-    return kIOReturnError;
 }
 
 
@@ -729,26 +962,43 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
 {
     OSNumber *		propertyPtr;
     unsigned short	theProperty;
-    AbsoluteTime        deadline;
-    
-    switch ( pm_vars->myCurrentState ) {
+    AbsoluteTime    deadline;
+
+    DEBUG_LOG("PowerChangeDone: %ld -> %ld\n", previousState, getPowerState());
+
+    switch ( getPowerState() ) {
         case SLEEP_STATE:
+			if ( previousState != ON_STATE )
+				break;
+
             if ( canSleep && sleepIsSupported ) 
             {
                 // re-enable this timer for next sleep
                 idleSleepPending = false;			
 
+                uint32_t secs, microsecs;
+                clock_get_calendar_microtime(&secs, &microsecs);
+                logtime(secs);
+                gIOLastSleepTime.tv_sec  = secs;
+                gIOLastSleepTime.tv_usec = microsecs;
+
+#if	HIBERNATION
                 IOLog("System %sSleep\n", gIOHibernateState ? "Safe" : "");
 
                 IOHibernateSystemHasSlept();
+#else
+                IOLog("System Sleep\n");
+#endif
 
-                pm_vars->thePlatform->sleepKernel();
+                getPlatform()->sleepKernel();
 
                 // The CPU(s) are off at this point. When they're awakened by CPU interrupt,
                 // code will resume execution here.
 
                 // Now we're waking...
+#if	HIBERNATION
                 IOHibernateSystemWake();
+#endif
 
                 // stay awake for at least 30 seconds
                 clock_interval_to_deadline(30, kSecondScale, &deadline);	
@@ -763,8 +1013,8 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 // sleep transition complete
                 gSleepOrShutdownPending = 0;
 
-				// trip the reset of the calendar clock
-				clock_wakeup_calendar();
+                // trip the reset of the calendar clock
+                clock_wakeup_calendar();
 
                 // get us some power
                 patriarch->wakeSystem();
@@ -773,7 +1023,9 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 tellClients(kIOMessageSystemWillPowerOn);
 
                 // tell the tree we're waking
+#if	HIBERNATION
                 IOLog("System %sWake\n", gIOHibernateState ? "SafeSleep " : "");
+#endif
                 systemWake();
                 
                 // Allow drivers to request extra processing time before clamshell
@@ -789,7 +1041,7 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 
                 // Find out what woke us
                 propertyPtr = OSDynamicCast(OSNumber,getProperty("WakeEvent"));
-                if ( propertyPtr ) {				
+                if ( propertyPtr ) {
                     theProperty = propertyPtr->unsigned16BitValue();
                     IOLog("Wake event %04x\n",theProperty);
                     if ( (theProperty & 0x0008) ||	//lid
@@ -805,12 +1057,13 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
                 }
                             
                 // Wake for thirty seconds
-                changePowerStateToPriv(ON_STATE);		
-                powerOverrideOffPriv();
+                changePowerStateToPriv(ON_STATE);
             } else {
                 // allow us to step up a power state
                 patriarch->sleepToDoze();
-                // and do it
+
+                // ignore children's request for higher power during doze.
+                powerOverrideOnPriv();
                 changePowerStateToPriv(DOZE_STATE);
             }
             break;
@@ -847,13 +1100,14 @@ void IOPMrootDomain::powerChangeDone ( unsigned long previousState )
 // **********************************************************************************
 void IOPMrootDomain::wakeFromDoze( void )
 {
-    if ( pm_vars->myCurrentState == DOZE_STATE ) 
+    if ( getPowerState() == DOZE_STATE ) 
     {
         // Reset sleep support till next sleep attempt.
         // A machine's support of sleep vs. doze can change over the course of
         // a running system, so we recalculate it before every sleep.
         setSleepSupported(0);
 
+        changePowerStateToPriv(ON_STATE);
         powerOverrideOffPriv();
 
         // early wake notification
@@ -904,6 +1158,8 @@ void IOPMrootDomain::publishFeature(
 
     supportedWhere &= kRD_AllPowerSources; // mask off any craziness!
 
+//    kprintf("IOPMrootDomain::publishFeature [\"%s\":%0x01x]\n", feature, supportedWhere);
+
     if(!supportedWhere) {
         // Feature isn't supported anywhere!
         return;
@@ -967,11 +1223,14 @@ void IOPMrootDomain::publishFeature(
 
     features->release();
 
-    // Notify EnergySaver and all those in user space so they might
-    // re-populate their feature specific UI
-    messageClients(kIOPMMessageFeatureChange, this);
-
     if(featuresDictLock) IOLockUnlock(featuresDictLock);    
+
+    // Notify EnergySaver and all those in user space so they might
+    // re-populate their feature specific UI    
+    if(pmPowerStateQueue) {
+        pmPowerStateQueue->featureChangeOccurred( 
+                            kIOPMMessageFeatureChange, this);
+    }
 }
 
 // *****************************************************************************
@@ -1083,8 +1342,11 @@ IOReturn IOPMrootDomain::removePublishedFeature( uint32_t removeFeatureID )
         setProperty(kRootDomainSupportedFeatures, features);
     
         // Notify EnergySaver and all those in user space so they might
-        // re-populate their feature specific UI
-        messageClients(kIOPMMessageFeatureChange, this);
+        // re-populate their feature specific UI    
+        if(pmPowerStateQueue) {
+            pmPowerStateQueue->featureChangeOccurred( 
+                                kIOPMMessageFeatureChange, this);
+        }
     } else {
         ret = kIOReturnNotFound;
     }
@@ -1249,7 +1511,6 @@ IOReturn IOPMrootDomain::registerPMSettingController(
         return kIOReturnBadArgument;
     }
 
-
     pmso = PMSettingObject::pmSettingObject(
                 (IOPMrootDomain *)this, func, target, 
                 refcon, supportedPowerSources, settings);
@@ -1274,12 +1535,12 @@ IOReturn IOPMrootDomain::registerPMSettingController(
         list->setObject(pmso);
     }
 
+    IORecursiveLockUnlock(settingsCtrlLock);
+    
     ret = kIOReturnSuccess;
 
     // Track this instance by its OSData ptr from now on  
     *handle = pmso;
-    
-    IORecursiveLockUnlock(settingsCtrlLock);
 
 bail_no_unlock:
     if(kIOReturnSuccess != ret) 
@@ -1291,6 +1552,7 @@ bail_no_unlock:
     return ret;
 }
 
+
 //******************************************************************************
 // sleepOnClamshellClosed
 //
@@ -1318,16 +1580,80 @@ void IOPMrootDomain::sendClientClamshellNotification ( void )
        shouldSleepOnClamshellClosed() ? kOSBooleanTrue : kOSBooleanFalse);
 
 
-    /* Argument to message is a bitfield of 
+    /* Argument to message is a bitfiel of 
      *      ( kClamshellStateBit | kClamshellSleepBit )
-     * Carry on the clamshell state change notification from an 
-     * independent thread.
      */
-    pmArbiter->clamshellStateChangeOccurred(
-            (uint32_t) ( (clamshellIsClosed ? kClamshellStateBit : 0)
-             | ( shouldSleepOnClamshellClosed() ? kClamshellSleepBit : 0)));
+    messageClients(kIOPMMessageClamshellStateChange,
+        (void *) ( (clamshellIsClosed ? kClamshellStateBit : 0)
+             | ( shouldSleepOnClamshellClosed() ? kClamshellSleepBit : 0)) );
 }
 
+//******************************************************************************
+// informCPUStateChange
+//
+// Call into PM CPU code so that CPU power savings may dynamically adjust for
+// running on battery, with the lid closed, etc.
+//
+// informCPUStateChange is a no-op on non x86 systems
+// only x86 has explicit support in the IntelCPUPowerManagement kext
+//******************************************************************************
+
+void IOPMrootDomain::informCPUStateChange(
+    uint32_t type, 
+    uint32_t value )
+{
+#ifdef __i386__
+
+    pmioctlVariableInfo_t varInfoStruct;                            
+    int                 pmCPUret = 0;
+    const char          *varNameStr = NULL;
+    int32_t             *varIndex   = NULL;
+
+    if (kInformAC == type) {
+        varNameStr = kIOPMRootDomainBatPowerCString;
+        varIndex = &idxPMCPULimitedPower;
+    } else if (kInformLid == type) {
+        varNameStr = kIOPMRootDomainLidCloseCString;
+        varIndex = &idxPMCPUClamshell;
+    } else {
+        return;
+    }
+    
+    // Set the new value!
+    // pmCPUControl will assign us a new ID if one doesn't exist yet
+    bzero(&varInfoStruct, sizeof(pmioctlVariableInfo_t));
+    varInfoStruct.varID         = *varIndex;
+    varInfoStruct.varType       = vBool;
+    varInfoStruct.varInitValue  = value;
+    varInfoStruct.varCurValue   = value;
+    strncpy( (char *)varInfoStruct.varName,
+             (const char *)varNameStr,
+             strlen(varNameStr) + 1 );                 
+    
+    // Set!
+    pmCPUret = pmCPUControl( PMIOCSETVARINFO, (void *)&varInfoStruct );
+
+    // pmCPU only assigns numerical id's when a new varName is specified
+    if ((0 == pmCPUret)
+        && (*varIndex == kCPUUnknownIndex))
+    {
+        // pmCPUControl has assigned us a new variable ID. 
+        // Let's re-read the structure we just SET to learn that ID.
+        pmCPUret = pmCPUControl( PMIOCGETVARNAMEINFO, (void *)&varInfoStruct );
+
+        if (0 == pmCPUret) 
+        {        
+            // Store it in idxPMCPUClamshell or idxPMCPULimitedPower
+            *varIndex = varInfoStruct.varID;
+        }
+    } 
+    
+    return;
+    
+#endif __i386__
+}
+
+
 //******************************************************************************
 // receivePowerNotification
 //
@@ -1354,7 +1680,8 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
     if (msg & kIOPMOverTemp) 
     {
         IOLog("PowerManagement emergency overtemp signal. Going to sleep!");
-        (void) sleepSystem ();
+        
+        privateSleepSystem (kIOPMThermalEmergencySleepKey);
     }
 
     /*
@@ -1364,7 +1691,7 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
     {
         IOService *pmu = waitForService(serviceMatching("ApplePMU"));
         pmu->callPlatformFunction("prepareForSleep", false, 0, 0, 0, 0);
-        pm_vars->thePlatform->sleepKernel();
+        getPlatform()->sleepKernel();
         pmu->callPlatformFunction("recoverFromSleep", false, 0, 0, 0, 0);
     }
 
@@ -1373,7 +1700,7 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
      */
     if (msg & kIOPMSleepNow) 
     {
-      (void) sleepSystem ();
+        privateSleepSystem (kIOPMSoftwareSleepKey);
     }
     
     /*
@@ -1381,7 +1708,7 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
      */
     if (msg & kIOPMPowerEmergency) 
     {
-      (void) sleepSystem ();
+        privateSleepSystem (kIOPMLowPowerSleepKey);
     }
 
     
@@ -1395,6 +1722,10 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
         clamshellIsClosed = false;
         clamshellExists = true;
                 
+        // Tell PMCPU
+        informCPUStateChange(kInformLid, 0);
+
+        // Tell general interest clients        
         sendClientClamshellNotification();
     } 
 
@@ -1409,6 +1740,10 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
         clamshellIsClosed = true;
         clamshellExists = true;
 
+        // Tell PMCPU
+        informCPUStateChange(kInformLid, 1);
+
+        // Tell general interest clients
         sendClientClamshellNotification();
         
         // And set eval_clamshell = so we can attempt 
@@ -1444,6 +1779,9 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
         acAdaptorConnect = (0 != (msg & kIOPMSetValue));
         msg &= ~(kIOPMSetACAdaptorConnected | kIOPMSetValue);
 
+        // Tell PMCPU
+        informCPUStateChange(kInformAC, !acAdaptorConnect);
+
         sendClientClamshellNotification();
 
         // Re-evaluate the lid state
@@ -1494,29 +1832,29 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
 
 
         // SLEEP!
-        sleepSystem();
+        privateSleepSystem (kIOPMClamshellSleepKey);
     }
 
     /*
      * Power Button
      */
     if (msg & kIOPMPowerButton) 
-    {				
+    {
         // toggle state of sleep/wake
         // are we dozing?
-        if ( pm_vars->myCurrentState == DOZE_STATE ) 
+        if ( getPowerState() == DOZE_STATE ) 
         {
             // yes, tell the tree we're waking 
             systemWake();
             // wake the Display Wrangler
-            reportUserInput();					
+            reportUserInput();
         }
         else {
             OSString *pbs = OSString::withCString("DisablePowerButtonSleep");
             // Check that power button sleep is enabled
             if( pbs ) {
                 if( kOSBooleanTrue != getProperty(pbs))
-                sleepSystem();
+                privateSleepSystem (kIOPMPowerButtonSleepKey);
             }
         }
     }
@@ -1538,7 +1876,7 @@ IOReturn IOPMrootDomain::receivePowerNotification (UInt32 msg)
     if (msg & kIOPMPreventSleep) {
         allowSleep = false;
 	    // are we dozing?
-        if ( pm_vars->myCurrentState == DOZE_STATE ) {
+        if ( getPowerState() == DOZE_STATE ) {
             // yes, tell the tree we're waking 
             systemWake();
             adjustPowerState();
@@ -1578,6 +1916,7 @@ void IOPMrootDomain::setSleepSupported( IOOptionBits flags )
 // requestPowerDomainState
 //
 // The root domain intercepts this call to the superclass.
+// Called on the PM work loop thread.
 //
 // If the clamp bit is not set in the desire, then the child doesn't need the power
 // state it's requesting; it just wants it.  The root ignores desires but not needs.
@@ -1586,63 +1925,122 @@ void IOPMrootDomain::setSleepSupported( IOOptionBits flags )
 // no power, we are on our way to idle sleep.
 //*********************************************************************************
 
-IOReturn IOPMrootDomain::requestPowerDomainState ( IOPMPowerFlags desiredState, IOPowerConnection * whichChild, unsigned long specification )
+IOReturn IOPMrootDomain::requestPowerDomainState (
+    IOPMPowerFlags      desiredState,
+    IOPowerConnection * whichChild,
+    unsigned long       specification )
 {
-    OSIterator                              *iter;
-    OSObject                                *next;
-    IOPowerConnection                       *connection;
-    unsigned long                           powerRequestFlag = 0;
-    IOPMPowerFlags                          editedDesire = desiredState;
+    OSIterator          *iter;
+    OSObject            *next;
+    IOPowerConnection   *connection;
+    unsigned long       powerRequestFlag = 0;
+    IOPMPowerFlags      editedDesire;
+
+#if DEBUG
+    IOService           *powerChild;
+    powerChild = (IOService *) whichChild->getChildEntry(gIOPowerPlane);
+#endif
 
-    // if they don't really need it, they don't get it
-    if ( !(desiredState & kIOPMPreventIdleSleep) ) {
-        editedDesire = 0;
-    }
+    DEBUG_LOG("RequestPowerDomainState: flags %lx, child %p [%s], spec %lx\n",
+        desiredState, powerChild, powerChild ? powerChild->getName() : "?",
+        specification);
+    
+    // Force the child's input power requirements to 0 unless the prevent
+    // idle-sleep flag is set. No input power flags map to our state 0.
+    // Our power clamp (deviceDesire) keeps the minimum power state at 2.
 
+    if (desiredState & kIOPMPreventIdleSleep)
+        editedDesire = desiredState;
+    else
+        editedDesire = 0;
 
-    IOLockLock(pm_vars->childLock);
+    // Recompute sleep supported flag (doze if not supported)
+    sleepIsSupported = true;
 
-    // recompute sleepIsSupported and see if all children are asleep
     iter = getChildIterator(gIOPowerPlane);
-    sleepIsSupported = true;
     if ( iter ) 
     {
         while ( (next = iter->getNextObject()) ) 
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) ) 
             {
+                // Ignore child that are in the process of joining.
+				if (connection->getReadyFlag() == false)
+					continue;
+
+                // Is this connection attached to the child that called
+                // requestPowerDomainState()?
+
                 if ( connection == whichChild ) 
                 {
-                    powerRequestFlag += editedDesire;
-                    if ( desiredState & kIOPMPreventSystemSleep ) 
-                    {
+                    // Yes, OR in the child's input power requirements.
+                    powerRequestFlag |= editedDesire;
+
+                    if ( desiredState & kIOPMPreventSystemSleep )
                         sleepIsSupported = false;
-                    }
-                } else {
-                    powerRequestFlag += connection->getDesiredDomainState();
-                    if ( connection->getPreventSystemSleepFlag() ) 
-                    {
+                }
+                else
+                {
+#if DEBUG
+                    powerChild = (IOService *) connection->getChildEntry(gIOPowerPlane);
+#endif
+                    DEBUG_LOG("  child %p, PState %ld, noIdle %d, noSleep %d, valid %d  %s\n",
+                        powerChild,
+                        connection->getDesiredDomainState(),
+                        connection->getPreventIdleSleepFlag(),
+                        connection->getPreventSystemSleepFlag(),
+                        connection->getReadyFlag(),
+                        powerChild ? powerChild->getName() : "?");
+
+                    // No, OR in the child's desired power domain state.
+                    // Which is our power state desired by this child.
+                    powerRequestFlag |= connection->getDesiredDomainState();
+
+                    if ( connection->getPreventSystemSleepFlag() )
                         sleepIsSupported = false;
-                    }
                 }
             }
         }
         iter->release();
     }
     
-    if ( (extraSleepDelay == 0) &&  (powerRequestFlag == 0) ) 
+    if ( !powerRequestFlag && !systemBooting ) 
     {
-        sleepASAP = true;
+	if (!wrangler)
+	{
+	    sleepASAP = false;
+	    changePowerStateToPriv(ON_STATE);
+	    if (idleSeconds)
+	    {
+		AbsoluteTime deadline;
+		// stay awake for at least idleSeconds
+		clock_interval_to_deadline(idleSeconds, kSecondScale, &deadline);	
+		thread_call_enter_delayed(extraSleepTimer, deadline);
+		// this gets turned off when we sleep again
+		idleSleepPending = true;
+	    }
+	}
+	else if (extraSleepDelay == 0)
+	{
+	    sleepASAP = true;
+	}
     }
     
-    // this may put the system to sleep
+    DEBUG_LOG("  sleepDelay %lx, mergedFlags %lx, sleepASAP %x, booting %x\n",
+        extraSleepDelay, powerRequestFlag, sleepASAP, systemBooting);
+
+    // Drop our power clamp to SLEEP_STATE when all devices become idle.
+    // Needed when the system sleep and display sleep timeouts are the same.
+    // Otherwise, the extra sleep timer will also drop our power clamp.
+
     adjustPowerState();
-    
-    IOLockUnlock(pm_vars->childLock);
 
-    editedDesire |= desiredState & kIOPMPreventSystemSleep;
+    editedDesire |= (desiredState & kIOPMPreventSystemSleep);
 
-    return super::requestPowerDomainState(editedDesire,whichChild,specification);
+    // If our power clamp has already dropped to SLEEP_STATE, and no child
+    // is keeping us at max power, then this will trigger idle sleep.
+
+    return super::requestPowerDomainState(editedDesire, whichChild, specification);
 }
 
 
@@ -1657,6 +2055,99 @@ IOOptionBits IOPMrootDomain::getSleepSupported( void )
 }
 
 
+//*********************************************************************************
+// handlePlatformHaltRestart
+//
+//*********************************************************************************
+
+struct HaltRestartApplierContext {
+	IOPMrootDomain *	RootDomain;
+	unsigned long		PowerState;
+	IOPMPowerFlags		PowerFlags;
+	UInt32				MessageType;
+	UInt32				Counter;
+};
+
+static void
+platformHaltRestartApplier( OSObject * object, void * context )
+{
+	IOPowerStateChangeNotification	notify;
+	HaltRestartApplierContext *		ctx;
+	AbsoluteTime					startTime;
+	UInt32							deltaTime;
+
+	ctx = (HaltRestartApplierContext *) context;
+	
+	memset(&notify, 0, sizeof(notify));
+    notify.powerRef    = (void *)ctx->Counter;
+    notify.returnValue = 0;
+    notify.stateNumber = ctx->PowerState;
+    notify.stateFlags  = ctx->PowerFlags;
+
+	clock_get_uptime(&startTime);
+    ctx->RootDomain->messageClient( ctx->MessageType, object, (void *)&notify );
+	deltaTime = computeDeltaTimeMS(&startTime);
+
+	if ((deltaTime > kPMHaltTimeoutMS) || (gIOKitDebug & kIOLogDebugPower))
+	{
+		_IOServiceInterestNotifier * notifier;
+		notifier = OSDynamicCast(_IOServiceInterestNotifier, object);
+
+		// IOService children of IOPMrootDomain are not instrumented.
+		// Only IORootParent currently falls under that group.
+
+		if (notifier)
+		{
+			HaltRestartLog("%s handler %p took %lu ms\n",
+				(ctx->MessageType == kIOMessageSystemWillPowerOff) ?
+					"PowerOff" : "Restart",
+				notifier->handler, deltaTime );
+		}
+	}
+
+	ctx->Counter++;
+}
+
+void IOPMrootDomain::handlePlatformHaltRestart( UInt32 pe_type )
+{
+	HaltRestartApplierContext	ctx;
+	AbsoluteTime				startTime;
+	UInt32						deltaTime;
+
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.RootDomain = this;
+
+	clock_get_uptime(&startTime);
+	switch (pe_type)
+	{
+		case kPEHaltCPU:
+			ctx.PowerState  = OFF_STATE;
+			ctx.MessageType = kIOMessageSystemWillPowerOff;
+			break;
+		
+		case kPERestartCPU:
+			ctx.PowerState  = RESTART_STATE;
+			ctx.MessageType = kIOMessageSystemWillRestart;
+			break;
+
+		default:
+			return;
+	}
+
+	// Notify legacy clients
+	applyToInterested(gIOPriorityPowerStateInterest, platformHaltRestartApplier, &ctx);
+
+	// Notify in power tree order
+	notifySystemShutdown(this, ctx.MessageType);
+
+	deltaTime = computeDeltaTimeMS(&startTime);
+	HaltRestartLog("%s all drivers took %lu ms\n",
+		(ctx.MessageType == kIOMessageSystemWillPowerOff) ?
+			"PowerOff" : "Restart",
+		deltaTime );
+}
+
+
 //*********************************************************************************
 // tellChangeDown
 //
@@ -1675,20 +2166,7 @@ bool IOPMrootDomain::tellChangeDown ( unsigned long stateNum )
             OSMetaClassSystemSleepOrWake( kIOMessageSystemWillSleep );
 
             return super::tellClientsWithResponse(kIOMessageSystemWillSleep);
-        case RESTART_STATE:
-            // Unsupported shutdown ordering hack on RESTART only
-            // For Bluetooth and USB (4368327)
-            super::tellClients(iokit_common_msg(0x759));
-
-            return super::tellClientsWithResponse(kIOMessageSystemWillRestart);
-        case OFF_STATE:
-            // Unsupported shutdown ordering hack on SHUTDOWN only
-            // For Bluetooth and USB (4554440)
-            super::tellClients(iokit_common_msg(0x749));
-
-            return super::tellClientsWithResponse(kIOMessageSystemWillPowerOff);
     }
-    // this shouldn't execute
     return super::tellChangeDown(stateNum);
 }
 
@@ -1722,6 +2200,16 @@ bool IOPMrootDomain::askChangeDown ( unsigned long )
 
 void IOPMrootDomain::tellNoChangeDown ( unsigned long )
 {
+    if (idleSeconds && !wrangler)
+    {
+	AbsoluteTime deadline;
+	sleepASAP = false;
+	// stay awake for at least idleSeconds
+	clock_interval_to_deadline(idleSeconds, kSecondScale, &deadline);	
+	thread_call_enter_delayed(extraSleepTimer, deadline);
+	// this gets turned off when we sleep again
+	idleSleepPending = true;
+    }
     return tellClients(kIOMessageSystemWillNotSleep);
 }
 
@@ -1739,11 +2227,13 @@ void IOPMrootDomain::tellChangeUp ( unsigned long stateNum)
 {
     if ( stateNum == ON_STATE ) 
     {
+#if	HIBERNATION
         // Direct callout into OSMetaClass so it can disable kmod unloads
         // during sleep/wake to prevent deadlocks.
         OSMetaClassSystemSleepOrWake( kIOMessageSystemHasPoweredOn );
 
 	IOHibernateSystemPostWake();
+#endif
         return tellClients(kIOMessageSystemHasPoweredOn);
     }
 }
@@ -1755,6 +2245,7 @@ void IOPMrootDomain::tellChangeUp ( unsigned long stateNum)
 
 void IOPMrootDomain::reportUserInput ( void )
 {
+#if !NO_KERNEL_HID
     OSIterator * iter;
 
     if(!wrangler) 
@@ -1769,6 +2260,7 @@ void IOPMrootDomain::reportUserInput ( void )
 
     if(wrangler)
         wrangler->activityTickle(0,0);
+#endif
 }
 
 //*********************************************************************************
@@ -1806,9 +2298,18 @@ IOReturn IOPMrootDomain::changePowerStateToPriv ( unsigned long ordinal )
 {
     IOReturn    ret;
 
-    if( (systemBooting || systemShutdown) && (ordinal == SLEEP_STATE) )
+    DEBUG_LOG("ChangePowerStateToPriv: power state %ld\n", ordinal);
+
+	if ( (getPowerState() == DOZE_STATE) && (ordinal != ON_STATE) )
+	{
+		return kIOReturnSuccess;
+	}
+
+    if( (userDisabledAllSleep || systemBooting || systemShutdown) 
+        && (ordinal == SLEEP_STATE) )
     {
-        kprintf("DANGER DANGER DANGER unexpected code path. aborting SLEEPSTATE change.\n");
+        DEBUG_LOG("  sleep denied: disableAllSleep %d, booting %d, shutdown %d\n",
+            userDisabledAllSleep, systemBooting, systemShutdown);
         super::changePowerStateToPriv(ON_STATE);
     }
 
@@ -1821,7 +2322,7 @@ IOReturn IOPMrootDomain::changePowerStateToPriv ( unsigned long ordinal )
                             NULL, NULL, NULL, NULL);
     
         // If the machine only supports doze, the callPlatformFunction call
-        // boils down toIOPMrootDomain::setSleepSupported(kPCICantSleep), 
+        // boils down to IOPMrootDomain::setSleepSupported(kPCICantSleep), 
         // otherwise nothing.
     }
 
@@ -1852,8 +2353,7 @@ IOReturn IOPMrootDomain::sysPowerDownHandler( void * target, void * refCon,
 
     switch (messageType) {
         case kIOMessageSystemWillSleep:
-            rootDomain->powerOverrideOnPriv();		// start ignoring children's requests
-                                                        // (fall through to other cases)
+            DEBUG_LOG("SystemWillSleep\n");
 
             // Interested applications have been notified of an impending power
             // change and have acked (when applicable).
@@ -1864,8 +2364,10 @@ IOReturn IOPMrootDomain::sysPowerDownHandler( void * target, void * refCon,
 
             // We will ack within 20 seconds
             params->returnValue = 20 * 1000 * 1000;
+#if	HIBERNATION
             if (gIOHibernateState)
                 params->returnValue += gIOHibernateFreeTime * 1000; 	//add in time we could spend freeing pages
+#endif
 
             if ( ! OSCompareAndSwap( 0, 1, &gSleepOrShutdownPending ) )
             {
@@ -1902,77 +2404,110 @@ IOReturn IOPMrootDomain::sysPowerDownHandler( void * target, void * refCon,
 //
 // Allows us to take action on display dim/undim.
 //
-// When the display goes dim we:
+// When the display sleeps we:
 // - Start the idle sleep timer
 // - set the quick spin down timeout
 //
-// On wake from display dim:
+// On wake from display sleep:
 // - Cancel the idle sleep timer
 // - restore the user's chosen spindown timer from the "quick" spin down value
 //*********************************************************************************
 
-IOReturn IOPMrootDomain::displayWranglerNotification( void * target, void * refCon,
-                                    UInt32 messageType, IOService * service,
-                                    void * messageArgument, vm_size_t argSize )
+IOReturn IOPMrootDomain::displayWranglerNotification(
+    void * target, void * refCon,
+    UInt32 messageType, IOService * service,
+    void * messageArgument, vm_size_t argSize )
 {
-    IOPMrootDomain *                rootDomain = OSDynamicCast(IOPMrootDomain, (IOService *)target);
-    AbsoluteTime                 deadline;
-    static bool                  deviceAlreadyPoweredOff = false;
+#if !NO_KERNEL_HID
+    IOPMrootDomain *    rootDomain = OSDynamicCast(IOPMrootDomain, (IOService *)target);
+    AbsoluteTime        deadline;
+    static int          displayPowerState = 4;
 
-    if(!rootDomain)
+    if (!rootDomain)
         return kIOReturnUnsupported;
 
     switch (messageType) {
        case kIOMessageDeviceWillPowerOff:
-            // The IODisplayWrangler has powered off either because of idle display sleep
-            // or force system sleep.
-            
-            // The display wrangler will send the DeviceWillPowerOff message 4 times until
-            // it gets into its lowest state. We only want to act on the first of those 4.
-            if( deviceAlreadyPoweredOff ) return kIOReturnUnsupported;
-
-           deviceAlreadyPoweredOff = true;
-
-           if( rootDomain->extraSleepDelay ) 
-           {
-                // start the extra sleep timer
-                clock_interval_to_deadline(rootDomain->extraSleepDelay*60, kSecondScale, &deadline );
+            DEBUG_LOG("DisplayWranglerWillPowerOff: new p-state %d\n",
+                displayPowerState - 1);
+
+            // The display wrangler has dropped power because of idle display sleep
+            // or force system sleep. We will receive 4 messages before the display
+            // wrangler reaches its lowest state. Act only when going to state 2.
+            //
+            // 4->3 Display Dim
+            // 3->2 Display Sleep
+            // 2->1 Not visible to user
+            // 1->0 Not visible to user
+
+            displayPowerState--;
+            if ( 2 != displayPowerState )
+                return kIOReturnUnsupported;
+
+            // We start a timer here if the System Sleep timer is greater than the
+            // Display Sleep timer. We kick off this timer when the display sleeps.
+            //
+            // Note that, although Display Dim timings may change adaptively accordingly
+            // to the user's activity patterns, Display Sleep _always_ occurs at the
+            // specified interval since last user activity.
+
+            if ( rootDomain->extraSleepDelay )
+            {
+                clock_interval_to_deadline(rootDomain->extraSleepDelay*60, kSecondScale, &deadline);
                 thread_call_enter_delayed(rootDomain->extraSleepTimer, deadline);
                 rootDomain->idleSleepPending = true;
+                DEBUG_LOG("  sleep timer set to expire in %ld min\n",
+                    rootDomain->extraSleepDelay);
             } else {
-                // accelerate disk spin down if spin down timer is non-zero (zero = never spin down)
-                // and if system sleep is non-Never
-                if( (0 != rootDomain->user_spindown) && (0 != rootDomain->sleepSlider) )
+                // Accelerate disk spindown if system sleep and display sleep
+                // sliders are set to the same value (e.g. both set to 5 min),
+                // and display is about to go dark. Check that spin down timer
+                // is non-zero (zero = never spin down) and system sleep is
+                // not set to never sleep.
+
+                if ( (0 != rootDomain->user_spindown) && (0 != rootDomain->sleepSlider) )
+                {
+                    DEBUG_LOG("  accelerate quick disk spindown, was %d min\n",
+                        rootDomain->user_spindown);
                     rootDomain->setQuickSpinDownTimeout();
+                }
             }
 
-             break;
+            break;
 
         case kIOMessageDeviceHasPoweredOn:
+            DEBUG_LOG("DisplayWranglerHasPoweredOn: previous p-state %d\n",
+                displayPowerState);
 
-            // The display has powered on either because of UI activity or wake from sleep/doze
-            deviceAlreadyPoweredOff = false;
+            // The display wrangler has powered on either because of user activity 
+            // or wake from sleep/doze.
+
+            displayPowerState = 4;
             rootDomain->adjustPowerState();
-            
 
-            // cancel any pending idle sleep
-            if(rootDomain->idleSleepPending) 
+            // cancel any pending idle sleep timers
+            if (rootDomain->idleSleepPending) 
             {
+                DEBUG_LOG("  extra-sleep timer stopped\n");
                 thread_call_cancel(rootDomain->extraSleepTimer);
                 rootDomain->idleSleepPending = false;
             }
 
-            // Change the spindown value back to the user's selection from our accelerated setting
-            if(0 != rootDomain->user_spindown)
+            // Change the spindown value back to the user's selection from our
+            // accelerated setting.
+            if (0 != rootDomain->user_spindown)
+            {
+                DEBUG_LOG("  restoring disk spindown to %d min\n",
+                    rootDomain->user_spindown);
                 rootDomain->restoreUserSpinDownTimeout();
-
-            // Put on the policy maker's on clamp.
+            }
 
             break;
 
          default:
              break;
      }
+#endif
      return kIOReturnUnsupported;
  }
 
@@ -1989,6 +2524,7 @@ bool IOPMrootDomain::displayWranglerPublished(
     void * refCon,
     IOService * newService)
 {
+#if !NO_KERNEL_HID
     IOPMrootDomain *rootDomain = 
             OSDynamicCast(IOPMrootDomain, (IOService *)target);
 
@@ -2003,11 +2539,10 @@ bool IOPMrootDomain::displayWranglerPublished(
     {
         return false;
     }
-    
+#endif
     return true;
 }
 
-
 //*********************************************************************************
 // batteryPublished
 //
@@ -2030,8 +2565,6 @@ bool IOPMrootDomain::batteryPublished(
     return (true);
 }
 
-
-
 //*********************************************************************************
 // adjustPowerState
 //
@@ -2059,24 +2592,478 @@ void IOPMrootDomain::adjustPowerState( void )
     if ( (sleepSlider == 0) 
         || !allowSleep 
         || systemBooting 
-        || systemShutdown )
+        || systemShutdown
+        || userDisabledAllSleep )
     {
-        if(systemBooting || systemShutdown) {
-            kprintf("adjusting power state to ON_STATE [2063] on grounds of systemBooting.\n");
-        }
-        
+        DEBUG_LOG("AdjustPowerState %ld -> ON: slider %ld, allowSleep %d, "
+            "booting %d, shutdown %d, userDisabled %d\n",
+            getPowerState(), sleepSlider, allowSleep, systemBooting,
+            systemShutdown, userDisabledAllSleep);
+
         changePowerStateToPriv(ON_STATE);
     } else {
         if ( sleepASAP ) 
         {
+            DEBUG_LOG("AdjustPowerState SLEEP\n");
+
+            /* Convenient place to run any code at idle sleep time
+             * IOPMrootDomain initiates an  idle sleep here
+             *
+             * Set last sleep cause accordingly.
+             */
+            setProperty(kRootDomainSleepReasonKey, kIOPMIdleSleepKey);
+        
             sleepASAP = false;
-            if ( !sleepIsSupported )
+            if ( !sleepIsSupported ) 
+            {
                 setSleepSupported( kPCICantSleep );
+                kprintf("Sleep prevented by kIOPMPreventSystemSleep flag\n");
+            }
             changePowerStateToPriv(SLEEP_STATE);
         }
     }
 }
 
+//*********************************************************************************
+// PMHaltWorker Class
+//
+//*********************************************************************************
+
+static unsigned int		gPMHaltBusyCount;
+static unsigned int		gPMHaltIdleCount;
+static int				gPMHaltDepth;
+static unsigned long    gPMHaltEvent;
+static IOLock *			gPMHaltLock  = 0;
+static OSArray *		gPMHaltArray = 0;
+static const OSSymbol * gPMHaltClientAcknowledgeKey = 0;
+
+PMHaltWorker * PMHaltWorker::worker( void )
+{
+	PMHaltWorker *	me;
+	IOThread		thread;
+
+	do {
+		me = OSTypeAlloc( PMHaltWorker );
+		if (!me || !me->init())
+			break;
+
+		me->lock = IOLockAlloc();
+		if (!me->lock)
+			break;
+
+		DEBUG_LOG("PMHaltWorker %p\n", me);
+		me->retain();	// thread holds extra retain
+		thread = IOCreateThread( &PMHaltWorker::main, me );
+		if (!thread)
+		{
+			me->release();
+			break;
+		}
+		return me;
+
+	} while (false);
+
+	if (me) me->release();
+	return 0;
+}
+
+void PMHaltWorker::free( void )
+{
+	DEBUG_LOG("PMHaltWorker free %p\n", this);
+	if (lock)
+	{
+		IOLockFree(lock);
+		lock = 0;
+	}
+	return OSObject::free();
+}
+
+void PMHaltWorker::main( void * arg )
+{
+	PMHaltWorker * me = (PMHaltWorker *) arg;
+
+	IOLockLock( gPMHaltLock );
+	gPMHaltBusyCount++;
+	me->depth = gPMHaltDepth;
+	IOLockUnlock( gPMHaltLock );
+
+	while (me->depth >= 0)
+	{
+		PMHaltWorker::work( me );
+
+		IOLockLock( gPMHaltLock );
+		if (++gPMHaltIdleCount >= gPMHaltBusyCount)
+		{
+			// This is the last thread to finish work on this level,
+			// inform everyone to start working on next lower level.
+			gPMHaltDepth--;
+			me->depth = gPMHaltDepth;
+			gPMHaltIdleCount = 0;
+			thread_wakeup((event_t) &gPMHaltIdleCount);
+		}
+		else
+		{
+			// One or more threads are still working on this level,
+			// this thread must wait.
+			me->depth = gPMHaltDepth - 1;
+			do {
+				IOLockSleep(gPMHaltLock, &gPMHaltIdleCount, THREAD_UNINT);
+			} while (me->depth != gPMHaltDepth);
+		}
+		IOLockUnlock( gPMHaltLock );
+	}
+
+	// No more work to do, terminate thread
+	DEBUG_LOG("All done for worker: %p (visits = %u)\n", me, me->visits);
+	thread_wakeup( &gPMHaltDepth );
+	me->release();
+}
+
+void PMHaltWorker::work( PMHaltWorker * me )
+{
+	IOService *		service;
+	OSSet *			inner;
+	AbsoluteTime	startTime;
+	UInt32			deltaTime;
+	bool			timeout;
+
+	while (true)
+	{
+		service = 0;
+		timeout = false;
+
+		// Claim an unit of work from the shared pool
+		IOLockLock( gPMHaltLock );
+		inner = (OSSet *)gPMHaltArray->getObject(me->depth);
+		if (inner)
+		{
+			service = (IOService *)inner->getAnyObject();
+			if (service)
+			{
+				service->retain();
+				inner->removeObject(service);
+			}
+		}
+		IOLockUnlock( gPMHaltLock );	
+		if (!service)
+			break;  // no more work at this depth
+
+		clock_get_uptime(&startTime);
+
+		if (!service->isInactive() &&
+			service->setProperty(gPMHaltClientAcknowledgeKey, me))
+		{
+			IOLockLock(me->lock);
+			me->startTime = startTime;
+			me->service   = service;
+			me->timeout   = false;
+			IOLockUnlock(me->lock);
+
+			service->systemWillShutdown( gPMHaltEvent );
+
+			// Wait for driver acknowledgement
+			IOLockLock(me->lock);
+			while (service->getProperty(gPMHaltClientAcknowledgeKey))
+			{
+				IOLockSleep(me->lock, me, THREAD_UNINT);			
+			}
+			me->service = 0;
+			timeout = me->timeout;
+			IOLockUnlock(me->lock);
+		}
+
+		deltaTime = computeDeltaTimeMS(&startTime);
+		if ((deltaTime > kPMHaltTimeoutMS) || timeout ||
+			(gIOKitDebug & kIOLogDebugPower))
+		{
+			HaltRestartLog("%s driver %s (%p) took %lu ms\n",
+				(gPMHaltEvent == kIOMessageSystemWillPowerOff) ?
+					"PowerOff" : "Restart",
+				service->getName(), service,
+				deltaTime );
+		}
+
+		service->release();
+		me->visits++;
+	}
+}
+
+void PMHaltWorker::checkTimeout( PMHaltWorker * me, AbsoluteTime * now )
+{
+	UInt64			nano;
+	AbsoluteTime	startTime;
+	AbsoluteTime	endTime;
+
+	endTime = *now;
+
+	IOLockLock(me->lock);
+	if (me->service && !me->timeout)
+	{
+		startTime = me->startTime;
+		nano = 0;
+	    if (CMP_ABSOLUTETIME(&endTime, &startTime) > 0)
+	    {
+			SUB_ABSOLUTETIME(&endTime, &startTime);
+			absolutetime_to_nanoseconds(endTime, &nano);
+	    }
+		if (nano > 3000000000ULL)
+		{
+			me->timeout = true;
+			HaltRestartLog("%s still waiting on %s\n",
+				(gPMHaltEvent == kIOMessageSystemWillPowerOff) ?
+					"PowerOff" : "Restart",
+				me->service->getName());
+		}
+	}
+	IOLockUnlock(me->lock);
+}
+
+//*********************************************************************************
+// acknowledgeSystemWillShutdown
+//
+// Acknowledgement from drivers that they have prepared for shutdown/restart.
+//*********************************************************************************
+
+void IOPMrootDomain::acknowledgeSystemWillShutdown( IOService * from )
+{
+	PMHaltWorker *	worker;
+	OSObject *		prop;
+
+	if (!from)
+		return;
+
+	//DEBUG_LOG("%s acknowledged\n", from->getName());
+	prop = from->copyProperty( gPMHaltClientAcknowledgeKey );
+	if (prop)
+	{
+		worker = (PMHaltWorker *) prop;
+		IOLockLock(worker->lock);
+		from->removeProperty( gPMHaltClientAcknowledgeKey );
+		thread_wakeup((event_t) worker);
+		IOLockUnlock(worker->lock);
+		worker->release();
+	}
+	else
+	{
+		DEBUG_LOG("%s acknowledged without worker property\n",
+			from->getName());
+	}
+}
+
+//*********************************************************************************
+// notifySystemShutdown
+//
+// Notify all objects in PM tree that system will shutdown or restart
+//*********************************************************************************
+
+static void
+notifySystemShutdown( IOService * root, unsigned long event )
+{
+#define PLACEHOLDER ((OSSet *)gPMHaltArray)
+	IORegistryIterator *	iter;
+	IORegistryEntry *		entry;
+	IOService *				node;
+	OSSet *					inner;
+	PMHaltWorker *			workers[kPMHaltMaxWorkers];
+	AbsoluteTime			deadline;
+	unsigned int			totalNodes = 0;
+	unsigned int			depth;
+	unsigned int			rootDepth;
+	unsigned int			numWorkers;
+	unsigned int			count;
+	int						waitResult;
+	void *					baseFunc;
+	bool					ok;
+
+	DEBUG_LOG("%s event = %lx\n", __FUNCTION__, event);
+
+	baseFunc = OSMemberFunctionCast(void *, root, &IOService::systemWillShutdown);
+
+	// Iterate the entire PM tree starting from root
+
+	rootDepth = root->getDepth( gIOPowerPlane );
+	if (!rootDepth) goto done;
+
+	// debug - for repeated test runs
+	while (PMHaltWorker::metaClass->getInstanceCount())
+		IOSleep(1);
+
+	if (!gPMHaltArray)
+	{
+		gPMHaltArray = OSArray::withCapacity(40);
+		if (!gPMHaltArray) goto done;
+	}
+	else // debug
+		gPMHaltArray->flushCollection();
+
+	if (!gPMHaltLock)
+	{
+		gPMHaltLock = IOLockAlloc();
+		if (!gPMHaltLock) goto done;
+	}
+
+	if (!gPMHaltClientAcknowledgeKey)
+	{
+		gPMHaltClientAcknowledgeKey =
+			OSSymbol::withCStringNoCopy("PMShutdown");
+		if (!gPMHaltClientAcknowledgeKey) goto done;
+	}
+
+	gPMHaltEvent = event;
+
+	// Depth-first walk of PM plane
+
+	iter = IORegistryIterator::iterateOver(
+		root, gIOPowerPlane, kIORegistryIterateRecursively);
+
+	if (iter)
+	{
+		while ((entry = iter->getNextObject()))
+		{
+			node = OSDynamicCast(IOService, entry);
+			if (!node)
+				continue;
+
+			if (baseFunc == 
+				OSMemberFunctionCast(void *, node, &IOService::systemWillShutdown))
+				continue;
+
+			depth = node->getDepth( gIOPowerPlane );
+			if (depth <= rootDepth)
+				continue;
+
+			ok = false;
+
+			// adjust to zero based depth
+			depth -= (rootDepth + 1);
+
+			// gPMHaltArray is an array of containers, each container
+			// refers to nodes with the same depth.
+
+			count = gPMHaltArray->getCount();
+			while (depth >= count)
+			{
+				// expand array and insert placeholders
+				gPMHaltArray->setObject(PLACEHOLDER);
+				count++;
+			}
+			count = gPMHaltArray->getCount();
+			if (depth < count)
+			{
+				inner = (OSSet *)gPMHaltArray->getObject(depth);
+				if (inner == PLACEHOLDER)
+				{
+					inner = OSSet::withCapacity(40);
+					if (inner)
+					{
+						gPMHaltArray->replaceObject(depth, inner);
+						inner->release();
+					}
+				}
+
+				// PM nodes that appear more than once in the tree will have
+				// the same depth, OSSet will refuse to add the node twice.
+				if (inner)
+					ok = inner->setObject(node);
+			}
+			if (!ok)
+				DEBUG_LOG("Skipped PM node %s\n", node->getName());
+		}
+		iter->release();
+	}
+
+	// debug only
+	for (int i = 0; (inner = (OSSet *)gPMHaltArray->getObject(i)); i++)
+	{
+		count = 0;
+		if (inner != PLACEHOLDER)
+			count = inner->getCount();
+		DEBUG_LOG("Nodes at depth %u = %u\n", i, count);
+	}
+
+	// strip placeholders (not all depths are populated)
+	numWorkers = 0;
+	for (int i = 0; (inner = (OSSet *)gPMHaltArray->getObject(i)); )
+	{
+		if (inner == PLACEHOLDER)
+		{
+			gPMHaltArray->removeObject(i);
+			continue;
+		}
+		count = inner->getCount();
+		if (count > numWorkers)
+			numWorkers = count;
+		totalNodes += count;
+		i++;
+	}
+
+	if (gPMHaltArray->getCount() == 0 || !numWorkers)
+		goto done;
+
+	gPMHaltBusyCount = 0;
+	gPMHaltIdleCount = 0;
+	gPMHaltDepth = gPMHaltArray->getCount() - 1;
+
+	// Create multiple workers (and threads)
+
+	if (numWorkers > kPMHaltMaxWorkers)
+		numWorkers = kPMHaltMaxWorkers;
+
+	DEBUG_LOG("PM nodes = %u, maxDepth = %u, workers = %u\n",
+		totalNodes, gPMHaltArray->getCount(), numWorkers);
+
+	for (unsigned int i = 0; i < numWorkers; i++)
+		workers[i] = PMHaltWorker::worker();
+
+	// Wait for workers to exhaust all available work
+
+	IOLockLock(gPMHaltLock);
+	while (gPMHaltDepth >= 0)
+	{
+		clock_interval_to_deadline(1000, kMillisecondScale, &deadline);
+
+		waitResult = IOLockSleepDeadline(
+			gPMHaltLock, &gPMHaltDepth, deadline, THREAD_UNINT);
+		if (THREAD_TIMED_OUT == waitResult)
+		{
+			AbsoluteTime now;
+			clock_get_uptime(&now);
+
+			IOLockUnlock(gPMHaltLock);
+			for (unsigned int i = 0 ; i < numWorkers; i++)
+			{
+				if (workers[i])
+					PMHaltWorker::checkTimeout(workers[i], &now);
+			}
+			IOLockLock(gPMHaltLock);
+		}
+	}
+	IOLockUnlock(gPMHaltLock);
+
+	// Release all workers
+
+	for (unsigned int i = 0; i < numWorkers; i++)
+	{
+		if (workers[i])
+			workers[i]->release();
+		// worker also retained by it's own thread
+	}
+
+done:
+	DEBUG_LOG("%s done\n", __FUNCTION__);
+	return;
+}
+
+#if DEBUG_TEST
+// debug - exercise notifySystemShutdown()
+bool IOPMrootDomain::serializeProperties( OSSerialize * s ) const
+{
+	IOPMrootDomain * root = (IOPMrootDomain *) this;
+	notifySystemShutdown( root, kIOMessageSystemWillPowerOff );
+    return( super::serializeProperties(s) );
+}
+#endif
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
@@ -2215,8 +3202,10 @@ bool IORootParent::start ( IOService * nub )
     mostRecentChange = ON_STATE;
     super::start(nub);
     PMinit();
+	youAreRoot();
     registerPowerDriver(this,patriarchPowerStates,number_of_power_states);
-    powerOverrideOnPriv();
+	wakeSystem();
+    powerOverrideOnPriv();	
     return true;
 }
 
diff --git a/iokit/Kernel/IOPMrootDomainInternal.h b/iokit/Kernel/IOPMrootDomainInternal.h
new file mode 100644
index 000000000..240b254bd
--- /dev/null
+++ b/iokit/Kernel/IOPMrootDomainInternal.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+/* DO NOT MODIFY THIS FILE
+ *
+ * This file is a partial mirror of
+ *    AppleIntelCPUPowerManagement/pmioctl.h
+ * Changes may only be made to the original, pmioctl.h.
+ * This file must be updated only when pmioctl.h changes.
+ */
+ 
+/*
+ * Defines the IOCTLs for dealing with the CPU power management KEXT.
+ */
+#ifndef _IOPMROOTDOMAINIOCTLS_H_
+#define _IOPMROOTDOMAINIOCTLS_H_
+
+#include <sys/ioccom.h>
+#include <i386/pmCPU.h>
+
+#define PMIOCGETVARIDINFO	_IOW('P', 25, uint64_t)
+#define PMIOCGETVARNAMEINFO	_IOW('P', 26, uint64_t)
+#define PMIOCSETVARINFO		_IOW('P', 27, uint64_t)
+
+/*
+ * Data structures used by IOCTLs
+ */
+#pragma pack(4)
+
+#define PMVARNAMELEN	16
+
+typedef enum
+{
+    vUnknown		= 0,		/* Unknown type */
+    vBool		= 1,		/* Boolean value */
+    vInt		= 2,		/* signed integer value */
+    vUInt		= 3,		/* Unsigned integer value */
+    vChars		= 4,		/* 8 characters */
+    vInvalid		= -1		/* invalid type */
+} pmioctlVarType_t;
+
+typedef struct pmioctlVaribleInfo
+{
+    uint32_t		varID;		/* ID of variable */
+    uint8_t		varName[PMVARNAMELEN+1];
+    pmioctlVarType_t	varType;	/* type of variable's value */
+    uint64_t		varInitValue;	/* variable's initial value */
+    uint64_t		varCurValue;	/* variable's current value */
+} pmioctlVariableInfo_t;
+
+#pragma pack()
+
+#endif /* _IOPMROOTDOMAINIOCTLS_H_ */
diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp
index 084975498..03d349e4f 100644
--- a/iokit/Kernel/IOPlatformExpert.cpp
+++ b/iokit/Kernel/IOPlatformExpert.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * HISTORY
@@ -35,6 +41,7 @@
 #include <IOKit/pwr_mgt/RootDomain.h>
 #include <IOKit/IOKitKeys.h>
 #include <IOKit/IOTimeStamp.h>
+#include <IOKit/IOUserClient.h>
 
 #include <IOKit/system.h>
 
@@ -43,13 +50,14 @@
 extern "C" {
 #include <machine/machine_routines.h>
 #include <pexpert/pexpert.h>
+#include <uuid/uuid.h>
 }
 
 /* Delay period for UPS halt */
 #define kUPSDelayHaltCPU_msec   (1000*60*5)
 
 void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg);
-static void getCStringForObject (OSObject * inObj, char * outStr);
+static void getCStringForObject(OSObject *inObj, char *outStr, size_t outStrLen);
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -243,6 +251,8 @@ int (*PE_halt_restart)(unsigned int type) = 0;
 
 int IOPlatformExpert::haltRestart(unsigned int type)
 {
+  if (type == kPEPanicSync) return 0;
+
   if (type == kPEHangCPU) while (1);
 
   if (type == kPEUPSDelayHaltCPU) {
@@ -254,7 +264,11 @@ int IOPlatformExpert::haltRestart(unsigned int type)
 
     type = kPERestartCPU;
   }
-  kprintf("platform halt restart\n");
+
+  // On ARM kPEPanicRestartCPU is supported in the drivers
+  if (type == kPEPanicRestartCPU)
+	  type = kPERestartCPU;
+  
   if (PE_halt_restart) return (*PE_halt_restart)(type);
   else return -1;
 }
@@ -687,7 +701,8 @@ void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg)
     if ( mkey->isEqualTo ("name") ) {
       char nameStr[64];
       nameStr[0] = 0;
-      getCStringForObject (inDictionary->getObject ("name"), nameStr );
+      getCStringForObject(inDictionary->getObject("name"), nameStr,
+		      sizeof(nameStr));
       if (strlen(nameStr) > 0)
         IOLog ("%s name is %s\n", inMsg, nameStr);
     }
@@ -700,7 +715,8 @@ void printDictionaryKeys (OSDictionary * inDictionary, char * inMsg)
   mcoll->release ();
 }
 
-static void getCStringForObject (OSObject * inObj, char * outStr)
+static void
+getCStringForObject(OSObject *inObj, char *outStr, size_t outStrLen)
 {
    char * buffer;
    unsigned int    len, i;
@@ -710,10 +726,11 @@ static void getCStringForObject (OSObject * inObj, char * outStr)
 
    char * objString = (char *) (inObj->getMetaClass())->getClassName();
 
-   if ((0 == strcmp(objString,"OSString")) || (0 == strcmp (objString, "OSSymbol")))
-     strcpy (outStr, ((OSString *)inObj)->getCStringNoCopy());
+   if ((0 == strncmp(objString, "OSString", sizeof("OSString"))) ||
+		   (0 == strncmp(objString, "OSSymbol", sizeof("OSSymbol"))))
+     strlcpy(outStr, ((OSString *)inObj)->getCStringNoCopy(), outStrLen);
 
-   else if (0 == strcmp(objString,"OSData")) {
+   else if (0 == strncmp(objString, "OSData", sizeof("OSData"))) {
      len = ((OSData *)inObj)->getLength();
      buffer = (char *)((OSData *)inObj)->getBytesNoCopy();
      if (buffer && (len > 0)) {
@@ -772,7 +789,6 @@ int PEGetPlatformEpoch(void)
 int PEHaltRestart(unsigned int type)
 {
   IOPMrootDomain    *pmRootDomain = IOService::getPMRootDomain();
-  bool              noWaitForResponses;
   AbsoluteTime      deadline;
   thread_call_t     shutdown_hang;
   unsigned int      tell_type;
@@ -800,15 +816,14 @@ int PEHaltRestart(unsigned int type)
         tell_type = type;
     }
 
-    noWaitForResponses = pmRootDomain->tellChangeDown2(tell_type); 
+    pmRootDomain->handlePlatformHaltRestart(tell_type); 
     /* This notification should have few clients who all do 
        their work synchronously.
              
        In this "shutdown notification" context we don't give
        drivers the option of working asynchronously and responding 
        later. PM internals make it very hard to wait for asynchronous
-       replies. In fact, it's a bad idea to even be calling
-       tellChangeDown2 from here at all.
+       replies.
      */
    }
 
@@ -842,6 +857,32 @@ void PESetGMTTimeOfDay(long secs)
 
 void IOPlatformExpert::registerNVRAMController(IONVRAMController * caller)
 {
+    OSData *          data;
+    IORegistryEntry * nvram;
+    OSString *        string;
+
+    nvram = IORegistryEntry::fromPath( "/options", gIODTPlane );
+    if ( nvram )
+    {
+        data = OSDynamicCast( OSData, nvram->getProperty( "platform-uuid" ) );
+        if ( data && data->getLength( ) == sizeof( uuid_t ) )
+        {
+            char uuid[ 36 + 1 ];
+            uuid_unparse( ( UInt8 * ) data->getBytesNoCopy( ), uuid );
+
+            string = OSString::withCString( uuid );
+            if ( string )
+            {
+                getProvider( )->setProperty( kIOPlatformUUIDKey, string );
+                publishResource( kIOPlatformUUIDKey, string );
+
+                string->release( );
+            }
+        }
+
+        nvram->release( );
+    }
+
     publishResource("IONVRAM");
 }
 
@@ -1047,7 +1088,7 @@ bool IODTPlatformExpert::getMachineName( char * name, int maxLength )
     ok = (0 != prop);
 
     if( ok )
-	strncpy( name, (const char *) prop->getBytesNoCopy(), maxLength );
+	strlcpy( name, (const char *) prop->getBytesNoCopy(), maxLength );
 
     return( ok );
 }
@@ -1212,7 +1253,7 @@ IOPlatformExpertDevice::initWithArgs(
     argsData[ 2 ] = p3;
     argsData[ 3 ] = p4;
 
-    setProperty("IOPlatformArgs", (void *)argsData, sizeof( argsData));
+    setProperty("IOPlatformArgs", (void *)argsData, sizeof(argsData));
 
     return( true);
 }
@@ -1222,16 +1263,57 @@ IOWorkLoop *IOPlatformExpertDevice::getWorkLoop() const
     return workLoop;
 }
 
-void IOPlatformExpertDevice::free()
+IOReturn IOPlatformExpertDevice::setProperties( OSObject * properties )
 {
-    if (workLoop)
-        workLoop->release();
+    OSDictionary * dictionary;
+    OSObject *     object;
+    IOReturn       status;
+
+    status = super::setProperties( properties );
+    if ( status != kIOReturnUnsupported ) return status;
+
+    status = IOUserClient::clientHasPrivilege( current_task( ), kIOClientPrivilegeAdministrator );
+    if ( status != kIOReturnSuccess ) return status;
+
+    dictionary = OSDynamicCast( OSDictionary, properties );
+    if ( dictionary == 0 ) return kIOReturnBadArgument;
+
+    object = dictionary->getObject( kIOPlatformUUIDKey );
+    if ( object )
+    {
+        IORegistryEntry * nvram;
+        OSString *        string;
+        uuid_t            uuid;
+
+        string = ( OSString * ) getProperty( kIOPlatformUUIDKey );
+        if ( string ) return kIOReturnNotPermitted;
+
+        string = OSDynamicCast( OSString, object );
+        if ( string == 0 ) return kIOReturnBadArgument;
+
+        status = uuid_parse( string->getCStringNoCopy( ), uuid );
+        if ( status != 0 ) return kIOReturnBadArgument;
+
+        nvram = IORegistryEntry::fromPath( "/options", gIODTPlane );
+        if ( nvram )
+        {
+            nvram->setProperty( "platform-uuid", uuid, sizeof( uuid_t ) );
+            nvram->release( );
+        }
+
+        setProperty( kIOPlatformUUIDKey, string );
+        publishResource( kIOPlatformUUIDKey, string );
+
+        return kIOReturnSuccess;
+    }
+
+    return kIOReturnUnsupported;
 }
 
-bool IOPlatformExpertDevice::attachToChild( IORegistryEntry * child,
-                                        const IORegistryPlane * plane )
+void IOPlatformExpertDevice::free()
 {
-    return IOService::attachToChild( child, plane );
+    if (workLoop)
+        workLoop->release();
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
diff --git a/iokit/Kernel/IOPowerConnection.cpp b/iokit/Kernel/IOPowerConnection.cpp
index cc3242765..db5a7696b 100644
--- a/iokit/Kernel/IOPowerConnection.cpp
+++ b/iokit/Kernel/IOPowerConnection.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
@@ -170,3 +176,23 @@ bool IOPowerConnection::getAwaitingAck ( void )
 {
     return awaitingAck;
 }
+
+
+// **********************************************************************************
+// setReadyFlag
+//
+// **********************************************************************************
+void IOPowerConnection::setReadyFlag( bool flag )
+{
+	readyFlag = flag;
+}
+
+
+// **********************************************************************************
+// getReadyFlag
+//
+// **********************************************************************************
+bool IOPowerConnection::getReadyFlag( void ) const
+{
+	return readyFlag;
+}
diff --git a/iokit/Kernel/IORangeAllocator.cpp b/iokit/Kernel/IORangeAllocator.cpp
index a52943ee3..d8a56aec6 100644
--- a/iokit/Kernel/IORangeAllocator.cpp
+++ b/iokit/Kernel/IORangeAllocator.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.
diff --git a/iokit/Kernel/IORegistryEntry.cpp b/iokit/Kernel/IORegistryEntry.cpp
index 4be7e3aa5..f039c28ad 100644
--- a/iokit/Kernel/IORegistryEntry.cpp
+++ b/iokit/Kernel/IORegistryEntry.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
@@ -48,7 +54,11 @@ OSDefineMetaClassAndStructors(IORegistryEntry, OSObject)
 #define kIORegPlaneChildSuffix		"ChildLinks"
 #define kIORegPlaneNameSuffix		"Name"
 #define kIORegPlaneLocationSuffix	"Location"
+
 #define kIORegPlaneParentSuffixLen	(sizeof(kIORegPlaneParentSuffix) - 1)
+#define kIORegPlaneChildSuffixLen	(sizeof(kIORegPlaneChildSuffix) - 1)
+#define kIORegPlaneNameSuffixLen	(sizeof(kIORegPlaneNameSuffix) - 1)
+#define kIORegPlaneLocationSuffixLen	(sizeof(kIORegPlaneLocationSuffix) - 1)
 
 static IORegistryEntry * gRegistryRoot;
 static OSDictionary * 	 gIORegistryPlanes;
@@ -173,22 +183,21 @@ const IORegistryPlane * IORegistryEntry::makePlane( const char * name )
     char		key[ kIOMaxPlaneName + 16 ];
     char *		end;
 
-    strncpy( key, name, kIOMaxPlaneName );
-    key[ kIOMaxPlaneName ] = 0;
-    end = key + strlen( name );
+    strlcpy( key, name, kIOMaxPlaneName + 1 );
+    end = key + strlen( key );
 
     nameKey = OSSymbol::withCString( key);
 
-    strcpy( end, kIORegPlaneParentSuffix );
+    strlcpy( end, kIORegPlaneParentSuffix, kIORegPlaneParentSuffixLen + 1 );
     parentKey = OSSymbol::withCString( key);
 
-    strcpy( end, kIORegPlaneChildSuffix );
+    strlcpy( end, kIORegPlaneChildSuffix, kIORegPlaneChildSuffixLen + 1 );
     childKey = OSSymbol::withCString( key);
 
-    strcpy( end, kIORegPlaneNameSuffix );
+    strlcpy( end, kIORegPlaneNameSuffix, kIORegPlaneNameSuffixLen + 1 );
     pathNameKey = OSSymbol::withCString( key);
 
-    strcpy( end, kIORegPlaneLocationSuffix );
+    strlcpy( end, kIORegPlaneLocationSuffix, kIORegPlaneLocationSuffixLen + 1 );
     pathLocationKey = OSSymbol::withCString( key);
 
     plane = new IORegistryPlane;
@@ -334,15 +343,15 @@ void IORegistryEntry::free( void )
 
 #if DEBUG_FREE
 #define msg ": attached at free()"
-    char buf[ strlen(msg) + 40 ];
+    int len = strlen(msg) + 40;
+    char buf[len];
 
     if( registryTable() && gIOServicePlane) {
         if( getParentSetReference( gIOServicePlane )
             || getChildSetReference( gIOServicePlane )) {
 
-            strncpy( buf, getName(), 32);
-            buf[32] = 0;
-            strcat( buf, msg );
+            strlcpy( buf, getName(), 32);
+            strlcat( buf, msg, len );
             IOPanic( buf );
         }
     }
@@ -839,7 +848,7 @@ bool IORegistryEntry::getPath(	char * path, int * length,
     IORegistryEntry *	parent;
     const OSSymbol *	alias;
     int			index;
-    int			len, maxLength, compLen;
+    int			len, maxLength, compLen, aliasLen;
     char *		nextComp;
     bool		ok;
 
@@ -853,16 +862,17 @@ bool IORegistryEntry::getPath(	char * path, int * length,
     len = plane->nameKey->getLength();
     if( len >= maxLength)
 	return( false);
-    strcpy( nextComp, plane->nameKey->getCStringNoCopy());
+    strlcpy( nextComp, plane->nameKey->getCStringNoCopy(), len + 1);
     nextComp[ len++ ] = ':';
     nextComp += len;
 
     if( (alias = hasAlias( plane ))) {
-	len += alias->getLength();
+	aliasLen = alias->getLength();
+	len += aliasLen;
 	ok = (maxLength > len);
 	*length = len;
 	if( ok)
-	    strcpy( nextComp, alias->getCStringNoCopy());
+	    strlcpy( nextComp, alias->getCStringNoCopy(), aliasLen + 1);
 	return( ok );
     }
 
@@ -905,9 +915,9 @@ bool IORegistryEntry::getPath(	char * path, int * length,
             nextComp = path + len;
 
             compLen = alias->getLength();
-            ok = (maxLength > len + compLen);
+            ok = (maxLength > (len + compLen));
             if( ok)
-                strcpy( nextComp, alias->getCStringNoCopy());
+                strlcpy( nextComp, alias->getCStringNoCopy(), compLen + 1);
         } else {
             compLen = maxLength - len;
             ok = entry->getPathComponent( nextComp + 1, &compLen, plane );
@@ -949,14 +959,14 @@ bool IORegistryEntry::getPathComponent( char * path, int * length,
     else
 	locLen = 0;
 
-    ok = ((len + locLen) < maxLength);
+    ok = ((len + locLen + 1) < maxLength);
     if( ok) {
-        strcpy( path, compName );
+        strlcpy( path, compName, len + 1 );
 	if( loc) {
             path += len;
             len += locLen;
             *path++ = '@';
-            strcpy( path, loc );
+            strlcpy( path, loc, locLen );
 	}
         *length = len;
     }
@@ -1117,8 +1127,7 @@ const char * IORegistryEntry::dealiasPath(
         {}
     end--;
     if( (end - path) < kIOMaxPlaneName) {
-        strncpy( temp, path, end - path );
-        temp[ end - path ] = 0;
+        strlcpy( temp, path, end - path + 1 );
 
         RLOCK;
         entry = IORegistryEntry::fromPath( "/aliases", plane );
@@ -1161,8 +1170,7 @@ IORegistryEntry * IORegistryEntry::fromPath(
 	// get plane name
         end = strchr( path, ':' );
 	if( end && ((end - path) < kIOMaxPlaneName)) {
-	    strncpy( temp, path, end - path );
-	    temp[ end - path ] = 0;
+	    strlcpy( temp, path, end - path + 1 );
             plane = getPlane( temp );
 	    path = end + 1;
 	}
@@ -1213,10 +1221,10 @@ IORegistryEntry * IORegistryEntry::fromPath(
 
 	if( opath && length) {
             // copy out residual path
-	    len2 = len + strlen( path );
-	    if( len2 < *length)
-                strcpy( opath + len, path );
-	    *length = len2;
+	    len2 = strlen( path );
+	    if( (len + len2) < *length)
+                strlcpy( opath + len, path, len2 + 1 );
+	    *length = (len + len2);
 
 	} else if( path[0])
 	    // no residual path => must be no tail for success
@@ -1550,12 +1558,16 @@ bool IORegistryEntry::inPlane( const IORegistryPlane * plane ) const
 	    const OSSymbol *key;
 
             while( (key = (OSSymbol *) iter->getNextObject()) ) {
-		const char *keysuffix;
+		size_t keysuffix;
 
 		// Get a pointer to this keys suffix
-		keysuffix = key->getCStringNoCopy()
-			  + key->getLength() - kIORegPlaneParentSuffixLen;
-		if( !strcmp(keysuffix, kIORegPlaneParentSuffix) ) {
+		keysuffix = key->getLength();
+		if (keysuffix <= kIORegPlaneParentSuffixLen)
+		    continue;
+		keysuffix -= kIORegPlaneParentSuffixLen;
+		if( !strncmp(key->getCStringNoCopy() + keysuffix, 
+				kIORegPlaneParentSuffix, 
+				kIORegPlaneParentSuffixLen + 1) ) {
 		    ret = true;
 		    break;
 		}
@@ -1851,7 +1863,7 @@ void IORegistryIterator::enterEntry( const IORegistryPlane * enterPlane )
     IORegCursor *	prev;
 
     prev = where;
-    where = (IORegCursor *) IOMalloc( sizeof( IORegCursor));
+    where = (IORegCursor *) IOMalloc( sizeof(IORegCursor));
     assert( where);
 
     if( where) {
@@ -1881,7 +1893,7 @@ bool IORegistryIterator::exitEntry( void )
     if( where != &start) {
 	gone = where;
         where = gone->next;
-        IOFree( gone, sizeof( IORegCursor));
+        IOFree( gone, sizeof(IORegCursor));
 	return( true);
 
     } else
diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp
index f5f4f57e0..f92941428 100644
--- a/iokit/Kernel/IOService.cpp
+++ b/iokit/Kernel/IOService.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include <IOKit/system.h>
@@ -140,6 +146,13 @@ static OSArray *		gIOFinalizeList;
 static SInt32			gIOConsoleUsersSeed;
 static OSData *			gIOConsoleUsersSeedValue;
 
+extern const OSSymbol *		gIODTPHandleKey;
+
+const OSSymbol *		gIOPlatformSleepActionKey;
+const OSSymbol *		gIOPlatformWakeActionKey;
+const OSSymbol *		gIOPlatformQuiesceActionKey;
+const OSSymbol *		gIOPlatformActiveActionKey;
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define LOCKREADNOTIFY()	\
@@ -270,6 +283,11 @@ void IOService::initialize( void )
     gIOConsoleSessionSecureInputPIDKey = OSSymbol::withCStringNoCopy( kIOConsoleSessionSecureInputPIDKey);
     gIOConsoleUsersSeedValue	= OSData::withBytesNoCopy(&gIOConsoleUsersSeed, sizeof(gIOConsoleUsersSeed));
 
+    gIOPlatformSleepActionKey	= OSSymbol::withCStringNoCopy(kIOPlatformSleepActionKey);
+    gIOPlatformWakeActionKey	= OSSymbol::withCStringNoCopy(kIOPlatformWakeActionKey);
+    gIOPlatformQuiesceActionKey	= OSSymbol::withCStringNoCopy(kIOPlatformQuiesceActionKey);
+    gIOPlatformActiveActionKey	= OSSymbol::withCStringNoCopy(kIOPlatformActiveActionKey);
+
     gNotificationLock	 	= IORecursiveLockAlloc();
 
     assert( gIOServicePlane && gIODeviceMemoryKey
@@ -1342,7 +1360,8 @@ void IOService::applyToInterested( const OSSymbol * typeOfInterest,
                                    OSObjectApplierFunction applier,
                                    void * context )
 {
-    applyToClients( (IOServiceApplierFunction) applier, context );
+    if (gIOGeneralInterest == typeOfInterest)
+	applyToClients( (IOServiceApplierFunction) applier, context );
     applyToInterestNotifiers(this, typeOfInterest, applier, context);
 }
 
@@ -2352,6 +2371,10 @@ void IOService::probeCandidates( OSOrderedSet * matches )
     OSObject 		*	nextMatch = 0;
     bool			started;
     bool			needReloc = false;
+#if CONFIG_MACF_KEXT
+    OSBoolean		*	isSandbox = 0;
+    bool			useSandbox = false;
+#endif
 #if IOMATCHDEBUG
     SInt64			debugFlags;
 #endif
@@ -2460,6 +2483,8 @@ void IOService::probeCandidates( OSOrderedSet * matches )
                 if( !symbol)
                     continue;
     
+                //IOLog("%s alloc (symbol %p props %p)\n", symbol->getCStringNoCopy(), symbol, props);
+
                 // alloc the driver instance
                 inst = (IOService *) OSMetaClass::allocClassWithName( symbol);
     
@@ -2486,7 +2511,10 @@ void IOService::probeCandidates( OSOrderedSet * matches )
                 if( 0 == category)
                     category = gIODefaultMatchCategoryKey;
                 inst->setProperty( gIOMatchCategoryKey, (OSObject *) category );
-    
+#if CONFIG_MACF_KEXT
+		isSandbox = OSDynamicCast(OSBoolean,
+                            props->getObject("IOKitForceMatch"));
+#endif
                 // attach driver instance
                 if( !(inst->attach( this )))
                         continue;
@@ -2503,6 +2531,21 @@ void IOService::probeCandidates( OSOrderedSet * matches )
     
                 newInst = inst->probe( this, &score );
                 inst->detach( this );
+#if CONFIG_MACF_KEXT
+		/*
+		 * If this is the Sandbox driver and it matched, this is a
+		 * disallowed device; toss any drivers that were already
+		 * matched.
+		 */
+		if (isSandbox && isSandbox->isTrue() && newInst != 0) {
+		    if (startDict != 0) {
+			startDict->flushCollection();
+			startDict->release();
+			startDict = 0;
+		    }
+		    useSandbox = true;
+		}
+#endif
                 if( 0 == newInst) {
 #if IOMATCHDEBUG
                     if( debugFlags & kIOLogProbe)
@@ -2541,6 +2584,13 @@ void IOService::probeCandidates( OSOrderedSet * matches )
             props->release();
             if( inst)
                 inst->release();
+#if CONFIG_MACF_KEXT
+	    /*
+	     * If we're forcing the sandbox, drop out of the loop.
+	     */
+	    if (isSandbox && isSandbox->isTrue() && useSandbox)
+		    break;
+#endif
         }
         familyMatches->release();
         familyMatches = 0;
@@ -2923,8 +2973,10 @@ UInt32 IOService::_adjustBusy( SInt32 delta )
 	    applyToInterestNotifiers( next, gIOBusyInterest, 
 				     &messageClientsApplier, &context );
 
+#if !NO_KEXTD
             if( nowQuiet && (next == gIOServiceRoot))
                 OSMetaClass::considerUnloads();
+#endif
         }
 
         delta = nowQuiet ? -1 : +1;
@@ -3009,9 +3061,18 @@ bool IOService::serializeProperties( OSSerialize * s ) const
 
 void _IOConfigThread::main( _IOConfigThread * self )
 {
-    _IOServiceJob *	job;
-    IOService 	*	nub;
-    bool		alive = true;
+    _IOServiceJob * job;
+    IOService 	*   nub;
+    bool	    alive = true;
+    kern_return_t   kr;
+    thread_precedence_policy_data_t precedence = { -1 };
+
+    kr = thread_policy_set(current_thread(), 
+			    THREAD_PRECEDENCE_POLICY, 
+			    (thread_policy_t) &precedence, 
+			    THREAD_PRECEDENCE_POLICY_COUNT);
+    if (KERN_SUCCESS != kr)
+	IOLog("thread_policy_set(%d)\n", kr);
 
     do {
 
@@ -3339,7 +3400,7 @@ IONotifier * IOService::addNotification(
 			void * target, void * ref,
 			SInt32 priority )
 {
-    OSIterator *		existing;
+    OSIterator *		existing = NULL;
     _IOServiceNotifier *	notify;
     IOService *			next;
 
@@ -3567,6 +3628,26 @@ OSDictionary * IOService::resourceMatching( const char * name,
     return( table );
 }
 
+OSDictionary * IOService::propertyMatching( const OSSymbol * key, const OSObject * value,
+			OSDictionary * table )
+{
+    OSDictionary * properties;
+
+    properties = OSDictionary::withCapacity( 2 );
+    if( !properties)
+	return( 0 );
+    properties->setObject( key, value );
+
+    if( !table)
+	table = OSDictionary::withCapacity( 2 );
+    if( table)
+        table->setObject( gIOPropertyMatchKey, properties );
+
+    properties->release();
+
+    return( table );
+}
+
 /*
  * _IOServiceNotifier
  */
@@ -3994,6 +4075,12 @@ bool IOService::passiveMatch( OSDictionary * table, bool changesOK )
 
             if( !(match = where->compareProperty( table, kIOBSDNameKey )))
                 break;
+            if( !(match = where->compareProperty( table, kIOBSDMajorKey )))
+                break;
+            if( !(match = where->compareProperty( table, kIOBSDMinorKey )))
+                break;
+            if( !(match = where->compareProperty( table, kIOBSDUnitKey )))
+                break;
 
             matchParent = false;
 
@@ -4572,8 +4659,8 @@ OSMetaClassDefineReservedUsed(IOService, 0);
 OSMetaClassDefineReservedUsed(IOService, 1);
 OSMetaClassDefineReservedUsed(IOService, 2);
 OSMetaClassDefineReservedUsed(IOService, 3);
+OSMetaClassDefineReservedUsed(IOService, 4);
 
-OSMetaClassDefineReservedUnused(IOService, 4);
 OSMetaClassDefineReservedUnused(IOService, 5);
 OSMetaClassDefineReservedUnused(IOService, 6);
 OSMetaClassDefineReservedUnused(IOService, 7);
diff --git a/iokit/Kernel/IOServicePM.cpp b/iokit/Kernel/IOServicePM.cpp
index 66e271062..776de768c 100644
--- a/iokit/Kernel/IOServicePM.cpp
+++ b/iokit/Kernel/IOServicePM.cpp
@@ -1,28 +1,32 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <IOKit/assert.h>
-
-#include <IOKit/IOCommandGate.h>
 #include <IOKit/IOKitDebug.h>
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMessage.h>
@@ -30,367 +34,591 @@
 #include <IOKit/IOService.h>
 #include <IOKit/IOTimerEventSource.h>
 #include <IOKit/IOWorkLoop.h>
+#include <IOKit/IOCommand.h>
 
-#include <IOKit/pwr_mgt/IOPMchangeNoteList.h>
+#include <IOKit/pwr_mgt/IOPMlog.h>
 #include <IOKit/pwr_mgt/IOPMinformee.h>
 #include <IOKit/pwr_mgt/IOPMinformeeList.h>
-#include <IOKit/pwr_mgt/IOPMlog.h>
 #include <IOKit/pwr_mgt/IOPowerConnection.h>
 #include <IOKit/pwr_mgt/RootDomain.h>
 
+#include <sys/proc.h>
 
 // Required for notification instrumentation
 #include "IOServicePrivate.h"
+#include "IOServicePMPrivate.h"
+#include "IOKitKernelInternal.h"
 
-#define super IORegistryEntry
+static void settle_timer_expired(thread_call_param_t, thread_call_param_t);
+static void PM_idle_timer_expired(OSObject *, IOTimerEventSource *);
+void        tellAppWithResponse(OSObject * object, void * context) { /*empty*/ }
+void        tellClientWithResponse(OSObject * object, void * context) { /*empty*/ }
+void        tellClient(OSObject * object, void * context);
+IOReturn    serializedAllowPowerChange(OSObject *, void *, void *, void *, void *);
 
-#define OUR_PMLog(t, a, b) \
-    do { pm_vars->thePlatform->PMLog(pm_vars->ourName, t, a, b); } while(0)
+static uint64_t computeTimeDeltaNS( const AbsoluteTime * start )
+{
+    AbsoluteTime    now;
+    uint64_t        nsec;
 
-static void ack_timer_expired(thread_call_param_t);
-static void settle_timer_expired(thread_call_param_t);
-static void PM_idle_timer_expired(OSObject *, IOTimerEventSource *);
-void tellAppWithResponse ( OSObject * object, void * context);
-void tellClientWithResponse ( OSObject * object, void * context);
-void tellClient ( OSObject * object, void * context);
-IOReturn serializedAllowPowerChange ( OSObject *, void *, void *, void *, void *);
-IOReturn serializedCancelPowerChange ( OSObject *, void *, void *, void *, void *);
+    clock_get_uptime(&now);
+    SUB_ABSOLUTETIME(&now, start);
+    absolutetime_to_nanoseconds(now, &nsec);
+    return nsec;
+}
 
-extern const IORegistryPlane * gIOPowerPlane;
+OSDefineMetaClassAndStructors(IOPMprot, OSObject)
 
+// log setPowerStates longer than (ns):
+#define LOG_SETPOWER_TIMES      (50ULL * 1000ULL * 1000ULL)
+// log app responses longer than (ns):
+#define LOG_APP_RESPONSE_TIMES  (100ULL * 1000ULL * 1000ULL)
 
-// and there's 1000 nanoseconds in a microsecond:
-#define ns_per_us 1000
+//*********************************************************************************
+// Globals
+//*********************************************************************************
 
+static bool                 gIOPMInitialized   = false;
+static IOItemCount          gIOPMBusyCount     = 0;
+static IOWorkLoop *         gIOPMWorkLoop      = 0;
+static IOPMRequestQueue *   gIOPMRequestQueue  = 0;
+static IOPMRequestQueue *   gIOPMReplyQueue    = 0;
+static IOPMRequestQueue *   gIOPMFreeQueue     = 0;
 
-// The current change note is processed by a state machine.
-// Inputs are acks from interested parties, ack from the controlling driver,
-// ack timeouts, settle timeout, and powerStateDidChange from the parent.
-// These are the states:
-enum {
-    kIOPM_OurChangeTellClientsPowerDown = 1,
-    kIOPM_OurChangeTellPriorityClientsPowerDown,
-    kIOPM_OurChangeNotifyInterestedDriversWillChange,
-    kIOPM_OurChangeSetPowerState,
-    kIOPM_OurChangeWaitForPowerSettle,
-    kIOPM_OurChangeNotifyInterestedDriversDidChange,
-    kIOPM_OurChangeFinish,
-    kIOPM_ParentDownTellPriorityClientsPowerDown_Immediate,
-    kIOPM_ParentDownNotifyInterestedDriversWillChange_Delayed,
-    kIOPM_ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate,
-    kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed,
-    kIOPM_ParentDownSetPowerState_Delayed,
-    kIOPM_ParentDownWaitForPowerSettle_Delayed,
-    kIOPM_ParentDownAcknowledgeChange_Delayed,
-    kIOPM_ParentUpSetPowerState_Delayed,
-    kIOPM_ParentUpSetPowerState_Immediate,
-    kIOPM_ParentUpWaitForSettleTime_Delayed,
-    kIOPM_ParentUpNotifyInterestedDriversDidChange_Delayed,
-    kIOPM_ParentUpAcknowledgePowerChange_Delayed,
-    kIOPM_Finished
-};
+//*********************************************************************************
+// Macros
+//*********************************************************************************
 
-// values of outofbandparameter
-enum {
-	kNotifyApps,
-	kNotifyPriority
-};
+#define PM_ERROR(x...)          do { kprintf(x); IOLog(x); } while (false)
+#define PM_DEBUG(x...)          do { kprintf(x); } while (false)
 
+#define PM_TRACE(x...)          do {  \
+	if (kIOLogDebugPower & gIOKitDebug) kprintf(x); } while (false)
 
-// used for applyToInterested
-struct context {
-    OSArray *	responseFlags;
-    UInt16	serialNumber;
-    UInt16 	counter;
-    UInt32	maxTimeRequested;
-    int		msgType;
-    IOService *	us;
-    IOLock *	flags_lock;
-    unsigned long stateNumber;
-    IOPMPowerFlags stateFlags;
-};
+#define PM_CONNECT(x...)
 
-// five minutes in microseconds
-#define FIVE_MINUTES 5*60*1000000
-#define k30seconds 30*1000000
+#define PM_ASSERT_IN_GATE(x)          \
+do {                                  \
+    assert(gIOPMWorkLoop->inGate());  \
+} while(false)
 
-/*
- There are two different kinds of power state changes.  One is initiated by a subclassed device object which has either
- decided to change power state, or its controlling driver has suggested it, or some other driver wants to use the
- idle device and has asked it to become usable.  The second kind of power state change is initiated by the power
- domain parent.  The two are handled slightly differently.
-
-There is a queue of so-called change notifications, or change notes for short.  Usually the queue is empty, and when
- it isn't, usually there is one change note in it, but since it's possible to have more than one power state change pending
- at one time, a queue is implemented.  Example:  the subclass device decides it's idle and initiates a change to a lower
- power state.  This causes interested parties to be notified, but they don't all acknowledge right away.  This causes the
- change note to sit in the queue until all the acks are received.  During this time, the device decides it isn't idle anymore and
- wants to raise power back up again.  This change can't be started, however, because the previous one isn't complete yet,
- so the second one waits in the queue.  During this time, the parent decides to lower or raise the power state of the entire
- power domain and notifies the device, and that notification goes into the queue, too, and can't be actioned until the
- others are.
+#define PM_LOCK()                     IOLockLock(fPMLock)
+#define PM_UNLOCK()                   IOLockUnlock(fPMLock)
+
+#define ns_per_us                     1000
+#define k30seconds                    (30*1000000)
+#define kMinAckTimeoutTicks           (10*1000000)
+#define kIOPMTardyAckSPSKey           "IOPMTardyAckSetPowerState"
+#define kIOPMTardyAckPSCKey           "IOPMTardyAckPowerStateChange"
+#define kPwrMgtKey                    "IOPowerManagement"
+
+#define OUR_PMLog(t, a, b) \
+    do { fPlatform->PMLog( fName, t, a, b); } while(0)
+
+#define NS_TO_MS(nsec)                ((int)((nsec) / 1000000ULL))
+
+//*********************************************************************************
+// PM machine states
+//*********************************************************************************
 
+enum {
+    kIOPM_OurChangeTellClientsPowerDown                 = 1,
+    kIOPM_OurChangeTellPriorityClientsPowerDown         = 2,
+    kIOPM_OurChangeNotifyInterestedDriversWillChange    = 3,
+    kIOPM_OurChangeSetPowerState                        = 4,
+    kIOPM_OurChangeWaitForPowerSettle                   = 5,
+    kIOPM_OurChangeNotifyInterestedDriversDidChange     = 6,
+    kIOPM_OurChangeFinish                               = 7,
+    kIOPM_ParentDownTellPriorityClientsPowerDown        = 8,
+    kIOPM_ParentDownNotifyInterestedDriversWillChange   = 9,
+    /* 10 not used */
+    kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange = 11,
+    kIOPM_ParentDownSetPowerState                       = 12,
+    kIOPM_ParentDownWaitForPowerSettle                  = 13,
+    kIOPM_ParentDownAcknowledgeChange                   = 14,
+    kIOPM_ParentUpSetPowerState                         = 15,
+    /* 16 not used */
+    kIOPM_ParentUpWaitForSettleTime                     = 17,
+    kIOPM_ParentUpNotifyInterestedDriversDidChange      = 18,
+    kIOPM_ParentUpAcknowledgePowerChange                = 19,
+    kIOPM_Finished                                      = 20,
+    kIOPM_DriverThreadCallDone                          = 21,
+    kIOPM_NotifyChildrenDone                            = 22
+};
+
+
+ /*
+ Power Management defines a few roles that drivers can play in their own, 
+ and other drivers', power management. We briefly define those here.
+ 
+ Many drivers implement their policy maker and power controller within the same 
+ IOService object, but that is not required. 
+ 
+== Policy Maker == 
+ * Virtual IOService PM methods a "policy maker" may implement
+    * maxCapabilityForDomainState()
+    * initialPowerStateForDomainState()
+    * powerStateForDomainState()
+    
+ * Virtual IOService PM methods a "policy maker" may CALL
+    * PMinit()
+ 
+== Power Controller ==
+ * Virtual IOService PM methods a "power controller" may implement
+    * setPowerState() 
+ 
+ * Virtual IOService PM methods a "power controller" may CALL
+    * joinPMtree()
+    * registerPowerDriver()
+ 
+=======================
+ There are two different kinds of power state changes.  
+    * One is initiated by a subclassed device object which has either decided
+      to change power state, or its controlling driver has suggested it, or
+      some other driver wants to use the idle device and has asked it to become
+      usable.  
+    * The second kind of power state change is initiated by the power domain 
+      parent.  
+ The two are handled through different code paths.
+ 
+ We maintain a queue of "change notifications," or change notes.
+    * Usually the queue is empty. 
+    * When it isn't, usually there is one change note in it 
+    * It's possible to have more than one power state change pending at one 
+        time, so a queue is implemented. 
+ Example:  
+    * The subclass device decides it's idle and initiates a change to a lower
+        power state. This causes interested parties to be notified, but they 
+        don't all acknowledge right away.  This causes the change note to sit 
+        in the queue until all the acks are received.  During this time, the 
+        device decides it isn't idle anymore and wants to raise power back up 
+        again.  This change can't be started, however, because the previous one 
+        isn't complete yet, so the second one waits in the queue.  During this 
+        time, the parent decides to lower or raise the power state of the entire
+        power domain and notifies the device, and that notification goes into 
+        the queue, too, and can't be actioned until the others are.
+ 
+ == SelfInitiated ==
  This is how a power change initiated by the subclass device is handled:
- First, all interested parties are notified of the change via their powerStateWillChangeTo method.  If they all don't
- acknowledge via return code, then we have to wait.  If they do, or when they finally all acknowledge via our
- acknowledgePowerChange method, then we can continue.  We call the controlling driver, instructing it to change to
- the new state.  Then we wait for power to settle.  If there is no settling-time, or after it has passed, we notify
- interested parties again, this time via their powerStateDidChangeTo methods.  When they have all acked, we're done.
- If we lowered power and don't need the power domain to be in its current power state, we suggest to the parent that
- it lower the power domain state.
-
- This is how a change to a lower power domain state initiated by the parent is handled:
- First, we figure out what power state we will be in when the new domain state is reached.  Then all interested parties are
- notified that we are moving to that new state.  When they have acknowledged, we call the controlling driver to assume
- that state and we wait for power to settle.  Then we acknowledge our preparedness to our parent.  When all its interested
- parties have acknowledged, it lowers power and then notifies its interested parties again.  When we get this call, we notify
- our interested parties that the power state has changed, and when they have all acknowledged, we're done.
-
- This is how a change to a higher power domain state initiated by the parent is handled:
- We figure out what power state we will be in when the new domain state is reached.  If it is different from our current
- state we acknowledge the parent.  When all the parent's interested parties have acknowledged, it raises power in the
-domain and waits for power to settle.  Then it  notifies everyone that the new state has been reached.  When we get this call,
- we call the controlling driver, instructing it  to assume the new state, and wait for power to settle.  Then we notify our interested
- parties.  When they all acknowledge  we are done.
-
- In either of the two cases above, it is possible that we will not be changing state even though the domain is.  Examples:
- A change to a lower domain state may not affect us because we are already in a low enough state, and
- We will not take advantage of a change to a higher domain state, because we have no need of the higher power.
- In such a case, there is nothing to do but acknowledge the parent.  So when the parent calls our powerDomainWillChange
- method, and we decide that we will not be changing state, we merely acknowledge the parent, via return code, and wait.
- When the parent subsequently calls powerStateDidChange, we acknowledge again via return code, and the change is complete.
-
- Power state changes are processed in a state machine, and since there are four varieties of power state changes, there are
- four major paths through the state machine:
-
- The fourth is nearly trivial.  In this path, the parent is changing the domain state, but we are not changing the device state.
- The change starts when the parent calls powerDomainWillChange.  All we do is acknowledge the parent.
-When the parent calls powerStateDidChange, we acknowledge the parent again, and we're done.
-
- The first is fairly simple.  It starts when a power domain child calls requestPowerDomainState and we decide to change power states
- to accomodate the child, or if our power-controlling driver calls changePowerStateTo, or if some other driver which is using our
- device calls makeUsable, or if a subclassed object calls changePowerStateToPriv.  These are all power changes initiated by us, not
- forced upon us by the parent.  We start by notifying interested parties.  If they all acknowledge via return code, we can go
- on to state "OurChangeSetPowerState".  Otherwise, we start the ack timer and wait for the stragglers to acknowlege by calling
- acknowledgePowerChange.  We move on to state "OurChangeSetPowerState" when all the stragglers have acknowledged,
- or when the ack timer expires on all those which didn't acknowledge.  In "OurChangeSetPowerState" we call the power-controlling
- driver to change the power state of the hardware.  If it returns saying it has done so, we go on to state "OurChangeWaitForPowerSettle".
- Otherwise, we have to wait for it, so we set the ack timer and wait.  When it calls acknowledgeSetPowerState, or when the
- ack timer expires, we go on.  In "OurChangeWaitForPowerSettle", we look in the power state array to see if there is any settle time required
- when changing from our current state to the new state.  If not, we go right away to "OurChangeNotifyInterestedDriversDidChange".  Otherwise, we
- set the settle timer and wait.  When it expires, we move on.  In "OurChangeNotifyInterestedDriversDidChange" state, we notify all our interested parties
- via their powerStateDidChange methods that we have finished changing power state.  If they all acknowledge via return
- code, we move on to "OurChangeFinish".  Otherwise we set the ack timer and wait.  When they have all acknowledged, or
- when the ack timer has expired for those that didn't, we move on to "OurChangeFinish", where we remove the used
- change note from the head of the queue and start the next one if one exists.
-
- Parent-initiated changes are more complex in the state machine.  First, power going up and power going down are handled
- differently, so they have different paths throught the state machine.  Second, we can acknowledge the parent's notification
- in two different ways, so each of the parent paths is really two.
-
- When the parent calls our powerDomainWillChange method, notifying us that it will lower power in the domain, we decide
- what state that will put our device in.  Then we embark on the state machine path "IOPMParentDownSetPowerState_Immediate"
- and "kIOPM_ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate", in which we notify interested parties of the upcoming change,  instruct our driver to make
- the change, check for settle time, and notify interested parties of the completed change.   If we get to the end of this path without
- stalling due to an interested party which didn't acknowledge via return code, due to the controlling driver not able to change
- state right away, or due to a non-zero settling time, then we return IOPMAckImplied to the parent, and we're done with the change.
- If we do stall in any of those states, we return IOPMWillAckLater to the parent and enter the parallel path "kIOPM_ParentDownSetPowerState_Delayed"
- "kIOPM_ParentDownWaitForPowerSettle_Delayed", and "kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed", where we continue with the same processing, except that at the end we
- acknowledge the parent explicitly via acknowledgePowerChange, and we're done with the change.
-Then when the parent calls us at powerStateDidChange we acknowledging via return code, because we have already made
- the power change.  In any case, when we are done we remove the used change note from the head of the queue and start on the next one.
-
- The case of the parent raising power in the domain is handled similarly in that there are parallel paths, one for no-stall
- that ends in implicit acknowleging the parent, and one that has stalled at least once that ends in explicit acknowledging
- the parent.  This case is different, though in that our device changes state in the second half, after the parent calls
- powerStateDidChange rather than before, as in the power-lowering case.
-
- When the parent calls our powerDomainWillChange method, notifying us that it will raise power in the domain, we acknowledge
- via return code, because there's really nothing we can do until the power is actually raised in the domain.
- When the parent calls us at powerStateDidChange, we start by notifying our interested parties.  If they all acknowledge via return code,
- we go on to" kIOPM_ParentUpSetPowerState_Immediate" to instruct the driver to raise its power level. After that, we check for any
- necessary settling time in "IOPMParentUpWaitForSettleTime_Immediate", and we notify all interested parties that power has changed
- in "IOPMParentUpNotifyInterestedDriversDidChange_Immediate".  If none of these operations stall, we acknowledge the parent via return code, release
- the change note, and start the next, if there is one.  If one of them does stall, we enter the parallel path  "kIOPM_ParentUpSetPowerState_Delayed",
- "kIOPM_ParentUpWaitForSettleTime_Delayed", "kIOPM_ParentUpNotifyInterestedDriversDidChange_Delayed", and "kIOPM_ParentUpAcknowledgePowerChange_Delayed", which ends with
- our explicit acknowledgement to the parent.
-
-*/
-
-
-const char priv_key[ ] = "Power Management private data";
-const char prot_key[ ] = "Power Management protected data";
+    -> First, all interested parties are notified of the change via their 
+       powerStateWillChangeTo method.  If they all don't acknowledge via return
+       code, then we have to wait.  If they do, or when they finally all
+       acknowledge via our acknowledgePowerChange method, then we can continue.  
+    -> We call the controlling driver, instructing it to change to the new state
+    -> Then we wait for power to settle. If there is no settling-time, or after 
+       it has passed, 
+    -> we notify interested parties again, this time via their 
+       powerStateDidChangeTo methods.  
+    -> When they have all acked, we're done.
+ If we lowered power and don't need the power domain to be in its current power 
+ state, we suggest to the parent that it lower the power domain state.
+ 
+ == PowerDomainDownInitiated ==
+How a change to a lower power domain state initiated by the parent is handled:
+    -> First, we figure out what power state we will be in when the new domain 
+        state is reached.  
+    -> Then all interested parties are notified that we are moving to that new 
+        state.  
+    -> When they have acknowledged, we call the controlling driver to assume
+        that state and we wait for power to settle.  
+    -> Then we acknowledge our preparedness to our parent.  When all its 
+        interested parties have acknowledged, 
+    -> it lowers power and then notifies its interested parties again.  
+    -> When we get this call, we notify our interested parties that the power 
+        state has changed, and when they have all acknowledged, we're done.
+ 
+ == PowerDomainUpInitiated ==
+How a change to a higher power domain state initiated by the parent is handled:
+    -> We figure out what power state we will be in when the new domain state is 
+        reached.  
+    -> If it is different from our current state we acknowledge the parent.  
+    -> When all the parent's interested parties have acknowledged, it raises 
+        power in the domain and waits for power to settle.  
+    -> Then it  notifies everyone that the new state has been reached.  
+    -> When we get this call, we call the controlling driver, instructing it to 
+        assume the new state, and wait for power to settle.
+    -> Then we notify our interested parties. When they all acknowledge we are 
+        done.
+ 
+ In either of the two power domain state cases above, it is possible that we 
+ will not be changing state even though the domain is. 
+ Examples:
+    * A change to a lower domain state may not affect us because we are already 
+        in a low enough state, 
+    * We will not take advantage of a change to a higher domain state, because 
+        we have no need of the higher power. In such cases, there is nothing to 
+        do but acknowledge the parent.  So when the parent calls our 
+        powerDomainWillChange method, and we decide that we will not be changing 
+        state, we merely acknowledge the parent, via return code, and wait.
+ When the parent subsequently calls powerStateDidChange, we acknowledge again 
+ via return code, and the change is complete.
+ 
+ == 4 Paths Through State Machine ==
+ Power state changes are processed in a state machine, and since there are four 
+ varieties of power state changes, there are four major paths through the state 
+ machine.
+ 
+ == 5. No Need To change ==
+ The fourth is nearly trivial.  In this path, the parent is changing the domain 
+ state, but we are not changing the device state. The change starts when the 
+ parent calls powerDomainWillChange.  All we do is acknowledge the parent. When 
+ the parent calls powerStateDidChange, we acknowledge the parent again, and 
+ we're done.
+ 
+ == 1. OurChange Down ==    XXX gvdl
+ The first is fairly simple.  It starts: 
+    * when a power domain child calls requestPowerDomainState and we decide to 
+        change power states to accomodate the child, 
+    * or if our power-controlling driver calls changePowerStateTo, 
+    * or if some other driver which is using our device calls makeUsable, 
+    * or if a subclassed object calls changePowerStateToPriv.  
+ These are all power changes initiated by us, not forced upon us by the parent.  
+ 
+ -> We start by notifying interested parties.  
+        -> If they all acknowledge via return code, we can go on to state 
+            "msSetPowerState".  
+        -> Otherwise, we start the ack timer and wait for the stragglers to 
+            acknowlege by calling acknowledgePowerChange.  
+            -> We move on to state "msSetPowerState" when all the 
+                stragglers have acknowledged, or when the ack timer expires on 
+                all those which didn't acknowledge.  
+ In "msSetPowerState" we call the power-controlling driver to change the 
+ power state of the hardware.  
+    -> If it returns saying it has done so, we go on to state 
+        "msWaitForPowerSettle".
+    -> Otherwise, we have to wait for it, so we set the ack timer and wait.  
+        -> When it calls acknowledgeSetPowerState, or when the ack timer 
+            expires, we go on.  
+ In "msWaitForPowerSettle", we look in the power state array to see if 
+ there is any settle time required when changing from our current state to the 
+ new state.  
+    -> If not, we go right away to "msNotifyInterestedDriversDidChange".  
+    -> Otherwise, we set the settle timer and wait. When it expires, we move on.  
+ In "msNotifyInterestedDriversDidChange" state, we notify all our 
+ interested parties via their powerStateDidChange methods that we have finished 
+ changing power state.  
+    -> If they all acknowledge via return code, we move on to "msFinish".  
+    -> Otherwise we set the ack timer and wait.  When they have all 
+        acknowledged, or when the ack timer has expired for those that didn't, 
+        we move on to "msFinish".
+ In "msFinish" we remove the used change note from the head of the queue 
+ and start the next one if one exists.
+
+ == 2. Parent Change Down ==
+ Start at Stage 2 of OurChange Down    XXX gvdl
+
+ == 3. Change Up ==
+ Start at Stage 4 of OurChange Down    XXX gvdl
+
+Note all parent requested changes need to acknowledge the power has changed to the parent when done.
+ */
 
+//*********************************************************************************
+// [public virtual] PMinit
+//
+// Initialize power management.
+//*********************************************************************************
 
 void IOService::PMinit ( void )
 {
-    if ( ! initialized ) {
-
-        // make space for our variables
-        pm_vars =  new IOPMprot;					
-        priv = new IOPMpriv;
-        pm_vars->init();
-        priv->init();
-        
-
-        // add pm_vars & priv to the properties
-        setProperty(prot_key, (OSObject *) pm_vars);			
-        setProperty(priv_key, (OSObject *) priv);
-
-        // then initialize them
-        priv->owner = this;
-        pm_vars->theNumberOfPowerStates = 0;				
-        priv->we_are_root = false;
-        pm_vars->theControllingDriver = NULL;
-        priv->our_lock = IOLockAlloc();
-        priv->flags_lock = IOLockAlloc();
-        priv->queue_lock = IOLockAlloc();
-        pm_vars->childLock = IOLockAlloc();
-        pm_vars->parentLock = IOLockAlloc();
-        priv->interestedDrivers = new IOPMinformeeList;
-        priv->interestedDrivers->initialize();
-        priv->changeList = new IOPMchangeNoteList;
-        priv->changeList->initialize();
-        pm_vars->aggressiveness = 0;
-        for (unsigned int i = 0; i <= kMaxType; i++) 
+    if ( !initialized )
+	{
+		if ( !gIOPMInitialized )
+		{
+			gIOPMWorkLoop = IOWorkLoop::workLoop();
+			if (gIOPMWorkLoop)
+			{
+				gIOPMRequestQueue = IOPMRequestQueue::create(
+					this, OSMemberFunctionCast(IOPMRequestQueue::Action,
+						this, &IOService::servicePMRequestQueue));
+
+				gIOPMReplyQueue = IOPMRequestQueue::create(
+					this, OSMemberFunctionCast(IOPMRequestQueue::Action,
+						this, &IOService::servicePMReplyQueue));
+
+				gIOPMFreeQueue = IOPMRequestQueue::create(
+					this, OSMemberFunctionCast(IOPMRequestQueue::Action,
+						this, &IOService::servicePMFreeQueue));
+
+				if (gIOPMWorkLoop->addEventSource(gIOPMRequestQueue) !=
+					kIOReturnSuccess)
+				{
+					gIOPMRequestQueue->release();
+					gIOPMRequestQueue = 0;
+				}
+
+				if (gIOPMWorkLoop->addEventSource(gIOPMReplyQueue) !=
+					kIOReturnSuccess)
+				{
+					gIOPMReplyQueue->release();
+					gIOPMReplyQueue = 0;
+				}
+
+				if (gIOPMWorkLoop->addEventSource(gIOPMFreeQueue) !=
+					kIOReturnSuccess)
+				{
+					gIOPMFreeQueue->release();
+					gIOPMFreeQueue = 0;
+				}
+			}
+
+			if (gIOPMRequestQueue && gIOPMReplyQueue && gIOPMFreeQueue)
+				gIOPMInitialized = true;
+		}
+		if (!gIOPMInitialized)
+			return;
+
+        pwrMgt = new IOServicePM;
+        pwrMgt->init();
+        setProperty(kPwrMgtKey, pwrMgt);
+
+        fOwner                      = this;
+        fWeAreRoot                  = false;
+        fPMLock                     = IOLockAlloc();
+        fInterestedDrivers          = new IOPMinformeeList;
+        fInterestedDrivers->initialize();
+        fDesiredPowerState          = 0;
+        fDriverDesire               = 0;
+        fDeviceDesire               = 0;
+        fInitialChange              = true;
+        fNeedToBecomeUsable         = false;
+        fPreviousRequest            = 0;
+        fDeviceOverrides            = false;
+        fMachineState               = kIOPM_Finished;
+        fIdleTimerEventSource       = NULL;
+        fActivityLock               = IOLockAlloc();
+        fClampOn                    = false;
+        fStrictTreeOrder            = false;
+        fActivityTicklePowerState   = -1;
+        fControllingDriver          = NULL;
+        fPowerStates                = NULL;
+        fNumberOfPowerStates        = 0;
+        fCurrentPowerState          = 0;
+        fParentsCurrentPowerFlags   = 0;
+        fMaxCapability              = 0;
+        fName                       = getName();
+        fPlatform                   = getPlatform();
+        fParentsKnowState           = false;
+        fSerialNumber               = 0;
+        fResponseArray              = NULL;
+        fDoNotPowerDown             = true;
+        fCurrentPowerConsumption    = kIOPMUnknown;
+
+        for (unsigned int i = 0; i <= kMaxType; i++)
         {
-	        pm_vars->current_aggressiveness_values[i] = 0;
-	        pm_vars->current_aggressiveness_valid[i] = false;
+	        fAggressivenessValue[i] = 0;
+	        fAggressivenessValid[i]  = false;
         }
-        pm_vars->myCurrentState =  0;
-        priv->imminentState = 0;
-        priv->ourDesiredPowerState = 0;
-        pm_vars->parentsCurrentPowerFlags = 0;
-        pm_vars->maxCapability = 0;
-        priv->driverDesire = 0;
-        priv->deviceDesire = 0;
-        priv->initial_change = true;
-        priv->need_to_become_usable = false;
-        priv->previousRequest = 0;
-        priv->device_overrides = false;
-        priv->machine_state = kIOPM_Finished;
-        priv->timerEventSrc = NULL;
-        priv->clampTimerEventSrc = NULL;
-        pm_vars->PMworkloop = NULL;
-        priv->activityLock = NULL;
-        pm_vars->ourName = getName();
-        pm_vars->thePlatform = getPlatform();
-        pm_vars->parentsKnowState = false;
-        assert( pm_vars->thePlatform != 0 );
-        priv->clampOn = false;
-        pm_vars->serialNumber = 0;
-        pm_vars->responseFlags = NULL;
-        pm_vars->doNotPowerDown = true;
-        pm_vars->PMcommandGate = NULL;
-        priv->ackTimer = thread_call_allocate(
-                        (thread_call_func_t)ack_timer_expired, 
-                        (thread_call_param_t)this);
-        priv->settleTimer = thread_call_allocate(
-                        (thread_call_func_t)settle_timer_expired, 
-                        (thread_call_param_t)this);
-        
+
+        fAckTimer = thread_call_allocate(
+			&IOService::ack_timer_expired, (thread_call_param_t)this);
+        fSettleTimer = thread_call_allocate(
+			&settle_timer_expired, (thread_call_param_t)this);
+		fDriverCallEntry = thread_call_allocate(
+			(thread_call_func_t) &IOService::pmDriverCallout, this);
+		assert(fDriverCallEntry);
+
+#if PM_VARS_SUPPORT
+        IOPMprot * prot = new IOPMprot;
+        if (prot)
+        {
+            prot->init();
+            prot->ourName = fName;
+            prot->thePlatform = fPlatform;
+            fPMVars = prot;
+            pm_vars = prot;
+		}
+#else
+        pm_vars = (IOPMprot *) true;
+#endif
+
         initialized = true;
     }
 }
 
-
 //*********************************************************************************
-// PMfree
+// [public] PMfree
 //
 // Free up the data created in PMinit, if it exists.
 //*********************************************************************************
+
 void IOService::PMfree ( void )
 {
-    if ( priv ) {
-        if (  priv->clampTimerEventSrc != NULL ) {
-            getPMworkloop()->removeEventSource(priv->clampTimerEventSrc);
-            priv->clampTimerEventSrc->release();
-            priv->clampTimerEventSrc = NULL;
-        }
-        if (  priv->timerEventSrc != NULL ) {
-            pm_vars->PMworkloop->removeEventSource(priv->timerEventSrc);
-            priv->timerEventSrc->release();
-            priv->timerEventSrc = NULL;
+	initialized = false;
+    pm_vars = 0;
+
+    if ( pwrMgt )
+	{
+		assert(fMachineState == kIOPM_Finished);
+		assert(fInsertInterestSet == NULL);
+		assert(fRemoveInterestSet == NULL);
+        assert(fNotifyChildArray  == NULL);
+
+        if ( fIdleTimerEventSource != NULL ) {
+            getPMworkloop()->removeEventSource(fIdleTimerEventSource);
+            fIdleTimerEventSource->release();
+            fIdleTimerEventSource = NULL;
         }
-        if ( priv->settleTimer ) {
-            thread_call_cancel(priv->settleTimer);
-            thread_call_free(priv->settleTimer);
-            priv->settleTimer = NULL;
+        if ( fSettleTimer ) {
+            thread_call_cancel(fSettleTimer);
+            thread_call_free(fSettleTimer);
+            fSettleTimer = NULL;
         }
-        if ( priv->ackTimer ) {
-            thread_call_cancel(priv->ackTimer);
-            thread_call_free(priv->ackTimer);
-            priv->ackTimer = NULL;
+        if ( fAckTimer ) {
+            thread_call_cancel(fAckTimer);
+            thread_call_free(fAckTimer);
+            fAckTimer = NULL;
         }
-        if ( priv->our_lock ) {
-            IOLockFree(priv->our_lock);
-            priv->our_lock = NULL;
+        if ( fDriverCallEntry ) {
+            thread_call_free(fDriverCallEntry);
+            fDriverCallEntry = NULL;
         }
-        if ( priv->flags_lock ) {
-            IOLockFree(priv->flags_lock);
-            priv->flags_lock = NULL;
+        if ( fPMLock ) {
+            IOLockFree(fPMLock);
+            fPMLock = NULL;
         }
-        if ( priv->activityLock ) {
-            IOLockFree(priv->activityLock);
-            priv->activityLock = NULL;
+        if ( fActivityLock ) {
+            IOLockFree(fActivityLock);
+            fActivityLock = NULL;
         }
-        priv->interestedDrivers->release();
-        priv->changeList->release();
-        // remove instance variables
-        priv->release();				
-    }
-    
-    if ( pm_vars ) {
-        if ( pm_vars->PMcommandGate ) {
-            if(pm_vars->PMworkloop)
-                pm_vars->PMworkloop->removeEventSource(pm_vars->PMcommandGate);
-            pm_vars->PMcommandGate->release();
-            pm_vars->PMcommandGate = NULL;
+		if ( fInterestedDrivers ) {
+			fInterestedDrivers->release();
+			fInterestedDrivers = NULL;
+		}
+		if ( fPMWorkQueue ) {
+			getPMworkloop()->removeEventSource(fPMWorkQueue);
+			fPMWorkQueue->release();
+			fPMWorkQueue = 0;
+		}
+		if (fDriverCallParamSlots && fDriverCallParamPtr) {
+			IODelete(fDriverCallParamPtr, DriverCallParam, fDriverCallParamSlots);
+			fDriverCallParamPtr = 0;
+			fDriverCallParamSlots = 0;
+		}
+        if ( fResponseArray ) {
+            fResponseArray->release();
+            fResponseArray = NULL;
         }
-        if ( pm_vars->PMworkloop ) {
-            // The work loop object returned from getPMworkLoop() is
-            // never retained, therefore it should not be released.
-            // pm_vars->PMworkloop->release();
-            pm_vars->PMworkloop = NULL;
+        if (fPowerStates && fNumberOfPowerStates) {
+            IODelete(fPowerStates, IOPMPowerState, fNumberOfPowerStates);
+            fNumberOfPowerStates = 0;
+            fPowerStates = NULL;
         }
-        if ( pm_vars->responseFlags ) {
-            pm_vars->responseFlags->release();
-            pm_vars->responseFlags = NULL;
-        }
-        // remove instance variables
-        pm_vars->release();				
+
+#if PM_VARS_SUPPORT
+		if (fPMVars)
+		{
+			fPMVars->release();
+			fPMVars = 0;
+		}
+#endif
+
+        pwrMgt->release();
+		pwrMgt = 0;
     }
 }
 
+//*********************************************************************************
+// [public virtual] joinPMtree
+//
+// A policy-maker calls its nub here when initializing, to be attached into
+// the power management hierarchy.  The default function is to call the
+// platform expert, which knows how to do it.  This method is overridden
+// by a nub subclass which may either know how to do it, or may need to
+// take other action.
+//
+// This may be the only "power management" method used in a nub,
+// meaning it may not be initialized for power management.
+//*********************************************************************************
+
+void IOService::joinPMtree ( IOService * driver )
+{
+    IOPlatformExpert * platform;
+
+    platform = getPlatform();
+    assert(platform != 0);
+    platform->PMRegisterDevice(this, driver);
+}
 
 //*********************************************************************************
-// PMstop
+// [public virtual] youAreRoot
 //
-// Disconnect the node from its parents and children in the Power Plane.
+// Power Managment is informing us that we are the root power domain.
+// The only difference between us and any other power domain is that
+// we have no parent and therefore never call it.
+//*********************************************************************************
+
+IOReturn IOService::youAreRoot ( void )
+{
+    fWeAreRoot = true;
+    fParentsKnowState = true;
+    attachToParent( getRegistryRoot(), gIOPowerPlane );
+    return IOPMNoErr;
+}
+
+//*********************************************************************************
+// [public virtual] PMstop
+//
+// Immediately stop driver callouts. Schedule an async stop request to detach
+// from power plane.
 //*********************************************************************************
+
 void IOService::PMstop ( void )
 {
-    OSIterator *	iter;
-    OSObject *		next;
+	IOPMRequest * request;
+
+	if (!initialized)
+		return;
+
+	// Schedule an async PMstop request, but immediately stop any further
+	// calls to the controlling or interested drivers. This device will
+	// continue to exist in the power plane and participate in power state
+	// changes until the PMstop async request is processed.
+
+	PM_LOCK();
+	fWillPMStop = true;
+    if (fDriverCallBusy)
+        PM_DEBUG("%s::PMstop() driver call busy\n", getName());
+    PM_UNLOCK();
+
+	request = acquirePMRequest( this, kIOPMRequestTypePMStop );
+	if (request)
+	{
+		PM_TRACE("[%s] %p PMstop\n", getName(), this);
+		submitPMRequest( request );
+	}
+}
+
+//*********************************************************************************
+// handlePMstop
+//
+// Disconnect the node from its parents and children in the Power Plane.
+//*********************************************************************************
+
+void IOService::handlePMstop ( IOPMRequest * request )
+{
+    OSIterator *		iter;
+    OSObject *			next;
     IOPowerConnection *	connection;
-    IOService *		theChild;
-    IOService *		theParent;
+    IOService *			theChild;
+    IOService *			theParent;
 
-    // remove the properties
-    removeProperty(prot_key);			
-    removeProperty(priv_key);
-    
-    // detach parents
-    iter = getParentIterator(gIOPowerPlane);	
+	PM_ASSERT_IN_GATE();
+	PM_TRACE("[%s] %p %s start\n", getName(), this, __FUNCTION__);
 
-    if ( iter ) 
+    // remove the property
+    removeProperty(kPwrMgtKey);			
+
+    // detach parents
+    iter = getParentIterator(gIOPowerPlane);
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
                 theParent = (IOService *)connection->copyParentEntry(gIOPowerPlane);
-                if ( theParent ) 
+                if ( theParent )
                 {
                     theParent->removePowerChild(connection);
                     theParent->release();
@@ -400,33 +628,29 @@ void IOService::PMstop ( void )
         iter->release();
     }
 
-    // detach IOConnections   
-    detachAbove( gIOPowerPlane );		
+    // detach IOConnections
+    detachAbove( gIOPowerPlane );
     
-    if ( pm_vars )
-    {
-        // no more power state changes
-        pm_vars->parentsKnowState = false;		
-    }
-    
-    // detach children
-    iter = getChildIterator(gIOPowerPlane);	
+    // no more power state changes
+    fParentsKnowState = false;
 
-    if ( iter ) 
+    // detach children
+    iter = getChildIterator(gIOPowerPlane);
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
                 theChild = ((IOService *)(connection->copyChildEntry(gIOPowerPlane)));
-                if ( theChild ) 
+                if ( theChild )
                 {
                     // detach nub from child
-                    connection->detachFromChild(theChild,gIOPowerPlane);	
+                    connection->detachFromChild(theChild, gIOPowerPlane);
                     theChild->release();
                 }
                 // detach us from nub
-                detachFromChild(connection,gIOPowerPlane);			
+                detachFromChild(connection, gIOPowerPlane);
             }
         }
         iter->release();
@@ -440,380 +664,478 @@ void IOService::PMstop ( void )
     // the object will be holding an extra retain on itself, and cannot
     // be freed.
 
-    if ( priv && priv->interestedDrivers )
+    if ( fInterestedDrivers )
     {
-        IOPMinformee * informee;
+		IOPMinformeeList *	list = fInterestedDrivers;
+        IOPMinformee *		item;
 
-        while (( informee = priv->interestedDrivers->firstInList() ))
-            deRegisterInterestedDriver( informee->whatObject );
-    }
-}
+		PM_LOCK();
+		while ((item = list->firstInList()))
+		{
+			list->removeFromList(item->whatObject);
+		}
+		PM_UNLOCK();
+	}
+
+	// Tell PM_idle_timer_expiration() to ignore idle timer.
+	fIdleTimerPeriod = 0;
 
+	fWillPMStop = false;
+	PM_TRACE("[%s] %p %s done\n", getName(), this, __FUNCTION__);
+}
 
 //*********************************************************************************
-// joinPMtree
-//
-// A policy-maker calls its nub here when initializing, to be attached into
-// the power management hierarchy.  The default function is to call the
-// platform expert, which knows how to do it.  This method is overridden
-// by a nub subclass which may either know how to do it, or may need
-// to take other action.
+// [public virtual] addPowerChild
 //
-// This may be the only "power management" method used in a nub,
-// meaning it may not be initialized for power management.
+// Power Management is informing us who our children are.
 //*********************************************************************************
-void IOService::joinPMtree ( IOService * driver )
+
+IOReturn IOService::addPowerChild ( IOService * child )
 {
-    IOPlatformExpert * thePlatform;
+	IOPowerConnection *	connection  = 0;
+	IOPMRequest *		requests[3] = {0, 0, 0};
+    OSIterator *		iter;
+	bool				ok = true;
+
+	if (!child)
+		return kIOReturnBadArgument;
+
+    if (!initialized || !child->initialized)
+		return IOPMNotYetInitialized;
+
+    OUR_PMLog( kPMLogAddChild, 0, 0 );
+
+	do {
+		// Is this child already one of our children?
+
+		iter = child->getParentIterator( gIOPowerPlane );
+		if ( iter )
+		{
+			IORegistryEntry *	entry;
+			OSObject *			next;
+
+			while ((next = iter->getNextObject()))
+			{
+				if ((entry = OSDynamicCast(IORegistryEntry, next)) &&
+					isChild(entry, gIOPowerPlane))
+				{
+					ok = false;
+					break;
+				}			
+			}
+			iter->release();
+		}
+		if (!ok)
+		{
+			PM_DEBUG("[%s] %s (%p) is already a child\n",
+				getName(), child->getName(), child);
+			break;
+		}
+
+		// Add the child to the power plane immediately, but the
+		// joining connection is marked as not ready.
+		// We want the child to appear in the power plane before
+		// returning to the caller, but don't want the caller to
+		// block on the PM work loop.
+
+		connection = new IOPowerConnection;
+		if (!connection)
+			break;
+
+		// Create a chain of PM requests to perform the bottom-half
+		// work from the PM work loop.
+
+		requests[0] = acquirePMRequest(
+					/* target */ this,
+					/* type */   kIOPMRequestTypeAddPowerChild1 );
+
+		requests[1] = acquirePMRequest(
+					/* target */ child,
+					/* type */   kIOPMRequestTypeAddPowerChild2 );
+
+		requests[2] = acquirePMRequest(
+					/* target */ this,
+					/* type */   kIOPMRequestTypeAddPowerChild3 );
+
+		if (!requests[0] || !requests[1] || !requests[2])
+			break;
+
+		requests[0]->setParentRequest( requests[1] );
+		requests[1]->setParentRequest( requests[2] );
+
+		connection->init();
+		connection->start(this);
+		connection->setAwaitingAck(false);
+		connection->setReadyFlag(false);
+
+		attachToChild( connection, gIOPowerPlane );
+		connection->attachToChild( child, gIOPowerPlane );
+
+		// connection needs to be released
+		requests[0]->fArg0 = connection;
+		requests[1]->fArg0 = connection;
+		requests[2]->fArg0 = connection;
+
+		submitPMRequest( requests, 3 );
+		return kIOReturnSuccess;
+	}
+	while (false);
 
-    thePlatform = getPlatform();
-    assert(thePlatform != 0 );
-    thePlatform->PMRegisterDevice(this,driver);
-}
+	if (connection)  connection->release();
+	if (requests[0]) releasePMRequest(requests[0]);
+	if (requests[1]) releasePMRequest(requests[1]);
+	if (requests[2]) releasePMRequest(requests[2]);
 
+	// silent failure, to prevent platform drivers from adding the child
+	// to the root domain.
+	return IOPMNoErr;
+}
 
 //*********************************************************************************
-// youAreRoot
+// [private] addPowerChild1
 //
-// Power Managment is informing us that we are the root power domain.
-// The only difference between us and any other power domain is that
-// we have no parent and therefore never call it.
+// Called on the power parent.
 //*********************************************************************************
-IOReturn IOService::youAreRoot ( void )
-{    
-    priv-> we_are_root = true;
-    pm_vars->parentsKnowState = true;
-    attachToParent( getRegistryRoot(),gIOPowerPlane );
-    
-    return IOPMNoErr;
-}
 
+void IOService::addPowerChild1 ( IOPMRequest * request )
+{
+	unsigned long tempDesire = 0;
+
+	// Make us temporary usable before adding the child.
+
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog( kPMLogMakeUsable, kPMLogMakeUsable, fDeviceDesire );
+
+	if (fControllingDriver && inPlane(gIOPowerPlane) && fParentsKnowState)
+	{
+		tempDesire = fNumberOfPowerStates - 1;
+	}
+
+	if (tempDesire && (fWeAreRoot || (fMaxCapability >= tempDesire)))
+	{
+		computeDesiredState( tempDesire );
+		changeState();
+	}
+}
 
 //*********************************************************************************
-// setPowerParent
+// [private] addPowerChild2
 //
-// Power Management is informing us who our parent is.
-// If we have a controlling driver, find out, given our newly-informed
-// power domain state, what state it would be in, and then tell it
-// to assume that state.
+// Called on the joining child. Blocked behind addPowerChild1.
 //*********************************************************************************
-IOReturn IOService::setPowerParent ( IOPowerConnection * theParent, bool stateKnown, IOPMPowerFlags currentState )
+
+void IOService::addPowerChild2 ( IOPMRequest * request )
 {
-    OSIterator *			iter;
-    OSObject *			next;
-    IOPowerConnection *	connection;
-    unsigned long		tempDesire;
+	IOPowerConnection * connection = (IOPowerConnection *) request->fArg0;
+	IOService *         parent;
+	IOPMPowerFlags		powerFlags;
+	bool				knowsState;
+	unsigned long		powerState;
+	unsigned long		tempDesire;
+
+	PM_ASSERT_IN_GATE();
+	parent = (IOService *) connection->getParentEntry(gIOPowerPlane);
+
+	if (!parent || !inPlane(gIOPowerPlane))
+	{
+		PM_DEBUG("[%s] addPowerChild2 not in power plane\n", getName());
+		return;
+	}
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogSetParent,stateKnown,currentState);
-    
-    IOLockLock(pm_vars->parentLock);
-    
-    if ( stateKnown && ((pm_vars->PMworkloop == NULL) || (pm_vars->PMcommandGate == NULL)) ) 
-    {
-        // we have a path to the root
-        // find out the workloop
-        getPMworkloop();
-        if ( pm_vars->PMworkloop != NULL ) 
-        {
-            if ( pm_vars->PMcommandGate == NULL ) 
-            {	
-                // and make our command gate
-                pm_vars->PMcommandGate = IOCommandGate::commandGate((OSObject *)this);
-                if ( pm_vars->PMcommandGate != NULL ) 
-                {
-                    pm_vars->PMworkloop->addEventSource(pm_vars->PMcommandGate);
-                }
-            }
-        }
-    }
-    
-    IOLockUnlock(pm_vars->parentLock);
+	// Parent will be waiting for us to complete this stage, safe to
+	// directly access parent's vars.
 
-    // set our connection data
-    theParent->setParentCurrentPowerFlags(currentState);	
-    theParent->setParentKnowsState(stateKnown);
+	knowsState = (parent->fPowerStates) && (parent->fParentsKnowState);
+	powerState = parent->fCurrentPowerState;
 
-    // combine parent knowledge
-    pm_vars->parentsKnowState = true;				
-    pm_vars->parentsCurrentPowerFlags = 0;
-    
-    iter = getParentIterator(gIOPowerPlane);
+	if (knowsState)
+		powerFlags = parent->fPowerStates[powerState].outputPowerCharacter;
+	else
+		powerFlags = 0;
 
-    if ( iter ) 
-    {
-        while ( (next = iter->getNextObject()) ) 
-        {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
-            {
-                pm_vars->parentsKnowState &= connection->parentKnowsState();
-                pm_vars->parentsCurrentPowerFlags |= connection->parentCurrentPowerFlags();
-            }
-        }
-        iter->release();
-    }
-    
-    if ( (pm_vars->theControllingDriver != NULL) &&
-         (pm_vars->parentsKnowState) ) 
+	// Set our power parent.
+
+    OUR_PMLog(kPMLogSetParent, knowsState, powerFlags);
+
+	setParentInfo( powerFlags, connection, knowsState );
+
+	connection->setReadyFlag(true);
+
+    if ( fControllingDriver && fParentsKnowState )
     {
-        pm_vars->maxCapability = pm_vars->theControllingDriver->maxCapabilityForDomainState(pm_vars->parentsCurrentPowerFlags);
+        fMaxCapability = fControllingDriver->maxCapabilityForDomainState(fParentsCurrentPowerFlags);
         // initially change into the state we are already in
-        tempDesire = priv->deviceDesire;			
-        priv->deviceDesire = pm_vars->theControllingDriver->initialPowerStateForDomainState(pm_vars->parentsCurrentPowerFlags);
-        computeDesiredState();
-        priv->previousRequest = 0xffffffff;
+        tempDesire = fControllingDriver->initialPowerStateForDomainState(fParentsCurrentPowerFlags);
+        computeDesiredState(tempDesire);
+        fPreviousRequest = 0xffffffff;
         changeState();
-        // put this back like before
-        priv->deviceDesire = tempDesire;    
     }
-    
-   return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// addPowerChild
+// [private] addPowerChild3
 //
-// Power Management is informing us who our children are.
+// Called on the parent. Blocked behind addPowerChild2.
 //*********************************************************************************
-IOReturn IOService::addPowerChild ( IOService * theChild )
-{
-    IOPowerConnection                       *connection;
-    unsigned int                            i;
 
-    if ( ! initialized ) 
-    {
-        // we're not a power-managed IOService
-        return IOPMNotYetInitialized;	
-    }
+void IOService::addPowerChild3 ( IOPMRequest * request )
+{
+	IOPowerConnection * connection = (IOPowerConnection *) request->fArg0;
+	IOService *         child;
+	unsigned int		i;
+
+	PM_ASSERT_IN_GATE();
+	child = (IOService *) connection->getChildEntry(gIOPowerPlane);
+
+	if (child && inPlane(gIOPowerPlane))
+	{
+		if (child->getProperty("IOPMStrictTreeOrder"))
+		{
+			PM_DEBUG("[%s] strict ordering enforced\n", getName());
+			fStrictTreeOrder = true;
+		}
+
+		for (i = 0; i <= kMaxType; i++)
+		{
+			if ( fAggressivenessValid[i] )
+			{
+				child->setAggressiveness(i, fAggressivenessValue[i]);
+			}
+		}
+	}
+	else
+	{
+		PM_DEBUG("[%s] addPowerChild3 not in power plane\n", getName());
+	}
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAddChild,0,0);
+	connection->release();
+}
 
-    // Put ourselves into a usable power state.
-    // We must be in an "on" power state, as our children must be able to access
-    // our hardware after joining the power plane.
-    temporaryMakeUsable();
-    
-    // make a nub
-    connection = new IOPowerConnection;			
+//*********************************************************************************
+// [public virtual deprecated] setPowerParent
+//
+// Power Management is informing us who our parent is.
+// If we have a controlling driver, find out, given our newly-informed
+// power domain state, what state it would be in, and then tell it
+// to assume that state.
+//*********************************************************************************
 
-    connection->init();
-    connection->start(this);
-    connection->setAwaitingAck(false);
-    
-    // connect it up
-    attachToChild( connection,gIOPowerPlane );			
-    connection->attachToChild( theChild,gIOPowerPlane );
-    connection->release();
-    
-    // tell it the current state of the power domain
-    if ( (pm_vars->theControllingDriver == NULL) ||		
-            ! (inPlane(gIOPowerPlane)) ||
-            ! (pm_vars->parentsKnowState) ) 
-    {
-        theChild->setPowerParent(connection,false,0);
-        if ( inPlane(gIOPowerPlane) ) 
-        {
-            for (i = 0; i <= kMaxType; i++) {
-                if ( pm_vars->current_aggressiveness_valid[i] ) 
-                {
-                    theChild->setAggressiveness (i, pm_vars->current_aggressiveness_values[i]);
-                }
-            }
-        }
-    } else {
-        theChild->setPowerParent(connection,true,pm_vars->thePowerStates[pm_vars->myCurrentState].outputPowerCharacter);
-        for (i = 0; i <= kMaxType; i++) 
-        {
-            if ( pm_vars->current_aggressiveness_valid[i] )
-            {
-                theChild->setAggressiveness (i, pm_vars->current_aggressiveness_values[i]);
-            }
-        }
-        // catch it up if change is in progress
-        add_child_to_active_change(connection);							
-    }
-    
-    return IOPMNoErr;
+IOReturn IOService::setPowerParent (
+	IOPowerConnection * theParent, bool stateKnown, IOPMPowerFlags powerFlags )
+{
+	return kIOReturnUnsupported;
 }
 
-
-//******************************************************************************
-// removePowerChild
+//*********************************************************************************
+// [public virtual] removePowerChild
 //
-//******************************************************************************
+// Called on a parent whose child is being removed by PMstop().
+//*********************************************************************************
+
 IOReturn IOService::removePowerChild ( IOPowerConnection * theNub )
 {
-    IORegistryEntry                         *theChild;
+    IORegistryEntry *	theChild;
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogRemoveChild,0,0);
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog( kPMLogRemoveChild, 0, 0 );
 
     theNub->retain();
     
     // detach nub from child
     theChild = theNub->copyChildEntry(gIOPowerPlane);			
-    if ( theChild ) 
+    if ( theChild )
     {
         theNub->detachFromChild(theChild, gIOPowerPlane);
         theChild->release();
     }
     // detach from the nub
-    detachFromChild(theNub,gIOPowerPlane);				
-    
-    // are we awaiting an ack from this child?
-    if ( theNub->getAwaitingAck() ) 
-    {
-        // yes, pretend we got one
-        theNub->setAwaitingAck(false);
-        if ( acquire_lock() ) 
-        {
-            if (priv->head_note_pendingAcks != 0 ) 
-            {
-                // that's one fewer ack to worry about
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                // is that the last?
-                if ( priv->head_note_pendingAcks == 0 ) 
-                {
-                    // yes, stop the timer
-                    stop_ack_timer();
-                    IOUnlock(priv->our_lock);
-                    // and now we can continue our power change
-                    all_acked();
-                } else {
-                    IOUnlock(priv->our_lock);
-                }
-            } else {
-                IOUnlock(priv->our_lock);
-            }
-        }
-    }
+    detachFromChild(theNub, gIOPowerPlane);
+
+    // Are we awaiting an ack from this child?
+    if ( theNub->getAwaitingAck() )
+	{
+		// yes, pretend we got one
+		theNub->setAwaitingAck(false);
+		if (fHeadNotePendingAcks != 0 )
+		{
+			// that's one fewer ack to worry about
+			fHeadNotePendingAcks--;
+
+			// is that the last?
+			if ( fHeadNotePendingAcks == 0 )
+			{
+				stop_ack_timer();
+			}
+		}
+	}
 
-    theNub->release();
+	theNub->release();
 
-    // if not fully initialized
-    if ( (pm_vars->theControllingDriver == NULL) ||
-                !(inPlane(gIOPowerPlane)) ||
-                !(pm_vars->parentsKnowState) ) 
-    {
-        // we can do no more
-        return IOPMNoErr;				
-    }
+	// Schedule a request to re-scan child desires and clamp bits.
+	if (!fWillAdjustPowerState)
+	{
+		IOPMRequest * request;
 
-    // Perhaps the departing child was holding up idle or system sleep - we 
-    // need to re-evaluate our childrens' requests. 
-    // Clear and re-calculate our kIOPMChildClamp and kIOPMChildClamp2 bits.
-    rebuildChildClampBits();
-    
-    // Change state if we can now tolerate lower power
-    computeDesiredState();
-    changeState();
+		request = acquirePMRequest( this, kIOPMRequestTypeAdjustPowerState );
+		if (request)
+		{
+			submitPMRequest( request );
+			fWillAdjustPowerState = true;
+		}
+	}
 
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// registerPowerDriver
+// [public virtual] registerPowerDriver
 //
 // A driver has called us volunteering to control power to our device.
-// If the power state array it provides is richer than the one we already
-// know about (supplied by an earlier volunteer), then accept the offer.
-// Notify all interested parties of our power state, which we now know.
 //*********************************************************************************
 
-IOReturn IOService::registerPowerDriver ( IOService * controllingDriver, IOPMPowerState* powerStates, unsigned long numberOfStates  )
+IOReturn IOService::registerPowerDriver (
+	IOService *			powerDriver,
+	IOPMPowerState *	powerStates,
+	unsigned long		numberOfStates )
 {
-    unsigned long                           i;
-    unsigned long                           tempDesire;
+	IOPMRequest *	 request;
+	IOPMPowerState * powerStatesCopy = 0;
 
-    if ( (numberOfStates > pm_vars->theNumberOfPowerStates) 
-                    && (numberOfStates > 1) ) 
-    {
-        if (  priv->changeList->currentChange() == -1 ) 
-        {
-            if ( controllingDriver != NULL ) 
-            {
-                if ( numberOfStates <= IOPMMaxPowerStates ) 
-                {
-                    switch ( powerStates[0].version  ) 
-                    {
-                        case 1:
-                            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriver,
-                                            (unsigned long)numberOfStates, (unsigned long)powerStates[0].version);
-                            for ( i = 0; i < numberOfStates; i++ ) 
-                            {
-                                pm_vars->thePowerStates[i] = powerStates[i];
-                            }
-                                break;
-                        case 2:
-                            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriver,
-                                            (unsigned long) numberOfStates,(unsigned long) powerStates[0].version);
-                            for ( i = 0; i < numberOfStates; i++ ) 
-                            {
-                                pm_vars->thePowerStates[i].version = powerStates[i].version;
-                                pm_vars->thePowerStates[i].capabilityFlags = powerStates[i].capabilityFlags;
-                                pm_vars->thePowerStates[i].outputPowerCharacter = powerStates[i].outputPowerCharacter;
-                                pm_vars->thePowerStates[i].inputPowerRequirement = powerStates[i].inputPowerRequirement;
-                                pm_vars->thePowerStates[i].staticPower = powerStates[i].staticPower;
-                                pm_vars->thePowerStates[i].unbudgetedPower = powerStates[i].unbudgetedPower;
-                                pm_vars->thePowerStates[i].powerToAttain = powerStates[i].powerToAttain;
-                                pm_vars->thePowerStates[i].timeToAttain = powerStates[i].timeToAttain;
-                                pm_vars->thePowerStates[i].settleUpTime = powerStates[i].settleUpTime;
-                                pm_vars->thePowerStates[i].timeToLower = powerStates[i].timeToLower;
-                                pm_vars->thePowerStates[i].settleDownTime = powerStates[i].settleDownTime;
-                                pm_vars->thePowerStates[i].powerDomainBudget = powerStates[i].powerDomainBudget;
-                            }
-                                break;
-                        default:
-                            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriverErr1,
-                                                                    (unsigned long)powerStates[0].version,0);
-                            return IOPMNoErr;
-                    }
+    if (!initialized)
+		return IOPMNotYetInitialized;
 
-                    // make a mask of all the character bits we know about
-                    pm_vars->myCharacterFlags = 0;	
-                    for ( i = 0; i < numberOfStates; i++ ) {
-                        pm_vars->myCharacterFlags |= pm_vars->thePowerStates[i].outputPowerCharacter;
-                    }
-                    
-                   pm_vars->theNumberOfPowerStates = numberOfStates;
-                    pm_vars->theControllingDriver = controllingDriver;
-                    if ( priv->interestedDrivers->findItem(controllingDriver) == NULL ) 
-                    {
-                        // register it as interested, unless already done
-                        registerInterestedDriver (controllingDriver );
-                    }
-                    if ( priv->need_to_become_usable ) {
-                        priv->need_to_become_usable = false;
-                        priv->deviceDesire = pm_vars->theNumberOfPowerStates - 1;
-                    }
+	// Validate arguments.
+	if (!powerStates || (numberOfStates < 2))
+	{
+		OUR_PMLog(kPMLogControllingDriverErr5, numberOfStates, 0);
+		return kIOReturnBadArgument;
+	}
 
-                    if ( inPlane(gIOPowerPlane) &&
-                         (pm_vars->parentsKnowState) ) {
-                        pm_vars->maxCapability = pm_vars->theControllingDriver->maxCapabilityForDomainState(pm_vars->parentsCurrentPowerFlags);
-                        // initially change into the state we are already in
-                        tempDesire = priv->deviceDesire;
-                        priv->deviceDesire = pm_vars->theControllingDriver->initialPowerStateForDomainState(pm_vars->parentsCurrentPowerFlags);
-                        computeDesiredState();
-                        changeState();
-                        // put this back like before
-                        priv->deviceDesire = tempDesire;
-                    }
-                } else {
-                    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriverErr2,(unsigned long)numberOfStates,0);
-                }
-            } else {
-                pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriverErr4,0,0);
-            }
-        }
-    }
-    else {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogControllingDriverErr5,(unsigned long)numberOfStates,0);
-    }
-    return IOPMNoErr;
+	if (!powerDriver)
+	{
+		OUR_PMLog(kPMLogControllingDriverErr4, 0, 0);
+		return kIOReturnBadArgument;
+	}
+
+	if (powerStates[0].version != kIOPMPowerStateVersion1)
+	{
+		OUR_PMLog(kPMLogControllingDriverErr1, powerStates[0].version, 0);
+		return kIOReturnBadArgument;
+	}
+
+	do {
+		// Make a copy of the supplied power state array.
+		powerStatesCopy = IONew(IOPMPowerState, numberOfStates);
+		if (!powerStatesCopy)
+			break;
+
+		bcopy( powerStates, powerStatesCopy,
+			sizeof(IOPMPowerState) * numberOfStates );
+
+		request = acquirePMRequest( this, kIOPMRequestTypeRegisterPowerDriver );
+		if (!request)
+			break;
+
+		powerDriver->retain();
+		request->fArg0 = (void *) powerDriver;
+		request->fArg1 = (void *) powerStatesCopy;
+		request->fArg2 = (void *) numberOfStates;
+
+		submitPMRequest( request );
+		return kIOReturnSuccess;
+	}
+	while (false);
+
+	if (powerStatesCopy)
+		IODelete(powerStatesCopy, IOPMPowerState, numberOfStates);
+	return kIOReturnNoMemory;
 }
 
 //*********************************************************************************
-// registerInterestedDriver
+// [private] handleRegisterPowerDriver
+//*********************************************************************************
+
+void IOService::handleRegisterPowerDriver ( IOPMRequest * request )
+{
+	IOService *			powerDriver    = (IOService *)      request->fArg0;
+	IOPMPowerState *	powerStates    = (IOPMPowerState *) request->fArg1;
+	unsigned long		numberOfStates = (unsigned long)    request->fArg2;
+    unsigned long		i;
+	IOService *			root;
+
+	PM_ASSERT_IN_GATE();
+	assert(powerStates);
+	assert(powerDriver);
+	assert(numberOfStates > 1);
+
+    if ( !fNumberOfPowerStates )
+    {
+		OUR_PMLog(kPMLogControllingDriver,
+			(unsigned long) numberOfStates,
+			(unsigned long) powerStates[0].version);
+
+        fPowerStates            = powerStates;
+		fNumberOfPowerStates    = numberOfStates;
+		fControllingDriver      = powerDriver;
+        fCurrentCapabilityFlags = fPowerStates[0].capabilityFlags;
+
+		// make a mask of all the character bits we know about
+		fOutputPowerCharacterFlags = 0;
+		for ( i = 0; i < numberOfStates; i++ ) {
+			fOutputPowerCharacterFlags |= fPowerStates[i].outputPowerCharacter;
+		}
+
+		// Register powerDriver as interested, unless already done.
+		// We don't want to register the default implementation since
+		// it does nothing. One ramification of not always registering
+		// is the one fewer retain count held.
+
+		root = getPlatform()->getProvider();
+		assert(root);
+		if (!root ||
+			((OSMemberFunctionCast(void (*)(void),
+				root, &IOService::powerStateDidChangeTo)) !=
+			((OSMemberFunctionCast(void (*)(void),
+				this, &IOService::powerStateDidChangeTo)))) ||
+			((OSMemberFunctionCast(void (*)(void),
+				root, &IOService::powerStateWillChangeTo)) !=
+			((OSMemberFunctionCast(void (*)(void),
+				this, &IOService::powerStateWillChangeTo)))))
+		{		
+			if (fInterestedDrivers->findItem(powerDriver) == NULL)
+			{
+				PM_LOCK();
+				fInterestedDrivers->appendNewInformee(powerDriver);
+				PM_UNLOCK();
+			}
+		}
+
+		if ( fNeedToBecomeUsable ) {
+			fNeedToBecomeUsable = false;
+			fDeviceDesire = fNumberOfPowerStates - 1;
+		}
+
+		if ( inPlane(gIOPowerPlane) && fParentsKnowState )
+		{
+			unsigned long tempDesire;
+			fMaxCapability = fControllingDriver->maxCapabilityForDomainState(fParentsCurrentPowerFlags);
+			// initially change into the state we are already in
+			tempDesire = fControllingDriver->initialPowerStateForDomainState(fParentsCurrentPowerFlags);
+			computeDesiredState(tempDesire);
+			changeState();
+		}
+	}
+	else
+	{
+		OUR_PMLog(kPMLogControllingDriverErr2, numberOfStates, 0);
+        IODelete(powerStates, IOPMPowerState, numberOfStates);
+	}
+
+	powerDriver->release();
+}
+
+//*********************************************************************************
+// [public virtual] registerInterestedDriver
 //
 // Add the caller to our list of interested drivers and return our current
 // power state.  If we don't have a power-controlling driver yet, we will
@@ -821,72 +1143,134 @@ IOReturn IOService::registerPowerDriver ( IOService * controllingDriver, IOPMPow
 // out what the current power state of the device is.
 //*********************************************************************************
 
-IOPMPowerFlags IOService::registerInterestedDriver ( IOService * theDriver )
+IOPMPowerFlags IOService::registerInterestedDriver ( IOService * driver )
 {
-    IOPMinformee                            *newInformee;
-    IOPMPowerFlags                          futureCapability;
+	IOPMRequest *	request;
+	bool			signal;
+
+	if (!initialized || !fInterestedDrivers)
+		return IOPMNotPowerManaged;
+
+	PM_LOCK();
+	signal = (!fInsertInterestSet && !fRemoveInterestSet);
+	if (fInsertInterestSet == NULL)
+		fInsertInterestSet = OSSet::withCapacity(4);
+	if (fInsertInterestSet)
+		fInsertInterestSet->setObject(driver);
+	PM_UNLOCK();
+
+	if (signal)
+	{
+		request = acquirePMRequest( this, kIOPMRequestTypeInterestChanged );
+		if (request)
+			submitPMRequest( request );
+	}
 
-    if (theDriver == NULL ) {
-        return 0;
-    }
+	// This return value cannot be trusted, but return a value
+	// for those clients that care.
 
-    // make new driver node
-    newInformee = new IOPMinformee;
-    newInformee->initialize(theDriver);
-    // add it to list of drivers
-    priv->interestedDrivers->addToList(newInformee);
+    OUR_PMLog(kPMLogInterestedDriver, kIOPMDeviceUsable, 2);
+    return kIOPMDeviceUsable;	
+}
 
-    if ( (pm_vars->theControllingDriver == NULL) ||
-                    !(inPlane(gIOPowerPlane)) ||
-                    !(pm_vars->parentsKnowState) ) 
-    {
-        // can't tell it a state yet
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogInterestedDriver,IOPMNotPowerManaged,0);
-        return IOPMNotPowerManaged;
-    }
+//*********************************************************************************
+// [public virtual] deRegisterInterestedDriver
+//*********************************************************************************
 
-    // can we notify new driver of a change in progress?
-    switch (priv->machine_state) {
-        case kIOPM_OurChangeSetPowerState:
-        case kIOPM_OurChangeFinish:
-        case kIOPM_ParentDownSetPowerState_Delayed:
-        case kIOPM_ParentDownAcknowledgeChange_Delayed:
-        case kIOPM_ParentUpSetPowerState_Delayed:
-        case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
-            // yes, remember what we tell it
-            futureCapability = priv->head_note_capabilityFlags;
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogInterestedDriver,(unsigned long)futureCapability,1);
-            // notify it
-            add_driver_to_active_change(newInformee);
-            // and return the same thing
-            return futureCapability;
-    }
+IOReturn IOService::deRegisterInterestedDriver ( IOService * driver )
+{
+	IOPMinformeeList *	list;
+    IOPMinformee *		item;
+	IOPMRequest *		request;
+	bool				signal;
+
+	if (!initialized || !fInterestedDrivers)
+		return IOPMNotPowerManaged;
+
+	PM_LOCK();
+	signal = (!fRemoveInterestSet && !fInsertInterestSet);
+	if (fRemoveInterestSet == NULL)
+		fRemoveInterestSet = OSSet::withCapacity(4);
+	if (fRemoveInterestSet)
+	{
+		fRemoveInterestSet->setObject(driver);
+
+		list = fInterestedDrivers;
+		item = list->findItem(driver);
+		if (item && item->active)
+		{
+			item->active = false;
+		}
+		if (fDriverCallBusy)
+            PM_DEBUG("%s::deRegisterInterestedDriver() driver call busy\n", getName());
+	}
+	PM_UNLOCK();
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogInterestedDriver,
-                    (unsigned long) pm_vars->thePowerStates[pm_vars->myCurrentState].capabilityFlags,2);
+	if (signal)
+	{
+		request = acquirePMRequest( this, kIOPMRequestTypeInterestChanged );
+		if (request)
+			submitPMRequest( request );
+	}
 
-    // no, return current capability
-    return  pm_vars->thePowerStates[pm_vars->myCurrentState].capabilityFlags;	
+	return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// deRegisterInterestedDriver
+// [private] handleInterestChanged
 //
+// Handle interest added or removed.
 //*********************************************************************************
-IOReturn IOService::deRegisterInterestedDriver ( IOService * theDriver )
+
+void IOService::handleInterestChanged( IOPMRequest * request )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogRemoveDriver,0,0);
+	IOService *			driver;
+    IOPMinformee *		informee;
+	IOPMinformeeList *	list = fInterestedDrivers;
+
+	PM_LOCK();
+
+	if (fInsertInterestSet)
+	{
+		while ((driver = (IOService *) fInsertInterestSet->getAnyObject()))
+		{
+			if ((list->findItem(driver) == NULL) &&
+				(!fRemoveInterestSet ||
+				 !fRemoveInterestSet->containsObject(driver)))
+			{
+				informee = list->appendNewInformee(driver);
+			}
+			fInsertInterestSet->removeObject(driver);
+		}
+		fInsertInterestSet->release();
+		fInsertInterestSet = 0;
+	}
 
-    // remove the departing driver
-    priv->interestedDrivers->removeFromList(theDriver);
+	if (fRemoveInterestSet)
+	{
+		while ((driver = (IOService *) fRemoveInterestSet->getAnyObject()))
+		{
+			informee = list->findItem(driver);
+			if (informee)
+			{
+				if (fHeadNotePendingAcks && informee->timer)
+				{
+					informee->timer = 0;
+					fHeadNotePendingAcks--;
+				}
+				list->removeFromList(driver);
+			}
+			fRemoveInterestSet->removeObject(driver);
+		}
+		fRemoveInterestSet->release();
+		fRemoveInterestSet = 0;
+	}
 
-    return IOPMNoErr;
+	PM_UNLOCK();
 }
 
-
 //*********************************************************************************
-// acknowledgePowerChange
+// [public virtual] acknowledgePowerChange
 //
 // After we notified one of the interested drivers or a power-domain child
 // of an impending change in power, it has called to say it is now
@@ -901,104 +1285,134 @@ IOReturn IOService::deRegisterInterestedDriver ( IOService * theDriver )
 
 IOReturn IOService::acknowledgePowerChange ( IOService * whichObject )
 {
-    IOPMinformee                                *ackingObject;
-    unsigned long	                            childPower = kIOPMUnknown;
-    IOService                                   *theChild;
+	IOPMRequest * request;
+
+    if (!initialized)
+		return IOPMNotYetInitialized;
+	if (!whichObject)
+		return kIOReturnBadArgument;
+
+	request = acquirePMRequest( this, kIOPMRequestTypeAckPowerChange );
+	if (!request)
+    {
+        PM_ERROR("%s::%s no memory\n", getName(), __FUNCTION__);
+		return kIOReturnNoMemory;
+    }
+
+	whichObject->retain();
+	request->fArg0 = whichObject;
+
+	submitPMRequest( request );
+    return IOPMNoErr;
+}
+
+//*********************************************************************************
+// [private] handleAcknowledgePowerChange
+//*********************************************************************************
+
+bool IOService::handleAcknowledgePowerChange ( IOPMRequest * request )
+{
+    IOPMinformee *		informee;
+    unsigned long		childPower = kIOPMUnknown;
+    IOService *			theChild;
+	IOService *			whichObject;
+	bool				all_acked  = false;
+
+	PM_ASSERT_IN_GATE();
+	whichObject = (IOService *) request->fArg0;
+	assert(whichObject);
 
     // one of our interested drivers?
-    ackingObject =  priv->interestedDrivers->findItem(whichObject);
-    if ( ackingObject == NULL ) 
+	informee = fInterestedDrivers->findItem( whichObject );
+    if ( informee == NULL )
     {
-        if ( ! isChild(whichObject,gIOPowerPlane) ) 
+        if ( !isChild(whichObject, gIOPowerPlane) )
         {
-             pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAcknowledgeErr1,0,0);
-             //kprintf("errant driver: %s\n",whichObject->getName());
-             // no, just return
-            return IOPMNoErr;
+			OUR_PMLog(kPMLogAcknowledgeErr1, 0, 0);
+			goto no_err;
         } else {
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogChildAcknowledge,priv->head_note_pendingAcks,0);
+            OUR_PMLog(kPMLogChildAcknowledge, fHeadNotePendingAcks, 0);
         }
     } else {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogDriverAcknowledge,priv->head_note_pendingAcks,0);
-    }
- 
-    if (! acquire_lock() ) 
-    {
-        return IOPMNoErr;
+        OUR_PMLog(kPMLogDriverAcknowledge, fHeadNotePendingAcks, 0);
     }
- 
-    if (priv->head_note_pendingAcks != 0 ) 
+
+    if ( fHeadNotePendingAcks != 0 )
     {
+        assert(fPowerStates != NULL);
+
          // yes, make sure we're expecting acks
-        if ( ackingObject != NULL ) 
+        if ( informee != NULL )
         {
             // it's an interested driver
             // make sure we're expecting this ack
-            if ( ackingObject->timer != 0 ) 
+            if ( informee->timer != 0 )
             {
-                // mark it acked
-                ackingObject->timer = 0;
-                // that's one fewer to worry about
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                // is that the last?
-                if ( priv->head_note_pendingAcks == 0 ) 
+#if LOG_SETPOWER_TIMES
+                if (informee->timer > 0)
                 {
-                    // yes, stop the timer
-                    stop_ack_timer();
-                    IOUnlock(priv->our_lock);
-                    // and now we can continue
-                    all_acked();
-                    return IOPMNoErr;
+                    uint64_t nsec = computeTimeDeltaNS(&informee->startTime);
+                    if (nsec > LOG_SETPOWER_TIMES)
+                        PM_DEBUG("%s::powerState%sChangeTo(%p, %s, %lu -> %lu) async took %d ms\n",
+                            informee->whatObject->getName(),
+                            (fDriverCallReason == kDriverCallInformPreChange) ? "Will" : "Did",
+                            informee->whatObject,
+                            fName, fCurrentPowerState, fHeadNoteState, NS_TO_MS(nsec));
                 }
+#endif
+                // mark it acked
+                informee->timer = 0;
+                // that's one fewer to worry about
+                fHeadNotePendingAcks--;
             } else {
                 // this driver has already acked
-                pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAcknowledgeErr2,0,0);
-                //kprintf("errant driver: %s\n",whichObject->getName());
+                OUR_PMLog(kPMLogAcknowledgeErr2, 0, 0);
             }
         } else {
             // it's a child
             // make sure we're expecting this ack
-            if ( ((IOPowerConnection *)whichObject)->getAwaitingAck() ) 
+            if ( ((IOPowerConnection *)whichObject)->getAwaitingAck() )
             {
                 // that's one fewer to worry about
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+                fHeadNotePendingAcks--;
                 ((IOPowerConnection *)whichObject)->setAwaitingAck(false);
                 theChild = (IOService *)whichObject->copyChildEntry(gIOPowerPlane);
-                if ( theChild ) 
+                if ( theChild )
                 {
                     childPower = theChild->currentPowerConsumption();
                     theChild->release();
                 }
-                if ( childPower == kIOPMUnknown ) 
+                if ( childPower == kIOPMUnknown )
                 {
-                    pm_vars->thePowerStates[priv->head_note_state].staticPower = kIOPMUnknown;
+                    fPowerStates[fHeadNoteState].staticPower = kIOPMUnknown;
                 } else {
-                    if ( pm_vars->thePowerStates[priv->head_note_state].staticPower != kIOPMUnknown ) 
+                    if ( fPowerStates[fHeadNoteState].staticPower != kIOPMUnknown )
                     {
-                        pm_vars->thePowerStates[priv->head_note_state].staticPower += childPower;
+                        fPowerStates[fHeadNoteState].staticPower += childPower;
                     }
                 }
-                // is that the last?
-                if ( priv->head_note_pendingAcks == 0 ) {
-                    // yes, stop the timer
-                    stop_ack_timer();
-                    IOUnlock(priv->our_lock);
-                    // and now we can continue
-                    all_acked();
-                    return IOPMNoErr;
-                }
             }
 	    }
+
+		if ( fHeadNotePendingAcks == 0 ) {
+			// yes, stop the timer
+			stop_ack_timer();
+			// and now we can continue
+			all_acked = true;
+		}
     } else {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAcknowledgeErr3,0,0);	// not expecting anybody to ack
-        //kprintf("errant driver: %s\n",whichObject->getName());
+        OUR_PMLog(kPMLogAcknowledgeErr3, 0, 0);	// not expecting anybody to ack
     }
-    IOUnlock(priv->our_lock);
-    return IOPMNoErr;
+
+no_err:
+	if (whichObject)
+		whichObject->release();
+
+    return all_acked;
 }
 
 //*********************************************************************************
-// acknowledgeSetPowerState
+// [public virtual] acknowledgeSetPowerState
 //
 // After we instructed our controlling driver to change power states,
 // it has called to say it has finished doing so.
@@ -1007,60 +1421,49 @@ IOReturn IOService::acknowledgePowerChange ( IOService * whichObject )
 
 IOReturn IOService::acknowledgeSetPowerState ( void )
 {
-    if (!acquire_lock()) 
-        return IOPMNoErr;
+	IOPMRequest * request;
 
-    IOReturn timer = priv->driver_timer;
-    if ( timer == -1 ) {
-        // driver is acking instead of using return code
-	OUR_PMLog(kPMLogDriverAcknowledgeSet, (UInt32) this, timer);
-        priv->driver_timer = 0;
-    }
-    else if ( timer > 0 ) {	// are we expecting this?
-	// yes, stop the timer
-	stop_ack_timer();
-	priv->driver_timer = 0;
-	OUR_PMLog(kPMLogDriverAcknowledgeSet, (UInt32) this, timer);
-	IOUnlock(priv->our_lock);
-	driver_acked();
-	return IOPMNoErr;
-    } else {
-	// not expecting this
-	OUR_PMLog(kPMLogAcknowledgeErr4, (UInt32) this, 0);
-    }
+    if (!initialized)
+		return IOPMNotYetInitialized;
 
-    IOUnlock(priv->our_lock);
-    return IOPMNoErr;
-}
+	request = acquirePMRequest( this, kIOPMRequestTypeAckSetPowerState );
+	if (!request)
+	{
+        PM_ERROR("%s::%s no memory\n", getName(), __FUNCTION__);
+		return kIOReturnNoMemory;
+	}
 
+	submitPMRequest( request );
+	return kIOReturnSuccess;
+}
 
 //*********************************************************************************
-// driver_acked
+// [private] adjustPowerState
 //
-// Either the controlling driver has called acknowledgeSetPowerState
-// or the acknowledgement timer has expired while waiting for that.
-// We carry on processing the current change note.
+// Child has signaled a change - child changed it's desire, new child added,
+// existing child removed. Adjust our power state accordingly.
 //*********************************************************************************
 
-void IOService::driver_acked ( void )
+void IOService::adjustPowerState( void )
 {
-
-    switch (priv->machine_state) {
-        case kIOPM_OurChangeWaitForPowerSettle:
-            OurChangeWaitForPowerSettle();
-            break;
-        case kIOPM_ParentDownWaitForPowerSettle_Delayed:
-            ParentDownWaitForPowerSettle_Delayed();
-            break;
-        case kIOPM_ParentUpWaitForSettleTime_Delayed:
-            ParentUpWaitForSettleTime_Delayed();
-            break;
-    }
+	PM_ASSERT_IN_GATE();
+	if (inPlane(gIOPowerPlane))
+	{
+		rebuildChildClampBits();
+		computeDesiredState();
+		if ( fControllingDriver && fParentsKnowState )
+			changeState();
+	}
+	else
+	{
+		PM_DEBUG("[%s] %s: not in power tree\n", getName(), __FUNCTION__);
+		return;
+	}
+	fWillAdjustPowerState = false;
 }
 
-
 //*********************************************************************************
-// powerDomainWillChangeTo
+// [public deprecated] powerDomainWillChangeTo
 //
 // Called by the power-hierarchy parent notifying of a new power state
 // in the power domain.
@@ -1069,81 +1472,113 @@ void IOService::driver_acked ( void )
 // kind of change is occuring in the domain.
 //*********************************************************************************
 
-IOReturn IOService::powerDomainWillChangeTo ( IOPMPowerFlags newPowerStateFlags, IOPowerConnection * whichParent )
+IOReturn IOService::powerDomainWillChangeTo (
+	IOPMPowerFlags		newPowerFlags,
+	IOPowerConnection *	whichParent )
 {
-    OSIterator                      *iter;
-    OSObject                        *next;
-    IOPowerConnection               *connection;
-    unsigned long                   newStateNumber;
-    IOPMPowerFlags                  combinedPowerFlags;
+	assert(false);
+	return kIOReturnUnsupported;
+}
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogWillChange,(unsigned long)newPowerStateFlags,0);
+//*********************************************************************************
+// [private] handlePowerDomainWillChangeTo
+//*********************************************************************************
 
-    if ( ! inPlane(gIOPowerPlane) ) 
-    {
-        // somebody goofed
-        return IOPMAckImplied;
-    }
+void IOService::handlePowerDomainWillChangeTo ( IOPMRequest * request )
+{
+	IOPMPowerFlags		newPowerFlags = (IOPMPowerFlags)      request->fArg0;
+	IOPowerConnection *	whichParent   = (IOPowerConnection *) request->fArg1;
+	bool				powerWillDrop = (bool)                request->fArg2;
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+    unsigned long		newPowerState;
+    IOPMPowerFlags		combinedPowerFlags;
+	bool				savedParentsKnowState;
+	IOReturn			result = IOPMAckImplied;
+
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog(kPMLogWillChange, newPowerFlags, 0);
+
+	if (!inPlane(gIOPowerPlane))
+	{
+		PM_DEBUG("[%s] %s: not in power tree\n", getName(), __FUNCTION__);
+		return;
+	}
 
-    IOLockLock(pm_vars->parentLock);
-    
-    if ( (pm_vars->PMworkloop == NULL) || (pm_vars->PMcommandGate == NULL) ) 
-    {
-        // we have a path to the root
-        getPMworkloop();
-        // so find out the workloop
-        if ( pm_vars->PMworkloop != NULL ) 
-        {
-            // and make our command gate
-            if ( pm_vars->PMcommandGate == NULL ) 
-            {
-                pm_vars->PMcommandGate = IOCommandGate::commandGate((OSObject *)this);
-                if ( pm_vars->PMcommandGate != NULL ) 
-                {
-                    pm_vars->PMworkloop->addEventSource(pm_vars->PMcommandGate);
-                }
-            }
-        }
-    }
-    
-    IOLockUnlock(pm_vars->parentLock);
+	savedParentsKnowState = fParentsKnowState;
 
-    // combine parents' power states
-    // to determine our maximum state within the new power domain
-    combinedPowerFlags = 0;
-    
-    iter = getParentIterator(gIOPowerPlane);
+    // Combine parents' power flags to determine our maximum state
+	// within the new power domain
+	combinedPowerFlags = 0;
 
-    if ( iter ) 
+    iter = getParentIterator(gIOPowerPlane);
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
-                if ( connection == whichParent ){
-                    combinedPowerFlags |= newPowerStateFlags;
-                } else {
+                if ( connection == whichParent )
+                    combinedPowerFlags |= newPowerFlags;
+                else
                     combinedPowerFlags |= connection->parentCurrentPowerFlags();
-                }
             }
         }
         iter->release();
     }
-    
-    if  ( pm_vars->theControllingDriver == NULL ) 
+
+    if  ( fControllingDriver )
     {
-        // we can't take any more action
-        return IOPMAckImplied;
-    }
-    newStateNumber = pm_vars->theControllingDriver->maxCapabilityForDomainState(combinedPowerFlags);
-    // make the change
-    return enqueuePowerChange(IOPMParentInitiated | IOPMDomainWillChange,
-                            newStateNumber,combinedPowerFlags,whichParent,newPowerStateFlags);
-}
+		newPowerState = fControllingDriver->maxCapabilityForDomainState(
+							combinedPowerFlags);
+
+		result = enqueuePowerChange(
+                 /* flags        */	IOPMParentInitiated | IOPMDomainWillChange,
+                 /* power state  */	newPowerState,
+				 /* domain state */	combinedPowerFlags,
+				 /* connection   */	whichParent,
+				 /* parent state */	newPowerFlags);
+	}
+
+	// If parent is dropping power, immediately update the parent's
+	// capability flags. Any future merging of parent(s) combined
+	// power flags should account for this power drop.
+
+	if (powerWillDrop)
+	{
+		setParentInfo(newPowerFlags, whichParent, true);
+	}
 
+	// Parent is expecting an ACK from us. If we did not embark on a state
+	// transition, when enqueuePowerChang() returns IOPMAckImplied. We are
+	// still required to issue an ACK to our parent.
+
+	if (IOPMAckImplied == result)
+	{
+		IOService * parent;
+		parent = (IOService *) whichParent->copyParentEntry(gIOPowerPlane);
+		assert(parent);
+		if ( parent )
+		{
+			parent->acknowledgePowerChange( whichParent );
+			parent->release();
+		}
+	}
+
+	// If the parent registers it's power driver late, then this is the
+	// first opportunity to tell our parent about our desire. 
+
+	if (!savedParentsKnowState && fParentsKnowState)
+	{
+		PM_TRACE("[%s] powerDomainWillChangeTo: parentsKnowState = true\n",
+			getName());
+		ask_parent( fDesiredPowerState );
+	}
+}
 
 //*********************************************************************************
-// powerDomainDidChangeTo
+// [public deprecated] powerDomainDidChangeTo
 //
 // Called by the power-hierarchy parent after the power state of the power domain
 // has settled at a new level.
@@ -1152,198 +1587,281 @@ IOReturn IOService::powerDomainWillChangeTo ( IOPMPowerFlags newPowerStateFlags,
 // kind of change is occuring in the domain.
 //*********************************************************************************
 
-IOReturn IOService::powerDomainDidChangeTo ( IOPMPowerFlags newPowerStateFlags, IOPowerConnection * whichParent )
+IOReturn IOService::powerDomainDidChangeTo (
+	IOPMPowerFlags		newPowerFlags,
+	IOPowerConnection *	whichParent )
 {
-    unsigned long newStateNumber;
+	assert(false);
+	return kIOReturnUnsupported;
+}
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogDidChange,newPowerStateFlags,0);
+//*********************************************************************************
+// [private] handlePowerDomainDidChangeTo
+//*********************************************************************************
 
-    setParentInfo(newPowerStateFlags,whichParent);
+void IOService::handlePowerDomainDidChangeTo ( IOPMRequest * request )
+{
+	IOPMPowerFlags		newPowerFlags = (IOPMPowerFlags)      request->fArg0;
+	IOPowerConnection *	whichParent   = (IOPowerConnection *) request->fArg1;
+    unsigned long		newPowerState;
+	bool				savedParentsKnowState;
+	IOReturn			result = IOPMAckImplied;
+
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog(kPMLogDidChange, newPowerFlags, 0);
+
+	if (!inPlane(gIOPowerPlane))
+	{
+		PM_DEBUG("[%s] %s: not in power tree\n", getName(), __FUNCTION__);
+		return;
+	}
 
-    if ( pm_vars->theControllingDriver == NULL ) {
-        return IOPMAckImplied;
-    }
+	savedParentsKnowState = fParentsKnowState;
 
-    newStateNumber = pm_vars->theControllingDriver->maxCapabilityForDomainState(pm_vars->parentsCurrentPowerFlags);
-    // tell interested parties about it
-    return enqueuePowerChange(IOPMParentInitiated | IOPMDomainDidChange,
-                    newStateNumber,pm_vars->parentsCurrentPowerFlags,whichParent,0);
-}
+    setParentInfo(newPowerFlags, whichParent, true);
+
+    if ( fControllingDriver )
+	{
+		newPowerState = fControllingDriver->maxCapabilityForDomainState(
+							fParentsCurrentPowerFlags);
+
+		result = enqueuePowerChange(
+				 /* flags        */	IOPMParentInitiated | IOPMDomainDidChange,
+                 /* power state  */	newPowerState,
+				 /* domain state */	fParentsCurrentPowerFlags,
+				 /* connection   */	whichParent,
+				 /* parent state */	0);
+	}
 
+	// Parent is expecting an ACK from us. If we did not embark on a state
+	// transition, when enqueuePowerChang() returns IOPMAckImplied. We are
+	// still required to issue an ACK to our parent.
+
+	if (IOPMAckImplied == result)
+	{
+		IOService * parent;
+		parent = (IOService *) whichParent->copyParentEntry(gIOPowerPlane);
+		assert(parent);
+		if ( parent )
+		{
+			parent->acknowledgePowerChange( whichParent );
+			parent->release();
+		}
+	}
+
+	// If the parent registers it's power driver late, then this is the
+	// first opportunity to tell our parent about our desire. 
+
+	if (!savedParentsKnowState && fParentsKnowState)
+	{
+		PM_TRACE("[%s] powerDomainDidChangeTo: parentsKnowState = true\n",
+			getName());
+		ask_parent( fDesiredPowerState );
+	}
+}
 
 //*********************************************************************************
-// setParentInfo
+// [private] setParentInfo
 //
 // Set our connection data for one specific parent, and then combine all the parent
 // data together.
 //*********************************************************************************
-
-void IOService::setParentInfo ( IOPMPowerFlags newPowerStateFlags, IOPowerConnection * whichParent )
+ 
+void IOService::setParentInfo (
+	IOPMPowerFlags		newPowerFlags,
+	IOPowerConnection * whichParent,
+	bool				knowsState )
 {
-    OSIterator                      *iter;
-    OSObject                        *next;
-    IOPowerConnection               *connection;
-    
-    // set our connection data
-    whichParent->setParentCurrentPowerFlags(newPowerStateFlags);
-    whichParent->setParentKnowsState(true);
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	conn;
 
-    IOLockLock(pm_vars->parentLock);
+	PM_ASSERT_IN_GATE();
+
+    // set our connection data
+    whichParent->setParentCurrentPowerFlags(newPowerFlags);
+    whichParent->setParentKnowsState(knowsState);
     
     // recompute our parent info
-    pm_vars->parentsCurrentPowerFlags = 0;
-    pm_vars->parentsKnowState = true;
+    fParentsCurrentPowerFlags = 0;
+    fParentsKnowState = true;
 
     iter = getParentIterator(gIOPowerPlane);
-
-    if ( iter ) 
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (conn = OSDynamicCast(IOPowerConnection, next)) )
             {
-                pm_vars->parentsKnowState &= connection->parentKnowsState();
-                pm_vars->parentsCurrentPowerFlags |= connection->parentCurrentPowerFlags();
+                fParentsKnowState &= conn->parentKnowsState();
+                fParentsCurrentPowerFlags |= conn->parentCurrentPowerFlags();
             }
         }
         iter->release();
     }
-    IOLockUnlock(pm_vars->parentLock);
 }
 
 //*********************************************************************************
-// rebuildChildClampBits
+// [private] rebuildChildClampBits
 //
 // The ChildClamp bits (kIOPMChildClamp & kIOPMChildClamp2) in our capabilityFlags
-// indicate that one of our children (or grandchildren or great-grandchildren or ...)
-// doesn't support idle or system sleep in its current state. Since we don't track the
-// origin of each bit, every time any child changes state we have to clear these bits 
-// and rebuild them.
+// indicate that one of our children (or grandchildren or great-grandchildren ...)
+// doesn't support idle or system sleep in its current state. Since we don't track
+// the origin of each bit, every time any child changes state we have to clear
+// these bits and rebuild them.
 //*********************************************************************************
 
-void IOService::rebuildChildClampBits(void)
+void IOService::rebuildChildClampBits ( void )
 {
-    unsigned long                       i;
-    OSIterator                          *iter;
-    OSObject                            *next;
-    IOPowerConnection                   *connection;
-    
-    
-    // A child's desires has changed.  We need to rebuild the child-clamp bits in our
-    // power state array.  Start by clearing the bits in each power state.
+    unsigned long		i;
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+	unsigned long		powerState;
+
+    // A child's desires has changed. We need to rebuild the child-clamp bits in
+	// our power state array. Start by clearing the bits in each power state.
     
-    for ( i = 0; i < pm_vars->theNumberOfPowerStates; i++ ) 
+    for ( i = 0; i < fNumberOfPowerStates; i++ )
     {
-        pm_vars->thePowerStates[i].capabilityFlags &= ~(kIOPMChildClamp | kIOPMChildClamp2);
+        fPowerStates[i].capabilityFlags &= ~(kIOPMChildClamp | kIOPMChildClamp2);
     }
 
-    // Now loop through the children.  When we encounter the calling child, save
-    // the computed state as this child's desire.  And while we're at it, set the ChildClamp bits
-    // in any of our states that some child has requested with clamp on.
+    // Loop through the children. When we encounter the calling child, save the
+	// computed state as this child's desire. And set the ChildClamp bits in any
+    // of our states that some child has clamp on.
 
     iter = getChildIterator(gIOPowerPlane);
-
-    if ( iter ) 
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
-                if ( connection->getPreventIdleSleepFlag() )
-                    pm_vars->thePowerStates[connection->getDesiredDomainState()].capabilityFlags |= kIOPMChildClamp;
-                if ( connection->getPreventSystemSleepFlag() )
-                    pm_vars->thePowerStates[connection->getDesiredDomainState()].capabilityFlags |= kIOPMChildClamp2;
+				if (connection->getReadyFlag() == false)
+				{
+					PM_CONNECT("[%s] %s: connection not ready\n",
+						getName(), __FUNCTION__);
+					continue;
+				}
+
+				powerState = connection->getDesiredDomainState();
+                if (powerState < fNumberOfPowerStates)
+                {
+                    if ( connection->getPreventIdleSleepFlag() )
+                        fPowerStates[powerState].capabilityFlags |= kIOPMChildClamp;
+                    if ( connection->getPreventSystemSleepFlag() )
+                        fPowerStates[powerState].capabilityFlags |= kIOPMChildClamp2;
+                }
             }
         }
         iter->release();
     }
-
 }
 
-
 //*********************************************************************************
-// requestPowerDomainState
+// [public virtual] requestPowerDomainState
 //
-// The kIOPMPreventIdleSleep and/or kIOPMPreventIdleSleep bits may be be set in the parameter.
-// It is not considered part of the state specification.
+// The child of a power domain calls it parent here to request power of a certain
+// character.
 //*********************************************************************************
-IOReturn IOService::requestPowerDomainState ( IOPMPowerFlags desiredState, IOPowerConnection * whichChild, unsigned long specification )
+
+IOReturn IOService::requestPowerDomainState (
+	IOPMPowerFlags		desiredState,
+	IOPowerConnection *	whichChild,
+	unsigned long		specification )
 {
-    unsigned long                           i;
-    unsigned long                           computedState;
-    unsigned long                           theDesiredState = desiredState & ~(kIOPMPreventIdleSleep | kIOPMPreventSystemSleep);
-    OSIterator                              *iter;
-    OSObject                                *next;
-    IOPowerConnection                       *connection;
+    unsigned long		i;
+    unsigned long		computedState;
+    unsigned long		theDesiredState;
+	IOService *			child;
+
+    if (!initialized)
+		return IOPMNotYetInitialized;
+
+	if (gIOPMWorkLoop->onThread() == false)
+	{
+		PM_DEBUG("[%s] called requestPowerDomainState\n", getName());
+		return kIOReturnSuccess;
+	}
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogRequestDomain,
-                                (unsigned long)desiredState,(unsigned long)specification);
+	theDesiredState = desiredState & ~(kIOPMPreventIdleSleep | kIOPMPreventSystemSleep);
 
-    if ( pm_vars->theControllingDriver == NULL) 
-    {
+    OUR_PMLog(kPMLogRequestDomain, desiredState, specification);
+
+	if (!isChild(whichChild, gIOPowerPlane))
+		return kIOReturnNotAttached;
+
+    if (fControllingDriver == NULL || !fPowerStates)
         return IOPMNotYetInitialized;
-    }
+
+	child = (IOService *) whichChild->getChildEntry(gIOPowerPlane);
+	assert(child);
 
     switch (specification) {
         case IOPMLowestState:
             i = 0;
-            while ( i < pm_vars->theNumberOfPowerStates ) 
+            while ( i < fNumberOfPowerStates )
             {
-                if ( ( pm_vars->thePowerStates[i].outputPowerCharacter & theDesiredState) == (theDesiredState & pm_vars->myCharacterFlags) ) 
+                if ( ( fPowerStates[i].outputPowerCharacter & theDesiredState) ==
+					 (theDesiredState & fOutputPowerCharacterFlags) )
                 {
                     break;
                 }
                 i++;
             }
-            if ( i >= pm_vars->theNumberOfPowerStates ) 
+            if ( i >= fNumberOfPowerStates )
             {
                 return IOPMNoSuchState;
             }
             break;
 
         case IOPMNextLowerState:
-            i = pm_vars->myCurrentState - 1;
-            while ( (int) i >= 0 ) 
+            i = fCurrentPowerState - 1;
+            while ( (int) i >= 0 )
             {
-                if ( ( pm_vars->thePowerStates[i].outputPowerCharacter & theDesiredState) == (theDesiredState & pm_vars->myCharacterFlags) ) 
+                if ( ( fPowerStates[i].outputPowerCharacter & theDesiredState) ==
+					 (theDesiredState & fOutputPowerCharacterFlags) )
                 {
                     break;
                 }
                 i--;
             }
-            if ( (int) i < 0 ) 
+            if ( (int) i < 0 )
             {
                 return IOPMNoSuchState;
             }
             break;
 
         case IOPMHighestState:
-            i = pm_vars->theNumberOfPowerStates;
-            while ( (int) i >= 0 ) 
+            i = fNumberOfPowerStates;
+            while ( (int) i >= 0 )
             {
                 i--;
-                if ( ( pm_vars->thePowerStates[i].outputPowerCharacter & theDesiredState) == (theDesiredState & pm_vars->myCharacterFlags) ) 
+                if ( ( fPowerStates[i].outputPowerCharacter & theDesiredState) ==
+					 (theDesiredState & fOutputPowerCharacterFlags) )
                 {
                     break;
                 }
             }
-            if ( (int) i < 0 ) 
+            if ( (int) i < 0 )
             {
                 return IOPMNoSuchState;
             }
             break;
 
         case IOPMNextHigherState:
-            i = pm_vars->myCurrentState + 1;
-            while ( i < pm_vars->theNumberOfPowerStates ) 
+            i = fCurrentPowerState + 1;
+            while ( i < fNumberOfPowerStates )
             {
-                if ( ( pm_vars->thePowerStates[i].outputPowerCharacter & theDesiredState) == (theDesiredState & pm_vars->myCharacterFlags) ) 
+                if ( ( fPowerStates[i].outputPowerCharacter & theDesiredState) ==
+					 (theDesiredState & fOutputPowerCharacterFlags) )
                 {
                     break;
                 }
                 i++;
             }
-            if ( i == pm_vars->theNumberOfPowerStates ) 
+            if ( i == fNumberOfPowerStates )
             {
                 return IOPMNoSuchState;
             }
@@ -1354,59 +1872,47 @@ IOReturn IOService::requestPowerDomainState ( IOPMPowerFlags desiredState, IOPow
     }
 
     computedState = i;
-    
-    IOLockLock(pm_vars->childLock);
 
-    // Now loop through the children.  When we encounter the calling child, save
-    // the computed state as this child's desire.
-    iter = getChildIterator(gIOPowerPlane);
+	// Clamp removed on the initial power request from a new child.
 
-    if ( iter ) 
-    {
-        while ( (next = iter->getNextObject()) ) 
-        {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
-            {
-                if ( connection == whichChild ) 
-                {
-                    connection->setDesiredDomainState(computedState);
-                    connection->setPreventIdleSleepFlag(desiredState & kIOPMPreventIdleSleep);
-                    connection->setPreventSystemSleepFlag(desiredState & kIOPMPreventSystemSleep);
-                    connection->setChildHasRequestedPower();
-                }
-            }
-        }
-        iter->release();
-    }
+	if (fClampOn && !whichChild->childHasRequestedPower())
+	{
+		PM_TRACE("[%s] %p power clamp removed (child = %p)\n",
+			getName(), this, whichChild);
+		fClampOn = false;
+		fDeviceDesire = 0;
+	}
 
-    // Since a child's power requirements may have changed, clear and rebuild 
-    // kIOPMChildClamp and kIOPMChildClamp2 (idle and system sleep clamps)
-    rebuildChildClampBits();
-        
-    IOLockUnlock(pm_vars->childLock);
-    
-    // this may be different now
-    computeDesiredState();
+	// Record the child's desires on the connection.
 
-    if ( inPlane(gIOPowerPlane) &&
-         (pm_vars->parentsKnowState) ) {
-         // change state if all children can now tolerate lower power
-        changeState();
-    }
-   
-    // are we clamped on, waiting for this child?
-    if ( priv->clampOn ) {
-        // yes, remove the clamp
-        priv->clampOn = false;
-        changePowerStateToPriv(0);
-    }
-   
-   return IOPMNoErr;
-}
+	whichChild->setDesiredDomainState( computedState );
+	whichChild->setPreventIdleSleepFlag( desiredState & kIOPMPreventIdleSleep );
+	whichChild->setPreventSystemSleepFlag( desiredState & kIOPMPreventSystemSleep );
+	whichChild->setChildHasRequestedPower();
+
+	if (whichChild->getReadyFlag() == false)
+		return IOPMNoErr;
+
+	// Issue a ping for us to re-evaluate all children desires and
+	// possibly change power state.
+
+	if (!fWillAdjustPowerState && !fDeviceOverrides)
+	{
+		IOPMRequest * childRequest;
+
+		childRequest = acquirePMRequest( this, kIOPMRequestTypeAdjustPowerState );
+		if (childRequest)
+		{
+			submitPMRequest( childRequest );
+			fWillAdjustPowerState = true;
+		}
+	}
 
+	return IOPMNoErr;
+}
 
 //*********************************************************************************
-// temporaryPowerClampOn
+// [public virtual] temporaryPowerClampOn
 //
 // A power domain wants to clamp its power on till it has children which
 // will thendetermine the power domain state.
@@ -1416,14 +1922,21 @@ IOReturn IOService::requestPowerDomainState ( IOPMPowerFlags desiredState, IOPow
 
 IOReturn IOService::temporaryPowerClampOn ( void )
 {
-    priv->clampOn = true;
-    makeUsable();
+	IOPMRequest * request;
+
+    if (!initialized)
+		return IOPMNotYetInitialized;
+
+	request = acquirePMRequest( this, kIOPMRequestTypeTemporaryPowerClamp );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	submitPMRequest( request );
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// makeUsable
+// [public virtual] makeUsable
 //
 // Some client of our device is asking that we become usable.  Although
 // this has not come from a subclassed device object, treat it exactly
@@ -1436,75 +1949,59 @@ IOReturn IOService::temporaryPowerClampOn ( void )
 
 IOReturn IOService::makeUsable ( void )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogMakeUsable,0,0);
+	IOPMRequest * request;
 
-    if ( pm_vars->theControllingDriver == NULL ) 
-    {
-        priv->need_to_become_usable = true;
-        return IOPMNoErr;
-    }
-    priv->deviceDesire = pm_vars->theNumberOfPowerStates - 1;
-    computeDesiredState();
-    if ( inPlane(gIOPowerPlane) && (pm_vars->parentsKnowState) ) 
-    {
-        return changeState();
-    }
+    if (!initialized)
+		return IOPMNotYetInitialized;
+
+    OUR_PMLog(kPMLogMakeUsable, 0, 0);
+
+	request = acquirePMRequest( this, kIOPMRequestTypeMakeUsable );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	submitPMRequest( request );
     return IOPMNoErr;
 }
 
-//******************************************************************************
-// temporaryMakeUsable
+//*********************************************************************************
+// [private] handleMakeUsable
 //
-// Private function, called by IOService::addPowerChild to ensure that the
-// device is temporarily in a usable power state so that attached power  
-// children may properly initialize.
-//******************************************************************************
+// Handle a request to become usable.
+//*********************************************************************************
 
-IOReturn IOService::temporaryMakeUsable ( void )
+void IOService::handleMakeUsable ( IOPMRequest * request )
 {
-    IOReturn        ret = kIOReturnSuccess;
-    unsigned long   tempDesire;
-    
-    pm_vars->thePlatform->PMLog( pm_vars->ourName,
-                                 PMlogMakeUsable,
-                                 PMlogMakeUsable,
-                                 priv->deviceDesire);
-
-    if ( pm_vars->theControllingDriver == NULL ) 
-    {
-        priv->need_to_become_usable = true;
-        return IOPMNoErr;
-    }
-    tempDesire = priv->deviceDesire;
-    priv->deviceDesire = pm_vars->theNumberOfPowerStates - 1;
-    computeDesiredState();
-    if ( inPlane(gIOPowerPlane) && (pm_vars->parentsKnowState) ) 
-    {
-        ret = changeState();
+	PM_ASSERT_IN_GATE();
+    if ( fControllingDriver )
+    {
+		fDeviceDesire = fNumberOfPowerStates - 1;
+		computeDesiredState();
+		if ( inPlane(gIOPowerPlane) && fParentsKnowState )
+		{
+			changeState();
+		}
+	}
+	else
+	{
+        fNeedToBecomeUsable = true;
     }
-    priv->deviceDesire = tempDesire;
-    return ret;
 }
 
-
 //*********************************************************************************
-// currentCapability
-//
+// [public virtual] currentCapability
 //*********************************************************************************
 
 IOPMPowerFlags IOService::currentCapability ( void )
 {
-    if ( pm_vars->theControllingDriver == NULL ) 
-    {
-        return 0;
-    } else {
-        return   pm_vars->thePowerStates[pm_vars->myCurrentState].capabilityFlags;
-    }
-}
+	if (!initialized)
+		return IOPMNotPowerManaged;
 
+    return fCurrentCapabilityFlags;
+}
 
 //*********************************************************************************
-// changePowerStateTo
+// [public virtual] changePowerStateTo
 //
 // For some reason, our power-controlling driver has decided it needs to change
 // power state.  We enqueue the power change so that appropriate parties
@@ -1513,24 +2010,65 @@ IOPMPowerFlags IOService::currentCapability ( void )
 
 IOReturn IOService::changePowerStateTo ( unsigned long ordinal )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogChangeStateTo,ordinal,0);
+	IOPMRequest * request;
 
-    if ( ordinal >= pm_vars->theNumberOfPowerStates ) 
-    {
-        return IOPMParameterError;
-    }
-    priv->driverDesire = ordinal;
-    computeDesiredState();
-    if ( inPlane(gIOPowerPlane) && (pm_vars->parentsKnowState) ) 
-    {
-        return changeState();
-    }
+	if (!initialized)
+		return IOPMNotYetInitialized;
+
+    OUR_PMLog(kPMLogChangeStateTo, ordinal, 0);
+
+	request = acquirePMRequest( this, kIOPMRequestTypeChangePowerStateTo );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	request->fArg0 = (void *) ordinal;
+	request->fArg1 = (void *) false;
 
+	// Avoid needless downwards power transitions by clamping power in
+	// computeDesiredState() until the delayed request is processed.
+
+	if (gIOPMWorkLoop->inGate())
+	{
+		fTempClampPowerState = max(fTempClampPowerState, ordinal);
+		fTempClampCount++;
+		request->fArg1 = (void *) true;
+	}
+
+	submitPMRequest( request );
     return IOPMNoErr;
 }
 
 //*********************************************************************************
-// changePowerStateToPriv
+// [private] handleChangePowerStateTo
+//*********************************************************************************
+
+void IOService::handleChangePowerStateTo ( IOPMRequest * request )
+{
+	unsigned long ordinal = (unsigned long) request->fArg0;
+
+	PM_ASSERT_IN_GATE();
+	if (request->fArg1)
+	{
+		assert(fTempClampCount != 0);
+		if (fTempClampCount)
+			fTempClampCount--;
+		if (!fTempClampCount)
+			fTempClampPowerState = 0;
+	}
+
+	if ( fControllingDriver && (ordinal < fNumberOfPowerStates))
+    {
+		fDriverDesire = ordinal;
+		computeDesiredState();
+		if ( inPlane(gIOPowerPlane) && fParentsKnowState )
+		{
+			changeState();
+		}
+	}
+}
+
+//*********************************************************************************
+// [public virtual] changePowerStateToPriv
 //
 // For some reason, a subclassed device object has decided it needs to change
 // power state.  We enqueue the power change so that appropriate parties
@@ -1539,76 +2077,157 @@ IOReturn IOService::changePowerStateTo ( unsigned long ordinal )
 
 IOReturn IOService::changePowerStateToPriv ( unsigned long ordinal )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogChangeStateToPriv,ordinal,0);
+	IOPMRequest * request;
 
-    if ( pm_vars->theControllingDriver == NULL) 
-    {
-        return IOPMNotYetInitialized;
-    }
-    if ( ordinal >= pm_vars->theNumberOfPowerStates ) 
-    {
-        return IOPMParameterError;
-    }
-    priv->deviceDesire = ordinal;
-    computeDesiredState();
-    if ( inPlane(gIOPowerPlane) && (pm_vars->parentsKnowState) ) 
-    {
-        return changeState();
-    }
+	if (!initialized)
+		return IOPMNotYetInitialized;
+
+	request = acquirePMRequest( this, kIOPMRequestTypeChangePowerStateToPriv );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	request->fArg0 = (void *) ordinal;
+	request->fArg1 = (void *) false;
+
+	// Avoid needless downwards power transitions by clamping power in
+	// computeDesiredState() until the delayed request is processed.
+
+	if (gIOPMWorkLoop->inGate())
+	{
+		fTempClampPowerState = max(fTempClampPowerState, ordinal);
+		fTempClampCount++;
+		request->fArg1 = (void *) true;
+	}
 
+	submitPMRequest( request );
     return IOPMNoErr;
 }
 
+//*********************************************************************************
+// [private] handleChangePowerStateToPriv
+//*********************************************************************************
+
+void IOService::handleChangePowerStateToPriv ( IOPMRequest * request )
+{
+	unsigned long ordinal = (unsigned long) request->fArg0;
+
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog(kPMLogChangeStateToPriv, ordinal, 0);
+	if (request->fArg1)
+	{
+		assert(fTempClampCount != 0);
+		if (fTempClampCount)
+			fTempClampCount--;
+		if (!fTempClampCount)
+			fTempClampPowerState = 0;
+	}
+
+	if ( fControllingDriver && (ordinal < fNumberOfPowerStates))
+	{
+		fDeviceDesire = ordinal;
+		computeDesiredState();
+		if ( inPlane(gIOPowerPlane) && fParentsKnowState )
+		{
+			changeState();
+		}
+	}
+}
 
 //*********************************************************************************
-// computeDesiredState
-//
+// [private] computeDesiredState
 //*********************************************************************************
 
-void IOService::computeDesiredState ( void )
+void IOService::computeDesiredState ( unsigned long tempDesire )
 {
-    OSIterator                      *iter;
-    OSObject                        *next;
-    IOPowerConnection               *connection;
-    unsigned long                   newDesiredState = 0;
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+    unsigned long		newDesiredState = 0;
+	unsigned long		childDesire = 0;
+	unsigned long		deviceDesire;
+
+	if (tempDesire)
+		deviceDesire = tempDesire;
+	else
+		deviceDesire = fDeviceDesire;
+
+	// If clamp is on, always override deviceDesire to max.
+
+	if (fClampOn && fNumberOfPowerStates)
+		deviceDesire = fNumberOfPowerStates - 1;
+
+    // Compute the maximum  of our children's desires,
+	// our controlling driver's desire, and the subclass device's desire.
 
-    // Compute the maximum  of our children's desires, our controlling driver's desire, and the subclass device's desire.
-    if ( !  priv->device_overrides ) 
+    if ( !fDeviceOverrides )
     {
         iter = getChildIterator(gIOPowerPlane);
-
-        if ( iter ) 
+        if ( iter )
         {
-            while ( (next = iter->getNextObject()) ) 
+            while ( (next = iter->getNextObject()) )
             {
-                if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+                if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
                 {
-                    if ( connection->getDesiredDomainState() > newDesiredState ) 
-                    {
-                        newDesiredState = connection->getDesiredDomainState();
-                    }
+					if (connection->getReadyFlag() == false)
+					{
+						PM_CONNECT("[%s] %s: connection not ready\n",
+							getName(), __FUNCTION__);
+						continue;
+					}
+	
+                    if (connection->getDesiredDomainState() > childDesire)
+                        childDesire = connection->getDesiredDomainState();
                 }
             }
             iter->release();
         }
-        
-        if (  priv->driverDesire > newDesiredState ) 
-        {
-            newDesiredState =  priv->driverDesire;
-        }
+
+        fChildrenDesire = childDesire;
+		newDesiredState = max(childDesire, fDriverDesire);
     }
 
-    if ( priv->deviceDesire > newDesiredState ) 
-    {
-        newDesiredState = priv->deviceDesire;
-    }
+	newDesiredState = max(deviceDesire, newDesiredState);
+	if (fTempClampCount && (fTempClampPowerState < fNumberOfPowerStates))
+		newDesiredState = max(fTempClampPowerState, newDesiredState);
+
+    fDesiredPowerState = newDesiredState;
+    
+    // Limit check against number of power states.
+
+    if (fNumberOfPowerStates == 0)
+        fDesiredPowerState = 0;
+    else if (fDesiredPowerState >= fNumberOfPowerStates)
+        fDesiredPowerState = fNumberOfPowerStates - 1;
 
-    priv->ourDesiredPowerState = newDesiredState;
-}
+	// Restart idle timer if stopped and deviceDesire has increased.
+
+	if (fDeviceDesire && fActivityTimerStopped)
+	{
+		fActivityTimerStopped = false;
+		start_PM_idle_timer();
+	}
+
+	// Invalidate cached tickle power state when desires change, and not
+	// due to a tickle request.  This invalidation must occur before the
+	// power state change to minimize races.  We want to err on the side
+	// of servicing more activity tickles rather than dropping one when
+	// the device is in a low power state.
+
+	if (fPMRequest && (fPMRequest->getType() != kIOPMRequestTypeActivityTickle) &&
+		(fActivityTicklePowerState != -1))
+	{
+		IOLockLock(fActivityLock);
+		fActivityTicklePowerState = -1;
+		IOLockUnlock(fActivityLock);
+	}
 
+	PM_TRACE("   NewState %ld, Child %ld, Driver %ld, Device %ld, Clamp %d (%ld)\n",
+		fDesiredPowerState, childDesire, fDriverDesire, deviceDesire,
+		fClampOn, fTempClampCount ? fTempClampPowerState : 0);
+}
 
 //*********************************************************************************
-// changeState
+// [private] changeState
 //
 // A subclass object, our controlling driver, or a power domain child
 // has asked for a different power state.  Here we compute what new
@@ -1617,124 +2236,98 @@ void IOService::computeDesiredState ( void )
 
 IOReturn IOService::changeState ( void )
 {
-    // if not fully initialized
-    if ( (pm_vars->theControllingDriver == NULL) ||
-                    !(inPlane(gIOPowerPlane)) ||
-                    !(pm_vars->parentsKnowState) ) 
-    {
-        // we can do no more
-        return IOPMNoErr;
-    }
-    
-    return enqueuePowerChange(IOPMWeInitiated,priv->ourDesiredPowerState,0,0,0);
-}
+	IOReturn result;
+
+	PM_ASSERT_IN_GATE();
+	assert(inPlane(gIOPowerPlane));
+	assert(fParentsKnowState);
+	assert(fControllingDriver);
+
+	result = enqueuePowerChange(
+			 /* flags        */	IOPMWeInitiated,
+			 /* power state  */	fDesiredPowerState,
+			 /* domain state */	0,
+			 /* connection   */	0,
+			 /* parent state */	0);
 
+	return result;
+}
 
 //*********************************************************************************
-// currentPowerConsumption
+// [public virtual] currentPowerConsumption
 //
 //*********************************************************************************
 
 unsigned long IOService::currentPowerConsumption ( void )
 {
-    if ( pm_vars->theControllingDriver == NULL ) 
-    {
+    if (!initialized)
         return kIOPMUnknown;
-    }
-    if ( pm_vars->thePowerStates[pm_vars->myCurrentState].capabilityFlags & kIOPMStaticPowerValid ) 
-    {
-        return  pm_vars->thePowerStates[pm_vars->myCurrentState].staticPower;
-    }
-    return kIOPMUnknown;
+
+    return fCurrentPowerConsumption;
+}
+
+//*********************************************************************************
+// [public virtual] getPMworkloop
+//*********************************************************************************
+
+IOWorkLoop * IOService::getPMworkloop ( void )
+{
+	return gIOPMWorkLoop;
 }
 
 //*********************************************************************************
-// activityTickle
+// [public virtual] activityTickle
 //
-// The activity tickle with parameter kIOPMSubclassPolicyis not handled
-// here and should have been intercepted by the subclass.
 // The tickle with parameter kIOPMSuperclassPolicy1 causes the activity
 // flag to be set, and the device state checked.  If the device has been
 // powered down, it is powered up again.
+// The tickle with parameter kIOPMSubclassPolicy is ignored here and
+// should be intercepted by a subclass.
 //*********************************************************************************
 
 bool IOService::activityTickle ( unsigned long type, unsigned long stateNumber )
 {
-    IOPMrootDomain                      *pmRootDomain;
-    AbsoluteTime                        uptime;
+	IOPMRequest *	request;
+	bool			noPowerChange = true;
 
-    if ( type == kIOPMSuperclassPolicy1 ) 
-    {
-        if ( pm_vars->theControllingDriver == NULL ) 
-        {
-            return true;
-        }
-        
-        if( priv->activityLock == NULL )
-        {
-            priv->activityLock = IOLockAlloc();
-        }
-        
-        IOTakeLock(priv->activityLock);
-        priv->device_active = true;
+    if ( initialized && stateNumber && (type == kIOPMSuperclassPolicy1) )
+	{
+        IOLockLock(fActivityLock);
 
-        clock_get_uptime(&uptime);
-        priv->device_active_timestamp = uptime;
+		// Record device activity for the idle timer handler.
 
-        if ( pm_vars->myCurrentState >= stateNumber) 
-        {
-            IOUnlock(priv->activityLock);
-            return true;
-        }
-        IOUnlock(priv->activityLock);
-                
-        // Transfer execution to the PM workloop
-        if( (pmRootDomain = getPMRootDomain()) )
-            pmRootDomain->unIdleDevice(this, stateNumber);
+        fDeviceActive = true;
+        clock_get_uptime(&fDeviceActiveTimestamp);
 
-        return false;
-    }
-    return true;
-}
+		// Record the last tickle power state.
+		// This helps to filter out redundant tickles as
+		// this function may be called from the data path.
 
-//*********************************************************************************
-// getPMworkloop
-//
-// A child is calling to get a pointer to the Power Management workloop.
-// We got it or get it from one of our parents.
-//*********************************************************************************
+		if (fActivityTicklePowerState < (long)stateNumber)
+		{
+			fActivityTicklePowerState = stateNumber;
+			noPowerChange = false;
 
-IOWorkLoop * IOService::getPMworkloop ( void )
-{
-    IOService                       *nub;
-    IOService                       *parent;
+			request = acquirePMRequest( this, kIOPMRequestTypeActivityTickle );
+			if (request)
+			{
+				request->fArg0 = (void *) stateNumber;	// power state
+				request->fArg1 = (void *) true;			// power rise
+				submitPMRequest(request);
+			}
+		}
 
-    if ( ! inPlane(gIOPowerPlane) ) 
-    {
-        return NULL;
-    }
-    // we have no workloop yet
-    if ( pm_vars->PMworkloop == NULL ) 
-    {
-        nub = (IOService *)copyParentEntry(gIOPowerPlane);
-        if ( nub ) 
-        {
-            parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
-            nub->release();
-            // ask one of our parents for the workloop
-            if ( parent ) 
-            {
-                pm_vars->PMworkloop = parent->getPMworkloop();
-                parent->release();
-            }
-        }
-    }
-    return  pm_vars->PMworkloop;
-}
+		IOLockUnlock(fActivityLock);
+	}
 
+	// Returns false if the activityTickle might cause a transition to a
+	// higher powered state, true otherwise.
+
+    return noPowerChange;
+}
 
 //*********************************************************************************
-// setIdleTimerPeriod
+// [public virtual] setIdleTimerPeriod
 //
 // A subclass policy-maker is going to use our standard idleness
 // detection service.  Make a command queue and an idle timer and
@@ -1742,34 +2335,34 @@ IOWorkLoop * IOService::getPMworkloop ( void )
 // start the timer.
 //*********************************************************************************
 
-IOReturn  IOService::setIdleTimerPeriod ( unsigned long period )
+IOReturn IOService::setIdleTimerPeriod ( unsigned long period )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMsetIdleTimerPeriod,period, 0);
+	IOWorkLoop * wl = getPMworkloop();
+
+    if (!initialized || !wl)
+		return IOPMNotYetInitialized;
+
+    OUR_PMLog(PMsetIdleTimerPeriod, period, 0);
 
-    priv->idle_timer_period = period;
+    fIdleTimerPeriod = period;
 
-    if ( period > 0 ) 
+    if ( period > 0 )
     {
-        if ( getPMworkloop() == NULL ) 
-        {
-            return kIOReturnError;
-        }
-        
        	// make the timer event
-        if (  priv->timerEventSrc == NULL ) 
-        {
-            priv->timerEventSrc = IOTimerEventSource::timerEventSource(this,
-                                                    PM_idle_timer_expired);
-            if ((!priv->timerEventSrc) ||
-                    (pm_vars->PMworkloop->addEventSource(priv->timerEventSrc) != kIOReturnSuccess) ) 
-            {
-                return kIOReturnError;
-            }
-        }
-
-        if ( priv->activityLock == NULL ) 
+        if ( fIdleTimerEventSource == NULL )
         {
-            priv->activityLock = IOLockAlloc();
+			IOTimerEventSource * timerSrc;
+
+			timerSrc = IOTimerEventSource::timerEventSource(
+				this, PM_idle_timer_expired);
+			
+            if (timerSrc && (wl->addEventSource(timerSrc) != kIOReturnSuccess))
+			{
+				timerSrc->release();
+				timerSrc = 0;
+			}
+
+            fIdleTimerEventSource = timerSrc;
         }
 
         start_PM_idle_timer();
@@ -1778,11 +2371,12 @@ IOReturn  IOService::setIdleTimerPeriod ( unsigned long period )
 }
 
 //******************************************************************************
-// nextIdleTimeout
+// [public virtual] nextIdleTimeout
 //
 // Returns how many "seconds from now" the device should idle into its
 // next lowest power state.
 //******************************************************************************
+
 SInt32 IOService::nextIdleTimeout(
     AbsoluteTime currentTime,
     AbsoluteTime lastActivity, 
@@ -1802,19 +2396,20 @@ SInt32 IOService::nextIdleTimeout(
     delta_secs = (SInt32)(delta_ns / NSEC_PER_SEC);
 
     // Be paranoid about delta somehow exceeding timer period.
-    if (delta_secs < (int) priv->idle_timer_period ) 
-        delay_secs = (int) priv->idle_timer_period - delta_secs;
+    if (delta_secs < (int) fIdleTimerPeriod )
+        delay_secs = (int) fIdleTimerPeriod - delta_secs;
     else
-        delay_secs = (int) priv->idle_timer_period;
+        delay_secs = (int) fIdleTimerPeriod;
     
     return (SInt32)delay_secs;
 }
 
 //******************************************************************************
-// start_PM_idle_timer
+// [public virtual] start_PM_idle_timer
 //
 // The parameter is a pointer to us.  Use it to call our timeout method.
 //******************************************************************************
+
 void IOService::start_PM_idle_timer ( void )
 {
     static const int                    maxTimeout = 100000;
@@ -1822,48 +2417,45 @@ void IOService::start_PM_idle_timer ( void )
     AbsoluteTime                        uptime;
     SInt32                              idle_in = 0;
 
-    IOLockLock(priv->activityLock);
+	if (!initialized || !fIdleTimerEventSource)
+		return;
+
+    IOLockLock(fActivityLock);
 
     clock_get_uptime(&uptime);
     
     // Subclasses may modify idle sleep algorithm
-    idle_in = nextIdleTimeout(uptime, 
-        priv->device_active_timestamp,
-        pm_vars->myCurrentState);
+    idle_in = nextIdleTimeout(uptime, fDeviceActiveTimestamp, fCurrentPowerState);
 
     // Check for out-of range responses
-    if(idle_in > maxTimeout)
+    if (idle_in > maxTimeout)
     {
         // use standard implementation
         idle_in = IOService::nextIdleTimeout(uptime,
-                        priv->device_active_timestamp,
-                        pm_vars->myCurrentState);
-    } else if(idle_in < minTimeout) {
-        // fire immediately
-        idle_in = 0;
+                        fDeviceActiveTimestamp,
+                        fCurrentPowerState);
+    } else if (idle_in < minTimeout) {
+        idle_in = fIdleTimerPeriod;
     }
 
-    priv->timerEventSrc->setTimeout(idle_in, NSEC_PER_SEC);
+    IOLockUnlock(fActivityLock);
 
-    IOLockUnlock(priv->activityLock);
-    return;
+	fIdleTimerEventSource->setTimeout(idle_in, NSEC_PER_SEC);
 }
 
-
 //*********************************************************************************
-// PM_idle_timer_expired
+// [private] PM_idle_timer_expired
 //
 // The parameter is a pointer to us.  Use it to call our timeout method.
 //*********************************************************************************
 
-void PM_idle_timer_expired(OSObject * ourSelves, IOTimerEventSource *)
+void PM_idle_timer_expired ( OSObject * ourSelves, IOTimerEventSource * )
 {
     ((IOService *)ourSelves)->PM_idle_timer_expiration();
 }
 
-
 //*********************************************************************************
-// PM_idle_timer_expiration
+// [public virtual] PM_idle_timer_expiration
 //
 // The idle timer has expired.  If there has been activity since the last
 // expiration, just restart the timer and return.  If there has not been
@@ -1872,69 +2464,64 @@ void PM_idle_timer_expired(OSObject * ourSelves, IOTimerEventSource *)
 
 void IOService::PM_idle_timer_expiration ( void )
 {
-    if ( ! initialized ) 
-    {
-        // we're unloading
+	IOPMRequest *	request;
+	bool			restartTimer = true;
+
+    if ( !initialized || !fIdleTimerPeriod )
         return;
-    }
 
-    if (  priv->idle_timer_period > 0 ) 
-    {
-        IOTakeLock(priv->activityLock);
-        if ( priv->device_active ) 
-        {
-            priv->device_active = false;
-            IOUnlock(priv->activityLock);
-            start_PM_idle_timer();
-            return;
-        }
-        if ( pm_vars->myCurrentState > 0 ) 
-        {
-        	
-        	unsigned long	newState = pm_vars->myCurrentState - 1;
-        	
-            IOUnlock(priv->activityLock);
-            changePowerStateToPriv(newState);
-            if ( newState >= priv->ourDesiredPowerState )
-            	start_PM_idle_timer();
-            return;
-        }
-        IOUnlock(priv->activityLock);
-        start_PM_idle_timer();
-    }
-}
+	IOLockLock(fActivityLock);
 
+	// Check for device activity (tickles) over last timer period.
 
-// **********************************************************************************
-// command_received
-//
-// We are un-idling a device due to its activity tickle. This routine runs on the
-// PM workloop, and is initiated by IOService::activityTickle.
-// We process all activityTickle state requests on the list.
-// **********************************************************************************
-void IOService::command_received ( void *statePtr , void *, void * , void * )
-{
-    unsigned long                       stateNumber;
+	if (fDeviceActive)
+	{
+		// Device was active - do not drop power, restart timer.
+		fDeviceActive = false;
+	}
+	else
+	{
+		// No device activity - drop power state by one level.
+		// Decrement the cached tickle power state when possible.
+		// This value may be (-1) before activityTickle() is called,
+		// but the power drop request must be issued regardless.
 
-    stateNumber = (unsigned long)statePtr;
+		if (fActivityTicklePowerState > 0)
+		{
+			fActivityTicklePowerState--;
+		}
 
-    // If not initialized, we're unloading
-    if ( ! initialized ) return;					
+		request = acquirePMRequest( this, kIOPMRequestTypeActivityTickle );
+		if (request)
+		{
+			request->fArg0 = (void *) 0;		// power state (irrelevant)
+			request->fArg1 = (void *) false;	// power drop
+			submitPMRequest( request );
 
-    if ( (pm_vars->myCurrentState < stateNumber) &&
-            (priv->imminentState < stateNumber) ) 
-    {
-        changePowerStateToPriv(stateNumber);
+			// Do not restart timer until after the tickle request has been
+			// processed.
 
-        // After we raise our state, re-schedule the idle timer.
-        if(priv->timerEventSrc)
-            start_PM_idle_timer();
+			restartTimer = false;
+		}
     }
+
+	IOLockUnlock(fActivityLock);
+
+	if (restartTimer)
+		start_PM_idle_timer();
 }
 
+//*********************************************************************************
+// [public virtual] command_received
+//
+//*********************************************************************************
+
+void IOService::command_received ( void *statePtr , void *, void * , void * )
+{
+}
 
 //*********************************************************************************
-// setAggressiveness
+// [public virtual] setAggressiveness
 //
 // Pass on the input parameters to all power domain children. All those which are
 // power domains will pass it on to their children, etc.
@@ -1942,29 +2529,39 @@ void IOService::command_received ( void *statePtr , void *, void * , void * )
 
 IOReturn IOService::setAggressiveness ( unsigned long type, unsigned long newLevel )
 {
-    OSIterator                          *iter;
-    OSObject                            *next;
-    IOPowerConnection                   *connection;
-    IOService                           *child;
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+    IOService *			child;
+
+    if (!initialized)
+		return IOPMNotYetInitialized;
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogSetAggressiveness,type, newLevel);
+    if (getPMRootDomain() == this)
+        OUR_PMLog(kPMLogSetAggressiveness, type, newLevel);
 
-    if ( type <= kMaxType ) 
+    if ( type <= kMaxType )
     {
-        pm_vars->current_aggressiveness_values[type] = newLevel;
-        pm_vars->current_aggressiveness_valid[type] = true;
+        fAggressivenessValue[type] = newLevel;
+        fAggressivenessValid[type]  = true;
     }
 
     iter = getChildIterator(gIOPowerPlane);
-
-    if ( iter ) 
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
+				if (connection->getReadyFlag() == false)
+				{
+					PM_CONNECT("[%s] %s: connection not ready\n",
+						getName(), __FUNCTION__);
+					continue;
+				}
+
                 child = ((IOService *)(connection->copyChildEntry(gIOPowerPlane)));
-                if ( child ) 
+                if ( child )
                 {
                     child->setAggressiveness(type, newLevel);
                     child->release();
@@ -1978,26 +2575,39 @@ IOReturn IOService::setAggressiveness ( unsigned long type, unsigned long newLev
 }
 
 //*********************************************************************************
-// getAggressiveness
+// [public virtual] getAggressiveness
 //
 // Called by the user client.
 //*********************************************************************************
 
 IOReturn IOService::getAggressiveness ( unsigned long type, unsigned long * currentLevel )
 {
-    if ( type > kMaxType ) 
+    if ( !initialized || (type > kMaxType) )
         return kIOReturnBadArgument;
 
-    if ( !pm_vars->current_aggressiveness_valid[type] )
+    if ( !fAggressivenessValid[type] )
         return kIOReturnInvalid;
  
-    *currentLevel = pm_vars->current_aggressiveness_values[type];
-    
+    *currentLevel = fAggressivenessValue[type];
+
     return kIOReturnSuccess;
 }
 
 //*********************************************************************************
-// systemWake
+// [public] getPowerState
+//
+//*********************************************************************************
+
+UInt32 IOService::getPowerState ( void )
+{
+    if (!initialized)
+        return 0;
+
+    return fCurrentPowerState;
+}
+
+//*********************************************************************************
+// [public virtual] systemWake
 //
 // Pass this to all power domain children. All those which are
 // power domains will pass it on to their children, etc.
@@ -2005,23 +2615,29 @@ IOReturn IOService::getAggressiveness ( unsigned long type, unsigned long * curr
 
 IOReturn IOService::systemWake ( void )
 {
-    OSIterator                          *iter;
-    OSObject                            *next;
-    IOPowerConnection                   *connection;
-    IOService                           *theChild;
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+    IOService *			theChild;
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogSystemWake,0, 0);
+    OUR_PMLog(kPMLogSystemWake, 0, 0);
 
     iter = getChildIterator(gIOPowerPlane);
-
-    if ( iter ) 
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
+				if (connection->getReadyFlag() == false)
+				{
+					PM_CONNECT("[%s] %s: connection not ready\n",
+						getName(), __FUNCTION__);
+					continue;
+				}
+
                 theChild = (IOService *)connection->copyChildEntry(gIOPowerPlane);
-                if ( theChild ) 
+                if ( theChild )
                 {
                 	theChild->systemWake();
                     theChild->release();
@@ -2031,9 +2647,9 @@ IOReturn IOService::systemWake ( void )
         iter->release();
     }
 
-    if ( pm_vars->theControllingDriver != NULL ) 
+    if ( fControllingDriver != NULL )
     {
-        if ( pm_vars->theControllingDriver->didYouWakeSystem() ) 
+        if ( fControllingDriver->didYouWakeSystem() )
         {
             makeUsable();
         }
@@ -2042,27 +2658,25 @@ IOReturn IOService::systemWake ( void )
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// temperatureCriticalForZone
-//
+// [public virtual] temperatureCriticalForZone
 //*********************************************************************************
 
 IOReturn IOService::temperatureCriticalForZone ( IOService * whichZone )
 {
-    IOService                       *theParent;
-    IOService                       *theNub;
+    IOService *	theParent;
+    IOService *	theNub;
     
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogCriticalTemp,0,0);
+    OUR_PMLog(kPMLogCriticalTemp, 0, 0);
 
-    if ( inPlane(gIOPowerPlane) && !(priv->we_are_root) ) 
+    if ( inPlane(gIOPowerPlane) && !fWeAreRoot )
     {
         theNub = (IOService *)copyParentEntry(gIOPowerPlane);
-        if ( theNub ) 
+        if ( theNub )
         {
             theParent = (IOService *)theNub->copyParentEntry(gIOPowerPlane);
             theNub->release();
-            if ( theParent ) 
+            if ( theParent )
             {
                 theParent->temperatureCriticalForZone(whichZone);
                 theParent->release();
@@ -2072,370 +2686,614 @@ IOReturn IOService::temperatureCriticalForZone ( IOService * whichZone )
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// powerOverrideOnPriv
-//
+// [public] powerOverrideOnPriv
 //*********************************************************************************
 
-
 IOReturn IOService::powerOverrideOnPriv ( void )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogOverrideOn,0,0);
+	IOPMRequest * request;
 
-    // turn on the override
-    priv->device_overrides = true;
-    computeDesiredState();
-    
-    // change state if that changed something
-    return changeState();
-}
+    if (!initialized)
+		return IOPMNotYetInitialized;
 
+	if (gIOPMWorkLoop->inGate())
+	{
+		fDeviceOverrides = true;
+		return IOPMNoErr;
+	}
+
+	request = acquirePMRequest( this, kIOPMRequestTypePowerOverrideOnPriv );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	submitPMRequest( request );
+    return IOPMNoErr;
+}
 
 //*********************************************************************************
-// powerOverrideOffPriv
-//
+// [public] powerOverrideOffPriv
 //*********************************************************************************
+
 IOReturn IOService::powerOverrideOffPriv ( void )
 {
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogOverrideOff,0,0);
+	IOPMRequest * request;
 
-    // turn off the override
-    priv->device_overrides = false;
-    computeDesiredState();
-    if( priv->clampOn)
-    {
-        return makeUsable();
-    } else {
-        // change state if that changed something
-        return changeState();
-    }
+    if (!initialized)
+		return IOPMNotYetInitialized;
+
+	if (gIOPMWorkLoop->inGate())
+	{
+		fDeviceOverrides = false;
+		return IOPMNoErr;
+	}
+
+	request = acquirePMRequest( this, kIOPMRequestTypePowerOverrideOffPriv );
+	if (!request)
+		return kIOReturnNoMemory;
+
+	submitPMRequest( request );
+    return IOPMNoErr;
 }
 
+//*********************************************************************************
+// [private] handlePowerOverrideChanged
+//*********************************************************************************
+
+void IOService::handlePowerOverrideChanged ( IOPMRequest * request )
+{
+	PM_ASSERT_IN_GATE();
+	if (request->getType() == kIOPMRequestTypePowerOverrideOnPriv)
+	{
+		OUR_PMLog(kPMLogOverrideOn, 0, 0);
+		fDeviceOverrides = true;
+    }
+	else
+	{
+		OUR_PMLog(kPMLogOverrideOff, 0, 0);
+		fDeviceOverrides = false;
+	}
+
+	if (fControllingDriver && inPlane(gIOPowerPlane) && fParentsKnowState)
+	{
+		computeDesiredState();
+		changeState();
+	}
+}
 
 //*********************************************************************************
-// enqueuePowerChange
-//
-// Allocate a new state change notification, initialize it with fields from the
-// caller, and add it to the tail of the list of pending power changes.
-//
-// If it is early enough in the list, and almost all the time it is the only one in
-// the list, start the power change.
-//
-// In rare instances, this change will preempt the previous change in the list.
-// If the previous change is un-actioned in any way (because we are still
-// processing an even earlier power change), and if both the previous change
-// in the list and this change are initiated by us (not the parent), then we
-// needn't perform the previous change, so we collapse the list a little.
+// [private] enqueuePowerChange
 //*********************************************************************************
 
 IOReturn IOService::enqueuePowerChange ( 
-    unsigned long flags,  
-    unsigned long whatStateOrdinal, 
-    unsigned long domainState, 
-    IOPowerConnection * whichParent, 
-    unsigned long singleParentState )
-{
-    long                            newNote;
-    long                            previousNote;
-
-    // Create and initialize the new change note
-
-    IOLockLock(priv->queue_lock);
-    newNote = priv->changeList->createChangeNote();
-    if ( newNote == -1 ) {
-        // uh-oh, our list is full
-        IOLockUnlock(priv->queue_lock);
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogEnqueueErr,0,0);
+    unsigned long		flags,  
+    unsigned long		whatStateOrdinal, 
+    unsigned long		domainState, 
+    IOPowerConnection *	whichParent, 
+    unsigned long		singleParentState )
+{
+	changeNoteItem		changeNote;
+	IOPMPowerState *	powerStatePtr;
+
+	PM_ASSERT_IN_GATE();
+	assert( fMachineState == kIOPM_Finished );
+    assert( whatStateOrdinal < fNumberOfPowerStates );
+
+    if (whatStateOrdinal >= fNumberOfPowerStates)
         return IOPMAckImplied;
-    }
 
-    priv->changeList->changeNote[newNote].newStateNumber = whatStateOrdinal;
-    priv->changeList->changeNote[newNote].outputPowerCharacter = pm_vars->thePowerStates[whatStateOrdinal].outputPowerCharacter;
-    priv->changeList->changeNote[newNote].inputPowerRequirement = pm_vars->thePowerStates[whatStateOrdinal].inputPowerRequirement;
-    priv->changeList->changeNote[newNote].capabilityFlags = pm_vars->thePowerStates[whatStateOrdinal].capabilityFlags;
-    priv->changeList->changeNote[newNote].flags = flags;
-    priv->changeList->changeNote[newNote].parent = NULL;
-    if (flags & IOPMParentInitiated ) 
+	powerStatePtr = &fPowerStates[whatStateOrdinal];
+
+    // Initialize the change note
+    changeNote.flags                 = flags;
+    changeNote.newStateNumber        = whatStateOrdinal;
+    changeNote.outputPowerCharacter  = powerStatePtr->outputPowerCharacter;
+    changeNote.inputPowerRequirement = powerStatePtr->inputPowerRequirement;
+    changeNote.capabilityFlags       = powerStatePtr->capabilityFlags;
+    changeNote.parent                = NULL;
+
+    if (flags & IOPMParentInitiated )
     {
-        priv->changeList->changeNote[newNote].domainState = domainState;
-        priv->changeList->changeNote[newNote].parent = whichParent;
-        whichParent->retain();
-        priv->changeList->changeNote[newNote].singleParentState = singleParentState;
+        changeNote.domainState       = domainState;
+        changeNote.parent            = whichParent;
+        changeNote.singleParentState = singleParentState;
     }
 
-    previousNote = priv->changeList->previousChangeNote(newNote);
+	if (flags & IOPMWeInitiated )
+	{
+		start_our_change(&changeNote);
+		return 0;
+	}
+	else
+	{
+		return start_parent_change(&changeNote);
+	}
+}
 
-    if ( previousNote == -1 ) 
-    {
+//*********************************************************************************
+// [private] notifyInterestedDrivers
+//*********************************************************************************
+
+bool IOService::notifyInterestedDrivers ( void )
+{
+	IOPMinformee *		informee;
+	IOPMinformeeList *	list = fInterestedDrivers;
+	DriverCallParam *	param;
+	IOItemCount			count;
+
+	PM_ASSERT_IN_GATE();
+	assert( fDriverCallBusy == false );
+	assert( fDriverCallParamCount == 0 );
+	assert( fHeadNotePendingAcks == 0 );
+
+	count = list->numberOfItems();
+	if (!count)
+		goto done;	// no interested drivers
+
+	// Allocate an array of interested drivers and their return values
+	// for the callout thread. Everything else is still "owned" by the
+	// PM work loop, which can run to process acknowledgePowerChange()
+	// responses.
+
+	param = (DriverCallParam *) fDriverCallParamPtr;
+	if (count > fDriverCallParamSlots)
+	{
+		if (fDriverCallParamSlots)
+		{
+			assert(fDriverCallParamPtr);
+			IODelete(fDriverCallParamPtr, DriverCallParam, fDriverCallParamSlots);
+			fDriverCallParamPtr = 0;
+			fDriverCallParamSlots = 0;
+		}
+
+		param = IONew(DriverCallParam, count);
+		if (!param)
+			goto done;	// no memory
+
+		fDriverCallParamPtr   = (void *) param;
+		fDriverCallParamSlots = count;
+	}
+
+	informee = list->firstInList();
+	assert(informee);
+	for (IOItemCount i = 0; i < count; i++)
+	{
+		informee->timer = -1;
+		param[i].Target = informee;
+		informee->retain();
+        informee = list->nextInList( informee );
+	}
+
+	fDriverCallParamCount = count;
+	fHeadNotePendingAcks = count;
+
+	// Machine state will be blocked pending callout thread completion.
+
+	PM_LOCK();
+	fDriverCallBusy = true;
+	PM_UNLOCK();
+	thread_call_enter( fDriverCallEntry );
+	return true;
+
+done:
+	// no interested drivers or did not schedule callout thread due to error.
+	return false;
+}
+
+//*********************************************************************************
+// [private] notifyInterestedDriversDone
+//*********************************************************************************
+
+void IOService::notifyInterestedDriversDone ( void )
+{
+	IOPMinformee *		informee;
+	IOItemCount			count;
+	DriverCallParam *	param;
+	IOReturn			result;
+
+	PM_ASSERT_IN_GATE();
+	param = (DriverCallParam *) fDriverCallParamPtr;
+	count = fDriverCallParamCount;
+
+	assert( fDriverCallBusy == false );
+	assert( fMachineState == kIOPM_DriverThreadCallDone );
+
+	if (param && count)
+	{
+		for (IOItemCount i = 0; i < count; i++, param++)
+		{
+			informee = (IOPMinformee *) param->Target;
+			result   = param->Result;
+
+			if ((result == IOPMAckImplied) || (result < 0))
+			{
+				// child return IOPMAckImplied
+				informee->timer = 0;
+				fHeadNotePendingAcks--;
+			}
+			else if (informee->timer)
+			{
+                assert(informee->timer == -1);
+
+                // Driver has not acked, and has returned a positive result.
+                // Enforce a minimum permissible timeout value.
+                // Make the min value large enough so timeout is less likely
+                // to occur if a driver misinterpreted that the return value
+                // should be in microsecond units.  And make it large enough
+                // to be noticeable if a driver neglects to ack.
+
+                if (result < kMinAckTimeoutTicks)
+                    result = kMinAckTimeoutTicks;
+
+                informee->timer = (result / (ACK_TIMER_PERIOD / ns_per_us)) + 1;
+			}
+			// else, child has already acked or driver has removed interest,
+            // and head_note_pendingAcks decremented.
+			// informee may have been removed from the interested drivers list,
+            // thus the informee must be retained across the callout.
+
+			informee->release();
+		}
+
+		fDriverCallParamCount = 0;
+
+		if ( fHeadNotePendingAcks )
+		{
+			OUR_PMLog(kPMLogStartAckTimer, 0, 0);
+			start_ack_timer();
+		}
+	}
+
+	// Hop back to original machine state path (from notifyAll)
+	fMachineState = fNextMachineState;
+
+	notifyChildren();
+}
+
+//*********************************************************************************
+// [private] notifyChildren
+//*********************************************************************************
+
+void IOService::notifyChildren ( void )
+{
+    OSIterator *		iter;
+    OSObject *			next;
+    IOPowerConnection *	connection;
+	OSArray *			children = 0;
 
-        // Queue is empty, we can start this change.
+	if (fStrictTreeOrder)
+		children = OSArray::withCapacity(8);
 
-        if (flags & IOPMWeInitiated ) 
+    // Sum child power consumption in notifyChild()
+    fPowerStates[fHeadNoteState].staticPower = 0;
+
+    iter = getChildIterator(gIOPowerPlane);
+    if ( iter )
+    {
+        while ((next = iter->getNextObject()))
         {
-            IOLockUnlock(priv->queue_lock);
-            start_our_change(newNote);
-            return 0;
-        } else {
-            IOLockUnlock(priv->queue_lock);
-            return start_parent_change(newNote);
+            if ((connection = OSDynamicCast(IOPowerConnection, next)))
+            {
+				if (connection->getReadyFlag() == false)
+				{
+					PM_CONNECT("[%s] %s: connection not ready\n",
+						getName(), __FUNCTION__);
+					continue;
+				}
+
+				if (children)
+					children->setObject( connection );
+				else
+					notifyChild( connection,
+						fDriverCallReason == kDriverCallInformPreChange );
+			}
         }
+        iter->release();
     }
 
-    // The queue is not empty.  Try to collapse this new change and the previous one in queue into one change.
-    // This is possible only if both changes are initiated by us, and neither has been started yet.
-    // Do this more than once if possible.
+	if (children)
+	{
+		if (children->getCount() == 0)
+		{
+			children->release();
+			children = 0;
+		}
+		else
+		{
+			assert(fNotifyChildArray == 0);
+			fNotifyChildArray = children;
+			fNextMachineState = fMachineState;
+			fMachineState     = kIOPM_NotifyChildrenDone;
+		}		
+	}
+}
 
-    // (A change is started iff it is at the head of the queue)
+//*********************************************************************************
+// [private] notifyChildrenDone
+//*********************************************************************************
 
-    while ( (previousNote != priv->head_note) &&  (previousNote != -1) &&
-            (priv->changeList->changeNote[newNote].flags &  priv->changeList->changeNote[previousNote].flags &  IOPMWeInitiated)  ) 
-    {
-        priv->changeList->changeNote[previousNote].outputPowerCharacter = priv->changeList->changeNote[newNote].outputPowerCharacter;
-        priv->changeList->changeNote[previousNote].inputPowerRequirement = priv->changeList->changeNote[newNote].inputPowerRequirement;
-        priv->changeList->changeNote[previousNote].capabilityFlags =priv-> changeList->changeNote[newNote].capabilityFlags;
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogCollapseQueue,priv->changeList->changeNote[newNote].newStateNumber,
-                                                                    priv->changeList->changeNote[previousNote].newStateNumber);
-        priv->changeList->changeNote[previousNote].newStateNumber = priv->changeList->changeNote[newNote].newStateNumber;
-        priv->changeList->releaseTailChangeNote();
-        newNote = previousNote;
-        previousNote = priv->changeList->previousChangeNote(newNote);
-    }
-    IOLockUnlock(priv->queue_lock);
-    // in any case, we can't start yet
-    return IOPMWillAckLater;
+void IOService::notifyChildrenDone ( void )
+{
+	PM_ASSERT_IN_GATE();
+	assert(fNotifyChildArray);
+	assert(fMachineState == kIOPM_NotifyChildrenDone);
+
+	// Interested drivers have all acked (if any), ack timer stopped.
+	// Notify one child, wait for it's ack, then repeat for next child.
+	// This is a workaround for some drivers with multiple instances at
+	// the same branch in the power tree, but the driver is slow to power
+	// up unless the tree ordering is observed. Problem observed only on
+	// system wake, not on system sleep.
+	//
+	// We have the ability to power off in reverse child index order.
+	// That works nicely on some machines, but not on all HW configs.
+
+	if (fNotifyChildArray->getCount())
+	{
+		IOPowerConnection *	connection;
+		connection = (IOPowerConnection *) fNotifyChildArray->getObject(0);
+		fNotifyChildArray->removeObject(0);
+		notifyChild( connection, fDriverCallReason == kDriverCallInformPreChange );
+	}
+	else
+	{
+		fNotifyChildArray->release();
+		fNotifyChildArray = 0;
+		fMachineState = fNextMachineState;
+	}
 }
 
 //*********************************************************************************
-// notifyAll
-//
-// Notify all interested parties either that a change is impending or that the
-// previously-notified change is done and power has settled.
-// The parameter identifies whether this is the
-// pre-change notification or the post-change notification.
-//
+// [private] notifyAll
 //*********************************************************************************
 
 IOReturn IOService::notifyAll ( bool is_prechange )
 {
-    IOPMinformee *		nextObject;
-    OSIterator *			iter;
-    OSObject *			next;
-    IOPowerConnection *	connection;
+	// Save the next machine_state to be restored by notifyInterestedDriversDone()
 
-    // To prevent acknowledgePowerChange from finishing the change note and 
-    // removing it from the queue if
-    // some driver calls it, we inflate the number of pending acks so it 
-    // cannot become zero.  We'll fix it later.
-    
-    if(!acquire_lock()) return IOPMAckImplied;
+	PM_ASSERT_IN_GATE();
+	fNextMachineState = fMachineState;
+	fMachineState     = kIOPM_DriverThreadCallDone;
+	fDriverCallReason = is_prechange ?
+						kDriverCallInformPreChange : kDriverCallInformPostChange;
 
-    OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
+	if (!notifyInterestedDrivers())
+		notifyInterestedDriversDone();
 
-    // OK, we will go through the lists of interested drivers and 
-    // power domain children and notify each one of this change.
-    
-    nextObject =  priv->interestedDrivers->firstInList();
-    while (  nextObject != NULL ) {
+	return IOPMWillAckLater;
+}
 
-        OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
+//*********************************************************************************
+// [private, static] pmDriverCallout
+//
+// Thread call context
+//*********************************************************************************
 
-        IOUnlock(priv->our_lock);
+IOReturn IOService::actionDriverCalloutDone (
+	OSObject * target,
+	void * arg0, void * arg1,
+	void * arg2, void * arg3 )
+{
+	IOServicePM * pwrMgt = (IOServicePM *) arg0;
 
-        inform(nextObject, is_prechange);
-        
-        if(!acquire_lock())
-        {
-            goto exit;
-        }
-        
-        nextObject  =  priv->interestedDrivers->nextInList(nextObject);
-    }
+	PM_LOCK();
+	fDriverCallBusy = false;
+	PM_UNLOCK();
 
-    // did they all ack?
-    if ( priv->head_note_pendingAcks > 1 ) {
-        // no
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-        start_ack_timer();
-    }
-    // either way
-    IOUnlock(priv->our_lock);
+	if (gIOPMReplyQueue)
+		gIOPMReplyQueue->signalWorkAvailable();
 
-    // notify children
-    iter = getChildIterator(gIOPowerPlane);
-    // summing their power consumption
-    pm_vars->thePowerStates[priv->head_note_state].staticPower = 0;
+	return kIOReturnSuccess;
+}
 
-    if ( iter && acquire_lock()) 
-    {
-        while ( (next = iter->getNextObject()) ) 
+void IOService::pmDriverCallout ( IOService * from )
+{
+	assert(from);
+	switch (from->fDriverCallReason)
+	{
+		case kDriverCallSetPowerState:
+			from->driverSetPowerState();
+			break;
+
+		case kDriverCallInformPreChange:
+		case kDriverCallInformPostChange:
+			from->driverInformPowerChange();
+			break;
+
+		default:
+			IOPanic("IOService::pmDriverCallout bad machine state");
+	}
+
+	gIOPMWorkLoop->runAction(actionDriverCalloutDone,
+		/* target */ from,
+		/* arg0   */ (void *) from->pwrMgt );
+}
+
+//*********************************************************************************
+// [private] driverSetPowerState
+//
+// Thread call context
+//*********************************************************************************
+
+void IOService::driverSetPowerState ( void )
+{
+	IOService *			driver;
+	unsigned long		powerState;
+	DriverCallParam *	param;
+	IOReturn			result;
+    AbsoluteTime        end;
+
+	assert( fDriverCallBusy );
+	param = (DriverCallParam *) fDriverCallParamPtr;
+	assert( param );
+	assert( fDriverCallParamCount == 1 );
+
+	driver = fControllingDriver;
+	powerState = fHeadNoteState;
+
+	if (!fWillPMStop)
+	{
+		OUR_PMLog(          kPMLogProgramHardware, (UInt32) this, powerState);
+        clock_get_uptime(&fDriverCallStartTime);
+		result = driver->setPowerState( powerState, this );
+        clock_get_uptime(&end);
+		OUR_PMLog((UInt32) -kPMLogProgramHardware, (UInt32) this, (UInt32) result);
+
+#if LOG_SETPOWER_TIMES
+        if ((result == IOPMAckImplied) || (result < 0))
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
-            {
-                OSAddAtomic(1, (SInt32*)&priv->head_note_pendingAcks);
-                
-                IOUnlock(priv->our_lock);
+            uint64_t    nsec;
 
-                notifyChild(connection, is_prechange);
-            
-                if(!acquire_lock())
-                {
-                    goto exit;
-                }
-            }
+            SUB_ABSOLUTETIME(&end, &fDriverCallStartTime);
+            absolutetime_to_nanoseconds(end, &nsec);
+            if (nsec > LOG_SETPOWER_TIMES)
+                PM_DEBUG("%s::setPowerState(%p, %lu -> %lu) took %d ms\n",
+                    fName, this, fCurrentPowerState, powerState, NS_TO_MS(nsec));
         }
-        iter->release();
-        IOUnlock(priv->our_lock);
-    }
+#endif
+	}
+	else
+		result = kIOPMAckImplied;
 
-    if (! acquire_lock() ) {
-        return IOPMNoErr;
-    }
-    // now make this real
-    OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-
-    // is it all acked?
-    if (priv->head_note_pendingAcks == 0 ) {
-        // yes, all acked
-        IOUnlock(priv->our_lock);
-        // return ack to parent
-        return IOPMAckImplied;
-    }
+	param->Result = result;
+}
 
-    // not all acked
-    IOUnlock(priv->our_lock);
+//*********************************************************************************
+// [private] driverInformPowerChange
+//
+// Thread call context
+//*********************************************************************************
 
-exit:           // unable to acquire_lock exit case
+void IOService::driverInformPowerChange ( void )
+{
+	IOItemCount			count;
+	IOPMinformee *		informee;
+	IOService *			driver;
+	IOReturn			result;
+	IOPMPowerFlags		powerFlags;
+	unsigned long		powerState;
+	DriverCallParam *	param;
+    AbsoluteTime        end;
+
+	assert( fDriverCallBusy );
+	param = (DriverCallParam *) fDriverCallParamPtr;
+	count = fDriverCallParamCount;
+	assert( count && param );
+
+	powerFlags = fHeadNoteCapabilityFlags;
+	powerState = fHeadNoteState;
+
+	for (IOItemCount i = 0; i < count; i++)
+	{
+		informee = (IOPMinformee *) param->Target;
+		driver   = informee->whatObject;
+
+		if (!fWillPMStop && informee->active)
+		{
+			if (fDriverCallReason == kDriverCallInformPreChange)
+			{
+				OUR_PMLog(kPMLogInformDriverPreChange, (UInt32) this, powerState);
+                clock_get_uptime(&informee->startTime);
+				result = driver->powerStateWillChangeTo(powerFlags, powerState, this);
+                clock_get_uptime(&end);
+				OUR_PMLog((UInt32)-kPMLogInformDriverPreChange, (UInt32) this, result);
+			}
+			else
+			{
+				OUR_PMLog(kPMLogInformDriverPostChange, (UInt32) this, powerState);
+                clock_get_uptime(&informee->startTime);
+				result = driver->powerStateDidChangeTo(powerFlags, powerState, this);
+                clock_get_uptime(&end);
+				OUR_PMLog((UInt32)-kPMLogInformDriverPostChange, (UInt32) this, result);
+			}
+
+#if LOG_SETPOWER_TIMES
+            if ((result == IOPMAckImplied) || (result < 0))
+            {
+                uint64_t nsec;
+
+                SUB_ABSOLUTETIME(&end, &informee->startTime);
+                absolutetime_to_nanoseconds(end, &nsec);
+                if (nsec > LOG_SETPOWER_TIMES)
+                    PM_DEBUG("%s::powerState%sChangeTo(%p, %s, %lu -> %lu) took %d ms\n",
+                        driver->getName(),
+                        (fDriverCallReason == kDriverCallInformPreChange) ? "Will" : "Did",
+                        driver, fName, fCurrentPowerState, powerState, NS_TO_MS(nsec));
+            }
+#endif
+		}
+		else
+			result = kIOPMAckImplied;
 
-    return IOPMWillAckLater;
+		param->Result = result;
+		param++;
+	}
 }
 
-
 //*********************************************************************************
-// notifyChild
+// [private] notifyChild
 //
 // Notify a power domain child of an upcoming power change.
-//
 // If the object acknowledges the current change, we return TRUE.
 //*********************************************************************************
 
 bool IOService::notifyChild ( IOPowerConnection * theNub, bool is_prechange )
 {
-    IOReturn                            k = IOPMAckImplied;
-    unsigned long                       childPower;
-    IOService                           *theChild;
-    
+    IOReturn		k = IOPMAckImplied;
+    unsigned long	childPower;
+    IOService *		theChild;
+	IOPMRequest *	childRequest;
+	int				requestType;
+
+	PM_ASSERT_IN_GATE();
     theChild = (IOService *)(theNub->copyChildEntry(gIOPowerPlane));
-    if(!theChild) 
+    if (!theChild)
     {
-        // The child has been detached since we grabbed the child iterator.
-        // Decrement pending_acks, already incremented in notifyAll,
-        // to account for this unexpected departure.
-
-        if( acquire_lock() ) 
-        {
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            IOUnlock(priv->our_lock); 
-        }
+		assert(false);
         return true;
     }
-    
+
     // Unless the child handles the notification immediately and returns
     // kIOPMAckImplied, we'll be awaiting their acknowledgement later.
+	fHeadNotePendingAcks++;
     theNub->setAwaitingAck(true);
     
-    if ( is_prechange ) 
-    {
-        k = theChild->powerDomainWillChangeTo(priv->head_note_outputFlags,theNub);
-    } else {
-        k = theChild->powerDomainDidChangeTo(priv->head_note_outputFlags,theNub);
-    }
-    
-    // did the return code ack?
-    if ( k == IOPMAckImplied ) 
-    {
-        // yes
-        if( acquire_lock() ) 
-        {
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            IOUnlock(priv->our_lock); 
-        }
-        
-        theNub->setAwaitingAck(false);
+	requestType = is_prechange ?
+		kIOPMRequestTypePowerDomainWillChange :
+		kIOPMRequestTypePowerDomainDidChange;
+
+	childRequest = acquirePMRequest( theChild, requestType );
+	if (childRequest)
+	{
+		childRequest->fArg0 = (void *) fHeadNoteOutputFlags;
+		childRequest->fArg1 = (void *) theNub;
+		childRequest->fArg2 = (void *) (fHeadNoteState < fCurrentPowerState);
+		theChild->submitPMRequest( childRequest );
+		k = IOPMWillAckLater;
+	}
+	else
+	{
+		k = IOPMAckImplied;
+		fHeadNotePendingAcks--;  
+		theNub->setAwaitingAck(false);
         childPower = theChild->currentPowerConsumption();
-        if ( childPower == kIOPMUnknown ) 
+        if ( childPower == kIOPMUnknown )
         {
-            pm_vars->thePowerStates[priv->head_note_state].staticPower = kIOPMUnknown;
+            fPowerStates[fHeadNoteState].staticPower = kIOPMUnknown;
         } else {
-            if ( pm_vars->thePowerStates[priv->head_note_state].staticPower != kIOPMUnknown ) 
+            if ( fPowerStates[fHeadNoteState].staticPower != kIOPMUnknown )
             {
-                pm_vars->thePowerStates[priv->head_note_state].staticPower += childPower;
+                fPowerStates[fHeadNoteState].staticPower += childPower;
             }
         }
-        theChild->release();
-        return true;
-    }
-    theChild->release();
-    return false;
-}
-
-
-//*********************************************************************************
-// inform
-//
-// Notify an interested driver of an upcoming power change.
-//
-// If the object acknowledges the current change, we return TRUE.
-//*********************************************************************************
-
-bool IOService::inform ( IOPMinformee * nextObject, bool is_prechange )
-{
-    IOReturn                            k = IOPMAckImplied;
-
-    // initialize this
-    nextObject->timer = -1;
-    
-    if ( is_prechange ) 
-    {
-        pm_vars->thePlatform->PMLog (pm_vars->ourName,PMlogInformDriverPreChange,
-                                    (unsigned long)priv->head_note_capabilityFlags,(unsigned long)priv->head_note_state);
-        k = nextObject->whatObject->powerStateWillChangeTo( priv->head_note_capabilityFlags,priv->head_note_state,this);
-    } else {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogInformDriverPostChange,
-                                    (unsigned long)priv->head_note_capabilityFlags,(unsigned long)priv->head_note_state);
-        k = nextObject->whatObject->powerStateDidChangeTo(priv->head_note_capabilityFlags,priv->head_note_state,this);
-    }
-    
-    // did it ack behind our back?
-    if ( nextObject->timer == 0 ) 
-    {
-        // yes
-        return true;
-    }
-    
-    if ( (k ==IOPMAckImplied)   // no, did the return code ack?
-      || (k < 0) )              // somebody goofed
-    {
-        // yes
-        nextObject->timer = 0;
-
-        if( acquire_lock() ) 
-        {
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            IOUnlock(priv->our_lock); 
-        }
-        return true;
     }
-    
-    // no, it's a timer
-    nextObject->timer = (k / (ACK_TIMER_PERIOD / ns_per_us)) + 1;
 
-    return false;
+    theChild->release();
+	return (k == IOPMAckImplied);
 }
 
-
 //*********************************************************************************
-// OurChangeTellClientsPowerDown
+// [private] OurChangeTellClientsPowerDown
 //
 // All registered applications and kernel clients have positively acknowledged our
 // intention of lowering power.  Here we notify them all that we will definitely
@@ -2445,22 +3303,12 @@ bool IOService::inform ( IOPMinformee * nextObject, bool is_prechange )
 
 void IOService::OurChangeTellClientsPowerDown ( void )
 {
-    // next state
-    priv->machine_state = kIOPM_OurChangeTellPriorityClientsPowerDown;
-    
-    // are we waiting for responses?
-    if ( tellChangeDown1(priv->head_note_state) ) 
-    {
-        // no, notify priority clients
-        OurChangeTellPriorityClientsPowerDown();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // allowCancelCommon() or ack timeout    
+    fMachineState = kIOPM_OurChangeTellPriorityClientsPowerDown;
+    tellChangeDown1(fHeadNoteState);
 }
 
-
 //*********************************************************************************
-// OurChangeTellPriorityClientsPowerDown
+// [private] OurChangeTellPriorityClientsPowerDown
 //
 // All registered applications and kernel clients have positively acknowledged our
 // intention of lowering power.  Here we notify "priority" clients that we are
@@ -2470,44 +3318,27 @@ void IOService::OurChangeTellClientsPowerDown ( void )
 
 void IOService::OurChangeTellPriorityClientsPowerDown ( void )
 {
-    // next state
-    priv->machine_state = kIOPM_OurChangeNotifyInterestedDriversWillChange;
-    // are we waiting for responses?
-    if ( tellChangeDown2(priv->head_note_state) ) 
-    {
-        // no, notify interested drivers
-        return OurChangeNotifyInterestedDriversWillChange();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // allowCancelCommon() or ack timeout    
+    fMachineState = kIOPM_OurChangeNotifyInterestedDriversWillChange;
+    tellChangeDown2(fHeadNoteState);
 }
 
-
 //*********************************************************************************
-// OurChangeNotifyInterestedDriversWillChange
+// [private] OurChangeNotifyInterestedDriversWillChange
 //
 // All registered applications and kernel clients have acknowledged our notification
 // that we are lowering power.  Here we notify interested drivers.  If we don't have
-// to wait for any of them to acknowledge, we instruct our power driver to make the change.
-// Otherwise, we do wait.
+// to wait for any of them to acknowledge, we instruct our power driver to make the
+// change. Otherwise, we do wait.
 //*********************************************************************************
 
 void IOService::OurChangeNotifyInterestedDriversWillChange ( void )
 {
-    // no, in case they don't all ack
-    priv->machine_state = kIOPM_OurChangeSetPowerState;
-    if ( notifyAll(true) == IOPMAckImplied ) 
-    {
-        // not waiting for responses
-        OurChangeSetPowerState();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout
+    fMachineState = kIOPM_OurChangeSetPowerState;
+    notifyAll( true );
 }
 
-
 //*********************************************************************************
-// OurChangeSetPowerState
+// [private] OurChangeSetPowerState
 //
 // All interested drivers have acknowledged our pre-change notification of a power
 // change we initiated.  Here we instruct our controlling driver to make
@@ -2518,27 +3349,16 @@ void IOService::OurChangeNotifyInterestedDriversWillChange ( void )
 
 void IOService::OurChangeSetPowerState ( void )
 {
-    priv->machine_state = kIOPM_OurChangeWaitForPowerSettle;
+    fNextMachineState = kIOPM_OurChangeWaitForPowerSettle;
+	fMachineState     = kIOPM_DriverThreadCallDone;
+	fDriverCallReason = kDriverCallSetPowerState;
 
-    IOLockLock(priv->our_lock);
-
-    if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
-    {
-        // it's done, carry on
-        IOLockUnlock(priv->our_lock);
-        OurChangeWaitForPowerSettle();
-    } else {
-        // it's not, wait for it
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-        start_ack_timer();
-        IOLockUnlock(priv->our_lock);
-        // execution will resume via ack_timer_ticked()
-    }
+	if (notifyControllingDriver() == false)
+		notifyControllingDriverDone();
 }
 
-
 //*********************************************************************************
-// OurChangeWaitForPowerSettle
+// [private] OurChangeWaitForPowerSettle
 //
 // Our controlling driver has changed power state on the hardware
 // during a power change we initiated.  Here we see if we need to wait
@@ -2549,19 +3369,16 @@ void IOService::OurChangeSetPowerState ( void )
 
 void IOService::OurChangeWaitForPowerSettle ( void )
 {
-    priv->settle_time = compute_settle_time();
-    if ( priv->settle_time == 0 ) 
+	fMachineState = kIOPM_OurChangeNotifyInterestedDriversDidChange;
+    fSettleTimeUS = compute_settle_time();
+    if ( fSettleTimeUS )
     {
-       OurChangeNotifyInterestedDriversDidChange();
-    } else {
-        priv->machine_state = kIOPM_OurChangeNotifyInterestedDriversDidChange;
-        startSettleTimer(priv->settle_time);
-    }
+		startSettleTimer(fSettleTimeUS);
+	}
 }
 
-
 //*********************************************************************************
-// OurChangeNotifyInterestedDriversDidChange
+// [private] OurChangeNotifyInterestedDriversDidChange
 //
 // Power has settled on a power change we initiated.  Here we notify
 // all our interested parties post-change.  If they all acknowledge, we're
@@ -2571,20 +3388,12 @@ void IOService::OurChangeWaitForPowerSettle ( void )
 
 void IOService::OurChangeNotifyInterestedDriversDidChange ( void )
 {
-    // in case they don't all ack
-    priv->machine_state = kIOPM_OurChangeFinish;
-    if ( notifyAll(false) == IOPMAckImplied ) 
-    {
-        // not waiting for responses
-        OurChangeFinish();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout
+    fMachineState = kIOPM_OurChangeFinish;
+    notifyAll(false);
 }
 
-
 //*********************************************************************************
-// OurChangeFinish
+// [private] OurChangeFinish
 //
 // Power has settled on a power change we initiated, and
 // all our interested parties have acknowledged.  We're
@@ -2596,59 +3405,8 @@ void IOService::OurChangeFinish ( void )
     all_done();
 }
 
-
-//*********************************************************************************
-// ParentDownTellPriorityClientsPowerDown_Immediate
-//
-// All applications and kernel clients have been notified of a power lowering
-// initiated by the parent and we didn't have to wait for any responses.  Here
-// we notify any priority clients.  If they all ack, we continue with the power change.
-// If at least one doesn't, we have to wait for it to acknowledge and then continue.
-//*********************************************************************************
-
-IOReturn IOService::ParentDownTellPriorityClientsPowerDown_Immediate ( void )
-{
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentDownNotifyInterestedDriversWillChange_Delayed;
-    // are we waiting for responses?
-    if ( tellChangeDown2(priv->head_note_state) ) 
-    {
-        // no, notify interested drivers
-        return ParentDownNotifyInterestedDriversWillChange_Immediate();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // allowCancelCommon() or ack timeout
-    return IOPMWillAckLater;
-}
-
-
-//*********************************************************************************
-// ParentDownTellPriorityClientsPowerDown_Immediate2
-//
-// All priority kernel clients have been notified of a power lowering
-// initiated by the parent and we didn't have to wait for any responses.  Here
-// we notify any interested drivers and power domain children.  If they all ack,
-// we continue with the power change.
-// If at least one doesn't, we have to wait for it to acknowledge and then continue.
-//*********************************************************************************
-
-IOReturn IOService::ParentDownNotifyInterestedDriversWillChange_Immediate ( void )
-{
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentDownSetPowerState_Delayed;
-    if ( notifyAll(true) == IOPMAckImplied ) 
-    {
-        // they did
-        return ParentDownSetPowerState_Immediate();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout
-    return IOPMWillAckLater;
-}
-
-
 //*********************************************************************************
-// ParentDownTellPriorityClientsPowerDown_Immediate4
+// [private] ParentDownTellPriorityClientsPowerDown
 //
 // All applications and kernel clients have been notified of a power lowering
 // initiated by the parent and we had to wait for responses.  Here
@@ -2656,24 +3414,14 @@ IOReturn IOService::ParentDownNotifyInterestedDriversWillChange_Immediate ( void
 // If at least one doesn't, we have to wait for it to acknowledge and then continue.
 //*********************************************************************************
 
-void IOService::ParentDownTellPriorityClientsPowerDown_Delayed ( void )
+void IOService::ParentDownTellPriorityClientsPowerDown ( void )
 {
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentDownNotifyInterestedDriversWillChange_Delayed;
-
-    // are we waiting for responses?
-    if ( tellChangeDown2(priv->head_note_state) ) 
-    {
-        // no, notify interested drivers
-        ParentDownNotifyInterestedDriversWillChange_Delayed();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // allowCancelCommon() or ack timeout
+    fMachineState = kIOPM_ParentDownNotifyInterestedDriversWillChange;
+	tellChangeDown2(fHeadNoteState);
 }
 
-
 //*********************************************************************************
-// ParentDownTellPriorityClientsPowerDown_Immediate5
+// [private] ParentDownNotifyInterestedDriversWillChange
 //
 // All applications and kernel clients have been notified of a power lowering
 // initiated by the parent and we had to wait for their responses.  Here we notify
@@ -2682,53 +3430,14 @@ void IOService::ParentDownTellPriorityClientsPowerDown_Delayed ( void )
 // If at least one doesn't, we have to wait for it to acknowledge and then continue.
 //*********************************************************************************
 
-void IOService::ParentDownNotifyInterestedDriversWillChange_Delayed ( void )
-{
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentDownSetPowerState_Delayed;
-    if ( notifyAll(true) == IOPMAckImplied ) 
-    {
-        // they did
-        ParentDownSetPowerState_Delayed();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout
-}
-
-
-//*********************************************************************************
-// ParentDownSetPowerState_Immediate
-//
-// All parties have acknowledged our pre-change notification of a power
-// lowering initiated by the parent.  Here we instruct our controlling driver
-// to put the hardware in the state it needs to be in when the domain is
-// lowered.  If it does so, we continue processing
-// (waiting for settle and acknowledging the parent.)
-// If it doesn't, we have to wait for it to acknowledge and then continue.
-//*********************************************************************************
-
-IOReturn IOService::ParentDownSetPowerState_Immediate ( void )
+void IOService::ParentDownNotifyInterestedDriversWillChange ( void )
 {
-    priv->machine_state = kIOPM_ParentDownWaitForPowerSettle_Delayed;
-
-    IOLockLock(priv->our_lock);
-
-    if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
-    {
-        // it's done, carry on
-        IOLockUnlock(priv->our_lock);
-        return ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate();
-    }
-    // it's not, wait for it
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-    start_ack_timer();
-    IOLockUnlock(priv->our_lock);
-    return IOPMWillAckLater;
+    fMachineState = kIOPM_ParentDownSetPowerState;
+	notifyAll( true );
 }
 
-
 //*********************************************************************************
-// ParentDownSetPowerState_Delayed
+// [private] ParentDownSetPowerState
 //
 // We had to wait for it, but all parties have acknowledged our pre-change
 // notification of a power lowering initiated by the parent.
@@ -2739,68 +3448,18 @@ IOReturn IOService::ParentDownSetPowerState_Immediate ( void )
 // If it doesn't, we have to wait for it to acknowledge and then continue.
 //*********************************************************************************
 
-void IOService::ParentDownSetPowerState_Delayed ( void )
+void IOService::ParentDownSetPowerState ( void )
 {
-    priv-> machine_state = kIOPM_ParentDownWaitForPowerSettle_Delayed;
-
-    IOLockLock(priv->our_lock);
-
-    if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
-    {
-        // it's done, carry on
-        IOLockUnlock(priv->our_lock);
-        ParentDownWaitForPowerSettle_Delayed();
-    } else {
-        // it's not, wait for it
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-        start_ack_timer();
-        IOLockUnlock(priv->our_lock);
-    }
-}
-
-
-//*********************************************************************************
-// ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate
-//
-// Our controlling driver has changed power state on the hardware
-// during a power change initiated by our parent.  Here we see if we need
-// to wait for power to settle before continuing.  If not, we continue
-// processing (acknowledging our preparedness to the parent).
-// If so, we wait and continue later.
-//*********************************************************************************
+	fNextMachineState = kIOPM_ParentDownWaitForPowerSettle;
+	fMachineState     = kIOPM_DriverThreadCallDone;
+	fDriverCallReason = kDriverCallSetPowerState;
 
-IOReturn IOService::ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate ( void )
-{
-    IOService * nub;
-    
-    priv->settle_time = compute_settle_time();
-    if ( priv->settle_time == 0 ) 
-    {
-        // store current state in case they don't all ack
-        priv->machine_state = kIOPM_ParentDownAcknowledgeChange_Delayed;
-        if ( notifyAll(false) == IOPMAckImplied ) 
-        {
-            // not waiting for responses
-            nub = priv->head_note_parent;
-            nub->retain();
-            all_done();
-            nub->release();
-            return IOPMAckImplied;
-        }
-        // If we are waiting for responses, execution will resume via 
-        // all_acked() or ack timeout        
-        return IOPMWillAckLater;
-   } else {
-        // let settle time elapse, then notify interest drivers of our power state change in ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed
-        priv->machine_state = kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed;
-        startSettleTimer(priv->settle_time);
-        return IOPMWillAckLater;
-   }
+	if (notifyControllingDriver() == false)
+		notifyControllingDriverDone();
 }
 
-
 //*********************************************************************************
-// ParentDownWaitForPowerSettle_Delayed
+// [private] ParentDownWaitForPowerSettle
 //
 // Our controlling driver has changed power state on the hardware
 // during a power change initiated by our parent.  We have had to wait
@@ -2811,52 +3470,31 @@ IOReturn IOService::ParentDownWaitForPowerSettleAndNotifyDidChange_Immediate ( v
 // If so, we wait and continue later.
 //*********************************************************************************
 
-void IOService::ParentDownWaitForPowerSettle_Delayed ( void )
+void IOService::ParentDownWaitForPowerSettle ( void )
 {
-    priv->settle_time = compute_settle_time();
-    if ( priv->settle_time == 0 ) 
+	fMachineState = kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange;
+    fSettleTimeUS = compute_settle_time();
+    if ( fSettleTimeUS )
     {
-        ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed();
-   } else {
-       priv->machine_state = kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed;
-       startSettleTimer(priv->settle_time);
-   }
+       startSettleTimer(fSettleTimeUS);
+	}
 }
 
-
 //*********************************************************************************
-// ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed
+// [private] ParentDownNotifyDidChangeAndAcknowledgeChange
 //
 // Power has settled on a power change initiated by our parent.  Here we
 // notify interested parties.
 //*********************************************************************************
 
-void IOService::ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed ( void )
+void IOService::ParentDownNotifyDidChangeAndAcknowledgeChange ( void )
 {
-    IORegistryEntry                         *nub;
-    IOService                               *parent;
-
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentDownAcknowledgeChange_Delayed;
-    if ( notifyAll(false) == IOPMAckImplied ) {
-        nub = priv->head_note_parent;
-        nub->retain();
-        all_done();
-        parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
-        if ( parent ) {
-            parent->acknowledgePowerChange((IOService *)nub);
-            parent->release();
-        }
-        nub->release();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout in ParentDownAcknowledgeChange_Delayed.
-    // Notice the duplication of code just above and in ParentDownAcknowledgeChange_Delayed.
+    fMachineState = kIOPM_ParentDownAcknowledgeChange;
+	notifyAll(false);	
 }
 
-
 //*********************************************************************************
-// ParentDownAcknowledgeChange_Delayed
+// [private] ParentDownAcknowledgeChange
 //
 // We had to wait for it, but all parties have acknowledged our post-change
 // notification of a power  lowering initiated by the parent.
@@ -2864,16 +3502,16 @@ void IOService::ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed ( void )
 // We are done with this change note, and we can start on the next one.
 //*********************************************************************************
 
-void IOService::ParentDownAcknowledgeChange_Delayed ( void )
+void IOService::ParentDownAcknowledgeChange ( void )
 {
-    IORegistryEntry                         *nub;
-    IOService                               *parent;
-    
-    nub = priv->head_note_parent;
+    IORegistryEntry *	nub;
+    IOService *			parent;
+
+    nub = fHeadNoteParent;
     nub->retain();
     all_done();
     parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
-    if ( parent ) 
+    if ( parent )
     {
         parent->acknowledgePowerChange((IOService *)nub);
         parent->release();
@@ -2881,9 +3519,8 @@ void IOService::ParentDownAcknowledgeChange_Delayed ( void )
     nub->release();
 }
 
-
 //*********************************************************************************
-// ParentUpSetPowerState_Delayed
+// [private] ParentUpSetPowerState
 //
 // Our parent has informed us via powerStateDidChange that it has
 // raised the power in our power domain, and we have had to wait
@@ -2895,85 +3532,18 @@ void IOService::ParentDownAcknowledgeChange_Delayed ( void )
 // If it doesn't, we have to wait for it to acknowledge and then continue.
 //*********************************************************************************
 
-void IOService::ParentUpSetPowerState_Delayed ( void )
+void IOService::ParentUpSetPowerState ( void )
 {
-    priv->machine_state = kIOPM_ParentUpWaitForSettleTime_Delayed;
- 
-    IOLockLock(priv->our_lock);
- 
-    if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
-    {
-        // it did it, carry on
-        IOLockUnlock(priv->our_lock);
-        ParentUpWaitForSettleTime_Delayed();
-    } else {
-        // it didn't, wait for it
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-        start_ack_timer();
-        IOLockUnlock(priv->our_lock);
-    }
-}
-
-
-//*********************************************************************************
-// ParentUpSetPowerState_Immediate
-//
-// Our parent has informed us via powerStateDidChange that it has
-// raised the power in our power domain.  Here we instruct our controlling
-// driver to program the hardware to take advantage of the higher domain
-// power.  If it does so, we continue processing
-// (waiting for settle and notifying interested parties post-change.)
-// If it doesn't, we have to wait for it to acknowledge and then continue.
-//*********************************************************************************
-
-IOReturn IOService::ParentUpSetPowerState_Immediate ( void )
-{
-    priv->machine_state = kIOPM_ParentUpWaitForSettleTime_Delayed;
-
-    IOLockLock(priv->our_lock);
-
-    if ( instruct_driver(priv->head_note_state) == IOPMAckImplied ) 
-    {
-        // it did it, carry on
-        IOLockUnlock(priv->our_lock);
-        return ParentUpWaitForSettleTime_Immediate();
-    }
-    else {
-        // it didn't, wait for it
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,0,0);
-        start_ack_timer();
-        IOLockUnlock(priv->our_lock);
-        return IOPMWillAckLater;
-    }
-}
-
-
-//*********************************************************************************
-// ParentUpWaitForSettleTime_Immediate
-//
-// Our controlling driver has changed power state on the hardware
-// during a power raise initiated by the parent.  Here we see if we need to wait
-// for power to settle before continuing.  If not, we continue processing
-// (notifying interested parties post-change).  If so, we wait and
-// continue later.
-//*********************************************************************************
+	fNextMachineState = kIOPM_ParentUpWaitForSettleTime;
+	fMachineState     = kIOPM_DriverThreadCallDone;
+	fDriverCallReason = kDriverCallSetPowerState;
 
-IOReturn IOService::ParentUpWaitForSettleTime_Immediate ( void )
-{
-    priv->settle_time = compute_settle_time();
-    if ( priv->settle_time == 0 ) 
-    {
-        return ParentUpNotifyInterestedDriversDidChange_Immediate();
-    } else {
-        priv->machine_state = kIOPM_ParentUpNotifyInterestedDriversDidChange_Delayed;
-        startSettleTimer(priv->settle_time);
-        return IOPMWillAckLater;
-    }
+	if (notifyControllingDriver() == false)
+		notifyControllingDriverDone();
 }
 
-
 //*********************************************************************************
-// ParentUpWaitForSettleTime_Delayed
+// [private] ParentUpWaitForSettleTime
 //
 // Our controlling driver has changed power state on the hardware
 // during a power raise initiated by the parent, but we had to wait for it.
@@ -2982,50 +3552,18 @@ IOReturn IOService::ParentUpWaitForSettleTime_Immediate ( void )
 // If so, we wait and continue later.
 //*********************************************************************************
 
-void IOService::ParentUpWaitForSettleTime_Delayed ( void )
-{
-    priv->settle_time = compute_settle_time();
-    if ( priv->settle_time == 0 ) 
-    {
-        ParentUpNotifyInterestedDriversDidChange_Delayed();
-    } else {
-        priv->machine_state = kIOPM_ParentUpNotifyInterestedDriversDidChange_Delayed;
-        startSettleTimer(priv->settle_time);
-    }
-}
-
-
-//*********************************************************************************
-// ParentUpNotifyInterestedDriversDidChange_Immediate
-//
-// No power settling was required on a power raise initiated by the parent.
-// Here we notify all our interested parties post-change.  If they all acknowledge,
-// we're done with this change note, and we can start on the next one.
-// Otherwise we have to wait for acknowledgements and finish up later.
-//*********************************************************************************
-
-IOReturn IOService::ParentUpNotifyInterestedDriversDidChange_Immediate ( void )
+void IOService::ParentUpWaitForSettleTime ( void )
 {
-    IOService * nub;
-    
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentUpAcknowledgePowerChange_Delayed;
-    if ( notifyAll(false) == IOPMAckImplied ) 
+	fMachineState = kIOPM_ParentUpNotifyInterestedDriversDidChange;
+    fSettleTimeUS = compute_settle_time();
+    if ( fSettleTimeUS )
     {
-        nub = priv->head_note_parent;
-        nub->retain();
-        all_done();
-        nub->release();
-        return IOPMAckImplied;
+        startSettleTimer(fSettleTimeUS);
     }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout in ParentUpAcknowledgePowerChange_Delayed.
-    return IOPMWillAckLater;
 }
 
-
 //*********************************************************************************
-// ParentUpNotifyInterestedDriversDidChange_Delayed
+// [private] ParentUpNotifyInterestedDriversDidChange
 //
 // Power has settled on a power raise initiated by the parent.
 // Here we notify all our interested parties post-change.  If they all acknowledge,
@@ -3033,37 +3571,30 @@ IOReturn IOService::ParentUpNotifyInterestedDriversDidChange_Immediate ( void )
 // Otherwise we have to wait for acknowledgements and finish up later.
 //*********************************************************************************
 
-void IOService::ParentUpNotifyInterestedDriversDidChange_Delayed ( void )
+void IOService::ParentUpNotifyInterestedDriversDidChange ( void )
 {
-    // in case they don't all ack
-    priv->machine_state = kIOPM_ParentUpAcknowledgePowerChange_Delayed;
-    if ( notifyAll(false) == IOPMAckImplied ) 
-    {
-        ParentUpAcknowledgePowerChange_Delayed();
-    }
-    // If we are waiting for responses, execution will resume via 
-    // all_acked() or ack timeout in ParentUpAcknowledgePowerChange_Delayed.
+    fMachineState = kIOPM_ParentUpAcknowledgePowerChange;
+	notifyAll(false);	
 }
 
-
 //*********************************************************************************
-// ParentUpAcknowledgePowerChange_Delayed
+// [private] ParentUpAcknowledgePowerChange
 //
 // All parties have acknowledged our post-change notification of a power
 // raising initiated by the parent.  Here we acknowledge the parent.
 // We are done with this change note, and we can start on the next one.
 //*********************************************************************************
 
-void IOService::ParentUpAcknowledgePowerChange_Delayed ( void )
+void IOService::ParentUpAcknowledgePowerChange ( void )
 {
-    IORegistryEntry                         *nub;
-    IOService                               *parent;
-    
-    nub = priv->head_note_parent;
+    IORegistryEntry *	nub;
+    IOService *			parent;
+
+    nub = fHeadNoteParent;
     nub->retain();
     all_done();
     parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
-    if ( parent ) 
+    if ( parent )
     {
         parent->acknowledgePowerChange((IOService *)nub);
         parent->release();
@@ -3071,9 +3602,8 @@ void IOService::ParentUpAcknowledgePowerChange_Delayed ( void )
     nub->release();
 }
 
-
 //*********************************************************************************
-// all_done
+// [private] all_done
 //
 // A power change is complete, and the used post-change note is at
 // the head of the queue.  Remove it and set myCurrentState to the result
@@ -3082,136 +3612,86 @@ void IOService::ParentUpAcknowledgePowerChange_Delayed ( void )
 
 void IOService::all_done ( void )
 {
-    unsigned long                           previous_state;
-    IORegistryEntry                         *nub;
-    IOService                               *parent;
-    
-    priv->machine_state = kIOPM_Finished;
+    unsigned long	previous_state;
+
+    fMachineState = kIOPM_Finished;
 
     // our power change
-    if ( priv->head_note_flags & IOPMWeInitiated ) 
+    if ( fHeadNoteFlags & IOPMWeInitiated )
     {
         // could our driver switch to the new state?
-        if ( !( priv->head_note_flags & IOPMNotDone) ) 
+        if ( !( fHeadNoteFlags & IOPMNotDone) )
         {
+			// we changed, tell our parent
+			if ( !fWeAreRoot )
+			{
+				ask_parent(fHeadNoteState);
+			}
+
             // yes, did power raise?
-            if ( pm_vars->myCurrentState < priv->head_note_state ) 
+            if ( fCurrentPowerState < fHeadNoteState )
             {
                 // yes, inform clients and apps
-                tellChangeUp (priv->head_note_state);
-            } else {
-                // no, if this lowers our
-                if ( !  priv->we_are_root ) 
-                {
-                    // power requirements, tell the parent
-                    ask_parent(priv->head_note_state);
-                }
+                tellChangeUp (fHeadNoteState);
             }
-            previous_state = pm_vars->myCurrentState;
+            previous_state = fCurrentPowerState;
             // either way
-            pm_vars->myCurrentState = priv->head_note_state;
-            priv->imminentState = pm_vars->myCurrentState;
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogChangeDone,(unsigned long)pm_vars->myCurrentState,0);
+            fCurrentPowerState = fHeadNoteState;
+#if PM_VARS_SUPPORT
+			fPMVars->myCurrentState = fCurrentPowerState;
+#endif
+            OUR_PMLog(kPMLogChangeDone, fCurrentPowerState, 0);
+
             // inform subclass policy-maker
-            powerChangeDone(previous_state);
+            if (!fWillPMStop && fParentsKnowState)
+                powerChangeDone(previous_state);
+            else
+                PM_DEBUG("%s::powerChangeDone() skipped\n", getName());
         }
     }
 
     // parent's power change
-    if ( priv->head_note_flags & IOPMParentInitiated) 
+    if ( fHeadNoteFlags & IOPMParentInitiated)
     {
-        if ( ((priv->head_note_flags & IOPMDomainWillChange) && (pm_vars->myCurrentState >= priv->head_note_state)) ||
-                 ((priv->head_note_flags & IOPMDomainDidChange) && (pm_vars->myCurrentState < priv->head_note_state)) ) 
+        if (((fHeadNoteFlags & IOPMDomainWillChange) && (fCurrentPowerState >= fHeadNoteState)) ||
+			((fHeadNoteFlags & IOPMDomainDidChange) && (fCurrentPowerState < fHeadNoteState)))
         {
             // did power raise?
-            if ( pm_vars->myCurrentState < priv->head_note_state ) 
+            if ( fCurrentPowerState < fHeadNoteState )
             {
                 // yes, inform clients and apps
-                tellChangeUp (priv->head_note_state);
+                tellChangeUp (fHeadNoteState);
             }
             // either way
-            previous_state = pm_vars->myCurrentState;
-            pm_vars->myCurrentState = priv->head_note_state;
-            priv->imminentState = pm_vars->myCurrentState;
-            pm_vars->maxCapability = pm_vars->theControllingDriver->maxCapabilityForDomainState(priv->head_note_domainState);
-
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogChangeDone,(unsigned long)pm_vars->myCurrentState,0);
-            // inform subclass policy-maker
-            powerChangeDone(previous_state);
-        }
-    }
+            previous_state = fCurrentPowerState;
+            fCurrentPowerState = fHeadNoteState;
+#if PM_VARS_SUPPORT
+			fPMVars->myCurrentState = fCurrentPowerState;
+#endif
+            fMaxCapability = fControllingDriver->maxCapabilityForDomainState(fHeadNoteDomainState);
 
-    IOLockLock(priv->queue_lock);
-    // we're done with this
-    priv->changeList->releaseHeadChangeNote();
-        
-    // start next one in queue
-    priv->head_note = priv->changeList->currentChange();
-    if ( priv->head_note != -1 ) 
-    {
+            OUR_PMLog(kPMLogChangeDone, fCurrentPowerState, 0);
 
-        IOLockUnlock(priv->queue_lock);
-        if (priv->changeList->changeNote[priv->head_note].flags & IOPMWeInitiated ) 
-        {
-            start_our_change(priv->head_note);
-        } else {
-            nub = priv->changeList->changeNote[priv->head_note].parent;
-            if (nub) nub->retain(); // might be released by start_parent_change()
-            if ( start_parent_change(priv->head_note) == IOPMAckImplied ) 
-            {
-                parent = (IOService *)nub->copyParentEntry(gIOPowerPlane);
-                if ( parent ) 
-                {
-                    parent->acknowledgePowerChange((IOService *)nub);
-                    parent->release();
-                }
-            }
-            if (nub) nub->release();
+            // inform subclass policy-maker
+            if (!fWillPMStop && fParentsKnowState)
+                powerChangeDone(previous_state);
+            else
+                PM_DEBUG("%s::powerChangeDone() skipped\n", getName());
         }
-    } else {
-        IOLockUnlock(priv->queue_lock);
     }
-}
-
-
-
-//*********************************************************************************
-// all_acked
-//
-// A driver or child has acknowledged our notification of an upcoming power
-// change, and this acknowledgement is the last one pending
-// before we change power or after changing power.
-//
-//*********************************************************************************
-
 
+    if (fCurrentPowerState < fNumberOfPowerStates)
+    {
+        const IOPMPowerState * powerStatePtr = &fPowerStates[fCurrentPowerState];
 
-void IOService::all_acked( void )
-{
-    switch (priv->machine_state) {
-       case kIOPM_OurChangeSetPowerState:
-           OurChangeSetPowerState();
-           break;
-       case kIOPM_OurChangeFinish:
-           OurChangeFinish();
-           break;
-       case kIOPM_ParentDownSetPowerState_Delayed:
-           ParentDownSetPowerState_Delayed();	
-           break;
-       case kIOPM_ParentDownAcknowledgeChange_Delayed:
-           ParentDownAcknowledgeChange_Delayed();
-           break;
-       case kIOPM_ParentUpSetPowerState_Delayed:
-           ParentUpSetPowerState_Delayed();
-           break;
-       case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
-           ParentUpAcknowledgePowerChange_Delayed();
-           break;
-   }
+        fCurrentCapabilityFlags = powerStatePtr->capabilityFlags;
+        if (fCurrentCapabilityFlags & kIOPMStaticPowerValid)
+            fCurrentPowerConsumption = powerStatePtr->staticPower;
+    }
 }
 
 //*********************************************************************************
-// settleTimerExpired
+// [public] settleTimerExpired
 //
 // Power has settled after our last change.  Notify interested parties that
 // there is a new power state.
@@ -3219,28 +3699,11 @@ void IOService::all_acked( void )
 
 void IOService::settleTimerExpired ( void )
 {
-    if ( ! initialized ) 
-    {
-        // we're unloading
-        return;
-    }
-
-    switch (priv->machine_state) {
-        case kIOPM_OurChangeNotifyInterestedDriversDidChange:
-            OurChangeNotifyInterestedDriversDidChange();
-            break;
-        case kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed:
-            ParentDownNotifyDidChangeAndAcknowledgeChange_Delayed();
-            break;
-        case kIOPM_ParentUpNotifyInterestedDriversDidChange_Delayed:
-            ParentUpNotifyInterestedDriversDidChange_Delayed();
-            break;
-    }
+	fSettleTimeUS = 0;
 }
 
-
 //*********************************************************************************
-// compute_settle_time
+// [private] compute_settle_time
 //
 // Compute the power-settling delay in microseconds for the
 // change from myCurrentState to head_note_state.
@@ -3248,29 +3711,31 @@ void IOService::settleTimerExpired ( void )
 
 unsigned long IOService::compute_settle_time ( void )
 {
-    unsigned long                           totalTime;
-    unsigned long                           i;
+    unsigned long	totalTime;
+    unsigned long	i;
+
+	PM_ASSERT_IN_GATE();
 
     // compute total time to attain the new state
     totalTime = 0;
-    i = pm_vars->myCurrentState;
+    i = fCurrentPowerState;
 
     // we're lowering power
-    if ( priv->head_note_state < pm_vars->myCurrentState ) 
+    if ( fHeadNoteState < fCurrentPowerState )
     {
-        while ( i > priv->head_note_state ) 
+        while ( i > fHeadNoteState )
         {
-            totalTime +=  pm_vars->thePowerStates[i].settleDownTime;
+            totalTime +=  fPowerStates[i].settleDownTime;
             i--;
         }
     }
 
     // we're raising power
-    if ( priv->head_note_state > pm_vars->myCurrentState ) 
+    if ( fHeadNoteState > fCurrentPowerState )
     {
-        while ( i < priv->head_note_state ) 
+        while ( i < fHeadNoteState )
         {
-            totalTime +=  pm_vars->thePowerStates[i+1].settleUpTime;
+            totalTime +=  fPowerStates[i+1].settleUpTime;
             i++;
         }
     }
@@ -3278,27 +3743,27 @@ unsigned long IOService::compute_settle_time ( void )
     return totalTime;
 }
 
-
 //*********************************************************************************
-// startSettleTimer
+// [private] startSettleTimer
 //
-// Enter with a power-settling delay in microseconds and start a nano-second
-// timer for that delay.
+// Enter a power-settling delay in microseconds and start a timer for that delay.
 //*********************************************************************************
 
 IOReturn IOService::startSettleTimer ( unsigned long delay )
 {
     AbsoluteTime	deadline;
-    
-    clock_interval_to_deadline(delay, kMicrosecondScale, &deadline);
+	boolean_t		pending;
 
-    thread_call_enter_delayed(priv->settleTimer, deadline);
+	retain();
+    clock_interval_to_deadline(delay, kMicrosecondScale, &deadline);
+    pending = thread_call_enter_delayed(fSettleTimer, deadline);
+	if (pending) release();
 
     return IOPMNoErr;
 }
 
 //*********************************************************************************
-// ack_timer_ticked
+// [public] ackTimerTick
 //
 // The acknowledgement timeout periodic timer has ticked.
 // If we are awaiting acks for a power change notification,
@@ -3307,592 +3772,378 @@ IOReturn IOService::startSettleTimer ( unsigned long delay )
 // If we are waiting for the controlling driver to change the power
 // state of the hardware, we decrement its timer word, and if it becomes
 // zero, we pretend the driver acknowledged.
+//
+// Returns true if the timer tick made it possible to advance to the next
+// machine state, false otherwise.
 //*********************************************************************************
 
 void IOService::ack_timer_ticked ( void )
 {
-    IOPMinformee * nextObject;
+	assert(false);
+}
 
-    if ( ! initialized ) 
-    {
-        // we're unloading
-        return;
-    }
+bool IOService::ackTimerTick( void )
+{
+    IOPMinformee *		nextObject;
+	bool				done = false;
 
-    if (! acquire_lock() ) 
-    {
-        return;
-    }
-    
-    switch (priv->machine_state) {
+	PM_ASSERT_IN_GATE();
+    switch (fMachineState) {
         case kIOPM_OurChangeWaitForPowerSettle:
-        case kIOPM_ParentDownWaitForPowerSettle_Delayed:
-        case kIOPM_ParentUpWaitForSettleTime_Delayed:
-            // are we waiting for our driver to make its change?
-            if ( priv->driver_timer != 0 ) {
-                // yes, tick once
-                priv->driver_timer -= 1;
-                // it's tardy, we'll go on without it
-                if ( priv->driver_timer == 0 ) 
+        case kIOPM_ParentDownWaitForPowerSettle:
+        case kIOPM_ParentUpWaitForSettleTime:
+            // are we waiting for controlling driver to acknowledge?
+            if ( fDriverTimer > 0 )
+            {
+                // yes, decrement timer tick
+                fDriverTimer--;
+                if ( fDriverTimer == 0 )
                 {
-                    IOUnlock(priv->our_lock);
-                    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogCtrlDriverTardy,0,0);
-                    driver_acked();
+                    // controlling driver is tardy
+                    uint64_t nsec = computeTimeDeltaNS(&fDriverCallStartTime);
+                    OUR_PMLog(kPMLogCtrlDriverTardy, 0, 0);
+                    setProperty(kIOPMTardyAckSPSKey, kOSBooleanTrue);
+                    PM_ERROR("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms\n",
+                        fName, this, fCurrentPowerState, fHeadNoteState, NS_TO_MS(nsec));
+
+                    if (gIOKitDebug & kIOLogDebugPower)
+                    {
+                        panic("%s::setPowerState(%p, %lu -> %lu) timed out after %d ms",
+                            fName, this, fCurrentPowerState, fHeadNoteState, NS_TO_MS(nsec));
+                    }
+                    else
+					{
+						// Unblock state machine and pretend driver has acked.
+						done = true;
+					}
                 } else {
                     // still waiting, set timer again
                     start_ack_timer();
-                    IOUnlock(priv->our_lock);
                 }
             }
-            else {
-                IOUnlock(priv->our_lock);
-            }
             break;
 
         case kIOPM_OurChangeSetPowerState:
         case kIOPM_OurChangeFinish:
-        case kIOPM_ParentDownSetPowerState_Delayed:
-        case kIOPM_ParentDownAcknowledgeChange_Delayed:
-        case kIOPM_ParentUpSetPowerState_Delayed:
-        case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
+        case kIOPM_ParentDownSetPowerState:
+        case kIOPM_ParentDownAcknowledgeChange:
+        case kIOPM_ParentUpSetPowerState:
+        case kIOPM_ParentUpAcknowledgePowerChange:
+		case kIOPM_NotifyChildrenDone:
             // are we waiting for interested parties to acknowledge?
-            if (priv->head_note_pendingAcks != 0 ) 
+            if ( fHeadNotePendingAcks != 0 )
             {
                 // yes, go through the list of interested drivers
-                nextObject =  priv->interestedDrivers->firstInList();
+                nextObject = fInterestedDrivers->firstInList();
                 // and check each one
-                while (  nextObject != NULL ) 
+                while (  nextObject != NULL )
                 {
-                    if ( nextObject->timer > 0 ) 
+                    if ( nextObject->timer > 0 )
                     {
-                        nextObject->timer -= 1;
+                        nextObject->timer--;
                         // this one should have acked by now
-                        if ( nextObject->timer == 0 ) 
+                        if ( nextObject->timer == 0 )
                         {
-                            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogIntDriverTardy,0,0);
-                            //kprintf("interested driver tardy: %s\n",nextObject->whatObject->getName());
-                            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
+                            uint64_t nsec = computeTimeDeltaNS(&nextObject->startTime);
+                            OUR_PMLog(kPMLogIntDriverTardy, 0, 0);
+                            nextObject->whatObject->setProperty(kIOPMTardyAckPSCKey, kOSBooleanTrue);
+                            PM_ERROR("%s::powerState%sChangeTo(%p, %s, %lu -> %lu) timed out after %d ms\n",
+                                nextObject->whatObject->getName(),
+                                (fDriverCallReason == kDriverCallInformPreChange) ? "Will" : "Did",
+                                nextObject->whatObject, fName, fCurrentPowerState, fHeadNoteState,
+                                NS_TO_MS(nsec));
+
+                            // Pretend driver has acked.
+                            fHeadNotePendingAcks--;
                         }
                     }
-                    nextObject  =  priv->interestedDrivers->nextInList(nextObject);
+                    nextObject = fInterestedDrivers->nextInList(nextObject);
                 }
 
                 // is that the last?
-                if ( priv->head_note_pendingAcks == 0 ) 
+                if ( fHeadNotePendingAcks == 0 )
                 {
-                    IOUnlock(priv->our_lock);
                     // yes, we can continue
-                    all_acked();
+					done = true;
                 } else {
                     // no, set timer again
                     start_ack_timer();
-                    IOUnlock(priv->our_lock);
                 }
-            } else {
-                IOUnlock(priv->our_lock);
             }
             break;
 
-        // apps didn't respond to parent-down notification
-        case kIOPM_ParentDownTellPriorityClientsPowerDown_Immediate:
-            IOUnlock(priv->our_lock);
-            IOLockLock(priv->flags_lock);
-            if (pm_vars->responseFlags) 
-            {
-                // get rid of this stuff
-                pm_vars->responseFlags->release();
-                pm_vars->responseFlags = NULL;
-            }
-            IOLockUnlock(priv->flags_lock);
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientTardy,0,5);
-            // carry on with the change
-            ParentDownTellPriorityClientsPowerDown_Delayed();
-            break;
-            
-        case kIOPM_ParentDownNotifyInterestedDriversWillChange_Delayed:
-            IOUnlock(priv->our_lock);
-            IOLockLock(priv->flags_lock);
-            if (pm_vars->responseFlags) 
-            {
-                // get rid of this stuff
-                pm_vars->responseFlags->release();
-                pm_vars->responseFlags = NULL;
-            }
-            IOLockUnlock(priv->flags_lock);
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientTardy,0,1);
-            // carry on with the change
-            ParentDownNotifyInterestedDriversWillChange_Delayed();
-            break;
-            
+        case kIOPM_ParentDownTellPriorityClientsPowerDown:
+        case kIOPM_ParentDownNotifyInterestedDriversWillChange:
         case kIOPM_OurChangeTellClientsPowerDown:
-            // apps didn't respond to our power-down request
-            IOUnlock(priv->our_lock);
-            IOLockLock(priv->flags_lock);
-            if (pm_vars->responseFlags) 
-            {
-                // get rid of this stuff
-                pm_vars->responseFlags->release();
-                pm_vars->responseFlags = NULL;
-            }
-            IOLockUnlock(priv->flags_lock);
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientTardy,0,2);
-            // rescind the request
-            tellNoChangeDown(priv->head_note_state);
-            // mark the change note un-actioned
-            priv->head_note_flags |= IOPMNotDone;
-            // and we're done
-            all_done();
-            break;
-            
         case kIOPM_OurChangeTellPriorityClientsPowerDown:
-            // clients didn't respond to our power-down note
-            IOUnlock(priv->our_lock);
-            IOLockLock(priv->flags_lock);
-            if (pm_vars->responseFlags) 
-            {
-                // get rid of this stuff
-                pm_vars->responseFlags->release();
-                pm_vars->responseFlags = NULL;
-            }
-            IOLockUnlock(priv->flags_lock);
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientTardy,0,4);
-            // carry on with the change
-            OurChangeTellPriorityClientsPowerDown();
-            break;
-            
         case kIOPM_OurChangeNotifyInterestedDriversWillChange:
-             // apps didn't respond to our power-down notification
-            IOUnlock(priv->our_lock);
-            IOLockLock(priv->flags_lock);
-            if (pm_vars->responseFlags) 
-            {
-                // get rid of this stuff
-                pm_vars->responseFlags->release();
-                pm_vars->responseFlags = NULL;
-            }
-            IOLockUnlock(priv->flags_lock);
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientTardy,0,3);
-            // carry on with the change
-            OurChangeNotifyInterestedDriversWillChange();
+			// apps didn't respond in time
+            cleanClientResponses(true);
+            OUR_PMLog(kPMLogClientTardy, 0, 1);
+			if (fMachineState == kIOPM_OurChangeTellClientsPowerDown)
+			{
+				// tardy equates to veto
+				fDoNotPowerDown = true;
+			}
+			done = true;
             break;
-            
+
         default:
-            // not waiting for acks
-            IOUnlock(priv->our_lock);
+            PM_TRACE("[%s] unexpected ack timer tick (state = %ld)\n",
+				getName(), fMachineState);
             break;
     }
+	return done;
 }
 
-
 //*********************************************************************************
-// start_ack_timer
-//
+// [private] start_ack_timer
 //*********************************************************************************
 
 void IOService::start_ack_timer ( void )
 {
-    AbsoluteTime                            deadline;
-
-    clock_interval_to_deadline(ACK_TIMER_PERIOD, kNanosecondScale, &deadline);
-    
-    thread_call_enter_delayed(priv->ackTimer, deadline);
+	start_ack_timer( ACK_TIMER_PERIOD, kNanosecondScale );
 }
 
+void IOService::start_ack_timer ( UInt32 interval, UInt32 scale )
+{
+    AbsoluteTime	deadline;
+	boolean_t		pending;
+
+    clock_interval_to_deadline(interval, scale, &deadline);
+
+	retain();
+    pending = thread_call_enter_delayed(fAckTimer, deadline);
+	if (pending) release();
+}
 
 //*********************************************************************************
-// stop_ack_timer
-//
+// [private] stop_ack_timer
 //*********************************************************************************
 
 void IOService::stop_ack_timer ( void )
 {
-    thread_call_cancel(priv->ackTimer);
-}
+	boolean_t		pending;
 
+    pending = thread_call_cancel(fAckTimer);
+	if (pending) release();
+}
 
 //*********************************************************************************
-// c-language timer expiration functions
+// [static] settleTimerExpired
 //
+// Inside PM work loop's gate.
 //*********************************************************************************
 
-static void ack_timer_expired ( thread_call_param_t us)
+IOReturn
+IOService::actionAckTimerExpired (
+	OSObject * target,
+	void * arg0, void * arg1,
+	void * arg2, void * arg3 )
 {
-    ((IOService *)us)->ack_timer_ticked();
-}
+	IOService * me = (IOService *) target;
+	bool		done;
 
+	// done will be true if the timer tick unblocks the machine state,
+	// otherwise no need to signal the work loop.
 
-static void settle_timer_expired ( thread_call_param_t us)
-{
-    ((IOService *)us)->settleTimerExpired();
-}
+	done = me->ackTimerTick();
+	if (done && gIOPMReplyQueue)
+		gIOPMReplyQueue->signalWorkAvailable();
 
+	return kIOReturnSuccess;
+}
 
 //*********************************************************************************
-// add_child_to_active_change
+// ack_timer_expired
 //
-// A child has just registered with us.  If there is
-// currently a change in progress, get the new party involved: if we
-// have notified all parties and are waiting for acks, notify the new
-// party.
+// Thread call function. Holds a retain while the callout is in flight.
 //*********************************************************************************
 
-IOReturn IOService::add_child_to_active_change ( IOPowerConnection * newObject )
+void
+IOService::ack_timer_expired ( thread_call_param_t arg0, thread_call_param_t arg1 )
 {
-    if (! acquire_lock() ) 
-    {
-        return IOPMNoErr;
-    }
-
-    switch (priv->machine_state) 
-    {
-        case kIOPM_OurChangeSetPowerState:
-        case kIOPM_ParentDownSetPowerState_Delayed:
-        case kIOPM_ParentUpSetPowerState_Delayed:
-            // one for this child and one to prevent
-            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
-            // incoming acks from changing our state
-            IOUnlock(priv->our_lock);
-            notifyChild(newObject, true);
-            if (! acquire_lock() ) 
-            {
-                // put it back
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                return IOPMNoErr;
-            }
-            // are we still waiting for acks?
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            if ( priv->head_note_pendingAcks == 0 ) 
-            {
-                // no, stop the timer
-                stop_ack_timer();
-                IOUnlock(priv->our_lock);
-                
-                // and now we can continue
-                all_acked();
-                return IOPMNoErr;
-            }
-            break;
-        case kIOPM_OurChangeFinish:
-        case kIOPM_ParentDownAcknowledgeChange_Delayed:
-        case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
-            // one for this child and one to prevent
-            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
-            // incoming acks from changing our state
-            IOUnlock(priv->our_lock);
-            notifyChild(newObject, false);
-            if (! acquire_lock() ) 
-            {
-                // put it back
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                return IOPMNoErr;
-            }
-            // are we still waiting for acks?
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            if ( priv->head_note_pendingAcks == 0 ) 
-            {
-                // no, stop the timer
-                stop_ack_timer();
-                IOUnlock(priv->our_lock);
+	IOService * me = (IOService *) arg0;
 
-                // and now we can continue
-                all_acked();
-                return IOPMNoErr;
-            }
-            break;
-    }
-    IOUnlock(priv->our_lock);
-    return IOPMNoErr;
+	if (gIOPMWorkLoop)
+	{
+		gIOPMWorkLoop->runAction(&actionAckTimerExpired, me);
+	}
+	me->release();
 }
 
-
 //*********************************************************************************
-// add_driver_to_active_change
+// settleTimerExpired
 //
-// An interested driver has just registered with us.  If there is
-// currently a change in progress, get the new party involved: if we
-// have notified all parties and are waiting for acks, notify the new
-// party.
+// Inside PM work loop's gate.
 //*********************************************************************************
 
-IOReturn IOService::add_driver_to_active_change ( IOPMinformee * newObject )
+static IOReturn
+settleTimerExpired (
+	OSObject * target,
+	void * arg0, void * arg1,
+	void * arg2, void * arg3 )
 {
-    if (! acquire_lock() ) 
-    {
-        return IOPMNoErr;
-    }
+	IOService * me = (IOService *) target;
+	me->settleTimerExpired();
+	return kIOReturnSuccess;
+}
 
-    switch (priv->machine_state) {
-        case kIOPM_OurChangeSetPowerState:
-        case kIOPM_ParentDownSetPowerState_Delayed:
-        case kIOPM_ParentUpSetPowerState_Delayed:
-            // one for this driver and one to prevent
-            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
-            // incoming acks from changing our state
-            IOUnlock(priv->our_lock);
-            // inform the driver
-            inform(newObject, true);
-            if (! acquire_lock() ) 
-            {
-                // put it back
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                return IOPMNoErr;
-            }
-            // are we still waiting for acks?
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            if ( priv->head_note_pendingAcks == 0 ) 
-            {
-                // no, stop the timer
-                stop_ack_timer();
-                IOUnlock(priv->our_lock);
+//*********************************************************************************
+// settle_timer_expired
+//
+// Thread call function. Holds a retain while the callout is in flight.
+//*********************************************************************************
 
-                // and now we can continue
-                all_acked();
-                return IOPMNoErr;
-            }
-            break;
-        case kIOPM_OurChangeFinish:
-        case kIOPM_ParentDownAcknowledgeChange_Delayed:
-        case kIOPM_ParentUpAcknowledgePowerChange_Delayed:
-            // one for this driver and one to prevent
-            OSAddAtomic(2, (SInt32*)&priv->head_note_pendingAcks);
-            // incoming acks from changing our state
-            IOUnlock(priv->our_lock);
-            // inform the driver
-            inform(newObject, false);
-            if (! acquire_lock() ) {
-                // put it back
-                OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-                return IOPMNoErr;
-            }
-            // are we still waiting for acks?
-            OSAddAtomic(-1, (SInt32*)&priv->head_note_pendingAcks);
-            if ( priv->head_note_pendingAcks == 0 ) {
-                // no, stop the timer
-                stop_ack_timer();
-                IOUnlock(priv->our_lock);
-
-                // and now we can continue
-                all_acked();
-                return IOPMNoErr;
-            }
-            break;
-    }
-    IOUnlock(priv->our_lock);
-    return IOPMNoErr;
-}
+static void
+settle_timer_expired ( thread_call_param_t arg0, thread_call_param_t arg1 )
+{
+	IOService * me = (IOService *) arg0;
 
+	if (gIOPMWorkLoop && gIOPMReplyQueue)
+	{
+		gIOPMWorkLoop->runAction(settleTimerExpired, me);
+		gIOPMReplyQueue->signalWorkAvailable();
+	}
+	me->release();
+}
 
 //*********************************************************************************
-// start_parent_change
-//
-// Here we begin the processing of a change note  initiated by our parent
-// which is at the head of the queue.
+// [private] start_parent_change
 //
-// It is possible for the change to be processed to completion and removed from the queue.
-// There are several possible interruptions to the processing, though, and they are:
-// we may have to wait for interested parties to acknowledge our pre-change notification,
-// we may have to wait for our controlling driver to change the hardware power state,
-// there may be a settling time after changing the hardware power state,
-// we may have to wait for interested parties to acknowledge our post-change notification,
-// we may have to wait for the acknowledgement timer expiration to substitute for the
-// acknowledgement from a failing driver.
-//*********************************************************************************
-
-IOReturn IOService::start_parent_change ( unsigned long queue_head )
-{
-    priv->head_note = queue_head;
-    priv->head_note_flags = priv-> changeList->changeNote[priv->head_note].flags;
-    priv->head_note_state =  priv->changeList->changeNote[priv->head_note].newStateNumber;
-    priv->imminentState = priv->head_note_state;
-    priv->head_note_outputFlags =  priv->changeList->changeNote[priv->head_note].outputPowerCharacter;
-    priv->head_note_domainState = priv->changeList->changeNote[priv->head_note].domainState;
-    priv->head_note_parent = priv->changeList->changeNote[priv->head_note].parent;
-    priv->head_note_capabilityFlags =  priv->changeList->changeNote[priv->head_note].capabilityFlags;
-
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartParentChange,
-                    (unsigned long)priv->head_note_state,(unsigned long)pm_vars->myCurrentState);
-
-    // if we need something and haven't told the parent, do so
-    ask_parent( priv->ourDesiredPowerState);
-
-    // power domain is lowering
-    if ( priv->head_note_state < pm_vars->myCurrentState ) 
+// Here we begin the processing of a power change initiated by our parent.
+//*********************************************************************************
+
+IOReturn IOService::start_parent_change ( const changeNoteItem * changeNote )
+{
+    fHeadNoteFlags           = changeNote->flags;
+    fHeadNoteState           = changeNote->newStateNumber;
+    fHeadNoteOutputFlags     = changeNote->outputPowerCharacter;
+    fHeadNoteDomainState     = changeNote->domainState;
+    fHeadNoteParent          = changeNote->parent;
+    fHeadNoteCapabilityFlags = changeNote->capabilityFlags;
+
+	PM_ASSERT_IN_GATE();
+    OUR_PMLog( kPMLogStartParentChange, fHeadNoteState, fCurrentPowerState );
+
+    // Power domain is lowering power
+    if ( fHeadNoteState < fCurrentPowerState )
     {
-        setParentInfo(priv->changeList->changeNote[priv->head_note].singleParentState,priv->head_note_parent);
-    	priv->initial_change = false;
-    	// tell apps and kernel clients
-        priv->machine_state = kIOPM_ParentDownTellPriorityClientsPowerDown_Immediate;
+		setParentInfo(
+			changeNote->singleParentState,
+			fHeadNoteParent, true );
 
-        // are we waiting for responses?
-        if ( tellChangeDown1(priv->head_note_state) ) 
-        {
-            // no, notify priority clients
-            return ParentDownTellPriorityClientsPowerDown_Immediate();
-        }
-        // yes
+    	// tell apps and kernel clients
+    	fInitialChange = false;
+        fMachineState = kIOPM_ParentDownTellPriorityClientsPowerDown;
+		tellChangeDown1(fHeadNoteState);
         return IOPMWillAckLater;
     }
 
-    // parent is raising power, we may or may not
-    if ( priv->head_note_state > pm_vars->myCurrentState ) 
+    // Power domain is raising power
+    if ( fHeadNoteState > fCurrentPowerState )
     {
-        if ( priv->ourDesiredPowerState > pm_vars->myCurrentState ) 
+		IOPMPowerState * powerStatePtr;
+
+        if ( fDesiredPowerState > fCurrentPowerState )
         {
-            if ( priv->ourDesiredPowerState < priv->head_note_state ) 
+            if ( fDesiredPowerState < fHeadNoteState )
             {
-                // we do, but not all the way
-                priv->head_note_state = priv->ourDesiredPowerState;
-                priv->imminentState = priv->head_note_state;
-                priv->head_note_outputFlags =   pm_vars->thePowerStates[priv->head_note_state].outputPowerCharacter;
-                priv->head_note_capabilityFlags =   pm_vars->thePowerStates[priv->head_note_state].capabilityFlags;
-                pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAmendParentChange,(unsigned long)priv->head_note_state,0);
+                // We power up, but not all the way
+                fHeadNoteState = fDesiredPowerState;
+				powerStatePtr = &fPowerStates[fHeadNoteState];
+                fHeadNoteOutputFlags = powerStatePtr->outputPowerCharacter;
+                fHeadNoteCapabilityFlags = powerStatePtr->capabilityFlags;
+                OUR_PMLog(kPMLogAmendParentChange, fHeadNoteState, 0);
              }
         } else {
-            // we don't
-            priv->head_note_state = pm_vars->myCurrentState;
-            priv->imminentState = priv->head_note_state;
-            priv->head_note_outputFlags =   pm_vars->thePowerStates[priv->head_note_state].outputPowerCharacter;
-            priv->head_note_capabilityFlags =   pm_vars->thePowerStates[priv->head_note_state].capabilityFlags;
-            pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAmendParentChange,(unsigned long)priv->head_note_state,0);
+            // We don't need to change
+            fHeadNoteState = fCurrentPowerState;
+			powerStatePtr = &fPowerStates[fHeadNoteState];
+            fHeadNoteOutputFlags = powerStatePtr->outputPowerCharacter;
+            fHeadNoteCapabilityFlags = powerStatePtr->capabilityFlags;
+            OUR_PMLog(kPMLogAmendParentChange, fHeadNoteState, 0);
         }
     }
 
-    if ( (priv->head_note_state > pm_vars->myCurrentState) &&
-                    (priv->head_note_flags & IOPMDomainDidChange) ) 
-    {
-        // changing up
-        priv->initial_change = false;
-        priv->machine_state = kIOPM_ParentUpSetPowerState_Delayed;
-        if (  notifyAll(true) == IOPMAckImplied ) {
-            return ParentUpSetPowerState_Immediate();
-        }
-        // they didn't all ack
+	if ((fHeadNoteState > fCurrentPowerState) &&
+		(fHeadNoteFlags & IOPMDomainDidChange))
+	{
+        // Parent did change up - start our change up
+        fInitialChange = false;
+        fMachineState = kIOPM_ParentUpSetPowerState;
+		notifyAll( true );
         return IOPMWillAckLater;
     }
 
     all_done();
-    // a null change or power will go up
     return IOPMAckImplied;
 }
 
-
 //*********************************************************************************
-// start_our_change
-//
-// Here we begin the processing of a change note  initiated by us
-// which is at the head of the queue.
+// [private] start_our_change
 //
-// It is possible for the change to be processed to completion and removed from the queue.
-// There are several possible interruptions to the processing, though, and they are:
-// we may have to wait for interested parties to acknowledge our pre-change notification,
-// changes initiated by the parent will wait in the middle for powerStateDidChange,
-// we may have to wait for our controlling driver to change the hardware power state,
-// there may be a settling time after changing the hardware power state,
-// we may have to wait for interested parties to acknowledge our post-change notification,
-// we may have to wait for the acknowledgement timer expiration to substitute for the
-// acknowledgement from a failing driver.
+// Here we begin the processing of a power change initiated by us.
 //*********************************************************************************
 
-void IOService::start_our_change ( unsigned long queue_head )
+void IOService::start_our_change ( const changeNoteItem * changeNote )
 {
-    priv->head_note = queue_head;
-    priv->head_note_flags =  priv->changeList->changeNote[priv->head_note].flags;
-    priv->head_note_state =  priv->changeList->changeNote[priv->head_note].newStateNumber;
-    priv->imminentState = priv->head_note_state;
-    priv->head_note_outputFlags =  priv->changeList->changeNote[priv->head_note].outputPowerCharacter;
-    priv->head_note_capabilityFlags =  priv->changeList->changeNote[priv->head_note].capabilityFlags;
+    fHeadNoteFlags           = changeNote->flags;
+    fHeadNoteState           = changeNote->newStateNumber;
+    fHeadNoteOutputFlags     = changeNote->outputPowerCharacter;
+    fHeadNoteCapabilityFlags = changeNote->capabilityFlags;
 
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartDeviceChange,
-                (unsigned long)priv->head_note_state,(unsigned long)pm_vars->myCurrentState);
+	PM_ASSERT_IN_GATE();
+
+    OUR_PMLog( kPMLogStartDeviceChange, fHeadNoteState, fCurrentPowerState );
 
     // can our driver switch to the new state?
-    if ( priv->head_note_capabilityFlags & IOPMNotAttainable ) 
+    if (( fHeadNoteCapabilityFlags & IOPMNotAttainable ) ||
+		((fMaxCapability < fHeadNoteState) && (!fWeAreRoot)))
     {
-        // no, ask the parent to do it then
-        if ( !  priv->we_are_root )
-        {
-            ask_parent(priv->head_note_state);
-        }
         // mark the change note un-actioned
-        priv-> head_note_flags |= IOPMNotDone;
-        // and we're done
-        all_done();
-        return;
-    }
-    
-    // is there enough power in the domain?
-    if ( (pm_vars->maxCapability < priv->head_note_state) && (!  priv->we_are_root) ) 
-    {
-        // no, ask the parent to raise it
-        if ( !  priv->we_are_root ) 
+        fHeadNoteFlags |= IOPMNotDone;
+
+        // no, ask the parent to do it then
+        if ( !fWeAreRoot )
         {
-            ask_parent(priv->head_note_state);
+            ask_parent(fHeadNoteState);
         }
-        // no, mark the change note un-actioned
-        priv->head_note_flags |= IOPMNotDone;
-        // and we're done
-        // till the parent raises power
         all_done();
         return;
     }
 
-    if ( !  priv->initial_change ) 
+    if ( !fInitialChange )
     {
-        if ( priv->head_note_state == pm_vars->myCurrentState ) 
+        if ( fHeadNoteState == fCurrentPowerState )
         {
             // we initiated a null change; forget it
             all_done();
             return;
         }
     }
-    priv->initial_change = false;
+    fInitialChange = false;
 
     // dropping power?
-    if ( priv->head_note_state < pm_vars->myCurrentState ) 
+    if ( fHeadNoteState < fCurrentPowerState )
     {
         // yes, in case we have to wait for acks
-        priv->machine_state = kIOPM_OurChangeTellClientsPowerDown;
-        pm_vars->doNotPowerDown = false;
+        fMachineState = kIOPM_OurChangeTellClientsPowerDown;
+        fDoNotPowerDown = false;
 
         // ask apps and kernel clients if we can drop power
-        pm_vars->outofbandparameter = kNotifyApps;
-        if ( askChangeDown(priv->head_note_state) ) 
-        {
-            // don't have to wait, did any clients veto?
-            if ( pm_vars->doNotPowerDown ) 
-            {
-                // yes, rescind the warning
-                tellNoChangeDown(priv->head_note_state);
-                // mark the change note un-actioned
-                priv-> head_note_flags |= IOPMNotDone;
-                // and we're done
-                all_done();
-            } else {
-                // no, tell'em we're dropping power
-                OurChangeTellClientsPowerDown();
-            }
-        }
+        fOutOfBandParameter = kNotifyApps;
+		askChangeDown(fHeadNoteState);
     } else {
-        // we are raising power
-        if ( !  priv->we_are_root ) 
-        {
-            // if this changes our power requirement, tell the parent
-            ask_parent(priv->head_note_state);
-        }
         // in case they don't all ack
-        priv->machine_state = kIOPM_OurChangeSetPowerState;
-
+        fMachineState = kIOPM_OurChangeSetPowerState;
         // notify interested drivers and children
-        if ( notifyAll(true) == IOPMAckImplied ) 
-        {
-            OurChangeSetPowerState();
-        }
+        notifyAll(true);
     }
 }
 
-
 //*********************************************************************************
-// ask_parent
+// [private] ask_parent
 //
 // Call the power domain parent to ask for a higher power state in the domain
 // or to suggest a lower power state.
@@ -3900,48 +4151,55 @@ void IOService::start_our_change ( unsigned long queue_head )
 
 IOReturn IOService::ask_parent ( unsigned long requestedState )
 {
-    OSIterator                          *iter;
-    OSObject                            *next;
-    IOPowerConnection                   *connection;
-    IOService                           *parent;
-    unsigned long                       ourRequest = pm_vars->thePowerStates[requestedState].inputPowerRequirement;
+    OSIterator *			iter;
+    OSObject *				next;
+    IOPowerConnection *		connection;
+    IOService *				parent;
+	const IOPMPowerState *	powerStatePtr;
+    unsigned long			ourRequest;
+
+	PM_ASSERT_IN_GATE();
+    if (requestedState >= fNumberOfPowerStates)
+        return IOPMNoErr;
 
-    if ( pm_vars->thePowerStates[requestedState].capabilityFlags & (kIOPMChildClamp | kIOPMPreventIdleSleep) ) 
+	powerStatePtr = &fPowerStates[requestedState];
+	ourRequest    = powerStatePtr->inputPowerRequirement;
+
+    if ( powerStatePtr->capabilityFlags & (kIOPMChildClamp | kIOPMPreventIdleSleep) )
     {
         ourRequest |= kIOPMPreventIdleSleep;
     }
-    if ( pm_vars->thePowerStates[requestedState].capabilityFlags & (kIOPMChildClamp2 | kIOPMPreventSystemSleep) ) 
+    if ( powerStatePtr->capabilityFlags & (kIOPMChildClamp2 | kIOPMPreventSystemSleep) )
     {
         ourRequest |= kIOPMPreventSystemSleep;
     }
-    
+
     // is this a new desire?
-    if ( priv->previousRequest == ourRequest )
+    if ( fPreviousRequest == ourRequest )
     {	
         // no, the parent knows already, just return
-        return IOPMNoErr;				
+        return IOPMNoErr;
     }
 
-    if (  priv->we_are_root ) 
+    if ( fWeAreRoot )
     {
         return IOPMNoErr;
     }
-    priv->previousRequest =  ourRequest;
+    fPreviousRequest = ourRequest;
 
     iter = getParentIterator(gIOPowerPlane);
-
-    if ( iter ) 
+    if ( iter )
     {
-        while ( (next = iter->getNextObject()) ) 
+        while ( (next = iter->getNextObject()) )
         {
-            if ( (connection = OSDynamicCast(IOPowerConnection,next)) ) 
+            if ( (connection = OSDynamicCast(IOPowerConnection, next)) )
             {
                 parent = (IOService *)connection->copyParentEntry(gIOPowerPlane);
                 if ( parent ) {
-                    if ( parent->requestPowerDomainState(ourRequest,connection,IOPMLowestState)!= IOPMNoErr ) 
+                    if ( parent->requestPowerDomainState(
+						ourRequest, connection, IOPMLowestState) != IOPMNoErr )
                     {
-                        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogRequestDenied,
-                                                                (unsigned long)priv->previousRequest,0);
+                        OUR_PMLog(kPMLogRequestDenied, fPreviousRequest, 0);
                     }
                     parent->release();
                 }
@@ -3953,96 +4211,110 @@ IOReturn IOService::ask_parent ( unsigned long requestedState )
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// instruct_driver
-//
-// Call the controlling driver and have it change the power state of the
-// hardware.  If it returns IOPMAckImplied, the change is complete, and
-// we return IOPMAckImplied.  Otherwise, it will ack when the change
-// is done; we return IOPMWillAckLater.
+// [private] notifyControllingDriver
 //*********************************************************************************
-IOReturn IOService::instruct_driver ( unsigned long newState )
-{
-    IOReturn delay;
 
-    // can our driver switch to the desired state?
-    if (  pm_vars->thePowerStates[newState].capabilityFlags & IOPMNotAttainable ) 
-    {
-        // no, so don't try
-        return IOPMAckImplied;
-    }
-
-    priv->driver_timer = -1;
+bool IOService::notifyControllingDriver ( void )
+{
+	DriverCallParam *	param;
+	unsigned long		powerState;
+
+	PM_ASSERT_IN_GATE();
+	assert( fDriverCallBusy == false );
+	assert( fDriverCallParamCount == 0  );
+	assert( fControllingDriver );
+
+	powerState = fHeadNoteState;
+    if (fPowerStates[powerState].capabilityFlags & IOPMNotAttainable )
+        return false;	// state not attainable
+
+	param = (DriverCallParam *) fDriverCallParamPtr;
+	if (!param)
+	{
+		param = IONew(DriverCallParam, 1);
+		if (!param)
+			return false;	// no memory
+
+		fDriverCallParamPtr   = (void *) param;
+		fDriverCallParamSlots = 1;
+	}
 
-    // yes, instruct it
-    IOLockUnlock(priv->our_lock);
-    OUR_PMLog(          kPMLogProgramHardware, (UInt32) this, newState);
-    delay = pm_vars->theControllingDriver->setPowerState( newState,this );
-    OUR_PMLog((UInt32) -kPMLogProgramHardware, (UInt32) this, (UInt32) delay);
-    IOLockLock(priv->our_lock);
+	param->Target = fControllingDriver;
+	fDriverCallParamCount = 1;
 
-    // it finished
-    if ( delay == IOPMAckImplied ) 
-    {
-        priv->driver_timer = 0;
-        return IOPMAckImplied;
-    }
+	fDriverTimer = -1;
 
-    // it acked behind our back
-    if ( priv->driver_timer == 0 ) 
-    {
-        return IOPMAckImplied;
-    }
+	// Machine state for this object will stall waiting for a reply
+	// from the callout thread.
 
-    // somebody goofed
-    if ( delay < 0 ) 
-    {
-        return IOPMAckImplied;
-    }
-    
-    // it didn't finish
-    priv->driver_timer = (delay / ( ACK_TIMER_PERIOD / ns_per_us )) + 1;
-    return IOPMWillAckLater;
+	PM_LOCK();
+	fDriverCallBusy = true;
+	PM_UNLOCK();
+	thread_call_enter( fDriverCallEntry );
+	return true;
 }
 
-
 //*********************************************************************************
-// acquire_lock
-//
-// We are acquiring the lock we use to protect our queue head from
-// simutaneous access by a thread which calls acknowledgePowerStateChange
-// or acknowledgeSetPowerState and the ack timer expiration thread.
-// Return TRUE if we acquire the lock, and the queue head didn't change
-// while we were acquiring the lock (and maybe blocked).
-// If there is no queue head, or it changes while we are blocked,
-// return FALSE with the lock unlocked.
+// [private] notifyControllingDriverDone
 //*********************************************************************************
 
-bool IOService::acquire_lock ( void )
+void IOService::notifyControllingDriverDone( void )
 {
-    long current_change_note;
-
-    current_change_note = priv->head_note;
-    if ( current_change_note == -1 ) {
-        return FALSE;
-    }
+	DriverCallParam *	param;
+	IOReturn			result;
+
+	PM_ASSERT_IN_GATE();
+	param = (DriverCallParam *) fDriverCallParamPtr;
+
+	assert( fDriverCallBusy == false );
+	assert( fMachineState == kIOPM_DriverThreadCallDone );
+
+	if (param)
+	{
+		assert(fDriverCallParamCount == 1);
+		
+		// the return value from setPowerState()
+		result = param->Result;
+
+		if ((result == IOPMAckImplied) || (result < 0))
+		{
+			// child return IOPMAckImplied
+			fDriverTimer = 0;
+		}
+		else if (fDriverTimer)
+		{
+			assert(fDriverTimer == -1);
+
+            // Driver has not acked, and has returned a positive result.
+            // Enforce a minimum permissible timeout value.
+            // Make the min value large enough so timeout is less likely
+            // to occur if a driver misinterpreted that the return value
+            // should be in microsecond units.  And make it large enough
+            // to be noticeable if a driver neglects to ack.
+
+            if (result < kMinAckTimeoutTicks)
+                result = kMinAckTimeoutTicks;
+
+            fDriverTimer = (result / (ACK_TIMER_PERIOD / ns_per_us)) + 1;
+		}
+		// else, child has already acked and driver_timer reset to 0.
+
+		fDriverCallParamCount = 0;
+
+		if ( fDriverTimer )
+		{
+			OUR_PMLog(kPMLogStartAckTimer, 0, 0);
+			start_ack_timer();
+		}
+	}
 
-    IOTakeLock(priv->our_lock);
-    if ( current_change_note == priv->head_note ) 
-    {
-        return TRUE;
-    } else {
-        // we blocked and something changed radically
-        // so there's nothing to do any more
-        IOUnlock(priv->our_lock);
-        return FALSE;
-    }
+	// Hop back to original machine state path.
+	fMachineState = fNextMachineState;
 }
 
-
 //*********************************************************************************
-// askChangeDown
+// [public virtual] askChangeDown
 //
 // Ask registered applications and kernel clients if we can change to a lower
 // power state.
@@ -4055,12 +4327,11 @@ bool IOService::acquire_lock ( void )
 
 bool IOService::askChangeDown ( unsigned long stateNum )
 {
-    return tellClientsWithResponse(kIOMessageCanDevicePowerOff);
+    return tellClientsWithResponse( kIOMessageCanDevicePowerOff );
 }
 
-
 //*********************************************************************************
-// tellChangeDown1
+// [public] tellChangeDown1
 //
 // Notify registered applications and kernel clients that we are definitely
 // dropping power.
@@ -4070,13 +4341,12 @@ bool IOService::askChangeDown ( unsigned long stateNum )
 
 bool IOService::tellChangeDown1 ( unsigned long stateNum )
 {
-    pm_vars->outofbandparameter = kNotifyApps;
+    fOutOfBandParameter = kNotifyApps;
     return tellChangeDown(stateNum);
 }
 
-
 //*********************************************************************************
-// tellChangeDown2
+// [public] tellChangeDown2
 //
 // Notify priority clients that we are definitely dropping power.
 //
@@ -4085,13 +4355,12 @@ bool IOService::tellChangeDown1 ( unsigned long stateNum )
 
 bool IOService::tellChangeDown2 ( unsigned long stateNum )
 {
-    pm_vars->outofbandparameter = kNotifyPriority;
+    fOutOfBandParameter = kNotifyPriority;
     return tellChangeDown(stateNum);
 }
 
-
 //*********************************************************************************
-// tellChangeDown
+// [public virtual] tellChangeDown
 //
 // Notify registered applications and kernel clients that we are definitely
 // dropping power.
@@ -4104,12 +4373,69 @@ bool IOService::tellChangeDown2 ( unsigned long stateNum )
 
 bool IOService::tellChangeDown ( unsigned long stateNum )
 {
-    return tellClientsWithResponse(kIOMessageDeviceWillPowerOff);
+    return tellClientsWithResponse( kIOMessageDeviceWillPowerOff );
+}
+
+//*********************************************************************************
+// cleanClientResponses
+//
+//*********************************************************************************
+
+static void logAppTimeouts ( OSObject * object, void * context)
+{
+    struct context  *theContext = (struct context *)context;
+    OSObject        *flag;
+
+    if( !OSDynamicCast( IOService, object) ) {
+        flag = theContext->responseFlags->getObject(theContext->counter);
+        if (kOSBooleanTrue != flag)
+        {
+            OSString * clientID = 0;
+            theContext->us->messageClient(theContext->msgType, object, &clientID);
+            PM_ERROR(theContext->errorLog, clientID ? clientID->getCStringNoCopy() : "");
+            if (clientID)
+                clientID->release();
+        }
+        theContext->counter += 1;
+    }
 }
 
+void IOService::cleanClientResponses ( bool logErrors )
+{
+    struct context theContext;
+
+    if (logErrors && fResponseArray) {
+        theContext.responseFlags    = fResponseArray;
+        theContext.serialNumber     = fSerialNumber;
+        theContext.counter          = 0;
+        theContext.msgType          = kIOMessageCopyClientID;
+        theContext.us               = this;
+        theContext.maxTimeRequested = 0;
+        theContext.stateNumber      = fHeadNoteState;
+        theContext.stateFlags       = fHeadNoteCapabilityFlags;
+        theContext.errorLog         = "PM notification timeout (%s)\n";
+
+        switch ( fOutOfBandParameter ) {
+            case kNotifyApps:
+                applyToInterested(gIOAppPowerStateInterest, logAppTimeouts, (void *) &theContext);
+            case kNotifyPriority:
+            default:
+                break;
+        }
+    }
+
+    if (fResponseArray) 
+    {
+        // get rid of this stuff
+        fResponseArray->release();
+        fResponseArray = NULL;
+    }
+
+    return;
+}
 
 //*********************************************************************************
-// tellClientsWithResponse
+// [public] tellClientsWithResponse
 //
 // Notify registered applications and kernel clients that we are definitely
 // dropping power.
@@ -4119,123 +4445,102 @@ bool IOService::tellChangeDown ( unsigned long stateNum )
 
 bool IOService::tellClientsWithResponse ( int messageType )
 {
-    struct context                          theContext;
-    AbsoluteTime                            deadline;
-    OSBoolean                               *aBool;
+    struct context	theContext;
+
+	PM_ASSERT_IN_GATE();
 
-    pm_vars->responseFlags = OSArray::withCapacity( 1 );
-    pm_vars->serialNumber += 1;
+    fResponseArray = OSArray::withCapacity( 1 );
+    fSerialNumber += 1;
     
-    theContext.responseFlags = pm_vars->responseFlags;
-    theContext.serialNumber = pm_vars->serialNumber;
-    theContext.flags_lock = priv->flags_lock;
-    theContext.counter = 1;
+    theContext.responseFlags = fResponseArray;
+    theContext.serialNumber = fSerialNumber;
+    theContext.counter = 0;
     theContext.msgType = messageType;
     theContext.us = this;
     theContext.maxTimeRequested = 0;
-    theContext.stateNumber = priv->head_note_state;
-    theContext.stateFlags = priv->head_note_capabilityFlags;
+    theContext.stateNumber = fHeadNoteState;
+    theContext.stateFlags = fHeadNoteCapabilityFlags;
 
-    IOLockLock(priv->flags_lock);
-
-    // position zero is false to
-    // prevent allowCancelCommon from succeeding
-    aBool = OSBoolean::withBoolean(false);
-    theContext.responseFlags->setObject(0,aBool);
-    aBool->release();
-    IOLockUnlock(priv->flags_lock);
-
-    switch ( pm_vars->outofbandparameter ) {
+    switch ( fOutOfBandParameter ) {
         case kNotifyApps:
-            applyToInterested(gIOAppPowerStateInterest,tellAppWithResponse,(void *)&theContext);
-            applyToInterested(gIOGeneralInterest,tellClientWithResponse,(void *)&theContext);
+            applyToInterested(gIOAppPowerStateInterest,
+				pmTellAppWithResponse, (void *)&theContext);
+            applyToInterested(gIOGeneralInterest,
+				pmTellClientWithResponse, (void *)&theContext);
             break;
         case kNotifyPriority:
-            applyToInterested(gIOPriorityPowerStateInterest,tellClientWithResponse,(void *)&theContext);
+            applyToInterested(gIOPriorityPowerStateInterest,
+				pmTellClientWithResponse, (void *)&theContext);
             break;
     }
-
-    if (! acquire_lock() ) 
-    {
-        return true;
-    }
-    IOLockLock(priv->flags_lock);
-    // now fix position zero
-    aBool = OSBoolean::withBoolean(true);
-    theContext.responseFlags->replaceObject(0,aBool);
-    aBool->release();
-    IOLockUnlock(priv->flags_lock);
     
     // do we have to wait for somebody?
-    if ( ! checkForDone() ) 
+    if ( !checkForDone() )
     {
-        // yes, start the ackTimer
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogStartAckTimer,theContext.maxTimeRequested,0);
-        clock_interval_to_deadline(theContext.maxTimeRequested / 1000, kMillisecondScale, &deadline);
-        
-        thread_call_enter_delayed(priv->ackTimer, deadline);
-    
-        IOUnlock(priv->our_lock);			
+        OUR_PMLog(kPMLogStartAckTimer,theContext.maxTimeRequested, 0);
+		start_ack_timer( theContext.maxTimeRequested / 1000, kMillisecondScale );	
         return false;
     }
-    
-    IOUnlock(priv->our_lock);
-    IOLockLock(priv->flags_lock);
-    
+
     // everybody responded
-    pm_vars->responseFlags->release();
-    pm_vars->responseFlags = NULL;
-    IOLockUnlock(priv->flags_lock);
+    fResponseArray->release();
+    fResponseArray = NULL;
+    // cleanClientResponses(false);
     
     return true;
 }
 
-
 //*********************************************************************************
-// tellAppWithResponse
+// [static private] pmTellAppWithResponse
 //
 // We send a message to an application, and we expect a response, so we compute a
 // cookie we can identify the response with.
 //*********************************************************************************
-void tellAppWithResponse ( OSObject * object, void * context)
+
+void IOService::pmTellAppWithResponse ( OSObject * object, void * context )
 {
-    struct context                      *theContext = (struct context *)context;
-    OSBoolean                           *aBool;
-    IOPMprot 				*pm_vars = theContext->us->pm_vars;
+    struct context *	theContext = (struct context *) context;
+    IOServicePM *		pwrMgt = theContext->us->pwrMgt;
+    AbsoluteTime        now;
 
-    if( OSDynamicCast( IOService, object) ) 
+    if( OSDynamicCast( IOService, object) )
     {
-	// Automatically 'ack' in kernel clients
-        IOLockLock(theContext->flags_lock);
-        aBool = OSBoolean::withBoolean(true);
-        theContext->responseFlags->setObject(theContext->counter,aBool);
-        aBool->release();
-        IOLockUnlock(theContext->flags_lock);
-
-	const char *who = ((IOService *) object)->getName();
-	pm_vars->thePlatform->PMLog(who,
-	    kPMLogClientAcknowledge, theContext->msgType, * (UInt32 *) object);
+		// Automatically 'ack' in kernel clients
+        theContext->responseFlags->setObject(theContext->counter, kOSBooleanTrue);
+
+		const char *who = ((IOService *) object)->getName();
+		fPlatform->PMLog(who,
+			kPMLogClientAcknowledge, theContext->msgType, * (UInt32 *) object);
     } else {
         UInt32 refcon = ((theContext->serialNumber & 0xFFFF)<<16)
-	              +  (theContext->counter & 0xFFFF);
-        IOLockLock(theContext->flags_lock);
-        aBool = OSBoolean::withBoolean(false);
-        theContext->responseFlags->setObject(theContext->counter,aBool);
-        aBool->release();
-        IOLockUnlock(theContext->flags_lock);
-
-	OUR_PMLog(kPMLogAppNotify, theContext->msgType, refcon);
-        theContext->us->messageClient(theContext->msgType,object,(void *)refcon);
-        if ( theContext->maxTimeRequested < k30seconds ) 
+	                  + (theContext->counter & 0xFFFF);
+		OUR_PMLog(kPMLogAppNotify, theContext->msgType, refcon);
+
+#if LOG_APP_RESPONSE_TIMES
+        OSNumber * num;
+        clock_get_uptime(&now);
+        num = OSNumber::withNumber(AbsoluteTime_to_scalar(&now), sizeof(uint64_t) * 8);
+        if (num)
+        {
+            theContext->responseFlags->setObject(theContext->counter, num);
+            num->release();
+        }
+        else
+#endif
+        theContext->responseFlags->setObject(theContext->counter, kOSBooleanFalse);
+
+        theContext->us->messageClient(theContext->msgType, object, (void *)refcon);
+        if ( theContext->maxTimeRequested < k30seconds )
         {
             theContext->maxTimeRequested = k30seconds;
         }
+
+        theContext->counter += 1;
     }
-    theContext->counter += 1;
 }
 
 //*********************************************************************************
-// tellClientWithResponse
+// [static private] pmTellClientWithResponse
 //
 // We send a message to an in-kernel client, and we expect a response, so we compute a
 // cookie we can identify the response with.
@@ -4244,33 +4549,29 @@ void tellAppWithResponse ( OSObject * object, void * context)
 // in the passed struct that it is currently ready, we won't wait for it to prepare.
 // If it tells us via the return code in the struct that it does need time, we will chill.
 //*********************************************************************************
-void tellClientWithResponse ( OSObject * object, void * context)
+
+void IOService::pmTellClientWithResponse ( OSObject * object, void * context )
 {
     struct context                          *theContext = (struct context *)context;
     IOPowerStateChangeNotification          notify;
     UInt32                                  refcon;
     IOReturn                                retCode;
-    OSBoolean                               *aBool;
     OSObject                                *theFlag;
-    
+
     refcon = ((theContext->serialNumber & 0xFFFF)<<16) + (theContext->counter & 0xFFFF);
-    IOLockLock(theContext->flags_lock);
-    aBool = OSBoolean::withBoolean(false);
-    theContext->responseFlags->setObject(theContext->counter,aBool);
-    aBool->release();
-    IOLockUnlock(theContext->flags_lock);
+    theContext->responseFlags->setObject(theContext->counter, kOSBooleanFalse);
 
-    IOPMprot *pm_vars = theContext->us->pm_vars;
+    IOServicePM * pwrMgt = theContext->us->pwrMgt;
     if (gIOKitDebug & kIOLogPower) {
-	OUR_PMLog(kPMLogClientNotify, refcon, (UInt32) theContext->msgType);
-	if (OSDynamicCast(IOService, object)) {
-	    const char *who = ((IOService *) object)->getName();
-	    pm_vars->thePlatform->PMLog(who,
-		    kPMLogClientNotify, * (UInt32 *) object, (UInt32) object);
-	} else if (OSDynamicCast(_IOServiceInterestNotifier, object)) {
-	    _IOServiceInterestNotifier *n = (_IOServiceInterestNotifier *) object;
-	    OUR_PMLog(kPMLogClientNotify, (UInt32) n->handler, 0);
-	}
+		OUR_PMLog(kPMLogClientNotify, refcon, (UInt32) theContext->msgType);
+		if (OSDynamicCast(IOService, object)) {
+			const char *who = ((IOService *) object)->getName();
+			fPlatform->PMLog(who,
+				kPMLogClientNotify, * (UInt32 *) object, (UInt32) object);
+		} else if (OSDynamicCast(_IOServiceInterestNotifier, object)) {
+			_IOServiceInterestNotifier *n = (_IOServiceInterestNotifier *) object;
+			OUR_PMLog(kPMLogClientNotify, (UInt32) n->handler, 0);
+		}
     }
 
     notify.powerRef = (void *)refcon;
@@ -4278,52 +4579,36 @@ void tellClientWithResponse ( OSObject * object, void * context)
     notify.stateNumber = theContext->stateNumber;
     notify.stateFlags = theContext->stateFlags;
     retCode = theContext->us->messageClient(theContext->msgType,object,(void *)&notify);
-    if ( retCode == kIOReturnSuccess ) 
+    if ( retCode == kIOReturnSuccess )
     {
-        if ( notify.returnValue == 0 ) 
+        if ( notify.returnValue == 0 )
         {
             // client doesn't want time to respond
-            IOLockLock(theContext->flags_lock);
-            aBool = OSBoolean::withBoolean(true);
-            // so set its flag true
-            theContext->responseFlags->replaceObject(theContext->counter,aBool);
-            aBool->release();
-            IOLockUnlock(theContext->flags_lock);
-	    OUR_PMLog(kPMLogClientAcknowledge, refcon, (UInt32) object);
+            theContext->responseFlags->replaceObject(theContext->counter, kOSBooleanTrue);
+			OUR_PMLog(kPMLogClientAcknowledge, refcon, (UInt32) object);
         } else {
-            IOLockLock(theContext->flags_lock);
-            
             // it does want time, and it hasn't responded yet
             theFlag = theContext->responseFlags->getObject(theContext->counter);
-            if ( theFlag != 0 ) 
+            if ( kOSBooleanTrue != theFlag ) 
             {
-                if ( ((OSBoolean *)theFlag)->isFalse() ) 
+                // so note its time requirement
+                if ( theContext->maxTimeRequested < notify.returnValue ) 
                 {
-                    // so note its time requirement
-                    if ( theContext->maxTimeRequested < notify.returnValue ) 
-                    {
-                        theContext->maxTimeRequested = notify.returnValue;
-                    }
+                    theContext->maxTimeRequested = notify.returnValue;
                 }
             }
-            IOLockUnlock(theContext->flags_lock);
         }
     } else {
-	OUR_PMLog(kPMLogClientAcknowledge, refcon, 0);
+		OUR_PMLog(kPMLogClientAcknowledge, refcon, 0);
         // not a client of ours
-        IOLockLock(theContext->flags_lock);
         // so we won't be waiting for response
-        aBool = OSBoolean::withBoolean(true);
-        theContext->responseFlags->replaceObject(theContext->counter,aBool);
-        aBool->release();
-        IOLockUnlock(theContext->flags_lock);
+        theContext->responseFlags->replaceObject(theContext->counter, kOSBooleanTrue);
     }
     theContext->counter += 1;
 }
 
-
 //*********************************************************************************
-// tellNoChangeDown
+// [public virtual] tellNoChangeDown
 //
 // Notify registered applications and kernel clients that we are not
 // dropping power.
@@ -4334,12 +4619,11 @@ void tellClientWithResponse ( OSObject * object, void * context)
 
 void IOService::tellNoChangeDown ( unsigned long )
 {
-    return tellClients(kIOMessageDeviceWillNotPowerOff);
+    return tellClients( kIOMessageDeviceWillNotPowerOff );
 }
 
-
 //*********************************************************************************
-// tellChangeUp
+// [public virtual] tellChangeUp
 //
 // Notify registered applications and kernel clients that we are raising power.
 //
@@ -4349,12 +4633,11 @@ void IOService::tellNoChangeDown ( unsigned long )
 
 void IOService::tellChangeUp ( unsigned long )
 {
-    return tellClients(kIOMessageDeviceHasPoweredOn);
+    return tellClients( kIOMessageDeviceHasPoweredOn );
 }
 
-
 //*********************************************************************************
-// tellClients
+// [public] tellClients
 //
 // Notify registered applications and kernel clients of something.
 //*********************************************************************************
@@ -4365,116 +4648,129 @@ void IOService::tellClients ( int messageType )
 
     theContext.msgType = messageType;
     theContext.us = this;
-    theContext.stateNumber = priv->head_note_state;
-    theContext.stateFlags = priv->head_note_capabilityFlags;
+    theContext.stateNumber = fHeadNoteState;
+    theContext.stateFlags = fHeadNoteCapabilityFlags;
 
-    applyToInterested(gIOAppPowerStateInterest,tellClient,(void *)&theContext);
-    applyToInterested(gIOGeneralInterest,tellClient,(void *)&theContext);
+    applyToInterested(gIOPriorityPowerStateInterest,tellClient,(void *)&theContext);
+    applyToInterested(gIOAppPowerStateInterest,tellClient, (void *)&theContext);
+    applyToInterested(gIOGeneralInterest,tellClient, (void *)&theContext);
 }
 
-
 //*********************************************************************************
-// tellClient
+// [global] tellClient
 //
 // Notify a registered application or kernel client of something.
 //*********************************************************************************
-void tellClient ( OSObject * object, void * context)
+
+void tellClient ( OSObject * object, void * context )
 {
-    struct context                              *theContext = (struct context *)context;
-    IOPowerStateChangeNotification              notify;
+    struct context *				theContext = (struct context *) context;
+    IOPowerStateChangeNotification	notify;
 
     notify.powerRef	= (void *) 0;
     notify.returnValue	= 0;
     notify.stateNumber	= theContext->stateNumber;
     notify.stateFlags	= theContext->stateFlags;
 
-    theContext->us->messageClient(theContext->msgType,object, &notify);
+    theContext->us->messageClient(theContext->msgType, object, &notify);
 }
 
+//*********************************************************************************
+// [private] checkForDone
+//*********************************************************************************
 
-// **********************************************************************************
-// checkForDone
-//
-// **********************************************************************************
 bool IOService::checkForDone ( void )
 {
-    int                                         i = 0;
-    OSObject                                    *theFlag;
+    int			i = 0;
+    OSObject *	theFlag;
 
-    IOLockLock(priv->flags_lock);
-    if ( pm_vars->responseFlags == NULL ) 
+    if ( fResponseArray == NULL )
     {
-        IOLockUnlock(priv->flags_lock);
         return true;
     }
     
-    for ( i = 0; ; i++ ) 
+    for ( i = 0; ; i++ )
     {
-        theFlag = pm_vars->responseFlags->getObject(i);
-        if ( theFlag == NULL ) 
+        theFlag = fResponseArray->getObject(i);
+        if ( theFlag == NULL )
         {
             break;
         }
-        if ( ((OSBoolean *)theFlag)->isFalse() ) 
+        if ( kOSBooleanTrue != theFlag ) 
         {
-            IOLockUnlock(priv->flags_lock);
             return false;
         }
     }
-    IOLockUnlock(priv->flags_lock);
     return true;
 }
 
+//*********************************************************************************
+// [public] responseValid
+//*********************************************************************************
 
-// **********************************************************************************
-// responseValid
-//
-// **********************************************************************************
-bool IOService::responseValid ( unsigned long x )
-{
-    UInt16 serialComponent;
-    UInt16 ordinalComponent;
-    OSObject * theFlag;
-    unsigned long refcon = (unsigned long)x;
-    OSBoolean * aBool;
-    
-    serialComponent = (refcon>>16) & 0xFFFF;
-    ordinalComponent = refcon & 0xFFFF;
-    
-    if ( serialComponent != pm_vars->serialNumber ) 
+bool IOService::responseValid ( unsigned long x, int pid )
+{
+    UInt16			serialComponent;
+    UInt16			ordinalComponent;
+    OSObject *		theFlag;
+    unsigned long	refcon = (unsigned long) x;
+
+    serialComponent  = (refcon >> 16) & 0xFFFF;
+    ordinalComponent = (refcon & 0xFFFF);
+
+    if ( serialComponent != fSerialNumber )
     {
         return false;
     }
     
-    IOLockLock(priv->flags_lock);
-    if ( pm_vars->responseFlags == NULL ) 
+    if ( fResponseArray == NULL )
     {
-        IOLockUnlock(priv->flags_lock);
         return false;
     }
     
-    theFlag = pm_vars->responseFlags->getObject(ordinalComponent);
+    theFlag = fResponseArray->getObject(ordinalComponent);
     
-    if ( theFlag == 0 ) 
+    if ( theFlag == 0 )
     {
-        IOLockUnlock(priv->flags_lock);
         return false;
     }
-    
-    if ( ((OSBoolean *)theFlag)->isFalse() ) 
+
+    OSNumber * num;
+    if ((num = OSDynamicCast(OSNumber, theFlag)))
+    {
+#if LOG_APP_RESPONSE_TIMES
+        AbsoluteTime	now;
+        AbsoluteTime	start;
+        uint64_t	nsec;
+
+        clock_get_uptime(&now);
+        AbsoluteTime_to_scalar(&start) = num->unsigned64BitValue();
+        SUB_ABSOLUTETIME(&now, &start);
+        absolutetime_to_nanoseconds(now, &nsec);
+
+        // > 100 ms
+        if (nsec > LOG_APP_RESPONSE_TIMES)
+        {
+            OSString * name = IOCopyLogNameForPID(pid);
+            PM_DEBUG("PM response took %d ms (%s)\n", NS_TO_MS(nsec),
+                name ? name->getCStringNoCopy() : "");
+            if (name)
+            name->release();
+        }
+#endif
+        theFlag = kOSBooleanFalse;
+    }
+
+    if ( kOSBooleanFalse == theFlag ) 
     {
-        aBool = OSBoolean::withBoolean(true);
-        pm_vars->responseFlags->replaceObject(ordinalComponent,aBool);
-        aBool->release();
+        fResponseArray->replaceObject(ordinalComponent, kOSBooleanTrue);
     }
     
-    IOLockUnlock(priv->flags_lock);
     return true;
 }
 
-
-// **********************************************************************************
-// allowPowerChange
+//*********************************************************************************
+// [public virtual] allowPowerChange
 //
 // Our power state is about to lower, and we have notified applications
 // and kernel clients, and one of them has acknowledged.  If this is the last to do
@@ -4482,41 +4778,46 @@ bool IOService::responseValid ( unsigned long x )
 //
 // We serialize this processing with timer expiration with a command gate on the
 // power management workloop, which the timer expiration is command gated to as well.
-// **********************************************************************************
+//*********************************************************************************
+
 IOReturn IOService::allowPowerChange ( unsigned long refcon )
 {
-    if ( ! initialized ) 
+	IOPMRequest * request;
+
+    if ( !initialized )
     {
         // we're unloading
         return kIOReturnSuccess;
     }
 
-    return pm_vars->PMcommandGate->runAction(serializedAllowPowerChange,(void *)refcon);
+	request = acquirePMRequest( this, kIOPMRequestTypeAllowPowerChange );
+	if (!request)
+	{
+        PM_ERROR("%s::%s no memory\n", getName(), __FUNCTION__);
+		return kIOReturnNoMemory;
+	}
+
+	request->fArg0 = (void *) refcon;
+	request->fArg1 = (void *) proc_selfpid();
+	submitPMRequest( request );
+
+	return kIOReturnSuccess;
 }
-    
-    
+
 IOReturn serializedAllowPowerChange ( OSObject *owner, void * refcon, void *, void *, void *)
 {
-    return ((IOService *)owner)->serializedAllowPowerChange2((unsigned long)refcon);
+	// [deprecated] public
+	return kIOReturnUnsupported;
 }
 
 IOReturn IOService::serializedAllowPowerChange2 ( unsigned long refcon )
 {
-    // response valid?
-    if ( ! responseValid(refcon) ) 
-    {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAcknowledgeErr5,refcon,0);
-        // no, just return
-        return kIOReturnSuccess;
-    }
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientAcknowledge,refcon,0);
-
-    return allowCancelCommon();
+	// [deprecated] public
+	return kIOReturnUnsupported;
 }
 
-
-// **********************************************************************************
-// cancelPowerChange
+//*********************************************************************************
+// [public virtual] cancelPowerChange
 //
 // Our power state is about to lower, and we have notified applications
 // and kernel clients, and one of them has vetoed the change.  If this is the last
@@ -4524,107 +4825,44 @@ IOReturn IOService::serializedAllowPowerChange2 ( unsigned long refcon )
 //
 // We serialize this processing with timer expiration with a command gate on the
 // power management workloop, which the timer expiration is command gated to as well.
-// **********************************************************************************
+//*********************************************************************************
+
 IOReturn IOService::cancelPowerChange ( unsigned long refcon )
 {
-    if ( ! initialized ) 
+	IOPMRequest * request;
+
+    if ( !initialized )
     {
         // we're unloading
         return kIOReturnSuccess;
     }
 
-    return pm_vars->PMcommandGate->runAction(serializedCancelPowerChange,(void *)refcon);
+	request = acquirePMRequest( this, kIOPMRequestTypeCancelPowerChange );
+	if (!request)
+	{
+        PM_ERROR("%s::%s no memory\n", getName(), __FUNCTION__);
+		return kIOReturnNoMemory;
+	}
+
+	request->fArg0 = (void *) refcon;
+	request->fArg1 = (void *) proc_selfpid();
+	submitPMRequest( request );
+
+	return kIOReturnSuccess;
 }
-    
-    
+
 IOReturn serializedCancelPowerChange ( OSObject *owner, void * refcon, void *, void *, void *)
 {
-    return ((IOService *)owner)->serializedCancelPowerChange2((unsigned long)refcon);
+	// [deprecated] public
+	return kIOReturnUnsupported;
 }
 
 IOReturn IOService::serializedCancelPowerChange2 ( unsigned long refcon )
 {
-    // response valid?
-    if ( ! responseValid(refcon) ) 
-    {
-        pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogAcknowledgeErr5,refcon,0);
-        // no, just return
-        return kIOReturnSuccess;
-    }
-    pm_vars->thePlatform->PMLog(pm_vars->ourName,PMlogClientCancel,refcon,0);
-    
-    pm_vars->doNotPowerDown = true;
-
-    return allowCancelCommon();
-}
-
-
-// **********************************************************************************
-// allowCancelCommon
-//
-// **********************************************************************************
-IOReturn IOService::allowCancelCommon ( void )
-{
-    if (! acquire_lock() ) 
-    {
-        return kIOReturnSuccess;
-    }
-
-    // is this the last response?
-    if ( checkForDone() ) 
-    {
-        // yes, stop the timer
-        stop_ack_timer();
-        IOUnlock(priv->our_lock);
-        IOLockLock(priv->flags_lock);
-        if ( pm_vars->responseFlags ) 
-        {
-            pm_vars->responseFlags->release();
-            pm_vars->responseFlags = NULL;
-        }
-        IOLockUnlock(priv->flags_lock);
-        switch (priv->machine_state) {
-            case kIOPM_OurChangeTellClientsPowerDown:
-                // our change, was it vetoed?
-                if ( ! pm_vars->doNotPowerDown ) 
-                {
-                    // no, we can continue
-                    OurChangeTellClientsPowerDown();
-                } else {
-                    // yes, rescind the warning
-                    tellNoChangeDown(priv->head_note_state);
-                    // mark the change note un-actioned
-                    priv->head_note_flags |= IOPMNotDone;
-
-                    // and we're done
-                    all_done();
-                }
-                break;
-            case kIOPM_OurChangeTellPriorityClientsPowerDown:
-                OurChangeTellPriorityClientsPowerDown();  
-                break;
-            case kIOPM_OurChangeNotifyInterestedDriversWillChange:
-                // our change, continue
-                OurChangeNotifyInterestedDriversWillChange();
-                break;
-            case kIOPM_ParentDownTellPriorityClientsPowerDown_Immediate:
-                // parent change, continue
-                ParentDownTellPriorityClientsPowerDown_Delayed();
-                break;
-            case kIOPM_ParentDownNotifyInterestedDriversWillChange_Delayed:
-                // parent change, continue
-                ParentDownNotifyInterestedDriversWillChange_Delayed();
-                break;
-        }
-    } else {
-        // not done yet
-        IOUnlock(priv->our_lock);
-    }
-
-    return kIOReturnSuccess;
+	// [deprecated] public
+	return kIOReturnUnsupported;
 }
 
-
 #if 0
 //*********************************************************************************
 // c_PM_clamp_Timer_Expired (C Func)
@@ -4632,24 +4870,23 @@ IOReturn IOService::allowCancelCommon ( void )
 // Called when our clamp timer expires...we will call the object method.
 //*********************************************************************************
 
-static void c_PM_Clamp_Timer_Expired (OSObject * client, IOTimerEventSource *)
+static void c_PM_Clamp_Timer_Expired ( OSObject * client, IOTimerEventSource * )
 {
     if (client)
         ((IOService *)client)->PM_Clamp_Timer_Expired ();
 }
 #endif
 
-
 //*********************************************************************************
 // PM_Clamp_Timer_Expired
 //
 // called when clamp timer expires...set power state to 0.
 //*********************************************************************************
 
-void IOService::PM_Clamp_Timer_Expired (void)
+void IOService::PM_Clamp_Timer_Expired ( void )
 {
 #if 0
-    if ( ! initialized ) 
+    if ( ! initialized )
     {
         // we're unloading
         return;
@@ -4659,49 +4896,49 @@ void IOService::PM_Clamp_Timer_Expired (void)
 #endif
 }
 
-//******************************************************************************
+//*********************************************************************************
 // clampPowerOn
 //
 // Set to highest available power state for a minimum of duration milliseconds
-//******************************************************************************
+//*********************************************************************************
 
 #define kFiveMinutesInNanoSeconds (300 * NSEC_PER_SEC)
 
-void IOService::clampPowerOn (unsigned long duration)
+void IOService::clampPowerOn ( unsigned long duration )
 {
 #if 0
-  changePowerStateToPriv (pm_vars->theNumberOfPowerStates-1);
+  changePowerStateToPriv (fNumberOfPowerStates-1);
 
-  if (  priv->clampTimerEventSrc == NULL ) {
-    priv->clampTimerEventSrc = IOTimerEventSource::timerEventSource(this,
+  if (  pwrMgt->clampTimerEventSrc == NULL ) {
+    pwrMgt->clampTimerEventSrc = IOTimerEventSource::timerEventSource(this,
                                                     c_PM_Clamp_Timer_Expired);
 
     IOWorkLoop * workLoop = getPMworkloop ();
 
-    if ( !priv->clampTimerEventSrc || !workLoop ||
-       ( workLoop->addEventSource(  priv->clampTimerEventSrc) != kIOReturnSuccess) ) {
+    if ( !pwrMgt->clampTimerEventSrc || !workLoop ||
+       ( workLoop->addEventSource(  pwrMgt->clampTimerEventSrc) != kIOReturnSuccess) ) {
 
     }
   }
 
-   priv->clampTimerEventSrc->setTimeout(300*USEC_PER_SEC, USEC_PER_SEC);
+   pwrMgt->clampTimerEventSrc->setTimeout(300*USEC_PER_SEC, USEC_PER_SEC);
 #endif
 }
 
 //*********************************************************************************
-// setPowerState
+// [public virtual] setPowerState
 //
 // Does nothing here.  This should be implemented in a subclass driver.
 //*********************************************************************************
 
-IOReturn IOService::setPowerState ( unsigned long powerStateOrdinal, IOService* whatDevice )
+IOReturn IOService::setPowerState (
+	unsigned long powerStateOrdinal, IOService * whatDevice )
 {
     return IOPMNoErr;
 }
 
-
 //*********************************************************************************
-// maxCapabilityForDomainState
+// [public virtual] maxCapabilityForDomainState
 //
 // Finds the highest power state in the array whose input power
 // requirement is equal to the input parameter.  Where a more intelligent
@@ -4712,13 +4949,14 @@ unsigned long IOService::maxCapabilityForDomainState ( IOPMPowerFlags domainStat
 {
    int i;
 
-   if (pm_vars->theNumberOfPowerStates == 0 ) 
+   if (fNumberOfPowerStates == 0 )
    {
        return 0;
    }
-   for ( i = (pm_vars->theNumberOfPowerStates)-1; i >= 0; i-- ) 
+   for ( i = fNumberOfPowerStates - 1; i >= 0; i-- )
    {
-       if (  (domainState & pm_vars->thePowerStates[i].inputPowerRequirement) == pm_vars->thePowerStates[i].inputPowerRequirement ) 
+       if ( (domainState & fPowerStates[i].inputPowerRequirement) ==
+			fPowerStates[i].inputPowerRequirement )
        {
            return i;
        }
@@ -4726,9 +4964,8 @@ unsigned long IOService::maxCapabilityForDomainState ( IOPMPowerFlags domainStat
    return 0;
 }
 
-
 //*********************************************************************************
-// initialPowerStateForDomainState
+// [public virtual] initialPowerStateForDomainState
 //
 // Finds the highest power state in the array whose input power
 // requirement is equal to the input parameter.  Where a more intelligent
@@ -4739,13 +4976,14 @@ unsigned long IOService::initialPowerStateForDomainState ( IOPMPowerFlags domain
 {
     int i;
 
-    if (pm_vars->theNumberOfPowerStates == 0 ) 
+    if (fNumberOfPowerStates == 0 )
     {
         return 0;
     }
-    for ( i = (pm_vars->theNumberOfPowerStates)-1; i >= 0; i-- ) 
+    for ( i = fNumberOfPowerStates - 1; i >= 0; i-- )
     {
-        if (  (domainState & pm_vars->thePowerStates[i].inputPowerRequirement) == pm_vars->thePowerStates[i].inputPowerRequirement ) 
+        if ( (domainState & fPowerStates[i].inputPowerRequirement) ==
+			fPowerStates[i].inputPowerRequirement )
         {
             return i;
         }
@@ -4753,9 +4991,8 @@ unsigned long IOService::initialPowerStateForDomainState ( IOPMPowerFlags domain
     return 0;
 }
 
-
 //*********************************************************************************
-// powerStateForDomainState
+// [public virtual] powerStateForDomainState
 //
 // Finds the highest power state in the array whose input power
 // requirement is equal to the input parameter.  Where a more intelligent
@@ -4766,13 +5003,14 @@ unsigned long IOService::powerStateForDomainState ( IOPMPowerFlags domainState )
 {
     int i;
 
-    if (pm_vars->theNumberOfPowerStates == 0 ) 
+    if (fNumberOfPowerStates == 0 )
     {
         return 0;
     }
-    for ( i = (pm_vars->theNumberOfPowerStates)-1; i >= 0; i-- ) 
+    for ( i = fNumberOfPowerStates - 1; i >= 0; i-- )
     {
-        if (  (domainState & pm_vars->thePowerStates[i].inputPowerRequirement) == pm_vars->thePowerStates[i].inputPowerRequirement ) 
+        if ( (domainState & fPowerStates[i].inputPowerRequirement) ==
+			fPowerStates[i].inputPowerRequirement )
         {
             return i;
         }
@@ -4780,9 +5018,8 @@ unsigned long IOService::powerStateForDomainState ( IOPMPowerFlags domainState )
     return 0;
 }
 
-
 //*********************************************************************************
-// didYouWakeSystem
+// [public virtual] didYouWakeSystem
 //
 // Does nothing here.  This should be implemented in a subclass driver.
 //*********************************************************************************
@@ -4792,34 +5029,32 @@ bool IOService::didYouWakeSystem  ( void )
     return false;
 }
 
-
 //*********************************************************************************
-// powerStateWillChangeTo
+// [public virtual] powerStateWillChangeTo
 //
 // Does nothing here.  This should be implemented in a subclass driver.
 //*********************************************************************************
 
-IOReturn IOService::powerStateWillChangeTo ( IOPMPowerFlags, unsigned long, IOService*)
+IOReturn IOService::powerStateWillChangeTo ( IOPMPowerFlags, unsigned long, IOService * )
 {
-    return 0;
+    return kIOPMAckImplied;
 }
 
-
 //*********************************************************************************
-// powerStateDidChangeTo
+// [public virtual] powerStateDidChangeTo
 //
 // Does nothing here.  This should be implemented in a subclass driver.
 //*********************************************************************************
 
-IOReturn IOService::powerStateDidChangeTo ( IOPMPowerFlags, unsigned long, IOService*)
+IOReturn IOService::powerStateDidChangeTo ( IOPMPowerFlags, unsigned long, IOService * )
 {
-    return 0;
+    return kIOPMAckImplied;
 }
 
-
 //*********************************************************************************
-// powerChangeDone
+// [public virtual] powerChangeDone
 //
+// Called from PM work loop thread.
 // Does nothing here.  This should be implemented in a subclass policy-maker.
 //*********************************************************************************
 
@@ -4827,142 +5062,861 @@ void IOService::powerChangeDone ( unsigned long )
 {
 }
 
-
 //*********************************************************************************
-// newTemperature
+// [public virtual] newTemperature
 //
 // Does nothing here.  This should be implemented in a subclass driver.
 //*********************************************************************************
 
 IOReturn IOService::newTemperature ( long currentTemp, IOService * whichZone )
-
 {
     return IOPMNoErr;
 }
 
+//*********************************************************************************
+// [public virtual] systemWillShutdown
+//
+// System shutdown and restart notification.
+//*********************************************************************************
 
-#undef super
-#define super OSObject
+void IOService::systemWillShutdown( IOOptionBits specifier )
+{
+	IOPMrootDomain * rootDomain = IOService::getPMRootDomain();
+	if (rootDomain)
+		rootDomain->acknowledgeSystemWillShutdown( this );
+}
 
-OSDefineMetaClassAndStructors(IOPMprot, OSObject)
 //*********************************************************************************
-// serialize
+// [private static] acquirePMRequest
+//*********************************************************************************
+
+IOPMRequest *
+IOService::acquirePMRequest( IOService * target, IOOptionBits requestType )
+{
+	IOPMRequest * request;
+
+	assert(target);
+
+	request = IOPMRequest::create();
+	if (request)
+	{
+		request->init( target, requestType );
+	}
+	return request;
+}
+
+//*********************************************************************************
+// [private static] releasePMRequest
+//*********************************************************************************
+
+void IOService::releasePMRequest( IOPMRequest * request )
+{
+	if (request)
+	{
+		request->reset();
+		request->release();
+	}
+}
+
+//*********************************************************************************
+// [private] submitPMRequest
+//*********************************************************************************
+
+void IOService::submitPMRequest( IOPMRequest * request )
+{
+	assert( request );
+	assert( gIOPMReplyQueue );
+	assert( gIOPMRequestQueue );
+
+	PM_TRACE("[+ %02lx] %p [%p %s] %p %p %p\n",
+		request->getType(), request,
+		request->getTarget(), request->getTarget()->getName(),
+		request->fArg0, request->fArg1, request->fArg2);
+
+	if (request->isReply())
+		gIOPMReplyQueue->queuePMRequest( request );
+	else
+		gIOPMRequestQueue->queuePMRequest( request );
+}
+
+void IOService::submitPMRequest( IOPMRequest ** requests, IOItemCount count )
+{
+	assert( requests );
+	assert( count > 0 );
+	assert( gIOPMRequestQueue );
+
+	for (IOItemCount i = 0; i < count; i++)
+	{
+		IOPMRequest * req = requests[i];
+		PM_TRACE("[+ %02lx] %p [%p %s] %p %p %p\n",
+			req->getType(), req,
+			req->getTarget(), req->getTarget()->getName(),
+			req->fArg0, req->fArg1, req->fArg2);
+	}
+
+	gIOPMRequestQueue->queuePMRequestChain( requests, count );
+}
+
+//*********************************************************************************
+// [private] servicePMRequestQueue
+//*********************************************************************************
+
+bool IOService::servicePMRequestQueue(
+	IOPMRequest *		request,
+	IOPMRequestQueue *	queue )
+{
+	// Calling PM methods without PMinit() is not allowed, fail the requests.
+
+	if (!initialized)
+	{
+		PM_DEBUG("[%s] %s: PM not initialized\n", getName(), __FUNCTION__);
+		goto done;
+	}
+
+	// Create an IOPMWorkQueue on demand, when the initial PM request is
+	// received.
+
+	if (!fPMWorkQueue)
+	{
+		// Allocate and attach an IOPMWorkQueue on demand to avoid taking
+		// the work loop lock in PMinit(), which may deadlock with certain
+		// drivers / families.
+
+		fPMWorkQueue = IOPMWorkQueue::create(
+			/* target */	this,
+			/* Work */		OSMemberFunctionCast(IOPMWorkQueue::Action, this,
+								&IOService::servicePMRequest),
+			/* Done */		OSMemberFunctionCast(IOPMWorkQueue::Action, this,
+								&IOService::retirePMRequest)
+			);
+
+		if (fPMWorkQueue &&
+			(gIOPMWorkLoop->addEventSource(fPMWorkQueue) != kIOReturnSuccess))
+		{
+			PM_ERROR("[%s] %s: addEventSource failed\n",
+				getName(), __FUNCTION__);
+			fPMWorkQueue->release();
+			fPMWorkQueue = 0;
+		}
+
+		if (!fPMWorkQueue)
+		{
+			PM_ERROR("[%s] %s: not ready (type %02lx)\n",
+				getName(), __FUNCTION__, request->getType());
+			goto done;
+		}
+	}
+
+	fPMWorkQueue->queuePMRequest(request);
+	return false;	// do not signal more
+
+done:
+	gIOPMFreeQueue->queuePMRequest( request );
+	return false;	// do not signal more
+}
+
+//*********************************************************************************
+// [private] servicePMFreeQueue
+//
+// Called by IOPMFreeQueue to recycle a completed request.
+//*********************************************************************************
+
+bool IOService::servicePMFreeQueue(
+	IOPMRequest *		request,
+	IOPMRequestQueue *	queue )
+{
+	bool more = request->hasParentRequest();
+	releasePMRequest( request );
+	return more;
+}
+
+//*********************************************************************************
+// [private] retirePMRequest
 //
-// Serialize protected instance variables for debug output.
+// Called by IOPMWorkQueue to retire a completed request.
 //*********************************************************************************
-bool IOPMprot::serialize(OSSerialize *s) const
+
+bool IOService::retirePMRequest( IOPMRequest * request, IOPMWorkQueue * queue )
 {
-    OSString * theOSString;
-    char * buffer;
-    char * ptr;
-    int     buf_size;
-    int i;
-    bool	rtn_code;
+	assert(request && queue);
 
-    // estimate how many bytes we need to present all power states
-    buf_size = 150      // beginning and end of string
-                + (275 * (int)theNumberOfPowerStates) // size per state
-                + 100;   // extra room just for kicks
+	PM_TRACE("[- %02lx] %p [%p %s] State %ld, Busy %ld\n",
+		request->getType(), request, this, getName(),
+		fMachineState, gIOPMBusyCount);
 
-    buffer = ptr = IONew(char, buf_size);
-    if(!buffer)
-        return false;
+	// Catch requests created by PM_idle_timer_expiration().
 
-    ptr += sprintf(ptr,"{ theNumberOfPowerStates = %d, ",(unsigned int)theNumberOfPowerStates);
+	if ((request->getType() == kIOPMRequestTypeActivityTickle) &&
+		(request->fArg1 == (void *) false))
+	{
+		// Idle timer power drop request completed.
+		// Restart the idle timer if deviceDesire can go lower, otherwise set
+		// a flag so we know to restart idle timer when deviceDesire goes up.
 
-    if ( theNumberOfPowerStates != 0 ) {
-        ptr += sprintf(ptr,"version %d, ",(unsigned int)thePowerStates[0].version);
-    }
+		if (fDeviceDesire > 0)
+			start_PM_idle_timer();
+		else
+			fActivityTimerStopped = true;
+	}
 
-    if ( theNumberOfPowerStates != 0 ) {
-        for ( i = 0; i < (int)theNumberOfPowerStates; i++ ) {
-            ptr += sprintf(ptr, "power state %d = { ",i);
-            ptr += sprintf(ptr,"capabilityFlags %08x, ",(unsigned int)thePowerStates[i].capabilityFlags);
-            ptr += sprintf(ptr,"outputPowerCharacter %08x, ",(unsigned int)thePowerStates[i].outputPowerCharacter);
-            ptr += sprintf(ptr,"inputPowerRequirement %08x, ",(unsigned int)thePowerStates[i].inputPowerRequirement);
-            ptr += sprintf(ptr,"staticPower %d, ",(unsigned int)thePowerStates[i].staticPower);
-            ptr += sprintf(ptr,"unbudgetedPower %d, ",(unsigned int)thePowerStates[i].unbudgetedPower);
-            ptr += sprintf(ptr,"powerToAttain %d, ",(unsigned int)thePowerStates[i].powerToAttain);
-            ptr += sprintf(ptr,"timeToAttain %d, ",(unsigned int)thePowerStates[i].timeToAttain);
-            ptr += sprintf(ptr,"settleUpTime %d, ",(unsigned int)thePowerStates[i].settleUpTime);
-            ptr += sprintf(ptr,"timeToLower %d, ",(unsigned int)thePowerStates[i].timeToLower);
-            ptr += sprintf(ptr,"settleDownTime %d, ",(unsigned int)thePowerStates[i].settleDownTime);
-            ptr += sprintf(ptr,"powerDomainBudget %d }, ",(unsigned int)thePowerStates[i].powerDomainBudget);
-        }
-    }
+	gIOPMFreeQueue->queuePMRequest( request );
+	return true;
+}
 
-    ptr += sprintf(ptr,"aggressiveness = %d, ",(unsigned int)aggressiveness);
-    ptr += sprintf(ptr,"myCurrentState = %d, ",(unsigned int)myCurrentState);
-    ptr += sprintf(ptr,"parentsCurrentPowerFlags = %08x, ",(unsigned int)parentsCurrentPowerFlags);
-    ptr += sprintf(ptr,"maxCapability = %d }",(unsigned int)maxCapability);
+//*********************************************************************************
+// [private] isPMBlocked
+//
+// Check if machine state transition is blocked.
+//*********************************************************************************
 
-    theOSString = OSString::withCString(buffer);
-    rtn_code = theOSString->serialize(s);
-    theOSString->release();
-    IODelete(buffer, char, buf_size);
+bool IOService::isPMBlocked ( IOPMRequest * request, int count )
+{
+	int	reason = 0;
+
+	do {
+		if (kIOPM_Finished == fMachineState)
+			break;
+
+		if (kIOPM_DriverThreadCallDone == fMachineState)
+		{
+            // 5 = kDriverCallInformPreChange
+            // 6 = kDriverCallInformPostChange
+            // 7 = kDriverCallSetPowerState
+			if (fDriverCallBusy) reason = 5 + fDriverCallReason;
+			break;
+		}
+
+		// Waiting on driver's setPowerState() timeout.
+		if (fDriverTimer)
+		{
+			reason = 1; break;
+		}
+
+		// Child or interested driver acks pending.
+		if (fHeadNotePendingAcks)
+		{
+			reason = 2; break;
+		}
+
+		// Waiting on apps or priority power interest clients.
+		if (fResponseArray)
+		{
+			reason = 3; break;
+		}
+
+		// Waiting on settle timer expiration.
+		if (fSettleTimeUS)
+		{
+			reason = 4; break;
+		}
+	} while (false);
+
+	fWaitReason = reason;
+
+	if (reason)
+	{
+		if (count)
+		{
+			PM_TRACE("[B %02lx] %p [%p %s] State %ld, Reason %d\n",
+				request->getType(), request, this, getName(),
+				fMachineState, reason);
+		}
+
+		return true;
+	}
 
-    return rtn_code;
+	return false;
 }
 
+//*********************************************************************************
+// [private] servicePMRequest
+//
+// Service a request from our work queue.
+//*********************************************************************************
+
+bool IOService::servicePMRequest( IOPMRequest * request, IOPMWorkQueue * queue )
+{
+	bool	done = false;
+	int		loop = 0;
+
+	assert(request && queue);
+
+	while (isPMBlocked(request, loop++) == false)
+	{
+		PM_TRACE("[W %02lx] %p [%p %s] State %ld\n",
+			request->getType(), request, this, getName(), fMachineState);
+
+		fPMRequest = request;
+
+		// Every PM machine states must be handled in one of the cases below.
+
+		switch ( fMachineState )
+		{
+			case kIOPM_Finished:
+				executePMRequest( request );
+				break;
+
+			case kIOPM_OurChangeTellClientsPowerDown:
+				// our change, was it vetoed?
+				if (fDesiredPowerState > fHeadNoteState)
+				{
+					PM_DEBUG("%s: idle cancel\n", fName);
+					fDoNotPowerDown = true;
+				}
+				if (!fDoNotPowerDown)
+				{
+					// no, we can continue
+					OurChangeTellClientsPowerDown();
+				}
+				else
+				{
+					// yes, rescind the warning
+					tellNoChangeDown(fHeadNoteState);
+					// mark the change note un-actioned
+					fHeadNoteFlags |= IOPMNotDone;
+					// and we're done
+					all_done();
+				}
+				break;
+
+			case kIOPM_OurChangeTellPriorityClientsPowerDown:
+				OurChangeTellPriorityClientsPowerDown();  
+				break;
+
+			case kIOPM_OurChangeNotifyInterestedDriversWillChange:
+				OurChangeNotifyInterestedDriversWillChange();
+				break;
+
+			case kIOPM_OurChangeSetPowerState:
+				OurChangeSetPowerState();
+				break;
+
+			case kIOPM_OurChangeWaitForPowerSettle:
+				OurChangeWaitForPowerSettle();
+				break;
+
+			case kIOPM_OurChangeNotifyInterestedDriversDidChange:
+				OurChangeNotifyInterestedDriversDidChange();
+				break;
+
+			case kIOPM_OurChangeFinish:
+				OurChangeFinish();
+				break;
+
+			case kIOPM_ParentDownTellPriorityClientsPowerDown:
+				ParentDownTellPriorityClientsPowerDown();
+				break;
+
+			case kIOPM_ParentDownNotifyInterestedDriversWillChange:
+				ParentDownNotifyInterestedDriversWillChange();
+				break;
+
+			case kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange:
+				ParentDownNotifyDidChangeAndAcknowledgeChange();
+				break;
+
+			case kIOPM_ParentDownSetPowerState:
+				ParentDownSetPowerState();	
+				break;
+
+			case kIOPM_ParentDownWaitForPowerSettle:
+				ParentDownWaitForPowerSettle();
+				break;
+
+			case kIOPM_ParentDownAcknowledgeChange:
+				ParentDownAcknowledgeChange();
+				break;
+
+			case kIOPM_ParentUpSetPowerState:
+				ParentUpSetPowerState();
+				break;
+
+			case kIOPM_ParentUpWaitForSettleTime:
+				ParentUpWaitForSettleTime();
+				break;
+
+			case kIOPM_ParentUpNotifyInterestedDriversDidChange:
+				ParentUpNotifyInterestedDriversDidChange();
+				break;
+
+			case kIOPM_ParentUpAcknowledgePowerChange:
+				ParentUpAcknowledgePowerChange();
+				break;
+
+			case kIOPM_DriverThreadCallDone:
+				if (fDriverCallReason == kDriverCallSetPowerState)
+					notifyControllingDriverDone();
+				else
+					notifyInterestedDriversDone();
+				break;
+
+			case kIOPM_NotifyChildrenDone:
+				notifyChildrenDone();
+				break;
+
+			default:
+				IOPanic("servicePMWorkQueue: unknown machine state");
+		}
+
+		fPMRequest = 0;
+
+		if (fMachineState == kIOPM_Finished)
+		{
+			//PM_TRACE("[%s] PM   End: Request %p (type %02lx)\n",
+			//	getName(), request, request->getType());
+			done = true;
+			break;
+		}
+	}
 
-#undef super
-#define super OSObject
+	return done;
+}
 
-OSDefineMetaClassAndStructors(IOPMpriv, OSObject)
 //*********************************************************************************
-// serialize
+// [private] executePMRequest
+//*********************************************************************************
+
+void IOService::executePMRequest( IOPMRequest * request )
+{
+	assert( kIOPM_Finished == fMachineState );
+
+	switch (request->getType())
+	{
+		case kIOPMRequestTypePMStop:
+			handlePMstop( request );
+			break;
+
+		case kIOPMRequestTypeAddPowerChild1:
+			addPowerChild1( request );
+			break;
+
+		case kIOPMRequestTypeAddPowerChild2:
+			addPowerChild2( request );
+			break;
+
+		case kIOPMRequestTypeAddPowerChild3:
+			addPowerChild3( request );
+			break;
+
+		case kIOPMRequestTypeRegisterPowerDriver:
+			handleRegisterPowerDriver( request );
+			break;
+
+		case kIOPMRequestTypeAdjustPowerState:
+			adjustPowerState();
+			break;
+
+		case kIOPMRequestTypeMakeUsable:
+			handleMakeUsable( request );
+			break;
+
+		case kIOPMRequestTypeTemporaryPowerClamp:
+			fClampOn = true;
+			handleMakeUsable( request );
+			break;
+
+		case kIOPMRequestTypePowerDomainWillChange:
+			handlePowerDomainWillChangeTo( request );
+			break;
+
+		case kIOPMRequestTypePowerDomainDidChange:
+			handlePowerDomainDidChangeTo( request );
+			break;
+
+		case kIOPMRequestTypeChangePowerStateTo:
+			handleChangePowerStateTo( request );
+			break;
+
+		case kIOPMRequestTypeChangePowerStateToPriv:
+			handleChangePowerStateToPriv( request );
+			break;
+
+		case kIOPMRequestTypePowerOverrideOnPriv:
+		case kIOPMRequestTypePowerOverrideOffPriv:
+			handlePowerOverrideChanged( request );
+			break;
+
+		case kIOPMRequestTypeActivityTickle:
+			if (request)
+			{
+				bool setDeviceDesire = false;
+
+				if (request->fArg1)
+				{
+					// power rise
+					if (fDeviceDesire < (unsigned long) request->fArg0)
+						setDeviceDesire = true;
+				}
+				else if (fDeviceDesire)
+				{
+					// power drop and deviceDesire is not zero
+					request->fArg0 = (void *) (fDeviceDesire - 1);
+					setDeviceDesire = true;
+				}
+
+				if (setDeviceDesire)
+				{
+					// handleChangePowerStateToPriv() does not check the
+					// request type, as long as the args are appropriate
+					// for kIOPMRequestTypeChangePowerStateToPriv.
+
+					request->fArg1 = (void *) false;
+					handleChangePowerStateToPriv( request );
+				}
+			}
+			break;
+
+		default:
+			IOPanic("executePMRequest: unknown request type");
+	}
+}
+
+//*********************************************************************************
+// [private] servicePMReplyQueue
+//*********************************************************************************
+
+bool IOService::servicePMReplyQueue( IOPMRequest * request, IOPMRequestQueue * queue )
+{
+	bool more = false;
+
+	assert( request && queue );
+	assert( request->isReply() );
+
+	PM_TRACE("[A %02lx] %p [%p %s] State %ld\n",
+		request->getType(), request, this, getName(), fMachineState);
+
+	switch ( request->getType() )
+	{
+		case kIOPMRequestTypeAllowPowerChange:
+		case kIOPMRequestTypeCancelPowerChange:
+			// Check if we are expecting this response.
+			if (responseValid((unsigned long) request->fArg0, (int) request->fArg1))
+			{
+				if (kIOPMRequestTypeCancelPowerChange == request->getType())
+					fDoNotPowerDown = true;
+
+				if (checkForDone())
+				{
+					stop_ack_timer();
+					if ( fResponseArray )
+					{
+						fResponseArray->release();
+						fResponseArray = NULL;
+					}
+					more = true;
+				}
+			}
+			break;
+
+		case kIOPMRequestTypeAckPowerChange:
+			more = handleAcknowledgePowerChange( request );
+			break;
+
+		case kIOPMRequestTypeAckSetPowerState:
+			if (fDriverTimer == -1)
+			{
+				// driver acked while setPowerState() call is in-flight.
+				// take this ack, return value from setPowerState() is irrelevant.
+				OUR_PMLog(kPMLogDriverAcknowledgeSet,
+					(UInt32) this, fDriverTimer);
+				fDriverTimer = 0;
+			}
+			else if (fDriverTimer > 0)
+			{
+				// expected ack, stop the timer
+				stop_ack_timer();
+
+#if LOG_SETPOWER_TIMES
+                uint64_t nsec = computeTimeDeltaNS(&fDriverCallStartTime);
+                if (nsec > LOG_SETPOWER_TIMES)
+                    PM_DEBUG("%s::setPowerState(%p, %lu -> %lu) async took %d ms\n",
+                        fName, this, fCurrentPowerState, fHeadNoteState, NS_TO_MS(nsec));
+#endif
+				OUR_PMLog(kPMLogDriverAcknowledgeSet, (UInt32) this, fDriverTimer);
+				fDriverTimer = 0;
+				more = true;
+			}
+			else
+			{
+				// unexpected ack
+				OUR_PMLog(kPMLogAcknowledgeErr4, (UInt32) this, 0);
+			}
+			break;
+
+		case kIOPMRequestTypeInterestChanged:
+			handleInterestChanged( request );
+			more = true;
+			break;
+
+		default:
+			IOPanic("servicePMReplyQueue: unknown reply type");
+	}
+
+	releasePMRequest( request );
+	return more;
+}
+
+//*********************************************************************************
+// IOPMRequest Class
+//
+// Requests from PM clients, and also used for inter-object messaging within PM.
+//*********************************************************************************
+
+OSDefineMetaClassAndStructors( IOPMRequest, IOCommand );
+
+IOPMRequest * IOPMRequest::create( void )
+{
+	IOPMRequest * me = OSTypeAlloc(IOPMRequest);
+	if (me && !me->init(0, kIOPMRequestTypeInvalid))
+	{
+		me->release();
+		me = 0;
+	}
+	return me;
+}
+
+bool IOPMRequest::init( IOService * target, IOOptionBits type )
+{
+	if (!IOCommand::init())
+		return false;
+
+	fType       = type;
+	fTarget     = target;
+	fParent     = 0;
+	fChildCount = 0;
+	fArg0 = fArg1 = fArg2 = 0;
+
+	if (fTarget)
+		fTarget->retain();
+
+	return true;
+}
+
+void IOPMRequest::reset( void )
+{
+	assert( fChildCount == 0 );
+
+	fType = kIOPMRequestTypeInvalid;
+
+	if (fParent)
+	{
+		fParent->fChildCount--;
+		fParent = 0;
+	}
+
+	if (fTarget)
+	{
+		fTarget->release();
+		fTarget = 0;
+	}
+}
+
+//*********************************************************************************
+// IOPMRequestQueue Class
 //
-// Serialize private instance variables for debug output.
+// Global queues. As PM-aware drivers load and unload, their IOPMWorkQueue's are
+// created and deallocated. IOPMRequestQueue are created once and never released.
 //*********************************************************************************
-bool IOPMpriv::serialize(OSSerialize *s) const
+
+OSDefineMetaClassAndStructors( IOPMRequestQueue, IOEventSource );
+
+IOPMRequestQueue * IOPMRequestQueue::create( IOService * inOwner, Action inAction )
 {
-    OSString *			theOSString;
-    bool			rtn_code;
-    char * 			buffer;
-    char * 			ptr;
-    IOPMinformee * 		nextObject;
+	IOPMRequestQueue * me = OSTypeAlloc(IOPMRequestQueue);
+	if (me && !me->init(inOwner, inAction))
+	{
+		me->release();
+		me = 0;
+	}
+	return me;
+}
 
-    buffer = ptr = IONew(char, 2000);
-    if(!buffer)
+bool IOPMRequestQueue::init( IOService * inOwner, Action inAction )
+{
+	if (!inAction || !IOEventSource::init(inOwner, (IOEventSourceAction)inAction))
         return false;
 
-    ptr += sprintf(ptr,"{ this object = %08x",(unsigned int)owner);
-    if ( we_are_root ) {
-        ptr += sprintf(ptr," (root)");
-    }
-    ptr += sprintf(ptr,", ");
+	queue_init(&fQueue);
+	fLock = IOLockAlloc();
+	return (fLock != 0);
+}
 
-    nextObject = interestedDrivers->firstInList();			// display interested drivers
-    while (  nextObject != NULL ) {
-        ptr += sprintf(ptr,"interested driver = %08x, ",(unsigned int)nextObject->whatObject);
-        nextObject  =  interestedDrivers->nextInList(nextObject);
-    }
+void IOPMRequestQueue::free( void )
+{
+	if (fLock)
+	{
+		IOLockFree(fLock);
+		fLock = 0;
+	}
+	return IOEventSource::free();
+}
 
-    if ( machine_state != kIOPM_Finished ) {
-        ptr += sprintf(ptr,"machine_state = %d, ",(unsigned int)machine_state);
-        ptr += sprintf(ptr,"driver_timer = %d, ",(unsigned int)driver_timer);
-        ptr += sprintf(ptr,"settle_time = %d, ",(unsigned int)settle_time);
-        ptr += sprintf(ptr,"head_note_flags = %08x, ",(unsigned int)head_note_flags);
-        ptr += sprintf(ptr,"head_note_state = %d, ",(unsigned int)head_note_state);
-        ptr += sprintf(ptr,"head_note_outputFlags = %08x, ",(unsigned int)head_note_outputFlags);
-        ptr += sprintf(ptr,"head_note_domainState = %08x, ",(unsigned int)head_note_domainState);
-        ptr += sprintf(ptr,"head_note_capabilityFlags = %08x, ",(unsigned int)head_note_capabilityFlags);
-        ptr += sprintf(ptr,"head_note_pendingAcks = %d, ",(unsigned int)head_note_pendingAcks);
-    }
+void IOPMRequestQueue::queuePMRequest( IOPMRequest * request )
+{
+	assert(request);
+	IOLockLock(fLock);
+	queue_enter(&fQueue, request, IOPMRequest *, fCommandChain);
+	IOLockUnlock(fLock);
+	if (workLoop) signalWorkAvailable();
+}
 
-    if ( device_overrides ) {
-        ptr += sprintf(ptr,"device overrides, ");
-    }
-    ptr += sprintf(ptr,"driverDesire = %d, ",(unsigned int)driverDesire);
-    ptr += sprintf(ptr,"deviceDesire = %d, ",(unsigned int)deviceDesire);
-    ptr += sprintf(ptr,"ourDesiredPowerState = %d, ",(unsigned int)ourDesiredPowerState);
-    ptr += sprintf(ptr,"previousRequest = %d }",(unsigned int)previousRequest);
+void
+IOPMRequestQueue::queuePMRequestChain( IOPMRequest ** requests, IOItemCount count )
+{
+	IOPMRequest * next;
+
+	assert(requests && count);
+	IOLockLock(fLock);
+	while (count--)
+	{
+		next = *requests;
+		requests++;
+		queue_enter(&fQueue, next, IOPMRequest *, fCommandChain);
+	}
+	IOLockUnlock(fLock);
+	if (workLoop) signalWorkAvailable();
+}
+
+bool IOPMRequestQueue::checkForWork( void )
+{
+    Action			dqAction = (Action) action;
+	IOPMRequest *	request;
+	IOService *		target;
+	bool			more = false;
+
+	IOLockLock( fLock );
+
+	while (!queue_empty(&fQueue))
+	{
+		queue_remove_first( &fQueue, request, IOPMRequest *, fCommandChain );		
+		IOLockUnlock( fLock );
+		target = request->getTarget();
+		assert(target);
+		more |= (*dqAction)( target, request, this );
+		IOLockLock( fLock );
+	}
+
+	IOLockUnlock( fLock );
+	return more;
+}
+
+void IOPMRequestQueue::signalWorkAvailable( void )
+{
+	IOEventSource::signalWorkAvailable();
+}
+
+//*********************************************************************************
+// IOPMWorkQueue Class
+//
+// Every object in the power plane that has handled a PM request, will have an
+// instance of IOPMWorkQueue allocated for it.
+//*********************************************************************************
 
-    theOSString =  OSString::withCString(buffer);
-    rtn_code = theOSString->serialize(s);
-    theOSString->release();
-    IODelete(buffer, char, 2000);
+OSDefineMetaClassAndStructors( IOPMWorkQueue, IOEventSource );
 
-    return rtn_code;
+IOPMWorkQueue *
+IOPMWorkQueue::create( IOService * inOwner, Action work, Action retire )
+{
+	IOPMWorkQueue * me = OSTypeAlloc(IOPMWorkQueue);
+	if (me && !me->init(inOwner, work, retire))
+	{
+		me->release();
+		me = 0;
+	}
+	return me;
+}
+
+bool IOPMWorkQueue::init( IOService * inOwner, Action work, Action retire )
+{
+	if (!work || !retire ||
+		!IOEventSource::init(inOwner, (IOEventSourceAction)0))
+		return false;
+
+	queue_init(&fWorkQueue);
+
+	fWorkAction   = work;
+	fRetireAction = retire;
+
+	return true;
+}
+
+void IOPMWorkQueue::queuePMRequest( IOPMRequest * request )
+{
+	assert( request );
+	assert( onThread() );
+
+	gIOPMBusyCount++;
+	queue_enter(&fWorkQueue, request, IOPMRequest *, fCommandChain);
+	checkForWork();
+}
+
+bool IOPMWorkQueue::checkForWork( void )
+{
+	IOPMRequest *	request;
+	IOService *		target = (IOService *) owner;
+	bool			done;
+
+	while (!queue_empty(&fWorkQueue))
+	{
+		request = (IOPMRequest *) queue_first(&fWorkQueue);
+		assert(request->getTarget() == target);
+		if (request->hasChildRequest()) break;
+		done = (*fWorkAction)( target, request, this );
+		if (!done) break;
+
+		assert(gIOPMBusyCount > 0);
+		if (gIOPMBusyCount) gIOPMBusyCount--;
+		queue_remove_first(&fWorkQueue, request, IOPMRequest *, fCommandChain);
+		(*fRetireAction)( target, request, this );
+	}
+
+	return false;
+}
+
+OSDefineMetaClassAndStructors(IOServicePM, OSObject)
+
+//*********************************************************************************
+// serialize
+//
+// Serialize IOServicePM for debugging.
+//*********************************************************************************
+
+static void
+setPMProperty( OSDictionary * dict, const char * key, unsigned long value )
+{
+    OSNumber * num = OSNumber::withNumber(value, sizeof(value) * 8);
+    if (num)
+    {
+        dict->setObject(key, num);
+        num->release();
+    }
 }
 
+bool IOServicePM::serialize( OSSerialize * s ) const
+{
+	OSDictionary *	dict;
+	bool			ok = false;
+
+	dict = OSDictionary::withCapacity(8);
+	if (dict)
+	{
+        setPMProperty( dict, "CurrentPowerState", CurrentPowerState );
+        if (DesiredPowerState != CurrentPowerState)
+            setPMProperty( dict, "DesiredPowerState", DesiredPowerState );
+        if (kIOPM_Finished != MachineState)
+            setPMProperty( dict, "MachineState", MachineState );
+        if (ChildrenDesire)
+            setPMProperty( dict, "ChildrenPowerState", ChildrenDesire );
+        if (DeviceDesire)
+            setPMProperty( dict, "DeviceChangePowerState", DeviceDesire );
+        if (DriverDesire)
+            setPMProperty( dict, "DriverChangePowerState", DriverDesire );
+        if (DeviceOverrides)
+            dict->setObject( "PowerOverrideOn", kOSBooleanTrue );
+
+		ok = dict->serialize(s);
+		dict->release();
+	}
+
+	return ok;
+}
diff --git a/iokit/Kernel/IOServicePMPrivate.h b/iokit/Kernel/IOServicePMPrivate.h
new file mode 100644
index 000000000..33af8328a
--- /dev/null
+++ b/iokit/Kernel/IOServicePMPrivate.h
@@ -0,0 +1,568 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _IOKIT_IOSERVICEPMPRIVATE_H
+#define _IOKIT_IOSERVICEPMPRIVATE_H
+
+/* Binary compatibility with drivers that access pm_vars */
+#define PM_VARS_SUPPORT     1
+
+/*! @class IOServicePM
+    @abstract Power management class.
+*/
+class IOServicePM : public OSObject
+{
+    friend class IOService;
+
+    OSDeclareDefaultStructors( IOServicePM )
+
+private:
+    /*! @var Owner
+        Points to object that called PMinit().  Used only for debugging.
+    */
+    IOService *             Owner;
+
+    /*! @var InterestedDrivers
+        List of interested drivers.
+    */
+    IOPMinformeeList *      InterestedDrivers;
+
+    /*! @var DriverTimer
+        How long to wait for controlling driver to acknowledge.
+    */
+    IOReturn                DriverTimer;
+
+    /*! @var AckTimer */
+    thread_call_t           AckTimer;
+
+    /*! @var SettleTimer */
+    thread_call_t           SettleTimer;
+
+    /*! @var MachineState
+        Current power management machine state.
+    */
+    unsigned long           MachineState;
+
+    /*! @var SettleTimeUS
+        Settle time after changing power state.
+    */
+    unsigned long           SettleTimeUS;
+
+    /*! @var HeadNoteFlags
+        The flags field for the current change note.
+    */
+    unsigned long           HeadNoteFlags;
+
+    /*! @var HeadNoteState
+        The newStateNumber field for the current change note.
+    */
+    unsigned long           HeadNoteState;
+
+    /*! @var HeadNoteOutputFlags
+        The outputPowerCharacter field for the current change note.
+    */
+    unsigned long           HeadNoteOutputFlags;
+
+    /*! @var HeadNoteDomainState
+        Power domain flags from parent. (only on parent change).
+    */
+    unsigned long           HeadNoteDomainState;
+
+    /*! @var HeadNoteParent
+        Pointer to initiating parent. (only on parent change).
+    */
+    IOPowerConnection *     HeadNoteParent;
+    
+    /*! @var HeadNoteCapabilityFlags
+        The capabilityFlags field for the current change note.
+    */
+    unsigned long           HeadNoteCapabilityFlags;
+
+    /*! @var HeadNotePendingAcks
+        Number of acks we are waiting for.
+    */
+    unsigned long           HeadNotePendingAcks;
+
+    /*! @var PMLock
+        PM state lock.
+    */
+    IOLock	*               PMLock;
+
+    /*! @var WeAreRoot
+        True if our owner is the root of the power tree.
+    */
+    bool                    WeAreRoot;
+
+    /*! @var InitialChange
+        Initialized to true, then set to false after the initial power change.
+    */
+    bool                    InitialChange;
+
+    /*! @var NeedToBecomeUsable
+        Someone has called makeUsable before we had a controlling driver.
+    */
+    bool                    NeedToBecomeUsable;
+
+    /*! @var DeviceOverrides
+        Ignore children and driver desires if true.
+    */
+    bool                    DeviceOverrides;
+
+    /*! @var ClampOn
+        Domain is clamped on until the first power child is added.
+    */
+    bool                    ClampOn;
+
+    /*! @var DeviceActive
+        True if device was active since last idle timer expiration.
+    */
+    bool                    DeviceActive;
+
+    /*! @var DoNotPowerDown
+        Keeps track of any negative responses from notified apps and clients.
+    */
+    bool					DoNotPowerDown;
+
+    /*! @var ParentsKnowState
+        True if all our parents know the state of their power domain.
+    */
+    bool					ParentsKnowState;
+
+    /*! @var DeviceActiveTimestamp
+        Time of last device activity.
+    */
+    AbsoluteTime            DeviceActiveTimestamp;
+
+    /*! @var ActivityLock
+        Used to protect activity flag.
+    */
+    IOLock *                ActivityLock;
+
+    /*! @var IdleTimerEventSource
+        An idle timer event source.
+    */
+    IOTimerEventSource *    IdleTimerEventSource;
+
+    /*! @var IdleTimerPeriod
+        Idle timer's period in seconds.
+    */
+    unsigned long           IdleTimerPeriod;
+
+    /*! @var DriverDesire
+        Power state desired by our controlling driver.
+    */
+    unsigned long           DriverDesire;
+
+    /*! @var DeviceDesire
+        Power state desired by a subclassed device object.
+    */
+    unsigned long           DeviceDesire;
+
+    /*! @var ChildrenDesire
+        Power state desired by all children.
+    */
+    unsigned long           ChildrenDesire;
+
+    /*! @var DesiredPowerState
+        This is the power state we desire currently.
+    */
+    unsigned long           DesiredPowerState;
+
+    /*! @var PreviousRequest
+        This is what our parent thinks our need is.
+    */
+    unsigned long           PreviousRequest;
+
+	/*! @var Name
+		Cache result from getName(), used in logging.
+	*/
+    const char *            Name;
+
+    /*! @var Platform
+        Cache result from getPlatform(), used in logging and registering.
+    */
+    IOPlatformExpert *      Platform;
+
+    /*! @var NumberOfPowerStates
+        Number of power states in the power array.
+    */
+    unsigned long           NumberOfPowerStates;
+
+    /*! @var PowerStates
+        Power state array.
+    */
+    IOPMPowerState *		PowerStates;
+
+    /*! @var ControllingDriver
+        The controlling driver.
+    */
+    IOService *				ControllingDriver;
+
+    /*! @var AggressivenessValues
+        Array of aggressiveness values.
+    */
+    unsigned long			AggressivenessValue[ kMaxType + 1 ];
+
+    /*! @var AggressivenessValid
+        True for aggressiveness values that are currently valid.
+    */
+    bool					AggressivenessValid[ kMaxType + 1 ];
+
+    /*! @var CurrentPowerState
+        The ordinal of our current power state.
+    */
+    unsigned long			CurrentPowerState;
+
+    /*! @var ParentsCurrentPowerFlags
+        Logical OR of power flags for each power domain parent.
+    */
+    IOPMPowerFlags			ParentsCurrentPowerFlags;
+
+    /*! @var MaxCapability
+        Ordinal of highest power state we can achieve in current power domain.
+    */
+    unsigned long			MaxCapability;
+
+    /*! @var OutputPowerCharacterFlags
+        Logical OR of all output power character flags in the array.
+    */
+    IOPMPowerFlags			OutputPowerCharacterFlags;
+
+    /*! @var SerialNumber
+        Used to uniquely identify power management notification to apps and clients.
+    */
+    UInt16					SerialNumber;
+
+    /*! @var ResponseArray
+        OSArray which manages responses from notified apps and clients.
+    */
+    OSArray *				ResponseArray;
+
+    /*! @var OutOfBandParameter
+        Used to communicate desired function to tellClientsWithResponse().
+        This is used because it avoids changing the signatures of the affected virtual methods. 
+    */
+    int						OutOfBandParameter;
+
+    AbsoluteTime            DriverCallStartTime;
+    IOPMPowerFlags          CurrentCapabilityFlags;
+    unsigned long           CurrentPowerConsumption;
+    unsigned long           TempClampPowerState;
+    unsigned long           TempClampCount;
+    IOPMWorkQueue *			PMWorkQueue;
+    IOPMRequest *			PMRequest;
+    OSSet *					InsertInterestSet;
+    OSSet *					RemoveInterestSet;
+    OSArray *				NotifyChildArray;
+    unsigned long			WaitReason;
+    unsigned long			NextMachineState;
+    thread_call_t			DriverCallEntry;
+    void *					DriverCallParamPtr;
+    IOItemCount				DriverCallParamCount;
+    IOItemCount				DriverCallParamSlots;
+    IOOptionBits			DriverCallReason;
+    long                    ActivityTicklePowerState;
+    bool					StrictTreeOrder;
+    bool					DriverCallBusy;
+    bool					ActivityTimerStopped;
+    bool					WillAdjustPowerState;
+    bool					WillPMStop;
+
+#if PM_VARS_SUPPORT
+    IOPMprot *				PMVars;
+#endif
+
+    /*! @function serialize
+        Serialize IOServicePM state for debug output.
+    */
+    virtual bool serialize( OSSerialize * s ) const;
+};
+
+#define fWeAreRoot                  pwrMgt->WeAreRoot
+#define fInterestedDrivers          pwrMgt->InterestedDrivers
+#define fDriverTimer                pwrMgt->DriverTimer
+#define fAckTimer                   pwrMgt->AckTimer
+#define fSettleTimer                pwrMgt->SettleTimer
+#define fMachineState               pwrMgt->MachineState
+#define fSettleTimeUS               pwrMgt->SettleTimeUS
+#define fHeadNoteFlags              pwrMgt->HeadNoteFlags
+#define fHeadNoteState              pwrMgt->HeadNoteState
+#define fHeadNoteOutputFlags        pwrMgt->HeadNoteOutputFlags
+#define fHeadNoteDomainState        pwrMgt->HeadNoteDomainState
+#define fHeadNoteParent             pwrMgt->HeadNoteParent
+#define fHeadNoteCapabilityFlags    pwrMgt->HeadNoteCapabilityFlags
+#define fHeadNotePendingAcks        pwrMgt->HeadNotePendingAcks
+#define fPMLock                     pwrMgt->PMLock
+#define fInitialChange              pwrMgt->InitialChange
+#define fNeedToBecomeUsable         pwrMgt->NeedToBecomeUsable
+#define fDeviceOverrides            pwrMgt->DeviceOverrides
+#define fClampOn                    pwrMgt->ClampOn
+#define fOwner                      pwrMgt->Owner
+#define fActivityLock               pwrMgt->ActivityLock
+#define fIdleTimerEventSource       pwrMgt->IdleTimerEventSource
+#define fIdleTimerPeriod            pwrMgt->IdleTimerPeriod
+#define fDeviceActive               pwrMgt->DeviceActive
+#define fDeviceActiveTimestamp      pwrMgt->DeviceActiveTimestamp
+#define fDriverDesire               pwrMgt->DriverDesire
+#define fDeviceDesire               pwrMgt->DeviceDesire
+#define fChildrenDesire             pwrMgt->ChildrenDesire
+#define fDesiredPowerState          pwrMgt->DesiredPowerState
+#define fPreviousRequest            pwrMgt->PreviousRequest
+#define fName                       pwrMgt->Name
+#define fPlatform                   pwrMgt->Platform
+#define fNumberOfPowerStates        pwrMgt->NumberOfPowerStates
+#define fPowerStates                pwrMgt->PowerStates
+#define fControllingDriver          pwrMgt->ControllingDriver
+#define fAggressivenessValue        pwrMgt->AggressivenessValue
+#define fAggressivenessValid        pwrMgt->AggressivenessValid
+#define fCurrentPowerState          pwrMgt->CurrentPowerState
+#define fParentsKnowState           pwrMgt->ParentsKnowState
+#define fParentsCurrentPowerFlags   pwrMgt->ParentsCurrentPowerFlags
+#define fMaxCapability              pwrMgt->MaxCapability
+#define fOutputPowerCharacterFlags  pwrMgt->OutputPowerCharacterFlags
+#define fSerialNumber               pwrMgt->SerialNumber
+#define fResponseArray              pwrMgt->ResponseArray
+#define fDoNotPowerDown             pwrMgt->DoNotPowerDown
+#define fOutOfBandParameter         pwrMgt->OutOfBandParameter
+#define fDriverCallStartTime        pwrMgt->DriverCallStartTime
+#define fCurrentCapabilityFlags     pwrMgt->CurrentCapabilityFlags
+#define fCurrentPowerConsumption    pwrMgt->CurrentPowerConsumption
+#define fTempClampPowerState        pwrMgt->TempClampPowerState
+#define fTempClampCount             pwrMgt->TempClampCount
+#define fPMWorkQueue                pwrMgt->PMWorkQueue
+#define fPMRequest                  pwrMgt->PMRequest
+#define fWaitReason                 pwrMgt->WaitReason
+#define fNextMachineState           pwrMgt->NextMachineState
+#define fDriverCallReason           pwrMgt->DriverCallReason
+#define fDriverCallEntry            pwrMgt->DriverCallEntry
+#define fDriverCallParamPtr         pwrMgt->DriverCallParamPtr
+#define fDriverCallParamCount       pwrMgt->DriverCallParamCount
+#define fDriverCallParamSlots       pwrMgt->DriverCallParamSlots
+#define fDriverCallBusy             pwrMgt->DriverCallBusy
+#define fWillPMStop                 pwrMgt->WillPMStop
+#define fActivityTickled            pwrMgt->ActivityTickled
+#define fInsertInterestSet          pwrMgt->InsertInterestSet
+#define fRemoveInterestSet          pwrMgt->RemoveInterestSet
+#define fStrictTreeOrder            pwrMgt->StrictTreeOrder
+#define fNotifyChildArray           pwrMgt->NotifyChildArray
+#define fWillAdjustPowerState       pwrMgt->WillAdjustPowerState
+#define fActivityTimerStopped       pwrMgt->ActivityTimerStopped
+#define fActivityTicklePowerState   pwrMgt->ActivityTicklePowerState
+#define fPMVars                     pwrMgt->PMVars
+
+/*!
+@defined ACK_TIMER_PERIOD
+@discussion When an IOService is waiting for acknowledgement to a power change
+notification from an interested driver or the controlling driver its ack timer
+is ticking every tenth of a second.
+(100000000 nanoseconds are one tenth of a second).
+*/
+#define ACK_TIMER_PERIOD 100000000
+
+#define IOPMParentInitiated     1   // this power change initiated by our  parent
+#define IOPMWeInitiated         2   // this power change initiated by this device
+#define IOPMNotDone             4   // we couldn't make this change
+#define IOPMNotInUse            8   // this list element not currently in use
+#define IOPMDomainWillChange    16  // change started by PowerDomainWillChangeTo
+#define IOPMDomainDidChange     32  // change started by PowerDomainDidChangeTo
+
+struct changeNoteItem {
+    unsigned long       flags;
+    unsigned long       newStateNumber;
+    IOPMPowerFlags      outputPowerCharacter;
+    IOPMPowerFlags      inputPowerRequirement;
+    IOPMPowerFlags      domainState;
+    IOPowerConnection * parent;
+    IOPMPowerFlags      singleParentState;
+    IOPMPowerFlags      capabilityFlags;
+};
+
+enum {
+    kDriverCallInformPreChange,
+    kDriverCallInformPostChange,
+    kDriverCallSetPowerState
+};
+
+struct DriverCallParam {
+    OSObject *  Target;
+    IOReturn    Result;
+};
+
+// values of outofbandparameter
+enum {
+    kNotifyApps,
+    kNotifyPriority
+};
+
+// used for applyToInterested
+struct context {
+    OSArray *      responseFlags;
+    UInt16         serialNumber;
+    UInt16         counter;
+    UInt32         maxTimeRequested;
+    int            msgType;
+    IOService *    us;
+    unsigned long  stateNumber;
+    IOPMPowerFlags stateFlags;
+    const char * errorLog;
+};
+
+//*********************************************************************************
+// PM command types
+//*********************************************************************************
+
+enum {
+    /* Command Types */
+    kIOPMRequestTypeInvalid                = 0x00,
+    kIOPMRequestTypePMStop                 = 0x01,
+    kIOPMRequestTypeAddPowerChild1         = 0x02,
+    kIOPMRequestTypeAddPowerChild2         = 0x03,
+    kIOPMRequestTypeAddPowerChild3         = 0x04,
+    kIOPMRequestTypeRegisterPowerDriver    = 0x05,
+    kIOPMRequestTypeAdjustPowerState       = 0x06,
+    kIOPMRequestTypeMakeUsable             = 0x07,
+    kIOPMRequestTypeTemporaryPowerClamp    = 0x08,
+    kIOPMRequestTypePowerDomainWillChange  = 0x09,
+    kIOPMRequestTypePowerDomainDidChange   = 0x0A,
+    kIOPMRequestTypeChangePowerStateTo     = 0x0B,
+    kIOPMRequestTypeChangePowerStateToPriv = 0x0C,
+    kIOPMRequestTypePowerOverrideOnPriv    = 0x0D,
+    kIOPMRequestTypePowerOverrideOffPriv   = 0x0E,
+    kIOPMRequestTypeActivityTickle         = 0x0F,
+    /* Reply Types */
+    kIOPMRequestTypeReplyStart             = 0x80,
+    kIOPMRequestTypeAckPowerChange         = 0x81,
+    kIOPMRequestTypeAckSetPowerState       = 0x82,
+    kIOPMRequestTypeAllowPowerChange       = 0x83,
+    kIOPMRequestTypeCancelPowerChange      = 0x84,
+    kIOPMRequestTypeInterestChanged        = 0x85
+};
+
+//*********************************************************************************
+// PM Helper Classes
+//*********************************************************************************
+
+class IOPMRequest : public IOCommand
+{
+    OSDeclareDefaultStructors( IOPMRequest )
+
+protected:
+    IOOptionBits        fType;          // request type
+    IOService *         fTarget;        // request target
+    IOPMRequest *       fParent;        // parent request
+    IOItemCount         fChildCount;    // wait if non-zero
+
+public:
+    void *              fArg0;
+    void *              fArg1;
+    void *              fArg2;
+
+    inline bool         hasChildRequest( void ) const
+    {
+        return (fChildCount != 0);
+    }
+
+    inline bool         hasParentRequest( void ) const
+    {
+        return (fParent != 0);
+    }
+
+    inline void         setParentRequest( IOPMRequest * parent )
+    {
+        if (!fParent)
+        {
+            fParent = parent;
+            fParent->fChildCount++;
+        }
+    }
+
+    inline IOOptionBits getType( void ) const
+    {
+        return fType;
+    }
+
+    inline bool         isReply( void ) const
+    {
+        return (fType > kIOPMRequestTypeReplyStart);
+    }
+
+    inline IOService *  getTarget( void ) const
+    {
+        return fTarget;
+    }
+
+    static IOPMRequest *create( void );
+
+    void    reset( void );
+
+    bool    init( IOService * owner, IOOptionBits type );
+};
+
+class IOPMRequestQueue : public IOEventSource
+{
+    OSDeclareDefaultStructors( IOPMRequestQueue )
+
+public:
+    typedef bool (*Action)( IOService *, IOPMRequest *, IOPMRequestQueue * );
+
+protected:
+    queue_head_t    fQueue;
+    IOLock *        fLock;
+
+    virtual bool checkForWork( void );
+    virtual void free( void );
+    virtual bool init( IOService * inOwner, Action inAction );
+
+public:
+    static  IOPMRequestQueue * create( IOService * inOwner, Action inAction );
+    void    queuePMRequest( IOPMRequest * request );
+    void    queuePMRequestChain( IOPMRequest ** requests, IOItemCount count );
+    void    signalWorkAvailable( void );
+};
+
+class IOPMWorkQueue : public IOEventSource
+{
+    OSDeclareDefaultStructors( IOPMWorkQueue )
+
+public:
+    typedef bool (*Action)( IOService *, IOPMRequest *, IOPMWorkQueue * );
+
+protected:
+    queue_head_t    fWorkQueue;
+    Action          fWorkAction;
+    Action          fRetireAction;
+
+    virtual bool checkForWork( void );
+    virtual bool init( IOService * inOwner, Action work, Action retire );
+
+public:
+    static IOPMWorkQueue * create( IOService * inOwner, Action work, Action retire );
+    void   queuePMRequest( IOPMRequest * request );
+};
+
+#endif /* !_IOKIT_IOSERVICEPMPRIVATE_H */
diff --git a/iokit/Kernel/IOServicePrivate.h b/iokit/Kernel/IOServicePrivate.h
index 213b2ea3e..577ddd7e1 100644
--- a/iokit/Kernel/IOServicePrivate.h
+++ b/iokit/Kernel/IOServicePrivate.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -128,11 +134,7 @@ public:
 };
 
 enum {
-#ifdef LESS_THREAD_CREATE
-    kMaxConfigThreads	= 4,
-#else
-    kMaxConfigThreads	= 32,
-#endif
+	kMaxConfigThreads	= CONFIG_MAX_THREADS,
 };
 
 enum {
diff --git a/iokit/Kernel/IOSharedDataQueue.cpp b/iokit/Kernel/IOSharedDataQueue.cpp
new file mode 100644
index 000000000..85bd0e3db
--- /dev/null
+++ b/iokit/Kernel/IOSharedDataQueue.cpp
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <IOKit/IOSharedDataQueue.h>
+#include <IOKit/IODataQueueShared.h>
+#include <IOKit/IOLib.h>
+#include <IOKit/IOMemoryDescriptor.h>
+
+#ifdef dequeue
+#undef dequeue
+#endif
+
+#define super IODataQueue
+
+OSDefineMetaClassAndStructors(IOSharedDataQueue, IODataQueue)
+
+IOSharedDataQueue *IOSharedDataQueue::withCapacity(UInt32 size)
+{
+    IOSharedDataQueue *dataQueue = new IOSharedDataQueue;
+
+    if (dataQueue) {
+        if  (!dataQueue->initWithCapacity(size)) {
+            dataQueue->release();
+            dataQueue = 0;
+        }
+    }
+
+    return dataQueue;
+}
+
+IOSharedDataQueue *IOSharedDataQueue::withEntries(UInt32 numEntries, UInt32 entrySize)
+{
+    IOSharedDataQueue *dataQueue = new IOSharedDataQueue;
+
+    if (dataQueue) {
+        if (!dataQueue->initWithEntries(numEntries, entrySize)) {
+            dataQueue->release();
+            dataQueue = 0;
+        }
+    }
+
+    return dataQueue;
+}
+
+Boolean IOSharedDataQueue::initWithCapacity(UInt32 size)
+{
+    IODataQueueAppendix *   appendix;
+    
+    if (!super::init()) {
+        return false;
+    }
+    
+    dataQueue = (IODataQueueMemory *)IOMallocAligned(round_page_32(size + DATA_QUEUE_MEMORY_HEADER_SIZE + DATA_QUEUE_MEMORY_APPENDIX_SIZE), PAGE_SIZE);
+    if (dataQueue == 0) {
+        return false;
+    }
+
+    dataQueue->queueSize    = size;
+    dataQueue->head         = 0;
+    dataQueue->tail         = 0;
+    
+    appendix            = (IODataQueueAppendix *)((UInt8 *)dataQueue + size + DATA_QUEUE_MEMORY_HEADER_SIZE);
+    appendix->version   = 0;
+    notifyMsg           = &(appendix->msgh);
+    setNotificationPort(MACH_PORT_NULL);
+
+    return true;
+}
+
+void IOSharedDataQueue::free()
+{
+    if (dataQueue) {
+        IOFreeAligned(dataQueue, round_page_32(dataQueue->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE + DATA_QUEUE_MEMORY_APPENDIX_SIZE));
+        dataQueue = NULL;
+    }
+
+    super::free();
+}
+
+IOMemoryDescriptor *IOSharedDataQueue::getMemoryDescriptor()
+{
+    IOMemoryDescriptor *descriptor = 0;
+
+    if (dataQueue != 0) {
+        descriptor = IOMemoryDescriptor::withAddress(dataQueue, dataQueue->queueSize + DATA_QUEUE_MEMORY_HEADER_SIZE + DATA_QUEUE_MEMORY_APPENDIX_SIZE, kIODirectionOutIn);
+    }
+
+    return descriptor;
+}
+
+
+IODataQueueEntry * IOSharedDataQueue::peek()
+{
+    IODataQueueEntry *entry = 0;
+
+    if (dataQueue && (dataQueue->head != dataQueue->tail)) {
+        IODataQueueEntry *  head		= 0;
+        UInt32              headSize    = 0;
+        UInt32              headOffset  = dataQueue->head;
+        UInt32              queueSize   = dataQueue->queueSize;
+
+        head 		= (IODataQueueEntry *)((char *)dataQueue->queue + headOffset);
+        headSize 	= head->size;
+        
+		// Check if there's enough room before the end of the queue for a header.
+        // If there is room, check if there's enough room to hold the header and
+        // the data.
+
+        if ((headOffset + DATA_QUEUE_ENTRY_HEADER_SIZE > queueSize) ||
+            ((headOffset + headSize + DATA_QUEUE_ENTRY_HEADER_SIZE) > queueSize))
+        {
+            // No room for the header or the data, wrap to the beginning of the queue.
+            entry = dataQueue->queue;
+        } else {
+            entry = head;
+        }
+    }
+
+    return entry;
+}
+
+Boolean IOSharedDataQueue::dequeue(void *data, UInt32 *dataSize)
+{
+    Boolean             retVal          = TRUE;
+    IODataQueueEntry *  entry           = 0;
+    UInt32              entrySize       = 0;
+    UInt32              newHeadOffset   = 0;
+
+    if (dataQueue) {
+        if (dataQueue->head != dataQueue->tail) {
+            IODataQueueEntry *  head		= 0;
+            UInt32              headSize    = 0;
+            UInt32              headOffset  = dataQueue->head;
+            UInt32              queueSize   = dataQueue->queueSize;
+
+            head 		= (IODataQueueEntry *)((char *)dataQueue->queue + headOffset);
+            headSize 	= head->size;
+            
+            // we wraped around to beginning, so read from there
+			// either there was not even room for the header
+			if ((headOffset + DATA_QUEUE_ENTRY_HEADER_SIZE > queueSize) ||
+				// or there was room for the header, but not for the data
+				((headOffset + headSize + DATA_QUEUE_ENTRY_HEADER_SIZE) > queueSize)) {
+                entry           = dataQueue->queue;
+                entrySize       = entry->size;
+                newHeadOffset   = entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE;
+            // else it is at the end
+            } else {
+                entry           = head;
+                entrySize       = entry->size;
+                newHeadOffset   = headOffset + entrySize + DATA_QUEUE_ENTRY_HEADER_SIZE;
+            }
+        }
+
+        if (entry) {
+            if (data) {
+                if (dataSize) {
+                    if (entrySize <= *dataSize) {
+                        memcpy(data, &(entry->data), entrySize);
+                        dataQueue->head = newHeadOffset;
+                    } else {
+                        retVal = FALSE;
+                    }
+                } else {
+                    retVal = FALSE;
+                }
+            } else {
+                dataQueue->head = newHeadOffset;
+            }
+
+            if (dataSize) {
+                *dataSize = entrySize;
+            }
+        } else {
+            retVal = FALSE;
+        }
+    } else {
+        retVal = FALSE;
+    }
+    
+    return retVal;
+}
+
+
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 0);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 1);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 2);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 3);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 4);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 5);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 6);
+OSMetaClassDefineReservedUnused(IOSharedDataQueue, 7);
diff --git a/iokit/Kernel/IOStartIOKit.cpp b/iokit/Kernel/IOStartIOKit.cpp
index 65c022bda..25c13cb0e 100644
--- a/iokit/Kernel/IOStartIOKit.cpp
+++ b/iokit/Kernel/IOStartIOKit.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998,1999 Apple Computer, Inc.  All rights reserved. 
@@ -49,8 +55,9 @@ extern void OSlibkernInit (void);
 extern void ml_hpet_cfg(uint32_t, uint32_t);
 
 #include <kern/clock.h>
+#include <sys/time.h>
 
-void IOKitResetTime( void )
+void IOKitInitializeTime( void )
 {
 	mach_timespec_t		t;
 
@@ -58,15 +65,26 @@ void IOKitResetTime( void )
 	t.tv_nsec = 0;
 	IOService::waitForService(
 		IOService::resourceMatching("IORTC"), &t );
-#ifndef i386
+#ifdef ppc
 	IOService::waitForService(
 		IOService::resourceMatching("IONVRAM"), &t );
-
 #endif
 
     clock_initialize_calendar();
 }
 
+void IOKitResetTime( void )
+{
+    uint32_t secs, microsecs;
+
+    clock_initialize_calendar();
+
+    clock_get_calendar_microtime(&secs, &microsecs);
+    gIOLastWakeTime.tv_sec  = secs;
+    gIOLastWakeTime.tv_usec = microsecs;
+}
+
+
 // From <osfmk/kern/debug.c>
 extern int debug_mode;
 
@@ -179,7 +197,32 @@ void StartIOKit( void * p1, void * p2, void * p3, void * p4 )
         gIOCatalogue->recordStartupExtensions();
 
         rootNub->registerService();
+
+#if !NO_KEXTD
+       /* Add a busy count to keep the registry busy until kextd has
+        * completely finished launching. This is decremented when kextd
+        * messages the kernel after the in-kernel linker has been
+        * removed and personalities have been sent.
+        */
+        IOService::getServiceRoot()->adjustBusy(1);
+#endif
     }
 }
 
+void
+IORegistrySetOSBuildVersion(char * build_version)
+{
+    IORegistryEntry * root = IORegistryEntry::getRegistryRoot();
+
+    if (root) {
+        if (build_version) {
+            root->setProperty(kOSBuildVersionKey, build_version);
+        } else {
+            root->removeProperty(kOSBuildVersionKey);
+        }
+    }
+
+    return;
+}
+
 }; /* extern "C" */
diff --git a/iokit/Kernel/IOStringFuncs.c b/iokit/Kernel/IOStringFuncs.c
index f7d9508b5..7b43926d3 100644
--- a/iokit/Kernel/IOStringFuncs.c
+++ b/iokit/Kernel/IOStringFuncs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*      Copyright (c) 1995 NeXT Computer, Inc.  All rights reserved.
  *
@@ -107,10 +113,7 @@ isdigit(char c)
  * alphabets and digits are each contiguous.
  */
 long
-strtol(nptr, endptr, base)
-	const char *nptr;
-	char **endptr;
-	register int base;
+strtol(const char *nptr, char **endptr, int base)
 {
 	register const char *s = nptr;
 	register unsigned long acc;
@@ -193,10 +196,7 @@ strtol(nptr, endptr, base)
 }
 
 unsigned long
-strtoul(nptr, endptr, base)
-	const char *nptr;
-	char **endptr;
-	register int base;
+strtoul(const char *nptr, char **endptr, int base)
 {
 	register const char *s = nptr;
 	register unsigned long acc;
@@ -264,10 +264,7 @@ strtoul(nptr, endptr, base)
  * alphabets and digits are each contiguous.
  */
 quad_t
-strtoq(nptr, endptr, base)
-	const char *nptr;
-	char **endptr;
-	register int base;
+strtoq(const char *nptr, char **endptr, int base)
 {
 	register const char *s;
 	register u_quad_t acc;
@@ -357,7 +354,7 @@ strtoq(nptr, endptr, base)
  * Ignores `locale' stuff.  Assumes that the upper and lower case
  * alphabets and digits are each contiguous.
  */
-uint64_t
+u_quad_t
 strtouq(const char *nptr,
 	char **endptr,
 	register int base)
diff --git a/iokit/Kernel/IOSyncer.cpp b/iokit/Kernel/IOSyncer.cpp
index c72281501..8379cc7ca 100644
--- a/iokit/Kernel/IOSyncer.cpp
+++ b/iokit/Kernel/IOSyncer.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOSyncer.cpp created by wgulland on 2000-02-02 */
 
diff --git a/iokit/Kernel/IOTimerEventSource.cpp b/iokit/Kernel/IOTimerEventSource.cpp
index b81757f38..aca7f3fbc 100644
--- a/iokit/Kernel/IOTimerEventSource.cpp
+++ b/iokit/Kernel/IOTimerEventSource.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -216,7 +222,7 @@ IOReturn IOTimerEventSource::setTimeoutUS(UInt32 us)
 IOReturn IOTimerEventSource::setTimeout(UInt32 interval, UInt32 scale_factor)
 {
     AbsoluteTime end;
-    
+
     clock_interval_to_deadline(interval, scale_factor, &end);
     return wakeAtTime(end);
 }
diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp
index d68a73632..bb451f8c4 100644
--- a/iokit/Kernel/IOUserClient.cpp
+++ b/iokit/Kernel/IOUserClient.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -29,10 +35,24 @@
 #include <IOKit/IOCatalogue.h>
 #include <IOKit/IOMemoryDescriptor.h>
 #include <IOKit/IOLib.h>
+#include <sys/proc.h>
 
 #include <IOKit/assert.h>
 
 #include "IOServicePrivate.h"
+#include "IOKitKernelInternal.h"
+
+#define SCALAR64(x) ((io_user_scalar_t)((unsigned int)x))
+#define SCALAR32(x) ((uint32_t )x)
+#define ARG32(x)    ((void *)SCALAR32(x))
+#define REF64(x)    ((io_user_reference_t)((natural_t)(x)))
+#define REF32(x)    ((int)(x))
+
+enum
+{
+    kIOUCAsync0Flags = 3ULL,
+    kIOUCAsync64Flag = 1ULL
+};
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -383,12 +403,12 @@ class IOServiceUserNotification : public IOUserNotification
 
     struct PingMsg {
         mach_msg_header_t		msgHdr;
-        OSNotificationHeader		notifyHeader;
+        OSNotificationHeader64		notifyHeader;
     };
 
     enum { kMaxOutstanding = 1024 };
 
-    PingMsg *		pingMsg;
+    PingMsg	*	pingMsg;
     vm_size_t		msgSize;
     OSArray 	*	newSet;
     OSObject	*	lastEntry;
@@ -397,7 +417,8 @@ class IOServiceUserNotification : public IOUserNotification
 public:
 
     virtual bool init( mach_port_t port, natural_t type,
-                       OSAsyncReference reference );
+                       void * reference, vm_size_t referenceSize,
+		       bool clientIs64 );
     virtual void free();
 
     static bool _handler( void * target,
@@ -415,16 +436,21 @@ class IOServiceMessageUserNotification : public IOUserNotification
         mach_msg_header_t		msgHdr;
 	mach_msg_body_t			msgBody;
 	mach_msg_port_descriptor_t	ports[1];
-        OSNotificationHeader		notifyHeader;
+        OSNotificationHeader64		notifyHeader;
     };
 
     PingMsg *		pingMsg;
     vm_size_t		msgSize;
+    uint8_t		clientIs64;
+    int			owningPID;
 
 public:
 
     virtual bool init( mach_port_t port, natural_t type,
-                       OSAsyncReference reference, vm_size_t extraSize );
+		       void * reference, vm_size_t referenceSize,
+		       vm_size_t extraSize,
+		       bool clientIs64 );
+
     virtual void free();
     
     static IOReturn _handler( void * target, void * ref,
@@ -505,13 +531,17 @@ OSDefineMetaClassAndStructors(IOServiceUserNotification, IOUserNotification)
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOServiceUserNotification::init( mach_port_t port, natural_t type,
-                                OSAsyncReference reference )
+				       void * reference, vm_size_t referenceSize,
+				       bool clientIs64 )
 {
     newSet = OSArray::withCapacity( 1 );
     if( !newSet)
         return( false );
 
-    msgSize = sizeof( PingMsg) + 0;
+    if (referenceSize > sizeof(OSAsyncReference64))
+        return( false );
+
+    msgSize = sizeof(PingMsg) - sizeof(OSAsyncReference64) + referenceSize;
     pingMsg = (PingMsg *) IOMalloc( msgSize);
     if( !pingMsg)
         return( false );
@@ -527,7 +557,7 @@ bool IOServiceUserNotification::init( mach_port_t port, natural_t type,
 
     pingMsg->notifyHeader.size = 0;
     pingMsg->notifyHeader.type = type;
-    bcopy( reference, pingMsg->notifyHeader.reference, sizeof(OSAsyncReference) );
+    bcopy( reference, pingMsg->notifyHeader.reference, referenceSize );
 
     return( super::init() );
 }
@@ -636,11 +666,19 @@ OSDefineMetaClassAndStructors(IOServiceMessageUserNotification, IOUserNotificati
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOServiceMessageUserNotification::init( mach_port_t port, natural_t type,
-                       OSAsyncReference reference, vm_size_t extraSize )
+				void * reference, vm_size_t referenceSize, vm_size_t extraSize,
+				bool client64 )
 {
 
-    extraSize += sizeof(IOServiceInterestContent);
-    msgSize = sizeof( PingMsg) + extraSize;
+    if (referenceSize > sizeof(OSAsyncReference64))
+        return( false );
+
+    clientIs64 = client64;
+
+    owningPID = proc_selfpid();
+
+    extraSize += sizeof(IOServiceInterestContent64);
+    msgSize = sizeof(PingMsg) - sizeof(OSAsyncReference64) + referenceSize + extraSize;
     pingMsg = (PingMsg *) IOMalloc( msgSize);
     if( !pingMsg)
         return( false );
@@ -663,7 +701,7 @@ bool IOServiceMessageUserNotification::init( mach_port_t port, natural_t type,
 
     pingMsg->notifyHeader.size 		= extraSize;
     pingMsg->notifyHeader.type 		= type;
-    bcopy( reference, pingMsg->notifyHeader.reference, sizeof(OSAsyncReference) );
+    bcopy( reference, pingMsg->notifyHeader.reference, referenceSize );
 
     return( super::init() );
 }
@@ -694,22 +732,39 @@ IOReturn IOServiceMessageUserNotification::handler( void * ref,
                                     UInt32 messageType, IOService * provider,
                                     void * messageArgument, vm_size_t argSize )
 {
-    kern_return_t		kr;
-    ipc_port_t 			thisPort, providerPort;
-    IOServiceInterestContent * 	data = (IOServiceInterestContent *)
-                                       pingMsg->notifyHeader.content;
+    kern_return_t		 kr;
+    ipc_port_t 			 thisPort, providerPort;
+    IOServiceInterestContent64 * data = (IOServiceInterestContent64 *)
+					((((uint8_t *) pingMsg) + msgSize) - pingMsg->notifyHeader.size);
+                                        // == pingMsg->notifyHeader.content;
+
+    if (kIOMessageCopyClientID == messageType)
+    {
+	*((void **) messageArgument) = IOCopyLogNameForPID(owningPID);
+	return (kIOReturnSuccess);
+    }
 
     data->messageType = messageType;
-    if( argSize == 0) {
-        argSize = sizeof( messageArgument);
-        data->messageArgument[0] = messageArgument;
-    } else {
+
+    if( argSize == 0)
+    {
+	data->messageArgument[0] = (io_user_reference_t) messageArgument;
+	if (clientIs64)
+	    argSize = sizeof(data->messageArgument[0]);
+	else
+	{
+	    data->messageArgument[0] |= (data->messageArgument[0] << 32);
+	    argSize = sizeof(messageArgument);
+	}
+    }
+    else
+    {
         if( argSize > kIOUserNotifyMaxMessageSize)
             argSize = kIOUserNotifyMaxMessageSize;
         bcopy( messageArgument, data->messageArgument, argSize );
     }
-    pingMsg->msgHdr.msgh_size = sizeof( PingMsg)
-        + sizeof( IOServiceInterestContent )
+    pingMsg->msgHdr.msgh_size = msgSize - pingMsg->notifyHeader.size
+        + sizeof( IOServiceInterestContent64 )
         - sizeof( data->messageArgument)
         + argSize;
 
@@ -752,11 +807,22 @@ void IOUserClient::setAsyncReference(OSAsyncReference asyncRef,
                                      mach_port_t wakePort,
                                      void *callback, void *refcon)
 {
-    asyncRef[kIOAsyncReservedIndex] = (natural_t) wakePort;
-    asyncRef[kIOAsyncCalloutFuncIndex] = (natural_t) callback;
+    asyncRef[kIOAsyncReservedIndex]      = ((natural_t) wakePort) 
+					 | (kIOUCAsync0Flags & asyncRef[kIOAsyncReservedIndex]);
+    asyncRef[kIOAsyncCalloutFuncIndex]   = (natural_t) callback;
     asyncRef[kIOAsyncCalloutRefconIndex] = (natural_t) refcon;
 }
 
+void IOUserClient::setAsyncReference64(OSAsyncReference64 asyncRef,
+					mach_port_t wakePort,
+					mach_vm_address_t callback, io_user_reference_t refcon)
+{
+    asyncRef[kIOAsyncReservedIndex]      = ((io_user_reference_t) wakePort)
+					 | (kIOUCAsync0Flags & asyncRef[kIOAsyncReservedIndex]);
+    asyncRef[kIOAsyncCalloutFuncIndex]   = (io_user_reference_t) callback;
+    asyncRef[kIOAsyncCalloutRefconIndex] = refcon;
+}
+
 inline OSDictionary * CopyConsoleUser(UInt32 uid)
 {
 	OSArray * array;
@@ -942,6 +1008,30 @@ IOMemoryMap * IOUserClient::mapClientMemory(
     return( map );
 }
 
+IOMemoryMap * IOUserClient::mapClientMemory64( 
+	IOOptionBits		type,
+	task_t			task,
+	IOOptionBits		mapFlags,
+	mach_vm_address_t	atAddress )
+{
+    IOReturn		err;
+    IOOptionBits	options = 0;
+    IOMemoryDescriptor * memory;
+    IOMemoryMap *	map = 0;
+
+    err = clientMemoryForType( (UInt32) type, &options, &memory );
+
+    if( memory && (kIOReturnSuccess == err)) {
+
+        options = (options & ~kIOMapUserOptionsMask)
+		| (mapFlags & kIOMapUserOptionsMask);
+	map = memory->createMappingInTask( task, atAddress, options );
+	memory->release();
+    }
+
+    return( map );
+}
+
 IOReturn IOUserClient::exportObjectToClient(task_t task,
 			OSObject *obj, io_object_t *clientObj)
 {
@@ -1007,39 +1097,98 @@ getTargetAndTrapForIndex(IOService ** targetP, UInt32 index)
 IOReturn IOUserClient::sendAsyncResult(OSAsyncReference reference,
                                        IOReturn result, void *args[], UInt32 numArgs)
 {
-    struct ReplyMsg {
-        mach_msg_header_t	msgHdr;
-        OSNotificationHeader	notifyHdr;
-        IOAsyncCompletionContent asyncContent;
-        void *			args[kMaxAsyncArgs];
+    OSAsyncReference64  reference64;
+    io_user_reference_t args64[kMaxAsyncArgs];
+    unsigned int        idx;
+
+    if (numArgs > kMaxAsyncArgs)
+        return kIOReturnMessageTooLarge;
+
+    for (idx = 0; idx < kOSAsyncRef64Count; idx++)
+	reference64[idx] = REF64(reference[idx]);
+
+    for (idx = 0; idx < numArgs; idx++)
+	args64[idx] = REF64(args[idx]);
+
+    return (sendAsyncResult64(reference64, result, args64, numArgs));
+}
+
+IOReturn IOUserClient::sendAsyncResult64(OSAsyncReference64 reference,
+                                        IOReturn result, io_user_reference_t args[], UInt32 numArgs)
+{
+    struct ReplyMsg
+    {
+	mach_msg_header_t msgHdr;
+	union
+	{
+	    struct
+	    {
+		OSNotificationHeader	 notifyHdr;
+		IOAsyncCompletionContent asyncContent;
+		uint32_t		 args[kMaxAsyncArgs];
+	    } msg32;
+	    struct
+	    {
+		OSNotificationHeader64	 notifyHdr;
+		IOAsyncCompletionContent asyncContent;
+		uint32_t		 pad;
+		io_user_reference_t	 args[kMaxAsyncArgs];
+	    } msg64;
+	} m;
     };
-    ReplyMsg replyMsg;
-    mach_port_t	replyPort;
+    ReplyMsg      replyMsg;
+    mach_port_t	  replyPort;
     kern_return_t kr;
 
     // If no reply port, do nothing.
-    replyPort = (mach_port_t) reference[0];
-    if(replyPort == MACH_PORT_NULL)
+    replyPort = (mach_port_t) (reference[0] & ~kIOUCAsync0Flags);
+    if (replyPort == MACH_PORT_NULL)
         return kIOReturnSuccess;
     
-    if(numArgs > kMaxAsyncArgs)
+    if (numArgs > kMaxAsyncArgs)
         return kIOReturnMessageTooLarge;
+
     replyMsg.msgHdr.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND /*remote*/,
-                                                0 /*local*/);
-    replyMsg.msgHdr.msgh_size =
-        sizeof(replyMsg) - (kMaxAsyncArgs-numArgs)*sizeof(void *);
+						0 /*local*/);
     replyMsg.msgHdr.msgh_remote_port = replyPort;
-    replyMsg.msgHdr.msgh_local_port = 0;
-    replyMsg.msgHdr.msgh_id = kOSNotificationMessageID;
+    replyMsg.msgHdr.msgh_local_port  = 0;
+    replyMsg.msgHdr.msgh_id          = kOSNotificationMessageID;
+    if (kIOUCAsync64Flag & reference[0])
+    {
+	replyMsg.msgHdr.msgh_size =
+	    sizeof(replyMsg.msgHdr) + sizeof(replyMsg.m.msg64) 
+	    - (kMaxAsyncArgs - numArgs) * sizeof(io_user_reference_t);
+	replyMsg.m.msg64.notifyHdr.size = sizeof(IOAsyncCompletionContent)
+					+ sizeof(uint32_t)
+					+ numArgs * sizeof(io_user_reference_t);
+	replyMsg.m.msg64.notifyHdr.type = kIOAsyncCompletionNotificationType;
+	bcopy(reference, replyMsg.m.msg64.notifyHdr.reference, sizeof(OSAsyncReference64));
+
+	replyMsg.m.msg64.asyncContent.result = result;
+	if (numArgs)
+	    bcopy(args, replyMsg.m.msg64.args, numArgs * sizeof(io_user_reference_t));
+    }
+    else
+    {
+	unsigned int idx;
+
+	replyMsg.msgHdr.msgh_size =
+	    sizeof(replyMsg.msgHdr) + sizeof(replyMsg.m.msg32)
+	    - (kMaxAsyncArgs - numArgs) * sizeof(uint32_t);
+
+	replyMsg.m.msg32.notifyHdr.size = sizeof(IOAsyncCompletionContent)
+					+ numArgs * sizeof(uint32_t);
+	replyMsg.m.msg32.notifyHdr.type = kIOAsyncCompletionNotificationType;
 
-    replyMsg.notifyHdr.size = sizeof(IOAsyncCompletionContent)
-                            + numArgs*sizeof(void *);
-    replyMsg.notifyHdr.type = kIOAsyncCompletionNotificationType;
-    bcopy( reference, replyMsg.notifyHdr.reference, sizeof(OSAsyncReference));
+	for (idx = 0; idx < kOSAsyncRefCount; idx++)
+	    replyMsg.m.msg32.notifyHdr.reference[idx] = REF32(reference[idx]);
+
+	replyMsg.m.msg32.asyncContent.result = result;
+
+	for (idx = 0; idx < numArgs; idx++)
+	    replyMsg.m.msg32.args[idx] = REF32(args[idx]);
+    }
 
-    replyMsg.asyncContent.result = result;
-    if(numArgs > 0)
-        bcopy(args, replyMsg.args, sizeof(void *)*numArgs);
      kr = mach_msg_send_from_kernel( &replyMsg.msgHdr,
             replyMsg.msgHdr.msgh_size);
     if( KERN_SUCCESS != kr)
@@ -1047,6 +1196,7 @@ IOReturn IOUserClient::sendAsyncResult(OSAsyncReference reference,
     return kr;
 }
 
+
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 extern "C" {
@@ -1109,7 +1259,7 @@ kern_return_t is_io_object_get_superclass(
 	my_cstr = superclass->getClassName();
 		
 	if (my_cstr) {
-		strncpy(class_name, my_cstr, sizeof(io_name_t)-1);
+		strlcpy(class_name, my_cstr, sizeof(io_name_t));
 		return( kIOReturnSuccess );
 	}
 	return (kIOReturnNotFound);
@@ -1148,7 +1298,7 @@ kern_return_t is_io_object_get_bundle_identifier(
 	
 	my_cstr = identifier->getCStringNoCopy();
 	if (my_cstr) {
-		strncpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t)-1);
+		strlcpy(bundle_name, identifier->getCStringNoCopy(), sizeof(io_name_t));
 		return( kIOReturnSuccess );
 	}
 
@@ -1171,7 +1321,7 @@ kern_return_t is_io_object_conforms_to(
 /* Routine io_object_get_retain_count */
 kern_return_t is_io_object_get_retain_count(
 	io_object_t object,
-	int *retainCount )
+	uint32_t *retainCount )
 {
     if( !object)
         return( kIOReturnBadArgument );
@@ -1252,7 +1402,7 @@ kern_return_t is_io_service_match_property_table_ool(
 	io_object_t service,
 	io_buf_ptr_t matching,
 	mach_msg_type_number_t matchingCnt,
-	natural_t *result,
+	kern_return_t *result,
 	boolean_t *matches )
 {
     kern_return_t	kr;
@@ -1304,7 +1454,7 @@ kern_return_t is_io_service_get_matching_services_ool(
 	mach_port_t master_port,
 	io_buf_ptr_t matching,
 	mach_msg_type_number_t matchingCnt,
-	natural_t *result,
+	kern_return_t *result,
 	io_object_t *existing )
 {
     kern_return_t	kr;
@@ -1324,14 +1474,14 @@ kern_return_t is_io_service_get_matching_services_ool(
     return( kr );
 }
 
-/* Routine io_service_add_notification */
-kern_return_t is_io_service_add_notification(
+static kern_return_t internal_io_service_add_notification(
 	mach_port_t master_port,
 	io_name_t notification_type,
 	io_string_t matching,
 	mach_port_t port,
-	io_async_ref_t reference,
-	mach_msg_type_number_t referenceCnt,
+	void * reference,
+	vm_size_t referenceSize,
+	bool client64,
 	io_object_t * notification )
 {
     IOServiceUserNotification *	userNotify = 0;
@@ -1371,7 +1521,7 @@ kern_return_t is_io_service_add_notification(
         userNotify = new IOServiceUserNotification;
 
         if( userNotify && !userNotify->init( port, userMsgType,
-                                             reference)) {
+                                             reference, referenceSize, client64)) {
             userNotify->release();
             userNotify = 0;
         }
@@ -1398,16 +1548,48 @@ kern_return_t is_io_service_add_notification(
     return( err );
 }
 
-/* Routine io_service_add_notification_ool */
-kern_return_t is_io_service_add_notification_ool(
+
+/* Routine io_service_add_notification */
+kern_return_t is_io_service_add_notification(
+	mach_port_t master_port,
+	io_name_t notification_type,
+	io_string_t matching,
+	mach_port_t port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	io_object_t * notification )
+{
+    return (internal_io_service_add_notification(master_port, notification_type, 
+		matching, port, &reference[0], sizeof(io_async_ref_t),
+		false, notification));
+}
+
+/* Routine io_service_add_notification_64 */
+kern_return_t is_io_service_add_notification_64(
+	mach_port_t master_port,
+	io_name_t notification_type,
+	io_string_t matching,
+	mach_port_t wake_port,
+	io_async_ref64_t reference,
+	mach_msg_type_number_t referenceCnt,
+	io_object_t *notification )
+{
+    return (internal_io_service_add_notification(master_port, notification_type, 
+		matching, wake_port, &reference[0], sizeof(io_async_ref64_t),
+		true, notification));
+}
+
+
+static kern_return_t internal_io_service_add_notification_ool(
 	mach_port_t master_port,
 	io_name_t notification_type,
 	io_buf_ptr_t matching,
 	mach_msg_type_number_t matchingCnt,
 	mach_port_t wake_port,
-	io_async_ref_t reference,
-	mach_msg_type_number_t referenceCnt,
-	natural_t *result,
+	void * reference,
+	vm_size_t referenceSize,
+	bool client64,
+	kern_return_t *result,
 	io_object_t *notification )
 {
     kern_return_t	kr;
@@ -1419,14 +1601,47 @@ kern_return_t is_io_service_add_notification_ool(
 
     if( KERN_SUCCESS == kr) {
         // must return success after vm_map_copyout() succeeds
-	*result = is_io_service_add_notification( master_port, notification_type,
-			(char *) data, wake_port, reference, referenceCnt, notification );
+	*result = internal_io_service_add_notification( master_port, notification_type,
+			(char *) data, wake_port, reference, referenceSize, client64, notification );
 	vm_deallocate( kernel_map, data, matchingCnt );
     }
 
     return( kr );
 }
 
+/* Routine io_service_add_notification_ool */
+kern_return_t is_io_service_add_notification_ool(
+	mach_port_t master_port,
+	io_name_t notification_type,
+	io_buf_ptr_t matching,
+	mach_msg_type_number_t matchingCnt,
+	mach_port_t wake_port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	kern_return_t *result,
+	io_object_t *notification )
+{
+    return (internal_io_service_add_notification_ool(master_port, notification_type, 
+		matching, matchingCnt, wake_port, &reference[0], sizeof(io_async_ref_t),
+		false, result, notification));
+}
+
+/* Routine io_service_add_notification_ool_64 */
+kern_return_t is_io_service_add_notification_ool_64(
+	mach_port_t master_port,
+	io_name_t notification_type,
+	io_buf_ptr_t matching,
+	mach_msg_type_number_t matchingCnt,
+	mach_port_t wake_port,
+	io_async_ref64_t reference,
+	mach_msg_type_number_t referenceCnt,
+	kern_return_t *result,
+	io_object_t *notification )
+{
+    return (internal_io_service_add_notification_ool(master_port, notification_type, 
+		matching, matchingCnt, wake_port, &reference[0], sizeof(io_async_ref64_t),
+		true, result, notification));
+}
 
 /* Routine io_service_add_notification_old */
 kern_return_t is_io_service_add_notification_old(
@@ -1441,13 +1656,14 @@ kern_return_t is_io_service_add_notification_old(
             matching, port, &ref, 1, notification ));
 }
 
-/* Routine io_service_add_message_notification */
-kern_return_t is_io_service_add_interest_notification(
+
+static kern_return_t internal_io_service_add_interest_notification(
         io_object_t _service,
         io_name_t type_of_interest,
         mach_port_t port,
-	io_async_ref_t reference,
-	mach_msg_type_number_t referenceCnt,
+	void * reference,
+	vm_size_t referenceSize,
+	bool client64,
         io_object_t * notification )
 {
 
@@ -1464,7 +1680,9 @@ kern_return_t is_io_service_add_interest_notification(
         userNotify = new IOServiceMessageUserNotification;
 
         if( userNotify && !userNotify->init( port, kIOServiceMessageNotificationType,
-                                             reference, kIOUserNotifyMaxMessageSize )) {
+                                             reference, referenceSize,
+					     kIOUserNotifyMaxMessageSize,
+					     client64 )) {
             userNotify->release();
             userNotify = 0;
         }
@@ -1487,6 +1705,33 @@ kern_return_t is_io_service_add_interest_notification(
     return( err );
 }
 
+/* Routine io_service_add_message_notification */
+kern_return_t is_io_service_add_interest_notification(
+        io_object_t service,
+        io_name_t type_of_interest,
+        mach_port_t port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+        io_object_t * notification )
+{
+    return (internal_io_service_add_interest_notification(service, type_of_interest,
+		    port, &reference[0], sizeof(io_async_ref_t), false, notification));
+}
+
+/* Routine io_service_add_interest_notification_64 */
+kern_return_t is_io_service_add_interest_notification_64(
+	io_object_t service,
+	io_name_t type_of_interest,
+	mach_port_t wake_port,
+	io_async_ref64_t reference,
+	mach_msg_type_number_t referenceCnt,
+	io_object_t *notification )
+{
+    return (internal_io_service_add_interest_notification(service, type_of_interest,
+		    wake_port, &reference[0], sizeof(io_async_ref64_t), true, notification));
+}
+
+
 /* Routine io_service_acknowledge_notification */
 kern_return_t is_io_service_acknowledge_notification(
 	io_object_t _service,
@@ -1534,7 +1779,7 @@ kern_return_t is_io_registry_get_root_entry(
 kern_return_t is_io_registry_create_iterator(
 	mach_port_t master_port,
 	io_name_t plane,
-	int options,
+	uint32_t options,
 	io_object_t *iterator )
 {
     if( master_port != master_device_port)
@@ -1550,7 +1795,7 @@ kern_return_t is_io_registry_create_iterator(
 kern_return_t is_io_registry_entry_create_iterator(
 	io_object_t registry_entry,
 	io_name_t plane,
-	int options,
+	uint32_t options,
 	io_object_t *iterator )
 {
     CHECK( IORegistryEntry, registry_entry, entry );
@@ -1711,7 +1956,7 @@ static kern_return_t copyoutkdata( void * data, vm_size_t len,
 kern_return_t is_io_registry_entry_get_property_bytes(
 	io_object_t registry_entry,
 	io_name_t property_name,
-	io_scalar_inband_t buf,
+	io_struct_inband_t buf,
 	mach_msg_type_number_t *dataCnt )
 {
     OSObject	*	obj;
@@ -1813,7 +2058,7 @@ kern_return_t is_io_registry_entry_get_property_recursively(
 	io_object_t registry_entry,
 	io_name_t plane,
 	io_name_t property_name,
-        int options,
+        uint32_t options,
 	io_buf_ptr_t *properties,
 	mach_msg_type_number_t *propertiesCnt )
 {
@@ -1886,7 +2131,7 @@ kern_return_t is_io_registry_entry_set_properties
 	io_object_t registry_entry,
 	io_buf_ptr_t properties,
 	mach_msg_type_number_t propertiesCnt,
-        natural_t * result)
+        kern_return_t * result)
 {
     OSObject *		obj;
     kern_return_t	err;
@@ -1948,7 +2193,7 @@ kern_return_t is_io_registry_entry_get_parent_iterator(
 /* Routine io_service_get_busy_state */
 kern_return_t is_io_service_get_busy_state(
 	io_object_t _service,
-	int *busyState )
+	uint32_t *busyState )
 {
     CHECK( IOService, _service, service );
 
@@ -1982,7 +2227,7 @@ kern_return_t is_io_service_wait_quiet(
 /* Routine io_service_request_probe */
 kern_return_t is_io_service_request_probe(
 	io_object_t _service,
-	int options )
+	uint32_t options )
 {
     CHECK( IOService, _service, service );
 
@@ -1994,7 +2239,7 @@ kern_return_t is_io_service_request_probe(
 kern_return_t is_io_service_open(
 	io_object_t _service,
 	task_t owningTask,
-	int connect_type,
+	uint32_t connect_type,
 	io_object_t *connection )
 {
     IOUserClient	*	client;
@@ -2017,11 +2262,11 @@ kern_return_t is_io_service_open(
 kern_return_t is_io_service_open_extended(
 	io_object_t _service,
 	task_t owningTask,
-	int connect_type,
+	uint32_t connect_type,
 	NDR_record_t ndr,
 	io_buf_ptr_t properties,
 	mach_msg_type_number_t propertiesCnt,
-        natural_t * result,
+        kern_return_t * result,
 	io_object_t *connection )
 {
     IOUserClient * client = 0;
@@ -2144,9 +2389,9 @@ kern_return_t is_io_connect_get_service(
 /* Routine io_connect_set_notification_port */
 kern_return_t is_io_connect_set_notification_port(
 	io_object_t connection,
-	int notification_type,
+	uint32_t notification_type,
 	mach_port_t port,
-	int reference)
+	uint32_t reference)
 {
     CHECK( IOUserClient, connection, client );
 
@@ -2154,35 +2399,49 @@ kern_return_t is_io_connect_set_notification_port(
 						reference ));
 }
 
-kern_return_t is_io_connect_map_memory(
-	io_object_t     connect,
-	int		type,
-	task_t		task,
-	vm_address_t *	mapAddr,
-	vm_size_t    *	mapSize,
-	int		flags )
+/* Routine io_connect_set_notification_port */
+kern_return_t is_io_connect_set_notification_port_64(
+	io_object_t connection,
+	uint32_t notification_type,
+	mach_port_t port,
+	io_user_reference_t reference)
+{
+    CHECK( IOUserClient, connection, client );
+
+    return( client->registerNotificationPort( port, notification_type,
+						reference ));
+}
+
+/* Routine io_connect_map_memory_into_task */
+kern_return_t is_io_connect_map_memory_into_task
+(
+	io_connect_t connection,
+	uint32_t memory_type,
+	task_t into_task,
+	mach_vm_address_t *address,
+	mach_vm_size_t *size,
+	uint32_t flags
+)
 {
     IOReturn		err;
     IOMemoryMap *	map;
 
-    CHECK( IOUserClient, connect, client );
+    CHECK( IOUserClient, connection, client );
 
-    map = client->mapClientMemory( type, task, flags, *mapAddr );
+    map = client->mapClientMemory64( memory_type, into_task, flags, *address );
 
     if( map) {
-        *mapAddr = map->getVirtualAddress();
-        if( mapSize)
-            *mapSize = map->getLength();
+        *address = map->getAddress();
+        if( size)
+            *size = map->getSize();
 
         if( client->sharedInstance
-	    || (task != current_task())) {
+	    || (into_task != current_task())) {
             // push a name out to the task owning the map,
             // so we can clean up maps
-#if IOASSERT
-	    mach_port_name_t name =
-#endif
-	    IOMachPort::makeSendRightForTask(
-                                    task, map, IKOT_IOKIT_OBJECT );
+	    mach_port_name_t name __unused =
+		IOMachPort::makeSendRightForTask(
+                                    into_task, map, IKOT_IOKIT_OBJECT );
             assert( name );
 
         } else {
@@ -2203,6 +2462,30 @@ kern_return_t is_io_connect_map_memory(
     return( err );
 }
 
+/* Routine is_io_connect_map_memory */
+kern_return_t is_io_connect_map_memory(
+	io_object_t     connect,
+	uint32_t	type,
+	task_t		task,
+	vm_address_t *	mapAddr,
+	vm_size_t    *	mapSize,
+	uint32_t	flags )
+{
+    IOReturn	      err;
+    mach_vm_address_t address;
+    mach_vm_size_t    size;
+
+    address = SCALAR64(*mapAddr);
+    size    = SCALAR64(*mapSize);
+
+    err = is_io_connect_map_memory_into_task(connect, type, task, &address, &size, flags);
+
+    *mapAddr = SCALAR32(address);
+    *mapSize = SCALAR32(size);
+
+    return (err);
+}
+
 IOMemoryMap * IOUserClient::removeMappingForDescriptor(IOMemoryDescriptor * mem)
 {
     OSIterator *  iter;
@@ -2230,27 +2513,29 @@ IOMemoryMap * IOUserClient::removeMappingForDescriptor(IOMemoryDescriptor * mem)
     return (map);
 }
 
-kern_return_t is_io_connect_unmap_memory(
-	io_object_t     connect,
-	int		type,
-	task_t		task,
-	vm_address_t 	mapAddr )
+/* Routine io_connect_unmap_memory_from_task */
+kern_return_t is_io_connect_unmap_memory_from_task
+(
+	io_connect_t connection,
+	uint32_t memory_type,
+	task_t from_task,
+	mach_vm_address_t address)
 {
     IOReturn		err;
     IOOptionBits	options = 0;
     IOMemoryDescriptor * memory;
     IOMemoryMap *	map;
 
-    CHECK( IOUserClient, connect, client );
+    CHECK( IOUserClient, connection, client );
 
-    err = client->clientMemoryForType( (UInt32) type, &options, &memory );
+    err = client->clientMemoryForType( (UInt32) memory_type, &options, &memory );
 
     if( memory && (kIOReturnSuccess == err)) {
 
         options = (options & ~kIOMapUserOptionsMask)
 		| kIOMapAnywhere | kIOMapReference;
 
-	map = memory->map( task, mapAddr, options );
+	map = memory->createMappingInTask( from_task, address, options );
 	memory->release();
         if( map)
 	{
@@ -2260,17 +2545,17 @@ kern_return_t is_io_connect_unmap_memory(
             IOLockUnlock( gIOObjectPortLock);
 
 	    mach_port_name_t name = 0;
-	    if (task != current_task())
-		name = IOMachPort::makeSendRightForTask( task, map, IKOT_IOKIT_OBJECT );
+	    if (from_task != current_task())
+		name = IOMachPort::makeSendRightForTask( from_task, map, IKOT_IOKIT_OBJECT );
 	    if (name)
 	    {
 		map->unmap();
-		err = iokit_mod_send_right( task, name, -2 );
+		err = iokit_mod_send_right( from_task, name, -2 );
 		err = kIOReturnSuccess;
 	    }
 	    else
 		IOMachPort::releasePortForObject( map, IKOT_IOKIT_OBJECT );
-	    if (task == current_task())
+	    if (from_task == current_task())
 		map->release();
         }
 	else
@@ -2280,6 +2565,22 @@ kern_return_t is_io_connect_unmap_memory(
     return( err );
 }
 
+kern_return_t is_io_connect_unmap_memory(
+	io_object_t     connect,
+	uint32_t	type,
+	task_t		task,
+	vm_address_t 	mapAddr )
+{
+    IOReturn		err;
+    mach_vm_address_t   address;
+    
+    address = SCALAR64(mapAddr);
+    
+    err = is_io_connect_unmap_memory_from_task(connect, type, task, mapAddr);
+
+    return (err);
+}
+
 
 /* Routine io_connect_add_client */
 kern_return_t is_io_connect_add_client(
@@ -2298,417 +2599,671 @@ kern_return_t is_io_connect_set_properties(
 	io_object_t connection,
 	io_buf_ptr_t properties,
 	mach_msg_type_number_t propertiesCnt,
-        natural_t * result)
+        kern_return_t * result)
 {
     return( is_io_registry_entry_set_properties( connection, properties, propertiesCnt, result ));
 }
 
 
-/* Routine io_connect_method_scalarI_scalarO */
-kern_return_t is_io_connect_method_scalarI_scalarO(
-	io_object_t	connect,
-	UInt32		index,
-        void *	 	input[],
-        IOByteCount	inputCount,
-        void *	 	output[],
-        IOByteCount *	outputCount )
+/* Routine io_user_client_method */
+kern_return_t is_io_connect_method
+(
+	io_connect_t connection,
+	uint32_t selector,
+	io_scalar_inband64_t scalar_input,
+	mach_msg_type_number_t scalar_inputCnt,
+	io_struct_inband_t inband_input,
+	mach_msg_type_number_t inband_inputCnt,
+	mach_vm_address_t ool_input,
+	mach_vm_size_t ool_input_size,
+	io_scalar_inband64_t scalar_output,
+	mach_msg_type_number_t *scalar_outputCnt,
+	io_struct_inband_t inband_output,
+	mach_msg_type_number_t *inband_outputCnt,
+	mach_vm_address_t ool_output,
+	mach_vm_size_t * ool_output_size
+)
 {
-    IOReturn 		err;
-    IOExternalMethod *	method;
-    IOService *		object;
-    IOMethod		func;
+    CHECK( IOUserClient, connection, client );
 
-    CHECK( IOUserClient, connect, client);
-    if( (method = client->getTargetAndMethodForIndex(&object, index))) {
-      do {
-        err = kIOReturnBadArgument;
-	if( kIOUCScalarIScalarO != (method->flags & kIOUCTypeMask))
-	    continue;
-	if( inputCount != method->count0)
-	    continue;
-	if( *outputCount != method->count1)
-	    continue;
+    IOExternalMethodArguments args;
+    IOReturn ret;
+    IOMemoryDescriptor * inputMD  = 0;
+    IOMemoryDescriptor * outputMD = 0;
 
-	func = method->func;
+    bzero(&args.__reserved[0], sizeof(args.__reserved));
+    args.version = kIOExternalMethodArgumentsCurrentVersion;
 
-	switch( inputCount) {
+    args.selector = selector;
 
-	    case 6:
-		err = (object->*func)(  input[0], input[1], input[2],
-					input[3], input[4], input[5] );
-		break;
-	    case 5:
-		err = (object->*func)(  input[0], input[1], input[2],
-					input[3], input[4], 
-					&output[0] );
-		break;
-	    case 4:
-		err = (object->*func)(  input[0], input[1], input[2],
-					input[3],
-					&output[0], &output[1] );
-		break;
-	    case 3:
-		err = (object->*func)(  input[0], input[1], input[2],
-					&output[0], &output[1], &output[2] );
-		break;
-	    case 2:
-		err = (object->*func)(  input[0], input[1],
-					&output[0], &output[1], &output[2],
-					&output[3] );
-		break;
-	    case 1:
-		err = (object->*func)(  input[0],
-					&output[0], &output[1], &output[2],
-					&output[3], &output[4] );
-		break;
-	    case 0:
-		err = (object->*func)(  &output[0], &output[1], &output[2],
-					&output[3], &output[4], &output[5] );
-		break;
+    args.asyncWakePort       = MACH_PORT_NULL;
+    args.asyncReference      = 0;
+    args.asyncReferenceCount = 0;
 
-	    default:
-		IOLog("%s: Bad method table\n", client->getName());
-	}
-      } while( false);
+    args.scalarInput = scalar_input;
+    args.scalarInputCount = scalar_inputCnt;
+    args.structureInput = inband_input;
+    args.structureInputSize = inband_inputCnt;
 
-    } else
-        err = kIOReturnUnsupported;
+    if (ool_input)
+	inputMD = IOMemoryDescriptor::withAddressRange(ool_input, ool_input_size, 
+						    kIODirectionOut, current_task());
 
-    return( err);
-}
+    args.structureInputDescriptor = inputMD;
 
-/* Routine io_connect_method_scalarI_structureO */
-kern_return_t is_io_connect_method_scalarI_structureO(
-	io_object_t	connect,
-	UInt32		index,
-        void *	 	input[],
-        IOByteCount	inputCount,
-        void *		output,
-        IOByteCount *	outputCount )
-{
-    IOReturn 		err;
-    IOExternalMethod *	method;
-    IOService *		object;
-    IOMethod		func;
+    args.scalarOutput = scalar_output;
+    args.scalarOutputCount = *scalar_outputCnt;
+    args.structureOutput = inband_output;
+    args.structureOutputSize = *inband_outputCnt;
 
-    CHECK( IOUserClient, connect, client);
+    if (ool_output)
+    {
+	outputMD = IOMemoryDescriptor::withAddressRange(ool_output, *ool_output_size, 
+						    kIODirectionIn, current_task());
+    }
 
-    if( (method = client->getTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-	if( kIOUCScalarIStructO != (method->flags & kIOUCTypeMask))
-	    continue;
-	if( inputCount != method->count0)
-	    continue;
-	if( (0xffffffff != method->count1)
-		&& (*outputCount != method->count1))
-	    continue;
+    args.structureOutputDescriptor = outputMD;
+    args.structureOutputDescriptorSize = *ool_output_size;
 
-	func = method->func;
+    ret = client->externalMethod( selector, &args );
 
-	switch( inputCount) {
+    *scalar_outputCnt = args.scalarOutputCount;
+    *inband_outputCnt = args.structureOutputSize;
+    *ool_output_size  = args.structureOutputDescriptorSize;
 
-	    case 5:
-		err = (object->*func)(  input[0], input[1], input[2],
-                                        input[3], input[4],
-                                        output );
-		break;
-	    case 4:
-		err = (object->*func)(  input[0], input[1], input[2],
-					input[3],
-					output, (void *)outputCount );
-		break;
-	    case 3:
-		err = (object->*func)(  input[0], input[1], input[2],
-					output, (void *)outputCount, 0 );
-		break;
-	    case 2:
-		err = (object->*func)(  input[0], input[1],
-					output, (void *)outputCount, 0, 0 );
-		break;
-	    case 1:
-		err = (object->*func)(  input[0],
-					output, (void *)outputCount, 0, 0, 0 );
-		break;
-	    case 0:
-		err = (object->*func)(  output, (void *)outputCount, 0, 0, 0, 0 );
-		break;
+    if (inputMD)
+	inputMD->release();
+    if (outputMD)
+	outputMD->release();
 
-	    default:
-		IOLog("%s: Bad method table\n", client->getName());
-	}
-      } while( false);
+    return (ret);
+}
 
-    } else
-        err = kIOReturnUnsupported;
+/* Routine io_async_user_client_method */
+kern_return_t is_io_connect_async_method
+(
+	io_connect_t connection,
+	mach_port_t wake_port,
+	io_async_ref64_t reference,
+	mach_msg_type_number_t referenceCnt,
+	uint32_t selector,
+	io_scalar_inband64_t scalar_input,
+	mach_msg_type_number_t scalar_inputCnt,
+	io_struct_inband_t inband_input,
+	mach_msg_type_number_t inband_inputCnt,
+	mach_vm_address_t ool_input,
+	mach_vm_size_t ool_input_size,
+	io_scalar_inband64_t scalar_output,
+	mach_msg_type_number_t *scalar_outputCnt,
+	io_struct_inband_t inband_output,
+	mach_msg_type_number_t *inband_outputCnt,
+	mach_vm_address_t ool_output,
+	mach_vm_size_t * ool_output_size
+)
+{
+    CHECK( IOUserClient, connection, client );
 
-    return( err);
+    IOExternalMethodArguments args;
+    IOReturn ret;
+    IOMemoryDescriptor * inputMD  = 0;
+    IOMemoryDescriptor * outputMD = 0;
+
+    bzero(&args.__reserved[0], sizeof(args.__reserved));
+    args.version = kIOExternalMethodArgumentsCurrentVersion;
+
+    reference[0]	     = (io_user_reference_t) wake_port;
+    if (vm_map_is_64bit(get_task_map(current_task()))) 
+	reference[0]	     |= kIOUCAsync64Flag;
+
+    args.selector = selector;
+
+    args.asyncWakePort       = wake_port;
+    args.asyncReference      = reference;
+    args.asyncReferenceCount = referenceCnt;
+
+    args.scalarInput = scalar_input;
+    args.scalarInputCount = scalar_inputCnt;
+    args.structureInput = inband_input;
+    args.structureInputSize = inband_inputCnt;
+
+    if (ool_input)
+	inputMD = IOMemoryDescriptor::withAddressRange(ool_input, ool_input_size,
+						    kIODirectionOut, current_task());
+
+    args.structureInputDescriptor = inputMD;
+
+    args.scalarOutput = scalar_output;
+    args.scalarOutputCount = *scalar_outputCnt;
+    args.structureOutput = inband_output;
+    args.structureOutputSize = *inband_outputCnt;
+
+    if (ool_output)
+    {
+	outputMD = IOMemoryDescriptor::withAddressRange(ool_output, *ool_output_size,
+						    kIODirectionIn, current_task());
+    }
+
+    args.structureOutputDescriptor = outputMD;
+    args.structureOutputDescriptorSize = *ool_output_size;
+
+    ret = client->externalMethod( selector, &args );
+
+    *inband_outputCnt = args.structureOutputSize;
+    *ool_output_size  = args.structureOutputDescriptorSize;
+
+    if (inputMD)
+	inputMD->release();
+    if (outputMD)
+	outputMD->release();
+
+    return (ret);
 }
 
-/* Routine io_connect_method_scalarI_structureI */
-kern_return_t is_io_connect_method_scalarI_structureI(
-	io_connect_t 	connect,
-	UInt32		index,
-        void *	 	input[],
-        IOByteCount	inputCount,
-        UInt8 *		inputStruct,
-        IOByteCount	inputStructCount )
+/* Routine io_connect_method_scalarI_scalarO */
+kern_return_t is_io_connect_method_scalarI_scalarO(
+	io_object_t	   connect,
+	uint32_t	   index,
+        io_scalar_inband_t       input,
+        mach_msg_type_number_t	 inputCount,
+        io_scalar_inband_t       output,
+        mach_msg_type_number_t * outputCount )
+{
+    IOReturn err;
+    uint32_t i;
+    io_scalar_inband64_t _input;
+    io_scalar_inband64_t _output;
+
+    mach_msg_type_number_t struct_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+	
+    err = is_io_connect_method(connect, index, 
+		    _input, inputCount, 
+		    NULL, 0,
+		    0, 0,
+		    _output, outputCount,
+		    NULL, &struct_outputCnt,
+		    0, &ool_output_size);
+
+    for (i = 0; i < *outputCount; i++)
+	output[i] = SCALAR32(_output[i]);
+
+    return (err);
+}
+
+kern_return_t shim_io_connect_method_scalarI_scalarO(
+	IOExternalMethod *	method,
+	IOService *		object,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	 inputCount,
+        io_user_scalar_t * output,
+        mach_msg_type_number_t * outputCount )
 {
-    IOReturn 		err;
-    IOExternalMethod *	method;
-    IOService *		object;
     IOMethod		func;
+    io_scalar_inband_t  _output;
+    IOReturn 		err;
+    err = kIOReturnBadArgument;
 
-    CHECK( IOUserClient, connect, client);
+    do {
 
-    if( (method = client->getTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-	if( kIOUCScalarIStructI != (method->flags & kIOUCTypeMask))
-	    continue;
-	if( (0xffffffff != method->count0)
-		&& (inputCount != method->count0))
+	if( inputCount != method->count0)
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
 	    continue;
-	if( (0xffffffff != method->count1)
-		&& (inputStructCount != method->count1))
+	}
+	if( *outputCount != method->count1)
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
 	    continue;
+	}
 
 	func = method->func;
 
 	switch( inputCount) {
 
+	    case 6:
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					ARG32(input[3]), ARG32(input[4]), ARG32(input[5]) );
+		break;
 	    case 5:
-		err = (object->*func)( input[0], input[1], input[2],
-					input[3], input[4], 
-					inputStruct );
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					ARG32(input[3]), ARG32(input[4]), 
+					&_output[0] );
 		break;
 	    case 4:
-		err = (object->*func)( input[0], input[1], input[2],
-					input[3],
-					inputStruct, (void *)inputStructCount );
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					ARG32(input[3]),
+					&_output[0], &_output[1] );
 		break;
 	    case 3:
-		err = (object->*func)( input[0], input[1], input[2],
-					inputStruct, (void *)inputStructCount,
-					0 );
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					&_output[0], &_output[1], &_output[2] );
 		break;
 	    case 2:
-		err = (object->*func)( input[0], input[1],
-					inputStruct, (void *)inputStructCount,
-					0, 0 );
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]),
+					&_output[0], &_output[1], &_output[2],
+					&_output[3] );
 		break;
 	    case 1:
-		err = (object->*func)( input[0],
-					inputStruct, (void *)inputStructCount,
-					0, 0, 0 );
+		err = (object->*func)(  ARG32(input[0]),
+					&_output[0], &_output[1], &_output[2],
+					&_output[3], &_output[4] );
 		break;
 	    case 0:
-		err = (object->*func)( inputStruct, (void *)inputStructCount,
-					0, 0, 0, 0 );
+		err = (object->*func)(  &_output[0], &_output[1], &_output[2],
+					&_output[3], &_output[4], &_output[5] );
 		break;
 
 	    default:
-		IOLog("%s: Bad method table\n", client->getName());
+		IOLog("%s: Bad method table\n", object->getName());
 	}
-      } while( false);
+    }
+    while( false);
 
-    } else
-        err = kIOReturnUnsupported;
+    uint32_t i;
+    for (i = 0; i < *outputCount; i++)
+	output[i] = SCALAR32(_output[i]);
 
     return( err);
 }
 
-/* Routine io_connect_method_structureI_structureO */
-kern_return_t is_io_connect_method_structureI_structureO(
+/* Routine io_async_method_scalarI_scalarO */
+kern_return_t is_io_async_method_scalarI_scalarO(
+	io_object_t	   connect,
+	mach_port_t wake_port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	uint32_t	   index,
+        io_scalar_inband_t       input,
+        mach_msg_type_number_t	 inputCount,
+        io_scalar_inband_t       output,
+        mach_msg_type_number_t * outputCount )
+{
+    IOReturn err;
+    uint32_t i;
+    io_scalar_inband64_t _input;
+    io_scalar_inband64_t _output;
+    io_async_ref64_t _reference;
+
+    for (i = 0; i < referenceCnt; i++)
+	_reference[i] = REF64(reference[i]);
+
+    mach_msg_type_number_t struct_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+
+    err = is_io_connect_async_method(connect, 
+		    wake_port, _reference, referenceCnt,
+		    index, 
+		    _input, inputCount, 
+		    NULL, 0,
+		    0, 0,
+		    _output, outputCount,
+		    NULL, &struct_outputCnt,
+		    0, &ool_output_size);
+
+    for (i = 0; i < *outputCount; i++)
+	output[i] = SCALAR32(_output[i]);
+
+    return (err);
+}
+/* Routine io_async_method_scalarI_structureO */
+kern_return_t is_io_async_method_scalarI_structureO(
 	io_object_t	connect,
-	UInt32		index,
-        UInt8 *		input,
-        IOByteCount	inputCount,
-        UInt8 *		output,
-        IOByteCount *	outputCount )
+	mach_port_t wake_port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	uint32_t	index,
+        io_scalar_inband_t input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
 {
-    IOReturn 		err;
-    IOExternalMethod *	method;
-    IOService *		object;
-    IOMethod		func;
+    uint32_t i;
+    io_scalar_inband64_t _input;
+    io_async_ref64_t _reference;
 
-    CHECK( IOUserClient, connect, client);
+    for (i = 0; i < referenceCnt; i++)
+	_reference[i] = REF64(reference[i]);
 
-    if( (method = client->getTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-	if( kIOUCStructIStructO != (method->flags & kIOUCTypeMask))
-	    continue;
-	if( (0xffffffff != method->count0)
-		&& (inputCount != method->count0))
-	    continue;
-	if( (0xffffffff != method->count1)
-		&& (*outputCount != method->count1))
-	    continue;
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
 
-	func = method->func;
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+	
+    return (is_io_connect_async_method(connect, 
+		    wake_port, _reference, referenceCnt,
+		    index,
+		    _input, inputCount, 
+		    NULL, 0,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    output, outputCount,
+		    0, &ool_output_size));
+}
 
-	if( method->count1) {
-	    if( method->count0) {
-		err = (object->*func)( input, output,
-					(void *)inputCount, outputCount, 0, 0 );
-	    } else {
-		err = (object->*func)( output, outputCount, 0, 0, 0, 0 );
-	    }
-	} else {
-		err = (object->*func)( input, (void *)inputCount, 0, 0, 0, 0 );
-	}
+/* Routine io_async_method_scalarI_structureI */
+kern_return_t is_io_async_method_scalarI_structureI(
+	io_connect_t		connect,
+	mach_port_t wake_port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	uint32_t		index,
+        io_scalar_inband_t	input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t	inputStruct,
+        mach_msg_type_number_t	inputStructCount )
+{
+    uint32_t i;
+    io_scalar_inband64_t _input;
+    io_async_ref64_t _reference;
 
-      } while( false);
+    for (i = 0; i < referenceCnt; i++)
+	_reference[i] = REF64(reference[i]);
 
-    } else
-        err = kIOReturnUnsupported;
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_msg_type_number_t inband_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
 
-    return( err);
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+	
+    return (is_io_connect_async_method(connect, 
+		    wake_port, _reference, referenceCnt,
+		    index,
+		    _input, inputCount, 
+		    inputStruct, inputStructCount,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    NULL, &inband_outputCnt,
+		    0, &ool_output_size));
 }
 
-kern_return_t is_io_async_method_scalarI_scalarO(
-        io_object_t	connect,
-        mach_port_t	wakePort,
-	io_async_ref_t		reference,
-	mach_msg_type_number_t	referenceCnt,
-        UInt32		index,
-        void *	 	input[],
-        IOByteCount	inputCount,
-        void *	 	output[],
-        IOByteCount *	outputCount )
+/* Routine io_async_method_structureI_structureO */
+kern_return_t is_io_async_method_structureI_structureO(
+	io_object_t	connect,
+	mach_port_t wake_port,
+	io_async_ref_t reference,
+	mach_msg_type_number_t referenceCnt,
+	uint32_t	index,
+        io_struct_inband_t		input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
+{
+    uint32_t i;
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+    io_async_ref64_t _reference;
+
+    for (i = 0; i < referenceCnt; i++)
+	_reference[i] = REF64(reference[i]);
+
+    return (is_io_connect_async_method(connect,
+		    wake_port, _reference, referenceCnt,
+		    index,
+		    NULL, 0, 
+		    input, inputCount,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    output, outputCount,
+		    0, &ool_output_size));
+}
+
+
+kern_return_t shim_io_async_method_scalarI_scalarO(
+	IOExternalAsyncMethod *	method,
+	IOService *		object,
+	mach_port_t             asyncWakePort,
+	io_user_reference_t *   asyncReference,
+	uint32_t                asyncReferenceCount,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	 inputCount,
+        io_user_scalar_t * output,
+        mach_msg_type_number_t * outputCount )
 {
-    IOReturn 		err;
-    IOExternalAsyncMethod *method;
-    IOService *		object;
     IOAsyncMethod	func;
+    uint32_t		i;
+    io_scalar_inband_t  _output;
+    IOReturn 		err;
+    io_async_ref_t	reference;
 
-    CHECK( IOUserClient, connect, client);
-    if( (method = client->getAsyncTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-        if( kIOUCScalarIScalarO != (method->flags & kIOUCTypeMask))
-            continue;
-        if( inputCount != method->count0)
-            continue;
-        if( *outputCount != method->count1)
-            continue;
+    for (i = 0; i < asyncReferenceCount; i++)
+	reference[i] = REF32(asyncReference[i]);
 
-        reference[0] = (natural_t) wakePort;
-        func = method->func;
+    err = kIOReturnBadArgument;
+
+    do {
+
+	if( inputCount != method->count0)
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( *outputCount != method->count1)
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
+
+	func = method->func;
 
         switch( inputCount) {
 
             case 6:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
-                                        input[3], input[4], input[5] );
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]), ARG32(input[4]), ARG32(input[5]) );
                 break;
             case 5:
                 err = (object->*func)(  reference,
-                                        input[0], input[1], input[2],
-                                        input[3], input[4],
-                                        &output[0] );
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]), ARG32(input[4]),
+                                        &_output[0] );
                 break;
             case 4:
                 err = (object->*func)(  reference,
-                                        input[0], input[1], input[2],
-                                        input[3],
-                                        &output[0], &output[1] );
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]),
+                                        &_output[0], &_output[1] );
                 break;
             case 3:
                 err = (object->*func)(  reference,
-                                        input[0], input[1], input[2],
-                                        &output[0], &output[1], &output[2] );
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        &_output[0], &_output[1], &_output[2] );
                 break;
             case 2:
                 err = (object->*func)(  reference,
-                                        input[0], input[1],
-                                        &output[0], &output[1], &output[2],
-                                        &output[3] );
+                                        ARG32(input[0]), ARG32(input[1]),
+                                        &_output[0], &_output[1], &_output[2],
+                                        &_output[3] );
                 break;
             case 1:
                 err = (object->*func)(  reference,
-                                        input[0],
-                                        &output[0], &output[1], &output[2],
-                                        &output[3], &output[4] );
+                                        ARG32(input[0]), 
+					&_output[0], &_output[1], &_output[2],
+                                        &_output[3], &_output[4] );
                 break;
             case 0:
                 err = (object->*func)(  reference,
-                                        &output[0], &output[1], &output[2],
-                                        &output[3], &output[4], &output[5] );
+                                        &_output[0], &_output[1], &_output[2],
+                                        &_output[3], &_output[4], &_output[5] );
                 break;
 
             default:
-                IOLog("%s: Bad method table\n", client->getName());
+                IOLog("%s: Bad method table\n", object->getName());
         }
-      } while( false);
+    }
+    while( false);
 
-    } else
-        err = kIOReturnUnsupported;
+    for (i = 0; i < *outputCount; i++)
+	output[i] = SCALAR32(_output[i]);
 
     return( err);
 }
 
-kern_return_t is_io_async_method_scalarI_structureO(
-        io_object_t	connect,
-        mach_port_t	wakePort,
-	io_async_ref_t		reference,
-	mach_msg_type_number_t	referenceCnt,
-        UInt32		index,
-        void *	 	input[],
-        IOByteCount	inputCount,
-        void *		output,
-        IOByteCount *	outputCount )
+
+/* Routine io_connect_method_scalarI_structureO */
+kern_return_t is_io_connect_method_scalarI_structureO(
+	io_object_t	connect,
+	uint32_t	index,
+        io_scalar_inband_t input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
 {
+    uint32_t i;
+    io_scalar_inband64_t _input;
+
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+	
+    return (is_io_connect_method(connect, index, 
+		    _input, inputCount, 
+		    NULL, 0,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    output, outputCount,
+		    0, &ool_output_size));
+}
+
+kern_return_t shim_io_connect_method_scalarI_structureO(
+
+	IOExternalMethod *	method,
+	IOService *		object,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
+{
+    IOMethod		func;
     IOReturn 		err;
-    IOExternalAsyncMethod *method;
-    IOService *		object;
+
+    err = kIOReturnBadArgument;
+
+    do {
+	if( inputCount != method->count0)
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (*outputCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
+
+	func = method->func;
+
+	switch( inputCount) {
+
+	    case 5:
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]), ARG32(input[4]),
+                                        output );
+		break;
+	    case 4:
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					ARG32(input[3]),
+					output, (void *)outputCount );
+		break;
+	    case 3:
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					output, (void *)outputCount, 0 );
+		break;
+	    case 2:
+		err = (object->*func)(  ARG32(input[0]), ARG32(input[1]),
+					output, (void *)outputCount, 0, 0 );
+		break;
+	    case 1:
+		err = (object->*func)(  ARG32(input[0]),
+					output, (void *)outputCount, 0, 0, 0 );
+		break;
+	    case 0:
+		err = (object->*func)(  output, (void *)outputCount, 0, 0, 0, 0 );
+		break;
+
+	    default:
+		IOLog("%s: Bad method table\n", object->getName());
+	}
+    }
+    while( false);
+
+    return( err);
+}
+
+
+kern_return_t shim_io_async_method_scalarI_structureO(
+	IOExternalAsyncMethod *	method,
+	IOService *		object,
+	mach_port_t             asyncWakePort,
+	io_user_reference_t *   asyncReference,
+	uint32_t                asyncReferenceCount,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
+{
     IOAsyncMethod	func;
+    uint32_t		i;
+    IOReturn 		err;
+    io_async_ref_t	reference;
 
-    CHECK( IOUserClient, connect, client);
+    for (i = 0; i < asyncReferenceCount; i++)
+	reference[i] = REF32(asyncReference[i]);
 
-    if( (method = client->getAsyncTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-        if( kIOUCScalarIStructO != (method->flags & kIOUCTypeMask))
-            continue;
-        if( inputCount != method->count0)
-            continue;
-        if( (0xffffffff != method->count1)
-                && (*outputCount != method->count1))
-            continue;
+    err = kIOReturnBadArgument;
+    do {
+	if( inputCount != method->count0)
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (*outputCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
 
-        reference[0] = (natural_t) wakePort;
-        func = method->func;
+	func = method->func;
 
         switch( inputCount) {
 
             case 5:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
-                                        input[3], input[4],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]), ARG32(input[4]),
                                         output );
                 break;
             case 4:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
-                                        input[3],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]),
                                         output, (void *)outputCount );
                 break;
             case 3:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
                                         output, (void *)outputCount, 0 );
                 break;
             case 2:
                 err = (object->*func)(	reference,
-                                        input[0], input[1],
+                                        ARG32(input[0]), ARG32(input[1]),
                                         output, (void *)outputCount, 0, 0 );
                 break;
             case 1:
                 err = (object->*func)(	reference,
-                                        input[0],
+                                        ARG32(input[0]),
                                         output, (void *)outputCount, 0, 0, 0 );
                 break;
             case 0:
@@ -2717,78 +3272,176 @@ kern_return_t is_io_async_method_scalarI_structureO(
                 break;
 
             default:
-                IOLog("%s: Bad method table\n", client->getName());
+                IOLog("%s: Bad method table\n", object->getName());
         }
-      } while( false);
+    }
+    while( false);
 
-    } else
-        err = kIOReturnUnsupported;
+    return( err);
+}
+
+/* Routine io_connect_method_scalarI_structureI */
+kern_return_t is_io_connect_method_scalarI_structureI(
+	io_connect_t		connect,
+	uint32_t		index,
+        io_scalar_inband_t	input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t	inputStruct,
+        mach_msg_type_number_t	inputStructCount )
+{
+    uint32_t i;
+    io_scalar_inband64_t _input;
+
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_msg_type_number_t inband_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+
+    for (i = 0; i < inputCount; i++)
+	_input[i] = SCALAR64(input[i]);
+	
+    return (is_io_connect_method(connect, index, 
+		    _input, inputCount, 
+		    inputStruct, inputStructCount,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    NULL, &inband_outputCnt,
+		    0, &ool_output_size));
+}
+
+kern_return_t shim_io_connect_method_scalarI_structureI(
+    IOExternalMethod *	method,
+    IOService *		object,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		inputStruct,
+        mach_msg_type_number_t	inputStructCount )
+{
+    IOMethod		func;
+    IOReturn		err = kIOReturnBadArgument;
+
+    do
+    {
+	if( (kIOUCVariableStructureSize != method->count0)
+		&& (inputCount != method->count0))
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (inputStructCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
+
+	func = method->func;
+
+	switch( inputCount) {
+
+	    case 5:
+		err = (object->*func)( ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					ARG32(input[3]), ARG32(input[4]), 
+					inputStruct );
+		break;
+	    case 4:
+		err = (object->*func)( ARG32(input[0]), ARG32(input[1]), (void *)  input[2],
+					ARG32(input[3]),
+					inputStruct, (void *)inputStructCount );
+		break;
+	    case 3:
+		err = (object->*func)( ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+					inputStruct, (void *)inputStructCount,
+					0 );
+		break;
+	    case 2:
+		err = (object->*func)( ARG32(input[0]), ARG32(input[1]),
+					inputStruct, (void *)inputStructCount,
+					0, 0 );
+		break;
+	    case 1:
+		err = (object->*func)( ARG32(input[0]),
+					inputStruct, (void *)inputStructCount,
+					0, 0, 0 );
+		break;
+	    case 0:
+		err = (object->*func)( inputStruct, (void *)inputStructCount,
+					0, 0, 0, 0 );
+		break;
+
+	    default:
+		IOLog("%s: Bad method table\n", object->getName());
+	}
+    }
+    while (false);
 
     return( err);
 }
 
-kern_return_t is_io_async_method_scalarI_structureI(
-            io_connect_t 	connect,
-            mach_port_t		wakePort,
-	    io_async_ref_t	    reference,
-	    mach_msg_type_number_t  referenceCnt,
-            UInt32		index,
-            void *	 	input[],
-            IOByteCount	inputCount,
-            UInt8 *		inputStruct,
-            IOByteCount	inputStructCount )
+kern_return_t shim_io_async_method_scalarI_structureI(
+	IOExternalAsyncMethod *	method,
+	IOService *		object,
+	mach_port_t             asyncWakePort,
+	io_user_reference_t *   asyncReference,
+	uint32_t                asyncReferenceCount,
+        const io_user_scalar_t * input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		inputStruct,
+        mach_msg_type_number_t	inputStructCount )
 {
-    IOReturn 		err;
-    IOExternalAsyncMethod *method;
-    IOService *		object;
     IOAsyncMethod	func;
+    uint32_t		i;
+    IOReturn		err = kIOReturnBadArgument;
+    io_async_ref_t	reference;
 
-    CHECK( IOUserClient, connect, client);
+    for (i = 0; i < asyncReferenceCount; i++)
+	reference[i] = REF32(asyncReference[i]);
 
-    if( (method = client->getAsyncTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-        if( kIOUCScalarIStructI != (method->flags & kIOUCTypeMask))
-            continue;
-        if( (0xffffffff != method->count0)
-                && (inputCount != method->count0))
-            continue;
-        if( (0xffffffff != method->count1)
-                && (inputStructCount != method->count1))
-            continue;
+    do
+    {
+	if( (kIOUCVariableStructureSize != method->count0)
+		&& (inputCount != method->count0))
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (inputStructCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
 
-        reference[0] = (natural_t) wakePort;
         func = method->func;
 
         switch( inputCount) {
 
             case 5:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
-                                        input[3], input[4],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]), ARG32(input[4]),
                                         inputStruct );
                 break;
             case 4:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
-                                        input[3],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
+                                        ARG32(input[3]),
                                         inputStruct, (void *)inputStructCount );
                 break;
             case 3:
                 err = (object->*func)(	reference,
-                                        input[0], input[1], input[2],
+                                        ARG32(input[0]), ARG32(input[1]), ARG32(input[2]),
                                         inputStruct, (void *)inputStructCount,
                                         0 );
                 break;
             case 2:
                 err = (object->*func)(	reference,
-                                        input[0], input[1],
+                                        ARG32(input[0]), ARG32(input[1]),
                                         inputStruct, (void *)inputStructCount,
                                         0, 0 );
                 break;
             case 1:
                 err = (object->*func)(	reference,
-                                        input[0],
+                                        ARG32(input[0]),
                                         inputStruct, (void *)inputStructCount,
                                         0, 0, 0 );
                 break;
@@ -2799,47 +3452,115 @@ kern_return_t is_io_async_method_scalarI_structureI(
                 break;
 
             default:
-                IOLog("%s: Bad method table\n", client->getName());
+                IOLog("%s: Bad method table\n", object->getName());
         }
-      } while( false);
+    }
+    while (false);
+
+    return( err);
+}
+
+/* Routine io_connect_method_structureI_structureO */
+kern_return_t is_io_connect_method_structureI_structureO(
+	io_object_t	connect,
+	uint32_t	index,
+        io_struct_inband_t		input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
+{
+    mach_msg_type_number_t scalar_outputCnt = 0;
+    mach_vm_size_t ool_output_size = 0;
+
+    return (is_io_connect_method(connect, index, 
+		    NULL, 0, 
+		    input, inputCount,
+		    0, 0,
+		    NULL, &scalar_outputCnt,
+		    output, outputCount,
+		    0, &ool_output_size));
+}
+
+kern_return_t shim_io_connect_method_structureI_structureO(
+    IOExternalMethod *	method,
+    IOService *		object,
+        io_struct_inband_t		input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
+{
+    IOMethod		func;
+    IOReturn 		err = kIOReturnBadArgument;
+
+    do 
+    {
+	if( (kIOUCVariableStructureSize != method->count0)
+		&& (inputCount != method->count0))
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (*outputCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
+
+	func = method->func;
+
+	if( method->count1) {
+	    if( method->count0) {
+		err = (object->*func)( input, output,
+					(void *)inputCount, outputCount, 0, 0 );
+	    } else {
+		err = (object->*func)( output, outputCount, 0, 0, 0, 0 );
+	    }
+	} else {
+		err = (object->*func)( input, (void *)inputCount, 0, 0, 0, 0 );
+	}
+    }
+    while( false);
 
-    } else
-        err = kIOReturnUnsupported;
 
     return( err);
 }
 
-kern_return_t is_io_async_method_structureI_structureO(
-        io_object_t	connect,
-        mach_port_t wakePort,
-	io_async_ref_t		reference,
-	mach_msg_type_number_t	referenceCnt,
-        UInt32		index,
-        UInt8 *		input,
-        IOByteCount	inputCount,
-        UInt8 *		output,
-        IOByteCount *	outputCount )
+kern_return_t shim_io_async_method_structureI_structureO(
+	IOExternalAsyncMethod *	method,
+	IOService *		object,
+	mach_port_t           asyncWakePort,
+	io_user_reference_t * asyncReference,
+	uint32_t              asyncReferenceCount,
+        io_struct_inband_t		input,
+        mach_msg_type_number_t	inputCount,
+        io_struct_inband_t		output,
+        mach_msg_type_number_t *	outputCount )
 {
-    IOReturn 		err;
-    IOExternalAsyncMethod *method;
-    IOService *		object;
     IOAsyncMethod	func;
+    uint32_t            i;
+    IOReturn 		err;
+    io_async_ref_t	reference;
 
-    CHECK( IOUserClient, connect, client);
+    for (i = 0; i < asyncReferenceCount; i++)
+	reference[i] = REF32(asyncReference[i]);
 
-    if( (method = client->getAsyncTargetAndMethodForIndex(&object, index)) ) {
-      do {
-        err = kIOReturnBadArgument;
-        if( kIOUCStructIStructO != (method->flags & kIOUCTypeMask))
-            continue;
-        if( (0xffffffff != method->count0)
-                && (inputCount != method->count0))
-            continue;
-        if( (0xffffffff != method->count1)
-                && (*outputCount != method->count1))
-            continue;
+    err = kIOReturnBadArgument;
+    do 
+    {
+	if( (kIOUCVariableStructureSize != method->count0)
+		&& (inputCount != method->count0))
+	{
+	    IOLog("%s: IOUserClient inputCount count mismatch\n", object->getName());
+	    continue;
+	}
+	if( (kIOUCVariableStructureSize != method->count1)
+		&& (*outputCount != method->count1))
+	{
+	    IOLog("%s: IOUserClient outputCount count mismatch\n", object->getName());
+	    continue;
+	}
 
-        reference[0] = (natural_t) wakePort;
         func = method->func;
 
         if( method->count1) {
@@ -2855,21 +3576,19 @@ kern_return_t is_io_async_method_structureI_structureO(
                 err = (object->*func)( reference,
                                        input, (void *)inputCount, 0, 0, 0, 0 );
         }
-
-      } while( false);
-
-    } else
-        err = kIOReturnUnsupported;
+    }
+    while( false);
 
     return( err);
 }
+
 /* Routine io_make_matching */
 kern_return_t is_io_make_matching(
-	mach_port_t 	master_port,
-	UInt32		type,
-	IOOptionBits	options,
-        UInt8 *		input,
-        IOByteCount	inputCount,
+	mach_port_t	    master_port,
+	uint32_t	    type,
+	uint32_t		options,
+        io_struct_inband_t	input,
+        mach_msg_type_number_t	inputCount,
 	io_string_t	matching )
 {
     OSSerialize * 	s;
@@ -2918,8 +3637,8 @@ kern_return_t is_io_make_matching(
 	    continue;
         } else
             strcpy( matching, s->text());
-
-    } while( false);
+    }
+    while( false);
 
     if( s)
 	s->release();
@@ -2932,10 +3651,10 @@ kern_return_t is_io_make_matching(
 /* Routine io_catalog_send_data */
 kern_return_t is_io_catalog_send_data(
         mach_port_t		master_port,
-        int                     flag,
+        uint32_t                flag,
         io_buf_ptr_t 		inData,
         mach_msg_type_number_t 	inDataCount,
-        natural_t *		result)
+        kern_return_t *		result)
 {
     OSObject * obj = 0;
     vm_offset_t data;
@@ -2947,7 +3666,7 @@ kern_return_t is_io_catalog_send_data(
         return kIOReturnNotPrivileged;
 
     // FIXME: This is a hack. Should have own function for removeKernelLinker()
-    if(flag != kIOCatalogRemoveKernelLinker && ( !inData || !inDataCount) )
+    if( (flag != kIOCatalogRemoveKernelLinker && flag != kIOCatalogKextdFinishedLaunching) && ( !inData || !inDataCount) )
         return kIOReturnBadArgument;
 
     if (inData) {
@@ -3030,6 +3749,21 @@ kern_return_t is_io_catalog_send_data(
             }
             break;
 
+        case kIOCatalogKextdFinishedLaunching: {
+#if !NO_KEXTD
+                static bool clearedBusy = false;
+                if (!clearedBusy) {
+                    IOService * serviceRoot = IOService::getServiceRoot();
+                    if (serviceRoot) {
+                        serviceRoot->adjustBusy(-1);
+                        clearedBusy = true;
+                    }
+                }
+#endif
+                kr = kIOReturnSuccess;
+            }
+            break;
+
         default:
             kr = kIOReturnBadArgument;
             break;
@@ -3044,7 +3778,7 @@ kern_return_t is_io_catalog_send_data(
 /* Routine io_catalog_terminate */
 kern_return_t is_io_catalog_terminate(
 	mach_port_t master_port,
-	int flag,
+	uint32_t flag,
 	io_name_t name )
 {
     kern_return_t	   kr;
@@ -3099,7 +3833,7 @@ kern_return_t is_io_catalog_terminate(
 /* Routine io_catalog_get_data */
 kern_return_t is_io_catalog_get_data(
         mach_port_t		master_port,
-        int                     flag,
+        uint32_t                flag,
         io_buf_ptr_t 		*outData,
         mach_msg_type_number_t 	*outDataCount)
 {
@@ -3143,7 +3877,7 @@ kern_return_t is_io_catalog_get_data(
 /* Routine io_catalog_get_gen_count */
 kern_return_t is_io_catalog_get_gen_count(
         mach_port_t		master_port,
-        int                     *genCount)
+        uint32_t                *genCount)
 {
     if( master_port != master_device_port)
         return kIOReturnNotPrivileged;
@@ -3178,7 +3912,7 @@ kern_return_t is_io_catalog_module_loaded(
 
 kern_return_t is_io_catalog_reset(
 	mach_port_t		master_port,
-	int			flag)
+	uint32_t		flag)
 {
     if( master_port != master_device_port)
         return kIOReturnNotPrivileged;
@@ -3223,9 +3957,147 @@ kern_return_t iokit_user_client_trap(struct iokit_user_client_trap_args *args)
     return result;
 }
 
+IOReturn IOUserClient::externalMethod( uint32_t selector, IOExternalMethodArguments * args,
+					IOExternalMethodDispatch * dispatch, OSObject * target, void * reference )
+{
+    IOReturn    err;
+    IOService * object;
+
+    if (dispatch)
+    {
+	uint32_t count;
+	count = dispatch->checkScalarInputCount;
+	if ((kIOUCVariableStructureSize != count) && (count != args->scalarInputCount))
+	{
+	    return (kIOReturnBadArgument);
+	}
+
+	count = dispatch->checkStructureInputSize;
+	if ((kIOUCVariableStructureSize != count) 
+	    && (count != ((args->structureInputDescriptor) 
+			    ? args->structureInputDescriptor->getLength() : args->structureInputSize)))
+	{
+	    return (kIOReturnBadArgument);
+	}
+
+	count = dispatch->checkScalarOutputCount;
+	if ((kIOUCVariableStructureSize != count) && (count != args->scalarOutputCount))
+	{
+	    return (kIOReturnBadArgument);
+	}
+
+	count = dispatch->checkStructureOutputSize;
+	if ((kIOUCVariableStructureSize != count) 
+	    && (count != ((args->structureOutputDescriptor) 
+			    ? args->structureOutputDescriptor->getLength() : args->structureOutputSize)))
+	{
+	    return (kIOReturnBadArgument);
+	}
+
+	if (dispatch->function)
+	    err = (*dispatch->function)(target, reference, args);
+	else
+	    err = kIOReturnNoCompletion;	    /* implementator can dispatch */
+
+	return (err);
+    }
+
+    // pre-Leopard API's don't do ool structs
+    if (args->structureInputDescriptor || args->structureOutputDescriptor)
+    {
+       err = kIOReturnIPCError;
+       return (err);
+    }
+
+    if (args->asyncWakePort)
+    {
+	IOExternalAsyncMethod *	method;
+
+	if( !(method = getAsyncTargetAndMethodForIndex(&object, selector)) )
+	    return (kIOReturnUnsupported);
+
+	switch (method->flags & kIOUCTypeMask)
+	{
+	    case kIOUCScalarIStructI:
+		err = shim_io_async_method_scalarI_structureI( method, object,
+					args->asyncWakePort, args->asyncReference, args->asyncReferenceCount,
+					args->scalarInput, args->scalarInputCount,
+					(char *)args->structureInput, args->structureInputSize );
+		break;
+
+	    case kIOUCScalarIScalarO:
+		err = shim_io_async_method_scalarI_scalarO( method, object,
+					args->asyncWakePort, args->asyncReference, args->asyncReferenceCount,
+					args->scalarInput, args->scalarInputCount,
+					args->scalarOutput, &args->scalarOutputCount );
+		break;
+
+	    case kIOUCScalarIStructO:
+		err = shim_io_async_method_scalarI_structureO( method, object,
+					args->asyncWakePort, args->asyncReference, args->asyncReferenceCount,
+					args->scalarInput, args->scalarInputCount,
+					(char *) args->structureOutput, &args->structureOutputSize );
+		break;
+
+
+	    case kIOUCStructIStructO:
+		err = shim_io_async_method_structureI_structureO( method, object,
+					args->asyncWakePort, args->asyncReference, args->asyncReferenceCount,
+					(char *)args->structureInput, args->structureInputSize,
+					(char *) args->structureOutput, &args->structureOutputSize );
+		break;
+
+	    default:
+		err = kIOReturnBadArgument;
+		break;
+	}
+    }
+    else
+    {
+	IOExternalMethod *	method;
+
+	if( !(method = getTargetAndMethodForIndex(&object, selector)) )
+	    return (kIOReturnUnsupported);
+
+	switch (method->flags & kIOUCTypeMask)
+	{
+	    case kIOUCScalarIStructI:
+		err = shim_io_connect_method_scalarI_structureI( method, object,
+					args->scalarInput, args->scalarInputCount,
+					(char *)args->structureInput, args->structureInputSize );
+		break;
+
+	    case kIOUCScalarIScalarO:
+		err = shim_io_connect_method_scalarI_scalarO( method, object,
+					args->scalarInput, args->scalarInputCount,
+					args->scalarOutput, &args->scalarOutputCount );
+		break;
+
+	    case kIOUCScalarIStructO:
+		err = shim_io_connect_method_scalarI_structureO( method, object,
+					args->scalarInput, args->scalarInputCount,
+					(char *) args->structureOutput, &args->structureOutputSize );
+		break;
+
+
+	    case kIOUCStructIStructO:
+		err = shim_io_connect_method_structureI_structureO( method, object,
+					(char *)args->structureInput, args->structureInputSize,
+					(char *) args->structureOutput, &args->structureOutputSize );
+		break;
+
+	    default:
+		err = kIOReturnBadArgument;
+		break;
+	}
+    }
+    return (err);
+}
+
+
 };	/* extern "C" */
 
-OSMetaClassDefineReservedUnused(IOUserClient, 0);
+OSMetaClassDefineReservedUsed(IOUserClient, 0);
 OSMetaClassDefineReservedUnused(IOUserClient, 1);
 OSMetaClassDefineReservedUnused(IOUserClient, 2);
 OSMetaClassDefineReservedUnused(IOUserClient, 3);
diff --git a/iokit/Kernel/IOWorkLoop.cpp b/iokit/Kernel/IOWorkLoop.cpp
index 81e03f4ab..09135d172 100644
--- a/iokit/Kernel/IOWorkLoop.cpp
+++ b/iokit/Kernel/IOWorkLoop.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
@@ -31,6 +37,7 @@ HISTORY
 #include <IOKit/IOInterruptEventSource.h>
 #include <IOKit/IOCommandGate.h>
 #include <IOKit/IOTimeStamp.h>
+#include <libkern/OSDebug.h>
 
 #define super OSObject
 
@@ -48,12 +55,21 @@ OSMetaClassDefineReservedUnused(IOWorkLoop, 6);
 OSMetaClassDefineReservedUnused(IOWorkLoop, 7);
 
 enum IOWorkLoopState { kLoopRestart = 0x1, kLoopTerminate = 0x2 };
+#ifdef __ppc__
 static inline void SETP(void *addr, unsigned int flag)
     { unsigned int *num = (unsigned int *) addr; *num |= flag; }
 static inline void CLRP(void *addr, unsigned int flag)
     { unsigned int *num = (unsigned int *) addr; *num &= ~flag; }
 static inline bool ISSETP(void *addr, unsigned int flag)
     { unsigned int *num = (unsigned int *) addr; return (*num & flag) != 0; }
+#else
+static inline void SETP(void *addr, unsigned int flag)
+    { unsigned char *num = (unsigned char *) addr; *num |= flag; }
+static inline void CLRP(void *addr, unsigned int flag)
+    { unsigned char *num = (unsigned char *) addr; *num &= ~flag; }
+static inline bool ISSETP(void *addr, unsigned int flag)
+    { unsigned char *num = (unsigned char *) addr; return (*num & flag) != 0; }
+#endif
 
 #define fFlags loopRestart
 
@@ -62,43 +78,70 @@ bool IOWorkLoop::init()
     // The super init and gateLock allocation MUST be done first
     if ( !super::init() )
         return false;
+	
+    if ( gateLock == NULL ) {
+        if ( !( gateLock = IORecursiveLockAlloc()) )
+            return false;
+    }
+	
+    if ( workToDoLock == NULL ) {
+        if ( !(workToDoLock = IOSimpleLockAlloc()) )
+            return false;
+        IOSimpleLockInit(workToDoLock);
+        workToDo = false;
+    }
 
-    if ( !(gateLock = IORecursiveLockAlloc()) )
-        return false;
-
-    if ( !(workToDoLock = IOSimpleLockAlloc()) )
-        return false;
-
-    controlG = IOCommandGate::
-	commandGate(this, OSMemberFunctionCast(IOCommandGate::Action,
-					this, &IOWorkLoop::_maintRequest));
-    if ( !controlG )
-        return false;
-
-    IOSimpleLockInit(workToDoLock);
-    workToDo = false;
-
-    // Point the controlGate at the workLoop.  Usually addEventSource
-    // does this automatically.  The problem is in this case addEventSource
-    // uses the control gate and it has to be bootstrapped.
-    controlG->setWorkLoop(this);
-    if (addEventSource(controlG) != kIOReturnSuccess)
-        return false;
+    if ( controlG == NULL ) {
+        controlG = IOCommandGate::commandGate(
+            this,
+            OSMemberFunctionCast(
+                IOCommandGate::Action,
+                this,
+                &IOWorkLoop::_maintRequest));
+
+        if ( !controlG )
+            return false;
+        // Point the controlGate at the workLoop.  Usually addEventSource
+        // does this automatically.  The problem is in this case addEventSource
+        // uses the control gate and it has to be bootstrapped.
+        controlG->setWorkLoop(this);
+        if (addEventSource(controlG) != kIOReturnSuccess)
+            return false;
+    }
 
-    IOThreadFunc cptr =
-	OSMemberFunctionCast(IOThreadFunc, this, &IOWorkLoop::threadMain);
-    workThread = IOCreateThread(cptr, this);
-    if (!workThread)
-        return false;
+    if ( workThread == NULL ) {
+        IOThreadFunc cptr = OSMemberFunctionCast(
+            IOThreadFunc,
+            this,
+            &IOWorkLoop::threadMain);
+        workThread = IOCreateThread(cptr, this);
+        if (!workThread)
+            return false;
+    }
 
     return true;
 }
 
 IOWorkLoop *
 IOWorkLoop::workLoop()
+{
+    return IOWorkLoop::workLoopWithOptions(0);
+}
+
+IOWorkLoop *
+IOWorkLoop::workLoopWithOptions(IOOptionBits options)
 {
     IOWorkLoop *me = new IOWorkLoop;
 
+    if (me && options) {
+	me->reserved = IONew(ExpansionData, 1);
+	if (!me->reserved) {
+	    me->release();
+	    return 0;
+	}
+	me->reserved->options = options;
+    }
+
     if (me && !me->init()) {
         me->release();
         return 0;
@@ -117,7 +160,7 @@ void IOWorkLoop::free()
 	IOInterruptState is;
 
 	// If we are here then we must be trying to shut down this work loop
-	// in this case disable all of the event source, mark the loop for
+	// in this case disable all of the event source, mark the loop
 	// as terminating and wakeup the work thread itself and return
 	// Note: we hold the gate across the entire operation mainly for the 
 	// benefit of our event sources so we can disable them cleanly.
@@ -143,9 +186,9 @@ void IOWorkLoop::free()
         }
         eventChain = 0;
 
-	// Either we have a partial initialisation to clean up
-	// or we the workThread itself is performing hari-kari.
-	// either way clean up all of our resources and return.
+	// Either we have a partial initialization to clean up
+	// or the workThread itself is performing hari-kari.
+	// Either way clean up all of our resources and return.
 	
 	if (controlG) {
 	    controlG->release();
@@ -161,6 +204,10 @@ void IOWorkLoop::free()
 	    IORecursiveLockFree(gateLock);
 	    gateLock = 0;
 	}
+	if (reserved) {
+	    IODelete(reserved, ExpansionData, 1);
+	    reserved = 0;
+	}
 
 	super::free();
     }
@@ -280,6 +327,7 @@ abort:
 
 /* virtual */ void IOWorkLoop::threadMain()
 {
+restartThread:
     do {
 	if ( !runEventSources() )
 	    goto exitThread;
@@ -288,10 +336,12 @@ abort:
         if ( !ISSETP(&fFlags, kLoopTerminate) && !workToDo) {
 	    assert_wait((void *) &workToDo, false);
 	    IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
-
-	    thread_continue_t cptr = OSMemberFunctionCast(
-		    thread_continue_t, this, &IOWorkLoop::threadMain);
+	    thread_continue_t cptr = NULL;
+	    if (!reserved || !(kPreciousStack & reserved->options))
+		cptr = OSMemberFunctionCast(
+			thread_continue_t, this, &IOWorkLoop::threadMain);
 	    thread_block_parameter(cptr, this);
+	    goto restartThread;
 	    /* NOTREACHED */
 	}
 
diff --git a/iokit/Kernel/PMmisc.cpp b/iokit/Kernel/PMmisc.cpp
index 0ab16cfbe..6f3f84827 100644
--- a/iokit/Kernel/PMmisc.cpp
+++ b/iokit/Kernel/PMmisc.cpp
@@ -1,31 +1,33 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/pwr_mgt/IOPM.h>
 #include <IOKit/IOService.h>
 
-extern "C" {
-extern void kprintf(const char *, ...);
-}
-
 static char rootDomain[ ] = "IOPMrootDomain";
 static char displayDevice[ ] = "IODisplayWrangler";
 static bool rootRegistered;
diff --git a/iokit/Kernel/RootDomainUserClient.cpp b/iokit/Kernel/RootDomainUserClient.cpp
index 47e55b009..170ab0eae 100644
--- a/iokit/Kernel/RootDomainUserClient.cpp
+++ b/iokit/Kernel/RootDomainUserClient.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
@@ -65,26 +71,75 @@ bool RootDomainUserClient::start( IOService * provider )
     return true;
 }
 
-IOReturn RootDomainUserClient::secureSleepSystem( int *return_code )
+IOReturn RootDomainUserClient::secureSleepSystem( uint32_t *return_code )
 {
+    IOByteCount     return_code_size = 1;
+
+    return secureSleepSystemOptions( NULL,      // inOptions
+                                     (void *)return_code, // returnCode
+                                     (void *)0,     // inSize
+                                     (void *)&return_code_size, // returnSize
+                                     NULL, NULL);
+}
+
+IOReturn RootDomainUserClient::secureSleepSystemOptions( 
+    void * p1, void * p2, void * p3,
+    void * p4, void * p5, void * p6 )
+{
+    void            *inOptions = (void *)p1; 
+    uint32_t        *returnCode = (uint32_t *)p2;
+    IOByteCount     inOptionsSize = (IOByteCount)p3;
+    IOByteCount     *returnCodeSize = (IOByteCount *)p4;
+
     int             local_priv = 0;
     int             admin_priv = 0;
     IOReturn        ret = kIOReturnNotPrivileged;
+    OSDictionary    *unserializedOptions =  NULL;
+    OSString        *unserializeErrorString = NULL;
 
     ret = clientHasPrivilege(fOwningTask, kIOClientPrivilegeLocalUser);
     local_priv = (kIOReturnSuccess == ret);
     
     ret = clientHasPrivilege(fOwningTask, kIOClientPrivilegeAdministrator);
     admin_priv = (kIOReturnSuccess == ret);
+    
+    *returnCodeSize = sizeof(uint32_t);
+    
+    if (inOptions)
+    {
+        unserializedOptions = OSDynamicCast( OSDictionary,
+                                             OSUnserializeXML((const char *)inOptions, &unserializeErrorString));
+    
+        if (!unserializedOptions) {
+            IOLog("IOPMRootDomain SleepSystem unserialization failure: %s\n", 
+                unserializeErrorString ? unserializeErrorString->getCStringNoCopy() : "Unknown");
+        }
+    }
+
+    if ( (local_priv || admin_priv) 
+          && fOwner ) 
+    {
+        if (unserializedOptions) 
+        {
+            // Publish Sleep Options in registry under root_domain
+            fOwner->setProperty( kRootDomainSleepOptionsKey, unserializedOptions);            
+
+            *returnCode = fOwner->sleepSystemOptions( unserializedOptions );
+
+            unserializedOptions->release();        
+        } else {
+            // No options
+            // Clear any pre-existing options
+            fOwner->removeProperty( kRootDomainSleepOptionsKey );
+
+            *returnCode = fOwner->sleepSystemOptions( NULL );        
+        }
 
-    if((local_priv || admin_priv) && fOwner) {
-        *return_code = fOwner->sleepSystem();
-        return kIOReturnSuccess;
     } else {
-        *return_code = kIOReturnNotPrivileged;
-        return kIOReturnSuccess;
+        *returnCode = kIOReturnNotPrivileged;
     }
 
+    return kIOReturnSuccess;
 }
 
 IOReturn RootDomainUserClient::secureSetAggressiveness( 
@@ -128,7 +183,7 @@ IOReturn RootDomainUserClient::clientClose( void )
 IOExternalMethod *
 RootDomainUserClient::getTargetAndMethodForIndex( IOService ** targetP, UInt32 index )
 {
-    static IOExternalMethod sMethods[] = {
+    static const IOExternalMethod sMethods[] = {
         { // kPMSetAggressiveness, 0
             (IOService *)1, (IOMethod)&RootDomainUserClient::secureSetAggressiveness, kIOUCScalarIScalarO, 2, 1
         },
@@ -150,9 +205,10 @@ RootDomainUserClient::getTargetAndMethodForIndex( IOService ** targetP, UInt32 i
         { // kPMRestartSystem, 6
             0, (IOMethod)&IOPMrootDomain::restartSystem, kIOUCScalarIScalarO, 0, 0
         },
-        { // kPMSetPreventative, 7
-            (IOService *)1, (IOMethod)&RootDomainUserClient::setPreventative, kIOUCScalarIScalarO, 2, 0
-        },
+        { // kPMSleepSystemOptions, 7
+            (IOService *)1, (IOMethod)&RootDomainUserClient::secureSleepSystemOptions, 
+            kIOUCStructIStructO, kIOUCVariableStructureSize, sizeof(uint32_t)
+        }
     };
     
     if(index >= kNumPMMethods)
@@ -163,10 +219,53 @@ RootDomainUserClient::getTargetAndMethodForIndex( IOService ** targetP, UInt32 i
         else
             *targetP = fOwner;
 
-        return &sMethods[index];
+        return (IOExternalMethod *)&sMethods[index];
     }
 }
 
+#if 0
+IOReturn RootDomainUserClient::externalMethod( uint32_t selector, IOExternalMethodArguments * args,
+						IOExternalMethodDispatch * dispatch, OSObject * target, void * reference )
+{
+    static const IOExternalMethodDispatch sMethods[] = {
+        { // kPMSetAggressiveness, 0
+            (IOService *)1, (IOMethod)&RootDomainUserClient::secureSetAggressiveness, kIOUCScalarIScalarO, 2, 1
+        },
+        { // kPMGetAggressiveness, 1
+            0, (IOMethod)&IOPMrootDomain::getAggressiveness, kIOUCScalarIScalarO, 1, 1
+        },
+        { // kPMSleepSystem, 2
+            (IOService *)1, (IOMethod)&RootDomainUserClient::secureSleepSystem, kIOUCScalarIScalarO, 0, 1
+        },
+        { // kPMAllowPowerChange, 3
+            0, (IOMethod)&IOPMrootDomain::allowPowerChange, kIOUCScalarIScalarO, 1, 0
+        },
+        { // kPMCancelPowerChange, 4
+            0, (IOMethod)&IOPMrootDomain::cancelPowerChange, kIOUCScalarIScalarO, 1, 0
+        },
+        { // kPMShutdownSystem, 5
+            0, (IOMethod)&IOPMrootDomain::shutdownSystem, kIOUCScalarIScalarO, 0, 0
+        },
+        { // kPMRestartSystem, 6
+            0, (IOMethod)&IOPMrootDomain::restartSystem, kIOUCScalarIScalarO, 0, 0
+        },
+        { // kPMSetPreventative, 7
+            (IOService *)1, (IOMethod)&RootDomainUserClient::setPreventative, kIOUCScalarIScalarO, 2, 0
+        },
+    };
+
+    if (selector > (sizeof(sMethods) / sizeof(sMethods[0])))
+	return (kIOReturnBadArgument);
+
+    if ((1 << selector) & ((1 << 0) | (1 << 7))
+	target = this;
+    else
+	target = fOwner;
+
+    return (super::externalMethod(selector, args, &sMethods[selector], target, 0));
+}
+#endif
+
 void 
 RootDomainUserClient::setPreventative(UInt32 on_off, UInt32 types_of_sleep)
 {
diff --git a/iokit/Kernel/RootDomainUserClient.h b/iokit/Kernel/RootDomainUserClient.h
index 6af6f379c..6f7356a85 100644
--- a/iokit/Kernel/RootDomainUserClient.h
+++ b/iokit/Kernel/RootDomainUserClient.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -44,8 +50,14 @@ private:
     IOPMrootDomain *	fOwner;
     task_t              fOwningTask;
 
-    IOReturn            secureSleepSystem( int *return_code );
-    IOReturn            secureSetAggressiveness( unsigned long type, unsigned long newLevel, int *return_code );
+    IOReturn            secureSleepSystem( uint32_t *return_code );
+    
+    IOReturn            secureSleepSystemOptions( void *p1, void *p2, void *p3,
+                                                  void *p4, void *p5, void *p6 );
+
+    IOReturn            secureSetAggressiveness( unsigned long type, 
+                                                 unsigned long newLevel, 
+                                                 int *return_code );
 
 public:
 
diff --git a/iokit/Kernel/i386/IOAsmSupport.s b/iokit/Kernel/i386/IOAsmSupport.s
index 4483a5d1f..ff7585311 100644
--- a/iokit/Kernel/i386/IOAsmSupport.s
+++ b/iokit/Kernel/i386/IOAsmSupport.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <architecture/i386/asm_help.h>
 
diff --git a/iokit/Kernel/i386/IOSharedLock.s b/iokit/Kernel/i386/IOSharedLock.s
index de84a9173..69183e016 100644
--- a/iokit/Kernel/i386/IOSharedLock.s
+++ b/iokit/Kernel/i386/IOSharedLock.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/machine/IOSharedLockImp.h>
 
diff --git a/iokit/Kernel/ppc/IOAsmSupport.s b/iokit/Kernel/ppc/IOAsmSupport.s
index 603f7fd73..56e068cc2 100644
--- a/iokit/Kernel/ppc/IOAsmSupport.s
+++ b/iokit/Kernel/ppc/IOAsmSupport.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  /*
  * Copyright (c) 1997-1998 Apple Computer, Inc.
diff --git a/iokit/Kernel/ppc/IODBDMA.cpp b/iokit/Kernel/ppc/IODBDMA.cpp
index 5a3d9d38e..f706e2809 100644
--- a/iokit/Kernel/ppc/IODBDMA.cpp
+++ b/iokit/Kernel/ppc/IODBDMA.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 Apple Computer, Inc.
diff --git a/iokit/Kernel/ppc/IOSharedLock.s b/iokit/Kernel/ppc/IOSharedLock.s
index de84a9173..69183e016 100644
--- a/iokit/Kernel/ppc/IOSharedLock.s
+++ b/iokit/Kernel/ppc/IOSharedLock.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/machine/IOSharedLockImp.h>
 
diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp
index 2cbe6f5be..37fe95833 100644
--- a/iokit/KernelConfigTables.cpp
+++ b/iokit/KernelConfigTables.cpp
@@ -1,23 +1,35 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 
@@ -29,6 +41,7 @@ const char * gIOKernelKmods =
 "{"
    "'com.apple.kernel'                         = '';"
    "'com.apple.kpi.bsd'                        = '';"
+   "'com.apple.kpi.dsep'                       = '';"
    "'com.apple.kpi.iokit'                      = '';"
    "'com.apple.kpi.libkern'                    = '';"
    "'com.apple.kpi.mach'                       = '';"
@@ -73,3 +86,4 @@ const char * gIOKernelConfigTables =
 #endif /* PPC */
 ")";
 
+
diff --git a/iokit/Makefile b/iokit/Makefile
index 1657e4a11..a5a3441af 100644
--- a/iokit/Makefile
+++ b/iokit/Makefile
@@ -7,14 +7,16 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
-INSTINC_SUBDIRS = include IOKit
+INSTINC_SUBDIRS = IOKit
 INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS}
 INSTINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS}
+INSTINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS}
 
 
 EXPINC_SUBDIRS = IOKit
 EXPINC_SUBDIRS_PPC = ${EXPINC_SUBDIRS}
 EXPINC_SUBDIRS_I386 = ${EXPINC_SUBDIRS}
+EXPINC_SUBDIRS_ARM = ${EXPINC_SUBDIRS}
 
 
 SETUP_SUBDIRS = conf
diff --git a/iokit/Tests/TestCollections.cpp b/iokit/Tests/TestCollections.cpp
index ea7f556f0..02813d3c1 100644
--- a/iokit/Tests/TestCollections.cpp
+++ b/iokit/Tests/TestCollections.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #if DEBUG
 #include "Tests.h"
diff --git a/iokit/Tests/TestContainers.cpp b/iokit/Tests/TestContainers.cpp
index 6641fc6a7..f4c86b8d9 100644
--- a/iokit/Tests/TestContainers.cpp
+++ b/iokit/Tests/TestContainers.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #if DEBUG
 #include "Tests.h"
diff --git a/iokit/Tests/TestDevice.cpp b/iokit/Tests/TestDevice.cpp
index effbab484..021f756f4 100644
--- a/iokit/Tests/TestDevice.cpp
+++ b/iokit/Tests/TestDevice.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #if DEBUG
 
diff --git a/iokit/Tests/Tests.cpp b/iokit/Tests/Tests.cpp
index 5e570cc27..405b20352 100644
--- a/iokit/Tests/Tests.cpp
+++ b/iokit/Tests/Tests.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
diff --git a/iokit/Tests/Tests.h b/iokit/Tests/Tests.h
index 99c6033fe..6c03b7a83 100644
--- a/iokit/Tests/Tests.h
+++ b/iokit/Tests/Tests.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/IOLib.h>
 
diff --git a/iokit/User/Makefile b/iokit/User/Makefile
index 5dcb75d06..e1c339083 100644
--- a/iokit/User/Makefile
+++ b/iokit/User/Makefile
@@ -7,7 +7,11 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
-COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
 do_all: 
 	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(MACH_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
diff --git a/iokit/bsddev/DINetBootHook.cpp b/iokit/bsddev/DINetBootHook.cpp
index d93e326c1..ebf591eb0 100644
--- a/iokit/bsddev/DINetBootHook.cpp
+++ b/iokit/bsddev/DINetBootHook.cpp
@@ -1,16 +1,46 @@
+/*
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 /*
  *  DINetBootHook.c
  *  DiskImages
  *
  *  Created by Byron Han on Sat Apr 13 2002.
- *  Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
  *	Revision History
  *
  *	$Log: DINetBootHook.cpp,v $
+ *	Revision 1.4  2005/07/29 21:49:57  lindak
+ *	Merge of branch "chardonnay" to pick up all chardonnay changes in Leopard
+ *	as of xnu-792.7.4
+ *
  *	Revision 1.3.1558.1  2005/06/24 01:47:25  lindak
  *	Bringing over all of the Karma changes into chardonnay.
- *
+ *	
  *	Revision 1.1.1.1  2005/02/24 21:48:06  akosut
  *	Import xnu-764 from Tiger8A395
  *	
@@ -149,9 +179,10 @@ int di_root_image(const char *path, char devname[], dev_t *dev_p)
 	}
 		
 	myDevName = OSDynamicCast(OSString, controller->getProperty(kDIRootImageDevNameKey));
-	if (myDevName)
-		strcpy(devname, myDevName->getCStringNoCopy());
-	else {
+	if (myDevName) {
+		/* rootdevice is 16 chars in bsd_init.c */
+		strlcpy(devname, myDevName->getCStringNoCopy(), 16);
+	} else {
 		IOLog("could not get %s\n", kDIRootImageDevNameKey);
 		res = kIOReturnError;
 		goto di_root_image_FAILED;
diff --git a/iokit/bsddev/DINetBootHook.h b/iokit/bsddev/DINetBootHook.h
index 8f2ffe411..d7ff1f0a1 100644
--- a/iokit/bsddev/DINetBootHook.h
+++ b/iokit/bsddev/DINetBootHook.h
@@ -8,9 +8,13 @@
  *	Revision History
  *
  *	$Log: DINetBootHook.h,v $
+ *	Revision 1.4  2005/07/29 21:49:57  lindak
+ *	Merge of branch "chardonnay" to pick up all chardonnay changes in Leopard
+ *	as of xnu-792.7.4
+ *
  *	Revision 1.3.1582.1  2005/06/24 01:47:25  lindak
  *	Bringing over all of the Karma changes into chardonnay.
- *
+ *	
  *	Revision 1.1.1.1  2005/02/24 21:48:06  akosut
  *	Import xnu-764 from Tiger8A395
  *	
diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp
index 1d9f9af62..7621a257f 100644
--- a/iokit/bsddev/IOKitBSDInit.cpp
+++ b/iokit/bsddev/IOKitBSDInit.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/IOBSD.h>
 #include <IOKit/IOLib.h>
@@ -30,6 +36,7 @@ extern "C" {
 
 #include <pexpert/pexpert.h>
 #include <kern/clock.h>
+#include <uuid/uuid.h>
 
 // how long to wait for matching root device, secs
 #define ROOTDEVICETIMEOUT	60
@@ -161,7 +168,8 @@ OSDictionary * IONetworkNamePrefixMatching( const char * prefix )
     OSDictionary *	 matching;
     OSDictionary *   propDict = 0;
     const OSSymbol * str      = 0;
-
+	char networkType[128];
+	
     do {
         matching = IOService::serviceMatching( "IONetworkInterface" );
         if ( matching == 0 )
@@ -179,6 +187,18 @@ OSDictionary * IONetworkNamePrefixMatching( const char * prefix )
         str->release();
         str = 0;
 
+		// see if we're contrained to netroot off of specific network type
+		if(PE_parse_boot_argn( "network-type", networkType, 128 ))
+		{
+			str = OSSymbol::withCString( networkType );
+			if(str)
+			{
+				propDict->setObject( "IONetworkRootType", str);
+				str->release();
+				str = 0;
+			}
+		}
+
         if ( matching->setObject( gIOPropertyMatchKey,
                                   (OSObject *) propDict ) != true )
             continue;
@@ -434,7 +454,7 @@ kern_return_t IOFindBSDRoot( char * rootName,
     UInt32		*ramdParms = 0;
 
     UInt32		flags = 0;
-    int			minor, major;
+    int			mnr, mjr;
     bool		findHFSChild = false;
     char *              mediaProperty = 0;
     char *		rdBootVar;
@@ -459,8 +479,8 @@ kern_return_t IOFindBSDRoot( char * rootName,
 	return( kIOReturnNoMemory );
     rdBootVar = str + kMaxPathBuf;
 
-    if (!PE_parse_boot_arg("rd", rdBootVar )
-     && !PE_parse_boot_arg("rootdev", rdBootVar ))
+    if (!PE_parse_boot_argn("rd", rdBootVar, kMaxBootVar )
+     && !PE_parse_boot_argn("rootdev", rdBootVar, kMaxBootVar ))
 	rdBootVar[0] = 0;
 
     do {
@@ -599,7 +619,7 @@ kern_return_t IOFindBSDRoot( char * rootName,
             uuid = (char *)IOMalloc( kMaxBootVar );
                   
             if ( uuid ) {
-                if (!PE_parse_boot_arg( "boot-uuid", uuid )) {
+                if (!PE_parse_boot_argn( "boot-uuid", uuid, kMaxBootVar )) {
                     panic( "rd=uuid but no boot-uuid=<value> specified" ); 
                 } 
                 uuidString = OSString::withCString( uuid );
@@ -618,8 +638,9 @@ kern_return_t IOFindBSDRoot( char * rootName,
     }
 
     if( !matching) {
-        OSString * astring;
-	// any HFS
+	OSString * astring;
+	// Match any HFS media
+	
         matching = IOService::serviceMatching( "IOMedia" );
         astring = OSString::withCStringNoCopy("Apple_HFS");
         if ( astring ) {
@@ -682,8 +703,8 @@ kern_return_t IOFindBSDRoot( char * rootName,
         service = (IOService *)service->getProperty(mediaProperty);
     }
 
-    major = 0;
-    minor = 0;
+    mjr = 0;
+    mnr = 0;
 
     // If the IOService we matched to is a subclass of IONetworkInterface,
     // then make sure it has been registered with BSD and has a BSD name
@@ -707,10 +728,10 @@ kern_return_t IOFindBSDRoot( char * rootName,
 	    strcpy( rootName, iostr->getCStringNoCopy() );
 	off = (OSNumber *) service->getProperty( kIOBSDMajorKey );
 	if( off)
-	    major = off->unsigned32BitValue();
+	    mjr = off->unsigned32BitValue();
 	off = (OSNumber *) service->getProperty( kIOBSDMinorKey );
 	if( off)
-	    minor = off->unsigned32BitValue();
+	    mnr = off->unsigned32BitValue();
 
 	if( service->metaCast( "IONetworkInterface" ))
 	    flags |= 1;
@@ -723,12 +744,12 @@ kern_return_t IOFindBSDRoot( char * rootName,
     }
 
     IOLog( "BSD root: %s", rootName );
-    if( major)
-	IOLog(", major %d, minor %d\n", major, minor );
+    if( mjr)
+	IOLog(", major %d, minor %d\n", mjr, mnr );
     else
 	IOLog("\n");
 
-    *root = makedev( major, minor );
+    *root = makedev( mjr, mnr );
     *oflags = flags;
 
     IOFree( str,  kMaxPathBuf + kMaxBootVar );
@@ -752,6 +773,20 @@ iofrootx:
     return( kIOReturnSuccess );
 }
 
+void IOSecureBSDRoot(const char * rootName)
+{
+#if CONFIG_EMBEDDED
+    IOPlatformExpert *pe;
+    const OSSymbol *functionName = OSSymbol::withCStringNoCopy("SecureRootName");
+    
+    while ((pe = IOService::getPlatform()) == 0) IOSleep(1 * 1000);
+    
+    pe->callPlatformFunction(functionName, false, (void *)rootName, (void *)0, (void *)0, (void *)0);
+    
+    functionName->release();
+#endif
+}
+
 void *
 IOBSDRegistryEntryForDeviceTree(char * path)
 {
@@ -783,4 +818,20 @@ IOBSDRegistryEntryGetData(void * entry, char * property_name,
     return (NULL);
 }
 
+kern_return_t IOBSDGetPlatformUUID( uuid_t uuid, mach_timespec_t timeout )
+{
+    IOService * resources;
+    OSString *  string;
+
+    resources = IOService::waitForService( IOService::resourceMatching( kIOPlatformUUIDKey ), &timeout );
+    if ( resources == 0 ) return KERN_OPERATION_TIMED_OUT;
+
+    string = ( OSString * ) IOService::getPlatform( )->getProvider( )->getProperty( kIOPlatformUUIDKey );
+    if ( string == 0 ) return KERN_NOT_SUPPORTED;
+
+    uuid_parse( string->getCStringNoCopy( ), uuid );
+
+    return KERN_SUCCESS;
+}
+
 } /* extern "C" */
diff --git a/iokit/bsddev/IOKitBSDInit.h b/iokit/bsddev/IOKitBSDInit.h
index 98606f650..3d900e66d 100644
--- a/iokit/bsddev/IOKitBSDInit.h
+++ b/iokit/bsddev/IOKitBSDInit.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef __cplusplus
 extern "C" {
diff --git a/iokit/conf/MASTER b/iokit/conf/MASTER
index 1e7c30b2e..bf820b20e 100644
--- a/iokit/conf/MASTER
+++ b/iokit/conf/MASTER
@@ -46,13 +46,49 @@
 #  IOKIT      = [iokitcpp debug]
 #
 #######################################################################
+#  SYSTEM SIZE CONFIGURATION (select exactly one)
+#
+#	xlarge = extra large scale system configuration
+#	large  = large scale system configuration
+#	medium = medium scale system configuration
+#	small  = small scale system configuration
+#	xsmall = extra small scale system configuration
+#	bsmall = special extra small scale system configuration
 #
 ident		IOKIT
 
+options		HIBERNATION	# system hibernation	# <hibernation>
 options		KERNOBJC	# Objective-C implementation	# <kernobjc>
 options		IOKITCPP	# C++ implementation		# <iokitcpp>
 options		KDEBUG		# kernel tracing		# <kdebug>
+options		NETWORKING	# kernel networking		# <networking>
+options		CRYPTO		# want crypto code		# <crypto>
+options		KPIDIRECT	# direct access			# <kpidirect>
+options     CONFIG_DTRACE # enable dtrace       # <config_dtrace>
+
 #makeoptions	LIBDRIVER = "libDriver_kern.o"			# <libdriver>
 #makeoptions	LIBOBJC   = "libkobjc.o"			# <kernobjc>
 
+#
+#  configurable kernel related resources 
+#
+options   CONFIG_MAX_THREADS=32		# <medium,large,xlarge>
+options   CONFIG_MAX_THREADS=32		# <small,xsmall>
+options   CONFIG_MAX_THREADS=32		# <bsmall>
+
+#
+#  configurable kernel - use these options to strip strings from panic
+#  and printf calls.
+#  no_panic_str - saves around 50K of kernel footprint.
+#  no_printf_str - saves around 45K of kernel footprint.
+#
+options   CONFIG_NO_PANIC_STRINGS		# <no_panic_str>
+options   CONFIG_NO_PRINTF_STRINGS		# <no_printf_str>
+options   CONFIG_NO_KPRINTF_STRINGS		# <no_kprintf_str>
+
+# configurable kernel - general switch to say we are building for an
+# embedded device
+#
+options   CONFIG_EMBEDDED                       # <config_embedded>
 
+options   MACH_ASSERT				# <mach_assert>
diff --git a/iokit/conf/MASTER.i386 b/iokit/conf/MASTER.i386
index 5d128418b..080b9dd74 100644
--- a/iokit/conf/MASTER.i386
+++ b/iokit/conf/MASTER.i386
@@ -1,15 +1,18 @@
-#
 ######################################################################
 #  
-#  Standard NeXT Research Configurations:
-#  -------- ---- -------- ---------------
+#  Standard Apple Mac OS Configurations:
+#  -------- ----- ------ ---------------
+#
+#  RELEASE	= [ intel mach iokitcpp hibernation medium crypto config_dtrace ]
+#  PROFILE	= [ RELEASE profile ]
+#  DEBUG	= [ RELEASE debug ]
 #
-#  RELEASE = [intel mach iokitcpp]
-#  PROFILE = [RELEASE profile]
-#  DEBUG = [intel mach iokitcpp debug]
+#  EMBEDDED	= [ intel mach iokitcpp hibernation no_kextd bsmall crypto ]
+#  DEVELOPMENT	= [ EMBEDDED config_dtrace ]
 #
 ######################################################################
 
 machine		"i386"					# <intel>
 cpu		"i386"					# <intel>
 
+options		NO_KEXTD				# <no_kextd>
diff --git a/iokit/conf/MASTER.ppc b/iokit/conf/MASTER.ppc
index 1bcd729c6..2318d5561 100644
--- a/iokit/conf/MASTER.ppc
+++ b/iokit/conf/MASTER.ppc
@@ -4,9 +4,10 @@
 #  Standard Apple MacOS X Configurations:
 #  -------- ---- -------- ---------------
 #
-#  RELEASE = [ppc mach iokitcpp]
+#  RELEASE = [ppc mach iokitcpp hibernation medium crypto config_dtrace]
+#  DEVELOPMENT = [ RELEASE ]
 #  PROFILE = [ RELEASE profile ]
-#  DEBUG = [ppc mach iokitcpp debug]
+#  DEBUG = [ RELEASE debug]
 #  RELEASE_TRACE = [ RELEASE kdebug ]
 #  DEBUG_TRACE   = [ DEBUG kdebug ]
 #
diff --git a/iokit/conf/Makefile b/iokit/conf/Makefile
index 34ca97c17..d52aee208 100644
--- a/iokit/conf/Makefile
+++ b/iokit/conf/Makefile
@@ -18,10 +18,14 @@ ifndef IOKIT_KERNEL_CONFIG
 export IOKIT_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(IOKIT_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -29,36 +33,32 @@ $(COMPOBJROOT)/$(IOKIT_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(IOKIT_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(IOKIT_KERNEL_CONFIG) $(IOKIT_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(IOKIT_KERNEL_CONFIG) $(IOKIT_KERNEL_CONFIG); \
 	);
 
 .ORDER: $(COMPOBJROOT)/$(IOKIT_KERNEL_CONFIG)/Makefile 
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(IOKIT_KERNEL_CONFIG)/Makefile 
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(IOKIT_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(IOKIT_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(IOKIT_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(IOKIT_KERNEL_CONFIG) \
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(IOKIT_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all:  do_all 
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/iokit/conf/Makefile.i386 b/iokit/conf/Makefile.i386
index 029888c26..0b9af95e3 100644
--- a/iokit/conf/Makefile.i386
+++ b/iokit/conf/Makefile.i386
@@ -27,8 +27,7 @@ OBJS_NO_WERROR=				\
 	IOMemoryCursor.cpo		\
 	IOMemoryDescriptor.cpo	\
 	IOPlatformExpert.cpo	\
-	IOPMPowerStateQueue.cpo		\
-	IOPMchangeNoteList.cpo		\
+	IOPMPowerStateQueue.cpo	\
 	IOPMrootDomain.cpo		\
 	IORangeAllocator.cpo	\
 	IORegistryEntry.cpo		\
diff --git a/iokit/conf/Makefile.template b/iokit/conf/Makefile.template
index 52571ea9d..c1c09f5b5 100644
--- a/iokit/conf/Makefile.template
+++ b/iokit/conf/Makefile.template
@@ -24,11 +24,13 @@ include $(MakeInc_cmd)
 include $(MakeInc_def)
 
 #
-# XXX: CFLAGS
+# XXX: CFLAGS   ARM_TODO: added __MBUF_TRANSITION_ for IONetworkingFamily
 #
-CFLAGS+= -DKERNEL -DDRIVER_PRIVATE				\
+CFLAGS+= -imacros meta_features.h -DKERNEL -DDRIVER_PRIVATE				\
 	-Wall -Wno-four-char-constants -fno-common		\
 	-DIOMATCHDEBUG=1 -DIOALLOCDEBUG=1			\
+	-D__MBUF_TRANSITION_					\
+	-imacros meta_features.h $(CFLAGS_INLINE_CONFIG)
 #-DIOKITDEBUG=-1
 
 CWARNFLAGS   += -Wno-unused-parameter -Wno-redundant-decls -Wno-nested-externs -Wno-write-strings
@@ -85,18 +87,15 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS)
-	@echo "creating $(COMPONENT).o"
-
-	$(SEG_HACK) __HIB IOHibernateRestoreKernel.o -o _IOHibernateRestoreKernel.o
-	mv _IOHibernateRestoreKernel.o IOHibernateRestoreKernel.o
-	$(SEG_HACK) __HIB WKdmDecompress.o -o _WKdmDecompress.o
-	mv _WKdmDecompress.o WKdmDecompress.o
-
-	@echo [ updating $(COMPONENT).o ${IOKIT_KERNEL_CONFIG} ]
-	$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
+	@echo LD $(COMPONENT)
+	$(_v)$(SEG_HACK) __HIB IOHibernateRestoreKernel.o -o _IOHibernateRestoreKernel.o
+	$(_v)mv _IOHibernateRestoreKernel.o IOHibernateRestoreKernel.o
+	$(_v)$(SEG_HACK) __HIB WKdmDecompress.o -o _WKdmDecompress.o
+	$(_v)mv _WKdmDecompress.o WKdmDecompress.o
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`
+	$(_v)${MD} -u Makedep -f -d `ls *.d`
 	
 	
 do_all: $(COMPONENT).o
@@ -107,4 +106,3 @@ do_build_all: do_depend
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/iokit/conf/files b/iokit/conf/files
index 45c5cc070..dea0585dc 100644
--- a/iokit/conf/files
+++ b/iokit/conf/files
@@ -2,14 +2,20 @@
 
 OPTIONS/iokitcpp					optional iokitcpp
 OPTIONS/kdebug						optional kdebug
+OPTIONS/networking					optional networking
+OPTIONS/kpidirect					optional kpidirect
+OPTIONS/hibernation					optional hibernation
+OPTIONS/crypto						optional crypto
+OPTIONS/config_dtrace				optional config_dtrace
+OPTIONS/mach_assert				optional mach_assert
 
 # libIOKit
 
 
-iokit/Kernel/WKdmCompress.c				optional iokitcpp
-iokit/Kernel/WKdmDecompress.c				optional iokitcpp
-iokit/Kernel/IOHibernateIO.cpp				optional iokitcpp
-iokit/Kernel/IOHibernateRestoreKernel.c			optional iokitcpp
+iokit/Kernel/WKdmCompress.c				optional hibernation
+iokit/Kernel/WKdmDecompress.c				optional hibernation
+iokit/Kernel/IOHibernateIO.cpp				optional hibernation
+iokit/Kernel/IOHibernateRestoreKernel.c			optional hibernation
 
 iokit/Kernel/IOLib.cpp					optional iokitcpp
 iokit/Kernel/IOLocks.cpp				optional iokitcpp
@@ -21,11 +27,9 @@ iokit/Kernel/IOStartIOKit.cpp				optional iokitcpp
 
 iokit/Kernel/IODeviceTreeSupport.cpp			optional iokitcpp
 
-iokit/Kernel/IOPMWorkArbiter.cpp                        optional iokitcpp
 iokit/Kernel/IORegistryEntry.cpp			optional iokitcpp
 iokit/Kernel/IOService.cpp				optional iokitcpp
 iokit/Kernel/IOServicePM.cpp				optional iokitcpp
-iokit/Kernel/IOPMchangeNoteList.cpp			optional iokitcpp
 iokit/Kernel/IOPMinformee.cpp				optional iokitcpp
 iokit/Kernel/IOPMinformeeList.cpp			optional iokitcpp
 iokit/Kernel/IOPMPowerStateQueue.cpp			optional iokitcpp
@@ -40,6 +44,7 @@ iokit/Kernel/IOCommandGate.cpp				optional iokitcpp
 iokit/Kernel/IOCommand.cpp				optional iokitcpp
 iokit/Kernel/IOCommandPool.cpp				optional iokitcpp
 iokit/Kernel/IOCommandQueue.cpp				optional iokitcpp
+iokit/Kernel/IODMAEventSource.cpp			optional iokitcpp
 iokit/Kernel/IOFilterInterruptEventSource.cpp		optional iokitcpp
 iokit/Kernel/IOTimerEventSource.cpp			optional iokitcpp
 
@@ -47,6 +52,7 @@ iokit/Kernel/IOTimerEventSource.cpp			optional iokitcpp
 iokit/Kernel/IOBufferMemoryDescriptor.cpp		optional iokitcpp
 iokit/Kernel/IODMACommand.cpp				optional iokitcpp
 iokit/Kernel/IODeviceMemory.cpp				optional iokitcpp
+iokit/Kernel/IOInterleavedMemoryDescriptor.cpp		optional iokitcpp
 iokit/Kernel/IOMapper.cpp				optional iokitcpp
 iokit/Kernel/IOCopyMapper.cpp				optional iokitcpp
 iokit/Kernel/IOMemoryCursor.cpp				optional iokitcpp
@@ -60,12 +66,14 @@ iokit/Kernel/IOCPU.cpp					optional iokitcpp
 
 iokit/Kernel/IONVRAM.cpp				optional iokitcpp
 
+iokit/Kernel/IODMAController.cpp			optional iokitcpp
 iokit/Kernel/IOInterruptController.cpp			optional iokitcpp
 
 iokit/Kernel/IOUserClient.cpp				optional iokitcpp
 
 iokit/Kernel/IOKitDebug.cpp				optional iokitcpp
 iokit/Kernel/IODataQueue.cpp				optional iokitcpp
+iokit/Kernel/IOSharedDataQueue.cpp			optional iokitcpp
 # iokit/Tests/Tests.cpp					optional iokitcpp
 # iokit/Tests/TestDevice.cpp				optional iokitcpp
 # iokit/Tests/TestContainers.cpp			optional iokitcpp
diff --git a/iokit/conf/tools/doconf/Makefile b/iokit/conf/tools/doconf/Makefile
index 2bf0b7a10..aa55a9419 100644
--- a/iokit/conf/tools/doconf/Makefile
+++ b/iokit/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,25 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/iokit/conf/tools/doconf/doconf.csh b/iokit/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/iokit/conf/tools/doconf/doconf.csh
+++ b/iokit/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/iokit/include/architecture/i386/kernBootStruct.h b/iokit/include/architecture/i386/kernBootStruct.h
deleted file mode 100644
index 5afc85d65..000000000
--- a/iokit/include/architecture/i386/kernBootStruct.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <architecture/i386/kernBootStruct.h> is going away use <pexpert/i386/boot.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <pexpert/i386/boot.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/assert.h b/iokit/include/assert.h
deleted file mode 100644
index 141036a77..000000000
--- a/iokit/include/assert.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <assert.h> is going away use IOKit/assert.h instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/assert.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/EventShmemLock.h b/iokit/include/bsddev/EventShmemLock.h
deleted file mode 100644
index 4ab305b6c..000000000
--- a/iokit/include/bsddev/EventShmemLock.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/EventShmemLock.h> is going away use <IOKit/machine/IOSharedLockImp.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/machine/IOSharedLockImp.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/Makefile b/iokit/include/bsddev/Makefile
deleted file mode 100644
index 3fc8d90bd..000000000
--- a/iokit/include/bsddev/Makefile
+++ /dev/null
@@ -1,41 +0,0 @@
-export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
-export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
-export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
-export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
-
-include $(MakeInc_cmd)
-include $(MakeInc_def)
-
-INSTINC_SUBDIRS = machine
-
-INSTINC_SUBDIRS_PPC = ppc
-
-INSTINC_SUBDIRS_I386 = i386
-
-EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
-
-EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
-
-EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
-
-DATAFILES = \
-	    EventShmemLock.h	\
-            ev_types.h		\
-            event.h		\
-            evio.h		\
-	    ev_keymap.h		\
-            evsio.h
-
-INSTALL_MI_LIST	= ${DATAFILES}
-
-INSTALL_MI_DIR = dev
-
-EXPORT_MI_LIST	= \
-
-EXPORT_MI_DIR = \
-
-
-include $(MakeInc_rule)
-include $(MakeInc_dir)
-
-
diff --git a/iokit/include/bsddev/ev_keymap.h b/iokit/include/bsddev/ev_keymap.h
deleted file mode 100644
index a6f9002a8..000000000
--- a/iokit/include/bsddev/ev_keymap.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/ev_keymap.h> is going away use <IOKit/hidsystem/ev_keymap.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/ev_keymap.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/ev_types.h b/iokit/include/bsddev/ev_types.h
deleted file mode 100644
index 91ffc7f01..000000000
--- a/iokit/include/bsddev/ev_types.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/ev_types.h> is going away use <IOKit/hidsystem/IOHIDTypes.h> instead
- 
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/IOHIDTypes.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/event.h b/iokit/include/bsddev/event.h
deleted file mode 100644
index 47520fab5..000000000
--- a/iokit/include/bsddev/event.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/event.h> is going away use <IOKit/hidsystem/IOLLEvent.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/IOLLEvent.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/evio.h b/iokit/include/bsddev/evio.h
deleted file mode 100644
index e797c040d..000000000
--- a/iokit/include/bsddev/evio.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/evio.h> is going away use <IOKit/hidsystem/IOHIDShared.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/IOHIDShared.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/evsio.h b/iokit/include/bsddev/evsio.h
deleted file mode 100644
index e3b799153..000000000
--- a/iokit/include/bsddev/evsio.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/evsio.h> is going away use <IOKit/hidsystem/IOHIDTypes.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/IOHIDTypes.h>
-
-/*
- * Identify this driver as one that uses the new driverkit and messaging API
- */
-#ifndef _NeXT_MACH_EVENT_DRIVER_
-#define _NeXT_MACH_EVENT_DRIVER_	(1)
-#endif /* _NeXT_MACH_EVENT_DRIVER_ */
-
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/i386/EventShmemLock.h b/iokit/include/bsddev/i386/EventShmemLock.h
deleted file mode 100644
index f8d4c1074..000000000
--- a/iokit/include/bsddev/i386/EventShmemLock.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/EventShmemLock.h> is going away use <IOKit/machine/IOSharedLockImp.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/EventShmemLock.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/i386/event.h b/iokit/include/bsddev/i386/event.h
deleted file mode 100644
index 588b60c12..000000000
--- a/iokit/include/bsddev/i386/event.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/event.h> is going away use <IOKit/hidsystem/IOLLEvent.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/event.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/i386/evio.h b/iokit/include/bsddev/i386/evio.h
deleted file mode 100644
index d577d8a9f..000000000
--- a/iokit/include/bsddev/i386/evio.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/evio.h> is going away use <IOKit/hidsystem/IOHIDShared.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/evio.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/i386/evsio.h b/iokit/include/bsddev/i386/evsio.h
deleted file mode 100644
index e70408425..000000000
--- a/iokit/include/bsddev/i386/evsio.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/evsio.h> is going away use <IOKit/hidsystem/IOHIDTypes.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/evsio.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/machine/EventShmemLock.h b/iokit/include/bsddev/machine/EventShmemLock.h
deleted file mode 100644
index 3d14ddf25..000000000
--- a/iokit/include/bsddev/machine/EventShmemLock.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _BSD_DEV_MACHINE_EVENTSHMEMLOCK_H_
-#define _BSD_DEV_MACHINE_EVENTSHMEMLOCK_H_
-
-
-#if defined (__ppc__)
-#include "bsd/dev/ppc/EventShmemLock.h"
-#elif defined (__i386__)
-#include "bsd/dev/i386/EventShmemLock.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _BSD_DEV_MACHINE_EVENTSHMEMLOCK_H_ */
diff --git a/iokit/include/bsddev/machine/Makefile b/iokit/include/bsddev/machine/Makefile
deleted file mode 100644
index 110ceda46..000000000
--- a/iokit/include/bsddev/machine/Makefile
+++ /dev/null
@@ -1,39 +0,0 @@
-export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
-export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
-export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
-export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
-
-include $(MakeInc_cmd)
-include $(MakeInc_def)
-
-INSTINC_SUBDIRS = \
-
-INSTINC_SUBDIRS_PPC = \
-
-INSTINC_SUBDIRS_I386 = \
-
-EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
-
-EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
-
-EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
-
-DATAFILES = \
-            evsio.h		\
-	    evio.h		\
-	    event.h		\
-	    EventShmemLock.h
-
-INSTALL_MI_LIST	= ${DATAFILES}
-
-INSTALL_MI_DIR = dev/machine
-
-EXPORT_MI_LIST	= \
-
-EXPORT_MI_DIR = \
-
-
-include $(MakeInc_rule)
-include $(MakeInc_dir)
-
-
diff --git a/iokit/include/bsddev/machine/event.h b/iokit/include/bsddev/machine/event.h
deleted file mode 100644
index 622867427..000000000
--- a/iokit/include/bsddev/machine/event.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _BSD_DEV_MACHINE_EVENT_H_
-#define _BSD_DEV_MACHINE_EVENT_H_
-
-
-#if defined (__ppc__)
-#include "bsd/dev/ppc/event.h"
-#elif defined (__i386__)
-#include "bsd/dev/i386/event.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _BSD_DEV_MACHINE_EVENT_H_ */
diff --git a/iokit/include/bsddev/machine/evio.h b/iokit/include/bsddev/machine/evio.h
deleted file mode 100644
index 447de6893..000000000
--- a/iokit/include/bsddev/machine/evio.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _BSD_DEV_MACHINE_EVIO_H_
-#define _BSD_DEV_MACHINE_EVIO_H_
-
-
-#if defined (__ppc__)
-#include "bsd/dev/ppc/evio.h"
-#elif defined (__i386__)
-#include "bsd/dev/i386/evio.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _BSD_DEV_MACHINE_EVIO_H_ */
diff --git a/iokit/include/bsddev/machine/evsio.h b/iokit/include/bsddev/machine/evsio.h
deleted file mode 100644
index 2dfcc948c..000000000
--- a/iokit/include/bsddev/machine/evsio.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _BSD_DEV_MACHINE_EVSIO_H_
-#define _BSD_DEV_MACHINE_EVSIO_H_
-
-
-#if defined (__ppc__)
-#include "bsd/dev/ppc/evsio.h"
-#elif defined (__i386__)
-#include "bsd/dev/i386/evsio.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _BSD_DEV_MACHINE_EVSIO_H_ */
diff --git a/iokit/include/bsddev/ppc/EventShmemLock.h b/iokit/include/bsddev/ppc/EventShmemLock.h
deleted file mode 100644
index 71d5658d3..000000000
--- a/iokit/include/bsddev/ppc/EventShmemLock.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/EventShmemLock.h> is going away use <IOKit/machine/IOSharedLockImp.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/EventShmemLock.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/bsddev/ppc/event.h b/iokit/include/bsddev/ppc/event.h
deleted file mode 100644
index 0abba8c7e..000000000
--- a/iokit/include/bsddev/ppc/event.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/event.h> is going away use <IOKit/hidsystem/IOLLEvent.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/event.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/ppc/evio.h b/iokit/include/bsddev/ppc/evio.h
deleted file mode 100644
index d577d8a9f..000000000
--- a/iokit/include/bsddev/ppc/evio.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/evio.h> is going away use <IOKit/hidsystem/IOHIDShared.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/evio.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/bsddev/ppc/evsio.h b/iokit/include/bsddev/ppc/evsio.h
deleted file mode 100644
index e70408425..000000000
--- a/iokit/include/bsddev/ppc/evsio.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <dev/machine/evsio.h> is going away use <IOKit/hidsystem/IOHIDTypes.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <dev/evsio.h>
-#endif /* __APPLE_API_OBSOLETE */
-
diff --git a/iokit/include/drivers/event_status_driver.h b/iokit/include/drivers/event_status_driver.h
deleted file mode 100644
index 1b421ddde..000000000
--- a/iokit/include/drivers/event_status_driver.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/******************************************************************************
-	event_status_driver.h
-	API for the events status driver.
-	This file contains public API.
-	mpaque 11Oct91
-	
-	Copyright 1991 NeXT Computer, Inc.
-	
-	Modified:
-	
-******************************************************************************/
-
-#warning include <drivers/event_status_driver.h> is going away use <IOKit/hidsystem/event_status_driver.h> instead
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#include <IOKit/hidsystem/event_status_driver.h>
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/include/mach/mach.h b/iokit/include/mach/mach.h
deleted file mode 100644
index 2289426e2..000000000
--- a/iokit/include/mach/mach.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#warning include <mach.h> is going away, please do not use it anymore.
-
-#include <sys/appleapiopts.h>
-
-#ifdef __APPLE_API_OBSOLETE
-#endif /* __APPLE_API_OBSOLETE */
diff --git a/iokit/mach-o/mach_header.h b/iokit/mach-o/mach_header.h
index 9dc84d842..8dab718c0 100644
--- a/iokit/mach-o/mach_header.h
+++ b/iokit/mach-o/mach_header.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.h
diff --git a/kgmacros b/kgmacros
index d1a666c74..f0f7e3df4 100644
--- a/kgmacros
+++ b/kgmacros
@@ -1,4 +1,3 @@
-#
 # Kernel gdb macros
 #
 #  These gdb macros should be useful during kernel development in
@@ -20,8 +19,8 @@ document kgm
 | These are the kernel gdb macros.  These gdb macros are intended to be
 | used when debugging a remote kernel via the kdp protocol.  Typically, you
 | would connect to your remote target like so:
-|     (gdb) target remote-kdp
-|     (gdb) attach <name-of-remote-host>
+| 		 (gdb) target remote-kdp
+| 		 (gdb) attach <name-of-remote-host>
 |
 | The following macros are available in this package:
 |     showversion    Displays a string describing the remote kernel version
@@ -63,6 +62,14 @@ document kgm
 |
 |     showpid        Display info about the process identified by pid
 |     showproc       Display info about the process identified by proc struct
+|     showprocinfo   Display detailed info about the process identified by proc struct
+|     showprocfiles  Given a proc_t pointer, display the list of open file descriptors
+|     showproclocks  Given a proc_t pointer, display the list of advisory file locks
+|     zombproc       Print out all procs in the zombie list
+|     allproc        Print out all process in the system not in the zombie list
+|     zombstacks     Print out all stacks of tasks that are exiting
+|
+|     showinitchild  Print out all processes in the system which are children of init process
 |
 |     showkmod	     Display info about a kernel module
 |     showkmodaddr   Given an address, display the kernel module and offset
@@ -96,8 +103,62 @@ document kgm
 |     readphys       Reads the specified untranslated address
 |     readphys64     Reads the specified untranslated 64-bit address
 |
-|     kdp-reboot     Restart remote target
-|	
+|     rtentry_showdbg Print the debug information of a route entry
+|     rtentry_trash  Walk the list of trash route entries
+|
+|     mbuf_walkpkt   Walk the mbuf packet chain (m_nextpkt)
+|     mbuf_walk      Walk the mbuf chain (m_next)
+|     mbuf_buf2slab  Find the slab structure of the corresponding buffer
+|     mbuf_buf2mca   Find the mcache audit structure of the corresponding mbuf
+|     mbuf_showmca   Print the contents of an mbuf mcache audit structure
+|     mbuf_showactive   Print all active/in-use mbuf objects
+|     mbuf_showinactive Print all freed/in-cache mbuf objects 
+|     mbuf_showall   Print all mbuf objects
+|     mbuf_slabs     Print all slabs in the group
+|     mbuf_slabstbl  Print slabs table
+|     mbuf_stat      Print extended mbuf allocator statistics
+|
+|     mcache_walkobj     Walk the mcache object chain (obj_next)
+|     mcache_stat        Print all mcaches in the system
+|     mcache_showcache   Display the number of objects in the cache
+|
+|     showbootermemorymap Dump phys memory map from EFI
+|
+|     systemlog      Display the kernel's printf ring buffer
+|
+|     showvnodepath      Print the path for a vnode
+|     showvnodelocks     Display list of advisory locks held/blocked on a vnode
+|     showallvols        Display a summary of mounted volumes
+|     showvnode          Display info about one vnode
+|     showvolvnodes      Display info about all vnodes of a given volume
+|     showvolbusyvnodes  Display info about busy (iocount!=0) vnodes of a given volume
+|     showallbusyvnodes  Display info about all busy (iocount!=0) vnodes
+|     showallvnodes      Display info about all vnodes
+|     print_vnode        Print out the fields of a vnode struct
+|     showprocvnodes     Print out all the open fds which are vnodes in a process
+|     showallprocvnodes  Print out all the open fds which are vnodes in any process
+|     showmountvnodes    Print the vnode list
+|     showmountallvnodes Print the vnode inactive list
+|     showworkqvnodes    Print the vnode worker list
+|     shownewvnodes      Print the new vnode list
+|
+|     ifconfig       display ifconfig-like output
+|     showifaddrs    show the list of addresses for the given ifp
+|     showifmultiaddrs show the list of multicast addresses for the given ifp
+|
+|     showallpmworkqueues   Display info about all IOPMWorkQueue objects
+|     showregistrypmstate   Display power management state for all IOPower registry entries
+|     showioservicepm       Display the IOServicePM object
+|     showstacksaftertask   showallstacks starting after a given task
+|     showstacksafterthread showallstacks starting after a given thread
+|
+|     showMCAstate	Print machine-check register state after MC exception.
+|
+|     showallgdbstacks	Cause GDB to trace all thread stacks
+|     showallgdbcorestacks Corefile equivalent of "showallgdbstacks"
+|     kdp-reenter	Schedule reentry into the debugger and continue.
+|     kdp-reboot	Restart remote target
+|
 | Type "help <macro>" for more specific help on a particular macro.
 | Type "show user <macro>" to see what the macro is really doing.
 end
@@ -119,9 +180,18 @@ Syntax: showversion
 | correctly.
 end
 
+set $kgm_mtype = ((struct mach_header)_mh_execute_header).cputype
+
+# This option tells gdb to relax its stack tracing heuristics
+# Useful for debugging across stack switches
+# (to the interrupt stack, for instance). Requires gdb-675 or greater.
+# Don't do this for arm as a workaround to 5486905
+if ($kgm_mtype != 12)
+	set backtrace sanity-checks off
+end
+
 set $kgm_dummy = &proc0
 set $kgm_dummy = &kmod
-set $kgm_mtype = ((struct mach_header)_mh_execute_header).cputype
 
 set $kgm_reg_depth = 0
 set $kgm_reg_plane = (void **) gIOServicePlane
@@ -132,6 +202,8 @@ set $kgm_show_object_addrs = 0
 set $kgm_show_object_retain = 0
 set $kgm_show_props = 0
 
+set $kgm_show_kmod_syms = 0
+
 define showkmodheader
     printf "kmod        address     size        "
     printf "id    refs     version  name\n"
@@ -192,13 +264,11 @@ end
 
 define showkmodaddr
     showkmodaddrint $arg0
-    printf "\n"
 end
 
 document showkmodaddr
+Syntax: (gdb) showkmodaddr <addr>
 | Given an address, print the offset and name for the kmod containing it
-| The following is the syntax:
-|     (gdb) showkmodaddr <addr>
 end
 
 define showkmod
@@ -206,9 +276,8 @@ define showkmod
     showkmodint $arg0
 end
 document showkmod
+Syntax: (gdb) showkmod <kmod>
 | Routine to print info about a kernel module
-| The following is the syntax:
-|     (gdb) showkmod <kmod>
 end
 
 define showallkmods
@@ -220,21 +289,20 @@ define showallkmods
     end
 end
 document showallkmods
+Syntax: (gdb) showallkmods
 | Routine to print a summary listing of all the kernel modules
-| The following is the syntax:
-|     (gdb) showallkmods
 end
 
 define showactheader
-    printf "            activation  "
-    printf "thread      pri  state  wait_queue  wait_event\n"
+    printf "            thread      "
+    printf "processor   pri  state  wait_queue  wait_event\n"
 end
 
 
 define showactint
 	printf "            0x%08x  ", $arg0
 	set $kgm_thread = *(struct thread *)$arg0
-	printf "0x%08x  ", $arg0
+	printf "0x%08x  ", $kgm_thread.last_processor
 	printf "%3d  ", $kgm_thread.sched_pri
 	set $kgm_state = $kgm_thread.state
 	if $kgm_state & 0x80
@@ -262,11 +330,17 @@ define showactint
 	    printf "W\t" 
 	    printf "0x%08x  ", $kgm_thread.wait_queue
 	    	if (((unsigned)$kgm_thread.wait_event > (unsigned)sectPRELINKB) \
-		    && ($arg1 != 2))
+		    && ($arg1 != 2) && ($kgm_show_kmod_syms == 0))
 			showkmodaddr $kgm_thread.wait_event
 	        else
 			output /a (unsigned) $kgm_thread.wait_event
 		end
+		if ($kgm_thread.uthread != 0)
+		    set $kgm_uthread = (struct uthread *)$kgm_thread.uthread
+		    if ($kgm_uthread->uu_wmesg != 0)
+			printf " \"%s\"", $kgm_uthread->uu_wmesg
+		    end
+		end
 	end
 	if $arg1 != 0
 	    if ($kgm_thread.kernel_stack != 0)
@@ -276,12 +350,21 @@ define showactint
 		printf "\n\t\tkernel_stack=0x%08x", $kgm_thread.kernel_stack
 		if ($kgm_mtype == 18)
 			set $mysp = $kgm_thread.machine.pcb->save_r1
-		else
+		end
+		if ($kgm_mtype == 7)
 			set $kgm_statep = (struct x86_kernel_state32 *) \
 				($kgm_thread->kernel_stack + 0x4000 \
 				 - sizeof(struct x86_kernel_state32))
 			set $mysp = $kgm_statep->k_ebp
 		end
+		if ($kgm_mtype == 12)
+			if ($arg0 == $r9)
+				set $mysp = $r7
+			else
+				set $kgm_statep = (struct arm_saved_state *)$kgm_thread.machine.kstackptr
+				set $mysp = $kgm_statep->r[7]
+			end
+		end
 		set $prevsp = $mysp - 16
 		printf "\n\t\tstacktop=0x%08x", $mysp
 		if ($kgm_mtype == 18)
@@ -295,26 +378,25 @@ define showactint
 		      && ((((unsigned) $mysp ^ (unsigned) $prevsp) < 0x2000) \
 		      || (((unsigned)$mysp < ((unsigned) ($kgm_thread->kernel_stack+0x4000))) \
 		      && ((unsigned)$mysp > (unsigned) ($kgm_thread->kernel_stack))))
-
-			if ((unsigned) $kgm_return > (unsigned) sectPRELINKB)
-			    showkmodaddr $kgm_return
-			else
-				if ((unsigned) $kgm_return > 0)
-				    output /a (unsigned) $kgm_return
-				end
-			end
 			printf "\n\t\t0x%08x  ", $mysp
 			if ($kgm_mtype == 18)
 				set $kgm_return = *($mysp + 8)
-			else
+			end
+			if ($kgm_mtype == 7)
+				set $kgm_return = *($mysp + 4)
+			end
+			if ($kgm_mtype == 12)
 				set $kgm_return = *($mysp + 4)
 			end
+			if (((unsigned) $kgm_return > (unsigned) sectPRELINKB) \
+			    && ($kgm_show_kmod_syms == 0))
+				showkmodaddr $kgm_return
+			else
+				output /a (unsigned) $kgm_return
+			end
 			set $prevsp = $mysp
 			set $mysp = * $mysp
 		end
-		if ((unsigned) $kgm_return > 0)
-		   output/a $kgm_return
-		end
 		set $kgm_return = 0
 		printf "\n\t\tstackbottom=0x%08x", $prevsp
 	    else
@@ -332,9 +414,8 @@ define showact
     showactint $arg0 0
 end
 document showact
+Syntax: (gdb) showact <activation> 
 | Routine to print out the state of a specific thread.
-| The following is the syntax:
-|     (gdb) showact <activation> 
 end
 
 
@@ -343,14 +424,13 @@ define showactstack
     showactint $arg0 1
 end
 document showactstack
+Syntax: (gdb) showactstack <activation> 
 | Routine to print out the stack of a specific thread.
-| The following is the syntax:
-|     (gdb) showactstack <activation> 
 end
 
 
 define showallthreads
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
         showtaskheader
@@ -363,17 +443,16 @@ define showallthreads
   	    set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
         end
 	printf "\n"
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
 end
 document showallthreads
+Syntax: (gdb) showallthreads
 | Routine to print out info about all threads in the system.
-| The following is the syntax:
-|     (gdb) showallthreads
 end
 
 define showcurrentthreads
-set $kgm_prp = processor_list
+set $kgm_prp = (struct processor *)processor_list
     while $kgm_prp != 0
 	if ($kgm_prp)->active_thread != 0
 	    set $kgm_actp = ($kgm_prp)->active_thread
@@ -387,14 +466,13 @@ set $kgm_prp = processor_list
     end
 end
 document showcurrentthreads
+Syntax: (gdb) showcurrentthreads
 | Routine to print out info about the thread running on each cpu.
-| The following is the syntax:
-|     (gdb) showcurrentthreads
 end
 
 set $decode_wait_events = 0
 define showallstacks
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
         showtaskheader
@@ -411,14 +489,13 @@ define showallstacks
   	    set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
         end
 	printf "\n"
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
 end
 
 document showallstacks
+Syntax: (gdb) showallstacks
 | Routine to print out the stack for each thread in the system.
-| The following is the syntax:
-|     (gdb) showallstacks
 | If the variable $decode_wait_events is non-zero, the routine attempts to
 | interpret thread wait_events as kernel module offsets, which can add to
 | processing time.
@@ -440,9 +517,8 @@ set $kgm_prp = processor_list
 end
 
 document showcurrentstacks
+Syntax: (gdb) showcurrentstacks
 | Routine to print out the thread running on each cpu (incl. its stack)
-| The following is the syntax:
-|     (gdb) showcurrentstacks
 end
 
 define showwaiterheader
@@ -671,9 +747,8 @@ define showmapvme
 	showvmint $arg0 1
 end
 document showmapvme
+Syntax: (gdb) showmapvme <vm_map>
 | Routine to print out a summary listing of all the entries in a vm_map
-| The following is the syntax:
-|     (gdb) showmapvme <vm_map>
 end
 
 
@@ -682,44 +757,41 @@ define showmap
 	showvmint $arg0 0
 end
 document showmap
+Syntax: (gdb) showmap <vm_map>
 | Routine to print out info about the specified vm_map
-| The following is the syntax:
-|     (gdb) showmap <vm_map>
 end
 
 define showallvm
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
         showtaskheader
 	showmapheader
 	showtaskint $kgm_taskp
 	showvmint $kgm_taskp->map 0
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
 end
 document showallvm
+Syntax: (gdb) showallvm
 | Routine to print a summary listing of all the vm maps
-| The following is the syntax:
-|     (gdb) showallvm
 end
 
 
 define showallvme
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
         showtaskheader
 	showmapheader
 	showtaskint $kgm_taskp
 	showvmint $kgm_taskp->map 1
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
 end
 document showallvme
+Syntax: (gdb) showallvme
 | Routine to print a summary listing of all the vm map entries
-| The following is the syntax:
-|     (gdb) showallvme
 end
 
 
@@ -827,9 +899,8 @@ define showipc
 	showipcint $kgm_isp 0
 end
 document showipc
+Syntax: (gdb) showipc <ipc_space>
 | Routine to print the status of the specified ipc space
-| The following is the syntax:
-|     (gdb) showipc <ipc_space>
 end
 
 define showrights
@@ -838,9 +909,8 @@ define showrights
 	showipcint $kgm_isp 1
 end
 document showrights
+Syntax: (gdb) showrights <ipc_space>
 | Routine to print a summary list of all the rights in a specified ipc space
-| The following is the syntax:
-|     (gdb) showrights <ipc_space>
 end
 
 
@@ -852,9 +922,8 @@ define showtaskipc
 	showipcint $kgm_taskp->itk_space 0
 end
 document showtaskipc
+Syntax: (gdb) showtaskipc <task>
 | Routine to print info about the ipc space for a task
-| The following is the syntax:
-|     (gdb) showtaskipc <task>
 end
 
 
@@ -866,44 +935,41 @@ define showtaskrights
 	showipcint $kgm_taskp->itk_space 1
 end
 document showtaskrights
+Syntax: (gdb) showtaskrights <task>
 | Routine to print info about the ipc rights for a task
-| The following is the syntax:
-|     (gdb) showtaskrights <task>
 end
 
 define showallipc
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_cur_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_cur_taskp != $kgm_head_taskp
         showtaskheader
         showipcheader
 	showtaskint $kgm_cur_taskp
 	showipcint $kgm_cur_taskp->itk_space 0
-    	set $kgm_cur_taskp = (struct task *)($kgm_cur_taskp->pset_tasks.next)
+    	set $kgm_cur_taskp = (struct task *)($kgm_cur_taskp->tasks.next)
     end
 end
 document showallipc
+Syntax: (gdb) showallipc
 | Routine to print a summary listing of all the ipc spaces
-| The following is the syntax:
-|     (gdb) showallipc
 end
 
 
 define showallrights
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_cur_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_cur_taskp != $kgm_head_taskp
         showtaskheader
         showipcheader
 	showtaskint $kgm_cur_taskp
 	showipcint $kgm_cur_taskp->itk_space 1
-    	set $kgm_cur_taskp = (struct task *)($kgm_cur_taskp->pset_tasks.next)
+    	set $kgm_cur_taskp = (struct task *)($kgm_cur_taskp->tasks.next)
     end
 end
 document showallrights
+Syntax: (gdb) showallrights
 | Routine to print a summary listing of all the ipc rights
-| The following is the syntax:
-|     (gdb) showallrights
 end
 
 
@@ -915,9 +981,8 @@ define showtaskvm
 	showvmint $kgm_taskp->map 0
 end
 document showtaskvm
+Syntax: (gdb) showtaskvm <task>
 | Routine to print out info about a task's vm_map
-| The following is the syntax:
-|     (gdb) showtaskvm <task>
 end
 
 define showtaskvme
@@ -928,9 +993,8 @@ define showtaskvme
 	showvmint $kgm_taskp->map 1
 end
 document showtaskvme
+Syntax: (gdb) showtaskvme <task>
 | Routine to print out info about a task's vm_map_entries
-| The following is the syntax:
-|     (gdb) showtaskvme <task>
 end
 
 
@@ -954,9 +1018,8 @@ define showtask
     showtaskint $arg0
 end
 document showtask
+Syntax (gdb) showtask <task>
 | Routine to print out info about a task.
-| The following is the syntax:
-|     (gdb) showtask <task>
 end
 
 
@@ -973,9 +1036,8 @@ define showtaskthreads
     end
 end
 document showtaskthreads
+Syntax: (gdb) showtaskthreads <task>
 | Routine to print info about the threads in a task.
-| The following is the syntax:
-|     (gdb) showtaskthreads <task>
 end
 
 
@@ -992,25 +1054,23 @@ define showtaskstacks
     end
 end
 document showtaskstacks
+Syntax: (gdb) showtaskstacks <task>
 | Routine to print out the stack for each thread in a task.
-| The following is the syntax:
-|     (gdb) showtaskstacks <task>
 end
 
 
 define showalltasks
     showtaskheader
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
 	showtaskint $kgm_taskp
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
 end
 document showalltasks
+Syntax: (gdb) showalltasks
 | Routine to print a summary listing of all the tasks
-| The following is the syntax:
-|     (gdb) showalltasks
 end
 
 
@@ -1031,7 +1091,7 @@ end
 
 define showpid
     showtaskheader
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
 	set $kgm_procp = (struct proc *)$kgm_taskp->bsd_info
@@ -1039,14 +1099,13 @@ define showpid
 	    showtaskint $kgm_taskp
 	    set $kgm_taskp = $kgm_head_taskp
 	else
-    	    set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	    set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
 	end
     end
 end
 document showpid
+Syntax: (gdb) showpid <pid>
 | Routine to print a single process by pid
-| The following is the syntax:
-|     (gdb) showpid <pid>
 end
 
 define showproc
@@ -1220,14 +1279,14 @@ define showportdestproc
 #   check against the previous cached value - this is slow
     if ($kgm_spacep != $kgm_destspacep)
 	set $kgm_destprocp = (struct proc *)0
-        set $kgm_head_taskp = &default_pset.tasks
+        set $kgm_head_taskp = &tasks
         set $kgm_desttaskp = (struct task *)($kgm_head_taskp->next)
         while (($kgm_destprocp == 0) && ($kgm_desttaskp != $kgm_head_taskp))
 	    set $kgm_destspacep = $kgm_desttaskp->itk_space
 	    if ($kgm_destspacep == $kgm_spacep)
 	       set $kgm_destprocp = (struct proc *)$kgm_desttaskp->bsd_info
 	    else
-    	       set $kgm_desttaskp = (struct task *)($kgm_desttaskp->pset_tasks.next)
+    	       set $kgm_desttaskp = (struct task *)($kgm_desttaskp->tasks.next)
             end
         end
     end
@@ -1400,13 +1459,12 @@ end
 printf "\n"
 end
 document zprint
+Syntax: (gdb) zprint
 | Routine to print a summary listing of all the kernel zones
-| The following is the syntax:
-|     (gdb) zprint
 end
 
 define showmtxgrp
-set $kgm_mtxgrp = (lck_grp_t *)$arg0
+set $kgm_mtxgrp = (struct _lck_grp_ *)$arg0
 
 if ($kgm_mtxgrp->lck_grp_mtxcnt)
 printf "0x%08x ", $kgm_mtxgrp
@@ -1422,22 +1480,21 @@ end
 
 define showallmtx
 printf "LCK GROUP       CNT         UTIL     MISS     WAIT NAME\n"
-set $kgm_mtxgrp_ptr = (lck_grp_t *)&lck_grp_queue
-set $kgm_mtxgrp_ptr = (lck_grp_t *)$kgm_mtxgrp_ptr->lck_grp_link.next
-while ($kgm_mtxgrp_ptr != (lck_grp_t *)&lck_grp_queue)
+set $kgm_mtxgrp_ptr = (struct _lck_grp_ *)&lck_grp_queue
+set $kgm_mtxgrp_ptr = (struct _lck_grp_ *)$kgm_mtxgrp_ptr->lck_grp_link.next
+while ($kgm_mtxgrp_ptr != (struct _lck_grp_ *)&lck_grp_queue)
 	showmtxgrp $kgm_mtxgrp_ptr
-	set $kgm_mtxgrp_ptr = (lck_grp_t *)$kgm_mtxgrp_ptr->lck_grp_link.next
+	set $kgm_mtxgrp_ptr = (struct _lck_grp_ *)$kgm_mtxgrp_ptr->lck_grp_link.next
 end
 printf "\n"
 end
 document showallmtx
+Syntax: (gdb) showallmtx
 | Routine to print a summary listing of all mutexes
-| The following is the syntax:
-|     (gdb) showallmtx
 end
 
 define showrwlckgrp
-set $kgm_rwlckgrp = (lck_grp_t *)$arg0
+set $kgm_rwlckgrp = (struct _lck_grp_ *)$arg0
 
 if ($kgm_rwlckgrp->lck_grp_rwcnt)
 printf "0x%08x ", $kgm_rwlckgrp
@@ -1453,24 +1510,47 @@ end
 
 define showallrwlck
 printf "LCK GROUP       CNT         UTIL     MISS     WAIT NAME\n"
-set $kgm_rwlckgrp_ptr = (lck_grp_t *)&lck_grp_queue
-set $kgm_rwlckgrp_ptr = (lck_grp_t *)$kgm_rwlckgrp_ptr->lck_grp_link.next
-while ($kgm_rwlckgrp_ptr != (lck_grp_t *)&lck_grp_queue)
+set $kgm_rwlckgrp_ptr = (struct _lck_grp_ *)&lck_grp_queue
+set $kgm_rwlckgrp_ptr = (struct _lck_grp_ *)$kgm_rwlckgrp_ptr->lck_grp_link.next
+while ($kgm_rwlckgrp_ptr != (struct _lck_grp_ *)&lck_grp_queue)
 	showrwlckgrp $kgm_rwlckgrp_ptr
-	set $kgm_rwlckgrp_ptr = (lck_grp_t *)$kgm_rwlckgrp_ptr->lck_grp_link.next
+	set $kgm_rwlckgrp_ptr = (struct _lck_grp_ *)$kgm_rwlckgrp_ptr->lck_grp_link.next
 end
 printf "\n"
 end
 document showallrwlck
+Syntax: (gdb) showallrwlck
 | Routine to print a summary listing of all read/writer locks
-| The following is the syntax:
-|     (gdb) showallrwlck
 end
 
 set $kdp_act_counter = 0
 
+set $r0_save	= 0
+set $r1_save	= 0
+set $r2_save	= 0
+set $r3_save	= 0
+set $r4_save	= 0
+set $r5_save	= 0
+set $r6_save	= 0
+set $r7_save	= 0
+set $r8_save	= 0
+set $r9_save	= 0
+set $r10_save	= 0
+set $r11_save	= 0
+set $r12_save	= 0
+set $sp_save	= 0
+set $lr_save	= 0
+set $pc_save	= 0
+
+define showcontext_int
+	echo Context switched, current instruction pointer: 
+	output/a $pc
+	echo \n
+end
+
 define switchtoact
 	set $newact = (struct thread *) $arg0
+	select 0
 	if ($newact->kernel_stack == 0)
 		echo This activation does not have a stack.\n
 		echo continuation:
@@ -1482,13 +1562,13 @@ define switchtoact
 			set $kdpstate = (struct savearea *) kdp.saved_state
 		end
 		set $kdp_act_counter = $kdp_act_counter + 1
-		set $newact = (struct thread *) $arg0
 		set (struct savearea *) kdp.saved_state=$newact->machine->pcb
 		flushregs
 		flushstack
 		set $pc=$newact->machine->pcb.save_srr0
 		update
-	else
+	end
+	if ($kgm_mtype == 7)
 		set $kdpstatep = (struct x86_saved_state32 *) kdp.saved_state
 		if ($kdp_act_counter == 0)
 			set $kdpstate = *($kdpstatep)
@@ -1508,7 +1588,46 @@ define switchtoact
 		set $pc = $kgm_statep->k_eip
 		update
 	end
+	if ($kgm_mtype == 12)
+		set $r0_save   = $r0
+		set $r1_save   = $r1
+		set $r2_save   = $r2
+		set $r3_save   = $r3
+		set $r4_save   = $r4
+		set $r5_save   = $r5
+		set $r6_save   = $r6
+		set $r7_save   = $r7
+		set $r8_save   = $r8
+		set $r9_save   = $r9
+		set $r10_save  = $r10
+		set $r11_save  = $r11
+		set $r12_save  = $r12
+		set $sp_save   = $sp
+		set $lr_save   = $lr
+		set $pc_save   = $pc
+		set $pc_ctx = load_reg+8
+		set $kgm_statep = (struct arm_saved_state *)((struct thread*)$arg0)->machine.kstackptr
+		set $r0 =  $kgm_statep->r[0]
+		set $r1 =  $kgm_statep->r[1]
+		set $r2 =  $kgm_statep->r[2]
+		set $r3 =  $kgm_statep->r[3]
+		set $r4 =  $kgm_statep->r[4]
+		set $r5 =  $kgm_statep->r[5]
+		set $r6 =  $kgm_statep->r[6]
+		set $r8 =  $kgm_statep->r[8]
+		set $r9 =  $kgm_statep->r[9]
+		set $r10 = $kgm_statep->r[10]
+		set $r11 = $kgm_statep->r[11]
+		set $r12 = $kgm_statep->r[12]
+		set $sp = $kgm_statep->sp
+		set $lr = $kgm_statep->lr
+		set $pc = $pc_ctx
+		set $r7 =  $kgm_statep->r[7]
+		flushregs
+		flushstack
+		end
 	end
+	showcontext_int
 end
 
 document switchtoact  
@@ -1521,6 +1640,7 @@ Syntax: switchtoact <address of activation>
 end     
 
 define switchtoctx
+	select 0
 	if ($kgm_mtype == 18)
 		if ($kdp_act_counter == 0)
 		set $kdpstate = (struct savearea *) kdp.saved_state
@@ -1531,6 +1651,44 @@ define switchtoctx
 		flushstack
 		set $pc=((struct savearea *) $arg0)->save_srr0
 		update
+	else
+	if ($kgm_mtype == 12)
+		set $r0_save   = $r0
+		set $r1_save   = $r1
+		set $r2_save   = $r2
+		set $r3_save   = $r3
+		set $r4_save   = $r4
+		set $r5_save   = $r5
+		set $r6_save   = $r6
+		set $r7_save   = $r7
+		set $r8_save   = $r8
+		set $r9_save   = $r9
+		set $r10_save  = $r10
+		set $r11_save  = $r11
+		set $r12_save  = $r12
+		set $sp_save   = $sp
+		set $lr_save   = $lr
+		set $pc_save   = $pc
+		set $kgm_statep = (struct arm_saved_state *)$arg0
+		set $r0 =  $kgm_statep->r[0]
+		set $r1 =  $kgm_statep->r[1]
+		set $r2 =  $kgm_statep->r[2]
+		set $r3 =  $kgm_statep->r[3]
+		set $r4 =  $kgm_statep->r[4]
+		set $r5 =  $kgm_statep->r[5]
+		set $r6 =  $kgm_statep->r[6]
+		set $r8 =  $kgm_statep->r[8]
+		set $r9 =  $kgm_statep->r[9]
+		set $r10 = $kgm_statep->r[10]
+		set $r11 = $kgm_statep->r[11]
+		set $r12 = $kgm_statep->r[12]
+		set $sp = $kgm_statep->sp
+		set $lr = $kgm_statep->lr
+		set $r7 =  $kgm_statep->r[7]
+		set $pc = $kgm_statep->pc
+		flushregs
+		flushstack
+		update
 	else
 		echo switchtoctx not implemented for this architecture.\n
 	end
@@ -1545,6 +1703,8 @@ Syntax: switchtoctx <address of pcb>
 end     
 
 define resetctx
+	select 0
+	if ($kdp_act_counter != 0)
 	if ($kgm_mtype == 18)
 		set (struct savearea *)kdp.saved_state=$kdpstate
 		flushregs
@@ -1552,7 +1712,8 @@ define resetctx
 		set $pc=((struct savearea *) kdp.saved_state)->save_srr0
 		update
 		set $kdp_act_counter = 0
-	else
+	end
+	if ($kgm_mtype == 7)
 		set $kdpstatep = (struct x86_saved_state32 *) kdp.saved_state
 		set *($kdpstatep)=$kdpstate
 		flushregs
@@ -1561,6 +1722,42 @@ define resetctx
 		update
 		set $kdp_act_counter = 0
 	end
+	if ($kgm_mtype == 12)
+		set $r0  = $r0_save
+		flushregs
+		set $r1  = $r1_save
+		flushregs
+		set $r2  = $r2_save
+		flushregs
+		set $r3  = $r3_save
+		flushregs
+		set $r4  = $r4_save
+		flushregs
+		set $r5  = $r5_save
+		flushregs
+		set $r6  = $r6_save
+		flushregs
+		set $r8  = $r8_save
+		flushregs
+		set $r9  = $r9_save
+		flushregs
+		set $r10 = $r10_save
+		flushregs
+		set $r11 = $r11_save
+		flushregs
+		set $r12 = $r12_save
+		flushregs
+		set $sp  = $sp_save
+		flushregs
+		set $lr  = $lr_save
+		flushregs
+		set $pc  = $pc_save
+		flushregs
+		set $r7  = $r7_save
+		flushregs
+	end
+	showcontext_int
+	end
 end     
         
 document resetctx
@@ -1570,6 +1767,18 @@ document resetctx
 | or "switchtoctx" commands.
 end     
 
+# This is a pre-hook for the continue command, to prevent inadvertent attempts 
+# to resume from the context switched to for examination.
+define hook-continue
+       resetctx
+end
+
+# This is a pre-hook for the detach command, to prevent inadvertent attempts 
+# to resume from the context switched to for examination.
+define hook-detach
+       resetctx
+end
+
 define resume_on
 	set noresume_on_disconnect = 0
 end
@@ -1706,21 +1915,21 @@ define showx86backtrace
 	x/i $kgm_cur_eip
 	set $kgm_prev_ebp = *((uint32_t *) $kgm_cur_ebp)
 	set $kgm_prev_eip = *((uint32_t *) ($kgm_cur_ebp + 4))
+	set $kgm_cur_ebp = 0
+	set $kgm_cur_eip = 0
 	set $kgm_frameno = 1
 	while $kgm_prev_ebp != 0
 		printf "%d: saved EBP: 0x%08x saved EIP: 0x%08x\n", $kgm_frameno, $kgm_prev_ebp, $kgm_prev_eip
 		x/i $kgm_prev_eip
-		set $kgm_cur_ebp = $kgm_prev_ebp
-		set $kgm_prev_ebp = *((uint32_t *) $kgm_cur_ebp)
-		set $kgm_prev_eip = *((uint32_t *) ($kgm_cur_ebp + 4))
+		set $kgm_prev_eip = *((uint32_t *) ($kgm_prev_ebp + 4))
+		set $kgm_prev_ebp = *((uint32_t *) $kgm_prev_ebp)
 		set $kgm_frameno = $kgm_frameno + 1
 	end
-	set $kgm_cur_ebp = 0
-	set $kgm_cur_eip = 0
 	set kdp_pmap = 0
 end
 
 define showuserstack
+		select 0
   		if ($kgm_mtype == 18)	
 		   if ($kdp_act_counter == 0)
 		      set $kdpstate = (struct savearea *) kdp.saved_state
@@ -1752,7 +1961,8 @@ define showuserstack
 		   end
 		else
 			set $newact = (struct thread *) $arg0
-			set $newiss = (x86_saved_state32_t *) ($newact->machine.pcb->iss)
+#This needs to identify 64-bit processes as well
+			set $newiss = (x86_saved_state32_t) ($newact->machine.pcb->iss.uss.ss_32)
 			set $checkpc = $newiss.eip
 			if ($checkpc == 0)
 			    echo This activation does not appear to have
@@ -1781,6 +1991,7 @@ end
 
 #Stopgap until gdb can generate the HOSTREBOOT packet
 define kdp-reboot
+#Alternatively, set *(*(unsigned **) 0x2498) = 1 (or 0x5498 on PPC)
        set flag_kdp_trigger_reboot = 1
        continue
 end
@@ -1796,7 +2007,7 @@ define sendcore
        set kdp_flag |= 0x40
        set panicd_ip_str = "$arg0"
        set panicd_specified = 1
-       set disableDebugOuput = 0
+       set disable_debug_output = 0
        set disableConsoleOutput = 0
        set logPanicDataToScreen = 1
        set reattach_wait = 1
@@ -1828,27 +2039,27 @@ Syntax: disablecore
 |configured to transmit coredumps through boot-args as well.
 end
 
-#Use of this macro requires the gdb submission from 3401283
 define switchtocorethread
+	set $newact = (struct thread *) $arg0
+	select 0
+	if ($newact->kernel_stack == 0)
+	   echo This thread does not have a stack.\n
+	   echo continuation:
+	   output/a (unsigned) $newact.continuation
+	   echo \n
+	else
 	if ($kgm_mtype == 18)
-		if ($kdp_act_counter == 0)
-		set $kdpstate = (struct savearea *) kdp.saved_state
-		end
-		set $kdp_act_counter = $kdp_act_counter + 1
-		set $newact = (struct thread *) $arg0
-		if ($newact->kernel_stack == 0)
-		   echo This thread does not have a stack.\n
-		   echo continuation:
-		   output/a (unsigned) $newact.continuation
-		   echo \n
-		else
-		   loadcontext $newact->machine->pcb
-# flushstack will be introduced in a gdb version > gdb-357
-		   flushstack
-		   set $pc = $newact->machine->pcb.save_srr0
-		end
+	   loadcontext $newact->machine->pcb
+	   flushstack
+	   set $pc = $newact->machine->pcb.save_srr0
 	else
-		echo switchtocorethread not implemented for this architecture.\n
+		set $kgm_cstatep = (struct x86_kernel_state32 *) \
+					($newact->kernel_stack + 0x4000 \
+					 - sizeof(struct x86_kernel_state32))
+		loadcontext $kgm_cstatep
+		flushstack
+ 	end
+	showcontext_int
 	end
 end
 
@@ -1864,51 +2075,80 @@ Syntax: switchtocorethread <address of activation>
 end
 
 define loadcontext
-       set $pc = $arg0.save_srr0
-       set $r1 = $arg0.save_r1
-       set $lr = $arg0.save_lr
-       
-       set $r2 = $arg0.save_r2
-       set $r3 = $arg0.save_r3
-       set $r4 = $arg0.save_r4
-       set $r5 = $arg0.save_r5
-       set $r6 = $arg0.save_r6
-       set $r7 = $arg0.save_r7
-       set $r8 = $arg0.save_r8
-       set $r9 = $arg0.save_r9
-       set $r10 = $arg0.save_r10
-       set $r11 = $arg0.save_r11
-       set $r12 = $arg0.save_r12
-       set $r13 = $arg0.save_r13
-       set $r14 = $arg0.save_r14
-       set $r15 = $arg0.save_r15
-       set $r16 = $arg0.save_r16
-       set $r17 = $arg0.save_r17
-       set $r18 = $arg0.save_r18
-       set $r19 = $arg0.save_r19
-       set $r20 = $arg0.save_r20
-       set $r21 = $arg0.save_r21
-       set $r22 = $arg0.save_r22
-       set $r23 = $arg0.save_r23
-       set $r24 = $arg0.save_r24
-       set $r25 = $arg0.save_r25
-       set $r26 = $arg0.save_r26
-       set $r27 = $arg0.save_r27
-       set $r28 = $arg0.save_r28
-       set $r29 = $arg0.save_r29
-       set $r30 = $arg0.save_r30
-       set $r31 = $arg0.save_r31
-       
-       set $cr = $arg0.save_cr
-       set $ctr = $arg0.save_ctr
+	select 0
+	if ($kgm_mtype == 18)
+	set $kgm_contextp = (struct savearea *) $arg0
+	set $pc = $kgm_contextp.save_srr0
+	set $r1 = $kgm_contextp.save_r1
+	set $lr = $kgm_contextp.save_lr
+
+	set $r2 = $kgm_contextp.save_r2
+	set $r3 = $kgm_contextp.save_r3
+	set $r4 = $kgm_contextp.save_r4
+	set $r5 = $kgm_contextp.save_r5
+	set $r6 = $kgm_contextp.save_r6
+	set $r7 = $kgm_contextp.save_r7
+	set $r8 = $kgm_contextp.save_r8
+	set $r9 = $kgm_contextp.save_r9
+	set $r10 = $kgm_contextp.save_r10
+	set $r11 = $kgm_contextp.save_r11
+	set $r12 = $kgm_contextp.save_r12
+	set $r13 = $kgm_contextp.save_r13
+	set $r14 = $kgm_contextp.save_r14
+	set $r15 = $kgm_contextp.save_r15
+	set $r16 = $kgm_contextp.save_r16
+	set $r17 = $kgm_contextp.save_r17
+	set $r18 = $kgm_contextp.save_r18
+	set $r19 = $kgm_contextp.save_r19
+	set $r20 = $kgm_contextp.save_r20
+	set $r21 = $kgm_contextp.save_r21
+	set $r22 = $kgm_contextp.save_r22
+	set $r23 = $kgm_contextp.save_r23
+	set $r24 = $kgm_contextp.save_r24
+	set $r25 = $kgm_contextp.save_r25
+	set $r26 = $kgm_contextp.save_r26
+	set $r27 = $kgm_contextp.save_r27
+	set $r28 = $kgm_contextp.save_r28
+	set $r29 = $kgm_contextp.save_r29
+	set $r30 = $kgm_contextp.save_r30
+	set $r31 = $kgm_contextp.save_r31
+
+	set $cr = $kgm_contextp.save_cr
+	set $ctr = $kgm_contextp.save_ctr
+       else
+		set $kgm_contextp = (struct x86_kernel_state32 *) $arg0
+		set $ebx = $kgm_contextp->k_ebx 
+		set $ebp = $kgm_contextp->k_ebp 
+		set $edi = $kgm_contextp->k_edi 
+		set $esi = $kgm_contextp->k_esi 
+		set $eip = $kgm_contextp->k_eip 
+		set $pc =  $kgm_contextp->k_eip
+	end
 end
 
 define resetcorectx
-       set $kgm_corecontext = (struct savearea *) kdp.saved_state
-       loadcontext $kgm_corecontext
-# Maintaining this act counter wouldn't be necessary if we just initialized
-# $kdpstate at the beginning of the macro..
-       set $kdp_act_counter = 0 
+	select 0
+	if ($kgm_mtype == 18)
+		set $kgm_corecontext = (struct savearea *) kdp.saved_state
+		loadcontext $kgm_corecontext
+	else
+	if ($kgm_mtype == 7)
+		set $kdpstatep = (struct x86_saved_state32 *) kdp.saved_state
+		set $ebx = $kdpstatep->ebx
+		set $ebp = $kdpstatep->ebp
+		set $edi = $kdpstatep->edi
+		set $esi = $kdpstatep->esi
+		set $eip = $kdpstatep->eip
+		set $eax = $kdpstatep->eax
+		set $ecx = $kdpstatep->ecx
+		set $edx = $kdpstatep->edx
+		flushregs
+		flushstack
+		set $pc = $kdpstatep->eip
+		update
+	end
+	end
+	showcontext_int
 end
 
 document resetcorectx
@@ -1952,6 +2192,12 @@ define showgdbthread
 	    printf "W\t" 
 	    printf "0x%08x  ", $kgm_thread.wait_queue
             output /a (unsigned) $kgm_thread.wait_event
+		if ($kgm_thread.uthread != 0)
+			set $kgm_uthread = (struct uthread *)$kgm_thread.uthread
+			if ($kgm_uthread->uu_wmesg != 0)
+				printf " \"%s\"", $kgm_uthread->uu_wmesg
+			end
+	    end
 	end
 	if $arg1 != 0
 	    if ($kgm_thread.kernel_stack != 0)
@@ -1961,15 +2207,28 @@ define showgdbthread
 		printf "\n\t\tkernel_stack=0x%08x", $kgm_thread.kernel_stack
 		if ($kgm_mtype == 18)
 			set $mysp = $kgm_thread.machine.pcb->save_r1
-		else
+		end
+		if ($kgm_mtype == 7)
 			set $kgm_statep = (struct x86_kernel_state32 *) \
 				($kgm_thread->kernel_stack + 0x4000 \
 				 - sizeof(struct x86_kernel_state32))
 			set $mysp = $kgm_statep->k_ebp
 		end
+		if ($kgm_mtype == 12)
+			if ($arg0 == $r9)
+				set $mysp = $r7
+			else
+                        	set $kgm_statep = (struct arm_saved_state *)$kgm_thread.machine.kstackptr
+                        	set $mysp = $kgm_statep->r[7]
+			end
+		end
 		set $prevsp = 0
 		printf "\n\t\tstacktop=0x%08x", $mysp
-		switchtoact $arg0
+		if ($arg2 == 0)
+			switchtoact $arg0
+		else
+			switchtocorethread $arg0
+		end
 	    	bt
 	    else
 		printf "\n\t\t\tcontinuation="
@@ -1988,7 +2247,7 @@ end
 #encountering such an error.
 
 define showallgdbstacks
-    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_head_taskp = &tasks
     set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
     while $kgm_taskp != $kgm_head_taskp
         showtaskheader
@@ -1997,11 +2256,11 @@ define showallgdbstacks
         set $kgm_actp = (struct thread *)($kgm_taskp->threads.next)
 	while $kgm_actp != $kgm_head_actp
 	    showactheader
-	    showgdbthread $kgm_actp 1
+	    showgdbthread $kgm_actp 1 0
   	    set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
         end
 	printf "\n"
-    	set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    	set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
     end
     resetctx
 end
@@ -2019,7 +2278,35 @@ Syntax: showallgdbstacks
 | errors.
 end
 
+define showallgdbcorestacks
+	select 0
+	set $kgm_head_taskp = &tasks
+	set $kgm_taskp = (struct task *)($kgm_head_taskp->next)
+        while $kgm_taskp != $kgm_head_taskp
+		showtaskheader
+		showtaskint $kgm_taskp
+		set $kgm_head_actp = &($kgm_taskp->threads)
+		set $kgm_actp = (struct thread *)($kgm_taskp->threads.next)
+		while $kgm_actp != $kgm_head_actp
+		showactheader
+		showgdbthread $kgm_actp 1 1
+		set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
+		end
+		printf "\n"
+		set $kgm_taskp = (struct task *)($kgm_taskp->tasks.next)
+	end
+	resetcorectx
+end
+
+
+document showallgdbcorestacks
+Syntax: showallgdbcorestacks
+|Corefile version of "showallgdbstacks"
+end
+
+
 define switchtouserthread
+		select 0
   		if ($kgm_mtype == 18)	
 		   if ($kdp_act_counter == 0)
 		      set $kdpstate = (struct savearea *) kdp.saved_state
@@ -2296,10 +2583,9 @@ define showobject
     printf "\n"
 end
 document showobject
+Syntax: (gdb) showobject <object address>
 | Show info about an OSObject - its vtable ptr and retain count.
 | If the object is a simple container class, more info will be shown.
-| The following is the syntax:
-|     (gdb) showobject <object address>
 end
 
 define dictget 
@@ -2355,8 +2641,8 @@ define showregistryentryrecurse
     if ($kgm_result != 0)
 	printf "%s", ((OSString *)$kgm_result)->string
     else
- 	if (((IOService*)$kgm_re)->pm_vars &&  ((IOService*)$kgm_re)->pm_vars->ourName)
- 	    printf "%s", ((IOService*)$kgm_re)->pm_vars->ourName
+ 	if (((IOService*)$kgm_re)->pwrMgt &&  ((IOService*)$kgm_re)->pwrMgt->Name)
+ 	    printf "%s", ((IOService*)$kgm_re)->pwrMgt->Name
 	else
 #	    printf ", guessclass "
 #	    guessclass $kgm_re
@@ -2425,9 +2711,8 @@ define showregistry
     showregistryentryint gRegistryRoot
 end
 document showregistry
+Syntax: (gdb) showregistry 
 | Show info about all registry entries in the current plane.
-| The following is the syntax:
-|     (gdb) showregistry 
 end
 
 define showregistryprops
@@ -2436,11 +2721,10 @@ define showregistryprops
     showregistryentryint gRegistryRoot
 end
 document showregistryprops
+Syntax: (gdb) showregistryprops 
 | Show info about all registry entries in the current plane, and their properties.
 | set $kgm_show_object_addrs = 1 and/or set $kgm_show_object_retain = 1 will display
 | more verbose information
-| The following is the syntax:
-|     (gdb) showregistryprops 
 end
 
 define showregistryentry
@@ -2449,9 +2733,8 @@ define showregistryentry
     showregistryentryint $arg0
 end
 document showregistryentry
+Syntax: (gdb) showregistryentry <object address>
 | Show info about a registry entry; its properties and descendants in the current plane.
-| The following is the syntax:
-|     (gdb) showregistryentry <object address>
 end
 
 define setregistryplane
@@ -2463,10 +2746,9 @@ define setregistryplane
     end
 end
 document setregistryplane
+Syntax: (gdb) setregistryplane <plane object address>
 | Set the plane to be used for the iokit registry macros. An argument of zero will 
 | display known planes.
-| The following is the syntax:
-|     (gdb) setregistryplane <plane object address>
 end
 
 define guessclass
@@ -2498,9 +2780,8 @@ define showallclasses
 end
 
 document showallclasses
+Syntax: (gdb) showallclasses
 | Show the instance counts and ivar size of all OSObject subclasses. See ioclasscount man page for details.
-| The following is the syntax:
-|     (gdb) showallclasses
 end
 
 define showioalloc
@@ -2511,9 +2792,38 @@ define showioalloc
 end
 
 document showioalloc
+Syntax: (gdb) showioalloc
 | Show some accounting of memory allocated by IOKit allocators. See ioalloccount man page for details.
-| The following is the syntax:
-|     (gdb) showioalloc
+end
+
+define showosobjecttracking
+    set $kgm_next = (OSObjectTracking *) gOSObjectTrackList.next
+    while $kgm_next != &gOSObjectTrackList
+	set $obj = (OSObject *) ($kgm_next+1)
+	showobject $obj
+	set $kgm_idx = 0
+	while $kgm_idx < (sizeof($kgm_next->bt) / sizeof($kgm_next->bt[0]))
+	    if ((unsigned) $kgm_next->bt[$kgm_idx] > (unsigned) sectPRELINKB)
+		showkmodaddr $kgm_next->bt[$kgm_idx]
+		printf "\n"
+	    else
+		if ((unsigned) $kgm_next->bt[$kgm_idx] > 0)
+		    output /a (unsigned) $kgm_next->bt[$kgm_idx]
+		    printf "\n"
+		end
+	    end
+	    set $kgm_idx = $kgm_idx + 1
+	end
+	printf "\n"
+	set $kgm_next = (OSObjectTracking *) $kgm_next->link.next
+    end
+end
+
+document showosobjecttracking
+Syntax: (gdb) showosobjecttracking
+| Show the list of tracked OSObject allocations with backtraces.
+| Boot with the kOSTraceObjectAlloc (0x00400000) io debug flag set. 
+| Set gOSObjectTrackThread to 1 or a thread_t to capture new OSObjects allocated by a thread or all threads.
 end
 
 define readphys
@@ -2543,3 +2853,2533 @@ document readphys64
 | addressed is displayed. While this fails if no physical page exists at the
 | given address, it must be used with caution.
 end
+
+define addkextsyms
+        shell ls $arg0/* | xargs -n 1 echo add-symbol-file > /tmp/gdb-syms
+        source /tmp/gdb-syms
+	set $kgm_show_kmod_syms = 1
+end
+
+document addkextsyms
+| Takes a directory of symbols for kexts generated with kextcache -y and loads them
+| into gdb.
+| (gdb) addkextsyms /path/to/symboldir
+end
+
+define showprocfiles
+    if ($argc == 1)
+	_showprocheader
+	_showprocfiles $arg0
+    else
+    	printf "| Usage:\n|\n"
+	help showprocfiles
+    end
+end
+document showprocfiles
+Syntax: (gdb) showprocfiles <proc_t>
+| Given a proc_t pointer, display the list of open file descriptors for the
+| referenced process.
+end
+
+define _showprocheader
+    printf "fd     fileglob    fg flags    fg type   fg data     info\n"
+    printf "-----  ----------  ----------  --------  ----------  -------------------\n"
+end
+
+define _showprocfiles
+    set $kgm_spf_filedesc = ((proc_t)$arg0)->p_fd
+    set $kgm_spf_last = $kgm_spf_filedesc->fd_lastfile
+    set $kgm_spf_ofiles = $kgm_spf_filedesc->fd_ofiles
+    set $kgm_spf_count = 0
+    while ($kgm_spf_count <= $kgm_spf_last)
+	if ($kgm_spf_ofiles[$kgm_spf_count] == 0)
+	    # DEBUG: For files that were open, but are now closed
+	    # printf "%-5d  FILEPROC_NULL\n", $kgm_spf_count
+	else
+	    # display fd #, fileglob address, fileglob flags
+	    set $kgm_spf_flags = $kgm_spf_ofiles[$kgm_spf_count].f_flags
+	    set $kgm_spf_fg = $kgm_spf_ofiles[$kgm_spf_count].f_fglob
+	    printf "%-5d  0x%08x  0x%08x  ", $kgm_spf_count, $kgm_spf_fg, $kgm_spf_flags
+	    # decode fileglob type
+	    set $kgm_spf_fgt = $kgm_spf_fg->fg_type
+	    if ($kgm_spf_fgt == 1)
+	    	printf "VNODE   "
+	    end
+	    if ($kgm_spf_fgt == 2)
+	    	printf "SOCKET  "
+	    end
+	    if ($kgm_spf_fgt == 3)
+	    	printf "PSXSHM  "
+	    end
+	    if ($kgm_spf_fgt == 4)
+	    	printf "PSXSEM  "
+	    end
+	    if ($kgm_spf_fgt == 5)
+	    	printf "KQUEUE  "
+	    end
+	    if ($kgm_spf_fgt == 6)
+	    	printf "PIPE    "
+	    end
+	    if ($kgm_spf_fgt == 7)
+	    	printf "FSEVENTS"
+	    end
+	    if ($kgm_spf_fgt < 1 || $kgm_spf_fgt > 7)
+	        printf "?: %-5d", $kgm_spf_fgt
+	    end
+
+	    # display fileglob data address and decode interesting fact(s)
+	    # about data, if we know any
+	    set $kgm_spf_fgd = $kgm_spf_fg->fg_data
+	    printf "  0x%08x  ", $kgm_spf_fgd
+	    if ($kgm_spf_fgt == 1)
+	    	set $kgm_spf_name = ((struct vnode *)$kgm_spf_fgd)->v_name
+		if ($kgm_spf_name == 0)
+		    printf "(null)"
+		else
+		    printf "%s", $kgm_spf_name
+		end
+	    end
+	    printf "\n"
+	end
+    	set $kgm_spf_count = $kgm_spf_count + 1
+    end
+end
+
+#
+# Show all the advisory file locks held by a process for each of the vnode
+# type files that it has open; do this by walking the per process open file
+# table and looking at any vnode type fileglob that has a non-NULL lock list
+# associated with it.
+#
+define showproclocks
+    if ($argc == 1)
+	_showproclocks $arg0
+    else
+    	printf "| Usage:\n|\n"
+	help showproclocks
+    end
+end
+document showproclocks
+Syntax: (gdb) showproclocks <proc_t>
+| Given a proc_t pointer, display the list of advisory file locks held by the
+| referenced process.
+end
+
+define _showproclocks
+    set $kgm_spl_filedesc = ((proc_t)$arg0)->p_fd
+    set $kgm_spl_last = $kgm_spl_filedesc->fd_lastfile
+    set $kgm_spl_ofiles = $kgm_spl_filedesc->fd_ofiles
+    set $kgm_spl_count = 0
+    set $kgm_spl_seen = 0
+    while ($kgm_spl_count <= $kgm_spl_last)
+	if ($kgm_spl_ofiles[$kgm_spl_count] == 0)
+	    # DEBUG: For files that were open, but are now closed
+	    # printf "%-5d  FILEPROC_NULL\n", $kgm_spl_count
+	else
+	    set $kgm_spl_fg = $kgm_spl_ofiles[$kgm_spl_count].f_fglob
+	    # decode fileglob type
+	    set $kgm_spl_fgt = $kgm_spl_fg->fg_type
+	    if ($kgm_spl_fgt == 1)
+		set $kgm_spl_fgd = $kgm_spl_fg->fg_data
+	    	set $kgm_spl_name = ((struct vnode *)$kgm_spl_fgd)->v_name
+		set $kgm_spl_vnode = ((vnode_t)$kgm_spl_fgd)
+		set $kgm_spl_lockiter = $kgm_spl_vnode->v_lockf
+		if ($kgm_spl_lockiter != 0)
+		    if ($kgm_spl_seen == 0)
+			_showvnodelockheader
+		    end
+		    set $kgm_spl_seen = $kgm_spl_seen + 1
+		    printf "( fd %d, name ", $kgm_spl_count
+		    if ($kgm_spl_name == 0)
+			printf "(null) )"
+		    else
+			printf "%s )\n", $kgm_spl_name
+		    end
+		    _showvnodelocks $kgm_spl_fgd
+		end
+	    end
+	end
+    	set $kgm_spl_count = $kgm_spf_count + 1
+    end
+    printf "%d total locks for 0x%08x\n", $kgm_spl_seen, $arg0
+end
+
+define showprocinfo
+    set $kgm_spi_proc = (proc_t)$arg0
+    printf "Process 0x%08x\n", $kgm_spi_proc
+    printf "   name %s\n", $kgm_spi_proc->p_comm
+	printf "   pid:%.8d", $kgm_spi_proc->p_pid
+	printf "   task:0x%.8x", $kgm_spi_proc->task
+	printf "   p_stat:%.1d", $kgm_spi_proc->p_stat
+	printf "   parent pid:%.8d", $kgm_spi_proc->p_ppid
+    # decode part of credential
+    set $kgm_spi_cred = $kgm_spi_proc->p_ucred
+    if ($kgm_spi_cred != 0)
+	printf "Cred: euid %d ruid %d svuid %d\n", $kgm_spi_cred->cr_uid, $kgm_spi_cred->cr_ruid, $kgm_spi_cred->cr_svuid
+    else
+    	printf "Cred: (null)\n"
+    end
+    # decode flags
+    set $kgm_spi_flag = $kgm_spi_proc->p_flag
+    printf "Flags: 0x%08x\n", $kgm_spi_flag
+    if ($kgm_spi_flag & 0x00000001)
+	printf "    0x00000001 - may hold advisory locks\n"
+    end
+    if ($kgm_spi_flag & 0x00000002)
+	printf "    0x00000002 - has a controlling tty\n"
+    end
+    if ($kgm_spi_flag & 0x00000004)
+	printf "    0x00000004 - process is 64 bit\n"
+    else
+	printf "   !0x00000004 - process is 32 bit\n"
+    end
+    if ($kgm_spi_flag & 0x00000008)
+	printf "    0x00000008 - no SIGCHLD on child stop\n"
+    end
+    if ($kgm_spi_flag & 0x00000010)
+	printf "    0x00000010 - waiting for child exec/exit\n"
+    end
+    if ($kgm_spi_flag & 0x00000020)
+	printf "    0x00000020 - has started profiling\n"
+    end
+    if ($kgm_spi_flag & 0x00000040)
+	printf "    0x00000040 - in select; wakeup/waiting danger\n"
+    end
+    if ($kgm_spi_flag & 0x00000080)
+	printf "    0x00000080 - was stopped and continued\n"
+    end
+    if ($kgm_spi_flag & 0x00000100)
+	printf "    0x00000100 - has set privileges since exec\n"
+    end
+    if ($kgm_spi_flag & 0x00000200)
+	printf "    0x00000200 - system process: no signals, stats, or swap\n"
+    end
+    if ($kgm_spi_flag & 0x00000400)
+	printf "    0x00000400 - timing out during a sleep\n"
+    end
+    if ($kgm_spi_flag & 0x00000800)
+	printf "    0x00000800 - debugged process being traced\n"
+    end
+    if ($kgm_spi_flag & 0x00001000)
+	printf "    0x00001000 - debugging process has waited for child\n"
+    end
+    if ($kgm_spi_flag & 0x00002000)
+	printf "    0x00002000 - exit in progress\n"
+    end
+    if ($kgm_spi_flag & 0x00004000)
+	printf "    0x00004000 - process has called exec\n"
+    end
+    if ($kgm_spi_flag & 0x00008000)
+	printf "    0x00008000 - owe process an addupc() XXX\n"
+    end
+    if ($kgm_spi_flag & 0x00010000)
+	printf "    0x00010000 - affinity for Rosetta children\n"
+    end
+    if ($kgm_spi_flag & 0x00020000)
+	printf "    0x00020000 - wants to run Rosetta\n"
+    end
+    if ($kgm_spi_flag & 0x00040000)
+	printf "    0x00040000 - has wait() in progress\n"
+    end
+    if ($kgm_spi_flag & 0x00080000)
+	printf "    0x00080000 - kdebug tracing on for this process\n"
+    end
+    if ($kgm_spi_flag & 0x00100000)
+	printf "    0x00100000 - blocked due to SIGTTOU or SIGTTIN\n"
+    end
+    if ($kgm_spi_flag & 0x00200000)
+	printf "    0x00200000 - has called reboot()\n"
+    end
+    if ($kgm_spi_flag & 0x00400000)
+	printf "    0x00400000 - is TBE state\n"
+    end
+    if ($kgm_spi_flag & 0x00800000)
+	printf "    0x00800000 - signal exceptions\n"
+    end
+    if ($kgm_spi_flag & 0x01000000)
+	printf "    0x01000000 - being branch traced\n"
+    end
+    if ($kgm_spi_flag & 0x02000000)
+	printf "    0x02000000 - has vfork() children\n"
+    end
+    if ($kgm_spi_flag & 0x04000000)
+	printf "    0x04000000 - not allowed to attach\n"
+    end
+    if ($kgm_spi_flag & 0x08000000)
+	printf "    0x08000000 - vfork() in progress\n"
+    end
+    if ($kgm_spi_flag & 0x10000000)
+	printf "    0x10000000 - no shared libraries\n"
+    end
+    if ($kgm_spi_flag & 0x20000000)
+	printf "    0x20000000 - force quota for root\n"
+    end
+    if ($kgm_spi_flag & 0x40000000)
+	printf "    0x40000000 - no zombies when children exit\n"
+    end
+    if ($kgm_spi_flag & 0x80000000)
+	printf "    0x80000000 - don't hang on remote FS ops\n"
+    end
+    # decode state
+    set $kgm_spi_state = $kgm_spi_proc->p_stat
+    printf "State: "
+    if ($kgm_spi_state == 1)
+    	printf "Idle\n"
+    end
+    if ($kgm_spi_state == 2)
+    	printf "Run\n"
+    end
+    if ($kgm_spi_state == 3)
+    	printf "Sleep\n"
+    end
+    if ($kgm_spi_state == 4)
+    	printf "Stop\n"
+    end
+    if ($kgm_spi_state == 5)
+    	printf "Zombie\n"
+    end
+    if ($kgm_spi_state == 6)
+    	printf "Reaping\n"
+    end
+    if ($kgm_spi_state < 1 || $kgm_spi_state > 6)
+    	printf "(Unknown)\n"
+    end
+end
+
+document showprocinfo
+Syntax: (gdb) showprocinfo <proc_t>
+| Displays name, pid, parent and task for a proc_t. Decodes cred, flag and p_stat fields.
+end
+
+#
+# dump the zombprocs 
+#
+define zombproc
+  set $basep = (struct proc  *)zombproc->lh_first
+  set $pp = $basep
+  while $pp
+	showprocinfo $pp
+      set $pp = $pp->p_list.le_next
+  end
+end
+
+document zombproc
+Syntax: (gdb) zombproc 
+| Routine to print out all procs in the zombie list
+end
+
+#
+# dump the zombstacks 
+#
+define zombstacks
+  set $basep = (struct proc  *)zombproc->lh_first
+  set $pp = $basep
+  while $pp
+	if $pp->p_stat != 5
+		showtaskstacks $pp->task
+	end
+     set $pp = $pp->p_list.le_next
+  end
+end
+
+document zombstacks
+Syntax: (gdb) zombstacks 
+| Routine to print out all stacks of tasks that are exiting
+end
+
+
+#
+# dump the allprocs
+#
+define allproc
+  set $basep = (struct proc  *)allproc->lh_first
+  set $pp = $basep
+  while $pp
+	showprocinfo $pp
+      set $pp = $pp->p_list.le_next
+  end
+end
+
+document allproc
+Syntax: (gdb) allproc 
+| Routine to print out all process in the system 
+| which are not in the zombie list
+end
+
+
+
+define print_vnode
+   set $vp = (struct vnode *)$arg0
+   printf "   "
+   printf " vp 0x%.8x", $vp
+   printf " use %d", $vp->v_usecount
+   printf " io %d", $vp->v_iocount
+   printf " kuse %d", $vp->v_kusecount
+   printf " type %d", $vp->v_type
+   printf " flg 0x%.8x", $vp->v_flag
+   printf " lflg 0x%.8x", $vp->v_lflag
+   printf " par 0x%.8x", $vp->v_parent
+   set $_name = (char *)$vp->v_name
+   if ($_name != 0)
+      printf " %s", $_name
+   end
+  if ($vp->v_type == VREG) && ($vp->v_un.vu_ubcinfo != 0)
+       printf " mapped %d", ($vp->v_un.vu_ubcinfo.ui_flags & 0x08) ? 1 : 0
+   end
+   printf "\n"
+end
+
+document print_vnode
+Syntax: (gdb) print_vnode <vnode>
+| Prints out the fields of a vnode struct
+end
+
+define showprocvnodes
+	set $pp = (struct proc *)$arg0
+	set $fdp = (struct filedesc *)$pp->p_fd
+	set $cvp = $fdp->fd_cdir
+	set $rvp = $fdp->fd_rdir
+	if $cvp
+		printf "Current Working Directory \n"
+		print_vnode $cvp
+		printf "\n"
+	end
+	if $rvp
+		printf "Current Root Directory \n"
+		print_vnode $rvp
+		printf "\n"
+	end
+	set $count = 0
+	set $fpp =  (struct fileproc **)($fdp->fd_ofiles)
+	set $fpo =  (char)($fdp->fd_ofileflags[0])
+	while $count < $fdp->fd_nfiles
+		#printf"fpp %x ", *$fpp
+		if *$fpp
+			set $fg =(struct fileglob *)((**$fpp)->f_fglob)
+			if  $fg && (($fg)->fg_type == 1) 
+				if $fdp->fd_ofileflags[$count] & 4
+					printf "U: "
+				else
+					printf " "
+				end
+				printf "fd = %d ", $count
+				print_vnode $fg->fg_data
+			end
+		end
+		set $fpp = $fpp + 1
+	 	set $count = $count + 1		
+	end
+end
+
+document showprocvnodes
+Syntax: (gdb) showprocvnodes <proc_address>
+| Routine to print out all the open fds
+| which are vnodes in a process
+end
+
+define showallprocvnodes
+  set $basep = (struct proc  *)allproc->lh_first
+  set $pp = $basep
+  while $pp
+	printf "============================================ \n"
+	showprocinfo $pp
+	showprocvnodes $pp
+  	set $pp = $pp->p_list.le_next
+  end
+end
+
+document showallprocvnodes
+Syntax: (gdb) showallprocvnodes
+| Routine to print out all the open fds
+| which are vnodes 
+end
+
+
+#   
+# dump the childrent of a proc 
+#
+define showinitchild
+  set $basep = (struct proc  *)initproc->p_children.lh_first
+  set $pp = $basep
+  while $pp
+	showprocinfo $pp
+      set $pp = $pp->p_sibling.le_next
+  end
+end
+
+document showinitchild
+Syntax: (gdb) showinitchild 
+| Routine to print out all processes in the system 
+| which are children of init process
+end
+
+
+define showmountallvnodes
+  set $mp = (struct mount *)$arg0
+  set $basevp = (struct vnode *)$mp->mnt_vnodelist.tqh_first
+  set $vp = $basevp
+  printf "____________________ Vnode list Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+  set $basevp = (struct vnode *)$mp->mnt_workerqueue.tqh_first
+  set $vp = $basevp
+  printf "____________________ Worker Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+  set $basevp = (struct vnode *)$mp->mnt_newvnodes.tqh_first
+  set $vp = $basevp
+  printf "____________________ New vnodes  Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+end
+document showmountallvnodes
+Syntax: showmountallvnodes <struct mount *>
+| Print the vnode inactive list
+end
+
+
+define showmountvnodes
+  set $mp = (struct mount *)$arg0
+  set $basevp = (struct vnode *)$mp->mnt_vnodelist.tqh_first
+  set $vp = $basevp
+  printf "____________________ Vnode list Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+end
+document showmountvnodes
+Syntax: showmountvnodes <struct mount *>
+| Print the vnode list
+end
+
+
+
+define showworkqvnodes
+  set $mp = (struct mount *)$arg0
+  set $basevp = (struct vnode *)$mp->mnt_workerqueue.tqh_first
+  set $vp = $basevp
+  printf "____________________ Worker Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+end
+document showworkqvnodes
+Syntax: showworkqvnodes <struct mount *>
+| Print the vnode worker list
+end
+
+
+define shownewvnodes
+  set $mp = (struct mount *)$arg0
+  set $basevp = (struct vnode *)$mp->mnt_newvnodes.tqh_first
+  set $vp = $basevp
+  printf "____________________ New vnodes  Queue ---------------\n"
+  while $vp
+      print_vnode $vp
+      set $vp = $vp->v_mntvnodes->tqe_next
+  end
+end
+
+document shownewvnodes
+Syntax: shownewvnodes <struct mount *>
+| Print the new vnode list
+end
+
+
+#
+# print mount point info
+define print_mount 
+   set $mp = (struct mount *)$arg0
+   printf "   "
+   printf " mp 0x%.8x", $mp
+   printf " flag %x", $mp->mnt_flag
+   printf " kern_flag %x", $mp->mnt_kern_flag
+   printf " lflag %x", $mp->mnt_lflag
+   printf " type:  %s", $mp->mnt_vfsstat.f_fstypename
+   printf " mnton:  %s", $mp->mnt_vfsstat.f_mntonname
+   printf " mntfrom:  %s", $mp->mnt_vfsstat.f_mntfromname
+   printf "\n"
+end
+
+define showallmounts
+	set $mp=(struct mount *)mountlist.tqh_first
+	while $mp
+	 	print_mount $mp
+		set $mp = $mp->mnt_list.tqe_next
+	end
+end
+	
+document showallmounts
+Syntax: showallmounts
+| Print all mount points
+end
+
+define pcprint
+	set $pc = $arg0
+	if ((unsigned int)$pc <= (unsigned int) $kgm_fkmodmax) && \
+	    ((unsigned int)$pc >= (unsigned int)$kgm_fkmodmin)
+		showkmodaddr $pc
+	else
+		output/a $pc
+	end
+end
+
+define mbuf_walkpkt
+	set $mp = (struct mbuf *)$arg0
+	set $cnt = 1
+	set $tot = 0
+	while $mp
+		printf "%4d: 0x%08x [len %4d, type %2d, ", $cnt, $mp, \
+		    $mp->m_hdr.mh_len, $mp->m_hdr.mh_type
+		if mclaudit != 0
+			mbuf_buf2mca $mp
+			printf ", "
+		end
+		set $tot = $tot + $mp->m_hdr.mh_len
+		printf "total %d]\n", $tot
+		set $mp = $mp->m_hdr.mh_nextpkt
+		set $cnt = $cnt + 1
+	end
+end
+
+document mbuf_walkpkt
+Syntax: (gdb) mbuf_walkpkt <addr>
+| Given an mbuf address, walk its m_nextpkt pointer
+end
+
+define mbuf_walk
+	set $mp = (struct mbuf *)$arg0
+	set $cnt = 1
+	set $tot = 0
+	while $mp
+		printf "%4d: 0x%08x [len %4d, type %2d, ", $cnt, $mp, \
+		    $mp->m_hdr.mh_len, $mp->m_hdr.mh_type
+		if mclaudit != 0
+			mbuf_buf2mca $mp
+			printf ", "
+		end
+		set $tot = $tot + $mp->m_hdr.mh_len
+		printf "total %d]\n", $tot
+		set $mp = $mp->m_hdr.mh_next
+		set $cnt = $cnt + 1
+	end
+end
+
+document mbuf_walk
+Syntax: (gdb) mbuf_walk <addr>
+| Given an mbuf address, walk its m_next pointer
+end
+
+define mbuf_buf2slab
+	set $addr = $arg0
+	set $gix = ((char *)$addr - (char *)mbutl) >> 20
+	set $ix = ((char *)$addr - (char *)mbutl) >> 11
+	set $slab = &slabstbl[$gix].slg_slab[$ix]
+	printf "0x%08x", $slab
+end
+
+document mbuf_buf2slab
+| Given an mbuf object, find its corresponding slab address.
+end
+
+define mbuf_buf2mca
+	set $addr = $arg0
+	set $ix = ((char *)$addr - (char *)mbutl) >> 11
+	set $clbase = ((union mcluster *)(mbutl + $ix))
+	set $mclidx = (((char *)$addr - (char *)$clbase) >> 8)
+	set $mca = mclaudit[$ix].cl_audit[$mclidx]
+	printf "mca: 0x%08x", $mca
+end
+
+document mbuf_buf2mca
+Syntax: (gdb) mbuf_buf2mca <addr>
+| Given an mbuf object, find its buffer audit structure address.
+| This requires mbuf buffer auditing to be turned on, by setting
+| the appropriate flags to the "mbuf_debug" boot-args parameter.
+end
+
+define mbuf_showmca
+	set language c
+	set $mca = (mcache_audit_t *)$arg0
+	set $cp = (mcache_t *)$mca->mca_cache
+	printf "object type:\t\t"
+	mbuf_mca_ctype $mca 1
+	printf "\ncontrolling mcache:\t%p (%s)\n", $mca->mca_cache, $cp->mc_name
+	if $mca->mca_uflags & $MB_SCVALID
+		set $ix = ((char *)$mca->mca_addr - (char *)mbutl) >> 11
+		set $clbase = ((union mcluster *)(mbutl + $ix))
+		set $mclidx = (((char *)$mca->mca_addr - (char *)$clbase) >> 8)
+		printf "mbuf obj:\t\t%p\n", $mca->mca_addr
+		printf "mbuf index:\t\t%d (out of 8) in cluster base %p\n", \
+		    $mclidx + 1, $clbase
+		if $mca->mca_uptr != 0
+			set $peer_mca = (mcache_audit_t *)$mca->mca_uptr
+			printf "paired cluster obj:\t%p (mca %p)\n", \
+			    $peer_mca->mca_addr, $peer_mca
+		end
+		printf "saved contents:\t\t%p (%d bytes)\n", \
+		    $mca->mca_contents, $mca->mca_contents_size
+	else
+		printf "cluster obj:\t\t%p\n", $mca->mca_addr
+		if $mca->mca_uptr != 0
+			set $peer_mca = (mcache_audit_t *)$mca->mca_uptr
+			printf "paired mbuf obj:\t%p (mca %p)\n", \
+			    $peer_mca->mca_addr, $peer_mca
+		end
+	end
+	printf "recent transaction for this buffer (thread %p):\n", \
+	    $mca->mca_thread
+	set $cnt = 0
+	while $cnt < $mca->mca_depth
+		set $pc = $mca->mca_stack[$cnt]
+		printf "%4d: ", $cnt + 1
+		pcprint $pc
+		printf "\n"
+		set $cnt = $cnt + 1
+	end
+	if $mca->mca_pdepth > 0
+		printf "previous transaction for this buffer (thread %p):\n", \
+		    $mca->mca_pthread
+	end
+	set $cnt = 0
+	while $cnt < $mca->mca_pdepth
+		set $pc = $mca->mca_pstack[$cnt]
+		printf "%4d: ", $cnt + 1
+		pcprint $pc
+		printf "\n"
+		set $cnt = $cnt + 1
+	end
+	set language auto
+end
+
+document mbuf_showmca
+Syntax: (gdb) mbuf_showmca <addr>
+| Given an mbuf/cluster buffer audit structure address, print the audit
+| records including the stack trace of the last buffer transaction.
+end
+
+set $MCF_NOCPUCACHE = 0x10
+
+define mcache_stat
+	set $head = (mcache_t *)mcache_head
+	set $mc = $head
+	printf "cache                        cache      cache    buf   buf    backing             (# of retries)     bufs\n"
+	printf "name                         state       addr   size align       zone     wait   nowait   failed  incache\n"
+	printf "------------------------- -------- ---------- ------ ----- ---------- -------------------------- --------\n"
+	while $mc != 0
+		set $bktsize = $mc->mc_cpu.cc_bktsize
+		printf "%-25s ", $mc->mc_name
+		if ($mc->mc_flags & $MCF_NOCPUCACHE)
+			printf "disabled"
+		else
+			if $mc->mc_purge_cnt > 0
+				printf " purging"
+			else
+				if $bktsize == 0
+					printf " offline"
+				else
+					printf "  online"
+				end
+			end
+		end
+		printf " 0x%08x %6d %5d ",$mc, \
+		    $mc->mc_bufsize, $mc->mc_align
+		if $mc->mc_slab_zone != 0
+			printf "0x%08x", $mc->mc_slab_zone
+		else
+			printf "    custom"
+		end
+		set $tot = 0
+		set $tot += $mc->mc_full.bl_total * $bktsize
+		set $ccp = (mcache_cpu_t *)$mc->mc_cpu
+		set $n = 0
+		while $n < ncpu
+			if $ccp->cc_objs > 0
+				set $tot += $ccp->cc_objs
+			end
+			if $ccp->cc_pobjs > 0
+				set $tot += $ccp->cc_pobjs
+			end
+			set $n += 1
+			set $ccp += 1
+		end
+		printf " %8d %8d %8d %8d", $mc->mc_wretry_cnt, \
+		    $mc->mc_nwretry_cnt, $mc->mc_nwfail_cnt, $tot
+		printf "\n"
+		set $mc = (mcache_t *)$mc->mc_list.le_next
+	end
+end
+
+document mcache_stat
+Syntax: (gdb) mcache_stat
+| Print all mcaches in the system.
+end
+
+define mcache_showzone
+	set $mc = (mcache_t *)$arg0
+	if $mc->mc_slab_zone != 0
+		printf "%p", $mc->mc_slab_zone
+	else
+		printf "   custom"
+end
+
+document mcache_showzone
+Syntax: (gdb) mcache_showzone <mcache_addr>
+| Print the type of backend (custom or zone) of a mcache.
+end
+
+define mcache_walkobj
+	set $p = (mcache_obj_t *)$arg0
+	set $cnt = 1
+	set $tot = 0
+	while $p
+		printf "%4d: 0x%08x\n", $cnt, $p,
+		set $p = $p->obj_next
+		set $cnt = $cnt + 1
+	end
+end
+
+document mcache_walkobj
+Syntax: (gdb) mcache_walkobj <addr>
+| Given a mcache object address, walk its obj_next pointer
+end
+
+define mcache_showcache
+	set $cp = (mcache_t *)$arg0
+	set $ccp = (mcache_cpu_t *)$cp->mc_cpu
+	set $bktsize = $cp->mc_cpu.cc_bktsize
+	set $cnt = 0
+	set $tot = 0
+	printf "Showing cache '%s':\n\n", $cp->mc_name
+	printf " CPU  cc_objs cc_pobjs    total\n"
+	printf "---- -------- -------- --------\n"
+	while $cnt < ncpu
+		set $objs = $ccp->cc_objs
+		if $objs <= 0
+			set $objs = 0
+		end
+		set $pobjs = $ccp->cc_pobjs
+		if $pobjs <= 0
+			set $pobjs = 0
+		end
+		set $tot_cpu = $objs + $pobjs
+		set $tot += $tot_cpu
+		printf "%4d %8d %8d %8d\n", $cnt, $objs, $pobjs, $tot_cpu
+		set $ccp += 1
+		set $cnt += 1
+	end
+	printf "                       ========\n"
+	printf "                       %8d\n", $tot
+	printf "\n"
+	set $tot += $cp->mc_full.bl_total * $bktsize
+	printf "Total # of full buckets (%d objs/bkt):\t%-8d\n", \
+	    $bktsize, $cp->mc_full.bl_total
+	printf "Total # of objects cached:\t\t%-8d\n", $tot
+end
+
+document mcache_showcache
+| Display the number of objects in the cache
+end
+
+set $NSLABSPMB = sizeof(mcl_slabg_t)/sizeof(mcl_slab_t)
+
+define mbuf_slabstbl
+	set $x = 0
+
+	printf "slot addr       slabs range\n"
+	printf "---- ---------- -----------------------\n"
+	while $x < maxslabgrp
+		set $slg = slabstbl[$x]
+		printf "%3d: ", $x
+		if $slg == 0
+			printf "-\n"
+		else
+			printf "%p [%p-%p]\n", $slg, &$slg->slg_slab[0], \
+			    &$slg->slg_slab[$NSLABSPMB-1]
+		end
+		set $x += 1
+	end
+end
+
+document mbuf_slabstbl
+| Display the mbuf slabs table
+end
+
+set $SLF_MAPPED=0x0001
+set $SLF_PARTIAL=0x0002
+set $SLF_DETACHED=0x0004
+
+define mbuf_slabs
+	set $slg = (mcl_slabg_t *)$arg0
+	set $x = 0
+
+	printf "slot addr       next       base        C  R  N   size flags\n"
+	printf "---- ---------- ---------- ---------- -- -- -- ------ -----\n"
+	while $x < $NSLABSPMB
+		set $sl = &$slg->slg_slab[$x]
+		printf "%3d: 0x%08x 0x%08x 0x%08x %2d %2d %2d %6d 0x%04x ", \
+		    $x + 1, $sl, $sl->sl_next, $sl->sl_base, $sl->sl_class, \
+		    $sl->sl_refcnt, $sl->sl_chunks, $sl->sl_len, \
+		    $sl->sl_flags
+		if $sl->sl_flags != 0
+			printf "<"
+			if $sl->sl_flags & $SLF_MAPPED
+				printf "mapped"
+			end
+			if $sl->sl_flags & $SLF_PARTIAL
+				printf ",partial"
+			end
+			if $sl->sl_flags & $SLF_DETACHED
+				printf ",detached"
+			end
+			printf ">"
+		end
+		printf "\n"
+		set $x += 1
+	end
+end
+
+document mbuf_slabs
+| Display all mbuf slabs in the group
+end
+
+define mbuf_stat
+	set $x = 0
+
+	printf "class               total   cached     uncached           inuse           failed   waiter notified    purge\n"
+	printf "name                 objs     objs     objs / slabs        objs      alloc count    count    count    count\n"
+	printf "---------------- -------- -------- ------------------- -------- ---------------- -------- -------- --------\n"
+	while $x < (sizeof(mbuf_table) / sizeof(mbuf_table[0]))
+		set $mbt = mbuf_table[$x]
+		set $mcs = (mb_class_stat_t *)mbuf_table[$x].mtbl_stats
+		set $tot = 0
+		set $mc = $mbt->mtbl_cache
+		set $bktsize = $mc->mc_cpu.cc_bktsize
+		set $tot += $mc->mc_full.bl_total * $bktsize
+		set $ccp = (mcache_cpu_t *)$mc->mc_cpu
+		set $n = 0
+		while $n < ncpu
+			if $ccp->cc_objs > 0
+				set $tot += $ccp->cc_objs
+			end
+			if $ccp->cc_pobjs > 0
+				set $tot += $ccp->cc_pobjs
+			end
+			set $n += 1
+			set $ccp += 1
+		end
+
+		printf "%-16s %8d %8d %8d / %-8d %8d %16llu %8d %8llu %8llu", \
+		    $mcs->mbcl_cname, $mcs->mbcl_total, $tot, \
+		    $mcs->mbcl_infree, $mcs->mbcl_slab_cnt, \
+		    ($mcs->mbcl_total - $tot - $mcs->mbcl_infree), \
+		    $mcs->mbcl_fail_cnt, $mc->mc_waiter_cnt, \
+		    $mcs->mbcl_notified, $mcs->mbcl_purge_cnt
+		printf "\n"
+		set $x += 1
+	end
+end
+
+document mbuf_stat
+| Print extended mbuf allocator statistics.
+end
+
+set $MB_INUSE = 0x1
+set $MB_COMP_INUSE = 0x2
+set $MB_SCVALID = 0x4
+
+set $MCLBYTES = 2048
+set $MSIZE = 256
+set $NBPG = 4096
+set $M16KCLBYTES = 16384
+
+define mbuf_mca_ctype
+	set $mca = (mcache_audit_t *)$arg0
+	set $vopt = $arg1
+	set $cp = $mca->mca_cache
+	set $class = (unsigned int)$cp->mc_private
+	set $csize = mbuf_table[$class].mtbl_stats->mbcl_size
+	set $done = 0
+	if $csize == $MSIZE
+		if $vopt
+			printf "M (mbuf) "
+		else
+			printf "M     "
+		end
+		set $done = 1
+	end
+	if !$done && $csize == $MCLBYTES
+		if $vopt
+			printf "CL (2K cluster) "
+		else
+			printf "CL    "
+		end
+		set $done = 1
+	end
+	if !$done && $csize == $NBPG
+		if $vopt
+			printf "BCL (4K cluster) "
+		else
+			printf "BCL   "
+		end
+		set $done = 1
+	end
+	if !$done && $csize == $M16KCLBYTES
+		if $vopt
+			printf "JCL (16K cluster) "
+		else
+			printf "JCL   "
+		end
+		set $done = 1
+	end
+	if !$done && $csize == ($MSIZE+$MCLBYTES)
+		if $mca->mca_uflags & $MB_SCVALID
+			if $mca->mca_uptr
+				printf "M+CL  "
+				if $vopt
+					printf "(paired mbuf, 2K cluster)"
+				end
+			else
+				printf "M-CL  "
+				if $vopt
+					printf "(unpaired mbuf, 2K cluster) "
+				end
+			end
+		else
+			if $mca->mca_uptr
+				printf "CL+M  "
+				if $vopt
+					printf "(paired 2K cluster, mbuf) "
+				end
+			else
+				printf "CL-M  "
+				if $vopt
+					printf "(paired 2K cluster, mbuf) "
+				end
+			end
+		end
+		set $done = 1
+	end
+	if !$done && $csize == ($MSIZE+$NBPG)
+		if $mca->mca_uflags & $MB_SCVALID
+			if $mca->mca_uptr
+				printf "M+BCL "
+				if $vopt
+					printf "(paired mbuf, 4K cluster) "
+				end
+			else
+				printf "M-BCL "
+				if $vopt
+					printf "(unpaired mbuf, 4K cluster) "
+				end
+			end
+		else
+			if $mca->mca_uptr
+				printf "BCL+M "
+				if $vopt
+					printf "(paired 4K cluster, mbuf) "
+				end
+			else
+				printf "BCL-M "
+				if $vopt
+					printf "(unpaired 4K cluster, mbuf) "
+				end
+			end
+		end
+		set $done = 1
+	end
+	if !$done && $csize == ($MSIZE+$M16KCLBYTES)
+		if $mca->mca_uflags & $MB_SCVALID
+			if $mca->mca_uptr
+				printf "M+JCL "
+				if $vopt
+					printf "(paired mbuf, 16K cluster) "
+				end
+			else
+				printf "M-JCL "
+				if $vopt
+					printf "(unpaired mbuf, 16K cluster) "
+				end
+			end
+		else
+			if $mca->mca_uptr
+				printf "JCL+M "
+				if $vopt
+					printf "(paired 16K cluster, mbuf) "
+				end
+			else
+				printf "JCL-M "
+				if $vopt
+					printf "(unpaired 16K cluster, mbuf) "
+				end
+			end
+		end
+		set $done = 1
+	end
+	if !$done
+		printf "unknown: %s ", $cp->mc_name
+	end
+end
+
+document mbuf_mca_ctype
+| This is a helper macro for mbuf_show{active,inactive,all} that prints
+| out the mbuf object type represented by a given mcache audit structure.
+end
+
+define mbuf_showactive
+	mbuf_walkallslabs 1 0
+end
+
+document mbuf_showactive
+Syntax: (gdb) mbuf_showactive
+| Walk the mbuf objects pool and print only the active ones; this
+| requires mbuf debugging to be turned on, by setting the appropriate flags
+| to the "mbuf_debug" boot-args parameter.  Active objects are those that
+| are outstanding (have not returned to the mbuf slab layer) and in use
+| by the client (have not been freed).
+end
+
+define mbuf_showinactive
+	mbuf_walkallslabs 0 1
+end
+
+document mbuf_showinactive
+Syntax: (gdb) mbuf_showinactive
+| Walk the mbuf objects pool and print only the inactive ones; this
+| requires mbuf debugging to be turned on, by setting the appropriate flags
+| to the "mbuf_debug" boot-args parameter.  Inactive objects are those that
+| are outstanding (have not returned to the mbuf slab layer) but have been
+| freed by the client, i.e. they still reside in the mcache layer ready to
+| be used for subsequent allocation requests.
+end
+
+define mbuf_showall
+	mbuf_walkallslabs 1 1
+end
+
+document mbuf_showall
+Syntax: (gdb) mbuf_showall
+| Walk the mbuf objects pool and print them all; this requires
+| mbuf debugging to be turned on, by setting the appropriate flags to the
+| "mbuf_debug" boot-args parameter.
+end
+
+define mbuf_mcaobjs
+end
+
+define mbuf_walkallslabs
+	set $show_a = $arg0
+	set $show_f = $arg1
+	set $x = 0
+	set $total = 0
+	set $total_a = 0
+	set $total_f = 0
+
+	printf "("
+	if $show_a && !$show_f
+		printf "Searching only for active "
+	end
+	if !$show_a && $show_f
+		printf "Searching only for inactive "
+	end
+	if $show_a && $show_f
+		printf "Displaying all "
+	end
+	printf "objects; this may take a while ...)\n\n"
+
+	printf "                slab        mca        obj        allocation\n"
+	printf "slot idx     address    address    address type        state\n"
+	printf "---- ---- ---------- ---------- ---------- ----- -----------\n"
+
+	while $x < slabgrp
+		set $slg = slabstbl[$x]
+		set $y = 0
+		set $stop = 0
+		while $y < $NSLABSPMB && $stop == 0
+			set $sl = &$slg->slg_slab[$y]
+			set $base = (char *)$sl->sl_base
+			set $ix = ($base - (char *)mbutl) >> 11
+			set $clbase = ((union mcluster *)(mbutl + $ix))
+			set $mclidx = ($base - (char *)$clbase) >> 8
+			set $mca = mclaudit[$ix].cl_audit[$mclidx]
+			set $first = 1
+
+			while $mca != 0 && $mca->mca_addr != 0
+				set $printmca = 0
+				if $mca->mca_uflags & ($MB_INUSE|$MB_COMP_INUSE)
+					set $total_a = $total_a + 1
+					set $printmca = $show_a
+				else
+					set $total_f = $total_f + 1
+					set $printmca = $show_f
+				end
+
+				if $printmca != 0
+					if $first == 1
+						printf "%4d %4d 0x%08x ", $x, $y, $sl
+					else
+						printf "                     "
+					end
+
+					printf "0x%08x 0x%08x ", $mca, $mca->mca_addr
+					mbuf_mca_ctype $mca 0
+					if $mca->mca_uflags & ($MB_INUSE|$MB_COMP_INUSE)
+						printf "active      "
+					else
+						printf "      freed "
+					end
+					if $first == 1
+						set $first = 0
+					end
+					printf "\n"
+					set $total = $total + 1
+				end
+				set $mca = $mca->mca_next
+			end
+			set $y += 1
+			if $slg->slg_slab[$y].sl_base == 0
+				set $stop = 1
+			end
+		end
+		set $x += 1
+	end
+	if $total && $show_a && $show_f
+		printf "\ntotal objects:\t%d\n", $total
+		printf "active/unfreed:\t%d\n", $total_a
+		printf "freed/in_cache:\t%d\n", $total_f
+	end
+end
+
+document mbuf_walkallslabs
+| Walk the mbuf objects pool; this requires mbuf debugging to be
+| turned on, by setting the appropriate flags to the "mbuf_debug" boot-args
+| parameter.  This is a backend routine for mbuf_show{active,inactive,all}.
+end
+
+define rtentry_trash
+	set $rtd = (struct rtentry_dbg *)rttrash_head.tqh_first
+	set $cnt = 0
+	while $rtd != 0
+		if $cnt == 0
+			printf "      rtentry_dbg  ref flags\n"
+			printf "      ------------ --- ----------\n"
+		end
+		printf "%4d: %p    %3d 0x%08x\n", $cnt + 1, $rtd, \
+		    $rtd->rtd_refhold_cnt - $rtd->rtd_refrele_cnt, \
+		    $rtd->rtd_entry.rt_flags
+		set $rtd = $rtd->rtd_trash_link.tqe_next
+		set $cnt = $cnt + 1
+	end
+end
+
+document rtentry_trash
+Syntax: (gdb) rtentry_trash
+| Walk the list of trash route entries; this requires route entry
+| debugging to be turned on, by setting the appropriate flags to the
+| "rte_debug" boot-args parameter.
+end
+
+set $RTD_TRSTACK_SIZE = 8
+set $RTD_REFHIST_SIZE = 4
+
+define rtentry_showdbg
+	set $rtd = (struct rtentry_dbg *)$arg0
+	set $cnt = 0
+
+	printf "Total holds: %d\n", $rtd->rtd_refhold_cnt
+	printf "Next hold slot: %d\n", $rtd->rtd_refhold_next
+	printf "Total releases: %d\n", $rtd->rtd_refrele_cnt
+	printf "Next release slot: %d\n", $rtd->rtd_refrele_next
+
+	set $ix = 0
+	while $ix < $RTD_TRSTACK_SIZE
+		set $pc = $rtd->rtd_alloc_stk_pc[$ix]
+		if $pc != 0
+			if $ix == 0
+				printf "\nAlloc (thread %p):\n", \
+				    $rtd->rtd_alloc_thread
+			end
+			printf "%4d: ", $ix + 1
+			pcprint $pc
+			printf "\n"
+		end
+		set $ix = $ix + 1
+	end
+	set $ix = 0
+	while $ix < $RTD_TRSTACK_SIZE
+		set $pc = $rtd->rtd_free_stk_pc[$ix]
+		if $pc != 0
+			if $ix == 0
+				printf "\nFree: (thread %p)\n", \
+				    $rtd->rtd_free_thread
+			end
+			printf "%4d: ", $ix + 1
+			pcprint $pc
+			printf "\n"
+		end
+		set $ix = $ix + 1
+	end
+	while $cnt < $RTD_REFHIST_SIZE
+		set $ix = 0
+		while $ix < $RTD_TRSTACK_SIZE
+			set $pc = $rtd->rtd_refhold[$cnt].pc[$ix]
+			if $pc != 0
+				if $ix == 0
+					printf "\nHold [%d] (thread %p):\n", \
+					    $cnt, $rtd->rtd_refhold[$cnt].th
+				end
+				printf "%4d: ", $ix + 1
+				pcprint $pc
+				printf "\n"
+			end
+			set $ix = $ix + 1
+		end
+		set $cnt = $cnt + 1
+	end
+	set $cnt = 0
+	while $cnt < $RTD_REFHIST_SIZE
+		set $ix = 0
+		while $ix < $RTD_TRSTACK_SIZE
+			set $pc = $rtd->rtd_refrele[$cnt].pc[$ix]
+			if $pc != 0
+				if $ix == 0
+					printf "\nRelease [%d] (thread %p):\n",\
+					    $cnt, $rtd->rtd_refrele[$cnt].th
+				end
+				printf "%4d: ", $ix + 1
+				pcprint $pc
+				printf "\n"
+			end
+			set $ix = $ix + 1
+		end
+		set $cnt = $cnt + 1
+	end
+end
+
+document rtentry_showdbg
+Syntax: (gdb) rtentry_showdbg <addr>
+| Given a route entry structure address, print the debug information
+| related to it.  This requires route entry debugging to be turned
+| on, by setting the appropriate flags to the "rte_debug" boot-args
+| parameter.
+end
+
+#
+# print all OSMalloc stats 
+
+define ostag_print
+set $kgm_tagp = (OSMallocTag)$arg0
+printf "0x%08x: ", $kgm_tagp
+printf "%8d ",$kgm_tagp->OSMT_refcnt
+printf "%8x ",$kgm_tagp->OSMT_state
+printf "%8x ",$kgm_tagp->OSMT_attr
+printf "%s ",$kgm_tagp->OSMT_name
+printf "\n"
+end
+
+
+define showosmalloc 
+printf "TAG          COUNT     STATE     ATTR     NAME\n"
+set $kgm_tagheadp = (OSMallocTag)&OSMalloc_tag_list
+    set $kgm_tagptr = (OSMallocTag )($kgm_tagheadp->OSMT_link.next)
+    while $kgm_tagptr != $kgm_tagheadp
+	ostag_print $kgm_tagptr
+	set $kgm_tagptr = (OSMallocTag)$kgm_tagptr->OSMT_link.next
+    end
+	printf "\n"
+end
+document showosmalloc
+Syntax: (gdb) showosmalloc
+| Print the outstanding allocation count by OSMallocTags.
+end
+
+
+define systemlog
+    if msgbufp->msg_bufc[msgbufp->msg_bufx] == 0
+        # The buffer hasn't wrapped, so take the easy (and fast!) path
+        printf "%s", msgbufp->msg_bufc
+    else
+        set $kgm_msgbuf = *msgbufp
+        set $kgm_syslog_bufsize = $kgm_msgbuf.msg_size
+        set $kgm_syslog_bufend = $kgm_msgbuf.msg_bufx
+        if $kgm_syslog_bufend >= $kgm_syslog_bufsize
+            set $kgm_syslog_bufend = 0
+        end
+    
+        # print older messages from msg_bufx to end of buffer
+        set $kgm_i = $kgm_syslog_bufend
+        while $kgm_i < $kgm_syslog_bufsize
+            set $kgm_syslog_char = $kgm_msgbuf.msg_bufc[$kgm_i]
+            if $kgm_syslog_char == 0
+                # break out of loop
+                set $kgm_i = $kgm_syslog_bufsize
+            else
+                printf "%c", $kgm_syslog_char
+            end
+            set $kgm_i = $kgm_i + 1
+        end
+        
+        # print newer messages from start of buffer to msg_bufx
+        set $kgm_i = 0
+        while $kgm_i < $kgm_syslog_bufend
+            set $kgm_syslog_char = $kgm_msgbuf.msg_bufc[$kgm_i]
+            printf "%c", $kgm_syslog_char
+            set $kgm_i = $kgm_i + 1
+        end
+    end
+    printf "\n"
+end
+document systemlog
+| Syntax: systemlog
+| Display the kernel's printf ring buffer
+end
+
+define printvnodepathint_recur
+	if $arg0 != 0
+		if ($arg0->v_flag & 0x000001) && ($arg0->v_mount != 0)
+			if $arg0->v_mount->mnt_vnodecovered != 0
+				printvnodepathint_recur  $arg0->v_mount->mnt_vnodecovered $arg0->v_mount->mnt_vnodecovered->v_name
+			end
+		else
+			printvnodepathint_recur $arg0->v_parent $arg0->v_parent->v_name
+			printf "/%s", $arg1
+		end
+	end
+end
+
+#
+# Show the locks held on a vnode by range, type, and holder.
+#
+define showvnodelocks
+    if ($argc == 1)
+    	_showvnodelockheader
+	_showvnodelocks $arg0
+    else
+    	printf "| Usage:\n|\n"
+	help showvnodelocks
+    end
+end
+document showvnodelocks
+| Given a vnodet pointer, display the list of advisory record locks for the
+| referenced pvnode.
+| The following is the syntax:
+|     (gdb) showvnodelocks <vnode_t>
+end
+
+define _showvnodelockheader
+    printf "*  type   W  held by        lock type  start               end\n"
+    printf "-  -----  -  -------------  ---------  ------------------  ------------------\n"
+end
+
+#
+# Macro to display a single lock; used to display both held locks and
+# blocked locks
+#
+define _showvnodelock
+    set $kgm_svl_lock = ((struct lockf *)$arg0)
+
+    # decode flags
+    set $kgm_svl_flags = $kgm_svl_lock->lf_flags
+    set $kgm_svl_type = $kgm_svl_lock->lf_type
+    if ($kgm_svl_flags & 0x20)
+	printf "flock"
+    end
+    if ($kgm_svl_flags & 0x40)
+	printf "posix"
+    end
+    if ($kgm_svl_flags & 0x80)
+	printf "prov "
+    end
+    if ($kgm_svl_flags & 0x10)
+	printf "  W  "
+    else
+	printf "  .  "
+    end
+
+    # POSIX file vs. advisory range locks
+    if ($kgm_svl_flags & 0x40)
+	set $kgm_svl_proc = (proc_t)$kgm_svl_lock->lf_id
+	printf "PID %8d   ", $kgm_svl_proc->p_pid
+    else
+	printf "ID 0x%08x  ", $kgm_svl_lock->lf_id
+    end
+
+    # lock type
+    if ($kgm_svl_type == 1)
+		printf "shared     "
+    else
+	if ($kgm_svl_type == 3)
+		printf "exclusive  "
+	else
+	    if ($kgm_svl_type == 2)
+		printf "unlock     "
+	    else
+		printf "unknown    "
+	    end
+	end
+    end
+
+    # start and stop
+    printf "0x%016x..", $kgm_svl_lock->lf_start
+    printf "0x%016x  ", $kgm_svl_lock->lf_end
+    printf "\n"
+end
+
+# Body of showvnodelocks, not including header
+define _showvnodelocks
+    set $kgm_svl_vnode = ((vnode_t)$arg0)
+    set $kgm_svl_lockiter = $kgm_svl_vnode->v_lockf
+    while ($kgm_svl_lockiter != 0)
+	# locks that are held
+    	printf "H  "
+    	_showvnodelock $kgm_svl_lockiter
+
+	# and any locks blocked by them
+	set $kgm_svl_blocker = $kgm_svl_lockiter->lf_blkhd.tqh_first
+	while ($kgm_svl_blocker != 0)
+	    printf ">  "
+	    _showvnodelock $kgm_svl_blocker
+	    set $kgm_svl_blocker = $kgm_svl_blocker->lf_block.tqe_next
+	end
+
+	# and on to the next one...
+    	set $kgm_svl_lockiter = $kgm_svl_lockiter->lf_next
+    end
+end
+
+define showvnodepath
+	set $vp = (struct vnode *)$arg0
+	if $vp != 0
+		if ($vp->v_flag & 0x000001) && ($vp->v_mount != 0) && ($vp->v_mount->mnt_flag & 0x00004000)
+			printf "/"
+		else
+			printvnodepathint_recur $vp $vp->v_name
+		end
+	end
+	printf "\n"
+end
+
+document showvnodepath
+Syntax: (gdb) showvnodepath <vnode>
+| Prints the path for a vnode
+end
+
+define printcolonhex
+    if ($argc == 2) 
+	set $addr = $arg0
+	set $count = $arg1
+	set $li = 0
+	while ($li < $count)
+	    if ($li == 0)
+		printf "%02x", (u_char)$addr[$li]
+	    end
+	    if ($li != 0)
+		printf ":%02x", (u_char)$addr[$li]
+	    end
+	    set $li = $li + 1
+	end
+    end
+end
+
+define showsockaddr_dl
+    set $sdl = (struct sockaddr_dl *)$arg0
+    printf "LINK "
+    if ($sdl == 0)
+	printf "(null)"
+    else
+	set $addr = $sdl->sdl_data + $sdl->sdl_nlen
+	set $count = $sdl->sdl_alen
+	printcolonhex $addr $count
+    end
+end
+
+define showsockaddr_unspec
+    set $sockaddr = (struct sockaddr *)$arg0
+    set $addr = $sockaddr->sa_data
+    set $count = $sockaddr->sa_len - 2
+    printf "UNSP "
+    printcolonhex $addr $count
+end
+
+define showsockaddr_at
+    set $sockaddr = (struct sockaddr *)$arg0
+    set $addr = $sockaddr->sa_data
+    set $count = $sockaddr->sa_len - 2
+    printf "ATLK "
+    printcolonhex $addr $count
+end
+
+define showsockaddr_in
+    set $sin = (struct sockaddr_in *)$arg0
+    set $sa_bytes = (unsigned char *)&($sin->sin_addr)
+    printf "IPV4 %d.%d.%d.%d", $sa_bytes[0], $sa_bytes[1], $sa_bytes[2], $sa_bytes[3]
+end
+
+define showsockaddr_in6
+    set $sin6 = (struct sockaddr_in6 *)$arg0
+    set $sa_bytes = $sin6->sin6_addr.__u6_addr.__u6_addr8
+    printf "IPV6 %02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x", $sa_bytes[0], $sa_bytes[1], $sa_bytes[2], $sa_bytes[3], $sa_bytes[4], $sa_bytes[5], $sa_bytes[6], $sa_bytes[7], $sa_bytes[8], $sa_bytes[9], $sa_bytes[10], $sa_bytes[11], $sa_bytes[12], $sa_bytes[13], $sa_bytes[14], $sa_bytes[15]
+end
+
+define showifmultiaddrs
+   set $ifp = (struct ifnet *)$arg0
+   set $if_multi = (struct ifmultiaddr *)$ifp->if_multiaddrs->lh_first
+   set $mymulti = $if_multi
+   set $myi = 0
+   while ($mymulti != 0)
+	printf "%2d. ", $myi
+	set $sa_family = $mymulti->ifma_addr.sa_family
+	if ($sa_family == 2)
+	    if ($mymulti->ifma_ll != 0)
+	    	showsockaddr_dl $mymulti->ifma_ll->ifma_addr
+	        printf " "
+	    end
+	    showsockaddr_in $mymulti->ifma_addr
+	end
+	if ($sa_family == 30)
+	    if ($mymulti->ifma_ll != 0)
+	    	showsockaddr_dl $mymulti->ifma_ll->ifma_addr
+	        printf " "
+	    end
+	    showsockaddr_in6 $mymulti->ifma_addr
+	end
+ 	if ($sa_family == 18)
+	    showsockaddr_dl $mymulti->ifma_addr
+	end
+	if ($sa_family == 0)
+	    showsockaddr_unspec $mymulti->ifma_addr 6
+	end
+	printf " [%d]", $mymulti->ifma_refcount
+	printf "\n"
+	set $mymulti = $mymulti->ifma_link.le_next
+	set $myi = $myi + 1
+   end
+end
+
+document showifmultiaddrs
+Syntax showifmultiaddrs <ifp>
+| show the (struct ifnet).if_multiaddrs list of multicast addresses for the given ifp
+end
+
+define showsockaddr
+    set $mysock = (struct sockaddr *)$arg0
+    set $showsockaddr_handled = 0
+    if ($mysock == 0)
+	printf "(null)"
+    else
+	if ($mysock->sa_family == 0)
+	    showsockaddr_unspec $mysock
+	    set $showsockaddr_handled = 1
+	end
+	if ($mysock->sa_family == 2)
+	    showsockaddr_in $mysock
+	    set $showsockaddr_handled = 1
+	end
+	if ($mysock->sa_family == 30)
+	    showsockaddr_in6 $mysock
+	    set $showsockaddr_handled = 1
+	end
+	if ($mysock->sa_family == 18)
+	    showsockaddr_dl $mysock
+	    set $showsockaddr_handled = 1
+	end
+	if ($mysock->sa_family == 16)
+	    showsockaddr_at $mysock
+	    set $showsockaddr_handled = 1
+	end
+	if ($showsockaddr_handled == 0)
+	    printf "%d ", $mysock->sa_family
+	    set $addr = $mysock->sa_data
+	    set $count = $mysock->sa_len
+	    printcolonhex $addr $count
+	end
+    end
+end
+
+define showifflags
+	set $flags = (u_short)$arg0
+	set $first = 1
+	printf "<"
+	if ($flags & 0x1)
+	    printf "UP"
+	    set $first = 0
+	end
+	if ($flags & 0x2)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "BROADCAST"
+	end
+	if ($flags & 0x4)
+	    printf "DEBUG"
+	end
+	if ($flags & 0x8)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "LOOPBACK"
+	end
+	if ($flags & 0x10)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "POINTTOPOINT"
+	end
+#	if ($flags & 0x20)
+#	    if ($first == 1)
+#		set $first = 0
+#	    else
+#	    	printf ","
+#	    end
+#	    printf "NOTRAILERS"
+#	end
+	if ($flags & 0x40)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "RUNNING"
+	end
+	if ($flags & 0x80)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "NOARP"
+	end
+	if ($flags & 0x100)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "PROMISC"
+	end
+	if ($flags & 0x200)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "ALLMULTI"
+	end
+	if ($flags & 0x400)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "OACTIVE"
+	end
+	if ($flags & 0x800)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "SIMPLEX"
+	end
+	if ($flags & 0x1000)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "LINK0"
+	end
+	if ($flags & 0x2000)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "LINK1"
+	end
+	if ($flags & 0x4000)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "LINK2-ALTPHYS"
+	end
+	if ($flags & 0x8000)
+	    if ($first == 1)
+		set $first = 0
+	    else
+	    	printf ","
+	    end
+	    printf "MULTICAST"
+	end
+	printf ">"
+end
+
+define showifaddrs
+   set $ifp = (struct ifnet *)$arg0
+   set $myifaddr = (struct ifaddr *)$ifp->if_addrhead->tqh_first
+   set $myi = 0
+   while ($myifaddr != 0)
+	printf "\t%d. ", $myi
+	showsockaddr $myifaddr->ifa_addr
+	printf " [%d]\n", $myifaddr->ifa_refcnt
+	set $myifaddr = $myifaddr->ifa_link->tqe_next
+	set $myi = $myi + 1
+   end
+end
+
+document showifaddrs
+Syntax: showifaddrs <ifp>
+| show the (struct ifnet).if_addrhead list of addresses for the given ifp
+end
+
+define ifconfig
+   set $ifconfig_all = 0
+   if ($argc == 1)
+	set $ifconfig_all = 1
+   end
+   set $ifp = (struct ifnet *)(ifnet->tqh_first)
+   while ($ifp != 0)
+	printf "%s%d: flags=%x", $ifp->if_name, $ifp->if_unit, (u_short)$ifp->if_flags
+	showifflags $ifp->if_flags
+	printf " mtu %d\n", $ifp->if_data.ifi_mtu
+	printf "\t(struct ifnet *)0x%x\n", $ifp
+	if ($ifconfig_all == 1) 
+	   showifaddrs $ifp
+        end
+	set $ifp = $ifp->if_link->tqe_next
+   end
+end
+document ifconfig
+Syntax: (gdb) ifconfig
+| display ifconfig-like output, and print the (struct ifnet *) pointers for further inspection
+end
+
+define showbpfdtab
+    set $myi = 0
+    while ($myi < bpf_dtab_size)
+	if (bpf_dtab[$myi] != 0)
+		printf "Address 0x%x, bd_next 0x%x\n", bpf_dtab[$myi], bpf_dtab[$myi]->bd_next
+		print *bpf_dtab[$myi]
+	end
+	set $myi = $myi + 1
+    end
+end
+
+define showallvols
+	printf "volume      mnt_data    mnt_devvp   typename    mountpoint\n"
+	set $kgm_vol = (mount_t) mountlist.tqh_first
+	while $kgm_vol
+		printf "0x%08x  ", $kgm_vol
+		printf "0x%08x  ", $kgm_vol->mnt_data
+		printf "0x%08x  ", $kgm_vol->mnt_devvp
+		if  ($kgm_vol->mnt_vtable->vfc_name[0] == 'h') && \
+		    ($kgm_vol->mnt_vtable->vfc_name[1] == 'f') && \
+		    ($kgm_vol->mnt_vtable->vfc_name[2] == 's') && \
+		    ($kgm_vol->mnt_vtable->vfc_name[3] == '\0')
+			set $kgm_hfsmount = \
+			    (struct hfsmount *) $kgm_vol->mnt_data
+			if $kgm_hfsmount->hfs_freezing_proc != 0
+				printf "FROZEN hfs  "
+			else
+				printf "hfs         "
+			end
+		else
+			printf "%-10s  ", $kgm_vol->mnt_vtable->vfc_name
+		end
+		printf "%s\n", $kgm_vol->mnt_vfsstat.f_mntonname
+		
+		set $kgm_vol = (mount_t) $kgm_vol->mnt_list.tqe_next
+	end
+end
+
+document showallvols
+Syntax: (gdb) showallvols
+| Display a summary of mounted volumes
+end
+
+define showvnodeheader
+	printf "vnode       usecount  iocount  v_data      vtype  parent      name\n"
+end
+
+define showvnodeint
+	set $kgm_vnode = (vnode_t) $arg0
+	printf "0x%08x  ", $kgm_vnode
+	printf "%8d  ", $kgm_vnode->v_usecount
+	printf "%7d  ", $kgm_vnode->v_iocount
+# print information about clean/dirty blocks?
+	printf "0x%08x  ", $kgm_vnode->v_data
+
+	# print the vtype, using the enum tag
+	set $kgm_vtype = $kgm_vnode->v_type
+	if $kgm_vtype == VNON
+		printf "VNON   "
+	end
+	if $kgm_vtype == VREG
+		printf "VREG   "
+	end
+	if $kgm_vtype == VDIR
+		printf "VDIR   "
+	end
+	if $kgm_vtype == VBLK
+		printf "VBLK   "
+	end
+	if $kgm_vtype == VCHR
+		printf "VCHR   "
+	end
+	if $kgm_vtype == VLNK
+		printf "VLNK   "
+	end
+	if $kgm_vtype == VSOCK
+		printf "VSOCK  "
+	end
+	if $kgm_vtype == VFIFO
+		printf "VFIFO  "
+	end
+	if $kgm_vtype == VBAD
+		printf "VBAD   "
+	end
+	if ($kgm_vtype < VNON) || ($kgm_vtype > VBAD)
+		printf "%5d  ", $kgm_vtype
+	end
+
+	printf "0x%08x  ", $kgm_vnode->v_parent
+	if $kgm_vnode->v_name != 0
+		printf "%s\n", $kgm_vnode->v_name
+	else
+		printf "\n"
+	end
+end
+
+define showvnode
+	showvnodeheader
+	showvnodeint $arg0
+end
+
+document showvnode
+Syntax: (gdb) showvnode <vnode>
+| Display info about one vnode
+end
+
+define showvolvnodes
+	showvnodeheader
+	set $kgm_vol = (mount_t) $arg0
+	set $kgm_vnode = (vnode_t) $kgm_vol.mnt_vnodelist.tqh_first
+	while $kgm_vnode
+		showvnodeint $kgm_vnode
+		set $kgm_vnode = (vnode_t) $kgm_vnode->v_mntvnodes.tqe_next
+	end
+end
+
+document showvolvnodes
+Syntax: (gdb) showvolvnodes <mouont_t>
+| Display info about all vnodes of a given mount_t
+end
+
+define showvolbusyvnodes
+	showvnodeheader
+	set $kgm_vol = (mount_t) $arg0
+	set $kgm_vnode = (vnode_t) $kgm_vol.mnt_vnodelist.tqh_first
+	while $kgm_vnode
+		if $kgm_vnode->v_iocount != 0
+			showvnodeint $kgm_vnode
+		end
+		set $kgm_vnode = (vnode_t) $kgm_vnode->v_mntvnodes.tqe_next
+	end
+end
+
+document showvolbusyvnodes
+Syntax: (gdb) showvolbusyvnodes <mount_t>
+| Display info about busy (iocount!=0) vnodes of a given mount_t
+end
+
+define showallbusyvnodes
+	showvnodeheader
+	set $kgm_vol = (mount_t) mountlist.tqh_first
+	while $kgm_vol
+		set $kgm_vnode = (vnode_t) $kgm_vol.mnt_vnodelist.tqh_first
+		while $kgm_vnode
+			if $kgm_vnode->v_iocount != 0
+				showvnodeint $kgm_vnode
+			end
+			set $kgm_vnode = (vnode_t) $kgm_vnode->v_mntvnodes.tqe_next
+		end
+		set $kgm_vol = (mount_t) $kgm_vol->mnt_list.tqe_next
+	end
+end
+
+document showallbusyvnodes
+Syntax: (gdb) showallbusyvnodes <vnode>
+| Display info about all busy (iocount!=0) vnodes
+end
+
+define showallvnodes
+	showvnodeheader
+	set $kgm_vol = (mount_t) mountlist.tqh_first
+	while $kgm_vol
+		set $kgm_vnode = (vnode_t) $kgm_vol.mnt_vnodelist.tqh_first
+		while $kgm_vnode
+			showvnodeint $kgm_vnode
+			set $kgm_vnode = (vnode_t) $kgm_vnode->v_mntvnodes.tqe_next
+		end
+		set $kgm_vol = (mount_t) $kgm_vol->mnt_list.tqe_next
+	end
+end
+
+document showallvnodes
+Syntax: (gdb) showallvnodes
+| Display info about all vnodes
+end
+
+define _showvnodelockheader
+    printf "*  type   W  held by        lock type  start               end\n"
+    printf "-  -----  -  -------------  ---------  ------------------  ------------------\n"
+end
+
+define _showvnodelock
+    set $kgm_svl_lock = ((struct lockf *)$arg0)
+
+    # decode flags
+    set $kgm_svl_flags = $kgm_svl_lock->lf_flags
+    set $kgm_svl_type = $kgm_svl_lock->lf_type
+    if ($kgm_svl_flags & 0x20)
+	printf "flock"
+    end
+    if ($kgm_svl_flags & 0x40)
+	printf "posix"
+    end
+    if ($kgm_svl_flags & 0x80)
+	printf "prov "
+    end
+    if ($kgm_svl_flags & 0x10)
+	printf "  W  "
+    else
+	printf "  .  "
+    end
+
+    # POSIX file vs. advisory range locks
+    if ($kgm_svl_flags & 0x40)
+	set $kgm_svl_proc = (proc_t)$kgm_svl_lock->lf_id
+	printf "PID %8d   ", $kgm_svl_proc->p_pid
+    else
+	printf "ID 0x%08x  ", $kgm_svl_lock->lf_id
+    end
+
+    # lock type
+    if ($kgm_svl_type == 1)
+		printf "shared     "
+    else
+	if ($kgm_svl_type == 3)
+		printf "exclusive  "
+	else
+	    if ($kgm_svl_type == 2)
+		printf "unlock     "
+	    else
+		printf "unknown    "
+	    end
+	end
+    end
+
+    # start and stop
+    printf "0x%016x..", $kgm_svl_lock->lf_start
+    printf "0x%016x  ", $kgm_svl_lock->lf_end
+    printf "\n"
+end
+# Body of showvnodelocks, not including header
+define _showvnodelocks
+    set $kgm_svl_vnode = ((vnode_t)$arg0)
+    set $kgm_svl_lockiter = $kgm_svl_vnode->v_lockf
+    while ($kgm_svl_lockiter != 0)
+	# locks that are held
+    	printf "H  "
+    	_showvnodelock $kgm_svl_lockiter
+
+	# and any locks blocked by them
+	set $kgm_svl_blocker = $kgm_svl_lockiter->lf_blkhd.tqh_first
+	while ($kgm_svl_blocker != 0)
+	    printf ">  "
+	    _showvnodelock $kgm_svl_blocker
+	    set $kgm_svl_blocker = $kgm_svl_blocker->lf_block.tqe_next
+	end
+
+	# and on to the next one...
+    	set $kgm_svl_lockiter = $kgm_svl_lockiter->lf_next
+    end
+end
+
+
+define showvnodelocks
+    if ($argc == 1)
+    	_showvnodelockheader
+	_showvnodelocks $arg0
+    else
+    	printf "| Usage:\n|\n"
+	help showvnodelocks
+    end
+end
+
+document showvnodelocks
+Syntax: (gdb) showvnodelocks <vnode_t>
+| Given a vnodet pointer, display the list of advisory record locks for the
+| referenced pvnodes
+end
+
+define showbootargs
+	printf "%s\n", (char*)((boot_args*)PE_state.bootArgs).CommandLine
+end
+
+document showbootargs
+Syntax: showbootargs
+| Display boot arguments passed to the target kernel
+end
+
+define showbootermemorymap
+       set $kgm_boot_args = kernelBootArgs
+       set $kgm_msize = kernelBootArgs->MemoryMapDescriptorSize
+       set $kgm_mcount = kernelBootArgs->MemoryMapSize / $kgm_msize
+       set $kgm_i = 0
+       
+       printf "Type       Physical Start   Number of Pages\n"
+       while $kgm_i < $kgm_mcount
+	     set $kgm_mptr = (EfiMemoryRange *)((unsigned long)kernelBootArgs->MemoryMap + $kgm_i * $kgm_msize)
+#	     p/x *$kgm_mptr
+	     if $kgm_mptr->Type == 0
+	       printf "reserved  "
+	     end
+	     if $kgm_mptr->Type == 1
+	       printf "LoaderCode"
+	     end
+	     if $kgm_mptr->Type == 2
+	       printf "LoaderData"
+	     end
+	     if $kgm_mptr->Type == 3
+	       printf "BS_code   "
+	     end
+	     if $kgm_mptr->Type == 4
+	       printf "BS_data   "
+	     end
+	     if $kgm_mptr->Type == 5
+	       printf "RT_code   "
+	     end
+	     if $kgm_mptr->Type == 6
+	       printf "RT_data   "
+	     end
+	     if $kgm_mptr->Type == 7
+	       printf "available "
+	     end
+	     if $kgm_mptr->Type == 8
+    	       printf "Unusable  "
+	     end
+	     if $kgm_mptr->Type == 9
+               printf "ACPI_recl "
+	     end
+	     if $kgm_mptr->Type == 10
+               printf "ACPI_NVS  "
+	     end
+	     if $kgm_mptr->Type == 11
+               printf "MemMapIO  "
+	     end
+	     if $kgm_mptr->Type == 12
+               printf "MemPortIO "
+	     end
+	     if $kgm_mptr->Type == 13
+               printf "PAL_code  "
+	     end
+	     if $kgm_mptr->Type > 13
+               printf "UNKNOWN   "
+	     end
+
+       	     printf " %016llx %016llx\n", $kgm_mptr->PhysicalStart, $kgm_mptr->NumberOfPages
+	     set $kgm_i = $kgm_i + 1
+       end
+end
+
+document showbootermemorymap
+Syntax: (gdb) showbootermemorymap
+| Prints out the phys memory map from kernelBootArgs
+end
+
+
+define showstacksaftertask
+    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_taskp = (struct task *)$arg0
+    while $kgm_taskp != $kgm_head_taskp
+        showtaskheader
+        showtaskint $kgm_taskp
+        set $kgm_head_actp = &($kgm_taskp->threads)
+        set $kgm_actp = (struct thread *)($kgm_taskp->threads.next) 
+        while $kgm_actp != $kgm_head_actp
+            showactheader
+            if ($decode_wait_events > 0)
+               showactint $kgm_actp 1
+            else
+               showactint $kgm_actp 2
+            end
+            set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
+        end
+        printf "\n"
+        set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next)
+    end
+end
+document showstacksaftertask
+Syntax: (gdb) showstacksaftertask <task>
+| Routine to print out all stacks (as in showallstacks) starting after a given task
+| Useful if that gdb refuses to print a certain task's stack.
+end
+
+define showpmworkqueueint
+    set $kgm_pm_wq = (IOPMWorkQueue *)$arg0
+    set $kgm_pm_node = (IOService *)$kgm_pm_wq->owner
+    printf "0x%08x  0x%08x  ", $kgm_pm_wq, $kgm_pm_node
+    printf "%02d ", $kgm_pm_node->pwrMgt->CurrentPowerState
+    printf "%02d ", $kgm_pm_node->pwrMgt->MachineState
+    printf "%02d  ", $kgm_pm_node->pwrMgt->WaitReason
+    printf "%s\n", $kgm_pm_node->pwrMgt->Name
+    set $kgm_pm_queue = &($kgm_pm_wq->fWorkQueue)
+    set $kgm_pm_req = (IOPMRequest *) $kgm_pm_queue->next
+    while ( (queue_entry_t) $kgm_pm_req != (queue_entry_t) $kgm_pm_queue)
+        printf "   Request  0x%08x [%02x]  Args ", $kgm_pm_req, $kgm_pm_req->fType
+        printf "0x%08x ", $kgm_pm_req->fArg0
+        printf "0x%08x ", $kgm_pm_req->fArg1
+        printf "0x%08x\n", $kgm_pm_req->fArg2
+        set $kgm_pm_req = (IOPMRequest *)$kgm_pm_req->fCommandChain.next
+    end
+end
+
+define showallpmworkqueues
+    set $kgm_pm_next = gIOPMWorkLoop->eventChain
+    printf "WorkQueue   Owner       PS MS WT  Name\n"
+    printf "--------------------------------------\n"
+    while ( $kgm_pm_next )
+        set $kgm_vt = *((void **) $kgm_pm_next)
+        if ($kgm_vt == _ZTV13IOPMWorkQueue)
+            showpmworkqueueint $kgm_pm_next
+        end
+        set $kgm_pm_next = $kgm_pm_next->eventChainNext
+    end
+end
+
+document showallpmworkqueues
+Syntax: (gdb) showallpmworkqueues
+| Display info about all IOPMWorkQueue objects
+end
+
+define showioservicepm
+	set $kgm_iopmpriv = (IOServicePM *)$arg0
+	printf "{ this object = %08x", $kgm_iopmpriv->Owner
+	if ( $kgm_iopmpriv->WeAreRoot )
+		printf " (root)"
+	end
+	printf ", "
+	
+	printf "MachineState = %d (", $kgm_iopmpriv->MachineState
+	if ( $kgm_iopmpriv->MachineState == 1 )
+		printf "kIOPM_OurChangeTellClientsPowerDown"
+	else
+		if ( $kgm_iopmpriv->MachineState == 2 )
+			printf "kIOPM_OurChangeTellPriorityClientsPowerDown"
+		else
+			if ( $kgm_iopmpriv->MachineState == 3 )
+				printf "kIOPM_OurChangeNotifyInterestedDriversWillChange"
+			else
+				if ( $kgm_iopmpriv->MachineState == 4 )
+					printf "kIOPM_OurChangeSetPowerState"
+				else
+					if ( $kgm_iopmpriv->MachineState == 5 )
+						printf "kIOPM_OurChangeWaitForPowerSettle"
+					else
+						if ( $kgm_iopmpriv->MachineState == 6 )
+							printf "kIOPM_OurChangeNotifyInterestedDriversDidChange"
+						else
+							if ( $kgm_iopmpriv->MachineState == 7 )
+								printf "kIOPM_OurChangeFinish"
+							else
+								if ( $kgm_iopmpriv->MachineState == 8 )
+									printf "kIOPM_ParentDownTellPriorityClientsPowerDown"
+								else
+									if ( $kgm_iopmpriv->MachineState == 9 )
+										printf "kIOPM_ParentDownNotifyInterestedDriversWillChange"
+									else
+										if ( $kgm_iopmpriv->MachineState == 10 )
+											printf "Unused_MachineState_10"
+										else
+											if ( $kgm_iopmpriv->MachineState == 11 )
+												printf "kIOPM_ParentDownNotifyDidChangeAndAcknowledgeChange"
+											else
+												if ( $kgm_iopmpriv->MachineState == 12 )
+													printf "kIOPM_ParentDownSetPowerState"
+												else
+													if ( $kgm_iopmpriv->MachineState == 13 )
+														printf "kIOPM_ParentDownWaitForPowerSettle"
+													else
+														if ( $kgm_iopmpriv->MachineState == 14 )
+															printf "kIOPM_ParentDownAcknowledgeChange"
+														else
+															if ( $kgm_iopmpriv->MachineState == 15)
+																printf "kIOPM_ParentUpSetPowerState"
+															else
+																if ( $kgm_iopmpriv->MachineState == 16)
+																	printf "Unused_MachineState_16"
+																else
+																	if ( $kgm_iopmpriv->MachineState == 17)
+																		printf "kIOPM_ParentUpWaitForSettleTime"
+																	else
+																		if ( $kgm_iopmpriv->MachineState == 18)
+																			printf "kIOPM_ParentUpNotifyInterestedDriversDidChange"
+																		else
+																			if ( $kgm_iopmpriv->MachineState == 19)
+																				printf "kIOPM_ParentUpAcknowledgePowerChange"
+																			else
+																				if ( $kgm_iopmpriv->MachineState == 20)
+																					printf "kIOPM_Finished"
+                                                                                else
+                                                                                    if ( $kgm_iopmpriv->MachineState == 21)
+                                                                                        printf "kIOPM_DriverThreadCallDone"
+                                                                                    else
+                                                                                        if ( $kgm_iopmpriv->MachineState == 22)
+                                                                                            printf "kIOPM_NotifyChildrenDone"
+                                                                                        end
+                                                                                    end
+																				end
+																			end
+																		end
+																	end
+																end
+															end
+														end
+													end
+												end
+											end
+										end
+									end
+								end
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+	printf "), "
+	
+	if ( $kgm_iopmpriv->MachineState != 20 )
+        printf "DriverTimer = %d, ",(unsigned int)$kgm_iopmpriv->DriverTimer
+        printf "SettleTime  = %d, ",(unsigned int)$kgm_iopmpriv->SettleTimeUS
+        printf "HeadNoteFlags = %08x, ",(unsigned int)$kgm_iopmpriv->HeadNoteFlags
+        printf "HeadNoteState = %d, ",(unsigned int)$kgm_iopmpriv->HeadNoteState
+        printf "HeadNoteOutputFlags = %08x, ",(unsigned int)$kgm_iopmpriv->HeadNoteOutputFlags
+        printf "HeadNoteDomainState = %08x, ",(unsigned int)$kgm_iopmpriv->HeadNoteDomainState
+        printf "HeadNoteCapabilityFlags = %08x, ",(unsigned int)$kgm_iopmpriv->HeadNoteCapabilityFlags
+        printf "HeadNotePendingAcks = %x, ",(unsigned int)$kgm_iopmpriv->HeadNotePendingAcks
+	end
+
+    if ( $kgm_iopmpriv->DeviceOverrides != 0 )
+        printf"DeviceOverrides, "
+    end
+	
+    printf "DriverDesire = %d, ",(unsigned int)$kgm_iopmpriv->DriverDesire
+    printf "DeviceDesire = %d, ",(unsigned int)$kgm_iopmpriv->DeviceDesire
+    printf "DesiredPowerState = %d, ",(unsigned int)$kgm_iopmpriv->DesiredPowerState
+    printf "PreviousRequest = %d }",(unsigned int)$kgm_iopmpriv->PreviousRequest
+end
+
+document showioservicepm
+Syntax: (gdb) showioservicepm <IOServicePM pointer>
+| Routine to dump the IOServicePM object
+end
+
+define showregistryentryrecursepmstate
+    set $kgm_re         = (IOService *)$arg1
+    set $kgm$arg0_stack = (unsigned long long) $arg2
+
+    if ($arg3)
+	set $kgm$arg0_stack = $kgm$arg0_stack | (1ULL << $kgm_reg_depth)
+    else
+	set $kgm$arg0_stack = $kgm$arg0_stack & ~(1ULL << $kgm_reg_depth)
+    end
+
+    dictget $kgm_re->fRegistryTable $kgm_childkey
+    set $kgm$arg0_child_array = (OSArray *) $kgm_result
+
+    if ($kgm$arg0_child_array)
+	set $kgm$arg0_child_count = $kgm$arg0_child_array->count
+    else
+	set $kgm$arg0_child_count = 0
+    end
+
+    if ($kgm$arg0_child_count)
+	set $kgm$arg0_stack = $kgm$arg0_stack | (2ULL << $kgm_reg_depth)
+    else
+	set $kgm$arg0_stack = $kgm$arg0_stack & ~(2ULL << $kgm_reg_depth)
+    end
+
+    indent $kgm_reg_depth $kgm$arg0_stack
+    printf "+-o "
+
+    dictget $kgm_re->fRegistryTable $kgm_namekey
+    if ($kgm_result == 0)
+	dictget $kgm_re->fRegistryTable gIONameKey
+    end
+    if ($kgm_result == 0)
+	dictget $kgm_re->fPropertyTable gIOClassKey
+    end
+
+    if ($kgm_result != 0)
+	printf "%s <%p>", ((OSString *)$kgm_result)->string, $kgm_re
+    else
+ 	if (((IOService*)$kgm_re)->pwrMgt &&  ((IOService*)$kgm_re)->pwrMgt->Name)
+ 	    printf "%s <%p>", ((IOService*)$kgm_re)->pwrMgt->Name, $kgm_re
+	else
+	    printf "?? <%p>", $kgm_re
+	end
+    end
+
+    if (((IOService*)$kgm_re)->pwrMgt )
+    	printf " Current Power State: %ld ", ((IOService*)$kgm_re)->pwrMgt->CurrentPowerState
+    	#printf " Mach State %ld", ((IOService*)$kgm_re)->pwrMgt->MachineState
+        showioservicepm ((IOService*)$kgm_re)->pwrMgt
+    end
+    printf "\n"
+   
+
+    # recurse
+    if ($kgm$arg0_child_count != 0)
+
+	set $kgm_reg_depth = $kgm_reg_depth + 1
+	set $kgm$arg0_child_idx = 0
+
+	while ($kgm$arg0_child_idx < $kgm$arg0_child_count)
+	    set $kgm_re = $kgm$arg0_child_array->array[$kgm$arg0_child_idx++]
+	    set $kgm_more_sib = ($kgm$arg0_child_idx < $kgm$arg0_child_count)
+	    showregistryentryrecursepmstate _$arg0 $kgm_re $kgm$arg0_stack $kgm_more_sib
+	end
+
+	set $kgm_reg_depth = $kgm_reg_depth - 1
+    end
+end
+
+define showregistryentryintpmstate
+    set $kgm_namekey   = (OSSymbol *) $kgm_reg_plane[2]
+    set $kgm_childkey  = (OSSymbol *) $kgm_reg_plane[4]
+    showregistryentryrecursepmstate _ $arg0 0 0
+end
+
+define showregistrypmstate
+#    setregistryplane gIOPowerPlane
+    set $kgm_reg_depth  = 0
+    set $kgm_show_props = 1
+    showregistryentryintpmstate gRegistryRoot
+end
+
+document showregistrypmstate
+Syntax: (gdb) showregistrypmstate
+| Routine to dump the PM state of each IOPower registry entry
+end
+
+define showstacksafterthread
+    set $kgm_head_taskp = &default_pset.tasks
+    set $kgm_actp = (struct thread *)$arg0
+    set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
+    set $kgm_taskp = (struct task *)$kgm_actp->task
+    while $kgm_taskp != $kgm_head_taskp
+        showtaskheader
+        showtaskint $kgm_taskp
+        set $kgm_head_actp = &($kgm_taskp->threads)
+        while $kgm_actp != $kgm_head_actp
+            showactheader
+            if ($decode_wait_events > 0)
+               showactint $kgm_actp 1
+            else
+               showactint $kgm_actp 2
+            end
+            set $kgm_actp = (struct thread *)($kgm_actp->task_threads.next)
+        end
+        printf "\n"
+        set $kgm_taskp = (struct task *)($kgm_taskp->pset_tasks.next) 
+    end
+end
+
+document showstacksafterthread
+Syntax: (gdb) showstacksafterthread <thread>
+| Routine to print out all stacks (as in showallstacks) starting after a given thread
+| Useful if that gdb refuses to print a certain task's stack.
+end
+
+define kdp-reenter
+       set kdp_reentry_deadline = ((unsigned) $arg0)*1000
+       continue
+end
+
+document kdp-reenter
+Syntax: (gdb) kdp-reenter <seconds>
+| Schedules reentry into the debugger after <seconds> seconds, and resumes
+| the target system.
+end
+
+define _if_present
+    if (!$arg0)
+        printf " not"
+    end
+    printf " present"
+end
+
+define showMCAstate
+    if ($kgm_mtype != 7)
+        printf "Not available for current architecture.\n"
+    else
+        printf "MCA"
+        _if_present mca_MCA_present
+        printf ", control MSR"
+        _if_present mca_control_MSR_present
+        printf ", threshold status"
+        _if_present mca_threshold_status_present
+        printf "\n%d error banks, ", mca_error_bank_count
+        printf "family code 0x%x, ", mca_family
+        printf "machine-check exception taken: %d\n", mca_exception_taken
+        set $kgm_cpu = 0
+        while cpu_data_ptr[$kgm_cpu] != 0
+            set $kgm_mcp = cpu_data_ptr[$kgm_cpu]->cpu_mca_state
+            if $kgm_mcp
+                printf "CPU %d:", $kgm_cpu
+                printf " mca_mcg_ctl: 0x%016llx", $kgm_mcp->mca_mcg_ctl
+                printf " mca_mcg_status: 0x%016llx\n", $kgm_mcp->mca_mcg_status.u64
+                printf "bank   "
+                printf "mca_mci_ctl        "
+                printf "mca_mci_status     "
+                printf "mca_mci_addr       "
+                printf "mca_mci_misc\n"
+                set $kgm_bank = 0
+                while $kgm_bank < mca_error_bank_count
+                    set $kgm_bp = &$kgm_mcp->mca_error_bank[$kgm_bank]
+                    printf " %2d:", $kgm_bank
+                    printf " 0x%016llx", $kgm_bp->mca_mci_ctl
+                    printf " 0x%016llx", $kgm_bp->mca_mci_status.u64
+                    printf " 0x%016llx", $kgm_bp->mca_mci_addr
+                    printf " 0x%016llx\n", $kgm_bp->mca_mci_misc
+                    set $kgm_bank = $kgm_bank + 1
+                end
+            end
+            set $kgm_cpu = $kgm_cpu + 1
+        end
+    end
+end
+
+document showMCAstate
+Syntax: showMCAstate
+| Print machine-check register state after MC exception.
+end
diff --git a/libkern/Makefile b/libkern/Makefile
index eca7e6882..3f1d09b7a 100644
--- a/libkern/Makefile
+++ b/libkern/Makefile
@@ -3,11 +3,6 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
-
-export COMP_LDFLAGS_COMPONENT_PPC = -Wl,-i_OSCompareAndSwap:_hw_compare_and_store \
-                                    -Wl,-i_OSDequeueAtomic:_hw_dequeue_atomic \
-                                    -Wl,-i_OSEnqueueAtomic:_hw_queue_atomic
-
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
@@ -19,6 +14,9 @@ INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS}
 
 INSTINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS}
 
+INSTINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS}
+
+
 EXPINC_SUBDIRS = \
 	libkern \
 	uuid
@@ -27,6 +25,8 @@ EXPINC_SUBDIRS_PPC = ${EXPINC_SUBDIRS}
 
 EXPINC_SUBDIRS_I386 = ${EXPINC_SUBDIRS}
 
+EXPINC_SUBDIRS_ARM = ${EXPINC_SUBDIRS}
+
 SETUP_SUBDIRS = conf
 
 COMP_SUBDIRS = conf
diff --git a/libkern/c++/OSArray.cpp b/libkern/c++/OSArray.cpp
index 6531060aa..c1b7ff66c 100644
--- a/libkern/c++/OSArray.cpp
+++ b/libkern/c++/OSArray.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOArray.m created by rsulack on Fri 12-Sep-1997 */
 /* IOArray.cpp converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/c++/OSBoolean.cpp b/libkern/c++/OSBoolean.cpp
index 42ddc73d2..732ce17a3 100644
--- a/libkern/c++/OSBoolean.cpp
+++ b/libkern/c++/OSBoolean.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSBoolean.cpp created by rsulack on Tue Oct 12 1999 */
 
diff --git a/libkern/c++/OSCPPDebug.cpp b/libkern/c++/OSCPPDebug.cpp
index 5cf9649d8..603ce7641 100644
--- a/libkern/c++/OSCPPDebug.cpp
+++ b/libkern/c++/OSCPPDebug.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <libkern/c++/OSCPPDebug.cpp>
diff --git a/libkern/c++/OSCollection.cpp b/libkern/c++/OSCollection.cpp
index ff54175ee..dc27d1d87 100644
--- a/libkern/c++/OSCollection.cpp
+++ b/libkern/c++/OSCollection.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOArray.h created by rsulack on Thu 11-Sep-1997 */
 
diff --git a/libkern/c++/OSCollectionIterator.cpp b/libkern/c++/OSCollectionIterator.cpp
index 8302a2d22..d44e0d500 100644
--- a/libkern/c++/OSCollectionIterator.cpp
+++ b/libkern/c++/OSCollectionIterator.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOArray.h created by rsulack on Thu 11-Sep-1997 */
 
diff --git a/libkern/c++/OSData.cpp b/libkern/c++/OSData.cpp
index 4804c8798..47def143b 100644
--- a/libkern/c++/OSData.cpp
+++ b/libkern/c++/OSData.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOData.m created by rsulack on Thu 25-Sep-1997 */
 
diff --git a/libkern/c++/OSDictionary.cpp b/libkern/c++/OSDictionary.cpp
index eb177a56d..d09743c1d 100644
--- a/libkern/c++/OSDictionary.cpp
+++ b/libkern/c++/OSDictionary.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSDictionary.m created by rsulack on Fri 12-Sep-1997 */
 /* OSDictionary.cpp converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/c++/OSIterator.cpp b/libkern/c++/OSIterator.cpp
index b462c88c6..dac01d78d 100644
--- a/libkern/c++/OSIterator.cpp
+++ b/libkern/c++/OSIterator.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/libkern/c++/OSMetaClass.cpp b/libkern/c++/OSMetaClass.cpp
index b2303e84d..f8ac932d7 100644
--- a/libkern/c++/OSMetaClass.cpp
+++ b/libkern/c++/OSMetaClass.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSMetaClass.cpp created by gvdl on Fri 1998-11-17 */
 
@@ -328,13 +334,13 @@ OSMetaClass::~OSMetaClass()
     }
 }
 
-void *OSMetaClass::operator new(size_t size) { return 0; }
+void *OSMetaClass::operator new(__unused size_t size) { return 0; }
 void OSMetaClass::retain() const { }
 void OSMetaClass::release() const { }
-void OSMetaClass::release(int when) const { }
-void OSMetaClass::taggedRetain(const void *tag) const { }
-void OSMetaClass::taggedRelease(const void *tag) const { }
-void OSMetaClass::taggedRelease(const void *tag, const int when) const { }
+void OSMetaClass::release(__unused int when) const { }
+void OSMetaClass::taggedRetain(__unused const void *tag) const { }
+void OSMetaClass::taggedRelease(__unused const void *tag) const { }
+void OSMetaClass::taggedRelease(__unused const void *tag, __unused const int when) const { }
 int  OSMetaClass::getRetainCount() const { return 0; }
 
 const char *OSMetaClass::getClassName() const
@@ -593,8 +599,8 @@ IOReturn OSMetaClassSystemSleepOrWake(UInt32 messageType)
 
 extern "C" kern_return_t kmod_unload_cache(void);
 
-static void _OSMetaClassConsiderUnloads(thread_call_param_t p0,
-                                        thread_call_param_t p1)
+static void _OSMetaClassConsiderUnloads(__unused thread_call_param_t p0,
+                                        __unused thread_call_param_t p1)
 {
     OSSet *kmodClasses;
     OSSymbol *kmodName;
@@ -855,7 +861,7 @@ OSDictionary * OSMetaClass::getClassDictionary()
     return 0;
 }
 
-bool OSMetaClass::serialize(OSSerialize *s) const
+bool OSMetaClass::serialize(__unused OSSerialize *s) const
 {
     panic("OSMetaClass::serialize(): Obsoleted\n");
     return false;
diff --git a/libkern/c++/OSNumber.cpp b/libkern/c++/OSNumber.cpp
index 99b309c7d..87f96c780 100644
--- a/libkern/c++/OSNumber.cpp
+++ b/libkern/c++/OSNumber.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOOffset.m created by rsulack on Wed 17-Sep-1997 */
 
@@ -145,15 +151,15 @@ bool OSNumber::serialize(OSSerialize *s) const
     
     if (s->previouslySerialized(this)) return true;
 
-    sprintf(temp, "integer size=\"%d\"", size); 
+    snprintf(temp, sizeof(temp), "integer size=\"%d\"", size); 
     if (!s->addXMLStartTag(this, temp)) return false;
     
     //XXX    sprintf(temp, "0x%qx", value);
     if ((value >> 32)) {
-        sprintf(temp, "0x%lx%08lx", (unsigned long)(value >> 32),
+        snprintf(temp, sizeof(temp), "0x%lx%08lx", (unsigned long)(value >> 32),
                     (unsigned long)(value & 0xFFFFFFFF));
     } else { 
-        sprintf(temp, "0x%lx", (unsigned long)value);
+        snprintf(temp, sizeof(temp), "0x%lx", (unsigned long)value);
     }
     if (!s->addString(temp)) return false;
 
diff --git a/libkern/c++/OSObject.cpp b/libkern/c++/OSObject.cpp
index 503b9504d..f92783f84 100644
--- a/libkern/c++/OSObject.cpp
+++ b/libkern/c++/OSObject.cpp
@@ -1,34 +1,45 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSObject.cpp created by gvdl on Fri 1998-11-17 */
 
 #include <libkern/c++/OSObject.h>
+#include <libkern/c++/OSArray.h>
 #include <libkern/c++/OSSerialize.h>
 #include <libkern/c++/OSLib.h>
+#include <libkern/OSDebug.h>
 #include <libkern/c++/OSCPPDebug.h>
+#include <IOKit/IOKitDebug.h>
 #include <libkern/OSAtomic.h>
 
 #include <libkern/c++/OSCollection.h>
 
+#include <kern/queue.h>
+
 __BEGIN_DECLS
 int debug_ivars_size;
 __END_DECLS
@@ -261,20 +272,91 @@ bool OSObject::serialize(OSSerialize *s) const
     return s->addXMLEndTag("string");
 }
 
+
+thread_t gOSObjectTrackThread;
+
+queue_head_t gOSObjectTrackList =
+    { (queue_t) &gOSObjectTrackList, (queue_t) &gOSObjectTrackList };
+
+lck_spin_t gOSObjectTrackLock;
+
+OSArray * OSFlushObjectTrackList(void)
+{
+    OSArray *     array;
+    queue_entry_t next;
+
+    array = OSArray::withCapacity(16);
+
+    lck_spin_lock(&gOSObjectTrackLock);
+    while (!queue_empty(&gOSObjectTrackList))
+    {
+	next = queue_first(&gOSObjectTrackList);
+	remque(next);
+	lck_spin_unlock(&gOSObjectTrackLock);
+	array->setObject((OSObject *) (next + 1));
+	lck_spin_lock(&gOSObjectTrackLock);
+    }
+    lck_spin_unlock(&gOSObjectTrackLock);
+
+    return (array);
+}
+
+struct OSObjectTracking
+{
+    queue_chain_t link;
+    void *	  bt[14];
+};
+
 void *OSObject::operator new(size_t size)
 {
-    void *mem = (void *) kalloc(size);
+    size_t tracking        = (gIOKitDebug & kOSTraceObjectAlloc) 
+			   ? sizeof(OSObjectTracking) : 0;
+    OSObjectTracking * mem = (OSObjectTracking *) kalloc(size + tracking);
+
     assert(mem);
+
+    if (tracking)
+    {
+	if ((((thread_t) 1) == gOSObjectTrackThread) || (current_thread() == gOSObjectTrackThread))
+	{
+	    (void) OSBacktrace(&mem->bt[0], sizeof(mem->bt) / sizeof(mem->bt[0]));
+	    lck_spin_lock(&gOSObjectTrackLock);
+	    enqueue_tail(&gOSObjectTrackList, &mem->link);
+	    lck_spin_unlock(&gOSObjectTrackLock);
+	}
+	else
+	    mem->link.next = 0;
+	mem++;
+    }
+
     bzero(mem, size);
 
     ACCUMSIZE(size);
 
-    return mem;
+    return (void *) mem;
 }
 
-void OSObject::operator delete(void *mem, size_t size)
+void OSObject::operator delete(void *_mem, size_t size)
 {
-    kfree(mem, size);
+    size_t             tracking = (gIOKitDebug & kOSTraceObjectAlloc)
+				? sizeof(OSObjectTracking) : 0;
+    OSObjectTracking * mem      = (OSObjectTracking *) _mem;
+
+    if (!mem)
+	return;
+
+    if (tracking)
+    {
+	mem--;
+	if (mem->link.next)
+	{
+	    lck_spin_lock(&gOSObjectTrackLock);
+	    remque(&mem->link);
+	    lck_spin_unlock(&gOSObjectTrackLock);
+	}
+    }
+
+    kfree(mem, size + tracking);
 
     ACCUMSIZE(-size);
 }
diff --git a/libkern/c++/OSObjectAsm.s b/libkern/c++/OSObjectAsm.s
index c2cf06dd8..eba1bc781 100644
--- a/libkern/c++/OSObjectAsm.s
+++ b/libkern/c++/OSObjectAsm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #if __GNUC__ >= 3
diff --git a/libkern/c++/OSOrderedSet.cpp b/libkern/c++/OSOrderedSet.cpp
index 34b07fbce..cabf763bb 100644
--- a/libkern/c++/OSOrderedSet.cpp
+++ b/libkern/c++/OSOrderedSet.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <libkern/c++/OSDictionary.h>
diff --git a/libkern/c++/OSRuntime.cpp b/libkern/c++/OSRuntime.cpp
index 3f24b8ee2..ecbaaaadf 100644
--- a/libkern/c++/OSRuntime.cpp
+++ b/libkern/c++/OSRuntime.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1997 Apple Computer, Inc.
@@ -316,8 +322,14 @@ kern_return_t OSRuntimeInitializeCPP(kmod_info_t *ki, void *)
 }
 
 static KMOD_LIB_DECL(__kernel__, 0);
+
+extern lck_spin_t  gOSObjectTrackLock;
+extern lck_grp_t * IOLockGroup;
+
 void OSlibkernInit(void)
 {
+    lck_spin_init(&gOSObjectTrackLock, IOLockGroup, LCK_ATTR_NULL);
+
     vm_address_t *headerArray = (vm_address_t *) getmachheaders();
 
     KMOD_INFO_NAME.address = headerArray[0]; assert(!headerArray[1]);
diff --git a/libkern/c++/OSSerialize.cpp b/libkern/c++/OSSerialize.cpp
index 88677c33e..6db4c90ee 100644
--- a/libkern/c++/OSSerialize.cpp
+++ b/libkern/c++/OSSerialize.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSSerialize.cpp created by rsulack on Wen 25-Nov-1998 */
 
@@ -82,7 +88,7 @@ bool OSSerialize::previouslySerialized(const OSMetaClassBase *o)
 	}
 
 	// build a tag
-	sprintf(temp, "%u", tag++);
+	snprintf(temp, sizeof(temp), "%u", tag++);
 	tagString = OSString::withCString(temp);
 
 	// add to tag dictionary
diff --git a/libkern/c++/OSSet.cpp b/libkern/c++/OSSet.cpp
index c09e49350..3aeb91037 100644
--- a/libkern/c++/OSSet.cpp
+++ b/libkern/c++/OSSet.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOSet.m created by rsulack on Thu 11-Jun-1998 */
 
diff --git a/libkern/c++/OSString.cpp b/libkern/c++/OSString.cpp
index fa7ed45ae..4b2cd6da4 100644
--- a/libkern/c++/OSString.cpp
+++ b/libkern/c++/OSString.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOString.m created by rsulack on Wed 17-Sep-1997 */
 /* IOString.cpp converted to C++ on Tue 1998-9-22 */
diff --git a/libkern/c++/OSSymbol.cpp b/libkern/c++/OSSymbol.cpp
index 039c9eec4..a5852153b 100644
--- a/libkern/c++/OSSymbol.cpp
+++ b/libkern/c++/OSSymbol.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOSymbol.cpp created by gvdl on Fri 1998-11-17 */
 
@@ -45,6 +51,24 @@ extern "C" {
 #define ACCUMSIZE(s)
 #endif
 
+#define INITIAL_POOL_SIZE  (exp2ml(1 + log2(kInitBucketCount)))
+
+#define GROW_FACTOR   (1)
+#define SHRINK_FACTOR (3)
+
+#define GROW_POOL()     do \
+    if (count * GROW_FACTOR > nBuckets) { \
+        reconstructSymbols(true); \
+    } \
+while (0)
+
+#define SHRINK_POOL()     do \
+    if (count * SHRINK_FACTOR < nBuckets && \
+        nBuckets > INITIAL_POOL_SIZE) { \
+        reconstructSymbols(false); \
+    } \
+while (0)
+
 class OSSymbolPool
 {
 private:
@@ -78,7 +102,8 @@ private:
     static unsigned long log2(unsigned int x);
     static unsigned long exp2ml(unsigned int x);
 
-    void reconstructSymbols();
+    void reconstructSymbols(void);
+    void reconstructSymbols(bool grow);
 
 public:
     static void *operator new(size_t size);
@@ -120,7 +145,7 @@ void OSSymbolPool::operator delete(void *mem, size_t size)
 bool OSSymbolPool::init()
 {
     count = 0;
-    nBuckets = exp2ml(1 + log2(kInitBucketCount));
+    nBuckets = INITIAL_POOL_SIZE;
     buckets = (Bucket *) kalloc(nBuckets * sizeof(Bucket));
     ACCUMSIZE(nBuckets * sizeof(Bucket));
     if (!buckets)
@@ -192,14 +217,35 @@ OSSymbol *OSSymbolPool::nextHashState(OSSymbolPoolState *stateP)
         return thisBucket->symbolP[stateP->j];
 }
 
-void OSSymbolPool::reconstructSymbols()
+void OSSymbolPool::reconstructSymbols(void)
 {
-    OSSymbolPool old(this);
+    this->reconstructSymbols(true);
+}
+
+void OSSymbolPool::reconstructSymbols(bool grow)
+{
+    unsigned int new_nBuckets = nBuckets;
     OSSymbol *insert;
     OSSymbolPoolState state;
 
-    nBuckets += nBuckets + 1;
+    if (grow) {
+        new_nBuckets += new_nBuckets + 1;
+    } else {
+       /* Don't shrink the pool below the default initial size.
+        */
+        if (nBuckets <= INITIAL_POOL_SIZE) {
+            return;
+        }
+        new_nBuckets = (new_nBuckets - 1) / 2;
+    }
+
+   /* Create old pool to iterate after doing above check, cause it
+    * gets finalized at return.
+    */
+    OSSymbolPool old(this);
+
     count = 0;
+    nBuckets = new_nBuckets;
     buckets = (Bucket *) kalloc(nBuckets * sizeof(Bucket));
     ACCUMSIZE(nBuckets * sizeof(Bucket));
     /* @@@ gvdl: Zero test and panic if can't set up pool */
@@ -275,8 +321,7 @@ OSSymbol *OSSymbolPool::insertSymbol(OSSymbol *sym)
         thisBucket->symbolP = list;
         thisBucket->count++;
         count++;
-        if (count > nBuckets)
-            reconstructSymbols();
+        GROW_POOL();
 
         return sym;
     }
@@ -298,8 +343,7 @@ OSSymbol *OSSymbolPool::insertSymbol(OSSymbol *sym)
     kfree(thisBucket->symbolP, j * sizeof(OSSymbol *));
     ACCUMSIZE(-(j * sizeof(OSSymbol *)));
     thisBucket->symbolP = list;
-    if (count > nBuckets)
-        reconstructSymbols();
+    GROW_POOL();
 
     return sym;
 }
@@ -325,6 +369,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
             thisBucket->symbolP = 0;
             count--;
             thisBucket->count--;
+            SHRINK_POOL();
             return;
         }
         return;
@@ -338,6 +383,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
 	    ACCUMSIZE(-(2 * sizeof(OSSymbol *)));
             count--;
             thisBucket->count--;
+            SHRINK_POOL();
             return;
         }
 
@@ -348,6 +394,7 @@ void OSSymbolPool::removeSymbol(OSSymbol *sym)
 	    ACCUMSIZE(-(2 * sizeof(OSSymbol *)));
             count--;
             thisBucket->count--;
+            SHRINK_POOL();
             return;
         }
         return;
diff --git a/libkern/c++/OSUnserialize.cpp b/libkern/c++/OSUnserialize.cpp
index db1237eb1..4568e6f8a 100644
--- a/libkern/c++/OSUnserialize.cpp
+++ b/libkern/c++/OSUnserialize.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*  OSUnserialize.y created by rsulack on Nov 21 1998 */
@@ -312,7 +318,7 @@ static const short yycheck[] = {     0,
 #define YYSTACK_USE_ALLOCA
 #define alloca __builtin_alloca
 #else /* not GNU C.  */
-#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386))
+#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386)) || defined (__arm)
 #define YYSTACK_USE_ALLOCA
 #include <alloca.h>
 #else /* not sparc */
@@ -1005,7 +1011,7 @@ yyerrlab:   /* here on detecting error */
 	{
 	  int size = 0;
 	  char *msg;
-	  int x, count;
+	  int x, count, len;
 
 	  count = 0;
 	  /* Start X at -yyn if nec to avoid negative indexes in yycheck.  */
@@ -1013,10 +1019,11 @@ yyerrlab:   /* here on detecting error */
 	       x < (sizeof(yytname) / sizeof(char *)); x++)
 	    if (yycheck[x + yyn] == x)
 	      size += strlen(yytname[x]) + 15, count++;
-	  msg = (char *) malloc(size + 15);
+	  len = size + 15;
+	  msg = (char *) malloc(len);
 	  if (msg != 0)
 	    {
-	      strcpy(msg, "parse error");
+	      strlcpy(msg, "parse error", len);
 
 	      if (count < 5)
 		{
@@ -1025,9 +1032,10 @@ yyerrlab:   /* here on detecting error */
 		       x < (sizeof(yytname) / sizeof(char *)); x++)
 		    if (yycheck[x + yyn] == x)
 		      {
-			strcat(msg, count == 0 ? ", expecting `" : " or `");
-			strcat(msg, yytname[x]);
-			strcat(msg, "'");
+			strlcat(msg, count == 0 ? ", expecting `" : " or `",
+					len);
+			strlcat(msg, yytname[x], len);
+			strlcat(msg, "'", len);
 			count++;
 		      }
 		}
diff --git a/libkern/c++/OSUnserialize.y b/libkern/c++/OSUnserialize.y
index 47da0c284..34b2ee963 100644
--- a/libkern/c++/OSUnserialize.y
+++ b/libkern/c++/OSUnserialize.y
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*  OSUnserialize.y created by rsulack on Nov 21 1998 */
diff --git a/libkern/c++/OSUnserializeXML.cpp b/libkern/c++/OSUnserializeXML.cpp
index 068e91f61..e543deec5 100644
--- a/libkern/c++/OSUnserializeXML.cpp
+++ b/libkern/c++/OSUnserializeXML.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -42,37 +48,111 @@
 //		 DO NOT EDIT OSUnserializeXML.cpp!
 //
 //			this means you!
-//
-//
-//
-//
-//
-//
+/* A Bison parser, made by GNU Bison 2.3.  */
+
+/* Skeleton implementation for Bison's Yacc-like parsers in C
 
-/*  A Bison parser, made from OSUnserializeXML.y
-    by GNU Bison version 1.28  */
+   Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+   Free Software Foundation, Inc.
 
-#define YYBISON 1  /* Identify Bison output.  */
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
 
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
+
+/* As a special exception, you may create a larger work that contains
+   part or all of the Bison parser skeleton and distribute that work
+   under terms of your choice, so long as that work isn't itself a
+   parser generator using the skeleton or a modified version thereof
+   as a parser skeleton.  Alternatively, if you modify or redistribute
+   the parser skeleton itself, you may (at your option) remove this
+   special exception, which will cause the skeleton and the resulting
+   Bison output files to be licensed under the GNU General Public
+   License without this special exception.
+
+   This special exception was added by the Free Software Foundation in
+   version 2.2 of Bison.  */
+
+/* C LALR(1) parser skeleton written by Richard Stallman, by
+   simplifying the original so-called "semantic" parser.  */
+
+/* All symbols defined below should begin with yy or YY, to avoid
+   infringing on user name space.  This should be done even for local
+   variables, as they might otherwise be expanded by user macros.
+   There are some unavoidable exceptions within include files to
+   define necessary library symbols; they are noted "INFRINGES ON
+   USER NAME SPACE" below.  */
+
+/* Identify Bison output.  */
+#define YYBISON 1
+
+/* Bison version.  */
+#define YYBISON_VERSION "2.3"
+
+/* Skeleton name.  */
+#define YYSKELETON_NAME "yacc.c"
+
+/* Pure parsers.  */
+#define YYPURE 1
+
+/* Using locations.  */
+#define YYLSP_NEEDED 0
+
+/* Substitute the variable and function names.  */
 #define yyparse OSUnserializeXMLparse
-#define yylex OSUnserializeXMLlex
+#define yylex   OSUnserializeXMLlex
 #define yyerror OSUnserializeXMLerror
-#define yylval OSUnserializeXMLlval
-#define yychar OSUnserializeXMLchar
+#define yylval  OSUnserializeXMLlval
+#define yychar  OSUnserializeXMLchar
 #define yydebug OSUnserializeXMLdebug
 #define yynerrs OSUnserializeXMLnerrs
-#define	ARRAY	257
-#define	BOOLEAN	258
-#define	DATA	259
-#define	DICTIONARY	260
-#define	IDREF	261
-#define	KEY	262
-#define	NUMBER	263
-#define	SET	264
-#define	STRING	265
-#define	SYNTAX_ERROR	266
-
-#line 55 "OSUnserializeXML.y"
+
+
+/* Tokens.  */
+#ifndef YYTOKENTYPE
+# define YYTOKENTYPE
+   /* Put the tokens into the symbol table, so that GDB and other debuggers
+      know about them.  */
+   enum yytokentype {
+     ARRAY = 258,
+     BOOLEAN = 259,
+     DATA = 260,
+     DICTIONARY = 261,
+     IDREF = 262,
+     KEY = 263,
+     NUMBER = 264,
+     SET = 265,
+     STRING = 266,
+     SYNTAX_ERROR = 267
+   };
+#endif
+/* Tokens.  */
+#define ARRAY 258
+#define BOOLEAN 259
+#define DATA 260
+#define DICTIONARY 261
+#define IDREF 262
+#define KEY 263
+#define NUMBER 264
+#define SET 265
+#define STRING 266
+#define SYNTAX_ERROR 267
+
+
+
+
+/* Copy the first part of user declarations.  */
+#line 61 "OSUnserializeXML.y"
 
 #include <string.h>
 #include <libkern/c++/OSMetaClass.h>
@@ -149,460 +229,1076 @@ extern unsigned long	strtoul(const char *, char **, int);
 #define realloc(a, s) kern_os_realloc(a, s)
 #define free(a) kern_os_free((void *)a)
 
-#ifndef YYSTYPE
-#define YYSTYPE int
+
+
+/* Enabling traces.  */
+#ifndef YYDEBUG
+# define YYDEBUG 0
 #endif
 
+/* Enabling verbose error messages.  */
+#ifdef YYERROR_VERBOSE
+# undef YYERROR_VERBOSE
+# define YYERROR_VERBOSE 1
+#else
+# define YYERROR_VERBOSE 0
+#endif
 
-#ifndef __cplusplus
-#ifndef __STDC__
-#define const
+/* Enabling the token table.  */
+#ifndef YYTOKEN_TABLE
+# define YYTOKEN_TABLE 0
 #endif
+
+#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
+typedef int YYSTYPE;
+# define yystype YYSTYPE /* obsolescent; will be withdrawn */
+# define YYSTYPE_IS_DECLARED 1
+# define YYSTYPE_IS_TRIVIAL 1
 #endif
 
 
 
-#define	YYFINAL		40
-#define	YYFLAG		-32768
-#define	YYNTBASE	19
-
-#define YYTRANSLATE(x) ((unsigned)(x) <= 266 ? yytranslate[x] : 33)
-
-static const char yytranslate[] = {     0,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,    15,
-    16,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-    17,     2,    18,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,    13,     2,    14,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
-     2,     2,     2,     2,     2,     1,     3,     4,     5,     6,
-     7,     8,     9,    10,    11,    12
-};
+/* Copy the second part of user declarations.  */
 
-#if YYDEBUG != 0
-static const short yyprhs[] = {     0,
-     0,     1,     3,     5,     7,     9,    11,    13,    15,    17,
-    19,    21,    24,    28,    30,    32,    35,    38,    40,    43,
-    47,    49,    52,    56,    58,    60,    63,    65,    67,    69,
-    71
-};
 
-static const short yyrhs[] = {    -1,
-    20,     0,    12,     0,    21,     0,    25,     0,    26,     0,
-    32,     0,    29,     0,    31,     0,    28,     0,    30,     0,
-    13,    14,     0,    13,    22,    14,     0,     6,     0,    23,
-     0,    22,    23,     0,    24,    20,     0,     8,     0,    15,
-    16,     0,    15,    27,    16,     0,     3,     0,    17,    18,
-     0,    17,    27,    18,     0,    10,     0,    20,     0,    27,
-    20,     0,     4,     0,     5,     0,     7,     0,     9,     0,
-    11,     0
-};
+/* Line 216 of yacc.c.  */
+#line 216 "OSUnserializeXML.tab.c"
 
+#ifdef short
+# undef short
 #endif
 
-#if YYDEBUG != 0
-static const short yyrline[] = { 0,
-   144,   147,   152,   157,   158,   159,   160,   161,   162,   163,
-   164,   177,   180,   183,   186,   187,   192,   201,   206,   209,
-   212,   215,   218,   221,   224,   227,   234,   237,   240,   243,
-   246
-};
+#ifdef YYTYPE_UINT8
+typedef YYTYPE_UINT8 yytype_uint8;
+#else
+typedef unsigned char yytype_uint8;
 #endif
 
+#ifdef YYTYPE_INT8
+typedef YYTYPE_INT8 yytype_int8;
+#elif (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+typedef signed char yytype_int8;
+#else
+typedef short int yytype_int8;
+#endif
 
-#if YYDEBUG != 0 || defined (YYERROR_VERBOSE)
+#ifdef YYTYPE_UINT16
+typedef YYTYPE_UINT16 yytype_uint16;
+#else
+typedef unsigned short int yytype_uint16;
+#endif
 
-static const char * const yytname[] = {   "$","error","$undefined.","ARRAY",
-"BOOLEAN","DATA","DICTIONARY","IDREF","KEY","NUMBER","SET","STRING","SYNTAX_ERROR",
-"'{'","'}'","'('","')'","'['","']'","input","object","dict","pairs","pair","key",
-"array","set","elements","boolean","data","idref","number","string", NULL
-};
+#ifdef YYTYPE_INT16
+typedef YYTYPE_INT16 yytype_int16;
+#else
+typedef short int yytype_int16;
 #endif
 
-static const short yyr1[] = {     0,
-    19,    19,    19,    20,    20,    20,    20,    20,    20,    20,
-    20,    21,    21,    21,    22,    22,    23,    24,    25,    25,
-    25,    26,    26,    26,    27,    27,    28,    29,    30,    31,
-    32
-};
+#ifndef YYSIZE_T
+# ifdef __SIZE_TYPE__
+#  define YYSIZE_T __SIZE_TYPE__
+# elif defined size_t
+#  define YYSIZE_T size_t
+# elif ! defined YYSIZE_T && (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+#  include <stddef.h> /* INFRINGES ON USER NAME SPACE */
+#  define YYSIZE_T size_t
+# else
+#  define YYSIZE_T unsigned int
+# endif
+#endif
 
-static const short yyr2[] = {     0,
-     0,     1,     1,     1,     1,     1,     1,     1,     1,     1,
-     1,     2,     3,     1,     1,     2,     2,     1,     2,     3,
-     1,     2,     3,     1,     1,     2,     1,     1,     1,     1,
-     1
-};
+#define YYSIZE_MAXIMUM ((YYSIZE_T) -1)
+
+#ifndef YY_
+# if YYENABLE_NLS
+#  if ENABLE_NLS
+#   include <libintl.h> /* INFRINGES ON USER NAME SPACE */
+#   define YY_(msgid) dgettext ("bison-runtime", msgid)
+#  endif
+# endif
+# ifndef YY_
+#  define YY_(msgid) msgid
+# endif
+#endif
+
+/* Suppress unused-variable warnings by "using" E.  */
+#if ! defined lint || defined __GNUC__
+# define YYUSE(e) ((void) (e))
+#else
+# define YYUSE(e) /* empty */
+#endif
 
-static const short yydefact[] = {     1,
-    21,    27,    28,    14,    29,    30,    24,    31,     3,     0,
-     0,     0,     2,     4,     5,     6,    10,     8,    11,     9,
-     7,    18,    12,     0,    15,     0,    19,    25,     0,    22,
-     0,    13,    16,    17,    20,    26,    23,     0,     0,     0
+/* Identity function, used to suppress warnings about constant conditions.  */
+#ifndef lint
+# define YYID(n) (n)
+#else
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static int
+YYID (int i)
+#else
+static int
+YYID (i)
+    int i;
+#endif
+{
+  return i;
+}
+#endif
+
+#if ! defined yyoverflow || YYERROR_VERBOSE
+
+/* The parser invokes alloca or malloc; define the necessary symbols.  */
+
+# ifdef YYSTACK_USE_ALLOCA
+#  if YYSTACK_USE_ALLOCA
+#   ifdef __GNUC__
+#    define YYSTACK_ALLOC __builtin_alloca
+#   elif defined __BUILTIN_VA_ARG_INCR
+#    include <alloca.h> /* INFRINGES ON USER NAME SPACE */
+#   elif defined _AIX
+#    define YYSTACK_ALLOC __alloca
+#   elif defined _MSC_VER
+#    include <malloc.h> /* INFRINGES ON USER NAME SPACE */
+#    define alloca _alloca
+#   else
+#    define YYSTACK_ALLOC alloca
+#    if ! defined _ALLOCA_H && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+#     include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
+#     ifndef _STDLIB_H
+#      define _STDLIB_H 1
+#     endif
+#    endif
+#   endif
+#  endif
+# endif
+
+# ifdef YYSTACK_ALLOC
+   /* Pacify GCC's `empty if-body' warning.  */
+#  define YYSTACK_FREE(Ptr) do { /* empty */; } while (YYID (0))
+#  ifndef YYSTACK_ALLOC_MAXIMUM
+    /* The OS might guarantee only one guard page at the bottom of the stack,
+       and a page size can be as small as 4096 bytes.  So we cannot safely
+       invoke alloca (N) if N exceeds 4096.  Use a slightly smaller number
+       to allow for a few compiler-allocated temporary stack slots.  */
+#   define YYSTACK_ALLOC_MAXIMUM 4032 /* reasonable circa 2006 */
+#  endif
+# else
+#  define YYSTACK_ALLOC YYMALLOC
+#  define YYSTACK_FREE YYFREE
+#  ifndef YYSTACK_ALLOC_MAXIMUM
+#   define YYSTACK_ALLOC_MAXIMUM YYSIZE_MAXIMUM
+#  endif
+#  if (defined __cplusplus && ! defined _STDLIB_H \
+       && ! ((defined YYMALLOC || defined malloc) \
+	     && (defined YYFREE || defined free)))
+#   include <stdlib.h> /* INFRINGES ON USER NAME SPACE */
+#   ifndef _STDLIB_H
+#    define _STDLIB_H 1
+#   endif
+#  endif
+#  ifndef YYMALLOC
+#   define YYMALLOC malloc
+#   if ! defined malloc && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+void *malloc (YYSIZE_T); /* INFRINGES ON USER NAME SPACE */
+#   endif
+#  endif
+#  ifndef YYFREE
+#   define YYFREE free
+#   if ! defined free && ! defined _STDLIB_H && (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+void free (void *); /* INFRINGES ON USER NAME SPACE */
+#   endif
+#  endif
+# endif
+#endif /* ! defined yyoverflow || YYERROR_VERBOSE */
+
+
+#if (! defined yyoverflow \
+     && (! defined __cplusplus \
+	 || (defined YYSTYPE_IS_TRIVIAL && YYSTYPE_IS_TRIVIAL)))
+
+/* A type that is properly aligned for any stack member.  */
+union yyalloc
+{
+  yytype_int16 yyss;
+  YYSTYPE yyvs;
+  };
+
+/* The size of the maximum gap between one aligned stack and the next.  */
+# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1)
+
+/* The size of an array large to enough to hold all stacks, each with
+   N elements.  */
+# define YYSTACK_BYTES(N) \
+     ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \
+      + YYSTACK_GAP_MAXIMUM)
+
+/* Copy COUNT objects from FROM to TO.  The source and destination do
+   not overlap.  */
+# ifndef YYCOPY
+#  if defined __GNUC__ && 1 < __GNUC__
+#   define YYCOPY(To, From, Count) \
+      __builtin_memcpy (To, From, (Count) * sizeof (*(From)))
+#  else
+#   define YYCOPY(To, From, Count)		\
+      do					\
+	{					\
+	  YYSIZE_T yyi;				\
+	  for (yyi = 0; yyi < (Count); yyi++)	\
+	    (To)[yyi] = (From)[yyi];		\
+	}					\
+      while (YYID (0))
+#  endif
+# endif
+
+/* Relocate STACK from its old location to the new one.  The
+   local variables YYSIZE and YYSTACKSIZE give the old and new number of
+   elements in the stack, and YYPTR gives the new location of the
+   stack.  Advance YYPTR to a properly aligned location for the next
+   stack.  */
+# define YYSTACK_RELOCATE(Stack)					\
+    do									\
+      {									\
+	YYSIZE_T yynewbytes;						\
+	YYCOPY (&yyptr->Stack, Stack, yysize);				\
+	Stack = &yyptr->Stack;						\
+	yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \
+	yyptr += yynewbytes / sizeof (*yyptr);				\
+      }									\
+    while (YYID (0))
+
+#endif
+
+/* YYFINAL -- State number of the termination state.  */
+#define YYFINAL  33
+/* YYLAST -- Last index in YYTABLE.  */
+#define YYLAST   108
+
+/* YYNTOKENS -- Number of terminals.  */
+#define YYNTOKENS  19
+/* YYNNTS -- Number of nonterminals.  */
+#define YYNNTS  15
+/* YYNRULES -- Number of rules.  */
+#define YYNRULES  32
+/* YYNRULES -- Number of states.  */
+#define YYNSTATES  40
+
+/* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX.  */
+#define YYUNDEFTOK  2
+#define YYMAXUTOK   267
+
+#define YYTRANSLATE(YYX)						\
+  ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK)
+
+/* YYTRANSLATE[YYLEX] -- Bison symbol number corresponding to YYLEX.  */
+static const yytype_uint8 yytranslate[] =
+{
+       0,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+      15,    16,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,    17,     2,    18,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,    13,     2,    14,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     2,     2,     2,     2,
+       2,     2,     2,     2,     2,     2,     1,     2,     3,     4,
+       5,     6,     7,     8,     9,    10,    11,    12
 };
 
-static const short yydefgoto[] = {    38,
-    28,    14,    24,    25,    26,    15,    16,    29,    17,    18,
-    19,    20,    21
+#if YYDEBUG
+/* YYPRHS[YYN] -- Index of the first RHS symbol of rule number YYN in
+   YYRHS.  */
+static const yytype_uint8 yyprhs[] =
+{
+       0,     0,     3,     4,     6,     8,    10,    12,    14,    16,
+      18,    20,    22,    24,    27,    31,    33,    35,    38,    41,
+      43,    46,    50,    52,    55,    59,    61,    63,    66,    68,
+      70,    72,    74
 };
 
-static const short yypact[] = {    45,
--32768,-32768,-32768,-32768,-32768,-32768,-32768,-32768,-32768,     4,
-    60,    -2,-32768,-32768,-32768,-32768,-32768,-32768,-32768,-32768,
--32768,-32768,-32768,     6,-32768,    90,-32768,-32768,    75,-32768,
-    29,-32768,-32768,-32768,-32768,-32768,-32768,    10,    17,-32768
+/* YYRHS -- A `-1'-separated list of the rules' RHS.  */
+static const yytype_int8 yyrhs[] =
+{
+      20,     0,    -1,    -1,    21,    -1,    12,    -1,    22,    -1,
+      26,    -1,    27,    -1,    33,    -1,    30,    -1,    32,    -1,
+      29,    -1,    31,    -1,    13,    14,    -1,    13,    23,    14,
+      -1,     6,    -1,    24,    -1,    23,    24,    -1,    25,    21,
+      -1,     8,    -1,    15,    16,    -1,    15,    28,    16,    -1,
+       3,    -1,    17,    18,    -1,    17,    28,    18,    -1,    10,
+      -1,    21,    -1,    28,    21,    -1,     4,    -1,     5,    -1,
+       7,    -1,     9,    -1,    11,    -1
 };
 
-static const short yypgoto[] = {-32768,
-     0,-32768,-32768,   -18,-32768,-32768,-32768,     7,-32768,-32768,
--32768,-32768,-32768
+/* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
+static const yytype_uint8 yyrline[] =
+{
+       0,   150,   150,   153,   158,   163,   164,   165,   166,   167,
+     168,   169,   170,   183,   186,   189,   192,   193,   198,   207,
+     212,   215,   218,   221,   224,   227,   230,   233,   240,   243,
+     246,   249,   252
 };
+#endif
 
+#if YYDEBUG || YYERROR_VERBOSE || YYTOKEN_TABLE
+/* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM.
+   First, the terminals, then, starting at YYNTOKENS, nonterminals.  */
+static const char *const yytname[] =
+{
+  "$end", "error", "$undefined", "ARRAY", "BOOLEAN", "DATA", "DICTIONARY",
+  "IDREF", "KEY", "NUMBER", "SET", "STRING", "SYNTAX_ERROR", "'{'", "'}'",
+  "'('", "')'", "'['", "']'", "$accept", "input", "object", "dict",
+  "pairs", "pair", "key", "array", "set", "elements", "boolean", "data",
+  "idref", "number", "string", 0
+};
+#endif
 
-#define	YYLAST		107
+# ifdef YYPRINT
+/* YYTOKNUM[YYLEX-NUM] -- Internal token number corresponding to
+   token YYLEX-NUM.  */
+static const yytype_uint16 yytoknum[] =
+{
+       0,   256,   257,   258,   259,   260,   261,   262,   263,   264,
+     265,   266,   267,   123,   125,    40,    41,    91,    93
+};
+# endif
 
+/* YYR1[YYN] -- Symbol number of symbol that rule YYN derives.  */
+static const yytype_uint8 yyr1[] =
+{
+       0,    19,    20,    20,    20,    21,    21,    21,    21,    21,
+      21,    21,    21,    22,    22,    22,    23,    23,    24,    25,
+      26,    26,    26,    27,    27,    27,    28,    28,    29,    30,
+      31,    32,    33
+};
 
-static const short yytable[] = {    13,
-     1,     2,     3,     4,     5,    33,     6,     7,     8,    39,
-    10,    22,    11,    22,    12,    30,    40,    23,    31,    32,
-     0,     0,     0,     0,     0,    34,     0,     0,    36,     0,
-    36,     1,     2,     3,     4,     5,     0,     6,     7,     8,
-     0,    10,     0,    11,     0,    12,    37,     1,     2,     3,
-     4,     5,     0,     6,     7,     8,     9,    10,     0,    11,
-     0,    12,     1,     2,     3,     4,     5,     0,     6,     7,
-     8,     0,    10,     0,    11,    27,    12,     1,     2,     3,
-     4,     5,     0,     6,     7,     8,     0,    10,     0,    11,
-    35,    12,     1,     2,     3,     4,     5,     0,     6,     7,
-     8,     0,    10,     0,    11,     0,    12
+/* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN.  */
+static const yytype_uint8 yyr2[] =
+{
+       0,     2,     0,     1,     1,     1,     1,     1,     1,     1,
+       1,     1,     1,     2,     3,     1,     1,     2,     2,     1,
+       2,     3,     1,     2,     3,     1,     1,     2,     1,     1,
+       1,     1,     1
 };
 
-static const short yycheck[] = {     0,
-     3,     4,     5,     6,     7,    24,     9,    10,    11,     0,
-    13,     8,    15,     8,    17,    18,     0,    14,    12,    14,
-    -1,    -1,    -1,    -1,    -1,    26,    -1,    -1,    29,    -1,
-    31,     3,     4,     5,     6,     7,    -1,     9,    10,    11,
-    -1,    13,    -1,    15,    -1,    17,    18,     3,     4,     5,
-     6,     7,    -1,     9,    10,    11,    12,    13,    -1,    15,
-    -1,    17,     3,     4,     5,     6,     7,    -1,     9,    10,
-    11,    -1,    13,    -1,    15,    16,    17,     3,     4,     5,
-     6,     7,    -1,     9,    10,    11,    -1,    13,    -1,    15,
-    16,    17,     3,     4,     5,     6,     7,    -1,     9,    10,
-    11,    -1,    13,    -1,    15,    -1,    17
+/* YYDEFACT[STATE-NAME] -- Default rule to reduce with in state
+   STATE-NUM when YYTABLE doesn't specify something else to do.  Zero
+   means the default is an error.  */
+static const yytype_uint8 yydefact[] =
+{
+       2,    22,    28,    29,    15,    30,    31,    25,    32,     4,
+       0,     0,     0,     0,     3,     5,     6,     7,    11,     9,
+      12,    10,     8,    19,    13,     0,    16,     0,    20,    26,
+       0,    23,     0,     1,    14,    17,    18,    21,    27,    24
 };
-#define YYPURE 1
 
-/* -*-C-*-  Note some compilers choke on comments on `#line' lines.  */
-#line 3 "/usr/share/bison.simple"
-/* This file comes from bison-1.28.  */
+/* YYDEFGOTO[NTERM-NUM].  */
+static const yytype_int8 yydefgoto[] =
+{
+      -1,    13,    29,    15,    25,    26,    27,    16,    17,    30,
+      18,    19,    20,    21,    22
+};
 
-/* Skeleton output parser for bison,
-   Copyright (C) 1984, 1989, 1990 Free Software Foundation, Inc.
+/* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
+   STATE-NUM.  */
+#define YYPACT_NINF -20
+static const yytype_int8 yypact[] =
+{
+      46,   -20,   -20,   -20,   -20,   -20,   -20,   -20,   -20,   -20,
+       4,    61,    -2,    10,   -20,   -20,   -20,   -20,   -20,   -20,
+     -20,   -20,   -20,   -20,   -20,     6,   -20,    91,   -20,   -20,
+      76,   -20,    30,   -20,   -20,   -20,   -20,   -20,   -20,   -20
+};
 
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 2, or (at your option)
-   any later version.
+/* YYPGOTO[NTERM-NUM].  */
+static const yytype_int8 yypgoto[] =
+{
+     -20,   -20,     0,   -20,   -20,   -19,   -20,   -20,   -20,     5,
+     -20,   -20,   -20,   -20,   -20
+};
 
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
+/* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
+   positive, shift that token.  If negative, reduce the rule which
+   number is the opposite.  If zero, do what YYDEFACT says.
+   If YYTABLE_NINF, syntax error.  */
+#define YYTABLE_NINF -1
+static const yytype_uint8 yytable[] =
+{
+      14,     1,     2,     3,     4,     5,    35,     6,     7,     8,
+      33,    10,    23,    11,    23,    12,    31,    32,    24,     0,
+      34,     0,     0,     0,     0,     0,     0,    36,     0,     0,
+      38,     0,    38,     1,     2,     3,     4,     5,     0,     6,
+       7,     8,     0,    10,     0,    11,     0,    12,    39,     1,
+       2,     3,     4,     5,     0,     6,     7,     8,     9,    10,
+       0,    11,     0,    12,     1,     2,     3,     4,     5,     0,
+       6,     7,     8,     0,    10,     0,    11,    28,    12,     1,
+       2,     3,     4,     5,     0,     6,     7,     8,     0,    10,
+       0,    11,    37,    12,     1,     2,     3,     4,     5,     0,
+       6,     7,     8,     0,    10,     0,    11,     0,    12
+};
 
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-/* As a special exception, when this file is copied by Bison into a
-   Bison output file, you may use that output file without restriction.
-   This special exception was added by the Free Software Foundation
-   in version 1.24 of Bison.  */
-
-/* This is the parser code that is written into each bison parser
-  when the %semantic_parser declaration is not specified in the grammar.
-  It was written by Richard Stallman by simplifying the hairy parser
-  used when %semantic_parser is specified.  */
-
-#ifndef YYSTACK_USE_ALLOCA
-#ifdef alloca
-#define YYSTACK_USE_ALLOCA
-#else /* alloca not defined */
-#ifdef __GNUC__
-#define YYSTACK_USE_ALLOCA
-#define alloca __builtin_alloca
-#else /* not GNU C.  */
-#if (!defined (__STDC__) && defined (sparc)) || defined (__sparc__) || defined (__sparc) || defined (__sgi) || (defined (__sun) && defined (__i386))
-#define YYSTACK_USE_ALLOCA
-#include <alloca.h>
-#else /* not sparc */
-/* We think this test detects Watcom and Microsoft C.  */
-/* This used to test MSDOS, but that is a bad idea
-   since that symbol is in the user namespace.  */
-#if (defined (_MSDOS) || defined (_MSDOS_)) && !defined (__TURBOC__)
-#if 0 /* No need for malloc.h, which pollutes the namespace;
-	 instead, just don't use alloca.  */
-#include <malloc.h>
-#endif
-#else /* not MSDOS, or __TURBOC__ */
-#if defined(_AIX)
-/* I don't know what this was needed for, but it pollutes the namespace.
-   So I turned it off.   rms, 2 May 1997.  */
-/* #include <malloc.h>  */
- #pragma alloca
-#define YYSTACK_USE_ALLOCA
-#else /* not MSDOS, or __TURBOC__, or _AIX */
-#if 0
-#ifdef __hpux /* haible@ilog.fr says this works for HPUX 9.05 and up,
-		 and on HPUX 10.  Eventually we can turn this on.  */
-#define YYSTACK_USE_ALLOCA
-#define alloca __builtin_alloca
-#endif /* __hpux */
-#endif
-#endif /* not _AIX */
-#endif /* not MSDOS, or __TURBOC__ */
-#endif /* not sparc */
-#endif /* not GNU C */
-#endif /* alloca not defined */
-#endif /* YYSTACK_USE_ALLOCA not defined */
-
-#ifdef YYSTACK_USE_ALLOCA
-#define YYSTACK_ALLOC alloca
-#else
-#define YYSTACK_ALLOC malloc
-#endif
+static const yytype_int8 yycheck[] =
+{
+       0,     3,     4,     5,     6,     7,    25,     9,    10,    11,
+       0,    13,     8,    15,     8,    17,    18,    12,    14,    -1,
+      14,    -1,    -1,    -1,    -1,    -1,    -1,    27,    -1,    -1,
+      30,    -1,    32,     3,     4,     5,     6,     7,    -1,     9,
+      10,    11,    -1,    13,    -1,    15,    -1,    17,    18,     3,
+       4,     5,     6,     7,    -1,     9,    10,    11,    12,    13,
+      -1,    15,    -1,    17,     3,     4,     5,     6,     7,    -1,
+       9,    10,    11,    -1,    13,    -1,    15,    16,    17,     3,
+       4,     5,     6,     7,    -1,     9,    10,    11,    -1,    13,
+      -1,    15,    16,    17,     3,     4,     5,     6,     7,    -1,
+       9,    10,    11,    -1,    13,    -1,    15,    -1,    17
+};
 
-/* Note: there must be only one dollar sign in this file.
-   It is replaced by the list of actions, each action
-   as one case of the switch.  */
+/* YYSTOS[STATE-NUM] -- The (internal number of the) accessing
+   symbol of state STATE-NUM.  */
+static const yytype_uint8 yystos[] =
+{
+       0,     3,     4,     5,     6,     7,     9,    10,    11,    12,
+      13,    15,    17,    20,    21,    22,    26,    27,    29,    30,
+      31,    32,    33,     8,    14,    23,    24,    25,    16,    21,
+      28,    18,    28,     0,    14,    24,    21,    16,    21,    18
+};
 
 #define yyerrok		(yyerrstatus = 0)
 #define yyclearin	(yychar = YYEMPTY)
-#define YYEMPTY		-2
+#define YYEMPTY		(-2)
 #define YYEOF		0
+
 #define YYACCEPT	goto yyacceptlab
-#define YYABORT 	goto yyabortlab
-#define YYERROR		goto yyerrlab1
-/* Like YYERROR except do call yyerror.
-   This remains here temporarily to ease the
-   transition to the new meaning of YYERROR, for GCC.
+#define YYABORT		goto yyabortlab
+#define YYERROR		goto yyerrorlab
+
+
+/* Like YYERROR except do call yyerror.  This remains here temporarily
+   to ease the transition to the new meaning of YYERROR, for GCC.
    Once GCC version 2 has supplanted version 1, this can go.  */
+
 #define YYFAIL		goto yyerrlab
+
 #define YYRECOVERING()  (!!yyerrstatus)
-#define YYBACKUP(token, value) \
+
+#define YYBACKUP(Token, Value)					\
 do								\
   if (yychar == YYEMPTY && yylen == 1)				\
-    { yychar = (token), yylval = (value);			\
-      yychar1 = YYTRANSLATE (yychar);				\
-      YYPOPSTACK;						\
+    {								\
+      yychar = (Token);						\
+      yylval = (Value);						\
+      yytoken = YYTRANSLATE (yychar);				\
+      YYPOPSTACK (1);						\
       goto yybackup;						\
     }								\
   else								\
-    { yyerror ("syntax error: cannot back up"); YYERROR; }	\
-while (0)
+    {								\
+      yyerror (YY_("syntax error: cannot back up")); \
+      YYERROR;							\
+    }								\
+while (YYID (0))
+
 
 #define YYTERROR	1
 #define YYERRCODE	256
 
-#ifndef YYPURE
-#define YYLEX		yylex()
+
+/* YYLLOC_DEFAULT -- Set CURRENT to span from RHS[1] to RHS[N].
+   If N is 0, then set CURRENT to the empty location which ends
+   the previous symbol: RHS[0] (always defined).  */
+
+#define YYRHSLOC(Rhs, K) ((Rhs)[K])
+#ifndef YYLLOC_DEFAULT
+# define YYLLOC_DEFAULT(Current, Rhs, N)				\
+    do									\
+      if (YYID (N))                                                    \
+	{								\
+	  (Current).first_line   = YYRHSLOC (Rhs, 1).first_line;	\
+	  (Current).first_column = YYRHSLOC (Rhs, 1).first_column;	\
+	  (Current).last_line    = YYRHSLOC (Rhs, N).last_line;		\
+	  (Current).last_column  = YYRHSLOC (Rhs, N).last_column;	\
+	}								\
+      else								\
+	{								\
+	  (Current).first_line   = (Current).last_line   =		\
+	    YYRHSLOC (Rhs, 0).last_line;				\
+	  (Current).first_column = (Current).last_column =		\
+	    YYRHSLOC (Rhs, 0).last_column;				\
+	}								\
+    while (YYID (0))
 #endif
 
-#ifdef YYPURE
-#ifdef YYLSP_NEEDED
-#ifdef YYLEX_PARAM
-#define YYLEX		yylex(&yylval, &yylloc, YYLEX_PARAM)
-#else
-#define YYLEX		yylex(&yylval, &yylloc)
+
+/* YY_LOCATION_PRINT -- Print the location on the stream.
+   This macro was not mandated originally: define only if we know
+   we won't break user code: when these are the locations we know.  */
+
+#ifndef YY_LOCATION_PRINT
+# if YYLTYPE_IS_TRIVIAL
+#  define YY_LOCATION_PRINT(File, Loc)			\
+     fprintf (File, "%d.%d-%d.%d",			\
+	      (Loc).first_line, (Loc).first_column,	\
+	      (Loc).last_line,  (Loc).last_column)
+# else
+#  define YY_LOCATION_PRINT(File, Loc) ((void) 0)
+# endif
 #endif
-#else /* not YYLSP_NEEDED */
+
+
+/* YYLEX -- calling `yylex' with the right arguments.  */
+
 #ifdef YYLEX_PARAM
-#define YYLEX		yylex(&yylval, YYLEX_PARAM)
+# define YYLEX yylex (&yylval, YYLEX_PARAM)
 #else
-#define YYLEX		yylex(&yylval)
+# define YYLEX yylex (&yylval)
 #endif
-#endif /* not YYLSP_NEEDED */
+
+/* Enable debugging if requested.  */
+#if YYDEBUG
+
+# ifndef YYFPRINTF
+#  include <stdio.h> /* INFRINGES ON USER NAME SPACE */
+#  define YYFPRINTF fprintf
+# endif
+
+# define YYDPRINTF(Args)			\
+do {						\
+  if (yydebug)					\
+    YYFPRINTF Args;				\
+} while (YYID (0))
+
+# define YY_SYMBOL_PRINT(Title, Type, Value, Location)			  \
+do {									  \
+  if (yydebug)								  \
+    {									  \
+      YYFPRINTF (stderr, "%s ", Title);					  \
+      yy_symbol_print (stderr,						  \
+		  Type, Value); \
+      YYFPRINTF (stderr, "\n");						  \
+    }									  \
+} while (YYID (0))
+
+
+/*--------------------------------.
+| Print this symbol on YYOUTPUT.  |
+`--------------------------------*/
+
+/*ARGSUSED*/
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static void
+yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep)
+#else
+static void
+yy_symbol_value_print (yyoutput, yytype, yyvaluep)
+    FILE *yyoutput;
+    int yytype;
+    YYSTYPE const * const yyvaluep;
 #endif
+{
+  if (!yyvaluep)
+    return;
+# ifdef YYPRINT
+  if (yytype < YYNTOKENS)
+    YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep);
+# else
+  YYUSE (yyoutput);
+# endif
+  switch (yytype)
+    {
+      default:
+	break;
+    }
+}
 
-/* If nonreentrant, generate the variables here */
 
-#ifndef YYPURE
+/*--------------------------------.
+| Print this symbol on YYOUTPUT.  |
+`--------------------------------*/
 
-int	yychar;			/*  the lookahead symbol		*/
-YYSTYPE	yylval;			/*  the semantic value of the		*/
-				/*  lookahead symbol			*/
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static void
+yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep)
+#else
+static void
+yy_symbol_print (yyoutput, yytype, yyvaluep)
+    FILE *yyoutput;
+    int yytype;
+    YYSTYPE const * const yyvaluep;
+#endif
+{
+  if (yytype < YYNTOKENS)
+    YYFPRINTF (yyoutput, "token %s (", yytname[yytype]);
+  else
+    YYFPRINTF (yyoutput, "nterm %s (", yytname[yytype]);
+
+  yy_symbol_value_print (yyoutput, yytype, yyvaluep);
+  YYFPRINTF (yyoutput, ")");
+}
+
+/*------------------------------------------------------------------.
+| yy_stack_print -- Print the state stack from its BOTTOM up to its |
+| TOP (included).                                                   |
+`------------------------------------------------------------------*/
 
-#ifdef YYLSP_NEEDED
-YYLTYPE yylloc;			/*  location data for the lookahead	*/
-				/*  symbol				*/
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static void
+yy_stack_print (yytype_int16 *bottom, yytype_int16 *top)
+#else
+static void
+yy_stack_print (bottom, top)
+    yytype_int16 *bottom;
+    yytype_int16 *top;
 #endif
+{
+  YYFPRINTF (stderr, "Stack now");
+  for (; bottom <= top; ++bottom)
+    YYFPRINTF (stderr, " %d", *bottom);
+  YYFPRINTF (stderr, "\n");
+}
+
+# define YY_STACK_PRINT(Bottom, Top)				\
+do {								\
+  if (yydebug)							\
+    yy_stack_print ((Bottom), (Top));				\
+} while (YYID (0))
 
-int yynerrs;			/*  number of parse errors so far       */
-#endif  /* not YYPURE */
 
-#if YYDEBUG != 0
-int yydebug;			/*  nonzero means print parse trace	*/
-/* Since this is uninitialized, it does not stop multiple parsers
-   from coexisting.  */
+/*------------------------------------------------.
+| Report that the YYRULE is going to be reduced.  |
+`------------------------------------------------*/
+
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static void
+yy_reduce_print (YYSTYPE *yyvsp, int yyrule)
+#else
+static void
+yy_reduce_print (yyvsp, yyrule)
+    YYSTYPE *yyvsp;
+    int yyrule;
 #endif
+{
+  int yynrhs = yyr2[yyrule];
+  int yyi;
+  unsigned long int yylno = yyrline[yyrule];
+  YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n",
+	     yyrule - 1, yylno);
+  /* The symbols being reduced.  */
+  for (yyi = 0; yyi < yynrhs; yyi++)
+    {
+      fprintf (stderr, "   $%d = ", yyi + 1);
+      yy_symbol_print (stderr, yyrhs[yyprhs[yyrule] + yyi],
+		       &(yyvsp[(yyi + 1) - (yynrhs)])
+		       		       );
+      fprintf (stderr, "\n");
+    }
+}
+
+# define YY_REDUCE_PRINT(Rule)		\
+do {					\
+  if (yydebug)				\
+    yy_reduce_print (yyvsp, Rule); \
+} while (YYID (0))
+
+/* Nonzero means print parse trace.  It is left uninitialized so that
+   multiple parsers can coexist.  */
+int yydebug;
+#else /* !YYDEBUG */
+# define YYDPRINTF(Args)
+# define YY_SYMBOL_PRINT(Title, Type, Value, Location)
+# define YY_STACK_PRINT(Bottom, Top)
+# define YY_REDUCE_PRINT(Rule)
+#endif /* !YYDEBUG */
 
-/*  YYINITDEPTH indicates the initial size of the parser's stacks	*/
 
+/* YYINITDEPTH -- initial size of the parser's stacks.  */
 #ifndef	YYINITDEPTH
-#define YYINITDEPTH 200
+# define YYINITDEPTH 200
 #endif
 
-/*  YYMAXDEPTH is the maximum size the stacks can grow to
-    (effective only if the built-in stack extension method is used).  */
+/* YYMAXDEPTH -- maximum size the stacks can grow to (effective only
+   if the built-in stack extension method is used).
 
-#if YYMAXDEPTH == 0
-#undef YYMAXDEPTH
-#endif
+   Do not make this value too large; the results are undefined if
+   YYSTACK_ALLOC_MAXIMUM < YYSTACK_BYTES (YYMAXDEPTH)
+   evaluated with infinite-precision integer arithmetic.  */
 
 #ifndef YYMAXDEPTH
-#define YYMAXDEPTH 10000
+# define YYMAXDEPTH 10000
 #endif
+
 
-/* Define __yy_memcpy.  Note that the size argument
-   should be passed with type unsigned int, because that is what the non-GCC
-   definitions require.  With GCC, __builtin_memcpy takes an arg
-   of type size_t, but it can handle unsigned int.  */
-
-#if __GNUC__ > 1		/* GNU C and GNU C++ define this.  */
-#define __yy_memcpy(TO,FROM,COUNT)	__builtin_memcpy(TO,FROM,COUNT)
-#else				/* not GNU C or C++ */
-#ifndef __cplusplus
-
-/* This is the most reliable way to avoid incompatibilities
-   in available built-in functions on various systems.  */
-static void
-__yy_memcpy (to, from, count)
-     char *to;
-     char *from;
-     unsigned int count;
+
+#if YYERROR_VERBOSE
+
+# ifndef yystrlen
+#  if defined __GLIBC__ && defined _STRING_H
+#   define yystrlen strlen
+#  else
+/* Return the length of YYSTR.  */
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static YYSIZE_T
+yystrlen (const char *yystr)
+#else
+static YYSIZE_T
+yystrlen (yystr)
+    const char *yystr;
+#endif
+{
+  YYSIZE_T yylen;
+  for (yylen = 0; yystr[yylen]; yylen++)
+    continue;
+  return yylen;
+}
+#  endif
+# endif
+
+# ifndef yystpcpy
+#  if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE
+#   define yystpcpy stpcpy
+#  else
+/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in
+   YYDEST.  */
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static char *
+yystpcpy (char *yydest, const char *yysrc)
+#else
+static char *
+yystpcpy (yydest, yysrc)
+    char *yydest;
+    const char *yysrc;
+#endif
 {
-  register char *f = from;
-  register char *t = to;
-  register int i = count;
+  char *yyd = yydest;
+  const char *yys = yysrc;
+
+  while ((*yyd++ = *yys++) != '\0')
+    continue;
 
-  while (i-- > 0)
-    *t++ = *f++;
+  return yyd - 1;
 }
+#  endif
+# endif
+
+# ifndef yytnamerr
+/* Copy to YYRES the contents of YYSTR after stripping away unnecessary
+   quotes and backslashes, so that it's suitable for yyerror.  The
+   heuristic is that double-quoting is unnecessary unless the string
+   contains an apostrophe, a comma, or backslash (other than
+   backslash-backslash).  YYSTR is taken from yytname.  If YYRES is
+   null, do not copy; instead, return the length of what the result
+   would have been.  */
+static YYSIZE_T
+yytnamerr (char *yyres, const char *yystr)
+{
+  if (*yystr == '"')
+    {
+      YYSIZE_T yyn = 0;
+      char const *yyp = yystr;
+
+      for (;;)
+	switch (*++yyp)
+	  {
+	  case '\'':
+	  case ',':
+	    goto do_not_strip_quotes;
+
+	  case '\\':
+	    if (*++yyp != '\\')
+	      goto do_not_strip_quotes;
+	    /* Fall through.  */
+	  default:
+	    if (yyres)
+	      yyres[yyn] = *yyp;
+	    yyn++;
+	    break;
+
+	  case '"':
+	    if (yyres)
+	      yyres[yyn] = '\0';
+	    return yyn;
+	  }
+    do_not_strip_quotes: ;
+    }
 
-#else /* __cplusplus */
+  if (! yyres)
+    return yystrlen (yystr);
 
-/* This is the most reliable way to avoid incompatibilities
-   in available built-in functions on various systems.  */
-static void
-__yy_memcpy (char *to, char *from, unsigned int count)
+  return yystpcpy (yyres, yystr) - yyres;
+}
+# endif
+
+/* Copy into YYRESULT an error message about the unexpected token
+   YYCHAR while in state YYSTATE.  Return the number of bytes copied,
+   including the terminating null byte.  If YYRESULT is null, do not
+   copy anything; just return the number of bytes that would be
+   copied.  As a special case, return 0 if an ordinary "syntax error"
+   message will do.  Return YYSIZE_MAXIMUM if overflow occurs during
+   size calculation.  */
+static YYSIZE_T
+yysyntax_error (char *yyresult, int yystate, int yychar)
 {
-  register char *t = to;
-  register char *f = from;
-  register int i = count;
+  int yyn = yypact[yystate];
 
-  while (i-- > 0)
-    *t++ = *f++;
+  if (! (YYPACT_NINF < yyn && yyn <= YYLAST))
+    return 0;
+  else
+    {
+      int yytype = YYTRANSLATE (yychar);
+      YYSIZE_T yysize0 = yytnamerr (0, yytname[yytype]);
+      YYSIZE_T yysize = yysize0;
+      YYSIZE_T yysize1;
+      int yysize_overflow = 0;
+      enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 };
+      char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM];
+      int yyx;
+
+# if 0
+      /* This is so xgettext sees the translatable formats that are
+	 constructed on the fly.  */
+      YY_("syntax error, unexpected %s");
+      YY_("syntax error, unexpected %s, expecting %s");
+      YY_("syntax error, unexpected %s, expecting %s or %s");
+      YY_("syntax error, unexpected %s, expecting %s or %s or %s");
+      YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s");
+# endif
+      char *yyfmt;
+      char const *yyf;
+      static char const yyunexpected[] = "syntax error, unexpected %s";
+      static char const yyexpecting[] = ", expecting %s";
+      static char const yyor[] = " or %s";
+      char yyformat[sizeof yyunexpected
+		    + sizeof yyexpecting - 1
+		    + ((YYERROR_VERBOSE_ARGS_MAXIMUM - 2)
+		       * (sizeof yyor - 1))];
+      char const *yyprefix = yyexpecting;
+
+      /* Start YYX at -YYN if negative to avoid negative indexes in
+	 YYCHECK.  */
+      int yyxbegin = yyn < 0 ? -yyn : 0;
+
+      /* Stay within bounds of both yycheck and yytname.  */
+      int yychecklim = YYLAST - yyn + 1;
+      int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS;
+      int yycount = 1;
+
+      yyarg[0] = yytname[yytype];
+      yyfmt = yystpcpy (yyformat, yyunexpected);
+
+      for (yyx = yyxbegin; yyx < yyxend; ++yyx)
+	if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR)
+	  {
+	    if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM)
+	      {
+		yycount = 1;
+		yysize = yysize0;
+		yyformat[sizeof yyunexpected - 1] = '\0';
+		break;
+	      }
+	    yyarg[yycount++] = yytname[yyx];
+	    yysize1 = yysize + yytnamerr (0, yytname[yyx]);
+	    yysize_overflow |= (yysize1 < yysize);
+	    yysize = yysize1;
+	    yyfmt = yystpcpy (yyfmt, yyprefix);
+	    yyprefix = yyor;
+	  }
+
+      yyf = YY_(yyformat);
+      yysize1 = yysize + yystrlen (yyf);
+      yysize_overflow |= (yysize1 < yysize);
+      yysize = yysize1;
+
+      if (yysize_overflow)
+	return YYSIZE_MAXIMUM;
+
+      if (yyresult)
+	{
+	  /* Avoid sprintf, as that infringes on the user's name space.
+	     Don't have undefined behavior even if the translation
+	     produced a string with the wrong number of "%s"s.  */
+	  char *yyp = yyresult;
+	  int yyi = 0;
+	  while ((*yyp = *yyf) != '\0')
+	    {
+	      if (*yyp == '%' && yyf[1] == 's' && yyi < yycount)
+		{
+		  yyp += yytnamerr (yyp, yyarg[yyi++]);
+		  yyf += 2;
+		}
+	      else
+		{
+		  yyp++;
+		  yyf++;
+		}
+	    }
+	}
+      return yysize;
+    }
 }
+#endif /* YYERROR_VERBOSE */
+
 
+/*-----------------------------------------------.
+| Release the memory associated to this symbol.  |
+`-----------------------------------------------*/
+
+/*ARGSUSED*/
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+static void
+yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep)
+#else
+static void
+yydestruct (yymsg, yytype, yyvaluep)
+    const char *yymsg;
+    int yytype;
+    YYSTYPE *yyvaluep;
 #endif
-#endif
+{
+  YYUSE (yyvaluep);
+
+  if (!yymsg)
+    yymsg = "Deleting";
+  YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp);
+
+  switch (yytype)
+    {
+
+      default:
+	break;
+    }
+}
 
-#line 217 "/usr/share/bison.simple"
 
-/* The user can define YYPARSE_PARAM as the name of an argument to be passed
-   into yyparse.  The argument should have type void *.
-   It should actually point to an object.
-   Grammar actions can access the variable by casting it
-   to the proper pointer type.  */
+/* Prevent warnings from -Wmissing-prototypes.  */
 
 #ifdef YYPARSE_PARAM
-#ifdef __cplusplus
-#define YYPARSE_PARAM_ARG void *YYPARSE_PARAM
-#define YYPARSE_PARAM_DECL
-#else /* not __cplusplus */
-#define YYPARSE_PARAM_ARG YYPARSE_PARAM
-#define YYPARSE_PARAM_DECL void *YYPARSE_PARAM;
-#endif /* not __cplusplus */
-#else /* not YYPARSE_PARAM */
-#define YYPARSE_PARAM_ARG
-#define YYPARSE_PARAM_DECL
-#endif /* not YYPARSE_PARAM */
-
-/* Prevent warning if -Wstrict-prototypes.  */
-#ifdef __GNUC__
-#ifdef YYPARSE_PARAM
-int yyparse (void *);
+#if defined __STDC__ || defined __cplusplus
+int yyparse (void *YYPARSE_PARAM);
 #else
-int yyparse (void);
+int yyparse ();
 #endif
+#else /* ! YYPARSE_PARAM */
+#if defined __STDC__ || defined __cplusplus
+int yyparse (void);
+#else
+int yyparse ();
 #endif
+#endif /* ! YYPARSE_PARAM */
 
-int
-yyparse(YYPARSE_PARAM_ARG)
-     YYPARSE_PARAM_DECL
-{
-  register int yystate;
-  register int yyn;
-  register short *yyssp;
-  register YYSTYPE *yyvsp;
-  int yyerrstatus;	/*  number of tokens to shift before error messages enabled */
-  int yychar1 = 0;		/*  lookahead token as an internal (translated) token number */
 
-  short	yyssa[YYINITDEPTH];	/*  the state stack			*/
-  YYSTYPE yyvsa[YYINITDEPTH];	/*  the semantic value stack		*/
 
-  short *yyss = yyssa;		/*  refer to the stacks thru separate pointers */
-  YYSTYPE *yyvs = yyvsa;	/*  to allow yyoverflow to reallocate them elsewhere */
 
-#ifdef YYLSP_NEEDED
-  YYLTYPE yylsa[YYINITDEPTH];	/*  the location stack			*/
-  YYLTYPE *yyls = yylsa;
-  YYLTYPE *yylsp;
 
-#define YYPOPSTACK   (yyvsp--, yyssp--, yylsp--)
+
+/*----------.
+| yyparse.  |
+`----------*/
+
+#ifdef YYPARSE_PARAM
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+int
+yyparse (void *YYPARSE_PARAM)
 #else
-#define YYPOPSTACK   (yyvsp--, yyssp--)
+int
+yyparse (YYPARSE_PARAM)
+    void *YYPARSE_PARAM;
 #endif
+#else /* ! YYPARSE_PARAM */
+#if (defined __STDC__ || defined __C99__FUNC__ \
+     || defined __cplusplus || defined _MSC_VER)
+int
+yyparse (void)
+#else
+int
+yyparse ()
 
-  int yystacksize = YYINITDEPTH;
-  int yyfree_stacks = 0;
-
-#ifdef YYPURE
-  int yychar;
-  YYSTYPE yylval;
-  int yynerrs;
-#ifdef YYLSP_NEEDED
-  YYLTYPE yylloc;
 #endif
+#endif
+{
+  /* The look-ahead symbol.  */
+int yychar;
+
+/* The semantic value of the look-ahead symbol.  */
+YYSTYPE yylval;
+
+/* Number of syntax errors so far.  */
+int yynerrs;
+
+  int yystate;
+  int yyn;
+  int yyresult;
+  /* Number of tokens to shift before error messages enabled.  */
+  int yyerrstatus;
+  /* Look-ahead token as an internal (translated) token number.  */
+  int yytoken = 0;
+#if YYERROR_VERBOSE
+  /* Buffer for error messages, and its allocated size.  */
+  char yymsgbuf[128];
+  char *yymsg = yymsgbuf;
+  YYSIZE_T yymsg_alloc = sizeof yymsgbuf;
 #endif
 
-  YYSTYPE yyval;		/*  the variable used to return		*/
-				/*  semantic values from the action	*/
-				/*  routines				*/
+  /* Three stacks and their tools:
+     `yyss': related to states,
+     `yyvs': related to semantic values,
+     `yyls': related to locations.
 
-  int yylen;
+     Refer to the stacks thru separate pointers, to allow yyoverflow
+     to reallocate them elsewhere.  */
 
-#if YYDEBUG != 0
-  if (yydebug)
-    fprintf(stderr, "Starting parse\n");
-#endif
+  /* The state stack.  */
+  yytype_int16 yyssa[YYINITDEPTH];
+  yytype_int16 *yyss = yyssa;
+  yytype_int16 *yyssp;
+
+  /* The semantic value stack.  */
+  YYSTYPE yyvsa[YYINITDEPTH];
+  YYSTYPE *yyvs = yyvsa;
+  YYSTYPE *yyvsp;
+
+
+
+#define YYPOPSTACK(N)   (yyvsp -= (N), yyssp -= (N))
+
+  YYSIZE_T yystacksize = YYINITDEPTH;
+
+  /* The variables used to return semantic value and location from the
+     action routines.  */
+  YYSTYPE yyval;
+
+
+  /* The number of symbols on the RHS of the reduced rule.
+     Keep to zero when no symbol should be popped.  */
+  int yylen = 0;
+
+  YYDPRINTF ((stderr, "Starting parse\n"));
 
   yystate = 0;
   yyerrstatus = 0;
@@ -614,595 +1310,557 @@ yyparse(YYPARSE_PARAM_ARG)
      so that they stay on the same level as the state stack.
      The wasted elements are never initialized.  */
 
-  yyssp = yyss - 1;
+  yyssp = yyss;
   yyvsp = yyvs;
-#ifdef YYLSP_NEEDED
-  yylsp = yyls;
-#endif
 
-/* Push a new state, which is found in  yystate  .  */
-/* In all cases, when you get here, the value and location stacks
-   have just been pushed. so pushing a state here evens the stacks.  */
-yynewstate:
+  goto yysetstate;
 
-  *++yyssp = yystate;
+/*------------------------------------------------------------.
+| yynewstate -- Push a new state, which is found in yystate.  |
+`------------------------------------------------------------*/
+ yynewstate:
+  /* In all cases, when you get here, the value and location stacks
+     have just been pushed.  So pushing a state here evens the stacks.  */
+  yyssp++;
 
-  if (yyssp >= yyss + yystacksize - 1)
-    {
-      /* Give user a chance to reallocate the stack */
-      /* Use copies of these so that the &'s don't force the real ones into memory. */
-      YYSTYPE *yyvs1 = yyvs;
-      short *yyss1 = yyss;
-#ifdef YYLSP_NEEDED
-      YYLTYPE *yyls1 = yyls;
-#endif
+ yysetstate:
+  *yyssp = yystate;
 
+  if (yyss + yystacksize - 1 <= yyssp)
+    {
       /* Get the current used size of the three stacks, in elements.  */
-      int size = yyssp - yyss + 1;
+      YYSIZE_T yysize = yyssp - yyss + 1;
 
 #ifdef yyoverflow
-      /* Each stack pointer address is followed by the size of
-	 the data in use in that stack, in bytes.  */
-#ifdef YYLSP_NEEDED
-      /* This used to be a conditional around just the two extra args,
-	 but that might be undefined if yyoverflow is a macro.  */
-      yyoverflow("parser stack overflow",
-		 &yyss1, size * sizeof (*yyssp),
-		 &yyvs1, size * sizeof (*yyvsp),
-		 &yyls1, size * sizeof (*yylsp),
-		 &yystacksize);
-#else
-      yyoverflow("parser stack overflow",
-		 &yyss1, size * sizeof (*yyssp),
-		 &yyvs1, size * sizeof (*yyvsp),
-		 &yystacksize);
-#endif
-
-      yyss = yyss1; yyvs = yyvs1;
-#ifdef YYLSP_NEEDED
-      yyls = yyls1;
-#endif
+      {
+	/* Give user a chance to reallocate the stack.  Use copies of
+	   these so that the &'s don't force the real ones into
+	   memory.  */
+	YYSTYPE *yyvs1 = yyvs;
+	yytype_int16 *yyss1 = yyss;
+
+
+	/* Each stack pointer address is followed by the size of the
+	   data in use in that stack, in bytes.  This used to be a
+	   conditional around just the two extra args, but that might
+	   be undefined if yyoverflow is a macro.  */
+	yyoverflow (YY_("memory exhausted"),
+		    &yyss1, yysize * sizeof (*yyssp),
+		    &yyvs1, yysize * sizeof (*yyvsp),
+
+		    &yystacksize);
+
+	yyss = yyss1;
+	yyvs = yyvs1;
+      }
 #else /* no yyoverflow */
+# ifndef YYSTACK_RELOCATE
+      goto yyexhaustedlab;
+# else
       /* Extend the stack our own way.  */
-      if (yystacksize >= YYMAXDEPTH)
-	{
-	  yyerror("parser stack overflow");
-	  if (yyfree_stacks)
-	    {
-	      free (yyss);
-	      free (yyvs);
-#ifdef YYLSP_NEEDED
-	      free (yyls);
-#endif
-	    }
-	  return 2;
-	}
+      if (YYMAXDEPTH <= yystacksize)
+	goto yyexhaustedlab;
       yystacksize *= 2;
-      if (yystacksize > YYMAXDEPTH)
+      if (YYMAXDEPTH < yystacksize)
 	yystacksize = YYMAXDEPTH;
-#ifndef YYSTACK_USE_ALLOCA
-      yyfree_stacks = 1;
-#endif
-      yyss = (short *) YYSTACK_ALLOC (yystacksize * sizeof (*yyssp));
-      __yy_memcpy ((char *)yyss, (char *)yyss1,
-		   size * (unsigned int) sizeof (*yyssp));
-      yyvs = (YYSTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yyvsp));
-      __yy_memcpy ((char *)yyvs, (char *)yyvs1,
-		   size * (unsigned int) sizeof (*yyvsp));
-#ifdef YYLSP_NEEDED
-      yyls = (YYLTYPE *) YYSTACK_ALLOC (yystacksize * sizeof (*yylsp));
-      __yy_memcpy ((char *)yyls, (char *)yyls1,
-		   size * (unsigned int) sizeof (*yylsp));
-#endif
+
+      {
+	yytype_int16 *yyss1 = yyss;
+	union yyalloc *yyptr =
+	  (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize));
+	if (! yyptr)
+	  goto yyexhaustedlab;
+	YYSTACK_RELOCATE (yyss);
+	YYSTACK_RELOCATE (yyvs);
+
+#  undef YYSTACK_RELOCATE
+	if (yyss1 != yyssa)
+	  YYSTACK_FREE (yyss1);
+      }
+# endif
 #endif /* no yyoverflow */
 
-      yyssp = yyss + size - 1;
-      yyvsp = yyvs + size - 1;
-#ifdef YYLSP_NEEDED
-      yylsp = yyls + size - 1;
-#endif
+      yyssp = yyss + yysize - 1;
+      yyvsp = yyvs + yysize - 1;
 
-#if YYDEBUG != 0
-      if (yydebug)
-	fprintf(stderr, "Stack size increased to %d\n", yystacksize);
-#endif
 
-      if (yyssp >= yyss + yystacksize - 1)
+      YYDPRINTF ((stderr, "Stack size increased to %lu\n",
+		  (unsigned long int) yystacksize));
+
+      if (yyss + yystacksize - 1 <= yyssp)
 	YYABORT;
     }
 
-#if YYDEBUG != 0
-  if (yydebug)
-    fprintf(stderr, "Entering state %d\n", yystate);
-#endif
+  YYDPRINTF ((stderr, "Entering state %d\n", yystate));
 
   goto yybackup;
- yybackup:
 
-/* Do appropriate processing given the current state.  */
-/* Read a lookahead token if we need one and don't already have one.  */
-/* yyresume: */
+/*-----------.
+| yybackup.  |
+`-----------*/
+yybackup:
 
-  /* First try to decide what to do without reference to lookahead token.  */
+  /* Do appropriate processing given the current state.  Read a
+     look-ahead token if we need one and don't already have one.  */
 
+  /* First try to decide what to do without reference to look-ahead token.  */
   yyn = yypact[yystate];
-  if (yyn == YYFLAG)
+  if (yyn == YYPACT_NINF)
     goto yydefault;
 
-  /* Not known => get a lookahead token if don't already have one.  */
-
-  /* yychar is either YYEMPTY or YYEOF
-     or a valid token in external form.  */
+  /* Not known => get a look-ahead token if don't already have one.  */
 
+  /* YYCHAR is either YYEMPTY or YYEOF or a valid look-ahead symbol.  */
   if (yychar == YYEMPTY)
     {
-#if YYDEBUG != 0
-      if (yydebug)
-	fprintf(stderr, "Reading a token: ");
-#endif
+      YYDPRINTF ((stderr, "Reading a token: "));
       yychar = YYLEX;
     }
 
-  /* Convert token to internal form (in yychar1) for indexing tables with */
-
-  if (yychar <= 0)		/* This means end of input. */
+  if (yychar <= YYEOF)
     {
-      yychar1 = 0;
-      yychar = YYEOF;		/* Don't call YYLEX any more */
-
-#if YYDEBUG != 0
-      if (yydebug)
-	fprintf(stderr, "Now at end of input.\n");
-#endif
+      yychar = yytoken = YYEOF;
+      YYDPRINTF ((stderr, "Now at end of input.\n"));
     }
   else
     {
-      yychar1 = YYTRANSLATE(yychar);
-
-#if YYDEBUG != 0
-      if (yydebug)
-	{
-	  fprintf (stderr, "Next token is %d (%s", yychar, yytname[yychar1]);
-	  /* Give the individual parser a way to print the precise meaning
-	     of a token, for further debugging info.  */
-#ifdef YYPRINT
-	  YYPRINT (stderr, yychar, yylval);
-#endif
-	  fprintf (stderr, ")\n");
-	}
-#endif
+      yytoken = YYTRANSLATE (yychar);
+      YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc);
     }
 
-  yyn += yychar1;
-  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != yychar1)
+  /* If the proper action on seeing token YYTOKEN is to reduce or to
+     detect an error, take that action.  */
+  yyn += yytoken;
+  if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken)
     goto yydefault;
-
   yyn = yytable[yyn];
-
-  /* yyn is what to do for this token type in this state.
-     Negative => reduce, -yyn is rule number.
-     Positive => shift, yyn is new state.
-       New state is final state => don't bother to shift,
-       just return success.
-     0, or most negative number => error.  */
-
-  if (yyn < 0)
+  if (yyn <= 0)
     {
-      if (yyn == YYFLAG)
+      if (yyn == 0 || yyn == YYTABLE_NINF)
 	goto yyerrlab;
       yyn = -yyn;
       goto yyreduce;
     }
-  else if (yyn == 0)
-    goto yyerrlab;
 
   if (yyn == YYFINAL)
     YYACCEPT;
 
-  /* Shift the lookahead token.  */
+  /* Count tokens shifted since error; after three, turn off error
+     status.  */
+  if (yyerrstatus)
+    yyerrstatus--;
 
-#if YYDEBUG != 0
-  if (yydebug)
-    fprintf(stderr, "Shifting token %d (%s), ", yychar, yytname[yychar1]);
-#endif
+  /* Shift the look-ahead token.  */
+  YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc);
 
-  /* Discard the token being shifted unless it is eof.  */
+  /* Discard the shifted token unless it is eof.  */
   if (yychar != YYEOF)
     yychar = YYEMPTY;
 
+  yystate = yyn;
   *++yyvsp = yylval;
-#ifdef YYLSP_NEEDED
-  *++yylsp = yylloc;
-#endif
 
-  /* count tokens shifted since error; after three, turn off error status.  */
-  if (yyerrstatus) yyerrstatus--;
-
-  yystate = yyn;
   goto yynewstate;
 
-/* Do the default action for the current state.  */
-yydefault:
 
+/*-----------------------------------------------------------.
+| yydefault -- do the default action for the current state.  |
+`-----------------------------------------------------------*/
+yydefault:
   yyn = yydefact[yystate];
   if (yyn == 0)
     goto yyerrlab;
+  goto yyreduce;
+
 
-/* Do a reduction.  yyn is the number of a rule to reduce with.  */
+/*-----------------------------.
+| yyreduce -- Do a reduction.  |
+`-----------------------------*/
 yyreduce:
+  /* yyn is the number of a rule to reduce with.  */
   yylen = yyr2[yyn];
-  if (yylen > 0)
-    yyval = yyvsp[1-yylen]; /* implement default value of the action */
-
-#if YYDEBUG != 0
-  if (yydebug)
-    {
-      int i;
-
-      fprintf (stderr, "Reducing via rule %d (line %d), ",
-	       yyn, yyrline[yyn]);
 
-      /* Print the symbols being reduced, and their result.  */
-      for (i = yyprhs[yyn]; yyrhs[i] > 0; i++)
-	fprintf (stderr, "%s ", yytname[yyrhs[i]]);
-      fprintf (stderr, " -> %s\n", yytname[yyr1[yyn]]);
-    }
-#endif
+  /* If YYLEN is nonzero, implement the default value of the action:
+     `$$ = $1'.
 
+     Otherwise, the following line sets YYVAL to garbage.
+     This behavior is undocumented and Bison
+     users should not rely upon it.  Assigning to YYVAL
+     unconditionally makes the parser a bit smaller, and it avoids a
+     GCC warning that YYVAL may be used uninitialized.  */
+  yyval = yyvsp[1-yylen];
 
-  switch (yyn) {
 
-case 1:
-#line 144 "OSUnserializeXML.y"
-{ yyerror("unexpected end of buffer");
+  YY_REDUCE_PRINT (yyn);
+  switch (yyn)
+    {
+        case 2:
+#line 150 "OSUnserializeXML.y"
+    { yyerror("unexpected end of buffer");
 				  YYERROR;
-				;
-    break;}
-case 2:
-#line 147 "OSUnserializeXML.y"
-{ STATE->parsedObject = yyvsp[0]->object;
-				  yyvsp[0]->object = 0;
-				  freeObject(STATE, yyvsp[0]);
+				;}
+    break;
+
+  case 3:
+#line 153 "OSUnserializeXML.y"
+    { STATE->parsedObject = (yyvsp[(1) - (1)])->object;
+				  (yyvsp[(1) - (1)])->object = 0;
+				  freeObject(STATE, (yyvsp[(1) - (1)]));
 				  YYACCEPT;
-				;
-    break;}
-case 3:
-#line 152 "OSUnserializeXML.y"
-{ yyerror("syntax error");
-				  YYERROR;
-				;
-    break;}
-case 4:
-#line 157 "OSUnserializeXML.y"
-{ yyval = buildDictionary(STATE, yyvsp[0]); ;
-    break;}
-case 5:
+				;}
+    break;
+
+  case 4:
 #line 158 "OSUnserializeXML.y"
-{ yyval = buildArray(STATE, yyvsp[0]); ;
-    break;}
-case 6:
-#line 159 "OSUnserializeXML.y"
-{ yyval = buildSet(STATE, yyvsp[0]); ;
-    break;}
-case 7:
-#line 160 "OSUnserializeXML.y"
-{ yyval = buildString(STATE, yyvsp[0]); ;
-    break;}
-case 8:
-#line 161 "OSUnserializeXML.y"
-{ yyval = buildData(STATE, yyvsp[0]); ;
-    break;}
-case 9:
-#line 162 "OSUnserializeXML.y"
-{ yyval = buildNumber(STATE, yyvsp[0]); ;
-    break;}
-case 10:
+    { yyerror("syntax error");
+				  YYERROR;
+				;}
+    break;
+
+  case 5:
 #line 163 "OSUnserializeXML.y"
-{ yyval = buildBoolean(STATE, yyvsp[0]); ;
-    break;}
-case 11:
+    { (yyval) = buildDictionary(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 6:
 #line 164 "OSUnserializeXML.y"
-{ yyval = retrieveObject(STATE, yyvsp[0]->idref);
-				  if (yyval) {
-				    yyval->object->retain();
+    { (yyval) = buildArray(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 7:
+#line 165 "OSUnserializeXML.y"
+    { (yyval) = buildSet(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 8:
+#line 166 "OSUnserializeXML.y"
+    { (yyval) = buildString(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 9:
+#line 167 "OSUnserializeXML.y"
+    { (yyval) = buildData(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 10:
+#line 168 "OSUnserializeXML.y"
+    { (yyval) = buildNumber(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 11:
+#line 169 "OSUnserializeXML.y"
+    { (yyval) = buildBoolean(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 12:
+#line 170 "OSUnserializeXML.y"
+    { (yyval) = retrieveObject(STATE, (yyvsp[(1) - (1)])->idref);
+				  if ((yyval)) {
+				    (yyval)->object->retain();
 				  } else { 
 				    yyerror("forward reference detected");
 				    YYERROR;
 				  }
-				  freeObject(STATE, yyvsp[0]);
-				;
-    break;}
-case 12:
-#line 177 "OSUnserializeXML.y"
-{ yyval = yyvsp[-1];
-				  yyval->elements = NULL;
-				;
-    break;}
-case 13:
-#line 180 "OSUnserializeXML.y"
-{ yyval = yyvsp[-2];
-				  yyval->elements = yyvsp[-1];
-				;
-    break;}
-case 16:
-#line 187 "OSUnserializeXML.y"
-{ yyval = yyvsp[0];
-				  yyval->next = yyvsp[-1];
-				;
-    break;}
-case 17:
-#line 192 "OSUnserializeXML.y"
-{ yyval = yyvsp[-1];
-				  yyval->key = (OSString *)yyval->object;
-				  yyval->object = yyvsp[0]->object;
-				  yyval->next = NULL; 
-				  yyvsp[0]->object = 0;
-				  freeObject(STATE, yyvsp[0]);
-				;
-    break;}
-case 18:
-#line 201 "OSUnserializeXML.y"
-{ yyval = buildString(STATE, yyvsp[0]); ;
-    break;}
-case 19:
-#line 206 "OSUnserializeXML.y"
-{ yyval = yyvsp[-1];
-				  yyval->elements = NULL;
-				;
-    break;}
-case 20:
-#line 209 "OSUnserializeXML.y"
-{ yyval = yyvsp[-2];
-				  yyval->elements = yyvsp[-1];
-				;
-    break;}
-case 22:
+				  freeObject(STATE, (yyvsp[(1) - (1)]));
+				;}
+    break;
+
+  case 13:
+#line 183 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (2)]);
+				  (yyval)->elements = NULL;
+				;}
+    break;
+
+  case 14:
+#line 186 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (3)]);
+				  (yyval)->elements = (yyvsp[(2) - (3)]);
+				;}
+    break;
+
+  case 17:
+#line 193 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(2) - (2)]);
+				  (yyval)->next = (yyvsp[(1) - (2)]);
+				;}
+    break;
+
+  case 18:
+#line 198 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (2)]);
+				  (yyval)->key = (OSString *)(yyval)->object;
+				  (yyval)->object = (yyvsp[(2) - (2)])->object;
+				  (yyval)->next = NULL; 
+				  (yyvsp[(2) - (2)])->object = 0;
+				  freeObject(STATE, (yyvsp[(2) - (2)]));
+				;}
+    break;
+
+  case 19:
+#line 207 "OSUnserializeXML.y"
+    { (yyval) = buildString(STATE, (yyvsp[(1) - (1)])); ;}
+    break;
+
+  case 20:
+#line 212 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (2)]);
+				  (yyval)->elements = NULL;
+				;}
+    break;
+
+  case 21:
 #line 215 "OSUnserializeXML.y"
-{ yyval = yyvsp[-1];
-				  yyval->elements = NULL;
-				;
-    break;}
-case 23:
-#line 218 "OSUnserializeXML.y"
-{ yyval = yyvsp[-2];
-				  yyval->elements = yyvsp[-1];
-				;
-    break;}
-case 25:
+    { (yyval) = (yyvsp[(1) - (3)]);
+				  (yyval)->elements = (yyvsp[(2) - (3)]);
+				;}
+    break;
+
+  case 23:
+#line 221 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (2)]);
+				  (yyval)->elements = NULL;
+				;}
+    break;
+
+  case 24:
 #line 224 "OSUnserializeXML.y"
-{ yyval = yyvsp[0]; 
-				  yyval->next = NULL; 
-				;
-    break;}
-case 26:
-#line 227 "OSUnserializeXML.y"
-{ yyval = yyvsp[0];
-				  yyval->next = yyvsp[-1];
-				;
-    break;}
-}
-   /* the action file gets copied in in place of this dollarsign */
-#line 543 "/usr/share/bison.simple"
-
-  yyvsp -= yylen;
-  yyssp -= yylen;
-#ifdef YYLSP_NEEDED
-  yylsp -= yylen;
-#endif
-
-#if YYDEBUG != 0
-  if (yydebug)
-    {
-      short *ssp1 = yyss - 1;
-      fprintf (stderr, "state stack now");
-      while (ssp1 != yyssp)
-	fprintf (stderr, " %d", *++ssp1);
-      fprintf (stderr, "\n");
+    { (yyval) = (yyvsp[(1) - (3)]);
+				  (yyval)->elements = (yyvsp[(2) - (3)]);
+				;}
+    break;
+
+  case 26:
+#line 230 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(1) - (1)]); 
+				  (yyval)->next = NULL; 
+				;}
+    break;
+
+  case 27:
+#line 233 "OSUnserializeXML.y"
+    { (yyval) = (yyvsp[(2) - (2)]);
+				  (yyval)->next = (yyvsp[(1) - (2)]);
+				;}
+    break;
+
+
+/* Line 1267 of yacc.c.  */
+#line 1600 "OSUnserializeXML.tab.c"
+      default: break;
     }
-#endif
+  YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc);
+
+  YYPOPSTACK (yylen);
+  yylen = 0;
+  YY_STACK_PRINT (yyss, yyssp);
 
   *++yyvsp = yyval;
 
-#ifdef YYLSP_NEEDED
-  yylsp++;
-  if (yylen == 0)
-    {
-      yylsp->first_line = yylloc.first_line;
-      yylsp->first_column = yylloc.first_column;
-      yylsp->last_line = (yylsp-1)->last_line;
-      yylsp->last_column = (yylsp-1)->last_column;
-      yylsp->text = 0;
-    }
-  else
-    {
-      yylsp->last_line = (yylsp+yylen-1)->last_line;
-      yylsp->last_column = (yylsp+yylen-1)->last_column;
-    }
-#endif
 
-  /* Now "shift" the result of the reduction.
-     Determine what state that goes to,
-     based on the state we popped back to
-     and the rule number reduced by.  */
+  /* Now `shift' the result of the reduction.  Determine what state
+     that goes to, based on the state we popped back to and the rule
+     number reduced by.  */
 
   yyn = yyr1[yyn];
 
-  yystate = yypgoto[yyn - YYNTBASE] + *yyssp;
-  if (yystate >= 0 && yystate <= YYLAST && yycheck[yystate] == *yyssp)
+  yystate = yypgoto[yyn - YYNTOKENS] + *yyssp;
+  if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp)
     yystate = yytable[yystate];
   else
-    yystate = yydefgoto[yyn - YYNTBASE];
+    yystate = yydefgoto[yyn - YYNTOKENS];
 
   goto yynewstate;
 
-yyerrlab:   /* here on detecting error */
 
-  if (! yyerrstatus)
-    /* If not already recovering from an error, report this error.  */
+/*------------------------------------.
+| yyerrlab -- here on detecting error |
+`------------------------------------*/
+yyerrlab:
+  /* If not already recovering from an error, report this error.  */
+  if (!yyerrstatus)
     {
       ++yynerrs;
-
-#ifdef YYERROR_VERBOSE
-      yyn = yypact[yystate];
-
-      if (yyn > YYFLAG && yyn < YYLAST)
-	{
-	  int size = 0;
-	  char *msg;
-	  int x, count;
-
-	  count = 0;
-	  /* Start X at -yyn if nec to avoid negative indexes in yycheck.  */
-	  for (x = (yyn < 0 ? -yyn : 0);
-	       x < (sizeof(yytname) / sizeof(char *)); x++)
-	    if (yycheck[x + yyn] == x)
-	      size += strlen(yytname[x]) + 15, count++;
-	  msg = (char *) malloc(size + 15);
-	  if (msg != 0)
-	    {
-	      strcpy(msg, "parse error");
-
-	      if (count < 5)
-		{
-		  count = 0;
-		  for (x = (yyn < 0 ? -yyn : 0);
-		       x < (sizeof(yytname) / sizeof(char *)); x++)
-		    if (yycheck[x + yyn] == x)
-		      {
-			strcat(msg, count == 0 ? ", expecting `" : " or `");
-			strcat(msg, yytname[x]);
-			strcat(msg, "'");
-			count++;
-		      }
-		}
-	      yyerror(msg);
-	      free(msg);
-	    }
-	  else
-	    yyerror ("parse error; also virtual memory exceeded");
-	}
-      else
-#endif /* YYERROR_VERBOSE */
-	yyerror("parse error");
+#if ! YYERROR_VERBOSE
+      yyerror (YY_("syntax error"));
+#else
+      {
+	YYSIZE_T yysize = yysyntax_error (0, yystate, yychar);
+	if (yymsg_alloc < yysize && yymsg_alloc < YYSTACK_ALLOC_MAXIMUM)
+	  {
+	    YYSIZE_T yyalloc = 2 * yysize;
+	    if (! (yysize <= yyalloc && yyalloc <= YYSTACK_ALLOC_MAXIMUM))
+	      yyalloc = YYSTACK_ALLOC_MAXIMUM;
+	    if (yymsg != yymsgbuf)
+	      YYSTACK_FREE (yymsg);
+	    yymsg = (char *) YYSTACK_ALLOC (yyalloc);
+	    if (yymsg)
+	      yymsg_alloc = yyalloc;
+	    else
+	      {
+		yymsg = yymsgbuf;
+		yymsg_alloc = sizeof yymsgbuf;
+	      }
+	  }
+
+	if (0 < yysize && yysize <= yymsg_alloc)
+	  {
+	    (void) yysyntax_error (yymsg, yystate, yychar);
+	    yyerror (yymsg);
+	  }
+	else
+	  {
+	    yyerror (YY_("syntax error"));
+	    if (yysize != 0)
+	      goto yyexhaustedlab;
+	  }
+      }
+#endif
     }
 
-  goto yyerrlab1;
-yyerrlab1:   /* here on error raised explicitly by an action */
+
 
   if (yyerrstatus == 3)
     {
-      /* if just tried and failed to reuse lookahead token after an error, discard it.  */
+      /* If just tried and failed to reuse look-ahead token after an
+	 error, discard it.  */
 
-      /* return failure if at end of input */
-      if (yychar == YYEOF)
-	YYABORT;
-
-#if YYDEBUG != 0
-      if (yydebug)
-	fprintf(stderr, "Discarding token %d (%s).\n", yychar, yytname[yychar1]);
-#endif
-
-      yychar = YYEMPTY;
+      if (yychar <= YYEOF)
+	{
+	  /* Return failure if at end of input.  */
+	  if (yychar == YYEOF)
+	    YYABORT;
+	}
+      else
+	{
+	  yydestruct ("Error: discarding",
+		      yytoken, &yylval);
+	  yychar = YYEMPTY;
+	}
     }
 
-  /* Else will try to reuse lookahead token
-     after shifting the error token.  */
+  /* Else will try to reuse look-ahead token after shifting the error
+     token.  */
+  goto yyerrlab1;
 
-  yyerrstatus = 3;		/* Each real token shifted decrements this */
 
-  goto yyerrhandle;
+/*---------------------------------------------------.
+| yyerrorlab -- error raised explicitly by YYERROR.  |
+`---------------------------------------------------*/
+yyerrorlab:
 
-yyerrdefault:  /* current state does not do anything special for the error token. */
+  /* Pacify compilers like GCC when the user code never invokes
+     YYERROR and the label yyerrorlab therefore never appears in user
+     code.  */
+  if (/*CONSTCOND*/ 0)
+     goto yyerrorlab;
 
-#if 0
-  /* This is wrong; only states that explicitly want error tokens
-     should shift them.  */
-  yyn = yydefact[yystate];  /* If its default is to accept any token, ok.  Otherwise pop it.*/
-  if (yyn) goto yydefault;
-#endif
+  /* Do not reclaim the symbols of the rule which action triggered
+     this YYERROR.  */
+  YYPOPSTACK (yylen);
+  yylen = 0;
+  YY_STACK_PRINT (yyss, yyssp);
+  yystate = *yyssp;
+  goto yyerrlab1;
 
-yyerrpop:   /* pop the current state because it cannot handle the error token */
 
-  if (yyssp == yyss) YYABORT;
-  yyvsp--;
-  yystate = *--yyssp;
-#ifdef YYLSP_NEEDED
-  yylsp--;
-#endif
+/*-------------------------------------------------------------.
+| yyerrlab1 -- common code for both syntax error and YYERROR.  |
+`-------------------------------------------------------------*/
+yyerrlab1:
+  yyerrstatus = 3;	/* Each real token shifted decrements this.  */
 
-#if YYDEBUG != 0
-  if (yydebug)
+  for (;;)
     {
-      short *ssp1 = yyss - 1;
-      fprintf (stderr, "Error: state stack now");
-      while (ssp1 != yyssp)
-	fprintf (stderr, " %d", *++ssp1);
-      fprintf (stderr, "\n");
-    }
-#endif
-
-yyerrhandle:
+      yyn = yypact[yystate];
+      if (yyn != YYPACT_NINF)
+	{
+	  yyn += YYTERROR;
+	  if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR)
+	    {
+	      yyn = yytable[yyn];
+	      if (0 < yyn)
+		break;
+	    }
+	}
 
-  yyn = yypact[yystate];
-  if (yyn == YYFLAG)
-    goto yyerrdefault;
+      /* Pop the current state because it cannot handle the error token.  */
+      if (yyssp == yyss)
+	YYABORT;
 
-  yyn += YYTERROR;
-  if (yyn < 0 || yyn > YYLAST || yycheck[yyn] != YYTERROR)
-    goto yyerrdefault;
 
-  yyn = yytable[yyn];
-  if (yyn < 0)
-    {
-      if (yyn == YYFLAG)
-	goto yyerrpop;
-      yyn = -yyn;
-      goto yyreduce;
+      yydestruct ("Error: popping",
+		  yystos[yystate], yyvsp);
+      YYPOPSTACK (1);
+      yystate = *yyssp;
+      YY_STACK_PRINT (yyss, yyssp);
     }
-  else if (yyn == 0)
-    goto yyerrpop;
 
   if (yyn == YYFINAL)
     YYACCEPT;
 
-#if YYDEBUG != 0
-  if (yydebug)
-    fprintf(stderr, "Shifting error token, ");
-#endif
-
   *++yyvsp = yylval;
-#ifdef YYLSP_NEEDED
-  *++yylsp = yylloc;
-#endif
+
+
+  /* Shift the error token.  */
+  YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp);
 
   yystate = yyn;
   goto yynewstate;
 
- yyacceptlab:
-  /* YYACCEPT comes here.  */
-  if (yyfree_stacks)
-    {
-      free (yyss);
-      free (yyvs);
-#ifdef YYLSP_NEEDED
-      free (yyls);
+
+/*-------------------------------------.
+| yyacceptlab -- YYACCEPT comes here.  |
+`-------------------------------------*/
+yyacceptlab:
+  yyresult = 0;
+  goto yyreturn;
+
+/*-----------------------------------.
+| yyabortlab -- YYABORT comes here.  |
+`-----------------------------------*/
+yyabortlab:
+  yyresult = 1;
+  goto yyreturn;
+
+#ifndef yyoverflow
+/*-------------------------------------------------.
+| yyexhaustedlab -- memory exhaustion comes here.  |
+`-------------------------------------------------*/
+yyexhaustedlab:
+  yyerror (YY_("memory exhausted"));
+  yyresult = 2;
+  /* Fall through.  */
 #endif
-    }
-  return 0;
 
- yyabortlab:
-  /* YYABORT comes here.  */
-  if (yyfree_stacks)
+yyreturn:
+  if (yychar != YYEOF && yychar != YYEMPTY)
+     yydestruct ("Cleanup: discarding lookahead",
+		 yytoken, &yylval);
+  /* Do not reclaim the symbols of the rule which action triggered
+     this YYABORT or YYACCEPT.  */
+  YYPOPSTACK (yylen);
+  YY_STACK_PRINT (yyss, yyssp);
+  while (yyssp != yyss)
     {
-      free (yyss);
-      free (yyvs);
-#ifdef YYLSP_NEEDED
-      free (yyls);
-#endif
+      yydestruct ("Cleanup: popping",
+		  yystos[*yyssp], yyvsp);
+      YYPOPSTACK (1);
     }
-  return 1;
+#ifndef yyoverflow
+  if (yyss != yyssa)
+    YYSTACK_FREE (yyss);
+#endif
+#if YYERROR_VERBOSE
+  if (yymsg != yymsgbuf)
+    YYSTACK_FREE (yymsg);
+#endif
+  /* Make sure YYID is used.  */
+  return YYID (yyresult);
 }
-#line 249 "OSUnserializeXML.y"
+
+
+#line 255 "OSUnserializeXML.y"
 
 
 int
@@ -1649,7 +2307,7 @@ yylex(YYSTYPE *lvalp, parser_state_t *state)
 			}
 
 			bool isHexFormat = false;
-			for (int i=0; i < attributeCount; i++) {
+			for (i=0; i < attributeCount; i++) {
 				if (!strcmp(attributes[i], "format") && !strcmp(values[i], "hex")) {
 					isHexFormat = true;
 					break;
@@ -2052,3 +2710,4 @@ OSUnserializeXML(const char *buffer, OSString **errorString)
 //
 //
 //
+
diff --git a/libkern/c++/OSUnserializeXML.y b/libkern/c++/OSUnserializeXML.y
index 33a18ef73..ca44aa099 100644
--- a/libkern/c++/OSUnserializeXML.y
+++ b/libkern/c++/OSUnserializeXML.y
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 1999-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -692,7 +698,7 @@ yylex(YYSTYPE *lvalp, parser_state_t *state)
 			}
 
 			bool isHexFormat = false;
-			for (int i=0; i < attributeCount; i++) {
+			for (i=0; i < attributeCount; i++) {
 				if (!strcmp(attributes[i], "format") && !strcmp(values[i], "hex")) {
 					isHexFormat = true;
 					break;
diff --git a/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp b/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp
index d9f86d6b2..80a132574 100644
--- a/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp
+++ b/libkern/c++/Tests/TestSerialization/test1/test1_main.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/OSBase.h>
 
diff --git a/libkern/c++/Tests/TestSerialization/test2/test2_main.cpp b/libkern/c++/Tests/TestSerialization/test2/test2_main.cpp
index 4ed533cbc..a8fe830e9 100644
--- a/libkern/c++/Tests/TestSerialization/test2/test2_main.cpp
+++ b/libkern/c++/Tests/TestSerialization/test2/test2_main.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/OSBase.h>
 
diff --git a/libkern/conf/MASTER b/libkern/conf/MASTER
index b87e166d0..611d12999 100644
--- a/libkern/conf/MASTER
+++ b/libkern/conf/MASTER
@@ -51,6 +51,16 @@
 #
 ident		LIBKERN
 
-options		KDEBUG		# kernel tracing		# <kdebug>
-options		GPROF		# kernel profiling		# <profile>
-options		LIBKERNCPP	# C++ implementation		# <libkerncpp>
+options		KDEBUG			# kernel tracing		# <kdebug>
+options		GPROF			# kernel profiling		# <profile>
+options		LIBKERNCPP		# C++ implementation	# <libkerncpp>
+options		NETWORKING		# kernel networking		# <networking>
+options     CONFIG_DTRACE 	# dtrace support		# <config_dtrace>
+options		CRYPTO			# cryptographic routines	# <ipsec,crypto>
+options		ZLIB		# zlib support			# <zlib>
+
+options		CONFIG_NO_PANIC_STRINGS			# <no_panic_str>
+options		CONFIG_NO_PRINTF_STRINGS		# <no_printf_str>
+options		CONFIG_NO_KPRINTF_STRINGS		# <no_kprintf_str>
+
+options		IPSEC			# IP security	# <ipsec>
diff --git a/libkern/conf/MASTER.i386 b/libkern/conf/MASTER.i386
index b61a6e538..c8a93d539 100644
--- a/libkern/conf/MASTER.i386
+++ b/libkern/conf/MASTER.i386
@@ -1,11 +1,13 @@
 ######################################################################
 #
-#  RELEASE = [intel mach libkerncpp]
-#  PROFILE = [RELEASE profile]
-#  DEBUG = [intel mach libkerncpp debug]
+#  RELEASE	= [ intel mach libkerncpp networking config_dtrace crypto zlib ]
+#  PROFILE	= [ RELEASE profile ]
+#  DEBUG	= [ RELEASE debug ]
+#
+#  EMBEDDED	= [ intel mach libkerncpp networking crypto zlib ]
+#  DEVELOPMENT	= [ EMBEDDED config_dtrace ]
 #
 ######################################################################
 
 machine		"i386"					# <intel>
 cpu		"i386"					# <intel>
-
diff --git a/libkern/conf/MASTER.ppc b/libkern/conf/MASTER.ppc
index 5c6b53d20..2b8994142 100644
--- a/libkern/conf/MASTER.ppc
+++ b/libkern/conf/MASTER.ppc
@@ -4,9 +4,10 @@
 #  Standard Apple MacOS X Configurations:
 #  -------- ---- -------- ---------------
 #
-#  RELEASE = [ppc mach libkerncpp]
+#  RELEASE = [ppc mach libkerncpp networking config_dtrace crypto zlib]
+#  DEVELOPMENT = [ RELEASE ]
 #  PROFILE = [RELEASE profile]
-#  DEBUG = [ppc mach libkerncpp debug]
+#  DEBUG = [RELEASE debug]
 #  RELEASE_TRACE = [ RELEASE kdebug ]
 #  DEBUG_TRACE   = [ DEBUG kdebug ]
 #
diff --git a/libkern/conf/Makefile b/libkern/conf/Makefile
index 739d5d807..f499460ea 100644
--- a/libkern/conf/Makefile
+++ b/libkern/conf/Makefile
@@ -18,10 +18,14 @@ ifndef LIBKERN_KERNEL_CONFIG
 export LIBKERN_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(LIBKERN_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -29,36 +33,32 @@ $(COMPOBJROOT)/$(LIBKERN_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(LIBKERN_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(LIBKERN_KERNEL_CONFIG) $(LIBKERN_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(LIBKERN_KERNEL_CONFIG) $(LIBKERN_KERNEL_CONFIG); \
 	);
 
 .ORDER: $(COMPOBJROOT)/$(LIBKERN_KERNEL_CONFIG)/Makefile
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(LIBKERN_KERNEL_CONFIG)/Makefile
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(LIBKERN_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(LIBKERN_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(LIBKERN_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(LIBKERN_KERNEL_CONFIG)	\
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(LIBKERN_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all: do_all 
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/libkern/conf/Makefile.i386 b/libkern/conf/Makefile.i386
index 712240adf..3dab7b368 100644
--- a/libkern/conf/Makefile.i386
+++ b/libkern/conf/Makefile.i386
@@ -26,9 +26,11 @@ OBJS_NO_WERROR=				\
 	OSSymbol.cpo			\
 	OSUnserialize.cpo		\
 	OSIterator.cpo			\
-	OSSet.cpo				\
-	scanf.o					\
-	OSUnserializeXML.cpo
+	OSSet.cpo			\
+	scanf.o				\
+	OSUnserializeXML.cpo		\
+	zlib.o				\
+	uuid.o
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
 
diff --git a/libkern/conf/Makefile.template b/libkern/conf/Makefile.template
index fc5e9da90..0a4000c19 100644
--- a/libkern/conf/Makefile.template
+++ b/libkern/conf/Makefile.template
@@ -26,8 +26,8 @@ include $(MakeInc_def)
 #
 # XXX: CFLAGS
 #
-CFLAGS+= -DKERNEL -DLIBKERN_KERNEL_PRIVATE -DOSALLOCDEBUG=1 \
-         -Wall -Wno-four-char-constants -fno-common
+CFLAGS+= -imacros meta_features.h -DKERNEL -DLIBKERN_KERNEL_PRIVATE -DOSALLOCDEBUG=1 \
+         -Wall -Wno-four-char-constants -fno-common  $(CFLAGS_INLINE_CONFIG)
 
 SFLAGS+= -DKERNEL
 
@@ -76,14 +76,13 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS)
-	@echo "creating $(COMPONENT).o"
-	@echo [ updating $(COMPONENT).o ${LIBKERN_KERNEL_CONFIG} ]
-	$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
+	@echo LD $(COMPONENT)
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} 
 
 do_all: $(COMPONENT).o
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`
+	$(_v)${MD} -u Makedep -f -d `ls *.d`
 	
 do_build_all: do_depend 
 
@@ -91,4 +90,3 @@ do_build_all: do_depend
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/libkern/conf/files b/libkern/conf/files
index 8c32688f5..f1c59a5a8 100644
--- a/libkern/conf/files
+++ b/libkern/conf/files
@@ -3,6 +3,10 @@
 OPTIONS/libkerncpp					optional libkerncpp
 OPTIONS/kdebug						optional kdebug
 OPTIONS/gprof						optional gprof
+OPTIONS/config_dtrace				optional config_dtrace
+OPTIONS/networking					optional networking
+OPTIONS/crypto						optional crypto
+OPTIONS/zlib						optional zlib
 
 # libkern
 
@@ -33,3 +37,11 @@ libkern/c++/OSUnserializeXML.cpp			optional libkerncpp
 libkern/stdio/scanf.c					standard
 
 libkern/uuid/uuid.c					standard
+
+libkern/zlib.c					optional zlib networking
+libkern/zlib.c					optional ipsec
+
+libkern/crypto/md5.c				optional crypto
+libkern/crypto/md5.c				optional networking
+libkern/crypto/sha1.c				optional crypto
+libkern/crypto/sha1.c				optional ipsec
diff --git a/libkern/conf/tools/doconf/Makefile b/libkern/conf/tools/doconf/Makefile
index 2bf0b7a10..aa55a9419 100644
--- a/libkern/conf/tools/doconf/Makefile
+++ b/libkern/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,25 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/libkern/conf/tools/doconf/doconf.csh b/libkern/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/libkern/conf/tools/doconf/doconf.csh
+++ b/libkern/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/libkern/crypto/md5.c b/libkern/crypto/md5.c
new file mode 100644
index 000000000..46e005986
--- /dev/null
+++ b/libkern/crypto/md5.c
@@ -0,0 +1,364 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * MD5.C - RSA Data Security, Inc., MD5 message-digest algorithm
+ *
+ * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
+ * rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * This code is the same as the code published by RSA Inc.  It has been
+ * edited for clarity and style only.
+ */
+
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <libkern/crypto/md5.h>
+
+#define	memset(x, y, z)	bzero(x, z);
+#define	memcpy(x, y, z)	bcopy(y, x, z)
+
+/*
+ * The digest algorithm interprets the input message as a sequence of 32-bit
+ * little-endian words.  We must reverse bytes in each word on PPC and other
+ * big-endian platforms, but not on little-endian ones.  When we can, we try
+ * to load each word at once.  We don't quite care about alignment, since
+ * x86/x64 allows us to do 4-byte loads on non 4-byte aligned addresses,
+ * and on PPC we do 1-byte loads anyway.
+ *
+ * We could check against __LITLE_ENDIAN__ to generalize the 4-byte load
+ * optimization, but that might not tell us whether or not we need 4-byte
+ * aligned loads.  Since we know that __i386__ and __x86_64__ are the two
+ * little-endian architectures that are not alignment-restrictive, we check
+ * explicitly against them below.  Note that the byte-reversing code for
+ * big-endian will still work on little-endian, albeit much slower.
+ */
+#if defined(__i386__) || defined(__x86_64__)
+#define	FETCH_32(p)	(*(const u_int32_t *)(p))
+#else
+#define	FETCH_32(p)						\
+	(((u_int32_t)*((const u_int8_t *)(p))) |		\
+	(((u_int32_t)*((const u_int8_t *)(p) + 1)) << 8) |	\
+	(((u_int32_t)*((const u_int8_t *)(p) + 2)) << 16) |	\
+	(((u_int32_t)*((const u_int8_t *)(p) + 3)) << 24))
+#endif /* __i386__ || __x86_64__ */
+
+/*
+ * Encodes input (u_int32_t) into output (unsigned char). Assumes len is
+ * a multiple of 4. This is not compatible with memcpy().
+ */
+static void
+Encode(unsigned char *output, u_int32_t *input, unsigned int len)
+{
+	unsigned int i, j;
+
+	for (i = 0, j = 0; j < len; i++, j += 4) {
+#if defined(__i386__) || defined(__x86_64__)
+		*(u_int32_t *)(output + j) = input[i];
+#else
+		output[j] = input[i] & 0xff;
+		output[j + 1] = (input[i] >> 8) & 0xff;
+		output[j + 2] = (input[i] >> 16) & 0xff;
+		output[j + 3] = (input[i] >> 24) & 0xff;
+#endif /* __i386__ || __x86_64__ */
+	}
+}
+
+static unsigned char PADDING[64] = { 0x80, /* zeros */ };
+
+/* F, G, H and I are basic MD5 functions. */
+#define	F(x, y, z)	((((y) ^ (z)) & (x)) ^ (z))
+#define	G(x, y, z)	((((x) ^ (y)) & (z)) ^ (y))
+#define	H(x, y, z)	((x) ^ (y) ^ (z))
+#define	I(x, y, z)	(((~(z)) | (x)) ^ (y))
+
+/* ROTATE_LEFT rotates x left n bits. */
+#define	ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+
+/*
+ * FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
+ * Rotation is separate from addition to prevent recomputation.
+ */
+#define	FF(a, b, c, d, x, s, ac) {					\
+	(a) += F((b), (c), (d)) + (x) + (unsigned long long)(ac);	\
+	(a) = ROTATE_LEFT((a), (s));					\
+	(a) += (b);							\
+}
+
+#define	GG(a, b, c, d, x, s, ac) {					\
+	(a) += G((b), (c), (d)) + (x) + (unsigned long long)(ac);	\
+	(a) = ROTATE_LEFT((a), (s));					\
+	(a) += (b);							\
+}
+
+#define	HH(a, b, c, d, x, s, ac) {					\
+	(a) += H((b), (c), (d)) + (x) + (unsigned long long)(ac);	\
+	(a) = ROTATE_LEFT((a), (s));					\
+	(a) += (b);							\
+}
+
+#define	II(a, b, c, d, x, s, ac) {					\
+	(a) += I((b), (c), (d)) + (x) + (unsigned long long)(ac);	\
+	(a) = ROTATE_LEFT((a), (s));					\
+	(a) += (b);							\
+}
+
+static void MD5Transform(u_int32_t, u_int32_t, u_int32_t, u_int32_t,
+    const u_int8_t [64], MD5_CTX *);
+
+/*
+ * MD5 initialization. Begins an MD5 operation, writing a new context.
+ */
+void
+MD5Init(MD5_CTX *context)
+{
+	context->count[0] = context->count[1] = 0;
+
+	/* Load magic initialization constants.  */
+	context->state[0] = 0x67452301UL;
+	context->state[1] = 0xefcdab89UL;
+	context->state[2] = 0x98badcfeUL;
+	context->state[3] = 0x10325476UL;
+}
+
+/*
+ * MD5 block update operation. Continues an MD5 message-digest
+ * operation, processing another message block, and updating the
+ * context.
+ */
+void
+MD5Update(MD5_CTX *context, const void *inpp, unsigned int inputLen)
+{
+	u_int32_t i, index, partLen;
+	const unsigned char *input = (const unsigned char *)inpp;
+
+	/* Compute number of bytes mod 64 */
+	index = (context->count[0] >> 3) & 0x3F;
+
+	/* Update number of bits */
+	if ((context->count[0] += (inputLen << 3)) < (inputLen << 3))
+		context->count[1]++;
+	context->count[1] += (inputLen >> 29);
+
+	partLen = 64 - index;
+
+	/* Transform as many times as possible. */
+	i = 0;
+	if (inputLen >= partLen) {
+		if (index != 0) {
+			memcpy(&context->buffer[index], input, partLen);
+			MD5Transform(context->state[0], context->state[1],
+			    context->state[2], context->state[3],
+			    context->buffer, context);
+			i = partLen;
+		}
+
+		for (; i + 63 < inputLen; i += 64)
+			MD5Transform(context->state[0], context->state[1],
+			    context->state[2], context->state[3],
+			    &input[i], context);
+
+		if (inputLen == i)
+			return;
+
+		index = 0;
+	}
+
+	/* Buffer remaining input */
+	memcpy(&context->buffer[index], &input[i], inputLen - i);
+}
+
+/*
+ * MD5 finalization. Ends an MD5 message-digest operation, writing the
+ * the message digest and zeroizing the context.
+ */
+void
+MD5Final(unsigned char digest[MD5_DIGEST_LENGTH], MD5_CTX *context)
+{
+	unsigned char bits[8];
+	u_int32_t index = (context->count[0] >> 3) & 0x3f;
+
+	/* Save number of bits */
+	Encode(bits, context->count, 8);
+
+	/* Pad out to 56 mod 64. */
+	MD5Update(context, PADDING, ((index < 56) ? 56 : 120) - index);
+
+	/* Append length (before padding) */
+	MD5Update(context, bits, 8);
+
+	/* Store state in digest */
+	Encode(digest, context->state, 16);
+
+	/* Zeroize sensitive information. */
+	memset(context, 0, sizeof (*context));
+}
+
+/*
+ * MD5 basic transformation. Transforms state based on block.
+ */
+static void
+MD5Transform(u_int32_t a, u_int32_t b, u_int32_t c, u_int32_t d,
+    const u_int8_t block[64], MD5_CTX *context)
+{
+	/* Register (instead of array) is a win in most cases */
+	register u_int32_t x0, x1, x2, x3, x4, x5, x6, x7;
+	register u_int32_t x8, x9, x10, x11, x12, x13, x14, x15;
+
+	x15 = FETCH_32(block + 60);
+	x14 = FETCH_32(block + 56);
+	x13 = FETCH_32(block + 52);
+	x12 = FETCH_32(block + 48);
+	x11 = FETCH_32(block + 44);
+	x10 = FETCH_32(block + 40);
+	x9  = FETCH_32(block + 36);
+	x8  = FETCH_32(block + 32);
+	x7  = FETCH_32(block + 28);
+	x6  = FETCH_32(block + 24);
+	x5  = FETCH_32(block + 20);
+	x4  = FETCH_32(block + 16);
+	x3  = FETCH_32(block + 12);
+	x2  = FETCH_32(block +  8);
+	x1  = FETCH_32(block +  4);
+	x0  = FETCH_32(block +  0);
+
+	/* Round 1 */
+#define	S11 7
+#define	S12 12
+#define	S13 17
+#define	S14 22
+	FF(a, b, c, d, x0,  S11, 0xd76aa478UL); /* 1 */
+	FF(d, a, b, c, x1,  S12, 0xe8c7b756UL); /* 2 */
+	FF(c, d, a, b, x2,  S13, 0x242070dbUL); /* 3 */
+	FF(b, c, d, a, x3,  S14, 0xc1bdceeeUL); /* 4 */
+	FF(a, b, c, d, x4,  S11, 0xf57c0fafUL); /* 5 */
+	FF(d, a, b, c, x5,  S12, 0x4787c62aUL); /* 6 */
+	FF(c, d, a, b, x6,  S13, 0xa8304613UL); /* 7 */
+	FF(b, c, d, a, x7,  S14, 0xfd469501UL); /* 8 */
+	FF(a, b, c, d, x8,  S11, 0x698098d8UL); /* 9 */
+	FF(d, a, b, c, x9,  S12, 0x8b44f7afUL); /* 10 */
+	FF(c, d, a, b, x10, S13, 0xffff5bb1UL); /* 11 */
+	FF(b, c, d, a, x11, S14, 0x895cd7beUL); /* 12 */
+	FF(a, b, c, d, x12, S11, 0x6b901122UL); /* 13 */
+	FF(d, a, b, c, x13, S12, 0xfd987193UL); /* 14 */
+	FF(c, d, a, b, x14, S13, 0xa679438eUL); /* 15 */
+	FF(b, c, d, a, x15, S14, 0x49b40821UL); /* 16 */
+
+	/* Round 2 */
+#define	S21 5
+#define	S22 9
+#define	S23 14
+#define	S24 20
+	GG(a, b, c, d, x1,  S21, 0xf61e2562UL); /* 17 */
+	GG(d, a, b, c, x6,  S22, 0xc040b340UL); /* 18 */
+	GG(c, d, a, b, x11, S23, 0x265e5a51UL); /* 19 */
+	GG(b, c, d, a, x0,  S24, 0xe9b6c7aaUL); /* 20 */
+	GG(a, b, c, d, x5,  S21, 0xd62f105dUL); /* 21 */
+	GG(d, a, b, c, x10, S22, 0x02441453UL); /* 22 */
+	GG(c, d, a, b, x15, S23, 0xd8a1e681UL); /* 23 */
+	GG(b, c, d, a, x4,  S24, 0xe7d3fbc8UL); /* 24 */
+	GG(a, b, c, d, x9,  S21, 0x21e1cde6UL); /* 25 */
+	GG(d, a, b, c, x14, S22, 0xc33707d6UL); /* 26 */
+	GG(c, d, a, b, x3,  S23, 0xf4d50d87UL); /* 27 */
+	GG(b, c, d, a, x8,  S24, 0x455a14edUL); /* 28 */
+	GG(a, b, c, d, x13, S21, 0xa9e3e905UL); /* 29 */
+	GG(d, a, b, c, x2,  S22, 0xfcefa3f8UL); /* 30 */
+	GG(c, d, a, b, x7,  S23, 0x676f02d9UL); /* 31 */
+	GG(b, c, d, a, x12, S24, 0x8d2a4c8aUL); /* 32 */
+
+	/* Round 3 */
+#define	S31 4
+#define	S32 11
+#define	S33 16
+#define	S34 23
+	HH(a, b, c, d, x5,  S31, 0xfffa3942UL); /* 33 */
+	HH(d, a, b, c, x8,  S32, 0x8771f681UL); /* 34 */
+	HH(c, d, a, b, x11, S33, 0x6d9d6122UL); /* 35 */
+	HH(b, c, d, a, x14, S34, 0xfde5380cUL); /* 36 */
+	HH(a, b, c, d, x1,  S31, 0xa4beea44UL); /* 37 */
+	HH(d, a, b, c, x4,  S32, 0x4bdecfa9UL); /* 38 */
+	HH(c, d, a, b, x7,  S33, 0xf6bb4b60UL); /* 39 */
+	HH(b, c, d, a, x10, S34, 0xbebfbc70UL); /* 40 */
+	HH(a, b, c, d, x13, S31, 0x289b7ec6UL); /* 41 */
+	HH(d, a, b, c, x0,  S32, 0xeaa127faUL); /* 42 */
+	HH(c, d, a, b, x3,  S33, 0xd4ef3085UL); /* 43 */
+	HH(b, c, d, a, x6,  S34, 0x04881d05UL); /* 44 */
+	HH(a, b, c, d, x9,  S31, 0xd9d4d039UL); /* 45 */
+	HH(d, a, b, c, x12, S32, 0xe6db99e5UL); /* 46 */
+	HH(c, d, a, b, x15, S33, 0x1fa27cf8UL); /* 47 */
+	HH(b, c, d, a, x2,  S34, 0xc4ac5665UL); /* 48 */
+
+	/* Round 4 */
+#define	S41 6
+#define	S42 10
+#define	S43 15
+#define	S44 21
+	II(a, b, c, d, x0,  S41, 0xf4292244UL); /* 49 */
+	II(d, a, b, c, x7,  S42, 0x432aff97UL); /* 50 */
+	II(c, d, a, b, x14, S43, 0xab9423a7UL); /* 51 */
+	II(b, c, d, a, x5,  S44, 0xfc93a039UL); /* 52 */
+	II(a, b, c, d, x12, S41, 0x655b59c3UL); /* 53 */
+	II(d, a, b, c, x3,  S42, 0x8f0ccc92UL); /* 54 */
+	II(c, d, a, b, x10, S43, 0xffeff47dUL); /* 55 */
+	II(b, c, d, a, x1,  S44, 0x85845dd1UL); /* 56 */
+	II(a, b, c, d, x8,  S41, 0x6fa87e4fUL); /* 57 */
+	II(d, a, b, c, x15, S42, 0xfe2ce6e0UL); /* 58 */
+	II(c, d, a, b, x6,  S43, 0xa3014314UL); /* 59 */
+	II(b, c, d, a, x13, S44, 0x4e0811a1UL); /* 60 */
+	II(a, b, c, d, x4,  S41, 0xf7537e82UL); /* 61 */
+	II(d, a, b, c, x11, S42, 0xbd3af235UL); /* 62 */
+	II(c, d, a, b, x2,  S43, 0x2ad7d2bbUL); /* 63 */
+	II(b, c, d, a, x9,  S44, 0xeb86d391UL); /* 64 */
+
+	context->state[0] += a;
+	context->state[1] += b;
+	context->state[2] += c;
+	context->state[3] += d;
+
+	/* Zeroize sensitive information. */
+	x15 = x14 = x13 = x12 = x11 = x10 = x9 = x8 = 0;
+	x7 = x6 = x5 = x4 = x3 = x2 = x1 = x0 = 0;
+}
diff --git a/libkern/crypto/sha1.c b/libkern/crypto/sha1.c
new file mode 100644
index 000000000..5e10e07b0
--- /dev/null
+++ b/libkern/crypto/sha1.c
@@ -0,0 +1,373 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * This SHA1 code is based on the basic framework from the reference
+ * implementation for MD5.  That implementation is Copyright (C)
+ * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ *
+ * Based on the FIPS 180-1: Secure Hash Algorithm (SHA-1) available at
+ * http://www.itl.nist.gov/div897/pubs/fip180-1.htm
+ */
+
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <libkern/crypto/sha1.h>
+
+#define	memset(x, y, z)	bzero(x, z);
+#define	memcpy(x, y, z)	bcopy(y, x, z)
+
+/* Internal mappings to the legacy sha1_ctxt structure. */
+#define	state	h.b32
+#define	bcount	c.b32
+#define	buffer	m.b8
+
+/*
+ * The digest algorithm interprets the input message as a sequence of 32-bit
+ * big-endian words.  We must reverse bytes in each word on x86/64 platforms,
+ * but not on big-endian ones such as PPC.  For performance, we take advantage
+ * of the bswap instruction on x86/64 to perform byte-reversal.  On PPC, we
+ * could do 4-byte load if the address is 4-byte aligned which should further
+ * improve the performance.  But for code simplicity, we punt and do 1-byte
+ * loads instead.
+ */
+#if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__)
+#define	FETCH_32(p) ({							\
+	register u_int32_t l = (u_int32_t)*((const u_int32_t *)(p));	\
+	__asm__ __volatile__("bswap %0" : "=r" (l) : "0" (l));		\
+	l;								\
+})
+#else
+#define	FETCH_32(p)							\
+	(((u_int32_t)*((const u_int8_t *)(p) + 3)) |			\
+	(((u_int32_t)*((const u_int8_t *)(p) + 2)) << 8) |		\
+	(((u_int32_t)*((const u_int8_t *)(p) + 1)) << 16) |		\
+	(((u_int32_t)*((const u_int8_t *)(p))) << 24))
+#endif /* __i386__ || __x86_64__ */
+
+/*
+ * Encodes input (u_int32_t) into output (unsigned char). Assumes len is
+ * a multiple of 4. This is not compatible with memcpy().
+ */
+static void
+Encode(unsigned char *output, u_int32_t *input, unsigned int len)
+{
+	unsigned int i, j;
+
+	for (i = 0, j = 0; j < len; i++, j += 4) {
+		output[j + 3] = input[i] & 0xff;
+		output[j + 2] = (input[i] >> 8) & 0xff;
+		output[j + 1] = (input[i] >> 16) & 0xff;
+		output[j] = (input[i] >> 24) & 0xff;
+	}
+}
+
+static unsigned char PADDING[64] = { 0x80, /* zeros */ };
+
+/* Constants from FIPS 180-1 */
+#define	K_00_19		0x5a827999UL
+#define	K_20_39		0x6ed9eba1UL
+#define	K_40_59		0x8f1bbcdcUL
+#define	K_60_79		0xca62c1d6UL
+
+/* F, G, H and I are basic SHA1 functions. */
+#define	F(b, c, d)	((((c) ^ (d)) & (b)) ^ (d))
+#define	G(b, c, d)	((b) ^ (c) ^ (d))
+#define	H(b, c, d)	(((b) & (c)) | (((b) | (c)) & (d)))
+
+/* ROTATE_LEFT rotates x left n bits. */
+#define	ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n))))
+
+/* R, R1-R4 are macros used during each transformation round. */
+#define R(f, k, v, w, x, y, z, i) {				\
+	(v) = ROTATE_LEFT(w, 5) + f(x, y, z) + (v) + (i) + (k);	\
+	(x) = ROTATE_LEFT(x, 30);				\
+}
+
+#define	R1(v, w, x, y, z, i)	R(F, K_00_19, v, w, x, y, z, i)
+#define	R2(v, w, x, y, z, i)	R(G, K_20_39, v, w, x, y, z, i)
+#define	R3(v, w, x, y, z, i)	R(H, K_40_59, v, w, x, y, z, i)
+#define	R4(v, w, x, y, z, i)	R(G, K_60_79, v, w, x, y, z, i)
+
+/* WUPDATE represents Wt variable that gets updated for steps 16-79 */
+#define	WUPDATE(p, q, r, s) {		\
+	(p) = ((q) ^ (r) ^ (s) ^ (p));	\
+	(p) = ROTATE_LEFT(p, 1);	\
+}
+
+static void SHA1Transform(u_int32_t, u_int32_t, u_int32_t, u_int32_t,
+    u_int32_t, const u_int8_t *, SHA1_CTX *);
+
+void SHA1Final_r(SHA1_CTX *, void *);
+
+/*
+ * SHA1 initialization. Begins a SHA1 operation, writing a new context.
+ */
+void
+SHA1Init(SHA1_CTX *context)
+{
+	context->bcount[0] = context->bcount[1] = 0;
+	context->count = 0;
+
+	/* Load magic initialization constants.  */
+	context->state[0] = 0x67452301UL;
+	context->state[1] = 0xefcdab89UL;
+	context->state[2] = 0x98badcfeUL;
+	context->state[3] = 0x10325476UL;
+	context->state[4] = 0xc3d2e1f0UL;
+}
+
+/*
+ * SHA1 block update operation. Continues a SHA1 message-digest
+ * operation, processing another message block, and updating the
+ * context.
+ */
+void
+SHA1Update(SHA1_CTX *context, const void *inpp, size_t inputLen)
+{
+	u_int32_t i, index, partLen;
+	const unsigned char *input = (const unsigned char *)inpp;
+
+	if (inputLen == 0)
+		return;
+
+	/* Compute number of bytes mod 64 */
+	index = (context->bcount[1] >> 3) & 0x3F;
+
+	/* Update number of bits */
+	if ((context->bcount[1] += (inputLen << 3)) < (inputLen << 3))
+		context->bcount[0]++;
+	context->bcount[0] += (inputLen >> 29);
+
+	partLen = 64 - index;
+
+	/* Transform as many times as possible. */
+	i = 0;
+	if (inputLen >= partLen) {
+		if (index != 0) {
+			memcpy(&context->buffer[index], input, partLen);
+			SHA1Transform(context->state[0], context->state[1],
+			    context->state[2], context->state[3],
+			    context->state[4], context->buffer, context);
+			i = partLen;
+		}
+
+		for (; i + 63 < inputLen; i += 64)
+			SHA1Transform(context->state[0], context->state[1],
+			    context->state[2], context->state[3],
+			    context->state[4], &input[i], context);
+
+		if (inputLen == i)
+			return;
+
+		index = 0;
+	}
+
+	/* Buffer remaining input */
+	memcpy(&context->buffer[index], &input[i], inputLen - i);
+}
+
+/*
+ * For backwards compatibility, sha1_result symbol is mapped to this
+ * routine since it's equivalent to SHA1Final with reversed parameters.
+ */
+void
+SHA1Final_r(SHA1_CTX *context, void *digest)
+{
+	SHA1Final(digest, context);
+}
+
+/*
+ * SHA1 finalization. Ends an SHA1 message-digest operation, writing the
+ * the message digest and zeroizing the context.
+ */
+void
+SHA1Final(void *digest, SHA1_CTX *context)
+{
+	unsigned char bits[8];
+	u_int32_t index = (context->bcount[1] >> 3) & 0x3f;
+
+	/* Save number of bits */
+	Encode(bits, context->bcount, 8);
+
+	/* Pad out to 56 mod 64. */
+	SHA1Update(context, PADDING, ((index < 56) ? 56 : 120) - index);
+
+	/* Append length (before padding) */
+	SHA1Update(context, bits, 8);
+
+	/* Store state in digest */
+	Encode(digest, context->state, 20);
+
+	/* Zeroize sensitive information. */
+	memset(context, 0, sizeof (*context));
+}
+
+/*
+ * SHA1 basic transformation. Transforms state based on block.
+ */
+static void
+SHA1Transform(u_int32_t a, u_int32_t b, u_int32_t c, u_int32_t d,
+    u_int32_t e, const u_int8_t block[64], SHA1_CTX *context)
+{
+	/* Register (instead of array) is a win in most cases */
+	register u_int32_t w0, w1, w2, w3, w4, w5, w6, w7;
+	register u_int32_t w8, w9, w10, w11, w12, w13, w14, w15;
+
+	w15 = FETCH_32(block + 60);
+	w14 = FETCH_32(block + 56);
+	w13 = FETCH_32(block + 52);
+	w12 = FETCH_32(block + 48);
+	w11 = FETCH_32(block + 44);
+	w10 = FETCH_32(block + 40);
+	w9  = FETCH_32(block + 36);
+	w8  = FETCH_32(block + 32);
+	w7  = FETCH_32(block + 28);
+	w6  = FETCH_32(block + 24);
+	w5  = FETCH_32(block + 20);
+	w4  = FETCH_32(block + 16);
+	w3  = FETCH_32(block + 12);
+	w2  = FETCH_32(block +  8);
+	w1  = FETCH_32(block +  4);
+	w0  = FETCH_32(block +  0);
+
+	/* Round 1 */
+					R1(e, a, b, c, d,  w0);		/*  0 */
+					R1(d, e, a, b, c,  w1);		/*  1 */
+					R1(c, d, e, a, b,  w2);		/*  2 */
+					R1(b, c, d, e, a,  w3);		/*  3 */
+					R1(a, b, c, d, e,  w4);		/*  4 */
+					R1(e, a, b, c, d,  w5);		/*  5 */
+					R1(d, e, a, b, c,  w6);		/*  6 */
+					R1(c, d, e, a, b,  w7);		/*  7 */
+					R1(b, c, d, e, a,  w8);		/*  8 */
+					R1(a, b, c, d, e,  w9);		/*  9 */
+					R1(e, a, b, c, d, w10);		/* 10 */
+					R1(d, e, a, b, c, w11);		/* 11 */
+					R1(c, d, e, a, b, w12);		/* 12 */
+					R1(b, c, d, e, a, w13);		/* 13 */
+					R1(a, b, c, d, e, w14);		/* 14 */
+					R1(e, a, b, c, d, w15);		/* 15 */
+	WUPDATE( w0, w13,  w8,  w2);	R1(d, e, a, b, c,  w0);		/* 16 */
+	WUPDATE( w1, w14,  w9,  w3);	R1(c, d, e, a, b,  w1);		/* 17 */
+	WUPDATE( w2, w15, w10,  w4);	R1(b, c, d, e, a,  w2);		/* 18 */
+	WUPDATE( w3,  w0, w11,  w5);	R1(a, b, c, d, e,  w3);		/* 19 */
+
+	/* Round 2 */
+	WUPDATE( w4,  w1, w12,  w6);	R2(e, a, b, c, d,  w4);		/* 20 */
+	WUPDATE( w5,  w2, w13,  w7);	R2(d, e, a, b, c,  w5);		/* 21 */
+	WUPDATE( w6,  w3, w14,  w8);	R2(c, d, e, a, b,  w6);		/* 22 */
+	WUPDATE( w7,  w4, w15,  w9);	R2(b, c, d, e, a,  w7);		/* 23 */
+	WUPDATE( w8,  w5,  w0, w10);	R2(a, b, c, d, e,  w8);		/* 24 */
+	WUPDATE( w9,  w6,  w1, w11);	R2(e, a, b, c, d,  w9);		/* 25 */
+	WUPDATE(w10,  w7,  w2, w12);	R2(d, e, a, b, c, w10);		/* 26 */
+	WUPDATE(w11,  w8,  w3, w13);	R2(c, d, e, a, b, w11);		/* 27 */
+	WUPDATE(w12,  w9,  w4, w14);	R2(b, c, d, e, a, w12);		/* 28 */
+	WUPDATE(w13, w10,  w5, w15);	R2(a, b, c, d, e, w13);		/* 29 */
+	WUPDATE(w14, w11,  w6,  w0);	R2(e, a, b, c, d, w14);		/* 30 */
+	WUPDATE(w15, w12,  w7,  w1);	R2(d, e, a, b, c, w15);		/* 31 */
+	WUPDATE( w0, w13,  w8,  w2);	R2(c, d, e, a, b,  w0);		/* 32 */
+	WUPDATE( w1, w14,  w9,  w3);	R2(b, c, d, e, a,  w1);		/* 33 */
+	WUPDATE( w2, w15, w10,  w4);	R2(a, b, c, d, e,  w2);		/* 34 */
+	WUPDATE( w3,  w0, w11,  w5);	R2(e, a, b, c, d,  w3);		/* 35 */
+	WUPDATE( w4,  w1, w12,  w6);	R2(d, e, a, b, c,  w4);		/* 36 */
+	WUPDATE( w5,  w2, w13,  w7);	R2(c, d, e, a, b,  w5);		/* 37 */
+	WUPDATE( w6,  w3, w14,  w8);	R2(b, c, d, e, a,  w6);		/* 38 */
+	WUPDATE( w7,  w4, w15,  w9);	R2(a, b, c, d, e,  w7);		/* 39 */
+
+	/* Round 3 */
+	WUPDATE( w8,  w5,  w0, w10);	R3(e, a, b, c, d,  w8);		/* 40 */
+	WUPDATE( w9,  w6,  w1, w11);	R3(d, e, a, b, c,  w9);		/* 41 */
+	WUPDATE(w10,  w7,  w2, w12);	R3(c, d, e, a, b, w10);		/* 42 */
+	WUPDATE(w11,  w8,  w3, w13);	R3(b, c, d, e, a, w11);		/* 43 */
+	WUPDATE(w12,  w9,  w4, w14);	R3(a, b, c, d, e, w12);		/* 44 */
+	WUPDATE(w13, w10,  w5, w15);	R3(e, a, b, c, d, w13);		/* 45 */
+	WUPDATE(w14, w11,  w6,  w0);	R3(d, e, a, b, c, w14);		/* 46 */
+	WUPDATE(w15, w12,  w7,  w1);	R3(c, d, e, a, b, w15);		/* 47 */
+	WUPDATE( w0, w13,  w8,  w2);	R3(b, c, d, e, a,  w0);		/* 48 */
+	WUPDATE( w1, w14,  w9,  w3);	R3(a, b, c, d, e,  w1);		/* 49 */
+	WUPDATE( w2, w15, w10,  w4);	R3(e, a, b, c, d,  w2);		/* 50 */
+	WUPDATE( w3,  w0, w11,  w5);	R3(d, e, a, b, c,  w3);		/* 51 */
+	WUPDATE( w4,  w1, w12,  w6);	R3(c, d, e, a, b,  w4);		/* 52 */
+	WUPDATE( w5,  w2, w13,  w7);	R3(b, c, d, e, a,  w5);		/* 53 */
+	WUPDATE( w6,  w3, w14,  w8);	R3(a, b, c, d, e,  w6);		/* 54 */
+	WUPDATE( w7,  w4, w15,  w9);	R3(e, a, b, c, d,  w7);		/* 55 */
+	WUPDATE( w8,  w5,  w0, w10);	R3(d, e, a, b, c,  w8);		/* 56 */
+	WUPDATE( w9,  w6,  w1, w11);	R3(c, d, e, a, b,  w9);		/* 57 */
+	WUPDATE(w10,  w7,  w2, w12);	R3(b, c, d, e, a, w10);		/* 58 */
+	WUPDATE(w11,  w8,  w3, w13);	R3(a, b, c, d, e, w11);		/* 59 */
+
+	WUPDATE(w12,  w9,  w4, w14);	R4(e, a, b, c, d, w12);		/* 60 */
+	WUPDATE(w13, w10,  w5, w15);	R4(d, e, a, b, c, w13);		/* 61 */
+	WUPDATE(w14, w11,  w6,  w0);	R4(c, d, e, a, b, w14);		/* 62 */
+	WUPDATE(w15, w12,  w7,  w1);	R4(b, c, d, e, a, w15);		/* 63 */
+	WUPDATE( w0, w13,  w8,  w2);	R4(a, b, c, d, e,  w0);		/* 64 */
+	WUPDATE( w1, w14,  w9,  w3);	R4(e, a, b, c, d,  w1);		/* 65 */
+	WUPDATE( w2, w15, w10,  w4);	R4(d, e, a, b, c,  w2);		/* 66 */
+	WUPDATE( w3,  w0, w11,  w5);	R4(c, d, e, a, b,  w3);		/* 67 */
+	WUPDATE( w4,  w1, w12,  w6);	R4(b, c, d, e, a,  w4);		/* 68 */
+	WUPDATE( w5,  w2, w13,  w7);	R4(a, b, c, d, e,  w5);		/* 69 */
+	WUPDATE( w6,  w3, w14,  w8);	R4(e, a, b, c, d,  w6);		/* 70 */
+	WUPDATE( w7,  w4, w15,  w9);	R4(d, e, a, b, c,  w7);		/* 71 */
+	WUPDATE( w8,  w5,  w0, w10);	R4(c, d, e, a, b,  w8);		/* 72 */
+	WUPDATE( w9,  w6,  w1, w11);	R4(b, c, d, e, a,  w9);		/* 73 */
+	WUPDATE(w10,  w7,  w2, w12);	R4(a, b, c, d, e, w10);		/* 74 */
+	WUPDATE(w11,  w8,  w3, w13);	R4(e, a, b, c, d, w11);		/* 75 */
+	WUPDATE(w12,  w9,  w4, w14);	R4(d, e, a, b, c, w12);		/* 76 */
+	WUPDATE(w13, w10,  w5, w15);	R4(c, d, e, a, b, w13);		/* 77 */
+	WUPDATE(w14, w11,  w6,  w0);	R4(b, c, d, e, a, w14);		/* 78 */
+	WUPDATE(w15, w12,  w7,  w1);	R4(a, b, c, d, e, w15);		/* 79 */
+
+	context->state[0] += a;
+	context->state[1] += b;
+	context->state[2] += c;
+	context->state[3] += d;
+	context->state[4] += e;
+
+	/* Zeroize sensitive information. */
+	w15 = w14 = w13 = w12 = w11 = w10 = w9 = w8 = 0;
+	w7 = w6 = w5 = w4 = w3 = w2 = w1 = w0 = 0;
+}
diff --git a/libkern/gen/OSAtomicOperations.c b/libkern/gen/OSAtomicOperations.c
index f924b7a45..bbdb0d970 100644
--- a/libkern/gen/OSAtomicOperations.c
+++ b/libkern/gen/OSAtomicOperations.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <libkern/OSAtomic.h>
@@ -26,12 +32,15 @@ enum {
 	false	= 0,
 	true	= 1
 };
-#define	NULL 0L
+
+#ifndef NULL
+#define NULL ((void *)0)
+#endif
 
 /*
  * atomic operations
  *	these are _the_ atomic operations, currently cast atop CompareAndSwap,
- *	which is implemented in assembler.  if we are worried about the cost of
+ *	which is implemented in assembler.	if we are worried about the cost of
  *	this layering (we shouldn't be), then all this stuff could be
  *	implemented in assembler, as it is in MacOS8/9
  *	(derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
@@ -47,34 +56,23 @@ enum {
 
 #ifndef __ppc__
 
-SInt32	OSAddAtomic(SInt32 amount, SInt32 * value)
-{
-	SInt32	oldValue;
-	SInt32	newValue;
-	
-	do {
-		oldValue = *value;
-		newValue = oldValue + amount;
-	} while (! OSCompareAndSwap((UInt32) oldValue, (UInt32) newValue, (UInt32 *) value));
-	
-	return oldValue;
-}
 
-SInt32	OSIncrementAtomic(SInt32 * value)
+SInt32	OSIncrementAtomic(volatile SInt32 * value)
 {
 	return OSAddAtomic(1, value);
 }
 
-SInt32	OSDecrementAtomic(SInt32 * value)
+SInt32	OSDecrementAtomic(volatile SInt32 * value)
 {
 	return OSAddAtomic(-1, value);
 }
 
 #ifdef CMPXCHG8B
-void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
+void *	OSDequeueAtomic(void * volatile * inList, SInt32 inOffset)
 {
-	void *	oldListHead;
-	void *	newListHead;
+	/* The _pointer_ is volatile, not the listhead itself */
+	void * volatile	oldListHead;
+	void * volatile	newListHead;
 	
 	do {
 		oldListHead = *inList;
@@ -82,28 +80,29 @@ void *	OSDequeueAtomic(void ** inList, SInt32 inOffset)
 			break;
 		}
 		
-		newListHead = *(void **) (((char *) oldListHead) + inOffset);
+		newListHead = *(void * volatile *) (((char *) oldListHead) + inOffset);
 	} while (! OSCompareAndSwap((UInt32)oldListHead,
-					(UInt32)newListHead, (UInt32 *)inList));
+					(UInt32)newListHead, (volatile UInt32 *)inList));
 	return oldListHead;
 }
 
-void	OSEnqueueAtomic(void ** inList, void * inNewLink, SInt32 inOffset)
+void	OSEnqueueAtomic(void * volatile * inList, void * inNewLink, SInt32 inOffset)
 {
-	void *	oldListHead;
-	void *	newListHead = inNewLink;
-	void **	newLinkNextPtr = (void **) (((char *) inNewLink) + inOffset);
+	/* The _pointer_ is volatile, not the listhead itself */
+	void * volatile	oldListHead;
+	void * volatile	newListHead = inNewLink;
+	void * volatile * newLinkNextPtr = (void * volatile *) (((char *) inNewLink) + inOffset);
 	
 	do {
 		oldListHead = *inList;
 		*newLinkNextPtr = oldListHead;
 	} while (! OSCompareAndSwap((UInt32)oldListHead, (UInt32)newListHead,
-					(UInt32 *)inList));
+					(volatile UInt32 *)inList));
 }
 #endif /* CMPXCHG8B */
 #endif	/* !__ppc__ */
 
-static UInt32	OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt32 * value)
+static UInt32	OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
 {
 	UInt32	oldValue;
 	UInt32	newValue;
@@ -116,41 +115,41 @@ static UInt32	OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask,
 	return oldValue;
 }
 
-UInt32	OSBitAndAtomic(UInt32 mask, UInt32 * value)
+UInt32	OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
 {
 	return OSBitwiseAtomic(mask, 0, 0, value);
 }
 
-UInt32	OSBitOrAtomic(UInt32 mask, UInt32 * value)
+UInt32	OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
 {
 	return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
 }
 
-UInt32	OSBitXorAtomic(UInt32 mask, UInt32 * value)
+UInt32	OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
 {
 	return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
 }
 
-static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, UInt8 * value8)
+static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
 {
-	UInt32		mask        = 0x000000ff;
-	UInt32		alignment   = ((UInt32) value8) & (sizeof(UInt32) - 1);
-    UInt32      shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
-    int			shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
-	UInt32 *	value32     = (UInt32 *) (value8 - alignment);
-    UInt32      oldValue;
-    UInt32      newValue;
+	UInt32				mask		= 0x000000ff;
+	UInt32				alignment	= ((UInt32) value8) & (sizeof(UInt32) - 1);
+	UInt32				shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
+	int					shift		= (UInt32) *(((UInt8 *) &shiftValues) + alignment);
+	volatile UInt32 *	value32		= (volatile UInt32 *) (value8 - alignment);
+	UInt32				oldValue;
+	UInt32				newValue;
 
-    mask <<= shift;
+	mask <<= shift;
 
-    oldValue = *value32;
-    oldValue = (oldValue & ~mask) | (oldValue8 << shift);
-    newValue = (oldValue & ~mask) | (newValue8 << shift);
+	oldValue = *value32;
+	oldValue = (oldValue & ~mask) | (oldValue8 << shift);
+	newValue = (oldValue & ~mask) | (newValue8 << shift);
 
 	return OSCompareAndSwap(oldValue, newValue, value32);
 }
 
-static Boolean	OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddress)
+static Boolean	OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
 {
 	UInt8		mask = 1;
 	UInt8		oldValue;
@@ -170,12 +169,12 @@ static Boolean	OSTestAndSetClear(UInt32 bit, Boolean wantSet, UInt8 * startAddre
 	return (oldValue & mask) == wantValue;
 }
 
-Boolean	OSTestAndSet(UInt32 bit, UInt8 * startAddress)
+Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
 {
 	return OSTestAndSetClear(bit, true, startAddress);
 }
 
-Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress)
+Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
 {
 	return OSTestAndSetClear(bit, false, startAddress);
 }
@@ -184,17 +183,17 @@ Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress)
  * silly unaligned versions
  */
 
-SInt8	OSIncrementAtomic8(SInt8 * value)
+SInt8	OSIncrementAtomic8(volatile SInt8 * value)
 {
 	return OSAddAtomic8(1, value);
 }
 
-SInt8	OSDecrementAtomic8(SInt8 * value)
+SInt8	OSDecrementAtomic8(volatile SInt8 * value)
 {
 	return OSAddAtomic8(-1, value);
 }
 
-SInt8	OSAddAtomic8(SInt32 amount, SInt8 * value)
+SInt8	OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
 {
 	SInt8	oldValue;
 	SInt8	newValue;
@@ -202,12 +201,12 @@ SInt8	OSAddAtomic8(SInt32 amount, SInt8 * value)
 	do {
 		oldValue = *value;
 		newValue = oldValue + amount;
-	} while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (UInt8 *) value));
+	} while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
 	
 	return oldValue;
 }
 
-static UInt8	OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt8 * value)
+static UInt8	OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
 {
 	UInt8	oldValue;
 	UInt8	newValue;
@@ -220,51 +219,51 @@ static UInt8	OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask,
 	return oldValue;
 }
 
-UInt8	OSBitAndAtomic8(UInt32 mask, UInt8 * value)
+UInt8	OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
 {
 	return OSBitwiseAtomic8(mask, 0, 0, value);
 }
 
-UInt8	OSBitOrAtomic8(UInt32 mask, UInt8 * value)
+UInt8	OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
 {
 	return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
 }
 
-UInt8	OSBitXorAtomic8(UInt32 mask, UInt8 * value)
+UInt8	OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
 {
 	return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
 }
 
-static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, UInt16 * value16)
+static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
 {
-	UInt32		mask        = 0x0000ffff;
-	UInt32		alignment   = ((UInt32) value16) & (sizeof(UInt32) - 1);
-    UInt32      shiftValues = (16 << 24) | (16 << 16);
-    UInt32		shift       = (UInt32) *(((UInt8 *) &shiftValues) + alignment);
-	UInt32 *	value32     = (UInt32 *) (((UInt32) value16) - alignment);
-    UInt32      oldValue;
-    UInt32      newValue;
+	UInt32				mask		= 0x0000ffff;
+	UInt32				alignment	= ((UInt32) value16) & (sizeof(UInt32) - 1);
+	UInt32				shiftValues = (16 << 24) | (16 << 16);
+	UInt32				shift		= (UInt32) *(((UInt8 *) &shiftValues) + alignment);
+	volatile UInt32 *	value32		= (volatile UInt32 *) (((UInt32) value16) - alignment);
+	UInt32				oldValue;
+	UInt32				newValue;
 
-    mask <<= shift;
+	mask <<= shift;
 
-    oldValue = *value32;
-    oldValue = (oldValue & ~mask) | (oldValue16 << shift);
-    newValue = (oldValue & ~mask) | (newValue16 << shift);
+	oldValue = *value32;
+	oldValue = (oldValue & ~mask) | (oldValue16 << shift);
+	newValue = (oldValue & ~mask) | (newValue16 << shift);
 
 	return OSCompareAndSwap(oldValue, newValue, value32);
 }
 
-SInt16	OSIncrementAtomic16(SInt16 * value)
+SInt16	OSIncrementAtomic16(volatile SInt16 * value)
 {
 	return OSAddAtomic16(1, value);
 }
 
-SInt16	OSDecrementAtomic16(SInt16 * value)
+SInt16	OSDecrementAtomic16(volatile SInt16 * value)
 {
 	return OSAddAtomic16(-1, value);
 }
 
-SInt16	OSAddAtomic16(SInt32 amount, SInt16 * value)
+SInt16	OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
 {
 	SInt16	oldValue;
 	SInt16	newValue;
@@ -272,12 +271,12 @@ SInt16	OSAddAtomic16(SInt32 amount, SInt16 * value)
 	do {
 		oldValue = *value;
 		newValue = oldValue + amount;
-	} while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (UInt16 *) value));
+	} while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
 	
 	return oldValue;
 }
 
-static UInt16	OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, UInt16 * value)
+static UInt16	OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
 {
 	UInt16	oldValue;
 	UInt16	newValue;
@@ -290,17 +289,17 @@ static UInt16	OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask
 	return oldValue;
 }
 
-UInt16	OSBitAndAtomic16(UInt32 mask, UInt16 * value)
+UInt16	OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
 {
 	return OSBitwiseAtomic16(mask, 0, 0, value);
 }
 
-UInt16	OSBitOrAtomic16(UInt32 mask, UInt16 * value)
+UInt16	OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
 {
 	return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
 }
 
-UInt16	OSBitXorAtomic16(UInt32 mask, UInt16 * value)
+UInt16	OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
 {
 	return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
 }
diff --git a/libkern/gen/OSDebug.cpp b/libkern/gen/OSDebug.cpp
index d8a8c2842..43544373f 100644
--- a/libkern/gen/OSDebug.cpp
+++ b/libkern/gen/OSDebug.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 // NOTE:  This file is only c++ so I can get static initialisers going
@@ -33,6 +39,8 @@
 #include <libkern/libkern.h>	// From bsd's libkern directory
 #include <mach/vm_param.h>
 
+#include <sys/kdebug.h>
+extern int etext;
 __BEGIN_DECLS
 // From osmfk/kern/thread.h but considered to be private
 extern vm_offset_t min_valid_stack_address(void);
@@ -42,10 +50,42 @@ extern vm_offset_t max_valid_stack_address(void);
 extern void kmod_dump_log(vm_offset_t *addr, unsigned int cnt);
 
 extern addr64_t kvtophys(vm_offset_t va);
+#if __arm__
+extern int copyinframe(vm_address_t fp, uint32_t *frame);
+#endif
+
 __END_DECLS
 
 static mutex_t *sOSReportLock = mutex_alloc(0);
 
+/* Use kernel_debug() to log a backtrace */ 
+void
+trace_backtrace(unsigned int debugid, unsigned int debugid2, int size, int data) {
+	void *bt[16];
+	const unsigned cnt = sizeof(bt) / sizeof(bt[0]);
+  	unsigned i;
+	int found = 0;
+
+	OSBacktrace(bt, cnt);	
+  
+	/* find first non-kernel frame */
+  	for (i = 3; i < cnt && bt[i]; i++) {
+ 		if (bt[i] > (void*)&etext) {
+			found = 1;
+  			break;
+		}
+	}
+	/* 
+	 * if there are non-kernel frames, only log these
+	 * otherwise, log everything but the first two
+	 */
+	if (!found) i=2;
+
+#define safe_bt(a) (int)(a<cnt ? bt[a] : 0)
+	kernel_debug(debugid, data, size, safe_bt(i), safe_bt(i+1), 0);
+	kernel_debug(debugid2, safe_bt(i+2), safe_bt(i+3), safe_bt(i+4), safe_bt(i+5), 0);
+}
+
 /* Report a message with a 4 entry backtrace - very slow */
 void
 OSReportWithBacktrace(const char *str, ...)
@@ -182,6 +222,30 @@ pad:
 
     for ( ; frame_index < maxAddrs; frame_index++)
 	    bt[frame_index] = (void *) 0;
+#elif __arm__
+    uint32_t i= 0;
+    uint32_t frameb[2];
+    uint32_t fp= 0;
+    
+    // get the current frame pointer for this thread
+    __asm__ volatile("mov %0,r7" : "=r" (fp)); 
+    
+    // now crawl up the stack recording the link value of each frame
+    do {
+      // check bounds
+      if ((fp == 0) || ((fp & 3) != 0) || (fp > VM_MAX_KERNEL_ADDRESS) || (fp < VM_MIN_KERNEL_ADDRESS)) {
+	break;
+      }
+      // safely read frame
+      if (copyinframe(fp, frameb) != 0) {
+	break;
+      }
+      
+      // No need to use copyin as this is always a kernel address, see check above
+      bt[i] = (void*)frameb[1];        // link register
+      fp = frameb[0]; 
+    } while (++i < maxAddrs);
+    frame= i;
 #else
 #error arch
 #endif
diff --git a/libkern/i386/OSAtomic.c b/libkern/i386/OSAtomic.c
deleted file mode 100644
index c5904c3ea..000000000
--- a/libkern/i386/OSAtomic.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
- /*
- * Copyright (c) 1999 Apple Computer, Inc.
- *
- *
- * HISTORY
- *
- * wgulland  23 Jul 99 - Bogus implementation!
- */
-#include <libkern/OSAtomic.h>
-
-#warning Bogus OSCompareAndSwap() for i386
-int OSCompareAndSwap( UInt32 oldVal,
-	UInt32 newVal, UInt32 * addr )
-{
-    /*
-     * This should be
-     * EAX = oldVal
-     * Lock:CMPXCHG addr, newVal
-     * But I don't know how to write that.
-     */
-    int ok;
-    if( (ok = (oldVal == *addr)))
-        *addr = newVal;
-    return( ok );
-}
-
diff --git a/libkern/i386/OSAtomic.s b/libkern/i386/OSAtomic.s
index 68b10523e..2be21d1a8 100644
--- a/libkern/i386/OSAtomic.s
+++ b/libkern/i386/OSAtomic.s
@@ -1,28 +1,34 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#;**************************************************************************
-#;* Boolean OSCompareAndSwap(SInt32 oldValue, SInt32 newValue, SInt32 * ptr) *
-#;**************************************************************************
+#;***************************************************************************
+#;* Boolean OSCompareAndSwap(SInt32 oldValue, SInt32 newValue, SInt32 *ptr) *
+#;***************************************************************************
 
 	.globl _OSCompareAndSwap
 
@@ -32,6 +38,67 @@ _OSCompareAndSwap:
 	movl		12(%esp), %ecx	#; ptr
 	lock
 	cmpxchgl	%edx, 0(%ecx)	#; CAS (eax is an implicit operand)
-	sete		%al				#; did CAS succeed? (TZ=1)
+	sete		%al			#; did CAS succeed? (TZ=1)
 	movzbl		%al, %eax		#; clear out the high bytes
 	ret
+
+#;*****************************************************************************
+#;* Boolean OSCompareAndSwap64(SInt64 oldValue, SInt64 newValue, SInt64 *ptr) *
+#;*****************************************************************************
+
+	.globl _OSCompareAndSwap64
+
+_OSCompareAndSwap64:
+	pushl		%edi
+	pushl		%ebx
+
+	movl		 4+8(%esp), %eax	#; low 32-bits of oldValue
+	movl		 8+8(%esp), %edx	#; high 32-bits of oldValue
+	movl		12+8(%esp), %ebx	#; low 32-bits of newValue
+	movl		16+8(%esp), %ecx	#; high 32-bits of newValue
+	movl		20+8(%esp), %edi	#; ptr
+	lock
+	cmpxchg8b	0(%edi)		#; CAS (eax:edx, ebx:ecx implicit)
+	sete		%al			#; did CAS succeed? (TZ=1)
+	movzbl		%al, %eax		#; clear out the high bytes
+
+	popl		%ebx
+	popl		%edi
+	ret
+
+#;*******************************************************
+#;* SInt64 OSAddAtomic64(SInt64 theAmount, SInt64 *ptr) *
+#;*******************************************************
+
+	.globl	_OSAddAtomic64
+_OSAddAtomic64:
+	pushl		%edi
+	pushl		%ebx
+
+	movl		12+8(%esp), %edi	#; ptr
+	movl		0(%edi), %eax		#; load low 32-bits of *ptr
+	movl		4(%edi), %edx		#; load high 32-bits of *ptr
+1:
+	movl		%eax, %ebx		
+	movl		%edx, %ecx		#; ebx:ecx := *ptr
+	addl		4+8(%esp), %ebx
+	adcl		8+8(%esp), %ecx		#; ebx:ecx := *ptr + theAmount
+	lock
+	cmpxchg8b	0(%edi)		#; CAS (eax:edx, ebx:ecx implicit)
+	jnz		1b		#; - failure: eax:edx re-loaded, retry
+					#; - success: old value in eax:edx
+	popl		%ebx
+	popl		%edi
+	ret
+
+#;*******************************************************
+#; SInt32 OSAddAtomic(SInt32 delta, SInt32 *address) 
+#;*******************************************************
+
+	.globl	_OSAddAtomic
+_OSAddAtomic:
+	movl	4(%esp), %eax		#; Load addend
+	movl	8(%esp), %ecx		#; Load address of operand
+	lock
+	xaddl	%eax, 0(%ecx)		#; Atomic exchange and add
+	ret
diff --git a/libkern/kmod/Makefile b/libkern/kmod/Makefile
index 3bad6a04f..7a50b88bd 100644
--- a/libkern/kmod/Makefile
+++ b/libkern/kmod/Makefile
@@ -7,29 +7,33 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
-COMPOBJROOT = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
 do_all: 
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(MACH_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";		\
+	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";		\
 	($(MKDIR) $(COMPOBJROOT)/kmod;					\
 	cd $(COMPOBJROOT)/kmod;						\
 	${MAKE} MAKEFILES=$(SOURCE)/Makefile.kmod			\
 		TARGET=$(TARGET)					\
 		do_build_all						\
 	);								\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(MACH_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
 
 do_build_all:	do_all
 
 do_install: 
-	@echo "[ $(SOURCE) ] Starting do_install  $(COMPONENT) $(MACH_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";		\
+	@echo "[ $(SOURCE) ] Starting do_install  $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";		\
 	($(MKDIR) $(COMPOBJROOT)/kmod;					\
 	cd $(COMPOBJROOT)/kmod;						\
 	${MAKE} MAKEFILES=$(SOURCE)/Makefile.kmod			\
 		TARGET=$(TARGET)					\
 		do_build_install					\
 	);								\
-	echo "[ $(SOURCE) ] Returning do_install $(COMPONENT) $(MACH_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+	echo "[ $(SOURCE) ] Returning do_install $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
 
 do_build_install:	do_install
 
diff --git a/libkern/kmod/Makefile.kmod b/libkern/kmod/Makefile.kmod
index 4adb8c5fb..fcc97f211 100644
--- a/libkern/kmod/Makefile.kmod
+++ b/libkern/kmod/Makefile.kmod
@@ -18,9 +18,16 @@ LIB_INSTALL_FLAGS = 	-p -m 444
 
 CFLAGS += -Wall -Wno-four-char-constants
 CFLAGS_PPC += -mlong-branch
+CFLAGS_ARM += -mlong-calls
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+COMPOBJROOT = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/kmod
+INSTOBJROOT = $(OBJROOT)/$(INSTALL_TYPE)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/kmod
+else
 COMPOBJROOT = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/kmod
 INSTOBJROOT = $(OBJROOT)/$(INSTALL_TYPE)_$(ARCH_CONFIG)/$(COMPONENT)/kmod
+endif
+
 
 KMOD_CFILES    = c_start.c c_stop.c
 KMODCPP_CFILES = cplus_start.c cplus_stop.c
@@ -44,14 +51,18 @@ do_build_all: $(COMPOBJROOT)/$(KMOD_NAME).a $(COMPOBJROOT)/$(KMODCPP_NAME).a
 $(INSTALL_DIR)/%.a: $(INSTOBJROOT)/%.a
 	@allarchs="";							\
 	for onearch in $(INSTALL_ARCHS); do				\
-            archdir=$(OBJROOT)/$(KERNEL_CONFIG)_$$onearch/$(COMPONENT);	\
-	    if [ -e $$archdir/kmod/$(*F).a ]; then			\
-		allarchs="$$allarchs $$archdir/kmod/$(*F).a";		\
+		if [ $(MACHINE_CONFIG) = DEFAULT ] ; then	\
+			archdir=$(OBJROOT)/$(KERNEL_CONFIG)_$${onearch}/$(COMPONENT);	\
+		else		\
+			archdir=$(OBJROOT)/$(KERNEL_CONFIG)_$${onearch}_$(MACHINE_CONFIG)/$(COMPONENT);	\
+		fi;		\
+	    if [ -e $${archdir}/kmod/$(*F).a ]; then			\
+		allarchs="$${allarchs} $${archdir}/kmod/$(*F).a";		\
 	    fi;								\
 	done;								\
 	$(RM) $@ || true;						\
 	${MKDIR} $(INSTALL_DIR) $(SYMROOT);				\
-        cmd="lipo $$allarchs -create -output $(SYMROOT)/$(*F).a";	\
+        cmd="lipo $${allarchs} -create -output $(SYMROOT)/$(*F).a";	\
 	echo $$cmd; eval $$cmd;						\
 	cmd="install $(LIB_INSTALL_FLAGS) $(SYMROOT)/$(*F).a $@";	\
 	echo $$cmd; eval $$cmd
diff --git a/libkern/kmod/c_start.c b/libkern/kmod/c_start.c
index ad33f01f4..720a70926 100644
--- a/libkern/kmod/c_start.c
+++ b/libkern/kmod/c_start.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
     Subtle combination of files and libraries make up the C++ runtime system for kernel modules.  We are dependant on the KernelModule kmod.make and CreateKModInfo.perl scripts to be exactly instep with both this library module and the libkmod module as well.
diff --git a/libkern/kmod/c_stop.c b/libkern/kmod/c_stop.c
index 3de526ad5..d71816041 100644
--- a/libkern/kmod/c_stop.c
+++ b/libkern/kmod/c_stop.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
     Subtle combination of files and libraries make up the C++ runtime system for kernel modules.  We are dependant on the KernelModule kmod.make and CreateKModInfo.perl scripts to be exactly instep with both this library module and the libkmod module as well.
diff --git a/libkern/kmod/cplus_start.c b/libkern/kmod/cplus_start.c
index 81bc3adc0..3bfb64198 100644
--- a/libkern/kmod/cplus_start.c
+++ b/libkern/kmod/cplus_start.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
     Subtle combination of files and libraries make up the C++ runtime system for kernel modules.  We are dependant on the KernelModule kmod.make and CreateKModInfo.perl scripts to be exactly instep with both this library module and the libkmod module as well.
diff --git a/libkern/kmod/cplus_stop.c b/libkern/kmod/cplus_stop.c
index 8151fb43f..fffd04044 100644
--- a/libkern/kmod/cplus_stop.c
+++ b/libkern/kmod/cplus_stop.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
     Subtle combination of files and libraries make up the C++ runtime system for kernel modules.  We are dependant on the KernelModule kmod.make and CreateKModInfo.perl scripts to be exactly instep with both this library module and the libkmod module as well.
diff --git a/libkern/libkern/Makefile b/libkern/libkern/Makefile
index 41c7ce91f..f9f411672 100644
--- a/libkern/libkern/Makefile
+++ b/libkern/libkern/Makefile
@@ -9,7 +9,8 @@ include $(MakeInc_def)
 
 INSTINC_SUBDIRS = \
         machine \
-	c++
+	c++ \
+	crypto
 
 INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS} \
         ppc
@@ -17,22 +18,28 @@ INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS} \
 INSTINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS} \
         i386
 
+INSTINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS} \
+        arm
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
 
 DATAFILES = \
 	OSAtomic.h	\
         OSBase.h	\
         OSByteOrder.h	\
+        _OSByteOrder.h	\
         OSDebug.h	\
         OSMalloc.h	\
         OSReturn.h	\
         OSTypes.h	\
 	locks.h		\
-	sysctl.h
+	sysctl.h	\
+	zlib.h
 
-INSTALL_MI_LIST	= OSByteOrder.h OSDebug.h OSReturn.h OSTypes.h 
+INSTALL_MI_LIST	= OSByteOrder.h _OSByteOrder.h OSDebug.h OSReturn.h OSTypes.h 
 
 INSTALL_MI_DIR = libkern
 
@@ -48,8 +55,8 @@ NEWVERS = $(SRCROOT)/config/newvers.pl
 
 version.h: version.h.template $(SRCROOT)/config/MasterVersion
 	@echo "Generating libkern/$@ from $<";
-	install $(DATA_INSTALL_FLAGS) $< $@
-	$(NEWVERS) $@;
+	$(_v)install $(DATA_INSTALL_FLAGS) $< $@
+	$(_v)$(NEWVERS) $@ > /dev/null;
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/libkern/libkern/OSAtomic.h b/libkern/libkern/OSAtomic.h
index 1870272b5..19fe32cc4 100644
--- a/libkern/libkern/OSAtomic.h
+++ b/libkern/libkern/OSAtomic.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
@@ -35,6 +41,41 @@
 extern "C" {
 #endif
 
+#if defined(__i386__)
+
+/*! @function OSCompareAndSwap64
+    @abstract 64-bit compare and swap operation.
+    @discussion See OSCompareAndSwap.
+*/
+extern Boolean OSCompareAndSwap64(UInt64 oldValue, UInt64 newValue,
+				   volatile UInt64 *address);
+
+/*! @function OSAddAtomic64
+    @abstract 64-bit atomic add operation.
+    @discussion See OSAddAtomic.
+*/
+extern SInt64 OSAddAtomic64(SInt64 theAmount, volatile SInt64 *address);
+
+/*! @function OSIncrementAtomic64
+    @abstract 64-bit increment.
+    @discussion See OSIncrementAtomic.
+*/
+inline static SInt64 OSIncrementAtomic64(volatile SInt64 *address)
+{
+	return OSAddAtomic64(1, address);
+}
+
+/*! @function OSDecrementAtomic64
+    @abstract 64-bit decrement.
+    @discussion See OSDecrementAtomic.
+*/
+inline static SInt64 OSDecrementAtomic64(volatile SInt64 *address)
+{
+	return OSAddAtomic64(-1, address);
+}
+
+#endif  /* defined(__i386__) */
+
 /*! @function OSCompareAndSwap
     @abstract Compare and swap operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
     @discussion The OSCompareAndSwap function compares the value at the specified address with oldVal. The value of newValue is written to the address only if oldValue and the value at the address are equal. OSCompareAndSwap returns true if newValue is written to the address; otherwise, it returns false.
@@ -45,7 +86,8 @@ extern "C" {
     @param address The 4-byte aligned address of the data to update atomically.
     @result true if newValue was written to the address. */
 
-extern Boolean OSCompareAndSwap( UInt32 oldValue, UInt32 newValue, UInt32 * address );
+extern Boolean OSCompareAndSwap(UInt32 oldValue, UInt32 newValue,
+				volatile UInt32 *address);
 
 /*! @function OSAddAtomic
     @abstract 32-bit add operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -56,7 +98,7 @@ extern Boolean OSCompareAndSwap( UInt32 oldValue, UInt32 newValue, UInt32 * addr
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the addition */
 
-extern SInt32	OSAddAtomic(SInt32 amount, SInt32 * address);
+extern SInt32	OSAddAtomic(SInt32 amount, volatile SInt32 * address);
 
 /*! @function OSAddAtomic16
     @abstract 16-bit add operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -67,7 +109,7 @@ extern SInt32	OSAddAtomic(SInt32 amount, SInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the addition */
 
-extern SInt16	OSAddAtomic16(SInt32 amount, SInt16 * address);
+extern SInt16	OSAddAtomic16(SInt32 amount, volatile SInt16 * address);
 
 /*! @function OSAddAtomic8
     @abstract 8-bit add operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -78,7 +120,7 @@ extern SInt16	OSAddAtomic16(SInt32 amount, SInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the addition */
 
-extern SInt8	OSAddAtomic8(SInt32 amount, SInt8 * address);
+extern SInt8	OSAddAtomic8(SInt32 amount, volatile SInt8 * address);
 
 /*! @function OSIncrementAtomic
     @abstract 32-bit increment operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -88,7 +130,7 @@ extern SInt8	OSAddAtomic8(SInt32 amount, SInt8 * address);
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the increment. */
 
-extern SInt32	OSIncrementAtomic(SInt32 * address);
+extern SInt32	OSIncrementAtomic(volatile SInt32 * address);
 
 /*! @function OSIncrementAtomic16
     @abstract 16-bit increment operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -98,7 +140,7 @@ extern SInt32	OSIncrementAtomic(SInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the increment. */
 
-extern SInt16	OSIncrementAtomic16(SInt16 * address);
+extern SInt16	OSIncrementAtomic16(volatile SInt16 * address);
 
 /*! @function OSIncrementAtomic8
     @abstract 8-bit increment operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -108,7 +150,7 @@ extern SInt16	OSIncrementAtomic16(SInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the increment. */
 
-extern SInt8	OSIncrementAtomic8(SInt8 * address);
+extern SInt8	OSIncrementAtomic8(volatile SInt8 * address);
 
 /*! @function OSDecrementAtomic
     @abstract 32-bit decrement operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -118,7 +160,7 @@ extern SInt8	OSIncrementAtomic8(SInt8 * address);
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the decrement. */
 
-extern SInt32	OSDecrementAtomic(SInt32 * address);
+extern SInt32	OSDecrementAtomic(volatile SInt32 * address);
 
 /*! @function OSDecrementAtomic16
     @abstract 16-bit decrement operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -128,7 +170,7 @@ extern SInt32	OSDecrementAtomic(SInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the decrement. */
 
-extern SInt16	OSDecrementAtomic16(SInt16 * address);
+extern SInt16	OSDecrementAtomic16(volatile SInt16 * address);
 
 /*! @function OSDecrementAtomic8
     @abstract 8-bit decrement operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -138,7 +180,7 @@ extern SInt16	OSDecrementAtomic16(SInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the decrement. */
 
-extern SInt8	OSDecrementAtomic8(SInt8 * address);
+extern SInt8	OSDecrementAtomic8(volatile SInt8 * address);
 
 /*! @function OSBitAndAtomic
     @abstract 32-bit logical and operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -149,7 +191,7 @@ extern SInt8	OSDecrementAtomic8(SInt8 * address);
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation */
 
-extern UInt32	OSBitAndAtomic(UInt32 mask, UInt32 * address);
+extern UInt32	OSBitAndAtomic(UInt32 mask, volatile UInt32 * address);
 
 /*! @function OSBitAndAtomic16
     @abstract 16-bit logical and operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -160,7 +202,7 @@ extern UInt32	OSBitAndAtomic(UInt32 mask, UInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt16	OSBitAndAtomic16(UInt32 mask, UInt16 * address);
+extern UInt16	OSBitAndAtomic16(UInt32 mask, volatile UInt16 * address);
 
 /*! @function OSBitAndAtomic8
     @abstract 8-bit logical and operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -171,7 +213,7 @@ extern UInt16	OSBitAndAtomic16(UInt32 mask, UInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt8	OSBitAndAtomic8(UInt32 mask, UInt8 * address);
+extern UInt8	OSBitAndAtomic8(UInt32 mask, volatile UInt8 * address);
 
 /*! @function OSBitOrAtomic
     @abstract 32-bit logical or operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -182,7 +224,7 @@ extern UInt8	OSBitAndAtomic8(UInt32 mask, UInt8 * address);
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt32	OSBitOrAtomic(UInt32 mask, UInt32 * address);
+extern UInt32	OSBitOrAtomic(UInt32 mask, volatile UInt32 * address);
 
 /*! @function OSBitOrAtomic16
     @abstract 16-bit logical or operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -193,7 +235,7 @@ extern UInt32	OSBitOrAtomic(UInt32 mask, UInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt16	OSBitOrAtomic16(UInt32 mask, UInt16 * address);
+extern UInt16	OSBitOrAtomic16(UInt32 mask, volatile UInt16 * address);
 
 /*! @function OSBitOrAtomic8
     @abstract 8-bit logical or operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -204,7 +246,7 @@ extern UInt16	OSBitOrAtomic16(UInt32 mask, UInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt8	OSBitOrAtomic8(UInt32 mask, UInt8 * address);
+extern UInt8	OSBitOrAtomic8(UInt32 mask, volatile UInt8 * address);
 
 /*! @function OSBitXorAtomic
     @abstract 32-bit logical xor operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -215,7 +257,7 @@ extern UInt8	OSBitOrAtomic8(UInt32 mask, UInt8 * address);
     @param address The 4-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt32	OSBitXorAtomic(UInt32 mask, UInt32 * address);
+extern UInt32	OSBitXorAtomic(UInt32 mask, volatile UInt32 * address);
 
 /*! @function OSBitXorAtomic16
     @abstract 16-bit logical xor operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -226,7 +268,7 @@ extern UInt32	OSBitXorAtomic(UInt32 mask, UInt32 * address);
     @param address The 2-byte aligned address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt16	OSBitXorAtomic16(UInt32 mask, UInt16 * address);
+extern UInt16	OSBitXorAtomic16(UInt32 mask, volatile UInt16 * address);
 
 /*! @function OSBitXorAtomic8
     @abstract 8-bit logical xor operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -237,7 +279,7 @@ extern UInt16	OSBitXorAtomic16(UInt32 mask, UInt16 * address);
     @param address The address of the value to update atomically.
     @result The value before the bitwise operation. */
 
-extern UInt8	OSBitXorAtomic8(UInt32 mask, UInt8 * address);
+extern UInt8	OSBitXorAtomic8(UInt32 mask, volatile UInt8 * address);
 
 /*! @function OSTestAndSet
     @abstract Bit test and set operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -248,7 +290,7 @@ extern UInt8	OSBitXorAtomic8(UInt32 mask, UInt8 * address);
     @param address The address of the byte to update atomically.
     @result true if the bit was already set, false otherwise. */
 
-extern Boolean	OSTestAndSet(UInt32 bit, UInt8 * startAddress);
+extern Boolean	OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress);
 
 /*! @function OSTestAndClear
     @abstract Bit test and clear operation, performed atomically with respect to all devices that participate in the coherency architecture of the platform.
@@ -259,7 +301,7 @@ extern Boolean	OSTestAndSet(UInt32 bit, UInt8 * startAddress);
     @param address The address of the byte to update atomically.
     @result true if the bit was already clear, false otherwise. */
 
-extern Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress);
+extern Boolean	OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress);
 
 #ifdef __ppc__
 /*! @function OSEnqueueAtomic
@@ -271,7 +313,7 @@ extern Boolean	OSTestAndClear(UInt32 bit, UInt8 * startAddress);
     @param element The list element to insert at the head of the list.
     @param elementNextFieldOffset The byte offset into the element where a pointer to the next element in the list is stored. */
 
-extern void	OSEnqueueAtomic(void ** listHead, void * element,
+extern void	OSEnqueueAtomic(void * volatile * listHead, void * element,
 				SInt32 elementNextFieldOffset);
 
 /*! @function OSDequeueAtomic
@@ -283,7 +325,7 @@ extern void	OSEnqueueAtomic(void ** listHead, void * element,
     @param elementNextFieldOffset The byte offset into the element where a pointer to the next element in the list is stored.
     @result A removed element, or zero if the list is empty. */
 
-extern void *	OSDequeueAtomic(void ** listHead,
+extern void *	OSDequeueAtomic(void * volatile * listHead,
 				SInt32 elementNextFieldOffset);
 #endif /* __ppc__ */
 
@@ -291,12 +333,20 @@ extern void *	OSDequeueAtomic(void ** listHead,
     @abstract The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices.
     @discussion The OSSynchronizeIO routine ensures orderly load and store operations to noncached memory mapped I/O devices. It executes the eieio instruction on PowerPC processors. */
 
+#if defined(__arm__) && defined(__thumb__)
+extern void	OSSynchronizeIO(void);
+#else
 static __inline__ void OSSynchronizeIO(void)
 {
 #if defined(__ppc__)
         __asm__ ("eieio");
 #endif
+#if defined(__arm__)
+	UInt32 temp = 0;
+	__asm__ volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (temp));
+#endif
 }
+#endif
 
 #if defined(__cplusplus)
 }
diff --git a/libkern/libkern/OSBase.h b/libkern/libkern/OSBase.h
index 2746f19af..651ba8728 100644
--- a/libkern/libkern/OSBase.h
+++ b/libkern/libkern/OSBase.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
diff --git a/libkern/libkern/OSByteOrder.h b/libkern/libkern/OSByteOrder.h
index f9b3a6a83..7b96a2264 100644
--- a/libkern/libkern/OSByteOrder.h
+++ b/libkern/libkern/OSByteOrder.h
@@ -1,50 +1,41 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _OS_OSBYTEORDER_H
 #define _OS_OSBYTEORDER_H
 
 #include <stdint.h>
+#include <libkern/_OSByteOrder.h>
 
 /* Macros for swapping constant values in the preprocessing stage. */
-#define OSSwapConstInt16(x) \
-    ((uint16_t)((((uint16_t)(x) & 0xff00) >> 8) | \
-                (((uint16_t)(x) & 0x00ff) << 8)))
-
-#define OSSwapConstInt32(x) \
-    ((uint32_t)((((uint32_t)(x) & 0xff000000) >> 24) | \
-                (((uint32_t)(x) & 0x00ff0000) >>  8) | \
-                (((uint32_t)(x) & 0x0000ff00) <<  8) | \
-                (((uint32_t)(x) & 0x000000ff) << 24)))
-
-#define OSSwapConstInt64(x) \
-    ((uint64_t)((((uint64_t)(x) & 0xff00000000000000ULL) >> 56) | \
-                (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
-                (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
-                (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
-                (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
-                (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
-                (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
-                (((uint64_t)(x) & 0x00000000000000ffULL) << 56)))
+#define OSSwapConstInt16(x)	__DARWIN_OSSwapConstInt16(x)
+#define OSSwapConstInt32(x)	__DARWIN_OSSwapConstInt32(x)
+#define OSSwapConstInt64(x)	__DARWIN_OSSwapConstInt64(x)
 
 #if defined(__GNUC__)
 
@@ -52,31 +43,22 @@
 #include <libkern/ppc/OSByteOrder.h>
 #elif (defined(__i386__) || defined(__x86_64__))
 #include <libkern/i386/OSByteOrder.h>
+#elif defined(__arm__)
+#include <libkern/arm/OSByteOrder.h>
 #else
 #include <libkern/machine/OSByteOrder.h>
 #endif
 
-#define OSSwapInt16(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt16(x) : _OSSwapInt16(x))
-
-#define OSSwapInt32(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt32(x) : _OSSwapInt32(x))
-
-#define OSSwapInt64(x) \
-    (__builtin_constant_p(x) ? OSSwapConstInt64(x) : _OSSwapInt64(x))
-
 #else /* ! __GNUC__ */
 
 #include <libkern/machine/OSByteOrder.h>
 
-#define OSSwapInt16(x) OSSwapConstInt16(x)
-
-#define OSSwapInt32(x) OSSwapConstInt32(x)
-
-#define OSSwapInt64(x) OSSwapConstInt64(x)
-
 #endif /* __GNUC__ */
 
+#define OSSwapInt16(x)	__DARWIN_OSSwapInt16(x)
+#define OSSwapInt32(x)	__DARWIN_OSSwapInt32(x)
+#define OSSwapInt64(x)	__DARWIN_OSSwapInt64(x)
+
 enum {
     OSUnknownByteOrder,
     OSLittleEndian,
diff --git a/libkern/libkern/OSCrossEndian.h b/libkern/libkern/OSCrossEndian.h
index 0b1e5aa18..0bbbf58e3 100644
--- a/libkern/libkern/OSCrossEndian.h
+++ b/libkern/libkern/OSCrossEndian.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/libkern/libkern/OSDebug.h b/libkern/libkern/OSDebug.h
index 4435dcdc3..afa0af160 100644
--- a/libkern/libkern/OSDebug.h
+++ b/libkern/libkern/OSDebug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
@@ -33,10 +39,20 @@
 
 __BEGIN_DECLS
 
+extern int log_leaks;
+
+/* Use kernel_debug() to log a backtrace */ 
+extern void trace_backtrace(unsigned int debugid, unsigned int debugid2, int size, int data);
 /* Report a message with a 4 entry backtrace - very slow */
 extern void OSReportWithBacktrace(const char *str, ...);
 extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
 
 __END_DECLS
 
+#define TRACE_MACHLEAKS(a,b,c,d)	\
+do {					\
+    if (log_leaks)			\
+        trace_backtrace(a,b,c,d);	\
+} while(0)
+
 #endif /* !_OS_OSDEBBUG_H */
diff --git a/libkern/libkern/OSMalloc.h b/libkern/libkern/OSMalloc.h
index 0390a7703..c54b90070 100644
--- a/libkern/libkern/OSMalloc.h
+++ b/libkern/libkern/OSMalloc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	LIBKERN_OSMALLOC_h
diff --git a/libkern/libkern/OSReturn.h b/libkern/libkern/OSReturn.h
index 5e925ac03..ace65fcc0 100644
--- a/libkern/libkern/OSReturn.h
+++ b/libkern/libkern/OSReturn.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
diff --git a/libkern/libkern/OSTypes.h b/libkern/libkern/OSTypes.h
index 020da8c12..3366f1447 100644
--- a/libkern/libkern/OSTypes.h
+++ b/libkern/libkern/OSTypes.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
@@ -37,7 +43,11 @@ typedef signed int         SInt;
 
 typedef unsigned char 	   UInt8;
 typedef unsigned short 	   UInt16;
-typedef unsigned long 	   UInt32;
+#if __LP64__
+typedef unsigned int       UInt32;
+#else
+typedef unsigned long      UInt32;
+#endif
 typedef unsigned long long UInt64;
 #if		defined(__BIG_ENDIAN__)
 typedef	struct UnsignedWide {
@@ -55,7 +65,11 @@ typedef	struct UnsignedWide {
 
 typedef signed char 	   SInt8;
 typedef signed short 	   SInt16;
-typedef signed long 	   SInt32;
+#if __LP64__
+typedef signed int         SInt32;
+#else
+typedef signed long        SInt32;
+#endif
 typedef signed long long   SInt64;
 #if		defined(__BIG_ENDIAN__)
 typedef	struct wide {
diff --git a/libkern/libkern/_OSByteOrder.h b/libkern/libkern/_OSByteOrder.h
new file mode 100644
index 000000000..e6a30a1d8
--- /dev/null
+++ b/libkern/libkern/_OSByteOrder.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OS__OSBYTEORDER_H
+#define _OS__OSBYTEORDER_H
+
+/*
+ * This header is normally included from <libkern/OSByteOrder.h>.  However,
+ * <sys/_endian.h> also includes this in the case of little-endian
+ * architectures, so that we can map OSByteOrder routines to the hton* and ntoh*
+ * macros.  This results in the asymmetry below; we only include
+ * <libkern/arch/_OSByteOrder.h> for little-endian architectures.
+ */
+
+#include <sys/_types.h>
+
+/* Macros for swapping constant values in the preprocessing stage. */
+#define __DARWIN_OSSwapConstInt16(x) \
+    ((__uint16_t)((((__uint16_t)(x) & 0xff00) >> 8) | \
+                (((__uint16_t)(x) & 0x00ff) << 8)))
+
+#define __DARWIN_OSSwapConstInt32(x) \
+    ((__uint32_t)((((__uint32_t)(x) & 0xff000000) >> 24) | \
+                (((__uint32_t)(x) & 0x00ff0000) >>  8) | \
+                (((__uint32_t)(x) & 0x0000ff00) <<  8) | \
+                (((__uint32_t)(x) & 0x000000ff) << 24)))
+
+#define __DARWIN_OSSwapConstInt64(x) \
+    ((__uint64_t)((((__uint64_t)(x) & 0xff00000000000000ULL) >> 56) | \
+                (((__uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
+                (((__uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
+                (((__uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
+                (((__uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
+                (((__uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
+                (((__uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
+                (((__uint64_t)(x) & 0x00000000000000ffULL) << 56)))
+
+#if defined(__GNUC__)
+
+#if defined(__i386__) || defined(__x86_64__)
+#include <libkern/i386/_OSByteOrder.h>
+#endif
+
+#if defined(__arm__)
+#include <libkern/arm/OSByteOrder.h>
+#endif
+
+
+#define __DARWIN_OSSwapInt16(x) \
+    (__builtin_constant_p(x) ? __DARWIN_OSSwapConstInt16(x) : _OSSwapInt16(x))
+
+#define __DARWIN_OSSwapInt32(x) \
+    (__builtin_constant_p(x) ? __DARWIN_OSSwapConstInt32(x) : _OSSwapInt32(x))
+
+#define __DARWIN_OSSwapInt64(x) \
+    (__builtin_constant_p(x) ? __DARWIN_OSSwapConstInt64(x) : _OSSwapInt64(x))
+
+#else /* ! __GNUC__ */
+
+#if defined(__i386__) || defined(__x86_64__)
+
+#if !defined(__DARWIN_OS_INLINE)
+# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+#        define __DARWIN_OS_INLINE static inline
+# elif defined(__MWERKS__) || defined(__cplusplus)
+#        define __DARWIN_OS_INLINE static inline
+# else
+#        define __DARWIN_OS_INLINE static __inline__
+# endif
+#endif
+
+__DARWIN_OS_INLINE
+uint16_t
+_OSSwapInt16(
+    uint16_t			data
+)
+{
+    return __DARWIN_OSSwapConstInt16(data);
+}
+
+__DARWIN_OS_INLINE
+uint32_t
+_OSSwapInt32(
+    uint32_t			data
+)
+{
+    return __DARWIN_OSSwapConstInt32(data);
+}
+
+__DARWIN_OS_INLINE
+uint64_t
+_OSSwapInt64(
+    uint64_t			data
+)
+{
+    return __DARWIN_OSSwapConstInt64(data);
+}
+#endif
+
+#define __DARWIN_OSSwapInt16(x) _OSSwapInt16(x)
+
+#define __DARWIN_OSSwapInt32(x) _OSSwapInt32(x)
+
+#define __DARWIN_OSSwapInt64(x) _OSSwapInt64(x)
+
+#endif /* __GNUC__ */
+
+#endif /* ! _OS__OSBYTEORDER_H */
diff --git a/libkern/libkern/c++/Makefile b/libkern/libkern/c++/Makefile
index c2660a21b..ccb46b4a8 100644
--- a/libkern/libkern/c++/Makefile
+++ b/libkern/libkern/c++/Makefile
@@ -12,12 +12,16 @@ INSTINC_SUBDIRS_PPC =
 
 INSTINC_SUBDIRS_I386 = 
 
+INSTINC_SUBDIRS_ARM = 
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
+
 DATAFILES = \
           OSArray.h \
           OSBoolean.h \
diff --git a/libkern/libkern/c++/OSArray.h b/libkern/libkern/c++/OSArray.h
index f70cdba68..7cc92dfaf 100644
--- a/libkern/libkern/c++/OSArray.h
+++ b/libkern/libkern/c++/OSArray.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOArray.h created by rsulack on Thu 11-Sep-1997 */
 /* IOArray.h converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/libkern/c++/OSBoolean.h b/libkern/libkern/c++/OSBoolean.h
index 58b3e36e6..107e53631 100644
--- a/libkern/libkern/c++/OSBoolean.h
+++ b/libkern/libkern/c++/OSBoolean.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSBoolean.cpp created by rsulack on Tue Oct 12 1999 */
 
diff --git a/libkern/libkern/c++/OSCPPDebug.h b/libkern/libkern/c++/OSCPPDebug.h
index 8d2eeeb10..1f750741d 100644
--- a/libkern/libkern/c++/OSCPPDebug.h
+++ b/libkern/libkern/c++/OSCPPDebug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <libkern/OSBase.h>
diff --git a/libkern/libkern/c++/OSCollection.h b/libkern/libkern/c++/OSCollection.h
index ef8f1da90..15a15fa36 100644
--- a/libkern/libkern/c++/OSCollection.h
+++ b/libkern/libkern/c++/OSCollection.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOCollection.h created by gvdl on Thu 1998-10-22 */
 
diff --git a/libkern/libkern/c++/OSCollectionIterator.h b/libkern/libkern/c++/OSCollectionIterator.h
index 387b6355d..a2d6d70ee 100644
--- a/libkern/libkern/c++/OSCollectionIterator.h
+++ b/libkern/libkern/c++/OSCollectionIterator.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOCollectionIterator.h created by gvdl on Fri 1998-10-30 */
 
diff --git a/libkern/libkern/c++/OSContainers.h b/libkern/libkern/c++/OSContainers.h
index 725a786fb..32a433fcd 100644
--- a/libkern/libkern/c++/OSContainers.h
+++ b/libkern/libkern/c++/OSContainers.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOContainers.h created by rsulack on Fri 26-Jun-1998 */
 
diff --git a/libkern/libkern/c++/OSData.h b/libkern/libkern/c++/OSData.h
index 33b8259af..e699dff1d 100644
--- a/libkern/libkern/c++/OSData.h
+++ b/libkern/libkern/c++/OSData.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOData.h created by rsulack on Wed 17-Sep-1997 */
 /* IOData.h converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/libkern/c++/OSDictionary.h b/libkern/libkern/c++/OSDictionary.h
index c09501cc4..30efb0010 100644
--- a/libkern/libkern/c++/OSDictionary.h
+++ b/libkern/libkern/c++/OSDictionary.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-1999 Apple Computer, Inc.  All rights reserved.
diff --git a/libkern/libkern/c++/OSEndianTypes.h b/libkern/libkern/c++/OSEndianTypes.h
index c427d5743..8a527d460 100644
--- a/libkern/libkern/c++/OSEndianTypes.h
+++ b/libkern/libkern/c++/OSEndianTypes.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  *
  * HISTORY
  *   gvdl    20050620    Created
diff --git a/libkern/libkern/c++/OSIterator.h b/libkern/libkern/c++/OSIterator.h
index 47fb89213..ad167c9d1 100644
--- a/libkern/libkern/c++/OSIterator.h
+++ b/libkern/libkern/c++/OSIterator.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1998-1999 Apple Computer, Inc.  All rights reserved. 
diff --git a/libkern/libkern/c++/OSLib.h b/libkern/libkern/c++/OSLib.h
index 57d5a720c..ad68370ba 100644
--- a/libkern/libkern/c++/OSLib.h
+++ b/libkern/libkern/c++/OSLib.h
@@ -1,31 +1,30 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
-/*
- * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved.
- *
- * HISTORY
- *
- */
-
 #ifndef _OS_OSLIB_H
 #define _OS_OSLIB_H
 
@@ -48,7 +47,11 @@ __BEGIN_DECLS
 __END_DECLS
 
 #ifndef NULL
+#if defined (__cplusplus)
 #define NULL 0
+#else
+#define NULL ((void *)0)
+#endif
 #endif
 
 #endif /* _OS_OSLIB_H  */
diff --git a/libkern/libkern/c++/OSMetaClass.h b/libkern/libkern/c++/OSMetaClass.h
index 5f2615c4e..fd4bc66b6 100644
--- a/libkern/libkern/c++/OSMetaClass.h
+++ b/libkern/libkern/c++/OSMetaClass.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _LIBKERN_OSMETACLASS_H
 #define _LIBKERN_OSMETACLASS_H
@@ -40,6 +46,16 @@ class OSSerialize;
 #define APPLE_KEXT_COMPATIBILITY __attribute__ ((apple_kext_compatibility))
 #endif
 
+#define APPLE_KEXT_VTABLE_PADDING   1
+
+#if APPLE_KEXT_VTABLE_PADDING
+#define APPLE_KEXT_PAD_METHOD	    virtual
+#define APPLE_KEXT_PAD_IMPL(index)  gMetaClass.reservedCalled(index)
+#else
+#define APPLE_KEXT_PAD_METHOD	    static
+#define APPLE_KEXT_PAD_IMPL(index)  
+#endif
+
 class OSMetaClassBase
 {
 public:
@@ -107,9 +123,10 @@ _ptmf2ptf(const OSMetaClassBase *self, void (OSMetaClassBase::*func)(void))
     } map;
 
     map.fIn = func;
-    if (map.fptmf2.fToff)
+    if (map.fptmf2.fToff) {
 	panic("Multiple inheritance is not supported");
-    else if (map.fptmf2.fVInd < 0) {
+	return 0;
+    } else if (map.fptmf2.fVInd < 0) {
 	// Not virtual, i.e. plain member func
 	return map.fptmf2.u.fPFN;
     } else {
@@ -628,13 +645,13 @@ public:
 
 #define OSMetaClassDeclareReservedUnused(classname, index)		\
     private:								\
-        virtual void _RESERVED ## classname ## index ()
+    APPLE_KEXT_PAD_METHOD void _RESERVED ## classname ## index ()
 
 #define OSMetaClassDeclareReservedUsed(classname, index)
 
 #define OSMetaClassDefineReservedUnused(classname, index)		\
 void classname ::_RESERVED ## classname ## index () 			\
-    { gMetaClass.reservedCalled(index); }
+    { APPLE_KEXT_PAD_IMPL(index); }
 
 #define OSMetaClassDefineReservedUsed(classname, index)
 
diff --git a/libkern/libkern/c++/OSNumber.h b/libkern/libkern/c++/OSNumber.h
index c72a91c98..df90e9ddd 100644
--- a/libkern/libkern/c++/OSNumber.h
+++ b/libkern/libkern/c++/OSNumber.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOOffset.h created by rsulack on Wed 17-Sep-1997 */
 /* IOOffset.h converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/libkern/c++/OSObject.h b/libkern/libkern/c++/OSObject.h
index a39913302..82fae0a6a 100644
--- a/libkern/libkern/c++/OSObject.h
+++ b/libkern/libkern/c++/OSObject.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 Copyright (c) 1998 Apple Computer, Inc.	 All rights reserved.
diff --git a/libkern/libkern/c++/OSOrderedSet.h b/libkern/libkern/c++/OSOrderedSet.h
index b9e211939..c457dbf6a 100644
--- a/libkern/libkern/c++/OSOrderedSet.h
+++ b/libkern/libkern/c++/OSOrderedSet.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _OS_OSORDEREDSET_H
@@ -45,7 +51,7 @@ public:
         @param obj1 An object from the collection.
         @param obj2 An object to be compared to obj1.
         @param ref The ordering context used by the sorting function as a hint for sorting.
-        @result Returns a comparison result of the object, a negative value if obj1 < obj2, 0 if obj1 == obj2, and a positive value if obj1 > obj2.
+        @result Returns a comparison result of the object, a negative value if obj1 > obj2, 0 if obj1 == obj2, and a positive value if obj1 < obj2.
      */
     typedef SInt32 (*OSOrderFunction)(const OSMetaClassBase * obj1,
                                       const OSMetaClassBase * obj2,
diff --git a/libkern/libkern/c++/OSSerialize.h b/libkern/libkern/c++/OSSerialize.h
index b27811dce..f8e6a0de8 100644
--- a/libkern/libkern/c++/OSSerialize.h
+++ b/libkern/libkern/c++/OSSerialize.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSSerialize.h created by rsulack on Wen 25-Nov-1998 */
 
diff --git a/libkern/libkern/c++/OSSet.h b/libkern/libkern/c++/OSSet.h
index e01401e7d..800dad227 100644
--- a/libkern/libkern/c++/OSSet.h
+++ b/libkern/libkern/c++/OSSet.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOSet.h created by rsulack on Thu 11-Jun-1998 */
 /* IOSet.h converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/libkern/c++/OSString.h b/libkern/libkern/c++/OSString.h
index 577807a86..deadb3b07 100644
--- a/libkern/libkern/c++/OSString.h
+++ b/libkern/libkern/c++/OSString.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOString.h created by rsulack on Wed 17-Sep-1997 */
 /* IOString.h converted to C++ by gvdl on Fri 1998-10-30 */
diff --git a/libkern/libkern/c++/OSSymbol.h b/libkern/libkern/c++/OSSymbol.h
index b3276aa48..da3d2534a 100644
--- a/libkern/libkern/c++/OSSymbol.h
+++ b/libkern/libkern/c++/OSSymbol.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* IOSymbol.h created by gvdl on Fri 1998-10-30 */
 /* IOSymbol must be created through the factory methods and thus is not subclassable. */
diff --git a/libkern/libkern/c++/OSUnserialize.h b/libkern/libkern/c++/OSUnserialize.h
index 540cf9149..52c4d04ea 100644
--- a/libkern/libkern/c++/OSUnserialize.h
+++ b/libkern/libkern/c++/OSUnserialize.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* OSUnserialize.h created by rsulack on Mon 23-Nov-1998 */
 
diff --git a/iokit/include/drivers/Makefile b/libkern/libkern/crypto/Makefile
similarity index 67%
rename from iokit/include/drivers/Makefile
rename to libkern/libkern/crypto/Makefile
index 4b77117cc..5c8103efa 100644
--- a/iokit/include/drivers/Makefile
+++ b/libkern/libkern/crypto/Makefile
@@ -6,11 +6,11 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
-INSTINC_SUBDIRS = 
+INSTINC_SUBDIRS =
 
-INSTINC_SUBDIRS_PPC = 
+INSTINC_SUBDIRS_PPC =
 
-INSTINC_SUBDIRS_I386 = 
+INSTINC_SUBDIRS_I386 =
 
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 
@@ -18,17 +18,15 @@ EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
-DATAFILES = \
-	    event_status_driver.h
+DATAFILES = md5.h sha1.h
 
-INSTALL_MI_LIST	= ${DATAFILES}
+INSTALL_KF_MI_LIST = ${DATAFILES}
 
-INSTALL_MI_DIR = drivers
+INSTALL_MI_DIR = libkern/crypto
 
-EXPORT_MI_LIST	= \
-
-EXPORT_MI_DIR = \
+EXPORT_MI_LIST	= ${DATAFILES}
 
+EXPORT_MI_DIR = libkern/crypto
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/libkern/libkern/crypto/md5.h b/libkern/libkern/crypto/md5.h
new file mode 100644
index 000000000..57e826af8
--- /dev/null
+++ b/libkern/libkern/crypto/md5.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * MD5.H - header file for MD5.C
+ */
+
+/*
+ * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
+ * rights reserved.
+ *
+ * License to copy and use this software is granted provided that it
+ * is identified as the "RSA Data Security, Inc. MD5 Message-Digest
+ * Algorithm" in all material mentioning or referencing this software
+ * or this function.
+ *
+ * License is also granted to make and use derivative works provided
+ * that such works are identified as "derived from the RSA Data
+ * Security, Inc. MD5 Message-Digest Algorithm" in all material
+ * mentioning or referencing the derived work.
+ *
+ * RSA Data Security, Inc. makes no representations concerning either
+ * the merchantability of this software or the suitability of this
+ * software for any particular purpose. It is provided "as is"
+ * without express or implied warranty of any kind.
+ *
+ * These notices must be retained in any copies of any part of this
+ * documentation and/or software.
+ */
+
+#ifndef _CRYPTO_MD5_H_
+#define	_CRYPTO_MD5_H_
+
+#include <sys/types.h>
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+
+#define	MD5_DIGEST_LENGTH	16
+
+/* MD5 context. */
+typedef struct {
+	u_int32_t state[4];	/* state (ABCD) */
+	u_int32_t count[2];	/* number of bits, modulo 2^64 (lsb first) */
+	unsigned char buffer[64];	/* input buffer */
+} MD5_CTX;
+
+extern void MD5Init(MD5_CTX *);
+extern void MD5Update(MD5_CTX *, const void *, unsigned int);
+extern void MD5Final(unsigned char [MD5_DIGEST_LENGTH], MD5_CTX *);
+
+__END_DECLS
+
+#endif /* _CRYPTO_MD5_H_ */
diff --git a/libkern/libkern/crypto/sha1.h b/libkern/libkern/crypto/sha1.h
new file mode 100644
index 000000000..47a6e11c3
--- /dev/null
+++ b/libkern/libkern/crypto/sha1.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _CRYPTO_SHA1_H_
+#define	_CRYPTO_SHA1_H_
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#define	SHA_DIGEST_LENGTH	20
+#define	SHA1_RESULTLEN		SHA_DIGEST_LENGTH
+
+typedef struct sha1_ctxt {
+	union {
+		u_int8_t	b8[20];
+		u_int32_t	b32[5];	/* state (ABCDE) */
+	} h;
+	union {
+		u_int8_t	b8[8];
+		u_int32_t	b32[2];
+		u_int64_t	b64[1];	/* # of bits, modulo 2^64 (msb first) */
+	} c;
+	union {
+		u_int8_t	b8[64];
+		u_int32_t	b32[16]; /* input buffer */
+	} m;
+	u_int8_t	count;		/* unused; for compatibility only */
+} SHA1_CTX;
+
+/* For compatibility with the other SHA-1 implementation. */
+#define sha1_init(c)		SHA1Init(c)
+#define sha1_loop(c, b, l)	SHA1Update(c, b, l)
+#define sha1_result(c, b)	SHA1Final(b, c)
+
+extern void SHA1Init(SHA1_CTX *);
+extern void SHA1Update(SHA1_CTX *, const void *, size_t);
+extern void SHA1Final(void *, SHA1_CTX *);
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif /*_CRYPTO_SHA1_H_*/
diff --git a/libkern/libkern/i386/Makefile b/libkern/libkern/i386/Makefile
index 1bdad11a7..3fc89f6c4 100644
--- a/libkern/libkern/i386/Makefile
+++ b/libkern/libkern/i386/Makefile
@@ -19,7 +19,8 @@ EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
 DATAFILES = \
-          OSByteOrder.h
+          OSByteOrder.h \
+	  _OSByteOrder.h
 
 INSTALL_MD_LIST	= ${DATAFILES}
 
diff --git a/libkern/libkern/i386/OSByteOrder.h b/libkern/libkern/i386/OSByteOrder.h
index 5783957f0..7f197367c 100644
--- a/libkern/libkern/i386/OSByteOrder.h
+++ b/libkern/libkern/i386/OSByteOrder.h
@@ -1,86 +1,39 @@
 /*
- * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1999-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _OS_OSBYTEORDERI386_H
 #define _OS_OSBYTEORDERI386_H
 
 #include <stdint.h>
+#include <libkern/i386/_OSByteOrder.h>
 
 #if !defined(OS_INLINE)
-# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-#        define OS_INLINE static inline
-# elif defined(__MWERKS__) || defined(__cplusplus)
-#        define OS_INLINE static inline
-# else
-#        define OS_INLINE static __inline__
-# endif
-#endif
-
-/* Generic byte swapping functions. */
-
-OS_INLINE
-uint16_t
-_OSSwapInt16(
-    uint16_t        data
-)
-{
-    return ((data << 8) | (data >> 8));
-}
-
-OS_INLINE
-uint32_t
-_OSSwapInt32(
-    uint32_t        data
-)
-{
-    __asm__ ("bswap   %0" : "+r" (data));
-    return data;
-}
-
-#if defined(__i386__)
-OS_INLINE
-uint64_t
-_OSSwapInt64(
-    uint64_t        data
-)
-{
-    __asm__ ("bswap   %%eax\n\t"
-             "bswap   %%edx\n\t" 
-             "xchgl   %%eax, %%edx"
-             : "+A" (data));
-    return data;
-}
-#elif defined(__x86_64__)
-OS_INLINE
-uint64_t
-_OSSwapInt64(
-    uint64_t        data
-)
-{
-    __asm__ ("bswap   %0" : "+r" (data));
-    return data;
-}
-#else
-#error Unknown architecture
+#define OS_INLINE	__DARWIN_OS_INLINE
 #endif
 
 /* Functions for byte reversed loads. */
diff --git a/libkern/libkern/i386/_OSByteOrder.h b/libkern/libkern/i386/_OSByteOrder.h
new file mode 100644
index 000000000..d84ce8f32
--- /dev/null
+++ b/libkern/libkern/i386/_OSByteOrder.h
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _OS__OSBYTEORDERI386_H
+#define _OS__OSBYTEORDERI386_H
+
+#if !defined(__DARWIN_OS_INLINE)
+# if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+#        define __DARWIN_OS_INLINE static inline
+# elif defined(__MWERKS__) || defined(__cplusplus)
+#        define __DARWIN_OS_INLINE static inline
+# else
+#        define __DARWIN_OS_INLINE static __inline__
+# endif
+#endif
+
+/* Generic byte swapping functions. */
+
+__DARWIN_OS_INLINE
+__uint16_t
+_OSSwapInt16(
+    __uint16_t        _data
+)
+{
+    return ((_data << 8) | (_data >> 8));
+}
+
+__DARWIN_OS_INLINE
+__uint32_t
+_OSSwapInt32(
+    __uint32_t        _data
+)
+{
+    __asm__ ("bswap   %0" : "+r" (_data));
+    return _data;
+}
+
+#if defined(__i386__)
+__DARWIN_OS_INLINE
+__uint64_t
+_OSSwapInt64(
+    __uint64_t        _data
+)
+{
+    __asm__ ("bswap   %%eax\n\t"
+             "bswap   %%edx\n\t" 
+             "xchgl   %%eax, %%edx"
+             : "+A" (_data));
+    return _data;
+}
+#elif defined(__x86_64__)
+__DARWIN_OS_INLINE
+__uint64_t
+_OSSwapInt64(
+    __uint64_t        _data
+)
+{
+    __asm__ ("bswap   %0" : "+r" (_data));
+    return _data;
+}
+#else
+#error Unknown architecture
+#endif
+
+#endif /* ! _OS__OSBYTEORDERI386_H */
diff --git a/libkern/libkern/locks.h b/libkern/libkern/locks.h
index 7e012f8f7..d25747139 100644
--- a/libkern/libkern/locks.h
+++ b/libkern/libkern/locks.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	LIBKERN_LOCKS_H
diff --git a/libkern/libkern/machine/Makefile b/libkern/libkern/machine/Makefile
index 461b7c379..9dbd25dce 100644
--- a/libkern/libkern/machine/Makefile
+++ b/libkern/libkern/machine/Makefile
@@ -12,12 +12,16 @@ INSTINC_SUBDIRS_PPC =
 
 INSTINC_SUBDIRS_I386 =
 
+INSTINC_SUBDIRS_ARM =
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
+
 DATAFILES = \
           OSByteOrder.h
 
diff --git a/libkern/libkern/machine/OSByteOrder.h b/libkern/libkern/machine/OSByteOrder.h
index 97f2a6fbd..48602154e 100644
--- a/libkern/libkern/machine/OSByteOrder.h
+++ b/libkern/libkern/machine/OSByteOrder.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _OS_OSBYTEORDERMACHINE_H
diff --git a/libkern/libkern/ppc/OSByteOrder.h b/libkern/libkern/ppc/OSByteOrder.h
index e545f5236..3241ba668 100644
--- a/libkern/libkern/ppc/OSByteOrder.h
+++ b/libkern/libkern/ppc/OSByteOrder.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _OS_OSBYTEORDERPPC_H
@@ -130,8 +136,8 @@ OSWriteSwapInt64(
 )
 {
     volatile uint64_t *addr = (volatile uint64_t *)((uintptr_t)base + byteOffset);
-    uint32_t hi = data >> 32;
-    uint32_t lo = data & 0xffffffff;
+    uint32_t hi = (uint32_t)(data >> 32);
+    uint32_t lo = (uint32_t)(data & 0xffffffff);
 
     __asm__ ("stwbrx %1, %4, %3\n\t"
              "stwbrx %2, %5, %3"
diff --git a/libkern/libkern/sysctl.h b/libkern/libkern/sysctl.h
index c798bf3af..9af0fa0cc 100644
--- a/libkern/libkern/sysctl.h
+++ b/libkern/libkern/sysctl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	LIBKERN_SYSCTL_H
diff --git a/libkern/libkern/version.h.template b/libkern/libkern/version.h.template
index 7f2569d4c..32793fe63 100644
--- a/libkern/libkern/version.h.template
+++ b/libkern/libkern/version.h.template
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * The contents of this file constitute Original Code as defined in and
  * are subject to the Apple Public Source License Version 1.1 (the
@@ -17,7 +17,7 @@
  * License for the specific language governing rights and limitations
  * under the License.
  *
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	LIBKERN_VERSION_H
@@ -86,6 +86,9 @@ extern const char osbuilder[];
 
 extern const char version[];
 
+#define OSVERSIZE 256
+extern char osversion[];
+
 
 #if defined(__cplusplus)
 }
diff --git a/libkern/libkern/zlib.h b/libkern/libkern/zlib.h
new file mode 100644
index 000000000..db7366f53
--- /dev/null
+++ b/libkern/libkern/zlib.h
@@ -0,0 +1,723 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/* $FreeBSD: src/sys/net/zlib.h,v 1.7 1999/12/29 04:38:38 peter Exp $	*/
+
+/*
+ * This file is derived from zlib.h and zconf.h from the zlib-1.1.4
+ * distribution by Jean-loup Gailly and Mark Adler.  The interface
+ * described in this file refers to the kernel zlib implementation
+ * of Mac OS X.
+ */
+
+/* zlib.h -- interface of the 'zlib' general purpose compression library
+  version 1.1.4, March 11th, 2002
+
+  Copyright (C) 1995-2002 Jean-loup Gailly and Mark Adler
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+
+  The data format used by the zlib library is described by RFCs (Request for
+  Comments) 1950 to 1952 in the files ftp://ds.internic.net/rfc/rfc1950.txt
+  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
+*/
+
+#ifndef _ZLIB_H
+#define _ZLIB_H
+
+#if __cplusplus
+extern "C" {
+#endif
+
+/* zconf.h -- configuration of the zlib compression library
+ * Copyright (C) 1995-2002 Jean-loup Gailly.
+ * For conditions of distribution and use, see copyright notice in zlib.h 
+ */
+
+#ifndef _ZCONF_H
+#define _ZCONF_H
+
+/* Maximum value for memLevel in deflateInit2 */
+#define MAX_MEM_LEVEL 9
+
+/* Maximum value for windowBits in deflateInit2 and inflateInit2 */
+#define MAX_WBITS   15 /* 32K LZ77 window */
+
+/* The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+ that is: 128K for windowBits=15  +  128K for memLevel = 8  (default values)
+ plus a few kilobytes for small objects.
+
+   The memory requirements for inflate are (in bytes) 1 << windowBits
+ that is, 32K for windowBits=15 (default value) plus a few kilobytes
+ for small objects.
+*/
+
+                        /* Type declarations */
+
+#define	OF(args)  args
+#define ZEXPORT
+#define ZEXPORTVA
+#define ZEXTERN extern
+#define FAR
+#define z_off_t  off_t
+
+typedef unsigned char  Byte;  /* 8 bits */
+typedef unsigned int   uInt;  /* 16 bits or more */
+typedef unsigned long  uLong; /* 32 bits or more */
+typedef Byte  Bytef;
+typedef char  charf;
+typedef int   intf;
+typedef uInt  uIntf;
+typedef uLong uLongf;
+typedef void  *voidpf;
+typedef void  *voidp;
+
+#endif /* _ZCONF_H */
+
+#define ZLIB_VERSION "1.1.4"
+
+/* 
+     The 'zlib' compression library provides in-memory compression and
+  decompression functions, including integrity checks of the uncompressed
+  data.  This version of the library supports only one compression method
+  (deflation) but other algorithms will be added later and will have the same
+  stream interface.
+
+     Compression can be done in a single step if the buffers are large
+  enough (for example if an input file is mmap'ed), or can be done by
+  repeated calls of the compression function.  In the latter case, the
+  application must provide more input and/or consume the output
+  (providing more output space) before each call.
+
+     The library does not install any signal handler. The decoder checks
+  the consistency of the compressed data, so the library should never
+  crash even in case of corrupted input.
+*/
+
+typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
+typedef void   (*free_func)  OF((voidpf opaque, voidpf address));
+
+struct internal_state;
+
+typedef struct z_stream_s {
+    Bytef    *next_in;  /* next input byte */
+    uInt     avail_in;  /* number of bytes available at next_in */
+    uLong    total_in;  /* total nb of input bytes read so far */
+
+    Bytef    *next_out; /* next output byte should be put there */
+    uInt     avail_out; /* remaining free space at next_out */
+    uLong    total_out; /* total nb of bytes output so far */
+
+    char     *msg;      /* last error message, NULL if no error */
+    struct internal_state FAR *state; /* not visible by applications */
+
+    alloc_func zalloc;  /* used to allocate the internal state */
+    free_func  zfree;   /* used to free the internal state */
+    voidpf     opaque;  /* private data object passed to zalloc and zfree */
+
+    int     data_type;  /* best guess about the data type: ascii or binary */
+    uLong   adler;      /* adler32 value of the uncompressed data */
+    uLong   reserved;   /* reserved for future use */
+} z_stream;
+
+typedef z_stream FAR *z_streamp;
+
+/*
+   The application must update next_in and avail_in when avail_in has
+   dropped to zero. It must update next_out and avail_out when avail_out
+   has dropped to zero. The application must initialize zalloc, zfree and
+   opaque before calling the init function. All other fields are set by the
+   compression library and must not be updated by the application.
+
+   The opaque value provided by the application will be passed as the first
+   parameter for calls of zalloc and zfree. This can be useful for custom
+   memory management. The compression library attaches no meaning to the
+   opaque value.
+
+   zalloc must return Z_NULL if there is not enough memory for the object.
+   If zlib is used in a multi-threaded application, zalloc and zfree must be
+   thread safe.
+
+   The fields total_in and total_out can be used for statistics or
+   progress reports. After compression, total_in holds the total size of
+   the uncompressed data and may be saved for use in the decompressor
+   (particularly if the decompressor wants to decompress everything in
+   a single step).
+*/
+
+                        /* constants */
+
+#define Z_NO_FLUSH      0
+#define Z_PARTIAL_FLUSH 1 /* will be removed, use Z_SYNC_FLUSH instead */
+#define Z_PACKET_FLUSH	2
+#define Z_SYNC_FLUSH    3
+#define Z_FULL_FLUSH    4
+#define Z_FINISH        5
+/* Allowed flush values; see deflate() below for details */
+
+#define Z_OK            0
+#define Z_STREAM_END    1
+#define Z_NEED_DICT     2
+#define Z_ERRNO        (-1)
+#define Z_STREAM_ERROR (-2)
+#define Z_DATA_ERROR   (-3)
+#define Z_MEM_ERROR    (-4)
+#define Z_BUF_ERROR    (-5)
+#define Z_VERSION_ERROR (-6)
+/* Return codes for the compression/decompression functions. Negative
+ * values are errors, positive values are used for special but normal events.
+ */
+
+#define Z_NO_COMPRESSION         0
+#define Z_BEST_SPEED             1
+#define Z_BEST_COMPRESSION       9
+#define Z_DEFAULT_COMPRESSION  (-1)
+/* compression levels */
+
+#define Z_FILTERED            1
+#define Z_HUFFMAN_ONLY        2
+#define Z_DEFAULT_STRATEGY    0
+/* compression strategy; see deflateInit2() below for details */
+
+#define Z_BINARY   0
+#define Z_ASCII    1
+#define Z_UNKNOWN  2
+/* Possible values of the data_type field */
+
+#define Z_DEFLATED   8
+/* The deflate compression method (the only one supported in this version) */
+
+#define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
+
+#define zlib_version zlibVersion()
+/* for compatibility with versions < 1.0.2 */
+
+                        /* basic functions */
+
+ZEXTERN const char * ZEXPORT zlibVersion OF((void));
+/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
+   If the first character differs, the library code actually used is
+   not compatible with the zlib.h header file used by the application.
+   This check is automatically made by deflateInit and inflateInit.
+ */
+
+/* 
+ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
+
+     Initializes the internal stream state for compression. The fields
+   zalloc, zfree and opaque must be initialized before by the caller.
+   If zalloc and zfree are set to Z_NULL, deflateInit updates them to
+   use default allocation functions.
+
+     The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
+   1 gives best speed, 9 gives best compression, 0 gives no compression at
+   all (the input data is simply copied a block at a time).
+   Z_DEFAULT_COMPRESSION requests a default compromise between speed and
+   compression (currently equivalent to level 6).
+
+     deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_STREAM_ERROR if level is not a valid compression level,
+   Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
+   with the version assumed by the caller (ZLIB_VERSION).
+   msg is set to null if there is no error message.  deflateInit does not
+   perform any compression: this will be done by deflate().
+*/
+
+
+ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
+/*
+    deflate compresses as much data as possible, and stops when the input
+  buffer becomes empty or the output buffer becomes full. It may introduce some
+  output latency (reading input without producing any output) except when
+  forced to flush.
+
+    The detailed semantics are as follows. deflate performs one or both of the
+  following actions:
+
+  - Compress more input starting at next_in and update next_in and avail_in
+    accordingly. If not all input can be processed (because there is not
+    enough room in the output buffer), next_in and avail_in are updated and
+    processing will resume at this point for the next call of deflate().
+
+  - Provide more output starting at next_out and update next_out and avail_out
+    accordingly. This action is forced if the parameter flush is non zero.
+    Forcing flush frequently degrades the compression ratio, so this parameter
+    should be set only when necessary (in interactive applications).
+    Some output may be provided even if flush is not set.
+
+  Before the call of deflate(), the application should ensure that at least
+  one of the actions is possible, by providing more input and/or consuming
+  more output, and updating avail_in or avail_out accordingly; avail_out
+  should never be zero before the call. The application can consume the
+  compressed output when it wants, for example when the output buffer is full
+  (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK
+  and with zero avail_out, it must be called again after making room in the
+  output buffer because there might be more output pending.
+
+    If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
+  flushed to the output buffer and the output is aligned on a byte boundary, so
+  that the decompressor can get all input data available so far. (In particular
+  avail_in is zero after the call if enough output space has been provided
+  before the call.)  Flushing may degrade compression for some compression
+  algorithms and so it should be used only when necessary.
+
+    If flush is set to Z_FULL_FLUSH, all output is flushed as with
+  Z_SYNC_FLUSH, and the compression state is reset so that decompression can
+  restart from this point if previous compressed data has been damaged or if
+  random access is desired. Using Z_FULL_FLUSH too often can seriously degrade
+  the compression.
+
+    If deflate returns with avail_out == 0, this function must be called again
+  with the same value of the flush parameter and more output space (updated
+  avail_out), until the flush is complete (deflate returns with non-zero
+  avail_out).
+
+    If the parameter flush is set to Z_FINISH, pending input is processed,
+  pending output is flushed and deflate returns with Z_STREAM_END if there
+  was enough output space; if deflate returns with Z_OK, this function must be
+  called again with Z_FINISH and more output space (updated avail_out) but no
+  more input data, until it returns with Z_STREAM_END or an error. After
+  deflate has returned Z_STREAM_END, the only possible operations on the
+  stream are deflateReset or deflateEnd.
+  
+    Z_FINISH can be used immediately after deflateInit if all the compression
+  is to be done in a single step. In this case, avail_out must be at least
+  0.1% larger than avail_in plus 12 bytes.  If deflate does not return
+  Z_STREAM_END, then it must be called again as described above.
+
+    deflate() sets strm->adler to the adler32 checksum of all input read
+  so far (that is, total_in bytes).
+
+    deflate() may update data_type if it can make a good guess about
+  the input data type (Z_ASCII or Z_BINARY). In doubt, the data is considered
+  binary. This field is only for information purposes and does not affect
+  the compression algorithm in any manner.
+
+    deflate() returns Z_OK if some progress has been made (more input
+  processed or more output produced), Z_STREAM_END if all input has been
+  consumed and all output has been produced (only when flush is set to
+  Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
+  if next_in or next_out was NULL), Z_BUF_ERROR if no progress is possible
+  (for example avail_in or avail_out was zero).
+*/
+
+
+ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
+/*
+     All dynamically allocated data structures for this stream are freed.
+   This function discards any unprocessed input and does not flush any
+   pending output.
+
+     deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
+   stream state was inconsistent, Z_DATA_ERROR if the stream was freed
+   prematurely (some input or output was discarded). In the error case,
+   msg may be set but then points to a static string (which must not be
+   deallocated).
+*/
+
+
+/* 
+ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
+
+     Initializes the internal stream state for decompression. The fields
+   next_in, avail_in, zalloc, zfree and opaque must be initialized before by
+   the caller. If next_in is not Z_NULL and avail_in is large enough (the exact
+   value depends on the compression method), inflateInit determines the
+   compression method from the zlib header and allocates all data structures
+   accordingly; otherwise the allocation will be deferred to the first call of
+   inflate.  If zalloc and zfree are set to Z_NULL, inflateInit updates them to
+   use default allocation functions.
+
+     inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
+   version assumed by the caller.  msg is set to null if there is no error
+   message. inflateInit does not perform any decompression apart from reading
+   the zlib header if present: this will be done by inflate().  (So next_in and
+   avail_in may be modified, but next_out and avail_out are unchanged.)
+*/
+
+
+ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
+/*
+    inflate decompresses as much data as possible, and stops when the input
+  buffer becomes empty or the output buffer becomes full. It may some
+  introduce some output latency (reading input without producing any output)
+  except when forced to flush.
+
+  The detailed semantics are as follows. inflate performs one or both of the
+  following actions:
+
+  - Decompress more input starting at next_in and update next_in and avail_in
+    accordingly. If not all input can be processed (because there is not
+    enough room in the output buffer), next_in is updated and processing
+    will resume at this point for the next call of inflate().
+
+  - Provide more output starting at next_out and update next_out and avail_out
+    accordingly.  inflate() provides as much output as possible, until there
+    is no more input data or no more space in the output buffer (see below
+    about the flush parameter).
+
+  Before the call of inflate(), the application should ensure that at least
+  one of the actions is possible, by providing more input and/or consuming
+  more output, and updating the next_* and avail_* values accordingly.
+  The application can consume the uncompressed output when it wants, for
+  example when the output buffer is full (avail_out == 0), or after each
+  call of inflate(). If inflate returns Z_OK and with zero avail_out, it
+  must be called again after making room in the output buffer because there
+  might be more output pending.
+
+    If the parameter flush is set to Z_SYNC_FLUSH, inflate flushes as much
+  output as possible to the output buffer. The flushing behavior of inflate is
+  not specified for values of the flush parameter other than Z_SYNC_FLUSH
+  and Z_FINISH, but the current implementation actually flushes as much output
+  as possible anyway.
+
+    inflate() should normally be called until it returns Z_STREAM_END or an
+  error. However if all decompression is to be performed in a single step
+  (a single call of inflate), the parameter flush should be set to
+  Z_FINISH. In this case all pending input is processed and all pending
+  output is flushed; avail_out must be large enough to hold all the
+  uncompressed data. (The size of the uncompressed data may have been saved
+  by the compressor for this purpose.) The next operation on this stream must
+  be inflateEnd to deallocate the decompression state. The use of Z_FINISH
+  is never required, but can be used to inform inflate that a faster routine
+  may be used for the single inflate() call.
+
+     If a preset dictionary is needed at this point (see inflateSetDictionary
+  below), inflate sets strm-adler to the adler32 checksum of the
+  dictionary chosen by the compressor and returns Z_NEED_DICT; otherwise 
+  it sets strm->adler to the adler32 checksum of all output produced
+  so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or
+  an error code as described below. At the end of the stream, inflate()
+  checks that its computed adler32 checksum is equal to that saved by the
+  compressor and returns Z_STREAM_END only if the checksum is correct.
+
+    inflate() returns Z_OK if some progress has been made (more input processed
+  or more output produced), Z_STREAM_END if the end of the compressed data has
+  been reached and all uncompressed output has been produced, Z_NEED_DICT if a
+  preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
+  corrupted (input stream not conforming to the zlib format or incorrect
+  adler32 checksum), Z_STREAM_ERROR if the stream structure was inconsistent
+  (for example if next_in or next_out was NULL), Z_MEM_ERROR if there was not
+  enough memory, Z_BUF_ERROR if no progress is possible or if there was not
+  enough room in the output buffer when Z_FINISH is used. In the Z_DATA_ERROR
+  case, the application may then call inflateSync to look for a good
+  compression block.
+*/
+
+
+ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
+/*
+     All dynamically allocated data structures for this stream are freed.
+   This function discards any unprocessed input and does not flush any
+   pending output.
+
+     inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
+   was inconsistent. In the error case, msg may be set but then points to a
+   static string (which must not be deallocated).
+*/
+
+                        /* Advanced functions */
+
+/*
+    The following functions are needed only in some special applications.
+*/
+
+/*   
+ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
+                                     int  level,
+                                     int  method,
+                                     int  windowBits,
+                                     int  memLevel,
+                                     int  strategy));
+
+     This is another version of deflateInit with more compression options. The
+   fields next_in, zalloc, zfree and opaque must be initialized before by
+   the caller.
+
+     The method parameter is the compression method. It must be Z_DEFLATED in
+   this version of the library.
+
+     The windowBits parameter is the base two logarithm of the window size
+   (the size of the history buffer).  It should be in the range 8..15 for this
+   version of the library. Larger values of this parameter result in better
+   compression at the expense of memory usage. The default value is 15 if
+   deflateInit is used instead.
+
+     The memLevel parameter specifies how much memory should be allocated
+   for the internal compression state. memLevel=1 uses minimum memory but
+   is slow and reduces compression ratio; memLevel=9 uses maximum memory
+   for optimal speed. The default value is 8. See zconf.h for total memory
+   usage as a function of windowBits and memLevel.
+
+     The strategy parameter is used to tune the compression algorithm. Use the
+   value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
+   filter (or predictor), or Z_HUFFMAN_ONLY to force Huffman encoding only (no
+   string match).  Filtered data consists mostly of small values with a
+   somewhat random distribution. In this case, the compression algorithm is
+   tuned to compress them better. The effect of Z_FILTERED is to force more
+   Huffman coding and less string matching; it is somewhat intermediate
+   between Z_DEFAULT and Z_HUFFMAN_ONLY. The strategy parameter only affects
+   the compression ratio but not the correctness of the compressed output even
+   if it is not set appropriately.
+
+      deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_STREAM_ERROR if a parameter is invalid (such as an invalid
+   method). msg is set to null if there is no error message.  deflateInit2 does
+   not perform any compression: this will be done by deflate().
+*/
+                            
+ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
+                                             const Bytef *dictionary,
+                                             uInt  dictLength));
+/*
+     Initializes the compression dictionary from the given byte sequence
+   without producing any compressed output. This function must be called
+   immediately after deflateInit, deflateInit2 or deflateReset, before any
+   call of deflate. The compressor and decompressor must use exactly the same
+   dictionary (see inflateSetDictionary).
+
+     The dictionary should consist of strings (byte sequences) that are likely
+   to be encountered later in the data to be compressed, with the most commonly
+   used strings preferably put towards the end of the dictionary. Using a
+   dictionary is most useful when the data to be compressed is short and can be
+   predicted with good accuracy; the data can then be compressed better than
+   with the default empty dictionary.
+
+     Depending on the size of the compression data structures selected by
+   deflateInit or deflateInit2, a part of the dictionary may in effect be
+   discarded, for example if the dictionary is larger than the window size in
+   deflate or deflate2. Thus the strings most likely to be useful should be
+   put at the end of the dictionary, not at the front.
+
+     Upon return of this function, strm->adler is set to the Adler32 value
+   of the dictionary; the decompressor may later use this value to determine
+   which dictionary has been used by the compressor. (The Adler32 value
+   applies to the whole dictionary even if only a subset of the dictionary is
+   actually used by the compressor.)
+
+     deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
+   parameter is invalid (such as NULL dictionary) or the stream state is
+   inconsistent (for example if deflate has already been called for this stream
+   or if the compression method is bsort). deflateSetDictionary does not
+   perform any compression: this will be done by deflate().
+*/
+
+ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
+                                    z_streamp source));
+/*
+     Sets the destination stream as a complete copy of the source stream.
+
+     This function can be useful when several compression strategies will be
+   tried, for example when there are several ways of pre-processing the input
+   data with a filter. The streams that will be discarded should then be freed
+   by calling deflateEnd.  Note that deflateCopy duplicates the internal
+   compression state which can be quite large, so this strategy is slow and
+   can consume lots of memory.
+
+     deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
+   (such as zalloc being NULL). msg is left unchanged in both source and
+   destination.
+*/
+
+ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
+/*
+     This function is equivalent to deflateEnd followed by deflateInit,
+   but does not free and reallocate all the internal compression state.
+   The stream will keep the same compression level and any other attributes
+   that may have been set by deflateInit2.
+
+      deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being NULL).
+*/
+
+ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
+				      int level,
+				      int strategy));
+/*
+     Dynamically update the compression level and compression strategy.  The
+   interpretation of level and strategy is as in deflateInit2.  This can be
+   used to switch between compression and straight copy of the input data, or
+   to switch to a different kind of input data requiring a different
+   strategy. If the compression level is changed, the input available so far
+   is compressed with the old level (and may be flushed); the new level will
+   take effect only at the next call of deflate().
+
+     Before the call of deflateParams, the stream state must be set as for
+   a call of deflate(), since the currently available input may have to
+   be compressed and flushed. In particular, strm->avail_out must be non-zero.
+
+     deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
+   stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR
+   if strm->avail_out was zero.
+*/
+
+/*   
+ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
+                                     int  windowBits));
+
+     This is another version of inflateInit with an extra parameter. The
+   fields next_in, avail_in, zalloc, zfree and opaque must be initialized
+   before by the caller.
+
+     The windowBits parameter is the base two logarithm of the maximum window
+   size (the size of the history buffer).  It should be in the range 8..15 for
+   this version of the library. The default value is 15 if inflateInit is used
+   instead. If a compressed stream with a larger window size is given as
+   input, inflate() will return with the error code Z_DATA_ERROR instead of
+   trying to allocate a larger window.
+
+      inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_STREAM_ERROR if a parameter is invalid (such as a negative
+   memLevel). msg is set to null if there is no error message.  inflateInit2
+   does not perform any decompression apart from reading the zlib header if
+   present: this will be done by inflate(). (So next_in and avail_in may be
+   modified, but next_out and avail_out are unchanged.)
+*/
+
+ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
+                                             const Bytef *dictionary,
+                                             uInt  dictLength));
+/*
+     Initializes the decompression dictionary from the given uncompressed byte
+   sequence. This function must be called immediately after a call of inflate
+   if this call returned Z_NEED_DICT. The dictionary chosen by the compressor
+   can be determined from the Adler32 value returned by this call of
+   inflate. The compressor and decompressor must use exactly the same
+   dictionary (see deflateSetDictionary).
+
+     inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
+   parameter is invalid (such as NULL dictionary) or the stream state is
+   inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
+   expected one (incorrect Adler32 value). inflateSetDictionary does not
+   perform any decompression: this will be done by subsequent calls of
+   inflate().
+*/
+
+ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
+/* 
+    Skips invalid compressed data until a full flush point (see above the
+  description of deflate with Z_FULL_FLUSH) can be found, or until all
+  available input is skipped. No output is provided.
+
+    inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
+  if no more input was provided, Z_DATA_ERROR if no flush point has been found,
+  or Z_STREAM_ERROR if the stream structure was inconsistent. In the success
+  case, the application may save the current current value of total_in which
+  indicates where valid compressed data was found. In the error case, the
+  application may repeatedly call inflateSync, providing more input each time,
+  until success or end of the input data.
+*/
+
+ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
+/*
+     This function is equivalent to inflateEnd followed by inflateInit,
+   but does not free and reallocate all the internal decompression state.
+   The stream will keep attributes that may have been set by inflateInit2.
+
+      inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being NULL).
+*/
+
+                        /* checksum functions */
+
+/*
+     These functions are not related to compression but are exported
+   anyway because they might be useful in applications using the
+   compression library.
+*/
+
+ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
+/*
+     Update a running Adler-32 checksum with the bytes buf[0..len-1] and
+   return the updated checksum. If buf is NULL, this function returns
+   the required initial value for the checksum.
+   An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
+   much faster. Usage example:
+
+     uLong adler = adler32(0L, Z_NULL, 0);
+
+     while (read_buffer(buffer, length) != EOF) {
+       adler = adler32(adler, buffer, length);
+     }
+     if (adler != original_adler) error();
+*/
+
+                        /* various hacks, don't look :) */
+
+/* deflateInit and inflateInit are macros to allow checking the zlib version
+ * and the compiler's view of z_stream:
+ */
+ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
+                                     const char *vers, int stream_size));
+ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
+                                     const char *vers, int stream_size));
+ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
+                                      int windowBits, int memLevel,
+                                      int strategy, const char *vers,
+                                      int stream_size));
+ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
+                                      const char *vers, int stream_size));
+#define deflateInit(strm, level) \
+        deflateInit_((strm), (level),       ZLIB_VERSION, sizeof(z_stream))
+#define inflateInit(strm) \
+        inflateInit_((strm),                ZLIB_VERSION, sizeof(z_stream))
+#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
+        deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
+                      (strategy),           ZLIB_VERSION, sizeof(z_stream))
+#define inflateInit2(strm, windowBits) \
+        inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
+
+ZEXTERN const char   * ZEXPORT zError           OF((int err));
+ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp z));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZLIB_H */
diff --git a/libkern/mach-o/loader.h b/libkern/mach-o/loader.h
index 85c23d70a..9a3f8b58a 100644
--- a/libkern/mach-o/loader.h
+++ b/libkern/mach-o/loader.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHO_LOADER_H_
 #define _MACHO_LOADER_H_
@@ -160,6 +166,8 @@ struct load_command {
 #define	LC_PREBOUND_DYLIB 0x10	/* modules prebound for a dynamicly */
 				/*  linked shared library */
 
+#define LC_UUID		0x1b	/* the uuid */
+
 /*
  * A variable length string in a load command is represented by an lc_str
  * union.  The strings are stored just after the load command structure and
@@ -758,6 +766,16 @@ struct dylib_reference {
     		  flags:8;	/* flags to indicate the type of reference */
 };
 
+/*
+ * The uuid load command contains a single 128-bit unique random number that
+ * identifies an object produced by the static link editor.
+ */
+struct uuid_command {
+    uint32_t	cmd;		/* LC_UUID */
+    uint32_t	cmdsize;	/* sizeof(struct uuid_command) */
+    uint8_t	uuid[16];	/* the 128-bit uuid */
+};
+
 /*
  * The symseg_command contains the offset and size of the GNU style
  * symbol table information as described in the header file <symseg.h>.
diff --git a/libkern/mach-o/mach_header.h b/libkern/mach-o/mach_header.h
index 31daf7349..991dde94b 100644
--- a/libkern/mach-o/mach_header.h
+++ b/libkern/mach-o/mach_header.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.h
diff --git a/libkern/ppc/OSAtomic.s b/libkern/ppc/OSAtomic.s
index f1834d998..aaa20f2dd 100644
--- a/libkern/ppc/OSAtomic.s
+++ b/libkern/ppc/OSAtomic.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  /*
  * Copyright (c) 1997-1998 Apple Computer, Inc.
diff --git a/libkern/ppc/bcmp.s b/libkern/ppc/bcmp.s
index 934ec4dcd..901850379 100644
--- a/libkern/ppc/bcmp.s
+++ b/libkern/ppc/bcmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 ;
 #include <ppc/asm.h>
diff --git a/libkern/ppc/memcmp.s b/libkern/ppc/memcmp.s
index 027af6bae..9968bf6f4 100644
--- a/libkern/ppc/memcmp.s
+++ b/libkern/ppc/memcmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 ;
 #include <ppc/asm.h>
diff --git a/libkern/ppc/strlen.s b/libkern/ppc/strlen.s
index def6639f3..0bb80cf99 100644
--- a/libkern/ppc/strlen.s
+++ b/libkern/ppc/strlen.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 ;
 ;
diff --git a/libkern/stdio/scanf.c b/libkern/stdio/scanf.c
index 99c5edbaa..0bc3a4363 100644
--- a/libkern/stdio/scanf.c
+++ b/libkern/stdio/scanf.c
@@ -1,14 +1,19 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
@@ -18,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1990, 1993
diff --git a/libkern/uuid/Makefile b/libkern/uuid/Makefile
index 03789630f..8b26dcefe 100644
--- a/libkern/uuid/Makefile
+++ b/libkern/uuid/Makefile
@@ -13,12 +13,16 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 # uuid.h is now installed by bsd/uuid/Makefile
 DATAFILES = \
 
diff --git a/libkern/uuid/uuid.c b/libkern/uuid/uuid.c
index 8b6573281..351de623a 100644
--- a/libkern/uuid/uuid.c
+++ b/libkern/uuid/uuid.c
@@ -49,6 +49,7 @@ UUID_DEFINE(UUID_NULL, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 static void
 read_node(uint8_t *node)
 {
+#if NETWORKING
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
@@ -65,6 +66,7 @@ read_node(uint8_t *node)
 		}
 	}
 	ifnet_head_done();
+#endif /* NETWORKING */
 
 	read_random(node, 6);
 	node[0] |= 0x01;
diff --git a/bsd/net/zlib.c b/libkern/zlib.c
similarity index 93%
rename from bsd/net/zlib.c
rename to libkern/zlib.c
index 7be8133ad..0377edb1f 100644
--- a/bsd/net/zlib.c
+++ b/libkern/zlib.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * This file is derived from various .h and .c files from the zlib-1.0.4
@@ -34,6 +40,7 @@
  * $FreeBSD: src/sys/net/zlib.c,v 1.10 1999/12/29 04:38:38 peter Exp $
  */
 
+#define STDC
 #define NO_DUMMY_DECL
 #define NO_ZCFUNCS
 #define MY_ZCALLOC
@@ -49,13 +56,11 @@
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 #ifndef _Z_UTIL_H
 #define _Z_UTIL_H
 
 #ifdef KERNEL
-#include <net/zlib.h>
+#include <libkern/zlib.h>
 #else
 #include "zlib.h"
 #endif
@@ -90,19 +95,24 @@
 #endif /* __KERNEL__ */
 #endif /* KERNEL */
 
-#ifndef local
-#  define local static
-#endif
-/* compile with -Dlocal if your debugger can't find static symbols */
-
 typedef unsigned char  uch;
 typedef uch FAR uchf;
 typedef unsigned short ush;
 typedef ush FAR ushf;
 typedef unsigned long  ulg;
 
-extern const char *z_errmsg[10]; /* indexed by 2-zlib_error */
 /* (size given to avoid silly warnings with Visual C++) */
+static const char *z_errmsg[10] = {  /* indexed by 2-zlib_error */
+"need dictionary",     /* Z_NEED_DICT       2  */
+"stream end",          /* Z_STREAM_END      1  */
+"",                    /* Z_OK              0  */
+"file error",          /* Z_ERRNO         (-1) */
+"stream error",        /* Z_STREAM_ERROR  (-2) */
+"data error",          /* Z_DATA_ERROR    (-3) */
+"insufficient memory", /* Z_MEM_ERROR     (-4) */
+"buffer error",        /* Z_BUF_ERROR     (-5) */
+"incompatible version",/* Z_VERSION_ERROR (-6) */
+""};
 
 #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)]
 
@@ -262,12 +272,12 @@ extern const char *z_errmsg[10]; /* indexed by 2-zlib_error */
 #  define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;}
 #  define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;}
 #else
-#  define Assert(cond,msg)
-#  define Trace(x)
-#  define Tracev(x)
-#  define Tracevv(x)
-#  define Tracec(c,x)
-#  define Tracecv(c,x)
+#  define Assert(cond,msg) do {} while(0)
+#  define Trace(x) do {} while(0)
+#  define Tracev(x) do {} while(0)
+#  define Tracevv(x) do {} while(0)
+#  define Tracec(c,x) do {} while(0)
+#  define Tracecv(c,x) do {} while(0)
 #endif
 
 
@@ -295,8 +305,6 @@ void   zcfree  OF((voidpf opaque, voidpf ptr));
    subject to change. Applications should only use zlib.h.
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 #ifndef _DEFLATE_H
 #define _DEFLATE_H
 
@@ -553,12 +561,12 @@ typedef struct deflate_state {
  */
 
         /* in trees.c */
-void _tr_init         OF((deflate_state *s));
-int  _tr_tally        OF((deflate_state *s, unsigned dist, unsigned lc));
-void _tr_flush_block  OF((deflate_state *s, charf *buf, ulg stored_len,
+static void _tr_init         OF((deflate_state *s));
+static int  _tr_tally        OF((deflate_state *s, unsigned dist, unsigned lc));
+static void _tr_flush_block  OF((deflate_state *s, charf *buf, ulg stored_len,
 			  int eof));
-void _tr_align        OF((deflate_state *s));
-void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len,
+static void _tr_align        OF((deflate_state *s));
+static void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len,
                           int eof));
 
 #define d_code(dist) \
@@ -655,8 +663,6 @@ void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len,
  *
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 /* #include "deflate.h" */
 
 const char deflate_copyright[] =
@@ -681,23 +687,23 @@ typedef enum {
 typedef block_state (*compress_func) OF((deflate_state *s, int flush));
 /* Compression function. Returns the block state after the call. */
 
-local void fill_window    OF((deflate_state *s));
-local block_state deflate_stored OF((deflate_state *s, int flush));
-local block_state deflate_fast   OF((deflate_state *s, int flush));
-local block_state deflate_slow   OF((deflate_state *s, int flush));
-local void lm_init        OF((deflate_state *s));
-local void putShortMSB    OF((deflate_state *s, uInt b));
-local void flush_pending  OF((z_streamp strm));
-local int read_buf        OF((z_streamp strm, Bytef *buf, unsigned size));
+static void fill_window    OF((deflate_state *s));
+static block_state deflate_stored OF((deflate_state *s, int flush));
+static block_state deflate_fast   OF((deflate_state *s, int flush));
+static block_state deflate_slow   OF((deflate_state *s, int flush));
+static void lm_init        OF((deflate_state *s));
+static void putShortMSB    OF((deflate_state *s, uInt b));
+static void flush_pending  OF((z_streamp strm));
+static int read_buf        OF((z_streamp strm, Bytef *buf, unsigned size));
 #ifdef ASMV
       void match_init OF((void)); /* asm code initialization */
       uInt longest_match  OF((deflate_state *s, IPos cur_match));
 #else
-local uInt longest_match  OF((deflate_state *s, IPos cur_match));
+static uInt longest_match  OF((deflate_state *s, IPos cur_match));
 #endif
 
 #ifdef DEBUG_ZLIB
-local  void check_match OF((deflate_state *s, IPos start, IPos match,
+static  void check_match OF((deflate_state *s, IPos start, IPos match,
                             int length));
 #endif
 
@@ -731,7 +737,7 @@ typedef struct config_s {
    compress_func func;
 } config;
 
-local const config configuration_table[10] = {
+static const config configuration_table[10] = {
 /*      good lazy nice chain */
 /* 0 */ {0,    0,  0,    0, deflate_stored},  /* store only */
 /* 1 */ {4,    4,  8,    4, deflate_fast}, /* maximum speed, no lazy matches */
@@ -797,28 +803,18 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */
     zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head));
 
 /* ========================================================================= */
-int ZEXPORT deflateInit_(strm, level, version, stream_size)
-    z_streamp strm;
-    int level;
-    const char *version;
-    int stream_size;
+int ZEXPORT
+deflateInit_(z_streamp strm, int level, const char *ver, int stream_size)
 {
     return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL,
-			 Z_DEFAULT_STRATEGY, version, stream_size);
+			 Z_DEFAULT_STRATEGY, ver, stream_size);
     /* To do: ignore strm->next_in if we use it as window */
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
-		  version, stream_size)
-    z_streamp strm;
-    int  level;
-    int  method;
-    int  windowBits;
-    int  memLevel;
-    int  strategy;
-    const char *version;
-    int stream_size;
+int ZEXPORT
+deflateInit2_(z_streamp strm, int level, int method, int windowBits,
+	      int memLevel, int strategy, const char *ver, int stream_size)
 {
     deflate_state *s;
     int noheader = 0;
@@ -829,7 +825,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
      * output size for (length,distance) codes is <= 24 bits.
      */
 
-    if (version == Z_NULL || version[0] != my_version[0] ||
+    if (ver == Z_NULL || ver[0] != my_version[0] ||
         stream_size != sizeof(z_stream)) {
 	return Z_VERSION_ERROR;
     }
@@ -900,10 +896,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy,
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
-    z_streamp strm;
-    const Bytef *dictionary;
-    uInt  dictLength;
+int ZEXPORT
+deflateSetDictionary(z_streamp strm, const Bytef *dictionary, uInt dictLength)
 {
     deflate_state *s;
     uInt length = dictLength;
@@ -941,8 +935,8 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateReset (strm)
-    z_streamp strm;
+int ZEXPORT
+deflateReset(z_streamp strm)
 {
     deflate_state *s;
     
@@ -971,10 +965,8 @@ int ZEXPORT deflateReset (strm)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateParams(strm, level, strategy)
-    z_streamp strm;
-    int level;
-    int strategy;
+int ZEXPORT
+deflateParams(z_streamp strm, int level, int strategy)
 {
     deflate_state *s;
     compress_func func;
@@ -1011,9 +1003,8 @@ int ZEXPORT deflateParams(strm, level, strategy)
  * IN assertion: the stream state is correct and there is enough room in
  * pending_buf.
  */
-local void putShortMSB (s, b)
-    deflate_state *s;
-    uInt b;
+static void
+putShortMSB(deflate_state *s, uInt b)
 {
     put_byte(s, (Byte)(b >> 8));
     put_byte(s, (Byte)(b & 0xff));
@@ -1025,8 +1016,8 @@ local void putShortMSB (s, b)
  * to avoid allocating a large strm->next_out buffer and copying into it.
  * (See also read_buf()).
  */
-local void flush_pending(strm)
-    z_streamp strm;
+static void
+flush_pending(z_streamp strm)
 {
 	deflate_state* s = (deflate_state*)strm->state;
     unsigned len = s->pending;
@@ -1046,9 +1037,8 @@ local void flush_pending(strm)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflate (strm, flush)
-    z_streamp strm;
-    int flush;
+int ZEXPORT
+deflate (z_streamp strm, int flush)
 {
     int old_flush; /* value of flush param for previous deflate call */
     deflate_state *s;
@@ -1180,8 +1170,8 @@ int ZEXPORT deflate (strm, flush)
 }
 
 /* ========================================================================= */
-int ZEXPORT deflateEnd (strm)
-    z_streamp strm;
+int ZEXPORT
+deflateEnd(z_streamp strm)
 {
 	deflate_state*	s;
     int status;
@@ -1212,9 +1202,8 @@ int ZEXPORT deflateEnd (strm)
  * To simplify the source, this is not supported for 16-bit MSDOS (which
  * doesn't have enough memory anyway to duplicate compression states).
  */
-int ZEXPORT deflateCopy (dest, source)
-    z_streamp dest;
-    z_streamp source;
+int ZEXPORT
+deflateCopy(z_streamp dest, z_streamp source)
 {
 #ifdef MAXSEG_64K
     return Z_STREAM_ERROR;
@@ -1274,10 +1263,8 @@ int ZEXPORT deflateCopy (dest, source)
  * allocating a large strm->next_in buffer and copying from it.
  * (See also flush_pending()).
  */
-local int read_buf(strm, buf, size)
-    z_streamp strm;
-    Bytef *buf;
-    unsigned size;
+static int
+read_buf(z_streamp strm, Bytef *buf, unsigned int size)
 {
     unsigned len = strm->avail_in;
 
@@ -1299,8 +1286,8 @@ local int read_buf(strm, buf, size)
 /* ===========================================================================
  * Initialize the "longest match" routines for a new zlib stream
  */
-local void lm_init (s)
-    deflate_state *s;
+static void
+lm_init(deflate_state *s)
 {
     s->window_size = (ulg)2L*s->w_size;
 
@@ -1338,14 +1325,13 @@ local void lm_init (s)
  * match.S. The code will be functionally equivalent.
  */
 #ifndef FASTEST
-local uInt longest_match(s, cur_match)
-    deflate_state *s;
-    IPos cur_match;                             /* current match */
+static uInt
+longest_match(deflate_state *s, IPos cur_match)
 {
     unsigned chain_length = s->max_chain_length;/* max hash chain length */
-    register Bytef *scan = s->window + s->strstart; /* current string */
-    register Bytef *match;                       /* matched string */
-    register int len;                           /* length of current match */
+    Bytef *scan = s->window + s->strstart; /* current string */
+    Bytef *match;                       /* matched string */
+    int len;                           /* length of current match */
     int best_len = s->prev_length;              /* best match length so far */
     int nice_match = s->nice_match;             /* stop if match long enough */
     IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
@@ -1360,13 +1346,13 @@ local uInt longest_match(s, cur_match)
     /* Compare two bytes at a time. Note: this is not always beneficial.
      * Try with and without -DUNALIGNED_OK to check.
      */
-    register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1;
-    register ush scan_start = *(ushf*)scan;
-    register ush scan_end   = *(ushf*)(scan+best_len-1);
+    Bytef *strend = s->window + s->strstart + MAX_MATCH - 1;
+    ush scan_start = *(ushf*)scan;
+    ush scan_end   = *(ushf*)(scan+best_len-1);
 #else
-    register Bytef *strend = s->window + s->strstart + MAX_MATCH;
-    register Byte scan_end1  = scan[best_len-1];
-    register Byte scan_end   = scan[best_len];
+    Bytef *strend = s->window + s->strstart + MAX_MATCH;
+    Byte scan_end1  = scan[best_len-1];
+    Byte scan_end   = scan[best_len];
 #endif
 
     /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16.
@@ -1480,14 +1466,13 @@ local uInt longest_match(s, cur_match)
 /* ---------------------------------------------------------------------------
  * Optimized version for level == 1 only
  */
-local uInt longest_match(s, cur_match)
-    deflate_state *s;
-    IPos cur_match;                             /* current match */
+static uInt
+longest_match(deflate_state *s, IPos cur_match)
 {
-    register Bytef *scan = s->window + s->strstart; /* current string */
-    register Bytef *match;                       /* matched string */
-    register int len;                           /* length of current match */
-    register Bytef *strend = s->window + s->strstart + MAX_MATCH;
+    Bytef *scan = s->window + s->strstart; /* current string */
+    Bytef *match;                       /* matched string */
+    int len;                           /* length of current match */
+    Bytef *strend = s->window + s->strstart + MAX_MATCH;
 
     /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16.
      * It is easy to get rid of this optimization if necessary.
@@ -1539,10 +1524,8 @@ local uInt longest_match(s, cur_match)
 /* ===========================================================================
  * Check that the match at match_start is indeed a match.
  */
-local void check_match(s, start, match, length)
-    deflate_state *s;
-    IPos start, match;
-    int length;
+static void
+check_match(deflate_state *s, IPos start, IPos match, int length)
 {
     /* check that the match is indeed a match */
     if (zmemcmp(s->window + match,
@@ -1573,11 +1556,11 @@ local void check_match(s, start, match, length)
  *    performed for at least two bytes (required for the zip translate_eol
  *    option -- not supported here).
  */
-local void fill_window(s)
-    deflate_state *s;
+static void
+fill_window(deflate_state *s)
 {
-    register unsigned n, m;
-    register Posf *p;
+    unsigned n, m;
+    Posf *p;
     unsigned more;    /* Amount of free space at the end of the window. */
     uInt wsize = s->w_size;
 
@@ -1693,9 +1676,8 @@ local void fill_window(s)
  * NOTE: this function should be optimized to avoid extra copying from
  * window to pending_buf.
  */
-local block_state deflate_stored(s, flush)
-    deflate_state *s;
-    int flush;
+static block_state
+deflate_stored(deflate_state *s, int flush)
 {
     /* Stored blocks are limited to 0xffff bytes, pending_buf is limited
      * to pending_buf_size, and each stored block has a 5 byte header:
@@ -1751,9 +1733,8 @@ local block_state deflate_stored(s, flush)
  * new strings in the dictionary only for unmatched strings or for short
  * matches. It is used only for the fast compression options.
  */
-local block_state deflate_fast(s, flush)
-    deflate_state *s;
-    int flush;
+static block_state
+deflate_fast(deflate_state *s, int flush)
 {
     IPos hash_head = NIL; /* head of the hash chain */
     int bflush;           /* set if current block must be flushed */
@@ -1847,9 +1828,8 @@ local block_state deflate_fast(s, flush)
  * evaluation for matches: a match is finally adopted only if there is
  * no better match at the next window position.
  */
-local block_state deflate_slow(s, flush)
-    deflate_state *s;
-    int flush;
+static block_state
+deflate_slow(deflate_state *s, int flush)
 {
     IPos hash_head = NIL;    /* head of hash chain */
     int bflush;              /* set if current block must be flushed */
@@ -1997,8 +1977,6 @@ local block_state deflate_slow(s, flush)
  *          Addison-Wesley, 1983. ISBN 0-201-06672-6.
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 /* #define GEN_TREES_H */
 
 /* #include "deflate.h" */
@@ -2026,16 +2004,16 @@ local block_state deflate_slow(s, flush)
 #define REPZ_11_138  18
 /* repeat a zero length 11-138 times  (7 bits of repeat count) */
 
-local const int extra_lbits[LENGTH_CODES] /* extra bits for each length code */
+static const int extra_lbits[LENGTH_CODES] /* extra bits for each length code */
    = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0};
 
-local const int extra_dbits[D_CODES] /* extra bits for each distance code */
+static const int extra_dbits[D_CODES] /* extra bits for each distance code */
    = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13};
 
-local const int extra_blbits[BL_CODES]/* extra bits for each bit length code */
+static const int extra_blbits[BL_CODES]/* extra bits for each bit length code */
    = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7};
 
-local const uch bl_order[BL_CODES]
+static const uch bl_order[BL_CODES]
    = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15};
 /* The lengths of the bit length codes are sent in order of decreasing
  * probability, to avoid transmitting the lengths for unused bit length codes.
@@ -2055,14 +2033,14 @@ local const uch bl_order[BL_CODES]
 #if defined(GEN_TREES_H) || !defined(STDC)
 /* non ANSI compilers may not accept trees.h */
 
-local ct_data *static_ltree = Z_NULL;
+static ct_data *static_ltree = Z_NULL;
 /* The static literal tree. Since the bit lengths are imposed, there is no
  * need for the L_CODES extra codes used during heap construction. However
  * The codes 286 and 287 are needed to build a canonical tree (see _tr_init
  * below).
  */
 
-local ct_data *static_dtree = Z_NULL;
+static ct_data *static_dtree = Z_NULL;
 /* The static distance tree. (Actually a trivial tree since all codes use
  * 5 bits.)
  */
@@ -2076,17 +2054,17 @@ uch *_dist_code = Z_NULL;
 uch *_length_code = Z_NULL;
 /* length code for each normalized match length (0 == MIN_MATCH) */
 
-local int *base_length = Z_NULL;
+static int *base_length = Z_NULL;
 /* First normalized length for each code (0 = MIN_MATCH) */
 
-local int *base_dist = Z_NULL;
+static int *base_dist = Z_NULL;
 /* First normalized distance for each code (0 = distance of 1) */
 
 #else
 /* +++ trees.h */
 /* header created automatically with -DGEN_TREES_H */
 
-local const ct_data static_ltree[L_CODES+2] = {
+static const ct_data static_ltree[L_CODES+2] = {
 {{ 12},{  8}}, {{140},{  8}}, {{ 76},{  8}}, {{204},{  8}}, {{ 44},{  8}},
 {{172},{  8}}, {{108},{  8}}, {{236},{  8}}, {{ 28},{  8}}, {{156},{  8}},
 {{ 92},{  8}}, {{220},{  8}}, {{ 60},{  8}}, {{188},{  8}}, {{124},{  8}},
@@ -2147,7 +2125,7 @@ local const ct_data static_ltree[L_CODES+2] = {
 {{163},{  8}}, {{ 99},{  8}}, {{227},{  8}}
 };
 
-local const ct_data static_dtree[D_CODES] = {
+static const ct_data static_dtree[D_CODES] = {
 {{ 0},{ 5}}, {{16},{ 5}}, {{ 8},{ 5}}, {{24},{ 5}}, {{ 4},{ 5}},
 {{20},{ 5}}, {{12},{ 5}}, {{28},{ 5}}, {{ 2},{ 5}}, {{18},{ 5}},
 {{10},{ 5}}, {{26},{ 5}}, {{ 6},{ 5}}, {{22},{ 5}}, {{14},{ 5}},
@@ -2201,12 +2179,12 @@ const uch _length_code[MAX_MATCH-MIN_MATCH+1]= {
 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28
 };
 
-local const int base_length[LENGTH_CODES] = {
+static const int base_length[LENGTH_CODES] = {
 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
 64, 80, 96, 112, 128, 160, 192, 224, 0
 };
 
-local const int base_dist[D_CODES] = {
+static const int base_dist[D_CODES] = {
     0,     1,     2,     3,     4,     6,     8,    12,    16,    24,
    32,    48,    64,    96,   128,   192,   256,   384,   512,   768,
  1024,  1536,  2048,  3072,  4096,  6144,  8192, 12288, 16384, 24576
@@ -2223,41 +2201,41 @@ struct static_tree_desc_s {
     int     max_length;          /* max bit length for the codes */
 };
 
-local static_tree_desc  static_l_desc =
+static static_tree_desc  static_l_desc =
 {NULL, extra_lbits, LITERALS+1, L_CODES, MAX_BITS};
 
-local static_tree_desc  static_d_desc =
+static static_tree_desc  static_d_desc =
 {NULL, extra_dbits, 0,          D_CODES, MAX_BITS};
 
-local static_tree_desc  static_bl_desc =
+static static_tree_desc  static_bl_desc =
 {(const ct_data *)0, extra_blbits, 0,   BL_CODES, MAX_BL_BITS};
 
 /* ===========================================================================
  * Local (static) routines in this file.
  */
 
-local int tr_static_init  OF((z_streamp z));
-local void init_block     OF((deflate_state *s));
-local void pqdownheap     OF((deflate_state *s, ct_data *tree, int k));
-local void gen_bitlen     OF((deflate_state *s, tree_desc *desc));
-local void gen_codes      OF((ct_data *tree, int max_code, ushf *bl_count));
-local void build_tree     OF((deflate_state *s, tree_desc *desc));
-local void scan_tree      OF((deflate_state *s, ct_data *tree, int max_code));
-local void send_tree      OF((deflate_state *s, ct_data *tree, int max_code));
-local int  build_bl_tree  OF((deflate_state *s));
-local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes,
+static int tr_static_init  OF((z_streamp z));
+static void init_block     OF((deflate_state *s));
+static void pqdownheap     OF((deflate_state *s, ct_data *tree, int k));
+static void gen_bitlen     OF((deflate_state *s, tree_desc *desc));
+static void gen_codes      OF((ct_data *tree, int max_code, ushf *bl_count));
+static void build_tree     OF((deflate_state *s, tree_desc *desc));
+static void scan_tree      OF((deflate_state *s, ct_data *tree, int max_code));
+static void send_tree      OF((deflate_state *s, ct_data *tree, int max_code));
+static int  build_bl_tree  OF((deflate_state *s));
+static void send_all_trees OF((deflate_state *s, int lcodes, int dcodes,
                               int blcodes));
-local void compress_block OF((deflate_state *s, ct_data *ltree,
+static void compress_block OF((deflate_state *s, ct_data *ltree,
                               ct_data *dtree));
-local void set_data_type  OF((deflate_state *s));
-local unsigned bi_reverse OF((unsigned value, int length));
-local void bi_windup      OF((deflate_state *s));
-local void bi_flush       OF((deflate_state *s));
-local void copy_block     OF((deflate_state *s, charf *buf, unsigned len,
+static void set_data_type  OF((deflate_state *s));
+static unsigned bi_reverse OF((unsigned value, int length));
+static void bi_windup      OF((deflate_state *s));
+static void bi_flush       OF((deflate_state *s));
+static void copy_block     OF((deflate_state *s, charf *buf, unsigned len,
                               int header));
 
 #ifdef GEN_TREES_H
-local void gen_trees_header OF((void));
+static void gen_trees_header OF((void));
 #endif
 
 #ifndef DEBUG_ZLIB
@@ -2284,12 +2262,9 @@ local void gen_trees_header OF((void));
  * IN assertion: length <= 16 and value fits in length bits.
  */
 #ifdef DEBUG_ZLIB
-local void send_bits      OF((deflate_state *s, int value, int length));
+static void send_bits      OF((deflate_state *s, int value, int length));
 
-local void send_bits(s, value, length)
-    deflate_state *s;
-    int value;  /* value to send */
-    int length; /* number of bits */
+static void send_bits(deflate_state *s, int value, int length)
 {
     Tracevv((stderr," l %2d v %4x ", length, value));
     Assert(length > 0 && length <= 15, "invalid length");
@@ -2341,12 +2316,14 @@ typedef struct {
 	int		base_dist[D_CODES];
 } __used_to_be_static;
 
+#if defined(GEN_TREES_H) || !defined(STDC)
 static __used_to_be_static	*static_storage = Z_NULL;
+#endif /* defined(GEN_TREES_H) || !defined(STDC) */
 
 /* ===========================================================================
  * Initialize the various 'constant' tables.
  */
-local int tr_static_init(
+static int tr_static_init(
 	z_streamp	z)
 {
 #if defined(GEN_TREES_H) || !defined(STDC)
@@ -2506,8 +2483,8 @@ void gen_trees_header()
 /* ===========================================================================
  * Initialize the tree data structures for a new zlib stream.
  */
-void _tr_init(s)
-    deflate_state *s;
+static void
+_tr_init(deflate_state *s)
 {
     tr_static_init(s->strm);
 
@@ -2535,8 +2512,8 @@ void _tr_init(s)
 /* ===========================================================================
  * Initialize a new block.
  */
-local void init_block(s)
-    deflate_state *s;
+static void
+init_block(deflate_state *s)
 {
     int n; /* iterates over tree elements */
 
@@ -2578,11 +2555,11 @@ local void init_block(s)
  * exchanging a node with the smallest of its two sons if necessary, stopping
  * when the heap property is re-established (each father smaller than its
  * two sons).
+ * ct_data *tree;  the tree to restore
+ * int k;               node to move down
  */
-local void pqdownheap(s, tree, k)
-    deflate_state *s;
-    ct_data *tree;  /* the tree to restore */
-    int k;               /* node to move down */
+static void
+pqdownheap(deflate_state *s, ct_data *tree, int k)
 {
     int v = s->heap[k];
     int j = k << 1;  /* left son of k */
@@ -2614,9 +2591,8 @@ local void pqdownheap(s, tree, k)
  *     The length opt_len is updated; static_len is also updated if stree is
  *     not null.
  */
-local void gen_bitlen(s, desc)
-    deflate_state *s;
-    tree_desc *desc;    /* the tree descriptor */
+static void
+gen_bitlen(deflate_state *s, tree_desc *desc)
 {
     ct_data *tree        = desc->dyn_tree;
     int max_code         = desc->max_code;
@@ -2700,11 +2676,13 @@ local void gen_bitlen(s, desc)
  * the given tree and the field len is set for all tree elements.
  * OUT assertion: the field code is set for all tree elements of non
  *     zero code length.
+ *
+ * ct_data *tree;             the tree to decorate
+ * int max_code;              largest code with non zero frequency
+ * ushf *bl_count;            number of codes at each bit length
  */
-local void gen_codes (tree, max_code, bl_count)
-    ct_data *tree;             /* the tree to decorate */
-    int max_code;              /* largest code with non zero frequency */
-    ushf *bl_count;            /* number of codes at each bit length */
+static void
+gen_codes(ct_data *tree, int max_code, ushf *bl_count)
 {
     ush next_code[MAX_BITS+1]; /* next code value for each bit length */
     ush code = 0;              /* running code value */
@@ -2743,9 +2721,8 @@ local void gen_codes (tree, max_code, bl_count)
  *     and corresponding code. The length opt_len is updated; static_len is
  *     also updated if stree is not null. The field max_code is set.
  */
-local void build_tree(s, desc)
-    deflate_state *s;
-    tree_desc *desc; /* the tree descriptor */
+static void
+build_tree(deflate_state *s, tree_desc *desc)
 {
     ct_data *tree         = desc->dyn_tree;
     const ct_data *stree  = desc->stat_desc->static_tree;
@@ -2829,11 +2806,12 @@ local void build_tree(s, desc)
 /* ===========================================================================
  * Scan a literal or distance tree to determine the frequencies of the codes
  * in the bit length tree.
+ *
+ * ct_data *tree;   the tree to be scanned
+ * int max_code;    and its largest code of non zero frequency
  */
-local void scan_tree (s, tree, max_code)
-    deflate_state *s;
-    ct_data *tree;   /* the tree to be scanned */
-    int max_code;    /* and its largest code of non zero frequency */
+static void
+scan_tree(deflate_state *s, ct_data *tree, int max_code)
 {
     int n;                     /* iterates over all tree elements */
     int prevlen = -1;          /* last emitted length */
@@ -2874,11 +2852,12 @@ local void scan_tree (s, tree, max_code)
 /* ===========================================================================
  * Send a literal or distance tree in compressed form, using the codes in
  * bl_tree.
+ *
+ * ct_data *tree; the tree to be scanned
+ * int max_code;       and its largest code of non zero frequency
  */
-local void send_tree (s, tree, max_code)
-    deflate_state *s;
-    ct_data *tree; /* the tree to be scanned */
-    int max_code;       /* and its largest code of non zero frequency */
+static void
+send_tree(deflate_state *s, ct_data *tree, int max_code)
 {
     int n;                     /* iterates over all tree elements */
     int prevlen = -1;          /* last emitted length */
@@ -2926,8 +2905,8 @@ local void send_tree (s, tree, max_code)
  * Construct the Huffman tree for the bit lengths and return the index in
  * bl_order of the last bit length code to send.
  */
-local int build_bl_tree(s)
-    deflate_state *s;
+static int
+build_bl_tree(deflate_state *s)
 {
     int max_blindex;  /* index of last bit length code of non zero freq */
 
@@ -2961,9 +2940,8 @@ local int build_bl_tree(s)
  * lengths of the bit length codes, the literal tree and the distance tree.
  * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4.
  */
-local void send_all_trees(s, lcodes, dcodes, blcodes)
-    deflate_state *s;
-    int lcodes, dcodes, blcodes; /* number of codes for each tree */
+static void
+send_all_trees(deflate_state *s, int lcodes, int dcodes, int blcodes)
 {
     int rank;                    /* index in bl_order */
 
@@ -2989,12 +2967,13 @@ local void send_all_trees(s, lcodes, dcodes, blcodes)
 
 /* ===========================================================================
  * Send a stored block
+ *
+ * charf *buf;       input block
+ * ulg stored_len;   length of input block
+ * int eof;          true if this is the last block for a file
  */
-void _tr_stored_block(s, buf, stored_len, eof)
-    deflate_state *s;
-    charf *buf;       /* input block */
-    ulg stored_len;   /* length of input block */
-    int eof;          /* true if this is the last block for a file */
+static void
+_tr_stored_block(deflate_state *s, charf *buf, ulg stored_len, int eof)
 {
     send_bits(s, (STORED_BLOCK<<1)+eof, 3);  /* send block type */
 #ifdef DEBUG_ZLIB
@@ -3015,8 +2994,8 @@ void _tr_stored_block(s, buf, stored_len, eof)
  * To simplify the code, we assume the worst case of last real code encoded
  * on one bit only.
  */
-void _tr_align(s)
-    deflate_state *s;
+static void
+_tr_align(deflate_state *s)
 {
     send_bits(s, STATIC_TREES<<1, 3);
     send_code(s, END_BLOCK, static_ltree);
@@ -3043,12 +3022,13 @@ void _tr_align(s)
 /* ===========================================================================
  * Determine the best encoding for the current block: dynamic trees, static
  * trees or store, and output the encoded block to the zip file.
+ *
+ * charf *buf;       input block, or NULL if too old
+ * ulg stored_len;   length of input block
+ * int eof;          true if this is the last block for a file
  */
-void _tr_flush_block(s, buf, stored_len, eof)
-    deflate_state *s;
-    charf *buf;       /* input block, or NULL if too old */
-    ulg stored_len;   /* length of input block */
-    int eof;          /* true if this is the last block for a file */
+static void
+_tr_flush_block(deflate_state *s, charf *buf, ulg stored_len, int eof)
 {
     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
     int max_blindex = 0;  /* index of last bit length code of non zero freq */
@@ -3143,11 +3123,12 @@ void _tr_flush_block(s, buf, stored_len, eof)
 /* ===========================================================================
  * Save the match info and tally the frequency counts. Return true if
  * the current block must be flushed.
+ *
+ * unsigned dist;  distance of matched string
+ * unsigned lc;    match length-MIN_MATCH or unmatched char (if dist==0)
  */
-int _tr_tally (s, dist, lc)
-    deflate_state *s;
-    unsigned dist;  /* distance of matched string */
-    unsigned lc;    /* match length-MIN_MATCH or unmatched char (if dist==0) */
+static int
+_tr_tally(deflate_state *s, unsigned int dist, unsigned int lc)
 {
     s->d_buf[s->last_lit] = (ush)dist;
     s->l_buf[s->last_lit++] = (uch)lc;
@@ -3193,11 +3174,12 @@ int _tr_tally (s, dist, lc)
 
 /* ===========================================================================
  * Send the block data compressed using the given Huffman trees
+ *
+ * ct_data *ltree; literal tree
+ * ct_data *dtree; distance tree
  */
-local void compress_block(s, ltree, dtree)
-    deflate_state *s;
-    ct_data *ltree; /* literal tree */
-    ct_data *dtree; /* distance tree */
+static void
+compress_block(deflate_state *s, ct_data *ltree, ct_data *dtree)
 {
     unsigned dist;      /* distance of matched string */
     int lc;             /* match length or unmatched char (if dist == 0) */
@@ -3247,8 +3229,8 @@ local void compress_block(s, ltree, dtree)
  * IN assertion: the fields freq of dyn_ltree are set and the total of all
  * frequencies does not exceed 64K (to fit in an int on 16 bit machines).
  */
-local void set_data_type(s)
-    deflate_state *s;
+static void
+set_data_type(deflate_state *s)
 {
     int n = 0;
     unsigned ascii_freq = 0;
@@ -3264,11 +3246,10 @@ local void set_data_type(s)
  * method would use a table)
  * IN assertion: 1 <= len <= 15
  */
-local unsigned bi_reverse(code, len)
-    unsigned code; /* the value to invert */
-    int len;       /* its bit length */
+static unsigned
+bi_reverse(unsigned code, int len)
 {
-    register unsigned res = 0;
+    unsigned res = 0;
     do {
         res |= code & 1;
         code >>= 1, res <<= 1;
@@ -3279,8 +3260,8 @@ local unsigned bi_reverse(code, len)
 /* ===========================================================================
  * Flush the bit buffer, keeping at most 7 bits in it.
  */
-local void bi_flush(s)
-    deflate_state *s;
+static void
+bi_flush(deflate_state *s)
 {
     if (s->bi_valid == 16) {
         put_short(s, s->bi_buf);
@@ -3296,8 +3277,8 @@ local void bi_flush(s)
 /* ===========================================================================
  * Flush the bit buffer and align the output on a byte boundary
  */
-local void bi_windup(s)
-    deflate_state *s;
+static void
+bi_windup(deflate_state *s)
 {
     if (s->bi_valid > 8) {
         put_short(s, s->bi_buf);
@@ -3314,12 +3295,13 @@ local void bi_windup(s)
 /* ===========================================================================
  * Copy a stored block, storing first the length and its
  * one's complement if requested.
+ *
+ * charf    *buf;    the input data
+ * unsigned len;     its length
+ * int      header;  true if block header must be written
  */
-local void copy_block(s, buf, len, header)
-    deflate_state *s;
-    charf    *buf;    /* the input data */
-    unsigned len;     /* its length */
-    int      header;  /* true if block header must be written */
+static void
+copy_block(deflate_state *s, charf *buf, unsigned int len, int header)
 {
     bi_windup(s);        /* align on byte boundary */
     s->last_eob_len = 8; /* enough lookahead for inflate */
@@ -3362,31 +3344,31 @@ local void copy_block(s, buf, len, header)
 struct inflate_blocks_state;
 typedef struct inflate_blocks_state FAR inflate_blocks_statef;
 
-extern inflate_blocks_statef * inflate_blocks_new OF((
+static inflate_blocks_statef * inflate_blocks_new OF((
     z_streamp z,
     check_func c,               /* check function */
     uInt w));                   /* window size */
 
-extern int inflate_blocks OF((
+static int inflate_blocks OF((
     inflate_blocks_statef *,
     z_streamp ,
     int));                      /* initial return code */
 
-extern void inflate_blocks_reset OF((
+static void inflate_blocks_reset OF((
     inflate_blocks_statef *,
     z_streamp ,
     uLongf *));                  /* check value on output */
 
-extern int inflate_blocks_free OF((
+static int inflate_blocks_free OF((
     inflate_blocks_statef *,
     z_streamp));
 
-extern void inflate_set_dictionary OF((
+static void inflate_set_dictionary OF((
     inflate_blocks_statef *s,
     const Bytef *d,  /* dictionary */
     uInt  n));       /* dictionary length */
 
-extern int inflate_blocks_sync_point OF((
+static int inflate_blocks_sync_point OF((
     inflate_blocks_statef *s));
 /* --- infblock.h */
 
@@ -3434,8 +3416,8 @@ typedef struct inflate_state {
 }inflate_state;
 
 
-int ZEXPORT inflateReset(z)
-z_streamp z;
+int ZEXPORT
+inflateReset(z_streamp z)
 {
 	inflate_state* s;
 	if (z == Z_NULL || z->state == Z_NULL)
@@ -3451,8 +3433,8 @@ z_streamp z;
 }
 
 
-int ZEXPORT inflateEnd(z)
-z_streamp z;
+int ZEXPORT
+inflateEnd(z_streamp z)
 {
   if (z == Z_NULL || z->state == Z_NULL || z->zfree == Z_NULL)
     return Z_STREAM_ERROR;
@@ -3465,14 +3447,11 @@ z_streamp z;
 }
 
 
-int ZEXPORT inflateInit2_(z, w, version, stream_size)
-z_streamp z;
-int w;
-const char *version;
-int stream_size;
+int ZEXPORT
+inflateInit2_(z_streamp z, int w, const char *ver, int stream_size)
 {
 	inflate_state*	s;
-  if (version == Z_NULL || version[0] != ZLIB_VERSION[0] ||
+  if (ver == Z_NULL || ver[0] != ZLIB_VERSION[0] ||
       stream_size != sizeof(z_stream))
       return Z_VERSION_ERROR;
 
@@ -3526,21 +3505,18 @@ int stream_size;
 }
 
 
-int ZEXPORT inflateInit_(z, version, stream_size)
-z_streamp z;
-const char *version;
-int stream_size;
+int ZEXPORT
+inflateInit_(z_streamp z, const char *ver, int stream_size)
 {
-  return inflateInit2_(z, DEF_WBITS, version, stream_size);
+  return inflateInit2_(z, DEF_WBITS, ver, stream_size);
 }
 
 
 #define NEEDBYTE {if(z->avail_in==0)return r;r=f;}
 #define NEXTBYTE (z->avail_in--,z->total_in++,*z->next_in++)
 
-int ZEXPORT inflate(z, f)
-z_streamp z;
-int f;
+int ZEXPORT
+inflate(z_streamp z, int f)
 {
   int r;
   uInt b;
@@ -3809,14 +3785,14 @@ struct inflate_huft_s {
    value below is more than safe. */
 #define MANY 1440
 
-extern int inflate_trees_bits OF((
+static int inflate_trees_bits OF((
     uIntf *,                    /* 19 code lengths */
     uIntf *,                    /* bits tree desired/actual depth */
     inflate_huft * FAR *,       /* bits tree result */
     inflate_huft *,             /* space for trees */
     z_streamp));                /* for messages */
 
-extern int inflate_trees_dynamic OF((
+static int inflate_trees_dynamic OF((
     uInt,                       /* number of literal/length codes */
     uInt,                       /* number of distance codes */
     uIntf *,                    /* that many (total) code lengths */
@@ -3827,7 +3803,7 @@ extern int inflate_trees_dynamic OF((
     inflate_huft *,             /* space for trees */
     z_streamp));                /* for messages */
 
-extern int inflate_trees_fixed OF((
+static int inflate_trees_fixed OF((
     uIntf *,                    /* literal desired/actual bit depth */
     uIntf *,                    /* distance desired/actual bit depth */
     inflate_huft * FAR *,       /* literal/length tree result */
@@ -3849,17 +3825,17 @@ extern int inflate_trees_fixed OF((
 struct inflate_codes_state;
 typedef struct inflate_codes_state FAR inflate_codes_statef;
 
-extern inflate_codes_statef *inflate_codes_new OF((
+static inflate_codes_statef *inflate_codes_new OF((
     uInt, uInt,
     inflate_huft *, inflate_huft *,
     z_streamp ));
 
-extern int inflate_codes OF((
+static int inflate_codes OF((
     inflate_blocks_statef *,
     z_streamp ,
     int));
 
-extern void inflate_codes_free OF((
+static void inflate_codes_free OF((
     inflate_codes_statef *,
     z_streamp ));
 
@@ -3953,10 +3929,15 @@ struct inflate_blocks_state {
 #define LOAD {LOADIN LOADOUT}
 
 /* masks for lower bits (size given to avoid silly warnings with Visual C++) */
-extern uInt inflate_mask[17];
+/* And'ing with mask[n] masks the lower n bits */
+static uInt inflate_mask[17] = {
+    0x0000,
+    0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff,
+    0x01ff, 0x03ff, 0x07ff, 0x0fff, 0x1fff, 0x3fff, 0x7fff, 0xffff
+};
 
 /* copy as much as possible from the sliding window to the output area */
-extern int inflate_flush OF((
+static int inflate_flush OF((
     inflate_blocks_statef *,
     z_streamp ,
     int));
@@ -3977,7 +3958,7 @@ struct inflate_codes_state {int dummy;}; /* for buggy compilers */
 #define bits word.what.Bits
 
 /* Table for deflate from PKZIP's appnote.txt. */
-local const uInt border[] = { /* Order of the bit length code lengths */
+static const uInt border[] = { /* Order of the bit length code lengths */
         16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
 
 /*
@@ -4026,7 +4007,7 @@ local const uInt border[] = { /* Order of the bit length code lengths */
  */
 
 
-void inflate_blocks_reset(s, z, c)
+static void inflate_blocks_reset(s, z, c)
 inflate_blocks_statef *s;
 z_streamp z;
 uLongf *c;
@@ -4047,7 +4028,7 @@ uLongf *c;
 }
 
 
-inflate_blocks_statef *inflate_blocks_new(z, c, w)
+static inflate_blocks_statef *inflate_blocks_new(z, c, w)
 z_streamp z;
 check_func c;
 uInt w;
@@ -4078,7 +4059,7 @@ uInt w;
 }
 
 
-int inflate_blocks(s, z, r)
+static int inflate_blocks(s, z, r)
 inflate_blocks_statef *s;
 z_streamp z;
 int r;
@@ -4330,7 +4311,7 @@ int r;
 }
 
 
-int inflate_blocks_free(s, z)
+static int inflate_blocks_free(s, z)
 inflate_blocks_statef *s;
 z_streamp z;
 {
@@ -4343,7 +4324,7 @@ z_streamp z;
 }
 
 
-void inflate_set_dictionary(s, d, n)
+static void inflate_set_dictionary(s, d, n)
 inflate_blocks_statef *s;
 const Bytef *d;
 uInt  n;
@@ -4357,7 +4338,7 @@ uInt  n;
  * by Z_SYNC_FLUSH or Z_FULL_FLUSH. 
  * IN assertion: s != Z_NULL
  */
-int inflate_blocks_sync_point(s)
+static int inflate_blocks_sync_point(s)
 inflate_blocks_statef *s;
 {
   return s->mode == LENS;
@@ -4395,7 +4376,8 @@ struct internal_state  {int dummy;}; /* for buggy compilers */
 #define bits word.what.Bits
 
 
-local int huft_build OF((
+static int
+huft_build OF((
     uIntf *,            /* code lengths in bits */
     uInt,               /* number of codes */
     uInt,               /* number of "simple" codes */
@@ -4408,18 +4390,18 @@ local int huft_build OF((
     uIntf * ));         /* space for values */
 
 /* Tables for deflate from PKZIP's appnote.txt. */
-local const uInt cplens[31] = { /* Copy lengths for literal codes 257..285 */
+static const uInt cplens[31] = { /* Copy lengths for literal codes 257..285 */
         3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31,
         35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0};
         /* see note #13 above about 258 */
-local const uInt cplext[31] = { /* Extra bits for literal codes 257..285 */
+static const uInt cplext[31] = { /* Extra bits for literal codes 257..285 */
         0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
         3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 112, 112}; /* 112==invalid */
-local const uInt cpdist[30] = { /* Copy offsets for distance codes 0..29 */
+static const uInt cpdist[30] = { /* Copy offsets for distance codes 0..29 */
         1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
         257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145,
         8193, 12289, 16385, 24577};
-local const uInt cpdext[30] = { /* Extra bits for distance codes */
+static const uInt cpdext[30] = { /* Extra bits for distance codes */
         0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,
         7, 7, 8, 8, 9, 9, 10, 10, 11, 11,
         12, 12, 13, 13};
@@ -4460,7 +4442,11 @@ local const uInt cpdext[30] = { /* Extra bits for distance codes */
 /* If BMAX needs to be larger than 16, then h and x[] should be uLong. */
 #define BMAX 15         /* maximum bit length of any code */
 
-local int huft_build(b, n, s, d, e, t, m, hp, hn, v)
+/* Given a list of code lengths and a maximum table size, make a set of
+   tables to decode that set of codes.  Return Z_OK on success, Z_BUF_ERROR
+   if the given code set is incomplete (the tables are still built in this
+   case), or Z_DATA_ERROR if the input is invalid. */
+#if 0
 uIntf *b;               /* code lengths in bits (all assumed <= BMAX) */
 uInt n;                 /* number of codes (assumed <= 288) */
 uInt s;                 /* number of simple-valued codes (0..s-1) */
@@ -4471,27 +4457,28 @@ uIntf *m;               /* maximum lookup bits, returns actual */
 inflate_huft *hp;       /* space for trees */
 uInt *hn;               /* hufts used in space */
 uIntf *v;               /* working area: values in order of bit length */
-/* Given a list of code lengths and a maximum table size, make a set of
-   tables to decode that set of codes.  Return Z_OK on success, Z_BUF_ERROR
-   if the given code set is incomplete (the tables are still built in this
-   case), or Z_DATA_ERROR if the input is invalid. */
-{
+#endif
 
+static int
+huft_build(uIntf *b, uInt n, uInt s, const uIntf *d, const uIntf *e,
+	   inflate_huft * FAR *t, uIntf *m, inflate_huft *hp, uInt *hn,
+	   uIntf *v)
+{
   uInt a;                       /* counter for codes of length k */
   uInt c[BMAX+1];               /* bit length count table */
   uInt f;                       /* i repeats in table every f entries */
   int g;                        /* maximum code length */
   int h;                        /* table level */
-  register uInt i;              /* counter, current code */
-  register uInt j;              /* counter */
-  register int k;               /* number of bits in current code */
+  uInt i;              /* counter, current code */
+  uInt j;              /* counter */
+  int k;               /* number of bits in current code */
   int l;                        /* bits per table (returned in m) */
   uInt mask;                    /* (1 << w) - 1, to avoid cc -O bug on HP */
-  register uIntf *p;            /* pointer into c[], b[], or v[] */
+  uIntf *p;            /* pointer into c[], b[], or v[] */
   inflate_huft *q;              /* points to current table */
   struct inflate_huft_s r;      /* table entry for structure assignment */
   inflate_huft *u[BMAX];        /* table stack */
-  register int w;               /* bits before this table == (l * h) */
+  int w;               /* bits before this table == (l * h) */
   uInt x[BMAX+1];               /* bit offsets, then code stack */
   uIntf *xp;                    /* pointer into x */
   int y;                        /* number of dummy codes added */
@@ -4660,7 +4647,7 @@ uIntf *v;               /* working area: values in order of bit length */
 }
 
 
-int inflate_trees_bits(c, bb, tb, hp, z)
+static int inflate_trees_bits(c, bb, tb, hp, z)
 uIntf *c;               /* 19 code lengths */
 uIntf *bb;              /* bits tree desired/actual depth */
 inflate_huft * FAR *tb; /* bits tree result */
@@ -4687,7 +4674,7 @@ z_streamp z;            /* for messages */
 }
 
 
-int inflate_trees_dynamic(nl, nd, c, bl, bd, tl, td, hp, z)
+static int inflate_trees_dynamic(nl, nd, c, bl, bd, tl, td, hp, z)
 uInt nl;                /* number of literal/length codes */
 uInt nd;                /* number of distance codes */
 uIntf *c;               /* that many (total) code lengths */
@@ -4753,13 +4740,13 @@ z_streamp z;            /* for messages */
 
 /* build fixed tables only once--keep them here */
 #ifdef BUILDFIXED
-local int fixed_built = 0;
+static int fixed_built = 0;
 #define FIXEDH 544      /* number of hufts used by fixed tables */
-local inflate_huft *fixed_mem = NULL;
-local uInt fixed_bl;
-local uInt fixed_bd;
-local inflate_huft *fixed_tl;
-local inflate_huft *fixed_td;
+static inflate_huft *fixed_mem = NULL;
+static uInt fixed_bl;
+static uInt fixed_bd;
+static inflate_huft *fixed_tl;
+static inflate_huft *fixed_td;
 #else
 /* +++ inffixed.h */
 /* inffixed.h -- table for decoding fixed codes
@@ -4771,9 +4758,9 @@ local inflate_huft *fixed_td;
    subject to change. Applications should only use zlib.h.
  */
 
-local uInt fixed_bl = 9;
-local uInt fixed_bd = 5;
-local inflate_huft fixed_tl[] = {
+static uInt fixed_bl = 9;
+static uInt fixed_bd = 5;
+static inflate_huft fixed_tl[] = {
     {{{96,7}},256}, {{{0,8}},80}, {{{0,8}},16}, {{{84,8}},115},
     {{{82,7}},31}, {{{0,8}},112}, {{{0,8}},48}, {{{0,9}},192},
     {{{80,7}},10}, {{{0,8}},96}, {{{0,8}},32}, {{{0,9}},160},
@@ -4903,7 +4890,7 @@ local inflate_huft fixed_tl[] = {
     {{{82,7}},27}, {{{0,8}},111}, {{{0,8}},47}, {{{0,9}},191},
     {{{0,8}},15}, {{{0,8}},143}, {{{0,8}},79}, {{{0,9}},255}
   };
-local inflate_huft fixed_td[] = {
+static inflate_huft fixed_td[] = {
     {{{80,5}},1}, {{{87,5}},257}, {{{83,5}},17}, {{{91,5}},4097},
     {{{81,5}},5}, {{{89,5}},1025}, {{{85,5}},65}, {{{93,5}},16385},
     {{{80,5}},3}, {{{88,5}},513}, {{{84,5}},33}, {{{92,5}},8193},
@@ -4917,7 +4904,7 @@ local inflate_huft fixed_td[] = {
 #endif
 
 
-int inflate_trees_fixed(bl, bd, tl, td, z)
+static int inflate_trees_fixed(bl, bd, tl, td, z)
 uIntf *bl;               /* literal desired/actual bit depth */
 uIntf *bd;               /* distance desired/actual bit depth */
 inflate_huft * FAR *tl;  /* literal/length tree result */
@@ -5006,7 +4993,7 @@ z_streamp z;             /* for memory allocation */
    subject to change. Applications should only use zlib.h.
  */
 
-extern int inflate_fast OF((
+static int inflate_fast OF((
     uInt,
     uInt,
     inflate_huft *,
@@ -5061,7 +5048,7 @@ struct inflate_codes_state {
 };
 
 
-inflate_codes_statef *inflate_codes_new(bl, bd, tl, td, z)
+static inflate_codes_statef *inflate_codes_new(bl, bd, tl, td, z)
 uInt bl, bd;
 inflate_huft *tl;
 inflate_huft *td; /* need separate declaration for Borland C++ */
@@ -5083,7 +5070,7 @@ z_streamp z;
 }
 
 
-int inflate_codes(s, z, r)
+static int inflate_codes(s, z, r)
 inflate_blocks_statef *s;
 z_streamp z;
 int r;
@@ -5222,7 +5209,7 @@ int r;
     case WASH:          /* o: got eob, possibly more output */
       if (k > 7)        /* return unused byte, if any */
       {
-        Assert(k < 16, "inflate_codes grabbed too many bytes")
+        Assert(k < 16, "inflate_codes grabbed too many bytes");
         k -= 8;
         n++;
         p--;            /* can always return one */
@@ -5247,7 +5234,7 @@ int r;
 }
 
 
-void inflate_codes_free(c, z)
+static void inflate_codes_free(c, z)
 inflate_codes_statef *c;
 z_streamp z;
 {
@@ -5272,16 +5259,8 @@ z_streamp z;
 struct inflate_codes_state {int dummy;}; /* for buggy compilers */
 #endif
 
-/* And'ing with mask[n] masks the lower n bits */
-uInt inflate_mask[17] = {
-    0x0000,
-    0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff,
-    0x01ff, 0x03ff, 0x07ff, 0x0fff, 0x1fff, 0x3fff, 0x7fff, 0xffff
-};
-
-
 /* copy as much as possible from the sliding window to the output area */
-int inflate_flush(s, z, r)
+static int inflate_flush(s, z, r)
 inflate_blocks_statef *s;
 z_streamp z;
 int r;
@@ -5378,7 +5357,7 @@ struct inflate_codes_state {int dummy;}; /* for buggy compilers */
    at least ten.  The ten bytes are six bytes for the longest length/
    distance pair plus four bytes for overloading the bit buffer. */
 
-int inflate_fast(bl, bd, tl, td, s, z)
+static int inflate_fast(bl, bd, tl, td, s, z)
 uInt bl, bd;
 inflate_huft *tl;
 inflate_huft *td; /* need separate declaration for Borland C++ */
@@ -5542,8 +5521,6 @@ z_streamp z;
  * For conditions of distribution and use, see copyright notice in zlib.h 
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 /* #include "zutil.h" */
 
 #ifndef NO_DUMMY_DECL
@@ -5554,19 +5531,6 @@ struct internal_state      {int dummy;}; /* for buggy compilers */
 extern void exit OF((int));
 #endif
 
-const char *z_errmsg[10] = {
-"need dictionary",     /* Z_NEED_DICT       2  */
-"stream end",          /* Z_STREAM_END      1  */
-"",                    /* Z_OK              0  */
-"file error",          /* Z_ERRNO         (-1) */
-"stream error",        /* Z_STREAM_ERROR  (-2) */
-"data error",          /* Z_DATA_ERROR    (-3) */
-"insufficient memory", /* Z_MEM_ERROR     (-4) */
-"buffer error",        /* Z_BUF_ERROR     (-5) */
-"incompatible version",/* Z_VERSION_ERROR (-6) */
-""};
-
-
 const char * ZEXPORT zlibVersion()
 {
     return ZLIB_VERSION;
@@ -5650,14 +5614,14 @@ void zmemzero(dest, len)
 #define MAX_PTR 10
 /* 10*64K = 640K */
 
-local int next_ptr = 0;
+static int next_ptr = 0;
 
 typedef struct ptr_table_s {
     voidpf org_ptr;
     voidpf new_ptr;
 } ptr_table;
 
-local ptr_table table[MAX_PTR];
+static ptr_table table[MAX_PTR];
 /* This table is used to remember the original form of pointers
  * to large buffers (64K). Such pointers are normalized with a zero offset.
  * Since MSDOS is not a preemptive multitasking OS, this table is not
@@ -5772,8 +5736,6 @@ void  zcfree (opaque, ptr)
  * For conditions of distribution and use, see copyright notice in zlib.h 
  */
 
-/* @(#) $Id: zlib.c,v 1.10.874.1 2005/06/24 01:47:11 lindak Exp $ */
-
 /* #include "zlib.h" */
 
 #define BASE 65521L /* largest prime smaller than 65536 */
@@ -5787,10 +5749,8 @@ void  zcfree (opaque, ptr)
 #define DO16(buf)   DO8(buf,0); DO8(buf,8);
 
 /* ========================================================================= */
-uLong ZEXPORT adler32(adler, buf, len)
-    uLong adler;
-    const Bytef *buf;
-    uInt len;
+uLong ZEXPORT
+adler32(uLong adler, const Bytef *buf, uInt len)
 {
     unsigned long s1 = adler & 0xffff;
     unsigned long s2 = (adler >> 16) & 0xffff;
diff --git a/libsa/Makefile b/libsa/Makefile
index 3e84f239e..8f7a3d4a4 100644
--- a/libsa/Makefile
+++ b/libsa/Makefile
@@ -10,11 +10,13 @@ include $(MakeInc_def)
 INSTINC_SUBDIRS = libsa
 INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS}
 INSTINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS}
+INSTINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS}
 
 
 EXPINC_SUBDIRS = libsa
 EXPINC_SUBDIRS_PPC = ${EXPINC_SUBDIRS}
 EXPINC_SUBDIRS_I386 = ${EXPINC_SUBDIRS}
+EXPINC_SUBDIRS_ARM = ${EXPINC_SUBDIRS}
 
 
 SETUP_SUBDIRS = conf
diff --git a/libsa/bootstrap.cpp b/libsa/bootstrap.cpp
index 8767a3a5c..7db64a172 100644
--- a/libsa/bootstrap.cpp
+++ b/libsa/bootstrap.cpp
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <IOKit/IOLib.h>
 #include <mach/kmod.h>
@@ -73,7 +79,9 @@ KLDBootstrap::KLDBootstrap() {
     add_from_mkext_function = &addExtensionsFromArchive;
     remove_startup_extension_function = &removeStartupExtension;
 
+#ifndef CONFIG_NOKLD
     kernelLinkerPresent = 1;
+#endif
 }
 
 /* The destructor frees all wired memory regions held
diff --git a/libsa/bsearch.c b/libsa/bsearch.c
index 2a0832467..03b81228d 100644
--- a/libsa/bsearch.c
+++ b/libsa/bsearch.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1990, 1993
@@ -75,7 +81,7 @@ static char sccsid[] = "@(#)bsearch.c	8.1 (Berkeley) 6/4/93";
  * look at item 3.
  */
 __private_extern__
-void * bsearch(
+const void * bsearch(
 	register const void *key,
 	const void *base0,
 	size_t nmemb,
@@ -91,9 +97,9 @@ void * bsearch(
 		p = base + (lim >> 1) * size;
 		cmp = (*compar)(key, p);
 		if (cmp == 0)
-			return ((void *)p);
+			return p;
 		if (cmp > 0) {	/* key > p: move right */
-			base = (char *)p + size;
+			base = (const char *)p + size;
 			lim--;
 		}		/* else move left */
 	}
diff --git a/libsa/c++rem3.c b/libsa/c++rem3.c
index dfea38871..44d76d683 100644
--- a/libsa/c++rem3.c
+++ b/libsa/c++rem3.c
@@ -339,7 +339,7 @@ static __inline__ void resetTo(ParseContext *c, CheckPoint *chk)
     c->fP = *chk;
 }
 
-static __inline__ const char *inCharFromCheck(ParseContext *c, CheckPoint *chk)
+static __inline__ const char *inCharFromCheck(CheckPoint *chk)
 {
     return chk->fInChar;
 }
@@ -519,7 +519,7 @@ static __inline__ Boolean isNext(ParseContext *c, char ch)
 }
 
 // Check the current input is ONE of the characters in str
-static Boolean charNext(ParseContext *c, char *str)
+static Boolean charNext(ParseContext *c, const char *str)
 {
     if (hasRemain(c, 1)) {
         char ch = peekNext(c);
@@ -542,7 +542,7 @@ static Boolean strNext(ParseContext *c, const char *str)
 
     do {
         if (!*str) {
-            c->fP.fInChar = (char *) cp;
+            c->fP.fInChar = (const char *) cp;
             return true;
         }
         else if (!*cp)
@@ -699,6 +699,7 @@ appendQualifiedClass(ParseContext *c, int entry)
 {
     BaseTypeData *iP, *oP, *sP, *endSP;
     const char *cp, *typeID;
+    char *cp_new;
     int sub, subEntry, prefixLen;
     int q_count;
 
@@ -714,8 +715,8 @@ appendQualifiedClass(ParseContext *c, int entry)
     for (q_count = 0; sP < endSP && (cp-typeID) < prefixLen; q_count++, sP++) {
         int count;
 
-        count = strtoul(cp, (char **) &cp, 10);
-        cp += count;
+        count = strtoul(cp, &cp_new, 10);
+	cp = cp_new + count;
 
         sP->fType = kNTClass;
         sP->fFundTypeID = typeID;
@@ -745,7 +746,7 @@ appendQualifiedClass(ParseContext *c, int entry)
 
         oP->fType = kNTSubstitute;	// Assume complete substitution
         oP->fLen = sub;
-        oP->fFundTypeID = 0;
+        oP->fFundTypeID = NULL;
 
         // We have a partial substitution so tag on the unmatched bit
         if (prefixLen != iP->fLen) {
@@ -792,7 +793,7 @@ appendType(ParseContext *c, int type)
     entry = tP->fStartEntry;
     numE  = tP->fNumEntries;
     lastEntry = entry + numE;
-    iP = 0;
+    iP = NULL;
     for (i = 0, found = false, sub = -1; i < numE; i++) {
         iP = &c->fInEntries[entry + i];
         switch (iP->fType) {
@@ -990,12 +991,14 @@ static Boolean parse_count(ParseContext *c, int *countP)
 {
     int count = 0;
     char ch;
+    char *newp;
 
     ch = peekNext(c);
     if (ch < '1' || ch > '9')
         return false;
 
-    count = strtol(c->fP.fInChar, (char **) &c->fP.fInChar, 10);
+    count = strtol(c->fP.fInChar, &newp, 10);
+    c->fP.fInChar = newp;
     if (countP)
         *countP = count;
 
@@ -1330,10 +1333,10 @@ static Boolean
 rotateFunction(ParseContext *c, int argStart, int retStart)
 {
     char returnTypeBuffer[MAX_RETURN_BUFFER];
-    int numArg, numRet;
-    int lenArg, lenRet;
+    unsigned int numArg, numRet;
+    unsigned int lenArg, lenRet;
     char *sArgP, *sRetP;
-    int i;
+    unsigned int i;
 
     TypeData *argTP = &c->fTypeList[argStart];
     TypeData *retTP = &c->fTypeList[retStart];
@@ -1393,7 +1396,7 @@ rotateFunction(ParseContext *c, int argStart, int retStart)
 // <function_type> ::= "F" <argument_types> "_" <type>
 static Boolean parse_function_type(ParseContext *c, Boolean forMethod)
 {
-    TypeData *bDictP = 0;
+    TypeData *bDictP = NULL;
     BaseTypeData *bP = c->fCurBaseP;
 
     int argTypeStart, retTypeStart;
@@ -1689,7 +1692,9 @@ abandonParse:
 static Boolean parse_declarators(ParseContext *c)
 {
     int count;
+    unsigned long l;
     BaseTypeData *dP;
+    char *newp;
 
     // Note we MUST go through the for loop at least once
     for (count = 0; ; count++) {
@@ -1728,8 +1733,9 @@ static Boolean parse_declarators(ParseContext *c)
             dP->fType = kNTArray;
 
             advance(c, 1); curDecl++;
-            curDecl = (void *)
-                strtoul(curDecl, (char **) &c->fP.fInChar, 10);
+            l = strtoul(curDecl, &newp, 10);
+            c->fP.fInChar = newp;
+            curDecl = (const char *)l;
             if (!curDecl)
                 goto abandonParse;
             dP->fFundTypeID = curDecl;
@@ -1873,7 +1879,7 @@ static Boolean parse_opinfo(ParseContext *c, const char **opInfoP)
     CheckPoint chk = *checkPoint(c);
     const char *op;
     char ch;
-    int i;
+    unsigned int i;
 
     if ('a' == (ch = peekNext(c))) {
         goto abandonParse;
@@ -2056,7 +2062,7 @@ static Boolean parse_mangled_name(ParseContext *c)
         if (atEnd(c))
             goto abandonParse;	// No Signature?
 
-        funcLen = inCharFromCheck(c, &dubBarChk) - func - 2;
+        funcLen = inCharFromCheck(&dubBarChk) - func - 2;
         if (parse_signature(c, func, funcLen, op))
             return true;
 
diff --git a/libsa/c++rem3.h b/libsa/c++rem3.h
index 9f37019d6..31842164a 100644
--- a/libsa/c++rem3.h
+++ b/libsa/c++rem3.h
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.0 (the 'License').  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License."
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * History:
diff --git a/libsa/catalogue.cpp b/libsa/catalogue.cpp
index abaf609dd..0aa46fce4 100644
--- a/libsa/catalogue.cpp
+++ b/libsa/catalogue.cpp
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libkern/c++/OSContainers.h>
 #include <IOKit/IODeviceTreeSupport.h>
@@ -53,7 +59,7 @@ extern int grade_binary(cpu_type_t exectype, cpu_subtype_t execsubtype);
 extern struct segment_command *getsegbyname(char *seg_name);
 // Return the address of the named section from the named Mach-O segment
 // from the currently executing 32 bit kernel, or NULL.
-extern struct section *getsectbyname(char *segname, char *sectname);
+extern struct section *getsectbyname(const char *segname, const char *sectname);
 };
 
 #define LOG_DELAY()
@@ -688,7 +694,7 @@ OSDictionary * readExtension(OSDictionary * propertyDict,
     OSString             * errorString = NULL;
     OSDictionary         * driverDict = NULL;
 
-    MemoryMapFileInfo * driverInfo = 0;
+    const MemoryMapFileInfo * driverInfo = 0;
     BootxDriverInfo * dataBuffer;
 
     kmod_info_t          * loaded_kmod = NULL;
@@ -715,10 +721,10 @@ OSDictionary * readExtension(OSDictionary * propertyDict,
         goto finish;
     }
 
-    driverInfo = (MemoryMapFileInfo *)
+    driverInfo = (const MemoryMapFileInfo *)
         bootxDriverDataObject->getBytesNoCopy(0,
         sizeof(MemoryMapFileInfo));
-#if defined (__ppc__)
+#if defined (__ppc__) || defined (__arm__)
     dataBuffer = (BootxDriverInfo *)ml_static_ptovirt(driverInfo->paddr);
 #elif defined (__i386__)
     dataBuffer = (BootxDriverInfo *)ml_boot_ptovirt(driverInfo->paddr);
@@ -925,7 +931,7 @@ finish:
 
 bool uncompressModule(OSData *compData, /* out */ OSData ** file) {
 
-    MkextEntryInfo *info = (MkextEntryInfo *) compData->getBytesNoCopy();
+    const MkextEntryInfo *info = (const MkextEntryInfo *) compData->getBytesNoCopy();
 
     return uncompressFile((u_int8_t *) info->base_address, 
 			  info->fileinfo, file);
@@ -936,7 +942,8 @@ bool uncompressModule(OSData *compData, /* out */ OSData ** file) {
 * Does the work of pulling extensions out of an mkext archive located
 * in memory.
 *********************************************************************/
-bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
+bool extractExtensionsFromArchive(const MemoryMapFileInfo * mkext_file_info,
+    bool vaddr,
     OSDictionary * extensions) {
 
     bool result = true;
@@ -959,14 +966,19 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
     OSData         * moduleInfo = 0;  // must release
     MkextEntryInfo   module_info;
 
-
-#if defined (__ppc__)
-    mkext_data = (mkext_header *)mkext_file_info->paddr;
+    if (vaddr) {
+	// addExtensionsFromArchive passes a kernel virtual address
+	mkext_data = (mkext_header *)mkext_file_info->paddr;
+    } else {
+#if defined (__ppc__) || defined (__arm__)
+	mkext_data = (mkext_header *)ml_static_ptovirt(mkext_file_info->paddr);
 #elif defined (__i386__)
-    mkext_data = (mkext_header *)ml_boot_ptovirt(mkext_file_info->paddr);
+	mkext_data = (mkext_header *)ml_boot_ptovirt(mkext_file_info->paddr);
 #else
 #error unsupported architecture
 #endif
+    }
+
     if (OSSwapBigToHostInt32(mkext_data->magic) != MKEXT_MAGIC ||
         OSSwapBigToHostInt32(mkext_data->signature) != MKEXT_SIGN) {
         IOLog("Error: Extension archive has invalid magic or signature.\n");
@@ -1096,7 +1108,7 @@ bool extractExtensionsFromArchive(MemoryMapFileInfo * mkext_file_info,
         } else {
             driverPlist = OSDynamicCast(OSDictionary,
                 OSUnserializeXML(
-                    (char *)driverPlistDataObject->getBytesNoCopy(),
+                    (const char *)driverPlistDataObject->getBytesNoCopy(),
                     &errorString));
             if (!driverPlist) {
                 IOLog("Error: Couldn't read XML property list "
@@ -1249,7 +1261,7 @@ bool readExtensions(OSDictionary * propertyDict,
 
     bool result = true;
     OSData * mkextDataObject = 0;      // don't release
-    MemoryMapFileInfo * mkext_file_info = 0; // don't free
+    const MemoryMapFileInfo * mkext_file_info = 0; // don't free
 
     mkextDataObject = OSDynamicCast(OSData,
         propertyDict->getObject(memory_map_name));
@@ -1264,13 +1276,13 @@ bool readExtensions(OSDictionary * propertyDict,
         goto finish;
     }
 
-    mkext_file_info = (MemoryMapFileInfo *)mkextDataObject->getBytesNoCopy();
+    mkext_file_info = (const MemoryMapFileInfo *)mkextDataObject->getBytesNoCopy();
     if (!mkext_file_info) {
         result = false;
         goto finish;
     }
 
-    result = extractExtensionsFromArchive(mkext_file_info, extensions);
+    result = extractExtensionsFromArchive(mkext_file_info, false /*physical*/, extensions);
 
 finish:
 
@@ -1415,7 +1427,7 @@ bool addExtensionsFromArchive(OSData * mkextDataObject) {
     */
     bootLoaderObjects->setObject(mkextDataObject);
 
-    result = extractExtensionsFromArchive(&mkext_file_info, extensions);
+    result = extractExtensionsFromArchive(&mkext_file_info, true /*virtual*/, extensions);
     if (!result) {
         IOLog("Error: Failed to extract extensions from archive.\n");
         LOG_DELAY();
@@ -1566,8 +1578,8 @@ bool recordStartupExtensions(void) {
 		    LOG_DELAY();
 		    continue;
 		}
-		UInt32 * prelink;
-		prelink = (UInt32 *) data->getBytesNoCopy();
+		const UInt32 * prelink;
+		prelink = (const UInt32 *) data->getBytesNoCopy();
 		kmod_info_t * kmod_info = (kmod_info_t *) OSReadBigInt32(prelink, 0);
 		// end of "file" is end of symbol sect
 		data = OSData::withBytesNoCopy((void *) kmod_info->address,
@@ -1999,7 +2011,7 @@ memory_map:
                   strlen(BOOTX_MULTIKEXT_PREFIX)) ) {
 
             OSData            * bootxDriverDataObject = NULL;
-            MemoryMapFileInfo * driverInfo = 0;
+            const MemoryMapFileInfo * driverInfo = 0;
 
             bootxDriverDataObject = OSDynamicCast(OSData,
                 propertyDict->getObject(keyValue));
@@ -2008,7 +2020,7 @@ memory_map:
             if (!bootxDriverDataObject) {
                 continue;
             }
-            driverInfo = (MemoryMapFileInfo *)
+            driverInfo = (const MemoryMapFileInfo *)
                 bootxDriverDataObject->getBytesNoCopy(0,
                 sizeof(MemoryMapFileInfo));
             IODTFreeLoaderInfo((char *)keyValue,
diff --git a/libsa/conf/MASTER b/libsa/conf/MASTER
index 09d80d909..a8ce6d996 100644
--- a/libsa/conf/MASTER
+++ b/libsa/conf/MASTER
@@ -53,3 +53,19 @@ ident		LIBSA
 
 options		KDEBUG		# kernel tracing		# <kdebug>
 options		GPROF		# kernel profiling		# <profile>
+
+options		CONFIG_NOLIBKLD	# kernel linker			# <no_kld>
+makeoptions	LIBKLD_PATH = "/usr/local/lib/libkld.a"		# <!arm>
+makeoptions	LIBKLD = " "					# <no_kld>
+makeoptions	LIBKLD = "${LIBKLD_PATH}"			# <!no_kld>
+
+options		MALLOC_RESET_GC					# <smaller_kld>
+options		MALLOC_KLD_VM_ALLOCATE				# <smaller_kld>
+# Use mach_vm_* calls for libsa kernel code, since we redefine vm_* for libkld
+options		vm_allocate = mach_vm_allocate			# <smaller_kld>
+options		vm_deallocate = mach_vm_deallocate		# <smaller_kld>
+options		CONFIG_DTRACE					# <config_dtrace>
+
+options		CONFIG_NO_PANIC_STRINGS				# <no_panic_str>
+options		CONFIG_NO_PRINTF_STRINGS			# <no_printf_str>
+options		CONFIG_NO_KPRINTF_STRINGS			# <no_kprintf_str>
diff --git a/libsa/conf/MASTER.i386 b/libsa/conf/MASTER.i386
index b61a6e538..383d26f5a 100644
--- a/libsa/conf/MASTER.i386
+++ b/libsa/conf/MASTER.i386
@@ -1,11 +1,13 @@
 ######################################################################
 #
-#  RELEASE = [intel mach libkerncpp]
-#  PROFILE = [RELEASE profile]
-#  DEBUG = [intel mach libkerncpp debug]
+#  RELEASE	= [ intel mach libkerncpp config_dtrace ]
+#  PROFILE	= [ RELEASE profile ]
+#  DEBUG	= [ RELEASE debug ]
+#
+#  EMBEDDED	= [ intel mach libkerncpp ]
+#  DEVELOPMENT	= [ EMBEDDED config_dtrace ]
 #
 ######################################################################
 
 machine		"i386"					# <intel>
 cpu		"i386"					# <intel>
-
diff --git a/libsa/conf/MASTER.ppc b/libsa/conf/MASTER.ppc
index 5c6b53d20..9995642ed 100644
--- a/libsa/conf/MASTER.ppc
+++ b/libsa/conf/MASTER.ppc
@@ -4,7 +4,8 @@
 #  Standard Apple MacOS X Configurations:
 #  -------- ---- -------- ---------------
 #
-#  RELEASE = [ppc mach libkerncpp]
+#  RELEASE		= [ppc mach libkerncpp config_dtrace]
+#  DEVELOPMENT	= [ RELEASE ]
 #  PROFILE = [RELEASE profile]
 #  DEBUG = [ppc mach libkerncpp debug]
 #  RELEASE_TRACE = [ RELEASE kdebug ]
diff --git a/libsa/conf/Makefile b/libsa/conf/Makefile
index fc29125a6..c7a74f71e 100644
--- a/libsa/conf/Makefile
+++ b/libsa/conf/Makefile
@@ -18,10 +18,14 @@ ifndef LIBSA_KERNEL_CONFIG
 export LIBSA_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(LIBSA_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -29,33 +33,30 @@ $(COMPOBJROOT)/$(LIBSA_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(LIBSA_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(LIBSA_KERNEL_CONFIG) $(LIBSA_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(LIBSA_KERNEL_CONFIG) $(LIBSA_KERNEL_CONFIG); \
 	);
 
 .ORDER: $(COMPOBJROOT)/$(LIBSA_KERNEL_CONFIG)/Makefile
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(LIBSA_KERNEL_CONFIG)/Makefile
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(LIBSA_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(LIBSA_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(LIBSA_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(LIBSA_KERNEL_CONFIG)	\
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(LIBSA_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all: do_all 
 
diff --git a/libsa/conf/Makefile.template b/libsa/conf/Makefile.template
index 0419939c7..9d966e91a 100644
--- a/libsa/conf/Makefile.template
+++ b/libsa/conf/Makefile.template
@@ -27,8 +27,8 @@ include $(MakeInc_def)
 #
 # XXX: CFLAGS
 #
-CFLAGS+= -DKERNEL -DLIBSA_KERNEL_PRIVATE \
-         -Wall -Wno-four-char-constants -fno-common
+CFLAGS+= -imacros meta_features.h -DKERNEL -DLIBSA_KERNEL_PRIVATE \
+         -Wall -Wno-four-char-constants -fno-common  $(CFLAGS_INLINE_CONFIG)
 
 SFLAGS+= -DKERNEL
 
@@ -77,16 +77,15 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS)
-	@echo "creating $(COMPONENT).o"
-	@echo [ updating $(COMPONENT).o ${LIBSA_KERNEL_CONFIG} ]
-	$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} /usr/local/lib/libkld.a
-	$(SEG_HACK) __KLD $(COMPONENT).o -o $(COMPONENT)_kld.o
-	mv $(COMPONENT)_kld.o $(COMPONENT).o
+	@echo LD $(COMPONENT)
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS} $(LIBKLD)
+	$(_v)$(SEG_HACK) __KLD $(COMPONENT).o -o $(COMPONENT)_kld.o
+	$(_v)$(LD) $(LDFLAGS_COMPONENT) $(COMPONENT)_kld.o -o $(COMPONENT).o
 
 do_all: $(COMPONENT).o
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`
+	$(_v)${MD} -u Makedep -f -d `ls *.d`
 	
 do_build_all: do_depend 
 
diff --git a/libsa/conf/files b/libsa/conf/files
index a6c36b9ce..9aa04c95b 100644
--- a/libsa/conf/files
+++ b/libsa/conf/files
@@ -3,6 +3,7 @@
 # OPTIONS/libkerncpp                                          optional libkerncpp
 # OPTIONS/kdebug                                              optional kdebug
 # OPTIONS/gprof                                               optional gprof
+OPTIONS/config_dtrace										optional config_dtrace
 
 # libsa
 
diff --git a/libsa/conf/tools/doconf/Makefile b/libsa/conf/tools/doconf/Makefile
index 2bf0b7a10..aa55a9419 100644
--- a/libsa/conf/tools/doconf/Makefile
+++ b/libsa/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,25 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/libsa/conf/tools/doconf/doconf.csh b/libsa/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/libsa/conf/tools/doconf/doconf.csh
+++ b/libsa/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/libsa/dgraph.c b/libsa/dgraph.c
index 199a9138e..03ecb3889 100644
--- a/libsa/dgraph.c
+++ b/libsa/dgraph.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifdef KERNEL
@@ -37,27 +43,18 @@
 #include "dgraph.h"
 #include "load.h"
 
+#ifdef KERNEL
+#include <sys/malloc.h>
+#endif
 
 static void __dgraph_entry_free(dgraph_entry_t * entry);
 
 #ifdef KERNEL
-/*******************************************************************************
-*
-*******************************************************************************/
-char * strdup(const char * string)
-{
-    char * dup = 0;
-    unsigned int length;
-
-    length = strlen(string);
-    dup = (char *)malloc((1+length) * sizeof(char));
-    if (!dup) {
-        return NULL;
-    }
-    strcpy(dup, string);
-    return dup;
-}
-
+#define dgstrdup(string) STRDUP(string, M_TEMP)
+#define dgfree(string) FREE(string, M_TEMP)
+#else
+#define dgstrdup strdup
+#define dgfree(string) free(string)
 #endif /* KERNEL */
 
 /*******************************************************************************
@@ -181,15 +178,15 @@ dgraph_error_t dgraph_init_with_arglist(
 static void __dgraph_entry_free(dgraph_entry_t * entry)
 {
     if (entry->name) {
-        free(entry->name);
+        dgfree(entry->name);
         entry->name = NULL;
     }
     if (entry->expected_kmod_name) {
-        free(entry->expected_kmod_name);
+        dgfree(entry->expected_kmod_name);
         entry->expected_kmod_name = NULL;
     }
     if (entry->expected_kmod_vers) {
-        free(entry->expected_kmod_vers);
+        dgfree(entry->expected_kmod_vers);
         entry->expected_kmod_vers = NULL;
     }
     if (entry->dependencies) {
@@ -198,7 +195,7 @@ static void __dgraph_entry_free(dgraph_entry_t * entry)
     }
     if (entry->symbols_malloc) {
         free((void *) entry->symbols_malloc);
-        entry->symbols_malloc = NULL;
+        entry->symbols_malloc = 0;
     }
     free(entry);
     return;
@@ -318,7 +315,7 @@ dgraph_entry_t ** fill_backward_load_order(
     dgraph_entry_t * first_entry,
     unsigned int * last_index /* out param */)
 {
-    int i;
+    unsigned int i;
     unsigned int scan_index = 0;
     unsigned int add_index = 0;
     dgraph_entry_t * scan_entry;
@@ -549,6 +546,10 @@ dgraph_entry_t * dgraph_add_dependent(
     void * object,
     size_t object_length,
     bool   object_is_kmem,
+#if CONFIG_MACF_KEXT
+    kmod_args_t user_data,
+    mach_msg_type_number_t user_data_length,
+#endif
 #endif /* KERNEL */
     const char * expected_kmod_name,
     const char * expected_kmod_vers,
@@ -622,12 +623,12 @@ dgraph_entry_t * dgraph_add_dependent(
         goto finish;
     }
     bzero(new_entry, sizeof(dgraph_entry_t));
-    new_entry->expected_kmod_name = strdup(expected_kmod_name);
+    new_entry->expected_kmod_name = dgstrdup(expected_kmod_name);
     if (!new_entry->expected_kmod_name) {
         error = 1;
         goto finish;
     }
-    new_entry->expected_kmod_vers = strdup(expected_kmod_vers);
+    new_entry->expected_kmod_vers = dgstrdup(expected_kmod_vers);
     if (!new_entry->expected_kmod_vers) {
         error = 1;
         goto finish;
@@ -657,8 +658,12 @@ dgraph_entry_t * dgraph_add_dependent(
     new_entry->object = object;
     new_entry->object_length = object_length;
     new_entry->object_is_kmem = object_is_kmem;
+#if CONFIG_MACF_KEXT
+    new_entry->user_data = user_data;
+    new_entry->user_data_length = user_data_length;
+#endif
 #endif /* KERNEL */
-    new_entry->name = strdup(name);
+    new_entry->name = dgstrdup(name);
     if (!new_entry->name) {
         error = 1;
         goto finish;
@@ -705,6 +710,10 @@ dgraph_entry_t * dgraph_add_dependency(
     void * object,
     size_t object_length,
     bool   object_is_kmem,
+#if CONFIG_MACF_KEXT
+    kmod_args_t user_data,
+    mach_msg_type_number_t user_data_length,
+#endif
 #endif /* KERNEL */
     const char * expected_kmod_name,
     const char * expected_kmod_vers,
@@ -742,6 +751,9 @@ dgraph_entry_t * dgraph_add_dependency(
     dependency = dgraph_add_dependent(dgraph, name,
 #ifdef KERNEL
          object, object_length, object_is_kmem,
+#if CONFIG_MACF_KEXT
+         user_data, user_data_length,
+#endif
 #endif /* KERNEL */
          expected_kmod_name, expected_kmod_vers, load_address,
          is_kernel_component);
diff --git a/libsa/dgraph.h b/libsa/dgraph.h
index 47e124098..9d7340e2e 100644
--- a/libsa/dgraph.h
+++ b/libsa/dgraph.h
@@ -1,3 +1,36 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 #ifndef __DGRAPH_H__
 #define __DGRAPH_H__
 
@@ -69,6 +102,12 @@ typedef struct dgraph_entry_t {
     int need_cleanup;  // true if load failed with kernel memory allocated
     kmod_t kmod_id;    // the id assigned by the kernel to a loaded kmod
 
+#if CONFIG_MACF_KEXT
+    // module-specific data from the plist
+    kmod_args_t user_data;
+    mach_msg_type_number_t user_data_length;
+#endif
+
 } dgraph_entry_t;
 
 typedef struct {
@@ -146,6 +185,10 @@ dgraph_entry_t * dgraph_add_dependent(
     void * object,
     size_t object_length,
     bool   object_is_kmem,
+#if CONFIG_MACF_KEXT
+    kmod_args_t user_data,
+    mach_msg_type_number_t user_data_length,
+#endif
 #endif /* KERNEL */
     const char * expected_kmod_name,
     const char * expected_kmod_vers,
@@ -160,12 +203,22 @@ dgraph_entry_t * dgraph_add_dependency(
     void * object,
     size_t object_length,
     bool   object_is_kmem,
+#if CONFIG_MACF_KEXT
+    kmod_args_t user_data,
+    mach_msg_type_number_t user_data_length,
+#endif 
 #endif /* KERNEL */
     const char * expected_kmod_name,
     const char * expected_kmod_vers,
     vm_address_t load_address,
     char is_kernel_component);
 
+dgraph_entry_t ** fill_backward_load_order(
+    dgraph_entry_t ** backward_load_order,
+    unsigned int * list_length,
+    dgraph_entry_t * first_entry,
+    unsigned int * last_index /* out param */);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/libsa/i386/setjmp.s b/libsa/i386/setjmp.s
index 9bbd90f4a..cb52668e3 100644
--- a/libsa/i386/setjmp.s
+++ b/libsa/i386/setjmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/libsa/kext.cpp b/libsa/kext.cpp
index e9c8f0954..48ef63d16 100644
--- a/libsa/kext.cpp
+++ b/libsa/kext.cpp
@@ -1,24 +1,38 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
+
 #include <libkern/c++/OSContainers.h>
 #include <IOKit/IOCatalogue.h>
 #include <IOKit/IOLib.h>
@@ -60,10 +74,14 @@ kmod_start_or_stop(
 extern kern_return_t kmod_retain(kmod_t id);
 extern kern_return_t kmod_release(kmod_t id);
 
-extern void flush_dcache(vm_offset_t addr, unsigned cnt, int phys);
-extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
+extern Boolean kmod_load_request(const char * moduleName, Boolean make_request);
 };
 
+extern kmod_args_t
+get_module_data(OSDictionary * kextPlist, mach_msg_type_number_t * datalen);
+
+extern struct mac_module_data *osdict_encode(OSDictionary *dict);
+
 #define DEBUG
 #ifdef DEBUG
 #define LOG_DELAY(x)    IODelay((x) * 1000000)
@@ -75,6 +93,11 @@ extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 #define VTRESET
 #endif /* DEBUG */
 
+
+#define KERNEL_PREFIX  "com.apple.kernel"
+#define KPI_PREFIX     "com.apple.kpi"
+
+
 /*********************************************************************
 *
 *********************************************************************/
@@ -144,7 +167,7 @@ bool getKext(
         *caller_owns_code = false;
     
         *code = (unsigned char *)kld_file_getaddr(bundleid,
-            (long *)&code_size_local);
+            (unsigned long *)&code_size_local);
         if (*code) {
             if (code_size) {
                 *code_size = code_size_local;
@@ -266,7 +289,7 @@ bool kextIsDependency(const char * kext_name, char * is_kernel) {
     OSData * compressedCode = 0;          // don't release
 
     if (is_kernel) {
-        *is_kernel = false;
+        *is_kernel = 0;
     }
 
    /* Get the dictionary of startup extensions.
@@ -307,7 +330,7 @@ bool kextIsDependency(const char * kext_name, char * is_kernel) {
         extPlist->getObject("OSKernelResource"));
     if (isKernelResourceObj && isKernelResourceObj->isTrue()) {
         if (is_kernel) {
-            *is_kernel = true;
+            *is_kernel = 1;
         }
     }
 
@@ -315,6 +338,8 @@ bool kextIsDependency(const char * kext_name, char * is_kernel) {
     compressedCode = OSDynamicCast(OSData,
         extDict->getObject("compressedCode"));
 
+   /* A kernel component that has code represents a KPI.
+    */
     if ((driverCode || compressedCode) && is_kernel && *is_kernel) {
         *is_kernel = 2;
     }
@@ -339,6 +364,8 @@ addDependenciesForKext(OSDictionary * kextPlist,
 {
     bool result = true;
     bool hasDirectKernelDependency = false;
+    bool hasKernelStyleDependency = false;
+    bool hasKPIStyleDependency = false;
     OSString * kextName = 0;  // don't release
     OSDictionary * libraries = 0;  // don't release
     OSCollectionIterator * keyIterator = 0; // must release
@@ -388,7 +415,7 @@ addDependenciesForKext(OSDictionary * kextPlist,
             if (!kextIsDependency(libraryName->getCStringNoCopy(),
                 &is_kernel_component)) {
 
-                is_kernel_component = false;
+                is_kernel_component = 0;
             }
 
             if (!skipKernelDependencies || !is_kernel_component) {
@@ -398,8 +425,25 @@ addDependenciesForKext(OSDictionary * kextPlist,
             if (!hasDirectKernelDependency && is_kernel_component) {
                 hasDirectKernelDependency = true;
             }
+
+           /* We already know from the kextIsDependency() call whether
+            * the dependency *itself* is kernel- or KPI-style, but since
+            * the declaration semantic is by bundle ID, we check that here
+            * instead.
+            */
+            if (strncmp(libraryName->getCStringNoCopy(),
+                KERNEL_PREFIX, strlen(KERNEL_PREFIX)) == 0) {
+
+                hasKernelStyleDependency = true;
+
+            } else if (strncmp(libraryName->getCStringNoCopy(),
+                KPI_PREFIX, strlen(KPI_PREFIX)) == 0) {
+
+                hasKPIStyleDependency = true;
+            }
         }
     }
+
     if (!hasDirectKernelDependency) {
         const OSSymbol * kernelName = 0;
 
@@ -415,7 +459,12 @@ addDependenciesForKext(OSDictionary * kextPlist,
         dependencyList->setObject(kernelName);
         kernelName->release();
 
-        IOLog("Extension \"%s\" has no kernel dependency.\n",
+        IOLog("Extension \"%s\" has no explicit kernel dependency; using version 6.0.\n",
+            kextName->getCStringNoCopy());
+
+    } else if (hasKernelStyleDependency && hasKPIStyleDependency) {
+        IOLog("Extension \"%s\" has immediate dependencies "
+            "on both com.apple.kernel and com.apple.kpi components; use only one style.\n",
             kextName->getCStringNoCopy());
     }
 
@@ -468,11 +517,16 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
     unsigned long code_length = 0;
     bool code_is_kmem = false;
     char * kmod_vers = 0; // from plist, don't free
-    char is_kernel_component = false;
+    char is_kernel_component = 0;
     dgraph_entry_t * dgraph_entry = 0; // don't free
     dgraph_entry_t * dgraph_dependency = 0; // don't free
     bool kext_is_dependency = true;
 
+#if CONFIG_MACF_KEXT
+    kmod_args_t user_data = 0;
+    mach_msg_type_number_t user_data_length;
+#endif
+
    /*****
     * Set up the root kmod.
     */
@@ -495,8 +549,16 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
         goto finish;
     }
 
+#if CONFIG_MACF_KEXT
+    // check kext for module data in the plist
+    user_data = get_module_data(kextPlist, &user_data_length);
+#endif
+
     dgraph_entry = dgraph_add_dependent(dgraph, kmod_name,
         code, code_length, code_is_kmem,
+#if CONFIG_MACF_KEXT
+        user_data, user_data_length,
+#endif
         kmod_name, kmod_vers,
         0 /* load_address not yet known */, is_kernel_component);
     if (!dgraph_entry) {
@@ -616,8 +678,16 @@ bool add_dependencies_for_kmod(const char * kmod_name, dgraph_t * dgraph)
                     goto finish;
                 }
 
+#if CONFIG_MACF_KEXT
+                // check kext for module data in the plist
+                // XXX - is this really needed?
+                user_data = get_module_data(kextPlist, &user_data_length);
+#endif
                 dgraph_dependency = dgraph_add_dependency(dgraph, dgraph_entry,
                     library_name, code, code_length, code_is_kmem,
+#if CONFIG_MACF_KEXT
+                    user_data, user_data_length,
+#endif
                     library_name, kmod_vers,
                     0 /* load_address not yet known */, is_kernel_component);
 
@@ -662,6 +732,12 @@ finish:
     }
     if (dependencyList)  dependencyList->release();
 
+#if CONFIG_MACF_KEXT
+    if (user_data && !result) {
+        vm_map_copy_discard((vm_map_copy_t)user_data);
+    }
+#endif
+
     return result;
 }
 
@@ -743,3 +819,71 @@ finish:
     }
     return result;
 }
+
+#define COM_APPLE  "com.apple."
+
+__private_extern__ void
+load_security_extensions (void)
+{
+    OSDictionary        * extensionsDict = NULL;  // don't release
+    OSCollectionIterator* keyIterator = NULL;     // must release
+    OSString            * key = NULL;             // don't release
+    OSDictionary        * extDict;                // don't release
+    OSDictionary        * extPlist;               // don't release
+    OSBoolean           * isSec = 0;              // don't release
+    Boolean             ret;
+
+    extensionsDict = getStartupExtensions();
+    if (!extensionsDict) {
+        IOLog("startup extensions dictionary is missing\n");
+        LOG_DELAY(1);
+        return;
+    }
+
+    keyIterator = OSCollectionIterator::withCollection(extensionsDict);
+    if (!keyIterator) {
+        IOLog("Error: Failed to allocate iterator for extensions.\n");
+        LOG_DELAY(1);
+        return;
+    }
+
+    while ((key = OSDynamicCast(OSString, keyIterator->getNextObject()))) {
+
+        const char * bundle_id = key->getCStringNoCopy();
+        
+       /* Skip extensions whose bundle IDs don't start with "com.apple.".
+        */
+        if (!bundle_id || (strncmp(bundle_id, COM_APPLE, strlen(COM_APPLE)) != 0)) {
+            continue;
+        }
+
+        extDict = OSDynamicCast(OSDictionary, extensionsDict->getObject(key));
+        if (!extDict) {
+            IOLog("extension \"%s\" cannot be found\n",
+                  key->getCStringNoCopy());
+            continue;
+        }
+
+        extPlist = OSDynamicCast(OSDictionary, extDict->getObject("plist"));
+        if (!extPlist) {
+            IOLog("extension \"%s\" has no info dictionary\n",
+                  key->getCStringNoCopy());
+            continue;
+        }
+
+        isSec = OSDynamicCast(OSBoolean,
+                             extPlist->getObject("AppleSecurityExtension"));
+        if (isSec && isSec->isTrue()) {
+            printf("Loading security extension %s\n", key->getCStringNoCopy());
+            ret = kmod_load_request(key->getCStringNoCopy(), false);
+            if (!ret) {
+                load_kernel_extension((char *)key->getCStringNoCopy());
+            }
+        }
+    }
+
+    if (keyIterator)
+        keyIterator->release();
+
+    return;
+}
diff --git a/libsa/kld_patch.c b/libsa/kld_patch.c
index 18892a019..811d0259e 100644
--- a/libsa/kld_patch.c
+++ b/libsa/kld_patch.c
@@ -1,25 +1,29 @@
 /*
- * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2001-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.0 (the 'License').  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License."
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * History:
@@ -36,6 +40,16 @@
 #include <libkern/OSByteOrder.h>
 #endif
 
+#ifdef CONFIG_NOLIBKLD
+int kld_address_func = 0;
+int kld_forget_symbol = 0;
+int kld_load_basefile_from_memory = 0;
+int kld_load_from_memory = 0;
+int kld_lookup = 0;
+int kld_set_link_options = 0;
+int kld_unload_all = 0;
+#endif
+
 #if KERNEL
 
 #include <stdarg.h>
@@ -58,7 +72,6 @@ enum { false = 0, true = 1 };
 
 extern void kld_error_vprintf(const char *format, va_list ap);
 
-__private_extern__ char *strstr(const char *in, const char *str);
 extern struct mach_header _mh_execute_header;
 extern struct segment_command *getsegbyname(char *seg_name);	// 32 bit only
 
@@ -163,7 +176,7 @@ extern struct segment_command *getsegbyname(char *seg_name);	// 32 bit only
 
 typedef struct Data {
     unsigned long fLength, fCapacity;
-    unsigned char *fData;
+    char *fData;
 } Data, *DataRef;
 
 struct sectionRecord {
@@ -187,7 +200,7 @@ struct patchRecord {
 
 struct relocRecord {
     void *fValue;
-    const struct nlist *fSymbol;
+    struct nlist *fSymbol;
     struct relocation_info *fRInfo;
     void *reserved;
 };
@@ -211,10 +224,10 @@ struct fileRecord {
     struct sectionRecord *fSections;
     vm_offset_t fVMAddr, fVMEnd;
     struct segment_command *fLinkEditSeg;
-    const char **fSymbToStringTable;
+    char **fSymbToStringTable;
     char *fStringBase;
     struct nlist *fSymbolBase;
-    const struct nlist *fLocalSyms;
+    struct nlist *fLocalSyms;
     unsigned int fNSects;
     int fNLocal;
     Boolean fIsKernel, fIsReloc, fIsIncrLink, fNoKernelExecutable, fIsKmem;
@@ -224,7 +237,7 @@ struct fileRecord {
 #if !KERNEL
     Boolean fSwapped;
 #endif
-    const char fPath[1];
+    char fPath[1];
 };
 
 static DataRef sFilesTable;
@@ -255,12 +268,12 @@ static __inline__ unsigned long DataGetLength(DataRef data)
     return data->fLength;
 }
 
-static __inline__ unsigned char *DataGetPtr(DataRef data)
+static __inline__ char *DataGetPtr(DataRef data)
 {
     return data->fData;
 }
 
-static __inline__ unsigned char *DataGetEndPtr(DataRef data)
+static __inline__ char *DataGetEndPtr(DataRef data)
 {
     return data->fData + data->fLength;
 }
@@ -285,11 +298,11 @@ static Boolean DataEnsureCapacity(DataRef data, unsigned long capacity)
 {
     // Don't bother to ever shrink a data object.
     if (capacity > data->fCapacity) {
-	unsigned char *newData;
+	char *newData;
 
 	capacity += kDataCapacityIncrement - 1;
 	capacity &= ~(kDataCapacityIncrement - 1);
-	newData = (unsigned char *) realloc(data->fData, capacity);
+	newData = (char *) realloc(data->fData, capacity);
 	if (!newData)
 	    return false;
 
@@ -345,7 +358,7 @@ static DataRef DataCreate(unsigned long capacity)
 	    data->fCapacity &= ~(kDataCapacityIncrement - 1);
 	}
 
-	data->fData = (unsigned char *) malloc(data->fCapacity);
+	data->fData = (char *) malloc(data->fCapacity);
 	if (!data->fData) {
 	    free(data);
 	    return NULL;
@@ -367,13 +380,13 @@ static void DataRelease(DataRef data)
     }
 }
 
-static __inline__ const char *
+static __inline__ char *
 symNameByIndex(const struct fileRecord *file, unsigned int symInd)
 {
     return file->fSymbToStringTable[symInd];
 }
 
-static __inline__  const char *
+static __inline__  char *
 symbolname(const struct fileRecord *file, const struct nlist *sym)
 {
     unsigned int index;
@@ -387,7 +400,7 @@ symbolname(const struct fileRecord *file, const struct nlist *sym)
         return symNameByIndex(file,  index);
 
     if (-1 == sym->n_un.n_strx)
-        return (const char *) sym->n_value;
+        return (char *) sym->n_value;
 
     // If the preceding tests fail then we have a getNewSymbol patch and
     // the file it refers to has already been patched as the n_strx is set
@@ -438,7 +451,7 @@ addFile(struct fileRecord *file, const char *path)
     }
 
     bcopy(file, newFile, sizeof(struct fileRecord) - 1);
-    strcpy((char *) newFile->fPath, path);
+    strlcpy((char *) newFile->fPath, path, strlen(path) + 1);
 
     return newFile;
 }
@@ -494,16 +507,66 @@ static void unmapFile(struct fileRecord *file)
 
 static void removeFile(struct fileRecord *file)
 {
+    int i, count;
+
     if (file->fClassList) {
-	DataRelease(file->fClassList);
+        struct metaClassRecord ** fileClasses =
+            (struct metaClassRecord **)DataGetPtr(file->fClassList);
+        
+        count = DataGetLength(file->fClassList) / sizeof(struct metaClassRecord *);
+
+        for (i = 0; i < count; i++) {
+            struct metaClassRecord * thisClass = fileClasses[i];
+
+            if (thisClass->fSuperName) {
+                free(thisClass->fSuperName);
+            }
+            if (thisClass->fPatchedVTable) {
+                free(thisClass->fPatchedVTable);
+            }
+
+            free(thisClass);
+        }
+        
+ 	DataRelease(file->fClassList);
 	file->fClassList = 0;
     }
 
+    // unmapFile() releases file->fSectData
+
+    if (file->fNewSymbols) {
+        struct nlist ** syms =
+            (struct nlist **)DataGetPtr(file->fNewSymbols);
+
+        count = DataGetLength(file->fNewSymbols) / sizeof(struct nlist *);
+
+        for (i = 0; i < count; i++) {
+            free(syms[i]);
+        }
+        DataRelease(file->fNewSymbols);
+        file->fNewSymbols = 0;
+    }
+
+    if (file->fNewStringBlocks) {
+        DataRef * stringBlocks = (DataRef *)DataGetPtr(file->fNewStringBlocks);
+        count = DataGetLength(file->fNewStringBlocks) / sizeof(DataRef);
+
+        for (i = 0; i < count; i++) {
+            DataRelease(stringBlocks[i]);
+        }
+
+        DataRelease(file->fNewStringBlocks);
+        file->fNewStringBlocks = 0;
+    }
+
+    // unmapFile() releases file->fSym2Strings
+
     unmapFile(file);
 
     free(file);
 }
 
+
 #if !KERNEL
 static Boolean
 mapObjectFile(struct fileRecord *file, const char *pathName)
@@ -798,7 +861,7 @@ static Boolean findBestArch(struct fileRecord *file, const char *pathName)
 #endif /* KERNEL */
 
     return_if(magic != MH_MAGIC,
-	false, ("%s isn't a valid mach-o\n", pathName));
+	false, ("%s isn't a valid mach-o (magic is %08x)\n", pathName, magic));
 
     return true;
 }
@@ -913,10 +976,10 @@ tryRemangleAgain:
 // mach-o files
 static Boolean parseSymtab(struct fileRecord *file, const char *pathName)
 {
-    const struct nlist *sym;
+    struct nlist *sym;
     unsigned int i, firstlocal, nsyms;
     unsigned long strsize;
-    const char *strbase;
+    char *strbase;
     Boolean foundOSObject, found295CPP, havelocal;
 
     // we found a link edit segment so recompute the bases
@@ -953,7 +1016,7 @@ static Boolean parseSymtab(struct fileRecord *file, const char *pathName)
     DataSetLength(file->fSym2Strings, nsyms * sizeof(const char *));
     return_if(!file->fSym2Strings, false, 
 	    ("Unable to allocate memory - symbol string trans\n", pathName));
-    file->fSymbToStringTable = (const char **) DataGetPtr(file->fSym2Strings);
+    file->fSymbToStringTable = (char **) DataGetPtr(file->fSym2Strings);
 
     // Search for the first non-stab symbol in table
     strsize = file->fSymtab->strsize;
@@ -963,7 +1026,7 @@ static Boolean parseSymtab(struct fileRecord *file, const char *pathName)
     found295CPP = foundOSObject = false;
     for (i = 0, sym = file->fSymbolBase; i < nsyms; i++, sym++) {
         long strx = sym->n_un.n_strx;
-        const char *symname = strbase + strx;
+        char *symname = strbase + strx;
         unsigned char n_type;
 
         return_if(((unsigned long) strx > strsize), false,
@@ -981,7 +1044,7 @@ static Boolean parseSymtab(struct fileRecord *file, const char *pathName)
 	if (file->fIsIncrLink && !file->fNSects)
 	{
 	    // symbol set
-	    struct nlist *patchsym = (struct nlist *) sym;
+	    struct nlist *patchsym = sym;
 	    const char * lookname;
 	    const struct nlist * realsym;
 
@@ -1109,11 +1172,11 @@ static Boolean parseSymtab(struct fileRecord *file, const char *pathName)
 
 // @@@ gvdl:  These functions need to be hashed they are
 // going to be way too slow for production code.
-static const struct nlist *
+static struct nlist *
 findSymbolByAddress(const struct fileRecord *file, void *entry)
 {
     // not quite so dumb linear search of all symbols
-    const struct nlist *sym;
+    struct nlist *sym;
     int i, nsyms;
 
     // First try to find the symbol in the most likely place which is the
@@ -1137,7 +1200,7 @@ findSymbolByAddress(const struct fileRecord *file, void *entry)
     return NULL;
 }
 
-static const struct nlist *
+static struct nlist *
 findSymbolByAddressInAllFiles(__unused const struct fileRecord * fromFile, 
 			    void *entry, const struct fileRecord **resultFile)
 {
@@ -1153,7 +1216,7 @@ findSymbolByAddressInAllFiles(__unused const struct fileRecord * fromFile,
 	    if ((((vm_offset_t)entry) >= files[i]->fVMAddr)
 	     && (((vm_offset_t)entry) <  files[i]->fVMEnd))
 	    {
-		const struct nlist * result;
+		struct nlist * result;
 		if (resultFile)
 		    *resultFile = files[i];
 		result = findSymbolByAddress(files[i], entry);
@@ -1198,7 +1261,7 @@ findSymbolByName(struct fileRecord *file, const char *symname)
 
         context.fSymname = symname;
         context.fFile = file;
-        return (struct nlist *)
+        return (const struct nlist *)
             bsearch(&context,
                     file->fLocalSyms, file->fNLocal, sizeof(struct nlist),
                     symbolSearch);
@@ -1206,9 +1269,9 @@ findSymbolByName(struct fileRecord *file, const char *symname)
 }
 
 static Boolean
-relocateSection(const struct fileRecord *file, struct sectionRecord *sectionRec)
+relocateSection(struct fileRecord *file, struct sectionRecord *sectionRec)
 {
-    const struct nlist *symbol;
+    struct nlist *symbol;
     const struct section *section;
     struct relocRecord *rec;
     struct relocation_info *rinfo;
@@ -1283,6 +1346,7 @@ relocateSection(const struct fileRecord *file, struct sectionRecord *sectionRec)
 		("Invalid relocation entry in %s - extern\n", file->fPath));
 	}
 	else {
+	    void * addr = *entry;
 	    /*
 	     * If the symbol is not in any section then it can't be a
 	     * pointer to a local segment and I don't care about it.
@@ -1295,7 +1359,6 @@ relocateSection(const struct fileRecord *file, struct sectionRecord *sectionRec)
 		("Invalid relocation entry in %s - local\n", file->fPath));
 
 	    // Find the symbol, if any, that backs this entry 
-	    void * addr = *entry;
 #if !KERNEL
 	    if (file->fSwapped)
 		addr = (void *) OSSwapInt32((uint32_t) addr);
@@ -1311,13 +1374,13 @@ relocateSection(const struct fileRecord *file, struct sectionRecord *sectionRec)
     }
 
     DataSetLength(sectionRec->fRelocCache, i * sizeof(struct relocRecord));
-    ((struct fileRecord *) file)->fImageDirty = true;
+    file->fImageDirty = true;
 
     return true;
 }
 
 static const struct nlist *
-findSymbolRefAtLocation(const struct fileRecord *file,
+findSymbolRefAtLocation(struct fileRecord *file,
 			struct sectionRecord *sctn, void **loc, const struct fileRecord **foundInFile)
 {
     const struct nlist * result;
@@ -1377,9 +1440,10 @@ addClass(struct fileRecord *file,
 	    (DataGetPtr(file->fClassList) + DataGetLength(file->fClassList));
 
 	// Copy the meta Class structure and string name into newClass and
-        // insert object at end of the file->fClassList and sMergeMetaClasses 
-	*newClass = *inClass;
-	strcpy(newClass->fClassName, cname);
+        // insert object at end of the file->fClassList and sMergeMetaClasses
+        memcpy(newClass, inClass, sizeof(*inClass));
+        // metaClassRecord declares fClassName[1]
+	strlcpy(newClass->fClassName, cname, strlen(cname) + sizeof(newClass->fClassName));
 	fileClasses[-1] = newClass;
 
 	return true;
@@ -1611,6 +1675,8 @@ static Boolean mergeOSObjectsForFile(const struct fileRecord *file)
 		    errprintf("duplicate class %s in %s & %s\n",
 			      list1[k]->fClassName,
 			      file->fPath, list2[j]->fFile->fPath);
+
+                    foundDuplicates = true;
 		}
 	    }
 	}
@@ -1731,7 +1797,7 @@ static Boolean resolveKernelVTable(struct metaClassRecord *metaClass)
 	if (file->fSwapped)
 	    addr = (void *) OSSwapInt32((uint32_t) addr);
 #endif
-	curPatch->fSymbol = (struct nlist *) 
+	curPatch->fSymbol =
 	    findSymbolByAddress(file, addr);
 	if (curPatch->fSymbol)
 	{
@@ -1740,7 +1806,7 @@ static Boolean resolveKernelVTable(struct metaClassRecord *metaClass)
 	}
 	else
 	{
-	    curPatch->fSymbol = (struct nlist *) 
+	    curPatch->fSymbol =
 		findSymbolByAddressInAllFiles(file, addr, &curPatch->fFile);
 	    if (!curPatch->fSymbol) {
 		errprintf("%s: !findSymbolByAddressInAllFiles(%p)\n",
@@ -1758,11 +1824,11 @@ static Boolean resolveKernelVTable(struct metaClassRecord *metaClass)
     return true;
 }
 
-static const char *addNewString(struct fileRecord *file, 
-                                const char *strname, int namelen)
+static char *addNewString(struct fileRecord *file, 
+                                const char *strname, unsigned int namelen)
 {
     DataRef strings = 0;
-    const char *newStr;
+    char *newStr;
 
     namelen++;	// Include terminating '\0';
 
@@ -1801,7 +1867,7 @@ static const char *addNewString(struct fileRecord *file,
 // reloc->fPatch must contain a valid pointer
 static struct nlist *
 getNewSymbol(struct fileRecord *file,
-	     const struct relocRecord *reloc, const char *supername)
+	     struct relocRecord *reloc, const char *supername)
 {
     unsigned int size, i;
     struct nlist **sym;
@@ -1841,14 +1907,14 @@ getNewSymbol(struct fileRecord *file,
 	// Mark the original symbol entry as having been processed.
 	// This means that we wont attempt to create the symbol again
 	// in the future if we come through a different path.
-        ((struct nlist *) reloc->fSymbol)->n_un.n_strx =
+        reloc->fSymbol->n_un.n_strx =
 	    -reloc->fSymbol->n_un.n_strx;    
 
         // Mark the old symbol as being potentially deletable I can use the
         // n_sect field as the input symbol must be of type N_UNDF which means
         // that the n_sect field must be set to NO_SECT otherwise it is an
         // invalid input file.
-        ((struct nlist *) reloc->fSymbol)->n_sect = (unsigned char) -1;
+        reloc->fSymbol->n_sect = (unsigned char) -1;
     }
 
     // If we are here we didn't find the symbol so create a new one now
@@ -1865,7 +1931,7 @@ getNewSymbol(struct fileRecord *file,
     // If we are here we didn't find the symbol so create a new one now
     return_if(!DataAppendBytes(file->fSym2Strings, &newStr, sizeof(newStr)),
             NULL, ("Unable to grow symbol table for %s\n", file->fPath));
-    file->fSymbToStringTable = (const char **) DataGetPtr(file->fSym2Strings);
+    file->fSymbToStringTable = (char **) DataGetPtr(file->fSym2Strings);
 
     // Offset the string index by the original string table size
     // and negate the address to indicate that this is a 'new' symbol
@@ -1884,19 +1950,19 @@ static struct nlist *
 fixOldSymbol(struct fileRecord *file,
 	     const struct relocRecord *reloc, const char *supername)
 {
-    unsigned int namelen;
+    unsigned int namelen, oldnamelen;
     struct nlist *sym = (struct nlist *) reloc->fSymbol;
-    const char *oldname = symbolname(file, sym);
+    char *oldname = symbolname(file, sym);
 
     // assert(sym->n_un.n_strx >= 0);
 
     namelen = strlen(supername);
 
     sym->n_un.n_strx = -sym->n_un.n_strx;
-    if (oldname && namelen < strlen(oldname))
+    if (oldname && namelen < (oldnamelen = strlen(oldname)))
     {
 	// Overwrite old string in string table
-	strcpy((char *) oldname, supername);
+        strlcpy((char *) oldname, supername, oldnamelen + 1);
         file->fSymbolsDirty = true; 
         return sym;
     }
@@ -2174,7 +2240,7 @@ static Boolean growImage(struct fileRecord *file, vm_size_t delta)
     // addr_old = macho_old + fixed_offset
     // addr_new = macho_new + fixed_offset	therefore:
     // addr_new = addr_old + (macho_new - macho_old)
-#define REBASE(addr, delta)	( ((vm_address_t) (addr)) += (delta) )
+#define REBASE(addr, delta)	( *(vm_address_t*)(&addr) += (delta) )
     delta = newMachO - startMachO;
 
     // Rebase the cached-in object 'struct symtab_command' pointer
@@ -2269,6 +2335,9 @@ DEBUG_LOG(("Linking 2 %s\n", file->fPath));	// @@@ gvdl:
 		void **entry;
 		struct nlist *repairSym;
     
+                return_if(!rec->fRInfo, false,
+                         ("Bad Mach-O file; cannot link\n"));
+
 		// Repair Damage to object image
 		entry = (void **) (sectionBase + rec->fRInfo->r_address);
 		*entry = rec->fValue;
@@ -2459,18 +2528,18 @@ kld_file_map(const char *pathName)
 #endif /* KERNEL */
 
     do {
-	const struct machOMapping {
+	struct machOMapping {
 	    struct mach_header h;
 	    struct load_command c[1];
 	} *machO;
-	const struct load_command *cmd;
+	struct load_command *cmd;
 	boolean_t lookVMRange;
         unsigned long i;
 
 	if (!findBestArch(&file, pathName))
 	    break;
 
-	machO = (const struct machOMapping *) file.fMachO;
+	machO = (struct machOMapping *) file.fMachO;
 	if (file.fMachOSize < machO->h.sizeofcmds)
 	    break;
 
@@ -2482,7 +2551,7 @@ kld_file_map(const char *pathName)
             if (cmd->cmd == LC_SYMTAB)
 		file.fSymtab = (struct symtab_command *) cmd;
 	    else if (cmd->cmd == LC_SEGMENT) {
-                struct segment_command *seg = (struct segment_command *) cmd;
+                struct segment_command *seg = (struct segment_command *)cmd;
                 int nsects = seg->nsects;
 
 		if (lookVMRange) {
@@ -2522,7 +2591,7 @@ kld_file_map(const char *pathName)
 	if (machO->h.flags & MH_INCRLINK) {
 
 	    file.fIsIncrLink = true;
-	    ((struct machOMapping *) machO)->h.flags &= ~MH_INCRLINK;
+	    machO->h.flags &= ~MH_INCRLINK;
 
 #if !KERNEL
 	    // the symtab fileoffset is the end of seg0's vmsize,
@@ -2586,7 +2655,7 @@ kld_file_map(const char *pathName)
     return false;
 }
 
-void *kld_file_getaddr(const char *pathName, long *size)
+void *kld_file_getaddr(const char *pathName, unsigned long *size)
 {
     struct fileRecord *file = getFile(pathName);
 
diff --git a/libsa/kld_patch.h b/libsa/kld_patch.h
index 773784031..13cb363f6 100644
--- a/libsa/kld_patch.h
+++ b/libsa/kld_patch.h
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.0 (the 'License').  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License."
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * History:
@@ -43,7 +47,7 @@ Boolean kld_file_debug_dump(const char *pathName, const char *outName);
 extern void *
     kld_file_lookupsymbol(const char *pathName, const char *symbolname);
 
-extern void *kld_file_getaddr(const char *pathName, long *size);
+extern void *kld_file_getaddr(const char *pathName, unsigned long *size);
 
 extern Boolean kld_file_merge_OSObjects(const char *pathName);
 
diff --git a/libsa/kmod.cpp b/libsa/kmod.cpp
deleted file mode 100644
index 2847e9f15..000000000
--- a/libsa/kmod.cpp
+++ /dev/null
@@ -1,1231 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <libsa/kmod.h>
-#include <libkern/c++/OSContainers.h>
-#include <IOKit/IOCatalogue.h>
-#include <IOKit/IOLib.h>
-#include <libsa/kmod.h>
-#include <libsa/catalogue.h>
-
-extern "C" {
-#include <mach-o/kld.h>
-#include <libsa/vers_rsrc.h>
-#include <libsa/stdlib.h>
-#include <mach/kmod.h>
-#include <vm/vm_kern.h>
-#include <mach/kern_return.h>
-#include <mach-o/fat.h>
-#include <mach_loader.h>
-};
-
-#include "kld_patch.h"
-
-
-extern "C" {
-extern kern_return_t
-kmod_create_internal(
-            kmod_info_t *info,
-            kmod_t *id);
-
-extern kern_return_t
-kmod_destroy_internal(kmod_t id);
-
-extern kern_return_t
-kmod_start_or_stop(
-    kmod_t id,
-    int start,
-    kmod_args_t *data,
-    mach_msg_type_number_t *dataCount);
-
-extern kern_return_t kmod_retain(kmod_t id);
-extern kern_return_t kmod_release(kmod_t id);
-
-extern void flush_dcache64(addr64_t addr, unsigned cnt, int phys);
-extern void invalidate_icache64(addr64_t addr, unsigned cnt, int phys);
-};
-
-
-#define LOG_DELAY()
-
-#define VTYELLOW	"\033[33m"
-#define VTRESET		"\033[0m"
-
-
-
-
-/*********************************************************************
-*
-*********************************************************************/
-bool verifyCompatibility(OSString * extName, OSString * requiredVersion)
-{
-    OSDictionary * extensionsDict;   // don't release
-    OSDictionary * extDict;          // don't release
-    OSDictionary * extPlist;         // don't release
-    OSString     * extVersion;       // don't release
-    OSString     * extCompatVersion; // don't release
-    UInt32 ext_version;
-    UInt32 ext_compat_version;
-    UInt32 required_version;
-
-   /* Get the dictionary of startup extensions.
-    * This is keyed by module name.
-    */
-    extensionsDict = getStartupExtensions();
-    if (!extensionsDict) {
-        IOLog("verifyCompatibility(): No extensions dictionary.\n");
-        return false;
-    }
-
-   /* Get the requested extension's dictionary entry and its property
-    * list, containing module dependencies.
-    */
-    extDict = OSDynamicCast(OSDictionary,
-        extensionsDict->getObject(extName));
-
-    if (!extDict) {
-        IOLog("verifyCompatibility(): "
-           "Extension \"%s\" cannot be found.\n",
-           extName->getCStringNoCopy());
-        return false;
-    }
-
-    extPlist = OSDynamicCast(OSDictionary, extDict->getObject("plist"));
-    if (!extPlist) {
-        IOLog("verifyCompatibility(): "
-            "Extension \"%s\" has no property list.\n",
-            extName->getCStringNoCopy());
-        return false;
-    }
-
-
-    extVersion = OSDynamicCast(OSString,
-        extPlist->getObject("CFBundleVersion"));
-    if (!extVersion) {
-        IOLog("verifyCompatibility(): "
-            "Extension \"%s\" has no \"CFBundleVersion\" property.\n",
-            extName->getCStringNoCopy());
-        return false;
-    }
-
-    extCompatVersion = OSDynamicCast(OSString,
-        extPlist->getObject("OSBundleCompatibleVersion"));
-    if (!extCompatVersion) {
-        IOLog("verifyCompatibility(): "
-            "Extension \"%s\" has no \"OSBundleCompatibleVersion\" property.\n",
-            extName->getCStringNoCopy());
-        return false;
-    }
-
-    if (!VERS_parse_string(requiredVersion->getCStringNoCopy(),
-         &required_version)) {
-        IOLog("verifyCompatibility(): "
-            "Can't parse required version \"%s\" of dependency %s.\n",
-            requiredVersion->getCStringNoCopy(),
-            extName->getCStringNoCopy());
-        return false;
-    }
-    if (!VERS_parse_string(extVersion->getCStringNoCopy(),
-         &ext_version)) {
-        IOLog("verifyCompatibility(): "
-            "Can't parse version \"%s\" of dependency %s.\n",
-            extVersion->getCStringNoCopy(),
-            extName->getCStringNoCopy());
-        return false;
-    }
-    if (!VERS_parse_string(extCompatVersion->getCStringNoCopy(),
-         &ext_compat_version)) {
-        IOLog("verifyCompatibility(): "
-            "Can't parse compatible version \"%s\" of dependency %s.\n",
-            extCompatVersion->getCStringNoCopy(),
-            extName->getCStringNoCopy());
-        return false;
-    }
-
-    if (required_version > ext_version || required_version < ext_compat_version) {
-        return false;
-    }
-
-    return true;
-}
-
-/*********************************************************************
-*********************************************************************/
-static
-Boolean kextIsADependency(OSString * name) {
-    Boolean result = true;
-    OSDictionary * extensionsDict = 0;    // don't release
-    OSDictionary * extDict = 0;           // don't release
-    OSDictionary * extPlist = 0;          // don't release
-    OSBoolean * isKernelResourceObj = 0;  // don't release
-    OSData * driverCode = 0;              // don't release
-    OSData * compressedCode = 0;          // don't release
-
-    extensionsDict = getStartupExtensions();
-    if (!extensionsDict) {
-        IOLog("kextIsADependency(): No extensions dictionary.\n");
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-    
-
-    extDict = OSDynamicCast(OSDictionary,
-        extensionsDict->getObject(name));
-    if (!extDict) {
-        IOLog("kextIsADependency(): "
-           "Extension \"%s\" cannot be found.\n",
-           name->getCStringNoCopy());
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    extPlist = OSDynamicCast(OSDictionary, extDict->getObject("plist"));
-    if (!extPlist) {
-        IOLog("getDependencyListForKmod(): "
-            "Extension \"%s\" has no property list.\n",
-            name->getCStringNoCopy());
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-   /* A kext that is a kernel component is still a dependency, as there
-    * are fake kmod entries for them.
-    */
-    isKernelResourceObj = OSDynamicCast(OSBoolean,
-        extPlist->getObject("OSKernelResource"));
-    if (isKernelResourceObj && isKernelResourceObj->isTrue()) {
-        result = true;
-        goto finish;
-    }
-
-    driverCode = OSDynamicCast(OSData, extDict->getObject("code"));
-    compressedCode = OSDynamicCast(OSData,
-        extDict->getObject("compressedCode"));
-
-    if (!driverCode && !compressedCode) {
-        result = false;
-        goto finish;
-    }
-
-finish:
-
-    return result;
-}
-
-/*********************************************************************
-* This function builds a uniqued, in-order list of modules that need
-* to be loaded in order for kmod_name to be successfully loaded. This
-* list ends with kmod_name itself.
-*********************************************************************/
-static
-OSArray * getDependencyListForKmod(const char * kmod_name) {
-
-    int error = 0;
-
-    OSDictionary * extensionsDict; // don't release
-    OSDictionary * extDict;        // don't release
-    OSDictionary * extPlist;       // don't release
-    OSString     * extName;        // don't release
-    OSArray      * dependencyList = NULL; // return value, caller releases
-    unsigned int   i;
-
-   /* These are used to remove duplicates from the dependency list.
-    */
-    OSArray      * originalList = NULL;     // must be released
-    OSDictionary * encounteredNames = NULL; // must be release
-
-
-   /* Get the dictionary of startup extensions.
-    * This is keyed by module name.
-    */
-    extensionsDict = getStartupExtensions();
-    if (!extensionsDict) {
-        IOLog("getDependencyListForKmod(): No extensions dictionary.\n");
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-    
-
-   /* Get the requested extension's dictionary entry and its property
-    * list, containing module dependencies.
-    */
-    extDict = OSDynamicCast(OSDictionary,
-        extensionsDict->getObject(kmod_name));
-
-    if (!extDict) {
-        IOLog("getDependencyListForKmod(): "
-           "Extension \"%s\" cannot be found.\n",
-           kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-
-    extPlist = OSDynamicCast(OSDictionary, extDict->getObject("plist"));
-    if (!extPlist) {
-        IOLog("getDependencyListForKmod(): "
-            "Extension \"%s\" has no property list.\n",
-            kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-
-
-   /* Verify that the retrieved entry's "CFBundleIdentifier" property exists.
-    * This will be added to the dependency list.
-    */
-    extName = OSDynamicCast(OSString,
-        extPlist->getObject("CFBundleIdentifier"));
-    if (!extName) {
-        IOLog("getDependencyListForKmod(): "
-            "Extension \"%s\" has no \"CFBundleIdentifier\" property.\n",
-            kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-
-    dependencyList = OSArray::withCapacity(10);
-    if (!dependencyList) {
-        IOLog("getDependencyListForKmod(): "
-            "Couldn't allocate dependency array for extension \"%s\".\n",
-            kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-
-
-   /* Okay, let's get started.
-    */
-    dependencyList->setObject(extName);
-
-
-   /* Here's a slightly tricky bit. This loop iterates through
-    * the dependency list until it runs off the end. Each time
-    * through, however, any number of dependencies can be added
-    * to the end of the list. Eventually some extensions won't
-    * have any more dependencies, no more names will be added
-    * to the list, and this loop will terminate.
-    */
-    for (i = 0; i < dependencyList->getCount(); i++) {
-
-        // None of these needs to be released, as they're all from plists.
-        OSString     * curName;
-        OSDictionary * curExtDict;
-        OSDictionary * curExtDepDict;
-        OSDictionary * curExtPlist;
-        OSString     * curDepName;
-
-
-       /* An arbitrary limit to prevent infinite loops.
-        */
-        if (i > 255) {
-            IOLog("getDependencyListForKmod(): "
-                "max dependency list length exceeded for "
-                "extension \"%s\".\n",
-                kmod_name);
-            LOG_DELAY();
-            error = 1;
-            goto finish;
-        }
-
-        curName = OSDynamicCast(OSString, dependencyList->getObject(i));
-
-        curExtDict = OSDynamicCast(OSDictionary,
-            extensionsDict->getObject(curName));
-        if (!curExtDict) {
-            IOLog("getDependencyListForKmod(): "
-                "Extension \"%s\", required for extension \"%s\", "
-                "is not available.\n",
-                curName->getCStringNoCopy(), kmod_name);
-            LOG_DELAY();
-            error = 1;
-            goto finish;
-        }
-
-        curExtPlist = OSDynamicCast(OSDictionary,
-            curExtDict->getObject("plist"));
-        if (!curExtPlist) {
-            IOLog("getDependencyListForKmod(): "
-                "Extension \"%s\", required for extension \"%s\", "
-                "has no property list.\n",
-                curName->getCStringNoCopy(), kmod_name);
-            LOG_DELAY();
-            error = 1;
-            goto finish;
-        }
-
-        curExtDepDict = OSDynamicCast(OSDictionary,
-              curExtPlist->getObject("OSBundleLibraries"));
-        if (curExtDepDict) {
-            OSCollectionIterator * keyIterator =
-                OSCollectionIterator::withCollection(curExtDepDict);
-
-            if (!keyIterator) {
-                IOLog("getDependencyListForKmod(): "
-                    "Couldn't allocate iterator for extension "
-                    "\"%s\".\n", kmod_name);
-                LOG_DELAY();
-                error = 1;
-                goto finish;
-            }
-            while ( (curDepName =
-                     OSDynamicCast(OSString,
-                         keyIterator->getNextObject())) ) {
-
-                OSString * requiredVersion = OSDynamicCast(OSString,
-                    curExtDepDict->getObject(curDepName));
-
-                if (!verifyCompatibility(curDepName, requiredVersion)) {
-                    IOLog("getDependencyListForKmod(): "
-                        "Dependency %s of %s is not compatible or is unavailable.\n",
-                        curDepName->getCStringNoCopy(),
-                        curName->getCStringNoCopy());
-                    LOG_DELAY();
-                    error = 1;
-                    goto finish;
-                }
-
-                dependencyList->setObject(curDepName);
-            }
-
-            keyIterator->release();
-        }
-    }
-
-
-   /*****
-    * The dependency list now exists in the reverse order of required loads,
-    * and may have duplicates. Now we turn the list around and remove
-    * duplicates.
-    */
-    originalList = dependencyList;
-    dependencyList = OSArray::withCapacity(originalList->getCount());
-    if (!dependencyList) {
-        IOLog("getDependenciesForKmod(): "
-              "Couldn't allocate reversal dependency list for extension "
-              "\"%s\".\n", kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-    encounteredNames = OSDictionary::withCapacity(originalList->getCount());
-    if (!encounteredNames) {
-        IOLog("getDependenciesForKmod(): "
-              "Couldn't allocate list of encountered names for extension "
-              "\"%s\".\n", kmod_name);
-        LOG_DELAY();
-        error = 1;
-        goto finish;
-    }
-
-
-   /* Go backward through the original list, using the encounteredNames
-    * dictionary to check for duplicates. We put originalList in as the
-    * value because we need some non-NULL value. Here we also drop any
-    * extensions that aren't proper dependencies (that is, any that are
-    * nonkernel kexts without code).
-    */
-    i = originalList->getCount();
-
-    if (i > 0) {
-        do {
-            i--;
-
-            OSString * item = OSDynamicCast(OSString,
-                originalList->getObject(i));
-
-            if ( (!encounteredNames->getObject(item)) &&
-                 kextIsADependency(item)) {
-
-                encounteredNames->setObject(item, originalList);
-                dependencyList->setObject(item);
-            }
-        } while (i > 0);
-    }
-
-
-finish:
-
-    if (originalList) {
-        originalList->release();
-    }
-    if (encounteredNames) {
-        encounteredNames->release();
-    }
-    if (error) {
-        if (dependencyList) {
-            dependencyList->release();
-            dependencyList = NULL;
-        }
-    }
-
-    return dependencyList;
-}
-
-
-/*********************************************************************
-*********************************************************************/
-/* Used in address_for_loaded_kmod.
- */
-static kmod_info_t * g_current_kmod_info = NULL;
-static const char * g_current_kmod_name = NULL;
-
-/* Globals to pass link buffer info from
- * address_for_loaded_kmod() and alloc_for_kmod()
- * to load_kmod().
- *
- * link_load_address is the address used to lay
- * down the linked code. It gets adjusted by the
- * pad between the headers size and a full page
- * multiple. If an error occurs this gets set to
- * zero so that the kld client code can detect
- * an address or allocation error even if kld
- * returns success.
- *
- * link_load_size is the size of the image as
- * created by kld_load_from_memory(). link_buffer_size
- * is the size of the buffer allocated for the final
- * laid-down image, and is adjusted by rounding the
- * load size and header size up to full-page multiples.
- *
- * link_buffer_address is set only by alloc_for_kmod();
- * its value is used as a check if kld_load_from_memory()
- * fails so that the buffer can be deallocated. 
- */
-static unsigned long link_load_address = 0;
-static unsigned long link_load_size = 0;
-static unsigned long link_buffer_size = 0;
-static unsigned long link_header_size = 0;
-static unsigned long link_buffer_address = 0;
-
-
-/*********************************************************************
-* This function is registered before kmod_load_from_memory() is
-* invoked to build symbol table entries for an already-loaded
-* kmod. This function just checks the g_current_kmod_info variable
-* to gets its load address, and futzes it by the header offset (pad).
-* See lower comments for more info on load address futzing.
-*********************************************************************/
-static
-unsigned long address_for_loaded_kmod(
-    unsigned long size,
-    unsigned long headers_size) {
-
-    unsigned long round_headers_size;
-    unsigned long headers_pad;
-
-    if (!g_current_kmod_info) {
-        IOLog("address_for_loaded_kmod(): No current kmod.\n");
-        LOG_DELAY();
-        link_load_address = 0;  // error sentinel for kld client
-        return 0;
-    }
-
-    round_headers_size = round_page_32(headers_size);
-    headers_pad = round_headers_size - headers_size;
-
-    link_load_address = (unsigned long)g_current_kmod_info->address +
-        headers_pad;
-
-    return link_load_address;
-}
-
-
-/*********************************************************************
-* This function is registered before kmod_load_from_memory() is
-* invoked to actually load a new kmod. It rounds up the header and
-* total sizes and vm_allocates a buffer for the kmod. Now, KLD doesn't
-* enforce any alignment of headers or segments, and we want to make
-* sure that the executable code of the kmod lies on a page boundary.
-* to do so, this function figures the pad between the actual header
-* size and the page-rounded header size, and returns that offset into
-* the allocated buffer. After kmod_load_from_memory() returns, its
-* caller will move the mach_header struct back to the beginning of the
-* allocated buffer so that the kmod_info_t structure contains the
-* correct address.
-*********************************************************************/
-static
-unsigned long alloc_for_kmod(
-    unsigned long size,
-    unsigned long headers_size) {
-
-    vm_address_t  buffer = 0;
-    kern_return_t k_result;
-
-    unsigned long round_headers_size;
-    unsigned long round_segments_size;
-    unsigned long round_size;
-    unsigned long headers_pad;
-
-    round_headers_size  = round_page_32(headers_size);
-    round_segments_size = round_page_32(size - headers_size);
-    round_size  = round_headers_size + round_segments_size;
-    headers_pad = round_headers_size - headers_size;
-
-    k_result = vm_allocate(kernel_map, (vm_offset_t *)&buffer,
-        round_size, VM_FLAGS_ANYWHERE);
-    if (k_result != KERN_SUCCESS) {
-        IOLog("alloc_for_kmod(): Can't allocate memory.\n");
-        LOG_DELAY();
-        link_buffer_address = 0;  // make sure it's clear
-        link_load_address = 0;    // error sentinel for kld client
-        return 0;
-    }
-
-    link_load_size = size;
-
-    link_buffer_address = buffer;
-    link_buffer_size = round_size;
-    link_header_size = headers_size; // NOT rounded!
-
-    link_load_address = link_buffer_address + headers_pad;
-
-    return link_load_address;
-}
-
-/*********************************************************************
-* This function reads the startup extensions dictionary to get the
-* address and length of the executable data for the requested kmod.
-*********************************************************************/
-static
-int map_and_patch(const char * kmod_name) {
-
-    char *address;
-
-    // Does the kld system already know about this kmod?
-    address = (char *) kld_file_getaddr(kmod_name, NULL);
-    if (address)
-	return 1;
-
-    // None of these needs to be released.
-    OSDictionary * extensionsDict;
-    OSDictionary * kmodDict;
-    OSData * compressedCode = 0;
-
-    // Driver Code may need to be released
-    OSData * driverCode;
-
-   /* Get the requested kmod's info dictionary from the global
-    * startup extensions dictionary.
-    */
-    extensionsDict = getStartupExtensions();
-    if (!extensionsDict) {
-        IOLog("map_and_patch(): No extensions dictionary.\n");
-        LOG_DELAY();
-        return 0;
-    }
-    
-    kmodDict = OSDynamicCast(OSDictionary,
-        extensionsDict->getObject(kmod_name));
-    if (!kmodDict) {
-        IOLog("map_and_patch(): "
-            "Extension \"%s\" cannot be found.\n", kmod_name);
-        LOG_DELAY();
-        return 0;
-    }
-
-    Boolean ret = false;
-
-    driverCode = OSDynamicCast(OSData, kmodDict->getObject("code"));
-    if (driverCode) {
-	ret =  kld_file_map(kmod_name,
-			    (unsigned char *) driverCode->getBytesNoCopy(),
-			    (size_t) driverCode->getLength(),
-			    /* isKmem */ false);
-    }
-    else {	// May be an compressed extension
-
-	// If we have a compressed segment the uncompressModule
-	// will return a new OSData object that points to the kmem_alloced
-	// memory.  Note we don't take a reference to driverCode so later
-	// when we release it we will actually free this driver.  Ownership
-	// of the kmem has been handed of to kld_file.
-	compressedCode = OSDynamicCast(OSData,
-	    kmodDict->getObject("compressedCode"));
-	if (!compressedCode) {
-	    IOLog("map_and_patch(): "
-		 "Extension \"%s\" has no \"code\" property.\n", kmod_name);
-	    LOG_DELAY();
-	    return 0;
-	}
-	if (!uncompressModule(compressedCode, &driverCode)) {
-	    IOLog("map_and_patch(): "
-		 "Extension \"%s\" Couldn't uncompress code.\n", kmod_name);
-	    LOG_DELAY();
-	    return 0;
-	}
-
-	unsigned char *driver = (unsigned char *) driverCode->getBytesNoCopy();
-	size_t driverSize = driverCode->getLength();
-
-	ret =  kld_file_map(kmod_name, driver, driverSize, /* isKmem */ true);
-	driverCode->release();
-	if (!ret)
-	    kmem_free(kernel_map, (vm_address_t) driver, driverSize);
-    }
-
-    if (!ret) {
-        IOLog("map_and_patch(): "
-              "Extension \"%s\" Didn't successfully load.\n", kmod_name);
-        LOG_DELAY();
-	return 0;
-    }
-
-    ret = TRUE;
-    if (!kld_file_patch_OSObjects(kmod_name)) {
-        IOLog("map_and_patch(): "
-              "Extension \"%s\" Error binding OSObjects.\n", kmod_name);
-        LOG_DELAY();
-        
-        // RY: Instead of returning here, set the return value.
-        // We still need to call kld_file_prepare_for_link because
-        // we might have patched files outside of the driver.  Don't
-        // worry, it will know to ignore the damaged file
-        ret = FALSE;
-    }
-
-    // Now repair any damage that the kld patcher may have done to the image
-    kld_file_prepare_for_link();
-
-    return ret;
-}
-
-/*********************************************************************
-*********************************************************************/
-bool stamp_kmod(const char * kmod_name, kmod_info_t * kmod_info) {
-    bool result = false;
-    OSDictionary * extensionsDict = NULL;  // don't release
-    OSDictionary * kmodDict = NULL;        // don't release
-    OSDictionary * plist = NULL;           // don't release
-    OSString     * versionString = NULL;   // don't release
-    const char   * plist_version = NULL;   // don't free
-
-    if (strlen(kmod_name) + 1 > KMOD_MAX_NAME) {
-        IOLog("stamp_kmod(): Kext identifier \"%s\" is too long.\n",
-            kmod_name);
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    strcpy(kmod_info->name, kmod_name);
-
-   /* Get the dictionary of startup extensions.
-    * This is keyed by module name.
-    */
-    extensionsDict = getStartupExtensions();
-    if (!extensionsDict) {
-        IOLog("stamp_kmod(): No extensions dictionary.\n");
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    kmodDict = OSDynamicCast(OSDictionary,
-        extensionsDict->getObject(kmod_name));
-    if (!kmodDict) {
-        IOLog("stamp_kmod(): Can't find record for kmod \"%s\".\n",
-            kmod_name);
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    plist = OSDynamicCast(OSDictionary,
-        kmodDict->getObject("plist"));
-    if (!kmodDict) {
-        IOLog("stamp_kmod(): Kmod \"%s\" has no property list.\n",
-            kmod_name);
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-   /*****
-    * Get the kext's version and stuff it into the kmod. This used
-    * to be a check that the kext & kmod had the same version, but
-    * now we just overwrite the kmod's version.
-    */
-
-    versionString = OSDynamicCast(OSString,
-        plist->getObject("CFBundleVersion"));
-    if (!versionString) {
-        IOLog("stamp_kmod(): Kmod \"%s\" has no \"CFBundleVersion\" "
-            "property.\n",
-            kmod_name);
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    plist_version = versionString->getCStringNoCopy();
-    if (!plist_version) {
-        IOLog("stamp_kmod(): Can't get C string for kext version.\n");
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    if (strlen(plist_version) + 1 > KMOD_MAX_NAME) {
-        IOLog("stamp_kmod(): Version \"%s\" of kext \"%s\" is too long.\n",
-            plist_version, kmod_name);
-        LOG_DELAY();
-        result = false;
-        goto finish;
-    }
-
-    strcpy(kmod_info->version, plist_version);
-
-    result = true;
-
-finish:
-
-    return result;
-}
-
-
-/*********************************************************************
-* This function takes a dependency list containing a series of
-* already-loaded module names, followed by a single name for a module
-* that hasn't yet been loaded. It invokes kld_load_from_memory() to
-* build symbol info for the already-loaded modules, and then finally
-* loads the actually requested module.
-*********************************************************************/
-static
-kern_return_t load_kmod(OSArray * dependencyList) {
-    kern_return_t result = KERN_SUCCESS;
-
-    unsigned int  num_dependencies = 0;
-    kmod_info_t ** kmod_dependencies = NULL;
-    unsigned int  i;
-    OSString    * requestedKmodName;   // don't release
-    const char  * requested_kmod_name;
-    OSString    * currentKmodName;     // don't release
-    char        * kmod_address;
-    unsigned long kmod_size;
-    struct mach_header * kmod_header;
-    unsigned long kld_result;
-    int           do_kld_unload = 0;
-    kmod_info_t * kmod_info_freeme = 0;
-    kmod_info_t * kmod_info = 0;
-    kmod_t        kmod_id;
-
-
-   /* Separate the requested kmod from its dependencies.
-    */
-    i = dependencyList->getCount();
-    if (i == 0) {
-        IOLog("load_kmod(): Called with empty list.\n");
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    } else {
-        i--;  // make i be the index of the last entry
-    }
-
-    requestedKmodName = OSDynamicCast(OSString, dependencyList->getObject(i));
-    if (!requestedKmodName) {
-        IOLog("load_kmod(): Called with invalid list of kmod names.\n");
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-    requested_kmod_name = requestedKmodName->getCStringNoCopy();
-    dependencyList->removeObject(i);
-
-   /* If the requested kmod is already loaded, there's no work to do.
-    */
-    kmod_info_freeme = kmod_lookupbyname_locked(requested_kmod_name);
-    if (kmod_info_freeme) {
-        // FIXME: Need to check for version mismatch if already loaded.
-        result = KERN_SUCCESS;
-        goto finish;
-    }
-
-
-   /* Do the KLD loads for the already-loaded modules in order to get
-    * their symbols.
-    */
-    kld_address_func(&address_for_loaded_kmod);
-
-    num_dependencies = dependencyList->getCount();
-    kmod_dependencies = (kmod_info_t **)kalloc(num_dependencies *
-        sizeof(kmod_info_t *));
-    if (!kmod_dependencies) {
-        IOLog("load_kmod(): Failed to allocate memory for dependency array "
-            "during load of kmod \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-    bzero(kmod_dependencies, num_dependencies *
-        sizeof(kmod_info_t *));
-
-    for (i = 0; i < num_dependencies; i++) {
-
-        currentKmodName = OSDynamicCast(OSString,
-            dependencyList->getObject(i));
-
-        if (!currentKmodName) {
-            IOLog("load_kmod(): Invalid dependency name at index %d for "
-                "kmod \"%s\".\n", i, requested_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        }
-
-        const char * current_kmod_name = currentKmodName->getCStringNoCopy();
-
-        // These globals are needed by the kld_address functions
-        g_current_kmod_info = kmod_lookupbyname_locked(current_kmod_name);
-        g_current_kmod_name = current_kmod_name;
-
-        if (!g_current_kmod_info) {
-            IOLog("load_kmod(): Missing dependency \"%s\".\n",
-                current_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        }
-
-       /* Record the current kmod as a dependency of the requested
-        * one. This will be used in building references after the
-        * load is complete.
-        */
-        kmod_dependencies[i] = g_current_kmod_info;
-
-        /* If the current kmod's size is zero it means that we have a
-         * fake in-kernel dependency.  If so then don't have to arrange
-         * for its symbol table to be reloaded as it is
-         * part of the kernel's symbol table..
-         */ 
-        if (!g_current_kmod_info->size)
-            continue;
-
-	if (!kld_file_merge_OSObjects(current_kmod_name)) {
-            IOLog("load_kmod(): Can't merge OSObjects \"%s\".\n",
-		current_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        }
-
-	kmod_address = (char *)
-	    kld_file_getaddr(current_kmod_name, (long *) &kmod_size);
-        if (!kmod_address) {
-
-            IOLog("load_kmod() failed for dependency kmod "
-                "\"%s\".\n", current_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        }
-
-        kld_result = kld_load_from_memory(&kmod_header,
-            current_kmod_name, kmod_address, kmod_size);
-
-        if (kld_result) {
-            do_kld_unload = 1;
-        }
-
-        if (!kld_result || !link_load_address) {
-            IOLog("kld_load_from_memory() failed for dependency kmod "
-                "\"%s\".\n", current_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        }
-
-        kld_forget_symbol("_kmod_info");
-    }
-
-   /*****
-    * Now that we've done all the dependencies, which should have already
-    * been loaded, we do the last requested module, which should not have
-    * already been loaded.
-    */
-    kld_address_func(&alloc_for_kmod);
-
-    g_current_kmod_name = requested_kmod_name;
-    g_current_kmod_info = 0;  // there is no kmod yet
-
-    if (!map_and_patch(requested_kmod_name)) {
-	IOLog("load_kmod: map_and_patch() failed for "
-	    "kmod \"%s\".\n", requested_kmod_name);
-	LOG_DELAY();
-	result = KERN_FAILURE;
-	goto finish;
-    }
-
-    kmod_address = (char *)
-	kld_file_getaddr(requested_kmod_name, (long *) &kmod_size);
-    if (!kmod_address) {
-        IOLog("load_kmod: kld_file_getaddr()  failed internal error "
-            "on \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-    kld_result = kld_load_from_memory(&kmod_header,
-			    requested_kmod_name, kmod_address, kmod_size);
-
-    if (kld_result) {
-        do_kld_unload = 1;
-    }
-
-    if (!kld_result || !link_load_address) {
-        IOLog("load_kmod(): kld_load_from_memory() failed for "
-            "kmod \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-
-   /* Copy the linked header and image into the vm_allocated buffer.
-    * Move each onto the appropriate page-aligned boundary as given
-    * by the global link_... variables.
-    */
-    bzero((char *)link_buffer_address, link_buffer_size);
-    // bcopy() is (from, to, length)
-    bcopy((char *)kmod_header, (char *)link_buffer_address, link_header_size);
-    bcopy((char *)kmod_header + link_header_size,
-        (char *)link_buffer_address + round_page_32(link_header_size),
-        link_load_size - link_header_size);
-
-
-   /* Get the kmod_info struct for the newly-loaded kmod.
-    */
-    if (!kld_lookup("_kmod_info", (unsigned long *)&kmod_info)) {
-        IOLog("kld_lookup() of \"_kmod_info\" failed for "
-            "kmod \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-
-    if (!stamp_kmod(requested_kmod_name, kmod_info)) {
-        // stamp_kmod() logs a meaningful message
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-
-   /* kld_lookup of _kmod_info yielded the actual linked address,
-    * so now that we've copied the data into its real place,
-    * we can set this stuff.
-    */
-    kmod_info->address = link_buffer_address;
-    kmod_info->size = link_buffer_size;
-    kmod_info->hdr_size = round_page_32(link_header_size);
-
-   /* We've written data and instructions, so *flush* the data cache
-    * and *invalidate* the instruction cache.
-    */
-    flush_dcache64((addr64_t)link_buffer_address, link_buffer_size, false);
-    invalidate_icache64((addr64_t)link_buffer_address, link_buffer_size, false);
-
-
-   /* Register the new kmod with the kernel proper.
-    */
-    if (kmod_create_internal(kmod_info, &kmod_id) != KERN_SUCCESS) {
-        IOLog("load_kmod(): kmod_create() failed for "
-            "kmod \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-#if DEBUG
-    IOLog("kmod id %d successfully created at 0x%lx, size %ld.\n",
-        (unsigned int)kmod_id, link_buffer_address, link_buffer_size);
-    LOG_DELAY();
-#endif /* DEBUG */
-
-   /* Record dependencies for the newly-loaded kmod.
-    */
-    for (i = 0; i < num_dependencies; i++) {
-        kmod_info_t * cur_dependency_info;
-        kmod_t packed_id;
-        cur_dependency_info = kmod_dependencies[i];
-        packed_id = KMOD_PACK_IDS(kmod_id, cur_dependency_info->id);
-        if (kmod_retain(packed_id) != KERN_SUCCESS) {
-            IOLog("load_kmod(): kmod_retain() failed for "
-                "kmod \"%s\".\n", requested_kmod_name);
-            LOG_DELAY();
-            kmod_destroy_internal(kmod_id);
-            result = KERN_FAILURE;
-            goto finish;
-        }
-    }
-
-   /* Start the kmod (which invokes constructors for I/O Kit
-    * drivers.
-    */
-    // kmod_start_or_stop(id, start?, user data, datalen)
-    if (kmod_start_or_stop(kmod_id, 1, 0, 0) != KERN_SUCCESS) {
-        IOLog("load_kmod(): kmod_start_or_stop() failed for "
-            "kmod \"%s\".\n", requested_kmod_name);
-        LOG_DELAY();
-        kmod_destroy_internal(kmod_id);
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-finish:
-
-    if (kmod_info_freeme) {
-        kfree((unsigned int)kmod_info_freeme, sizeof(kmod_info_t));
-    }
-
-   /* Only do a kld_unload_all() if at least one load happened.
-    */
-    if (do_kld_unload) {
-        kld_unload_all(/* deallocate sets */ 1);
-    }
-
-   /* If the link failed, blow away the allocated link buffer.
-    */
-    if (result != KERN_SUCCESS && link_buffer_address) {
-        vm_deallocate(kernel_map, link_buffer_address, link_buffer_size);
-    }
-
-    if (kmod_dependencies) {
-        for (i = 0; i < num_dependencies; i++) {
-            if (kmod_dependencies[i]) {
-                kfree((unsigned int)kmod_dependencies[i], sizeof(kmod_info_t));
-            }
-        }
-        kfree((unsigned int)kmod_dependencies,
-            num_dependencies * sizeof(kmod_info_t *));
-    }
-
-   /* Reset these static global variables for the next call.
-    */
-    g_current_kmod_name = NULL;
-    g_current_kmod_info = NULL;
-    link_buffer_address = 0;
-    link_load_address = 0;
-    link_load_size = 0;
-    link_buffer_size = 0;
-    link_header_size = 0;
-
-    return result;
-}
-
-
-/*********************************************************************
-* This is the function that IOCatalogue calls in order to load a kmod.
-* It first checks whether the kmod is already loaded. If the kmod
-* isn't loaded, this function builds a dependency list and calls
-* load_kmod() repeatedly to guarantee that each dependency is in fact
-* loaded.
-*********************************************************************/
-__private_extern__
-kern_return_t load_kernel_extension(char * kmod_name) {
-    kern_return_t result = KERN_SUCCESS;
-    kmod_info_t * kmod_info = 0;  // must free
-    OSArray * dependencyList = NULL;     // must release
-    OSArray * curDependencyList = NULL;  // must release
-
-   /* See if the kmod is already loaded.
-    */
-    kmod_info = kmod_lookupbyname_locked(kmod_name);
-    if (kmod_info) {  // NOT checked
-        result = KERN_SUCCESS;
-        goto finish;
-    }
-
-   /* It isn't loaded; build a dependency list and
-    * load those.
-    */
-    unsigned int count;
-    unsigned int i;
-    dependencyList = getDependencyListForKmod(kmod_name);
-    if (!dependencyList) {
-        IOLog("load_kernel_extension(): "
-            "Can't get dependencies for kernel extension \"%s\".\n",
-            kmod_name);
-        LOG_DELAY();
-        result = KERN_FAILURE;
-        goto finish;
-    }
-
-    count = dependencyList->getCount();
-    for (i = 0; i < count; i++) {
-        kern_return_t load_result;
-        OSString * curKmodName;  // don't release
-        const char * cur_kmod_name;
-
-        curKmodName = OSDynamicCast(OSString,
-            dependencyList->getObject(i));
-        cur_kmod_name = curKmodName->getCStringNoCopy();
-        curDependencyList = getDependencyListForKmod(cur_kmod_name);
-        if (!curDependencyList) {
-            IOLog("load_kernel_extension(): "
-                "Can't get dependencies for kernel extension \"%s\".\n",
-                cur_kmod_name);
-            LOG_DELAY();
-            result = KERN_FAILURE;
-            goto finish;
-        } else {
-            load_result = load_kmod(curDependencyList);
-            if (load_result != KERN_SUCCESS) {
-                IOLog("load_kernel_extension(): "
-                    "load_kmod() failed for kmod \"%s\".\n",
-                    cur_kmod_name);
-                LOG_DELAY();
-                result = load_result;
-                goto finish;
-            }
-            curDependencyList->release();
-            curDependencyList = NULL;
-        }
-    }
-
-
-finish:
-
-    if (kmod_info) {
-        kfree((unsigned int)kmod_info, sizeof(kmod_info_t));
-    }
-
-    if (dependencyList) {
-        dependencyList->release();
-        dependencyList = NULL;
-    }
-    if (curDependencyList) {
-        curDependencyList->release();
-        curDependencyList = NULL;
-    }
-
-    return result;
-}
diff --git a/libsa/libsa/i386/setjmp.h b/libsa/libsa/i386/setjmp.h
index 0ab263dbf..33232dade 100644
--- a/libsa/libsa/i386/setjmp.h
+++ b/libsa/libsa/i386/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/libsa/libsa/kext.h b/libsa/libsa/kext.h
index f734469df..8c10b1122 100644
--- a/libsa/libsa/kext.h
+++ b/libsa/libsa/kext.h
@@ -5,7 +5,10 @@ extern "C" {
 #include <mach/kern_return.h>
 
 __private_extern__ kern_return_t load_kernel_extension(char * kmod_name);
+__private_extern__ void load_security_extensions (void);
+
 
 #ifdef __cplusplus
 };
 #endif /* __cplusplus */
+
diff --git a/libsa/libsa/malloc.h b/libsa/libsa/malloc.h
index 0f70d2a39..4183eaff7 100644
--- a/libsa/libsa/malloc.h
+++ b/libsa/libsa/malloc.h
@@ -2,7 +2,6 @@
 #define _LIBSA_MALLOC_H_
 
 #include <sys/cdefs.h>
-#include "stdlib.h"
 
 __BEGIN_DECLS
 
diff --git a/libsa/libsa/ppc/setjmp.h b/libsa/libsa/ppc/setjmp.h
index 54d17d51b..a5c0492ae 100644
--- a/libsa/libsa/ppc/setjmp.h
+++ b/libsa/libsa/ppc/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/libsa/libsa/setjmp.h b/libsa/libsa/setjmp.h
index ed021de4e..7c13293ea 100644
--- a/libsa/libsa/setjmp.h
+++ b/libsa/libsa/setjmp.h
@@ -1,37 +1,43 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _LIBSA_SETJMP_H
 #define _LIBSA_SETJMP_H
 
-
 #if defined (__ppc__)
 #include "ppc/setjmp.h"
 #elif defined (__i386__)
 #include "i386/setjmp.h"
+#elif defined (__arm__)
+#include "arm/setjmp.h"
 #else
 #error architecture not supported
 #endif
 
-
 __private_extern__ int setjmp(
         jmp_buf       jmp_buf);
 
diff --git a/libsa/libsa/stdlib.h b/libsa/libsa/stdlib.h
index df136e5d7..304f0807e 100644
--- a/libsa/libsa/stdlib.h
+++ b/libsa/libsa/stdlib.h
@@ -10,11 +10,15 @@ typedef __darwin_size_t	size_t;
 #endif
 
 #ifndef NULL
-#define NULL   (0)
+#if defined (__cplusplus)
+#define NULL 0
+#else
+#define NULL ((void *)0)
+#endif
 #endif
 
 
-__private_extern__ char *kld_basefile_name;
+__private_extern__ const char *kld_basefile_name;
 
 
 __BEGIN_DECLS
@@ -27,6 +31,7 @@ __private_extern__ void   malloc_reset(void); // Destroy all memory regions
 __private_extern__ void * realloc(void * address, size_t new_size);
 
 __private_extern__ char * strrchr(const char *cp, int ch);
+__private_extern__ char * strstr(const char *in, const char *str);
 
 __private_extern__ void qsort(
     void * array,
@@ -34,19 +39,13 @@ __private_extern__ void qsort(
     size_t member_size,
     int (*)(const void *, const void *));
 
-__private_extern__ void * bsearch(
+__private_extern__ const void * bsearch(
     register const void *key,
     const void *base0,
     size_t nmemb,
     register size_t size,
     register int (*compar)(const void *, const void *));
 
-
-/* These are defined in the kernel.
- */
-extern long     strtol(const char *, char **, int);
-extern unsigned long strtoul(const char *, char **, int);
-
 __END_DECLS
 
 #endif /* _LIBSA_STDLIB_H_ */
diff --git a/libsa/load.c b/libsa/load.c
index 2d64905b4..89486e1bb 100644
--- a/libsa/load.c
+++ b/libsa/load.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 /***************
@@ -523,6 +529,13 @@ kload_error __kload_keep_symbols(dgraph_entry_t * entry)
     unsigned long		idx, ncmds;
     vm_size_t	  		size;
     vm_address_t  		mem;
+    struct load_cmds {
+	struct mach_header     hdr;
+	struct segment_command seg;
+	struct symtab_command  symcmd;
+    };
+    struct load_cmds * cmd;
+    unsigned int symtabsize;
 
     if (entry->symbols)
 	return kload_error_none;
@@ -545,14 +558,6 @@ kload_error __kload_keep_symbols(dgraph_entry_t * entry)
 
     symcmd = (struct symtab_command *) seg;
 
-    struct load_cmds {
-	struct mach_header     hdr;
-	struct segment_command seg;
-	struct symtab_command  symcmd;
-    };
-    struct load_cmds * cmd;
-    unsigned int symtabsize;
-
     symtabsize = symcmd->stroff + symcmd->strsize - symcmd->symoff;
 
     size = sizeof(struct load_cmds) + symtabsize;
@@ -578,7 +583,7 @@ kload_error __kload_keep_symbols(dgraph_entry_t * entry)
 
     cmd->seg.cmd 	= LC_SEGMENT;
     cmd->seg.cmdsize 	= sizeof(struct segment_command);
-    strcpy(cmd->seg.segname, SEG_LINKEDIT);
+    strlcpy(cmd->seg.segname, SEG_LINKEDIT, sizeof(cmd->seg.segname));
     cmd->seg.vmaddr 	= 0;
     cmd->seg.vmsize 	= 0;
     cmd->seg.fileoff 	= cmd->symcmd.symoff;
@@ -608,7 +613,7 @@ kload_error __kload_keep_symbols(dgraph_entry_t * entry)
     if (log_level >= kload_log_level_load_details)
     {
 	kload_log_message("__kload_keep_symbols %s, nsyms %ld, 0x%x bytes" KNL, 
-			    entry->name, symcmd->nsyms, size);
+			    entry->name, (unsigned long)symcmd->nsyms, size);
     }
 
     entry->symbols	  = mem;
@@ -743,13 +748,16 @@ kload_error __kload_load_modules(dgraph_t * dgraph
 {
     kload_error result = kload_error_none;
 #ifndef KERNEL
-    long int kernel_size = 0;
+    unsigned long int kernel_size = 0;
     kern_return_t mach_result = KERN_SUCCESS;
+#else
+	const char *kernel_file = "(kernel)";
 #endif /* not KERNEL */
-    char * kernel_base_addr = 0;
+	char *kernel_base_addr = NULL;
     int kld_result;
     Boolean cleanup_kld_loader = false;
     unsigned int i;
+    char opaque_now = false;
 
    /* We have to map all object files to get their CFBundleIdentifier
     * names.
@@ -813,7 +821,6 @@ kload_error __kload_load_modules(dgraph_t * dgraph
     }
 #else /* KERNEL */
 
-    const char * kernel_file = "(kernel)";
     kernel_base_addr = (char *) &_mh_execute_header;
 
 #endif /* not KERNEL */
@@ -844,7 +851,6 @@ kload_error __kload_load_modules(dgraph_t * dgraph
     }
 
     cleanup_kld_loader = true;
-    char opaque_now = false;
 
     for (i = 0; i < dgraph->length; i++) {
         dgraph_entry_t * current_entry = dgraph->load_order[i];
@@ -988,7 +994,7 @@ kload_error __kload_load_modules(dgraph_t * dgraph
                  (interactive_level == 2) ) {
 
                 int approve = (*__kload_approve_func)(1,
-                    "\nStart module %s (ansering no will abort the load)",
+                    "\nStart module %s (answering no will abort the load)",
                     current_entry->name);
 
                 if (approve > 0) {
@@ -1319,10 +1325,10 @@ kload_error __kload_load_module(dgraph_t * dgraph,
     * resident inside the kmod.
     */
     bzero(local_kmod_info->name, sizeof(local_kmod_info->name));
-    strcpy(local_kmod_info->name, entry->expected_kmod_name);
+    strlcpy(local_kmod_info->name, entry->expected_kmod_name, sizeof(local_kmod_info->name));
 
     bzero(local_kmod_info->version, sizeof(local_kmod_info->version));
-    strcpy(local_kmod_info->version, entry->expected_kmod_vers);
+    strlcpy(local_kmod_info->version, entry->expected_kmod_vers, sizeof(local_kmod_info->version));
 
     if (log_level >= kload_log_level_details) {
         kload_log_message("kmod name: %s" KNL, local_kmod_info->name);
@@ -2231,10 +2237,10 @@ kload_error __kload_output_patches(
             }
 
             patch_filename = allocated_filename;
-            strcpy(patch_filename, patch_dir);
-            strcat(patch_filename, "/");
-            strcat(patch_filename, entry->expected_kmod_name);
-            strcat(patch_filename, __KLOAD_PATCH_EXTENSION);
+            strlcpy(patch_filename, patch_dir, length);
+            strlcat(patch_filename, "/", length);
+            strlcat(patch_filename, entry->expected_kmod_name, length);
+            strlcat(patch_filename, __KLOAD_PATCH_EXTENSION, length);
 
             output_patch = 1;
             file_check = kload_file_exists(patch_filename);
@@ -2384,6 +2390,9 @@ kload_error __kload_start_module(dgraph_entry_t * entry) {
 #ifndef KERNEL
     void * kmod_control_args = 0;
     int num_args = 0;
+#elif CONFIG_MACF_KEXT
+    kmod_args_t kmod_args = entry->user_data;
+    mach_msg_type_number_t arg_size = entry->user_data_length;
 #endif /* not KERNEL */
 
     if (!entry->do_load) {
@@ -2394,6 +2403,8 @@ kload_error __kload_start_module(dgraph_entry_t * entry) {
 #ifndef KERNEL
     mach_result = kmod_control(G_kernel_priv_port,
 	    entry->kmod_id, KMOD_CNTL_START, &kmod_control_args, &num_args);
+#elif CONFIG_MACF_KEXT
+    mach_result = kmod_start_or_stop(entry->kmod_id, 1, &kmod_args, &arg_size);
 #else
     mach_result = kmod_start_or_stop(entry->kmod_id, 1, 0, 0);
 #endif /* not KERNEL */
diff --git a/libsa/mach.c b/libsa/mach.c
index 1c04135fe..1507b6a5b 100644
--- a/libsa/mach.c
+++ b/libsa/mach.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libsa/mach/mach.h>
 #include <sys/systm.h>
diff --git a/libsa/mach_loader.h b/libsa/mach_loader.h
index eed913974..7f76237f2 100644
--- a/libsa/mach_loader.h
+++ b/libsa/mach_loader.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Copyright (C) 1992, NeXT, Inc.
@@ -52,6 +58,7 @@ typedef struct _load_result {
     /* boolean_t */	unixproc	:1,
     			dynlinker	:1,
 					:0;
+    unsigned int	csflags;
 } load_result_t;
 
 
diff --git a/libsa/malloc.c b/libsa/malloc.c
index 300583b59..248fd9d94 100644
--- a/libsa/malloc.c
+++ b/libsa/malloc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <string.h>
 
@@ -28,13 +34,13 @@
 #include <kern/kalloc.h>
 #include <kern/lock.h>
 #include <kern/assert.h> 
+#include <kern/zalloc.h>
+#include <kern/debug.h>
 
 #include <vm/vm_kern.h>
 
 #include "libsa/malloc.h"
 
-extern void panic(const char *string, ...);
-
 /*********************************************************************
 * Structure for a client memory block. Contains linked-list pointers,
 * a size field giving the TOTAL size of the block, including this
@@ -49,7 +55,7 @@ typedef struct malloc_block {
 	unsigned int		malSize;
 } malloc_block;
 
-static malloc_block malAnchor = {&malAnchor, &malAnchor, 0, 0};
+static malloc_block malAnchor = {&malAnchor, &malAnchor, NULL, 0};
 
 static int malInited = 0;
 static mutex_t *malloc_lock;
@@ -72,7 +78,7 @@ void * malloc(size_t size) {
 	
 	rmem = (nmem + 15) & -16;					/* Round to 16 byte boundary */
 	amem = (malloc_block *)rmem;				/* Point to the block */
-	amem->malActl = nmem;					/* Set the actual address */
+	amem->malActl = (void *)nmem;					/* Set the actual address */
 	amem->malSize = nsize;						/* Size */
 	
 	mutex_lock(malloc_lock);
@@ -162,6 +168,12 @@ void malloc_reset(void) {
 	mutex_unlock(malloc_lock);				/* Unlock now */
 
 	mutex_free(malloc_lock);
+	
+#ifdef MALLOC_RESET_GC
+	/* Force garbage collection of zones, since we've thrashed through a lot of memory */
+	zone_gc();
+#endif
+
     return;
 
 } /* malloc_reset() */
@@ -194,4 +206,43 @@ void * realloc(void * address, size_t new_client_size) {
 
 } /* realloc() */
 
+#ifdef MALLOC_KLD_VM_ALLOCATE
+#undef vm_allocate
+#undef vm_deallocate
+
+/*
+ * Wrap vm_allocate calls made by kld in malloc/free so that the memory
+ * is all released when we jettison kld.  Make other VM calls used by kld
+ * no-op, since we don't need them.
+ */
+__private_extern__
+kern_return_t vm_allocate(vm_map_t target_task, vm_address_t *address, vm_size_t size, int flags)
+{
+    assert(flags & VM_FLAGS_ANYWHERE);
+    assert(target_task == kernel_map);
+
+    *address = (vm_address_t)malloc(size);
+    bzero(*address, size);
+
+    return KERN_SUCCESS;
+}
 
+__private_extern__
+kern_return_t vm_deallocate(vm_map_t target_task, vm_address_t address, vm_size_t size)
+{
+    free(address);
+    return KERN_SUCCESS;
+}
+
+__private_extern__
+kern_return_t vm_protect(vm_map_t target_task, vm_address_t address, vm_size_t size, boolean_t set_maximum, vm_prot_t new_protection)
+{
+    return KERN_SUCCESS;
+}
+
+__private_extern__
+kern_return_t vm_msync(vm_map_t target_task, vm_address_t address, vm_size_t size, vm_sync_t sync_flags)
+{
+    return KERN_SUCCESS;
+}
+#endif
diff --git a/libsa/misc.c b/libsa/misc.c
index c5b5a99ab..f158145b4 100644
--- a/libsa/misc.c
+++ b/libsa/misc.c
@@ -1,33 +1,38 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <libsa/mach/mach.h>
 #include <sys/systm.h>
 #include <stdarg.h>
 #include <vm/vm_kern.h>
-#include <libsa/stdlib.h>
 
 
 __private_extern__
-char *kld_basefile_name = "(memory-resident kernel)";
+const char *kld_basefile_name = "(memory-resident kernel)";
 
 
 /* from osfmk/kern/printf.c */
diff --git a/libsa/mkext.c b/libsa/mkext.c
index 9747f5382..a5f1096e7 100644
--- a/libsa/mkext.c
+++ b/libsa/mkext.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <string.h>
 #if KERNEL
diff --git a/libsa/ppc/setjmp.s b/libsa/ppc/setjmp.s
index 7177d23c7..9e3c71136 100644
--- a/libsa/ppc/setjmp.s
+++ b/libsa/ppc/setjmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/libsa/sort.c b/libsa/sort.c
index 2ed7ab3f0..4c08ad409 100644
--- a/libsa/sort.c
+++ b/libsa/sort.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-
  * Copyright (c) 1991, 1993
@@ -163,7 +169,7 @@ __private_extern__
 void qsort(void * vbase, size_t nmemb, size_t size,
 	int (*compar)(const void *, const void *)) {
 
-	register int cnt, i, j, l;
+	register unsigned int cnt, i, j, l;
 	register char tmp, *tmp1, *tmp2;
 	char *base, *k, *p, *t;
 
diff --git a/libsa/strrchr.c b/libsa/strrchr.c
index 8c57859e6..d41768813 100644
--- a/libsa/strrchr.c
+++ b/libsa/strrchr.c
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.0 (the 'License').  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License."
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1993
@@ -62,6 +66,8 @@
 /* static char sccsid[] = "@(#)rindex.c	8.1 (Berkeley) 6/4/93"; */
 #endif /* LIBC_SCCS and not lint */
 
+#include <libsa/stdlib.h>
+
 __private_extern__ char * strrchr(const char *cp, int ch)
 {
     char *save;
diff --git a/libsa/strstr.c b/libsa/strstr.c
index b5be864af..f6ca51ecf 100644
--- a/libsa/strstr.c
+++ b/libsa/strstr.c
@@ -1,25 +1,29 @@
 /*
  * Copyright (c) 2001 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * "Portions Copyright (c) 1999 Apple Computer, Inc.  All Rights
- * Reserved.  This file contains Original Code and/or Modifications of
- * Original Code as defined in and that are subject to the Apple Public
- * Source License Version 1.0 (the 'License').  You may not use this file
- * except in compliance with the License.  Please obtain a copy of the
- * License at http://www.apple.com/publicsource and read it before using
- * this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License."
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1988, 1993
diff --git a/libsa/vers_rsrc.c b/libsa/vers_rsrc.c
index 409b71644..f3db0d5d2 100644
--- a/libsa/vers_rsrc.c
+++ b/libsa/vers_rsrc.c
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
 #ifndef KERNEL
 #include <libc.h>
 #include "vers_rsrc.h"
@@ -70,8 +97,8 @@ static int __VERS_isreleasestate(char c) {
 }
 
 
-static VERS_stage __VERS_stage_for_string(char ** string_p) {
-    char * string;
+static VERS_stage __VERS_stage_for_string(const char ** string_p) {
+    const char * string;
 
     if (!string_p || !*string_p) {
         return VERS_invalid;
@@ -121,7 +148,7 @@ static VERS_stage __VERS_stage_for_string(char ** string_p) {
     return VERS_invalid;
 }
 
-static char * __VERS_string_for_stage(VERS_stage stage) {
+static const char * __VERS_string_for_stage(VERS_stage stage) {
     switch (stage) {
       case VERS_invalid:     return "?"; break;
       case VERS_development: return "d"; break;
@@ -144,13 +171,13 @@ VERS_version VERS_parse_string(const char * vers_string) {
     VERS_version vers_revision = 0;
     VERS_version vers_stage = 0;
     VERS_version vers_stage_level = 0;
-    char * current_char_p;
+    const char * current_char_p;
 
     if (!vers_string || *vers_string == '\0') {
         return -1;
     }
 
-    current_char_p = (char *)&vers_string[0];
+    current_char_p = (const char *)&vers_string[0];
 
    /*****
     * Check for an initial digit of the major release number.
@@ -353,7 +380,7 @@ int VERS_string(char * buffer, UInt32 length, VERS_version vers) {
     VERS_version vers_revision = 0;
     VERS_version vers_stage = 0;
     VERS_version vers_stage_level = 0;
-    char * stage_string = NULL;  // don't free
+    const char * stage_string = NULL;  // don't free
 
    /* No buffer or length less than longest possible vers string,
     * return 0.
@@ -365,7 +392,7 @@ int VERS_string(char * buffer, UInt32 length, VERS_version vers) {
     bzero(buffer, length * sizeof(char));
 
     if (vers < 0) {
-        strcpy(buffer, "(invalid)");
+        strlcpy(buffer, "(invalid)", length);
         return 1;
     }
 
@@ -387,30 +414,31 @@ int VERS_string(char * buffer, UInt32 length, VERS_version vers) {
         ( (vers_major * VERS_MAJOR_MULT) + (vers_minor * VERS_MINOR_MULT) +
           (vers_revision * VERS_REVISION_MULT) + (vers_stage * VERS_STAGE_MULT));
 
-    cpos = sprintf(buffer, "%lu", (UInt32)vers_major);
+    cpos = snprintf(buffer, length, "%lu", (UInt32)vers_major);
 
    /* Always include the minor version; it just looks weird without.
     */
     buffer[cpos] = '.';
     cpos++;
-    cpos += sprintf(buffer+cpos, "%lu", (UInt32)vers_minor);
+    cpos += snprintf(buffer+cpos, length - cpos, "%lu", (UInt32)vers_minor);
 
    /* The revision is displayed only if nonzero.
     */
     if (vers_revision) {
         buffer[cpos] = '.';
         cpos++;
-        cpos += sprintf(buffer+cpos, "%lu", (UInt32)vers_revision);
+        cpos += snprintf(buffer+cpos, length - cpos, "%lu",
+			(UInt32)vers_revision);
     }
 
     stage_string = __VERS_string_for_stage(vers_stage);
     if (stage_string && stage_string[0]) {
-        strcat(buffer, stage_string);
+        strlcat(buffer, stage_string, length);
         cpos += strlen(stage_string);
     }
 
     if (vers_stage < VERS_release) {
-        sprintf(buffer+cpos, "%lu", (UInt32)vers_stage_level);
+        snprintf(buffer+cpos, length - cpos, "%lu", (UInt32)vers_stage_level);
     }
 
     return 1;
diff --git a/libsyscall/BSDmakefile b/libsyscall/BSDmakefile
new file mode 100644
index 000000000..699706fb6
--- /dev/null
+++ b/libsyscall/BSDmakefile
@@ -0,0 +1,137 @@
+.ifndef DSTROOT
+DSTROOT != x=`pwd`/DSTROOT && mkdir -p $$x && echo $$x
+.endif
+.ifndef OBJROOT
+OBJROOT != x=`pwd`/OBJROOT && mkdir -p $$x && echo $$x
+.endif
+.ifndef SRCROOT
+SRCROOT != dirname `pwd`
+.endif
+.ifndef SYMROOT
+SYMROOT != x=`pwd`/SYMROOT && mkdir -p $$x && echo $$x
+.endif
+ARCH != arch
+.ifndef RC_ARCHS
+RC_ARCHS = $(ARCH)
+RC_$(RC_ARCHS) = 1
+.endif
+NARCHS != echo $(RC_ARCHS) | wc -w
+LIBSYS = $(NEXT_ROOT)/usr/local/lib/system
+NJOBS != perl -e '$$n = `/usr/sbin/sysctl -n hw.ncpu`; printf "%d\n", $$n < 2 ? 2 : ($$n * 1.5)'
+BSDMAKE = bsdmake -f Makefile
+BSDMAKEJ = $(BSDMAKE) -j $(NJOBS)
+
+# This variables are to guarantee that the left-hand side of an expression is
+# always a variable
+dynamic = dynamic
+static = static
+
+# Remove the arch stuff, since we know better here.
+LOCAL_CFLAGS != echo $(RC_CFLAGS) | sed 's/ *-arch [^ ][^ ]*//g'
+
+FORMS := debug dynamic profile static
+
+all: setup build
+
+build: build-debug build-dynamic build-profile build-static
+
+# These are the non B&I defaults
+.ifndef RC_ProjectName
+install: installhdrs install-all
+
+.else # RC_ProjectName
+
+install: setup
+.for F in $(FORMS)
+install: BI-install-$(F)
+.endfor # FORMS
+install:
+	install -c -m 444 $(OBJROOT)/sys/libsyscall.list $(DSTROOT)/usr/local/lib/system
+.endif # RC_ProjectName
+
+.for F in $(FORMS)
+.if $(dynamic) == $(F)
+SUFFIX$(F) =
+.else
+SUFFIX$(F) = _$(F)
+.endif
+LIPOARGS$(F) != perl -e 'printf "%s\n", join(" ", map(qq(-arch $$_ \"$(OBJROOT)/obj.$$_/libsyscall$(SUFFIX$(F)).a\"), qw($(RC_ARCHS))))'
+
+.for A in $(RC_ARCHS)
+build-$(F): build-$(A)-$(F)
+.endfor # RC_ARCHS
+build-$(F):
+.if $(NARCHS) == 1
+	cp -p "$(OBJROOT)/obj.$(RC_ARCHS)/libsyscall$(SUFFIX$(F)).a" "$(SYMROOT)"
+.else
+	lipo -create $(LIPOARGS$(F)) -output $(SYMROOT)/libsyscall$(SUFFIX$(F)).a
+.endif
+
+.for A in $(RC_ARCHS)
+build-$(A)-$(F):
+	mkdir -p $(OBJROOT)/obj.$(A) && \
+	MAKEOBJDIR="$(OBJROOT)/obj.$(A)" MACHINE_ARCH="$(A)" \
+		DSTROOT='$(DSTROOT)' OBJROOT='$(OBJROOT)' SYMROOT='$(SYMROOT)' \
+		MAKEFLAGS="" CFLAGS="-arch $(A) $(LOCAL_CFLAGS)" $(BSDMAKEJ) libsyscall$(SUFFIX$(F)).a
+.endfor # RC_ARCHS
+.endfor # FORMS
+
+installhdrs:
+	MAKEOBJDIR="$(OBJROOT)" DESTDIR="$(DSTROOT)" MAKEFLAGS="" \
+		DSTROOT='$(DSTROOT)' OBJROOT='$(OBJROOT)' SYMROOT='$(SYMROOT)' \
+		$(BSDMAKE) installhdrs
+.for A in $(RC_ARCHS)
+	mkdir -p "$(OBJROOT)/obj.$(A)" && \
+	MAKEOBJDIR="$(OBJROOT)/obj.$(A)" MACHINE_ARCH="$(A)" \
+		DSTROOT='$(DSTROOT)' OBJROOT='$(OBJROOT)' SYMROOT='$(SYMROOT)' \
+		MAKEFLAGS="" $(BSDMAKE) installhdrs-md
+.endfor # RC_ARCHS
+
+.for F in $(FORMS)
+BI-install-$(F): build-$(F)
+	mkdir -p $(DSTROOT)/usr/local/lib/system
+	if [ -f "$(SYMROOT)/libsyscall$(SUFFIX$(F)).a" ]; then \
+		echo "Installing libsyscall$(SUFFIX$(F)).a" && \
+		install -c -m 444 "$(SYMROOT)/libsyscall$(SUFFIX$(F)).a" \
+			$(DSTROOT)/usr/local/lib/system && \
+		ranlib "$(DSTROOT)/usr/local/lib/system/libsyscall$(SUFFIX$(F)).a"; \
+	fi
+.endfor # FORMS
+
+install-man:
+	mkdir -p $(DSTROOT)/usr/share/man/man2
+	MAKEOBJDIR="$(OBJROOT)" DESTDIR="$(DSTROOT)" \
+		DSTROOT='$(DSTROOT)' OBJROOT='$(OBJROOT)' SYMROOT='$(SYMROOT)' \
+		MACHINE_ARCH="$(ARCH)" MAKEFLAGS="" $(BSDMAKE) all-man maninstall
+
+install-all: setup build install-man
+.for F in $(FORMS)
+install-all: BI-install-$(F)
+.endfor # FORMS
+
+clean:
+.for F in $(FORMS)
+	rm -f $(OBJROOT)/libsyscall$(SUFFIX$(F)).a
+.endfor # FORMS
+.for A in $(RC_ARCHS)
+	rm -rf $(OBJROOT)/obj.$(A)
+.endfor # RC_ARCHS
+
+INCLUDEDIR = $(OBJROOT)/include
+SYSDIR = $(OBJROOT)/sys
+
+setup: $(INCLUDEDIR) $(SYSDIR)
+
+USR-INCLUDE = /usr/include
+MOD-HEADERS = architecture/ppc/mode_independent_asm.h architecture/i386/asm_help.h
+
+$(INCLUDEDIR):
+	mkdir -p $(INCLUDEDIR)
+.for h in $(MOD-HEADERS)
+	mkdir -p $(INCLUDEDIR)/$(h:H)
+	sed 's/\.globl/.private_extern/g' $(USR-INCLUDE)/$(h) > $(INCLUDEDIR)/$(h)
+.endfor # MOD-HEADERS
+
+$(SYSDIR):
+	mkdir -p $(SYSDIR)
+	$(SRCROOT)/libsyscall/create-syscalls.pl $(SRCROOT)/bsd/kern/syscalls.master $(SRCROOT)/libsyscall/custom $(SYSDIR)
diff --git a/libsyscall/GNUmakefile b/libsyscall/GNUmakefile
new file mode 100644
index 000000000..6965e8628
--- /dev/null
+++ b/libsyscall/GNUmakefile
@@ -0,0 +1,8 @@
+# This GNUmakefile is only used when running "make" by-hand; it is not
+# used by buildit or XBS
+
+all:
+	@bsdmake
+
+.DEFAULT:
+	@bsdmake $@
diff --git a/libsyscall/Makefile b/libsyscall/Makefile
new file mode 100644
index 000000000..9b193ef21
--- /dev/null
+++ b/libsyscall/Makefile
@@ -0,0 +1,55 @@
+#	@(#)Makefile	8.2 (Berkeley) 2/3/94
+# $FreeBSD: src/lib/libc/Makefile,v 1.31 2001/08/13 21:48:43 peter Exp $
+#
+# All library objects contain rcsid strings by default; they may be
+# excluded as a space-saving measure.  To produce a library that does
+# not contain these strings, delete -DLIBC_RCS and -DSYSLIBC_RCS
+# from CFLAGS below.  To remove these strings from just the system call
+# stubs, remove just -DSYSLIBC_RCS from CFLAGS.
+#
+# Yes, we build everything with -g, and strip it out later...
+#
+LIB=syscall
+SHLIB_MAJOR= 1
+SHLIB_MINOR= 0
+.if (${MACHINE_ARCH} == unknown)
+MACHINE_ARCH != /usr/bin/arch
+.endif 
+.if !empty $(MACHINE_ARCH:M*64)
+LP64 = 1
+.endif
+CC = gcc
+.ifdef ALTFRAMEWORKSPATH
+PRIVINC = -F${ALTFRAMEWORKSPATH} -I${ALTFRAMEWORKSPATH}/System.framework/PrivateHeaders
+.else
+PRIVINC = -I${NEXT_ROOT}/System/Library/Frameworks/System.framework/PrivateHeaders
+.endif
+CFLAGS += ${PRIVINC}
+CFLAGS += -no-cpp-precomp -force_cpusubtype_ALL
+CFLAGS += -fno-common -pipe -Wmost -g
+AINC= -no-cpp-precomp -force_cpusubtype_ALL
+AINC+= -arch ${MACHINE_ARCH} -g
+CLEANFILES+=tags
+INSTALL_PIC_ARCHIVE=	yes
+PRECIOUSLIB=	yes
+
+# workaround for 3649783
+AINC += -fdollars-in-identifiers
+
+# If these aren't set give it expected defaults
+DESTDIR ?= ${DSTROOT}
+MAKEOBJDIR ?= ${OBJROOT}
+
+# add version string
+SRCS += libsyscall_version.c
+libsyscall_version.c:
+	/Developer/Makefiles/bin/version.pl Libsyscall > $@
+
+CFLAGS += -I${SYMROOT}
+.include "${.CURDIR}/Makefile.inc"
+.PATH: ${SYMROOT}
+.include "Makefile.xbs"
+.if exists(/usr/share/mk/bsd.init.mk)
+.include <bsd.init.mk>
+.endif
+.include <bsd.man.mk>
diff --git a/libsyscall/Makefile.inc b/libsyscall/Makefile.inc
new file mode 100644
index 000000000..0bc95b71c
--- /dev/null
+++ b/libsyscall/Makefile.inc
@@ -0,0 +1,52 @@
+# $FreeBSD: src/lMakefile.inc,v 1.7 2001/04/04 18:17:25 tmm Exp $
+#
+# This file contains make rules that are shared by libc and libc_r.
+#
+# Define (empty) variables so that make doesn't give substitution
+# errors if the included makefiles don't change these:
+MDSRCS=
+MISRCS=
+MDASM=
+MIASM=
+NOASM=
+
+# SUPPRESSSRCS is used to prevent machine-independent files from being
+# built, when a machine-dependent file defines multiple symbols.
+# Use MDSRCS to block one file, and SUPPRESSSRCS to block the others.
+SUPPRESSSRCS=
+
+# set object file suffix
+.if make(lib${LIB}_static.a)
+OBJSUFFIX = o
+.endif
+.if make(lib${LIB}_profile.a)
+OBJSUFFIX = po
+.endif
+.if make(lib${LIB}_debug.a)
+OBJSUFFIX = do
+.endif
+.if make(lib${LIB}.a)
+OBJSUFFIX = So
+.endif
+
+.if exists(${OBJROOT}/sys/Makefile.inc)
+.include "${OBJROOT}/sys/Makefile.inc"
+.endif
+.include "${.CURDIR}/include/Makefile.inc"
+.include "${.CURDIR}/mach/Makefile.inc"
+
+# If there are no machine dependent sources, append all the
+# machine-independent sources:
+.if empty(MDSRCS)
+SRCS+=	${MISRCS}
+.else
+# Append machine-dependent sources, then append machine-independent sources
+# for which there is no machine-dependent variant, and not being suppressed.
+SRCS+=	${MDSRCS}
+_SUPPRESS= ${MDSRCS} ${SUPPRESSSRCS}
+.for _src in ${MISRCS}
+.if ${_SUPPRESS:R:M${_src:R}} == ""
+SRCS+=	${_src}
+.endif
+.endfor
+.endif
diff --git a/libsyscall/Makefile.xbs b/libsyscall/Makefile.xbs
new file mode 100644
index 000000000..62721adbd
--- /dev/null
+++ b/libsyscall/Makefile.xbs
@@ -0,0 +1,128 @@
+BSDMAKE = bsdmake -f Makefile
+
+.PATH: .
+.MAIN: all
+all: lib${LIB}.a lib${LIB}_static.a lib${LIB}_debug.a lib${LIB}_profile.a
+install: installhdrs install_lib${LIB}.a install_lib${LIB}_static.a \
+	 install_lib${LIB}_profile.a install_lib${LIB}_debug.a maninstall
+
+.SUFFIXES:
+.SUFFIXES: .o .po .So .do
+.SUFFIXES: .S .s .c .cc .cpp .cxx .m .C
+.SUFFIXES: .defs .h
+.SUFFIXES: User.c User.o User.po User.So User.do
+.SUFFIXES: Server.c Server.o Server.po Server.So Server.do
+
+OBJS+=  ${SRCS:N*.h:R:S/$/.o/g}
+DOBJS+= ${OBJS:.o=.do}
+POBJS+= ${OBJS:.o=.po} ${STATICOBJS:.o=.po}
+SOBJS+= ${OBJS:.o=.So}
+
+#### Standard C Rules #################################################
+.c.o User.cUser.o Server.cServer.o:
+	${CC} -static ${PRECFLAGS-${.IMPSRC:T}} ${CFLAGS} \
+	    ${CFLAGS-${.IMPSRC:T}} -Os ${OPTIMIZE-${.IMPSRC:T}} \
+	    -c ${.IMPSRC} -o ${.TARGET}
+.c.po User.cUser.po Server.cServer.po:
+	${CC} -pg ${PRECFLAGS-${.IMPSRC:T}} -DPROFILE ${CFLAGS} \
+	    ${CFLAGS-${.IMPSRC:T}} -Os ${OPTIMIZE-${.IMPSRC:T}} \
+	    -c ${.IMPSRC} -o ${.TARGET}
+.c.So User.cUser.So Server.cServer.So: 
+	${CC} ${PRECFLAGS-${.IMPSRC:T}} ${CFLAGS} ${CFLAGS-${.IMPSRC:T}} \
+	    -Os ${OPTIMIZE-${.IMPSRC:T}} -c ${.IMPSRC} -o ${.TARGET}
+.c.do User.cUser.do Server.cServer.do:
+	${CC} -g ${PRECFLAGS-${.IMPSRC:T}} -DDEBUG ${CFLAGS} \
+	    ${CFLAGS-${.IMPSRC:T}} -c ${.IMPSRC} -o ${.TARGET}
+
+#### Standard Assembler Rules #########################################
+.s.o .S.o:
+	${CC} -static -x assembler-with-cpp ${PRECFLAGS-${.IMPSRC:T}} \
+	    ${AINC} ${CFLAGS:M-[BIDFU]*} ${CFLAGS-${.IMPSRC:T}:M-[BIDFU]*} \
+	    -Os ${OPTIMIZE-${.IMPSRC:T}} -c ${.IMPSRC} -o ${.TARGET}
+.s.po .S.po:
+	${CC} -pg -x assembler-with-cpp ${PRECFLAGS-${.IMPSRC:T}} -DPROFILE \
+	    ${AINC} ${CFLAGS:M-[BIDFU]*} ${CFLAGS-${.IMPSRC:T}:M-[BIDFU]*} \
+	    -Os ${OPTIMIZE-${.IMPSRC:T}} -c ${.IMPSRC} -o ${.TARGET}
+.s.So .S.So:
+	${CC} -x assembler-with-cpp ${PRECFLAGS-${.IMPSRC:T}} \
+	    ${AINC} ${CFLAGS:M-[BIDFU]*} ${CFLAGS-${.IMPSRC:T}:M-[BIDFU]*} \
+	    -Os ${OPTIMIZE-${.IMPSRC:T}} -c ${.IMPSRC} -o ${.TARGET}
+.s.do .S.do:
+	${CC} -g -x assembler-with-cpp ${PRECFLAGS-${.IMPSRC:T}} -DDEBUG \
+	    ${AINC} ${CFLAGS:M-[BIDFU]*} ${CFLAGS-${.IMPSRC:T}:M-[BIDFU]*} \
+	    -c ${.IMPSRC} -o ${.TARGET}
+
+#### mig Rules ########################################################
+.defs.h .defsUser.c .defsServer.c:
+	mig -arch ${MACHINE_ARCH} -cc ${CC} -user ${.PREFIX}User.c -server ${.PREFIX}Server.c -header ${.PREFIX}.h ${.IMPSRC}
+
+gen_mig_defs: ${SRVMIGHDRS} ${MIGHDRS}
+gen_md_mig_defs: ${MD_MIGHDRS}
+
+#### Library Rules ####################################################
+lib${LIB}_static.a:: ${OBJS} ${STATICOBJS} 
+	@${ECHO} building static ${LIB} library
+	@rm -f lib${LIB}_static.a
+	@${AR} cq lib${LIB}_static.a `lorder ${OBJS} ${STATICOBJS} | tsort -q` ${ARADD}
+	${RANLIB} lib${LIB}_static.a
+
+lib${LIB}_profile.a:: ${POBJS} ${POBJS2}
+	@${ECHO} building profiled ${LIB} library
+	@rm -f lib${LIB}_profile.a
+	@${AR} cq lib${LIB}_profile.a `lorder ${POBJS} | tsort -q` ${ARADD}
+	${RANLIB} lib${LIB}_profile.a
+
+lib${LIB}_debug.a:: ${DOBJS} ${DOBJS2}
+	@${ECHO} building debug ${LIB} library
+	@rm -f lib${LIB}_debug.a
+	@${AR} cq lib${LIB}_debug.a `lorder ${DOBJS} | tsort -q` ${ARADD}
+	${RANLIB} lib${LIB}_debug.a
+
+lib${LIB}.a:: ${SOBJS} ${SOBJS2}
+	@${ECHO} building standard ${LIB} library
+	@rm -f lib${LIB}.a
+	@${AR} cq lib${LIB}.a `lorder ${SOBJS} | tsort -q` ${ARADD}
+	${RANLIB} lib${LIB}.a
+
+CLEANFILES += ${DOBJS} lib${LIB}_static.a lib${LIB}_profile.a lib${LIB}_debug.a
+
+INCDIR = ${DESTDIR}/usr/include
+LOCINCDIR = ${DESTDIR}/usr/local/include
+SYSTEMFRAMEWORK = ${DESTDIR}/System/Library/Frameworks/System.framework
+PRIVHDRS = ${SYSTEMFRAMEWORK}/Versions/B/PrivateHeaders
+PRIVHDRSPPC = ${PRIVHDRS}/architecture/ppc
+KERNELFRAMEWORK = ${DESTDIR}/System/Library/Frameworks/Kernel.framework
+PRIVKERNELHDRS = ${KERNELFRAMEWORK}/Versions/A/PrivateHeaders
+
+installhdrs-md: gen_md_mig_defs
+	mkdir -p ${INCDIR}/mach/${MACHINE_ARCH}
+	${INSTALL} -o 0 -c -m 444 ${MD_MIGHDRS} ${INCDIR}/mach/${MACHINE_ARCH}
+	mkdir -p ${PRIVHDRSPPC}
+	${INSTALL} -c -m 444 ${PRIVHDRSPPCHDRS} ${PRIVHDRSPPC}
+
+installhdrs: gen_mig_defs
+	mkdir -p ${INCDIR}/mach
+	mkdir -p ${INCDIR}/servers
+	${INSTALL} -o 0 -c -m 444 ${MACH_INSTHDRS} ${INCDIR}/mach
+	${INSTALL} -o 0 -c -m 444 ${SRVHDRS} ${INCDIR}/servers
+	@for i in `find ${DESTDIR}/usr/include/mach ${DESTDIR}/usr/include/servers -name \*.h`; do \
+	    x=`fgrep '<mach/mig.h>' $$i | uniq -d`; \
+	    if [ -n "$$x" ]; then \
+		echo patching $$i; \
+		ed - $$i < ${SRCROOT}/libsyscall/fixdups.ed; \
+	    fi; \
+	done
+
+install_lib${LIB}_static.a:
+	${INSTALL} -c -m 444 lib${LIB}_static.a ${DESTDIR}/usr/local/lib/system/
+install_lib${LIB}_profile.a:
+	${INSTALL} -c -m 444 lib${LIB}_profile.a ${DESTDIR}/usr/local/lib/system
+install_lib${LIB}_debug.a:
+	${INSTALL} -c -m 444 lib${LIB}_debug.a ${DESTDIR}/usr/local/lib/system/
+install_lib${LIB}.a:
+	${INSTALL} -c -m 444 lib${LIB}.a ${DESTDIR}/usr/local/lib/system/
+
+clean:
+	rm -f ${OBJS} ${POBJS} ${DOBJS} ${SOBJS} ${CLEANFILES}
+	rm -f lib${LIB}.a lib${LIB}_static.a lib${LIB}_profile.a \
+		lib${LIB}_debug.a 
diff --git a/libsyscall/create-syscalls.pl b/libsyscall/create-syscalls.pl
new file mode 100755
index 000000000..f54f344ed
--- /dev/null
+++ b/libsyscall/create-syscalls.pl
@@ -0,0 +1,265 @@
+#!/usr/bin/perl
+#
+# Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+#
+# @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+# 
+# This file contains Original Code and/or Modifications of Original Code
+# as defined in and that are subject to the Apple Public Source License
+# Version 2.0 (the 'License'). You may not use this file except in
+# compliance with the License. Please obtain a copy of the License at
+# http://www.opensource.apple.com/apsl/ and read it before using this
+# file.
+# 
+# The Original Code and all software distributed under the License are
+# distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+# EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+# INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+# Please see the License for the specific language governing rights and
+# limitations under the License.
+# 
+# @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+#
+##########################################################################
+#
+# % create-syscalls.pl syscalls.master custom-directory out-directory
+#
+# This script fills the the out-directory with a Makefile.inc and *.s
+# files to create the double-underbar syscall stubs.  It reads the
+# syscall.master file to get the symbol names and number of arguments,
+# and whether Libsystem should automatically create the (non-double-underbar)
+# stubs if Libc doesn't provide a wrapper.  Which system calls will get
+# the automatic treatment is writen to the libsyscall.list file, also
+# written to the out-directory.
+#
+# The custom-directory contains:
+# 1. SYS.h - used by the automatically created *.s and custom files
+# 2. custom.s - contains architecture specific additional system calls and
+#    auxilliary routines (like cerror)
+# 3. special case double-underbar stub files - which are copied into
+#    the out-directory
+#
+# The BSDmakefile copies /usr/include/architecture/ppc/emode_independent_asm.h
+# and /usr/include/architecture/i386/asm_help.h to $(OBJDIR)/include,
+# replacing .globl with .private_extern.  These headers, along with SYS.h
+# make the double-underbar syscall stub private_extern, so that then become
+# static in the resulting libSystem.dylib.
+#
+##########################################################################
+
+use strict;
+use File::Basename ();
+use File::Copy ();
+use File::Spec;
+use IO::File;
+
+my $MyName = File::Basename::basename($0);
+
+my @CustomSrc = qw(custom.s);
+
+my @Copy = (qw(SYS.h), @CustomSrc);
+my $CustomDir;
+my %NoStub;
+my $OutDir;
+my %Stub = (
+    quota => [4, 0],	# unimplemented
+    setquota => [2, 0],	# unimplemented
+    syscall => [0, 0],	# custom/__syscall.s will be used
+);
+my $StubFile = 'libsyscall.list';
+# size in bytes of known types (only used for i386)
+my %TypeBytes = (
+    'caddr_t'		=> 4,
+    'gid_t'		=> 4,
+    'id_t'		=> 4,
+    'idtype_t'		=> 4,
+    'int'		=> 4,
+    'int32_t'		=> 4,
+    'int64_t'		=> 8,
+    'key_t'		=> 4,
+    'long'		=> 4,
+    'mode_t'		=> 4,
+    'off_t'		=> 8,
+    'pid_t'		=> 4,
+    'semun_t'		=> 4,
+    'sigset_t'		=> 4,
+    'size_t'		=> 4,
+    'socklen_t'		=> 4,
+    'ssize_t'		=> 4,
+    'time_t'		=> 4,
+    'u_int'		=> 4,
+    'u_long'		=> 4,
+    'uid_t'		=> 4,
+    'uint32_t'		=> 4,
+    'uint64_t'		=> 8,
+    'user_addr_t'	=> 4,
+    'user_long_t'	=> 4,
+    'user_size_t'	=> 4,
+    'user_ssize_t'	=> 4,
+    'user_ulong_t'	=> 4,
+);
+
+##########################################################################
+# Make a __xxx.s file: if it exists in the $CustomDir, just copy it, otherwise
+# create one.  We define the macro __SYSCALL_I386_ARG_BYTES so that SYS.h could
+# use that to define __SYSCALL dependent on the arguments' total size.
+##########################################################################
+sub make_s {
+    my($name, $args, $bytes) = @_;
+    local $_;
+    my $pseudo = $name;
+    $pseudo = '__' . $pseudo unless $pseudo =~ /^__/;
+    my $file = $pseudo . '.s';
+    my $custom = File::Spec->join($CustomDir, $file);
+    my $path = File::Spec->join($OutDir, $file);
+    if(-f $custom) {
+	File::Copy::copy($custom, $path) || die "$MyName: copy($custom, $path): $!\n";
+	print "Copying $path\n";
+    } else {
+	my $f = IO::File->new($path, 'w');
+	die "$MyName: $path: $!\n" unless defined($f);
+	print $f "#define __SYSCALL_I386_ARG_BYTES $bytes\n\n";
+	print $f "#include \"SYS.h\"\n\n";
+	print $f "__SYSCALL($pseudo, $name, $args)\n";
+	print "Creating $path\n";
+    }
+    return $file;
+}
+
+sub usage {
+    die "Usage: $MyName syscalls.master custom-directory out-directory\n";
+}
+
+##########################################################################
+# Read the syscall.master file and collect the system call names and number
+# of arguments.  It looks for the NO_SYSCALL_STUB quailifier following the
+# prototype to determine if no automatic stub should be created by Libsystem.
+# System call name that are already prefixed with double-underbar are set as
+# if the NO_SYSCALL_STUB qualifier were specified (whether it is or not).
+#
+# For the #if lines in syscall.master, all macros are assumed to be defined,
+# except COMPAT_GETFSSTAT (assumed undefined).
+##########################################################################
+sub readmaster {
+    my $file = shift;
+    local $_;
+    my $f = IO::File->new($file, 'r');
+    die "$MyName: $file: $!\n" unless defined($f);
+    my $line = 0;
+    my $skip = 0;
+    while(<$f>) {
+	$line++;
+	if(/^#\s*endif/) {
+	    $skip = 0;
+	    next;
+	}
+	if(/^#\s*else/) {
+	    $skip = -$skip;
+	    next;
+	}
+	chomp;
+	if(/^#\s*if\s+(\S+)$/) {
+	    $skip = ($1 eq 'COMPAT_GETFSSTAT') ? -1 : 1;
+	    next;
+	}
+	next if $skip < 0;
+	next unless /^\d/;
+	s/^[^{]*{\s*//;
+	s/\s*}.*$//; # }
+	die "$MyName: no function prototype on line $line\n" unless length($_) > 0 && /;$/;
+	my $no_syscall_stub = /\)\s*NO_SYSCALL_STUB\s*;/;
+	my($name, $args) = /\s(\S+)\s*\(([^)]*)\)/;
+	next if $name =~ /e?nosys/;
+	$args =~ s/^\s+//;
+	$args =~ s/\s+$//;
+	my $argbytes = 0;
+	my $nargs = 0;
+	if($args ne '' && $args ne 'void') {
+	    my @a = split(',', $args);
+	    $nargs = scalar(@a);
+	    # Calculate the size of all the arguments (only used for i386)
+	    for my $type (@a) {
+		$type =~ s/\s*\w+$//; # remove the argument name
+		if($type =~ /\*$/) {
+		    $argbytes += 4; # a pointer type
+		} else {
+		    $type =~ s/^.*\s//; # remove any type qualifier, like unsigned
+		    my $b = $TypeBytes{$type};
+		    die "$MyName: $name: unknown type '$type'\n" unless defined($b);
+		    $argbytes += $b;
+		}
+	    }
+	}
+	if($no_syscall_stub || $name =~ /^__/) {
+	    $NoStub{$name} = [$nargs, $argbytes];
+	} else {
+	    $Stub{$name} = [$nargs, $argbytes];
+	}
+    }
+}
+
+usage() unless scalar(@ARGV) == 3;
+$CustomDir = $ARGV[1];
+die "$MyName: $CustomDir: No such directory\n" unless -d $CustomDir;
+$OutDir = $ARGV[2];
+die "$MyName: $OutDir: No such directory\n" unless -d $OutDir;
+
+readmaster($ARGV[0]);
+
+##########################################################################
+# copy the files specified in @Copy from the $CustomDir to $OutDir
+##########################################################################
+for(@Copy) {
+    my $custom = File::Spec->join($CustomDir, $_);
+    my $path = File::Spec->join($OutDir, $_);
+    File::Copy::copy($custom, $path) || die "$MyName: copy($custom, $path): $!\n";
+}
+
+##########################################################################
+# make all the *.s files
+##########################################################################
+my @src;
+my($k, $v);
+while(($k, $v) = each(%Stub)) {
+    push(@src, make_s($k, @$v));
+}
+while(($k, $v) = each(%NoStub)) {
+    push(@src, make_s($k, @$v));
+}
+
+##########################################################################
+# create the Makefile.inc file from the list for files in @src and @CustomSrc
+##########################################################################
+my $path = File::Spec->join($OutDir, 'Makefile.inc');
+my $f = IO::File->new($path, 'w');
+die "$MyName: $path: $!\n" unless defined($f);
+print $f ".PATH: $OutDir\n\n";
+print $f "SYSCALLSRCS= " . join(" \\\n\t", sort(@src, @CustomSrc)) . "\n\n";
+print $f "MDSRCS+= \$(SYSCALLSRCS)\n\n";
+print $f ".for S in \$(SYSCALLSRCS)\n";
+print $f "PRECFLAGS-\$(S)+= -I\$(OBJROOT)/include\n";
+print $f ".endfor\n";
+undef $f;
+
+##########################################################################
+# create the libsyscall.list file for Libsystem to use.  For the one that
+# should not have auto-generated stubs, the line begins with #.
+##########################################################################
+$path = File::Spec->join($OutDir, $StubFile);
+$f = IO::File->new($path, 'w');
+die "$MyName: $path: $!\n" unless defined($f);
+# Add the %NoStub entries to %Stub, appending '#' to the name, so we can sort
+while(($k, $v) = each(%NoStub)) {
+    $k =~ s/^__//;
+    $Stub{"$k#"} = $v;
+}
+for(sort(keys(%Stub))) {
+    $k = $_;
+    if($k =~ s/#$//) {
+	printf $f "#___%s\t%s\n", $k, $Stub{$_}->[0];
+    } else {
+	printf $f "___%s\t%s\n", $_, $Stub{$_}->[0];
+    }
+}
+undef $f;
diff --git a/libsyscall/custom/SYS.h b/libsyscall/custom/SYS.h
new file mode 100644
index 000000000..3f95dbbaf
--- /dev/null
+++ b/libsyscall/custom/SYS.h
@@ -0,0 +1,395 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved.
+ *
+ *	File:	SYS.h
+ *
+ *	Definition of the user side of the UNIX system call interface
+ *	for M98K.
+ *
+ *	Errors are flagged by the location of the trap return (ie., which
+ *	instruction is executed upon rfi):
+ *
+ *		SC PC + 4:	Error (typically branch to cerror())
+ *		SC PC + 8:	Success
+ *
+ * HISTORY
+ * 18-Nov-92	Ben Fathi (benf@next.com)
+ *	Ported to m98k.
+ *
+ *  9-Jan-92	Peter King (king@next.com)
+ *	Created.
+ */
+
+#include <sys/syscall.h>
+
+#ifndef SYS_setquota
+#define SYS_setquota	148
+#endif
+#ifndef SYS_quota
+#define SYS_quota	149
+#endif
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+#include <architecture/ppc/mode_independent_asm.h>
+
+/* From rhapsody kernel mach/ppc/syscall_sw.h */
+#define	kernel_trap_args_0
+#define	kernel_trap_args_1
+#define	kernel_trap_args_2
+#define	kernel_trap_args_3
+#define	kernel_trap_args_4
+#define	kernel_trap_args_5
+#define	kernel_trap_args_6
+#define	kernel_trap_args_7
+#define	kernel_trap_args_8
+/* End of rhapsody kernel mach/ppc/syscall_sw.h */
+
+/*
+ * Macros.
+ */
+
+#define	SYSCALL(name, nargs)			\
+	.globl	cerror				@\
+    MI_ENTRY_POINT(_##name)     @\
+	kernel_trap_args_##nargs    @\
+	li	r0,SYS_##name			@\
+	sc                          @\
+	b	1f                      @\
+	blr                         @\
+1:	MI_BRANCH_EXTERNAL(cerror)
+
+
+#define	SYSCALL_NONAME(name, nargs)		\
+	.globl	cerror				@\
+	kernel_trap_args_##nargs    @\
+	li	r0,SYS_##name			@\
+	sc                          @\
+	b	1f                      @\
+	b	2f                      @\
+1:	MI_BRANCH_EXTERNAL(cerror)  @\
+2:
+
+
+#define	PSEUDO(pseudo, name, nargs)		\
+    .private_extern  _##pseudo           @\
+    .text                       @\
+    .align  2                   @\
+_##pseudo:                      @\
+	SYSCALL_NONAME(name, nargs)
+
+#define __SYSCALL(pseudo, name, nargs)	\
+    PSEUDO(pseudo, name, nargs)	@\
+    blr
+
+#elif defined(__i386__)
+
+#include <architecture/i386/asm_help.h>
+#include <mach/i386/syscall_sw.h>
+
+/*
+ * We have two entry points. int's is used for syscalls which need to preserve
+ * %ecx across the call, or return a 64-bit value in %eax:%edx. sysenter is used
+ * for the majority of syscalls which just return a value in %eax.
+ */
+
+#define UNIX_SYSCALL_SYSENTER		call __sysenter_trap
+#define UNIX_SYSCALL(name, nargs)			\
+	.globl	cerror					;\
+LEAF(_##name, 0)					;\
+	movl	$ SYS_##name, %eax			;\
+	UNIX_SYSCALL_SYSENTER				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)  				;\
+2:
+
+#define UNIX_SYSCALL_INT(name, nargs)			\
+	.globl	cerror					;\
+LEAF(_##name, 0)					;\
+	movl	$ SYS_##name, %eax			;\
+	UNIX_SYSCALL_TRAP				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)  				;\
+2:
+
+#if defined(__SYSCALL_I386_ARG_BYTES) && ((__SYSCALL_I386_ARG_BYTES >= 4) && (__SYSCALL_I386_ARG_BYTES <= 20))
+#define UNIX_SYSCALL_NONAME(name, nargs)			\
+	movl	$(SYS_##name | (__SYSCALL_I386_ARG_BYTES << I386_SYSCALL_ARG_BYTES_SHIFT)), %eax		;\
+	UNIX_SYSCALL_SYSENTER					;\
+	jnb	2f						;\
+	BRANCH_EXTERN(cerror)					;\
+2:
+#else /* __SYSCALL_I386_ARG_BYTES < 4 || > 20 */
+#define UNIX_SYSCALL_NONAME(name, nargs)		\
+	.globl	cerror					;\
+	movl	$ SYS_##name, %eax			;\
+	UNIX_SYSCALL_SYSENTER				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)				;\
+2:
+#endif
+
+#define UNIX_SYSCALL_INT_NONAME(name, nargs)		\
+	.globl	cerror					;\
+	movl	$ SYS_##name, %eax			;\
+	UNIX_SYSCALL_TRAP				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)  				;\
+2:
+
+#define PSEUDO(pseudo, name, nargs)			\
+LEAF(_##pseudo, 0)					;\
+	UNIX_SYSCALL_NONAME(name, nargs)
+
+#define PSEUDO_INT(pseudo, name, nargs)			\
+LEAF(_##pseudo, 0)					;\
+	UNIX_SYSCALL_INT_NONAME(name, nargs)
+
+#define __SYSCALL(pseudo, name, nargs)			\
+	PSEUDO(pseudo, name, nargs)			;\
+	ret
+
+#define __SYSCALL_INT(pseudo, name, nargs)		\
+	PSEUDO_INT(pseudo, name, nargs)			;\
+	ret
+
+#elif defined(__x86_64__)
+
+#include <architecture/i386/asm_help.h>
+#include <mach/i386/syscall_sw.h>
+
+#define UNIX_SYSCALL_SYSCALL	\
+	movq	%rcx, %r10		;\
+	syscall
+
+#define UNIX_SYSCALL(name, nargs)			\
+	.globl	cerror					;\
+LEAF(_##name, 0)					;\
+	movl	$ SYSCALL_CONSTRUCT_UNIX(SYS_##name), %eax	;\
+	UNIX_SYSCALL_SYSCALL				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)  				;\
+2:
+
+#define UNIX_SYSCALL_NONAME(name, nargs)		\
+	.globl	cerror					;\
+	movl	$ SYSCALL_CONSTRUCT_UNIX(SYS_##name), %eax	;\
+	UNIX_SYSCALL_SYSCALL				;\
+	jnb	2f					;\
+	BRANCH_EXTERN(cerror)  				;\
+2:
+
+#define PSEUDO(pseudo, name, nargs)			\
+LEAF(_##pseudo, 0)					;\
+	UNIX_SYSCALL_NONAME(name, nargs)
+
+#define __SYSCALL(pseudo, name, nargs)			\
+	PSEUDO(pseudo, name, nargs)			;\
+	ret
+
+#elif defined(__arm__)
+
+#define SWI_SYSCALL 0x80	// from <mach/vm_param.h>
+
+/*
+ * ARM system call interface:
+ *
+ * swi 0x80
+ * args: r0-r6
+ * return code: r0
+ * on error, carry bit is set in the psr, otherwise carry bit is cleared.
+ */
+
+/*
+ * Macros.
+ */
+
+/*
+ * until we update the architecture project, these live here
+ */
+
+#if defined(__DYNAMIC__)
+#define MI_GET_ADDRESS(reg,var)  \
+	ldr	reg, 4f					;\
+3:	ldr	reg, [pc, reg]				;\
+	b	5f					;\
+4:	.long	6f - (3b + 8)				;\
+5:							;\
+	.non_lazy_symbol_pointer			;\
+6:							;\
+	.indirect_symbol var				;\
+	.long 0						;\
+	.text						;\
+	.align 2
+#else
+#define MI_GET_ADDRESS(reg,var)  \
+	ldr	reg, 3f	;\
+	b	4f	;\
+3:	.long var	;\
+4:
+#endif
+
+#if defined(__DYNAMIC__)
+#define MI_BRANCH_EXTERNAL(var)				\
+	.globl	var								;\
+	MI_GET_ADDRESS(ip, var)				;\
+ 	bx	ip
+#else
+#define MI_BRANCH_EXTERNAL(var)				;\
+	.globl	var								;\
+ 	b	var
+#endif
+
+#if defined(__DYNAMIC__)
+#define MI_CALL_EXTERNAL(var)    \
+	.globl	var				;\
+	MI_GET_ADDRESS(ip,var)	;\
+	mov	lr, pc		;\
+	bx	ip
+#else
+#define MI_CALL_EXTERNAL(var)				\
+	.globl	var								;\
+ 	bl	var
+#endif
+
+#define MI_ENTRY_POINT(name)				\
+	.align 2	;\
+	.globl  name							;\
+	.text									;\
+name:
+
+/* load the syscall number into r12 and trap */
+#define DO_SYSCALL(num)		\
+	.if (((num) & 0xff) == (num)) 	       				;\
+	mov		r12, #(num)		       			;\
+	.elseif (((num) & 0x3fc) == (num))				;\
+	mov		r12, #(num)					;\
+	.else								;\
+	mov		r12, #((num) & 0xffffff00)	/* top half of the syscall number */ ;\
+	orr		r12, r12, #((num) & 0xff)	/* bottom half */ ;\
+	.endif								;\
+	swi		#SWI_SYSCALL
+
+/* simple syscalls (0 to 4 args) */
+#define	SYSCALL_0to4(name)					\
+	MI_ENTRY_POINT(_##name)					;\
+	DO_SYSCALL(SYS_##name)					;\
+	bxcc	lr								/* return if carry is clear (no error) */ ; \
+1:	MI_BRANCH_EXTERNAL(cerror)
+
+/* syscalls with 5 args is different, because of the single arg register load */
+#define	SYSCALL_5(name)						\
+	MI_ENTRY_POINT(_##name)					;\
+	mov		ip, sp							/* save a pointer to the args */ ; \
+	stmfd	sp!, { r4-r5 }					/* save r4-r5 */ ;\
+	ldr		r4, [ip]						/* load 5th arg */ ; \
+	DO_SYSCALL(SYS_##name)					;\
+	ldmfd	sp!, { r4-r5 }					/* restore r4-r5 */ ; \
+	bxcc	lr								/* return if carry is clear (no error) */ ; \
+1:	MI_BRANCH_EXTERNAL(cerror)
+
+/* syscalls with 6 to 8 args */
+#define SYSCALL_6to8(name, save_regs, arg_regs) \
+	MI_ENTRY_POINT(_##name)					;\
+	mov		ip, sp							/* save a pointer to the args */ ; \
+	stmfd	sp!, { save_regs }				/* callee saved regs */ ;\
+	ldmia	ip, { arg_regs }				/* load arg regs */ ; \
+	DO_SYSCALL(SYS_##name)					;\
+	ldmfd	sp!, { save_regs }				/* restore callee saved regs */ ; \
+	bxcc	lr								/* return if carry is clear (no error) */ ; \
+1:	MI_BRANCH_EXTERNAL(cerror)
+
+#define COMMA ,
+
+#define SYSCALL_0(name)						SYSCALL_0to4(name)
+#define SYSCALL_1(name)						SYSCALL_0to4(name)
+#define SYSCALL_2(name)						SYSCALL_0to4(name)
+#define SYSCALL_3(name)						SYSCALL_0to4(name)
+#define SYSCALL_4(name)						SYSCALL_0to4(name)
+/* SYSCALL_5 declared above */
+#define SYSCALL_6(name)						SYSCALL_6to8(name, r4-r5, r4-r5)
+#define SYSCALL_7(name)						SYSCALL_6to8(name, r4-r6 COMMA r8, r4-r6)
+#define SYSCALL_8(name)						SYSCALL_6to8(name, r4-r6 COMMA r8, r4-r6 COMMA r8)
+
+/* select the appropriate syscall code, based on the number of arguments */
+#define SYSCALL(name, nargs)	SYSCALL_##nargs(name)
+
+#define	SYSCALL_NONAME_0to4(name)			\
+	DO_SYSCALL(SYS_##name)					;\
+	bcc		1f								/* branch if carry bit is clear (no error) */ ; \
+	MI_BRANCH_EXTERNAL(cerror)				/* call cerror */ ; \
+1:
+
+#define	SYSCALL_NONAME_5(name)				\
+	mov		ip, sp 							/* save a pointer to the args */ ; \
+	stmfd	sp!, { r4-r5 }					/* save r4-r5 */ ;\
+	ldr		r4, [ip]						/* load 5th arg */ ; \
+	DO_SYSCALL(SYS_##name)					;\
+	ldmfd	sp!, { r4-r5 }					/* restore r4-r7 */ ; \
+	bcc		1f								/* branch if carry bit is clear (no error) */ ; \
+	MI_BRANCH_EXTERNAL(cerror)				/* call cerror */ ; \
+1:
+
+#define	SYSCALL_NONAME_6to8(name, save_regs, arg_regs)	\
+	mov		ip, sp 							/* save a pointer to the args */ ; \
+	stmfd	sp!, { save_regs }				/* callee save regs */ ;\
+	ldmia	ip, { arg_regs }				/* load arguments */ ; \
+	DO_SYSCALL(SYS_##name)					;\
+	ldmfd	sp!, { save_regs }				/* restore callee saved regs */ ; \
+	bcc		1f								/* branch if carry bit is clear (no error) */ ; \
+	MI_BRANCH_EXTERNAL(cerror)				/* call cerror */ ; \
+1:
+
+#define SYSCALL_NONAME_0(name)				SYSCALL_NONAME_0to4(name)
+#define SYSCALL_NONAME_1(name)				SYSCALL_NONAME_0to4(name)
+#define SYSCALL_NONAME_2(name)				SYSCALL_NONAME_0to4(name)
+#define SYSCALL_NONAME_3(name)				SYSCALL_NONAME_0to4(name)
+#define SYSCALL_NONAME_4(name)				SYSCALL_NONAME_0to4(name)
+/* SYSCALL_NONAME_5 declared above */
+#define SYSCALL_NONAME_6(name)				SYSCALL_NONAME_6to8(name, r4-r5, r4-r5)
+#define SYSCALL_NONAME_7(name)				SYSCALL_NONAME_6to8(name, r4-r6 COMMA r8, r4-r6)
+#define SYSCALL_NONAME_8(name)				SYSCALL_NONAME_6to8(name, r4-r6 COMMA r8, r4-r6 COMMA r8)
+
+/* select the appropriate syscall code, based on the number of arguments */
+#define SYSCALL_NONAME(name, nargs)	SYSCALL_NONAME_##nargs(name)
+
+#define	PSEUDO(pseudo, name, nargs)			\
+	.globl	_##pseudo						;\
+	.text									;\
+	.align  2								;\
+_##pseudo:									;\
+	SYSCALL_NONAME(name, nargs)
+
+#define __SYSCALL(pseudo, name, nargs)		\
+	PSEUDO(pseudo, name, nargs)				;\
+	bx lr
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__fork.s b/libsyscall/custom/__fork.s
new file mode 100644
index 000000000..227812ecc
--- /dev/null
+++ b/libsyscall/custom/__fork.s
@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved.
+ *
+ *	File:	libc/ppc/sys/fork.s
+ *
+ * HISTORY
+ * 18-Nov-92  Ben Fathi (benf@next.com)
+ *	Created from M88K sources
+ *
+ * 11-Jan-92  Peter King (king@next.com)
+ *	Created from M68K sources
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+/* We use mode-independent "g" opcodes such as "srgi".  These expand
+ * into word operations when targeting __ppc__, and into doubleword
+ * operations when targeting __ppc64__.
+ */
+#include <architecture/ppc/mode_independent_asm.h>
+
+MI_ENTRY_POINT(___fork)
+    MI_PUSH_STACK_FRAME
+    
+    MI_CALL_EXTERNAL(__cthread_fork_prepare)
+    
+	li      r0,SYS_fork
+	sc                      // do the fork
+	b       Lbotch			// error return
+
+	cmpwi	r4,0            // parent (r4==0) or child (r4==1) ?
+	beq     Lparent         // parent, since r4==0
+
+                            
+/* Here if we are the child.  */
+
+#if defined(__DYNAMIC__)
+    .cstring
+LC3:
+	.ascii	"__dyld_fork_child\0"
+    .text
+	.align 2
+	mflr	r0
+	bcl     20,31,1f
+1:	mflr	r3
+	mtlr	r0
+	addis	r3,r3,ha16(LC3-1b)
+	addi	r3,r3,lo16(LC3-1b)
+	addi 	r4,r1,SF_LOCAL1
+	bl      __dyld_func_lookup
+	lg      r3,SF_LOCAL1(r1)
+	mtspr 	ctr,r3
+	bctrl	
+#endif
+
+    li      r9,0
+    MI_GET_ADDRESS(r8,__current_pid)
+    stw     r9,0(r8)            // clear cached pid in child
+    
+	MI_CALL_EXTERNAL(__cthread_fork_child)
+    
+	li	r3,0        // flag for "we are the child"
+	b	Lreturn
+
+
+/* Here if we are the parent, with:
+ *  r3 = child's pid
+ */
+Lparent:
+	stg     r3,SF_LOCAL2(r1)	// save child pid in stack
+    
+    b       Lparent_return      // clean up and return child's pid
+
+
+/* Here if the fork() syscall failed.  We're still the parent.  */
+
+Lbotch:	
+
+	MI_CALL_EXTERNAL(cerror)
+    li      r3,-1               // get an error return code
+	stg     r3,SF_LOCAL2(r1)	// save return code in stack
+    
+	/*
+	 * We use cthread_fork_parent() to clean up after a fork error
+	 * (unlock cthreads and mailloc packages) so the parent
+	 * process can Malloc() after fork() errors without
+	 * deadlocking.
+	 */
+     
+Lparent_return:
+	MI_CALL_EXTERNAL(__cthread_fork_parent)
+	lg      r3,SF_LOCAL2(r1)    // return -1 on error, child's pid on success
+    
+Lreturn:
+    MI_POP_STACK_FRAME_AND_RETURN
+
+#elif defined(__i386__)
+
+LEAF(___fork, 0)
+	subl  $28, %esp   // Align the stack, with 16 bytes of extra padding that we'll need
+	CALL_EXTERN(__cthread_fork_prepare)
+
+	movl 	$ SYS_fork,%eax; 	// code for fork -> eax
+	UNIX_SYSCALL_TRAP		// do the system call
+	jnc	L1			// jump if CF==0
+
+	CALL_EXTERN(cerror)
+	CALL_EXTERN(__cthread_fork_parent)
+	movl	$-1,%eax
+	addl	$28, %esp   // restore the stack
+	ret
+	
+L1:
+	orl	%edx,%edx	// CF=OF=0,  ZF set if zero result	
+	jz	L2		// parent, since r1 == 0 in parent, 1 in child
+	
+	//child here...
+#if defined(__DYNAMIC__)
+// Here on the child side of the fork we need to tell the dynamic linker that
+// we have forked.  To do this we call __dyld_fork_child in the dyanmic
+// linker.  But since we can't dynamically bind anything until this is done we
+// do this by using the private extern __dyld_func_lookup() function to get the
+// address of __dyld_fork_child (the 'C' code equivlent):
+//
+//	_dyld_func_lookup("__dyld_fork_child", &address);
+//	address();
+//
+.cstring
+LC0:
+	.ascii "__dyld_fork_child\0"
+
+.text
+	leal	0x8(%esp),%eax		// get the address where we're going to store the pointer
+	movl	%eax, 0x4(%esp)		// copy the address of the pointer
+	call	1f
+1:	popl	%eax
+	leal	LC0-1b(%eax),%eax
+	movl 	%eax, 0x0(%esp)		// copy the name of the function to look up
+	call 	__dyld_func_lookup
+	movl	0x8(%esp),%eax		// move the value returned in address parameter
+	call	*%eax		// call __dyld_fork_child indirectly
+#endif
+	xorl	%eax, %eax
+	REG_TO_EXTERN(%eax, __current_pid)
+	CALL_EXTERN(__cthread_fork_child)
+
+	xorl	%eax,%eax	// zero eax
+	addl	$28, %esp   // restore the stack
+	ret
+
+	//parent here...
+L2:
+	movl	%eax, 0xc(%esp)		// save pid
+
+	CALL_EXTERN_AGAIN(__cthread_fork_parent)
+	movl	0xc(%esp), %eax		// return pid
+	addl	$28, %esp   // restore the stack
+	ret		
+
+#elif defined(__x86_64__)
+
+LEAF(___fork, 0)
+	subq  $24, %rsp   // Align the stack, plus room for local storage
+	CALL_EXTERN(__cthread_fork_prepare)
+
+	movl 	$ SYSCALL_CONSTRUCT_UNIX(SYS_fork),%eax; // code for fork -> rax
+	UNIX_SYSCALL_TRAP		// do the system call
+	jnc	L1			// jump if CF==0
+
+	CALL_EXTERN(cerror)
+	CALL_EXTERN(__cthread_fork_parent)
+	movq	$-1, %rax
+	addq	$24, %rsp   // restore the stack
+	ret
+	
+L1:
+	orl	%edx,%edx	// CF=OF=0,  ZF set if zero result	
+	jz	L2		// parent, since r1 == 0 in parent, 1 in child
+	
+	//child here...
+#if defined(__DYNAMIC__)
+// Here on the child side of the fork we need to tell the dynamic linker that
+// we have forked.  To do this we call __dyld_fork_child in the dyanmic
+// linker.  But since we can't dynamically bind anything until this is done we
+// do this by using the private extern __dyld_func_lookup() function to get the
+// address of __dyld_fork_child (the 'C' code equivlent):
+//
+//	_dyld_func_lookup("__dyld_fork_child", &address);
+//	address();
+//
+.cstring
+LC0:
+	.ascii "__dyld_fork_child\0"
+
+.text
+	leaq	8(%rsp),%rsi		// get the address where we're going to store the pointer
+	leaq 	LC0(%rip), %rdi		// copy the name of the function to look up
+	call 	__dyld_func_lookup
+	call	*8(%rsp)		// call __dyld_fork_child indirectly
+#endif
+	xorq	%rax, %rax
+	REG_TO_EXTERN(%rax, __current_pid)
+	CALL_EXTERN(__cthread_fork_child)
+
+	xorq	%rax,%rax	// zero rax
+	addq	$24, %rsp   // restore the stack
+	ret
+
+	//parent here...
+L2:
+	movl	%eax, 16(%rsp)		// save pid
+
+	CALL_EXTERN_AGAIN(__cthread_fork_parent)
+	movl	16(%rsp), %eax		// return pid
+	addq	$24, %rsp   // restore the stack
+	ret		
+
+#elif defined(__arm__)
+	
+	.globl	cerror
+	MI_ENTRY_POINT(_fork)
+	stmfd	sp!, {r4, r7, lr}
+	add	r7, sp, #4
+	MI_CALL_EXTERNAL(__cthread_fork_prepare)
+	mov	r1, #1					// prime results
+	mov	r12, #SYS_fork
+	swi	#SWI_SYSCALL				// make the syscall
+	bcs	Lbotch					// error?
+	cmp	r1, #0					// parent (r1=0) or child(r1=1)
+	beq	Lparent
+
+	//child here...
+	MI_GET_ADDRESS(r3, __current_pid)
+	mov	r0, #0
+	str	r0, [r3]		// clear cached pid in child
+
+#if defined(__DYNAMIC__)
+// Here on the child side of the fork we need to tell the dynamic linker that
+// we have forked.  To do this we call __dyld_fork_child in the dyanmic
+// linker.  But since we can't dynamicly bind anything until this is done we
+// do this by using the private extern __dyld_func_lookup() function to get the
+// address of __dyld_fork_child (the 'C' code equivlent):
+//
+//	_dyld_func_lookup("__dyld_fork_child", &address);
+//	address();
+//
+	.cstring
+	.align 2
+LC0:
+	.ascii "__dyld_fork_child\0"
+.text
+.align 2
+	sub	sp, sp, #4				// allocate space for the address parameter
+	mov	r1, sp					// get the address of the allocated space
+	ldr	r0, LP0					// get the name of the function to look up
+L0:	add	r0, pc, r0
+	bl	__dyld_func_lookup
+	mov	lr, pc
+	ldr	pc, [sp], #4				// call __dyld_fork_child indirectly and pop
+#endif
+	MI_CALL_EXTERNAL(__cthread_fork_child)		// let child get ready
+	mov	r0, #0
+        ldmfd   sp!, {r4, r7, pc}
+
+Lbotch:
+	MI_CALL_EXTERNAL(cerror)			// jump here on error
+	mov	r0,#-1					// set the error
+	// fall thru
+	
+Lparent:	
+	mov	r4, r0					// save child pid
+	MI_CALL_EXTERNAL(__cthread_fork_parent)
+	mov	r0, r4					// restore child pid
+	ldmfd   sp!, {r4, r7, pc}			// pop and return
+
+	.align 2
+#if defined(__DYNAMIC__)
+LP0:
+	.long	LC0-(L0+8)
+#endif
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__getpid.s b/libsyscall/custom/__getpid.s
new file mode 100644
index 000000000..a6cc5a5f4
--- /dev/null
+++ b/libsyscall/custom/__getpid.s
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+        .data
+        .globl  __current_pid
+        .align  2
+__current_pid:
+        .long 0
+
+MI_ENTRY_POINT(___getpid)
+#if defined(__DYNAMIC__)
+        mflr    r0              // note we cannot use MI_GET_ADDRESS...
+        bcl    20,31,1f         // ...because we define __current_pid
+1:
+        mflr    r5
+        mtlr    r0
+        addis   r5, r5, ha16(__current_pid - 1b)
+        addi    r5, r5, lo16(__current_pid - 1b)
+#else
+	lis	r5,hi16(__current_pid)
+	ori	r5,r5,lo16(__current_pid)
+#endif
+        lwz     r3,0(r5)		// get the cached pid
+        cmpwi 	r3,0			// if positive,
+        bgtlr++                 // return it
+	
+        SYSCALL_NONAME(getpid, 0)
+
+        lwarx	r4,0,r5			// see if we can cache it
+        cmpwi	r4,0			// we can't if there are any...
+        blt--	1f              // ...vforks in progress
+
+        stwcx.	r3,0,r5			// ignore cache conflicts
+        blr
+1:
+        li      r6,-4           // on 970, cancel the reservation using red zone...
+        stwcx.  r3,r6,r1        // ...to avoid an errata
+        blr
+
+#elif defined(__i386__)
+
+	.data
+	.private_extern __current_pid
+__current_pid:
+	.long 0
+L__current_pid_addr = __current_pid
+
+#if defined(__DYNAMIC__)
+#define GET_CURRENT_PID				\
+	call	0f				; \
+0:						; \
+	popl	%ecx				; \
+	leal	L__current_pid_addr-0b(%ecx), %ecx
+
+#define __current_pid (%ecx)
+
+#else
+#define GET_CURRENT_PID
+#endif
+
+/*
+ * If __current_pid is > 0, return it, else make syscall.
+ * If __current_pid is 0, cache result of syscall.
+ */
+TEXT
+LEAF(___getpid, 0)
+	GET_CURRENT_PID
+	movl		__current_pid, %eax
+	testl		%eax, %eax
+	jle		1f
+	ret
+1:
+	UNIX_SYSCALL_NONAME(getpid, 0)
+	movl		%eax, %edx
+	xorl		%eax, %eax
+	GET_CURRENT_PID
+	lock
+	cmpxchgl	%edx, __current_pid
+	movl		%edx, %eax
+	ret
+
+#elif defined(__x86_64__)
+
+	.data
+	.private_extern __current_pid
+__current_pid:
+	.long 0
+
+/*
+ * If __current_pid is > 0, return it, else make syscall.
+ * If __current_pid is 0, cache result of syscall.
+ */
+TEXT
+LEAF(___getpid, 0)
+	movl	__current_pid(%rip), %eax
+	testl	%eax, %eax
+	jle		1f
+	ret
+1:
+	UNIX_SYSCALL_NONAME(getpid, 0)
+	movl		%eax, %edx
+	xorl		%eax, %eax
+	leaq		__current_pid(%rip), %rcx
+	lock
+	cmpxchgl	%edx, (%rcx)
+	movl		%edx, %eax
+	ret
+
+#elif defined(__arm__)
+
+#include <arm/arch.h>
+	
+	.data
+	.globl	__current_pid
+	.align 2
+__current_pid:
+	/* Cached pid.  Possible values:
+	 *	0:		no value cached
+	 *	> 0:		cached PID of current process
+	 *	< 0:		negative number of vforks in progress
+	 *	INT_MIN:	for pre-ARMv6, "looking" value (0x80000000)
+	 */
+	.long 0
+
+MI_ENTRY_POINT(_getpid)
+	ldr	r3, L__current_pid
+L1:	add	r3, pc, r3		// r3 = &__current_pid
+	ldr	r0, [r3]		// get the cached pid
+	cmp	r0, #0
+	bxgt	lr			// if positive, return it
+
+	SYSCALL_NONAME(getpid, 0)
+
+#ifdef _ARM_ARCH_6
+	ldrex	r2, [r3]		// see if we can cache it
+	cmp	r2, #0			// we can't if there are any...
+	bxlt	lr			// ...vforks in progress
+	strex	r2, r0, [r3]		// ignore conflicts
+#else
+	mov	r1, #0x80000000		// load "looking" value
+	swp	r2, r1, [r3]		// look at the value, lock others out
+	cmp	r2, r1			// anyone else trying to look?
+	bxeq	lr			// yes, so return immediately/	
+	cmp	r2, #0			// see if we can cache it
+	streq	r0, [r3]		// if zero, we can
+	strne	r2, [r3]		// otherwise restore previous value
+#endif
+		
+	bx	lr
+				
+L__current_pid:	
+	.long	__current_pid - (L1+8)		
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__gettimeofday.s b/libsyscall/custom/__gettimeofday.s
new file mode 100644
index 000000000..aa1a7d1f0
--- /dev/null
+++ b/libsyscall/custom/__gettimeofday.s
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright 1998 Apple Computer, Inc. */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+/* This syscall is special cased: the timeval is returned in r3/r4.
+ * Note also that the "seconds" field of the timeval is a long, so
+ * it's size is mode dependent.
+ */
+MI_ENTRY_POINT(___gettimeofday)
+    mr      r12,r3              // save ptr to timeval
+    SYSCALL_NONAME(gettimeofday,0)
+	stg     r3,0(r12)           // "stw" in 32-bit mode, "std" in 64-bit mode
+	stw     r4,GPR_BYTES(r12)
+	li      r3,0
+	blr
+
+#elif defined(__i386__)
+
+/*
+ *	This syscall is special cased: the timeval is returned in eax/edx.
+ */
+LABEL(___gettimeofday)
+    UNIX_SYSCALL_INT_NONAME(gettimeofday,0)
+	mov		4(%esp),%ecx
+	mov		%eax,(%ecx)
+	mov		%edx,4(%ecx)
+	xor		%eax,%eax
+	ret
+
+#elif defined(__x86_64__)
+
+/*
+ *	This syscall is special cased: the timeval is returned in rax:rdx.
+ */
+LABEL(___gettimeofday)
+    UNIX_SYSCALL_NONAME(gettimeofday,0)
+    movq	%rax, (%rdi)
+    movl	%edx, 8(%rdi)
+    xorl	%eax, %eax
+    ret
+
+#elif defined(__arm__)
+/*
+ * This syscall is special cased: the timeval is returned in r0/r1.
+ */
+MI_ENTRY_POINT(___gettimeofday)
+    mov     r3, r0                  // save ptr to timeval
+    SYSCALL_NONAME(gettimeofday,2)
+    stmia   r3, { r0, r1 }
+    mov     r0, #0
+    bx      lr
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__lseek.s b/libsyscall/custom/__lseek.s
new file mode 100644
index 000000000..b2d9215a3
--- /dev/null
+++ b/libsyscall/custom/__lseek.s
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__) || defined(__x86_64__)
+
+__SYSCALL(__lseek, lseek, 3)
+
+#elif defined(__i386__)
+
+__SYSCALL_INT(__lseek, lseek, 3)
+
+#elif defined(__arm__)
+
+__SYSCALL(__lseek, lseek, 4)
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__pipe.s b/libsyscall/custom/__pipe.s
new file mode 100644
index 000000000..b0eaf9ea2
--- /dev/null
+++ b/libsyscall/custom/__pipe.s
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+MI_ENTRY_POINT(___pipe)
+    mr      r12,r3              // save fildes across syscall
+	SYSCALL_NONAME(pipe, 0)
+	stw     r3,0(r12)
+	stw     r4,4(r12)
+	li      r3,0
+	blr
+
+#elif defined(__i386__)
+
+PSEUDO_INT(__pipe, pipe, 0)
+	movl	4(%esp),%ecx
+	movl	%eax,(%ecx)
+	movl	%edx,4(%ecx)
+	xorl	%eax,%eax
+	ret
+
+#elif defined(__x86_64__)
+
+PSEUDO(__pipe, pipe, 0)
+	movl	%eax, (%rdi)
+	movl	%edx, 4(%rdi)
+	xorl	%eax, %eax
+	ret
+
+#elif defined(__arm__)
+
+MI_ENTRY_POINT(_pipe)
+	mov		r3,r0              // save fildes across syscall
+	SYSCALL_NONAME(pipe, 0)
+	str     r0, [r3, #0]
+	str     r1, [r3, #4]
+	mov		r0,#0
+	bx		lr
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__ptrace.s b/libsyscall/custom/__ptrace.s
new file mode 100644
index 000000000..400b75497
--- /dev/null
+++ b/libsyscall/custom/__ptrace.s
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+MI_ENTRY_POINT(___ptrace)
+	li      r7,0
+    MI_GET_ADDRESS(r8,_errno)
+    stw     r7,0(r8)
+    SYSCALL_NONAME(ptrace, 4)
+	blr
+
+#elif defined(__i386__)
+
+	.globl	_errno
+
+LEAF(___ptrace, 0)
+	xorl	%eax,%eax
+	REG_TO_EXTERN(%eax,_errno)
+UNIX_SYSCALL_NONAME(ptrace, 4)
+	ret
+
+#elif defined(__x86_64__)
+
+	.globl	_errno
+
+LEAF(___ptrace, 0)
+	xorq	%rax,%rax
+	REG_TO_EXTERN(%rax,_errno)
+UNIX_SYSCALL_NONAME(ptrace, 4)
+	ret
+
+#elif defined(__arm__)
+
+MI_ENTRY_POINT(_ptrace)
+	MI_GET_ADDRESS(ip,_errno)
+	str	r8, [sp, #-4]!
+	mov     r8, #0
+	str     r8, [ip]
+	ldr	r8, [sp], #4	
+	SYSCALL_NONAME(ptrace, 4)
+	bx		lr
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__sigaltstack.s b/libsyscall/custom/__sigaltstack.s
new file mode 100644
index 000000000..f4ce94d63
--- /dev/null
+++ b/libsyscall/custom/__sigaltstack.s
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__) || defined(__x86_64__)
+
+__SYSCALL(__sigaltstack, sigaltstack, 3)
+
+#elif defined(__i386__)
+
+__SYSCALL_INT(__sigaltstack, sigaltstack, 3)
+
+#elif defined(__arm__)
+
+__SYSCALL(__sigaltstack, sigaltstack, 3)
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__sigreturn.s b/libsyscall/custom/__sigreturn.s
new file mode 100644
index 000000000..64a4adb57
--- /dev/null
+++ b/libsyscall/custom/__sigreturn.s
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__) || defined(__x86_64__)
+
+__SYSCALL(__sigreturn, sigreturn, 2)
+
+#elif defined(__i386__)
+
+__SYSCALL_INT(__sigreturn, sigreturn, 2)
+
+#elif defined(__arm__)
+
+__SYSCALL(__sigreturn, sigreturn, 2)
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__syscall.s b/libsyscall/custom/__syscall.s
new file mode 100644
index 000000000..88f1f08e0
--- /dev/null
+++ b/libsyscall/custom/__syscall.s
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved. */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+__SYSCALL(__syscall, syscall, 7)
+
+#elif defined(__i386__)
+
+LEAF(___syscall, 0)
+	popl	%ecx		// ret addr
+	popl	%eax		// syscall number
+	pushl	%ecx
+	UNIX_SYSCALL_TRAP
+	movl	(%esp),%edx	// add one element to stack so
+	pushl	%ecx		// caller "pop" will work
+	jnb	2f
+	BRANCH_EXTERN(cerror)
+2:
+END(___syscall)
+
+#elif defined(__x86_64__)
+
+// For x86-64, the kernel slides the argument list for us.
+// The number of arguments here is variable, but our macros ignore
+// that value anyway.
+__SYSCALL(__syscall, syscall, 0);
+
+#elif defined(__arm__)
+
+__SYSCALL(__syscall, syscall, 7)
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/__vfork.s b/libsyscall/custom/__vfork.s
new file mode 100644
index 000000000..c6e559ccd
--- /dev/null
+++ b/libsyscall/custom/__vfork.s
@@ -0,0 +1,246 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
+ *
+ *	File:	libc/ppc/sys/vfork.s
+ *
+ * HISTORY
+ * 23-Jun-1998	Umesh Vaishampayan (umeshv@apple.com)
+ *	Created from fork.s
+ *
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+/* We use mode-independent "g" opcodes such as "srgi", and/or
+ * mode-independent macros such as MI_GET_ADDRESS.  These expand
+ * into word operations when targeting __ppc__, and into doubleword
+ * operations when targeting __ppc64__.
+ */
+#include <architecture/ppc/mode_independent_asm.h>
+
+/* In vfork(), the child runs in parent's address space.  */
+
+
+MI_ENTRY_POINT(___vfork)
+    MI_GET_ADDRESS(r5,__current_pid)  // get address of __current_pid in r5
+2:
+	lwarx	r6,0,r5			// don't cache pid across vfork
+	cmpwi	r6,0
+	ble--	3f              // is another vfork in progress
+	li      r6,0			// if not, erase the stored pid
+3:	
+	addi	r6,r6,-1		// count the parallel vforks in
+	stwcx.	r6,0,r5			// negative cached pid values
+	bne--	2b
+	
+	li      r0,SYS_vfork
+	sc
+	b       Lbotch			// error return
+
+	cmpwi	r4,0
+	beq     Lparent			// parent, since a1 == 0 in parent,
+
+	li      r3,0			// child
+	blr
+
+Lparent:                    // r3 == child's pid
+	lwarx	r6,0,r5			// we're back, decrement vfork count
+	addi	r6,r6,1
+	stwcx.	r6,0,r5
+	bne--	Lparent
+	blr                     // return pid
+
+Lbotch:
+	lwarx	r6,0,r5			// never went, decrement vfork count
+	addi	r6,r6,1
+	stwcx.	r6,0,r5
+	bne--	Lbotch
+
+	MI_BRANCH_EXTERNAL(cerror)
+
+#elif defined(__i386__)
+
+#if defined(__DYNAMIC__)
+#define GET_CURRENT_PID	PICIFY(__current_pid)
+
+        NON_LAZY_STUB(__current_pid)
+#define __current_pid	(%edx)
+#else
+#define GET_CURRENT_PID
+#endif
+
+/*
+ * If __current_pid >= 0, we want to put a -1 in there
+ * otherwise we just decrement it
+ */
+
+LEAF(___vfork, 0)
+	GET_CURRENT_PID
+	movl		__current_pid, %eax
+0:
+	xorl		%ecx, %ecx
+	testl		%eax, %eax
+	cmovs		%eax, %ecx
+	decl		%ecx
+	lock
+	cmpxchgl	%ecx, __current_pid
+	jne		0b
+	popl		%ecx
+	movl		$(SYS_vfork), %eax	// code for vfork -> eax
+	UNIX_SYSCALL_TRAP			// do the system call
+	jnb		L1                     	// jump if CF==0
+	GET_CURRENT_PID
+	lock
+	incl		__current_pid
+	pushl		%ecx
+	BRANCH_EXTERN(cerror)
+
+L1:
+	testl		%edx, %edx		// CF=OF=0,  ZF set if zero result
+	jz		L2			// parent, since r1 == 0 in parent, 1 in child
+	xorl		%eax, %eax		// zero eax
+	jmp		*%ecx
+
+L2:
+	GET_CURRENT_PID
+	lock
+	incl		__current_pid
+	jmp		*%ecx
+
+#elif defined(__x86_64__)
+
+/*
+ * If __current_pid >= 0, we want to put a -1 in there
+ * otherwise we just decrement it
+ */
+
+LEAF(___vfork, 0)
+	movq		__current_pid@GOTPCREL(%rip), %rax
+	movl		(%rax), %eax
+0:
+	xorl		%ecx, %ecx
+	testl		%eax, %eax
+	cmovs		%eax, %ecx
+	subl		$1, %ecx
+	movq		__current_pid@GOTPCREL(%rip), %rdx
+	lock
+	cmpxchgl	%ecx, (%rdx)
+	jne		0b
+	popq		%rdi			// return address in %rdi
+	movq		$ SYSCALL_CONSTRUCT_UNIX(SYS_vfork), %rax	// code for vfork -> rax
+	UNIX_SYSCALL_TRAP			// do the system call
+	jnb		L1					// jump if CF==0
+	movq		__current_pid@GOTPCREL(%rip), %rcx
+	lock
+	addq		$1, (%rcx)
+	movq		(%rcx), %rdi
+	BRANCH_EXTERN(cerror)
+
+L1:
+	testl		%edx, %edx		// CF=OF=0,  ZF set if zero result
+	jz		L2			// parent, since r1 == 0 in parent, 1 in child
+	xorq		%rax, %rax		// zero rax
+	jmp		*%rdi
+
+L2:
+	movq		__current_pid@GOTPCREL(%rip), %rdx
+	lock
+	addq		$1, (%rdx)
+	jmp		*%rdi
+
+#elif defined(__arm__)
+
+#include <arm/arch.h>
+		
+	.globl	cerror
+	MI_ENTRY_POINT(_vfork)
+
+	MI_GET_ADDRESS(r3, __current_pid)	// get address of __current_pid
+#ifdef _ARM_ARCH_6
+L0:	
+	ldrex	r1, [r3]
+	subs	r1, r1, #1			// if __current_pid <= 0, decrement it
+	movpl	r1, #-1				// otherwise put -1 in there
+	strex	r2, r1, [r3]
+	cmp	r2, #0
+	bne	L0
+#else
+	mov	r2, #0x80000000			// load "looking" value
+L0:	
+	swp	r1, r2, [r3]			// look at the value, lock others out
+	cmp	r1, r2				// anyone else trying to look?
+	beq	L0				// yes, so wait our turn
+        subs    r1, r1, #1                      // if __current_pid <= 0, decrement it
+	movpl   r1, #-1                         // otherwise put -1 in there
+	str	r1, [r3]
+#endif
+		
+	mov	r1, #1					// prime results
+	mov	r12, #SYS_vfork
+	swi	#SWI_SYSCALL				// make the syscall
+	bcs	Lbotch					// error?
+	cmp	r1, #0					// parent (r1=0) or child(r1=1)
+	beq	Lparent
+
+	//child here...
+	mov	r0, #0
+	bx	lr					// return
+
+Lbotch:
+	MI_CALL_EXTERNAL(cerror)			// jump here on error
+	mov	r0,#-1					// set the error
+	// reload values clobbered by cerror (so we can treat them as live in Lparent)
+	MI_GET_ADDRESS(r3, __current_pid)		// get address of __current_pid
+#ifndef _ARM_ARCH_6
+	mov	r2, #0x80000000			// load "looking" value
+#endif
+	// fall thru
+	
+Lparent:	
+#ifdef _ARM_ARCH_6
+	ldrex	r1, [r3]
+	add	r1, r1, #1			// we're back, decrement vfork count
+	strex	r2, r1, [r3]
+	cmp	r2, #0
+	bne	Lparent
+#else
+	swp	r1, r2, [r3]			// look at the value, lock others out
+	cmp	r1, r2				// anyone else trying to look?
+	beq	Lparent				// yes, so wait our turn
+	add	r1, r1, #1			// we're back, decrement vfork count
+	str	r1, [r3]
+#endif
+
+	bx	lr					// return
+
+#else
+#error Unsupported architecture
+#endif
diff --git a/libsyscall/custom/custom.s b/libsyscall/custom/custom.s
new file mode 100644
index 000000000..c18fabc0e
--- /dev/null
+++ b/libsyscall/custom/custom.s
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* Copyright (c) 1992 NeXT Computer, Inc.  All rights reserved.
+ */
+
+#include "SYS.h"
+
+#if defined(__ppc__) || defined(__ppc64__)
+
+/* We use mode-independent "g" opcodes such as "srgi", and/or
+ * mode-independent macros such as MI_GET_ADDRESS.  These expand
+ * into word operations when targeting __ppc__, and into doubleword
+ * operations when targeting __ppc64__.
+ */
+#include <architecture/ppc/mode_independent_asm.h>
+
+    .globl  _errno
+
+MI_ENTRY_POINT(cerror)
+    MI_PUSH_STACK_FRAME
+    MI_GET_ADDRESS(r12,_errno)
+    stw     r3,0(r12)               /* save syscall return code in global */
+    MI_CALL_EXTERNAL(_cthread_set_errno_self)
+    li      r3,-1                   /* then bug return value */
+    li      r4,-1                   /* in case we're returning a long-long in 32-bit mode, etc */
+    MI_POP_STACK_FRAME_AND_RETURN
+
+
+    .globl _processor_facilities_used
+    .align 2
+_processor_facilities_used:
+    li	r0,0x7FF3
+    sc
+    blr
+
+#elif defined(__i386__)
+
+	.globl	_errno
+
+LABEL(cerror)
+	REG_TO_EXTERN(%eax, _errno)
+	mov		%esp,%edx
+	andl	$0xfffffff0,%esp
+	subl	$16,%esp
+	movl	%edx,4(%esp)
+	movl	%eax,(%esp)
+	CALL_EXTERN(_cthread_set_errno_self)
+	movl	4(%esp),%esp
+	movl	$-1,%eax
+	movl	$-1,%edx /* in case a 64-bit value is returned */
+	ret
+
+	.private_extern __sysenter_trap
+	ALIGN
+__sysenter_trap:
+	popl %edx
+	movl %esp, %ecx
+	sysenter
+
+#elif defined(__x86_64__)
+
+	.globl	_errno
+
+LABEL(cerror)
+	REG_TO_EXTERN(%rax, _errno)
+	mov		%rsp,%rdx
+	andq	$-16,%rsp
+	subq	$16,%rsp
+	// Preserve the original stack
+	movq	%rdx,(%rsp)
+	movq	%rax,%rdi
+	CALL_EXTERN(_cthread_set_errno_self)
+	// Restore the original stack
+	movq	(%rsp),%rsp
+	movq	$-1,%rax
+	movq	$-1,%rdx /* in case a 128-bit value is returned */
+	ret
+
+#elif defined(__arm__)
+
+	.globl	_errno
+
+MI_ENTRY_POINT(cerror)
+	stmfd	sp!, {r7, lr}
+	mov		r7, sp
+	MI_GET_ADDRESS(r3,_errno)
+	str		r0, [r3]
+	MI_CALL_EXTERNAL(_cthread_set_errno_self)
+	mov		r0, #-1
+	mov		r1, #-1
+	ldmfd	sp!, {r7, pc}
+
+#else
+#error Unsupported architecture
+#endif
+
+#if defined(__i386__) || defined(__x86_64__)
+
+	.globl _i386_get_ldt
+	ALIGN
+_i386_get_ldt:
+	movl    $6,%eax
+	MACHDEP_SYSCALL_TRAP
+	jnb	2f
+	jmp	cerror
+2:	ret
+
+
+	.globl _i386_set_ldt
+	ALIGN
+_i386_set_ldt:
+	movl    $5,%eax
+	MACHDEP_SYSCALL_TRAP
+	jnb	2f
+	jmp	cerror
+2:	ret
+
+#endif
diff --git a/libsyscall/fixdups.ed b/libsyscall/fixdups.ed
new file mode 100644
index 000000000..009f6d917
--- /dev/null
+++ b/libsyscall/fixdups.ed
@@ -0,0 +1,2 @@
+/<mach\/mig\.h>/;.+1,$g/<mach\/mig\.h>/d
+w
diff --git a/libsyscall/include/Makefile.inc b/libsyscall/include/Makefile.inc
new file mode 100644
index 000000000..7bf41dc30
--- /dev/null
+++ b/libsyscall/include/Makefile.inc
@@ -0,0 +1 @@
+PRIVHDRSPPCHDRS += ${.CURDIR}/include/processor_facilities.h
diff --git a/libsyscall/include/processor_facilities.h b/libsyscall/include/processor_facilities.h
new file mode 100644
index 000000000..7ba9747bc
--- /dev/null
+++ b/libsyscall/include/processor_facilities.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/* Does the current CPU have Altivec support? */
+extern int _cpu_has_altivec;
+
+/* What processor facilities is the current thread using? */
+#define floatUsed	0x40000000
+#define vectorUsed	0x20000000
+
+extern int processor_facilities_used(void);
diff --git a/libsyscall/mach/Makefile.inc b/libsyscall/mach/Makefile.inc
new file mode 100644
index 000000000..4efe509ce
--- /dev/null
+++ b/libsyscall/mach/Makefile.inc
@@ -0,0 +1,73 @@
+# machine-dependent mach sources
+.if exists(${.CURDIR}/mach/${MACHINE_ARCH}/Makefile.inc)
+.include "${.CURDIR}/mach/${MACHINE_ARCH}/Makefile.inc"
+.endif
+
+.PATH: ${.CURDIR}/mach
+
+.include "${.CURDIR}/mach/headers/Makefile.inc"
+.include "${.CURDIR}/mach/servers/Makefile.inc"
+
+MD_MIGDEFS += task.defs \
+	thread_act.defs 
+
+MD_MIGHDRS += ${MD_MIGDEFS:.defs=.h}
+
+MIGDEFS += \
+	clock.defs \
+	clock_priv.defs \
+	exc.defs \
+	host_priv.defs \
+	host_security.defs \
+	ledger.defs \
+	lock_set.defs \
+	mach_port.defs \
+	mach_host.defs \
+	mach_vm.defs \
+	processor.defs \
+	processor_set.defs \
+	vm_map.defs
+
+MIGHDRS = ${MIGDEFS:.defs=.h}
+MIGHDRS += clock_reply.h
+MACH_INSTHDRS += ${MIGHDRS}
+
+# These files are generated from the .defs files
+MIGSRCS = ${MIGDEFS:.defs=User.c} ${MD_MIGDEFS:.defs=User.c}
+
+MISRCS += ${MIGSRCS} \
+	bootstrap_ports.c \
+	clock_sleep.c \
+	error_codes.c \
+	excServer.c \
+	excUser.c \
+	exc_catcher.c \
+	exc_catcher_state.c \
+	exc_catcher_state_identity.c \
+	fprintf_stderr.c \
+	mig_allocate.c \
+	mig_deallocate.c \
+	mig_reply_setup.c  \
+	mig_strncpy.c \
+	mach_error.c \
+	mach_error_string.c \
+	mach_init.c \
+	mach_init_libSystem.c \
+	mach_init_ports.c \
+	mach_msg.c \
+	mach_traps.s \
+	ms_thread_switch.c  \
+	notifyUser.c \
+	panic.c  \
+	port_obj.c  \
+	sbrk.c  \
+	semaphore.c  \
+	slot_name.c 
+
+CLEANFILES += ${MIGHDRS} ${MIGSRCS} ${MD_MIGDEFS:.defs=Server.c} \
+		${MIGDEFS:.defs=Server.c} exc.h excUser.c excServer.c \
+		notify.h notifyUser.c notifyServer.c
+
+MAN2 += brk.2
+
+MLINKS += brk.2 sbrk.2
diff --git a/libsyscall/mach/bootstrap_ports.c b/libsyscall/mach/bootstrap_ports.c
new file mode 100644
index 000000000..33399332b
--- /dev/null
+++ b/libsyscall/mach/bootstrap_ports.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/mach.h>
+
+/*
+ * Stub out the old bootstrap_ports() API, as some applications need
+ * it to exist.  We do not publish a prototype for this, and the stub
+ * WILL disappear in a future release.
+ */
+kern_return_t
+bootstrap_ports(
+	mach_port_t	bootstrap,
+	mach_port_t	*priv_host,
+	mach_port_t	*device_master,
+	mach_port_t	*wired_ledger,
+	mach_port_t	*paged_ledger,
+	mach_port_t	*host_security)
+{
+	return KERN_FAILURE;
+}
+
diff --git a/libsyscall/mach/brk.2 b/libsyscall/mach/brk.2
new file mode 100644
index 000000000..9ea4f61c2
--- /dev/null
+++ b/libsyscall/mach/brk.2
@@ -0,0 +1,150 @@
+.\"	$NetBSD: brk.2,v 1.7 1995/02/27 12:31:57 cgd Exp $
+.\"
+.\" Copyright (c) 1980, 1991, 1993
+.\"	The Regents of the University of California.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 3. All advertising materials mentioning features or use of this software
+.\"    must display the following acknowledgement:
+.\"	This product includes software developed by the University of
+.\"	California, Berkeley and its contributors.
+.\" 4. Neither the name of the University nor the names of its contributors
+.\"    may be used to endorse or promote products derived from this software
+.\"    without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"     @(#)brk.2	8.2 (Berkeley) 12/11/93
+.\"
+.Dd December 11, 1993
+.Dt BRK 2
+.Os BSD 4
+.Sh NAME
+.Nm brk ,
+.Nm sbrk
+.Nd change data segment size
+.Sh SYNOPSIS
+.Fd #include <unistd.h>
+.Ft void *
+.Fn brk "const void *addr"
+.Ft void *
+.Fn sbrk "int incr"
+.Sh DESCRIPTION
+.Bf -symbolic
+The brk and sbrk functions are historical curiosities
+left over from earlier days before the advent of virtual memory management.
+.Ef
+The
+.Fn brk
+function
+sets the break or lowest address
+of a process's data segment (uninitialized data) to
+.Fa addr
+(immediately above bss).
+Data addressing is restricted between
+.Fa addr
+and the lowest stack pointer to the stack segment.
+Memory is allocated by
+.Fa brk
+in page size pieces;
+if
+.Fa addr
+is not evenly divisible by the system page size, it is
+increased to the next page boundary.
+.Pp
+.\" The
+.\" .Nm sbrk
+.\" function
+.\" allocates chunks of
+.\" .Fa incr
+.\" bytes
+.\" to the process's data space
+.\" and returns an address pointer.
+.\" The
+.\" .Xr malloc 3
+.\" function utilizes
+.\" .Nm sbrk .
+.\" .Pp
+The current value of the program break is reliably returned by
+.Dq Li sbrk(0)
+(see also 
+.Xr end 3 ) .
+The
+.Xr getrlimit 2
+system call may be used to determine
+the maximum permissible size of the
+.Em data
+segment;
+it will not be possible to set the break
+beyond the
+.Em rlim_max
+value returned from a call to
+.Xr getrlimit ,
+e.g.
+.Dq qetext + rlp\(->rlim_max.
+(see
+.Xr end 3
+for the definition of
+.Em etext ) .
+.Sh RETURN VALUES
+.Nm Brk
+returns a pointer to the new end of memory if successful;
+otherwise -1 with
+.Va errno
+set to indicate why the allocation failed.
+The
+.Nm sbrk
+function returns a pointer to the base of the new storage if successful;
+otherwise -1 with
+.Va errno
+set to indicate why the allocation failed.
+.Sh ERRORS
+.Xr Sbrk
+will fail and no additional memory will be allocated if
+one of the following are true:
+.Bl -tag -width Er
+.It Bq Er ENOMEM
+The limit, as set by
+.Xr setrlimit 2 ,
+was exceeded.
+.It Bq Er ENOMEM
+The maximum possible size of a data segment (compiled into the
+system) was exceeded.
+.It Bq Er ENOMEM
+Insufficient space existed in the swap area
+to support the expansion.
+.El
+.Sh SEE ALSO
+.Xr execve 2 ,
+.Xr getrlimit 2 ,
+.Xr malloc 3 ,
+.Xr mmap 2 ,
+.Xr end 3
+.Sh BUGS
+Setting the break may fail due to a temporary lack of
+swap space.  It is not possible to distinguish this
+from a failure caused by exceeding the maximum size of
+the data segment without consulting 
+.Xr getrlimit .
+.Sh HISTORY
+A
+.Fn brk
+function call appeared in 
+.At v7 .
diff --git a/libsyscall/mach/clock.defs b/libsyscall/mach/clock.defs
new file mode 100644
index 000000000..9d4257b8e
--- /dev/null
+++ b/libsyscall/mach/clock.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/clock.defs>
diff --git a/libsyscall/mach/clock_priv.defs b/libsyscall/mach/clock_priv.defs
new file mode 100644
index 000000000..8595f8296
--- /dev/null
+++ b/libsyscall/mach/clock_priv.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/clock_priv.defs>
diff --git a/libsyscall/mach/clock_reply.defs b/libsyscall/mach/clock_reply.defs
new file mode 100644
index 000000000..6586f2644
--- /dev/null
+++ b/libsyscall/mach/clock_reply.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/clock_reply.defs>
diff --git a/libsyscall/mach/clock_sleep.c b/libsyscall/mach/clock_sleep.c
new file mode 100644
index 000000000..6470f2713
--- /dev/null
+++ b/libsyscall/mach/clock_sleep.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/message.h>
+#include <mach/mach_syscalls.h>
+#include <mach/clock_types.h>
+
+kern_return_t clock_sleep(mach_port_t clock_name,
+                          sleep_type_t clock_type,
+                          mach_timespec_t sleep_time,
+                          mach_timespec_t *wake_time) {
+
+    return clock_sleep_trap(clock_name, clock_type, sleep_time.tv_sec, sleep_time.tv_nsec, wake_time);
+}
diff --git a/libsyscall/mach/err_iokit.sub b/libsyscall/mach/err_iokit.sub
new file mode 100755
index 000000000..2230ecc57
--- /dev/null
+++ b/libsyscall/mach/err_iokit.sub
@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * The contents of this file constitute Original Code as defined in and
+ * are subject to the Apple Public Source License Version 1.1 (the
+ * "License").  You may not use this file except in compliance with the
+ * License.  Please obtain a copy of the License at
+ * http://www.apple.com/publicsource and read it before using this file.
+ * 
+ * This Original Code and all software distributed under the License are
+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *	File: 	err_iokit.sub
+ *	Author:	Godfrey van der Linden, Apple Computer, Inc
+ *	Date:	Sept, 2005
+ *
+ *	error codes for Mach and Unix kernels
+ */
+
+#include <Kernel/IOKit/IOReturn.h>
+#include <IOKit/usb/USB.h>
+#include <IOKit/firewire/IOFireWireLib.h>
+
+static struct error_sparse_map err_codes_iokit_common_map[] = {
+    err_code_map_entry(kIOReturnInvalid, kIOReturnInvalid ),
+    err_code_map_entry(kIOReturnError,   kIOReturnNotFound),
+};
+
+static const char * err_codes_iokit_common[] = {
+    "(iokit/common) invalid - should never be seen",			// 0x001
+
+    "(iokit/common) general error",					// 0x2bc
+    "(iokit/common) can't allocate memory",				// 0x2bd
+    "(iokit/common) resource shortage",					// 0x2be
+    "(iokit/common) error during IPC",					// 0x2bf
+    "(iokit/common) no such device",					// 0x2c0
+    "(iokit/common) privilege violation",				// 0x2c1
+    "(iokit/common) invalid argument",					// 0x2c2
+    "(iokit/common) device read locked",				// 0x2c3
+    "(iokit/common) device write locked",				// 0x2c4
+    "(iokit/common) exclusive access and device already open",		// 0x2c5
+    "(iokit/common) sent/received messages had different msg_id",	// 0x2c6
+    "(iokit/common) unsupported function",				// 0x2c7
+    "(iokit/common) misc. VM failure",					// 0x2c8
+    "(iokit/common) internal error",					// 0x2c9
+    "(iokit/common) General I/O error",					// 0x2ca
+    "(iokit/common) ???",						// 0x2cb
+    "(iokit/common) can't acquire lock",				// 0x2cc
+    "(iokit/common) device not open",					// 0x2cd
+    "(iokit/common) read not supported",				// 0x2ce
+    "(iokit/common) write not supported",				// 0x2cf
+    "(iokit/common) alignment error",					// 0x2d0
+    "(iokit/common) Media Error",					// 0x2d1
+    "(iokit/common) device(s) still open",				// 0x2d2
+    "(iokit/common) rld failure",					// 0x2d3
+    "(iokit/common) DMA failure",					// 0x2d4
+    "(iokit/common) Device Busy",					// 0x2d5
+    "(iokit/common) I/O Timeout",					// 0x2d6
+    "(iokit/common) device offline",					// 0x2d7
+    "(iokit/common) not ready",						// 0x2d8
+    "(iokit/common) device not attached",				// 0x2d9
+    "(iokit/common) no DMA channels left",				// 0x2da
+    "(iokit/common) no space for data",					// 0x2db
+    "(iokit/common) ???",						// 0x2dc
+    "(iokit/common) port already exists",				// 0x2dd
+    "(iokit/common) can't wire down physical memory",			// 0x2de
+    "(iokit/common) no interrupt attached",				// 0x2df
+    "(iokit/common) no DMA frames enqueued",				// 0x2e0
+    "(iokit/common) oversized msg received on interrupt port",		// 0x2e1
+    "(iokit/common) not permitted",					// 0x2e2
+    "(iokit/common) no power to device",				// 0x2e3
+    "(iokit/common) media not present",					// 0x2e4
+    "(iokit/common) media not formatted",				// 0x2e5
+    "(iokit/common) no such mode",					// 0x2e6
+    "(iokit/common) data underrun",					// 0x2e7
+    "(iokit/common) data overrun",					// 0x2e8
+    "(iokit/common) the device is not working properly!",		// 0x2e9
+    "(iokit/common) a completion routine is required",			// 0x2ea
+    "(iokit/common) operation aborted",					// 0x2eb
+    "(iokit/common) bus bandwidth would be exceeded",			// 0x2ec
+    "(iokit/common) device not responding",				// 0x2ed
+    "(iokit/common) isochronous I/O request for distant past!",		// 0x2ee
+    "(iokit/common) isochronous I/O request for distant future",	// 0x2ef
+    "(iokit/common) data was not found",				// 0x2f0
+};
+
+static struct error_sparse_map err_codes_iokit_usb_map[] = {
+    err_code_map_entry(kIOUSBCRCErr, kIOUSBDataToggleErr),
+    err_code_map_entry(kIOUSBPIDCheckErr, kIOUSBWrongPIDErr),
+    err_code_map_entry(kIOUSBReserved1Err, kIOUSBLinkErr),
+    err_code_map_entry(kIOUSBDeviceNotHighSpeed, kIOUSBTransactionTimeout),
+    err_code_map_entry(kIOUSBConfigNotFound,     kIOUSBEndpointNotFound),
+    err_code_map_entry(kIOUSBNotEnoughPowerErr,  kIOUSBUnknownPipeErr),
+};
+
+// error codes with in 0xe0004000
+static const char * err_codes_iokit_usb[] = {
+    "(iokit/usb) USB Controller Error: bad CRC received",		// 0x001
+    "(iokit/usb) USB Controller Error:  bitstuffing",			// 0x002
+    "(iokit/usb) USB Controller Error: Bad data toggle",		// 0x003
+
+    "(iokit/usb) USB Controller Error: PID CRC error",			// 0x006
+    "(iokit/usb) USB Controller Error: Bad or wrong PID",		// 0x007
+
+    "(iokit/usb) Reserved",						// 0x00a
+    "(iokit/usb) Reserved",						// 0x00b
+    "(iokit/usb) USB Controller: Buffer Overrun",			// 0x00c
+    "(iokit/usb) USB Controller: Buffer Underrun ",			// 0x00d
+    "(iokit/usb) Transaction was not sent on the bus",			// 0x00e
+    "(iokit/usb) Transaction was not sent on the bus",			// 0x00f
+    "(iokit/usb) USB Controller: Link Error",				// 0x010
+
+    "(iokit/usb) device is not high speed, ehci driver returns error",	// 0x049
+    "(iokit/usb) synchronous usb request on the workloop thread",	// 0x04a
+    "(iokit/usb) high speed hub error during split transaction",	// 0x04b
+    "(iokit/usb) user land isoch call with unprepared frame-list",	// 0x04c
+    "(iokit/usb) user land isoch call with unprepared data buffer",	// 0x04d
+    "(iokit/usb) interface ref not recognized",				// 0x050
+    "(iokit/usb) pipe has stalled, error needs to be cleared",		// 0x04f
+    "(iokit/usb) transaction has been returned to the caller",		// 0x050
+    "(iokit/usb) transaction timed out",				// 0x051
+
+    "(iokit/usb) configuration not found",				// 0x056
+    "(iokit/usb) endpoint not found",					// 0x057
+
+    "(iokit/usb) not enough power for selected configuration",		// 0x05d
+    "(iokit/usb) not enough pipes in interface",			// 0x05e
+    "(iokit/usb) no async port",					// 0x05f
+    "(iokit/usb) too many pipes",					// 0x060
+    "(iokit/usb) pipe ref not recognized",				// 0x061
+};
+
+static struct error_sparse_map err_codes_iokit_fw_map[] = {
+    err_code_map_entry(kIOConfigNoEntry, kIOFireWireBusReset),
+    err_code_map_entry(kIOFireWireBogusDCLProgram, kIOFireWireCompleting),
+    err_code_map_entry(kIOFWMessageServiceIsRequestingClose, kIOFWMessageTopologyChanged),
+};
+
+// error codes with in 0xe0008000
+static const char * err_codes_iokit_fw[] = {
+    "(iokit/firewire) can't find requested entry in config ROM",	// 001
+    "(iokit/firewire) command pending (internal)",			// 002
+    "(iokit/firewire) DCL callback is final callback (internal)",	// 003
+    "(iokit/firewire) device rom changed, ref is no longer valid",	// 004
+    "(iokit/firewire) kIOFireWireAlreadyRegistered",			// 005
+    "(iokit/firewire) kIOFireWireMultipleTalkers",			// 006
+    "(iokit/firewire) kIOFireWireChannelActive",			// 007
+    "(iokit/firewire) kIOFireWireNoListenerOrTalker",			// 008
+    "(iokit/firewire) kIOFireWireNoChannels",				// 009
+    "(iokit/firewire) requested isoch channel is in use",		// 00A
+    "(iokit/firewire) kIOFireWireSeparateBus",				// 00B
+    "(iokit/firewire) kIOFireWireBadSelfIDs",				// 00C
+    "(iokit/firewire) kIOFireWireLowCableVoltage",			// 00D
+    "(iokit/firewire) kIOFireWireInsufficientPower",			// 00E
+    "(iokit/firewire) all transaction labels are in use",		// 00f
+    "(iokit/firewire) rcode 0, resp_complete",				// 010
+    "(iokit/firewire) rcode 1, reserved",				// 011
+    "(iokit/firewire) rcode 2, reserved",				// 012
+    "(iokit/firewire) rcode 3, reserved",				// 013
+    "(iokit/firewire) rcode 4, resp_conflict_error",			// 014
+    "(iokit/firewire) rcode 5, resp_data_error",			// 015
+    "(iokit/firewire) rcode 6, resp_type_error",			// 016
+    "(iokit/firewire) rcode 7, resp_address_error",			// 017
+    "(iokit/firewire) rcode 8, reserved",				// 018
+    "(iokit/firewire) rcode 9, reserved",				// 019
+    "(iokit/firewire) rcode 10, reserved",				// 01a
+    "(iokit/firewire) rcode 11, reserved",				// 01b
+    "(iokit/firewire) rcode 12, reserved",				// 01c
+    "(iokit/firewire) rcode 13, reserved",				// 01d
+    "(iokit/firewire) rcode 14, reserved",				// 01e
+    "(iokit/firewire) rcode 15, reserved",				// 01f
+    "(iokit/firewire) bus reset occurred, generation does not match",	// 020
+
+    "(iokit/firewire) DCL program contains an illegal DCL",		// 101
+    "(iokit/firewire) kIOFireWireTalkingAndListening",			// 102
+    "(iokit/firewire) system sleep, DCL program stopped",		// 103
+    "(iokit/firewire) command is actively completing (internal)",	// 104
+
+    "(iokit/firewire) device offline, please close device object",	// 7d0
+    "(iokit/firewire) more power may be available",			// 7d1
+    "(iokit/firewire) IOFireWire plane in registry updated",		// 7d2
+};
+
+// error codes with in 0xe0020000
+static const char * err_codes_iokit_bluetooth[] = {
+    "(iokit/bluetooth) unknown error",					// 000
+    "(iokit/bluetooth) interrupted operation, hardware reset",		// 001
+    "(iokit/bluetooth) connection to device already exists",		// 002
+    "(iokit/bluetooth) no HCI controller",				// 003
+    "(iokit/bluetooth) changing power states is unsupported",		// 004
+};
+
+static const struct error_sparse_map err_iokit_sub_map[] = {
+    err_sub_map_entry(sub_iokit_common,          sub_iokit_pmu),
+    err_sub_map_entry(sub_iokit_vendor_specific, sub_iokit_reserved)
+};
+
+#define err_iokit_null_sub	{ "(iokit/?", 0 }
+static struct error_subsystem err_iokit_sub[] =
+{
+    /* 0 */ {
+	"(iokit/common)",			// 0xe0000000
+	errlib_count(err_codes_iokit_common),
+	err_codes_iokit_common,
+	err_codes_iokit_common_map,
+	errlib_count(err_codes_iokit_common_map),
+    },
+    /*  1 */ {
+	"(iokit/usb)",				// 0xe0004000
+	errlib_count(err_codes_iokit_usb),
+	err_codes_iokit_usb,
+	err_codes_iokit_usb_map,
+	errlib_count(err_codes_iokit_usb_map),
+    },
+    /*  2 */ {
+	"(iokit/firewire)",			// 0xe0008000
+	errlib_count(err_codes_iokit_fw),
+	err_codes_iokit_fw,
+	err_codes_iokit_fw_map,
+	errlib_count(err_codes_iokit_fw_map),
+    },
+    /*  3 */ err_iokit_null_sub,		// 0xe000c000
+    /*  4 */ { "(iokit/blkstorage)", 0 },	// 0xe0010000
+    /*  5 */ { "(iokit/graphics)",   0 },	// 0xe0014000
+    /*  6 */ err_iokit_null_sub,		// 0xe0018000
+    /*  7 */ err_iokit_null_sub,		// 0xe001c000
+    /*  8 */ {
+	"(iokit/bluetooth)",			// 0xe0020000
+	errlib_count(err_codes_iokit_bluetooth),
+	err_codes_iokit_bluetooth,
+	NULL, 0,
+    },
+    /*  9 */ { "(iokit/pmu)",        0 },	// 0xe0024000
+    /* -2 */ { "(iokit/vendor)",     0 },	// 0xe0028000
+    /* -1 */ { "(iokit/reserved)",   0 },	// 0xe002c000
+};
+
+
+/* vim: set ft=c ts=8 sw=4: */
diff --git a/libsyscall/mach/err_ipc.sub b/libsyscall/mach/err_ipc.sub
new file mode 100644
index 000000000..f5c0905f4
--- /dev/null
+++ b/libsyscall/mach/err_ipc.sub
@@ -0,0 +1,98 @@
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	err_ipc.sub
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar, 1988
+ *
+ *	Definitions of error strings for original IPC
+ */
+
+static const char * err_codes_send[] = {
+	"(ipc/send) unknown error",		/* -100 */
+	"(ipc/send) invalid memory",		/* -101 */
+	"(ipc/send) invalid port",		/* -102 */
+	"(ipc/send) timed out",			/* -103 */
+	"(ipc/send) unused error",		/* -104 */
+	"(ipc/send) will notify",		/* -105 */
+	"(ipc/send) notify in progress",	/* -106 */	
+	"(ipc/send) kernel refused message",	/* -107 */
+	"(ipc/send) send interrupted",		/* -108 */
+	"(ipc/send) send message too large",	/* -109 */
+	"(ipc/send) send message too small",	/* -110 */
+	"(ipc/send) message size changed while being copied",	/* -111 */
+};
+
+static const char * err_codes_rcv[] = {
+	"(ipc/rcv) unknown error",			/* -200 */
+	"(ipc/rcv) invalid memory",			/* -201 */
+	"(ipc/rcv) invalid port",			/* -202 */
+	"(ipc/rcv) receive timed out",			/* -203 */
+	"(ipc/rcv) message too large",			/* -204 */
+	"(ipc/rcv) no space for message data",		/* -205 */
+	"(ipc/rcv) only sender remaining",		/* -206 */
+	"(ipc/rcv) receive interrupted",		/* -207 */
+	"(ipc/rcv) port receiver changed or port became enabled", /* -208 */
+};
+
+static const char 	* err_codes_mig[] = {
+	"(ipc/mig) type check failure in message interface",	/* 0 (-300) */
+	"(ipc/mig) wrong return message ID",			/* 1 */
+	"(ipc/mig) server detected error",			/* 2 */
+	"(ipc/mig) bad message ID",				/* 3 */
+	"(ipc/mig) server found wrong arguments",		/* 4 */
+	"(ipc/mig) no reply should be sent",			/* 5 */
+	"(ipc/mig) server raised exception",			/* 6 */
+	"(ipc/mig) user specified array not large enough for return info",	/* 7 */
+};
+
+/*	err_ipc subsystems      	*/
+static const struct error_subsystem err_ipc_sub[] = {
+	/* ipc/0; */
+	{
+		"(ipc/send)",
+		errlib_count(err_codes_send),
+		err_codes_send,
+	},
+	/* ipc/1; */
+	{
+		"(ipc/rcv)",
+		errlib_count(err_codes_rcv),
+		err_codes_rcv,
+
+	},
+	/* ipc/2 */
+	{
+		"(ipc/mig)",
+		errlib_count(err_codes_mig),
+		err_codes_mig,
+	},
+};
diff --git a/libsyscall/mach/err_kern.sub b/libsyscall/mach/err_kern.sub
new file mode 100644
index 000000000..f00943599
--- /dev/null
+++ b/libsyscall/mach/err_kern.sub
@@ -0,0 +1,182 @@
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File: 	err_kern.sub
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar, 1988
+ *
+ *	error codes for Mach and Unix kernels
+ */
+
+static const char * err_codes_kern[] = {
+	"(os/kern) successful",			/* 0 */
+	"(os/kern) invalid address",
+	"(os/kern) protection failure",
+	"(os/kern) no space available",
+	"(os/kern) invalid argument",
+	"(os/kern) failure",			/* 5 */
+	"(os/kern) resource shortage",
+	"(os/kern) not receiver",
+	"(os/kern) no access",
+	"(os/kern) memory failure",
+	"(os/kern) memory error",		/* 10 */
+	"(os/kern) already in set",
+	"(os/kern) not in set",
+	"(os/kern) name exists",
+	"(os/kern) aborted",
+	"(os/kern) invalid name",		/* 15 */
+	"(os/kern) invalid task",
+	"(os/kern) invalid right",
+	"(os/kern) invalid value",
+	"(os/kern) urefs overflow",
+	"(os/kern) invalid capability",		/* 20 */
+	"(os/kern) right exists",
+	"(os/kern) invalid host",
+	"(os/kern) memory present",
+	"(os/kern) memory data moved",
+	"(os/kern) memory restart copy",	/* 25 */
+	"(os/kern) invalid processor set",
+	"(os/kern) policy limit",
+	"(os/kern) invalid policy",
+	"(os/kern) invalid object",
+	"(os/kern) already waiting",		/* 30 */
+	"(os/kern) default set",
+	"(os/kern) exception protected",
+	"(os/kern) invalid ledger",
+	"(os/kern) invalid memory control",
+	"(os/kern) invalid security",		/* 35 */
+	"(os/kern) not depressed",
+	"(os/kern) object terminated",
+	"(os/kern) lock set destroyed",
+	"(os/kern) lock unstable",
+	"(os/kern) lock owned by another",	/* 40 */
+	"(os/kern) lock owned by self",
+	"(os/kern) semaphore destroyed",
+	"(os/kern) RPC terminated",
+	"(os/kern) terminate orphan",
+	"(os/kern) let orphan continue",	/* 45 */
+	"(os/kern) service not supported",
+	"(os/kern) remote node down",
+};
+
+static const char * err_codes_unix[] = {
+	NO_SUCH_ERROR,
+	"(os/unix) no rights to object",
+	"(os/unix) file or directory does not exist",
+	"(os/unix) no such process",
+	"(os/unix) interrupted system call",
+	"(os/unix) i/o error",
+	"(os/unix) device does not exist",
+	"(os/unix) argument list is too long",
+	"(os/unix) invalid executable object format",
+	"(os/unix) bad file descriptor number",
+	"(os/unix) no child processes are present",
+	"(os/unix) no more processes are available",
+	"(os/unix) insufficient memory",
+	"(os/unix) access denied",
+	"(os/unix) memory access fault",
+	"(os/unix) block device required for operation",
+	"(os/unix) mount device busy",
+	"(os/unix) file already exists",
+	"(os/unix) cross device link",
+	"(os/unix) device does not exist",
+	"(os/unix) object is not a directory",
+	"(os/unix) object is a directory",
+	"(os/unix) invalid argument",
+	"(os/unix) internal file table overflow",
+	"(os/unix) maximum number of open files reached",
+	"(os/unix) object is not a tty-like device",
+	"(os/unix) executable object is in use",
+	"(os/unix) file is too large",
+	"(os/unix) no space is left on device",
+	"(os/unix) illegal seek attempt",
+	"(os/unix) read-only file system",
+	"(os/unix) too many links",
+	"(os/unix) broken pipe",
+	"(os/unix) argument is too large",
+	"(os/unix) result is out of range",
+	"(os/unix) operation on device would block",
+	"(os/unix) operation is now in progress",
+	"(os/unix) operation is already in progress",	
+	"(os/unix) socket operation attempted on non-socket object",
+	"(os/unix) destination address is required",
+	"(os/unix) message is too long",
+	"(os/unix) protocol type is incorrect for socket",
+	"(os/unix) protocol type is not availaible",
+	"(os/unix) protocol type is not supported",
+	"(os/unix) socket type is not supported",
+	"(os/unix) operation is not supported on sockets",
+	"(os/unix) protocol family is not supported",
+	"(os/unix) address family is not supported by protocol family",
+	"(os/unix) address is already in use",
+	"(os/unix) can't assign requested address",
+	"(os/unix) network is down",
+	"(os/unix) network is unreachable",
+	"(os/unix) network dropped connection on reset",
+	"(os/unix) software aborted connection",
+	"(os/unix) connection reset by peer",
+	"(os/unix) no buffer space is available",
+	"(os/unix) socket is already connected",
+	"(os/unix) socket is not connected",
+	"(os/unix) can't send after socket shutdown",
+	"(os/unix) too many references; can't splice",
+	"(os/unix) connection timed out",
+	"(os/unix) connection was refused",
+	"(os/unix) too many levels of symbolic links",
+	"(os/unix) file name exceeds system maximum limit",
+	"(os/unix) host is down",
+	"(os/unix) there is no route to host",
+	"(os/unix) directory is not empty",
+	"(os/unix) quota on number of processes exceeded",
+	"(os/unix) quota on number of users exceeded",
+	"(os/unix) quota on available disk space exceeded",
+};
+
+static const struct error_subsystem err_os_sub[] = {
+	{
+	"(os/kern)",
+	errlib_count(err_codes_kern),
+	err_codes_kern,
+	},
+	{
+	"(os/?)",
+	0,
+	},
+	{
+	"(os/?)",
+	0,
+	},
+	{
+	"(os/unix)",
+	errlib_count(err_codes_unix),
+	err_codes_unix,
+	},
+};
diff --git a/libsyscall/mach/err_mach_ipc.sub b/libsyscall/mach/err_mach_ipc.sub
new file mode 100644
index 000000000..c9e6a79c9
--- /dev/null
+++ b/libsyscall/mach/err_mach_ipc.sub
@@ -0,0 +1,119 @@
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	err_mach_ipc.sub
+ *	Author:	Richard Draves, Carnegie Mellon University
+ *	Date:	Jan, 1990
+ *
+ *	Error string definitions for the new Mach IPC
+ */
+
+static const char * err_codes_mach_send[] = {
+	/* 0 */	"(ipc/send) no error",
+	/* 1 */	"(ipc/send) send in progress",
+	/* 2 */	"(ipc/send) invalid data",
+	/* 3 */	"(ipc/send) invalid destination port",
+	/* 4 */	"(ipc/send) timed out",
+	/* 5 */	"(ipc/send) will notify",
+	/* 6 */	"(ipc/send) notify in progress",
+	/* 7 */	"(ipc/send) interrupted",
+	/* 8 */	"(ipc/send) msg too small",
+	/* 9 */	"(ipc/send) invalid reply port",
+       /* 10 */	"(ipc/send) invalid port right",
+       /* 11 */	"(ipc/send) invalid notify port",
+       /* 12 */	"(ipc/send) invalid memory",
+       /* 13 */	"(ipc/send) no msg buffer",
+       /* 14 */	"(ipc/send) no notify possible",
+       /* 15 */	"(ipc/send) invalid msg-type",
+       /* 16 */	"(ipc/send) invalid msg-header",
+       /* 17 */ "(ipc/send) invalid msg-trailer",
+       /* 18 */ "(ipc/send) DIPC transport failure",
+       /* 19 */ "(ipc/send) DIPC port migrated",
+       /* 20 */ "(ipc/send) DIPC resend failed",
+       /* 21 */ "(ipc/send) out-of-line buffer too large",
+};
+
+static const char * err_codes_mach_rcv[] = {
+	/* 0 */	"(ipc/rcv) no error",
+	/* 1 */	"(ipc/rcv) receive in progress",
+	/* 2 */	"(ipc/rcv) invalid name",
+	/* 3 */	"(ipc/rcv) timed out",
+	/* 4 */	"(ipc/rcv) msg too large",
+	/* 5 */	"(ipc/rcv) interrupted",
+	/* 6 */	"(ipc/rcv) port changed",
+	/* 7 */	"(ipc/rcv) invalid notify port",
+	/* 8 */	"(ipc/rcv) invalid data",
+	/* 9 */	"(ipc/rcv) port died",
+       /* 10 */	"(ipc/rcv) port in set",
+       /* 11 */	"(ipc/rcv) header error",
+       /* 12 */	"(ipc/rcv) body error",
+       /* 13 */	"(ipc/rcv) invalid scatter list entry",
+       /* 14 */	"(ipc/rcv) overwrite region too small",
+       /* 15 */ "(ipc/rcv) invalid msg-trailer",
+       /* 16 */ "(ipc/rcv) DIPC transport error",
+};
+
+static const char 	* err_codes_mach_mig[] = {
+	/* 0 */	"(ipc/mig) client type check failure",
+	/* 1 */	"(ipc/mig) wrong reply message ID",
+	/* 2 */	"(ipc/mig) server detected error",
+	/* 3 */	"(ipc/mig) bad request message ID",
+	/* 4 */	"(ipc/mig) server type check failure",
+	/* 5 */	"(ipc/mig) no reply should be sent",
+	/* 6 */	"(ipc/mig) server raised exception",
+	/* 7 */	"(ipc/mig) array not large enough",
+	/* 8 */	"(ipc/mig) server died",
+	/* 9 */	"(ipc/mig) unknown trailer format",
+};
+
+/*	err_mach_ipc subsystems      	*/
+static const struct error_subsystem err_mach_ipc_sub[] = {
+	/* ipc/0; */
+	{
+		"(ipc/send)",
+		errlib_count(err_codes_mach_send),
+		err_codes_mach_send,
+	},
+	/* ipc/1; */
+	{
+		"(ipc/rcv)",
+		errlib_count(err_codes_mach_rcv),
+		err_codes_mach_rcv,
+
+	},
+	/* ipc/2 */
+	{
+		"(ipc/mig)",
+		errlib_count(err_codes_mach_mig),
+		err_codes_mach_mig,
+	},
+
+};
diff --git a/libsyscall/mach/err_server.sub b/libsyscall/mach/err_server.sub
new file mode 100644
index 000000000..acac59a0e
--- /dev/null
+++ b/libsyscall/mach/err_server.sub
@@ -0,0 +1,357 @@
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	err_server.sub
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar, 1988
+ *
+ *	Definitions of Servers error strings
+ */
+
+static const char * err_codes_netname[] = {			/* 0 */
+		"(server/netname) name is not yours",
+		"(server/netname) name not checked in",
+		"(server/netname) no such host",
+		"(server/netname) host not found",
+};
+static const char * err_codes_env_mgr[] = {			/* 1 */
+		NO_SUCH_ERROR,
+		"(server/env_mgr) variable not found",
+		"(server/env_mgr) wrong type of variable",
+		"(server/env_mgr) unknown port",
+		"(server/env_mgr) read only environment",
+		"(server/env_mgr) no more connections available",
+		"(server/env_mgr) port table full",
+		"(server/env_mgr) attempting to enter a null port ",
+};
+static const char * err_codes_execd[] = {			/* 2 */
+		NO_SUCH_ERROR,
+		"(server/execd) could not find file to run",
+		"(server/execd) userid or password incorrect",
+		"(server/execd) fork failed",
+};
+static const char * err_codes_netmemory[] = {			/* 3 */
+		"(server/netmemory) successful",
+		"(server/netmemory) invalid argument",
+		"(server/netmemory) resource shortage",
+};
+static const char * err_codes_ufs[] = {				/* 4 */
+		NO_SUCH_ERROR,
+/* XXX		"(server/ufs) invalid port", */
+};
+
+static const char * err_codes_task_master[] = {			/* 5 */
+		"(server/task_master) GENERIC ERROR",
+		"(server/task_master) invalid tm_task port",
+		"(server/task_master) invalid task id",
+		"(server/task_master) invalid kernel port",
+		"(server/task_master) invalid job group",
+		"(server/task_master) invalid action",
+};
+
+static const char * err_codes_ns[] = {				/* 6 */
+		"(server/ns) GENERIC ERROR",
+		"(server/ns) invalid handle",
+		"(server/ns) name not found",
+		"(server/ns) name already exists",
+		"(server/ns) name too long",
+		"(server/ns) path too long",
+		"(server/ns) invalid name",
+		"(server/ns) not a directory",
+		"(server/ns) is a directory",
+		"(server/ns) directory not empty",
+		"(server/ns) infinite retry loop in resolver",
+		"(server/ns) infinite forwarding loop in resolver",
+		"(server/ns) invalid prefix",
+		"(server/ns) prefix table overflow",
+		"(server/ns) bad format for directory",
+		"(server/ns) unknown entry type",
+		"(server/ns) invalid generation",
+		"(server/ns) entry not reserved",
+};
+
+static const char 	* err_codes_io[] = {			/* 7 */
+		"(server/io) GENERIC ERROR",
+		"(server/io) invalid offset",
+		"(server/io) invalid size",
+		"(server/io) invalid mode",
+		"(server/io) invalid strategy",
+		"(server/io) operation rejected under current I/O strategy",
+};
+
+static const char * err_codes_auth[] = {			/* 8 */
+		"(server/auth) GENERIC ERROR",
+		"(server/auth) bad private port",
+		"(server/auth) bad name",
+		"(server/auth) not primary",
+		"(server/auth) bad pauthsword",
+		"(server/auth) bad group",
+		"(server/auth) duplicate id",
+		"(server/auth) duplicate name",
+		"(server/auth) not secondary",
+};
+
+static const char * err_codes_us[] = {				/* 9 */
+		"(server/us) GENERIC ERROR",
+		"(server/us) unknown error",
+		"(server/us) object not found",
+		"(server/us) object exists",
+		"(server/us) object busy",
+		"(server/us) object not started",
+		"(server/us) object dead",
+		"(server/us) invalid args",
+		"(server/us) invalid access",
+		"(server/us) invalid format",
+		"(server/us) invalid buffer size",
+		"(server/us) access denied",
+		"(server/us) resource exhausted",
+		"(server/us) quota exceeded",
+		"(server/us) limit exceeded",
+		"(server/us) not implemented",
+		"(server/us) not supported",
+		"(server/us) hardware error",
+		"(server/us) retry required",
+		"(server/us) not authenticated",
+		"(server/us) exclusive access",
+		"(server/us) timeout",
+		"(server/us) bad reference count",
+		"(server/us) internal error",
+};
+
+static const char * err_codes_sunrpc[] = {			/* 10 */
+		"(server/sunrpc) GENERIC ERROR",
+		"(server/sunrpc) cannot encode arguments",
+		"(server/sunrpc) cannot decode results",
+		"(server/sunrpc) failure in sending call",
+		"(server/sunrpc) failure in receiving result",
+		"(server/sunrpc) call timed out",
+		"(server/sunrpc) rpc versions not compatible",
+		"(server/sunrpc) authentication error",
+		"(server/sunrpc) program not available",
+		"(server/sunrpc) program version mismatched",
+		"(server/sunrpc) procedure unavailable",
+		"(server/sunrpc) decode arguments error",
+		"(server/sunrpc) generic other problem",
+		"(server/sunrpc) unknown host name",
+		"(server/sunrpc) portmapper failed",
+		"(server/sunrpc) remote program not registered",
+		"(server/sunrpc) unspecified error",
+		"(server/sunrpc) unknown protocol",
+};
+
+static const char	* err_codes_machobj[] = {		/* 11 */
+		"(server/object system) GENERIC ERROR",
+		"(server/object system) object not found",
+		"(server/object system) no such operation",
+		"(server/object system) undefined ipc method arguments",
+		"(server/object system) too many arguments to method",
+		"(server/object system) bad ipc message format",
+};
+
+static const char 	* err_codes_loader[] = {		/* 12 */
+		"(server/loader) GENERIC ERROR",
+		"(server/loader) object file not relocated",
+		"(server/loader) unknown file type",
+		"(server/loader) symbol not found",
+		"(server/loader) symbol multiply defined",
+		"(server/loader) memory region overlap",
+};
+
+
+static const char	* err_codes_exception[] = {		/* 13 */
+	"(server/exception) GENERIC ERROR",
+	"(server/exception) invalid access",
+	"(server/exception) invalid instruction",
+	"(server/exception) arithmetic exception",
+	"(server/exception) emulation exception",
+	"(server/exception) software exception",
+	"(server/exception) breakpoint exception",
+};
+
+static const char	* err_codes_ux_signal[] = {		/* 14 */
+	"(server/unix-signal) GENERIC ERROR",
+	"(server/unix-signal) hangup",
+	"(server/unix-signal) interrupt",
+	"(server/unix-signal) quit",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) kill",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) undefined",
+	"(server/unix-signal) system error",
+	"(server/unix-signal) pipe signal",
+	"(server/unix-signal) alarm",
+	"(server/unix-signal) terminate",
+	"(server/unix-signal) urgent i/o",
+	"(server/unix-signal) stop",
+	"(server/unix-signal) terminal stop",
+	"(server/unix-signal) continue",
+	"(server/unix-signal) child death",
+	"(server/unix-signal) tty input",
+	"(server/unix-signal) tty output",
+	"(server/unix-signal) i/o signal",
+	"(server/unix-signal) cpu time limit exceeded",
+	"(server/unix-signal) file size exceeded",
+	"(server/unix-signal) virtual alarm",
+	"(server/unix-signal) profile signal",
+	"(server/unix-signal) window size change",
+	"(server/unix-signal) user-defined signal 1",
+	"(server/unix-signal) user-defined signal 2",
+};
+
+static const char	* err_codes_xkernel[] = {		/* 15 */
+	"(server/xkernel) GENERIC ERROR",
+	"(server/xkernel) map full",
+	"(server/xkernel) inconsistent bind",
+	"(server/xkernel) cannot resolve",
+	"(server/xkernel) cannot unbind",
+	"(server/xkernel) invalid type",
+	"(server/xkernel) invalid opcode",
+	"(server/xkernel) buffer too small",
+	"(server/xkernel) invalid ev code",
+	"(server/xkernel) event not registered",
+	"(server/xkernel) invalid open",
+	"(server/xkernel) already open",
+	"(server/xkernel) bad addr",
+};
+
+
+/*	err_server subsystems      	*/
+static const struct error_subsystem err_server_sub[] = {
+	/* server/0; */
+	{
+		"(server/netname)",
+		errlib_count(err_codes_netname),
+		err_codes_netname,
+	},
+	/* server/1; */
+	{
+		"(server/env_mgr)",
+		errlib_count(err_codes_env_mgr),
+		err_codes_env_mgr,
+	},
+	/* server/2; */
+	{
+		"(server/execd)",
+		errlib_count(err_codes_execd),
+		err_codes_execd,
+	},
+	/* server/3; */
+	{
+		"(server/netmemory)",
+		errlib_count(err_codes_netmemory),
+		err_codes_netmemory,
+	},
+	/* server/4; */
+	{
+		"(server/ufs)",
+		errlib_count(err_codes_ufs),
+		err_codes_ufs,
+	},
+	/* server/5; */
+	{
+		"(server/task_master)",
+		errlib_count(err_codes_task_master),
+		err_codes_task_master,
+	},
+	/* server/6; */
+	{
+		"(server/ns)",
+		errlib_count(err_codes_ns),
+		err_codes_ns,
+	},
+
+	/* server/7; i/o subsystem */
+	{
+		"(server/io)",
+		errlib_count(err_codes_io),
+		err_codes_io,
+	},
+
+	/* server/8; authentication server */
+	{
+		"(server/auth)",
+		errlib_count(err_codes_auth),
+		err_codes_auth,
+	},
+
+	/* server/9; generic US system */
+	{
+		"(server/us)",
+		errlib_count(err_codes_us),
+		err_codes_us,
+	},
+
+	/* server/10; SUN RPC package */
+	{
+		"(server/sunrpc)",
+		errlib_count(err_codes_sunrpc),
+		err_codes_sunrpc,
+	},
+
+	/* server/11; MachObject system */
+	{
+		"(server/object system)",
+		errlib_count(err_codes_machobj),
+		err_codes_machobj,
+	},
+
+	/* server/12; loader */
+	{
+		"(server/loader)",
+		errlib_count(err_codes_loader),
+		err_codes_loader,
+	},
+
+	/* server/13; mach exception */
+	{
+		"(server/exception)",
+		errlib_count(err_codes_exception),
+		err_codes_exception,
+	},
+
+	/* server/14; unix signal */
+	{
+		"(server/unix-signal)",
+		errlib_count(err_codes_ux_signal),
+		err_codes_ux_signal,
+	},
+
+	/* server/15; xkernel */
+	{
+		"(server/xkernel)",
+		errlib_count(err_codes_xkernel),
+		err_codes_xkernel,
+	},
+
+};
diff --git a/libsyscall/mach/err_us.sub b/libsyscall/mach/err_us.sub
new file mode 100644
index 000000000..121fd38d8
--- /dev/null
+++ b/libsyscall/mach/err_us.sub
@@ -0,0 +1,45 @@
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	err_us.sub
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar, 1988
+ *
+ *	A place to define User errors
+ */
+
+
+/*	err_us subsystems      	*/
+static const struct error_subsystem err_us_sub[] = {
+	{
+		NULL,
+	},
+};
diff --git a/libsyscall/mach/error_codes.c b/libsyscall/mach/error_codes.c
new file mode 100644
index 000000000..4fdf8306c
--- /dev/null
+++ b/libsyscall/mach/error_codes.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	error_codes.c
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar, 1988
+ *
+ *      Generic error code interface
+ */
+
+#include <mach/error.h>
+#include "errorlib.h"
+#include "err_iokit.sub"
+#include "err_ipc.sub"
+#include "err_kern.sub"
+#include "err_mach_ipc.sub"
+#include "err_server.sub"
+#include "err_us.sub"
+
+__private_extern__ struct error_system _mach_errors[err_max_system+1] = {
+	/* 0; err_kern */
+	{
+		errlib_count(err_os_sub),
+		"(operating system/?) unknown subsystem error",
+		err_os_sub,
+	},
+	/* 1; err_us */
+	{
+		errlib_count(err_us_sub),
+		"(user space/?) unknown subsystem error",
+		err_us_sub,
+	},
+	/* 2; err_server */
+	{
+		errlib_count(err_server_sub),
+		"(server/?) unknown subsystem error",
+		err_server_sub,
+	},
+	/* 3 (& 3f); err_ipc */
+	{
+		errlib_count(err_ipc_sub),
+		"(ipc/?) unknown subsystem error",
+		err_ipc_sub,
+	},
+	/* 4; err_mach_ipc */
+	{
+		errlib_count(err_mach_ipc_sub),
+		"(ipc/?) unknown subsystem error",
+		err_mach_ipc_sub,
+	},
+
+	/* 0x05 */ errorlib_system_null,
+	/* 0x06 */ errorlib_system_null, /* 0x07 */ errorlib_system_null,
+	/* 0x08 */ errorlib_system_null, /* 0x09 */ errorlib_system_null,
+	/* 0x0a */ errorlib_system_null, /* 0x0b */ errorlib_system_null,
+	/* 0x0c */ errorlib_system_null, /* 0x0d */ errorlib_system_null,
+	/* 0x0e */ errorlib_system_null, /* 0x0f */ errorlib_system_null,
+
+	/* 0x10 */ errorlib_system_null, /* 0x11 */ errorlib_system_null,
+	/* 0x12 */ errorlib_system_null, /* 0x13 */ errorlib_system_null,
+	/* 0x14 */ errorlib_system_null, /* 0x15 */ errorlib_system_null,
+	/* 0x16 */ errorlib_system_null, /* 0x17 */ errorlib_system_null,
+	/* 0x18 */ errorlib_system_null, /* 0x19 */ errorlib_system_null,
+	/* 0x1a */ errorlib_system_null, /* 0x1b */ errorlib_system_null,
+	/* 0x1c */ errorlib_system_null, /* 0x1d */ errorlib_system_null,
+	/* 0x1e */ errorlib_system_null, /* 0x1f */ errorlib_system_null,
+
+	/* 0x20 */ errorlib_system_null, /* 0x21 */ errorlib_system_null,
+	/* 0x22 */ errorlib_system_null, /* 0x23 */ errorlib_system_null,
+	/* 0x24 */ errorlib_system_null, /* 0x25 */ errorlib_system_null,
+	/* 0x26 */ errorlib_system_null, /* 0x27 */ errorlib_system_null,
+	/* 0x28 */ errorlib_system_null, /* 0x29 */ errorlib_system_null,
+	/* 0x2a */ errorlib_system_null, /* 0x2b */ errorlib_system_null,
+	/* 0x2c */ errorlib_system_null, /* 0x2d */ errorlib_system_null,
+	/* 0x2e */ errorlib_system_null, /* 0x2f */ errorlib_system_null,
+
+	/* 0x30 */ errorlib_system_null, /* 0x31 */ errorlib_system_null,
+	/* 0x32 */ errorlib_system_null, /* 0x33 */ errorlib_system_null,
+	/* 0x34 */ errorlib_system_null, /* 0x35 */ errorlib_system_null,
+	/* 0x36 */ errorlib_system_null, /* 0x37 */ errorlib_system_null,
+
+	/* 0x38; err_iokit */
+	{
+		errlib_count(err_iokit_sub),
+		"(iokit/?) unknown subsystem error",
+		err_iokit_sub,
+		err_iokit_sub_map,
+		errlib_count(err_iokit_sub_map)
+	},
+
+	                                 /* 0x39 */ errorlib_system_null,
+	/* 0x3a */ errorlib_system_null, /* 0x3b */ errorlib_system_null,
+	/* 0x3c */ errorlib_system_null, /* 0x3d */ errorlib_system_null,
+	/* 0x3e */ errorlib_system_null, /* 0x3f */ errorlib_system_null,
+};
+
+// int error_system_count = errlib_count(_mach_errors);
diff --git a/libsyscall/mach/errorlib.h b/libsyscall/mach/errorlib.h
new file mode 100644
index 000000000..0ed4a27c8
--- /dev/null
+++ b/libsyscall/mach/errorlib.h
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	errorlib.h
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar. 1988
+ *
+ *	Error bases for subsytems errors.
+ */
+
+#include <stddef.h>
+
+#include <mach/error.h>
+#include <mach/boolean.h>
+
+#define	MACH_IPC_SEND_MOD	(err_mach_ipc|err_sub(0))
+#define	MACH_IPC_RCV_MOD	(err_mach_ipc|err_sub(1))
+#define	MACH_IPC_MIG_MOD	(err_mach_ipc|err_sub(2))
+
+#define	IPC_SEND_MOD		(err_ipc|err_sub(0))
+#define	IPC_RCV_MOD		(err_ipc|err_sub(1))
+#define	IPC_MIG_MOD		(err_ipc|err_sub(2))
+
+#define	SERV_NETNAME_MOD	(err_server|err_sub(0))
+#define	SERV_ENV_MOD		(err_server|err_sub(1))
+#define	SERV_EXECD_MOD		(err_server|err_sub(2))
+
+
+#define	NO_SUCH_ERROR		"unknown error code"
+
+struct error_sparse_map {
+	unsigned		start;
+	unsigned		end;
+};
+#define err_sub_map_entry(start, end)  { err_get_sub(start),  err_get_sub(end) }
+#define err_code_map_entry(start, end) { err_get_code(start), err_get_code(end) }
+
+struct error_subsystem {
+	const char			* subsys_name;
+	int				max_code;
+	const char			* * codes;
+	const struct error_sparse_map	*map_table;
+	int				map_count;
+};
+#define errorlib_system_null	{ 0, NULL, NULL, NULL, 0 }
+
+struct error_system {
+	int				max_sub;
+	const char			* bad_sub;
+	const struct error_subsystem	* subsystem;
+	const struct error_sparse_map	* map_table;
+	int				map_count;
+};
+#define errorlib_sub_null	{ NULL, 0, NULL, NULL, 0 }
+
+__private_extern__ struct error_system 	_mach_errors[err_max_system+1];
+
+__private_extern__ char *mach_error_string_int(mach_error_t, boolean_t *);
+
+#define	errlib_count(s)		(sizeof(s)/sizeof(s[0]))
diff --git a/libsyscall/mach/exc.defs b/libsyscall/mach/exc.defs
new file mode 100644
index 000000000..b32f16a9d
--- /dev/null
+++ b/libsyscall/mach/exc.defs
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#define catch_ internal_catch_
+#include <mach/exc.defs>
diff --git a/libsyscall/mach/exc_catcher.c b/libsyscall/mach/exc_catcher.c
new file mode 100644
index 000000000..507cb1000
--- /dev/null
+++ b/libsyscall/mach/exc_catcher.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * catch_exception_raise will be implemented by user programs
+ * This implementation is provided to resolve the reference in
+ * exc_server().
+ */
+
+#include <mach/boolean.h>
+#include <mach/message.h>
+#include <mach/exception.h>
+#include <mach/mig_errors.h>
+#include <mach-o/dyld.h>
+#include <stdlib.h>
+
+__private_extern__ kern_return_t internal_catch_exception_raise (
+    mach_port_t exception_port,
+    mach_port_t thread,
+    mach_port_t task,
+    exception_type_t exception,
+    exception_data_t code,
+    mach_msg_type_number_t codeCnt)
+{
+#if defined(__DYNAMIC__)
+    static int checkForFunction = 0;
+    /* This will be non-zero if the user has defined this function */
+    static kern_return_t (*func)(mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t);
+    if (checkForFunction == 0) {
+        checkForFunction = 1;
+        _dyld_lookup_and_bind("_catch_exception_raise", (unsigned long *)&func, (void **)0);
+    }
+    if (func == 0) {
+        /* The user hasn't defined catch_exception_raise in their binary */
+        abort();
+    }
+    return (*func)(exception_port, thread, task, exception, code, codeCnt);
+#else
+    extern kern_return_t catch_exception_raise(mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t);
+    return catch_exception_raise(exception_port, thread, task, exception, code, codeCnt);
+#endif
+}
+
diff --git a/libsyscall/mach/exc_catcher_state.c b/libsyscall/mach/exc_catcher_state.c
new file mode 100644
index 000000000..c372d1c20
--- /dev/null
+++ b/libsyscall/mach/exc_catcher_state.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * catch_exception_raise_state will be implemented by user programs
+ * This implementation is provided to resolve the reference in
+ * exc_server().
+ */
+
+#include <mach/boolean.h>
+#include <mach/message.h>
+#include <mach/exception.h>
+#include <mach/mig_errors.h>
+#include <mach-o/dyld.h>
+#include <stdlib.h>
+
+__private_extern__ kern_return_t internal_catch_exception_raise_state (
+        mach_port_t exception_port,
+        exception_type_t exception,
+        exception_data_t code,
+        mach_msg_type_number_t codeCnt,
+        int *flavor,
+        thread_state_t old_state,
+        mach_msg_type_number_t old_stateCnt,
+        thread_state_t new_state,
+        mach_msg_type_number_t *new_stateCnt)
+{
+#if defined(__DYNAMIC__)
+    static int checkForFunction = 0;
+    /* This will be non-zero if the user has defined this function */
+    static kern_return_t (*func)(mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t, int *, thread_state_t, mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
+    if (checkForFunction == 0) {
+        checkForFunction = 1;
+        _dyld_lookup_and_bind("_catch_exception_raise_state", (unsigned long *)&func, (void **)0);
+    }
+    if (func == 0) {
+        /* The user hasn't defined catch_exception_raise in their binary */
+        abort();
+    }
+    return (*func)(exception_port, exception, code, codeCnt, flavor, old_state, old_stateCnt, new_state, new_stateCnt);
+#else
+    extern kern_return_t catch_exception_raise_state(mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t, int *, thread_state_t, mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
+    return catch_exception_raise_state(exception_port, exception, code, codeCnt, flavor, old_state, old_stateCnt, new_state, new_stateCnt);
+#endif
+}
+
diff --git a/libsyscall/mach/exc_catcher_state_identity.c b/libsyscall/mach/exc_catcher_state_identity.c
new file mode 100644
index 000000000..139b772c2
--- /dev/null
+++ b/libsyscall/mach/exc_catcher_state_identity.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * catch_exception_raise_state_identity will be implemented by user programs
+ * This implementation is provided to resolve the reference in
+ * exc_server().
+ */
+
+#include <mach/boolean.h>
+#include <mach/message.h>
+#include <mach/exception.h>
+#include <mach/mig_errors.h>
+#include <mach-o/dyld.h>
+#include <stdlib.h>
+
+__private_extern__ kern_return_t internal_catch_exception_raise_state_identity (
+        mach_port_t exception_port,
+        mach_port_t thread,
+        mach_port_t task,
+        exception_type_t exception,
+        exception_data_t code,
+        mach_msg_type_number_t codeCnt,
+        int *flavor,
+        thread_state_t old_state,
+        mach_msg_type_number_t old_stateCnt,
+        thread_state_t new_state,
+        mach_msg_type_number_t *new_stateCnt)
+{
+#if defined(__DYNAMIC__)
+    static int checkForFunction = 0;
+    /* This will be non-zero if the user has defined this function */
+    static kern_return_t (*func)(mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t, int *, thread_state_t, mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
+    if (checkForFunction == 0) {
+        checkForFunction = 1;
+        _dyld_lookup_and_bind("_catch_exception_raise_state_identity", (unsigned long *)&func, (void **)0);
+    }
+    if (func == 0) {
+        /* The user hasn't defined catch_exception_raise in their binary */
+        abort();
+    }
+    return (*func)(exception_port, thread, task, exception, code, codeCnt, flavor, old_state, old_stateCnt, new_state, new_stateCnt);
+#else
+    extern kern_return_t catch_exception_raise_state_identity(mach_port_t, mach_port_t, mach_port_t, exception_type_t, exception_data_t, mach_msg_type_number_t, int *, thread_state_t, mach_msg_type_number_t, thread_state_t, mach_msg_type_number_t *);
+    return catch_exception_raise_state_identity(exception_port, thread, task, exception, code, codeCnt, flavor, old_state, old_stateCnt, new_state, new_stateCnt);
+#endif
+}
+
diff --git a/libsyscall/mach/externs.h b/libsyscall/mach/externs.h
new file mode 100644
index 000000000..765140455
--- /dev/null
+++ b/libsyscall/mach/externs.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+#ifndef _MACH_EXTERNS_H_
+#define _MACH_EXTERNS_H_
+
+#include <mach/mach_types.h>
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+extern void mig_init(void *);
+extern void mach_init_ports(void);
+__END_DECLS
+
+#endif /* _MACH_EXTERNS_H_ */
diff --git a/libsyscall/mach/fprintf_stderr.c b/libsyscall/mach/fprintf_stderr.c
new file mode 100644
index 000000000..e89df1136
--- /dev/null
+++ b/libsyscall/mach/fprintf_stderr.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_FREE_COPYRIGHT@
+ * 
+ */
+
+#include <mach/mach.h>
+#include <mach/mach_init.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+int (*vprintf_stderr_func)(const char *format, va_list ap);
+
+
+/* This function allows the writing of a mach error message to an
+ * application-controllable output method, the default being to
+ * use printf if no other method is specified by the application.
+ *
+ * To override, set the global (static) function pointer vprintf_stderr to
+ * a function which takes the same parameters as vprintf.
+ */
+
+int fprintf_stderr(const char *format, ...)
+{
+        va_list args;
+	int retval;
+
+	va_start(args, format);
+	if (vprintf_stderr_func == NULL)
+		retval = vprintf(format, args);
+	else
+		retval = (*vprintf_stderr_func)(format, args);
+	va_end(args);
+
+	return retval;
+}
diff --git a/libsyscall/mach/headers/Makefile.inc b/libsyscall/mach/headers/Makefile.inc
new file mode 100644
index 000000000..f747c3d76
--- /dev/null
+++ b/libsyscall/mach/headers/Makefile.inc
@@ -0,0 +1,10 @@
+MACH_INSTHDRS += mach.h \
+                 mach_error.h \
+                 mach_init.h \
+                 mach_interface.h \
+                 port_obj.h \
+                 sync.h \
+                 task.h \
+                 thread_act.h \
+                 vm_task.h
+MACH_INSTHDRS := ${MACH_INSTHDRS:S/^/${.CURDIR}\/mach\/headers\//}
diff --git a/libsyscall/mach/headers/errorlib.h b/libsyscall/mach/headers/errorlib.h
new file mode 100644
index 000000000..0c3cc64c2
--- /dev/null
+++ b/libsyscall/mach/headers/errorlib.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	errorlib.h
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar. 1988
+ *
+ *	Error bases for subsytems errors.
+ */
+
+#include <mach/error.h>
+
+#define	MACH_IPC_SEND_MOD	(err_mach_ipc|err_sub(0))
+#define	MACH_IPC_RCV_MOD	(err_mach_ipc|err_sub(1))
+#define	MACH_IPC_MIG_MOD	(err_mach_ipc|err_sub(2))
+
+#define	IPC_SEND_MOD		(err_ipc|err_sub(0))
+#define	IPC_RCV_MOD		(err_ipc|err_sub(1))
+#define	IPC_MIG_MOD		(err_ipc|err_sub(2))
+
+#define	SERV_NETNAME_MOD	(err_server|err_sub(0))
+#define	SERV_ENV_MOD		(err_server|err_sub(1))
+#define	SERV_EXECD_MOD		(err_server|err_sub(2))
+
+
+#define	NO_SUCH_ERROR		"unknown error code"
+
+struct error_subsystem {
+	char			* subsys_name;
+	int			max_code;
+	char			* * codes;
+};
+
+struct error_system {
+	int			max_sub;
+	char			* bad_sub;
+	struct error_subsystem	* subsystem;
+};
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+extern	struct error_system 	errors[err_max_system+1];
+__END_DECLS
+
+#define	errlib_count(s)		(sizeof(s)/sizeof(s[0]))
diff --git a/libsyscall/mach/headers/mach.h b/libsyscall/mach/headers/mach.h
new file mode 100644
index 000000000..0b3b1a0a5
--- /dev/null
+++ b/libsyscall/mach/headers/mach.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 1999-2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+/* 
+ *  Includes all the types that a normal user
+ *  of Mach programs should need
+ */
+
+#ifndef	_MACH_H_
+#define	_MACH_H_
+
+#define __MACH30__
+#define MACH_IPC_FLAVOR UNTYPED
+
+#include <mach/std_types.h>
+#include <mach/mach_types.h>
+#include <mach/mach_interface.h>
+#include <mach/mach_port.h>
+#include <mach/mach_init.h>
+#include <mach/mach_host.h>
+#include <mach/thread_switch.h>
+
+#include <mach/rpc.h>  		/* for compatibility only */
+#include <mach/mig.h>
+
+#include <mach/mig_errors.h>
+#include <mach/mach_error.h>
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+/*
+ * Standard prototypes
+ */
+extern void			panic_init(mach_port_t);
+extern void			panic(const char *, ...);
+
+extern void			safe_gets(char *,
+					  char *,
+					  int);
+
+extern void			slot_name(cpu_type_t,
+					  cpu_subtype_t,
+					  char **,
+					  char **);
+
+extern void			mig_reply_setup(mach_msg_header_t *,
+						mach_msg_header_t *);
+
+extern void			mach_msg_destroy(mach_msg_header_t *);
+
+extern mach_msg_return_t	mach_msg_receive(mach_msg_header_t *);
+
+extern mach_msg_return_t	mach_msg_send(mach_msg_header_t *);
+
+extern mach_msg_return_t	mach_msg_server_once(boolean_t (*)
+						     (mach_msg_header_t *,
+						      mach_msg_header_t *),
+						     mach_msg_size_t,
+						     mach_port_t,
+						     mach_msg_options_t);
+extern mach_msg_return_t	mach_msg_server(boolean_t (*)
+						(mach_msg_header_t *,
+						 mach_msg_header_t *),
+						mach_msg_size_t,
+						mach_port_t,
+						mach_msg_options_t);
+
+/*
+ * Prototypes for compatibility
+ */
+extern kern_return_t	clock_get_res(mach_port_t,
+				      clock_res_t *);
+extern kern_return_t	clock_set_res(mach_port_t,
+				      clock_res_t);
+
+extern kern_return_t	clock_sleep(mach_port_t,
+				    int,
+				    mach_timespec_t,
+				    mach_timespec_t *);
+__END_DECLS
+
+#endif	/* _MACH_H_ */
diff --git a/libsyscall/mach/headers/mach_error.h b/libsyscall/mach/headers/mach_error.h
new file mode 100644
index 000000000..cb1acb01e
--- /dev/null
+++ b/libsyscall/mach/headers/mach_error.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS 
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie the
+ * rights to redistribute these changes.
+ */
+
+/*
+ *	File:	mach_error.h
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar. 1988
+ *
+ *	Definitions of routines in mach_error.c
+ */
+
+#ifndef	_MACH_ERROR_
+#define	_MACH_ERROR_	1
+
+#include <mach/error.h>
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+char		*mach_error_string(
+/*
+ *	Returns a string appropriate to the error argument given
+ */
+	mach_error_t error_value
+				);
+
+void		mach_error(
+/*
+ *	Prints an appropriate message on the standard error stream
+ */
+	char 		*str,
+	mach_error_t	error_value
+				);
+
+char		*mach_error_type(
+/*
+ *	Returns a string with the error system, subsystem and code
+ */
+	mach_error_t	error_value
+				);
+__END_DECLS
+
+#endif	/* _MACH_ERROR_ */
diff --git a/libsyscall/mach/headers/mach_init.h b/libsyscall/mach/headers/mach_init.h
new file mode 100644
index 000000000..15a3830d1
--- /dev/null
+++ b/libsyscall/mach/headers/mach_init.h
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987,1986 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+/*
+ *	Items provided by the Mach environment initialization.
+ */
+
+#ifndef	_MACH_INIT_
+#define	_MACH_INIT_	1
+
+#include <mach/mach_types.h>
+#include <stdarg.h>
+
+#include <sys/cdefs.h>
+
+/*
+ *	Kernel-related ports; how a task/thread controls itself
+ */
+
+__BEGIN_DECLS
+extern mach_port_t mach_task_self(void);
+extern mach_port_t mach_host_self(void);
+extern mach_port_t mach_thread_self(void);
+extern kern_return_t host_page_size(host_t, vm_size_t *);
+
+extern	mach_port_t	mach_task_self_;
+#define	mach_task_self() mach_task_self_
+#define	current_task()	mach_task_self()
+
+__END_DECLS
+#include <mach/mach_traps.h>
+__BEGIN_DECLS
+
+/*
+ *	Other important ports in the Mach user environment
+ */
+
+extern	mach_port_t	bootstrap_port;
+extern	mach_port_t	name_server_port;
+extern	mach_port_t	environment_port;
+extern	mach_port_t	service_port;
+
+/*
+ *	Where these ports occur in the "mach_ports_register"
+ *	collection... only servers or the runtime library need know.
+ */
+
+#define	NAME_SERVER_SLOT	0
+#define	ENVIRONMENT_SLOT	1
+#define SERVICE_SLOT		2
+
+#define	MACH_PORTS_SLOTS_USED	3
+
+/*
+ *	Globally interesting numbers.
+ *	These macros assume vm_page_size is a power-of-2.
+ */
+
+extern	vm_size_t	vm_page_size;
+extern	vm_size_t	vm_page_mask;
+extern	int		vm_page_shift;
+
+#define trunc_page(x)	((x) & (~(vm_page_size - 1)))
+#define round_page(x)	trunc_page((x) + (vm_page_size - 1))
+
+/*
+ *	fprintf_stderr uses vprintf_stderr_func to produce
+ *	error messages, this can be overridden by a user
+ *	application to point to a user-specified output function
+ */
+extern int (*vprintf_stderr_func)(const char *format, va_list ap);
+__END_DECLS
+
+#endif	/* _MACH_INIT_ */
diff --git a/libsyscall/mach/headers/mach_interface.h b/libsyscall/mach/headers/mach_interface.h
new file mode 100644
index 000000000..af939cb3b
--- /dev/null
+++ b/libsyscall/mach/headers/mach_interface.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (C) Apple Computer 1998
+ * ALL Rights Reserved
+ */
+/*
+ * This file represents the interfaces that used to come
+ * from creating the user headers from the mach.defs file.
+ * Because mach.defs was decomposed, this file now just
+ * wraps up all the new interface headers generated from
+ * each of the new .defs resulting from that decomposition.
+ */
+#ifndef	_MACH_INTERFACE_H_
+#define _MACH_INTERFACE_H_
+
+#include <mach/clock_priv.h>
+#include <mach/host_priv.h>
+#include <mach/host_security.h>
+#include <mach/ledger.h>
+#include <mach/lock_set.h>
+#include <mach/processor.h>
+#include <mach/processor_set.h>
+#include <mach/semaphore.h>
+#include <mach/task.h>
+#include <mach/thread_act.h>
+#include <mach/vm_map.h>
+
+#endif /* _MACH_INTERFACE_H_ */
diff --git a/libsyscall/mach/headers/port_obj.h b/libsyscall/mach/headers/port_obj.h
new file mode 100644
index 000000000..d2b5f89a8
--- /dev/null
+++ b/libsyscall/mach/headers/port_obj.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/*
+ * Define a service to map from a kernel-generated port name
+ * to server-defined "type" and "value" data to be associated
+ * with the port.
+ */
+
+#ifndef PORT_OBJ_H
+#define PORT_OBJ_H
+
+#include <mach/port.h>
+
+struct port_obj_tentry {
+	void *pos_value;
+	int pos_type;
+};
+
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+extern void port_obj_init(int);
+__END_DECLS
+
+extern struct port_obj_tentry *port_obj_table;
+extern int port_obj_table_size;
+
+#ifndef PORT_OBJ_ASSERT
+
+#define port_set_obj_value_type(pname, value, type)	\
+do {							\
+	int ndx;					\
+							\
+	if (!port_obj_table)				\
+		port_obj_init(port_obj_table_size);	\
+	ndx = MACH_PORT_INDEX(pname);			\
+	port_obj_table[ndx].pos_value = (value);	\
+	port_obj_table[ndx].pos_type = (type);		\
+} while (0)
+
+#define port_get_obj_value(pname)			\
+	(port_obj_table[MACH_PORT_INDEX(pname)].pos_value)
+
+#define port_get_obj_type(pname)			\
+	(port_obj_table[MACH_PORT_INDEX(pname)].pos_type)
+
+#else	/* PORT_OBJ_ASSERT */
+
+#define port_set_obj_value_type(pname, value, type)	\
+do {							\
+	int ndx;					\
+							\
+	if (!port_obj_table)				\
+		port_obj_init(port_obj_table_size);	\
+	ndx = MACH_PORT_INDEX(pname);			\
+	assert(ndx > 0);				\
+	assert(ndx < port_obj_table_size);		\
+	port_obj_table[ndx].pos_value = (value);	\
+	port_obj_table[ndx].pos_type = (type);		\
+} while (0)
+
+#define port_get_obj_value(pname)				\
+	((MACH_PORT_INDEX(pname) < (unsigned)port_obj_table_size) ?	\
+	port_obj_table[MACH_PORT_INDEX(pname)].pos_value :	\
+	(panic("port_get_obj_value: index too big"), (void *)-1))
+
+#define port_get_obj_type(pname)				\
+	((MACH_PORT_INDEX(pname) < (unsigned)port_obj_table_size) ?	\
+	port_obj_table[MACH_PORT_INDEX(pname)].pos_type :	\
+	(panic("port_get_obj_type: index too big"), -1))
+
+#endif	/* PORT_OBJ_ASSERT */
+
+#endif	/* PORT_OBJ_H */
diff --git a/libsyscall/mach/headers/sync.h b/libsyscall/mach/headers/sync.h
new file mode 100644
index 000000000..0a567c244
--- /dev/null
+++ b/libsyscall/mach/headers/sync.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+** This file contains compatibilty wrapper header for things that used
+** to be generated from mach/sync.defs.  Now that code is split into two
+** different interface generator files, so include the two resulting
+** headers here.
+*/
+#include <mach/semaphore.h>
+#include <mach/lock_set.h>
diff --git a/libsyscall/mach/headers/task.h b/libsyscall/mach/headers/task.h
new file mode 100644
index 000000000..a919ee664
--- /dev/null
+++ b/libsyscall/mach/headers/task.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#if defined(__i386__)
+#include <mach/i386/task.h>
+#elif defined(__x86_64__)
+#include <mach/x86_64/task.h>
+#elif defined(__ppc__)
+#include <mach/ppc/task.h>
+#elif defined(__ppc64__)
+#include <mach/ppc64/task.h>
+#else
+#error unknown architecture
+#endif 
diff --git a/libsyscall/mach/headers/thread_act.h b/libsyscall/mach/headers/thread_act.h
new file mode 100644
index 000000000..2696b626d
--- /dev/null
+++ b/libsyscall/mach/headers/thread_act.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#if defined(__i386__)
+#include <mach/i386/thread_act.h>
+#elif defined(__x86_64__)
+#include <mach/x86_64/thread_act.h>
+#elif defined(__ppc__)
+#include <mach/ppc/thread_act.h>
+#elif defined(__ppc64__)
+#include <mach/ppc64/thread_act.h>
+#else
+#error unknown architecture
+#endif 
diff --git a/libsyscall/mach/headers/vm_task.h b/libsyscall/mach/headers/vm_task.h
new file mode 100644
index 000000000..d2401ace0
--- /dev/null
+++ b/libsyscall/mach/headers/vm_task.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+** This file contains compatibilty wrapper header for things that are
+** generated from mach/vm_map.defs into mach/vm_map.h. 
+** 
+** This file will go away eventually - please switch.
+*/
+#include <mach/vm_map.h>
diff --git a/libsyscall/mach/host_priv.defs b/libsyscall/mach/host_priv.defs
new file mode 100644
index 000000000..d3f7a283f
--- /dev/null
+++ b/libsyscall/mach/host_priv.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/host_priv.defs>
diff --git a/libsyscall/mach/host_security.defs b/libsyscall/mach/host_security.defs
new file mode 100644
index 000000000..9e4b571c5
--- /dev/null
+++ b/libsyscall/mach/host_security.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/host_security.defs>
diff --git a/libsyscall/mach/i386/Makefile.inc b/libsyscall/mach/i386/Makefile.inc
new file mode 100644
index 000000000..4afb1ae3a
--- /dev/null
+++ b/libsyscall/mach/i386/Makefile.inc
@@ -0,0 +1,3 @@
+.PATH: ${.CURDIR}/mach/i386
+
+MDSRCS += mach_absolute_time.S
diff --git a/libsyscall/mach/i386/mach_absolute_time.S b/libsyscall/mach/i386/mach_absolute_time.S
new file mode 100644
index 000000000..71e746235
--- /dev/null
+++ b/libsyscall/mach/i386/mach_absolute_time.S
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+
+
+	.text
+	.align	2
+	.globl	_mach_absolute_time
+_mach_absolute_time:
+	movl	$(_COMM_PAGE_NANOTIME), %eax
+	jmpl	*%eax
diff --git a/libsyscall/mach/ledger.defs b/libsyscall/mach/ledger.defs
new file mode 100644
index 000000000..a97546ad9
--- /dev/null
+++ b/libsyscall/mach/ledger.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/ledger.defs>
diff --git a/libsyscall/mach/lock_set.defs b/libsyscall/mach/lock_set.defs
new file mode 100644
index 000000000..fb52dda15
--- /dev/null
+++ b/libsyscall/mach/lock_set.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/lock_set.defs>
diff --git a/libsyscall/mach/mach_error.c b/libsyscall/mach/mach_error.c
new file mode 100644
index 000000000..f3722a573
--- /dev/null
+++ b/libsyscall/mach/mach_error.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+/*
+ * 	File:	mach_error.c
+ *	Author:	Douglas Orr, Carnegie Mellon University
+ *	Date:	Mar 1988
+ *
+ *      interprets structured mach error codes and prints
+ *      or returns a descriptive string.
+ */
+
+#include <stdio.h>
+#include <mach/mach_error.h>
+#include <mach/boolean.h>
+#include "errorlib.h"
+
+int fprintf_stderr(const char *format, ...);
+
+void
+mach_error( str, err )	
+	char	*str;
+	mach_error_t		err;
+{
+	char * err_str;
+	char buf[1024];
+	boolean_t diag;
+
+	err_str=mach_error_string_int(err, &diag);
+
+	if ( diag ) {
+		sprintf( buf, "%s %s (%x)", mach_error_type(err), err_str, err );
+		err_str = buf;
+	}
+
+	fprintf_stderr("%s %s\n", str, err_str);
+}
diff --git a/libsyscall/mach/mach_error_string.c b/libsyscall/mach/mach_error_string.c
new file mode 100644
index 000000000..9240629d9
--- /dev/null
+++ b/libsyscall/mach/mach_error_string.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <limits.h>
+
+#include <mach/boolean.h>
+#include <mach/error.h>
+#include <mach/mach_error.h>
+
+#include "errorlib.h"
+#include "externs.h"
+
+static void do_compat(mach_error_t *);
+
+static void
+do_compat(mach_error_t *org_err)
+{
+	mach_error_t		err = *org_err;
+
+	/* 
+	 * map old error numbers to 
+	 * to new error sys & subsystem 
+	 */
+
+	if ((-200 < err) && (err <= -100))
+		err = -(err + 100) | IPC_SEND_MOD;
+	else if ((-300 < err) && (err <= -200))
+		err = -(err + 200) | IPC_RCV_MOD;
+	else if ((-400 < err) && (err <= -300))
+		err = -(err + 300) | MACH_IPC_MIG_MOD;
+	else if ((1000 <= err) && (err < 1100))
+		err = (err - 1000) | SERV_NETNAME_MOD;
+	else if ((1600 <= err) && (err < 1700))
+		err = (err - 1600) | SERV_ENV_MOD;
+	else if ((27600 <= err) && (err < 27700))
+		err = (err - 27600) | SERV_EXECD_MOD;
+
+	*org_err = err;
+}
+
+static int
+err_sparse_mapit(int old, const struct error_sparse_map *map_table, int mapcnt)
+{
+	boolean_t found = FALSE;
+	int ret = 0, i;
+
+	for (i = 0; i < mapcnt; i++) {
+		struct error_sparse_map entry = map_table[i];
+
+		if (entry.start <= old && old <= entry.end) {
+			ret += old - entry.start;
+			found = TRUE;
+			break;
+		}
+		ret += entry.end - entry.start + 1;
+	}
+
+	return (found)? ret : INT_MAX;
+}
+
+char *
+mach_error_type(mach_error_t err)
+{
+	const struct error_system *sys_p;
+	int sub, system;
+
+	do_compat( &err );
+
+	system = err_get_system(err);
+	sys_p = &_mach_errors[system];
+	sub = err_get_sub(err);
+
+	if (system <= err_max_system && sys_p->map_table)
+		sub = err_sparse_mapit(sub, sys_p->map_table, sys_p->map_count);
+
+	if (system > err_max_system || sub >= sys_p->max_sub)
+	    return((char *)"(?/?)");
+	return (char *) (sys_p->subsystem[sub].subsys_name);
+}
+
+boolean_t mach_error_full_diag = FALSE;
+
+__private_extern__ char *
+mach_error_string_int(mach_error_t err, boolean_t *diag)
+{
+	const struct error_system *sys_p;
+	const struct error_subsystem *sub_p;
+	int sub, system, code;
+
+	do_compat( &err );
+
+	system = err_get_system(err);
+	sys_p = &_mach_errors[system];
+	sub = err_get_sub(err);
+	code = err_get_code(err);
+
+	*diag = TRUE;
+
+	if (system > err_max_system)
+	    return((char *)"(?/?) unknown error system");
+	else if (sys_p->map_table)
+		sub = err_sparse_mapit(sub, sys_p->map_table, sys_p->map_count);
+
+	if (sub >= sys_p->max_sub)
+	    return((char *)sys_p->bad_sub);
+
+	sub_p = &sys_p->subsystem[sub];
+	if (sub_p->map_table)
+		code = err_sparse_mapit(code, sub_p->map_table, sub_p->map_count);
+	if (code >= sub_p->max_code)
+	    return ((char *)NO_SUCH_ERROR);
+
+	*diag = mach_error_full_diag;
+	return( (char *)sub_p->codes[code] );
+}
+
+char *
+mach_error_string(mach_error_t err)
+{
+	boolean_t diag;
+
+	return mach_error_string_int( err, &diag );
+
+}
+
+/* vim: set ts=4: */
diff --git a/libsyscall/mach/mach_host.defs b/libsyscall/mach/mach_host.defs
new file mode 100644
index 000000000..cf6f53279
--- /dev/null
+++ b/libsyscall/mach/mach_host.defs
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/mach_host.defs>
+
+import <mach/mach_init.h>;  /* for host_page_size() */
diff --git a/libsyscall/mach/mach_init.c b/libsyscall/mach/mach_init.c
new file mode 100644
index 000000000..ce2eeed8c
--- /dev/null
+++ b/libsyscall/mach/mach_init.c
@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/mach.h>
+#include <mach/boolean.h>
+#include <mach/machine/ndr_def.h>
+#include <mach/mach_traps.h>
+#include <mach/mach_host.h>
+#include <mach/mach_init.h>
+#include <mach/vm_param.h>
+#include "externs.h"
+
+mach_port_t	mach_task_self_ = MACH_PORT_NULL;
+mach_port_t     mach_host_self_ = MACH_PORT_NULL;
+
+__private_extern__ kern_return_t _host_mach_msg_trap_return_;
+
+vm_size_t	vm_page_size;
+vm_size_t	vm_page_mask;
+int		vm_page_shift;
+
+/*
+ * Forward internal declarations for automatic mach_init during
+ * fork() implementation.
+ */
+/* fork() calls through atfork_child_routine */
+void (*_atfork_child_routine)(void);
+
+static void mach_atfork_child_routine(void);
+static boolean_t first = TRUE;
+static void (*previous_atfork_child_routine)(void);
+static boolean_t mach_init_inited = FALSE;
+extern int mach_init(void);
+extern void _pthread_set_self(void *);
+extern void cthread_set_self(void *);
+extern void __libc_init(void); /* Libc initialization routine */
+
+kern_return_t
+host_page_size(__unused host_t host, vm_size_t *out_page_size)
+{
+	*out_page_size = PAGE_SIZE;
+	return KERN_SUCCESS;
+}
+
+static void mach_atfork_child_routine(void)
+{
+	/*
+	 * If an (*_atfork_child_routine)() was registered when
+	 * mach_init was first called, then call that routine
+	 * prior to performing our re-initialization. This ensures
+	 * that the post-fork handlers are called in exactly the
+	 * same order as the crt0 (exec) handlers. Any library 
+	 * that makes use of the _atfork_child_routine must follow
+	 * the same technique.
+	 */
+	if (previous_atfork_child_routine) {
+		(*previous_atfork_child_routine)();
+	}
+	mach_init_inited = FALSE;
+	mach_init();
+}
+
+mach_port_t
+mach_host_self(void)
+{
+        return(host_self_trap());
+}
+
+int mach_init_doit(int forkchild)
+{
+	host_t host;
+
+	/*
+	 *	Get the important ports into the cached values,
+	 *	as required by "mach_init.h".
+	 */
+	 
+	mach_task_self_ = task_self_trap();
+	host = host_self_trap();
+
+
+	if (!forkchild) {
+		/*
+		 * Set up the post-fork child handler in the libc stub
+		 * to invoke this routine if this process forks. Save the
+		 * previous value in order that we can call that handler
+		 * prior to performing our postfork work.
+		 */
+            
+		first = FALSE;
+		previous_atfork_child_routine = _atfork_child_routine;
+		_atfork_child_routine = mach_atfork_child_routine;
+                _pthread_set_self(0);
+                cthread_set_self(0);
+	}
+
+	/*
+	 *	Initialize the single mig reply port
+	 */
+
+	mig_init(0);
+
+	/*
+	 *	Cache some other valuable system constants
+	 */
+
+	(void)host_page_size(host, &vm_page_size);
+	vm_page_mask = vm_page_size - 1;
+	if (vm_page_size == 0) {
+		/* guard against unlikely craziness */
+		vm_page_shift = 0;
+	} else {
+		/*
+		 * Unfortunately there's no kernel interface to get the
+		 * vm_page_shift, but it's easy enough to calculate.
+		 */
+		for (vm_page_shift = 0;
+		     (vm_page_size & (1 << vm_page_shift)) == 0;
+		     vm_page_shift++)
+			continue;
+	}
+
+	mach_port_deallocate(mach_task_self_, host);
+
+	mach_init_ports();
+
+#if WE_REALLY_NEED_THIS_GDB_HACK
+	/*
+	 * Check to see if GDB wants us to stop
+	 */
+	{
+	  task_user_data_data_t	user_data;
+	  mach_msg_type_number_t	user_data_count = TASK_USER_DATA_COUNT;
+	  
+	user_data.user_data = 0;
+	(void)task_info(mach_task_self_, TASK_USER_DATA,
+		(task_info_t)&user_data, &user_data_count);
+#define MACH_GDB_RUN_MAGIC_NUMBER 1
+#ifdef	MACH_GDB_RUN_MAGIC_NUMBER	
+	  /* This magic number is set in mach-aware gdb 
+	   *  for RUN command to allow us to suspend user's
+	   *  executable (linked with this libmach!) 
+	   *  with the code below.
+	* This hack should disappear when gdb improves.
+	*/
+	if ((int)user_data.user_data == MACH_GDB_RUN_MAGIC_NUMBER) {
+	    kern_return_t ret;
+	    user_data.user_data = 0;
+	    
+	    ret = task_suspend (mach_task_self_);
+	    if (ret != KERN_SUCCESS) {
+		while(1) (void)task_terminate(mach_task_self_);
+	    }
+	}
+#undef MACH_GDB_RUN_MAGIC_NUMBER  
+#endif /* MACH_GDB_RUN_MAGIC_NUMBER */
+	}
+#endif /* WE_REALLY_NEED_THIS_GDB_HACK */
+
+	/*
+         * Reserve page 0 so that the program doesn't get it as
+	 * the result of a vm_allocate() or whatever.
+	 */
+	{
+		vm_offset_t zero_page_start;
+
+		zero_page_start = 0;
+		(void)vm_map(mach_task_self_, &zero_page_start, vm_page_size,
+			     0, FALSE, MEMORY_OBJECT_NULL, 0, TRUE,
+			     VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_COPY);
+		/* ignore result, we don't care if it failed */
+	}
+
+	return(0);
+}
+
+
+
+
+/* 
+ * mach_init() is called explicitly in static executables (including dyld)
+ * It is called implicitly by libSystem_initializer() in dynamic executables
+ */
+int mach_init(void)
+{
+	int ret;
+
+	if (mach_init_inited)
+		return(0);
+	mach_init_inited = TRUE;
+	ret = mach_init_doit(0);
+
+	return ret;
+}
+
+
+
+
+/* called by _cthread_fork_child() */
+int fork_mach_init(void)
+{
+	/* called only from child */
+	return(mach_init_doit(1));
+}
+
+#undef	mach_task_self
+
+mach_port_t
+mach_task_self(void)
+{
+	return(task_self_trap());
+}
+
+mach_port_t
+mach_thread_self(void)
+{
+	return(thread_self_trap());
+}
diff --git a/libsyscall/mach/mach_init_libSystem.c b/libsyscall/mach/mach_init_libSystem.c
new file mode 100644
index 000000000..898f82bf2
--- /dev/null
+++ b/libsyscall/mach/mach_init_libSystem.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef __DYNAMIC__
+struct ProgramVars; /* forward reference */
+
+extern void pthread_init(void);				// from libc.a
+extern void __libc_init(const struct ProgramVars* vars);	// from libc.a
+extern void __keymgr_initializer(void);		// from libkeymgr.a
+extern void _dyld_initializer(void);		// from libdyld.a
+
+/*
+ * libsyscall_initializer() initializes all of libSystem.dylib <rdar://problem/4892197>
+ */
+static __attribute__((constructor)) 
+void libSystem_initializer(int argc, const char* argv[], const char* envp[], const char* apple[], const struct ProgramVars* vars)
+{
+	mach_init();
+	pthread_init();
+	__libc_init(vars);
+	__keymgr_initializer();
+	_dyld_initializer();
+}
+
+/*  
+ *  Old crt1.o glue used to call through mach_init_routine which was used to initialize libSystem.
+ *  LibSystem now auto-initializes but mach_init_routine is left for binary compatibility.
+ */
+static void mach_init_old() {}
+void (*mach_init_routine)(void) = &mach_init_old;
+
+#endif /* __DYNAMIC__ */
diff --git a/libsyscall/mach/mach_init_ports.c b/libsyscall/mach/mach_init_ports.c
new file mode 100644
index 000000000..fcb6d2227
--- /dev/null
+++ b/libsyscall/mach/mach_init_ports.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/mach.h>
+#include <stdlib.h>
+#include "externs.h"
+
+mach_port_t	bootstrap_port = MACH_PORT_NULL;
+mach_port_t	name_server_port = MACH_PORT_NULL;
+mach_port_t	environment_port = MACH_PORT_NULL;
+mach_port_t	service_port = MACH_PORT_NULL;
+semaphore_t	clock_sem = MACH_PORT_NULL;
+mach_port_t	clock_port = MACH_PORT_NULL;
+mach_port_t thread_recycle_port = MACH_PORT_NULL;
+
+void
+mach_init_ports(void)
+{
+	mach_port_array_t	ports;
+	mach_msg_type_number_t	ports_count;
+	kern_return_t		kr;
+	host_t			host;
+
+	/*
+	 *	Find those ports important to every task.
+	 */
+	kr = task_get_special_port(mach_task_self(),
+				   TASK_BOOTSTRAP_PORT,
+				   &bootstrap_port);
+	if (kr != KERN_SUCCESS)
+	    return;
+
+        /* Get the clock service port for nanosleep */
+	host = mach_host_self();
+        kr = host_get_clock_service(host, SYSTEM_CLOCK, &clock_port);
+        if (kr != KERN_SUCCESS) {
+            abort();
+	}
+        kr = semaphore_create(mach_task_self(), &clock_sem, SYNC_POLICY_FIFO, 0);
+        if (kr != KERN_SUCCESS) {
+            abort();
+        }
+	mach_port_deallocate(mach_task_self(), host);
+        kr = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &thread_recycle_port);
+        if (kr != KERN_SUCCESS) {
+            abort();
+        }
+
+	/*
+	 *	Find the options service ports.
+	 *	XXX - Don't need these on Darwin, should go away.
+	 */
+	kr = mach_ports_lookup(mach_task_self(), &ports,
+			       &ports_count);
+	if (kr == KERN_SUCCESS) {
+		if (ports_count >= MACH_PORTS_SLOTS_USED) {
+			name_server_port = ports[NAME_SERVER_SLOT];
+			environment_port = ports[ENVIRONMENT_SLOT];
+			service_port     = ports[SERVICE_SLOT];
+		}
+
+		/* get rid of out-of-line data */
+		(void) vm_deallocate(mach_task_self(),
+			     (vm_offset_t) ports,
+			     (vm_size_t) (ports_count * sizeof *ports));
+	}
+}
+
+#ifdef notdef
+/* will have problems with dylib build --> not needed anyway */
+#ifndef	lint
+/*
+ *	Routines which our library must suck in, to avoid
+ *	a later library from referencing them and getting
+ *	the wrong version.
+ */
+extern void _replacements(void);
+
+void
+_replacements(void)
+{
+	(void)sbrk(0);			/* Pull in our sbrk/brk */
+	(void)malloc(0);		/* Pull in our malloc package */
+}
+#endif	/* lint */
+#endif /* notdef */
diff --git a/libsyscall/mach/mach_msg.c b/libsyscall/mach/mach_msg.c
new file mode 100644
index 000000000..644313d6b
--- /dev/null
+++ b/libsyscall/mach/mach_msg.c
@@ -0,0 +1,619 @@
+/*
+ * Copyright (c) 1999-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <stdlib.h>
+#include <mach/mach.h>
+#include <mach/boolean.h>
+#include <mach/kern_return.h>
+#include <mach/message.h>
+#include <mach/mig_errors.h>
+#include <mach/vm_statistics.h>
+
+#define MACH_MSG_TRAP(msg, opt, ssize, rsize, rname, to, not) \
+	 mach_msg_trap((msg), (opt), (ssize), (rsize), (rname), (to), (not))
+
+#define LIBMACH_OPTIONS	(MACH_SEND_INTERRUPT|MACH_RCV_INTERRUPT)
+
+/*
+ *	Routine:	mach_msg
+ *	Purpose:
+ *		Send and/or receive a message.  If the message operation
+ *		is interrupted, and the user did not request an indication
+ *		of that fact, then restart the appropriate parts of the
+ * 		operation.
+ */
+mach_msg_return_t
+mach_msg(msg, option, send_size, rcv_size, rcv_name, timeout, notify)
+	mach_msg_header_t *msg;
+	mach_msg_option_t option;
+	mach_msg_size_t send_size;
+	mach_msg_size_t rcv_size;
+	mach_port_t rcv_name;
+	mach_msg_timeout_t timeout;
+	mach_port_t notify;
+{
+	mach_msg_return_t mr;
+
+	/*
+	 * Consider the following cases:
+	 *	1) Errors in pseudo-receive (eg, MACH_SEND_INTERRUPTED
+	 *	plus special bits).
+	 *	2) Use of MACH_SEND_INTERRUPT/MACH_RCV_INTERRUPT options.
+	 *	3) RPC calls with interruptions in one/both halves.
+	 *
+	 * We refrain from passing the option bits that we implement
+	 * to the kernel.  This prevents their presence from inhibiting
+	 * the kernel's fast paths (when it checks the option value).
+	 */
+
+	mr = MACH_MSG_TRAP(msg, option &~ LIBMACH_OPTIONS,
+			   send_size, rcv_size, rcv_name,
+			   timeout, notify);
+	if (mr == MACH_MSG_SUCCESS)
+		return MACH_MSG_SUCCESS;
+
+	if ((option & MACH_SEND_INTERRUPT) == 0)
+		while (mr == MACH_SEND_INTERRUPTED)
+			mr = MACH_MSG_TRAP(msg,
+				option &~ LIBMACH_OPTIONS,
+				send_size, rcv_size, rcv_name,
+				timeout, notify);
+
+	if ((option & MACH_RCV_INTERRUPT) == 0)
+		while (mr == MACH_RCV_INTERRUPTED)
+			mr = MACH_MSG_TRAP(msg,
+				option &~ (LIBMACH_OPTIONS|MACH_SEND_MSG),
+				0, rcv_size, rcv_name,
+				timeout, notify);
+
+	return mr;
+}
+
+/*
+ *	Routine:	mach_msg_overwrite
+ *	Purpose:
+ *		Send and/or receive a message.  If the message operation
+ *		is interrupted, and the user did not request an indication
+ *		of that fact, then restart the appropriate parts of the
+ * 		operation.
+ *
+ *		Distinct send and receive buffers may be specified.  If
+ *		no separate receive buffer is specified, the msg parameter
+ *		will be used for both send and receive operations.
+ *
+ *		In addition to a distinct receive buffer, that buffer may
+ *		already contain scatter control information to direct the
+ *		receiving of the message.
+ */
+mach_msg_return_t
+mach_msg_overwrite(msg, option, send_size, rcv_limit, rcv_name, timeout, 
+		   notify, rcv_msg, rcv_scatter_size)
+	mach_msg_header_t *msg;
+	mach_msg_option_t option;
+	mach_msg_size_t send_size;
+	mach_msg_size_t rcv_limit;
+	mach_port_t rcv_name;
+	mach_msg_timeout_t timeout;
+	mach_port_t notify;
+	mach_msg_header_t *rcv_msg;
+	mach_msg_size_t rcv_scatter_size;
+{
+	mach_msg_return_t mr;
+
+	/*
+	 * Consider the following cases:
+	 *	1) Errors in pseudo-receive (eg, MACH_SEND_INTERRUPTED
+	 *	plus special bits).
+	 *	2) Use of MACH_SEND_INTERRUPT/MACH_RCV_INTERRUPT options.
+	 *	3) RPC calls with interruptions in one/both halves.
+	 *
+	 * We refrain from passing the option bits that we implement
+	 * to the kernel.  This prevents their presence from inhibiting
+	 * the kernel's fast paths (when it checks the option value).
+	 */
+
+	mr = mach_msg_overwrite_trap(msg, option &~ LIBMACH_OPTIONS,
+			   send_size, rcv_limit, rcv_name,
+			   timeout, notify, rcv_msg, rcv_scatter_size);
+	if (mr == MACH_MSG_SUCCESS)
+		return MACH_MSG_SUCCESS;
+
+	if ((option & MACH_SEND_INTERRUPT) == 0)
+		while (mr == MACH_SEND_INTERRUPTED)
+			mr = mach_msg_overwrite_trap(msg,
+				option &~ LIBMACH_OPTIONS,
+				send_size, rcv_limit, rcv_name,
+				timeout, notify, rcv_msg, rcv_scatter_size);
+
+	if ((option & MACH_RCV_INTERRUPT) == 0)
+		while (mr == MACH_RCV_INTERRUPTED)
+			mr = mach_msg_overwrite_trap(msg,
+				option &~ (LIBMACH_OPTIONS|MACH_SEND_MSG),
+				0, rcv_limit, rcv_name,
+				timeout, notify, rcv_msg, rcv_scatter_size);
+
+	return mr;
+}
+
+
+mach_msg_return_t
+mach_msg_send(mach_msg_header_t *msg)
+{
+	return mach_msg(msg, MACH_SEND_MSG,
+			msg->msgh_size, 0, MACH_PORT_NULL,
+			MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+}
+
+mach_msg_return_t
+mach_msg_receive(mach_msg_header_t *msg)
+{
+	return mach_msg(msg, MACH_RCV_MSG,
+			0, msg->msgh_size, msg->msgh_local_port,
+			MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+}
+
+
+static void
+mach_msg_destroy_port(mach_port_t port, mach_msg_type_name_t type)
+{
+    if (MACH_PORT_VALID(port)) switch (type) {
+      case MACH_MSG_TYPE_MOVE_SEND:
+      case MACH_MSG_TYPE_MOVE_SEND_ONCE:
+	/* destroy the send/send-once right */
+	(void) mach_port_deallocate(mach_task_self(), port);
+	break;
+
+      case MACH_MSG_TYPE_MOVE_RECEIVE:
+	/* destroy the receive right */
+	(void) mach_port_mod_refs(mach_task_self(), port,
+				  MACH_PORT_RIGHT_RECEIVE, -1);
+	break;
+
+      case MACH_MSG_TYPE_MAKE_SEND:
+	/* create a send right and then destroy it */
+	(void) mach_port_insert_right(mach_task_self(), port,
+				      port, MACH_MSG_TYPE_MAKE_SEND);
+	(void) mach_port_deallocate(mach_task_self(), port);
+	break;
+
+      case MACH_MSG_TYPE_MAKE_SEND_ONCE:
+	/* create a send-once right and then destroy it */
+	(void) mach_port_extract_right(mach_task_self(), port,
+				       MACH_MSG_TYPE_MAKE_SEND_ONCE,
+				       &port, &type);
+	(void) mach_port_deallocate(mach_task_self(), port);
+	break;
+    }
+}
+
+static void
+mach_msg_destroy_memory(vm_offset_t addr, vm_size_t size)
+{
+    if (size != 0)
+	(void) vm_deallocate(mach_task_self(), addr, size);
+}
+
+
+/*
+ *	Routine:	mach_msg_destroy
+ *	Purpose:
+ *		mach_msg_destroy is useful in two contexts.
+ *
+ *		First, it can deallocate all port rights and
+ *		out-of-line memory in a received message.
+ *		When a server receives a request it doesn't want,
+ *		it needs this functionality.
+ *
+ *		Second, it can mimic the side-effects of a msg-send
+ *		operation.  The effect is as if the message were sent
+ *		and then destroyed inside the kernel.  When a server
+ *		can't send a reply (because the client died),
+ *		it needs this functionality.
+ */
+void
+mach_msg_destroy(mach_msg_header_t *msg)
+{
+    mach_msg_bits_t mbits = msg->msgh_bits;
+
+    /*
+     *	The msgh_local_port field doesn't hold a port right.
+     *	The receive operation consumes the destination port right.
+     */
+
+    mach_msg_destroy_port(msg->msgh_remote_port, MACH_MSGH_BITS_REMOTE(mbits));
+
+    if (mbits & MACH_MSGH_BITS_COMPLEX) {
+	mach_msg_body_t		*body;
+	mach_msg_descriptor_t	*saddr, *eaddr;
+	
+    	body = (mach_msg_body_t *) (msg + 1);
+    	saddr = (mach_msg_descriptor_t *) 
+			((mach_msg_base_t *) msg + 1);
+    	eaddr =  saddr + body->msgh_descriptor_count;
+
+	for  ( ; saddr < eaddr; saddr++) {
+	    switch (saddr->type.type) {
+	    
+	        case MACH_MSG_PORT_DESCRIPTOR: {
+		    mach_msg_port_descriptor_t *dsc;
+
+		    /* 
+		     * Destroy port rights carried in the message 
+		     */
+		    dsc = &saddr->port;
+		    mach_msg_destroy_port(dsc->name, dsc->disposition);		
+		    break;
+	        }
+
+	        case MACH_MSG_OOL_DESCRIPTOR : {
+		    mach_msg_ool_descriptor_t *dsc;
+
+		    /* 
+		     * Destroy memory carried in the message 
+		     */
+		    dsc = &saddr->out_of_line;
+		    if (dsc->deallocate) {
+		        mach_msg_destroy_memory((vm_offset_t)dsc->address,
+						dsc->size);
+		    }
+		    break;
+	        }
+
+	        case MACH_MSG_OOL_PORTS_DESCRIPTOR : {
+		    mach_port_t             		*ports;
+		    mach_msg_ool_ports_descriptor_t	*dsc;
+		    mach_msg_type_number_t   		j;
+
+		    /*
+		     * Destroy port rights carried in the message 
+		     */
+		    dsc = &saddr->ool_ports;
+		    ports = (mach_port_t *) dsc->address;
+		    for (j = 0; j < dsc->count; j++, ports++)  {
+		        mach_msg_destroy_port(*ports, dsc->disposition);
+		    }
+
+		    /* 
+		     * Destroy memory carried in the message 
+		     */
+		    if (dsc->deallocate) {
+		        mach_msg_destroy_memory((vm_offset_t)dsc->address, 
+					dsc->count * sizeof(mach_port_t));
+		    }
+		    break;
+	        }
+	    }
+	}
+    }
+}
+
+/*
+ *	Routine:	mach_msg_server_once
+ *	Purpose:
+ *		A simple generic server function.  It allows more flexibility
+ *		than mach_msg_server by processing only one message request
+ *		and then returning to the user.  Note that more in the way
+ * 		of error codes are returned to the user; specifically, any
+ * 		failing error from mach_msg calls will be returned
+ *		(though errors from the demux routine or the routine it
+ *		calls will not be).
+ */
+mach_msg_return_t
+mach_msg_server_once(
+	boolean_t (*demux)(mach_msg_header_t *, mach_msg_header_t *),
+	mach_msg_size_t max_size,
+	mach_port_t rcv_name,
+	mach_msg_options_t options)
+{
+	mig_reply_error_t *bufRequest, *bufReply;
+	mach_msg_size_t request_size;
+	mach_msg_size_t request_alloc;
+	mach_msg_size_t trailer_alloc;
+	mach_msg_size_t reply_alloc;
+	mach_msg_return_t mr;
+	kern_return_t kr;
+	mach_port_t self = mach_task_self();
+
+	options &= ~(MACH_SEND_MSG|MACH_RCV_MSG);
+
+	trailer_alloc = REQUESTED_TRAILER_SIZE(options);
+	request_alloc = round_page(max_size + trailer_alloc);
+
+	request_size = (options & MACH_RCV_LARGE) ?
+    		   request_alloc : max_size + trailer_alloc;
+
+	reply_alloc = round_page((options & MACH_SEND_TRAILER) ? 
+			     (max_size + MAX_TRAILER_SIZE) :
+			     max_size);
+
+	kr = vm_allocate(self,
+		     (vm_address_t *)&bufReply,
+		     reply_alloc,
+		     VM_MAKE_TAG(VM_MEMORY_MACH_MSG)|TRUE);
+	if (kr != KERN_SUCCESS)
+		return kr;    
+
+	for (;;) {
+		mach_msg_size_t new_request_alloc;
+
+		kr = vm_allocate(self,
+			 (vm_address_t *)&bufRequest,
+			 request_alloc,
+			 VM_MAKE_TAG(VM_MEMORY_MACH_MSG)|TRUE);
+		if (kr != KERN_SUCCESS) {
+			vm_deallocate(self,
+					(vm_address_t)bufReply,
+			  		reply_alloc);
+			return kr;
+		}    
+	
+		mr = mach_msg(&bufRequest->Head, MACH_RCV_MSG|options,
+					  0, request_size, rcv_name,
+					  MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+	
+		if (!((mr == MACH_RCV_TOO_LARGE) && (options & MACH_RCV_LARGE)))
+			break;
+
+		new_request_alloc = round_page(bufRequest->Head.msgh_size +
+									   trailer_alloc);
+		vm_deallocate(self,
+				(vm_address_t) bufRequest,
+				request_alloc);
+		request_size = request_alloc = new_request_alloc;
+	}
+
+	if (mr == MACH_MSG_SUCCESS) {
+	/* we have a request message */
+
+		(void) (*demux)(&bufRequest->Head, &bufReply->Head);
+
+		if (!(bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
+			if (bufReply->RetCode == MIG_NO_REPLY)
+				bufReply->Head.msgh_remote_port = MACH_PORT_NULL;
+			else if ((bufReply->RetCode != KERN_SUCCESS) &&
+					 (bufRequest->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
+				/* destroy the request - but not the reply port */
+				bufRequest->Head.msgh_remote_port = MACH_PORT_NULL;
+				mach_msg_destroy(&bufRequest->Head);
+			}
+		}
+
+		/*
+		 *	We don't want to block indefinitely because the client
+		 *	isn't receiving messages from the reply port.
+		 *	If we have a send-once right for the reply port, then
+		 *	this isn't a concern because the send won't block.
+		 *	If we have a send right, we need to use MACH_SEND_TIMEOUT.
+		 *	To avoid falling off the kernel's fast RPC path unnecessarily,
+		 *	we only supply MACH_SEND_TIMEOUT when absolutely necessary.
+		 */
+		if (bufReply->Head.msgh_remote_port != MACH_PORT_NULL) {
+
+			mr = mach_msg(&bufReply->Head,
+				(MACH_MSGH_BITS_REMOTE(bufReply->Head.msgh_bits) ==
+				 MACH_MSG_TYPE_MOVE_SEND_ONCE) ?
+				MACH_SEND_MSG|options :
+				MACH_SEND_MSG|MACH_SEND_TIMEOUT|options,
+				bufReply->Head.msgh_size, 0, MACH_PORT_NULL,
+				MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+		
+			if ((mr != MACH_SEND_INVALID_DEST) &&
+				(mr != MACH_SEND_TIMED_OUT))
+				goto done_once;
+			mr = MACH_MSG_SUCCESS;
+		}
+		if (bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)
+			mach_msg_destroy(&bufReply->Head);
+	}
+
+ done_once:
+	(void)vm_deallocate(self,
+			(vm_address_t) bufRequest,
+			request_alloc);
+	(void)vm_deallocate(self,
+			(vm_address_t) bufReply,
+			reply_alloc);
+	return mr;
+}
+
+/*
+ *	Routine:	mach_msg_server
+ *	Purpose:
+ *		A simple generic server function.  Note that changes here
+ * 		should be considered for duplication above.
+ */
+mach_msg_return_t
+mach_msg_server(
+	boolean_t (*demux)(mach_msg_header_t *, mach_msg_header_t *),
+	mach_msg_size_t max_size,
+	mach_port_t rcv_name,
+	mach_msg_options_t options)
+{
+	mig_reply_error_t *bufRequest, *bufReply;
+	mach_msg_size_t request_size;
+	mach_msg_size_t new_request_alloc;
+	mach_msg_size_t request_alloc;
+	mach_msg_size_t trailer_alloc;
+	mach_msg_size_t reply_alloc;
+	mach_msg_return_t mr;
+	kern_return_t kr;
+	mach_port_t self = mach_task_self();
+
+	options &= ~(MACH_SEND_MSG|MACH_RCV_MSG|MACH_RCV_OVERWRITE);
+
+	reply_alloc = round_page((options & MACH_SEND_TRAILER) ? 
+			     (max_size + MAX_TRAILER_SIZE) : max_size);
+
+	kr = vm_allocate(self,
+		     (vm_address_t *)&bufReply,
+		     reply_alloc,
+		     VM_MAKE_TAG(VM_MEMORY_MACH_MSG)|TRUE);
+	if (kr != KERN_SUCCESS) 
+		return kr;
+
+	request_alloc = 0;
+	trailer_alloc = REQUESTED_TRAILER_SIZE(options);
+	new_request_alloc = round_page(max_size + trailer_alloc);
+
+	request_size = (options & MACH_RCV_LARGE) ?
+    		   new_request_alloc : max_size + trailer_alloc;
+
+	for (;;) {
+		if (request_alloc < new_request_alloc) {
+			request_alloc = new_request_alloc;
+			kr = vm_allocate(self,
+			     	(vm_address_t *)&bufRequest,
+			     	request_alloc,
+			     	VM_MAKE_TAG(VM_MEMORY_MACH_MSG)|TRUE);
+			if (kr != KERN_SUCCESS) {
+				vm_deallocate(self,
+						(vm_address_t)bufReply,
+						reply_alloc);
+				return kr;
+			}
+		}
+		
+		mr = mach_msg(&bufRequest->Head, MACH_RCV_MSG|options,
+				0, request_size, rcv_name,
+				MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+	
+		while (mr == MACH_MSG_SUCCESS) {
+			/* we have another request message */
+
+			(void) (*demux)(&bufRequest->Head, &bufReply->Head);
+
+			if (!(bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
+				if (bufReply->RetCode == MIG_NO_REPLY)
+					bufReply->Head.msgh_remote_port = MACH_PORT_NULL;
+				else if ((bufReply->RetCode != KERN_SUCCESS) &&
+					(bufRequest->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)) {
+					/* destroy the request - but not the reply port */
+					bufRequest->Head.msgh_remote_port = MACH_PORT_NULL;
+					mach_msg_destroy(&bufRequest->Head);
+				}
+			}
+
+			/*
+			 * We don't want to block indefinitely because the client
+			 * isn't receiving messages from the reply port.
+			 * If we have a send-once right for the reply port, then
+			 * this isn't a concern because the send won't block.
+			 * If we have a send right, we need to use MACH_SEND_TIMEOUT.
+			 * To avoid falling off the kernel's fast RPC path,
+			 * we only supply MACH_SEND_TIMEOUT when absolutely necessary.
+			 */
+			if (bufReply->Head.msgh_remote_port != MACH_PORT_NULL) {
+				if (request_alloc == reply_alloc) {
+					mig_reply_error_t *bufTemp;
+
+					mr = mach_msg(
+						&bufReply->Head,
+						(MACH_MSGH_BITS_REMOTE(bufReply->Head.msgh_bits) ==
+						 MACH_MSG_TYPE_MOVE_SEND_ONCE) ?
+						 MACH_SEND_MSG|MACH_RCV_MSG|options :
+						 MACH_SEND_MSG|MACH_RCV_MSG|MACH_SEND_TIMEOUT|options,
+						bufReply->Head.msgh_size, request_size, rcv_name,
+						MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+
+					/* swap request and reply */
+					bufTemp = bufRequest;
+					bufRequest = bufReply;
+					bufReply = bufTemp;
+
+				} else {
+					mr = mach_msg_overwrite(
+						&bufReply->Head,
+						(MACH_MSGH_BITS_REMOTE(bufReply->Head.msgh_bits) ==
+						 MACH_MSG_TYPE_MOVE_SEND_ONCE) ?
+						 MACH_SEND_MSG|MACH_RCV_MSG|options :
+						 MACH_SEND_MSG|MACH_RCV_MSG|MACH_SEND_TIMEOUT|options,
+						bufReply->Head.msgh_size, request_size, rcv_name,
+						MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL,
+						&bufRequest->Head, 0);
+				}
+				
+				if ((mr != MACH_SEND_INVALID_DEST) &&
+					(mr != MACH_SEND_TIMED_OUT))
+					continue;
+			}
+			if (bufReply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX)
+				mach_msg_destroy(&bufReply->Head);
+
+			mr = mach_msg(&bufRequest->Head, MACH_RCV_MSG|options,
+						  0, request_size, rcv_name,
+						  MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL);
+
+		} /* while (mr == MACH_MSG_SUCCESS) */
+		
+		if ((mr == MACH_RCV_TOO_LARGE) && (options & MACH_RCV_LARGE)) {
+			new_request_alloc = round_page(bufRequest->Head.msgh_size +
+								trailer_alloc);
+			request_size = new_request_alloc;
+			vm_deallocate(self,
+						  (vm_address_t) bufRequest,
+						  request_alloc);
+			continue;
+		}
+
+		break;
+
+    } /* for(;;) */
+
+    (void)vm_deallocate(self,
+			(vm_address_t) bufRequest,
+			request_alloc);
+    (void)vm_deallocate(self,
+			(vm_address_t) bufReply,
+			reply_alloc);
+    return mr;
+}
diff --git a/libsyscall/mach/mach_port.defs b/libsyscall/mach/mach_port.defs
new file mode 100644
index 000000000..1fc33c241
--- /dev/null
+++ b/libsyscall/mach/mach_port.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/mach_port.defs>
diff --git a/libsyscall/mach/mach_traps.s b/libsyscall/mach/mach_traps.s
new file mode 100644
index 000000000..4470f3ca6
--- /dev/null
+++ b/libsyscall/mach/mach_traps.s
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987,1986 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/syscall_sw.h>
diff --git a/libsyscall/mach/mach_vm.defs b/libsyscall/mach/mach_vm.defs
new file mode 100644
index 000000000..6aa46f933
--- /dev/null
+++ b/libsyscall/mach/mach_vm.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/mach_vm.defs>
diff --git a/libsyscall/mach/mig_allocate.c b/libsyscall/mach/mig_allocate.c
new file mode 100644
index 000000000..14b8a2933
--- /dev/null
+++ b/libsyscall/mach/mig_allocate.c
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon 
+ * the rights to redistribute these changes.
+ */
+
+/*
+ * Memory allocation routine for MiG interfaces.
+ */
+#include <mach/mach.h>
+
+void
+mig_allocate(vm_address_t *addr_p, vm_size_t size)
+{
+	if (vm_allocate(mach_task_self(),
+			addr_p,
+			size,
+			VM_MAKE_TAG(VM_MEMORY_MACH_MSG)|TRUE)
+	    != KERN_SUCCESS)
+		*addr_p = 0;
+}
diff --git a/libsyscall/mach/mig_deallocate.c b/libsyscall/mach/mig_deallocate.c
new file mode 100644
index 000000000..bbcf15e6b
--- /dev/null
+++ b/libsyscall/mach/mig_deallocate.c
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 1999-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon 
+ * the rights to redistribute these changes.
+ */
+
+/*
+ * Memory deallocation routine for MiG interfaces.
+ */
+#include <mach/mach.h>
+
+void
+mig_deallocate(vm_address_t addr, vm_size_t size)
+{
+	(void) vm_deallocate(mach_task_self(),
+			addr,
+			size);
+}
diff --git a/libsyscall/mach/mig_reply_setup.c b/libsyscall/mach/mig_reply_setup.c
new file mode 100644
index 000000000..96df01b1d
--- /dev/null
+++ b/libsyscall/mach/mig_reply_setup.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon 
+ * the rights to redistribute these changes.
+ */
+
+/*
+ * Routine to set up a MiG reply message from a request message.
+ *
+ * Knows about the MiG reply message ID convention:
+ *	reply_id = request_id + 100
+ *
+ * For typed IPC sets up the RetCode type field.  Does NOT set a
+ * return code value.
+ */
+
+#include <mach/mach.h>
+#include <mach/message.h>
+#include <mach/mig_errors.h>
+
+void
+mig_reply_setup(mach_msg_header_t *request, mach_msg_header_t *reply)
+{
+#define	InP	(request)
+#define	OutP	((mig_reply_error_t *) reply)
+
+	OutP->Head.msgh_bits =
+		MACH_MSGH_BITS(MACH_MSGH_BITS_LOCAL(InP->msgh_bits), 0);
+	OutP->Head.msgh_size = sizeof(mig_reply_error_t);
+	OutP->Head.msgh_remote_port = InP->msgh_local_port;
+	OutP->Head.msgh_local_port  = MACH_PORT_NULL;
+	OutP->Head.msgh_id = InP->msgh_id + 100;
+	OutP->NDR = NDR_record;
+}
diff --git a/libsyscall/mach/mig_strncpy.c b/libsyscall/mach/mig_strncpy.c
new file mode 100644
index 000000000..4366563fa
--- /dev/null
+++ b/libsyscall/mach/mig_strncpy.c
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+/*
+ * mig_strncpy.c - by Joshua Block
+ *
+ * mig_strncpy -- Bounded string copy.  Does what the library routine strncpy
+ * OUGHT to do:  Copies the (null terminated) string in src into dest, a 
+ * buffer of length len.  Assures that the copy is still null terminated
+ * and doesn't overflow the buffer, truncating the copy if necessary.
+ *
+ * Parameters:
+ * 
+ *     dest - Pointer to destination buffer.
+ * 
+ *     src - Pointer to source string.
+ * 
+ *     len - Length of destination buffer.
+ *
+ * Result:
+ *	length of string copied, INCLUDING the trailing 0.
+ */
+#include <mach/mig_errors.h>
+
+int
+mig_strncpy(
+    register char *dest,
+    register const char *src,
+    register int len)
+{
+    register int i;
+
+    if (len <= 0)
+	return 0;
+
+    for (i=1; i<len; i++)
+	if (! (*dest++ = *src++))
+	    return i;
+
+    *dest = '\0';
+    return i;
+}
diff --git a/libsyscall/mach/ms_thread_switch.c b/libsyscall/mach/ms_thread_switch.c
new file mode 100644
index 000000000..e7769c11a
--- /dev/null
+++ b/libsyscall/mach/ms_thread_switch.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/mach.h>
+#include <mach/message.h>
+#include <mach/rpc.h>
+#include <mach/mach_syscalls.h>
+#include <mach/thread_switch.h>
+
+extern kern_return_t syscall_thread_switch(mach_port_name_t, int, mach_msg_timeout_t);	// From pthread_internals.h
+
+kern_return_t
+thread_switch(
+	mach_port_t		thread,
+	int			option,
+	mach_msg_timeout_t	option_time)
+{
+	kern_return_t result;
+	
+		result = syscall_thread_switch(thread, option, option_time);
+	return (result);
+}
diff --git a/libsyscall/mach/notify.defs b/libsyscall/mach/notify.defs
new file mode 100644
index 000000000..609633e5c
--- /dev/null
+++ b/libsyscall/mach/notify.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/notify.defs>
diff --git a/libsyscall/mach/panic.c b/libsyscall/mach/panic.c
new file mode 100644
index 000000000..d0a658643
--- /dev/null
+++ b/libsyscall/mach/panic.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+
+#include <mach/mach.h>
+#include <mach/mach_host.h>
+#include <stdarg.h>
+#include <stdio.h>
+
+static mach_port_t master_host_port;
+
+void
+panic_init(mach_port_t port)
+{
+	master_host_port = port;
+}
+
+/*VARARGS1*/
+void
+panic(const char *s, ...)
+{
+	va_list listp;
+
+	printf("panic: ");
+	va_start(listp, s);
+	vprintf(s, listp);
+	va_end(listp);
+	printf("\n");
+
+#define RB_DEBUGGER	0x1000	/* enter debugger NOW */
+	(void) host_reboot(master_host_port, RB_DEBUGGER);
+
+	/* 4279008 - don't return */
+	abort();
+}
diff --git a/libsyscall/mach/port_obj.c b/libsyscall/mach/port_obj.c
new file mode 100644
index 000000000..1951d1ce5
--- /dev/null
+++ b/libsyscall/mach/port_obj.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+/*
+ * Define a service to map from a kernel-generated port name
+ * to server-defined "type" and "value" data to be associated
+ * with the port.
+ */
+#include <mach/port_obj.h>
+#include <mach/mach.h>
+
+#define DEFAULT_TABLE_SIZE	(64 * 1024)
+
+struct port_obj_tentry *port_obj_table;
+int port_obj_table_size = DEFAULT_TABLE_SIZE;
+
+void port_obj_init(
+	int maxsize)
+{
+	kern_return_t kr;
+
+	kr = vm_allocate(mach_task_self(),
+		(vm_offset_t *)&port_obj_table,
+		(vm_size_t)(maxsize * sizeof (*port_obj_table)),
+		TRUE);
+	if (kr != KERN_SUCCESS)
+		panic("port_obj_init: can't vm_allocate");
+}
diff --git a/libsyscall/mach/ppc/Makefile.inc b/libsyscall/mach/ppc/Makefile.inc
new file mode 100644
index 000000000..faa3b19e8
--- /dev/null
+++ b/libsyscall/mach/ppc/Makefile.inc
@@ -0,0 +1,3 @@
+.PATH: ${.CURDIR}/mach/ppc
+
+MDSRCS += mach_absolute_time.s
diff --git a/libsyscall/mach/ppc/mach_absolute_time.s b/libsyscall/mach/ppc/mach_absolute_time.s
new file mode 100644
index 000000000..2f4da835f
--- /dev/null
+++ b/libsyscall/mach/ppc/mach_absolute_time.s
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#define	__APPLE_API_PRIVATE
+#include <machine/cpu_capabilities.h>
+#undef	__APPLE_API_PRIVATE
+
+.text
+.align 4
+.globl _mach_absolute_time
+_mach_absolute_time:
+    ba	_COMM_PAGE_ABSOLUTE_TIME
diff --git a/libsyscall/mach/ppc64/Makefile.inc b/libsyscall/mach/ppc64/Makefile.inc
new file mode 100644
index 000000000..302f57141
--- /dev/null
+++ b/libsyscall/mach/ppc64/Makefile.inc
@@ -0,0 +1,4 @@
+# searching ppc directory as a fallback to avoid unnecessary code duplication
+.PATH: ${.CURDIR}/mach/ppc
+
+MDSRCS += mach_absolute_time.s
diff --git a/libsyscall/mach/processor.defs b/libsyscall/mach/processor.defs
new file mode 100644
index 000000000..72da66329
--- /dev/null
+++ b/libsyscall/mach/processor.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/processor.defs>
diff --git a/libsyscall/mach/processor_set.defs b/libsyscall/mach/processor_set.defs
new file mode 100644
index 000000000..53af54c25
--- /dev/null
+++ b/libsyscall/mach/processor_set.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/processor_set.defs>
diff --git a/libsyscall/mach/sbrk.c b/libsyscall/mach/sbrk.c
new file mode 100644
index 000000000..702534a1b
--- /dev/null
+++ b/libsyscall/mach/sbrk.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *	File:	sbrk.c
+ *
+ *	Unix compatibility for sbrk system call.
+ *
+ * HISTORY
+ * 09-Mar-90  Gregg Kellogg (gk) at NeXT.
+ *	include <kern/mach_interface.h> instead of <kern/mach.h>
+ *
+ * 14-Feb-89  Avadis Tevanian (avie) at NeXT.
+ *	Total rewrite using a fixed area of VM from break region.
+ */
+
+#include <mach/mach.h>		/* for vm_allocate, vm_offset_t */
+#include <mach/vm_statistics.h>
+
+static int sbrk_needs_init = TRUE;
+static vm_size_t sbrk_region_size = 4*1024*1024; /* Well, what should it be? */
+static vm_address_t sbrk_curbrk;
+
+void *sbrk(size)
+	int	size;
+{
+	kern_return_t	ret;
+
+	if (sbrk_needs_init) {
+		sbrk_needs_init = FALSE;
+		/*
+		 *	Allocate a big region to simulate break region.
+		 */
+		ret =  vm_allocate(mach_task_self(), &sbrk_curbrk, sbrk_region_size,
+				  VM_MAKE_TAG(VM_MEMORY_SBRK)|TRUE);
+		if (ret != KERN_SUCCESS)
+			return((void *)-1);
+	}
+	
+	if (size <= 0)
+		return((void *)sbrk_curbrk);
+	else if (size > sbrk_region_size)
+		return((void *)-1);
+	sbrk_curbrk += size;
+	sbrk_region_size -= size;
+	return((void *)(sbrk_curbrk - size));
+}
+
+void *brk(x)
+	void *x;
+{
+	return((void *)-1);
+}
+
diff --git a/libsyscall/mach/semaphore.c b/libsyscall/mach/semaphore.c
new file mode 100644
index 000000000..5d7b45e11
--- /dev/null
+++ b/libsyscall/mach/semaphore.c
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/boolean.h>
+#include <mach/message.h>
+#include <mach/kern_return.h>
+#include <mach/mach_traps.h>
+#include <mach/mach_types.h>
+#include <mach/clock_types.h>
+
+kern_return_t semaphore_signal(
+    mach_port_t signal_semaphore)
+{
+        return semaphore_signal_trap(signal_semaphore);
+}
+
+kern_return_t semaphore_signal_all(
+    mach_port_t signal_semaphore)
+{
+        return semaphore_signal_all_trap(signal_semaphore);
+}
+
+kern_return_t semaphore_signal_thread(
+    mach_port_t signal_semaphore,
+    mach_port_t thread_act)
+{
+        return semaphore_signal_thread_trap(signal_semaphore, thread_act);
+}
+
+kern_return_t semaphore_wait (
+    mach_port_t wait_semaphore)
+{
+        return semaphore_wait_trap(wait_semaphore);
+}
+
+kern_return_t semaphore_timedwait (
+    mach_port_t wait_semaphore,
+    mach_timespec_t wait_time)
+{
+        return semaphore_timedwait_trap(wait_semaphore,
+                                        wait_time.tv_sec,
+                                        wait_time.tv_nsec);
+}
+
+kern_return_t semaphore_wait_signal (
+    mach_port_t wait_semaphore,
+    mach_port_t signal_semaphore)
+{
+        return semaphore_wait_signal_trap(wait_semaphore, signal_semaphore);
+}
+
+kern_return_t semaphore_timedwait_signal (
+    mach_port_t wait_semaphore,
+    mach_port_t signal_semaphore,
+    mach_timespec_t wait_time)
+{
+        return semaphore_timedwait_signal_trap(wait_semaphore,
+                                               signal_semaphore,
+                                               wait_time.tv_sec,
+                                               wait_time.tv_nsec);
+}
diff --git a/libsyscall/mach/servers/Makefile.inc b/libsyscall/mach/servers/Makefile.inc
new file mode 100644
index 000000000..848379a88
--- /dev/null
+++ b/libsyscall/mach/servers/Makefile.inc
@@ -0,0 +1,16 @@
+.PATH: ${.CURDIR}/${MACHINE_ARCH}/mach/servers ${.CURDIR}/mach/servers
+
+SRVMIGDEFS += netname.defs
+
+SRVMIGHDRS = ${SRVMIGDEFS:S/.defs$/.h/}
+#SRVMIGHDRS = ${SRVMIGDEFS:S/.defs$/.h/:S/^/${.CURDIR}\/mach\/servers\//}
+SRVMIGSRCS = ${SRVMIGDEFS:S/.defs$/User.c/}
+
+SRVHDRS = netname_defs.h key_defs.h nm_defs.h ls_defs.h 
+SRVHDRS := ${SRVHDRS:S/^/${.CURDIR}\/mach\/servers\//}
+SRVHDRS += ${SRVMIGHDRS}
+
+MISRCS+= ${SRVMIGDEFS:S/.defs$/User.defs/}
+
+CLEANFILES += ${SRVMIGHDRS} ${SRVMIGHDRS:S/.h$/User.c/} \
+		${SRVMIGHDRS:S/.h$/Server.c/}
diff --git a/libsyscall/mach/servers/key_defs.h b/libsyscall/mach/servers/key_defs.h
new file mode 100644
index 000000000..5d46904f3
--- /dev/null
+++ b/libsyscall/mach/servers/key_defs.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1987 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+
+/*
+ * Definitions of encryption keys etc..
+ */
+
+/*
+ * HISTORY:
+ *  5-Jun-87  Robert Sansom (rds) at Carnegie Mellon University
+ *	Added macros to convert keys between network and host order.
+ *
+ * 12-Apr-87  Robert Sansom (rds) at Carnegie Mellon University
+ *	Added KEY_IS_NULL.
+ *
+ *  2-Feb-87  Robert Sansom (rds) at Carnegie Mellon University
+ *	Added KEY_EQUAL.
+ *
+ *  5-Nov-86  Robert Sansom (rds) at Carnegie-Mellon University
+ *	Started.
+ *
+ */
+
+#ifndef	_KEY_DEFS_
+#define	_KEY_DEFS_
+
+/*
+ * An encrytion key.
+ */
+typedef union {
+    unsigned char	key_bytes[16];
+    unsigned long	key_longs[4];
+} key_t, *key_ptr_t;
+
+#define KEY_EQUAL(key1, key2) 					\
+    ((key1.key_longs[0] == key2.key_longs[0])			\
+	&& (key1.key_longs[1] == key2.key_longs[1])		\
+	&& (key1.key_longs[2] == key2.key_longs[2])		\
+	&& (key1.key_longs[3] == key2.key_longs[3]))
+
+#define KEY_IS_NULL(key)					\
+    (((key).key_longs[0] == 0) && ((key).key_longs[1] == 0)	\
+	&& ((key).key_longs[2] == 0) && ((key).key_longs[3] == 0))
+
+
+/*
+ * Macros to convert keys between network and host byte order.
+ */
+#define NTOH_KEY(key) {							\
+    (key).key_longs[0] = ntohl((key).key_longs[0]);			\
+    (key).key_longs[1] = ntohl((key).key_longs[1]);			\
+    (key).key_longs[2] = ntohl((key).key_longs[2]);			\
+    (key).key_longs[3] = ntohl((key).key_longs[3]);			\
+}
+
+#define HTON_KEY(key) {							\
+    (key).key_longs[0] = htonl((key).key_longs[0]);			\
+    (key).key_longs[1] = htonl((key).key_longs[1]);			\
+    (key).key_longs[2] = htonl((key).key_longs[2]);			\
+    (key).key_longs[3] = htonl((key).key_longs[3]);			\
+}
+
+/*
+ * Structure used to transmit or store a token or a key.
+ */
+typedef union {
+    key_t	si_key;
+    key_t	si_token;
+} secure_info_t, *secure_info_ptr_t;
+
+/*
+ * Security Level of ports and messages.
+ */
+#define PORT_NOT_SECURE		0
+#define MESSAGE_NOT_SECURE	0
+
+#endif	/* _KEY_DEFS_ */
diff --git a/libsyscall/mach/servers/ls_defs.h b/libsyscall/mach/servers/ls_defs.h
new file mode 100644
index 000000000..5415727fb
--- /dev/null
+++ b/libsyscall/mach/servers/ls_defs.h
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1989 Carnegie-Mellon University
+ * Copyright (c) 1988 Carnegie-Mellon University
+ * Copyright (c) 1987 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+
+/*
+ * Definitions for the logstat module.
+ */
+
+
+#ifndef	_LS_DEFS_
+#define	_LS_DEFS_
+
+#include <sys/types.h>
+
+/*
+ * Definition for a log record.
+ */
+typedef	struct	{
+	long	code;
+	long	thread;
+	long	a1;
+	long	a2;
+	long	a3;
+	long	a4;
+	long	a5;
+	long	a6;
+} log_rec_t;
+
+typedef	log_rec_t	*log_ptr_t;
+
+/*
+ * Statistics record.
+ */
+typedef	struct {
+	int	datagram_pkts_sent;
+	int	datagram_pkts_rcvd;
+	int	srr_requests_sent;
+	int	srr_bcasts_sent;
+	int	srr_requests_rcvd;
+	int	srr_bcasts_rcvd;
+	int	srr_replies_sent;
+	int	srr_replies_rcvd;
+	int	srr_retries_sent;
+	int	srr_retries_rcvd;
+	int	srr_cfailures_sent;
+	int	srr_cfailures_rcvd;
+	int	deltat_dpkts_sent;
+	int	deltat_acks_rcvd;
+	int	deltat_dpkts_rcvd;
+	int	deltat_acks_sent;
+	int	deltat_oldpkts_rcvd;
+	int	deltat_oospkts_rcvd;
+	int	deltat_retries_sent;
+	int	deltat_retries_rcvd;
+	int	deltat_cfailures_sent;
+	int	deltat_cfailures_rcvd;
+	int	deltat_aborts_sent;
+	int	deltat_aborts_rcvd;
+	int	vmtp_requests_sent;
+	int	vmtp_requests_rcvd;
+	int	vmtp_replies_sent;
+	int	vmtp_replies_rcvd;
+	int	ipc_in_messages;
+	int	ipc_out_messages;
+	int	ipc_unblocks_sent;
+	int	ipc_unblocks_rcvd;
+	int	pc_requests_sent;
+	int	pc_requests_rcvd;
+	int	pc_replies_rcvd;
+	int	pc_startups_rcvd;
+	int	nn_requests_sent;
+	int	nn_requests_rcvd;
+	int	nn_replies_rcvd;
+	int	po_ro_hints_sent;
+	int	po_ro_hints_rcvd;
+	int	po_token_requests_sent;
+	int	po_token_requests_rcvd;
+	int	po_token_replies_rcvd;
+	int	po_xfer_requests_sent;
+	int	po_xfer_requests_rcvd;
+	int	po_xfer_replies_rcvd;
+	int	po_deaths_sent;
+	int	po_deaths_rcvd;
+	int	ps_requests_sent;
+	int	ps_requests_rcvd;
+	int	ps_replies_rcvd;
+	int	ps_auth_requests_sent;
+	int	ps_auth_requests_rcvd;
+	int	ps_auth_replies_rcvd;
+	int	mallocs_or_vm_allocates;
+	int	mem_allocs;
+	int	mem_deallocs;
+	int	mem_allocobjs;
+	int	mem_deallocobjs;
+	int	pkts_encrypted;
+	int	pkts_decrypted;
+	int	vmtp_segs_encrypted;
+	int	vmtp_segs_decrypted;
+	int	tcp_requests_sent;
+	int	tcp_replies_sent;
+	int	tcp_requests_rcvd;
+	int	tcp_replies_rcvd;
+	int	tcp_send;
+	int	tcp_recv;
+	int	tcp_connect;
+	int	tcp_accept;
+	int	tcp_close;
+} stat_t;
+
+typedef	stat_t	*stat_ptr_t;
+
+
+/*
+ * Debugging flags record.
+ */
+typedef	struct {
+	int	print_level;
+	int	ipc_in;
+	int	ipc_out;
+	int	tracing;
+	int	vmtp;
+	int	netname;
+	int	deltat;
+	int	tcp;
+	int	mem;
+} debug_t;
+
+typedef	debug_t	*debug_ptr_t;
+
+
+/*
+ * Parameters record.
+ */
+typedef struct {
+    	int	srr_max_tries;
+	int	srr_retry_sec;
+	int	srr_retry_usec;
+    	int	deltat_max_tries;
+	int	deltat_retry_sec;
+	int	deltat_retry_usec;
+	int	deltat_msg_life;
+	int	pc_checkup_interval;
+	int	crypt_algorithm;
+	int	transport_default;
+	int	conf_network;
+	int	conf_netport;
+	int	timer_quantum;
+	int	tcp_conn_steady;
+	int	tcp_conn_opening;
+	int	tcp_conn_max;
+	int	compat;
+	int	syslog;
+	int	old_nmmonitor;
+} param_t;
+
+typedef param_t *param_ptr_t;
+
+
+/*
+ * Port statistics record.
+ */
+typedef struct {
+	u_int	port_id;
+	u_int	alive;
+	u_int	nport_id_high;
+	u_int	nport_id_low;
+	u_int	nport_receiver;
+	u_int	nport_owner;
+	u_int	messages_sent;
+	u_int	messages_rcvd;
+	u_int	send_rights_sent;
+	u_int	send_rights_rcvd_sender;
+	u_int	send_rights_rcvd_recown;
+	u_int	rcv_rights_xferd;
+	u_int	own_rights_xferd;
+	u_int	all_rights_xferd;
+	u_int	tokens_sent;
+	u_int	tokens_requested;
+	u_int	xfer_hints_sent;
+	u_int	xfer_hints_rcvd;
+} port_stat_t, *port_stat_ptr_t;
+
+extern port_stat_ptr_t	port_stat_cur;
+extern port_stat_ptr_t	port_stat_end;
+extern struct mutex	port_stat_lock;
+
+
+/*
+ * Types for the mem_list operation.
+ *
+ * XXX These must be faked, because we cannot include mem.h here
+ * (mutual includes).
+ */
+typedef char			*mem_class_ptr_t;
+typedef char			*mem_nam_ptr_t;
+typedef int			*mem_bucket_ptr_t;
+
+	
+/*
+ * Definitions for print_level.
+ */
+#define	LS_PRINT_NEVER		5
+#define	LS_PRINT_LOG		3
+#define	LS_PRINT_ALWAYS		0
+
+#endif	/* _LS_DEFS_ */
diff --git a/libsyscall/mach/servers/netname.defs b/libsyscall/mach/servers/netname.defs
new file mode 100644
index 000000000..f92903498
--- /dev/null
+++ b/libsyscall/mach/servers/netname.defs
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1987 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+
+/*
+ * Mig definitions for Network Name Service.
+ */
+
+/*
+ * HISTORY:
+ * 30-May-87  Robert Sansom (rds) at Carnegie Mellon University
+ *	Changes for the new mig.
+ *
+ * 20-Oct-86  Robert Sansom (rds) at Carnegie Mellon University
+ *	New network name service interface.
+ *	Added a serverprefix.
+ */
+
+#include <mach/std_types.defs>
+
+import <servers/netname_defs.h>;
+
+subsystem netname 1040;
+
+serverprefix _;
+
+
+type netname_name_t	= c_string[*:80];
+
+routine netname_check_in(server_port	: mach_port_t;
+			port_name	: netname_name_t;
+			signature	: mach_port_t;
+			port_id		: mach_port_t);
+
+routine netname_look_up(server_port	: mach_port_t;
+			host_name	: netname_name_t;
+			port_name	: netname_name_t;
+		    out port_id		: mach_port_t);
+
+routine netname_check_out(server_port	: mach_port_t;
+			port_name	: netname_name_t;
+			signature	: mach_port_t);
+
+routine netname_version(server_port	: mach_port_t;
+		    out	version		: netname_name_t);
+
diff --git a/libsyscall/mach/servers/netname_defs.h b/libsyscall/mach/servers/netname_defs.h
new file mode 100644
index 000000000..0c39ad41d
--- /dev/null
+++ b/libsyscall/mach/servers/netname_defs.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1987 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+
+/*
+ * Definitions for the mig interface to the network name service.
+ */
+
+/*
+ * HISTORY:
+ * 28-Jul-88  Mary R. Thompson (mrt) at Carnegie Mellon
+ *	Copied definitions of NAME_NOT_YOURS and NAME_NOT_CHECKED_IN
+ *	from the old netname_defs.h so that old code would not break
+ *
+ *  8-Mar-88  Daniel Julin (dpj) at Carnegie-Mellon University
+ *	Added NETNAME_INVALID_PORT.
+ *
+ * 28-Feb-88  Daniel Julin (dpj) at Carnegie-Mellon University
+ *	Added NETNAME_PENDING.
+ *
+ * 23-Dec-86  Robert Sansom (rds) at Carnegie Mellon University
+ *	Copied from the previous version of the network server.
+ *
+ */
+
+#ifndef	_NETNAME_DEFS_
+#define	_NETNAME_DEFS_
+
+#define NETNAME_SUCCESS		(0)
+#define	NETNAME_PENDING		(-1)
+#define NETNAME_NOT_YOURS	(1000)
+#define NAME_NOT_YOURS		(1000)
+#define NETNAME_NOT_CHECKED_IN	(1001)
+#define NAME_NOT_CHECKED_IN	(1001)
+#define NETNAME_NO_SUCH_HOST	(1002)
+#define NETNAME_HOST_NOT_FOUND	(1003)
+#define	NETNAME_INVALID_PORT	(1004)
+
+typedef char netname_name_t[80];
+
+#endif /* NETNAME_DEFS_ */
diff --git a/libsyscall/mach/servers/nm_defs.h b/libsyscall/mach/servers/nm_defs.h
new file mode 100644
index 000000000..80baa6ab2
--- /dev/null
+++ b/libsyscall/mach/servers/nm_defs.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1987 Carnegie-Mellon University
+ * All rights reserved.  The CMU software License Agreement specifies
+ * the terms and conditions for use and redistribution.
+ */
+
+/*
+ * Random definitions for the network service that everyone needs!
+ */
+
+/*
+ * HISTORY:
+ * 27-Mar-90  Gregg Kellogg (gk) at NeXT
+ *	include <sys/netport.h> rather than <sys/ipc_netport.h>
+ *
+ * 24-Aug-88  Daniel Julin (dpj) at Carnegie-Mellon University
+ *	Replace sys/mach_ipc_netport.h with kern/ipc_netport.h. Sigh.
+ *
+ * 24-May-88  Daniel Julin (dpj) at Carnegie-Mellon University
+ *	Replace mach_ipc_vmtp.h with mach_ipc_netport.h.
+ *
+ *  4-Sep-87  Daniel Julin (dpj) at Carnegie-Mellon University
+ *	Fixed for new kernel include files which declare a lot
+ *	of network server stuff internally, because of the NETPORT
+ *	option.
+ *
+ *  5-Nov-86  Robert Sansom (rds) at Carnegie-Mellon University
+ *	Started.
+ *
+ */
+
+#ifndef	_NM_DEFS_
+#define	_NM_DEFS_
+
+/*
+ * netaddr_t is declared with the kernel files,
+ * in <sys/netport.h>.
+ */
+#include	<sys/netport.h>
+
+#ifdef	notdef
+typedef unsigned long	netaddr_t;
+#endif	/* notdef */
+
+typedef union {
+    struct {
+	unsigned char ia_net_owner;
+	unsigned char ia_net_node_type;
+	unsigned char ia_host_high;
+	unsigned char ia_host_low;
+    } ia_bytes;
+    netaddr_t ia_netaddr;
+} ip_addr_t;
+
+#endif	/* _NM_DEFS_ */
+
diff --git a/libsyscall/mach/slot_name.c b/libsyscall/mach/slot_name.c
new file mode 100644
index 000000000..a059c1c59
--- /dev/null
+++ b/libsyscall/mach/slot_name.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *	File:	slot_name.c
+ *	Author:	Avadis Tevanian, Jr.
+ *
+ *	Copyright (C) 1987, Avadis Tevanian, Jr.
+ *
+ *	Convert machine slot values to human readable strings.
+ *
+ * HISTORY
+ * 26-Jan-88  Mary Thompson (mrt) at Carnegie Mellon
+ *	added case for CUP_SUBTYPE_RT_APC
+ *
+ * 28-Feb-87  Avadis Tevanian (avie) at Carnegie-Mellon University
+ *	Created.
+ *
+ */
+
+#include <mach/mach.h>
+#include <mach-o/arch.h>
+#include <stddef.h>
+
+/*
+ *	Convert the specified cpu_type/cpu_subtype pair to their
+ *	human readable form.
+ */
+void slot_name(cpu_type, cpu_subtype, cpu_name, cpu_subname)
+        cpu_type_t	cpu_type;
+        cpu_subtype_t	cpu_subtype;
+        char		**cpu_name, **cpu_subname;
+{
+        register char	*name = "Unknown CPU";
+        register char	*subname = "";
+        const NXArchInfo *ai = NXGetArchInfoFromCpuType(cpu_type, cpu_subtype);
+        if (ai != NULL) {
+            name = (char *)ai->name;
+            subname = (char *)ai->description;
+        }
+        *cpu_name = name;
+        *cpu_subname = subname;
+}
+
+kern_return_t msg_rpc(void) {
+	return KERN_FAILURE;
+}
+
+kern_return_t msg_send(void) {
+	return KERN_FAILURE;
+}
+
+kern_return_t msg_receive(void) {
+	return KERN_FAILURE;
+}
+
+mach_port_t task_self_(void) {
+	return mach_task_self();
+}
+
+mach_port_t host_self(void) {
+	return mach_host_self();
+}
+ 
diff --git a/libsyscall/mach/task.defs b/libsyscall/mach/task.defs
new file mode 100644
index 000000000..e35accca3
--- /dev/null
+++ b/libsyscall/mach/task.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/task.defs>
diff --git a/libsyscall/mach/thread_act.defs b/libsyscall/mach/thread_act.defs
new file mode 100644
index 000000000..2e064ff4b
--- /dev/null
+++ b/libsyscall/mach/thread_act.defs
@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <mach/thread_act.defs>
diff --git a/libsyscall/mach/vm_map.defs b/libsyscall/mach/vm_map.defs
new file mode 100644
index 000000000..d0562dbed
--- /dev/null
+++ b/libsyscall/mach/vm_map.defs
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#if defined(__LP64__)
+/*
+ * In an LP64 environment, the traditional Mach VM interface names are
+ * really just a second instance of the "wide" Mach VM interfaces.
+ *
+ * The _MACH_VM_PUBLISH_AS_LOCAL_ flag triggers mach_vm.defs to export
+ * the local names instead.
+ */
+#define _MACH_VM_PUBLISH_AS_LOCAL_
+#include <mach/mach_vm.defs>
+#else
+#include <mach/vm_map.defs>
+#endif
diff --git a/libsyscall/mach/x86_64/Makefile.inc b/libsyscall/mach/x86_64/Makefile.inc
new file mode 100644
index 000000000..475e5a5b8
--- /dev/null
+++ b/libsyscall/mach/x86_64/Makefile.inc
@@ -0,0 +1,3 @@
+.PATH: ${.CURDIR}/mach/x86_64
+
+MDSRCS += mach_absolute_time.S
diff --git a/libsyscall/mach/x86_64/mach_absolute_time.S b/libsyscall/mach/x86_64/mach_absolute_time.S
new file mode 100644
index 000000000..7c53025b6
--- /dev/null
+++ b/libsyscall/mach/x86_64/mach_absolute_time.S
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#include <machine/cpu_capabilities.h>
+
+
+	.text
+	.align	2
+	.globl	_mach_absolute_time
+_mach_absolute_time:
+	movq	$(_COMM_PAGE_NANOTIME), %rax
+	jmp	*%rax
diff --git a/makedefs/MakeInc.cmd b/makedefs/MakeInc.cmd
index 52ee80340..6af46de7f 100644
--- a/makedefs/MakeInc.cmd
+++ b/makedefs/MakeInc.cmd
@@ -10,6 +10,7 @@ CP = /bin/cp
 LN = /bin/ln -fs
 CAT = /bin/cat
 MKDIR = /bin/mkdir -p
+FIND = /usr/bin/find
 
 TAR = /usr/bin/gnutar
 STRIP = /usr/bin/strip
@@ -23,3 +24,9 @@ SEG_HACK = $(NEXT_ROOT)/usr/local/bin/seg_hack
 UNIFDEF   = /usr/bin/unifdef
 DECOMMENT = /usr/local/bin/decomment
 
+DSYMUTIL = /usr/bin/dsymutil
+CTFCONVERT = /usr/local/bin/ctfconvert
+CTFMERGE = /usr/local/bin/ctfmerge
+CTFSCRUB = /usr/local/bin/ctfdump -r
+
+# vim: set ft=make:
diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def
index 937a465c3..b08213bc0 100644
--- a/makedefs/MakeInc.def
+++ b/makedefs/MakeInc.def
@@ -20,7 +20,7 @@ endif
 # Component List
 #
 ifndef COMPONENT_LIST 
-export COMPONENT_LIST 	= osfmk bsd iokit pexpert libkern libsa
+export COMPONENT_LIST 	= osfmk bsd libkern iokit pexpert libsa security
 export COMPONENT_LIST_UC := $(shell printf "%s" "$(COMPONENT_LIST)" | $(TR) a-z A-Z)
 endif
 ifndef	COMPONENT
@@ -35,8 +35,10 @@ endif
 
 # Architecture options
 #
-#  supported configurations : PPC I386
-#
+ifndef SUPPORTED_ARCH_CONFIGS
+export SUPPORTED_ARCH_CONFIGS = PPC I386 ARM
+endif
+
 ifndef ARCH_CONFIGS
 ifdef RC_ARCHS
 export ARCH_CONFIGS 	:= $(shell printf "%s" "$(RC_ARCHS)" | $(TR) a-z A-Z)
@@ -50,40 +52,131 @@ export ARCH_CONFIG_LC 	:= $(shell printf "%s" "$(ARCH_CONFIG)" | $(TR) A-Z a-z)
 endif
 endif
 
-
+#
 # Kernel Configuration options  
 #
-# supported configurations : RELEASE DEBUG PROFILE
-# By default, make wll build RELEASE, otherwise the value of KERNEL_CONFIG
-# will be used as kernel configuration. If KERNEL_CONFIGS (plural) is set
-# it will override KERNEL_CONFIG. Make sure to set KERNEL_CONFIGS because
-# build_all rule loops over it when building.
+ifndef SUPPORTED_KERNEL_CONFIGS
+export SUPPORTED_KERNEL_CONFIGS = RELEASE DEVELOPMENT DEBUG PROFILE
+endif
 
-ifndef KERNEL_CONFIGS
-ifndef KERNEL_CONFIG
-export KERNEL_CONFIGS 	= RELEASE
-else
-export KERNEL_CONFIGS = $(KERNEL_CONFIG)
+ifndef DEFAULT_KERNEL_CONFIG
+export DEFAULT_KERNEL_CONFIG = RELEASE
 endif
+
+# If KERNEL_CONFIGS is specified it should override KERNEL_CONFIG.
+# If KERNEL_CONFIG is specified it will override the default. Will quit with
+# error if more than one config is specified.
+# If DEFAULT_KERNEL_CONFIG is not specified then it will be built RELEASE.
+ifndef KERNEL_CONFIGS
+	ifndef KERNEL_CONFIG
+	export KERNEL_CONFIGS 	= $(DEFAULT_KERNEL_CONFIG)
+	else
+	export KERNEL_CONFIGS	= $(KERNEL_CONFIG)
+	endif
 endif
 
 ifndef KERNEL_CONFIG
 export KERNEL_CONFIG 	= $(firstword $(KERNEL_CONFIGS))
 endif
 
+ifneq ($(words $(KERNEL_CONFIG)), 1)
+$(error There were $(words $(KERNEL_CONFIG)) parameters passed to KERNEL_CONFIG = $(KERNEL_CONFG). \
+		Are you sure? To specify multiple configurations please use KERNEL_CONFIGS)
+endif
+
 #
-# Kernel Configuration to install
+# Machine Configuration options  
 #
-#  supported install architecture : PPC I386
+# ppc supported configurations : none
+# i386 supported configurations : none
+# arm supported configurations : LN2410SBC MX31ADS INTEGRATORCP S5I3000SMDK S5L8900XFPGA S5L8900XRB OLOCREEK
 #
-export INSTALL_TYPE 	= RELEASE
-ifndef INSTALL_ARCHS
-ifdef RC_ARCHS
-export INSTALL_ARCHS 	= $(ARCH_CONFIGS)
+ifndef SUPPORTED_MACHINE_CONFIGS
+export SUPPORTED_MACHINE_CONFIGS = LN2410SBC MX31ADS INTEGRATORCP S5I3000SMDK S5L8900XFPGA S5L8900XRB OLOCREEK DEFAULT
+endif
+
+export DEFAULT_ARM_MACHINE_CONFIG 	= S5L8900XRB
+
+ifndef MACHINE_CONFIG
+export MACHINE_CONFIG 	= DEFAULT
+endif
+
+ifndef MACHINE_FLAGS_LN2410SBC
+export MACHINE_FLAGS_LN2410SBC = -DARM_BOARD_CONFIG_LN2410_920T
+endif
+ifndef MACHINE_FLAGS_MX31ADS
+export MACHINE_FLAGS_MX31ADS = -DARM_BOARD_CONFIG_MX31ADS_1136JFS
+endif
+ifndef MACHINE_FLAGS_INTEGRATORCP
+export MACHINE_FLAGS_INTEGRATORCP = -DARM_BOARD_CONFIG_INTEGRATORCP_1136JFS
+endif
+ifndef MACHINE_FLAGS_S5I3000SMDK
+export MACHINE_FLAGS_S5I3000SMDK = -DARM_BOARD_CONFIG_S5I3000SMDK_1176JZFS
+endif
+ifndef MACHINE_FLAGS_S5L8900XFPGA
+export MACHINE_FLAGS_S5L8900XFPGA = -DARM_BOARD_CONFIG_S5L8900XFPGA_1136JFS
+endif
+ifndef MACHINE_FLAGS_S5L8900XRB
+export MACHINE_FLAGS_S5L8900XRB = -DARM_BOARD_CONFIG_S5L8900XRB
+endif
+ifndef MACHINE_FLAGS_OLOCREEK
+export MACHINE_FLAGS_OLOCREEK = -DARM_BOARD_CONFIG_OLOCREEK
+endif
+ifndef MACHINE_FLAGS_DEFAULT
+export MACHINE_FLAGS_DEFAULT =
+endif
+
+#
+# Target configuration options.  NOTE - target configurations will 
+# override ARCH_CONFIGS and KERNEL_CONFIGS.
+#
+# Target configs come in groups of three parameters.  The first is the 
+# kernel configuration, the second is the architecture configuration,
+# and the third is the machine configuration.  You may pass in as
+# many groups of configurations as you wish.  Each item passed in is
+# seperated by whitespace.
+#
+# Example:
+#	TARGET_CONFIGS="release ppc default debug i386 default release arm MX31ADS"
+# Parameters may be in upper or lower case (they are converted to upper).
+#
+# "default" parameter is a special case.  It means use the default value for 
+# that parameter.  Here are the default values for each configuration:
+#
+# default kernel configuration = DEFAULT_KERNEL_CONFIG
+# default architecture configuration = system architecture where you are running make.
+# default machine configuration for ppc = none at this time.
+# default machine configuration for i386 = none at this time.
+# default machine configuration for arm = "S5L8900XRB".
+#
+ifndef TARGET_CONFIGS_UC
+ifdef TARGET_CONFIGS
+	export TARGET_CONFIGS_UC = $(strip $(shell printf "%s" "$(TARGET_CONFIGS)" | $(TR) a-z A-Z))
+	export MACHINE_CONFIG	= $(word 3, $(TARGET_CONFIGS_UC))
+	export DEFAULT_KERNEL_CONFIG = $(word 1, $(TARGET_CONFIGS_UC))
 else
-export INSTALL_ARCHS 	= $(ARCH_CONFIGS)
+	# generate TARGET_CONFIGS using KERNEL_CONFIGS and ARCH_CONFIGS and MACHINE_CONFIG (which defaults to "DEFAULT")
+	temp_list = $(foreach my_kern_config, $(KERNEL_CONFIGS), $(my_kern_config) arch_slot $(MACHINE_CONFIG))
+	export TARGET_CONFIGS = $(strip $(foreach my_arch_config, $(ARCH_CONFIGS), $(subst arch_slot,$(my_arch_config),$(temp_list))))
+	export TARGET_CONFIGS_UC := $(shell printf "%s" "$(TARGET_CONFIGS)" | $(TR) a-z A-Z)
+	export MACHINE_CONFIG	= $(word 3, $(TARGET_CONFIGS_UC))
+	export DEFAULT_KERNEL_CONFIG = $(word 1, $(TARGET_CONFIGS_UC))
 endif
-export INSTALL_ARCHS_LC := $(shell printf "%s" "$(ARCH_CONFIGS)" | $(TR) A-Z a-z)
+endif
+
+export MACHINE_CONFIG_LC := $(shell printf "%s" "$(MACHINE_CONFIG)" | $(TR) A-Z a-z)
+export KERNEL_CONFIG_LC := $(shell printf "%s" "$(KERNEL_CONFIG)" | $(TR) A-Z a-z)
+
+#
+# Kernel Configuration to install
+#
+#  supported install architecture : PPC I386 ARM
+#
+export INSTALL_TYPE 	= $(DEFAULT_KERNEL_CONFIG)
+
+ifndef INSTALL_ARCHS
+export INSTALL_ARCHS 	= $(strip $(foreach my_config, $(SUPPORTED_ARCH_CONFIGS), $(findstring $(my_config), $(TARGET_CONFIGS_UC))))
+export INSTALL_ARCHS_LC := $(shell printf "%s" "$(INSTALL_ARCHS)" | $(TR) A-Z a-z)
 endif
 
 export INSTALL_ARCH_DEFAULT	= PPC
@@ -91,7 +184,7 @@ export INSTALL_ARCH_DEFAULT	= PPC
 #
 # Standard defines list
 #
-export DEFINES = -DAPPLE -DNeXT -DKERNEL -DKERNEL_PRIVATE -DXNU_KERNEL_PRIVATE -DPRIVATE -D__MACHO__=1 -Dvolatile=__volatile $(IDENT)
+export DEFINES = -DAPPLE -DKERNEL -DKERNEL_PRIVATE -DXNU_KERNEL_PRIVATE -DPRIVATE -D__MACHO__=1 -Dvolatile=__volatile $(IDENT)
 
 #
 # Compiler command
@@ -128,14 +221,27 @@ export CXXWARNFLAGS ?= $(CXXWARNFLAGS_STD)
 
 
 #
-# Setup for parallel sub-makes when doing an RC build
+# Setup for parallel sub-makes based on 2 times number of logical CPUs
 #
 ifndef MAKEJOBS
-ifeq "YES" "$(RC_XBS)"
-export MAKEJOBS = --jobs=3
-endif
+export MAKEJOBS = --jobs=$(shell expr `/usr/sbin//sysctl -n hw.logicalcpu` \* 2)
 endif
 
+#
+# Default ARCH_FLAGS, for use with compiler/linker/assembler/mig drivers
+
+ARCH_FLAGS_PPC		  = -arch ppc
+ARCH_FLAGS_I386		  = -arch i386
+ARCH_FLAGS_ARM		  = $($(addsuffix $(MACHINE_CONFIG),ARCH_FLAGS_ARM_))
+
+ARCH_FLAGS_ARM_LN2410SBC	  = -arch arm
+ARCH_FLAGS_ARM_MX31ADS		  = -arch armv6
+ARCH_FLAGS_ARM_INTEGRATORCP	  = -arch armv6
+ARCH_FLAGS_ARM_S5I3000SMDK	  = -arch armv6
+ARCH_FLAGS_ARM_S5L8900XFPGA	  = -arch armv6
+ARCH_FLAGS_ARM_S5L8900XRB	  = -arch armv6
+ARCH_FLAGS_ARM_OLOCREEK	 	  = -arch arm
+
 #
 # Default CFLAGS
 #
@@ -143,26 +249,67 @@ ifdef RC_CFLAGS
 export OTHER_CFLAGS	= $(subst $(addprefix -arch ,$(RC_ARCHS)),,$(RC_CFLAGS))
 endif
 
-export CFLAGS_GEN = -static -g -nostdinc -nostdlib -no-cpp-precomp \
-	-fno-builtin -finline -fno-keep-inline-functions -msoft-float \
-	-fsigned-bitfields $(OTHER_CFLAGS) -force_cpusubtype_ALL
+export DSYMBUILDDIR = ./Contents/Resources/DWARF/
+
+#
+# We must not use -fno-keep-inline-functions, or it will remove the dtrace
+# probes from the kernel.
+#
+export CFLAGS_GEN = -static $(DEBUG_CFLAGS) -nostdinc -nostdlib -no-cpp-precomp \
+	-fno-builtin -finline -msoft-float \
+	-fsigned-bitfields $(OTHER_CFLAGS)
 
 export CFLAGS_RELEASE 	= 
+export CFLAGS_DEVELOPMENT 	=
 export CFLAGS_DEBUG 	= 
 export CFLAGS_PROFILE 	=  -pg
 
-export CFLAGS_PPC 	= -arch ppc -Dppc -DPPC -D__PPC__ -DPAGE_SIZE_FIXED
-export CFLAGS_I386 	= -arch i386 -Di386 -DI386 -D__I386__ -DPAGE_SIZE_FIXED
+ifeq ($(ARCH_CONFIG),ARM)
+BUILD_STABS = 1
+endif
+
+ifeq ($(BUILD_STABS),1)
+export CFLAGS_PPC 	= -Dppc -DPPC -D__PPC__ -DPAGE_SIZE_FIXED \
+				-mno-altivec -gstabs+ -force_cpusubtype_ALL
+export CFLAGS_I386 	= -Di386 -DI386 -D__I386__ \
+				-DPAGE_SIZE_FIXED -gstabs+ -force_cpusubtype_ALL
+export CFLAGS_ARM 	= -Darm -DARM -D__ARM__ -DPAGE_SIZE_FIXED \
+	-fno-strict-aliasing -gstabs+ -fno-keep-inline-functions
+export BUILD_DWARF = 0
+export BUILD_STABS = 1
+else
+export CFLAGS_PPC 	= -Dppc -DPPC -D__PPC__ -DPAGE_SIZE_FIXED \
+				-mno-altivec -gdwarf-2 -force_cpusubtype_ALL
+export CFLAGS_I386 	= -Di386 -DI386 -D__I386__ \
+				-DPAGE_SIZE_FIXED -gdwarf-2 -force_cpusubtype_ALL
+export CFLAGS_ARM 	= -Darm -DARM -D__ARM__ -DPAGE_SIZE_FIXED \
+	-fno-strict-aliasing -gdwarf-2 -fno-keep-inline-functions
+export BUILD_DWARF = 1
+export BUILD_STABS = 0
+endif
+ifeq (-arch armv6,$(ARCH_FLAGS_ARM))
+CFLAGS_ARM		+= -mthumb
+endif
+
+export CFLAGS_RELEASEPPC = -O2 -mcpu=750 -mmultiple
+export CFLAGS_RELEASE_TRACEPPC = -O2 -mcpu=750 -mmultiple
+export CFLAGS_DEVELOPMENTPPC = -O2 -mcpu=750 -mmultiple
+export CFLAGS_DEBUGPPC = -O2 -mcpu=750 -mmultiple
+export CFLAGS_PROFILEPPC = -O2 -mcpu=750 -mmultiple
 
-export CFLAGS_RELEASEPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
-export CFLAGS_RELEASE_TRACEPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
-export CFLAGS_DEBUGPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
-export CFLAGS_PROFILEPPC = -O2 -mcpu=750 -mmultiple -fschedule-insns
 export CFLAGS_RELEASEI386 = -Os
+export CFLAGS_DEVELOPMENTI386 = -Os
 export CFLAGS_DEBUGI386 = -Os
 export CFLAGS_PROFILEI386 = -Os
 
+export CFLAGS_RELEASEARM = -O2
+export CFLAGS_DEVELOPMENTARM = -O2
+export CFLAGS_DEBUGARM = -O2
+export CFLAGS_PROFILEARM = -O2
+
 export CFLAGS 	= $(CFLAGS_GEN) \
+		  $($(addsuffix $(MACHINE_CONFIG),MACHINE_FLAGS_)) \
+		  $($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
 		  $($(addsuffix $(ARCH_CONFIG),CFLAGS_)) \
 		  $($(addsuffix $(KERNEL_CONFIG),CFLAGS_)) \
 		  $($(addsuffix $(ARCH_CONFIG), $(addsuffix $(KERNEL_CONFIG),CFLAGS_))) \
@@ -187,16 +334,20 @@ S_KCC	= $(CC)
 #
 # Default SFLAGS
 #
-export SFLAGS_GEN = -static -D__ASSEMBLER__ -force_cpusubtype_ALL $(OTHER_CFLAGS)
+export SFLAGS_GEN = -static -D__ASSEMBLER__ $(OTHER_CFLAGS)
 
 export SFLAGS_RELEASE 	= 
+export SFLAGS_DEVELOPMENT 	= 
 export SFLAGS_DEBUG 	= 
 export SFLAGS_PROFILE 	= 
 
-export SFLAGS_PPC 	= $(CFLAGS_PPC)
-export SFLAGS_I386 	= $(CFLAGS_I386)
+export SFLAGS_PPC 	= $(CFLAGS_PPC) -force_cpusubtype_ALL
+export SFLAGS_I386	= $(CFLAGS_I386)
+export SFLAGS_ARM	= $(CFLAGS_ARM)
 
 export SFLAGS 	= $(SFLAGS_GEN) \
+		  $($(addsuffix $(MACHINE_CONFIG),MACHINE_FLAGS_)) \
+		  $($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
 		  $($(addsuffix $(ARCH_CONFIG),SFLAGS_)) \
 		  $($(addsuffix $(KERNEL_CONFIG),SFLAGS_)) \
 		  $(DEFINES)
@@ -212,22 +363,23 @@ LD	= $(KC++) -nostdlib
 export LDFLAGS_COMPONENT_GEN = -static -r $(COMP_LDFLAGS_COMPONENT_GEN)
 
 export LDFLAGS_COMPONENT_RELEASE =  $(COMP_LDFLAGS_COMPONENT_RELEASE)
+export LDFLAGS_COMPONENT_DEVELOPMENT =  $(COMP_LDFLAGS_COMPONENT_DEVELOPMENT)
 export LDFLAGS_COMPONENT_DEBUG   =  $(COMP_LDFLAGS_COMPONENT_DEBUG)
 export LDFLAGS_COMPONENT_PROFILE =  $(COMP_LDFLAGS_COMPONENT_PROFILE)
 
-export LDFLAGS_COMPONENT_PPC 	= -arch ppc $(COMP_LDFLAGS_COMPONENT_PPC)
-export LDFLAGS_COMPONENT_I386 	= -arch i386 $(COMP_LDFLAGS_COMPONENT_i386)
+export LDFLAGS_COMPONENT_PPC 	= $(COMP_LDFLAGS_COMPONENT_PPC) -force_cpusubtype_ALL
+export LDFLAGS_COMPONENT_I386 	= $(COMP_LDFLAGS_COMPONENT_i386)
+export LDFLAGS_COMPONENT_ARM 	= $(COMP_LDFLAGS_COMPONENT_ARM)
 
 export LDFLAGS_COMPONENT	= $(LDFLAGS_COMPONENT_GEN) \
+		  $($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
 		  $($(addsuffix $(ARCH_CONFIG),LDFLAGS_COMPONENT_)) \
 		  $($(addsuffix $(KERNEL_CONFIG),LDFLAGS_COMPONENT_))
 
 export LDFLAGS_KERNEL_GEN = \
 	-static \
 	-fapple-kext \
-	-force_cpusubtype_ALL \
 	-Wl,-e,__start \
-	-Wl,-segalign,0x1000 \
 	-Wl,-sectalign,__TEXT,__text,0x1000 \
 	-Wl,-sectalign,__DATA,__common,0x1000 \
 	-Wl,-sectalign,__DATA,__bss,0x1000 \
@@ -236,22 +388,30 @@ export LDFLAGS_KERNEL_GEN = \
 	-Wl,-sectcreate,__PRELINK,__info,/dev/null
 
 export LDFLAGS_KERNEL_RELEASE 	=
+export LDFLAGS_KERNEL_DEVELOPMENT 	=
 #  -noseglinkedit
 export LDFLAGS_KERNEL_DEBUG 	= 
 export LDFLAGS_KERNEL_PROFILE 	= 
 
 export LDFLAGS_KERNEL_PPC      = \
-	-arch ppc \
+	-force_cpusubtype_ALL \
+	-Wl,-new_linker \
 	-Wl,-segaddr,__VECTORS,0x0 \
 	-Wl,-segaddr,__HIB,0x7000  \
 	-Wl,-segaddr,__TEXT,0xe000
  
 export LDFLAGS_KERNEL_I386     = \
-	-arch i386 \
+	-Wl,-new_linker \
 	-Wl,-segaddr,__HIB,0x100000 \
-	-Wl,-segaddr,__TEXT,0x111000
+	-Wl,-segaddr,__TEXT,0x111000 
+
+export LDFLAGS_KERNEL_ARM     = \
+	-Wl,-segaddr,__HIB,0xC0000000 \
+	-Wl,-segaddr,__TEXT,0xC0008000
 
 export LDFLAGS_KERNEL	= $(LDFLAGS_KERNEL_GEN) \
+		  $($(addsuffix $(MACHINE_CONFIG),MACHINE_FLAGS_)) \
+		  $($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_)) \
 		  $($(addsuffix $(ARCH_CONFIG),LDFLAGS_KERNEL_)) \
 		  $($(addsuffix $(KERNEL_CONFIG),LDFLAGS_KERNEL_))
 
@@ -275,7 +435,7 @@ export INCFLAGS 	= $(INCFLAGS_LOCAL) $(INCFLAGS_GEN) $(INCFLAGS_IMPORT) $(INCFLA
 #
 # Default MIGFLAGS
 #
-export MIGFLAGS	= $(DEFINES) $(INCFLAGS) $($(addsuffix $(ARCH_CONFIG),CFLAGS_))
+export MIGFLAGS	= $(DEFINES) $(INCFLAGS) $($(addsuffix $(ARCH_CONFIG),CFLAGS_)) $($(addsuffix $(ARCH_CONFIG),ARCH_FLAGS_))
 
 #
 # Default VPATH
@@ -289,7 +449,7 @@ export VPATH_GEN	= .:$(SOURCE):
 export VPATH	 	= $(VPATH_GEN)$(VPATH_IMPORT)$(VPATH_EXTERN)$(VPATH_MAKEFILE)
 
 #
-# Macros that control installation of kernel and it's header files
+# Macros that control installation of kernel and its header files
 #
 # install flags for header files
 # 
@@ -316,10 +476,10 @@ KRESDIR = $(KINCFRAME)/Versions/$(KINCVERS)/Resources
 
 XNU_PRIVATE_UNIFDEF = -UMACH_KERNEL_PRIVATE -UBSD_KERNEL_PRIVATE -UIOKIT_KERNEL_PRIVATE -ULIBKERN_KERNEL_PRIVATE -ULIBSA_KERNEL_PRIVATE -UPEXPERT_KERNEL_PRIVATE -UXNU_KERNEL_PRIVATE
 
-SPINCFRAME_UNIFDEF = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UKERNEL -DPRIVATE
-SINCFRAME_UNIFDEF  = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UKERNEL -UPRIVATE
-KPINCFRAME_UNIFDEF = $(XNU_PRIVATE_UNIFDEF) -DKERNEL_PRIVATE -DPRIVATE -DKERNEL
-KINCFRAME_UNIFDEF  = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UPRIVATE -DKERNEL
+SPINCFRAME_UNIFDEF = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UKERNEL -DPRIVATE -U_OPEN_SOURCE_
+SINCFRAME_UNIFDEF  = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UKERNEL -UPRIVATE -D_OPEN_SOURCE_
+KPINCFRAME_UNIFDEF = $(XNU_PRIVATE_UNIFDEF) -DKERNEL_PRIVATE -DPRIVATE -DKERNEL -U_OPEN_SOURCE_
+KINCFRAME_UNIFDEF  = $(XNU_PRIVATE_UNIFDEF) -UKERNEL_PRIVATE -UPRIVATE -DKERNEL -D_OPEN_SOURCE_
 
 #
 # Compononent Header file destinations
@@ -330,6 +490,7 @@ EXPDIR = EXPORT_HDRS/$(COMPONENT)
 # Strip Flags
 #
 export STRIP_FLAGS_RELEASE	= -S -x 
+export STRIP_FLAGS_DEVELOPMENT	= -S -x 
 export STRIP_FLAGS_RELEASE_TRACE	= -S -x 
 export STRIP_FLAGS_DEBUG	= -S 
 export STRIP_FLAGS_DEBUG_TRACE	= -S 
@@ -337,11 +498,35 @@ export STRIP_FLAGS_PROFILE	= -S -x
 
 export STRIP_FLAGS 	= $($(addsuffix $(KERNEL_CONFIG),STRIP_FLAGS_)) 
 
+#
+# dsymutil flags
+#
+export DSYMUTIL_FLAGS_I386 = --arch=i386
+export DSYMUTIL_FLAGS_PPC = --arch=ppc
+export DSYMUTIL_FLAGS_ARM = --arch=arm
+
+export DSYMUTIL_FLAGS = $($(addsuffix $(ARCH_CONFIG),DSYMUTIL_FLAGS_))
+
 #
 # Man Page destination
 #
 MANDIR = usr/share/man
 
+##
+# Verbosity
+##
+ifeq ($(RC_XBS),YES)
+VERBOSE = YES
+else
+VERBOSE = NO
+endif
+
+ifeq ($(VERBOSE),YES)
+_v = 
+else
+_v = @
+endif
+
 #
 #  This must be here before any rules are possibly defined by the
 #  machine dependent makefile fragment so that a plain "make" command
@@ -351,3 +536,4 @@ MANDIR = usr/share/man
 
 default: all
 
+# vim: set ft=make:
diff --git a/makedefs/MakeInc.dir b/makedefs/MakeInc.dir
index 06f191b1f..f3e156b2e 100644
--- a/makedefs/MakeInc.dir
+++ b/makedefs/MakeInc.dir
@@ -1,9 +1,13 @@
 #
 # Install kernel header files
 #
+ifeq ($(RC_ProjectName),Libsyscall)
+installhdrs:
+	bsdmake -C libsyscall installhdrs
+else # xnu
 installhdrs: exporthdrs installhdrs_mi installhdrs_md 
-	@echo "[ $(SRCROOT) ] make installhdrs installing Kernel.framework"; \
-	kincpath=$(DSTROOT)/$(KINCDIR);					\
+	@echo "[ $(SRCROOT) ] make installhdrs installing Kernel.framework"
+	$(_v)kincpath=$(DSTROOT)/$(KINCDIR);					\
 	krespath=$(DSTROOT)/$(KRESDIR);					\
 	kframepath=$(DSTROOT)/$(KINCFRAME);				\
 	[ -d $$krespath ] || $(MKDIR) $$krespath;			\
@@ -19,6 +23,7 @@ installhdrs: exporthdrs installhdrs_mi installhdrs_md
 	[ -d $(DSTROOT)/$(KPINCDIR) ] || $(MKDIR) $(DSTROOT)/$(KPINCDIR);	\
 	cd $$kframepath; [ -L PrivateHeaders ] ||		 	\
 	    $(LN) Versions/Current/PrivateHeaders PrivateHeaders;
+endif
 
 #
 # Install header files order
@@ -29,8 +34,7 @@ installhdrs: exporthdrs installhdrs_mi installhdrs_md
 # Install machine independent header files
 #
 installhdrs_mi:
-	@echo "[ $(SOURCE) ] make installhdrs_mi ";			\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
 	kernel_config=$(INSTALL_TYPE);					\
 	arch_config=$(INSTALL_ARCH_DEFAULT);					\
 	installinc_dir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
@@ -47,19 +51,28 @@ installhdrs_mi:
 # Install machine dependent kernel header files 
 #
 installhdrs_md:
-	@echo "[ $(SOURCE) ] make installhdrs_md ";			\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
 	kernel_config=$(INSTALL_TYPE);					\
+	machine_config=$(MACHINE_CONFIG);		\
 	for arch_config in $(INSTALL_ARCHS);				\
 	do								\
-	[ -d ${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path} ] || \
-	$(MKDIR) ${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
-	${MAKE} -C ${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path}	\
+	if [ $${arch_config} = ARM ] ; then	\
+		if [ $${machine_config} = DEFAULT ] ; then	\
+			machine_config=$(DEFAULT_ARM_MACHINE_CONFIG);		\
+		fi;		\
+	fi;		\
+	if [ $${machine_config} = DEFAULT ] ; then	\
+		objpath=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
+	else		\
+		objpath=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config}/$${rel_path};	\
+	fi;		\
+	[ -d $${objpath} ] || $(MKDIR) $${objpath};	\
+	${MAKE} -C $${objpath}	\
 		KERNEL_CONFIG=$${kernel_config}				\
 		ARCH_CONFIG=$${arch_config}				\
 		MAKEFILES=${SOURCE}/Makefile				\
 		SOURCE=${SOURCE}/					\
-		TARGET=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path}/	\
+		TARGET=$${objpath}/			\
 		build_installhdrs_md;					\
 	done;
 
@@ -69,8 +82,8 @@ installhdrs_md:
 do_installhdrs_mi:
 
 build_installhdrs_mi::
-	@echo "[ $(SOURCE) ] make build_installhdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	for installinc_subdir in $(INSTINC_SUBDIRS);			\
+	@echo "[ $(SOURCE) ] make build_installhdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
+	$(_v)for installinc_subdir in $(INSTINC_SUBDIRS);			\
 	do								\
 		[ -d $${installinc_subdir} ] || $(MKDIR) $${installinc_subdir};		\
 		${MAKE} -C $${installinc_subdir}			\
@@ -87,8 +100,8 @@ build_installhdrs_mi::
 do_installhdrs_md:
 
 build_installhdrs_md::
-	@echo "[ $(SOURCE) ] make installhdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	for installinc_subdir in $($(addprefix INSTINC_SUBDIRS_, $(ARCH_CONFIG)));		\
+	@echo "[ $(SOURCE) ] make installhdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
+	$(_v)for installinc_subdir in $($(addprefix INSTINC_SUBDIRS_, $(ARCH_CONFIG)));		\
 	do								\
 		[ -d $${installinc_subdir} ] || $(MKDIR) $${installinc_subdir};			\
 		${MAKE} -C $${installinc_subdir}			\
@@ -115,8 +128,7 @@ exporthdrs: exporthdrs_mi exporthdrs_md
 do_exporthdrs_mi:
 
 exporthdrs_mi:
-	@echo "[ ${SOURCE} ] make exporthdrs_mi ";			\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
 	kernel_config=$(INSTALL_TYPE);					\
 	arch_config=$(INSTALL_ARCH_DEFAULT);					\
 	exportinc_dir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
@@ -132,21 +144,57 @@ exporthdrs_mi:
 #
 # Install machine dependent kernel header files 
 #
+# Note - installation of machine dependent kernel header files only occurs for architecture
+# defined in INSTALL_TYPE.  We use skipit variable to skip over architectures that are not
+# equal to what is in the INSTALL_TYPE variable.
+# TARGET_CONFIGS_UC variable holds sets of three configuration options.  The first item in the
+# set is the kernel configuration.  The second item in the set is the architecture and the 
+# third item is the machine configuration.  There may be multiple sets to build.
 exporthdrs_md:
-	@echo "[ $(SOURCE) ] make exporthdrs_md ";			\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
-	kernel_config=$(INSTALL_TYPE);					\
-	for arch_config in $(ARCH_CONFIGS);				\
-	do								\
-	exportinc_dir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
-	[ -d $${exportinc_dir} ] || $(MKDIR) $${exportinc_dir};					\
-	${MAKE} -C $${exportinc_dir}					\
-		KERNEL_CONFIG=$${kernel_config}				\
-		ARCH_CONFIG=$${arch_config}				\
-		MAKEFILES=${SOURCE}/Makefile				\
-		SOURCE=${SOURCE}/					\
-		TARGET=$${exportinc_dir}/				\
-		build_exporthdrs_md;					\
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	my_counter=1;		\
+	for my_config in $(TARGET_CONFIGS_UC);				\
+	do		\
+	if [ $${my_counter} -eq 1 ] ; then	\
+		skipit=0;			\
+		my_counter=2;		\
+		kernel_config=$${my_config};		\
+		if [ $${kernel_config} = DEFAULT ] ; then	\
+			kernel_config=$(DEFAULT_KERNEL_CONFIG);		\
+		fi;		\
+		if [ $${kernel_config} != $(INSTALL_TYPE) ] ; then	\
+			skipit=1;	\
+		fi;		\
+	elif [ $${my_counter} -eq 2 ] ; then	\
+		my_counter=3;		\
+		arch_config=$${my_config};		\
+		if [ $${arch_config} = DEFAULT ] ; then	\
+			arch_config=`arch | $(TR) a-z A-Z`;		\
+		fi;		\
+	else		\
+		my_counter=1;		\
+		machine_config=$${my_config};		\
+		if [ $${skipit} -eq 0 ] ; then	\
+			if [ $${arch_config} = ARM ] ; then	\
+				if [ $${machine_config} = DEFAULT ] ; then	\
+					machine_config=$(DEFAULT_ARM_MACHINE_CONFIG);		\
+				fi;		\
+			fi;		\
+			if [ $${machine_config} = DEFAULT ] ; then	\
+				exportinc_dir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
+			else		\
+				exportinc_dir=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config}/$${rel_path};	\
+			fi;		\
+			[ -d $${exportinc_dir} ] || $(MKDIR) $${exportinc_dir};					\
+			${MAKE} -C $${exportinc_dir}					\
+				KERNEL_CONFIG=$${kernel_config}				\
+				ARCH_CONFIG=$${arch_config}				\
+				MAKEFILES=${SOURCE}/Makefile				\
+				SOURCE=${SOURCE}/					\
+				TARGET=$${exportinc_dir}/				\
+				build_exporthdrs_md;					\
+		fi;		\
+	fi;		\
 	done;
 
 #
@@ -155,8 +203,7 @@ exporthdrs_md:
 do_exporthdrs_mi:
 
 build_exporthdrs_mi:
-	@echo "[ $(SOURCE) ] make build_exporthdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	_TMP_EXPINC_SUBDIRS="$(EXPINC_SUBDIRS)";			\
+	$(_v)_TMP_EXPINC_SUBDIRS="$(EXPINC_SUBDIRS)";			\
 	for exportinc_subdir in $${_TMP_EXPINC_SUBDIRS};		\
 	do								\
 		[ -d $${exportinc_subdir} ] || $(MKDIR) $${exportinc_subdir};				\
@@ -174,8 +221,7 @@ build_exporthdrs_mi:
 do_exporthdrs_md:
 
 build_exporthdrs_md:
-	@echo "[ $(SOURCE) ] make exporthdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	_TMP_exportinc_subdir="$($(addprefix EXPINC_SUBDIRS_, $(ARCH_CONFIG)))";	\
+	$(_v)_TMP_exportinc_subdir="$($(addprefix EXPINC_SUBDIRS_, $(ARCH_CONFIG)))";	\
 	for exportinc_subdir in $${_TMP_exportinc_subdir};			\
 	do								\
 		[ -d $${exportinc_subdir} ] || $(MKDIR) $${exportinc_subdir};				\
@@ -191,8 +237,7 @@ build_exporthdrs_md:
 # Setup pass for  all architectures for all Configuration/Architecture options
 #
 setup:
-	@echo "[ $(SOURCE) ] make setup";					\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
 	for kernel_config in $(KERNEL_CONFIGS);				\
 	do								\
 	for arch_config in $(ARCH_CONFIGS);				\
@@ -212,8 +257,7 @@ setup:
 do_build_setup:
 
 build_setup:
-	@echo "[ $(SOURCE) ] make build_setup $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	_TMP_setup_subdir="$(SETUP_SUBDIRS) $($(addprefix SETUP_SUBDIRS_, $(ARCH_CONFIG)))";	\
+	$(_v)_TMP_setup_subdir="$(SETUP_SUBDIRS) $($(addprefix SETUP_SUBDIRS_, $(ARCH_CONFIG)))";	\
 	for setup_subdir in $${_TMP_setup_subdir};		\
 	do							\
 	[ -d $${setup_subdir} ] || $(MKDIR) $${setup_subdir};				\
@@ -229,27 +273,58 @@ build_setup:
 #
 # Build all architectures for all Configuration/Architecture options
 #
+# Note - TARGET_CONFIGS_UC variable holds sets of three configuration options.  The first 
+# item in the set is the kernel configuration.  The second item in the set is the architecture  
+# and the third item is the machine configuration.  There may be multiple sets to build.
+#
+ifeq ($(RC_ProjectName),Libsyscall)
+all:
+	bsdmake -C libsyscall install
+else # xnu
 ifeq ($(COMPONENT), .)
 all: exporthdrs
 else
 all:
 endif
-	@echo "[ $(SOURCE) ] make all";					\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
-	for kernel_config in $(KERNEL_CONFIGS);				\
-	do								\
-	for arch_config in $(ARCH_CONFIGS);				\
-	do								\
-	build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path}; \
-	[ -d $${build_subdir} ] || $(MKDIR) $${build_subdir};					\
-	${MAKE} -C $${build_subdir}					\
-		KERNEL_CONFIG=$${kernel_config}				\
-		ARCH_CONFIG=$${arch_config}				\
-		MAKEFILES=${SOURCE}/Makefile 				\
-		SOURCE=${SOURCE}/					\
-		build_all;						\
-	done;								\
+	$(_v)my_counter=1;		\
+	for my_config in $(TARGET_CONFIGS_UC);				\
+	do		\
+	if [ $${my_counter} -eq 1 ] ; then	\
+		my_counter=2;		\
+		kernel_config=$${my_config};		\
+		if [ $${kernel_config} = DEFAULT ] ; then	\
+			kernel_config=$(DEFAULT_KERNEL_CONFIG);		\
+		fi;		\
+	elif [ $${my_counter} -eq 2 ] ; then	\
+		my_counter=3;		\
+		arch_config=$${my_config};		\
+		if [ $${arch_config} = DEFAULT ] ; then	\
+			arch_config=`arch | $(TR) a-z A-Z`;		\
+		fi;		\
+	else		\
+		my_counter=1;		\
+		machine_config=$${my_config};		\
+		if [ $${arch_config} = ARM ] ; then	\
+			if [ $${machine_config} = DEFAULT ] ; then	\
+				machine_config=$(DEFAULT_ARM_MACHINE_CONFIG);		\
+			fi;		\
+		fi;		\
+		if [ $${machine_config} = DEFAULT ] ; then	\
+			build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path}; \
+		else		\
+			build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config}/$${rel_path}; \
+		fi;		\
+		[ -d $${build_subdir} ] || $(MKDIR) $${build_subdir};					\
+			${MAKE} -C $${build_subdir}					\
+				KERNEL_CONFIG=$${kernel_config}				\
+				ARCH_CONFIG=$${arch_config}			\
+				MACHINE_CONFIG=$${machine_config}			\
+				MAKEFILES=${SOURCE}/Makefile 				\
+				SOURCE=${SOURCE}/					\
+				build_all;						\
+	fi;		\
 	done;
+endif
 
 #
 # Build all architectures for all Configuration/Architecture options
@@ -257,8 +332,11 @@ endif
 do_build_all:
 
 build_all: 
-	@TARGET=${OBJROOT}/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT);	\
-	echo "[ $(SOURCE) ] make build_all $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $${TARGET}";	\
+	$(_v)if [ $(MACHINE_CONFIG) = DEFAULT ] ; then  \
+		TARGET=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT);	\
+	else		\
+		TARGET="$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)";	\
+	fi;		\
 	_TMP_comp_subdir="$(COMP_SUBDIRS) $($(addprefix COMP_SUBDIRS_, $(ARCH_CONFIG)))";	\
 	for comp_subdir in $${_TMP_comp_subdir};			\
 	do								\
@@ -279,28 +357,55 @@ build_all:
 			SOURCE=${SOURCE}$${comp_subdir}/		\
 			TARGET=$${TARGET}				\
 		build_all;						\
-	done;								\
+	done;								
 	
 
 #
 # Build all architectures for all Configuration/Architecture options
 #
+# Note - TARGET_CONFIGS_UC variable holds sets of three configuration options.  The first 
+# item in the set is the kernel configuration.  The second item is the architecture  
+# and the third item is the machine configuration.  There may be multiple sets to build.
+#
 mach_kernel:
-	@echo "[ $(SOURCE) ] make mach_kernel";				\
-	for kernel_config in $(KERNEL_CONFIGS);				\
-	do								\
-	for arch_config in $(ARCH_CONFIGS);				\
-	do								\
-	build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config};	\
-	[ -d $${build_subdir} ] || $(MKDIR) $${build_subdir};					\
-	${MAKE} -C $${build_subdir}					\
-		KERNEL_CONFIG=$${kernel_config}				\
-		ARCH_CONFIG=$${arch_config}				\
-		MAKEFILES=${SOURCE}/Makefile 				\
-		SOURCE=${SOURCE}/					\
-		TARGET=$${build_subdir}/				\
-		build_mach_kernel;					\
-	done;								\
+	$(_v)my_counter=1;		\
+	for my_config in $(TARGET_CONFIGS_UC);				\
+	do		\
+	if [ $${my_counter} -eq 1 ] ; then	\
+		my_counter=2;		\
+		kernel_config=$${my_config};		\
+		if [ $${kernel_config} = DEFAULT ] ; then	\
+			kernel_config=$(DEFAULT_KERNEL_CONFIG);		\
+		fi;		\
+	elif [ $${my_counter} -eq 2 ] ; then	\
+		my_counter=3;		\
+		arch_config=$${my_config};		\
+		if [ $${arch_config} = DEFAULT ] ; then	\
+			arch_config=`arch | $(TR) a-z A-Z`;		\
+		fi;		\
+	else		\
+		my_counter=1;		\
+		machine_config=$${my_config};		\
+		if [ $${arch_config} = ARM ] ; then	\
+			if [ $${machine_config} = DEFAULT ] ; then	\
+				machine_config=$(DEFAULT_ARM_MACHINE_CONFIG);		\
+			fi;		\
+		fi;		\
+		if [ $${machine_config} = DEFAULT ] ; then	\
+			build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config};	\
+		else		\
+			build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config};	\
+		fi;		\
+		[ -d $${build_subdir} ] || $(MKDIR) $${build_subdir};					\
+		${MAKE} -C $${build_subdir}					\
+			KERNEL_CONFIG=$${kernel_config}				\
+			ARCH_CONFIG=$${arch_config}				\
+			MACHINE_CONFIG=$${machine_config}			\
+			MAKEFILES=${SOURCE}/Makefile 				\
+			SOURCE=${SOURCE}/					\
+			TARGET=$${build_subdir}/				\
+			build_mach_kernel;					\
+	fi;		\
 	done;
 
 #
@@ -309,8 +414,7 @@ mach_kernel:
 do_build_mach_kernel:
 
 build_mach_kernel: 
-	@echo "[ $(SOURCE) ] make build_mach_kernel $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	${MAKE} ${MAKEJOBS} do_build_mach_kernel;
+	$(_v)${MAKE} ${MAKEJOBS} do_build_mach_kernel;
 
 
 #
@@ -323,23 +427,74 @@ build_mach_kernel:
 # Install kernel based on RC_ARCHS for all INSTALL_TYPES
 # Install kernel header files based on RC_ARCHS
 #
-install: installhdrs all installman
-	@echo "[ $(SOURCE) ] make install";				\
-	rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+install: installhdrs all installman installmachinekernels
+ifeq ($(RC_ProjectName),Libsyscall)
+# nothing to do
+else # xnu
+	$(_v)rel_path=$(shell $(RELPATH) $(SRCROOT) $(SOURCE));			\
+	machine_config=$(MACHINE_CONFIG);		\
 	for kernel_config in $(INSTALL_TYPE);				\
 	do								\
 	for arch_config in $(INSTALL_ARCHS);				\
 	do								\
-	install_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
+	if [ $${arch_config} = ARM ] ; then	\
+		if [ $${machine_config} = DEFAULT ] ; then	\
+			machine_config=$(DEFAULT_ARM_MACHINE_CONFIG);		\
+		fi;		\
+	fi;		\
+	if [ $${machine_config} = DEFAULT ] ; then	\
+		install_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}/$${rel_path};	\
+	else		\
+		install_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config}/$${rel_path};	\
+	fi;		\
 	[ -d $${install_subdir} ] || $(MKDIR) $${install_subdir};					\
 	${MAKE} -C $${install_subdir}					\
 		KERNEL_CONFIG=$${kernel_config}				\
 		ARCH_CONFIG=$${arch_config}				\
+		MACHINE_CONFIG=$${machine_config}			\
 		MAKEFILES=${SOURCE}/Makefile 				\
 		SOURCE=${SOURCE}/					\
 		build_install;						\
 	done;								\
 	done;
+endif
+
+installmachinekernels:
+	@echo "[ $(SOURCE) ] make installmachinekernels";				\
+	my_counter=1;		\
+	for my_config in $(TARGET_CONFIGS_UC);				\
+	do		\
+	if [ $${my_counter} -eq 1 ] ; then	\
+		my_counter=2;		\
+		kernel_config=$${my_config};		\
+		if [ $${kernel_config} = DEFAULT ] ; then	\
+			kernel_config=$(DEFAULT_KERNEL_CONFIG);		\
+		fi;		\
+	elif [ $${my_counter} -eq 2 ] ; then	\
+		my_counter=3;		\
+		arch_config=$${my_config};		\
+		if [ $${arch_config} = DEFAULT ] ; then	\
+			arch_config=`arch | $(TR) a-z A-Z`;		\
+		fi;		\
+	else		\
+		my_counter=1;		\
+		machine_config=$${my_config};		\
+		if [ $${machine_config} != DEFAULT ] ; then	\
+			build_subdir=${OBJROOT}/$${kernel_config}_$${arch_config}_$${machine_config};	\
+			install_file_list=mach.`printf "%s" "$${kernel_config}" | $(TR) A-Z a-z`.`printf "%s" "$${machine_config}" | $(TR) A-Z a-z`;	\
+			[ -d $${build_subdir} ] || $(MKDIR) $${build_subdir};					\
+			${MAKE} -C $${build_subdir}					\
+				INSTALL_FILE_LIST=$${install_file_list}			\
+				KERNEL_CONFIG=$${kernel_config}				\
+				ARCH_CONFIG=$${arch_config}				\
+				MACHINE_CONFIG=$${machine_config}			\
+				MAKEFILES=${SOURCE}/Makefile 				\
+				SOURCE=${SOURCE}/					\
+				TARGET=$${build_subdir}/				\
+				do_build_install;					\
+		fi;		\
+	fi;		\
+	done;
 
 #
 # Install for all architectures for all Configuration/Architecture options
@@ -349,8 +504,11 @@ setup_build_install:
 do_build_install:
 
 build_install: 
-	@TARGET=${OBJROOT}/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT);	\
-	echo "[ $(SOURCE) ] make build_install $(COMPONENT) $(KERNEL_CONFIG) $(ARCH_CONFIG) $${TARGET}";	\
+	$(_v)if [ $(MACHINE_CONFIG) = DEFAULT ] ; then  \
+		TARGET=${OBJROOT}/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT);	\
+	else		\
+		TARGET="$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)";	\
+	fi;		\
 	${MAKE} TARGET=$${TARGET} setup_build_install;			\
 	kernel_config=$(KERNEL_CONFIG);					\
 	for install_subdir in $(INST_SUBDIRS);				\
@@ -370,8 +528,7 @@ build_install:
 # Install source tree
 #
 installsrc:
-	pax -rw . ${SRCROOT}
-
+	$(_v)(tar -c --mode go=r,+X --no-ignore-case --exclude .svn --exclude cscope.\* --exclude BUILD --exclude \*~ -f - .) | (cd $(SRCROOT) && tar --no-same-owner -xf -)
 
 
 #
@@ -407,8 +564,11 @@ cscope: cscope.files
 
 tags: cscope.files
 	@echo "Building ctags"
-	@-xargs ctags -dtw < cscope.files 2> /dev/null || \
+	@-sed 1d cscope.files | xargs ctags -dtw 2> /dev/null || \
 		echo "Phantom files detected!" 2>&1 > /dev/null
+	@-[ -f TAGS ] || ${MAKE} TAGS
+
+TAGS: cscope.files
 	@echo "Building etags"
 	@-cat cscope.files | etags -l auto -S - 2> /dev/null
 
@@ -416,19 +576,24 @@ tags: cscope.files
 # Install Man Pages
 #
 installman:
-	@echo "[ $(SRCROOT) ] Installing man pages"; \
-	manpath=$(DSTROOT)/$(MANDIR);   \
+ifeq ($(RC_ProjectName),Libsyscall)
+	bsdmake -C libsyscall install-man
+else # xnu
+	@echo "[ $(SRCROOT) ] Installing man pages"
+	$(_v)manpath=$(DSTROOT)/$(MANDIR);   \
 	[ -d $$manpath ] || $(MKDIR) $$manpath;             \
 	${MAKE} MAKEFILES=${SOURCE}/Makefile \
 		SOURCE=${SOURCE}/ \
 		TARGET=${DSTROOT}/ \
 		build_installman
+	${SOURCE}/config/compress-man-pages.pl ${DSTROOT}/${MANDIR}
+endif
 
 do_installman:
 
 build_installman:
-	@echo "[ $(SOURCE) ] make build_installman";    \
-	if [ -n "$(strip $(INSTMAN_SUBDIRS))" ]; then \
+	@echo "[ $(SOURCE) ] make build_installman"
+	$(_v)if [ -n "$(strip $(INSTMAN_SUBDIRS))" ]; then \
 		for installman_subdir in $(INSTMAN_SUBDIRS); do \
 			${MAKE} -C $${installman_subdir} -r \
 				MAKEFILES=$(SOURCE)$${installman_subdir}/Makefile \
@@ -440,3 +605,5 @@ build_installman:
 	if [ -n "$(strip $(INSTALL_MAN_LIST))" ]; then \
 		${MAKE} ${MAKEJOBS} do_installman; \
 	fi
+
+# vim: set ft=make:
diff --git a/makedefs/MakeInc.rule b/makedefs/MakeInc.rule
index d24d1fb75..add2ecf97 100644
--- a/makedefs/MakeInc.rule
+++ b/makedefs/MakeInc.rule
@@ -50,11 +50,17 @@ ifndef INSTALL_KF_MD_GEN_LIST
     INSTALL_KF_MD_GEN_LIST = $(EXPORT_MD_GEN_LIST)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+	OBJPATH = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)
+else
+	OBJPATH = $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)
+endif
+
 INSTALL_MI_GEN_FILES = $(addprefix $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR)/, $(INSTALL_MI_GEN_LIST))
 
 $(INSTALL_MI_GEN_FILES): $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -75,8 +81,8 @@ $(INSTALL_MI_GEN_FILES): $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR)/% : %
 INSTALL_KF_MI_GEN_FILES = $(addprefix  $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR)/, $(INSTALL_KF_MI_GEN_LIST))
 
 $(INSTALL_KF_MI_GEN_FILES): $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR)/% : %
-	@true echo Installing $< in $(midir $@);			\
-	[ -d $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR);	\
+	@true echo Installing $< in $(midir $@)
+	$(_v)[ -d $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -97,8 +103,8 @@ $(INSTALL_KF_MI_GEN_FILES): $(DSTROOT)/$(KINCDIR)/$(EXPORT_MI_DIR)/% : %
 INSTALL_MI_GEN_LCL_FILES = $(addprefix $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR)/, $(INSTALL_MI_LCL_GEN_LIST))
 
 $(INSTALL_MI_GEN_LCL_FILES): $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -119,8 +125,8 @@ $(INSTALL_MI_GEN_LCL_FILES): $(DSTROOT)/$(LCLDIR)/$(INSTALL_MI_DIR)/% : %
 INSTALL_KF_MI_LCL_GEN_FILES = $(addprefix  $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR)/, $(INSTALL_KF_MI_LCL_GEN_LIST))
 
 $(INSTALL_KF_MI_LCL_GEN_FILES): $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -141,8 +147,8 @@ $(INSTALL_KF_MI_LCL_GEN_FILES): $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MI_DIR)/% : %
 INSTALL_MD_GEN_INC_FILES = $(addprefix $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR)/, $(INSTALL_MD_GEN_LIST))
 
 $(INSTALL_MD_GEN_INC_FILES): $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -163,8 +169,8 @@ $(INSTALL_MD_GEN_INC_FILES): $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR)/% : %
 INSTALL_KF_MD_GEN_FILES = $(addprefix $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR)/, $(INSTALL_KF_MD_GEN_LIST))
 
 $(INSTALL_KF_MD_GEN_FILES): $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KINCDIR)/$(EXPORT_MD_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -186,8 +192,8 @@ INSTALL_MD_LCL_FILES = $(addprefix $(SOURCE), $(INSTALL_MD_LCL_LIST))
 INSTALL_MD_GEN_LCL_FILES = $(addprefix $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR)/, $(INSTALL_MD_LCL_GEN_LIST))
 
 $(INSTALL_MD_GEN_LCL_FILES): $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(LCLDIR)/$(INSTALL_MD_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -209,8 +215,8 @@ INSTALL_KF_MD_LCL_FILES = $(addprefix $(SOURCE), $(INSTALL_KF_MD_LCL_LIST))
 INSTALL_KF_MD_LCL_GEN_FILES = $(addprefix $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR)/, $(INSTALL_KF_MD_LCL_GEN_LIST))
 
 $(INSTALL_KF_MD_LCL_GEN_FILES): $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR)/% : %
-	@true echo Installing $< in $(dir $@);			\
-	[ -d $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR);	\
+	@true echo Installing $< in $(dir $@)
+	$(_v)[ -d $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR);	\
 	filename=`$(BASENAME) $<`;				\
 	filename_strip=$(addsuffix .strip,$${filename});	\
 	$(RM) $(RMFLAGS) $@;					\
@@ -229,11 +235,10 @@ $(INSTALL_KF_MD_LCL_GEN_FILES): $(DSTROOT)/$(KPINCDIR)/$(EXPORT_MD_DIR)/% : %
 	fi;
 
 setup_installhdrs_mi: 
-	@echo "[ $(SOURCE) ] make setup_installhdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_installhdrs_mi: $(INSTALL_MI_GEN_FILES) $(INSTALL_MI_GEN_LCL_FILES) $(INSTALL_KF_MI_GEN_FILES) $(INSTALL_KF_MI_LCL_GEN_FILES)
-	@true echo "[ $(SOURCE) ] make do_installhdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	$(MKDIR) ./incmidir ./pincmidir ./kincmidir ./kpincmidir;				\
+	@true echo "[ $(SOURCE) ] make do_installhdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
+	$(_v)$(MKDIR) ./incmidir ./pincmidir ./kincmidir ./kpincmidir;				\
 	if [ -n "$(strip $(INSTALL_MI_LIST))" ]; then			\
 	    if [ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR) ]; then	\
 		(cd $(DSTROOT)/$(INCDIR)/$(INSTALL_MI_DIR);$(RM) $(RMFLAGS) $(INSTALL_MI_LIST) );       \
@@ -325,11 +330,10 @@ do_installhdrs_mi: $(INSTALL_MI_GEN_FILES) $(INSTALL_MI_GEN_LCL_FILES) $(INSTALL
 	$(RM) -rf ./incmidir ./pincmidir ./kincmidir ./kpincmidir;
 
 setup_installhdrs_md:
-	@echo "[ $(SOURCE) ] make setup_installhdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_installhdrs_md: $(INSTALL_MD_GEN_INC_FILES) $(INSTALL_MD_GEN_LCL_FILES) $(INSTALL_KF_MD_GEN_FILES) $(INSTALL_KF_MD_LCL_GEN_FILES)
-	@true echo "[ $(SOURCE) ] make do_installhdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	$(MKDIR) ./incdir ./pincdir ./kincdir ./kpincdir;				\
+	@true echo "[ $(SOURCE) ] make do_installhdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
+	$(_v)$(MKDIR) ./incdir ./pincdir ./kincdir ./kpincdir;				\
 	if [ -n "$(strip $(INSTALL_MD_LIST))" ]; then			\
 	    if [ -d $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR) ]; then	\
 		(cd $(DSTROOT)/$(INCDIR)/$(INSTALL_MD_DIR);$(RM) $(RMFLAGS) $(INSTALL_MD_LIST) );       \
@@ -430,8 +434,8 @@ EXPORT_MI_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR)/, $(EXPO
 EXPORT_MI_GEN_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR)/, $(EXPORT_MI_GEN_LIST))
 
 $(EXPORT_MI_INC_FILES) $(EXPORT_MI_GEN_INC_FILES): $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR)/% : %
-	@true echo Exporting $< in $(dir $@);			\
-	[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR);	\
+	@true echo Exporting $< in $(dir $@)
+	$(_v)[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR);	\
 	${CP} -p $< $(dir $@);	\
 
 
@@ -439,18 +443,16 @@ EXPORT_MD_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR)/, $(EXPO
 EXPORT_MD_GEN_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR)/, $(EXPORT_MD_GEN_LIST))
 
 $(EXPORT_MD_INC_FILES) $(EXPORT_MD_GEN_INC_FILES): $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR)/% : %
-	@true echo Exporting $< in $(dir $@);			\
-	[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR);	\
+	@true echo Exporting $< in $(dir $@)
+	$(_v)[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR);	\
 	${CP} -p $< $(dir $@);	\
 
 setup_exporthdrs_mi:
-	@echo "[ $(SOURCE) ] make setup_exporthdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_exporthdrs_mi: $(EXPORT_MI_INC_FILES) $(EXPORT_MI_GEN_INC_FILES)
 	@true echo "[ $(SOURCE) ] make do_exporthdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_exporthdrs_md:
-	@echo "[ $(SOURCE) ] make setup_exporthdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_exporthdrs_md: $(EXPORT_MD_INC_FILES) $(EXPORT_MD_GEN_INC_FILES)
 	@true echo "[ $(SOURCE) ] make do_exporthdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
@@ -461,8 +463,8 @@ EXPORT_MI_INC_FILES = $(addprefix $(SOURCE), $(EXPORT_MI_LIST))
 EXPORT_MI_GEN_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR)/, $(EXPORT_MI_GEN_LIST))
 
 $(EXPORT_MI_GEN_INC_FILES): $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR)/% : %
-	@true echo Exporting $< in $(dir $@);			\
-	[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR);	\
+	@true echo Exporting $< in $(dir $@)
+	$(_v)[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR);	\
 	${CP} -p $< $(dir $@);	\
 
 
@@ -470,16 +472,15 @@ EXPORT_MD_INC_FILES = $(addprefix $(SOURCE), $(EXPORT_MD_LIST))
 EXPORT_MD_GEN_INC_FILES = $(addprefix $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR)/, $(EXPORT_MD_GEN_LIST))
 
 $(EXPORT_MD_GEN_INC_FILES): $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR)/% : %
-	@true echo Exporting $< in $(dir $@);			\
-	[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR);	\
+	@true echo Exporting $< in $(dir $@)
+	$(_v)[ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR) ] ||$(MKDIR) $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR);	\
 	${CP} -p $< $(dir $@);	\
 
 setup_exporthdrs_mi:
-	@echo "[ $(SOURCE) ] make setup_exporthdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_exporthdrs_mi: $(EXPORT_MI_GEN_INC_FILES)
 	@true echo "[ $(SOURCE) ] make do_exporthdrs_mi $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
-	@if [ -n "$(strip $(EXPORT_MI_LIST))" ]; then					\
+	$(_v)if [ -n "$(strip $(EXPORT_MI_LIST))" ]; then					\
 		if [ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR) ]; then			\
 			(cd $(OBJROOT)/$(EXPDIR)/$(EXPORT_MI_DIR);$(RM) $(RMFLAGS) $(EXPORT_MI_LIST) );       \
 		else								\
@@ -489,11 +490,10 @@ do_exporthdrs_mi: $(EXPORT_MI_GEN_INC_FILES)
 	fi
 
 setup_exporthdrs_md:
-	@echo "[ $(SOURCE) ] make setup_exporthdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_exporthdrs_md: $(EXPORT_MD_GEN_INC_FILES)
 	@true echo "[ $(SOURCE) ] make do_exporthdrs_md $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
-	@if [ -n "$(strip $(EXPORT_MD_LIST))" ]; then					\
+	$(_v)if [ -n "$(strip $(EXPORT_MD_LIST))" ]; then					\
 		if [ -d $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR) ]; then			\
 			(cd $(OBJROOT)/$(EXPDIR)/$(EXPORT_MD_DIR);$(RM) $(RMFLAGS) $(EXPORT_MD_LIST) );       \
 		else								\
@@ -512,6 +512,7 @@ endif
 #
 # Compilation rules to generate .o from .s
 #
+
 COMP_SOBJ_FILES = $(addprefix $(TARGET)$(COMP_OBJ_DIR), $(COMP_SOBJ_LIST))
 
 $(COMP_SOBJ_FILES): $(TARGET)$(COMP_OBJ_DIR)%.o : %.s
@@ -522,27 +523,29 @@ $(COMP_SOBJ_FILES): $(TARGET)$(COMP_OBJ_DIR)%.o : %.s
 
 S_RULE_1A=@ls /
 S_RULE_1B= ${patsubst %.o,%.s,${@}} > /dev/null
-S_RULE_2= ${S_KCC} -E -MD ${SFLAGS} -DASSEMBLER $(INCFLAGS) $< \
+S_RULE_2=$(_v)${S_KCC} -E -MD ${SFLAGS} -DASSEMBLER $(INCFLAGS) $< \
                > $(patsubst %.o, %.pp, ${@}); \
 	sed '/^\#/d' $(patsubst %.o, %.pp, ${@}) > $(patsubst %.o, %.s, ${@});
-S_RULE_3= ${S_KCC} ${SFLAGS} ${_HOST_AS_FLAGS}  -c $(patsubst %.o, %.s, ${@});\
-       ${RM} ${_RMFLAGS_} $(patsubst %.o, %.pp, ${@}) $(patsubst %.o,%.s,${@})
-
-#
-# Compilation rules to generate .o from .c
-#
-COMP_COBJ_FILES = $(addprefix $(TARGET)$(COMP_OBJ_DIR), $(COMP_COBJ_LIST))
-
-$(COMP_COBJ_FILES): $(TARGET)$(COMP_OBJ_DIR)%.o : %.c
-	${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} $<
+ifeq    ($(BUILD_STABS),1)
+S_RULE_3=$(_v)${S_KCC} ${SFLAGS} ${_HOST_AS_FLAGS}  -c $(patsubst %.o, %.s, ${@});\
+       ${RM} ${_RMFLAGS_} $(patsubst %.o, %.pp, ${@}) $(patsubst %.o,%.s,${@});
+else
+S_RULE_3=$(_v)${S_KCC} ${SFLAGS} ${_HOST_AS_FLAGS}  -c $(patsubst %.o, %.s, ${@});\
+       ${RM} ${_RMFLAGS_} $(patsubst %.o, %.pp, ${@}) $(patsubst %.o,%.s,${@});\
+  ${CTFCONVERT} -l xnu -v -o $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf $@ > /dev/null && $(CTFSCRUB) `cat $(SRCROOT)/config/DtraceIgnored.symbols` $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf || true;
+S_RULE_4=@echo ASM $@
+endif
 
 #
 # Compilation rules to generate .o from .c for normal files
-#
-C_RULE_1A=${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${CWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
+C_RULE_1A=$(_v)${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${CWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
 C_RULE_1B=$*.c
-C_RULE_2=
+C_RULE_2=@echo CC $@
+ifeq    ($(BUILD_STABS),1)
 C_RULE_3=
+else
+C_RULE_3=$(_v)${CTFCONVERT} -l xnu -v -o $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf $@ > /dev/null && $(CTFSCRUB) `cat $(SRCROOT)/config/DtraceIgnored.symbols` $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf || true;
+endif
 C_RULE_4=
 
 #
@@ -557,9 +560,9 @@ C_RULE_4_D=${C_RULE_4}
 #
 # Compilation rules to generate .o from .m
 #
-M_RULE_1A=${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${MWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
+M_RULE_1A=$(_v)${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${MWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
 M_RULE_1B=$*.m
-M_RULE_2=
+M_RULE_2=@echo CC $@
 M_RULE_3=
 M_RULE_4=
 
@@ -567,11 +570,15 @@ M_RULE_4=
 # Compilation rules to generate .co from .cp or .cpo from .cpp
 #   The config tool slickly changes the last source filename char to 'o'
 #   for the object filename.
-#
-P_RULE_1A=${KC++} -o $@ -c ${CXXFLAGS} ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${CXXWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
+P_RULE_1A=$(_v)${KC++} -o $@ -c ${CXXFLAGS} ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS} ${CXXWARNFLAGS}} -MD ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} 
 P_RULE_1B=$(<F)
-P_RULE_2=@sed 's/.c.o: /.cpo: /' $(@:.cpo=.d) > $(@:.cpo=.d~) && mv $(@:.cpo=.d~) $(@:.cpo=.d)
-P_RULE_3=
+P_RULE_2=$(_v)sed 's/.c.o: /.cpo: /' $(@:.cpo=.d) > $(@:.cpo=.d~) && mv $(@:.cpo=.d~) $(@:.cpo=.d)
+P_RULE_3=@echo C++ $@
+ifeq    ($(BUILD_STABS),1)
+P_RULE_4=
+else
+P_RULE_4=$(_v)${CTFCONVERT} -l xnu -v -o $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf $@ > /dev/null && $(CTFSCRUB) `cat $(SRCROOT)/config/DtraceIgnored.symbols` $(TARGET)$(COMP_OBJ_DIR)/$(KERNEL_CONFIG)/$@.ctf || true;
+endif
 P_RULE_4=
 
 #
@@ -582,15 +589,13 @@ LD_COMPONENT_OBJ_FILES = $(addprefix $(TARGET)$(COMP_OBJ_DIR), $(LD_COMPONENT_OB
 COMPONENT_IMAGE_FILE = $(addprefix $(TARGET), $(COMPONENT_IMAGE))
 
 $(COMPONENT_IMAGE_FILE): $(LD_COMPONENT_OBJ_FILES)
-	@echo "[ creating $(COMPONENT_IMAGE) ]"
-	$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT_IMAGE_FILE) ${LD_COMPONENT_OBJ_FILES};
-	(cd $(TARGET)$(COMP_OBJ_DIR); ${MD} -u Makedep -f -d `ls *.d`);
+	@echo LD $@
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT) -o $(COMPONENT_IMAGE_FILE) ${LD_COMPONENT_OBJ_FILES};
+	$(_v)(cd $(TARGET)$(COMP_OBJ_DIR); ${MD} -u Makedep -f -d `ls *.d`);
 
 setup_build_all: 
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_all: $(COMP_FILES) $(COMP_COBJ_FILES) $(COMP_SOBJ_FILES) $(COMPONENT_IMAGE_FILE)
-	@true echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 ifeq ($(COMPONENT), .)
 do_build_all: do_build_mach_kernel
@@ -599,32 +604,51 @@ endif
 #
 # mach_kernel building rules
 #
-do_build_mach_kernel: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/kgmacros
-	@echo "[ building mach_kernel ]";
-	@install $(DATA_INSTALL_FLAGS) $(SRCROOT)/config/version.c $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c;
-	@$(SRCROOT)/config/newvers.pl $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c;
-	${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS}} ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.c -o $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.o
-	$(LD) $(LDFLAGS_KERNEL) $(addprefix $(TARGET)/,$(foreach component,$(COMPONENT_LIST), $(addprefix $(component)/$(firstword $($(addsuffix _KERNEL_CONFIG, $(shell printf "%s" "$(component)" | tr a-z A-Z))) $(KERNEL_CONFIG))/, $(addsuffix .o, $(component))))) $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/version.o -o $(TARGET)/mach_kernel.sys $(LD_KERNEL_LIBS);	\
-	$(STRIP) $(STRIP_FLAGS) $(TARGET)/mach_kernel.sys -o $(TARGET)/mach_kernel;
-
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/kgmacros: $(SRCROOT)/kgmacros
-	cp $? $@
+do_build_mach_kernel: $(OBJPATH)/kgmacros
+	$(_v)install $(DATA_INSTALL_FLAGS) $(SRCROOT)/config/version.c $(OBJPATH)/version.c;
+	$(_v)$(SRCROOT)/config/newvers.pl $(OBJPATH)/version.c > /dev/null;
+	@echo CC version.o
+	$(_v)${KCC} -c ${filter-out ${${join $@,_CFLAGS_RM}}, ${CFLAGS}} ${${join $@,_CFLAGS_ADD}} ${INCFLAGS} ${${join $@,_INCFLAGS}} $(OBJPATH)/version.c -o $(OBJPATH)/version.o
+	@echo LD mach_kernel.sys
+	$(_v)$(LD) $(LDFLAGS_KERNEL) $(addprefix $(TARGET)/,$(foreach component,$(COMPONENT_LIST), $(addprefix $(component)/$(firstword $($(addsuffix _KERNEL_CONFIG, $(shell printf $(component) | tr a-z A-Z))) $(KERNEL_CONFIG))/, $(addsuffix .o, $(component))))) $(OBJPATH)/version.o -o $(TARGET)/mach_kernel.sys $(LD_KERNEL_LIBS)
+	@echo DSYMUTIL mach_kernel.sys
+	$(_v)if [ $(BUILD_DWARF)  -eq  1 ]; then \
+		$(DSYMUTIL) $(DSYMUTIL_FLAGS) $(TARGET)/mach_kernel.sys -o $(TARGET)/mach_kernel.sys.dSYM > /dev/null; \
+	fi;
+	@echo STRIP mach_kernel
+	$(_v)$(STRIP) $(STRIP_FLAGS) $(TARGET)/mach_kernel.sys -o $(TARGET)/mach_kernel
 
-#
-# Generic Install rules
-#
-INSTALL_FILE_FILES = $(addprefix $(DSTROOT)$(INSTALL_FILE_DIR), $(INSTALL_FILE_LIST))
+$(OBJPATH)/kgmacros: $(SRCROOT)/kgmacros
+	$(_v)$(CP) $? $@
 
-force_file_install:
+# Special rules to install machine configuration variants
 
-$(INSTALL_FILE_FILES): $(DSTROOT)$(INSTALL_FILE_DIR)% : $(TARGET)/% force_file_install
+$(DSTROOT)$(INSTALL_FILE_DIR)mach.$(KERNEL_CONFIG_LC).$(MACHINE_CONFIG_LC): $(TARGET)/mach_kernel force_file_install
 	@echo Installing $< in $@;
 	@if [ ! -e $(DSTROOT)$(INSTALL_FILE_DIR) ]; then	\
 		$(MKDIR) $(DSTROOT)$(INSTALL_FILE_DIR);		\
 	fi;							\
 	if [ "`echo $(INSTALL_ARCHS_LC) | wc -w`" -eq 1 ]; then	\
 		$(RM) $(RMFLAGS) $@;				\
-		install $(FILE_INSTALL_FLAGS) $< $(dir $@);		\
+		install $(FILE_INSTALL_FLAGS) $< $@;		\
+	else							\
+		if [ ! -e $@ ]; then				\
+			echo >empty_file_$(notdir $@);			\
+			lipo_arg="$(subst _empty_file, empty_file_$(notdir $@),$(foreach lipo_arch,$(INSTALL_ARCHS_LC), $(addprefix -arch , $(addsuffix _empty_file, $(lipo_arch)))))"; \
+			$(LIPO) $${lipo_arg} -create -output $@;	\
+			$(RM) $(RMFLAGS) empty_file_$(notdir $@);		\
+		fi;							\
+		$(LIPO) $@ -replace $(ARCH_CONFIG_LC)  $< -o $@;	\
+	fi
+
+$(SYMROOT)$(INSTALL_FILE_DIR)mach.$(KERNEL_CONFIG_LC).$(MACHINE_CONFIG_LC): $(TARGET)/mach_kernel.sys force_file_install
+	@echo Installing $< in $@;
+	@if [ ! -e $(SYMROOT)$(INSTALL_FILE_DIR) ]; then	\
+		$(MKDIR) $(SYMROOT)$(INSTALL_FILE_DIR);		\
+	fi;							\
+	if [ "`echo $(INSTALL_ARCHS_LC) | wc -w`" -eq 1 ]; then	\
+		$(RM) $(RMFLAGS) $@;				\
+		install $(FILE_INSTALL_FLAGS) $< $@;		\
 	else							\
 		if [ ! -e $@ ]; then				\
 			echo >empty_file_$(notdir $@);			\
@@ -635,34 +659,115 @@ $(INSTALL_FILE_FILES): $(DSTROOT)$(INSTALL_FILE_DIR)% : $(TARGET)/% force_file_i
 		$(LIPO) $@ -replace $(ARCH_CONFIG_LC)  $< -o $@;	\
 	fi
 
+
+#
+# Generic Install rules
+#
+INSTALL_FILE_FILES = $(addprefix $(DSTROOT)$(INSTALL_FILE_DIR), $(INSTALL_FILE_LIST))
+INSTALL_FILE_FILES_GENERIC = $(filter-out $(DSTROOT)$(INSTALL_FILE_DIR)mach.$(KERNEL_CONFIG_LC).$(MACHINE_CONFIG_LC), $(INSTALL_FILE_FILES))
+
+force_file_install:
+
+$(INSTALL_FILE_FILES_GENERIC): $(DSTROOT)$(INSTALL_FILE_DIR)% : $(TARGET)/% force_file_install
+	@echo Installing $< in $@;
+	$(_v)if [ ! -e $(DSTROOT)$(INSTALL_FILE_DIR) ]; then		\
+		$(MKDIR) $(DSTROOT)$(INSTALL_FILE_DIR);			\
+	fi;								\
+	if [ "`echo $(INSTALL_ARCHS_LC) | wc -w`" -eq 1 ]; then		\
+		$(RM) $(RMFLAGS) $@;					\
+		install $(FILE_INSTALL_FLAGS) $< $@;		\
+	else								\
+		if [ ! -e $@ ]; then					\
+			echo >empty_file_$(notdir $@);			\
+			lipo_arg="$(subst _empty_file, empty_file_$(notdir $@),$(foreach lipo_arch,$(INSTALL_ARCHS_LC), $(addprefix -arch , $(addsuffix _empty_file, $(lipo_arch)))))"; \
+			$(LIPO) $${lipo_arg} -create -output $@;	\
+			$(RM) $(RMFLAGS) empty_file_$(notdir $@);	\
+		fi;							\
+		$(LIPO) $@ -replace $(ARCH_CONFIG_LC)  $< -o $@;	\
+	fi;								\
+	if [ $(BUILD_DWARF) -eq 1 ]; then				\
+		if [ "`echo $(INSTALL_ARCHS_LC) | wc -w`" -eq 1 ]; then	\
+			$(CP) -f $< $<.ctfsys;				\
+			$(FIND) $(OBJPATH)/ -name \*.ctf -size 0	\
+				-exec $(RM) -rf {} \;	;		\
+			$(CTFMERGE) -l xnu -o $<.ctfsys 		\
+				$(OBJPATH)/*/$(KERNEL_CONFIG)/*.*o.ctf || true;	\
+			install $(FILE_INSTALL_FLAGS) $<.ctfsys $(dir $@); \
+		else							\
+			if [ ! -e $@.ctfsys ]; then			\
+				echo >empty_file_$(notdir $@);		\
+				lipo_arg="$(subst _empty_file, empty_file_$(notdir $@),$(foreach lipo_arch,$(INSTALL_ARCHS_LC), $(addprefix -arch , $(addsuffix _empty_file, $(lipo_arch)))))"; \
+				$(LIPO) $${lipo_arg} -create -output $@.ctfsys;\
+				$(RM) $(RMFLAGS) empty_file_$(notdir $@);\
+			fi;						\
+			$(FIND) $(OBJPATH)/ -name \*.ctf -size 0	\
+				-exec $(RM) -rf {} \;	;		\
+			$(CP) -f $< $<.ctfsys;				\
+			$(CTFMERGE) -l xnu -o $<.ctfsys			\
+				$(OBJPATH)/*/$(KERNEL_CONFIG)/*.*o.ctf || true;	\
+			$(LIPO) $@.ctfsys -replace $(ARCH_CONFIG_LC)	\
+				$<.ctfsys -o $@.ctfsys;			\
+		fi;							\
+	fi
+
 INSTALL_FILESYS_FILES = $(addprefix $(SYMROOT)$(INSTALL_FILE_DIR), $(INSTALL_FILE_LIST))
+INSTALL_FILESYS_FILES_GENERIC = $(filter-out $(SYMROOT)$(INSTALL_FILE_DIR)mach.$(KERNEL_CONFIG_LC).$(MACHINE_CONFIG_LC), $(INSTALL_FILESYS_FILES))
 
 force_filesys_install:
 
-$(INSTALL_FILESYS_FILES): $(SYMROOT)$(INSTALL_FILE_DIR)% : $(TARGET)/%.sys force_filesys_install
+$(INSTALL_FILESYS_FILES_GENERIC): $(SYMROOT)$(INSTALL_FILE_DIR)% : $(TARGET)/%.sys force_filesys_install
 	@echo Installing $< in $@;
-	@if [ ! -e $(SYMROOT)$(INSTALL_FILE_DIR) ]; then	\
+	$(_v)if [ ! -e $(SYMROOT)$(INSTALL_FILE_DIR) ]; then	\
 		$(MKDIR) $(SYMROOT)$(INSTALL_FILE_DIR);		\
 	fi;							\
 	if [ "`echo $(INSTALL_ARCHS_LC) | wc -w`" -eq 1 ]; then	\
 		$(RM) $(RMFLAGS) $@;				\
-		install $(INSTALL_FLAGS) $< $(dir $@);		\
+		install $(INSTALL_FLAGS) $< $@;			\
+		if [ $(BUILD_DWARF) -eq 1 ]; then			\
+			$(DSYMUTIL) $(DSYMUTIL_FLAGS)			\
+				$(TARGET)/mach_kernel.sys		\
+				-o $(TARGET)/mach_kernel.sys.dSYM;	\
+			$(RM) -rf $@.dSYM;				\
+			$(MKDIR) -p -m 0755 $@.dSYM/$(DSYMBUILDDIR);	\
+			install $(INSTALL_FLAGS)			\
+				$<.dSYM/$(DSYMBUILDDIR)/$(notdir $<)	\
+				$@.dSYM/$(DSYMBUILDDIR)/$(notdir $@);	\
+		fi;							\
 	else							\
 		if [ ! -e $@ ]; then				\
-			echo >empty_filesys_$(notdir $@);			\
+			echo >empty_filesys_$(notdir $@);	\
 			lipo_arg="$(subst _empty_file, empty_filesys_$(notdir $@),$(foreach lipo_arch,$(INSTALL_ARCHS_LC), $(addprefix -arch , $(addsuffix _empty_file, $(lipo_arch)))))"; \
 			$(LIPO) $${lipo_arg} -create -output $@;	\
-			$(RM) $(RMFLAGS) empty_filesys_$(notdir $@);		\
+			$(RM) $(RMFLAGS) empty_filesys_$(notdir $@);	\
 		fi;							\
 		$(LIPO) $@ -replace $(ARCH_CONFIG_LC)  $< -o $@;	\
+									\
+		if [ $(BUILD_DWARF) -eq 1 ]; then			       \
+			if [ ! -e $@.dSYM/$(DSYMBUILDDIR)/$(notdir $@) ]; then \
+				echo >empty_filesys_$(notdir $@);	       \
+				lipo_arg="$(subst _empty_file, empty_filesys_$(notdir $@),$(foreach lipo_arch,$(INSTALL_ARCHS_LC), $(addprefix -arch , $(addsuffix _empty_file, $(lipo_arch)))))"; \
+				$(MKDIR) -p -m 0755 $@.dSYM/$(DSYMBUILDDIR);   \
+				$(LIPO) $${lipo_arg} -create		       \
+					-output				       \
+					$@.dSYM/$(DSYMBUILDDIR)/$(notdir $@);  \
+				$(RM) $(RMFLAGS) empty_filesys_$(notdir $@);   \
+			fi;						       \
+			$(DSYMUTIL) $(DSYMUTIL_FLAGS)			\
+				$(TARGET)/mach_kernel.sys		\
+				-o $(TARGET)/mach_kernel.sys.dSYM;	\
+			$(LIPO) $@.dSYM/$(DSYMBUILDDIR)/$(notdir $@)	       \
+				-replace $(ARCH_CONFIG_LC)		       \
+				$<.dSYM/$(DSYMBUILDDIR)/$(notdir $<)	       \
+				-o $@.dSYM/$(DSYMBUILDDIR)/$(notdir $@);       \
+		fi;							       \
 	fi
-	cp $(SOURCE)kgmacros $(SYMROOT)$(INSTALL_FILE_DIR)
+	$(CP) $(SOURCE)kgmacros $(SYMROOT)$(INSTALL_FILE_DIR)
 
 INSTALL_DATA_FILES = $(addprefix $(DSTROOT)$(INSTALL_DATA_DIR), $(INSTALL_DATA_LIST))
 
 $(INSTALL_DATA_FILES): $(DSTROOT)$(INSTALL_DATA_DIR)% : $(SOURCE)/%
 	@echo Installing $< in $@;
-	@[ -d $(dir $@) ] ||$(MKDIR) $(dir $@);		\
+	$(_v)[ -d $(dir $@) ] ||$(MKDIR) $(dir $@);		\
 	$(RM) $(RMFLAGS) $@;		\
 	install $(DATA_INSTALL_FLAGS) $< $(dir $@);
 
@@ -676,13 +781,11 @@ INSTALL_MAN_FILES = $(addprefix $(DSTROOT)/$(MANDIR)/$(INSTALL_MAN_DIR)/, $(INST
 
 do_installman: $(INSTALL_MAN_FILES)
 	@echo "[ $(SOURCE) ] make do_installman"
-	@if [ -n "$(strip $(INSTALL_MAN_LIST))" ]; then \
+	$(_v)if [ -n "$(strip $(INSTALL_MAN_LIST))" ]; then \
 		man_dir=$(DSTROOT)/$(MANDIR)/$(INSTALL_MAN_DIR); \
 		if [ -d $$man_dir ]; then       \
-			cur_dir=`pwd`;  \
-			cd $$man_dir;   \
-			$(RM) $(RMFLAGS) $(INSTALL_MAN_LIST) $(INSTALL_MAN_LINKS); \
-			cd $$cur_dir;   \
+			( cd $$man_dir;   \
+			$(RM) $(RMFLAGS) $(INSTALL_MAN_LIST) $(INSTALL_MAN_LINKS)); \
 		else                    \
 			$(MKDIR) $$man_dir;     \
 		fi;                             \
@@ -703,8 +806,8 @@ do_installman: $(INSTALL_MAN_FILES)
 	fi
 
 $(INSTALL_MAN_FILES): $(DSTROOT)/$(MANDIR)/$(INSTALL_MAN_DIR)/% : %
-	@true echo Installing $< in $(dir $@);                  \
-	$(MKDIR) $(DSTROOT)/$(MANDIR)/$(INSTALL_MAN_DIR);       \
+	@true echo Installing $< in $(dir $@)
+	$(_v)$(MKDIR) $(DSTROOT)/$(MANDIR)/$(INSTALL_MAN_DIR);       \
 	$(RM) $(RMFLAGS) $@;                                    \
 	install $(INSTALL_FLAGS) $< $(dir $@);
 
@@ -712,3 +815,4 @@ ifeq    ($(INCL_MAKEDEP), TRUE)
 -include Makedep
 endif
 
+# vim: set ft=make:
diff --git a/osfmk/Makefile b/osfmk/Makefile
index 77661426e..b39ff9c49 100644
--- a/osfmk/Makefile
+++ b/osfmk/Makefile
@@ -3,6 +3,9 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
+export COMP_LDFLAGS_COMPONENT_PPC = -Wl,-i_OSCompareAndSwap:_hw_compare_and_store \
+                                    -Wl,-i_OSDequeueAtomic:_hw_dequeue_atomic \
+                                    -Wl,-i_OSEnqueueAtomic:_hw_queue_atomic
 
 include $(MakeInc_cmd)
 include $(MakeInc_def)
@@ -16,8 +19,11 @@ INSTINC_SUBDIRS = \
 	ipc \
 	machine \
 	UserNotification \
+	gssd \
+	lockd \
 	vm \
-	libsa
+	libsa \
+	kdp
 
 INSTINC_SUBDIRS_PPC = \
 	mach \
@@ -27,6 +33,10 @@ INSTINC_SUBDIRS_I386 = \
 	mach	\
 	i386
 
+INSTINC_SUBDIRS_ARM = \
+	mach	\
+	arm
+
 EXPINC_SUBDIRS = \
 	mach	\
 	device \
@@ -37,8 +47,11 @@ EXPINC_SUBDIRS = \
 	ipc \
 	machine \
 	UserNotification \
+	gssd \
+	lockd \
 	vm \
-	libsa
+	libsa \
+	kdp
 
 EXPINC_SUBDIRS_PPC = \
 	mach \
@@ -48,6 +61,10 @@ EXPINC_SUBDIRS_I386 = \
 	mach	\
 	i386
 
+EXPINC_SUBDIRS_ARM = \
+	mach	\
+	arm
+
 SETUP_SUBDIRS = 	\
 	conf
 
@@ -59,5 +76,3 @@ INST_SUBDIRS =	\
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/osfmk/UserNotification/KUNCUserNotifications.c b/osfmk/UserNotification/KUNCUserNotifications.c
index 21ef970a0..afd0acc92 100644
--- a/osfmk/UserNotification/KUNCUserNotifications.c
+++ b/osfmk/UserNotification/KUNCUserNotifications.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/port.h>
@@ -110,7 +116,11 @@ UNDAlertCompletedWithResult_rpc (
         UNDReplyRef 		reply,
         int 			result,
         xmlData_t		keyRef,		/* raw XML bytes */
+#ifdef KERNEL_CF
         mach_msg_type_number_t	keyLen)
+#else
+        __unused mach_msg_type_number_t	keyLen)
+#endif
 {
 #ifdef KERNEL_CF
 	CFStringRef		xmlError = NULL;
@@ -182,7 +192,7 @@ UNDNotificationCreated_rpc (
 
 
 KUNCUserNotificationID
-KUNCGetNotificationID()
+KUNCGetNotificationID(void)
 {
 	UNDReplyRef reply;
 
diff --git a/osfmk/UserNotification/KUNCUserNotifications.h b/osfmk/UserNotification/KUNCUserNotifications.h
index 6b1f816b8..a7524e64a 100644
--- a/osfmk/UserNotification/KUNCUserNotifications.h
+++ b/osfmk/UserNotification/KUNCUserNotifications.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __USERNOTIFICATION_KUNCUSERNOTIFICATIONS_H
@@ -205,7 +211,7 @@ typedef void
 (*KUNCUserNotificationCallBack)(
 	int		contextKey,
 	int		responseFlags,
-	void		*xmlData);
+	const void	*xmlData);
 
 /*
  * Get a notification ID
diff --git a/osfmk/UserNotification/Makefile b/osfmk/UserNotification/Makefile
index d1f0b286f..edc1f17b4 100644
--- a/osfmk/UserNotification/Makefile
+++ b/osfmk/UserNotification/Makefile
@@ -12,12 +12,16 @@ INSTINC_SUBDIRS_PPC =
 
 INSTINC_SUBDIRS_I386 =
 
+INSTINC_SUBDIRS_ARM =
+
 EXPINC_SUBDIRS =
 
 EXPINC_SUBDIRS_PPC =
 
 EXPINC_SUBDIRS_I386 =
 
+EXPINC_SUBDIRS_ARM =
+
 MIG_TYPES = \
 	UNDTypes.defs
 
@@ -72,7 +76,8 @@ ${COMP_FILES} : ${MIG_TYPES}
         
 ${MIG_KUSRC} : \
 	%.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
 		-user    $*.c              \
 		-header  $*.h              \
 		-server  /dev/null		\
@@ -81,7 +86,8 @@ ${MIG_KUSRC} : \
 
 ${MIG_KSSRC}: \
 	%Server.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
 		-user    /dev/null              \
 		-header  /dev/null              \
 		-server  $*Server.c		\
diff --git a/osfmk/UserNotification/UNDReply.defs b/osfmk/UserNotification/UNDReply.defs
index 58d613224..49e00ec23 100644
--- a/osfmk/UserNotification/UNDReply.defs
+++ b/osfmk/UserNotification/UNDReply.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 subsystem
@@ -37,3 +43,5 @@ simpleroutine UNDAlertCompletedWithResult_rpc(
 simpleroutine UNDNotificationCreated_rpc(
 				reply: UNDReplyRef;
 			in	userLandNotificationKey: int);
+
+/* vim: set ft=c : */
diff --git a/osfmk/UserNotification/UNDRequest.defs b/osfmk/UserNotification/UNDRequest.defs
index 959faa9f3..9a3a7c954 100644
--- a/osfmk/UserNotification/UNDRequest.defs
+++ b/osfmk/UserNotification/UNDRequest.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 subsystem
@@ -110,4 +116,6 @@ routine UNDDisplayAlertSimple_rpc(
                         in	defaultButtonTitle:UNDLabel;
                         in	alternateButtonTitle:UNDLabel;
                         in	otherButtonTitle:UNDLabel;
-			out	response: int);
+			out	response: unsigned);
+
+/* vim: set ft=c : */
diff --git a/osfmk/UserNotification/UNDTypes.defs b/osfmk/UserNotification/UNDTypes.defs
index 8ccefc684..0a040bee3 100644
--- a/osfmk/UserNotification/UNDTypes.defs
+++ b/osfmk/UserNotification/UNDTypes.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _USERNOTIFICATION_UNDTYPES_DEFS_
@@ -58,3 +64,5 @@ type UNDReplyRef = mach_port_t
 import <UserNotification/UNDTypes.h>;
 
 #endif /* _USERNOTIFICATION_UNDTYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/UserNotification/UNDTypes.h b/osfmk/UserNotification/UNDTypes.h
index 02838bf34..01d2960b8 100644
--- a/osfmk/UserNotification/UNDTypes.h
+++ b/osfmk/UserNotification/UNDTypes.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef __USERNOTIFICATION_UNDTYPES_H
diff --git a/osfmk/chud/chud_cpu.c b/osfmk/chud/chud_cpu.c
index d8e0ef4a0..9652dcf92 100644
--- a/osfmk/chud/chud_cpu.c
+++ b/osfmk/chud/chud_cpu.c
@@ -1,50 +1,49 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
+
 #include <mach/mach_types.h>
 #include <mach/mach_host.h>
 
 #include <kern/host.h>
 #include <kern/processor.h>
 #include <kern/cpu_data.h>
-
+#include <kern/machine.h>
 #include <machine/machine_routines.h>
 
 #include <chud/chud_xnu.h>
 
 #pragma mark **** cpu count ****
 
-__private_extern__ int
-chudxnu_avail_cpu_count(void)
+__private_extern__ int 
+chudxnu_logical_cpu_count(void)
 {
-    host_basic_info_data_t hinfo;
-    kern_return_t kr;
-    mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
-
-    kr = host_info(host_self(), HOST_BASIC_INFO, (integer_t *)&hinfo, &count);
-    if(kr == KERN_SUCCESS) {
-        return hinfo.avail_cpus;
-    } else {
-        return 0;
-    }
+	return machine_info.logical_cpu_max;
 }
 
 __private_extern__ int
@@ -58,29 +57,16 @@ chudxnu_phys_cpu_count(void)
     if(kr == KERN_SUCCESS) {
         return hinfo.max_cpus;
     } else {
-        return 0;
+        return 1;  // fall back to 1, 0 doesn't make sense at all
     }
 }
 
-__private_extern__
-int chudxnu_cpu_number(void)
+__private_extern__ int
+chudxnu_cpu_number(void)
 {
     return cpu_number();
 }
 
-#pragma mark **** branch trace buffer ****
-
-extern int pc_trace_buf[1024];
-
-__private_extern__ uint32_t *
-chudxnu_get_branch_trace_buffer(uint32_t *entries)
-{
-    if(entries) {
-        *entries = sizeof(pc_trace_buf)/sizeof(int);
-    }
-    return pc_trace_buf;
-}
-
 #pragma mark **** interrupts enable/disable ****
 
 __private_extern__ boolean_t
@@ -126,3 +112,13 @@ chudxnu_get_preemption_level(void)
 {
 	return get_preemption_level();
 }
+
+#pragma mark *** deprecated ***
+
+//DEPRECATED
+__private_extern__ int
+chudxnu_avail_cpu_count(void)
+{
+	return machine_info.logical_cpu;
+}
+
diff --git a/osfmk/chud/chud_dtrace.h b/osfmk/chud/chud_dtrace.h
new file mode 100644
index 000000000..1cc06a511
--- /dev/null
+++ b/osfmk/chud/chud_dtrace.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef __CHUD_DTRACE_H__
+#define __CHUD_DTRACE_H__
+
+/* Definitions for arguments to the chud() dtrace builtin */
+
+#define CHUD_DTRACE_START_SHARK		((uint64_t)0x1ULL)
+#define CHUD_DTRACE_STOP_SHARK		((uint64_t)0x2ULL)
+#define CHUD_DTRACE_RECORD_SAMPLE	((uint64_t)0x3ULL)
+#define CHUD_DTRACE_SIGNPOST_POINT	((uint64_t)0x4ULL)
+#define CHUD_DTRACE_SIGNPOST_START	((uint64_t)0x5ULL)
+#define CHUD_DTRACE_SIGNPOST_END	((uint64_t)0x6ULL)
+
+#endif /* __CHUD_DTRACE_H__ */
diff --git a/osfmk/chud/chud_glue.c b/osfmk/chud/chud_glue.c
index ba3aa48a9..26f1a70ce 100644
--- a/osfmk/chud/chud_glue.c
+++ b/osfmk/chud/chud_glue.c
@@ -1,22 +1,28 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
diff --git a/osfmk/chud/chud_memory.c b/osfmk/chud/chud_memory.c
index 9eeabb69e..f71c0333e 100644
--- a/osfmk/chud/chud_memory.c
+++ b/osfmk/chud/chud_memory.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/vm_param.h>
@@ -26,15 +32,16 @@
 
 extern unsigned int IODefaultCacheBits(addr64_t pa);
 extern unsigned int vm_page_free_count;
+extern unsigned int vm_page_inactive_count;
 
-__private_extern__
-uint64_t chudxnu_avail_memory_size(void)
+__private_extern__ uint64_t 
+chudxnu_avail_memory_size(void)
 {
     return max_mem;
 }
 
-__private_extern__
-uint64_t chudxnu_phys_memory_size(void)
+__private_extern__ uint64_t 
+chudxnu_phys_memory_size(void)
 {
     return mem_actual;
 }
@@ -49,14 +56,28 @@ uint64_t chudxnu_free_memory_size(void)
     return (uint64_t)vm_page_free_count * (uint64_t)page_size;
 }
 
+/*
+ * This function is not intended to be valid for any amount of time,
+ * it is just an instantaneous snapshot of the current inactive memory size.
+ */
+__private_extern__
+uint64_t chudxnu_inactive_memory_size(void)
+{
+	return (uint64_t)vm_page_inactive_count * (uint64_t)page_size;
+}
+
+#pragma mark *** DEPRECATED ***
+
+// DEPRECATED
 __private_extern__
 vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size)
 {
     return ml_io_map(phys_addr, size); // XXXXX limited to first 2GB XXXXX
 }
 
-__private_extern__
-uint32_t chudxnu_phys_addr_wimg(uint64_t phys_addr)
+// DEPRECATED
+__private_extern__ uint32_t
+chudxnu_phys_addr_wimg(uint64_t phys_addr)
 {
     return IODefaultCacheBits(phys_addr);
 }
diff --git a/osfmk/chud/chud_osfmk_callback.c b/osfmk/chud/chud_osfmk_callback.c
index 7dc865a5a..4afe955b4 100644
--- a/osfmk/chud/chud_osfmk_callback.c
+++ b/osfmk/chud/chud_osfmk_callback.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -31,6 +37,8 @@
 #include <kern/kalloc.h>
 #include <kern/thread.h>
 
+#include <libkern/OSAtomic.h>
+
 #include <machine/machine_routines.h>
 #include <machine/cpu_data.h>
 
@@ -38,6 +46,7 @@
 #include <chud/chud_xnu_private.h>
 
 #pragma mark **** timer ****
+
 __private_extern__ chud_timer_t
 chudxnu_timer_alloc(chudxnu_timer_callback_func_t func, uint32_t param0)
 {
@@ -72,62 +81,40 @@ chudxnu_timer_free(chud_timer_t timer)
     return KERN_SUCCESS;
 }
 
-#pragma mark **** thread timer - DEPRECATED ****
-
-static thread_call_t thread_timer_call = NULL;
-static chudxnu_thread_timer_callback_func_t thread_timer_callback_fn = NULL;
+static chudxnu_dtrace_callback_t 
+	dtrace_callback = (chudxnu_dtrace_callback_t) NULL;
 
-static void chudxnu_private_thread_timer_callback(
-		thread_call_param_t param0,
-		thread_call_param_t param1)
+kern_return_t
+chudxnu_dtrace_callback(uint64_t selector, uint64_t *args, uint32_t count)
 {
-#pragma unused (param1)
-    chudxnu_thread_timer_callback_func_t fn = thread_timer_callback_fn;
-    
-    if(thread_timer_call) {
-        thread_call_free(thread_timer_call);
-        thread_timer_call = NULL;
-
-        if(fn) {
-            (fn)((uint32_t)param0);
-        }
-    }
+	/* it's not an error if no callback is hooked up */
+	kern_return_t ret = KERN_SUCCESS;
+
+	/* Make a local stack copy of the function ptr */
+	chudxnu_dtrace_callback_t fn = dtrace_callback;
+
+	if(fn) {
+		ret = fn(selector, args, count);
+	}
+
+	return ret;
 }
 
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_thread_timer_callback_enter(
-		chudxnu_thread_timer_callback_func_t func,
-		uint32_t param,
-		uint32_t time,
-		uint32_t units)
+__private_extern__ void
+chudxnu_dtrace_callback_enter(chudxnu_dtrace_callback_t fn)
 {
-    if(!thread_timer_call) {
-        uint64_t t_delay;
-        thread_timer_callback_fn = func;
-
-        thread_timer_call = thread_call_allocate(
-				(thread_call_func_t)
-					chudxnu_private_thread_timer_callback,
-				(thread_call_param_t)
-					param);
-        clock_interval_to_deadline(time, units, &t_delay);
-        thread_call_enter_delayed(thread_timer_call, t_delay);
-        return KERN_SUCCESS;
-    } else {
-        return KERN_FAILURE; // thread timer call already pending
-    }
+	chudxnu_dtrace_callback_t old_fn = dtrace_callback;
+
+	/* Atomically clear the call back */
+	while(!OSCompareAndSwap((UInt32)old_fn, (UInt32)fn, 
+		(volatile UInt32 *) &dtrace_callback)) {
+		old_fn = dtrace_callback;
+	}
 }
 
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_thread_timer_callback_cancel(void)
+__private_extern__ void
+chudxnu_dtrace_callback_cancel(void)
 {
-    if(thread_timer_call) {
-    	thread_call_cancel(thread_timer_call);
-        thread_call_free(thread_timer_call);
-        thread_timer_call = NULL;
-    }
-    thread_timer_callback_fn = NULL;
-    return KERN_SUCCESS;
+	chudxnu_dtrace_callback_enter(NULL);
 }
+
diff --git a/osfmk/chud/chud_thread.c b/osfmk/chud/chud_thread.c
index bfaf88fd7..13f03940b 100644
--- a/osfmk/chud/chud_thread.c
+++ b/osfmk/chud/chud_thread.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -31,35 +37,66 @@
 
 #include <chud/chud_xnu.h>
 #include <chud/chud_xnu_private.h>
+#include <chud/chud_thread.h>
 
 #include <machine/machine_routines.h>
 
+#include <libkern/OSAtomic.h>
+
 // include the correct file to find real_ncpus
 #if defined(__i386__) || defined(__x86_64__)
 #	include <i386/mp.h>	
-#endif // i386 or x86_64
-
-#if defined(__ppc__) || defined(__ppc64__)
+#elif defined(__ppc__) || defined(__ppc64__)
 #	include <ppc/cpu_internal.h>
-#endif // ppc or ppc64
+#elif defined(__arm__)
+#	include <arm/cpu_internal.h>
+#else
+// fall back on declaring it extern.  The linker will sort us out.
+extern unsigned int real_ncpus;
+#endif
+
+// Mask for supported options
+#define T_CHUD_BIND_OPT_MASK (-1UL)
 
 #pragma mark **** thread binding ****
 
+/*
+ * This method will bind a given thread to the requested CPU starting at the
+ * next time quantum.  If the thread is the current thread, this method will
+ * force a thread_block().  The result is that if you call this method on the
+ * current thread, you will be on the requested CPU when this method returns.
+ */
 __private_extern__ kern_return_t
-chudxnu_bind_thread(thread_t thread, int cpu)
+chudxnu_bind_thread(thread_t thread, int cpu, __unused int options)
 {
     processor_t proc = NULL;
-	
-	if(cpu >= real_ncpus) // sanity check
+
+	if(cpu < 0 || (unsigned int)cpu >= real_ncpus) // sanity check
 		return KERN_FAILURE;
+
+	// temporary restriction until after phase 2 of the scheduler
+	if(thread != current_thread())
+		return KERN_FAILURE; 
 	
 	proc = cpu_to_processor(cpu);
 
+	/* 
+	 * Potentially racey, but mainly to prevent bind to shutdown
+	 * processor.
+	 */
 	if(proc && !(proc->state == PROCESSOR_OFF_LINE) &&
-			   !(proc->state == PROCESSOR_SHUTDOWN)) {
-		/* disallow bind to shutdown processor */
-		thread_bind(thread, proc);
-		if(thread==current_thread()) {
+			!(proc->state == PROCESSOR_SHUTDOWN)) {
+		
+		thread_bind(proc);
+
+		/*
+		 * If we're trying to bind the current thread, and
+		 * we're not on the target cpu, and not at interrupt
+		 * context, block the current thread to force a
+		 * reschedule on the target CPU.
+		 */
+		if(thread == current_thread() && 
+			!(ml_at_interrupt_context() && cpu_number() == cpu)) {
 			(void)thread_block(THREAD_CONTINUE_NULL);
 		}
 		return KERN_SUCCESS;
@@ -68,16 +105,29 @@ chudxnu_bind_thread(thread_t thread, int cpu)
 }
 
 __private_extern__ kern_return_t
-chudxnu_unbind_thread(thread_t thread)
+chudxnu_unbind_thread(thread_t thread, __unused int options)
 {
-    thread_bind(thread, PROCESSOR_NULL);
+	if(thread == current_thread())
+		thread_bind(PROCESSOR_NULL);
     return KERN_SUCCESS;
 }
 
+__private_extern__ boolean_t
+chudxnu_thread_get_idle(thread_t thread) {
+	/* 
+	 * Instantaneous snapshot of the idle state of
+	 * a given thread.
+	 *
+	 * Should be called only on an interrupted or 
+	 * suspended thread to avoid a race.
+	 */
+	return ((thread->state & TH_IDLE) == TH_IDLE);
+}
+
 #pragma mark **** task and thread info ****
 
-__private_extern__
-boolean_t chudxnu_is_64bit_task(task_t task)
+__private_extern__ boolean_t
+chudxnu_is_64bit_task(task_t task)
 {
 	return (task_has_64BitAddr(task));
 }
@@ -100,23 +150,18 @@ chudxnu_private_processor_set_things(
 	vm_size_t size, size_needed;
 	void  *addr;
 
-	if (pset == PROCESSOR_SET_NULL)
+	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
 		return (KERN_INVALID_ARGUMENT);
 
-	size = 0; addr = 0;
+	size = 0; addr = NULL;
 
 	for (;;) {
-		pset_lock(pset);
-		if (!pset->active) {
-			pset_unlock(pset);
-
-			return (KERN_FAILURE);
-		}
+		mutex_lock(&tasks_threads_lock);
 
 		if (type == THING_TASK)
-			maxthings = pset->task_count;
+			maxthings = tasks_count;
 		else
-			maxthings = pset->thread_count;
+			maxthings = threads_count;
 
 		/* do we have the memory we need? */
 
@@ -124,8 +169,7 @@ chudxnu_private_processor_set_things(
 		if (size_needed <= size)
 			break;
 
-		/* unlock the pset and allocate more memory */
-		pset_unlock(pset);
+		mutex_unlock(&tasks_threads_lock);
 
 		if (size != 0)
 			kfree(addr, size);
@@ -145,13 +189,13 @@ chudxnu_private_processor_set_things(
 
 	case THING_TASK:
 	{
-		task_t		task, *tasks = (task_t *)addr;
+		task_t		task, *task_list = (task_t *)addr;
 
-		for (task = (task_t)queue_first(&pset->tasks);
-				!queue_end(&pset->tasks, (queue_entry_t)task);
-					task = (task_t)queue_next(&task->pset_tasks)) {
+		for (task = (task_t)queue_first(&tasks);
+				!queue_end(&tasks, (queue_entry_t)task);
+					task = (task_t)queue_next(&task->tasks)) {
 			task_reference_internal(task);
-			tasks[actual++] = task;
+			task_list[actual++] = task;
 		}
 
 		break;
@@ -159,27 +203,27 @@ chudxnu_private_processor_set_things(
 
 	case THING_THREAD:
 	{
-		thread_t	thread, *threads = (thread_t *)addr;
+		thread_t	thread, *thread_list = (thread_t *)addr;
 
-		for (i = 0, thread = (thread_t)queue_first(&pset->threads);
-				!queue_end(&pset->threads, (queue_entry_t)thread);
-					thread = (thread_t)queue_next(&thread->pset_threads)) {
+		for (i = 0, thread = (thread_t)queue_first(&threads);
+				!queue_end(&threads, (queue_entry_t)thread);
+					thread = (thread_t)queue_next(&thread->threads)) {
 			thread_reference_internal(thread);
-			threads[actual++] = thread;
+			thread_list[actual++] = thread;
 		}
 
 		break;
 	}
 	}
 		
-	pset_unlock(pset);
+	mutex_unlock(&tasks_threads_lock);
 
 	if (actual < maxthings)
 		size_needed = actual * sizeof (mach_port_t);
 
 	if (actual == 0) {
 		/* no things, so return null pointer and deallocate memory */
-		*thing_list = 0;
+		*thing_list = NULL;
 		*count = 0;
 
 		if (size != 0)
@@ -197,19 +241,19 @@ chudxnu_private_processor_set_things(
 
 				case THING_TASK:
 				{
-					task_t		*tasks = (task_t *)addr;
+					task_t		*task_list = (task_t *)addr;
 
 					for (i = 0; i < actual; i++)
-						task_deallocate(tasks[i]);
+						task_deallocate(task_list[i]);
 					break;
 				}
 
 				case THING_THREAD:
 				{
-					thread_t	*threads = (thread_t *)addr;
+					thread_t	*thread_list = (thread_t *)addr;
 
 					for (i = 0; i < actual; i++)
-						thread_deallocate(threads[i]);
+						thread_deallocate(thread_list[i]);
 					break;
 				}
 				}
@@ -238,7 +282,7 @@ chudxnu_private_task_threads(
     	mach_msg_type_number_t  *count)
 {
 	mach_msg_type_number_t	actual;
-	thread_t				*threads;
+	thread_t				*thread_list;
 	thread_t				thread;
 	vm_size_t				size, size_needed;
 	void					*addr;
@@ -247,7 +291,7 @@ chudxnu_private_task_threads(
 	if (task == TASK_NULL)
 		return (KERN_INVALID_ARGUMENT);
 
-	size = 0; addr = 0;
+	size = 0; addr = NULL;
 
 	for (;;) {
 		task_lock(task);
@@ -282,14 +326,14 @@ chudxnu_private_task_threads(
 	}
 
 	/* OK, have memory and the task is locked & active */
-	threads = (thread_t *)addr;
+	thread_list = (thread_t *)addr;
 
 	i = j = 0;
 
 	for (thread = (thread_t)queue_first(&task->threads); i < actual;
 				++i, thread = (thread_t)queue_next(&thread->task_threads)) {
 		thread_reference_internal(thread);
-		threads[j++] = thread;
+		thread_list[j++] = thread;
 	}
 
 	assert(queue_end(&task->threads, (queue_entry_t)thread));
@@ -303,7 +347,7 @@ chudxnu_private_task_threads(
 	if (actual == 0) {
 		/* no threads, so return null pointer and deallocate memory */
 
-		*threads_out = 0;
+		*threads_out = NULL;
 		*count = 0;
 
 		if (size != 0)
@@ -318,17 +362,17 @@ chudxnu_private_task_threads(
 			newaddr = kalloc(size_needed);
 			if (newaddr == 0) {
 				for (i = 0; i < actual; ++i)
-					thread_deallocate(threads[i]);
+					thread_deallocate(thread_list[i]);
 				kfree(addr, size);
 				return (KERN_RESOURCE_SHORTAGE);
 			}
 
 			bcopy(addr, newaddr, size_needed);
 			kfree(addr, size);
-			threads = (thread_t *)newaddr;
+			thread_list = (thread_t *)newaddr;
 		}
 
-		*threads_out = threads;
+		*threads_out = thread_list;
 		*count = actual;
 	}
 
@@ -341,7 +385,7 @@ chudxnu_all_tasks(
 	task_array_t		*task_list,
 	mach_msg_type_number_t	*count)
 {
-	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)task_list, count, THING_TASK);	
+	return chudxnu_private_processor_set_things(&pset0, (mach_port_t **)task_list, count, THING_TASK);	
 }
 
 __private_extern__ kern_return_t
@@ -365,13 +409,12 @@ chudxnu_free_task_list(
 		return KERN_FAILURE;
 	}
 }
-
 __private_extern__ kern_return_t
 chudxnu_all_threads(
 	thread_array_t		*thread_list,
 	mach_msg_type_number_t	*count)
 {
-	return chudxnu_private_processor_set_things(&default_pset, (mach_port_t **)thread_list, count, THING_THREAD);
+	return chudxnu_private_processor_set_things(&pset0, (mach_port_t **)thread_list, count, THING_THREAD);
 }
 
 __private_extern__ kern_return_t
@@ -433,6 +476,7 @@ chudxnu_thread_info(
 	return thread_info(thread, flavor, thread_info_out, thread_info_count);
 }
 
+
 __private_extern__ kern_return_t
 chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp)
 {
@@ -440,3 +484,31 @@ chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp)
     return KERN_SUCCESS;
 }
 
+/* thread marking stuff */
+
+__private_extern__ boolean_t 
+chudxnu_thread_get_marked(thread_t thread) 
+{
+	if(thread)
+		return ((thread->t_chud & T_CHUD_MARKED) != 0);
+	return FALSE;
+}
+
+__private_extern__ boolean_t
+chudxnu_thread_set_marked(thread_t thread, boolean_t new_value)
+{
+	boolean_t old_val;
+
+	if(thread) {
+		if(new_value) {
+			// set the marked bit
+			old_val = OSBitOrAtomic(T_CHUD_MARKED, (UInt32 *) &(thread->t_chud));
+		} else {
+			// clear the marked bit
+			old_val = OSBitAndAtomic(~T_CHUD_MARKED, (UInt32 *) &(thread->t_chud));
+		}
+		return (old_val & T_CHUD_MARKED) == T_CHUD_MARKED;
+	}
+	return FALSE;
+}
+
diff --git a/osfmk/chud/chud_thread.h b/osfmk/chud/chud_thread.h
new file mode 100644
index 000000000..ab335816f
--- /dev/null
+++ b/osfmk/chud/chud_thread.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _CHUD_THREAD_H_
+#define _CHUD_THREAD_H_
+
+/* Flags for the t_chud element of a thread_t structure. */
+#define T_CHUD_MARKED   0x1	/* this thread is marked by CHUD */
+#define T_IN_CHUD       0x2	/* this thread is already in a CHUD handler */
+
+#endif /* _CHUD_THREAD_H_ */
diff --git a/osfmk/chud/chud_xnu.h b/osfmk/chud/chud_xnu.h
index 992aa65f3..9577e8088 100644
--- a/osfmk/chud/chud_xnu.h
+++ b/osfmk/chud/chud_xnu.h
@@ -1,29 +1,34 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _CHUD_XNU_H_
 #define _CHUD_XNU_H_
 
-
 #include <stdint.h>
 #include <mach/boolean.h>
 #include <mach/mach_types.h>
@@ -50,14 +55,13 @@ extern boolean_t chudxnu_is_64bit_task(task_t task);
 // ********************************************************************************
 // thread
 // ********************************************************************************
-extern kern_return_t chudxnu_bind_thread(thread_t thread, int cpu);
-extern kern_return_t chudxnu_unbind_thread(thread_t thread);
+extern kern_return_t chudxnu_bind_thread(thread_t thread, int cpu, int options);
+extern kern_return_t chudxnu_unbind_thread(thread_t thread, int options);
 
 extern kern_return_t chudxnu_thread_get_state(thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t *count, boolean_t user_only);
 extern kern_return_t chudxnu_thread_set_state(thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count, boolean_t user_only);
 extern kern_return_t chudxnu_thread_user_state_available(thread_t thread);
 
-extern kern_return_t chudxnu_thread_get_callstack(thread_t thread, uint32_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
 extern kern_return_t chudxnu_thread_get_callstack64(thread_t thread, uint64_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
 
 extern task_t chudxnu_current_task(void);
@@ -76,6 +80,10 @@ extern kern_return_t chudxnu_thread_info(  thread_t thread, thread_flavor_t flav
 
 extern kern_return_t chudxnu_thread_last_context_switch(thread_t thread, uint64_t *timestamp);
 
+extern boolean_t chudxnu_thread_set_marked(thread_t thread, boolean_t marked);
+extern boolean_t chudxnu_thread_get_marked(thread_t thread);
+extern boolean_t chudxnu_thread_get_idle(thread_t thread);
+
 #pragma mark **** memory ****
 // ********************************************************************************
 // memory
@@ -84,24 +92,18 @@ extern kern_return_t chudxnu_thread_last_context_switch(thread_t thread, uint64_
 extern uint64_t chudxnu_avail_memory_size(void);
 extern uint64_t chudxnu_phys_memory_size(void);
 extern uint64_t chudxnu_free_memory_size(void);
-
-extern vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size);
-
-extern uint32_t chudxnu_phys_addr_wimg(uint64_t phys_addr);
+extern uint64_t chudxnu_inactive_memory_size(void);
 
 #pragma mark **** cpu ****
 // ********************************************************************************
 // cpu
 // ********************************************************************************
-extern int chudxnu_avail_cpu_count(void);
+extern int chudxnu_logical_cpu_count(void);
 extern int chudxnu_phys_cpu_count(void);
 extern int chudxnu_cpu_number(void);
 
 extern kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable);
 
-extern kern_return_t chudxnu_enable_cpu_nap(int cpu, boolean_t enable);
-extern boolean_t chudxnu_cpu_nap_enabled(int cpu);
-
 extern boolean_t chudxnu_get_interrupts_enabled(void);
 extern boolean_t chudxnu_set_interrupts_enabled(boolean_t enable);
 extern boolean_t chudxnu_at_interrupt_context(void);
@@ -114,22 +116,9 @@ extern int chudxnu_get_preemption_level(void);
 extern kern_return_t chudxnu_set_shadowed_spr(int cpu, int spr, uint32_t val);
 extern kern_return_t chudxnu_set_shadowed_spr64(int cpu, int spr, uint64_t val);
 
-extern uint32_t chudxnu_get_orig_cpu_l2cr(int cpu);
-extern uint32_t chudxnu_get_orig_cpu_l3cr(int cpu);
-
-extern kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p);
-extern kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p);
-extern kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val);
-extern kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val);
-
-extern void chudxnu_flush_caches(void);
-extern void chudxnu_enable_caches(boolean_t enable);
-
 extern kern_return_t chudxnu_perfmon_acquire_facility(task_t);
 extern kern_return_t chudxnu_perfmon_release_facility(task_t);
 
-extern uint32_t * chudxnu_get_branch_trace_buffer(uint32_t *entries);
-
 typedef struct {
     uint32_t hwResets;
     uint32_t hwMachineChecks;
@@ -155,13 +144,8 @@ typedef struct {
     uint32_t hwInstrumentations;
 } rupt_counters_t;
 
-extern kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts);
-extern kern_return_t chudxnu_clear_cpu_rupt_counters(int cpu);
-
-extern kern_return_t chudxnu_passup_alignment_exceptions(boolean_t enable);
-
-extern kern_return_t chudxnu_scom_read(uint32_t reg, uint64_t *data);
-extern kern_return_t chudxnu_scom_write(uint32_t reg, uint64_t data);
+extern kern_return_t chudxnu_get_cpu_interrupt_counters(int cpu, rupt_counters_t *rupts);
+extern kern_return_t chudxnu_clear_cpu_interrupt_counters(int cpu);
 
 #pragma mark **** callbacks ****
 // ********************************************************************************
@@ -210,7 +194,6 @@ extern kern_return_t chudxnu_interrupt_callback_cancel(void);
 typedef kern_return_t (*chudxnu_perfmon_ast_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate,  mach_msg_type_number_t count);
 extern kern_return_t chudxnu_perfmon_ast_callback_enter(chudxnu_perfmon_ast_callback_func_t func);
 extern kern_return_t chudxnu_perfmon_ast_callback_cancel(void);
-extern kern_return_t chudxnu_perfmon_ast_send(void);
 extern kern_return_t chudxnu_perfmon_ast_send_urgent(boolean_t urgent);
 
 // cpusig callback - one callback for system
@@ -233,25 +216,47 @@ extern kern_return_t chudxnu_timer_callback_cancel(chud_timer_t timer);
 extern kern_return_t chudxnu_timer_free(chud_timer_t timer);
 
 // CHUD systemcall callback - one callback for system
-//typedef kern_return_t (*chudxnu_syscall_callback_func_t)(thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count); // v2
 typedef kern_return_t (*chudxnu_syscall_callback_func_t)(uint32_t code, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4);
 extern kern_return_t chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func);
 extern kern_return_t chudxnu_syscall_callback_cancel(void);
 
+// DTrace Triggering
+typedef kern_return_t (*chudxnu_dtrace_callback_t)(uint64_t selector, uint64_t *args, uint32_t count);
+extern kern_return_t chudxnu_dtrace_callback(uint64_t selector, uint64_t *args, uint32_t count);
+extern void chudxnu_dtrace_callback_enter(chudxnu_dtrace_callback_t fn);
+extern void chudxnu_dtrace_callback_cancel(void);
+
 // ********************************************************************************
 // DEPRECATED
 // ********************************************************************************
-extern kern_return_t chudxnu_bind_current_thread(int cpu);
+extern kern_return_t chudxnu_perfmon_ast_send(void);
+extern kern_return_t chudxnu_thread_get_callstack(thread_t thread, uint32_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
 
-extern kern_return_t chudxnu_unbind_current_thread(void);
+extern vm_offset_t chudxnu_io_map(uint64_t phys_addr, vm_size_t size);
+extern uint32_t chudxnu_phys_addr_wimg(uint64_t phys_addr);
 
-extern kern_return_t chudxnu_current_thread_get_callstack(uint32_t *callStack, mach_msg_type_number_t *count, boolean_t user_only);
-                        
-extern thread_t chudxnu_current_act(void);
+extern int chudxnu_avail_cpu_count(void);
 
-// thread timer callback - one callback for system
-typedef kern_return_t (*chudxnu_thread_timer_callback_func_t)(uint32_t param);
-extern kern_return_t chudxnu_thread_timer_callback_enter(chudxnu_thread_timer_callback_func_t func, uint32_t param, uint32_t time, uint32_t units);
-extern kern_return_t chudxnu_thread_timer_callback_cancel(void);
+extern kern_return_t chudxnu_enable_cpu_nap(int cpu, boolean_t enable);
+extern boolean_t chudxnu_cpu_nap_enabled(int cpu);
+
+extern uint32_t chudxnu_get_orig_cpu_l2cr(int cpu);
+extern uint32_t chudxnu_get_orig_cpu_l3cr(int cpu);
+
+extern kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p);
+extern kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p);
+extern kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val);
+extern kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val);
+
+extern kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts);
+extern kern_return_t chudxnu_clear_cpu_rupt_counters(int cpu);
+
+extern kern_return_t chudxnu_passup_alignment_exceptions(boolean_t enable);
+
+extern kern_return_t chudxnu_scom_read(uint32_t reg, uint64_t *data);
+extern kern_return_t chudxnu_scom_write(uint32_t reg, uint64_t data);
+
+extern void chudxnu_flush_caches(void);
+extern void chudxnu_enable_caches(boolean_t enable);
 
 #endif /* _CHUD_XNU_H_ */
diff --git a/osfmk/chud/chud_xnu_glue.h b/osfmk/chud/chud_xnu_glue.h
index ab912af79..876c72513 100644
--- a/osfmk/chud/chud_xnu_glue.h
+++ b/osfmk/chud/chud_xnu_glue.h
@@ -1,29 +1,37 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *           
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #if defined (__ppc__)
 #include "ppc/chud_xnu_glue.h"
 #elif defined (__i386__)
 #include "i386/chud_xnu_glue.h"
+#elif defined (__arm__)
+#include "arm/chud_xnu_glue.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/chud/chud_xnu_private.h b/osfmk/chud/chud_xnu_private.h
index 3028a70d3..2908bfccb 100644
--- a/osfmk/chud/chud_xnu_private.h
+++ b/osfmk/chud/chud_xnu_private.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _CHUD_XNU_PRIVATE_H_
@@ -31,6 +37,8 @@
 #include "chud/ppc/chud_xnu_private.h"
 #elif defined (__i386__)
 #include "chud/i386/chud_xnu_private.h"
+#elif defined (__arm__)
+#include "chud/arm/chud_xnu_private.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/chud/i386/chud_cpu_asm.h b/osfmk/chud/i386/chud_cpu_asm.h
index ee9308606..6f5938897 100644
--- a/osfmk/chud/i386/chud_cpu_asm.h
+++ b/osfmk/chud/i386/chud_cpu_asm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _CHUD_CPU_ASM_H_
diff --git a/osfmk/chud/i386/chud_cpu_asm.s b/osfmk/chud/i386/chud_cpu_asm.s
index 838afe22e..aed4310e5 100644
--- a/osfmk/chud/i386/chud_cpu_asm.s
+++ b/osfmk/chud/i386/chud_cpu_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #define ASSEMBLER
diff --git a/osfmk/chud/i386/chud_cpu_i386.c b/osfmk/chud/i386/chud_cpu_i386.c
index 12d91f3b4..8510249dc 100644
--- a/osfmk/chud/i386/chud_cpu_i386.c
+++ b/osfmk/chud/i386/chud_cpu_i386.c
@@ -1,24 +1,31 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 #include <mach/mach_types.h>
 #include <mach/mach_host.h>
 
@@ -29,6 +36,8 @@
 #include <i386/machine_routines.h>
 #include <i386/perfmon.h>
 #include <i386/mp.h>
+#include <i386/trap.h>
+#include <mach/i386/syscall_sw.h>
 
 #include <chud/chud_xnu.h>
 
@@ -40,7 +49,7 @@ extern kern_return_t processor_exit(processor_t     processor); // osfmk/kern/pr
 __private_extern__
 kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable)
 {
-    chudxnu_unbind_thread(current_thread());
+    chudxnu_unbind_thread(current_thread(), 0);
 	
     if(cpu < 0 || (unsigned int)cpu >= real_ncpus) // sanity check
         return KERN_FAILURE;
@@ -52,41 +61,14 @@ kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable)
 			return KERN_FAILURE;
 
         if(enable) {
-			// make sure it isn't already running
-			if(processor->state == PROCESSOR_OFF_LINE || 
-				processor->state == PROCESSOR_SHUTDOWN) {
-				return processor_start(processor);
-			}
-			return KERN_SUCCESS;	// it's already running
+            return processor_start(processor);
         } else {
-			// make sure it hasn't already exited
-			if(processor->state != PROCESSOR_OFF_LINE &&
-				processor->state != PROCESSOR_SHUTDOWN) {
-				return processor_exit(processor);
-			}
-			return KERN_SUCCESS;
+            return processor_exit(processor);
         }
     }
     return KERN_FAILURE;
 }
 
-#pragma mark **** cache flush ****
-
-__private_extern__
-void
-chudxnu_flush_caches(void)
-{
-/* XXX */
-}
-
-__private_extern__
-void
-chudxnu_enable_caches(boolean_t enable)
-{
-#pragma unused (enable)
-/* XXX */
-}
-
 #pragma mark **** perfmon facility ****
 
 __private_extern__ kern_return_t
@@ -101,10 +83,10 @@ chudxnu_perfmon_release_facility(task_t task)
     return pmc_release(task);
 }
 
-#pragma mark **** rupt counters ****
+#pragma mark **** interrupt counters ****
 
 __private_extern__ kern_return_t
-chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
+chudxnu_get_cpu_interrupt_counters(int cpu, rupt_counters_t *rupts)
 {
     if(cpu < 0 || (unsigned int)cpu >= real_ncpus) { // sanity check
         return KERN_FAILURE;
@@ -115,28 +97,43 @@ chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
         cpu_data_t	*per_proc;
 
         per_proc = cpu_data_ptr[cpu];
-        rupts->hwResets = 0;
-        rupts->hwMachineChecks = 0;
+		// For now, we'll call an NMI a 'reset' interrupt
+        rupts->hwResets = per_proc->cpu_hwIntCnt[T_NMI];
+        rupts->hwMachineChecks = per_proc->cpu_hwIntCnt[T_MACHINE_CHECK];
         rupts->hwDSIs = 0;
         rupts->hwISIs = 0;
+		// we could accumulate 0x20-0x7f, but that'd likely overflow...
         rupts->hwExternals = 0;
-        rupts->hwAlignments = 0;
+		// This appears to be wrong.
+        rupts->hwAlignments = 0; //per_proc->cpu_hwIntCnt[0x11];
         rupts->hwPrograms = 0;
-        rupts->hwFloatPointUnavailable = 0;
-        rupts->hwDecrementers = 0;
-        rupts->hwIOErrors = 0;
-        rupts->hwSystemCalls = 0;
-        rupts->hwTraces = 0;
+        rupts->hwFloatPointUnavailable = per_proc->cpu_hwIntCnt[T_NO_FPU];
+		// osfmk/i386/mp.h
+        rupts->hwDecrementers = per_proc->cpu_hwIntCnt[LAPIC_VECTOR(TIMER)];
+		// LAPIC_ERROR == IO ERROR??
+        rupts->hwIOErrors = per_proc->cpu_hwIntCnt[LAPIC_VECTOR(ERROR)];
+
+		// accumulate all system call types
+		// osfmk/mach/i386/syscall_sw.h
+        rupts->hwSystemCalls = per_proc->cpu_hwIntCnt[UNIX_INT]  +
+			per_proc->cpu_hwIntCnt[MACH_INT] +
+			per_proc->cpu_hwIntCnt[MACHDEP_INT] +
+			per_proc->cpu_hwIntCnt[DIAG_INT];
+
+        rupts->hwTraces = per_proc->cpu_hwIntCnt[T_DEBUG]; // single steps == traces??
         rupts->hwFloatingPointAssists = 0;
-        rupts->hwPerformanceMonitors = 0;
+		// osfmk/i386/mp.h
+        rupts->hwPerformanceMonitors =
+			per_proc->cpu_hwIntCnt[LAPIC_VECTOR(PERFCNT)];
         rupts->hwAltivecs = 0;
-        rupts->hwInstBreakpoints = 0;
+        rupts->hwInstBreakpoints = per_proc->cpu_hwIntCnt[T_INT3];
         rupts->hwSystemManagements = 0;
         rupts->hwAltivecAssists = 0;
-        rupts->hwThermal = 0;
+        rupts->hwThermal = per_proc->cpu_hwIntCnt[LAPIC_VECTOR(THERMAL)];
         rupts->hwSoftPatches = 0;
         rupts->hwMaintenances = 0;
-        rupts->hwInstrumentations = 0;
+		// Watchpoint == instrumentation
+		rupts->hwInstrumentations = per_proc->cpu_hwIntCnt[T_WATCHPOINT]; 
 
         ml_set_interrupts_enabled(oldlevel);
         return KERN_SUCCESS;
@@ -146,15 +143,32 @@ chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
 }
 
 __private_extern__ kern_return_t
-chudxnu_clear_cpu_rupt_counters(int cpu)
+chudxnu_clear_cpu_interrupt_counters(int cpu)
 {
     if(cpu < 0 || (unsigned int)cpu >= real_ncpus) { // sanity check
         return KERN_FAILURE;
     }
+	cpu_data_t	*per_proc;
+
+	per_proc = cpu_data_ptr[cpu];
+
+	bzero((char *)per_proc->cpu_hwIntCnt, sizeof(uint32_t)*256);
 
-/*
- * XXX
- *    bzero((char *)&(cpu_data_ptr[cpu]->hwCtrs), sizeof(struct hwCtrs));
- */
     return KERN_SUCCESS;
 }
+
+#pragma mark *** deprecated ***
+
+//DEPRECATED
+__private_extern__ kern_return_t
+chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
+{
+	return chudxnu_get_cpu_interrupt_counters(cpu, rupts);
+}
+
+//DEPRECATED
+__private_extern__ kern_return_t
+chudxnu_clear_cpu_rupt_counters(int cpu)
+{
+	return chudxnu_clear_cpu_interrupt_counters(cpu);
+}
diff --git a/osfmk/chud/i386/chud_osfmk_callback_i386.c b/osfmk/chud/i386/chud_osfmk_callback_i386.c
index 9800764df..a3f12cf15 100644
--- a/osfmk/chud/i386/chud_osfmk_callback_i386.c
+++ b/osfmk/chud/i386/chud_osfmk_callback_i386.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -37,6 +43,7 @@
 
 #include <chud/chud_xnu.h>
 #include <chud/chud_xnu_private.h>
+#include <chud/chud_thread.h>
 
 #include <i386/misc_protos.h>
 #include <i386/mp.h>
@@ -59,11 +66,9 @@ void chudxnu_cancel_all_callbacks(void)
     chudxnu_interrupt_callback_cancel();
     chudxnu_perfmon_ast_callback_cancel();
     chudxnu_kdebug_callback_cancel();
-    chudxnu_thread_timer_callback_cancel();
     chudxnu_trap_callback_cancel();
-#if XXX
 	chudxnu_syscall_callback_cancel();
-#endif
+	chudxnu_dtrace_callback_cancel();
 }
 
 static chudcpu_data_t chudcpu_boot_cpu;
@@ -72,7 +77,6 @@ chudxnu_cpu_alloc(boolean_t boot_processor)
 {
 	chudcpu_data_t	*chud_proc_info;
 
-
 	if (boot_processor) {
 		chud_proc_info = &chudcpu_boot_cpu;
 	} else {
@@ -217,7 +221,7 @@ static chudxnu_trap_callback_func_t trap_callback_fn = NULL;
 
 static kern_return_t
 chudxnu_private_trap_callback(
-	int			trapno,
+	int trapno,
 	void			*regs,
 	int			unused1,
 	int			unused2)
@@ -225,34 +229,50 @@ chudxnu_private_trap_callback(
 #pragma unused (regs)
 #pragma unused (unused1)
 #pragma unused (unused2)
-    kern_return_t retval = KERN_FAILURE;
+	kern_return_t retval = KERN_FAILURE;
 	chudxnu_trap_callback_func_t fn = trap_callback_fn;
 
-        if(fn) {
-		boolean_t			oldlevel;
-		x86_thread_state_t	state;	// once we have an 64bit- independent way to determine if a thread is
-									// running kernel code, we'll switch to x86_thread_state_t.
-		mach_msg_type_number_t		count;	
-
+	if(fn) {
+		boolean_t oldlevel;
+		x86_thread_state_t state;
+		mach_msg_type_number_t count;
+		thread_t thread = current_thread();
+		
 		oldlevel = ml_set_interrupts_enabled(FALSE);
+		
+		/* prevent reentry into CHUD when dtracing */
+		if(thread->t_chud & T_IN_CHUD) {
+			/* restore interrupts */
+			ml_set_interrupts_enabled(oldlevel);
+
+			return KERN_FAILURE;	// not handled - pass off to dtrace
+		}
+
+		/* update the chud state bits */
+		thread->t_chud |= T_IN_CHUD;
 
 		count = x86_THREAD_STATE_COUNT;
-		if(chudxnu_thread_get_state(current_thread(),
+		
+		if(chudxnu_thread_get_state(thread,
 				x86_THREAD_STATE,
 				(thread_state_t)&state,
 				&count,
 				FALSE) == KERN_SUCCESS) {
-	  
-		  retval = (fn)(
-				  trapno,
-				  x86_THREAD_STATE,
-				  (thread_state_t)&state,
-				  count);
+		  
+					retval = (fn)(
+						trapno,
+						x86_THREAD_STATE,
+						(thread_state_t)&state,
+						count);
 		}
-    ml_set_interrupts_enabled(oldlevel);
+
+		/* no longer in CHUD */
+		thread->t_chud &= ~(T_IN_CHUD);
+
+		ml_set_interrupts_enabled(oldlevel);
 	}
 
-    return retval;
+	return retval;
 }
 
 __private_extern__ kern_return_t
@@ -305,7 +325,7 @@ chudxnu_private_chud_ast_callback(
 		x86_thread_state_t state;
 		mach_msg_type_number_t count;
 		count = x86_THREAD_STATE_COUNT;
-		
+
 		if (chudxnu_thread_get_state(
 			current_thread(),
 			x86_THREAD_STATE,
@@ -542,49 +562,3 @@ chudxnu_cpusig_send(int otherCPU, uint32_t request_code)
 	return retval;
 }
 
-#ifdef XXX
-#pragma mark **** CHUD syscall (PPC) ****
-
-typedef int (*PPCcallEnt)(struct savearea *save);
-extern PPCcallEnt      PPCcalls[];
-
-static chudxnu_syscall_callback_func_t syscall_callback_fn = NULL;
-
-static int
-chudxnu_private_syscall_callback(struct savearea *ssp)
-{
-	if(ssp) {
-		if(syscall_callback_fn) {
-			struct ppc_thread_state64 state;
-			kern_return_t retval;
-			mach_msg_type_number_t count = PPC_THREAD_STATE64_COUNT;
-			chudxnu_copy_savearea_to_threadstate(PPC_THREAD_STATE64, (thread_state_t)&state, &count, ssp);
-			ssp->save_r3 = (syscall_callback_fn)(PPC_THREAD_STATE64, (thread_state_t)&state, count);
-		} else {
-			ssp->save_r3 = KERN_FAILURE;
-		}
-	}
-	
-    return 1; // check for ASTs (always)
-}
-
-__private_extern__ kern_return_t
-chudxnu_syscall_callback_enter(chudxnu_syscall_callback_func_t func)
-{
-	syscall_callback_fn = func;
-	PPCcalls[9] = chudxnu_private_syscall_callback;
-    __asm__ volatile("eieio");	/* force order */
-    __asm__ volatile("sync");	/* force to memory */
-    return KERN_SUCCESS;
-}
-
-__private_extern__ kern_return_t
-chudxnu_syscall_callback_cancel(void)
-{
-	syscall_callback_fn = NULL;
-	PPCcalls[9] = NULL;
-    __asm__ volatile("eieio");	/* force order */
-    __asm__ volatile("sync");	/* force to memory */
-    return KERN_SUCCESS;
-}
-#endif
diff --git a/osfmk/chud/i386/chud_thread_i386.c b/osfmk/chud/i386/chud_thread_i386.c
index ef90c2379..c2b36c794 100644
--- a/osfmk/chud/i386/chud_thread_i386.c
+++ b/osfmk/chud/i386/chud_thread_i386.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -49,11 +55,11 @@ chudxnu_thread_user_state_available(thread_t thread)
 
 __private_extern__ kern_return_t
 chudxnu_thread_get_state(
-	thread_t	 	thread, 
-	thread_flavor_t	 	flavor,
-	thread_state_t	 	tstate,
-	mach_msg_type_number_t	*count,
-	boolean_t	 	user_only)
+						 thread_t	 	thread, 
+						 thread_flavor_t	 	flavor,
+						 thread_state_t	 	tstate,
+						 mach_msg_type_number_t	*count,
+						 boolean_t	 	user_only)
 {
 	if (user_only) {
 		/* We can't get user state for kernel threads */
@@ -65,7 +71,7 @@ chudxnu_thread_get_state(
 		// i386 machine_thread_get_kern_state() is different from the PPC version which returns
 		// the previous save area - user or kernel - rather than kernel or NULL if no kernel
 		// interrupt state available
-	
+		
 		// the real purpose of this branch is the following:
 		// the user doesn't care if the thread states are user or kernel, he
 		// just wants the thread state, so we need to determine the proper one
@@ -75,7 +81,7 @@ chudxnu_thread_get_state(
 			// state. we still need to determine if this interrupt happened in
 			// kernel or user context
 			if(USER_STATE(thread) == current_cpu_datap()->cpu_int_state &&
-				current_cpu_datap()->cpu_interrupt_level == 1) {
+			   current_cpu_datap()->cpu_interrupt_level == 1) {
 				// interrupt happened in user land
 				return machine_thread_get_state(thread, flavor, tstate, count);
 			} else {
@@ -91,32 +97,39 @@ chudxnu_thread_get_state(
 
 __private_extern__ kern_return_t
 chudxnu_thread_set_state(
-	thread_t		thread, 
-	thread_flavor_t		flavor,
-	thread_state_t		tstate,
-	mach_msg_type_number_t	count,
-	boolean_t 		user_only)
+						 thread_t		thread, 
+						 thread_flavor_t		flavor,
+						 thread_state_t		tstate,
+						 mach_msg_type_number_t	count,
+						 boolean_t 		user_only)
 {
 #pragma unused (user_only)
 	return machine_thread_set_state(thread, flavor, tstate, count);
 }
 
 #pragma mark **** task memory read/write ****
-    
+
 __private_extern__ kern_return_t
 chudxnu_task_read(
-	task_t		task,
-	void		*kernaddr,
-	uint64_t	usraddr,
-	vm_size_t	size)
+				  task_t		task,
+				  void		*kernaddr,
+				  uint64_t	usraddr,
+				  vm_size_t	size)
 {
 	kern_return_t ret = KERN_SUCCESS;
-    
-	if(current_task()==task) {
-		if(ml_at_interrupt_context()) {
-			return KERN_FAILURE; // can't do copyin on interrupt stack
-		}
+	boolean_t old_level;
 	
+	if(ml_at_interrupt_context()) {
+		return KERN_FAILURE; // Can't look at tasks on interrupt stack
+	}
+
+	/*
+	 * pmap layer requires interrupts to be on
+	 */
+	old_level = ml_set_interrupts_enabled(TRUE);
+	
+	if(current_task()==task) {	
+		
 		if(copyin(usraddr, kernaddr, size)) {
 			ret = KERN_FAILURE;
 		}
@@ -125,23 +138,32 @@ chudxnu_task_read(
 		ret = vm_map_read_user(map, usraddr, kernaddr, size);
 	}
     
+	ml_set_interrupts_enabled(old_level);
+
 	return ret;
 }
-			
+
 __private_extern__ kern_return_t
 chudxnu_task_write(
-	task_t		task,
-	uint64_t	useraddr,
-	void		*kernaddr,
-	vm_size_t	size)
+				   task_t		task,
+				   uint64_t	useraddr,
+				   void		*kernaddr,
+				   vm_size_t	size)
 {
 	kern_return_t ret = KERN_SUCCESS;
-    
-	if(current_task()==task) {    
-		if(ml_at_interrupt_context()) {
-			return KERN_FAILURE; // can't do copyout on interrupt stack
-		}
+	boolean_t old_level;
+	
+	if(ml_at_interrupt_context()) {
+		return KERN_FAILURE; // can't poke into tasks on interrupt stack
+	}
+
+	/*
+	 * pmap layer requires interrupts to be on
+	 */
+	old_level = ml_set_interrupts_enabled(TRUE);
 	
+	if(current_task()==task) {    
+		
 		if(copyout(kernaddr, useraddr, size)) {
 			ret = KERN_FAILURE;
 		}
@@ -149,102 +171,34 @@ chudxnu_task_write(
 		vm_map_t map = get_task_map(task);
 		ret = vm_map_write_user(map, kernaddr, useraddr, size);
 	}		
-    
+	
+	ml_set_interrupts_enabled(old_level);
+
 	return ret;
 }
 
 __private_extern__ kern_return_t
 chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
 {
-    while(size>0) {
-		ppnum_t pp;
-		addr64_t phys_addr;    
-		
-		/* Get the page number */
-		pp = pmap_find_phys(kernel_pmap, srcaddr);
-		if(!pp) {
-			return KERN_FAILURE;	/* Not mapped... */
-		}
-		
-		/* Shove in the page offset */
-		phys_addr = ((addr64_t)pp << 12) |
-				(srcaddr & 0x0000000000000FFFULL);
-		if(phys_addr >= mem_actual) {
-			return KERN_FAILURE;	/* out of range */
-		}
-		
-		if((phys_addr&0x1) || size==1) {
-			*((uint8_t *)dstaddr) =
-				ml_phys_read_byte_64(phys_addr);
-			dstaddr = ((uint8_t *)dstaddr) + 1;
-			srcaddr += sizeof(uint8_t);
-			size -= sizeof(uint8_t);
-		} else if((phys_addr&0x3) || size<=2) {
-			*((uint16_t *)dstaddr) =
-				ml_phys_read_half_64(phys_addr);
-			dstaddr = ((uint16_t *)dstaddr) + 1;
-			srcaddr += sizeof(uint16_t);
-			size -= sizeof(uint16_t);
-		} else {
-			*((uint32_t *)dstaddr) =
-				ml_phys_read_word_64(phys_addr);
-			dstaddr = ((uint32_t *)dstaddr) + 1;
-			srcaddr += sizeof(uint32_t);
-			size -= sizeof(uint32_t);
-		}
-    }
-    return KERN_SUCCESS;
+	return (ml_nofault_copy(srcaddr, (vm_offset_t) dstaddr, size) == size ?
+			KERN_SUCCESS: KERN_FAILURE);
 }
 
 __private_extern__ kern_return_t
 chudxnu_kern_write(
-	vm_offset_t	dstaddr,
-	void		*srcaddr,
-	vm_size_t	size)
+				   vm_offset_t	dstaddr,
+				   void		*srcaddr,
+				   vm_size_t	size)
 {
-	while(size>0) {
-		ppnum_t pp;
-		addr64_t phys_addr;    
-		
-		/* Get the page number */
-		pp = pmap_find_phys(kernel_pmap, dstaddr);
-		if(!pp) {
-			return KERN_FAILURE;	/* Not mapped... */
-		}
-		
-		/* Shove in the page offset */
-		phys_addr = ((addr64_t)pp << 12) |
-				(dstaddr & 0x0000000000000FFFULL);
-		if(phys_addr > mem_actual) {
-			return KERN_FAILURE;	/* out of range */
-		}
-		
-		if((phys_addr&0x1) || size==1) {
-			ml_phys_write_byte_64(phys_addr, *((uint8_t *)srcaddr));
-			srcaddr = ((uint8_t *)srcaddr) + 1;
-			dstaddr += sizeof(uint8_t);
-			size -= sizeof(uint8_t);
-		} else if((phys_addr&0x3) || size<=2) {
-			ml_phys_write_half_64(phys_addr, *((uint16_t *)srcaddr));
-			srcaddr = ((uint16_t *)srcaddr) + 1;
-			dstaddr += sizeof(uint16_t);
-			size -= sizeof(uint16_t);
-		} else {
-			ml_phys_write_word_64(phys_addr, *((uint32_t *)srcaddr));
-			srcaddr = ((uint32_t *)srcaddr) + 1;
-			dstaddr += sizeof(uint32_t);
-			size -= sizeof(uint32_t);
-		}
-    }
-    
-    return KERN_SUCCESS;
+	return (ml_nofault_copy((vm_offset_t) srcaddr, dstaddr, size) == size ?
+			KERN_SUCCESS: KERN_FAILURE);
 }
 
 #define VALID_STACK_ADDRESS(supervisor, addr, minKernAddr, maxKernAddr)   (supervisor ? (addr>=minKernAddr && addr<=maxKernAddr) : TRUE)
 // don't try to read in the hole
 #define VALID_STACK_ADDRESS64(supervisor, addr, minKernAddr, maxKernAddr) \
-	(supervisor ? (addr >= minKernAddr && addr <= maxKernAddr) : \
-		(addr != 0 && (addr <= 0x00007FFFFFFFFFFFULL || addr >= 0xFFFF800000000000ULL)))
+(supervisor ? (addr >= minKernAddr && addr <= maxKernAddr) : \
+(addr != 0 && (addr <= 0x00007FFFFFFFFFFFULL || addr >= 0xFFFF800000000000ULL)))
 
 typedef struct _cframe64_t {
 	uint64_t	prevFP;		// can't use a real pointer here until we're a 64 bit kernel
@@ -259,6 +213,164 @@ typedef struct _cframe_t {
 	uint32_t		args[0];
 } cframe_t;
 
+extern void * find_user_regs(thread_t);
+extern x86_saved_state32_t *find_kern_regs(thread_t);
+
+static kern_return_t do_backtrace32(
+	task_t task,
+	thread_t thread,
+	x86_saved_state32_t *regs, 
+	uint64_t *frames,
+	mach_msg_type_number_t *start_idx,
+	mach_msg_type_number_t max_idx,
+	boolean_t supervisor)
+{
+	uint32_t tmpWord = 0UL;
+	uint64_t currPC = (uint64_t) regs->eip;
+	uint64_t currFP = (uint64_t) regs->ebp;
+	uint64_t prevPC = 0ULL;
+	uint64_t prevFP = 0ULL;
+	uint64_t kernStackMin = thread->kernel_stack;
+    uint64_t kernStackMax = kernStackMin + KERNEL_STACK_SIZE;
+	mach_msg_type_number_t ct = *start_idx;
+	kern_return_t kr = KERN_FAILURE;
+
+	if(ct >= max_idx)
+		return KERN_RESOURCE_SHORTAGE;	// no frames traced
+	
+	frames[ct++] = currPC;
+
+	// build a backtrace of this 32 bit state.
+	while(VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
+		cframe_t *fp = (cframe_t *) (uint32_t) currFP;
+
+        if(!currFP) {
+            currPC = 0;
+            break;
+        }
+
+        if(ct >= max_idx) {
+			*start_idx = ct;
+            return KERN_RESOURCE_SHORTAGE;
+        }
+
+		/* read our caller */
+		if(supervisor) {
+			kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
+		} else {
+			kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
+		}
+
+		if(kr != KERN_SUCCESS) {
+			currPC = 0ULL;
+			break;
+		}
+
+		currPC = (uint64_t) tmpWord;    // promote 32 bit address
+
+        /* 
+         * retrive contents of the frame pointer and advance to the next stack
+         * frame if it's valid 
+         */
+        prevFP = 0;
+		if(supervisor) {
+			kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
+		} else {
+			kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
+		}
+		prevFP = (uint64_t) tmpWord;    // promote 32 bit address
+
+        if(prevFP) {
+            frames[ct++] = currPC;
+            prevPC = currPC;
+        }
+        if(prevFP < currFP) {
+            break;
+        } else {
+            currFP = prevFP;
+        }	
+	}
+
+	*start_idx = ct;
+	return KERN_SUCCESS;
+}
+
+static kern_return_t do_backtrace64(
+	task_t task,
+	thread_t thread,
+	x86_saved_state64_t *regs, 
+	uint64_t *frames,
+	mach_msg_type_number_t *start_idx,
+	mach_msg_type_number_t max_idx,
+	boolean_t supervisor)
+{
+	uint64_t currPC = regs->isf.rip;
+	uint64_t currFP = regs->rbp;
+	uint64_t prevPC = 0ULL;
+	uint64_t prevFP = 0ULL;
+	uint64_t kernStackMin = (uint64_t)thread->kernel_stack;
+    uint64_t kernStackMax = (uint64_t)kernStackMin + KERNEL_STACK_SIZE;
+	mach_msg_type_number_t ct = *start_idx;
+	kern_return_t kr = KERN_FAILURE;
+
+	if(*start_idx >= max_idx)
+		return KERN_RESOURCE_SHORTAGE;	// no frames traced
+	
+	frames[ct++] = currPC;
+
+	// build a backtrace of this 32 bit state.
+	while(VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax)) {
+		// this is the address where caller lives in the user thread
+		uint64_t caller = currFP + sizeof(uint64_t);
+
+        if(!currFP) {
+            currPC = 0;
+            break;
+        }
+
+        if(ct >= max_idx) {
+			*start_idx = ct;
+            return KERN_RESOURCE_SHORTAGE;
+        }
+
+		/* read our caller */
+		if(supervisor) {
+			kr = KERN_FAILURE;
+		} else {
+			kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));
+		}
+
+		if(kr != KERN_SUCCESS) {
+			currPC = 0ULL;
+			break;
+		}
+
+        /* 
+         * retrive contents of the frame pointer and advance to the next stack
+         * frame if it's valid 
+         */
+        prevFP = 0;
+		if(supervisor) {
+			kr = KERN_FAILURE;
+		} else {
+			kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
+		}
+
+        if(VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
+            frames[ct++] = currPC;
+            prevPC = currPC;
+        }
+        if(prevFP < currFP) {
+            break;
+        } else {
+            currFP = prevFP;
+        }	
+	}
+
+	*start_idx = ct;
+	return KERN_SUCCESS;
+}
+
 __private_extern__
 kern_return_t chudxnu_thread_get_callstack64(
 	thread_t		thread,
@@ -266,249 +378,177 @@ kern_return_t chudxnu_thread_get_callstack64(
 	mach_msg_type_number_t	*count,
 	boolean_t		user_only)
 {
-    kern_return_t kr = KERN_FAILURE;
-	kern_return_t ret = KERN_SUCCESS;
+	kern_return_t kr = KERN_FAILURE;
     task_t task = thread->task;
     uint64_t currPC = 0;
-	uint64_t prevPC = 0;
-    uint64_t currFP = 0;
-    uint64_t prevFP = 0;
-    uint64_t rsp = 0;
-    uint64_t kernStackMin = min_valid_stack_address();
-    uint64_t kernStackMax = max_valid_stack_address();
-    uint64_t *buffer = callstack;
-    int bufferIndex = 0;
-    int bufferMaxIndex = *count;
-    boolean_t supervisor = FALSE;
-	boolean_t is64bit = FALSE;
-	void * t_regs;
-	
-	if (user_only) {
-		/* We can't get user state for kernel threads */
-		if (task == kernel_task) {
+	boolean_t supervisor = FALSE;
+    mach_msg_type_number_t bufferIndex = 0;
+    mach_msg_type_number_t bufferMaxIndex = *count;
+	x86_saved_state_t *tagged_regs = NULL;		// kernel register state
+	x86_saved_state64_t *regs64 = NULL;
+	x86_saved_state32_t *regs32 = NULL;
+	x86_saved_state32_t *u_regs32 = NULL;
+	x86_saved_state64_t *u_regs64 = NULL;
+
+	if(ml_at_interrupt_context()) {
+		
+		if(user_only) {
+			/* can't backtrace user state on interrupt stack. */
 			return KERN_FAILURE;
 		}
-        t_regs = USER_STATE(thread);
-		
-		if(is_saved_state64(t_regs)) {
-			void *int_state = current_cpu_datap()->cpu_int_state;
-			x86_saved_state64_t *s64 = saved_state64(t_regs);
+
+		/* backtracing at interrupt context? */
+		 if(thread == current_thread() && current_cpu_datap()->cpu_int_state) {
+			/* 
+			 * Locate the registers for the interrupted thread, assuming it is
+			 * current_thread(). 
+			 */
+			tagged_regs = current_cpu_datap()->cpu_int_state;
 			
-			if(int_state) {	// are we on an interrupt that happened in user land
-				supervisor = !(t_regs == int_state && current_cpu_datap()->cpu_interrupt_level == 1);
+			if(is_saved_state64(tagged_regs)) {
+				/* 64 bit registers */
+				regs64 = saved_state64(tagged_regs);
+				supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
 			} else {
-				if(s64) {
-					supervisor = ((s64->isf.cs & SEL_PL) != SEL_PL_U);
-				} else {
-					// assume 32 bit kernel
-					supervisor = FALSE;	
-				}
+				/* 32 bit registers */
+				regs32 = saved_state32(tagged_regs);
+				supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
 			}
-			is64bit = TRUE;
-		} else {
-			x86_saved_state32_t *regs;
+		} 
+	}
 
-			regs = saved_state32(t_regs);
-			
-			// find out if we're in supervisor mode
-			supervisor = ((regs->cs & SEL_PL) != SEL_PL_U);
-			is64bit = FALSE;
+	if(!tagged_regs) {
+		/* 
+		 * not at interrupt context, or tracing a different thread than
+		 * current_thread() at interrupt context 
+		 */
+		tagged_regs = USER_STATE(thread);
+		if(is_saved_state64(tagged_regs)) {
+			/* 64 bit registers */
+			regs64 = saved_state64(tagged_regs);
+			supervisor = ((regs64->isf.cs & SEL_PL) != SEL_PL_U);
+		} else {
+			/* 32 bit registers */
+			regs32 = saved_state32(tagged_regs);
+			supervisor = ((regs32->cs & SEL_PL) != SEL_PL_U);
 		}
-    } else {
-    	t_regs = current_cpu_datap()->cpu_int_state;
-		x86_saved_state32_t *regs;
+	}
 
-        regs = saved_state32(t_regs);
-		
-		// find out if we're in supervisor mode
-        supervisor = ((regs->cs & SEL_PL) != SEL_PL_U);
-		is64bit = FALSE;
-    }
-	
-	if(is64bit) {
-		x86_saved_state64_t *regs = saved_state64(t_regs);
-		
+	*count = 0; 
+
+	if(supervisor) {
+		// the caller only wants a user callstack.
 		if(user_only) {
-			/* cant get user state for kernel threads */
-			if(task == kernel_task) {
-				return KERN_FAILURE;
-			}
-			regs = USER_REGS64(thread);
-		} 
-		
-		currPC = regs->isf.rip;
-		currFP = regs->rbp;
-		
-		if(!currPC)
-		{
-			*count = 0;
+			// bail - we've only got kernel state
 			return KERN_FAILURE;
 		}
-		
-		bufferIndex = 0;
-		
-		//allot space for saving %rsp on the 
-		//bottom of the stack for user callstacks
-		if(!supervisor)
-			bufferMaxIndex = bufferMaxIndex - 1;    
-			
-		if(bufferMaxIndex < 1) {
-			*count = 0;
-			return KERN_RESOURCE_SHORTAGE;
+	} else {
+		// regs32(64) is not in supervisor mode.
+		u_regs32 = regs32;
+		u_regs64 = regs64;
+		regs32 = NULL;
+		regs64 = NULL;
+	}
+
+	if (user_only) {
+		/* we only want to backtrace the user mode */
+		if(!(u_regs32 || u_regs64)) {
+			/* no user state to look at */
+			return KERN_FAILURE;
 		}
-		buffer[bufferIndex++] = currPC; // save RIP on the top of the stack
-
-		// now make a 64bit back trace
-		while (VALID_STACK_ADDRESS64(supervisor, currFP, kernStackMin, kernStackMax))
-		{
-			// this is the address where caller lives in the user thread
-			uint64_t caller = currFP + sizeof(uint64_t);
-			if(!currFP) {
-				currPC = 0;
-				break;
-			}
-			
-			if(bufferIndex >= bufferMaxIndex) {
-				*count = bufferMaxIndex;
-				return KERN_RESOURCE_SHORTAGE;
-			}
+	}
 
-			/* read our caller */
-			kr = chudxnu_task_read(task, &currPC, caller, sizeof(uint64_t));    
-			
-			if(kr != KERN_SUCCESS) {
-				currPC = 0;
-				break;
-			}
-			
-			/* 
-			 * retrive contents of the frame pointer and advance to the next stack
-			 * frame if it's valid 
-			 */
-			prevFP = 0;
-			kr = chudxnu_task_read(task, &prevFP, currFP, sizeof(uint64_t));
-			
-			if(kr != KERN_SUCCESS) {
-				currPC = 0;
-				break;
-			}
+	/* 
+	 * Order of preference for top of stack:
+	 * 64 bit kernel state (not likely)
+	 * 32 bit kernel state
+	 * 64 bit user land state
+	 * 32 bit user land state
+	 */
+
+	if(regs64) {
+		currPC = regs64->isf.rip;
+	} else if(regs32) {
+		currPC = (uint64_t) regs32->eip;
+	} else if(u_regs64) {
+		currPC = u_regs64->isf.rip;
+	} else if(u_regs32) {
+		currPC = (uint64_t) u_regs32->eip;
+	}
 	
-			if(VALID_STACK_ADDRESS64(supervisor, prevFP, kernStackMin, kernStackMax)) {
-				buffer[bufferIndex++] = currPC;
-				prevPC = currPC;
-			}
-			if(prevFP < currFP) {
-				break;
-			} else {
-				currFP = prevFP;
-			}
-		}
+	if(!currPC) {
+		/* no top of the stack, bail out */
+		return KERN_FAILURE;
+	}
 
-		// append (rsp) on the bottom of the callstack
-		kr = chudxnu_task_read(task, &rsp, (addr64_t) regs->isf.rsp, sizeof(uint64_t));
-		if(kr == KERN_SUCCESS) {
-			buffer[bufferIndex++] = rsp;
-		}
-    } else {
-		/* !thread_is_64bit() */
-		/* we grab 32 bit frames and silently promote them to 64 bits */
-		uint32_t tmpWord = 0;
-		x86_saved_state32_t *regs = NULL;
-		
-		if(user_only) {
-			/* cant get user state for kernel threads */
-			if(task == kernel_task || supervisor) {
-				return 0x11;
-			}
-			regs = USER_REGS32(thread);
-		} else {
-			regs = saved_state32(current_cpu_datap()->cpu_int_state);
-		}
+	bufferIndex = 0;
 		
-		if(regs == NULL) {
-			*count = 0;
-			return 0x12;
+	if(bufferMaxIndex < 1) {
+		*count = 0;
+		return KERN_RESOURCE_SHORTAGE;
+	}
+
+	/* backtrace kernel */
+	if(regs64) {
+		uint64_t rsp = 0ULL;
+
+		// backtrace the 64bit side.
+		kr = do_backtrace64(task, thread, regs64, callstack, &bufferIndex, 
+			bufferMaxIndex, TRUE);
+
+		if(KERN_SUCCESS == chudxnu_kern_read(&rsp, (addr64_t) regs64->isf.rsp, sizeof(uint64_t)) && 
+			bufferIndex < bufferMaxIndex) {
+			callstack[bufferIndex++] = rsp;
 		}
 
-		currPC = (uint64_t) regs->eip;
-		currFP = (uint64_t) regs->ebp;
+	} else if(regs32) {
+		uint32_t esp = 0UL;
+
+		// backtrace the 32bit side.
+		kr = do_backtrace32(task, thread, regs32, callstack, &bufferIndex, 
+			bufferMaxIndex, TRUE);
 		
-		bufferIndex = 0;
-		//if(!supervisor)
-		//	bufferMaxIndex = bufferMaxIndex - 1;    //allot space for saving %rsp on the stack for user callstacks
-		if(bufferMaxIndex < 1) {
-			*count = 0;
-			return KERN_RESOURCE_SHORTAGE;
+		if(KERN_SUCCESS == chudxnu_kern_read(&esp, (addr64_t) regs32->uesp, sizeof(uint32_t)) && 
+			bufferIndex < bufferMaxIndex) {
+			callstack[bufferIndex++] = (uint64_t) esp;
 		}
-		buffer[bufferIndex++] = currPC; // save EIP on the top of the stack
+	} else if(u_regs64) {
+		/* backtrace user land */
+		uint64_t rsp = 0ULL;
+		
+		kr = do_backtrace64(task, thread, u_regs64, callstack, &bufferIndex, 
+			bufferMaxIndex, FALSE);
 
-		// now make a 64bit back trace from 32 bit stack frames
-		while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax))
-		{
-			cframe_t *fp = (cframe_t *) (uint32_t) currFP;
+		if(KERN_SUCCESS == chudxnu_task_read(task, &rsp, (addr64_t) u_regs64->isf.rsp, sizeof(uint64_t)) && 
+			bufferIndex < bufferMaxIndex) {
+			callstack[bufferIndex++] = rsp;
+		}
 
-			if(bufferIndex >= bufferMaxIndex) {
-				*count = bufferMaxIndex;
-				return KERN_RESOURCE_SHORTAGE;
-			}
+	} else if(u_regs32) {
+		uint32_t esp = 0UL;
+		
+		kr = do_backtrace32(task, thread, u_regs32, callstack, &bufferIndex, 
+			bufferMaxIndex, FALSE);
 
-			/* read the next frame */
-			if(supervisor) {
-				kr = chudxnu_kern_read(&tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));
-			} else {
-				kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t) &fp->caller, sizeof(uint32_t));    
-			}
-			
-			if(kr != KERN_SUCCESS) {
-				currPC = 0;
-				break;
-			}
-			
-			currPC = (uint64_t) tmpWord;	// promote 32 bit address
-			
-			/* 
-			 * retrive contents of the frame pointer and advance to the next stack
-			 * frame if it's valid 
-			 */
-			prevFP = 0;
-			if(supervisor) {
-				kr = chudxnu_kern_read(&tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
-			} else {
-				kr = chudxnu_task_read(task, &tmpWord, (vm_offset_t)&fp->prev, sizeof(uint32_t));
-			}
-			prevFP = (uint64_t) tmpWord;	// promote 32 bit address
-			
-			if(prevFP) {
-				buffer[bufferIndex++] = currPC;
-				prevPC = currPC;
-			}
-			if(prevFP < currFP) {
-				break;
-			} else {
-				currFP = prevFP;
-			}
+		if(KERN_SUCCESS == chudxnu_task_read(task, &esp, (addr64_t) u_regs32->uesp, sizeof(uint32_t)) && 
+			bufferIndex < bufferMaxIndex) {
+			callstack[bufferIndex++] = (uint64_t) esp;
 		}
+	}
 
-		// append (esp) on the bottom of the callstack
-		if(!supervisor) {
-			kr = chudxnu_task_read(task, &tmpWord, regs->uesp, sizeof(uint32_t));
-			if(kr == KERN_SUCCESS) {
-				rsp = (uint64_t) tmpWord;	// promote 32 bit address
-				buffer[bufferIndex++] = rsp;
-			}
-		}
-    }
-    
     *count = bufferIndex;
-    return ret;
+    return kr;
 }
 
+#pragma mark **** DEPRECATED ****
+
+// DEPRECATED
 __private_extern__ kern_return_t
 chudxnu_thread_get_callstack(
-	thread_t		thread, 
-	uint32_t		*callStack,
-	mach_msg_type_number_t	*count,
-	boolean_t		user_only)
+							 thread_t		thread, 
+							 uint32_t		*callStack,
+							 mach_msg_type_number_t	*count,
+							 boolean_t		user_only)
 {
 	kern_return_t   kr;
 	task_t          task = thread->task;
@@ -517,14 +557,14 @@ chudxnu_thread_get_callstack(
 	uint32_t        prevFP = 0;
 	uint32_t        prevPC = 0;
     uint32_t        esp = 0;
-	uint32_t        kernStackMin = min_valid_stack_address();
-	uint32_t        kernStackMax = max_valid_stack_address();
+	uint32_t        kernStackMin = thread->kernel_stack;
+	uint32_t        kernStackMax = kernStackMin + KERNEL_STACK_SIZE;
 	uint32_t       *buffer = callStack;
 	int             bufferIndex = 0;
 	int             bufferMaxIndex = *count;
 	boolean_t       supervisor;
 	x86_saved_state32_t *regs = NULL;
-
+	
     if (user_only) {
 		/* We can't get user state for kernel threads */
 		if (task == kernel_task) {
@@ -534,7 +574,7 @@ chudxnu_thread_get_callstack(
     } else {
     	regs = saved_state32(current_cpu_datap()->cpu_int_state);
     }
-
+	
     if (regs == NULL) {
         *count = 0;
 		return KERN_FAILURE;
@@ -553,45 +593,45 @@ chudxnu_thread_get_callstack(
 		return KERN_RESOURCE_SHORTAGE;
 	}
 	buffer[bufferIndex++] = currPC; //save PC in position 0.
-
+	
 	// Now, fill buffer with stack backtraces.
 	while (VALID_STACK_ADDRESS(supervisor, currFP, kernStackMin, kernStackMax)) {
 		cframe_t	*fp = (cframe_t *) currFP;
-
+		
 		if (bufferIndex >= bufferMaxIndex) {
 			*count = bufferMaxIndex;
 			return KERN_RESOURCE_SHORTAGE;
 		}
-
+		
 		if (supervisor) {
 			kr = chudxnu_kern_read(
-					&currPC,
-					(vm_offset_t) &fp->caller,
-					sizeof(currPC));
+								   &currPC,
+								   (vm_offset_t) &fp->caller,
+								   sizeof(currPC));
 		} else {
 			kr = chudxnu_task_read(
-					task,
-					&currPC,
-					(vm_offset_t) &fp->caller,
-					sizeof(currPC));
+								   task,
+								   &currPC,
+								   (vm_offset_t) &fp->caller,
+								   sizeof(currPC));
 		}
 		if (kr != KERN_SUCCESS)
 			break;
-
+		
 		//retrieve the contents of the frame pointer
 		// and advance to the prev stack frame if it's valid
 		prevFP = 0;
 		if (supervisor) {
 			kr = chudxnu_kern_read(
-					&prevFP,
-					(vm_offset_t) &fp->prev,
-					sizeof(prevFP));
+								   &prevFP,
+								   (vm_offset_t) &fp->prev,
+								   sizeof(prevFP));
 		} else {
 			kr = chudxnu_task_read(
-					task,
-					&prevFP,
-					(vm_offset_t) &fp->prev,
-					sizeof(prevFP));
+								   task,
+								   &prevFP,
+								   (vm_offset_t) &fp->prev,
+								   sizeof(prevFP));
 		}
 		if (prevFP) {
 			buffer[bufferIndex++] = currPC;
@@ -603,7 +643,7 @@ chudxnu_thread_get_callstack(
 			currFP = prevFP;
 		}
 	}
-
+	
 	// put the stack pointer on the bottom of the backtrace
     if(!supervisor) {
         kr = chudxnu_task_read(task, &esp, regs->uesp, sizeof(uint32_t));
@@ -611,41 +651,8 @@ chudxnu_thread_get_callstack(
             buffer[bufferIndex++] = esp;
         }
     }
-
+	
 	*count = bufferIndex;
 	return KERN_SUCCESS;
 }
 
-
-#pragma mark **** DEPRECATED ****
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_bind_current_thread(int cpu)
-{
-	return chudxnu_bind_thread(current_thread(), cpu);
-}
-
-// DEPRECATED
-kern_return_t chudxnu_unbind_current_thread(void)
-{
-	return chudxnu_unbind_thread(current_thread());
-}
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_current_thread_get_callstack(
-	uint32_t		*callStack,
-	mach_msg_type_number_t	*count,
-	boolean_t		user_only)
-{
-	return chudxnu_thread_get_callstack(
-			current_thread(), callStack, count, user_only);
-}
-
-// DEPRECATED
-__private_extern__
-thread_t chudxnu_current_act(void)
-{
-	return chudxnu_current_thread();
-}
diff --git a/osfmk/chud/i386/chud_xnu_glue.h b/osfmk/chud/i386/chud_xnu_glue.h
index ba3aa48a9..26f1a70ce 100644
--- a/osfmk/chud/i386/chud_xnu_glue.h
+++ b/osfmk/chud/i386/chud_xnu_glue.h
@@ -1,22 +1,28 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
diff --git a/osfmk/chud/i386/chud_xnu_private.h b/osfmk/chud/i386/chud_xnu_private.h
index 5fe82ee2a..4377af6ef 100644
--- a/osfmk/chud/i386/chud_xnu_private.h
+++ b/osfmk/chud/i386/chud_xnu_private.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_CHUD_XNU_PRIVATE_H_
@@ -34,18 +40,18 @@
  * to do it ourselves.
  */ 
 typedef struct {
-	struct queue_entry	req_entry;	/* Must be first */
-	uint32_t		req_type;
-	uint32_t		req_code;
-	volatile uint32_t	req_sync;
+	struct queue_entry req_entry;	/* Must be first */
+	uint32_t req_type;
+	uint32_t req_code;
+	volatile uint32_t req_sync;
 } chudcpu_signal_request_t;
 
 typedef struct {
 	void					*cpu_chud_fn_tablep;
-	timer_call_data_t			cpu_timer_call;
+	timer_call_data_t		cpu_timer_call;
 	uint64_t				t_deadline;
 	chudxnu_cpu_timer_callback_func_t	cpu_timer_callback_fn;
-	mpqueue_head_t				cpu_request_queue;
+	mpqueue_head_t			cpu_request_queue;
 } chudcpu_data_t;
 /* NB: cpu_chud_fn_tablep is expected to be the first member, at offset 0 */
 
diff --git a/osfmk/chud/ppc/chud_cpu_asm.h b/osfmk/chud/ppc/chud_cpu_asm.h
index a422a3a6c..a385f7664 100644
--- a/osfmk/chud/ppc/chud_cpu_asm.h
+++ b/osfmk/chud/ppc/chud_cpu_asm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _CHUD_CPU_ASM_H_
diff --git a/osfmk/chud/ppc/chud_cpu_asm.s b/osfmk/chud/ppc/chud_cpu_asm.s
index f9e6cc4f1..81482361a 100644
--- a/osfmk/chud/ppc/chud_cpu_asm.s
+++ b/osfmk/chud/ppc/chud_cpu_asm.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
-#define ASSEMBLER
 #include <chud/ppc/chud_spr.h>
 #include <ppc/asm.h>
 #include <mach/kern_return.h>
diff --git a/osfmk/chud/ppc/chud_cpu_ppc.c b/osfmk/chud/ppc/chud_cpu_ppc.c
index 6bd2df97b..b7e53cd3e 100644
--- a/osfmk/chud/ppc/chud_cpu_ppc.c
+++ b/osfmk/chud/ppc/chud_cpu_ppc.c
@@ -1,24 +1,31 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 #include <mach/mach_types.h>
 #include <mach/mach_host.h>
 
@@ -58,7 +65,7 @@ extern kern_return_t processor_exit(processor_t     processor); // osfmk/kern/pr
 __private_extern__
 kern_return_t chudxnu_enable_cpu(int cpu, boolean_t enable)
 {
-    chudxnu_unbind_thread(current_thread());
+    chudxnu_unbind_thread(current_thread(), 0);
 
     if(cpu<0 || cpu>=chudxnu_phys_cpu_count()) { // check sanity of cpu argument
         return KERN_FAILURE;
@@ -128,7 +135,7 @@ kern_return_t chudxnu_set_shadowed_spr(int cpu, int spr, uint32_t val)
 		cpu = chudxnu_cpu_number();
 		didBind = FALSE;
     } else {
-    chudxnu_bind_thread(current_thread(), cpu);
+    chudxnu_bind_thread(current_thread(), cpu, 0);
 		didBind = TRUE;
     }
 
@@ -246,7 +253,7 @@ kern_return_t chudxnu_set_shadowed_spr(int cpu, int spr, uint32_t val)
     }
 
     if(didBind) {
-    chudxnu_unbind_thread(current_thread());
+    chudxnu_unbind_thread(current_thread(), 0);
     }
     
     return retval;
@@ -268,7 +275,7 @@ kern_return_t chudxnu_set_shadowed_spr64(int cpu, int spr, uint64_t val)
 		cpu = chudxnu_cpu_number();
 		didBind = FALSE;
     } else {
-    chudxnu_bind_thread(current_thread(), cpu);
+    chudxnu_bind_thread(current_thread(), cpu, 0);
 		didBind = TRUE;
     }
 
@@ -327,7 +334,7 @@ kern_return_t chudxnu_set_shadowed_spr64(int cpu, int spr, uint64_t val)
     }
 
     if(didBind) {
-    chudxnu_unbind_thread(current_thread());
+    chudxnu_unbind_thread(current_thread(), 0);
     }
 
     return retval;
@@ -361,8 +368,8 @@ kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p)
     uint32_t val = 0xFFFFFFFF;
 
     /* bind to requested CPU */
-    if(cpu>=0) { // cpu<0 means don't bind
-		if(chudxnu_bind_thread(current_thread(), cpu)!=KERN_SUCCESS) {
+    if(cpu>=0 && !(ml_at_interrupt_context() && cpu_number() == cpu)) { // cpu<0 means don't bind
+		if(chudxnu_bind_thread(current_thread(), cpu, 0)!=KERN_SUCCESS) {
 			return KERN_INVALID_ARGUMENT;
 		}
     }
@@ -577,7 +584,7 @@ kern_return_t chudxnu_read_spr(int cpu, int spr, uint32_t *val_p)
     chudxnu_set_interrupts_enabled(oldlevel); /* enable interrupts */
 
     if(cpu>=0) { // cpu<0 means don't bind
-		chudxnu_unbind_thread(current_thread());
+		chudxnu_unbind_thread(current_thread(), 0);
     }
 
     *val_p = val;
@@ -592,8 +599,8 @@ kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p)
     boolean_t oldlevel;
 
     /* bind to requested CPU */
-    if(cpu>=0) { // cpu<0 means don't bind
-		if(chudxnu_bind_thread(current_thread(), cpu)!=KERN_SUCCESS) {
+    if(cpu>=0 && !(ml_at_interrupt_context() && cpu_number() == cpu)) { // cpu<0 means don't bind
+		if(chudxnu_bind_thread(current_thread(), cpu, 0)!=KERN_SUCCESS) {
 			return KERN_INVALID_ARGUMENT;
 		}
     }
@@ -667,7 +674,7 @@ kern_return_t chudxnu_read_spr64(int cpu, int spr, uint64_t *val_p)
     chudxnu_set_interrupts_enabled(oldlevel); /* enable interrupts */
 
     if(cpu>=0) { // cpu<0 means don't bind
-		chudxnu_unbind_thread(current_thread());
+		chudxnu_unbind_thread(current_thread(), 0);
     }
 
     return retval;
@@ -680,8 +687,8 @@ kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val)
     boolean_t oldlevel;
 
     /* bind to requested CPU */
-    if(cpu>=0) { // cpu<0 means don't bind
-		if(chudxnu_bind_thread(current_thread(), cpu)!=KERN_SUCCESS) {
+    if(cpu>=0 && !(ml_at_interrupt_context() && cpu_number() == cpu)) { // cpu<0 means don't bind
+		if(chudxnu_bind_thread(current_thread(), cpu, 0)!=KERN_SUCCESS) {
 			return KERN_INVALID_ARGUMENT;
 		}
     }
@@ -948,7 +955,7 @@ kern_return_t chudxnu_write_spr(int cpu, int spr, uint32_t val)
     chudxnu_set_interrupts_enabled(oldlevel); /* re-enable interrupts */
 	
     if(cpu>=0) { // cpu<0 means don't bind
-		chudxnu_unbind_thread(current_thread());
+		chudxnu_unbind_thread(current_thread(), 0);
     }
   
     return retval;
@@ -962,8 +969,8 @@ kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val)
     uint64_t *val_p = &val;
 
     /* bind to requested CPU */
-    if(cpu>=0) { // cpu<0 means don't bind
-		if(chudxnu_bind_thread(current_thread(), cpu)!=KERN_SUCCESS) {
+    if(cpu>=0 && !(ml_at_interrupt_context() && cpu_number() == cpu)) { // cpu<0 means don't bind
+		if(chudxnu_bind_thread(current_thread(), cpu, 0)!=KERN_SUCCESS) {
 			return KERN_INVALID_ARGUMENT;
 		}
     }
@@ -1064,31 +1071,12 @@ kern_return_t chudxnu_write_spr64(int cpu, int spr, uint64_t val)
     chudxnu_set_interrupts_enabled(oldlevel); /* re-enable interrupts */
 
     if(cpu>=0) { // cpu<0 means don't bind
-		chudxnu_unbind_thread(current_thread());
+		chudxnu_unbind_thread(current_thread(), 0);
     }
  
     return retval;
 }
 
-#pragma mark **** cache flush ****
-
-__private_extern__
-void chudxnu_flush_caches(void)
-{
-    cacheInit();
-}
-
-__private_extern__
-void chudxnu_enable_caches(boolean_t enable)
-{
-    if(!enable) {
-        cacheInit();
-        cacheDisable();
-    } else {
-        cacheInit();
-    }
-}
-
 #pragma mark **** perfmon facility ****
 
 __private_extern__
@@ -1106,7 +1094,7 @@ kern_return_t chudxnu_perfmon_release_facility(task_t task)
 #pragma mark **** rupt counters ****
 
 __private_extern__
-kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
+kern_return_t chudxnu_get_cpu_interrupt_counters(int cpu, rupt_counters_t *rupts)
 {
     if(cpu<0 || cpu>=chudxnu_phys_cpu_count()) { // check sanity of cpu argument
         return KERN_FAILURE;
@@ -1148,7 +1136,7 @@ kern_return_t chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts)
 }
 
 __private_extern__
-kern_return_t chudxnu_clear_cpu_rupt_counters(int cpu)
+kern_return_t chudxnu_clear_cpu_interrupt_counters(int cpu)
 {
     if(cpu<0 || cpu>=chudxnu_phys_cpu_count()) { // check sanity of cpu argument
         return KERN_FAILURE;
@@ -1183,3 +1171,36 @@ kern_return_t chudxnu_scom_write(uint32_t reg, uint64_t data)
 	ml_scom_write(reg, data);
 	return KERN_SUCCESS;
 }
+
+#pragma mark *** deprecated ***
+
+//DEPRECATED
+__private_extern__ kern_return_t
+chudxnu_get_cpu_rupt_counters(int cpu, rupt_counters_t *rupts) {
+	return chudxnu_get_cpu_interrupt_counters(cpu, rupts);
+}
+
+//DEPRECATED
+__private_extern__ kern_return_t
+chudxnu_clear_cpu_rupt_counters(int cpu) {
+	return chudxnu_clear_cpu_interrupt_counters(cpu);
+}
+
+//DEPRECATED
+__private_extern__
+void chudxnu_flush_caches(void)
+{
+	cacheInit();
+}
+
+//DEPRECATED
+__private_extern__
+void chudxnu_enable_caches(boolean_t enable)
+{
+	if(!enable) {
+		cacheInit();
+		cacheDisable();
+	} else {
+		cacheInit();
+	}
+}
diff --git a/osfmk/chud/ppc/chud_osfmk_callback_ppc.c b/osfmk/chud/ppc/chud_osfmk_callback_ppc.c
index 54177dc3e..44c893c47 100644
--- a/osfmk/chud/ppc/chud_osfmk_callback_ppc.c
+++ b/osfmk/chud/ppc/chud_osfmk_callback_ppc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <stdint.h>
@@ -49,8 +55,8 @@ void chudxnu_cancel_all_callbacks(void)
     chudxnu_perfmon_ast_callback_cancel();
     chudxnu_cpusig_callback_cancel();
     chudxnu_kdebug_callback_cancel();
-    chudxnu_thread_timer_callback_cancel();
 	chudxnu_syscall_callback_cancel();
+	chudxnu_dtrace_callback_cancel();
 }
 
 static chudcpu_data_t chudcpu_boot_cpu;
@@ -81,7 +87,9 @@ void chudxnu_per_proc_free(void *per_proc_chud)
 	}
 }
 
-static void chudxnu_private_cpu_timer_callback(timer_call_param_t param0, timer_call_param_t param1)
+static void
+chudxnu_private_cpu_timer_callback(__unused timer_call_param_t param0,
+				   __unused timer_call_param_t param1)
 {
     chudcpu_data_t	*chud_proc_info;
     boolean_t oldlevel;
@@ -192,7 +200,10 @@ static chudxnu_trap_callback_func_t trap_callback_fn = NULL;
                              (t==T_INSTRUMENTATION) ? 0x2000 : \
                              0x0)
 
-static kern_return_t chudxnu_private_trap_callback(int trapno, struct savearea *ssp, unsigned int dsisr, unsigned int dar)
+static kern_return_t
+chudxnu_private_trap_callback(int trapno, struct savearea *ssp,
+			      __unused unsigned int dsisr,
+			      __unused addr64_t dar)
 {
     boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
     kern_return_t retval = KERN_FAILURE;
@@ -236,7 +247,11 @@ kern_return_t chudxnu_trap_callback_cancel(void)
 #pragma mark **** ast ****
 static chudxnu_perfmon_ast_callback_func_t perfmon_ast_callback_fn = NULL;
 
-static kern_return_t chudxnu_private_chud_ast_callback(int trapno, struct savearea *ssp, unsigned int dsisr, unsigned int dar)
+static kern_return_t
+chudxnu_private_chud_ast_callback(__unused int trapno,
+				  __unused struct savearea *ssp,
+				  __unused unsigned int dsisr,
+				  __unused addr64_t dar)
 {
     boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
     ast_t *myast = ast_pending();
@@ -332,7 +347,10 @@ kern_return_t chudxnu_perfmon_ast_send(void)
 static chudxnu_interrupt_callback_func_t interrupt_callback_fn = NULL;
 //extern perfCallback perfIntHook; /* function hook into interrupt() */
 
-static kern_return_t chudxnu_private_interrupt_callback(int trapno, struct savearea *ssp, unsigned int dsisr, unsigned int dar)
+static kern_return_t
+chudxnu_private_interrupt_callback(int trapno, struct savearea *ssp,
+				   __unused unsigned int dsisr,
+				   __unused addr64_t dar)
 {
     chudxnu_interrupt_callback_func_t fn = interrupt_callback_fn;
     
@@ -370,7 +388,10 @@ kern_return_t chudxnu_interrupt_callback_cancel(void)
 static chudxnu_cpusig_callback_func_t cpusig_callback_fn = NULL;
 extern perfCallback perfCpuSigHook; /* function hook into cpu_signal_handler() */
 
-static kern_return_t chudxnu_private_cpu_signal_handler(int request, struct savearea *ssp, unsigned int arg0, unsigned int arg1)
+static kern_return_t
+chudxnu_private_cpu_signal_handler(int request, struct savearea *ssp,
+				   __unused unsigned int arg0,
+				   __unused addr64_t arg1)
 {
     chudxnu_cpusig_callback_func_t fn = cpusig_callback_fn;
     
diff --git a/osfmk/chud/ppc/chud_spr.h b/osfmk/chud/ppc/chud_spr.h
index f268c15e3..479f664be 100644
--- a/osfmk/chud/ppc/chud_spr.h
+++ b/osfmk/chud/ppc/chud_spr.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _CHUD_SPR_H_
diff --git a/osfmk/chud/ppc/chud_thread_ppc.c b/osfmk/chud/ppc/chud_thread_ppc.c
index 170a98bd9..36fdc1e19 100644
--- a/osfmk/chud/ppc/chud_thread_ppc.c
+++ b/osfmk/chud/ppc/chud_thread_ppc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -40,13 +46,6 @@
 #include <ppc/machine_routines.h>
 #include <ppc/fpu_protos.h>
 
-// forward declarations
-extern kern_return_t machine_thread_get_kern_state( thread_t                thread,
-													thread_flavor_t         flavor,
-													thread_state_t          tstate,
-													mach_msg_type_number_t  *count);
-
-
 #pragma mark **** thread state ****
 
 __private_extern__
@@ -334,6 +333,11 @@ __private_extern__
 kern_return_t chudxnu_task_read(task_t task, void *kernaddr, uint64_t usraddr, vm_size_t size)
 {
     kern_return_t ret = KERN_SUCCESS;
+
+	if(ml_at_interrupt_context()) {
+		// can't do this on an interrupt stack
+		return KERN_FAILURE;
+	}
     
 	if(!chudxnu_is_64bit_task(task)) { // clear any cruft out of upper 32-bits for 32-bit tasks
 		usraddr &= 0x00000000FFFFFFFFULL;
@@ -343,15 +347,13 @@ kern_return_t chudxnu_task_read(task_t task, void *kernaddr, uint64_t usraddr, v
 		thread_t      cur_thr = current_thread();
 		vm_offset_t   recover_handler = cur_thr->recover; 
 		
-		if(ml_at_interrupt_context()) {
-			return KERN_FAILURE; // can't do copyin on interrupt stack
-		}
-	
 		if(copyin(usraddr, kernaddr, size)) {
 			ret = KERN_FAILURE;
 		}
+
 		cur_thr->recover = recover_handler;
     } else {
+		
 		vm_map_t map = get_task_map(task);
 		ret = vm_map_read_user(map, usraddr, kernaddr, size);
     }
@@ -363,7 +365,12 @@ __private_extern__
 kern_return_t chudxnu_task_write(task_t task, uint64_t useraddr, void *kernaddr, vm_size_t size)
 {
     kern_return_t ret = KERN_SUCCESS;
-    
+ 
+	if(ml_at_interrupt_context()) {
+		// can't do this on an interrupt stack
+		return KERN_FAILURE;
+	}
+
 	if(!chudxnu_is_64bit_task(task)) { // clear any cruft out of upper 32-bits for 32-bit tasks
 		useraddr &= 0x00000000FFFFFFFFULL;
 	}
@@ -372,15 +379,12 @@ kern_return_t chudxnu_task_write(task_t task, uint64_t useraddr, void *kernaddr,
 		thread_t      cur_thr = current_thread();
 		vm_offset_t   recover_handler = cur_thr->recover; 
 					
-		if(ml_at_interrupt_context()) {
-			return KERN_FAILURE; // can't do copyout on interrupt stack
-		}
-	
 		if(copyout(kernaddr, useraddr, size)) {
 			ret = KERN_FAILURE;
 		}
 		cur_thr->recover = recover_handler;
     } else {
+		
 		vm_map_t map = get_task_map(task);
 		ret = vm_map_write_user(map, kernaddr, useraddr, size);
     }		
@@ -391,76 +395,15 @@ kern_return_t chudxnu_task_write(task_t task, uint64_t useraddr, void *kernaddr,
 __private_extern__
 kern_return_t chudxnu_kern_read(void *dstaddr, vm_offset_t srcaddr, vm_size_t size)
 {
-    while(size>0) {
-		ppnum_t pp;
-		addr64_t phys_addr;    
-		
-		pp = pmap_find_phys(kernel_pmap, srcaddr);			/* Get the page number */
-		if(!pp) {
-			return KERN_FAILURE;					/* Not mapped... */
-		}
-		
-		phys_addr = ((addr64_t)pp << 12) | (srcaddr & 0x0000000000000FFFULL);	/* Shove in the page offset */
-		if(phys_addr >= mem_actual) {
-			return KERN_FAILURE;					/* out of range */
-		}
-		
-		if((phys_addr&0x1) || size==1) {
-			*((uint8_t *)dstaddr) = ml_phys_read_byte_64(phys_addr);
-			((uint8_t *)dstaddr)++;
-			srcaddr += sizeof(uint8_t);
-			size -= sizeof(uint8_t);
-		} else if((phys_addr&0x3) || size<=2) {
-			*((uint16_t *)dstaddr) = ml_phys_read_half_64(phys_addr);
-			((uint16_t *)dstaddr)++;
-			srcaddr += sizeof(uint16_t);
-			size -= sizeof(uint16_t);
-		} else {
-			*((uint32_t *)dstaddr) = ml_phys_read_word_64(phys_addr);
-			((uint32_t *)dstaddr)++;
-			srcaddr += sizeof(uint32_t);
-			size -= sizeof(uint32_t);
-		}
-    }
-    return KERN_SUCCESS;
+	return (ml_nofault_copy(srcaddr, (vm_offset_t) dstaddr, size) == size ?
+	    KERN_SUCCESS: KERN_FAILURE);
 }
 
 __private_extern__
-kern_return_t chudxnu_kern_write(vm_offset_t dstaddr, void *srcaddr, vm_size_t size)
+kern_return_t chudxnu_kern_write(vm_offset_t dstaddr, void *srcaddr, vm_size_t	size)
 {
-    while(size>0) {
-		ppnum_t pp;
-		addr64_t phys_addr;    
-		
-		pp = pmap_find_phys(kernel_pmap, dstaddr);			/* Get the page number */
-		if(!pp) {
-			return KERN_FAILURE;					/* Not mapped... */
-		}
-		
-		phys_addr = ((addr64_t)pp << 12) | (dstaddr & 0x0000000000000FFFULL);	/* Shove in the page offset */
-		if(phys_addr >= mem_actual) {
-			return KERN_FAILURE;					/* out of range */
-		}
-		
-		if((phys_addr&0x1) || size==1) {
-			ml_phys_write_byte_64(phys_addr, *((uint8_t *)srcaddr));
-			((uint8_t *)srcaddr)++;
-			dstaddr += sizeof(uint8_t);
-			size -= sizeof(uint8_t);
-		} else if((phys_addr&0x3) || size<=2) {
-			ml_phys_write_half_64(phys_addr, *((uint16_t *)srcaddr));
-			((uint16_t *)srcaddr)++;
-			dstaddr += sizeof(uint16_t);
-			size -= sizeof(uint16_t);
-		} else {
-			ml_phys_write_word_64(phys_addr, *((uint32_t *)srcaddr));
-			((uint32_t *)srcaddr)++;
-			dstaddr += sizeof(uint32_t);
-			size -= sizeof(uint32_t);
-		}
-    }
-    
-    return KERN_SUCCESS;
+	return (ml_nofault_copy((vm_offset_t) srcaddr, dstaddr, size) == size ?
+	    KERN_SUCCESS: KERN_FAILURE);
 }
 
 // chudxnu_thread_get_callstack gathers a raw callstack along with any information needed to
@@ -507,8 +450,8 @@ kern_return_t chudxnu_thread_get_callstack64(	thread_t thread,
     uint64_t currPC, currLR, currR0;
     uint64_t framePointer;
     uint64_t prevPC = 0;
-    uint64_t kernStackMin = min_valid_stack_address();
-    uint64_t kernStackMax = max_valid_stack_address();
+    uint64_t kernStackMin = thread->kernel_stack;
+    uint64_t kernStackMax = kernStackMin + KERNEL_STACK_SIZE;
     uint64_t *buffer = callStack;
     uint32_t tmpWord;
     int bufferIndex = 0;
@@ -530,8 +473,7 @@ kern_return_t chudxnu_thread_get_callstack64(	thread_t thread,
 
     supervisor = SUPERVISOR_MODE(sv->save_srr1);
     if(supervisor) {
-#warning assuming kernel task is always 32-bit
-		is64Bit = FALSE;
+		is64Bit = FALSE; /* XXX assuming task is always 32-bit */
     } else {
 		is64Bit = chudxnu_is_64bit_task(task);
     }
@@ -638,6 +580,9 @@ kern_return_t chudxnu_thread_get_callstack64(	thread_t thread,
     return KERN_SUCCESS;
 }
 
+#pragma mark **** DEPRECATED ****
+
+// DEPRECATED
 __private_extern__
 kern_return_t chudxnu_thread_get_callstack( thread_t thread, 
 					    uint32_t *callStack,
@@ -650,8 +595,8 @@ kern_return_t chudxnu_thread_get_callstack( thread_t thread,
     uint64_t currPC, currLR, currR0;
     uint64_t framePointer;
     uint64_t prevPC = 0;
-    uint64_t kernStackMin = min_valid_stack_address();
-    uint64_t kernStackMax = max_valid_stack_address();
+    uint64_t kernStackMin = thread->kernel_stack;
+    uint64_t kernStackMax = kernStackMin + KERNEL_STACK_SIZE;
     uint32_t *buffer = callStack;
     uint32_t tmpWord;
     int bufferIndex = 0;
@@ -673,8 +618,7 @@ kern_return_t chudxnu_thread_get_callstack( thread_t thread,
 
     supervisor = SUPERVISOR_MODE(sv->save_srr1);
     if(supervisor) {
-#warning assuming kernel task is always 32-bit
-		is64Bit = FALSE;
+		is64Bit = FALSE; /* XXX assuming kernel task is always 32-bit */
     } else {
 		is64Bit = chudxnu_is_64bit_task(task);
     }
@@ -781,33 +725,3 @@ kern_return_t chudxnu_thread_get_callstack( thread_t thread,
     return KERN_SUCCESS;
 }
 
-#pragma mark **** DEPRECATED ****
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_bind_current_thread(int cpu)
-{
-	return chudxnu_bind_thread(current_thread(), cpu);
-}
-
-// DEPRECATED
-kern_return_t chudxnu_unbind_current_thread(void)
-{
-	return chudxnu_unbind_thread(current_thread());
-}
-
-// DEPRECATED
-__private_extern__
-kern_return_t chudxnu_current_thread_get_callstack(	uint32_t *callStack,
-													mach_msg_type_number_t *count,
-													boolean_t user_only)
-{
-	return chudxnu_thread_get_callstack(current_thread(), callStack, count, user_only);
-}
-
-// DEPRECATED
-__private_extern__
-thread_t chudxnu_current_act(void)
-{
-	return chudxnu_current_thread();
-}
diff --git a/osfmk/chud/ppc/chud_xnu_glue.h b/osfmk/chud/ppc/chud_xnu_glue.h
index ba3aa48a9..26f1a70ce 100644
--- a/osfmk/chud/ppc/chud_xnu_glue.h
+++ b/osfmk/chud/ppc/chud_xnu_glue.h
@@ -1,22 +1,28 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
diff --git a/osfmk/chud/ppc/chud_xnu_private.h b/osfmk/chud/ppc/chud_xnu_private.h
index d41d672c6..e3ffaf075 100644
--- a/osfmk/chud/ppc/chud_xnu_private.h
+++ b/osfmk/chud/ppc/chud_xnu_private.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _PPC_CHUD_XNU_PRIVATE_H_
diff --git a/osfmk/conf/MASTER b/osfmk/conf/MASTER
index feca58dec..5f7d87d6d 100644
--- a/osfmk/conf/MASTER
+++ b/osfmk/conf/MASTER
@@ -38,6 +38,16 @@
 #	configurations.
 #
 #######################################################################
+#  SYSTEM SIZE CONFIGURATION (select exactly one)
+#
+#	xlarge = extra large scale system configuration
+#	large  = large scale system configuration
+#	medium = medium scale system configuration
+#	small  = small scale system configuration
+#	xsmall = extra small scale system configuration
+#	bsmall = special extra small scale system configuration 
+#
+#######################################################################
 #
 # Basic compilation options.
 #
@@ -75,7 +85,7 @@ options		ADVISORY_PAGEOUT
 #	consistency of various algorithms in the kernel. The performance impact
 #	of this option is significant.
 #
-options		MACH_ASSERT	#		# <debug>
+options		MACH_ASSERT	#		# <mach_assert>
 #
 # MACH_DEBUG enables the mach_debug_server, a message interface used to
 #	retrieve or control various statistics. This interface may expose data
@@ -84,8 +94,8 @@ options		MACH_ASSERT	#		# <debug>
 #	Other options here enable information retrieval for specific subsystems
 #
 options		MACH_DEBUG	#		#
+options		MACH_IPC_DEBUG	#		#
 #
-options		MACH_IPC_DEBUG	#		# <debug>
 options		MACH_VM_DEBUG	#		# <debug>
 #
 # MACH_MP_DEBUG control the possible dead locks that may occur by controlling
@@ -98,6 +108,8 @@ options		MACH_MP_DEBUG	#		# <debug>
 #	operations on each element.
 #
 options		ZONE_DEBUG	#		# <debug>
+#
+options		ZONE_ALIAS_ADDR	#		# <zone_alias_addr>
 # 
 # XPR_DEBUG enables the gathering of data through the XPR macros inserted
 #	into various subsystems. This option is normally only enabled for
@@ -119,7 +131,12 @@ options		MACH_LDEBUG	#		# <debug>
 #
 options		KDEBUG		# kernel tracing	# <kdebug>
 
-#	
+#
+# CONFIG_DTRACE enables code needed to support DTrace. Currently this is
+# only used for delivery of traps/interrupts to DTrace.
+#
+options		CONFIG_DTRACE	#		# <config_dtrace>
+
 # MACH_COUNTERS enables code that handles various counters in the system.
 # 
 options		MACH_COUNTERS	#		# <debug>
@@ -134,11 +151,6 @@ options		UPL_DEBUG	# 		# <debug>
 #
 ##########################################################
 #
-# MACH_PROF enables the profiling server, a message interface used to
-#	retrieve profiling statistics.
-#
-#options	MACH_PROF		#		# <mach_prof>
-#
 # MACH_IPC_STATS controls the collection of statistics in the MACH IPC
 #	subsystem.
 #
@@ -167,3 +179,54 @@ options		UPL_DEBUG	# 		# <debug>
 #
 options		MACH_COUNTERS		#		# <stats>
 
+#
+#  configuration option for including cypto code
+#
+options		CRYPTO                  # <crypto>
+
+# HIBERNATION - include hibernation code
+#
+options		HIBERNATION		#		# <hibernation>
+
+#
+#  configurable kernel related resources (CONFIG_THREAD_MAX needs to stay in 
+#  sync with bsd/conf/MASTER until we fix the config system... todo XXX
+#
+options   CONFIG_THREAD_MAX=2560		# <medium,large,xlarge>
+options   CONFIG_THREAD_MAX=1536		# <small,xsmall>
+options   CONFIG_THREAD_MAX=1024		# <bsmall>
+
+options   CONFIG_TASK_MAX=1024			# <medium,large,xlarge>
+options   CONFIG_TASK_MAX=768			# <small,xsmall>
+options   CONFIG_TASK_MAX=512			# <bsmall>
+
+options   CONFIG_ZONE_MAP_MIN=12582912	# <medium,large,xlarge>
+options   CONFIG_ZONE_MAP_MIN=6291456	# <small,xsmall>
+options   CONFIG_ZONE_MAP_MIN=1048576	# <bsmall>
+
+options   CONFIG_TOKEN_QUEUE_SMALL=1		# <bsmall>
+options   CONFIG_TOKEN_QUEUE_SMALL=0		# <xsmall,small,medium,large,xlarge>
+
+#
+#  configurable kernel - use these options to strip strings from panic
+#  and printf calls.
+#  no_panic_str - saves around 50K of kernel footprint.
+#  no_printf_str - saves around 45K of kernel footprint.
+#
+options   CONFIG_NO_PANIC_STRINGS		# <no_panic_str>
+options   CONFIG_NO_PRINTF_STRINGS		# <no_printf_str>
+options   CONFIG_NO_KPRINTF_STRINGS		# <no_kprintf_str>
+
+# configurable kernel - general switch to say we are building for an
+# embedded device
+#
+options   CONFIG_EMBEDDED                       # <config_embedded>
+
+#  jettison_kernel_linker - jettison kernel linker after kernel init; don't wait for kextd to launch
+options   CONFIG_JETTISON_KERNEL_LINKER		# <jettison_kernel_linker>
+
+# vc_progress_white - make the progress gear white instead of black
+options	  CONFIG_VC_PROGRESS_WHITE		# <vc_progress_white>
+
+# secure_kernel - secure kernel from user programs
+options		SECURE_KERNEL		# <secure_kernel>
diff --git a/osfmk/conf/MASTER.i386 b/osfmk/conf/MASTER.i386
index 1e9f170b6..01731eae5 100644
--- a/osfmk/conf/MASTER.i386
+++ b/osfmk/conf/MASTER.i386
@@ -9,12 +9,15 @@
 #  Standard Apple MacOS X Configurations:
 #  -------- ---- -------- ---------------
 #
-#  osfmk = [intel pc mach small event vol pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug mk30 mk30_i386]
-#  RELEASE = [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386]
-#  DEBUG_KDP = [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 osf_debug debug]
-#  DEBUG= [intel pc iokit mach_pe mach mach_kdp small event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 osf_debug debug mach_kdb]
+#  RELEASE = [ medium intel pc iokit mach_pe mach mach_kdp event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 hibernation crypto config_dtrace]
+#  DEBUG_KDP = [ RELEASE osf_debug debug ]
+#  DEBUG= [ RELEASE osf_debug debug mach_kdb mach_assert ]
 #  PROFILE = [ RELEASE profile ]
 #
+#  EMBEDDED_BASE = [ bsmall intel pc iokit mach_pe mach mach_kdp event vol hd pst gdb fixpri simple_clock mkernserv uxpr kernstack ipc_compat ipc_debug fb mk30 mk30_i386 hibernation crypto ]
+#  EMBEDDED = [ EMBEDDED_BASE no_printf_str no_kprintf_str no_kdebug ]
+#  DEVELOPMENT = [ EMBEDDED_BASE mach_assert config_dtrace ]
+#
 ######################################################################
 #
 machine		"i386"						# <intel>
@@ -53,3 +56,7 @@ options		MACH_KDB	#				# <mach_kdb>
 options		MACH_KDP	# KDP				# <mach_kdp>
 options		PAE
 options		X86_64
+options		DISPATCH_COUNTS
+
+options		CONFIG_MACF		# Mandatory Access Control Framework
+#options	CONFIG_MACF_MACH	# MACF applied to Mach services
diff --git a/osfmk/conf/MASTER.ppc b/osfmk/conf/MASTER.ppc
index 850c57ddd..d655eea9e 100644
--- a/osfmk/conf/MASTER.ppc
+++ b/osfmk/conf/MASTER.ppc
@@ -9,9 +9,10 @@
 #  Standard Apple MacOS X Configurations:
 #  -------- ---- -------- ---------------
 #
-#  RELEASE       = [ mach_bsd mach_kdp iokit mach_pe ppc mach              ]
+#  RELEASE       = [ medium mach_bsd mach_kdp iokit mach_pe ppc mach hibernation crypto config_dtrace ]
+#  DEVELOPMENT	 = [ RELEASE ]
 #  RELEASE_TRACE = [ RELEASE kdebug ]
-#  DEBUG         = [ RELEASE mach_kdb debug ]
+#  DEBUG         = [ RELEASE mach_kdb debug mach_assert ]
 #  DEBUG_TRACE   = [ DEBUG kdebug ]
 #  PROFILE       = [ RELEASE profile ]
 #
@@ -54,3 +55,8 @@ options		MACH_PE		#		# <mach_pe>
 # XXX for bringup, turns on mac disklabels, 
 # and some other nice stuff for the diskshim
 options		POWERMAC
+
+options		DISPATCH_COUNTS
+
+options		CONFIG_MACF		# Mandatory Access Control Framework
+#options	CONFIG_MACF_MACH	# MACF applied to Mach services
diff --git a/osfmk/conf/Makefile b/osfmk/conf/Makefile
index 212081efc..cc38548ba 100644
--- a/osfmk/conf/Makefile
+++ b/osfmk/conf/Makefile
@@ -3,241 +3,6 @@ export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
 export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
 export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 
-#
-# VM should be warning free
-#
-export device_vm.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export device_vm.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export memory_object.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export memory_object.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export task_working_set.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export task_working_set.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_debug.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_debug.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_external.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_external.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_fault.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_fault.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_init.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_init.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_kern.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_kern.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_map.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_map.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_object.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_object.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_pageout.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_pageout.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_resident.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_resident.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_shared_memory_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_shared_memory_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export bsd_vm.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export bsd_vm.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export default_pager.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export default_pager.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export dp_backing_store.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export dp_backing_store.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export dp_memory_object.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export dp_memory_object.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export default_pager_alerts_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export default_pager_alerts_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export default_pager_alerts_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export default_pager_alerts_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export memory_object_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export memory_object_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export memory_object_control_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export memory_object_control_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export memory_object_default_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export memory_object_default_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export memory_object_name_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export memory_object_name_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export upl_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export upl_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export vm_map_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export vm_map_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-
-#
-# ipc should be warning free
-#
-export ipc_entry.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_entry.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_hash.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_hash.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_init.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_init.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_kmsg.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_kmsg.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_mqueue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_mqueue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_notify.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_notify.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_object.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_object.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_port.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_port.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_pset.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_pset.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_right.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_right.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_space.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_space.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_splay.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_splay.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_table.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_table.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_debug.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_debug.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export mach_msg.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export mach_msg.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_port.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_port.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mig_log.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mig_log.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_clock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_clock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_host.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_host.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export ipc_kobject.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export ipc_kobject.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_mig.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_mig.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ipc_sync.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ipc_sync.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export ipc_tt.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export ipc_tt.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export sync_lock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export sync_lock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export sync_sema.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export sync_sema.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export mach_port_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export mach_port_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export lock_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export lock_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export semaphore_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export semaphore_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-
-#
-# kern should be warning free (almost)
-#
-# export debug.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export debug.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-# export printf.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export printf.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-# export xpr.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export xpr.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-# export mk_sp.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export mk_sp.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-# export syscall_emulation.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export syscall_emulation.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-# export bsd_kern.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-# export bsd_kern.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#
-export ast.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ast.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export clock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export clock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export counters.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export counters.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export exception.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export exception.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export host.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export host.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export host_notify.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export host_notify.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export kalloc.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export kalloc.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export ledger.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export ledger.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export locks.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export locks.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_clock.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_clock.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_factor.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_factor.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export machine.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export machine.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mk_timer.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mk_timer.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export profile.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export profile.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export priority.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export priority.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export processor.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export processor.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export processor_data.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export processor_data.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export queue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export queue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export sched_prim.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export sched_prim.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export sscanf.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export sscanf.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export stack.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export stack.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export startup.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export startup.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export syscall_subr.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export syscall_subr.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export syscall_sw.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export syscall_sw.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export task.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export task.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export task_policy.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export task_policy.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export task_swap.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export task_swap.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export thread.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export thread.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread_act.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread_act.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export thread_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export thread_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread_policy.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread_policy.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export thread_swap.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export thread_swap.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export timer.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export timer.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export timer_call.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export timer_call.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export wait_queue.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export wait_queue.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export zalloc.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export zalloc.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export clock_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export clock_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export clock_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export clock_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export clock_reply_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export clock_reply_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export exc_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export exc_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export exc_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export exc_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export host_priv_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export host_priv_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export host_security_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export host_security_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export ledger_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export ledger_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export mach_host_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export mach_host_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export mach_notify_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export mach_notify_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export processor_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export processor_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export processor_set_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export processor_set_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-export prof_user.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-export prof_user.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export task_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export task_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-#export thread_act_server.o_CFLAGS_RM=$(CWARNFLAGS_STD)
-#export thread_act_server.o_CFLAGS_ADD=-Werror $(CWARNFLAGS_STD)
-
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
@@ -252,10 +17,14 @@ ifndef OSFMK_KERNEL_CONFIG
 export OSFMK_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/Makefile:  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -263,41 +32,37 @@ $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/Makefile:  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(OSFMK_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(OSFMK_KERNEL_CONFIG) $(OSFMK_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(OSFMK_KERNEL_CONFIG) $(OSFMK_KERNEL_CONFIG); \
 	);
 
 $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/platforms.h: $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/Makefile
-	(cd $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG);			\
+	$(_v)(cd $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG);			\
 	${RM} $@;						\
 	${LN} cputypes.h $@;					\
 	)
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/Makefile \
 		$(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)/platforms.h 
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(OSFMK_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(OSFMK_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(OSFMK_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(OSFMK_KERNEL_CONFIG)	\
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(OSFMK_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all: do_all 
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/osfmk/conf/Makefile.i386 b/osfmk/conf/Makefile.i386
index f6c752838..c3c05a7d7 100644
--- a/osfmk/conf/Makefile.i386
+++ b/osfmk/conf/Makefile.i386
@@ -11,113 +11,29 @@ CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
 
 # Objects that don't compile cleanly:
 OBJS_NO_WERROR=				\
-	pms.o				\
-	etimer.o			\
-	ioconf.o			\
 	UNDRequest.o			\
-	KUNCUserNotifications.o		\
-	panic_dialog.o			\
-	panic_image.o			\
-	rendered_numbers.o		\
-	video_console.o			\
-	iokit_rpc.o			\
-	subrs.o				\
-	kdp.o				\
-	kdp_udp.o			\
-	bsd_kern.o			\
-	debug.o				\
-	kmod.o				\
-	mk_sp.o				\
-	printf.o			\
-	syscall_emulation.o		\
-	UNDReplyServer.o	\
-	ipc_kobject.o		\
-	ipc_tt.o			\
-	machine.o			\
-	processor.o			\
-	stack.o				\
-	sched_prim.o		\
-	sync_lock.o			\
-	task.o				\
-	syscall_subr.o		\
-	thread.o			\
-	thread_call.o		\
-	timer_call.o		\
-	hibernate.o			\
-	clock_server.o		\
-	clock_priv_server.o	\
-	exc_user.o			\
-	wait_queue.o		\
-	exc_server.o		\
-	host_priv_server.o	\
-	host_priv_server.o	\
-	ledger_server.o		\
-	host_security_server.o \
-	lock_set_server.o	\
-	mach_host_server.o	\
-	mach_port_server.o	\
-	processor_server.o	\
-	processor_set_server.o \
-	semaphore_server.o	\
-	task_server.o		\
-	thread_act_server.o	\
-	device_server.o		\
-	pmap.o				\
-	cpuid.o				\
-	bsd_i386.o			\
-	loose_ends.o		\
-	mp_desc.o			\
-	perfmon.o			\
-	pcb.o				\
-	rtclock.o			\
-	trap.o				\
-	bbclock.o			\
-	mp.o				\
-	acpi.o				\
-	serial_console.o	\
-	text_console.o		\
-	mtrr.o				\
-	hibernate_i386.o	\
-	mach_msg.o	\
-	mach_header.o \
-	chud_osfmk_callback_i386.o \
-	chud_thread.o 		   \
-	chud_thread_i386.o 	   \
-	chud_cpu.o 		   \
-	db_access.o 		   \
-	db_aout.o		   \
-	db_break.o 		   \
-	db_command.o 		   \
-	db_cond.o 		   \
-	db_disasm.o		   \
-	db_examine.o 		   \
-	db_expr.o 		   \
-	db_ext_symtab.o 	   \
-	db_input.o 		   \
-	db_interface.o		   \
-	db_lex.o 		   \
-	db_macro.o 		   \
-	db_output.o 		   \
-	db_print.o 		   \
-	db_run.o 		   \
-	db_sym.o 		   \
-	db_task_thread.o 	   \
-	db_trace.o		   \
-	db_trap.o 		   \
-	db_variables.o 		   \
-	db_watch.o 		   \
-	db_write_cmd.o		   \
-	xpr.o
+	db_examine.o			\
+	db_macro.o			\
+	db_print.o			\
+	db_sym.o			\
+	db_variables.o			\
+	db_disasm.o			\
+	db_interface.o			\
+	db_trace.o			\
+	loose_ends.o			\
+	gssd_mach.o			\
+	mp.o
 
 OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
 
 $(OBJS_WERROR):		WERROR=-Werror
 
 # Files that must go in the __HIB segment:
-HIB_FILES=                              \
+UNCONFIGURED_HIB_FILES=					\
            hibernate_restore.o          \
            gdt.o                        \
            idt.o
+HIB_FILES=$(filter $(UNCONFIGURED_HIB_FILES),$(OBJS))
 
 ######################################################################
 #END	Machine dependent Makefile fragment for i386
diff --git a/osfmk/conf/Makefile.ppc b/osfmk/conf/Makefile.ppc
index 80f3bd542..d63b86c89 100644
--- a/osfmk/conf/Makefile.ppc
+++ b/osfmk/conf/Makefile.ppc
@@ -2,6 +2,32 @@
 #BEGIN	Machine dependent Makefile fragment for ppc
 ######################################################################
 
+#
+# ppc should be (mostly) warning free
+#
+CFLAGS+= $(WERROR)
+CWARNFLAGS= $(filter-out -Wbad-function-cast, $(CWARNFLAGS_STD))
+
+# Objects that don't compile cleanly:
+OBJS_NO_WERROR=                         \
+	UNDRequest.o			\
+	machine_routines.o		\
+	db_examine.o			\
+	db_macro.o			\
+	db_print.o			\
+	db_sym.o			\
+	db_variables.o			\
+	ppc_disasm.o			\
+	db_disasm.o			\
+	db_trace.o			\
+	db_low_trace.o			\
+	gssd_mach.o			\
+	kdp_machdep.o
+
+OBJS_WERROR=$(filter-out $(OBJS_NO_WERROR),$(OBJS))
+
+$(OBJS_WERROR):         WERROR=-Werror
+
 #
 # KDB support
 #
@@ -18,8 +44,9 @@ ppc_disasm.c ppc_disasm.h : $(SRCROOT)/osfmk/ppc/ppc_disasm.i makedis
 db_disasm.o : ppc_disasm.h
 
 # Files that must go in the __HIB segment:
-HIB_FILES=                              \
+UNCONFIGURED_HIB_FILES=                              \
            hibernate_restore.o
+HIB_FILES=$(filter $(UNCONFIGURED_HIB_FILES),$(OBJS))
 
 ######################################################################
 #END	Machine dependent Makefile fragment for ppc
diff --git a/osfmk/conf/Makefile.template b/osfmk/conf/Makefile.template
index aab83c2ec..d3596adef 100644
--- a/osfmk/conf/Makefile.template
+++ b/osfmk/conf/Makefile.template
@@ -26,7 +26,7 @@ include $(MakeInc_def)
 #
 # XXX: CFLAGS
 #
-CFLAGS+= -DMACH_KERNEL_PRIVATE
+CFLAGS+= -imacros meta_features.h -DMACH_KERNEL_PRIVATE $(CFLAGS_INLINE_CONFIG)
 
 #
 # Directories for mig generated files
@@ -36,7 +36,9 @@ COMP_SUBDIRS = \
 	device \
 	mach_debug \
 	mach \
-	UserNotification
+	UserNotification \
+	gssd \
+	lockd
 
 COMP_SUBDIRS_I386 = \
 	mach
@@ -81,19 +83,18 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS) assym.s
-	@echo "[ creating $(COMPONENT).o ]"
-	for hib_file in ${HIB_FILES};		\
+	$(_v)for hib_file in ${HIB_FILES};		\
 	do	\
                 $(SEG_HACK) __HIB $${hib_file} -o $${hib_file}__; \
                 mv $${hib_file}__ $${hib_file} ; \
 	done;
-	@echo [ updating $(COMPONENT).o ${OSFMK_KERNEL_CONFIG} ]
-	$(LD)  $(LDFLAGS_COMPONENT)  -o $(COMPONENT).o ${LDOBJS}
+	@echo LD $(COMPONENT)
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT)  -o $(COMPONENT).o ${LDOBJS}
 
 do_all: $(COMPONENT).o
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`;
+	$(_v)${MD} -u Makedep -f -d `ls *.d`;
 	
 do_build_all: do_depend
 
@@ -102,10 +103,10 @@ do_build_all: do_depend
 # dependency generation
 
 genassym.o: $(SOURCE_DIR)/$(COMPONENT)/$(ARCH_CONFIG_LC)/genassym.c
-	${KCC} ${CFLAGS} -MD ${_HOST_EXTRA_CFLAGS} -S -o ${@} -c ${INCFLAGS} $<
+	$(_v)${KCC} ${CFLAGS} -MD ${_HOST_EXTRA_CFLAGS} -S -o ${@} -c ${INCFLAGS} $<
 
 assym.s: genassym.o
-	sed -e '/#DEFINITION#/!d' -e 's/^.*#DEFINITION#//' -e 's/\$$//' genassym.o > ${@}
+	$(_v)sed -e '/#DEFINITION#/!d' -e 's/^.*#DEFINITION#//' -e 's/\$$//' -e 'p' -e 's/#//2' -e 's/[^A-Za-z0-9_]*\([A-Za-z0-9_]*\)/ \1_NUM/2' genassym.o > ${@}
 
 ${SOBJS}: assym.s
 
@@ -114,4 +115,3 @@ ${SOBJS}: assym.s
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/osfmk/conf/files b/osfmk/conf/files
index e9738b439..9bfe3b7dd 100644
--- a/osfmk/conf/files
+++ b/osfmk/conf/files
@@ -27,7 +27,9 @@
 # the rights to redistribute these changes.
 #
 
-OPTIONS/dli			optional dli
+OPTIONS/hibernation		optional hibernation
+OPTIONS/crypto			optional crypto
+OPTIONS/dli				optional dli
 OPTIONS/kdebug			optional kdebug
 OPTIONS/mach_assert		optional mach_assert
 OPTIONS/mach_debug		optional mach_debug
@@ -40,7 +42,6 @@ OPTIONS/norma_ether		optional norma_ether
 OPTIONS/norma_scsi		optional norma_scsi
 OPTIONS/mach_cluster_stats	optional mach_cluster_stats
 OPTIONS/mach_counters		optional mach_counters
-OPTIONS/mach_host		optional mach_host
 OPTIONS/mach_ipc_debug		optional mach_ipc_debug
 OPTIONS/mach_ipc_test		optional mach_ipc_test
 OPTIONS/mach_kdb		optional mach_kdb
@@ -50,7 +51,6 @@ OPTIONS/mach_kprof		optional mach_kprof
 OPTIONS/mach_ldebug		optional mach_ldebug
 OPTIONS/mach_mp_debug		optional mach_mp_debug
 OPTIONS/mach_pagemap		optional mach_pagemap
-OPTIONS/mach_prof		optional mach_prof
 OPTIONS/mach_rt			optional mach_rt
 OPTIONS/advisory_pageout	optional advisory_pageout
 #
@@ -68,10 +68,13 @@ OPTIONS/bootstrap_symbols	optional bootstrap_symbols
 OPTIONS/fast_tas		optional fast_tas
 OPTIONS/power_save		optional power_save
 OPTIONS/zone_debug		optional zone_debug
+OPTIONS/zone_alias_addr		optional zone_alias_addr
 OPTIONS/vm_cpm			optional vm_cpm
 OPTIONS/task_swapper		optional task_swapper
 OPTIONS/stack_usage		optional stack_usage
 
+OPTIONS/config_dtrace		optional config_dtrace
+
 # Default pager and system pager files, to be moved to separate component
  
 osfmk/default_pager/default_pager.c	standard
@@ -80,6 +83,16 @@ osfmk/default_pager/dp_memory_object.c	standard
 ./default_pager/default_pager_alerts_user.c	standard
 ./default_pager/default_pager_object_server.c	standard
 
+#
+# gssd files
+#
+./gssd/gssd_mach.c				standard
+
+#
+# lockd files
+#
+./lockd/lockd_mach.c				standard
+
 #
 # UserNotification files
 #
@@ -123,16 +136,17 @@ osfmk/ipc/ipc_right.c			standard
 osfmk/ipc/ipc_space.c			standard
 osfmk/ipc/ipc_splay.c			standard
 osfmk/ipc/ipc_table.c			standard
+osfmk/ipc/ipc_labelh.c			standard
 osfmk/ipc/mach_debug.c		standard
 osfmk/ipc/mach_msg.c			standard
 osfmk/ipc/mach_port.c			standard
 osfmk/ipc/mig_log.c			optional mig_debug
+osfmk/kern/affinity.c			standard
 osfmk/kern/ast.c			standard
 osfmk/kern/clock.c			standard
 osfmk/kern/clock_oldops.c		standard
 osfmk/kern/counters.c			standard
 osfmk/kern/debug.c			standard
-osfmk/kern/etimer.c			standard
 osfmk/kern/exception.c		standard
 osfmk/kern/host.c			standard
 osfmk/kern/host_notify.c		standard
@@ -149,9 +163,7 @@ osfmk/kern/mach_clock.c		standard
 osfmk/kern/machine.c			standard
 osfmk/kern/mk_sp.c			standard
 osfmk/kern/mk_timer.c		standard
-osfmk/kern/pms.c			standard
 osfmk/kern/page_decrypt.c	standard
-osfmk/kern/profile.c			standard
 osfmk/kern/printf.c			standard
 osfmk/kern/priority.c			standard
 osfmk/kern/processor.c		standard
@@ -159,6 +171,7 @@ osfmk/kern/processor_data.c		standard
 osfmk/kern/queue.c			standard
 osfmk/kern/sched_average.c		standard
 osfmk/kern/sched_prim.c		standard
+osfmk/kern/security.c		optional config_macf
 osfmk/kern/stack.c			standard
 osfmk/kern/startup.c			standard
 osfmk/kern/sync_lock.c		standard
@@ -179,7 +192,8 @@ osfmk/kern/wait_queue.c		standard
 osfmk/kern/xpr.c			optional xpr_debug
 osfmk/kern/zalloc.c			standard
 osfmk/kern/bsd_kern.c		optional mach_bsd
-osfmk/kern/hibernate.c		standard
+osfmk/kern/hibernate.c		optional hibernation
+osfmk/kern/symbols.c			standard
 ./mach/clock_server.c			standard
 ./mach/clock_priv_server.c		standard
 ./mach/clock_reply_user.c		standard
@@ -189,6 +203,8 @@ osfmk/kern/hibernate.c		standard
 ./mach/host_security_server.c		standard
 ./mach/ledger_server.c			standard
 ./mach/lock_set_server.c		standard
+./mach/mach_exc_user.c			standard
+./mach/mach_exc_server.c		optional mach_bsd
 ./mach/mach_host_server.c		standard
 ./mach/mach_notify_user.c		standard
 ./mach/mach_port_server.c		standard
@@ -199,6 +215,7 @@ osfmk/kern/hibernate.c		standard
 ./mach/memory_object_name_server.c	standard
 ./mach/upl_server.c			standard
 ./mach/audit_triggers_user.c		standard
+./mach/task_access_user.c          	standard
 #
 # For now, no external pagers
 #
@@ -207,27 +224,29 @@ osfmk/kern/hibernate.c		standard
 #
 ./mach/processor_server.c		standard
 ./mach/processor_set_server.c		standard
-./mach/prof_user.c			optional mach_prof
 ./mach/semaphore_server.c		standard
 ./mach/task_server.c			standard
 ./mach/thread_act_server.c		standard
 ./mach/vm_map_server.c			standard
+./mach/security_server.c		optional config_macf
+
 osfmk/mach-o/mach_header.c		standard
+
 osfmk/vm/device_vm.c			standard
 osfmk/vm/memory_object.c		standard
-osfmk/vm/task_working_set.c		standard
 osfmk/vm/vm_debug.c			standard
-osfmk/vm/vm_external.c                optional mach_pagemap
+osfmk/vm/vm_external.c			optional mach_pagemap
 osfmk/vm/vm_fault.c			standard
 osfmk/vm/vm_init.c			standard
 osfmk/vm/vm_kern.c			standard
 osfmk/vm/vm_map.c			standard
 osfmk/vm/vm_object.c			standard
 osfmk/vm/vm_pageout.c			standard
-osfmk/vm/vm_resident.c		standard
-osfmk/vm/vm_shared_memory_server.c	standard
+osfmk/vm/vm_resident.c			standard
+osfmk/vm/vm_shared_region.c		standard
 osfmk/vm/vm_user.c			standard
 osfmk/vm/bsd_vm.c			optional mach_bsd
+osfmk/vm/vm_purgeable.c			standard
 
 #
 # IOKit files, for a while
diff --git a/osfmk/conf/files.i386 b/osfmk/conf/files.i386
index 2a3399191..421ab6358 100644
--- a/osfmk/conf/files.i386
+++ b/osfmk/conf/files.i386
@@ -1,7 +1,7 @@
 OPTIONS/show_space		optional show_space
-OPTIONS/gdb			optional gdb
+OPTIONS/gdb				optional gdb
 OPTIONS/iplmeas			optional iplmeas
-OPTIONS/fb			optional fb
+OPTIONS/fb				optional fb
 
 
 #machdep/i386/unix_signal.c	standard
@@ -18,9 +18,10 @@ OPTIONS/fddi		optional fddi
 
 osfmk/vm/vm_apple_protect.c	 standard
 
-osfmk/i386/hi_res_clock_map.c 	optional hi_res_clock
+#osfmk/i386/hi_res_clock_map.c 	optional hi_res_clock
 
 osfmk/i386/pmap.c		standard
+#osfmk/i386/read_fault.c	standard
 
 
 osfmk/ddb/db_aout.c		optional mach_kdb
@@ -35,9 +36,11 @@ osfmk/i386/bzero.s		standard
 osfmk/i386/cpu.c		standard
 osfmk/i386/cpuid.c		standard
 osfmk/i386/cpu_threads.c	standard
+osfmk/i386/cpu_topology.c	standard
 osfmk/i386/db_disasm.c	optional mach_kdb
 osfmk/i386/db_interface.c	optional mach_kdb
 osfmk/i386/db_trace.c		optional mach_kdb
+osfmk/i386/etimer.c		standard
 osfmk/i386/fpu.c		standard
 osfmk/i386/gcc.s		standard
 osfmk/i386/gdt.c		standard
@@ -45,9 +48,7 @@ osfmk/i386/i386_lock.s	standard
 osfmk/i386/i386_init.c		standard
 osfmk/i386/i386_vm_init.c	standard
 osfmk/i386/idt.s		standard
-osfmk/i386/io_emulate.c	standard
 osfmk/i386/io_map.c		standard
-osfmk/i386/iopb.c		standard
 osfmk/i386/ktss.c		standard
 osfmk/i386/ldt.c		standard
 osfmk/i386/loose_ends.c	standard
@@ -61,7 +62,7 @@ osfmk/i386/machine_routines_asm.s	standard
 osfmk/i386/machine_check.c	standard
 osfmk/i386/mcount.s		optional profile
 osfmk/i386/mp_desc.c		standard
-osfmk/i386/ntoh.s		standard
+#osfmk/i386/ntoh.s		standard
 osfmk/i386/perfmon.c		standard
 osfmk/i386/pcb.c		standard
 osfmk/i386/phys.c		standard
@@ -82,22 +83,20 @@ osfmk/i386/commpage/pthreads.s	standard
 osfmk/i386/commpage/cacheflush.s	standard
 osfmk/i386/commpage/commpage_gettimeofday.s	standard
 osfmk/i386/commpage/bcopy_scalar.s	standard
-osfmk/i386/commpage/bcopy_sse3.s	standard
-osfmk/i386/commpage/bcopy_sse4.s	standard
-osfmk/i386/commpage/bcopy_sse4_64.s	standard
+osfmk/i386/commpage/bcopy_sse2.s	standard
+osfmk/i386/commpage/bcopy_sse3x.s	standard
+osfmk/i386/commpage/bcopy_sse3x_64.s	standard
 osfmk/i386/commpage/bzero_scalar.s	standard
-osfmk/i386/commpage/bzero_sse3.s	standard
-osfmk/i386/commpage/bzero_sse3_64.s	standard
-osfmk/i386/commpage/memset_pattern_sse3.s	standard
-osfmk/i386/commpage/memset_pattern_sse3_64.s	standard
-osfmk/i386/commpage/longcopy_sse4.s	standard
-osfmk/i386/commpage/longcopy_sse4_64.s	standard
+osfmk/i386/commpage/bzero_sse2.s	standard
+osfmk/i386/commpage/bzero_sse2_64.s	standard
+osfmk/i386/commpage/memset_pattern_sse2.s	standard
+osfmk/i386/commpage/memset_pattern_sse2_64.s	standard
+osfmk/i386/commpage/longcopy_sse3x.s	standard
+osfmk/i386/commpage/longcopy_sse3x_64.s	standard
 osfmk/i386/commpage/commpage_sigs.c	standard
 
-osfmk/i386/AT386/autoconf.c	standard
 osfmk/i386/AT386/conf.c		standard
 osfmk/i386/AT386/model_dep.c	standard
-osfmk/i386/AT386/physmem.c	optional physmem device-driver
 
 osfmk/i386/mp.c			standard
 osfmk/i386/mp_slave_boot.s	standard
@@ -127,19 +126,22 @@ osfmk/kdp/ml/i386/kdp_machdep.c	optional	mach_kdp
 
 osfmk/kdp/ml/i386/kdp_vm.c	optional	mach_kdp
 
-osfmk/i386/hibernate_i386.c		standard
-osfmk/i386/hibernate_restore.s		standard
+osfmk/i386/hibernate_i386.c		optional hibernation
+osfmk/i386/hibernate_restore.s		optional hibernation
 
 osfmk/chud/i386/chud_osfmk_callback_i386.c	standard
 osfmk/chud/i386/chud_cpu_i386.c			standard
 osfmk/chud/i386/chud_thread_i386.c		standard
 
+osfmk/i386/vmx/vmx_cpu.c			standard
+osfmk/i386/vmx/vmx_shims.c			standard
+
 # DUMMIES TO FORCE GENERATION OF .h FILES
-osfmk/OPTIONS/ln		optional ln
-osfmk/OPTIONS/eisa		optional eisa
-osfmk/OPTIONS/himem		optional himem
-osfmk/OPTIONS/ec		optional ec
-osfmk/OPTIONS/hi_res_clock	optional hi_res_clock
+#osfmk/OPTIONS/ln		optional ln
+#osfmk/OPTIONS/eisa		optional eisa
+#osfmk/OPTIONS/himem		optional himem
+#osfmk/OPTIONS/ec		optional ec
+#osfmk/OPTIONS/hi_res_clock	optional hi_res_clock
 
 
 osfmk/i386/startup64.c			optional x86_64
diff --git a/osfmk/conf/files.ppc b/osfmk/conf/files.ppc
index ffc13d1d8..070e02f48 100644
--- a/osfmk/conf/files.ppc
+++ b/osfmk/conf/files.ppc
@@ -59,12 +59,14 @@ osfmk/ppc/Emulate.s			standard
 osfmk/ppc/Emulate64.s		standard
 osfmk/ppc/AltiAssist.s		standard
 osfmk/ppc/conf.c			standard
+osfmk/ppc/etimer.c		standard
 osfmk/ppc/rtclock.c			standard
 osfmk/ppc/Diagnostics.c		standard
 osfmk/ppc/PPCcalls.c		standard
 osfmk/ppc/vmachmon.c		standard
 osfmk/ppc/vmachmon_asm.s	standard
-osfmk/ppc/pmsCPU.c			standard
+osfmk/ppc/pms.c			standard
+osfmk/ppc/pmsCPU.c		standard
 
 osfmk/ppc/Firmware.s		standard
 osfmk/ppc/FirmwareC.c		standard
@@ -109,8 +111,8 @@ osfmk/console/panic_dialog.c	optional	vc device-driver
 osfmk/console/video_console.c	optional	vc device-driver
 osfmk/console/ppc/video_scroll.s	optional	vc device-driver
 
-osfmk/ppc/hibernate_ppc.c		standard
-osfmk/ppc/hibernate_restore.s		standard
+osfmk/ppc/hibernate_ppc.c			optional hibernation
+osfmk/ppc/hibernate_restore.s		optional hibernation
 
 # DUMMIES TO FORCE GENERATION OF .h FILES
 OPTIONS/bm			optional bm
diff --git a/osfmk/conf/tools/doconf/Makefile b/osfmk/conf/tools/doconf/Makefile
index 2bf0b7a10..aa55a9419 100644
--- a/osfmk/conf/tools/doconf/Makefile
+++ b/osfmk/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,25 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/osfmk/conf/tools/doconf/doconf.csh b/osfmk/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/osfmk/conf/tools/doconf/doconf.csh
+++ b/osfmk/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/osfmk/console/i386/kdasm.s b/osfmk/console/i386/kdasm.s
index 2b9fa2bcd..e52429b33 100644
--- a/osfmk/console/i386/kdasm.s
+++ b/osfmk/console/i386/kdasm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* CMU_ENDHIST */
 /* 
diff --git a/osfmk/console/i386/serial_console.c b/osfmk/console/i386/serial_console.c
index a8bc82d29..37f29e101 100644
--- a/osfmk/console/i386/serial_console.c
+++ b/osfmk/console/i386/serial_console.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <i386/mp.h>
@@ -25,6 +31,7 @@
 #include <i386/machine_cpu.h>
 #include <i386/machine_routines.h>
 #include <i386/misc_protos.h>
+#include <i386/serial_io.h>
 #include <vm/vm_kern.h>
 #include <console/video_console.h>
 #include <console/serial_protos.h>
@@ -56,11 +63,15 @@ extern void serial_putc(int);
 
 static void _serial_putc(int, int, int);
 
-int vcgetc(int, int, boolean_t, boolean_t);
-
-console_ops_t cons_ops[] = {
-    {_serial_putc, _serial_getc},
-    {vcputc, vcgetc}
+struct console_ops cons_ops[] = {
+	{
+		.putc = _serial_putc,
+		.getc = _serial_getc,
+	},
+	{
+		.putc = vcputc,
+		.getc = vcgetc,
+	},
 };
 
 uint32_t nconsops = (sizeof cons_ops / sizeof cons_ops[0]);
@@ -177,7 +188,7 @@ _cnputc(char c)
 	 * non-reentrant.
 	 */
 	mp_disable_preemption();
-	if (!hw_lock_to(&cnputc_lock, LockTimeOut*10)) {
+	if (!hw_lock_to(&cnputc_lock, LockTimeOutTSC)) {
 	/* If we timed out on the lock, and we're in the debugger,
 	 * break the lock.
 	 */
diff --git a/osfmk/console/i386/text_console.c b/osfmk/console/i386/text_console.c
index 95a83b8f3..3a1ff1f7b 100644
--- a/osfmk/console/i386/text_console.c
+++ b/osfmk/console/i386/text_console.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -72,14 +78,14 @@ typedef short  csrpos_t;    /* cursor position, ONE_SPACE bytes per char */
 /*
  * Globals.
  */
-static short     vga_idx_reg      = 0;   /* location of VGA index register */
-static short     vga_io_reg       = 0;   /* location of VGA data register */
-static short     vga_cols         = 80;  /* number of columns */
-static short     vga_rows         = 25;  /* number of rows */
-static char      vga_attr         = 0;   /* current character attribute */
-static char      vga_attr_rev     = 0;   /* current reverse attribute */
-static char      vga_cursor_start = 0;   /* cached cursor start scan line */
-static char *    vram_start       = 0;   /* VM start of VGA frame buffer */
+static short     vga_idx_reg;		/* location of VGA index register */
+static short     vga_io_reg;		/* location of VGA data register */
+static short     vga_cols = 80;		/* number of columns */
+static short     vga_rows = 25;		/* number of rows */
+static char      vga_attr;		/* current character attribute */
+static char      vga_attr_rev;		/* current reverse attribute */
+static char      vga_cursor_start;	/* cached cursor start scan line */
+static unsigned char *vram_start;	/* VM start of VGA frame buffer */
 
 /*
  * Functions in kdasm.s.
@@ -209,7 +215,7 @@ vga_init(int cols, int rows, unsigned char * addr)
  * Scroll the screen up 'n' character lines.
  */
 void
-tc_scroll_up( int lines, __unused int top, __unused int bottom )
+tc_scroll_up(int lines, __unused unsigned int top, __unused unsigned int bottom)
 {
     csrpos_t  to;
     csrpos_t  from;
@@ -233,7 +239,8 @@ tc_scroll_up( int lines, __unused int top, __unused int bottom )
  * Scrolls the screen down 'n' character lines.
  */
 void
-tc_scroll_down( int lines, __unused int top, __unused int bottom )
+tc_scroll_down(int lines, __unused unsigned int top,
+	       __unused unsigned int bottom)
 {
     csrpos_t  to;
     csrpos_t  from;
@@ -312,7 +319,7 @@ tc_update_color( int color, int fore )
  * Show the hardware cursor.
  */
 void
-tc_show_cursor( int x, int y )
+tc_show_cursor(unsigned int x, unsigned int y)
 {
     set_cursor_position( XY_TO_CSRPOS(x, y) );
     set_cursor_enable( TRUE );
@@ -324,7 +331,7 @@ tc_show_cursor( int x, int y )
  * Hide the hardware cursor.
  */
 void
-tc_hide_cursor( __unused int x, __unused int y )
+tc_hide_cursor(__unused unsigned int x, __unused unsigned int y)
 {
     set_cursor_enable( FALSE );
 }
@@ -336,8 +343,8 @@ tc_hide_cursor( __unused int x, __unused int y )
  * relative to the current cursor position.
  */
 void
-tc_clear_screen(int x, int y, __unused int top, __unused int bottom,
-		int operation)
+tc_clear_screen(unsigned int x, unsigned int y, __unused unsigned int top,
+		__unused unsigned int bottom, int operation)
 {
     csrpos_t start;
     int      count;
@@ -368,7 +375,7 @@ tc_clear_screen(int x, int y, __unused int top, __unused int bottom,
  * and attributes.
  */
 void
-tc_paint_char(int x, int y, unsigned char ch, int attrs,
+tc_paint_char(unsigned int x, unsigned int y, unsigned char ch, int attrs,
 	      __unused unsigned char ch_previous, __unused int attrs_previous)
 {
     char my_attr = vga_attr;
diff --git a/osfmk/console/i386/text_console.h b/osfmk/console/i386/text_console.h
index f0921a20a..f971aaf19 100644
--- a/osfmk/console/i386/text_console.h
+++ b/osfmk/console/i386/text_console.h
@@ -1,34 +1,42 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _TEXT_CONSOLE_H_
 #define _TEXT_CONSOLE_H_
 
-void tc_paint_char(int x, int y, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous);
-void tc_scroll_down(int lines, int top, int bottom);
-void tc_scroll_up(int lines, int top, int bottom);
-void tc_clear_screen(int x, int y, int top, int bottom, int operation);
-void tc_show_cursor(int x, int y);
-void tc_hide_cursor(int x, int y); 
+void tc_paint_char(unsigned int x, unsigned int y, unsigned char ch, int attrs,
+		unsigned char ch_previous, int attrs_previous);
+void tc_scroll_down(int lines, unsigned int top, unsigned int bottom);
+void tc_scroll_up(int lines, unsigned int top, unsigned int bottom);
+void tc_clear_screen(unsigned int x, unsigned int y, unsigned int top,
+		unsigned int bottom, int operation);
+void tc_show_cursor(unsigned int x, unsigned int y);
+void tc_hide_cursor(unsigned int x, unsigned int y); 
 void tc_enable(boolean_t enable);
 void tc_initialize(struct vc_info * vinfo_p);
 void tc_update_color(int color, int fore);
diff --git a/osfmk/console/i386/video_scroll.c b/osfmk/console/i386/video_scroll.c
index 41538c872..0f607a651 100644
--- a/osfmk/console/i386/video_scroll.c
+++ b/osfmk/console/i386/video_scroll.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <types.h>
diff --git a/osfmk/console/iso_font.c b/osfmk/console/iso_font.c
index f0d45891a..e82fb3ff9 100644
--- a/osfmk/console/iso_font.c
+++ b/osfmk/console/iso_font.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/console/panic_dialog.c b/osfmk/console/panic_dialog.c
index 6c56b8e9c..1f4a94e1f 100644
--- a/osfmk/console/panic_dialog.c
+++ b/osfmk/console/panic_dialog.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <vc.h>
@@ -28,7 +34,7 @@
 #include <mach/mach_time.h>
 #include <sys/errno.h>
 #include <string.h>
-
+#include <machine/machlimits.h>
 
 extern struct vc_info vinfo;
 extern boolean_t panicDialogDesired;
@@ -37,17 +43,18 @@ extern boolean_t panicDialogDesired;
 
 void panic_ui_initialize(const unsigned char * system_clut);
 int panic_dialog_set_image( const unsigned char * ptr, unsigned int size );
-void panic_dialog_get_image( unsigned char ** ptr, unsigned int * size );
+void panic_dialog_get_image(const unsigned char **ptr, unsigned int *size);
 void draw_panic_dialog( void );
 void panic_dialog_test( void );
 
 static int panic_dialog_verify( const struct panicimage * data, unsigned int size );
 static int pixels_needed_to_blit_digit( int digit );
 static void blit_digit( int digit );
-static char * strnstr(const char * s, const char * find, size_t slen);
+static const char * strnstr(const char * s, const char * find, size_t slen);
 static void dim_screen(void);
-static void panic_blit_rect(unsigned int x, unsigned int y, unsigned int width, unsigned int height,
-							int transparent, unsigned char * dataPtr );
+static void panic_blit_rect(unsigned int x, unsigned int y, unsigned int width,
+			    unsigned int height, int transparent,
+			    const unsigned char * dataPtr);
 
 static int panic_info_x;
 static int panic_info_y;
@@ -60,14 +67,16 @@ static const struct panicimage * panic_dialog = NULL;		/* the active panic dialo
 static const unsigned char * panic_dialog_data = NULL;		/* where the image data starts */
 static const unsigned char * panic_dialog_clut = NULL;		/* where the clut used for the image starts */
 
-static unsigned char * curr_image_ptr = NULL;				/* If NULL, the default panic dialog is active */
+static const unsigned char *curr_image_ptr; /* If NULL, the default panic
+					       dialog is active */
 static unsigned int curr_image_size = 0;
 
 #define FONT_WIDTH	8
 #define FONT_HEIGHT	16
 static unsigned short rendered_font[FONT_HEIGHT][FONT_WIDTH];
 
-static char versionbuf[20];		/* ####.###~###\0 */
+#define VERSIONBUF_LEN 20
+static char versionbuf[VERSIONBUF_LEN];		/* ####.###~###\0 */
 
 #define isdigit(d) ((d) >= '0' && (d) <= '9')
 
@@ -81,31 +90,29 @@ static char versionbuf[20];		/* ####.###~###\0 */
 
 extern unsigned char iso_font[];
 extern const char version[];
-extern unsigned int panic_caller;
 
 void
 panic_ui_initialize(const unsigned char * system_clut)
 {
-	char vstr[20];
-
+	char vstr[VERSIONBUF_LEN];
 
 	panic_dialog_set_image( NULL, 0 );
 
 	active_clut = system_clut;
 
-	strcpy(vstr, "custom");
+	strlcpy(vstr, "custom", VERSIONBUF_LEN);
 
 	/* Convert xnu-####.###.obj~### into ####.###~### */
 
 	if (version) {
-		char * versionpos = strnstr(version, "xnu-", 20);
+		const char *versionpos = strnstr(version, "xnu-", VERSIONBUF_LEN);
 
 		if (versionpos) {
 			int len, i;
 
 			vstr[0] = '\0';
 
-			for (i=0,len=4;len<20;len++) {
+			for (i = 0, len = 4; len < VERSIONBUF_LEN; len++) {
 				if (isdigit(versionpos[len]) || versionpos[len] == '.') {	/* extract ####.###. */
 					vstr[i++] = versionpos[len];
 					continue;
@@ -116,7 +123,7 @@ panic_ui_initialize(const unsigned char * system_clut)
 			if ( versionpos[len-1] == '.' )     		/* remove trailing period if present */
 				i--;
 
-			for (;len<20;len++) {						/* skip to next digit if present */
+			for (; len < VERSIONBUF_LEN; len++) {						/* skip to next digit if present */
 				if ( !isdigit(versionpos[len]) )
 					continue;
 				break;
@@ -124,7 +131,7 @@ panic_ui_initialize(const unsigned char * system_clut)
 
 			if ( versionpos[len-1] == '~' ) {				/* extract ~### if present */
 				vstr[i++] = versionpos[len-1];
-				for (;len<20;len++) {						/* extract ### */
+				for (; len < VERSIONBUF_LEN; len++) {						/* extract ### */
 					if ( isdigit(versionpos[len]) ) {
 						vstr[i++] = versionpos[len];
 						continue;
@@ -137,7 +144,7 @@ panic_ui_initialize(const unsigned char * system_clut)
 		}
 	}
 
-	strcpy(versionbuf, vstr);
+	strlcpy(versionbuf, vstr, VERSIONBUF_LEN);
 }
 
 
@@ -148,12 +155,12 @@ panic_dialog_test( void )
 	boolean_t o_panicDialogDrawn = panicDialogDrawn;
 	boolean_t o_panicDialogDesired = panicDialogDesired;
 	unsigned int o_logPanicDataToScreen = logPanicDataToScreen;
-	unsigned int o_panic_caller = panic_caller;
+	unsigned long o_panic_caller = panic_caller;
 	unsigned int o_panicDebugging = panicDebugging;
 
 
 	panicDebugging = TRUE;
-	panic_caller = (unsigned int) __builtin_return_address(0);
+	panic_caller = (unsigned long)(char *)__builtin_return_address(0);
 	logPanicDataToScreen = FALSE;
 	panicDialogDesired = TRUE;
 	panicDialogDrawn = FALSE;
@@ -194,8 +201,9 @@ draw_panic_dialog( void )
 			pd_y = (vinfo.v_height/2) - panic_dialog->pd_height/2;
 		
 			/*  draw panic dialog at pd_x/pd_y */
-			panic_blit_rect( pd_x, pd_y, panic_dialog->pd_width, panic_dialog->pd_height, 
-										 0, (unsigned char*) panic_dialog_data);
+			panic_blit_rect(pd_x, pd_y, panic_dialog->pd_width,
+					panic_dialog->pd_height, 0,
+					panic_dialog_data);
 		
 			panic_dialog_count = 0;		/* number of info items to display at the bottom of dialog */
 
@@ -397,7 +405,7 @@ panic_dialog_set_image( const unsigned char * ptr, unsigned int size )
 		newsize = sizeof(struct panicimage) + newimage->pd_dataSize;
 	}
 	else {
-		newimage = (struct panicimage *) ptr;
+		newimage = (const struct panicimage *)ptr;
 		newsize = size;
 	}
 
@@ -408,7 +416,7 @@ panic_dialog_set_image( const unsigned char * ptr, unsigned int size )
 	panic_dialog_data = &panic_dialog->data[0];
 	panic_dialog_clut = &panic_dialog->data[panic_dialog->pd_dataSize-CLUT_SIZE];
 
-	curr_image_ptr = (unsigned char *) ptr;
+	curr_image_ptr = ptr;
 	curr_image_size = size;
 
 	return (0);
@@ -421,7 +429,7 @@ panic_dialog_set_image( const unsigned char * ptr, unsigned int size )
  */
 
 void
-panic_dialog_get_image( unsigned char ** ptr, unsigned int * size )
+panic_dialog_get_image(const unsigned char ** ptr, unsigned int * size )
 {
 	*ptr =  curr_image_ptr;
 	*size = curr_image_size;
@@ -462,13 +470,18 @@ panic_dialog_verify( const struct panicimage * newimage, unsigned int size )
 
 
 static const struct rendered_num * find_rendered_digit( int digit );
-static void panic_blit_rect_8(	unsigned int x, unsigned int y, unsigned int width, unsigned int height,
-								int transparent, unsigned char * dataPtr );
-static void panic_blit_rect_16(	unsigned int x, unsigned int y, unsigned int width, unsigned int height,
-								int transparent, unsigned char * dataPtr );
-static void panic_blit_rect_32(	unsigned int x, unsigned int y, unsigned int width, unsigned int height,
-								int transparent, unsigned char * dataPtr );
-static int decode_rle( unsigned char * dataPtr, unsigned int * quantity, unsigned int * depth, unsigned char ** value );
+static void panic_blit_rect_8(unsigned int x, unsigned int y,
+			      unsigned int width, unsigned int height,
+			      int transparent, const unsigned char *dataPtr);
+static void panic_blit_rect_16(unsigned int x, unsigned int y,
+			       unsigned int width, unsigned int height,
+			       int transparent, const unsigned char *dataPtr);
+static void panic_blit_rect_32(unsigned int x, unsigned int y,
+			       unsigned int width, unsigned int height,
+			       int transparent, const unsigned char *dataPtr);
+static int decode_rle(const unsigned char *dataPtr,
+		      unsigned int *quantity, unsigned int *depth,
+		      const unsigned char **value);
 
 
 /* Utilities to convert 8 bit/gray */
@@ -479,7 +492,7 @@ static unsigned char findbestgray( unsigned int color24 );
 static int isActiveClutOK( void );
 
 static int
-pixels_needed_to_blit_digit( int digit )
+pixels_needed_to_blit_digit(__unused int digit )
 {
 	return FONT_WIDTH;
 }
@@ -506,7 +519,8 @@ find_rendered_digit( int digit )
 static void 
 blit_digit( int digit )
 {
-	unsigned char * raw_data = (unsigned char *) find_rendered_digit( digit );
+	const unsigned char *raw_data =
+		(const unsigned char *)find_rendered_digit(digit);
 	unsigned width = FONT_WIDTH, height = FONT_HEIGHT;
 	int row;
 
@@ -531,9 +545,9 @@ blit_digit( int digit )
 
 
 static void 
-panic_blit_rect(	unsigned int x, unsigned int y,
-					unsigned int width, unsigned int height,
-					int transparent, unsigned char * dataPtr )
+panic_blit_rect(unsigned int x, unsigned int y, unsigned int width,
+		unsigned int height, int transparent,
+		const unsigned char *dataPtr)
 {
 	if(!vinfo.v_depth)
 		return;
@@ -557,15 +571,15 @@ panic_blit_rect(	unsigned int x, unsigned int y,
  */
 
 static void 
-panic_blit_rect_8(	unsigned int x, unsigned int y,
-					unsigned int width, unsigned int height,
-					int transparent, unsigned char * dataPtr )
+panic_blit_rect_8(unsigned int x, unsigned int y, unsigned int width,
+		  unsigned int height, __unused int transparent,
+		  const unsigned char * dataPtr)
 {
 	volatile unsigned char * dst;
 	unsigned int line, col, i;
 	static int clutOK = -1;
 	unsigned int data, quantity, depth;
-	unsigned char * value;
+	const unsigned char *value;
 	
 
 	if ( clutOK == -1 )
@@ -609,16 +623,16 @@ panic_blit_rect_8(	unsigned int x, unsigned int y,
  *	pixel values (RGB) and writes each pixel to the screen.
  */
 
- static void 
- panic_blit_rect_16(	unsigned int x, unsigned int y,
-			 			unsigned int width, unsigned int height,
-			 			int transparent, unsigned char * dataPtr )
- {
+static void 
+panic_blit_rect_16(unsigned int x, unsigned int y, unsigned int width,
+		   unsigned int height, __unused int transparent,
+		   const unsigned char *dataPtr)
+{
 
 	volatile unsigned short * dst;
 	unsigned int line, col, i;
 	unsigned int  quantity, index, data, depth;
-	unsigned char * value;
+	const unsigned char *value;
 
 	dst = (volatile unsigned short *) (vinfo.v_baseaddr +
 									  (y * vinfo.v_rowbytes) +
@@ -659,15 +673,15 @@ panic_blit_rect_8(	unsigned int x, unsigned int y,
  *	 writes it to the screen.
  */	
 
- static void 
- panic_blit_rect_32(	unsigned int x, unsigned int y,
-			 			unsigned int width, unsigned int height,
-			 			int transparent, unsigned char * dataPtr )
- {
+static void 
+panic_blit_rect_32(unsigned int x, unsigned int y, unsigned int width,
+		   unsigned int height, __unused int transparent,
+		   const unsigned char *dataPtr)
+{
 	volatile unsigned int * dst;
 	unsigned int line, col, i;
 	unsigned int quantity, index, data, depth;
-	unsigned char * value;
+	const unsigned char *value;
 
 
 	dst = (volatile unsigned int *) (vinfo.v_baseaddr +
@@ -730,7 +744,8 @@ panic_blit_rect_8(	unsigned int x, unsigned int y,
 */
 
 static int
-decode_rle( unsigned char * dataPtr, unsigned int * quantity, unsigned int * depth, unsigned char ** value )
+decode_rle(const unsigned char *dataPtr, unsigned int *quantity,
+	   unsigned int *depth, const unsigned char **value )
 {
 	unsigned int mask;
 	int i, runlen, runsize;
@@ -789,14 +804,15 @@ dim_screen(void)
 
 	for (row = p ; row < endp ; row += rowlongs) {
 		for (p = &row[0], col = 0; col < rowline; col++) {
-			*p++ = (*p >> 1) & mask;
+			*p = (*p >> 1) & mask;
+			++p;
 		}
 	}
 }
 
 
 /* From user mode Libc - this ought to be in a library */
-static char *
+static const char *
 strnstr(const char * s, const char * find, size_t slen)
 {
   char c, sc;
@@ -814,7 +830,7 @@ strnstr(const char * s, const char * find, size_t slen)
     } while (strncmp(s, find, len) != 0);
     s--; 
   }       
-  return ((char *)s);
+  return s;
 } 
 
 /*
@@ -863,7 +879,7 @@ findbestgray( unsigned int color24 )
 	}
 
 	/* Did we fail to find any grays ? */
-	if ( bestindex == -1 ) {
+	if (ULONG_MAX == bestindex) {
 		/* someday we should look for the best color match */
 		/* but for now just return the gray as the index */
 		/* at least there might be something readble on the display */
@@ -890,7 +906,7 @@ color24togray8( unsigned int color24 )
     Gray = (R*30) + (G*59) + (B*11);
     gray8 = (unsigned char) ((Gray + 50) / 100);
     return gray8;
-}       
+}
 
 
 static unsigned char
diff --git a/osfmk/console/panic_ui/genimage.c b/osfmk/console/panic_ui/genimage.c
index 2c1ecd110..533e6b32b 100644
--- a/osfmk/console/panic_ui/genimage.c
+++ b/osfmk/console/panic_ui/genimage.c
@@ -1518,7 +1518,7 @@ WriteQTRawFile( FILE * ostream, unsigned char * data, int height, int width, int
 	image_header.vRes =  72 << 16;
 	image_header.dataSize = csize;
 	image_header.frameCount = 1;
-	strcpy(image_header.name, " None"); 
+	strlcpy(image_header.name, " None", sizeof(image_header.name)); 
 	image_header.name[0] = 4;
 	image_header.depth = depth*8;
 	image_header.clutID = (depth==1) ? 8 : -1;
diff --git a/osfmk/console/ppc/serial_console.c b/osfmk/console/ppc/serial_console.c
index bb5e7f87a..a7ec89d97 100644
--- a/osfmk/console/ppc/serial_console.c
+++ b/osfmk/console/ppc/serial_console.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -60,10 +66,8 @@ const int console_unit = 0;
 const uint32_t console_chan_default = CONSOLE_PORT;
 #define console_chan (console_chan_default) /* ^ cpu_number()) */
 
-
 #define MP_SAFE_CONSOLE 1	/* Set this to 1 to allow more than 1 processor to print at once */
 #if MP_SAFE_CONSOLE
-
 struct ppcbfr {													/* Controls multiple processor output */
 	unsigned int 	pos;										/* Current position in buffer */
 	unsigned int	noprompt;									/* Set if we skip the prompt */
@@ -75,14 +79,17 @@ typedef struct ppcbfr ppcbfr_t;
 ppcbfr_t cbfr_boot_cpu;											/* Get one for boot cpu */
 volatile unsigned int cbfpend;									/* A buffer is pending output */
 volatile unsigned int sconowner=-1;								/* Mark who's actually writing */
-
-#endif
-
-#define OPS(putc, getc, nosplputc, nosplgetc) putc, getc
-
-console_ops_t cons_ops[] = {
-	{OPS(scc_putc, scc_getc, no_spl_scputc, no_spl_scgetc)},
-	{OPS(vcputc, vcgetc, no_spl_vcputc, no_spl_vcgetc)},
+#endif /* MP_SAFE_CONSOLE */
+
+struct console_ops cons_ops[] = {
+	{
+		.putc = scc_putc,
+		.getc = scc_getc,
+	},
+	{
+		.putc = vcputc,
+		.getc = vcgetc,
+	},
 };
 
 uint32_t nconsops = (sizeof cons_ops / sizeof cons_ops[0]);
@@ -130,12 +137,12 @@ void cnputcusr(char c) {										/* Echo input character directly */
 	s=splhigh();
 	procinfo = getPerProc();
 
-	hw_atomic_add(&(procinfo->debugger_holdoff), 1);			/* Don't allow debugger entry just now (this is a HACK) */
+	(void)hw_atomic_add(&(procinfo->debugger_holdoff), 1); /* Don't allow debugger entry just now (this is a HACK) */
 
 	_cnputc( c);												/* Echo the character */
 	if(c=='\n') _cnputc( '\r');									/* Add a return if we had a new line */
 
-	hw_atomic_sub(&(procinfo->debugger_holdoff), 1);			/* Don't allow debugger entry just now (this is a HACK) */
+	(void)hw_atomic_sub(&(procinfo->debugger_holdoff), 1); /* Don't allow debugger entry just now (this is a HACK) */
 	splx(s);
 	return;
 }
@@ -165,13 +172,13 @@ cnputc(char c)
 	cpu = procinfo->cpu_number;
 	cbfr = procinfo->pp_cbfr;
 
-	hw_atomic_add(&(procinfo->debugger_holdoff), 1);			/* Don't allow debugger entry just now (this is a HACK) */
+	(void)hw_atomic_add(&(procinfo->debugger_holdoff), 1); /* Don't allow debugger entry just now (this is a HACK) */
 
 	ourbit = 1 << cpu;											/* Make a mask for just us */
 	if(debugger_cpu != -1) {									/* Are we in the debugger with empty buffers? */
 	
 		while(sconowner != cpu) {								/* Anyone but us? */
-			hw_compare_and_store(-1, cpu, (unsigned int *)&sconowner);	/* Try to mark it for us if idle */
+			hw_compare_and_store(-1, cpu, &sconowner);	/* Try to mark it for us if idle */
 		}
 	
 		_cnputc( c);											/* Yeah, just write it */
@@ -179,7 +186,7 @@ cnputc(char c)
 			_cnputc( '\r');										/* Yeah, just add a return */
 			
 		sconowner=-1;											/* Mark it idle */	
-		hw_atomic_sub(&(procinfo->debugger_holdoff), 1);			/* Don't allow debugger entry just now (this is a HACK) */
+		(void)hw_atomic_sub(&(procinfo->debugger_holdoff), 1); /* Don't allow debugger entry just now (this is a HACK) */
 		
 		splx(s);
 		return;													/* Leave... */
@@ -222,11 +229,11 @@ cnputc(char c)
 				
 		while(1) {												/* Loop until we see who's doing this */
 			oldpend=cbfpend;									/* Get the currentest pending buffer flags */
-			if(hw_compare_and_store(oldpend, oldpend|ourbit, (unsigned int *)&cbfpend))	/* Swap ours on if no change */
+			if(hw_compare_and_store(oldpend, oldpend|ourbit, &cbfpend))	/* Swap ours on if no change */
 				break;											/* Bail the loop if it worked */
 		}
 		
-		if(!hw_compare_and_store(-1, cpu, (unsigned int *)&sconowner)) {	/* See if someone else has this, and take it if not */
+		if(!hw_compare_and_store(-1, cpu, &sconowner)) {	/* See if someone else has this, and take it if not */
 			procinfo->debugger_holdoff = 0;						/* Allow debugger entry (this is a HACK) */
 			splx(s);											/* Let's take some 'rupts now */
 			return;												/* We leave here, 'cause another processor is already writing the buffers */
@@ -268,17 +275,17 @@ cnputc(char c)
 						
 					cbfr_cpu->pos=0;								/* Reset the buffer pointer */
 		
-					while(!hw_compare_and_store(cbfpend, cbfpend&~(1<<sccpu), (unsigned int *)&cbfpend));	/* Swap it off */
+					while(!hw_compare_and_store(cbfpend, cbfpend&~(1<<sccpu), &cbfpend));	/* Swap it off */
 				}
 			}
 			sconowner=-1;										/* Set the writer to idle */
 			sync();												/* Insure that everything's done */
-			if(hw_compare_and_store(0, 0, (unsigned int *)&cbfpend)) break;	/* If there are no new buffers, we are done... */
-			if(!hw_compare_and_store(-1, cpu, (unsigned int *)&sconowner)) break;	/* If this isn't idle anymore, we're done */
+			if(hw_compare_and_store(0, 0, &cbfpend)) break;	/* If there are no new buffers, we are done... */
+			if(!hw_compare_and_store(-1, cpu, &sconowner)) break;	/* If this isn't idle anymore, we're done */
 	
 		}
 	}
-	hw_atomic_sub(&(procinfo->debugger_holdoff), 1);					/* Don't allow debugger entry just now (this is a HACK) */
+	(void)hw_atomic_sub(&(procinfo->debugger_holdoff), 1); /* Don't allow debugger entry just now (this is a HACK) */
 	splx(s);													/* Let's take some 'rupts now */
 
 #else  /* MP_SAFE_CONSOLE */
@@ -290,14 +297,14 @@ cnputc(char c)
 }
 
 int
-cngetc()
+cngetc(void)
 {
 	return cons_ops[cons_ops_index].getc(console_unit, console_chan,
 					     TRUE, FALSE);
 }
 
 int
-cnmaygetc()
+cnmaygetc(void)
 {
 	return cons_ops[cons_ops_index].getc(console_unit, console_chan,
 					     FALSE, FALSE);
diff --git a/osfmk/console/ppc/video_scroll.s b/osfmk/console/ppc/video_scroll.s
index db4ba535a..77e4ffcfc 100644
--- a/osfmk/console/ppc/video_scroll.s
+++ b/osfmk/console/ppc/video_scroll.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/console/serial_general.c b/osfmk/console/serial_general.c
index 16ce74940..a2d78e79a 100644
--- a/osfmk/console/serial_general.c
+++ b/osfmk/console/serial_general.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -34,7 +40,6 @@
 #include <kern/thread.h>
 #include <console/serial_protos.h>
 
-extern unsigned int disableSerialOuput;
 extern void cons_cinput(char ch);		/* The BSD routine that gets characters */
 
 unsigned int serialmode;				/* Serial mode keyboard and console control */
@@ -50,7 +55,8 @@ serial_keyboard_init(void)
 	kern_return_t	result;
 	thread_t		thread;
 
-	if(!(serialmode & 2)) return;		/* Leave if we do not want a serial console */
+	if(!(serialmode & 2)) /* Leave if we do not want a serial console */
+		return;
 
 	kprintf("Serial keyboard started\n");
 	result = kernel_thread_start_priority((thread_continue_t)serial_keyboard_start, NULL, MAXPRI_KERNEL, &thread);
@@ -63,10 +69,9 @@ serial_keyboard_init(void)
 void
 serial_keyboard_start(void)
 {
-
-	serial_keyboard_poll();				/* Go see if there are any characters pending now */
+	/* Go see if there are any characters pending now */
+	serial_keyboard_poll();
 	panic("serial_keyboard_start: we can't get back here\n");
-
 }
 
 void
@@ -76,9 +81,10 @@ serial_keyboard_poll(void)
 	uint64_t next;
 
 
-	while(1) {							/* Do this for a while */
+	while(1) {
 		chr = _serial_getc(0, 1, 0, 1);	/* Get a character if there is one */
-		if(chr < 0) break;				/* The serial buffer is empty */
+		if(chr < 0) /* The serial buffer is empty */
+			break;
 		cons_cinput((char)chr);			/* Buffer up the character */
 	}
 
@@ -89,13 +95,14 @@ serial_keyboard_poll(void)
 	panic("serial_keyboard_poll: Shouldn't never ever get here...\n");
 }
 
-boolean_t console_is_serial()
+boolean_t
+console_is_serial(void)
 {
-	return cons_ops_index == SERIAL_CONS_OPS;
+	return (cons_ops_index == SERIAL_CONS_OPS);
 }
 
 int
-switch_to_video_console()
+switch_to_video_console(void)
 {
 	int old_cons_ops = cons_ops_index;
 	cons_ops_index = VC_CONS_OPS;
@@ -103,7 +110,7 @@ switch_to_video_console()
 }
 
 int
-switch_to_serial_console()
+switch_to_serial_console(void)
 {
 	int old_cons_ops = cons_ops_index;
 	cons_ops_index = SERIAL_CONS_OPS;
diff --git a/osfmk/console/serial_protos.h b/osfmk/console/serial_protos.h
index 4d6279d70..89966977b 100644
--- a/osfmk/console/serial_protos.h
+++ b/osfmk/console/serial_protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,7 +32,8 @@
  * @APPLE_FREE_COPYRIGHT@
  */
 
-
+#ifndef _CONSOLE_SERIAL_PROTOS_H_
+#define _CONSOLE_SERIAL_PROTOS_H_
 
 void serial_keyboard_init(void);
 void serial_keyboard_start(void);
@@ -35,20 +42,21 @@ void serial_keyboard_poll(void);
 extern uint32_t serialmode;
 extern uint32_t cons_ops_index;
 extern uint32_t nconsops;
+extern unsigned int disable_serial_output;
 
-extern int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw);
+int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw);
 
-extern boolean_t console_is_serial(void);
-extern int switch_to_serial_console(void);
-extern int switch_to_video_console(void);
-extern void	 switch_to_old_console(int old_console);
+boolean_t console_is_serial(void);
+int switch_to_serial_console(void);
+int switch_to_video_console(void);
+void switch_to_old_console(int old_console);
 
 struct console_ops {
-	int	(*putc)(int, int, int);
+	void	(*putc)(int, int, int);
 	int	(*getc)(int, int, boolean_t, boolean_t);
 } console_ops;
-typedef struct console_ops console_ops_t;
-
 
 #define SERIAL_CONS_OPS 0
 #define VC_CONS_OPS 1
+
+#endif /* _CONSOLE_SERIAL_PROTOS_H_ */
diff --git a/osfmk/console/video_console.c b/osfmk/console/video_console.c
index 133097adb..c42f0df5a 100644
--- a/osfmk/console/video_console.c
+++ b/osfmk/console/video_console.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -85,6 +91,7 @@
 #include <vc.h>
 
 #include <console/video_console.h>
+#include <console/serial_protos.h>
 
 #include <kern/kern_types.h>
 #include <kern/kalloc.h>
@@ -96,11 +103,14 @@
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <machine/io_map_entries.h>
+#include <machine/machine_cpu.h>
 
 #include <pexpert/pexpert.h>
 
 #include "iso_font.c"
 
+#include "sys/msgbuf.h"
+
 /*
  * Generic Console (Front-End)
  * ---------------------------
@@ -120,22 +130,69 @@ static boolean_t vm_initialized = FALSE;
 static struct {
 	void (*initialize)(struct vc_info * info);
 	void (*enable)(boolean_t enable);
-	void (*paint_char)(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous);
-	void (*clear_screen)(int xx, int yy, int top, int bottom, int which);
-	void (*scroll_down)(int num, int top, int bottom);
-	void (*scroll_up)(int num, int top, int bottom);
-	void (*hide_cursor)(int xx, int yy);
-	void (*show_cursor)(int xx, int yy);
+	void (*paint_char)(unsigned int xx, unsigned int yy, unsigned char ch,
+			   int attrs, unsigned char ch_previous,
+			   int attrs_previous);
+	void (*clear_screen)(unsigned int xx, unsigned int yy, unsigned int top,
+			     unsigned int bottom, int which);
+	void (*scroll_down)(int num, unsigned int top, unsigned int bottom);
+	void (*scroll_up)(int num, unsigned int top, unsigned int bottom);
+	void (*hide_cursor)(unsigned int xx, unsigned int yy);
+	void (*show_cursor)(unsigned int xx, unsigned int yy);
 	void (*update_color)(int color, boolean_t fore);
 } gc_ops;
 
-static unsigned char * gc_buffer_attributes = NULL;
-static unsigned char * gc_buffer_characters = NULL;
-static unsigned char * gc_buffer_colorcodes = NULL;
-static unsigned long   gc_buffer_columns    = 0;
-static unsigned long   gc_buffer_rows       = 0;
-static unsigned long   gc_buffer_size       = 0;
-decl_simple_lock_data(,gc_buffer_lock)
+static unsigned char *gc_buffer_attributes;
+static unsigned char *gc_buffer_characters;
+static unsigned char *gc_buffer_colorcodes;
+static unsigned long gc_buffer_columns;
+static unsigned long gc_buffer_rows;
+static unsigned long gc_buffer_size;
+
+#ifdef __i386__
+decl_simple_lock_data(static, vcputc_lock);
+
+#define VCPUTC_LOCK_INIT()				\
+MACRO_BEGIN						\
+	simple_lock_init(&vcputc_lock, 0);		\
+MACRO_END
+
+#define VCPUTC_LOCK_LOCK()				\
+MACRO_BEGIN						\
+	boolean_t istate = ml_get_interrupts_enabled();	\
+	while (!simple_lock_try(&vcputc_lock))		\
+	{						\
+		if (!istate)				\
+			handle_pending_TLB_flushes();	\
+		cpu_pause();				\
+	}						\
+MACRO_END
+
+#define VCPUTC_LOCK_UNLOCK()				\
+MACRO_BEGIN						\
+	simple_unlock(&vcputc_lock);			\
+MACRO_END
+#else
+static hw_lock_data_t vcputc_lock;
+
+#define VCPUTC_LOCK_INIT()				\
+MACRO_BEGIN						\
+	hw_lock_init(&vcputc_lock);			\
+MACRO_END
+
+#define VCPUTC_LOCK_LOCK()				\
+MACRO_BEGIN						\
+	if (!hw_lock_to(&vcputc_lock, LockTimeOut*10))	\
+	{						\
+		panic("VCPUTC_LOCK_LOCK");		\
+	}						\
+MACRO_END
+
+#define VCPUTC_LOCK_UNLOCK()				\
+MACRO_BEGIN						\
+	hw_lock_unlock(&vcputc_lock);			\
+MACRO_END
+#endif
 
 /*
 # Attribute codes: 
@@ -157,18 +214,22 @@ decl_simple_lock_data(,gc_buffer_lock)
 #define COLOR_CODE_GET(code, fore)        (((code) & ((fore) ? 0xF0 : 0x0F))            >> ((fore) ? 4 : 0))
 #define COLOR_CODE_SET(code, color, fore) (((code) & ((fore) ? 0x0F : 0xF0)) | ((color) << ((fore) ? 4 : 0)))
 
-static unsigned char gc_color_code = 0;
+static unsigned char gc_color_code;
 
 /* VT100 state: */
 #define MAXPARS	16
-static int gc_x = 0, gc_y = 0, gc_savex, gc_savey;
-static int gc_par[MAXPARS], gc_numpars, gc_hanging_cursor, gc_attr, gc_saveattr;
+static unsigned int gc_x, gc_y, gc_savex, gc_savey;
+static unsigned int gc_par[MAXPARS], gc_numpars, gc_hanging_cursor, gc_attr, gc_saveattr;
 
 /* VT100 tab stops & scroll region */
 static char gc_tab_stops[255];
-static int  gc_scrreg_top, gc_scrreg_bottom;
+static unsigned int gc_scrreg_top, gc_scrreg_bottom;
 
+#ifdef CONFIG_VC_PROGRESS_WHITE
+enum { kProgressAcquireDelay = 0 /* secs */ };
+#else
 enum { kProgressAcquireDelay = 5 /* secs */ };
+#endif
 
 enum vt100state_e {
 	ESnormal,		/* Nothing yet                             */
@@ -190,12 +251,14 @@ static int gc_charset_select = 0, gc_save_charset_s = 0;
 static int gc_charset[2] = { 0, 0 };
 static int gc_charset_save[2] = { 0, 0 };
 
-static void gc_clear_line(int xx, int yy, int which);
-static void gc_clear_screen(int xx, int yy, int top, int bottom, int which);
+static void gc_clear_line(unsigned int xx, unsigned int yy, int which);
+static void gc_clear_screen(unsigned int xx, unsigned int yy, int top,
+		unsigned int bottom, int which);
 static void gc_enable(boolean_t enable);
-static void gc_hide_cursor(int xx, int yy);
+static void gc_hide_cursor(unsigned int xx, unsigned int yy);
 static void gc_initialize(struct vc_info * info);
-static void gc_paint_char(int xx, int yy, unsigned char ch, int attrs);
+static void gc_paint_char(unsigned int xx, unsigned int yy, unsigned char ch,
+		int attrs);
 static void gc_putchar(char ch);
 static void gc_putc_askcmd(unsigned char ch);
 static void gc_putc_charsetcmd(int charset, unsigned char ch);
@@ -205,20 +268,18 @@ static void gc_putc_getpars(unsigned char ch);
 static void gc_putc_gotpars(unsigned char ch);
 static void gc_putc_normal(unsigned char ch);
 static void gc_putc_square(unsigned char ch);
-static void gc_refresh_screen(void);
 static void gc_reset_screen(void);
 static void gc_reset_tabs(void);
 static void gc_reset_vt100(void);
-static void gc_scroll_down(int num, int top, int bottom);
-static void gc_scroll_up(int num, int top, int bottom);
-static void gc_show_cursor(int xx, int yy);
+static void gc_scroll_down(int num, unsigned int top, unsigned int bottom);
+static void gc_scroll_up(int num, unsigned int top, unsigned int bottom);
+static void gc_show_cursor(unsigned int xx, unsigned int yy);
 static void gc_update_color(int color, boolean_t fore);
-extern int  vcputc(int l, int u, int c);
 
 static void 
-gc_clear_line(int xx, int yy, int which)
+gc_clear_line(unsigned int xx, unsigned int yy, int which)
 {
-	int     start, end, i;
+	unsigned int start, end, i;
 
 	/*
 	 * This routine runs extremely slowly.  I don't think it's
@@ -240,24 +301,21 @@ gc_clear_line(int xx, int yy, int which)
 		start = 0;
 		end = vinfo.v_columns-1;
 		break;
+	default:
+		return;
 	}
 
 	for (i = start; i <= end; i++) {
 		gc_paint_char(i, yy, ' ', ATTR_NONE);
 	}
-
 }
 
 static void 
-gc_clear_screen(int xx, int yy, int top, int bottom, int which)
+gc_clear_screen(unsigned int xx, unsigned int yy, int top, unsigned int bottom,
+		int which)
 {
-	spl_t    s;
-
         if (!gc_buffer_size) return;
 
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( xx < gc_buffer_columns && yy < gc_buffer_rows && bottom <= gc_buffer_rows )
 	{
 		unsigned long start, end;
@@ -275,40 +333,42 @@ gc_clear_screen(int xx, int yy, int top, int bottom, int which)
 				start = (top * gc_buffer_columns);
 				end = (bottom * gc_buffer_columns) - 1;
 				break;
+			default:
+				start = 0;
+				end = 0;
+				break;
 		}
 
-		memset(gc_buffer_attributes + start, 0x00, end - start + 1);
-		memset(gc_buffer_characters + start, 0x00, end - start + 1);
+		memset(gc_buffer_attributes + start, ATTR_NONE, end - start + 1);
+		memset(gc_buffer_characters + start, ' ', end - start + 1);
 		memset(gc_buffer_colorcodes + start, gc_color_code, end - start + 1);
 	}
 
-	simple_unlock(&gc_buffer_lock);
-	splx(s);
-
 	gc_ops.clear_screen(xx, yy, top, bottom, which);
 }
 
 static void
 gc_enable( boolean_t enable )
 {
-	unsigned char * buffer_attributes;
-	unsigned char * buffer_characters;
-	unsigned char * buffer_colorcodes;
-	unsigned long   buffer_columns;
-	unsigned long   buffer_rows;
-	unsigned long   buffer_size;
-
+	unsigned char *buffer_attributes = NULL;
+	unsigned char *buffer_characters = NULL;
+	unsigned char *buffer_colorcodes = NULL;
+	unsigned long buffer_columns = 0;
+	unsigned long buffer_rows = 0;
+	unsigned long buffer_size = 0;
 	spl_t s;
 
 	if ( enable == FALSE )
 	{
-		disableConsoleOutput = TRUE;
+		// only disable console output if it goes to the graphics console
+		if ( console_is_serial() == FALSE )
+			disableConsoleOutput = TRUE;
 		gc_enabled           = FALSE;
 		gc_ops.enable(FALSE);
 	}
 
 	s = splhigh( );
-	simple_lock( &gc_buffer_lock );
+	VCPUTC_LOCK_LOCK( );
 
 	if ( gc_buffer_size )
 	{
@@ -324,7 +384,7 @@ gc_enable( boolean_t enable )
 		gc_buffer_rows       = 0;
 		gc_buffer_size       = 0;
 
-		simple_unlock( &gc_buffer_lock );
+		VCPUTC_LOCK_UNLOCK( );
 		splx( s );
 
 		kfree( buffer_attributes, buffer_size );
@@ -333,7 +393,7 @@ gc_enable( boolean_t enable )
 	}
 	else
 	{
-		simple_unlock( &gc_buffer_lock );
+		VCPUTC_LOCK_UNLOCK( );
 		splx( s );
 	}
 
@@ -365,15 +425,15 @@ gc_enable( boolean_t enable )
 				}
 				else
 				{
-					memset( buffer_attributes, 0x00, buffer_size );
-					memset( buffer_characters, 0x00, buffer_size );
-					memset( buffer_colorcodes, 0x0F, buffer_size );
+					memset( buffer_attributes, ATTR_NONE, buffer_size );
+					memset( buffer_characters, ' ', buffer_size );
+					memset( buffer_colorcodes, COLOR_CODE_SET( 0, COLOR_FOREGROUND, TRUE ), buffer_size );
 				}
 			}
 		}
 
 		s = splhigh( );
-		simple_lock( &gc_buffer_lock );
+		VCPUTC_LOCK_LOCK( );
 
 		gc_buffer_attributes = buffer_attributes;
 		gc_buffer_characters = buffer_characters;
@@ -382,10 +442,13 @@ gc_enable( boolean_t enable )
 		gc_buffer_rows       = buffer_rows;
 		gc_buffer_size       = buffer_size;
 
-		simple_unlock( &gc_buffer_lock );
+		gc_reset_screen();
+
+		VCPUTC_LOCK_UNLOCK( );
 		splx( s );
 
-		gc_reset_screen();
+		gc_ops.clear_screen(gc_x, gc_y, 0, vinfo.v_rows, 2);
+		gc_ops.show_cursor(gc_x, gc_y);
 
 		gc_ops.enable(TRUE);
 		gc_enabled           = TRUE;
@@ -394,13 +457,8 @@ gc_enable( boolean_t enable )
 }
 
 static void
-gc_hide_cursor(int xx, int yy)
+gc_hide_cursor(unsigned int xx, unsigned int yy)
 {
-	spl_t s;
-
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( xx < gc_buffer_columns && yy < gc_buffer_rows )
 	{
 		unsigned long index = (yy * gc_buffer_columns) + xx;
@@ -409,9 +467,6 @@ gc_hide_cursor(int xx, int yy)
 		unsigned char colorcode = gc_buffer_colorcodes[index];
 		unsigned char colorcodesave = gc_color_code;
 
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_update_color(COLOR_CODE_GET(colorcode, TRUE ), TRUE );
 		gc_update_color(COLOR_CODE_GET(colorcode, FALSE), FALSE);
 
@@ -422,9 +477,6 @@ gc_hide_cursor(int xx, int yy)
 	}
 	else
 	{
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_ops.hide_cursor(xx, yy);
 	}
 }
@@ -435,7 +487,7 @@ gc_initialize(struct vc_info * info)
 	if ( gc_initialized == FALSE )
 	{
 		/* Init our lock */
-		simple_lock_init(&gc_buffer_lock, 0);
+		VCPUTC_LOCK_INIT();
 
 		gc_initialized = TRUE;
 	}
@@ -447,13 +499,8 @@ gc_initialize(struct vc_info * info)
 }
 
 static void
-gc_paint_char(int xx, int yy, unsigned char ch, int attrs)
+gc_paint_char(unsigned int xx, unsigned int yy, unsigned char ch, int attrs)
 {
-	spl_t s;
-
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( xx < gc_buffer_columns && yy < gc_buffer_rows )
 	{
 		unsigned long index = (yy * gc_buffer_columns) + xx;
@@ -463,9 +510,6 @@ gc_paint_char(int xx, int yy, unsigned char ch, int attrs)
 		gc_buffer_colorcodes[index] = gc_color_code;
 	}
 
-	simple_unlock(&gc_buffer_lock);
-	splx(s);
-
 	gc_ops.paint_char(xx, yy, ch, attrs, 0, 0);
 }
 
@@ -507,18 +551,17 @@ gc_putchar(char ch)
 	}
 
 	if (gc_x >= vinfo.v_columns) {
-		gc_x = vinfo.v_columns - 1;
-	}
-	if (gc_x < 0) {
-		gc_x = 0;
+		if (0 == vinfo.v_columns)
+			gc_x = 0;
+		else
+			gc_x = vinfo.v_columns - 1;
 	}
 	if (gc_y >= vinfo.v_rows) {
-		gc_y = vinfo.v_rows - 1;
-	}
-	if (gc_y < 0) {
-		gc_y = 0;
+		if (0 == vinfo.v_rows)
+			gc_y = 0;
+		else
+			gc_y = vinfo.v_rows - 1;
 	}
-
 }
 
 static void
@@ -575,7 +618,7 @@ gc_putc_charsizecmd(unsigned char ch)
 			break;
 		case '8' :	/* fill 'E's */
 			{
-				int xx, yy;
+				unsigned int xx, yy;
 				for (yy = 0; yy < vinfo.v_rows; yy++)
 					for (xx = 0; xx < vinfo.v_columns; xx++)
 						gc_paint_char(xx, yy, 'E', ATTR_NONE);
@@ -687,7 +730,7 @@ gc_putc_getpars(unsigned char ch)
 static void 
 gc_putc_gotpars(unsigned char ch)
 {
-	int     i;
+	unsigned int i;
 
 	if (ch < ' ') {
 		/* special case for vttest for handling cursor
@@ -714,9 +757,12 @@ gc_putc_gotpars(unsigned char ch)
 			gc_x = vinfo.v_columns-1;
 		break;
 	case 'D':		/* Left			 */
-		gc_x -= gc_par[0] ? gc_par[0] : 1;
-		if (gc_x < 0)
+		if (gc_par[0] > gc_x)
 			gc_x = 0;
+		else if (gc_par[0])
+			gc_x -= gc_par[0];
+		else if (gc_x)
+			--gc_x;
 		break;
 	case 'H':		/* Set cursor position	 */
 	case 'f':
@@ -728,7 +774,6 @@ gc_putc_gotpars(unsigned char ch)
 		break;
 	case 'X':		/* clear p1 characters */
 		if (gc_numpars) {
-			int i;
 			for (i = gc_x; i < gc_x + gc_par[0]; i++)
 				gc_paint_char(i, gc_y, ' ', ATTR_NONE);
 		}
@@ -747,8 +792,6 @@ gc_putc_gotpars(unsigned char ch)
 				break;				
 			case 3:	/* Clear every tabs */
 				{
-					int i;
-
 					for (i = 0; i <= vinfo.v_columns; i++)
 						gc_tab_stops[i] = 0;
 				}
@@ -801,8 +844,6 @@ gc_putc_gotpars(unsigned char ch)
 		/* ensure top < bottom, and both within limits */
 		if ((gc_numpars > 0) && (gc_par[0] < vinfo.v_rows)) {
 			gc_scrreg_top = gc_par[0] ? gc_par[0] - 1 : 0;
-			if (gc_scrreg_top < 0)
-				gc_scrreg_top = 0;
 		} else {
 			gc_scrreg_top = 0;
 		}
@@ -909,59 +950,17 @@ gc_putc_square(unsigned char ch)
 
 }
 
-static void 
-gc_refresh_screen(void)
-{
-	spl_t s;
-
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
-	if ( gc_buffer_size )
-	{
-		unsigned char colorcodesave = gc_color_code;
-		unsigned long column, row;
-		unsigned long index;
-
-		for ( index = 0, row = 0 ; row < gc_buffer_rows ; row++ )
-		{
-			for ( column = 0 ; column < gc_buffer_columns ; index++, column++ )
-			{
-				if ( gc_buffer_colorcodes[index] != gc_color_code )
-				{
-					gc_update_color(COLOR_CODE_GET(gc_buffer_colorcodes[index], TRUE ), TRUE );
-					gc_update_color(COLOR_CODE_GET(gc_buffer_colorcodes[index], FALSE), FALSE);
-				}
-
-				gc_ops.paint_char(column, row, gc_buffer_characters[index], gc_buffer_attributes[index], 0, 0);
-			}
-		}
-
-		if ( colorcodesave != gc_color_code )
-		{
-			gc_update_color(COLOR_CODE_GET(colorcodesave, TRUE ), TRUE );
-			gc_update_color(COLOR_CODE_GET(colorcodesave, FALSE), FALSE);
-		}
-	}
-        
-	simple_unlock(&gc_buffer_lock);
-	splx(s);
-}
-
 static void
 gc_reset_screen(void)
 {
-	gc_hide_cursor(gc_x, gc_y);
 	gc_reset_vt100();
 	gc_x = gc_y = 0;
-	gc_clear_screen(gc_x, gc_y, 0, vinfo.v_rows, 2);
-	gc_show_cursor(gc_x, gc_y);
 }
 
 static void
 gc_reset_tabs(void)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i<= vinfo.v_columns; i++) {
 		gc_tab_stops[i] = ((i % 8) == 0);
@@ -985,15 +984,10 @@ gc_reset_vt100(void)
 }
 
 static void 
-gc_scroll_down(int num, int top, int bottom)
+gc_scroll_down(int num, unsigned int top, unsigned int bottom)
 {
-	spl_t s;
-
         if (!gc_buffer_size) return;
 
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( bottom <= gc_buffer_rows )
 	{
 		unsigned char colorcodesave = gc_color_code;
@@ -1050,32 +1044,59 @@ gc_scroll_down(int num, int top, int bottom)
 			gc_update_color(COLOR_CODE_GET(colorcodesave, FALSE), FALSE);
 		}
 
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
+		/* Now set the freed up lines to the background colour */
+
+		for ( row = top ; row < top + num ; row++ )
+		{
+			index = row * gc_buffer_columns;
+
+			for ( column = 0 ; column < gc_buffer_columns ; index++, column++ )
+			{
+				if ( gc_buffer_attributes[index] != ATTR_NONE     || 
+				     gc_buffer_characters[index] != ' '           || 
+				     gc_buffer_colorcodes[index] != gc_color_code )
+				{
+					if ( gc_buffer_colorcodes[index] != gc_color_code )
+					{
+						gc_ops.paint_char( /* xx             */ column,
+						                   /* yy             */ row,
+						                   /* ch             */ ' ',
+						                   /* attrs          */ ATTR_NONE,
+						                   /* ch_previous    */ 0,
+						                   /* attrs_previous */ 0 );
+					}
+					else
+					{
+						gc_ops.paint_char( /* xx             */ column,
+						                   /* yy             */ row,
+						                   /* ch             */ ' ',
+						                   /* attrs          */ ATTR_NONE,
+						                   /* ch_previous    */ gc_buffer_characters[index],
+						                   /* attrs_previous */ gc_buffer_attributes[index] );
+					}
+
+					gc_buffer_attributes[index] = ATTR_NONE;
+					gc_buffer_characters[index] = ' ';
+					gc_buffer_colorcodes[index] = gc_color_code;
+				}
+			}
+		}
 	}
 	else
 	{
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_ops.scroll_down(num, top, bottom);
-	}
 
-	/* Now set the freed up lines to the background colour */
+		/* Now set the freed up lines to the background colour */
 
-	gc_clear_screen(vinfo.v_columns - 1, top + num - 1, top, bottom, 1);
+		gc_clear_screen(vinfo.v_columns - 1, top + num - 1, top, bottom, 1);
+	}
 }
 
 static void 
-gc_scroll_up(int num, int top, int bottom)
+gc_scroll_up(int num, unsigned int top, unsigned int bottom)
 {
-	spl_t s;
-
         if (!gc_buffer_size) return;
 
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( bottom <= gc_buffer_rows )
 	{
 		unsigned char colorcodesave = gc_color_code;
@@ -1122,7 +1143,6 @@ gc_scroll_up(int num, int top, int bottom)
 					gc_buffer_attributes[index] = gc_buffer_attributes[index + jump];
 					gc_buffer_characters[index] = gc_buffer_characters[index + jump];
 					gc_buffer_colorcodes[index] = gc_buffer_colorcodes[index + jump];
-
 				}
 			}
 		}
@@ -1133,30 +1153,57 @@ gc_scroll_up(int num, int top, int bottom)
 			gc_update_color(COLOR_CODE_GET(colorcodesave, FALSE), FALSE);
 		}
 
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
+		/* Now set the freed up lines to the background colour */
+
+		for ( row = bottom - num ; row < bottom ; row++ )
+		{
+			index = row * gc_buffer_columns;
+
+			for ( column = 0 ; column < gc_buffer_columns ; index++, column++ )
+			{
+				if ( gc_buffer_attributes[index] != ATTR_NONE     || 
+				     gc_buffer_characters[index] != ' '           || 
+				     gc_buffer_colorcodes[index] != gc_color_code )
+				{
+					if ( gc_buffer_colorcodes[index] != gc_color_code )
+					{
+						gc_ops.paint_char( /* xx             */ column,
+						                   /* yy             */ row,
+						                   /* ch             */ ' ',
+						                   /* attrs          */ ATTR_NONE,
+						                   /* ch_previous    */ 0,
+						                   /* attrs_previous */ 0 );
+					}
+					else
+					{
+						gc_ops.paint_char( /* xx             */ column,
+						                   /* yy             */ row,
+						                   /* ch             */ ' ',
+						                   /* attrs          */ ATTR_NONE,
+						                   /* ch_previous    */ gc_buffer_characters[index],
+						                   /* attrs_previous */ gc_buffer_attributes[index] );
+					}
+
+					gc_buffer_attributes[index] = ATTR_NONE;
+					gc_buffer_characters[index] = ' ';
+					gc_buffer_colorcodes[index] = gc_color_code;
+				}
+			}
+		}
 	}
 	else
 	{
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_ops.scroll_up(num, top, bottom);
-	}
 
-	/* Now set the freed up lines to the background colour */
+		/* Now set the freed up lines to the background colour */
 
-	gc_clear_screen(0, bottom - num, top, bottom, 0);
+		gc_clear_screen(0, bottom - num, top, bottom, 0);
+	}
 }
 
 static void
-gc_show_cursor(int xx, int yy)
+gc_show_cursor(unsigned int xx, unsigned int yy)
 {
-	spl_t s;
-
-	s = splhigh();
-	simple_lock(&gc_buffer_lock);
-
 	if ( xx < gc_buffer_columns && yy < gc_buffer_rows )
 	{
 		unsigned long index = (yy * gc_buffer_columns) + xx;
@@ -1165,9 +1212,6 @@ gc_show_cursor(int xx, int yy)
 		unsigned char colorcode = gc_buffer_colorcodes[index];
 		unsigned char colorcodesave = gc_color_code;
 
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_update_color(COLOR_CODE_GET(colorcode, FALSE), TRUE );
 		gc_update_color(COLOR_CODE_GET(colorcode, TRUE ), FALSE);
 
@@ -1178,9 +1222,6 @@ gc_show_cursor(int xx, int yy)
 	}
 	else
 	{
-		simple_unlock(&gc_buffer_lock);
-		splx(s);
-
 		gc_ops.show_cursor(xx, yy);
 	}
 }
@@ -1192,17 +1233,23 @@ gc_update_color(int color, boolean_t fore)
 	gc_ops.update_color(color, fore);
 }
 
-int
-vcputc(int l, int u, int c)
+void
+vcputc(__unused int l, __unused int u, int c)
 {
 	if ( gc_enabled || debug_mode )
-        {
+	{
+		spl_t s;
+
+		s = splhigh();
+		VCPUTC_LOCK_LOCK();
+
 		gc_hide_cursor(gc_x, gc_y);
 		gc_putchar(c);
 		gc_show_cursor(gc_x, gc_y);
-	}
 
-	return 0;
+		VCPUTC_LOCK_UNLOCK();
+		splx(s);
+	}
 }
 
 /*
@@ -1248,22 +1295,9 @@ static int vc_rendered_char_size = 0;
 #define REN_MAX_DEPTH	32
 static unsigned char vc_rendered_char[ISO_CHAR_HEIGHT * ((REN_MAX_DEPTH / 8) * ISO_CHAR_WIDTH)];
 
-static void vc_clear_screen(int xx, int yy, int scrreg_top, int scrreg_bottom, int which);
-static void vc_enable(boolean_t enable);
-static void vc_initialize(struct vc_info * vinfo_p);
-static void vc_paint_char(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous);
-static void vc_paint_char_8(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous); 
-static void vc_paint_char_16(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous); 
-static void vc_paint_char_32(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous);
-static void vc_render_char(unsigned char ch, unsigned char *renderptr, short newdepth);
-static void vc_render_font(short newdepth);
-static void vc_reverse_cursor(int xx, int yy);
-static void vc_scroll_down(int num, int scrreg_top, int scrreg_bottom);
-static void vc_scroll_up(int num, int scrreg_top, int scrreg_bottom);
-static void vc_update_color(int color, boolean_t fore);
-
 static void 
-vc_clear_screen(int xx, int yy, int scrreg_top, int scrreg_bottom, int which)
+vc_clear_screen(unsigned int xx, unsigned int yy, unsigned int scrreg_top,
+		unsigned int scrreg_bottom, int which)
 {
 	unsigned long *p, *endp, *row;
 	int      linelongs, col;
@@ -1311,43 +1345,52 @@ vc_clear_screen(int xx, int yy, int scrreg_top, int scrreg_bottom, int which)
 }
 
 static void
-vc_enable(boolean_t enable)
+vc_initialize(__unused struct vc_info * vinfo_p)
 {
-	if ( enable )
-	{
-		vc_render_font(vinfo.v_depth);
-	}
-}
 
-static void
-vc_initialize(struct vc_info * vinfo_p)
-{
 	vinfo.v_rows = vinfo.v_height / ISO_CHAR_HEIGHT;
 	vinfo.v_columns = vinfo.v_width / ISO_CHAR_WIDTH;
 	vinfo.v_rowscanbytes = (vinfo.v_depth / 8) * vinfo.v_width;
 }
 
 static void
-vc_paint_char(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous)
+vc_render_char(unsigned char ch, unsigned char *renderptr, short newdepth)
 {
-	if( !vinfo.v_depth)
-		return;
+	union {
+		unsigned char  *charptr;
+		unsigned short *shortptr;
+		unsigned long  *longptr;
+	} current; 	/* current place in rendered font, multiple types. */
+	unsigned char *theChar;	/* current char in iso_font */
+	int line;
 
-	switch( vinfo.v_depth) {
-		case 8:
-			vc_paint_char_8(xx, yy, ch, attrs, ch_previous, attrs_previous);
-			break;
-		case 16:
-			vc_paint_char_16(xx, yy, ch, attrs, ch_previous, attrs_previous);
-			break;
-		case 32:
-			vc_paint_char_32(xx, yy, ch, attrs, ch_previous, attrs_previous);
-			break;
+	current.charptr = renderptr;
+	theChar = iso_font + (ch * ISO_CHAR_HEIGHT);
+
+	for (line = 0; line < ISO_CHAR_HEIGHT; line++) {
+		unsigned char mask = 1;
+		do {
+			switch (newdepth) {
+			case 8: 
+				*current.charptr++ = (*theChar & mask) ? 0xFF : 0;
+				break;
+			case 16:
+				*current.shortptr++ = (*theChar & mask) ? 0xFFFF : 0;
+				break;
+
+			case 32: 
+				*current.longptr++ = (*theChar & mask) ? 0xFFFFFFFF : 0;
+				break;
+			}
+			mask <<= 1;
+		} while (mask);	/* while the single bit drops to the right */
+		theChar++;
 	}
 }
 
 static void
-vc_paint_char_8(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous) 
+vc_paint_char_8(unsigned int xx, unsigned int yy, unsigned char ch, int attrs,
+		__unused unsigned char ch_previous, __unused int attrs_previous)
 {
 	unsigned long *theChar;
 	unsigned long *where;
@@ -1402,7 +1445,9 @@ vc_paint_char_8(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_pr
 }
 
 static void
-vc_paint_char_16(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous) 
+vc_paint_char_16(unsigned int xx, unsigned int yy, unsigned char ch, int attrs,
+		 __unused unsigned char ch_previous,
+		 __unused int attrs_previous) 
 {
 	unsigned long *theChar;
 	unsigned long *where;
@@ -1453,7 +1498,8 @@ vc_paint_char_16(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_p
 }
 
 static void
-vc_paint_char_32(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_previous, int attrs_previous) 
+vc_paint_char_32(unsigned int xx, unsigned int yy, unsigned char ch, int attrs,
+		 unsigned char ch_previous, int attrs_previous) 
 {
 	unsigned long *theChar;
 	unsigned long *theCharPrevious;
@@ -1515,39 +1561,24 @@ vc_paint_char_32(int xx, int yy, unsigned char ch, int attrs, unsigned char ch_p
 }
 
 static void
-vc_render_char(unsigned char ch, unsigned char *renderptr, short newdepth)
+vc_paint_char(unsigned int xx, unsigned int yy, unsigned char ch, int attrs,
+	      unsigned char ch_previous, int attrs_previous)
 {
-	union {
-		unsigned char  *charptr;
-		unsigned short *shortptr;
-		unsigned long  *longptr;
-	} current; 	/* current place in rendered font, multiple types. */
-
-	unsigned char *theChar;	/* current char in iso_font */
-
-	int line;
-
-	current.charptr = renderptr;
-	theChar = iso_font + (ch * ISO_CHAR_HEIGHT);
+	if(!vinfo.v_depth)
+		return;
 
-	for (line = 0; line < ISO_CHAR_HEIGHT; line++) {
-		unsigned char mask = 1;
-		do {
-			switch (newdepth) {
-				case 8: 
-					*current.charptr++ = (*theChar & mask) ? 0xFF : 0;
-					break;
-				case 16:
-					*current.shortptr++ = (*theChar & mask) ? 0xFFFF : 0;
-					break;
-
-				case 32: 
-					*current.longptr++ = (*theChar & mask) ? 0xFFFFFFFF : 0;
-					break;
-			}
-			mask <<= 1;
-		} while (mask);	/* while the single bit drops to the right */
-		theChar++;
+	switch(vinfo.v_depth) {
+	case 8:
+		vc_paint_char_8(xx, yy, ch, attrs, ch_previous, attrs_previous);
+		break;
+	case 16:
+		vc_paint_char_16(xx, yy, ch, attrs, ch_previous,
+				 attrs_previous);
+		break;
+	case 32:
+		vc_paint_char_32(xx, yy, ch, attrs, ch_previous,
+				 attrs_previous);
+		break;
 	}
 }
 
@@ -1557,6 +1588,10 @@ vc_render_font(short newdepth)
 	static short olddepth = 0;
 
 	int charindex;	/* index in ISO font */
+	unsigned char *rendered_font;
+	unsigned long rendered_font_size;
+	int rendered_char_size;
+	spl_t s;
 
 	if (vm_initialized == FALSE) {
 		return;	/* nothing to do */
@@ -1564,28 +1599,61 @@ vc_render_font(short newdepth)
 	if (olddepth == newdepth && vc_rendered_font) {
 		return;	/* nothing to do */
 	}
-	if (vc_rendered_font) {
-		kfree(vc_rendered_font, vc_rendered_font_size);
+
+	s = splhigh();
+	VCPUTC_LOCK_LOCK();
+
+	rendered_font      = vc_rendered_font;
+	rendered_font_size = vc_rendered_font_size;
+	rendered_char_size = vc_rendered_char_size;
+
+	vc_rendered_font      = NULL;
+	vc_rendered_font_size = 0;
+	vc_rendered_char_size = 0;
+
+	VCPUTC_LOCK_UNLOCK();
+	splx(s);
+
+	if (rendered_font) {
+		kfree(rendered_font, rendered_font_size);
+		rendered_font = NULL;
 	}
 
-	vc_rendered_char_size = ISO_CHAR_HEIGHT * ((newdepth / 8) * ISO_CHAR_WIDTH);
-	vc_rendered_font_size = (ISO_CHAR_MAX-ISO_CHAR_MIN+1) * vc_rendered_char_size;
-        vc_rendered_font = (unsigned char *) kalloc(vc_rendered_font_size);
+	if (newdepth) {
+		rendered_char_size = ISO_CHAR_HEIGHT * ((newdepth / 8) * ISO_CHAR_WIDTH);
+		rendered_font_size = (ISO_CHAR_MAX-ISO_CHAR_MIN+1) * rendered_char_size;
+		rendered_font = (unsigned char *) kalloc(rendered_font_size);
+	}
 
-	if (vc_rendered_font == NULL) {
-		vc_rendered_font_size = 0;
+	if (rendered_font == NULL) {
 		return;
 	}
 
 	for (charindex = ISO_CHAR_MIN; charindex <= ISO_CHAR_MAX; charindex++) {
-		vc_render_char(charindex, vc_rendered_font + (charindex * vc_rendered_char_size), newdepth);
+		vc_render_char(charindex, rendered_font + (charindex * rendered_char_size), newdepth);
 	}
 
 	olddepth = newdepth;
+
+	s = splhigh();
+	VCPUTC_LOCK_LOCK();
+
+	vc_rendered_font      = rendered_font;
+	vc_rendered_font_size = rendered_font_size;
+	vc_rendered_char_size = rendered_char_size;
+
+	VCPUTC_LOCK_UNLOCK();
+	splx(s);
 }
 
 static void
-vc_reverse_cursor(int xx, int yy)
+vc_enable(boolean_t enable)
+{
+	vc_render_font(enable ? vinfo.v_depth : 0);
+}
+
+static void
+vc_reverse_cursor(unsigned int xx, unsigned int yy)
 {
 	unsigned long *where;
 	int line, col;
@@ -1616,7 +1684,7 @@ vc_reverse_cursor(int xx, int yy)
 }
 
 static void 
-vc_scroll_down(int num, int scrreg_top, int scrreg_bottom)
+vc_scroll_down(int num, unsigned int scrreg_top, unsigned int scrreg_bottom)
 {
 	unsigned long *from, *to,  linelongs, i, line, rowline, rowscanline;
 
@@ -1649,7 +1717,7 @@ vc_scroll_down(int num, int scrreg_top, int scrreg_bottom)
 }
 
 static void 
-vc_scroll_up(int num, int scrreg_top, int scrreg_bottom)
+vc_scroll_up(int num, unsigned int scrreg_top, unsigned int scrreg_bottom)
 {
 	unsigned long *from, *to, linelongs, i, line, rowline, rowscanline;
 
@@ -1742,7 +1810,7 @@ static void vc_blit_rect_32( int x, int y, int width, int height,
                              unsigned int * backBuffer, boolean_t save, boolean_t static_alpha );
 extern void vc_display_icon( vc_progress_element * desc, const unsigned char * data );
 extern void vc_progress_initialize( vc_progress_element * desc, const unsigned char * data, const unsigned char * clut );
-static void vc_progress_set( boolean_t enable, uint32_t delay );
+static void vc_progress_set(boolean_t enable, uint32_t vc_delay);
 static void vc_progress_task( void * arg0, void * arg );
 
 static void vc_blit_rect(	int x, int y,
@@ -1769,12 +1837,11 @@ static void vc_blit_rect(	int x, int y,
     }
 }
 
-static void vc_blit_rect_8(	int x, int y,
-                                int width, int height, 
-                                const unsigned char * dataPtr,
-                                const unsigned char * alphaPtr,
-                                unsigned char * backPtr,
-                                boolean_t save, boolean_t static_alpha )
+static void
+vc_blit_rect_8(int x, int y, int width, int height,
+	       const unsigned char * dataPtr, const unsigned char * alphaPtr,
+	       __unused unsigned char * backPtr, __unused boolean_t save,
+	       __unused boolean_t static_alpha)
 {
     volatile unsigned char * dst;
     int line, col;
@@ -1795,6 +1862,21 @@ static void vc_blit_rect_8(	int x, int y,
     }
 }
 
+/* For ARM, 16-bit is 565 (RGB); it is 1555 (XRGB) on other platforms */
+
+#define CLUT_MASK_R	0xf8
+#define CLUT_MASK_G	0xf8
+#define CLUT_MASK_B	0xf8
+#define CLUT_SHIFT_R	<< 7
+#define CLUT_SHIFT_G	<< 2
+#define CLUT_SHIFT_B	>> 3
+#define MASK_R		0x7c00
+#define MASK_G		0x03e0
+#define MASK_B		0x001f
+#define MASK_R_8	0x3fc00
+#define MASK_G_8	0x01fe0
+#define MASK_B_8	0x000ff
+
 static void vc_blit_rect_16(	int x, int y,
                                 int width, int height,
                                 const unsigned char * dataPtr,
@@ -1804,7 +1886,7 @@ static void vc_blit_rect_16(	int x, int y,
 {
     volatile unsigned short * dst;
     int line, col;
-    unsigned int data, index, alpha, back;
+    unsigned int data = 0, index = 0, alpha, back;
 
     dst = (volatile unsigned short *)(vinfo.v_baseaddr +
                                     (y * vinfo.v_rowbytes) +
@@ -1823,28 +1905,35 @@ static void vc_blit_rect_16(	int x, int y,
                 data = 0;
 		if( dataPtr != 0) {
                     if( vc_clut[index + 0] > alpha)
-                        data |= (((vc_clut[index + 0] - alpha) & 0xf8) << 7);
+                        data |= (((vc_clut[index + 0] - alpha) & CLUT_MASK_R) CLUT_SHIFT_R);
                     if( vc_clut[index + 1] > alpha)
-                        data |= (((vc_clut[index + 1] - alpha) & 0xf8) << 2);
+                        data |= (((vc_clut[index + 1] - alpha) & CLUT_MASK_G) CLUT_SHIFT_G);
                     if( vc_clut[index + 2] > alpha)
-                        data |= (((vc_clut[index + 2] - alpha) & 0xf8) >> 3);
+                        data |= (((vc_clut[index + 2] - alpha) & CLUT_MASK_B) CLUT_SHIFT_B);
 		}
+#ifdef CONFIG_VC_PROGRESS_WHITE
+		else {
+		    data |= (((0xff - alpha) & CLUT_MASK_R) CLUT_SHIFT_R);
+		    data |= (((0xff - alpha) & CLUT_MASK_G) CLUT_SHIFT_G);
+		    data |= (((0xff - alpha) & CLUT_MASK_B) CLUT_SHIFT_B);
+		}
+#endif
 
                 if( save) {
                     back = *(dst + col);
                     if ( !static_alpha)
                         *backPtr++ = back;
-                        back = (((((back & 0x7c00) * alpha) + 0x3fc00) >> 8) & 0x7c00)
-                             | (((((back & 0x03e0) * alpha) + 0x01fe0) >> 8) & 0x03e0)
-                             | (((((back & 0x001f) * alpha) + 0x000ff) >> 8) & 0x001f);
+                        back = (((((back & MASK_R) * alpha) + MASK_R_8) >> 8) & MASK_R)
+                             | (((((back & MASK_G) * alpha) + MASK_G_8) >> 8) & MASK_G)
+                             | (((((back & MASK_B) * alpha) + MASK_B_8) >> 8) & MASK_B);
                     if ( static_alpha)
                         *backPtr++ = back;
                 } else {
                     back = *backPtr++;
                     if ( !static_alpha) {
-                        back = (((((back & 0x7c00) * alpha) + 0x3fc00) >> 8) & 0x7c00)
-                             | (((((back & 0x03e0) * alpha) + 0x01fe0) >> 8) & 0x03e0)
-                             | (((((back & 0x001f) * alpha) + 0x000ff) >> 8) & 0x001f);
+                        back = (((((back & MASK_R) * alpha) + MASK_R_8) >> 8) & MASK_R)
+                             | (((((back & MASK_G) * alpha) + MASK_G_8) >> 8) & MASK_G)
+                             | (((((back & MASK_B) * alpha) + MASK_B_8) >> 8) & MASK_B);
                     }
                 }
 
@@ -1852,9 +1941,9 @@ static void vc_blit_rect_16(	int x, int y,
 
             } else
                 if( dataPtr != 0) {
-            	    data = ( (0xf8 & (vc_clut[index + 0])) << 7)
-                           | ( (0xf8 & (vc_clut[index + 1])) << 2)
-                           | ( (0xf8 & (vc_clut[index + 2])) >> 3);
+            	    data = ( (CLUT_MASK_R & (vc_clut[index + 0])) CLUT_SHIFT_R)
+                           | ( (CLUT_MASK_G & (vc_clut[index + 1])) CLUT_SHIFT_G)
+                           | ( (CLUT_MASK_B & (vc_clut[index + 2])) CLUT_SHIFT_B);
 		}
 
             *(dst + col) = data;
@@ -1872,7 +1961,7 @@ static void vc_blit_rect_32(	int x, int y,
 {
     volatile unsigned int * dst;
     int line, col;
-    unsigned int data, index, alpha, back;
+    unsigned int data = 0, index = 0, alpha, back;
 
     dst = (volatile unsigned int *) (vinfo.v_baseaddr +
                                     (y * vinfo.v_rowbytes) +
@@ -1897,6 +1986,13 @@ static void vc_blit_rect_32(	int x, int y,
                     if( vc_clut[index + 2] > alpha)
                         data |= ((vc_clut[index + 2] - alpha));
 		}
+#ifdef CONFIG_VC_PROGRESS_WHITE
+		else {
+		    data |= (0xff - alpha) << 16;
+		    data |= (0xff - alpha) << 8;
+		    data |= (0xff - alpha);
+		}
+#endif
 
                 if( save) {
                     back = *(dst + col);
@@ -1960,6 +2056,8 @@ vc_progress_initialize( vc_progress_element * desc,
     vc_clut = clut;
     vc_clut8 = clut;
 
+    simple_lock_init(&vc_progress_lock, 0);
+
     vc_progress = desc;
     vc_progress_data = data;
     if( 2 & vc_progress->flags)
@@ -1972,15 +2070,13 @@ vc_progress_initialize( vc_progress_element * desc,
 
     clock_interval_to_absolutetime_interval(vc_progress->time, 1000 * 1000, &abstime);
     vc_progress_interval = abstime;
-
-    simple_lock_init(&vc_progress_lock, 0);
 }
 
 static void
-vc_progress_set( boolean_t enable, uint32_t delay )
+vc_progress_set(boolean_t enable, uint32_t vc_delay)
 {
     spl_t	     s;
-    void             *saveBuf = 0;
+    void             *saveBuf = NULL;
     vm_size_t        saveLen = 0;
     unsigned int     count;
     unsigned int     index;
@@ -2016,9 +2112,9 @@ vc_progress_set( boolean_t enable, uint32_t delay )
 
 	    case 16 :
 		buf16 = (unsigned short *) saveBuf;
-		pdata16 = ((vc_clut[0x01 * 3 + 0] & 0xf8) << 7)
-		       | ((vc_clut[0x01 * 3 + 0] & 0xf8) << 2)
-		       | ((vc_clut[0x01 * 3 + 0] & 0xf8) >> 3);
+		pdata16 = ((vc_clut[0x01 * 3 + 0] & CLUT_MASK_R) CLUT_SHIFT_R)
+		       | ((vc_clut[0x01 * 3 + 0] & CLUT_MASK_G) CLUT_SHIFT_G)
+		       | ((vc_clut[0x01 * 3 + 0] & CLUT_MASK_B) CLUT_SHIFT_B);
 		for( count = 0; count < saveLen / 2; count++)
 		    buf16[count] = pdata16;
 		break;
@@ -2043,17 +2139,19 @@ vc_progress_set( boolean_t enable, uint32_t delay )
             vc_needsave      = TRUE;
             vc_saveunder     = saveBuf;
             vc_saveunder_len = saveLen;
-            saveBuf	     = 0;
+            saveBuf	     = NULL;
             saveLen 	     = 0;
 
-            clock_interval_to_deadline(delay, 1000 * 1000 * 1000 /*second scale*/, &vc_progress_deadline);
+            clock_interval_to_deadline(vc_delay,
+				       1000 * 1000 * 1000 /*second scale*/,
+				       &vc_progress_deadline);
             thread_call_enter_delayed(&vc_progress_call, vc_progress_deadline);
 
         } else {
             if( vc_saveunder) {
                 saveBuf      = vc_saveunder;
                 saveLen      = vc_saveunder_len;
-                vc_saveunder = 0;
+                vc_saveunder = NULL;
                 vc_saveunder_len = 0;
             }
 
@@ -2068,7 +2166,8 @@ vc_progress_set( boolean_t enable, uint32_t delay )
         kfree( saveBuf, saveLen );
 }
 
-static void vc_progress_task( void * arg0, void * arg )
+static void
+vc_progress_task(__unused void *arg0, void *arg)
 {
     spl_t		s;
     int			count = (int) arg;
@@ -2118,6 +2217,7 @@ static void vc_progress_task( void * arg0, void * arg )
 
 static boolean_t gc_acquired      = FALSE;
 static boolean_t gc_graphics_boot = FALSE;
+static boolean_t gc_desire_text   = FALSE;
 
 static unsigned int lastVideoPhys   = 0;
 static unsigned int lastVideoVirt   = 0;
@@ -2125,52 +2225,55 @@ static unsigned int lastVideoSize   = 0;
 static boolean_t    lastVideoMapped = FALSE;
 
 void
-initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
+initialize_screen(PE_Video * boot_vinfo, unsigned int op)
 {
-	unsigned int fbsize;
-	unsigned int newVideoVirt;
+	unsigned int fbsize = 0;
+	unsigned int newVideoVirt = 0;
+	boolean_t graphics_now;
 	ppnum_t fbppage;
 
 	if ( boot_vinfo )
 	{
+		struct vc_info new_vinfo = vinfo;
+
 //        	bcopy((const void *)boot_vinfo, (void *)&boot_video_info, sizeof(boot_video_info));
 
 		/* 
 		 *	First, check if we are changing the size and/or location of the framebuffer
 		 */
-		vinfo.v_name[0]  = 0;
-		vinfo.v_width    = boot_vinfo->v_width;
-		vinfo.v_height   = boot_vinfo->v_height;
-		vinfo.v_depth    = boot_vinfo->v_depth;
-		vinfo.v_rowbytes = boot_vinfo->v_rowBytes;
-		vinfo.v_physaddr = boot_vinfo->v_baseAddr;		/* Get the physical address */
+		new_vinfo.v_name[0]  = 0;
+		new_vinfo.v_width    = boot_vinfo->v_width;
+		new_vinfo.v_height   = boot_vinfo->v_height;
+		new_vinfo.v_depth    = boot_vinfo->v_depth;
+		new_vinfo.v_rowbytes = boot_vinfo->v_rowBytes;
+		new_vinfo.v_physaddr = boot_vinfo->v_baseAddr;		/* Get the physical address */
 #ifdef __i386__
-                vinfo.v_type     = boot_vinfo->v_display;
+                new_vinfo.v_type     = boot_vinfo->v_display;
 #else
-                vinfo.v_type = 0;
+                new_vinfo.v_type = 0;
 #endif
-    
- 
-                kprintf("initialize_screen: b=%08X, w=%08X, h=%08X, r=%08X, d=%08X\n",                  /* (BRINGUP) */
-                        vinfo.v_physaddr, vinfo.v_width,  vinfo.v_height,  vinfo.v_rowbytes, vinfo.v_type);     /* (BRINGUP) */
+     
+		if (!lastVideoMapped)
+		    kprintf("initialize_screen: b=%08lX, w=%08lX, h=%08lX, r=%08lX, d=%08lX\n",                  /* (BRINGUP) */
+			    new_vinfo.v_physaddr, new_vinfo.v_width,  new_vinfo.v_height,  new_vinfo.v_rowbytes, new_vinfo.v_type);     /* (BRINGUP) */
 
 #ifdef __i386__
-                if ( (vinfo.v_type == VGA_TEXT_MODE) )
+                if ( (new_vinfo.v_type == VGA_TEXT_MODE) )
                 {
-                    if (vinfo.v_physaddr == 0) {
-                        vinfo.v_physaddr = 0xb8000;
-                        vinfo.v_width = 80;
-                        vinfo.v_height = 25;
-                        vinfo.v_depth = 8;
-                        vinfo.v_rowbytes = 0x8000;
+                    if (new_vinfo.v_physaddr == 0) {
+                        new_vinfo.v_physaddr = 0xb8000;
+                        new_vinfo.v_width = 80;
+                        new_vinfo.v_height = 25;
+                        new_vinfo.v_depth = 8;
+                        new_vinfo.v_rowbytes = 0x8000;
                     }
                 }
 #endif /* __i386__ */
 
-		if (!vinfo.v_physaddr)							/* Check to see if we have a framebuffer */
+		if (!new_vinfo.v_physaddr)							/* Check to see if we have a framebuffer */
 		{
 			kprintf("initialize_screen: No video - forcing serial mode\n");		/* (BRINGUP) */
-			vinfo.v_depth = 0;						/* vc routines are nop */
+			new_vinfo.v_depth = 0;						/* vc routines are nop */
 			(void)switch_to_serial_console();				/* Switch into serial mode */
 			gc_graphics_boot = FALSE;					/* Say we are not in graphics mode */
 			disableConsoleOutput = FALSE;					/* Allow printfs to happen */
@@ -2187,46 +2290,69 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 			    fbppage = pmap_find_phys(kernel_pmap, (addr64_t)boot_vinfo->v_baseAddr);	/* Get the physical address of frame buffer */
 			    if(!fbppage)						/* Did we find it? */
 			    {
-				    panic("initialize_screen: Strange framebuffer - addr = %08X\n", boot_vinfo->v_baseAddr);
+				    panic("initialize_screen: Strange framebuffer - addr = %08X\n", (uint32_t)boot_vinfo->v_baseAddr);
 			    }
-			    vinfo.v_physaddr = (fbppage << 12) | (boot_vinfo->v_baseAddr & PAGE_MASK);			/* Get the physical address */
+			    new_vinfo.v_physaddr = (fbppage << 12) | (boot_vinfo->v_baseAddr & PAGE_MASK);			/* Get the physical address */
 		    }
     
-		    fbsize = round_page_32(vinfo.v_height * vinfo.v_rowbytes);			/* Remember size */
+		    if (boot_vinfo->v_length != 0)
+			    fbsize = round_page_32(boot_vinfo->v_length);
+		    else
+			    fbsize = round_page_32(new_vinfo.v_height * new_vinfo.v_rowbytes);			/* Remember size */
+
     
-		    if ((lastVideoPhys != vinfo.v_physaddr) || (fbsize > lastVideoSize))		/* Did framebuffer change location or get bigger? */
+		    if ((lastVideoPhys != new_vinfo.v_physaddr) || (fbsize > lastVideoSize))		/* Did framebuffer change location or get bigger? */
 		    {
 			    unsigned int
 #if FALSE
-			    flags = (vinfo.v_type == VGA_TEXT_MODE) ? VM_WIMG_IO : VM_WIMG_WCOMB;
+			    flags = (new_vinfo.v_type == VGA_TEXT_MODE) ? VM_WIMG_IO : VM_WIMG_WCOMB;
 #else
 			    flags = VM_WIMG_IO;
 #endif
-			    newVideoVirt = io_map_spec((vm_offset_t)vinfo.v_physaddr, fbsize, flags);	/* Allocate address space for framebuffer */
-    
-			    if (lastVideoVirt)							/* Was the framebuffer mapped before? */
-			    {
+			    newVideoVirt = io_map_spec((vm_offset_t)new_vinfo.v_physaddr, fbsize, flags);	/* Allocate address space for framebuffer */
+    		    }
+		}
+
+		if (newVideoVirt != 0)
+		    new_vinfo.v_baseaddr = newVideoVirt + boot_vinfo->v_offset;				/* Set the new framebuffer address */
+		else
+		    new_vinfo.v_baseaddr = lastVideoVirt + boot_vinfo->v_offset;				/* Set the new framebuffer address */
+	
+		/* Update the vinfo structure atomically with respect to the vc_progress task if running */
+		if (vc_progress)
+		{
+		    simple_lock(&vc_progress_lock);
+		    vinfo = new_vinfo;
+		    simple_unlock(&vc_progress_lock);
+		}
+		else
+		{
+		    vinfo = new_vinfo;
+		}
+
+		// If we changed the virtual address, remove the old mapping
+		if (newVideoVirt != 0)
+		{
+			if (lastVideoVirt)							/* Was the framebuffer mapped before? */
+			{
 #if FALSE
-                                    if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
+				if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
 #endif
-				    {
-					    pmap_remove(kernel_pmap, trunc_page_64(lastVideoVirt),
-						    round_page_64(lastVideoVirt + lastVideoSize));	/* Toss mappings */
-				    }
-                                    if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
-				    {
-					    kmem_free(kernel_map, lastVideoVirt, lastVideoSize);	/* Toss kernel addresses */
-				    }
-			    }
-			    lastVideoPhys = vinfo.v_physaddr;					/* Remember the framebuffer address */
-			    lastVideoSize = fbsize;							/* Remember the size */
-			    lastVideoVirt = newVideoVirt;						/* Remember the virtual framebuffer address */
-			    lastVideoMapped  = (NULL != kernel_map);
-		    }
+				{
+					pmap_remove(kernel_pmap, trunc_page_64(lastVideoVirt),
+						round_page_64(lastVideoVirt + lastVideoSize));	/* Toss mappings */
+				}
+				if(lastVideoMapped)                            /* Was this not a special pre-VM mapping? */
+				{
+					kmem_free(kernel_map, lastVideoVirt, lastVideoSize);	/* Toss kernel addresses */
+				}
+			}
+			lastVideoPhys = new_vinfo.v_physaddr;					/* Remember the framebuffer address */
+			lastVideoSize = fbsize;							/* Remember the size */
+			lastVideoVirt = newVideoVirt;						/* Remember the virtual framebuffer address */
+			lastVideoMapped  = (NULL != kernel_map);
 		}
 
-		vinfo.v_baseaddr = lastVideoVirt;				/* Set the new framebuffer address */
-
 #ifdef __i386__
 		if ( (vinfo.v_type == VGA_TEXT_MODE) )
 		{
@@ -2270,6 +2396,7 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 		case kPEGraphicsMode:
 			panicDialogDesired = TRUE;
 			gc_graphics_boot = TRUE;
+			gc_desire_text = FALSE;
 			break;
 
 		case kPETextMode:
@@ -2279,10 +2406,11 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 
 		case kPEAcquireScreen:
 			if ( gc_acquired ) break;
-
-			vc_progress_set( gc_graphics_boot, kProgressAcquireDelay );
-			gc_enable( !gc_graphics_boot );
+			graphics_now = gc_graphics_boot && !gc_desire_text;
+			vc_progress_set( graphics_now, kProgressAcquireDelay );
+			gc_enable( !graphics_now );
 			gc_acquired = TRUE;
+			gc_desire_text = FALSE;
 			break;
 
 		case kPEEnableScreen:
@@ -2290,8 +2418,14 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 			break;
 
 		case kPETextScreen:
+			if ( console_is_serial() ) break;
+
 			panicDialogDesired = FALSE;
-			if ( gc_acquired      == FALSE ) break;
+			if ( gc_acquired == FALSE )
+			{
+				gc_desire_text = TRUE;
+				break;
+			}
 			if ( gc_graphics_boot == FALSE ) break;
 
 			vc_progress_set( FALSE, 0 );
@@ -2304,6 +2438,7 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 
 		case kPEReleaseScreen:
 			gc_acquired = FALSE;
+			gc_desire_text = FALSE;
 			gc_enable( FALSE );
 			vc_progress_set( FALSE, 0 );
 
@@ -2318,33 +2453,23 @@ initialize_screen(Boot_Video * boot_vinfo, unsigned int op)
 #endif /* GRATEFULDEBUGGER */
 }
 
-void
-refresh_screen(void)
-{
-	if ( gc_enabled )
-	{
-		gc_refresh_screen();
-		gc_show_cursor(gc_x, gc_y);
-	}
-}
+void vcattach(void); /* XXX gcc 4 warning cleanup */
 
 void
 vcattach(void)
 {
-	extern struct { long msg_magic; long msg_bufx; long msg_bufr; char msg_bufc[]; } * msgbufp;
-
 	vm_initialized = TRUE;
 
 	if ( gc_graphics_boot == FALSE )
 	{
-		unsigned int index;
+		long index;
 
 		if ( gc_acquired )
 		{
-			initialize_screen( 0, kPEReleaseScreen );
+			initialize_screen(NULL, kPEReleaseScreen);
 		}
 
-		initialize_screen( 0, kPEAcquireScreen );
+		initialize_screen(NULL, kPEAcquireScreen);
 
 		for ( index = 0 ; index < msgbufp->msg_bufx ; index++ )
 		{
diff --git a/osfmk/console/video_console.h b/osfmk/console/video_console.h
index 125300533..1851b8756 100644
--- a/osfmk/console/video_console.h
+++ b/osfmk/console/video_console.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -31,9 +37,7 @@
 
 #include <device/device_types.h>
 
-int vcputc(	int		l,
-		int		u,
-		int		c );
+void vcputc(int, int, int);
 
 int vcgetc(	int		l,
 		int		u,
diff --git a/osfmk/ddb/db_access.c b/osfmk/ddb/db_access.c
index 7556e8357..fb0512bac 100644
--- a/osfmk/ddb/db_access.c
+++ b/osfmk/ddb/db_access.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_access.h b/osfmk/ddb/db_access.h
index 89e92c114..518a1b648 100644
--- a/osfmk/ddb/db_access.h
+++ b/osfmk/ddb/db_access.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_aout.c b/osfmk/ddb/db_aout.c
index 08c781adc..b55857be5 100644
--- a/osfmk/ddb/db_aout.c
+++ b/osfmk/ddb/db_aout.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -127,37 +133,37 @@ aout_db_compare_symbols(
 
 int db_sorting_limit = 50000;
 
+extern boolean_t getsymtab(char *, vm_offset_t *, int *, vm_offset_t *,
+		vm_size_t *);
+
 boolean_t
 aout_db_sym_init(
 	char *	symtab,		/* pointer to start of symbol table */
-	char *	esymtab,	/* pointer to end of string table,
+	__unused char *esymtab,	/* pointer to end of string table,
 				   for checking - may be rounded up to
 				   integer boundary */
-	char *	name,
+	const char *name,
 	char *	task_addr)	/* use for this task only */
 {
 	struct nlist	*sym_start, *sym_end, *dbsym_start, *dbsym_end;
 	struct nlist	*sp;
 	char *strtab, *dbstrtab;
-	int	strlen;
+	int	db_strlen;
 	char *estrtab, *dbestrtab;
 	unsigned long	minsym = ~0;
 	unsigned long	maxsym = 0;
 	boolean_t	sorted;
 	boolean_t	sorting;
-	extern boolean_t getsymtab(char *, 
-		vm_offset_t *, int *,
-        	vm_offset_t *,  vm_size_t *);
 	int nsyms;
 
 
 	if (!getsymtab(symtab, 
 		(vm_offset_t *)&sym_start, &nsyms, 
-		(vm_offset_t *)&strtab, (vm_size_t *)&strlen)) {
+		(vm_offset_t *)&strtab, (vm_size_t *)&db_strlen)) {
 		return(FALSE);
 	}
 	sym_end = sym_start + nsyms;
-	estrtab = strtab + strlen;
+	estrtab = strtab + db_strlen;
 	
 /*
  *	We haven't actually started up VM yet, so we can just steal some pages to 
@@ -171,7 +177,7 @@ aout_db_sym_init(
 	bcopy(strtab, dbstrtab, (unsigned int)estrtab - (unsigned int)strtab);	/* Copy strings */
 
 	dbsym_end = dbsym_start + nsyms;
-	dbestrtab = dbstrtab + strlen;
+	dbestrtab = dbstrtab + db_strlen;
 
 	sorting = ((dbsym_end - dbsym_start) < db_sorting_limit);
 	
@@ -179,7 +185,7 @@ aout_db_sym_init(
 	    register long strx;
 	    strx = sp->n_un.n_strx;
 	    if (strx != 0) {
-		if (strx > strlen) {
+		if (strx > db_strlen) {
 		    sp->n_un.n_name = 0;
 		    continue;
 		}
@@ -198,7 +204,7 @@ aout_db_sym_init(
 
 	if (sorting) {
 		db_qsort((char *) dbsym_start, dbsym_end - dbsym_start,
-		         sizeof (struct nlist), aout_db_order_symbols);
+		         sizeof(struct nlist), aout_db_order_symbols);
 		sorted = TRUE;
 	} else
 		sorted = FALSE;
@@ -363,7 +369,7 @@ aout_db_qualified_print_completion(
 	struct nlist	*sp;
 	struct nlist	*sp1;
 	struct nlist	*ep;
-	struct nlist	*ep1;
+	struct nlist	*ep1 = NULL;
 	struct nlist	*fp = 0;
 	int		symlen;
 	int		nsym = 0;
@@ -398,8 +404,9 @@ aout_db_qualified_print_completion(
 		if (cur->n_other) {
 		   if (sp1 == sp)
 			sp1 = cur;
-		   if (strcmp(&cur->n_un.n_name[cur->n_other - 1],
-			      &new->n_un.n_name[new->n_other - 1]) < 0)
+		   if (strncmp(&cur->n_un.n_name[cur->n_other - 1],
+			      &new->n_un.n_name[new->n_other - 1],
+			      symlen) < 0)
 			new = cur;
 		   else
 			ep1 = cur;
@@ -676,8 +683,8 @@ aout_db_search_symbol(
 	    target.n_value = off;
 	    target.n_un.n_name = (char *) 0;
 	    target.n_other = (char) 0;
-	    db_qsort_limit_search((char *) &target, (char **) &sp, (char **) &ep,
-			          sizeof (struct nlist), aout_db_compare_symbols);
+	    db_qsort_limit_search((char *)&target, (char **)&sp, (char **)&ep,
+			          sizeof(struct nlist), aout_db_compare_symbols);
 	    first_pass = TRUE;
 	}
 
@@ -738,7 +745,7 @@ aout_db_symbol_values(
 }
 
 #define X_DB_MAX_DIFF	8	/* maximum allowable diff at the end of line */
-extern int 	db_search_maxoff;	/* maximum acceptable offset */
+extern unsigned int db_search_maxoff;	/* maximum acceptable offset */
 
 /*
  * search symbol by value
@@ -779,8 +786,8 @@ aout_db_search_by_addr(
 		target.n_value = addr;
 		target.n_un.n_name = (char *) 0;
 		target.n_other = (char) 0;
-		db_qsort_limit_search((char *) &target, (char **) &sp,
-				      (char **) &ep, sizeof (struct nlist),
+		db_qsort_limit_search((char *)&target, (char **)&sp,
+				      (char **)&ep, sizeof(struct nlist),
 				      aout_db_compare_symbols);
 		first_pass = TRUE;
 	}
@@ -926,7 +933,7 @@ aout_db_search_by_addr(
 boolean_t
 aout_db_line_at_pc(
 	db_symtab_t	*stab,
-	db_sym_t	sym,
+	__unused db_sym_t	sym,
 	char		**file,
 	int		*line,
 	db_expr_t	pc)
@@ -942,14 +949,13 @@ aout_db_line_at_pc(
 	return(found && func && *file);
 }
 
+extern struct mach_header _mh_execute_header;
 /*
  * Initialization routine for a.out files.
  */
 void
 aout_db_init(void)
 {
-	extern struct mach_header _mh_execute_header;
-
 	aout_db_sym_init((char *) &_mh_execute_header,
 		(char *)0, "mach", (char *)0);
 }
diff --git a/osfmk/ddb/db_aout.h b/osfmk/ddb/db_aout.h
index 28b9f4255..3a1ab3836 100644
--- a/osfmk/ddb/db_aout.h
+++ b/osfmk/ddb/db_aout.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -32,11 +38,7 @@
 #include <machine/db_machdep.h>		/* data types */
 #include <ddb/db_sym.h>			/* db_symtab_t */
 
-boolean_t aout_db_sym_init(
-	char *	symtab,
-	char *	esymtab,
-	char *	name,
-	char *	task_addr);
+boolean_t aout_db_sym_init(char *, char *, const char *, char *);
 
 db_sym_t aout_db_lookup(
 	db_symtab_t	*stab,
diff --git a/osfmk/ddb/db_break.c b/osfmk/ddb/db_break.c
index f4c841f17..91267047b 100644
--- a/osfmk/ddb/db_break.c
+++ b/osfmk/ddb/db_break.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -44,7 +50,7 @@
 #include <ddb/db_expr.h>
 #include <ddb/db_output.h>		/* For db_printf() */
 #include <ddb/db_task_thread.h>
-
+#include <kern/thread.h>
 
 #define	NBREAKPOINTS	100
 #define NTHREAD_LIST	(NBREAKPOINTS*3)
@@ -221,7 +227,7 @@ db_find_thread_breakpoint_here(
 	bkpt = db_find_breakpoint(task, (db_addr_t)addr);
 	if (bkpt == 0)
 	    return(0);
-	return(db_find_thread_breakpoint(bkpt, current_act()));
+	return(db_find_thread_breakpoint(bkpt, current_thread()));
 }
 
 db_thread_breakpoint_t
@@ -436,7 +442,7 @@ db_set_breakpoints(void)
 	register db_breakpoint_t bkpt;
 	register task_t	task;
 	db_expr_t	inst;
-	thread_t	cur_act = current_act();
+	thread_t	cur_act = current_thread();
 	task_t		cur_task =
 				(cur_act) ?
 					cur_act->task : TASK_NULL;
@@ -479,7 +485,7 @@ db_clear_breakpoints(void)
 	register db_breakpoint_t bkpt, *bkptp;
 	register task_t	 task;
 	db_expr_t inst;
-	thread_t	 cur_act = current_act();
+	thread_t	 cur_act = current_thread();
 	task_t	 cur_task = (cur_act) ?
 			cur_act->task: TASK_NULL;
 
@@ -694,7 +700,7 @@ db_delete_cmd(void)
 
 	if (t == tHASH) {
 	    db_thread_breakpoint_t tbp;
-	    db_breakpoint_t bkpt;
+	    db_breakpoint_t bkpt = 0;
 
 	    if (db_read_token() != tNUMBER) {
 		db_printf("Bad break point number #%s\n", db_tok_string);
@@ -745,11 +751,8 @@ db_delete_cmd(void)
 #include <mach/machine/vm_param.h>
 
 void
-db_breakpoint_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_breakpoint_cmd(db_expr_t addr, __unused boolean_t have_addr, db_expr_t count,
+		  char *modif)
 {
 	register int n;
 	thread_t thr_act;
@@ -757,7 +760,7 @@ db_breakpoint_cmd(
 	boolean_t task_bpt = db_option(modif, 'T');
 	boolean_t user_space;
 
-	if (count == -1)
+	if (count == (uint64_t)-1)
 	    count = 1;
 #if 0 /* CHECKME */
 	if (!task_bpt && db_option(modif,'t'))
@@ -806,7 +809,8 @@ db_breakpoint_cmd(
 
 /* list breakpoints */
 void
-db_listbreak_cmd(void)
+db_listbreak_cmd(__unused db_expr_t addr, __unused boolean_t have_addr,
+		 __unused db_expr_t count, __unused char *modif)
 {
 	db_list_breakpoints();
 }
diff --git a/osfmk/ddb/db_break.h b/osfmk/ddb/db_break.h
index 1f3552348..f456b2517 100644
--- a/osfmk/ddb/db_break.h
+++ b/osfmk/ddb/db_break.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -229,10 +235,10 @@ void db_delete_cmd(void);
 
 void db_breakpoint_cmd(
 	db_expr_t	addr,
-	int		have_addr,
+	boolean_t	have_addr,
 	db_expr_t	count,
 	char *		modif);
 
-void db_listbreak_cmd(void);
+void db_listbreak_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
 #endif	/* !_DDB_DB_BREAK_H_ */
diff --git a/osfmk/ddb/db_coff.h b/osfmk/ddb/db_coff.h
index b0cad757e..57f3b7ddb 100644
--- a/osfmk/ddb/db_coff.h
+++ b/osfmk/ddb/db_coff.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -52,7 +58,7 @@
 boolean_t coff_db_sym_init(
 	char *	symtab,
 	char *	esymtab,
-	char *	name,
+	const char *	name,
 	char *	task_addr);
 
 db_sym_t coff_db_lookup(
diff --git a/osfmk/ddb/db_command.c b/osfmk/ddb/db_command.c
index 03fe35f3b..f1962e8d1 100644
--- a/osfmk/ddb/db_command.c
+++ b/osfmk/ddb/db_command.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -80,7 +86,9 @@
 #include <ddb/db_cond.h>
 #include <ddb/db_examine.h>
 #include <ddb/db_expr.h>
+#if defined(__ppc__)
 #include <ppc/db_low_trace.h>
+#endif
 #include <ddb/db_macro.h>
 #include <ddb/db_print.h>
 #include <ddb/db_run.h>
@@ -142,8 +150,6 @@ void db_command(
 
 void db_help_cmd(void);
 
-void db_output_prompt(void);
-
 void db_fncall(void);
 
 void db_cmd_list(struct db_command *table);
@@ -173,7 +179,7 @@ db_cmd_search(
 
 	for (cmd = table; cmd->name != 0; cmd++) {
 	    register char *lp;
-	    register char *rp;
+	    const char *rp;
 	    register int  c;
 
 	    lp = name;
@@ -212,9 +218,9 @@ db_cmd_search(
 void
 db_cmd_list(struct db_command *table)
 {
-	register struct db_command *new;
-	register struct db_command *old;
-	register struct db_command *cur;
+	struct db_command *new;
+	struct db_command *old;
+	struct db_command *cur;
 	unsigned int l;
 	unsigned int len;
 
@@ -228,9 +234,9 @@ db_cmd_list(struct db_command *table)
 	    new = (struct db_command *)0;
 	    for (cur = table; cur->name != 0; cur++)
 		if ((new == (struct db_command *)0 ||
-		     strcmp(cur->name, new->name) < 0) &&
+		     strncmp(cur->name, new->name, strlen(cur->name)) < 0) &&
 		    (old == (struct db_command *)0 ||
-		     strcmp(cur->name, old->name) > 0))
+		     strncmp(cur->name, old->name, strlen(cur->name)) > 0))
 		    new = cur;
 	    if (new == (struct db_command *)0)
 		    return;
@@ -252,7 +258,7 @@ db_command(
 	char		modif[TOK_STRING_SIZE];
 	char		*modifp = &modif[0];
 	db_expr_t	addr, count;
-	boolean_t	have_addr;
+	boolean_t	have_addr = FALSE;
 	int		result;
 
 	t = db_read_token();
@@ -324,7 +330,7 @@ db_command(
 			db_flush_lex();
 			return;
 		    }
-		    strcpy(modif, db_tok_string);
+		    strlcpy(modif, db_tok_string, TOK_STRING_SIZE);
 		}
 		else {
 		    db_unread_token(t);
@@ -382,7 +388,7 @@ db_command(
 	}
 	*last_cmdp = cmd;
 	*last_countp = count;
-	strcpy(last_modifp, modifp);
+	strlcpy(last_modifp, modifp, TOK_STRING_SIZE);
 }
 
 void
@@ -402,138 +408,437 @@ db_command_list(
 extern void	db_system_stats(void);
 
 struct db_command db_show_all_cmds[] = {
-	{ "acts",	db_show_all_acts,			0,	0 },
-	{ "spaces",	db_show_all_spaces,			0,	0 },
-	{ "tasks",	db_show_all_acts,			0,	0 },
+	{
+		.name = "acts",
+		.fcn = db_show_all_acts,
+	},
+	{
+		.name = "spaces",
+		.fcn = db_show_all_spaces,
+	},
+	{
+		.name = "tasks",
+		.fcn = db_show_all_acts,
+	},
 	/* temporary alias for sanity preservation */
-	{ "threads",	db_show_all_acts,			0,	0 },
-	{ "zones",	db_show_all_zones,			0,	0 },
-	{ "vmtask",	db_show_all_task_vm,			0,	0 },
-	{ (char *)0 }
+	{
+		.name ="threads",
+		db_show_all_acts,
+	},
+	{
+		.name = "zones",
+		.fcn = db_show_all_zones,
+	},
+	{
+		.name = "vmtask",
+		.fcn = db_show_all_task_vm,
+	},
+	{
+		.name = (const char *)NULL,
+       	},
 };
 
 /* XXX */
 
 extern void		db_show_thread_log(void);
-extern void		db_show_one_lock(lock_t*);
 extern void		db_show_etap_log(db_expr_t, int, db_expr_t, char *);
 
 struct db_command db_show_cmds[] = {
-	{ "all",	0,				0,	db_show_all_cmds },
-	{ "registers",	db_show_regs,			0,	0 },
-	{ "variables",	(db_func) db_show_variable,	CS_OWN,	0 },
-	{ "breaks",	(db_func) db_listbreak_cmd,	0,	0 },
-	{ "watches",	(db_func) db_listwatch_cmd,	0,	0 },
-	{ "task",	db_show_one_task,		0,	0 },
-	{ "act",	db_show_one_act,		0,	0 },
-	{ "shuttle",	db_show_shuttle,		0,	0 },
+	{
+		.name = "all",
+		.more = db_show_all_cmds
+	},
+	{
+		.name = "registers",
+		.fcn = db_show_regs,
+	},
+	{
+		.name = "variables",
+		.fcn = db_show_variable,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "breaks",
+		.fcn = db_listbreak_cmd,
+	},
+	{
+		.name = "watches",
+		.fcn = db_listwatch_cmd,
+	},
+	{
+		.name = "task",
+		.fcn = db_show_one_task,
+	},
+	{
+		.name = "act",
+		.fcn = db_show_one_act,
+	},
+	{
+		.name = "shuttle",
+		.fcn = db_show_shuttle,
+	},
 #if 0
-	{ "thread",	db_show_one_thread,		0,	0 },
+	{
+		.name = "thread",
+		.fcn = db_show_one_thread,
+	},
 #endif
-	{ "vmtask",	db_show_one_task_vm,		0,	0 },
-	{ "macro",	(db_func) db_show_macro,	CS_OWN, 0 },
-	{ "runq",	(db_func) db_show_runq,		0,	0 },
-	{ "map",	(db_func) vm_map_print,		0,	0 },
-	{ "object",	(db_func) vm_object_print,	0,	0 },
-	{ "page",	(db_func) vm_page_print,	0,	0 },
-	{ "copy",	(db_func) vm_map_copy_print,	0,	0 },
-	{ "port",	(db_func) ipc_port_print,	0,	0 },
-	{ "pset",	(db_func) ipc_pset_print,	0,	0 },
-	{ "kmsg",	(db_func) ipc_kmsg_print,	0,	0 },
-	{ "msg",	(db_func) ipc_msg_print,	0,	0 },
-	{ "ipc_port",	db_show_port_id,		0,	0 },
-#if	NORMA_VM
-	{ "xmm_obj",	(db_func) xmm_obj_print,	0,	0 },
-	{ "xmm_reply",	(db_func) xmm_reply_print,	0,	0 },
+	{
+		.name = "vmtask",
+		.fcn = db_show_one_task_vm,
+	},
+	{
+		.name = "macro",
+		.fcn = (db_func)db_show_macro,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "runq",
+		.fcn = (db_func)db_show_runq,
+	},
+	{
+		.name = "map",
+		.fcn = (db_func)vm_map_print,
+	},
+	{
+		.name = "object",
+		.fcn = vm_object_print,
+	},
+	{
+		.name = "page",
+		.fcn = (db_func)vm_page_print,
+	},
+	{
+		.name = "copy",
+		.fcn = (db_func)vm_map_copy_print,
+	},
+	{
+		.name = "port",
+		.fcn = (db_func)ipc_port_print,
+	},
+	{
+		.name = "pset",
+		.fcn = (db_func)ipc_pset_print,
+	},
+	{
+		.name = "kmsg",
+		.fcn = (db_func)ipc_kmsg_print,
+	},
+	{
+		.name = "msg",
+		.fcn = (db_func)ipc_msg_print,
+	},
+	{
+		.name = "ipc_port",
+		.fcn = db_show_port_id,
+	},
+#if NORMA_VM
+	{
+		.name = "xmm_obj",
+		.fcn = (db_func)xmm_obj_print,
+	},
+	{
+		.name = "xmm_reply",
+		.fcn = (db_func)xmm_reply_print,
+	},
 #endif	/* NORMA_VM */
-#if	TRACE_BUFFER
-	{ "tr",		db_show_tr,			0,	0 },
+#if TRACE_BUFFER
+	{
+		.name = "tr",
+		.fcn = db_show_tr,
+	},
 #endif	/* TRACE_BUFFER */
-	{ "space",	db_show_one_space,		0,	0 },
-	{ "system",	(db_func) db_system_stats,	0,	0 },
-	{ "zone",	db_show_one_zone,		0,	0 },
-	{ "lock",	(db_func)db_show_one_lock,	0,	0 },
-	{ "mutex_lock",	(db_func)db_show_one_mutex,	0,	0 },
-	{ "simple_lock", (db_func)db_show_one_simple_lock,	0,	0 },
-	{ "thread_log", (db_func)db_show_thread_log,	0,	0 },
-	{ "shuttle",	db_show_shuttle,		0,	0 },
-	{ (char *)0, }
+	{
+		.name = "space",
+		.fcn = db_show_one_space,
+	},
+	{
+		.name = "system",
+		.fcn = (db_func)db_system_stats,
+	},
+	{
+		.name = "zone",
+		.fcn = db_show_one_zone,
+	},
+	{
+		.name = "lock",
+		.fcn = (db_func)db_show_one_lock,
+	},
+	{
+		.name = "mutex_lock",
+		.fcn = (db_func)db_show_one_mutex,
+	},
+	{
+		.name = "simple_lock",
+		.fcn = (db_func)db_show_one_simple_lock,
+	},
+	{
+		.name = "thread_log",
+		(db_func)db_show_thread_log,
+	},
+	{
+		.name = "shuttle",
+		.fcn = db_show_shuttle,
+	},
+	{
+		.name = (const char *)NULL,
+	},
 };
 
 #define	db_switch_cpu kdb_on
-extern void	db_switch_cpu(int);
 
 struct db_command db_command_table[] = {
 #if DB_MACHINE_COMMANDS
-
-/* this must be the first entry, if it exists */
-	{ "machine",	0,				0,			0 },
-#endif
-	{ "print",	(db_func) db_print_cmd,		CS_OWN,			0 },
-	{ "examine",	db_examine_cmd,			CS_MORE|CS_SET_DOT,	0 },
-	{ "x",		db_examine_cmd,			CS_MORE|CS_SET_DOT,	0 },
-	{ "xf",		db_examine_forward,		CS_SET_DOT,		0 },
-	{ "xb",		db_examine_backward,		CS_SET_DOT,		0 },
-	{ "search",	(db_func) db_search_cmd,	CS_OWN|CS_SET_DOT,	0 },
-	{ "set",	(db_func) db_set_cmd,		CS_OWN,			0 },
-	{ "write",	db_write_cmd,			CS_MORE|CS_SET_DOT,	0 },
-	{ "w",		db_write_cmd,			CS_MORE|CS_SET_DOT,	0 },
-	{ "delete",	(db_func) db_delete_cmd,	CS_OWN,			0 },
-	{ "d",		(db_func) db_delete_cmd,	CS_OWN,			0 },
-	{ "break",	db_breakpoint_cmd,		CS_MORE,		0 },
-	{ "dwatch",	db_deletewatch_cmd,		CS_MORE,		0 },
-	{ "watch",	db_watchpoint_cmd,		CS_MORE,		0 },
-	{ "step",	db_single_step_cmd,		0,			0 },
-	{ "s",		db_single_step_cmd,		0,			0 },
-	{ "continue",	db_continue_cmd,		0,			0 },
-	{ "c",		db_continue_cmd,		0,			0 },
-	{ "gdb",	db_continue_gdb,		0,			0 },
-	{ "until",	db_trace_until_call_cmd,	0,			0 },
+	/* this must be the first entry, if it exists */
+	{
+		.name = "machine",
+	},
+#endif /* DB_MACHINE_COMMANDS */
+	{
+		.name = "print",
+		.fcn = (db_func)db_print_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "examine",
+		.fcn = db_examine_cmd,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "x",
+		.fcn = db_examine_cmd,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "xf",
+		.fcn = db_examine_forward,
+		.flag = CS_SET_DOT,
+	},
+	{
+		.name = "xb",
+		.fcn = db_examine_backward,
+		.flag = CS_SET_DOT,
+	},
+	{
+		.name = "search",
+		.fcn = (db_func)db_search_cmd,
+		.flag = CS_OWN|CS_SET_DOT,
+	},
+	{
+		.name = "set",
+		.fcn = (db_func)db_set_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "write",
+		.fcn = db_write_cmd,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "w",
+		.fcn = db_write_cmd,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "delete",
+		.fcn = (db_func)db_delete_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "d",
+		.fcn = (db_func)db_delete_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "break",
+		.fcn = db_breakpoint_cmd,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "dwatch",
+		.fcn = db_deletewatch_cmd,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "watch",
+		.fcn = db_watchpoint_cmd,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "step",
+		.fcn = db_single_step_cmd,
+	},
+	{
+		.name = "s",
+		.fcn = db_single_step_cmd,
+	},
+	{
+		.name = "continue",
+		.fcn = db_continue_cmd,
+	},
+	{
+		.name = "c",
+		.fcn = db_continue_cmd,
+	},
+	{
+		.name = "gdb",
+		.fcn = db_continue_gdb,
+	},
+	{
+		.name = "until",
+		.fcn = db_trace_until_call_cmd,
+	},
 
 	/* As per request of DNoveck, CR1550, leave this disabled	*/
 #if 0	/* until CR1440 is fixed, to avoid toe-stubbing			*/
-	{ "next",	db_trace_until_matching_cmd,	0,			0 },
+	{
+		.name = "next",
+		.fcn = db_trace_until_matching_cmd,
+	},
 #endif
-	{ "match",	db_trace_until_matching_cmd,	0	,		0 },
-	{ "trace",	db_stack_trace_cmd,		0,			0 },
-	{ "cond",	(db_func) db_cond_cmd,		CS_OWN,			0 },
-	{ "call",	(db_func) db_fncall,		CS_OWN,			0 },
-	{ "macro",	(db_func) db_def_macro_cmd,	CS_OWN,			0 },
-	{ "dmacro",	(db_func) db_del_macro_cmd,	CS_OWN,			0 },
-	{ "show",	0,				0,			db_show_cmds },
-	{ "cpu",	(db_func) db_switch_cpu,	0,			0 },
-	{ "dr",		db_display_real,		CS_MORE|CS_SET_DOT,	0 },
-	{ "di",		db_display_iokit,		CS_MORE,			0 },
-	{ "dk",		db_display_kmod,		CS_MORE,			0 },
-
-	{ "reboot",	(db_func) db_reboot,		0,			0 },
+	{
+		.name = "match",
+		.fcn = db_trace_until_matching_cmd,
+	},
+	{
+		.name = "trace",
+		.fcn = db_stack_trace_cmd,
+	},
+	{
+		.name = "cond",
+		.fcn = (db_func)db_cond_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "call",
+		.fcn = (db_func)db_fncall,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "macro",
+		.fcn = (db_func)db_def_macro_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "dmacro",
+		.fcn = (db_func)db_del_macro_cmd,
+		.flag = CS_OWN,
+	},
+	{
+		.name = "show",
+		.more = db_show_cmds
+	},
+	{
+		.name = "cpu",
+		.fcn = (db_func)db_switch_cpu,
+	},
+	{
+		.name = "dr",
+		.fcn = db_display_real,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "di",
+		.fcn = db_display_iokit,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "dk",
+		.fcn = db_display_kmod,
+		.flag = CS_MORE,
+	},
+
+	{
+		.name = "reboot",
+		(db_func)db_reboot,
+	},
 #if !defined(__ppc__)	
-	{ "pm",		db_pmgr,				CS_MORE,			0 },
-	{ "na",		db_nap,					CS_MORE,			0 },
-	{ "ms",		db_msr,					CS_MORE,			0 },
-	{ "cp",		db_cpuid,				CS_MORE,			0 },
-	{ "da",		db_apic,				CS_MORE,			0 },
-	{ "ts",		db_test,				CS_MORE,			0 },
-	{ "dn",		db_intcnt,				CS_MORE,			0 },
-	{ "hp",		db_hpet,				CS_MORE,			0 },
-	{ "cf",		db_cfg,					CS_MORE,			0 },
-	{ "dt",		db_dtimers,				CS_MORE,			0 },
-#endif
+	{
+		.name = "ms",
+		.fcn = db_msr,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "cp",
+		.fcn = db_cpuid,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "da",
+		.fcn = db_apic,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "hp",
+		.fcn = db_hpet,
+		.flag = CS_MORE,
+	},
+#endif /* !__ppc__ */
 #if defined(__ppc__)	
-	{ "lt",		db_low_trace,			CS_MORE|CS_SET_DOT,	0 },
-	{ "dl",		db_display_long,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dc",		db_display_char,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dv",		db_display_virtual,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dm",		db_display_mappings,	CS_MORE|CS_SET_DOT,	0 },
-	{ "dh",		db_display_hash,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dp",		db_display_pmap,		CS_MORE,			0 },
-	{ "ds",		db_display_save,		CS_MORE|CS_SET_DOT,	0 },
-	{ "dx",		db_display_xregs,		CS_MORE|CS_SET_DOT,	0 },
-	{ "gs",		db_gsnoop,				CS_MORE,			0 },
-	{ "cm",		db_check_mappings,		CS_MORE,			0 },
-	{ "cp",		db_check_pmaps,			CS_MORE,			0 },
-#endif
-	{ (char *)0, }
+	{
+		.name = "lt",
+		.fcn = db_low_trace,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dl",
+		.fcn = db_display_long,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dc",
+		.fcn = db_display_char,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dv",
+		.fcn = db_display_virtual,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dm",
+		.fcn = db_display_mappings,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dh",
+		.fcn = db_display_hash,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dp",
+		.fcn = db_display_pmap,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "ds",
+		.fcn = db_display_save,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "dx",
+		.fcn = db_display_xregs,
+		.flag = CS_MORE|CS_SET_DOT,
+	},
+	{
+		.name = "gs",
+		.fcn = db_gsnoop,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "cm",
+		.fcn = db_check_mappings,
+		.flag = CS_MORE,
+	},
+	{
+		.name = "cp",
+		.fcn = db_check_pmaps,
+		.flag = CS_MORE,
+	},
+#endif /* __ppc__ */
+	{
+		.name = (const char *)NULL,
+	},
 };
 
 /* this function should be called to install the machine dependent
@@ -563,14 +868,14 @@ db_help_cmd(void)
 
 int	(*ddb_display)(void);
 
+extern int db_output_line;
+extern int db_macro_level;
+
 void
 db_command_loop(void)
 {
 	jmp_buf_t db_jmpbuf;
 	jmp_buf_t *prev = db_recover;
-	extern int db_output_line;
-	extern int db_macro_level;
-	extern int db_indent;
 
 	/*
 	 * Initialize 'prev' and 'next' to dot.
@@ -622,8 +927,6 @@ db_exec_cmd_nest(
 void
 db_error(const char *s)
 {
-	extern int db_macro_level;
-
 	db_macro_level = 0;
 	if (db_recover) {
 	    if (s > (char *)1)
@@ -661,7 +964,7 @@ db_fncall(void)
 	    db_flush_lex();
 	    return;
 	}
-	func = (uint32_t (*) (uint32_t, ...)) fn_addr;
+	func = (uint32_t (*) (uint32_t, ...))(unsigned long)fn_addr;
 
 	t = db_read_token();
 	if (t == tLPAREN) {
@@ -704,7 +1007,7 @@ db_option(
 	const char	*modif,
 	int		option)
 {
-	register char *p;
+	const char *p;
 
 	for (p = modif; *p; p++)
 	    if (*p == option)
diff --git a/osfmk/ddb/db_command.h b/osfmk/ddb/db_command.h
index 8c8443506..102dd97ea 100644
--- a/osfmk/ddb/db_command.h
+++ b/osfmk/ddb/db_command.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,13 +69,13 @@
 #include <machine/db_machdep.h>
 #include <db_machine_commands.h>
 
-typedef void	(*db_func)(db_expr_t, int, db_expr_t, char *);
+typedef void	(*db_func)(db_expr_t, boolean_t, db_expr_t, char *);
 
 /*
  * Command table
  */
 struct db_command {
-	char *	name;		/* command name */
+	const char *name;	/* command name */
 	db_func	fcn;		/* function to call */
 	int	flag;		/* extra info: */
 #define	CS_OWN		0x1	    /* non-standard syntax */
diff --git a/osfmk/ddb/db_cond.c b/osfmk/ddb/db_cond.c
index f5aa3af0c..e5199b914 100644
--- a/osfmk/ddb/db_cond.c
+++ b/osfmk/ddb/db_cond.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -141,6 +147,8 @@ db_cond_free(db_thread_breakpoint_t bkpt)
 	}
 }
 
+extern jmp_buf_t *db_recover;
+
 boolean_t
 db_cond_check(db_thread_breakpoint_t bkpt)
 {
@@ -148,7 +156,6 @@ db_cond_check(db_thread_breakpoint_t bkpt)
 	db_expr_t value;
 	int	  t;
 	jmp_buf_t db_jmpbuf;
-	extern 	  jmp_buf_t *db_recover;
 
 	if (bkpt->tb_cond <= 0) {		/* no condition */
 		if (--(bkpt->tb_count) > 0)
diff --git a/osfmk/ddb/db_cond.h b/osfmk/ddb/db_cond.h
index cb178648d..4e8c98a21 100644
--- a/osfmk/ddb/db_cond.h
+++ b/osfmk/ddb/db_cond.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_examine.c b/osfmk/ddb/db_examine.c
index 49404a8d1..7093604cf 100644
--- a/osfmk/ddb/db_examine.c
+++ b/osfmk/ddb/db_examine.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -53,7 +59,7 @@
  *	Author: David B. Golub, Carnegie Mellon University
  *	Date:	7/90
  */
-#include <string.h>			/* For strcpy() */
+#include <string.h>			/* For strlcpy() */
 #include <mach/boolean.h>
 #include <machine/db_machdep.h>
 
@@ -93,23 +99,21 @@ int db_examine_width(
 	int *items,
 	int *remainder);
 
+extern char db_last_modifier[];
+
 /*
  * Examine (print) data.
  */
 void
-db_examine_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_examine_cmd(db_expr_t addr, __unused boolean_t have_addr, db_expr_t count,
+	       char *modif)
 {
 	thread_t	thr_act;
-	extern char	db_last_modifier[];
 
 	if (modif[0] != '\0')
-	    strcpy(db_examine_format, modif);
+	    strlcpy(db_examine_format, modif, TOK_STRING_SIZE);
 
-	if (count == -1)
+	if (count == (db_expr_t)-1)
 	    count = 1;
 	db_examine_count = count;
 	if (db_option(modif, 't')) {
@@ -119,7 +123,7 @@ db_examine_cmd(
 		return;
 	} else
 	  if (db_option(modif,'u'))
-	    thr_act = current_act();
+	    thr_act = current_thread();
 	  else
 	    thr_act = THREAD_NULL;
 
@@ -129,22 +133,16 @@ db_examine_cmd(
 }
 
 void
-db_examine_forward(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_examine_forward(__unused db_expr_t addr, __unused boolean_t have_addr,
+		   __unused db_expr_t count, __unused char *modif)
 {
 	db_examine(db_next, db_examine_format, db_examine_count,
 				db_act_to_task(db_examine_act));
 }
 
 void
-db_examine_backward(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_examine_backward(__unused db_expr_t addr, __unused boolean_t have_addr,
+		    __unused db_expr_t count, __unused char *modif)
 {
 	db_examine(db_examine_prev_addr - (db_next - db_examine_prev_addr),
 			 db_examine_format, db_examine_count,
@@ -189,9 +187,9 @@ db_examine(
 	int		width;
 	int		leader;
 	int		items;
-	int		nitems;
+	int		nitems = 0;
 	char *		fp;
-	db_addr_t	next_addr;
+	db_addr_t	next_addr = 0;
 	int		sz;
 
 	db_examine_prev_addr = addr;
@@ -276,7 +274,7 @@ db_examine(
 				break;
 			    case 'r':	/* signed, current radix */
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
@@ -302,7 +300,7 @@ db_examine(
 			    case 'X':	/* unsigned hex */
 			    case 'x':	/* unsigned hex */
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
@@ -333,7 +331,7 @@ db_examine(
 				break;
 			    case 'z':	/* signed hex */
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
@@ -358,7 +356,7 @@ db_examine(
 				break;
 			    case 'd':	/* signed decimal */
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
@@ -384,7 +382,7 @@ db_examine(
 			    case 'U':	/* unsigned decimal */
 			    case 'u':
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
@@ -409,7 +407,7 @@ db_examine(
 				break;
 			    case 'o':	/* unsigned octal */
 				for (sz = size, next_addr = addr;
-				     sz >= sizeof (db_expr_t);
+				     sz >= (signed)sizeof (db_expr_t);
 				     sz -= sizeof (db_expr_t)) {
 				    if (nitems-- == 0) {
 					db_putchar('\n');
diff --git a/osfmk/ddb/db_examine.h b/osfmk/ddb/db_examine.h
index e59698ed6..08f026d2c 100644
--- a/osfmk/ddb/db_examine.h
+++ b/osfmk/ddb/db_examine.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -66,23 +72,11 @@
 /* Prototypes for functions exported by this module.
  */
 
-void db_examine_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_examine_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_examine_forward(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_examine_forward(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_examine_backward(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_examine_backward(db_expr_t, boolean_t, db_expr_t, char *);
 
 void db_examine(
 	db_addr_t	addr,
diff --git a/osfmk/ddb/db_expr.c b/osfmk/ddb/db_expr.c
index 46bf7fc09..d2b571f66 100644
--- a/osfmk/ddb/db_expr.c
+++ b/osfmk/ddb/db_expr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -161,7 +167,7 @@ db_term(db_expr_t *valuep)
 	    return (TRUE);
 	case tSTRING:
 	    {
-		    static db_tok_offset = 0;
+		    static int db_tok_offset = 0;
 		    char *sp, *cp;
 
 		    sp = (char *)db_tok_string + db_tok_offset;
@@ -223,7 +229,6 @@ db_unary(db_expr_t *valuep)
 	int	  size;
 	boolean_t u_opt, t_opt;
 	task_t	  task;
-	extern	  task_t db_default_task;
 
 	t = db_read_token();
 	if (t == tMINUS) {
@@ -358,7 +363,7 @@ db_shift_expr(db_expr_t *valuep)
 		db_error(0);
 		/*NOTREACHED*/
 	    }
-	    if (rhs < 0) {
+	    if ((int64_t)rhs < 0) {
 		db_error("Negative shift amount\n");
 		/*NOTREACHED*/
 	    }
diff --git a/osfmk/ddb/db_expr.h b/osfmk/ddb/db_expr.h
index a57dca83b..080e1a6c6 100644
--- a/osfmk/ddb/db_expr.h
+++ b/osfmk/ddb/db_expr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_ext_symtab.c b/osfmk/ddb/db_ext_symtab.c
index 58cba851d..4cb7c02ae 100644
--- a/osfmk/ddb/db_ext_symtab.c
+++ b/osfmk/ddb/db_ext_symtab.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_input.c b/osfmk/ddb/db_input.c
index 7283a4519..650f05bb6 100644
--- a/osfmk/ddb/db_input.c
+++ b/osfmk/ddb/db_input.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -234,9 +240,7 @@ int	db_hist_ignore_dups = 0;	/* don't duplicate commands in hist */
 
 /* Prototypes for functions local to this file.  XXX -- should be static!
  */
-void db_putstring(
-	char	*s,
-	int	count);
+void db_putstring(const char *s, int count);
 
 void db_putnchars(
 	int	c,
@@ -257,9 +261,7 @@ boolean_t db_inputchar(int c);
 extern jmp_buf_t	*db_recover;
 
 void
-db_putstring(
-	char	*s,
-	int	count)
+db_putstring(const char *s, int count)
 {
 	while (--count >= 0)
 	    cnputc(*s++);
@@ -367,18 +369,18 @@ db_inputchar(int c)
 	char *start;
 	char *restart;
 	jmp_buf_t db_jmpbuf;
-	jmp_buf_t *db_prev;
+	jmp_buf_t *local_prev;
 	char *p;
 	int len;
 
 	switch(db_completion) {
 	case -1:
 	    db_putchar('\n');
-	    db_prev = db_recover;
+	    local_prev = db_recover;
 	    if (_setjmp(db_recover = &db_jmpbuf) == 0 &&
 		(c == 'y' || c == ' ' || c == '\t'))
 		    db_print_completion(db_tok_string);
-	    db_recover = db_prev;
+	    db_recover = local_prev;
 	    db_completion = 0;
 	    db_reset_more();
 	    db_output_prompt();
@@ -449,7 +451,7 @@ db_inputchar(int c)
 		len = strlen(db_tok_string) - (start - restart);
 		if (db_completion == 1 &&
 		    (db_le == db_lc ||
-		     (db_le > db_lc) && *db_lc != ' '))
+		     ((db_le > db_lc) && *db_lc != ' ')))
 		    len++;
 		for (p = db_le - 1; p >= db_lc; p--)
 		    *(p + len) = *p;
@@ -464,7 +466,7 @@ db_inputchar(int c)
 			cnputc(*sym);
 		    if (db_completion == 1) {
 			if (db_le == db_lc ||
-			    (db_le > db_lc) && *db_lc != ' ') {
+			    ((db_le > db_lc) && *db_lc != ' ')) {
 			    cnputc(' ');
 			    *db_lc++ = ' ';
 			}
@@ -558,7 +560,6 @@ db_inputchar(int c)
 #if DB_HISTORY_SIZE != 0
 		db_history_curr = db_history_last;
 		if (c == CTRL('g') && db_hist_search) {
-			char *p;
 			for (p = db_hist_search_string, db_le = db_lbuf_start;
 			     *p; ) {
 				*db_le++ = *p++;
@@ -619,7 +620,6 @@ db_inputchar(int c)
 			cnputc('\007');
 			db_history_curr = old_history_curr;
 		} else {
-			register char *p;
 			INC_DB_CURR();
 			db_delete_line();
 			for (p = db_history_curr, db_le = db_lbuf_start;
@@ -671,7 +671,6 @@ db_inputchar(int c)
 		if (db_history_curr != db_history_last) {
 			INC_DB_CURR();
 			if (db_history_curr != db_history_last) {
-				register char *p;
 				db_delete_line();
 				for (p = db_history_curr,
 				     db_le = db_lbuf_start; *p;) {
@@ -709,7 +708,7 @@ db_inputchar(int c)
 #if DB_HISTORY_SIZE != 0
 		/* Check if it same than previous line */
 		if (db_history_prev) {
-			register char *pp, *pc;
+			char *pc;
 
 			/* Is it unmodified */
 			for (p = db_history_prev, pc = db_lbuf_start;
diff --git a/osfmk/ddb/db_input.h b/osfmk/ddb/db_input.h
index c1a89bffd..107bfcdc8 100644
--- a/osfmk/ddb/db_input.h
+++ b/osfmk/ddb/db_input.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_lex.c b/osfmk/ddb/db_lex.c
index efafadfab..d79305736 100644
--- a/osfmk/ddb/db_lex.c
+++ b/osfmk/ddb/db_lex.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -146,7 +152,7 @@
 /*
  * Lexical analyzer.
  */
-#include <string.h>			/* For strcpy(), strncmp(), strlen() */
+#include <string.h>			/* For strlcpy(), strlcmp(), strlen() */
 #include <ddb/db_lex.h>
 #include <ddb/db_command.h>
 #include <ddb/db_input.h>
@@ -154,8 +160,8 @@
 
 char	db_line[DB_LEX_LINE_SIZE];
 char	db_last_line[DB_LEX_LINE_SIZE];
-char	*db_lp, *db_endlp;
-char	*db_last_lp;
+const char *db_lp, *db_endlp;
+const char *db_last_lp;
 int	db_look_char = 0;
 db_expr_t db_look_token = 0;
 
@@ -167,7 +173,7 @@ void db_unread_char(int c);
 
 
 int
-db_read_line(char *repeat_last)
+db_read_line(const char *repeat_last)
 {
 	int	i;
 
@@ -176,11 +182,11 @@ db_read_line(char *repeat_last)
 	    return (0);	/* EOI */
 	if (repeat_last) {
 	    if (strncmp(db_line, repeat_last, strlen(repeat_last)) == 0) {
-		strcpy(db_line, db_last_line);
+		strlcpy(db_line, db_last_line, DB_LEX_LINE_SIZE);
 		db_printf("%s", db_line);
 		i = strlen(db_line);
 	    } else if (db_line[0] != '\n' && db_line[0] != 0)
-		strcpy(db_last_line, db_line);
+		strlcpy(db_last_line, db_line, DB_LEX_LINE_SIZE);
 	}
 	db_lp = db_line;
 	db_endlp = db_lp + i;
@@ -199,9 +205,7 @@ db_flush_line(void)
 }
 
 void
-db_switch_input(
-	char	*buffer,
-	int	size)
+db_switch_input(const char *buffer, int	size)
 {
 	db_lp = buffer;
 	db_last_lp = db_lp;
@@ -296,7 +300,7 @@ db_skip_to_eol(void)
 	register int skip;
 	register int t;
 	register int n;
-	register char *p;
+	const char *p;
 
 	t = db_read_token();
 	p = db_last_lp;
@@ -562,7 +566,7 @@ db_lex(void)
 	    case '?':
 		return (tQUESTION);
 	    case -1:
-		strcpy(db_tok_string, "<EOL>");
+		strlcpy(db_tok_string, "<EOL>", TOK_STRING_SIZE);
 		return (tEOF);
 	}
 	db_printf("Bad character '%c'\n", c);
diff --git a/osfmk/ddb/db_lex.h b/osfmk/ddb/db_lex.h
index 6280d3f7b..2b327246f 100644
--- a/osfmk/ddb/db_lex.h
+++ b/osfmk/ddb/db_lex.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -146,8 +152,8 @@
 struct db_lex_context {
 	int  l_char;		/* peek char */
 	int  l_token;		/* peek token */
-	char *l_ptr;		/* line pointer */
-	char *l_eptr;		/* line end pointer */
+	const char *l_ptr;	/* line pointer */
+	const char *l_eptr;	/* line end pointer */
 };
 
 extern db_expr_t db_tok_number;
@@ -191,11 +197,9 @@ extern db_expr_t db_radix;
 
 /* Prototypes for functions exported by this module.
  */
-int db_read_line(char *repeat_last);
+int db_read_line(const char *);
 
-void db_switch_input(
-	char	*buffer,
-	int	size);
+void db_switch_input(const char *, int);
 
 void db_save_lex_context(struct db_lex_context *lp);
 
diff --git a/osfmk/ddb/db_macro.c b/osfmk/ddb/db_macro.c
index 5365f9c6d..9e9e6cf98 100644
--- a/osfmk/ddb/db_macro.c
+++ b/osfmk/ddb/db_macro.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -50,7 +56,7 @@
 /*
  */
 #include <kern/thread.h>
-#include <string.h>			/* For strcmp(), strcpy() */
+#include <string.h>			/* For strncmp(), strlcpy() */
 
 #include <machine/db_machdep.h>
 #include <ddb/db_command.h>
@@ -91,7 +97,7 @@ db_lookup_macro(char *name)
 	for (mp = db_user_macro; mp < &db_user_macro[DB_NUSER_MACRO]; mp++) {
 	    if (mp->m_name[0] == 0)
 		continue;
-	    if (strcmp(mp->m_name, name) == 0)
+	    if (strncmp(mp->m_name, name, TOK_STRING_SIZE) == 0)
 		return(mp);
 	}
 	return(0);
@@ -119,7 +125,7 @@ db_def_macro_cmd(void)
 		db_error("ddb: internal error(macro)\n");
 		/* NOTREACHED */
 	    db_macro_free--;
-	    strcpy(mp->m_name, db_tok_string);
+	    strlcpy(mp->m_name, db_tok_string, TOK_STRING_SIZE);
 	}
 	for (c = db_read_char(); c == ' ' || c == '\t'; c = db_read_char());
 	for (p = mp->m_lbuf; c > 0; c = db_read_char())
@@ -131,7 +137,7 @@ db_def_macro_cmd(void)
 void
 db_del_macro_cmd(void)
 {
-	register struct db_user_macro *mp;
+	struct db_user_macro *mp = NULL;
 
 	if (db_read_token() != tIDENT 
 	    || (mp = db_lookup_macro(db_tok_string)) == 0) {
@@ -157,7 +163,7 @@ db_show_macro(void)
 	for (mp = db_user_macro; mp < &db_user_macro[DB_NUSER_MACRO]; mp++) {
 	    if (mp->m_name[0] == 0)
 		continue;
-	    if (name && strcmp(mp->m_name, name))
+	    if (name && strncmp(mp->m_name, name, TOK_STRING_SIZE))
 		continue;
 	    db_printf("%s: %s", mp->m_name, mp->m_lbuf);
 	}
@@ -189,11 +195,8 @@ db_exec_macro(char *name)
 }
 
 int
-db_arg_variable(
-	struct db_variable	*vp,
-	db_expr_t		*valuep,
-	int			flag,
-	db_var_aux_param_t	ap)
+db_arg_variable(__unused struct db_variable *vp, db_expr_t *valuep, int flag,
+		db_var_aux_param_t ap)
 {
 	db_expr_t value;
 	char *name;
diff --git a/osfmk/ddb/db_macro.h b/osfmk/ddb/db_macro.h
index ecd0dbed8..2a21e44de 100644
--- a/osfmk/ddb/db_macro.h
+++ b/osfmk/ddb/db_macro.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_output.c b/osfmk/ddb/db_output.c
index 9be9c4eaf..6e4b29a8b 100644
--- a/osfmk/ddb/db_output.c
+++ b/osfmk/ddb/db_output.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -141,7 +147,7 @@ db_reset_more()
 static void
 db_more(void)
 {
-	register  char *p;
+	const char *p;
 	boolean_t quit_output = FALSE;
 
 	for (p = "--db_more--"; *p; p++)
diff --git a/osfmk/ddb/db_output.h b/osfmk/ddb/db_output.h
index 38a834fee..39052640c 100644
--- a/osfmk/ddb/db_output.h
+++ b/osfmk/ddb/db_output.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_print.c b/osfmk/ddb/db_print.c
index 9bb1c9064..1a6fac39b 100644
--- a/osfmk/ddb/db_print.c
+++ b/osfmk/ddb/db_print.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -120,12 +126,6 @@ ipc_port_t db_lookup_port(
 	thread_t	thr_act,
 	int		id);
 
-static void db_print_port_id(
-	int		id,
-	ipc_port_t	port,
-	unsigned	bits,
-	int		n);
-
 void db_print_act(
 	thread_t 	thr_act,
 	int		act_id,
@@ -146,11 +146,8 @@ void db_system_stats(void);
 
 
 void
-db_show_regs(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		*modif)
+db_show_regs(db_expr_t addr, boolean_t have_addr, __unused db_expr_t count,
+	     char *modif)
 {
 	register struct db_variable *regp;
 	db_expr_t	value;
@@ -164,9 +161,9 @@ db_show_regs(
 	aux_param.thr_act = THREAD_NULL;
 	if (db_option(modif, 't')) {
 	    if (have_addr) {
-		if (!db_check_act_address_valid((thread_t)addr))
+		if (!db_check_act_address_valid((thread_t)(unsigned long)addr))
 		    return;
-		aux_param.thr_act = (thread_t)addr;
+		aux_param.thr_act = (thread_t)(unsigned long)addr;
 	    } else
 	        aux_param.thr_act = db_default_act;
 	    if (aux_param.thr_act != THREAD_NULL)
@@ -244,9 +241,7 @@ db_act_stat(
 }
 
 char *
-db_act_swap_stat(
-	register thread_t	thr_act,
-	char			*status)
+db_act_swap_stat(__unused thread_t thr_act, char *status)
 {
 	register char *p = status;
 	*p++ = 0;
@@ -254,8 +249,7 @@ db_act_swap_stat(
 	return status;
 }
 
-char	*policy_list[] = { "TS", "RR", "??", "FF",
-			   "??", "??", "??", "BE"};
+const char *policy_list[] = { "TS", "RR", "??", "FF", "??", "??", "??", "BE"};
 
 void
 db_print_act(
@@ -266,7 +260,7 @@ db_print_act(
 	thread_t athread;
 	char status[8];
 	char swap_status[3];
-	char *indent = "";
+	const char *indent = "";
 	int	 policy;
 
 	if (!thr_act) {
@@ -330,7 +324,7 @@ db_print_act(
 		if (!athread->kernel_stack) {
 		    if (athread->continuation) {
 			db_printf("(");
-			db_task_printsym((db_addr_t)athread->continuation,
+			db_task_printsym((db_addr_t)(unsigned long)athread->continuation,
 						DB_STGY_ANY, kernel_task);
 			db_printf(")");
 		    } else {
@@ -441,10 +435,7 @@ db_print_task(
 }
 
 void
-db_print_space(
-	task_t	task,
-	int	task_id,
-	int	flag)
+db_print_space(task_t task, int task_id, __unused int flag)
 {
 	ipc_space_t space;
 	thread_t act = (thread_t)queue_first(&task->threads);
@@ -460,11 +451,8 @@ db_print_space(
 }
 
 void
-db_print_task_vm(
-	task_t		task,
-	int		task_id,
-	boolean_t	title,
-	char		*modif)
+db_print_task_vm(task_t task, int task_id, boolean_t title,
+		 __unused char *modif)
 {
 	vm_map_t	map;
 	pmap_t		pmap;
@@ -479,7 +467,7 @@ db_print_task_vm(
 	map = task->map;
 	pmap = vm_map_pmap(map);
 
-	size = db_vm_map_total_size(map);
+	size = db_vm_map_total_size((unsigned long)map);
 	resident = pmap->stats.resident_count;
 	wired = pmap->stats.wired_count;
 
@@ -495,11 +483,8 @@ db_print_task_vm(
 
 
 void
-db_show_one_task_vm(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		*modif)
+db_show_one_task_vm(db_expr_t addr, boolean_t have_addr,
+		    __unused db_expr_t count, char *modif)
 {
 	thread_t	thread;
 	task_t		task;
@@ -514,7 +499,7 @@ db_show_one_task_vm(
 		}
 		task = thread->task;
 	} else {
-		task = (task_t) addr;
+		task = (task_t)(unsigned long)addr;
 	}
 
 	task_id = db_lookup_task(task);
@@ -527,19 +512,15 @@ db_show_one_task_vm(
 }
 
 void
-db_show_all_task_vm(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		*modif)
+db_show_all_task_vm(__unused db_expr_t addr, __unused boolean_t have_addr,
+		    __unused db_expr_t count, char *modif)
 {
 	task_t		task;
 	int		task_id;
 	boolean_t	title = TRUE;
-	processor_set_t	pset = &default_pset;
 
 	task_id = 0;
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		db_print_task_vm(task, task_id, title, modif);
 		title = FALSE;
 		task_id++;
@@ -547,16 +528,12 @@ db_show_all_task_vm(
 }
 
 void
-db_show_all_acts(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_all_acts(__unused db_expr_t addr, __unused boolean_t have_addr,
+		 __unused db_expr_t count, char *modif)
 {
 	task_t task;
 	int task_id;
 	int flag;
-	processor_set_t pset = &default_pset;
 
 	flag = OPTION_TASK_TITLE|OPTION_INDENT;
 	if (db_option(modif, 'u'))
@@ -565,7 +542,7 @@ db_show_all_acts(
 	    flag |= OPTION_LONG;
 
 	task_id = 0;
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		db_print_task(task, task_id, flag);
 		flag &= ~OPTION_TASK_TITLE;
 		task_id++;
@@ -575,11 +552,8 @@ db_show_all_acts(
 }
 
 void
-db_show_one_space(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_one_space(db_expr_t addr, boolean_t have_addr,
+		  __unused db_expr_t count, char *modif)
 {
 	int		flag;
 	int		task_id;
@@ -598,7 +572,7 @@ db_show_one_space(
 		/*NOTREACHED*/
 	    }
 	} else
-	    task = (task_t) addr;
+	    task = (task_t)(unsigned long)addr;
 
 	if ((task_id = db_lookup_task(task)) < 0) {
 	    db_printf("bad task address 0x%llx\n", (unsigned long long)addr);
@@ -611,16 +585,12 @@ db_show_one_space(
 }
 
 void
-db_show_all_spaces(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_all_spaces(__unused db_expr_t addr, __unused boolean_t have_addr,
+		   __unused db_expr_t count, char *modif)
 {
 	task_t task;
 	int task_id = 0;
 	int flag;
-	processor_set_t pset = &default_pset;
 
 	flag = OPTION_TASK_TITLE|OPTION_INDENT;
 	if (db_option(modif, 'u'))
@@ -629,7 +599,7 @@ db_show_all_spaces(
 	    flag |= OPTION_LONG;
 
 	db_printf(" ID: TASK     SPACE    MAP               COUNT\n");
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		db_print_space(task, task_id, flag);
 		task_id++;
 	}
@@ -642,12 +612,11 @@ db_task_from_space(
 {
 	task_t task;
 	int tid = 0;
-	processor_set_t pset = &default_pset;
 
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		if (task->itk_space == space) {
 			*task_id = tid;
-			return (db_addr_t)task;
+			return (db_addr_t)(unsigned long)task;
 		}
 		tid++;
 	}
@@ -656,11 +625,8 @@ db_task_from_space(
 }
 
 void
-db_show_one_act(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_one_act(db_expr_t addr, boolean_t have_addr, __unused db_expr_t	count,
+		char *modif)
 {
 	int		flag;
 	int		act_id;
@@ -679,7 +645,7 @@ db_show_one_act(
 		/*NOTREACHED*/
 	    }
 	} else
-	    thr_act = (thread_t) addr;
+	    thr_act = (thread_t)(unsigned long)addr;
 
 	if ((act_id = db_lookup_act(thr_act)) < 0) {
 	    db_printf("bad thr_act address %#llX\n", (unsigned long long)addr);
@@ -708,11 +674,8 @@ db_show_one_act(
 }
 
 void
-db_show_one_task(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_one_task(db_expr_t addr, boolean_t have_addr,
+		 __unused db_expr_t count, char *modif)
 {
 	int		flag;
 	int		task_id;
@@ -731,7 +694,7 @@ db_show_one_task(
 		/*NOTREACHED*/
 	    }
 	} else
-	    task = (task_t) addr;
+	    task = (task_t)(unsigned long)addr;
 
 	if ((task_id = db_lookup_task(task)) < 0) {
 	    db_printf("bad task address 0x%llX\n", (unsigned long long)addr);
@@ -743,16 +706,13 @@ db_show_one_task(
 }
 
 void
-db_show_shuttle(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_shuttle(db_expr_t addr, boolean_t have_addr, __unused db_expr_t	count,
+		__unused char *modif)
 {
 	thread_t			thread;
 
 	if (have_addr)
-	    thread = (thread_t) addr;
+	    thread = (thread_t)(unsigned long)addr;
 	else {
 	    thread = current_thread();
 	    if (thread == THREAD_NULL) {
@@ -782,12 +742,8 @@ void db_reset_print_entry(
 }
 
 void
-db_print_one_entry(
-	ipc_entry_t		entry,
-	int			index,
-	mach_port_name_t	name,
-	boolean_t		is_pset,
-	ipc_space_t		space)
+db_print_one_entry(ipc_entry_t entry, int index, mach_port_name_t name,
+		   boolean_t is_pset, __unused ipc_space_t space)
 {
 	ipc_port_t aport = (ipc_port_t)entry->ie_object;
 	ipc_entry_bits_t bits;
@@ -876,7 +832,7 @@ db_lookup_port(
 	if (thr_act == THREAD_NULL)
 	    return(0);
 	space = thr_act->task->itk_space;
-	if (id < 0 || id >= space->is_table_size)
+	if (id < 0 || (unsigned)id >= space->is_table_size)
 	    return(0);
 	entry = &space->is_table[id];
 	if (entry->ie_bits & MACH_PORT_TYPE_PORT_RIGHTS)
@@ -884,26 +840,9 @@ db_lookup_port(
 	return(0);
 }
 
-static void
-db_print_port_id(
-	int		id,
-	ipc_port_t	port,
-	unsigned	bits,
-	int		n)
-{
-	if (n != 0 && n % 3 == 0)
-	    db_printf("\n");
-	db_printf("\tport%d(%s,%x)", id,
-		(bits & MACH_PORT_TYPE_RECEIVE)? "r":
-		(bits & MACH_PORT_TYPE_SEND)? "s": "S", port);
-}
-
 void
-db_show_port_id(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_port_id(db_expr_t addr, boolean_t have_addr, __unused db_expr_t count,
+		char *modif)
 {
 	thread_t thr_act;
 
@@ -914,7 +853,7 @@ db_show_port_id(
 		/*NOTREACHED*/
 	    }
 	} else
-	    thr_act = (thread_t) addr;
+	    thr_act = (thread_t)(unsigned long)addr;
 	if (db_lookup_act(thr_act) < 0) {
 	    db_printf("Bad thr_act address 0x%llX\n", addr);
 	    db_error(0);
@@ -924,14 +863,13 @@ db_show_port_id(
 	    db_printf("\n");
 }
 
+extern void db_sched(void);
 /*
  *	Useful system state when the world has hung.
  */
 void
-db_system_stats()
+db_system_stats(void)
 {
-	extern void	db_sched(void);
-
 	db_sched();
 	iprintf("\n");
 	db_vm();
@@ -944,28 +882,23 @@ db_system_stats()
 void db_show_one_runq(run_queue_t runq);
 
 void
-db_show_runq(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_show_runq(__unused db_expr_t addr, __unused boolean_t have_addr,
+	     __unused db_expr_t count, __unused char *modif)
 {
-	processor_set_t pset = &default_pset;
 	processor_t proc;
 	run_queue_t runq;
 	boolean_t showedany = FALSE;
 
-	queue_iterate(&pset->processors, proc, processor_t, processors) {
+	for (proc = processor_list; proc != PROCESSOR_NULL; proc = proc->processor_list) {
 		runq = &proc->runq;
 		if (runq->count > 0) {
-		    db_printf("PROCESSOR %x IN SET %x\n", proc, pset);
+		    db_printf("PROCESSOR %x IN SET %x\n", proc, proc->processor_set);
 		    db_show_one_runq(runq);
 		    showedany = TRUE;
 		}
 	}
-	runq = &pset->runq;
-	if (runq->count > 0) {
-		db_printf("PROCESSOR SET %x\n", pset);
+	if (rt_runq.count > 0) {
+		db_printf("REAL TIME\n");
 		db_show_one_runq(runq);
 		showedany = TRUE;
 	}
diff --git a/osfmk/ddb/db_print.h b/osfmk/ddb/db_print.h
index e3e622889..aa5d55fff 100644
--- a/osfmk/ddb/db_print.h
+++ b/osfmk/ddb/db_print.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -198,4 +204,6 @@ void db_show_runq(
 	db_expr_t	count,
 	char *		modif);
 
+void db_show_one_lock(lock_t *);
+
 #endif	/* !_DDB_DB_PRINT_H_ */
diff --git a/osfmk/ddb/db_run.c b/osfmk/ddb/db_run.c
index 4a00102db..c0e98e1ab 100644
--- a/osfmk/ddb/db_run.c
+++ b/osfmk/ddb/db_run.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -101,19 +107,19 @@ void db_clear_task_single_step(
 extern jmp_buf_t *db_recover;
 boolean_t db_step_again(void);
 
+static db_addr_t db_stop_pc;
 boolean_t
 db_stop_at_pc(
 	boolean_t	*is_breakpoint,
 	task_t		task,
 	task_t		space)
 {
-	register  db_addr_t	pc;
 	register  db_thread_breakpoint_t bkpt;
 
 	db_clear_task_single_step(DDB_REGS, space);
 	db_clear_breakpoints();
 	db_clear_watchpoints();
-	pc = PC_REGS(DDB_REGS);
+	db_stop_pc = PC_REGS(DDB_REGS);
 
 #ifdef	FIXUP_PC_AFTER_BREAK
 	if (*is_breakpoint) {
@@ -122,14 +128,14 @@ db_stop_at_pc(
 	     * machine requires it.
 	     */
 	    FIXUP_PC_AFTER_BREAK
-	    pc = PC_REGS(DDB_REGS);
+	    db_stop_pc = PC_REGS(DDB_REGS);
 	}
 #endif
 
 	/*
 	 * Now check for a breakpoint at this address.
 	 */
-	bkpt = db_find_thread_breakpoint_here(space, pc);
+	bkpt = db_find_thread_breakpoint_here(space, db_stop_pc);
 	if (bkpt) {
 	    if (db_cond_check(bkpt)) {
 		*is_breakpoint = TRUE;
@@ -148,7 +154,7 @@ db_stop_at_pc(
 	if (db_run_mode == STEP_ONCE) {
 	    if (--db_loop_count > 0) {
 		if (db_sstep_print) {
-		    db_print_loc_and_inst(pc, task);
+		    db_print_loc_and_inst(db_stop_pc, task);
 		}
 		return (FALSE);	/* continue */
 	    }
@@ -157,7 +163,9 @@ db_stop_at_pc(
 	    jmp_buf_t *prev;
 	    jmp_buf_t db_jmpbuf;
 	    /* WARNING: the following assumes an instruction fits an int */
-	    db_expr_t ins = db_get_task_value(pc, sizeof(int), FALSE, space);
+		db_expr_t ins;
+	   
+		ins = db_get_task_value(db_stop_pc, sizeof(int), FALSE, space);
 
 	    /* continue until matching return */
 
@@ -175,7 +183,7 @@ db_stop_at_pc(
 				db_last_inst_count = db_inst_count;
 				for (i = db_call_depth; --i > 0; )
 				    db_printf("  ");
-				db_print_loc_and_inst(pc, task);
+				db_print_loc_and_inst(db_stop_pc, task);
 				db_printf("\n");
 		    	    }
 		        }
@@ -190,7 +198,8 @@ db_stop_at_pc(
 	}
 	if (db_run_mode == STEP_CALLT) {
 	    /* WARNING: the following assumes an instruction fits an int */
-	    db_expr_t ins = db_get_task_value(pc, sizeof(int), FALSE, space);
+		db_expr_t ins;
+		ins = db_get_task_value(db_stop_pc, sizeof(int), FALSE, space);
 
 	    /* continue until call or return */
 
@@ -201,7 +210,7 @@ db_stop_at_pc(
 				return (FALSE);	/* continue */
 	    }
 	}
-	if (db_find_breakpoint_here(space, pc))
+	if (db_find_breakpoint_here(space, db_stop_pc))
 		return(FALSE);
 	db_run_mode = STEP_NONE;
 	return (TRUE);
@@ -212,7 +221,11 @@ db_restart_at_pc(
 	boolean_t	watchpt,
 	task_t	  	task)
 {
-	register db_addr_t pc = PC_REGS(DDB_REGS), brpc;
+	db_addr_t pc = PC_REGS(DDB_REGS);
+#ifdef	SOFTWARE_SSTEP
+	db_addr_t brpc;
+#endif
+
 
 	if ((db_run_mode == STEP_COUNT) ||
 	    (db_run_mode == STEP_RETURN) ||
@@ -278,9 +291,7 @@ db_step_again(void)
 }
 
 void
-db_single_step(
-	db_regs_t	*regs,
-	task_t	  	task)
+db_single_step(db_regs_t *regs, __unused task_t task)
 {
 	if (db_run_mode == STEP_CONTINUE) {
 	    db_run_mode = STEP_INVISIBLE;
@@ -405,15 +416,12 @@ extern int	db_cmd_loop_done;
 
 /* single-step */
 void
-db_single_step_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_single_step_cmd(__unused db_expr_t addr, __unused boolean_t have_addr,
+		   db_expr_t count, char *modif)
 {
 	boolean_t	print = FALSE;
 
-	if (count == -1)
+	if (count == (db_expr_t)-1)
 	    count = 1;
 
 	if (modif[0] == 'p')
@@ -432,11 +440,8 @@ db_single_step_cmd(
 
 /* trace and print until call/return */
 void
-db_trace_until_call_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_trace_until_call_cmd(__unused db_expr_t addr, __unused boolean_t have_addr,
+			__unused db_expr_t count, char *modif)
 {
 	boolean_t	print = FALSE;
 
@@ -454,11 +459,10 @@ db_trace_until_call_cmd(
 }
 
 void
-db_trace_until_matching_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_trace_until_matching_cmd(__unused db_expr_t addr,
+			    __unused boolean_t have_addr,
+			    __unused db_expr_t count,
+			    char *modif)
 {
 	boolean_t	print = FALSE;
 
@@ -478,11 +482,8 @@ db_trace_until_matching_cmd(
 
 /* continue */
 void
-db_continue_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_continue_cmd(__unused db_expr_t addr, __unused boolean_t have_addr,
+		__unused db_expr_t count, __unused char *modif)
 {
 	/*
 	 * Though "cont/c" works fairly well, it's not really robust
@@ -510,22 +511,16 @@ db_continue_cmd(
 /*
  * Switch to gdb
  */
-void
-db_to_gdb(
-	void)
+static void
+db_to_gdb(void)
 {
-	extern unsigned int switch_debugger;
-
-	switch_debugger=1;
+	switch_debugger = 1;
 }
 
 /* gdb */
 void    
-db_continue_gdb(
-	db_expr_t	addr, 
-	int		have_addr,
-	db_expr_t	count,   
-	char *		modif)
+db_continue_gdb(__unused db_expr_t addr, __unused boolean_t have_addr,
+		__unused db_expr_t count, __unused char *modif)
 {
 	db_to_gdb();
 	db_run_mode = STEP_CONTINUE;
diff --git a/osfmk/ddb/db_run.h b/osfmk/ddb/db_run.h
index a3544402d..c568f0dec 100644
--- a/osfmk/ddb/db_run.h
+++ b/osfmk/ddb/db_run.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -75,35 +81,15 @@ void db_single_step(
 	db_regs_t	*regs,
 	task_t	  	task);
 
-void db_single_step_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_single_step_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_trace_until_call_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_trace_until_call_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_trace_until_matching_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_trace_until_matching_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_continue_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_continue_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_continue_gdb(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_continue_gdb(db_expr_t, boolean_t, db_expr_t, char *);
 
 boolean_t db_in_single_step(void);
 
diff --git a/osfmk/ddb/db_sym.c b/osfmk/ddb/db_sym.c
index b6c01ca52..b99f9674f 100644
--- a/osfmk/ddb/db_sym.c
+++ b/osfmk/ddb/db_sym.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -58,6 +64,7 @@
 #include <string.h>			/* For strcpy(), strcmp() */
 #include <mach/std_types.h>
 #include <kern/misc_protos.h>		/* For printf() */
+#include <kern/kalloc.h>
 #include <ddb/db_sym.h>
 #include <ddb/db_task_thread.h>
 #include <ddb/db_command.h>
@@ -72,7 +79,7 @@
  */
 #define	MAXNOSYMTABS	6
 
-db_symtab_t	db_symtabs[MAXNOSYMTABS] = {{0}};
+db_symtab_t db_symtabs[MAXNOSYMTABS];
 int db_nsymtab = 0;
 
 db_symtab_t	*db_last_symtab;
@@ -146,6 +153,7 @@ ddb_init(void)
 	db_machdep_init();
 }
 
+extern vm_map_t kernel_map;
 /*
  * Add symbol table, with given name, to list of symbol tables.
  */
@@ -153,8 +161,8 @@ boolean_t
 db_add_symbol_table(
 	int		type,
 	char		*start,
-	char		*end,
-	char		*name,
+	char		*db_end,
+	const char	*name,
 	char		*ref,
 	char		*map_pointer,
 	unsigned long	minsym,
@@ -162,7 +170,6 @@ db_add_symbol_table(
 	boolean_t	sorted)
 {
 	register db_symtab_t *st;
-	extern vm_map_t kernel_map;
 
 	if (db_nsymtab >= MAXNOSYMTABS)
 	    return (FALSE);
@@ -170,11 +177,11 @@ db_add_symbol_table(
 	st = &db_symtabs[db_nsymtab];
 	st->type = type;
 	st->start = start;
-	st->end = end;
+	st->end = db_end;
 	st->private = ref;
-	if (map_pointer == (char *)kernel_map || 
-	    (VM_MAX_ADDRESS <= VM_MIN_KERNEL_ADDRESS &&
-	     VM_MIN_KERNEL_ADDRESS <= minsym))
+	if (map_pointer == (char *)kernel_map ||
+	    (VM_MIN_KERNEL_ADDRESS - VM_MAX_ADDRESS > 0 &&
+	     minsym - VM_MIN_KERNEL_ADDRESS > 0))
 		st->map_pointer = 0;
 	else
 		st->map_pointer = map_pointer;
@@ -208,11 +215,13 @@ db_qualify(
 	register char	*s;
 
 	s = tmp;
-	while (*s++ = *symtabname++) {
+	while ((*s++ = *symtabname++)) {
+		;
 	}
 	s[-1] = ':';
 	*s++ = ':';
-	while (*s++ = *symname++) {
+	while ((*s++ = *symname++)) {
+		;
 	}
 	return tmp;
 }
@@ -226,14 +235,14 @@ db_eqname(
 {
 	if (!strcmp(src, dst))
 	    return (TRUE);
-	if (src[0] == c)
+	if (src[0] == (char)c)
 	    return (!strcmp(src+1,dst));
 	return (FALSE);
 }
 
 boolean_t
 db_value_of_name(
-	char		*name,
+	const char	*name,
 	db_expr_t	*valuep)
 {
 	db_sym_t	sym;
@@ -256,10 +265,6 @@ db_print_completion(
 	int symtab_start = 0;
 	int symtab_end = db_nsymtab;
 	register char *cp;
-	int nsym = 0;
-	char *name = (char *)0;
-	int len;
-	int toadd;
 
 	/*
 	 * Look for, remove, and remember any symbol table specifier.
@@ -359,13 +364,13 @@ db_lookup_incomplete(
  * otherwise, all symbol tables will be searched.
  */
 db_sym_t
-db_lookup(char *symstr)
+db_lookup(const char *symstr)
 {
 	db_sym_t sp;
-	register int i;
+	int i;
 	int symtab_start = 0;
 	int symtab_end = db_nsymtab;
-	register char *cp;
+	char *cp;
 
 	/*
 	 * Look for, remove, and remember any symbol table specifier.
@@ -392,7 +397,7 @@ db_lookup(char *symstr)
 	 * Return on first match.
 	 */
 	for (i = symtab_start; i < symtab_end; i++) {
-		if (sp = X_db_lookup(&db_symtabs[i], symstr)) {
+		if ((sp = X_db_lookup(&db_symtabs[i], symstr))) {
 			db_last_symtab = &db_symtabs[i];
 			return sp;
 		}
@@ -697,7 +702,7 @@ again:
 db_sym_t
 db_search_task_symbol_and_line(
 	register db_addr_t	val,
-	db_strategy_t		strategy,
+	__unused db_strategy_t	strategy,
 	db_expr_t		*offp,
 	char			**filenamep,
 	int			*linenump,
@@ -766,7 +771,7 @@ void
 db_symbol_values(
 	db_symtab_t	*stab,
 	db_sym_t	sym,
-	char		**namep,
+	const char		**namep,
 	db_expr_t	*valuep)
 {
 	db_expr_t	value;
@@ -973,7 +978,7 @@ qsort_swap(
 	char *aa, *bb;
 	char ctemp;
 
-	for (; size >= sizeof (int); size -= sizeof (int), a++, b++) {
+	for (; size >= (signed)sizeof (int); size -= sizeof (int), a++, b++) {
 		temp = *a;
 		*a = *b;
 		*b = temp;
@@ -999,7 +1004,8 @@ qsort_rotate(
 	char *aa, *bb, *cc;
 	char ctemp;
 
-	for (; size >= sizeof (int); size -= sizeof (int), a++, b++, c++) {
+	for (; size >= (signed)sizeof(int);
+			size -= sizeof(int), a++, b++, c++) {
 		temp = *a;
 		*a = *c;
 		*c = *b;
@@ -1193,7 +1199,7 @@ void
 db_qsort_limit_search(
 	char	*target,
 	char	**start,
-	char	**end,
+	char	**db_end,
 	int	eltsize,
 	int	(*compfun)(char *, char *))
 {
@@ -1203,7 +1209,7 @@ db_qsort_limit_search(
 	int comp;
 
 	oleft = left = *start;
-	oright = right = *end;
+	oright = right = *db_end;
 	part = (char *) 0;
 
 	while (left < right) {
@@ -1234,7 +1240,7 @@ db_qsort_limit_search(
 			     left > *start && (*compfun)(left, part) == 0;
 			     left -= eltsize);
 			for (right = part + eltsize;
-			     right < *end && (*compfun)(right, part) == 0;
+			     right < *db_end && (*compfun)(right, part) == 0;
 			     right += eltsize);
 			oright = right;
 			oleft = left;
@@ -1244,9 +1250,9 @@ db_qsort_limit_search(
 	
 	if (qsort_search_debug)
 		printf("[ Limited from %x-%x to %x-%x in %d iters ]\n",
-			  *start, *end, oleft, oright, nbiter);
+			  *start, *db_end, oleft, oright, nbiter);
 	*start = oleft;
-	*end = oright;
+	*db_end = oright;
 }
 
 void
@@ -1257,13 +1263,13 @@ bubble_sort(
 	int	(*compfun)(char *, char *))
 {
 	boolean_t sorted;
-	char *end;
+	char *b_end;
 	register char *p;
 
-	end = table + ((nbelts-1) * eltsize);
+	b_end = table + ((nbelts-1) * eltsize);
 	do {
 		sorted = TRUE;
-		for (p = table; p < end; p += eltsize) {
+		for (p = table; p < b_end; p += eltsize) {
 			if ((*compfun)(p, p + eltsize) > 0) {
 				qsort_swap((int *) p, (int *) (p + eltsize),
 					   eltsize);
@@ -1289,6 +1295,7 @@ db_install_inks(
 	}
 }
 
+extern void db_clone_offsetXXX(char *, long);
 
 void
 db_clone_symtabXXX(
@@ -1300,7 +1307,6 @@ db_clone_symtabXXX(
 	char *		memp;
 	vm_size_t	size;
 	long		offset;
-	extern void db_clone_offsetXXX(char *, long);
 
 	if (db_nsymtab >= MAXNOSYMTABS) {
 	    db_printf("db_clone_symtab: Too Many Symbol Tables\n");
@@ -1378,93 +1384,83 @@ static void no_init(void)
 	db_printf("Non-existent code for ddb init\n");
 }
 
-static boolean_t no_sym_init(
-	char *start,
-	char *end,
-	char *name,
-	char *task_addr)
+static boolean_t
+no_sym_init(__unused char *nstart, __unused char *nend, const char *name,
+	    __unused char *task_addr)
 {
 	db_printf("Non-existent code for init of symtab %s\n", name);
 	return FALSE;
 }
 
-static db_sym_t no_lookup(
-	db_symtab_t *stab,
-	char *symstr)
+static db_sym_t
+no_lookup(__unused db_symtab_t *stab, char *symstr)
 {
 	db_printf("Bogus lookup of symbol %s\n", symstr);
 	return DB_SYM_NULL;
 }
 
-static db_sym_t no_search(
-	db_symtab_t *stab,
-	db_addr_t off,
-	db_strategy_t strategy,
-	db_expr_t *diffp)
+static db_sym_t
+no_search(__unused db_symtab_t *stab, db_addr_t off,
+	  __unused db_strategy_t strategy, __unused db_expr_t *diffp)
 {
 	db_printf("Bogus search for offset %#llXn", (unsigned long long)off);
 	return DB_SYM_NULL;
 }
 
-static boolean_t no_line_at_pc(
-	db_symtab_t *stab,
-	db_sym_t sym,
-	char **file,
-	int *line,
-	db_expr_t pc)
+static boolean_t
+no_line_at_pc(__unused db_symtab_t *stab, __unused db_sym_t sym,
+	      __unused char **file, __unused int *line, db_expr_t pc)
 {
 	db_printf("Bogus search for pc %#llX\n", (unsigned long long)pc);
 	return FALSE;
 }
 
-static void no_symbol_values(
-	db_sym_t sym,
-	char **namep,
-	db_expr_t *valuep)
+static void
+no_symbol_values(__unused db_sym_t sym, char **namep, db_expr_t *valuep)
 {
 	db_printf("Bogus symbol value resolution\n");
 	if (namep) *namep = NULL;
 	if (valuep) *valuep = 0;
 }
 
-static db_sym_t no_search_by_addr(
-	db_symtab_t *stab,
-	db_addr_t off,
-	char **file,
-	char **func,
-	int *line,
-	db_expr_t *diffp,
-	int *args)
+static db_sym_t
+no_search_by_addr(__unused db_symtab_t *stab, db_addr_t off,
+		  __unused char **file, __unused char **func,
+		  __unused int *line, __unused db_expr_t *diffp,
+		  __unused int *args)
 {
 	db_printf("Bogus search for address %#llX\n", (unsigned long long)off);
 	return DB_SYM_NULL;
 }
 	
 int
-no_print_completion(
-	db_symtab_t	*stab,
-	char		*symstr	)
+no_print_completion(__unused db_symtab_t *stab, __unused char *symstr)
 {
 	db_printf("Bogus print completion: not supported\n");
 	return 0;
 }
 
 int
-no_lookup_incomplete(
-	db_symtab_t	*stab,
-	char		*symstr,
-	char		**name,
-	int		*len,
-	int		*toadd)
+no_lookup_incomplete(__unused db_symtab_t *stab,
+		     __unused char *symstr, __unused char **name,
+		     __unused int *len, __unused int *toadd)
 {
 	db_printf("Bogus lookup incomplete: not supported\n");
 	return 0;
 }
 
 #define NONE	\
-	{ no_init, no_sym_init, no_lookup, no_search, \
-	  no_line_at_pc, no_symbol_values, no_search_by_addr, \
-		  no_print_completion, no_lookup_incomplete}
+	{	\
+		.init = no_init, \
+		.sym_init = no_sym_init, \
+		.lookup = no_lookup, \
+		.search_symbol = no_search, \
+		.line_at_pc = no_line_at_pc, \
+		.symbol_values = no_symbol_values, \
+		.search_by_addr = no_search_by_addr, \
+		.print_completion = no_print_completion, \
+		.lookup_incomplete = no_lookup_incomplete, \
+	}
 
 struct db_sym_switch x_db[] = {
 
@@ -1472,17 +1468,33 @@ struct db_sym_switch x_db[] = {
 #ifdef	DB_NO_AOUT
 	NONE,
 #else	/* DB_NO_AOUT */
-	{ aout_db_init, aout_db_sym_init, aout_db_lookup, aout_db_search_symbol,
-	  aout_db_line_at_pc, aout_db_symbol_values, aout_db_search_by_addr,
-	  aout_db_print_completion, aout_db_lookup_incomplete},
+	{
+		.init = aout_db_init,
+		.sym_init = aout_db_sym_init,
+		.lookup = aout_db_lookup,
+		.search_symbol = aout_db_search_symbol,
+		.line_at_pc = aout_db_line_at_pc,
+		.symbol_values = aout_db_symbol_values,
+		.search_by_addr = aout_db_search_by_addr,
+		.print_completion = aout_db_print_completion,
+		.lookup_incomplete = aout_db_lookup_incomplete,
+	},
 #endif	/* DB_NO_AOUT */
 
 #ifdef	DB_NO_COFF
 	NONE,
 #else	/* DB_NO_COFF */
-	{ coff_db_init, coff_db_sym_init, coff_db_lookup, coff_db_search_symbol,
-	  coff_db_line_at_pc, coff_db_symbol_values, coff_db_search_by_addr,
-	  coff_db_print_completion, coff_db_lookup_incomplete },
+	{
+		.init = coff_db_init,
+		.sym_init = coff_db_sym_init,
+		.lookup = coff_db_lookup,
+		.search_symbol = coff_db_search_symbol,
+		.line_at_pc = coff_db_line_at_pc,
+		.symbol_values = coff_db_symbol_values,
+		.search_by_addr = coff_db_search_by_addr,
+		.print_completion = coff_db_print_completion,
+		.lookup_incomplete = coff_db_lookup_incomplete,
+	},
 #endif	/* DB_NO_COFF */
 
 	/* Machdep, not inited here */
diff --git a/osfmk/ddb/db_sym.h b/osfmk/ddb/db_sym.h
index 4252e069b..3749e758e 100644
--- a/osfmk/ddb/db_sym.h
+++ b/osfmk/ddb/db_sym.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -120,7 +126,7 @@ extern boolean_t db_add_symbol_table(
 	int		type,
 	char		*start,
 	char		*end,
-	char		*name,
+	const char	*name,
 	char		*ref,
 	char		*map_pointer,
 	unsigned long	minsym,
@@ -131,10 +137,10 @@ extern void db_install_inks(
 	vm_offset_t	base);
 
 extern boolean_t db_value_of_name(
-	char		*name,
+	const char	*name,
 	db_expr_t	*valuep);
 
-extern db_sym_t db_lookup(char *symstr);
+extern db_sym_t db_lookup(const char *symstr);
 
 extern char * db_get_sym(
 	db_expr_t	* off);
@@ -188,7 +194,7 @@ extern db_sym_t db_search_task_symbol_and_line(
 extern void db_symbol_values(
 	db_symtab_t	*stab,
 	db_sym_t	sym,
-	char		**namep,
+	const char		**namep,
 	db_expr_t	*valuep);
 
 extern void db_task_printsym(
@@ -277,7 +283,7 @@ extern struct db_sym_switch {
 	boolean_t	(*sym_init)(
 				char *start,
 				char *end,
-				char *name,
+				const char *name,
 				char *task_addr
 				);
 
diff --git a/osfmk/ddb/db_task_thread.c b/osfmk/ddb/db_task_thread.c
index 09229702e..7e7420c14 100644
--- a/osfmk/ddb/db_task_thread.c
+++ b/osfmk/ddb/db_task_thread.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -92,15 +98,11 @@ db_lookup_task(task_t target_task)
 {
 	register task_t task;
 	register int task_id;
-	register processor_set_t pset = &default_pset;
-	register int npset = 0;
 
 	task_id = 0;
-	if (npset++ >= DB_MAX_PSETS)
-		return(-1);
-	if (queue_first(&pset->tasks) == 0)
+	if (queue_first(&tasks) == 0)
 		return(-1);
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		if (target_task == task)
 		    return(task_id);
 		if (task_id++ >= DB_MAX_TASKID)
@@ -141,15 +143,11 @@ db_lookup_act(thread_t target_act)
 {
 	register int act_id;
 	register task_t task;
-	register processor_set_t pset = &default_pset;
 	register int ntask = 0;
-	register int npset = 0;
 
-	if (npset++ >= DB_MAX_PSETS)
+	if (queue_first(&tasks) == 0)
 		return(-1);
-	if (queue_first(&pset->tasks) == 0)
-		return(-1);
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		if (ntask++ > DB_MAX_TASKID)
 		    return(-1);
 		if (task->thread_count == 0)
@@ -180,19 +178,15 @@ db_check_act_address_valid(thread_t thr_act)
  * convert task_id(queue postion) to task address
  */
 task_t
-db_lookup_task_id(register task_id)
+db_lookup_task_id(int task_id)
 {
 	register task_t task;
-	register processor_set_t pset = &default_pset;
-	register int npset = 0;
 
 	if (task_id > DB_MAX_TASKID)
 	    return(TASK_NULL);
-	if (npset++ >= DB_MAX_PSETS)
-		return(TASK_NULL);
-	if (queue_first(&pset->tasks) == 0)
+	if (queue_first(&tasks) == 0)
 		return(TASK_NULL);
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		if (task_id-- <= 0)
 			return(task);
 	}
@@ -235,7 +229,7 @@ db_get_next_act(
 
 	*actp = THREAD_NULL;
 	if (db_expression(&value)) {
-	    thr_act = (thread_t) value;
+	    thr_act = (thread_t)(unsigned long)value;
 	    if (!db_check_act_address_valid(thr_act)) {
 		db_flush_lex();
 		return(FALSE);
@@ -267,11 +261,8 @@ db_init_default_act(void)
  * in the command line
  */
 int
-db_set_default_act(
-	struct db_variable	*vp,
-	db_expr_t		*valuep,
-	int			flag,
-	db_var_aux_param_t	ap)			/* unused */
+db_set_default_act(__unused struct db_variable *vp, db_expr_t *valuep,
+		   int flag, __unused db_var_aux_param_t ap)
 {
 	thread_t	thr_act;
 	int		task_id;
@@ -290,10 +281,10 @@ db_set_default_act(
 	}
 
 	if (flag != DB_VAR_SET) {
-	    *valuep = (db_expr_t) db_default_act;
+	    *valuep = (db_expr_t)(unsigned long)db_default_act;
 	    return(0);
 	}
-	thr_act = (thread_t) *valuep;
+	thr_act = (thread_t)(unsigned long)*valuep;
 	if (thr_act != THREAD_NULL && !db_check_act_address_valid(thr_act))
 	    db_error(0);
 	    /* NOTREACHED */
@@ -307,11 +298,8 @@ db_set_default_act(
  * convert $taskXXX[.YYY] type DDB variable to task or thread address
  */
 int
-db_get_task_act(
-	struct db_variable	*vp,
-	db_expr_t		*valuep,
-	int			flag,
-	db_var_aux_param_t	ap)
+db_get_task_act(__unused struct db_variable *vp, db_expr_t *valuep, int flag,
+		db_var_aux_param_t ap)
 {
 	task_t	 		task;
 	thread_t		thr_act;
@@ -335,7 +323,7 @@ db_get_task_act(
 	    /* NOTREACHED */
 	}
 	if (ap->level <= 1) {
-	    *valuep = (db_expr_t) task;
+	    *valuep = (db_expr_t)(unsigned long)task;
 	    return(0);
 	}
 	if ((thr_act = db_lookup_act_id(task, ap->suffix[1])) == THREAD_NULL){
@@ -344,6 +332,6 @@ db_get_task_act(
 	    db_error(0);
 	    /* NOTREACHED */
 	}
-	*valuep = (db_expr_t) thr_act;
+	*valuep = (db_expr_t)(unsigned long)thr_act;
 	return(0);
 }
diff --git a/osfmk/ddb/db_task_thread.h b/osfmk/ddb/db_task_thread.h
index 2c465c157..12f9ac6b4 100644
--- a/osfmk/ddb/db_task_thread.h
+++ b/osfmk/ddb/db_task_thread.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_trap.c b/osfmk/ddb/db_trap.c
index d95ff4536..41acbd325 100644
--- a/osfmk/ddb/db_trap.c
+++ b/osfmk/ddb/db_trap.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -76,18 +82,15 @@ extern int		db_inst_count;
 extern int		db_load_count;
 extern int		db_store_count;
 
+static task_t task_space;
+static task_t task;
 void
-db_task_trap(
-	int		type,
-	int		code,
-	boolean_t	user_space)
+db_task_trap(__unused int type, __unused int code, boolean_t user_space)
 {
 	jmp_buf_t db_jmpbuf;
 	jmp_buf_t *prev;
 	boolean_t	bkpt;
 	boolean_t	watchpt;
-	task_t		task;
-	task_t		task_space;
 
 	task = db_current_task();
 	task_space = db_target_space(current_thread(), user_space);
diff --git a/osfmk/ddb/db_trap.h b/osfmk/ddb/db_trap.h
index 60fad74b3..79554c85b 100644
--- a/osfmk/ddb/db_trap.h
+++ b/osfmk/ddb/db_trap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/db_variables.c b/osfmk/ddb/db_variables.c
index 163e98d63..0fe14d1e6 100644
--- a/osfmk/ddb/db_variables.c
+++ b/osfmk/ddb/db_variables.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -66,7 +72,6 @@
 #include <ddb/db_macro.h>
 #include <ddb/db_output.h>		/* For db_printf() */
 
-extern db_expr_t	db_radix;
 extern db_expr_t	db_max_width;
 extern db_expr_t	db_tab_stop_width;
 extern db_expr_t	db_max_line;
@@ -79,21 +84,65 @@ extern db_expr_t	db_auto_completion;
 db_expr_t	db_work[DB_NWORK];	/* work variable */
 
 struct db_variable db_vars[] = {
-	{ "maxoff",	(db_expr_t*)&db_maxoff,	FCN_NULL },
-	{ "autowrap",	&db_auto_wrap,		FCN_NULL },
-	{ "completion",	&db_auto_completion,	FCN_NULL },
-	{ "maxwidth",	&db_max_width,		FCN_NULL },
-	{ "radix",	&db_radix,		FCN_NULL },
-	{ "tabstops",	&db_tab_stop_width,	FCN_NULL },
-	{ "lines",	&db_max_line,		FCN_NULL },
-	{ "thr_act",	0,			db_set_default_act	},
-	{ "task",	0,			db_get_task_act,
-	  1,		2,			-1,	-1		},
-	{ "work",	&db_work[0],		FCN_NULL,
-	  1,		1,			0,	DB_NWORK-1	},
-	{ "arg",	0,			db_arg_variable,
-	  1,		1,			1,	DB_MACRO_NARGS,
-	  1,		0,	DB_MACRO_LEVEL-1,	(int *)&db_macro_level	},
+	{
+		.name = "maxoff",
+		.valuep = (db_expr_t*)&db_maxoff,
+	},
+	{
+		.name = "autowrap",
+		.valuep = &db_auto_wrap,
+	},
+	{
+		.name = "completion",
+		.valuep = &db_auto_completion,
+	},
+	{
+		.name = "maxwidth",
+		.valuep = &db_max_width,
+	},
+	{
+		.name = "radix",
+		.valuep = &db_radix,
+	},
+	{
+		.name = "tabstops",
+		.valuep = &db_tab_stop_width,
+	},
+	{
+		.name = "lines",
+		.valuep = &db_max_line,
+	},
+	{
+		.name = "thr_act",
+		.fcn = db_set_default_act,
+	},
+	{
+		.name = "task",
+		.fcn = db_get_task_act,
+		.min_level = 1,
+		.max_level = 2,
+		.low = -1,
+		.high = -1,
+	},
+	{
+		.name = "work",
+		.valuep = &db_work[0],
+		.min_level = 1,
+		.max_level = 1,
+		.high = DB_NWORK - 1,
+	},
+	{
+		.name = "arg",
+		.fcn = db_arg_variable,
+		.min_level = 1,
+		.max_level = 1,
+		.low = 1,
+		.high = DB_MACRO_NARGS,
+		.hidden_level = 1,
+		.hidden_low = 0,
+		.hidden_high = DB_MACRO_LEVEL - 1,
+		.hidden_levelp = (int *)&db_macro_level,
+	},
 };
 struct db_variable *db_evars = db_vars + sizeof(db_vars)/sizeof(db_vars[0]);
 
@@ -102,27 +151,19 @@ struct db_variable *db_evars = db_vars + sizeof(db_vars)/sizeof(db_vars[0]);
 /* Prototypes for functions local to this file.
  */
 
-static char *db_get_suffix(
-	register char	*suffix,
-	short		*suffix_value);
+static const char *db_get_suffix(const char *, short *);
 
-static boolean_t db_cmp_variable_name(
-	struct db_variable		*vp,
-	char				*name,
-	register db_var_aux_param_t	ap);
+static boolean_t db_cmp_variable_name(struct db_variable *, const char *,
+		db_var_aux_param_t);
 
 static int db_find_variable(
 	struct db_variable	**varp,
 	db_var_aux_param_t	ap);
 
-static int db_set_variable(db_expr_t value);
-
 void db_list_variable(void);
 
-static char *
-db_get_suffix(
-	register char	*suffix,
-	short		*suffix_value)
+static const char *
+db_get_suffix(const char *suffix, short *suffix_value)
 {
 	register int value;
 
@@ -138,13 +179,11 @@ db_get_suffix(
 }
 
 static boolean_t
-db_cmp_variable_name(
-	struct db_variable		*vp,
-	char				*name,
-	register db_var_aux_param_t	ap)
+db_cmp_variable_name(struct db_variable *vp, const char *name,
+		     db_var_aux_param_t ap)
 {
-	register char *var_np, *np;
-	register int level;
+	const char *var_np, *np;
+	int level;
 	
 	for (np = name, var_np = vp->name; *var_np; ) {
 	    if (*np++ != *var_np++)
@@ -217,22 +256,6 @@ db_get_variable(db_expr_t *valuep)
 	return (1);
 }
 
-static int
-db_set_variable(db_expr_t value)
-{
-	struct db_variable *vp;
-	struct db_var_aux_param aux_param;
-	char		modif[TOK_STRING_SIZE];
-
-	aux_param.modif = modif;
-	if (!db_find_variable(&vp, &aux_param))
-	    return (0);
-
-	db_read_write_variable(vp, &value, DB_VAR_SET, &aux_param);
-
-	return (1);
-}
-
 void
 db_read_write_variable(
 	struct db_variable	*vp,
@@ -247,7 +270,7 @@ db_read_write_variable(
 
 	if (ap == 0) {
 	    ap = &aux_param;
-	    ap->modif = "";
+	    ap->modif = NULL;
 	    ap->level = 0;
 	    ap->thr_act = THREAD_NULL;
 	}
@@ -389,18 +412,17 @@ db_show_one_variable(void)
 	struct db_variable *cur;
 	unsigned int len;
 	unsigned int sl;
-	unsigned int slen;
-	short h;
+	unsigned int slen = 0;
+	short h = 0;
 	short i;
-	short j;
+	unsigned short j;
 	short k;
 	short low;
-	int hidden_level;
+	int hidden_level = 0;
 	struct db_var_aux_param aux_param;
-	char *p;
-	char *q;
+	const char *p = NULL, *q;
 	char *name;
-	db_addr_t offset;
+	db_addr_t offset = 0;
 
 	for (cur = db_vars; cur < db_evars; cur++)
 	    if (db_cmp_variable_name(cur, db_tok_string, &aux_param))
@@ -529,16 +551,17 @@ db_show_one_variable(void)
 }
 
 void
-db_show_variable(void)
+db_show_variable(__unused db_expr_t addr, __unused boolean_t have_addr,
+		 __unused db_expr_t count, __unused char *modif)
 {
 	struct db_variable *cur;
 	unsigned int l;
 	unsigned int len;
 	unsigned int sl;
 	unsigned int slen;
-	short h;
+	short h = 0;
 	short i;
-	short j;
+	unsigned short j;
 	short k;
 	int t;
 	int t1;
@@ -592,7 +615,7 @@ db_show_variable(void)
 		len = l + 1;
 	}
 
-	aux_param.modif = "";
+	aux_param.modif = NULL;
 	aux_param.level = 1;
 	aux_param.thr_act = THREAD_NULL;
 
diff --git a/osfmk/ddb/db_variables.h b/osfmk/ddb/db_variables.h
index 2b13c746e..8d0b1c817 100644
--- a/osfmk/ddb/db_variables.h
+++ b/osfmk/ddb/db_variables.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -179,7 +185,7 @@ typedef struct db_var_aux_param	*db_var_aux_param_t;
  * Debugger variables.
  */
 struct db_variable {
-	char	*name;		/* Name of variable */
+	const char *name;	/* Name of variable */
 	db_expr_t *valuep;	/* pointer to value of variable */
 				/* function to call when reading/writing */
 	int	(*fcn)(struct db_variable *,db_expr_t *,int,db_var_aux_param_t);
@@ -243,7 +249,7 @@ void db_set_cmd(void);
 
 void db_show_one_variable(void);
 
-void db_show_variable(void);
+void db_show_variable(db_expr_t, boolean_t, db_expr_t, char *);
 
 db_variable_t db_find_reg_name(char	*s);
 
diff --git a/osfmk/ddb/db_watch.c b/osfmk/ddb/db_watch.c
index 3a99ac5ab..f4a740028 100644
--- a/osfmk/ddb/db_watch.c
+++ b/osfmk/ddb/db_watch.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -237,7 +243,7 @@ db_get_task(
 	user_space = db_option(modif, 'T');
 	if (user_space) {
 	    if (db_expression(&value)) {
-		task = (task_t)value;
+		task = (task_t)(unsigned long)value;
 		if (db_lookup_task(task) < 0) {
 		    db_printf("bad task address %X\n", task);
 		    return(-1);
@@ -263,11 +269,8 @@ db_get_task(
 
 /* Delete watchpoint */
 void
-db_deletewatch_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_deletewatch_cmd(db_expr_t addr, __unused boolean_t have_addr,
+		   __unused db_expr_t count, char *modif)
 {
 	task_t		task;
 
@@ -278,11 +281,8 @@ db_deletewatch_cmd(
 
 /* Set watchpoint */
 void
-db_watchpoint_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_watchpoint_cmd(db_expr_t addr, __unused boolean_t have_addr,
+		  __unused db_expr_t count, char *modif)
 {
 	vm_size_t	size;
 	db_expr_t	value;
@@ -299,7 +299,8 @@ db_watchpoint_cmd(
 
 /* list watchpoints */
 void
-db_listwatch_cmd(void)
+db_listwatch_cmd(__unused db_expr_t addr, __unused boolean_t have_addr,
+		 __unused db_expr_t count, __unused char *modif)
 {
 	db_list_watchpoints();
 }
diff --git a/osfmk/ddb/db_watch.h b/osfmk/ddb/db_watch.h
index 2d37eb46f..6ec420e36 100644
--- a/osfmk/ddb/db_watch.h
+++ b/osfmk/ddb/db_watch.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -136,19 +142,11 @@ typedef struct db_watchpoint {
 /* Prototypes for functions exported by this module.
  */
 
-void db_deletewatch_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_deletewatch_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_watchpoint_cmd(
-	db_expr_t	addr,
-	int		have_addr,
-	db_expr_t	count,
-	char *		modif);
+void db_watchpoint_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
-void db_listwatch_cmd(void);
+void db_listwatch_cmd(db_expr_t, boolean_t, db_expr_t, char *);
 
 void db_clear_watchpoints(void);
 
diff --git a/osfmk/ddb/db_write_cmd.c b/osfmk/ddb/db_write_cmd.c
index ad6a3c3e4..00b00e513 100644
--- a/osfmk/ddb/db_write_cmd.c
+++ b/osfmk/ddb/db_write_cmd.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -73,11 +79,8 @@
  * Write to file.
  */
 void
-db_write_cmd(
-	db_expr_t	address,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char *		modif)
+db_write_cmd(db_expr_t address, __unused boolean_t have_addr,
+	     __unused db_expr_t count, char *modif)
 {
 	register db_addr_t	addr;
 	register db_expr_t	old_value;
diff --git a/osfmk/ddb/db_write_cmd.h b/osfmk/ddb/db_write_cmd.h
index 21fa8ce65..1987c278b 100644
--- a/osfmk/ddb/db_write_cmd.h
+++ b/osfmk/ddb/db_write_cmd.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/makedis.c b/osfmk/ddb/makedis.c
index b397eb7c4..f25201a43 100644
--- a/osfmk/ddb/makedis.c
+++ b/osfmk/ddb/makedis.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/nlist.h b/osfmk/ddb/nlist.h
index 54d868455..a677d771a 100644
--- a/osfmk/ddb/nlist.h
+++ b/osfmk/ddb/nlist.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/orig/db_print.c b/osfmk/ddb/orig/db_print.c
index 59c51daa9..28aa7908e 100644
--- a/osfmk/ddb/orig/db_print.c
+++ b/osfmk/ddb/orig/db_print.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/stab.h b/osfmk/ddb/stab.h
index 328bd6d59..514ffb0a9 100644
--- a/osfmk/ddb/stab.h
+++ b/osfmk/ddb/stab.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/tr.c b/osfmk/ddb/tr.c
index b1f7b75ea..c33b0239c 100644
--- a/osfmk/ddb/tr.c
+++ b/osfmk/ddb/tr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ddb/tr.h b/osfmk/ddb/tr.h
index 4288e3de5..13ff98fde 100644
--- a/osfmk/ddb/tr.h
+++ b/osfmk/ddb/tr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/default_pager/Makefile b/osfmk/default_pager/Makefile
index b8e1ccbf7..b0c68f345 100644
--- a/osfmk/default_pager/Makefile
+++ b/osfmk/default_pager/Makefile
@@ -47,7 +47,8 @@ ${MIGINCLUDES} : ${MIG_TYPES}
 
 ${MIG_UUHDRS} : \
 	%.h : %.defs
-	$(MIG) $(MIGFLAGS)		\
+	@echo MIG $@
+	$(_v)$(MIG) $(MIGFLAGS)		\
 		-server /dev/null	\
 		-user /dev/null		\
 		-header $@ 		\
@@ -57,7 +58,8 @@ ${MIG_UUHDRS} : \
 
 ${MIG_USHDRS} : \
 	%_server.h : %.defs
-	$(MIG) $(MIGFLAGS)		\
+	@echo MIG $@
+	$(_v)$(MIG) $(MIGFLAGS)		\
 		-server /dev/null	\
 		-user /dev/null		\
 		-header /dev/null	\
@@ -111,7 +113,8 @@ ${COMP_FILES} : ${MIG_TYPES}
         
 ${MIG_KUSRC} : \
 	%_user.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
 		-user    $*_user.c              \
 		-header  $*.h              \
 		-server  /dev/null		\
@@ -122,7 +125,8 @@ ${MIG_KUSRC} : \
         
 ${MIG_KSSRC}: \
 	%_server.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
 		-user    /dev/null              \
 		-header  /dev/null              \
 		-server  $*_server.c		\
diff --git a/osfmk/default_pager/default_pager.c b/osfmk/default_pager/default_pager.c
index 9373974d6..429c4c601 100644
--- a/osfmk/default_pager/default_pager.c
+++ b/osfmk/default_pager/default_pager.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -123,8 +129,6 @@ void	default_pager_thread(void *);
 void	default_pager_initialize(void);
 boolean_t	dp_parse_argument(char *);	/* forward; */
 unsigned int	d_to_i(char *);			/* forward; */
-boolean_t       strprefix(register const char *s1, register const char *s2);
-
 
 extern int vstruct_def_clshift;
 
@@ -312,22 +316,6 @@ start_def_pager( __unused char *bs_device )
 	return (0);
 }
 
-/*
- * Return TRUE if string 2 is a prefix of string 1.
- */     
-boolean_t       
-strprefix(register const char *s1, register const char *s2)
-{               
-        register int    c;
-                
-        while ((c = *s2++) != '\0') {
-            if (c != *s1++) 
-                return (FALSE);
-        }       
-        return (TRUE);
-}
-
-
 kern_return_t
 default_pager_info(
 	memory_object_default_t	pager,
@@ -373,7 +361,7 @@ default_pager_info_64(
 
 
 void
-default_pager_initialize()
+default_pager_initialize(void)
 {
 	kern_return_t		kr;
 	__unused static char	here[] = "default_pager_initialize";
diff --git a/osfmk/default_pager/default_pager_alerts.defs b/osfmk/default_pager/default_pager_alerts.defs
index bbf143bd1..5629769c3 100644
--- a/osfmk/default_pager/default_pager_alerts.defs
+++ b/osfmk/default_pager/default_pager_alerts.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * File:	default_pager/default_pager_alerts.defs
@@ -41,3 +47,4 @@ simpleroutine	default_pager_space_alert(
 		alert_port		: mach_port_t;
 	in	flags			: int);
 
+/* vim: set ft=c : */
diff --git a/osfmk/default_pager/default_pager_internal.h b/osfmk/default_pager/default_pager_internal.h
index b01a94cae..0d9fb9c9a 100644
--- a/osfmk/default_pager/default_pager_internal.h
+++ b/osfmk/default_pager/default_pager_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -156,8 +162,8 @@ extern int	debug_mask;
 
 #else	/* DEFAULT_PAGER_DEBUG */
 
-#define DP_DEBUG(level, args)
-#define ASSERT(clause)
+#define DP_DEBUG(level, args) do {} while(0)
+#define ASSERT(clause) do {} while(0)
 
 #endif	/* DEFAULT_PAGER_DEBUG */
 
@@ -165,8 +171,6 @@ extern int	debug_mask;
 extern char *mach_error_string(kern_return_t);
 #endif
 
-#define MIN(a,b)	(((a) < (b)) ? (a) : (b))
-
 #define	PAGER_SUCCESS	0
 #define	PAGER_FULL	1
 #define	PAGER_ERROR	2
@@ -590,7 +594,7 @@ typedef struct vstruct {
 	unsigned int		vs_writers;	/* Writes in progress */
 
 #ifdef MACH_KERNEL
-	int
+	unsigned int
 	/* boolean_t */		vs_waiting_seqno:1,	/* to wait on seqno */
 	/* boolean_t */		vs_waiting_read:1, 	/* waiting on reader? */
 	/* boolean_t */		vs_waiting_write:1,	/* waiting on writer? */
@@ -791,8 +795,9 @@ extern boolean_t	bs_add_device(char *,
 extern vstruct_t	ps_vstruct_create(vm_size_t);
 extern void		ps_vstruct_dealloc(vstruct_t);
 extern kern_return_t	pvs_cluster_read(vstruct_t,
-					vm_offset_t,
-					vm_size_t);
+					 vm_offset_t,
+					 vm_size_t,
+					 void *);
 extern kern_return_t	vs_cluster_write(vstruct_t,
 					 upl_t,
 					 upl_offset_t,
diff --git a/osfmk/default_pager/default_pager_object.defs b/osfmk/default_pager/default_pager_object.defs
index 8caa3d652..18b84c8a8 100644
--- a/osfmk/default_pager/default_pager_object.defs
+++ b/osfmk/default_pager/default_pager_object.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -137,4 +143,4 @@ routine default_pager_info_64(
 		default_pager		: mach_port_t;
 	out	info			: default_pager_info_64_t);
 
-
+/* vim: set ft=c : */
diff --git a/osfmk/default_pager/default_pager_types.defs b/osfmk/default_pager/default_pager_types.defs
index 5abe8f5c9..7d98c1ebf 100644
--- a/osfmk/default_pager/default_pager_types.defs
+++ b/osfmk/default_pager/default_pager_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -44,3 +50,5 @@ type backing_store_info_t		= array[*:20] of integer_t;
 import <default_pager/default_pager_types.h>;
 
 #endif	/* _MACH_DEFAULT_PAGER_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/default_pager/default_pager_types.h b/osfmk/default_pager/default_pager_types.h
index 86ad227f6..2c37e4b04 100644
--- a/osfmk/default_pager/default_pager_types.h
+++ b/osfmk/default_pager/default_pager_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/default_pager/diag.h b/osfmk/default_pager/diag.h
index 72f9c184d..ad39f3b0e 100644
--- a/osfmk/default_pager/diag.h
+++ b/osfmk/default_pager/diag.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -29,7 +35,7 @@
 	((void) ((cond) ? 0 : panic("%sassertion: %s", my_name, # cond)))
 #endif
 #ifndef ASSERTIONS
-#define assert(cond)
+#define assert(cond) do {} while(0)
 #endif
 #endif
 
diff --git a/osfmk/default_pager/dp_backing_store.c b/osfmk/default_pager/dp_backing_store.c
index 60136048e..78c99073f 100644
--- a/osfmk/default_pager/dp_backing_store.c
+++ b/osfmk/default_pager/dp_backing_store.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -77,6 +83,7 @@
 #include <vm/vm_object.h>
 #include <vm/vm_protos.h>
 
+
 /* LP64todo - need large internal object support */
 
 /*
@@ -722,8 +729,8 @@ ps_delete(
 			error = vm_object_upl_request(transfer_object, 
 				(vm_object_offset_t)0, VM_SUPER_CLUSTER,
 				&upl, NULL, &count,
-				UPL_NO_SYNC | UPL_CLEAN_IN_PLACE
-					    | UPL_SET_INTERNAL);
+				UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_LITE | UPL_SET_INTERNAL);
+
 			if(error == KERN_SUCCESS) {
 				error = ps_vstruct_transfer_from_segment(
 							vs, ps, upl);
@@ -2509,52 +2516,88 @@ pvs_object_data_provided(
 
 }
 
+static memory_object_offset_t   last_start;
+static vm_size_t		last_length;
+
 kern_return_t
 pvs_cluster_read(
 	vstruct_t	vs,
 	vm_offset_t	vs_offset,
-	vm_size_t	cnt)
+	vm_size_t	cnt,
+        void		*fault_info)
 {
-	upl_t			upl;
 	kern_return_t		error = KERN_SUCCESS;
-	int					size;
+	unsigned int		size;
 	unsigned int		residual;
 	unsigned int		request_flags;
-	int					seg_index;
-	int					pages_in_cl;
+	int			seg_index;
+	int			pages_in_cl;
 	int	                cl_size;
 	int	                cl_mask;
-	int					cl_index;
-	int			xfer_size;
+	int			cl_index;
+	unsigned int		xfer_size;
+	vm_offset_t		orig_vs_offset;
 	vm_offset_t       ps_offset[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT];
 	paging_segment_t        psp[(VM_SUPER_CLUSTER / PAGE_SIZE) >> VSTRUCT_DEF_CLSHIFT];
 	struct clmap		clmap;
+	upl_t			upl;
+	unsigned int		page_list_count;
+	memory_object_offset_t	start;
 
 	pages_in_cl = 1 << vs->vs_clshift;
 	cl_size = pages_in_cl * vm_page_size;
 	cl_mask = cl_size - 1;
 
-	/*
-	 * This loop will be executed multiple times until the entire
-	 * request has been satisfied... if the request spans cluster
-	 * boundaries, the clusters will be checked for logical continunity,
-	 * if contiguous the I/O request will span multiple clusters, otherwise
-	 * it will be broken up into the minimal set of I/O's
-	 *
-	 * If there are holes in a request (either unallocated pages in a paging
-	 * segment or an unallocated paging segment), we stop
-	 * reading at the hole, inform the VM of any data read, inform
-	 * the VM of an unavailable range, then loop again, hoping to
-	 * find valid pages later in the requested range.  This continues until
-	 * the entire range has been examined, and read, if present.
-	 */
-
 #if	USE_PRECIOUS
-	request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_PRECIOUS | UPL_RET_ONLY_ABSENT;
+	request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_PRECIOUS | UPL_RET_ONLY_ABSENT | UPL_SET_LITE;
 #else
-	request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_RET_ONLY_ABSENT;
+	request_flags = UPL_NO_SYNC |  UPL_CLEAN_IN_PLACE | UPL_RET_ONLY_ABSENT | UPL_SET_LITE;
 #endif
+	cl_index = (vs_offset & cl_mask) / vm_page_size;
+
+        if ((ps_clmap(vs, vs_offset & ~cl_mask, &clmap, CL_FIND, 0, 0) == (vm_offset_t)-1) ||
+	    !CLMAP_ISSET(clmap, cl_index)) {
+	        /*
+		 * the needed page doesn't exist in the backing store...
+		 * we don't want to try to do any I/O, just abort the
+		 * page and let the fault handler provide a zero-fill
+		 */
+		if (cnt == 0) {
+			/*
+			 * The caller was just poking at us to see if
+			 * the page has been paged out.  No need to 
+			 * mess with the page at all.
+			 * Just let the caller know we don't have that page.
+			 */
+			return KERN_FAILURE;
+		}
+
+		page_list_count = 0;
+
+		memory_object_super_upl_request(vs->vs_control,	(memory_object_offset_t)vs_offset,
+						PAGE_SIZE, PAGE_SIZE, 
+						&upl, NULL, &page_list_count,
+						request_flags);
+
+		if (clmap.cl_error)
+		        upl_abort(upl, UPL_ABORT_ERROR);
+		else
+		        upl_abort(upl, UPL_ABORT_UNAVAILABLE);
+		upl_deallocate(upl);
 
+		return KERN_SUCCESS;
+	}
+
+	if (cnt == 0) {
+		/*
+		 * The caller was just poking at us to see if
+		 * the page has been paged out.  No need to 
+		 * mess with the page at all.
+		 * Just let the caller know we do have that page.
+		 */
+		return KERN_SUCCESS;
+	}
+		
 	assert(dp_encryption_inited);
 	if (dp_encryption) {
 		/*
@@ -2564,34 +2607,55 @@ pvs_cluster_read(
 		 */
 		request_flags |= UPL_ENCRYPT;
 	}
+	orig_vs_offset = vs_offset;
 
-	while (cnt && (error == KERN_SUCCESS)) {
-	        int     	ps_info_valid;
-		unsigned int	page_list_count;
+	start = (memory_object_offset_t)vs_offset;
+	assert(cnt != 0);
+	cnt = VM_SUPER_CLUSTER;
 
-		if((vs_offset & cl_mask) && 
-			(cnt > (VM_SUPER_CLUSTER - 
-				(vs_offset & cl_mask)))) {
+	/*
+	 * determine how big a speculative I/O we should try for...
+	 */
+	if (memory_object_cluster_size(vs->vs_control, &start, &cnt, (memory_object_fault_info_t)fault_info) == KERN_SUCCESS) {
+		assert(vs_offset >= (vm_offset_t) start &&
+		       vs_offset < (vm_offset_t) (start + cnt));
+	        vs_offset = (vm_offset_t)start;
+	} else
+	        cnt = PAGE_SIZE;
+
+	last_start = start;
+	last_length = cnt;
+
+	/*
+	 * This loop will be executed multiple times until the entire
+	 * range has been looked at or we issue an I/O... if the request spans cluster
+	 * boundaries, the clusters will be checked for logical continunity,
+	 * if contiguous the I/O request will span multiple clusters...
+	 * at most only 1 I/O will be issued... it will encompass the original offset
+	 */
+	while (cnt && error == KERN_SUCCESS) {
+	        int     ps_info_valid;
+
+		if ((vs_offset & cl_mask) && (cnt > (VM_SUPER_CLUSTER - (vs_offset & cl_mask)))) {
 			size = VM_SUPER_CLUSTER;
 			size -= vs_offset & cl_mask;
-	        } else if (cnt > VM_SUPER_CLUSTER) {
+	        } else if (cnt > VM_SUPER_CLUSTER)
 		        size = VM_SUPER_CLUSTER;
-		} else {
+		else
 		        size = cnt;
-		}
+
 		cnt -= size;
 
 		ps_info_valid = 0;
 		seg_index     = 0;
 
 		while (size > 0 && error == KERN_SUCCESS) {
-		        int           abort_size;
+		        unsigned int  abort_size;
 			int           failed_size;
 			int           beg_pseg;
 			int           beg_indx;
 			vm_offset_t   cur_offset;
 
-
 			if ( !ps_info_valid) {
 			        ps_offset[seg_index] = ps_clmap(vs, vs_offset & ~cl_mask, &clmap, CL_FIND, 0, 0);
 				psp[seg_index]       = CLMAP_PS(clmap);
@@ -2604,26 +2668,12 @@ pvs_cluster_read(
 				abort_size = cl_size - (vs_offset & cl_mask);
 				abort_size = MIN(abort_size, size);
 
-				page_list_count = 0;
-				memory_object_super_upl_request(
-					vs->vs_control,
-					(memory_object_offset_t)vs_offset,
-					abort_size, abort_size, 
-					&upl, NULL, &page_list_count,
-					request_flags);
-
-			        if (clmap.cl_error) {
- 				        upl_abort(upl, UPL_ABORT_ERROR);
-				} else {
-				        upl_abort(upl, UPL_ABORT_UNAVAILABLE);
-				}
-				upl_deallocate(upl);
-
-				size       -= abort_size;
-				vs_offset  += abort_size;
+				size      -= abort_size;
+				vs_offset += abort_size;
 
 				seg_index++;
 				ps_info_valid = 0;
+
 				continue;
 			}
 			cl_index = (vs_offset & cl_mask) / vm_page_size;
@@ -2637,24 +2687,8 @@ pvs_cluster_read(
 				abort_size += vm_page_size;
 			}
 			if (abort_size) {
-			        /*
-				 * Let VM system know about holes in clusters.
-				 */
-			        GSTAT(global_stats.gs_pages_unavail += atop_32(abort_size));
-
-				page_list_count = 0;
-				memory_object_super_upl_request(
-					vs->vs_control,
-					(memory_object_offset_t)vs_offset,
-					abort_size, abort_size, 
-					&upl, NULL, &page_list_count,
-					request_flags);
-
-				upl_abort(upl, UPL_ABORT_UNAVAILABLE);
-				upl_deallocate(upl);
-
-				size       -= abort_size;
-				vs_offset  += abort_size;
+				size      -= abort_size;
+				vs_offset += abort_size;
 
 				if (cl_index == pages_in_cl) {
 				        /*
@@ -2663,6 +2697,7 @@ pvs_cluster_read(
 					 */
 				        seg_index++;
 					ps_info_valid = 0;
+
 					continue;
 				}
 				if (size == 0)
@@ -2683,8 +2718,7 @@ pvs_cluster_read(
 			 */
 			for (xfer_size = 0; xfer_size < size; ) {
 
-			        while (cl_index < pages_in_cl 
-						&& xfer_size < size) {
+			        while (cl_index < pages_in_cl && xfer_size < size) {
 				        /*
 					 * accumulate allocated pages within 
 					 * a physical segment
@@ -2699,12 +2733,11 @@ pvs_cluster_read(
 					} else
 					        break;
 				}
-				if (cl_index < pages_in_cl 
-						|| xfer_size >= size) {
+				if (cl_index < pages_in_cl || xfer_size >= size) {
 				        /*
 					 * we've hit an unallocated page or 
-					 * the end of this request... go fire
-					 * the I/O
+					 * the end of this request... see if
+					 * it's time to fire the I/O
 					 */
 				        break;
 				}
@@ -2715,10 +2748,7 @@ pvs_cluster_read(
 				 */
 				seg_index++;
 				  
-				ps_offset[seg_index] = 
-					ps_clmap(vs, 
-						cur_offset & ~cl_mask, 
-						&clmap, CL_FIND, 0, 0);
+				ps_offset[seg_index] = ps_clmap(vs, cur_offset & ~cl_mask, &clmap, CL_FIND, 0, 0);
 				psp[seg_index] = CLMAP_PS(clmap);
 				ps_info_valid = 1;
 
@@ -2729,36 +2759,52 @@ pvs_cluster_read(
 					 * the one we're currently in, or it's 
 					 * in a different paging file, or
 					 * it hasn't been allocated....
-					 * we stop here and generate the I/O
+					 * we stop this run and go check
+					 * to see if it's time to fire the I/O
 					 */
 				        break;
 				}
 				/*
 				 * start with first page of the next physical
-				 *  segment
+				 * segment
 				 */
 				cl_index = 0;
 			}
-			if (xfer_size) {
+			if (xfer_size == 0) {
 			        /*
-				 * we have a contiguous range of allocated pages
-				 * to read from
+				 * no I/O to generate for this segment
 				 */
-				page_list_count = 0;
-			        memory_object_super_upl_request(vs->vs_control,
-					(memory_object_offset_t)vs_offset,
-					xfer_size, xfer_size, 
-					&upl, NULL, &page_list_count,
-					request_flags | UPL_SET_INTERNAL);
-
-				error = ps_read_file(psp[beg_pseg], 
-					upl, (upl_offset_t) 0, 
-					ps_offset[beg_pseg] + 
-						(beg_indx * vm_page_size), 
-					xfer_size, &residual, 0);
-			} else
 			        continue;
+			}
+			if (cur_offset <= orig_vs_offset) {
+			        /*
+				 * we've hit a hole in our speculative cluster
+				 * before the offset that we're really after...
+				 * don't issue the I/O since it doesn't encompass
+				 * the original offset and we're looking to only
+				 * pull in the speculative pages if they can be
+				 * made part of a single I/O
+				 */
+			        size      -= xfer_size;
+				vs_offset += xfer_size;
 
+				continue;
+			}
+			/*
+			 * we have a contiguous range of allocated pages
+			 * to read from that encompasses the original offset
+			 */
+			page_list_count = 0;
+			memory_object_super_upl_request(vs->vs_control, (memory_object_offset_t)vs_offset,
+							xfer_size, xfer_size, 
+							&upl, NULL, &page_list_count,
+							request_flags | UPL_SET_INTERNAL | UPL_NOBLOCK);
+
+			error = ps_read_file(psp[beg_pseg], 
+					     upl, (upl_offset_t) 0, 
+					     ps_offset[beg_pseg] + (beg_indx * vm_page_size), 
+					     xfer_size, &residual, 0);
+			
 			failed_size = 0;
 
 			/*
@@ -2779,65 +2825,64 @@ pvs_cluster_read(
 				 * supplied data is deallocated from the pager's
 				 *  address space.
 				 */
-			        pvs_object_data_provided(
-					vs, upl, vs_offset, xfer_size);
+			        pvs_object_data_provided(vs, upl, vs_offset, xfer_size);
 			} else {
 			        failed_size = xfer_size;
 
 				if (error == KERN_SUCCESS) {
-				        if ((signed) residual == xfer_size) {
-				        /*
-					 * If a read operation returns no error
-					 * and no data moved, we turn it into
-					 * an error, assuming we're reading at
-					 * or beyong EOF.
-					 * Fall through and error the entire
-					 * range.
-					 */
+				        if (residual == xfer_size) {
+					        /*
+						 * If a read operation returns no error
+						 * and no data moved, we turn it into
+						 * an error, assuming we're reading at
+						 * or beyong EOF.
+						 * Fall through and error the entire range.
+						 */
 					        error = KERN_FAILURE;
 					} else {
-				        /*
-					 * Otherwise, we have partial read. If
-					 * the part read is a integral number
-					 * of pages supply it. Otherwise round
-					 * it up to a page boundary, zero fill
-					 * the unread part, and supply it.
-					 * Fall through and error the remainder
-					 * of the range, if any.
-					 */
-					        int fill, lsize;
+					        /*
+						 * Otherwise, we have partial read. If
+						 * the part read is a integral number
+						 * of pages supply it. Otherwise round
+						 * it up to a page boundary, zero fill
+						 * the unread part, and supply it.
+						 * Fall through and error the remainder
+						 * of the range, if any.
+						 */
+					        int fill;
+						unsigned int lsize;
+
+						fill = residual & ~vm_page_size;
+						lsize = (xfer_size - residual) + fill;
 
-						fill = residual 
-							& ~vm_page_size;
-						lsize = (xfer_size - residual)
-									 + fill;
-						pvs_object_data_provided(
-							vs, upl, 
-							vs_offset, lsize);
+						pvs_object_data_provided(vs, upl, vs_offset, lsize);
 
 						if (lsize < xfer_size) {
-						        failed_size = 
-							    xfer_size - lsize;
+						        failed_size = xfer_size - lsize;
 							error = KERN_FAILURE;
 						}
 					}
 				} 
 			}
-			/*
-			 * If there was an error in any part of the range, tell
-			 * the VM. Note that error is explicitly checked again
-			 *  since it can be modified above.
-			 */
 			if (error != KERN_SUCCESS) {
+			        /*
+				 * There was an error in some part of the range, tell
+				 * the VM. Note that error is explicitly checked again
+				 * since it can be modified above.
+				 */
 				BS_STAT(psp[beg_pseg]->ps_bs,
-					psp[beg_pseg]->ps_bs->bs_pages_in_fail 
-						+= atop_32(failed_size));
+					psp[beg_pseg]->ps_bs->bs_pages_in_fail += atop_32(failed_size));
 			}
-			size       -= xfer_size;
-			vs_offset  += xfer_size;
+			/*
+			 * we've issued a single I/O that encompassed the original offset
+			 * at this point we either met our speculative request length or 
+			 * we ran into a 'hole' (i.e. page not present in the cluster, cluster
+			 * not present or not physically contiguous to the previous one), so
+			 * we're done issuing I/O at this point
+			 */
+			return (error);
 		}
-
-	} /* END while (cnt && (error == 0)) */
+	}
 	return error;
 }
 
@@ -2892,13 +2937,13 @@ vs_cluster_write(
 
 			request_flags = UPL_NOBLOCK |
 				UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | 
-				UPL_NO_SYNC | UPL_SET_INTERNAL;
+				UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE;
 		} else {
 			super_size = VM_SUPER_CLUSTER;
 
 			request_flags = UPL_NOBLOCK | UPL_CLEAN_IN_PLACE |
 				UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM | 
-				UPL_NO_SYNC | UPL_SET_INTERNAL;
+				UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE;
 		}
 
 		if (!dp_encryption_inited) {
@@ -3776,9 +3821,10 @@ ps_read_file(
 
 	f_offset = (vm_object_offset_t)(ps->ps_offset + offset);
 	
-	/* for transfer case we need to pass uploffset and flags */
-	error = vnode_pagein(ps->ps_vnode, 
-				   upl, upl_offset, f_offset, (vm_size_t)size, flags | UPL_NORDAHEAD, NULL);
+	/*
+	 * for transfer case we need to pass uploffset and flags
+	 */
+	error = vnode_pagein(ps->ps_vnode, upl, upl_offset, f_offset, (vm_size_t)size, flags, NULL);
 
 	/* The vnode_pagein semantic is somewhat at odds with the existing   */
 	/* device_read semantic.  Partial reads are not experienced at this  */
@@ -3821,9 +3867,7 @@ ps_write_file(
 		 */
 		upl_encrypt(upl, upl_offset, size);
 	}
-
-	if (vnode_pageout(ps->ps_vnode,
-				upl, upl_offset, f_offset, (vm_size_t)size, flags, NULL))
+	if (vnode_pageout(ps->ps_vnode,	upl, upl_offset, f_offset, (vm_size_t)size, flags, NULL))
 	        result = KERN_FAILURE;
 	else
 	        result = KERN_SUCCESS;
diff --git a/osfmk/default_pager/dp_memory_object.c b/osfmk/default_pager/dp_memory_object.c
index 77ebe0662..4690f5d3e 100644
--- a/osfmk/default_pager/dp_memory_object.c
+++ b/osfmk/default_pager/dp_memory_object.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -576,7 +582,8 @@ dp_memory_object_data_request(
 	memory_object_t		mem_obj,
 	memory_object_offset_t	offset,
 	vm_size_t		length,
-	__unused vm_prot_t		protection_required)
+	__unused vm_prot_t	protection_required,
+        memory_object_fault_info_t	fault_info)
 {
 	vstruct_t		vs;
 
@@ -627,7 +634,7 @@ dp_memory_object_data_request(
 	if ((offset & vm_page_mask) != 0 || (length & vm_page_mask) != 0)
 		Panic("bad alignment");
 
-	pvs_cluster_read(vs, (vm_offset_t)offset, length);
+	pvs_cluster_read(vs, (vm_offset_t)offset, length, fault_info);
 
 	vs_finish_read(vs);
 
diff --git a/osfmk/device/Makefile b/osfmk/device/Makefile
index f856daeed..817a680d8 100644
--- a/osfmk/device/Makefile
+++ b/osfmk/device/Makefile
@@ -43,7 +43,8 @@ COMP_FILES	= ${DEVICE_FILES}
 .ORDER: ${DEVICE_FILES}
 
 ${DEVICE_FILES}: device.defs
-	${MIG} ${MIGFLAGS} ${MIGKSFLAGS}	\
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKSFLAGS}	\
 	-header /dev/null			\
 	-user /dev/null				\
 	-sheader device_server.h		\
@@ -51,7 +52,6 @@ ${DEVICE_FILES}: device.defs
 	$<
 
 
+
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/osfmk/device/device.defs b/osfmk/device/device.defs
index 54e4f0f70..32052fec1 100644
--- a/osfmk/device/device.defs
+++ b/osfmk/device/device.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -67,6 +73,10 @@ subsystem
 #include <mach/clock_types.defs>
 #include <mach/clock_types.defs>
 
+#if !__LP64__
+#    define __ILP32__ 1
+#endif
+
 import <device/device_types.h>;
 
 serverprefix	is_;
@@ -76,13 +86,34 @@ type reply_port_t = MACH_MSG_TYPE_MAKE_SEND_ONCE | polymorphic
 
 #if	IOKIT
 
-type io_name_t          = c_string[*:128];
-type io_string_t        = c_string[*:512];
-type io_struct_inband_t = array[*:4096] of char;
-type io_scalar_inband_t = array[*:16] of int;
-type io_async_ref_t 	= array[*:8] of natural_t;
-type io_buf_ptr_t	= ^array[] of MACH_MSG_TYPE_INTEGER_8;
-type NDR_record_t       = struct[8] of char;
+type io_name_t            = c_string[*:128];
+type io_string_t          = c_string[*:512];
+type io_struct_inband_t   = array[*:4096] of char;
+type io_buf_ptr_t	  = ^array[] of MACH_MSG_TYPE_INTEGER_8;
+type NDR_record_t         = struct[8] of char;
+
+#if KERNEL
+type io_user_scalar_t     = uint64_t;
+type io_user_reference_t  = uint64_t;
+type io_scalar_inband_t   = array[*:16] of int;
+type io_async_ref_t 	  = array[*:8]  of natural_t;
+type io_scalar_inband64_t = array[*:16] of io_user_scalar_t;
+type io_async_ref64_t 	  = array[*:8]  of io_user_reference_t;
+#elif __LP64__
+type io_user_scalar_t     = uint64_t;
+type io_user_reference_t  = uint64_t;
+type io_scalar_inband_t   = array[*:16] of io_user_scalar_t;
+type io_async_ref_t 	  = array[*:8]  of io_user_reference_t;
+type io_scalar_inband64_t = array[*:16] of io_user_scalar_t;
+type io_async_ref64_t 	  = array[*:8]  of io_user_reference_t;
+#else
+type io_user_scalar_t     = int;
+type io_user_reference_t  = natural_t;
+type io_scalar_inband_t   = array[*:16] of io_user_scalar_t;
+type io_async_ref_t 	  = array[*:8]  of io_user_reference_t;
+type io_scalar_inband64_t = array[*:16] of uint64_t;
+type io_async_ref64_t 	  = array[*:8]  of uint64_t;
+#endif // __LP64__
 
 type io_object_t = mach_port_t
 		ctype: mach_port_t
@@ -137,7 +168,7 @@ routine io_registry_entry_get_property(
 routine io_registry_create_iterator(
 	    master_port		: mach_port_t;
 	in  plane		: io_name_t;
-	in  options		: int;
+	in  options		: uint32_t;
 	out iterator		: io_object_t
 	);
 
@@ -186,7 +217,7 @@ routine io_registry_entry_get_parent_iterator(
 routine io_service_open(
 	    service		: io_object_t;
 	in  owningTask		: task_t;
-	in  connect_type	: int;
+	in  connect_type	: uint32_t;
 	out connection		: io_connect_t
 	);
 
@@ -199,21 +230,26 @@ routine io_connect_get_service(
 	out service		: io_object_t
 	);
 
+#if  KERNEL_SERVER || __ILP32__
 routine io_connect_set_notification_port(
 	    connection		: io_connect_t;
-	in  notification_type	: int;
+	in  notification_type	: uint32_t;
 	in  port		: mach_port_make_send_t;
-	in  reference		: int
+	in  reference		: uint32_t
 	);
 
 routine io_connect_map_memory(
 	    connection		: io_connect_t;
-	in  memory_type		: int;
+	in  memory_type		: uint32_t;
 	in  into_task		: task_t;
 	inout address		: vm_address_t;
 	inout size		: vm_size_t;
-	in  flags		: int
+	in  flags		: uint32_t
 	);
+#else
+skip;
+skip;
+#endif
 
 routine io_connect_add_client(
 	    connection		: io_connect_t;
@@ -223,37 +259,43 @@ routine io_connect_add_client(
 routine io_connect_set_properties(
 	    connection		: io_connect_t;
 	in  properties		: io_buf_ptr_t, physicalcopy;
-        out result		: natural_t
+        out result		: kern_return_t
 	);
 
-
+#if  KERNEL_SERVER || (__ILP32__ && !MAP_32B_METHODS)
 routine io_connect_method_scalarI_scalarO(
 	    connection		: io_connect_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	out output		: io_scalar_inband_t, CountInOut
 	);
 
 routine io_connect_method_scalarI_structureO(
 	    connection		: io_connect_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	out output		: io_struct_inband_t, CountInOut
 	);
 
 routine io_connect_method_scalarI_structureI(
 	    connection		: io_connect_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	in  inputStruct		: io_struct_inband_t
 	);
 
 routine io_connect_method_structureI_structureO(
 	    connection		: io_connect_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_struct_inband_t;
 	out output		: io_struct_inband_t, CountInOut
 	);
+#else
+skip;
+skip;
+skip;
+skip;
+#endif
 
 routine io_registry_entry_get_path(
 	    registry_entry	: io_object_t;
@@ -269,7 +311,7 @@ routine io_registry_get_root_entry(
 routine io_registry_entry_set_properties(
 	    registry_entry	: io_object_t;
 	in  properties		: io_buf_ptr_t, physicalcopy;
-        out result		: natural_t
+        out result		: kern_return_t
 	);
 
 routine io_registry_entry_in_plane(
@@ -280,12 +322,12 @@ routine io_registry_entry_in_plane(
 
 routine io_object_get_retain_count(
 	    object		: io_object_t;
-	out retainCount		: int
+	out retainCount		: uint32_t
 	);
 
 routine io_service_get_busy_state(
 	    service		: io_object_t;
-	out busyState		: int
+	out busyState		: uint32_t
 	);
 
 routine io_service_wait_quiet(
@@ -296,7 +338,7 @@ routine io_service_wait_quiet(
 routine io_registry_entry_create_iterator(
 	    registry_entry	: io_object_t;
 	in  plane		: io_name_t;
-	in  options		: int;
+	in  options		: uint32_t;
 	out iterator		: io_object_t
 	);
 
@@ -307,34 +349,34 @@ routine io_iterator_is_valid(
 
 routine io_make_matching(
 	    master_port		: mach_port_t;
-	in  of_type		: int;
-	in  options		: int;
+	in  of_type		: uint32_t;
+	in  options		: uint32_t;
 	in  input		: io_struct_inband_t;
 	out matching		: io_string_t
 	);
 
 routine io_catalog_send_data(
 	    master_port		: mach_port_t;
-        in  flag                : int;
+        in  flag                : uint32_t;
 	in  inData		: io_buf_ptr_t;
-        out result		: natural_t
+        out result		: kern_return_t
 	);
 
 routine io_catalog_terminate(
 	    master_port		: mach_port_t;
-        in  flag                : int;
+        in  flag                : uint32_t;
 	in  name		: io_name_t
 	);
 
 routine io_catalog_get_data(
 	    master_port		: mach_port_t;
-        in  flag                : int;
+        in  flag                : uint32_t;
 	out outData		: io_buf_ptr_t
 	);
 
 routine io_catalog_get_gen_count(
 	    master_port		: mach_port_t;
-        out genCount            : int
+        out genCount            : uint32_t
 	);
 
 routine io_catalog_module_loaded(
@@ -344,12 +386,12 @@ routine io_catalog_module_loaded(
 
 routine io_catalog_reset(
 	    master_port		: mach_port_t;
-	in  flag		: int
+	in  flag		: uint32_t
 	);
 
 routine io_service_request_probe(
 	    service		: io_object_t;
-	in  options		: int
+	in  options		: uint32_t
 	);
 
 routine io_registry_entry_get_name_in_plane(
@@ -364,42 +406,47 @@ routine io_service_match_property_table(
 	out matches		: boolean_t
 	);
 
+#if  KERNEL_SERVER || (__ILP32__ && !MAP_32B_ASYNC_METHODS)
 routine io_async_method_scalarI_scalarO(
 	    connection		: io_connect_t;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	out output		: io_scalar_inband_t, CountInOut
 	);
-
 routine io_async_method_scalarI_structureO(
 	    connection		: io_connect_t;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	out output		: io_struct_inband_t, CountInOut
 	);
-
 routine io_async_method_scalarI_structureI(
 	    connection		: io_connect_t;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_scalar_inband_t;
 	in  inputStruct		: io_struct_inband_t
 	);
-
 routine io_async_method_structureI_structureO(
 	    connection		: io_connect_t;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
-	in  selector		: int;
+	in  selector		: uint32_t;
 	in  input		: io_struct_inband_t;
 	out output		: io_struct_inband_t, CountInOut
 	);
+#else
+skip;
+skip;
+skip;
+skip;
+#endif
 
+#if KERNEL_SERVER || __ILP32__
 routine io_service_add_notification(
 	    master_port		: mach_port_t;
 	in  notification_type	: io_name_t;
@@ -408,20 +455,23 @@ routine io_service_add_notification(
 	in  reference		: io_async_ref_t;
 	out notification	: io_object_t
 	);
-
 routine io_service_add_interest_notification(
 	    service		: io_object_t;
 	in  type_of_interest	: io_name_t;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
 	out notification	: io_object_t
-	);
-
+        );
 routine io_service_acknowledge_notification(
-	    service		: io_object_t;
-	in  notify_ref		: natural_t;
-	in  response		: natural_t
-	);
+           service             : io_object_t;
+       in  notify_ref          : natural_t;
+       in  response            : natural_t
+        );
+#else
+skip;
+skip;
+skip;
+#endif
 
 routine io_connect_get_notification_semaphore(
 	    connection		: io_connect_t;
@@ -429,12 +479,16 @@ routine io_connect_get_notification_semaphore(
 	out semaphore		: semaphore_t
 	);
 
+#if KERNEL_SERVER || __ILP32__
 routine io_connect_unmap_memory(
 	    connection		: io_connect_t;
-	in  memory_type		: int;
+	in  memory_type		: uint32_t;
 	in  into_task		: task_t;
 	in  address		: vm_address_t
 	);
+#else
+skip;
+#endif
 
 routine io_registry_entry_get_location_in_plane(
 	    registry_entry	: io_object_t;
@@ -446,7 +500,7 @@ routine io_registry_entry_get_property_recursively(
 	    registry_entry	: io_object_t;
 	in  plane		: io_name_t;
 	in  property_name	: io_name_t;
-	in  options		: int;
+	in  options		: uint32_t;
 	out properties		: io_buf_ptr_t, physicalcopy
 	);
 
@@ -459,26 +513,30 @@ routine io_service_get_state(
 routine io_service_get_matching_services_ool(
 	    master_port		: mach_port_t;
 	in  matching		: io_buf_ptr_t, physicalcopy;
-        out result		: natural_t;
+        out result		: kern_return_t;
 	out existing		: io_object_t
 	);
 
 routine io_service_match_property_table_ool(
 	    service		: io_object_t;
 	in  matching		: io_buf_ptr_t, physicalcopy;
-        out result		: natural_t;
+        out result		: kern_return_t;
 	out matches		: boolean_t
 	);
 
+#if KERNEL_SERVER || __ILP32__
 routine io_service_add_notification_ool(
 	    master_port		: mach_port_t;
 	in  notification_type	: io_name_t;
 	in  matching		: io_buf_ptr_t, physicalcopy;
 	in  wake_port		: mach_port_make_send_t;
 	in  reference		: io_async_ref_t;
-        out result		: natural_t;
+        out result		: kern_return_t;
 	out notification	: io_object_t
 	);
+#else
+skip;
+#endif
 
 routine io_object_get_superclass(
 	    master_port		: mach_port_t;
@@ -495,13 +553,110 @@ routine io_object_get_bundle_identifier(
 routine io_service_open_extended(
 	    service		: io_object_t;
 	in  owningTask		: task_t;
-	in  connect_type	: int;
+	in  connect_type	: uint32_t;
 	in  ndr			: NDR_record_t;
 	in  properties		: io_buf_ptr_t, physicalcopy;
-        out result		: natural_t;
+        out result		: kern_return_t;
 	out connection		: io_connect_t
 	);
 
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+routine io_connect_map_memory_into_task(
+	    connection		: io_connect_t;
+	in  memory_type		: uint32_t;
+	in  into_task		: task_t;
+	inout address		: mach_vm_address_t;
+	inout size		: mach_vm_size_t;
+	in  flags		: uint32_t
+	);
+
+routine io_connect_unmap_memory_from_task(
+	    connection		: io_connect_t;
+	in  memory_type		: uint32_t;
+	in  from_task		: task_t;
+	in  address		: mach_vm_address_t
+	);
+
+routine io_connect_method(
+	    connection		: io_connect_t;
+	in  selector		: uint32_t;
+
+	in  scalar_input	: io_scalar_inband64_t;
+	in  inband_input	: io_struct_inband_t;
+	in  ool_input		: mach_vm_address_t;
+	in  ool_input_size	: mach_vm_size_t;
+
+	out scalar_output	: io_scalar_inband64_t, CountInOut;
+	out inband_output	: io_struct_inband_t, CountInOut;
+	in  ool_output		: mach_vm_address_t;
+	inout ool_output_size	: mach_vm_size_t
+	);
+
+routine io_connect_async_method(
+	    connection		: io_connect_t;
+	in  wake_port		: mach_port_make_send_t;
+	in  reference		: io_async_ref64_t;
+	in  selector		: uint32_t;
+
+	in  scalar_input	: io_scalar_inband64_t;
+	in  inband_input	: io_struct_inband_t;
+	in  ool_input		: mach_vm_address_t;
+	in  ool_input_size	: mach_vm_size_t;
+
+	out scalar_output	: io_scalar_inband64_t, CountInOut;
+	out inband_output	: io_struct_inband_t, CountInOut;
+	in  ool_output		: mach_vm_address_t;
+	inout  ool_output_size	: mach_vm_size_t
+	);
+
+
+#if KERNEL_SERVER || __LP64__
+
+#if KERNEL_SERVER
+#define FUNC_NAME(name) name ## _64
+#else
+#define FUNC_NAME(name) name
 #endif
 
+routine FUNC_NAME(io_connect_set_notification_port)(
+	    connection		: io_connect_t;
+	in  notification_type	: uint32_t;
+	in  port		: mach_port_make_send_t;
+	in  reference		: io_user_reference_t
+	);
+
+routine FUNC_NAME(io_service_add_notification)(
+	    master_port		: mach_port_t;
+	in  notification_type	: io_name_t;
+	in  matching		: io_string_t;
+	in  wake_port		: mach_port_make_send_t;
+	in  reference		: io_async_ref64_t;
+	out notification	: io_object_t
+	);
+
+routine FUNC_NAME(io_service_add_interest_notification)(
+	    service		: io_object_t;
+	in  type_of_interest	: io_name_t;
+	in  wake_port		: mach_port_make_send_t;
+	in  reference		: io_async_ref64_t;
+	out notification	: io_object_t
+	);
+
+routine FUNC_NAME(io_service_add_notification_ool)(
+	    master_port		: mach_port_t;
+	in  notification_type	: io_name_t;
+	in  matching		: io_buf_ptr_t, physicalcopy;
+	in  wake_port		: mach_port_make_send_t;
+	in  reference		: io_async_ref64_t;
+        out result		: kern_return_t;
+	out notification	: io_object_t
+	);
+
+
+#endif /* KERNEL_SERVER || __LP64__ */
+
+#endif /* IOKIT */
 
+/* vim: set ft=c : */
diff --git a/osfmk/device/device_init.c b/osfmk/device/device_init.c
index 55904f4e2..05e76bc0e 100644
--- a/osfmk/device/device_init.c
+++ b/osfmk/device/device_init.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/device/device_port.h b/osfmk/device/device_port.h
index 13a285a82..8948ed53f 100644
--- a/osfmk/device/device_port.h
+++ b/osfmk/device/device_port.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/device/device_types.defs b/osfmk/device/device_types.defs
index b499e8d8a..f88b7ae33 100644
--- a/osfmk/device/device_types.defs
+++ b/osfmk/device/device_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -97,3 +103,5 @@ type io_done_queue_t	= mach_port_t
 import <device/device_types.h>;
 
 #endif	/* _DEVICE_DEVICE_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/device/device_types.h b/osfmk/device/device_types.h
index a3245ee1c..dcbd7940d 100644
--- a/osfmk/device/device_types.h
+++ b/osfmk/device/device_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -77,11 +83,32 @@ typedef char *	io_buf_ptr_t;
 #ifdef IOKIT
 
 /* must match device_types.defs */
-typedef	char		io_name_t[128];
-typedef	char		io_string_t[512];	
-typedef char 		io_struct_inband_t[4096];
-typedef int		io_scalar_inband_t[16];
-typedef natural_t	io_async_ref_t[8];
+typedef	char			io_name_t[128];
+typedef	char			io_string_t[512];	
+typedef char 			io_struct_inband_t[4096];
+
+#if KERNEL
+typedef uint64_t		io_user_scalar_t;
+typedef uint64_t		io_user_reference_t;
+typedef int 			io_scalar_inband_t[16];
+typedef natural_t		io_async_ref_t[8];
+typedef io_user_scalar_t	io_scalar_inband64_t[16];
+typedef io_user_reference_t	io_async_ref64_t[8];
+#elif __LP64__
+typedef uint64_t		io_user_scalar_t;
+typedef uint64_t		io_user_reference_t;
+typedef io_user_scalar_t	io_scalar_inband_t[16];
+typedef io_user_reference_t	io_async_ref_t[8];
+typedef io_user_scalar_t	io_scalar_inband64_t[16];
+typedef io_user_reference_t	io_async_ref64_t[8];
+#else
+typedef int			io_user_scalar_t;
+typedef natural_t		io_user_reference_t;
+typedef io_user_scalar_t	io_scalar_inband_t[16];
+typedef io_user_reference_t	io_async_ref_t[8];
+typedef uint64_t		io_scalar_inband64_t[16];
+typedef uint64_t		io_async_ref64_t[8];
+#endif // __LP64__
 
 #ifdef MACH_KERNEL
 
diff --git a/osfmk/device/iokit_rpc.c b/osfmk/device/iokit_rpc.c
index 20622aa84..667c1323f 100644
--- a/osfmk/device/iokit_rpc.c
+++ b/osfmk/device/iokit_rpc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach_kdb.h>
 #include <zone_debug.h>
@@ -29,7 +35,7 @@
 #include <mach/port.h>
 #include <mach/vm_param.h>
 #include <mach/notify.h>
-#include <mach/mach_host_server.h>
+//#include <mach/mach_host_server.h>
 #include <mach/mach_types.h>
 
 #include <machine/machparam.h>		/* spl definitions */
@@ -91,6 +97,43 @@ iokit_client_memory_for_type(
 
 extern ppnum_t IOGetLastPageNumber(void);
 
+/*
+ * Functions imported by iokit:IOUserClient.cpp
+ */
+
+extern ipc_port_t iokit_alloc_object_port( io_object_t obj,
+			ipc_kobject_type_t type );
+
+extern kern_return_t iokit_destroy_object_port( ipc_port_t port );
+
+extern mach_port_name_t iokit_make_send_right( task_t task,
+				io_object_t obj, ipc_kobject_type_t type );
+
+extern kern_return_t iokit_mod_send_right( task_t task, mach_port_name_t name, mach_port_delta_t delta );
+
+extern io_object_t iokit_lookup_connect_ref(io_object_t clientRef, ipc_space_t task);
+
+extern io_object_t iokit_lookup_connect_ref_current_task(io_object_t clientRef);
+
+extern void iokit_retain_port( ipc_port_t port );
+extern void iokit_release_port( ipc_port_t port );
+
+extern kern_return_t iokit_switch_object_port( ipc_port_t port, io_object_t obj, ipc_kobject_type_t type );
+
+/*
+ * Functions imported by iokit:IOMemoryDescriptor.cpp
+ */
+
+extern kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t pa,
+                                 mach_vm_size_t length, unsigned int mapFlags);
+
+extern kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t length);
+
+extern kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
+					mach_vm_size_t length, unsigned int options);
+
+extern unsigned int IODefaultCacheBits(addr64_t pa);
+
 /*
  * Lookup a device by its port.
  * Doesn't consume the naked send right; produces a device reference.
@@ -231,10 +274,6 @@ iokit_make_connect_port(
     return( sendPort);
 }
 
-
-EXTERN ipc_port_t
-iokit_alloc_object_port( io_object_t obj, ipc_kobject_type_t type );
-
 int gIOKitPortCount;
 
 EXTERN ipc_port_t
@@ -338,7 +377,7 @@ iokit_no_senders( mach_no_senders_notification_t * notification )
 {
     ipc_port_t		port;
     io_object_t		obj = NULL;
-    ipc_kobject_type_t	type;
+    ipc_kobject_type_t	type = IKOT_NONE;
     ipc_port_t		notify;
 
     port = (ipc_port_t) notification->not_header.msgh_remote_port;
@@ -389,7 +428,7 @@ iokit_notify( mach_msg_header_t * msg )
         case MACH_NOTIFY_SEND_ONCE:
         case MACH_NOTIFY_DEAD_NAME:
         default:
-            printf("iokit_notify: strange notification %ld\n", msg->msgh_id);
+            printf("iokit_notify: strange notification %d\n", msg->msgh_id);
             return FALSE;
     }
 }
@@ -397,8 +436,7 @@ iokit_notify( mach_msg_header_t * msg )
 /* need to create a pmap function to generalize */
 unsigned int IODefaultCacheBits(addr64_t pa)
 {
-
-      return(pmap_cache_attributes(pa >> PAGE_SHIFT));
+	return(pmap_cache_attributes(pa >> PAGE_SHIFT));
 }
 
 kern_return_t IOMapPages(vm_map_t map, mach_vm_address_t va, mach_vm_address_t pa,
@@ -450,9 +488,13 @@ kern_return_t IOUnmapPages(vm_map_t map, mach_vm_address_t va, mach_vm_size_t le
     return( KERN_SUCCESS );
 }
 
-kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
-					mach_vm_size_t length, unsigned int options)
+kern_return_t IOProtectCacheMode(vm_map_t __unused map, mach_vm_address_t __unused va,
+					mach_vm_size_t __unused length, unsigned int __unused options)
 {
+#if __ppc__
+    // can't remap block mappings, but ppc doesn't speculatively read from WC
+#else
+
     mach_vm_size_t off;
     vm_prot_t	   prot;
     unsigned int   flags;
@@ -463,7 +505,7 @@ kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
 
     switch (options & kIOMapCacheMask)
     {
-    /* What cache mode do we need? */
+	// what cache mode do we need?
 	case kIOMapDefaultCache:
 	default:
 	    return (KERN_INVALID_ARGUMENT);
@@ -484,9 +526,6 @@ kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
 	    flags = VM_WIMG_COPYBACK;
 	    break;
     }
-#if __ppc__
-    // can't remap block mappings, but ppc doesn't speculative read from WC
-#else
 
     //  enter each page's physical address in the target map
     for (off = 0; off < length; off += page_size)
@@ -504,17 +543,19 @@ kern_return_t IOProtectCacheMode(vm_map_t map, mach_vm_address_t va,
 ppnum_t IOGetLastPageNumber(void)
 {
     ppnum_t	 lastPage, highest = 0;
+    unsigned int idx;
 
 #if __ppc__
-    int idx;
     for (idx = 0; idx < pmap_mem_regions_count; idx++)
     {
 	lastPage = pmap_mem_regions[idx].mrEnd;
 #elif __i386__
-    unsigned int idx;
     for (idx = 0; idx < pmap_memory_region_count; idx++)
     {
 	lastPage = pmap_memory_regions[idx].end - 1;
+#elif __arm__
+    if (0) /* XXX */
+    {
 #else
 #error arch
 #endif
@@ -528,6 +569,6 @@ ppnum_t IOGetLastPageNumber(void)
 void IOGetTime( mach_timespec_t * clock_time);
 void IOGetTime( mach_timespec_t * clock_time)
 {
-	clock_get_system_nanotime(&clock_time->tv_sec, &clock_time->tv_nsec);
+	clock_get_system_nanotime(&clock_time->tv_sec, (uint32_t *) &clock_time->tv_nsec);
 }
 
diff --git a/osfmk/device/subrs.c b/osfmk/device/subrs.c
index 3fa97189b..b9aafe509 100644
--- a/osfmk/device/subrs.c
+++ b/osfmk/device/subrs.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -89,12 +95,48 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
+
+/*
+ * Copyright (c) 1998 Todd C. Miller <Todd.Miller@courtesan.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+ * THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  * Random device subroutines and stubs.
  */
 
 #include <vm/vm_kern.h>
 #include <kern/misc_protos.h>
+#include <libsa/stdlib.h>
+#include <sys/malloc.h>
 
 /* String routines, from CMU */
 #ifdef	strcpy
@@ -115,14 +157,16 @@
  *      It returns < 0 if the first differing character is smaller
  *      in s1 than in s2 or if s1 is shorter than s2 and the
  *      contents are identical upto the length of s1.
+ * Deprecation Warning:
+ *	strcmp() is being deprecated. Please use strncmp() instead.
  */
 
 int
 strcmp(
-        register const char *s1,
-        register const char *s2)
+        const char *s1,
+        const char *s2)
 {
-        register unsigned int a, b;
+        unsigned int a, b;
 
         do {
                 a = *s1++;
@@ -145,11 +189,11 @@ strcmp(
 
 int
 strncmp(
-        register const char *s1,
-        register const char *s2,
+        const char *s1,
+        const char *s2,
         size_t n)
 {
-        register unsigned int a, b;
+        unsigned int a, b;
 
         while (n != 0) {
                 a = *s1++;
@@ -214,14 +258,16 @@ strncasecmp(const char *s1, const char *s2, size_t n)
  *      strcpy copies the contents of the string "from" including
  *      the null terminator to the string "to". A pointer to "to"
  *      is returned.
+ * Deprecation Warning: 
+ *	strcpy() is being deprecated. Please use strlcpy() instead.
  */
 
 char *
 strcpy(
-        register char *to,
-        register const char *from)
+        char *to,
+        const char *from)
 {
-        register char *ret = to;
+        char *ret = to;
 
         while ((*to++ = *from++) != '\0')
                 continue;
@@ -267,8 +313,7 @@ strncpy(
  */
 
 int
-atoi(
-	u_char  *cp)
+atoi(const char *cp)
 {
         int     number;
 
@@ -295,8 +340,8 @@ atoi_term(
 	char	*p,	/* IN */
 	char	**t)	/* OUT */
 {
-        register int n;
-        register int f;
+        int n;
+        int f;
 
         n = 0;
         f = 0;
@@ -328,10 +373,10 @@ atoi_term(
  * Taken from archive/kern-stuff/sbf_machine.c in 
  * seatbelt. 
  * inputs:
- *     s       string whose length is to be measured
- *     max     maximum length of string to search for null
+ * 	s	string whose length is to be measured
+ *	max	maximum length of string to search for null
  * outputs:
- *     length of s or max; whichever is smaller
+ *	length of s or max; whichever is smaller
  */
 size_t 
 strnlen(const char *s, size_t max) {
@@ -342,8 +387,6 @@ strnlen(const char *s, size_t max) {
 	return p - s;
 }
 
-
-
 /*
  * convert an integer to an ASCII string.
  * inputs:
@@ -360,8 +403,8 @@ itoa(
 	char	*str)
 {
         char    digits[11];
-        register char *dp;
-        register char *cp = str;
+        char *dp;
+        char *cp = str;
 
         if (num == 0) {
             *cp++ = '0';
@@ -381,17 +424,143 @@ itoa(
 	return str;
 }
 
+/* 
+ * Deprecation Warning:
+ *	strcat() is being deprecated. Please use strlcat() instead.
+ */
 char *
 strcat(
-	register char *dest,
-	register const char *src)
+	char *dest,
+	const char *src)
 {
 	char *old = dest;
 
 	while (*dest)
 		++dest;
-	while (*dest++ = *src++)
+	while ((*dest++ = *src++))
 		;
 	return (old);
 }
 
+/*
+ * Appends src to string dst of size siz (unlike strncat, siz is the
+ * full size of dst, not space left).  At most siz-1 characters
+ * will be copied.  Always NUL terminates (unless siz <= strlen(dst)).
+ * Returns strlen(src) + MIN(siz, strlen(initial dst)).
+ * If retval >= siz, truncation occurred.
+ */
+size_t
+strlcat(char *dst, const char *src, size_t siz)
+{
+	char *d = dst;
+	const char *s = src;
+	size_t n = siz;
+	size_t dlen;
+
+	/* Find the end of dst and adjust bytes left but don't go past end */
+	while (n-- != 0 && *d != '\0')
+		d++;
+	dlen = d - dst;
+	n = siz - dlen;
+
+	if (n == 0)
+		return(dlen + strlen(s));
+	while (*s != '\0') {
+		if (n != 1) {
+			*d++ = *s;
+			n--;
+		}
+		s++;
+	}
+	*d = '\0';
+
+	return(dlen + (s - src));       /* count does not include NUL */
+}
+
+/*
+ * Copy src to string dst of size siz.  At most siz-1 characters
+ * will be copied.  Always NUL terminates (unless siz == 0).
+ * Returns strlen(src); if retval >= siz, truncation occurred.
+ */
+size_t
+strlcpy(char *dst, const char *src, size_t siz)
+{
+	char *d = dst;
+	const char *s = src;
+	size_t n = siz;
+
+	/* Copy as many bytes as will fit */
+	if (n != 0 && --n != 0) {
+		do {
+			if ((*d++ = *s++) == 0)
+				break;
+		} while (--n != 0);
+	}
+
+	/* Not enough room in dst, add NUL and traverse rest of src */
+	if (n == 0) {
+		if (siz != 0)
+			*d = '\0';		/* NUL-terminate dst */
+		while (*s++)
+			;
+	}
+
+	return(s - src - 1);	/* count does not include NUL */
+}
+
+/*
+ * STRDUP
+ *
+ * Description: The STRDUP function allocates sufficient memory for a copy
+ *              of the string "string", does the copy, and returns a pointer
+ *              it. The pointer may subsequently be used as an argument to
+ *              the macro FREE().
+ *
+ * Parameters:  string		String to be duplicated
+ *              type		type of memory to be allocated (normally
+ *              		M_TEMP)
+ *
+ * Returns:     char *          A pointer to the newly allocated string with
+ *                              duplicated contents in it.
+ *
+ *              NULL		If MALLOC() fails.
+ *
+ * Note:        This function can *not* be called from interrupt context as
+ *              it calls MALLOC with M_WAITOK.  In fact, you really
+ *              shouldn't be doing string manipulation in interrupt context
+ *              ever.
+ *
+ *              This function name violates the kernel style(9) guide
+ *              by being all caps.  This was done on purpose to emphasize
+ *              one should use FREE() with the allocated buffer.
+ *
+ */
+inline char *
+STRDUP(const char *string, int type)
+{
+	size_t len;
+	char *copy;   
+
+	len = strlen(string) + 1;
+	MALLOC(copy, char *, len, type, M_WAITOK);
+	if (copy == NULL)
+		return (NULL);
+	bcopy(string, copy, len);
+	return (copy); 
+}
+
+/*
+ * Return TRUE(1) if string 2 is a prefix of string 1.
+ */     
+int       
+strprefix(register const char *s1, register const char *s2)
+{               
+        register int    c;
+                
+        while ((c = *s2++) != '\0') {
+            if (c != *s1++) 
+                return (0);
+        }       
+        return (1);
+}
+
diff --git a/osfmk/gssd/Makefile b/osfmk/gssd/Makefile
new file mode 100644
index 000000000..2f7167424
--- /dev/null
+++ b/osfmk/gssd/Makefile
@@ -0,0 +1,65 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+INSTINC_SUBDIRS =
+
+INSTINC_SUBDIRS_PPC = 
+
+INSTINC_SUBDIRS_I386 =
+
+EXPINC_SUBDIRS =
+
+EXPINC_SUBDIRS_PPC =
+
+EXPINC_SUBDIRS_I386 =
+
+MIG_DEFS = gssd_mach.defs
+
+DATAFILES = gssd_mach_types.h ${MIG_DEFS}
+
+INSTALL_MI_LIST =
+
+INSTALL_MI_LCL_LIST	= ${DATAFILES} 
+
+INSTALL_MI_GEN_LIST = 
+
+INSTALL_MI_DIR = gssd
+
+EXPORT_MI_LIST	= \
+	${DATAFILES}
+
+EXPORT_MI_GEN_LIST = gssd_mach.h
+
+EXPORT_MI_DIR = gssd
+
+#
+# Build path
+#       
+INCFLAGS_MAKEFILE= -I..
+
+MIGKUFLAGS	= -DMACH_KERNEL_PRIVATE -DKERNEL_USER=1 -maxonstack 1024
+
+MIG_KUHDRS = gssd_mach.h
+
+MIG_KUSRC = gssd_mach.c gssd_mach.h
+
+COMP_FILES = ${MIG_KUSRC}
+
+${COMP_FILES} : gssd_mach.defs
+
+${MIG_KUSRC} : gssd_mach.defs
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKUFLAGS}	\
+		-user    $*.c			\
+		-header  $*.h			\
+		-server  /dev/null		\
+		-sheader /dev/null		\
+		$<
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/osfmk/gssd/gssd_mach.defs b/osfmk/gssd/gssd_mach.defs
new file mode 100644
index 000000000..5a8cf1f64
--- /dev/null
+++ b/osfmk/gssd/gssd_mach.defs
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/std_types.defs>
+#include <mach/mach_types.defs>
+
+#ifdef KERNEL
+import <gssd/gssd_mach_types.h>;
+#else
+import <gssd_mach_types.h>;
+#endif
+
+type mechtype = int32_t;
+type string_t = c_string[*:1024];
+type byte_buffer = array [] of uint8_t;
+type gssd_verifier =  uint64_t;
+type gid_list = array [*:16] of uint32_t;
+
+subsystem
+#if KERNEL_USER
+KernelUser
+#endif
+gssd_mach  999;
+
+serverprefix svc_;
+
+routine mach_gss_init_sec_context(
+	server			: mach_port_t;
+	in  mech		: mechtype;
+	in  intoken		: byte_buffer;
+	in  uid			: uint32_t;
+	in  princ_namestr	: string_t;
+	in  svc_namestr		: string_t;
+	in  flags		: uint32_t;
+	inout verifier		: gssd_verifier;
+	inout context		: uint32_t;
+	inout cred_handle	: uint32_t;
+	out  key		: byte_buffer, dealloc;
+	out outtoken		: byte_buffer, dealloc;
+	out major_stat		: uint32_t;
+	out minor_stat		: uint32_t
+);
+
+routine mach_gss_accept_sec_context(
+	server			: mach_port_t;
+	in  intoken		: byte_buffer;
+	in  svc_namestr		: string_t;
+	in  flags		: uint32_t;
+	inout verifier		: gssd_verifier;
+	inout context		: uint32_t;
+	inout cred_handle	: uint32_t;
+	out uid			: uint32_t;
+	out gids		: gid_list;
+	out  key		: byte_buffer, dealloc;
+	out outtoken		: byte_buffer, dealloc;
+	out major_stat		: uint32_t;
+	out minor_stat		: uint32_t
+);
+
+simpleroutine mach_gss_log_error(
+	server			: mach_port_t;
+	in  mnt			: string_t;
+	in  uid			: uint32_t;
+	in  source		: string_t;
+	in  major_stat		: uint32_t;
+	in  minor_stat		: uint32_t
+);
diff --git a/osfmk/gssd/gssd_mach_types.h b/osfmk/gssd/gssd_mach_types.h
new file mode 100644
index 000000000..0d97fb398
--- /dev/null
+++ b/osfmk/gssd/gssd_mach_types.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#ifndef _GSSD_MACH_TYPES_H_
+#define _GSSD_MACH_TYPES_H_
+
+typedef enum mechtype { DEFAULT_MECH = 0, KRB5_MECH = 0, SPNEGO_MECH } mechtype;
+typedef char *string_t;
+typedef uint8_t *byte_buffer;
+typedef uint64_t gssd_verifier;
+typedef uint32_t *gid_list;
+
+#define GSSD_GSS_FLAGS_MASK	0x1FF
+/* The following need to correspond to GSS_C_*_FLAG in gssapi.h */
+#define GSSD_DELEG_FLAG		1
+#define GSSD_MUTUAL_FLAG		2
+#define GSSD_REPLAY_FLAG		4
+#define GSSD_SEQUENCE_FLAG	8
+#define GSSD_CONF_FLAG		16
+#define GSSD_INTEG_FLAG		32
+#define GSSD_ANON_FLAG		64
+#define GSSD_PROT_FLAG		128
+#define GSSD_TRANS_FLAG		256
+
+#define GSSD_FLAGS_SHIFT		16
+#define GSSD_NO_DEFAULT		(1 << GSSD_FLAGS_SHIFT) // Only use principal from uid
+#define GSSD_NO_CANON		(2 << GSSD_FLAGS_SHIFT) // Don't canononicalize host names
+#define GSSD_NO_HOME_ACCESS	(4 << GSSD_FLAGS_SHIFT) // Dont access home directory
+#define GSSD_NO_UI		(8 << GSSD_FLAGS_SHIFT) // Don't bring up UI
+#define GSSD_WIN2K_HACK		(128 << GSSD_FLAGS_SHIFT) // Hack for Win2K
+
+
+#endif /* _GSSD_MACH_TYPES_H_ */
diff --git a/osfmk/i386/AT386/autoconf.c b/osfmk/i386/AT386/autoconf.c
deleted file mode 100644
index 7a5d3d7f6..000000000
--- a/osfmk/i386/AT386/autoconf.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/* FIXME - Remove when device/ is completely pulled out of OSFMK */
-int	dev_indirect_count	= 0;
-int	dev_indirect_list 	= 0;
diff --git a/osfmk/i386/AT386/conf.c b/osfmk/i386/AT386/conf.c
index 3be143d73..598a237c7 100644
--- a/osfmk/i386/AT386/conf.c
+++ b/osfmk/i386/AT386/conf.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/AT386/config.h b/osfmk/i386/AT386/config.h
deleted file mode 100644
index 44fd99948..000000000
--- a/osfmk/i386/AT386/config.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- * 
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:38  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:39  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.4.1  1996/04/17  17:48:30  davidp
- * 	Needed for SVR4 device drivers
- * 	[1996/04/11  13:03:48  davidp]
- *
- * Revision 1.1.1.2  1996/04/10  12:28:49  calvert
- * 	Needed for SVR4 device drivers
- * 
- * $EndLog$
- */
diff --git a/osfmk/i386/AT386/cram.h b/osfmk/i386/AT386/cram.h
deleted file mode 100644
index a56bef22f..000000000
--- a/osfmk/i386/AT386/cram.h
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:38  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:39  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.6.1  1994/09/23  01:33:48  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:16:06  ezf]
- *
- * Revision 1.1.2.2  1993/06/02  23:19:06  jeffc
- * 	Added to OSF/1 R1.3 from NMK15.0.
- * 	[1993/06/02  21:01:43  jeffc]
- * 
- * Revision 1.1  1992/09/30  02:26:37  robert
- * 	Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.5  91/05/14  16:22:13  mrt
- * 	Correcting copyright
- * 
- * Revision 2.4  91/02/05  17:16:50  mrt
- * 	Changed to new Mach copyright
- * 	[91/02/01  17:42:45  mrt]
- * 
- * Revision 2.3  90/11/26  14:49:30  rvb
- * 	jsb bet me to XMK34, sigh ...
- * 	[90/11/26            rvb]
- * 	Synched 2.5 & 3.0 at I386q (r1.5.1.3) & XMK35 (r2.3)
- * 	[90/11/15            rvb]
- * 
- * Revision 2.2  90/05/03  15:41:48  dbg
- * 	First checkin.
- * 
- * Revision 1.5.1.2  90/02/28  15:49:23  rvb
- * 	Fix numerous typo's in Olivetti disclaimer.
- * 	[90/02/28            rvb]
- * 
- * Revision 1.5.1.1  90/01/08  13:31:57  rvb
- * 	Add Olivetti copyright.
- * 	[90/01/08            rvb]
- * 
- * Revision 1.5  89/09/25  12:26:32  rvb
- * 	File was provided by Intel 9/18/89.
- * 	[89/09/23            rvb]
- * 
- */
-/* CMU_ENDHIST */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-
-/*
- * cram.h
- */
-
-/*
- * Copyright 1988, 1989 by Olivetti Advanced Technology Center, Inc.,
- * Cupertino, California.
- * 
- * 		All Rights Reserved
- * 
- * Permission to use, copy, modify, and distribute this software and
- * its documentation for any purpose and without fee is hereby
- * granted, provided that the above copyright notice appears in all
- * copies and that both the copyright notice and this permission notice
- * appear in supporting documentation, and that the name of Olivetti
- * not be used in advertising or publicity pertaining to distribution
- * of the software without specific, written prior permission.
- * 
- * OLIVETTI DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
- * IN NO EVENT SHALL OLIVETTI BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
- * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUR OF OR IN CONNECTION
- * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- * 
- */
-
-/* 
- * outb(CMOS_ADDR, addr);
- * result = inb(CMOS_DATA);
- * 
- * where "addr" tells what value you want to read (some are listed 
- * below).  Interrupts should be disabled while you do this.
- */
-
-/* I/O ports */
-
-#define CMOS_ADDR	0x70		/* port for CMOS ram address */
-#define CMOS_DATA	0x71		/* port for CMOS ram data */
-
-
-/* Addresses, related masks, and potential results */
-
-#define CMOS_EB		0x14		/* read Equipment Byte */
-#define CM_SCRMSK	0x30		/* mask for EB query to get screen */
-#define CM_EGA_VGA	0x00		/* "not CGA or MONO" */
-#define CM_CGA_40	0x10
-#define CM_CGA_80	0x20
-#define CM_MONO_80	0x30
-
diff --git a/osfmk/i386/AT386/himem.c b/osfmk/i386/AT386/himem.c
deleted file mode 100644
index c01c1f37e..000000000
--- a/osfmk/i386/AT386/himem.c
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_FREE_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:38  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:39  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.3.17.7  1995/08/21  20:33:13  devrcs
- * 	ri-osc CR1547:  Fix himem buffer translation to cope with non
- * 	page-aligned addresses.
- * 	[1995/08/08  16:51:58  bolinger]
- *
- * Revision 1.3.17.6  1995/02/24  15:51:12  alanl
- * 	DIPC:  Merge from nmk17b2 to nmk18b8.
- * 	Notes:  lock package cleanup.
- * 	[95/01/23            alanl]
- * 	[95/02/24            alanl]
- * 
- * Revision 1.3.17.5  1995/01/26  22:14:52  ezf
- * 	removed extraneous CMU CR
- * 	[1995/01/26  20:24:45  ezf]
- * 
- * Revision 1.3.17.4  1995/01/10  04:51:04  devrcs
- * 	mk6 CR801 - merge up from nmk18b4 to nmk18b7
- * 	* Rev 1.3.17.3  1994/10/21  18:41:39  joe
- * 	  Added ETAP support
- * 	[1994/12/09  20:37:48  dwm]
- * 
- * 	mk6 CR764 - s/spinlock/simple_lock/ (name change only)
- * 	[1994/11/10  05:25:33  dwm]
- * 
- * 	mk6 CR668 - 1.3b26 merge
- * 	* Revision 1.3.5.8  1994/05/06  18:44:06  tmt
- * 	Fix prototypes for new device signatures.
- * 	* Revision 1.3.5.6  1993/12/10  18:08:15  jeffc
- * 	CR10305 -- locking bug in himem_reserve(): change call to
- * 	vm_page_free to VM_PAGE_FREE.
- * 	* Revision 1.3.5.5  1993/11/19  17:56:58  jeffc
- * 	CR10125 -- Uninitialized lock in himem_convert. Add himem_init
- * 	CR9461 -- Locking bug in himem_convert - must retake lock after
- * 	thread_sleep.
- * 	* End1.3merge
- * 	[1994/11/04  09:07:39  dwm]
- * 
- * Revision 1.3.17.1  1994/06/14  03:04:20  toshi
- * 	Merge MK6 and NMK17
- * 	[1994/06/14  01:06:55  toshi]
- * 
- * Revision 1.3.15.2  1994/06/08  21:14:24  dswartz
- * 	Preemption merge.
- * 	[1994/06/08  21:12:29  dswartz]
- * 
- * Revision 1.3.15.1  1994/05/19  20:30:23  dwm
- * 	mk6 CR 74.  Locking bug in himem_reserve(): use VM_PAGE_FREE.
- * 	mk6 CR 9461.  Init hil_lock used by himem_convert();
- * 	retake lock after sleeping.
- * 	[1994/05/19  20:30:07  dwm]
- * 
- * Revision 1.3.11.1  1994/02/09  07:27:07  bernadat
- * 	Added himem_init() for module initialization.
- * 	[93/08/12            paire]
- * 
- * 	Take back hil_lock lock on return from thread_sleep()
- * 	[93/07/16            bernadat]
- * 
- * 	Add vm_page_gobble() calls where needed. (dwm bug #542)
- * 	Change from NORMA_MK14.6 [1993/02/09  22:24:00  dwm]
- * 	[93/07/16            bernadat]
- * 	[94/02/08            bernadat]
- * 
- * Revision 1.3.5.4  1993/08/09  19:37:19  dswartz
- * 	Add ANSI prototypes - CR#9523
- * 	[1993/08/06  17:50:02  dswartz]
- * 
- * Revision 1.3.5.3  1993/08/03  22:21:26  bernard
- * 	CR#9523 - ANSI prototype fixes.
- * 	[1993/08/03  15:34:10  bernard]
- * 
- * Revision 1.3.5.2  1993/06/09  02:25:18  gm
- * 	CR9157 - Find himem.h in the right place.
- * 	[1993/05/28  17:27:23  brezak]
- * 
- * Revision 1.3  1993/04/19  16:09:46  devrcs
- * 	make endif tags ansi compliant/include files
- * 	[1993/02/20  21:46:44  david]
- * 
- * 	Print an appropriate message when going out of HIMEM pages.
- * 	[93/01/26            bernadat]
- * 
- * Revision 1.2  1992/11/25  01:07:08  robert
- * 	integrate changes below for norma_14
- * 	[1992/11/13  19:28:44  robert]
- * 
- * $EndLog$
- */
-
-/*
- * support of memory above 16 Megs for DMA limited to memory
- * below 16 Megs. Copies high memory lo low memory before DMA
- * write operations and does the reverse at completion time for
- * DMA read operations
- */
-
-#include <cpus.h>
-#include <platforms.h>
-#include <kern/lock.h>
-#include <mach/vm_param.h>
-#include <vm/vm_page.h>
-#include <i386/AT386/himem.h>
-#include <kern/kalloc.h>
-#include <kern/spl.h>
-#include <mach/boolean.h>
-#include <kern/misc_protos.h>
-#include <i386/AT386/misc_protos.h>
-#include <i386/pmap.h>
-
-hil_t		hil_head;
-decl_simple_lock_data(,hil_lock)
-
-#if	HIMEM_STATS
-int himem_request;	/* number of requests */
-int himem_used;		/* number of times used */
-#endif	/* HIMEM_STATS */
-
-void
-himem_init(
-	void)
-{
-	simple_lock_init(&hil_lock, 0);
-}
-
-/* 
- * Called by drivers, this indicates himem that this driver might need
- * to allocate as many as npages pages in a single I/O DMA transfer
- */
-
-void
-himem_reserve(
-	int		npages)
-{
-	register		i = 0;
-	vm_page_t		free_head = VM_PAGE_NULL;
-	vm_page_t		low;
-	hil_t			hil;
-	spl_t			ipl;
-	extern pmap_paddr_t_t	avail_end;
-
-	if (avail_end <= HIGH_MEM)
-		return;
-	kprintf("looking for low mem pages\n");
-	hil = (hil_t)kalloc(npages*sizeof(struct himem_link));
-	if (hil == (hil_t)0) 
-		panic("himem_reserve: kalloc failed\n");
-
-	for (i=0; i < npages-1; i++)
-		(hil+i)->next = hil+i+1;
-
-	/*
-	 * This is the only way of getting low physical pages 
-	 * wtithout changing VM internals
-	 */
-	for (i=0; i != npages;) {
-		if ((low = vm_page_grab()) == VM_PAGE_NULL)
-			panic("No low memory pages for himem\n");
-		vm_page_gobble(low); /* mark as consumed internally */
-		if (_high_mem_page(low->phys_addr)) {
-			low->pageq.next = (queue_entry_t)free_head;
-			free_head = low;
-		} else {
-			(hil+i)->low_page = low->phys_addr;
-			i++;
-		}
-	}
-	kprintf("freeing high pages back\n");
-	for (low = free_head; low; low = free_head) {
-		free_head = (vm_page_t) low->pageq.next;
-		VM_PAGE_FREE(low);
-        }
-
-	ipl = splhi();
-	simple_lock(&hil_lock);
-	(hil+npages-1)->next = hil_head;
-	hil_head = hil;
-	simple_unlock(&hil_lock);
-	splx(ipl);
-}
-
-/*
- * Called by driver at DMA initialization time. Converts a high memory
- * physical page to a low memory one. If operation is a write, 
- * [phys_addr, phys_addr+length-1] is copied to new page. Caller must
- * provide a pointer to a pointer to a himem_list. This is used to store
- * all the conversions and is use at completion time to revert the pages.
- * This pointer must point to a null hil_t value for the call on the first
- * page of a DMA transfer.
- */
-
-vm_offset_t
-himem_convert(
-	vm_offset_t	phys_addr,
-	vm_size_t	length,
-	int		io_op,
-	hil_t		*hil)
-{
-	hil_t		h;
-	spl_t		ipl;
-	vm_offset_t	offset = phys_addr & (I386_PGBYTES - 1);
-
-	assert (offset + length <= I386_PGBYTES);
-
-	ipl = splhi();
-	simple_lock(&hil_lock);
-	while (!(h = hil_head)) { 
-		printf("WARNING: out of HIMEM pages\n");
-		thread_sleep_simple_lock((event_t)&hil_head,
-					 simple_lock_addr(hil_lock),
-					 THREAD_UNINT);
-		/* hil_lock relocked */
-	}
-	hil_head = hil_head->next;
-	simple_unlock(&hil_lock);
-	splx(ipl);
-	
-	h->high_addr = phys_addr;
-
-	if (io_op == D_WRITE) {
-	  bcopy_phys((addr64_t)phys_addr, (addr64_t)(h->low_page + offset),
-		length);
-	  h->length = 0;
-	} else {
-	  h->length = length;
-	}
-	h->offset = offset;
-
-	assert(!*hil || (*hil)->high_addr);
-
-	h->next = *hil;
-	*hil = h;
-	return(h->low_page + offset);
-}
-
-/*
- * Called by driver at DMA completion time. Converts a list of low memory
- * physical page to the original high memory one. If operation was read, 
- * [phys_addr, phys_addr+lenght-1] is copied to original page
- */
-
-void
-himem_revert(
-	hil_t		hil)
-{
-	hil_t		next;
-	boolean_t	wakeup = FALSE;
-	spl_t		ipl;
-
-	while(hil) {
-		if (hil->length) {
-			bcopy_phys((addr64_t)hil->low_page + hil->offset),
-				(addr64_t)(hil->high_addr),
-			      hil->length);
-		}
-		hil->high_addr = 0;
-		hil->length = 0;
-		hil->offset = 0;
-		next = hil->next;
-		ipl = splhi();
-		simple_lock(&hil_lock);
-		if (!(hil->next = hil_head))
-			wakeup = TRUE;
-		hil_head = hil;
-		simple_unlock(&hil_lock);
-		splx(ipl);
-		hil = next;
-	}
-	if (wakeup)
-		thread_wakeup((event_t)&hil_head);
-}
diff --git a/osfmk/i386/AT386/himem.h b/osfmk/i386/AT386/himem.h
deleted file mode 100644
index c0ae26d10..000000000
--- a/osfmk/i386/AT386/himem.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_FREE_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:38  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:39  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.3.11.4  1995/12/15  10:49:49  bernadat
- * 	cbus includes moved to busses/cbus
- * 	[95/12/15            bernadat]
- *
- * Revision 1.3.11.3  1995/08/21  20:33:23  devrcs
- * 	ri-osc CR1547:  Fix himem buffer translation to cope with non
- * 	page-aligned addresses.
- * 	[1995/08/08  16:52:06  bolinger]
- * 
- * Revision 1.3.11.2  1995/01/26  22:14:56  ezf
- * 	removed extraneous CMU CR
- * 	[1995/01/26  20:24:48  ezf]
- * 
- * Revision 1.3.9.2  1994/06/08  21:14:27  dswartz
- * 	Preemption merge.
- * 	[1994/06/08  21:12:31  dswartz]
- * 
- * Revision 1.3.9.1  1994/05/19  20:30:30  dwm
- * 	mk6 CR 80.  Add himem_init prototype.
- * 	[1994/05/19  20:30:10  dwm]
- * 
- * Revision 1.3.2.3  1993/08/09  19:37:21  dswartz
- * 	Add ANSI prototypes - CR#9523
- * 	[1993/08/06  17:50:06  dswartz]
- * 
- * Revision 1.3.2.2  1993/06/09  02:25:24  gm
- * 	CR9176 - ANSI C violations: trailing tokens on CPP
- * 	directives, extra semicolons after decl_ ..., asm keywords
- * 	[1993/06/07  18:58:40  jeffc]
- * 
- * 	Added to OSF/1 R1.3 from NMK15.0.
- * 	[1993/06/02  21:01:57  jeffc]
- * 
- * Revision 1.3  1993/04/19  16:09:54  devrcs
- * 	Use free copyright
- * 	[1993/03/03  12:12:12  bernadat]
- * 
- * Revision 1.2  1992/11/25  01:07:16  robert
- * 	integrate changes below for norma_14
- * 	[1992/11/13  19:28:57  robert]
- * 
- * $EndLog$
- */
-
-#ifndef _I386AT_HIMEM_H_
-#define _I386AT_HIMEM_H_
-
-/*
- * support of memory above 16 Megs for DMA limited to memory
- * below 16 Megs.
- */
-
-#include <platforms.h>
-
-#define HIMEM_STATS 0
-
-#if	HIMEM_STATS
-extern int himem_request;
-extern int himem_used;
-#endif	/* HIMEM_STATS */
-
-struct himem_link {
-	struct himem_link *next;
-	vm_offset_t	high_addr;	/* physical address */
-	vm_offset_t	low_page;	/* physical page */
-	vm_offset_t offset;		/* offset on page */
-	vm_size_t	length;
-};
- 
-typedef struct himem_link *hil_t;	
-
-
-#define HIGH_MEM		((vm_offset_t) 0xf00000)
-
-#define _high_mem_page(x)	((vm_offset_t)(x) >= HIGH_MEM)
-
-
-#if	HIMEM_STATS
-#define high_mem_page(x) \
-	(++himem_request && _high_mem_page(x) && ++himem_used)
-
-#else	/* HIMEM_STATS */
-#define high_mem_page(x) 	_high_mem_page(x)
-#endif	/* HIMEM_STATS */
-
-extern void		himem_init(void);
-extern void		himem_reserve(
-				int		npages);
-extern vm_offset_t	himem_convert(
-				vm_offset_t	paddr,
-				vm_size_t	len,
-				int		op,
-				hil_t		* hil);
-extern void		himem_revert(
-				hil_t		hil);
-
-#endif /* _I386AT_HIMEM_H_ */
diff --git a/osfmk/i386/AT386/machdep.mk b/osfmk/i386/AT386/machdep.mk
deleted file mode 100644
index 5fe42b283..000000000
--- a/osfmk/i386/AT386/machdep.mk
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# @OSF_COPYRIGHT@
-#
-#
-# HISTORY
-# 
-# Revision 1.1.1.1  1998/09/22 21:05:39  wsanchez
-# Import of Mac OS X kernel (~semeria)
-#
-# Revision 1.1.1.1  1998/03/07 02:25:38  wsanchez
-# Import of OSF Mach kernel (~mburg)
-#
-# Revision 1.1.17.1  1996/11/29  16:56:53  stephen
-# 	nmklinux_1.0b3_shared into pmk1.1
-# 	Export "kd.h".
-# 	[96/01/29            barbou]
-#
-# Revision 1.1.8.1  1994/09/23  01:45:12  ezf
-# 	change marker to not FREE
-# 	[1994/09/22  21:19:36  ezf]
-# 
-# Revision 1.1.4.2  1993/06/04  15:12:16  jeffc
-# 	CR9193 - Added export for "machine" headers.
-# 	Created from mach_kernel/include/i386/AT386/machdep.mk.
-# 	[1993/06/04  13:40:42  jeffc]
-# 
-# Revision 1.3  93/04/19  16:17:41  devrcs
-# 	updated for ODE 1.2 reno make
-# 	[1993/02/20  21:37:01  david]
-# 
-# $EndLog$
-
-AT386_DATAFILES =	disk.h kd.h
-
-#AT386_DATAFILES =	atbus.h			\
-#			blitreg.h		\
-#			blituser.h		\
-#			blitvar.h		\
-#			comreg.h		\
-#			cram.h			\
-#			disk.h			\
-#			eisa.h			\
-#			himem.h			\
-#			fdreg.h			\
-#			hdreg.h			\
-#			i8250.h			\
-#			i82586.h		\
-#			if_3c501.h		\
-#			if_3c503.h		\
-#			if_ns8390.h		\
-#			if_pc586.h		\
-#			if_wd8003.h		\
-#			kd.h			\
-#			kd_queue.h		\
-#			kdsoft.h		\
-#			rtc.h
diff --git a/osfmk/i386/AT386/misc_protos.h b/osfmk/i386/AT386/misc_protos.h
deleted file mode 100644
index 8c6c7e604..000000000
--- a/osfmk/i386/AT386/misc_protos.h
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-
-#ifndef	_AT386_MISC_PROTOS_H_
-#define	_AT386_MISC_PROTOS_H_
-
-#include <pexpert/i386/boot.h>	/* for KernelBootArgs_t */
-
-/*
- * i386/AT386/model_dep.c
- */
-
-extern void		i386_init(void);
-extern void		machine_init(void);
-extern void		machine_startup(void);
-
-/*
- * i386/AT386/kd.c
- */
-
-extern void		cninit(void);
-extern void		kdreboot(void);
-
-/*
- * i386/db_interface.c
- */
-
-extern void		kdb_console(void);
-
-/*
- * i386/bcopy.s
- */
-
-extern void		bcopy16(
-				char		* from,
-				char		* to,
-				int		count);
-
-typedef	void		(*i386_intr_t)(void);
-#endif
-
diff --git a/osfmk/i386/AT386/model_dep.c b/osfmk/i386/AT386/model_dep.c
index b28fd974e..eb86e791e 100644
--- a/osfmk/i386/AT386/model_dep.c
+++ b/osfmk/i386/AT386/model_dep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -85,6 +91,8 @@
 #include <i386/machine_routines.h>
 #include <i386/pmCPU.h>
 #include <i386/postcode.h>
+#include <i386/trap.h>
+#include <i386/mp.h>		/* mp_rendezvous_break_lock */
 #include <architecture/i386/pio.h> /* inb() */
 #include <pexpert/i386/boot.h>
 #if	MACH_KDB
@@ -117,31 +125,25 @@ static void machine_conf(void);
 extern int		default_preemption_rate;
 extern int		max_unsafe_quanta;
 extern int		max_poll_quanta;
-extern int		idlehalt;
 extern unsigned int	panic_is_inited;
 
 int db_run_mode;
 
-static int packAsc (uint8_t *inbuf, unsigned int length);
-
 volatile int pbtcpu = -1;
 hw_lock_data_t pbtlock;		/* backtrace print lock */
 uint32_t pbtcnt = 0;
 
-extern const char version[];
-
 typedef struct _cframe_t {
     struct _cframe_t	*prev;
     unsigned		caller;
     unsigned		args[0];
 } cframe_t;
 
-void panic_i386_backtrace(void *_frame, int nframes);
-
-static unsigned panic_io_port = 0;
+static unsigned panic_io_port;
+static unsigned	commit_paniclog_to_nvram;
 
 void
-machine_startup()
+machine_startup(void)
 {
 	int	boot_arg;
 
@@ -152,12 +154,22 @@ machine_startup()
 
 	if (PE_parse_boot_arg("debug", &boot_arg)) {
 		if (boot_arg & DB_HALT) halt_in_debugger=1;
-		if (boot_arg & DB_PRT) disableDebugOuput=FALSE; 
+		if (boot_arg & DB_PRT) disable_debug_output=FALSE; 
 		if (boot_arg & DB_SLOG) systemLogDiags=TRUE; 
 		if (boot_arg & DB_NMI) panicDebugging=TRUE; 
 		if (boot_arg & DB_LOG_PI_SCRN) logPanicDataToScreen=TRUE; 
 	}
 
+	if (!PE_parse_boot_arg("nvram_paniclog", &commit_paniclog_to_nvram))
+		commit_paniclog_to_nvram = 1;
+
+	/*
+	 * Entering the debugger will put the CPUs into a "safe"
+	 * power mode.
+	 */
+	if (PE_parse_boot_arg("pmsafe_debug", &boot_arg))
+	    pmsafe_debug = boot_arg;
+
 #if NOTYET
 	hw_lock_init(&debugger_lock);	/* initialize debugger lock */
 #endif
@@ -210,25 +222,6 @@ machine_startup()
 		panic_io_port = boot_arg & 0xffff;
 	}
 
-/*
- *	fn is used to force napping.
- *		fn=0 means no napping allowed
- *		fn=1 means forces napping on, normal C2 and C4 transitions
- *		fn=2 means forces napping on, but C4 is disabled
- *		fn=3 means forces napping on, but use halt
- *		fn=4 means forces napping on and will always use C4
- *
- *		Note that this will take effect only when the system normally starts napping.
- *
- */ 
-
-	if (!PE_parse_boot_arg("fn", &forcenap)) forcenap = 0;	/* If force nap not set, make 0 */
-	else {
-		if(forcenap < 5) forcenap = forcenap + 1;		/* See comments above for decode, this is set to fn + 1 */
-		else forcenap = 0;								/* Clear for error case */
-	}
-	machine_nap_policy();								/* Make sure the nap policy reflects the user's choice */
-
 	machine_conf();
 
 #if NOTYET
@@ -362,7 +355,7 @@ efi_set_tables_64(EFI_SYSTEM_TABLE_64 * system_table)
     uint32_t hdr_cksum;
     uint32_t cksum;
 
-    kprintf("Processing 64-bit EFI tables at 0x%x\n", (unsigned int)system_table);
+    kprintf("Processing 64-bit EFI tables at %p\n", system_table);
     do {
         if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 	    kprintf("Bad EFI system table signature\n");
@@ -383,8 +376,8 @@ efi_set_tables_64(EFI_SYSTEM_TABLE_64 * system_table)
 	gPEEFISystemTable     = system_table;
 
         kprintf("RuntimeServices table at 0x%qx\n", system_table->RuntimeServices);
-        runtime = (EFI_RUNTIME_SERVICES_64 *) (uint32_t)system_table->RuntimeServices; // XXX
-        kprintf("Checking runtime services table 0x%x\n", runtime);
+        runtime = (EFI_RUNTIME_SERVICES_64 *) (uintptr_t)system_table->RuntimeServices; // XXX
+        kprintf("Checking runtime services table %p\n", runtime);
 	if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
 	    kprintf("Bad EFI runtime table signature\n");
 	    break;
@@ -414,7 +407,7 @@ efi_set_tables_32(EFI_SYSTEM_TABLE * system_table)
     uint32_t hdr_cksum;
     uint32_t cksum;
 
-    kprintf("Processing 32-bit EFI tables at 0x%x\n", (unsigned int)system_table);
+    kprintf("Processing 32-bit EFI tables at %p\n", system_table);
     do {
         if (system_table->Hdr.Signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 	    kprintf("Bad EFI system table signature\n");
@@ -434,6 +427,7 @@ efi_set_tables_32(EFI_SYSTEM_TABLE * system_table)
 
 	gPEEFISystemTable     = system_table;
 
+	kprintf("RuntimeServices table at %p\n", system_table->RuntimeServices);
         runtime = (EFI_RUNTIME_SERVICES *) system_table->RuntimeServices;
 	if (runtime->Hdr.Signature != EFI_RUNTIME_SERVICES_SIGNATURE) {
 	    kprintf("Bad EFI runtime table signature\n");
@@ -468,7 +462,7 @@ efi_init(void)
 
     do
     {
-        vm_offset_t vm_size, vm_addr;
+	vm_offset_t vm_size, vm_addr;
 	vm_map_offset_t phys_addr;
 	EfiMemoryRange *mptr;
 	unsigned int msize, mcount;
@@ -516,7 +510,7 @@ hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_o
 	return;
     do
     {
-	vm_offset_t vm_size, vm_addr;
+        vm_offset_t vm_size, vm_addr;
 	vm_map_offset_t phys_addr;
 	EfiMemoryRange *mptr;
 	unsigned int msize, mcount;
@@ -527,10 +521,10 @@ hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_o
 
 	system_table_offset += ptoa_32(args->efiRuntimeServicesPageStart);
 
-	kprintf("Old system table %p, new %p\n",
-	    args->efiSystemTable,    (void *) system_table_offset);
+	kprintf("Old system table 0x%x, new 0x%x\n",
+	    (uint32_t)args->efiSystemTable,    system_table_offset);
 
-	args->efiSystemTable    = (uint32_t) system_table_offset;
+	args->efiSystemTable    = system_table_offset;
 
 	kprintf("Old map:\n");
 	msize = args->MemoryMapDescriptorSize;
@@ -543,7 +537,7 @@ hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_o
 		vm_addr =   (vm_offset_t) mptr->VirtualStart;
 		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
 
-		kprintf("mapping[%d] %qx @ %x, %x\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
+		kprintf("mapping[%u] %qx @ %x, %llu\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
 	    }
 	}
 
@@ -561,7 +555,7 @@ hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t system_table_o
 		vm_addr =   (vm_offset_t) mptr->VirtualStart;
 		phys_addr = (vm_map_offset_t) mptr->PhysicalStart;
 
-		kprintf("mapping[%d] %qx @ %x, %x\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
+		kprintf("mapping[%u] %qx @ %x, %llu\n", mptr->Type, phys_addr, vm_addr, mptr->NumberOfPages);
 
 		pmap_map(vm_addr, phys_addr, phys_addr + round_page(vm_size),
 			 (mptr->Type == kEfiRuntimeServicesCode) ? VM_PROT_READ | VM_PROT_EXECUTE : VM_PROT_READ|VM_PROT_WRITE,
@@ -662,6 +656,7 @@ halt_all_cpus(boolean_t reboot)
 	while(1);
 }
 
+ 
 /* Issue an I/O port read if one has been requested - this is an event logic
  * analyzers can use as a trigger point.
  */
@@ -678,7 +673,7 @@ panic_io_port_read(void) {
 static void
 machine_halt_cpu(__unused void *arg) {
 	panic_io_port_read();
-	__asm__ volatile("hlt");
+	pmCPUHalt(PM_HALT_DEBUG);
 }
 
 void
@@ -723,39 +718,45 @@ Debugger(
 		 */
 
 		if( debug_buf_size > 0) {
-		    /* Do not compress the panic log 
-		     * or save to NVRAM unless kernel debugging 
-		     * is disabled. The NVRAM shim doesn't
-		     * sync to the store until haltRestart is called.
-		     */
-		    if (!panicDebugging) {
+		  /* Optionally sync the panic log, if any, to NVRAM
+		   * This is the default.
+		   */
+		    if (commit_paniclog_to_nvram) {
 			unsigned int bufpos;
-			
+
                         debug_putc(0);
 
 			/* Now call the compressor */
 			/* XXX Consider using the WKdm compressor in the
 			 * future, rather than just packing - would need to
 			 * be co-ordinated with crashreporter, which decodes
-			 * this post-restart.
+			 * this post-restart. The compressor should be
+			 * capable of in-place compression.
 			 */
-			bufpos = packAsc ((uint8_t *)debug_buf,
-			    (unsigned int) (debug_buf_ptr - debug_buf) );
+			bufpos = packA(debug_buf,
+			    (unsigned int) (debug_buf_ptr - debug_buf), debug_buf_size);
 			/* If compression was successful,
 			 * use the compressed length
 			 */
-			if (bufpos) {
-			    debug_buf_ptr = debug_buf + bufpos;
-                        }
+			pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf);
+
 			/* Save panic log to non-volatile store
 			 * Panic info handler must truncate data that is 
 			 * too long for this platform.
 			 * This call must save data synchronously,
 			 * since we can subsequently halt the system.
 			 */
-                        pi_size = debug_buf_ptr - debug_buf;
+			kprintf("Attempting to commit panic log to NVRAM\n");
                         pi_size = PESavePanicInfo((unsigned char *)debug_buf,
 			    pi_size );
+
+			/* Uncompress in-place, to permit examination of
+			 * the panic log by debuggers.
+			 */
+
+			if (bufpos) {
+			  unpackA(debug_buf, bufpos);
+			}
                     }
                 }
 		draw_panic_dialog();
@@ -765,12 +766,16 @@ Debugger(
 			 * that a panic occurred while in that codepath.
 			 */
 			mp_rendezvous_break_lock();
+#if CONFIG_EMBEDDED
+			PEHaltRestart(kPEPanicRestartCPU);
+#else
 			/* Force all CPUs to disable interrupts and HLT.
 			 * We've panicked, and shouldn't depend on the
 			 * PEHaltRestart() mechanism, which relies on several
 			 * bits of infrastructure.
 			 */
 			mp_rendezvous_no_intrs(machine_halt_cpu, NULL);
+#endif
 			/* NOT REACHED */
 		}
         }
@@ -810,32 +815,6 @@ struct pasc {
 
 typedef struct pasc pasc_t;
 
-static int packAsc (unsigned char *inbuf, unsigned int length)
-{
-  unsigned int i, j = 0;
-  unsigned int extra;
-  pasc_t pack;
-
-  for (i = 0; i < length; i+=8)
-    {
-      pack.a = inbuf[i];
-      pack.b = inbuf[i+1];
-      pack.c = inbuf[i+2];
-      pack.d = inbuf[i+3];
-      pack.e = inbuf[i+4];
-      pack.f = inbuf[i+5];
-      pack.g = inbuf[i+6];
-      pack.h = inbuf[i+7];
-      bcopy ((char *) &pack, inbuf + j, 7);
-      j += 7;
-    }
-  extra = (i - length);
-  if (extra > 0) {
-    inbuf[j - extra] &= (0xFF << (8-extra));
-  }
-  return j-((extra == 7) ? 6 : extra);
-}
-
 /* Routines for address - symbol translation. Not called unless the "keepsyms"
  * boot-arg is supplied.
  */
@@ -861,11 +840,14 @@ panic_print_macho_symbol_name(struct mach_header *mh, vm_address_t search)
         if (cmd->cmd == LC_SEGMENT) {
             struct segment_command *orig_sg = (struct segment_command *) cmd;
             
-            if (strcmp(SEG_TEXT, orig_sg->segname) == 0)
+            if (strncmp(SEG_TEXT, orig_sg->segname,
+				    sizeof(orig_sg->segname)) == 0)
                 orig_ts = orig_sg;
-            else if (strcmp(SEG_LINKEDIT, orig_sg->segname) == 0)
+            else if (strncmp(SEG_LINKEDIT, orig_sg->segname,
+				    sizeof(orig_sg->segname)) == 0)
                 orig_le = orig_sg;
-            else if (strcmp("", orig_sg->segname) == 0)
+            else if (strncmp("", orig_sg->segname,
+				    sizeof(orig_sg->segname)) == 0)
                 orig_ts = orig_sg; /* kexts have a single unnamed segment */
         }
         else if (cmd->cmd == LC_SYMTAB)
@@ -906,9 +888,9 @@ panic_print_macho_symbol_name(struct mach_header *mh, vm_address_t search)
     
     if (bestsym != NULL) {
         if (diff != 0) {
-            kdb_printf("%s + 0x%08x ", bestsym, diff);
+            kdb_printf("%s + 0x%08x \n", bestsym, diff);
         } else {
-            kdb_printf("%s ", bestsym);
+            kdb_printf("%s \n", bestsym);
         }
         return 1;
     }
@@ -930,7 +912,7 @@ panic_print_kmod_symbol_name(vm_address_t search)
     }
     if (current_kmod != NULL) {
         /* if kexts had symbol table loaded, we'd call search_symbol_name again; alas, they don't */
-        kdb_printf("%s + %d ", current_kmod->name, search - current_kmod->address);
+        kdb_printf("%s + %d \n", current_kmod->name, search - current_kmod->address);
     }
 }
 
@@ -969,14 +951,14 @@ panic_i386_backtrace(void *_frame, int nframes)
 		/* Spin on print backtrace lock, which serializes output
 		 * Continue anyway if a timeout occurs.
 		 */
-		hw_lock_to(&pbtlock, LockTimeOut*100);
+		hw_lock_to(&pbtlock, LockTimeOutTSC);
 		pbtcpu = cpu_number();
 	}
 
 	PE_parse_boot_arg("keepsyms", &keepsyms);
 
 	kdb_printf("Backtrace, "
-	    "Format - Frame : Return Address (4 potential args on stack) ");
+	    "Format - Frame : Return Address (4 potential args on stack) \n");
 
 	for (frame_index = 0; frame_index < nframes; frame_index++) {
 		vm_offset_t curframep = (vm_offset_t) frame;
@@ -995,13 +977,12 @@ panic_i386_backtrace(void *_frame, int nframes)
 			goto invalid;
 		}
 
-		kdb_printf("\n0x%x : 0x%x ",
-		    frame, frame->caller);
+		kdb_printf("%p : 0x%x ", frame, frame->caller);
 		if (frame_index < DUMPFRAMES)
 			raddrs[frame_index] = frame->caller;
 
 		if (kvtophys((vm_offset_t)&(frame->args[3])))
-			kdb_printf("(0x%x 0x%x 0x%x 0x%x) ",
+			kdb_printf("(0x%x 0x%x 0x%x 0x%x) \n",
 			    frame->args[0], frame->args[1],
 			    frame->args[2], frame->args[3]);
 
@@ -1027,7 +1008,7 @@ panic_i386_backtrace(void *_frame, int nframes)
 	goto out;
 
 invalid:
-	kdb_printf("Backtrace terminated-invalid frame pointer 0x%x\n",frame);
+	kdb_printf("Backtrace terminated-invalid frame pointer %p\n",frame);
 out:
 
 	/* Identify kernel modules in the backtrace and display their
@@ -1037,8 +1018,7 @@ out:
 	if (frame_index)
 		kmod_dump((vm_offset_t *)&raddrs[0], frame_index);
 
-	kdb_printf("\nKernel version:\n%s\n\n",version);
-
+	panic_display_system_configuration();
 	/* Release print backtrace lock, to permit other callers in the
 	 * event of panics on multiple processors.
 	 */
diff --git a/osfmk/i386/AT386/physmem_entries.h b/osfmk/i386/AT386/physmem_entries.h
deleted file mode 100644
index ffb48fa50..000000000
--- a/osfmk/i386/AT386/physmem_entries.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_FREE_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:39  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:38  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.4.1  1996/11/29  16:56:56  stephen
- * 	nmklinux_1.0b3_shared into pmk1.1
- * 	Created. Prototypes for the "physmem" device.
- * 	[1996/11/22  15:25:06  barbou]
- *
- * $EndLog$
- */
-
-extern io_return_t	physmem_open(
-				dev_t		dev,
-				dev_mode_t	flag,
-				io_req_t	ior);
-extern void		physmem_close(
-				dev_t		dev);
-extern io_return_t	physmem_read(
-				dev_t		dev,
-				io_req_t	ior);
-extern io_return_t	physmem_write(
-				dev_t		dev,
-				io_req_t	ior);
-extern io_return_t	physmem_getstat(
-				dev_t		dev,
-				dev_flavor_t	flavor,
-				dev_status_t	data,
-				mach_msg_type_number_t * count);
-extern io_return_t	physmem_setstat(
-				dev_t		dev,
-				dev_flavor_t	flavor,
-				dev_status_t	data,
-				mach_msg_type_number_t	count);
-extern vm_offset_t	physmem_mmap(
-				dev_t		dev,
-				vm_offset_t	off,
-				vm_prot_t	prot);
-extern io_return_t	phsymem_async_in(
-				dev_t		dev,
-				ipc_port_t	rcv_port,
-				int		pri,
-				filter_t	*filter,
-				mach_msg_type_number_t fcount,
-				device_t	device);
-extern void		physmem_reset(
-				dev_t		dev);
-extern boolean_t	phsymem_port_death(
-				dev_t		dev,
-				ipc_port_t	port);
-extern io_return_t	physmem_dev_info(
-				dev_t		dev,
-				dev_flavor_t	flavor,
-				char		* info);
diff --git a/osfmk/i386/Diagnostics.c b/osfmk/i386/Diagnostics.c
index eae3c6e66..34209af9e 100644
--- a/osfmk/i386/Diagnostics.c
+++ b/osfmk/i386/Diagnostics.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -72,24 +78,76 @@ diagWork        dgWork;
 uint64_t        lastNapClear = 0ULL;
 uint64_t        lastRuptClear = 0ULL;
 
-typedef struct pmdata {
-	uint64_t        pmNapDur;	/* Time since last query */
-	pmStats_t       pmd;	/* Powermanagement statistics */
-}               pmdata;
-
-
 
 int 
-diagCall64(__unused x86_saved_state_t * regs)
+diagCall64(x86_saved_state_t * state)
 {
-        panic("diagCall not yet supported for 64 bit tasks\n");
+	uint64_t        curpos, i, j;
+	uint64_t        selector, data;
+	uint64_t        currNap, durNap;
+	x86_saved_state64_t	*regs;
+
+	assert(is_saved_state64(state));
+	regs = saved_state64(state);
+
+	if (!(dgWork.dgFlags & enaDiagSCs))
+		return 0;	/* If not enabled, cause an exception */
+
+	selector = regs->rdi;
+
+	switch (selector) {	/* Select the routine */
+	case dgRuptStat:	/* Suck Interruption statistics */
+		data = regs->rsi; /* Get the number of processors */
+
+		if (data == 0) { /* If no location is specified for data, clear all
+				  * counts
+				  */
+			for (i = 0; i < real_ncpus; i++) {	/* Cycle through
+								 * processors */
+				for (j = 0; j < 256; j++)
+					cpu_data_ptr[i]->cpu_hwIntCnt[j] = 0;
+			}
+
+			lastRuptClear = mach_absolute_time();	/* Get the time of clear */
+			return 1;	/* Normal return */
+		}
+
+		(void) copyout((char *) &real_ncpus, data, sizeof(real_ncpus));	/* Copy out number of
+										 * processors */
+
+		currNap = mach_absolute_time();	/* Get the time now */
+		durNap = currNap - lastRuptClear;	/* Get the last interval
+							 * duration */
+		if (durNap == 0)
+			durNap = 1;	/* This is a very short time, make it
+					 * bigger */
+
+		curpos = data + sizeof(real_ncpus);	/* Point to the next
+							 * available spot */
+
+		for (i = 0; i < real_ncpus; i++) {	/* Move 'em all out */
+			(void) copyout((char *) &durNap, curpos, 8);	/* Copy out the time
+									 * since last clear */
+			(void) copyout((char *) &cpu_data_ptr[i]->cpu_hwIntCnt, curpos + 8, 256 * sizeof(uint32_t));	/* Copy out interrupt
+															 * data for this
+															 * processor */
+			curpos = curpos + (256 * sizeof(uint32_t) + 8);	/* Point to next out put
+									 * slot */
+		}
+		break;
+
+	default:		/* Handle invalid ones */
+		return 0;	/* Return an exception */
+
+	}
+
+	return 1;		/* Normal non-ast check return */
 }
 
 
 int 
 diagCall(x86_saved_state_t * state)
 {
-
 	uint32_t        stk, curpos, i, j;
 	uint32_t        selector, data;
 	int             err;
@@ -108,7 +166,6 @@ diagCall(x86_saved_state_t * state)
 		return 0;	/* Failed to fetch stack */
 	}
 	switch (selector) {	/* Select the routine */
-
 	case dgRuptStat:	/* Suck Interruption statistics */
 
 		err = copyin((user_addr_t) (stk + 8), (char *) &data, sizeof(uint32_t));	/* Get the selector */
@@ -124,7 +181,6 @@ diagCall(x86_saved_state_t * state)
 			lastRuptClear = mach_absolute_time();	/* Get the time of clear */
 			return 1;	/* Normal return */
 		}
-		err = copyin((user_addr_t) (stk + 8), (char *) &data, sizeof(uint32_t));	/* Get the selector */
 
 		(void) copyout((char *) &real_ncpus, data, sizeof(real_ncpus));	/* Copy out number of
 										 * processors */
diff --git a/osfmk/i386/Diagnostics.h b/osfmk/i386/Diagnostics.h
index bebd43f20..c8d385c7b 100644
--- a/osfmk/i386/Diagnostics.h
+++ b/osfmk/i386/Diagnostics.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/i386/Makefile b/osfmk/i386/Makefile
index 2ce73da27..a5fa0188e 100644
--- a/osfmk/i386/Makefile
+++ b/osfmk/i386/Makefile
@@ -13,6 +13,7 @@ EXPORT_ONLY_FILES = 	\
 		    cpu_number.h \
 		    cpu_capabilities.h	\
 		    cpu_data.h \
+		    cpu_topology.h \
 		    cpuid.h \
 		    eflags.h \
 		    hpet.h \
@@ -32,12 +33,12 @@ EXPORT_ONLY_FILES = 	\
 		    seg.h \
 		    simple_lock.h \
 		    tsc.h \
-		    tss.h
+		    tss.h \
+		    vmx.h
 
 INSTALL_MD_DIR = i386
 
-INSTALL_MD_LIST = eflags.h \
-		  user_ldt.h
+INSTALL_MD_LIST = eflags.h user_ldt.h
 
 INSTALL_MD_LCL_LIST = cpu_capabilities.h
 
diff --git a/osfmk/i386/_setjmp.s b/osfmk/i386/_setjmp.s
index cde520aea..faa486515 100644
--- a/osfmk/i386/_setjmp.s
+++ b/osfmk/i386/_setjmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/acpi.c b/osfmk/i386/acpi.c
index 26ad0dc4a..539a82fde 100644
--- a/osfmk/i386/acpi.c
+++ b/osfmk/i386/acpi.c
@@ -1,47 +1,57 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <i386/misc_protos.h>
+#include <i386/cpu_data.h>
 #include <i386/proc_reg.h>
 #include <i386/pmap.h>
 #include <i386/mtrr.h>
+#include <i386/vmx/vmx_cpu.h>
 #include <i386/acpi.h>
 #include <i386/fpu.h>
 #include <i386/mp.h>
 #include <i386/mp_desc.h>
+#include <i386/serial_io.h>
+#include <i386/hpet.h>
 #include <i386/machine_check.h>
 
 #include <kern/cpu_data.h>
+#include <console/serial_protos.h>
 
+#if HIBERNATION
 #include <IOKit/IOHibernatePrivate.h>
+#endif
 #include <IOKit/IOPlatformExpert.h>
 
 extern void	acpi_sleep_cpu(acpi_sleep_callback, void * refcon);
 extern char acpi_wake_start[];
 extern char	acpi_wake_end[];
 
-extern int	serial_init(void);
-extern unsigned int disableSerialOuput;
-
 extern void        set_kbd_leds(int leds);
 
 extern void 	fpinit(void);
@@ -61,137 +71,166 @@ acpi_install_wake_handler(void)
 	return ACPI_WAKE_ADDR;
 }
 
-typedef struct acpi_hibernate_callback_data {
-    acpi_sleep_callback func;
-    void *refcon;
-} acpi_hibernate_callback_data;
+#if HIBERNATION
+struct acpi_hibernate_callback_data {
+	acpi_sleep_callback func;
+	void *refcon;
+};
+typedef struct acpi_hibernate_callback_data acpi_hibernate_callback_data_t;
 
 static void
 acpi_hibernate(void *refcon)
 {
-    boolean_t dohalt;
+	uint32_t mode;
 
-    acpi_hibernate_callback_data *data = (acpi_hibernate_callback_data *)refcon;
+	acpi_hibernate_callback_data_t *data =
+		(acpi_hibernate_callback_data_t *)refcon;
 
-    if (current_cpu_datap()->cpu_hibernate) {
-
-        dohalt = hibernate_write_image();
-	if (dohalt)
+	if (current_cpu_datap()->cpu_hibernate) 
 	{
-	    // off
-	    HIBLOG("power off\n");
-	    if (PE_halt_restart) 
-		(*PE_halt_restart)(kPEHaltCPU);
-	}
-	else
-	{
-	    // sleep
-	    HIBLOG("sleep\n");
+		cpu_IA32e_enable(current_cpu_datap());
+
+		mode = hibernate_write_image();
+
+		if( mode == kIOHibernatePostWriteHalt )
+		{
+			// off
+			HIBLOG("power off\n");
+			if (PE_halt_restart) (*PE_halt_restart)(kPEHaltCPU);
+		}
+		else if( mode == kIOHibernatePostWriteRestart )
+		{
+			// restart
+			HIBLOG("restart\n");
+			if (PE_halt_restart) (*PE_halt_restart)(kPERestartCPU);
+		}
+		else
+		{
+			// sleep
+			HIBLOG("sleep\n");
+	
+			// should we come back via regular wake, set the state in memory.
+			cpu_datap(0)->cpu_hibernate = 0;			
+		}
+
+		/*
+		 * If we're in 64-bit mode, drop back into legacy mode during sleep.
+		 */
+		cpu_IA32e_disable(current_cpu_datap());
 
-	    // should we come back via regular wake, set the state in memory.
-	    cpu_datap(0)->cpu_hibernate = 0;
 	}
-    }
 
-    (data->func)(data->refcon);
+	(data->func)(data->refcon);
 
-    /* should never get here! */
+	/* should never get here! */
 }
+#endif
 
 static uint64_t		acpi_sleep_abstime;
 
 void
 acpi_sleep_kernel(acpi_sleep_callback func, void *refcon)
 {
-    acpi_hibernate_callback_data data;
-    boolean_t did_hibernate;
+#if HIBERNATION
+	acpi_hibernate_callback_data_t data;
+	boolean_t did_hibernate;
+#endif
 
-    kprintf("acpi_sleep_kernel hib=%d\n", current_cpu_datap()->cpu_hibernate);
+	kprintf("acpi_sleep_kernel hib=%d\n",
+			current_cpu_datap()->cpu_hibernate);
 
-    /* shutdown local APIC before passing control to BIOS */
-    lapic_shutdown();
+	/* shutdown local APIC before passing control to BIOS */
+	lapic_shutdown();
 
-    data.func = func;
-    data.refcon = refcon;
+#if HIBERNATION
+	data.func = func;
+	data.refcon = refcon;
+#endif
 
-    /* Save HPET state */
-    hpet_save();
+	/* Save HPET state */
+	hpet_save();
 
-    /*
-     * If we're in 64-bit mode, drop back into legacy mode during sleep.
-     */
-    if (cpu_mode_is64bit()) {
+	/* 
+	 * Turn off VT, otherwise switching to legacy mode will fail
+	 */
+	vmx_suspend();
+
+	/*
+	 * If we're in 64-bit mode, drop back into legacy mode during sleep.
+	 */
 	cpu_IA32e_disable(current_cpu_datap());
-	kprintf("acpi_sleep_kernel legacy mode re-entered\n");
-    }
 
 	acpi_sleep_abstime = mach_absolute_time();
-	
-    /*
-     * Save master CPU state and sleep platform.
-     * Will not return until platform is woken up,
-     * or if sleep failed.
-     */
-    acpi_sleep_cpu(acpi_hibernate, &data);
-
-    /* reset UART if kprintf is enabled */
-    if (FALSE == disableSerialOuput)
-	    serial_init();
-
-    kprintf("ret from acpi_sleep_cpu hib=%d\n", current_cpu_datap()->cpu_hibernate);
-
-    if (current_cpu_datap()->cpu_hibernate) {
-	int i;
-	for (i=0; i<PMAP_NWINDOWS; i++) {
-	    *current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0;
+
+	/*
+	 * Save master CPU state and sleep platform.
+	 * Will not return until platform is woken up,
+	 * or if sleep failed.
+	 */
+#if HIBERNATION
+	acpi_sleep_cpu(acpi_hibernate, &data);
+#else
+	acpi_sleep_cpu(func, refcon);
+#endif
+
+	/* reset UART if kprintf is enabled */
+	if (FALSE == disable_serial_output)
+		serial_init();
+
+#if HIBERNATION
+	if (current_cpu_datap()->cpu_hibernate) {
+		int i;
+		for (i = 0; i < PMAP_NWINDOWS; i++)
+			*current_cpu_datap()->cpu_pmap->mapwindow[i].prv_CMAP = 0;
+		current_cpu_datap()->cpu_hibernate = 0;
+		did_hibernate = TRUE;
+
+	} else
+#endif 
+	{
+		did_hibernate = FALSE;
 	}
-	current_cpu_datap()->cpu_hibernate = 0;
-	did_hibernate = TRUE;
 
-    } else {
-	did_hibernate = FALSE;
-    }
+	/* Re-enable mode (including 64-bit if applicable) */
+	cpu_mode_init(current_cpu_datap());
 
-    /* Re-enable 64-bit mode if necessary. */
-    if (cpu_mode_is64bit()) {
-	cpu_IA32e_enable(current_cpu_datap());
-	cpu_desc_load64(current_cpu_datap());
-	kprintf("acpi_sleep_kernel 64-bit mode re-enabled\n");
-	fast_syscall_init64();
-    } else {
-	fast_syscall_init();
-    }
+	/* Re-enable machine check handling */
+	mca_cpu_init();
 
-    /* Re-enable machine check handling */
-    mca_cpu_init();
+	/* restore MTRR settings */
+	mtrr_update_cpu();
 
-    /* restore MTRR settings */
-    mtrr_update_cpu();
+	/* 
+	 * Restore VT mode
+	 */
+	vmx_resume();
 
-    /* set up PAT following boot processor power up */
-    pat_init();
+	/* set up PAT following boot processor power up */
+	pat_init();
 
 	/* let the realtime clock reset */
 	rtc_sleep_wakeup(acpi_sleep_abstime);
 
-    if (did_hibernate) {
-        hibernate_machine_init();
-    }
-        
-    /* re-enable and re-init local apic */
-    if (lapic_probe())
-	lapic_init();
+	if (did_hibernate)
+		hibernate_machine_init();
+
+	/* re-enable and re-init local apic */
+	if (lapic_probe())
+		lapic_init();
 
-    /* Restore HPET state */
-    hpet_restore();
+	/* Restore HPET state */
+	hpet_restore();
 
 	/* Restart tick interrupts from the LAPIC timer */
 	rtc_lapic_start_ticking();
 
-    fpinit();
-    clear_fpu();
+	fpinit();
+	clear_fpu();
+
+#if HIBERNATION
+	if (did_hibernate)
+		enable_preemption();
 
-    if (did_hibernate) {
-        enable_preemption();
-    }
+	kprintf("ret from acpi_sleep_cpu hib=%d\n", did_hibernate);
+#endif
 }
diff --git a/osfmk/i386/acpi.h b/osfmk/i386/acpi.h
index bdb544058..ec5ce3ea4 100644
--- a/osfmk/i386/acpi.h
+++ b/osfmk/i386/acpi.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_ACPI_H_
diff --git a/osfmk/i386/acpi_wakeup.s b/osfmk/i386/acpi_wakeup.s
index 9dc633a76..0d20c8063 100644
--- a/osfmk/i386/acpi_wakeup.s
+++ b/osfmk/i386/acpi_wakeup.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <i386/asm.h>
diff --git a/osfmk/i386/apic.h b/osfmk/i386/apic.h
index 40e558ca9..6cce0663c 100644
--- a/osfmk/i386/apic.h
+++ b/osfmk/i386/apic.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/arch_types.h b/osfmk/i386/arch_types.h
index c04037860..832a3096f 100644
--- a/osfmk/i386/arch_types.h
+++ b/osfmk/i386/arch_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/asm.h b/osfmk/i386/asm.h
index fcaffc055..280a8fae3 100644
--- a/osfmk/i386/asm.h
+++ b/osfmk/i386/asm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -67,7 +73,7 @@
 #endif	/* MACH_KERNEL || _KERNEL */
 
 
-#define S_PC	 0(%esp)
+#define S_PC	 (%esp)
 #define S_ARG0	 4(%esp)
 #define S_ARG1	 8(%esp)
 #define S_ARG2	12(%esp)
@@ -77,7 +83,7 @@
 #define FRAME	pushl %ebp; movl %esp, %ebp
 #define EMARF	leave
 
-#define B_LINK	 0(%ebp)
+#define B_LINK	 (%ebp)
 #define B_PC	 4(%ebp)
 #define B_ARG0	 8(%ebp)
 #define B_ARG1	12(%ebp)
diff --git a/osfmk/i386/asm64.h b/osfmk/i386/asm64.h
index a47fcf369..4b4c9252a 100644
--- a/osfmk/i386/asm64.h
+++ b/osfmk/i386/asm64.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* Helper macros for 64-bit mode switching */
@@ -48,9 +54,9 @@
  * Long jump to 32-bit compatibility mode from 64-bit space.
  */
 #define ENTER_COMPAT_MODE()			\
-	ljmp	*4f				;\
-4:	.long	5f				;\
+	ljmp	*(%rip)				;\
+	.long	4f				;\
 	.word	KERNEL_CS			;\
 	.code32					;\
-5:
+4:
 
diff --git a/osfmk/i386/ast.h b/osfmk/i386/ast.h
index 46f5db9fe..727695a39 100644
--- a/osfmk/i386/ast.h
+++ b/osfmk/i386/ast.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/ast_check.c b/osfmk/i386/ast_check.c
index 2999f58f2..9272e2246 100644
--- a/osfmk/i386/ast_check.c
+++ b/osfmk/i386/ast_check.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/ast_types.h b/osfmk/i386/ast_types.h
index 6ca56f471..68dad8dc5 100644
--- a/osfmk/i386/ast_types.h
+++ b/osfmk/i386/ast_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/bcopy.s b/osfmk/i386/bcopy.s
index fe9b9b883..73642a449 100644
--- a/osfmk/i386/bcopy.s
+++ b/osfmk/i386/bcopy.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -65,7 +71,7 @@ memcpy_common:
 	cld
 /* move longs*/
 	movl	%edx,%ecx
-	sarl	$2,%ecx
+	shrl	$2,%ecx
 	rep
 	movsl
 /* move bytes*/
@@ -97,7 +103,7 @@ ENTRY(bcopy16)
 /* move words */
 0:	cld
 	movl	%edx,%ecx
-	sarl	$1,%ecx
+	shrl	$1,%ecx
 	rep
 	movsw
 /* move bytes */
diff --git a/osfmk/i386/bsd_i386.c b/osfmk/i386/bsd_i386.c
index 75690eb3b..562b0b392 100644
--- a/osfmk/i386/bsd_i386.c
+++ b/osfmk/i386/bsd_i386.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	MACH_BSD
 #include <mach_rt.h>
@@ -52,7 +58,6 @@
 #include <i386/tss.h>
 #include <i386/user_ldt.h>
 #include <i386/fpu.h>
-#include <i386/iopb_entries.h>
 #include <i386/machdep_call.h>
 #include <i386/misc_protos.h>
 #include <i386/cpu_data.h>
@@ -63,13 +68,9 @@
 #include <mach/i386/syscall_sw.h>
 #include <sys/syscall.h>
 #include <sys/kdebug.h>
-#include <sys/ktrace.h>
 #include <sys/errno.h>
 #include <../bsd/sys/sysent.h>
 
-extern struct proc *current_proc(void);
-extern struct proc * kernproc;
-
 kern_return_t
 thread_userstack(
     thread_t,
@@ -77,7 +78,7 @@ thread_userstack(
     thread_state_t,
     unsigned int,
     mach_vm_offset_t *,
-    int *
+	int *
 );
 
 kern_return_t
@@ -102,6 +103,8 @@ thread_compose_cthread_desc(unsigned int addr, pcb_t pcb);
 
 void IOSleep(int);
 
+void thread_set_cthreadself(thread_t thread, uint64_t pself, int isLP64);
+
 /*
  * thread_userstack:
  *
@@ -110,98 +113,96 @@ void IOSleep(int);
  */
 kern_return_t
 thread_userstack(
-		 __unused thread_t   thread,
-		 int                 flavor,
-		 thread_state_t      tstate,
-		 __unused unsigned int        count,
-		 user_addr_t    *user_stack,
-		 int		*customstack
-		 )
+    __unused thread_t   thread,
+    int                 flavor,
+    thread_state_t      tstate,
+    __unused unsigned int        count,
+    user_addr_t    *user_stack,
+	int					*customstack
+)
 {
-        if (customstack)
-	        *customstack = 0;
-
-        switch (flavor) {
-	    case OLD_i386_THREAD_STATE:
-	    case x86_THREAD_STATE32:
-	    {
-	        x86_thread_state32_t *state25;
-
-                state25 = (x86_thread_state32_t *) tstate;
-
-		if (state25->esp)
-			*user_stack = state25->esp;
-		else 
-			*user_stack = VM_USRSTACK32;
-		if (customstack && state25->esp)
-			*customstack = 1;
-		else
-			*customstack = 0;
-		break;
-	    }
+	if (customstack)
+		*customstack = 0;
 
-	    case x86_THREAD_STATE64:
-	    {
-	        x86_thread_state64_t *state25;
+	switch (flavor) {
+	case x86_THREAD_STATE32:
+		{
+			x86_thread_state32_t *state25;
+
+			state25 = (x86_thread_state32_t *) tstate;
+
+			if (state25->esp)
+				*user_stack = state25->esp;
+			else 
+				*user_stack = VM_USRSTACK32;
+			if (customstack && state25->esp)
+				*customstack = 1;
+			else
+				*customstack = 0;
+			break;
+		}
 
-                state25 = (x86_thread_state64_t *) tstate;
+	case x86_THREAD_STATE64:
+		{
+			x86_thread_state64_t *state25;
+
+			state25 = (x86_thread_state64_t *) tstate;
+
+			if (state25->rsp)
+				*user_stack = state25->rsp;
+			else 
+				*user_stack = VM_USRSTACK64;
+			if (customstack && state25->rsp)
+				*customstack = 1;
+			else
+				*customstack = 0;
+			break;
+		}
 
-		if (state25->rsp)
-			*user_stack = state25->rsp;
-		else 
-			*user_stack = VM_USRSTACK64;
-		if (customstack && state25->rsp)
-			*customstack = 1;
-		else
-			*customstack = 0;
-		break;
-	    }
+	default:
+		return (KERN_INVALID_ARGUMENT);
+	}
 
-	    default :
-                return (KERN_INVALID_ARGUMENT);
-        }
-                
-        return (KERN_SUCCESS);
-}    
+	return (KERN_SUCCESS);
+}
 
 
 kern_return_t
 thread_entrypoint(
-		  __unused thread_t   thread,
-		  int                 flavor,
-		  thread_state_t      tstate,
-		  __unused unsigned int        count,
-		  mach_vm_offset_t    *entry_point
-		  )
+    __unused thread_t   thread,
+    int                 flavor,
+    thread_state_t      tstate,
+    __unused unsigned int        count,
+    mach_vm_offset_t    *entry_point
+)
 { 
-        /*
+	/*
 	 * Set a default.
 	 */
-        if (*entry_point == 0)
-	        *entry_point = VM_MIN_ADDRESS;
-		
+	if (*entry_point == 0)
+		*entry_point = VM_MIN_ADDRESS;
+
 	switch (flavor) {
-	    case OLD_i386_THREAD_STATE:
-	    case x86_THREAD_STATE32:
-	    {
-	        x86_thread_state32_t *state25;
+	case x86_THREAD_STATE32:
+		{
+			x86_thread_state32_t *state25;
 
-		state25 = (x86_thread_state32_t *) tstate;
-		*entry_point = state25->eip ? state25->eip: VM_MIN_ADDRESS;
-		break;
-	    }
+			state25 = (i386_thread_state_t *) tstate;
+			*entry_point = state25->eip ? state25->eip: VM_MIN_ADDRESS;
+			break;
+		}
 
-	    case x86_THREAD_STATE64:
-	    {
-	        x86_thread_state64_t *state25;
+	case x86_THREAD_STATE64:
+		{
+			x86_thread_state64_t *state25;
 
-		state25 = (x86_thread_state64_t *) tstate;
-		*entry_point = state25->rip ? state25->rip: VM_MIN_ADDRESS64;
-		break;
-	    }
-    }
-    return (KERN_SUCCESS);
-}   
+			state25 = (x86_thread_state64_t *) tstate;
+			*entry_point = state25->rip ? state25->rip: VM_MIN_ADDRESS64;
+			break;
+		}
+	}
+	return (KERN_SUCCESS);
+}
 
 
 /*
@@ -222,21 +223,21 @@ machine_thread_dup(
 	    (parent_pcb = parent->machine.pcb) == NULL)
 		return (KERN_FAILURE);
 	/*
-	 * Copy over the i386_saved_state registers
+	 * Copy over the x86_saved_state registers
 	 */
 	if (cpu_mode_is64bit()) {
-	        if (thread_is_64bit(parent))
-		        bcopy(USER_REGS64(parent), USER_REGS64(child), sizeof(x86_saved_state64_t));
+		if (thread_is_64bit(parent))
+			bcopy(USER_REGS64(parent), USER_REGS64(child), sizeof(x86_saved_state64_t));
 		else
-		        bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state_compat32_t));
+			bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state_compat32_t));
 	} else
-	        bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state32_t));
+		bcopy(USER_REGS32(parent), USER_REGS32(child), sizeof(x86_saved_state32_t));
 
 	/*
 	 * Check to see if parent is using floating point
 	 * and if so, copy the registers to the child
 	 */
-        fpu_dup_fxstate(parent, child);
+	fpu_dup_fxstate(parent, child);
 
 #ifdef	MACH_BSD
 	/*
@@ -250,13 +251,13 @@ machine_thread_dup(
 	 * FIXME - should a user specified LDT, TSS and V86 info
 	 * be duplicated as well?? - probably not.
 	 */
-        // duplicate any use LDT entry that was set I think this is appropriate.
+	// duplicate any use LDT entry that was set I think this is appropriate.
         if (parent_pcb->uldt_selector!= 0) {
 	        child_pcb->uldt_selector = parent_pcb->uldt_selector;
 		child_pcb->uldt_desc = parent_pcb->uldt_desc;
-        }
+	}
 #endif
-            
+
 	return (KERN_SUCCESS);
 }
 
@@ -268,18 +269,17 @@ void thread_set_child(thread_t child, int pid);
 void
 thread_set_child(thread_t child, int pid)
 {
+	if (thread_is_64bit(child)) {
+		x86_saved_state64_t	*iss64;
 
-        if (thread_is_64bit(child)) {
-	        x86_saved_state64_t	*iss64;
-		
 		iss64 = USER_REGS64(child);
 
 		iss64->rax = pid;
 		iss64->rdx = 1;
 		iss64->isf.rflags &= ~EFL_CF;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(child);
 
 		iss32->eax = pid;
@@ -290,21 +290,21 @@ thread_set_child(thread_t child, int pid)
 
 
 void thread_set_parent(thread_t parent, int pid);
+
 void
 thread_set_parent(thread_t parent, int pid)
 {
+	if (thread_is_64bit(parent)) {
+		x86_saved_state64_t	*iss64;
 
-        if (thread_is_64bit(parent)) {
-	        x86_saved_state64_t	*iss64;
-		
 		iss64 = USER_REGS64(parent);
 
 		iss64->rax = pid;
 		iss64->rdx = 0;
 		iss64->isf.rflags &= ~EFL_CF;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(parent);
 
 		iss32->eax = pid;
@@ -314,27 +314,19 @@ thread_set_parent(thread_t parent, int pid)
 }
 
 
-
 /*
  * System Call handling code
  */
 
-extern struct proc * i386_current_proc(void);
-
 extern long fuword(vm_offset_t);
 
 
-/* following implemented in bsd/dev/i386/unix_signal.c */
-int __pthread_cset(struct sysent *);
-
-void __pthread_creset(struct sysent *);
-
 
 void
 machdep_syscall(x86_saved_state_t *state)
 {
 	int			args[machdep_call_count];
-        int			trapno;
+	int			trapno;
 	int			nargs;
 	machdep_call_t		*entry;
 	x86_saved_state32_t	*regs;
@@ -348,7 +340,7 @@ machdep_syscall(x86_saved_state_t *state)
 #endif
 
 	if (trapno < 0 || trapno >= machdep_call_count) {
-	        regs->eax = (unsigned int)kern_invalid(NULL);
+		regs->eax = (unsigned int)kern_invalid(NULL);
 
 		thread_exception_return();
 		/* NOTREACHED */
@@ -357,50 +349,50 @@ machdep_syscall(x86_saved_state_t *state)
 	nargs = entry->nargs;
 
 	if (nargs != 0) {
-	        if (copyin((user_addr_t) regs->uesp + sizeof (int),
-			   (char *) args, (nargs * sizeof (int)))) {
-		        regs->eax = KERN_INVALID_ADDRESS;
+		if (copyin((user_addr_t) regs->uesp + sizeof (int),
+				(char *) args, (nargs * sizeof (int)))) {
+			regs->eax = KERN_INVALID_ADDRESS;
 
 			thread_exception_return();
 			/* NOTREACHED */
 		}
 	}
 	switch (nargs) {
-	    case 0:
-	        regs->eax = (*entry->routine.args_0)();
+	case 0:
+		regs->eax = (*entry->routine.args_0)();
 		break;
-	    case 1:
+	case 1:
 		regs->eax = (*entry->routine.args_1)(args[0]);
 		break;
-	    case 2:
-		regs->eax = (*entry->routine.args_2)(args[0], args[1]);
+	case 2:
+		regs->eax = (*entry->routine.args_2)(args[0],args[1]);
 		break;
-	    case 3:
-	        if (!entry->bsd_style)
-		        regs->eax = (*entry->routine.args_3)(args[0], args[1], args[2]);
+	case 3:
+		if (!entry->bsd_style)
+			regs->eax = (*entry->routine.args_3)(args[0],args[1],args[2]);
 		else {
-		        int	error;
-			int	rval;
+			int	error;
+			uint32_t	rval;
 
-		        error = (*entry->routine.args_bsd_3)(&rval, args[0], args[1], args[2]);
+			error = (*entry->routine.args_bsd_3)(&rval, args[0], args[1], args[2]);
 			if (error) {
-			        regs->eax = error;
+				regs->eax = error;
 				regs->efl |= EFL_CF;	/* carry bit */
 			} else {
-			        regs->eax = rval;
-			        regs->efl &= ~EFL_CF;
+				regs->eax = rval;
+				regs->efl &= ~EFL_CF;
 			}
 		}
 		break;
-	    case 4:
+	case 4:
 		regs->eax = (*entry->routine.args_4)(args[0], args[1], args[2], args[3]);
 		break;
 
-	    default:
-	        panic("machdep_syscall: too many args");
+	default:
+		panic("machdep_syscall: too many args");
 	}
 	if (current_thread()->funnel_lock)
-	        (void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
+		(void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
 
 	thread_exception_return();
 	/* NOTREACHED */
@@ -410,7 +402,7 @@ machdep_syscall(x86_saved_state_t *state)
 void
 machdep_syscall64(x86_saved_state_t *state)
 {
-        int			trapno;
+	int			trapno;
 	machdep_call_t		*entry;
 	x86_saved_state64_t	*regs;
 
@@ -420,7 +412,7 @@ machdep_syscall64(x86_saved_state_t *state)
 	trapno = regs->rax & SYSCALL_NUMBER_MASK;
 
 	if (trapno < 0 || trapno >= machdep_call_count) {
-	        regs->rax = (unsigned int)kern_invalid(NULL);
+		regs->rax = (unsigned int)kern_invalid(NULL);
 
 		thread_exception_return();
 		/* NOTREACHED */
@@ -428,17 +420,17 @@ machdep_syscall64(x86_saved_state_t *state)
 	entry = &machdep_call_table64[trapno];
 
 	switch (entry->nargs) {
-	    case 0:
-	        regs->rax = (*entry->routine.args_0)();
+	case 0:
+		regs->rax = (*entry->routine.args_0)();
 		break;
-	    case 1:
+	case 1:
 		regs->rax = (*entry->routine.args64_1)(regs->rdi);
 		break;
-	    default:
-	        panic("machdep_syscall64: too many args");
+	default:
+		panic("machdep_syscall64: too many args");
 	}
 	if (current_thread()->funnel_lock)
-	        (void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
+		(void) thread_funnel_set(current_thread()->funnel_lock, FALSE);
 
 	thread_exception_return();
 	/* NOTREACHED */
@@ -470,15 +462,15 @@ thread_compose_cthread_desc(unsigned int addr, pcb_t pcb)
 kern_return_t
 thread_set_cthread_self(uint32_t self)
 {
-   current_thread()->machine.pcb->cthread_self = (uint64_t) self;
-   
-   return (KERN_SUCCESS);
+	current_thread()->machine.pcb->cthread_self = (uint64_t) self;
+
+	return (KERN_SUCCESS);
 }
 
 kern_return_t
 thread_get_cthread_self(void)
 {
-    return ((kern_return_t)current_thread()->machine.pcb->cthread_self);
+	return ((kern_return_t)current_thread()->machine.pcb->cthread_self);
 }
 
 kern_return_t
@@ -496,6 +488,37 @@ thread_fast_set_cthread_self(uint32_t self)
 	return (USER_CTHREAD);
 }
 
+void 
+thread_set_cthreadself(thread_t thread, uint64_t pself, int isLP64)
+{
+	if (isLP64 == 0) {
+		pcb_t			pcb;
+		x86_saved_state32_t	*iss;
+
+		pcb = (pcb_t)thread->machine.pcb;
+		thread_compose_cthread_desc(pself, pcb);
+		pcb->cthread_self = (uint64_t) pself; /* preserve old func too */
+		iss = saved_state32(pcb->iss);
+		iss->gs = USER_CTHREAD;
+	} else {
+		pcb_t			pcb;
+		x86_saved_state64_t	*iss;
+
+		pcb = thread->machine.pcb;
+
+	/* check for canonical address, set 0 otherwise  */
+		if (!IS_USERADDR64_CANONICAL(pself))
+			pself = 0ULL;
+		pcb->cthread_self = pself;
+
+		/* XXX for 64-in-32 */
+		iss = saved_state64(pcb->iss);
+		iss->gs = USER_CTHREAD;
+		thread_compose_cthread_desc((uint32_t) pself, pcb);
+	}
+}
+
+
 kern_return_t
 thread_fast_set_cthread_self64(uint64_t self)
 {
@@ -549,41 +572,41 @@ thread_fast_set_cthread_self64(uint64_t self)
 kern_return_t
 thread_set_user_ldt(uint32_t address, uint32_t size, uint32_t flags)
 {
-    pcb_t pcb;
-    struct fake_descriptor temp;
-    int mycpu;
-
-    if (flags != 0)
-	return -1;		// flags not supported
-    if (size > 0xFFFFF)
-	return -1;		// size too big, 1 meg is the limit
-
-    mp_disable_preemption();
-    mycpu = cpu_number();
-    
-    // create a "fake" descriptor so we can use fix_desc()
-    // to build a real one...
-    //   32 bit default operation size
-    //   standard read/write perms for a data segment
-    pcb = (pcb_t)current_thread()->machine.pcb;
-    temp.offset = address;
-    temp.lim_or_seg = size;
-    temp.size_or_wdct = SZ_32;
-    temp.access = ACC_P|ACC_PL_U|ACC_DATA_W;
+	pcb_t pcb;
+	struct fake_descriptor temp;
+	int mycpu;
+
+	if (flags != 0)
+		return -1;		// flags not supported
+	if (size > 0xFFFFF)
+		return -1;		// size too big, 1 meg is the limit
+
+	mp_disable_preemption();
+	mycpu = cpu_number();
+
+	// create a "fake" descriptor so we can use fix_desc()
+	// to build a real one...
+	//   32 bit default operation size
+	//   standard read/write perms for a data segment
+	pcb = (pcb_t)current_thread()->machine.pcb;
+	temp.offset = address;
+	temp.lim_or_seg = size;
+	temp.size_or_wdct = SZ_32;
+	temp.access = ACC_P|ACC_PL_U|ACC_DATA_W;
 
-    // turn this into a real descriptor
-    fix_desc(&temp,1);
+	// turn this into a real descriptor
+	fix_desc(&temp,1);
 
-    // set up our data in the pcb
-    pcb->uldt_desc = *(struct real_descriptor*)&temp;
-    pcb->uldt_selector = USER_SETTABLE;		// set the selector value
+	// set up our data in the pcb
+	pcb->uldt_desc = *(struct real_descriptor*)&temp;
+	pcb->uldt_selector = USER_SETTABLE;		// set the selector value
 
-    // now set it up in the current table...
-    *ldt_desc_p(USER_SETTABLE) = *(struct real_descriptor*)&temp;
+	// now set it up in the current table...
+	*ldt_desc_p(USER_SETTABLE) = *(struct real_descriptor*)&temp;
 
-    mp_enable_preemption();
+	mp_enable_preemption();
 
-    return USER_SETTABLE;
+	return USER_SETTABLE;
 }
 
 #endif	/* MACH_BSD */
@@ -603,7 +626,6 @@ struct mach_call_args {
 	syscall_arg_t arg9;
 };
 
-
 static kern_return_t
 mach_call_arg_munger32(uint32_t sp, int nargs, int call_number, struct mach_call_args *args);
 
@@ -614,37 +636,34 @@ mach_call_arg_munger32(uint32_t sp, int nargs, int call_number, struct mach_call
 	unsigned int args32[9];
 
 	if (copyin((user_addr_t)(sp + sizeof(int)), (char *)args32, nargs * sizeof (int)))
-	        return KERN_INVALID_ARGUMENT;
+		return KERN_INVALID_ARGUMENT;
 
 	switch (nargs) {
-	    case 9: args->arg9 = args32[8];
-	    case 8: args->arg8 = args32[7];
-	    case 7: args->arg7 = args32[6];
-	    case 6: args->arg6 = args32[5];
-	    case 5: args->arg5 = args32[4];
-	    case 4: args->arg4 = args32[3];
-	    case 3: args->arg3 = args32[2];
-	    case 2: args->arg2 = args32[1];
-	    case 1: args->arg1 = args32[0];
+	case 9: args->arg9 = args32[8];
+	case 8: args->arg8 = args32[7];
+	case 7: args->arg7 = args32[6];
+	case 6: args->arg6 = args32[5];
+	case 5: args->arg5 = args32[4];
+	case 4: args->arg4 = args32[3];
+	case 3: args->arg3 = args32[2];
+	case 2: args->arg2 = args32[1];
+	case 1: args->arg1 = args32[0];
 	}
 	if (call_number == 90) {
-	        /* munge_l for mach_wait_until_trap() */
-	        args->arg1 = (((uint64_t)(args32[0])) | ((((uint64_t)(args32[1]))<<32)));
+		/* munge_l for mach_wait_until_trap() */
+		args->arg1 = (((uint64_t)(args32[0])) | ((((uint64_t)(args32[1]))<<32)));
 	}
 	if (call_number == 93) {
-	        /* munge_wl for mk_timer_arm_trap() */
-	        args->arg2 = (((uint64_t)(args32[1])) | ((((uint64_t)(args32[2]))<<32)));
+		/* munge_wl for mk_timer_arm_trap() */
+		args->arg2 = (((uint64_t)(args32[1])) | ((((uint64_t)(args32[2]))<<32)));
 	}
 
 	return KERN_SUCCESS;
 }
 
 
-__private_extern__ void
-mach_call_munger(x86_saved_state_t *state);
+__private_extern__ void mach_call_munger(x86_saved_state_t *state);
 
-
-__private_extern__
 void
 mach_call_munger(x86_saved_state_t *state)
 {
@@ -664,47 +683,42 @@ mach_call_munger(x86_saved_state_t *state)
 #endif
 
 	if (call_number < 0 || call_number >= mach_trap_count) {
-	        i386_exception(EXC_SYSCALL, call_number, 1);
+		i386_exception(EXC_SYSCALL, call_number, 1);
 		/* NOTREACHED */
 	}
 	mach_call = (mach_call_t)mach_trap_table[call_number].mach_trap_function;
-	
+
 	if (mach_call == (mach_call_t)kern_invalid) {
-	        i386_exception(EXC_SYSCALL, call_number, 1);
+		i386_exception(EXC_SYSCALL, call_number, 1);
 		/* NOTREACHED */
 	}
-	argc = mach_trap_table[call_number].mach_trap_arg_count;
 
+	argc = mach_trap_table[call_number].mach_trap_arg_count;
 	if (argc) {
-	        retval = mach_call_arg_munger32(regs->uesp, argc, call_number, &args);
-		
+		retval = mach_call_arg_munger32(regs->uesp, argc, call_number, &args);
 		if (retval != KERN_SUCCESS) {
-		        regs->eax = retval;
-	
+			regs->eax = retval;
+
 			thread_exception_return();
 			/* NOTREACHED */
 		}
 	}
 	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
-			      (int) args.arg1, (int) args.arg2, (int) args.arg3, (int) args.arg4, 0);
-	
+			(int) args.arg1, (int) args.arg2, (int) args.arg3, (int) args.arg4, 0);
+
 	retval = mach_call(&args);
 
 	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END,
-			      retval, 0, 0, 0, 0);
+			retval, 0, 0, 0, 0);
 	regs->eax = retval;
-	
+
 	thread_exception_return();
 	/* NOTREACHED */
 }
 
 
+__private_extern__ void mach_call_munger64(x86_saved_state_t *regs);
 
-__private_extern__ void
-mach_call_munger64(x86_saved_state_t *state);
-
-
-__private_extern__
 void
 mach_call_munger64(x86_saved_state_t *state)
 {
@@ -718,8 +732,10 @@ mach_call_munger64(x86_saved_state_t *state)
 
 	call_number = regs->rax & SYSCALL_NUMBER_MASK;
 
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | DBG_FUNC_START,
-			      (int) regs->rdi, (int) regs->rsi, (int) regs->rdx, (int) regs->r10, 0);
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,
+					   (call_number)) | DBG_FUNC_START,
+			      (int) regs->rdi, (int) regs->rsi,
+			      (int) regs->rdx, (int) regs->r10, 0);
 	
 	if (call_number < 0 || call_number >= mach_trap_count) {
 	        i386_exception(EXC_SYSCALL, regs->rax, 1);
@@ -747,7 +763,8 @@ mach_call_munger64(x86_saved_state_t *state)
 	}
 	regs->rax = (uint64_t)mach_call((void *)(&regs->rdi));
 	
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,(call_number)) | DBG_FUNC_END,
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_SC,
+					   (call_number)) | DBG_FUNC_END,
 			      (int)regs->rax, 0, 0, 0, 0);
 
 	thread_exception_return();
@@ -755,7 +772,6 @@ mach_call_munger64(x86_saved_state_t *state)
 }
 
 
-
 /*
  * thread_setuserstack:
  *
@@ -767,15 +783,15 @@ thread_setuserstack(
 	thread_t	thread,
 	mach_vm_address_t	user_stack)
 {
-        if (thread_is_64bit(thread)) {
-	        x86_saved_state64_t	*iss64;
+	if (thread_is_64bit(thread)) {
+		x86_saved_state64_t	*iss64;
 
 		iss64 = USER_REGS64(thread);
 
 		iss64->isf.rsp = (uint64_t)user_stack;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(thread);
 
 		iss32->uesp = CAST_DOWN(unsigned int, user_stack);
@@ -793,8 +809,8 @@ thread_adjuserstack(
 	thread_t	thread,
 	int		adjust)
 {
-        if (thread_is_64bit(thread)) {
-	        x86_saved_state64_t	*iss64;
+	if (thread_is_64bit(thread)) {
+		x86_saved_state64_t	*iss64;
 
 		iss64 = USER_REGS64(thread);
 
@@ -802,8 +818,8 @@ thread_adjuserstack(
 
 		return iss64->isf.rsp;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(thread);
 
 		iss32->uesp += adjust;
@@ -821,15 +837,15 @@ thread_adjuserstack(
 void
 thread_setentrypoint(thread_t thread, mach_vm_address_t entry)
 {
-        if (thread_is_64bit(thread)) {
-	        x86_saved_state64_t	*iss64;
+	if (thread_is_64bit(thread)) {
+		x86_saved_state64_t	*iss64;
 
 		iss64 = USER_REGS64(thread);
 
 		iss64->isf.rip = (uint64_t)entry;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(thread);
 
 		iss32->eip = CAST_DOWN(unsigned int, entry);
@@ -837,37 +853,78 @@ thread_setentrypoint(thread_t thread, mach_vm_address_t entry)
 }
 
 
-void
+kern_return_t
 thread_setsinglestep(thread_t thread, int on)
 {
-        if (thread_is_64bit(thread)) {
-	        x86_saved_state64_t	*iss64;
+	if (thread_is_64bit(thread)) {
+		x86_saved_state64_t	*iss64;
 
 		iss64 = USER_REGS64(thread);
 
 		if (on)
-		        iss64->isf.rflags |= EFL_TF;
+			iss64->isf.rflags |= EFL_TF;
 		else
-		        iss64->isf.rflags &= ~EFL_TF;
+			iss64->isf.rflags &= ~EFL_TF;
 	} else {
-	        x86_saved_state32_t	*iss32;
-		
+		x86_saved_state32_t	*iss32;
+
 		iss32 = USER_REGS32(thread);
 
 		if (on)
-		        iss32->efl |= EFL_TF;
+			iss32->efl |= EFL_TF;
 		else
-		        iss32->efl &= ~EFL_TF;
+			iss32->efl &= ~EFL_TF;
 	}
+	
+	return (KERN_SUCCESS);
 }
 
 
 
 /* XXX this should be a struct savearea so that CHUD will work better on x86 */
 void *
-find_user_regs(
-	thread_t        thread)
+find_user_regs(thread_t thread)
 {
 	return USER_STATE(thread);
 }
 
+void *
+get_user_regs(thread_t th)
+{
+	if (th->machine.pcb)
+		return(USER_STATE(th));
+	else {
+		printf("[get_user_regs: thread does not have pcb]");
+		return NULL;
+	}
+}
+
+#if CONFIG_DTRACE
+/*
+ * DTrace would like to have a peek at the kernel interrupt state, if available.
+ * Based on osfmk/chud/i386/chud_thread_i386.c:chudxnu_thread_get_state(), which see.
+ */
+x86_saved_state32_t *find_kern_regs(thread_t);
+
+x86_saved_state32_t *
+find_kern_regs(thread_t thread)
+{
+	if (thread == current_thread() && 
+		NULL != current_cpu_datap()->cpu_int_state &&
+		!(USER_STATE(thread) == current_cpu_datap()->cpu_int_state &&
+		  current_cpu_datap()->cpu_interrupt_level == 1)) {
+
+		return saved_state32(current_cpu_datap()->cpu_int_state);
+	} else {
+		return NULL;
+	}
+}
+
+vm_offset_t dtrace_get_cpu_int_stack_top(void);
+
+vm_offset_t
+dtrace_get_cpu_int_stack_top(void)
+{
+	return current_cpu_datap()->cpu_int_stack_top;
+}
+#endif
diff --git a/osfmk/i386/bzero.s b/osfmk/i386/bzero.s
index e1d668695..28bdf2217 100644
--- a/osfmk/i386/bzero.s
+++ b/osfmk/i386/bzero.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/commpage/atomic.s b/osfmk/i386/commpage/atomic.s
index 790e08f38..a8d6a6b52 100644
--- a/osfmk/i386/commpage/atomic.s
+++ b/osfmk/i386/commpage/atomic.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
@@ -147,6 +153,86 @@ Latomic_add32_up:
 	ret
 
     COMMPAGE_DESCRIPTOR(atomic_add32_up,_COMM_PAGE_ATOMIC_ADD32,kUP,0)
+    
+    
+// OSMemoryBarrier()
+// These are used both in 32 and 64-bit mode.  We use a fence even on UP
+// machines, so this function can be used with nontemporal stores.
+
+Lmemory_barrier:
+	lock
+	addl	$0,(%esp)
+	ret
+	
+    COMMPAGE_DESCRIPTOR(memory_barrier,_COMM_PAGE_MEMORY_BARRIER,0,kHasSSE2);
+
+Lmemory_barrier_sse2:
+	mfence
+	ret
+	
+    COMMPAGE_DESCRIPTOR(memory_barrier_sse2,_COMM_PAGE_MEMORY_BARRIER,kHasSSE2,0);
+    
+
+/*
+ *	typedef	volatile struct {
+ *		void	*opaque1;  <-- ptr to 1st queue element or null
+ *		long	 opaque2;  <-- generation count
+ *	} OSQueueHead;
+ *
+ * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
+ */
+
+LAtomicEnqueue:
+	pushl	%edi
+	pushl	%esi
+	pushl	%ebx
+	movl	16(%esp),%edi	// %edi == ptr to list head
+	movl	20(%esp),%ebx	// %ebx == new
+	movl	24(%esp),%esi	// %esi == offset
+	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
+	movl	4(%edi),%edx	// %edx == current generation count
+1:
+	movl	%eax,(%ebx,%esi)// link to old list head from new element
+	movl	%edx,%ecx
+	incl	%ecx		// increment generation count
+	lock			// always lock for now...
+	cmpxchg8b (%edi)	// ...push on new element
+	jnz	1b
+	popl	%ebx
+	popl	%esi
+	popl	%edi
+	ret
+	
+    COMMPAGE_DESCRIPTOR(AtomicEnqueue,_COMM_PAGE_ENQUEUE,0,0)
+	
+	
+/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
+
+LAtomicDequeue:
+	pushl	%edi
+	pushl	%esi
+	pushl	%ebx
+	movl	16(%esp),%edi	// %edi == ptr to list head
+	movl	20(%esp),%esi	// %esi == offset
+	movl	(%edi),%eax	// %eax == ptr to 1st element in Q
+	movl	4(%edi),%edx	// %edx == current generation count
+1:
+	testl	%eax,%eax	// list empty?
+	jz	2f		// yes
+	movl	(%eax,%esi),%ebx // point to 2nd in Q
+	movl	%edx,%ecx
+	incl	%ecx		// increment generation count
+	lock			// always lock for now...
+	cmpxchg8b (%edi)	// ...pop off 1st element
+	jnz	1b
+2:
+	popl	%ebx
+	popl	%esi
+	popl	%edi
+	ret			// ptr to 1st element in Q still in %eax
+	
+    COMMPAGE_DESCRIPTOR(AtomicDequeue,_COMM_PAGE_DEQUEUE,0,0)
+
 
 
 /************************* x86_64 versions follow **************************/
@@ -297,3 +383,55 @@ Latomic_add64_up_64:
 	ret
 
     COMMPAGE_DESCRIPTOR(atomic_add64_up_64,_COMM_PAGE_ATOMIC_ADD64,kUP,0)
+
+
+/*
+ *	typedef	volatile struct {
+ *		void	*opaque1;  <-- ptr to 1st queue element or null
+ *		long	 opaque2;  <-- generation count
+ *	} OSQueueHead;
+ *
+ * void  OSAtomicEnqueue( OSQueueHead *list, void *new, size_t offset);
+ */
+
+	.code64
+LAtomicEnqueue_64:		// %rdi == list head, %rsi == new, %rdx == offset
+	pushq	%rbx
+	movq	%rsi,%rbx	// %rbx == new
+	movq	%rdx,%rsi	// %rsi == offset
+	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
+	movq	8(%rdi),%rdx	// %rdx == current generation count
+1:
+	movq	%rax,(%rbx,%rsi)// link to old list head from new element
+	movq	%rdx,%rcx
+	incq	%rcx		// increment generation count
+	lock			// always lock for now...
+	cmpxchg16b (%rdi)	// ...push on new element
+	jnz	1b
+	popq	%rbx
+	ret
+	
+    COMMPAGE_DESCRIPTOR(AtomicEnqueue_64,_COMM_PAGE_ENQUEUE,0,0)
+	
+	
+/* void* OSAtomicDequeue( OSQueueHead *list, size_t offset); */
+
+	.code64
+LAtomicDequeue_64:		// %rdi == list head, %rsi == offset
+	pushq	%rbx
+	movq	(%rdi),%rax	// %rax == ptr to 1st element in Q
+	movq	8(%rdi),%rdx	// %rdx == current generation count
+1:
+	testq	%rax,%rax	// list empty?
+	jz	2f		// yes
+	movq	(%rax,%rsi),%rbx // point to 2nd in Q
+	movq	%rdx,%rcx
+	incq	%rcx		// increment generation count
+	lock			// always lock for now...
+	cmpxchg16b (%rdi)	// ...pop off 1st element
+	jnz	1b
+2:
+	popq	%rbx
+	ret			// ptr to 1st element in Q still in %rax
+	
+    COMMPAGE_DESCRIPTOR(AtomicDequeue_64,_COMM_PAGE_DEQUEUE,0,0)
diff --git a/osfmk/i386/commpage/bcopy_scalar.s b/osfmk/i386/commpage/bcopy_scalar.s
index d89188d7c..dfedd73b3 100644
--- a/osfmk/i386/commpage/bcopy_scalar.s
+++ b/osfmk/i386/commpage/bcopy_scalar.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*-
diff --git a/osfmk/i386/commpage/bcopy_sse3.s b/osfmk/i386/commpage/bcopy_sse2.s
similarity index 91%
rename from osfmk/i386/commpage/bcopy_sse3.s
rename to osfmk/i386/commpage/bcopy_sse2.s
index 21d7b8a12..5e5fa35c6 100644
--- a/osfmk/i386/commpage/bcopy_sse3.s
+++ b/osfmk/i386/commpage/bcopy_sse2.s
@@ -1,30 +1,36 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
         
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /*
- * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE3
+ * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE2
  * and 64-byte cache lines, such as Core and Core 2.
  *
  * The following #defines are tightly coupled to the u-architecture:
@@ -40,7 +46,7 @@
  
         .text
         .align 5, 0x90
-Lbcopy_sse3:				// void bcopy(const void *src, void *dst, size_t len)
+Lbcopy_sse2:				// void bcopy(const void *src, void *dst, size_t len)
 	pushl	%ebp			// set up a frame for backtraces
 	movl	%esp,%ebp
         pushl   %esi
@@ -467,4 +473,4 @@ LReverseUnalignedLoop:                  // loop over 64-byte chunks
         jmp     LReverseShort           // copy remaining 0..63 bytes and done
 
 
-	COMMPAGE_DESCRIPTOR(bcopy_sse3,_COMM_PAGE_BCOPY,kHasSSE2+kCache64,kHasSupplementalSSE3)
+	COMMPAGE_DESCRIPTOR(bcopy_sse2,_COMM_PAGE_BCOPY,kHasSSE2+kCache64,kHasSupplementalSSE3)
diff --git a/osfmk/i386/commpage/bcopy_sse4.s b/osfmk/i386/commpage/bcopy_sse3x.s
similarity index 94%
rename from osfmk/i386/commpage/bcopy_sse4.s
rename to osfmk/i386/commpage/bcopy_sse3x.s
index a56ee219d..418635d9b 100644
--- a/osfmk/i386/commpage/bcopy_sse4.s
+++ b/osfmk/i386/commpage/bcopy_sse3x.s
@@ -1,31 +1,37 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
         
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /*
- * The bcopy/memcpy loops, tuned for Pentium-M class processors with SSE4
- * and 64-byte cache lines.
+ * The bcopy/memcpy loops, tuned for Pentium-M class processors with
+ * Supplemental SSE3 and 64-byte cache lines.
  *
  * The following #defines are tightly coupled to the u-architecture:
  */
@@ -40,7 +46,7 @@
         .text
         .align	5, 0x90
 LZero:
-Lbcopy_sse4:				// void bcopy(const void *src, void *dst, size_t len)
+Lbcopy_sse3x:				// void bcopy(const void *src, void *dst, size_t len)
 	pushl	%ebp			// set up a frame for backtraces
 	movl	%esp,%ebp
         pushl   %esi
@@ -796,4 +802,4 @@ LReverseUnalignedLoop:                  // loop over 64-byte chunks
         jmp     LReverseShort           // copy remaining 0..63 bytes and done
 
 
-	COMMPAGE_DESCRIPTOR(bcopy_sse4,_COMM_PAGE_BCOPY,kHasSSE3+kHasSupplementalSSE3+kCache64,0)
+	COMMPAGE_DESCRIPTOR(bcopy_sse3x,_COMM_PAGE_BCOPY,kHasSSE2+kHasSupplementalSSE3+kCache64,0)
diff --git a/osfmk/i386/commpage/bcopy_sse4_64.s b/osfmk/i386/commpage/bcopy_sse3x_64.s
similarity index 94%
rename from osfmk/i386/commpage/bcopy_sse4_64.s
rename to osfmk/i386/commpage/bcopy_sse3x_64.s
index a055422ab..eae8b0ce8 100644
--- a/osfmk/i386/commpage/bcopy_sse4_64.s
+++ b/osfmk/i386/commpage/bcopy_sse3x_64.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
         
 #include <machine/cpu_capabilities.h>
@@ -25,7 +31,7 @@
 
 /*
  * The bcopy/memcpy loops, tuned for 64-bit Pentium-M class processors with 
- * SSE4 and 64-byte cache lines.  This is the 64-bit version.
+ * Supplemental SSE3 and 64-byte cache lines.  This is the 64-bit version.
  *
  * The following #defines are tightly coupled to the u-architecture:
  */
@@ -41,7 +47,7 @@
 	.code64
         .align 5, 0x90
 LZero:
-Lbcopy_sse4_64:				// void bcopy(const void *src, void *dst, size_t len)
+Lbcopy_sse3x_64:				// void bcopy(const void *src, void *dst, size_t len)
 	pushq	%rbp			// set up a frame for backtraces
 	movq	%rsp,%rbp
 	movq	%rsi,%rax		// copy dest ptr
@@ -788,4 +794,4 @@ LReverseUnalignedLoop:                  // loop over 64-byte chunks
         jmp     LReverseShort           // copy remaining 0..63 bytes and done
 
 
-	COMMPAGE_DESCRIPTOR(bcopy_sse4_64,_COMM_PAGE_BCOPY,kHasSSE3+kHasSupplementalSSE3+kCache64,0)
+	COMMPAGE_DESCRIPTOR(bcopy_sse3x_64,_COMM_PAGE_BCOPY,kHasSSE2+kHasSupplementalSSE3+kCache64,0)
diff --git a/osfmk/i386/commpage/bzero_scalar.s b/osfmk/i386/commpage/bzero_scalar.s
index 9e3195681..7839413d7 100644
--- a/osfmk/i386/commpage/bzero_scalar.s
+++ b/osfmk/i386/commpage/bzero_scalar.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1993 Winning Strategies, Inc.
diff --git a/osfmk/i386/commpage/bzero_sse3.s b/osfmk/i386/commpage/bzero_sse2.s
similarity index 72%
rename from osfmk/i386/commpage/bzero_sse3.s
rename to osfmk/i386/commpage/bzero_sse2.s
index a7a6963eb..49c94750d 100644
--- a/osfmk/i386/commpage/bzero_sse3.s
+++ b/osfmk/i386/commpage/bzero_sse2.s
@@ -1,30 +1,36 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /*
- * Bzero, tuned for Pentium-M class processors with SSE3
+ * Bzero, tuned for Pentium-M class processors with SSE2
  * and 64-byte cache lines.
  *
  * This routine is also used for memset(p,0,n), which is a common case
@@ -38,7 +44,7 @@
 
         .text
         .align  5, 0x90
-Lbzero_sse3:                            // void	bzero(void *b, size_t len);
+Lbzero_sse2:                            // void	bzero(void *b, size_t len);
 	pushl	%ebp			// set up a frame for backtraces
 	movl	%esp,%ebp
         pushl   %edi
@@ -155,4 +161,4 @@ LVeryLong:
 	jmp	Lshort
 
 
-	COMMPAGE_DESCRIPTOR(bzero_sse3,_COMM_PAGE_BZERO,kHasSSE2,0)
+	COMMPAGE_DESCRIPTOR(bzero_sse2,_COMM_PAGE_BZERO,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/bzero_sse3_64.s b/osfmk/i386/commpage/bzero_sse2_64.s
similarity index 74%
rename from osfmk/i386/commpage/bzero_sse3_64.s
rename to osfmk/i386/commpage/bzero_sse2_64.s
index 98d012f3b..d82d77e6f 100644
--- a/osfmk/i386/commpage/bzero_sse3_64.s
+++ b/osfmk/i386/commpage/bzero_sse2_64.s
@@ -1,30 +1,36 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /*
- * Bzero, tuned for Pentium-M class processors with SSE3
+ * Bzero, tuned for Pentium-M class processors with SSE2
  * and 64-byte cache lines.  This is the 64-bit version.
  *
  * This routine is also used for memset(p,0,n), which is a common case
@@ -39,7 +45,7 @@
         .text
 	.code64
         .align  5, 0x90
-Lbzero_sse3_64:                         // void	bzero(void *b, size_t len);
+Lbzero_sse2_64:                         // void	bzero(void *b, size_t len);
 	pushq	%rbp			// set up a frame for backtraces
 	movq	%rsp,%rbp
         xorl    %eax,%eax               // set fill data to 0
@@ -155,4 +161,4 @@ LVeryLong:
 	jmp	Lshort
 
 
-	COMMPAGE_DESCRIPTOR(bzero_sse3_64,_COMM_PAGE_BZERO,kHasSSE3,0)
+	COMMPAGE_DESCRIPTOR(bzero_sse2_64,_COMM_PAGE_BZERO,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/cacheflush.s b/osfmk/i386/commpage/cacheflush.s
index 876597fac..00632d656 100644
--- a/osfmk/i386/commpage/cacheflush.s
+++ b/osfmk/i386/commpage/cacheflush.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
@@ -30,16 +36,17 @@
 // 32-bit version
 
 Lsys_flush_dcache:
-	movl	4(%esp),%ecx		// get length
-	movl	8(%esp),%edx		// get ptr
+	movl	8(%esp),%ecx		// get length
+	movl	4(%esp),%edx		// get ptr
 	testl	%ecx,%ecx		// length 0?
 	jz	2f			// yes
 	mfence				// ensure previous stores make it to memory
+	clflush	-1(%edx,%ecx)		// make sure last line is flushed
 1:
 	clflush	(%edx)			// flush a line
 	addl	$64,%edx
 	subl	$64,%ecx
-	jnc	1b
+	ja	1b
 	mfence				// make sure memory is updated before we return
 2:
 	ret
@@ -54,11 +61,12 @@ Lsys_flush_dcache_64:			// %rdi = ptr,  %rsi = length
 	testq	%rsi,%rsi		// length 0?
 	jz	2f			// yes
 	mfence				// ensure previous stores make it to memory
+	clflush	-1(%rdi,%rsi)		// make sure last line is flushed
 1:
 	clflush	(%rdi)			// flush a line
 	addq	$64,%rdi
 	subq	$64,%rsi
-	jnc	1b
+	ja	1b
 	mfence				// make sure memory is updated before we return
 2:
 	ret
diff --git a/osfmk/i386/commpage/commpage.c b/osfmk/i386/commpage/commpage.c
index 5e347ac29..b2bd5af50 100644
--- a/osfmk/i386/commpage/commpage.c
+++ b/osfmk/i386/commpage/commpage.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -44,6 +50,8 @@
 #include <mach/vm_map.h>
 #include <i386/machine_routines.h>
 #include <i386/misc_protos.h>
+#include <i386/tsc.h>
+#include <i386/cpu_data.h>
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 #include <machine/pmap.h>
@@ -61,8 +69,8 @@ extern	commpage_descriptor*	commpage_64_routines[];
 extern	commpage_descriptor sigdata_descriptor;
 extern	commpage_descriptor *ba_descriptors[];
 
-extern vm_map_t	com_region_map32;	// the shared submap, set up in vm init
-extern vm_map_t	com_region_map64;	// the shared submap, set up in vm init
+extern vm_map_t	commpage32_map;	// the shared submap, set up in vm init
+extern vm_map_t	commpage64_map;	// the shared submap, set up in vm init
 
 char	*commPagePtr32 = NULL;		// virtual addr in kernel map of 32-bit commpage
 char	*commPagePtr64 = NULL;		// ...and of 64-bit commpage
@@ -70,9 +78,6 @@ int     _cpu_capabilities = 0;          // define the capability vector
 
 int	noVMX = 0;		/* if true, do not set kHasAltivec in ppc _cpu_capabilities */
 
-void*	dsmos_blobs[3];         /* ptrs to the system integrity data in each commpage */
-int	dsmos_blob_count = 0;
-
 static uintptr_t next;			// next available byte in comm page
 static int     	cur_routine;		// comm page address of "current" routine
 static int     	matched;		// true if we've found a match for "current" routine
@@ -80,6 +85,9 @@ static int     	matched;		// true if we've found a match for "current" routine
 static char    *commPagePtr;		// virtual addr in kernel map of commpage we are working on
 static size_t	commPageBaseOffset;	// add to 32-bit runtime address to get offset in commpage
 
+static	commpage_time_data	*time_data32 = NULL;
+static	commpage_time_data	*time_data64 = NULL;
+
 /* Allocate the commpage and add to the shared submap created by vm:
  * 	1. allocate a page in the kernel map (RW)
  *	2. wire it down
@@ -89,10 +97,10 @@ static size_t	commPageBaseOffset;	// add to 32-bit runtime address to get offset
 
 static  void*
 commpage_allocate( 
-	vm_map_t	submap,			// com_region_map32 or com_region_map64
+	vm_map_t	submap,			// commpage32_map or commpage_map64
 	size_t		area_used )		// _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
 {
-	vm_offset_t	kernel_addr;		// address of commpage in kernel map
+	vm_offset_t	kernel_addr = 0;	// address of commpage in kernel map
 	vm_offset_t	zero = 0;
 	vm_size_t	size = area_used;	// size actually populated
 	vm_map_entry_t	entry;
@@ -101,7 +109,7 @@ commpage_allocate(
 	if (submap == NULL)
 		panic("commpage submap is null");
 
-	if (vm_allocate(kernel_map,&kernel_addr,area_used,VM_FLAGS_ANYWHERE))
+	if (vm_map(kernel_map,&kernel_addr,area_used,0,VM_FLAGS_ANYWHERE,NULL,0,FALSE,VM_PROT_ALL,VM_PROT_ALL,VM_INHERIT_NONE))
 		panic("cannot allocate commpage");
 
 	if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+area_used,VM_PROT_DEFAULT,FALSE))
@@ -122,7 +130,7 @@ commpage_allocate(
 	if (mach_make_memory_entry( kernel_map,		// target map
 				    &size,		// size 
 				    kernel_addr,	// offset (address in kernel map)
-				    VM_PROT_DEFAULT,	// map it RW
+				    VM_PROT_ALL,	// map it RWX
 				    &handle,		// this is the object handle we get
 				    NULL ))		// parent_entry (what is this?)
 		panic("cannot make entry for commpage");
@@ -135,8 +143,8 @@ commpage_allocate(
 			handle,				// port is the memory entry we just made
 			0,                              // offset (map 1st page in memory entry)
 			FALSE,                          // copy
-			VM_PROT_READ,                   // cur_protection (R-only in user map)
-			VM_PROT_READ,                   // max_protection
+			VM_PROT_READ|VM_PROT_EXECUTE,   // cur_protection (R-only in user map)
+			VM_PROT_READ|VM_PROT_EXECUTE,   // max_protection
 			VM_INHERIT_SHARE ))             // inheritance
 		panic("cannot map commpage");
 
@@ -185,6 +193,12 @@ commpage_init_cpu_capabilities( void )
 	ml_cpu_get_info(&cpu_info);
 	
 	switch (cpu_info.vector_unit) {
+		case 8:
+			bits |= kHasSSE4_2;
+			/* fall thru */
+		case 7:
+			bits |= kHasSSE4_1;
+			/* fall thru */
 		case 6:
 			bits |= kHasSupplementalSSE3;
 			/* fall thru */
@@ -227,11 +241,14 @@ commpage_init_cpu_capabilities( void )
 	if (cpu_mode_is64bit())			// k64Bit means processor is 64-bit capable
 		bits |= k64Bit;
 
+	if (tscFreq <= SLOW_TSC_THRESHOLD)	/* is TSC too slow for _commpage_nanotime?  */
+		bits |= kSlow;
+
 	_cpu_capabilities = bits;		// set kernel version for use by drivers etc
 }
 
 int
-_get_cpu_capabilities()
+_get_cpu_capabilities(void)
 {
 	return _cpu_capabilities;
 }
@@ -247,7 +264,7 @@ commpage_stuff(
     void	*dest = commpage_addr_of(address);
     
     if ((uintptr_t)dest < next)
-        panic("commpage overlap at address 0x%x, 0x%x < 0x%x", address, dest, next);
+        panic("commpage overlap at address 0x%x, %p < 0x%lx", address, dest, next);
     
     bcopy(source,dest,length);
     
@@ -299,7 +316,7 @@ commpage_stuff_routine(
     
     if (rd->commpage_address != cur_routine) {
         if ((cur_routine!=0) && (matched==0))
-            panic("commpage no match for last, next address %08x", rd->commpage_address);
+            panic("commpage no match for last, next address %08lx", rd->commpage_address);
         cur_routine = rd->commpage_address;
         matched = 0;
     }
@@ -309,7 +326,7 @@ commpage_stuff_routine(
     
     if ((must == rd->musthave) && (cant == 0)) {
         if (matched)
-            panic("commpage multiple matches for address %08x", rd->commpage_address);
+            panic("commpage multiple matches for address %08lx", rd->commpage_address);
         matched = 1;
         
         commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length);
@@ -323,12 +340,13 @@ commpage_stuff_routine(
 
 static void
 commpage_populate_one( 
-	vm_map_t	submap,		// com_region_map32 or com_region_map64
+	vm_map_t	submap,		// commpage32_map or compage64_map
 	char **		kernAddressPtr,	// &commPagePtr32 or &commPagePtr64
 	size_t		area_used,	// _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
 	size_t		base_offset,	// will become commPageBaseOffset
 	commpage_descriptor** commpage_routines, // list of routine ptrs for this commpage
 	boolean_t	legacy,		// true if 32-bit commpage
+	commpage_time_data** time_data,	// &time_data32 or &time_data64
 	const char*	signature )	// "commpage 32-bit" or "commpage 64-bit"
 {
    	short   c2;
@@ -343,6 +361,8 @@ commpage_populate_one(
 	commPagePtr = (char *)commpage_allocate( submap, (vm_size_t) area_used );
 	*kernAddressPtr = commPagePtr;				// save address either in commPagePtr32 or 64
 	commPageBaseOffset = base_offset;
+	
+	*time_data = commpage_addr_of( _COMM_PAGE_TIME_DATA_START );
 
 	/* Stuff in the constants.  We move things into the comm page in strictly
 	* ascending order, so we can check for overlap and panic if so.
@@ -384,7 +404,7 @@ commpage_populate_one(
 		panic("commpage no match on last routine");
 
 	if (next > (uintptr_t)_COMM_PAGE_END)
-		panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%08x", next, (uintptr_t)commPagePtr);
+		panic("commpage overflow: next = 0x%08lx, commPagePtr = 0x%08lx", next, (uintptr_t)commPagePtr);
 
 	if ( legacy ) {
 		next = (uintptr_t) NULL;
@@ -394,10 +414,6 @@ commpage_populate_one(
 		next = (uintptr_t) NULL;
 		commpage_stuff_routine(&sigdata_descriptor);
 	}	
-
-	/* salt away a ptr to the system integrity data in this commpage */
-	dsmos_blobs[dsmos_blob_count++] = 
-		commpage_addr_of( _COMM_PAGE_SYSTEM_INTEGRITY );
 }
 
 
@@ -413,23 +429,27 @@ commpage_populate( void )
 {
 	commpage_init_cpu_capabilities();
 	
-	commpage_populate_one(	com_region_map32, 
+	commpage_populate_one(	commpage32_map, 
 				&commPagePtr32,
 				_COMM_PAGE32_AREA_USED,
 				_COMM_PAGE32_BASE_ADDRESS,
 				commpage_32_routines, 
 				TRUE,			/* legacy (32-bit) commpage */
+				&time_data32,
 				"commpage 32-bit");
 	pmap_commpage32_init((vm_offset_t) commPagePtr32, _COMM_PAGE32_BASE_ADDRESS, 
 			   _COMM_PAGE32_AREA_USED/INTEL_PGBYTES);
+			   
+	time_data64 = time_data32;			/* if no 64-bit commpage, point to 32-bit */
 
 	if (_cpu_capabilities & k64Bit) {
-		commpage_populate_one(	com_region_map64, 
+		commpage_populate_one(	commpage64_map, 
 					&commPagePtr64,
 					_COMM_PAGE64_AREA_USED,
 					_COMM_PAGE32_START_ADDRESS, /* because kernel is built 32-bit */
 					commpage_64_routines, 
 					FALSE,		/* not a legacy commpage */
+					&time_data64,
 					"commpage 64-bit");
 		pmap_commpage64_init((vm_offset_t) commPagePtr64, _COMM_PAGE64_BASE_ADDRESS, 
 				   _COMM_PAGE64_AREA_USED/INTEL_PGBYTES);
@@ -437,3 +457,101 @@ commpage_populate( void )
 
 	rtc_nanotime_init_commpage();
 }
+
+
+/* Update commpage nanotime information.  Note that we interleave
+ * setting the 32- and 64-bit commpages, in order to keep nanotime more
+ * nearly in sync between the two environments.
+ *
+ * This routine must be serialized by some external means, ie a lock.
+ */
+
+void
+commpage_set_nanotime(
+	uint64_t	tsc_base,
+	uint64_t	ns_base,
+	uint32_t	scale,
+	uint32_t	shift )
+{
+	commpage_time_data	*p32 = time_data32;
+	commpage_time_data	*p64 = time_data64;
+	static uint32_t	generation = 0;
+	uint32_t	next_gen;
+	
+	if (p32 == NULL)		/* have commpages been allocated yet? */
+		return;
+		
+	if ( generation != p32->nt_generation )
+		panic("nanotime trouble 1");	/* possibly not serialized */
+	if ( ns_base < p32->nt_ns_base )
+		panic("nanotime trouble 2");
+	if ((shift != 32) && ((_cpu_capabilities & kSlow)==0) )
+		panic("nanotime trouble 3");
+		
+	next_gen = ++generation;
+	if (next_gen == 0)
+		next_gen = ++generation;
+	
+	p32->nt_generation = 0;		/* mark invalid, so commpage won't try to use it */
+	p64->nt_generation = 0;
+	
+	p32->nt_tsc_base = tsc_base;
+	p64->nt_tsc_base = tsc_base;
+	
+	p32->nt_ns_base = ns_base;
+	p64->nt_ns_base = ns_base;
+	
+	p32->nt_scale = scale;
+	p64->nt_scale = scale;
+	
+	p32->nt_shift = shift;
+	p64->nt_shift = shift;
+	
+	p32->nt_generation = next_gen;	/* mark data as valid */
+	p64->nt_generation = next_gen;
+}
+
+
+/* Disable commpage gettimeofday(), forcing commpage to call through to the kernel.  */
+
+void
+commpage_disable_timestamp( void )
+{
+	time_data32->gtod_generation = 0;
+	time_data64->gtod_generation = 0;
+}
+
+
+/* Update commpage gettimeofday() information.  As with nanotime(), we interleave
+ * updates to the 32- and 64-bit commpage, in order to keep time more nearly in sync 
+ * between the two environments.
+ *
+ * This routine must be serializeed by some external means, ie a lock.
+ */
+ 
+ void
+ commpage_set_timestamp(
+	uint64_t	abstime,
+	uint64_t	secs )
+{
+	commpage_time_data	*p32 = time_data32;
+	commpage_time_data	*p64 = time_data64;
+	static uint32_t	generation = 0;
+	uint32_t	next_gen;
+	
+	next_gen = ++generation;
+	if (next_gen == 0)
+		next_gen = ++generation;
+	
+	p32->gtod_generation = 0;		/* mark invalid, so commpage won't try to use it */
+	p64->gtod_generation = 0;
+	
+	p32->gtod_ns_base = abstime;
+	p64->gtod_ns_base = abstime;
+	
+	p32->gtod_sec_base = secs;
+	p64->gtod_sec_base = secs;
+	
+	p32->gtod_generation = next_gen;	/* mark data as valid */
+	p64->gtod_generation = next_gen;
+}
diff --git a/osfmk/i386/commpage/commpage.h b/osfmk/i386/commpage/commpage.h
index 2a5aaae11..0288a906c 100644
--- a/osfmk/i386/commpage/commpage.h
+++ b/osfmk/i386/commpage/commpage.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_COMMPAGE_H
@@ -66,15 +72,31 @@ typedef	struct	commpage_descriptor	{
 } commpage_descriptor;
 
 
+/* Warning: following structure must match the layout of the commpage.  */
+/* This is the data starting at _COMM_PAGE_TIME_DATA_START, ie for nanotime() and gettimeofday() */
+
+typedef	volatile struct	commpage_time_data	{
+	uint64_t	nt_tsc_base;				// _COMM_PAGE_NT_TSC_BASE
+	uint32_t	nt_scale;				// _COMM_PAGE_NT_SCALE
+	uint32_t	nt_shift;				// _COMM_PAGE_NT_SHIFT
+	uint64_t	nt_ns_base;				// _COMM_PAGE_NT_NS_BASE
+	uint32_t	nt_generation;				// _COMM_PAGE_NT_GENERATION
+	uint32_t	gtod_generation;			// _COMM_PAGE_GTOD_GENERATION
+	uint64_t	gtod_ns_base;				// _COMM_PAGE_GTOD_NS_BASE
+	uint64_t	gtod_sec_base;				// _COMM_PAGE_GTOD_SEC_BASE
+} commpage_time_data;
+
+
 extern	char	*commPagePtr32;				// virt address of 32-bit commpage in kernel map
 extern	char	*commPagePtr64;				// ...and of 64-bit commpage
 
-extern	void	_commpage_set_timestamp(uint64_t abstime, uint64_t secs);
-#define commpage_set_timestamp(x, y, z)		_commpage_set_timestamp((x), (y))
+extern	void	commpage_set_timestamp(uint64_t abstime, uint64_t secs);
+
+extern	void	commpage_disable_timestamp( void );
 
 extern  void	commpage_set_nanotime(uint64_t tsc_base, uint64_t ns_base, uint32_t scale, uint32_t shift);
 
-#include <kern/page_decrypt.h>
+extern	void	commpage_sched_gen_inc(void);
 
 #endif	/* __ASSEMBLER__ */
 
diff --git a/osfmk/i386/commpage/commpage_asm.s b/osfmk/i386/commpage/commpage_asm.s
index f4ff05ed6..e9604430c 100644
--- a/osfmk/i386/commpage/commpage_asm.s
+++ b/osfmk/i386/commpage/commpage_asm.s
@@ -1,110 +1,60 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
 
-        .text
-        .align  2, 0x90
-	.globl	__commpage_set_timestamp
-/* extern void	_commpage_set_timestamp(uint64_t abstime, uint64_t secs); */
-__commpage_set_timestamp:
-	push	%ebp
-	mov	%esp,%ebp
-
-	mov	_commPagePtr32,%ecx
-	sub	$ _COMM_PAGE32_BASE_ADDRESS,%ecx
-	mov	_commPagePtr64,%edx			/* point to 64-bit commpage too */
-	mov	%edx,%eax
-	sub	$ _COMM_PAGE32_START_ADDRESS,%edx	/* because kernel is built 32-bit */
-	test	%eax,%eax
-	cmovz	%ecx,%edx				/* if no 64-bit commpage, point to 32 with both */
-
-	movl	$0,_COMM_PAGE_TIMEENABLE(%ecx)
-	movl	$0,_COMM_PAGE_TIMEENABLE(%edx)
 
-	mov	8(%ebp),%eax
-	or	12(%ebp),%eax
-	je	1f
-
-	mov	8(%ebp),%eax
-	mov	%eax,_COMM_PAGE_TIMEBASE(%ecx)
-	mov	%eax,_COMM_PAGE_TIMEBASE(%edx)
-	mov	12(%ebp),%eax
-	mov	%eax,_COMM_PAGE_TIMEBASE+4(%ecx)
-	mov	%eax,_COMM_PAGE_TIMEBASE+4(%edx)
-
-	mov	16(%ebp),%eax
-	mov	%eax,_COMM_PAGE_TIMESTAMP(%ecx)
-	mov	%eax,_COMM_PAGE_TIMESTAMP(%edx)
-	mov	20(%ebp),%eax
-	mov	%eax,_COMM_PAGE_TIMESTAMP+4(%ecx)
-	mov	%eax,_COMM_PAGE_TIMESTAMP+4(%edx)
-
-	movl	$1,_COMM_PAGE_TIMEENABLE(%ecx)
-	movl	$1,_COMM_PAGE_TIMEENABLE(%edx)
-1:
-	pop	%ebp
-	ret
+/*
+ * extern void	commpage_sched_gen_inc(void);
+ */
+	.text
+	.align  2, 0x90
+	.globl	_commpage_sched_gen_inc
 
-        .text
-        .align  2, 0x90
-	.globl	_commpage_set_nanotime
-/* extern void	commpage_set_nanotime(uint64_t tsc_base, uint64_t ns_base, uint32_t scale, uint32_t shift); */
-_commpage_set_nanotime:
+_commpage_sched_gen_inc:
 	push	%ebp
 	mov	%esp,%ebp
 
-	mov	_commPagePtr32,%ecx
-	testl	%ecx,%ecx
+	/* Increment 32-bit commpage field if present */
+	mov	_commPagePtr32,%edx
+	testl	%edx,%edx
 	je	1f
+	sub	$(_COMM_PAGE32_BASE_ADDRESS),%edx
+	lock
+	incl	_COMM_PAGE_SCHED_GEN(%edx)
 
-	sub	$(_COMM_PAGE_BASE_ADDRESS),%ecx
-	mov	_commPagePtr64,%edx			/* point to 64-bit commpage too */
-	mov	%edx,%eax
-	sub	$ _COMM_PAGE32_START_ADDRESS,%edx	/* because kernel is built 32-bit */
-	test	%eax,%eax
-	cmovz	%ecx,%edx				/* if no 64-bit commpage, point to 32 with both */
-
-	mov	8(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_TSC_BASE(%ecx)
-	mov	%eax,_COMM_PAGE_NT_TSC_BASE(%edx)
-	mov	12(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_TSC_BASE+4(%ecx)
-	mov	%eax,_COMM_PAGE_NT_TSC_BASE+4(%edx)
-
-	mov	24(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_SCALE(%ecx)
-	mov	%eax,_COMM_PAGE_NT_SCALE(%edx)
-
-	mov	28(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_SHIFT(%ecx)
-	mov	%eax,_COMM_PAGE_NT_SHIFT(%edx)
-
-	mov	16(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_NS_BASE(%ecx)
-	mov	%eax,_COMM_PAGE_NT_NS_BASE(%edx)
-	mov	20(%ebp),%eax
-	mov	%eax,_COMM_PAGE_NT_NS_BASE+4(%ecx)
-	mov	%eax,_COMM_PAGE_NT_NS_BASE+4(%edx)
+	/* Increment 64-bit commpage field if present */
+	mov	_commPagePtr64,%edx
+	testl	%edx,%edx
+	je	1f
+	sub	$(_COMM_PAGE32_START_ADDRESS),%edx
+	lock
+	incl	_COMM_PAGE_SCHED_GEN(%edx)
 1:
 	pop	%ebp
 	ret
@@ -121,6 +71,10 @@ _commpage_32_routines:
 	.long	CPN(compare_and_swap32_up)
 	.long	CPN(compare_and_swap64_mp)
 	.long	CPN(compare_and_swap64_up)
+	.long	CPN(AtomicEnqueue)
+	.long	CPN(AtomicDequeue)
+	.long	CPN(memory_barrier)
+	.long	CPN(memory_barrier_sse2)
 	.long	CPN(atomic_add32_mp)
 	.long	CPN(atomic_add32_up)
 	.long	CPN(mach_absolute_time)
@@ -140,14 +94,14 @@ _commpage_32_routines:
 	.long	CPN(bit_test_and_clear_mp)
 	.long	CPN(bit_test_and_clear_up)
 	.long	CPN(bzero_scalar)
-	.long	CPN(bzero_sse3)
+	.long	CPN(bzero_sse2)
 	.long	CPN(bcopy_scalar)
-	.long	CPN(bcopy_sse3)
-	.long	CPN(bcopy_sse4)
-	.long	CPN(old_nanotime)
-	.long	CPN(memset_pattern_sse3)
-	.long	CPN(longcopy_sse4)
+	.long	CPN(bcopy_sse2)
+	.long	CPN(bcopy_sse3x)
+	.long	CPN(memset_pattern_sse2)
+	.long	CPN(longcopy_sse3x)
 	.long	CPN(nanotime)
+	.long	CPN(nanotime_slow)
 	.long	0
 
 
@@ -161,6 +115,9 @@ _commpage_64_routines:
 	.long	CPN(compare_and_swap32_up_64)
 	.long	CPN(compare_and_swap64_mp_64)
 	.long	CPN(compare_and_swap64_up_64)
+	.long	CPN(AtomicEnqueue_64)
+	.long	CPN(AtomicDequeue_64)
+	.long	CPN(memory_barrier_sse2)	/* same routine as 32-bit version */
 	.long	CPN(atomic_add32_mp_64)
 	.long	CPN(atomic_add32_up_64)
 	.long	CPN(atomic_add64_mp_64)
@@ -180,11 +137,10 @@ _commpage_64_routines:
 	.long	CPN(bit_test_and_set_up_64)
 	.long	CPN(bit_test_and_clear_mp_64)
 	.long	CPN(bit_test_and_clear_up_64)
-	.long	CPN(bzero_sse3_64)
-	.long	CPN(bcopy_sse4_64)
-	.long	CPN(old_nanotime_64)
-	.long	CPN(memset_pattern_sse3_64)
-	.long	CPN(longcopy_sse4_64)
+	.long	CPN(bzero_sse2_64)
+	.long	CPN(bcopy_sse3x_64)
+	.long	CPN(memset_pattern_sse2_64)
+	.long	CPN(longcopy_sse3x_64)
 	.long	CPN(nanotime_64)
 	.long	0
 
diff --git a/osfmk/i386/commpage/commpage_gettimeofday.s b/osfmk/i386/commpage/commpage_gettimeofday.s
index 55bfd27ba..a50dd5410 100644
--- a/osfmk/i386/commpage/commpage_gettimeofday.s
+++ b/osfmk/i386/commpage/commpage_gettimeofday.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
@@ -34,29 +40,25 @@ Lgettimeofday:
 	push	%ebp
 	mov	%esp,%ebp
 	push	%esi
-	push	%edi
 	push	%ebx
 
 0:
-	cmp	$0,_COMM_PAGE_TIMEENABLE
-	je	4f
-	mov	_COMM_PAGE_TIMEBASE,%esi
-	mov	_COMM_PAGE_TIMEBASE+4,%edi
-	mov	_COMM_PAGE_TIMESTAMP,%ebx
+	movl	_COMM_PAGE_GTOD_GENERATION,%esi	/* get generation (0 if disabled) */
+	testl	%esi,%esi			/* disabled? */
+	jz	4f
 
 	mov	$ _COMM_PAGE_NANOTIME,%eax
-	call	*%eax		/* get ns in %edx:%eax */
+	call	*%eax				/* get ns in %edx:%eax */
 
-	cmp	_COMM_PAGE_TIMEBASE,%esi
-	jne	0b
-	cmp	_COMM_PAGE_TIMEBASE+4,%edi
-	jne	0b
-	cmp	$0,_COMM_PAGE_TIMEENABLE
-	je	4f
 	
+	sub	_COMM_PAGE_GTOD_NS_BASE,%eax
+	sbb	_COMM_PAGE_GTOD_NS_BASE+4,%edx
+	mov	_COMM_PAGE_GTOD_SEC_BASE,%ebx	/* load all the data before checking generation */
 	mov	$ NSEC_PER_SEC,%ecx
-	sub	%esi,%eax
-	sbb	%edi,%edx
+	
+	cmpl	_COMM_PAGE_GTOD_GENERATION,%esi	/* has time data changed out from under us? */
+	jne	0b
+	
 	div	%ecx
 	add	%eax,%ebx
 
@@ -72,7 +74,6 @@ Lgettimeofday:
 
 3:
 	pop	%ebx
-	pop	%edi
 	pop	%esi
 	pop	%ebp
 	ret
@@ -91,21 +92,21 @@ Lgettimeofday_64:			// %rdi = ptr to timeval
 	pushq	%rbp			// set up a frame for backtraces
 	movq	%rsp,%rbp
 	movq	%rdi,%r9		// save ptr to timeval
-	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_TIMEBASE),%r10
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_TIME_DATA_START),%r10
 0:
-	cmpl	$0,_TIMEENABLE(%r10)	// is data valid? (test _COMM_PAGE_TIMEENABLE)
-	jz	4f			// no
-	movq	_TIMEBASE(%r10),%r11	// get _COMM_PAGE_TIMEBASE
+	movl	_GTOD_GENERATION(%r10),%r11d	// get generation (0 if disabled)
+	testl	%r11d,%r11d		// disabled?
+	jz	4f
+	
 	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_NANOTIME),%rax
 	call	*%rax			// get %rax <- nanotime(), preserving %r9, %r10 and %r11
-	movl	_TIMESTAMP(%r10),%r8d	// get _COMM_PAGE_TIMESTAMP
-	cmpq	_TIMEBASE(%r10),%r11	// has _COMM_PAGE_TIMEBASE changed?
-	jne	0b			// loop until we have consistent data
-	cmpl	$0,_TIMEENABLE(%r10)	// is data valid? (test _COMM_PAGE_TIMEENABLE)
-	jz	4f			// no
+	
+	movl	_GTOD_SEC_BASE(%r10),%r8d	// get _COMM_PAGE_TIMESTAMP
+	subq	_GTOD_NS_BASE(%r10),%rax	// generate nanoseconds since timestamp
+	cmpl	_GTOD_GENERATION(%r10),%r11d	// has data changed out from under us?
+	jne	0b
 	
 	movl	$ NSEC_PER_SEC,%ecx
-	subq	%r11,%rax		// generate nanoseconds since timestamp
 	movq	%rax,%rdx
 	shrq	$32,%rdx		// get high half of delta in %edx
 	divl	%ecx			// %eax <- seconds since timestamp, %edx <- nanoseconds
diff --git a/osfmk/i386/commpage/commpage_mach_absolute_time.s b/osfmk/i386/commpage/commpage_mach_absolute_time.s
index 71592ec0e..3ea04c5f1 100644
--- a/osfmk/i386/commpage/commpage_mach_absolute_time.s
+++ b/osfmk/i386/commpage/commpage_mach_absolute_time.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
@@ -36,54 +42,107 @@ Lmach_absolute_time:
 
 	COMMPAGE_DESCRIPTOR(mach_absolute_time,_COMM_PAGE_ABSOLUTE_TIME,0,0)
 
-/* Nanotime is being moved out of the way of bcopy in the commpage.
- * First we put it in both places, old and new.  Then, when all the build
- * trains have rebuilt libSystem, we can remove the deprecated instance.
- */
  
 /* return nanotime in %edx:%eax */
-Lold_nanotime:
+
 Lnanotime:
-	push	%ebp
-	mov	%esp,%ebp
-	push	%esi
-	push	%edi
-	push	%ebx
+	pushl	%ebp
+	movl	%esp,%ebp
+	pushl	%esi
+	pushl	%ebx
 
 0:
-	mov	_COMM_PAGE_NT_TSC_BASE,%esi
-	mov	_COMM_PAGE_NT_TSC_BASE+4,%edi
+	movl	_COMM_PAGE_NT_GENERATION,%esi	/* get generation (0 if being changed) */
+	testl	%esi,%esi			/* if being updated, loop until stable */
+	jz	0b
 
-	rdtsc
-	sub	%esi,%eax
-	sbb	%edi,%edx
+	rdtsc					/* get TSC in %edx:%eax */
+	subl	_COMM_PAGE_NT_TSC_BASE,%eax
+	sbbl	_COMM_PAGE_NT_TSC_BASE+4,%edx
 
-	mov	_COMM_PAGE_NT_SCALE,%ecx
+	movl	_COMM_PAGE_NT_SCALE,%ecx
 
-	mov	%edx,%ebx
+	movl	%edx,%ebx
 	mull	%ecx
-	mov	%ebx,%eax
-	mov	%edx,%ebx
+	movl	%ebx,%eax
+	movl	%edx,%ebx
 	mull	%ecx
-	add	%ebx,%eax
-	adc	$0,%edx
+	addl	%ebx,%eax
+	adcl	$0,%edx
+
+	addl	_COMM_PAGE_NT_NS_BASE,%eax
+	adcl	_COMM_PAGE_NT_NS_BASE+4,%edx
+
+	cmpl	_COMM_PAGE_NT_GENERATION,%esi	/* have the parameters changed? */
+	jne	0b				/* yes, loop until stable */
+
+	popl	%ebx
+	popl	%esi
+	popl	%ebp
+	ret
+
+	COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,0,kSlow)
+
+
+/* nanotime routine for machines slower than ~1Gz (SLOW_TSC_THRESHOLD) */
+Lnanotime_slow:
+	push	%ebp
+	mov	%esp,%ebp
+	push	%esi
+	push	%edi
+	push	%ebx
+
+0:
+	movl	_COMM_PAGE_NT_GENERATION,%esi
+	testl	%esi,%esi			/* if generation is 0, data being changed */
+	jz	0b				/* so loop until stable */
+
+	rdtsc					/* get TSC in %edx:%eax */
+	subl	_COMM_PAGE_NT_TSC_BASE,%eax
+	sbbl	_COMM_PAGE_NT_TSC_BASE+4,%edx
+
+	pushl	%esi				/* save generation */
+	/*
+	 * Do the math to convert tsc ticks to nanoseconds.  We first
+	 * do long multiply of 1 billion times the tsc.  Then we do
+	 * long division by the tsc frequency
+	 */
+	mov	$1000000000, %ecx		/* number of nanoseconds in a second */
+	mov	%edx, %ebx
+	mul	%ecx
+	mov	%edx, %edi
+	mov	%eax, %esi
+	mov	%ebx, %eax
+	mul	%ecx
+	add	%edi, %eax
+	adc	$0, %edx			/* result in edx:eax:esi */
+	mov	%eax, %edi
+	mov	_COMM_PAGE_NT_SHIFT,%ecx	/* overloaded as the low 32 tscFreq */
+	xor	%eax, %eax
+	xchg	%edx, %eax
+	div	%ecx
+	xor	%eax, %eax
+	mov	%edi, %eax
+	div	%ecx
+	mov	%eax, %ebx
+	mov	%esi, %eax
+	div	%ecx
+	mov	%ebx, %edx			/* result in edx:eax */
+	popl	%esi				/* recover generation */
 
 	add	_COMM_PAGE_NT_NS_BASE,%eax
 	adc	_COMM_PAGE_NT_NS_BASE+4,%edx
 
-	cmp	_COMM_PAGE_NT_TSC_BASE,%esi
-	jne	0b
-	cmp	_COMM_PAGE_NT_TSC_BASE+4,%edi
-	jne	0b
+	cmpl	_COMM_PAGE_NT_GENERATION,%esi	/* have the parameters changed? */
+	jne	0b				/* yes, loop until stable */
 
 	pop	%ebx
 	pop	%edi
 	pop	%esi
 	pop	%ebp
-	ret
+	ret					/* result in edx:eax */
 
-	COMMPAGE_DESCRIPTOR(nanotime,_COMM_PAGE_NANOTIME,0,0)
-	COMMPAGE_DESCRIPTOR(old_nanotime,_COMM_PAGE_OLD_NANOTIME,0,0)
+	COMMPAGE_DESCRIPTOR(nanotime_slow,_COMM_PAGE_NANOTIME,kSlow,0)
 
 
 /* The 64-bit version.  We return the 64-bit nanotime in %rax,
@@ -92,26 +151,26 @@ Lnanotime:
 	.text
 	.align	2
 	.code64
-Lold_nanotime_64:
-Lnanotime_64:				// NB: must preserve r9, r10, and r11
-	pushq	%rbp			// set up a frame for backtraces
+Lnanotime_64:					// NB: must preserve r9, r10, and r11
+	pushq	%rbp				// set up a frame for backtraces
 	movq	%rsp,%rbp
-	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_NT_TSC_BASE),%rsi
+	movq	$_COMM_PAGE_32_TO_64(_COMM_PAGE_TIME_DATA_START),%rsi
 1:
-	movq	_NT_TSC_BASE(%rsi),%r8	// r8  := base_tsc
-	rdtsc				// edx:eax := tsc
-	shlq	$32,%rdx		// rax := ((edx << 32) | eax), ie 64-bit tsc
+	movl	_NT_GENERATION(%rsi),%r8d	// get generation
+	testl	%r8d,%r8d			// if 0, data is being changed...
+	jz	1b				// ...so loop until stable
+	rdtsc					// edx:eax := tsc
+	shlq	$32,%rdx			// rax := ((edx << 32) | eax), ie 64-bit tsc
 	orq	%rdx,%rax
-	subq	%r8, %rax		// rax := (tsc - base_tsc)
+	subq	_NT_TSC_BASE(%rsi), %rax	// rax := (tsc - base_tsc)
 	movl	_NT_SCALE(%rsi),%ecx
-	mulq	%rcx			// rdx:rax := (tsc - base_tsc) * scale
-	shrdq	$32,%rdx,%rax		// _COMM_PAGE_NT_SHIFT is always 32
-	addq	_NT_NS_BASE(%rsi),%rax	// (((tsc - base_tsc) * scale) >> 32) + ns_base
+	mulq	%rcx				// rdx:rax := (tsc - base_tsc) * scale
+	shrdq	$32,%rdx,%rax			// _COMM_PAGE_NT_SHIFT is always 32
+	addq	_NT_NS_BASE(%rsi),%rax		// (((tsc - base_tsc) * scale) >> 32) + ns_base
 	
-	cmpq	_NT_TSC_BASE(%rsi),%r8	// did the data change during computation?
+	cmpl	_NT_GENERATION(%rsi),%r8d	// did the data change during computation?
 	jne	1b
 	popq	%rbp
 	ret
 
-	COMMPAGE_DESCRIPTOR(nanotime_64,_COMM_PAGE_NANOTIME,0,0)
-	COMMPAGE_DESCRIPTOR(old_nanotime_64,_COMM_PAGE_OLD_NANOTIME,0,0)
+	COMMPAGE_DESCRIPTOR(nanotime_64,_COMM_PAGE_NANOTIME,0,kSlow)
diff --git a/osfmk/i386/commpage/commpage_sigs.c b/osfmk/i386/commpage/commpage_sigs.c
index 8b7e7d404..0c100a276 100644
--- a/osfmk/i386/commpage/commpage_sigs.c
+++ b/osfmk/i386/commpage/commpage_sigs.c
@@ -17,120 +17,120 @@ static unsigned int sigdata[] =
 	0x2000804e, 0xf8ff0039, 0x00006038, 0xad09887c, 0x2000804e, 0x2000804e, 
 	0x06004018, 0x00000000, 0x6d6f635f, 0x65726170, 0x646e615f, 0x6177735f, 
 	0x62343670, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6d656d5f, 
-	0x5f746573, 0x74746170, 0x006e7265, 0x06004018, 0x10000071, 0x000009c8, 
-	0x080029c8, 0x14008241, 0x000008d8, 0x080028d8, 0xf0ff8438, 0x10000839, 
-	0xbfd18054, 0x0000807f, 0xbe068454, 0x70008241, 0xec47007c, 0xa603097c, 
-	0x2000c038, 0x3400be41, 0x000008d8, 0x080028d8, 0x100008d8, 0x180028d8, 
-	0x200008d8, 0x280028d8, 0x300008d8, 0x380028d8, 0x40000839, 0xdcff0042, 
-	0x34000048, 0xec47007c, 0xec47067c, 0x000008d8, 0x080028d8, 0x100008d8, 
-	0x180028d8, 0x200008d8, 0x280028d8, 0x300008d8, 0x380028d8, 0x40000839, 
-	0xd4ff0042, 0x3fe18054, 0x3e078454, 0x2000824d, 0xa603097c, 0x000008d8, 
-	0x080028d8, 0x10000839, 0xf4ff0042, 0x2000804e, 0x2000804e, 0x06004018, 
-	0x00000000, 0x6d656d5f, 0x5f746573, 0x74746170, 0x006e7265, 0x06004018, 
-	0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 0x75716e65, 0x00657565, 
-	0x06004018, 0x2818c07c, 0x2e29c47c, 0x00000060, 0x2d19807c, 0x2000e24d, 
-	0xecffff4b, 0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 
-	0x75716e65, 0x00657565, 0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 
-	0x5f63696d, 0x75716564, 0x00657565, 0x06004018, 0x781b657c, 0x2828607c, 
-	0x0000032c, 0x2000824d, 0x2e20c37c, 0x2d29c07c, 0xecffc240, 0x00000060, 
-	0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 0x75716564, 
-	0x00657565, 0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 
-	0x72726162, 0x00726569, 0x06004018, 0x2000804e, 0x06004018, 0x00000000, 
-	0x6f74615f, 0x5f63696d, 0x72726162, 0x00726569, 0x06004018, 0x06004018, 
-	0x01000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 0x00003233, 0x06004018, 
-	0x2820a07c, 0x142ac37c, 0x2d21c07c, 0xf4ffc240, 0x7833c37c, 0x2000804e, 
-	0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 0x00003233, 
-	0x06004018, 0x06004018, 0x01000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 
-	0x745f6574, 0x00656d69, 0x06004018, 0x00004018, 0x04006000, 0x00000000, 
-	0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 0x5f656d69, 
-	0x68676968, 0x00000000, 0x00004018, 0x00004018, 0x04008000, 0x00000000, 
-	0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 0x5f656d69, 
-	0x00776f6c, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x63616d5f, 
-	0x62615f68, 0x756c6f73, 0x745f6574, 0x00656d69, 0x06004018, 0x06004018, 
-	0x01000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x00007972, 0x06004018, 
-	0x05004018, 0x04006000, 0x05004018, 0x00004018, 0x04006000, 0x00000000, 
-	0x00000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x775f7972, 0x70706172, 
-	0x00007265, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 
-	0x6f6c5f6e, 0x745f6b63, 0x00007972, 0x06004018, 0x06004018, 0x01000000, 
-	0x6970735f, 0x6f6c5f6e, 0x00006b63, 0x06004018, 0x05004018, 0x04006000, 
-	0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x6970735f, 
-	0x6f6c5f6e, 0x775f6b63, 0x70706172, 0x00007265, 0x00004018, 0x2000804e, 
-	0x06004018, 0x00000000, 0x6970735f, 0x6f6c5f6e, 0x00006b63, 0x06004018, 
-	0x06004018, 0x01000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 0x00000000, 
-	0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 0x00000000, 
-	0x00000000, 0x00000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 0x00000000, 
-	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 0x6e755f6e, 
-	0x6b636f6c, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6874705f, 
-	0x64616572, 0x7465675f, 0x63657073, 0x63696669, 0x00000000, 0x06004018, 
-	0x02004018, 0xc082ffff, 0x02004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x5f746573, 0x74746170, 0x006e7265, 0x06004018, 0xc0008428, 0xa642407c, 
+	0x00804064, 0x10000571, 0xa643007c, 0x1000a038, 0xce48007c, 0x2000c038, 
+	0x7c008441, 0x30002039, 0x10008241, 0xce41007c, 0x10000839, 0xf0ff8438, 
+	0xbed18054, 0xec45007c, 0xbe068454, 0xffff0030, 0x40004039, 0x60006039, 
+	0xa603097c, 0xec45067c, 0x10000048, 0x00000060, 0x00000060, 0x00000060, 
+	0xec450a7c, 0xec450b7c, 0xce41007c, 0xce41057c, 0xce41067c, 0xce41097c, 
+	0x40000839, 0xe4ff0042, 0xce41007c, 0xce41057c, 0xce41067c, 0xce41097c, 
+	0x40000839, 0x7fd98054, 0xfe068454, 0x18008241, 0xa603097c, 0xce41007c, 
+	0xce41057c, 0x20000839, 0xf4ff0042, 0xa643407c, 0x2000804e, 0x2000804e, 
+	0x06004018, 0x00000000, 0x6d656d5f, 0x5f746573, 0x74746170, 0x006e7265, 
+	0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 0x75716e65, 
+	0x00657565, 0x06004018, 0x2818c07c, 0x2e29c47c, 0x00000060, 0x2d19807c, 
+	0x2000e24d, 0xecffff4b, 0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 
+	0x5f63696d, 0x75716e65, 0x00657565, 0x06004018, 0x06004018, 0x01000000, 
+	0x6f74615f, 0x5f63696d, 0x75716564, 0x00657565, 0x06004018, 0x781b657c, 
+	0x2828607c, 0x0000032c, 0x2000824d, 0x2e20c37c, 0x2d29c07c, 0xecffc240, 
+	0x00000060, 0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 
+	0x75716564, 0x00657565, 0x06004018, 0x06004018, 0x01000000, 0x6f74615f, 
+	0x5f63696d, 0x72726162, 0x00726569, 0x06004018, 0x2000804e, 0x06004018, 
+	0x00000000, 0x6f74615f, 0x5f63696d, 0x72726162, 0x00726569, 0x06004018, 
+	0x06004018, 0x01000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 0x00003233, 
+	0x06004018, 0x2820a07c, 0x142ac37c, 0x2d21c07c, 0xf4ffc240, 0x7833c37c, 
+	0x2000804e, 0x06004018, 0x00000000, 0x6f74615f, 0x5f63696d, 0x5f646461, 
+	0x00003233, 0x06004018, 0x06004018, 0x01000000, 0x63616d5f, 0x62615f68, 
+	0x756c6f73, 0x745f6574, 0x00656d69, 0x06004018, 0x00004018, 0x04006000, 
+	0x00000000, 0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 
+	0x5f656d69, 0x68676968, 0x00000000, 0x00004018, 0x00004018, 0x04008000, 
+	0x00000000, 0x00000000, 0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 
+	0x5f656d69, 0x00776f6c, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x63616d5f, 0x62615f68, 0x756c6f73, 0x745f6574, 0x00656d69, 0x06004018, 
+	0x06004018, 0x01000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x00007972, 
+	0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 0x04006000, 
+	0x00000000, 0x00000000, 0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x775f7972, 
+	0x70706172, 0x00007265, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6970735f, 0x6f6c5f6e, 0x745f6b63, 0x00007972, 0x06004018, 0x06004018, 
+	0x01000000, 0x6970735f, 0x6f6c5f6e, 0x00006b63, 0x06004018, 0x05004018, 
+	0x04006000, 0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 
+	0x6970735f, 0x6f6c5f6e, 0x775f6b63, 0x70706172, 0x00007265, 0x00004018, 
+	0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 0x6f6c5f6e, 0x00006b63, 
+	0x06004018, 0x06004018, 0x01000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 
+	0x00000000, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 
+	0x00000000, 0x00000000, 0x00000000, 0x6970735f, 0x6e755f6e, 0x6b636f6c, 
+	0x00000000, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6970735f, 
+	0x6e755f6e, 0x6b636f6c, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 
 	0x6874705f, 0x64616572, 0x7465675f, 0x63657073, 0x63696669, 0x00000000, 
-	0x06004018, 0x06004018, 0x01000000, 0x7465675f, 0x656d6974, 0x6164666f, 
-	0x00000079, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x00004018, 
-	0x04006000, 0x00000000, 0x00000000, 0x7465675f, 0x656d6974, 0x6164666f, 
-	0x72775f79, 0x65707061, 0x00000072, 0x00004018, 0x2000804e, 0x06004018, 
-	0x00000000, 0x7465675f, 0x656d6974, 0x6164666f, 0x00000079, 0x06004018, 
-	0x06004018, 0x01000000, 0x7379735f, 0x6163645f, 0x5f656863, 0x73756c66, 
-	0x00000068, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x05004018, 
-	0x04008000, 0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 
-	0x7379735f, 0x6163645f, 0x5f656863, 0x73756c66, 0x00000068, 0x00004018, 
-	0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 0x6163645f, 0x5f656863, 
-	0x73756c66, 0x00000068, 0x06004018, 0x06004018, 0x01000000, 0x7379735f, 
+	0x06004018, 0x02004018, 0xc082ffff, 0x02004018, 0x2000804e, 0x06004018, 
+	0x00000000, 0x6874705f, 0x64616572, 0x7465675f, 0x63657073, 0x63696669, 
+	0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x7465675f, 0x656d6974, 
+	0x6164666f, 0x00000079, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 
+	0x00004018, 0x04006000, 0x00000000, 0x00000000, 0x7465675f, 0x656d6974, 
+	0x6164666f, 0x72775f79, 0x65707061, 0x00000072, 0x00004018, 0x2000804e, 
+	0x06004018, 0x00000000, 0x7465675f, 0x656d6974, 0x6164666f, 0x00000079, 
+	0x06004018, 0x06004018, 0x01000000, 0x7379735f, 0x6163645f, 0x5f656863, 
+	0x73756c66, 0x00000068, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 
+	0x05004018, 0x04008000, 0x05004018, 0x00004018, 0x00000000, 0x00000000, 
+	0x00000000, 0x7379735f, 0x6163645f, 0x5f656863, 0x73756c66, 0x00000068, 
+	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 0x6163645f, 
+	0x5f656863, 0x73756c66, 0x00000068, 0x06004018, 0x06004018, 0x01000000, 
+	0x7379735f, 0x6163695f, 0x5f656863, 0x61766e69, 0x6164696c, 0x00006574, 
+	0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x05004018, 0x04008000, 
+	0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x7379735f, 
+	0x6163695f, 0x5f656863, 0x61766e69, 0x6164696c, 0x775f6574, 0x70706172, 
+	0x00007265, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 
 	0x6163695f, 0x5f656863, 0x61766e69, 0x6164696c, 0x00006574, 0x06004018, 
-	0x05004018, 0x04006000, 0x05004018, 0x05004018, 0x04008000, 0x05004018, 
-	0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x7379735f, 0x6163695f, 
-	0x5f656863, 0x61766e69, 0x6164696c, 0x775f6574, 0x70706172, 0x00007265, 
-	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x7379735f, 0x6163695f, 
-	0x5f656863, 0x61766e69, 0x6164696c, 0x00006574, 0x06004018, 0x06004018, 
-	0x01000000, 0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 0x06004018, 
-	0x02004018, 0x8085ffff, 0x02004018, 0x2000804e, 0x06004018, 0x00000000, 
-	0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 0x06004018, 0x06004018, 
-	0x01000000, 0x657a625f, 0x00006f72, 0x06004018, 0x05004018, 0x04006000, 
-	0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x00004018, 0x00000000, 
-	0x00000000, 0x00000000, 0x657a625f, 0x00006f72, 0x00004018, 0x2000804e, 
-	0x06004018, 0x00000000, 0x657a625f, 0x00006f72, 0x06004018, 0x06004018, 
-	0x01000000, 0x6f63625f, 0x00007970, 0x06004018, 0x05004018, 0x04006000, 
-	0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x05004018, 0x0400a000, 
-	0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x6f63625f, 
-	0x00007970, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6f63625f, 
-	0x00007970, 0x06004018, 0x06004018, 0x01000000, 0x6d656d5f, 0x65766f6d, 
-	0x00000000, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 0x05004018, 
-	0x04008000, 0x05004018, 0x05004018, 0x0400a000, 0x05004018, 0x00004018, 
-	0x00000000, 0x00000000, 0x00000000, 0x6d656d5f, 0x65766f6d, 0x00000000, 
-	0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6d656d5f, 0x65766f6d, 
-	0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6e616e5f, 0x6d69746f, 
-	0x00000065, 0x06004018, 0x00004018, 0x04006000, 0x00000000, 0x00000000, 
-	0x6e616e5f, 0x6d69746f, 0x69685f65, 0x00006867, 0x00004018, 0x00004018, 
-	0x04008000, 0x00000000, 0x00000000, 0x6e616e5f, 0x6d69746f, 0x6f6c5f65, 
-	0x00000077, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6e616e5f, 
-	0x6d69746f, 0x00000065, 0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 
-	0x736d5f63, 0x6e655367, 0x00000064, 0x06004018, 0x00004018, 0x00000000, 
-	0x00000400, 0x00000000, 0x6a626f5f, 0x736d5f63, 0x6e655367, 0x00000064, 
-	0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x736d5f63, 0x6e655367, 
-	0x00000064, 0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 0x73615f63, 
-	0x6e676973, 0x6176695f, 0x00000072, 0x06004018, 0x00004018, 0x00000000, 
-	0x00000400, 0x00000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6176695f, 
-	0x65675f72, 0x6972656e, 0x00000063, 0x00004018, 0x06004018, 0x00000000, 
-	0x6a626f5f, 0x73615f63, 0x6e676973, 0x6176695f, 0x00000072, 0x06004018, 
-	0x06004018, 0x01000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6f6c675f, 
-	0x006c6162, 0x06004018, 0x00004018, 0x00000000, 0x00000400, 0x00000000, 
-	0x6a626f5f, 0x73615f63, 0x6e676973, 0x6f6c675f, 0x5f6c6162, 0x656e6567, 
-	0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x73615f63, 
-	0x6e676973, 0x6f6c675f, 0x006c6162, 0x06004018, 0x06004018, 0x01000000, 
-	0x6a626f5f, 0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x00747361, 
-	0x06004018, 0x00004018, 0x00000000, 0x00000400, 0x00000000, 0x6a626f5f, 
-	0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x5f747361, 0x656e6567, 
-	0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x73615f63, 
-	0x6e676973, 0x7274735f, 0x43676e6f, 0x00747361, 0x06004018, 
+	0x06004018, 0x01000000, 0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 
+	0x06004018, 0x02004018, 0x8085ffff, 0x02004018, 0x2000804e, 0x06004018, 
+	0x00000000, 0x6874705f, 0x64616572, 0x6c65735f, 0x00000066, 0x06004018, 
+	0x06004018, 0x01000000, 0x657a625f, 0x00006f72, 0x06004018, 0x05004018, 
+	0x04006000, 0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x00004018, 
+	0x00000000, 0x00000000, 0x00000000, 0x657a625f, 0x00006f72, 0x00004018, 
+	0x2000804e, 0x06004018, 0x00000000, 0x657a625f, 0x00006f72, 0x06004018, 
+	0x06004018, 0x01000000, 0x6f63625f, 0x00007970, 0x06004018, 0x05004018, 
+	0x04006000, 0x05004018, 0x05004018, 0x04008000, 0x05004018, 0x05004018, 
+	0x0400a000, 0x05004018, 0x00004018, 0x00000000, 0x00000000, 0x00000000, 
+	0x6f63625f, 0x00007970, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6f63625f, 0x00007970, 0x06004018, 0x06004018, 0x01000000, 0x6d656d5f, 
+	0x65766f6d, 0x00000000, 0x06004018, 0x05004018, 0x04006000, 0x05004018, 
+	0x05004018, 0x04008000, 0x05004018, 0x05004018, 0x0400a000, 0x05004018, 
+	0x00004018, 0x00000000, 0x00000000, 0x00000000, 0x6d656d5f, 0x65766f6d, 
+	0x00000000, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 0x6d656d5f, 
+	0x65766f6d, 0x00000000, 0x06004018, 0x06004018, 0x01000000, 0x6e616e5f, 
+	0x6d69746f, 0x00000065, 0x06004018, 0x00004018, 0x04006000, 0x00000000, 
+	0x00000000, 0x6e616e5f, 0x6d69746f, 0x69685f65, 0x00006867, 0x00004018, 
+	0x00004018, 0x04008000, 0x00000000, 0x00000000, 0x6e616e5f, 0x6d69746f, 
+	0x6f6c5f65, 0x00000077, 0x00004018, 0x2000804e, 0x06004018, 0x00000000, 
+	0x6e616e5f, 0x6d69746f, 0x00000065, 0x06004018, 0x06004018, 0x01000000, 
+	0x6a626f5f, 0x736d5f63, 0x6e655367, 0x00000064, 0x06004018, 0x00004018, 
+	0x00000000, 0x00000400, 0x00000000, 0x6a626f5f, 0x736d5f63, 0x6e655367, 
+	0x00000064, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 0x736d5f63, 
+	0x6e655367, 0x00000064, 0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 
+	0x73615f63, 0x6e676973, 0x6176695f, 0x00000072, 0x06004018, 0x00004018, 
+	0x00000000, 0x00000400, 0x00000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 
+	0x6176695f, 0x65675f72, 0x6972656e, 0x00000063, 0x00004018, 0x06004018, 
+	0x00000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6176695f, 0x00000072, 
+	0x06004018, 0x06004018, 0x01000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 
+	0x6f6c675f, 0x006c6162, 0x06004018, 0x00004018, 0x00000000, 0x00000400, 
+	0x00000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x6f6c675f, 0x5f6c6162, 
+	0x656e6567, 0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 
+	0x73615f63, 0x6e676973, 0x6f6c675f, 0x006c6162, 0x06004018, 0x06004018, 
+	0x01000000, 0x6a626f5f, 0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 
+	0x00747361, 0x06004018, 0x00004018, 0x00000000, 0x00000400, 0x00000000, 
+	0x6a626f5f, 0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x5f747361, 
+	0x656e6567, 0x00636972, 0x00004018, 0x06004018, 0x00000000, 0x6a626f5f, 
+	0x73615f63, 0x6e676973, 0x7274735f, 0x43676e6f, 0x00747361, 0x06004018, 
 };
 commpage_descriptor sigdata_descriptor =
 { sigdata, sizeof(sigdata), 0xffff3000, 0, 0 };
 
 static unsigned int badata[] =
 {
-	0xaa3aff4b, 0x3a3aff4b, 0xca39ff4b, 0x6e39ff4b, 0x0230ff4b, 0x5a32ff4b, 
-	0xae32ff4b, 0x0a33ff4b, 0x4633ff4b, 0x9633ff4b, 0x3634ff4b, 0xaa34ff4b, 
-	0x1235ff4b, 0x7e35ff4b, 0xd635ff4b, 0x4a36ff4b, 0xce36ff4b, 0x6637ff4b, 
-	0xae37ff4b, 0x0e38ff4b, 0x7a38ff4b, 0x6630ff4b, 0xde30ff4b, 0x5e31ff4b, 
+	0xae3aff4b, 0x3e3aff4b, 0xce39ff4b, 0x7239ff4b, 0x0230ff4b, 0x5e32ff4b, 
+	0xb232ff4b, 0x0e33ff4b, 0x4a33ff4b, 0x9a33ff4b, 0x3a34ff4b, 0xae34ff4b, 
+	0x1635ff4b, 0x8235ff4b, 0xda35ff4b, 0x4e36ff4b, 0xd236ff4b, 0x6a37ff4b, 
+	0xb237ff4b, 0x1238ff4b, 0x7e38ff4b, 0x6630ff4b, 0xde30ff4b, 0x5e31ff4b, 
 };
 static commpage_descriptor badata_descriptor_ary[] =
 {
diff --git a/osfmk/i386/commpage/longcopy_sse4.s b/osfmk/i386/commpage/longcopy_sse3x.s
similarity index 84%
rename from osfmk/i386/commpage/longcopy_sse4.s
rename to osfmk/i386/commpage/longcopy_sse3x.s
index 86a2d2e39..973eef434 100644
--- a/osfmk/i386/commpage/longcopy_sse4.s
+++ b/osfmk/i386/commpage/longcopy_sse3x.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
          
 #include <machine/cpu_capabilities.h>
@@ -26,7 +32,7 @@
 
 /*
  * The bcopy/memcpy loops for very long operands, tuned for Pentium-M
- * class processors with SSE4 and 64-byte cache lines.
+ * class processors with Supplemental SSE3 and 64-byte cache lines.
  *
  * The following #defines are tightly coupled to the u-architecture:
  */
@@ -49,7 +55,7 @@
  
         .text
         .align	5, 0x90
-Llongcopy_sse4:				// void longcopy(const void *dest, void *sou, size_t len)
+Llongcopy_sse3x:			// void longcopy(const void *dest, void *sou, size_t len)
 	pushl	%ebp			// set up a frame for backtraces
 	movl	%esp,%ebp
         pushl   %esi
@@ -212,4 +218,4 @@ LVeryLongChunkEnd:
 	ret
 
 	/* always match for now, as commpage_stuff_routine() will panic if no match */
-	COMMPAGE_DESCRIPTOR(longcopy_sse4, _COMM_PAGE_LONGCOPY, 0 ,0)
+	COMMPAGE_DESCRIPTOR(longcopy_sse3x, _COMM_PAGE_LONGCOPY, 0 ,0)
diff --git a/osfmk/i386/commpage/longcopy_sse4_64.s b/osfmk/i386/commpage/longcopy_sse3x_64.s
similarity index 83%
rename from osfmk/i386/commpage/longcopy_sse4_64.s
rename to osfmk/i386/commpage/longcopy_sse3x_64.s
index a55c6551e..b04592701 100644
--- a/osfmk/i386/commpage/longcopy_sse4_64.s
+++ b/osfmk/i386/commpage/longcopy_sse3x_64.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
         
 #include <machine/cpu_capabilities.h>
@@ -26,7 +32,7 @@
 
 /*
  * The bcopy/memcpy loops for very long operands, tuned for 64-bit
- * Pentium-M class processors with SSE4 and 64-byte cache lines.
+ * Pentium-M class processors with Supplemental SSE3 and 64-byte cache lines.
  * This is the 64-bit version.
  *
  * The following #defines are tightly coupled to the u-architecture:
@@ -52,7 +58,7 @@
         .text
 	.code64
         .align 5, 0x90
-Llongcopy_sse4_64:			// void longcopy(const void *dest, void *sou, size_t len)
+Llongcopy_sse3x_64:			// void longcopy(const void *dest, void *sou, size_t len)
 	pushq	%rbp			// set up a frame for backtraces
 	movq	%rsp,%rbp
 	movl	%edi,%eax		// copy dest ptr
@@ -202,4 +208,4 @@ LVeryLongChunkEnd:
 	ret
 
 	/* always match for now, as commpage_stuff_routine() will panic if no match */
-	COMMPAGE_DESCRIPTOR(longcopy_sse4_64, _COMM_PAGE_LONGCOPY, 0 ,0)
+	COMMPAGE_DESCRIPTOR(longcopy_sse3x_64, _COMM_PAGE_LONGCOPY, 0 ,0)
diff --git a/osfmk/i386/commpage/memset_pattern_sse3.s b/osfmk/i386/commpage/memset_pattern_sse2.s
similarity index 79%
rename from osfmk/i386/commpage/memset_pattern_sse3.s
rename to osfmk/i386/commpage/memset_pattern_sse2.s
index 58fbe9fc6..c3f4ecdd9 100644
--- a/osfmk/i386/commpage/memset_pattern_sse3.s
+++ b/osfmk/i386/commpage/memset_pattern_sse2.s
@@ -1,30 +1,36 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /* The common path for nonzero memset and the memset_pattern routines,
- * tuned for Pentium-M class processors with SSE3 and 64-byte cache lines.
+ * tuned for Pentium-M class processors with SSE2 and 64-byte cache lines.
  * This is used by the following functions:
  *
  *	void *memset(void *b, int c, size_t len);                   // when c!=0
@@ -51,7 +57,7 @@
 
         .text
         .align  5, 0x90
-Lmemset_pattern_sse3:
+Lmemset_pattern_sse2:
         cmpl    $(kShort),%edx		// long enough to bother aligning?
         ja	LNotShort		// yes
 	jmp	LShort			// no
@@ -176,4 +182,4 @@ LNoMoreChunks:
 	jge	LLoopBy16		// yes
 	jmp	LLessThan16		// handle up to 15 remaining bytes
 
-	COMMPAGE_DESCRIPTOR(memset_pattern_sse3,_COMM_PAGE_MEMSET_PATTERN,kHasSSE2,0)
+	COMMPAGE_DESCRIPTOR(memset_pattern_sse2,_COMM_PAGE_MEMSET_PATTERN,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/memset_pattern_sse3_64.s b/osfmk/i386/commpage/memset_pattern_sse2_64.s
similarity index 80%
rename from osfmk/i386/commpage/memset_pattern_sse3_64.s
rename to osfmk/i386/commpage/memset_pattern_sse2_64.s
index ab2f42fc9..b3d64a692 100644
--- a/osfmk/i386/commpage/memset_pattern_sse3_64.s
+++ b/osfmk/i386/commpage/memset_pattern_sse2_64.s
@@ -1,30 +1,36 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <machine/cpu_capabilities.h>
 #include <machine/commpage.h>
 
 /* The common path for nonzero memset and the memset_pattern routines,
- * tuned for Pentium-M class processors with SSE3 and 64-byte cache lines.
+ * tuned for Pentium-M class processors with SSE2 and 64-byte cache lines.
  * This is the 64-bit bersion.  It is used by the following functions:
  *
  *	void *memset(void *b, int c, size_t len);                   // when c!=0
@@ -53,7 +59,7 @@
         .text
         .align  5, 0x90
 	.code64
-Lmemset_pattern_sse3_64:
+Lmemset_pattern_sse2_64:
         cmpq    $(kShort),%rdx		// long enough to bother aligning?
         ja	LNotShort		// yes
 	jmp	LShort			// no
@@ -178,4 +184,4 @@ LNoMoreChunks:
 	jge	LLoopBy16		// yes
 	jmp	LLessThan16		// handle up to 15 remaining bytes
 
-	COMMPAGE_DESCRIPTOR(memset_pattern_sse3_64,_COMM_PAGE_MEMSET_PATTERN,kHasSSE3,0)
+	COMMPAGE_DESCRIPTOR(memset_pattern_sse2_64,_COMM_PAGE_MEMSET_PATTERN,kHasSSE2,0)
diff --git a/osfmk/i386/commpage/pthreads.s b/osfmk/i386/commpage/pthreads.s
index 4aa2675b4..484857e13 100644
--- a/osfmk/i386/commpage/pthreads.s
+++ b/osfmk/i386/commpage/pthreads.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/i386/commpage/spinlocks.s b/osfmk/i386/commpage/spinlocks.s
index f8d6329fa..e582635a0 100644
--- a/osfmk/i386/commpage/spinlocks.s
+++ b/osfmk/i386/commpage/spinlocks.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/i386/cpu.c b/osfmk/i386/cpu.c
index 588377b1f..b0f87d7da 100644
--- a/osfmk/i386/cpu.c
+++ b/osfmk/i386/cpu.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File:	i386/cpu.c
@@ -29,7 +35,6 @@
 #include <kern/misc_protos.h>
 #include <kern/machine.h>
 #include <mach/processor_info.h>
-#include <i386/mp.h>
 #include <i386/machine_cpu.h>
 #include <i386/machine_routines.h>
 #include <i386/pmap.h>
@@ -48,7 +53,7 @@ cpu_control(
 	processor_info_t	info,
 	unsigned int		count)
 {
-	printf("cpu_control(%d,0x%x,%d) not implemented\n",
+	printf("cpu_control(%d,%p,%d) not implemented\n",
 		slot_num, info, count);
 	return (KERN_FAILURE);
 }
@@ -71,7 +76,7 @@ cpu_info(
 	processor_info_t	info,
 	unsigned int		*count)
 {
-	printf("cpu_info(%d,%d,0x%x,0x%x) not implemented\n",
+	printf("cpu_info(%d,%d,%p,%p) not implemented\n",
 		flavor, slot_num, info, count);
 	return (KERN_FAILURE);
 }
@@ -79,11 +84,11 @@ cpu_info(
 void
 cpu_sleep(void)
 {
-	cpu_data_t	*proc_info = current_cpu_datap();
+	cpu_data_t	*cdp = current_cpu_datap();
 
-	proc_info->cpu_running = FALSE;
+	i386_deactivate_cpu();
 
-	PE_cpu_machine_quiesce(proc_info->cpu_id);
+	PE_cpu_machine_quiesce(cdp->cpu_id);
 
 	cpu_thread_halt();
 }
@@ -96,7 +101,7 @@ cpu_init(void)
 	cdp->cpu_type = cpuid_cputype();
 	cdp->cpu_subtype = cpuid_cpusubtype();
 
-	cdp->cpu_running = TRUE;
+	i386_activate_cpu();
 }
 
 kern_return_t
@@ -120,8 +125,17 @@ cpu_start(
 
 void
 cpu_exit_wait(
-	__unused int cpu)
+	int cpu)
 {
+    	cpu_data_t	*cdp = cpu_datap(cpu);
+
+	simple_lock(&x86_topo_lock);
+	while (!cdp->lcpu.halted) {
+	    simple_unlock(&x86_topo_lock);
+	    cpu_pause();
+	    simple_lock(&x86_topo_lock);
+	}
+	simple_unlock(&x86_topo_lock);
 }
 
 void
@@ -141,6 +155,9 @@ cpu_machine_init(
 	}
 #endif
 	ml_init_interrupt();
+
+	/* for every CPU, get the VT specs */
+	vmx_get_specs();
 }
 
 processor_t
diff --git a/osfmk/i386/cpu_affinity.h b/osfmk/i386/cpu_affinity.h
new file mode 100644
index 000000000..d65614a90
--- /dev/null
+++ b/osfmk/i386/cpu_affinity.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_CPU_AFFINITY_H_
+#define _I386_CPU_AFFINITY_H_
+
+#include <i386/cpu_topology.h>
+
+typedef struct x86_affinity_set
+{
+    struct x86_affinity_set	*next;	/* Forward link */
+    struct x86_cpu_cache	*cache;	/* The L2 cache concerned */	
+    processor_set_t		pset;	/* The processor set container */
+    uint32_t			num;	/* Logical id */
+} x86_affinity_set_t;
+
+extern x86_affinity_set_t *x86_affinities;	/* root of all affinities */
+
+extern int		ml_get_max_affinity_sets(void);
+extern processor_set_t	ml_affinity_to_pset(uint32_t affinity_num);
+
+#endif /* _I386_CPU_AFFINITY_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/cpu_capabilities.h b/osfmk/i386/cpu_capabilities.h
index 6270145f1..31aee1589 100644
--- a/osfmk/i386/cpu_capabilities.h
+++ b/osfmk/i386/cpu_capabilities.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	PRIVATE
 
@@ -42,10 +48,12 @@
 #define	kCache64			0x00000020
 #define	kCache128			0x00000040
 #define	kFastThreadLocalStorage		0x00000080	/* TLS ptr is kept in a user-mode-readable register */
-#define kHasSupplementalSSE3	0x00000100
-#define kHasMNI				kHasSupplementalSSE3
+#define kHasSupplementalSSE3		0x00000100
 #define	k64Bit				0x00000200	/* processor supports EM64T (not what mode you're running in) */
+#define	kHasSSE4_1			0x00000400
+#define	kHasSSE4_2			0x00000800
 
+#define	kSlow				0x00004000	/* tsc < nanosecond */
 #define	kUP				0x00008000	/* set if (kNumCPUs == 1) */
 #define	kNumCPUs			0x00FF0000	/* number of CPUs (see _NumCPUs() below) */
 
@@ -87,6 +95,11 @@ int _NumCPUs( void )
 #define _COMM_PAGE64_START_ADDRESS	( _COMM_PAGE64_BASE_ADDRESS )		/* address traditional commpage code starts on */
 #define _COMM_PAGE64_AREA_USED		( 2 * 4096 )				/* this is the amt actually populated */
 
+/* no need for an Objective-C area on Intel */
+#define _COMM_PAGE32_OBJC_SIZE		0ULL
+#define _COMM_PAGE32_OBJC_BASE		0ULL
+#define _COMM_PAGE64_OBJC_SIZE		0ULL
+#define _COMM_PAGE64_OBJC_BASE		0ULL
 
 #if defined(__i386__)
 
@@ -111,13 +124,15 @@ int _NumCPUs( void )
  
 #define _COMM_PAGE_SIGNATURE		(_COMM_PAGE_START_ADDRESS+0x000)	/* first few bytes are a signature */
 #define _COMM_PAGE_VERSION		(_COMM_PAGE_START_ADDRESS+0x01E)	/* 16-bit version# */
-#define	_COMM_PAGE_THIS_VERSION		6					/* version of the commarea format */
+#define	_COMM_PAGE_THIS_VERSION		7					/* version of the commarea format */
   
 #define _COMM_PAGE_CPU_CAPABILITIES	(_COMM_PAGE_START_ADDRESS+0x020)	/* uint32_t _cpu_capabilities */
 #define _COMM_PAGE_NCPUS		(_COMM_PAGE_START_ADDRESS+0x022)	/* uint8_t number of configured CPUs */
 #define _COMM_PAGE_CACHE_LINESIZE	(_COMM_PAGE_START_ADDRESS+0x026)	/* uint16_t cache line size */
 
-#define _COMM_PAGE_UNUSED1		(_COMM_PAGE_START_ADDRESS+0x028)	/* 24 unused bytes */
+#define _COMM_PAGE_SCHED_GEN		(_COMM_PAGE_START_ADDRESS+0x028)	/* uint32_t scheduler generation number (count of pre-emptions) */
+
+#define _COMM_PAGE_UNUSED1		(_COMM_PAGE_START_ADDRESS+0x02c)	/* 20 unused bytes */
  
 #if defined(__i386__)								/* following are not defined in 64-bit */
 #define _COMM_PAGE_2_TO_52		(_COMM_PAGE_START_ADDRESS+0x040)	/* double float constant 2**52 */
@@ -126,22 +141,27 @@ int _NumCPUs( void )
 #define _COMM_PAGE_UNUSED2		(_COMM_PAGE_START_ADDRESS+0x040)	/* 16 unused bytes */
 #endif
 
+#define	_COMM_PAGE_TIME_DATA_START	(_COMM_PAGE_START_ADDRESS+0x050)	/* base of offsets below (_NT_SCALE etc) */
 #define _COMM_PAGE_NT_TSC_BASE		(_COMM_PAGE_START_ADDRESS+0x050)	/* used by nanotime() */
 #define _COMM_PAGE_NT_SCALE		(_COMM_PAGE_START_ADDRESS+0x058)	/* used by nanotime() */
 #define _COMM_PAGE_NT_SHIFT		(_COMM_PAGE_START_ADDRESS+0x05c)	/* used by nanotime() */
 #define _COMM_PAGE_NT_NS_BASE		(_COMM_PAGE_START_ADDRESS+0x060)	/* used by nanotime() */
- 
-#define _COMM_PAGE_TIMEBASE		(_COMM_PAGE_START_ADDRESS+0x068)	/* used by gettimeofday() */
-#define _COMM_PAGE_TIMESTAMP		(_COMM_PAGE_START_ADDRESS+0x070)	/* used by gettimeofday() */
-#define _COMM_PAGE_TIMEENABLE		(_COMM_PAGE_START_ADDRESS+0x078)	/* used by gettimeofday() */
+#define _COMM_PAGE_NT_GENERATION	(_COMM_PAGE_START_ADDRESS+0x068)	/* used by nanotime() */
+#define _COMM_PAGE_GTOD_GENERATION	(_COMM_PAGE_START_ADDRESS+0x06c)	/* used by gettimeofday() */
+#define _COMM_PAGE_GTOD_NS_BASE		(_COMM_PAGE_START_ADDRESS+0x070)	/* used by gettimeofday() */
+#define _COMM_PAGE_GTOD_SEC_BASE	(_COMM_PAGE_START_ADDRESS+0x078)	/* used by gettimeofday() */
+
+/* Warning: kernel commpage.h has a matching c typedef for the following.  They must be kept in sync.  */
+/* These offsets are from _COMM_PAGE_TIME_DATA_START */
 
-#define	_NT_TSC_BASE			0					/* offsets into nanotime data */
+#define	_NT_TSC_BASE			0
 #define	_NT_SCALE			8
 #define	_NT_SHIFT			12
 #define	_NT_NS_BASE			16
-#define	_TIMEBASE			0					/* offsets into gettimeofday data */
-#define	_TIMESTAMP			8
-#define	_TIMEENABLE			16
+#define	_NT_GENERATION			24
+#define	_GTOD_GENERATION		28
+#define	_GTOD_NS_BASE			32
+#define	_GTOD_SEC_BASE			40
  
  /* jump table (jmp to this address, which may be a branch to the actual code somewhere else) */
  /* When new jump table entries are added, corresponding symbols should be added below         */
@@ -154,7 +174,7 @@ int _NumCPUs( void )
 #define _COMM_PAGE_ATOMIC_ADD32         (_COMM_PAGE_START_ADDRESS+0x1a0)	/* add atomic word */
 #define _COMM_PAGE_ATOMIC_ADD64         (_COMM_PAGE_START_ADDRESS+0x1c0)	/* add atomic doubleword */
 
-#define	_COMM_PAGE_UNUSED3		(_COMM_PAGE_START_ADDRESS+0x1e0)	/* 32 unused bytes */
+#define	_COMM_PAGE_UNUSED4		(_COMM_PAGE_START_ADDRESS+0x1e0)	/* 32 unused bytes */
 
 #define _COMM_PAGE_ABSOLUTE_TIME	(_COMM_PAGE_START_ADDRESS+0x200)	/* mach_absolute_time() */
 #define _COMM_PAGE_SPINLOCK_TRY		(_COMM_PAGE_START_ADDRESS+0x220)	/* spinlock_try() */
@@ -166,7 +186,7 @@ int _NumCPUs( void )
 #define _COMM_PAGE_FLUSH_ICACHE		(_COMM_PAGE_START_ADDRESS+0x520)	/* sys_icache_invalidate() */
 #define _COMM_PAGE_PTHREAD_SELF		(_COMM_PAGE_START_ADDRESS+0x580)	/* pthread_self() */
 
-#define _COMM_PAGE_UNUSED4		(_COMM_PAGE_START_ADDRESS+0x5a0)	/* 32 unused bytes */
+#define _COMM_PAGE_UNUSED5		(_COMM_PAGE_START_ADDRESS+0x5a0)	/* 32 unused bytes */
 
 #define _COMM_PAGE_RELINQUISH		(_COMM_PAGE_START_ADDRESS+0x5c0)	/* used by spinlocks */ 
 #define _COMM_PAGE_BTS		        (_COMM_PAGE_START_ADDRESS+0x5e0)	/* bit test-and-set */
@@ -176,12 +196,13 @@ int _NumCPUs( void )
 #define _COMM_PAGE_BCOPY		(_COMM_PAGE_START_ADDRESS+0x780)	/* bcopy() */
 #define	_COMM_PAGE_MEMCPY		(_COMM_PAGE_START_ADDRESS+0x7a0)	/* memcpy() */
 #define	_COMM_PAGE_MEMMOVE		(_COMM_PAGE_START_ADDRESS+0x7a0)	/* memmove() */
+#define	_COMM_PAGE_BCOPY_END		(_COMM_PAGE_START_ADDRESS+0xfff)	/* used by rosetta */
 
-#define	_COMM_PAGE_OLD_NANOTIME		(_COMM_PAGE_START_ADDRESS+0xf80)	/* old nanotime location (deprecated) */
 #define	_COMM_PAGE_MEMSET_PATTERN       (_COMM_PAGE_START_ADDRESS+0x1000)	/* used by nonzero memset() */
 #define	_COMM_PAGE_LONGCOPY		(_COMM_PAGE_START_ADDRESS+0x1200)	/* used by bcopy() for very long operands */
+#define	_COMM_PAGE_LONGCOPY_END		(_COMM_PAGE_START_ADDRESS+0x15ff)	/* used by rosetta */
 
-#define _COMM_PAGE_SYSTEM_INTEGRITY	(_COMM_PAGE_START_ADDRESS+0x1600)	/* system integrity data, 256 bytes */
+#define _COMM_PAGE_UNUSED6		(_COMM_PAGE_START_ADDRESS+0x1600)	/* unused */
 
 #define	_COMM_PAGE_NANOTIME		(_COMM_PAGE_START_ADDRESS+0x1700)	/* nanotime() */
 
@@ -225,7 +246,6 @@ symbol_name: nop
 	CREATE_COMM_PAGE_SYMBOL(___bcopy, _COMM_PAGE_BCOPY)
 	CREATE_COMM_PAGE_SYMBOL(___memcpy, _COMM_PAGE_MEMCPY)
 /*	CREATE_COMM_PAGE_SYMBOL(___memmove, _COMM_PAGE_MEMMOVE) */
-	CREATE_COMM_PAGE_SYMBOL(___old_nanotime, _COMM_PAGE_OLD_NANOTIME)
 	CREATE_COMM_PAGE_SYMBOL(___memset_pattern, _COMM_PAGE_MEMSET_PATTERN)
 	CREATE_COMM_PAGE_SYMBOL(___longcopy, _COMM_PAGE_LONGCOPY)
 	CREATE_COMM_PAGE_SYMBOL(___nanotime, _COMM_PAGE_NANOTIME)
diff --git a/osfmk/i386/cpu_data.h b/osfmk/i386/cpu_data.h
index 8ad34142a..e061888d7 100644
--- a/osfmk/i386/cpu_data.h
+++ b/osfmk/i386/cpu_data.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -39,13 +45,16 @@
 #include <mach/i386/thread_status.h>
 #include <i386/rtclock.h>
 #include <i386/pmCPU.h>
+#include <i386/cpu_topology.h>
+
+#include <i386/vmx/vmx_cpu.h>
 
 /*
  * Data structures referenced (anonymously) from per-cpu data:
  */
-struct cpu_core;
 struct cpu_cons_buffer;
 struct cpu_desc_table;
+struct mca_state;
 
 
 /*
@@ -57,13 +66,21 @@ typedef struct rtclock_timer {
 	boolean_t	has_expired;
 } rtclock_timer_t;
 
-typedef struct {
+typedef struct rtc_nanotime {
 	uint64_t	tsc_base;		/* timestamp */
 	uint64_t	ns_base;		/* nanoseconds */
 	uint32_t	scale;			/* tsc -> nanosec multiplier */
 	uint32_t	shift;			/* tsc -> nanosec shift/div */
+						/* shift is overloaded with
+						 * lower 32bits of tsc_freq
+						 * on slower machines (SLOW_TSC_THRESHOLD) */
+	uint32_t	generation;		/* 0 == being updated */
+	uint32_t	spare1;
 } rtc_nanotime_t;
 
+#define	SLOW_TSC_THRESHOLD	1000067800	/* TSC is too slow for regular nanotime() algorithm */
+
+
 typedef struct {
 	struct i386_tss         *cdi_ktss;
 #if    MACH_KDB
@@ -129,15 +146,15 @@ typedef struct cpu_data
 	int			cpu_running;
 	uint64_t		rtclock_intr_deadline;
 	rtclock_timer_t		rtclock_timer;
-        boolean_t		cpu_is64bit;
-        task_map_t		cpu_task_map;
-        addr64_t		cpu_task_cr3;
-        addr64_t		cpu_active_cr3;
-        addr64_t		cpu_kernel_cr3;
+	boolean_t		cpu_is64bit;
+	task_map_t		cpu_task_map;
+	addr64_t		cpu_task_cr3;
+	addr64_t		cpu_active_cr3;
+	addr64_t		cpu_kernel_cr3;
 	cpu_uber_t		cpu_uber;
 	void			*cpu_chud;
 	void			*cpu_console_buf;
-	struct cpu_core		*cpu_core;		/* cpu's parent core */
+	struct x86_lcpu		lcpu;
 	struct processor	*cpu_processor;
 	struct cpu_pmap		*cpu_pmap;
 	struct cpu_desc_table	*cpu_desc_tablep;
@@ -147,40 +164,48 @@ typedef struct cpu_data
 #ifdef MACH_KDB
 	/* XXX Untested: */
 	int			cpu_db_pass_thru;
-	vm_offset_t	cpu_db_stacks;
-	void		*cpu_kdb_saved_state;
-	spl_t		cpu_kdb_saved_ipl;
+	vm_offset_t		cpu_db_stacks;
+	void			*cpu_kdb_saved_state;
+	spl_t			cpu_kdb_saved_ipl;
 	int			cpu_kdb_is_slave;
 	int			cpu_kdb_active;
 #endif /* MACH_KDB */
 	boolean_t		cpu_iflag;
 	boolean_t		cpu_boot_complete;
 	int			cpu_hibernate;
-	pmsd			pms;					/* Power Management Stepper control */
-	uint64_t		rtcPop;			/* when the etimer wants a timer pop */
-	
-	vm_offset_t	cpu_copywindow_base;
-	uint64_t	*cpu_copywindow_pdp;
-	
-	vm_offset_t	cpu_physwindow_base;
-	uint64_t	*cpu_physwindow_ptep;
-	void 		*cpu_hi_iss;
-	boolean_t	cpu_tlb_invalid;
-	
-	uint64_t	*cpu_pmHpet;		/* Address of the HPET for this processor */
-	uint32_t	cpu_pmHpetVec;		/* Interrupt vector for HPET for this processor */
-/*	Statistics */
-	pmStats_t	cpu_pmStats;		/* Power management data */
-	uint32_t	cpu_hwIntCnt[256];		/* Interrupt counts */
-	
+
+	vm_offset_t		cpu_copywindow_base;
+	uint64_t		*cpu_copywindow_pdp;
+
+	vm_offset_t		cpu_physwindow_base;
+	uint64_t		*cpu_physwindow_ptep;
+	void 			*cpu_hi_iss;
+	boolean_t		cpu_tlb_invalid;
+	uint32_t		cpu_hwIntCnt[256];		/* Interrupt counts */
 	uint64_t		cpu_dr7; /* debug control register */
+	uint64_t		cpu_int_event_time;	/* intr entry/exit time */
+	vmx_cpu_t		cpu_vmx;		/* wonderful world of virtualization */
+	struct mca_state	*cpu_mca_state;		/* State at MC fault */
+	uint64_t		cpu_uber_arg_store;	/* Double mapped address
+							 * of current thread's
+							 * uu_arg array.
+							 */
+	uint64_t		cpu_uber_arg_store_valid; /* Double mapped
+							   * address of pcb
+							   * arg store
+							   * validity flag.
+							   */
+
+							  
 } cpu_data_t;
 
 extern cpu_data_t	*cpu_data_ptr[];  
 extern cpu_data_t	cpu_data_master;  
 
 /* Macro to generate inline bodies to retrieve per-cpu data fields. */
+#ifndef offsetof
 #define offsetof(TYPE,MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif /* offsetof */
 #define CPU_DATA_GET(member,type)					\
 	type ret;							\
 	__asm__ volatile ("movl %%gs:%P1,%0"				\
@@ -233,11 +258,6 @@ get_cpu_phys_number(void)
 {
 	CPU_DATA_GET(cpu_phys_number,int)
 }
-static inline struct
-cpu_core * get_cpu_core(void)
-{
-	CPU_DATA_GET(cpu_core,struct cpu_core *)
-}
 
 static inline void
 disable_preemption(void)
diff --git a/osfmk/i386/cpu_number.h b/osfmk/i386/cpu_number.h
index 81feb8eda..eeda73ff3 100644
--- a/osfmk/i386/cpu_number.h
+++ b/osfmk/i386/cpu_number.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/cpu_threads.c b/osfmk/i386/cpu_threads.c
index 0a4c3d5e2..ad8867f53 100644
--- a/osfmk/i386/cpu_threads.c
+++ b/osfmk/i386/cpu_threads.c
@@ -1,31 +1,42 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <vm/vm_kern.h>
+#include <kern/kalloc.h>
 #include <mach/machine.h>
 #include <i386/cpu_threads.h>
 #include <i386/cpuid.h>
 #include <i386/machine_cpu.h>
 #include <i386/lock.h>
 #include <i386/perfmon.h>
+#include <i386/pmCPU.h>
+
+#define bitmask(h,l)	((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
+#define bitfield(x,h,l)	(((x) & bitmask(h,l)) >> l)
 
 /*
  * Kernel parameter determining whether threads are halted unconditionally
@@ -34,81 +45,548 @@
  */
 int idlehalt = 1;
 
+x86_pkg_t	*x86_pkgs	= NULL;
+uint32_t	num_packages	= 0;
+uint32_t	num_Lx_caches[MAX_CACHE_DEPTH]	= { 0 };
+
+static x86_pkg_t	*free_pkgs	= NULL;
+static x86_core_t	*free_cores	= NULL;
+
+static x86_cpu_cache_t	*x86_caches	= NULL;
+static uint32_t		num_caches	= 0;
+
+decl_simple_lock_data(, x86_topo_lock);
+
+static x86_cpu_cache_t *
+x86_cache_alloc(void)
+{
+    x86_cpu_cache_t	*cache;
+    int			i;
+
+    if (x86_caches == NULL) {
+	cache = kalloc(sizeof(x86_cpu_cache_t) + (MAX_CPUS * sizeof(x86_lcpu_t *)));
+	if (cache == NULL)
+	    return(NULL);
+    } else {
+	cache = x86_caches;
+	x86_caches = cache->next;
+	cache->next = NULL;
+    }
+
+    bzero(cache, sizeof(x86_cpu_cache_t));
+    cache->next = NULL;
+    cache->maxcpus = MAX_CPUS;
+    for (i = 0; i < cache->maxcpus; i += 1) {
+	cache->cpus[i] = NULL;
+    }
+
+    num_caches += 1;
+
+    return(cache);
+}
+
+static void
+x86_cache_free(x86_cpu_cache_t *cache)
+{
+    num_caches -= 1;
+    if (cache->level > 0 && cache->level <= MAX_CACHE_DEPTH)
+	num_Lx_caches[cache->level - 1] -= 1;
+    cache->next = x86_caches;
+    x86_caches = cache;
+}
+
+/*
+ * This returns a list of cache structures that represent the
+ * caches for a CPU.  Some of the structures may have to be
+ * "freed" if they are actually shared between CPUs.
+ */
+static x86_cpu_cache_t *
+x86_cache_list(void)
+{
+    x86_cpu_cache_t	*root	= NULL;
+    x86_cpu_cache_t	*cur	= NULL;
+    x86_cpu_cache_t	*last	= NULL;
+    uint32_t		index;
+    uint32_t		cache_info[4];
+    uint32_t		nsets;
+
+    do_cpuid(0, cache_info);
+
+    if (cache_info[eax] < 4) {
+	/*
+	 * Processor does not support deterministic
+	 * cache information. Don't report anything
+	 */
+	return NULL;
+    }
+
+    for (index = 0; ; index += 1) {
+	cache_info[eax] = 4;
+	cache_info[ecx] = index;
+	cache_info[ebx] = 0;
+	cache_info[edx] = 0;
+
+	cpuid(cache_info);
+
+	/*
+	 * See if all levels have been queried.
+	 */
+	if (bitfield(cache_info[eax], 4, 0) == 0)
+	    break;
+
+	cur = x86_cache_alloc();
+	if (cur == NULL) {
+	    break;
+	}
+
+	cur->type = bitfield(cache_info[eax], 4, 0);
+	cur->level = bitfield(cache_info[eax], 7, 5);
+	cur->nlcpus = bitfield(cache_info[eax], 25, 14) + 1;
+	cur->line_size = bitfield(cache_info[ebx], 11, 0) + 1;
+	cur->partitions = bitfield(cache_info[ebx], 21, 12) + 1;
+	cur->ways = bitfield(cache_info[ebx], 31, 22) + 1;
+	nsets = bitfield(cache_info[ecx], 31, 0) + 1;
+	cur->cache_size = cur->line_size * cur->ways * cur->partitions * nsets;
+
+	if (last == NULL) {
+	    root = cur;
+	    last = cur;
+	} else {
+	    last->next = cur;
+	    last = cur;
+	}
+
+	num_Lx_caches[cur->level - 1] += 1;
+    }
+
+    return(root);
+}
 
 static boolean_t
 cpu_is_hyperthreaded(void)
 {
-	if  (cpuid_features() & CPUID_FEATURE_HTT)
-		return (cpuid_info()->cpuid_logical_per_package /
-			cpuid_info()->cpuid_cores_per_package) > 1;
-	else
-		return FALSE;
+    if  (cpuid_features() & CPUID_FEATURE_HTT)
+	return (cpuid_info()->cpuid_logical_per_package /
+		cpuid_info()->cpuid_cores_per_package) > 1;
+    else
+	return FALSE;
+}
+
+static void
+x86_lcpu_init(int cpu)
+{
+    cpu_data_t		*cpup;
+    x86_lcpu_t		*lcpu;
+    int			i;
+
+    cpup = cpu_datap(cpu);
+
+    lcpu = &cpup->lcpu;
+    lcpu->lcpu = lcpu;
+    lcpu->cpu  = cpup;
+    lcpu->next = NULL;
+    lcpu->core = NULL;
+    lcpu->lnum = cpu;
+    lcpu->pnum = cpup->cpu_phys_number;
+    lcpu->halted = FALSE;	/* XXX is this correct? */
+    lcpu->idle   = FALSE;	/* XXX is this correct? */
+    for (i = 0; i < MAX_CACHE_DEPTH; i += 1)
+	lcpu->caches[i] = NULL;
+
+    lcpu->master = (lcpu->pnum == (unsigned int) master_cpu);
+    lcpu->primary = (lcpu->pnum % cpuid_info()->cpuid_logical_per_package) == 0;
+}
+
+static x86_core_t *
+x86_core_alloc(int cpu)
+{
+    x86_core_t	*core;
+    cpu_data_t	*cpup;
+    uint32_t	cpu_in_pkg;
+    uint32_t	lcpus_per_core;
+
+    cpup = cpu_datap(cpu);
+
+    simple_lock(&x86_topo_lock);
+    if (free_cores != NULL) {
+	core = free_cores;
+	free_cores = core->next;
+	core->next = NULL;
+	simple_unlock(&x86_topo_lock);
+    } else {
+	simple_unlock(&x86_topo_lock);
+	core = kalloc(sizeof(x86_core_t));
+	if (core == NULL)
+	    panic("x86_core_alloc() kalloc of x86_core_t failed!\n");
+    }
+
+    bzero((void *) core, sizeof(x86_core_t));
+
+    cpu_in_pkg = cpu % cpuid_info()->cpuid_logical_per_package;
+    lcpus_per_core = cpuid_info()->cpuid_logical_per_package /
+		     cpuid_info()->cpuid_cores_per_package;
+
+    core->pcore_num = cpup->cpu_phys_number / lcpus_per_core;
+    core->lcore_num = core->pcore_num % cpuid_info()->cpuid_cores_per_package;
+
+    core->flags = X86CORE_FL_PRESENT | X86CORE_FL_READY;
+
+    return(core);
+}
+
+static void
+x86_core_free(x86_core_t *core)
+{
+    simple_lock(&x86_topo_lock);
+    core->next = free_cores;
+    free_cores = core;
+    simple_unlock(&x86_topo_lock);
+}
+
+static x86_pkg_t *
+x86_package_find(int cpu)
+{
+    x86_pkg_t	*pkg;
+    cpu_data_t	*cpup;
+    uint32_t	pkg_num;
+
+    cpup = cpu_datap(cpu);
+
+    pkg_num = cpup->cpu_phys_number / cpuid_info()->cpuid_logical_per_package;
+
+    pkg = x86_pkgs;
+    while (pkg != NULL) {
+	if (pkg->ppkg_num == pkg_num)
+	    break;
+	pkg = pkg->next;
+    }
+
+    return(pkg);
+}
+
+static x86_core_t *
+x86_core_find(int cpu)
+{
+    x86_core_t	*core;
+    x86_pkg_t	*pkg;
+    cpu_data_t	*cpup;
+    uint32_t	core_num;
+
+    cpup = cpu_datap(cpu);
+
+    core_num = cpup->cpu_phys_number
+	       / (cpuid_info()->cpuid_logical_per_package
+		  / cpuid_info()->cpuid_cores_per_package);
+
+    pkg = x86_package_find(cpu);
+    if (pkg == NULL)
+	return(NULL);
+
+    core = pkg->cores;
+    while (core != NULL) {
+	if (core->pcore_num == core_num)
+	    break;
+	core = core->next;
+    }
+
+    return(core);
+}
+
+static void
+x86_core_add_lcpu(x86_core_t *core, x86_lcpu_t *lcpu)
+{
+    x86_cpu_cache_t	*list;
+    x86_cpu_cache_t	*cur;
+    x86_core_t		*cur_core;
+    x86_lcpu_t		*cur_lcpu;
+    boolean_t		found;
+    int			level;
+    int			i;
+    uint32_t		cpu_mask;
+
+    assert(core != NULL);
+    assert(lcpu != NULL);
+
+    /*
+     * Add the cache data to the topology.
+     */
+    list = x86_cache_list();
+
+    simple_lock(&x86_topo_lock);
+
+    while (list != NULL) {
+	/*
+	 * Remove the cache from the front of the list.
+	 */
+	cur = list;
+	list = cur->next;
+	cur->next = NULL;
+	level = cur->level - 1;
+
+	/*
+	 * If the cache isn't shared then just put it where it
+	 * belongs.
+	 */
+	if (cur->nlcpus == 1) {
+	    goto found_first;
+	}
+
+	/*
+	 * We'll assume that all of the caches at a particular level
+	 * have the same sharing.  So if we have a cache already at
+	 * this level, we'll just skip looking for the match.
+	 */
+	if (lcpu->caches[level] != NULL) {
+	    x86_cache_free(cur);
+	    continue;
+	}
+
+	/*
+	 * This is a shared cache, so we have to figure out if
+	 * this is the first time we've seen this cache.  We do
+	 * this by searching through the package and seeing if
+	 * a related core is already describing this cache.
+	 *
+	 * NOTE: This assumes that CPUs whose ID mod <# sharing cache>
+	 * are indeed sharing the cache.
+	 */
+	cpu_mask = lcpu->pnum & ~(cur->nlcpus - 1);
+	cur_core = core->package->cores;
+	found = FALSE;
+
+	while (cur_core != NULL && !found) {
+	    cur_lcpu = cur_core->lcpus;
+	    while (cur_lcpu != NULL && !found) {
+		if ((cur_lcpu->pnum & ~(cur->nlcpus - 1)) == cpu_mask) {
+		    lcpu->caches[level] = cur_lcpu->caches[level];
+		    found = TRUE;
+		    x86_cache_free(cur);
+
+		    /*
+		     * Put the new CPU into the list of the cache.
+		     */
+		    cur = lcpu->caches[level];
+		    for (i = 0; i < cur->nlcpus; i += 1) {
+			if (cur->cpus[i] == NULL) {
+			    cur->cpus[i] = lcpu;
+			    break;
+			}
+		    }
+		}
+		cur_lcpu = cur_lcpu->next;
+	    }
+
+	    cur_core = cur_core->next;
+	}
+
+	if (!found) {
+found_first:
+	    cur->next = lcpu->caches[level];
+	    lcpu->caches[level] = cur;
+	    cur->cpus[0] = lcpu;
+	}
+    }
+
+    /*
+     * Add the Logical CPU to the core.
+     */
+    lcpu->next = core->lcpus;
+    lcpu->core = core;
+    core->lcpus = lcpu;
+    core->num_lcpus += 1;
+
+    simple_unlock(&x86_topo_lock);
+}
+
+static x86_pkg_t *
+x86_package_alloc(int cpu)
+{
+    x86_pkg_t	*pkg;
+    cpu_data_t	*cpup;
+
+    cpup = cpu_datap(cpu);
+
+    simple_lock(&x86_topo_lock);
+    if (free_pkgs != NULL) {
+	pkg = free_pkgs;
+	free_pkgs = pkg->next;
+	pkg->next = NULL;
+	simple_unlock(&x86_topo_lock);
+    } else {
+	simple_unlock(&x86_topo_lock);
+	pkg = kalloc(sizeof(x86_pkg_t));
+	if (pkg == NULL)
+	    panic("x86_package_alloc() kalloc of x86_pkg_t failed!\n");
+    }
+
+    bzero((void *) pkg, sizeof(x86_pkg_t));
+
+    pkg->ppkg_num = cpup->cpu_phys_number
+		    / cpuid_info()->cpuid_logical_per_package;
+
+    pkg->lpkg_num = num_packages;
+    atomic_incl((long *) &num_packages, 1);
+
+    pkg->flags = X86PKG_FL_PRESENT | X86PKG_FL_READY;
+    return(pkg);
+}
+
+static void
+x86_package_free(x86_pkg_t *pkg)
+{
+    simple_lock(&x86_topo_lock);
+    pkg->next = free_pkgs;
+    free_pkgs = pkg;
+    atomic_decl((long *) &num_packages, 1);
+    simple_unlock(&x86_topo_lock);
+}
+
+static void
+x86_package_add_core(x86_pkg_t *pkg, x86_core_t *core)
+{
+    assert(pkg != NULL);
+    assert(core != NULL);
+
+    core->next = pkg->cores;
+    core->package = pkg;
+    pkg->cores = core;
+    pkg->num_cores += 1;
 }
 
 void *
 cpu_thread_alloc(int cpu)
 {
-	int		core_base_cpu;
-	int		ret;
-	cpu_core_t	*core;
+    x86_core_t	*core;
+    x86_pkg_t	*pkg;
+    cpu_data_t	*cpup;
+    uint32_t	phys_cpu;
 
+    cpup = cpu_datap(cpu);
+
+    phys_cpu = cpup->cpu_phys_number;
+
+    x86_lcpu_init(cpu);
+
+    /*
+     * Assume that all cpus have the same features.
+     */
+    if (cpu_is_hyperthreaded()) {
+	cpup->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
+    } else {
+	cpup->cpu_threadtype = CPU_THREADTYPE_NONE;
+    }
+
+    /*
+     * Only allow one to manipulate the topology at a time.
+     */
+    simple_lock(&x86_topo_lock);
+
+    /*
+     * Get the core for this logical CPU.
+     */
+  core_again:
+    core = x86_core_find(cpu);
+    if (core == NULL) {
 	/*
- 	 * Assume that all cpus have the same features.
+	 * Core structure hasn't been created yet, do it now.
+	 *
+	 * Get the package that the core is part of.
 	 */
-	if (cpu_is_hyperthreaded()) {
-		/*
-		 * Get the cpu number of the base thread in the core.
-		 */
-		core_base_cpu = cpu_to_core_cpu(cpu);
-		cpu_datap(cpu)->cpu_threadtype = CPU_THREADTYPE_INTEL_HTT;
-	} else {
-		core_base_cpu = cpu;
-		cpu_datap(cpu)->cpu_threadtype = CPU_THREADTYPE_NONE;
+      package_again:
+	pkg = x86_package_find(cpu);
+	if (pkg == NULL) {
+	    /*
+	     * Package structure hasn't been created yet, do it now.
+	     */
+	    simple_unlock(&x86_topo_lock);
+	    pkg = x86_package_alloc(cpu);
+	    simple_lock(&x86_topo_lock);
+	    if (x86_package_find(cpu) != NULL) {
+		x86_package_free(pkg);
+		goto package_again;
+	    }
+	    
+	    /*
+	     * Add the new package to the global list of packages.
+	     */
+	    pkg->next = x86_pkgs;
+	    x86_pkgs = pkg;
 	}
 
-	core = (cpu_core_t *) cpu_to_core(core_base_cpu);
-	if (core == NULL) {
-		ret = kmem_alloc(kernel_map,
-				 (void *) &core, sizeof(cpu_core_t));
-		if (ret != KERN_SUCCESS)
-			panic("cpu_thread_alloc() kmem_alloc ret=%d\n", ret);
-		bzero((void *) core, sizeof(cpu_core_t));
+	/*
+	 * Allocate the core structure now.
+	 */
+	simple_unlock(&x86_topo_lock);
+	core = x86_core_alloc(cpu);
+	simple_lock(&x86_topo_lock);
+	if (x86_core_find(cpu) != NULL) {
+	    x86_core_free(core);
+	    goto core_again;
+	}
 
-		core->base_cpu = core_base_cpu;
+	/*
+	 * Add it to the package.
+	 */
+	x86_package_add_core(pkg, core);
+	machine_info.physical_cpu_max += 1;
 
-		atomic_incl((long *) &machine_info.physical_cpu_max, 1);
+	/*
+	 * Allocate performance counter structure.
+	 */
+	simple_unlock(&x86_topo_lock);
+	core->pmc = pmc_alloc();
+	simple_lock(&x86_topo_lock);
+    }
+    
+    /*
+     * Done manipulating the topology, so others can get in.
+     */
+    machine_info.logical_cpu_max += 1;
+    simple_unlock(&x86_topo_lock);
 
-		/* Allocate performance counter data area (if available) */
-		core->pmc = pmc_alloc();
-	}
-	atomic_incl((long *) &machine_info.logical_cpu_max, 1);
+    x86_core_add_lcpu(core, &cpup->lcpu);
 
-	return (void *) core;
+    return (void *) core;
 }
 
 void
 cpu_thread_init(void)
 {
-	int		my_cpu = get_cpu_number();
-	cpu_core_t	*my_core;
+    int		my_cpu	= get_cpu_number();
+    cpu_data_t	*cpup	= current_cpu_datap();
+    x86_core_t	*core;
+    static int	initialized = 0;
+
+    /*
+     * If we're the boot processor, we do all of the initialization of
+     * the CPU topology infrastructure.
+     */
+    if (my_cpu == master_cpu && !initialized) {
+	simple_lock_init(&x86_topo_lock, 0);
 
 	/*
-	 * If we're the boot processor we allocate the core structure here.
-	 * Otherwise the core has already been allocated (by the boot cpu).
+	 * Put this logical CPU into the physical CPU topology.
 	 */
-	if (my_cpu == master_cpu)
-		cpu_to_core(master_cpu) = cpu_thread_alloc(master_cpu);
+	cpup->lcpu.core = cpu_thread_alloc(my_cpu);
+
+	initialized = 1;
+    }
 
-	my_core = cpu_core();
-	if (my_core == NULL)
-		panic("cpu_thread_init() no core allocated for cpu %d", my_cpu);
+    /*
+     * Do the CPU accounting.
+     */
+    core = cpup->lcpu.core;
+    simple_lock(&x86_topo_lock);
+    machine_info.logical_cpu += 1;
+    if (core->active_lcpus == 0)
+	machine_info.physical_cpu += 1;
+    core->active_lcpus += 1;
+    cpup->lcpu.halted = FALSE;
+    cpup->lcpu.idle   = FALSE;
+    simple_unlock(&x86_topo_lock);
 
-	atomic_incl((long *) &my_core->active_threads, 1);
-	atomic_incl((long *) &machine_info.logical_cpu, 1);
-	/* Note: cpus are started serially so this isn't as racey as it looks */
-	if (my_core->num_threads == 0)
-		atomic_incl((long *) &machine_info.physical_cpu, 1);
-	atomic_incl((long *) &my_core->num_threads, 1);
+    pmCPUMarkRunning(cpup);
+    etimer_resync_deadlines();
 }
 
 /*
@@ -118,12 +596,25 @@ cpu_thread_init(void)
 void
 cpu_thread_halt(void)
 {
-	cpu_core_t	*my_core = cpu_core();
+    x86_core_t	*core;
+    cpu_data_t	*cpup = current_cpu_datap();
 
-	atomic_decl((long *) &machine_info.logical_cpu, 1);
-	atomic_decl((long *) &my_core->active_threads, 1);
-	if (atomic_decl_and_test((long *) &my_core->num_threads, 1))
-		atomic_decl((long *) &machine_info.physical_cpu, 1);
+    simple_lock(&x86_topo_lock);
+    machine_info.logical_cpu -= 1;
+    cpup->lcpu.idle   = TRUE;
+    core = cpup->lcpu.core;
+    core->active_lcpus -= 1;
+    if (core->active_lcpus == 0)
+	machine_info.physical_cpu -= 1;
+    simple_unlock(&x86_topo_lock);
 
-	cpu_halt();
+    /*
+     * Let the power management code determine the best way to "stop"
+     * the processor.
+     */
+    ml_set_interrupts_enabled(FALSE);
+    while (1) {
+	pmCPUHalt(PM_HALT_NORMAL);
+    }
+    /* NOT REACHED */
 }
diff --git a/osfmk/i386/cpu_threads.h b/osfmk/i386/cpu_threads.h
index 5f7931cba..e22f231cf 100644
--- a/osfmk/i386/cpu_threads.h
+++ b/osfmk/i386/cpu_threads.h
@@ -1,56 +1,64 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _I386_CPU_THREADS_H_
 #define _I386_CPU_THREADS_H_
 
-#include <i386/mp.h>
 #include <i386/cpu_data.h>
+#include <i386/mp.h>
+#include <i386/cpu_topology.h>
 
-struct pmc;
-
-typedef struct {
-	int		base_cpu;	/* Number of the cpu first in core */
-	int		num_threads;	/* Number of threads (logical cpus) */
-	int		active_threads;	/* Number of non-halted thredas */
-	struct pmc	*pmc;		/* Pointer to perfmon data */
-} cpu_core_t;
-
+/*
+ * These are defined here rather than in cpu_topology.h so as to keep
+ * cpu_topology.h from having a dependency on cpu_data.h.
+ */
 #define CPU_THREAD_MASK			0x00000001
 #define cpu_to_core_lapic(cpu)		(cpu_to_lapic[cpu] & ~CPU_THREAD_MASK)
 #define cpu_to_core_cpu(cpu)		(lapic_to_cpu[cpu_to_core_lapic(cpu)])
 #define cpu_to_logical_cpu(cpu)		(cpu_to_lapic[cpu] & CPU_THREAD_MASK)
 #define cpu_is_core_cpu(cpu)		(cpu_to_logical_cpu(cpu) == 0)
 
-#define cpu_to_core(cpu)		(cpu_datap(cpu)->cpu_core)
+#define cpu_to_lcpu(cpu)		(&cpu_datap(cpu)->lcpu)
+#define cpu_to_core(cpu)		(cpu_to_lcpu(cpu)->core)
+#define cpu_to_package(cpu)		(cpu_to_core(cpu)->package)
 
 /* Fast access: */
-#define cpu_core()			((cpu_core_t *) get_cpu_core())
+#define x86_lcpu()			(&current_cpu_datap()->lcpu)
+#define x86_core()			(x86_lcpu()->core)
+#define x86_package()			(x86_core()->package)
 
 #define cpu_is_same_core(cpu1,cpu2)	(cpu_to_core(cpu1) == cpu_to_core(cpu2))
+#define cpu_is_same_package(cpu1,cpu2)	(cpu_to_package(cpu1) == cpu_to_package(cpu2))
+#define cpus_share_cache(cpu1,cpu2,_cl) (cpu_to_lcpu(cpu1)->caches[_cl] == cpu_to_lcpu(cpu2)->caches[_cl])
+
+extern decl_simple_lock_data(, x86_topo_lock);
 
 extern void *cpu_thread_alloc(int);
 extern void cpu_thread_init(void);
 extern void cpu_thread_halt(void);
 
-extern int idlehalt;
-
 #endif /* _I386_CPU_THREADS_H_ */
diff --git a/osfmk/i386/cpu_topology.c b/osfmk/i386/cpu_topology.c
new file mode 100644
index 000000000..56b3b43a9
--- /dev/null
+++ b/osfmk/i386/cpu_topology.c
@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mach/machine.h>
+#include <mach/processor.h>
+#include <kern/kalloc.h>
+#include <i386/cpu_affinity.h>
+#include <i386/cpu_topology.h>
+#include <i386/cpu_data.h>
+#include <i386/cpu_threads.h>
+#include <i386/machine_cpu.h>
+#include <i386/machine_routines.h>
+#include <i386/lock.h>
+#include <i386/mp.h>
+
+//#define TOPO_DEBUG 1
+#if TOPO_DEBUG
+#define DBG(x...)	kprintf("DBG: " x)
+#else
+#define DBG(x...)
+#endif
+
+__private_extern__ void qsort(
+    void * array,
+    size_t nmembers,
+    size_t member_size,
+    int (*)(const void *, const void *));
+
+static int lapicid_cmp(const void *x, const void *y);
+static x86_affinity_set_t *find_cache_affinity(x86_cpu_cache_t *L2_cachep);
+
+x86_affinity_set_t	*x86_affinities = NULL;
+static int		x86_affinity_count = 0;
+
+/*
+ * cpu_topology_start() is called after all processors have been registered
+ * but before any non-boot processor id started.
+ * We establish canonical logical processor numbering - logical cpus must be
+ * contiguous, zero-based and assigned in physical (local apic id) order.
+ * This step is required because the discovery/registration order is
+ * non-deterministic - cores are registered in differing orders over boots.
+ * Enforcing canonical numbering simplifies identification
+ * of processors - in particular, for stopping/starting from CHUD.
+ */ 
+void
+cpu_topology_start(void)
+{
+	int		ncpus = machine_info.max_cpus;
+	int		i;
+	boolean_t	istate;
+
+	assert(machine_info.physical_cpu == 1);
+	assert(machine_info.logical_cpu == 1);
+	assert(master_cpu == 0);
+	assert(cpu_number() == 0);
+	assert(cpu_datap(0)->cpu_number == 0);
+	
+	/* Lights out for this */
+	istate = ml_set_interrupts_enabled(FALSE);
+
+#ifdef TOPO_DEBUG
+	DBG("cpu_topology_start() %d cpu%s registered\n",
+		ncpus, (ncpus > 1) ? "s" : "");
+	for (i = 0; i < ncpus; i++) {
+		cpu_data_t	*cpup = cpu_datap(i);
+		DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
+			i, (unsigned) cpup, cpup->cpu_phys_number);
+	}
+#endif
+	/*
+	 * Re-order the cpu_data_ptr vector sorting by physical id.
+	 * Skip the boot processor, it's required to be correct.
+	 */
+	if (ncpus > 1) {
+		qsort((void *) &cpu_data_ptr[1],
+			ncpus - 1,
+			sizeof(cpu_data_t *),
+			lapicid_cmp);
+	}
+#ifdef TOPO_DEBUG
+	DBG("cpu_topology_start() after sorting:\n");
+	for (i = 0; i < ncpus; i++) {
+		cpu_data_t	*cpup = cpu_datap(i);
+		DBG("\tcpu_data[%d]:0x%08x local apic 0x%x\n",
+			i, (unsigned) cpup, cpup->cpu_phys_number);
+	}
+#endif
+
+	/*
+	 * Fix up logical numbers and reset the map kept by the lapic code.
+	 */
+	for (i = 1; i < ncpus; i++) {
+		cpu_data_t	*cpup = cpu_datap(i);
+
+		if (cpup->cpu_number != i) {
+			kprintf("cpu_datap(%d):0x%08x local apic id 0x%x "
+				"remapped from %d\n",
+				i, (unsigned) cpup, cpup->cpu_phys_number,
+				cpup->cpu_number);
+		}
+		cpup->cpu_number = i;
+		cpup->lcpu.lnum = i;
+		lapic_cpu_map(cpup->cpu_phys_number, i);
+	}
+
+	ml_set_interrupts_enabled(istate);
+
+	/*
+	 * Iterate over all logical cpus finding or creating the affinity set
+	 * for their L2 cache. Each affinity set possesses a processor set
+	 * into which each logical processor is added.
+	 */
+	DBG("cpu_topology_start() creating affinity sets:\n");
+	for (i = 0; i < ncpus; i++) {
+		cpu_data_t		*cpup = cpu_datap(i);
+		x86_lcpu_t		*lcpup = cpu_to_lcpu(i);
+		x86_cpu_cache_t		*L2_cachep;
+		x86_affinity_set_t	*aset;
+
+		L2_cachep = lcpup->caches[CPU_CACHE_DEPTH_L2];
+		assert(L2_cachep->type == CPU_CACHE_TYPE_UNIF);
+		aset = find_cache_affinity(L2_cachep); 
+		if (aset == NULL) {
+			aset = (x86_affinity_set_t *) kalloc(sizeof(*aset));
+			if (aset == NULL)
+				panic("cpu_topology_start() failed aset alloc");
+			aset->next = x86_affinities;
+			x86_affinities = aset;
+			aset->num = x86_affinity_count++;
+			aset->cache = L2_cachep;
+			aset->pset = (i == master_cpu) ?
+					processor_pset(master_processor) :
+					pset_create(pset_node_root());
+			if (aset->pset == PROCESSOR_SET_NULL)
+				panic("cpu_topology_start: pset_create");
+			DBG("\tnew set %p(%d) pset %p for cache %p\n",
+				aset, aset->num, aset->pset, aset->cache);
+		}
+
+		DBG("\tprocessor_init set %p(%d) lcpup %p(%d) cpu %p processor %p\n",
+			aset, aset->num, lcpup, lcpup->lnum, cpup, cpup->cpu_processor);
+
+		if (i != master_cpu)
+			processor_init(cpup->cpu_processor, i, aset->pset);
+	}
+
+	/*
+	 * Finally we start all processors (including the boot cpu we're
+	 * running on).
+	 */
+	DBG("cpu_topology_start() processor_start():\n");
+	for (i = 0; i < ncpus; i++) {
+		DBG("\tlcpu %d\n", cpu_datap(i)->cpu_number);
+		processor_start(cpu_datap(i)->cpu_processor); 
+	}
+}
+
+static int
+lapicid_cmp(const void *x, const void *y)
+{
+	cpu_data_t	*cpu_x = *((cpu_data_t **)(uintptr_t)x);
+	cpu_data_t	*cpu_y = *((cpu_data_t **)(uintptr_t)y);
+
+	DBG("lapicid_cmp(%p,%p) (%d,%d)\n",
+		x, y, cpu_x->cpu_phys_number, cpu_y->cpu_phys_number);
+	if (cpu_x->cpu_phys_number < cpu_y->cpu_phys_number)
+		return -1;
+	if (cpu_x->cpu_phys_number == cpu_y->cpu_phys_number)
+		return 0;
+	return 1;
+}
+
+static x86_affinity_set_t *
+find_cache_affinity(x86_cpu_cache_t *l2_cachep)
+{
+	x86_affinity_set_t	*aset;
+
+	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
+		if (l2_cachep == aset->cache)
+			break;
+	}
+	return aset;			
+}
+
+int
+ml_get_max_affinity_sets(void)
+{
+	return x86_affinity_count;
+}
+
+processor_set_t
+ml_affinity_to_pset(uint32_t affinity_num) 
+{
+	x86_affinity_set_t	*aset;
+
+	for (aset = x86_affinities; aset != NULL; aset = aset->next) {
+		if (affinity_num == aset->num)
+			break;
+	}
+	return (aset == NULL) ? PROCESSOR_SET_NULL : aset->pset;			
+	
+}
+
+uint64_t
+ml_cpu_cache_size(unsigned int level)
+{
+	x86_cpu_cache_t	*cachep;
+
+	if (level == 0) {
+		return machine_info.max_mem;
+	} else if ( 1 <= level && level <= 3) {
+		cachep = current_cpu_datap()->lcpu.caches[level-1];
+		return cachep ? cachep->cache_size : 0;
+	} else {
+		return 0;
+	}
+}
+
+uint64_t
+ml_cpu_cache_sharing(unsigned int level)
+{
+	x86_cpu_cache_t	*cachep;
+
+	if (level == 0) {
+		return machine_info.max_cpus;
+	} else if ( 1 <= level && level <= 3) {
+		cachep = current_cpu_datap()->lcpu.caches[level-1];
+		return cachep ? cachep->nlcpus : 0;
+	} else {
+		return 0;
+	}
+}
+
diff --git a/osfmk/i386/cpu_topology.h b/osfmk/i386/cpu_topology.h
new file mode 100644
index 000000000..86729b935
--- /dev/null
+++ b/osfmk/i386/cpu_topology.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _I386_CPU_TOPOLOGY_H_
+#define _I386_CPU_TOPOLOGY_H_
+
+/*
+ * This was originally part of cpu_threads.h.  It was split out so that
+ * these structures could be referenced without pulling in all of the headers
+ * required for the definition of cpu_data.  These data structures are
+ * used by KEXTs in order to deal with the physical topology.
+ *
+ * NOTE: this header must stand on its own as much as possible
+ * and not be dependent upon any unexported, kernel-private header.
+ */
+
+/*
+ * Cache structure that can be used to identify the cache heirarchy.
+ */
+typedef struct x86_cpu_cache
+{
+    struct x86_cpu_cache *next;		/* next cache at this level/lcpu */
+    uint8_t		maxcpus;	/* maximum # of cpus that can share */
+    uint8_t		nlcpus;		/* # of logical cpus sharing this cache */
+    uint8_t		type;		/* type of cache */
+    uint8_t		level;		/* level of cache */
+    uint16_t		ways;		/* # of ways in cache */
+    uint16_t		partitions;	/* # of partitions in cache */
+    uint16_t		line_size;	/* size of a cache line */
+    uint32_t		cache_size;	/* total size of cache */
+    struct x86_lcpu	*cpus[0];	/* cpus sharing this cache */
+} x86_cpu_cache_t;
+
+#define CPU_CACHE_TYPE_DATA	1	/* data cache */
+#define CPU_CACHE_TYPE_INST	2	/* instruction cache */
+#define CPU_CACHE_TYPE_UNIF	3	/* unified cache */
+
+#define CPU_CACHE_DEPTH_L1	0
+#define CPU_CACHE_DEPTH_L2	1
+#define CPU_CACHE_DEPTH_L3	2
+
+#define MAX_CACHE_DEPTH		3	/* deepest cache */
+
+struct pmc;
+struct cpu_data;
+
+typedef struct x86_lcpu
+{
+    struct x86_lcpu	*next;	/* next logical cpu in core */
+    struct x86_lcpu	*lcpu;	/* pointer back to self */
+    struct x86_core	*core;	/* core containing the logical cpu */
+    struct cpu_data	*cpu;	/* cpu_data structure */
+    uint32_t		lnum;	/* logical cpu number */
+    uint32_t		pnum;	/* physical cpu number */
+    boolean_t		master;	/* logical cpu is the master (boot) CPU */
+    boolean_t		primary;/* logical cpu is primary CPU in package */
+    boolean_t		halted;	/* logical cpu is halted */
+    boolean_t		idle;	/* logical cpu is idle */
+    uint64_t		rtcPop;	/* when etimer wants a timer pop */
+    uint64_t		rtcDeadline;
+    x86_cpu_cache_t	*caches[MAX_CACHE_DEPTH];
+} x86_lcpu_t;
+
+#define X86CORE_FL_PRESENT	0x80000000	/* core is present */
+#define X86CORE_FL_READY	0x40000000	/* core struct is init'd */
+#define X86CORE_FL_HALTED	0x00008000	/* core is halted */
+#define X86CORE_FL_IDLE		0x00004000	/* core is idle */
+
+typedef struct x86_core
+{
+    struct x86_core	*next;		/* next core in package */
+    struct x86_lcpu	*lcpus;		/* list of logical cpus in core */
+    struct x86_pkg	*package;	/* package containing core */
+    uint32_t		flags;
+    uint32_t		lcore_num;	/* logical core # (unique to package) */
+    uint32_t		pcore_num;	/* physical core # (globally unique) */
+    uint32_t		num_lcpus;	/* Number of logical cpus */
+    uint32_t		active_lcpus;	/* Number of non-halted cpus */
+    struct pmc		*pmc;		/* Pointer to perfmon data */
+    struct hpetTimer	*Hpet;		/* Address of the HPET for this core */
+    uint32_t		HpetVec;	/* Interrupt vector for HPET */
+    uint64_t		HpetInt;	/* Number of HPET Interrupts */
+    uint64_t		HpetCmp;	/* HPET Comparitor */
+    uint64_t		HpetCfg;	/* HPET configuration */
+    uint64_t		HpetTime;
+    void		*pmStats;	/* Power management stats for core */
+    void		*pmState;	/* Power management state for core */
+} x86_core_t;
+
+#define X86PKG_FL_PRESENT	0x80000000	/* package is present */
+#define X86PKG_FL_READY		0x40000000	/* package struct init'd */
+#define X86PKG_FL_HAS_HPET	0x10000000	/* package has HPET assigned */
+#define X86PKG_FL_HALTED	0x00008000	/* package is halted */
+#define X86PKG_FL_IDLE		0x00004000	/* package is idle */
+
+typedef struct x86_pkg
+{
+    struct x86_pkg	*next;		/* next package */
+    struct x86_core	*cores;		/* list of cores in package */
+    uint32_t		flags;
+    uint32_t		lpkg_num;	/* logical package # */
+    uint32_t		ppkg_num;	/* physical package # */
+    uint32_t		num_cores;	/* number of cores in package */
+    struct hpetTimer	*Hpet;		/* address of HPET for this package */
+    uint32_t		HpetVec;	/* Interrupt vector for HPET */
+    uint64_t		HpetInt;	/* Number of HPET interrupts */
+    uint64_t		HpetCmp;	/* HPET comparitor */
+    uint64_t		HpetCfg;	/* HPET configuration */
+    uint64_t		HpetTime;
+    void		*pmStats;	/* Power Management stats for package*/
+    void		*pmState;	/* Power Management state for package*/
+} x86_pkg_t;
+
+extern x86_pkg_t	*x86_pkgs;	/* root of all CPU packages */
+
+/* Called after cpu discovery */
+extern void		cpu_topology_start(void);
+
+extern int idlehalt;
+
+#endif /* _I386_CPU_TOPOLOGY_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/cpuid.c b/osfmk/i386/cpuid.c
index 1ae202d47..a23ed95df 100644
--- a/osfmk/i386/cpuid.c
+++ b/osfmk/i386/cpuid.c
@@ -1,29 +1,36 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
 #include <platforms.h>
 #include <mach_kdb.h>
+#include <vm/vm_page.h>
 #include <pexpert/pexpert.h>
 
 #include "cpuid.h"
@@ -47,283 +54,28 @@
 
 /*
  * CPU identification routines.
- *
- * Note that this code assumes a processor that supports the
- * 'cpuid' instruction.
  */
 
-static unsigned int	cpuid_maxcpuid;
-
 static i386_cpu_info_t	*cpuid_cpu_infop = NULL;
 static i386_cpu_info_t	cpuid_cpu_info;
 
-uint32_t		cpuid_feature;		/* XXX obsolescent for compat */
-
-/*
- * We only identify Intel CPUs here.  Adding support
- * for others would be straightforward.
- */
-static void	set_cpu_generic(i386_cpu_info_t *);
-static void	set_cpu_intel(i386_cpu_info_t *);
-static void	set_cpu_amd(i386_cpu_info_t *);
-static void	set_cpu_nsc(i386_cpu_info_t *);
-static void	set_cpu_unknown(i386_cpu_info_t *);
-
-struct {
-	const char	*vendor;
-	void		(* func)(i386_cpu_info_t *);
-} cpu_vendors[] = {
-	{CPUID_VID_INTEL,	set_cpu_intel},
-	{CPUID_VID_AMD,         set_cpu_amd},
-	{CPUID_VID_NSC,         set_cpu_nsc},
-	{0,			set_cpu_unknown}
-};
-
-void
-cpuid_get_info(i386_cpu_info_t *info_p)
-{
-	uint32_t	cpuid_result[4];
-	int		i;
-
-	bzero((void *)info_p, sizeof(i386_cpu_info_t));
-
-	/* do cpuid 0 to get vendor */
-	do_cpuid(0, cpuid_result);
-	cpuid_maxcpuid = cpuid_result[eax];
-	bcopy((char *)&cpuid_result[ebx], &info_p->cpuid_vendor[0], 4); /* ug */
-	bcopy((char *)&cpuid_result[ecx], &info_p->cpuid_vendor[8], 4);
-	bcopy((char *)&cpuid_result[edx], &info_p->cpuid_vendor[4], 4);
-	info_p->cpuid_vendor[12] = 0;
-
-	/* look up vendor */
-	for (i = 0; ; i++) {
-		if ((cpu_vendors[i].vendor == 0) ||
-		    (!strcmp(cpu_vendors[i].vendor, info_p->cpuid_vendor))) {
-			cpu_vendors[i].func(info_p);
-			break;
-		}
-	}
-}
-
-/*
- * Cache descriptor table. Each row has the form:
- *	   (descriptor_value,		cache,	size,		linesize,
- * 				description)
- * Note: the CACHE_DESC macro does not expand description text in the kernel.
- */
-static cpuid_cache_desc_t cpuid_cache_desc_tab[] = {
-CACHE_DESC(CPUID_CACHE_ITLB_4K, 	Lnone,	0,		0, \
-	"Instruction TLB, 4K, pages 4-way set associative, 64 entries"),
-CACHE_DESC(CPUID_CACHE_ITLB_4M, 	Lnone,	0,		0, \
-	"Instruction TLB, 4M, pages 4-way set associative, 2 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_4K, 	Lnone,	0,		0, \
-	"Data TLB, 4K pages, 4-way set associative, 64 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_4M, 	Lnone,	0,		0, \
-	"Data TLB, 4M pages, 4-way set associative, 8 entries"),
-CACHE_DESC(CPUID_CACHE_ITLB_64, 	Lnone,	0,		0, \
-	"Instruction TLB, 4K and 2M or 4M pages, 64 entries"),
-CACHE_DESC(CPUID_CACHE_ITLB_128, 	Lnone,	0,		0, \
-	"Instruction TLB, 4K and 2M or 4M pages, 128 entries"),
-CACHE_DESC(CPUID_CACHE_ITLB_256, 	Lnone,	0,		0, \
-	"Instruction TLB, 4K and 2M or 4M pages, 256 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_64,		Lnone,	0,		0, \
-	"Data TLB, 4K and 4M pages, 64 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_128,	Lnone,	0,		0, \
-	"Data TLB, 4K and 4M pages, 128 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_256,	Lnone,	0,		0, \
-	"Data TLB, 4K and 4M pages, 256 entries"),
-CACHE_DESC(CPUID_CACHE_ITLB_4K_128_4,      Lnone,  0,              0, \
-        "Instruction TLB, 4K pages, 4-way set associative, 128 entries"),
-CACHE_DESC(CPUID_CACHE_DTLB_4K_128_4,      Lnone,  0,              0, \
-        "Data TLB, 4K pages, 4-way set associative, 128 entries"),
-CACHE_DESC(CPUID_CACHE_ICACHE_8K,	L1I,	8*1024, 	32, \
-	"Instruction L1 cache, 8K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_8K,	L1D,	8*1024, 	32, \
-	"Data L1 cache, 8K, 2-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_ICACHE_16K,	L1I,	16*1024,	 32, \
-	"Instruction L1 cache, 16K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_16K,	L1D,	16*1024, 	32, \
-	"Data L1 cache, 16K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_8K_4_64,	L1D,	8*1024,		64, \
-	"Data L1 cache, 8K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_16K_4_64,	L1D,	16*1024,	64, \
-	"Data L1 cache, 16K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_32K_4_64,	L1D,	32*1024,	64, \
-	"Data L1 cache, 32K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_32K,      L1D,    32*1024,        64, \
-        "Data L1 cache, 32K, 8-way set assocative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_ICACHE_32K,      L1I,    32*1024,        64, \
-        "Instruction L1 cache, 32K, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_DCACHE_16K_8_64,    L1D,    16*1024,        64, \
-        "Data L1 cache, 16K, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_TRACE_12K_8,	L1I,	12*1024,	64, \
-	"Trace cache, 12K-uop, 8-way set associative"),
-CACHE_DESC(CPUID_CACHE_TRACE_16K_8,	L1I,	16*1024,	64, \
-	"Trace cache, 16K-uop, 8-way set associative"),
-CACHE_DESC(CPUID_CACHE_TRACE_32K_8,	L1I,	32*1024,	64, \
-	"Trace cache, 32K-uop, 8-way set associative"),
-CACHE_DESC(CPUID_CACHE_L2_128K,	L2U,	128*1024,	32, \
-	"Unified L2 cache, 128K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_256K,	L2U,	128*1024,	32, \
-	"Unified L2 cache, 256K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_512K,	L2U,	512*1024,	32, \
-	"Unified L2 cache, 512K, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_1M_4,	L2U,	1*1024*1024,	32, \
-	"Unified L2 cache, 1M, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_2M_4,	L2U,	2*1024*1024,	32, \
-	"Unified L2 cache, 2M, 4-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_4M_16_64,	L2U,	4*1024*1024,	64, \
-	"Unified L2 cache, 4M, 16-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_128K_8_64_2,	L2U,	128*1024,	64, \
-	"Unified L2 cache, 128K, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_256K_8_64_2,	L2U,	256*1024,	64, \
-	"Unified L2 cache, 256K, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_512K_8_64_2,	L2U,	512*1024,	64, \
-	"Unified L2 cache, 512K, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_1M_8_64_2,	L2U,	1*1024*1024,	64, \
-	"Unified L2 cache, 1M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_256K_8_32,	L2U,	256*1024,	32, \
-	"Unified L2 cache, 256K, 8-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_512K_8_32,	L2U,	512*1024,	32, \
-	"Unified L2 cache, 512K, 8-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_1M_8_32,	L2U,	1*1024*1024,	32, \
-	"Unified L2 cache, 1M, 8-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_2M_8_32,	L2U,	2*1024*1024,	32, \
-	"Unified L2 cache, 2M, 8-way set associative, 32byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_1M_4_64,  L2U,    1*1024*1024,    64, \
-        "Unified L2 cache, 1M, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_2M_8_64,    L2U,    2*1024*1024,    64, \
-        "Unified L2 cache, 2M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_512K_2_64,L2U,    512*1024,       64, \
-        "Unified L2 cache, 512K, 2-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_512K_4_64,L2U,    512*1024,       64, \
-        "Unified L2 cache, 512K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_1M_8_64,  L2U,    1*1024*1024,    64, \
-        "Unified L2 cache, 1M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_128K_S4,  L2U,    128*1024,       64, \
-        "Unified L2 sectored cache, 128K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_128K_S2,  L2U,    128*1024,       64, \
-        "Unified L2 sectored cache, 128K, 2-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L2_256K_S4,  L2U,    256*1024,       64, \
-        "Unified L2 sectored cache, 256K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L3_512K,    L3U,    512*1024,       64, \
-        "Unified L3 cache, 512K, 4-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L3_1M,      L3U,    1*1024*1024,    64, \
-        "Unified L3 cache, 1M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L3_2M,      L3U,    2*1024*1024,    64, \
-        "Unified L3 cache, 2M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_L3_4M,      L3U,    4*1024*1024,    64, \
-        "Unified L3 cache, 4M, 8-way set associative, 64byte line size"),
-CACHE_DESC(CPUID_CACHE_PREFETCH_64,     Lnone,  0,              0,  \
-        "64-Byte Prefetching"),
-CACHE_DESC(CPUID_CACHE_PREFETCH_128,    Lnone,  0,              0,  \
-        "128-Byte Prefetching"),
-CACHE_DESC(CPUID_CACHE_NOCACHE, Lnone, 0, 0, \
-        "No L2 cache or, if valid L2 cache, no L3 cache"),
-CACHE_DESC(CPUID_CACHE_NULL, Lnone, 0, 0, \
-	(char *)0),
-};
-
-static const char * get_intel_model_string( i386_cpu_info_t * info_p, cpu_type_t* type, cpu_subtype_t* subtype)
-{
-	*type = CPU_TYPE_X86;
-	*subtype = CPU_SUBTYPE_X86_ARCH1;
-
-    /* check for brand id string */
-    switch(info_p->cpuid_brand) {
-        case CPUID_BRAND_UNSUPPORTED:
-            /* brand ID not supported; use alternate method. */
-            switch(info_p->cpuid_family) {
-                case CPUID_FAMILY_486:
-                    return "Intel 486";
-                case CPUID_FAMILY_586:
-                    return "Intel Pentium";
-                case CPUID_FAMILY_686:
-                    switch(info_p->cpuid_model) {
-                        case CPUID_MODEL_P6:
-                            return "Intel Pentium Pro";
-                        case CPUID_MODEL_PII:
-                            return "Intel Pentium II";
-                        case CPUID_MODEL_P65:
-                        case CPUID_MODEL_P66:
-                            return "Intel Celeron";
-                        case CPUID_MODEL_P67:
-                        case CPUID_MODEL_P68:
-                        case CPUID_MODEL_P6A:
-                        case CPUID_MODEL_P6B:
-                            return "Intel Pentium III";
-                        case CPUID_MODEL_PM9:
-                        case CPUID_MODEL_PMD:
-                            return "Intel Pentium M";
-                        default:
-                            return "Unknown Intel P6 Family";
-                    }
-                case CPUID_FAMILY_EXTENDED:
-                    switch (info_p->cpuid_extfamily) {
-                        case CPUID_EXTFAMILY_PENTIUM4:
-			    *subtype = CPU_SUBTYPE_PENTIUM_4;
-                            return "Intel Pentium 4";
-						default:
-		                    return "Unknown Intel Extended Family";
-                    }
-                default:
-                    return "Unknown Intel Family";
-            }
-            break;
-        case CPUID_BRAND_CELERON_1:
-        case CPUID_BRAND_CELERON_A:
-        case CPUID_BRAND_CELERON_14:
-            return "Intel Celeron";
-        case CPUID_BRAND_PENTIUM_III_2:
-        case CPUID_BRAND_PENTIUM_III_4:
-            return "Pentium III";
-        case CPUID_BRAND_PIII_XEON:
-			if (info_p->cpuid_signature == 0x6B1) {
-				return "Intel Celeron";
-			} else {
-				return "Intel Pentium III Xeon";
-			}
-        case CPUID_BRAND_PENTIUM_III_M:
-            return "Mobile Intel Pentium III-M";
-        case CPUID_BRAND_M_CELERON_7:
-        case CPUID_BRAND_M_CELERON_F:
-        case CPUID_BRAND_M_CELERON_13:
-        case CPUID_BRAND_M_CELERON_17:
-            return "Mobile Intel Celeron";
-        case CPUID_BRAND_PENTIUM4_8:
-        case CPUID_BRAND_PENTIUM4_9:
-	    *subtype = CPU_SUBTYPE_PENTIUM_4;
-            return "Intel Pentium 4";
-        case CPUID_BRAND_XEON:
-            return "Intel Xeon";
-        case CPUID_BRAND_XEON_MP:
-            return "Intel Xeon MP";
-        case CPUID_BRAND_PENTIUM4_M:
-			if (info_p->cpuid_signature == 0xF13) {
-				return "Intel Xeon";
-			} else {
-				*subtype = CPU_SUBTYPE_PENTIUM_4;
-				return "Mobile Intel Pentium 4";
-			}
-        case CPUID_BRAND_CELERON_M:
-            return "Intel Celeron M";
-        case CPUID_BRAND_PENTIUM_M:
-            return "Intel Pentium M";
-        case CPUID_BRAND_MOBILE_15:
-        case CPUID_BRAND_MOBILE_17:
-            return "Mobile Intel";
-    }        
-    return "Unknown Intel";
-}
-
-static void set_intel_cache_info( i386_cpu_info_t * info_p )
+/* this function is Intel-specific */
+static void
+cpuid_set_cache_info( i386_cpu_info_t * info_p )
 {
 	uint32_t	cpuid_result[4];
-        uint32_t        l1d_cache_linesize = 0;
+	uint32_t	reg[4];
+	uint32_t	index;
+	uint32_t	linesizes[LCACHE_MAX];
 	unsigned int	i;
 	unsigned int	j;
+	boolean_t	cpuid_deterministic_supported = FALSE;
 
-	/* get processor cache descriptor info */
+	bzero( linesizes, sizeof(linesizes) );
+
+	/* Get processor cache descriptor info using leaf 2.  We don't use
+	 * this internally, but must publish it for KEXTs.
+	 */
 	do_cpuid(2, cpuid_result);
 	for (j = 0; j < 4; j++) {
 		if ((cpuid_result[j] >> 31) == 1) 	/* bit31 is validity */
@@ -343,232 +95,143 @@ static void set_intel_cache_info( i386_cpu_info_t * info_p )
 		}
 	}
 
-	/* decode the descriptors looking for L1/L2/L3 size info */
-	for (i = 1; i < sizeof(info_p->cache_info); i++) {
-		cpuid_cache_desc_t	*descp;
-		uint8_t			desc = info_p->cache_info[i];
-
-		if (desc == CPUID_CACHE_NULL)
-			continue;
-		for (descp = cpuid_cache_desc_tab;
-			descp->value != CPUID_CACHE_NULL; descp++) {
-			if (descp->value != desc)
-				continue;
-			info_p->cache_size[descp->type] = descp->size;
-			if (descp->type == L2U)
-				info_p->cache_linesize = descp->linesize;
-                        if (descp->type == L1D)
-                                l1d_cache_linesize = descp->linesize;
-			break;
-		}
-	}
-	/* For P-IIIs, L2 could be 256k or 512k but we can't tell */ 
-	if (info_p->cache_size[L2U] == 0 &&
-	    info_p->cpuid_family == 0x6 && info_p->cpuid_model == 0xb) {
-		info_p->cache_size[L2U] = 256*1024;
-		info_p->cache_linesize = 32;
-	}
-        /* If we have no L2 cache, use the L1 data cache line size */
-        if (info_p->cache_size[L2U] == 0)
-            info_p->cache_linesize = l1d_cache_linesize;
-
 	/*
-	 * Get cache sharing info if available.
+	 * Get cache info using leaf 4, the "deterministic cache parameters."
+	 * Most processors Mac OS X supports implement this flavor of CPUID.
+	 * Loop over each cache on the processor.
 	 */
 	do_cpuid(0, cpuid_result);
-	if (cpuid_result[eax] >= 4) {
-		uint32_t	reg[4];
-		uint32_t	index;
-		for (index = 0;; index++) {
-			/*
-			 * Scan making calls for cpuid with %eax = 4
-			 * to get info about successive cache levels
-			 * until a null type is returned.
-			 */
-			cache_type_t	type = Lnone;
-			uint32_t	cache_type;
-			uint32_t	cache_level;
-			uint32_t	cache_sharing;
-
-			reg[eax] = 4;		/* cpuid request 4 */
-			reg[ecx] = index;	/* index starting at 0 */
-			cpuid(reg);
+	if (cpuid_result[eax] >= 4)
+		cpuid_deterministic_supported = TRUE;
+
+	for (index = 0; cpuid_deterministic_supported; index++) {
+		cache_type_t	type = Lnone;
+		uint32_t	cache_type;
+		uint32_t	cache_level;
+		uint32_t	cache_sharing;
+		uint32_t	cache_linesize;
+		uint32_t	cache_sets;
+		uint32_t	cache_associativity;
+		uint32_t	cache_size;
+		uint32_t	cache_partitions;
+		uint32_t	colors;
+		
+		reg[eax] = 4;		/* cpuid request 4 */
+		reg[ecx] = index;	/* index starting at 0 */
+		cpuid(reg);
 //kprintf("cpuid(4) index=%d eax=%p\n", index, reg[eax]);
-			cache_type = bitfield(reg[eax], 4, 0);
-			if (cache_type == 0)
-				break;		/* done with cache info */
-			cache_level   = bitfield(reg[eax],  7,  5);
-			cache_sharing = bitfield(reg[eax], 25, 14);
-			info_p->cpuid_cores_per_package = 
-					bitfield(reg[eax], 31, 26) + 1;
-			switch (cache_level) {
-			case 1:
-				type = cache_type == 1 ? L1D :
-				       cache_type == 2 ? L1I :
-							 Lnone;
-				break;
-			case 2:
-				type = cache_type == 3 ? L2U :
-							 Lnone;
-				break;
-			case 3:
-				type = cache_type == 3 ? L3U :
-							 Lnone;
-			}
-			if (type != Lnone)
-				info_p->cache_sharing[type] = cache_sharing + 1;
-		} 
-	}
-}
-
-static void set_cpu_intel( i386_cpu_info_t * info_p )
-{
-    set_cpu_generic(info_p);
-    set_intel_cache_info(info_p);
-    info_p->cpuid_model_string = get_intel_model_string(info_p, &info_p->cpuid_cpu_type, &info_p->cpuid_cpu_subtype);
-}
-
-static const char * get_amd_model_string( i386_cpu_info_t * info_p, cpu_type_t* type, cpu_subtype_t* subtype )
-{
-	*type = CPU_TYPE_X86;
-	*subtype = CPU_SUBTYPE_X86_ARCH1;
-
-    /* check for brand id string */
-    switch (info_p->cpuid_family)
-    {
-        case CPUID_FAMILY_486:
-            switch (info_p->cpuid_model) {
-                case CPUID_MODEL_AM486_DX:
-                case CPUID_MODEL_AM486_DX2:
-                case CPUID_MODEL_AM486_DX2WB:
-                case CPUID_MODEL_AM486_DX4:
-                case CPUID_MODEL_AM486_DX4WB:
-                    return "Am486";
-                case CPUID_MODEL_AM486_5X86:
-                case CPUID_MODEL_AM486_5X86WB:
-                    return "Am5x86";
-            }
-            break;
-        case CPUID_FAMILY_586:
-            switch (info_p->cpuid_model) {
-                case CPUID_MODEL_K5M0:
-                case CPUID_MODEL_K5M1:
-                case CPUID_MODEL_K5M2:
-                case CPUID_MODEL_K5M3:
-                    return "AMD-K5";
-                case CPUID_MODEL_K6M6:
-                case CPUID_MODEL_K6M7:
-                    return "AMD-K6";
-                case CPUID_MODEL_K6_2:
-                    return "AMD-K6-2";
-                case CPUID_MODEL_K6_III:
-                    return "AMD-K6-III";
-            }
-            break;
-        case CPUID_FAMILY_686:
-            switch (info_p->cpuid_model) {
-                case CPUID_MODEL_ATHLON_M1:
-                case CPUID_MODEL_ATHLON_M2:
-                case CPUID_MODEL_ATHLON_M4:
-                case CPUID_MODEL_ATHLON_M6:
-                case CPUID_MODEL_ATHLON_M8:
-                case CPUID_MODEL_ATHLON_M10:
-                    return "AMD Athlon";
-                case CPUID_MODEL_DURON_M3:
-                case CPUID_MODEL_DURON_M7:
-                    return "AMD Duron";
-                default:
-                    return "Unknown AMD Athlon";
-            }
-        case CPUID_FAMILY_EXTENDED:
-            switch (info_p->cpuid_model) {
-                case CPUID_MODEL_ATHLON64:
-                    return "AMD Athlon 64";
-                case CPUID_MODEL_OPTERON:
-                    return "AMD Opteron";
-                default:
-                    return "Unknown AMD-64";
-            }
-    }
-    return "Unknown AMD";
-}
-
-static void set_amd_cache_info( i386_cpu_info_t * info_p )
-{
-    uint32_t	cpuid_result[4];
-
-    /* It would make sense to fill in info_p->cache_info with complete information
-     * on the TLBs and data cache associativity, lines, etc, either by mapping
-     * to the Intel tags (if possible), or replacing cache_info with a generic
-     * mechanism.  But right now, nothing makes use of that information (that I know
-     * of).
-     */
-
-    /* L1 Cache and TLB Information */
-    do_cpuid(0x80000005, cpuid_result);
-    
-    /* EAX: TLB Information for 2-Mbyte and 4-MByte Pages */
-    /* (ignore) */
-    
-    /* EBX: TLB Information for 4-Kbyte Pages */
-    /* (ignore) */
-    
-    /* ECX: L1 Data Cache Information */
-    info_p->cache_size[L1D] = ((cpuid_result[ecx] >> 24) & 0xFF) * 1024;
-    info_p->cache_linesize = (cpuid_result[ecx] & 0xFF);
-    
-    /* EDX: L1 Instruction Cache Information */
-    info_p->cache_size[L1I] = ((cpuid_result[edx] >> 24) & 0xFF) * 1024;
-
-    /* L2 Cache Information */
-    do_cpuid(0x80000006, cpuid_result);
-    
-    /* EAX: L2 TLB Information for 2-Mbyte and 4-Mbyte Pages */
-    /* (ignore) */
-    
-    /* EBX: L2 TLB Information for 4-Kbyte Pages */
-    /* (ignore) */
-    
-    /* ECX: L2 Cache Information */
-    info_p->cache_size[L2U] = ((cpuid_result[ecx] >> 16) & 0xFFFF) * 1024;
-    if (info_p->cache_size[L2U] > 0)
-        info_p->cache_linesize = cpuid_result[ecx] & 0xFF;
-}
+		cache_type = bitfield(reg[eax], 4, 0);
+		if (cache_type == 0)
+			break;		/* no more caches */
+		cache_level  		= bitfield(reg[eax],  7,  5);
+		cache_sharing	 	= bitfield(reg[eax], 25, 14) + 1;
+		info_p->cpuid_cores_per_package 
+					= bitfield(reg[eax], 31, 26) + 1;
+		cache_linesize		= bitfield(reg[ebx], 11,  0) + 1;
+		cache_partitions	= bitfield(reg[ebx], 21, 12) + 1;
+		cache_associativity	= bitfield(reg[ebx], 31, 22) + 1;
+		cache_sets 		= bitfield(reg[ecx], 31,  0) + 1;
+				
+		/* Map type/levels returned by CPUID into cache_type_t */
+		switch (cache_level) {
+		case 1:
+			type = cache_type == 1 ? L1D :
+			       cache_type == 2 ? L1I :
+						 Lnone;
+			break;
+		case 2:
+			type = cache_type == 3 ? L2U :
+						 Lnone;
+			break;
+		case 3:
+			type = cache_type == 3 ? L3U :
+						 Lnone;
+			break;
+		default:
+			type = Lnone;
+		}
+		
+		/* The total size of a cache is:
+		 *	( linesize * sets * associativity )
+		 */
+		if (type != Lnone) {
+			cache_size = cache_linesize * cache_sets * cache_associativity;
+			info_p->cache_size[type] = cache_size;
+			info_p->cache_sharing[type] = cache_sharing;
+			info_p->cache_partitions[type] = cache_partitions;
+			linesizes[type] = cache_linesize;
+			
+			/* Compute the number of page colors for this cache,
+			 * which is:
+			 *	( linesize * sets ) / page_size
+			 *
+			 * To help visualize this, consider two views of a
+			 * physical address.  To the cache, it is composed
+			 * of a line offset, a set selector, and a tag.
+			 * To VM, it is composed of a page offset, a page
+			 * color, and other bits in the pageframe number:
+			 *
+			 *           +-----------------+---------+--------+
+			 *  cache:   |       tag       |   set   | offset |
+			 *           +-----------------+---------+--------+
+			 *
+			 *           +-----------------+-------+----------+
+			 *  VM:      |    don't care   | color | pg offset|
+			 *           +-----------------+-------+----------+
+			 *
+			 * The color is those bits in (set+offset) not covered
+			 * by the page offset.
+			 */
+			 colors = ( cache_linesize * cache_sets ) >> 12;
+			 
+			 if ( colors > vm_cache_geometry_colors )
+				vm_cache_geometry_colors = colors;
+		}
+	} 
+	
+	/*
+	 * If deterministic cache parameters are not available, use
+	 * something else
+	 */
+	if (info_p->cpuid_cores_per_package == 0) {
+		info_p->cpuid_cores_per_package = 1;
 
-static void set_cpu_amd( i386_cpu_info_t * info_p )
-{
-    set_cpu_generic(info_p);
-    set_amd_cache_info(info_p);
-    info_p->cpuid_model_string = get_amd_model_string(info_p, &info_p->cpuid_cpu_type, &info_p->cpuid_cpu_subtype);
-}
+		/* cpuid define in 1024 quantities */
+		info_p->cache_size[L2U] = info_p->cpuid_cache_size * 1024;
+		info_p->cache_sharing[L2U] = 1;
+		info_p->cache_partitions[L2U] = 1;
 
-static void set_cpu_nsc( i386_cpu_info_t * info_p )
-{
-    set_cpu_generic(info_p);
-    set_amd_cache_info(info_p);
-
-    /* check for brand id string */
-    if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX1) {
-        info_p->cpuid_model_string = "AMD Geode GX1";
-    } else if (info_p->cpuid_family == CPUID_FAMILY_586 && info_p->cpuid_model == CPUID_MODEL_GX2) {
-        info_p->cpuid_model_string = "AMD Geode GX";
-    } else {
-        info_p->cpuid_model_string = "Unknown National Semiconductor";
-    }
-    info_p->cpuid_cpu_type = CPU_TYPE_X86;
-    info_p->cpuid_cpu_subtype = CPU_SUBTYPE_X86_ARCH1;
+		linesizes[L2U] = info_p->cpuid_cache_linesize;
+	}
+	
+	/*
+	 * What linesize to publish?  We use the L2 linesize if any,
+	 * else the L1D.
+	 */
+	if ( linesizes[L2U] )
+		info_p->cache_linesize = linesizes[L2U];
+	else if (linesizes[L1D])
+		info_p->cache_linesize = linesizes[L1D];
+	else panic("no linesize");
 }
 
 static void
-set_cpu_generic(i386_cpu_info_t *info_p)
+cpuid_set_generic_info(i386_cpu_info_t *info_p)
 {
-	uint32_t	cpuid_result[4];
+	uint32_t	cpuid_reg[4];
 	uint32_t	max_extid;
         char            str[128], *p;
 
+	/* do cpuid 0 to get vendor */
+	do_cpuid(0, cpuid_reg);
+	bcopy((char *)&cpuid_reg[ebx], &info_p->cpuid_vendor[0], 4); /* ug */
+	bcopy((char *)&cpuid_reg[ecx], &info_p->cpuid_vendor[8], 4);
+	bcopy((char *)&cpuid_reg[edx], &info_p->cpuid_vendor[4], 4);
+	info_p->cpuid_vendor[12] = 0;
+
 	/* get extended cpuid results */
-	do_cpuid(0x80000000, cpuid_result);
-	max_extid = cpuid_result[eax];
+	do_cpuid(0x80000000, cpuid_reg);
+	max_extid = cpuid_reg[eax];
 
 	/* check to see if we can get brand string */
 	if (max_extid >= 0x80000004) {
@@ -576,57 +239,142 @@ set_cpu_generic(i386_cpu_info_t *info_p)
 		 * The brand string 48 bytes (max), guaranteed to
 		 * be NUL terminated.
 		 */
-		do_cpuid(0x80000002, cpuid_result);
-		bcopy((char *)cpuid_result, &str[0], 16);
-		do_cpuid(0x80000003, cpuid_result);
-		bcopy((char *)cpuid_result, &str[16], 16);
-		do_cpuid(0x80000004, cpuid_result);
-		bcopy((char *)cpuid_result, &str[32], 16);
+		do_cpuid(0x80000002, cpuid_reg);
+		bcopy((char *)cpuid_reg, &str[0], 16);
+		do_cpuid(0x80000003, cpuid_reg);
+		bcopy((char *)cpuid_reg, &str[16], 16);
+		do_cpuid(0x80000004, cpuid_reg);
+		bcopy((char *)cpuid_reg, &str[32], 16);
 		for (p = str; *p != '\0'; p++) {
 			if (*p != ' ') break;
 		}
-		strncpy(info_p->cpuid_brand_string,
-			p, sizeof(info_p->cpuid_brand_string)-1);
-		info_p->cpuid_brand_string[sizeof(info_p->cpuid_brand_string)-1] = '\0';
+		strlcpy(info_p->cpuid_brand_string,
+			p, sizeof(info_p->cpuid_brand_string));
 
-                if (!strcmp(info_p->cpuid_brand_string, CPUID_STRING_UNKNOWN)) {
+                if (!strncmp(info_p->cpuid_brand_string, CPUID_STRING_UNKNOWN,
+			     min(sizeof(info_p->cpuid_brand_string),
+				 strlen(CPUID_STRING_UNKNOWN) + 1))) {
                     /*
-                     * This string means we have a BIOS-programmable brand string,
-                     * and the BIOS couldn't figure out what sort of CPU we have.
+                     * This string means we have a firmware-programmable brand string,
+                     * and the firmware couldn't figure out what sort of CPU we have.
                      */
                     info_p->cpuid_brand_string[0] = '\0';
                 }
 	}
     
+	/* Get cache and addressing info. */
+	if (max_extid >= 0x80000006) {
+		do_cpuid(0x80000006, cpuid_reg);
+		info_p->cpuid_cache_linesize   = bitfield(cpuid_reg[ecx], 7, 0);
+		info_p->cpuid_cache_L2_associativity =
+						 bitfield(cpuid_reg[ecx],15,12);
+		info_p->cpuid_cache_size       = bitfield(cpuid_reg[ecx],31,16);
+		do_cpuid(0x80000008, cpuid_reg);
+		info_p->cpuid_address_bits_physical =
+						 bitfield(cpuid_reg[eax], 7, 0);
+		info_p->cpuid_address_bits_virtual =
+						 bitfield(cpuid_reg[eax],15, 8);
+	}
+
 	/* get processor signature and decode */
-	do_cpuid(1, cpuid_result);
-	info_p->cpuid_signature = cpuid_result[eax];
-	info_p->cpuid_stepping  = bitfield(cpuid_result[eax],  3,  0);
-	info_p->cpuid_model     = bitfield(cpuid_result[eax],  7,  4);
-	info_p->cpuid_family    = bitfield(cpuid_result[eax], 11,  8);
-	info_p->cpuid_type      = bitfield(cpuid_result[eax], 13, 12);
-	info_p->cpuid_extmodel  = bitfield(cpuid_result[eax], 19, 16);
-	info_p->cpuid_extfamily = bitfield(cpuid_result[eax], 27, 20);
-	info_p->cpuid_brand     = bitfield(cpuid_result[ebx],  7,  0);
-	info_p->cpuid_logical_per_package =
-				  bitfield(cpuid_result[ebx], 23, 16);
-	info_p->cpuid_features  = quad(cpuid_result[ecx], cpuid_result[edx]);
+	do_cpuid(1, cpuid_reg);
+	info_p->cpuid_signature = cpuid_reg[eax];
+	info_p->cpuid_stepping  = bitfield(cpuid_reg[eax],  3,  0);
+	info_p->cpuid_model     = bitfield(cpuid_reg[eax],  7,  4);
+	info_p->cpuid_family    = bitfield(cpuid_reg[eax], 11,  8);
+	info_p->cpuid_type      = bitfield(cpuid_reg[eax], 13, 12);
+	info_p->cpuid_extmodel  = bitfield(cpuid_reg[eax], 19, 16);
+	info_p->cpuid_extfamily = bitfield(cpuid_reg[eax], 27, 20);
+	info_p->cpuid_brand     = bitfield(cpuid_reg[ebx],  7,  0);
+	info_p->cpuid_features  = quad(cpuid_reg[ecx], cpuid_reg[edx]);
+
+	/* Fold extensions into family/model */
+	if (info_p->cpuid_family == 0x0f)
+		info_p->cpuid_family += info_p->cpuid_extfamily;
+	if (info_p->cpuid_family == 0x0f || info_p->cpuid_family== 0x06)
+		info_p->cpuid_model += (info_p->cpuid_extmodel << 4);
+
+	if (info_p->cpuid_features & CPUID_FEATURE_HTT)
+		info_p->cpuid_logical_per_package =
+				bitfield(cpuid_reg[ebx], 23, 16);
+	else
+		info_p->cpuid_logical_per_package = 1;
 
 	if (max_extid >= 0x80000001) {
-		do_cpuid(0x80000001, cpuid_result);
+		do_cpuid(0x80000001, cpuid_reg);
 		info_p->cpuid_extfeatures =
-				quad(cpuid_result[ecx], cpuid_result[edx]);
+				quad(cpuid_reg[ecx], cpuid_reg[edx]);
+	}
+
+	if (info_p->cpuid_extfeatures && CPUID_FEATURE_MONITOR) {
+		/*
+		 * Extract the Monitor/Mwait Leaf info:
+		 */
+		do_cpuid(5, cpuid_reg);
+		info_p->cpuid_mwait_linesize_min = cpuid_reg[eax];
+		info_p->cpuid_mwait_linesize_max = cpuid_reg[ebx];
+		info_p->cpuid_mwait_extensions   = cpuid_reg[ecx];
+		info_p->cpuid_mwait_sub_Cstates  = cpuid_reg[edx];
+
+		/*
+		 * And the thermal and Power Leaf while we're at it:
+		 */
+		do_cpuid(6, cpuid_reg);
+		info_p->cpuid_thermal_sensor =
+					bitfield(cpuid_reg[eax], 0, 0);
+		info_p->cpuid_thermal_dynamic_acceleration =
+					bitfield(cpuid_reg[eax], 1, 1);
+		info_p->cpuid_thermal_thresholds =
+					bitfield(cpuid_reg[ebx], 3, 0);
+		info_p->cpuid_thermal_ACNT_MCNT =
+					bitfield(cpuid_reg[ecx], 0, 0);
+
+		/*
+		 * And the Architectural Performance Monitoring Leaf:
+		 */
+		do_cpuid(0xa, cpuid_reg);
+		info_p->cpuid_arch_perf_version =
+					bitfield(cpuid_reg[eax], 7, 0);
+		info_p->cpuid_arch_perf_number =
+					bitfield(cpuid_reg[eax],15, 8);
+		info_p->cpuid_arch_perf_width =
+					bitfield(cpuid_reg[eax],23,16);
+		info_p->cpuid_arch_perf_events_number =
+					bitfield(cpuid_reg[eax],31,24);
+		info_p->cpuid_arch_perf_events =
+					cpuid_reg[ebx];
+		info_p->cpuid_arch_perf_fixed_number =
+					bitfield(cpuid_reg[edx], 4, 0);
+		info_p->cpuid_arch_perf_fixed_width =
+					bitfield(cpuid_reg[edx],12, 5);
+
 	}
 
 	return;
 }
 
-static void
-set_cpu_unknown(__unused i386_cpu_info_t *info_p)
+void
+cpuid_set_info(void)
 {
-    info_p->cpuid_model_string = "Unknown";
-}
+	bzero((void *)&cpuid_cpu_info, sizeof(cpuid_cpu_info));
+
+	cpuid_set_generic_info(&cpuid_cpu_info);
 
+	/* verify we are running on a supported CPU */
+	if ((strncmp(CPUID_VID_INTEL, cpuid_cpu_info.cpuid_vendor,
+		     min(strlen(CPUID_STRING_UNKNOWN) + 1,
+			 sizeof(cpuid_cpu_info.cpuid_vendor)))) ||
+	   (cpuid_cpu_info.cpuid_family != 6) ||
+	   (cpuid_cpu_info.cpuid_model < 13))
+		panic("Unsupported CPU");
+
+	cpuid_cpu_info.cpuid_cpu_type = CPU_TYPE_X86;
+	cpuid_cpu_info.cpuid_cpu_subtype = CPU_SUBTYPE_X86_ARCH1;
+
+	cpuid_set_cache_info(&cpuid_cpu_info);
+
+	cpuid_cpu_info.cpuid_model_string = ""; /* deprecated */
+}
 
 static struct {
 	uint64_t	mask;
@@ -667,12 +415,11 @@ static struct {
 	{CPUID_FEATURE_SMX,     "SMX"},
 	{CPUID_FEATURE_EST,     "EST"},
 	{CPUID_FEATURE_TM2,     "TM2"},
-	{CPUID_FEATURE_MNI,     "MNI"},
+	{CPUID_FEATURE_SSSE3,   "SSSE3"},
 	{CPUID_FEATURE_CID,     "CID"},
 	{CPUID_FEATURE_CX16,    "CX16"},
 	{CPUID_FEATURE_xTPR,    "TPR"},
 	{CPUID_FEATURE_PDCM,    "PDCM"},
-	{CPUID_FEATURE_DCA,     "DCA"},
 	{CPUID_FEATURE_SSE4_1,  "SSE4.1"},
 	{CPUID_FEATURE_SSE4_2,  "SSE4.2"},
 	{CPUID_FEATURE_POPCNT,  "POPCNT"},
@@ -691,7 +438,7 @@ cpuid_info(void)
 {
 	/* Set-up the cpuid_indo stucture lazily */
 	if (cpuid_cpu_infop == NULL) {
-		cpuid_get_info(&cpuid_cpu_info);
+		cpuid_set_info();
 		cpuid_cpu_infop = &cpuid_cpu_info;
 	}
 	return cpuid_cpu_infop;
@@ -741,6 +488,26 @@ cpuid_get_extfeature_names(uint64_t extfeatures, char *buf, unsigned buf_len)
 	return buf;
 }
 
+
+#if CONFIG_NO_KPRINTF_STRINGS
+void
+cpuid_feature_display(
+	__unused const char	*header)
+{
+}
+
+void
+cpuid_extfeature_display(
+	__unused const char	*header)
+{
+}
+
+void
+cpuid_cpu_display(
+	__unused const char	*header)
+{
+}
+#else /* CONFIG_NO_KPRINTF_STRINGS */
 void
 cpuid_feature_display(
 	const char	*header)
@@ -776,10 +543,11 @@ void
 cpuid_cpu_display(
 	const char	*header)
 {
-    if (cpuid_info()->cpuid_brand_string[0] != '\0') {
+    if (cpuid_cpu_info.cpuid_brand_string[0] != '\0') {
 	kprintf("%s: %s\n", header, cpuid_cpu_info.cpuid_brand_string);
     }
 }
+#endif /* !CONFIG_NO_KPRINTF_STRINGS */
 
 unsigned int
 cpuid_family(void)
@@ -810,10 +578,10 @@ cpuid_features(void)
 		    /* check for boot-time fpu limitations */
 			if (PE_parse_boot_arg("_fpu", &fpu_arg[0])) {
 				printf("limiting fpu features to: %s\n", fpu_arg);
-				if (!strncmp("387", fpu_arg, sizeof "387") || !strncmp("mmx", fpu_arg, sizeof "mmx")) {
+				if (!strncmp("387", fpu_arg, sizeof("387")) || !strncmp("mmx", fpu_arg, sizeof("mmx"))) {
 					printf("no sse or sse2\n");
 					cpuid_cpu_info.cpuid_features &= ~(CPUID_FEATURE_SSE | CPUID_FEATURE_SSE2 | CPUID_FEATURE_FXSR);
-				} else if (!strncmp("sse", fpu_arg, sizeof "sse")) {
+				} else if (!strncmp("sse", fpu_arg, sizeof("sse"))) {
 					printf("no sse2\n");
 					cpuid_cpu_info.cpuid_features &= ~(CPUID_FEATURE_SSE2);
 				}
@@ -829,11 +597,6 @@ cpuid_extfeatures(void)
 	return cpuid_info()->cpuid_extfeatures;
 }
  
-void
-cpuid_set_info(void)
-{
-	cpuid_get_info(&cpuid_cpu_info);
-}
 
 #if MACH_KDB
 
diff --git a/osfmk/i386/cpuid.h b/osfmk/i386/cpuid.h
index acaa81cd1..efc6bb239 100644
--- a/osfmk/i386/cpuid.h
+++ b/osfmk/i386/cpuid.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -35,17 +41,8 @@
 
 #ifdef __APPLE_API_PRIVATE
 
-#define	CPUID_VID_SIZE		12
 #define	CPUID_VID_INTEL		"GenuineIntel"
-#define	CPUID_VID_UMC		"UMC UMC UMC "
 #define	CPUID_VID_AMD		"AuthenticAMD"
-#define	CPUID_VID_CYRIX		"CyrixInstead"
-#define	CPUID_VID_NEXGEN	"NexGenDriven"
-#define CPUID_VID_CENTAUR       "CentaurHauls"
-#define CPUID_VID_RISE          "RiseRiseRise"
-#define CPUID_VID_SIS           "SiS SiS SiS "
-#define CPUID_VID_TRANSMETA     "GenuineTMx86"
-#define CPUID_VID_NSC           "Geode by NSC"
 
 #define CPUID_STRING_UNKNOWN    "Unknown CPU Typ"
 
@@ -86,7 +83,7 @@
 #define CPUID_FEATURE_TM      _Bit(29)	/* Thermal Monitor (TM1) */
 #define CPUID_FEATURE_PBE     _Bit(31)	/* Pend Break Enable */
 
-#define CPUID_FEATURE_SSE3    _HBit(0)	/* Prescott New Inst. */
+#define CPUID_FEATURE_SSE3    _HBit(0)	/* Streaming SIMD extensions 3 */
 #define CPUID_FEATURE_MONITOR _HBit(3)	/* Monitor/mwait */
 #define CPUID_FEATURE_DSCPL   _HBit(4)	/* Debug Store CPL */
 #define CPUID_FEATURE_VMX     _HBit(5)	/* VMX */
@@ -94,14 +91,12 @@
 #define CPUID_FEATURE_EST     _HBit(7)	/* Enhanced SpeedsTep (GV3) */
 #define CPUID_FEATURE_TM2     _HBit(8)	/* Thermal Monitor 2 */
 #define CPUID_FEATURE_SSSE3   _HBit(9)	/* Supplemental SSE3 instructions */
-#define CPUID_FEATURE_MNI     CPUID_FEATURE_SSSE3
 #define CPUID_FEATURE_CID     _HBit(10)	/* L1 Context ID */
 #define CPUID_FEATURE_CX16    _HBit(13)	/* CmpXchg16b instruction */
 #define CPUID_FEATURE_xTPR    _HBit(14)	/* Send Task PRiority msgs */
 #define CPUID_FEATURE_PDCM    _HBit(15)	/* Perf/Debug Capability MSR */
-#define CPUID_FEATURE_DCA     _HBit(18)	/* Direct Cache Control */
 #define CPUID_FEATURE_SSE4_1  _HBit(19)	/* Streaming SIMD extensions 4.1 */
-#define CPUID_FEATURE_SSE4_2  _HBit(20)	/* Streaming SIMD extensions 4.2 */
+#define CPUID_FEATURE_SSE4_2  _HBit(20)	/* Streaming SIMD extensions 4.1 */
 #define CPUID_FEATURE_POPCNT  _HBit(23)	/* POPCNT instruction */
 
 /*
@@ -114,145 +109,6 @@
 
 #define CPUID_EXTFEATURE_LAHF	   _HBit(20)	/* LAFH/SAHF instructions */
 
-
-#define	CPUID_TYPE_OEM		    0x0	/* Original processor */
-#define	CPUID_TYPE_OVERDRIVE	    0x1	/* Overdrive processor */
-#define	CPUID_TYPE_DUAL		    0x2	/* Can be used as dual processor */
-#define	CPUID_TYPE_RESERVED	    0x3	/* Reserved */
-
-#define	CPUID_FAMILY_386	    0x3	/* Intel 386 (not part of CPUID) */
-
-#define	CPUID_MODEL_I386_DX	    0x0	/* Intel 386 (not part of CPUID) */
-
-#define	CPUID_FAMILY_486	    0x4	/* Intel 486 */
-
-#define	CPUID_MODEL_I486_DX	    0x0	/* Intel 486DX */
-#define	CPUID_MODEL_I486_DX_S	    0x1	/* Intel 486DX-S */
-#define	CPUID_MODEL_I486_SX	    0x2	/* Intel 486SX */
-#define	CPUID_MODEL_I486_DX2	    0x3	/* Intel 486DX2 */
-#define	CPUID_MODEL_I486_SL	    0x4	/* Intel 486SL */
-#define	CPUID_MODEL_I486_SX2	    0x5	/* Intel 486SX2 */
-#define	CPUID_MODEL_I486_DX2WB	    0x7	/* Intel 486DX2WB */
-#define	CPUID_MODEL_I486_DX4	    0x8	/* Intel 486DX4 */
-#define	CPUID_MODEL_I486_DX4WB	    0x9	/* Intel 486DX4WB */
-
-#define	CPUID_MODEL_AM486_DX	    0x1	/* AMD 486DX */
-#define	CPUID_MODEL_AM486_DX2	    0x3	/* AMD 486DX2 */
-#define	CPUID_MODEL_AM486_DX2WB	    0x7	/* AMD 486DX2WB */
-#define	CPUID_MODEL_AM486_DX4	    0x8	/* AMD 486DX4 */
-#define	CPUID_MODEL_AM486_DX4WB	    0x9	/* AMD 486DX4WB */
-#define	CPUID_MODEL_AM486_5X86	    0xE	/* AMD 5x86 */
-#define	CPUID_MODEL_AM486_5X86WB    0xF	/* AMD 5x86WB */
-
-#define CPUID_MODEL_MEDIAGX         0x4 /* Cyrix MediaGX */
-#define	CPUID_MODEL_CYRIX5X86	    0x9	/* CYRIX 5X86 */
-
-#define	CPUID_FAMILY_586            0x5	/* Intel Pentium, AMD K5/K6*/
-
-#define	CPUID_MODEL_UMC5D	    0x1	/* UMC U5D */
-#define	CPUID_MODEL_UMC5S	    0x2	/* UMC U5S */
-#define	CPUID_MODEL_UMC486_DX2	    0x3	/* UMC U486_DX2 */
-#define	CPUID_MODEL_UMC486_SX2	    0x5	/* UMC U486_SX2 */
-
-#define	CPUID_MODEL_P5A		    0x0	/* Intel P5 60/66 Step A */
-#define	CPUID_MODEL_P5		    0x1	/* Intel P5 60/66 */
-#define	CPUID_MODEL_P54		    0x2	/* Intel P5 75/80/100/120/133/166 */
-#define	CPUID_MODEL_P24T	    0x3	/* Intel P5 Overdrive 63/83 */
-
-#define CPUID_MODEL_K5M0            0x0 /* AMD-K5 Model 0 */
-#define CPUID_MODEL_K5M1            0x1 /* AMD-K5 Model 1 */
-#define CPUID_MODEL_K5M2            0x2 /* AMD-K5 Model 2 */
-#define CPUID_MODEL_K5M3            0x3 /* AMD-K5 Model 3 */
-#define CPUID_MODEL_K6M6            0x6 /* AMD-K6 Model 6 */
-#define CPUID_MODEL_K6M7            0x7 /* AMD-K6 Model 7 */
-#define CPUID_MODEL_K6_2            0x8 /* AMD-K6-2 Model 8 */
-#define CPUID_MODEL_K6_III          0x9 /* AMD-K6-III Model 9 */
-
-#define CPUID_MODEL_CYRIX_M1              0x2 /* Cyrix M1 */
-#define CPUID_MODEL_MEDIAGX_MMX           0x4 /* Cyrix MediaGX MMX Enhanced */
-
-#define	CPUID_FAMILY_686            0x6	/* Intel Pentium Pro, II, III; AMD Athlon */
-
-#define	CPUID_MODEL_P6		    0x1	/* Intel P6 */
-#define	CPUID_MODEL_PII		    0x3	/* Intel PII */
-#define CPUID_MODEL_P65             0x5 /* Intel PII/Xeon/Celeron model 5 */
-#define CPUID_MODEL_P66             0x6 /* Intel Celeron model 6 */
-#define CPUID_MODEL_P67             0x7 /* Intel PIII/Xeon model 7 */
-#define CPUID_MODEL_P68             0x8 /* Intel PIII/Xeon/Celeron model 8 */
-#define CPUID_MODEL_PM9             0x9 /* Intel Pentium M model 9 */
-#define CPUID_MODEL_P6A             0xA /* Intel PIII Xeon model A */
-#define CPUID_MODEL_P6B             0xB /* Intel PIII model B */
-#define CPUID_MODEL_PMD             0xD /* Intel Pentium M model D */
-#define CPUID_MODEL_CORE            0xE /* Intel Core Solo & Duo */
-#define CPUID_MODEL_YONAH           CPUID_MODEL_YONAH
-#define CPUID_MODEL_CORE2	    0xF /* Intel Core2 Duo */
-#define CPUID_MODEL_MEROM           CPUID_MODEL_CORE2
-#define CPUID_MODEL_PENRYN          0x17
-
-#define CPUID_MODEL_ATHLON_M1       0x1 /* AMD Athlon Model 1 */
-#define CPUID_MODEL_ATHLON_M2       0x2 /* AMD Athlon Model 2 */
-#define CPUID_MODEL_DURON_M3        0x3 /* AMD Duron Model 3 */
-#define CPUID_MODEL_ATHLON_M4       0x4 /* AMD Athlon Model 4 */
-#define CPUID_MODEL_ATHLON_M6       0x6 /* (Mobile) AMD Athlon/Duron MP/XP/4 Model 6 */
-#define CPUID_MODEL_DURON_M7        0x7 /* (Mobile) AMD Duron Model 7 */
-#define CPUID_MODEL_ATHLON_M8       0x8 /* (Mobile) Athlon XP/MP/XP-M Model 8 */
-#define CPUID_MODEL_ATHLON_M10      0xA /* (Mobile) AMD Athlon XP/MP/XP-M/XP-M(LV) Model 10 */
-
-#define CPUID_MODEL_CYRIX_M2        0x0 /* Cyrix M2 */
-#define CPUID_MODEL_CYRIX_MII       0x2 /* VIA Cyrix MII (6x86MX) */
-#define CPUID_MODEL_VIA_CYRIX_M2    0x5 /* VIA C3 Cyrix M2 */
-#define CPUID_MODEL_WINCHIP_C5A     0x6 /* VIA C3 WinChip C5A */
-#define CPUID_MODEL_WINCHIP_C5BC    0x7 /* VIA C3 WinChip C5B/C5C */
-#define CPUID_MODEL_WINCHIP_C5N     0x8 /* VIA C3 WinChip C5N */
-#define CPUID_MODEL_WINCHIP_C5XLP   0x9 /* VIA C3 WinChip C5P */
-
-#define CPUID_MODEL_NX586           0x0 /* NexGen Nx586 */
-
-#define CPUID_MODEL_RISE_MP6_0      0x0 /* Rise mP6 */
-#define CPUID_MODEL_RISE_MP6_2      0x2 /* Rise mP6 */
-
-#define CPUID_MODEL_SIS_55X         0x0 /* SIS 55x */
-
-#define CPUID_MODEL_TM_CRUSOE       0x4 /* Transmeta Crusoe TM3x00 and TM5x00 */
-
-#define CPUID_MODEL_CENTAUR_C6            0x4 /* Centaur C6 */
-#define CPUID_MODEL_CENTAUR_C2            0x8 /* Centaur C2 */
-#define CPUID_MODEL_CENTAUR_C3            0x9 /* Centaur C3 */
-
-#define CPUID_MODEL_GX1               0x4 /* AMD Geode GX1 */
-#define CPUID_MODEL_GX2               0x5 /* AMD Geode GX */
-
-#define	CPUID_FAMILY_ITANIUM        0x7	/* Intel Intanium */
-#define CPUID_FAMILY_EXTENDED       0xF /* Intel Pentium 4, Itanium II */
-
-#define CPUID_EXTFAMILY_PENTIUM4    0x0 /* Intel Pentium 4 */
-#define CPUID_EXTFAMILY_ITANIUM2    0x1 /* Intel Itanium 2 */
-
-#define CPUID_MODEL_ATHLON64        0x4 /* AMD Athlon 64 Model 4 */
-#define CPUID_MODEL_OPTERON         0x5 /* AMD Opteron Model 4 */
-
-#define CPUID_BRAND_UNSUPPORTED     0x00
-#define CPUID_BRAND_CELERON_1       0x01 /* Intel Celeron */
-#define CPUID_BRAND_PENTIUM_III_2   0x02 /* Intel Pentium III */
-#define CPUID_BRAND_PIII_XEON       0x03 /* Intel Pentium III Xeon / Celeron */
-#define CPUID_BRAND_PENTIUM_III_4   0x04 /* Intel Pentium III */
-#define CPUID_BRAND_PENTIUM_III_M   0x05 /* Mobile Intel Pentium III-M */
-#define CPUID_BRAND_M_CELERON_7     0x07 /* Mobile Intel Celeron */
-#define CPUID_BRAND_PENTIUM4_8      0x08 /* Intel Pentium 4 */
-#define CPUID_BRAND_PENTIUM4_9      0x09 /* Intel Pentium 4 */
-#define CPUID_BRAND_CELERON_A       0x0A /* Intel Celeron */
-#define CPUID_BRAND_XEON            0x0B /* Intel Xeon (MP) */
-#define CPUID_BRAND_XEON_MP         0x0C /* Intel Xeon MP */
-#define CPUID_BRAND_PENTIUM4_M      0x0E /* Mobile Intel Pentium 4-M / Xeon */
-#define CPUID_BRAND_M_CELERON_F     0x0F /* Mobile Intel Celeron */
-#define CPUID_BRAND_MOBILE_17       0x11 /* Mobile Genuine Intel */
-#define CPUID_BRAND_CELERON_M       0x12 /* Intel Celeron M */
-#define CPUID_BRAND_M_CELERON_13    0x13 /* Mobile Intel Celeron */
-#define CPUID_BRAND_CELERON_14      0x14 /* Intel Celeron */
-#define CPUID_BRAND_MOBILE_15       0x15 /* Mobile Genuine Intel */
-#define CPUID_BRAND_PENTIUM_M       0x16 /* Intel Pentium M */
-#define CPUID_BRAND_M_CELERON_17    0x17 /* Mobile Intel Celeron */
-
 #define	CPUID_CACHE_SIZE	16	/* Number of descriptor vales */
 
 #define	CPUID_CACHE_NULL	   0x00	/* NULL */
@@ -281,6 +137,7 @@
 #define	CPUID_CACHE_L2_2M_4	   0x45	/* L2: 2M, 4-way */
 #define CPUID_CACHE_L3_4M_4_64     0x46 /* L3:  4M,  4-way, 64 bytes */
 #define CPUID_CACHE_L3_8M_8_64     0x47 /* L3:  8M,  8-way, 64 bytes*/
+#define CPUID_CACHE_L2_3M_12_64    0x48 /* L3:  3M,  8-way, 64 bytes*/
 #define CPUID_CACHE_L2_4M_16_64    0x49 /* L2:  4M, 16-way, 64 bytes */
 #define CPUID_CACHE_L2_6M_12_64    0x4A /* L2:  6M, 12-way, 64 bytes */
 #define CPUID_CACHE_L2_8M_16_64    0x4B /* L2:  8M, 16-way, 64 bytes */
@@ -290,6 +147,8 @@
 #define CPUID_CACHE_ITLB_64        0x50 /* Instruction TLB: 64 entries */
 #define CPUID_CACHE_ITLB_128       0x51 /* Instruction TLB: 128 entries */
 #define CPUID_CACHE_ITLB_256       0x52 /* Instruction TLB: 256 entries */
+#define CPUID_CACHE_DTLB_4M_16_4   0x56 /* Data TLB: 4M, 16 entries, 4-way */
+#define CPUID_CACHE_DTLB_4K_16_4   0x56 /* Data TLB: 4K, 16 entries, 4-way */
 #define CPUID_CACHE_DTLB_64        0x5B /* Data TLB: 64 entries */
 #define CPUID_CACHE_DTLB_128       0x5C /* Data TLB: 128 entries */
 #define CPUID_CACHE_DTLB_256       0x5D /* Data TLB: 256 entries */
@@ -317,10 +176,13 @@
 #define CPUID_CACHE_ITLB_4M_4_4    0xB1 /* ITLB: 4MB,   4 entries, 4-way, or  */
 #define CPUID_CACHE_ITLB_2M_8_4    0xB1 /* ITLB: 2MB,   8 entries, 4-way */
 #define CPUID_CACHE_DTLB_4K_128_4  0xB3 /* DTLB: 4KB, 128 entries, 4-way */
-#define CPUID_CACHE_DTLB_4K_256_4  0xB3 /* DTLB: 4KB, 256 entries, 4-way */
+#define CPUID_CACHE_DTLB_4K_256_4  0xB4 /* DTLB: 4KB, 256 entries, 4-way */
 #define CPUID_CACHE_PREFETCH_64    0xF0 /* 64-Byte Prefetching */
 #define CPUID_CACHE_PREFETCH_128   0xF1 /* 128-Byte Prefetching */
 
+#define CPUID_MWAIT_EXTENSION	_Bit(0)	/* enumeration of WMAIT extensions */
+#define CPUID_MWAIT_BREAK	_Bit(1)	/* interrupts are break events	   */
+
 #ifndef ASSEMBLER
 #include <stdint.h>
 #include <mach/mach_types.h>
@@ -354,8 +216,8 @@ do_cpuid(uint32_t selector, uint32_t *data)
 }
 
 /*
- * Cache ID descriptor structure.
- * Note: description string absent in kernel.
+ * Cache ID descriptor structure, used to parse CPUID leaf 2.
+ * Note: not used in kernel.
  */
 typedef enum { Lnone, L1I, L1D, L2U, L3U, LCACHE_MAX } cache_type_t ; 
 typedef struct {
@@ -401,9 +263,40 @@ typedef struct {
 	uint32_t	cpuid_cores_per_package;
 	uint32_t	cpuid_logical_per_package;
 	uint32_t	cache_sharing[LCACHE_MAX];
+	uint32_t	cache_partitions[LCACHE_MAX];
 
-	cpu_type_t		cpuid_cpu_type;			/* <mach/machine.h> */
+	cpu_type_t	cpuid_cpu_type;			/* <mach/machine.h> */
 	cpu_subtype_t	cpuid_cpu_subtype;		/* <mach/machine.h> */	
+
+	/* Monitor/mwait Leaf: */
+	uint32_t	cpuid_mwait_linesize_min;
+	uint32_t	cpuid_mwait_linesize_max;
+	uint32_t	cpuid_mwait_extensions;
+	uint32_t	cpuid_mwait_sub_Cstates;
+
+	/* Thermal and Power Management Leaf: */
+	boolean_t	cpuid_thermal_sensor;
+	boolean_t	cpuid_thermal_dynamic_acceleration;
+	uint32_t	cpuid_thermal_thresholds;
+	boolean_t	cpuid_thermal_ACNT_MCNT;
+
+	/* Architectural Performance Monitoring Leaf: */
+	uint8_t		cpuid_arch_perf_version;
+	uint8_t		cpuid_arch_perf_number;
+	uint8_t		cpuid_arch_perf_width;
+	uint8_t		cpuid_arch_perf_events_number;
+	uint32_t	cpuid_arch_perf_events;
+	uint8_t		cpuid_arch_perf_fixed_number;
+	uint8_t		cpuid_arch_perf_fixed_width;
+	
+	/* Cache details: */
+	uint32_t	cpuid_cache_linesize;
+	uint32_t	cpuid_cache_L2_associativity;
+	uint32_t	cpuid_cache_size;
+
+	/* Virtual and physical address aize: */
+	uint32_t	cpuid_address_bits_physical;
+	uint32_t	cpuid_address_bits_virtual;
 } i386_cpu_info_t;
 
 #ifdef __cplusplus
diff --git a/osfmk/i386/cswitch.s b/osfmk/i386/cswitch.s
index 31ca053b8..0668c465e 100644
--- a/osfmk/i386/cswitch.s
+++ b/osfmk/i386/cswitch.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/db_disasm.c b/osfmk/i386/db_disasm.c
index 67e73801f..c68d65237 100644
--- a/osfmk/i386/db_disasm.c
+++ b/osfmk/i386/db_disasm.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/db_gcc_aout.c b/osfmk/i386/db_gcc_aout.c
index 15fd0003d..93ddcb531 100644
--- a/osfmk/i386/db_gcc_aout.c
+++ b/osfmk/i386/db_gcc_aout.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * COPYRIGHT NOTICE
diff --git a/osfmk/i386/db_interface.c b/osfmk/i386/db_interface.c
index 55cf5aa62..7390ddf04 100644
--- a/osfmk/i386/db_interface.c
+++ b/osfmk/i386/db_interface.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -255,7 +261,9 @@ kdb_trap(
 	    {
 	    	extern int dr_addr[];
 		int addr;
-	    	int status = dr6();
+	    	uint32_t status;
+
+		__asm__ volatile ("movl %%dr6, %0" : "=r" (status));
 
 		if (status & 0xf) {	/* hmm hdw break */
 			addr =	status & 0x8 ? dr_addr[3] :
diff --git a/osfmk/i386/db_machdep.h b/osfmk/i386/db_machdep.h
index dbcf9eb43..fda25c022 100644
--- a/osfmk/i386/db_machdep.h
+++ b/osfmk/i386/db_machdep.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -154,18 +160,21 @@ extern void		db_reboot(
 				db_expr_t	count,
 				char		*modif);
 
-extern void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_display_real(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_display_iokit(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_cpuid(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_msr(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_apic(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_test(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_intcnt(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_display_hpet(hpetReg_t *hpt);
-extern void db_hpet(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_cfg(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-extern void db_dtimers(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+extern void db_display_kmod(db_expr_t addr, boolean_t have_addr,
+			    db_expr_t count, char *modif);
+extern void db_display_real(db_expr_t addr, boolean_t have_addr,
+			    db_expr_t count, char *modif);
+extern void db_display_iokit(db_expr_t addr, boolean_t have_addr,
+			     db_expr_t count, char * modif);
+extern void db_cpuid(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		     char *modif);
+extern void db_msr(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		   char *modif);
+extern void db_apic(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		    char *modif);
+extern void db_display_hpet(hpetReg_t *);
+extern void db_hpet(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		    char *modif);
 
 /* macros for printing OS server dependent task name */
 
@@ -190,7 +199,6 @@ extern int		kdb_trap(
 				x86_saved_state32_t	*regs);
 extern boolean_t	db_trap_from_asm(
 				x86_saved_state32_t *regs);
-extern int		dr6(void);
 extern void		kdb_on(
 				int			cpu);
 
diff --git a/osfmk/i386/db_trace.c b/osfmk/i386/db_trace.c
index 49d6a465e..b828b615c 100644
--- a/osfmk/i386/db_trace.c
+++ b/osfmk/i386/db_trace.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -339,6 +345,9 @@ struct interrupt_frame {
 	int		  if_efl;	/* saved efl(iret_i) */
 };
 
+extern const char *trap_type[];
+extern int	TRAP_TYPES;
+
 /* 
  * Figure out the next frame up in the call stack.  
  * For trap(), we print the address of the faulting instruction and 
@@ -357,66 +366,65 @@ db_nextframe(
 	int			frame_type,	/* in */
 	thread_t		thr_act)	/* in */
 {
-        x86_saved_state32_t	*iss32;
-	extern char *	trap_type[];
-	extern int	TRAP_TYPES;
-
+	x86_saved_state32_t	*iss32;
 	struct interrupt_frame *ifp;
 	task_t task = (thr_act != THREAD_NULL)? thr_act->task: TASK_NULL;
 
 	switch(frame_type) {
 	case TRAP:
-	    /*
-	     * We know that trap() has 1 argument and we know that
-	     * it is an (x86_saved_state32_t *).
-	     */
-	    iss32 = (x86_saved_state32_t *)
-	                 db_get_task_value((int)&((*fp)->f_arg0),4,FALSE,task);
-	    
-	    if (iss32->trapno >= 0 && iss32->trapno < TRAP_TYPES) {
-	            db_printf(">>>>> %s trap at ",
-			      trap_type[iss32->trapno]);
-	    } else {
-	            db_printf(">>>>> trap (number %d) at ",
-			      iss32->trapno & 0xffff);
-	    }
-	    db_task_printsym(iss32->eip, DB_STGY_PROC, task);
-	    db_printf(" <<<<<\n");
-	    *fp = (struct i386_frame *)iss32->ebp;
-	    *ip = (db_addr_t)iss32->eip;
-	    break;
+		/*
+		 * We know that trap() has 1 argument and we know that
+		 * it is an (strcut x86_saved_state32_t *).
+		 */
+		iss32 = (x86_saved_state32_t *)
+			db_get_task_value((int)&((*fp)->f_arg0),4,FALSE,task);
+
+		if (iss32->trapno >= 0 && iss32->trapno < TRAP_TYPES) {
+			db_printf(">>>>> %s trap at ",
+					trap_type[iss32->trapno]);
+		} else {
+			db_printf(">>>>> trap (number %d) at ",
+					iss32->trapno & 0xffff);
+		}
+		db_task_printsym(iss32->eip, DB_STGY_PROC, task);
+		db_printf(" <<<<<\n");
+		*fp = (struct i386_frame *)iss32->ebp;
+		*ip = (db_addr_t)iss32->eip;
+		break;
+
 	case INTERRUPT:
-	    if (*lfp == 0) {
-		db_printf(">>>>> interrupt <<<<<\n");
-		goto miss_frame;
-	    }
-	    db_printf(">>>>> interrupt at "); 
-	    ifp = (struct interrupt_frame *)(*lfp);
-	    *fp = ifp->if_frame;
-	    if (ifp->if_iretaddr == db_return_to_iret_symbol_value) {
-	            *ip = ((x86_saved_state32_t *) ifp->if_edx)->eip;
-	    } else
-	            *ip = (db_addr_t) ifp->if_eip;
-	    db_task_printsym(*ip, DB_STGY_PROC, task);
-	    db_printf(" <<<<<\n");
-	    break;
+		if (*lfp == 0) {
+			db_printf(">>>>> interrupt <<<<<\n");
+			goto miss_frame;
+		}
+		db_printf(">>>>> interrupt at "); 
+		ifp = (struct interrupt_frame *)(*lfp);
+		*fp = ifp->if_frame;
+		if (ifp->if_iretaddr == db_return_to_iret_symbol_value) {
+			*ip = ((x86_saved_state32_t *)ifp->if_edx)->eip;
+		} else
+			*ip = (db_addr_t)ifp->if_eip;
+		db_task_printsym(*ip, DB_STGY_PROC, task);
+		db_printf(" <<<<<\n");
+		break;
+
 	case SYSCALL:
-	    if (thr_act != THREAD_NULL && thr_act->machine.pcb) {
-		    iss32 = (x86_saved_state32_t *)thr_act->machine.pcb->iss;
+		if (thr_act != THREAD_NULL && thr_act->machine.pcb) {
+			iss32 = (x86_saved_state32_t *)thr_act->machine.pcb->iss;
 
-		    *ip = (db_addr_t)(iss32->eip);
-		    *fp = (struct i386_frame *)(iss32->ebp);
-	    }
-	    break;
-	    /* falling down for unknown case */
-	default:
-	miss_frame:
-	    *ip = (db_addr_t)
-		db_get_task_value((int)&(*fp)->f_retaddr, 4, FALSE, task);
-	    *lfp = *fp;
-	    *fp = (struct i386_frame *)
-		db_get_task_value((int)&(*fp)->f_frame, 4, FALSE, task);
-	    break;
+			*ip = (db_addr_t)(iss32->eip);
+			*fp = (struct i386_frame *)(iss32->ebp);
+		}
+		break;
+
+	default:	/* falling down for unknown case */
+miss_frame:
+		*ip = (db_addr_t)
+			db_get_task_value((int)&(*fp)->f_retaddr, 4, FALSE, task);
+		*lfp = *fp;
+		*fp = (struct i386_frame *)
+			db_get_task_value((int)&(*fp)->f_frame, 4, FALSE, task);
+		break;
 	}
 }
 
@@ -501,7 +509,7 @@ db_stack_trace_cmd(
 	if (count == -1)
 	    count = 65535;
 
-    next_thread:
+next_thread:
 	top_act = THREAD_NULL;
 
 	user_frame = 0;
@@ -559,8 +567,8 @@ db_stack_trace_cmd(
 
 		    iss32 = (x86_saved_state32_t *)th->machine.pcb->iss;
 
-		    frame = (struct i386_frame *) (iss32->ebp);
-		    callpc = (db_addr_t) (iss32->eip);
+			frame = (struct i386_frame *) (iss32->ebp);
+			callpc = (db_addr_t) (iss32->eip);
 
 		} else {
 		    int cpu;
@@ -580,8 +588,8 @@ db_stack_trace_cmd(
 			     */
 		            iss32 = (x86_saved_state32_t *)th->machine.pcb->iss;
 
-			    frame = (struct i386_frame *) (iss32->ebp);
-			    callpc = (db_addr_t) (iss32->eip);
+				    frame = (struct i386_frame *) (iss32->ebp);
+				    callpc = (db_addr_t) (iss32->eip);
 		    } else {
 			    if (cpu == real_ncpus) {
 			    register struct x86_kernel_state32 *iks;
@@ -612,12 +620,12 @@ db_stack_trace_cmd(
 
 			    iss32 = (x86_saved_state32_t *)cpu_datap(cpu)->cpu_kdb_saved_state;
 
-			    frame = (struct i386_frame *) (iss32->ebp);
-			    callpc = (db_addr_t) (iss32->eip);
+				    frame = (struct i386_frame *) (iss32->ebp);
+				    callpc = (db_addr_t) (iss32->eip);
+			    }
 			}
 		    }
 	        }
-	    }
 	} else {
 	    frame = (struct i386_frame *)addr;
 	    th = (db_default_act)? db_default_act: current_thread();
@@ -752,7 +760,7 @@ db_stack_trace_cmd(
 	    }
 	    db_printf("\n");
 
-	next_frame:
+next_frame:
 	    lastcallpc = callpc;
 	    db_nextframe(&lastframe, &frame, &callpc, frame_type,
 			 (user_frame) ? th : THREAD_NULL);
@@ -788,7 +796,7 @@ db_stack_trace_cmd(
 	    }
 	}
 
-    thread_done:
+thread_done:
 	if (trace_all_threads) {
 	    if (top_act != THREAD_NULL)
 		th = top_act;
@@ -810,15 +818,18 @@ extern boolean_t kdp_trans_off;
  *		
  *		dr [entaddr]
  */
-void db_display_real(db_expr_t addr, __unused int have_addr,  __unused db_expr_t count,  __unused char * modif) {
-
+void
+db_display_real(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	int				i;
 	unsigned int xbuf[8];
 	unsigned read_result = 0;
 /* Print 256 bytes */
 	for(i=0; i<8; i++) {
 
-/* Do a physical read using kdp_vm_read(), rather than replicating the same
+/*
+ * Do a physical read using kdp_vm_read(), rather than replicating the same
  * facility
  */
 		kdp_trans_off = 1;
@@ -839,10 +850,11 @@ void db_display_real(db_expr_t addr, __unused int have_addr,  __unused db_expr_t
 /*
  *	Displays all of the kmods in the system.
  *
-  *	dk
+ *		dk
  */
 void 
-db_display_kmod(__unused db_expr_t addr, __unused int have_addr, __unused db_expr_t count, __unused char *modif)
+db_display_kmod(__unused db_expr_t addr, __unused boolean_t have_addr,
+		__unused db_expr_t count, __unused char *modif)
 {
 
 	kmod_info_t    *kmd;
@@ -859,6 +871,10 @@ db_display_kmod(__unused db_expr_t addr, __unused int have_addr, __unused db_exp
 			kmd, kmd->address, strt, end, kmd->name, kmd->version);
 		kmd = kmd->next;
 	}
+}
 
-	return;
+void
+db_display_iokit(__unused db_expr_t addr, __unused boolean_t have_addr,
+		__unused db_expr_t count, __unused char *modif)
+{
 }
diff --git a/osfmk/i386/eflags.h b/osfmk/i386/eflags.h
index 64e200fa7..9dc64783e 100644
--- a/osfmk/i386/eflags.h
+++ b/osfmk/i386/eflags.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/endian.h b/osfmk/i386/endian.h
index 730c5c7b6..88fbc2cd5 100644
--- a/osfmk/i386/endian.h
+++ b/osfmk/i386/endian.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/etimer.c b/osfmk/i386/etimer.c
new file mode 100644
index 000000000..ddbd77ffc
--- /dev/null
+++ b/osfmk/i386/etimer.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+/*
+ *	File:		etimer.c
+ *	Purpose:	Routines for handling the machine independent
+ *				event timer.
+ */
+
+#include <mach/mach_types.h>
+
+#include <kern/clock.h>
+#include <kern/thread.h>
+#include <kern/processor.h>
+#include <kern/macro_help.h>
+#include <kern/spl.h>
+#include <kern/etimer.h>
+#include <kern/pms.h>
+
+#include <machine/commpage.h>
+#include <machine/machine_routines.h>
+
+#include <sys/kdebug.h>
+#include <i386/cpu_data.h>
+#include <i386/cpu_topology.h>
+#include <i386/cpu_threads.h>
+
+/* XXX from <arch>/rtclock.c */
+clock_timer_func_t		rtclock_timer_expire;
+
+/*
+ * 	Event timer interrupt.
+ *
+ * XXX a drawback of this implementation is that events serviced earlier must not set deadlines
+ *     that occur before the entire chain completes.
+ *
+ * XXX a better implementation would use a set of generic callouts and iterate over them
+ */
+void
+etimer_intr(
+__unused int inuser,
+__unused uint64_t iaddr)
+{
+	uint64_t		abstime;
+	rtclock_timer_t		*mytimer;
+	cpu_data_t		*pp;
+	x86_lcpu_t		*lcpu;
+
+	pp = current_cpu_datap();
+	lcpu = x86_lcpu();
+
+	mytimer = &pp->rtclock_timer;				/* Point to the event timer */
+	abstime = mach_absolute_time();				/* Get the time now */
+
+	/* is it time for power management state change? */	
+	if (pmCPUGetDeadline(pp) <= abstime) {
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_START, 0, 0, 0, 0, 0);
+		pmCPUDeadline(pp);
+	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+		abstime = mach_absolute_time();			/* Get the time again since we ran a bit */
+	}
+
+	/* has a pending clock timer expired? */
+	if (mytimer->deadline <= abstime) {			/* Have we expired the deadline? */
+		mytimer->has_expired = TRUE;			/* Remember that we popped */
+		mytimer->deadline = EndOfAllTime;		/* Set timer request to the end of all time in case we have no more events */
+		(*rtclock_timer_expire)(abstime);		/* Process pop */
+		mytimer->has_expired = FALSE;
+	}
+
+	/* schedule our next deadline */
+	lcpu->rtcPop = EndOfAllTime;				/* any real deadline will be earlier */
+	etimer_resync_deadlines();
+}
+
+/*
+ * Set the clock deadline; called by the thread scheduler.
+ */
+void etimer_set_deadline(uint64_t deadline)
+{
+	rtclock_timer_t		*mytimer;
+	spl_t			s;
+	cpu_data_t		*pp;
+
+	s = splclock();					/* no interruptions */
+	pp = current_cpu_datap();
+
+	mytimer = &pp->rtclock_timer;			/* Point to the timer itself */
+	mytimer->deadline = deadline;			/* Set the new expiration time */
+
+	etimer_resync_deadlines();
+
+	splx(s);
+}
+
+/*
+ * Re-evaluate the outstanding deadlines and select the most proximate.
+ *
+ * Should be called at splclock.
+ */
+void
+etimer_resync_deadlines(void)
+{
+	uint64_t		deadline;
+	uint64_t		pmdeadline;
+	rtclock_timer_t		*mytimer;
+	spl_t			s = splclock();
+	cpu_data_t		*pp;
+	x86_lcpu_t		*lcpu;
+
+	pp = current_cpu_datap();
+	lcpu = x86_lcpu();
+	deadline = ~0ULL;
+
+	/*
+	 * If we have a clock timer set sooner, pop on that.
+	 */
+	mytimer = &pp->rtclock_timer;
+	if (!mytimer->has_expired && mytimer->deadline > 0)
+		deadline = mytimer->deadline;
+
+	/*
+	 * If we have a power management deadline, see if that's earlier.
+	 */
+	pmdeadline = pmCPUGetDeadline(pp);
+	if (pmdeadline > 0 && pmdeadline < deadline)
+	    deadline = pmdeadline;
+
+	/*
+	 * Go and set the "pop" event.
+	 */
+	if (deadline > 0) {
+		int     decr;
+		uint64_t now;
+
+		now = mach_absolute_time();
+		decr = setPop(deadline);
+
+		if (deadline < now)
+		        lcpu->rtcPop = now + decr;
+		else
+		        lcpu->rtcPop = deadline;
+
+		lcpu->rtcDeadline = lcpu->rtcPop;
+
+		KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, decr, 2, 0, 0, 0);
+	}
+	splx(s);
+}
diff --git a/osfmk/i386/exec.h b/osfmk/i386/exec.h
index 72526d180..2de2ea0cb 100644
--- a/osfmk/i386/exec.h
+++ b/osfmk/i386/exec.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/flipc_page.h b/osfmk/i386/flipc_page.h
index 542c8a446..8236be548 100644
--- a/osfmk/i386/flipc_page.h
+++ b/osfmk/i386/flipc_page.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/fpu.c b/osfmk/i386/fpu.c
index 403002322..618f2d9b0 100644
--- a/osfmk/i386/fpu.c
+++ b/osfmk/i386/fpu.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -167,15 +173,12 @@ init_fpu(void)
         {
 	    /* Use FPU save/restore instructions if available */
 		if (cpuid_features() & CPUID_FEATURE_FXSR) {
-		        fp_kind = FP_FXSR;
+	    	fp_kind = FP_FXSR;
 			set_cr4(get_cr4() | CR4_FXS);
-			printf("Enabling XMM register save/restore");
 			/* And allow SIMD instructions if present */
 			if (cpuid_features() & CPUID_FEATURE_SSE) {
-		    	printf(" and SSE/SSE2");
 		    	set_cr4(get_cr4() | CR4_XMM);
 			}
-			printf(" opcodes\n");
 	    } else
 			panic("fpu is not FP_FXSR");
 
@@ -226,8 +229,7 @@ fpu_module_init(void)
  * Called only when thread terminating - no locking necessary.
  */
 void
-fpu_free(fps)
-	struct x86_fpsave_state *fps;
+fpu_free(struct x86_fpsave_state *fps)
 {
 	fp_state_free(fps);
 }
@@ -243,7 +245,7 @@ fpu_free(fps)
  */
 kern_return_t
 fpu_set_fxstate(
-	thread_t	thr_act,
+	thread_t		thr_act,
 	thread_state_t	tstate)
 {
 	struct x86_fpsave_state	*ifps;
@@ -252,7 +254,7 @@ fpu_set_fxstate(
 	pcb_t	pcb;
 
 	if (fp_kind == FP_NO)
-	        return KERN_FAILURE;
+	    return KERN_FAILURE;
 
 	state = (x86_float_state64_t *)tstate;
 
@@ -260,58 +262,63 @@ fpu_set_fxstate(
 	pcb = thr_act->machine.pcb;
 
 	if (state == NULL) {
-	        /*
-		 * new FPU state is 'invalid'.
-		 * Deallocate the fp state if it exists.
-		 */
-	        simple_lock(&pcb->lock);
+	    /*
+	     * new FPU state is 'invalid'.
+	     * Deallocate the fp state if it exists.
+	     */
+	    simple_lock(&pcb->lock);
 
 		ifps = pcb->ifps;
 		pcb->ifps = 0;
 
-		simple_unlock(&pcb->lock);
+	    simple_unlock(&pcb->lock);
 
 		if (ifps != 0)
-		        fp_state_free(ifps);
+		fp_state_free(ifps);
 	} else {
-	        /*
-		 * Valid state.  Allocate the fp state if there is none.
-		 */
-	        new_ifps = 0;
+	    /*
+	     * Valid state.  Allocate the fp state if there is none.
+	     */
+	    new_ifps = 0;
 	Retry:
-		simple_lock(&pcb->lock);
+	    simple_lock(&pcb->lock);
 
 		ifps = pcb->ifps;
-		if (ifps == 0) {
-		        if (new_ifps == 0) {
-			        simple_unlock(&pcb->lock);
-				new_ifps = fp_state_alloc();
-				goto Retry;
-			}
-			ifps = new_ifps;
-			new_ifps = 0;
-			pcb->ifps = ifps;
+	    if (ifps == 0) {
+		if (new_ifps == 0) {
+		    simple_unlock(&pcb->lock);
+		    new_ifps = fp_state_alloc();
+		    goto Retry;
 		}
-		/*
-		 * now copy over the new data.
-		 */
-		bcopy((char *)&state->fpu_fcw,
+		ifps = new_ifps;
+		new_ifps = 0;
+			pcb->ifps = ifps;
+	    }
+	    /*
+	     * now copy over the new data.
+	     */
+            bcopy((char *)&state->fpu_fcw,
 		      (char *)&ifps->fx_save_state, sizeof(struct x86_fx_save));
 
 		/* XXX The layout of the state set from user-space may need to be
 		 * validated for consistency.
 		 */
 		ifps->fp_save_layout = thread_is_64bit(thr_act) ? FXSAVE64 : FXSAVE32;
+		/* Mark the thread's floating point status as non-live. */
+		/* Temporarily disabled: radar 4647827
+		 * ifps->fp_valid = TRUE;
+		 */
+
 		/*
 		 * Clear any reserved bits in the MXCSR to prevent a GPF
 		 * when issuing an FXRSTOR.
 		 */
-		ifps->fx_save_state.fx_MXCSR &= mxcsr_capability_mask;
+	    ifps->fx_save_state.fx_MXCSR &= mxcsr_capability_mask;
 
-		simple_unlock(&pcb->lock);
+	    simple_unlock(&pcb->lock);
 
-		if (new_ifps != 0)
-		        fp_state_free(new_ifps);
+	    if (new_ifps != 0)
+		fp_state_free(new_ifps);
 	}
 	return KERN_SUCCESS;
 }
@@ -324,7 +331,7 @@ fpu_set_fxstate(
  */
 kern_return_t
 fpu_get_fxstate(
-	thread_t	thr_act,
+	thread_t				thr_act,
 	thread_state_t	tstate)
 {
 	struct x86_fpsave_state	*ifps;
@@ -333,7 +340,7 @@ fpu_get_fxstate(
 	pcb_t	pcb;
 
 	if (fp_kind == FP_NO)
-	    return KERN_FAILURE;
+		return KERN_FAILURE;
 
 	state = (x86_float_state64_t *)tstate;
 
@@ -344,10 +351,10 @@ fpu_get_fxstate(
 
 	ifps = pcb->ifps;
 	if (ifps == 0) {
-	        /*
+		/*
 		 * No valid floating-point state.
 		 */
-	        bcopy((char *)&starting_fp_state.fx_save_state,
+		bcopy((char *)&starting_fp_state.fx_save_state,
 		      (char *)&state->fpu_fcw, sizeof(struct x86_fx_save));
 
 		simple_unlock(&pcb->lock);
@@ -358,9 +365,8 @@ fpu_get_fxstate(
 	 * Make sure we`ve got the latest fp state info
 	 * If the live fpu state belongs to our target
 	 */
-	if (thr_act == current_thread())
-	{
-	        boolean_t	intr;
+	if (thr_act == current_thread()) {
+		boolean_t	intr;
 
 		intr = ml_set_interrupts_enabled(FALSE);
 
@@ -381,6 +387,7 @@ fpu_get_fxstate(
 }
 
 
+
 /*
  * the child thread is 'stopped' with the thread
  * mutex held and is currently not known by anyone
@@ -428,6 +435,10 @@ fpu_dup_fxstate(
 			      (char *)&(child->machine.pcb->ifps->fx_save_state), sizeof(struct x86_fx_save));
 
 			new_ifps->fp_save_layout = ppcb->ifps->fp_save_layout;
+			/* Mark the new fp saved state as non-live. */
+			/* Temporarily disabled: radar 4647827
+			 * new_ifps->fp_valid = TRUE;
+			 */
 			/*
 			 * Clear any reserved bits in the MXCSR to prevent a GPF
 			 * when issuing an FXRSTOR.
@@ -467,8 +478,8 @@ fpinit(void)
 	fldcw(control);
 
 	/* Initialize SSE/SSE2 */
-	__builtin_ia32_ldmxcsr(0x1f80);
-}
+		__builtin_ia32_ldmxcsr(0x1f80);
+	}
 
 /*
  * Coprocessor not present.
@@ -478,6 +489,15 @@ void
 fpnoextflt(void)
 {
 	boolean_t	intr;
+	thread_t	thr_act;
+	pcb_t		pcb;
+	struct x86_fpsave_state *ifps = 0;
+
+	thr_act = current_thread();
+	pcb = thr_act->machine.pcb;
+
+	if (pcb->ifps == 0 && !get_interrupt_level())
+	        ifps = fp_state_alloc();
 
 	intr = ml_set_interrupts_enabled(FALSE);
 
@@ -488,16 +508,22 @@ fpnoextflt(void)
 		 * Save current coprocessor context if valid
 		 * Initialize coprocessor live context
 		 */
-		fp_save(current_thread());
+		fp_save(thr_act);
 		fpinit();
 	} else {
+	        if (pcb->ifps == 0) {
+		        pcb->ifps = ifps;
+			ifps = 0;
+		}
 		/*
 		 * Load this thread`s state into coprocessor live context.
 		 */
-		fp_load(current_thread());
+		fp_load(thr_act);
 	}
-
 	(void)ml_set_interrupts_enabled(intr);
+
+	if (ifps)
+	        fp_state_free(ifps);
 }
 
 /*
@@ -516,7 +542,7 @@ fpextovrflt(void)
 	intr = ml_set_interrupts_enabled(FALSE);
 
 	if (get_interrupt_level())
-		panic("FPU segment overrun exception at interrupt context\n");
+		panic("FPU segment overrun exception  at interrupt context\n");
 	if (current_task() == kernel_task)
 		panic("FPU segment overrun exception in kernel thread context\n");
 
diff --git a/osfmk/i386/fpu.h b/osfmk/i386/fpu.h
index 9effc417f..cd0b78906 100644
--- a/osfmk/i386/fpu.h
+++ b/osfmk/i386/fpu.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -48,9 +54,6 @@
  * the rights to redistribute these changes.
  */
 
-/*
- */
-
 #ifndef	_I386_FPU_H_
 #define	_I386_FPU_H_
 
@@ -101,9 +104,8 @@ extern void		fxrstor64(struct x86_fx_save *);
 #define	fldcw(control) \
 	__asm__ volatile("fldcw %0" : : "m" (*(unsigned short *) &(control)) )
 
-extern unsigned short		fnstsw(void);
-
-extern __inline__ unsigned short fnstsw(void)
+static inline unsigned short
+fnstsw(void)
 {
 	unsigned short status;
 	__asm__ volatile("fnstsw %0" : "=ma" (status));
@@ -152,6 +154,7 @@ static inline void fpu_save_context(thread_t thread)
 		clear_ts();
 		/* registers are in FPU - save to memory */
 		ifps->fp_valid = TRUE;
+
 		if (!thread_is_64bit(thread) || is_saved_state32(thread->machine.pcb->iss)) {
 			/* save the compatibility/legacy mode XMM+x87 state */
 			fxsave(&ifps->fx_save_state);
diff --git a/osfmk/i386/gcc.s b/osfmk/i386/gcc.s
index 6537364ce..06a6fb13c 100644
--- a/osfmk/i386/gcc.s
+++ b/osfmk/i386/gcc.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/gdb_defs.h b/osfmk/i386/gdb_defs.h
index c8942d570..c039f2ee8 100644
--- a/osfmk/i386/gdb_defs.h
+++ b/osfmk/i386/gdb_defs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/gdt.c b/osfmk/i386/gdt.c
index 500c34495..4150e7c0d 100644
--- a/osfmk/i386/gdt.c
+++ b/osfmk/i386/gdt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,39 +69,39 @@
 struct fake_descriptor master_gdt[GDTSZ] __attribute__ ((aligned (4096))) = {
 	[SEL_TO_INDEX(KERNEL_CS)] {	/* kernel code */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
+		  0xfffff,
+		  SZ_32|SZ_G,
 		ACC_P|ACC_PL_K|ACC_CODE_R,
 	},
 	[SEL_TO_INDEX(KERNEL_DS)] {	/* kernel data */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_K|ACC_DATA_W
+		  0xfffff,
+		  SZ_32|SZ_G,
+		  ACC_P|ACC_PL_K|ACC_DATA_W
 	},
 	[SEL_TO_INDEX(KERNEL_LDT)] {	/* local descriptor table */
 		(uint32_t) &master_ldt,
-		LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
-		0,
-		ACC_P|ACC_PL_K|ACC_LDT
+		  LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
+		  0,
+		  ACC_P|ACC_PL_K|ACC_LDT
 	},				/* The slot KERNEL_LDT_2 is reserved. */
 	[SEL_TO_INDEX(KERNEL_TSS)] {	/* TSS for this processor */
 		(uint32_t) &master_ktss,
-		sizeof(struct i386_tss)-1,
-		0,
-		ACC_P|ACC_PL_K|ACC_TSS
+		  sizeof(struct i386_tss)-1,
+		  0,
+		  ACC_P|ACC_PL_K|ACC_TSS
 	},				/* The slot KERNEL_TSS_2 is reserved. */
 	[SEL_TO_INDEX(CPU_DATA_GS)] {	/* per-CPU current thread address */
 		(uint32_t) &cpu_data_master,
-		sizeof(cpu_data_t)-1,
-		SZ_32,
-		ACC_P|ACC_PL_K|ACC_DATA_W
+		  sizeof(cpu_data_t)-1,
+		  SZ_32,
+		  ACC_P|ACC_PL_K|ACC_DATA_W
 	},
 	[SEL_TO_INDEX(USER_LDT)] {	/* user local descriptor table */
 		(uint32_t) &master_ldt,
-		LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
-		0,
-		ACC_P|ACC_PL_K|ACC_LDT
+		  LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
+		  0,
+		  ACC_P|ACC_PL_K|ACC_LDT
 	},
 	[SEL_TO_INDEX(KERNEL64_CS)] {	/* kernel 64-bit code */ 
 		0,
@@ -112,9 +118,9 @@ struct fake_descriptor master_gdt[GDTSZ] __attribute__ ((aligned (4096))) = {
 #if	MACH_KDB
 	[SEL_TO_INDEX(DEBUG_TSS)] {	/* TSS for this processor */
 		(uint32_t)&master_dbtss,
-		sizeof(struct i386_tss)-1,
-		0,
-		ACC_P|ACC_PL_K|ACC_TSS
+		  sizeof(struct i386_tss)-1,
+		  0,
+		  ACC_P|ACC_PL_K|ACC_TSS
 	},
 #endif	/* MACH_KDB */
 };
diff --git a/osfmk/i386/genassym.c b/osfmk/i386/genassym.c
index 56cffc191..a723585bb 100644
--- a/osfmk/i386/genassym.c
+++ b/osfmk/i386/genassym.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -83,6 +89,11 @@
 #include <i386/mp_desc.h>
 #include <pexpert/i386/boot.h>
 
+#if	CONFIG_DTRACE
+#define NEED_DTRACE_DEFS
+#include <../bsd/sys/lockstat.h>
+#endif
+
 /*
  * genassym.c is used to produce an
  * assembly file which, intermingled with unuseful assembly code,
@@ -118,6 +129,7 @@ main(
 {
 
 	DECLARE("AST_URGENT",		AST_URGENT);
+	DECLARE("AST_BSD",			AST_BSD);
 
 	/* Simple Lock structure */
 	DECLARE("SLOCK_ILK",	offsetof(usimple_lock_t, interlock));
@@ -141,8 +153,30 @@ main(
 	DECLARE("MUTEX_TAG",	MUTEX_TAG);
 #endif	/* MACH_LDEBUG */
 	DECLARE("MUTEX_IND",	LCK_MTX_TAG_INDIRECT);
+	DECLARE("MUTEX_DESTROYED", LCK_MTX_TAG_DESTROYED);
+	DECLARE("MUTEX_LOCKED_AS_SPIN",	MUTEX_LOCKED_AS_SPIN);
 	DECLARE("MUTEX_ITAG",	offsetof(lck_mtx_t *, lck_mtx_tag));
 	DECLARE("MUTEX_PTR",	offsetof(lck_mtx_t *, lck_mtx_ptr));
+	DECLARE("MUTEX_ASSERT_OWNED",	LCK_MTX_ASSERT_OWNED);
+	DECLARE("MUTEX_ASSERT_NOTOWNED",LCK_MTX_ASSERT_NOTOWNED);
+	/* Per-mutex statistic element */
+	DECLARE("MTX_ACQ_TSC",	offsetof(lck_mtx_ext_t *, lck_mtx_stat));
+
+	/* Mutex group statistics elements */
+	DECLARE("MUTEX_GRP",	offsetof(lck_mtx_ext_t *, lck_mtx_grp));
+	
+	DECLARE("GRP_MTX_STAT_UTIL",	offsetof(lck_grp_t *, lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt));
+	DECLARE("GRP_MTX_STAT_MISS",	offsetof(lck_grp_t *, lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt));
+	DECLARE("GRP_MTX_STAT_WAIT",	offsetof(lck_grp_t *, lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt));
+	/*
+	 * The use of this field is somewhat at variance with the alias.
+	 */
+	DECLARE("GRP_MTX_STAT_DIRECT_WAIT",	offsetof(lck_grp_t *, lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt));
+
+	DECLARE("GRP_MTX_STAT_HELD_MAX",	offsetof(lck_grp_t *, lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_max));
+	/* Reader writer lock types */
+	DECLARE("RW_SHARED",    LCK_RW_TYPE_SHARED);
+	DECLARE("RW_EXCL",      LCK_RW_TYPE_EXCLUSIVE);
 
 	DECLARE("TH_RECOVER",		offsetof(thread_t, recover));
 	DECLARE("TH_CONTINUATION",	offsetof(thread_t, continuation));
@@ -153,11 +187,14 @@ main(
 	DECLARE("TASK_SYSCALLS_MACH",	offsetof(struct task *, syscalls_mach));
 	DECLARE("TASK_SYSCALLS_UNIX",	offsetof(struct task *, syscalls_unix));
 
+	DECLARE("TASK_VTIMERS",			offsetof(struct task *, vtimers));
+
 	/* These fields are being added on demand */
 	DECLARE("ACT_MACH_EXC_PORT",
 		offsetof(thread_t, exc_actions[EXC_MACH_SYSCALL].port));
 
 	DECLARE("ACT_TASK",	offsetof(thread_t, task));
+	DECLARE("ACT_AST",	offsetof(thread_t, ast));
 	DECLARE("ACT_PCB",	offsetof(thread_t, machine.pcb));
 	DECLARE("ACT_SPF",	offsetof(thread_t, machine.specFlags));
 	DECLARE("ACT_MAP",	offsetof(thread_t, map));
@@ -374,6 +411,8 @@ main(
 		offsetof(cpu_data_t *,cpu_processor));
         DECLARE("CPU_INT_STATE",
 		offsetof(cpu_data_t *, cpu_int_state));
+        DECLARE("CPU_INT_EVENT_TIME",
+		offsetof(cpu_data_t *, cpu_int_event_time));
 
         DECLARE("CPU_HI_ISS",
 		offsetof(cpu_data_t *, cpu_hi_iss));
@@ -397,6 +436,11 @@ main(
 		offsetof(cpu_data_t *, cpu_uber.cu_isf));
 	DECLARE("CPU_UBER_TMP",
 		offsetof(cpu_data_t *, cpu_uber.cu_tmp));
+	DECLARE("CPU_UBER_ARG_STORE",
+		offsetof(cpu_data_t *, cpu_uber_arg_store));
+	DECLARE("CPU_UBER_ARG_STORE_VALID",
+		offsetof(cpu_data_t *, cpu_uber_arg_store_valid));
+
 	DECLARE("CPU_DR7",
 		offsetof(cpu_data_t *, cpu_dr7));
 
@@ -470,6 +514,8 @@ main(
 		offsetof(rtc_nanotime_t *, scale));
 	DECLARE("RNT_SHIFT",
 		offsetof(rtc_nanotime_t *, shift));
+	DECLARE("RNT_GENERATION",
+		offsetof(rtc_nanotime_t *, generation));
 
 	/* values from kern/timer.h */
 	DECLARE("TIMER_LOW",
@@ -482,13 +528,23 @@ main(
 	DECLARE("TIMER_TSTAMP",
 		offsetof(struct timer *, tstamp));
 
-	DECLARE("CURRENT_TIMER",
-		offsetof(struct processor *, processor_data.current_timer));
+	DECLARE("THREAD_TIMER",
+		offsetof(struct processor *, processor_data.thread_timer));
 #endif
+	DECLARE("KERNEL_TIMER",
+		offsetof(struct processor *, processor_data.kernel_timer));
 	DECLARE("SYSTEM_TIMER",
 		offsetof(struct thread *, system_timer));
 	DECLARE("USER_TIMER",
 		offsetof(struct thread *, user_timer));
+	DECLARE("SYSTEM_STATE",
+			offsetof(struct processor *, processor_data.system_state));
+	DECLARE("USER_STATE",
+			offsetof(struct processor *, processor_data.user_state));
+	DECLARE("IDLE_STATE",
+			offsetof(struct processor *, processor_data.idle_state));
+	DECLARE("CURRENT_STATE",
+			offsetof(struct processor *, processor_data.current_state));
 
 	DECLARE("OnProc", OnProc);
 
@@ -507,5 +563,24 @@ main(
 	DECLARE("TIM2_CONF",	offsetof(hpetReg_t *, TIM2_CONF));
 	DECLARE("TIM2_COMP",	offsetof(hpetReg_t *, TIM2_COMP));
 
+#if	CONFIG_DTRACE
+	DECLARE("LS_LCK_MTX_LOCK_ACQUIRE", LS_LCK_MTX_LOCK_ACQUIRE);
+	DECLARE("LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE", LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE);
+	DECLARE("LS_LCK_MTX_UNLOCK_RELEASE", LS_LCK_MTX_UNLOCK_RELEASE);
+	DECLARE("LS_LCK_MTX_TRY_LOCK_ACQUIRE", LS_LCK_MTX_TRY_LOCK_ACQUIRE);
+	DECLARE("LS_LCK_RW_LOCK_SHARED_ACQUIRE", LS_LCK_RW_LOCK_SHARED_ACQUIRE);
+	DECLARE("LS_LCK_RW_DONE_RELEASE", LS_LCK_RW_DONE_RELEASE);
+	DECLARE("LS_LCK_MTX_EXT_LOCK_ACQUIRE", LS_LCK_MTX_EXT_LOCK_ACQUIRE);
+	DECLARE("LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE", LS_LCK_MTX_TRY_EXT_LOCK_ACQUIRE);
+	DECLARE("LS_LCK_MTX_EXT_UNLOCK_RELEASE", LS_LCK_MTX_EXT_UNLOCK_RELEASE);
+
+	DECLARE("LS_MUTEX_LOCK_ACQUIRE", LS_MUTEX_LOCK_ACQUIRE);
+	DECLARE("LS_MUTEX_TRY_SPIN_ACQUIRE", LS_MUTEX_TRY_SPIN_ACQUIRE);
+	DECLARE("LS_MUTEX_TRY_LOCK_ACQUIRE", LS_MUTEX_TRY_LOCK_ACQUIRE);
+	DECLARE("LS_MUTEX_UNLOCK_RELEASE", LS_MUTEX_UNLOCK_RELEASE);
+	DECLARE("LS_MUTEX_LOCK_SPIN_ACQUIRE", LS_MUTEX_LOCK_SPIN_ACQUIRE);
+	DECLARE("LS_MUTEX_CONVERT_SPIN_ACQUIRE", LS_MUTEX_CONVERT_SPIN_ACQUIRE);
+#endif
+
 	return (0);
 }
diff --git a/osfmk/i386/hi_res_clock.h b/osfmk/i386/hi_res_clock.h
deleted file mode 100644
index f9c006310..000000000
--- a/osfmk/i386/hi_res_clock.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-
-#ifndef	_MACHINE_HI_RES_CLOCK_H_
-#define	_MACHINE_HI_RES_CLOCK_H_
-
-#include <types.h>
-#include <mach/i386/vm_types.h>
-
-extern vm_offset_t hi_res_clk_mmap(
-	dev_t		dev,
-	vm_offset_t	off,
-	int		prot);
-
-extern void clock_thread_386(
-	int	loop_iterations_per_tick,
-	int	*high_res_clock);
-
-#endif	/* _MACHINE_HI_RES_CLOCK_H_ */
diff --git a/osfmk/i386/hi_res_clock_map.c b/osfmk/i386/hi_res_clock_map.c
deleted file mode 100644
index 66ca626ff..000000000
--- a/osfmk/i386/hi_res_clock_map.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:36  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:37  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.7.1  1994/09/23  01:54:44  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:23:10  ezf]
- *
- * Revision 1.1.2.2  1993/08/24  09:39:55  rod
- * 	Created for iX86 common high resolution clock common code.  CR #9400.
- * 	[1993/08/17  11:26:08  rod]
- * 
- * $EndLog$
- */
-
-#include <vm/pmap.h>
-#include <i386/hi_res_clock.h>
-
-extern int *high_res_clock;
-
-vm_offset_t
-hi_res_clk_mmap(
-	dev_t		dev,
-	vm_offset_t	off,
-	int		prot)
-{
-               if (prot & VM_PROT_WRITE) return (-1);
-               return (i386_btop(pmap_extract(pmap_kernel(), 
-		                              (vm_offset_t) high_res_clock)));
-}
diff --git a/osfmk/i386/hibernate_i386.c b/osfmk/i386/hibernate_i386.c
index d8f14ea46..4409b74bc 100644
--- a/osfmk/i386/hibernate_i386.c
+++ b/osfmk/i386/hibernate_i386.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/machine.h>
@@ -57,7 +63,7 @@ hibernate_page_list_allocate(void)
     hibernate_bitmap_t	    dram_ranges[MAX_BANKS];
     boot_args *		    args = (boot_args *) PE_state.bootArgs;
 
-    mptr = args->MemoryMap;
+    mptr = (EfiMemoryRange *)args->MemoryMap;
     if (args->MemoryMapDescriptorSize == 0)
 	panic("Invalid memory map descriptor size");
     msize = args->MemoryMapDescriptorSize;
diff --git a/osfmk/i386/hibernate_restore.s b/osfmk/i386/hibernate_restore.s
index 564b0ec0b..756de43a4 100644
--- a/osfmk/i386/hibernate_restore.s
+++ b/osfmk/i386/hibernate_restore.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <i386/asm.h>
@@ -166,7 +172,7 @@ LEXT(hibernate_machine_entrypoint)
         POSTCODE(0x3)
         
 	movl    %cr4,%eax
-        orl     $(CR4_PAE|CR4_PGE),%eax
+        orl     $(CR4_PAE),%eax
         movl    %eax,%cr4               /* enable page size extensions */
 
 	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
@@ -253,7 +259,7 @@ LEXT(hibernate_restore_phys_page)
 	cld
 	/* move longs*/
 	movl	%edx,%ecx
-	sarl	$2,%ecx
+	shrl	$2,%ecx
 	rep
 	movsl
 	/* move bytes*/
diff --git a/osfmk/i386/hpet.c b/osfmk/i386/hpet.c
index c6eec35ab..52c70d903 100644
--- a/osfmk/i386/hpet.c
+++ b/osfmk/i386/hpet.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <string.h>
@@ -49,8 +55,13 @@
 #include <i386/tsc.h>
 #include <i386/hpet.h>
 #include <i386/pmCPU.h>
+#include <i386/cpu_topology.h>
+#include <i386/cpu_threads.h>
 #include <pexpert/device_tree.h>
 #if	MACH_KDB
+#include <i386/db_machdep.h>
+#endif
+#if	MACH_KDB
 #include <ddb/db_aout.h>
 #include <ddb/db_access.h>
 #include <ddb/db_sym.h>
@@ -83,12 +94,88 @@ uint64_t hpet2bus = 0;
 uint32_t rcbaArea = 0;			
 uint32_t rcbaAreap = 0;			
 
+static int (*hpet_req)(uint32_t apicid, void *arg, hpetRequest_t *hpet) = NULL;
+static void *hpet_arg = NULL;
+
 #if DEBUG
 #define DBG(x...)	kprintf("DBG: " x)
 #else
 #define DBG(x...)
 #endif
 
+int
+hpet_register_callback(int (*hpet_reqst)(uint32_t apicid,
+					 void *arg,
+					 hpetRequest_t *hpet),
+		       void *arg)
+{
+    hpet_req = hpet_reqst;
+    hpet_arg = arg;
+    return(0);
+}
+
+/*
+ * This routine is called to obtain an HPET and have it assigned
+ * to a CPU.  It returns 0 if successful and non-zero if one could
+ * not be assigned.
+ */
+int
+hpet_request(uint32_t cpu)
+{
+    hpetRequest_t	hpetReq;
+    int			rc;
+    x86_lcpu_t		*lcpu;
+    x86_core_t		*core;
+    x86_pkg_t		*pkg;
+    boolean_t		enabled;
+
+    if (hpet_req == NULL) {
+	return(-1);
+    }
+
+    rc = (*hpet_req)(ml_get_apicid(cpu), hpet_arg, &hpetReq);
+    if (rc != 0) {
+	return(rc);
+    }
+
+    enabled = ml_set_interrupts_enabled(FALSE);
+    lcpu = cpu_to_lcpu(cpu);
+    core = lcpu->core;
+    pkg  = core->package;
+
+    /*
+     * Compute the address of the HPET.
+     */
+    core->Hpet = (hpetTimer_t *)((uint8_t *)hpetArea + hpetReq.hpetOffset);
+    core->HpetVec = hpetReq.hpetVector;
+
+    /*
+     * Enable interrupts
+     */
+    core->Hpet->Config |= Tn_INT_ENB_CNF;
+
+    /*
+     * Save the configuration
+     */
+    core->HpetCfg = core->Hpet->Config;
+    core->HpetCmp = 0;
+
+    /*
+     * If the CPU is the "primary" for the package, then
+     * add the HPET to the package too.
+     */
+    if (lcpu->primary) {
+	pkg->Hpet = core->Hpet;
+	pkg->HpetCfg = core->HpetCfg;
+	pkg->HpetCmp = core->HpetCmp;
+	pkg->flags |= X86PKG_FL_HAS_HPET;
+    }
+
+    ml_set_interrupts_enabled(enabled);
+
+    return(0);
+}
+
 /*
  * Map the RCBA area.
  */
@@ -111,8 +198,6 @@ void
 hpet_init(void)
 {
 	unsigned int	*xmod;
-	uint64_t	now;
-	uint64_t	initialHPET;
 
 	map_rcbaArea();
 
@@ -204,14 +289,14 @@ hpet_init(void)
 	 * Convert current TSC to HPET value,
 	 * set it, and start it ticking.
 	 */
-	now = mach_absolute_time();
-	initialHPET = tmrCvt(now, hpetCvtn2t);
-	((hpetReg_t *)hpetArea)->MAIN_CNT = initialHPET;
+	uint64_t currtsc = rdtsc64();
+	uint64_t tscInHPET = tmrCvt(currtsc, tsc2hpet);
+	((hpetReg_t *)hpetArea)->MAIN_CNT = tscInHPET;
 	hpetcon = hpetcon | 1;
 	((hpetReg_t *)hpetArea)->GEN_CONF = hpetcon;
-	kprintf("HPET started: now = %08X.%08X, HPET = %08X.%08X\n",
-		(uint32_t)(now >> 32), (uint32_t)now,
-		(uint32_t)(initialHPET >> 32), (uint32_t)initialHPET);
+	kprintf("HPET started: TSC = %08X.%08X, HPET = %08X.%08X\n", 
+		(uint32_t)(currtsc >> 32), (uint32_t)currtsc,
+		(uint32_t)(tscInHPET >> 32), (uint32_t)tscInHPET);
 
 #if MACH_KDB
 	db_display_hpet((hpetReg_t *)hpetArea);	/* (BRINGUP) */
@@ -239,56 +324,87 @@ hpet_get_info(hpetInfo_t *info)
 
 /*
  * This routine is called by the HPET driver
- * when it assigns an HPET timer to a processor
+ * when it assigns an HPET timer to a processor.
+ *
+ * XXX with the new callback into the HPET driver,
+ * this routine will be deprecated.
  */
-
 void
 ml_hpet_cfg(uint32_t cpu, uint32_t hpetVect)
 {
-	uint64_t *hpetVaddr;
-	uint64_t hpetcnf;
+	uint64_t	*hpetVaddr;
+	hpetTimer_t	*hpet;
+	x86_lcpu_t	*lcpu;
+	x86_core_t	*core;
+	x86_pkg_t	*pkg;
+	boolean_t	enabled;
 	
 	if(cpu > 1) {
 		panic("ml_hpet_cfg: invalid cpu = %d\n", cpu);
 	}
 
+	lcpu = cpu_to_lcpu(cpu);
+	core = lcpu->core;
+	pkg  = core->package;
+
+	/*
+	 * Only deal with the primary CPU for the package.
+	 */
+	if (!lcpu->primary)
+	    return;
+
+	enabled = ml_set_interrupts_enabled(FALSE);
+
 	/* Calculate address of the HPET for this processor */
 	hpetVaddr = (uint64_t *)(((uint32_t)&(((hpetReg_t *)hpetArea)->TIM1_CONF)) + (cpu << 5));
+	hpet = (hpetTimer_t *)hpetVaddr;
 
-	DBG("ml_hpet_cfg: HPET for cpu %d at %08X, vector = %d\n",
+	DBG("ml_hpet_cfg: HPET for cpu %d at %p, vector = %d\n",
 	     cpu, hpetVaddr, hpetVect);
-	
+
 	/* Save the address and vector of the HPET for this processor */
-	cpu_data_ptr[cpu]->cpu_pmHpet = (uint64_t *)hpetVaddr;
-	cpu_data_ptr[cpu]->cpu_pmHpetVec = hpetVect;
+	core->Hpet = hpet;
+	core->HpetVec = hpetVect;
 
-	/* Enable the interruptions now that we have a vector */
-	hpetcnf = *hpetVaddr;
-	hpetcnf = hpetcnf | Tn_INT_ENB_CNF;
-	*hpetVaddr = hpetcnf;
+	/*
+	 * Enable interrupts
+	 */
+	core->Hpet->Config |= Tn_INT_ENB_CNF;
 
 	/* Save the configuration */
-	cpu_data_ptr[cpu]->cpu_pmStats.pmHpetCfg = hpetcnf;
-	cpu_data_ptr[cpu]->cpu_pmStats.pmHpetCmp = 0;
+	core->HpetCfg = core->Hpet->Config;
+	core->HpetCmp = 0;
 
-	/* See if nap policy has changed now */
-	machine_nap_policy();
+	/*
+	 * We're only doing this for the primary CPU, so go
+	 * ahead and add the HPET to the package too.
+	 */
+	pkg->Hpet = core->Hpet;
+	pkg->HpetVec = core->HpetVec;
+	pkg->HpetCfg = core->HpetCfg;
+	pkg->HpetCmp = core->HpetCmp;
+	pkg->flags |= X86PKG_FL_HAS_HPET;
 
+	ml_set_interrupts_enabled(enabled);
 }
 
 /*
  * This is the HPET interrupt handler.
  *
- * We really don't want to be here, but so far, I haven't figured out
- * a way to cancel the interrupt. Hopefully, some day we will figure out
- * how to do that or switch all timers to the HPET.
+ * It just hands off to the power management code so that the
+ * appropriate things get done there.
  */
 int
 HPETInterrupt(void)
 {
 
 	/* All we do here is to bump the count */
-	current_cpu_datap()->cpu_pmStats.pmHPETRupt++;
+	x86_package()->HpetInt++;
+
+	/*
+	 * Let power management do it's thing.
+	 */
+	pmHPETInterrupt();
 
 	/* Return and show that the 'rupt has been handled... */
 	return 1;
@@ -297,7 +413,8 @@ HPETInterrupt(void)
 
 static hpetReg_t saved_hpet;
 
-void hpet_save( void )
+void
+hpet_save(void)
 {
 	hpetReg_t	*from = (hpetReg_t *) hpetArea;
 	hpetReg_t	*to = &saved_hpet;
@@ -312,7 +429,8 @@ void hpet_save( void )
 	to->MAIN_CNT  = from->MAIN_CNT;
 }
 
-void hpet_restore( void )
+void
+hpet_restore(void)
 {
 	hpetReg_t	*from = &saved_hpet;
 	hpetReg_t	*to = (hpetReg_t *) hpetArea;
@@ -338,7 +456,7 @@ void hpet_restore( void )
 	to->TIM2_CONF = from->TIM2_CONF;
 	to->TIM2_COMP = from->TIM2_COMP;
 	to->GINTR_STA = -1ULL;
-	to->MAIN_CNT  = tmrCvt(mach_absolute_time(), hpetCvtn2t);
+	to->MAIN_CNT  = from->MAIN_CNT;
 
 	to->GEN_CONF = from->GEN_CONF;
 }
@@ -382,9 +500,8 @@ db_hpet(__unused db_expr_t addr, __unused int have_addr, __unused db_expr_t coun
 }
 
 void
-db_display_hpet(hpetReg_t * hpt)
+db_display_hpet(hpetReg_t *hpt)
 {
-
 	uint64_t        cmain;
 
 	cmain = hpt->MAIN_CNT;	/* Get the main timer */
@@ -422,9 +539,5 @@ db_display_hpet(hpetReg_t * hpt)
 
 	db_printf("\nHPET Frequency = %d.%05dMHz\n",
 	  (uint32_t) (hpetFreq / 1000000), (uint32_t) (hpetFreq % 1000000));
-
-	return;
-
 }
-
 #endif
diff --git a/osfmk/i386/hpet.h b/osfmk/i386/hpet.h
index f3595dce1..72656d06a 100644
--- a/osfmk/i386/hpet.h
+++ b/osfmk/i386/hpet.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef KERNEL_PRIVATE
 #ifndef _I386_HPET_H_
@@ -56,19 +62,34 @@ typedef struct hpetReg {
 } hpetReg;
 typedef struct 	hpetReg hpetReg_t;
 
+typedef struct hpetTimer {
+	uint64_t	Config;		/* Timer config and capabilities */
+	uint64_t	Compare;	/* Timer comparitor */
+} hpetTimer_t;
+
 struct hpetInfo
 {
-    uint64_t	hpetCvtt2n;
-    uint64_t	hpetCvtn2t;
-    uint64_t	tsc2hpet;
-    uint64_t	hpet2tsc;
-    uint64_t	bus2hpet;
-    uint64_t	hpet2bus;
-    uint32_t	rcbaArea;
-    uint32_t	rcbaAreap;
+	uint64_t	hpetCvtt2n;
+	uint64_t	hpetCvtn2t;
+	uint64_t	tsc2hpet;
+	uint64_t	hpet2tsc;
+	uint64_t	bus2hpet;
+	uint64_t	hpet2bus;
+	uint32_t	rcbaArea;
+	uint32_t	rcbaAreap;
 };
 typedef struct hpetInfo hpetInfo_t;
 
+struct hpetRequest
+{
+	uint32_t	flags;
+	uint32_t	hpetOffset;
+	uint32_t	hpetVector;
+};
+typedef struct hpetRequest hpetRequest_t;
+
+#define HPET_REQFL_64BIT	0x00000001	/* Timer is 64 bits */
+
 extern uint64_t hpetFemto;
 extern uint64_t hpetFreq;
 extern uint64_t hpetCvtt2n;
@@ -91,6 +112,9 @@ extern void hpet_restore(void);
 extern int HPETInterrupt(void);
 #endif
 
+extern int hpet_register_callback(int (*hpet_reqst)(uint32_t apicid, void *arg, hpetRequest_t *hpet), void *arg);
+extern int hpet_request(uint32_t cpu);
+
 extern uint64_t rdHPET(void);
 extern void hpet_get_info(hpetInfo_t *info);
 
diff --git a/osfmk/i386/hw_defs.h b/osfmk/i386/hw_defs.h
new file mode 100644
index 000000000..0fac10f3c
--- /dev/null
+++ b/osfmk/i386/hw_defs.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _I386_HW_DEFS_H_
+#define _I386_HW_DEFS_H_
+
+
+#define pmMwaitC1 	0x00
+#define pmMwaitC2	0x10
+#define pmMwaitC3 	0x20
+#define pmMwaitC4 	0x30
+#define pmMwaitBrInt 0x1
+
+#define pmBase 		0x400
+#define pmCtl1 		0x04
+#define pmCtl2 		0x20
+#define pmC3Res 	0x54
+#define pmStatus 	0x00
+#define msrTSC 		0x10
+
+#endif /* _I386_HW_DEFS_H_ */
diff --git a/osfmk/i386/hw_lock_types.h b/osfmk/i386/hw_lock_types.h
index 4bf586e78..1cf2d99dc 100644
--- a/osfmk/i386/hw_lock_types.h
+++ b/osfmk/i386/hw_lock_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/i386_init.c b/osfmk/i386/i386_init.c
index 39fbd996a..2c0df6147 100644
--- a/osfmk/i386/i386_init.c
+++ b/osfmk/i386/i386_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -86,6 +92,7 @@
 #include <i386/pmCPU.h>
 #include <i386/tsc.h>
 #include <i386/hpet.h>
+#include <i386/locks.h> /* LcksOpts */
 #if	MACH_KDB
 #include <ddb/db_aout.h>
 #endif /* MACH_KDB */
@@ -93,6 +100,8 @@
 
 static boot_args *kernelBootArgs;
 
+int debug_task;
+
 extern int disableConsoleOutput;
 extern const char version[];
 extern const char version_variant[];
@@ -100,9 +109,6 @@ extern int nx_enabled;
 
 extern int noVMX;	/* if set, rosetta should not emulate altivec */
 
-void cpu_stack_set(void);
-
-
 /*
  *	Cpu initialization.  Running virtual, but without MACH VM
  *	set up.  First C routine called.
@@ -137,9 +143,6 @@ i386_init(vm_offset_t boot_args_start)
 	cpu_init();
 	postcode(CPU_INIT_D);
 
-	/* init processor performance control */
-	pmsInit();
-	
 	PE_init_platform(FALSE, kernelBootArgs);
 	postcode(PE_INIT_PLATFORM_D);
 
@@ -167,11 +170,7 @@ i386_init(vm_offset_t boot_args_start)
 
 	kprintf("version_variant = %s\n", version_variant);
 	kprintf("version         = %s\n", version);
-
-	/*   
-	 * VM initialization, after this we're using page tables...
-	 * The maximum number of cpus must be set beforehand.
-	 */
+	
 	if (!PE_parse_boot_arg("maxmem", &maxmem))
 		maxmemtouse=0;
 	else
@@ -188,6 +187,9 @@ i386_init(vm_offset_t boot_args_start)
 	if (!PE_parse_boot_arg("himemory_mode", &vm_himemory_mode))
 	        vm_himemory_mode = 0;
 
+	if (!PE_parse_boot_arg("immediate_NMI", &force_immediate_debugger_NMI))
+		force_immediate_debugger_NMI = FALSE;
+
 	/*
 	 * At this point we check whether we are a 64-bit processor
 	 * and that we're not restricted to legacy mode, 32-bit operation.
@@ -196,15 +198,24 @@ i386_init(vm_offset_t boot_args_start)
 	if (cpuid_extfeatures() & CPUID_EXTFEATURE_EM64T) {
 		kprintf("EM64T supported");
 		if (PE_parse_boot_arg("-legacy", &legacy_mode)) {
-		 	kprintf(" but legacy mode forced\n");
+			kprintf(" but legacy mode forced\n");
 		} else {
 			IA32e = TRUE;
 			kprintf(" and will be enabled\n");
 		}
 	}
+
 	if (!(cpuid_extfeatures() & CPUID_EXTFEATURE_XD))
 		nx_enabled = 0;
 
+	/* Obtain "lcks" options:this currently controls lock statistics */
+	if (!PE_parse_boot_arg("lcks", &LcksOpts))
+		LcksOpts = 0;
+
+	/*   
+	 * VM initialization, after this we're using page tables...
+	 * The maximum number of cpus must be set beforehand.
+	 */
 	i386_vm_init(maxmemtouse, IA32e, kernelBootArgs);
 
 	if ( ! PE_parse_boot_arg("novmx", &noVMX))
@@ -223,5 +234,4 @@ i386_init(vm_offset_t boot_args_start)
 	thread_bootstrap();
 
 	machine_startup();
-
 }
diff --git a/osfmk/i386/i386_lock.s b/osfmk/i386/i386_lock.s
index 838fd0ed8..d8ffd44e4 100644
--- a/osfmk/i386/i386_lock.s
+++ b/osfmk/i386/i386_lock.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -33,6 +39,9 @@
 #include <platforms.h>
 #include <mach_ldebug.h>
 #include <i386/asm.h>
+#include <i386/eflags.h>
+#include <i386/trap.h>
+#include <config_dtrace.h>
 
 #include "assym.s"
 
@@ -47,7 +56,7 @@
 
 #if	BUILD_STACK_FRAMES
 
-/* STack-frame-relative: */
+/* Stack-frame-relative: */
 #define	L_PC		B_PC
 #define	L_ARG0		B_ARG0
 #define	L_ARG1		B_ARG1
@@ -226,6 +235,90 @@
 #endif	/* MACH_LDEBUG */
 
 
+#define PREEMPTION_DISABLE				\
+	incl	%gs:CPU_PREEMPTION_LEVEL
+	
+	
+#define	PREEMPTION_ENABLE				\
+	decl	%gs:CPU_PREEMPTION_LEVEL	;	\
+	jne	9f				;	\
+	pushf					;	\
+	testl	$ EFL_IF,(%esp)			;	\
+	je	8f				;	\
+	cli					;	\
+	movl	%gs:CPU_PENDING_AST,%eax	;	\
+	testl	$ AST_URGENT,%eax		;	\
+	je	8f				;	\
+	movl	%gs:CPU_INTERRUPT_LEVEL,%eax	;	\
+	testl	%eax,%eax			;	\
+	jne	8f				;	\
+	popf					;	\
+	int	$(T_PREEMPT)			;	\
+	jmp	9f				;	\
+8:							\
+	popf					;	\
+9:	
+
+	
+
+#if	CONFIG_DTRACE
+#define	LOCKSTAT_LABEL(lab) \
+	.data				;\
+	.globl	lab			;\
+	lab:				;\
+	.long 9f			;\
+	.text				;\
+	9:
+
+	.globl	_lockstat_probe
+	.globl	_lockstat_probemap
+
+#define	LOCKSTAT_RECORD(id, lck) \
+	push	%ebp					;	\
+	mov	%esp,%ebp				;	\
+	sub	$0x38,%esp	/* size of dtrace_probe args */ ; \
+	movl	_lockstat_probemap + (id * 4),%eax	;	\
+	test	%eax,%eax				;	\
+	je	9f					;	\
+	movl	$0,36(%esp)				;	\
+	movl	$0,40(%esp)				;	\
+	movl	$0,28(%esp)				;	\
+	movl	$0,32(%esp)				;	\
+	movl	$0,20(%esp)				;	\
+	movl	$0,24(%esp)				;	\
+	movl	$0,12(%esp)				;	\
+	movl	$0,16(%esp)				;	\
+	movl	lck,4(%esp)	/* copy lock pointer to arg 1 */ ; \
+	movl	$0,8(%esp)				;	\
+	movl	%eax,(%esp) 				; 	\
+	call	*_lockstat_probe			;	\
+9:	leave
+	/* ret - left to subsequent code, e.g. return values */
+
+#define	LOCKSTAT_RECORD2(id, lck, arg) \
+	push	%ebp					;	\
+	mov	%esp,%ebp				;	\
+	sub	$0x38,%esp	/* size of dtrace_probe args */ ; \
+	movl	_lockstat_probemap + (id * 4),%eax	;	\
+	test	%eax,%eax				;	\
+	je	9f					;	\
+	movl	$0,36(%esp)				;	\
+	movl	$0,40(%esp)				;	\
+	movl	$0,28(%esp)				;	\
+	movl	$0,32(%esp)				;	\
+	movl	$0,20(%esp)				;	\
+	movl	$0,24(%esp)				;	\
+	movl	$0,12(%esp)				;	\
+	movl	$0,16(%esp)				;	\
+	movl	lck,4(%esp)	/* copy lock pointer to arg 1 */ ; \
+	movl	arg,8(%esp)				;	\
+	movl	%eax,(%esp) 				; 	\
+	call	*_lockstat_probe			;	\
+9:	leave
+	/* ret - left to subsequent code, e.g. return values */
+#endif
+
+
 /*
  *	void hw_lock_init(hw_lock_t)
  *
@@ -233,7 +326,18 @@
  */
 LEAF_ENTRY(hw_lock_init)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
-	movl	$0,0(%edx)		/* clear the lock */
+	movl	$0,(%edx)		/* clear the lock */
+	LEAF_RET
+
+
+/*
+ *	void hw_lock_byte_init(uint8_t *)
+ *
+ *	Initialize a hardware byte lock.
+ */
+LEAF_ENTRY(hw_lock_byte_init)
+	movl	L_ARG0,%edx		/* fetch lock pointer */
+	movb	$0,(%edx)		/* clear the lock */
 	LEAF_RET
 
 /*
@@ -246,12 +350,12 @@ LEAF_ENTRY(hw_lock_lock)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
 
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
-	DISABLE_PREEMPTION
+	PREEMPTION_DISABLE
 1:
-	movl	0(%edx), %eax
+	movl	(%edx), %eax
 	testl	%eax,%eax		/* lock locked? */
 	jne	3f			/* branch if so */
-	lock; cmpxchgl	%ecx,0(%edx)	/* try to acquire the HW lock */
+	lock; cmpxchgl	%ecx,(%edx)	/* try to acquire the HW lock */
 	jne	3f
 	movl	$1,%eax			/* In case this was a timeout call */
 	LEAF_RET			/* if yes, then nothing left to do */
@@ -259,6 +363,28 @@ LEAF_ENTRY(hw_lock_lock)
 	PAUSE				/* pause for hyper-threading */
 	jmp	1b			/* try again */
 
+/*
+ *	void	hw_lock_byte_lock(uint8_t *lock_byte)
+ *
+ *	Acquire byte sized lock operand, spinning until it becomes available.
+ *	MACH_RT:  also return with preemption disabled.
+ */
+
+LEAF_ENTRY(hw_lock_byte_lock)
+	movl	L_ARG0,%edx		/* Load lock pointer */
+	PREEMPTION_DISABLE
+	movl	$1, %ecx		/* Set lock value */
+1:
+	movb	(%edx), %al		/* Load byte at address */
+	testb	%al,%al			/* lock locked? */
+	jne	3f			/* branch if so */
+	lock; cmpxchgb	%cl,(%edx)	/* attempt atomic compare exchange */
+	jne	3f
+	LEAF_RET			/* if yes, then nothing left to do */
+3:
+	PAUSE				/* pause for hyper-threading */
+	jmp	1b			/* try again */
+
 /*
  *	unsigned int hw_lock_to(hw_lock_t, unsigned int)
  *
@@ -273,11 +399,11 @@ LEAF_ENTRY(hw_lock_to)
 	 * Attempt to grab the lock immediately
 	 * - fastpath without timeout nonsense.
 	 */
-	DISABLE_PREEMPTION
-	movl	0(%edx), %eax
+	PREEMPTION_DISABLE
+	movl	(%edx), %eax
 	testl	%eax,%eax		/* lock locked? */
 	jne	2f			/* branch if so */
-	lock; cmpxchgl	%ecx,0(%edx)	/* try to acquire the HW lock */
+	lock; cmpxchgl	%ecx,(%edx)	/* try to acquire the HW lock */
 	jne	2f			/* branch on failure */
 	movl	$1,%eax
 	LEAF_RET
@@ -306,7 +432,7 @@ LEAF_ENTRY(hw_lock_to)
 	mov	$(INNER_LOOP_COUNT),%edx
 5:
 	PAUSE				/* pause for hyper-threading */
-	movl	0(%edi),%eax		/* spin checking lock value in cache */
+	movl	(%edi),%eax		/* spin checking lock value in cache */
 	testl	%eax,%eax
 	je	6f			/* zero => unlocked, try to grab it */
 	decl	%edx			/* decrement inner loop count */
@@ -331,7 +457,7 @@ LEAF_ENTRY(hw_lock_to)
 	 * after contention.
 	 */
 	movl	%gs:CPU_ACTIVE_THREAD,%edx
-	lock; cmpxchgl	%edx,0(%edi)	/* try to acquire the HW lock */
+	lock; cmpxchgl	%edx,(%edi)	/* try to acquire the HW lock */
 	jne	4b			/* no - spin again */
 	movl	$1,%eax			/* yes */
 	pop	%ebx
@@ -346,10 +472,22 @@ LEAF_ENTRY(hw_lock_to)
  */
 LEAF_ENTRY(hw_lock_unlock)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
-	movl	$0,0(%edx)		/* clear the lock */
-	ENABLE_PREEMPTION
+	movl	$0,(%edx)		/* clear the lock */
+	PREEMPTION_ENABLE
 	LEAF_RET
+/*
+ *	void hw_lock_byte_unlock(uint8_t *lock_byte)
+ *
+ *	Unconditionally release byte sized lock operand.
+ *	MACH_RT:  release preemption level.
+ */
 
+LEAF_ENTRY(hw_lock_byte_unlock)
+	movl	L_ARG0,%edx		/* Load lock pointer */
+	movb	$0,(%edx)		/* Clear the lock byte */
+	PREEMPTION_ENABLE
+	LEAF_RET
+	
 /*
  *	void i386_lock_unlock_with_flush(hw_lock_t)
  *
@@ -364,10 +502,10 @@ LEAF_ENTRY(hw_lock_unlock)
  */
 LEAF_ENTRY(i386_lock_unlock_with_flush)
 	movl	L_ARG0,%edx		/* Fetch lock pointer */
-	movl	$0,0(%edx)		/* Clear the lock */
+	movl	$0,(%edx)		/* Clear the lock */
 	mfence				/* Serialize prior stores */
-	clflush	0(%edx)			/* Write back and invalidate line */
-	ENABLE_PREEMPTION
+	clflush	(%edx)			/* Write back and invalidate line */
+	PREEMPTION_ENABLE
 	LEAF_RET
 
 /*
@@ -378,18 +516,18 @@ LEAF_ENTRY(hw_lock_try)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
 
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
-	DISABLE_PREEMPTION
-	movl	0(%edx),%eax
+	PREEMPTION_DISABLE
+	movl	(%edx),%eax
 	testl	%eax,%eax
 	jne	1f
-	lock; cmpxchgl	%ecx,0(%edx)	/* try to acquire the HW lock */
+	lock; cmpxchgl	%ecx,(%edx)	/* try to acquire the HW lock */
 	jne	1f
 
 	movl	$1,%eax			/* success */
 	LEAF_RET
 
 1:
-	ENABLE_PREEMPTION		/* failure:  release preemption... */
+	PREEMPTION_ENABLE		/* failure:  release preemption... */
 	xorl	%eax,%eax		/* ...and return failure */
 	LEAF_RET
 
@@ -401,7 +539,7 @@ LEAF_ENTRY(hw_lock_try)
 LEAF_ENTRY(hw_lock_held)
 	movl	L_ARG0,%edx		/* fetch lock pointer */
 
-	movl	0(%edx),%eax		/* check lock value */
+	movl	(%edx),%eax		/* check lock value */
 	testl	%eax,%eax
 	movl	$1,%ecx
 	cmovne	%ecx,%eax		/* 0 => unlocked, 1 => locked */
@@ -423,26 +561,185 @@ LEAF_ENTRY(mutex_init)
 
 	LEAF_RET
 
-NONLEAF_ENTRY2(mutex_lock,_mutex_lock)
+/*
+ * Reader-writer lock fastpaths. These currently exist for the
+ * shared lock acquire and release paths (where they reduce overhead
+ * considerably)--more can be added as necessary (DRK).
+ */
+
+/*
+ * These should reflect the layout of the bitfield embedded within
+ * the lck_rw_t structure (see i386/locks.h).
+ */
+#define LCK_RW_INTERLOCK 0x1
+#define LCK_RW_WANT_UPGRADE 0x2
+#define LCK_RW_WANT_WRITE 0x4
+#define LCK_R_WAITING 0x8
+#define LCK_W_WAITING 0x10
+
+#define	RW_LOCK_SHARED_MASK ((LCK_RW_INTERLOCK<<16) |	\
+	((LCK_RW_WANT_UPGRADE|LCK_RW_WANT_WRITE) << 24))
+/*
+ *		void lck_rw_lock_shared(lck_rw_t*)
+ *
+ */
+
+Entry(lck_rw_lock_shared)
+	movl	S_ARG0, %edx
+1:
+	movl	(%edx), %eax		/* Load state bitfield and interlock */
+	testl	$(RW_LOCK_SHARED_MASK), %eax	/* Eligible for fastpath? */
+	jne	3f
+	movl	%eax, %ecx
+	incl	%ecx				/* Increment reader refcount */
+	lock
+	cmpxchgl %ecx, (%edx)			/* Attempt atomic exchange */
+	jne	2f
+
+#if	CONFIG_DTRACE
+	/*
+	 * Dtrace lockstat event: LS_LCK_RW_LOCK_SHARED_ACQUIRE
+	 * Implemented by swapping between return and no-op instructions.
+	 * See bsd/dev/dtrace/lockstat.c.
+	 */
+	LOCKSTAT_LABEL(_lck_rw_lock_shared_lockstat_patch_point)
+	ret
+	/* Fall thru when patched, counting on lock pointer in %edx  */
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, %edx)
+#endif
+	ret
+
+2:
+	PAUSE
+	jmp	1b
+3:
+	jmp	EXT(lck_rw_lock_shared_gen)
+
+
+/*
+ *		lck_rw_type_t lck_rw_done(lck_rw_t*)
+ *
+ */
+
+.data
+rwl_release_error_str:
+	.asciz	"Releasing non-exclusive RW lock without a reader refcount!"
+.text
+
+#define RW_LOCK_RELEASE_MASK ((LCK_RW_INTERLOCK<<16) |	\
+	((LCK_RW_WANT_UPGRADE|LCK_RW_WANT_WRITE|LCK_R_WAITING|LCK_W_WAITING) << 24))
+Entry(lck_rw_done)
+	movl	S_ARG0,	%edx
+1:
+	movl	(%edx), %eax		/* Load state bitfield and interlock */
+	testl	$(RW_LOCK_RELEASE_MASK), %eax	/* Eligible for fastpath? */
+	jne	3f
+	movl	%eax, %ecx
+	/* Assert refcount */
+	testl	$(0xFFFF), %ecx
+	jne	5f
+	movl	$(rwl_release_error_str), S_ARG0
+	jmp	EXT(panic)
+5:
+	decl	%ecx			/* Decrement reader count */
+	lock
+	cmpxchgl %ecx, (%edx)
+	jne	2f
+	movl	$(RW_SHARED), %eax	/* Indicate that the lock was shared */
+#if	CONFIG_DTRACE
+	/* Dtrace lockstat probe: LS_RW_DONE_RELEASE as reader */
+	LOCKSTAT_LABEL(_lck_rw_done_lockstat_patch_point)
+	ret
+	/*
+	 * Note: Dtrace's convention is 0 ==> reader, which is
+	 * a different absolute value than $(RW_SHARED)
+	 * %edx contains the lock address already from the above
+	 */
+	LOCKSTAT_RECORD2(LS_LCK_RW_DONE_RELEASE, %edx, $0)
+	movl	$(RW_SHARED), %eax	/* Indicate that the lock was shared */
+#endif
+	ret
+
+2:
+	PAUSE
+	jmp	1b
+3:
+	jmp	EXT(lck_rw_done_gen)
+
+
+NONLEAF_ENTRY2(mutex_lock_spin,_mutex_lock_spin)
 
 	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
 
 	CHECK_MUTEX_TYPE()
 	CHECK_NO_SIMPLELOCKS()
 	CHECK_PREEMPTION_LEVEL()
 
-	pushf				/* save interrupt state */
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Lmls_ilk_loop		/* no, go spin */
+Lmls_retry:
 	cli				/* disable interrupts */
-Lml_retry:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Lml_get_hw:
+	/* eax == 0 at this point */
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Lmls_ilk_fail		/* branch on failure to spin loop */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex locked? */
+	jne	Lml_fail		/* yes, fall back to a normal mutex lock */
+	movl	$(MUTEX_LOCKED_AS_SPIN),M_LOCKED	/* indicate ownership as a spin lock */
+	
+#if	MACH_LDEBUG
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	movl	%ecx,M_THREAD
+	movl	B_PC,%ecx
+	movl	%ecx,M_PC
+#endif
+	PREEMPTION_DISABLE
+	popf				/* restore interrupt state */
+	leave				/* return with the interlock held */
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_lock_spin_lockstat_patch_point)
+	ret
+	/* %edx contains the lock address from above */
+	LOCKSTAT_RECORD(LS_MUTEX_LOCK_SPIN_ACQUIRE, %edx)
+#endif
+	ret
+	
+Lmls_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Lmls_ilk_loop:
+	PAUSE
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Lml_ilk_fail		/* no - take the slow path */
+	je	Lmls_retry		/* yes, go for it */
+	jmp	Lmls_ilk_loop		/* no, keep spinning */
+
+
+NONLEAF_ENTRY2(mutex_lock,_mutex_lock)
+
+	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
+
+	CHECK_MUTEX_TYPE()
+	CHECK_NO_SIMPLELOCKS()
+	CHECK_PREEMPTION_LEVEL()
+
+	movl	M_ILK,%eax		/* is interlock held */
+	testl	%eax,%eax
+	jne	Lml_ilk_loop		/* yes, go do the spin loop */
+Lml_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Lml_get_hw		/* branch on failure to retry */
+	jne	Lml_ilk_fail		/* branch on failure to spin loop */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
@@ -456,7 +753,6 @@ Lml_acquire:
 	movl	B_PC,%ecx
 	movl	%ecx,M_PC
 #endif
-
 	cmpw	$0,M_WAITERS		/* are there any waiters? */
 	jne	Lml_waiters		/* yes, more work to do */
 Lml_return:
@@ -464,9 +760,19 @@ Lml_return:
 	movl	%eax,M_ILK
 
 	popf				/* restore interrupt state */
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_lock_lockstat_patch_point)
+	ret
+	/* %edx still contains the lock pointer */
+	LOCKSTAT_RECORD(LS_MUTEX_LOCK_ACQUIRE, %edx)
+#endif
+	ret
 
-	NONLEAF_RET
-
+	/*
+	 * We got the mutex, but there are waiters.  Update information
+	 * on waiters.
+	 */
 Lml_waiters:
 	pushl	%edx			/* save mutex address */
 	pushl	%edx
@@ -475,18 +781,21 @@ Lml_waiters:
 	popl	%edx			/* restore mutex address */
 	jmp	Lml_return
 
+Lml_restart:
 Lml_ilk_fail:
-	/*
-	 * Slow path: call out to do the spinning.
-	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Lml_retry		/* try again */
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Lml_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Lml_retry		/* yes, go try to grab it */
+	jmp	Lml_ilk_loop		/* no - keep spinning */
 
 Lml_fail:
 	/*
-	 n Check if the owner is on another processor and therefore
+	 * Check if the owner is on another processor and therefore
 	 * we should try to spin before blocking.
 	 */
 	testl	$(OnProc),ACT_SPF(%ecx)
@@ -506,22 +815,21 @@ Lml_fail:
 	pushf				/* restore interrupt state */
 
 	push	%edx			/* lock address */
-	call	EXT(lck_mtx_lock_spin)	/* call out to do spinning */
+	call	EXT(lck_mtx_lock_spinwait)	/* call out to do spinning */
 	addl	$4,%esp
 	movl	B_ARG0,%edx		/* refetch mutex address */
 
-	/* Re-acquire interlock */
-	cli				/* disable interrupts */
+	/* Re-acquire interlock - interrupts currently enabled */
+	movl	M_ILK,%eax		/* is interlock held */
+	testl	%eax,%eax
+	jne	Lml_ilk_reloop		/* yes, go do the spin loop */
 Lml_reget_retry:
+	cli				/* disable interrupts */
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Lml_reget_hw:
-	movl	M_ILK,%eax		/* read interlock */
-	testl	%eax,%eax		/* unlocked? */
-	jne	Lml_ilk_refail		/* no - slow path */
-
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Lml_reget_hw		/* branch on failure to retry */
+	jne	Lml_ilk_refail		/* branch on failure to spin loop */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex free? */
@@ -532,43 +840,112 @@ Lml_block:
 	pushl	M_LOCKED
 	pushl	%edx			/* push mutex address */
 	call	EXT(lck_mtx_lock_wait)	/* wait for the lock */
-	addl	$8,%esp
+	addl	$8,%esp			/* returns with interlock dropped */
 	movl	B_ARG0,%edx		/* refetch mutex address */
-	cli				/* ensure interrupts disabled */
-	jmp	Lml_retry		/* and try again */
+	jmp	Lml_restart		/* and start over */
 
 Lml_ilk_refail:
-	/*
-	 * Slow path: call out to do the spinning.
-	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Lml_reget_retry		/* try again */
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
 
-NONLEAF_ENTRY2(mutex_try,_mutex_try)	
+Lml_ilk_reloop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Lml_reget_retry		/* yes, go try to grab it */
+	jmp	Lml_ilk_reloop		/* no - keep spinning */
+
+	
+
+NONLEAF_ENTRY2(mutex_try_spin,_mutex_try_spin)	
 
 	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
 
 	CHECK_MUTEX_TYPE()
 	CHECK_NO_SIMPLELOCKS()
 
-	pushf				/* save interrupt state */
+	movl	M_ILK,%eax
+	testl	%eax,%eax		/* is the interlock held? */
+	jne	Lmts_ilk_loop		/* yes, go to spin loop */
+Lmts_retry:
 	cli				/* disable interrupts */
-Lmt_retry:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Lmt_get_hw:
+	/* eax == 0 at this point */
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Lmts_ilk_fail		/* branch on failure to spin loop */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex locked? */
+	jne	Lmt_fail		/* yes, we lose */
+Lmts_acquire:
+	movl	$(MUTEX_LOCKED_AS_SPIN),M_LOCKED	/* indicate ownership as a spin lock */
+
+#if	MACH_LDEBUG
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	movl	%ecx,M_THREAD
+	movl	B_PC,%ecx
+	movl	%ecx,M_PC
+#endif
+	PREEMPTION_DISABLE		/* no, return with interlock held */
+	popf				/* restore interrupt state */
+	movl	$1,%eax
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_try_spin_lockstat_patch_point)
+	ret
+	/* %edx inherits the lock pointer from above */
+	LOCKSTAT_RECORD(LS_MUTEX_TRY_SPIN_ACQUIRE, %edx)
+	movl	$1,%eax
+#endif
+	ret
+
+Lmts_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Lmts_ilk_loop:
+	PAUSE
+	/*
+	 * need to do this check outside of the interlock in
+	 * case this lock is held as a simple lock which means
+	 * we won't be able to take the interlock
+ 	 */
+	movl	M_LOCKED,%eax
+	testl	%eax,%eax		/* is the mutex locked? */
+	jne	Lmt_fail_no_ilk		/* yes, go return failure */
+
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Lmt_ilk_fail		/* no - slow path */
+	je	Lmts_retry		/* yes, go try to grab it */
+	jmp	Lmts_ilk_loop		/* keep spinning */
+
+
+
+NONLEAF_ENTRY2(mutex_try,_mutex_try)	
+
+	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
+
+	CHECK_MUTEX_TYPE()
+	CHECK_NO_SIMPLELOCKS()
+
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Lmt_ilk_loop		/* yes, go try to grab it */
+Lmt_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Lmt_get_hw		/* branch on failure to retry */
+	jne	Lmt_ilk_fail		/* branch on failure to spin loop */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
 	jne	Lmt_fail		/* yes, we lose */
+Lmt_acquire:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
@@ -577,8 +954,7 @@ Lmt_get_hw:
 	movl	B_PC,%ecx
 	movl	%ecx,M_PC
 #endif
-
-	cmpl	$0,M_WAITERS		/* are there any waiters? */
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
 	jne	Lmt_waiters		/* yes, more work to do */
 Lmt_return:
 	xorl	%eax,%eax
@@ -586,8 +962,15 @@ Lmt_return:
 	popf				/* restore interrupt state */
 
 	movl	$1,%eax
-
-	NONLEAF_RET
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_try_lockstat_patch_point)
+	ret
+	/* inherit the lock pointer in %edx from above */
+	LOCKSTAT_RECORD(LS_MUTEX_TRY_LOCK_ACQUIRE, %edx)
+	movl	$1,%eax
+#endif
+	ret
 
 Lmt_waiters:
 	pushl	%edx			/* save mutex address */
@@ -598,69 +981,152 @@ Lmt_waiters:
 	jmp	Lmt_return
 
 Lmt_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Lmt_ilk_loop:
+	PAUSE
 	/*
-	 * Slow path: call out to do the spinning.
-	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Lmt_retry		/* try again */
+	 * need to do this check outside of the interlock in
+	 * case this lock is held as a simple lock which means
+	 * we won't be able to take the interlock
+ 	 */
+	movl	M_LOCKED,%eax		/* get lock owner */
+	testl	%eax,%eax		/* is the mutex locked? */
+	jne	Lmt_fail_no_ilk		/* yes, go return failure */
+
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Lmt_retry		/* yes, go try to grab it */
+	jmp	Lmt_ilk_loop		/* no - keep spinning */
 
 Lmt_fail:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
 
+Lmt_fail_no_ilk:
+	xorl	%eax,%eax
 	popf				/* restore interrupt state */
+	NONLEAF_RET
 
-	xorl	%eax,%eax
 
-	NONLEAF_RET
+
+LEAF_ENTRY(mutex_convert_spin)
+	movl	L_ARG0,%edx		/* fetch lock pointer */
+
+	movl	M_LOCKED,%ecx		/* is this the spin variant of the mutex */
+	cmpl	$(MUTEX_LOCKED_AS_SPIN),%ecx
+	jne	Lmcs_exit		/* already owned as a mutex, just return */
+
+	movl	M_ILK,%ecx		/* convert from spin version to mutex */
+	movl	%ecx,M_LOCKED		/* take control of the mutex */
+	
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Lmcs_waiters		/* yes, more work to do */
+
+Lmcs_return:
+	xorl	%ecx,%ecx
+	movl	%ecx,M_ILK		/* clear interlock */
+	PREEMPTION_ENABLE
+Lmcs_exit:
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_convert_spin_lockstat_patch_point)
+	ret
+	/* inherit %edx from above */
+	LOCKSTAT_RECORD(LS_MUTEX_CONVERT_SPIN_ACQUIRE, %edx)
+#endif
+	ret
+
+
+Lmcs_waiters:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_acquire)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Lmcs_return
+
+	
 
 NONLEAF_ENTRY(mutex_unlock)
 	movl	B_ARG0,%edx		/* fetch lock pointer */
 
-	CHECK_MUTEX_TYPE()
-	CHECK_THREAD(M_THREAD)
+	movl	M_LOCKED,%ecx		/* is this the spin variant of the mutex */
+	cmpl	$(MUTEX_LOCKED_AS_SPIN),%ecx
+	jne	Lmu_enter		/* no, go treat like a real mutex */
+
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Lmus_wakeup		/* yes, more work to do */
+
+Lmus_drop_ilk:	
+	xorl	%ecx,%ecx
+	movl	%ecx,M_LOCKED		/* yes, clear the spin indicator */
+	movl	%ecx,M_ILK		/* release the interlock */
+	PREEMPTION_ENABLE		/* and re-enable preemption */
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_unlock_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx from above */
+	LOCKSTAT_RECORD(LS_MUTEX_UNLOCK_RELEASE, %edx)
+#endif
+	ret
 
+Lmus_wakeup:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx			/* push mutex address */
+	call	EXT(lck_mtx_unlockspin_wakeup)	/* yes, wake a thread */
+	addl	$4,%esp
+	popl	%edx			/* restore mutex pointer */
+	jmp	Lmus_drop_ilk
+
+Lmu_enter:
 	pushf				/* save interrupt state */
-	cli				/* disable interrupts */
-Lmu_retry:
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Lmu_get_hw:
+	CHECK_MUTEX_TYPE()
+	CHECK_THREAD(M_THREAD)
+
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Lmu_ilk_fail		/* no - slow path */
+	jne	Lmu_ilk_loop		/* yes, go try to grab it */
+Lmu_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Lmu_get_hw		/* branch on failure to retry */
+	jne	Lmu_ilk_fail		/* branch on failure to spin loop */
 
 	cmpw	$0,M_WAITERS		/* are there any waiters? */
 	jne	Lmu_wakeup		/* yes, more work to do */
 
 Lmu_doit:
-
 #if	MACH_LDEBUG
 	movl	$0,M_THREAD		/* disown thread */
 #endif
-
 	xorl	%ecx,%ecx
 	movl	%ecx,M_LOCKED		/* unlock the mutex */
-
-	movl	%ecx,M_ILK
-
+	movl	%ecx,M_ILK		/* release the interlock */
 	popf				/* restore interrupt state */
-
-	NONLEAF_RET
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_mutex_unlock2_lockstat_patch_point)
+	ret
+	/* inherit %edx from above */
+	LOCKSTAT_RECORD(LS_MUTEX_UNLOCK_RELEASE, %edx)
+#endif
+	ret
 
 Lmu_ilk_fail:
-	/*
-	 * Slow path: call out to do the spinning.
-	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Lmu_retry		/* try again */
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Lmu_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Lmu_retry		/* yes, go try to grab it */
+	jmp	Lmu_ilk_loop		/* no - keep spinning */
 
 Lmu_wakeup:
 	pushl	M_LOCKED
@@ -670,35 +1136,210 @@ Lmu_wakeup:
 	movl	B_ARG0,%edx		/* restore lock pointer */
 	jmp	Lmu_doit
 
+/*
+ *	void lck_mtx_assert(lck_mtx_t* l, unsigned int)
+ *	void _mutex_assert(mutex_t, unsigned int)
+ *	Takes the address of a lock, and an assertion type as parameters.
+ *	The assertion can take one of two forms determine by the type
+ *	parameter: either the lock is held by the current thread, and the
+ *	type is	LCK_MTX_ASSERT_OWNED, or it isn't and the type is
+ *	LCK_MTX_ASSERT_NOT_OWNED. Calls panic on assertion failure.
+ *	
+ */
+
+Entry(lck_mtx_assert)
+Entry(_mutex_assert)
+	movl	S_ARG0,%edx			/* Load lock address */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* Load current thread */
+
+	cmpl	$(MUTEX_IND),M_ITAG		/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx			/* If so, take indirection */
+
+	movl	M_LOCKED,%eax			/* Load lock word */
+	cmpl	$(MUTEX_LOCKED_AS_SPIN),%eax	/* check for spin variant */
+	cmove	M_ILK,%eax			/* yes, spin lock owner is in the interlock */
+
+	cmpl	$(MUTEX_ASSERT_OWNED),S_ARG1	/* Determine assert type */
+	jne	2f				/* Assert ownership? */
+	cmpl	%eax,%ecx			/* Current thread match? */
+	jne	3f				/* no, go panic */
+1:						/* yes, we own it */
+	ret					/* just return */
+2:
+	cmpl	%eax,%ecx			/* Current thread match? */
+	jne	1b				/* No, return */
+	movl	%edx,S_ARG1			/* Prep assertion failure */
+	movl	$(mutex_assert_owned_str),S_ARG0
+	jmp	4f
+3:
+	movl	%edx,S_ARG1			/* Prep assertion failure */
+	movl	$(mutex_assert_not_owned_str),S_ARG0
+4:
+	jmp	EXT(panic)
+
+.data
+mutex_assert_not_owned_str:
+	.asciz	"mutex (%p) not owned\n"
+mutex_assert_owned_str:
+	.asciz	"mutex (%p) owned\n"
+.text
+
+/* This preprocessor define controls whether the R-M-W update of the
+ * per-group statistics elements are atomic (LOCK-prefixed)
+ * Enabled by default.
+ */
+#define ATOMIC_STAT_UPDATES 1
+
+#if defined(ATOMIC_STAT_UPDATES)
+#define LOCK_IF_ATOMIC_STAT_UPDATES lock
+#else
+#define LOCK_IF_ATOMIC_STAT_UPDATES
+#endif /* ATOMIC_STAT_UPDATES */
+
+
 /*
  * lck_mtx_lock()
  * lck_mtx_try_lock()
  * lck_mutex_unlock()
+ * lck_mtx_lock_spin()
+ * lck_mtx_convert_spin()
  *
- * These are variants of mutex_lock(), mutex_try() and mutex_unlock() without
+ * These are variants of mutex_lock(), mutex_try(), mutex_unlock()
+ * mutex_lock_spin and mutex_convert_spin without
  * DEBUG checks (which require fields not present in lck_mtx_t's).
  */
-NONLEAF_ENTRY(lck_mtx_lock)
+
+NONLEAF_ENTRY(lck_mtx_lock_spin)
 
 	movl	B_ARG0,%edx		/* fetch lock pointer */
-	cmpl	$(MUTEX_IND),M_ITAG	/* is this indirect? */
-	cmove	M_PTR,%edx		/* yes - take indirection */
+	pushf				/* save interrupt state */
 
 	CHECK_NO_SIMPLELOCKS()
 	CHECK_PREEMPTION_LEVEL()
 
-	pushf				/* save interrupt state */
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llmls_eval_ilk		/* no, go see if indirect */
+Llmls_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+	/* eax == 0 at this point */
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Llmls_ilk_fail		/* branch on failure to spin loop */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex locked? */
+	jne	Llml_fail		/* yes, fall back to a normal mutex */
+
+Llmls_acquire:	
+	movl	$(MUTEX_LOCKED_AS_SPIN),M_LOCKED	/* indicate ownership as a spin lock */
+	PREEMPTION_DISABLE
+	popf				/* restore interrupt state */
+	NONLEAF_RET			/* return with the interlock held */
+
+Llmls_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Llmls_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llmls_retry		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llmls_ilk_loop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
+	 */
+	popl	%edx
+	jmp	Llmls_ilk_loop
+
+
+Llmls_eval_ilk:
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+	jne	Llmls_ilk_loop		/* If not, go to spin loop */
+
+Llmls_lck_ext:
+	pushl	%esi			/* Used to hold the lock group ptr */
+	pushl	%edi			/* Used for stat update records */
+	movl	MUTEX_GRP(%edx),%esi	/* Load lock group */
+	xorl	%edi,%edi		/* Clear stat update records */
+	/* 64-bit increment of acquire attempt statistic (per-group) */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	addl	$1, GRP_MTX_STAT_UTIL(%esi)
+	jnc	1f
+	incl	GRP_MTX_STAT_UTIL+4(%esi)
+1:
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llmls_ext_ilk_loop	/* no, go to spin loop */
+Llmls_ext_retry:
 	cli				/* disable interrupts */
-Llml_retry:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Llml_get_hw:
+	/* eax == 0 at this point */
+	lock; cmpxchgl %ecx,M_ILK	/* atomic compare and exchange */
+	jne     Llmls_ext_ilk_fail	/* branch on failure to retry */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl   %ecx,%ecx		/* is the mutex locked? */
+	jne	Llml_ext_fail		/* yes, we lose */
+
+	popl	%edi
+	popl	%esi
+	jmp	Llmls_acquire
+
+Llmls_ext_ilk_fail:
+	/*
+	 * Slow path: call out to do the spinning.
+	 */
+	movl	8(%esp),%ecx
+	pushl	%ecx
+	popf				/* restore interrupt state */
+	
+Llmls_ext_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llmls_ext_retry		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llmls_ext_ilk_loop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
+	 */
+	popl	%edx
+	jmp	Llmls_ext_ilk_loop	/* no - keep spinning  */
+
+	
+
+NONLEAF_ENTRY(lck_mtx_lock)
+
+	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
+
+	CHECK_NO_SIMPLELOCKS()
+	CHECK_PREEMPTION_LEVEL()
+
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Llml_ilk_fail		/* no - slow path */
+	jne	Llml_eval_ilk		/* no, go see if indirect */
+Llml_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Llml_get_hw		/* branch on failure to retry */
+	jne	Llml_ilk_fail		/* branch on failure to spin loop */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
@@ -707,15 +1348,21 @@ Llml_acquire:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
-	cmpl	$0,M_WAITERS		/* are there any waiters? */
-	jne	Llml_waiters		/* yes, more work to do */
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Lml_waiters		/* yes, more work to do */
 Llml_return:
 	xorl	%eax,%eax
 	movl	%eax,M_ILK
 
 	popf				/* restore interrupt state */
-
-	NONLEAF_RET
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_lck_mtx_lock_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE, %edx)
+#endif
+	ret
 
 Llml_waiters:
 	pushl	%edx			/* save mutex address */
@@ -725,14 +1372,27 @@ Llml_waiters:
 	popl	%edx			/* restore mutex address */
 	jmp	Llml_return
 
+Llml_restart:
 Llml_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Llml_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llml_retry		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llml_ilk_loop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
 	/*
-	 * Slow path: call out to do the spinning.
+	 * shouldn't return from here, but just in case
 	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Llml_retry		/* try again */
+	popl	%edx
+	jmp	Llml_ilk_loop		/* no - keep spinning  */
 
 Llml_fail:
 	/*
@@ -754,25 +1414,23 @@ Llml_fail:
 	movl	%eax,M_ILK		/* zero interlock */
 	popf
 	pushf				/* restore interrupt state */
-
 	pushl	%edx			/* save mutex address */
 	pushl	%edx
-	call	EXT(lck_mtx_lock_spin)
+	call	EXT(lck_mtx_lock_spinwait)
 	addl	$4,%esp
 	popl	%edx			/* restore mutex address */
 
 	/* Re-acquire interlock */
-	cli				/* disable interrupts */
-Llml_reget_retry:
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx
-
-Llml_reget_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Llml_ilk_refail		/* no - slow path */
+	jne	Llml_ilk_refail		/* no, go to spin loop */
+Llml_reget_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Llml_reget_hw		/* branch on failure to retry */
+	jne	Llml_ilk_refail		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex free? */
@@ -783,50 +1441,366 @@ Llml_block:
 	pushl	%edx			/* save mutex address */
 	pushl	M_LOCKED
 	pushl	%edx			/* push mutex address */
+	/*
+	 * N.B.: lck_mtx_lock_wait is called here with interrupts disabled
+	 * Consider reworking.
+	 */
 	call	EXT(lck_mtx_lock_wait)	/* wait for the lock */
 	addl	$8,%esp
 	popl	%edx			/* restore mutex address */
-	cli				/* ensure interrupts disabled */
-	jmp	Llml_retry		/* and try again */
+	jmp	Llml_restart		/* and start over */
 
 Llml_ilk_refail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state on stack */
+
+Llml_ilk_reloop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llml_reget_retry	/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llml_ilk_reloop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
 	/*
-	 * Slow path: call out to do the spinning.
+	 * shouldn't return from here, but just in case
 	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Llml_reget_retry	/* try again */
+	popl	%edx
+	jmp	Llml_ilk_reloop		/* no - keep spinning  */
 
-NONLEAF_ENTRY(lck_mtx_try_lock)
+
+Llml_eval_ilk:
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+	jne	Llml_ilk_loop		/* If not, go to spin loop */
+
+/*
+ * Entry into statistics codepath for lck_mtx_lock:
+ * EDX: real lock pointer
+ * first dword on stack contains flags
+ */
+
+/* Enable this preprocessor define to record the first miss alone
+ * By default, we count every miss, hence multiple misses may be
+ * recorded for a single lock acquire attempt via lck_mtx_lock
+ */
+#undef LOG_FIRST_MISS_ALONE	
+
+/*
+ * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
+ * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
+ * as a 64-bit quantity (this matches the existing PowerPC implementation,
+ * and the new x86 specific statistics are also maintained as 32-bit
+ * quantities).
+ */
+	
+Llml_lck_ext:
+	pushl	%esi			/* Used to hold the lock group ptr */
+	pushl	%edi			/* Used for stat update records */
+	movl	MUTEX_GRP(%edx),%esi	/* Load lock group */
+	xorl	%edi,%edi		/* Clear stat update records */
+	/* 64-bit increment of acquire attempt statistic (per-group) */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	addl	$1, GRP_MTX_STAT_UTIL(%esi)
+	jnc	1f
+	incl	GRP_MTX_STAT_UTIL+4(%esi)
+1:
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llml_ext_ilk_loop	/* no, go to spin loop */
+Llml_ext_get_hw:
+	cli
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+	/* eax == 0 at this point */
+	lock; cmpxchgl %ecx,M_ILK	/* atomic compare and exchange */
+	jne	Llml_ext_ilk_fail	/* branch on failure to retry */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex locked? */
+	jne	Llml_ext_fail		/* yes, we lose */
+
+Llml_ext_acquire:
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	movl	%ecx,M_LOCKED
+
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Llml_ext_waiters	/* yes, more work to do */
+Llml_ext_return:
+	xorl	%eax,%eax
+	movl	%eax,M_ILK
+
+	popl	%edi
+	popl	%esi
+	popf				/* restore interrupt state */
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_lck_mtx_lock_ext_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_ACQUIRE, %edx)
+#endif
+	ret
+
+Llml_ext_waiters:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_acquire)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Llml_ext_return
+
+Llml_ext_restart:
+Llml_ext_ilk_fail:
+	movl	8(%esp),%ecx
+	pushl	%ecx
+	popf				/* restore interrupt state */
+
+Llml_ext_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llml_ext_get_hw		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llml_ext_ilk_loop	/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
+	 */
+	popl	%edx
+	jmp	Llml_ext_ilk_loop
+
+
+Llml_ext_fail:
+#ifdef LOG_FIRST_MISS_ALONE
+	testl	$1, %edi
+	jnz	1f
+#endif /* LOG_FIRST_MISS_ALONE */
+	/* Record that a lock acquire attempt missed (per-group statistic) */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	incl	GRP_MTX_STAT_MISS(%esi)
+#ifdef LOG_FIRST_MISS_ALONE
+	orl	$1, %edi
+#endif /* LOG_FIRST_MISS_ALONE */
+1:
+	/*
+	 * Check if the owner is on another processor and therefore
+	 * we should try to spin before blocking.
+	 */
+	testl	$(OnProc),ACT_SPF(%ecx)
+	jnz	2f
+	/*
+	 * Record the "direct wait" statistic, which indicates if a
+	 * miss proceeded to block directly without spinning--occurs
+	 * if the owner of the mutex isn't running on another processor
+	 * at the time of the check.
+	 */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	incl	GRP_MTX_STAT_DIRECT_WAIT(%esi)
+	jmp	Llml_ext_block
+2:
+	/*
+	 * Here if owner is on another processor:
+	 *  - release the interlock
+	 *  - spin on the holder until release or timeout
+	 *  - in either case re-acquire the interlock
+	 *  - if released, acquire it
+	 *  - otherwise drop thru to block.
+	 */
+	xorl	%eax,%eax
+	movl	%eax,M_ILK		/* zero interlock */
+
+	pushl	8(%esp)			/* Make another copy of EFLAGS image */
+	popf				/* Restore interrupt state */
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_spinwait)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+
+	/* Re-acquire interlock */
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llml_ext_ilk_refail	/* no, go to spin loop */
+Llml_ext_reget_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+
+	/* eax == 0 at this point */
+	lock; cmpxchgl %ecx,M_ILK	/* atomic compare and exchange */
+	jne	Llml_ext_ilk_refail	/* branch on failure to spin loop */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex free? */
+	je	Llml_ext_acquire	/* yes, acquire */
+	
+Llml_ext_block:
+	/* If we wanted to count waits just once per lock acquire, we'd
+	 * skip over the stat update here
+	 */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	/* Record that a lock miss proceeded to block */
+	incl	GRP_MTX_STAT_WAIT(%esi) 
+1:
+	CHECK_MYLOCK(M_THREAD)
+	pushl	%edx			/* save mutex address */
+	pushl	M_LOCKED
+	pushl	%edx			/* push mutex address */
+	/*
+	 * N.B.: lck_mtx_lock_wait is called here with interrupts disabled
+	 * Consider reworking.
+	 */
+	call	EXT(lck_mtx_lock_wait)	/* wait for the lock */
+	addl	$8,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Llml_ext_restart	/* and start over */
+
+Llml_ext_ilk_refail:
+	movl	8(%esp),%ecx
+	pushl	%ecx
+	popf				/* restore interrupt state */
+	
+Llml_ext_ilk_reloop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llml_ext_reget_retry	/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llml_ext_ilk_reloop	/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
+	 */
+	popl	%edx
+	jmp	Llml_ext_ilk_reloop
+
+	
+
+NONLEAF_ENTRY(lck_mtx_try_lock_spin)
 
 	movl	B_ARG0,%edx		/* fetch lock pointer */
-	cmpl	$(MUTEX_IND),M_ITAG	/* is this indirect? */
-	cmove	M_PTR,%edx		/* yes - take indirection */
+	pushf				/* save interrupt state */
 
 	CHECK_NO_SIMPLELOCKS()
 	CHECK_PREEMPTION_LEVEL()
 
-	pushf				/* save interrupt state */
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	jne	Llmts_eval_ilk		/* no, go see if indirect */
+Llmts_retry:
 	cli				/* disable interrupts */
-Llmt_retry:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Llmt_get_hw:
+	/* eax == 0 at this point */
+	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
+	jne	Llmts_ilk_fail		/* branch on failure to retry */
+
+	movl	M_LOCKED,%ecx		/* get lock owner */
+	testl	%ecx,%ecx		/* is the mutex locked? */
+	jne	Llmt_fail		/* yes, we lose */
+
+	movl	$(MUTEX_LOCKED_AS_SPIN),M_LOCKED	/* no, indicate ownership as a spin lock */
+	PREEMPTION_DISABLE		/* and return with interlock held */
+
+	movl	$1,%eax			/* return success */
+	popf				/* restore interrupt state */
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_lck_mtx_try_lock_spin_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE, %edx)
+	movl	$1,%eax			/* return success */
+#endif
+	ret
+
+Llmts_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state */
+	
+Llmts_ilk_loop:
+	PAUSE
+	/*
+	 * need to do this check outside of the interlock in
+	 * case this lock is held as a simple lock which means
+	 * we won't be able to take the interlock
+ 	 */
+	movl	M_LOCKED,%eax		/* get lock owner */
+	testl	%eax,%eax		/* is the mutex locked? */
+	jne	Llmt_fail_no_ilk	/* yes, go return failure */
+
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llmts_retry		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llmts_ilk_loop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
+	 */
+	popl	%edx
+	jmp	Llmts_ilk_loop
+
+Llmts_eval_ilk:
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+	jne	Llmts_ilk_loop		/* If not, go to spin loop */
+
+	/*
+	 * bump counter on indirect lock
+	 */
+	pushl	%esi			/* Used to hold the lock group ptr */
+	movl	MUTEX_GRP(%edx),%esi	/* Load lock group */
+	/* 64-bit increment of acquire attempt statistic (per-group) */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	addl	$1, GRP_MTX_STAT_UTIL(%esi)
+	jnc	1f
+	incl	GRP_MTX_STAT_UTIL+4(%esi)
+1:
+	popl	%esi
+	jmp	Llmts_ilk_loop
+
+
+	
+NONLEAF_ENTRY(lck_mtx_try_lock)
+
+	movl	B_ARG0,%edx		/* fetch lock pointer */
+	pushf				/* save interrupt state */
+
+	CHECK_NO_SIMPLELOCKS()
+	CHECK_PREEMPTION_LEVEL()
+
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Llmt_ilk_fail		/* no - slow path */
+	jne	Llmt_eval_ilk		/* no, go see if indirect */
+Llmt_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Llmt_get_hw		/* branch on failure to retry */
+	jne	Llmt_ilk_fail		/* branch on failure to retry */
 
 	movl	M_LOCKED,%ecx		/* get lock owner */
 	testl	%ecx,%ecx		/* is the mutex locked? */
 	jne	Llmt_fail		/* yes, we lose */
+Llmt_acquire:
 	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 	movl	%ecx,M_LOCKED
 
-	cmpl	$0,M_WAITERS		/* are there any waiters? */
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
 	jne	Llmt_waiters		/* yes, more work to do */
 Llmt_return:
 	xorl	%eax,%eax
@@ -835,7 +1809,16 @@ Llmt_return:
 	popf				/* restore interrupt state */
 
 	movl	$1,%eax			/* return success */
-	NONLEAF_RET
+	leave
+#if	CONFIG_DTRACE
+	/* Dtrace probe: LS_LCK_MTX_TRY_LOCK_ACQUIRE */
+	LOCKSTAT_LABEL(_lck_mtx_try_lock_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx from above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_TRY_LOCK_ACQUIRE, %edx)
+	movl	$1,%eax			/* return success */
+#endif
+	ret
 
 Llmt_waiters:
 	pushl	%edx			/* save mutex address */
@@ -846,41 +1829,168 @@ Llmt_waiters:
 	jmp	Llmt_return
 
 Llmt_ilk_fail:
+	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state */
+	
+Llmt_ilk_loop:
+	PAUSE
 	/*
-	 * Slow path: call out to do the spinning.
+	 * need to do this check outside of the interlock in
+	 * case this lock is held as a simple lock which means
+	 * we won't be able to take the interlock
+ 	 */
+	movl	M_LOCKED,%eax		/* get lock owner */
+	testl	%eax,%eax		/* is the mutex locked? */
+	jne	Llmt_fail_no_ilk	/* yes, go return failure */
+
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llmt_retry		/* yes - go try to grab it */
+
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llmt_ilk_loop		/* no - keep spinning  */
+
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
+	/*
+	 * shouldn't return from here, but just in case
 	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Llmt_retry		/* try again */
+	popl	%edx
+	jmp	Llmt_ilk_loop
 
 Llmt_fail:
-	xorl	%eax,%eax
+	xorl	%eax,%eax		/* Zero interlock value */
 	movl	%eax,M_ILK
 
+Llmt_fail_no_ilk:
 	popf				/* restore interrupt state */
 
-	xorl	%eax,%eax		/* return failure */
+	cmpl	%edx,B_ARG0
+	jne	Llmt_fail_indirect
+
+	xorl	%eax,%eax
+	/* Note that we don't record a dtrace event for trying and missing */
+	NONLEAF_RET
+
+Llmt_fail_indirect:	
+	pushl	%esi			/* Used to hold the lock group ptr */
+	movl	MUTEX_GRP(%edx),%esi	/* Load lock group */
+
+	/* Record mutex acquire attempt miss statistic */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	incl	GRP_MTX_STAT_MISS(%esi)
+
+	popl	%esi
+	xorl	%eax,%eax
 	NONLEAF_RET
 
+Llmt_eval_ilk:
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+	jne	Llmt_ilk_loop		/* If not, go to spin loop */
+
+	/*
+	 * bump counter for indirect lock
+  	 */
+	pushl	%esi			/* Used to hold the lock group ptr */
+	movl	MUTEX_GRP(%edx),%esi	/* Load lock group */
+
+	/* 64-bit increment of acquire attempt statistic (per-group) */
+	LOCK_IF_ATOMIC_STAT_UPDATES
+	addl	$1, GRP_MTX_STAT_UTIL(%esi)
+	jnc	1f
+	incl	GRP_MTX_STAT_UTIL+4(%esi)
+1:
+	pop	%esi
+	jmp	Llmt_ilk_loop
+
+
+
+LEAF_ENTRY(lck_mtx_convert_spin)
+	movl	L_ARG0,%edx		/* fetch lock pointer */
+
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+
+	movl	M_LOCKED,%ecx		/* is this the spin variant of the mutex */
+	cmpl	$(MUTEX_LOCKED_AS_SPIN),%ecx
+	jne	Llmcs_exit		/* already owned as a mutex, just return */
+
+	movl	M_ILK,%ecx		/* convert from spin version to mutex */
+	movl	%ecx,M_LOCKED		/* take control of the mutex */
+
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Llmcs_waiters		/* yes, more work to do */
+
+Llmcs_return:
+	xorl	%ecx,%ecx
+	movl	%ecx,M_ILK		/* clear interlock */
+	PREEMPTION_ENABLE
+Llmcs_exit:
+	LEAF_RET
+
+Llmcs_waiters:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx
+	call	EXT(lck_mtx_lock_acquire)
+	addl	$4,%esp
+	popl	%edx			/* restore mutex address */
+	jmp	Llmcs_return
+	
+	
+
 NONLEAF_ENTRY(lck_mtx_unlock)
 
 	movl	B_ARG0,%edx		/* fetch lock pointer */
-	cmpl	$(MUTEX_IND),M_ITAG	/* is this indirect? */
-	cmove	M_PTR,%edx		/* yes - take indirection */
 
+	cmpl	$(MUTEX_IND),M_ITAG	/* Is this an indirect mutex? */
+	cmove	M_PTR,%edx		/* If so, take indirection */
+
+	movl	M_LOCKED,%ecx		/* is this the spin variant of the mutex */
+	cmpl	$(MUTEX_LOCKED_AS_SPIN),%ecx
+	jne	Llmu_enter		/* no, go treat like a real mutex */
+
+	cmpw	$0,M_WAITERS		/* are there any waiters? */
+	jne	Llmus_wakeup		/* yes, more work to do */
+
+Llmu_drop_ilk:
+	xorl	%eax,%eax
+	movl	%eax,M_LOCKED		/* clear spin indicator */
+	movl	%eax,M_ILK		/* release the interlock */
+
+	PREEMPTION_ENABLE		/* and re-enable preemption */
+	leave
+#if	CONFIG_DTRACE
+	/* Dtrace: LS_LCK_MTX_UNLOCK_RELEASE */
+	LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx from above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, %edx)
+#endif
+	ret
+	
+Llmus_wakeup:
+	pushl	%edx			/* save mutex address */
+	pushl	%edx			/* push mutex address */
+	call	EXT(lck_mtx_unlockspin_wakeup)	/* yes, wake a thread */
+	addl	$4,%esp
+	popl	%edx			/* restore mutex pointer */
+	jmp	Llmu_drop_ilk
+
+
+Llmu_enter:	
 	pushf				/* save interrupt state */
-	cli				/* disable interrupts */
-Llmu_retry:
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
-Llmu_get_hw:
 	movl	M_ILK,%eax		/* read interlock */
 	testl	%eax,%eax		/* unlocked? */
-	jne	Llmu_ilk_fail		/* no - slow path */
+	jne	Llmu_ilk_loop		/* no - go to spin loop */
+Llmu_retry:
+	cli				/* disable interrupts */
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
 
+	/* eax == 0 at this point */
 	lock; cmpxchgl	%ecx,M_ILK	/* atomic compare and exchange */
-	jne	Llmu_get_hw		/* branch on failure to retry */
+	jne	Llmu_ilk_fail		/* branch on failure to spin loop */
 
 	cmpw	$0,M_WAITERS		/* are there any waiters? */
 	jne	Llmu_wakeup		/* yes, more work to do */
@@ -888,21 +1998,38 @@ Llmu_get_hw:
 Llmu_doit:
 	xorl	%ecx,%ecx
 	movl	%ecx,M_LOCKED		/* unlock the mutex */
+	movl	%ecx,M_ILK		/* clear the interlock */
 
-	movl	%ecx,M_ILK
+	popf				/* restore interrupt state */
+	leave
+#if	CONFIG_DTRACE
+	LOCKSTAT_LABEL(_lck_mtx_unlock2_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, %edx)
+#endif
+	ret
 
+Llmu_ilk_fail:
 	popf				/* restore interrupt state */
+	pushf				/* resave interrupt state */
+	
+Llmu_ilk_loop:
+	PAUSE
+	movl	M_ILK,%eax		/* read interlock */
+	testl	%eax,%eax		/* unlocked? */
+	je	Llmu_retry		/* yes - go try to grab it */
 
-	NONLEAF_RET
+	cmpl	$(MUTEX_DESTROYED),%eax	/* check to see if its marked destroyed */
+	jne	Llmu_ilk_loop		/* no - keep spinning  */
 
-Llmu_ilk_fail:
+	pushl	%edx
+	call	EXT(lck_mtx_interlock_panic)
 	/*
-	 * Slow path: call out to do the spinning.
+	 * shouldn't return from here, but just in case
 	 */
-	pushl	%edx			/* lock address */
-	call	EXT(lck_mtx_interlock_spin)
-	popl	%edx			/* lock pointer */
-	jmp	Llmu_retry		/* try again */
+	popl	%edx
+	jmp	Llmu_ilk_loop
 
 Llmu_wakeup:
 	pushl	%edx			/* save mutex address */
@@ -911,7 +2038,23 @@ Llmu_wakeup:
 	call	EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */
 	addl	$8,%esp
 	popl	%edx			/* restore mutex pointer */
-	jmp	Llmu_doit
+	xorl	%ecx,%ecx
+	movl	%ecx,M_LOCKED		/* unlock the mutex */
+
+	movl	%ecx,M_ILK
+
+	popf				/* restore interrupt state */
+
+	leave
+#if	CONFIG_DTRACE
+	/* Dtrace: LS_LCK_MTX_EXT_UNLOCK_RELEASE */
+	LOCKSTAT_LABEL(_lck_mtx_ext_unlock_lockstat_patch_point)
+	ret
+	/* inherit lock pointer in %edx from above */
+	LOCKSTAT_RECORD(LS_LCK_MTX_EXT_UNLOCK_RELEASE, %edx)
+#endif
+	ret
+
 
 LEAF_ENTRY(lck_mtx_ilk_unlock)
 	movl	L_ARG0,%edx		/* no indirection here */
@@ -920,7 +2063,8 @@ LEAF_ENTRY(lck_mtx_ilk_unlock)
 	movl	%eax,M_ILK
 
 	LEAF_RET
-	
+
+
 LEAF_ENTRY(_disable_preemption)
 #if	MACH_RT
 	_DISABLE_PREEMPTION
@@ -1019,6 +2163,7 @@ LEAF_ENTRY(i_bit_clear)
 	btr	%edx,(%eax)
 	LEAF_RET
 
+
 LEAF_ENTRY(bit_lock)
 	movl	L_ARG0,%ecx
 	movl	L_ARG1,%eax
@@ -1028,6 +2173,7 @@ LEAF_ENTRY(bit_lock)
 	jb	1b
 	LEAF_RET
 
+
 LEAF_ENTRY(bit_lock_try)
 	movl	L_ARG0,%ecx
 	movl	L_ARG1,%eax
@@ -1045,3 +2191,71 @@ LEAF_ENTRY(bit_unlock)
 	lock
 	btr	%ecx,(%eax)
 	LEAF_RET
+
+/*
+ * Atomic primitives, prototyped in kern/simple_lock.h
+ */
+LEAF_ENTRY(hw_atomic_add)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	L_ARG1, %eax		/* Load addend */
+	movl	%eax, %edx
+	lock
+	xaddl	%eax, (%ecx)		/* Atomic exchange and add */
+	addl	%edx, %eax		/* Calculate result */
+	LEAF_RET
+
+LEAF_ENTRY(hw_atomic_sub)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	L_ARG1, %eax		/* Load subtrahend */
+	negl	%eax
+	movl	%eax, %edx
+	lock
+	xaddl	%eax, (%ecx)		/* Atomic exchange and add */
+	addl	%edx, %eax		/* Calculate result */
+	LEAF_RET
+
+LEAF_ENTRY(hw_atomic_or)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	(%ecx), %eax
+1:
+	movl	L_ARG1, %edx		/* Load mask */
+	orl	%eax, %edx
+	lock
+	cmpxchgl	%edx, (%ecx)	/* Atomic CAS */
+	jne	1b
+	movl	%edx, %eax		/* Result */
+	LEAF_RET
+/*
+ * A variant of hw_atomic_or which doesn't return a value.
+ * The implementation is thus comparatively more efficient.
+ */
+
+LEAF_ENTRY(hw_atomic_or_noret)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	L_ARG1, %edx		/* Load mask */
+	lock
+	orl	%edx, (%ecx)		/* Atomic OR */
+	LEAF_RET
+
+LEAF_ENTRY(hw_atomic_and)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	(%ecx), %eax
+1:
+	movl	L_ARG1, %edx		/* Load mask */
+	andl	%eax, %edx
+	lock
+	cmpxchgl	%edx, (%ecx)	/* Atomic CAS */
+	jne	1b
+	movl	%edx, %eax		/* Result */
+	LEAF_RET
+/*
+ * A variant of hw_atomic_and which doesn't return a value.
+ * The implementation is thus comparatively more efficient.
+ */
+
+LEAF_ENTRY(hw_atomic_and_noret)
+	movl	L_ARG0, %ecx		/* Load address of operand */
+	movl	L_ARG1, %edx		/* Load mask */
+	lock
+	andl	%edx, (%ecx)		/* Atomic OR */
+	LEAF_RET
diff --git a/osfmk/i386/i386_lowmem.h b/osfmk/i386/i386_lowmem.h
index 8a1fe906a..ffbb00b57 100644
--- a/osfmk/i386/i386_lowmem.h
+++ b/osfmk/i386/i386_lowmem.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_LOWMEM_H_
diff --git a/osfmk/i386/i386_vm_init.c b/osfmk/i386/i386_vm_init.c
index 2cf027867..9123aa771 100644
--- a/osfmk/i386/i386_vm_init.c
+++ b/osfmk/i386/i386_vm_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -78,7 +84,6 @@
 
 vm_size_t	mem_size = 0; 
 vm_offset_t	first_avail = 0;/* first after page tables */
-vm_offset_t	last_addr;
 
 uint64_t	max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
 uint64_t        mem_actual;
@@ -96,7 +101,7 @@ uint32_t	bounce_pool_size = 0;
 static void	reserve_bouncepool(uint32_t);
 
 
-pmap_paddr_t	avail_start, avail_end;
+pmap_paddr_t     avail_start, avail_end;
 vm_offset_t	virtual_avail, virtual_end;
 static pmap_paddr_t	avail_remaining;
 vm_offset_t     static_memory_end = 0;
@@ -253,7 +258,7 @@ i386_vm_init(uint64_t	maxmem,
 		        pmap_type = mptr->Type;
 		}
 
-		kprintf("EFI region: type = %d/%d,  base = 0x%x,  top = 0x%x\n", mptr->Type, pmap_type, base, top);
+		kprintf("EFI region: type = %u/%d,  base = 0x%x,  top = 0x%x\n", mptr->Type, pmap_type, base, top);
 
 		if (maxpg) {
 		        if (base >= maxpg)
@@ -264,11 +269,8 @@ i386_vm_init(uint64_t	maxmem,
 		/*
 		 * handle each region
 		 */
-		if (kEfiACPIMemoryNVS == pmap_type) {
-		        prev_pmptr = 0;
-			continue;
-		} else if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
-			   pmap_type != kEfiConventionalMemory) {
+		if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
+		    pmap_type != kEfiConventionalMemory) {
 		        prev_pmptr = 0;
 			continue;
 		} else {
@@ -394,11 +396,36 @@ i386_vm_init(uint64_t	maxmem,
 	 */
 	if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
 		ppnum_t discarded_pages  = (sane_size - maxmem) >> I386_PGSHIFT;
-		sane_size                = maxmem;
+		ppnum_t	highest_pn = 0;
+		ppnum_t	cur_alloc  = 0;
+		uint64_t	pages_to_use;
+		unsigned	cur_region = 0;
+
+		sane_size = maxmem;
+
 		if (avail_remaining > discarded_pages)
 			avail_remaining -= discarded_pages;
 		else
 			avail_remaining = 0;
+		
+		pages_to_use = avail_remaining;
+
+		while (cur_region < pmap_memory_region_count && pages_to_use) {
+		        for (cur_alloc = pmap_memory_regions[cur_region].alloc;
+			     cur_alloc < pmap_memory_regions[cur_region].end && pages_to_use;
+			     cur_alloc++) {
+			        if (cur_alloc > highest_pn)
+				        highest_pn = cur_alloc;
+				pages_to_use--;
+			}
+			if (pages_to_use == 0)
+			        pmap_memory_regions[cur_region].end = cur_alloc;
+
+			cur_region++;
+		}
+		pmap_memory_region_count = cur_region;
+
+		avail_end = i386_ptob(highest_pn + 1);
 	}
 
 	/*
@@ -411,7 +438,7 @@ i386_vm_init(uint64_t	maxmem,
 	        mem_size = (vm_size_t)sane_size;
 	max_mem = sane_size;
 
-	kprintf("Physical memory %d MB\n", sane_size/MEG);
+	kprintf("Physical memory %llu MB\n", sane_size/MEG);
 
 	if (!PE_parse_boot_arg("max_valid_dma_addr", &maxdmaaddr))
 	        max_valid_dma_address = 1024ULL * 1024ULL * 4096ULL;
@@ -439,9 +466,9 @@ i386_vm_init(uint64_t	maxmem,
 		        reserve_bouncepool(maxbouncepoolsize);
 
 		if (maxloreserve)
-			vm_lopage_poolsize = maxloreserve / PAGE_SIZE;
+		        vm_lopage_poolsize = maxloreserve / PAGE_SIZE;
 	}
-
+	
 	/*
 	 *	Initialize kernel physical map.
 	 *	Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
@@ -486,7 +513,7 @@ pmap_valid_page(
 
 	assert(pn);
 	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
-                if ( (pn >= pmptr->base) && (pn <= pmptr->end) && pmptr->type == kEfiConventionalMemory )
+	        if ( (pn >= pmptr->base) && (pn <= pmptr->end) )
 	                return TRUE;
 	}
 	return FALSE;
@@ -504,7 +531,7 @@ reserve_bouncepool(uint32_t bounce_pool_wanted)
 	pages_needed = bounce_pool_wanted / PAGE_SIZE;
 
 	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
-	        if ( (pmptr->type == kEfiConventionalMemory) && ((pmptr->end - pmptr->alloc) >= pages_needed) ) {
+	        if ( (pmptr->end - pmptr->alloc) >= pages_needed ) {
 		        if ( (lowest == NULL) || (pmptr->alloc < lowest->alloc) )
 			        lowest = pmptr;
 		}
diff --git a/osfmk/i386/idt.s b/osfmk/i386/idt.s
index ea73007fd..bc45d5d4a 100644
--- a/osfmk/i386/idt.s
+++ b/osfmk/i386/idt.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,7 +69,7 @@
  */
 #define	IDT_BASE_ENTRY(vec,seg,type) \
 	.data			;\
-	.long	EXT(vec) - EXT(hi_remap_text) + HIGH_MEM_BASE ;\
+	.long	EXT(vec) - EXT(hi_remap_text) + HIGH_MEM_BASE ; \
 	.word	seg		;\
 	.byte	0		;\
 	.byte	type		;\
@@ -71,7 +77,7 @@
 
 #define	IDT_BASE_ENTRY_INT(vec,seg,type) \
 	.data			;\
-	.long	vec - EXT(hi_remap_text) + HIGH_MEM_BASE ;\
+	.long	vec - EXT(hi_remap_text) + HIGH_MEM_BASE ; \
 	.word	seg		;\
 	.byte	0		;\
 	.byte	type		;\
@@ -79,7 +85,7 @@
 
 #define	IDT_BASE_ENTRY_TG(vec,seg,type) \
 	.data			;\
-	.long	0		;\
+	.long	0		; \
 	.word	seg		;\
 	.byte	0		;\
 	.byte	type		;\
@@ -316,7 +322,7 @@ INTERRUPT(0x7b)
 INTERRUPT(0x7c)
 INTERRUPT(0x7d)
 INTERRUPT(0x7e)
-INTERRUPT(0x7f)
+EXCEP_USR(0x7f, t_dtrace_ret)
 
 EXCEP_SPC_USR(0x80,hi_unix_scall)
 EXCEP_SPC_USR(0x81,hi_mach_scall)
@@ -542,15 +548,13 @@ EXT(ret_popl_ds):
 EXT(ret_iret):
         iret                            /* return from interrupt */
 fast_exit:
-	popl    %edx                    /* user return eip */
-        popl    %ecx                    /* pop and toss cs */
+	popl	%edx			/* user return eip */
+	popl	%ecx			/* pop and toss cs */
 	andl	$(~EFL_IF),(%esp)	/* clear intrs enabled, see sti below */
-        popf                            /* flags - carry denotes failure */
-        popl    %ecx                    /* user return esp */
+	popf				/* flags - carry denotes failure */
+	popl	%ecx			/* user return esp */
 	sti				/* interrupts enabled after sysexit */
-        sysexit
-
-/*******************************************************************************************************/
+	sysexit
 
 		
 Entry(hi_unix_scall)
@@ -606,10 +610,9 @@ Entry(hi_sysenter)
 	pushf				/* flags */
 	/*
 	* Clear, among others, the Nested Task (NT) flags bit;
-	* This is cleared by INT, but not by sysenter, which only
-	* clears RF, VM and IF.
+	* This is cleared by INT, but not by SYSENTER.
 	*/
-	pushl	$0
+	pushl   $0
 	popfl
 	pushl	$(SYSENTER_CS)		/* cs */
 hi_sysenter_2:
@@ -624,8 +627,8 @@ enter_lohandler:
 	pushl   %es
         pushl   %fs
         pushl   %gs
-enter_lohandler1:
 	pushl	$(SS_32)		/* 32-bit state flavor */
+enter_lohandler1:
 	mov	%ss,%eax
 	mov	%eax,%ds
 	mov	%eax,%fs
@@ -657,7 +660,7 @@ enter_lohandler1:
 2:
 	movl	R_TRAPNO(%esp),%ecx			// Get the interrupt vector
 	addl	$1,%gs:hwIntCnt(,%ecx,4)	// Bump the count
-	jmp	*%ebx
+	jmp		*%ebx
 
 	
 /*
@@ -830,6 +833,7 @@ push_fs:
 push_gs:
 	pushl	%gs			/* restore gs. */
 push_none:
+	pushl	$(SS_32)		/* 32-bit state flavor */
 	movl	%eax,R_TRAPNO(%esp)	/* set trap number */
 	movl	%edx,R_ERR(%esp)	/* set error code */
 					/* now treat as fault from user */
diff --git a/osfmk/i386/idt64.s b/osfmk/i386/idt64.s
index 7f9cdbb07..64df3f215 100644
--- a/osfmk/i386/idt64.s
+++ b/osfmk/i386/idt64.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <i386/asm.h>
 #include <i386/asm64.h>
@@ -35,7 +41,6 @@
  */
 #define	LO_ALLINTRS		EXT(lo_allintrs)
 #define	LO_ALLTRAPS		EXT(lo_alltraps)
-#define	LO_SYSENTER		EXT(lo_sysenter)
 #define	LO_SYSCALL		EXT(lo_syscall)
 #define	LO_UNIX_SCALL		EXT(lo_unix_scall)
 #define	LO_MACH_SCALL		EXT(lo_mach_scall)
@@ -168,7 +173,7 @@ EXCEP64_IST(0x0c,db_task_stk_fault64,1)
 EXCEP64_SPC(0x0c,hi64_stack_fault)
 #endif
 EXCEP64_SPC(0x0d,hi64_gen_prot)
-EXCEP64_ERR(0x0e,t64_page_fault)
+EXCEP64_SPC(0x0e, hi64_page_fault)
 EXCEPTION64(0x0f,t64_trap_0f)
 EXCEPTION64(0x10,t64_fpu_err)
 EXCEPTION64(0x11,t64_trap_11)
@@ -287,7 +292,7 @@ INTERRUPT64(0x7b)
 INTERRUPT64(0x7c)
 INTERRUPT64(0x7d)
 INTERRUPT64(0x7e)
-INTERRUPT64(0x7f)
+EXCEP64_USR(0x7f, t64_dtrace_ret)
 
 EXCEP64_SPC_USR(0x80,hi64_unix_scall)
 EXCEP64_SPC_USR(0x81,hi64_mach_scall)
@@ -481,9 +486,9 @@ Entry(lo64_ret_to_user)
 	movl	ACT_PCB_IDS(%ecx),%eax	/* Obtain this thread's debug state */
 	cmpl	$0,%eax			/* Is there a debug register context? */
 	je	2f 			/* branch if not */
-	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP	/* Are we a 64-bit task? */
+	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP /* Are we a 32-bit task? */
 	jne	1f
-	movl	DS_DR0(%eax), %ecx	/* If not, load the 32 bit DRs */
+	movl	DS_DR0(%eax), %ecx	/* If so, load the 32 bit DRs */
 	movl	%ecx, %db0
 	movl	DS_DR1(%eax), %ecx
 	movl	%ecx, %db1
@@ -614,14 +619,14 @@ EXT(ret32_iret):
         iretq				/* return from interrupt */
 
 L_fast_exit:
-	pop	%rdx                    /* user return eip */
-        pop	%rcx                    /* pop and toss cs */
+	pop	%rdx			/* user return eip */
+	pop	%rcx			/* pop and toss cs */
 	andl	$(~EFL_IF), (%rsp)	/* clear interrupts enable, sti below */
-        popf                            /* flags - carry denotes failure */
-        pop	%rcx                    /* user return esp */
+	popf				/* flags - carry denotes failure */
+	pop	%rcx			/* user return esp */
 	.code32
 	sti				/* interrupts enabled after sysexit */
-        sysexit				/* 32-bit sysexit */
+	sysexit				/* 32-bit sysexit */
 	.code64
 
 L_64bit_return:
@@ -674,7 +679,7 @@ L_sysret:
 	mov	ISF64_RIP-16(%rsp), %rcx
 	mov	ISF64_RFLAGS-16(%rsp), %r11
 	mov	ISF64_RSP-16(%rsp), %rsp
-        sysretq				/* return from system call */
+	sysretq				/* return from system call */
 
 /*
  * Common path to enter locore handlers.
@@ -750,7 +755,15 @@ L_syscall_continue:
 	movl	$(0), ISF64_TRAPNO(%rsp)	/* trapno */
 	movl	$(LO_SYSCALL), ISF64_TRAPFN(%rsp)
 	jmp	L_64bit_enter		/* this can only be a 64-bit task */
-	
+
+
+L_32bit_enter_check:
+	/*
+	 * Check we're not a confused 64-bit user.
+	 */
+	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
+	jne	L_64bit_entry_reject
+	jmp	L_32bit_enter
 /*
  * sysenter entry point
  * Requires user code to set up:
@@ -771,10 +784,9 @@ Entry(hi64_sysenter)
 	push	%rcx			/* uesp */
 	pushf				/* flags */
 	/*
-	* Clear, among others, the Nested Task (NT) flags bit;
-	* This is cleared by INT, but not by sysenter, which only
-	* clears RF, VM and IF.
-	*/
+	 * Clear, among others, the Nested Task (NT) flags bit;
+	 * this is zeroed by INT, but not by SYSENTER.
+	 */
 	push	$0
 	popf
 	push	$(SYSENTER_CS)		/* cs */
@@ -783,16 +795,39 @@ L_sysenter_continue:
 	push	%rdx			/* eip */
 	push	%rax			/* err/eax - syscall code */
 	push	$(0)
-	movl	$(LO_SYSENTER), ISF64_TRAPFN(%rsp)
 	orl	$(EFL_IF), ISF64_RFLAGS(%rsp)
-
-L_32bit_enter_check:
-	/*
-	 * Check we're not a confused 64-bit user.
-	 */
-	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
-	jne	L_64bit_entry_reject
-	/* fall through to 32-bit handler: */
+	movl	$(LO_MACH_SCALL), ISF64_TRAPFN(%rsp)
+	testl	%eax, %eax
+	js	L_32bit_enter_check
+	movl	$(LO_UNIX_SCALL), ISF64_TRAPFN(%rsp)
+ 	cmpl	$(TASK_MAP_32BIT), %gs:CPU_TASK_MAP
+ 	jne	L_64bit_entry_reject
+/* If the caller (typically LibSystem) has recorded the cumulative size of
+ * the arguments in EAX, copy them over from the user stack directly.
+ * We recover from exceptions inline--if the copy loop doesn't complete
+ * due to an exception, we fall back to copyin from compatibility mode.
+ * We can potentially extend this mechanism to mach traps as well (DRK).
+ */
+L_sysenter_copy_args:
+	testl	$(I386_SYSCALL_ARG_BYTES_MASK), %eax
+	jz	L_32bit_enter
+	xor	%r9, %r9
+	mov	%gs:CPU_UBER_ARG_STORE, %r8
+	movl	%eax, %r9d
+	mov	%gs:CPU_UBER_ARG_STORE_VALID, %r12
+	xor	%r10, %r10
+	shrl	$(I386_SYSCALL_ARG_DWORDS_SHIFT), %r9d
+	andl	$(I386_SYSCALL_ARG_DWORDS_MASK), %r9d
+	movl	$0, (%r12)
+EXT(hi64_sysenter_user_arg_copy):
+0:
+	movl	4(%rcx, %r10, 4), %r11d
+	movl	%r11d, (%r8, %r10, 4)
+	incl	%r10d
+	decl	%r9d
+	jnz	0b
+	movl	$1, (%r12)
+	/* Fall through to 32-bit handler */
 
 L_32bit_enter:
 	/*
@@ -958,6 +993,16 @@ L_64bit_enter_after_fault:
 
 	jmp	L_enter_lohandler2
 
+Entry(hi64_page_fault)
+	push	$(T_PAGE_FAULT)
+	movl	$(LO_ALLTRAPS), 4(%rsp)
+	cmpl	$(KERNEL_UBER_BASE_HI32), ISF64_RIP+4(%rsp)
+	jne	L_enter_lohandler
+	cmpl	$(EXT(hi64_sysenter_user_arg_copy)), ISF64_RIP(%rsp)
+	jne	L_enter_lohandler
+	mov	ISF64_RSP(%rsp), %rsp
+	jmp	L_32bit_enter
+
 /*
  * Debug trap.  Check for single-stepping across system call into
  * kernel.  If this is the case, taking the debug trap has turned
@@ -1075,6 +1120,10 @@ trap_check_kernel_exit:
 	cmpl	$(EXT(ret64_iret)), 16(%rsp)
 	je	L_fault_iret64
 
+	cmpl	$(EXT(hi64_sysenter_user_arg_copy)), ISF64_RIP(%rsp)
+	jne	hi64_take_trap
+	mov	ISF64_RSP(%rsp), %rsp
+	jmp	L_32bit_enter
 hi64_take_trap:
 	jmp	L_enter_lohandler
 
diff --git a/osfmk/i386/intel_read_fault.h b/osfmk/i386/intel_read_fault.h
deleted file mode 100644
index fbecc7966..000000000
--- a/osfmk/i386/intel_read_fault.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-
-#ifndef _INTEL_H_
-#define	_INTEL_H_
-
-#include <mach/kern_return.h>
-#include <vm/vm_map.h>
-
-/*
- * Exported functions
- */
-extern kern_return_t intel_read_fault(
-		vm_map_t	map,
-		vm_offset_t	vaddr);
-
-#endif	/* _INTEL_H_ */
diff --git a/osfmk/i386/io_emulate.c b/osfmk/i386/io_emulate.c
deleted file mode 100644
index a84c041bc..000000000
--- a/osfmk/i386/io_emulate.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/*
- */
-
-#include <platforms.h>
-#include <mach/boolean.h>
-#include <mach/port.h>
-#include <kern/thread.h>
-#include <kern/task.h>
-
-#include <ipc/ipc_port.h>
-#include <ipc/ipc_space.h>
-#include <ipc/ipc_right.h>
-#include <ipc/ipc_object.h>
-#include <ipc/ipc_entry.h>
-
-#include <i386/thread.h>
-#include <i386/io_port.h>
-#include <i386/io_emulate.h>
-#include <i386/iopb_entries.h>
-
-#if 1
-int
-emulate_io(
-	__unused x86_saved_state32_t		*regs,
-	__unused int				opcode,
-	__unused int				io_port)
-{
-	/* At the moment, we are not allowing I/O emulation 
-	 *
- 	 * FIXME - this should probably change due to 
-	 * the Window Server's need to map I/O ports into its space.
-	 */
-
-	return EM_IO_ERROR;
-}
-#else
-int
-emulate_io(
-	struct i386_saved_state	*regs,
-	int			opcode,
-	int			io_port)
-{
-	thread_t	thread = current_thread();
-	at386_io_lock_state();
-
-	if (iopl_emulate(regs, opcode, io_port))
-	    return EM_IO_DONE;
-
-	if (iopb_check_mapping(thread, iopl_device))
-	    return EM_IO_ERROR;
-
-	/*
-	 *	Check for send rights to the IOPL device port.
-	 */
-	if (iopl_device_port == IP_NULL)
-	    return EM_IO_ERROR;
-	{
-	    ipc_space_t	space = current_space();
-	    mach_port_name_t	name;
-	    ipc_entry_t	entry;
-	    boolean_t	has_rights = FALSE;
-	    ipc_entry_bits_t *capability;
-
-	    is_write_lock(space);
-	    assert(space->is_active);
-
-	    if (ipc_right_reverse(space, (ipc_object_t) iopl_device_port,
-				  &name, &entry, &capability)) {
-		/* iopl_device_port is locked and active */
-		if (capability[space->server_id] & MACH_PORT_TYPE_SEND)
-		    has_rights = TRUE;
-		ip_unlock(iopl_device_port);
-	    }
-
-	    is_write_unlock(space);
-	    if (!has_rights) {
-		return EM_IO_ERROR;
-	    }
-	}
-
-	/*
-	 * Map the IOPL port set into the thread.
-	 */
-
-	if (i386_io_port_add(thread, iopl_device)
-	    != KERN_SUCCESS) 
-		return EM_IO_ERROR;
-
-	/*
-	 * Make the thread use its IO_TSS to get the IO permissions;
-	 * it may not have had one before this.
-	 */
-	act_machine_switch_pcb(thread);
-
-	return EM_IO_RETRY;
-}
-#endif
diff --git a/osfmk/i386/io_emulate.h b/osfmk/i386/io_emulate.h
deleted file mode 100644
index 2eb6ce595..000000000
--- a/osfmk/i386/io_emulate.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon 
- * the rights to redistribute these changes.
- */
-
-/*
- */
-
-#ifndef	_I386_IO_EMULATE_H_
-#define	_I386_IO_EMULATE_H_
-
-/*
- * Return codes from IO emulation.
- */
-extern int	emulate_io(
-			x86_saved_state32_t	*regs,
-			int			opcode,
-			int			io_port);
-
-#define	EM_IO_DONE	0	/* IO instruction executed, proceed */
-#define	EM_IO_RETRY	1	/* IO port mapped, retry instruction */
-#define	EM_IO_ERROR	2	/* IO port not mapped */
-
-#endif	/* _I386_IO_EMULATE_H_ */
diff --git a/osfmk/i386/io_map.c b/osfmk/i386/io_map.c
index 988b0f8cd..40892578b 100644
--- a/osfmk/i386/io_map.c
+++ b/osfmk/i386/io_map.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/io_map_entries.h b/osfmk/i386/io_map_entries.h
index 338cb6954..0913ba060 100644
--- a/osfmk/i386/io_map_entries.h
+++ b/osfmk/i386/io_map_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/io_port.h b/osfmk/i386/io_port.h
index 4d0684917..0dccd723f 100644
--- a/osfmk/i386/io_port.h
+++ b/osfmk/i386/io_port.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/iopb.c b/osfmk/i386/iopb.c
deleted file mode 100644
index 6ae4acb7a..000000000
--- a/osfmk/i386/iopb.c
+++ /dev/null
@@ -1,662 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-
-/*
- */
-
-/*
- * Code to manipulate IO permission bitmaps.
- */
- 
-
-
-#include <mach/boolean.h>
-#include <mach/kern_return.h>
-
-#include <ipc/ipc_port.h>
-
-#include <kern/kalloc.h>
-#include <kern/lock.h>
-#include <kern/queue.h>
-#include <kern/thread.h>
-#include <kern/misc_protos.h>
-
-
-#include <i386/io_port.h>
-#include <i386/iopb.h>
-#include <i386/seg.h>
-#include <i386/iopb_entries.h>
-
-void
-iopb_init(void)
-{
-}
-
-
-void
-iopb_destroy(__unused iopb_tss_t      io_tss)
-{
-}
-
-#if 0				/* Code removed until better solution comes on board */
-/*
- * A set of ports for an IO device.
- */
-struct io_port {
-	device_t	device;		/* Mach device */
-	queue_chain_t	dev_list;	/* link in device list */
-	queue_chain_t	io_use_list;	/* List of threads that use it */
-	io_reg_t	*io_port_list;	/* list of IO ports that use it */
-					/* list ends with IO_REG_NULL */
-};
-typedef struct io_port	*io_port_t;
-
-/*
- * Lookup table for device -> io_port mapping
- * (a linked list - I don't expect too many)
- */
-queue_head_t	device_to_io_port_list;
-
-/*
- * Cross-reference:
- *	all threads that have IO ports mapped
- *	all IO ports that have threads mapped
- */
-struct io_use {
-	queue_chain_t	psq;	/* Links from port set */
-	queue_chain_t	tsq;	/* links from tss */
-	io_port_t	ps;	/* Port set */
-	iopb_tss_t	ts;	/* Task segment */
-};
-typedef	struct io_use	*io_use_t;
-
-/*
- * Big lock for the whole mess.
- */
-decl_simple_lock_data(,iopb_lock)
-
-/* Forward */
-
-extern void		io_bitmap_init(
-				isa_iopb		bp);
-extern void		io_bitmap_set(
-				isa_iopb		bp,
-				io_reg_t		*bit_list);
-extern void		io_bitmap_clear(
-				isa_iopb		bp,
-				io_reg_t		*bit_list);
-extern io_port_t	device_to_io_port_lookup(
-				device_t		device);
-extern void		io_tss_init(
-				iopb_tss_t		io_tss);
-
-
-/*
- * Initialize the package.
- */
-void
-iopb_init(void)
-{
-	queue_init(&device_to_io_port_list);
-	simple_lock_init(&iopb_lock, 0);
-}
-
-/*
- * Initialize bitmap (set all bits to OFF == 1)
- */
-void
-io_bitmap_init(
-	isa_iopb	bp)
-{
-	register unsigned char *b;
-	register int	s;
-
-	s = sizeof(isa_iopb);
-	b = bp;
-
-	do {
-	    *b++ = ~0;
-	} while (--s >= 0);
-}
-
-/*
- * Set selected bits in bitmap to ON == 0
- */
-void
-io_bitmap_set(
-	isa_iopb	bp,
-	io_reg_t	*bit_list)
-{
-	io_reg_t	io_bit;
-
-	while ((io_bit = *bit_list++) != IO_REG_NULL) {
-	    bp[io_bit>>3] &= ~(1 << (io_bit & 0x7));
-	}
-}
-
-/*
- * Set selected bits in bitmap to OFF == 1
- */
-void
-io_bitmap_clear(
-	isa_iopb	bp,
-	io_reg_t	*bit_list)
-{
-	io_reg_t	io_bit;
-
-	while ((io_bit = *bit_list++) != IO_REG_NULL) {
-	    bp[io_bit>>3] |= (1 << (io_bit & 0x7));
-	}
-}
-
-/*
- * Lookup an io-port set by device
- */
-io_port_t
-device_to_io_port_lookup(
-	device_t	device)
-{
-	register io_port_t io_port;
-
-	queue_iterate(&device_to_io_port_list, io_port, io_port_t, dev_list) {
-	    if (io_port->device == device) {
-		return io_port;
-	    }
-	}
-	return 0;
-}
-
-/*
- * [exported]
- * Create an io_port set
- */
-void
-io_port_create(
-	device_t	device,
-	io_reg_t	*io_port_list)
-{
-	register io_port_t io_port, old_io_port;
-
-	io_port = (io_port_t) kalloc(sizeof(struct io_port));
-
-	simple_lock(&iopb_lock);
-	if (device_to_io_port_lookup(device) != 0) {
-	    simple_unlock(&iopb_lock);
-	    kfree((vm_offset_t) io_port, sizeof(struct io_port));
-	    return;
-	}
-
-	io_port->device = device;
-	queue_init(&io_port->io_use_list);
-	io_port->io_port_list = io_port_list;
-
-	/*
-	 * Enter in lookup list.
-	 */
-	queue_enter(&device_to_io_port_list, io_port, io_port_t, dev_list);
-
-	simple_unlock(&iopb_lock);
-}
-
-/*
- * [exported]
- * Destroy an io port set, removing any IO mappings.
- */
-void
-io_port_destroy(
-	device_t	device)
-{
-	io_port_t	io_port;
-	io_use_t	iu;
-
-	simple_lock(&iopb_lock);
-	io_port = device_to_io_port_lookup(device);
-	if (io_port == 0) {
-	    simple_unlock(&iopb_lock);
-	    return;
-	}
-
-	queue_iterate(&io_port->io_use_list, iu, io_use_t, psq) {
-	    iopb_tss_t	io_tss;
-	    io_tss = iu->ts;
-	    io_bitmap_clear(io_tss->bitmap, io_port->io_port_list);
-	    queue_remove(&io_tss->io_port_list, iu, io_use_t, tsq);
-	}
-	queue_remove(&device_to_io_port_list, io_port, io_port_t, dev_list);
-	simple_unlock(&iopb_lock);
-
-	while (!queue_empty(&io_port->io_use_list)) {
-	    iu = (io_use_t) queue_first(&io_port->io_use_list);
-	    queue_remove(&io_port->io_use_list, iu, io_use_t, psq);
-	    kfree((vm_offset_t)iu, sizeof(struct io_use));
-	}
-
-	kfree((vm_offset_t)io_port, sizeof(struct io_port));
-}
-
-/*
- * Initialize an IO TSS.
- */
-void
-io_tss_init(
-	iopb_tss_t	io_tss)
-{
-	vm_offset_t	addr = (vm_offset_t) io_tss;
-	vm_size_t	size = (char *)&io_tss->barrier - (char *)io_tss;
-
-	bzero((char *)&io_tss->tss, sizeof(struct i386_tss));
-	io_tss->tss.io_bit_map_offset
-			= (char *)&io_tss->bitmap - (char *)io_tss;
-	io_tss->tss.ss0 = KERNEL_DS;
-	io_bitmap_init(io_tss->bitmap);
-	io_tss->barrier = ~0;
-	queue_init(&io_tss->io_port_list);
-	addr |= LINEAR_KERNEL_ADDRESS;
-	io_tss->iopb_desc[0] = ((size-1) & 0xffff)
-		| ((addr & 0xffff) << 16);
-	io_tss->iopb_desc[1] = ((addr & 0x00ff0000) >> 16)
-		| ((ACC_TSS|ACC_PL_K|ACC_P) << 8)
-		| ((size-1) & 0x000f0000)
-		| (addr & 0xff000000);
-}
-
-/*
- * [exported]
- * Create an IOPB_TSS
- */
-iopb_tss_t
-iopb_create(void)
-{
-	register iopb_tss_t ts;
-
-	ts = (iopb_tss_t) kalloc(sizeof (struct iopb_tss));
-	io_tss_init(ts);
-	return (ts);
-}
-
-/*
- * [exported]
- * Destroy an IOPB_TSS
- */
-void
-iopb_destroy(
-	iopb_tss_t	io_tss)
-{
-	io_use_t	iu;
-	io_port_t	io_port;
-
-	simple_lock(&iopb_lock);
-
-	queue_iterate(&io_tss->io_port_list, iu, io_use_t, tsq) {
-	    io_port = iu->ps;
-	    /* skip bitmap clear - entire bitmap will vanish */
-	    queue_remove(&io_port->io_use_list, iu, io_use_t, psq);
-	}
-
-	simple_unlock(&iopb_lock);
-
-	while (!queue_empty(&io_tss->io_port_list)) {
-	    iu = (io_use_t) queue_first(&io_tss->io_port_list);
-	    queue_remove(&io_tss->io_port_list, iu, io_use_t, tsq);
-	    kfree((vm_offset_t)iu, sizeof(struct io_use));
-	}
-
-	kfree((vm_offset_t)io_tss, sizeof(struct iopb_tss));
-}
-
-/*
- * Add an IO mapping to a thread.
- */
-kern_return_t
-i386_io_port_add(
-	thread_t	thread,
-	device_t	device)
-{
-	pcb_t		pcb;
-	iopb_tss_t	io_tss, new_io_tss;
-	io_port_t	io_port;
-	io_use_t	iu, old_iu;
-
-	if (thread == THREAD_NULL
-	 || device == DEVICE_NULL)
-	    return KERN_INVALID_ARGUMENT;
-
-	pcb = thread->machine.pcb;
-
-	new_io_tss = 0;
-	iu = (io_use_t) kalloc(sizeof(struct io_use));
-
-    Retry:
-	simple_lock(&iopb_lock);
-
-	/* find the io_port_t for the device */
-	io_port = device_to_io_port_lookup(device);
-	if (io_port == 0) {
-	    /*
-	     * Device does not have IO ports available.
-	     */
-	    simple_unlock(&iopb_lock);
-	    if (new_io_tss)
-		kfree((vm_offset_t)new_io_tss, sizeof(struct iopb_tss));
-	    kfree((vm_offset_t) iu, sizeof(struct io_use));
-	    return KERN_INVALID_ARGUMENT;
-	}
-
-	/* Have the IO port. */
-
-	/* Make sure the thread has a TSS. */
-
-	simple_lock(&pcb->lock);
-	io_tss = pcb->io_tss;
-	if (io_tss == 0) {
-	    if (new_io_tss == 0) {
-		/*
-		 * Allocate an IO-tss.
-		 */
-		simple_unlock(&pcb->lock);
-		simple_unlock(&iopb_lock);
-
-		new_io_tss = (iopb_tss_t) kalloc(sizeof(struct iopb_tss));
-		io_tss_init(new_io_tss);
-
-		goto Retry;
-	    }
-	    io_tss = new_io_tss;
-	    pcb->io_tss = io_tss;
-	    new_io_tss = 0;
-	}
-
-	/*
-	 * Have io_port and io_tss.
-	 * See whether device is already mapped.
-	 */
-	queue_iterate(&io_tss->io_port_list, old_iu, io_use_t, tsq) {
-	    if (old_iu->ps == io_port) {
-		/*
-		 * Already mapped.
-		 */
-		simple_unlock(&pcb->lock);
-		simple_unlock(&iopb_lock);
-
-		kfree((vm_offset_t)iu, sizeof(struct io_use));
-		if (new_io_tss)
-		    kfree((vm_offset_t)new_io_tss, sizeof(struct iopb_tss));
-		return KERN_SUCCESS;
-	    }
-	}
-
-	/*
-	 * Add mapping.
-	 */
-	iu->ps = io_port;
-	iu->ts = io_tss;
-	queue_enter(&io_port->io_use_list, iu, io_use_t, psq);
-	queue_enter(&io_tss->io_port_list, iu, io_use_t, tsq);
-	io_bitmap_set(io_tss->bitmap, io_port->io_port_list);
-
-	simple_unlock(&pcb->lock);
-	simple_unlock(&iopb_lock);
-
-	if (new_io_tss)
-	    kfree((vm_offset_t)new_io_tss, sizeof(struct iopb_tss));
-	return KERN_SUCCESS;
-
-}
-
-/*
- * Remove an IO mapping from a thread.
- */
-kern_return_t
-i386_io_port_remove(
-	thread_t	thread,
-	device_t	device)
-{
-	pcb_t		pcb;
-	iopb_tss_t	io_tss;
-	io_port_t	io_port;
-	io_use_t	iu;
-
-	if (thread == THREAD_NULL
-	 || device == DEVICE_NULL)
-	    return KERN_INVALID_ARGUMENT;
-
-	pcb = thread->machine.pcb;
-
-	simple_lock(&iopb_lock);
-
-	/* find the io_port_t for the device */
-
-	io_port = device_to_io_port_lookup(device);
-	if (io_port == 0) {
-	    /*
-	     * Device does not have IO ports available.
-	     */
-	    simple_unlock(&iopb_lock);
-	    return KERN_INVALID_ARGUMENT;
-	}
-
-	simple_lock(&pcb->lock);
-	io_tss = pcb->io_tss;
-	if (io_tss == 0) {
-	    simple_unlock(&pcb->lock);
-	    simple_unlock(&iopb_lock);
-	    return KERN_INVALID_ARGUMENT;	/* not mapped */
-	}
-
-	/*
-	 * Find the mapping.
-	 */
-	queue_iterate(&io_tss->io_port_list, iu, io_use_t, tsq) {
-	    if (iu->ps == io_port) {
-		/*
-		 * Found mapping.  Remove it.
-		 */
-		io_bitmap_clear(io_tss->bitmap, io_port->io_port_list);
-
-		queue_remove(&io_port->io_use_list, iu, io_use_t, psq);
-		queue_remove(&io_tss->io_port_list, iu, io_use_t, tsq);
-
-		simple_unlock(&pcb->lock);
-		simple_unlock(&iopb_lock);
-
-		kfree((vm_offset_t)iu, sizeof(struct io_use));
-
-		return KERN_SUCCESS;
-	    }
-	}
-
-	/*
-	 * No mapping.
-	 */
-	return KERN_INVALID_ARGUMENT;
-}
-
-/*
- * Return the IO ports mapped into a thread.
- */
-
-kern_return_t
-i386_io_port_list(thread, list, list_count)
-	thread_t	thread;
-	device_t	**list;
-	unsigned int	*list_count;
-{
-	register pcb_t	pcb;
-	register iopb_tss_t io_tss;
-	unsigned int	count, alloc_count;
-	device_t	*devices;
-	vm_size_t	size_needed, size;
-	vm_offset_t	addr;
-	int		i;
-
-	if (thread == THREAD_NULL)
-	    return KERN_INVALID_ARGUMENT;
-
-	pcb = thread->machine.pcb;
-
-	alloc_count = 16;		/* a guess */
-
-	do {
-	    size_needed = alloc_count * sizeof(ipc_port_t);
-	    if (size_needed <= size)
-		break;
-
-	    if (size != 0)
-		kfree(addr, size);
-
-	    assert(size_needed > 0);
-	    size = size_needed;
-
-	    addr = kalloc(size);
-	    if (addr == 0)
-		return KERN_RESOURCE_SHORTAGE;
-
-	    devices = (device_t *)addr;
-	    count = 0;
-
-	    simple_lock(&iopb_lock);
-	    simple_lock(&pcb->lock);
-	    io_tss = pcb->io_tss;
-	    if (io_tss != 0) {
-		register io_use_t iu;
-
-		queue_iterate(&io_tss->io_port_list, iu, io_use_t, tsq) {
-		    if (++count < alloc_count) {
-			*devices = iu->ps->device;
-			device_reference(*devices);
-			devices++;
-		    }
-		}
-	    }
-	    simple_unlock(&pcb->lock);
-	    simple_unlock(&iopb_lock);
-	} while (count > alloc_count);
-
-	if (count == 0) {
-	    /*
-	     * No IO ports
-	     */
-	    *list = 0;
-	    *list_count = 0;
-
-	    if (size != 0)
-		kfree(addr, size);
-	}
-	else {
-	    /*
-	     * If we allocated too much, must copy.
-	     */
-	    size_needed = count * sizeof(ipc_port_t);
-	    if (size_needed < size) {
-		vm_offset_t	new_addr;
-
-		new_addr = kalloc(size_needed);
-		if (new_addr == 0) {
-		    for (i = 0; i < count; i++)
-			device_deallocate(devices[i]);
-		    kfree(addr, size);
-		    return KERN_RESOURCE_SHORTAGE;
-		}
-
-		bcopy((char *)addr, (char *)new_addr, size_needed);
-		kfree(addr, size);
-		devices = (device_t *)new_addr;
-	    }
-
-	    for (i = 0; i < count; i++)
-		((ipc_port_t *)devices)[i] =
-			convert_device_to_port(devices[i]);
-	}
-	*list = devices;
-	*list_count = count;
-
-	return KERN_SUCCESS;
-}
-
-/*
- * Check whether an IO device is mapped to a particular thread.
- * Used to support the 'iopl' device automatic mapping.
- */
-boolean_t
-iopb_check_mapping(
-	thread_t	thread,
-	device_t	device)
-{
-	pcb_t		pcb;
-	io_port_t	io_port;
-	io_use_t	iu;
-
-	pcb = thread->machine.pcb;
-
-	simple_lock(&iopb_lock);
-
-	/* Find the io port for the device */
-
-	io_port = device_to_io_port_lookup(device);
-	if (io_port == 0) {
-	    simple_unlock(&iopb_lock);
-	    return FALSE;
-	}
-
-	/* Look up the mapping in the device`s mapping list. */
-
-	queue_iterate(&io_port->io_use_list, iu, io_use_t, psq) {
-	    if (iu->ts == pcb->io_tss) {
-		/*
-		 * Device is mapped.
-		 */
-		simple_unlock(&iopb_lock);
-		return TRUE;
-	    }
-	}
-	simple_unlock(&iopb_lock);
-	return FALSE;
-}
-#endif
diff --git a/osfmk/i386/iopb.h b/osfmk/i386/iopb.h
index bd9705115..e73d13933 100644
--- a/osfmk/i386/iopb.h
+++ b/osfmk/i386/iopb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/iopb_entries.h b/osfmk/i386/iopb_entries.h
deleted file mode 100644
index c6b638d2f..000000000
--- a/osfmk/i386/iopb_entries.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-
-#if 0
-extern boolean_t	iopb_check_mapping(
-					thread_t	thread,
-					device_t	device);
-extern kern_return_t	i386_io_port_add(
-					thread_t	thread,
-					device_t	device);
-extern kern_return_t	i386_io_port_remove(
-					thread_t	thread,
-					device_t	device);
-extern kern_return_t	i386_io_port_list(
-					thread_t	thread,
-					device_t	** list,
-					unsigned int	* list_count);
-#endif
-extern void		iopb_init(void);
-extern iopb_tss_t	iopb_create(void);
-extern void		iopb_destroy(
-					iopb_tss_t	iopb);
diff --git a/osfmk/i386/ipl.h b/osfmk/i386/ipl.h
index 1eded56e4..76c8c2d1d 100644
--- a/osfmk/i386/ipl.h
+++ b/osfmk/i386/ipl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/ktss.c b/osfmk/i386/ktss.c
index 24d031229..f0b69591c 100644
--- a/osfmk/i386/ktss.c
+++ b/osfmk/i386/ktss.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -55,7 +61,6 @@
  *
  * We don't use the i386 task switch mechanism.  We need a TSS
  * only to hold the kernel stack pointer for the current thread.
- *
  */
 #include <i386/tss.h>
 #include <i386/seg.h>
@@ -110,8 +115,6 @@ struct x86_64_tss master_ktss64 __attribute__ ((aligned (4096))) = {
 	.io_bit_map_offset = 0x0FFF,
 };
 #endif	/* X86_64 */
- 
-
 
 /*
  * Task structure for double-fault handler:
diff --git a/osfmk/i386/ldt.c b/osfmk/i386/ldt.c
index 2ad1ec04b..428bf9226 100644
--- a/osfmk/i386/ldt.c
+++ b/osfmk/i386/ldt.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,27 +69,27 @@
 struct fake_descriptor	master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = {
 	[SEL_TO_INDEX(SYSENTER_CS)] {	/* kernel code (sysenter) */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_K|ACC_CODE_R
+	  0xfffff,
+	  SZ_32|SZ_G,
+	  ACC_P|ACC_PL_K|ACC_CODE_R
 	},
 	[SEL_TO_INDEX(SYSENTER_DS)] {	/* kernel data (sysenter) */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_K|ACC_DATA_W
+	  0xfffff,
+	  SZ_32|SZ_G,
+	  ACC_P|ACC_PL_K|ACC_DATA_W
 	},
 	[SEL_TO_INDEX(USER_CS)] {	/* user code segment */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_U|ACC_CODE_R
+	  0xfffff,
+ 	  SZ_32|SZ_G,
+	  ACC_P|ACC_PL_U|ACC_CODE_R
 	},
 	[SEL_TO_INDEX(USER_DS)] {	/* user data segment */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_U|ACC_DATA_W
+	  0xfffff,
+	  SZ_32|SZ_G,
+	  ACC_P|ACC_PL_U|ACC_DATA_W
 	},
 	[SEL_TO_INDEX(USER64_CS)] {	/* user 64-bit code segment */
 		0,
@@ -93,8 +99,8 @@ struct fake_descriptor	master_ldt[LDTSZ] __attribute__ ((aligned (4096))) = {
 	},
 	[SEL_TO_INDEX(USER_CTHREAD)] {	/* user cthread segment */
 		0,
-		0xfffff,
-		SZ_32|SZ_G,
-		ACC_P|ACC_PL_U|ACC_DATA_W
+	  0xfffff,
+	  SZ_32|SZ_G,
+	  ACC_P|ACC_PL_U|ACC_DATA_W
 	},
 };
diff --git a/osfmk/i386/lock.h b/osfmk/i386/lock.h
index 1f11129e7..f497a82b2 100644
--- a/osfmk/i386/lock.h
+++ b/osfmk/i386/lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1998 Apple Computer
@@ -89,7 +95,8 @@ typedef struct {
 
 typedef lck_rw_t lock_t;
 
-extern unsigned int LockTimeOut;			/* Number of hardware ticks of a lock timeout */
+extern unsigned int LockTimeOutTSC;	/* Lock timeout in TSC ticks */
+extern unsigned int LockTimeOut;	/* Lock timeout in absolute time */ 
 
 
 #if defined(__GNUC__)
@@ -180,7 +187,7 @@ atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
 {
 	uint32_t res = old;
 
-	asm volatile(
+	__asm__ volatile(
 		"lock;	cmpxchgl	%1,%2;	\n\t"
 		"	setz		%%al;	\n\t"
 		"	movzbl		%%al,%0"
@@ -226,7 +233,7 @@ static inline void	atomic_decl(volatile long * p, long delta)
 static inline int	atomic_decl_and_test(volatile long * p, long delta)
 {
 	uint8_t	ret;
-	asm volatile (
+	__asm__ volatile (
 		"	lock		\n\t"
 		"	subl	%1,%2	\n\t"
 		"	sete	%0"
diff --git a/osfmk/i386/locks.h b/osfmk/i386/locks.h
index c262da07c..0ad756ed2 100644
--- a/osfmk/i386/locks.h
+++ b/osfmk/i386/locks.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_I386_LOCKS_H_
@@ -65,7 +71,7 @@ typedef struct _lck_mtx_ {
 		} lck_mtxd;
 		struct {
 			unsigned int			lck_mtxi_tag;
-			struct _lck_mtx_ext_	*lck_mtxi_ptr;
+			struct _lck_mtx_ext_		*lck_mtxi_ptr;
 			unsigned int			lck_mtxi_pad8;
 		} lck_mtxi;
 	} lck_mtx_sw;
@@ -82,11 +88,17 @@ typedef struct _lck_mtx_ {
 #define	LCK_MTX_TAG_INDIRECT			0x00001007	/* lock marked as Indirect  */
 #define	LCK_MTX_TAG_DESTROYED			0x00002007	/* lock marked as Destroyed */
 
+#define MUTEX_LOCKED_AS_SPIN			0x00004001	/* used to indicate that the mutex */
+                                                                /* was acquired as a spin lock - stored in lck_mtxd_locked */
 /* Adaptive spin before blocking */
 extern unsigned int	MutexSpin;
-extern void		lck_mtx_lock_spin(lck_mtx_t *lck);
+extern void		lck_mtx_lock_spinwait(lck_mtx_t *lck);
+
+extern void		lck_mtx_interlock_panic(lck_mtx_t *lck);
 
-extern void		lck_mtx_interlock_spin(lck_mtx_t *lck);
+extern void		hw_lock_byte_init(uint8_t *lock_byte);
+extern void		hw_lock_byte_lock(uint8_t *lock_byte);
+extern void		hw_lock_byte_unlock(uint8_t *lock_byte);
 
 typedef struct {
 	unsigned int		type;
@@ -105,7 +117,7 @@ typedef struct _lck_mtx_ext_ {
 	struct _lck_grp_	*lck_mtx_grp;
 	unsigned int		lck_mtx_attr;
 	lck_mtx_deb_t		lck_mtx_deb;
-	lck_mtx_stat_t		lck_mtx_stat;
+	uint64_t		lck_mtx_stat;
 } lck_mtx_ext_t;
 
 #define	LCK_MTX_ATTR_DEBUG	0x1
@@ -124,17 +136,25 @@ typedef struct __lck_mtx_t__	lck_mtx_t;
 #endif
 
 #ifdef	MACH_KERNEL_PRIVATE
+#pragma pack(1)		/* Make sure the structure stays as we defined it */
 typedef struct {
-	hw_lock_data_t		interlock;
-	volatile unsigned int
-						read_count:16,	/* No. of accepted readers */
-						want_upgrade:1,	/* Read-to-write upgrade waiting */
-			want_write:1,	/* Writer waiting or locked for write */
-						waiting:1,		/* Someone is sleeping on lock */
-			can_sleep:1,	/* Can attempts to lock go to sleep? */
-			read_priority:1;/* New read takes piority over write */
-	unsigned int		lck_rw_tag;
+	volatile uint16_t	lck_rw_shared_count;	/* No. of accepted readers */
+	uint8_t	lck_rw_interlock; 			/* Interlock byte */
+	volatile uint8_t
+				lck_rw_priv_excl:1,	/* Writers prioritized if set */
+				lck_rw_want_upgrade:1,	/* Read-to-write upgrade waiting */
+				lck_rw_want_write:1,	/* Writer waiting or locked for write */
+				lck_r_waiting:1,	/* Reader is sleeping on lock */
+				lck_w_waiting:1,	/* Writer is sleeping on lock */
+				lck_rw_can_sleep:1,	/* Can attempts to lock go to sleep? */
+				lck_rw_pad6:2; 		/* padding */
+
+	unsigned int		lck_rw_tag; /* This can be obsoleted when stats
+					     * are in
+					     */
+	unsigned int		lck_rw_pad8;
 } lck_rw_t;
+#pragma pack()
 
 #define	LCK_RW_ATTR_DEBUG	0x1
 #define	LCK_RW_ATTR_DEBUGb	0
diff --git a/osfmk/i386/locks_i386.c b/osfmk/i386/locks_i386.c
index be8e5e45b..38d332b00 100644
--- a/osfmk/i386/locks_i386.c
+++ b/osfmk/i386/locks_i386.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -82,6 +88,16 @@
 
 #include <sys/kdebug.h>
 
+/*
+ * We need only enough declarations from the BSD-side to be able to
+ * test if our probe is active, and to call __dtrace_probe().  Setting
+ * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
+ */
+#if	CONFIG_DTRACE
+#define NEED_DTRACE_DEFS
+#include <../bsd/sys/lockstat.h>
+#endif
+
 #define	LCK_RW_LCK_EXCLUSIVE_CODE	0x100
 #define	LCK_RW_LCK_EXCLUSIVE1_CODE	0x101
 #define	LCK_RW_LCK_SHARED_CODE		0x102
@@ -94,7 +110,7 @@
 #define	ANY_LOCK_DEBUG	(USLOCK_DEBUG || LOCK_DEBUG || MUTEX_DEBUG)
 
 unsigned int LcksOpts=0;
-unsigned int lock_wait_time[2] = { (unsigned int)-1, 100 } ;
+unsigned int lock_wait_time[2] = { (unsigned int)-1, 0 } ;
 
 /* Forwards */
 
@@ -162,6 +178,16 @@ int		usld_lock_common_checks(usimple_lock_t, char *);
 #define	USLDBG(stmt)
 #endif	/* USLOCK_DEBUG */
 
+/*
+ * Forward definitions
+ */
+
+void lck_rw_lock_shared_gen(
+	lck_rw_t	*lck);
+
+lck_rw_type_t lck_rw_done_gen(
+	lck_rw_t	*lck);
+
 /*
  *      Routine:        lck_spin_alloc_init
  */
@@ -248,7 +274,7 @@ boolean_t
 lck_spin_try_lock(
 	lck_spin_t	*lck)
 {
-	return(usimple_lock_try((usimple_lock_t) lck));
+	return((boolean_t)usimple_lock_try((usimple_lock_t) lck));
 }
 
 /*
@@ -282,13 +308,13 @@ usimple_lock(
 	usimple_lock_t	l)
 {
 #ifndef	MACHINE_SIMPLE_LOCK
-	pc_t		pc = NULL;
+	DECL_PC(pc);
 
 	OBTAIN_PC(pc, l);
 	USLDBG(usld_lock_pre(l, pc));
 
-	if(!hw_lock_to(&l->interlock, LockTimeOut))	/* Try to get the lock with a timeout */ 
-		panic("simple lock deadlock detection - l=%08X, cpu=%d, ret=%08X", l, cpu_number(), pc);
+	if(!hw_lock_to(&l->interlock, LockTimeOutTSC))	/* Try to get the lock with a timeout */ 
+		panic("simple lock deadlock detection: lock=%p, cpu=%d, owning thread=0x%x", l, cpu_number(), l->interlock.lock_data);
 
 	USLDBG(usld_lock_post(l, pc));
 #else
@@ -337,8 +363,8 @@ usimple_lock_try(
 	usimple_lock_t	l)
 {
 #ifndef	MACHINE_SIMPLE_LOCK
-	DECL_PC(pc);
 	unsigned int	success;
+	DECL_PC(pc);
 
 	OBTAIN_PC(pc, l);
 	USLDBG(usld_lock_try_pre(l, pc));
@@ -438,11 +464,11 @@ usld_lock_pre(
 
 	if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread &&
 	    l->debug.lock_thread == (void *) current_thread()) {
-		printf("%s:  lock 0x%x already locked (at 0x%x) by",
-		      caller, (integer_t) l, l->debug.lock_pc);
-		printf(" current thread 0x%x (new attempt at pc 0x%x)\n",
+		printf("%s:  lock %p already locked (at %p) by",
+		      caller, l, l->debug.lock_pc);
+		printf(" current thread %p (new attempt at pc %p)\n",
 		       l->debug.lock_thread, pc);
-		panic(caller);
+		panic("%s", caller);
 	}
 	mp_disable_preemption();
 	usl_trace(l, cpu_number(), pc, caller);
@@ -511,13 +537,13 @@ usld_unlock(
 		panic("%s:  lock 0x%x hasn't been taken",
 		      caller, (integer_t) l);
 	if (l->debug.lock_thread != (void *) current_thread())
-		panic("%s:  unlocking lock 0x%x, owned by thread 0x%x",
+		panic("%s:  unlocking lock 0x%x, owned by thread %p",
 		      caller, (integer_t) l, l->debug.lock_thread);
 	if (l->debug.lock_cpu != mycpu) {
 		printf("%s:  unlocking lock 0x%x on cpu 0x%x",
 		       caller, (integer_t) l, mycpu);
 		printf(" (acquired on cpu 0x%x)\n", l->debug.lock_cpu);
-		panic(caller);
+		panic("%s", caller);
 	}
 	usl_trace(l, mycpu, pc, caller);
 
@@ -666,12 +692,13 @@ lock_init(
 	__unused unsigned short	tag,
 	__unused unsigned short	tag1)
 {
-	hw_lock_init(&l->interlock);
-	l->want_write = FALSE;
-	l->want_upgrade = FALSE;
-	l->read_count = 0;
-	l->can_sleep = can_sleep;
+	hw_lock_byte_init(&l->lck_rw_interlock);
+	l->lck_rw_want_write = FALSE;
+	l->lck_rw_want_upgrade = FALSE;
+	l->lck_rw_shared_count = 0;
+	l->lck_rw_can_sleep = can_sleep;
 	l->lck_rw_tag = tag;
+	l->lck_rw_priv_excl = 1;
 }
 
 
@@ -713,7 +740,7 @@ lock_read(
  *		already requested an upgrade to a write lock,
  *		no lock is held upon return.
  *
- *		Returns TRUE if the upgrade *failed*.
+ *		Returns FALSE if the upgrade *failed*.
  */
 
 boolean_t
@@ -770,14 +797,14 @@ lck_rw_init(
 	lck_attr_t	*lck_attr = (attr != LCK_ATTR_NULL) ?
 					attr : &LockDefaultLckAttr;
 
-	hw_lock_init(&lck->interlock);
-	lck->want_write = FALSE;
-	lck->want_upgrade = FALSE;
-	lck->read_count = 0;
-	lck->can_sleep = TRUE;
+	hw_lock_byte_init(&lck->lck_rw_interlock);
+	lck->lck_rw_want_write = FALSE;
+	lck->lck_rw_want_upgrade = FALSE;
+	lck->lck_rw_shared_count = 0;
+	lck->lck_rw_can_sleep = TRUE;
 	lck->lck_rw_tag = 0;
-	lck->read_priority = (lck_attr->lck_attr_val &
-				 LCK_ATTR_RW_SHARED_PRIORITY) != 0;
+	lck->lck_rw_priv_excl = ((lck_attr->lck_attr_val &
+				LCK_ATTR_RW_SHARED_PRIORITY) == 0);
 
 	lck_grp_reference(grp);
 	lck_grp_lckcnt_incr(grp, LCK_TYPE_RW);
@@ -806,6 +833,11 @@ lck_rw_destroy(
 
 #define DECREMENTER_TIMEOUT 1000000
 
+#define RW_LOCK_READER_EVENT(x)		\
+		((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_tag))))
+
+#define RW_LOCK_WRITER_EVENT(x)		\
+		((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_pad8))))
 
 /*
  * We need to disable interrupts while holding the mutex interlock
@@ -818,7 +850,7 @@ lck_interlock_lock(lck_rw_t *lck)
 	boolean_t	istate;
 
 	istate = ml_set_interrupts_enabled(FALSE);	
-	hw_lock_lock(&lck->interlock);
+	hw_lock_byte_lock(&lck->lck_rw_interlock);
 
 	return istate;
 }
@@ -826,11 +858,10 @@ lck_interlock_lock(lck_rw_t *lck)
 static void
 lck_interlock_unlock(lck_rw_t *lck, boolean_t istate)
 {               
-	hw_lock_unlock(&lck->interlock);
+	hw_lock_byte_unlock(&lck->lck_rw_interlock);
 	ml_set_interrupts_enabled(istate);
 }
 
-
 /*
  * This inline is used when busy-waiting for an rw lock.
  * If interrupts were disabled when the lock primitive was called,
@@ -853,68 +884,98 @@ lck_rw_lock_exclusive(
 	lck_rw_t	*lck)
 {
 	int	   i;
-	boolean_t		lock_miss = FALSE;
 	wait_result_t	res;
 #if	MACH_LDEBUG
 	int				decrementer;
 #endif	/* MACH_LDEBUG */
 	boolean_t	istate;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	istate = lck_interlock_lock(lck);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw_shared_count;
+#endif
 
 #if	MACH_LDEBUG
 	decrementer = DECREMENTER_TIMEOUT;
 #endif	/* MACH_LDEBUG */
 
 	/*
-	 *	Try to acquire the want_write bit.
+	 *	Try to acquire the lck_rw_want_write bit.
 	 */
-	while (lck->want_write) {
-		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_START, (int)lck, 0, 0, 0, 0);
+	while (lck->lck_rw_want_write) {
 
-		if (!lock_miss) {
-			lock_miss = TRUE;
+		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_START, (int)lck, 0, 0, 0, 0);
+		/*
+		 * Either sleeping or spinning is happening, start
+		 * a timing of our delay interval now.
+		 */
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = -1;
 		}
+#endif
 
-		i = lock_wait_time[lck->can_sleep ? 1 : 0];
+
+		i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
 		if (i != 0) {
 			lck_interlock_unlock(lck, istate);
 #if	MACH_LDEBUG
 			if (!--decrementer)
-				Debugger("timeout - want_write");
+				Debugger("timeout - lck_rw_want_write");
 #endif	/* MACH_LDEBUG */
-			while (--i != 0 && lck->want_write)
+			while (--i != 0 && lck->lck_rw_want_write)
 				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
-		if (lck->can_sleep && lck->want_write) {
-			lck->waiting = TRUE;
-			res = assert_wait((event_t) lck, THREAD_UNINT);
+		if (lck->lck_rw_can_sleep && lck->lck_rw_want_write) {
+			lck->lck_w_waiting = TRUE;
+			res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
 				lck_interlock_unlock(lck, istate);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				istate = lck_interlock_lock(lck);
 			}
 		}
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_END, (int)lck, res, 0, 0, 0);
 	}
-	lck->want_write = TRUE;
+	lck->lck_rw_want_write = TRUE;
 
 	/* Wait for readers (and upgrades) to finish */
 
 #if	MACH_LDEBUG
 	decrementer = DECREMENTER_TIMEOUT;
 #endif	/* MACH_LDEBUG */
-	while ((lck->read_count != 0) || lck->want_upgrade) {
-		if (!lock_miss) {
-			lock_miss = TRUE;
-		}
+	while ((lck->lck_rw_shared_count != 0) || lck->lck_rw_want_upgrade) {
 
-		i = lock_wait_time[lck->can_sleep ? 1 : 0];
+		i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_START,
-			     (int)lck, lck->read_count, lck->want_upgrade, i, 0);
+			     (int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, i, 0);
+
+#if	CONFIG_DTRACE
+		/*
+		 * Either sleeping or spinning is happening, start
+		 * a timing of our delay interval now.  If we set it
+		 * to -1 we don't have accurate data so we cannot later
+		 * decide to record a dtrace spin or sleep event.
+		 */
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (i != 0) {
 			lck_interlock_unlock(lck, istate);
@@ -922,53 +983,84 @@ lck_rw_lock_exclusive(
 			if (!--decrementer)
 				Debugger("timeout - wait for readers");
 #endif	/* MACH_LDEBUG */
-			while (--i != 0 && (lck->read_count != 0 ||
-					    lck->want_upgrade))
+			while (--i != 0 && (lck->lck_rw_shared_count != 0 ||
+					    lck->lck_rw_want_upgrade))
 				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
-		if (lck->can_sleep && (lck->read_count != 0 || lck->want_upgrade)) {
-			lck->waiting = TRUE;
-			res = assert_wait((event_t) lck, THREAD_UNINT);
+		if (lck->lck_rw_can_sleep && (lck->lck_rw_shared_count != 0 || lck->lck_rw_want_upgrade)) {
+			lck->lck_w_waiting = TRUE;
+			res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
 				lck_interlock_unlock(lck, istate);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				istate = lck_interlock_lock(lck);
 			}
 		}
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_END,
-			     (int)lck, lck->read_count, lck->want_upgrade, res, 0);
+			     (int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, res, 0);
 	}
 
 	lck_interlock_unlock(lck, istate);
+#if	CONFIG_DTRACE
+	/*
+	 * Decide what latencies we suffered that are Dtrace events.
+	 * If we have set wait_interval, then we either spun or slept.
+	 * At least we get out from under the interlock before we record
+	 * which is the best we can do here to minimize the impact
+	 * of the tracing.
+	 * If we have set wait_interval to -1, then dtrace was not enabled when we
+	 * started sleeping/spinning so we don't record this event.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lck,
+			    mach_absolute_time() - wait_interval, 1);
+		} else {
+			/*
+			 * For the blocking case, we also record if when we blocked
+			 * it was held for read or write, and how many readers.
+			 * Notice that above we recorded this before we dropped
+			 * the interlock so the count is accurate.
+			 */
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 1,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lck, 1);
+#endif
 }
 
 
 /*
- *      Routine:        lck_rw_done
+ *      Routine:        lck_rw_done_gen
  */
 lck_rw_type_t
-lck_rw_done(
+lck_rw_done_gen(
 	lck_rw_t	*lck)
 {
-	boolean_t	do_wakeup = FALSE;
+	boolean_t	wakeup_readers = FALSE;
+	boolean_t	wakeup_writers = FALSE;
 	lck_rw_type_t	lck_rw_type;
 	boolean_t	istate;
 
-
 	istate = lck_interlock_lock(lck);
 
-	if (lck->read_count != 0) {
+	if (lck->lck_rw_shared_count != 0) {
 		lck_rw_type = LCK_RW_TYPE_SHARED;
-		lck->read_count--;
+		lck->lck_rw_shared_count--;
 	}
 	else {	
 		lck_rw_type = LCK_RW_TYPE_EXCLUSIVE;
-		if (lck->want_upgrade) 
-			lck->want_upgrade = FALSE;
+		if (lck->lck_rw_want_upgrade) 
+			lck->lck_rw_want_upgrade = FALSE;
 		else 
-			lck->want_write = FALSE;
+			lck->lck_rw_want_write = FALSE;
 	}
 
 	/*
@@ -979,15 +1071,29 @@ lck_rw_done(
 	 *		if there are still readers, they can't proceed
 	 */
 
-	if (lck->waiting && (lck->read_count == 0)) {
-		lck->waiting = FALSE;
-		do_wakeup = TRUE;
+	if (lck->lck_rw_shared_count == 0) {
+		if (lck->lck_w_waiting) {
+			lck->lck_w_waiting = FALSE;
+			wakeup_writers = TRUE;
+		} 
+		if (!(lck->lck_rw_priv_excl && wakeup_writers == TRUE) && 
+				lck->lck_r_waiting) {
+			lck->lck_r_waiting = FALSE;
+			wakeup_readers = TRUE;
+		}
 	}
 
 	lck_interlock_unlock(lck, istate);
 
-	if (do_wakeup)
-		thread_wakeup((event_t) lck);
+	if (wakeup_readers) 
+		thread_wakeup(RW_LOCK_READER_EVENT(lck));
+	if (wakeup_writers) 
+		thread_wakeup(RW_LOCK_WRITER_EVENT(lck));
+
+#if CONFIG_DTRACE
+	LOCKSTAT_RECORD(LS_LCK_RW_DONE_RELEASE, lck, (lck_rw_type == LCK_RW_TYPE_EXCLUSIVE ? 1 : 0));
+#endif
+
 	return(lck_rw_type);
 }
 
@@ -1061,10 +1167,10 @@ lck_rw_lock(
 
 
 /*
- *	Routine:	lck_rw_lock_shared
+ *	Routine:	lck_rw_lock_shared_gen
  */
 void
-lck_rw_lock_shared(
+lck_rw_lock_shared_gen(
 	lck_rw_t	*lck)
 {
 	int		i;
@@ -1073,20 +1179,34 @@ lck_rw_lock_shared(
 	int		decrementer;
 #endif	/* MACH_LDEBUG */
 	boolean_t	istate;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	istate = lck_interlock_lock(lck);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw_shared_count;
+#endif
 
 #if	MACH_LDEBUG
 	decrementer = DECREMENTER_TIMEOUT;
 #endif	/* MACH_LDEBUG */
-	while ((lck->want_write && (lck->read_priority ?
-					lck->read_count == 0 : TRUE)) ||
-	       lck->want_upgrade) {
+	while ((lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
+	    ((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl)) {
 
-		i = lock_wait_time[lck->can_sleep ? 1 : 0];
+		i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
-			     (int)lck, lck->want_write, lck->want_upgrade, i, 0);
+			     (int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, i, 0);
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = -1;
+		}
+#endif
 
 		if (i != 0) {
 			lck_interlock_unlock(lck, istate);
@@ -1095,32 +1215,45 @@ lck_rw_lock_shared(
 				Debugger("timeout - wait no writers");
 #endif	/* MACH_LDEBUG */
 			while (--i != 0 &&
-				((lck->want_write && (lck->read_priority ?
-					lck->read_count == 0 : TRUE)) ||
-				 lck->want_upgrade))
+			    (lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
+			       ((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl))
 				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
-		if (lck->can_sleep &&
-			((lck->want_write && (lck->read_priority ?
-						lck->read_count == 0 : TRUE)) ||
-			  lck->want_upgrade)) {
-			lck->waiting = TRUE;
-			res = assert_wait((event_t) lck, THREAD_UNINT);
+		if (lck->lck_rw_can_sleep &&
+		    (lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
+		    ((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl)) {
+			lck->lck_r_waiting = TRUE;
+			res = assert_wait(RW_LOCK_READER_EVENT(lck), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
 				lck_interlock_unlock(lck, istate);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				istate = lck_interlock_lock(lck);
 			}
 		}
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_END,
-			     (int)lck, lck->want_write, lck->want_upgrade, res, 0);
+			     (int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, res, 0);
 	}
 
-	lck->read_count++;
+	lck->lck_rw_shared_count++;
 
 	lck_interlock_unlock(lck, istate);
+#if	CONFIG_DTRACE
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 0,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0);
+#endif
 }
 
 
@@ -1132,7 +1265,7 @@ lck_rw_lock_shared(
  *		already requested an upgrade to a write lock,
  *		no lock is held upon return.
  *
- *		Returns TRUE if the upgrade *failed*.
+ *		Returns FALSE if the upgrade *failed*.
  */
 
 boolean_t
@@ -1146,74 +1279,105 @@ lck_rw_lock_shared_to_exclusive(
 	int		   decrementer;
 #endif	/* MACH_LDEBUG */
 	boolean_t	istate;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep = 0;
+#endif
 
 	istate = lck_interlock_lock(lck);
 
-	lck->read_count--;	
+	lck->lck_rw_shared_count--;	
 
-	if (lck->want_upgrade) {
+	if (lck->lck_rw_want_upgrade) {
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_START,
-			     (int)lck, lck->read_count, lck->want_upgrade, 0, 0);
+			     (int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0);
 
 		/*
 		 *	Someone else has requested upgrade.
 		 *	Since we've released a read lock, wake
 		 *	him up.
 		 */
-		if (lck->waiting && (lck->read_count == 0)) {
-			lck->waiting = FALSE;
+		if (lck->lck_w_waiting && (lck->lck_rw_shared_count == 0)) {
+			lck->lck_w_waiting = FALSE;
 			do_wakeup = TRUE;
 		}
 
 		lck_interlock_unlock(lck, istate);
 
-		if (do_wakeup)
-			thread_wakeup((event_t) lck);
+		if (do_wakeup) 
+			thread_wakeup(RW_LOCK_WRITER_EVENT(lck));
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_END,
-			     (int)lck, lck->read_count, lck->want_upgrade, 0, 0);
+			     (int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0);
 
-		return (TRUE);
+		return (FALSE);
 	}
 
-	lck->want_upgrade = TRUE;
+	lck->lck_rw_want_upgrade = TRUE;
 
 #if	MACH_LDEBUG
 	decrementer = DECREMENTER_TIMEOUT;
 #endif	/* MACH_LDEBUG */
-	while (lck->read_count != 0) {
-		i = lock_wait_time[lck->can_sleep ? 1 : 0];
+	while (lck->lck_rw_shared_count != 0) {
+#if	CONFIG_DTRACE
+		if (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+			readers_at_sleep = lck->lck_rw_shared_count;
+		} else {
+			wait_interval = -1;
+		}
+#endif
+		i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | DBG_FUNC_START,
-			     (int)lck, lck->read_count, i, 0, 0);
+			     (int)lck, lck->lck_rw_shared_count, i, 0, 0);
 
 		if (i != 0) {
 			lck_interlock_unlock(lck, istate);
 #if	MACH_LDEBUG
 			if (!--decrementer)
-				Debugger("timeout - read_count");
+				Debugger("timeout - lck_rw_shared_count");
 #endif	/* MACH_LDEBUG */
-			while (--i != 0 && lck->read_count != 0)
+			while (--i != 0 && lck->lck_rw_shared_count != 0)
 				lck_rw_lock_pause(istate);
 			istate = lck_interlock_lock(lck);
 		}
 
-		if (lck->can_sleep && lck->read_count != 0) {
-			lck->waiting = TRUE;
-			res = assert_wait((event_t) lck, THREAD_UNINT);
+		if (lck->lck_rw_can_sleep && lck->lck_rw_shared_count != 0) {
+			lck->lck_w_waiting = TRUE;
+			res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
 			if (res == THREAD_WAITING) {
 				lck_interlock_unlock(lck, istate);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				istate = lck_interlock_lock(lck);
 			}
 		}
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | DBG_FUNC_END,
-			     (int)lck, lck->read_count, 0, 0, 0);
+			     (int)lck, lck->lck_rw_shared_count, 0, 0, 0);
 	}
 
 	lck_interlock_unlock(lck, istate);
+#if	CONFIG_DTRACE
+	/*
+	 * We infer whether we took the sleep/spin path above by checking readers_at_sleep.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1 && readers_at_sleep) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 1,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
 
-	return (FALSE);
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lck, 1);
+#endif
+	return (TRUE);
 }
 
 /*
@@ -1223,33 +1387,44 @@ void
 lck_rw_lock_exclusive_to_shared(
 	lck_rw_t	*lck)
 {
-	boolean_t	do_wakeup = FALSE;
+	boolean_t	wakeup_readers = FALSE;
+	boolean_t	wakeup_writers = FALSE;
 	boolean_t	istate;
 
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_START,
-			     (int)lck, lck->want_write, lck->want_upgrade, 0, 0);
+			     (int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, 0, 0);
 
 	istate = lck_interlock_lock(lck);
 
-	lck->read_count++;
-	if (lck->want_upgrade)
-		lck->want_upgrade = FALSE;
+	lck->lck_rw_shared_count++;
+	if (lck->lck_rw_want_upgrade)
+		lck->lck_rw_want_upgrade = FALSE;
 	else
-	 	lck->want_write = FALSE;
-
-	if (lck->waiting) {
-		lck->waiting = FALSE;
-		do_wakeup = TRUE;
+	 	lck->lck_rw_want_write = FALSE;
+
+	if (lck->lck_w_waiting) {
+		lck->lck_w_waiting = FALSE;
+		wakeup_writers = TRUE;
+	} 
+	if (!(lck->lck_rw_priv_excl && wakeup_writers == TRUE) && 
+			lck->lck_r_waiting) {
+		lck->lck_r_waiting = FALSE;
+		wakeup_readers = TRUE;
 	}
 
 	lck_interlock_unlock(lck, istate);
 
-	if (do_wakeup)
-		thread_wakeup((event_t) lck);
+	if (wakeup_readers)
+		thread_wakeup(RW_LOCK_READER_EVENT(lck));
+	if (wakeup_writers)
+		thread_wakeup(RW_LOCK_WRITER_EVENT(lck));
 
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_END,
-			     (int)lck, lck->want_write, lck->want_upgrade, lck->read_count, 0);
+			     (int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, lck->lck_rw_shared_count, 0);
 
+#if CONFIG_DTRACE
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, lck, 0);
+#endif
 }
 
 
@@ -1286,7 +1461,7 @@ lck_rw_try_lock_exclusive(
 
 	istate = lck_interlock_lock(lck);
 
-	if (lck->want_write || lck->want_upgrade || lck->read_count) {
+	if (lck->lck_rw_want_write || lck->lck_rw_want_upgrade || lck->lck_rw_shared_count) {
 		/*
 		 *	Can't get lock.
 		 */
@@ -1298,10 +1473,13 @@ lck_rw_try_lock_exclusive(
 	 *	Have lock.
 	 */
 
-	lck->want_write = TRUE;
+	lck->lck_rw_want_write = TRUE;
 
 	lck_interlock_unlock(lck, istate);
 
+#if CONFIG_DTRACE
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lck, 1);
+#endif
 	return(TRUE);
 }
 
@@ -1320,19 +1498,54 @@ lck_rw_try_lock_shared(
 	boolean_t	istate;
 
 	istate = lck_interlock_lock(lck);
-
-	if (lck->want_write || lck->want_upgrade) {
+/* No reader priority check here... */
+	if (lck->lck_rw_want_write || lck->lck_rw_want_upgrade) {
 		lck_interlock_unlock(lck, istate);
 		return(FALSE);
 	}
 
-	lck->read_count++;
+	lck->lck_rw_shared_count++;
 
 	lck_interlock_unlock(lck, istate);
 
+#if CONFIG_DTRACE
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lck, 0);
+#endif
 	return(TRUE);
 }
 
+void
+lck_rw_assert(
+	lck_rw_t	*lck,
+	unsigned int	type)
+{
+	switch (type) {
+	case LCK_RW_ASSERT_SHARED:
+		if (lck->lck_rw_shared_count != 0) {
+			return;
+		}
+		break;
+	case LCK_RW_ASSERT_EXCLUSIVE:
+		if ((lck->lck_rw_want_write ||
+		     lck->lck_rw_want_upgrade) &&
+		    lck->lck_rw_shared_count == 0) {
+			return;
+		}
+		break;
+	case LCK_RW_ASSERT_HELD:
+		if (lck->lck_rw_want_write ||
+		    lck->lck_rw_want_upgrade ||
+		    lck->lck_rw_shared_count != 0) {
+			return;
+		}
+		break;
+	default:
+		break;
+	}
+
+	panic("rw lock (%p) not held (mode=%u)\n", lck, type);
+}
+
 /*
  *      Routine:        lck_mtx_alloc_init
  */
@@ -1370,20 +1583,17 @@ lck_mtx_ext_init(
 	lck_grp_t	*grp,
 	lck_attr_t	*attr)
 {
-	lck->lck_mtx.lck_mtx_ilk = 0;
-	lck->lck_mtx.lck_mtx_locked = 0;
-	lck->lck_mtx.lck_mtx_waiters = 0;
-	lck->lck_mtx.lck_mtx_pri = 0;
-	lck->lck_mtx_attr = 0;
+	bzero((void *)lck, sizeof(lck_mtx_ext_t));
 
 	if ((attr->lck_attr_val) & LCK_ATTR_DEBUG) {
-		lck->lck_mtx_deb.pc = 0;
-		lck->lck_mtx_deb.thread = 0;
 		lck->lck_mtx_deb.type = MUTEX_TAG;
 		lck->lck_mtx_attr |= LCK_MTX_ATTR_DEBUG;
 	}
 
 	lck->lck_mtx_grp = grp;
+
+	if (grp->lck_grp_attr & LCK_GRP_ATTR_STAT)
+		 lck->lck_mtx_attr |= LCK_MTX_ATTR_STAT;
 }
 
 /*
@@ -1396,10 +1606,16 @@ lck_mtx_init(
 	lck_attr_t	*attr)
 {
 	lck_mtx_ext_t	*lck_ext;
+	lck_attr_t	*lck_attr;
+
+	if (attr != LCK_ATTR_NULL)
+		lck_attr = attr;
+	else
+		lck_attr = &LockDefaultLckAttr;
 
-	if ((attr != LCK_ATTR_NULL) && ((attr->lck_attr_val) & LCK_ATTR_DEBUG)) {
+	if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) {
 		if ((lck_ext = (lck_mtx_ext_t *)kalloc(sizeof(lck_mtx_ext_t))) != 0) {
-			lck_mtx_ext_init(lck_ext, grp, attr);	
+			lck_mtx_ext_init(lck_ext, grp, lck_attr);	
 			lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT;
 			lck->lck_mtx_ptr = lck_ext;
 		}
@@ -1413,6 +1629,37 @@ lck_mtx_init(
 	lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX);
 }
 
+/*
+ *      Routine:        lck_mtx_init_ext
+ */
+void
+lck_mtx_init_ext(
+	lck_mtx_t	*lck,
+	lck_mtx_ext_t	*lck_ext,
+	lck_grp_t	*grp,
+	lck_attr_t	*attr)
+{
+	lck_attr_t	*lck_attr;
+
+	if (attr != LCK_ATTR_NULL)
+		lck_attr = attr;
+	else
+		lck_attr = &LockDefaultLckAttr;
+
+	if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) {
+		lck_mtx_ext_init(lck_ext, grp, lck_attr);
+		lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT;
+		lck->lck_mtx_ptr = lck_ext;
+	} else {
+		lck->lck_mtx_ilk = 0;
+		lck->lck_mtx_locked = 0;
+		lck->lck_mtx_waiters = 0;
+		lck->lck_mtx_pri = 0;
+	}
+	lck_grp_reference(grp);
+	lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX);
+}
+
 /*
  *      Routine:        lck_mtx_destroy
  */
@@ -1435,17 +1682,7 @@ lck_mtx_destroy(
 }
 
 /*
- *      Routine:        lck_mtx_assert
- */
-void
-lck_mtx_assert(
-	__unused lck_mtx_t	*lck,
-	__unused unsigned int	type)
-{
-}
-
-/*
- * Routine: 	lck_mtx_lock_spin
+ * Routine: 	lck_mtx_lock_spinwait
  *
  * Invoked trying to acquire a mutex when there is contention but
  * the holder is running on another processor. We spin for up to a maximum
@@ -1454,11 +1691,11 @@ lck_mtx_assert(
  * Called with the interlock unlocked.
  */
 void
-lck_mtx_lock_spin(
+lck_mtx_lock_spinwait(
 	lck_mtx_t		*lck)
 {
 	thread_t		holder;
-	lck_mtx_t		*mutex;
+	volatile lck_mtx_t	*mutex;
 	uint64_t		deadline;
 
 	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
@@ -1466,67 +1703,79 @@ lck_mtx_lock_spin(
 	else
 		mutex = &lck->lck_mtx_ptr->lck_mtx;
 
-        KERNEL_DEBUG(
-		MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_SPIN) | DBG_FUNC_START,
+	KERNEL_DEBUG(
+		MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_SPIN) | DBG_FUNC_NONE,
 		(int)lck, (int)mutex->lck_mtx_locked, 0, 0, 0);
 
 	deadline = mach_absolute_time() + MutexSpin;
 	/*
 	 * Spin while:
 	 *   - mutex is locked, and
+	 *   - its locked as a spin lock, or
 	 *   - owner is running on another processor, and
-	 *   - owner is not is the idle delay, and
+	 *   - owner (processor) is not idling, and
 	 *   - we haven't spun for long enough.
 	 */
-	while ((holder = (thread_t) mutex->lck_mtx_locked) != NULL &&
-	       (holder->machine.specFlags & OnProc) != 0 &&
-	       (holder->options & TH_OPT_DELAYIDLE) == 0 &&
-	       mach_absolute_time() < deadline)
-		cpu_pause();
+	while ((holder = (thread_t) mutex->lck_mtx_locked) != NULL) {
+	        if ((holder == (thread_t)MUTEX_LOCKED_AS_SPIN) ||
+		    ((holder->machine.specFlags & OnProc) != 0 &&
+		     (holder->state & TH_IDLE) == 0 &&
+		     mach_absolute_time() < deadline)) {
+		        cpu_pause();
+			continue;
+		}
+		break;
+	}
+#if	CONFIG_DTRACE
+	/*
+	 * We've already kept a count via deadline of how long we spun.
+	 * If dtrace is active, then we compute backwards to decide how
+	 * long we spun.
+	 *
+	 * Note that we record a different probe id depending on whether
+	 * this is a direct or indirect mutex.  This allows us to 
+	 * penalize only lock groups that have debug/stats enabled
+	 * with dtrace processing if desired.
+	 */
+	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) {
+		LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_SPIN, lck,
+		    mach_absolute_time() - (deadline - MutexSpin));
+	} else {
+		LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_SPIN, lck,
+		    mach_absolute_time() - (deadline - MutexSpin));
+	}
+	/* The lockstat acquire event is recorded by the assembly code beneath us. */
+#endif
 }
 
 /*
- * Called from assembly code when a mutex interlock is held.
- * We spin here re-checking the interlock but panic if we timeout.
- * Note: here with interrupts disabled.
+ * Called from assembly code when a destroyed mutex is detected
+ * during a lock/unlock/try/convert
  */
+
 void
-lck_mtx_interlock_spin(
-	lck_mtx_t		*lck)
+lck_mtx_interlock_panic(
+			lck_mtx_t		*lck)
 {
-	lck_mtx_t		*mutex;
-	uint64_t		deadline;
-
-	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
-		mutex = lck;
-	else
-		mutex = &lck->lck_mtx_ptr->lck_mtx;
-
-	deadline = mach_absolute_time() + LockTimeOut;
-	while (mutex->lck_mtx_ilk != 0) {
-		cpu_pause();
-		if (mach_absolute_time() > deadline)
-			panic("interlock timeout for mutex %p", lck);
-	}
-
+        panic("trying to interlock destroyed mutex %p", lck);
 }
 
-#if	MACH_KDB
 
-void	db_show_one_lock(lock_t  *);
+#if	MACH_KDB
 
 void
 db_show_one_lock(
 	lock_t  *lock)
 {
 	db_printf("Read_count = 0x%x, %swant_upgrade, %swant_write, ",
-		  lock->read_count,
-		  lock->want_upgrade ? "" : "!",
-		  lock->want_write ? "" : "!");
+		  lock->lck_rw_shared_count,
+		  lock->lck_rw_want_upgrade ? "" : "!",
+		  lock->lck_rw_want_write ? "" : "!");
 	db_printf("%swaiting, %scan_sleep\n", 
-		  lock->waiting ? "" : "!", lock->can_sleep ? "" : "!");
+		  (lock->lck_r_waiting || lock->lck_w_waiting) ? "" : "!", 
+		  lock->lck_rw_can_sleep ? "" : "!");
 	db_printf("Interlock:\n");
-	db_show_one_simple_lock((db_expr_t) ((vm_offset_t)simple_lock_addr(lock->interlock)),
+	db_show_one_simple_lock((db_expr_t) ((vm_offset_t)simple_lock_addr(lock->lck_rw_interlock)),
 			TRUE, (db_expr_t)0, (char *)0);
 }
 
@@ -1571,36 +1820,6 @@ mutex_free(
 	kfree(m, sizeof(mutex_t));
 }
 
-/*
- *	Routine:	_mutex_assert
- */
-void
-_mutex_assert (
-	mutex_t		*mutex,
-	unsigned int	what)
-{
-
-	thread_t	thread = current_thread();
-	thread_t	holder;
-
-        if (panicstr != NULL)
-	        return;
-
-	holder = (thread_t) mutex->lck_mtx.lck_mtx_locked;
-
-	switch (what) {
-	case MA_OWNED:
-	        if (thread != holder)
-		        panic("mutex %x not owned\n", mutex);
-		break;
-
-        case MA_NOTOWNED:
-	        if (thread == holder)
-		        panic("mutex %x owned\n", mutex);
-	        break;
-	}
-
-}
 
 #if	MACH_KDB
 /*
diff --git a/osfmk/i386/locore.s b/osfmk/i386/locore.s
index 2b6b1c718..29e356922 100644
--- a/osfmk/i386/locore.s
+++ b/osfmk/i386/locore.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -217,99 +223,146 @@ Entry(timer_grab)
  */
 
 /*
- * Low 32-bits of nanotime returned in %eax.
+ * Nanotime returned in %edx:%eax.
  * Computed from tsc based on the scale factor
  * and an implicit 32 bit shift.
  *
- * Uses %esi, %edi, %ebx, %ecx and %edx.
+ * Uses %eax, %ebx, %ecx, %edx, %esi, %edi.
  */
 #define RNT_INFO		_rtc_nanotime_info
-#define NANOTIME32											  \
-0:	movl	RNT_INFO+RNT_TSC_BASE,%esi						 ;\
-	movl	RNT_INFO+RNT_TSC_BASE+4,%edi					 ;\
-	rdtsc													 ;\
-	subl	%esi,%eax					/* tsc - tsc_base */ ;\
-	sbbl	%edi,%edx										 ;\
-	movl	RNT_INFO+RNT_SCALE,%ecx							 ;\
-	movl	%edx,%ebx					/* delta * scale */  ;\
-	mull	%ecx											 ;\
-	movl	%ebx,%eax										 ;\
-	movl	%edx,%ebx										 ;\
-	mull	%ecx											 ;\
-	addl	%ebx,%eax										 ;\
-	addl	RNT_INFO+RNT_NS_BASE,%eax	/* add ns_base */	 ;\
-	cmpl	RNT_INFO+RNT_TSC_BASE,%esi						 ;\
-	jne		0b												 ;\
-	cmpl	RNT_INFO+RNT_TSC_BASE+4,%edi					 ;\
-	jne		0b
+#define NANOTIME														  \
+0:	movl	RNT_INFO+RNT_TSC_BASE,%esi									; \
+	movl	RNT_INFO+RNT_TSC_BASE+4,%edi								; \
+	rdtsc																; \
+	subl	%esi,%eax						/* tsc - tsc_base */		; \
+	sbbl	%edi,%edx													; \
+	movl	RNT_INFO+RNT_SCALE,%ecx										; \
+	movl	%edx,%ebx						/* delta * scale */			; \
+	mull	%ecx														; \
+	movl	%ebx,%eax													; \
+	movl	%edx,%ebx													; \
+	mull	%ecx														; \
+	addl	%ebx,%eax													; \
+	adcl	$0,%edx							/* add carry into hi */		; \
+	addl	RNT_INFO+RNT_NS_BASE,%eax		/* add ns_base lo */		; \
+	adcl	RNT_INFO+RNT_NS_BASE+4,%edx		/* add ns_base hi */		; \
+	cmpl	RNT_INFO+RNT_TSC_BASE,%esi									; \
+	jne	0b									/* repeat if changed */		; \
+	cmpl	RNT_INFO+RNT_TSC_BASE+4,%edi								; \
+	jne	0b
 
 /*
- * Add 32-bit ns delta in register dreg to timer pointed to by register treg.
+ * Add 64-bit delta in register dreg : areg to timer pointed to by register treg.
  */
-#define TIMER_UPDATE(treg,dreg)						      \
-	addl	TIMER_LOW(treg),dreg		/* add delta low bits     */ ;\
-	adcl	$0,TIMER_HIGHCHK(treg)		/* add carry check bits   */ ;\
-	movl	dreg,TIMER_LOW(treg)		/* store updated low bit  */ ;\
-	movl	TIMER_HIGHCHK(treg),dreg	/* copy high check bits	  */ ;\
-	movl    dreg,TIMER_HIGH(treg)		/*   to high bita         */
+#define TIMER_UPDATE(treg,dreg,areg)									  \
+	addl	TIMER_LOW(treg),areg		/* add low bits */				; \
+	adcl	dreg,TIMER_HIGH(treg)		/* add carry high bits */		; \
+	movl	areg,TIMER_LOW(treg)		/* store updated low bit */		; \
+	movl	TIMER_HIGH(treg),dreg		/* copy high bits */			; \
+	movl    dreg,TIMER_HIGHCHK(treg)	/* to high check */
 
 /*
  * Add time delta to old timer and start new.
  */
-#define TIMER_EVENT(old,new)                                                  \
-	NANOTIME32				/* eax low bits nanosecs  */ ;\
-	movl	%gs:CPU_PROCESSOR,%ecx		/* get current processor  */ ;\
-	movl 	CURRENT_TIMER(%ecx),%ecx	/* get current timer      */ ;\
-	movl	%eax,%edx			/* save timestamp in %edx */ ;\
-	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time   */ ;\
-	TIMER_UPDATE(%ecx,%eax)			/* update timer struct    */ ;\
-	addl	$(new##_TIMER-old##_TIMER),%ecx	/* point to new timer     */ ;\
-	movl	%edx,TIMER_TSTAMP(%ecx)		/* set timestamp          */ ;\
-	movl	%gs:CPU_PROCESSOR,%edx		/* get current processor  */ ;\
-	movl	%ecx,CURRENT_TIMER(%edx)	/* set current timer      */
-
+#define TIMER_EVENT(old,new)											  \
+	NANOTIME							/* edx:eax nanosecs */			; \
+	movl	%eax,%esi					/* save timestamp */			; \
+	movl	%edx,%edi					/* save timestamp */			; \
+	movl	%gs:CPU_PROCESSOR,%ebx		/* get current processor */		; \
+	movl 	THREAD_TIMER(%ebx),%ecx		/* get current timer */			; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	addl	$(new##_TIMER-old##_TIMER),%ecx	/* point to new timer */	; \
+	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */				; \
+	movl	%ecx,THREAD_TIMER(%ebx)		/* set current timer */			; \
+	movl	%esi,%eax					/* restore timestamp */			; \
+	movl	%edi,%edx					/* restore timestamp */			; \
+	movl	CURRENT_STATE(%ebx),%ecx	/* current state */				; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	addl	$(new##_STATE-old##_STATE),%ecx /* point to new state */	; \
+	movl	%ecx,CURRENT_STATE(%ebx)	/* set current state */			; \
+	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */
 
 /*
  * Update time on user trap entry.
- * Uses %eax,%ecx,%edx,%esi.
+ * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
  */
 #define	TIME_TRAP_UENTRY	TIMER_EVENT(USER,SYSTEM)
 
 /*
  * update time on user trap exit.
- * Uses %eax,%ecx,%edx,%esi.
+ * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
  */
 #define	TIME_TRAP_UEXIT		TIMER_EVENT(SYSTEM,USER)
 
 /*
  * update time on interrupt entry.
- * Uses %eax,%ecx,%edx,%esi.
- */
-#define	TIME_INT_ENTRY \
-	NANOTIME32				/* eax low bits nanosecs  */ ;\
-	movl	%gs:CPU_PROCESSOR,%ecx		/* get current processor  */ ;\
-	movl 	CURRENT_TIMER(%ecx),%ecx	/* get current timer      */ ;\
-	movl	%eax,%edx			/* save timestamp in %edx */ ;\
-	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time   */ ;\
-	TIMER_UPDATE(%ecx,%eax)			/* update timer struct    */ ;\
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */ ;\
-	addl	$(SYSTEM_TIMER),%ecx		/* point to sys timer     */ ;\
-	movl	%edx,TIMER_TSTAMP(%ecx)		/* set timestamp          */
+ * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
+ * Saves processor state info on stack.
+ */
+#define	TIME_INT_ENTRY													  \
+	NANOTIME							/* edx:eax nanosecs */			; \
+	movl	%eax,%gs:CPU_INT_EVENT_TIME		/* save in cpu data */		; \
+	movl	%edx,%gs:CPU_INT_EVENT_TIME+4	/* save in cpu data */		; \
+	movl	%eax,%esi					/* save timestamp */			; \
+	movl	%edx,%edi					/* save timestamp */			; \
+	movl	%gs:CPU_PROCESSOR,%ebx		/* get current processor */		; \
+	movl 	THREAD_TIMER(%ebx),%ecx		/* get current timer */			; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	movl	KERNEL_TIMER(%ebx),%ecx		/* point to kernel timer */		; \
+	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */				; \
+	movl	%esi,%eax					/* restore timestamp */			; \
+	movl	%edi,%edx					/* restore timestamp */			; \
+	movl	CURRENT_STATE(%ebx),%ecx	/* get current state */			; \
+	pushl	%ecx						/* save state */				; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	leal	IDLE_STATE(%ebx),%eax		/* get idle state */			; \
+	cmpl	%eax,%ecx					/* compare current state */		; \
+	je		0f							/* skip if equal */				; \
+	leal	SYSTEM_STATE(%ebx),%ecx		/* get system state */			; \
+	movl	%ecx,CURRENT_STATE(%ebx)	/* set current state */			; \
+0:	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */
 
 /*
  * update time on interrupt exit.
- * Uses %eax, %ecx, %edx, %esi.
- */
-#define	TIME_INT_EXIT \
-	NANOTIME32				/* eax low bits nanosecs  */ ;\
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */ ;\
-	addl	$(SYSTEM_TIMER),%ecx		/* point to sys timer 	  */ ;\
-	movl	%eax,%edx			/* save timestamp in %edx */ ;\
-	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time   */ ;\
-	TIMER_UPDATE(%ecx,%eax)			/* update timer struct    */ ;\
-	movl	%gs:CPU_PROCESSOR,%ecx		/* get current processor  */ ;\
-	movl	CURRENT_TIMER(%ecx),%ecx	/* interrupted timer      */ ;\
-	movl	%edx,TIMER_TSTAMP(%ecx)		/* set timestamp          */
+ * Uses %eax,%ebx,%ecx,%edx,%esi,%edi.
+ * Restores processor state info from stack.
+ */
+#define	TIME_INT_EXIT													  \
+	NANOTIME							/* edx:eax nanosecs */			; \
+	movl	%eax,%gs:CPU_INT_EVENT_TIME		/* save in cpu data */		; \
+	movl	%edx,%gs:CPU_INT_EVENT_TIME+4	/* save in cpu data */		; \
+	movl	%eax,%esi					/* save timestamp */			; \
+	movl	%edx,%edi					/* save timestamp */			; \
+	movl	%gs:CPU_PROCESSOR,%ebx		/* get current processor */		; \
+	movl	KERNEL_TIMER(%ebx),%ecx		/* point to kernel timer */		; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	movl	THREAD_TIMER(%ebx),%ecx		/* interrupted timer */			; \
+	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */				; \
+	movl	%esi,%eax					/* restore timestamp */			; \
+	movl	%edi,%edx					/* restore timestamp */			; \
+	movl	CURRENT_STATE(%ebx),%ecx	/* get current state */			; \
+	subl	TIMER_TSTAMP(%ecx),%eax		/* compute elapsed time */		; \
+	sbbl	TIMER_TSTAMP+4(%ecx),%edx	/* compute elapsed time */		; \
+	TIMER_UPDATE(%ecx,%edx,%eax)		/* update timer */				; \
+	popl	%ecx						/* restore state */				; \
+	movl	%ecx,CURRENT_STATE(%ebx)	/* set current state */			; \
+	movl	%esi,TIMER_TSTAMP(%ecx)		/* set timestamp */				; \
+	movl	%edi,TIMER_TSTAMP+4(%ecx)	/* set timestamp */
 
 #endif /* STAT_TIME */
 
@@ -407,7 +460,7 @@ Entry(db_task_gen_prot)
 Entry(db_task_start)
 	movl	%esp,%edx
 	subl	$(ISS32_SIZE),%edx
-	movl	%edx,%esp		/* allocate i386_saved_state on stack */
+	movl	%edx,%esp		/* allocate x86_saved_state on stack */
 	movl	%eax,R_ERR(%esp)
 	movl	%ebx,R_TRAPNO(%esp)
 	pushl	%edx
@@ -454,10 +507,10 @@ Entry(call_continuation)
 	pushl	%eax
 	call	EXT(thread_terminate)
 	
-
+	
 	
 /*******************************************************************************************************
- *	
+ *
  * All 64 bit task 'exceptions' enter lo_alltraps:
  *	esp	-> x86_saved_state_t
  * 
@@ -477,11 +530,26 @@ Entry(lo_alltraps)
 	jne	1f
 	movl	R64_CS(%esp),%eax	/* 64-bit user mode */
 1:
-	testb	$3,%eax
+	testb	$3,%al
 	jz	trap_from_kernel
 						/* user mode trap */
 	TIME_TRAP_UENTRY
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	movl	ACT_TASK(%ecx),%ebx
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp			/* switch to kernel stack */
 	sti
@@ -504,6 +572,7 @@ LEXT(return_from_trap)
 	sti				/* interrupts always enabled on return to user mode */
 
 	pushl	%ebx			/* save PCB stack */
+	xorl	%ebp,%ebp		/* Clear framepointer */
 	CCALL1(i386_astintr, $0)	/* take the AST */
 	cli
 	popl	%esp			/* switch back to PCB stack (w/exc link) */
@@ -518,7 +587,7 @@ LEXT(ret_to_user)
 	jmp	EXT(lo64_ret_to_user)
 
 
-	
+
 /*
  * Trap from kernel mode.  No need to switch stacks.
  * Interrupts must be off here - we will set them to state at time of trap
@@ -526,7 +595,12 @@ LEXT(ret_to_user)
  */
 trap_from_kernel:
 	movl	%esp, %eax		/* saved state addr */
-	CCALL1(kernel_trap, %eax)	/* to kernel trap routine */
+	pushl	R_EIP(%esp)		/* Simulate a CALL from fault point */
+	pushl   %ebp			/* Extend framepointer chain */
+	movl	%esp, %ebp
+	CCALL1(kernel_trap, %eax)	/* Call kernel trap handler */
+	popl	%ebp
+	addl	$4, %esp
 	cli
 
 	movl	%gs:CPU_PENDING_AST,%eax		/* get pending asts */
@@ -591,6 +665,21 @@ Entry(lo_allintrs)
 	
 	TIME_INT_ENTRY			/* do timing */
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx
+	movl	ACT_TASK(%ecx),%ebx
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	incl	%gs:CPU_PREEMPTION_LEVEL
 	incl	%gs:CPU_INTERRUPT_LEVEL
 
@@ -601,9 +690,6 @@ Entry(lo_allintrs)
 	xorl	%eax,%eax
 	movl	%eax,%gs:CPU_INT_STATE	/* clear intr state pointer */
 
-	.globl	EXT(return_to_iret)
-LEXT(return_to_iret)			/* (label for kdb_kintr and hardclock) */
-
 	decl	%gs:CPU_INTERRUPT_LEVEL
 	decl	%gs:CPU_PREEMPTION_LEVEL
 
@@ -632,7 +718,7 @@ LEXT(return_to_iret)			/* (label for kdb_kintr and hardclock) */
 	jne	3f
 	movl	R64_CS(%esp),%eax	/* 64-bit user mode */
 3:
-	testb	$3,%eax			/* user mode, */
+	testb	$3,%al			/* user mode, */
 	jnz	ast_from_interrupt_user	/* go handle potential ASTs */
 	/*
 	 * we only want to handle preemption requests if
@@ -670,7 +756,7 @@ int_from_intstack:
 	incl	%gs:CPU_PREEMPTION_LEVEL
 	incl	%gs:CPU_INTERRUPT_LEVEL
 
-	movl	%esp, %edx		/* i386_saved_state */
+	movl	%esp, %edx		/* x86_saved_state */
 	CCALL1(PE_incoming_interrupt, %edx)
 
 	decl	%gs:CPU_INTERRUPT_LEVEL
@@ -696,7 +782,7 @@ ast_from_interrupt_user:
  * 32bit Tasks
  * System call entries via INTR_GATE or sysenter:
  *
- *	esp	 -> i386_saved_state_t
+ *	esp	 -> x86_saved_state32_t
  *	cr3	 -> kernel directory
  *	esp	 -> low based stack
  *	gs	 -> CPU_DATA_GS
@@ -718,42 +804,83 @@ Entry(lo_sysenter)
 						/* > 0 => unix */
 	
 Entry(lo_unix_scall)
-        TIME_TRAP_UENTRY
+	TIME_TRAP_UENTRY
+
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+	addl	$1,TASK_SYSCALLS_UNIX(%ebx)	/* increment call count   */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
 
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
 	sti
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
-	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
-	addl	$1,TASK_SYSCALLS_UNIX(%ecx)	/* increment call count   */
 
 	CCALL1(unix_syscall, %ebx)
 	/*
 	 * always returns through thread_exception_return
 	 */
-	
+
 
 Entry(lo_mach_scall)
 	TIME_TRAP_UENTRY
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+	addl	$1,TASK_SYSCALLS_MACH(%ebx)	/* increment call count   */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
 	sti
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
-	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
-	addl	$1,TASK_SYSCALLS_MACH(%ecx)	/* increment call count   */
 
 	CCALL1(mach_call_munger, %ebx)
 	/*
 	 * always returns through thread_exception_return
 	 */
 
-	
+
 Entry(lo_mdep_scall)
-        TIME_TRAP_UENTRY
+	TIME_TRAP_UENTRY
+
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
 
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
@@ -763,21 +890,37 @@ Entry(lo_mdep_scall)
 	/*
 	 * always returns through thread_exception_return
 	 */
-	
+
 
 Entry(lo_diag_scall)
 	TIME_TRAP_UENTRY
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx	// Get the address of the kernel stack
 	xchgl	%ebx,%esp		// Switch to it, saving the previous
 
 	CCALL1(diagCall, %ebx)		// Call diagnostics
-	cli				// Disable interruptions just in case they were enabled
-	popl	%esp			// Get back the original stack
 	
 	cmpl	$0,%eax			// What kind of return is this?
-	jne	EXT(return_to_user)	// Normal return, do not check asts...
-				
+	je	2f
+	cli				// Disable interruptions just in case they were enabled
+	popl	%esp			// Get back the original stack
+	jmp	EXT(return_to_user)	// Normal return, do not check asts...
+2:	
 	CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
 		// pass what would be the diag syscall
 		// error return - cause an exception
@@ -818,47 +961,94 @@ Entry(lo_syscall)
 	cmpl	$(SYSCALL_CLASS_DIAG<<SYSCALL_CLASS_SHIFT), %ebx
 	je	EXT(lo64_diag_scall)
 
+	movl	%gs:CPU_KERNEL_STACK,%ebx
+	xchgl	%ebx,%esp		/* switch to kernel stack */
+
+	sti
+
 	/* Syscall class unknown */
 	CCALL3(i386_exception, $(EXC_SYSCALL), %eax, $1)
 	/* no return */
 
+
 Entry(lo64_unix_scall)
-        TIME_TRAP_UENTRY
+	TIME_TRAP_UENTRY
+
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+	addl	$1,TASK_SYSCALLS_UNIX(%ebx)	/* increment call count   */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
 
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
 	sti
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
-	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
-	addl	$1,TASK_SYSCALLS_UNIX(%ecx)	/* increment call count   */
 
 	CCALL1(unix_syscall64, %ebx)
 	/*
 	 * always returns through thread_exception_return
 	 */
-	
+
 
 Entry(lo64_mach_scall)
 	TIME_TRAP_UENTRY
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+	addl	$1,TASK_SYSCALLS_MACH(%ebx)	/* increment call count   */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
 	sti
-	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
-	movl	ACT_TASK(%ecx),%ecx		/* point to current task  */
-	addl	$1,TASK_SYSCALLS_MACH(%ecx)	/* increment call count   */
 
 	CCALL1(mach_call_munger64, %ebx)
 	/*
 	 * always returns through thread_exception_return
 	 */
 
-	
+
+
 Entry(lo64_mdep_scall)
-        TIME_TRAP_UENTRY
+	TIME_TRAP_UENTRY
 
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
+	
+1:
 	movl	%gs:CPU_KERNEL_STACK,%ebx
 	xchgl	%ebx,%esp		/* switch to kernel stack */
 
@@ -868,29 +1058,42 @@ Entry(lo64_mdep_scall)
 	/*
 	 * always returns through thread_exception_return
 	 */
-	
+
 
 Entry(lo64_diag_scall)
 	TIME_TRAP_UENTRY
 
-	movl	%gs:CPU_KERNEL_STACK,%ebx	// Get the address of the kernel stack
-	xchgl	%ebx,%esp		// Switch to it, saving the previous
+	movl	%gs:CPU_ACTIVE_THREAD,%ecx	/* get current thread     */
+	movl	ACT_TASK(%ecx),%ebx			/* point to current task  */
+
+	/* Check for active vtimers in the current task */
+	cmpl	$0,TASK_VTIMERS(%ebx)
+	jz		1f
+
+	/* Set a pending AST */
+	orl		$(AST_BSD),%gs:CPU_PENDING_AST
+
+	/* Set a thread AST (atomic) */
+	lock
+	orl		$(AST_BSD),ACT_AST(%ecx)
 	
-	pushl	%ebx			// Push the previous stack
+1:
+	movl	%gs:CPU_KERNEL_STACK,%ebx // Get the address of the kernel stack
+	xchgl	%ebx,%esp		// Switch to it, saving the previous
+
 	CCALL1(diagCall64, %ebx)	// Call diagnostics
+		
+	cmpl	$0,%eax			// What kind of return is this?
+	je	2f
 	cli				// Disable interruptions just in case they were enabled
 	popl	%esp			// Get back the original stack
-	
-	cmpl	$0,%eax			// What kind of return is this?
-	jne	EXT(return_to_user)	// Normal return, do not check asts...
-				
+	jmp	EXT(return_to_user)	// Normal return, do not check asts...
+2:	
 	CCALL3(i386_exception, $EXC_SYSCALL, $0x6000, $1)
+		// pass what would be the diag syscall
+		// error return - cause an exception
 	/* no return */
-	
 
-			
-/******************************************************************************************************
-			
 /**/
 /*
  * Utility routines.
@@ -903,20 +1106,20 @@ Entry(lo64_diag_scall)
  * arg1:	kernel address
  * arg2:	byte count
  */
-ENTRY(copyinphys_user)
+Entry(copyinphys_user)
 	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
 	mov	%cx,%ds
 
-ENTRY(copyinphys_kern)
+Entry(copyinphys_kern)
 	movl	$(PHYS_WINDOW_SEL),%ecx	/* physical access through kernel window */
 	mov	%cx,%es
 	jmp	copyin_common
 
-ENTRY(copyin_user)
+Entry(copyin_user)
 	movl	$(USER_WINDOW_SEL),%ecx	/* user data segment access through kernel window */
 	mov	%cx,%ds
 
-ENTRY(copyin_kern)
+Entry(copyin_kern)
 
 copyin_common:
 	pushl	%esi
@@ -1068,7 +1271,6 @@ copyout_fail:
 	movl	$(EFAULT),%eax		/* return error for failure */
 	jmp	copyout_ret		/* pop frame and return */
 
-
 /*
  * io register must not be used on slaves (no AT bus)
  */
@@ -1351,70 +1553,6 @@ rdmsr_fail:
 	RECOVERY_SECTION
 	RECOVER_TABLE_END
 
-
-
-ENTRY(dr6)
-	movl	%db6, %eax
-	ret
-
-/*	dr<i>(address, type, len, persistence)
- */
-ENTRY(dr0)
-	movl	S_ARG0, %eax
-	movl	%eax,EXT(dr_addr)
-	movl	%eax, %db0
-	movl	$0, %ecx
-	jmp	0f
-ENTRY(dr1)
-	movl	S_ARG0, %eax
-	movl	%eax,EXT(dr_addr)+1*4
-	movl	%eax, %db1
-	movl	$2, %ecx
-	jmp	0f
-ENTRY(dr2)
-	movl	S_ARG0, %eax
-	movl	%eax,EXT(dr_addr)+2*4
-	movl	%eax, %db2
-	movl	$4, %ecx
-	jmp	0f
-
-ENTRY(dr3)
-	movl	S_ARG0, %eax
-	movl	%eax,EXT(dr_addr)+3*4
-	movl	%eax, %db3
-	movl	$6, %ecx
-
-0:
-	pushl	%ebp
-	movl	%esp, %ebp
-
-	movl	%db7, %edx
-	movl	%edx,EXT(dr_addr)+4*4
-	andl	dr_msk(,%ecx,2),%edx	/* clear out new entry */
-	movl	%edx,EXT(dr_addr)+5*4
-	movzbl	B_ARG3, %eax
-	andb	$3, %al
-	shll	%cl, %eax
-	orl	%eax, %edx
-
-	movzbl	B_ARG1, %eax
-	andb	$3, %al
-	addb	$0x10, %cl
-	shll	%cl, %eax
-	orl	%eax, %edx
-
-	movzbl	B_ARG2, %eax
-	andb	$3, %al
-	addb	$0x2, %cl
-	shll	%cl, %eax
-	orl	%eax, %edx
-
-	movl	%edx, %db7
-	movl	%edx,EXT(dr_addr)+7*4
-	movl	%edx, %eax
-	leave
-	ret
-
 	.data
 dr_msk:
 	.long	~0x000f0003
@@ -1427,15 +1565,6 @@ ENTRY(dr_addr)
 
 	.text
 
-ENTRY(get_cr0)
-	movl	%cr0, %eax
-	ret
-
-ENTRY(set_cr0)
-	movl	4(%esp), %eax
-	movl	%eax, %cr0
-	ret
-
 #ifndef	SYMMETRY
 
 /*
diff --git a/osfmk/i386/loose_ends.c b/osfmk/i386/loose_ends.c
index ad6dca447..d5ed99e51 100644
--- a/osfmk/i386/loose_ends.c
+++ b/osfmk/i386/loose_ends.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -88,7 +94,6 @@ extern void		pmap_set_reference(ppnum_t pn);
 extern void		mapping_set_mod(ppnum_t pa); 
 extern void		mapping_set_ref(ppnum_t pn);
 
-extern void		fillPage(ppnum_t pa, unsigned int fill);
 extern void		ovbcopy(const char	*from,
 				char		*to,
 				vm_size_t	nbytes);
@@ -98,6 +103,13 @@ void machine_callstack(natural_t *buf, vm_size_t callstack_max);
 #define value_64bit(value)  ((value) & 0xFFFFFFFF00000000LL)
 #define low32(x)  ((unsigned int)((x) & 0x00000000FFFFFFFFLL))
 
+void
+bzero_phys_nc(
+	      addr64_t src64,
+	      vm_size_t bytes)
+{
+  bzero_phys(src64,bytes);
+}
 
 void
 bzero_phys(
@@ -105,22 +117,14 @@ bzero_phys(
 	   vm_size_t bytes)
 {
         mapwindow_t *map;
-	pt_entry_t save;
 
         mp_disable_preemption();
-	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD));
-	if (map == 0) {
-	        panic("bzero_phys: CMAP busy");
-	}
-	save = *map->prv_CMAP;
 
-	invlpg((uintptr_t)map->prv_CADDR);
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)src64 & PG_FRAME) | INTEL_PTE_REF | INTEL_PTE_MOD));
 
 	bzero((void *)((uintptr_t)map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)), bytes);
 
-	if (save != *map->prv_CMAP)
-	        panic("bzero_phys: CMAP changed");
-	*map->prv_CMAP = 0;
+	pmap_put_mapwindow(map);
 
 	mp_enable_preemption();
 }
@@ -137,7 +141,6 @@ bcopy_phys(
 	   vm_size_t bytes)
 {
         mapwindow_t *src_map, *dst_map;
-	pt_entry_t save1, save2;
 
 	/* ensure we stay within a page */
 	if ( ((((uint32_t)src64 & (NBPG-1)) + bytes) > NBPG) || ((((uint32_t)dst64 & (NBPG-1)) + bytes) > NBPG) ) {
@@ -149,23 +152,11 @@ bcopy_phys(
 	dst_map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | ((pmap_paddr_t)dst64 & PG_FRAME) |
 						  INTEL_PTE_REF | INTEL_PTE_MOD));
 
-	if (src_map == 0 || dst_map == 0) {
-	        panic("bcopy_phys: CMAP busy");
-	}
-	save1 = *src_map->prv_CMAP;
-	save2 = *dst_map->prv_CMAP;
-
-	invlpg((uintptr_t)src_map->prv_CADDR);
-	invlpg((uintptr_t)dst_map->prv_CADDR);
-
 	bcopy((void *) ((uintptr_t)src_map->prv_CADDR | ((uint32_t)src64 & INTEL_OFFMASK)),
 	      (void *) ((uintptr_t)dst_map->prv_CADDR | ((uint32_t)dst64 & INTEL_OFFMASK)), bytes);
 
-	if ( (save1 != *src_map->prv_CMAP) || (save2 != *dst_map->prv_CMAP))
-	        panic("bcopy_phys CMAP changed");
-
-	*src_map->prv_CMAP = 0;
-	*dst_map->prv_CMAP = 0;
+	pmap_put_mapwindow(src_map);
+	pmap_put_mapwindow(dst_map);
 
 	mp_enable_preemption();
 }
@@ -206,16 +197,10 @@ ml_phys_read_data(pmap_paddr_t paddr, int size )
 {
         mapwindow_t *map;
 	unsigned int result;
-	pt_entry_t save;
 
 	mp_disable_preemption();
-	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
-	if (map == 0) {
-		panic("ml_phys_read_data: CMAP busy");
-	}
 
-	save = *map->prv_CMAP;
-	invlpg((uintptr_t)map->prv_CADDR);
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
 
         switch (size) {
             unsigned char s1;
@@ -233,10 +218,8 @@ ml_phys_read_data(pmap_paddr_t paddr, int size )
             result = *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
             break;
         }
+        pmap_put_mapwindow(map);
 
-	if (save != *map->prv_CMAP)
-	        panic("ml_phys_read_data CMAP changed");
-        *map->prv_CMAP = 0;
 	mp_enable_preemption();
 
         return result;
@@ -247,31 +230,23 @@ ml_phys_read_long_long(pmap_paddr_t paddr )
 {
         mapwindow_t *map;
 	unsigned long long result;
-	pt_entry_t save;
 
 	mp_disable_preemption();
-	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
-
-	if (map == 0) {
-		panic("ml_phys_read_long_long: CMAP busy");
-	}
 
-	save = *map->prv_CMAP;
-	invlpg((uintptr_t)map->prv_CADDR);
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | (paddr & PG_FRAME) | INTEL_PTE_REF));
 
 	result = *(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK));
 
-	if (save != *map->prv_CMAP)
-	        panic("ml_phys_read_long_long CMAP changed");
-	*map->prv_CMAP = 0;
+        pmap_put_mapwindow(map);
+
 	mp_enable_preemption();
 
-	return result;
+        return result;
 }
 
 
 
-unsigned int ml_phys_read(vm_offset_t paddr)
+unsigned int ml_phys_read( vm_offset_t paddr)
 {
         return ml_phys_read_data((pmap_paddr_t)paddr, 4);
 }
@@ -331,19 +306,12 @@ static void
 ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
 {
         mapwindow_t *map;
-	pt_entry_t save;
 
 	mp_disable_preemption();
+
 	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
 					  INTEL_PTE_REF | INTEL_PTE_MOD));
 
-	if (map == 0) {
-		panic("ml_phys_write_data: CMAP busy");
-	}
-
-	save = *map->prv_CMAP;
-	invlpg((uintptr_t)map->prv_CADDR);
-
         switch (size) {
         case 1:
 	    *(unsigned char *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = (unsigned char)data;
@@ -356,10 +324,7 @@ ml_phys_write_data(pmap_paddr_t paddr, unsigned long data, int size)
 	    *(unsigned int *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data;
             break;
         }
-
-	if (save != *map->prv_CMAP)
-	        panic("ml_phys_write_data CMAP changed");
-	*map->prv_CMAP = 0;
+        pmap_put_mapwindow(map);
 
 	mp_enable_preemption();
 }
@@ -368,23 +333,16 @@ static void
 ml_phys_write_long_long(pmap_paddr_t paddr, unsigned long long data)
 {
         mapwindow_t *map;
-	pt_entry_t save;
 
 	mp_disable_preemption();
+
 	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (paddr & PG_FRAME) | 
 					      INTEL_PTE_REF | INTEL_PTE_MOD));
-	if (map == 0) {
-		panic("ml_phys_write_data: CMAP busy");
-	}
-
-	save = *map->prv_CMAP;
-	invlpg((uintptr_t)map->prv_CADDR);
 
 	*(unsigned long long *)((uintptr_t)map->prv_CADDR | ((uint32_t)paddr & INTEL_OFFMASK)) = data;
 
-	if (save != *map->prv_CMAP)
-	        panic("ml_phys_write_data CMAP changed");
-	*map->prv_CMAP = 0;
+        pmap_put_mapwindow(map);
+
 	mp_enable_preemption();
 }
 
@@ -453,9 +411,12 @@ void ml_phys_write_double_64(addr64_t paddr64, unsigned long long data)
 boolean_t
 ml_probe_read(vm_offset_t paddr, unsigned int *val)
 {
-        *val = ml_phys_read((pmap_paddr_t)paddr);
+    if ((PAGE_SIZE - (paddr & PAGE_MASK)) < 4)
+        return FALSE;
+
+    *val = ml_phys_read((pmap_paddr_t)paddr);
 
-	return TRUE;
+    return TRUE;
 }
 
 /*
@@ -467,9 +428,11 @@ ml_probe_read(vm_offset_t paddr, unsigned int *val)
 boolean_t 
 ml_probe_read_64(addr64_t paddr64, unsigned int *val)
 {
-        *val = ml_phys_read_64((pmap_paddr_t)paddr64);
+    if ((PAGE_SIZE - (paddr64 & PAGE_MASK)) < 4)
+        return FALSE;
 
-	return TRUE;
+    *val = ml_phys_read_64((pmap_paddr_t)paddr64);
+    return TRUE;
 }
 
 
@@ -493,9 +456,7 @@ int bcmp(
 }
 
 int
-memcmp(s1, s2, n)
-	const void *s1, *s2;
-	size_t n;
+memcmp(const void *s1, const void *s2, size_t n)
 {
 	if (n != 0) {
 		const unsigned char *p1 = s1, *p2 = s2;
@@ -526,80 +487,11 @@ strlen(
 }
 
 uint32_t
-hw_atomic_add(
-	uint32_t	*dest,
-	uint32_t	delt)
-{
-	uint32_t	oldValue;
-	uint32_t	newValue;
-	
-	do {
-		oldValue = *dest;
-		newValue = (oldValue + delt);
-	} while (!OSCompareAndSwap((UInt32)oldValue,
-									(UInt32)newValue, (UInt32 *)dest));
-	
-	return newValue;
-}
-
-uint32_t
-hw_atomic_sub(
-	uint32_t	*dest,
-	uint32_t	delt)
-{
-	uint32_t	oldValue;
-	uint32_t	newValue;
-	
-	do {
-		oldValue = *dest;
-		newValue = (oldValue - delt);
-	} while (!OSCompareAndSwap((UInt32)oldValue,
-									(UInt32)newValue, (UInt32 *)dest));
-	
-	return newValue;
-}
-
-uint32_t
-hw_atomic_or(
-	uint32_t	*dest,
-	uint32_t	mask)
-{
-	uint32_t	oldValue;
-	uint32_t	newValue;
-	
-	do {
-		oldValue = *dest;
-		newValue = (oldValue | mask);
-	} while (!OSCompareAndSwap((UInt32)oldValue,
-									(UInt32)newValue, (UInt32 *)dest));
-	
-	return newValue;
-}
-
-uint32_t
-hw_atomic_and(
-	uint32_t	*dest,
-	uint32_t	mask)
+hw_compare_and_store(uint32_t oldval, uint32_t newval, volatile uint32_t *dest)
 {
-	uint32_t	oldValue;
-	uint32_t	newValue;
-	
-	do {
-		oldValue = *dest;
-		newValue = (oldValue & mask);
-	} while (!OSCompareAndSwap((UInt32)oldValue,
-									(UInt32)newValue, (UInt32 *)dest));
-	
-	return newValue;
-}
-
-uint32_t
-hw_compare_and_store(
-	uint32_t	oldval,
-	uint32_t	newval,
-	uint32_t	*dest)
-{
-	return OSCompareAndSwap((UInt32)oldval, (UInt32)newval, (UInt32 *)dest);
+	return OSCompareAndSwap((UInt32)oldval,
+				(UInt32)newval,
+				(volatile UInt32 *)dest);
 }
 
 #if	MACH_ASSERT
@@ -616,31 +508,26 @@ void machine_callstack(
 
 #endif	/* MACH_ASSERT */
 
+void fillPage(ppnum_t pa, unsigned int fill)
+{
+        mapwindow_t *map;
+	pmap_paddr_t src;
+	int i;
+	int cnt = PAGE_SIZE/sizeof(unsigned int);
+	unsigned int *addr;
 
+	mp_disable_preemption();
 
+	src = i386_ptob(pa);
+	map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
+					      INTEL_PTE_REF | INTEL_PTE_MOD));
 
-void fillPage(ppnum_t pa, unsigned int fill)
-{
-    mapwindow_t *map;
-    pmap_paddr_t src;
-    int i;
-    int cnt = PAGE_SIZE/sizeof(unsigned int);
-    unsigned int *addr;
-
-    mp_disable_preemption();
-    src = i386_ptob(pa);
-    map = pmap_get_mapwindow((pt_entry_t)(INTEL_PTE_VALID | INTEL_PTE_RW | (src & PG_FRAME) | 
-					  INTEL_PTE_REF | INTEL_PTE_MOD));
-    if (map == 0) {
-        panic("fillPage: CMAP busy");
-    }
-    invlpg((uintptr_t)map->prv_CADDR);
+	for (i = 0, addr = (unsigned int *)map->prv_CADDR; i < cnt ; i++ )
+	        *addr++ = fill;
 
-    for (i = 0, addr = (unsigned int *)map->prv_CADDR; i < cnt ; i++ )
-        *addr++ = fill;
+	pmap_put_mapwindow(map);
 
-    *map->prv_CMAP = 0;
-    mp_enable_preemption();
+	mp_enable_preemption();
 }
 
 static inline void __sfence(void)
@@ -676,9 +563,6 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
         addr   = pa - offset;
 
         map = pmap_get_mapwindow((pt_entry_t)(i386_ptob(atop_64(addr)) | INTEL_PTE_VALID));
-        if (map == 0) {
-                panic("cache_flush_page_phys: CMAP busy");
-        }
 
         count += offset;
         offset = addr & ((addr64_t) (page_size - 1));
@@ -688,9 +572,6 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
         {
             if (chunk > count)
                 chunk = count;
-
-            *map->prv_CMAP = (pt_entry_t)(i386_ptob(atop_64(addr)) | INTEL_PTE_VALID);
-            invlpg((uintptr_t)map->prv_CADDR);
     
             for (; offset < chunk; offset += linesize)
                 __clflush((void *)(((uintptr_t)map->prv_CADDR) + offset));
@@ -699,10 +580,15 @@ void dcache_incoherent_io_store64(addr64_t pa, unsigned int count)
             addr  += chunk;
             chunk  = page_size;
             offset = 0;
+
+	    if (count) {
+	        pmap_store_pte(map->prv_CMAP, (pt_entry_t)(i386_ptob(atop_64(addr)) | INTEL_PTE_VALID));
+		invlpg((uintptr_t)map->prv_CADDR);
+	    }
         }
         while (count);
 
-        *map->prv_CMAP = 0;
+        pmap_put_mapwindow(map);
 
         (void) ml_set_interrupts_enabled(istate);
 
@@ -758,19 +644,13 @@ cache_flush_page_phys(ppnum_t pa)
 	istate = ml_set_interrupts_enabled(FALSE);
 
         map = pmap_get_mapwindow((pt_entry_t)(i386_ptob(pa) | INTEL_PTE_VALID));
-	if (map == 0) {
-		panic("cache_flush_page_phys: CMAP busy");
-        }
-
-	invlpg((uintptr_t)map->prv_CADDR);
 
 	for (i = 0, cacheline_addr = (unsigned char *)map->prv_CADDR;
 	     i < cachelines_in_page;
 	     i++, cacheline_addr += cacheline_size) {
 		__clflush((void *) cacheline_addr);
 	}
-
-        *map->prv_CMAP = 0;
+        pmap_put_mapwindow(map);
 
 	(void) ml_set_interrupts_enabled(istate);
 
@@ -778,18 +658,6 @@ cache_flush_page_phys(ppnum_t pa)
 }
 
 
-void exit_funnel_section(void)
-{
-        thread_t thread;
-
-	thread = current_thread();
-
-        if (thread->funnel_lock)
-	        (void) thread_funnel_set(thread->funnel_lock, FALSE);
-}
-
-
-
 /*
  * the copy engine has the following characteristics
  *   - copyio handles copies to/from user or kernel space
@@ -883,7 +751,8 @@ void inval_copy_windows(thread_t thread)
 
 
 static int
-copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
+copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
+       vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
 {
         thread_t	thread;
 	pmap_t		pmap;
@@ -905,14 +774,20 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes
 
 	thread = current_thread();
 
-	KERNEL_DEBUG(debug_type | DBG_FUNC_START, (int)(user_addr >> 32), (int)user_addr, (int)nbytes, thread->machine.copyio_state, 0);
+	KERNEL_DEBUG(debug_type | DBG_FUNC_START, (int)(user_addr >> 32), (int)user_addr,
+		     (int)nbytes, thread->machine.copyio_state, 0);
 
 	if (nbytes == 0) {
-	        KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)user_addr, (int)kernel_addr, (int)nbytes, 0, 0);
+	        KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)user_addr,
+			     (unsigned)kernel_addr, (unsigned)nbytes, 0, 0);
 	        return (0);
 	}
         pmap = thread->map->pmap;
 
+#if CONFIG_DTRACE
+	thread->machine.specFlags |= CopyIOActive;
+#endif /* CONFIG_DTRACE */
+
         if (pmap == kernel_pmap || use_kernel_map) {
 
 	        kern_vaddr = (vm_offset_t)user_addr;
@@ -939,14 +814,21 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes
 		        error = copyoutphys_kern(kernel_addr, kern_vaddr, nbytes);
 			break;
 		}
-		KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)kern_vaddr, (int)kernel_addr, (int)nbytes, error | 0x80000000, 0);
+		KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)kern_vaddr,
+			     (unsigned)kernel_addr, (unsigned)nbytes,
+			     error | 0x80000000, 0);
+
+#if CONFIG_DTRACE
+	thread->machine.specFlags &= ~CopyIOActive;
+#endif /* CONFIG_DTRACE */
 
 		return (error);
 	}
 	user_base = user_addr & ~((user_addr_t)(NBPDE - 1));
 	user_offset = user_addr & (NBPDE - 1);
 
-	KERNEL_DEBUG(debug_type | DBG_FUNC_NONE, (int)(user_base >> 32), (int)user_base, (int)user_offset, 0, 0);
+	KERNEL_DEBUG(debug_type | DBG_FUNC_NONE, (int)(user_base >> 32), (int)user_base,
+		     (int)user_offset, 0, 0);
 
 	cnt = NBPDE - user_offset;
 
@@ -994,7 +876,9 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes
 
 		        copyio_state = WINDOWS_DIRTY;
 
-			KERNEL_DEBUG(0xeff70040 | DBG_FUNC_NONE, window_index, (int)user_base, (int)updp, (int)kpdp, 0);
+			KERNEL_DEBUG(0xeff70040 | DBG_FUNC_NONE, window_index,
+				     (unsigned)user_base, (unsigned)updp,
+				     (unsigned)kpdp, 0);
 
 		}
 #if JOE_DEBUG
@@ -1022,7 +906,8 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes
 		}
 		user_offset += (window_index * NBPDE);
 
-		KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (int)user_offset, (int)kernel_addr, cnt, 0, 0);
+		KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (unsigned)user_offset,
+			     (unsigned)kernel_addr, cnt, 0, 0);
 
 	        switch (copy_type) {
 
@@ -1107,7 +992,12 @@ copyio(int copy_type, user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes
 done:
 	thread->machine.copyio_state = WINDOWS_CLOSED;
 
-	KERNEL_DEBUG(debug_type | DBG_FUNC_END, (int)user_addr, (int)kernel_addr, (int)nbytes, error, 0);
+	KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)user_addr,
+		     (unsigned)kernel_addr, (unsigned)nbytes, error, 0);
+
+#if CONFIG_DTRACE
+	thread->machine.specFlags &= ~CopyIOActive;
+#endif /* CONFIG_DTRACE */
 
 	return (error);
 }
@@ -1117,7 +1007,7 @@ static int
 copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 {
         pmap_paddr_t paddr;
-	user_addr_t  vaddr;
+	user_addr_t vaddr;
 	char        *window_offset;
 	pt_entry_t  pentry;
 	int         ctype;
@@ -1189,7 +1079,7 @@ copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 			 * flushing the tlb after it reloaded the page table from machine.physwindow_pte
 			 */
 			istate = ml_set_interrupts_enabled(FALSE);
-			*(current_cpu_datap()->cpu_physwindow_ptep) = pentry;
+			pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), pentry);
 			(void) ml_set_interrupts_enabled(istate);
 
 			invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);
@@ -1208,8 +1098,6 @@ copyio_phys(addr64_t source, addr64_t sink, vm_size_t csize, int which)
 	return (retval);
 }
 
-
-
 int
 copyinmsg(const user_addr_t user_addr, char *kernel_addr, vm_size_t nbytes)
 {
@@ -1233,23 +1121,24 @@ copyinstr(const user_addr_t user_addr,  char *kernel_addr, vm_size_t nbytes, vm_
 int
 copyoutmsg(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 {
-        return (copyio(COPYOUT, user_addr, (char *)kernel_addr, nbytes, NULL, 0));
+        return (copyio(COPYOUT, user_addr, kernel_addr, nbytes, NULL, 0));
 }
 
 int
-copyout(const char *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
+copyout(const void *kernel_addr, user_addr_t user_addr, vm_size_t nbytes)
 {
-        return (copyio(COPYOUT, user_addr, (char *)kernel_addr, nbytes, NULL, 0));
+        return (copyio(COPYOUT, user_addr, kernel_addr, nbytes, NULL, 0));
 }
 
 
-kern_return_t copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
+kern_return_t
+copypv(addr64_t src64, addr64_t snk64, unsigned int size, int which)
 {
 	unsigned int lop, csize;
 	int bothphys = 0;
 	
-
-	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (int)src64, (int)snk64, size, which, 0);
+	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_START, (unsigned)src64,
+		     (unsigned)snk64, size, which, 0);
 
 	if ((which & (cppvPsrc | cppvPsnk)) == 0 )				/* Make sure that only one is virtual */
 		panic("copypv: no more than 1 parameter may be virtual\n");	/* Not allowed */
@@ -1303,11 +1192,19 @@ kern_return_t copypv(addr64_t src64, addr64_t snk64, unsigned int size, int whic
 		if (which & cppvFsnk)
 		        flush_dcache64(snk64, csize, 1);	/* If requested, flush sink after move */
 #endif
-		size  -= csize;					/* Calculate what is left */
+		size   -= csize;					/* Calculate what is left */
 		snk64 += csize;					/* Bump sink to next physical address */
 		src64 += csize;					/* Bump source to next physical address */
 	}
-	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (int)src64, (int)snk64, size, which, 0);
+	KERNEL_DEBUG(0xeff7004c | DBG_FUNC_END, (unsigned)src64,
+		     (unsigned)snk64, size, which, 0);
 
 	return KERN_SUCCESS;
 }
+
+#if !MACH_KDP
+void
+kdp_register_callout(void)
+{
+}
+#endif
diff --git a/osfmk/i386/lowglobals.h b/osfmk/i386/lowglobals.h
index 22bfed701..a8e5ddd52 100644
--- a/osfmk/i386/lowglobals.h
+++ b/osfmk/i386/lowglobals.h
@@ -1,24 +1,31 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 /*
  *		Header files for the Low Memory Globals (lg) 
  */
@@ -47,16 +54,21 @@ typedef struct lowglo {
 
 	unsigned char	lgVerCode[8];		/* 0x2000 System verification code */
 	unsigned long long lgZero;		/* 0x2008 Double constant 0 */
-	uint32_t	lgRsv010[3];		/* 0x2010 Reserved */
+	uint32_t	lgRsv010;		/* 0x2010 Reserved */
+	uint32_t	lgCHUDXNUfnStart;	/* 0x2014 CHUD XNU function glue
+						 * table */
+	uint32_t	lgRsv018;		/* 0x2018 Reserved */
 	uint32_t	lgVersion;		/* 0x201C Pointer to kernel version string */
 	uint32_t	lgRsv020[280];		/* 0X2020 Reserved */
 	uint32_t	lgKmodptr;		/* 0x2480 Pointer to kmod, debugging aid */
 	uint32_t	lgTransOff;		/* 0x2484 Pointer to kdp_trans_off, debugging aid */
-	uint32_t	lgRsv028;		/* 0x2488 Reserved */
+	uint32_t	lgReadIO;		/* 0x2488 Pointer to kdp_read_io, debugging aid */
 	uint32_t	lgDevSlot1;		/* 0x248C For developer use */
 	uint32_t	lgDevSlot2;		/* 0x2490 For developer use */
-	uint32_t	lgRsv494[731];		/* Reserved - push to 1 page */
+	uint32_t	lgOSVersion;		/* 0x2494 Pointer to OS version string */
+	uint32_t	lgRebootFlag;		/* 0x2498 Pointer to debugger reboot trigger */
+	uint32_t	lgRsv49C[729];		/* 0x549C Reserved - push to 1 page */
 } lowglo;
-
-
+#pragma pack()
+extern lowglo lowGlo;
 #endif /* _LOW_MEMORY_GLOBALS_H_ */
diff --git a/osfmk/i386/lowmem_vectors.s b/osfmk/i386/lowmem_vectors.s
index bf4cd2941..b274b8ee5 100644
--- a/osfmk/i386/lowmem_vectors.s
+++ b/osfmk/i386/lowmem_vectors.s
@@ -1,24 +1,31 @@
 /*
  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 /*
  * @OSF_COPYRIGHT@
  */
@@ -50,6 +57,7 @@
 
 #include <platforms.h>
 #include <mach_kdb.h>
+#include <mach_kdp.h>
 
 #include <i386/asm.h>
 #include <i386/proc_reg.h>
@@ -71,13 +79,24 @@ EXT(lowGlo):
 	.long   0			/* 0x2008 Double constant 0 */
 	.long   0
 	.long	0			/* 0x2010 Reserved */
-	.long	0			/* 0x2014 Reserved */
+	.long	0			/* 0x2014 Zero */
 	.long	0			/* 0x2018 Reserved */
 	.long	EXT(version)		/* 0x201C Pointer to kernel version string */
 	.fill	280, 4, 0		/* 0x2020 Reserved */
 	.long	EXT(kmod)		/* 0x2480 Pointer to kmod, debugging aid */
+#if MACH_KDP
 	.long	EXT(kdp_trans_off)	/* 0x2484 Pointer to kdp_trans_off, debugging aid */
+	.long	EXT(kdp_read_io)	/* 0x2488 Pointer to kdp_read_io, debugging aid */
+#else
+	.long	0			/* 0x2484 Reserved */
 	.long	0			/* 0x2488 Reserved */
+#endif
 	.long	0			/* 0x248C Reserved for developer use */
 	.long	0			/* 0x2490 Reserved for developer use */
-	.fill	731, 4, 0
+	.long	EXT(osversion)		/* 0x2494 Pointer to osversion string */
+#if MACH_KDP
+	.long	EXT(flag_kdp_trigger_reboot) /* 0x2498 Pointer to debugger reboot trigger */
+#else
+	.long	0			/* 0x2498 Reserved */
+#endif
+	.fill	729, 4, 0
diff --git a/osfmk/i386/machdep_call.c b/osfmk/i386/machdep_call.c
index 1cebbe4e0..a21a9a792 100644
--- a/osfmk/i386/machdep_call.c
+++ b/osfmk/i386/machdep_call.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
diff --git a/osfmk/i386/machdep_call.h b/osfmk/i386/machdep_call.h
index cab1fb7e2..bfc7c55ec 100644
--- a/osfmk/i386/machdep_call.h
+++ b/osfmk/i386/machdep_call.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1992 NeXT Computer, Inc.
@@ -38,7 +44,8 @@ typedef union {
 	kern_return_t		(*args_3)(uint32_t,uint32_t,uint32_t);
 	kern_return_t		(*args_4)(uint32_t,uint32_t,uint32_t,uint32_t);
 	kern_return_t		(*args_var)(uint32_t,...);
-	int			(*args_bsd_3)(int *, uint32_t,uint32_t,uint32_t);
+	int			(*args_bsd_3)(uint32_t *, uint32_t,
+					      uint32_t, uint32_t);
 } machdep_call_routine_t;
 
 #define MACHDEP_CALL_ROUTINE(func, args)	\
@@ -46,7 +53,7 @@ typedef union {
 	
 #define MACHDEP_CALL_ROUTINE64(func, args)	\
 	{ { .args64_ ## args = func }, args, 0 }
-	
+
 #define MACHDEP_BSD_CALL_ROUTINE(func, args)	\
 	{ { .args_bsd_ ## args = func }, args, 1 }
 
@@ -67,8 +74,8 @@ extern kern_return_t		thread_fast_set_cthread_self(uint32_t);
 extern kern_return_t		thread_fast_set_cthread_self64(uint64_t);
 extern kern_return_t		thread_set_user_ldt(uint32_t,uint32_t,uint32_t);
 
-extern int			i386_set_ldt(int *,uint32_t,uint32_t,uint32_t);
-extern int			i386_get_ldt(int *,uint32_t,uint32_t,uint32_t);
+extern int		i386_set_ldt(uint32_t *,uint32_t,uint32_t,uint32_t);
+extern int		i386_get_ldt(uint32_t *,uint32_t,uint32_t,uint32_t);
 
 extern void			machdep_syscall(x86_saved_state_t *);
 extern void			machdep_syscall64(x86_saved_state_t *);
diff --git a/osfmk/i386/machine_check.c b/osfmk/i386/machine_check.c
index 08fbb1116..8abf223f9 100644
--- a/osfmk/i386/machine_check.c
+++ b/osfmk/i386/machine_check.c
@@ -26,6 +26,7 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#include <kern/kalloc.h>
 #include <i386/cpu_data.h>
 #include <i386/cpuid.h>
 #include <i386/machine_check.h>
@@ -36,12 +37,29 @@
 static boolean_t	mca_initialized = FALSE;
 static boolean_t	mca_MCE_present = FALSE;
 static boolean_t	mca_MCA_present = FALSE;
+static uint32_t		mca_family = 0;
 static unsigned int	mca_error_bank_count = 0;
 static boolean_t	mca_control_MSR_present = FALSE;
 static boolean_t	mca_threshold_status_present = FALSE;
 static boolean_t	mca_extended_MSRs_present = FALSE;
 static unsigned int	mca_extended_MSRs_count = 0;
 static ia32_mcg_cap_t	ia32_mcg_cap;
+static boolean_t	mca_exception_taken = FALSE;
+
+decl_simple_lock_data(static, mca_lock);
+
+typedef struct {
+	ia32_mci_ctl_t		mca_mci_ctl;
+	ia32_mci_status_t	mca_mci_status;
+	ia32_mci_misc_t		mca_mci_misc;
+	ia32_mci_addr_t		mca_mci_addr;
+} mca_mci_bank_t;
+
+typedef struct mca_state {
+	ia32_mcg_ctl_t		mca_mcg_ctl;
+	ia32_mcg_status_t	mca_mcg_status;
+	mca_mci_bank_t		mca_error_bank[0];
+} mca_state_t;
 
 static void
 mca_get_availability(void)
@@ -51,6 +69,7 @@ mca_get_availability(void)
 
 	mca_MCE_present = (features & CPUID_FEATURE_MCE) != 0;
 	mca_MCA_present = (features & CPUID_FEATURE_MCA) != 0;
+	mca_family = family;
 	
 	/*
 	 * If MCA, the number of banks etc is reported by the IA32_MCG_CAP MSR.
@@ -79,6 +98,7 @@ mca_cpu_init(void)
 	if (!mca_initialized) {
 		mca_get_availability();
 		mca_initialized = TRUE;
+		simple_lock_init(&mca_lock, 0);
 	}
 
 	if (mca_MCA_present) {
@@ -87,7 +107,7 @@ mca_cpu_init(void)
 		if (mca_control_MSR_present)
 			wrmsr64(IA32_MCG_CTL, IA32_MCG_CTL_ENABLE);
 	
-		switch (cpuid_family()) {
+		switch (mca_family) {
 		case 0x06:
 			/* Enable all but mc0 */
 			for (i = 1; i < mca_error_bank_count; i++)
@@ -115,6 +135,68 @@ mca_cpu_init(void)
 	}
 }
 
+void
+mca_cpu_alloc(cpu_data_t	*cdp)
+{
+	vm_size_t	mca_state_size;
+
+	/*
+	 * Allocate space for an array of error banks.
+	 */
+	mca_state_size = sizeof(mca_state_t) +
+				sizeof(mca_mci_bank_t) * mca_error_bank_count;
+	cdp->cpu_mca_state = kalloc(mca_state_size);
+	if (cdp->cpu_mca_state == NULL) {
+		printf("mca_cpu_alloc() failed for cpu %d\n", cdp->cpu_number);
+		return;
+	}
+	bzero((void *) cdp->cpu_mca_state, mca_state_size);
+
+	/*
+	 * If the boot processor is yet have its allocation made,
+	 * do this now.
+	 */
+	if (cpu_datap(master_cpu)->cpu_mca_state == NULL)
+		mca_cpu_alloc(cpu_datap(master_cpu));
+}
+
+static void
+mca_save_state(void)
+{
+	mca_state_t	*mca_state;
+	mca_mci_bank_t  *bank;
+	unsigned int	i;
+
+	assert(!ml_get_interrupts_enabled() || get_preemption_level() > 0);
+
+	mca_state = (mca_state_t *) current_cpu_datap()->cpu_mca_state;
+	if  (mca_state == NULL)
+		return;
+
+	mca_state->mca_mcg_ctl = mca_control_MSR_present ?
+					rdmsr64(IA32_MCG_CTL) : 0ULL;	
+	mca_state->mca_mcg_status.u64 = rdmsr64(IA32_MCG_STATUS);
+
+ 	bank = (mca_mci_bank_t *) &mca_state->mca_error_bank[0];
+	for (i = 0; i < mca_error_bank_count; i++, bank++) {
+		bank->mca_mci_ctl        = rdmsr64(IA32_MCi_CTL(i));	
+		bank->mca_mci_status.u64 = rdmsr64(IA32_MCi_STATUS(i));	
+		if (!bank->mca_mci_status.bits.val)
+			continue;
+		bank->mca_mci_misc = (bank->mca_mci_status.bits.miscv)?
+					rdmsr64(IA32_MCi_MISC(i)) : 0ULL;	
+		bank->mca_mci_addr = (bank->mca_mci_status.bits.addrv)?
+					rdmsr64(IA32_MCi_ADDR(i)) : 0ULL;	
+	} 
+}
+
+void
+mca_check_save(void)
+{
+	if (mca_exception_taken)
+		mca_save_state();
+}
+
 static void mca_dump_64bit_state(void)
 {
 	kdb_printf("Extended Machine Check State:\n");
@@ -226,11 +308,18 @@ mca_dump(void)
 {
 	ia32_mcg_status_t	status;
 
+	mca_exception_taken = TRUE;
+	mca_save_state();
+
+	/* Serialize in case of multiple simultaneous machine-checks */
+	simple_lock(&mca_lock);
+
 	/*
 	 * Report machine-check capabilities:
 	 */
 	kdb_printf(
-		"Machine-check capabilities 0x%016qx:\n", ia32_mcg_cap.u64);
+		"Machine-check capabilities (cpu %d) 0x%016qx:\n",
+		cpu_number(), ia32_mcg_cap.u64);
 	kdb_printf(
 		" %d error-reporting banks\n%s%s", mca_error_bank_count,
 		IF(mca_control_MSR_present,
@@ -265,4 +354,6 @@ mca_dump(void)
 		else
 			mca_dump_32bit_state();
 	}
+
+	simple_unlock(&mca_lock);
 }
diff --git a/osfmk/i386/machine_check.h b/osfmk/i386/machine_check.h
index 3ec287bbe..853fe36f4 100644
--- a/osfmk/i386/machine_check.h
+++ b/osfmk/i386/machine_check.h
@@ -189,8 +189,10 @@ typedef uint64_t	ia32_mci_misc_t;
 #define IA32_MCG_R14		(0x196)
 #define IA32_MCG_R15		(0x197)
 
+extern void	mca_cpu_alloc(cpu_data_t *cdp);
 extern void	mca_cpu_init(void);
 extern void	mca_dump(void);
+extern void	mca_check_save(void);
 
 #endif	/* _I386_MACHINE_CHECK_H_ */
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/i386/machine_cpu.h b/osfmk/i386/machine_cpu.h
index 21ee6880b..b3143b07e 100644
--- a/osfmk/i386/machine_cpu.h
+++ b/osfmk/i386/machine_cpu.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _I386_MACHINE_CPU_H_
 #define _I386_MACHINE_CPU_H_
diff --git a/osfmk/i386/machine_routines.c b/osfmk/i386/machine_routines.c
index 99bb21870..4ffb4ddc3 100644
--- a/osfmk/i386/machine_routines.c
+++ b/osfmk/i386/machine_routines.c
@@ -1,28 +1,36 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 #include <i386/machine_routines.h>
 #include <i386/io_map_entries.h>
 #include <i386/cpuid.h>
 #include <i386/fpu.h>
+#include <mach/processor.h>
 #include <kern/processor.h>
 #include <kern/machine.h>
 #include <kern/cpu_data.h>
@@ -32,11 +40,12 @@
 #include <i386/machine_cpu.h>
 #include <i386/mp.h>
 #include <i386/mp_events.h>
-#include <i386/cpu_threads.h>
 #include <i386/pmap.h>
 #include <i386/misc_protos.h>
 #include <i386/pmCPU.h>
 #include <i386/proc_reg.h>
+#include <i386/tsc.h>
+#include <i386/cpu_threads.h>
 #include <mach/vm_param.h>
 #if MACH_KDB
 #include <i386/db_machdep.h>
@@ -49,23 +58,21 @@
 #include <ddb/db_expr.h>
 #endif
 
-#define MIN(a,b) ((a)<(b)? (a) : (b))
-
 #if DEBUG
 #define DBG(x...)	kprintf("DBG: " x)
 #else
 #define DBG(x...)
 #endif
 
-extern void	initialize_screen(Boot_Video *, unsigned int);
 extern thread_t	Shutdown_context(thread_t thread, void (*doshutdown)(processor_t),processor_t  processor);
 extern void 	wakeup(void *);
 extern unsigned KernelRelocOffset;
 
 static int max_cpus_initialized = 0;
 
-unsigned int	LockTimeOut = 12500000;
-unsigned int	MutexSpin = 0;
+unsigned int	LockTimeOut;
+unsigned int	LockTimeOutTSC;
+unsigned int	MutexSpin;
 
 #define MAX_CPUS_SET    0x1
 #define MAX_CPUS_WAIT   0x2
@@ -131,6 +138,12 @@ ml_static_mfree(
 	     vaddr_cur += PAGE_SIZE) {
 		ppn = pmap_find_phys(kernel_pmap, (addr64_t)vaddr_cur);
 		if (ppn != (vm_offset_t)NULL) {
+		        kernel_pmap->stats.resident_count++;
+			if (kernel_pmap->stats.resident_count >
+			    kernel_pmap->stats.resident_max) {
+				kernel_pmap->stats.resident_max =
+					kernel_pmap->stats.resident_count;
+			}
 			pmap_remove(kernel_pmap, (addr64_t)vaddr_cur, (addr64_t)(vaddr_cur+PAGE_SIZE));
 			vm_page_create(ppn,(ppn+1));
 			vm_page_wire_count--;
@@ -146,6 +159,46 @@ vm_offset_t ml_vtophys(
 	return	kvtophys(vaddr);
 }
 
+/*
+ *	Routine:        ml_nofault_copy
+ *	Function:	Perform a physical mode copy if the source and
+ *			destination have valid translations in the kernel pmap.
+ *			If translations are present, they are assumed to
+ *			be wired; i.e. no attempt is made to guarantee that the
+ *			translations obtained remained valid for
+ *			the duration of the copy process.
+ */
+
+vm_size_t ml_nofault_copy(
+	vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size)
+{
+	addr64_t cur_phys_dst, cur_phys_src;
+	uint32_t count, nbytes = 0;
+
+	while (size > 0) {
+		if (!(cur_phys_src = kvtophys(virtsrc)))
+			break;
+		if (!(cur_phys_dst = kvtophys(virtdst)))
+			break;
+		if (!pmap_valid_page(i386_btop(cur_phys_dst)) || !pmap_valid_page(i386_btop(cur_phys_src)))
+			break;
+		count = PAGE_SIZE - (cur_phys_src & PAGE_MASK);
+		if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK)))
+			count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK);
+		if (count > size)
+			count = size;
+
+		bcopy_phys(cur_phys_src, cur_phys_dst, count);
+
+		nbytes += count;
+		virtsrc += count;
+		virtdst += count;
+		size -= count;
+	}
+
+	return nbytes;
+}
+
 /* Interrupt handling */
 
 /* Initialize Interrupts */
@@ -203,12 +256,9 @@ void ml_cause_interrupt(void)
 
 void ml_thread_policy(
 	thread_t thread,
-	unsigned policy_id,
+__unused	unsigned policy_id,
 	unsigned policy_info)
 {
-	if (policy_id == MACHINE_GROUP)
-		thread_bind(thread, master_processor);
-
 	if (policy_info & MACHINE_NETWORK_WORKLOOP) {
 		spl_t		s = splsched();
 
@@ -238,14 +288,15 @@ void ml_install_interrupt_handler(
 
 	(void) ml_set_interrupts_enabled(current_state);
 
-	initialize_screen(0, kPEAcquireScreen);
+	initialize_screen(NULL, kPEAcquireScreen);
 }
 
 
 void
 machine_idle(void)
 {
-	cpu_core_t	*my_core = cpu_core();
+	x86_core_t	*my_core = x86_core();
+	cpu_data_t	*my_cpu  = current_cpu_datap();
 	int		others_active;
 
 	/*
@@ -255,18 +306,23 @@ machine_idle(void)
 	 * core doesn't go into a low-power mode.
 	 * For 4/4, we set a null "active cr3" while idle.
 	 */
+	if (my_core == NULL || my_cpu == NULL)
+	    goto out;
+
 	others_active = !atomic_decl_and_test(
-				(long *) &my_core->active_threads, 1);
+				(long *) &my_core->active_lcpus, 1);
+	my_cpu->lcpu.idle = TRUE;
 	if (idlehalt || others_active) {
 		DBGLOG(cpu_handle, cpu_number(), MP_IDLE);
 		MARK_CPU_IDLE(cpu_number());
-		machine_idle_cstate();
+		machine_idle_cstate(FALSE);
 		MARK_CPU_ACTIVE(cpu_number());
 		DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE);
-	} else {
-		__asm__ volatile("sti");
 	}
-	atomic_incl((long *) &my_core->active_threads, 1);
+	my_cpu->lcpu.idle = FALSE;
+	atomic_incl((long *) &my_core->active_lcpus, 1);
+  out:
+	__asm__ volatile("sti");
 }
 
 void
@@ -278,11 +334,12 @@ machine_signal_idle(
 
 thread_t        
 machine_processor_shutdown(
-	thread_t               thread,
-	void                   (*doshutdown)(processor_t),
-	processor_t    processor)
+	thread_t	thread,
+	void		(*doshutdown)(processor_t),
+	processor_t	processor)
 {
-        fpu_save_context(thread);
+	vmx_suspend();
+	fpu_save_context(thread);
 	return(Shutdown_context(thread, doshutdown, processor));
 }
 
@@ -319,7 +376,11 @@ ml_processor_register(
 		goto failed;
 
 	if (!boot_cpu) {
-		this_cpu_datap->cpu_core = cpu_thread_alloc(target_cpu);
+		this_cpu_datap->lcpu.core = cpu_thread_alloc(this_cpu_datap->cpu_number);
+		if (this_cpu_datap->lcpu.core == NULL)
+			goto failed;
+
+		pmCPUStateInit();
 
 		this_cpu_datap->cpu_pmap = pmap_cpu_alloc(boot_cpu);
 		if (this_cpu_datap->cpu_pmap == NULL)
@@ -328,12 +389,25 @@ ml_processor_register(
 		this_cpu_datap->cpu_processor = cpu_processor_alloc(boot_cpu);
 		if (this_cpu_datap->cpu_processor == NULL)
 			goto failed;
-		processor_init(this_cpu_datap->cpu_processor, target_cpu);
+		/*
+		 * processor_init() deferred to topology start
+		 * because "slot numbers" a.k.a. logical processor numbers
+	 	 * are not yet finalized.
+		 */
 	}
 
 	*processor_out = this_cpu_datap->cpu_processor;
 	*ipi_handler = NULL;
 
+	if (target_cpu == machine_info.max_cpus - 1) {
+		/*
+		 * All processors are now registered but not started (except
+		 * for this "in-limbo" boot processor). We call to the machine
+		 * topology code to finalize and activate the topology.
+		 */
+		cpu_topology_start();
+	}
+
 	return KERN_SUCCESS;
 
 failed:
@@ -358,7 +432,11 @@ ml_cpu_get_info(ml_cpu_info_t *cpu_infop)
 	 * As distinct from whether the cpu has these capabilities.
 	 */
 	os_supports_sse = get_cr4() & CR4_XMM;
-	if ((cpuid_features() & CPUID_FEATURE_MNI) && os_supports_sse)
+	if ((cpuid_features() & CPUID_FEATURE_SSE4_2) && os_supports_sse)
+		cpu_infop->vector_unit = 8;
+	else if ((cpuid_features() & CPUID_FEATURE_SSE4_1) && os_supports_sse)
+		cpu_infop->vector_unit = 7;
+	else if ((cpuid_features() & CPUID_FEATURE_SSSE3) && os_supports_sse)
 		cpu_infop->vector_unit = 6;
 	else if ((cpuid_features() & CPUID_FEATURE_SSE3) && os_supports_sse)
 		cpu_infop->vector_unit = 5;
@@ -404,7 +482,7 @@ ml_init_max_cpus(unsigned long max_cpus)
         if (max_cpus_initialized != MAX_CPUS_SET) {
                 if (max_cpus > 0 && max_cpus <= MAX_CPUS) {
 			/*
-			 * Note: max_cpus is the number of enable processors
+			 * Note: max_cpus is the number of enabled processors
 			 * that ACPI found; max_ncpus is the maximum number
 			 * that the kernel supports or that the "cpus="
 			 * boot-arg has set. Here we take int minimum.
@@ -443,13 +521,10 @@ ml_init_lock_timeout(void)
 	uint64_t	abstime;
 	uint32_t	mtxspin; 
 
-	/*
-	 * XXX As currently implemented for x86, LockTimeOut should be a
-	 * cycle (tsc) count not an absolute time (nanoseconds) -
-	 * but it's of the right order.
-	 */
+	/* LockTimeOut is absolutetime, LockTimeOutTSC is in TSC ticks */
 	nanoseconds_to_absolutetime(NSEC_PER_SEC>>2, &abstime);
-	LockTimeOut = (unsigned int)abstime;
+	LockTimeOut = (uint32_t) abstime;
+	LockTimeOutTSC = (uint32_t) tmrCvt(abstime, tscFCvtn2t);
 
 	if (PE_parse_boot_arg("mtxspin", &mtxspin)) {
 		if (mtxspin > USEC_PER_SEC>>4)
@@ -481,29 +556,6 @@ ml_cpu_down(void)
 	return;
 }
 
-/* Stubs for pc tracing mechanism */
-
-int *pc_trace_buf;
-int pc_trace_cnt = 0;
-
-int
-set_be_bit(void)
-{
-  return(0);
-}
-
-int
-clr_be_bit(void)
-{
-  return(0);
-}
-
-int
-be_tracing(void)
-{
-  return(0);
-}
-
 /*
  * The following are required for parts of the kernel
  * that cannot resolve these functions as inlines:
@@ -523,52 +575,6 @@ current_thread(void)
   return(current_thread_fast());
 }
 
-/*
- * Set the worst-case time for the C4 to C2 transition.
- * The maxdelay parameter is in nanoseconds. 
- */
- 
-void
-ml_set_maxsnoop(uint32_t maxdelay)
-{
-	C4C2SnoopDelay = maxdelay;	/* Set the transition time */ 
-   	machine_nap_policy();		/* Adjust the current nap state */
-}
-
-
-/*
- * Get the worst-case time for the C4 to C2 transition.  Returns nanoseconds.
- */
-
-unsigned
-ml_get_maxsnoop(void)
-{
-	return C4C2SnoopDelay;		/* Set the transition time */
-}
-
-
-uint32_t
-ml_get_maxbusdelay(void)
-{
-    return maxBusDelay;
-}
-
-/*
- * Set the maximum delay time allowed for snoop on the bus.
- *
- * Note that this value will be compared to the amount of time that it takes
- * to transition from a non-snooping power state (C4) to a snooping state (C2).
- * If maxBusDelay is less than C4C2SnoopDelay,
- * we will not enter the lowest power state.
- */
- 
-void
-ml_set_maxbusdelay(uint32_t mdelay)
-{
-	maxBusDelay = mdelay;		/* Set the delay */
-	machine_nap_policy();		/* Adjust the current nap state */
-}
-
 
 boolean_t ml_is64bit(void) {
 
@@ -612,6 +618,12 @@ void ml_fp_setvalid(boolean_t value)
         fp_setvalid(value);
 }
 
+uint64_t ml_cpu_int_event_time(void)
+{
+	return current_cpu_datap()->cpu_int_event_time;
+}
+
+
 #if MACH_KDB
 
 /*
diff --git a/osfmk/i386/machine_routines.h b/osfmk/i386/machine_routines.h
index d9060ecf0..d746ed642 100644
--- a/osfmk/i386/machine_routines.h
+++ b/osfmk/i386/machine_routines.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -192,6 +198,9 @@ void ml_static_mfree(
 vm_offset_t ml_vtophys(
 	vm_offset_t vaddr);
 
+vm_size_t ml_nofault_copy(
+	vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size);
+
 /* Struct for ml_cpu_get_info */
 struct ml_cpu_info {
 	unsigned long		vector_unit;
@@ -209,6 +218,10 @@ typedef struct ml_cpu_info ml_cpu_info_t;
 /* Get processor info */
 void ml_cpu_get_info(ml_cpu_info_t *ml_cpu_info);
 
+/* Machine topology info */
+uint64_t ml_cpu_cache_size(unsigned int level);	
+uint64_t ml_cpu_cache_sharing(unsigned int level);	
+
 #endif /* __APPLE_API_UNSTABLE */
 
 #ifdef __APPLE_API_PRIVATE
@@ -240,6 +253,10 @@ void bzero_phys(
 	addr64_t phys_address,
 	uint32_t length);
 
+void bzero_phys_nc(
+	addr64_t phys_address,
+	uint32_t length);
+
 void ml_thread_policy(
 	thread_t thread,
 	unsigned policy_id,
@@ -261,18 +278,21 @@ int ml_get_max_cpus(
 extern void	ml_cpu_up(void);
 extern void	ml_cpu_down(void);
 
-extern int	set_be_bit(void);
-extern int	clr_be_bit(void);
-extern int	be_tracing(void);
-
+/*
+ * The following are in pmCPU.c not machine_routines.c.
+ */
 extern void ml_set_maxsnoop(uint32_t maxdelay);
 extern unsigned ml_get_maxsnoop(void);
 extern void ml_set_maxbusdelay(uint32_t mdelay);
 extern uint32_t ml_get_maxbusdelay(void);
+
+
 extern void ml_hpet_cfg(uint32_t cpu, uint32_t hpetVect);
 
 extern uint64_t tmrCvt(uint64_t time, uint64_t conversion);
 
+extern uint64_t ml_cpu_int_event_time(void);
+
 #endif /* __APPLE_API_PRIVATE */
 
 __END_DECLS
diff --git a/osfmk/i386/machine_routines_asm.s b/osfmk/i386/machine_routines_asm.s
index b017f473a..1e41a89d6 100644
--- a/osfmk/i386/machine_routines_asm.s
+++ b/osfmk/i386/machine_routines_asm.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include <i386/asm.h>
@@ -28,17 +34,6 @@
 #include <i386/apic.h>
 #include <assym.s>
 
-#define	PA(addr)	(addr)
-#define	VA(addr)	(addr)
-	
-/*
- * GAS won't handle an intersegment jump with a relocatable offset.
- */
-#define	LJMP(segment,address)	\
-	.byte	0xea		;\
-	.long	address		;\
-	.word	segment
-
 /*
 **      ml_get_timebase()
 **
@@ -58,7 +53,6 @@ ENTRY(ml_get_timebase)
 			
 			ret
 
-
 /*
  *  	Convert between various timer units 
  *
@@ -136,16 +130,20 @@ LEXT(tmrCvt)
 			popl	%ebx					// Restore a volatile
 			popl	%ebp					// Restore a volatile
 
-			ret								// Leave...
+			ret						// Leave...
 
-			.globl	EXT(rtc_nanotime_store)
+			.globl	EXT(_rtc_nanotime_store)
 			.align	FALIGN
 
-LEXT(rtc_nanotime_store)
-		push	%ebp
-		mov		%esp,%ebp
+LEXT(_rtc_nanotime_store)
+		push		%ebp
+		movl		%esp,%ebp
+		push		%esi
 
-		mov		32(%ebp),%edx
+		mov		32(%ebp),%edx				/* get ptr to rtc_nanotime_info */
+		
+		movl		RNT_GENERATION(%edx),%esi		/* get current generation */
+		movl		$0,RNT_GENERATION(%edx)			/* flag data as being updated */
 
 		mov		8(%ebp),%eax
 		mov		%eax,RNT_TSC_BASE(%edx)
@@ -162,35 +160,142 @@ LEXT(rtc_nanotime_store)
 		mov		%eax,RNT_NS_BASE(%edx)
 		mov		20(%ebp),%eax
 		mov		%eax,RNT_NS_BASE+4(%edx)
+		
+		incl		%esi					/* next generation */
+		jnz		1f
+		incl		%esi					/* skip 0, which is a flag */
+1:		movl		%esi,RNT_GENERATION(%edx)		/* update generation and make usable */
 
+		pop		%esi
 		pop		%ebp
 		ret
 
-			.globl	EXT(rtc_nanotime_load)
-			.align	FALIGN
 
-LEXT(rtc_nanotime_load)
-		push	%ebp
-		mov		%esp,%ebp
+/* unint64_t _rtc_nanotime_read( rtc_nanotime_t *rntp, int slow );
+ *
+ * This is the same as the commpage nanotime routine, except that it uses the
+ * kernel internal "rtc_nanotime_info" data instead of the commpage data.  The two copies
+ * of data (one in the kernel and one in user space) are kept in sync by rtc_nanotime_update().
+ *
+ * There are actually two versions of the algorithm, one each for "slow" and "fast"
+ * processors.  The more common "fast" algorithm is:
+ *
+ *	nanoseconds = (((rdtsc - rnt_tsc_base) * rnt_tsc_scale) / 2**32) - rnt_ns_base;
+ *
+ * Of course, the divide by 2**32 is a nop.  rnt_tsc_scale is a constant computed during initialization:
+ *
+ *	rnt_tsc_scale = (10e9 * 2**32) / tscFreq;
+ *
+ * The "slow" algorithm uses long division:
+ *
+ *	nanoseconds = (((rdtsc - rnt_tsc_base) * 10e9) / tscFreq) - rnt_ns_base;
+ *
+ * Since this routine is not synchronized and can be called in any context, 
+ * we use a generation count to guard against seeing partially updated data.  In addition,
+ * the _rtc_nanotime_store() routine -- just above -- zeroes the generation before
+ * updating the data, and stores the nonzero generation only after all other data has been
+ * stored.  Because IA32 guarantees that stores by one processor must be seen in order
+ * by another, we can avoid using a lock.  We spin while the generation is zero.
+ *
+ * In accordance with the ABI, we return the 64-bit nanotime in %edx:%eax.
+ */
+ 
+		.globl	EXT(_rtc_nanotime_read)
+		.align	FALIGN
+LEXT(_rtc_nanotime_read)
+		pushl		%ebp
+		movl		%esp,%ebp
+		pushl		%esi
+		pushl		%edi
+		pushl		%ebx
+		movl		8(%ebp),%edi				/* get ptr to rtc_nanotime_info */
+		movl		12(%ebp),%eax				/* get "slow" flag */
+		testl		%eax,%eax
+		jnz		Lslow
+		
+		/* Processor whose TSC frequency is faster than SLOW_TSC_THRESHOLD */
+0:
+		movl		RNT_GENERATION(%edi),%esi		/* get generation (0 if being changed) */
+		testl		%esi,%esi				/* if being changed, loop until stable */
+		jz		0b
+
+		rdtsc							/* get TSC in %edx:%eax */
+		subl		RNT_TSC_BASE(%edi),%eax
+		sbbl		RNT_TSC_BASE+4(%edi),%edx
 
-		mov		8(%ebp),%ecx
-		mov		12(%ebp),%edx
+		movl		RNT_SCALE(%edi),%ecx
 
-		mov		RNT_TSC_BASE(%ecx),%eax
-		mov		%eax,RNT_TSC_BASE(%edx)
-		mov		RNT_TSC_BASE+4(%ecx),%eax
-		mov		%eax,RNT_TSC_BASE+4(%edx)
+		movl		%edx,%ebx
+		mull		%ecx
+		movl		%ebx,%eax
+		movl		%edx,%ebx
+		mull		%ecx
+		addl		%ebx,%eax
+		adcl		$0,%edx
 
-		mov		RNT_SCALE(%ecx),%eax
-		mov		%eax,RNT_SCALE(%edx)
+		addl		RNT_NS_BASE(%edi),%eax
+		adcl		RNT_NS_BASE+4(%edi),%edx
 
-		mov		RNT_SHIFT(%ecx),%eax
-		mov		%eax,RNT_SHIFT(%edx)
+		cmpl		RNT_GENERATION(%edi),%esi		/* have the parameters changed? */
+		jne		0b					/* yes, loop until stable */
 
-		mov		RNT_NS_BASE(%ecx),%eax
-		mov		%eax,RNT_NS_BASE(%edx)
-		mov		RNT_NS_BASE+4(%ecx),%eax
-		mov		%eax,RNT_NS_BASE+4(%edx)
+		popl		%ebx
+		popl		%edi
+		popl		%esi
+		popl		%ebp
+		ret
+
+		/* Processor whose TSC frequency is slower than or equal to SLOW_TSC_THRESHOLD */
+Lslow:
+		movl		RNT_GENERATION(%edi),%esi		/* get generation (0 if being changed) */
+		testl		%esi,%esi				/* if being changed, loop until stable */
+		jz		Lslow
+		pushl		%esi					/* save generation */
+		pushl		RNT_SHIFT(%edi)				/* save low 32 bits of tscFreq */
+
+		rdtsc							/* get TSC in %edx:%eax */
+		subl		RNT_TSC_BASE(%edi),%eax
+		sbbl		RNT_TSC_BASE+4(%edi),%edx
+
+		/*
+		* Do the math to convert tsc ticks to nanoseconds.  We first
+		* do long multiply of 1 billion times the tsc.  Then we do
+		* long division by the tsc frequency
+		*/
+		mov		$1000000000, %ecx			/* number of nanoseconds in a second */
+		mov		%edx, %ebx
+		mul		%ecx
+		mov		%edx, %edi
+		mov		%eax, %esi
+		mov		%ebx, %eax
+		mul		%ecx
+		add		%edi, %eax
+		adc		$0, %edx				/* result in edx:eax:esi */
+		mov		%eax, %edi
+		popl		%ecx					/* get low 32 tscFreq */
+		xor		%eax, %eax
+		xchg		%edx, %eax
+		div		%ecx
+		xor		%eax, %eax
+		mov		%edi, %eax
+		div		%ecx
+		mov		%eax, %ebx
+		mov		%esi, %eax
+		div		%ecx
+		mov		%ebx, %edx				/* result in edx:eax */
+		
+		movl		8(%ebp),%edi				/* recover ptr to rtc_nanotime_info */
+		popl		%esi					/* recover generation */
 
+		addl		RNT_NS_BASE(%edi),%eax
+		adcl		RNT_NS_BASE+4(%edi),%edx
+
+		cmpl		RNT_GENERATION(%edi),%esi		/* have the parameters changed? */
+		jne		Lslow					/* yes, loop until stable */
+
+		pop		%ebx
+		pop		%edi
+		pop		%esi
 		pop		%ebp
-		ret
+		ret							/* result in edx:eax */
+
diff --git a/osfmk/i386/machine_rpc.h b/osfmk/i386/machine_rpc.h
index b444de089..5e29e9504 100644
--- a/osfmk/i386/machine_rpc.h
+++ b/osfmk/i386/machine_rpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/machlimits.h b/osfmk/i386/machlimits.h
index 5aefbace2..b3e775f56 100644
--- a/osfmk/i386/machlimits.h
+++ b/osfmk/i386/machlimits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/machparam.h b/osfmk/i386/machparam.h
index 908137f32..1096d6e5a 100644
--- a/osfmk/i386/machparam.h
+++ b/osfmk/i386/machparam.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/mcount.s b/osfmk/i386/mcount.s
index 12f1ddb68..822d643e0 100644
--- a/osfmk/i386/mcount.s
+++ b/osfmk/i386/mcount.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #define __NO_UNDERSCORES__
diff --git a/osfmk/i386/misc_protos.h b/osfmk/i386/misc_protos.h
index 26b7edd84..8d9036a15 100644
--- a/osfmk/i386/misc_protos.h
+++ b/osfmk/i386/misc_protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -34,15 +40,16 @@ struct cpu_data;
 extern void		i386_init(vm_offset_t);
 extern void		i386_macho_zerofill(void);
 extern void		i386_vm_init(
-				uint64_t		maxmem,
-				boolean_t 		IA32e,
-				struct boot_args	*args);
+				uint64_t,
+				boolean_t,
+				struct boot_args *);
 extern void		cpu_IA32e_enable(struct cpu_data *);
 extern void		cpu_IA32e_disable(struct cpu_data *);
 extern void		ml_load_desc64(void);
 extern void		ml_64bit_wrmsr64(uint32_t msr, uint64_t value);
-extern void             cpu_window_init(int);
 extern void		ml_64bit_lldt(int);
+extern void             cpu_userwindow_init(int);
+extern void             cpu_physwindow_init(int);
 
 extern void		machine_startup(void);
 
@@ -52,6 +59,7 @@ extern void		interrupt_processor(
 				int		cpu);
 extern void		mp_probe_cpus(void);
 extern void		panic_io_port_read(void);
+
 extern void		remote_kdb(void);
 extern void		clear_kdb_intr(void);
 extern void             draw_panic_dialog(void);
@@ -118,10 +126,10 @@ extern void	rtc_clock_stepping(
 extern void	rtc_clock_stepped(
 			uint32_t new_frequency,
 			uint32_t old_frequency);
-extern void	rtc_clock_napped(
-			uint64_t);
+extern void	rtc_clock_napped(uint64_t);
 
 extern void     x86_lowmem_free(void);
 
+thread_t Switch_context(thread_t, thread_continue_t, thread_t);
 
-#endif	/* _I386_MISC_PROTOS_H_ */
+#endif /* _I386_MISC_PROTOS_H_ */
diff --git a/osfmk/i386/mp.c b/osfmk/i386/mp.c
index 55f7576bf..fb7afc4d0 100644
--- a/osfmk/i386/mp.c
+++ b/osfmk/i386/mp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,6 +63,7 @@
 #include <i386/machine_cpu.h>
 #include <i386/misc_protos.h>
 #include <i386/mtrr.h>
+#include <i386/vmx/vmx_cpu.h>
 #include <i386/postcode.h>
 #include <i386/perfmon.h>
 #include <i386/cpu_threads.h>
@@ -65,6 +72,7 @@
 #include <i386/machine_routines.h>
 #include <i386/pmCPU.h>
 #include <i386/hpet.h>
+#include <i386/machine_check.h>
 
 #include <chud/chud_xnu.h>
 #include <chud/chud_xnu_private.h>
@@ -112,14 +120,16 @@ volatile boolean_t	mp_kdb_trap = FALSE;
 volatile long	mp_kdb_ncpus = 0;
 #endif
 
-static void	mp_kdp_wait(void);
+static void	mp_kdp_wait(boolean_t flush);
 static void	mp_rendezvous_action(void);
+static void 	mp_broadcast_action(void);
 
-static int	NMIInterruptHandler(void *regs);
+static int		NMIInterruptHandler(x86_saved_state_t *regs);
 static boolean_t	cpu_signal_pending(int cpu, mp_event_t event);
-static void	cpu_NMI_interrupt(int cpu);
+static void		cpu_NMI_interrupt(int cpu);
 
 boolean_t 	smp_initialized = FALSE;
+boolean_t	force_immediate_debugger_NMI = FALSE;
 
 decl_simple_lock_data(,mp_kdp_lock);
 
@@ -137,6 +147,15 @@ static volatile long	mp_rv_entry    __attribute__((aligned(64)));
 static volatile long	mp_rv_exit     __attribute__((aligned(64)));
 static volatile long	mp_rv_complete __attribute__((aligned(64)));
 
+/* Variables needed for MP broadcast. */
+static void        (*mp_bc_action_func)(void *arg);
+static void        *mp_bc_func_arg;
+static int     mp_bc_ncpus;
+static volatile long   mp_bc_count;
+decl_mutex_data(static, mp_bc_lock);
+
+static void	mp_cpus_call_action(void); 
+
 int		lapic_to_cpu[MAX_CPUS];
 int		cpu_to_lapic[MAX_CPUS];
 
@@ -235,6 +254,7 @@ smp_init(void)
 	simple_lock_init(&mp_kdp_lock, 0);
 	simple_lock_init(&mp_rv_lock, 0);
 	mutex_init(&mp_cpu_boot_lock, 0);
+	mutex_init(&mp_bc_lock, 0);
 	console_init();
 
 	/* Local APIC? */
@@ -611,7 +631,7 @@ lapic_interrupt(int interrupt, x86_saved_state_t *state)
 	int	retval = 0;
 
 	/* Did we just field an interruption for the HPET comparator? */
-	if(current_cpu_datap()->cpu_pmHpetVec == ((uint32_t)interrupt - 0x40)) {
+	if(x86_core()->HpetVec == ((uint32_t)interrupt - 0x40)) {
 		/* Yes, go handle it... */
 		retval = HPETInterrupt();
 		/* Was it really handled? */
@@ -778,8 +798,7 @@ intel_startCPU(
 		cpu_shutdown();
 		return KERN_SUCCESS;
 	} else {
-		kprintf("Started cpu %d (lapic id %p)\n", slot_num, lapic);
-		printf("Started CPU %02d\n", slot_num);
+		kprintf("Started cpu %d (lapic id %08x)\n", slot_num, lapic);
 		return KERN_SUCCESS;
 	}
 }
@@ -865,7 +884,7 @@ cpu_signal_handler(x86_saved_state_t *regs)
  * XXX 64-bit state?
  */
 			sync_iss_to_iks(saved_state32(regs));
-			mp_kdp_wait();
+			mp_kdp_wait(TRUE);
 		} else
 #endif	/* MACH_KDP */
 		if (i_bit(MP_TLB_FLUSH, my_word)) {
@@ -888,10 +907,18 @@ cpu_signal_handler(x86_saved_state_t *regs)
 			DBGLOG(cpu_handle,my_cpu,MP_RENDEZVOUS);
 			i_bit_clear(MP_RENDEZVOUS, my_word);
 			mp_rendezvous_action();
+		} else if (i_bit(MP_BROADCAST, my_word)) {
+			DBGLOG(cpu_handle,my_cpu,MP_BROADCAST);
+			i_bit_clear(MP_BROADCAST, my_word);
+			mp_broadcast_action();
 		} else if (i_bit(MP_CHUD, my_word)) {
 			DBGLOG(cpu_handle,my_cpu,MP_CHUD);
 			i_bit_clear(MP_CHUD, my_word);
 			chudxnu_cpu_signal_handler();
+		} else if (i_bit(MP_CALL, my_word)) {
+			DBGLOG(cpu_handle,my_cpu,MP_CALL);
+			i_bit_clear(MP_CALL, my_word);
+			mp_cpus_call_action();
 		}
 	} while (*my_word);
 
@@ -899,66 +926,78 @@ cpu_signal_handler(x86_saved_state_t *regs)
 
 }
 
-
-/* We want this to show up in backtraces, so mark it noinline
+/* We want this to show up in backtraces, hence marked noinline.
  */
 static int __attribute__((noinline))
-NMIInterruptHandler(void *regs)
+NMIInterruptHandler(x86_saved_state_t *regs)
 {
 	boolean_t state = ml_set_interrupts_enabled(FALSE);
 	sync_iss_to_iks_unconditionally(regs);
-	mp_kdp_wait();
+	mp_kdp_wait(FALSE);
 	(void) ml_set_interrupts_enabled(state);
 	return 1;
 }
 
 #ifdef	MP_DEBUG
 extern int	max_lock_loops;
+int		trappedalready = 0;	/* (BRINGUP */
 #endif	/* MP_DEBUG */
 
-int trappedalready = 0;	/* (BRINGUP */
-
-void
-cpu_interrupt(int cpu)
+static void
+i386_cpu_IPI(int cpu)
 {
 	boolean_t	state;
 	
+#ifdef	MP_DEBUG
 	if(cpu_datap(cpu)->cpu_signals & 6) {	/* (BRINGUP) */
-		kprintf("cpu_interrupt: sending enter debugger signal (%08X) to cpu %d\n", cpu_datap(cpu)->cpu_signals, cpu);
+		kprintf("i386_cpu_IPI: sending enter debugger signal (%08X) to cpu %d\n", cpu_datap(cpu)->cpu_signals, cpu);
 	}
-
-	if (smp_initialized) {
+#endif	/* MP_DEBUG */
 
 #if MACH_KDB
-//		if(!trappedalready && (cpu_datap(cpu)->cpu_signals & 6)) {	/* (BRINGUP) */
-//			if(kdb_cpu != cpu_number()) {
-//				trappedalready = 1;
-//				panic("cpu_interrupt: sending enter debugger signal (%08X) to cpu %d and I do not own debugger, owner = %08X\n", 
-//					cpu_datap(cpu)->cpu_signals, cpu, kdb_cpu);
-//			}
-//		}
+#ifdef	MP_DEBUG
+	if(!trappedalready && (cpu_datap(cpu)->cpu_signals & 6)) {	/* (BRINGUP) */
+		if(kdb_cpu != cpu_number()) {
+			trappedalready = 1;
+			panic("i386_cpu_IPI: sending enter debugger signal (%08X) to cpu %d and I do not own debugger, owner = %08X\n", 
+				cpu_datap(cpu)->cpu_signals, cpu, kdb_cpu);
+		}
+	}
+#endif	/* MP_DEBUG */
 #endif
 
-		/* Wait for previous interrupt to be delivered... */
+	/* Wait for previous interrupt to be delivered... */
 #ifdef	MP_DEBUG
-		int     pending_busy_count = 0;
-		while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) {
-			if (++pending_busy_count > max_lock_loops)
-				panic("cpus_interrupt() deadlock\n");
+	int     pending_busy_count = 0;
+	while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) {
+		if (++pending_busy_count > max_lock_loops)
+			panic("i386_cpu_IPI() deadlock\n");
 #else
-		while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) {
+	while (LAPIC_REG(ICR) & LAPIC_ICR_DS_PENDING) {
 #endif	/* MP_DEBUG */
-			cpu_pause();
-		}
-
-		state = ml_set_interrupts_enabled(FALSE);
-		LAPIC_REG(ICRD) =
-			cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT;
-		LAPIC_REG(ICR)  =
-			LAPIC_VECTOR(INTERPROCESSOR) | LAPIC_ICR_DM_FIXED;
-		(void) ml_set_interrupts_enabled(state);
+		cpu_pause();
 	}
 
+	state = ml_set_interrupts_enabled(FALSE);
+	LAPIC_REG(ICRD) =
+		cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT;
+	LAPIC_REG(ICR)  =
+		LAPIC_VECTOR(INTERPROCESSOR) | LAPIC_ICR_DM_FIXED;
+	(void) ml_set_interrupts_enabled(state);
+}
+
+/*
+ * cpu_interrupt is really just to be used by the scheduler to
+ * get a CPU's attention it may not always issue an IPI.  If an
+ * IPI is always needed then use i386_cpu_IPI.
+ */
+void
+cpu_interrupt(int cpu)
+{
+	if (smp_initialized
+	    && pmCPUExitIdle(cpu_datap(cpu))) {
+		i386_cpu_IPI(cpu);
+	}
 }
 
 /*
@@ -971,16 +1010,16 @@ cpu_NMI_interrupt(int cpu)
 
 	if (smp_initialized) {
 		state = ml_set_interrupts_enabled(FALSE);
+/* Program the interrupt command register */
 		LAPIC_REG(ICRD) =
 			cpu_to_lapic[cpu] << LAPIC_ICRD_DEST_SHIFT;
-/* The vector is ignored in this case, the other CPU will come in on the
+/* The vector is ignored in this case--the target CPU will enter on the
  * NMI vector.
  */
 		LAPIC_REG(ICR)  =
 			LAPIC_VECTOR(INTERPROCESSOR) | LAPIC_ICR_DM_NMI;
 		(void) ml_set_interrupts_enabled(state);
 	}
-
 }
 
 void
@@ -999,7 +1038,7 @@ i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode)
 	DBGLOG(cpu_signal, cpu, event);
 	
 	i_bit_set(event, signals);
-	cpu_interrupt(cpu);
+	i386_cpu_IPI(cpu);
 	if (mode == SYNC) {
 	   again:
 		tsc_timeout = rdtsc64() + (1000*1000*1000);
@@ -1016,12 +1055,18 @@ i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode)
 	        KERNEL_DEBUG(0xef800020 | DBG_FUNC_END, cpu, 0, 0, 0, 0);
 }
 
+/*
+ * Send event to all running cpus.
+ * Called with the topology locked.
+ */
 void
 i386_signal_cpus(mp_event_t event, mp_sync_t mode)
 {
 	unsigned int	cpu;
 	unsigned int	my_cpu = cpu_number();
 
+	assert(hw_lock_held(&x86_topo_lock));
+
 	for (cpu = 0; cpu < real_ncpus; cpu++) {
 		if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
 			continue;
@@ -1029,12 +1074,18 @@ i386_signal_cpus(mp_event_t event, mp_sync_t mode)
 	}
 }
 
+/*
+ * Return the number of running cpus.
+ * Called with the topology locked.
+ */
 int
 i386_active_cpus(void)
 {
 	unsigned int	cpu;
 	unsigned int	ncpus = 0;
 
+	assert(hw_lock_held(&x86_topo_lock));
+
 	for (cpu = 0; cpu < real_ncpus; cpu++) {
 		if (cpu_datap(cpu)->cpu_running)
 			ncpus++;
@@ -1059,17 +1110,20 @@ i386_active_cpus(void)
 static void
 mp_rendezvous_action(void)
 {
+	boolean_t intrs_enabled;
 
 	/* setup function */
 	if (mp_rv_setup_func != NULL)
 		mp_rv_setup_func(mp_rv_func_arg);
+
+	intrs_enabled = ml_get_interrupts_enabled();
+
 	/* spin on entry rendezvous */
 	atomic_incl(&mp_rv_entry, 1);
 	while (mp_rv_entry < mp_rv_ncpus) {
-		boolean_t intr = ml_set_interrupts_enabled(FALSE);
-		/* poll for pesky tlb flushes */
-		handle_pending_TLB_flushes();
-		ml_set_interrupts_enabled(intr);
+		/* poll for pesky tlb flushes if interrupts disabled */
+		if (!intrs_enabled)
+			handle_pending_TLB_flushes();
 		cpu_pause();
 	}
 	/* action function */
@@ -1077,8 +1131,11 @@ mp_rendezvous_action(void)
 		mp_rv_action_func(mp_rv_func_arg);
 	/* spin on exit rendezvous */
 	atomic_incl(&mp_rv_exit, 1);
-	while (mp_rv_exit < mp_rv_ncpus)
+	while (mp_rv_exit < mp_rv_ncpus) {
+		if (!intrs_enabled)
+			handle_pending_TLB_flushes();
 		cpu_pause();
+	}
 
 	/* teardown function */
 	if (mp_rv_teardown_func != NULL)
@@ -1120,9 +1177,12 @@ mp_rendezvous(void (*setup_func)(void *),
 
 	/*
 	 * signal other processors, which will call mp_rendezvous_action()
+	 * with interrupts disabled
 	 */
+	simple_lock(&x86_topo_lock);
 	mp_rv_ncpus = i386_active_cpus();
 	i386_signal_cpus(MP_RENDEZVOUS, ASYNC);
+	simple_unlock(&x86_topo_lock);
 
 	/* call executor function on this cpu */
 	mp_rendezvous_action();
@@ -1136,6 +1196,12 @@ mp_rendezvous(void (*setup_func)(void *),
 		cpu_pause();
 	}
 	
+	/* Tidy up */
+	mp_rv_setup_func = NULL;
+	mp_rv_action_func = NULL;
+	mp_rv_teardown_func = NULL;
+	mp_rv_func_arg = NULL;
+
 	/* release lock */
 	simple_unlock(&mp_rv_lock);
 }
@@ -1184,16 +1250,236 @@ handle_pending_TLB_flushes(void)
 {
 	volatile int	*my_word = &current_cpu_datap()->cpu_signals;
 
-        if (i_bit(MP_TLB_FLUSH, my_word)) {
-	        DBGLOG(cpu_handle, cpu_number(), MP_TLB_FLUSH);
+	if (i_bit(MP_TLB_FLUSH, my_word)) {
+		DBGLOG(cpu_handle, cpu_number(), MP_TLB_FLUSH);
 		i_bit_clear(MP_TLB_FLUSH, my_word);
 		pmap_update_interrupt();
 	}
 }
 
+/*
+ * This is called from cpu_signal_handler() to process an MP_CALL signal.
+ */
+static void
+mp_cpus_call_action(void)
+{
+	if (mp_rv_action_func != NULL)
+		mp_rv_action_func(mp_rv_func_arg);
+	atomic_incl(&mp_rv_complete, 1);
+}
+
+/*
+ * mp_cpus_call() runs a given function on cpus specified in a given cpu mask.
+ * If the mode is SYNC, the function is called serially on the target cpus
+ * in logical cpu order. If the mode is ASYNC, the function is called in
+ * parallel over the specified cpus.
+ * The action function may be NULL.
+ * The cpu mask may include the local cpu. Offline cpus are ignored.
+ * Return does not occur until the function has completed on all cpus.
+ * The return value is the number of cpus on which the function was called.
+ */
+cpu_t
+mp_cpus_call(
+	cpumask_t	cpus,
+	mp_sync_t	mode,
+        void		(*action_func)(void *),
+        void		*arg)
+{
+	cpu_t		cpu;
+	boolean_t	intrs_enabled = ml_get_interrupts_enabled();
+	boolean_t	call_self = FALSE;
+
+	if (!smp_initialized) {
+		if ((cpus & CPUMASK_SELF) == 0)
+			return 0;
+		if (action_func != NULL) {
+			(void) ml_set_interrupts_enabled(FALSE);
+			action_func(arg);
+			ml_set_interrupts_enabled(intrs_enabled);
+		}
+		return 1;
+	}
+		
+	/* obtain rendezvous lock */
+	simple_lock(&mp_rv_lock);
+
+	/* Use the rendezvous data structures for this call */
+	mp_rv_action_func = action_func;
+	mp_rv_func_arg = arg;
+	mp_rv_ncpus = 0;
+	mp_rv_complete = 0;
+
+	simple_lock(&x86_topo_lock);
+	for (cpu = 0; cpu < (cpu_t) real_ncpus; cpu++) {
+		if (((cpu_to_cpumask(cpu) & cpus) == 0) ||
+		    !cpu_datap(cpu)->cpu_running)
+			continue;
+		if (cpu == (cpu_t) cpu_number()) {
+			/*
+			 * We don't IPI ourself and if calling asynchronously,
+			 * we defer our call until we have signalled all others.
+			 */
+			call_self = TRUE;
+			if (mode == SYNC && action_func != NULL) {
+				(void) ml_set_interrupts_enabled(FALSE);
+				action_func(arg);
+				ml_set_interrupts_enabled(intrs_enabled);
+			}
+		} else {
+			/*
+			 * Bump count of other cpus called and signal this cpu.
+			 * Note: we signal asynchronously regardless of mode
+			 * because we wait on mp_rv_complete either here
+			 * (if mode == SYNC) or later (if mode == ASYNC).
+			 * While spinning, poll for TLB flushes if interrupts
+			 * are disabled.
+			 */
+			mp_rv_ncpus++;
+			i386_signal_cpu(cpu, MP_CALL, ASYNC);
+			if (mode == SYNC) {
+				simple_unlock(&x86_topo_lock);
+				while (mp_rv_complete < mp_rv_ncpus) {
+					if (!intrs_enabled)
+						handle_pending_TLB_flushes();
+					cpu_pause();
+				}
+				simple_lock(&x86_topo_lock);
+			}
+		}
+	}
+	simple_unlock(&x86_topo_lock);
+
+	/*
+	 * If calls are being made asynchronously,
+	 * make the local call now if needed, and then
+	 * wait for all other cpus to finish their calls.
+	 */
+	if (mode == ASYNC) {
+		if (call_self && action_func != NULL) {
+			(void) ml_set_interrupts_enabled(FALSE);
+			action_func(arg);
+			ml_set_interrupts_enabled(intrs_enabled);
+		}
+		while (mp_rv_complete < mp_rv_ncpus) {
+			if (!intrs_enabled)
+				handle_pending_TLB_flushes();
+			cpu_pause();
+		}
+	}
+	
+	/* Determine the number of cpus called */
+	cpu = mp_rv_ncpus + (call_self ? 1 : 0);
+
+	simple_unlock(&mp_rv_lock);
+
+	return cpu;
+}
+
+static void
+mp_broadcast_action(void)
+{
+   /* call action function */
+   if (mp_bc_action_func != NULL)
+       mp_bc_action_func(mp_bc_func_arg);
+
+   /* if we're the last one through, wake up the instigator */
+   if (atomic_decl_and_test((volatile long *)&mp_bc_count, 1))
+       thread_wakeup(((event_t)(unsigned int *) &mp_bc_count));
+}
+
+/*
+ * mp_broadcast() runs a given function on all active cpus.
+ * The caller blocks until the functions has run on all cpus.
+ * The caller will also block if there is another pending braodcast.
+ */
+void
+mp_broadcast(
+         void (*action_func)(void *),
+         void *arg)
+{
+   if (!smp_initialized) {
+       if (action_func != NULL)
+	           action_func(arg);
+       return;
+   }
+       
+   /* obtain broadcast lock */
+   mutex_lock(&mp_bc_lock);
+
+   /* set static function pointers */
+   mp_bc_action_func = action_func;
+   mp_bc_func_arg = arg;
+
+   assert_wait(&mp_bc_count, THREAD_UNINT);
+
+   /*
+    * signal other processors, which will call mp_broadcast_action()
+    */
+   simple_lock(&x86_topo_lock);
+   mp_bc_ncpus = i386_active_cpus();   /* total including this cpu */
+   mp_bc_count = mp_bc_ncpus;
+   i386_signal_cpus(MP_BROADCAST, ASYNC);
+
+   /* call executor function on this cpu */
+   mp_broadcast_action();
+   simple_unlock(&x86_topo_lock);
+
+   /* block for all cpus to have run action_func */
+   if (mp_bc_ncpus > 1)
+       thread_block(THREAD_CONTINUE_NULL);
+   else
+       clear_wait(current_thread(), THREAD_AWAKENED);
+       
+   /* release lock */
+   mutex_unlock(&mp_bc_lock);
+}
+
+void
+i386_activate_cpu(void)
+{
+	cpu_data_t	*cdp = current_cpu_datap();
+
+	assert(!ml_get_interrupts_enabled());
+
+	if (!smp_initialized) {
+		cdp->cpu_running = TRUE;
+		return;
+	}
+
+	simple_lock(&x86_topo_lock);
+	cdp->cpu_running = TRUE;
+	simple_unlock(&x86_topo_lock);
+}
+
+void
+i386_deactivate_cpu(void)
+{
+	cpu_data_t	*cdp = current_cpu_datap();
+
+	assert(!ml_get_interrupts_enabled());
+
+	simple_lock(&x86_topo_lock);
+	cdp->cpu_running = FALSE;
+	simple_unlock(&x86_topo_lock);
+
+	/*
+	 * In case a rendezvous/braodcast/call was initiated to this cpu
+	 * before we cleared cpu_running, we must perform any actions due.
+	 */
+	if (i_bit(MP_RENDEZVOUS, &cdp->cpu_signals))
+		mp_rendezvous_action();
+	if (i_bit(MP_BROADCAST, &cdp->cpu_signals))
+		mp_broadcast_action();
+	if (i_bit(MP_CALL, &cdp->cpu_signals))
+		mp_cpus_call_action();
+	cdp->cpu_signals = 0;			/* all clear */
+}
+
+int	pmsafe_debug	= 1;
+
 #if	MACH_KDP
 volatile boolean_t	mp_kdp_trap = FALSE;
-volatile long		mp_kdp_ncpus;
+volatile unsigned long		mp_kdp_ncpus;
 boolean_t		mp_kdp_state;
 
 
@@ -1215,10 +1501,13 @@ mp_kdp_enter(void)
 	mp_kdp_state = ml_set_interrupts_enabled(FALSE);
 	simple_lock(&mp_kdp_lock);
 
+	if (pmsafe_debug)
+	    pmSafeMode(&current_cpu_datap()->lcpu, PM_SAFE_FL_SAFE);
+
 	while (mp_kdp_trap) {
 		simple_unlock(&mp_kdp_lock);
 		DBG("mp_kdp_enter() race lost\n");
-		mp_kdp_wait();
+		mp_kdp_wait(TRUE);
 		simple_lock(&mp_kdp_lock);
 	}
 	mp_kdp_ncpus = 1;	/* self */
@@ -1229,43 +1518,57 @@ mp_kdp_enter(void)
 	 * Deliver a nudge to other cpus, counting how many
 	 */
 	DBG("mp_kdp_enter() signaling other processors\n");
-	for (ncpus = 1, cpu = 0; cpu < real_ncpus; cpu++) {
-		if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
-			continue;
-		ncpus++;
-		i386_signal_cpu(cpu, MP_KDP, ASYNC);
-	}
-	/*
-	 * Wait other processors to synchronize
-	 */
-	DBG("mp_kdp_enter() waiting for (%d) processors to suspend\n", ncpus);
+	if (force_immediate_debugger_NMI == FALSE) {
+		for (ncpus = 1, cpu = 0; cpu < real_ncpus; cpu++) {
+			if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
+				continue;
+			ncpus++;
+			i386_signal_cpu(cpu, MP_KDP, ASYNC);
+		}
+		/*
+		 * Wait other processors to synchronize
+		 */
+		DBG("mp_kdp_enter() waiting for (%d) processors to suspend\n", ncpus);
 
-	tsc_timeout = rdtsc64() + (ncpus * 100 * 1000 * 1000);
+		/*
+		 * This timeout is rather arbitrary; we don't want to NMI
+		 * processors that are executing at potentially
+		 * "unsafe-to-interrupt" points such as the trampolines,
+		 * but neither do we want to lose state by waiting too long.
+		 */
+		tsc_timeout = rdtsc64() + (ncpus * 1000 * 1000);
 
-	while (mp_kdp_ncpus != ncpus && rdtsc64() < tsc_timeout) {
-	        /*
-		 * A TLB shootdown request may be pending... this would
-		 * result in the requesting processor waiting in
-		 * PMAP_UPDATE_TLBS() until this processor deals with it.
-		 * Process it, so it can now enter mp_kdp_wait()
+		while (mp_kdp_ncpus != ncpus && rdtsc64() < tsc_timeout) {
+			/*
+			 * A TLB shootdown request may be pending--this would
+			 * result in the requesting processor waiting in
+			 * PMAP_UPDATE_TLBS() until this processor deals with it.
+			 * Process it, so it can now enter mp_kdp_wait()
+			 */
+			handle_pending_TLB_flushes();
+			cpu_pause();
+		}
+		/* If we've timed out, and some processor(s) are still unresponsive,
+		 * interrupt them with an NMI via the local APIC.
 		 */
-	        handle_pending_TLB_flushes();
-		cpu_pause();
+		if (mp_kdp_ncpus != ncpus) {
+			for (cpu = 0; cpu < real_ncpus; cpu++) {
+				if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
+					continue;
+				if (cpu_signal_pending(cpu, MP_KDP))
+					cpu_NMI_interrupt(cpu);
+			}
+		}
 	}
-/* If we've timed out, and some processor(s) are still unresponsive,
- * interrupt them with an NMI via the local APIC.
- */
-	if (mp_kdp_ncpus != ncpus) {
+	else
 		for (cpu = 0; cpu < real_ncpus; cpu++) {
 			if (cpu == my_cpu || !cpu_datap(cpu)->cpu_running)
 				continue;
-			if (cpu_signal_pending(cpu, MP_KDP))
-			    cpu_NMI_interrupt(cpu);
+			cpu_NMI_interrupt(cpu);
 		}
-	}
 
-	DBG("mp_kdp_enter() %d processors done %s\n",
-		mp_kdp_ncpus, (mp_kdp_ncpus == ncpus) ? "OK" : "timed out");
+	DBG("mp_kdp_enter() %u processors done %s\n",
+	    mp_kdp_ncpus, (mp_kdp_ncpus == ncpus) ? "OK" : "timed out");
 	
 	postcode(MP_KDP_ENTER);
 }
@@ -1280,25 +1583,37 @@ cpu_signal_pending(int cpu, mp_event_t event)
 		retval = TRUE;
 	return retval;
 }
+ 
 
 static void
-mp_kdp_wait(void)
+mp_kdp_wait(boolean_t flush)
 {
 	DBG("mp_kdp_wait()\n");
-
+	/* If an I/O port has been specified as a debugging aid, issue a read */
 	panic_io_port_read();
 
-	atomic_incl(&mp_kdp_ncpus, 1);
+	/* If we've trapped due to a machine-check, save MCA registers */
+	mca_check_save();
+
+	if (pmsafe_debug)
+	    pmSafeMode(&current_cpu_datap()->lcpu, PM_SAFE_FL_SAFE);
+
+	atomic_incl((volatile long *)&mp_kdp_ncpus, 1);
 	while (mp_kdp_trap) {
 	        /*
-		 * a TLB shootdown request may be pending... this would result in the requesting
-		 * processor waiting in PMAP_UPDATE_TLBS() until this processor deals with it.
+		 * A TLB shootdown request may be pending--this would result
+		 * in the requesting processor waiting in PMAP_UPDATE_TLBS()
+		 * until this processor handles it.
 		 * Process it, so it can now enter mp_kdp_wait()
 		 */
-	        handle_pending_TLB_flushes();
-
+		if (flush)
+			handle_pending_TLB_flushes();
 		cpu_pause();
 	}
+
+	if (pmsafe_debug)
+	    pmSafeMode(&current_cpu_datap()->lcpu, PM_SAFE_FL_NORMAL);
+
 	atomic_decl((volatile long *)&mp_kdp_ncpus, 1);
 	DBG("mp_kdp_wait() done\n");
 }
@@ -1323,6 +1638,10 @@ mp_kdp_exit(void)
 
 		cpu_pause();
 	}
+
+	if (pmsafe_debug)
+	    pmSafeMode(&current_cpu_datap()->lcpu, PM_SAFE_FL_NORMAL);
+
 	DBG("mp_kdp_exit() done\n");
 	(void) ml_set_interrupts_enabled(mp_kdp_state);
 	postcode(0);
@@ -1391,6 +1710,7 @@ mp_kdb_wait(void)
 {
 	DBG("mp_kdb_wait()\n");
 
+	/* If an I/O port has been specified as a debugging aid, issue a read */
 	panic_io_port_read();
 
 	atomic_incl(&mp_kdb_ncpus, 1);
@@ -1438,6 +1758,7 @@ mp_kdb_exit(void)
 
 		cpu_pause();
 	}
+
 	DBG("mp_kdb_exit() done\n");
 }
 
@@ -1461,13 +1782,8 @@ i386_init_slave(void)
 		get_cpu_number(), get_cpu_phys_number());
 
 	assert(!ml_get_interrupts_enabled());
-	if (cpu_mode_is64bit()) {
-		cpu_IA32e_enable(current_cpu_datap());
-		cpu_desc_load64(current_cpu_datap());
-		fast_syscall_init64();
-	} else {
-		fast_syscall_init();
-	}
+
+	cpu_mode_init(current_cpu_datap());
 
 	mca_cpu_init();
 
@@ -1479,9 +1795,12 @@ i386_init_slave(void)
 
 	mtrr_update_cpu();
 
+	/* resume VT operation */
+	vmx_resume();
+
 	pat_init();
 
-	cpu_thread_init();
+	cpu_thread_init();	/* not strictly necessary */
 
 	cpu_init();	/* Sets cpu_running which starter cpu waits for */ 
 
diff --git a/osfmk/i386/mp.h b/osfmk/i386/mp.h
index 6477ca377..062d2a488 100644
--- a/osfmk/i386/mp.h
+++ b/osfmk/i386/mp.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -186,6 +192,8 @@ extern	int	kdb_debug;
 extern	int	kdb_active[];
 
 extern	volatile boolean_t mp_kdp_trap;
+extern boolean_t force_immediate_debugger_NMI;
+
 extern	void	mp_kdp_enter(void);
 extern	void	mp_kdp_exit(void);
 
@@ -206,6 +214,42 @@ extern void mp_rendezvous_no_intrs(
 		void *arg);
 extern void mp_rendezvous_break_lock(void);
 
+/*
+ * All cpu broadcast.
+ * Called from thread context, this blocks until all active cpus have
+ * run action_func:
+ */
+extern void mp_broadcast(
+		void (*action_func)(void *),
+		void *arg);
+
+typedef uint32_t cpu_t;
+typedef uint32_t cpumask_t;
+static inline cpumask_t
+cpu_to_cpumask(cpu_t cpu)
+{
+	return (cpu < 32) ? (1 << cpu) : 0;
+}
+#define CPUMASK_ALL	0xffffffff
+#define CPUMASK_SELF	cpu_to_cpumask(cpu_number())
+#define CPUMASK_OTHERS	(CPUMASK_ALL & ~CPUMASK_SELF)
+
+/*
+ * Invoke a function (possibly NULL) on a set of cpus specified by a mask.
+ * The mask may include the local cpu.
+ * If the mode is:
+ *	- ASYNC: other cpus make their calls in parallel.
+ * 	- SYNC: the calls are performed serially in logical cpu order.
+ * This call returns when the function has been run on all specified cpus.
+ * The return value is the number of cpus on which the call was made.
+ * The action function is called with interrupts disabled.
+ */
+extern cpu_t mp_cpus_call(
+		cpumask_t	cpus,
+		mp_sync_t	mode,
+		void		(*action_func)(void *),
+		void		*arg);
+
 __END_DECLS
 
 #if MP_DEBUG
diff --git a/osfmk/i386/mp_desc.c b/osfmk/i386/mp_desc.c
index 40eaa28e4..12a071c2c 100644
--- a/osfmk/i386/mp_desc.c
+++ b/osfmk/i386/mp_desc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -51,7 +57,6 @@
 /*
  */
 
-
 #include <kern/cpu_number.h>
 #include <kern/kalloc.h>
 #include <kern/cpu_data.h>
@@ -66,7 +71,7 @@
 #include <i386/misc_protos.h>
 #include <i386/mp.h>
 #include <i386/pmap.h>
-#include <i386/cpu_threads.h>
+#include <i386/machine_check.h>
 
 #include <kern/misc_protos.h>
 
@@ -100,10 +105,9 @@ extern void *hi_remap_text;
 #define HI_TEXT(lo_text)	\
 	(((uint32_t)&lo_text - (uint32_t)&hi_remap_text) + HIGH_MEM_BASE)
 
-extern void	hi_sysenter(void);
-extern void	hi64_sysenter(void);
-extern void	hi64_syscall(void);
-
+extern void hi_sysenter(void);
+extern void hi64_sysenter(void);
+extern void hi64_syscall(void);
 
 /*
  * Multiprocessor i386/i486 systems use a separate copy of the
@@ -210,7 +214,6 @@ cpu_desc_init(
 				pmap_index_to_virt(HIGH_FIXED_IDT);
 	    cdi->cdi_ldt = (struct fake_descriptor *)
 				pmap_index_to_virt(HIGH_FIXED_LDT_BEGIN);
-
 	} else {
 
 	    vm_offset_t	cpu_hi_desc;
@@ -241,10 +244,10 @@ cpu_desc_init(
 	     */
 	    cdi->cdi_ldt  = (struct fake_descriptor *)
 				pmap_cpu_high_shared_remap(
-				    cdp->cpu_number,
-				    HIGH_CPU_LDT_BEGIN,
-				    (vm_offset_t) cdp->cpu_ldtp,
-				    HIGH_CPU_LDT_END - HIGH_CPU_LDT_BEGIN + 1);
+				cdp->cpu_number,
+				HIGH_CPU_LDT_BEGIN,
+				(vm_offset_t) cdp->cpu_ldtp,
+				HIGH_CPU_LDT_END - HIGH_CPU_LDT_BEGIN + 1);
 
 	    /*
 	     * Copy the tables
@@ -303,7 +306,8 @@ cpu_desc_init(
 	    cdt->ktss.ss0 = KERNEL_DS;
 	    cdt->ktss.io_bit_map_offset = 0x0FFF;	/* no IO bitmap */
 
-	    cpu_window_init(cdp->cpu_number);
+	    cpu_userwindow_init(cdp->cpu_number);
+	    cpu_physwindow_init(cdp->cpu_number);
 
 	}
 
@@ -319,95 +323,97 @@ cpu_desc_init64(
 	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
 
 	if (is_boot_cpu) {
-	    /*
-	     * Master CPU uses the tables built at boot time.
-	     * Just set the index pointers to the low memory space.
-	     * Note that in 64-bit mode these are addressed in the
-	     * double-mapped window (uber-space).
-	     */
-	    cdi->cdi_ktss = (struct i386_tss *) &master_ktss64;
-	    cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
-	    cdi->cdi_gdt  = master_gdt;
-	    cdi->cdi_idt  = (struct fake_descriptor *) &master_idt64;
-	    cdi->cdi_ldt  = (struct fake_descriptor *) &master_ldt;
-
-	    /* Replace the expanded LDT and TSS slots in the GDT: */
-	    *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
-		kernel_ldt_desc64;
-	    *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
-		kernel_tss_desc64;
+		/*
+		 * Master CPU uses the tables built at boot time.
+		 * Just set the index pointers to the low memory space.
+		 * Note that in 64-bit mode these are addressed in the
+		 * double-mapped window (uber-space).
+		 */
+		cdi->cdi_ktss = (struct i386_tss *) &master_ktss64;
+		cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
+		cdi->cdi_gdt  = master_gdt;
+		cdi->cdi_idt  = (struct fake_descriptor *) &master_idt64;
+		cdi->cdi_ldt  = (struct fake_descriptor *) &master_ldt;
+
+		/* Replace the expanded LDT and TSS slots in the GDT: */
+		*(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
+			kernel_ldt_desc64;
+		*(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
+			kernel_tss_desc64;
 
-	    /*
-	     * Fix up the expanded descriptors for 64-bit.
-	     */
-	    fix_desc64((void *) &master_idt64, IDTSZ);
-	    fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
-	    fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
+		/*
+		 * Fix up the expanded descriptors for 64-bit.
+		 */
+		fix_desc64((void *) &master_idt64, IDTSZ);
+		fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
+		fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
 
-	    /*
-	     * Set the double-fault stack as IST1 in the 64-bit TSS 
-	     */
-	    master_ktss64.ist1 = UBER64(df_task_stack_end);
+		/*
+		 * Set the double-fault stack as IST1 in the 64-bit TSS
+		 */
+		master_ktss64.ist1 = UBER64(df_task_stack_end);
 
 	} else {
-	    /*
-	     * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel 
-	     * heap (cpu_desc_table) and double-mapped in uber-space (over 4GB).
-	     * LDT descriptors are mapped into a separate area.
-	     */
-	    cdi->cdi_gdt  = cdt->gdt;
-	    cdi->cdi_idt  = (struct fake_descriptor *) cdt->idt;
-	    cdi->cdi_ktss = (struct i386_tss *) &cdt->ktss;
-	    cdi->cdi_sstk = (vm_offset_t) &cdt->sstk.top;
-	    cdi->cdi_ldt  = cdp->cpu_ldtp;
-
-	    /*
-	     * Copy the tables
-	     */
-	    bcopy((char *)master_idt64,
-		  (char *)cdt->idt,
-		  sizeof(master_idt64));
-	    bcopy((char *)master_gdt,
-		  (char *)cdt->gdt,
-		  sizeof(master_gdt));
-	    bcopy((char *)master_ldt,
-		  (char *)cdp->cpu_ldtp,
-		  sizeof(master_ldt));
-	    bcopy((char *)&master_ktss64,
-		  (char *)&cdt->ktss,
-		  sizeof(struct x86_64_tss));
+		/*
+		 * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel 
+		 * heap (cpu_desc_table) and double-mapped in uber-space
+		 * (over 4GB).
+		 * LDT descriptors are mapped into a separate area.
+		 */
+		cdi->cdi_gdt  = (struct fake_descriptor *)cdt->gdt;
+		cdi->cdi_idt  = (struct fake_descriptor *)cdt->idt;
+		cdi->cdi_ktss = (struct i386_tss *)&cdt->ktss;
+		cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top;
+		cdi->cdi_ldt  = cdp->cpu_ldtp;
 
-	    /*
-	     * Fix up the entries in the GDT to point to
-	     * this LDT and this TSS.
-	     */
-	    kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
-	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
-	    		kernel_ldt_desc64;
-	    fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
+		/*
+		 * Copy the tables
+		 */
+		bcopy((char *)master_idt64,
+				(char *)cdt->idt,
+				sizeof(master_idt64));
+		bcopy((char *)master_gdt,
+				(char *)cdt->gdt,
+				sizeof(master_gdt));
+		bcopy((char *)master_ldt,
+				(char *)cdp->cpu_ldtp,
+				sizeof(master_ldt));
+		bcopy((char *)&master_ktss64,
+				(char *)&cdt->ktss,
+				sizeof(struct x86_64_tss));
 
-	    kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
-	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
-	    		kernel_ldt_desc64;
-	    fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
+		/*
+		 * Fix up the entries in the GDT to point to
+		 * this LDT and this TSS.
+		 */
+		kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
+		*(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
+			kernel_ldt_desc64;
+		fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
 
-	    kernel_tss_desc64.offset[0] = (vm_offset_t) cdi->cdi_ktss;
-	    *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
-	    		kernel_tss_desc64;
-	    fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
+		kernel_ldt_desc64.offset[0] = (vm_offset_t) cdi->cdi_ldt;
+		*(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
+			kernel_ldt_desc64;
+		fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
 
-	    cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern;
-	    cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp;
-	    fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1);
+		kernel_tss_desc64.offset[0] = (vm_offset_t) cdi->cdi_ktss;
+		*(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
+			kernel_tss_desc64;
+		fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
 
-	    /* Set double-fault stack as IST1 */
-	    cdt->ktss.ist1 = UBER64(cdt->dfstk + sizeof(cdt->dfstk));
+		cdt->gdt[sel_idx(CPU_DATA_GS)] = cpudata_desc_pattern;
+		cdt->gdt[sel_idx(CPU_DATA_GS)].offset = (vm_offset_t) cdp;
+		fix_desc(&cdt->gdt[sel_idx(CPU_DATA_GS)], 1);
 
-	    /*
-	     * Allocate copyio windows.
-	     */
-	    cpu_window_init(cdp->cpu_number);
+		/* Set double-fault stack as IST1 */
+		cdt->ktss.ist1 = UBER64((unsigned long)cdt->dfstk
+					 + sizeof(cdt->dfstk));
 
+		/*
+		 * Allocate copyio windows.
+		 */
+		cpu_userwindow_init(cdp->cpu_number);
+		cpu_physwindow_init(cdp->cpu_number);
 	}
 
 	/* Require that the top of the sysenter stack is 16-byte aligned */
@@ -418,7 +424,7 @@ cpu_desc_init64(
 /*
  * Set MSRs for sysenter/sysexit for 64-bit.
  */
-void
+static void
 fast_syscall_init64(void)
 {
 	wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS); 
@@ -439,17 +445,18 @@ fast_syscall_init64(void)
 	/*
 	 * Emulate eflags cleared by sysenter but note that
 	 * we also clear the trace trap to avoid the complications
-	 * of single-stepping into a syscall. We also clear
-	 * the nested task bit to avoid a spurious "task switch"
-	 * on IRET.
+	 * of single-stepping into a syscall. The nested task bit
+	 * is also cleared to avoid a spurious "task switch"
+	 * should we choose to return via an IRET.
 	 */
 	wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
 
 	/*
-	 * Set the Kermel GS base MSR to point to per-cpu data in uber-space.
+	 * Set the Kernel GS base MSR to point to per-cpu data in uber-space.
 	 * The uber-space handler (hi64_syscall) uses the swapgs instruction.
 	 */
-	wrmsr64(MSR_IA32_KERNEL_GS_BASE, UBER64(current_cpu_datap()));
+	wrmsr64(MSR_IA32_KERNEL_GS_BASE,
+		UBER64((unsigned long)current_cpu_datap()));
 	kprintf("fast_syscall_init64() KERNEL_GS_BASE=0x%016llx\n",
 		rdmsr64(MSR_IA32_KERNEL_GS_BASE));
 }
@@ -457,7 +464,7 @@ fast_syscall_init64(void)
 /*
  * Set MSRs for sysenter/sysexit
  */
-void
+static void
 fast_syscall_init(void)
 {
 	wrmsr(MSR_IA32_SYSENTER_CS, SYSENTER_CS, 0); 
@@ -543,6 +550,9 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 		goto abort;
 	}
 
+	/* Machine-check shadow register allocation. */
+	mca_cpu_alloc(cdp);
+
 	simple_lock(&cpu_lock);
 	if (real_ncpus >= max_ncpus) {
 		simple_unlock(&cpu_lock);
@@ -553,8 +563,8 @@ cpu_data_alloc(boolean_t is_boot_cpu)
 	real_ncpus++;
 	simple_unlock(&cpu_lock);
 
-	kprintf("cpu_data_alloc(%d) 0x%x desc_table: 0x%x "
-		"ldt: 0x%x "
+	kprintf("cpu_data_alloc(%d) %p desc_table: %p "
+		"ldt: %p "
 		"int_stack: 0x%x-0x%x\n",
 		cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
 		cdp->cpu_int_stack_top - INTSTACK_SIZE, cdp->cpu_int_stack_top);
@@ -592,29 +602,27 @@ valid_user_segment_selectors(uint16_t cs,
 
 
 static vm_offset_t user_window_base = 0;
-static vm_offset_t phys_window_base = 0;
 
 void
-cpu_window_init(int cpu)
+cpu_userwindow_init(int cpu)
 {
 	cpu_data_t		*cdp = cpu_data_ptr[cpu];
-	cpu_desc_index_t	*cdi;
+	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
         vm_offset_t 		user_window;
-        vm_offset_t 		phys_window;
         vm_offset_t 		vaddr;
 	int			num_cpus;
 
 	num_cpus = ml_get_max_cpus();
 
 	if (cpu >= num_cpus)
-	        panic("copy_window_init: cpu > num_cpus");
+	        panic("cpu_userwindow_init: cpu > num_cpus");
 
 	if (user_window_base == 0) {
 
 	        if (vm_allocate(kernel_map, &vaddr,
 				(NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE,
 				VM_FLAGS_ANYWHERE) != KERN_SUCCESS)
-		        panic("copy_window_init: "
+		        panic("cpu_userwindow_init: "
 				"couldn't allocate user map window");
 
 		/*
@@ -639,19 +647,10 @@ cpu_window_init(int cpu)
 				(vaddr +
 				 ((NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE)) -
 				 user_window);
-
-	        if (vm_allocate(kernel_map, &phys_window_base,
-				PAGE_SIZE * num_cpus, VM_FLAGS_ANYWHERE)
-					!= KERN_SUCCESS)
-		        panic("copy_window_init: "
-				"couldn't allocate phys map window");
 	}
 
 	user_window = user_window_base + (cpu * NCOPY_WINDOWS * NBPDE);
-	phys_window = phys_window_base + (cpu * PAGE_SIZE);
 
-	cdi = &cdp->cpu_desc_index;
-	  
 	cdp->cpu_copywindow_base = user_window;
 	cdp->cpu_copywindow_pdp  = pmap_pde(kernel_pmap, user_window);
 
@@ -660,15 +659,28 @@ cpu_window_init(int cpu)
 
 	fix_desc(&cdi->cdi_gdt[sel_idx(USER_WINDOW_SEL)], 1);
 
-	cdp->cpu_physwindow_base = phys_window;
+}
 
-	/*
-	 * make sure the page that encompasses the
-	 * pte pointer we're interested in actually
-	 * exists in the page table
-	 */
+void
+cpu_physwindow_init(int cpu)
+{
+	cpu_data_t		*cdp = cpu_data_ptr[cpu];
+	cpu_desc_index_t	*cdi = &cdp->cpu_desc_index;
+        vm_offset_t 		phys_window;
+
+	if (vm_allocate(kernel_map, &phys_window,
+			PAGE_SIZE, VM_FLAGS_ANYWHERE)
+				!= KERN_SUCCESS)
+	        panic("cpu_physwindow_init: couldn't allocate phys map window");
+
+        /*
+         * make sure the page that encompasses the
+         * pte pointer we're interested in actually
+         * exists in the page table
+         */
 	pmap_expand(kernel_pmap, phys_window);
 
+	cdp->cpu_physwindow_base = phys_window;
 	cdp->cpu_physwindow_ptep = vtopte(phys_window);
 
 	cdi->cdi_gdt[sel_idx(PHYS_WINDOW_SEL)] = physwindow_desc_pattern;
@@ -715,3 +727,20 @@ cpu_desc_load64(cpu_data_t *cdp)
 
 	kprintf("64-bit descriptor tables loaded\n");
 }
+
+void
+cpu_mode_init(cpu_data_t *cdp)
+{
+	if (cpu_mode_is64bit()) {
+		cpu_IA32e_enable(cdp);
+		cpu_desc_load64(cdp);
+		fast_syscall_init64();
+	} else {
+		fast_syscall_init();
+	}
+
+	/* Call for per-cpu pmap mode initialization */
+	pmap_cpu_init();
+
+}
+
diff --git a/osfmk/i386/mp_desc.h b/osfmk/i386/mp_desc.h
index c7916f1c9..d9ada40cb 100644
--- a/osfmk/i386/mp_desc.h
+++ b/osfmk/i386/mp_desc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -116,8 +122,8 @@ extern void	cpu_desc_init64(
 			boolean_t	is_boot_cpu);
 extern void	cpu_desc_load64(
 			cpu_data_t	*cdp);
-extern void	fast_syscall_init(void);
-extern void	fast_syscall_init64(void);
+extern void	cpu_mode_init(
+			cpu_data_t	*cdp);
 
 static inline boolean_t
 valid_user_data_selector(uint16_t selector)
diff --git a/osfmk/i386/mp_events.h b/osfmk/i386/mp_events.h
index aa3b4f2ee..43257779a 100644
--- a/osfmk/i386/mp_events.h
+++ b/osfmk/i386/mp_events.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef __AT386_MP_EVENTS__
 #define	__AT386_MP_EVENTS__
@@ -37,6 +43,8 @@ typedef enum {
 	MP_IDLE,
 	MP_UNIDLE,
 	MP_CHUD,
+	MP_BROADCAST,
+	MP_CALL,
 	MP_LAST
 } mp_event_t;
 
@@ -50,9 +58,11 @@ const char *mp_event_name[] = {	\
 	"MP_IDLE",		\
 	"MP_UNIDLE",		\
 	"MP_CHUD",		\
+	"MP_BROADCAST",		\
+	"MP_CALL",		\
 	"MP_LAST"		\
 }
-	
+
 typedef enum { SYNC, ASYNC } mp_sync_t;
 
 __BEGIN_DECLS
@@ -60,6 +70,8 @@ __BEGIN_DECLS
 extern void	i386_signal_cpu(int cpu, mp_event_t event, mp_sync_t mode);
 extern void	i386_signal_cpus(mp_event_t event, mp_sync_t mode);
 extern int	i386_active_cpus(void);
+extern void	i386_activate_cpu(void);
+extern void	i386_deactivate_cpu(void);
 
 __END_DECLS
 
diff --git a/osfmk/i386/mp_slave_boot.h b/osfmk/i386/mp_slave_boot.h
index 06949fff1..d011444e7 100644
--- a/osfmk/i386/mp_slave_boot.h
+++ b/osfmk/i386/mp_slave_boot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/mp_slave_boot.s b/osfmk/i386/mp_slave_boot.s
index c1a8d8ccf..15a9d811c 100644
--- a/osfmk/i386/mp_slave_boot.s
+++ b/osfmk/i386/mp_slave_boot.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/mtrr.c b/osfmk/i386/mtrr.c
index fe718c39d..cd818d511 100644
--- a/osfmk/i386/mtrr.c
+++ b/osfmk/i386/mtrr.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/kern_return.h>
@@ -221,8 +227,6 @@ mtrr_msr_dump(void)
 void
 mtrr_init(void)
 {
-	i386_cpu_info_t	* infop = cpuid_info();
-
 	/* no reason to init more than once */
 	if (mtrr_initialized == TRUE)
 		return;
@@ -231,41 +235,6 @@ mtrr_init(void)
 	if ((cpuid_features() & CPUID_FEATURE_MTRR) == 0)
         	return;  /* no MTRR feature */
 
-	/* cpu vendor/model specific handling */
-	if (!strncmp(infop->cpuid_vendor, CPUID_VID_AMD, sizeof(CPUID_VID_AMD)))
-	{
-		/* Check for AMD Athlon 64 and Opteron */
-		if (cpuid_family() == 0xF)
-		{
-			uint32_t cpuid_result[4];
-
-			/* check if cpu support Address Sizes function */
-			do_cpuid(0x80000000, cpuid_result);
-			if (cpuid_result[0] >= 0x80000008)
-			{
-				int bits;
-
-				do_cpuid(0x80000008, cpuid_result);
-				DBG("MTRR: AMD 8000_0008 EAX = %08x\n",
-					cpuid_result[0]);
-
-				/*
-				 * Function 8000_0008 (Address Sizes) EAX
-				 * Bits  7-0 : phys address size
-				 * Bits 15-8 : virt address size
-				 */
-				bits = cpuid_result[0] & 0xFF;
-				if ((bits < 36) || (bits > 64))
-				{
-					printf("MTRR: bad address size\n");
-					return; /* bogus size */
-				}
-
-				mtrr_phys_mask = PHYS_BITS_TO_MASK(bits);
-			}
-		}
-	}
-
 	/* use a lock to serialize MTRR changes */
 	bzero((void *)&mtrr_state, sizeof(mtrr_state));
 	simple_lock_init(&mtrr_lock, 0);
diff --git a/osfmk/i386/mtrr.h b/osfmk/i386/mtrr.h
index 7222200b8..a749ad4ea 100644
--- a/osfmk/i386/mtrr.h
+++ b/osfmk/i386/mtrr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _I386_MTRR_H_
diff --git a/osfmk/i386/net_filter.c b/osfmk/i386/net_filter.c
deleted file mode 100644
index cda06b521..000000000
--- a/osfmk/i386/net_filter.c
+++ /dev/null
@@ -1,1554 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1993 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-
-#include <kern/zalloc.h>
-
-#define	NET_REG_EAX	0	/* data segment register */
-#define	NET_REG_EDX	1	/* header segment register */
-#define	NET_REG_EBX	2	/* free register */
-#define	NET_REG_ESI	3	/* free register */
-#define	NET_REG_EDI	4	/* free register */
-#define	NET_REG_MAX	5	/* number of available registers */
-
-struct net_opt {
-	filter_t	val;	/* value */
-	unsigned char	reg;	/* register associated */
-	unsigned char	used;	/* how many times it will be used */
-};
-
-boolean_t net_filter_enable = FALSE;
-
-/*
- * Forward declarations.
- */
-void net_filter_optimize(
-	struct net_opt	net_o[],
-	unsigned 	net_len,
-	int		reg[],
-	unsigned	nbreg);
-
-/*
- *	Compilation of a source network filter into i386 instructions.
- */
-filter_fct_t
-net_filter_alloc(
-	filter_t	*fpstart,
-	unsigned int	fplen,
-	unsigned int	*len)
-{
-	filter_t	*fp;
-	unsigned int	op;
-	unsigned int	arg;
-	unsigned char	*p;
-	filter_fct_t	top;
-	unsigned char	*pend;
-	unsigned char	*pend_old;
-	unsigned int	loop;
-	unsigned int	use_header;
-	unsigned int	use_data;
-	unsigned int	push_ecx;
-	int		reg[NET_REG_MAX];
-	struct net_opt	net_o[NET_MAX_FILTER];
-	int		net_i;
-	unsigned	net_j;
-	unsigned	i;
-	unsigned	push;
-	unsigned	false_pad;
-	struct net_opt	*pn;
-
-	/*
-	 * Addresses of end_true and end_false from the end of the program.
-	 */
-#define	PEND_TRUE	(pend_old - (11 + push + false_pad))
-#define	PEND_FALSE	(pend_old - (4 + push))
-
-	/*
-	 * Don't produce anything if net_filter generation is not enabled.
-	 */
-	if (!net_filter_enable) {
-		*len = 0;
-		return ((filter_fct_t)0);
-	}
-
-	/*
-	 * called as (*filter)(data, data_count, header)
-	 *
-	 * %esp -> stack;
-	 * %ecx -> arg;
-	 * %eax -> data (if needed);
-	 * %edx -> header (if needed);
-	 */
-	loop = 0;
-	p = (unsigned char *)0;
-	pend = 0;
-	use_header = 0;
-	use_data = 0;
-	net_j = 0;
-	false_pad = sizeof(int) - 1;
-
-	/*
-	 * The compiler needs 3 passes to produce the compiled net_filter:
-	 * 0) compute without optimization the maximum size of the object
-	 *	routine (one run),
-	 * 1) try to reduce the size of the object procedure (many runs),
-	 * 2) produce final object code (one run).
-	 */
-	for (;;) {
-		if (loop == 0)
-			pend += 14;
-
-		else if (loop == 1) {
-			if (reg[NET_REG_EBX] == -1) {
-				/* push and pop it */
-				pend++;
-				push = 1;
-			} else
-				push = 0;
-			if (reg[NET_REG_ESI] == -1) {
-				/* push and pop it */
-				pend++;
-				push++;
-			}
-			if (reg[NET_REG_EDI] == -1) {
-				/* push and pop it */
-				pend++;
-				push++;
-			}
-			if (push) {
-				/* restore %esp */
-				push += 3;
-			}
-
-			if (use_data)
-				pend += 3;
-			if (use_header)
-				pend += 3;
-			pend += 8;
-
-		} else {
-			*p++ = 0x55;	/* pushl %ebp */
-			*p++ = 0x89;	/* movl %esp, %ebp */
-			*p++ = 0xE5;
-			if (reg[NET_REG_EBX] == -1)
-				*p++ = 0x53;	/* pushl %ebx */
-			if (reg[NET_REG_ESI] == -1)
-				*p++ = 0x56;	/* pushl %esi */
-			if (reg[NET_REG_EDI] == -1)
-				*p++ = 0x57;	/* pushl %edi */
-			*p++ = 0xB9;	/* movl $1, %ecx */
-			*p++ = 0x01;
-			*p++ = 0x00;
-			*p++ = 0x00;
-			*p++ = 0x00;
-			if (use_data) {
-				*p++ = 0x8B;	/* movl 0x8(%ebp), %eax */
-				*p++ = 0x45;
-				*p++ = 0x08;
-			}
-			if (use_header) {
-				*p++ = 0x8B;	/* movl 0x10(%ebp), %edx */
-				*p++ = 0x55;
-				*p++ = 0x10;
-			}
-		}
-		push_ecx = 1;
-		net_i = -1;
-
-		fp = fpstart;
-		while (fp - fpstart < fplen)
-		{
-			arg = *fp++;
-			op = NETF_OP(arg);
-			arg = NETF_ARG(arg);
-
-			switch (arg) {
-			case NETF_NOPUSH:
-				/*
-				 * arg = *sp++;
-				 */
-				if (push_ecx) {
-					push_ecx = 0;
-					break;
-				}
-				if (loop < 2)
-					pend++;
-				else
-					*p++ = 0x59;	/* popl %ecx */
-				break;
-
-			case NETF_PUSHZERO:
-				/*
-				 * arg = 0;
-				 */
-				if (loop < 2) {
-					if (push_ecx) {
-						pend++;
-						push_ecx = 0;
-					}
-					pend += 2;
-				} else {
-					if (push_ecx) {
-						*p++ = 0x51; /* pushl %ecx */
-						push_ecx = 0;
-					}
-					*p++ = 0x31;	/* xorl %ecx, %ecx */
-					*p++ = 0xC9;
-				}
-				break;
-
-			case NETF_PUSHLIT:
-				/*
-				 * arg = *fp++;
-				 */
-				if (loop < 2) {
-					if (push_ecx) {
-						pend++;
-						push_ecx = 0;
-					}
-					pend += 5;
-				} else {
-					if (push_ecx) {
-						*p++ = 0x51; /* pushl %ecx */
-						push_ecx = 0;
-					}
-					*p++ = 0xB9;	/* movl *fp, %ecx */
-					*p++ = *(unsigned char *)fp;
-					*p++ = *(((unsigned char *)fp) + 1);
-					*p++ = 0x0;
-					*p++ = 0x0;
-				}
-				fp++;
-				break;
-
-			case NETF_PUSHIND:
-				/*
-				 * arg = *sp++;
-				 * if (arg >= data_count)
-				 *     return FALSE;
-				 * arg = data_word[arg];
-				 */
-				if (loop < 2) {
-					if (push_ecx)
-						push_ecx = 0;
-					else
-						pend++;
-					if (loop == 0)
-						use_data = 1;
-					if (loop == 0 ||
-					    PEND_FALSE - (pend + 5) >= 128)
-						pend += 14;
-					else
-						pend += 10;
-					break;
-				}
-
-				if (push_ecx)
-					push_ecx = 0;
-				else
-					*p++ = 0x59; /* popl %ecx */
-				*p++ = 0x39; /* cmpl 0xC(%ebp), %ecx */
-				*p++ = 0x4D;
-				*p++ = 0x0C;
-
-				if (PEND_FALSE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* jle end_false */
-					*p++ = 0x8E;
-					*(p+0) =  PEND_FALSE - (p + 4);
-					*(p+1) = (PEND_FALSE - (p + 4)) >> 8;
-					*(p+2) = (PEND_FALSE - (p + 4)) >> 16;
-					*(p+3) = (PEND_FALSE - (p + 4)) >> 24;
-					p += 4;
-
-				} else {
-					*p++ = 0x7E; /* jle end_false */
-					*p = PEND_FALSE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x0F; /* movzwl 0(%eax,%ecx,2), %ecx */
-				*p++ = 0xB7;
-				*p++ = 0x4C;
-				*p++ = 0x48;
-				*p++ = 0x00;
-				break;
-
-			case NETF_PUSHHDRIND:
-				/*
-				 * arg = *sp++;
-				 * if (arg >= (NET_HDW_HDR_MAX /
-				 *             sizeof(unsigned short))
-				 *     return FALSE;
-				 * arg = header_word[arg];
-				 */
-				if (loop < 2) {
-					if (push_ecx)
-						push_ecx = 0;
-					else
-						pend++;
-					if (loop == 0)
-						use_header = 1;
-					if (loop == 0 ||
-					    PEND_FALSE - (pend + 8) >= 128)
-						pend += 17;
-					else
-						pend += 13;
-					break;
-				}
-
-				if (push_ecx)
-					push_ecx = 0;
-				else
-					*p++ = 0x59;	/* popl %ecx */
-				*p++ = 0x81;	/* cmpl %ecx, <value> */
-				*p++ = 0xF9;
-				*p++ =  NET_HDW_HDR_MAX /
-					sizeof(unsigned short);
-				*p++ = (NET_HDW_HDR_MAX /
-					sizeof(unsigned short)) >> 8;
-				*p++ = (NET_HDW_HDR_MAX /
-					sizeof(unsigned short)) >> 16;
-				*p++ = (NET_HDW_HDR_MAX /
-					sizeof(unsigned short)) >> 24;
-
-				if (PEND_FALSE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* jge end_false */
-					*p++ = 0x8D;
-					*(p+0) =  PEND_FALSE - (p + 4);
-					*(p+1) = (PEND_FALSE - (p + 4)) >> 8;
-					*(p+2) = (PEND_FALSE - (p + 4)) >> 16;
-					*(p+3) = (PEND_FALSE - (p + 4)) >> 24;
-					p += 4;
-
-				} else {
-					*p++ = 0x7D;	/* jge end_false */
-					*p = PEND_FALSE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x0F; /* movzwl 0(%edx,%ecx,2), %ecx */
-				*p++ = 0xB7;
-				*p++ = 0x4C;
-				*p++ = 0x4A;
-				*p++ = 0x00;
-				break;
-
-			default:
-				if (arg >= NETF_PUSHSTK) {
-					arg -= NETF_PUSHSTK;
-					/*
-					 * arg = sp[arg];
-					 */
-					arg <<= 2;
-					if (loop < 2) {
-						if (push_ecx) {
-							pend++;
-							push_ecx = 0;
-						}
-						pend += (arg < 128) ? 4 : 7;
-						break;
-					}
-
-					if (push_ecx) {
-						*p++ = 0x51; /* pushl %ecx */
-						push_ecx = 0;
-					}
-					*p++ = 0x8B; /* movl arg(%esp), %ecx */
-					if (arg < 128) {
-						*p++ = 0x4C;
-						*p++ = 0x24;
-						*p++ = arg;
-					} else {
-						*p++ = 0x8C;
-						*p++ = 0x24;
-						*p++ = arg;
-						*p++ = arg >> 8;
-						*p++ = arg >> 16;
-						*p++ = arg >> 24;
-					}
-				
-				} else if (arg >= NETF_PUSHHDR) {
-					arg -= NETF_PUSHHDR;
-					/*
-					 * arg = header_word[arg];
-					 */
-					arg <<= 1;
-					if (loop < 2) {
-						if (push_ecx) {
-							pend++;
-							push_ecx = 0;
-						}
-						if (loop == 0) {
-							use_header = 1;
-							net_o[net_j++].val =
-							    arg + NETF_PUSHHDR;
-						} else {
-							net_i++;
-							assert(net_i < net_j);
-							pn = &net_o[net_i];
-							assert(reg[NET_REG_EDX]
-							       == -2);
-							assert(pn->used == 0 ||
-							       reg[pn->reg]
-							       != -2);
-							assert(pn->val == arg +
-							       NETF_PUSHHDR);
-							if (pn->used > 0 &&
-							    reg[pn->reg] >= 0 &&
-							    net_o[reg[pn->reg]]
-							    .val == pn->val) {
-								pend += 2;
-								break;
-							}
-						}
-						pend += (arg < 128) ? 5 : 8;
-						if (loop == 1 && pn->used > 1 &&
-						    (reg[pn->reg] < 0 ||
-						     net_o[reg[pn->reg]].val !=
-						     pn->val)) {
-							reg[pn->reg] = net_i;
-							pend += 2;
-						}
-						break;
-					}
-
-					if (push_ecx) {
-						*p++ = 0x51; /* pushl %ecx */
-						push_ecx = 0;
-					}
-
-					net_i++;
-					assert(net_i < net_j);
-					pn = &net_o[net_i];
-					assert(reg[NET_REG_EDX] == -2);
-					assert(pn->used == 0 ||
-					       reg[pn->reg] != -2);
-					assert(pn->val == arg + NETF_PUSHHDR);
-					if (pn->used > 0 &&
-					    reg[pn->reg] >= 0 &&
-					    net_o[reg[pn->reg]].val ==
-					    pn->val) {
-						*p++ = 0x89;
-						switch (pn->reg) {
-						case NET_REG_EAX:
-							/* movl %eax, %ecx */
-							*p++ = 0xC1;
-							break;
-
-						case NET_REG_EBX:
-							/* movl %ebx, %ecx */
-							*p++ = 0xD9;
-							break;
-
-						case NET_REG_ESI:
-							/* movl %esi, %ecx */
-							*p++ = 0xF1;
-							break;
-
-						case NET_REG_EDI:
-							/* movl %edi, %ecx */
-							*p++ = 0xF9;
-							break;
-						}
-						break;
-					}
-
-					*p++ = 0x0F;/* movzwl arg(%edx),%ecx */
-					*p++ = 0xB7;
-					if (arg < 128) {
-						*p++ = 0x4C;
-						*p++ = 0x22;
-						*p++ = arg;
-					} else {
-						*p++ = 0x8C;
-						*p++ = 0x22;
-						*p++ = arg;
-						*p++ = arg >> 8;
-						*p++ = arg >> 16;
-						*p++ = arg >> 24;
-					}
-
-					if (pn->used > 1 &&
-					    (reg[pn->reg] == -1 ||
-					     net_o[reg[pn->reg]].val !=
-					     pn->val)) {
-						reg[pn->reg] = net_i;
-						*p++ = 0x89;
-						assert(net_o[net_i].reg !=
-						       NET_REG_EDX);
-						switch (net_o[net_i].reg) {
-						case NET_REG_EAX:
-							/* movl %ecx, %eax */
-							*p++ = 0xC8;
-							break;
-						case NET_REG_EBX:
-							/* movl %ecx, %ebx */
-							*p++ = 0xCB;
-							break;
-						case NET_REG_ESI:
-							/* movl %ecx, %esi */
-							*p++ = 0xCE;
-							break;
-						case NET_REG_EDI:
-							/* movl %ecx, %edi */
-							*p++ = 0xCF;
-							break;
-						}
-					}
-
-				} else {
-					arg -= NETF_PUSHWORD;
-					/*
-					 *     if (arg >= data_count)
-					 *         return FALSE;
-					 *     arg = data_word[arg];
-					 */
-					if (loop < 2) {
-						if (push_ecx) {
-							pend++;
-							push_ecx = 0;
-						}
-						if (loop == 0) {
-							use_data = 1;
-							net_o[net_j++].val =
-							    arg + NETF_PUSHWORD;
-						} else {
-							net_i++;
-							assert(net_i < net_j);
-							pn = &net_o[net_i];
-							assert(reg[NET_REG_EAX]
-							       == -2);
-							assert(pn->used == 0 ||
-							       reg[pn->reg]
-							       != -2);
-							assert(pn->val == arg +
-							       NETF_PUSHWORD);
-							if (pn->used > 0 &&
-							    reg[pn->reg] >= 0 &&
-							    net_o[reg[pn->reg]]
-							    .val == pn->val) {
-								pend += 2;
-								break;
-							}
-						}
-						arg <<= 1;
-						pend += (arg < 128) ? 4 : 7;
-						if (loop == 0 ||
-						    (PEND_FALSE -
-						     (pend + 2)) >= 128)
-							pend += 6;
-						else
-							pend += 2;
-
-						if (arg < 128)
-							pend += 5;
-						else
-							pend += 8;
-						if (loop == 1 && pn->used > 1 &&
-						    (reg[pn->reg] < 0 ||
-						     net_o[reg[pn->reg]].val !=
-						     pn->val)) {
-							reg[pn->reg] = net_i;
-							pend += 2;
-						}
-						break;
-					}
-
-					if (push_ecx) {
-						*p++ = 0x51; /* pushl %ecx */
-						push_ecx = 0;
-					}
-
-					net_i++;
-					assert(net_i < net_j);
-					pn = &net_o[net_i];
-					assert(reg[NET_REG_EAX] == -2);
-					assert(pn->used == 0 ||
-					       reg[pn->reg] != -2);
-					assert(pn->val == arg + NETF_PUSHWORD);
-					if (pn->used > 0 &&
-					    reg[pn->reg] >= 0 &&
-					    net_o[reg[pn->reg]].val ==
-					    pn->val) {
-						*p++ = 0x89;
-						switch (pn->reg) {
-						case NET_REG_EDX:
-							/* movl %edx, %ecx */
-							*p++ = 0xD1;
-							break;
-
-						case NET_REG_EBX:
-							/* movl %ebx, %ecx */
-							*p++ = 0xD9;
-							break;
-
-						case NET_REG_ESI:
-							/* movl %esi, %ecx */
-							*p++ = 0xF1;
-							break;
-
-						case NET_REG_EDI:
-							/* movl %edi, %ecx */
-							*p++ = 0xF9;
-							break;
-						}
-						break;
-					}
-
-					/* cmpl 0xC(%ebp), <arg> */
-					arg <<= 1;
-					if (arg < 128) {
-						*p++ = 0x83;
-						*p++ = 0x7D;
-						*p++ = 0x0C;
-						*p++ = arg;
-					} else {
-						*p++ = 0x81;
-						*p++ = 0x7D;
-						*p++ = 0x0C;
-						*p++ = arg;
-						*p++ = arg >> 8;
-						*p++ = arg >> 16;
-						*p++ = arg >> 24;
-					}
-
-					if (PEND_FALSE - (p + 2) >= 128) {
-						*p++ = 0x0F;/* jle end_false */
-						*p++ = 0x8E;
-						*(p+0) =  PEND_FALSE - (p + 4);
-						*(p+1) = (PEND_FALSE - (p + 4))
-							  >> 8;
-						*(p+2) = (PEND_FALSE - (p + 4))
-							  >> 16;
-						*(p+3) = (PEND_FALSE - (p + 4))
-							  >> 24;
-						p += 4;
-
-					} else {
-						*p++ = 0x7E;/* jle end_false */
-						*p = PEND_FALSE - (p + 1);
-						p++;
-					}
-
-					*p++ = 0x0F;/* movzwl arg(%eax),%ecx */
-					*p++ = 0xB7;
-					if (arg < 128) {
-						*p++ = 0x4C;
-						*p++ = 0x20;
-						*p++ = arg;
-					} else {
-						*p++ = 0x8C;
-						*p++ = 0x20;
-						*p++ = arg;
-						*p++ = arg >> 8;
-						*p++ = arg >> 16;
-						*p++ = arg >> 24;
-					}
-
-					if (pn->used > 1 &&
-					    (reg[pn->reg] == -1 ||
-					     net_o[reg[pn->reg]].val !=
-					     pn->val)) {
-						reg[pn->reg] = net_i;
-						*p++ = 0x89;
-						assert(net_o[net_i].reg !=
-						       NET_REG_EAX);
-						switch (net_o[net_i].reg) {
-						case NET_REG_EDX:
-							/* movl %ecx, %edx */
-							*p++ = 0xCA;
-							break;
-						case NET_REG_EBX:
-							/* movl %ecx, %ebx */
-							*p++ = 0xCB;
-							break;
-						case NET_REG_ESI:
-							/* movl %ecx, %esi */
-							*p++ = 0xCE;
-							break;
-						case NET_REG_EDI:
-							/* movl %ecx, %edi */
-							*p++ = 0xCF;
-							break;
-						}
-					}
-				}
-				break;
-			}
-
-			switch (op) {
-			case NETF_OP(NETF_NOP):
-				/*
-				 * *--sp = arg;
-				 */
-				push_ecx = 1;
-				break;
-
-			case NETF_OP(NETF_AND):
-				/*
-				 * *sp &= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0x21;	/* andl (%esp), %ecx */
-					*p++ = 0x0C;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_OR):
-				/*
-				 * *sp |= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0x09;	/* orl (%esp), %ecx */
-					*p++ = 0x0C;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_XOR):
-				/*
-				 * sp ^= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0x31;	/* xorl (%esp), %ecx */
-					*p++ = 0x0C;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_EQ):
-				/*
-				 * *sp = (*sp == arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x74;	/* je .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_NEQ):
-				/*
-				 * *sp = (*sp != arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x75;	/* jne .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_LT):
-				/*
-				 * *sp = (*sp < arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x7C;	/* jl .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_LE):
-				/*
-				 * *sp = (*sp <= arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x7E;	/* jle .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_GT):
-				/*
-				 * *sp = (*sp > arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x7F;	/* jg .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_GE):
-				/*
-				 * *sp = (*sp >= arg);
-				 */
-				if (loop < 2) {
-					pend += 14;
-					/*
-					 * Pad to longword boundary (cf dissas).
-					 */
-					if (i = ((pend - (unsigned char *)0) &
-						 (sizeof(int) - 1)))
-						pend += (sizeof(int) - i);
-					pend += 7;
-					break;
-				}
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				i = ((p - (unsigned char *)top) + 11) &
-					(sizeof(int) - 1);
-				*p++ = 0x7D;	/* jge .+9+<pad> */
-				*p++ = 0x09 + (i ? sizeof(int) - i : 0);
-				*p++ = 0xC7;	/* movl $0, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-
-				i = ((p - (unsigned char *)top) + 2) &
-					(sizeof(int) - 1);
-				*p++ = 0xEB;	/* jmp .+7+<pad> */
-				*p++ = 0x07 + (i ? sizeof(int) - i : 0);
-
-				/*
-				 * Pad to longword boundary (cf dissas).
-				 */
-				if (i = (p - (unsigned char *)top) &
-				    (sizeof(int) - 1))
-					while (i++ < sizeof(int))
-						*p++ = 0x90; /* nop */
-				*p++ = 0xC7;	/* movl $1, 0(%esp) */
-				*p++ = 0x04;
-				*p++ = 0x24;
-				*p++ = 0x01;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				*p++ = 0x00;
-				break;
-
-			case NETF_OP(NETF_COR):
-				/*
-				 * if (*sp++ == arg)
-				 *     return (TRUE);
-				 */
-				if (loop < 2) {
-					if (loop == 0 ||
-					    PEND_TRUE - (pend + 5) >= 128)
-						pend += 12;
-					else
-						pend += 8;
-					break;
-				}
-
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				if (PEND_TRUE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* je end_true */
-					*p++ = 0x84;
-					*(p+0) =  PEND_TRUE - (p + 4);
-					*(p+1) = (PEND_TRUE - (p + 4)) >> 8;
-					*(p+2) = (PEND_TRUE - (p + 4)) >> 16;
-					*(p+3) = (PEND_TRUE - (p + 4)) >> 24;
-					p += 4;
-
-				} else {
-					*p++ = 0x74;	/* je end_true */
-					*p = PEND_TRUE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x83;	/* addl $4, %esp */
-				*p++ = 0xC4;
-				*p++ = 0x04;
-				break;
-
-			case NETF_OP(NETF_CAND):
-				/*
-				 * if (*sp++ != arg)
-				 *     return (FALSE);
-				 */
-				if (loop < 2) {
-					if (loop == 0 ||
-					    PEND_FALSE - (pend + 5) >= 128)
-						pend += 12;
-					else
-						pend += 8;
-					break;
-				}
-
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				if (PEND_FALSE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* jne end_false */
-					*p++ = 0x85;
-					*(p+0) =  PEND_FALSE - (p + 4);
-					*(p+1) = (PEND_FALSE - (p + 4)) >> 8;
-					*(p+2) = (PEND_FALSE - (p + 4)) >> 16;
-					*(p+3) = (PEND_FALSE - (p + 4)) >> 24;
-					p += 4;
-
-				} else {
-					*p++ = 0x75;	/* jne end_false */
-					*p = PEND_FALSE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x83;	/* addl $4, %esp */
-				*p++ = 0xC4;
-				*p++ = 0x04;
-				break;
-
-			case NETF_OP(NETF_CNOR):
-				/*
-				 * if (*sp++ == arg)
-				 *     return (FALSE);
-				 */
-				if (loop < 2) {
-					if (loop == 0 ||
-					    PEND_FALSE - (pend + 5) >= 128)
-						pend += 12;
-					else
-						pend += 8;
-					break;
-				}
-
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				if (PEND_FALSE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* je end_false */
-					*p++ = 0x84;
-					*(p+0) =  PEND_FALSE - (p + 4);
-					*(p+1) = (PEND_FALSE - (p + 4)) >> 8;
-					*(p+2) = (PEND_FALSE - (p + 4)) >> 16;
-					*(p+3) = (PEND_FALSE - (p + 4)) >> 24;
-					p += 4;
-				} else {
-					*p++ = 0x74;	/* je end_false */
-					*p = PEND_FALSE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x83;	/* addl $4, %esp */
-				*p++ = 0xC4;
-				*p++ = 0x04;
-				break;
-
-			case NETF_OP(NETF_CNAND):
-				/*
-				 * if (*sp++ != arg)
-				 *     return (TRUE);
-				 */
-				if (loop < 2) {
-					if (loop == 0 ||
-					    PEND_TRUE - (pend + 5) >= 128)
-						pend += 12;
-					else
-						pend += 8;
-					break;
-				}
-
-				*p++ = 0x39;	/* cmpl (%esp), %ecx */
-				*p++ = 0x0C;
-				*p++ = 0x24;
-
-				if (PEND_TRUE - (p + 2) >= 128) {
-					*p++ = 0x0F;	/* jne end_true */
-					*p++ = 0x85;
-					*(p+0) =  PEND_TRUE - (p + 4);
-					*(p+1) = (PEND_TRUE - (p + 4)) >> 8;
-					*(p+2) = (PEND_TRUE - (p + 4)) >> 16;
-					*(p+3) = (PEND_TRUE - (p + 4)) >> 24;
-					p += 4;
-
-				} else {
-					*p++ = 0x75;	/* jne end_true */
-					*p = PEND_TRUE - (p + 1);
-					p++;
-				}
-
-				*p++ = 0x83;	/* addl $4, %esp */
-				*p++ = 0xC4;
-				*p++ = 0x04;
-				break;
-
-			case NETF_OP(NETF_LSH):
-				/*
-				 * *sp <<= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0xD3;	/* sall (%esp), %cl */
-					*p++ = 0x24;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_RSH):
-				/*
-				 * *sp >>= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0xD3;	/* sarl (%esp), %cl */
-					*p++ = 0x3C;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_ADD):
-				/*
-				 * *sp += arg;
-				 */ 
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0x01;	/* addl (%esp), %ecx */
-					*p++ = 0x0C;
-					*p++ = 0x24;
-				}
-				break;
-
-			case NETF_OP(NETF_SUB):
-				/*
-				 * *sp -= arg;
-				 */
-				if (loop < 2)
-					pend += 3;
-				else {
-					*p++ = 0x29;	/* subl (%esp), %ecx */
-					*p++ = 0x0C;
-					*p++ = 0x24;
-				}
-				break;
-			}
-		}
-
-		/*
-		 * return ((*sp) ? TRUE : FALSE);
-		 */
-		if (loop < 2) {
-			if (push_ecx) {
-				pend += 12;
-				push_ecx = 0;
-			} else
-				pend += 13;
-			/*
-			 * Pad to longword boundary (cf dissas).
-			 */
-			i = (pend - (unsigned char *)0) & (sizeof(int) - 1);
-			false_pad = i ? sizeof(int) - i : 0;
-			pend += 4 + push + false_pad;
-		} else {
-			if (push_ecx) {
-				*p++ = 0x83;	/* cmpl %ecx, $0 */
-				*p++ = 0xF9;
-				*p++ = 0x00;
-				push_ecx = 0;
-			} else {
-				*p++ = 0x83;	/* cmpl (%esp), $0 */
-				*p++ = 0x3C;
-				*p++ = 0x24;
-				*p++ = 0x00;
-			}
-
-			i = ((p - (unsigned char *)top) + 9) &
-				(sizeof(int) - 1);
-			false_pad = i ? sizeof(int) - i : 0;
-			*p++ = 0x74;	/* je end_false */
-			*p++ = 0x07 + false_pad;
-
-			*p++ = 0xB8;	/* movl $1, %eax */
-			*p++ = 0x01;
-			*p++ = 0x00;
-			*p++ = 0x00;
-			*p++ = 0x00;
-
-			*p++ = 0xEB;	/* jmp .+2+<pad> */
-			*p++ = 0x02 + false_pad;
-
-			/*
-			 * Pad to longword boundary (cf dissas).
-			 */
-			for (i = 0; i < false_pad; i++)
-				*p++ = 0x90; /* nop */
-			*p++ = 0x31;	/* xorl %eax, %eax */
-			*p++ = 0xC0;
-			if (push) {
-				*p++ = 0x8D;	/* leal -<push>(%ebx), %esp */
-				*p++ = 0x65;
-				*p++ = -((push - 3) * 4);
-			}
-			if (reg[NET_REG_EDI] >= 0)
-				*p++ = 0x5F;	/* pop %edi */
-			if (reg[NET_REG_ESI] >= 0)
-				*p++ = 0x5E;	/* pop %esi */
-			if (reg[NET_REG_EBX] >= 0)
-				*p++ = 0x5B;	/* pop %ebx */
-			*p++ = 0xC9;	/* leave */
-			*p++ = 0xC3;	/* ret */
-		}
-
-		/*
-		 * Prepare next loop if any.
-		 */
-		if (loop == 2)
-			break;
-
-		if (loop == 1 && pend == pend_old) {
-			loop = 2;
-			*len = pend - (unsigned char *)0;
-			top = (filter_fct_t)kalloc(*len);
-			p = (unsigned char *)top;
-			pend_old = p + (pend - (unsigned char *)0);
-		} else {
-			if (loop == 0) {
-				loop = 1;
-				/*
-				 * Compute and optimize free registers usage.
-				 */
-				for (i = 0; i < NET_REG_MAX; i++)
-					reg[i] = -1;
-				if (use_data)
-					reg[NET_REG_EAX] = -2;
-				if (use_header)
-					reg[NET_REG_EDX] = -2;
-				net_filter_optimize(net_o, net_j,
-						    reg, NET_REG_MAX);
-			}
-			pend_old = pend;
-			pend = 0;
-		}
-		for (i = 0; i < NET_REG_MAX; i++)
-			if (reg[i] != -2)
-				reg[i] = -1;
-	}
-	return (top);
-}
-
-void
-net_filter_free(
-	filter_fct_t	fp,
-	unsigned int	len)
-{
-	kfree((vm_offset_t)fp, len);
-}
-
-/*
- * Try to compute how to use (if needed) extra registers to store
- * values read more than once.
- *
- * Input : 	net_o is an array of used values (only .val is valid).
- *		net_len is the length of net_o.
- *		reg is an array of available registers (-2 ==> used register).
- *		nbreg is the maximum number of registers.
- *
- * Output :	net_o is an array of register usage.
- *			.used == 0 ==> do not used any register.
- *			.used >= 2 ==> how many times the .reg register
- *						will be used.
- *		reg is an array of used registers.
- *			== -2 ==> unused or unavailable register.
- *			>=  0 ==> used register.
- *
- * N.B. This procedure is completely machine-independent and should take place
- *		in a file of the device directory.
- */
-void
-net_filter_optimize(
-	struct net_opt	net_o[],
-	unsigned 	net_len,
-	int		reg[],
-	unsigned	nbreg)
-{
-	unsigned	i;
-	unsigned	j;
-	unsigned	nbnet;
-	unsigned	avail;
-	unsigned	used;
-	unsigned	first;
-	unsigned	max;
-	unsigned	last;
-	struct net_opt	*p;
-	struct net_opt	*q;
-
-	avail = 0;
-	for (i = 0; i < nbreg; i++)
-		if (reg[i] != -2)
-			avail++;
-	if (avail == 0)
-		return;
-
-	/*
-	 * First step: set up used field.
-	 */
-	p = &net_o[net_len];
-	while (p != net_o) {
-		for (q = p--; q < &net_o[net_len]; q++)
-			if (q->val == p->val) {
-				p->used = q->used + 1;
-				break;
-			}
-		if (q == &net_o[net_len])
-			p->used = 1;
-	}
-
-	/*
-	 * Second step: choose best register and update used field.
-	 */
-	if (net_len > 0) {
-		if (net_o[0].used == 1)
-			used = net_o[0].used = 0;
-		else {
-			net_o[0].reg = 0;
-			used = 1;
-		}
-
-		for (p = &net_o[1]; p < &net_o[net_len]; p++) {
-			max = 0;
-			first = avail;
-			for (i = 0; i < avail; i++) {
-				q = p;
-				j = 0;
-				while (q-- != net_o)
-					if (q->used > 0 && q->reg == i) {
-						if (q->used == 1)
-							first = i;
-						j = 1;
-						break;
-					}
-				if (j == 0)
-					continue;
-
-				if (q->val == p->val) {
-					p->reg = i;
-					break;
-				}
-
-				if (p->used == 1)
-					continue;
-
-				if (first == avail && used == avail) {
-					j = 1;
-					for (q = p+1; q->val != p->val; p++)
-						j++;
-					if (j > max) {
-						max = j;
-						last = i;
-					}
-				}
-			}
-			if (i < avail)
-				continue;
-
-			if (p->used > 1) {
-				if (first != avail)
-					p->reg = first;
-				else if (used < avail)
-					p->reg = used++;
-				else
-					p->reg = last;
-			} else
-				p->used = 0;
-		}
-	}
-
-	/*
-	 * Third step: associate correct register number and keep max value.
-	 */
-	for (p = net_o; p < &net_o[net_len]; p++) {
-		if (p->used == 0)
-			continue;
-		i = first = 0;
-		for (;;) {
-			if (reg[i] != -2) {
-				if (first == p->reg) {
-					p->reg = i;
-					break;
-				}
-				first++;
-			}
-			i++;
-		}
-	}
-
-	/*
-	 * Forth step: invalidate useless registers.
-	 */
-	if (net_len == 0) {
-		for (i = 0; i < nbreg; i++)
-			if (reg[i] != -2)
-				reg[i] = -2;
-
-	} else if (used < avail) {
-		first = 0;
-		for (i = 0; i < nbreg; i++)
-			if (reg[i] != -2)
-				if (first >= used)
-					reg[i] = -2;
-				else
-					first++;
-	}
-}
diff --git a/osfmk/i386/ntoh.h b/osfmk/i386/ntoh.h
deleted file mode 100644
index 2f9e26ce2..000000000
--- a/osfmk/i386/ntoh.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-#ifndef	_MACHINE_NTOH_H_
-#define	_MACHINE_NTOH_H_
-
-#ifdef	BYTE_MSF
-#undef	BYTE_MSF	/* i386 byte order is *NOT* Most Significant First */
-#endif
-
-extern unsigned long ntohl(
-		unsigned long arg);
-
-extern unsigned long htonl(
-		unsigned long arg);
-
-extern unsigned short ntohs(
-		unsigned short arg);
-
-extern unsigned short htons(
-		unsigned short arg);
-#endif	/* _MACHINE_NTOH_H_ */
diff --git a/osfmk/i386/ntoh.s b/osfmk/i386/ntoh.s
deleted file mode 100644
index 47868223d..000000000
--- a/osfmk/i386/ntoh.s
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:37  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:38  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.6.1  1994/09/23  01:59:41  ezf
- * 	change marker to not FREE
- * 	[1994/09/22  21:25:28  ezf]
- *
- * Revision 1.1.2.2  1993/06/02  23:26:45  jeffc
- * 	Added to OSF/1 R1.3 from NMK15.0.
- * 	[1993/06/02  21:06:38  jeffc]
- * 
- * Revision 1.1  1992/09/30  02:43:08  robert
- * 	Initial revision
- * 
- * $EndLog$
- */
-/* CMU_HIST */
-/*
- * Revision 2.4  91/05/14  16:12:50  mrt
- * 	Correcting copyright
- * 
- * Revision 2.3  91/02/14  15:04:55  mrt
- * 	Changed to new Mach copyright
- * 
- * 
- * Revision 2.2  90/05/03  15:34:56  dbg
- * 	First checkin.
- * 
- * 	New a.out and coff compatible .s files.
- * 	[89/10/16            rvb]
- *
- * Revision 1.3  89/02/26  12:35:37  gm0w
- * 	Changes for cleanup.
- * 
- * 16-Feb-89  Robert Baron (rvb) at Carnegie-Mellon University
- *	Created.
- *
- */
-/* CMU_ENDHIST */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/*
- */
-
-#include <i386/asm.h>
-
-Entry(ntohl)
-ENTRY(htonl)
-	movl	4(%esp), %eax
-	rorw	$8, %ax
-	ror	$16,%eax
-	rorw	$8, %ax
-	ret
-
-
-Entry(ntohs)
-ENTRY(htons)
-	movzwl	4(%esp), %eax
-	rorw	$8, %ax
-	ret
diff --git a/osfmk/i386/pcb.c b/osfmk/i386/pcb.c
index d3cab35aa..04f150453 100644
--- a/osfmk/i386/pcb.c
+++ b/osfmk/i386/pcb.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -87,11 +93,13 @@
 #include <i386/tss.h>
 #include <i386/user_ldt.h>
 #include <i386/fpu.h>
-#include <i386/iopb_entries.h>
 #include <i386/mp_desc.h>
 #include <i386/cpu_data.h>
+#include <i386/misc_protos.h>
 #include <i386/machine_routines.h>
 
+#include <machine/commpage.h>
+
 /*
  * Maps state flavor to number of words in the state:
  */
@@ -115,28 +123,18 @@ unsigned int _MachineStateCount[] = {
 	x86_DEBUG_STATE_COUNT
 };
 
-zone_t		iss_zone32;		/* zone for 32bit saved_state area */
-zone_t		iss_zone64;		/* zone for 64bit saved_state area */
-zone_t		ids_zone32;		/* zone for 32bit debug_state area */
-zone_t		ids_zone64;		/* zone for 64bit debug_state area */
-
+zone_t		iss_zone;		/* zone for saved_state area */
+zone_t		ids_zone;		/* zone for debug_state area */
 
 /* Forward */
 
 void		act_machine_throughcall(thread_t thr_act);
-user_addr_t	get_useraddr(void);
 void		act_machine_return(int);
-void		act_machine_sv_free(thread_t, int);
 
-extern thread_t		Switch_context(
-				thread_t			old,
-				thread_continue_t	cont,
-				thread_t			new);
 extern void		Thread_continue(void);
 extern void		Load_context(
 				thread_t			thread);
 
-
 static void
 get_exception_state32(thread_t thread, x86_exception_state32_t *es);
 
@@ -258,7 +256,7 @@ set_debug_state32(thread_t thread, x86_debug_state32_t *ds)
 	ids = pcb->ids;
 
 	if (ids == NULL) {
-		ids = zalloc(ids_zone32);
+		ids = zalloc(ids_zone);
 		bzero(ids, sizeof *ids);
 
 		simple_lock(&pcb->lock);
@@ -268,7 +266,7 @@ set_debug_state32(thread_t thread, x86_debug_state32_t *ds)
 			simple_unlock(&pcb->lock);
 		} else {
 			simple_unlock(&pcb->lock);
-			zfree(ids_zone32, ids);
+			zfree(ids_zone, ids);
 		}
 	}
 
@@ -319,7 +317,7 @@ set_debug_state64(thread_t thread, x86_debug_state64_t *ds)
 	ids = pcb->ids;
 
 	if (ids == NULL) {
-		ids = zalloc(ids_zone64);
+		ids = zalloc(ids_zone);
 		bzero(ids, sizeof *ids);
 
 		simple_lock(&pcb->lock);
@@ -329,7 +327,7 @@ set_debug_state64(thread_t thread, x86_debug_state64_t *ds)
 			simple_unlock(&pcb->lock);
 		} else {
 			simple_unlock(&pcb->lock);
-			zfree(ids_zone64, ids);
+			zfree(ids_zone, ids);
 		}
 	}
 
@@ -423,8 +421,7 @@ void
 consider_machine_adjust(void)
 {
 }
-
-
+extern void *get_bsduthreadarg(thread_t th);
 
 static void
 act_machine_switch_pcb( thread_t new )
@@ -441,7 +438,6 @@ act_machine_switch_pcb( thread_t new )
 
 	if (!cpu_mode_is64bit()) {
 		x86_saved_state32_tagged_t	*hi_iss32;
-
 		/*
 		 *	Save a pointer to the top of the "kernel" stack -
 		 *	actually the place in the PCB where a trap into
@@ -450,7 +446,7 @@ act_machine_switch_pcb( thread_t new )
 		hi_iss = (vm_offset_t)((unsigned long)
 			pmap_cpu_high_map_vaddr(cpu_number(), HIGH_CPU_ISS0) |
 			((unsigned long)pcb->iss & PAGE_MASK));
-	 
+
 		cdp->cpu_hi_iss = (void *)hi_iss;
 
 		pmap_high_map(pcb->iss_pte0, HIGH_CPU_ISS0);
@@ -467,13 +463,15 @@ act_machine_switch_pcb( thread_t new )
 		*(vm_offset_t *) current_sstk() = hi_pcb_stack_top;
 
 		current_ktss()->esp0 = hi_pcb_stack_top;
-/* XXX: This check is performed against the thread save state flavor rather than the
- * task's 64-bit feature flag because of the thread/task 64-bit state divergence
- * that can arise in task_set_64bit() on x86. When that is addressed, we can
- * revert to checking the task 64 bit feature flag. The assert below is retained
- * for that reason.
- */
+
 	} else if (is_saved_state64(pcb->iss)) {
+		/*
+		 * The test above is performed against the thread save state
+		 * flavor and not task's 64-bit feature flag because of the
+		 * thread/task 64-bit state divergence that can arise in
+		 * task_set_64bit() x86: the task state is changed before
+		 * the individual thread(s).
+		 */
 	        x86_saved_state64_tagged_t	*iss64;
 		vm_offset_t			isf;
 
@@ -500,13 +498,14 @@ act_machine_switch_pcb( thread_t new )
 		 */
 		*current_sstk64() = UBER64(pcb_stack_top);
 
-		cdp->cpu_task_map = new->map->pmap->pm_kernel_cr3 ?
-					TASK_MAP_64BIT_SHARED : TASK_MAP_64BIT;
+		cdp->cpu_task_map = new->map->pmap->pm_task_map; 
 
 		/*
 		 * Enable the 64-bit user code segment, USER64_CS.
+		 * Disable the 32-bit user code segment, USER_CS.
 		 */
 		ldt_desc_p(USER64_CS)->access |= ACC_PL_U;
+		ldt_desc_p(USER_CS)->access &= ~ACC_PL_U;
 
 	} else {
 		x86_saved_state_compat32_t	*iss32compat;
@@ -533,11 +532,19 @@ act_machine_switch_pcb( thread_t new )
 		current_ktss64()->rsp0 = UBER64(pcb_stack_top);
 
 		cdp->cpu_task_map = TASK_MAP_32BIT;
+		/* Precalculate pointers to syscall argument store, for use
+		 * in the trampolines.
+		 */
+		cdp->cpu_uber_arg_store = UBER64((vm_offset_t)get_bsduthreadarg(new));
+		cdp->cpu_uber_arg_store_valid = UBER64((vm_offset_t)&pcb->arg_store_valid);
+		pcb->arg_store_valid = 0;
 
 		/*
 		 * Disable USER64_CS
+		 * Enable USER_CS
 		 */
 		ldt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
+		ldt_desc_p(USER_CS)->access |= ACC_PL_U;
 	}
 
 	/*
@@ -570,6 +577,12 @@ act_machine_switch_pcb( thread_t new )
 		 */
 		user_ldt_set(new);
 	}
+
+	/*
+	 * Bump the scheduler generation count in the commpage.
+	 * This can be read by user code to detect its preemption.
+	 */
+	commpage_sched_gen_inc();
 }
 
 /*
@@ -617,15 +630,13 @@ machine_switch_context(
 	 *	Load the rest of the user state for the new thread
 	 */
 	act_machine_switch_pcb(new);
-	KERNEL_DEBUG_CONSTANT(
-		MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
-		(int)old, (int)new, old->sched_pri, new->sched_pri, 0);
+
 	return(Switch_context(old, continuation, new));
 }
 
 /*
  * act_machine_sv_free
- * release saveareas associated with an act. if flag is true, release
+ * release saveareas associated with an act.  if flag is true, release
  * user level savearea(s) too, else don't
  */
 void
@@ -642,18 +653,18 @@ kern_return_t
 machine_thread_state_initialize(
 	thread_t thread)
 {
-        /*
-	 * If there's an fpu save area, free it.
-	 * The initialized state will then be lazily faulted-in, if required.
-	 * And if we're target, re-arm the no-fpu trap.
-	 */
+    /*
+     * If there's an fpu save area, free it.
+     * The initialized state will then be lazily faulted-in, if required.
+     * And if we're target, re-arm the no-fpu trap.
+     */
         if (thread->machine.pcb->ifps) {
-	        (void) fpu_set_fxstate(thread, NULL);
+	(void) fpu_set_fxstate(thread, NULL);
 
-		if (thread == current_thread())
-		        clear_fpu();
-	}
-	return KERN_SUCCESS;
+    	if (thread == current_thread())
+	   clear_fpu();
+    }
+    return  KERN_SUCCESS;
 }
 
 uint32_t
@@ -687,7 +698,7 @@ get_eflags_exportmask(void)
  *			   for either 32bit or 64bit tasks
  */
 
-
+ 
 static void
 get_exception_state64(thread_t thread, x86_exception_state64_t *es)
 {
@@ -785,7 +796,6 @@ set_thread_state64(thread_t thread, x86_thread_state64_t *ts)
 	saved_state->r14 = ts->r14;
 	saved_state->r15 = ts->r15;
 	saved_state->rax = ts->rax;
-	saved_state->rax = ts->rax;
 	saved_state->rbx = ts->rbx;
 	saved_state->rcx = ts->rcx;
 	saved_state->rdx = ts->rdx;
@@ -861,6 +871,69 @@ get_thread_state64(thread_t thread, x86_thread_state64_t *ts)
 }
 
 
+void
+thread_set_wq_state32(thread_t thread, thread_state_t tstate)
+{
+        x86_thread_state32_t	*state;
+        x86_saved_state32_t	*saved_state;
+	thread_t curth = current_thread();
+
+	saved_state = USER_REGS32(thread);
+	state = (x86_thread_state32_t *)tstate;
+	
+	if (curth != thread)
+	        thread_lock(thread);
+
+	saved_state->eip = state->eip;
+	saved_state->eax = state->eax;
+	saved_state->ebx = state->ebx;
+	saved_state->ecx = state->ecx;
+	saved_state->edx = state->edx;
+	saved_state->edi = state->edi;
+	saved_state->esi = state->esi;
+	saved_state->uesp = state->esp;
+	saved_state->efl = EFL_USER_SET;
+
+	saved_state->cs = USER_CS;
+	saved_state->ss = USER_DS;
+	saved_state->ds = USER_DS;
+	saved_state->es = USER_DS;
+
+	if (curth != thread)
+	        thread_unlock(thread);
+}
+
+
+void
+thread_set_wq_state64(thread_t thread, thread_state_t tstate)
+{
+        x86_thread_state64_t	*state;
+        x86_saved_state64_t	*saved_state;
+	thread_t curth = current_thread();
+
+	saved_state = USER_REGS64(thread);
+	state = (x86_thread_state64_t *)tstate;
+	
+	if (curth != thread)
+	        thread_lock(thread);
+
+	saved_state->rdi = state->rdi;
+	saved_state->rsi = state->rsi;
+	saved_state->rdx = state->rdx;
+	saved_state->rcx = state->rcx;
+	saved_state->r8  = state->r8;
+	saved_state->r9  = state->r9;
+
+	saved_state->isf.rip = state->rip;
+	saved_state->isf.rsp = state->rsp;
+	saved_state->isf.cs = USER64_CS;
+	saved_state->isf.rflags = EFL_USER_SET;
+
+	if (curth != thread)
+	        thread_unlock(thread);
+}
+
+
 
 /*
  *	act_machine_set_state:
@@ -875,28 +948,29 @@ machine_thread_set_state(
 	thread_state_t tstate,
 	mach_msg_type_number_t count)
 {
-
-	switch (flavor)
+	switch (flavor) {
+	case x86_SAVED_STATE32:
 	{
-	    case x86_SAVED_STATE32:
-	    {
 		x86_saved_state32_t	*state;
 		x86_saved_state32_t	*saved_state;
 
 		if (count < x86_SAVED_STATE32_COUNT)
-		        return(KERN_INVALID_ARGUMENT);
+			return(KERN_INVALID_ARGUMENT);
+        
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
 
 		state = (x86_saved_state32_t *) tstate;
 
 		/* Check segment selectors are safe */
 		if (!valid_user_segment_selectors(state->cs,
-						  state->ss,
-						  state->ds,
-						  state->es,
-						  state->fs,
-						  state->gs))
-		        return KERN_INVALID_ARGUMENT;
-		
+					state->ss,
+					state->ds,
+					state->es,
+					state->fs,
+					state->gs))
+			return KERN_INVALID_ARGUMENT;
+
 		saved_state = USER_REGS32(thr_act);
 
 		/*
@@ -935,15 +1009,18 @@ machine_thread_set_state(
 		saved_state->fs = state->fs;
 		saved_state->gs = state->gs;
 		break;
-	    }
+	}
 
-	    case x86_SAVED_STATE64:
-	    {
+	case x86_SAVED_STATE64:
+	{
 		x86_saved_state64_t	*state;
 		x86_saved_state64_t	*saved_state;
 
 		if (count < x86_SAVED_STATE64_COUNT)
-		        return(KERN_INVALID_ARGUMENT);
+			return(KERN_INVALID_ARGUMENT);
+
+		if (!thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
 
 		state = (x86_saved_state64_t *) tstate;
 
@@ -987,22 +1064,21 @@ machine_thread_set_state(
 
 		saved_state->isf.rflags = (state->isf.rflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
 
-	        /*
+		/*
 		 * User setting segment registers.
 		 * Code and stack selectors have already been
 		 * checked.  Others will be reset by 'sys'
 		 * if they are not valid.
 		 */
-	        saved_state->isf.cs = state->isf.cs;
+		saved_state->isf.cs = state->isf.cs;
 		saved_state->isf.ss = state->isf.ss;
 		saved_state->fs = state->fs;
 		saved_state->gs = state->gs;
-
 		break;
-	    }
+	}
 
-	    case x86_FLOAT_STATE32:
-	    {
+	case x86_FLOAT_STATE32:
+	{
 		if (count != x86_FLOAT_STATE32_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
@@ -1010,10 +1086,10 @@ machine_thread_set_state(
 			return(KERN_INVALID_ARGUMENT);
 
 		return fpu_set_fxstate(thr_act, tstate);
-	    }
+	}
 
-	    case x86_FLOAT_STATE64:
-	    {
+	case x86_FLOAT_STATE64:
+	{
 		if (count != x86_FLOAT_STATE64_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
@@ -1021,73 +1097,71 @@ machine_thread_set_state(
 			return(KERN_INVALID_ARGUMENT);
 
 		return fpu_set_fxstate(thr_act, tstate);
-	    }
+	}
 
-	    case x86_FLOAT_STATE:
-	    {
-	        x86_float_state_t	*state;
+	case x86_FLOAT_STATE:
+	{   
+		x86_float_state_t       *state;
 
 		if (count != x86_FLOAT_STATE_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
 		state = (x86_float_state_t *)tstate;
-
 		if (state->fsh.flavor == x86_FLOAT_STATE64 && state->fsh.count == x86_FLOAT_STATE64_COUNT &&
 		    thread_is_64bit(thr_act)) {
-		        return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
+			return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
 		}
 		if (state->fsh.flavor == x86_FLOAT_STATE32 && state->fsh.count == x86_FLOAT_STATE32_COUNT &&
 		    !thread_is_64bit(thr_act)) {
-		        return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs32);
+			return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs32); 
 		}
 		return(KERN_INVALID_ARGUMENT);
-	    }
-
-
+	}
 
-	    case OLD_i386_THREAD_STATE: 
-	    case x86_THREAD_STATE32: 
-	    {
+	case x86_THREAD_STATE32: 
+	{
 		if (count != x86_THREAD_STATE32_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
 		if (thread_is_64bit(thr_act))
 			return(KERN_INVALID_ARGUMENT);
-		
+
 		return set_thread_state32(thr_act, (x86_thread_state32_t *)tstate);
-	    }
+	}
 
-	    case x86_THREAD_STATE64: 
-	    {
+	case x86_THREAD_STATE64: 
+	{
 		if (count != x86_THREAD_STATE64_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
-		if ( !thread_is_64bit(thr_act))
+		if (!thread_is_64bit(thr_act))
 			return(KERN_INVALID_ARGUMENT);
 
 		return set_thread_state64(thr_act, (x86_thread_state64_t *)tstate);
-	    }
 
-	    case x86_THREAD_STATE: 
-	    {
-	        x86_thread_state_t	*state;
+	}
+	case x86_THREAD_STATE:
+	{
+		x86_thread_state_t      *state;
 
 		if (count != x86_THREAD_STATE_COUNT)
 			return(KERN_INVALID_ARGUMENT);
 
 		state = (x86_thread_state_t *)tstate;
 
-		if (state->tsh.flavor == x86_THREAD_STATE64 && state->tsh.count == x86_THREAD_STATE64_COUNT &&
+		if (state->tsh.flavor == x86_THREAD_STATE64 &&
+		    state->tsh.count == x86_THREAD_STATE64_COUNT &&
 		    thread_is_64bit(thr_act)) {
-		        return set_thread_state64(thr_act, &state->uts.ts64);
-		} else if (state->tsh.flavor == x86_THREAD_STATE32 && state->tsh.count == x86_THREAD_STATE32_COUNT &&
+			return set_thread_state64(thr_act, &state->uts.ts64);
+		} else if (state->tsh.flavor == x86_THREAD_STATE32 &&
+			   state->tsh.count == x86_THREAD_STATE32_COUNT &&
 			   !thread_is_64bit(thr_act)) {
-		        return set_thread_state32(thr_act, &state->uts.ts32);
+			return set_thread_state32(thr_act, &state->uts.ts32);
 		} else
-		        return(KERN_INVALID_ARGUMENT);
+			return(KERN_INVALID_ARGUMENT);
 
 		break;
-	    }
+	}
 	case x86_DEBUG_STATE32:
 	{
 		x86_debug_state32_t *state;
@@ -1160,6 +1234,7 @@ machine_thread_get_state(
 	thread_state_t tstate,
 	mach_msg_type_number_t *count)
 {
+
 	switch (flavor)  {
 
 	    case THREAD_STATE_FLAVOR_LIST:
@@ -1197,6 +1272,9 @@ machine_thread_get_state(
 		if (*count < x86_SAVED_STATE32_COUNT)
 		        return(KERN_INVALID_ARGUMENT);
 
+		if (thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
 		state = (x86_saved_state32_t *) tstate;
 		saved_state = USER_REGS32(thr_act);
 
@@ -1221,6 +1299,9 @@ machine_thread_get_state(
 		if (*count < x86_SAVED_STATE64_COUNT)
 		        return(KERN_INVALID_ARGUMENT);
 
+		if (!thread_is_64bit(thr_act))
+			return(KERN_INVALID_ARGUMENT);
+
 		state = (x86_saved_state64_t *)tstate;
 		saved_state = USER_REGS64(thr_act);
 
@@ -1291,8 +1372,6 @@ machine_thread_get_state(
 		return(kret);
 	    }
 
-
-	    case OLD_i386_THREAD_STATE: 
 	    case x86_THREAD_STATE32: 
 	    {
 		if (*count < x86_THREAD_STATE32_COUNT)
@@ -1456,7 +1535,7 @@ machine_thread_get_state(
 		*count = x86_DEBUG_STATE_COUNT;
 		break;
 	}
-	    default:
+	default:
 		return(KERN_INVALID_ARGUMENT);
 	}
 
@@ -1478,16 +1557,14 @@ machine_thread_get_kern_state(
 		return KERN_FAILURE;
 
 	switch(flavor) {
-        
-		case x86_THREAD_STATE32: 
+		case x86_THREAD_STATE32:
 		{
-
 			x86_thread_state32_t	*state;
 			x86_saved_state32_t	*saved_state;
 
 			if (*count < x86_THREAD_STATE32_COUNT)
-				return(KERN_INVALID_ARGUMENT);
-			
+				return(KERN_INVALID_ARGUMENT);     
+
 			state = (x86_thread_state32_t *)tstate;
 
 			assert(is_saved_state32(current_cpu_datap()->cpu_int_state));
@@ -1511,55 +1588,55 @@ machine_thread_get_kern_state(
 			state->es = saved_state->es & 0xffff;
 			state->fs = saved_state->fs & 0xffff;
 			state->gs = saved_state->gs & 0xffff;
-			
+
 			*count = x86_THREAD_STATE32_COUNT;
 
 			return KERN_SUCCESS;
 		}
-		break;	// for completeness
+		break;
 
 		case x86_THREAD_STATE:
 		{
 			// wrap a 32 bit thread state into a 32/64bit clean thread state
-			x86_thread_state_t		*state;
-			x86_saved_state32_t		*saved_state;
-
-			if(*count < x86_THREAD_STATE_COUNT)
-				return (KERN_INVALID_ARGUMENT);
-
-			state = (x86_thread_state_t *)tstate;
-			assert(is_saved_state32(current_cpu_datap()->cpu_int_state));
-			saved_state = saved_state32(current_cpu_datap()->cpu_int_state);
-
-			state->tsh.flavor = x86_THREAD_STATE32;
-			state->tsh.count = x86_THREAD_STATE32_COUNT;
-			
-			/* 
-			 * General registers.
-			 */
-
-			state->uts.ts32.eax = saved_state->eax;
-			state->uts.ts32.ebx = saved_state->ebx;
-			state->uts.ts32.ecx = saved_state->ecx;
-			state->uts.ts32.edx = saved_state->edx;
-			state->uts.ts32.edi = saved_state->edi;
-			state->uts.ts32.esi = saved_state->esi;
-			state->uts.ts32.ebp = saved_state->ebp;
-			state->uts.ts32.esp = saved_state->uesp;
-			state->uts.ts32.eflags = saved_state->efl;
-			state->uts.ts32.eip = saved_state->eip;
-			state->uts.ts32.cs = saved_state->cs;
-			state->uts.ts32.ss = saved_state->ss;
-			state->uts.ts32.ds = saved_state->ds & 0xffff;
-			state->uts.ts32.es = saved_state->es & 0xffff;
-			state->uts.ts32.fs = saved_state->fs & 0xffff;
-			state->uts.ts32.gs = saved_state->gs & 0xffff;
-	
-			*count = x86_THREAD_STATE_COUNT;
-			return KERN_SUCCESS;
+            x86_thread_state_t      *state;
+            x86_saved_state32_t     *saved_state;
+
+            if(*count < x86_THREAD_STATE_COUNT)
+                return (KERN_INVALID_ARGUMENT);
+
+            state = (x86_thread_state_t *)tstate;
+            assert(is_saved_state32(current_cpu_datap()->cpu_int_state));
+            saved_state = saved_state32(current_cpu_datap()->cpu_int_state);
+
+            state->tsh.flavor = x86_THREAD_STATE32;
+            state->tsh.count = x86_THREAD_STATE32_COUNT;
+
+            /* 
+             * General registers.
+             */
+
+            state->uts.ts32.eax = saved_state->eax;
+            state->uts.ts32.ebx = saved_state->ebx;
+            state->uts.ts32.ecx = saved_state->ecx;
+            state->uts.ts32.edx = saved_state->edx;
+            state->uts.ts32.edi = saved_state->edi;
+            state->uts.ts32.esi = saved_state->esi;
+            state->uts.ts32.ebp = saved_state->ebp;
+            state->uts.ts32.esp = saved_state->uesp;
+            state->uts.ts32.eflags = saved_state->efl;
+            state->uts.ts32.eip = saved_state->eip;
+            state->uts.ts32.cs = saved_state->cs;
+            state->uts.ts32.ss = saved_state->ss;
+            state->uts.ts32.ds = saved_state->ds & 0xffff;
+            state->uts.ts32.es = saved_state->es & 0xffff;
+            state->uts.ts32.fs = saved_state->fs & 0xffff;
+            state->uts.ts32.gs = saved_state->gs & 0xffff;
+
+            *count = x86_THREAD_STATE_COUNT;
+            return KERN_SUCCESS;
 		}
 		break;
-	}	
+	}
 	return KERN_FAILURE;
 }
 
@@ -1582,22 +1659,27 @@ machine_thread_create(
 	thread->machine.physwindow_pte = 0;
 	thread->machine.physwindow_busy = 0;
 
+	/*
+	 * Allocate pcb only if required.
+	 */
+	if (pcb->sf == NULL) {
+		pcb->sf = zalloc(iss_zone);
+		if (pcb->sf == NULL)
+			panic("iss_zone");
+	}
+
         if (task_has_64BitAddr(task)) {
 		x86_sframe64_t		*sf64;
 
-	        sf64 = (x86_sframe64_t *)zalloc(iss_zone64);
-
-		if (sf64 == NULL)
-		        panic("iss_zone64");
-		pcb->sf = (void *)sf64;
+		sf64 = (x86_sframe64_t *) pcb->sf;
 
 		bzero((char *)sf64, sizeof(x86_sframe64_t));
 
 		iss = (x86_saved_state_t *) &sf64->ssf;
 		iss->flavor = x86_SAVED_STATE64;
 		/*
-		 *	Guarantee that the bootstrapped thread will be in user
-		 *	mode.
+		 *      Guarantee that the bootstrapped thread will be in user
+		 *      mode.
 		 */
 		iss->ss_64.isf.rflags = EFL_USER_SET;
 		iss->ss_64.isf.cs = USER64_CS;
@@ -1606,12 +1688,9 @@ machine_thread_create(
 		iss->ss_64.gs = USER_DS;
 	} else {
 		if (cpu_mode_is64bit()) {
-			x86_sframe_compat32_t	   *sfc32;
+			x86_sframe_compat32_t      *sfc32;
 
-			sfc32 = (x86_sframe_compat32_t *)zalloc(iss_zone32);
-			if (sfc32 == NULL)
-			        panic("iss_zone32");
-			pcb->sf = (void *)sfc32;
+			sfc32 = (x86_sframe_compat32_t *)pcb->sf;
 
 			bzero((char *)sfc32, sizeof(x86_sframe_compat32_t));
 
@@ -1619,21 +1698,17 @@ machine_thread_create(
 			iss->flavor = x86_SAVED_STATE32;
 #if DEBUG
 			{
-			x86_saved_state_compat32_t *xssc;
+				x86_saved_state_compat32_t *xssc;
 
-			xssc  = (x86_saved_state_compat32_t *) iss;
-			xssc->pad_for_16byte_alignment[0] = 0x64326432;
-			xssc->pad_for_16byte_alignment[1] = 0x64326432;
+				xssc  = (x86_saved_state_compat32_t *) iss;
+				xssc->pad_for_16byte_alignment[0] = 0x64326432;
+				xssc->pad_for_16byte_alignment[1] = 0x64326432;
 			}
 #endif
 		} else {
-		        x86_sframe32_t	*sf32;
+			x86_sframe32_t  *sf32;
 
-			sf32 = (x86_sframe32_t *)zalloc(iss_zone32);
-
-			if (sf32 == NULL)
-			        panic("iss_zone32");
-			pcb->sf = (void *)sf32;
+			sf32 = (x86_sframe32_t *) pcb->sf;
 
 			bzero((char *)sf32, sizeof(x86_sframe32_t));
 
@@ -1641,8 +1716,8 @@ machine_thread_create(
 			iss->flavor = x86_SAVED_STATE32;
 		}
 		/*
-		 *	Guarantee that the bootstrapped thread will be in user
-		 *	mode.
+		 *      Guarantee that the bootstrapped thread will be in user
+		 *      mode.
 		 */
 		iss->ss_32.cs = USER_CS;
 		iss->ss_32.ss = USER_DS;
@@ -1651,6 +1726,7 @@ machine_thread_create(
 		iss->ss_32.fs = USER_DS;
 		iss->ss_32.gs = USER_DS;
 		iss->ss_32.efl = EFL_USER_SET;
+
 	}
 	pcb->iss = iss;
 
@@ -1663,6 +1739,7 @@ machine_thread_create(
 	pcb->uldt_selector = 0;
 
 	pcb->iss_pte0 = (uint64_t)pte_kernel_rw(kvtophys((vm_offset_t)pcb->iss));
+	pcb->arg_store_valid = 0;
 
 	if (0 == (paddr = pa_to_pte(kvtophys((vm_offset_t)(pcb->iss) + PAGE_SIZE))))
 	        pcb->iss_pte1 = INTEL_PTE_INVALID;
@@ -1686,40 +1763,36 @@ machine_thread_destroy(
 	if (pcb->ifps != 0)
 		fpu_free(pcb->ifps);
 	if (pcb->sf != 0) {
-	        if (thread_is_64bit(thread))
-		        zfree(iss_zone64, pcb->sf);
-		else
-		        zfree(iss_zone32, pcb->sf);
+		zfree(iss_zone, pcb->sf);
 		pcb->sf = 0;
 	}
 	if (pcb->ids) {
-	        if (thread_is_64bit(thread))
-		        zfree(ids_zone64, pcb->ids);
-		else
-		        zfree(ids_zone32, pcb->ids);
+		zfree(ids_zone, pcb->ids);
+		pcb->ids = NULL;
 	}
 	thread->machine.pcb = (pcb_t)0;
 
 }
 
 void
-machine_thread_switch_addrmode(thread_t thread, int oldmode_is64bit)
+machine_thread_switch_addrmode(thread_t thread)
 {
-	register pcb_t	pcb = thread->machine.pcb;
+	/*
+	 * We don't want to be preempted until we're done
+	 * - particularly if we're switching the current thread
+	 */
+	disable_preemption();
 
-        assert(pcb);
-	
-	if (pcb->sf != 0) {
-	        if (oldmode_is64bit)
-		        zfree(iss_zone64, pcb->sf);
-		else
-		        zfree(iss_zone32, pcb->sf);
-	}
+	/*
+	 * Reset the state saveareas.
+	 */
 	machine_thread_create(thread, thread->task);
 
 	/* If we're switching ourselves, reset the pcb addresses etc. */
 	if (thread == current_thread())
 		act_machine_switch_pcb(thread);
+
+	enable_preemption();
 }
 
 
@@ -1751,7 +1824,13 @@ machine_thread_terminate_self(void)
 }
 
 void
-act_machine_return(int code)
+act_machine_return(
+#if CONFIG_NO_PANIC_STRINGS
+		__unused int code
+#else
+		int code
+#endif
+		)
 {
 	/*
 	 * This code is called with nothing locked.
@@ -1778,41 +1857,32 @@ act_machine_return(int code)
 void
 machine_thread_init(void)
 {
-        if (cpu_mode_is64bit()) {
-	        iss_zone64 = zinit(sizeof(x86_sframe64_t),
-				   THREAD_MAX * sizeof(x86_sframe64_t),
-				   THREAD_CHUNK * sizeof(x86_sframe64_t),
-				   "x86_64 saved state");
-
+	if (cpu_mode_is64bit()) {
 		assert(sizeof(x86_sframe_compat32_t) % 16 == 0);
-	        iss_zone32 = zinit(sizeof(x86_sframe_compat32_t),
-				   THREAD_MAX * sizeof(x86_sframe_compat32_t),
-				   THREAD_CHUNK * sizeof(x86_sframe_compat32_t),
-				   "x86_32 saved state");
-
-	        ids_zone32 = zinit(sizeof(x86_debug_state32_t),
-				   THREAD_MAX * (sizeof(x86_debug_state32_t)),
-				   THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
-				   "x86_32 debug state");
-	        ids_zone64 = zinit(sizeof(x86_debug_state64_t),
-				   THREAD_MAX * sizeof(x86_debug_state64_t),
-				   THREAD_CHUNK * sizeof(x86_debug_state64_t),
-				   "x86_64 debug state");
+		iss_zone = zinit(sizeof(x86_sframe64_t),
+				THREAD_MAX * sizeof(x86_sframe64_t),
+				THREAD_CHUNK * sizeof(x86_sframe64_t),
+				"x86_64 saved state");
+
+	        ids_zone = zinit(sizeof(x86_debug_state64_t),
+				 THREAD_MAX * sizeof(x86_debug_state64_t),
+				 THREAD_CHUNK * sizeof(x86_debug_state64_t),
+				 "x86_64 debug state");
 
 	} else {
-	        iss_zone32 = zinit(sizeof(x86_sframe32_t),
-				   THREAD_MAX * sizeof(x86_sframe32_t),
-				   THREAD_CHUNK * sizeof(x86_sframe32_t),
-				   "x86 saved state");
-	        ids_zone32 = zinit(sizeof(x86_debug_state32_t),
-				   THREAD_MAX * (sizeof(x86_debug_state32_t)),
-				   THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
-				   "x86 debug state");
+		iss_zone = zinit(sizeof(x86_sframe32_t),
+				THREAD_MAX * sizeof(x86_sframe32_t),
+				THREAD_CHUNK * sizeof(x86_sframe32_t),
+				"x86 saved state");
+	        ids_zone = zinit(sizeof(x86_debug_state32_t),
+				THREAD_MAX * (sizeof(x86_debug_state32_t)),
+				THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
+				"x86 debug state");
 	}
 	fpu_module_init();
-	iopb_init();
 }
 
+
 /*
  * Some routines for debugging activation code
  */
@@ -1823,42 +1893,42 @@ int		dump_act(thread_t thr_act);
 static void
 dump_handlers(thread_t thr_act)
 {
-    ReturnHandler *rhp = thr_act->handlers;
-    int	counter = 0;
-
-    printf("\t");
-    while (rhp) {
-	if (rhp == &thr_act->special_handler){
-	    if (rhp->next)
-		printf("[NON-Zero next ptr(%x)]", rhp->next);
-	    printf("special_handler()->");
-	    break;
-	}
-	printf("hdlr_%d(%x)->",counter,rhp->handler);
-	rhp = rhp->next;
-	if (++counter > 32) {
-		printf("Aborting: HUGE handler chain\n");
-		break;
+	ReturnHandler *rhp = thr_act->handlers;
+	int	counter = 0;
+
+	printf("\t");
+	while (rhp) {
+		if (rhp == &thr_act->special_handler){
+			if (rhp->next)
+				printf("[NON-Zero next ptr(%p)]", rhp->next);
+			printf("special_handler()->");
+			break;
+		}
+		printf("hdlr_%d(%p)->", counter, rhp->handler);
+		rhp = rhp->next;
+		if (++counter > 32) {
+			printf("Aborting: HUGE handler chain\n");
+			break;
+		}
 	}
-    }
-    printf("HLDR_NULL\n");
+	printf("HLDR_NULL\n");
 }
 
 void
 dump_regs(thread_t thr_act)
 {
-        if (thr_act->machine.pcb == NULL)
-	        return;
+	if (thr_act->machine.pcb == NULL)
+		return;
 
-        if (thread_is_64bit(thr_act)) {
-	        x86_saved_state64_t	*ssp;
+	if (thread_is_64bit(thr_act)) {
+		x86_saved_state64_t	*ssp;
 
 		ssp = USER_REGS64(thr_act);
 
 		panic("dump_regs: 64bit tasks not yet supported");
 
 	} else {
-	        x86_saved_state32_t	*ssp;
+		x86_saved_state32_t	*ssp;
 
 		ssp = USER_REGS32(thr_act);
 
@@ -1866,10 +1936,10 @@ dump_regs(thread_t thr_act)
 		 * Print out user register state
 		 */
 		printf("\tRegs:\tedi=%x esi=%x ebp=%x ebx=%x edx=%x\n",
-		    ssp->edi, ssp->esi, ssp->ebp, ssp->ebx, ssp->edx);
+			ssp->edi, ssp->esi, ssp->ebp, ssp->ebx, ssp->edx);
 
 		printf("\t\tecx=%x eax=%x eip=%x efl=%x uesp=%x\n",
-		    ssp->ecx, ssp->eax, ssp->eip, ssp->efl, ssp->uesp);
+			ssp->ecx, ssp->eax, ssp->eip, ssp->efl, ssp->uesp);
 
 		printf("\t\tcs=%x ss=%x\n", ssp->cs, ssp->ss);
 	}
@@ -1881,21 +1951,22 @@ dump_act(thread_t thr_act)
 	if (!thr_act)
 		return(0);
 
-	printf("thread(0x%x)(%d): task=%x(%d)\n",
-	       thr_act, thr_act->ref_count,
-	       thr_act->task,   thr_act->task   ? thr_act->task->ref_count : 0);
+	printf("thread(%p)(%d): task=%p(%d)\n",
+			thr_act, thr_act->ref_count,
+			thr_act->task,
+			thr_act->task   ? thr_act->task->ref_count : 0);
 
 	printf("\tsusp=%d user_stop=%d active=%x ast=%x\n",
-		       thr_act->suspend_count, thr_act->user_stop_count,
-		       thr_act->active, thr_act->ast);
-	printf("\tpcb=%x\n", thr_act->machine.pcb);
+			thr_act->suspend_count, thr_act->user_stop_count,
+			thr_act->active, thr_act->ast);
+	printf("\tpcb=%p\n", thr_act->machine.pcb);
 
 	if (thr_act->kernel_stack) {
-	    vm_offset_t stack = thr_act->kernel_stack;
+		vm_offset_t stack = thr_act->kernel_stack;
 
-	    printf("\tk_stk %x  eip %x ebx %x esp %x iss %x\n",
-		stack, STACK_IKS(stack)->k_eip, STACK_IKS(stack)->k_ebx,
-		STACK_IKS(stack)->k_esp, STACK_IEL(stack)->saved_state);
+		printf("\tk_stk %x  eip %x ebx %x esp %x iss %p\n",
+			stack, STACK_IKS(stack)->k_eip, STACK_IKS(stack)->k_ebx,
+			STACK_IKS(stack)->k_esp, STACK_IEL(stack)->saved_state);
 	}
 
 	dump_handlers(thr_act);
@@ -1909,7 +1980,7 @@ get_useraddr(void)
         thread_t thr_act = current_thread();
  
 	if (thr_act->machine.pcb == NULL) 
-	        return (0);
+		return(0);
 
         if (thread_is_64bit(thr_act)) {
 	        x86_saved_state64_t	*iss64;
@@ -2006,10 +2077,6 @@ machine_stack_handoff(thread_t old,
 	PMAP_SWITCH_CONTEXT(old, new, cpu_number());
 	act_machine_switch_pcb(new);
 
-	KERNEL_DEBUG_CONSTANT(
-		MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_HANDOFF)|DBG_FUNC_NONE,
-		old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
-
 	machine_set_current_thread(new);
 
 	return;
@@ -2035,31 +2102,31 @@ struct x86_act_context64 {
 void *
 act_thread_csave(void)
 {
-        kern_return_t kret;
+	kern_return_t kret;
 	mach_msg_type_number_t val;
-        thread_t thr_act = current_thread();
+	thread_t thr_act = current_thread();
 
-        if (thread_is_64bit(thr_act)) {
-	        struct x86_act_context64 *ic64;
+	if (thread_is_64bit(thr_act)) {
+		struct x86_act_context64 *ic64;
 
-	        ic64 = (struct x86_act_context64 *)kalloc(sizeof(struct x86_act_context64));
+		ic64 = (struct x86_act_context64 *)kalloc(sizeof(struct x86_act_context64));
 
 		if (ic64 == (struct x86_act_context64 *)NULL)
-		        return((void *)0);
+			return((void *)0);
 
 		val = x86_SAVED_STATE64_COUNT; 
 		kret = machine_thread_get_state(thr_act, x86_SAVED_STATE64,
-						(thread_state_t) &ic64->ss, &val);
+				(thread_state_t) &ic64->ss, &val);
 		if (kret != KERN_SUCCESS) {
-		        kfree(ic64, sizeof(struct x86_act_context64));
+			kfree(ic64, sizeof(struct x86_act_context64));
 			return((void *)0);
 		}
 		val = x86_FLOAT_STATE64_COUNT; 
 		kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE64,
-						(thread_state_t) &ic64->fs, &val);
+				(thread_state_t) &ic64->fs, &val);
 
 		if (kret != KERN_SUCCESS) {
-		        kfree(ic64, sizeof(struct x86_act_context64));
+			kfree(ic64, sizeof(struct x86_act_context64));
 			return((void *)0);
 		}
 
@@ -2075,25 +2142,25 @@ act_thread_csave(void)
 		return(ic64);
 
 	} else {
-	        struct x86_act_context32 *ic32;
+		struct x86_act_context32 *ic32;
 
-	        ic32 = (struct x86_act_context32 *)kalloc(sizeof(struct x86_act_context32));
+		ic32 = (struct x86_act_context32 *)kalloc(sizeof(struct x86_act_context32));
 
 		if (ic32 == (struct x86_act_context32 *)NULL)
-		        return((void *)0);
+			return((void *)0);
 
 		val = x86_SAVED_STATE32_COUNT; 
 		kret = machine_thread_get_state(thr_act, x86_SAVED_STATE32,
-						(thread_state_t) &ic32->ss, &val);
+				(thread_state_t) &ic32->ss, &val);
 		if (kret != KERN_SUCCESS) {
-		        kfree(ic32, sizeof(struct x86_act_context32));
+			kfree(ic32, sizeof(struct x86_act_context32));
 			return((void *)0);
 		}
 		val = x86_FLOAT_STATE32_COUNT; 
 		kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE32,
-						(thread_state_t) &ic32->fs, &val);
+				(thread_state_t) &ic32->fs, &val);
 		if (kret != KERN_SUCCESS) {
-		        kfree(ic32, sizeof(struct x86_act_context32));
+			kfree(ic32, sizeof(struct x86_act_context32));
 			return((void *)0);
 		}
 
@@ -2118,7 +2185,7 @@ act_thread_catt(void *ctx)
 	kern_return_t kret;
 
 	if (ctx == (void *)NULL)
-	        return;
+				return;
 
         if (thread_is_64bit(thr_act)) {
 	        struct x86_act_context64 *ic64;
@@ -2128,8 +2195,8 @@ act_thread_catt(void *ctx)
 		kret = machine_thread_set_state(thr_act, x86_SAVED_STATE64,
 						(thread_state_t) &ic64->ss, x86_SAVED_STATE64_COUNT);
 		if (kret == KERN_SUCCESS) {
-		        machine_thread_set_state(thr_act, x86_FLOAT_STATE64,
-						 (thread_state_t) &ic64->fs, x86_FLOAT_STATE64_COUNT);
+			        machine_thread_set_state(thr_act, x86_FLOAT_STATE64,
+							 (thread_state_t) &ic64->fs, x86_FLOAT_STATE64_COUNT);
 		}
 		kfree(ic64, sizeof(struct x86_act_context64));
 	} else {
@@ -2157,3 +2224,13 @@ void act_thread_cfree(__unused void *ctx)
 {
 	/* XXX - Unused */
 }
+void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid);
+void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid) {
+	thread->machine.pcb->arg_store_valid = valid;
+}
+
+boolean_t x86_sysenter_arg_store_isvalid(thread_t thread);
+
+boolean_t x86_sysenter_arg_store_isvalid(thread_t thread) {
+	return (thread->machine.pcb->arg_store_valid);
+}
diff --git a/osfmk/i386/perfmon.c b/osfmk/i386/perfmon.c
index bf8172d46..61e107963 100644
--- a/osfmk/i386/perfmon.c
+++ b/osfmk/i386/perfmon.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/std_types.h>
@@ -38,9 +44,10 @@
 #define	DBG(x...)
 #endif
 
-decl_simple_lock_data(,pmc_lock)
-static task_t	pmc_owner = TASK_NULL;
-static int	pmc_thread_count = 0;
+static decl_simple_lock_data(,pmc_lock)
+static task_t		pmc_owner = TASK_NULL;
+static int		pmc_thread_count = 0;
+static boolean_t	pmc_inited = FALSE;
 
 /* PMC Facility Owner:
  * TASK_NULL - no one owns it
@@ -200,13 +207,28 @@ typedef struct {
 	pmc_machine_t	machine_type;		/* P6 or P4/Xeon */
 	uint32_t	msr_counter_base;	/* First counter MSR */
 	uint32_t	msr_control_base;	/* First control MSR */
-	boolean_t	reserved[18];		/* Max-sized arrays... */
-	pmc_ovf_func_t	*ovf_func[18];
+	union {
+	    struct {
+		boolean_t	reserved[2];
+		pmc_ovf_func_t	*ovf_func[2];
+	    } P6;
+	    struct {
+		boolean_t	reserved[2];
+		pmc_ovf_func_t	*ovf_func[2];
+		uint32_t	msr_global_ctrl;
+		uint32_t	msr_global_ovf_ctrl;
+		uint32_t	msr_global_status;
+	    } Core;
+	    struct {
+		boolean_t	reserved[18];
+		pmc_ovf_func_t	*ovf_func[18];
 #ifdef DEBUG
-	pmc_cccr_t	cccr_shadow[18];	/* Last cccr values set */
-	pmc_counter_t	counter_shadow[18];	/* Last counter values set */
-	uint32_t	ovfs_unexpected[18];	/* Count of unexpected intrs */
+		pmc_cccr_t	cccr_shadow[18];	/* Last cccr set */
+		pmc_counter_t	counter_shadow[18];	/* Last counter set */
+		uint32_t	ovfs_unexpected[18];	/* Unexpected intrs */
 #endif
+	    } P4;
+	};
 } pmc_table_t;
 
 static pmc_machine_t
@@ -222,7 +244,12 @@ _pmc_machine_type(void)
 
 	switch (infop->cpuid_family) {
 	case 0x6:
-		return pmc_P6;
+		switch (infop->cpuid_model) {
+		case 15:
+			return pmc_Core;
+		default:
+			return pmc_P6;
+		}
 	case 0xf:
 		return pmc_P4_Xeon;
 	default:
@@ -233,7 +260,7 @@ _pmc_machine_type(void)
 static void
 pmc_p4_intr(void *state)
 {
-	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
+	pmc_table_t	*pmc_table = (pmc_table_t *) x86_core()->pmc;
 	uint32_t	cccr_addr;
 	pmc_cccr_t	cccr;
 	pmc_id_t	id;
@@ -244,28 +271,28 @@ pmc_p4_intr(void *state)
 	 * with a registered overflow function.
 	 */
 	for (id = 0; id <= pmc_table->id_max; id++) {
-		if (!pmc_table->reserved[id])
+		if (!pmc_table->P4.reserved[id])
 			continue;
 		cccr_addr = pmc_table->msr_control_base + id;
 		cccr.u_u64 = rdmsr64(cccr_addr);
 #ifdef DEBUG
-		pmc_table->cccr_shadow[id] = cccr;
-		*((uint64_t *) &pmc_table->counter_shadow[id]) =
+		pmc_table->P4.cccr_shadow[id] = cccr;
+		pmc_table->P4.counter_shadow[id].u64 =
 			rdmsr64(pmc_table->msr_counter_base + id);
 #endif
 		if (cccr.u_htt.ovf == 0)
 			continue;
 		if ((cccr.u_htt.ovf_pmi_t0 == 1 && my_logical_cpu == 0) ||
 		    (cccr.u_htt.ovf_pmi_t1 == 1 && my_logical_cpu == 1)) {
-			if (pmc_table->ovf_func[id]) {
-				(*pmc_table->ovf_func[id])(id, state);
+			if (pmc_table->P4.ovf_func[id]) {
+				(*pmc_table->P4.ovf_func[id])(id, state);
 				/* func expected to clear overflow */
 				continue;
 			}
 		}
 		/* Clear overflow for unexpected interrupt */
 #ifdef DEBUG
-		pmc_table->ovfs_unexpected[id]++;
+		pmc_table->P4.ovfs_unexpected[id]++;
 #endif
 	}
 }
@@ -273,7 +300,7 @@ pmc_p4_intr(void *state)
 static void
 pmc_p6_intr(void *state)
 {
-	pmc_table_t	*pmc_table = (pmc_table_t *) cpu_core()->pmc;
+	pmc_table_t	*pmc_table = (pmc_table_t *) x86_core()->pmc;
 	pmc_id_t	id;
 
 	/*
@@ -281,8 +308,34 @@ pmc_p6_intr(void *state)
 	 * so call all registered functions.
 	 */
 	for (id = 0; id <= pmc_table->id_max; id++)
-		if (pmc_table->reserved[id] && pmc_table->ovf_func[id])
-			(*pmc_table->ovf_func[id])(id, state);
+		if (pmc_table->P6.reserved[id] && pmc_table->P6.ovf_func[id])
+			(*pmc_table->P6.ovf_func[id])(id, state);
+}
+
+static void
+pmc_core_intr(void *state)
+{
+	pmc_table_t	*pmc_table = (pmc_table_t *) x86_core()->pmc;
+	pmc_id_t	id;
+	pmc_global_status_t	ovf_status;
+
+	ovf_status.u64 = rdmsr64(pmc_table->Core.msr_global_status);
+	/*
+	 * Scan through table for reserved counters with overflow and
+	 * with a registered overflow function.
+	 */
+	for (id = 0; id <= pmc_table->id_max; id++) {
+		if (!pmc_table->Core.reserved[id])
+			continue;
+		if ((id == 0 && ovf_status.fld.PMC0_overflow) ||
+		    (id == 1 && ovf_status.fld.PMC1_overflow)) {
+			if (pmc_table->Core.ovf_func[id]) {
+				(*pmc_table->Core.ovf_func[id])(id, state);
+				/* func expected to clear overflow */
+				continue;
+			}
+		}
+	}
 }
 
 void *
@@ -292,34 +345,52 @@ pmc_alloc(void)
 	pmc_table_t	*pmc_table;
 	pmc_machine_t	pmc_type;
 
+	if (!pmc_inited) {
+		simple_lock_init(&pmc_lock, 0);
+		pmc_inited = TRUE;
+	}
+
 	pmc_type = _pmc_machine_type();
 	if (pmc_type == pmc_none) {
 		return NULL;
 	}
 	
-		ret = kmem_alloc(kernel_map,
-			(void *) &pmc_table, sizeof(pmc_table_t));
-		if (ret != KERN_SUCCESS)
-			panic("pmc_init() kmem_alloc returned %d\n", ret);
-		bzero((void *)pmc_table, sizeof(pmc_table_t));
-
-		pmc_table->machine_type = pmc_type;
-		switch (pmc_type) {
-		case pmc_P4_Xeon:
-			pmc_table->id_max = 17;
-			pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
-			pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
-			lapic_set_pmi_func(&pmc_p4_intr);
-			break;
-		case pmc_P6:
-			pmc_table->id_max = 1;
-			pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
-			pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
-			lapic_set_pmi_func(&pmc_p6_intr);
-			break;
-		default:
-			break;
-		}
+	ret = kmem_alloc(kernel_map,
+		(void *) &pmc_table, sizeof(pmc_table_t));
+	if (ret != KERN_SUCCESS)
+		panic("pmc_init() kmem_alloc returned %d\n", ret);
+	bzero((void *)pmc_table, sizeof(pmc_table_t));
+
+	pmc_table->machine_type = pmc_type;
+	switch (pmc_type) {
+	case pmc_P4_Xeon:
+		pmc_table->id_max = 17;
+		pmc_table->msr_counter_base = MSR_COUNTER_ADDR(0);
+		pmc_table->msr_control_base = MSR_CCCR_ADDR(0);
+		lapic_set_pmi_func(&pmc_p4_intr);
+		break;
+	case pmc_Core:
+		pmc_table->id_max = 1;
+		pmc_table->msr_counter_base = MSR_IA32_PMC(0);
+		pmc_table->msr_control_base = MSR_IA32_PERFEVTSEL(0);
+		pmc_table->Core.msr_global_ctrl = MSR_PERF_GLOBAL_CTRL;
+		pmc_table->Core.msr_global_ovf_ctrl = MSR_PERF_GLOBAL_OVF_CTRL;
+		pmc_table->Core.msr_global_status = MSR_PERF_GLOBAL_STATUS;
+		lapic_set_pmi_func(&pmc_core_intr);
+		break;
+	case pmc_P6:
+		pmc_table->id_max = 1;
+		pmc_table->msr_counter_base = MSR_P6_COUNTER_ADDR(0);
+		pmc_table->msr_control_base = MSR_P6_PES_ADDR(0);
+		lapic_set_pmi_func(&pmc_p6_intr);
+		break;
+	default:
+		break;
+	}
+	DBG("pmc_alloc() type=%d msr_counter_base=%p msr_control_base=%p\n",
+		pmc_table->machine_type,
+	(void *) pmc_table->msr_counter_base,
+	(void *) pmc_table->msr_control_base);
 	return (void *) pmc_table;
 }
 
@@ -327,24 +398,28 @@ pmc_alloc(void)
 static inline pmc_table_t *
 pmc_table_valid(pmc_id_t id)
 {
-	cpu_core_t	*my_core = cpu_core();
-	pmc_table_t	*pmc_table;
+	x86_core_t	*my_core = x86_core();
+	pmc_table_t	*pmc;
 
-	assert(my_core);
+	assert(my_core != NULL);
 	
-	pmc_table = (pmc_table_t *) my_core->pmc;
-	return (pmc_table == NULL ||
-		id > pmc_table->id_max ||
-		!pmc_table->reserved[id]) ? NULL : pmc_table;
+	pmc = (pmc_table_t *) my_core->pmc;
+	if ((pmc == NULL) ||
+	    (id > pmc->id_max) ||
+	    (pmc->machine_type == pmc_P4_Xeon && !pmc->P4.reserved[id]) ||
+	    (pmc->machine_type == pmc_P6      && !pmc->P6.reserved[id]) ||
+	    (pmc->machine_type == pmc_Core    && !pmc->Core.reserved[id]))
+		return NULL;
+	return pmc;
 }
 
 int
 pmc_machine_type(pmc_machine_t *type)
 {
-	cpu_core_t	*my_core = cpu_core();
+	x86_core_t	*my_core = x86_core();
 	pmc_table_t	*pmc_table;
 
-	assert(my_core);
+	assert(my_core != NULL);
 
 	pmc_table = (pmc_table_t *) my_core->pmc;
 	if (pmc_table == NULL)
@@ -358,22 +433,42 @@ pmc_machine_type(pmc_machine_t *type)
 int
 pmc_reserve(pmc_id_t id)
 {
-	cpu_core_t	*my_core = cpu_core();
+	x86_core_t	*my_core = x86_core();
 	pmc_table_t	*pmc_table;
 
-	assert(my_core);
+	assert(my_core != NULL);
 
 	pmc_table = (pmc_table_t *) my_core->pmc;
 	if (pmc_table == NULL)
 		return KERN_FAILURE;
 	if (id > pmc_table->id_max)
 		return KERN_INVALID_ARGUMENT;
-	if (pmc_table->reserved[id])
+	switch (pmc_table->machine_type) {
+	case pmc_P4_Xeon:
+		if (pmc_table->P4.reserved[id])
+			return KERN_FAILURE;
+		pmc_table->P4.reserved[id] = TRUE;
+		return KERN_SUCCESS;
+	case pmc_P6:
+		if (pmc_table->P6.reserved[id])
+			return KERN_FAILURE;
+		pmc_table->P6.reserved[id] = TRUE;
+		return KERN_SUCCESS;
+	case pmc_Core:
+		if (pmc_table->Core.reserved[id])
+			return KERN_FAILURE;
+		pmc_table->Core.reserved[id] = TRUE;
+		pmc_global_ctrl_t ctrl;
+		ctrl.u64 = rdmsr64(pmc_table->Core.msr_global_ctrl);
+		if (id == 0)
+			ctrl.fld.PMC0_enable = 1;
+		else
+			ctrl.fld.PMC1_enable = 1;
+		wrmsr64(pmc_table->Core.msr_global_ctrl, ctrl.u64);
+		return KERN_SUCCESS;
+	default:
 		return KERN_FAILURE;
-
-	pmc_table->reserved[id] = TRUE;
-
-	return KERN_SUCCESS;
+	}
 }
 
 boolean_t
@@ -391,8 +486,29 @@ pmc_free(pmc_id_t id)
 		return KERN_INVALID_ARGUMENT;
 
 	pmc_cccr_write(id, 0x0ULL);
-	pmc_table->reserved[id] = FALSE;
-	pmc_table->ovf_func[id] = NULL;
+	switch (pmc_table->machine_type) {
+	case pmc_P4_Xeon:
+		pmc_table->P4.reserved[id] = FALSE;
+		pmc_table->P4.ovf_func[id] = NULL;
+		break;
+	case pmc_P6:
+		pmc_table->P6.reserved[id] = FALSE;
+		pmc_table->P6.ovf_func[id] = NULL;
+		break;
+	case pmc_Core:
+		pmc_table->Core.reserved[id] = FALSE;
+		pmc_table->Core.ovf_func[id] = NULL;
+		pmc_global_ctrl_t ctrl;
+		ctrl.u64 = rdmsr64(pmc_table->Core.msr_global_ctrl);
+		if (id == 0)
+			ctrl.fld.PMC0_enable = 0;
+		else
+			ctrl.fld.PMC1_enable = 0;
+		wrmsr64(pmc_table->Core.msr_global_ctrl, ctrl.u64);
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
 
 	return KERN_SUCCESS;
 }
@@ -463,10 +579,11 @@ pmc_evtsel_read(pmc_id_t id, pmc_evtsel_t *evtsel)
 	if (pmc_table == NULL)
 		return KERN_INVALID_ARGUMENT;
 	
-	if (pmc_table->machine_type != pmc_P6)
+	if (!(pmc_table->machine_type == pmc_P6 ||
+	      pmc_table->machine_type == pmc_Core))
 		return KERN_FAILURE;
 	
-	*(uint64_t *)evtsel = rdmsr64(pmc_table->msr_control_base + id);
+	evtsel->u64 = rdmsr64(pmc_table->msr_control_base + id);
 
 	return KERN_SUCCESS;
 }
@@ -479,10 +596,11 @@ pmc_evtsel_write(pmc_id_t id, pmc_evtsel_t *evtsel)
 	if (pmc_table == NULL)
 		return KERN_INVALID_ARGUMENT;
 
-	if (pmc_table->machine_type != pmc_P4_Xeon)
+	if (!(pmc_table->machine_type == pmc_P6 ||
+	      pmc_table->machine_type == pmc_Core))
 		return KERN_FAILURE;
 	
-	wrmsr64(pmc_table->msr_control_base + id, *(uint64_t *)evtsel);
+	wrmsr64(pmc_table->msr_control_base + id, evtsel->u64);
 
 	return KERN_SUCCESS;
 }
@@ -543,7 +661,19 @@ pmc_set_ovf_func(pmc_id_t id, pmc_ovf_func_t func)
 	if (pmc_table == NULL)
 		return KERN_INVALID_ARGUMENT;
 
-	pmc_table->ovf_func[id] = func;
+	switch (pmc_table->machine_type) {
+	case pmc_P4_Xeon:
+		pmc_table->P4.ovf_func[id] = func;
+		break;
+	case pmc_P6:
+		pmc_table->P6.ovf_func[id] = func;
+		break;
+	case pmc_Core:
+		pmc_table->Core.ovf_func[id] = func;
+		break;
+	default:
+		return KERN_INVALID_ARGUMENT;
+	}
 
 	return KERN_SUCCESS;
 }
@@ -553,6 +683,9 @@ pmc_acquire(task_t task)
 {
 	kern_return_t retval = KERN_SUCCESS;
   
+	if (!pmc_inited)
+		return KERN_FAILURE;
+
 	simple_lock(&pmc_lock);
   
 	if(pmc_owner == task) {
@@ -597,6 +730,9 @@ pmc_release(task_t task)
 	kern_return_t retval = KERN_SUCCESS;
 	task_t old_pmc_owner = pmc_owner;
   
+	if (!pmc_inited)
+		return KERN_FAILURE;
+
 	simple_lock(&pmc_lock);
   
 	if(task != pmc_owner) {
diff --git a/osfmk/i386/perfmon.h b/osfmk/i386/perfmon.h
index 3c2906b7c..b943d1817 100644
--- a/osfmk/i386/perfmon.h
+++ b/osfmk/i386/perfmon.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _I386_PERFMON_H_
 #define _I386_PERFMON_H_
@@ -58,30 +64,103 @@ pmc_is_available(void)
 #define	PMC_COUNTER_COUNTER		field(0,39)
 #define	PMC_COUNTER_RESERVED		field(40,64)
 #define PMC_COUNTER_MAX			((uint64_t) PMC_COUNTER_COUNTER)
-typedef struct {
+typedef union {
+    struct {
 	uint64_t	counter		: 40;
 	uint64_t	reserved	: 24;
+     }		fld;
+     uint64_t	u64;
 } pmc_counter_t;
-#define	PMC_COUNTER_ZERO { 0, 0 }
+#define	PMC_COUNTER_ZERO { .u64 = 0 }
 
 
 /*
- * There are 2 basic flavors of PMCsL: P6 and P4/Xeon:
+ * There are 2 basic flavors of PMCs: P6 (or Core) and P4/Xeon:
  */
 typedef enum {
-	pmc_none,
+	pmc_none = 0,
 	pmc_P6,
 	pmc_P4_Xeon,
+	pmc_Core,
 	pmc_unknown
 } pmc_machine_t;
 
+#define MSR_PERF_FIXED_CTR(n)		(0x309 + (n))
+#define MSR_PERF_INST_RETIRED		MSR_PERF_FIXED_CTR(0)
+#define MSR_PERF_CPU_CLK_UNHALTED_CORE	MSR_PERF_FIXED_CTR(1)
+#define MSR_PERF_CPU_CLK_UNHALTED_REF	MSR_PERF_FIXED_CTR(2)
+
+#define	MSR_PERF_FIXED_CTR_CTRL		(0x38d)
+typedef union {
+    struct {
+	uint64_t	FIXED_CTR0_enable	:2;
+	uint64_t	reserved0		:1;
+	uint64_t	FIXED_CTR0_pmi		:1;
+	uint64_t	FIXED_CTR1_enable	:2;
+	uint64_t	reserved1		:1;
+	uint64_t	FIXED_CTR1_pmi		:1;
+	uint64_t	FIXED_CTR2_enable	:2;
+	uint64_t	reserved2		:1;
+	uint64_t	FIXED_CTR2_pmi		:1;
+     }		fld;
+     uint64_t	u64;
+} pmc_fixed_ctr_ctrl_t;
+
+#define	MSR_PERF_GLOBAL_STATUS		(0x38e)
+typedef union {
+    struct {
+	uint64_t	PMC0_overflow		: 1;
+	uint64_t	PMC1_overflow		: 1;
+	uint64_t	reserved1		: 30;
+	uint64_t	FIXED_CTR0_overflow	: 1;
+	uint64_t	FIXED_CTR1_overflow	: 1;
+	uint64_t	FIXED_CTR2_overflow	: 1;
+	uint64_t	reserved2		: 27;
+	uint64_t	ovf_buffer		: 1;
+	uint64_t	cond_changed		: 1;
+     }		fld;
+     uint64_t	u64;
+} pmc_global_status_t;
+
+#define	MSR_PERF_GLOBAL_CTRL		(0x38f)
+typedef union {
+    struct {
+	uint64_t	PMC0_enable		: 1;
+	uint64_t	PMC1_enable		: 1;
+	uint64_t	reserved1		: 30;
+	uint64_t	FIXED_CTR0_enable	: 1;
+	uint64_t	FIXED_CTR1_enable	: 1;
+	uint64_t	FIXED_CTR2_enable	: 1;
+     }		fld;
+     uint64_t	u64;
+} pmc_global_ctrl_t;
+
+#define	MSR_PERF_GLOBAL_OVF_CTRL	(0x390)
+typedef union {
+    struct {
+	uint64_t	PMC0_clr_overflow	: 1;
+	uint64_t	PMC1_clr_overflow	: 1;
+	uint64_t	reserved1		: 30;
+	uint64_t	FIXED_CTR0_clr_overflow	: 1;
+	uint64_t	FIXED_CTR1_clr_overflow	: 1;
+	uint64_t	FIXED_CTR2_clr_overflow	: 1;
+	uint64_t	reserved2		: 27;
+	uint64_t	clr_ovf_buffer		: 1;
+	uint64_t	clr_cond_changed	: 1;
+     }		fld;
+     uint64_t	u64;
+} pmc_global_ovf_ctrl;
+
 /*
  * P6 MSRs...
  */
 #define	MSR_P6_COUNTER_ADDR(n)	(0x0c1 + (n))
 #define	MSR_P6_PES_ADDR(n)	(0x186 + (n))
+#define MSR_IA32_PMC(n)		(0x0c1 + (n))
+#define MSR_IA32_PERFEVTSEL(n)	(0x186 + (n))
 
-typedef struct {
+typedef union {
+    struct {
 	uint64_t	event_select	: 8;
 	uint64_t	umask		: 8;
 	uint64_t	usr		: 1;
@@ -93,8 +172,45 @@ typedef struct {
 	uint64_t	en		: 1;
 	uint64_t	inv		: 1;
 	uint64_t	cmask		: 8;
+     }		fld;
+     uint64_t	u64;
 } pmc_evtsel_t;
-#define PMC_EVTSEL_ZERO	((pmc_evtsel_t){ 0,0,0,0,0,0,0,0,0,0,0 }) 
+#define PMC_EVTSEL_ZERO	{ .u64 = 0ULL }
+	
+
+/*
+ * Non-architectural event selectors. See Vol 3b, section 18.13:   
+ */
+#define PMC_EVTSEL_ALLCORES	(bit(15)|bit(14))
+#define PMC_EVTSEL_THISCORE	(bit(14))
+#define PMC_EVTSEL_ALLAGENTS	(bit(14))
+#define PMC_EVTSEL_THISAGENT	(0ULL) }
+#define PMC_EVTSEL_PREFETCH_ALL	(bit(13)|bit(12))
+#define PMC_EVTSEL_PREFETCH_HW	(bit(12))
+#define PMC_EVTSEL_PREFETCH_NOHW (0ULL)
+#define PMC_EVTSEL_MESI_MOD	(bit(11))
+#define PMC_EVTSEL_MESI_EXCL	(bit(10))
+#define PMC_EVTSEL_MESI_SHRD	(bit(9))
+#define PMC_EVTSEL_MESI_INVAL	(bit(8))
+
+#define PMC_EVTSEL_SNOOP_HITM	(bit(11))
+#define PMC_EVTSEL_SNOOP_HIT	(bit(9))
+#define PMC_EVTSEL_SNOOP_CLEAN	(bit(8))
+#define PMC_EVTSEL_SNOOP_CMP2I	(bit(9))
+#define PMC_EVTSEL_SNOOP_CMP2S	(bit(8))
+
+#define PMC_CORE_ITLB_MISS_RETIRED			\
+	{ .fld.event_select = 0xc9, .fld.umask = 0 }
+#define PMC_CORE_MEM_LOAD_RETIRED_L1D_MISS		\
+	{ .fld.event_select = 0xcb, .fld.umask = 1 }
+#define PMC_CORE_MEM_LOAD_RETIRED_L1D_LINE_MISS		\
+	{ .fld.event_select = 0xcb, .fld.umask = 2 }
+#define PMC_CORE_MEM_LOAD_RETIRED_L2_MISS		\
+	{ .fld.event_select = 0xcb, .fld.umask = 4 }
+#define PMC_CORE_MEM_LOAD_RETIRED_L2_LINE_MISS		\
+	{ .fld.event_select = 0xcb, .fld.umask = 8 }
+#define PMC_CORE_MEM_LOAD_RETIRED_DTLB_MISS		\
+	{ .fld.event_select = 0xcb, .fld.umask = 0x10 }
 
 #define	MSR_P6_PERFCTR0	0
 #define	MSR_P6_PERFCTR1	1
diff --git a/osfmk/i386/phys.c b/osfmk/i386/phys.c
index 390d5a014..7269aa283 100644
--- a/osfmk/i386/phys.c
+++ b/osfmk/i386/phys.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -80,7 +86,6 @@
 #include <i386/tss.h>
 #include <i386/user_ldt.h>
 #include <i386/fpu.h>
-#include <i386/iopb_entries.h>
 #include <i386/misc_protos.h>
 
 /*
@@ -91,6 +96,7 @@ pmap_zero_page(
 	       ppnum_t pn)
 {
 	assert(pn != vm_page_fictitious_addr);
+	assert(pn != vm_page_guard_addr);
 	bzero_phys((addr64_t)i386_ptob(pn), PAGE_SIZE);
 }
 
@@ -105,6 +111,7 @@ pmap_zero_part_page(
 	vm_size_t       len)
 {
 	assert(pn != vm_page_fictitious_addr);
+	assert(pn != vm_page_guard_addr);
 	assert(offset + len <= PAGE_SIZE);
 	bzero_phys((addr64_t)(i386_ptob(pn) + offset), len);
 }
@@ -124,6 +131,8 @@ pmap_copy_part_page(
 
 	assert(psrc != vm_page_fictitious_addr);
 	assert(pdst != vm_page_fictitious_addr);
+	assert(psrc != vm_page_guard_addr);
+	assert(pdst != vm_page_guard_addr);
 
 	src = i386_ptob(psrc);
 	dst = i386_ptob(pdst);
@@ -150,19 +159,17 @@ pmap_copy_part_lpage(
         mapwindow_t *map;
 
 	assert(pdst != vm_page_fictitious_addr);
+	assert(pdst != vm_page_guard_addr);
 	assert((dst_offset + len) <= PAGE_SIZE);
 
         mp_disable_preemption();
 
         map = pmap_get_mapwindow(INTEL_PTE_VALID | INTEL_PTE_RW | (i386_ptob(pdst) & PG_FRAME) | 
                                  INTEL_PTE_REF | INTEL_PTE_MOD);
-	if (map == 0) {
-	        panic("pmap_copy_part_lpage");
-        }
-	invlpg((uintptr_t)map->prv_CADDR);
 
 	memcpy((void *) (map->prv_CADDR + (dst_offset & INTEL_OFFMASK)), (void *) src, len);
-	*map->prv_CMAP = 0;
+
+	pmap_put_mapwindow(map);
 
 	mp_enable_preemption();
 }
@@ -181,19 +188,17 @@ pmap_copy_part_rpage(
         mapwindow_t *map;
 
 	assert(psrc != vm_page_fictitious_addr);
+	assert(psrc != vm_page_guard_addr);
 	assert((src_offset + len) <= PAGE_SIZE);
 
         mp_disable_preemption();
 
         map = pmap_get_mapwindow(INTEL_PTE_VALID | INTEL_PTE_RW | (i386_ptob(psrc) & PG_FRAME) | 
                                  INTEL_PTE_REF);
-	if (map == 0) {
-	        panic("pmap_copy_part_rpage");
-        }
-	invlpg((uintptr_t) map->prv_CADDR);
 
 	memcpy((void *) dst, (void *) (map->prv_CADDR + (src_offset & INTEL_OFFMASK)), len);
-	*map->prv_CMAP = 0;
+
+	pmap_put_mapwindow(map);
 
 	mp_enable_preemption();
 }
@@ -209,14 +214,15 @@ kvtophys(
 {
         pt_entry_t *ptep;
 	pmap_paddr_t pa;
-	
+ 
 	mp_disable_preemption();
 	if ((ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)addr)) == PT_ENTRY_NULL) {
-	        pa = 0;
+	  pa = 0;
 	} else {
-	        pa =  pte_to_pa(*ptep) | (addr & INTEL_OFFMASK);
+	  pa =  pte_to_pa(*ptep) | (addr & INTEL_OFFMASK);
 	}
 	mp_enable_preemption_no_check();
+
 	return ((addr64_t)pa);
 }
 
diff --git a/osfmk/i386/pic.h b/osfmk/i386/pic.h
deleted file mode 100644
index 1c9311a52..000000000
--- a/osfmk/i386/pic.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-
-/*
-Copyright (c) 1988,1989 Prime Computer, Inc.  Natick, MA 01760
-All Rights Reserved.
-
-Permission to use, copy, modify, and distribute this
-software and its documentation for any purpose and
-without fee is hereby granted, provided that the above
-copyright notice appears in all copies and that both the
-copyright notice and this permission notice appear in
-supporting documentation, and that the name of Prime
-Computer, Inc. not be used in advertising or publicity
-pertaining to distribution of the software without
-specific, written prior permission.
-
-THIS SOFTWARE IS PROVIDED "AS IS", AND PRIME COMPUTER,
-INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
-SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  IN
-NO EVENT SHALL PRIME COMPUTER, INC.  BE LIABLE FOR ANY
-SPECIAL, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
-DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
-PROFITS, WHETHER IN ACTION OF CONTRACT, NEGLIGENCE, OR
-OTHER TORTIOUS ACTION, ARISING OUR OF OR IN CONNECTION
-WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-
-#ifndef	_I386_PIC_H_
-#define	_I386_PIC_H_
-
-#include <platforms.h>
-
-#define NINTR	0x10
-#define	NPICS	0x02
-
-/*
-** The following are definitions used to locate the PICs in the system
-*/
-
-#define ADDR_PIC_BASE		0x20
-#define OFF_ICW			0x00
-#define OFF_OCW			0x01
-#define SIZE_PIC		0x80
-
-/*
-** The following banks of definitions ICW1, ICW2, ICW3, and ICW4 are used
-** to define the fields of the various ICWs for initialisation of the PICs 
-*/
-
-/*
-**	ICW1				
-*/
-
-#define ICW_TEMPLATE		0x10
-
-#define LEVL_TRIGGER		0x08
-#define EDGE_TRIGGER		0x00
-#define ADDR_INTRVL4		0x04
-#define ADDR_INTRVL8		0x00
-#define SINGLE__MODE		0x02
-#define CASCADE_MODE		0x00
-#define ICW4__NEEDED		0x01
-#define NO_ICW4_NEED		0x00
-
-/*
-**	ICW2
-*/
-
-#define	PICM_VECTBASE		0x40 
-#define PICS_VECTBASE		PICM_VECTBASE + 0x08
-
-/*
-**	ICW3				
-*/
-
-#define SLAVE_ON_IR0		0x01
-#define SLAVE_ON_IR1		0x02
-#define SLAVE_ON_IR2		0x04
-#define SLAVE_ON_IR3		0x08
-#define SLAVE_ON_IR4		0x10
-#define SLAVE_ON_IR5		0x20
-#define SLAVE_ON_IR6		0x40
-#define SLAVE_ON_IR7		0x80
-
-#define I_AM_SLAVE_0		0x00
-#define I_AM_SLAVE_1		0x01
-#define I_AM_SLAVE_2		0x02
-#define I_AM_SLAVE_3		0x03
-#define I_AM_SLAVE_4		0x04
-#define I_AM_SLAVE_5		0x05
-#define I_AM_SLAVE_6		0x06
-#define I_AM_SLAVE_7		0x07
-
-/*
-**	ICW4				
-*/
-
-#define SNF_MODE_ENA		0x10
-#define SNF_MODE_DIS		0x00
-#define BUFFERD_MODE		0x08
-#define NONBUFD_MODE		0x00
-#define AUTO_EOI_MOD		0x02
-#define NRML_EOI_MOD		0x00
-#define I8086_EMM_MOD		0x01
-#define SET_MCS_MODE		0x00
-
-/*
-**	OCW1				
-*/
-#define PICM_MASK		0xFF
-#define	PICS_MASK		0xFF
-/*
-**	OCW2				
-*/
-
-#define NON_SPEC_EOI		0x20
-#define SPECIFIC_EOI		0x30
-#define ROT_NON_SPEC		0x50
-#define SET_ROT_AEOI		0x40
-#define RSET_ROTAEOI		0x00
-#define ROT_SPEC_EOI		0x70
-#define SET_PRIORITY		0x60
-#define NO_OPERATION		0x20
-
-#define SEND_EOI_IR0		0x00
-#define SEND_EOI_IR1		0x01
-#define SEND_EOI_IR2		0x02
-#define SEND_EOI_IR3		0x03
-#define SEND_EOI_IR4		0x04
-#define SEND_EOI_IR5		0x05
-#define SEND_EOI_IR6		0x06
-#define SEND_EOI_IR7		0x07
- 
-/*
-**	OCW3				
-*/
-
-#define OCW_TEMPLATE		0x08
-#define SPECIAL_MASK		0x40
-#define MASK_MDE_SET		0x20
-#define MASK_MDE_RST		0x00
-#define POLL_COMMAND		0x04
-#define NO_POLL_CMND		0x00
-#define READ_NEXT_RD		0x02
-#define READ_IR_ONRD		0x00
-#define READ_IS_ONRD		0x01
-
-#endif	/* _I386_PIC_H_ */
diff --git a/osfmk/i386/pio.h b/osfmk/i386/pio.h
index 28e59196b..48cf259ae 100644
--- a/osfmk/i386/pio.h
+++ b/osfmk/i386/pio.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/pit.h b/osfmk/i386/pit.h
deleted file mode 100644
index 1bfb05bf9..000000000
--- a/osfmk/i386/pit.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/* 
- */
-
-/*
-  Copyright 1988, 1989 by Intel Corporation, Santa Clara, California.
-
-		All Rights Reserved
-
-Permission to use, copy, modify, and distribute this software and
-its documentation for any purpose and without fee is hereby
-granted, provided that the above copyright notice appears in all
-copies and that both the copyright notice and this permission notice
-appear in supporting documentation, and that the name of Intel
-not be used in advertising or publicity pertaining to distribution
-of the software without specific, written prior permission.
-
-INTEL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
-INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS,
-IN NO EVENT SHALL INTEL BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
-CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
-NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-
-#include <platforms.h>
-/* Definitions for 8254 Programmable Interrupt Timer ports on AT 386 */
-#define PITCTR0_PORT	0x40		/* counter 0 port */	
-#define PITCTR1_PORT	0x41		/* counter 1 port */	
-#define PITCTR2_PORT	0x42		/* counter 2 port */	
-#define PITCTL_PORT	0x43		/* PIT control port */
-#define PITAUX_PORT	0x61		/* PIT auxiliary port */
-/* bits used in auxiliary control port for timer 2 */
-#define PITAUX_GATE2	0x01		/* aux port, PIT gate 2 input */
-#define PITAUX_OUT2	0x02		/* aux port, PIT clock out 2 enable */
-
-/* Following are used for Timer 0 */
-#define PIT_C0          0x00            /* select counter 0 */
-#define PIT_LOADMODE	0x30		/* load least significant byte followed
-					 * by most significant byte */
-#define PIT_NDIVMODE	0x04		/*divide by N counter */
-#define PIT_SQUAREMODE	0x06		/* square-wave mode */
-
-/* Used for Timer 1. Used for delay calculations in countdown mode */
-#define PIT_C1          0x40            /* select counter 1 */
-#define PIT_READMODE	0x30		/* read or load least significant byte
-					 * followed by most significant byte */
-#define PIT_RATEMODE	0x06		/* square-wave mode for USART */
-
-/*
- * Clock speed for the timer in hz divided by the constant HZ
- * (defined in param.h)
- */
-#define CLKNUM		1193182		/* formerly 1193167 */
-
-#if	EXL
-/* added micro-timer support.   --- csy */
-typedef struct time_latch {
-		time_t	ticks;          /* time in HZ since boot */
-		time_t	uticks;         /* time in 1.25 MHZ */
-/* don't need these two for now.   --- csy */
-/*		time_t  secs;           \* seconds since boot */
-/*		time_t  epochsecs;      \* seconds since epoch */
-	} time_latch;
-/* a couple in-line assembly codes for efficiency. */
-asm  int   intr_disable()
-{
-     pushfl
-     cli
-}
-
-asm  int   intr_restore()
-{
-     popfl
-}
-
-#endif	/* EXL */
diff --git a/osfmk/i386/pmCPU.c b/osfmk/i386/pmCPU.c
index 3659de1c8..1f12073fb 100644
--- a/osfmk/i386/pmCPU.c
+++ b/osfmk/i386/pmCPU.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -34,19 +40,10 @@
 #include <i386/proc_reg.h>
 #include <kern/pms.h>
 #include <kern/processor.h>
+#include <i386/cpu_threads.h>
 #include <i386/pmCPU.h>
 #include <i386/cpuid.h>
 #include <i386/rtclock.h>
-#if MACH_KDB
-#include <i386/db_machdep.h>
-#include <ddb/db_aout.h>
-#include <ddb/db_access.h>
-#include <ddb/db_sym.h>
-#include <ddb/db_variables.h>
-#include <ddb/db_command.h>
-#include <ddb/db_output.h>
-#include <ddb/db_expr.h>
-#endif
 
 extern int disableConsoleOutput;
 
@@ -62,142 +59,137 @@ pmDispatch_t	*pmDispatch	= NULL;
  */
 static pmInitState_t	pmInitState;
 
+static uint32_t		pmInitDone	= 0;
+
 /*
  * Nap control variables:
  */
-uint32_t napCtl = 0;			/* Defaults to neither napping
-					   nor halting */
 uint32_t forcenap = 0;			/* Force nap (fn) boot-arg controls */
-uint32_t maxBusDelay = 0xFFFFFFFF;	/* Maximum memory bus delay that
-					   I/O devices can tolerate
-					   before errors (nanoseconds) */
-uint32_t C4C2SnoopDelay = 0;		/* C4 to C2 transition time -
-					   time before a C4 system
-					   can snoop (nanoseconds) */
-
-/*
- * We are being asked to set PState (sel).
- */
-void
-pmsCPUSet(uint32_t sel)
-{
-    if (pmDispatch != NULL && pmDispatch->pmsCPUSet != NULL)
-	(*pmDispatch->pmsCPUSet)(sel);
-    else
-	pmInitState.PState = sel;
-}
 
 /*
- * This code configures the initial step tables.  It should be called after
- * the timebase frequency is initialized.
- *
- * Note that this is not used in normal operation.  It is strictly for
- * debugging/testing purposes.
+ * Do any initialization needed
  */
 void
-pmsCPUConf(void)
+pmsInit(void)
 {
+    static int		initialized	= 0;
 
-    if (pmDispatch != NULL && pmDispatch->pmsCPUConf != NULL)
-	(*pmDispatch->pmsCPUConf)();
-}
-
-/*
- * Machine-dependent initialization.
- */
-void
-pmsCPUMachineInit(void)
-{
     /*
      * Initialize some of the initial state to "uninitialized" until
      * it gets set with something more useful.  This allows the KEXT
      * to determine if the initial value was actually set to something.
      */
-    pmInitState.PState = -1;
-    pmInitState.PLimit = -1;
+    if (!initialized) {
+	pmInitState.PState = -1;
+	pmInitState.PLimit = -1;
+	pmInitState.maxBusDelay = -1;
+	initialized = 1;
+    }
 
-    if (pmDispatch != NULL && pmDispatch->pmsCPUMachineInit != NULL)
-	(*pmDispatch->pmsCPUMachineInit)();
+    if (pmDispatch != NULL && pmDispatch->pmsInit != NULL)
+	(*pmDispatch->pmsInit)();
 }
 
 /*
- * This function should be called once for each processor to force the
- * processor to the correct initial voltage and frequency.
+ * Start the power management stepper on all processors
+ *
+ * All processors must be parked.  This should be called when the hardware
+ * is ready to step.  Probably only at boot and after wake from sleep.
+ *
  */
 void
-pmsCPUInit(void)
+pmsStart(void)
 {
-    pmsCPUMachineInit();
-    if (pmDispatch != NULL && pmDispatch->pmsCPUInit != NULL)
-	(*pmDispatch->pmsCPUInit)();
+    if (pmDispatch != NULL && pmDispatch->pmsStart != NULL)
+	(*pmDispatch->pmsStart)();
 }
 
 /*
- * Broadcast a change to all processing including ourselves.
+ * Park the stepper execution.  This will force the stepper on this
+ * processor to abandon its current step and stop.  No changes to the
+ * hardware state is made and any previous step is lost.
+ *	
+ * This is used as the initial state at startup and when the step table
+ * is being changed.
+ *
  */
 void
-pmsCPURun(uint32_t nstep)
+pmsPark(void)
 {
-    if (pmDispatch != NULL && pmDispatch->pmsCPURun != NULL)
-	(*pmDispatch->pmsCPURun)(nstep);
+    if (pmDispatch != NULL && pmDispatch->pmsPark != NULL)
+	(*pmDispatch->pmsPark)();
 }
 
 /*
- * Return the current state of a core.
- */
-uint32_t
-pmsCPUQuery(void)
-{
-    if (pmDispatch != NULL && pmDispatch->pmsCPUQuery != NULL)
-	return((*pmDispatch->pmsCPUQuery)());
-
-    /*
-     * Return a non-sense value.
-     */
-    return((~0) << 16);
-}
-
-/*
- * Return the current state of the package.
+ * Control the Power Management Stepper.
+ * Called from user state by the superuser.
+ * Interrupts disabled.
+ *
+ * This interface is deprecated and is now a no-op.
  */
-uint32_t
-pmsCPUPackageQuery(void)
+kern_return_t
+pmsControl(__unused uint32_t request, __unused user_addr_t reqaddr,
+	   __unused uint32_t reqsize)
 {
-    if (pmDispatch != NULL && pmDispatch->pmsCPUPackageQuery != NULL)
-	return((*pmDispatch->pmsCPUPackageQuery)());
-
-    /*
-     * Return a non-sense value.
-     */
-    return((~0) << 16);
+    return(KERN_SUCCESS);
 }
 
 /*
- * Force the CPU package to the lowest power level.  This is a low-level
- * interface meant to be called from the panic or debugger code to bring
- * the CPU to a safe power level for unmanaged operation.
+ * Broadcast a change to all processors including ourselves.
  *
- * Note that while this will bring an entire package to a safe level, it
- * cannot affect other packages.  As a general rule, this should be run on
- * every code as part of entering the debugger or on the panic path.
+ * Interrupts disabled.
  */
 void
-pmsCPUYellowFlag(void)
+pmsRun(uint32_t nstep)
 {
-    if (pmDispatch != NULL && pmDispatch->pmsCPUYellowFlag != NULL)
-	(*pmDispatch->pmsCPUYellowFlag)();
+    if (pmDispatch != NULL && pmDispatch->pmsRun != NULL)
+	(*pmDispatch->pmsRun)(nstep);
 }
 
 /*
- * Restore the CPU to the power state it was in before a yellow flag.
+ * Build the tables needed for the stepper.  This includes both the step
+ * definitions and the step control table.
+ *
+ * We most absolutely need to be parked before this happens because we're
+ * going to change the table.  We also have to be complte about checking
+ * for errors.  A copy is always made because we don't want to be crippled
+ * by not being able to change the table or description formats.
+ *
+ * We pass in a table of external functions and the new stepper def uses
+ * the corresponding indexes rather than actual function addresses.  This
+ * is done so that a proper table can be built with the control syscall.
+ * It can't supply addresses, so the index has to do.  We internalize the
+ * table so our caller does not need to keep it.  Note that passing in a 0
+ * will use the current function table.  Also note that entry 0 is reserved
+ * and must be 0, we will check and fail the build.
+ *
+ * The platformData parameter is a 32-bit word of data that is passed unaltered
+ * to the set function.
+ *
+ * The queryFunc parameter is the address of a function that will return the
+ * current state of the platform. The format of the data returned is the same
+ * as the platform specific portions of pmsSetCmd, i.e., pmsXClk, pmsVoltage,
+ * and any part of pmsPowerID that is maintained by the platform hardware
+ * (an example would be the values of the gpios that correspond to pmsPowerID).
+ * The value should be constructed by querying hardware rather than returning
+ * a value cached by software. One of the intents of this function is to help
+ * recover lost or determine initial power states.
+ *
  */
-void
-pmsCPUGreenFlag(void)
+kern_return_t
+pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab,
+	 uint32_t platformData, pmsQueryFunc_t queryFunc)
 {
-    if (pmDispatch != NULL && pmDispatch->pmsCPUGreenFlag != NULL)
-	(*pmDispatch->pmsCPUGreenFlag)();
+    kern_return_t	rc	= 0;
+
+    if (pmDispatch != NULL && pmDispatch->pmsBuild != NULL)
+	rc = (*pmDispatch->pmsBuild)(pd, pdsize, functab,
+				     platformData, queryFunc);
+
+    return(rc);
 }
 
+
 /*
  * Load a new ratio/VID table.
  *
@@ -248,45 +240,20 @@ pmsCPUSetPStateLimit(uint32_t limit)
 void
 power_management_init(void)
 {
-    uint32_t	cpuModel;
-    uint32_t	cpuFamily;
-    uint32_t	xcpuid[4];
+    static boolean_t	initialized	= FALSE;
 
     /*
      * Initialize the lock for the KEXT initialization.
      */
-    simple_lock_init(&pm_init_lock, 0);
-
-    /*
-     * XXX
-     *
-     * The following is a hack to disable power management on some systems
-     * until the KEXT is done.  This is strictly temporary!!!
-     */
-    do_cpuid(1, xcpuid);
-    cpuFamily = (xcpuid[eax] >> 8) & 0xf;
-    cpuModel  = (xcpuid[eax] >> 4) & 0xf;
-
-    if (cpuFamily != 0x6 || cpuModel < 0xe)
-	pmDispatch = NULL;
+    if (!initialized) {
+	simple_lock_init(&pm_init_lock, 0);
+	initialized = TRUE;
+    }
 
     if (pmDispatch != NULL && pmDispatch->cstateInit != NULL)
 	(*pmDispatch->cstateInit)();
 }
 
-/*
- * This function will update the system nap policy.  It should be called
- * whenever conditions change: when the system is ready to being napping
- * and if something changes the rules (e.g. a sysctl altering the policy
- * for debugging).
- */
-void
-machine_nap_policy(void)
-{
-    if (pmDispatch != NULL && pmDispatch->cstateNapPolicy != NULL)
-	napCtl = (*pmDispatch->cstateNapPolicy)(forcenap, napCtl);
-}
-
 /*
  * ACPI calls the following routine to set/update mwait hints.  A table
  * (possibly null) specifies the available Cstates and their hints, all
@@ -315,155 +282,411 @@ Cstate_table_set(Cstate_hint_t *tablep, unsigned int nstates)
     return(KERN_SUCCESS);
 }
 
-static inline void
-sti(void) {
-	__asm__ volatile ( "sti" : : : "memory");
-}
-
 /*
  * Called when the CPU is idle.  It will choose the best C state to
  * be in.
  */
 void
-machine_idle_cstate(void)
+machine_idle_cstate(boolean_t halted)
 {
-    if (pmDispatch != NULL && pmDispatch->cstateMachineIdle != NULL)
-	(*pmDispatch->cstateMachineIdle)(napCtl);
-    else {
-	sti();
+	if (pmInitDone
+	    && pmDispatch != NULL
+	    && pmDispatch->cstateMachineIdle != NULL)
+		(*pmDispatch->cstateMachineIdle)(!halted ?
+						 0x7FFFFFFFFFFFFFFFULL : 0ULL);
+	else if (halted) {
+	    /*
+	     * If no power managment and a processor is taken off-line,
+	     * then invalidate the cache and halt it (it will not be able
+	     * to be brought back on-line without resetting the CPU).
+	     */
+	    __asm__ volatile ( "wbinvd; hlt" );
+	} else {
+	    /*
+	     * If no power management, re-enable interrupts and halt.
+	     * This will keep the CPU from spinning through the scheduler
+	     * and will allow at least some minimal power savings (but it
+	     * may cause problems in some MP configurations w.r.t to the
+	     * APIC stopping during a P-State transition).
+	     */
+	    __asm__ volatile ( "sti; hlt" );
+	}
+}
+
+/*
+ * Called when the CPU is to be halted.  It will choose the best C-State
+ * to be in.
+ */
+void
+pmCPUHalt(uint32_t reason)
+{
+
+    switch (reason) {
+    case PM_HALT_DEBUG:
+	__asm__ volatile ("wbinvd; hlt");
+	break;
+
+    case PM_HALT_PANIC:
+	__asm__ volatile ("cli; wbinvd; hlt");
+	break;
+
+    case PM_HALT_NORMAL:
+    default:
+	__asm__ volatile ("cli");
+
+	if (pmInitDone
+	    && pmDispatch != NULL
+	    && pmDispatch->pmCPUHalt != NULL) {
+	    (*pmDispatch->pmCPUHalt)();
+	} else {
+	    cpu_data_t	*cpup	= current_cpu_datap();
+
+	    /*
+	     * If no power managment and a processor is taken off-line,
+	     * then invalidate the cache and halt it (it will not be able
+	     * to be brought back on-line without resetting the CPU).
+	     */
+	    __asm__ volatile ("wbinvd");
+	    cpup->lcpu.halted = TRUE;
+	    __asm__ volatile ( "wbinvd; hlt" );
+	}
+	break;
     }
 }
 
-static pmStats_t *
-pmsCPUStats(void)
+/*
+ * Called to initialize the power management structures for the CPUs.
+ */
+void
+pmCPUStateInit(void)
+{
+    if (pmDispatch != NULL && pmDispatch->pmCPUStateInit != NULL)
+	(*pmDispatch->pmCPUStateInit)();
+}
+
+static void
+pmInitComplete(void)
+{
+    pmInitDone = 1;
+}
+
+static x86_lcpu_t *
+pmGetLogicalCPU(int cpu)
 {
-    cpu_data_t	*pp;
+    return(cpu_to_lcpu(cpu));
+}
+
+static x86_lcpu_t *
+pmGetMyLogicalCPU(void)
+{
+    cpu_data_t	*cpup	= current_cpu_datap();
 
-    pp = current_cpu_datap();
-    return(&pp->cpu_pmStats);
+    return(&cpup->lcpu);
+}
+
+static x86_core_t *
+pmGetCore(int cpu)
+{
+    return(cpu_to_core(cpu));
 }
 
-static pmsd *
-pmsCPUStepperData(void)
+static x86_core_t *
+pmGetMyCore(void)
 {
-    cpu_data_t	*pp;
+    cpu_data_t	*cpup	= current_cpu_datap();
 
-    pp = current_cpu_datap();
-    return(&pp->pms);
+    return(cpup->lcpu.core);
 }
 
-static uint64_t *
-CPUHPETAddr(void)
+static x86_pkg_t *
+pmGetPackage(int cpu)
 {
-    cpu_data_t	*pp;
-    pp = current_cpu_datap();
-    return(pp->cpu_pmHpet);
+    return(cpu_to_package(cpu));
+}
+
+static x86_pkg_t *
+pmGetMyPackage(void)
+{
+    cpu_data_t	*cpup	= current_cpu_datap();
+
+    return(cpup->lcpu.core->package);
+}
+
+static void
+pmLockCPUTopology(int lock)
+{
+    if (lock) {
+	simple_lock(&x86_topo_lock);
+    } else {
+	simple_unlock(&x86_topo_lock);
+    }
 }
 
 /*
- * Called by the power management kext to register itself and to get the
- * callbacks it might need into other power management functions.
+ * Called to get the next deadline that has been set by the
+ * power management code.
  */
-void
-pmRegister(pmDispatch_t *cpuFuncs, pmCallBacks_t *callbacks)
-{
-    if (callbacks != NULL) {
-	callbacks->Park        = pmsPark;
-	callbacks->Run         = pmsRun;
-	callbacks->RunLocal    = pmsRunLocal;
-	callbacks->SetStep     = pmsSetStep;
-	callbacks->NapPolicy   = machine_nap_policy;
-	callbacks->Build       = pmsBuild;
-	callbacks->Stats       = pmsCPUStats;
-	callbacks->StepperData = pmsCPUStepperData;
-	callbacks->HPETAddr    = CPUHPETAddr;
-	callbacks->InitState   = &pmInitState;
-	callbacks->resetPop    = resetPop;
-    }
+uint64_t
+pmCPUGetDeadline(cpu_data_t *cpu)
+{
+    uint64_t	deadline	= EndOfAllTime;
 
-    if (cpuFuncs != NULL)
-	pmDispatch = cpuFuncs;
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->GetDeadline != NULL)
+	deadline = (*pmDispatch->GetDeadline)(&cpu->lcpu);
+
+    return(deadline);
 }
 
 /*
- * Unregisters the power management functions from the kext.
+ * Called to determine if the supplied deadline or the power management
+ * deadline is sooner.  Returns which ever one is first.
  */
-void
-pmUnRegister(pmDispatch_t *cpuFuncs)
+uint64_t
+pmCPUSetDeadline(cpu_data_t *cpu, uint64_t deadline)
 {
-    if (cpuFuncs != NULL && pmDispatch == cpuFuncs)
-	pmDispatch = NULL;
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->SetDeadline != NULL)
+	deadline = (*pmDispatch->SetDeadline)(&cpu->lcpu, deadline);
+
+    return(deadline);
 }
 
-#if MACH_KDB
 /*
- * XXX stubs for now
+ * Called when a power management deadline expires.
  */
 void
-db_cfg(__unused db_expr_t addr,
-       __unused int have_addr,
-       __unused db_expr_t count,
-       __unused char *modif)
+pmCPUDeadline(cpu_data_t *cpu)
 {
-    return;
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->Deadline != NULL)
+	(*pmDispatch->Deadline)(&cpu->lcpu);
+}
+
+/*
+ * Called to get a CPU out of idle.
+ */
+boolean_t
+pmCPUExitIdle(cpu_data_t *cpu)
+{
+    boolean_t		do_ipi;
+
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->exitIdle != NULL)
+	do_ipi = (*pmDispatch->exitIdle)(&cpu->lcpu);
+    else
+	do_ipi = TRUE;
+
+    return(do_ipi);
 }
 
+/*
+ * Called when a CPU is being restarted after being powered off (as in S3).
+ */
 void
-db_display_iokit(__unused db_expr_t addr,
-		 __unused int have_addr,
-		 __unused db_expr_t count,
-		 __unused char *modif)
+pmCPUMarkRunning(cpu_data_t *cpu)
 {
-    return;
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->markCPURunning != NULL)
+	(*pmDispatch->markCPURunning)(&cpu->lcpu);
 }
 
+/*
+ * Called from the HPET interrupt handler to perform the
+ * necessary power management work.
+ */
 void
-db_dtimers(__unused db_expr_t addr,
-	   __unused int have_addr,
-	   __unused db_expr_t count,
-	   __unused char *modif)
+pmHPETInterrupt(void)
 {
-    return;
+    if (pmInitDone
+	&& pmDispatch != NULL
+	&& pmDispatch->HPETInterrupt != NULL)
+	(*pmDispatch->HPETInterrupt)();
+}
+
+/*
+ * Called to get/set CPU power management state.
+ */
+int
+pmCPUControl(uint32_t cmd, void *datap)
+{
+    int		rc	= -1;
+
+    if (pmDispatch != NULL
+	&& pmDispatch->pmCPUControl != NULL)
+	rc = (*pmDispatch->pmCPUControl)(cmd, datap);
+
+    return(rc);
 }
 
+/*
+ * Set the worst-case time for the C4 to C2 transition.
+ * No longer does anything.
+ */
 void
-db_intcnt(__unused db_expr_t addr,
-	  __unused int have_addr,
-	  __unused db_expr_t count,
-	  __unused char *modif)
+ml_set_maxsnoop(__unused uint32_t maxdelay)
 {
-    return;
 }
 
+
+/*
+ * Get the worst-case time for the C4 to C2 transition.  Returns nanoseconds.
+ */
+unsigned
+ml_get_maxsnoop(void)
+{
+    uint64_t	max_snoop	= 0;
+
+    if (pmDispatch != NULL
+	&& pmDispatch->getMaxSnoop != NULL)
+	max_snoop = pmDispatch->getMaxSnoop();
+
+    return((unsigned)(max_snoop & 0xffffffff));
+}
+
+
+uint32_t
+ml_get_maxbusdelay(void)
+{
+    uint64_t	max_delay	= 0;
+
+    if (pmDispatch != NULL
+	&& pmDispatch->getMaxBusDelay != NULL)
+	max_delay = pmDispatch->getMaxBusDelay();
+
+    return((uint32_t)(max_delay & 0xffffffff));
+}
+
+/*
+ * Set the maximum delay time allowed for snoop on the bus.
+ *
+ * Note that this value will be compared to the amount of time that it takes
+ * to transition from a non-snooping power state (C4) to a snooping state (C2).
+ * If maxBusDelay is less than C4C2SnoopDelay,
+ * we will not enter the lowest power state.
+ */
 void
-db_nap(__unused db_expr_t addr,
-       __unused int have_addr,
-       __unused db_expr_t count,
-       __unused char *modif)
+ml_set_maxbusdelay(uint32_t mdelay)
 {
-    return;
+    uint64_t	maxdelay	= mdelay;
+
+    if (pmDispatch != NULL
+	&& pmDispatch->setMaxBusDelay != NULL)
+	pmDispatch->setMaxBusDelay(maxdelay);
+    else
+	pmInitState.maxBusDelay = maxdelay;
 }
 
+/*
+ * Put a CPU into "safe" mode with respect to power.
+ *
+ * Some systems cannot operate at a continuous "normal" speed without
+ * exceeding the thermal design.  This is called per-CPU to place the
+ * CPUs into a "safe" operating mode.
+ */
 void
-db_pmgr(__unused db_expr_t addr,
-	__unused int have_addr,
-	__unused db_expr_t count,
-	__unused char *modif)
+pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags)
+{
+    if (pmDispatch != NULL
+	&& pmDispatch->pmCPUSafeMode != NULL)
+	pmDispatch->pmCPUSafeMode(lcpu, flags);
+    else {
+	/*
+	 * Do something reasonable if the KEXT isn't present.
+	 *
+	 * We only look at the PAUSE and RESUME flags.  The other flag(s)
+	 * will not make any sense without the KEXT, so just ignore them.
+	 *
+	 * We set the halted flag in the LCPU structure to indicate
+	 * that this CPU isn't to do anything.  If it's the CPU we're
+	 * currently running on, then spin until the halted flag is
+	 * reset.
+	 */
+	if (flags & PM_SAFE_FL_PAUSE) {
+	    lcpu->halted = TRUE;
+	    if (lcpu == x86_lcpu()) {
+		while (lcpu->halted)
+		    cpu_pause();
+	    }
+	}
+	
+	/*
+	 * Clear the halted flag for the specified CPU, that will
+	 * get it out of it's spin loop.
+	 */
+	if (flags & PM_SAFE_FL_RESUME) {
+	    lcpu->halted = FALSE;
+	}
+    }
+}
+
+/*
+ * Returns the root of the package tree.
+ */
+static x86_pkg_t *
+pmGetPkgRoot(void)
+{
+    return(x86_pkgs);
+}
+
+static boolean_t
+pmCPUGetHibernate(int cpu)
 {
-    return;
+    return(cpu_datap(cpu)->cpu_hibernate);
 }
 
+static processor_t
+pmLCPUtoProcessor(int lcpu)
+{
+    return(cpu_datap(lcpu)->cpu_processor);
+}
+
+/*
+ * Called by the power management kext to register itself and to get the
+ * callbacks it might need into other kernel functions.  This interface
+ * is versioned to allow for slight mis-matches between the kext and the
+ * kernel.
+ */
 void
-db_test(__unused db_expr_t addr,
-	__unused int have_addr,
-	__unused db_expr_t count,
-	__unused char *modif)
+pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs,
+	       pmCallBacks_t *callbacks)
 {
-    return;
+    if (callbacks != NULL && version == PM_DISPATCH_VERSION) {
+	callbacks->InitState   = &pmInitState;
+	callbacks->setRTCPop   = setPop;
+	callbacks->resyncDeadlines = etimer_resync_deadlines;
+	callbacks->initComplete= pmInitComplete;
+	callbacks->GetLCPU     = pmGetLogicalCPU;
+	callbacks->GetCore     = pmGetCore;
+	callbacks->GetPackage  = pmGetPackage;
+	callbacks->GetMyLCPU   = pmGetMyLogicalCPU;
+	callbacks->GetMyCore   = pmGetMyCore;
+	callbacks->GetMyPackage= pmGetMyPackage;
+	callbacks->CoresPerPkg = cpuid_info()->cpuid_cores_per_package;
+	callbacks->GetPkgRoot  = pmGetPkgRoot;
+	callbacks->LockCPUTopology = pmLockCPUTopology;
+	callbacks->GetHibernate    = pmCPUGetHibernate;
+	callbacks->LCPUtoProcessor = pmLCPUtoProcessor;
+    }
+
+    if (cpuFuncs != NULL) {
+	pmDispatch = cpuFuncs;
+    }
 }
 
+/*
+ * Unregisters the power management functions from the kext.
+ */
 void
-db_getpmgr(__unused pmData_t *pmj)
+pmUnRegister(pmDispatch_t *cpuFuncs)
 {
+    if (cpuFuncs != NULL && pmDispatch == cpuFuncs) {
+	pmDispatch = NULL;
+    }
 }
-#endif
+
diff --git a/osfmk/i386/pmCPU.h b/osfmk/i386/pmCPU.h
index 3ee626fea..1ef88c3fe 100644
--- a/osfmk/i386/pmCPU.h
+++ b/osfmk/i386/pmCPU.h
@@ -1,54 +1,48 @@
 /*
- * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef KERNEL_PRIVATE
 #ifndef _I386_PMCPU_H_
 #define _I386_PMCPU_H_
 
 #include <kern/pms.h>
+#include <i386/cpu_topology.h>
 
 #ifndef ASSEMBLER
 
-typedef enum { C1, C2, C3, C4, Hlt, C3Res, All, Cnum } pm_Cstate_t;
-typedef struct pmStats {
-	uint64_t	pmNapCnt[Cnum];		/* Total nap calls for states */
-	uint64_t	pmNapTime[Cnum];	/* Total nap time for states */
-	uint64_t	pmNapC2HPET;		/* Total nap time for C2 using HPET for stats */
-	uint64_t	pmNapC4HPET;		/* Total nap time for C4 using HPET for stats */
-	uint64_t	pmNapHPETPops;		/* Number of times we detect HPET popping */
-	uint64_t	pmHPETRupt;		/* Number of HPET interruptions */
-	uint32_t	pmCurC3Res;		/* Current value of the C3 residency timer */
-	uint32_t	pmLastApic;		/* Last value of apic timer */
-	uint32_t	pmNewApic;		/* New value of apic timer */
-	uint64_t	pmHpetTim;		/* Time to next interrupt in HPET ticks */
-	uint64_t	pmHpetCmp;		/* HPET comparator */
-	uint64_t	pmHpetCfg;		/* HPET configuration */
-	uint64_t	pmLSNs;			/* (TEST) Last set nanotime */
-	uint64_t	pmLLHpet;		/* (TEST) Last loaded HPET */
-} pmStats_t;
-
 #define MAX_PSTATES	32			/* architectural limit */
 
-typedef enum { Cn1, Cn2, Cn3, Cn4, Cnmax } Cstate_number_t;
-typedef struct {
+typedef enum
+{
+    Cn1, Cn2, Cn3, Cn4, Cn5, Cn6, CnHlt, Cn0, CnRun, Cnmax
+} Cstate_number_t;
+
+typedef struct
+{
 	Cstate_number_t	number;
 	uint32_t	hint;
 } Cstate_hint_t;
@@ -70,6 +64,12 @@ typedef struct 	pmData pmData_t;
 #define cfgDat 		0xCFC
 #define lpcCfg 		(0x80000000 | (0 << 16) | (31 << 11) | (0 << 8))
 
+/*
+ * This value should be changed each time that pmDsipatch_t or pmCallBacks_t
+ * changes.
+ */
+#define PM_DISPATCH_VERSION	7
+
 /*
  * Dispatch table for functions that get installed when the power
  * management KEXT loads.
@@ -79,25 +79,37 @@ typedef struct
     /*
      * The following are the stepper table interfaces.
      */
-    void	(*pmsCPUMachineInit)(void);
-    void	(*pmsCPUInit)(void);
-    void	(*pmsCPUSet)(uint32_t sel);
-    void	(*pmsCPUConf)(void);
-    void	(*pmsCPURun)(uint32_t nstep);
-    uint32_t	(*pmsCPUQuery)(void);
-    uint32_t	(*pmsCPUPackageQuery)(void);
-    void	(*pmsCPUYellowFlag)(void);
-    void	(*pmsCPUGreenFlag)(void);
-    kern_return_t	(*pmsCPULoadVIDTable)(uint16_t *tablep, int nstates);
+    int			(*pmCPUStateInit)(void);
+    void		(*pmsInit)(void);
+    void		(*pmsStart)(void);
+    void		(*pmsPark)(void);
     kern_return_t	(*pmsCPUSetPStateLimit)(uint32_t limit);
 
+    /*
+     * The following are legacy stepper interfaces.
+     */
+    void		(*pmsRun)(uint32_t nstep);
+    kern_return_t	(*pmsBuild)(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc);
+    kern_return_t	(*pmsCPULoadVIDTable)(uint16_t *tablep, int nstates);
+
     /*
      * The following are the 'C' State interfaces.
      */
-    void	(*cstateInit)(void);
-    void	(*cstateMachineIdle)(uint32_t napCtl);
+    void		(*cstateInit)(void);
+    uint64_t		(*cstateMachineIdle)(uint64_t maxIdleDuration);
     kern_return_t	(*cstateTableSet)(Cstate_hint_t *tablep, unsigned int nstates);
-    uint32_t	(*cstateNapPolicy)(uint32_t forcenap, uint32_t napCtl);
+    uint64_t		(*GetDeadline)(x86_lcpu_t *lcpu);
+    uint64_t		(*SetDeadline)(x86_lcpu_t *lcpu, uint64_t);
+    void		(*Deadline)(x86_lcpu_t *lcpu);
+    boolean_t		(*exitIdle)(x86_lcpu_t *lcpu);
+    void		(*markCPURunning)(x86_lcpu_t *lcpu);
+    void		(*HPETInterrupt)(void);
+    int			(*pmCPUControl)(uint32_t cmd, void *datap);
+    void		(*pmCPUHalt)(void);
+    uint64_t		(*getMaxSnoop)(void);
+    void		(*setMaxBusDelay)(uint64_t time);
+    uint64_t		(*getMaxBusDelay)(void);
+    void		(*pmCPUSafeMode)(x86_lcpu_t *lcpu, uint32_t flags);
 } pmDispatch_t;
 
 typedef struct {
@@ -107,35 +119,63 @@ typedef struct {
     uint32_t		VIDTableCount;
     Cstate_hint_t	CStates[Cnmax];
     uint32_t		CStatesCount;
+    uint64_t		maxBusDelay;
 } pmInitState_t;
 
 typedef struct {
-    void	(*Park)(void);
-    void	(*Run)(uint32_t nstep);
-    void	(*RunLocal)(uint32_t nstep);
-    void	(*SetStep)(uint32_t nstep, int dir);
-    void	(*NapPolicy)(void);
-    kern_return_t	(*Build)(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc);
-    pmStats_t	*(*Stats)(void);
-    pmsd	*(*StepperData)(void);
-    uint64_t	*(*HPETAddr)(void);
+    uint64_t		*(*HPETAddr)(void);
     pmInitState_t	*InitState;
-    void	(*resetPop)(void);
+    int			(*setRTCPop)(uint64_t time);
+    void		(*resyncDeadlines)(void);
+    void		(*initComplete)(void);
+    x86_lcpu_t		*(*GetLCPU)(int cpu);
+    x86_core_t		*(*GetCore)(int cpu);
+    x86_pkg_t		*(*GetPackage)(int cpu);
+    x86_lcpu_t		*(*GetMyLCPU)(void);
+    x86_core_t		*(*GetMyCore)(void);
+    x86_pkg_t		*(*GetMyPackage)(void);
+    uint32_t		CoresPerPkg;
+    x86_pkg_t		*(*GetPkgRoot)(void);
+    void		(*LockCPUTopology)(int lock);
+    boolean_t		(*GetHibernate)(int cpu);
+    processor_t		(*LCPUtoProcessor)(int lcpu);
 } pmCallBacks_t;
 
 extern pmDispatch_t	*pmDispatch;
 
-extern uint32_t		maxBusDelay;
-extern uint32_t		C4C2SnoopDelay;
 extern uint32_t		forcenap;
 
 void power_management_init(void);
 void machine_nap_policy(void);
 kern_return_t Cstate_table_set(Cstate_hint_t *tablep, unsigned int nstates);
-void machine_idle_cstate(void);
-void pmRegister(pmDispatch_t *cpuFuncs, pmCallBacks_t *callbacks);
+void machine_idle_cstate(boolean_t halted);
+void pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs,
+		    pmCallBacks_t *callbacks);
 void pmUnRegister(pmDispatch_t *cpuFuncs);
+void pmCPUStateInit(void);
+uint64_t pmCPUGetDeadline(struct cpu_data *cpu);
+uint64_t pmCPUSetDeadline(struct cpu_data *cpu, uint64_t deadline);
+void pmCPUDeadline(struct cpu_data *cpu);
+boolean_t pmCPUExitIdle(struct cpu_data *cpu);
+void pmCPUMarkRunning(struct cpu_data *cpu);
+void pmHPETInterrupt(void);
+int pmCPUControl(uint32_t cmd, void *datap);
+void pmCPUHalt(uint32_t reason);
+
+#define PM_HALT_NORMAL		0		/* normal halt path */
+#define PM_HALT_DEBUG		1		/* debug code wants to halt */
+#define PM_HALT_PANIC		2		/* panic code wants to halt */
+
+void pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags);
+
+#define PM_SAFE_FL_NORMAL	0x00000001	/* put CPU into "normal" power mode */
+#define PM_SAFE_FL_SAFE		0x00000002	/* put CPU into a "safe" power mode */
+#define PM_SAFE_FL_PAUSE	0x00000010	/* pause execution on the CPU */
+#define PM_SAFE_FL_RESUME	0x00000020	/* resume execution on the CPU */
+
+extern int pmsafe_debug;
 
 #endif /* ASSEMBLER */
+
 #endif /* _I386_PMCPU_H_ */
 #endif /* KERNEL_PRIVATE */
diff --git a/osfmk/i386/pmap.c b/osfmk/i386/pmap.c
index 7224204c9..36dae2f3e 100644
--- a/osfmk/i386/pmap.c
+++ b/osfmk/i386/pmap.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -87,11 +93,14 @@
 #include <mach_kdb.h>
 #include <mach_ldebug.h>
 
+#include <libkern/OSAtomic.h>
+
 #include <mach/machine/vm_types.h>
 
 #include <mach/boolean.h>
 #include <kern/thread.h>
 #include <kern/zalloc.h>
+#include <kern/queue.h>
 
 #include <kern/lock.h>
 #include <kern/kalloc.h>
@@ -117,7 +126,11 @@
 #include <i386/machine_cpu.h>
 #include <i386/mp_slave_boot.h>
 #include <i386/seg.h>
+#include <i386/serial_io.h>
 #include <i386/cpu_capabilities.h>
+#include <i386/machine_routines.h>
+#include <i386/proc_reg.h>
+#include <i386/tsc.h>
 
 #if	MACH_KDB
 #include <ddb/db_command.h>
@@ -126,8 +139,6 @@
 #include <ddb/db_print.h>
 #endif	/* MACH_KDB */
 
-#include <kern/xpr.h>
-
 #include <vm/vm_protos.h>
 
 #include <i386/mp.h>
@@ -135,6 +146,13 @@
 
 #include <sys/kdebug.h>
 
+/* #define DEBUGINTERRUPTS 1  uncomment to ensure pmap callers have interrupts enabled */
+#ifdef DEBUGINTERRUPTS
+#define pmap_intr_assert() {if (processor_avail_count > 1 && !ml_get_interrupts_enabled()) panic("pmap interrupt assert %s, %d",__FILE__, __LINE__);}
+#else
+#define pmap_intr_assert()
+#endif
+
 #ifdef IWANTTODEBUG
 #undef	DEBUG
 #define DEBUG 1
@@ -142,6 +160,17 @@
 #include <i386/postcode.h>
 #endif /* IWANTTODEBUG */
 
+//#define PMAP_TRACES 1
+#ifdef	PMAP_TRACES
+boolean_t	pmap_trace = FALSE;
+#define PMAP_TRACE(x,a,b,c,d,e)						\
+	if (pmap_trace) {						\
+		KERNEL_DEBUG_CONSTANT(x,a,b,c,d,e);			\
+	}
+#else
+#define PMAP_TRACE(x,a,b,c,d,e)	KERNEL_DEBUG(x,a,b,c,d,e)
+#endif /* PMAP_TRACES */
+
 /*
  * Forward declarations for internal functions.
  */
@@ -153,26 +182,22 @@ void		pmap_expand_pdpt(
 			pmap_t		map,
 			vm_map_offset_t	v);
 
-void		pmap_expand(
-			pmap_t		map,
-			vm_map_offset_t	v);
-
-static void	pmap_remove_range(
+void	pmap_remove_range(
 			pmap_t		pmap,
 			vm_map_offset_t	va,
 			pt_entry_t	*spte,
 			pt_entry_t	*epte);
 
 void		phys_attribute_clear(
-			ppnum_t	phys,
+			ppnum_t		phys,
 			int		bits);
 
-boolean_t	phys_attribute_test(
-			ppnum_t	phys,
+int		phys_attribute_test(
+			ppnum_t		phys,
 			int		bits);
 
 void		phys_attribute_set(
-			ppnum_t	phys,
+			ppnum_t		phys,
 			int		bits);
 
 void		pmap_set_reference(
@@ -186,6 +211,7 @@ void		pmap_movepage(
 boolean_t	phys_page_exists(
 			ppnum_t pn);
 
+
 #ifdef PMAP_DEBUG
 void dump_pmap(pmap_t);
 void dump_4GB_pdpt(pmap_t p);
@@ -195,9 +221,22 @@ void dump_4GB_pdpt_thread(thread_t tp);
 #define	iswired(pte)	((pte) & INTEL_PTE_WIRED)
 
 int nx_enabled = 1;			/* enable no-execute protection */
+int allow_data_exec  = VM_ABI_32;	/* 32-bit apps may execute data by default, 64-bit apps may not */
+int allow_stack_exec = 0;		/* No apps may execute from the stack by default */
 
 int cpu_64bit  = 0;
 
+/*
+ * when spinning through pmap_remove
+ * ensure that we don't spend too much
+ * time with preemption disabled.
+ * I'm setting the current threshold
+ * to 20us
+ */
+#define MAX_PREEMPTION_LATENCY_NS 20000
+
+uint64_t max_preemption_latency_tsc = 0;
+
 
 /*
  *	Private data structures.
@@ -206,54 +245,202 @@ int cpu_64bit  = 0;
 /*
  *	For each vm_page_t, there is a list of all currently
  *	valid virtual mappings of that page.  An entry is
- *	a pv_entry_t; the list is the pv_table.
+ *	a pv_rooted_entry_t; the list is the pv_table.
+ *
+ *      N.B.  with the new combo rooted/hashed scheme it is
+ *      only possibly to remove individual non-rooted entries
+ *      if they are found via the hashed chains as there is no
+ *      way to unlink the singly linked hashed entries if navigated to
+ *      via the queue list off the rooted entries.  Think of it as
+ *      hash/walk/pull, keeping track of the prev pointer while walking
+ *      the singly linked hash list.  All of this is to save memory and
+ *      keep both types of pv_entries as small as possible.
  */
 
+/*
+
+PV HASHING Changes - JK 1/2007
+
+Pve's establish physical to virtual mappings.  These are used for aliasing of a 
+physical page to (potentially many) virtual addresses within pmaps. In the previous 
+implementation the structure of the pv_entries (each 16 bytes in size) was
+
 typedef struct pv_entry {
-	struct pv_entry	*next;		/* next pv_entry */
-	pmap_t		pmap;		/* pmap where mapping lies */
-	vm_map_offset_t	va;		/* virtual address for mapping */
+    struct pv_entry_t    next;
+    pmap_t                    pmap;
+    vm_map_offset_t   va;
 } *pv_entry_t;
 
-#define PV_ENTRY_NULL	((pv_entry_t) 0)
+An initial array of these is created at boot time, one per physical page of memory, 
+indexed by the physical page number. Additionally, a pool of entries is created from a 
+pv_zone to be used as needed by pmap_enter() when it is creating new mappings.  
+Originally, we kept this pool around because the code in pmap_enter() was unable to 
+block if it needed an entry and none were available - we'd panic.  Some time ago I 
+restructured the pmap_enter() code so that for user pmaps it can block while zalloc'ing 
+a pv structure and restart, removing a panic from the code (in the case of the kernel 
+pmap we cannot block and still panic, so, we keep a separate hot pool for use only on 
+kernel pmaps).  The pool has not been removed since there is a large performance gain 
+keeping freed pv's around for reuse and not suffering the overhead of zalloc for every new pv we need.
+
+As pmap_enter() created new mappings it linked the new pve's for them off the fixed 
+pv array for that ppn (off the next pointer).  These pve's are accessed for several 
+operations, one of them being address space teardown.  In that case, we basically do this
+
+	for (every page/pte in the space) {
+		calc pve_ptr from the ppn in the pte
+		for (every pv in the list for the ppn) {
+			if (this pv is for this pmap/vaddr) {
+				do housekeeping
+				unlink/free the pv
+			}
+		}
+	}
+
+The problem arose when we were running, say 8000 (or even 2000) apache or other processes 
+and one or all terminate. The list hanging off each pv array entry could have thousands of 
+entries.  We were continuously linearly searching each of these lists as we stepped through 
+the address space we were tearing down.  Because of the locks we hold, likely taking a cache 
+miss for each node,  and interrupt disabling for MP issues the system became completely 
+unresponsive for many seconds while we did this.
+
+Realizing that pve's are accessed in two distinct ways (linearly running the list by ppn 
+for operations like pmap_page_protect and finding and modifying/removing a single pve as 
+part of pmap_enter processing) has led to modifying the pve structures and databases.
+
+There are now two types of pve structures.  A "rooted" structure which is basically the 
+original structure accessed in an array by ppn, and a ''hashed'' structure accessed on a 
+hash list via a hash of [pmap, vaddr].  These have been designed with the two goals of 
+minimizing wired memory and making the lookup of a ppn faster.  Since a vast majority of 
+pages in the system are not aliased and hence represented by a single pv entry I've kept 
+the rooted entry size as small as possible because there is one of these dedicated for 
+every physical page of memory.  The hashed pve's are larger due to the addition of the hash 
+link and the ppn entry needed for matching while running the hash list to find the entry we 
+are looking for.  This way, only systems that have lots of aliasing (like 2000+ httpd procs) 
+will pay the extra memory price. Both structures have the same first three fields allowing 
+some simplification in the code.
+
+They have these shapes
+
+typedef struct pv_rooted_entry {
+        queue_head_t qlink;
+        vm_map_offset_t va;
+        pmap_t          pmap;
+} *pv_rooted_entry_t;
+
+
+typedef struct pv_hashed_entry {
+  queue_head_t qlink;
+  vm_map_offset_t va;
+  pmap_t        pmap;
+  ppnum_t ppn;
+  struct pv_hashed_entry *nexth;
+} *pv_hashed_entry_t;
+
+The main flow difference is that the code is now aware of the rooted entry and the hashed 
+entries.  Code that runs the pv list still starts with the rooted entry and then continues 
+down the qlink onto the hashed entries.  Code that is looking up a specific pv entry first 
+checks the rooted entry and then hashes and runs the hash list for the match. The hash list 
+lengths are much smaller than the original pv lists that contained all aliases for the specific ppn.
 
-pv_entry_t	pv_head_table;		/* array of entries, one per page */
+*/
+
+typedef struct pv_rooted_entry {     /* first three entries must match pv_hashed_entry_t */
+        queue_head_t qlink;
+	vm_map_offset_t	va;		/* virtual address for mapping */
+	pmap_t		pmap;		/* pmap where mapping lies */
+} *pv_rooted_entry_t;
+
+#define PV_ROOTED_ENTRY_NULL	((pv_rooted_entry_t) 0)
+
+pv_rooted_entry_t	pv_head_table;		/* array of entries, one per page */
+
+typedef struct pv_hashed_entry {     /* first three entries must match pv_rooted_entry_t */
+  queue_head_t qlink;
+  vm_map_offset_t va;
+  pmap_t        pmap;
+  ppnum_t ppn;
+  struct pv_hashed_entry *nexth;
+} *pv_hashed_entry_t;
+
+#define PV_HASHED_ENTRY_NULL ((pv_hashed_entry_t)0)
+
+#define NPVHASH 4095   /* MUST BE 2^N - 1 */
+pv_hashed_entry_t     *pv_hash_table;  /* hash lists */
+
+uint32_t npvhash = 0;
+
+/* #define PV_DEBUG 1   uncomment to enable some PV debugging code */
+#ifdef PV_DEBUG
+#define CHK_NPVHASH() if(0 == npvhash) panic("npvhash uninitialized");
+#else
+#define CHK_NPVHASH()
+#endif
 
 /*
  *	pv_list entries are kept on a list that can only be accessed
  *	with the pmap system locked (at SPLVM, not in the cpus_active set).
- *	The list is refilled from the pv_list_zone if it becomes empty.
+ *	The list is refilled from the pv_hashed_list_zone if it becomes empty.
  */
-pv_entry_t	pv_free_list;		/* free list at SPLVM */
-decl_simple_lock_data(,pv_free_list_lock)
+pv_rooted_entry_t	pv_free_list = PV_ROOTED_ENTRY_NULL;		/* free list at SPLVM */
+pv_hashed_entry_t	pv_hashed_free_list = PV_HASHED_ENTRY_NULL;
+pv_hashed_entry_t      pv_hashed_kern_free_list = PV_HASHED_ENTRY_NULL;
+decl_simple_lock_data(,pv_hashed_free_list_lock)
+decl_simple_lock_data(,pv_hashed_kern_free_list_lock)
+decl_simple_lock_data(,pv_hash_table_lock)
+
 int pv_free_count = 0;
-#define PV_LOW_WATER_MARK 5000
-#define PV_ALLOC_CHUNK 2000
+int pv_hashed_free_count = 0;
+int pv_kern_free_count = 0;
+int pv_hashed_kern_free_count = 0;
+#define PV_HASHED_LOW_WATER_MARK 5000
+#define PV_HASHED_KERN_LOW_WATER_MARK 100
+#define PV_HASHED_ALLOC_CHUNK 2000
+#define PV_HASHED_KERN_ALLOC_CHUNK 50
 thread_call_t  mapping_adjust_call;
 static thread_call_data_t  mapping_adjust_call_data;
-int mappingrecurse = 0;
-
-#define	PV_ALLOC(pv_e) { \
-	simple_lock(&pv_free_list_lock); \
-	if ((pv_e = pv_free_list) != 0) { \
-	    pv_free_list = pv_e->next; \
-            pv_free_count--; \
-            if (pv_free_count < PV_LOW_WATER_MARK) \
+uint32_t mappingrecurse = 0;
+
+#define	PV_HASHED_ALLOC(pvh_e) { \
+	simple_lock(&pv_hashed_free_list_lock); \
+	if ((pvh_e = pv_hashed_free_list) != 0) { \
+	  pv_hashed_free_list = (pv_hashed_entry_t)pvh_e->qlink.next;	\
+            pv_hashed_free_count--; \
+            if (pv_hashed_free_count < PV_HASHED_LOW_WATER_MARK) \
+              if (hw_compare_and_store(0,1,(u_int *)&mappingrecurse)) \
+                thread_call_enter(mapping_adjust_call); \
+	} \
+	simple_unlock(&pv_hashed_free_list_lock); \
+}
+
+#define	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt) {	\
+	simple_lock(&pv_hashed_free_list_lock); \
+	pvh_et->qlink.next = (queue_entry_t)pv_hashed_free_list;	\
+	pv_hashed_free_list = pvh_eh; \
+        pv_hashed_free_count += pv_cnt; \
+	simple_unlock(&pv_hashed_free_list_lock); \
+}
+
+#define	PV_HASHED_KERN_ALLOC(pvh_e) { \
+	simple_lock(&pv_hashed_kern_free_list_lock); \
+	if ((pvh_e = pv_hashed_kern_free_list) != 0) { \
+	  pv_hashed_kern_free_list = (pv_hashed_entry_t)pvh_e->qlink.next;	\
+            pv_hashed_kern_free_count--; \
+            if (pv_hashed_kern_free_count < PV_HASHED_KERN_LOW_WATER_MARK) \
               if (hw_compare_and_store(0,1,(u_int *)&mappingrecurse)) \
                 thread_call_enter(mapping_adjust_call); \
 	} \
-	simple_unlock(&pv_free_list_lock); \
+	simple_unlock(&pv_hashed_kern_free_list_lock); \
 }
 
-#define	PV_FREE(pv_e) { \
-	simple_lock(&pv_free_list_lock); \
-	pv_e->next = pv_free_list; \
-	pv_free_list = pv_e; \
-        pv_free_count++; \
-	simple_unlock(&pv_free_list_lock); \
+#define	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt) {	 \
+	simple_lock(&pv_hashed_kern_free_list_lock); \
+	pvh_et->qlink.next = (queue_entry_t)pv_hashed_kern_free_list;	\
+	pv_hashed_kern_free_list = pvh_eh; \
+        pv_hashed_kern_free_count += pv_cnt; \
+	simple_unlock(&pv_hashed_kern_free_list_lock); \
 }
 
-zone_t		pv_list_zone;		/* zone of pv_entry structures */
+zone_t		pv_hashed_list_zone;	/* zone of pv_hashed_entry structures */
 
 static zone_t pdpt_zone;
 
@@ -266,41 +453,49 @@ static zone_t pdpt_zone;
 char	*pv_lock_table;		/* pointer to array of bits */
 #define pv_lock_table_size(n)	(((n)+BYTE_SIZE-1)/BYTE_SIZE)
 
+char    *pv_hash_lock_table;
+#define pv_hash_lock_table_size(n)  (((n)+BYTE_SIZE-1)/BYTE_SIZE)
+
 /*
  *	First and last physical addresses that we maintain any information
  *	for.  Initialized to zero so that pmap operations done before
  *	pmap_init won't touch any non-existent structures.
  */
-pmap_paddr_t	vm_first_phys = (pmap_paddr_t) 0;
-pmap_paddr_t	vm_last_phys  = (pmap_paddr_t) 0;
 boolean_t	pmap_initialized = FALSE;/* Has pmap_init completed? */
 
 static struct vm_object kptobj_object_store;
 static vm_object_t kptobj;
 
 /*
- *	Index into pv_head table, its lock bits, and the modify/reference
- *	bits starting at vm_first_phys.
+ *	Index into pv_head table, its lock bits, and the modify/reference and managed bits
  */
 
-#define pa_index(pa)	(i386_btop(pa - vm_first_phys))
+#define pa_index(pa)	(i386_btop(pa))
+#define ppn_to_pai(ppn)	((int)ppn)
 
 #define pai_to_pvh(pai)		(&pv_head_table[pai])
 #define lock_pvh_pai(pai)	bit_lock(pai, (void *)pv_lock_table)
 #define unlock_pvh_pai(pai)	bit_unlock(pai, (void *)pv_lock_table)
 
+#define pvhashidx(pmap, va) (((uint32_t)pmap ^ ((uint32_t)((uint64_t)va >> PAGE_SHIFT) & 0xFFFFFFFF)) & npvhash)
+#define pvhash(idx)         (&pv_hash_table[idx])
+
+#define lock_hash_hash(hash)	        bit_lock(hash, (void *)pv_hash_lock_table)
+#define unlock_hash_hash(hash)	bit_unlock(hash, (void *)pv_hash_lock_table)
+
 /*
  *	Array of physical page attribites for managed pages.
  *	One byte per physical page.
  */
 char	*pmap_phys_attributes;
+unsigned int	last_managed_page = 0;
 
 /*
  *	Physical page attributes.  Copy bits from PTE definition.
  */
 #define	PHYS_MODIFIED	INTEL_PTE_MOD	/* page modified */
 #define	PHYS_REFERENCED	INTEL_PTE_REF	/* page referenced */
-#define PHYS_NCACHE	INTEL_PTE_NCACHE
+#define PHYS_MANAGED	INTEL_PTE_VALID /* page is managed */
 
 /*
  *	Amount of virtual memory mapped by one
@@ -314,7 +509,7 @@ uint64_t pde_mapped_size;
  */
 
 /*
- *	Locking Protocols:
+ *	Locking Protocols: (changed 2/2007 JK)
  *
  *	There are two structures in the pmap module that need locking:
  *	the pmaps themselves, and the per-page pv_lists (which are locked
@@ -324,87 +519,55 @@ uint64_t pde_mapped_size;
  *	pmap_remove_all and pmap_copy_on_write operate on a physical page
  *	basis and want to do the locking in the reverse order, i.e. lock
  *	a pv_list and then go through all the pmaps referenced by that list.
- *	To protect against deadlock between these two cases, the pmap_lock
- *	is used.  There are three different locking protocols as a result:
- *
- *  1.  pmap operations only (pmap_extract, pmap_access, ...)  Lock only
- *		the pmap.
  *
- *  2.  pmap-based operations (pmap_enter, pmap_remove, ...)  Get a read
- *		lock on the pmap_lock (shared read), then lock the pmap
- *		and finally the pv_lists as needed [i.e. pmap lock before
- *		pv_list lock.]
- *
- *  3.  pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
- *		Get a write lock on the pmap_lock (exclusive write); this
- *		also guaranteees exclusive access to the pv_lists.  Lock the
- *		pmaps as needed.
- *
- *	At no time may any routine hold more than one pmap lock or more than
- *	one pv_list lock.  Because interrupt level routines can allocate
- *	mbufs and cause pmap_enter's, the pmap_lock and the lock on the
- *	kernel_pmap can only be held at splhigh.
- */
-
-/*
- *	We raise the interrupt level to splvm, to block interprocessor
- *	interrupts during pmap operations.  We mark the cpu's cr3 inactive
- *	while interrupts are blocked.
+ *      The system wide pmap lock has been removed. Now, paths take a lock
+ *      on the pmap before changing its 'shape' and the reverse order lockers
+ *      (coming in by phys ppn) take a lock on the corresponding pv and then
+ *      retest to be sure nothing changed during the window before they locked
+ *      and can then run up/down the pv lists holding the list lock. This also
+ *      lets the pmap layer run (nearly completely) interrupt enabled, unlike
+ *      previously.
  */
-#define SPLVM(spl)	{						\
-	spl = splhigh();						\
-	CPU_CR3_MARK_INACTIVE();					\
-}
 
-#define SPLX(spl)	{						\
-	if (current_cpu_datap()->cpu_tlb_invalid)			\
-	    process_pmap_updates();					\
-	CPU_CR3_MARK_ACTIVE();						\
-	splx(spl);							\
-}
-	    
 /*
- *	Lock on pmap system
+ * pmap locking
  */
-lock_t	pmap_system_lock;
 
-#define PMAP_READ_LOCK(pmap, spl) {	\
-	SPLVM(spl);			\
-	lock_read(&pmap_system_lock);	\
+#define PMAP_LOCK(pmap) {	\
 	simple_lock(&(pmap)->lock);	\
 }
 
-#define PMAP_WRITE_LOCK(spl) {		\
-	SPLVM(spl);			\
-	lock_write(&pmap_system_lock);	\
-}
-
-#define PMAP_READ_UNLOCK(pmap, spl) {		\
+#define PMAP_UNLOCK(pmap) {		\
 	simple_unlock(&(pmap)->lock);		\
-	lock_read_done(&pmap_system_lock);	\
-	SPLX(spl);				\
 }
 
-#define PMAP_WRITE_UNLOCK(spl) {		\
-	lock_write_done(&pmap_system_lock);	\
-	SPLX(spl);				\
+/*
+ * PV locking
+ */
+
+#define LOCK_PVH(index)		{       \
+    mp_disable_preemption();           \
+    lock_pvh_pai(index);               \
 }
 
-#define PMAP_WRITE_TO_READ_LOCK(pmap) {		\
-	simple_lock(&(pmap)->lock);		\
-	lock_write_to_read(&pmap_system_lock);	\
+#define UNLOCK_PVH(index)  {      \
+    unlock_pvh_pai(index);        \
+    mp_enable_preemption();       \
 }
 
-#define LOCK_PVH(index)		lock_pvh_pai(index)
+/*
+ * PV hash locking
+ */
 
-#define UNLOCK_PVH(index)	unlock_pvh_pai(index)
+#define LOCK_PV_HASH(hash)         lock_hash_hash(hash)
+
+#define UNLOCK_PV_HASH(hash)       unlock_hash_hash(hash)
 
 #if	USLOCK_DEBUG
 extern int	max_lock_loops;
-extern int	disableSerialOuput;
 #define LOOP_VAR							\
 	unsigned int	loop_count;					\
-	loop_count = disableSerialOuput ? max_lock_loops		\
+	loop_count = disable_serial_output ? max_lock_loops		\
 					: max_lock_loops*100
 #define LOOP_CHECK(msg, pmap)						\
 	if (--loop_count == 0) {					\
@@ -447,7 +610,7 @@ struct zone	*pmap_zone;		/* zone of pmap structures */
 
 int		pmap_debug = 0;		/* flag for debugging prints */
 
-unsigned int	inuse_ptepages_count = 0;	/* debugging */
+unsigned int	inuse_ptepages_count = 0;
 
 addr64_t	kernel64_cr3;
 boolean_t	no_shared_cr3 = FALSE;	/* -no_shared_cr3 boot arg */
@@ -466,25 +629,47 @@ extern char end;
 
 static int nkpt;
 extern uint32_t lowGlo;
-extern void *version;
 
 pt_entry_t     *DMAP1, *DMAP2;
 caddr_t         DADDR1;
 caddr_t         DADDR2;
 
-#if  DEBUG_ALIAS
-#define PMAP_ALIAS_MAX 32
-struct pmap_alias {
-        vm_offset_t rpc;
-        pmap_t pmap;
-        vm_map_offset_t va;
-        int cookie;
-#define PMAP_ALIAS_COOKIE 0xdeadbeef
-} pmap_aliasbuf[PMAP_ALIAS_MAX];
-int pmap_alias_index = 0;
-extern vm_offset_t get_rpc();
+static inline
+void pmap_pvh_unlink(pv_hashed_entry_t pv);
+
+/*
+ * unlinks the pv_hashed_entry_t pvh from the singly linked hash chain.
+ * properly deals with the anchor.
+ * must be called with the hash locked, does not unlock it
+ */
+
+static inline
+void pmap_pvh_unlink(pv_hashed_entry_t pvh)
+{
+  pv_hashed_entry_t curh;
+  pv_hashed_entry_t *pprevh;
+  int pvhash_idx;
+
+  CHK_NPVHASH();
+  pvhash_idx = pvhashidx(pvh->pmap, pvh->va);
 
-#endif  /* DEBUG_ALIAS */
+  pprevh = pvhash(pvhash_idx);
+
+#if PV_DEBUG
+  if (NULL == *pprevh) panic("pvh_unlink null anchor"); /* JK DEBUG */
+#endif
+  curh = *pprevh;
+
+  while (PV_HASHED_ENTRY_NULL != curh) {
+    if (pvh == curh)
+      break;
+    pprevh = &curh->nexth;
+    curh = curh->nexth;
+  }
+  if (PV_HASHED_ENTRY_NULL == curh) panic("pmap_pvh_unlink no pvh");
+  *pprevh = pvh->nexth;
+  return;
+}
 
 /*
  * for legacy, returns the address of the pde entry.
@@ -561,7 +746,7 @@ pmap64_pdpt(pmap_t pmap, vm_map_offset_t vaddr)
 			  ((vm_offset_t)((vaddr>>PDPTSHIFT)&(NPDPTPG-1))));
 	}
 
-	return (0);
+	return (NULL);
 }
 
 /*
@@ -606,9 +791,18 @@ pmap64_pde(pmap_t pmap, vm_map_offset_t vaddr)
 			  ((vm_offset_t)((vaddr>>PDSHIFT)&(NPDPG-1))));
 	}
 
-	return (0);
+	return (NULL);
 }
 
+/*
+ * Because the page tables (top 3 levels) are mapped into per cpu windows,
+ * callers must either disable interrupts or disable preemption before calling
+ * one of the pte mapping routines (e.g. pmap_pte()) as the returned vaddr
+ * is in one of those mapped windows and that cannot be allowed to change until
+ * the caller is done using the returned pte pointer. When done, the caller
+ * restores interrupts or preemption to its previous state after which point the
+ * vaddr for the returned pte can no longer be used
+ */
 
 
 /*
@@ -634,10 +828,12 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
 	pde = pmap_pde(pmap,vaddr);
 
 	if (pde && ((*pde & INTEL_PTE_VALID))) {
-	  if (pmap == kernel_pmap) {
-	    return (vtopte(vaddr)); /* compat kernel still has pte's mapped */
-	  }
-
+	    if (pmap == kernel_pmap)
+	        return (vtopte(vaddr)); /* compat kernel still has pte's mapped */
+#if TESTING
+	    if (ml_get_interrupts_enabled() && get_preemption_level() == 0)
+	        panic("pmap_pte: unsafe call");
+#endif
 	        assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
 
 		newpf = *pde & PG_FRAME;
@@ -660,9 +856,9 @@ pmap_pte(pmap_t pmap, vm_map_offset_t vaddr)
 			  ((vm_offset_t)i386_btop(vaddr) & (NPTEPG-1)));
 	}
 
-	return(0);
+	return(NULL);
 }
-
+	
 
 /*
  *	Map memory at initialization.  The physical addresses being
@@ -695,7 +891,6 @@ pmap_map(
  *	Back-door routine for mapping kernel VM at initialization.  
  * 	Useful for mapping memory outside the range
  *      Sets no-cache, A, D.
- *	[vm_first_phys, vm_last_phys) (i.e., devices).
  *	Otherwise like pmap_map.
  */
 vm_offset_t
@@ -708,6 +903,7 @@ pmap_map_bd(
 {
 	pt_entry_t	template;
 	pt_entry_t	*pte;
+	spl_t           spl;
 
 	template = pa_to_pte(start_addr)
 		| INTEL_PTE_REF
@@ -725,11 +921,13 @@ pmap_map_bd(
 	    template |= INTEL_PTE_WRITE;
 
 	while (start_addr < end_addr) {
+	        spl = splhigh();
 		pte = pmap_pte(kernel_pmap, (vm_map_offset_t)virt);
 		if (pte == PT_ENTRY_NULL) {
 			panic("pmap_map_bd: Invalid kernel address\n");
 		}
 		pmap_store_pte(pte, template);
+		splx(spl);
 		pte_increment_pa(template);
 		virt += PAGE_SIZE;
 		start_addr += PAGE_SIZE;
@@ -746,6 +944,25 @@ extern  vm_offset_t     etext;
 extern  void            *sectHIBB;
 extern  int             sectSizeHIB;
 
+void
+pmap_cpu_init(void)
+{
+	/*
+	 * Here early in the life of a processor (from cpu_mode_init()).
+	 * If we're not in 64-bit mode, enable the global TLB feature.
+	 * Note: regardless of mode we continue to set the global attribute
+	 * bit in ptes for all (32-bit) global pages such as the commpage.
+	 */
+	if (!cpu_64bit) {
+		set_cr4(get_cr4() | CR4_PGE);
+	}
+
+	/*
+	 * Initialize the per-cpu, TLB-related fields.
+	 */
+	current_cpu_datap()->cpu_active_cr3 = kernel_pmap->pm_cr3;
+	current_cpu_datap()->cpu_tlb_invalid = FALSE;
+}
 
 vm_offset_t
 pmap_high_shared_remap(enum high_fixed_addresses e, vm_offset_t va, int sz)
@@ -754,8 +971,10 @@ pmap_high_shared_remap(enum high_fixed_addresses e, vm_offset_t va, int sz)
   pt_entry_t *ptep;
   pmap_paddr_t pa;
   int i;
+  spl_t s;
 
   assert(0 == (va & PAGE_MASK));  /* expecting page aligned */
+  s = splhigh();
   ptep = pmap_pte(kernel_pmap, (vm_map_offset_t)ve);
 
   for (i=0; i< sz; i++) {
@@ -769,6 +988,7 @@ pmap_high_shared_remap(enum high_fixed_addresses e, vm_offset_t va, int sz)
     va+= PAGE_SIZE;
     ptep++;
   }
+  splx(s);
   return ve;
 }
 
@@ -800,13 +1020,16 @@ pmap_init_high_shared(void)
 	vm_offset_t haddr;
         struct __gdt_desc_struct gdt_desc = {0,0,0};
 	struct __idt_desc_struct idt_desc = {0,0,0};
+	spl_t s;
 #if MACH_KDB
 	struct i386_tss *ttss;
 #endif
 
 	kprintf("HIGH_MEM_BASE 0x%x fixed per-cpu begin 0x%x\n", 
 		HIGH_MEM_BASE,pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN));
+	s = splhigh();
 	pte_unique_base = pmap_pte(kernel_pmap, (vm_map_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN));
+	splx(s);
 
 	if (i386_btop(&hi_remap_etext - &hi_remap_text + 1) >
 				HIGH_FIXED_TRAMPS_END - HIGH_FIXED_TRAMPS + 1)
@@ -911,6 +1134,7 @@ pmap_bootstrap(
 	int i;
 	int wpkernel, boot_arg;
 	pdpt_entry_t *pdpt;
+	spl_t s;
 
 	vm_last_addr = VM_MAX_KERNEL_ADDRESS;	/* Set the highest address
 						 * known to VM */
@@ -924,7 +1148,7 @@ pmap_bootstrap(
 	kernel_pmap = &kernel_pmap_store;
 	kernel_pmap->ref_count = 1;
 	kernel_pmap->nx_enabled = FALSE;
-	kernel_pmap->pm_64bit = 0;
+	kernel_pmap->pm_task_map = TASK_MAP_32BIT;
 	kernel_pmap->pm_obj = (vm_object_t) NULL;
 	kernel_pmap->dirbase = (pd_entry_t *)((unsigned int)IdlePTD | KERNBASE);
 	kernel_pmap->pdirbase = (pmap_paddr_t)((int)IdlePTD);
@@ -953,7 +1177,9 @@ pmap_bootstrap(
 	high_shared_pde = *pmap_pde(kernel_pmap, HIGH_MEM_BASE);
 	/* make sure G bit is on for high shared pde entry */
 	high_shared_pde |= INTEL_PTE_GLOBAL;
+	s = splhigh();
 	pmap_store_pte(pmap_pde(kernel_pmap, HIGH_MEM_BASE), high_shared_pde);
+	splx(s);
 
 	nkpt = NKPT;
 	inuse_ptepages_count += NKPT;
@@ -983,21 +1209,28 @@ pmap_bootstrap(
 	SYSMAP(caddr_t, DMAP1, DADDR1, 1);
 	SYSMAP(caddr_t, DMAP2, DADDR2, 1);  /* XXX temporary - can remove */
 
-
-	lock_init(&pmap_system_lock,
-		  FALSE,		/* NOT a sleep lock */
-		  0, 0);
-
 	virtual_avail = va;
 
+	if (PE_parse_boot_arg("npvhash", &npvhash)) {
+	  if (0 != ((npvhash+1) & npvhash)) {
+	    kprintf("invalid hash %d, must be ((2^N)-1), using default %d\n",npvhash,NPVHASH);
+	    npvhash = NPVHASH;
+	  }
+	} else {
+	  npvhash = NPVHASH;
+	}
+	printf("npvhash=%d\n",npvhash);
+
 	wpkernel = 1;
 	if (PE_parse_boot_arg("wpkernel", &boot_arg)) {
 		if (boot_arg == 0)
 			wpkernel = 0;
 	}
 
+	s = splhigh();
+
 	/* Remap kernel text readonly unless the "wpkernel" boot-arg is present
-	 * and set to 0.
+ 	 * and set to 0.
 	 */
 	if (wpkernel)
 	{
@@ -1019,14 +1252,17 @@ pmap_bootstrap(
 
 	/* map lowmem global page into fixed addr 0x2000 */
 	if (0 == (pte = pmap_pte(kernel_pmap,0x2000))) panic("lowmem pte");
-
+	assert(0 == ((vm_offset_t) &lowGlo & PAGE_MASK)); /* make sure it is defined on page boundary */
 	pmap_store_pte(pte, kvtophys((vm_offset_t)&lowGlo)|INTEL_PTE_VALID|INTEL_PTE_REF|INTEL_PTE_MOD|INTEL_PTE_WIRED|INTEL_PTE_RW);
+	splx(s);
 	flush_tlb();
 
 	simple_lock_init(&kernel_pmap->lock, 0);
-	simple_lock_init(&pv_free_list_lock, 0);
+	simple_lock_init(&pv_hashed_free_list_lock, 0);
+	simple_lock_init(&pv_hashed_kern_free_list_lock, 0);
+	simple_lock_init(&pv_hash_table_lock,0);
 
-        pmap_init_high_shared();
+	pmap_init_high_shared();
 
 	pde_mapped_size = PDE_MAPPED_SIZE;
 
@@ -1062,20 +1298,20 @@ pmap_bootstrap(
 	  pmap_store_pte((ppml4+KERNEL_UBER_PML4_INDEX), *(ppml4+0));
 
 	  kernel64_cr3 = (addr64_t) kernel_pmap->pm_cr3;
-	  cpu_IA32e_enable(current_cpu_datap());
-	  current_cpu_datap()->cpu_is64bit = TRUE;
-	  /* welcome to a 64 bit world */
 
-	  /* Re-initialize and load descriptors */
+	  /* Re-initialize descriptors and prepare to switch modes */
 	  cpu_desc_init64(&cpu_data_master, TRUE);
-	  cpu_desc_load64(&cpu_data_master);
-	  fast_syscall_init64();
+	  current_cpu_datap()->cpu_is64bit = TRUE;
+	  current_cpu_datap()->cpu_active_cr3 = kernel64_cr3;
 
 	  pde_mapped_size = 512*4096 ; 
 
 	  ml_set_interrupts_enabled(istate);
-
 	}
+
+	/* Set 64-bit mode if required. */
+	cpu_mode_init(&cpu_data_master);
+
 	kernel_pmap->pm_hold = (vm_offset_t)kernel_pmap->pm_pml4;
 
 	kprintf("Kernel virtual space from 0x%x to 0x%x.\n",
@@ -1093,7 +1329,13 @@ pmap_bootstrap(
 	 */
 	if (PE_parse_boot_arg("-no_shared_cr3", &no_shared_cr3)) {
 		kprintf("Shared kernel address space disabled\n");
-	}
+	}	
+
+#ifdef	PMAP_TRACES
+	if (PE_parse_boot_arg("-pmap_trace", &pmap_trace)) {
+		kprintf("Kernel traces for pmap operations enabled\n");
+	}	
+#endif	/* PMAP_TRACES */
 }
 
 void
@@ -1117,18 +1359,23 @@ pmap_init(void)
 	vm_offset_t		addr;
 	register vm_size_t	s;
 	vm_map_offset_t		vaddr;
-	ppnum_t			ppn;
+	ppnum_t ppn;
 
 	/*
 	 *	Allocate memory for the pv_head_table and its lock bits,
 	 *	the modify bit array, and the pte_page table.
 	 */
 
-	/* zero bias all these arrays now instead of off avail_start
-	   so we cover all memory */
+	/*
+	 * zero bias all these arrays now instead of off avail_start
+	 * so we cover all memory
+	 */
+
 	npages = i386_btop(avail_end);
-	s = (vm_size_t) (sizeof(struct pv_entry) * npages
-				+ pv_lock_table_size(npages)
+	s = (vm_size_t) (sizeof(struct pv_rooted_entry) * npages
+			 + (sizeof (struct pv_hashed_entry_t *) * (npvhash+1))
+			 + pv_lock_table_size(npages)
+			 + pv_hash_lock_table_size((npvhash+1))
 				+ npages);
 
 	s = round_page(s);
@@ -1137,16 +1384,47 @@ pmap_init(void)
 
 	memset((char *)addr, 0, s);
 
+#if PV_DEBUG
+	if (0 == npvhash) panic("npvhash not initialized");
+#endif
+
 	/*
 	 *	Allocate the structures first to preserve word-alignment.
 	 */
-	pv_head_table = (pv_entry_t) addr;
+	pv_head_table = (pv_rooted_entry_t) addr;
 	addr = (vm_offset_t) (pv_head_table + npages);
 
+	pv_hash_table = (pv_hashed_entry_t *)addr;
+	addr = (vm_offset_t) (pv_hash_table + (npvhash + 1));
+
 	pv_lock_table = (char *) addr;
 	addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages));
 
+	pv_hash_lock_table = (char *) addr;
+	addr = (vm_offset_t) (pv_hash_lock_table + pv_hash_lock_table_size((npvhash+1)));
+
 	pmap_phys_attributes = (char *) addr;
+	{
+	        unsigned int i;
+		unsigned int pn;
+		ppnum_t  last_pn;
+		pmap_memory_region_t *pmptr = pmap_memory_regions;
+
+		last_pn = i386_btop(avail_end);
+
+		for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
+		        if (pmptr->type == kEfiConventionalMemory) {
+			        for (pn = pmptr->base; pn <= pmptr->end; pn++) {
+				        if (pn < last_pn) {
+					        pmap_phys_attributes[pn] |= PHYS_MANAGED;
+
+						if (pn > last_managed_page)
+						        last_managed_page = pn;
+					}
+				}
+			}
+		}
+	}
 
 	/*
 	 *	Create the zone of physical maps,
@@ -1154,24 +1432,13 @@ pmap_init(void)
 	 */
 	s = (vm_size_t) sizeof(struct pmap);
 	pmap_zone = zinit(s, 400*s, 4096, "pmap"); /* XXX */
-	s = (vm_size_t) sizeof(struct pv_entry);
-	pv_list_zone = zinit(s, 10000*s, 4096, "pv_list"); /* XXX */
+	s = (vm_size_t) sizeof(struct pv_hashed_entry);
+	pv_hashed_list_zone = zinit(s, 10000*s, 4096, "pv_list"); /* XXX */
 	s = 63;
 	pdpt_zone = zinit(s, 400*s, 4096, "pdpt"); /* XXX */
 
-	/*
-	 *	Only now, when all of the data structures are allocated,
-	 *	can we set vm_first_phys and vm_last_phys.  If we set them
-	 *	too soon, the kmem_alloc_wired above will try to use these
-	 *	data structures and blow up.
-	 */
-
-	/* zero bias this now so we cover all memory */
-	vm_first_phys = 0;
-	vm_last_phys = avail_end;
-
 	kptobj = &kptobj_object_store;
-	_vm_object_allocate((vm_object_size_t)NKPDE, kptobj);
+	_vm_object_allocate((vm_object_size_t)(NPGPTD*NPTDPG), kptobj);
 	kernel_pmap->pm_obj = kptobj;
 
 	/* create pv entries for kernel pages mapped by low level
@@ -1180,24 +1447,26 @@ pmap_init(void)
 
 	vaddr = (vm_map_offset_t)0;
 	for (ppn = 0; ppn < i386_btop(avail_start) ; ppn++ ) {
-	  pv_entry_t	pv_e;
+	  pv_rooted_entry_t	pv_e;
 
 	  pv_e = pai_to_pvh(ppn);
 	  pv_e->va = vaddr;
 	  vaddr += PAGE_SIZE;
-	  kernel_pmap->stats.resident_count++;
 	  pv_e->pmap = kernel_pmap;
-	  pv_e->next = PV_ENTRY_NULL;
+	  queue_init(&pv_e->qlink);
 	}
 
 	pmap_initialized = TRUE;
 
 	/*
-	 *	Initializie pmap cache.
+	 *	Initialize pmap cache.
 	 */
 	pmap_cache_list = PMAP_NULL;
 	pmap_cache_count = 0;
 	simple_lock_init(&pmap_cache_lock, 0);
+
+	max_preemption_latency_tsc = tmrCvt((uint64_t)MAX_PREEMPTION_LATENCY_NS, tscFCvtn2t);
+
 }
 
 void
@@ -1213,37 +1482,77 @@ x86_lowmem_free(void)
 }
 
 
-#define valid_page(x) (pmap_initialized && pmap_valid_page(x))
+#define managed_page(x) ( (unsigned int)x <= last_managed_page && (pmap_phys_attributes[x] & PHYS_MANAGED) )
 
+/*
+ * this function is only used for debugging fron the vm layer
+ */
 boolean_t
 pmap_verify_free(
 		 ppnum_t pn)
 {
-        pmap_paddr_t	phys;
-	pv_entry_t	pv_h;
+	pv_rooted_entry_t	pv_h;
 	int		pai;
-	spl_t		spl;
 	boolean_t	result;
 
 	assert(pn != vm_page_fictitious_addr);
-	phys = (pmap_paddr_t)i386_ptob(pn);
+
 	if (!pmap_initialized)
 		return(TRUE);
 
-	if (!pmap_valid_page(pn))
-		return(FALSE);
+	if (pn == vm_page_guard_addr)
+		return TRUE;
 
-	PMAP_WRITE_LOCK(spl);
+	pai = ppn_to_pai(pn);
+	if (!managed_page(pai))
+		return(FALSE);
+	pv_h = pai_to_pvh(pn);
+	result = (pv_h->pmap == PMAP_NULL);
+	return(result);
+}
 
-	pai = pa_index(phys);
-	pv_h = pai_to_pvh(pai);
+boolean_t
+pmap_is_empty(
+       pmap_t          pmap,
+       vm_map_offset_t vstart,
+       vm_map_offset_t vend)
+{
+	vm_map_offset_t offset;
+	ppnum_t         phys_page;
 
-	result = (pv_h->pmap == PMAP_NULL);
-	PMAP_WRITE_UNLOCK(spl);
+	if (pmap == PMAP_NULL) {
+		return TRUE;
+	}
+	for (offset = vstart;
+	     offset < vend;
+	     offset += PAGE_SIZE_64) {
+		phys_page = pmap_find_phys(pmap, offset);
+		if (phys_page) {
+			if (pmap != kernel_pmap &&
+			    pmap->pm_task_map == TASK_MAP_32BIT &&
+			    offset >= HIGH_MEM_BASE) {
+				/*
+				 * The "high_shared_pde" is used to share
+				 * the entire top-most 2MB of address space
+				 * between the kernel and all 32-bit tasks.
+				 * So none of this can be removed from 32-bit
+				 * tasks.
+				 * Let's pretend there's nothing up
+				 * there...
+				 */
+				return TRUE;
+			}
+			kprintf("pmap_is_empty(%p,0x%llx,0x%llx): "
+				"page %d at 0x%llx\n",
+				pmap, vstart, vend, phys_page, offset);
+			return FALSE;
+		}
+	}
 
-	return(result);
+	return TRUE;
 }
 
+
 /*
  *	Create and return a physical map.
  *
@@ -1259,18 +1568,21 @@ pmap_verify_free(
 pmap_t
 pmap_create(
 	    vm_map_size_t	sz,
-	    boolean_t  		is_64bit)
+	    boolean_t		is_64bit)
 {
-        register pmap_t			p;
+	pmap_t			p;
 	int		i;
 	vm_offset_t	va;
 	vm_size_t	size;
 	pdpt_entry_t    *pdpt;
 	pml4_entry_t    *pml4p;
-	int template;
 	pd_entry_t      *pdp;
+	int template;
 	spl_t s;
 
+	PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
+		   (int) (sz>>32), (int) sz, (int) is_64bit, 0, 0);
+
 	size = (vm_size_t) sz;
 
 	/*
@@ -1283,101 +1595,149 @@ pmap_create(
 
 	p = (pmap_t) zalloc(pmap_zone);
 	if (PMAP_NULL == p)
-	  panic("pmap_create zalloc");
+		panic("pmap_create zalloc");
 
 	/* init counts now since we'll be bumping some */
 	simple_lock_init(&p->lock, 0);
 	p->stats.resident_count = 0;
+	p->stats.resident_max = 0;
 	p->stats.wired_count = 0;
 	p->ref_count = 1;
 	p->nx_enabled = 1;
-	p->pm_64bit = is_64bit;
-	p->pm_kernel_cr3 = FALSE;
 	p->pm_shared = FALSE;
 
+	assert(!is_64bit || cpu_64bit);
+	p->pm_task_map = is_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;;
+
 	if (!cpu_64bit) {
-	  /* legacy 32 bit setup */
-	  /* in the legacy case the pdpt layer is hardwired to 4 entries and each
-	   * entry covers 1GB of addr space */
-	  if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->dirbase), NBPTD))
-	    panic("pmap_create kmem_alloc_wired");
-	  p->pm_hold = (vm_offset_t)zalloc(pdpt_zone);
-	  if ((vm_offset_t)NULL == p->pm_hold) {
-	    panic("pdpt zalloc");
-	  }
-	  pdpt = (pdpt_entry_t *) (( p->pm_hold + 31) & ~31);
-	  p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)pdpt);
-	  if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPGPTD*NPTDPG))))
-	    panic("pmap_create vm_object_allocate");
+		/* legacy 32 bit setup */
+		/* in the legacy case the pdpt layer is hardwired to 4 entries and each
+		 * entry covers 1GB of addr space */
+		if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->dirbase), NBPTD))
+			panic("pmap_create kmem_alloc_wired");
+		p->pm_hold = (vm_offset_t)zalloc(pdpt_zone);
+		if ((vm_offset_t)NULL == p->pm_hold) {
+			panic("pdpt zalloc");
+		}
+		pdpt = (pdpt_entry_t *) (( p->pm_hold + 31) & ~31);
+		p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)pdpt);
+		if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPGPTD*NPTDPG))))
+			panic("pmap_create vm_object_allocate");
 
-	  memset((char *)p->dirbase, 0, NBPTD);
+		memset((char *)p->dirbase, 0, NBPTD);
 
-	  va = (vm_offset_t)p->dirbase;
-	  p->pdirbase = kvtophys(va);
+		va = (vm_offset_t)p->dirbase;
+		p->pdirbase = kvtophys(va);
 
-	  template = cpu_64bit ? INTEL_PTE_VALID|INTEL_PTE_RW|INTEL_PTE_USER|INTEL_PTE_REF : INTEL_PTE_VALID;
-	  for (i = 0; i< NPGPTD; i++, pdpt++) {
-	    pmap_paddr_t pa;
-	    pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
-	    pmap_store_pte(pdpt, pa | template);
-	  }
+		template = cpu_64bit ? INTEL_PTE_VALID|INTEL_PTE_RW|INTEL_PTE_USER|INTEL_PTE_REF : INTEL_PTE_VALID;
+		for (i = 0; i< NPGPTD; i++, pdpt++ ) {
+			pmap_paddr_t pa;
+			pa = (pmap_paddr_t) kvtophys(va + i386_ptob(i));
+			pmap_store_pte(pdpt, pa | template);
+		}
 
-	  /* map the high shared pde */
-	  pmap_store_pte(pmap_pde(p, HIGH_MEM_BASE), high_shared_pde);
+		/* map the high shared pde */
+		s = splhigh();
+		pmap_store_pte(pmap_pde(p, HIGH_MEM_BASE), high_shared_pde);
+		splx(s);
 
 	} else {
+	        /* 64 bit setup  */
 
-	  /* 64 bit setup  */
+	        /* alloc the pml4 page in kernel vm */
+	        if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->pm_hold), PAGE_SIZE))
+		        panic("pmap_create kmem_alloc_wired pml4");
 
-	  /* alloc the pml4 page in kernel vm */
-	  if (KERN_SUCCESS != kmem_alloc_wired(kernel_map, (vm_offset_t *)(&p->pm_hold), PAGE_SIZE))
-	    panic("pmap_create kmem_alloc_wired pml4");
+	        memset((char *)p->pm_hold, 0, PAGE_SIZE);
+		p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_hold);
 
-	  memset((char *)p->pm_hold, 0, PAGE_SIZE);
-	  p->pm_cr3 = (pmap_paddr_t)kvtophys((vm_offset_t)p->pm_hold);
+	        vm_page_lock_queues();
+		inuse_ptepages_count++;
+		vm_page_unlock_queues();
 
-	  inuse_ptepages_count++;
-	  p->stats.resident_count++;
-	  p->stats.wired_count++;
+		/* allocate the vm_objs to hold the pdpt, pde and pte pages */
 
-	/* allocate the vm_objs to hold the pdpt, pde and pte pages */
+		if (NULL == (p->pm_obj_pml4 = vm_object_allocate((vm_object_size_t)(NPML4PGS))))
+			panic("pmap_create pdpt obj");
 
-	if (NULL == (p->pm_obj_pml4 = vm_object_allocate((vm_object_size_t)(NPML4PGS))))
-	  panic("pmap_create pdpt obj");
+		if (NULL == (p->pm_obj_pdpt = vm_object_allocate((vm_object_size_t)(NPDPTPGS))))
+			panic("pmap_create pdpt obj");
 
-	if (NULL == (p->pm_obj_pdpt = vm_object_allocate((vm_object_size_t)(NPDPTPGS))))
-	  panic("pmap_create pdpt obj");
+		if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPDEPGS))))
+			panic("pmap_create pte obj");
 
-	if (NULL == (p->pm_obj = vm_object_allocate((vm_object_size_t)(NPDEPGS))))
-	  panic("pmap_create pte obj");
+		/* uber space points to uber mapped kernel */
+		s = splhigh();
+		pml4p = pmap64_pml4(p, 0ULL);
+		pmap_store_pte((pml4p+KERNEL_UBER_PML4_INDEX),*kernel_pmap->pm_pml4);
 
-	/* uber space points to uber mapped kernel */
-	s = splhigh();
-	pml4p = pmap64_pml4(p, 0ULL);
-	pmap_store_pte((pml4p+KERNEL_UBER_PML4_INDEX),*kernel_pmap->pm_pml4);
-	if (!is_64bit) {
-	  while ((pdp = pmap64_pde(p, (uint64_t)HIGH_MEM_BASE)) == PD_ENTRY_NULL) {
-	    splx(s);
-	    pmap_expand_pdpt(p, (uint64_t)HIGH_MEM_BASE); /* need room for another pde entry */
-	    s = splhigh();
-	  }
-	  pmap_store_pte(pdp, high_shared_pde);
-	}
 
-	splx(s);
+		if (!is_64bit) {
+			while ((pdp = pmap64_pde(p, (uint64_t)HIGH_MEM_BASE)) == PD_ENTRY_NULL) {
+				splx(s);
+				pmap_expand_pdpt(p, (uint64_t)HIGH_MEM_BASE); /* need room for another pde entry */
+				s = splhigh();
+			}
+			pmap_store_pte(pdp, high_shared_pde);
+		}
+		splx(s);
 	}
 
+	PMAP_TRACE(PMAP_CODE(PMAP__CREATE) | DBG_FUNC_START,
+		   (int) p, is_64bit, 0, 0, 0);
+
 	return(p);
 }
 
+/*
+ * The following routines implement the shared address optmization for 64-bit
+ * users with a 4GB page zero.
+ *
+ * pmap_set_4GB_pagezero()
+ *	is called in the exec and fork paths to mirror the kernel's
+ *	mapping in the bottom 4G of the user's pmap. The task mapping changes
+ *	from TASK_MAP_64BIT to TASK_MAP_64BIT_SHARED. This routine returns
+ *	without doing anything if the -no_shared_cr3 boot-arg is set.
+ *
+ * pmap_clear_4GB_pagezero()
+ *	is called in the exec/exit paths to undo this mirror. The task mapping
+ *	reverts to TASK_MAP_64BIT. In addition, we switch to the kernel's
+ *	CR3 by calling pmap_load_kernel_cr3(). 
+ *
+ * pmap_load_kernel_cr3()
+ *	loads cr3 with the kernel's page table. In addition to being called
+ * 	by pmap_clear_4GB_pagezero(), it is used both prior to teardown and
+ *	when we go idle in the context of a shared map.
+ *
+ * Further notes on per-cpu data used:
+ *
+ *	cpu_kernel_cr3	is the cr3 for the kernel's pmap.
+ *			This is loaded in a trampoline on entering the kernel
+ *			from a 32-bit user (or non-shared-cr3 64-bit user).
+ *	cpu_task_cr3	is the cr3 for the current thread.
+ *			This is loaded in a trampoline as we exit the kernel.
+ *	cpu_active_cr3	reflects the cr3 currently loaded.
+ *			However, the low order bit is set when the
+ *			processor is idle or interrupts are disabled
+ *			while the system pmap lock is held. It is used by
+ *			tlb shoot-down.
+ *	cpu_task_map	indicates whether the task cr3 belongs to
+ *			a 32-bit, a 64-bit or a 64-bit shared map.
+ *			The latter allows the avoidance of the cr3 load
+ *			on kernel entry and exit.
+ *	cpu_tlb_invalid	set TRUE when a tlb flush is requested.
+ *			If the cr3 is "inactive" (the cpu is idle or the
+ *			system-wide pmap lock is held) this not serviced by
+ *			an IPI but at time when the cr3 becomes "active".
+ */ 
+
 void
 pmap_set_4GB_pagezero(pmap_t p)
 {
-	int		spl;
 	pdpt_entry_t	*user_pdptp;
 	pdpt_entry_t	*kern_pdptp;
 
-	assert(p->pm_64bit);
+	assert(p->pm_task_map != TASK_MAP_32BIT);
 
 	/* Kernel-shared cr3 may be disabled by boot arg. */
 	if (no_shared_cr3)
@@ -1386,64 +1746,59 @@ pmap_set_4GB_pagezero(pmap_t p)
 	/*
 	 * Set the bottom 4 3rd-level pte's to be the kernel's.
 	 */
-	spl = splhigh();
+	PMAP_LOCK(p);
 	while ((user_pdptp = pmap64_pdpt(p, 0x0)) == PDPT_ENTRY_NULL) {
-		splx(spl);
+		PMAP_UNLOCK(p);
 		pmap_expand_pml4(p, 0x0);
-		spl = splhigh();
+		PMAP_LOCK(p);
 	}
 	kern_pdptp = kernel_pmap->pm_pdpt;
 	pmap_store_pte(user_pdptp+0, *(kern_pdptp+0));
 	pmap_store_pte(user_pdptp+1, *(kern_pdptp+1));
 	pmap_store_pte(user_pdptp+2, *(kern_pdptp+2));
 	pmap_store_pte(user_pdptp+3, *(kern_pdptp+3));
-
-	p->pm_kernel_cr3 = TRUE;
-
-	splx(spl);
-
-}
-
-void
-pmap_load_kernel_cr3(void)
-{
-	uint32_t	kernel_cr3;
-
-	assert(!ml_get_interrupts_enabled());
-
-	/*
-	 * Reload cr3 with the true kernel cr3.
-	 * Note: kernel's pml4 resides below 4GB physical.
-	 */
-	kernel_cr3 = current_cpu_datap()->cpu_kernel_cr3;
-	set_cr3(kernel_cr3);
-	current_cpu_datap()->cpu_active_cr3 = kernel_cr3;
-	current_cpu_datap()->cpu_task_map = TASK_MAP_32BIT;
-	current_cpu_datap()->cpu_tlb_invalid = FALSE;
-	__asm__ volatile("mfence");
+	p->pm_task_map = TASK_MAP_64BIT_SHARED;
+	PMAP_UNLOCK(p);
 }
 
 void
 pmap_clear_4GB_pagezero(pmap_t p)
 {
-	int		spl;
 	pdpt_entry_t	*user_pdptp;
 
-	if (!p->pm_kernel_cr3)
+	if (p->pm_task_map != TASK_MAP_64BIT_SHARED)
 		return;
 
-	spl = splhigh();
+	PMAP_LOCK(p);
+
+	p->pm_task_map = TASK_MAP_64BIT;
+
+	pmap_load_kernel_cr3();
+
 	user_pdptp = pmap64_pdpt(p, 0x0);
 	pmap_store_pte(user_pdptp+0, 0);
 	pmap_store_pte(user_pdptp+1, 0);
 	pmap_store_pte(user_pdptp+2, 0);
 	pmap_store_pte(user_pdptp+3, 0);
 
-	p->pm_kernel_cr3 = FALSE;
+	PMAP_UNLOCK(p);
+}
 
-	pmap_load_kernel_cr3();
+void
+pmap_load_kernel_cr3(void)
+{
+	uint64_t	kernel_cr3;
 
-	splx(spl);
+	assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+
+	/*
+	 * Reload cr3 with the true kernel cr3.
+	 */
+	kernel_cr3 = current_cpu_datap()->cpu_kernel_cr3;
+	set64_cr3(kernel_cr3);
+	current_cpu_datap()->cpu_active_cr3 = kernel_cr3;
+	current_cpu_datap()->cpu_tlb_invalid = FALSE;
+	__asm__ volatile("mfence");
 }
 
 /*
@@ -1457,17 +1812,17 @@ pmap_destroy(
 	register pmap_t	p)
 {
 	register int		c;
-	spl_t                   s;
-#if 0
-	register pt_entry_t	*pdep;
-	register vm_page_t	m;
-#endif
 
 	if (p == PMAP_NULL)
 		return;
-	SPLVM(s);
-	simple_lock(&p->lock);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_START,
+		   (int) p, 0, 0, 0, 0);
+
+	PMAP_LOCK(p);
+
 	c = --p->ref_count;
+
 	if (c == 0) {
 		/* 
 		 * If some cpu is not using the physical pmap pointer that it
@@ -1476,84 +1831,57 @@ pmap_destroy(
 		 * physically on the right pmap:
 		 */
 		PMAP_UPDATE_TLBS(p,
-				 VM_MIN_ADDRESS,
-				 VM_MAX_KERNEL_ADDRESS);
-
+				 0x0ULL,
+				 0xFFFFFFFFFFFFF000ULL);
 	}
-	simple_unlock(&p->lock);
-	SPLX(s);
+
+	PMAP_UNLOCK(p);
 
 	if (c != 0) {
-	    return;	/* still in use */
+		PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
+			   (int) p, 1, 0, 0, 0);
+	        return;	/* still in use */
 	}
 
 	/*
 	 *	Free the memory maps, then the
 	 *	pmap structure.
 	 */
-
 	if (!cpu_64bit) {
-#if 0
-	pdep = (pt_entry_t *)p->dirbase;
+	        vm_page_lock_queues();
+		inuse_ptepages_count -= p->pm_obj->resident_page_count;
+		vm_page_unlock_queues();
 
-	while (pdep < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)]) {
-	    int ind;
+		kmem_free(kernel_map, (vm_offset_t)p->dirbase, NBPTD);
+		zfree(pdpt_zone, (void *)p->pm_hold);
 
-	    if (*pdep & INTEL_PTE_VALID) {
-	        ind = pdep - (pt_entry_t *)&p->dirbase[0];
+		vm_object_deallocate(p->pm_obj);
+	} else {
+	        /* 64 bit */
+	        int inuse_ptepages = 0;
 
-		vm_object_lock(p->pm_obj);
-		m = vm_page_lookup(p->pm_obj, (vm_object_offset_t)ind);
-		if (m == VM_PAGE_NULL) {
-		    panic("pmap_destroy: pte page not in object");
-		}
-		vm_page_lock_queues();
-		vm_page_free(m);
-		inuse_ptepages_count--;
+		/* free 64 bit mode structs */
+		inuse_ptepages++;
+		kmem_free(kernel_map, (vm_offset_t)p->pm_hold, PAGE_SIZE);
 
-		vm_object_unlock(p->pm_obj);
-		vm_page_unlock_queues();
+		inuse_ptepages += p->pm_obj_pml4->resident_page_count;
+		vm_object_deallocate(p->pm_obj_pml4);
 
-		/*
-		 *	Clear pdes, this might be headed for the cache.
-		 */
-		pmap_store_pte(pdep, 0);
-		pdep++;
-	    }
-	    else {
-	      pmap_store_pte(pdep, 0);
-	      pdep++;
-	    }
-	
-	}
-#else
-	inuse_ptepages_count -= p->pm_obj->resident_page_count;
-#endif
-	vm_object_deallocate(p->pm_obj);
-	  kmem_free(kernel_map, (vm_offset_t)p->dirbase, NBPTD);
-	  zfree(pdpt_zone, (void *)p->pm_hold);
-	} else {
-
-	  /* 64 bit */
-
-	  pmap_unmap_sharedpage(p);
-
-	  /* free 64 bit mode structs */
-	  inuse_ptepages_count--;
-	  kmem_free(kernel_map, (vm_offset_t)p->pm_hold, PAGE_SIZE);
+		inuse_ptepages += p->pm_obj_pdpt->resident_page_count;
+		vm_object_deallocate(p->pm_obj_pdpt);
 
-	  inuse_ptepages_count -= p->pm_obj_pml4->resident_page_count;
-	  vm_object_deallocate(p->pm_obj_pml4);
-
-	  inuse_ptepages_count -= p->pm_obj_pdpt->resident_page_count;
-	  vm_object_deallocate(p->pm_obj_pdpt);
-
-	  inuse_ptepages_count -= p->pm_obj->resident_page_count;
-	  vm_object_deallocate(p->pm_obj);
+		inuse_ptepages += p->pm_obj->resident_page_count;
+		vm_object_deallocate(p->pm_obj);
 
+		vm_page_lock_queues();
+		inuse_ptepages_count -= inuse_ptepages;
+		vm_page_unlock_queues();
 	}
-
 	zfree(pmap_zone, p);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__DESTROY) | DBG_FUNC_END,
+		   0, 0, 0, 0, 0);
+
 }
 
 /*
@@ -1564,14 +1892,11 @@ void
 pmap_reference(
 	register pmap_t	p)
 {
-	spl_t	s;
 
 	if (p != PMAP_NULL) {
-		SPLVM(s);
-		simple_lock(&p->lock);
+	        PMAP_LOCK(p);
 		p->ref_count++;
-		simple_unlock(&p->lock);
-		SPLX(s);
+		PMAP_UNLOCK(p);;
 	}
 }
 
@@ -1587,7 +1912,7 @@ pmap_reference(
  *	Assumes that the pte-page exists.
  */
 
-static void
+void
 pmap_remove_range(
 	pmap_t			pmap,
 	vm_map_offset_t		start_vaddr,
@@ -1595,14 +1920,32 @@ pmap_remove_range(
 	pt_entry_t		*epte)
 {
 	register pt_entry_t	*cpte;
+	pv_hashed_entry_t       pvh_et = PV_HASHED_ENTRY_NULL;
+	pv_hashed_entry_t       pvh_eh = PV_HASHED_ENTRY_NULL;
+	pv_hashed_entry_t       pvh_e;
+	int			pvh_cnt = 0;
 	int			num_removed, num_unwired, num_found;
 	int			pai;
 	pmap_paddr_t		pa;
-	vm_map_offset_t         vaddr;
+	vm_map_offset_t		vaddr;
+	int                     pvhash_idx;
+	uint32_t                pv_cnt;
 
 	num_removed = 0;
 	num_unwired = 0;
-	num_found = 0;
+	num_found   = 0;
+
+	if (pmap != kernel_pmap &&
+	    pmap->pm_task_map == TASK_MAP_32BIT &&
+	    start_vaddr >= HIGH_MEM_BASE) {
+		/*
+		 * The range is in the "high_shared_pde" which is shared
+		 * between the kernel and all 32-bit tasks.  It holds
+		 * the 32-bit commpage but also the trampolines, GDT, etc...
+		 * so we can't let user tasks remove anything from it.
+		 */
+		return;
+	}
 
 	/* invalidate the PTEs first to "freeze" them */
 	for (cpte = spte, vaddr = start_vaddr;
@@ -1617,8 +1960,9 @@ pmap_remove_range(
 	    if (iswired(*cpte))
 		num_unwired++;
 
-	    if (!valid_page(i386_btop(pa))) {
+	    pai = pa_index(pa);
 
+	    if (!managed_page(pai)) {
 		/*
 		 *	Outside range of managed physical memory.
 		 *	Just remove the mappings.
@@ -1627,13 +1971,13 @@ pmap_remove_range(
 		continue;
 	    }
 
-	    /* invalidate the PTE */
+	    /* invalidate the PTE */ 
 	    pmap_update_pte(cpte, *cpte, (*cpte & ~INTEL_PTE_VALID));
 	}
 
-	if (0 == num_found) {
-	  /* nothing was changed, we're done */
-	  goto update_counts;
+	if (num_found == 0) {
+		/* nothing was changed: we're done */
+	        goto update_counts;
 	}
 
 	/* propagate the invalidates to other CPUs */
@@ -1652,6 +1996,12 @@ pmap_remove_range(
 
 	    LOCK_PVH(pai);
 
+	    pa = pte_to_pa(*cpte);
+	    if (pa == 0) {
+	      UNLOCK_PVH(pai);
+	      continue;
+	    }
+	      
 	    num_removed++;
 
 	    /*
@@ -1660,7 +2010,7 @@ pmap_remove_range(
 	     */
 	    /* remember reference and change */
 	    pmap_phys_attributes[pai] |=
-	      (char)(*cpte & (PHYS_MODIFIED|PHYS_REFERENCED));
+		    (char)(*cpte & (PHYS_MODIFIED | PHYS_REFERENCED));
 	    /* completely invalidate the PTE */
 	    pmap_store_pte(cpte, 0);
 
@@ -1669,50 +2019,110 @@ pmap_remove_range(
 	     *	this physical page.
 	     */
 	    {
-		register pv_entry_t	pv_h, prev, cur;
+	      pv_rooted_entry_t	pv_h;
+	      pv_hashed_entry_t *pprevh;
+	      ppnum_t ppn = (ppnum_t)pai;
 
 		pv_h = pai_to_pvh(pai);
-		if (pv_h->pmap == PMAP_NULL) {
-		    panic("pmap_remove: null pv_list!");
-		}
-		if (pv_h->va == vaddr && pv_h->pmap == pmap) {
+		pvh_e = PV_HASHED_ENTRY_NULL;
+		if (pv_h->pmap == PMAP_NULL)
+		    panic("pmap_remove_range: null pv_list!");
+
+		if (pv_h->va == vaddr && pv_h->pmap == pmap) { /* rooted or not */
 		    /*
-		     * Header is the pv_entry.  Copy the next one
-		     * to header and free the next one (we cannot
-		     * free the header)
+		     * Header is the pv_rooted_entry. We can't free that. If there is a queued
+		     * entry after this one we remove that
+		     * from the ppn queue, we remove it from the hash chain
+		     * and copy it to the rooted entry. Then free it instead.
 		     */
-		    cur = pv_h->next;
-		    if (cur != PV_ENTRY_NULL) {
-			*pv_h = *cur;
-			PV_FREE(cur);
+
+		  pvh_e = (pv_hashed_entry_t)queue_next(&pv_h->qlink);
+		  if (pv_h != (pv_rooted_entry_t)pvh_e) {  /* any queued after rooted? */
+		    CHK_NPVHASH();
+		    pvhash_idx = pvhashidx(pvh_e->pmap,pvh_e->va);
+		    LOCK_PV_HASH(pvhash_idx);
+		    remque(&pvh_e->qlink);
+		    {
+		      pprevh = pvhash(pvhash_idx);
+		      if (PV_HASHED_ENTRY_NULL == *pprevh) {
+			panic("pmap_remove_range empty hash removing rooted pv");
+		      }
 		    }
-		    else {
-			pv_h->pmap = PMAP_NULL;
+		    pmap_pvh_unlink(pvh_e);
+		    UNLOCK_PV_HASH(pvhash_idx);
+		    pv_h->pmap = pvh_e->pmap;
+		    pv_h->va = pvh_e->va;   /* dispose of pvh_e */
+		  } else {  /* none queued after rooted */
+		    pv_h->pmap = PMAP_NULL;
+		    pvh_e = PV_HASHED_ENTRY_NULL;
+		  }   /* any queued after rooted */
+
+		} else { /* rooted or not */
+		  /* not removing rooted pv. find it on hash chain, remove from ppn queue and
+		   * hash chain and free it */
+		  CHK_NPVHASH();
+		  pvhash_idx = pvhashidx(pmap,vaddr);
+		  LOCK_PV_HASH(pvhash_idx);
+		  pprevh = pvhash(pvhash_idx);
+		  if (PV_HASHED_ENTRY_NULL == *pprevh) {
+		    panic("pmap_remove_range empty hash removing hashed pv");
 		    }
-		}
-		else {
-		    cur = pv_h;
-		    do {
-			prev = cur;
-			if ((cur = prev->next) == PV_ENTRY_NULL) {
-			  panic("pmap-remove: mapping not in pv_list!");
-			}
-		    } while (cur->va != vaddr || cur->pmap != pmap);
-		    prev->next = cur->next;
-		    PV_FREE(cur);
-		}
+		  pvh_e = *pprevh;
+		  pmap_pv_hashlist_walks++;
+		  pv_cnt = 0;
+		  while (PV_HASHED_ENTRY_NULL != pvh_e) {
+			pv_cnt++;
+			if (pvh_e->pmap == pmap && pvh_e->va == vaddr && pvh_e->ppn == ppn) break;
+			pprevh = &pvh_e->nexth;
+			pvh_e = pvh_e->nexth;
+		  }
+		  pmap_pv_hashlist_cnts += pv_cnt;
+		  if (pmap_pv_hashlist_max < pv_cnt) pmap_pv_hashlist_max = pv_cnt;
+		  if (PV_HASHED_ENTRY_NULL == pvh_e) panic("pmap_remove_range pv not on hash");
+		  *pprevh = pvh_e->nexth;
+		  remque(&pvh_e->qlink);
+		  UNLOCK_PV_HASH(pvhash_idx);
+
+		} /* rooted or not */
+
 		UNLOCK_PVH(pai);
-	    }
+
+		if (pvh_e != PV_HASHED_ENTRY_NULL) {
+		  pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+		  pvh_eh = pvh_e;
+
+		  if (pvh_et == PV_HASHED_ENTRY_NULL) {
+		    pvh_et = pvh_e;
+		  }
+
+		  pvh_cnt++;
+		}
+
+	    } /* removing mappings for this phy page */
+	} /* for loop */
+	
+	if (pvh_eh != PV_HASHED_ENTRY_NULL) {
+	    PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pvh_cnt);
 	}
 
- update_counts:
+update_counts:
 	/*
 	 *	Update the counts
 	 */
+#if TESTING
+	if (pmap->stats.resident_count < num_removed)
+	        panic("pmap_remove_range: resident_count");
+#endif
 	assert(pmap->stats.resident_count >= num_removed);
-	pmap->stats.resident_count -= num_removed;
+	OSAddAtomic(-num_removed, (SInt32 *) &pmap->stats.resident_count);
+
+#if TESTING
+	if (pmap->stats.wired_count < num_unwired)
+	        panic("pmap_remove_range: wired_count");
+#endif
 	assert(pmap->stats.wired_count >= num_unwired);
-	pmap->stats.wired_count -= num_unwired;
+	OSAddAtomic(-num_unwired, (SInt32 *) &pmap->stats.wired_count);
+
 	return;
 }
 
@@ -1745,36 +2155,91 @@ pmap_remove(
 	addr64_t	s64,
 	addr64_t	e64)
 {
-	spl_t			spl;
-	register pt_entry_t	*pde;
-	register pt_entry_t	*spte, *epte;
-	addr64_t		l64;
-	addr64_t    		orig_s64;
+	pt_entry_t	*pde;
+	pt_entry_t	*spte, *epte;
+	addr64_t	l64;
+	addr64_t    	orig_s64;
+	uint64_t        deadline;
+
+	pmap_intr_assert();
 
 	if (map == PMAP_NULL || s64 == e64)
 		return;
+ 
+	PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_START,
+		   (int) map,
+		   (int) (s64>>32), (int) s64,
+		   (int) (e64>>32), (int) e64);
 
-	PMAP_READ_LOCK(map, spl);
+	PMAP_LOCK(map);
+
+#if 0
+	/*
+	 * Check that address range in the kernel does not overlap the stacks.
+	 * We initialize local static min/max variables once to avoid making
+	 * 2 function calls for every remove. Note also that these functions
+	 * both return 0 before kernel stacks have been initialized, and hence
+	 * the panic is not triggered in this case.
+	 */
+	if (map == kernel_pmap) {
+		static vm_offset_t	kernel_stack_min = 0;
+		static vm_offset_t	kernel_stack_max = 0;
+
+		if (kernel_stack_min == 0) {
+			kernel_stack_min = min_valid_stack_address();
+			kernel_stack_max = max_valid_stack_address();
+		}
+		if  ((kernel_stack_min <= s64 && s64 <  kernel_stack_max) ||
+		     (kernel_stack_min <  e64 && e64 <= kernel_stack_max))
+			panic("pmap_remove() attempted in kernel stack");
+	}
+#else
+
+	/*
+	 * The values of kernel_stack_min and kernel_stack_max are no longer
+	 * relevant now that we allocate kernel stacks anywhere in the kernel map,
+	 * so the old code above no longer applies.  If we wanted to check that
+	 * we weren't removing a mapping of a page in a kernel stack we'd have to
+	 * mark the PTE with an unused bit and check that here.
+	 */
+
+#endif
+
+	deadline = rdtsc64() + max_preemption_latency_tsc;
 
 	orig_s64 = s64;
 
 	while (s64 < e64) {
+
 	    l64 = (s64 + pde_mapped_size) & ~(pde_mapped_size-1);
 	    if (l64 > e64)
 		l64 = e64;
 	    pde = pmap_pde(map, s64);
+
 	    if (pde && (*pde & INTEL_PTE_VALID)) {
 	        spte = (pt_entry_t *)pmap_pte(map, (s64 & ~(pde_mapped_size-1)));
 		spte = &spte[ptenum(s64)];
 		epte = &spte[intel_btop(l64-s64)];
+
 		pmap_remove_range(map, s64, spte, epte);
 	    }
 	    s64 = l64;
 	    pde++;
+
+	    if (s64 < e64 && rdtsc64() >= deadline) {
+	      PMAP_UNLOCK(map)
+		PMAP_LOCK(map)
+
+	      deadline = rdtsc64() + max_preemption_latency_tsc;
+	    }
+
 	}
-	PMAP_UPDATE_TLBS(map, orig_s64, e64);
 
-	PMAP_READ_UNLOCK(map, spl);
+	PMAP_UNLOCK(map);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__REMOVE) | DBG_FUNC_END,
+		   (int) map, 0, 0, 0, 0);
+
 }
 
 /*
@@ -1789,24 +2254,36 @@ pmap_page_protect(
         ppnum_t         pn,
 	vm_prot_t	prot)
 {
-	pv_entry_t		pv_h, prev;
-	register pv_entry_t	pv_e;
-	register pt_entry_t	*pte;
+	pv_hashed_entry_t		pvh_eh = PV_HASHED_ENTRY_NULL;
+	pv_hashed_entry_t		pvh_et = PV_HASHED_ENTRY_NULL;
+	pv_hashed_entry_t       nexth;
+	int			pvh_cnt = 0;
+	pv_rooted_entry_t		pv_h;
+	pv_rooted_entry_t		pv_e;
+	pv_hashed_entry_t       pvh_e;
+	pt_entry_t		*pte;
 	int			pai;
 	register pmap_t		pmap;
-	spl_t			spl;
 	boolean_t		remove;
-	pmap_paddr_t            phys;
+	int                     pvhash_idx;
 
+	pmap_intr_assert();
 	assert(pn != vm_page_fictitious_addr);
+	if (pn == vm_page_guard_addr)
+		return;
+
+	pai = ppn_to_pai(pn);
 
-	if (!valid_page(pn)) {
+	if (!managed_page(pai)) {
 	    /*
 	     *	Not a managed page.
 	     */
 	    return;
 	}
 
+	PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_START,
+		   (int) pn, (int) prot, 0, 0, 0);
+
 	/*
 	 * Determine the new protection.
 	 */
@@ -1821,115 +2298,132 @@ pmap_page_protect(
 		remove = TRUE;
 		break;
 	}
-	phys = (pmap_paddr_t)i386_ptob(pn);
-	pai = pa_index(phys);
-	pv_h = pai_to_pvh(pai);
 
+	pv_h = pai_to_pvh(pai);
 
-	/*
-	 *	Lock the pmap system first, since we will be changing
-	 *	several pmaps.
-	 */
-	PMAP_WRITE_LOCK(spl);
+	LOCK_PVH(pai);
 
 	/*
 	 * Walk down PV list, changing or removing all mappings.
-	 * We do not have to lock the pv_list because we have
-	 * the entire pmap system locked.
 	 */
 	if (pv_h->pmap != PMAP_NULL) {
 
-	        prev = pv_e = pv_h;
-
-		do {
-		        register vm_map_offset_t vaddr;
-
-			pmap = pv_e->pmap;
-			/*
-			 * Lock the pmap to block pmap_extract and similar routines.
-			 */
-			simple_lock(&pmap->lock);
+	    pv_e = pv_h;
+	    pvh_e = (pv_hashed_entry_t)pv_e; /* cheat */
 
-			vaddr = pv_e->va;
-			pte = pmap_pte(pmap, vaddr);
-			if(0 == pte) {
-			  kprintf("pmap_page_protect pmap 0x%x pn 0x%x vaddr 0x%llx\n",pmap, pn, vaddr);
-			  panic("pmap_page_protect");
-			}
-			/*
-			 * Consistency checks.
-			 */
-			/* assert(*pte & INTEL_PTE_VALID); XXX */
-			/* assert(pte_to_phys(*pte) == phys); */
+	    do {
+	        register vm_map_offset_t vaddr;
+		pmap = pv_e->pmap;
 
+		vaddr = pv_e->va;
+		pte = pmap_pte(pmap, vaddr);
+		
+		if (0 == pte) {
+		    kprintf("pmap_page_protect pmap %p pn 0x%x vaddr 0x%llx\n",pmap, pn, vaddr);
+		    panic("pmap_page_protect");
+		}
 
-			/*
-			 * Remove the mapping if new protection is NONE
-			 * or if write-protecting a kernel mapping.
-			 */
-			if (remove || pmap == kernel_pmap) {
-			        /*
-				 * Remove the mapping, collecting any modify bits.
-				 */
-			        pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_VALID));
+		nexth = (pv_hashed_entry_t)queue_next(&pvh_e->qlink);  /* if there is one */
 
-				PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
+		/*
+		 * Remove the mapping if new protection is NONE
+		 * or if write-protecting a kernel mapping.
+		 */
+		if (remove || pmap == kernel_pmap) {
+		    /*
+		     * Remove the mapping, collecting any modify bits.
+		     */
+		    pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_VALID));
 
-			        pmap_phys_attributes[pai] |= *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
+		    PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
 
-				pmap_store_pte(pte, 0);
+		    pmap_phys_attributes[pai] |= *pte & (PHYS_MODIFIED|PHYS_REFERENCED);
 
+		    pmap_store_pte(pte, 0);
 
-				//XXX breaks DEBUG build		    assert(pmap->stats.resident_count >= 1);
-				pmap->stats.resident_count--;
+#if TESTING
+		    if (pmap->stats.resident_count < 1)
+		        panic("pmap_page_protect: resident_count");
+#endif
+		    assert(pmap->stats.resident_count >= 1);
+		    OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count);
 
-				/*
-				 * Remove the pv_entry.
-				 */
-				if (pv_e == pv_h) {
-				        /*
-					 * Fix up head later.
-					 */
-				        pv_h->pmap = PMAP_NULL;
-				}
-				else {
-				        /*
-					 * Delete this entry.
-					 */
-				        prev->next = pv_e->next;
-					PV_FREE(pv_e);
-				}
-			} else {
-			        /*
-				 * Write-protect.
-				 */
-			        pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_WRITE));
-				PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
-				/*
-				 * Advance prev.
-				 */
-				prev = pv_e;
-			}
+		    /*
+		     * Deal with the pv_rooted_entry.
+		     */
 
-			simple_unlock(&pmap->lock);
+		    if (pv_e == pv_h) {
+			/*
+			 * Fix up head later.
+			 */
+			pv_h->pmap = PMAP_NULL;
+		    }
+		    else {
+			/*
+			 * Delete this entry.
+			 */
+		      CHK_NPVHASH();
+		      pvhash_idx = pvhashidx(pvh_e->pmap,pvh_e->va);
+		      LOCK_PV_HASH(pvhash_idx);
+		      remque(&pvh_e->qlink);
+		      pmap_pvh_unlink(pvh_e);
+		      UNLOCK_PV_HASH(pvhash_idx);
+
+		      pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+			pvh_eh = pvh_e;
+
+			if (pvh_et == PV_HASHED_ENTRY_NULL)
+			    pvh_et = pvh_e;
+			pvh_cnt++;
+		    }
+		} else {
+		    /*
+		     * Write-protect.
+		     */
+		    pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_WRITE));
+		    PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
+		}
 
-		} while ((pv_e = prev->next) != PV_ENTRY_NULL);
+		pvh_e = nexth;
+	    } while ((pv_e = (pv_rooted_entry_t)nexth) != pv_h);
 
-		/*
-		 * If pv_head mapping was removed, fix it up.
-		 */
-		if (pv_h->pmap == PMAP_NULL) {
-		        pv_e = pv_h->next;
+	    /*
+	     * If pv_head mapping was removed, fix it up.
+	     */
 
-			if (pv_e != PV_ENTRY_NULL) {
-			        *pv_h = *pv_e;
-				PV_FREE(pv_e);
-			}
+	    if (pv_h->pmap == PMAP_NULL) {
+	      pvh_e = (pv_hashed_entry_t)queue_next(&pv_h->qlink);
+
+	      if (pvh_e != (pv_hashed_entry_t)pv_h) {
+		CHK_NPVHASH();
+		pvhash_idx = pvhashidx(pvh_e->pmap,pvh_e->va);
+		LOCK_PV_HASH(pvhash_idx);
+		remque(&pvh_e->qlink);
+		pmap_pvh_unlink(pvh_e);
+		UNLOCK_PV_HASH(pvhash_idx);
+		  pv_h->pmap = pvh_e->pmap;
+		  pv_h->va = pvh_e->va;
+		  pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+		    pvh_eh = pvh_e;
+
+		    if (pvh_et == PV_HASHED_ENTRY_NULL)
+		        pvh_et = pvh_e;
+		    pvh_cnt++;
 		}
+	    }
 	}
-	PMAP_WRITE_UNLOCK(spl);
+	if (pvh_eh != PV_HASHED_ENTRY_NULL) {
+	    PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pvh_cnt);
+	}
+
+	UNLOCK_PVH(pai);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__PAGE_PROTECT) | DBG_FUNC_END,
+		   0, 0, 0, 0, 0);
+
 }
 
+
 /*
  *	Routine:
  *		pmap_disconnect
@@ -1942,7 +2436,7 @@ pmap_page_protect(
 unsigned int pmap_disconnect(
 	ppnum_t pa)
 {
-	pmap_page_protect(pa, 0);				/* disconnect the page */
+	pmap_page_protect(pa, 0);			/* disconnect the page */
 	return (pmap_get_refmod(pa));			/* return ref/chg status */
 }
 
@@ -1962,9 +2456,10 @@ pmap_protect(
 	register pt_entry_t	*spte, *epte;
 	vm_map_offset_t		lva;
 	vm_map_offset_t		orig_sva;
-	spl_t		spl;
 	boolean_t	set_NX;
-	int num_found = 0;
+	int		num_found = 0;
+
+	pmap_intr_assert();
 
 	if (map == PMAP_NULL)
 		return;
@@ -1974,13 +2469,17 @@ pmap_protect(
 		return;
 	}
 
+	PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_START,
+		   (int) map,
+		   (int) (sva>>32), (int) sva,
+		   (int) (eva>>32), (int) eva);
+
 	if ( (prot & VM_PROT_EXECUTE) || !nx_enabled || !map->nx_enabled )
 	        set_NX = FALSE;
 	else
 	        set_NX = TRUE;
 
-	SPLVM(spl);
-	simple_lock(&map->lock);
+	PMAP_LOCK(map);
 
 	orig_sva = sva;
 	while (sva < eva) {
@@ -1994,20 +2493,20 @@ pmap_protect(
 		epte = &spte[intel_btop(lva-sva)];
 
 		while (spte < epte) {
+
 		    if (*spte & INTEL_PTE_VALID) {
 		      
 		        if (prot & VM_PROT_WRITE)
-			  pmap_update_pte(spte, *spte, (*spte | INTEL_PTE_WRITE));
+			    pmap_update_pte(spte, *spte, (*spte | INTEL_PTE_WRITE));
 			else
-			  pmap_update_pte(spte, *spte, (*spte & ~INTEL_PTE_WRITE));
+			    pmap_update_pte(spte, *spte, (*spte & ~INTEL_PTE_WRITE));
 
 			if (set_NX == TRUE)
-			  pmap_update_pte(spte, *spte, (*spte | INTEL_PTE_NX));
+			    pmap_update_pte(spte, *spte, (*spte | INTEL_PTE_NX));
 			else
-			  pmap_update_pte(spte, *spte, (*spte & ~INTEL_PTE_NX));
+			    pmap_update_pte(spte, *spte, (*spte & ~INTEL_PTE_NX));
 
 			num_found++;
-
 		    }
 		    spte++;
 		}
@@ -2015,10 +2514,13 @@ pmap_protect(
 	    sva = lva;
 	}
 	if (num_found)
-	  PMAP_UPDATE_TLBS(map, orig_sva, eva);
+	    PMAP_UPDATE_TLBS(map, orig_sva, eva);
+
+	PMAP_UNLOCK(map);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__PROTECT) | DBG_FUNC_END,
+		   0, 0, 0, 0, 0);
 
-	simple_unlock(&map->lock);
-	SPLX(spl);
 }
 
 /* Map a (possibly) autogenned block */
@@ -2032,13 +2534,13 @@ pmap_map_block(
 	int		attr,
 	__unused unsigned int	flags)
 {
-        uint32_t page;
+    uint32_t page;
 
-	for (page = 0; page < size; page++) {
-	        pmap_enter(pmap, va, pa, prot, attr, TRUE);
-		va += PAGE_SIZE;
-		pa++;
-	}
+    for (page = 0; page < size; page++) {
+	pmap_enter(pmap, va, pa, prot, attr, TRUE);
+	va += PAGE_SIZE;
+	pa++;
+    }
 }
 
 
@@ -2057,34 +2559,41 @@ pmap_map_block(
 void
 pmap_enter(
 	register pmap_t		pmap,
-	vm_map_offset_t		vaddr,
+ 	vm_map_offset_t		vaddr,
 	ppnum_t                 pn,
 	vm_prot_t		prot,
 	unsigned int 		flags,
 	boolean_t		wired)
 {
 	register pt_entry_t	*pte;
-	register pv_entry_t	pv_h;
+	register pv_rooted_entry_t	pv_h;
 	register int		pai;
-	pv_entry_t		pv_e;
+	pv_hashed_entry_t		pvh_e;
+	pv_hashed_entry_t		pvh_new;
+	pv_hashed_entry_t       *hashp;
 	pt_entry_t		template;
-	spl_t			spl;
 	pmap_paddr_t		old_pa;
-	pmap_paddr_t            pa = (pmap_paddr_t)i386_ptob(pn);
+	pmap_paddr_t             pa = (pmap_paddr_t)i386_ptob(pn);
 	boolean_t		need_tlbflush = FALSE;
 	boolean_t		set_NX;
-	char                    oattr;
-
-	XPR(0x80000000, "%x/%x: pmap_enter %x/%qx/%x\n",
-	    current_thread(),
-	    current_thread(), 
-	    pmap, vaddr, pn);
+	char			oattr;
+	int                     pvhash_idx;
+	uint32_t                pv_cnt;
+	boolean_t               old_pa_locked;
 
+	pmap_intr_assert();
 	assert(pn != vm_page_fictitious_addr);
 	if (pmap_debug)
 		printf("pmap(%qx, %x)\n", vaddr, pn);
 	if (pmap == PMAP_NULL)
 		return;
+	if (pn == vm_page_guard_addr)
+		return;
+
+	PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_START,
+		   (int) pmap,
+		   (int) (vaddr>>32), (int) vaddr,
+		   (int) pn, prot);
 
 	if ( (prot & VM_PROT_EXECUTE) || !nx_enabled || !pmap->nx_enabled )
 	        set_NX = FALSE;
@@ -2098,9 +2607,12 @@ pmap_enter(
 	 *	and allocate one.  Then we will retry, throughing away
 	 *	the allocated entry later (if we no longer need it).
 	 */
-	pv_e = PV_ENTRY_NULL;
 
-	PMAP_READ_LOCK(pmap, spl);
+	pvh_new = PV_HASHED_ENTRY_NULL;
+Retry:
+	pvh_e = PV_HASHED_ENTRY_NULL;
+
+	PMAP_LOCK(pmap);
 
 	/*
 	 *	Expand pmap to include this pte.  Assume that
@@ -2112,22 +2624,42 @@ pmap_enter(
 		/*
 		 *	Must unlock to expand the pmap.
 		 */
-		PMAP_READ_UNLOCK(pmap, spl);
-
+	        PMAP_UNLOCK(pmap);
 		pmap_expand(pmap, vaddr); /* going to grow pde level page(s) */
+		PMAP_LOCK(pmap);
+	}
 
-		PMAP_READ_LOCK(pmap, spl);
+	old_pa = pte_to_pa(*pte);
+	pai = pa_index(old_pa);
+	old_pa_locked = FALSE;
+
+	/*
+	 * if we have a previous managed page, lock the pv entry now. after
+	 * we lock it, check to see if someone beat us to the lock and if so
+	 * drop the lock
+	 */
+
+	if ((0 != old_pa) && managed_page(pai)) {
+	  LOCK_PVH(pai);
+	  old_pa_locked = TRUE;
+	  old_pa = pte_to_pa(*pte);
+	  if (0 == old_pa) {
+	    UNLOCK_PVH(pai);  /* some other path beat us to it */
+	    old_pa_locked = FALSE;
+	  }
 	}
+
+
 	/*
-	 *	Special case if the physical page is already mapped
+	 *	Special case if the incoming physical page is already mapped
 	 *	at this address.
 	 */
-	old_pa = pte_to_pa(*pte);
 	if (old_pa == pa) {
+
 	    /*
 	     *	May be changing its wired attribute or protection
 	     */
-	
+
 	    template = pa_to_pte(pa) | INTEL_PTE_VALID;
 
 	    if(VM_MEM_NOT_CACHEABLE == (flags & (VM_MEM_NOT_CACHEABLE | VM_WIMG_USE_DEFAULT))) {
@@ -2147,17 +2679,21 @@ pmap_enter(
 	    if (wired) {
 		template |= INTEL_PTE_WIRED;
 		if (!iswired(*pte))
-		    pmap->stats.wired_count++;
+		    OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
 	    }
 	    else {
 		if (iswired(*pte)) {
 		    assert(pmap->stats.wired_count >= 1);
-		    pmap->stats.wired_count--;
+		    OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
 		}
 	    }
 
-	    /* store modified PTE and preserve RC bits */
+	    /* store modified PTE and preserve RC bits */ 
 	    pmap_update_pte(pte, *pte, template | (*pte & (INTEL_PTE_REF | INTEL_PTE_MOD)));
+	    if (old_pa_locked) {
+	      UNLOCK_PVH(pai);
+	      old_pa_locked = FALSE;
+	    }
 	    need_tlbflush = TRUE;
 	    goto Done;
 	}
@@ -2169,12 +2705,6 @@ pmap_enter(
 	 *	   2) Add pvlist entry for new mapping
 	 *	   3) Enter new mapping.
 	 *
-	 *      SHARING FAULTS IS HORRIBLY BROKEN
-	 *	SHARING_FAULTS complicates this slightly in that it cannot
-	 *	replace the mapping, but must remove it (because adding the
-	 *	pvlist entry for the new mapping may remove others), and
-	 *	hence always enters the new mapping at step 3)
-	 *
 	 *	If the old physical page is not managed step 1) is skipped
 	 *	(except for updating the TLBs), and the mapping is
 	 *	overwritten at step 3).  If the new physical page is not
@@ -2189,261 +2719,211 @@ pmap_enter(
 	     *	to overwrite the old one.
 	     */
 
-	  /* invalidate the PTE */
-	  pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_VALID));
-	  /* propagate the invalidate everywhere */
-	  PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
-	  /* remember reference and change */
-	  oattr = (char)(*pte & (PHYS_MODIFIED | PHYS_REFERENCED));
-	  /* completely invalidate the PTE */
-	  pmap_store_pte(pte,0);
-
-	    if (valid_page(i386_btop(old_pa))) {
+	    /* invalidate the PTE */ 
+	    pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_VALID));
+	    /* propagate invalidate everywhere */
+	    PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
+	    /* remember reference and change */
+	    oattr = (char)(*pte & (PHYS_MODIFIED | PHYS_REFERENCED));
+	    /* completely invalidate the PTE */
+	    pmap_store_pte(pte, 0);
 
-		pai = pa_index(old_pa);
-		LOCK_PVH(pai);
+	    if (managed_page(pai)) {
 
+#if TESTING
+	        if (pmap->stats.resident_count < 1)
+		    panic("pmap_enter: resident_count");
+#endif
 		assert(pmap->stats.resident_count >= 1);
-		pmap->stats.resident_count--;
+		OSAddAtomic(-1, (SInt32 *) &pmap->stats.resident_count);
+
 	    	if (iswired(*pte)) {
+
+#if TESTING
+	            if (pmap->stats.wired_count < 1)
+		        panic("pmap_enter: wired_count");
+#endif
 		    assert(pmap->stats.wired_count >= 1);
-		    pmap->stats.wired_count--;
+		    OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
 		}
 
 		pmap_phys_attributes[pai] |= oattr;
 		/*
 		 *	Remove the mapping from the pvlist for
 		 *	this physical page.
+		 *      We'll end up with either a rooted pv or a
+		 *      hashed pv
 		 */
 		{
-		    register pv_entry_t	prev, cur;
 
 		    pv_h = pai_to_pvh(pai);
+
 		    if (pv_h->pmap == PMAP_NULL) {
 			panic("pmap_enter: null pv_list!");
 		    }
 
 		    if (pv_h->va == vaddr && pv_h->pmap == pmap) {
 			/*
-			 * Header is the pv_entry.  Copy the next one
-			 * to header and free the next one (we cannot
+			 * Header is the pv_rooted_entry.  
+			 * If there is a next one, copy it to the
+			 * header and free the next one (we cannot
 			 * free the header)
 			 */
-			cur = pv_h->next;
-			if (cur != PV_ENTRY_NULL) {
-			    *pv_h = *cur;
-			    pv_e = cur;
-			}
-			else {
-			    pv_h->pmap = PMAP_NULL;
+		      pvh_e = (pv_hashed_entry_t)queue_next(&pv_h->qlink);
+		      if (pvh_e != (pv_hashed_entry_t)pv_h) {
+			pvhash_idx = pvhashidx(pvh_e->pmap, pvh_e->va);
+			LOCK_PV_HASH(pvhash_idx);
+			  remque(&pvh_e->qlink);
+			  pmap_pvh_unlink(pvh_e);
+			  UNLOCK_PV_HASH(pvhash_idx);
+			  pv_h->pmap = pvh_e->pmap;
+			  pv_h->va = pvh_e->va;
 			}
+		      else {
+			pv_h->pmap = PMAP_NULL;
+			pvh_e = PV_HASHED_ENTRY_NULL;
+		      }
 		    }
 		    else {
-			cur = pv_h;
-			do {
-			    prev = cur;
-			    if ((cur = prev->next) == PV_ENTRY_NULL) {
-			        panic("pmap_enter: mapping not in pv_list!");
-			    }
-			} while (cur->va != vaddr || cur->pmap != pmap);
-			prev->next = cur->next;
-			pv_e = cur;
+		      pv_hashed_entry_t *pprevh;
+		      ppnum_t old_ppn;
+		      /* wasn't the rooted pv - hash, find it, and unlink it */
+		      old_ppn = (ppnum_t)pa_index(old_pa);
+		      CHK_NPVHASH();
+		      pvhash_idx = pvhashidx(pmap,vaddr);
+		      LOCK_PV_HASH(pvhash_idx);
+		      pprevh = pvhash(pvhash_idx);
+#if PV_DEBUG
+		      if (NULL==pprevh)panic("pmap enter 1");
+#endif
+		      pvh_e = *pprevh;
+		      pmap_pv_hashlist_walks++;
+		      pv_cnt = 0;
+		      while (PV_HASHED_ENTRY_NULL != pvh_e) {
+			pv_cnt++;
+			if (pvh_e->pmap == pmap && pvh_e->va == vaddr && pvh_e->ppn == old_ppn) break;
+			pprevh = &pvh_e->nexth;
+			pvh_e = pvh_e->nexth;
+		      }
+		      pmap_pv_hashlist_cnts += pv_cnt;
+		      if (pmap_pv_hashlist_max < pv_cnt) pmap_pv_hashlist_max = pv_cnt;
+		      if (PV_HASHED_ENTRY_NULL == pvh_e) panic("pmap_enter: pv not in hash list");
+		      if(NULL==pprevh)panic("pmap enter 2");
+		      *pprevh = pvh_e->nexth;
+		      remque(&pvh_e->qlink);
+		      UNLOCK_PV_HASH(pvhash_idx);
 		    }
 		}
-		UNLOCK_PVH(pai);
 	    }
 	    else {
 
 		/*
-		 *	old_pa is not managed.  Pretend it's zero so code
-		 *	at Step 3) will enter new mapping (overwriting old
-		 *	one).  Do removal part of accounting.
+		 *	old_pa is not managed.
+		 *	Do removal part of accounting.
 		 */
-		old_pa = (pmap_paddr_t) 0;
 
 		if (iswired(*pte)) {
 		    assert(pmap->stats.wired_count >= 1);
-		    pmap->stats.wired_count--;
+		    OSAddAtomic(-1, (SInt32 *) &pmap->stats.wired_count);
 		}
 	    }
-	    need_tlbflush = TRUE;
-        
 	}
 
-	if (valid_page(i386_btop(pa))) {
+	/*
+	 * if we had a previously managed paged locked, unlock it now
+	 */
+
+	if (old_pa_locked) {
+	  UNLOCK_PVH(pai);
+	  old_pa_locked = FALSE;
+	}
+
+	pai = pa_index(pa);     /* now working with new incoming phys page */
+	if (managed_page(pai)) {
 
 	    /*
 	     *	Step 2) Enter the mapping in the PV list for this
 	     *	physical page.
 	     */
+	    pv_h = pai_to_pvh(pai);
 
-	    pai = pa_index(pa);
-
-
-#if SHARING_FAULTS /* this is horribly broken , do not enable */
-RetryPvList:
-	    /*
-	     * We can return here from the sharing fault code below
-	     * in case we removed the only entry on the pv list and thus
-	     * must enter the new one in the list header.
-	     */
-#endif /* SHARING_FAULTS */
 	    LOCK_PVH(pai);
-	    pv_h = pai_to_pvh(pai);
 
 	    if (pv_h->pmap == PMAP_NULL) {
 		/*
-		 *	No mappings yet
+		 *	No mappings yet, use  rooted pv
 		 */
 		pv_h->va = vaddr;
 		pv_h->pmap = pmap;
-		pv_h->next = PV_ENTRY_NULL;
+		queue_init(&pv_h->qlink);
 	    }
 	    else {
-#if	DEBUG
-		{
-		    /*
-		     * check that this mapping is not already there
-		     * or there is no alias for this mapping in the same map
-		     */
-		    pv_entry_t	e = pv_h;
-		    while (e != PV_ENTRY_NULL) {
-			if (e->pmap == pmap && e->va == vaddr)
-                            panic("pmap_enter: already in pv_list");
-			e = e->next;
-		    }
-		}
-#endif	/* DEBUG */
-#if SHARING_FAULTS /* broken, do not enable */
-                {
-                    /*
-                     * do sharing faults.
-                     * if we find an entry on this pv list in the same address
-		     * space, remove it.  we know there will not be more
-		     * than one. 
-		     */
-		    pv_entry_t	e = pv_h;
-                    pt_entry_t      *opte;
-
-		    while (e != PV_ENTRY_NULL) {
-			if (e->pmap == pmap) {
-                            /*
-			     *	Remove it, drop pv list lock first.
-			     */
-                            UNLOCK_PVH(pai);
-
-                            opte = pmap_pte(pmap, e->va);
-                            assert(opte != PT_ENTRY_NULL);
-                            /*
-			     *	Invalidate the translation buffer,
-			     *	then remove the mapping.
-			     */
-                             pmap_remove_range(pmap, e->va, opte,
-                                                      opte + 1);
-
-			     PMAP_UPDATE_TLBS(pmap, e->va, e->va + PAGE_SIZE);
-
-			     /*
-			      * We could have remove the head entry,
-			      * so there could be no more entries
-			      * and so we have to use the pv head entry.
-			      * so, go back to the top and try the entry
-			      * again.
-			      */
-			     goto RetryPvList;
-			}
-                        e = e->next;
-		    }
-
-		    /*
-                     * check that this mapping is not already there
-                     */
-		    e = pv_h;
-		    while (e != PV_ENTRY_NULL) {
-			if (e->pmap == pmap)
-                            panic("pmap_enter: alias in pv_list");
-			e = e->next;
-		    }
-		}
-#endif /* SHARING_FAULTS */
-#if DEBUG_ALIAS
-                {
-                    /*
-                     * check for aliases within the same address space.
-                     */
-		    pv_entry_t	e = pv_h;
-                    vm_offset_t     rpc = get_rpc();
-
-		    while (e != PV_ENTRY_NULL) {
-			if (e->pmap == pmap) {
-                            /*
-                             * log this entry in the alias ring buffer
-			     * if it's not there already.
-                             */
-                            struct pmap_alias *pma;
-                            int ii, logit;
-
-                            logit = TRUE;
-                            for (ii = 0; ii < pmap_alias_index; ii++) {
-                                if (pmap_aliasbuf[ii].rpc == rpc) {
-                                    /* found it in the log already */
-                                    logit = FALSE;
-                                    break;
-				}
-			    }
-                            if (logit) {
-                                pma = &pmap_aliasbuf[pmap_alias_index];
-                                pma->pmap = pmap;
-                                pma->va = vaddr;
-                                pma->rpc = rpc;
-                                pma->cookie = PMAP_ALIAS_COOKIE;
-                                if (++pmap_alias_index >= PMAP_ALIAS_MAX)
-                                    panic("pmap_enter: exhausted alias log");
-			    }
-			}
-                        e = e->next;
-		    }
-		}
-#endif /* DEBUG_ALIAS */
 		/*
-		 *	Add new pv_entry after header.
+		 *	Add new pv_hashed_entry after header.
 		 */
-		if (pv_e == PV_ENTRY_NULL) {
-		    PV_ALLOC(pv_e);
-		    if (pv_e == PV_ENTRY_NULL) {
-		      panic("pmap no pv_e's");
+		if ((PV_HASHED_ENTRY_NULL == pvh_e) && pvh_new) {
+		  pvh_e = pvh_new;
+		  pvh_new = PV_HASHED_ENTRY_NULL;  /* show we used it */
+		} else if (PV_HASHED_ENTRY_NULL == pvh_e) {
+		  PV_HASHED_ALLOC(pvh_e);
+		  if (PV_HASHED_ENTRY_NULL == pvh_e) {
+		    /* the pv list is empty.
+		     * if we are on the kernel pmap we'll use one of the special private
+		     * kernel pv_e's, else, we need to unlock everything, zalloc a pv_e,
+		     * and restart bringing in the pv_e with us.
+		     */
+		    if (kernel_pmap == pmap) {
+		      PV_HASHED_KERN_ALLOC(pvh_e);
+		    } else {
+		      UNLOCK_PVH(pai);
+		      PMAP_UNLOCK(pmap);
+		      pvh_new = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
+		      goto Retry;
 		    }
+		  }
 		}
-		pv_e->va = vaddr;
-		pv_e->pmap = pmap;
-		pv_e->next = pv_h->next;
-		pv_h->next = pv_e;
+
+		if (PV_HASHED_ENTRY_NULL == pvh_e) panic("pvh_e exhaustion");
+		pvh_e->va = vaddr;
+		pvh_e->pmap = pmap;
+		pvh_e->ppn = pn;
+		CHK_NPVHASH();
+		pvhash_idx = pvhashidx(pmap,vaddr);
+		LOCK_PV_HASH(pvhash_idx);
+		insque(&pvh_e->qlink, &pv_h->qlink);
+		hashp = pvhash(pvhash_idx);
+#if PV_DEBUG
+		if(NULL==hashp)panic("pmap_enter 4");
+#endif
+		pvh_e->nexth = *hashp;
+		*hashp = pvh_e;
+		UNLOCK_PV_HASH(pvhash_idx);
+
 		/*
 		 *	Remember that we used the pvlist entry.
 		 */
-		pv_e = PV_ENTRY_NULL;
+		pvh_e = PV_HASHED_ENTRY_NULL;
 	    }
-	    UNLOCK_PVH(pai);
 
 	    /*
 	     * only count the mapping
 	     * for 'managed memory'
 	     */
-	    pmap->stats.resident_count++;
+	    OSAddAtomic(+1, (SInt32 *) &pmap->stats.resident_count);
+	    if (pmap->stats.resident_count > pmap->stats.resident_max) {
+		    pmap->stats.resident_max = pmap->stats.resident_count;
+	    }
 	}
 
 	/*
 	 * Step 3) Enter the mapping.
-	 */
-
-
-	/*
+	 *
 	 *	Build a template to speed up entering -
 	 *	only the pfn changes.
 	 */
 	template = pa_to_pte(pa) | INTEL_PTE_VALID;
 
-	if(flags & VM_MEM_NOT_CACHEABLE) {
+	if (flags & VM_MEM_NOT_CACHEABLE) {
 		if(!(flags & VM_MEM_GUARDED))
 			template |= INTEL_PTE_PTA;
 		template |= INTEL_PTE_NCACHE;
@@ -2459,19 +2939,32 @@ RetryPvList:
 
 	if (wired) {
 		template |= INTEL_PTE_WIRED;
-		pmap->stats.wired_count++;
+		OSAddAtomic(+1, (SInt32 *) &pmap->stats.wired_count);
 	}
 	pmap_store_pte(pte, template);
 
+	/* if this was a managed page we delayed unlocking the pv until here
+	 * to prevent pmap_page_protect et al from finding it until the pte
+	 * has been stored */
+
+	if (managed_page(pai)) {
+	  UNLOCK_PVH(pai);
+	}
+
 Done:
 	if (need_tlbflush == TRUE)
 	        PMAP_UPDATE_TLBS(pmap, vaddr, vaddr + PAGE_SIZE);
 
-	if (pv_e != PV_ENTRY_NULL) {
-	        PV_FREE(pv_e);
+	if (pvh_e != PV_HASHED_ENTRY_NULL) {
+	        PV_HASHED_FREE_LIST(pvh_e, pvh_e, 1);
+	}
+
+	if (pvh_new != PV_HASHED_ENTRY_NULL) {
+	  PV_HASHED_KERN_FREE_LIST(pvh_new, pvh_new, 1);
 	}
 
-	PMAP_READ_UNLOCK(pmap, spl);
+	PMAP_UNLOCK(pmap);
+	PMAP_TRACE(PMAP_CODE(PMAP__ENTER) | DBG_FUNC_END, 0, 0, 0, 0, 0);
 }
 
 /*
@@ -2488,14 +2981,12 @@ pmap_change_wiring(
 	boolean_t	wired)
 {
 	register pt_entry_t	*pte;
-	spl_t			spl;
 
-#if 1
 	/*
 	 *	We must grab the pmap system lock because we may
 	 *	change a pte_page queue.
 	 */
-	PMAP_READ_LOCK(map, spl);
+	PMAP_LOCK(map);
 
 	if ((pte = pmap_pte(map, vaddr)) == PT_ENTRY_NULL)
 		panic("pmap_change_wiring: pte missing");
@@ -2504,26 +2995,19 @@ pmap_change_wiring(
 	    /*
 	     *	wiring down mapping
 	     */
-	    map->stats.wired_count++;
+	    OSAddAtomic(+1, (SInt32 *) &map->stats.wired_count);
 	    pmap_update_pte(pte, *pte, (*pte | INTEL_PTE_WIRED));
-	    pte++;
 	}
 	else if (!wired && iswired(*pte)) {
 	    /*
 	     *	unwiring mapping
 	     */
 	    assert(map->stats.wired_count >= 1);
-	    map->stats.wired_count--;
+	    OSAddAtomic(-1, (SInt32 *) &map->stats.wired_count);
 	    pmap_update_pte(pte, *pte, (*pte & ~INTEL_PTE_WIRED));
-	    pte++;
 	}
 
-	PMAP_READ_UNLOCK(map, spl);
-
-#else
-	return;
-#endif
-
+	PMAP_UNLOCK(map);
 }
 
 ppnum_t
@@ -2565,6 +3049,7 @@ pmap_extract(
 
 	paddr = (vm_offset_t)0;
 	ppn = pmap_find_phys(pmap, vaddr);
+
 	if (ppn) {
 	        paddr = ((vm_offset_t)i386_ptob(ppn)) | (vaddr & INTEL_OFFMASK);
 	}
@@ -2586,9 +3071,9 @@ pmap_expand_pml4(
 	if (kernel_pmap == map) panic("expand kernel pml4");
 
 	spl = splhigh();
-	  pml4p = pmap64_pml4(map, vaddr);
-	  splx(spl);
-	  if (PML4_ENTRY_NULL == pml4p) panic("pmap_expand_pml4 no pml4p");
+	pml4p = pmap64_pml4(map, vaddr);
+	splx(spl);
+	if (PML4_ENTRY_NULL == pml4p) panic("pmap_expand_pml4 no pml4p");
 
 	/*
 	 *	Allocate a VM page for the pml4 page
@@ -2604,53 +3089,47 @@ pmap_expand_pml4(
 	pa = i386_ptob(pn);
 	i = pml4idx(map, vaddr);
 
-	vm_object_lock(map->pm_obj_pml4);
-#if 0 /* DEBUG */
-	if (0 != vm_page_lookup(map->pm_obj_pml4, (vm_object_offset_t)i)) {
-	  kprintf("pmap_expand_pml4: obj_pml4 not empty, pmap 0x%x pm_obj_pml4 0x%x vaddr 0x%llx i 0x%llx\n",
-		  map, map->pm_obj_pml4, vaddr, i);
-	}
-#endif
-	vm_page_insert(m, map->pm_obj_pml4, (vm_object_offset_t)i);
+	/*
+	 *	Zero the page.
+	 */
+	pmap_zero_page(pn);
 
 	vm_page_lock_queues();
 	vm_page_wire(m);
-
-	vm_page_unlock_queues();
-	vm_object_unlock(map->pm_obj_pml4);
 	inuse_ptepages_count++;
-	map->stats.resident_count++;
-	map->stats.wired_count++;
+	vm_page_unlock_queues();
 
-	/*
-	 *	Zero the page.
-	 */
-	pmap_zero_page(pn);
+	/* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
+	vm_object_lock(map->pm_obj_pml4);
 
-	PMAP_READ_LOCK(map, spl);
+	PMAP_LOCK(map);
 	/*
 	 *	See if someone else expanded us first
 	 */
 	if (pmap64_pdpt(map, vaddr) != PDPT_ENTRY_NULL) {
-		PMAP_READ_UNLOCK(map, spl);
-		vm_object_lock(map->pm_obj_pml4);
+	        PMAP_UNLOCK(map);
+		vm_object_unlock(map->pm_obj_pml4);
+
 		vm_page_lock_queues();
 		vm_page_free(m);
 		inuse_ptepages_count--;
-		map->stats.resident_count--;
-		map->stats.wired_count--;
-
 		vm_page_unlock_queues();
-		vm_object_unlock(map->pm_obj_pml4);
+
 		return;
 	}
 
+#if 0 /* DEBUG */
+       if (0 != vm_page_lookup(map->pm_obj_pml4, (vm_object_offset_t)i)) {
+	       panic("pmap_expand_pml4: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
+		     map, map->pm_obj_pml4, vaddr, i);
+       }
+#endif
+	vm_page_insert(m, map->pm_obj_pml4, (vm_object_offset_t)i);
+	vm_object_unlock(map->pm_obj_pml4);
+
 	/*
 	 *	Set the page directory entry for this page table.
-	 *	If we have allocated more than one hardware page,
-	 *	set several page directory entries.
 	 */
-
 	pml4p = pmap64_pml4(map, vaddr); /* refetch under lock */
 
 	pmap_store_pte(pml4p, pa_to_pte(pa)
@@ -2658,7 +3137,7 @@ pmap_expand_pml4(
 				| INTEL_PTE_USER
 				| INTEL_PTE_WRITE);
 
-	PMAP_READ_UNLOCK(map, spl);
+	PMAP_UNLOCK(map);
 
 	return;
 
@@ -2679,13 +3158,12 @@ pmap_expand_pdpt(
 	if (kernel_pmap == map) panic("expand kernel pdpt");
 
 	spl = splhigh();
-	  while ((pdptp = pmap64_pdpt(map, vaddr)) == PDPT_ENTRY_NULL) {
-	    splx(spl);
-	    pmap_expand_pml4(map, vaddr); /* need room for another pdpt entry */
-	    spl = splhigh();
-	  }
-	  splx(spl);
-
+	while ((pdptp = pmap64_pdpt(map, vaddr)) == PDPT_ENTRY_NULL) {
+	        splx(spl);
+		pmap_expand_pml4(map, vaddr); /* need room for another pdpt entry */
+		spl = splhigh();
+	}
+	splx(spl);
 
 	/*
 	 *	Allocate a VM page for the pdpt page
@@ -2701,53 +3179,47 @@ pmap_expand_pdpt(
 	pa = i386_ptob(pn);
 	i = pdptidx(map, vaddr);
 
-	vm_object_lock(map->pm_obj_pdpt);
-#if 0 /* DEBUG */
-	if (0 != vm_page_lookup(map->pm_obj_pdpt, (vm_object_offset_t)i)) {
-	  kprintf("pmap_expand_pdpt: obj_pdpt not empty, pmap 0x%x pm_obj_pdpt 0x%x vaddr 0x%llx i 0x%llx\n",
-		  map, map->pm_obj_pdpt, vaddr, i);
-	}
-#endif
-	vm_page_insert(m, map->pm_obj_pdpt, (vm_object_offset_t)i);
+	/*
+	 *	Zero the page.
+	 */
+	pmap_zero_page(pn);
 
 	vm_page_lock_queues();
 	vm_page_wire(m);
-
-	vm_page_unlock_queues();
-	vm_object_unlock(map->pm_obj_pdpt);
 	inuse_ptepages_count++;
-	map->stats.resident_count++;
-	map->stats.wired_count++;
+	vm_page_unlock_queues();
 
-	/*
-	 *	Zero the page.
-	 */
-	pmap_zero_page(pn);
+	/* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
+	vm_object_lock(map->pm_obj_pdpt);
 
-	PMAP_READ_LOCK(map, spl);
+	PMAP_LOCK(map);
 	/*
 	 *	See if someone else expanded us first
 	 */
 	if (pmap64_pde(map, vaddr) != PD_ENTRY_NULL) {
-		PMAP_READ_UNLOCK(map, spl);
-		vm_object_lock(map->pm_obj_pdpt);
+		PMAP_UNLOCK(map);
+		vm_object_unlock(map->pm_obj_pdpt);
+
 		vm_page_lock_queues();
 		vm_page_free(m);
 		inuse_ptepages_count--;
-		map->stats.resident_count--;
-		map->stats.wired_count--;
-
 		vm_page_unlock_queues();
-		vm_object_unlock(map->pm_obj_pdpt);
+
 		return;
 	}
 
+#if 0 /* DEBUG */
+       if (0 != vm_page_lookup(map->pm_obj_pdpt, (vm_object_offset_t)i)) {
+	       panic("pmap_expand_pdpt: obj not empty, pmap %p pm_obj %p vaddr 0x%llx i 0x%llx\n",
+		     map, map->pm_obj_pdpt, vaddr, i);
+       }
+#endif
+	vm_page_insert(m, map->pm_obj_pdpt, (vm_object_offset_t)i);
+	vm_object_unlock(map->pm_obj_pdpt);
+
 	/*
 	 *	Set the page directory entry for this page table.
-	 *	If we have allocated more than one hardware page,
-	 *	set several page directory entries.
 	 */
-
 	pdptp = pmap64_pdpt(map, vaddr); /* refetch under lock */
 
 	pmap_store_pte(pdptp, pa_to_pte(pa)
@@ -2755,7 +3227,7 @@ pmap_expand_pdpt(
 				| INTEL_PTE_USER
 				| INTEL_PTE_WRITE);
 
-	PMAP_READ_UNLOCK(map, spl);
+	PMAP_UNLOCK(map);
 
 	return;
 
@@ -2796,18 +3268,15 @@ pmap_expand(
 	 */
 
 	if (cpu_64bit && (map != kernel_pmap)) {
-	  spl = splhigh();
-	  while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
-	    splx(spl);
-	    pmap_expand_pdpt(map, vaddr); /* need room for another pde entry */
-	    spl = splhigh();
-	  }
-	  splx(spl);
-	} else {
-	  pdp = pmap_pde(map, vaddr);
+	        spl = splhigh();
+		while ((pdp = pmap64_pde(map, vaddr)) == PD_ENTRY_NULL) {
+		        splx(spl);
+			pmap_expand_pdpt(map, vaddr); /* need room for another pde entry */
+			spl = splhigh();
+		}
+		splx(spl);
 	}
 
-
 	/*
 	 *	Allocate a VM page for the pde entries.
 	 */
@@ -2822,59 +3291,60 @@ pmap_expand(
 	pa = i386_ptob(pn);
 	i = pdeidx(map, vaddr);
 
-	vm_object_lock(map->pm_obj);
-#if 0 /* DEBUG */
-	if (0 != vm_page_lookup(map->pm_obj, (vm_object_offset_t)i)) {
-	  kprintf("pmap_expand: obj not empty, pmap 0x%x pm_obj 0x%x vaddr 0x%llx i 0x%llx\n",
-		  map, map->pm_obj, vaddr, i);
-	}
-#endif
-	vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
+	/*
+	 *	Zero the page.
+	 */
+	pmap_zero_page(pn);
 
 	vm_page_lock_queues();
 	vm_page_wire(m);
 	inuse_ptepages_count++;
-
 	vm_page_unlock_queues();
-	vm_object_unlock(map->pm_obj);
 
-	/*
-	 *	Zero the page.
-	 */
-	pmap_zero_page(pn);
+	/* Take the oject lock (mutex) before the PMAP_LOCK (spinlock) */
+	vm_object_lock(map->pm_obj);
 
-	PMAP_READ_LOCK(map, spl);
+	PMAP_LOCK(map);
 	/*
 	 *	See if someone else expanded us first
 	 */
+
 	if (pmap_pte(map, vaddr) != PT_ENTRY_NULL) {
-		PMAP_READ_UNLOCK(map, spl);
-		vm_object_lock(map->pm_obj);
+		PMAP_UNLOCK(map);
+		vm_object_unlock(map->pm_obj);
 
 		vm_page_lock_queues();
 		vm_page_free(m);
 		inuse_ptepages_count--;
-
 		vm_page_unlock_queues();
-		vm_object_unlock(map->pm_obj);
+
 		return;
 	}
 
-	pdp = pmap_pde(map, vaddr); /* refetch while locked */
+#if 0 /* DEBUG */
+       if (0 != vm_page_lookup(map->pm_obj, (vm_object_offset_t)i)) {
+	       panic("pmap_expand: obj not empty, pmap 0x%x pm_obj 0x%x vaddr 0x%llx i 0x%llx\n",
+		     map, map->pm_obj, vaddr, i);
+       }
+#endif
+	vm_page_insert(m, map->pm_obj, (vm_object_offset_t)i);
+	vm_object_unlock(map->pm_obj);
 
 	/*
-	 *	Set the page directory entry for this page table.
-	 *	If we have allocated more than one hardware page,
-	 *	set several page directory entries.
+	 * refetch while locked 
 	 */
 
+	pdp = pmap_pde(map, vaddr);
+
+	/*
+	 *	Set the page directory entry for this page table.
+	 */
 	pmap_store_pte(pdp, pa_to_pte(pa)
 				| INTEL_PTE_VALID
 				| INTEL_PTE_USER
 				| INTEL_PTE_WRITE);
-	    
 
-	PMAP_READ_UNLOCK(map, spl);
+	PMAP_UNLOCK(map);
 
 	return;
 }
@@ -2904,6 +3374,10 @@ pmap_sync_page_attributes_phys(ppnum_t pa)
 	cache_flush_page_phys(pa);
 }
 
+
+
+#ifdef CURRENTLY_UNUSED_AND_UNTESTED
+
 int	collect_ref;
 int	collect_unref;
 
@@ -2925,7 +3399,6 @@ pmap_collect(
 	register pt_entry_t	*pdp, *ptp;
 	pt_entry_t		*eptp;
 	int			wired;
-	spl_t                   spl;
 
 	if (p == PMAP_NULL)
 		return;
@@ -2936,7 +3409,7 @@ pmap_collect(
 	/*
 	 *	Garbage collect map.
 	 */
-	PMAP_READ_LOCK(p, spl);
+	PMAP_LOCK(p);
 
 	for (pdp = (pt_entry_t *)p->dirbase;
 	     pdp < (pt_entry_t *)&p->dirbase[(UMAXPTDI+1)];
@@ -2979,7 +3452,7 @@ pmap_collect(
 		     */
 		    pmap_store_pte(pdp, 0x0);
 		 
-		    PMAP_READ_UNLOCK(p, spl);
+		    PMAP_UNLOCK(p);
 
 		    /*
 		     * And free the pte page itself.
@@ -2988,37 +3461,39 @@ pmap_collect(
 			register vm_page_t m;
 
 			vm_object_lock(p->pm_obj);
+
 			m = vm_page_lookup(p->pm_obj,(vm_object_offset_t)(pdp - (pt_entry_t *)&p->dirbase[0]));
 			if (m == VM_PAGE_NULL)
 			    panic("pmap_collect: pte page not in object");
+
 			vm_page_lock_queues();
 			vm_page_free(m);
 			inuse_ptepages_count--;
 			vm_page_unlock_queues();
+
 			vm_object_unlock(p->pm_obj);
 		    }
 
-		    PMAP_READ_LOCK(p, spl);
+		    PMAP_LOCK(p);
 		}
 	      }
 	   }
 	}
-	PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
 
-	PMAP_READ_UNLOCK(p, spl);
+	PMAP_UPDATE_TLBS(p, 0x0, 0xFFFFFFFFFFFFF000ULL);
+	PMAP_UNLOCK(p);
 	return;
 
 }
+#endif
 
 
 void
-pmap_copy_page(src, dst)
-	ppnum_t src;
-	ppnum_t dst;
+pmap_copy_page(ppnum_t src, ppnum_t dst)
 {
-        bcopy_phys((addr64_t)i386_ptob(src),
-		   (addr64_t)i386_ptob(dst),
-		   PAGE_SIZE);
+  bcopy_phys((addr64_t)i386_ptob(src),
+	     (addr64_t)i386_ptob(dst),
+	     PAGE_SIZE);
 }
 
 
@@ -3053,35 +3528,36 @@ pmap_pageable(
  */
 void
 phys_attribute_clear(
-	ppnum_t	pn,
+	ppnum_t		pn,
 	int		bits)
 {
-	pv_entry_t		pv_h;
-	register pv_entry_t	pv_e;
+	pv_rooted_entry_t		pv_h;
+	register pv_hashed_entry_t	pv_e;
 	register pt_entry_t	*pte;
 	int			pai;
 	register pmap_t		pmap;
-	spl_t			spl;
-	pmap_paddr_t            phys;
 
+	pmap_intr_assert();
 	assert(pn != vm_page_fictitious_addr);
-	if (!valid_page(pn)) {
+	if (pn == vm_page_guard_addr)
+		return;
+
+	pai = ppn_to_pai(pn);
+
+	if (!managed_page(pai)) {
 	    /*
 	     *	Not a managed page.
 	     */
 	    return;
 	}
 
-	/*
-	 *	Lock the pmap system first, since we will be changing
-	 *	several pmaps.
-	 */
+	PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_START,
+		   (int) pn, bits, 0, 0, 0);
 
-	PMAP_WRITE_LOCK(spl);
-	phys = i386_ptob(pn);
-	pai = pa_index(phys);
 	pv_h = pai_to_pvh(pai);
 
+	LOCK_PVH(pai);
+
 	/*
 	 * Walk down PV list, clearing all modify or reference bits.
 	 * We do not have to lock the pv_list because we have
@@ -3091,70 +3567,74 @@ phys_attribute_clear(
 	    /*
 	     * There are some mappings.
 	     */
-	    for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
 
+	  pv_e = (pv_hashed_entry_t)pv_h;
+
+	  do {
 		pmap = pv_e->pmap;
-		/*
-		 * Lock the pmap to block pmap_extract and similar routines.
-		 */
-		simple_lock(&pmap->lock);
 
 		{
-		    register vm_map_offset_t va;
+		    vm_map_offset_t va;
 
 		    va = pv_e->va;
-
 		    /*
 		     * first make sure any processor actively
-		     * using this pmap fluses its TLB state
+		     * using this pmap, flushes its TLB state
 		     */
 
 		    PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
 
-		/*
-		 * Clear modify or reference bits.
-		 */
+		    /*
+		     * Clear modify and/or reference bits.
+		     */
 
 		    pte = pmap_pte(pmap, va);
 		    pmap_update_pte(pte, *pte, (*pte & ~bits));
 
 		}
-		simple_unlock(&pmap->lock);
 
-	    }
-	}
+		pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
 
+	  } while (pv_e != (pv_hashed_entry_t)pv_h);
+	}
 	pmap_phys_attributes[pai] &= ~bits;
 
-	PMAP_WRITE_UNLOCK(spl);
+	UNLOCK_PVH(pai);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__ATTRIBUTE_CLEAR) | DBG_FUNC_END,
+		   0, 0, 0, 0, 0);
+
 }
 
 /*
  *	Check specified attribute bits.
  */
-boolean_t
+int
 phys_attribute_test(
-	ppnum_t	pn,
+	ppnum_t		pn,
 	int		bits)
 {
-	pv_entry_t		pv_h;
-	register pv_entry_t	pv_e;
+	pv_rooted_entry_t		pv_h;
+	register pv_hashed_entry_t	pv_e;
 	register pt_entry_t	*pte;
 	int			pai;
 	register pmap_t		pmap;
-	spl_t			spl;
-	pmap_paddr_t            phys;
+	int			attributes = 0;
 
+	pmap_intr_assert();
 	assert(pn != vm_page_fictitious_addr);
-	if (!valid_page(pn)) {
+	if (pn == vm_page_guard_addr)
+		return 0;
+
+	pai = ppn_to_pai(pn);
+
+	if (!managed_page(pai)) {
 	    /*
 	     *	Not a managed page.
 	     */
-	    return (FALSE);
+	    return (0);
 	}
 
-	phys = i386_ptob(pn);
-	pai = pa_index(phys);
 	/*
 	 * super fast check...  if bits already collected
 	 * no need to take any locks...
@@ -3162,23 +3642,18 @@ phys_attribute_test(
 	 * the lock in case they got pulled in while
 	 * we were waiting for the lock
 	 */
-	if (pmap_phys_attributes[pai] & bits)
-	    return (TRUE);
+	if ( (pmap_phys_attributes[pai] & bits) == bits)
+	    return (bits);
+
 	pv_h = pai_to_pvh(pai);
 
-	/*
-	 *	Lock the pmap system first, since we will be checking
-	 *	several pmaps.
-	 */
-	PMAP_WRITE_LOCK(spl);
+	LOCK_PVH(pai);
 
-	if (pmap_phys_attributes[pai] & bits) {
-	    PMAP_WRITE_UNLOCK(spl);
-	    return (TRUE);
-	}
+	attributes = pmap_phys_attributes[pai] & bits;
 
 	/*
-	 * Walk down PV list, checking all mappings.
+	 * Walk down PV list, checking the mappings until we
+	 * reach the end or we've found the attributes we've asked for
 	 * We do not have to lock the pv_list because we have
 	 * the entire pmap system locked.
 	 */
@@ -3186,44 +3661,37 @@ phys_attribute_test(
 	    /*
 	     * There are some mappings.
 	     */
-	    for (pv_e = pv_h; pv_e != PV_ENTRY_NULL; pv_e = pv_e->next) {
+	  pv_e = (pv_hashed_entry_t)pv_h;
+	  if (attributes != bits) do {
 
-		pmap = pv_e->pmap;
-		/*
-		 * Lock the pmap to block pmap_extract and similar routines.
-		 */
-		simple_lock(&pmap->lock);
+	        pmap = pv_e->pmap;
 
 		{
-		    register vm_map_offset_t va;
+		    vm_map_offset_t va;
 
 		    va = pv_e->va;
-		    pte = pmap_pte(pmap, va);
+		    /*
+		     * first make sure any processor actively
+		     * using this pmap, flushes its TLB state
+		     */
+		    PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE);
 
-#if	0
 		    /*
-		     * Consistency checks.
+		     * pick up modify and/or reference bits from this mapping
 		     */
-		    assert(*pte & INTEL_PTE_VALID);
-		    /* assert(pte_to_phys(*pte) == phys); */
-#endif
-		}
 
-		/*
-		 * Check modify or reference bits.
-		 */
-		{
-			if (*pte++ & bits) {
-			    simple_unlock(&pmap->lock);
-			    PMAP_WRITE_UNLOCK(spl);
-			    return (TRUE);
-			}
+		    pte = pmap_pte(pmap, va);
+		    attributes |= *pte & bits;
+
 		}
-		simple_unlock(&pmap->lock);
-	    }
+
+		pv_e = (pv_hashed_entry_t)queue_next(&pv_e->qlink);
+
+	    } while ((attributes != bits) && (pv_e != (pv_hashed_entry_t)pv_h));
 	}
-	PMAP_WRITE_UNLOCK(spl);
-	return (FALSE);
+
+	UNLOCK_PVH(pai);
+	return (attributes);
 }
 
 /*
@@ -3231,29 +3699,30 @@ phys_attribute_test(
  */
 void
 phys_attribute_set(
-	ppnum_t	pn,
+	ppnum_t		pn,
 	int		bits)
 {
-	int			spl;
-	pmap_paddr_t   phys;
+	int		pai;
 
+	pmap_intr_assert();
 	assert(pn != vm_page_fictitious_addr);
-	if (!valid_page(pn)) {
+	if (pn == vm_page_guard_addr)
+		return;
+
+	pai = ppn_to_pai(pn);
+
+	if (!managed_page(pai)) {
 	    /*
 	     *	Not a managed page.
 	     */
 	    return;
 	}
 
-	/*
-	 *	Lock the pmap system and set the requested bits in
-	 *	the phys attributes array.  Don't need to bother with
-	 *	ptes because the test routine looks here first.
-	 */
-	phys = i386_ptob(pn);
-	PMAP_WRITE_LOCK(spl);
-	pmap_phys_attributes[pa_index(phys)] |= bits;
-	PMAP_WRITE_UNLOCK(spl);
+	LOCK_PVH(pai);
+
+	pmap_phys_attributes[pai] |= bits;
+
+	UNLOCK_PVH(pai);
 }
 
 /*
@@ -3288,7 +3757,10 @@ boolean_t
 pmap_is_modified(
 		 ppnum_t pn)
 {
-	return (phys_attribute_test(pn, PHYS_MODIFIED));
+        if (phys_attribute_test(pn, PHYS_MODIFIED))
+	        return TRUE;
+
+	return FALSE;
 }
 
 /*
@@ -3321,7 +3793,10 @@ boolean_t
 pmap_is_referenced(
 		   ppnum_t pn)
 {
-	return (phys_attribute_test(pn, PHYS_REFERENCED));
+        if (phys_attribute_test(pn, PHYS_REFERENCED))
+	        return TRUE;
+
+	return FALSE;
 }
 
 /*
@@ -3332,8 +3807,17 @@ pmap_is_referenced(
 unsigned int
 pmap_get_refmod(ppnum_t pa)
 {
-	return (   ((phys_attribute_test(pa,   PHYS_MODIFIED))?   VM_MEM_MODIFIED : 0)
-			 | ((phys_attribute_test(pa, PHYS_REFERENCED))? VM_MEM_REFERENCED : 0));
+        int	refmod;
+	unsigned int retval = 0;
+
+	refmod = phys_attribute_test(pa, PHYS_MODIFIED | PHYS_REFERENCED);
+
+	if (refmod & PHYS_MODIFIED)
+	        retval |= VM_MEM_MODIFIED;
+	if (refmod & PHYS_REFERENCED)
+	        retval |= VM_MEM_REFERENCED;
+
+	return (retval);
 }
 
 /*
@@ -3351,63 +3835,6 @@ pmap_clear_refmod(ppnum_t pa, unsigned int mask)
 	phys_attribute_clear(pa, x86Mask);
 }
 
-/*
- *	Set the modify bit on the specified range
- *	of this map as requested.
- *
- *	This optimization stands only if each time the dirty bit
- *	in vm_page_t is tested, it is also tested in the pmap.
- */
-void
-pmap_modify_pages(
-	pmap_t		map,
-	vm_map_offset_t	sva,
-	vm_map_offset_t	eva)
-{
-	spl_t			spl;
-	register pt_entry_t	*pde;
-	register pt_entry_t	*spte, *epte;
-	vm_map_offset_t		lva;
-	vm_map_offset_t		orig_sva;
-
-	if (map == PMAP_NULL)
-		return;
-
-	PMAP_READ_LOCK(map, spl);
-
-	orig_sva = sva;
-	while (sva && sva < eva) {
-	    lva = (sva + pde_mapped_size) & ~(pde_mapped_size-1);
-	    if (lva > eva)
-		lva = eva;
-	    pde = pmap_pde(map, sva);
-	    if (pde && (*pde & INTEL_PTE_VALID)) {
-	      spte = (pt_entry_t *)pmap_pte(map, (sva & ~(pde_mapped_size-1)));
-		if (lva) {
-		   spte = &spte[ptenum(sva)];
-		   epte = &spte[intel_btop(lva-sva)];
-	        } else {
-		   epte = &spte[intel_btop(pde_mapped_size)];
-		   spte = &spte[ptenum(sva)];
-	        }
-		while (spte < epte) {
-		    if (*spte & INTEL_PTE_VALID) {
-			pmap_store_pte(spte, *spte
-						| INTEL_PTE_MOD
-						| INTEL_PTE_WRITE);
-		    }
-		    spte++;
-		}
-	    }
-	    sva = lva;
-	    pde++;
-	}
-	PMAP_UPDATE_TLBS(map, orig_sva, eva);
-
-	PMAP_READ_UNLOCK(map, spl);
-}
-
-
 void 
 invalidate_icache(__unused vm_offset_t	addr,
 		  __unused unsigned	cnt,
@@ -3423,12 +3850,38 @@ flush_dcache(__unused vm_offset_t	addr,
 	return;
 }
 
+#if CONFIG_DTRACE
+/*
+ * Constrain DTrace copyin/copyout actions
+ */
+extern kern_return_t dtrace_copyio_preflight(addr64_t);
+extern kern_return_t dtrace_copyio_postflight(addr64_t);
+
+kern_return_t dtrace_copyio_preflight(__unused addr64_t va)
+{
+	thread_t thread = current_thread();
+
+	if (current_map() == kernel_map)
+		return KERN_FAILURE;
+	else if (thread->machine.specFlags & CopyIOActive)
+		return KERN_FAILURE;
+	else
+		return KERN_SUCCESS;
+}
+ 
+kern_return_t dtrace_copyio_postflight(__unused addr64_t va)
+{
+	return KERN_SUCCESS;
+}
+#endif /* CONFIG_DTRACE */
+
 #if	MACH_KDB
 
 /* show phys page mappings and attributes */
 
 extern void	db_show_page(pmap_paddr_t pa);
 
+#if 0
 void
 db_show_page(pmap_paddr_t pa)
 {
@@ -3440,7 +3893,7 @@ db_show_page(pmap_paddr_t pa)
 	pv_h = pai_to_pvh(pai);
 
 	attr = pmap_phys_attributes[pai];
-	printf("phys page %x ", pa);
+	printf("phys page %llx ", pa);
 	if (attr & PHYS_MODIFIED)
 		printf("modified, ");
 	if (attr & PHYS_REFERENCED)
@@ -3451,12 +3904,14 @@ db_show_page(pmap_paddr_t pa)
 		printf(" not mapped\n");
 	for (; pv_h; pv_h = pv_h->next)
 		if (pv_h->pmap)
-			printf("%x in pmap %x\n", pv_h->va, pv_h->pmap);
+			printf("%llx in pmap %p\n", pv_h->va, pv_h->pmap);
 }
+#endif
 
 #endif /* MACH_KDB */
 
 #if	MACH_KDB
+#if 0
 void db_kvtophys(vm_offset_t);
 void db_show_vaddrs(pt_entry_t  *);
 
@@ -3498,7 +3953,7 @@ db_show_vaddrs(
 			continue;
 		}
 		pdecnt++;
-		ptep = (pt_entry_t *) ((*pdep) & ~INTEL_OFFMASK);
+		ptep = (pt_entry_t *) ((unsigned long)(*pdep) & ~INTEL_OFFMASK);
 		db_printf("dir[%4d]: 0x%x\n", y, *pdep);
 		for (x = 0; x < NPTEPG; x++, ptep++) {
 			if (((tmp = *ptep) & INTEL_PTE_VALID) == 0) {
@@ -3516,6 +3971,7 @@ db_show_vaddrs(
 	db_printf("total: %d tables, %d page table entries.\n", pdecnt, ptecnt);
 
 }
+#endif
 #endif	/* MACH_KDB */
 
 #include <mach_vm_debug.h>
@@ -3555,37 +4011,67 @@ boolean_t
 phys_page_exists(
 		 ppnum_t pn)
 {
-	pmap_paddr_t     phys;
-
 	assert(pn != vm_page_fictitious_addr);
 
 	if (!pmap_initialized)
 		return (TRUE);
-	phys = (pmap_paddr_t) i386_ptob(pn);
-	if (!pmap_valid_page(pn))
+
+	if (pn == vm_page_guard_addr)
+		return FALSE;
+
+	if (!managed_page(ppn_to_pai(pn)))
 		return (FALSE);
 
 	return TRUE;
 }
 
 void
-mapping_free_prime()
+mapping_free_prime(void)
 {
 	int             i;
-	pv_entry_t      pv_e;
+	pv_hashed_entry_t      pvh_e;
+	pv_hashed_entry_t      pvh_eh;
+	pv_hashed_entry_t      pvh_et;
+	int		pv_cnt;
+
+	pv_cnt = 0;
+	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
+	for (i = 0; i < (5 * PV_HASHED_ALLOC_CHUNK); i++) {
+		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
+
+		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+		pvh_eh = pvh_e;
+
+		if (pvh_et == PV_HASHED_ENTRY_NULL)
+		        pvh_et = pvh_e;
+		pv_cnt++;
+	}
+	PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 
-	for (i = 0; i < (5 * PV_ALLOC_CHUNK); i++) {
-		pv_e = (pv_entry_t) zalloc(pv_list_zone);
-		PV_FREE(pv_e);
+	pv_cnt = 0;
+	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
+	for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK; i++) {
+		pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
+
+		pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+		pvh_eh = pvh_e;
+
+		if (pvh_et == PV_HASHED_ENTRY_NULL)
+		        pvh_et = pvh_e;
+		pv_cnt++;
 	}
+	PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
+
 }
 
 void
-mapping_adjust()
+mapping_adjust(void)
 {
-	pv_entry_t      pv_e;
+	pv_hashed_entry_t      pvh_e;
+	pv_hashed_entry_t      pvh_eh;
+	pv_hashed_entry_t      pvh_et;
+	int		pv_cnt;
 	int             i;
-	int             spl;
 
 	if (mapping_adjust_call == NULL) {
 		thread_call_setup(&mapping_adjust_call_data,
@@ -3593,14 +4079,37 @@ mapping_adjust()
 				  (thread_call_param_t) NULL);
 		mapping_adjust_call = &mapping_adjust_call_data;
 	}
-	/* XXX  rethink best way to do locking here */
-	if (pv_free_count < PV_LOW_WATER_MARK) {
-		for (i = 0; i < PV_ALLOC_CHUNK; i++) {
-			pv_e = (pv_entry_t) zalloc(pv_list_zone);
-			SPLVM(spl);
-			PV_FREE(pv_e);
-			SPLX(spl);
+
+	pv_cnt = 0;
+	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
+	if (pv_hashed_kern_free_count < PV_HASHED_KERN_LOW_WATER_MARK) {
+		for (i = 0; i < PV_HASHED_KERN_ALLOC_CHUNK; i++) {
+			pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
+
+			pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+			pvh_eh = pvh_e;
+
+			if (pvh_et == PV_HASHED_ENTRY_NULL)
+			        pvh_et = pvh_e;
+			pv_cnt++;
+		}
+		PV_HASHED_KERN_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
+	}
+
+	pv_cnt = 0;
+	pvh_eh = pvh_et = PV_HASHED_ENTRY_NULL;
+	if (pv_hashed_free_count < PV_HASHED_LOW_WATER_MARK) {
+		for (i = 0; i < PV_HASHED_ALLOC_CHUNK; i++) {
+			pvh_e = (pv_hashed_entry_t) zalloc(pv_hashed_list_zone);
+
+			pvh_e->qlink.next = (queue_entry_t)pvh_eh;
+			pvh_eh = pvh_e;
+
+			if (pvh_et == PV_HASHED_ENTRY_NULL)
+			        pvh_et = pvh_e;
+			pv_cnt++;
 		}
+		PV_HASHED_FREE_LIST(pvh_eh, pvh_et, pv_cnt);
 	}
 	mappingrecurse = 0;
 }
@@ -3608,84 +4117,49 @@ mapping_adjust()
 void
 pmap_commpage32_init(vm_offset_t kernel_commpage, vm_offset_t user_commpage, int cnt)
 {
-  int i;
-  pt_entry_t *opte, *npte;
-  pt_entry_t pte;
-
-
-  for (i = 0; i < cnt; i++) {
-    opte = pmap_pte(kernel_pmap, (vm_map_offset_t)kernel_commpage);
-    if (0 == opte) panic("kernel_commpage");
-    pte = *opte | INTEL_PTE_USER|INTEL_PTE_GLOBAL;
-    pte &= ~INTEL_PTE_WRITE; // ensure read only
-    npte = pmap_pte(kernel_pmap, (vm_map_offset_t)user_commpage);
-    if (0 == npte) panic("user_commpage");
-    pmap_store_pte(npte, pte);
-    kernel_commpage += INTEL_PGBYTES;
-    user_commpage += INTEL_PGBYTES;
-  }
+	int i;
+	pt_entry_t *opte, *npte;
+	pt_entry_t pte;
+	spl_t s;
+
+	for (i = 0; i < cnt; i++) {
+	        s = splhigh();
+		opte = pmap_pte(kernel_pmap, (vm_map_offset_t)kernel_commpage);
+		if (0 == opte)
+			panic("kernel_commpage");
+		pte = *opte | INTEL_PTE_USER|INTEL_PTE_GLOBAL;
+		pte &= ~INTEL_PTE_WRITE; // ensure read only
+		npte = pmap_pte(kernel_pmap, (vm_map_offset_t)user_commpage);
+		if (0 == npte)
+			panic("user_commpage");
+		pmap_store_pte(npte, pte);
+		splx(s);
+		kernel_commpage += INTEL_PGBYTES;
+		user_commpage += INTEL_PGBYTES;
+	}
 }
 
+
 #define PMAP_COMMPAGE64_CNT  (_COMM_PAGE64_AREA_USED/PAGE_SIZE)
 pt_entry_t pmap_commpage64_ptes[PMAP_COMMPAGE64_CNT];
 
 void
 pmap_commpage64_init(vm_offset_t kernel_commpage, __unused vm_map_offset_t user_commpage, int cnt)
 {
-  spl_t s;
-  int i;
-  pt_entry_t *kptep;
-
-  s = splhigh();
-  for (i = 0; i< cnt; i++) {
-    kptep = pmap_pte(kernel_pmap, (uint64_t)kernel_commpage + (i*PAGE_SIZE));
-    if ((0 == kptep) || (0 == (*kptep & INTEL_PTE_VALID))) panic("pmap_commpage64_init pte");
-    pmap_commpage64_ptes[i] = ((*kptep & ~INTEL_PTE_WRITE) | INTEL_PTE_USER);
-  }
-  splx(s);
-
-}
-
-void
-pmap_map_sharedpage(__unused task_t task, pmap_t p)
-{
-  pt_entry_t *ptep;
-  spl_t s;
-  int i;
-
-  if (!p->pm_64bit) return;
-  /* setup high 64 bit commpage */
-  s = splhigh();  
-  while ((ptep = pmap_pte(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS)) == PD_ENTRY_NULL) {
-    splx(s);
-    pmap_expand(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS);
-    s = splhigh();
-  }
+    int i;
+    pt_entry_t *kptep;
 
-  for (i = 0; i< PMAP_COMMPAGE64_CNT; i++) {
-    ptep = pmap_pte(p, (uint64_t)_COMM_PAGE64_BASE_ADDRESS + (i*PAGE_SIZE));
-    if (0 == ptep) panic("pmap_map_sharedpage");
-    pmap_store_pte(ptep, pmap_commpage64_ptes[i]);
-  }
-  splx(s);
+    PMAP_LOCK(kernel_pmap);
 
+    for (i = 0; i < cnt; i++) {
+        kptep = pmap_pte(kernel_pmap, (uint64_t)kernel_commpage + (i*PAGE_SIZE));
+	if ((0 == kptep) || (0 == (*kptep & INTEL_PTE_VALID)))
+	    panic("pmap_commpage64_init pte");
+	pmap_commpage64_ptes[i] = ((*kptep & ~INTEL_PTE_WRITE) | INTEL_PTE_USER);
+    }
+    PMAP_UNLOCK(kernel_pmap);
 }
 
-void
-pmap_unmap_sharedpage(pmap_t pmap)
-{
-  spl_t s;
-  pt_entry_t *ptep;
-  int i;
-
-  if (!pmap->pm_64bit) return;
-  s = splhigh();
-  for (i = 0; i< PMAP_COMMPAGE64_CNT; i++) {
-    ptep = pmap_pte(pmap, (uint64_t)_COMM_PAGE64_BASE_ADDRESS + (i*PAGE_SIZE));
-  if (ptep) pmap_store_pte(ptep, 0);
-  }
-  splx(s);
-}
 
 static cpu_pmap_t		cpu_pmap_master;
 
@@ -3728,11 +4202,14 @@ pmap_cpu_alloc(boolean_t is_boot_cpu)
 		address = (vm_offset_t)mapaddr;
 
 		for (i = 0; i < PMAP_NWINDOWS; i++, address += PAGE_SIZE) {
+		  spl_t s;
+		        s = splhigh();
 			while ((pte = pmap_pte(kernel_pmap, (vm_map_offset_t)address)) == 0)
 				pmap_expand(kernel_pmap, (vm_map_offset_t)address);
 			* (int *) pte = 0; 
 			cp->mapwindow[i].prv_CADDR = (caddr_t) address;
 			cp->mapwindow[i].prv_CMAP = pte;
+			splx(s);
 		}
 		vm_map_unlock(kernel_map);
 	}
@@ -3758,13 +4235,8 @@ pmap_get_mapwindow(pt_entry_t pentry)
 {
     mapwindow_t *mp;
     int i;
-    boolean_t	istate;
 
-    /*
-     * can be called from hardware interrupt context
-     * so we need to protect the lookup process
-     */
-    istate = ml_set_interrupts_enabled(FALSE);
+    assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
 
     /*
      * Note: 0th map reserved for pmap_pte()
@@ -3773,18 +4245,33 @@ pmap_get_mapwindow(pt_entry_t pentry)
             mp = &current_cpu_datap()->cpu_pmap->mapwindow[i];
 
 	    if (*mp->prv_CMAP == 0) {
-	            *mp->prv_CMAP = pentry;
-		    break;
+	            pmap_store_pte(mp->prv_CMAP, pentry);
+
+		    invlpg((uintptr_t)mp->prv_CADDR);
+
+		    return (mp);
 	    }
     }
-    if (i >= PMAP_NWINDOWS)
-            mp = NULL;
-    (void) ml_set_interrupts_enabled(istate);
-    
-    return (mp);
+    panic("pmap_get_mapwindow: no windows available");
+
+    return NULL;
+}
+
+
+void
+pmap_put_mapwindow(mapwindow_t *mp)
+{
+    pmap_store_pte(mp->prv_CMAP, 0);
 }
 
 
+/*
+ * The Intel platform can nest at the PDE level, so NBPDE (i.e. 2MB) at a time,
+ * on a NBPDE boundary.
+ */
+uint64_t pmap_nesting_size_min = NBPDE;
+uint64_t pmap_nesting_size_max = 0 - (uint64_t)NBPDE; /* no limit, really... */
+
 /*
  *	kern_return_t pmap_nest(grand, subord, vstart, size)
  *
@@ -3808,85 +4295,76 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t n
 		
         vm_map_offset_t	vaddr, nvaddr;
 	pd_entry_t	*pde,*npde;
-	unsigned int	i, need_flush;
-	unsigned int	num_pde;
-	spl_t		s;
+	unsigned int	i;
+	uint64_t	num_pde;
 
 	// do validity tests
-
-	if(size & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this for multiples of 256MB */
+	if (size & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
+	if(vstart & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
+	if(nstart & (pmap_nesting_size_min-1)) return KERN_INVALID_VALUE;
 	if((size >> 28) > 65536)  return KERN_INVALID_VALUE;	/* Max size we can nest is 16TB */
-	if(vstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this aligned to 256MB */
-	if(nstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE;	/* We can only do this aligned to 256MB */
-	if(size == 0) {	   
+	if(size == 0) {
 		panic("pmap_nest: size is invalid - %016llX\n", size);
 	}
-	if ((size >> 28) != 1) panic("pmap_nest: size 0x%llx must be 0x%x", size, NBPDE);
-
-	subord->pm_shared = TRUE;
 
-	// prepopulate subord pmap pde's if necessary
+	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
+		   (int) grand, (int) subord,
+		   (int) (vstart>>32), (int) vstart, 0);
 
-	if (cpu_64bit) {
-	  s = splhigh();
-	  while (PD_ENTRY_NULL == (npde = pmap_pde(subord, nstart))) {
-	    splx(s);
-	    pmap_expand(subord, nstart);
-	    s = splhigh();
-	  }
-	  splx(s);
-	}
-
-	PMAP_READ_LOCK(subord,s);
+	subord->pm_shared = TRUE;
 	nvaddr = (vm_map_offset_t)nstart;
-	need_flush = 0;
 	num_pde = size >> PDESHIFT;
 
-	for (i=0;i<num_pde;i++) {
+	PMAP_LOCK(subord);
+	for (i = 0; i < num_pde; i++) {
 	  npde = pmap_pde(subord, nvaddr);
-	  if ((0 == npde) || (*npde++ & INTEL_PTE_VALID) == 0) {
-	    PMAP_READ_UNLOCK(subord,s);
+	  while (0 == npde || ((*npde & INTEL_PTE_VALID) == 0)) {
+	    PMAP_UNLOCK(subord);
 	    pmap_expand(subord, nvaddr); // pmap_expand handles races
-	    PMAP_READ_LOCK(subord,s);
-	    need_flush++;
+	    PMAP_LOCK(subord);
+	    npde = pmap_pde(subord, nvaddr);
 	  }
 	  nvaddr += NBPDE;
 	}
 
-	if (need_flush) {
-	  nvaddr = (vm_map_offset_t)nstart;
-	  PMAP_UPDATE_TLBS(subord, nvaddr, nvaddr + (1 << 28) -1 );
-	}
-	PMAP_READ_UNLOCK(subord,s);
+	PMAP_UNLOCK(subord);
 
-	// copy pde's from subord pmap into grand pmap
+	vaddr = (vm_map_offset_t)vstart;
 
-	if (cpu_64bit) {
-	  s = splhigh();
-	  while (PD_ENTRY_NULL == (pde = pmap_pde(grand, vstart))) {
-	    splx(s);
-	    pmap_expand(grand, vstart);
-	    s = splhigh();
-	  }
-	  splx(s);
-	}
+	PMAP_LOCK(grand);
 
-	PMAP_READ_LOCK(grand,s);
-	vaddr = (vm_map_offset_t)vstart;
-	for (i=0;i<num_pde;i++,pde++) {
+	for (i = 0;i < num_pde; i++) {
 	  pd_entry_t tpde;
+
 	  npde = pmap_pde(subord, nstart);
-	  if (npde == 0) panic("pmap_nest: no npde, subord 0x%x nstart 0x%llx", subord, nstart);
+	  if (npde == 0)
+		  panic("pmap_nest: no npde, subord %p nstart 0x%llx", subord, nstart);
 	  tpde = *npde;
 	  nstart += NBPDE;
 	  pde = pmap_pde(grand, vaddr);
-	  if (pde == 0) panic("pmap_nest: no pde, grand  0x%x vaddr 0x%llx", grand, vaddr);
+/* Legacy mode does not require expansion.
+ * DRK: consider a debug mode test to verify that no PTEs are extant within
+ * this range.
+ */
+	  if ((0 == pde) && cpu_64bit) {
+	    PMAP_UNLOCK(grand);
+	    pmap_expand_pdpt(grand, vaddr);
+	    PMAP_LOCK(grand);
+	    pde = pmap_pde(grand, vaddr);
+	  }
+
+	  if (pde == 0)
+		  panic("pmap_nest: no pde, grand  %p vaddr 0x%llx", grand, vaddr);
 	  vaddr += NBPDE;
 	  pmap_store_pte(pde, tpde);
 	}
-	PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1 << 28) -1 );
 
-	PMAP_READ_UNLOCK(grand,s);
+	/* XXX FBDP: why do we need to flush here ? */
+	PMAP_UPDATE_TLBS(grand, vstart, vstart + size - 1);
+
+	PMAP_UNLOCK(grand);
+
+	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
 
 	return KERN_SUCCESS;
 }
@@ -3902,30 +4380,48 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t n
  *	segment sized ranges.
  */
 
-kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr) {
+kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
 			
-	spl_t s;
 	pd_entry_t *pde;
 	unsigned int i;
 	unsigned int num_pde;
+	addr64_t vstart, vend;
+
+	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_START,
+		   (int) grand, 
+		   (int) (vaddr>>32), (int) vaddr, 0, 0);
 
-	PMAP_READ_LOCK(grand,s);
+	if ((size & (pmap_nesting_size_min-1)) ||
+	    (vaddr & (pmap_nesting_size_min-1))) {
+		panic("pmap_unnest(%p,0x%llx,0x%llx): unaligned...\n",
+		      grand, vaddr, size);
+	}
+
+	/* align everything to PDE boundaries */
+	vstart = vaddr & ~(NBPDE-1);
+	vend = (vaddr + size + NBPDE - 1) & ~(NBPDE-1);
+	size = vend - vstart;
+
+	PMAP_LOCK(grand);
 
 	// invalidate all pdes for segment at vaddr in pmap grand
 
-	num_pde = (1<<28) >> PDESHIFT;
+	num_pde = size >> PDESHIFT;
 
+	vaddr = vstart;
 	for (i=0;i<num_pde;i++,pde++) {
 	  pde = pmap_pde(grand, (vm_map_offset_t)vaddr);
-	  if (pde == 0) panic("pmap_unnest: no pde, grand 0x%x vaddr 0x%llx\n", grand, vaddr);
+	  if (pde == 0) panic("pmap_unnest: no pde, grand %p vaddr 0x%llx\n", grand, vaddr);
 	  pmap_store_pte(pde, (pd_entry_t)0);
 	  vaddr += NBPDE;
 	}
-	PMAP_UPDATE_TLBS(grand, vaddr, vaddr + (1<<28) -1 );
+	PMAP_UPDATE_TLBS(grand, vstart, vend);
 
-	PMAP_READ_UNLOCK(grand,s);
+	PMAP_UNLOCK(grand);
 		
-	return KERN_SUCCESS;								/* Bye, bye, butterfly... */
+	PMAP_TRACE(PMAP_CODE(PMAP__NEST) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+	return KERN_SUCCESS;
 }
 
 void
@@ -3985,7 +4481,7 @@ vm_offset_t pmap_high_map(pt_entry_t pte, enum high_cpu_types e)
 
   a = e + HIGH_CPU_END * cpu_number();
   vaddr = (vm_offset_t)pmap_index_to_virt(HIGH_FIXED_CPUS_BEGIN + a);
-  *(pte_unique_base + a) = pte;
+  pmap_store_pte(pte_unique_base + a, pte);
 
   /* TLB flush for this page for this  cpu */
   invlpg((uintptr_t)vaddr);
@@ -4014,7 +4510,8 @@ pmap_flush_tlbs(pmap_t	pmap)
 	boolean_t	flush_self = FALSE;
 	uint64_t	deadline;
 
-	assert(!ml_get_interrupts_enabled());
+	assert((processor_avail_count < 2) ||
+	       (ml_get_interrupts_enabled() && get_preemption_level() != 0));
 
 	/*
 	 * Scan other cpus for matching active or task CR3.
@@ -4026,8 +4523,8 @@ pmap_flush_tlbs(pmap_t	pmap)
 	for (cpu = 0, cpu_bit = 1; cpu < real_ncpus; cpu++, cpu_bit <<= 1) {
 		if (!cpu_datap(cpu)->cpu_running)
 			continue;
-		if ((cpu_datap(cpu)->cpu_task_cr3   == pmap_cr3) ||
-		    (CPU_GET_ACTIVE_CR3(cpu) == pmap_cr3) ||
+		if ((cpu_datap(cpu)->cpu_task_cr3 == pmap_cr3) ||
+		    (CPU_GET_ACTIVE_CR3(cpu)      == pmap_cr3) ||
 		    (pmap->pm_shared) ||
 		    ((pmap == kernel_pmap) &&
 		     (!CPU_CR3_IS_ACTIVE(cpu) ||
@@ -4046,9 +4543,10 @@ pmap_flush_tlbs(pmap_t	pmap)
 		}
 	}
 
-	if (cpus_to_signal) {
-	        KERNEL_DEBUG(0xef800024 | DBG_FUNC_START, cpus_to_signal, 0, 0, 0, 0);
+	PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS) | DBG_FUNC_START,
+		   (int) pmap, cpus_to_signal, flush_self, 0, 0);
 
+	if (cpus_to_signal) {
 		deadline = mach_absolute_time() + LockTimeOut;
 		/*
 		 * Wait for those other cpus to acknowledge
@@ -4061,16 +4559,17 @@ pmap_flush_tlbs(pmap_t	pmap)
 				        cpus_to_signal &= ~cpu_bit;
 					break;
 				}
-				if (mach_absolute_time() > deadline)
-				        panic("pmap_flush_tlbs() "
-					      "timeout pmap=%p cpus_to_signal=%p",
-					      pmap, cpus_to_signal);
+				if (mach_absolute_time() > deadline) {
+					force_immediate_debugger_NMI = TRUE;
+				        panic("pmap_flush_tlbs() timeout: "
+								"cpu %d failing to respond to interrupts, pmap=%p cpus_to_signal=%lx",
+								cpu, pmap, cpus_to_signal);
+				}
 				cpu_pause();
 			}
 		        if (cpus_to_signal == 0)
 			        break;
 		}
-	        KERNEL_DEBUG(0xef800024 | DBG_FUNC_END, cpus_to_signal, 0, 0, 0, 0);
 	}
 
 	/*
@@ -4082,11 +4581,16 @@ pmap_flush_tlbs(pmap_t	pmap)
 	if (flush_self)
 		flush_tlb();
 
+
+	PMAP_TRACE(PMAP_CODE(PMAP__FLUSH_TLBS) | DBG_FUNC_END,
+		   (int) pmap, cpus_to_signal, flush_self, 0, 0);
 }
 
 void
 process_pmap_updates(void)
 {
+	assert(ml_get_interrupts_enabled() == 0 || get_preemption_level() != 0);
+
 	flush_tlb();
 
 	current_cpu_datap()->cpu_tlb_invalid = FALSE;
@@ -4096,19 +4600,19 @@ process_pmap_updates(void)
 void
 pmap_update_interrupt(void)
 {
-        KERNEL_DEBUG(0xef800028 | DBG_FUNC_START, 0, 0, 0, 0, 0);
-
-	assert(!ml_get_interrupts_enabled());
+        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_START,
+		   0, 0, 0, 0, 0);
 
 	process_pmap_updates();
 
-        KERNEL_DEBUG(0xef800028 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+        PMAP_TRACE(PMAP_CODE(PMAP__UPDATE_INTERRUPT) | DBG_FUNC_END,
+		   0, 0, 0, 0, 0);
 }
 
 
 unsigned int pmap_cache_attributes(ppnum_t pn) {
 
-	if (!pmap_valid_page(pn))
+	if (!managed_page(ppn_to_pai(pn)))
 	        return (VM_WIMG_IO);
 
 	return (VM_WIMG_COPYBACK);
diff --git a/osfmk/i386/pmap.h b/osfmk/i386/pmap.h
index 67bccdd41..90f9ff8d7 100644
--- a/osfmk/i386/pmap.h
+++ b/osfmk/i386/pmap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -167,7 +173,7 @@ pmap_store_pte(pt_entry_t *entryp, pt_entry_t value)
 	 * If the compare succeeds, the new value will have been stored.
 	 * Otherwise, the old value changed and reloaded, so try again.
 	 */
-	asm volatile(
+	__asm__ volatile(
 		"	movl	(%0), %%eax	\n\t"
 		"	movl	4(%0), %%edx	\n\t"
 		"1:				\n\t"
@@ -212,6 +218,7 @@ pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new)
 #define pmap_update_pte(entryp, old, new) \
 	while (!pmap_cmpx_pte((entryp), (old), (new)))
 
+
 /* in 64 bit spaces, the number of each type of page in the page tables */
 #define NPML4PGS        (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t))))
 #define NPDPTPGS        (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t))))
@@ -385,6 +392,10 @@ extern pdpt_entry_t *IdlePDPT64;
 extern addr64_t     kernel64_cr3;
 extern boolean_t    no_shared_cr3;
 
+extern uint64_t pmap_pv_hashlist_walks;
+extern uint64_t pmap_pv_hashlist_cnts;
+extern uint32_t pmap_pv_hashlist_max;
+
 /*
  * virtual address to page table entry and
  * to physical address. Likewise for alternate address space.
@@ -415,9 +426,7 @@ struct pmap {
         vm_object_t     pm_obj;         /* object to hold pde's */
 	int		ref_count;	/* reference count */
         int		nx_enabled;
-        boolean_t       pm_64bit;
-        boolean_t       pm_kernel_cr3;
-        boolean_t       pm_shared;
+        task_map_t      pm_task_map;
 	decl_simple_lock_data(,lock)	/* lock on map */
 	struct pmap_statistics	stats;	/* map statistics */
 	vm_offset_t     pm_hold;        /* true pdpt zalloc addr */
@@ -427,6 +436,7 @@ struct pmap {
 	vm_object_t     pm_obj_pdpt;    /* holds pdpt pages */
 	vm_object_t     pm_obj_pml4;    /* holds pml4 pages */
 	vm_object_t     pm_obj_top;     /* holds single top level page */
+        boolean_t       pm_shared;
 };
 
 
@@ -455,6 +465,8 @@ typedef struct cpu_pmap {
 
 
 extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry);
+extern void         pmap_put_mapwindow(mapwindow_t *map);
+
 
 typedef struct pmap_memory_regions {
   ppnum_t base;
@@ -472,7 +484,7 @@ extern pmap_memory_region_t pmap_memory_regions[];
 
 static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) {
     	current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3);
-	current_cpu_datap()->cpu_task_map = tpmap->pm_64bit ? TASK_MAP_64BIT : TASK_MAP_32BIT;
+	current_cpu_datap()->cpu_task_map = tpmap->pm_task_map;
 }
 
 /*
@@ -489,6 +501,10 @@ extern void		pmap_update_interrupt(void);
 extern addr64_t		(kvtophys)(
 				vm_offset_t	addr);
 
+extern void		pmap_expand(
+				pmap_t		pmap,
+				vm_map_offset_t	addr);
+
 extern pt_entry_t	*pmap_pte(
 				struct pmap	*pmap,
 				vm_map_offset_t	addr);
@@ -557,13 +573,8 @@ extern void		pmap_map_block(
 extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys);
 extern void flush_dcache(vm_offset_t addr, unsigned count, int phys);
 extern ppnum_t          pmap_find_phys(pmap_t map, addr64_t va);
-extern void pmap_sync_page_data_phys(ppnum_t pa);
-extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 
-extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
+extern void pmap_cpu_init(void);
 extern void pmap_disable_NX(pmap_t pmap);
 extern void pmap_set_4GB_pagezero(pmap_t pmap);
 extern void pmap_clear_4GB_pagezero(pmap_t pmap);
@@ -618,9 +629,10 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
         set_dirbase(tpmap, my_cpu);					\
 }
 
-#define PMAP_DEACTIVATE_MAP(map, my_cpu)                                \
-        if (current_cpu_datap()->cpu_task_map == TASK_MAP_64BIT_SHARED) \
-            pmap_load_kernel_cr3();
+#define PMAP_DEACTIVATE_MAP(map, my_cpu)				\
+	if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED)	\
+		pmap_load_kernel_cr3();
+	
 
 #define PMAP_ACTIVATE_USER(th, my_cpu)  {				\
         spl_t		spl;						\
@@ -651,7 +663,7 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
                 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) {	\
 	                updp = pmap_pde(new_th->map->pmap,		\
                               new_th->machine.copy_window[i].user_base);\
-	                pmap_store_pte(kpdp,  updp ? *updp : 0);        \
+                        pmap_store_pte(kpdp, updp ? *updp : 0);		\
                 }							\
                 kpdp++;							\
         }								\
@@ -661,8 +673,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
         else								\
                 new_th->machine.copyio_state = WINDOWS_DIRTY;		\
         if (new_th->machine.physwindow_pte) {				\
-                pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),	\
-				 new_th->machine.physwindow_pte);	\
+	  pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep),	\
+			       new_th->machine.physwindow_pte);	        \
                 if (need_flush == 0)					\
                         invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\
         }								\
@@ -697,8 +709,8 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
 #define CPU_CR3_IS_ACTIVE(cpu)						\
 	((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0)
 
-#define CPU_GET_ACTIVE_CR3(cpu)                                         \
-        (cpu_datap(cpu)->cpu_active_cr3 & ~1)
+#define CPU_GET_ACTIVE_CR3(cpu)						\
+	(cpu_datap(cpu)->cpu_active_cr3 & ~1)
 
 #define MARK_CPU_IDLE(my_cpu)	{					\
 	/*								\
@@ -747,12 +759,18 @@ extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_s
 
 
 #define pmap_resident_count(pmap)	((pmap)->stats.resident_count)
+#define pmap_resident_max(pmap)		((pmap)->stats.resident_max)
 #define	pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr)
 #define	pmap_attribute(pmap,addr,size,attr,value) \
 					(KERN_INVALID_ADDRESS)
 #define	pmap_attribute_cache_sync(addr,size,attr,value) \
 					(KERN_INVALID_ADDRESS)
 
+#define MACHINE_PMAP_IS_EMPTY 1
+extern boolean_t pmap_is_empty(pmap_t		pmap,
+			       vm_map_offset_t	start,
+			       vm_map_offset_t	end);
+
 #endif	/* ASSEMBLER */
 
 
diff --git a/osfmk/i386/postcode.h b/osfmk/i386/postcode.h
index ed74d9dc2..42a4627b0 100644
--- a/osfmk/i386/postcode.h
+++ b/osfmk/i386/postcode.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_I386_POSTCODE_H_
@@ -28,7 +34,7 @@
 #endif
 
 /* Define this to delay about 1 sec after posting each code */
-/* #define POSTCODE_DELAY 1 */
+//#define POSTCODE_DELAY 1
 
 /* The POSTCODE is port 0x80 */
 #define	POSTPORT 0x80
@@ -41,7 +47,6 @@
  * Macro to output byte value to postcode, destoying register al.
  * Additionally, if POSTCODE_DELAY, spin for about a second.
  */
-
 #if POSTCODE_DELAY
 #define POSTCODE_AL			\
         outb    %al,$(POSTPORT);	\
@@ -167,7 +172,7 @@ postcode(uint8_t	xx)
 #endif
 }
 #else
-#define postcode(xx)
+#define postcode(xx) do {} while(0)
 #endif
 #endif
 
diff --git a/osfmk/i386/proc_reg.h b/osfmk/i386/proc_reg.h
index 27ca1e7ed..ee5fe6de1 100644
--- a/osfmk/i386/proc_reg.h
+++ b/osfmk/i386/proc_reg.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -139,6 +145,7 @@
 /*
  * CR4
  */
+#define CR4_VMXE 0x00002000	/* Enable VMX operation */
 #define CR4_FXS 0x00000200    	/* SSE/SSE2 OS supports FXSave */
 #define CR4_XMM 0x00000400    	/* SSE/SSE2 instructions supported in OS */
 #define CR4_PGE 0x00000080    	/* p6:   Page Global Enable */
@@ -232,6 +239,8 @@ static inline void lldt(unsigned int seg)
 
 #ifdef MACH_KERNEL_PRIVATE
 extern void flush_tlb64(void);
+extern uint64_t get64_cr3(void);
+extern void set64_cr3(uint64_t);
 static inline void flush_tlb(void)
 {
 	unsigned long	cr3_temp;
@@ -313,6 +322,12 @@ __END_DECLS
 #define MSR_IA32_APIC_BASE_ENABLE	(1<<11)
 #define MSR_IA32_APIC_BASE_BASE		(0xfffff<<12)
 
+#define MSR_IA32_FEATURE_CONTROL	0x3a
+#define MSR_IA32_FEATCTL_LOCK		(1<<0)
+#define MSR_IA32_FEATCTL_VMXON_SMX	(1<<1)
+#define MSR_IA32_FEATCTL_VMXON		(1<<2)
+#define MSR_IA32_FEATCTL_CSTATE_SMI	(1<<16)
+
 #define MSR_IA32_UCODE_WRITE		0x79
 #define MSR_IA32_UCODE_REV		0x8b
 
@@ -368,12 +383,23 @@ __END_DECLS
 #define MSR_IA32_MTRR_FIX4K_F0000	0x26e
 #define MSR_IA32_MTRR_FIX4K_F8000	0x26f
 
+#define MSR_IA32_VMX_BASE		0x480
+#define MSR_IA32_VMX_BASIC		MSR_IA32_VMX_BASE
+#define MSR_IA32_VMXPINBASED_CTLS		MSR_IA32_VMX_BASE+1
+#define MSR_IA32_PROCBASED_CTLS		MSR_IA32_VMX_BASE+2
+#define MSR_IA32_VMX_EXIT_CTLS		MSR_IA32_VMX_BASE+3
+#define MSR_IA32_VMX_ENTRY_CTLS		MSR_IA32_VMX_BASE+4
+#define MSR_IA32_VMX_MISC		MSR_IA32_VMX_BASE+5
+#define MSR_IA32_VMX_CR0_FIXED0		MSR_IA32_VMX_BASE+6
+#define MSR_IA32_VMX_CR0_FIXED1		MSR_IA32_VMX_BASE+7
+#define MSR_IA32_VMX_CR4_FIXED0		MSR_IA32_VMX_BASE+8
+#define MSR_IA32_VMX_CR4_FIXED1		MSR_IA32_VMX_BASE+9
 
 #define	MSR_IA32_EFER		0xC0000080
-#define	MSR_IA32_EFER_SCE		0x00000001
-#define	MSR_IA32_EFER_LME		0x00000100
-#define	MSR_IA32_EFER_LMA		0x00000400
-#define	MSR_IA32_EFER_NXE		0x00000800
+#define	MSR_IA32_EFER_SCE	0x00000001
+#define	MSR_IA32_EFER_LME	0x00000100
+#define	MSR_IA32_EFER_LMA	0x00000400
+#define	MSR_IA32_EFER_NXE	0x00000800
 
 #define	MSR_IA32_STAR		0xC0000081
 #define	MSR_IA32_LSTAR		0xC0000082
diff --git a/osfmk/i386/read_fault.c b/osfmk/i386/read_fault.c
deleted file mode 100644
index 67dc98358..000000000
--- a/osfmk/i386/read_fault.c
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-/*
- */
-
-#include <kern/thread.h>
-#include <vm/vm_fault.h>
-#include <mach/kern_return.h>
-#include <mach/vm_behavior.h>
-#include <vm/vm_map.h>
-#include <vm/vm_object.h>
-#include <vm/vm_page.h>
-#include <vm/pmap.h>
-
-#include <i386/intel_read_fault.h>
-
-#include <kern/macro_help.h>
-
-/*
- *	Expansion of vm_fault for read fault in kernel mode.
- *	Must enter the mapping as writable, since the i386
- *	(and i860 in i386 compatability mode) ignores write
- *	protection in kernel mode.
- *
- *	Note that this routine can be called for pmap's other
- *	than the kernel_pmap, in which case it just enters
- *	a read-only mapping.  (See e.g. kernel_trap().)
- */
-kern_return_t
-intel_read_fault(
-	vm_map_t	map,
-	vm_offset_t	vaddr)
-{
-	vm_map_version_t	version;	/* Map version for
-						   verification */
-	vm_object_t		object;		/* Top-level object */
-	vm_object_offset_t	offset;		/* Top-level offset */
-	vm_prot_t		prot;		/* Protection for mapping */
-	vm_behavior_t		behavior;	/* Expected paging behavior */
-	vm_map_offset_t		lo_offset, hi_offset;
-	vm_page_t		result_page;	/* Result of vm_fault_page */
-	vm_page_t		top_page;	/* Placeholder page */
-	boolean_t		wired;		/* Is map region wired? */
-	kern_return_t		result;
-	register vm_page_t	m;
-	vm_map_t		map_pmap;
-	vm_map_t                original_map = map;
-	thread_t                cur_thread;
-	boolean_t               funnel_set;
-	funnel_t                *curflock = NULL;
-
-	cur_thread = current_thread();
-	if ((cur_thread->funnel_state & TH_FN_OWNED) == TH_FN_OWNED) {
-		funnel_set = TRUE;
-		curflock = cur_thread->funnel_lock;
-		thread_funnel_set( curflock , FALSE);
-	} else {
-		funnel_set = FALSE;
-	}
-
-    RetryFault:
-
-	map = original_map;
-
-	/*
-	 *	Find the backing store object and offset into it
-	 *	to begin search.
-	 */
-	vm_map_lock_read(map);
-	result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
-				      &object, &offset, &prot, &wired,
-				      &behavior, &lo_offset, 
-				      &hi_offset, &map_pmap);
-	
-	vm_map_unlock_read(map);
-
-	if (result != KERN_SUCCESS) {
-		if (funnel_set)
-			thread_funnel_set( curflock, TRUE);
-		return (result);
-	}
-
-	if(map_pmap != map) {
-		vm_map_reference(map_pmap);
-		vm_map_unlock_read(map_pmap);
-	}
-
-	/*
-	 *	Make a reference to this object to prevent its
-	 *	disposal while we are playing with it.
-	 */
-	assert(object->ref_count > 0);
-	object->ref_count++;
-	vm_object_res_reference(object);
-	vm_object_paging_begin(object);
-
-	result = vm_fault_page(object, offset, VM_PROT_READ, FALSE,
-			       THREAD_ABORTSAFE,
-			       lo_offset, hi_offset, behavior,
-			       &prot, &result_page, &top_page, (int *)0,
-			       0, map->no_zero_fill, FALSE, map, vaddr);
-
-	if (result != VM_FAULT_SUCCESS) {
-	    vm_object_deallocate(object);
-	    if(map_pmap != map) {
-			vm_map_deallocate(map_pmap);
-	   }
-
-	    switch (result) {
-		case VM_FAULT_RETRY:
-		    goto RetryFault;
-		case VM_FAULT_INTERRUPTED:
-			if (funnel_set)
-				thread_funnel_set( curflock, TRUE);
-		    return (KERN_SUCCESS);
-		case VM_FAULT_MEMORY_SHORTAGE:
-		    VM_PAGE_WAIT();
-		    goto RetryFault;
-		case VM_FAULT_FICTITIOUS_SHORTAGE:
-		    vm_page_more_fictitious();
-		    goto RetryFault;
-		case VM_FAULT_MEMORY_ERROR:
-		    return (KERN_MEMORY_ERROR);
-	    }
-	}
-
-	m = result_page;
-
-	/*
-	 *	How to clean up the result of vm_fault_page.  This
-	 *	happens whether the mapping is entered or not.
-	 */
-
-#define UNLOCK_AND_DEALLOCATE				\
-	MACRO_BEGIN					\
-	vm_fault_cleanup(m->object, top_page);		\
-	vm_object_deallocate(object);			\
-	MACRO_END
-
-	/*
-	 *	What to do with the resulting page from vm_fault_page
-	 *	if it doesn't get entered into the physical map:
-	 */
-
-#define RELEASE_PAGE(m)					\
-	MACRO_BEGIN					\
-	PAGE_WAKEUP_DONE(m);				\
-	vm_page_lock_queues();				\
-	if (!m->active && !m->inactive)			\
-		vm_page_activate(m);			\
-	vm_page_unlock_queues();			\
-	MACRO_END
-
-	/*
-	 *	We must verify that the maps have not changed.
-	 */
-	vm_object_unlock(m->object);
-
-	if ((map != original_map) || !vm_map_verify(map, &version)) {
-	    vm_object_t		retry_object;
-	    vm_object_offset_t	retry_offset;
-	    vm_prot_t		retry_prot;
-
-		if (map != map_pmap) {
-			vm_map_deallocate(map_pmap);
-		}
-	    
-		map = original_map;
-		vm_map_lock_read(map);
-
-	    result = vm_map_lookup_locked(&map, vaddr, VM_PROT_READ, &version,
-				&retry_object, &retry_offset, &retry_prot,
-				&wired, &behavior, &lo_offset, 
-				&hi_offset, &map_pmap);
-
-	    if (result != KERN_SUCCESS) {
-	        vm_map_unlock_read(map);
-			vm_object_lock(m->object);
-			RELEASE_PAGE(m);
-			UNLOCK_AND_DEALLOCATE;
-			if (funnel_set)
-				thread_funnel_set( curflock, TRUE);
-			return (result);
-	    }
-
-		if (map != map_pmap) {
-			vm_map_reference(map_pmap);
-		}
-
-	    vm_object_unlock(retry_object);
-
-	    if (retry_object != object || retry_offset != offset) {
-			vm_object_lock(m->object);
-			RELEASE_PAGE(m);
-	        vm_map_unlock_read(map);
-	        if(map_pmap != map) {
-		   		vm_map_unlock_read(map_pmap);
-		   		vm_map_deallocate(map_pmap);
-			}
-			UNLOCK_AND_DEALLOCATE;
-			goto RetryFault;
-	    }
-	}
-
-	/*
-	 *	Put the page in the physical map.
-	 */
-
-	PMAP_ENTER(map_pmap->pmap, vaddr, m, VM_PROT_READ, PMAP_DEFAULT_CACHE, wired);
-
-	if(map_pmap != map) {
-		vm_map_unlock_read(map_pmap);
-		vm_map_deallocate(map_pmap);
-	}
-	
-	vm_object_lock(m->object);
-	vm_page_lock_queues();
-	if (!m->active && !m->inactive)
-		vm_page_activate(m);
-	m->reference = TRUE;
-	vm_page_unlock_queues();
-
-	vm_map_verify_done(map, &version);
-	PAGE_WAKEUP_DONE(m);
-
-	UNLOCK_AND_DEALLOCATE;
-
-#undef	UNLOCK_AND_DEALLOCATE
-#undef	RELEASE_PAGE
-	if (funnel_set)
-		thread_funnel_set( curflock, TRUE);
-	return (KERN_SUCCESS);
-}
-
diff --git a/osfmk/i386/rtclock.c b/osfmk/i386/rtclock.c
index ebcb3610c..bad2abbe7 100644
--- a/osfmk/i386/rtclock.c
+++ b/osfmk/i386/rtclock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -50,7 +56,6 @@
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>		/* for kernel_map */
 #include <i386/ipl.h>
-#include <i386/pit.h>
 #include <architecture/i386/pio.h>
 #include <i386/misc_protos.h>
 #include <i386/proc_reg.h>
@@ -69,9 +74,6 @@
 #include <i386/hpet.h>
 #include <i386/rtclock.h>
 
-#define MAX(a,b) (((a)>(b))?(a):(b))
-#define MIN(a,b) (((a)>(b))?(b):(a))
-
 #define NSEC_PER_HZ			(NSEC_PER_SEC / 100) /* nsec per tick */
 
 #define UI_CPUFREQ_ROUNDING_FACTOR	10000000
@@ -91,46 +93,19 @@ extern clock_timer_func_t	rtclock_timer_expire;
 static void	rtc_set_timescale(uint64_t cycles);
 static uint64_t	rtc_export_speed(uint64_t cycles);
 
-extern void		rtc_nanotime_store(
+extern void		_rtc_nanotime_store(
 					uint64_t		tsc,
 					uint64_t		nsec,
 					uint32_t		scale,
 					uint32_t		shift,
 					rtc_nanotime_t	*dst);
 
-extern void		rtc_nanotime_load(
-					rtc_nanotime_t	*src,
-					rtc_nanotime_t	*dst);
-
-rtc_nanotime_t	rtc_nanotime_info;
+extern uint64_t		_rtc_nanotime_read(
+					rtc_nanotime_t	*rntp,
+					int		slow );
 
-/*
- * tsc_to_nanoseconds:
- *
- * Basic routine to convert a raw 64 bit TSC value to a
- * 64 bit nanosecond value.  The conversion is implemented
- * based on the scale factor and an implicit 32 bit shift.
- */
-static inline uint64_t
-_tsc_to_nanoseconds(uint64_t value)
-{
-    asm volatile("movl	%%edx,%%esi	;"
-				 "mull	%%ecx		;"
-				 "movl	%%edx,%%edi	;"
-				 "movl	%%esi,%%eax	;"
-				 "mull	%%ecx		;"
-				 "addl	%%edi,%%eax	;"	
-				 "adcl	$0,%%edx	 "
-				 		: "+A" (value) : "c" (rtc_nanotime_info.scale) : "esi", "edi");
-
-    return (value);
-}
+rtc_nanotime_t	rtc_nanotime_info = {0,0,0,0,1,0};
 
-uint64_t
-tsc_to_nanoseconds(uint64_t value)
-{
-	return _tsc_to_nanoseconds(value);
-}
 
 static uint32_t
 deadline_to_decrementer(
@@ -150,20 +125,12 @@ deadline_to_decrementer(
 void
 rtc_lapic_start_ticking(void)
 {
-	uint64_t	abstime;
-	uint64_t	first_tick;
-	cpu_data_t      *cdp = current_cpu_datap();
-
-	abstime = mach_absolute_time();
-	rtclock_tick_interval = NSEC_PER_HZ;
-
-	first_tick = abstime + rtclock_tick_interval;
-	cdp->rtclock_intr_deadline = first_tick;
+	x86_lcpu_t	*lcpu = x86_lcpu();
 
 	/*
 	 * Force a complete re-evaluation of timer deadlines.
 	 */
-	cdp->rtcPop = EndOfAllTime;
+	lcpu->rtcPop = EndOfAllTime;
 	etimer_resync_deadlines();
 }
 
@@ -217,7 +184,7 @@ _rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base)
 {
 	uint64_t	tsc = rdtsc64();
 
-	rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
+	_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
 }
 
 static void
@@ -246,57 +213,22 @@ rtc_nanotime_init_commpage(void)
 	splx(s);
 }
 
-/*
- * rtc_nanotime_update:
- *
- * Update the nanotime info from the base time.  Since
- * the base value might be from a lower resolution clock,
- * we compare it to the TSC derived value, and use the
- * greater of the two values.
- *
- * N.B. In comparison to the above init routine, this assumes
- * that the TSC has remained monotonic compared to the tsc_base
- * value, which is not the case after S3 sleep.
- */
-static inline void
-_rtc_nanotime_update(rtc_nanotime_t *rntp, uint64_t	base)
-{
-	uint64_t	nsecs, tsc = rdtsc64();
-
-	nsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base);
-	rtc_nanotime_store(tsc, MAX(nsecs, base), rntp->scale, rntp->shift, rntp);
-}
-
-static void
-rtc_nanotime_update(
-	uint64_t		base)
-{
-	rtc_nanotime_t	*rntp = &rtc_nanotime_info;
-
-	assert(!ml_get_interrupts_enabled());
-        
-	_rtc_nanotime_update(rntp, base);
-	rtc_nanotime_set_commpage(rntp);
-}
-
 /*
  * rtc_nanotime_read:
  *
  * Returns the current nanotime value, accessable from any
  * context.
  */
-static uint64_t
+static inline uint64_t
 rtc_nanotime_read(void)
 {
-	rtc_nanotime_t	rnt, *rntp = &rtc_nanotime_info;
-	uint64_t		result;
-
-	do {
-		rtc_nanotime_load(rntp, &rnt);
-		result = rnt.ns_base + _tsc_to_nanoseconds(rdtsc64() - rnt.tsc_base);
-	} while (rntp->tsc_base != rnt.tsc_base);
-
-	return (result);
+	
+#if CONFIG_EMBEDDED
+	if (gPEClockFrequencyInfo.timebase_frequency_hz > SLOW_TSC_THRESHOLD)
+		return	_rtc_nanotime_read( &rtc_nanotime_info, 1 );	/* slow processor */
+	else
+#endif
+	return	_rtc_nanotime_read( &rtc_nanotime_info, 0 );	/* assume fast processor */
 }
 
 /*
@@ -304,15 +236,24 @@ rtc_nanotime_read(void)
  *
  * Invoked from power manangement when we have awoken from a nap (C3/C4)
  * during which the TSC lost counts.  The nanotime data is updated according
- * to the provided nanosecond base value.
+ * to the provided value which indicates the number of nanoseconds that the
+ * TSC was not counting.
  *
  * The caller must guarantee non-reentrancy.
  */
 void
 rtc_clock_napped(
-	uint64_t		base)
+	uint64_t		delta)
 {
-	rtc_nanotime_update(base);
+	rtc_nanotime_t	*rntp = &rtc_nanotime_info;
+	uint32_t	generation;
+
+	assert(!ml_get_interrupts_enabled());
+	generation = rntp->generation;
+	rntp->generation = 0;
+	rntp->ns_base += delta;
+	rntp->generation = ((generation + 1) != 0) ? (generation + 1) : 1;
+	rtc_nanotime_set_commpage(rntp);
 }
 
 void
@@ -326,7 +267,7 @@ void
 rtc_clock_stepped(__unused uint32_t new_frequency,
 		  __unused uint32_t old_frequency)
 {
-	panic("rtc_clock_stepping unsupported");
+	panic("rtc_clock_stepped unsupported");
 }
 
 /*
@@ -405,7 +346,11 @@ static void
 rtc_set_timescale(uint64_t cycles)
 {
 	rtc_nanotime_info.scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles;
-	rtc_nanotime_info.shift = 32;
+
+	if (cycles <= SLOW_TSC_THRESHOLD)
+		rtc_nanotime_info.shift = cycles;
+	else
+		rtc_nanotime_info.shift = 32;
 
 	rtc_nanotime_init(0);
 }
@@ -489,7 +434,7 @@ clock_gettimeofday_set_commpage(
 
 	*secs += epoch;
 
-	commpage_set_timestamp(abstime - remain, *secs, NSEC_PER_SEC);
+	commpage_set_timestamp(abstime - remain, *secs);
 }
 
 void
@@ -518,13 +463,15 @@ rtclock_intr(
 	boolean_t	user_mode = FALSE;
 	uint64_t	abstime;
 	uint32_t	latency;
-	cpu_data_t	*pp = current_cpu_datap();
+	x86_lcpu_t	*lcpu = x86_lcpu();
 
 	assert(get_preemption_level() > 0);
 	assert(!ml_get_interrupts_enabled());
 
 	abstime = rtc_nanotime_read();
-	latency = (uint32_t) abstime - pp->rtcPop;
+	latency = (uint32_t)(abstime - lcpu->rtcDeadline);
+	if (abstime < lcpu->rtcDeadline)
+		latency = 1;
 
 	if (is_saved_state64(tregs) == TRUE) {
 	        x86_saved_state64_t	*regs;
@@ -574,23 +521,6 @@ setPop(
 }
 
 
-void
-resetPop(void)
-{
-	uint64_t	now;
-	uint32_t	decr;
-	uint64_t	count;
-	cpu_data_t	*cdp = current_cpu_datap();
-
-	now = rtc_nanotime_read();
-
-	decr = deadline_to_decrementer(cdp->rtcPop, now);
-
-	count = tmrCvt(decr, busFCvtn2t);
-	lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t)count);
-}
-
-
 uint64_t
 mach_absolute_time(void)
 {
diff --git a/osfmk/i386/rtclock.h b/osfmk/i386/rtclock.h
index cc5d76402..904b38786 100644
--- a/osfmk/i386/rtclock.h
+++ b/osfmk/i386/rtclock.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -36,8 +42,6 @@
 
 #include <kern/etimer.h>
 
-void resetPop(void);
-
 struct cpu_data;
 
 #endif /* _I386_RTCLOCK_H_ */
diff --git a/osfmk/i386/sched_param.h b/osfmk/i386/sched_param.h
index 7aecc1b1f..5a5677f53 100644
--- a/osfmk/i386/sched_param.h
+++ b/osfmk/i386/sched_param.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/seg.h b/osfmk/i386/seg.h
index eca69d6f3..1c0ee14bb 100644
--- a/osfmk/i386/seg.h
+++ b/osfmk/i386/seg.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,13 +53,11 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
-/*
- */
-
 #ifndef	_I386_SEG_H_
 #define	_I386_SEG_H_
 
 #include <mach_kdb.h>
+#ifndef ASSEMBLER
 #include <stdint.h>
 #include <mach/vm_types.h>
 #include <architecture/i386/sel.h>
@@ -89,7 +93,8 @@ selector_to_sel(uint16_t selector)
 }
 
 #define LDTSZ		8192		/* size of the kernel ldt in entries */
-#define	LDTSZ_MIN	17		/* kernel ldt entries used by the system */
+#define	LDTSZ_MIN	SEL_TO_INDEX(USER_SETTABLE)
+					/* kernel ldt entries */
 
 #if	MACH_KDB
 #define	GDTSZ		19
@@ -102,8 +107,6 @@ selector_to_sel(uint16_t selector)
  */
 #define	IDTSZ		256
 
-#ifndef	__ASSEMBLER__
-
 #include <sys/cdefs.h>
 
 /*
@@ -311,6 +314,7 @@ __END_DECLS
 #define	DEBUG_TSS	0x90		/* debug TSS (uniprocessor) */
 #endif
 
+#ifndef __ASSEMBLER__
 struct __gdt_desc_struct {
   unsigned short size;
   unsigned long address __attribute__((packed));
@@ -322,6 +326,6 @@ struct __idt_desc_struct {
   unsigned long address __attribute__((packed));
   unsigned short pad;
 } __attribute__ ((packed));
-
+#endif /* __ASSEMBLER__ */
 
 #endif	/* _I386_SEG_H_ */
diff --git a/osfmk/i386/serial_io.h b/osfmk/i386/serial_io.h
new file mode 100644
index 000000000..58e75e4a3
--- /dev/null
+++ b/osfmk/i386/serial_io.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/*
+ * @APPLE_FREE_COPYRIGHT@
+ */
+
+#ifndef _I386_SERIAL_IO_H_
+#define _I386_SERIAL_IO_H_
+
+#include <console/serial_protos.h>
+
+int serial_init(void);
+
+#endif /* _I386_SERIAL_IO_H_ */
diff --git a/osfmk/i386/setjmp.h b/osfmk/i386/setjmp.h
index 0f98cd323..57787d549 100644
--- a/osfmk/i386/setjmp.h
+++ b/osfmk/i386/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/setjmp.s b/osfmk/i386/setjmp.s
index 5c8af20b3..f6581f3ac 100644
--- a/osfmk/i386/setjmp.s
+++ b/osfmk/i386/setjmp.s
@@ -1,21 +1,27 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
diff --git a/osfmk/i386/simple_lock.h b/osfmk/i386/simple_lock.h
index e823308c9..ad2c33893 100644
--- a/osfmk/i386/simple_lock.h
+++ b/osfmk/i386/simple_lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/stab.h b/osfmk/i386/stab.h
index 897f79e2c..c0f3f2013 100644
--- a/osfmk/i386/stab.h
+++ b/osfmk/i386/stab.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/start.s b/osfmk/i386/start.s
index eafe900f6..3218d3d15 100644
--- a/osfmk/i386/start.s
+++ b/osfmk/i386/start.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -116,7 +122,7 @@ LEXT(idtptr)
 	.word	Times(8,IDTSZ)-1
 	.long	EXT(master_idt)
 
-        /* back to the regular __DATA section. */
+          /* back to the regular __DATA section. */
 
           .section __DATA, __data
 
@@ -286,7 +292,7 @@ Ls1:	popl	%esi			// Get return address
 	movl	KSIZE(%ebp), %ecx	// Set ecx to kernbootstruct ksize
 	addl	$(NBPG-1), %ecx		// Add NBPG-1 to ecx
 	andl	$(-NBPG), %ecx		// Truncate  ecx to a page aligned addr
-	sarl	$2, %ecx		// Divide ecx by 4
+	shrl	$2, %ecx		// Divide ecx by 4
 	movl	%esp, (PA(EXT(KernelRelocOffset)))(%esp)
 					// Store relocation offset
 	movl	%edi, KADDR(%ebp)	// Relocate kaddr in kernbootstruct
@@ -501,7 +507,7 @@ fix_ldt_ret:
 	movl	%eax, %cr3
 
 	movl	%cr4, %eax
-	orl	$(CR4_PAE|CR4_PGE), %eax
+	orl	$(CR4_PAE), %eax
 	movl	%eax, %cr4
 
 	movl	$0x80000001, %eax
@@ -594,7 +600,7 @@ LEXT(slave_pstart)
  */
 #ifdef PAE
 	movl	%cr4, %eax
-	orl	$(CR4_PAE|CR4_PGE), %eax
+	orl	$(CR4_PAE), %eax
 	movl	%eax, %cr4
 
 	movl	$(MSR_IA32_EFER), %ecx			/* MSR number in ecx */
diff --git a/osfmk/i386/start64.s b/osfmk/i386/start64.s
index 25b70d602..e822e6c12 100644
--- a/osfmk/i386/start64.s
+++ b/osfmk/i386/start64.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <platforms.h>
@@ -151,13 +157,48 @@ Entry(flush_tlb64)
 	EMARF
 	ret
 
+Entry(set64_cr3)
+
+	FRAME
+
+	movl	B_ARG0, %eax
+	movl	B_ARG1, %edx
+
+	ENTER_64BIT_MODE()
+
+	/* %rax = %edx:%eax */
+	shl	$32, %rax
+	shrd	$32, %rdx, %rax
+
+	mov	%rax, %cr3
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
+Entry(get64_cr3)
+
+	FRAME
+
+	ENTER_64BIT_MODE()
+
+	mov	%cr3, %rax
+	mov	%rax, %rdx
+	shr	$32, %rdx		// %edx:%eax = %cr3
+
+	ENTER_COMPAT_MODE()
+
+	EMARF
+	ret
+
 /* FXSAVE and FXRSTOR operate in a mode dependent fashion, hence these variants.
-* Must be called with interrupts disabled.
-* We clear pending x87 exceptions here; this is technically incorrect, since we should
-* propagate those to the user, but the compatibility mode kernel is currently not
-* prepared to handle exceptions originating in 64-bit kernel mode. However, it may be possible
-* to work around this should it prove necessary.
-*/
+ * Must be called with interrupts disabled.
+ * We clear pending x87 exceptions here; this is technically incorrect, since we
+ * should propagate those to the user, but the compatibility mode kernel is
+ * currently not prepared to handle exceptions originating in 64-bit kernel mode.
+ * However, it may be possible to work around this should it prove necessary.
+ */
 
 Entry(fxsave64)
 	movl		S_ARG0,%eax
diff --git a/osfmk/i386/startup64.c b/osfmk/i386/startup64.c
index dd8250f68..0998530b1 100644
--- a/osfmk/i386/startup64.c
+++ b/osfmk/i386/startup64.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <string.h>
@@ -63,25 +69,52 @@
 void
 cpu_IA32e_enable(cpu_data_t *cdp)
 {
-	uint32_t	cr0 = get_cr0();
-        uint64_t	efer = rdmsr64(MSR_IA32_EFER);
-
 	assert(!ml_get_interrupts_enabled());
 
-	postcode(CPU_IA32_ENABLE_ENTRY);
-
-	/* Turn paging off - works because we're identity mapped */
-	set_cr0(cr0 & ~CR0_PG);
+	if (!cdp->cpu_is64bit ||
+	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
+		return;
 
-	/* pop in new top level phys pg addr */
-	set_cr3((vm_offset_t) kernel64_cr3);
-
-	wrmsr64(MSR_IA32_EFER, efer | MSR_IA32_EFER_LME); /* set  mode */
-
-	/* Turn paging on */
-        set_cr0(cr0 | CR0_PG);
+	postcode(CPU_IA32_ENABLE_ENTRY);
 
-	/* this call is required to re-activate paging */
+	/* 
+	 * The following steps are performed by inlines so that
+	 * we can be assured we don't use the stack or any other
+	 * non-identity mapped data while paging is turned off...
+	 */
+	/* Turn paging off */
+	asm volatile(
+		"mov	%%cr0, %%eax	\n\t"
+		"andl	%0, %%eax	\n\t"
+		"mov	%%eax, %%cr0	\n\t"
+		:
+		: "i" (~CR0_PG)
+		: "eax" );
+
+	/* Pop new top level phys pg addr into CR3 */
+	asm volatile(
+		"mov	%%eax, %%cr3	\n\t"
+		:
+		: "a" ((uint32_t) kernel64_cr3));
+
+	/* Turn on the 64-bit mode bit */
+	asm volatile(
+		"rdmsr			\n\t"
+		"orl	%1, %%eax	\n\t"
+		"wrmsr			\n\t"
+		:
+		: "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME)
+		: "eax", "edx");
+
+	/* Turn paging on again */
+	asm volatile(
+		"mov	%%cr0, %%eax	\n\t"
+		"orl	%0, %%eax	\n\t"
+		"mov	%%eax, %%cr0	\n\t"
+		:
+		: "i" (CR0_PG)
+		: "eax" );
+	
 	kprintf("cpu_IA32e_enable(%p)\n", cdp);
 
 	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
@@ -95,28 +128,52 @@ cpu_IA32e_enable(cpu_data_t *cdp)
 void
 cpu_IA32e_disable(cpu_data_t *cdp)
 {
-	uint32_t	cr0 = get_cr0();
-        uint64_t	efer = rdmsr64(MSR_IA32_EFER);
-
 	assert(!ml_get_interrupts_enabled());
 
 	postcode(CPU_IA32_DISABLE_ENTRY);
 
-	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
-		panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA clear on entry");
-
-	/* Turn paging off - works because we're identity mapped */
-	set_cr0(cr0 & ~CR0_PG);
-
-	/* pop in legacy top level phys pg addr */
-	set_cr3((vm_offset_t) lo_kernel_cr3);
-
-	wrmsr64(MSR_IA32_EFER, efer & ~MSR_IA32_EFER_LME); /* reset mode */
-
-	/* Turn paging on */
-        set_cr0(cr0 | CR0_PG);
-
-	/* this call is required to re-activate paging */
+	if (!cdp->cpu_is64bit ||
+	    (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
+		return;
+
+	/* 
+	 * The following steps are performed by inlines so that
+	 * we can be assured we don't use the stack or any other
+	 * non-identity mapped data while paging is turned off...
+	 */
+	/* Turn paging off */
+	asm volatile(
+		"mov	%%cr0, %%eax	\n\t"
+		"andl	%0, %%eax	\n\t"
+		"mov	%%eax, %%cr0	\n\t"
+		:
+		: "i" (~CR0_PG)
+		: "eax" );
+
+	/* Pop legacy top level phys pg addr into CR3 */
+	asm volatile(
+		"mov	%%eax, %%cr3	\n\t"
+		:
+		: "a" ((uint32_t) lo_kernel_cr3));
+
+	/* Turn off the 64-bit mode bit */
+	asm volatile(
+		"rdmsr			\n\t"
+		"andl	%1, %%eax	\n\t"
+		"wrmsr			\n\t"
+		:
+		: "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME)
+		: "eax", "edx");
+
+	/* Turn paging on again */
+	asm volatile(
+		"mov	%%cr0, %%eax	\n\t"
+		"orl	%0, %%eax	\n\t"
+		"mov	%%eax, %%cr0	\n\t"
+		:
+		: "i" (CR0_PG)
+		: "eax" );
+	
 	kprintf("cpu_IA32e_disable(%p)\n", cdp);
 
 	if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
@@ -203,12 +260,12 @@ dump_frame32(x86_saved_state_compat32_t *scp)
 	unsigned int	i;
 	uint32_t	*ip = (uint32_t *) scp;
 
-	kprintf("dump_frame32(0x%08x):\n", scp);
+	kprintf("dump_frame32(%p):\n", scp);
 	
 	for (i = 0;
 	     i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t);
 	     i++, ip++)
-		kprintf("0x%08x: 0x%08x\n", ip, *ip);
+		kprintf("%p: 0x%08x\n", ip, *ip);
 
 	kprintf("scp->isf64.err:    0x%016llx\n", scp->isf64.err);
 	kprintf("scp->isf64.rip:    0x%016llx\n", scp->isf64.rip);
@@ -251,7 +308,7 @@ dump_frame64(x86_saved_state64_t *sp)
 	for (i = 0;
 	     i < sizeof(x86_saved_state64_t)/sizeof(uint64_t);
 	     i++, ip++)
-		kprintf("0x%08x: 0x%016x\n", ip, *ip);
+		kprintf("%p: 0x%016llx\n", ip, *ip);
 
 	kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno);
 	kprintf("sp->isf.trapfn: 0x%08x\n", sp->isf.trapfn);
@@ -293,7 +350,7 @@ dump_gdt(void *gdtp)
 	unsigned int	i;
 	uint32_t	*ip = (uint32_t *) gdtp;
 
-	kprintf("GDT:\n", ip);
+	kprintf("GDT:\n");
 	for (i = 0; i < GDTSZ; i++, ip += 2) {
 		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
@@ -306,7 +363,7 @@ dump_ldt(void *ldtp)
 	unsigned int	i;
 	uint32_t	*ip = (uint32_t *) ldtp;
 
-	kprintf("LDT:\n", ip);
+	kprintf("LDT:\n");
 	for (i = 0; i < LDTSZ_MIN; i++, ip += 2) {
 		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
@@ -319,7 +376,7 @@ dump_idt(void *idtp)
 	unsigned int	i;
 	uint32_t	*ip = (uint32_t *) idtp;
 
-	kprintf("IDT64:\n", ip);
+	kprintf("IDT64:\n");
 	for (i = 0; i < 16; i++, ip += 4) {
 		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 		kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
@@ -334,7 +391,7 @@ dump_tss(void *tssp)
 	unsigned int	i;
 	uint32_t	*ip = (uint32_t *) tssp;
 
-	kprintf("TSS64:\n", ip);
+	kprintf("TSS64:\n");
 	for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) {
 		kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 	}
diff --git a/osfmk/i386/task.h b/osfmk/i386/task.h
index c15ac0963..b4218d665 100644
--- a/osfmk/i386/task.h
+++ b/osfmk/i386/task.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -52,4 +58,6 @@
  * Machine dependant task fields
  */
 
+#include <i386/user_ldt.h>
+
 #define MACHINE_TASK struct user_ldt *       i386_ldt;
diff --git a/osfmk/i386/thread.h b/osfmk/i386/thread.h
index d23a9d087..85a82e880 100644
--- a/osfmk/i386/thread.h
+++ b/osfmk/i386/thread.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -131,13 +137,14 @@ typedef struct pcb {
 #ifdef	MACH_BSD
 	uint64_t	cthread_self;		/* for use of cthread package */
         struct real_descriptor cthread_desc;
-	unsigned long	uldt_selector;		/* user ldt selector to set */
-	struct real_descriptor uldt_desc;	/* the actual user setable ldt data */
+	unsigned long  uldt_selector;          /* user ldt selector to set */
+	struct real_descriptor uldt_desc;      /* the actual user setable ldt data */
 #endif
 	decl_simple_lock_data(,lock);
 	uint64_t	iss_pte0;
 	uint64_t	iss_pte1;
 	void		*ids;
+	uint32_t	arg_store_valid;
 } *pcb_t;
 
 
@@ -165,6 +172,9 @@ struct machine_thread {
 
 	uint32_t	specFlags;
 #define		OnProc	0x1
+#if CONFIG_DTRACE
+#define		CopyIOActive 0x2 /* Checked to ensure DTrace actions do not re-enter copyio(). */
+#endif /* CONFIG_DTRACE */
   
         struct {
 	        user_addr_t	user_base;
diff --git a/osfmk/i386/timer.h b/osfmk/i386/timer.h
index 713bb946a..8f7bbdc60 100644
--- a/osfmk/i386/timer.h
+++ b/osfmk/i386/timer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/trap.c b/osfmk/i386/trap.c
index 7c9aa04f1..bec974a5d 100644
--- a/osfmk/i386/trap.c
+++ b/osfmk/i386/trap.c
@@ -1,56 +1,63 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
- * @OSF_COPYRIGHT@
- */
+* @OSF_COPYRIGHT@
+*/
 /* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
+* Mach Operating System
+* Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
+* All Rights Reserved.
+* 
+* Permission to use, copy, modify and distribute this software and its
+* documentation is hereby granted, provided that both the copyright
+* notice and this permission notice appear in all copies of the
+* software, derivative works or modified versions, and any portions
+* thereof, and that both notices appear in supporting documentation.
+* 
+* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+* CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+* 
+* Carnegie Mellon requests users of this software to return to
+* 
+*  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+*  School of Computer Science
+*  Carnegie Mellon University
+*  Pittsburgh PA 15213-3890
+* 
+* any improvements or extensions that they make and grant Carnegie Mellon
+* the rights to redistribute these changes.
+*/
 /*
- */
+*/
+
 /*
- * Hardware trap/fault handler.
+* Hardware trap/fault handler.
  */
 
 #include <mach_kdb.h>
@@ -99,7 +106,6 @@
 
 #include <string.h>
 
-#include <i386/io_emulate.h>
 #include <i386/postcode.h>
 #include <i386/mp_desc.h>
 #include <i386/proc_reg.h>
@@ -116,6 +122,13 @@ static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip);
 perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */
 perfCallback perfASTHook  = NULL; /* Pointer to CHUD AST hook routine */
 
+#if CONFIG_DTRACE
+/* See <rdar://problem/4613924> */
+perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
+
+extern boolean_t dtrace_tally_fault(user_addr_t);
+#endif
+
 void
 thread_syscall_return(
         kern_return_t ret)
@@ -196,17 +209,17 @@ user_page_fault_continue(
 			 kern_return_t	kr)
 {
 	thread_t	thread = current_thread();
-	x86_saved_state_t *regs = USER_STATE(thread);
 	ast_t		*myast;
 	boolean_t	intr;
 	user_addr_t	vaddr;
 #if	MACH_KDB
+	x86_saved_state_t *regs = USER_STATE(thread);
 	int		err;
 	int		trapno;
-#endif
 
 	assert((is_saved_state32(regs) && !thread_is_64bit(thread)) ||
 	       (is_saved_state64(regs) &&  thread_is_64bit(thread)));
+#endif
 
         if (thread_is_64bit(thread)) {
 	        x86_saved_state64_t	*uregs;
@@ -240,8 +253,8 @@ user_page_fault_continue(
 		    (err & T_PF_WRITE) &&
 		    db_find_watchpoint(thread->map,
 				       (vm_offset_t)vaddr,
-				       regs))
-			kdb_trap(T_WATCHPOINT, 0, regs);
+				       saved_state32(regs)))
+			kdb_trap(T_WATCHPOINT, 0, saved_state32(regs));
 #endif	/* MACH_KDB */
 		intr = ml_set_interrupts_enabled(FALSE);
 		myast = ast_pending();
@@ -258,7 +271,7 @@ user_page_fault_continue(
 
 #if	MACH_KDB
 	if (debug_all_traps_with_kdb &&
-	    kdb_trap(trapno, err, regs)) {
+	    kdb_trap(trapno, err, saved_state32(regs))) {
 		thread_exception_return();
 		/*NOTREACHED*/
 	}
@@ -282,6 +295,7 @@ extern struct recovery	recover_table_end[];
 const char *	trap_type[] = {TRAP_NAMES};
 unsigned 	TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]);
 
+
 static inline void
 reset_dr7(void)
 {
@@ -304,7 +318,7 @@ kernel_trap(
 	int			code;
 	user_addr_t		vaddr;
 	int			type;
-	vm_map_t		map;
+	vm_map_t		map = 0;	/* protected by T_PAGE_FAULT */
 	kern_return_t		result = KERN_FAILURE;
 	thread_t		thread;
 	ast_t			*myast;
@@ -350,17 +364,28 @@ kernel_trap(
 			return;
 		}
 	}
+
+#if CONFIG_DTRACE
+	if (tempDTraceTrapHook) {
+		if (tempDTraceTrapHook(type, state, 0, 0) == KERN_SUCCESS) {
+			/*
+			 * If it succeeds, we are done...
+			 */
+			return;
+		}
+	}
+#endif /* CONFIG_DTRACE */
+
 	/*
 	 * we come here with interrupts off as we don't want to recurse
 	 * on preemption below.  but we do want to re-enable interrupts
 	 * as soon we possibly can to hold latency down
 	 */
 	if (T_PREEMPT == type) {
+	        ast_taken(AST_PREEMPTION, FALSE);
 
-	        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
+		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE,
 				      0, 0, 0, kern_ip, 0);
-
-	        ast_taken(AST_PREEMPTION, FALSE);
 		return;
 	}
 	
@@ -425,8 +450,13 @@ kernel_trap(
 	        fpSSEexterrflt();
 		return;
  	    case T_DEBUG:
+#if MACH_KDP
 		    if ((saved_state->efl & EFL_TF) == 0
-		    && !kdp_has_active_watchpoints) {
+		    && !kdp_has_active_watchpoints)
+#else
+		    if ((saved_state->efl & EFL_TF) == 0)
+#endif
+		    {
 			    /* We've somehow encountered a debug
 			     * register match that does not belong
 			     * to the kernel debugger.
@@ -434,7 +464,7 @@ kernel_trap(
 			     */
 			    reset_dr7();
 			    return;
-			    }
+		    }
 		    goto debugger_entry;
 	    case T_PAGE_FAULT:
 		/*
@@ -472,6 +502,19 @@ kernel_trap(
 		}
 #endif	/* MACH_KDB */
 
+#if CONFIG_DTRACE
+		if (thread->options & TH_OPT_DTRACE) {	/* Executing under dtrace_probe? */
+			if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */
+				/*
+				 * DTrace has "anticipated" the possibility of this fault, and has
+				 * established the suitable recovery state. Drop down now into the
+				 * recovery handling code in "case T_GENERAL_PROTECTION:". 
+				 */
+				goto FALL_THROUGH;
+			}
+		}
+#endif /* CONFIG_DTRACE */
+
 		result = vm_fault(map,
 				  vm_map_trunc_page(vaddr),
 				  prot,
@@ -527,6 +570,9 @@ look_for_watchpoints:
 		/*
 		 * fall through
 		 */
+#if CONFIG_DTRACE
+FALL_THROUGH:
+#endif /* CONFIG_DTRACE */
 
 	    case T_GENERAL_PROTECTION:
 		/*
@@ -578,10 +624,9 @@ restart_debugger:
                 if (current_debugger != KDB_CUR_DB) {
 			if (kdp_i386_trap(type, saved_state, result, vaddr))
 				return;
-		}
+		} else {
 #endif /* MACH_KDP */
 #if MACH_KDB
-		else
 			if (kdb_trap(type, code, saved_state)) {
 				if (switch_debugger) {
 					current_debugger = KDP_CUR_DB;
@@ -591,6 +636,9 @@ restart_debugger:
 				return;
 			}
 #endif /* MACH_KDB */
+#if MACH_KDP
+		}
+#endif
 	}
 
 	panic_trap(saved_state);
@@ -616,6 +664,10 @@ panic_trap(x86_saved_state32_t *regs)
 	uint32_t	cr3 = get_cr3();
 	uint32_t	cr4 = get_cr4();
 
+	/*
+	 * Issue an I/O port read if one has been requested - this is an
+	 * event logic analyzers can use as a trigger point.
+	 */
 	panic_io_port_read();
 
 	kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip);
@@ -624,16 +676,17 @@ panic_trap(x86_saved_state32_t *regs)
 
 	if (regs->trapno < TRAP_TYPES)
 	        trapname = trap_type[regs->trapno];
-
-	panic("Unresolved kernel trap (CPU %d, Type %d=%s), registers:\n"
+#undef panic
+	panic("Kernel trap at 0x%08x, type %d=%s, registers:\n"
 	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
 	      "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
-	      "EFL: 0x%08x, EIP: 0x%08x, CS:  0x%08x, DS:  0x%08x\n",
-	      cpu_number(), regs->trapno, trapname, cr0, cr2, cr3, cr4,
+	      "EFL: 0x%08x, EIP: 0x%08x, CS:  0x%08x, DS:  0x%08x\n"
+	      "Error code: 0x%08x\n",
+	      regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4,
 	      regs->eax,regs->ebx,regs->ecx,regs->edx,
 	      regs->cr2,regs->ebp,regs->esi,regs->edi,
-	      regs->efl,regs->eip,regs->cs, regs->ds);
+	      regs->efl,regs->eip,regs->cs, regs->ds, regs->err);
 	/*
 	 * This next statement is not executed,
 	 * but it's needed to stop the compiler using tail call optimization
@@ -651,16 +704,25 @@ extern void     kprintf_break_lock(void);
  * Kernel stack overflow is one route here.
  */
 void
-panic_double_fault(int code)
+panic_double_fault(
+#if CONFIG_NO_PANIC_STRINGS
+		__unused int code
+#else
+		int code
+#endif
+		)
 {
+#if MACH_KDP || !CONFIG_NO_PANIC_STRINGS
 	struct i386_tss *my_ktss = current_ktss();
+#endif
 
 	/* Set postcode (DEBUG only) */
 	postcode(PANIC_DOUBLE_FAULT);
 
-/* Issue an I/O port read if one has been requested - this is an event logic
- * analyzers can use as a trigger point.
- */
+	/*
+	 * Issue an I/O port read if one has been requested - this is an
+	 * event logic analyzers can use as a trigger point.
+	 */
 	panic_io_port_read();
 
 	/*
@@ -676,13 +738,13 @@ panic_double_fault(int code)
 	panic_i386_backtrace((void *) my_ktss->ebp, 10);
 #endif
 
-	panic("Double fault (CPU:%d, thread:%p, code:0x%x),"
+	panic("Double fault at 0x%08x, thread:%p, code:0x%x, "
 	      "registers:\n"
 	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
 	      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
 	      "EFL: 0x%08x, EIP: 0x%08x\n",
-	      cpu_number(), current_thread(), code,
+	      my_ktss->eip, current_thread(), code,
 	      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 	      my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
 	      my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
@@ -694,13 +756,27 @@ panic_double_fault(int code)
  * Called from locore on a special reserved stack after a machine-check
  */
 void
-panic_machine_check(int code)
+panic_machine_check(
+#if CONFIG_NO_PANIC_STRINGS
+		__unused int code
+#else
+		int code
+#endif
+		)
 {
+#if !CONFIG_NO_PANIC_STRINGS
 	struct i386_tss *my_ktss = current_ktss();
+#endif
 
 	/* Set postcode (DEBUG only) */
 	postcode(PANIC_MACHINE_CHECK);
 
+	/*
+	 * Issue an I/O port read if one has been requested - this is an
+	 * event logic analyzers can use as a trigger point.
+	 */
+	panic_io_port_read();
+
 	/*
 	 * Break kprintf lock in case of recursion,
 	 * and record originally faulted instruction address.
@@ -715,13 +791,13 @@ panic_machine_check(int code)
 	/*
 	 * And that's all folks, we don't attempt recovery...
 	 */
-	panic("Machine-check (CPU:%d, thread:%p, code:0x%x),"
+	panic("Machine-check at 0x%08x, thread:%p, code:0x%x, "
 	      "registers:\n"
 	      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 	      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
 	      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
 	      "EFL: 0x%08x, EIP: 0x%08x\n",
-	      cpu_number(), current_thread(), code,
+	      my_ktss->eip, current_thread(), code,
 	      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 	      my_ktss->eax, my_ktss->ebx, my_ktss->ecx, my_ktss->edx,
 	      my_ktss->esp, my_ktss->ebp, my_ktss->esi, my_ktss->edi,
@@ -734,6 +810,12 @@ panic_double_fault64(x86_saved_state_t *esp)
 	/* Set postcode (DEBUG only) */
 	postcode(PANIC_DOUBLE_FAULT);
 
+	/*
+	 * Issue an I/O port read if one has been requested - this is an
+	 * event logic analyzers can use as a trigger point.
+	 */
+	panic_io_port_read();
+
 	/*
 	 * Break kprintf lock in case of recursion,
 	 * and record originally faulted instruction address.
@@ -744,8 +826,10 @@ panic_double_fault64(x86_saved_state_t *esp)
 	 * Dump the interrupt stack frame at last kernel entry.
 	 */
 	if (is_saved_state64(esp)) {
+#if !CONFIG_NO_PANIC_STRINGS
 		x86_saved_state64_t	*ss64p = saved_state64(esp);
-		panic("Double fault (CPU:%d, thread:%p, trapno:0x%x, err:0x%qx),"
+#endif
+		panic("Double fault thread:%p, trapno:0x%x, err:0x%qx, "
 		      "registers:\n"
 		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 		      "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
@@ -753,7 +837,7 @@ panic_double_fault64(x86_saved_state_t *esp)
 		      "R8:  0x%016qx, R9:  0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
 		      "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
 		      "RFL: 0x%016qx, RIP: 0x%016qx, CR2: 0x%016qx\n",
-		      cpu_number(), current_thread(), ss64p->isf.trapno, ss64p->isf.err,
+		      current_thread(), ss64p->isf.trapno, ss64p->isf.err,
 		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 		      ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
 		      ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
@@ -761,14 +845,16 @@ panic_double_fault64(x86_saved_state_t *esp)
 		      ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
 		      ss64p->isf.rflags, ss64p->isf.rip, ss64p->cr2);
 	} else {
+#if !CONFIG_NO_PANIC_STRINGS
 		x86_saved_state32_t	*ss32p = saved_state32(esp);
-		panic("Double fault (CPU:%d, thread:%p, trapno:0x%x, err:0x%x),"
+#endif
+		panic("Double fault at 0x%08x, thread:%p, trapno:0x%x, err:0x%x),"
 		      "registers:\n"
 		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 		      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
 		      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
 		      "EFL: 0x%08x, EIP: 0x%08x\n",
-		      cpu_number(), current_thread(), ss32p->trapno, ss32p->err,
+		      ss32p->eip, current_thread(), ss32p->trapno, ss32p->err,
 		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 		      ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
 		      ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
@@ -785,6 +871,12 @@ panic_machine_check64(x86_saved_state_t *esp)
 	/* Set postcode (DEBUG only) */
 	postcode(PANIC_MACHINE_CHECK);
 
+	/*
+	 * Issue an I/O port read if one has been requested - this is an
+	 * event logic analyzers can use as a trigger point.
+	 */
+	panic_io_port_read();
+
 	/*
 	 * Break kprintf lock in case of recursion,
 	 * and record originally faulted instruction address.
@@ -800,8 +892,10 @@ panic_machine_check64(x86_saved_state_t *esp)
 	 * And that's all folks, we don't attempt recovery...
 	 */
 	if (is_saved_state64(esp)) {
+#if !CONFIG_NO_PANIC_STRINGS
 		x86_saved_state64_t	*ss64p = saved_state64(esp);
-		panic("Machine Check (CPU:%d, thread:%p, trapno:0x%x, err:0x%qx),"
+#endif
+		panic("Machine Check thread:%p, trapno:0x%x, err:0x%qx, "
 		      "registers:\n"
 		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 		      "RAX: 0x%016qx, RBX: 0x%016qx, RCX: 0x%016qx, RDX: 0x%016qx\n"
@@ -809,7 +903,7 @@ panic_machine_check64(x86_saved_state_t *esp)
 		      "R8:  0x%016qx, R9:  0x%016qx, R10: 0x%016qx, R11: 0x%016qx\n"
 		      "R12: 0x%016qx, R13: 0x%016qx, R14: 0x%016qx, R15: 0x%016qx\n"
 		      "RFL: 0x%016qx, RIP: 0x%016qx\n",
-		      cpu_number(), current_thread(), ss64p->isf.trapno, ss64p->isf.err,
+		      current_thread(), ss64p->isf.trapno, ss64p->isf.err,
 		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 		      ss64p->rax, ss64p->rbx, ss64p->rcx, ss64p->rdx,
 		      ss64p->isf.rsp, ss64p->rbp, ss64p->rsi, ss64p->rdi,
@@ -817,14 +911,16 @@ panic_machine_check64(x86_saved_state_t *esp)
 		      ss64p->r12, ss64p->r13, ss64p->r14, ss64p->r15,
 		      ss64p->isf.rflags, ss64p->isf.rip);
 	} else {
+#if !CONFIG_NO_PANIC_STRINGS
 		x86_saved_state32_t	*ss32p = saved_state32(esp);
-		panic("Machine Check (CPU:%d, thread:%p, trapno:0x%x, err:0x%x),"
+#endif
+		panic("Machine Check at 0x%08x, thread:%p, trapno:0x%x, err:0x%x, "
 		      "registers:\n"
 		      "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n"
 		      "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n"
 		      "ESP: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n"
 		      "EFL: 0x%08x, EIP: 0x%08x\n",
-		      cpu_number(), current_thread(), ss32p->trapno, ss32p->err,
+		      ss32p->eip, current_thread(), ss32p->trapno, ss32p->err,
 		      get_cr0(), get_cr2(), get_cr3(), get_cr4(),
 		      ss32p->eax, ss32p->ebx, ss32p->ecx, ss32p->edx,
 		      ss32p->uesp, ss32p->ebp, ss32p->esi, ss32p->edi,
@@ -832,6 +928,10 @@ panic_machine_check64(x86_saved_state_t *esp)
 	}
 }
 
+#if CONFIG_DTRACE
+extern kern_return_t dtrace_user_probe(x86_saved_state_t *);
+#endif
+
 /*
  *	Trap from user mode.
  */
@@ -839,18 +939,17 @@ void
 user_trap(
 	x86_saved_state_t *saved_state)
 {
-	int		exc;
-	int		code;
-	int		err;
-	unsigned int	subcode;
-	int		type;
-	user_addr_t	vaddr;
-	vm_prot_t	prot;
-	thread_t	thread = current_thread();
-	ast_t		*myast;
-	boolean_t       intr;
-	kern_return_t	kret;
-	user_addr_t	rip;
+	int			exc;
+	int			err;
+	mach_exception_code_t 	code;
+	mach_exception_subcode_t subcode;
+	int			type;
+	user_addr_t		vaddr;
+	vm_prot_t		prot;
+	thread_t		thread = current_thread();
+	ast_t			*myast;
+	kern_return_t		kret;
+	user_addr_t		rip;
 
 	assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) ||
 	       (is_saved_state64(saved_state) &&  thread_is_64bit(thread)));
@@ -865,7 +964,7 @@ user_trap(
 		vaddr = (user_addr_t)regs->cr2;
 		rip   = (user_addr_t)regs->isf.rip;
 	} else {
-	        x86_saved_state32_t	*regs;
+		x86_saved_state32_t	*regs;
 
 		regs = saved_state32(saved_state);
 
@@ -901,6 +1000,12 @@ user_trap(
 			return;	/* If it succeeds, we are done... */
 	}
 
+	/*
+	 * DTrace does not consume all user traps, only INT_3's for now.
+	 * Avoid needlessly calling tempDTraceTrapHook here, and let the
+	 * INT_3 case handle them.
+	 */
+
 	switch (type) {
 
 	    case T_DIVIDE_ERROR:
@@ -941,6 +1046,10 @@ user_trap(
 			break;
 		}
 	    case T_INT3:
+#if CONFIG_DTRACE
+		if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
+			return; /* If it succeeds, we are done... */
+#endif
 		exc = EXC_BREAKPOINT;
 		code = EXC_I386_BPT;
 		break;
@@ -961,15 +1070,14 @@ user_trap(
 		break;
 
 	    case T_NO_FPU:
-	    case 32:		/* XXX */
 		fpnoextflt();
 		return;
 
 	    case T_FPU_FAULT:
-		fpextovrflt();
+		fpextovrflt(); /* Propagates exception directly, doesn't return */
 		return;
 
-	    case 10:		/* invalid TSS == iret with NT flag set */
+	    case T_INVALID_TSS:	/* invalid TSS == iret with NT flag set */
 		exc = EXC_BAD_INSTRUCTION;
 		code = EXC_I386_INVTSSFLT;
 		subcode = err;
@@ -988,7 +1096,21 @@ user_trap(
 		break;
 
 	    case T_GENERAL_PROTECTION:
-		exc = EXC_BAD_INSTRUCTION;
+		/*
+		 * There's a wide range of circumstances which generate this
+		 * class of exception. From user-space, many involve bad
+		 * addresses (such as a non-canonical 64-bit address).
+		 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV).
+		 * The trouble is cr2 doesn't contain the faulting address;
+		 * we'd need to decode the faulting instruction to really
+		 * determine this. We'll leave that to debuggers.
+		 * However, attempted execution of privileged instructions
+		 * (e.g. cli) also generate GP faults and so we map these to
+		 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than
+		 * EXC_BAD_INSTRUCTION which is more accurate. We just can't
+		 * win!
+		 */ 
+		exc = EXC_BAD_ACCESS;
 		code = EXC_I386_GPFLT;
 		subcode = err;
 		break;
@@ -1012,37 +1134,44 @@ user_trap(
 		break;
 
 	    case T_SSE_FLOAT_ERROR:
-	        fpSSEexterrflt();
+		fpSSEexterrflt(); /* Propagates exception directly, doesn't return */
 		return;
 
 
 	    case T_FLOATING_POINT_ERROR:
-		fpexterrflt();
+		fpexterrflt(); /* Propagates exception directly, doesn't return */
 		return;
 
+	    case T_DTRACE_RET:
+#if CONFIG_DTRACE
+		if (dtrace_user_probe(saved_state) == KERN_SUCCESS)
+			return; /* If it succeeds, we are done... */
+#endif
+		/*
+		 * If we get an INT 0x7f when we do not expect to,
+		 * treat it as an illegal instruction
+		 */
+		exc = EXC_BAD_INSTRUCTION;
+		code = EXC_I386_INVOP;
+		break;
+
 	    default:
 #if     MACH_KGDB
 		Debugger("Unanticipated user trap");
 		return;
 #endif  /* MACH_KGDB */
 #if	MACH_KDB
-		if (kdb_trap(type, err, saved_state))
+		if (kdb_trap(type, err, saved_state32(saved_state)))
 		    return;
 #endif	/* MACH_KDB */
-		panic("user trap");
+		panic("Unexpected user trap, type %d", type);
 		return;
 	}
-	intr = ml_set_interrupts_enabled(FALSE);
-	myast = ast_pending();
-	while (*myast & AST_ALL) {
-	        ast_taken(AST_ALL, intr);
-		ml_set_interrupts_enabled(FALSE);
-		myast = ast_pending();
-	}
-	ml_set_interrupts_enabled(intr);
-
+	/* Note: Codepaths that directly return from user_trap() have pending
+	 * ASTs processed in locore
+	 */
 	i386_exception(exc, code, subcode);
-	/*NOTREACHED*/
+	/* NOTREACHED */
 }
 
 
@@ -1083,10 +1212,10 @@ i386_astintr(int preemption)
 void
 i386_exception(
 	int	exc,
-	int	code,
-	int	subcode)
+	mach_exception_code_t code,
+	mach_exception_subcode_t subcode)
 {
-	exception_data_type_t   codes[EXCEPTION_CODE_MAX];
+	mach_exception_data_type_t   codes[EXCEPTION_CODE_MAX];
 
 	codes[0] = code;		/* new exception interface */
 	codes[1] = subcode;
@@ -1111,24 +1240,24 @@ kernel_preempt_check(void)
 	myast = ast_pending();
 
 	if ((*myast & AST_URGENT) && intr == TRUE && get_interrupt_level() == 0) {
-	        /*
+		/*
 		 * can handle interrupts and preemptions 
 		 * at this point
 		 */
-	        ml_set_interrupts_enabled(intr);
+		ml_set_interrupts_enabled(intr);
 
 		/*
 		 * now cause the PRE-EMPTION trap
 		 */
 		__asm__ volatile ("     int     $0xff");
 	} else {
-	        /*
+		/*
 		 * if interrupts were already disabled or
 		 * we're in an interrupt context, we can't
 		 * preempt...  of course if AST_URGENT
 		 * isn't set we also don't want to
 		 */
-	        ml_set_interrupts_enabled(intr);
+		ml_set_interrupts_enabled(intr);
 	}
 }
 
@@ -1223,16 +1352,14 @@ sync_iss_to_iks(x86_saved_state32_t *saved_state)
  * or user space.
  */
 void
-sync_iss_to_iks_unconditionally(__unused x86_saved_state32_t *saved_state) {
+sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) {
 	struct x86_kernel_state32 *iks;
 	vm_offset_t kstack;
-	boolean_t record_active_regs = FALSE;
 
 	if ((kstack = current_thread()->kernel_stack) != 0) {
-
 		iks = STACK_IKS(kstack);
-				/*
-		 * Show the trap handler path
+		/*
+		 * Display the trap handler path.
 		 */
 		__asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx));
 		__asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp));
@@ -1240,9 +1367,8 @@ sync_iss_to_iks_unconditionally(__unused x86_saved_state32_t *saved_state) {
 		__asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi));
 		__asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi));
 		/*
-		 * "Current" instruction pointer
+		 * "Current" instruction pointer.
 		 */
 		__asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip));
-
 	}
 }
diff --git a/osfmk/i386/trap.h b/osfmk/i386/trap.h
index 44eded817..2488dd2a8 100644
--- a/osfmk/i386/trap.h
+++ b/osfmk/i386/trap.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -66,7 +72,7 @@
 #define	T_NO_FPU		7		/* no floating point */
 #define	T_DOUBLE_FAULT		8		/* double fault */
 #define	T_FPU_FAULT		9
-/*				10 */
+#define T_INVALID_TSS		10
 #define	T_SEGMENT_NOT_PRESENT	11
 #define	T_STACK_FAULT		12
 #define	T_GENERAL_PROTECTION	13
@@ -76,6 +82,8 @@
 #define	T_WATCHPOINT		17
 #define T_MACHINE_CHECK		18
 #define T_SSE_FLOAT_ERROR       19
+/*                          20-126 */
+#define T_DTRACE_RET            127
 #define T_PREEMPT		255
 
 #define TRAP_NAMES "divide error", "debug trap", "NMI", "breakpoint", \
@@ -83,7 +91,7 @@
 		   "no coprocessor", "double fault", "coprocessor overrun", \
 		   "invalid TSS", "segment not present", "stack bounds", \
 		   "general protection", "page fault", "(reserved)", \
-		   "coprocessor error", "watchpoint", "(reserved)", "SSE floating point"
+		   "coprocessor error", "watchpoint", "machine check", "SSE floating point"
 
 /*
  * Page-fault trap codes.
@@ -103,13 +111,14 @@
 #include <i386/thread.h>
 
 extern void		i386_exception(
-				int		exc,
-				int		code,
-				int		subcode);
+				int			exc,
+				mach_exception_code_t	code,
+				mach_exception_subcode_t subcode);
 
 extern void		sync_iss_to_iks(x86_saved_state32_t *regs);
 
-extern void		sync_iss_to_iks_unconditionally(x86_saved_state32_t *regs);
+extern void		sync_iss_to_iks_unconditionally(
+				x86_saved_state_t	*regs);
 
 extern void		kernel_trap(x86_saved_state_t *regs);
 
@@ -136,13 +145,13 @@ extern perfCallback perfTrapHook;
 extern perfCallback perfASTHook;
 extern perfCallback perfIntHook;
 
+extern void		panic_i386_backtrace(void *, int);
 #if MACH_KDP
 extern boolean_t	kdp_i386_trap(
 				unsigned int,
 				x86_saved_state32_t *,
 				kern_return_t,
 				vm_offset_t);
-extern void		panic_i386_backtrace(void *, int);
 #endif /* MACH_KDP */
 #endif	/* !ASSEMBLER && MACH_KERNEL */
 
diff --git a/osfmk/i386/tsc.c b/osfmk/i386/tsc.c
index b6741c24d..724bb0f9f 100644
--- a/osfmk/i386/tsc.c
+++ b/osfmk/i386/tsc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -46,16 +52,13 @@
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>		/* for kernel_map */
 #include <i386/ipl.h>
-#include <i386/pit.h>
 #include <architecture/i386/pio.h>
 #include <i386/misc_protos.h>
 #include <i386/proc_reg.h>
 #include <i386/machine_cpu.h>
 #include <i386/mp.h>
-#include <i386/cpuid.h>
 #include <i386/cpu_data.h>
-#include <i386/cpu_threads.h>
-#include <i386/perfmon.h>
+#include <i386/cpuid.h>
 #include <i386/machine_routines.h>
 #include <pexpert/pexpert.h>
 #include <machine/limits.h>
@@ -71,6 +74,11 @@ uint64_t	tscFCvtt2n = 0;
 uint64_t	tscFCvtn2t = 0;
 uint64_t	tscGranularity = 0;
 uint64_t	bus2tsc = 0;
+uint64_t	busFreq = 0;
+
+#define bit(n)		(1ULL << (n))
+#define bitmask(h,l)	((bit(h)|(bit(h)-1)) & ~(bit(l)-1))
+#define bitfield(x,h,l)	(((x) & bitmask(h,l)) >> l)
 
 /* Decimal powers: */
 #define kilo (1000ULL)
@@ -79,6 +87,8 @@ uint64_t	bus2tsc = 0;
 #define Tera (kilo * Giga)
 #define Peta (kilo * Tera)
 
+#define CPU_FAMILY_PENTIUM_M	(0x6)
+
 static const char	FSB_Frequency_prop[] = "FSBFrequency";
 /*
  * This routine extracts the front-side bus frequency in Hz from
@@ -90,14 +100,15 @@ EFI_FSB_frequency(void)
 	uint64_t	frequency = 0;
 	DTEntry		entry;
 	void		*value;
-	int		size;
+	unsigned int	size;
 
 	if (DTLookupEntry(0, "/efi/platform", &entry) != kSuccess) {
 		kprintf("EFI_FSB_frequency: didn't find /efi/platform\n");
 		return 0;
 	}
 	if (DTGetProperty(entry,FSB_Frequency_prop,&value,&size) != kSuccess) {
-		kprintf("EFI_FSB_frequency: property %s not found\n");
+		kprintf("EFI_FSB_frequency: property %s not found\n",
+			FSB_Frequency_prop);
 		return 0;
 	}
 	if (size == sizeof(uint64_t)) {
@@ -121,72 +132,78 @@ EFI_FSB_frequency(void)
 void
 tsc_init(void)
 {
-    uint64_t	busFreq;
-    uint64_t	busFCvtInt;
-    uint32_t	cpuFamily  = cpuid_info()->cpuid_family;
-
-    /*
-     * Get the FSB frequency and conversion factors.
-     */
-    busFreq = EFI_FSB_frequency();
-    if (busFreq != 0) {
-	busFCvtt2n = ((1 * Giga) << 32) / busFreq;
-	busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
-	busFCvtInt = tmrCvt(1 * Peta, 0xFFFFFFFFFFFFFFFFULL / busFreq); 
-    } else {
-	panic("rtclock_init: EFI not supported!\n");
-    }
-	
-    kprintf(" BUS: Frequency = %6d.%04dMHz, "
-	    "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, "
-	    "cvtInt = %08X.%08X\n",
-	    (uint32_t)(busFreq / Mega),
-	    (uint32_t)(busFreq % Mega), 
-	    (uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
-	    (uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t,
-	    (uint32_t)(busFCvtInt >> 32), (uint32_t)busFCvtInt);
-
-    /*
-     * Get the TSC increment.  The TSC is incremented by this
-     * on every bus tick.  Calculate the TSC conversion factors
-     * to and from nano-seconds.
-     */
-    if (cpuFamily == CPUID_FAMILY_686) {
-	uint64_t	prfsts;
-	
-	prfsts = rdmsr64(IA32_PERF_STS);
-	tscGranularity = (uint32_t)(prfsts >> BusRatioShift) & BusRatioMask;
-    } else {
-	panic("rtclock_init: unknown CPU family: 0x%X\n",
-	      cpuFamily);
-    }
-	
-    tscFCvtt2n = busFCvtt2n / (uint64_t)tscGranularity;
-    tscFreq = ((1 * Giga)  << 32) / tscFCvtt2n;
-    tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
-
-    kprintf(" TSC: Frequency = %6d.%04dMHz, "
-	    "cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %d\n",
-	    (uint32_t)(tscFreq / Mega),
-	    (uint32_t)(tscFreq % Mega), 
-	    (uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
-	    (uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
-	    tscGranularity);
-
-    /*
-     * Calculate conversion from BUS to TSC
-     */
-    bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
+	uint64_t	busFCvtInt = 0;
+	boolean_t	N_by_2_bus_ratio = FALSE;
+
+	/*
+	 * Get the FSB frequency and conversion factors.
+	 */
+	busFreq = EFI_FSB_frequency();
+	if (busFreq != 0) {
+		busFCvtt2n = ((1 * Giga) << 32) / busFreq;
+		busFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / busFCvtt2n;
+		busFCvtInt = tmrCvt(1 * Peta, 0xFFFFFFFFFFFFFFFFULL / busFreq); 
+	} else {
+		panic("rtclock_init: EFI not supported!\n");
+	}
+
+	kprintf(" BUS: Frequency = %6d.%04dMHz, "
+			"cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, "
+			"cvtInt = %08X.%08X\n",
+			(uint32_t)(busFreq / Mega),
+			(uint32_t)(busFreq % Mega), 
+			(uint32_t)(busFCvtt2n >> 32), (uint32_t)busFCvtt2n,
+			(uint32_t)(busFCvtn2t >> 32), (uint32_t)busFCvtn2t,
+			(uint32_t)(busFCvtInt >> 32), (uint32_t)busFCvtInt);
+
+	/*
+	 * Get the TSC increment.  The TSC is incremented by this
+	 * on every bus tick.  Calculate the TSC conversion factors
+	 * to and from nano-seconds.
+	 */
+	if (cpuid_info()->cpuid_family == CPU_FAMILY_PENTIUM_M) {
+		uint64_t	prfsts;
+
+		prfsts = rdmsr64(IA32_PERF_STS);
+		tscGranularity = (uint32_t)bitfield(prfsts, 44, 40);
+		N_by_2_bus_ratio = prfsts & bit(46);
+
+	} else {
+		panic("rtclock_init: unknown CPU family: 0x%X\n",
+			cpuid_info()->cpuid_family);
+	}
+
+	if (N_by_2_bus_ratio)
+		tscFCvtt2n = busFCvtt2n * 2 / (uint64_t)tscGranularity;
+	else
+		tscFCvtt2n = busFCvtt2n / (uint64_t)tscGranularity;
+
+	tscFreq = ((1 * Giga)  << 32) / tscFCvtt2n;
+	tscFCvtn2t = 0xFFFFFFFFFFFFFFFFULL / tscFCvtt2n;
+
+	kprintf(" TSC: Frequency = %6d.%04dMHz, "
+			"cvtt2n = %08X.%08X, cvtn2t = %08X.%08X, gran = %lld\n",
+			(uint32_t)(tscFreq / Mega),
+			(uint32_t)(tscFreq % Mega), 
+			(uint32_t)(tscFCvtt2n >> 32), (uint32_t)tscFCvtt2n,
+			(uint32_t)(tscFCvtn2t >> 32), (uint32_t)tscFCvtn2t,
+			tscGranularity);
+
+	/*
+	 * Calculate conversion from BUS to TSC
+	 */
+	bus2tsc = tmrCvt(busFCvtt2n, tscFCvtn2t);
 }
 
 void
 tsc_get_info(tscInfo_t *info)
 {
-    info->busFCvtt2n     = busFCvtt2n;
-    info->busFCvtn2t     = busFCvtn2t;
-    info->tscFreq        = tscFreq;
-    info->tscFCvtt2n     = tscFCvtt2n;
-    info->tscFCvtn2t     = tscFCvtn2t;
-    info->tscGranularity = tscGranularity;
-    info->bus2tsc        = bus2tsc;
+	info->busFCvtt2n     = busFCvtt2n;
+	info->busFCvtn2t     = busFCvtn2t;
+	info->tscFreq        = tscFreq;
+	info->tscFCvtt2n     = tscFCvtt2n;
+	info->tscFCvtn2t     = tscFCvtn2t;
+	info->tscGranularity = tscGranularity;
+	info->bus2tsc        = bus2tsc;
+	info->busFreq        = busFreq;
 }
diff --git a/osfmk/i386/tsc.h b/osfmk/i386/tsc.h
index 71d2d6527..79ece7085 100644
--- a/osfmk/i386/tsc.h
+++ b/osfmk/i386/tsc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -34,8 +40,6 @@
 #ifndef _I386_TSC_H_
 #define _I386_TSC_H_
 
-#define BusRatioShift 40
-#define BusRatioMask 0x1F
 #define IA32_PERF_STS 0x198
 
 extern uint64_t	busFCvtt2n;
@@ -45,6 +49,7 @@ extern uint64_t tscFCvtt2n;
 extern uint64_t tscFCvtn2t;
 extern uint64_t tscGranularity;
 extern uint64_t bus2tsc;
+extern uint64_t busFreq;
 
 struct tscInfo
 {
@@ -55,6 +60,7 @@ uint64_t	tscFCvtt2n;
 uint64_t	tscFCvtn2t;
 uint64_t	tscGranularity;
 uint64_t	bus2tsc;
+uint64_t	busFreq;
 };
 typedef struct tscInfo tscInfo_t;
 
diff --git a/osfmk/i386/tss.h b/osfmk/i386/tss.h
index 85dc5333b..1426d5a41 100644
--- a/osfmk/i386/tss.h
+++ b/osfmk/i386/tss.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/user_ldt.c b/osfmk/i386/user_ldt.c
index 57bb34fc9..87aab0426 100644
--- a/osfmk/i386/user_ldt.c
+++ b/osfmk/i386/user_ldt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -82,7 +88,7 @@ static void user_ldt_set_action(void *);
 
 int
 i386_set_ldt(
-	int			*retval,
+	uint32_t		*retval,
 	uint32_t		start_sel,
 	uint32_t		descs,	/* out */
 	uint32_t		num_sels)
@@ -100,7 +106,7 @@ i386_set_ldt(
 	    && (start_sel < min_selector || start_sel >= LDTSZ))
 	    return EINVAL;
 	if (start_sel != LDT_AUTO_ALLOC
-	    && (uint64_t)start_sel + (uint64_t)num_sels > LDTSZ)
+	    && (uint64_t)start_sel + (uint64_t)num_sels > LDTSZ) /* cast to uint64_t to detect wrap-around */
 	    return EINVAL;
 
 	task_lock(task);
@@ -276,7 +282,7 @@ i386_set_ldt(
 
 int
 i386_get_ldt(
-	int			*retval,
+	uint32_t		*retval,
 	uint32_t		start_sel,
 	uint32_t		descs,	/* out */
 	uint32_t		num_sels)
diff --git a/osfmk/i386/user_ldt.h b/osfmk/i386/user_ldt.h
index e11cd3072..e83c16dca 100644
--- a/osfmk/i386/user_ldt.h
+++ b/osfmk/i386/user_ldt.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/vm_tuning.h b/osfmk/i386/vm_tuning.h
index ba65481ba..5465f801e 100644
--- a/osfmk/i386/vm_tuning.h
+++ b/osfmk/i386/vm_tuning.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/i386/vmx.h b/osfmk/i386/vmx.h
new file mode 100644
index 000000000..132baf088
--- /dev/null
+++ b/osfmk/i386/vmx.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#ifndef _I386_VMX_H_
+#define _I386_VMX_H_
+
+/*
+ * Error codes
+ */
+#define VMX_OK			0 /* all ok */
+#define VMX_UNSUPPORTED	1 /* VT unsupported or disabled on 1+ cores */
+#define VMX_INUSE		2 /* VT is being exclusively used already */
+
+/* SPI */
+int host_vmxon(int exclusive);
+void host_vmxoff(void);
+
+#endif
diff --git a/osfmk/i386/vmx/vmx_asm.h b/osfmk/i386/vmx/vmx_asm.h
new file mode 100644
index 000000000..78722ec86
--- /dev/null
+++ b/osfmk/i386/vmx/vmx_asm.h
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#ifndef _I386_VMX_ASM_H_
+#define _I386_VMX_ASM_H_
+ 
+#include <mach/machine/vm_types.h>
+#include <mach/boolean.h>
+#include <kern/assert.h>
+#include <i386/eflags.h>
+#include <i386/seg.h>
+
+#ifndef DEBUG
+#include <debug.h>
+#endif
+
+#define VMX_FAIL_INVALID	-1
+#define VMX_FAIL_VALID		-2
+#define VMX_SUCCEED			0
+
+static inline void enter_64bit_mode(void) {
+	__asm__ __volatile__ (
+		".byte   0xea    /* far jump longmode */	\n\t"
+		".long   1f					\n\t"
+		".word   %P0					\n\t"
+		".code64					\n\t"
+		"1:"
+		:: "i" (KERNEL64_CS)
+	);
+}
+static inline void enter_compat_mode(void) {
+	asm(
+		"ljmp    *4f					\n\t"
+	"4:							\n\t"
+		".long   5f					\n\t"
+		".word   %P0					\n\t"
+		".code32					\n\t"
+	"5:"
+		:: "i" (KERNEL_CS)
+	);
+}
+
+#define __VMXOFF(res)						\
+	__asm__ __volatile__ (					\
+		"vmxoff		\n\t"				\
+		"cmovcl %2, %0	\n\t"	/* CF = 1, ZF = 0 */	\
+		"cmovzl %3, %0"		/* CF = 0, ZF = 1 */	\
+		: "=&r" (res)				\
+		: "0" (VMX_SUCCEED),				\
+		  "r" (VMX_FAIL_INVALID),			\
+		  "r" (VMX_FAIL_VALID)				\
+		: "memory", "cc"				\
+	)
+
+#define __VMXON(addr, res)					\
+	__asm__ __volatile__ (					\
+		"vmxon %4	\n\t"				\
+		"cmovcl %2, %0	\n\t"	/* CF = 1, ZF = 0 */	\
+		"cmovzl %3, %0"		/* CF = 0, ZF = 1 */	\
+		: "=&r" (res)					\
+		: "0" (VMX_SUCCEED),				\
+		  "r" (VMX_FAIL_INVALID),			\
+		  "r" (VMX_FAIL_VALID),				\
+		  "m" (*addr)					\
+		: "memory", "cc"				\
+	);
+
+
+/*
+ *	__vmxoff -- Leave VMX Operation
+ *
+ */
+static inline int
+__vmxoff(void)
+{
+	int result;
+	if (ml_is64bit()) {
+		/* don't put anything between these lines! */
+		enter_64bit_mode();
+		__VMXOFF(result);
+		enter_compat_mode();
+	} else {
+		__VMXOFF(result);
+	}
+	return result;
+}
+
+/*
+ *	__vmxon -- Enter VMX Operation
+ *
+ */
+ static inline int
+__vmxon(addr64_t *v)
+ {
+	int result;
+	if (ml_is64bit()) {
+		/* don't put anything between these lines! */
+		enter_64bit_mode();
+		__VMXON(v, result);
+		enter_compat_mode();
+	} else {
+		__VMXON(v, result);
+	}
+	return result;
+}
+
+/*
+ * VMX Capability Registers (VCR)
+ *
+ */
+#define VMX_VCR_VMCS_MEM_TYPE_BIT	50
+#define VMX_VCR_VMCS_MEM_TYPE_MASK	0xF
+
+#define VMX_VCR_VMCS_SIZE_BIT		32
+#define VMX_VCR_VMCS_SIZE_MASK		0x01FFF
+#define VMX_VCR_VMCS_REV_ID		0x00000000FFFFFFFFLL
+
+#define VMX_VCR_ACT_HLT_BIT		6
+#define VMX_VCR_ACT_HLT_MASK		0x1
+#define VMX_VCR_ACT_SHUTDOWN_BIT	7
+#define VMX_VCR_ACT_SHUTDOWN_MASK	0x1
+#define VMX_VCR_ACT_SIPI_BIT		8
+#define VMX_VCR_ACT_SIPI_MASK		0x1
+#define VMX_VCR_ACT_CSTATE_BIT		9
+#define VMX_VCR_ACT_CSTATE_MASK		0x1
+#define VMX_VCR_CR3_TARGS_BIT		16
+#define VMX_VCR_CR3_TARGS_MASK		0xFF
+#define VMX_VCR_MAX_MSRS_BIT		25
+#define VMX_VCR_MAX_MSRS_MASK		0x7
+#define VMX_VCR_MSEG_ID_BIT		32
+#define VMX_VCR_MSEG_ID_MASK		0xFFFFFFFF
+
+#endif	/* _I386_VMX_H_ */
diff --git a/osfmk/i386/vmx/vmx_cpu.c b/osfmk/i386/vmx/vmx_cpu.c
new file mode 100644
index 000000000..31c16ca9f
--- /dev/null
+++ b/osfmk/i386/vmx/vmx_cpu.c
@@ -0,0 +1,373 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <pexpert/pexpert.h>
+#include <i386/cpuid.h>
+#include <i386/cpu_data.h>
+#include <i386/proc_reg.h>
+#include <i386/vmx.h>
+#include <i386/vmx/vmx_asm.h>
+#include <i386/vmx/vmx_shims.h>
+#include <i386/vmx/vmx_cpu.h>
+#include <i386/mtrr.h>
+#include <mach/mach_host.h>             /* for host_info() */
+#include <i386/mp.h>
+
+#define VMX_KPRINTF(x...) /* kprintf("vmx: " x) */
+
+int vmx_use_count = 0;
+int vmx_exclusive = 0;
+decl_simple_lock_data(static,vmx_use_count_lock)
+
+/* -----------------------------------------------------------------------------
+   vmx_is_available()
+	Is the VMX facility available on this CPU?
+   -------------------------------------------------------------------------- */
+static inline boolean_t
+vmx_is_available(void)
+{
+	return (0 != (cpuid_features() & CPUID_FEATURE_VMX));
+}
+
+/* -----------------------------------------------------------------------------
+   vmxon_is_enabled()
+	Is the VMXON instruction enabled on this CPU?
+   -------------------------------------------------------------------------- */
+static inline boolean_t
+vmxon_is_enabled(void)
+{
+	return (vmx_is_available() &&
+		(rdmsr64(MSR_IA32_FEATURE_CONTROL) & MSR_IA32_FEATCTL_VMXON));
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_is_cr0_valid()
+	Is CR0 valid for executing VMXON on this CPU?
+   -------------------------------------------------------------------------- */
+static inline boolean_t
+vmx_is_cr0_valid(vmx_specs_t *specs)
+{
+	uint32_t cr0 = get_cr0();
+	return (0 == ((~cr0 & specs->cr0_fixed_0)|(cr0 & ~specs->cr0_fixed_1)));
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_is_cr4_valid()
+	Is CR4 valid for executing VMXON on this CPU?
+   -------------------------------------------------------------------------- */
+static inline boolean_t
+vmx_is_cr4_valid(vmx_specs_t *specs)
+{
+	uint32_t cr4 = get_cr4();
+	return (0 == ((~cr4 & specs->cr4_fixed_0)|(cr4 & ~specs->cr4_fixed_1)));
+}
+
+static void
+vmx_init(void)
+{
+	uint64_t msr_image;
+
+	if (!vmx_is_available())
+		return;
+
+	/*
+	 * We don't count on EFI initializing MSR_IA32_FEATURE_CONTROL
+	 * and turning VMXON on and locking the bit, so we do that now.
+	 */
+	msr_image = rdmsr64(MSR_IA32_FEATURE_CONTROL);
+	if (0 == ((msr_image & MSR_IA32_FEATCTL_LOCK)))
+		wrmsr64(MSR_IA32_FEATURE_CONTROL,
+			(msr_image |
+			 MSR_IA32_FEATCTL_VMXON |
+			 MSR_IA32_FEATCTL_LOCK));
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_get_specs()
+	Obtain VMX facility specifications for this CPU and
+	enter them into the vmx_specs_t structure. If VMX is not available or
+	disabled on this CPU, set vmx_present to false and return leaving
+	the remainder of the vmx_specs_t uninitialized. 
+   -------------------------------------------------------------------------- */
+void
+vmx_get_specs()
+{
+	vmx_specs_t *specs = &current_cpu_datap()->cpu_vmx.specs;
+	uint64_t msr_image;
+	
+	/* this is called once for every CPU, but the lock doesn't care :-) */
+	simple_lock_init(&vmx_use_count_lock, 0);
+
+	vmx_init();
+
+	/*
+	 * if we have read the data on boot, we won't read it
+	 *  again on wakeup, otherwise *bad* things will happen
+	 */
+	if (specs->initialized)
+		return;
+	else
+		specs->initialized = TRUE;
+
+	/* See if VMX is present, return if it is not */
+	specs->vmx_present = vmx_is_available() && vmxon_is_enabled();
+	if (!specs->vmx_present)
+		return;
+
+#define bitfield(x,f)	((x >> f##_BIT) & f##_MASK)
+	/* Obtain and decode VMX general capabilities */	
+	msr_image = rdmsr64(MSR_IA32_VMX_BASIC);
+	specs->vmcs_id       = msr_image & VMX_VCR_VMCS_REV_ID;
+	specs->vmcs_mem_type = bitfield(msr_image, VMX_VCR_VMCS_MEM_TYPE) != 0;
+	specs->vmcs_size = bitfield(msr_image, VMX_VCR_VMCS_SIZE);
+							  
+	/* Obtain allowed settings for pin-based execution controls */
+	msr_image = rdmsr64(MSR_IA32_VMXPINBASED_CTLS);
+	specs->pin_exctls_0 = msr_image & 0xFFFFFFFF;
+	specs->pin_exctls_1 = msr_image >> 32;
+	
+	/* Obtain allowed settings for processor-based execution controls */
+	msr_image = rdmsr64(MSR_IA32_PROCBASED_CTLS);
+	specs->proc_exctls_0 = msr_image & 0xFFFFFFFF;
+	specs->proc_exctls_1 = msr_image >> 32;
+	
+	/* Obtain allowed settings for VM-exit controls */
+	msr_image = rdmsr64(MSR_IA32_VMX_EXIT_CTLS);
+	specs->exit_ctls_0 = msr_image & 0xFFFFFFFF;
+	specs->exit_ctls_1 = msr_image >> 32;
+	
+	/* Obtain allowed settings for VM-entry controls */
+	msr_image = rdmsr64(MSR_IA32_VMX_ENTRY_CTLS);
+	specs->enter_ctls_0 = msr_image & 0xFFFFFFFF;
+	specs->enter_ctls_0 = msr_image >> 32;
+	
+	/* Obtain and decode miscellaneous capabilities */
+	msr_image = rdmsr64(MSR_IA32_VMX_MISC);
+	specs->act_halt     = bitfield(msr_image, VMX_VCR_ACT_HLT) != 0;
+	specs->act_shutdown = bitfield(msr_image, VMX_VCR_ACT_SHUTDOWN) != 0;
+	specs->act_SIPI     = bitfield(msr_image, VMX_VCR_ACT_SIPI) != 0;
+	specs->act_CSTATE   = bitfield(msr_image, VMX_VCR_ACT_CSTATE) != 0;
+	specs->cr3_targs    = bitfield(msr_image, VMX_VCR_CR3_TARGS);
+	specs->max_msrs     = 512 * (1 + bitfield(msr_image, VMX_VCR_MAX_MSRS));
+	specs->mseg_id      = bitfield(msr_image, VMX_VCR_MSEG_ID);
+	
+	/* Obtain VMX-fixed bits in CR0 */
+	specs->cr0_fixed_0 = rdmsr64(MSR_IA32_VMX_CR0_FIXED0) & 0xFFFFFFFF;
+	specs->cr0_fixed_1 = rdmsr64(MSR_IA32_VMX_CR0_FIXED1) & 0xFFFFFFFF;
+	
+	/* Obtain VMX-fixed bits in CR4 */
+	specs->cr4_fixed_0 = rdmsr64(MSR_IA32_VMX_CR4_FIXED0) & 0xFFFFFFFF;
+	specs->cr4_fixed_1 = rdmsr64(MSR_IA32_VMX_CR4_FIXED1) & 0xFFFFFFFF;
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_on()
+	Enter VMX root operation on this CPU.
+   -------------------------------------------------------------------------- */
+static void
+vmx_on(void)
+{
+	vmx_cpu_t *cpu = &current_cpu_datap()->cpu_vmx;
+	addr64_t vmxon_region_paddr;
+	int result;
+
+	vmx_init();
+	
+	assert(cpu->specs.vmx_present);
+
+	if (NULL == cpu->vmxon_region)
+		panic("vmx_on: VMXON region not allocated");
+	vmxon_region_paddr = vmx_paddr(cpu->vmxon_region);
+
+	/*
+	 * Enable VMX operation.
+	 */
+	set_cr4(get_cr4() | CR4_VMXE);
+	
+	assert(vmx_is_cr0_valid(&cpu->specs));
+	assert(vmx_is_cr4_valid(&cpu->specs));
+	
+	if ((result = __vmxon(&vmxon_region_paddr)) != VMX_SUCCEED) {
+		panic("vmx_on: unexpected return %d from __vmxon()", result);
+	}
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_off()
+	Leave VMX root operation on this CPU.
+   -------------------------------------------------------------------------- */
+static void
+vmx_off(void)
+{
+	int result;
+	
+	/* Tell the CPU to release the VMXON region */
+	if ((result = __vmxoff()) != VMX_SUCCEED) {
+		panic("vmx_off: unexpected return %d from __vmxoff()", result);
+	}
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_allocate_vmxon_regions()
+	Allocate, clear and init VMXON regions for all CPUs.
+   -------------------------------------------------------------------------- */
+static void
+vmx_allocate_vmxon_regions(void)
+{
+	unsigned int i;
+	
+	for (i=0; i<real_ncpus; i++) {
+		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
+
+		/* The size is defined to be always <= 4K, so we just allocate a page */
+		cpu->vmxon_region = vmx_pcalloc();
+		if (NULL == cpu->vmxon_region)
+			panic("vmx_allocate_vmxon_regions: unable to allocate VMXON region");
+		*(uint32_t*)(cpu->vmxon_region) = cpu->specs.vmcs_id;
+	}
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_free_vmxon_regions()
+	Free VMXON regions for all CPUs.
+   -------------------------------------------------------------------------- */
+static void
+vmx_free_vmxon_regions(void)
+{
+	unsigned int i;
+
+	for (i=0; i<real_ncpus; i++) {
+		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
+
+		vmx_pfree(cpu->vmxon_region);
+		cpu->vmxon_region = NULL;
+	}
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_globally_available()
+	Checks whether VT can be turned on for all CPUs.
+   -------------------------------------------------------------------------- */
+static boolean_t
+vmx_globally_available(void)
+{
+	unsigned int i;
+	
+	boolean_t available = TRUE;
+
+	for (i=0; i<real_ncpus; i++) {
+		vmx_cpu_t *cpu = &cpu_datap(i)->cpu_vmx;
+
+		if (!cpu->specs.vmx_present)
+			available = FALSE;
+	}
+	VMX_KPRINTF("VMX available: %d\n", available);
+	return available;
+}
+
+
+/* -----------------------------------------------------------------------------
+   vmx_turn_on()
+	Turn on VT operation on all CPUs.
+   -------------------------------------------------------------------------- */
+int
+host_vmxon(boolean_t exclusive)
+{
+	int error;
+
+	if (!vmx_globally_available())
+		return VMX_UNSUPPORTED;
+
+	simple_lock(&vmx_use_count_lock);
+
+	if (vmx_exclusive) {
+		error = VMX_INUSE;
+		goto out;
+	}
+	vmx_use_count++;
+	if (vmx_use_count == 1) { /* was turned off before */
+		vmx_allocate_vmxon_regions();
+		mp_rendezvous(NULL, (void (*)(void *))vmx_on, NULL, NULL);
+	}
+	vmx_exclusive = exclusive;
+
+	VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
+	error = VMX_OK;
+out:
+	simple_unlock(&vmx_use_count_lock);
+
+	return error;
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_turn_off()
+	Turn off VT operation on all CPUs.
+   -------------------------------------------------------------------------- */
+void
+host_vmxoff()
+{
+	simple_lock(&vmx_use_count_lock);
+
+	if (vmx_use_count) {
+		vmx_use_count--;
+		vmx_exclusive = 0;
+		if (!vmx_use_count) {
+			mp_rendezvous(NULL, (void (*)(void *))vmx_off, NULL, NULL);
+			vmx_free_vmxon_regions();
+		}
+	}
+
+	simple_unlock(&vmx_use_count_lock);
+
+	VMX_KPRINTF("VMX use count: %d\n", vmx_use_count);
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_suspend()
+	Turn off VT operation on this CPU if it was on.
+	Called when a CPU goes offline.
+   -------------------------------------------------------------------------- */
+void
+vmx_suspend()
+{
+	VMX_KPRINTF("vmx_suspend\n");
+	if (vmx_use_count)
+		vmx_off();
+}
+
+/* -----------------------------------------------------------------------------
+   vmx_suspend()
+	Restore the previous VT state. Called when CPU comes back online.
+   -------------------------------------------------------------------------- */
+void
+vmx_resume()
+{
+	VMX_KPRINTF("vmx_resume\n");
+	if (vmx_use_count)
+		vmx_on();
+}
diff --git a/osfmk/i386/vmx/vmx_cpu.h b/osfmk/i386/vmx/vmx_cpu.h
new file mode 100644
index 000000000..255ba421c
--- /dev/null
+++ b/osfmk/i386/vmx/vmx_cpu.h
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#ifndef _I386_VMX_CPU_H_
+#define _I386_VMX_CPU_H_
+
+#include <mach/machine/vm_types.h>
+#include <mach/boolean.h>
+#include <i386/vmx/vmx_asm.h>
+
+/*
+ * Physical CPU's VMX specifications
+ *
+ */
+typedef struct vmx_specs {
+	boolean_t	initialized;	/* the specs have already been read */
+	boolean_t	vmx_present;	/* VMX feature available and enabled */
+	uint32_t	vmcs_id;	/* VMCS revision identifier */
+	uint8_t		vmcs_mem_type;	/* VMCS memory type, (see enum above) */
+	uint16_t	vmcs_size;	/* VMCS region size in bytes */
+	boolean_t	act_halt;	/* HLT activity state supported */
+	boolean_t	act_shutdown;	/* shutdown activity state supported */
+	boolean_t	act_SIPI;	/* wait-for-SIPI activity supported */
+	boolean_t	act_CSTATE;	/* C-state activity state supported */
+	uint8_t		cr3_targs;	/* CR3 target values supported */
+	uint32_t	max_msrs;	/* max MSRs to load/store on VMX transition */
+	uint32_t	mseg_id;	/* MSEG revision identifier for SMI */
+	/*
+	 * Allowed settings for these controls are specified by
+	 * a pair of bitfields: 0-settings contain 0 bits
+	 * corresponding to controls thay may be 0; 1-settings
+	 * contain 1 bits corresponding to controls that may be 1.
+	 */
+	uint32_t	pin_exctls_0;	/* allowed 0s pin-based controls */
+	uint32_t	pin_exctls_1;	/* allowed 1s pin-based controls */
+	
+	uint32_t	proc_exctls_0;	/* allowed 0s proc-based controls */
+	uint32_t	proc_exctls_1;	/* allowed 1s proc-based controls */
+	
+	uint32_t	sec_exctls_0;	/* allowed 0s 2ndary proc-based ctrls */
+	uint32_t	sec_exctls_1;	/* allowed 1s 2ndary proc-based ctrls */
+	
+	uint32_t	exit_ctls_0;	/* allowed 0s VM-exit controls */
+	uint32_t	exit_ctls_1;	/* allowed 1s VM-exit controls */
+	
+	uint32_t	enter_ctls_0;	/* allowed 0s VM-entry controls */
+	uint32_t	enter_ctls_1;	/* allowed 1s VM-entry controls */
+
+	/*
+	 * Fixed control register bits are specified by a pair of
+	 * bitfields: 0-settings contain 0 bits corresponding to
+	 * CR bits that may be 0; 1-settings contain 1 bits
+	 * corresponding to CR bits that may be 1.
+	 */
+	uint32_t	cr0_fixed_0;	/* allowed 0-settings for CR0 */
+	uint32_t	cr0_fixed_1;	/* allowed 1-settings for CR0 */
+	
+	uint32_t	cr4_fixed_0;	/* allowed 0-settings for CR4 */
+	uint32_t	cr4_fixed_1;	/* allowed 1-settings for CR4 */
+} vmx_specs_t;
+
+typedef struct vmx_cpu {
+	vmx_specs_t	specs;		/* this phys CPU's VMX specifications */
+	void		*vmxon_region;	/* the logical address of the VMXON region page */
+} vmx_cpu_t;
+
+void vmx_get_specs(void);
+void vmx_resume(void);
+void vmx_suspend(void);
+
+#endif	/* _I386_VMX_CPU_H_ */
diff --git a/osfmk/i386/vmx/vmx_shims.c b/osfmk/i386/vmx/vmx_shims.c
new file mode 100644
index 000000000..f0fe27733
--- /dev/null
+++ b/osfmk/i386/vmx/vmx_shims.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <string.h>
+#include <kern/assert.h> 
+#include <mach/i386/vm_param.h>
+#include <mach/i386/kern_return.h>
+#include <vm/vm_kern.h>
+#include <i386/pmap.h>
+#include "vmx_shims.h"
+
+void *
+vmx_pcalloc(void)
+{
+	char 		   *pptr;
+	kern_return_t	ret;
+	ret = kmem_alloc_wired(kernel_map, (vm_offset_t *)&pptr, PAGE_SIZE);
+	if (ret != KERN_SUCCESS) return (NULL);
+	bzero(pptr, PAGE_SIZE);
+	return (pptr);
+}
+
+addr64_t
+vmx_paddr(void *va)
+{
+	return (ptoa_64(pmap_find_phys(kernel_pmap, (addr64_t)(uint32_t)va)));
+}
+
+void
+vmx_pfree(void *va)
+{
+	kmem_free(kernel_map, (vm_offset_t)va, PAGE_SIZE);
+}
diff --git a/osfmk/i386/vmx/vmx_shims.h b/osfmk/i386/vmx/vmx_shims.h
new file mode 100644
index 000000000..979396699
--- /dev/null
+++ b/osfmk/i386/vmx/vmx_shims.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+ 
+#ifndef _I386_VMX_SHIMS_H_
+#define _I386_VMX_SHIMS_H_
+
+#include <mach/machine/vm_types.h>
+#include <mach/boolean.h>
+#include <mach/vm_types.h>
+
+void *vmx_pcalloc(void);
+addr64_t vmx_paddr(void *);
+void vmx_pfree(void *);
+
+#endif	/* _I386_VMX_SHIMS_H_ */
diff --git a/osfmk/i386/xpr.h b/osfmk/i386/xpr.h
index 06b132787..9221068a7 100644
--- a/osfmk/i386/xpr.h
+++ b/osfmk/i386/xpr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_entry.c b/osfmk/ipc/ipc_entry.c
index 00814406f..b468943ac 100644
--- a/osfmk/ipc/ipc_entry.c
+++ b/osfmk/ipc/ipc_entry.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_entry.h b/osfmk/ipc/ipc_entry.h
index 70f99b476..e1c01d154 100644
--- a/osfmk/ipc/ipc_entry.h
+++ b/osfmk/ipc/ipc_entry.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_hash.c b/osfmk/ipc/ipc_hash.c
index 46462a593..19f36a890 100644
--- a/osfmk/ipc/ipc_hash.c
+++ b/osfmk/ipc/ipc_hash.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -633,6 +639,17 @@ ipc_hash_init(void)
 
 #if	MACH_IPC_DEBUG
 
+/*
+ *	Routine:	ipc_hash_size
+ *	Purpose:
+ *		Return the size of the global reverse hash table.
+ */
+natural_t
+ipc_hash_size(void)
+{
+	return ipc_hash_global_size;
+}
+
 /*
  *	Routine:	ipc_hash_info
  *	Purpose:
@@ -648,7 +665,7 @@ ipc_hash_init(void)
 ipc_hash_index_t
 ipc_hash_info(
 	hash_info_bucket_t	*info,
-	mach_msg_type_number_t count)
+	natural_t		count)
 {
 	ipc_hash_index_t i;
 
diff --git a/osfmk/ipc/ipc_hash.h b/osfmk/ipc/ipc_hash.h
index 00f4bde5c..987e54013 100644
--- a/osfmk/ipc/ipc_hash.h
+++ b/osfmk/ipc/ipc_hash.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -118,17 +124,17 @@ extern void ipc_hash_local_insert(
 	ipc_entry_t		entry);
 
 /* Initialize the reverse hash table implementation */
-extern void ipc_hash_init(void);
+extern void ipc_hash_init(void) __attribute__((section("__TEXT, initcode")));
 
 #include <mach_ipc_debug.h>
 
 #if	MACH_IPC_DEBUG
 
 #include <mach_debug/hash_info.h>
-
+extern natural_t ipc_hash_size(void);
 extern natural_t ipc_hash_info(
 	hash_info_bucket_t	*info,
-	mach_msg_type_number_t count);
+	natural_t count);
 
 #endif	/* MACH_IPC_DEBUG */
 
diff --git a/osfmk/ipc/ipc_init.c b/osfmk/ipc/ipc_init.c
index 47c0ffc00..1267d06e8 100644
--- a/osfmk/ipc/ipc_init.c
+++ b/osfmk/ipc/ipc_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -88,6 +101,7 @@
 #include <ipc/ipc_table.h>
 
 #include <mach/machine/ndr_def.h>   /* NDR_record */
+#include <ipc/ipc_labelh.h>
 
 vm_map_t ipc_kernel_map;
 vm_size_t ipc_kernel_map_size = 1024 * 1024;
@@ -169,11 +183,19 @@ ipc_bootstrap(void)
 	 * elements at the processor-level to avoid the locking.
 	 */
 	ipc_kmsg_zone = zinit(IKM_SAVED_KMSG_SIZE,
-			      ipc_port_max * MACH_PORT_QLIMIT_MAX *
+			      ipc_port_max * MACH_PORT_QLIMIT_DEFAULT *
 			      IKM_SAVED_KMSG_SIZE,
 			      IKM_SAVED_KMSG_SIZE,
 			      "ipc kmsgs");
 
+#if CONFIG_MACF_MACH
+	ipc_labelh_zone = 
+		zinit(sizeof(struct ipc_labelh),
+		      ipc_port_max * sizeof(struct ipc_labelh),
+		      sizeof(struct ipc_labelh),
+		      "label handles");
+#endif
+
 	/* create special spaces */
 
 	kr = ipc_space_create_special(&ipc_space_kernel);
diff --git a/osfmk/ipc/ipc_init.h b/osfmk/ipc/ipc_init.h
index dbe2831f2..b590d8a3e 100644
--- a/osfmk/ipc/ipc_init.h
+++ b/osfmk/ipc/ipc_init.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -116,9 +122,9 @@ extern int ipc_pset_max;
  */
 
 /* IPC initialization needed before creation of kernel task */
-extern void ipc_bootstrap(void);
+extern void ipc_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 /* Remaining IPC initialization */
-extern void ipc_init(void);
+extern void ipc_init(void) __attribute__((section("__TEXT, initcode")));
 
 #endif	/* _IPC_IPC_INIT_H_ */
diff --git a/osfmk/ipc/ipc_kmsg.c b/osfmk/ipc/ipc_kmsg.c
index 9f524e034..92a903209 100644
--- a/osfmk/ipc/ipc_kmsg.c
+++ b/osfmk/ipc/ipc_kmsg.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -97,6 +110,8 @@
 #include <ipc/ipc_hash.h>
 #include <ipc/ipc_table.h>
 
+#include <security/mac_mach_internal.h>
+
 #include <string.h>
 
 #ifdef ppc
@@ -262,6 +277,7 @@ ipc_kmsg_alloc(
 				    max_expanded_size -
 				    msg_and_trailer_size);
 	}
+
 	return(kmsg);
 }
 
@@ -284,6 +300,13 @@ ipc_kmsg_free(
 	mach_msg_size_t size = kmsg->ikm_size;
 	ipc_port_t port;
 
+#if CONFIG_MACF_MACH
+	if (kmsg->ikm_sender != NULL) {
+		task_deallocate(kmsg->ikm_sender);
+		kmsg->ikm_sender = NULL;
+	}
+#endif
+
 	/*
 	 * Check to see if the message is bound to the port.  If so,
 	 * mark it not in use.  If the port isn't already dead, then
@@ -389,8 +412,8 @@ ipc_kmsg_rmqueue(
 		prev->ikm_next = next;
 	}
 	/* XXX Temporary debug logic */
-	assert(kmsg->ikm_next = IKM_BOGUS);
-	assert(kmsg->ikm_prev = IKM_BOGUS);
+	assert((kmsg->ikm_next = IKM_BOGUS) == IKM_BOGUS);
+	assert((kmsg->ikm_prev = IKM_BOGUS) == IKM_BOGUS);
 }
 
 /*
@@ -644,6 +667,13 @@ ipc_kmsg_clean(
 		ipc_kmsg_clean_body(kmsg, body->msgh_descriptor_count,
 				    (mach_msg_descriptor_t *)(body + 1));
 	}
+
+#if CONFIG_MACF_MACH
+	if (kmsg->ikm_sender != NULL) {
+		task_deallocate(kmsg->ikm_sender);
+		kmsg->ikm_sender = NULL;
+	}
+#endif
 }
 
 /*
@@ -747,6 +777,19 @@ ipc_kmsg_get(
 		(unsigned int)kmsg->ikm_header->msgh_remote_port, 
 		(unsigned int)kmsg->ikm_header->msgh_local_port, 0); 
 #endif
+
+#if CONFIG_MACF_MACH
+	/* XXX - why do we zero sender labels here instead of in mach_msg()? */
+	task_t cur = current_task();
+	if (cur) {
+		task_reference(cur);
+		kmsg->ikm_sender = cur;
+	} else
+		trailer->msgh_labels.sender = 0;
+#else
+	  trailer->msgh_labels.sender = 0;
+#endif
+
 	*kmsgp = kmsg;
 	return MACH_MSG_SUCCESS;
 }
@@ -832,6 +875,11 @@ ipc_kmsg_get_from_kernel(
 	trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
 	trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
 
+	trailer->msgh_labels.sender = 0;
+
+#if CONFIG_MACF_MACH
+	kmsg->ikm_sender = NULL;
+#endif
 	*kmsgp = kmsg;
 	return MACH_MSG_SUCCESS;
 }
@@ -864,6 +912,9 @@ ipc_kmsg_send(
 	port = (ipc_port_t) kmsg->ikm_header->msgh_remote_port;
 	assert(IP_VALID(port));
 
+	if ((option & ~(MACH_SEND_TIMEOUT|MACH_SEND_ALWAYS)) != 0)
+		printf("ipc_kmsg_send: bad option 0x%x\n", option);
+
 	ip_lock(port);
 
 	if (port->ip_receiver == ipc_space_kernel) {
@@ -1036,6 +1087,7 @@ ipc_kmsg_copyin_header(
 	ipc_object_t dest_port, reply_port;
 	ipc_port_t dest_soright, reply_soright;
 	ipc_port_t notify_port;
+	ipc_entry_t entry;
 
 	if ((mbits != msg->msgh_bits) ||
 	    (!MACH_MSG_TYPE_PORT_ANY_SEND(dest_type)) ||
@@ -1053,9 +1105,33 @@ ipc_kmsg_copyin_header(
 	if (!MACH_PORT_VALID(dest_name))
 		goto invalid_dest;
 
-	if (notify != MACH_PORT_NULL) {
-		ipc_entry_t entry;
+#if CONFIG_MACF_MACH
+	/*
+	 * We do the port send check here instead of in ipc_kmsg_send()
+	 * because copying the header involves copying the port rights too
+	 * and we need to do the send check before anything is actually copied.
+	 */
+	entry = ipc_entry_lookup(space, dest_name);
+	if (entry != IE_NULL) {
+		int error = 0;
+		ipc_port_t port = (ipc_port_t) entry->ie_object;
+		if (port == IP_NULL)
+			goto invalid_dest;
+		ip_lock(port);
+		if (ip_active(port)) {
+			task_t self = current_task();
+			tasklabel_lock(self);
+			error = mac_port_check_send(&self->maclabel,
+			    &port->ip_label);
+			tasklabel_unlock(self);
+		}
+		ip_unlock(port);
+		if (error != 0)
+			goto invalid_dest;
+	}
+#endif
 
+	if (notify != MACH_PORT_NULL) {
 		if ((entry = ipc_entry_lookup(space, notify)) == IE_NULL) {
 			is_write_unlock(space);
 			return MACH_SEND_INVALID_NOTIFY;
@@ -1070,7 +1146,6 @@ ipc_kmsg_copyin_header(
 		notify_port = IP_NULL;
 
 	if (dest_name == reply_name) {
-		ipc_entry_t entry;
 		mach_port_name_t name = dest_name;
 
 		/*
@@ -1226,8 +1301,6 @@ ipc_kmsg_copyin_header(
 			}
 		}
 	} else if (!MACH_PORT_VALID(reply_name)) {
-		ipc_entry_t entry;
-
 		/*
 		 *	No reply port!  This is an easy case
 		 *	to make atomic.  Just copyin the destination.
@@ -1433,7 +1506,7 @@ ipc_kmsg_copyin_body(
 	user_desc_sizes = (dsc_count <= DESC_COUNT_SMALL) ?
 	    &desc_size_space : kalloc(dsc_count * sizeof(vm_size_t));
 	if (user_desc_sizes == NULL) {
-	    ipc_kmsg_clean_partial(kmsg,0,0,0,0);
+	    ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
 	    return KERN_RESOURCE_SHORTAGE;
 	}
     }
@@ -1443,7 +1516,7 @@ ipc_kmsg_copyin_body(
      * physical copies and possible contraction of the descriptors from
      * processes with pointers larger than the kernel's.
      */
-    daddr = 0;
+    daddr = NULL;
     for (i = 0; i < dsc_count; i++) {
 	daddr = naddr;
 
@@ -1467,7 +1540,7 @@ ipc_kmsg_copyin_body(
 
 	if (naddr > (mach_msg_descriptor_t *)
 	    ((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size)) {
-	    ipc_kmsg_clean_partial(kmsg,0,0,0,0);
+	    ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
 	    mr = MACH_SEND_MSG_TOO_SMALL;
 	    goto out;
 	}
@@ -1486,7 +1559,7 @@ ipc_kmsg_copyin_body(
 		/*
 		 * Invalid copy option
 		 */
-		ipc_kmsg_clean_partial(kmsg,0,0,0,0);
+		ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
 		mr = MACH_SEND_INVALID_TYPE;
 		goto out;
 	    }
@@ -1505,7 +1578,7 @@ ipc_kmsg_copyin_body(
 		    /*
 		     * Per message kernel memory limit exceeded
 		     */
-		    ipc_kmsg_clean_partial(kmsg,0,0,0,0);
+		    ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
 		    mr = MACH_MSG_VM_KERNEL;
 		    goto out;
 		}
@@ -1521,7 +1594,7 @@ ipc_kmsg_copyin_body(
     if (space_needed) {
 	if (vm_allocate(ipc_kernel_copy_map, &paddr, space_needed, VM_FLAGS_ANYWHERE) !=
 	    KERN_SUCCESS) {
-	    ipc_kmsg_clean_partial(kmsg,0,0,0,0);
+	    ipc_kmsg_clean_partial(kmsg, 0, NULL, 0, 0);
 		mr = MACH_MSG_VM_KERNEL;
 		goto out;
 	}
@@ -1616,7 +1689,7 @@ ipc_kmsg_copyin_body(
 		dsc->type = dsc_type;
 
 		if (length == 0) {
-		    dsc->address = 0;
+		    dsc->address = NULL;
 		} else if ((length >= MSG_OOL_SIZE_SMALL) &&
 			   (copy_options == MACH_MSG_PHYSICAL_COPY) && !dealloc) {
 
@@ -1714,13 +1787,16 @@ ipc_kmsg_copyin_body(
 		dsc->copy = copy_option;
 		dsc->type = daddr->type.type;
 		dsc->count = count;
+		dsc->address = NULL;  /* for now */
+
+		result_disp = ipc_object_copyin_type(user_disp);
+		dsc->disposition = result_disp;
 
 		/* calculate length of data in bytes, rounding up */
 		length = count * sizeof(mach_port_name_t);
 		
 		if (length == 0) {
 		    complex = TRUE;
-		    dsc->address = (void *) 0;
 		    break;
 		}
 
@@ -1741,13 +1817,9 @@ ipc_kmsg_copyin_body(
 		    (void) mach_vm_deallocate(map, addr, (mach_vm_size_t)length);
 		}
 		
+       		objects = (ipc_object_t *) data;
 		dsc->address = data;
 		
-		result_disp = ipc_object_copyin_type(user_disp);
-		dsc->disposition = result_disp;
-
-		objects = (ipc_object_t *) data;
-		
 		for ( j = 0; j < count; j++) {
 		    mach_port_name_t port = (mach_port_name_t) objects[j];
 		    ipc_object_t object;
@@ -1766,6 +1838,7 @@ ipc_kmsg_copyin_body(
 		    	    	ipc_object_destroy(object, result_disp);
 			}
 			kfree(data, length);
+			dsc->address = NULL;
 			mr = MACH_SEND_INVALID_RIGHT;
 			break;
 		    }
@@ -2094,6 +2167,7 @@ ipc_kmsg_copyout_header(
 			ipc_port_request_index_t request;
 
 			if (!space->is_active) {
+				printf("ipc_kmsg_copyout_header: dead space\n");
 				is_write_unlock(space);
 				return (MACH_RCV_HEADER_ERROR|
 					MACH_MSG_IPC_SPACE);
@@ -2103,6 +2177,7 @@ ipc_kmsg_copyout_header(
 				notify_port = ipc_port_lookup_notify(space,
 								     notify);
 				if (notify_port == IP_NULL) {
+					printf("ipc_kmsg_copyout_header: no notify port\n");
 					is_write_unlock(space);
 					return MACH_RCV_INVALID_NOTIFY;
 				}
@@ -2157,12 +2232,15 @@ ipc_kmsg_copyout_header(
 				if (kr != KERN_SUCCESS) {
 					/* space is unlocked */
 
-					if (kr == KERN_RESOURCE_SHORTAGE)
+					if (kr == KERN_RESOURCE_SHORTAGE) {
+						printf("ipc_kmsg_copyout_header: can't grow kernel ipc space\n");
 						return (MACH_RCV_HEADER_ERROR|
 							MACH_MSG_IPC_KERNEL);
-					else
+					} else {
+						printf("ipc_kmsg_copyout_header: can't grow user ipc space\n");
 						return (MACH_RCV_HEADER_ERROR|
 							MACH_MSG_IPC_SPACE);
+					}
 				}
 				/* space is locked again; start over */
 
@@ -2200,9 +2278,11 @@ ipc_kmsg_copyout_header(
 
 				kr = ipc_port_dngrow(reply, ITS_SIZE_NONE);
 				/* port is unlocked */
-				if (kr != KERN_SUCCESS)
+				if (kr != KERN_SUCCESS) {
+					printf("ipc_kmsg_copyout_header: can't grow kernel ipc space2\n");
 					return (MACH_RCV_HEADER_ERROR|
 						MACH_MSG_IPC_KERNEL);
+				}
 
 				is_write_lock(space);
 				continue;
@@ -2239,6 +2319,7 @@ ipc_kmsg_copyout_header(
 
 		is_read_lock(space);
 		if (!space->is_active) {
+			printf("ipc_kmsg_copyout_header: dead space2\n");
 			is_read_unlock(space);
 			return MACH_RCV_HEADER_ERROR|MACH_MSG_IPC_SPACE;
 		}
@@ -2249,11 +2330,13 @@ ipc_kmsg_copyout_header(
 			/* must check notify even though it won't be used */
 
 			if ((entry = ipc_entry_lookup(space, notify)) == IE_NULL) {
+				printf("ipc_kmsg_copyout_header: ipc_entry_lookup failed\n");
 				is_read_unlock(space);
 				return MACH_RCV_INVALID_NOTIFY;
 			}
 	
 			if ((entry->ie_bits & MACH_PORT_TYPE_RECEIVE) == 0) {
+				printf("ipc_kmsg_copyout_header: MACH_PORT_TYPE_RECEIVE not set!\n");
 				is_read_unlock(space);
 				return MACH_RCV_INVALID_NOTIFY;
 			}
@@ -2835,8 +2918,10 @@ ipc_kmsg_copyout(
 	mach_msg_return_t mr;
 
 	mr = ipc_kmsg_copyout_header(kmsg->ikm_header, space, notify);
-	if (mr != MACH_MSG_SUCCESS)
+	if (mr != MACH_MSG_SUCCESS) {
+		printf("ipc_kmsg_copyout: ipc_kmsg_copyout_header failed: %d\n", mr);
 		return mr;
+	}
 
 	if (kmsg->ikm_header->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
 		mr = ipc_kmsg_copyout_body(kmsg, space, map, slist);
@@ -3320,7 +3405,7 @@ ipc_msg_print(
 	}
 
 	if (msgh->msgh_local_port) {
-		printf("%slocal=0x%x(", needs_comma ? "," : "",
+		printf("%slocal=%p(", needs_comma ? "," : "",
 		       msgh->msgh_local_port);
 		ipc_print_type_name(MACH_MSGH_BITS_LOCAL(msgh->msgh_bits));
 		printf(")\n");
diff --git a/osfmk/ipc/ipc_kmsg.h b/osfmk/ipc/ipc_kmsg.h
index dc58188a8..0c07ed8ea 100644
--- a/osfmk/ipc/ipc_kmsg.h
+++ b/osfmk/ipc/ipc_kmsg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -80,12 +93,14 @@
  *	of the message.
  */
 
+struct ipc_labelh;
 
 struct ipc_kmsg {
 	struct ipc_kmsg *ikm_next;
 	struct ipc_kmsg *ikm_prev;
 	ipc_port_t ikm_prealloc;	/* port we were preallocated from */
 	mach_msg_size_t ikm_size;
+	struct ipc_labelh *ikm_sender;
 	mach_msg_header_t *ikm_header;
 };
 
@@ -127,11 +142,11 @@ MACRO_BEGIN								\
 	ip_release(port);						\
 MACRO_END
 
-
 #define	ikm_init(kmsg, size)						\
 MACRO_BEGIN								\
 	(kmsg)->ikm_size = (size);					\
 	(kmsg)->ikm_prealloc = IP_NULL;					\
+        (kmsg)->ikm_sender = NULL;					\
 	assert((kmsg)->ikm_prev = (kmsg)->ikm_next = IKM_BOGUS);	\
 MACRO_END
 
@@ -389,3 +404,4 @@ extern void ipc_kmsg_free_scatter(
         mach_msg_size_t		slist_size);
 
 #endif	/* _IPC_IPC_KMSG_H_ */
+
diff --git a/osfmk/ipc/ipc_labelh.c b/osfmk/ipc/ipc_labelh.c
new file mode 100644
index 000000000..e6763aa36
--- /dev/null
+++ b/osfmk/ipc/ipc_labelh.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ipc/ipc_space.h>
+#include <ipc/ipc_port.h>
+#include <ipc/ipc_labelh.h>
+#include <kern/ipc_kobject.h>
+#include <mach/security.h>
+#include <security/mac_mach_internal.h>
+
+#if CONFIG_MACF_MACH
+zone_t ipc_labelh_zone;
+
+/*
+ * Create a new label handle in the task described by the specified space.
+ * The specified label is used in the label handle.  The associated port
+ * name is copied out to namep and the task is granted send and receive rights.
+ */
+kern_return_t
+labelh_new_user(ipc_space_t space, struct label *inl, mach_port_name_t *namep)
+{
+	kern_return_t kr;
+	ipc_labelh_t lh;
+	ipc_entry_t entry;
+	ipc_port_t port;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return (KERN_INVALID_TASK);
+
+	/* XXX - perform entrypoint check here? */
+
+	/*
+	 * Note: the calling task will have a receive right for the port.
+	 * This is different from label handles that reference tasks
+	 * where the kernel holds the receive right and the caller only
+	 * gets a send right.
+	 */
+	kr = ipc_port_alloc(space, namep, &port);
+	if (kr != KERN_SUCCESS)
+		return (kr);
+	ip_reference(port);	/* ipc_port_alloc() does not add a reference */
+
+	/* Convert right to MACH_PORT_TYPE_SEND_RECEIVE */
+	port->ip_mscount++;
+	port->ip_srights++;
+	is_write_lock(space);
+	entry = ipc_entry_lookup(space, *namep);
+	if (entry != IE_NULL)
+		entry->ie_bits |= MACH_PORT_TYPE_SEND;
+	is_write_unlock(space);
+
+	/* Allocate new label handle, insert port and label. */
+	lh = (ipc_labelh_t)zalloc(ipc_labelh_zone);
+	lh_lock_init(lh);
+	lh->lh_port = port;
+	lh->lh_label = *inl;
+	lh->lh_type = LABELH_TYPE_USER;
+	lh->lh_references = 1;		/* unused for LABELH_TYPE_USER */
+
+	/* Must call ipc_kobject_set() with port unlocked. */
+	ip_unlock(lh->lh_port);
+	ipc_kobject_set(lh->lh_port, (ipc_kobject_t)lh, IKOT_LABELH);
+
+	return (KERN_SUCCESS);
+}
+
+kern_return_t
+mac_label_new(ipc_space_t space, mach_port_name_t *namep, labelstr_t labelstr)
+{
+	struct label inl;
+	kern_return_t kr;
+
+	mac_task_label_init(&inl);
+	if (mac_task_label_internalize(&inl, labelstr))
+		return (KERN_INVALID_ARGUMENT);
+
+	kr = labelh_new_user(space, &inl, namep);
+	if (kr != KERN_SUCCESS) {
+		mac_task_label_destroy(&inl);
+		return (kr);
+	}
+
+	return (KERN_SUCCESS);
+}
+
+/*
+ * This function should be used to allocate label handles
+ * that are stored in other kernel objects, such as tasks.
+ * They must be released along with that object.
+ * The caller gets one reference, which can be applied to either the
+ * port or the ipc_label_t structure itself.
+ */
+ipc_labelh_t
+labelh_new(int canblock)
+{
+	ipc_labelh_t lh;
+
+	lh = (ipc_labelh_t)zalloc_canblock(ipc_labelh_zone, canblock);
+	lh_lock_init(lh);
+	lh->lh_port = ipc_port_alloc_kernel();
+	lh->lh_type = LABELH_TYPE_KERN;
+	lh->lh_references = 1;
+	ipc_kobject_set(lh->lh_port, (ipc_kobject_t)lh, IKOT_LABELH);
+
+	return (lh);
+}
+
+/*
+ * Call with old label handle locked.
+ * Returned label handle is unlocked.
+ */
+ipc_labelh_t
+labelh_duplicate(ipc_labelh_t old)
+{
+	ipc_labelh_t lh;
+
+	lh = labelh_new(0);
+	ip_lock(lh->lh_port);
+	mac_task_label_init(&lh->lh_label);
+	mac_task_label_copy(&old->lh_label, &lh->lh_label);
+	ip_unlock(lh->lh_port);
+	return (lh);
+}
+
+/*
+ * Call with old label handle locked.
+ * Returned label handle is locked.
+ */
+ipc_labelh_t
+labelh_modify(ipc_labelh_t old)
+{
+	ipc_labelh_t lh;
+
+	/*
+	 * A label handle may only have a single reference. 
+	 * If there are no other references this is a no-op.
+	 * Otherwise, make a copy we can write to and return it.
+	 */
+	if (old->lh_references == 1)
+		return (old);
+	lh = labelh_duplicate(old);
+	lh_release(old);
+	lh_check_unlock(old);
+	lh_lock(lh);
+	return (lh);
+}
+
+/*
+ * Add or drop a reference on an (unlocked) label handle.
+ */
+ipc_labelh_t
+labelh_reference(ipc_labelh_t lh)
+{
+	lh_lock(lh);
+	lh_reference(lh);
+	lh_unlock(lh);
+	return (lh);
+}
+
+/*
+ * Release a reference on an (unlocked) label handle.
+ */
+void
+labelh_release(ipc_labelh_t lh)
+{
+	lh_lock(lh);
+	lh_release(lh);
+	lh_check_unlock(lh);
+}
+
+/*
+ * Deallocate space associated with the label handle backed by the
+ * specified port.  For kernel-allocated label handles the
+ * label handle reference count should be 0.  For user-allocated
+ * handles the ref count is not used (it was initialized to 1).
+ */
+void
+labelh_destroy(ipc_port_t port)
+{
+	ipc_labelh_t lh = (ipc_labelh_t) port->ip_kobject;
+
+	mac_task_label_destroy(&lh->lh_label);
+	zfree(ipc_labelh_zone, (vm_offset_t)lh);
+}
+#else
+kern_return_t
+mac_label_new(__unused ipc_space_t space, 
+	      __unused mach_port_name_t *namep,
+	      __unused labelstr_t labelstr)
+{
+	return KERN_FAILURE;
+}
+#endif /* MAC_MACH */
diff --git a/osfmk/ipc/ipc_labelh.h b/osfmk/ipc/ipc_labelh.h
new file mode 100644
index 000000000..b9ca1f5bd
--- /dev/null
+++ b/osfmk/ipc/ipc_labelh.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _IPC_LABELH_H_
+#define _IPC_LABELH_H_
+
+#include <kern/lock.h>
+#include <ipc/ipc_port.h>
+#include <security/_label.h>
+
+#if CONFIG_MACF_MACH
+typedef struct ipc_labelh
+{
+	natural_t         lh_references;
+	int               lh_type;
+	struct label      lh_label;
+	ipc_port_t        lh_port;
+	decl_mutex_data(,	lh_lock_data)
+} *ipc_labelh_t;
+
+#define	LABELH_TYPE_KERN	0
+#define	LABELH_TYPE_USER	1
+
+void labelh_destroy(ipc_port_t port);
+ipc_labelh_t labelh_duplicate(ipc_labelh_t old);
+ipc_labelh_t labelh_modify(ipc_labelh_t old);
+ipc_labelh_t labelh_new(int canblock);
+kern_return_t labelh_new_user(ipc_space_t, struct label *, mach_port_name_t *);
+void labelh_release(ipc_labelh_t lh);
+ipc_labelh_t labelh_reference(ipc_labelh_t lh);
+
+#define lh_reference(lh)	((lh)->lh_references++)
+#define lh_release(lh)						\
+MACRO_BEGIN							\
+	assert((lh)->lh_references > 0);			\
+	(lh)->lh_references--;					\
+MACRO_END
+
+extern zone_t ipc_labelh_zone;
+
+#define lh_lock_init(lh)	mutex_init(&(lh)->lh_lock_data, 0)
+#define lh_lock(lh)		mutex_lock(&(lh)->lh_lock_data)
+#define lh_unlock(lh)		mutex_unlock(&(lh)->lh_lock_data)
+
+/*
+ * Check the number of references the label handle has left.
+ * If there are 0 references and this is a kernel-allocated
+ * label handle, deallocate the associated port.  The
+ * storage space for the label handle will be deallocated
+ * as part of the port destruction.  User-allocated label
+ * handles are destroyed along with their ports.
+ */
+#define lh_check_unlock(lh)					\
+MACRO_BEGIN							\
+	_VOLATILE_ natural_t _refs = (lh)->lh_references;	\
+								\
+	lh_unlock(lh);						\
+	if (_refs == 0 && (lh)->lh_type == LABELH_TYPE_KERN)	\
+		ipc_port_dealloc_kernel((lh)->lh_port);		\
+MACRO_END
+
+#endif /* MAC_MACH */
+#endif /* _IPC_LABELH_H_ */
diff --git a/osfmk/ipc/ipc_machdep.h b/osfmk/ipc/ipc_machdep.h
index f528d8913..52e9533e1 100644
--- a/osfmk/ipc/ipc_machdep.h
+++ b/osfmk/ipc/ipc_machdep.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_mqueue.c b/osfmk/ipc/ipc_mqueue.c
index 9150acbc0..66c3db6f1 100644
--- a/osfmk/ipc/ipc_mqueue.c
+++ b/osfmk/ipc/ipc_mqueue.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -56,6 +62,13 @@
  *
  *	Functions to manipulate IPC message queues.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2006 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
+    
 
 #include <mach/port.h>
 #include <mach/message.h>
@@ -79,6 +92,10 @@
 
 #include <ddb/tr.h>
 
+#if CONFIG_MACF_MACH
+#include <security/mac_mach_internal.h>
+#endif
+
 int ipc_mqueue_full;		/* address is event for queue space */
 int ipc_mqueue_rcv;		/* address is event for message arrival */
 
@@ -539,7 +556,6 @@ ipc_mqueue_receive_results(wait_result_t saved_wait_result)
 {
 	thread_t     		self = current_thread();
 	mach_msg_option_t	option = self->ith_option;
-	kern_return_t		mr;
 
 	/*
 	 * why did we wake up?
@@ -563,8 +579,6 @@ ipc_mqueue_receive_results(wait_result_t saved_wait_result)
 		 * We do not need to go select a message, somebody
 		 * handed us one (or a too-large indication).
 		 */
-		mr = MACH_MSG_SUCCESS;
-
 		switch (self->ith_state) {
 		case MACH_RCV_SCATTER_SMALL:
 		case MACH_RCV_TOO_LARGE:
@@ -641,9 +655,15 @@ ipc_mqueue_receive(
 	thread_t                self;
 	uint64_t				deadline;
 	spl_t                   s;
+#if CONFIG_MACF_MACH
+	ipc_labelh_t lh;
+	task_t task;
+	int rc;
+#endif
 
 	s = splsched();
 	imq_lock(mqueue);
+	self = current_thread();
 	
 	if (imq_is_set(mqueue)) {
 		wait_queue_link_t wql;
@@ -672,7 +692,7 @@ ipc_mqueue_receive(
 			if (!imq_lock_try(port_mq)) {
 				imq_unlock(mqueue);
 				splx(s);
-				delay(1);
+				mutex_pause(0);
 				s = splsched();
 				imq_lock(mqueue);
 				goto search_set; /* start again at beginning - SMP */
@@ -698,6 +718,21 @@ ipc_mqueue_receive(
 
 			ipc_mqueue_select(port_mq, option, max_size);
 			imq_unlock(port_mq);
+#if CONFIG_MACF_MACH
+			if (self->ith_kmsg != NULL &&
+			    self->ith_kmsg->ikm_sender != NULL) {
+				lh = self->ith_kmsg->ikm_sender->label;
+				task = current_task();
+				tasklabel_lock(task);
+				ip_lock(lh->lh_port);
+				rc = mac_port_check_receive(&task->maclabel,
+				    &lh->lh_label);
+				ip_unlock(lh->lh_port);
+				tasklabel_unlock(task);
+				if (rc)
+					self->ith_state = MACH_RCV_INVALID_DATA;
+			}
+#endif
 			splx(s);
 			return;
 			
@@ -712,6 +747,21 @@ ipc_mqueue_receive(
 		if (ipc_kmsg_queue_first(kmsgs) != IKM_NULL) {
 			ipc_mqueue_select(mqueue, option, max_size);
 			imq_unlock(mqueue);
+#if CONFIG_MACF_MACH
+			if (self->ith_kmsg != NULL &&
+			    self->ith_kmsg->ikm_sender != NULL) {
+				lh = self->ith_kmsg->ikm_sender->label;
+				task = current_task();
+				tasklabel_lock(task);
+				ip_lock(lh->lh_port);
+				rc = mac_port_check_receive(&task->maclabel,
+				    &lh->lh_label);
+				ip_unlock(lh->lh_port);
+				tasklabel_unlock(task);
+				if (rc)
+					self->ith_state = MACH_RCV_INVALID_DATA;
+			}
+#endif
 			splx(s);
 			return;
 		}
@@ -722,7 +772,6 @@ ipc_mqueue_receive(
 	 * block on (whether the set's or the local port's) is
 	 * still locked.
 	 */
-	self = current_thread();
 	if (option & MACH_RCV_TIMEOUT) {
 		if (rcv_timeout == 0) {
 			imq_unlock(mqueue);
diff --git a/osfmk/ipc/ipc_mqueue.h b/osfmk/ipc/ipc_mqueue.h
index 2670f5c5c..3993c5d05 100644
--- a/osfmk/ipc/ipc_mqueue.h
+++ b/osfmk/ipc/ipc_mqueue.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_notify.c b/osfmk/ipc/ipc_notify.c
index c445269b6..bbffba8bc 100644
--- a/osfmk/ipc/ipc_notify.c
+++ b/osfmk/ipc/ipc_notify.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -82,7 +88,7 @@ ipc_notify_port_deleted(
 
 	kr = mach_notify_port_deleted(port, name);
 	if (kr != KERN_SUCCESS) {
-		printf("dropped port-deleted (0x%08x, 0x%x)\n", port, name);
+		printf("dropped port-deleted (%p, 0x%x)\n", port, name);
 		ipc_port_release_sonce(port);
 	}
 }
@@ -108,7 +114,7 @@ ipc_notify_port_destroyed(
 
 	kr = mach_notify_port_destroyed(port, right);
 	if (kr != KERN_SUCCESS) {
-		printf("dropped port-destroyed (0x%08x, 0x%08x)\n",
+		printf("dropped port-destroyed (%p, %p)\n",
 		       port, right);
 		ipc_port_release_sonce(port);
 		ipc_port_release_receive(right);
@@ -133,7 +139,7 @@ ipc_notify_no_senders(
 
 	kr = mach_notify_no_senders(port, mscount);
 	if (kr != KERN_SUCCESS) {
-		printf("dropped no-senders (0x%08x, %u)\n", port, mscount);
+		printf("dropped no-senders (%p, %u)\n", port, mscount);
 		ipc_port_release_sonce(port);
 	}
 }
@@ -155,7 +161,7 @@ ipc_notify_send_once(
 
 	kr = mach_notify_send_once(port);
 	if (kr != KERN_SUCCESS) {
-		printf("dropped send-once (0x%08x)\n", port);
+		printf("dropped send-once (%p)\n", port);
 		ipc_port_release_sonce(port);
 	}
 }
@@ -178,7 +184,7 @@ ipc_notify_dead_name(
 
 	kr = mach_notify_dead_name(port, name);
 	if (kr != KERN_SUCCESS) {
-		printf("dropped dead-name (0x%08x, 0x%x)\n", port, name);
+		printf("dropped dead-name (%p, 0x%x)\n", port, name);
 		ipc_port_release_sonce(port);
 	}
 }
diff --git a/osfmk/ipc/ipc_notify.h b/osfmk/ipc/ipc_notify.h
index a4064e0c9..a2da35065 100644
--- a/osfmk/ipc/ipc_notify.h
+++ b/osfmk/ipc/ipc_notify.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_object.c b/osfmk/ipc/ipc_object.c
index c5b03d101..0ca7095ee 100644
--- a/osfmk/ipc/ipc_object.c
+++ b/osfmk/ipc/ipc_object.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -67,6 +80,7 @@
 
 #include <kern/kern_types.h>
 #include <kern/misc_protos.h>
+#include <kern/ipc_kobject.h>
 
 #include <ipc/ipc_types.h>
 #include <ipc/port.h>
@@ -77,6 +91,9 @@
 #include <ipc/ipc_right.h>
 #include <ipc/ipc_notify.h>
 #include <ipc/ipc_pset.h>
+#include <ipc/ipc_labelh.h>
+
+#include <security/mac_mach_internal.h>
 
 zone_t ipc_object_zones[IOT_NUMBER];
 
@@ -328,6 +345,9 @@ ipc_object_alloc(
 		ipc_port_t port = (ipc_port_t)object;
 
 		bzero((char *)port, sizeof(*port));
+#if CONFIG_MACF_MACH
+		mac_port_label_init(&port->ip_label);
+#endif
 	} else if (otype == IOT_PORT_SET) {
 		ipc_pset_t pset = (ipc_pset_t)object;
 
@@ -396,6 +416,9 @@ ipc_object_alloc_name(
 		ipc_port_t port = (ipc_port_t)object;
 
 		bzero((char *)port, sizeof(*port));
+#if CONFIG_MACF_MACH
+		mac_port_label_init(&port->ip_label);
+#endif
 	} else if (otype == IOT_PORT_SET) {
 		ipc_pset_t pset = (ipc_pset_t)object;
 
@@ -972,7 +995,31 @@ ipc_object_rename(
 	return kr;
 }
 
-#if	MACH_ASSERT
+/*
+ * Get a label out of a port, to be used by a kernel call
+ * that takes a security label as a parameter. In this case, we want
+ * to use the label stored in the label handle and not the label on its
+ * port.
+ *
+ * The port should be locked for this call. The lock protecting
+ * label handle contents should not be necessary, as they can only
+ * be modified when a label handle with one reference is a task label.
+ * User allocated label handles can never be modified.
+ */
+#if CONFIG_MACF_MACH
+struct label *io_getlabel (ipc_object_t objp)
+{
+	ipc_port_t port = (ipc_port_t)objp;
+
+	assert(io_otype(objp) == IOT_PORT);
+
+	if (ip_kotype(port) == IKOT_LABELH)
+		return &((ipc_labelh_t) port->ip_kobject)->lh_label;
+	else
+		return &port->ip_label;
+}
+#endif
+#if MACH_ASSERT || CONFIG_MACF_MACH
 /*
  *	Check whether the object is a port if so, free it.  But
  *	keep track of that fact.
@@ -989,16 +1036,20 @@ io_free(
 #if	MACH_ASSERT
 		ipc_port_track_dealloc(port);
 #endif	/* MACH_ASSERT */
+
+#if CONFIG_MACF_MACH
+		/* Port label should have been initialized after creation. */
+		mac_port_label_destroy(&port->ip_label);
+#endif	  
 	}
 	zfree(ipc_object_zones[otype], object);
 }
-#endif	/* MACH_ASSERT */
+#endif		/* MACH_ASSER || MAC */
 
 #include <mach_kdb.h>
 #if	MACH_KDB
 
 #include <ddb/db_output.h>
-#include <kern/ipc_kobject.h>
 
 #define	printf	kdbprintf 
 
@@ -1042,6 +1093,9 @@ const char *ikot_print_array[IKOT_MAX_TYPE] = {
 	"(IOKIT_OBJECT)     ",	/* 30 */
 	"(UPL)              ",
 	"(MEM_OBJ_CONTROL)  ",
+#if CONFIG_MACF_MACH
+	"(LABELH)           ",
+#endif
 /*
  * Add new entries here.
  * Please keep in sync with kern/ipc_kobject.h
diff --git a/osfmk/ipc/ipc_object.h b/osfmk/ipc/ipc_object.h
index 1eb392c29..a3863ffa8 100644
--- a/osfmk/ipc/ipc_object.h
+++ b/osfmk/ipc/ipc_object.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -135,7 +147,7 @@ extern zone_t ipc_object_zones[IOT_NUMBER];
 #define	io_alloc(otype)		\
 		((ipc_object_t) zalloc(ipc_object_zones[(otype)]))
 
-#if	MACH_ASSERT
+#if MACH_ASSERT || CONFIG_MACF_MACH
 /*
  *	Call the routine for io_free so that checking can be performed.
  */
@@ -143,10 +155,10 @@ extern void	io_free(
 			unsigned int	otype,
 			ipc_object_t	object);
 
-#else	/* MACH_ASSERT */
-#define	io_free(otype, io)	\
-		zfree(ipc_object_zones[(otype)], (io))
-#endif	/* MACH_ASSERT */
+#else	/* MACH_ASSERT || MAC_MACH */
+#define io_free(otype, io)  \
+	zfree(ipc_object_zones[(otype)], (io))
+#endif	/* MACH_ASSERT || MAC_MACH */
 
 /*
  * Here we depend on the ipc_object being first within the ipc_common_data,
@@ -199,6 +211,16 @@ MACRO_BEGIN								\
 	(io)->io_references--;						\
 MACRO_END
 
+/*   
+ * Retrieve a label for use in a kernel call that takes a security
+ * label as a parameter. If necessary, io_getlabel acquires internal
+ * (not io_lock) locks, and io_unlocklabel releases them.
+ */
+
+struct label;
+extern struct label *io_getlabel (ipc_object_t obj);
+#define io_unlocklabel(obj)
+
 /*
  * Exported interfaces
  */
diff --git a/osfmk/ipc/ipc_port.c b/osfmk/ipc/ipc_port.c
index 99a43f5bf..b531befff 100644
--- a/osfmk/ipc/ipc_port.c
+++ b/osfmk/ipc/ipc_port.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -80,6 +92,8 @@
 #include <ipc/ipc_print.h>
 #include <ipc/ipc_table.h>
 
+#include <security/mac_mach_internal.h>
+
 #if	MACH_KDB
 #include <machine/db_machdep.h>
 #include <ddb/db_command.h>
@@ -499,6 +513,14 @@ ipc_port_alloc(
 
 	ipc_port_init(port, space, name);
 
+#if CONFIG_MACF_MACH
+	task_t issuer = current_task();
+	tasklabel_lock2 (issuer, space->is_task);
+	mac_port_label_associate(&issuer->maclabel, &space->is_task->maclabel,
+			 &port->ip_label);
+	tasklabel_unlock2 (issuer, space->is_task);
+#endif
+
 	*namep = name;
 	*portp = port;
 
@@ -538,6 +560,14 @@ ipc_port_alloc_name(
 
 	ipc_port_init(port, space, name);
 
+#if CONFIG_MACF_MACH
+	task_t issuer = current_task();
+	tasklabel_lock2 (issuer, space->is_task);
+	mac_port_label_associate(&issuer->maclabel, &space->is_task->maclabel,
+			 &port->ip_label);
+	tasklabel_unlock2 (issuer, space->is_task);
+#endif
+
 	*portp = port;
 
 	return KERN_SUCCESS;
@@ -1117,6 +1147,18 @@ ipc_port_alloc_special(
 
 	ipc_port_init(port, space, 1);
 
+#if CONFIG_MACF_MACH
+	/* Currently, ipc_port_alloc_special is used for two things:
+	 * - Reply ports for messages from the kernel
+	 * - Ports for communication with the kernel (e.g. task ports)
+	 * Since both of these would typically be labelled as kernel objects,
+	 * we will use a new entry point for this purpose, as current_task()
+	 * is often wrong (i.e. not kernel_task) or null.
+	 */
+	mac_port_label_init(&port->ip_label);
+	mac_port_label_associate_kernel(&port->ip_label, space == ipc_space_reply);
+#endif
+
 	return port;
 }
 
diff --git a/osfmk/ipc/ipc_port.h b/osfmk/ipc/ipc_port.h
index 587f14654..6b409dc2d 100644
--- a/osfmk/ipc/ipc_port.h
+++ b/osfmk/ipc/ipc_port.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -77,6 +89,8 @@
 #include <ipc/ipc_mqueue.h>
 #include <ipc/ipc_space.h>
 
+#include <security/_label.h>
+
 /*
  *  A receive right (port) can be in four states:
  *	1) dead (not active, ip_timestamp has death time)
@@ -141,6 +155,10 @@ struct ipc_port {
 	unsigned long	ip_spares[IP_NSPARES]; /* for debugging */
 #endif	/* MACH_ASSERT */
 	int		alias;
+
+//#if MAC
+        struct label    ip_label;
+//#endif
 };
 
 
diff --git a/osfmk/ipc/ipc_print.h b/osfmk/ipc/ipc_print.h
index 8b4277f1d..8365a1ee4 100644
--- a/osfmk/ipc/ipc_print.h
+++ b/osfmk/ipc/ipc_print.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_pset.c b/osfmk/ipc/ipc_pset.c
index d2983415a..bf805af55 100644
--- a/osfmk/ipc/ipc_pset.c
+++ b/osfmk/ipc/ipc_pset.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_pset.h b/osfmk/ipc/ipc_pset.h
index 0ee2afc84..c575a71ad 100644
--- a/osfmk/ipc/ipc_pset.h
+++ b/osfmk/ipc/ipc_pset.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_right.c b/osfmk/ipc/ipc_right.c
index 9f228a43a..1d9d16596 100644
--- a/osfmk/ipc/ipc_right.c
+++ b/osfmk/ipc/ipc_right.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -73,6 +86,7 @@
 #include <ipc/ipc_right.h>
 #include <ipc/ipc_notify.h>
 #include <ipc/ipc_table.h>
+#include <security/mac_mach_internal.h>
 
 /*
  *	Routine:	ipc_right_lookup_write
@@ -1316,23 +1330,66 @@ ipc_right_copyin_check(
 	mach_msg_type_name_t		msgt_name)
 {
 	ipc_entry_bits_t bits;
+	ipc_port_t port;
+#if CONFIG_MACF_MACH
+	task_t self = current_task();
+	int rc = 0;
+#endif
 
 	bits= entry->ie_bits;
 	assert(space->is_active);
 
 	switch (msgt_name) {
 	    case MACH_MSG_TYPE_MAKE_SEND:
+		if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
+			return FALSE;
+
+#if CONFIG_MACF_MACH
+		port = (ipc_port_t) entry->ie_object;
+		ip_lock(port);
+		tasklabel_lock(self);
+		rc = mac_port_check_make_send(&self->maclabel, &port->ip_label);                tasklabel_unlock(self);
+		ip_unlock(port);
+		if (rc)
+			return FALSE;
+#endif
+		break;
+
 	    case MACH_MSG_TYPE_MAKE_SEND_ONCE:
+		if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
+			return FALSE;
+
+#if CONFIG_MACF_MACH
+		port = (ipc_port_t) entry->ie_object;
+		ip_lock(port);
+		tasklabel_lock(self);
+		rc = mac_port_check_make_send_once(&self->maclabel, &port->ip_label);
+		tasklabel_unlock(self);
+		ip_unlock(port);
+		if (rc)
+			return FALSE;
+#endif
+		break;
+
 	    case MACH_MSG_TYPE_MOVE_RECEIVE:
 		if ((bits & MACH_PORT_TYPE_RECEIVE) == 0)
 			return FALSE;
 
+#if CONFIG_MACF_MACH
+		port = (ipc_port_t) entry->ie_object;
+		ip_lock(port);
+		tasklabel_lock(self);
+		rc = mac_port_check_move_receive(&self->maclabel, &port->ip_label);
+		tasklabel_unlock(self);
+		ip_unlock(port);
+		if (rc)
+                        return FALSE;
+#endif
 		break;
 
 	    case MACH_MSG_TYPE_COPY_SEND:
 	    case MACH_MSG_TYPE_MOVE_SEND:
 	    case MACH_MSG_TYPE_MOVE_SEND_ONCE: {
-		ipc_port_t port;
 		boolean_t active;
 
 		if (bits & MACH_PORT_TYPE_DEAD_NAME)
@@ -1346,6 +1403,30 @@ ipc_right_copyin_check(
 
 		ip_lock(port);
 		active = ip_active(port);
+#if CONFIG_MACF_MACH
+		tasklabel_lock(self);
+		switch (msgt_name) {
+		case MACH_MSG_TYPE_COPY_SEND:
+			rc = mac_port_check_copy_send(&self->maclabel,
+			    &port->ip_label);
+			break;
+		case MACH_MSG_TYPE_MOVE_SEND:
+			rc = mac_port_check_move_send(&self->maclabel,
+			    &port->ip_label);
+			break;
+		case MACH_MSG_TYPE_MOVE_SEND_ONCE:
+			rc = mac_port_check_move_send_once(&self->maclabel,
+			    &port->ip_label);
+			break;
+		default:
+			panic("ipc_right_copyin_check: strange rights");
+		}
+		tasklabel_unlock(self);
+		if (rc) {
+			ip_unlock(port);
+			return FALSE;
+		}
+#endif
 		ip_unlock(port);
 
 		if (!active) {
@@ -1404,6 +1485,10 @@ ipc_right_copyin(
 	ipc_port_t		*sorightp)
 {
 	ipc_entry_bits_t bits;
+#if CONFIG_MACF_MACH
+	task_t self = current_task();
+	int    rc;
+#endif
 	
 	bits = entry->ie_bits;
 
@@ -1424,6 +1509,16 @@ ipc_right_copyin(
 		assert(port->ip_receiver_name == name);
 		assert(port->ip_receiver == space);
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock(self);
+		rc = mac_port_check_make_send(&self->maclabel, &port->ip_label);
+		tasklabel_unlock(self);
+		if (rc) {
+			ip_unlock(port);
+			return KERN_NO_ACCESS;
+		}
+#endif
+
 		port->ip_mscount++;
 		port->ip_srights++;
 		ip_reference(port);
@@ -1448,6 +1543,16 @@ ipc_right_copyin(
 		assert(port->ip_receiver_name == name);
 		assert(port->ip_receiver == space);
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock(self);
+		rc = mac_port_check_make_send_once(&self->maclabel, &port->ip_label);
+		tasklabel_unlock(self);
+		if (rc) {
+			ip_unlock(port);
+			return KERN_NO_ACCESS;
+		}
+#endif
+
 		port->ip_sorights++;
 		ip_reference(port);
 		ip_unlock(port);
@@ -1472,6 +1577,17 @@ ipc_right_copyin(
 		assert(port->ip_receiver_name == name);
 		assert(port->ip_receiver == space);
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock(self);
+		rc = mac_port_check_move_receive(&self->maclabel,
+						 &port->ip_label);
+		tasklabel_unlock(self);
+		if (rc) {
+			ip_unlock(port);
+			return KERN_NO_ACCESS;
+		}
+#endif
+
 		if (bits & MACH_PORT_TYPE_SEND) {
 			assert(IE_BITS_TYPE(bits) ==
 					MACH_PORT_TYPE_SEND_RECEIVE);
@@ -1524,6 +1640,16 @@ ipc_right_copyin(
 		}
 		/* port is locked and active */
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock(self);
+		rc = mac_port_check_copy_send(&self->maclabel, &port->ip_label);
+		tasklabel_unlock(self);
+		if (rc) {
+			ip_unlock(port);
+			return KERN_NO_ACCESS;
+		}
+#endif
+
 		if ((bits & MACH_PORT_TYPE_SEND) == 0) {
 			assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
 			assert(port->ip_sorights > 0);
@@ -1566,6 +1692,17 @@ ipc_right_copyin(
 		}
 		/* port is locked and active */
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock (self);
+		rc = mac_port_check_copy_send (&self->maclabel, &port->ip_label);
+		tasklabel_unlock (self);
+		if (rc)
+		  {
+		    ip_unlock (port);
+		    return KERN_NO_ACCESS;
+		  }
+#endif
+
 		if ((bits & MACH_PORT_TYPE_SEND) == 0) {
 			assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND_ONCE);
 			assert(port->ip_sorights > 0);
@@ -1632,6 +1769,17 @@ ipc_right_copyin(
 		}
 		/* port is locked and active */
 
+#if CONFIG_MACF_MACH
+		tasklabel_lock (self);
+		rc = mac_port_check_copy_send (&self->maclabel, &port->ip_label);
+		tasklabel_unlock (self);
+		if (rc)
+		  {
+		    ip_unlock (port);
+		    return KERN_NO_ACCESS;
+		  }
+#endif
+
 		if ((bits & MACH_PORT_TYPE_SEND_ONCE) == 0) {
 			assert(bits & MACH_PORT_TYPE_SEND);
 			assert(port->ip_srights > 0);
@@ -1803,6 +1951,10 @@ ipc_right_copyin_two(
 	mach_port_urefs_t urefs;
 	ipc_port_t port;
 	ipc_port_t dnrequest = IP_NULL;
+#if CONFIG_MACF_MACH
+	task_t self = current_task();
+	int    rc;
+#endif
 
 	assert(space->is_active);
 
@@ -1823,6 +1975,16 @@ ipc_right_copyin_two(
 	}
 	/* port is locked and active */
 
+#if CONFIG_MACF_MACH
+	tasklabel_lock(self);
+	rc = mac_port_check_copy_send(&self->maclabel, &port->ip_label);
+	tasklabel_unlock(self);
+	if (rc) {
+		ip_unlock(port);
+		return KERN_NO_ACCESS;
+	}
+#endif
+
 	assert(port->ip_srights > 0);
 
 	if (urefs == 2) {
@@ -1899,6 +2061,9 @@ ipc_right_copyout(
 {
 	ipc_entry_bits_t bits;
 	ipc_port_t port;
+#if CONFIG_MACF_MACH
+	int rc;
+#endif
 
 	bits = entry->ie_bits;
 
@@ -1915,6 +2080,19 @@ ipc_right_copyout(
 		assert(IE_BITS_TYPE(bits) == MACH_PORT_TYPE_NONE);
 		assert(port->ip_sorights > 0);
 
+#if CONFIG_MACF_MACH
+		if (space->is_task) {
+			tasklabel_lock(space->is_task);
+			rc = mac_port_check_hold_send_once(&space->is_task->maclabel,
+							   &port->ip_label);
+			tasklabel_unlock(space->is_task);
+
+			if (rc) {
+				ip_unlock(port);
+				return KERN_NO_ACCESS;
+			}
+		}
+#endif
 		/* transfer send-once right and ref to entry */
 		ip_unlock(port);
 
@@ -1924,6 +2102,20 @@ ipc_right_copyout(
 	    case MACH_MSG_TYPE_PORT_SEND:
 		assert(port->ip_srights > 0);
 
+#if CONFIG_MACF_MACH
+		if (space->is_task) {
+			tasklabel_lock(space->is_task);
+			rc = mac_port_check_hold_send(&space->is_task->maclabel,
+						      &port->ip_label);
+			tasklabel_unlock(space->is_task);
+
+			if (rc) {
+				ip_unlock(port);
+				return KERN_NO_ACCESS;
+			}
+		}
+#endif
+
 		if (bits & MACH_PORT_TYPE_SEND) {
 			mach_port_urefs_t urefs = IE_BITS_UREFS(bits);
 
@@ -1978,6 +2170,20 @@ ipc_right_copyout(
 		assert(port->ip_receiver_name == MACH_PORT_NULL);
 		dest = port->ip_destination;
 
+#if CONFIG_MACF_MACH
+		if (space->is_task) {
+			tasklabel_lock(space->is_task);
+			rc = mac_port_check_hold_receive(&space->is_task->maclabel,
+							 &port->ip_label);
+			tasklabel_unlock(space->is_task);
+
+			if (rc) {
+				ip_unlock(port);
+				return KERN_NO_ACCESS;
+			}
+		}
+#endif
+
 		port->ip_receiver_name = name;
 		port->ip_receiver = space;
 
diff --git a/osfmk/ipc/ipc_right.h b/osfmk/ipc/ipc_right.h
index 0623ba61c..b7affe7b3 100644
--- a/osfmk/ipc/ipc_right.h
+++ b/osfmk/ipc/ipc_right.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_space.c b/osfmk/ipc/ipc_space.c
index c88fbe0de..434580250 100644
--- a/osfmk/ipc/ipc_space.c
+++ b/osfmk/ipc/ipc_space.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -171,6 +183,7 @@ ipc_space_create(
 	space->is_tree_total = 0;
 	space->is_tree_small = 0;
 	space->is_tree_hash = 0;
+	space->is_task = NULL;
 
 	*spacep = space;
 	return KERN_SUCCESS;
diff --git a/osfmk/ipc/ipc_space.h b/osfmk/ipc/ipc_space.h
index 757ff469d..792fe6894 100644
--- a/osfmk/ipc/ipc_space.h
+++ b/osfmk/ipc/ipc_space.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -69,7 +81,7 @@
 #include <sys/appleapiopts.h>
 
 #ifdef __APPLE_API_PRIVATE
-#ifdef MACH_KERNEL_PRIVATE
+#if MACH_KERNEL_PRIVATE
 #include <mach_kdb.h>
 #include <kern/macro_help.h>
 #include <kern/kern_types.h>
@@ -113,6 +125,8 @@ struct ipc_space {
 	ipc_entry_num_t is_tree_small;	/* # of small entries in the tree */
 	ipc_entry_num_t is_tree_hash;	/* # of hashed entries in the tree */
 	boolean_t is_fast;              /* for is_fast_space() */
+
+	task_t is_task;                 /* associated task */
 };
 
 #define	IS_NULL			((ipc_space_t) 0)
diff --git a/osfmk/ipc/ipc_splay.c b/osfmk/ipc/ipc_splay.c
index 06cc17444..e0fec7699 100644
--- a/osfmk/ipc/ipc_splay.c
+++ b/osfmk/ipc/ipc_splay.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_splay.h b/osfmk/ipc/ipc_splay.h
index a25350d5d..03cfe0d70 100644
--- a/osfmk/ipc/ipc_splay.h
+++ b/osfmk/ipc/ipc_splay.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_table.c b/osfmk/ipc/ipc_table.c
index 24452704a..090bc78b2 100644
--- a/osfmk/ipc/ipc_table.c
+++ b/osfmk/ipc/ipc_table.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/ipc_table.h b/osfmk/ipc/ipc_table.h
index fee1cb97d..1c5eb1bcf 100644
--- a/osfmk/ipc/ipc_table.h
+++ b/osfmk/ipc/ipc_table.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -103,7 +109,7 @@ extern ipc_table_size_t ipc_table_entries;
 extern ipc_table_size_t ipc_table_dnrequests;
 
 /* Initialize IPC capabilities table storage */
-extern void ipc_table_init(void);
+extern void ipc_table_init(void) __attribute__((section("__TEXT, initcode")));
 
 /*
  *	Note that ipc_table_alloc, ipc_table_realloc, and ipc_table_free
diff --git a/osfmk/ipc/ipc_types.h b/osfmk/ipc/ipc_types.h
index 77629a865..a7ac3475e 100644
--- a/osfmk/ipc/ipc_types.h
+++ b/osfmk/ipc/ipc_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/mach_debug.c b/osfmk/ipc/mach_debug.c
index 0d3699afa..ed8745e3f 100644
--- a/osfmk/ipc/mach_debug.c
+++ b/osfmk/ipc/mach_debug.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -159,10 +165,11 @@ host_ipc_hash_info(
 	hash_info_bucket_array_t		*infop,
 	mach_msg_type_number_t 		*countp)
 {
+	vm_map_copy_t copy;
 	vm_offset_t addr;
-	vm_size_t size = 0;
+	vm_size_t size;
 	hash_info_bucket_t *info;
-	unsigned int potential, actual;
+	natural_t count;
 	kern_return_t kr;
 
 	if (host == HOST_NULL)
@@ -170,53 +177,24 @@ host_ipc_hash_info(
 
 	/* start with in-line data */
 
-	info = *infop;
-	potential = *countp;
-
-	for (;;) {
-		actual = ipc_hash_info(info, potential);
-		if (actual <= potential)
-			break;
-
-		/* allocate more memory */
-
-		if (info != *infop)
-			kmem_free(ipc_kernel_map, addr, size);
-
-		size = round_page(actual * sizeof *info);
-		kr = kmem_alloc_pageable(ipc_kernel_map, &addr, size);
-		if (kr != KERN_SUCCESS)
-			return KERN_RESOURCE_SHORTAGE;
-
-		info = (hash_info_bucket_t *) addr;
-		potential = size/sizeof *info;
-	}
-
-	if (info == *infop) {
-		/* data fit in-line; nothing to deallocate */
-
-		*countp = actual;
-	} else if (actual == 0) {
-		kmem_free(ipc_kernel_map, addr, size);
-
-		*countp = 0;
-	} else {
-		vm_map_copy_t copy;
-		vm_size_t used;
-
-		used = round_page(actual * sizeof *info);
+	count = ipc_hash_size();
+	size = round_page(count * sizeof(hash_info_bucket_t));
+	kr = kmem_alloc_pageable(ipc_kernel_map, &addr, size);
+	if (kr != KERN_SUCCESS)
+		return KERN_RESOURCE_SHORTAGE;
 
-		if (used != size)
-			kmem_free(ipc_kernel_map, addr + used, size - used);
+	info = (hash_info_bucket_t *) addr;
+	count = ipc_hash_info(info, count);
 
-		kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr, 
-				   (vm_map_size_t)used, TRUE, &copy);
-		assert(kr == KERN_SUCCESS);
+	if (size > count * sizeof(hash_info_bucket_t))
+		bzero((char *)&info[count], size - count * sizeof(hash_info_bucket_t));
 
-		*infop = (hash_info_bucket_t *) copy;
-		*countp = actual;
-	}
+	kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)addr, 
+			   (vm_map_size_t)size, TRUE, &copy);
+	assert(kr == KERN_SUCCESS);
 
+	*infop = (hash_info_bucket_t *) copy;
+	*countp = count;
 	return KERN_SUCCESS;
 }
 #endif /* MACH_IPC_DEBUG */
@@ -249,26 +227,26 @@ mach_port_space_info(
 #else
 kern_return_t
 mach_port_space_info(
-	ipc_space_t				space,
-	ipc_info_space_t			*infop,
+	ipc_space_t			space,
+	ipc_info_space_t		*infop,
 	ipc_info_name_array_t		*tablep,
 	mach_msg_type_number_t 		*tableCntp,
-	ipc_info_tree_name_array_t		*treep,
+	ipc_info_tree_name_array_t	*treep,
 	mach_msg_type_number_t 		*treeCntp)
 {
 	ipc_info_name_t *table_info;
-	unsigned int table_potential, table_actual;
 	vm_offset_t table_addr;
-	vm_size_t table_size;
+	vm_size_t table_size, table_size_needed;
 	ipc_info_tree_name_t *tree_info;
-	unsigned int tree_potential, tree_actual;
 	vm_offset_t tree_addr;
-	vm_size_t tree_size;
+	vm_size_t tree_size, tree_size_needed;
 	ipc_tree_entry_t tentry;
 	ipc_entry_t table;
 	ipc_entry_num_t tsize;
 	mach_port_index_t index;
 	kern_return_t kr;
+	vm_map_copy_t copy;
+
 
 	if (space == IS_NULL)
 		return KERN_INVALID_TASK;
@@ -276,78 +254,58 @@ mach_port_space_info(
 	/* start with in-line memory */
 
 	table_size = 0;
-	table_info = *tablep;
-	table_potential = *tableCntp;
 	tree_size = 0;
-	tree_info = *treep;
-	tree_potential = *treeCntp;
 
 	for (;;) {
 		is_read_lock(space);
 		if (!space->is_active) {
 			is_read_unlock(space);
-			if (table_info != *tablep)
+			if (table_size != 0)
 				kmem_free(ipc_kernel_map,
 					  table_addr, table_size);
-			if (tree_info != *treep)
+			if (tree_size != 0)
 				kmem_free(ipc_kernel_map,
 					  tree_addr, tree_size);
 			return KERN_INVALID_TASK;
 		}
 
-		table_actual = space->is_table_size;
-		tree_actual = space->is_tree_total;
+		table_size_needed = round_page(space->is_table_size
+					       * sizeof(ipc_info_name_t));
+		tree_size_needed = round_page(space->is_tree_total
+					      * sizeof(ipc_info_tree_name_t));
 
-		if ((table_actual <= table_potential) &&
-		    (tree_actual <= tree_potential))
+		if ((table_size_needed == table_size) &&
+		    (tree_size_needed == tree_size))
 			break;
 
 		is_read_unlock(space);
 
-		if (table_actual > table_potential) {
-			if (table_info != *tablep)
-				kmem_free(ipc_kernel_map,
-					  table_addr, table_size);
-
-			table_size = round_page(table_actual *
-						sizeof *table_info);
-			kr = kmem_alloc(ipc_kernel_map,
-					&table_addr, table_size);
+		if (table_size != table_size_needed) {
+			if (table_size != 0)
+				kmem_free(ipc_kernel_map, table_addr, table_size);
+			kr = kmem_alloc(ipc_kernel_map,	&table_addr, table_size_needed);
 			if (kr != KERN_SUCCESS) {
-				if (tree_info != *treep)
-					kmem_free(ipc_kernel_map,
-						  tree_addr, tree_size);
-
+				if (tree_size != 0)
+					kmem_free(ipc_kernel_map, tree_addr, tree_size);
 				return KERN_RESOURCE_SHORTAGE;
 			}
-
-			table_info = (ipc_info_name_t *) table_addr;
-			table_potential = table_size/sizeof *table_info;
+			table_size = table_size_needed;
 		}
-
-		if (tree_actual > tree_potential) {
-			if (tree_info != *treep)
-				kmem_free(ipc_kernel_map,
-					  tree_addr, tree_size);
-
-			tree_size = round_page(tree_actual *
-					       sizeof *tree_info);
-			kr = kmem_alloc(ipc_kernel_map,
-					&tree_addr, tree_size);
+		if (tree_size != tree_size_needed) {
+			if (tree_size != 0)
+				kmem_free(ipc_kernel_map, tree_addr, tree_size);
+			kr = kmem_alloc(ipc_kernel_map, &tree_addr, tree_size_needed);
 			if (kr != KERN_SUCCESS) {
-				if (table_info != *tablep)
-					kmem_free(ipc_kernel_map,
-						  table_addr, table_size);
-
+				if (table_size != 0)
+					kmem_free(ipc_kernel_map, table_addr, table_size);
 				return KERN_RESOURCE_SHORTAGE;
 			}
-
-			tree_info = (ipc_info_tree_name_t *) tree_addr;
-			tree_potential = tree_size/sizeof *tree_info;
+			tree_size = tree_size_needed;
 		}
 	}
 	/* space is read-locked and active; we have enough wired memory */
 
+	/* get the overall space info */
 	infop->iis_genno_mask = MACH_PORT_NGEN(MACH_PORT_DEAD);
 	infop->iis_table_size = space->is_table_size;
 	infop->iis_table_next = space->is_table_next->its_size;
@@ -355,9 +313,10 @@ mach_port_space_info(
 	infop->iis_tree_small = space->is_tree_small;
 	infop->iis_tree_hash = space->is_tree_hash;
 
+	/* walk the table for this space */
 	table = space->is_table;
 	tsize = space->is_table_size;
-
+	table_info = (ipc_info_name_array_t)table_addr;
 	for (index = 0; index < tsize; index++) {
 		ipc_info_name_t *iin = &table_info[index];
 		ipc_entry_t entry = &table[index];
@@ -367,12 +326,16 @@ mach_port_space_info(
 		iin->iin_name = MACH_PORT_MAKE(index, IE_BITS_GEN(bits));
 		iin->iin_collision = (bits & IE_BITS_COLLISION) ? TRUE : FALSE;
 		iin->iin_type = IE_BITS_TYPE(bits);
+		if (entry->ie_request)
+			iin->iin_type |= MACH_PORT_TYPE_DNREQUEST;
 		iin->iin_urefs = IE_BITS_UREFS(bits);
 		iin->iin_object = (vm_offset_t) entry->ie_object;
 		iin->iin_next = entry->ie_next;
 		iin->iin_hash = entry->ie_index;
 	}
 
+	/* walk the splay tree for this space */
+	tree_info = (ipc_info_tree_name_array_t)tree_addr;
 	for (tentry = ipc_splay_traverse_start(&space->is_tree), index = 0;
 	     tentry != ITE_NULL;
 	     tentry = ipc_splay_traverse_next(&space->is_tree, FALSE)) {
@@ -386,6 +349,8 @@ mach_port_space_info(
 		iin->iin_name = tentry->ite_name;
 		iin->iin_collision = (bits & IE_BITS_COLLISION) ? TRUE : FALSE;
 		iin->iin_type = IE_BITS_TYPE(bits);
+		if (entry->ie_request)
+			iin->iin_type |= MACH_PORT_TYPE_DNREQUEST;
 		iin->iin_urefs = IE_BITS_UREFS(bits);
 		iin->iin_object = (vm_offset_t) entry->ie_object;
 		iin->iin_next = entry->ie_next;
@@ -405,82 +370,43 @@ mach_port_space_info(
 	ipc_splay_traverse_finish(&space->is_tree);
 	is_read_unlock(space);
 
-	if (table_info == *tablep) {
-		/* data fit in-line; nothing to deallocate */
-
-		*tableCntp = table_actual;
-	} else if (table_actual == 0) {
-		kmem_free(ipc_kernel_map, table_addr, table_size);
-
-		*tableCntp = 0;
-	} else {
-		vm_size_t size_used, rsize_used;
-		vm_map_copy_t copy;
-
-		/* kmem_alloc doesn't zero memory */
-
-		size_used = table_actual * sizeof *table_info;
-		rsize_used = round_page(size_used);
-
-		if (rsize_used != table_size)
-			kmem_free(ipc_kernel_map,
-				  table_addr + rsize_used,
-				  table_size - rsize_used);
-
-		if (size_used != rsize_used)
-			bzero((char *) (table_addr + size_used),
-			      rsize_used - size_used);
+	/* prepare the table out-of-line data for return */
+	if (table_size > 0) {
+		if (table_size > infop->iis_table_size * sizeof(ipc_info_name_t))
+			bzero((char *)&table_info[infop->iis_table_size],
+			      table_size - infop->iis_table_size * sizeof(ipc_info_name_t));
 
 		kr = vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(table_addr),
-				   vm_map_round_page(table_addr + rsize_used), FALSE);
+				   vm_map_round_page(table_addr + table_size), FALSE);
 		assert(kr == KERN_SUCCESS);
-
 		kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)table_addr, 
-				   (vm_map_size_t)rsize_used, TRUE, &copy);
+				   (vm_map_size_t)table_size, TRUE, &copy);
 		assert(kr == KERN_SUCCESS);
-
-		*tablep = (ipc_info_name_t *) copy;
-		*tableCntp = table_actual;
-	}
-
-	if (tree_info == *treep) {
-		/* data fit in-line; nothing to deallocate */
-
-		*treeCntp = tree_actual;
-	} else if (tree_actual == 0) {
-		kmem_free(ipc_kernel_map, tree_addr, tree_size);
-
-		*treeCntp = 0;
+		*tablep = (ipc_info_name_t *)copy;
+		*tableCntp = infop->iis_table_size;
 	} else {
-		vm_size_t size_used, rsize_used;
-		vm_map_copy_t copy;
-
-		/* kmem_alloc doesn't zero memory */
-
-		size_used = tree_actual * sizeof *tree_info;
-		rsize_used = round_page(size_used);
-
-		if (rsize_used != tree_size)
-			kmem_free(ipc_kernel_map,
-				  tree_addr + rsize_used,
-				  tree_size - rsize_used);
+		*tablep = (ipc_info_name_t *)0;
+		*tableCntp = 0;
+	}
 
-		if (size_used != rsize_used)
-			bzero((char *) (tree_addr + size_used),
-			      rsize_used - size_used);
+	/* prepare the tree out-of-line data for return */
+	if (tree_size > 0) {
+		if (tree_size > infop->iis_tree_size * sizeof(ipc_info_tree_name_t))
+			bzero((char *)&tree_info[infop->iis_tree_size],
+			      tree_size - infop->iis_tree_size * sizeof(ipc_info_tree_name_t));
 
 		kr = vm_map_unwire(ipc_kernel_map, vm_map_trunc_page(tree_addr),
-				   vm_map_round_page(tree_addr + rsize_used), FALSE);
+				   vm_map_round_page(tree_addr + tree_size), FALSE);
 		assert(kr == KERN_SUCCESS);
-
-		kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)tree_addr,
-				   (vm_map_size_t)rsize_used, TRUE, &copy);
+		kr = vm_map_copyin(ipc_kernel_map, (vm_map_address_t)tree_addr, 
+				   (vm_map_size_t)tree_size, TRUE, &copy);
 		assert(kr == KERN_SUCCESS);
-
-		*treep = (ipc_info_tree_name_t *) copy;
-		*treeCntp = tree_actual;
+		*treep = (ipc_info_tree_name_t *)copy;
+		*treeCntp = infop->iis_tree_size;
+	} else {
+		*treep = (ipc_info_tree_name_t *)0;
+		*treeCntp = 0;
 	}
-
 	return KERN_SUCCESS;
 }
 #endif /* MACH_IPC_DEBUG */
@@ -593,6 +519,9 @@ mach_port_kernel_object(
 	ipc_port_t port;
 	kern_return_t kr;
 
+	if (space == IS_NULL)
+		return KERN_INVALID_TASK;
+
 	kr = ipc_right_lookup_read(space, name, &entry);
 	if (kr != KERN_SUCCESS)
 		return kr;
diff --git a/osfmk/ipc/mach_msg.c b/osfmk/ipc/mach_msg.c
index c7c22bfbd..065f3b40f 100644
--- a/osfmk/ipc/mach_msg.c
+++ b/osfmk/ipc/mach_msg.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -93,9 +106,11 @@
 #include <ipc/ipc_entry.h>
 
 #include <machine/machine_routines.h>
+#include <security/mac_mach_internal.h>
 
 #include <sys/kdebug.h>
 
+
 #ifndef offsetof
 #define offsetof(type, member)  ((size_t)(&((type *)0)->member))
 #endif /* offsetof */
@@ -268,7 +283,7 @@ mach_msg_receive_results(void)
 	ipc_kmsg_t        kmsg = self->ith_kmsg;
 	mach_port_seqno_t seqno = self->ith_seqno;
 
-	mach_msg_format_0_trailer_t *trailer;
+	mach_msg_max_trailer_t *trailer;
 
 	ipc_object_release(object);
 
@@ -296,12 +311,63 @@ mach_msg_receive_results(void)
 	  goto out;
 	}
 
-	trailer = (mach_msg_format_0_trailer_t *)
+	trailer = (mach_msg_max_trailer_t *)
 			((vm_offset_t)kmsg->ikm_header +
 			round_msg(kmsg->ikm_header->msgh_size));
 	if (option & MACH_RCV_TRAILER_MASK) {
 		trailer->msgh_seqno = seqno;
 		trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
+
+
+		if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_AV)) {
+#if CONFIG_MACF_MACH
+		  if (kmsg->ikm_sender != NULL &&
+		    IP_VALID(kmsg->ikm_header->msgh_remote_port) &&
+		    mac_port_check_method(kmsg->ikm_sender,
+		    &kmsg->ikm_sender->maclabel,
+		    &((ipc_port_t)kmsg->ikm_header->msgh_remote_port)->ip_label,
+		    kmsg->ikm_header->msgh_id) == 0)
+		      trailer->msgh_ad = 1;
+		  else
+#endif
+		      trailer->msgh_ad = 0;
+		}
+
+		/*
+		 * The ipc_kmsg_t holds a reference to the label of a label
+		 * handle, not the port. We must get a reference to the port
+		 * and a send right to copyout to the receiver.
+		 */
+
+		if (option & MACH_RCV_TRAILER_ELEMENTS (MACH_RCV_TRAILER_LABELS)) {
+#if CONFIG_MACF_MACH
+		  if (kmsg->ikm_sender != NULL) {
+		    ipc_labelh_t  lh = kmsg->ikm_sender->label;
+		    kern_return_t kr;
+
+		    ip_lock(lh->lh_port);
+		    lh->lh_port->ip_mscount++;
+		    lh->lh_port->ip_srights++;
+		    ip_reference(lh->lh_port);
+		    ip_unlock(lh->lh_port);
+
+		    kr = ipc_object_copyout(space, (ipc_object_t)lh->lh_port,
+					    MACH_MSG_TYPE_PORT_SEND, 0,
+					    &trailer->msgh_labels.sender);
+		    if (kr != KERN_SUCCESS) {
+		      ip_lock(lh->lh_port);
+		      ip_release(lh->lh_port);
+		      ip_check_unlock(lh->lh_port);
+
+		      trailer->msgh_labels.sender = 0;
+		    }
+		  } else {
+		    trailer->msgh_labels.sender = 0;
+		  }
+#else
+		    trailer->msgh_labels.sender = 0;
+#endif
+		}
 	}
 
 	/*
@@ -559,7 +625,11 @@ unsigned int c_mmot_kernel_send = 0;		/* kernel server calls	*/
 
 #endif	/* !HOTPATH_DEBUG */
 
+#if CONFIG_MACF_MACH
+boolean_t enable_hotpath = FALSE;	/* XXX - push MAC into HOTPATH too */
+#else
 boolean_t enable_hotpath = TRUE;	/* Patchable, just in case ...	*/
+#endif
 #endif	/* HOTPATH_ENABLE */
 
 /*
@@ -585,16 +655,15 @@ mach_msg_overwrite_trap(
 	mach_port_name_t	notify = args->notify;
 	mach_vm_address_t	rcv_msg_addr = args->rcv_msg;
         mach_msg_size_t		scatter_list_size = 0; /* NOT INITIALIZED - but not used in pactice */
-	mach_port_seqno_t temp_seqno = 0;
+	__unused mach_port_seqno_t temp_seqno = 0;
 
-	register mach_msg_header_t *hdr;
 	mach_msg_return_t  mr = MACH_MSG_SUCCESS;
+#if	ENABLE_HOTPATH
 	/* mask out some of the options before entering the hot path */
 	mach_msg_option_t  masked_option = 
 		option & ~(MACH_SEND_TRAILER|MACH_RCV_TRAILER_MASK|MACH_RCV_LARGE);
+	register mach_msg_header_t *hdr;
 
-#if	ENABLE_HOTPATH
-	/* BEGINNING OF HOT PATH */
 	if ((masked_option == (MACH_SEND_MSG|MACH_RCV_MSG)) && enable_hotpath) {
 		thread_t self = current_thread();
 		mach_msg_format_0_trailer_t *trailer;
@@ -1092,120 +1161,88 @@ mach_msg_overwrite_trap(
 		  register ipc_mqueue_t dest_mqueue;
 		  wait_queue_t waitq;
 		  thread_t receiver;
-		  processor_t processor;
-		  boolean_t still_running;
 		  spl_t s;
 
 		  s = splsched();
-		  processor = current_processor();
-		  if (processor->current_pri >= BASEPRI_RTQUEUES)
-			  goto abort_send_receive1;
-
 		  dest_mqueue = &dest_port->ip_messages;
 		  waitq = &dest_mqueue->imq_wait_queue;
 		  imq_lock(dest_mqueue);
 
-		  wait_queue_peek64_locked(waitq, IPC_MQUEUE_RECEIVE, &receiver, &waitq);
-		  /* queue still locked, thread locked - but still on q */
+	    get_next_receiver:
+		  receiver = wait_queue_wakeup64_identity_locked(waitq,
+								IPC_MQUEUE_RECEIVE, 
+								THREAD_AWAKENED,
+								FALSE);
+		  /* queue still locked, receiver thread locked (if any) */
 
 		  if (	receiver == THREAD_NULL ) {
-		  abort_send_receive:
-			imq_unlock(dest_mqueue);
-		  abort_send_receive1:
-			splx(s);
-			ip_unlock(dest_port);
-			ipc_object_release(rcv_object);
-			HOT(c_mmot_cold_032++);
-			goto slow_send;
-		  }
-
-		  assert(receiver->state & TH_WAIT);
-		  assert(receiver->wait_queue == waitq);
-		  assert(receiver->wait_event == IPC_MQUEUE_RECEIVE);
-		
-		  /*
-		   * Make sure that the scheduling restrictions of the receiver
-		   * are consistent with a handoff here (if it comes down to that).
-		   */
-		  if (	receiver->sched_pri >= BASEPRI_RTQUEUES ||
-			  	receiver->processor_set != processor->processor_set ||
-				(receiver->bound_processor != PROCESSOR_NULL &&
-				 receiver->bound_processor != processor)) {
-			HOT(c_mmot_cold_033++);
-		fall_off:
-			thread_unlock(receiver);
-			if (waitq != &dest_mqueue->imq_wait_queue)
-				wait_queue_unlock(waitq);
-			goto abort_send_receive;
-		  }
+			  imq_unlock(dest_mqueue);
+			  splx(s);
 
-		  /*
-		   * Check that the receiver can stay on the hot path.
-		   */
-		  if (ipc_kmsg_copyout_size(kmsg, receiver->map) + 
-			  REQUESTED_TRAILER_SIZE(receiver->ith_option) > receiver->ith_msize) {
-			/*
-			 *	The receiver can't accept the message.
-			 */
-			HOT(c_mmot_bad_rcvr++);
-			goto fall_off;
+			  ip_unlock(dest_port);
+			  ipc_object_release(rcv_object);
+			  HOT(c_mmot_cold_032++);
+			  goto slow_send;
 		  }
 
 		  /*
-		   * Before committing to the handoff, make sure that we are
-		   * really going to block (i.e. there are no messages already
-		   * queued for us.  This violates lock ordering, so make sure
-		   * we don't deadlock. After the trylock succeeds below, we
-		   * may have up to 3 message queues locked:
-		   *	- the dest port mqueue
-		   * 	- a portset mqueue (where waiting receiver was found)
-		   *    - finally our own rcv_mqueue
+		   * Check that the receiver can handle the size of the message.
+		   * If not, and the receiver just wants to be informed of that
+		   * fact, set it running and try to find another thread.
 		   *
-		   * JMM - Need to make this check appropriate for portsets as
-		   * well before re-enabling them.
+		   * If he didn't want the "too large" message left on the queue,
+		   * give it to him anyway, he'll consume it as part of his receive
+		   * processing.
 		   */
-		  if (!imq_lock_try(rcv_mqueue)) {
-			goto fall_off;
-		  }
-		  if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
-			imq_unlock(rcv_mqueue);
-			splx(s);
-			HOT(c_mmot_cold_033++);
-			goto fall_off;
+		  if (receiver->ith_msize < 
+		      ipc_kmsg_copyout_size(kmsg, receiver->map) + 
+		      REQUESTED_TRAILER_SIZE(receiver->ith_option))
+		  {
+			  receiver->ith_msize = kmsg->ikm_header->msgh_size;
+			  receiver->ith_state = MACH_RCV_TOO_LARGE;
+
+			  if ((receiver->ith_option & MACH_RCV_LARGE) != 0) {
+				  receiver->ith_kmsg = IKM_NULL;
+				  receiver->ith_seqno = 0;
+				  thread_unlock(receiver);
+				  HOT(c_mmot_bad_rcvr++);
+				  goto get_next_receiver;
+			  }
+		  } else {
+			  receiver->ith_state = MACH_MSG_SUCCESS;
 		  }
 
-		  /* At this point we are committed to do the "handoff". */
+		  /* At this point we are committed to do the message handoff. */
 		  c_mach_msg_trap_switch_fast++;
-		  
-		  /*
-		   * Go ahead and pull the receiver from the waitq.  If the
-		   * waitq wasn't the one for the mqueue, unlock it.
-		   */
-		  wait_queue_pull_thread_locked(waitq,
-								receiver,
-								(waitq != &dest_mqueue->imq_wait_queue));
 
 		  /*
 		   *	Store the kmsg and seqno where the receiver can pick it up.
+		   *	and set it running.
 		   */
-		  receiver->ith_state = MACH_MSG_SUCCESS;
 		  receiver->ith_kmsg = kmsg;
 		  receiver->ith_seqno = dest_mqueue->imq_seqno++;
-
-		  /*
-		   * Unblock the receiver.  If it was still running on another
-		   * CPU, we'll give it a chance to run with the message where
-		   * it is (and just select someother thread to run here).
-		   * Otherwise, we'll invoke it here as part of the handoff.
-		   */
-		  still_running = thread_unblock(receiver, THREAD_AWAKENED);
-
 		  thread_unlock(receiver);
 
 		  imq_unlock(dest_mqueue);
 		  ip_unlock(dest_port);
 		  current_task()->messages_sent++;
+		 
+		  /*
+		   * Now prepare to wait on our receive queue.  But we have to make
+		   * sure the queue doesn't already have messages. If it does, we'll
+		   * have to do a slow receive.
+		   *
+		   * JMM - Need to make this check appropriate for portsets as
+		   * well before re-enabling them.
+		   */
+		  imq_lock(rcv_mqueue);
 
+		  if (ipc_kmsg_queue_first(&rcv_mqueue->imq_messages) != IKM_NULL) {
+			imq_unlock(rcv_mqueue);
+			splx(s);
+			HOT(c_mmot_cold_033++);
+			goto slow_receive;
+		  }
 
 		  /*
 		   *	Put self on receive port's queue.
@@ -1228,19 +1265,8 @@ mach_msg_overwrite_trap(
 										self);
 		  thread_unlock(self);
 		  imq_unlock(rcv_mqueue);
-
-		  /*
-		   * If the receiving thread wasn't still running, we switch directly
-		   * to it here.  Otherwise we let the scheduler pick something for
-		   * here.  In either case, block this thread as though it had called
-		   * ipc_mqueue_receive.
-		   */
-		  if (still_running) {
-			  splx(s);
-			  thread_block(ipc_mqueue_receive_continue);
-		  } else {
-			  thread_run(self, ipc_mqueue_receive_continue, NULL, receiver);
-		  }
+		  splx(s);
+		  thread_block(ipc_mqueue_receive_continue);
 		  /* NOTREACHED */
 		}
 
@@ -1711,13 +1737,13 @@ mach_msg_overwrite_trap(
 		/* hold ref for rcv_object */
 
 		/*
-		 * slow_receive:
 		 *
 		 *	Now we have sent the request and copied in rcv_name,
 		 *	and hold ref for rcv_object (to keep mqueue alive).
 		 *  Just receive a reply and try to get back to fast path.
 		 */
 
+		slow_receive:
 		self->ith_continuation = (void (*)(mach_msg_return_t))0;
 		ipc_mqueue_receive(rcv_mqueue,
 				   MACH_MSG_OPTION_NONE,
@@ -1730,22 +1756,22 @@ mach_msg_overwrite_trap(
 
 		ipc_object_release(rcv_object);
 
-		  if (mr != MACH_MSG_SUCCESS) {
+		if (mr != MACH_MSG_SUCCESS) {
 		    return(mr);
-		  }
+		}
 
-		  kmsg = self->ith_kmsg;
-		  hdr = kmsg->ikm_header;
-		  send_size = hdr->msgh_size;
-		  trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
-							     round_msg(send_size));
-		  if (option & MACH_RCV_TRAILER_MASK) {
+		kmsg = self->ith_kmsg;
+		hdr = kmsg->ikm_header;
+		send_size = hdr->msgh_size;
+		trailer = (mach_msg_format_0_trailer_t *) ((vm_offset_t) hdr +
+												   round_msg(send_size));
+		if (option & MACH_RCV_TRAILER_MASK) {
 		    trailer->msgh_seqno = temp_seqno;	
 		    trailer->msgh_trailer_size = REQUESTED_TRAILER_SIZE(option);
-		  }
-		  dest_port = (ipc_port_t) hdr->msgh_remote_port;
-		  HOT(c_mmot_cold_055++);
-		  goto fast_copyout;
+		}
+		dest_port = (ipc_port_t) hdr->msgh_remote_port;
+		HOT(c_mmot_cold_055++);
+		goto fast_copyout;
 
 	    slow_copyout:
 		/*
diff --git a/osfmk/ipc/mach_port.c b/osfmk/ipc/mach_port.c
index fb27cd9a1..e220fc9fb 100644
--- a/osfmk/ipc/mach_port.c
+++ b/osfmk/ipc/mach_port.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -82,7 +95,11 @@
 #include <ipc/ipc_pset.h>
 #include <ipc/ipc_right.h>
 #include <ipc/ipc_kmsg.h>
+#include <ipc/ipc_labelh.h>
 #include <kern/misc_protos.h>
+#include <security/mac_mach_internal.h>
+
+#include <mach/security_server.h>
 
 /*
  * Forward declarations
@@ -1775,3 +1792,177 @@ task_set_port_space(
 	return kr;
 }
 
+/*
+ * Get a (new) label handle representing the given port's port label.
+ */
+#if CONFIG_MACF_MACH
+kern_return_t
+mach_get_label(
+	ipc_space_t		space,
+	mach_port_name_t	name,
+	mach_port_name_t	*outlabel)
+{
+	ipc_entry_t entry;
+	ipc_port_t port;
+	struct label outl;
+	kern_return_t kr;
+	int dead;
+
+	if (!MACH_PORT_VALID(name))
+		return KERN_INVALID_NAME;
+
+	/* Lookup the port name in the task's space. */
+	kr = ipc_right_lookup_write(space, name, &entry);
+	if (kr != KERN_SUCCESS)
+		return kr;
+
+	port = (ipc_port_t) entry->ie_object;
+	dead = ipc_right_check(space, port, name, entry);
+	if (dead) {
+		is_write_unlock(space);
+		return KERN_INVALID_RIGHT;
+	}
+	/* port is now locked */
+
+	is_write_unlock(space);
+	/* Make sure we are not dealing with a label handle. */
+	if (ip_kotype(port) == IKOT_LABELH) {
+		/* already is a label handle! */
+		ip_unlock(port);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	/* Copy the port label and stash it in a new label handle. */
+	mac_port_label_init(&outl);
+	mac_port_label_copy(&port->ip_label, &outl); 
+	kr = labelh_new_user(space, &outl, outlabel);
+	ip_unlock(port);
+
+	return KERN_SUCCESS;
+}
+#else
+kern_return_t
+mach_get_label(
+	 __unused ipc_space_t		space,
+	 __unused mach_port_name_t	name,
+	 __unused mach_port_name_t	*outlabel)
+{
+	return KERN_INVALID_ARGUMENT;
+}
+#endif
+
+/*
+ * also works on label handles
+ */
+#if CONFIG_MACF_MACH
+kern_return_t
+mach_get_label_text(
+	ipc_space_t		space,
+	mach_port_name_t	name,
+	labelstr_t		policies,
+	labelstr_t		outlabel)
+{
+	ipc_entry_t entry;
+	kern_return_t kr;
+	struct label *l;
+	int dead;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	if (!MACH_PORT_VALID(name))
+		return KERN_INVALID_NAME;
+
+	kr = ipc_right_lookup_write(space, name, &entry);
+	if (kr != KERN_SUCCESS)
+		return kr;
+
+	dead = ipc_right_check(space, (ipc_port_t) entry->ie_object, name,
+	    entry);
+	if (dead) {
+		is_write_unlock(space);
+		return KERN_INVALID_RIGHT;
+	}
+	/* object (port) is now locked */
+
+	is_write_unlock (space);
+	l = io_getlabel(entry->ie_object);
+
+	mac_port_label_externalize(l, policies, outlabel, 512, 0);
+
+	io_unlocklabel(entry->ie_object);
+	io_unlock(entry->ie_object);
+	return KERN_SUCCESS;
+}
+#else
+kern_return_t
+mach_get_label_text(
+	__unused ipc_space_t		space,
+	__unused mach_port_name_t	name,
+	__unused labelstr_t		policies,
+	__unused labelstr_t		outlabel)
+{
+	return KERN_INVALID_ARGUMENT;
+}
+#endif
+
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mach_set_port_label(
+	ipc_space_t		space,
+	mach_port_name_t	name,
+	labelstr_t		labelstr)
+{
+	ipc_entry_t entry;
+	kern_return_t kr;
+	struct label inl;
+	ipc_port_t port;
+	int rc;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	if (!MACH_PORT_VALID(name))
+		return KERN_INVALID_NAME;
+
+	mac_port_label_init(&inl);
+	rc = mac_port_label_internalize(&inl, labelstr);
+	if (rc)
+		return KERN_INVALID_ARGUMENT;
+
+	kr = ipc_right_lookup_write(space, name, &entry);
+	if (kr != KERN_SUCCESS)
+		return kr;
+
+	if (io_otype(entMACry->ie_object) != IOT_PORT) {
+		is_write_unlock(space);
+		return KERN_INVALID_RIGHT;
+	}
+
+	port = (ipc_port_t) entry->ie_object;
+	ip_lock(port);
+
+	tasklabel_lock(space->is_task);
+	rc = mac_port_check_label_update(&space->is_task->maclabel,
+				    &port->ip_label, &inl);
+	tasklabel_unlock(space->is_task);
+	if (rc)
+		kr = KERN_NO_ACCESS;
+	else
+		mac_port_label_copy(&inl, &port->ip_label);
+
+	ip_unlock(port);
+	is_write_unlock(space);
+	return kr;
+}
+#else
+kern_return_t
+mach_set_port_label(
+	ipc_space_t		space __unused,
+	mach_port_name_t	name __unused,
+	labelstr_t		labelstr __unused)
+{
+	return KERN_INVALID_ARGUMENT;
+}
+#endif
diff --git a/osfmk/ipc/mig_log.c b/osfmk/ipc/mig_log.c
index 9d9e85e55..86b8ae672 100644
--- a/osfmk/ipc/mig_log.c
+++ b/osfmk/ipc/mig_log.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ipc/port.h b/osfmk/ipc/port.h
index 52e25a645..6e0b8e2d1 100644
--- a/osfmk/ipc/port.h
+++ b/osfmk/ipc/port.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kdp/Makefile b/osfmk/kdp/Makefile
index a71bf45c6..38d35f441 100644
--- a/osfmk/kdp/Makefile
+++ b/osfmk/kdp/Makefile
@@ -13,13 +13,18 @@ INSTINC_SUBDIRS_PPC = \
 
 INSTINC_SUBDIRS_I386 = \
 
+INSTINC_SUBDIRS_ARM = \
+
 EXPINC_SUBDIRS = \
 
 EXPINC_SUBDIRS_PPC = \
 
 EXPINC_SUBDIRS_I386 = \
 
+EXPINC_SUBDIRS_ARM = \
+
 DATAFILES = \
+		kdp_callout.h \
 		kdp_en_debugger.h
 		
 EXPORT_MI_LIST	= ${DATAFILES}
diff --git a/osfmk/kdp/kdp.c b/osfmk/kdp/kdp.c
index a0f3d496e..357f19ec7 100644
--- a/osfmk/kdp/kdp.c
+++ b/osfmk/kdp/kdp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -69,7 +75,7 @@ static kdp_dispatch_t
 /*10 */ kdp_breakpoint_remove,
 /*11 */	kdp_regions,
 /*12 */ kdp_reattach,
-/*13 */ kdp_reboot
+/*13 */ (kdp_dispatch_t)kdp_reboot
     };
     
 kdp_glob_t	kdp;
@@ -93,6 +99,7 @@ static unsigned int breakpoints_initialized = 0;
 
 int reattach_wait = 0;
 int noresume_on_disconnect = 0;
+extern unsigned int return_on_panic;
 
 #define MAXCOMLEN 16
 
@@ -115,13 +122,14 @@ struct thread_snapshot {
 typedef struct thread_snapshot *thread_snapshot_t;
 
 extern int
-machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
 extern int
-machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
 extern int
 proc_pid(void *p);
 extern void
-proc_name(int pid, char *buf, int size);
+proc_name_kdp(task_t  task, char *buf, int size);
+
 extern void
 kdp_snapshot_postflight(void);
 
@@ -129,7 +137,7 @@ static int
 pid_from_task(task_t task);
 
 int
-kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced);
+kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced);
 
 extern char version[];
 
@@ -142,8 +150,8 @@ kdp_packet(
 {
     static unsigned	aligned_pkt[1538/sizeof(unsigned)+1]; // max ether pkt
     kdp_pkt_t		*rd = (kdp_pkt_t *)&aligned_pkt;
-    int			plen = *len;
-    unsigned int	req;
+    size_t		plen = *len;
+    kdp_req_t		req;
     boolean_t		ret;
     
 #if DO_ALIGN
@@ -152,7 +160,7 @@ kdp_packet(
     rd = (kdp_pkt_t *)pkt;
 #endif
     if (plen < sizeof (rd->hdr) || rd->hdr.len != plen) {
-	printf("kdp_packet bad len pkt %d hdr %d\n", plen, rd->hdr.len);
+	printf("kdp_packet bad len pkt %lu hdr %d\n", plen, rd->hdr.len);
 
 	return (FALSE);
     }
@@ -165,7 +173,7 @@ kdp_packet(
     }
     
     req = rd->hdr.request;
-    if ((req < KDP_CONNECT) || (req > KDP_HOSTREBOOT)) {
+    if (req > KDP_HOSTREBOOT) {
 	printf("kdp_packet bad request %x len %d seq %x key %x\n",
 	    rd->hdr.request, rd->hdr.len, rd->hdr.seq, rd->hdr.key);
 
@@ -182,8 +190,8 @@ kdp_packet(
 static boolean_t
 kdp_unknown(
     kdp_pkt_t		*pkt,
-    int			*len,
-    unsigned short	*reply_port
+    __unused int	*len,
+    __unused unsigned short	*reply_port
 )
 {
     kdp_pkt_t		*rd = (kdp_pkt_t *)pkt;
@@ -202,7 +210,7 @@ kdp_connect(
 )
 {
     kdp_connect_req_t	*rq = &pkt->connect_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_connect_reply_t	*rp = &pkt->connect_reply;
 
     if (plen < sizeof (*rq))
@@ -245,7 +253,7 @@ kdp_disconnect(
 )
 {
     kdp_disconnect_req_t	*rq = &pkt->disconnect_req;
-    int				plen = *len;
+    size_t			plen = *len;
     kdp_disconnect_reply_t	*rp = &pkt->disconnect_reply;
 
     if (plen < sizeof (*rq))
@@ -262,6 +270,9 @@ kdp_disconnect(
     kdp.is_halted = kdp.is_conn = FALSE;
     kdp.exception_seq = kdp.conn_seq = 0;
 
+    if ((panicstr != NULL) && (return_on_panic == 0))
+	    reattach_wait = 1;
+
     if (noresume_on_disconnect == 1) {
 	reattach_wait = 1;
 	noresume_on_disconnect = 0;
@@ -286,7 +297,6 @@ kdp_reattach(
 )
 {
   kdp_reattach_req_t            *rq = &pkt->reattach_req;
-  kdp_disconnect_reply_t	*rp = &pkt->disconnect_reply;
 
   kdp.is_conn = TRUE;
   kdp_disconnect(pkt, len, reply_port);
@@ -303,7 +313,7 @@ kdp_hostinfo(
 )
 {
     kdp_hostinfo_req_t	*rq = &pkt->hostinfo_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_hostinfo_reply_t *rp = &pkt->hostinfo_reply;
 
     if (plen < sizeof (*rq))
@@ -328,7 +338,7 @@ kdp_suspend(
 )
 {
     kdp_suspend_req_t	*rq = &pkt->suspend_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_suspend_reply_t *rp = &pkt->suspend_reply;
 
     if (plen < sizeof (*rq))
@@ -355,7 +365,7 @@ kdp_resumecpus(
 )
 {
     kdp_resumecpus_req_t	*rq = &pkt->resumecpus_req;
-    int			plen = *len;
+    size_t			plen = *len;
     kdp_resumecpus_reply_t 	*rp = &pkt->resumecpus_reply;
 
     if (plen < sizeof (*rq))
@@ -382,7 +392,7 @@ kdp_writemem(
 )
 {
     kdp_writemem_req_t	*rq = &pkt->writemem_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_writemem_reply_t *rp = &pkt->writemem_reply;
     int 		cnt;
 
@@ -415,10 +425,10 @@ kdp_readmem(
 )
 {
     kdp_readmem_req_t	*rq = &pkt->readmem_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_readmem_reply_t *rp = &pkt->readmem_reply;
     int			cnt;
-#if __i386__
+#if __i386__ || __arm__
     void		*pversion = &version;
 #endif
     if (plen < sizeof (*rq))
@@ -432,19 +442,22 @@ kdp_readmem(
     else {
 	unsigned int	n = rq->nbytes;
 
-	dprintf(("kdp_readmem addr %x size %d\n", rq->address, rq->nbytes));
-#if __i386__
+	dprintf(("kdp_readmem addr %x size %d\n", rq->address, n));
+#if __i386__ || __arm__
 	/* XXX This is a hack to facilitate the "showversion" macro
-	 * on i386, which is used to obtain the kernel version without
+	 * on i386/ARM, which is used to obtain the kernel version without
 	 * symbols - a pointer to the version string should eventually
 	 * be pinned at a fixed address when an equivalent of the
 	 * VECTORS segment (loaded at a fixed load address, and contains
-	 * a table) is implemented on x86, as with PPC.
+	 * a table) is implemented on these architectures, as with PPC.
+	 * N.B.: x86 now has a low global page, and the version indirection
+	 * is pinned at 0x201C. We retain the 0x501C address override
+	 * for compatibility.
 	 */
 	if (rq->address == (void *)0x501C)
 		rq->address = &pversion;
 #endif
-	cnt = kdp_vm_read((caddr_t)rq->address, (caddr_t)rp->data, rq->nbytes);
+	cnt = kdp_vm_read((caddr_t)rq->address, (caddr_t)rp->data, n);
 	rp->error = KDPERR_NO_ERROR;
 
 	rp->hdr.len += cnt;
@@ -464,7 +477,7 @@ kdp_maxbytes(
 )
 {
     kdp_maxbytes_req_t	*rq = &pkt->maxbytes_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_maxbytes_reply_t *rp = &pkt->maxbytes_reply;
 
     if (plen < sizeof (*rq))
@@ -491,9 +504,8 @@ kdp_version(
 )
 {
     kdp_version_req_t	*rq = &pkt->version_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_version_reply_t *rp = &pkt->version_reply;
-    kdp_region_t	*r;	
 
     if (plen < sizeof (*rq))
 	return (FALSE);
@@ -504,7 +516,7 @@ kdp_version(
     dprintf(("kdp_version\n"));
 
     rp->version = KDP_VERSION;
-#ifdef	__ppc__
+#if	__ppc__
     if (!(kdp_flag & KDP_BP_DIS))
       rp->feature = KDP_FEATURE_BP;
     else
@@ -527,7 +539,7 @@ kdp_regions(
 )
 {
     kdp_regions_req_t	*rq = &pkt->regions_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_regions_reply_t *rp = &pkt->regions_reply;
     kdp_region_t	*r;	
 
@@ -542,7 +554,7 @@ kdp_regions(
     r = rp->regions;
     rp->nregions = 0;
 
-    (vm_offset_t)r->address = 0;
+    r->address = NULL;
     r->nbytes = 0xffffffff;
 
     r->protection = VM_PROT_ALL; r++; rp->nregions++;
@@ -563,7 +575,7 @@ kdp_writeregs(
 )
 {
     kdp_writeregs_req_t	*rq = &pkt->writeregs_req;
-    int			plen = *len;
+    size_t		plen = *len;
     int			size;
     kdp_writeregs_reply_t *rp = &pkt->writeregs_reply;
 
@@ -590,7 +602,7 @@ kdp_readregs(
 )
 {
     kdp_readregs_req_t	*rq = &pkt->readregs_req;
-    int			plen = *len;
+    size_t		plen = *len;
     kdp_readregs_reply_t *rp = &pkt->readregs_reply;
     int			size;
 
@@ -618,7 +630,7 @@ kdp_breakpoint_set(
 {
   kdp_breakpoint_req_t	*rq = &pkt->breakpoint_req;
   kdp_breakpoint_reply_t *rp = &pkt->breakpoint_reply;
-  int			plen = *len;
+  size_t		plen = *len;
   int                   cnt, i;
   unsigned int          old_instruction = 0;
   unsigned int breakinstr = kdp_ml_get_breakinsn();
@@ -681,7 +693,7 @@ kdp_breakpoint_remove(
 {
   kdp_breakpoint_req_t	*rq = &pkt->breakpoint_req;
   kdp_breakpoint_reply_t *rp = &pkt->breakpoint_reply;
-  int			plen = *len;
+  size_t		plen = *len;
   int                   cnt,i;
 
   if (plen < sizeof (*rq))
@@ -711,7 +723,7 @@ kdp_breakpoint_remove(
 }
 
 boolean_t
-kdp_remove_all_breakpoints()
+kdp_remove_all_breakpoints(void)
 {
   int i;
   boolean_t breakpoint_found = FALSE;
@@ -747,28 +759,33 @@ static int pid_from_task(task_t task)
 }
 
 int
-kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced)
+kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size, unsigned trace_options, uint32_t *pbytesTraced)
 {
-	uint32_t tracepos = (uint32_t) tracebuf;
-	uint32_t tracebound = tracepos + tracebuf_size;
+	char *tracepos = (char *) tracebuf;
+	char *tracebound = tracepos + tracebuf_size;
 	uint32_t tracebytes = 0;
 	int error = 0;
-	
-	processor_set_t pset = &default_pset;
+
 	task_t task = TASK_NULL;
 	thread_t thread = THREAD_NULL;
 	int nframes = trace_options;
 	thread_snapshot_t tsnap = NULL;
 	unsigned framesize = 2 * sizeof(vm_offset_t);
+	struct task ctask;
+	struct thread cthread;
 
 	if ((nframes <= 0) || nframes > MAX_FRAMES)
 		nframes = MAX_FRAMES;
 
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
+		if ((task == NULL) || (ml_nofault_copy((vm_offset_t) task, (vm_offset_t) &ctask, sizeof(struct task)) != sizeof(struct task)))
+			goto error_exit;
 		/* Trace everything, unless a process was specified */
 		if ((pid == -1) || (pid == pid_from_task(task)))
 			queue_iterate(&task->threads, thread, thread_t, task_threads){
-				if ((tracepos + 4 * sizeof(struct thread_snapshot)) > tracebound) {
+				if ((thread == NULL) || (ml_nofault_copy((vm_offset_t) thread, (vm_offset_t) &cthread, sizeof(struct thread)) != sizeof(struct thread)))
+					goto error_exit;
+				if (((tracepos + 4 * sizeof(struct thread_snapshot)) > tracebound)) {
 					error = -1;
 					goto error_exit;
 				}
@@ -783,7 +800,7 @@ kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size, unsigned trace
 				tsnap->continuation = thread->continuation;
 /* Add the BSD process identifiers */
 				if ((tsnap->pid = pid_from_task(task)) != -1)
-					proc_name(tsnap->pid, tsnap->p_comm, MAXCOMLEN + 1);
+					proc_name_kdp(task, tsnap->p_comm, MAXCOMLEN + 1);
 				else
 					tsnap->p_comm[0] = '\0';
 
@@ -822,7 +839,7 @@ error_exit:
 	/* Release stack snapshot wait indicator */
 	kdp_snapshot_postflight();
 
-	*pbytesTraced = tracepos - tracebuf;
+	*pbytesTraced = tracepos - (char *) tracebuf;
 
 	return error;
 }
diff --git a/osfmk/kdp/kdp.h b/osfmk/kdp/kdp.h
index da4991194..9d5b91352 100644
--- a/osfmk/kdp/kdp.h
+++ b/osfmk/kdp/kdp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/kdp/kdp_callout.h b/osfmk/kdp/kdp_callout.h
new file mode 100644
index 000000000..dab21ae3a
--- /dev/null
+++ b/osfmk/kdp/kdp_callout.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+typedef enum {
+	KDP_EVENT_ENTER,
+	KDP_EVENT_EXIT,
+	KDP_EVENT_PANICLOG
+} kdp_event_t;
+
+typedef void (*kdp_callout_fn_t)(void *arg, kdp_event_t event);
+
+/*
+ * Register fn(arg, event) to be called at kdp entry/exit.
+ * The callouts are made in a single-threaded environment, interrupts are
+ * disabled and processors other than the callout processor quiesced.
+ * N.B. callouts are strictly limited in what they can do: they must execute
+ * with interrupts disabled and they can't call back into the kernel for any
+ * non-trivial service.
+ */
+extern void kdp_register_callout(kdp_callout_fn_t fn, void *arg);
+
diff --git a/osfmk/kdp/kdp_core.h b/osfmk/kdp/kdp_core.h
index 801a1c7af..aa54f350d 100644
--- a/osfmk/kdp/kdp_core.h
+++ b/osfmk/kdp/kdp_core.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* HISTORY
diff --git a/osfmk/kdp/kdp_en_debugger.h b/osfmk/kdp/kdp_en_debugger.h
index 6904c399d..dd63d30e2 100644
--- a/osfmk/kdp/kdp_en_debugger.h
+++ b/osfmk/kdp/kdp_en_debugger.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/kdp/kdp_internal.h b/osfmk/kdp/kdp_internal.h
index c513c91da..88d72cfda 100644
--- a/osfmk/kdp/kdp_internal.h
+++ b/osfmk/kdp/kdp_internal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -151,3 +157,8 @@ unsigned int
 kdp_ml_get_breakinsn(
     void
 );
+
+extern void
+kdp_ml_enter_debugger(
+	void
+);
diff --git a/osfmk/kdp/kdp_private.h b/osfmk/kdp/kdp_private.h
index e9399002a..c4ef6bf8a 100644
--- a/osfmk/kdp/kdp_private.h
+++ b/osfmk/kdp/kdp_private.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/kdp/kdp_protocol.h b/osfmk/kdp/kdp_protocol.h
index 8768c9c87..1eb174cec 100644
--- a/osfmk/kdp/kdp_protocol.h
+++ b/osfmk/kdp/kdp_protocol.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/kdp/kdp_udp.c b/osfmk/kdp/kdp_udp.c
index df1ec3f67..a513398fe 100644
--- a/osfmk/kdp/kdp_udp.c
+++ b/osfmk/kdp/kdp_udp.c
@@ -1,24 +1,31 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
+
 /*
  * Copyright (c) 1982, 1986, 1993
  *      The Regents of the University of California.  All rights reserved.
@@ -95,13 +102,13 @@ static const char
 
 volatile int kdp_flag = 0;
 
-static kdp_send_t kdp_en_send_pkt = 0;
-static kdp_receive_t kdp_en_recv_pkt = 0;
+static kdp_send_t kdp_en_send_pkt;
+static kdp_receive_t kdp_en_recv_pkt;
 
 
 static u_long kdp_current_ip_address = 0;
 static struct ether_addr kdp_current_mac_address = {{0, 0, 0, 0, 0, 0}};
-static void *kdp_current_ifp = 0;
+static void *kdp_current_ifp;
 
 static void kdp_handler( void *);
 
@@ -114,7 +121,6 @@ static volatile boolean_t panicd_specified = FALSE;
 static boolean_t router_specified = FALSE;
 static unsigned int panicd_port = CORE_REMOTE_PORT;
 
-/* As in bsd/net/ether_if_module.c */
 static struct ether_addr etherbroadcastaddr = {{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}};
 
 static struct ether_addr router_mac = {{0, 0, 0 , 0, 0, 0}};
@@ -137,12 +143,11 @@ static char panicd_ip_str[20];
 static char router_ip_str[20];
 
 static unsigned int panic_block = 0;
-static volatile unsigned int kdp_trigger_core_dump = 0;
-static volatile unsigned int flag_kdp_trigger_reboot = 0;
+volatile unsigned int kdp_trigger_core_dump = 0;
+__private_extern__ volatile unsigned int flag_kdp_trigger_reboot = 0;
 
 extern unsigned int not_in_kdp;
 
-extern unsigned long panic_caller;
 extern unsigned int disableConsoleOutput;
 
 extern int 		kdp_vm_read( caddr_t, caddr_t, unsigned int);
@@ -160,6 +165,8 @@ static void 		kdp_arp_reply(struct ether_arp *);
 static void 		kdp_process_arp_reply(struct ether_arp *);
 static boolean_t 	kdp_arp_resolve(uint32_t, struct ether_addr *);
 
+static volatile unsigned	kdp_reentry_deadline;
+
 static boolean_t	gKDPDebug = FALSE;
 #define KDP_DEBUG(...) if (gKDPDebug) printf(__VA_ARGS__);
 
@@ -172,6 +179,8 @@ static uint32_t stack_snapshot_bufsize;
 static int stack_snapshot_pid;
 static uint32_t stack_snapshot_options;
 
+static unsigned int old_debugger;
+
 void
 kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size,
     uint32_t options);
@@ -180,7 +189,7 @@ void
 kdp_snapshot_postflight(void);
 
 extern int
-kdp_stackshot(int pid, uint32_t tracebuf, uint32_t tracebuf_size,
+kdp_stackshot(int pid, void *tracebuf, uint32_t tracebuf_size,
     unsigned trace_options, uint32_t *pbytesTraced);
 
 int
@@ -189,43 +198,63 @@ kdp_stack_snapshot_geterror(void);
 int
 kdp_stack_snapshot_bytes_traced(void);
 
+static thread_call_t
+kdp_timer_call;
+
+static void
+kdp_ml_enter_debugger_wrapper(__unused void *param0, __unused void *param1) {
+	kdp_ml_enter_debugger();
+}
+
+static void
+kdp_timer_callout_init(void) {
+	kdp_timer_call = thread_call_allocate(kdp_ml_enter_debugger_wrapper, NULL);
+}
+
+
 void
 kdp_register_send_receive(
 	kdp_send_t	send, 
 	kdp_receive_t	receive)
 {
-	unsigned int	debug=0;
+	unsigned int	debug = 0;
 
 	kdp_en_send_pkt = send;
 	kdp_en_recv_pkt = receive;
 
 	debug_log_init();
 
+	kdp_timer_callout_init();
+
 	PE_parse_boot_arg("debug", &debug);
 
+
+	if (!debug)
+		return;
+
 	if (debug & DB_KDP_BP_DIS)
 		kdp_flag |= KDP_BP_DIS;   
 	if (debug & DB_KDP_GETC_ENA)
 		kdp_flag |= KDP_GETC_ENA;   
 	if (debug & DB_ARP)
-	  kdp_flag |= KDP_ARP;
+		kdp_flag |= KDP_ARP;
 
 	if (debug & DB_KERN_DUMP_ON_PANIC)
-	  kdp_flag |= KDP_PANIC_DUMP_ENABLED;
+		kdp_flag |= KDP_PANIC_DUMP_ENABLED;
 	if (debug & DB_KERN_DUMP_ON_NMI)
-	  kdp_flag |= PANIC_CORE_ON_NMI;
-	
+		kdp_flag |= PANIC_CORE_ON_NMI;
+
 	if (debug & DB_DBG_POST_CORE)
-	  kdp_flag |= DBG_POST_CORE;
-	
+		kdp_flag |= DBG_POST_CORE;
+
 	if (debug & DB_PANICLOG_DUMP)
-	  kdp_flag |= PANIC_LOG_DUMP;
-	
+		kdp_flag |= PANIC_LOG_DUMP;
+
 	if (PE_parse_boot_arg ("_panicd_ip", panicd_ip_str))
-	  panicd_specified = TRUE;
+		panicd_specified = TRUE;
 
 	if (PE_parse_boot_arg ("_router_ip", router_ip_str))
-	  router_specified = TRUE;
+		router_specified = TRUE;
 
 	if (!PE_parse_boot_arg ("panicd_port", &panicd_port))
 		panicd_port = CORE_REMOTE_PORT;
@@ -233,7 +262,7 @@ kdp_register_send_receive(
 	kdp_flag |= KDP_READY;
 	if (current_debugger == NO_CUR_DB)
 		current_debugger = KDP_CUR_DB;
-	if (halt_in_debugger) {
+	if ((kdp_current_ip_address != 0) && halt_in_debugger) {
 		kdp_call(); 
 		halt_in_debugger=0;
 	}
@@ -260,12 +289,22 @@ kdp_snapshot_preflight(int pid, void * tracebuf, uint32_t tracebuf_size, uint32_
 	stack_snapshot_bufsize = tracebuf_size;
 	stack_snapshot_options = options;
 	kdp_snapshot++;
+	/* Mark this debugger as active, since the polled mode driver that 
+	 * ordinarily does this may not be enabled (yet), or since KDB may be
+	 * the primary debugger.
+	 */
+	old_debugger = current_debugger;
+	if (old_debugger != KDP_CUR_DB) {
+		current_debugger = KDP_CUR_DB;
+	}
 }
 
 void
 kdp_snapshot_postflight(void)
 {
 	kdp_snapshot--;
+	if ((kdp_en_send_pkt == NULL) || (old_debugger == KDB_CUR_DB))
+		current_debugger = old_debugger;
 }
 
 int
@@ -280,6 +319,14 @@ kdp_stack_snapshot_bytes_traced(void)
 	return stack_snapshot_bytes_traced;
 }
 
+static void
+kdp_schedule_debugger_reentry(unsigned interval) {
+	uint64_t deadline;;
+
+	clock_interval_to_deadline(interval, 1000 * 1000, &deadline);
+	thread_call_enter_delayed(kdp_timer_call, deadline);
+}
+
 static void
 enaddr_copy(
 	void	*src,
@@ -332,7 +379,7 @@ kdp_reply(
 #else
 	ui = (struct udpiphdr *)&pkt.data[pkt.off];
 #endif
-	ui->ui_next = ui->ui_prev = 0;
+	ui->ui_next = ui->ui_prev = NULL;
 	ui->ui_x1 = 0;
 	ui->ui_pr = IPPROTO_UDP;
 	ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
@@ -400,7 +447,7 @@ kdp_send(
 #else
     ui = (struct udpiphdr *)&pkt.data[pkt.off];
 #endif
-    ui->ui_next = ui->ui_prev = 0;
+    ui->ui_next = ui->ui_prev = NULL;
     ui->ui_x1 = 0;
     ui->ui_pr = IPPROTO_UDP;
     ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
@@ -449,7 +496,7 @@ kdp_set_interface(void *ifp)
 }
 
 void *
-kdp_get_interface()
+kdp_get_interface(void)
 {
 	return kdp_current_ifp;
 }
@@ -461,6 +508,10 @@ kdp_set_ip_and_mac_addresses(
 {
 	kdp_current_ip_address = ipaddr->s_addr;
 	kdp_current_mac_address = *macaddr;
+	if ((current_debugger == KDP_CUR_DB) && halt_in_debugger) {
+		kdp_call();
+		halt_in_debugger=0;
+	}
 }
 
 void
@@ -857,6 +908,7 @@ kdp_connection_wait(void)
             
 	printf("\nWaiting for remote debugger connection.\n");
 
+
 	if (reattach_wait == 0) {
 		if((kdp_flag & KDP_GETC_ENA) && (0 != kdp_getc()))
 		{
@@ -992,6 +1044,14 @@ kdp_raise_exception(
 {
     int			index;
 
+    /* Was a system trace requested ? */
+    if (kdp_snapshot && (!panic_active()) && (panic_caller == 0)) {
+	    stack_snapshot_ret = kdp_stackshot(stack_snapshot_pid,
+	    stack_snapshot_buf, stack_snapshot_bufsize,
+	    stack_snapshot_options, &stack_snapshot_bytes_traced);
+	    return;
+    }
+
     disable_preemption();
 
     if (saved_state == 0) 
@@ -1017,14 +1077,6 @@ kdp_raise_exception(
     if (pkt.input)
 	kdp_panic("kdp_raise_exception");
 
-    /* Was a system trace requested ? */
-    if (kdp_snapshot && (panicstr == ((char *) 0)) && (panic_caller == 0) && !kdp.is_conn) {
-	    /* XXX This should be reworked to take a pointer to the buffer */
-	    stack_snapshot_ret = kdp_stackshot(stack_snapshot_pid,
-	    (uint32_t) stack_snapshot_buf, stack_snapshot_bufsize,
-	    stack_snapshot_options, &stack_snapshot_bytes_traced);
-	    goto exit_raise_exception;
-    }
 	    
     if (((kdp_flag & KDP_PANIC_DUMP_ENABLED) || (kdp_flag & PANIC_LOG_DUMP))
 	&& (panicstr != (char *) 0)) {
@@ -1035,7 +1087,7 @@ kdp_raise_exception(
       if ((kdp_flag & PANIC_CORE_ON_NMI) && (panicstr == (char *) 0) &&
 	  !kdp.is_conn) {
 
-	disableDebugOuput = disableConsoleOutput = FALSE;
+	disable_debug_output = disableConsoleOutput = FALSE;
 	kdp_panic_dump();
 
 	if (!(kdp_flag & DBG_POST_CORE))
@@ -1072,6 +1124,7 @@ kdp_raise_exception(
 	kdp_flag &= ~PANIC_LOG_DUMP;
 	kdp_flag |= KDP_PANIC_DUMP_ENABLED;
 	kdp_panic_dump();
+	kdp_trigger_core_dump = 0;
       }
 
 /* Trigger a reboot if the user has set this flag through the
@@ -1084,7 +1137,13 @@ kdp_raise_exception(
 	    /* If we're still around, reset the flag */
 	    flag_kdp_trigger_reboot = 0;
     }
-	    
+
+    if (kdp_reentry_deadline) {
+	    kdp_schedule_debugger_reentry(kdp_reentry_deadline);
+	    printf("Debugger re-entry scheduled in %d milliseconds\n", kdp_reentry_deadline);
+	    kdp_reentry_deadline = 0;
+    }
+
     kdp_sync_cache();
 
     if (reattach_wait == 1)
@@ -1122,7 +1181,7 @@ create_panic_header(unsigned int request, const char *corename,
 #else
 	ui = (struct udpiphdr *)&pkt.data[pkt.off];
 #endif
-	ui->ui_next = ui->ui_prev = 0;
+	ui->ui_next = ui->ui_prev = NULL;
 	ui->ui_x1 = 0;
 	ui->ui_pr = IPPROTO_UDP;
 	ui->ui_len = htons((u_short)pkt.len + sizeof (struct udphdr));
@@ -1159,14 +1218,12 @@ create_panic_header(unsigned int request, const char *corename,
   
 	if (request == KDP_WRQ)
 	{
-		register char *cp;
+		char *cp;
 
 		cp = coreh->th_u.tu_rpl;
-		strcpy (cp, corename);
-		cp += strlen(corename);
+		cp += strlcpy (cp, corename, KDP_MAXPACKET);
 		*cp++ = '\0';
-		strcpy (cp, mode);
-		cp+= modelen;
+		cp += strlcpy (cp, mode, KDP_MAXPACKET - strlen(corename));
 		*cp++ = '\0';
 	}
 	else
@@ -1336,7 +1393,7 @@ strnstr(char *s, const char *find, size_t slen)
     } while (strncmp(s, find, len) != 0);
     s--;
   }
-  return ((char *)s);
+  return (s);
 }
 
 extern char version[];
@@ -1353,13 +1410,12 @@ extern char version[];
 static int 
 kdp_get_xnu_version(char *versionbuf)
 {
-
 	char *versionpos;
 	char vstr[20];
 	int retval = -1;
 	char *vptr;
 
-	strcpy(vstr, "custom");
+	strlcpy(vstr, "custom", 10);
 	if (version) {
 		if (kdp_vm_read(version, versionbuf, 95)) {
 			versionbuf[94] = '\0';
@@ -1378,7 +1434,7 @@ kdp_get_xnu_version(char *versionbuf)
 			}
 		}
 	}
-	strcpy(versionbuf, vstr);
+	strlcpy(versionbuf, vstr, KDP_MAXPACKET);
 	return retval;
 }
 
@@ -1387,7 +1443,7 @@ extern int snprintf(char *str, size_t size, const char *format, ...);
 
 /* Primary dispatch routine for the system dump */
 void 
-kdp_panic_dump()
+kdp_panic_dump(void)
 {
 	char corename[50];
 	char coreprefix[10];
diff --git a/osfmk/kdp/kdp_udp.h b/osfmk/kdp/kdp_udp.h
index 040c1d83c..debbc571f 100644
--- a/osfmk/kdp/kdp_udp.h
+++ b/osfmk/kdp/kdp_udp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 1982, 1986, 1993
diff --git a/osfmk/kdp/ml/i386/kdp_machdep.c b/osfmk/kdp/ml/i386/kdp_machdep.c
index 09e93e2b4..07c3145a2 100644
--- a/osfmk/kdp/ml/i386/kdp_machdep.c
+++ b/osfmk/kdp/ml/i386/kdp_machdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include <mach_kdp.h>
@@ -28,6 +34,7 @@
 #include <i386/trap.h>
 #include <i386/mp.h>
 #include <kdp/kdp_internal.h>
+#include <kdp/kdp_callout.h>
 #include <mach-o/loader.h>
 #include <mach-o/nlist.h>
 #include <IOKit/IOPlatformExpert.h> /* for PE_halt_restart */
@@ -37,6 +44,7 @@
 #include <i386/thread.h>
 #include <vm/vm_map.h>
 #include <i386/pmap.h>
+#include <kern/kalloc.h>
 
 #define KDP_TEST_HARNESS 0
 #if KDP_TEST_HARNESS
@@ -57,13 +65,18 @@ void		kdp_setstate(i386_thread_state_t *);
 void		kdp_print_phys(int);
 
 int
-machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
 
 int
-machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
+
+unsigned
+machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len);
 
 extern unsigned kdp_vm_read(caddr_t src, caddr_t dst, unsigned len);
 
+static void	kdp_callouts(kdp_event_t event);
+
 void
 kdp_exception(
     unsigned char	*pkt,
@@ -193,11 +206,10 @@ kdp_machine_read_regs(
     __unused int *size
 )
 {
-    static struct i386_float_state  null_fpstate;
+    static x86_float_state32_t  null_fpstate;
 
     switch (flavor) {
 
-    case OLD_i386_THREAD_STATE:
     case x86_THREAD_STATE32:
 	dprintf(("kdp_readregs THREAD_STATE\n"));
 	kdp_getstate((x86_thread_state32_t *)data);
@@ -227,7 +239,6 @@ kdp_machine_write_regs(
 {
     switch (flavor) {
 
-    case OLD_i386_THREAD_STATE:
     case x86_THREAD_STATE32:
 	dprintf(("kdp_writeregs THREAD_STATE\n"));
 	kdp_setstate((x86_thread_state32_t *)data);
@@ -267,7 +278,11 @@ kdp_machine_hostinfo(
 
 void
 kdp_panic(
+#if CONFIG_NO_KPRINTF_STRINGS
+    __unused const char		*msg
+#else
     const char		*msg
+#endif
 )
 {
     kprintf("kdp panic: %s\n", msg);    
@@ -299,7 +314,7 @@ kdp_intr_enbl(int s)
 }
 
 int
-kdp_getc()
+kdp_getc(void)
 {
 	return	cnmaygetc();
 }
@@ -319,18 +334,18 @@ void print_saved_state(void *state)
 	kprintf("pc = 0x%x\n", saved_state->eip);
 	kprintf("cr2= 0x%x\n", saved_state->cr2);
 	kprintf("rp = TODO FIXME\n");
-	kprintf("sp = 0x%x\n", saved_state);
+	kprintf("sp = %p\n", saved_state);
 
 }
 
 void
-kdp_sync_cache()
+kdp_sync_cache(void)
 {
 	return;	/* No op here. */
 }
 
 void
-kdp_call()
+kdp_call(void)
 {
 	__asm__ volatile ("int	$3");	/* Let the processor do the work */
 }
@@ -367,7 +382,7 @@ kdp_print_phys(int src)
 
 boolean_t
 kdp_i386_trap(
-    unsigned int	trapno,
+    unsigned int		trapno,
     x86_saved_state32_t	*saved_state,
     kern_return_t	result,
     vm_offset_t		va
@@ -376,13 +391,19 @@ kdp_i386_trap(
     unsigned int exception, subcode = 0, code;
 
     if (trapno != T_INT3 && trapno != T_DEBUG) {
-    	kprintf("unexpected kernel trap 0x%x eip 0x%x cr2 0x%x \n",
+    	kprintf("Debugger: Unexpected kernel trap number: "
+		"0x%x, EIP: 0x%x, CR2: 0x%x\n",
 		trapno, saved_state->eip, saved_state->cr2);
 	if (!kdp.is_conn)
 	    return FALSE;
     }	
 
     mp_kdp_enter();
+    kdp_callouts(KDP_EVENT_ENTER);
+
+    if (saved_state->efl & EFL_TF) {
+	    enable_preemption_no_check();
+    }
 
     switch (trapno) {
     
@@ -448,7 +469,13 @@ kdp_i386_trap(
     }
 
     kdp_raise_exception(exception, code, subcode, saved_state);
+    /* If the instruction single step bit is set, disable kernel preemption
+     */
+    if (saved_state->efl & EFL_TF) {
+	    disable_preemption();
+    }
 
+    kdp_callouts(KDP_EVENT_EXIT);
     mp_kdp_exit();
 
     return TRUE;
@@ -467,10 +494,11 @@ kdp_ml_get_breakinsn(void)
   return 0xcc;
 }
 extern pmap_t kdp_pmap;
+extern uint32_t kdp_src_high32;
 
 #define RETURN_OFFSET 4
 int
-machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p)
 {
 	uint32_t *tracebuf = (uint32_t *)tracepos;
 	uint32_t fence = 0;
@@ -486,8 +514,8 @@ machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, in
 		
 		iss32 = USER_REGS32(thread);
 
-		init_eip = iss32->eip;
-		stackptr = iss32->ebp;
+			init_eip = iss32->eip;
+			stackptr = iss32->ebp;
 
 		/* This bound isn't useful, but it doesn't hinder us*/
 		stacklimit = 0xffffffff;
@@ -503,7 +531,7 @@ machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, in
 
 	for (framecount = 0; framecount < nframes; framecount++) {
 
-		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+		if ((uint32_t)(tracebound - ((char *)tracebuf)) < (4 * framesize)) {
 			tracebuf--;
 			break;
 		}
@@ -539,11 +567,142 @@ machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, in
 
 	kdp_pmap = 0;
 
-	return ((uint32_t) tracebuf - tracepos);
+	return (uint32_t) (((char *) tracebuf) - tracepos);
+}
+
+#define RETURN_OFFSET64	8
+/* Routine to encapsulate the 64-bit address read hack*/
+unsigned
+machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len)
+{
+	uint32_t kdp_vm_read_low32;
+	unsigned retval;
+	
+	kdp_src_high32 = srcaddr >> 32;
+	kdp_vm_read_low32 = srcaddr & 0x00000000FFFFFFFFUL;
+	retval = kdp_vm_read((caddr_t)kdp_vm_read_low32, dstaddr, len);
+	kdp_src_high32 = 0;
+	return retval;
 }
 
-/* This is a stub until the x86 64-bit model becomes clear */
 int
-machine_trace_thread64(__unused thread_t thread, __unused uint32_t tracepos, __unused uint32_t tracebound, __unused int nframes, __unused boolean_t user_p) {
-	return 0;
+machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p)
+{
+	uint64_t *tracebuf = (uint64_t *)tracepos;
+	uint32_t fence = 0;
+	addr64_t stackptr = 0;
+	uint64_t stacklimit = 0xfc000000;
+	int framecount = 0;
+	addr64_t init_rip = 0;
+	addr64_t prevsp = 0;
+	unsigned framesize = 2 * sizeof(addr64_t);
+	
+	if (user_p) {
+		x86_saved_state64_t	*iss64;
+		iss64 = USER_REGS64(thread);
+		init_rip = iss64->isf.rip;
+		stackptr = iss64->rbp;
+		stacklimit = 0xffffffffffffffffULL;
+		kdp_pmap = thread->task->map->pmap;
+	}
+	else {
+		/* DRK: This would need to adapt for a 64-bit kernel, if any */
+		stackptr = STACK_IKS(thread->kernel_stack)->k_ebp;
+		init_rip = STACK_IKS(thread->kernel_stack)->k_eip;
+	}
+
+	*tracebuf++ = init_rip;
+
+	for (framecount = 0; framecount < nframes; framecount++) {
+
+		if ((uint32_t)(tracebound - ((char *)tracebuf)) < (4 * framesize)) {
+			tracebuf--;
+			break;
+		}
+
+		*tracebuf++ = stackptr;
+
+		if (!stackptr || (stackptr == fence)){
+			break;
+		}
+		if (stackptr < prevsp) {
+			break;
+		}
+		if (stackptr & 0x0000003) {
+			break;
+		}
+		if (stackptr > stacklimit) {
+			break;
+		}
+
+		if (machine_read64(stackptr + RETURN_OFFSET64, (caddr_t) tracebuf, sizeof(addr64_t)) != sizeof(addr64_t)) {
+			break;
+		}
+		tracebuf++;
+
+		prevsp = stackptr;
+		if (machine_read64(stackptr, (caddr_t) &stackptr, sizeof(addr64_t)) != sizeof(addr64_t)) {
+			*tracebuf++ = 0;
+			break;
+		}
+	}
+
+	kdp_pmap = NULL;
+
+	return (uint32_t) (((char *) tracebuf) - tracepos);
+}
+
+static struct kdp_callout {
+	struct kdp_callout	*callout_next;
+	kdp_callout_fn_t	callout_fn;
+	void			*callout_arg;
+} *kdp_callout_list = NULL;
+
+
+/*
+ * Called from kernel context to register a kdp event callout.
+ */
+void
+kdp_register_callout(
+	kdp_callout_fn_t	fn,
+	void			*arg)
+{
+	struct kdp_callout	*kcp;
+	struct kdp_callout	*list_head;
+
+	kcp = kalloc(sizeof(*kcp));
+	if (kcp == NULL)
+		panic("kdp_register_callout() kalloc failed");
+
+	kcp->callout_fn  = fn;
+	kcp->callout_arg = arg;
+
+	/* Lock-less list insertion using compare and exchange. */
+	do {
+		list_head = kdp_callout_list;
+		kcp->callout_next = list_head;
+	} while(!atomic_cmpxchg((uint32_t *) &kdp_callout_list,
+				(uint32_t) list_head,
+				(uint32_t) kcp));
+}
+
+/*
+ * Called at exception/panic time when extering or exiting kdp.  
+ * We are single-threaded at this time and so we don't use locks.
+ */
+static void
+kdp_callouts(kdp_event_t event)
+{
+	struct kdp_callout	*kcp = kdp_callout_list;
+
+	while (kcp) {
+		kcp->callout_fn(kcp->callout_arg, event); 
+		kcp = kcp->callout_next;
+	}	
+}
+
+void
+kdp_ml_enter_debugger(void)
+{
+	__asm__ __volatile__("int3");
 }
diff --git a/osfmk/kdp/ml/i386/kdp_vm.c b/osfmk/kdp/ml/i386/kdp_vm.c
index 3d1aaa7e4..9b3b85ad9 100644
--- a/osfmk/kdp/ml/i386/kdp_vm.c
+++ b/osfmk/kdp/ml/i386/kdp_vm.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/mach_types.h>
 #include <mach/vm_attributes.h>
@@ -41,8 +47,9 @@
 unsigned kdp_vm_read( caddr_t, caddr_t, unsigned);
 unsigned kdp_vm_write( caddr_t, caddr_t, unsigned);
 
-boolean_t kdp_trans_off = 0;
-uint32_t kdp_src_high32 = 0;
+boolean_t kdp_read_io;
+boolean_t kdp_trans_off;
+uint32_t kdp_src_high32;
 extern pmap_paddr_t avail_start, avail_end;
 
 extern void bcopy_phys(addr64_t from, addr64_t to, int size);
@@ -86,8 +93,7 @@ kdp_vtophys(
 {
 	addr64_t    pa;
 	ppnum_t pp;
-/* Clear high 32 - pmap_find_phys() may panic() otherwise */
-	va &= 0xFFFFFFFFULL;
+
 	pp = pmap_find_phys(pmap, va);
 	if(!pp) return 0;
 	
@@ -131,8 +137,8 @@ unsigned kdp_vm_read(
 		if(!(cur_phys_dst = kdp_vtophys(kernel_pmap, cur_virt_dst)))
 			goto exit;
 
-		/* Validate physical page numbers when performing a crashdump */
-		if (not_in_kdp == 0)
+		/* Validate physical page numbers unless kdp_read_io is set */
+		if (kdp_read_io == FALSE)
 			if (!pmap_valid_page(i386_btop(cur_phys_dst)) || !pmap_valid_page(i386_btop(cur_phys_src)))
 				goto exit;
 
diff --git a/osfmk/kdp/ml/ppc/kdp_asm.s b/osfmk/kdp/ml/ppc/kdp_asm.s
index 764b48580..cdc0cfc5f 100644
--- a/osfmk/kdp/ml/ppc/kdp_asm.s
+++ b/osfmk/kdp/ml/ppc/kdp_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach_kdb.h>
 #include <mach_kgdb.h>
diff --git a/osfmk/kdp/ml/ppc/kdp_machdep.c b/osfmk/kdp/ml/ppc/kdp_machdep.c
index f2d2df2e5..14a88fcb0 100644
--- a/osfmk/kdp/ml/ppc/kdp_machdep.c
+++ b/osfmk/kdp/ml/ppc/kdp_machdep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include <mach/mach_types.h>
@@ -26,10 +32,10 @@
 #include <ppc/proc_reg.h>
 #include <kdp/kdp_internal.h>
 #include <ppc/savearea.h>
+#include <ppc/misc_protos.h>
 #include <kern/debug.h>
 #include <IOKit/IOPlatformExpert.h>
 
-
 #include <kern/thread.h>
 #include <ppc/thread.h>
 #include <vm/vm_map.h>
@@ -44,7 +50,6 @@
 
 void print_saved_state(void *);
 void kdp_call(void);
-void kdp_trap( unsigned int, struct savearea	*saved_state);
 int kdp_getc(void);
 boolean_t kdp_call_kdb(void);
 
@@ -55,10 +60,10 @@ extern uint32_t kdp_src_high32;
 extern unsigned kdp_vm_read(caddr_t src, caddr_t dst, unsigned len);
 
 int
-machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
 
 int
-machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p);
+machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p);
 
 unsigned
 machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len);
@@ -111,7 +116,7 @@ kdp_exception_ack(
     kdp_exception_ack_t	aligned_pkt;
     kdp_exception_ack_t	*rq = (kdp_exception_ack_t *)&aligned_pkt;
 
-    if (len < sizeof (*rq))
+    if ((size_t)len < sizeof (*rq))
 	return(FALSE);
 	
     bcopy((char *)pkt, (char *)rq, sizeof(*rq));
@@ -232,7 +237,7 @@ kdp_getintegerstate64(
 
 kdp_error_t
 kdp_machine_read_regs(
-    unsigned int cpu,
+    __unused unsigned int cpu,
     unsigned int flavor,
     char *data,
     int *size
@@ -364,10 +369,10 @@ kdp_setintegerstate64(
 
 kdp_error_t
 kdp_machine_write_regs(
-    unsigned int cpu,
+    __unused unsigned int cpu,
     unsigned int flavor,
     char *data,
-    int *size
+    __unused int *size
 )
 {
     switch (flavor) {
@@ -431,6 +436,7 @@ kdp_panic(
     while(1) {}
 }
 
+extern void halt_all_cpus(boolean_t);
 
 void
 kdp_reboot(void)
@@ -458,8 +464,6 @@ kdp_intr_enbl(int s)
 void
 kdp_us_spin(int usec)
 {
-    extern void delay(int);
-
     delay(usec/100);
 }
 
@@ -477,7 +481,7 @@ void print_saved_state(void *state)
 }
 
 void
-kdp_call()
+kdp_call(void)
 {
 	Debugger("inline call to debugger(machine_startup)");
 }
@@ -539,7 +543,7 @@ int kdp_trap_codes[] = {
 };
 
 int
-kdp_getc()
+kdp_getc(void)
 {
 	return(cnmaygetc());
 }
@@ -559,7 +563,6 @@ kdp_trap(
 {
 	unsigned int *fp;
 	unsigned int sp;
-	struct savearea *state;
 
 	if (kdp_noisy) {
 		if (kdp_backtrace) {
@@ -607,35 +610,34 @@ kdp_call_kdb(
 	return(TRUE);
 }
 
-void kdp_print_registers(struct savearea *state)
+static void kdp_print_registers(struct savearea *state)
 {
 	int i;
 	for (i=0; i<32; i++) {
 		if ((i % 8) == 0)
 			printf("\n%4d :",i);
-			printf(" %08x",*(&state->save_r0+i));
+			printf(" %08llx",*(&state->save_r0+i));
 	}
 	printf("\n");
 	printf("cr        = 0x%08x\t\t",state->save_cr);
-	printf("xer       = 0x%08x\n",state->save_xer);
-	printf("lr        = 0x%08x\t\t",state->save_lr);
-	printf("ctr       = 0x%08x\n",state->save_ctr);
-	printf("srr0(iar) = 0x%08x\t\t",state->save_srr0);
+	printf("xer       = 0x%08llx\n",state->save_xer);
+	printf("lr        = 0x%08llx\t\t",state->save_lr);
+	printf("ctr       = 0x%08llx\n",state->save_ctr);
+	printf("srr0(iar) = 0x%08llx\t\t",state->save_srr0);
 	printf("srr1(msr) = 0x%08B\n",state->save_srr1,
 		"\x10\x11""EE\x12PR\x13""FP\x14ME\x15""FE0\x16SE\x18"
 		"FE1\x19""AL\x1a""EP\x1bIT\x1c""DT");
 	printf("\n");
 }
 
+void kdp_print_backtrace(unsigned, struct savearea *);
+
 void
 kdp_print_backtrace(
     unsigned int                exception,
     struct savearea     *saved_state)
 {
-	extern void kdp_print_registers(struct savearea *);
-	extern void print_backtrace(struct savearea *);
-
-	disableDebugOuput = FALSE;
+	disable_debug_output = FALSE;
 	debug_mode = TRUE;
 	printf("re-entering kdp:\n");
 	printf("vector=%x, \n", exception/4);
@@ -653,7 +655,7 @@ unsigned int kdp_ml_get_breakinsn(void)
 #define LR_OFFSET64 16
 
 int
-machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+machine_trace_thread(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p)
 {
 	uint32_t *tracebuf = (uint32_t *)tracepos;
 	uint32_t fence = 0;
@@ -681,7 +683,7 @@ machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, in
 
 	for (framecount = 0; framecount < nframes; framecount++) {
 /* Bounds check */
-		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+		if ((uint32_t) (tracebound - ((char *)tracebuf)) < (4 * framesize)) {
 			tracebuf--;
 			break;
 		}
@@ -716,8 +718,8 @@ machine_trace_thread(thread_t thread, uint32_t tracepos, uint32_t tracebound, in
 		}
 	}
 /* Reset the target pmap */
-	kdp_pmap = 0;
-	return ((uint32_t) tracebuf - tracepos);
+	kdp_pmap = NULL;
+	return (uint32_t) (((char *) tracebuf) - tracepos);
 }
 
 /* Routine to encapsulate the 64-bit address read hack*/
@@ -735,7 +737,7 @@ machine_read64(addr64_t srcaddr, caddr_t dstaddr, uint32_t len)
 }
 
 int
-machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound, int nframes, boolean_t user_p)
+machine_trace_thread64(thread_t thread, char *tracepos, char *tracebound, int nframes, boolean_t user_p)
 {
 	uint64_t *tracebuf = (uint64_t *)tracepos;
 	uint32_t fence = 0;
@@ -761,7 +763,7 @@ machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound,
 
 	for (framecount = 0; framecount < nframes; framecount++) {
 
-		if ((tracebound - ((uint32_t) tracebuf)) < (4 * framesize)) {
+		if ((uint32_t)(tracebound - ((char *)tracebuf)) < (4 * framesize)) {
 			tracebuf--;
 			break;
 		}
@@ -793,7 +795,13 @@ machine_trace_thread64(thread_t thread, uint32_t tracepos, uint32_t tracebound,
 		}
 	}
 
-	kdp_pmap = 0;
+	kdp_pmap = NULL;
+	return (uint32_t) (((char *) tracebuf) - tracepos);
+}
+
 
-	return ((uint32_t) tracebuf - tracepos);
+void
+kdp_ml_enter_debugger(void)
+{
+	__asm__ __volatile__("tw 4,r3,r3");
 }
diff --git a/osfmk/kdp/ml/ppc/kdp_misc.s b/osfmk/kdp/ml/ppc/kdp_misc.s
index 4bfd0d049..a007a296b 100644
--- a/osfmk/kdp/ml/ppc/kdp_misc.s
+++ b/osfmk/kdp/ml/ppc/kdp_misc.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <debug.h>
 #include <ppc/asm.h>
diff --git a/osfmk/kdp/ml/ppc/kdp_vm.c b/osfmk/kdp/ml/ppc/kdp_vm.c
index 12f9c9fd1..2c0120e69 100644
--- a/osfmk/kdp/ml/ppc/kdp_vm.c
+++ b/osfmk/kdp/ml/ppc/kdp_vm.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/mach_types.h>
 #include <mach/vm_attributes.h>
@@ -31,6 +37,7 @@
 #include <ppc/pmap.h>
 #include <ppc/mappings.h> 
 #include <ppc/cpu_data.h>
+#include <ppc/misc_protos.h>
 
 #include <mach/thread_status.h>
 #include <mach-o/loader.h>
@@ -48,10 +55,10 @@
 #include <mach/vm_map.h>
 
 
-pmap_t kdp_pmap=0;
-boolean_t kdp_trans_off=0;
-boolean_t kdp_read_io  =0;
-uint32_t kdp_src_high32 = 0;
+pmap_t kdp_pmap;
+boolean_t kdp_trans_off;
+boolean_t kdp_read_io;
+uint32_t kdp_src_high32;
 
 unsigned kdp_vm_read( caddr_t, caddr_t, unsigned);
 unsigned kdp_vm_write( caddr_t, caddr_t, unsigned);
@@ -59,14 +66,12 @@ unsigned kdp_vm_write( caddr_t, caddr_t, unsigned);
 extern vm_offset_t sectTEXTB, sectDATAB, sectLINKB, sectPRELINKB;
 extern int sectSizeTEXT, sectSizeDATA, sectSizeLINK, sectSizePRELINK;
 
-/* XXX prototypes which should be in a commmon header file */
-addr64_t	kdp_vtophys(pmap_t pmap, addr64_t va);
-int		kern_dump(void);
-int		kdp_dump_trap(int type, struct savearea *regs);
+static addr64_t	kdp_vtophys(pmap_t pmap, addr64_t va);
+int             kern_dump(void);
 
 typedef struct {
   int	flavor;			/* the number for this flavor */
-  int	count;			/* count of ints in this flavor */
+  mach_msg_type_number_t	count;	/* count of ints in this flavor */
 } mythread_state_flavor_t;
 
 static mythread_state_flavor_t thread_flavor_array[] = {
@@ -90,7 +95,7 @@ char command_buffer[512];
 /*
  *
  */
-addr64_t
+static addr64_t
 kdp_vtophys(
 	pmap_t pmap,
 	addr64_t va)
@@ -290,7 +295,7 @@ kdp_dump_trap(
 	      __unused struct savearea *regs)
 {
   printf ("An unexpected trap (type %d) occurred during the kernel dump, terminating.\n", type);
-  kdp_send_crashdump_pkt (KDP_EOF, NULL, 0, ((void *) 0));
+  kdp_send_crashdump_pkt(KDP_EOF, NULL, 0, ((void *) 0));
   abort_panic_transfer();
   kdp_flag &= ~KDP_PANIC_DUMP_ENABLED;
   kdp_flag &= ~PANIC_CORE_ON_NMI;
@@ -325,10 +330,10 @@ kern_dump(void)
   mythread_state_flavor_t flavors[MAX_TSTATE_FLAVORS];
   vm_size_t	nflavors;
   vm_size_t	i;
-  int nesting_depth = 0;
+  uint32_t nesting_depth = 0;
   kern_return_t	kret = 0;
   struct vm_region_submap_info_64 vbr;
-  int vbrcount  = 0;
+  mach_msg_type_number_t vbrcount  = 0;
   tir_t tir1;
 
   int panic_error = 0;
@@ -390,17 +395,17 @@ kern_dump(void)
    * to begin data transmission 
    */
 
-   if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { 
+   if ((panic_error = kdp_send_crashdump_pkt(KDP_SEEK, NULL, sizeof(nfoffset) , &nfoffset)) < 0) { 
      printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); 
      return -1; 
    } 
 
-   if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct mach_header), (caddr_t) mh) < 0)) {
+   if ((panic_error = kdp_send_crashdump_data(KDP_DATA, NULL, sizeof(struct mach_header), (caddr_t) mh) < 0)) {
      printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
      return -1 ;
    }
 
-   if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) {
+   if ((panic_error = kdp_send_crashdump_pkt(KDP_SEEK, NULL, sizeof(foffset) , &foffset) < 0)) {
      printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
      return (-1);
    }
@@ -492,12 +497,12 @@ kern_dump(void)
     sc->initprot = prot;
     sc->nsects = 0;
 
-    if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
+    if ((panic_error = kdp_send_crashdump_pkt(KDP_SEEK, NULL, sizeof(hoffset) , &hoffset)) < 0) { 
 	printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error); 
 	return -1; 
       } 
     
-    if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, sizeof(struct segment_command) , (caddr_t) sc)) < 0) {
+    if ((panic_error = kdp_send_crashdump_data(KDP_DATA, NULL, sizeof(struct segment_command) , (caddr_t) sc)) < 0) {
 	printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
 	return -1 ;
       }
@@ -508,14 +513,14 @@ kern_dump(void)
 
     if ((vbr.user_tag != VM_MEMORY_IOKIT)) {
       
-      if ((panic_error = kdp_send_crashdump_pkt (KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) {
+      if ((panic_error = kdp_send_crashdump_pkt(KDP_SEEK, NULL, sizeof(foffset) , &foffset)) < 0) {
 	  printf ("kdp_send_crashdump_pkt failed with error %d\n", panic_error);
 	  return (-1);
 	}
 
       txstart = vmoffset;
 
-      if ((panic_error = kdp_send_crashdump_data (KDP_DATA, NULL, size, (caddr_t) txstart)) < 0)	{
+      if ((panic_error = kdp_send_crashdump_data(KDP_DATA, NULL, size, (caddr_t) txstart)) < 0)	{
 	  printf ("kdp_send_crashdump_data failed with error %d\n", panic_error);
 	  return -1 ;
 	}
diff --git a/osfmk/kern/Makefile b/osfmk/kern/Makefile
index 383bac18c..c07d5f29b 100644
--- a/osfmk/kern/Makefile
+++ b/osfmk/kern/Makefile
@@ -10,6 +10,7 @@ include $(MakeInc_def)
 DATAFILES =
 
 EXPORT_ONLY_FILES = \
+	affinity.h \
 	assert.h \
 	clock.h \
 	cpu_number.h \
@@ -24,8 +25,8 @@ EXPORT_ONLY_FILES = \
 	host.h \
 	mach_param.h \
 	macro_help.h \
-	pms.h \
 	page_decrypt.h \
+	pms.h \
 	processor.h \
 	queue.h \
 	sched_prim.h \
diff --git a/osfmk/kern/affinity.c b/osfmk/kern/affinity.c
new file mode 100644
index 000000000..1b319c753
--- /dev/null
+++ b/osfmk/kern/affinity.c
@@ -0,0 +1,552 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <kern/affinity.h>
+#include <kern/task.h>
+#include <kern/kalloc.h>
+#include <machine/cpu_affinity.h>
+
+/*
+ * Affinity involves 2 objects:
+ * - affinity namespace:
+ *	shared by a task family, this controls affinity tag lookup and
+ *	allocation; it anchors all affinity sets in one namespace
+ * - affinity set:
+ * 	anchors all threads with membership of this affinity set
+ *	and which share an affinity tag in the owning namespace.
+ * 
+ * Locking:
+ * - The task lock protects the creation of an affinity namespace.
+ * - The affinity namespace mutex protects the inheritance of a namespace
+ *   and its thread membership. This includes its destruction when the task
+ *   reference count goes to zero.
+ * - The thread mutex protects a thread's affinity set membership, but in
+ *   addition, the thread_lock is taken to write thread->affinity_set since this
+ *   field (representng the active affinity set) is read by the scheduler.
+ * 
+ * The lock ordering is: task lock, thread mutex, namespace mutex, thread lock.
+ */
+
+#if AFFINITY_DEBUG
+#define DBG(x...)	kprintf("DBG: " x)
+#else
+#define DBG(x...)
+#endif
+
+struct affinity_space {
+	mutex_t		aspc_lock;
+	uint32_t	aspc_task_count;
+	queue_head_t	aspc_affinities;
+};
+typedef struct affinity_space *affinity_space_t;
+
+static affinity_space_t affinity_space_alloc(void);
+static void affinity_space_free(affinity_space_t aspc);
+static affinity_set_t affinity_set_alloc(void);
+static void affinity_set_free(affinity_set_t aset);
+static affinity_set_t affinity_set_find(affinity_space_t aspc, uint32_t tag);
+static void affinity_set_place(affinity_space_t aspc, affinity_set_t aset);
+static void affinity_set_add(affinity_set_t aset, thread_t thread);
+static affinity_set_t affinity_set_remove(affinity_set_t aset, thread_t thread);
+
+/*
+ * The following globals may be modified by the sysctls
+ *   kern.affinity_sets_enabled	- disables hinting if cleared
+ *   kern.affinity_sets_mapping	- controls cache distribution policy
+ * See bsd/kern_sysctl.c
+ */
+boolean_t	affinity_sets_enabled = TRUE;
+int		affinity_sets_mapping = 1;
+
+boolean_t
+thread_affinity_is_supported(void)
+{
+	return (ml_get_max_affinity_sets() != 0);
+}
+
+
+/*
+ * thread_affinity_get() 
+ * Return the affinity tag for a thread. 
+ * Called with the thread mutex held.
+ */
+uint32_t
+thread_affinity_get(thread_t thread)
+{
+	uint32_t tag;
+
+	if (thread->affinity_set != NULL)
+		tag = thread->affinity_set->aset_tag;
+	else
+		tag = THREAD_AFFINITY_TAG_NULL;
+
+	return tag;
+}
+
+
+/*
+ * thread_affinity_set() 
+ * Place a thread in an affinity set identified by a tag.
+ * Called with thread referenced but not locked.
+ */
+kern_return_t
+thread_affinity_set(thread_t thread, uint32_t tag)
+{
+	affinity_set_t		aset;
+	affinity_set_t		empty_aset = NULL;
+	affinity_space_t	aspc;
+	affinity_space_t	new_aspc = NULL;
+
+	DBG("thread_affinity_set(%p,%u)\n", thread, tag);
+
+	task_lock(thread->task);
+	aspc = thread->task->affinity_space;
+	if (aspc == NULL) {
+		task_unlock(thread->task);
+		new_aspc = affinity_space_alloc();
+		if (new_aspc == NULL)
+			return KERN_RESOURCE_SHORTAGE;
+		task_lock(thread->task);
+		if (thread->task->affinity_space == NULL) {
+			thread->task->affinity_space = new_aspc;
+			new_aspc = NULL;
+		}
+		aspc = thread->task->affinity_space;
+	}
+	task_unlock(thread->task);
+	if (new_aspc)
+		affinity_space_free(new_aspc);
+
+	thread_mtx_lock(thread);
+	if (!thread->active) {
+		/* Beaten to lock and the thread is dead */
+		thread_mtx_unlock(thread);
+		return KERN_TERMINATED;
+	}
+
+	mutex_lock(&aspc->aspc_lock);
+	aset = thread->affinity_set;
+	if (aset != NULL) {
+		/*
+		 * Remove thread from current affinity set
+		 */
+		DBG("thread_affinity_set(%p,%u) removing from aset %p\n",
+			thread, tag, aset);
+		empty_aset = affinity_set_remove(aset, thread);
+	}
+
+	if (tag != THREAD_AFFINITY_TAG_NULL) {
+		aset = affinity_set_find(aspc, tag);
+		if (aset != NULL) {
+			/*
+			 * Add thread to existing affinity set
+			 */
+			DBG("thread_affinity_set(%p,%u) found aset %p\n",
+				thread, tag, aset);
+		} else {
+			/*
+			 * Use the new affinity set, add this thread
+			 * and place it in a suitable processor set.
+			 */
+			if (empty_aset != NULL) {
+				aset = empty_aset;
+				empty_aset = NULL;
+			} else {
+				aset = affinity_set_alloc();
+				if (aset == NULL) {
+					mutex_unlock(&aspc->aspc_lock);
+					thread_mtx_unlock(thread);
+					return KERN_RESOURCE_SHORTAGE;
+				}
+			}
+			DBG("thread_affinity_set(%p,%u) (re-)using aset %p\n",
+				thread, tag, aset);
+			aset->aset_tag = tag;
+			affinity_set_place(aspc, aset);
+		}
+		affinity_set_add(aset, thread);
+	}
+
+	mutex_unlock(&aspc->aspc_lock);
+	thread_mtx_unlock(thread);
+
+	/*
+	 * If we wound up not using an empty aset we created,
+	 * free it here.
+	 */
+	if (empty_aset != NULL)
+		affinity_set_free(empty_aset);
+
+	if (thread == current_thread())
+	        thread_block(THREAD_CONTINUE_NULL);
+
+	return KERN_SUCCESS;
+}
+
+/*
+ * task_affinity_create()
+ * Called from task create.
+ */
+void
+task_affinity_create(task_t parent_task, task_t child_task)
+{
+	affinity_space_t	aspc = parent_task->affinity_space;
+
+	DBG("task_affinity_create(%p,%p)\n", parent_task, child_task);
+
+	assert(aspc);
+
+	/*
+	 * Bump the task reference count on the shared namespace and
+	 * give it to the child.
+	 */
+	mutex_lock(&aspc->aspc_lock);
+	aspc->aspc_task_count++;
+	child_task->affinity_space = aspc;
+	mutex_unlock(&aspc->aspc_lock);
+}
+
+/*
+ * task_affinity_deallocate()
+ * Called from task_deallocate() when there's a namespace to dereference.
+ */
+void
+task_affinity_deallocate(task_t	task)
+{
+	affinity_space_t	aspc = task->affinity_space;
+
+	DBG("task_affinity_deallocate(%p) aspc %p task_count %d\n",
+		task, aspc, aspc->aspc_task_count);
+
+	mutex_lock(&aspc->aspc_lock);
+	if (--(aspc->aspc_task_count) == 0) {
+		assert(queue_empty(&aspc->aspc_affinities));
+		mutex_unlock(&aspc->aspc_lock);
+		affinity_space_free(aspc);
+	} else {
+		mutex_unlock(&aspc->aspc_lock);
+	}
+}
+
+/*
+ * task_affinity_info()
+ * Return affinity tag info (number, min, max) for the task.
+ */
+kern_return_t
+task_affinity_info(
+	task_t			task,
+	task_info_t		task_info_out,
+	mach_msg_type_number_t	*task_info_count)
+{
+	affinity_set_t			aset;
+	affinity_space_t		aspc;
+	task_affinity_tag_info_t	info;
+
+	*task_info_count = TASK_AFFINITY_TAG_INFO_COUNT;
+	info = (task_affinity_tag_info_t) task_info_out;
+	info->set_count = 0;
+	info->task_count = 0;
+	info->min = THREAD_AFFINITY_TAG_NULL;
+	info->max = THREAD_AFFINITY_TAG_NULL;
+
+	task_lock(task);
+	aspc = task->affinity_space;
+	if (aspc) {
+		mutex_lock(&aspc->aspc_lock);
+		queue_iterate(&aspc->aspc_affinities,
+				 aset, affinity_set_t, aset_affinities) {	
+			info->set_count++;
+			if (info->min == THREAD_AFFINITY_TAG_NULL ||
+			    aset->aset_tag < (uint32_t) info->min)
+				info->min = aset->aset_tag;
+			if (info->max == THREAD_AFFINITY_TAG_NULL ||
+			    aset->aset_tag > (uint32_t) info->max)
+				info->max = aset->aset_tag;
+		}
+		info->task_count = aspc->aspc_task_count;
+		mutex_unlock(&aspc->aspc_lock);
+	}
+	task_unlock(task);
+	return KERN_SUCCESS;
+}
+
+/*
+ * Called from thread_dup() during fork() with child's mutex held.
+ * Set the child into the parent's affinity set.
+ * Note the affinity space is shared.
+ */
+void
+thread_affinity_dup(thread_t parent, thread_t child)
+{
+	affinity_set_t			aset;
+	affinity_space_t		aspc;
+
+	thread_mtx_lock(parent);
+	aset = parent->affinity_set;
+	DBG("thread_affinity_dup(%p,%p) aset %p\n", parent, child, aset);
+	if (aset == NULL) {
+		thread_mtx_unlock(parent);
+		return;
+	}
+
+	aspc = aset->aset_space;
+	assert(aspc == parent->task->affinity_space);
+	assert(aspc == child->task->affinity_space);
+
+	mutex_lock(&aspc->aspc_lock);
+	affinity_set_add(aset, child);
+	mutex_unlock(&aspc->aspc_lock);
+
+	thread_mtx_unlock(parent);
+}
+
+/*
+ * thread_affinity_terminate() 
+ * Remove thread from any affinity set.
+ * Called with the thread mutex locked.
+ */
+void
+thread_affinity_terminate(thread_t thread)
+{
+	affinity_set_t		aset = thread->affinity_set;
+	affinity_space_t	aspc;
+
+	DBG("thread_affinity_terminate(%p)\n", thread);
+
+	aspc = aset->aset_space;
+	mutex_lock(&aspc->aspc_lock);
+	if (affinity_set_remove(aset, thread)) {
+		affinity_set_free(aset);
+	}
+	mutex_unlock(&aspc->aspc_lock);
+}
+
+/*
+ * Create an empty affinity namespace data structure.
+ */
+static affinity_space_t
+affinity_space_alloc(void) 
+{
+	affinity_space_t	aspc;
+
+	aspc = (affinity_space_t) kalloc(sizeof(struct affinity_space));
+	if (aspc == NULL)
+		return NULL;
+
+	mutex_init(&aspc->aspc_lock, 0);
+	queue_init(&aspc->aspc_affinities);
+	aspc->aspc_task_count = 1;
+
+	DBG("affinity_space_create() returns %p\n", aspc);
+	return aspc;
+}
+
+/*
+ * Destroy the given empty affinity namespace data structure.
+ */
+static void
+affinity_space_free(affinity_space_t aspc)
+{
+	assert(queue_empty(&aspc->aspc_affinities));
+
+	DBG("affinity_space_free(%p)\n", aspc);
+	kfree(aspc, sizeof(struct affinity_space));
+}
+
+
+/*
+ * Create an empty affinity set data structure
+ * entering it into a list anchored by the owning task.
+ */
+static affinity_set_t
+affinity_set_alloc(void) 
+{
+	affinity_set_t	aset;
+
+	aset = (affinity_set_t) kalloc(sizeof(struct affinity_set));
+	if (aset == NULL)
+		return NULL;
+
+	aset->aset_thread_count = 0;
+	queue_init(&aset->aset_affinities);
+	queue_init(&aset->aset_threads);
+	aset->aset_num = 0;
+	aset->aset_pset = PROCESSOR_SET_NULL;
+	aset->aset_space = NULL;
+
+	DBG("affinity_set_create() returns %p\n", aset);
+	return aset;
+}
+
+/*
+ * Destroy the given empty affinity set data structure
+ * after removing it from the parent task.
+ */
+static void
+affinity_set_free(affinity_set_t aset)
+{
+	assert(queue_empty(&aset->aset_threads));
+
+	DBG("affinity_set_free(%p)\n", aset);
+	kfree(aset, sizeof(struct affinity_set));
+}
+
+/*
+ * Add a thread to an affinity set.
+ * The caller must have the thread mutex and space locked.
+ */
+static void
+affinity_set_add(affinity_set_t aset, thread_t thread)
+{
+	spl_t	s;
+
+	DBG("affinity_set_add(%p,%p)\n", aset, thread);
+	queue_enter(&aset->aset_threads,
+		thread, thread_t, affinity_threads);
+	aset->aset_thread_count++;
+	s = splsched();
+	thread_lock(thread);
+	thread->affinity_set = affinity_sets_enabled ? aset : NULL;
+	thread_unlock(thread);
+	splx(s);
+}
+
+/*
+ * Remove a thread from an affinity set returning the set if now empty.
+ * The caller must have the thread mutex and space locked.
+ */
+static affinity_set_t
+affinity_set_remove(affinity_set_t aset, thread_t thread)
+{
+	spl_t	s;
+
+	s = splsched();
+	thread_lock(thread);
+	thread->affinity_set = NULL;
+	thread_unlock(thread);
+	splx(s);
+
+	aset->aset_thread_count--;
+	queue_remove(&aset->aset_threads,
+		thread, thread_t, affinity_threads);
+	if (queue_empty(&aset->aset_threads)) {
+		queue_remove(&aset->aset_space->aspc_affinities,
+				aset, affinity_set_t, aset_affinities);
+		assert(aset->aset_thread_count == 0);
+		aset->aset_tag = THREAD_AFFINITY_TAG_NULL;
+		aset->aset_num = 0;
+		aset->aset_pset = PROCESSOR_SET_NULL;
+		aset->aset_space = NULL;
+		DBG("affinity_set_remove(%p,%p) set now empty\n", aset, thread);
+		return aset;
+	} else {
+		DBG("affinity_set_remove(%p,%p)\n", aset, thread);
+		return NULL;
+	}
+}
+
+/*
+ * Find an affinity set in the parent task with the given affinity tag.
+ * The caller must have the space locked.
+ */
+static affinity_set_t
+affinity_set_find(affinity_space_t space, uint32_t tag)
+{
+	affinity_set_t	aset;
+
+	queue_iterate(&space->aspc_affinities,
+			 aset, affinity_set_t, aset_affinities) {	
+		if (aset->aset_tag == tag) {
+			DBG("affinity_set_find(%p,%u) finds %p\n",
+		 	    space, tag, aset);
+			return aset;
+		}
+	}
+	DBG("affinity_set_find(%p,%u) not found\n", space, tag);
+	return NULL;
+}
+
+/*
+ * affinity_set_place() assigns an affinity set to a suitable processor_set.
+ * The selection criteria is:
+ *  - the set currently occupied by the least number of affinities
+ *    belonging to the owning the task.
+ * The caller must have the space locked.
+ */
+static void
+affinity_set_place(affinity_space_t aspc, affinity_set_t new_aset)
+{
+	unsigned int	num_cpu_asets = ml_get_max_affinity_sets();
+	unsigned int	set_occupancy[num_cpu_asets];
+	unsigned int	i;
+	unsigned int	i_least_occupied;
+	affinity_set_t	aset;
+
+	for (i = 0; i < num_cpu_asets; i++)
+		set_occupancy[i] = 0;
+
+	/*
+	 * Scan the affinity sets calculating the number of sets
+	 * occupy the available physical affinities.
+	 */
+	queue_iterate(&aspc->aspc_affinities,
+			 aset, affinity_set_t, aset_affinities) {
+		set_occupancy[aset->aset_num]++;
+	}
+
+	/*
+	 * Find the least occupied set (or the first empty set).
+	 * To distribute placements somewhat, start searching from
+	 * a cpu affinity chosen randomly per namespace:
+	 *   [(unsigned int)aspc % 127] % num_cpu_asets
+	 * unless this mapping policy is overridden.
+	 */
+	if (affinity_sets_mapping == 0)
+		i_least_occupied = 0;
+	else
+		i_least_occupied = ((unsigned int)aspc % 127) % num_cpu_asets;
+	for (i = 0; i < num_cpu_asets; i++) {
+		unsigned int	j = (i_least_occupied + i) % num_cpu_asets;
+		if (set_occupancy[j] == 0) {
+			i_least_occupied = j;
+			break;
+		}
+		if (set_occupancy[j] < set_occupancy[i_least_occupied])
+			i_least_occupied = j;
+	}
+	new_aset->aset_num = i_least_occupied;
+	new_aset->aset_pset = ml_affinity_to_pset(i_least_occupied);
+
+	/* Add the new affinity set to the group */
+	new_aset->aset_space = aspc;
+	queue_enter(&aspc->aspc_affinities,
+			new_aset, affinity_set_t, aset_affinities);
+
+	DBG("affinity_set_place(%p,%p) selected affinity %u pset %p\n",
+	    aspc, new_aset, new_aset->aset_num, new_aset->aset_pset);
+}
diff --git a/osfmk/kern/affinity.h b/osfmk/kern/affinity.h
new file mode 100644
index 000000000..e634e7122
--- /dev/null
+++ b/osfmk/kern/affinity.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef	_KERN_CPU_AFFINITY_H_
+#define	_KERN_CPU_AFFINITY_H_
+
+#include <kern/queue.h>
+#include <kern/processor.h>
+#include <kern/lock.h>
+
+/*
+ * An affinity set object represents a set of threads identified by the user
+ * to be sharing (cache) affinity. A task may have multiple affinity sets
+ * defined. Each set has dis-affinity other sets. Tasks related by inheritance
+ * may share the same affinity set namespace.
+ * Affinity sets are used to advise (hint) thread placement.
+ */
+struct affinity_set {
+	struct affinity_space *aset_space;	/* namespace */
+	queue_chain_t	aset_affinities;	/* links affinities in group */
+	queue_head_t	aset_threads;		/* threads in affinity set */
+	uint32_t	aset_thread_count;	/* num threads in set */
+	uint32_t	aset_tag;		/* user-assigned tag */
+	uint32_t	aset_num;		/* kernel-assigned affinity */
+	processor_set_t	aset_pset;		/* processor set */
+};
+
+extern boolean_t	thread_affinity_is_supported(void);
+extern kern_return_t	thread_affinity_set(thread_t thread, uint32_t tag);
+extern uint32_t		thread_affinity_get(thread_t thread);
+extern void		thread_affinity_dup(thread_t parent, thread_t child);
+extern void		thread_affinity_terminate(thread_t thread);
+extern void		task_affinity_create(
+					task_t,
+					task_t);
+extern void		task_affinity_deallocate(
+					task_t);
+extern kern_return_t	task_affinity_info(
+					task_t,
+					task_info_t,
+					mach_msg_type_number_t  *);
+
+#endif	/* _KERN_CPU_AFFINITY_H_ */
diff --git a/osfmk/kern/assert.h b/osfmk/kern/assert.h
index fc390b5aa..1b528a4b3 100644
--- a/osfmk/kern/assert.h
+++ b/osfmk/kern/assert.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -65,23 +71,23 @@
 __BEGIN_DECLS
 /* Assert error */
 extern void	Assert(
-			const char	*file,
-			int		line,
-			const char	*expression);
+	const char	*file,
+	int		line,
+	const char	*expression);
 __END_DECLS
 
 #if	MACH_ASSERT
 
 #define assert(ex)  \
-    ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
+	((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
 #define	assert_static(x)	assert(x)
 
 #define __assert_only
 
 #else	/* MACH_ASSERT */
 
-#define assert(ex)		((void)0)
-#define assert_static(ex)
+#define assert(ex) ((void)0)
+#define assert_static(ex) do {} while (0)
 
 #define __assert_only __unused
 
diff --git a/osfmk/kern/ast.c b/osfmk/kern/ast.c
index 4fff10303..8aaff637f 100644
--- a/osfmk/kern/ast.c
+++ b/osfmk/kern/ast.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -184,12 +190,10 @@ void
 ast_check(
 	processor_t		processor)
 {
-	register thread_t		thread = processor->active_thread;
-
-	processor->current_pri = thread->sched_pri;
 	if (	processor->state == PROCESSOR_RUNNING		||
 			processor->state == PROCESSOR_SHUTDOWN		) {
-		register ast_t		preempt;
+		thread_t			thread = processor->active_thread;
+		ast_t				preempt;
 
 		/*
 		 *	Propagate thread ast to processor.
diff --git a/osfmk/kern/ast.h b/osfmk/kern/ast.h
index e6f951ae4..93567aca7 100644
--- a/osfmk/kern/ast.h
+++ b/osfmk/kern/ast.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -176,11 +182,11 @@ MACRO_END
  *  be followed by ast_propagate().
  */
 #define thread_ast_set(act, reason)		\
-						(hw_atomic_or(&(act)->ast, (reason)))
+						((void)hw_atomic_or(&(act)->ast, (reason)))
 #define thread_ast_clear(act, reason)	\
-						(hw_atomic_and(&(act)->ast, ~(reason)))
+						((void)hw_atomic_and(&(act)->ast, ~(reason)))
 #define thread_ast_clear_all(act)		\
-						(hw_atomic_and(&(act)->ast, AST_NONE))
+						((void)hw_atomic_and(&(act)->ast, AST_NONE))
 
 #ifdef MACH_BSD
 
diff --git a/osfmk/kern/bits.c b/osfmk/kern/bits.c
index 085ccd9a6..b6cfb2043 100644
--- a/osfmk/kern/bits.c
+++ b/osfmk/kern/bits.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/bsd_kern.c b/osfmk/kern/bsd_kern.c
index a9a805928..d09e4ea6e 100644
--- a/osfmk/kern/bsd_kern.c
+++ b/osfmk/kern/bsd_kern.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/mach_types.h>
 
@@ -51,9 +57,10 @@ boolean_t thread_should_abort(thread_t);
 boolean_t current_thread_aborted(void);
 void task_act_iterate_wth_args(task_t, void(*)(thread_t, void *), void *);
 void ipc_port_release(ipc_port_t);
-boolean_t is_thread_active(thread_t);
 kern_return_t get_signalact(task_t , thread_t *, int);
 int get_vmsubmap_entries(vm_map_t, vm_object_offset_t, vm_object_offset_t);
+void syscall_exit_funnelcheck(void);
+
 
 /*
  *
@@ -63,6 +70,14 @@ void  *get_bsdtask_info(task_t t)
 	return(t->bsd_info);
 }
 
+/*
+ *
+ */
+void *get_bsdthreadtask_info(thread_t th)
+{
+	return(th->task != TASK_NULL ? th->task->bsd_info : NULL);
+}
+
 /*
  *
  */
@@ -117,15 +132,15 @@ get_signalact(
 
 	for (inc  = (thread_t)queue_first(&task->threads);
 			!queue_end(&task->threads, (queue_entry_t)inc); ) {
-                thread_mtx_lock(inc);
-                if (inc->active  && 
-	                    (inc->state & (TH_ABORT|TH_ABORT_SAFELY)) != TH_ABORT) {
-                    thread = inc;
-					break;
-                }
-                thread_mtx_unlock(inc);
-
-				inc = (thread_t)queue_next(&inc->task_threads);
+		thread_mtx_lock(inc);
+		if (inc->active &&
+				(inc->sched_mode & TH_MODE_ISABORTED) != TH_MODE_ABORT) {
+			thread = inc;
+			break;
+		}
+		thread_mtx_unlock(inc);
+
+		inc = (thread_t)queue_next(&inc->task_threads);
 	}
 
 	if (result_out) 
@@ -169,7 +184,7 @@ check_actforsig(
 			thread_mtx_lock(inc);
 
 			if (inc->active  && 
-				(inc->state & (TH_ABORT|TH_ABORT_SAFELY)) != TH_ABORT) {
+					(inc->sched_mode & TH_MODE_ISABORTED) != TH_MODE_ABORT) {
 				result = KERN_SUCCESS;
 				break;
 			}
@@ -290,43 +305,6 @@ task_t	get_threadtask(thread_t th)
 	return(th->task);
 }
 
-
-/*
- *
- */
-boolean_t is_thread_idle(thread_t th)
-{
-	return((th->state & TH_IDLE) == TH_IDLE);
-}
-
-/*
- *
- */
-boolean_t is_thread_running(thread_t th)
-{
-	return((th->state & TH_RUN) == TH_RUN);
-}
-
-/*
- *
- */
-thread_t
-getshuttle_thread(
-	thread_t	th)
-{
-	return(th);
-}
-
-/*
- *
- */
-thread_t
-getact_thread(
-	thread_t	th)
-{
-	return(th);
-}
-
 /*
  *
  */
@@ -444,7 +422,7 @@ boolean_t
 thread_should_abort(
 	thread_t th)
 {
-	return ((th->state & (TH_ABORT|TH_ABORT_SAFELY)) == TH_ABORT);
+	return ((th->sched_mode & TH_MODE_ISABORTED) == TH_MODE_ABORT);
 }
 
 /*
@@ -462,14 +440,14 @@ current_thread_aborted (
 	thread_t th = current_thread();
 	spl_t s;
 
-	if ((th->state & (TH_ABORT|TH_ABORT_SAFELY)) == TH_ABORT &&
+	if ((th->sched_mode & TH_MODE_ISABORTED) == TH_MODE_ABORT &&
 			(th->options & TH_OPT_INTMASK) != THREAD_UNINT)
 		return (TRUE);
-	if (th->state & TH_ABORT_SAFELY) {
+	if (th->sched_mode & TH_MODE_ABORTSAFELY) {
 		s = splsched();
 		thread_lock(th);
-		if (th->state & TH_ABORT_SAFELY)
-			th->state &= ~(TH_ABORT|TH_ABORT_SAFELY);
+		if (th->sched_mode & TH_MODE_ABORTSAFELY)
+			th->sched_mode &= ~TH_MODE_ISABORTED;
 		thread_unlock(th);
 		splx(s);
 	}
@@ -505,13 +483,6 @@ ipc_port_release(
 	ipc_object_release(&(port)->ip_object);
 }
 
-boolean_t
-is_thread_active(
-	thread_t th)
-{
-	return(th->active);
-}
-
 void
 astbsd_on(void)
 {
@@ -531,13 +502,14 @@ fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo)
 	vm_map_t map;
 	task_absolutetime_info_data_t   tinfo;
 	thread_t thread;
-	int numrunning = 0;
+	int cswitch = 0, numrunning = 0;
 	
 	map = (task == kernel_task)? kernel_map: task->map;
 
 	ptinfo->pti_virtual_size  = map->size;
-	ptinfo->pti_resident_size  = (mach_vm_size_t)(pmap_resident_count(map->pmap)
-				   * PAGE_SIZE);
+	ptinfo->pti_resident_size =
+		(mach_vm_size_t)(pmap_resident_count(map->pmap))
+		* PAGE_SIZE_64;
 
 	task_lock(task);
 
@@ -553,6 +525,7 @@ fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo)
 
 		if ((thread->state & TH_RUN) == TH_RUN)
 			numrunning++;
+		cswitch += thread->c_switch;
 		tval = timer_grab(&thread->user_timer);
 		tinfo.threads_user += tval;
 		tinfo.total_user += tval;
@@ -574,7 +547,7 @@ fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo)
 	ptinfo->pti_messages_received = task->messages_received;
 	ptinfo->pti_syscalls_mach = task->syscalls_mach;
 	ptinfo->pti_syscalls_unix = task->syscalls_unix;
-	ptinfo->pti_csw = task->csw;
+	ptinfo->pti_csw = task->c_switch + cswitch;
 	ptinfo->pti_threadnum = task->thread_count;
 	ptinfo->pti_numrunning = numrunning;
 	ptinfo->pti_priority = task->priority;
@@ -583,10 +556,11 @@ fill_taskprocinfo(task_t task, struct proc_taskinfo_internal * ptinfo)
 }
 
 int 
-fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo)
+fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_internal * ptinfo, void * vpp, int *vidp)
 {
 	thread_t  thact;
-	int err=0, count;
+	int err=0;
+	mach_msg_type_number_t count;
 	thread_basic_info_data_t basic_info;
 	kern_return_t kret;
 
@@ -594,7 +568,7 @@ fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_interna
 
 	for (thact  = (thread_t)queue_first(&task->threads);
 			!queue_end(&task->threads, (queue_entry_t)thact); ) {
-#if defined(__ppc__)
+#if defined(__ppc__) || defined(__arm__)
 		if (thact->machine.cthread_self == thaddr)
 #elif defined (__i386__)
 		if (thact->machine.pcb->cthread_self == thaddr)
@@ -604,7 +578,7 @@ fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_interna
 		{
 		
 			count = THREAD_BASIC_INFO_COUNT;
-			if ((kret = thread_info_internal(thact, THREAD_BASIC_INFO, &basic_info, &count)) != KERN_SUCCESS) {
+			if ((kret = thread_info_internal(thact, THREAD_BASIC_INFO, (thread_info_t)&basic_info, &count)) != KERN_SUCCESS) {
 				err = 1;
 				goto out;	
 			}
@@ -625,6 +599,8 @@ fill_taskthreadinfo(task_t task, uint64_t thaddr, struct proc_threadinfo_interna
 			ptinfo->pth_priority = thact->priority;
 			ptinfo->pth_maxpriority = thact->max_priority;
 			
+			if ((vpp != NULL) && (thact->uthread != NULL)) 
+				bsd_threadcdir(thact->uthread, vpp, vidp);
 			err = 0;
 			goto out; 
 		}
@@ -651,7 +627,7 @@ fill_taskthreadlist(task_t task, void * buffer, int thcount)
 
 	for (thact  = (thread_t)queue_first(&task->threads);
 			!queue_end(&task->threads, (queue_entry_t)thact); ) {
-#if defined(__ppc__)
+#if defined(__ppc__) || defined(__arm__)
 		thaddr = thact->machine.cthread_self;
 #elif defined (__i386__)
 		thaddr = thact->machine.pcb->cthread_self;
@@ -677,3 +653,13 @@ get_numthreads(task_t task)
 	return(task->thread_count);
 }
 
+void 
+syscall_exit_funnelcheck(void)
+{
+        thread_t thread;
+
+	thread = current_thread();
+
+        if (thread->funnel_lock)
+		panic("syscall exit with funnel held\n");
+}
diff --git a/osfmk/kern/call_entry.h b/osfmk/kern/call_entry.h
index ad846bf5c..0990ad142 100644
--- a/osfmk/kern/call_entry.h
+++ b/osfmk/kern/call_entry.h
@@ -2,23 +2,29 @@
  * Copyright (c) 1993-1995, 1999-2000 Apple Computer, Inc.
  * All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Private declarations for thread-based callouts.
diff --git a/osfmk/kern/clock.c b/osfmk/kern/clock.c
index 4085c5883..4a762bec5 100644
--- a/osfmk/kern/clock.c
+++ b/osfmk/kern/clock.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -41,6 +47,12 @@
 #include <mach/mach_traps.h>
 #include <mach/mach_time.h>
 
+uint32_t	hz_tick_interval = 1;
+
+#if CONFIG_DTRACE
+static void clock_track_calend_nowait(void);
+#endif
+
 decl_simple_lock_data(static,clock_lock)
 
 /*
@@ -53,9 +65,35 @@ decl_simple_lock_data(static,clock_lock)
  *	where CONV converts absolute time units into seconds and a fraction.
  */
 static struct clock_calend {
-	uint64_t			epoch;
-	uint64_t			offset;
-}					clock_calend;
+
+	uint64_t	epoch;
+	uint64_t	offset;
+	int64_t		adjtotal;	/* Nanosecond remaining total adjustment */
+	uint64_t	adjdeadline;	/* Absolute time value for next adjustment period */
+	uint32_t	adjinterval;	/* Absolute time interval of adjustment period */
+	int32_t		adjdelta;	/* Nanosecond time delta for this adjustment period */
+	uint64_t	adjstart;	/* Absolute time value for start of this adjustment period */
+	uint32_t	adjoffset;	/* Absolute time offset for this adjustment period as absolute value */
+	uint32_t	adjactive;
+	timer_call_data_t	adjcall;
+} clock_calend;
+
+#if CONFIG_DTRACE
+/*
+ *	Unlocked calendar flipflop; this is used to track a clock_calend such
+ *	that we can safely access a snapshot of a valid  clock_calend structure
+ *	without needing to take any locks to do it.
+ *
+ *	The trick is to use a generation count and set the low bit when it is
+ *	being updated/read; by doing this, we guarantee, through use of the
+ *	hw_atomic functions, that the generation is incremented when the bit
+ *	is cleared atomically (by using a 1 bit add).
+ */
+static struct unlocked_clock_calend {
+	struct clock_calend	calend;		/* copy of calendar */
+	uint32_t		gen;		/* generation count */
+} flipflop[ 2];
+#endif
 
 /*
  *	Calendar adjustment variables and values.
@@ -64,18 +102,6 @@ static struct clock_calend {
 #define calend_adjskew		(40 * NSEC_PER_USEC)	/* "standard" skew, ns / period */
 #define	calend_adjbig		(NSEC_PER_SEC)			/* use 10x skew above adjbig ns */
 
-static uint64_t			calend_adjstart;		/* Absolute time value for start of this adjustment period */
-static uint32_t			calend_adjoffset;		/* Absolute time offset for this adjustment period as absolute value */
-
-static int32_t			calend_adjdelta;		/* Nanosecond time delta for this adjustment period */
-static int64_t			calend_adjtotal;		/* Nanosecond remaining total adjustment */
-
-static uint64_t			calend_adjdeadline;		/* Absolute time value for next adjustment period */
-static uint32_t			calend_adjinterval;		/* Absolute time interval of adjustment period */
-
-static timer_call_data_t	calend_adjcall;
-static uint32_t				calend_adjactive;
-
 static uint32_t		calend_set_adjustment(
 						int32_t			*secs,
 						int32_t			*microsecs);
@@ -117,7 +143,7 @@ clock_config(void)
 {
 	simple_lock_init(&clock_lock, 0);
 
-	timer_call_setup(&calend_adjcall, (timer_call_func_t)calend_adjust_call, NULL);
+	timer_call_setup(&clock_calend.adjcall, (timer_call_func_t)calend_adjust_call, NULL);
 	thread_call_setup(&calend_wakecall, (thread_call_func_t)IOKitResetTime, NULL);
 
 	clock_oldconfig();
@@ -153,7 +179,10 @@ clock_timebase_init(void)
 	uint64_t	abstime;
 
 	nanoseconds_to_absolutetime(calend_adjperiod, &abstime);
-	calend_adjinterval = abstime;
+	clock_calend.adjinterval = abstime;
+
+	nanoseconds_to_absolutetime(NSEC_PER_SEC / 100, &abstime);
+	hz_tick_interval = abstime;
 
 	sched_timebase_init();
 }
@@ -200,16 +229,16 @@ clock_get_calendar_microtime(
 
 	now = mach_absolute_time();
 
-	if (calend_adjdelta < 0) {
+	if (clock_calend.adjdelta < 0) {
 		uint32_t	t32;
 
-		if (now > calend_adjstart) {
-			t32 = now - calend_adjstart;
+		if (now > clock_calend.adjstart) {
+			t32 = now - clock_calend.adjstart;
 
-			if (t32 > calend_adjoffset)
-				now -= calend_adjoffset;
+			if (t32 > clock_calend.adjoffset)
+				now -= clock_calend.adjoffset;
 			else
-				now = calend_adjstart;
+				now = clock_calend.adjstart;
 		}
 	}
 
@@ -246,16 +275,16 @@ clock_get_calendar_nanotime(
 
 	now = mach_absolute_time();
 
-	if (calend_adjdelta < 0) {
+	if (clock_calend.adjdelta < 0) {
 		uint32_t	t32;
 
-		if (now > calend_adjstart) {
-			t32 = now - calend_adjstart;
+		if (now > clock_calend.adjstart) {
+			t32 = now - clock_calend.adjstart;
 
-			if (t32 > calend_adjoffset)
-				now -= calend_adjoffset;
+			if (t32 > clock_calend.adjoffset)
+				now -= clock_calend.adjoffset;
 			else
-				now = calend_adjstart;
+				now = clock_calend.adjstart;
 		}
 	}
 
@@ -294,19 +323,19 @@ clock_gettimeofday(
 
 	now = mach_absolute_time();
 
-	if (calend_adjdelta >= 0) {
+	if (clock_calend.adjdelta >= 0) {
 		clock_gettimeofday_set_commpage(now, clock_calend.epoch, clock_calend.offset, secs, microsecs);
 	}
 	else {
 		uint32_t	t32;
 
-		if (now > calend_adjstart) {
-			t32 = now - calend_adjstart;
+		if (now > clock_calend.adjstart) {
+			t32 = now - clock_calend.adjstart;
 
-			if (t32 > calend_adjoffset)
-				now -= calend_adjoffset;
+			if (t32 > clock_calend.adjoffset)
+				now -= clock_calend.adjoffset;
 			else
-				now = calend_adjstart;
+				now = clock_calend.adjstart;
 		}
 
 		now += clock_calend.offset;
@@ -347,7 +376,7 @@ clock_set_calendar_microtime(
 	s = splclock();
 	simple_lock(&clock_lock);
 
-    commpage_set_timestamp(0,0,0);
+	commpage_disable_timestamp();
 
 	/*
 	 *	Calculate the new calendar epoch based on
@@ -370,7 +399,7 @@ clock_set_calendar_microtime(
 	/*
 	 *	Cancel any adjustment in progress.
 	 */
-	calend_adjdelta = calend_adjtotal = 0;
+	clock_calend.adjdelta = clock_calend.adjtotal = 0;
 
 	simple_unlock(&clock_lock);
 
@@ -385,6 +414,10 @@ clock_set_calendar_microtime(
 	 *	Send host notifications.
 	 */
 	host_notify_calendar_change();
+	
+#if CONFIG_DTRACE
+	clock_track_calend_nowait();
+#endif
 }
 
 /*
@@ -406,7 +439,7 @@ clock_initialize_calendar(void)
 	s = splclock();
 	simple_lock(&clock_lock);
 
-    commpage_set_timestamp(0,0,0);
+	commpage_disable_timestamp();
 
 	if ((int32_t)secs >= (int32_t)clock_boottime) {
 		/*
@@ -431,7 +464,7 @@ clock_initialize_calendar(void)
 		/*
 		 *	 Cancel any adjustment in progress.
 		 */
-		calend_adjdelta = calend_adjtotal = 0;
+		clock_calend.adjdelta = clock_calend.adjtotal = 0;
 	}
 
 	simple_unlock(&clock_lock);
@@ -441,6 +474,10 @@ clock_initialize_calendar(void)
 	 *	Send host notifications.
 	 */
 	host_notify_calendar_change();
+	
+#if CONFIG_DTRACE
+	clock_track_calend_nowait();
+#endif
 }
 
 /*
@@ -478,13 +515,13 @@ clock_adjtime(
 
 	interval = calend_set_adjustment(secs, microsecs);
 	if (interval != 0) {
-		calend_adjdeadline = mach_absolute_time() + interval;
-		if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
-			calend_adjactive++;
+		clock_calend.adjdeadline = mach_absolute_time() + interval;
+		if (!timer_call_enter(&clock_calend.adjcall, clock_calend.adjdeadline))
+			clock_calend.adjactive++;
 	}
 	else
-	if (timer_call_cancel(&calend_adjcall))
-		calend_adjactive--;
+	if (timer_call_cancel(&clock_calend.adjcall))
+		clock_calend.adjactive--;
 
 	simple_unlock(&clock_lock);
 	splx(s);
@@ -501,11 +538,11 @@ calend_set_adjustment(
 
 	total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC;
 
-    commpage_set_timestamp(0,0,0);
+	commpage_disable_timestamp();
 
 	now = mach_absolute_time();
 
-	ototal = calend_adjtotal;
+	ototal = clock_calend.adjtotal;
 
 	if (total != 0) {
 		int32_t		delta = calend_adjskew;
@@ -517,7 +554,7 @@ calend_set_adjustment(
 				delta = total;
 
 			nanoseconds_to_absolutetime((uint64_t)delta, &t64);
-			calend_adjoffset = t64;
+			clock_calend.adjoffset = t64;
 		}
 		else {
 			if (total < -calend_adjbig)
@@ -526,19 +563,19 @@ calend_set_adjustment(
 			if (delta < total)
 				delta = total;
 
-			calend_adjstart = now;
+			clock_calend.adjstart = now;
 
 			nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
-			calend_adjoffset = t64;
+			clock_calend.adjoffset = t64;
 		}
 
-		calend_adjtotal = total;
-		calend_adjdelta = delta;
+		clock_calend.adjtotal = total;
+		clock_calend.adjdelta = delta;
 
-		interval = calend_adjinterval;
+		interval = clock_calend.adjinterval;
 	}
 	else
-		calend_adjdelta = calend_adjtotal = 0;
+		clock_calend.adjdelta = clock_calend.adjtotal = 0;
 
 	if (ototal != 0) {
 		*secs = ototal / NSEC_PER_SEC;
@@ -547,6 +584,10 @@ calend_set_adjustment(
 	else
 		*secs = *microsecs = 0;
 
+#if CONFIG_DTRACE
+	clock_track_calend_nowait();
+#endif
+	
 	return (interval);
 }
 
@@ -559,14 +600,14 @@ calend_adjust_call(void)
 	s = splclock();
 	simple_lock(&clock_lock);
 
-	if (--calend_adjactive == 0) {
+	if (--clock_calend.adjactive == 0) {
 		interval = calend_adjust();
 		if (interval != 0) {
 			clock_deadline_for_periodic_event(interval, mach_absolute_time(),
-																&calend_adjdeadline);
+																&clock_calend.adjdeadline);
 
-			if (!timer_call_enter(&calend_adjcall, calend_adjdeadline))
-				calend_adjactive++;
+			if (!timer_call_enter(&clock_calend.adjcall, clock_calend.adjdeadline))
+				clock_calend.adjactive++;
 		}
 	}
 
@@ -581,41 +622,45 @@ calend_adjust(void)
 	int32_t			delta;
 	uint32_t		interval = 0;
 
-    commpage_set_timestamp(0,0,0);
+	commpage_disable_timestamp();
 
 	now = mach_absolute_time();
 
-	delta = calend_adjdelta;
+	delta = clock_calend.adjdelta;
 
 	if (delta > 0) {
-		clock_calend.offset += calend_adjoffset;
+		clock_calend.offset += clock_calend.adjoffset;
 
-		calend_adjtotal -= delta;
-		if (delta > calend_adjtotal) {
-			calend_adjdelta = delta = calend_adjtotal;
+		clock_calend.adjtotal -= delta;
+		if (delta > clock_calend.adjtotal) {
+			clock_calend.adjdelta = delta = clock_calend.adjtotal;
 
 			nanoseconds_to_absolutetime((uint64_t)delta, &t64);
-			calend_adjoffset = t64;
+			clock_calend.adjoffset = t64;
 		}
 	}
 	else
 	if (delta < 0) {
-		clock_calend.offset -= calend_adjoffset;
+		clock_calend.offset -= clock_calend.adjoffset;
 
-		calend_adjtotal -= delta;
-		if (delta < calend_adjtotal) {
-			calend_adjdelta = delta = calend_adjtotal;
+		clock_calend.adjtotal -= delta;
+		if (delta < clock_calend.adjtotal) {
+			clock_calend.adjdelta = delta = clock_calend.adjtotal;
 
 			nanoseconds_to_absolutetime((uint64_t)-delta, &t64);
-			calend_adjoffset = t64;
+			clock_calend.adjoffset = t64;
 		}
 
-		if (calend_adjdelta != 0)
-			calend_adjstart = now;
+		if (clock_calend.adjdelta != 0)
+			clock_calend.adjstart = now;
 	}
+	
+	if (clock_calend.adjdelta != 0)
+		interval = clock_calend.adjinterval;
 
-	if (calend_adjdelta != 0)
-		interval = calend_adjinterval;
+#if CONFIG_DTRACE
+	clock_track_calend_nowait();
+#endif
 
 	return (interval);
 }
@@ -747,3 +792,101 @@ clock_deadline_for_periodic_event(
 			*deadline = abstime + interval;
 	}
 }
+
+#if CONFIG_DTRACE
+
+/*
+ * clock_get_calendar_nanotime_nowait
+ *
+ * Description:	Non-blocking version of clock_get_calendar_nanotime()
+ *
+ * Notes:	This function operates by separately tracking calendar time
+ *		updates using a two element structure to copy the calendar
+ *		state, which may be asynchronously modified.  It utilizes
+ *		barrier instructions in the tracking process and in the local
+ *		stable snapshot process in order to ensure that a consistent
+ *		snapshot is used to perform the calculation.
+ */
+void
+clock_get_calendar_nanotime_nowait(
+	uint32_t			*secs,
+	uint32_t			*nanosecs)
+{
+	int i = 0;
+	uint64_t		now;
+	struct unlocked_clock_calend stable;
+
+	for (;;) {
+		stable = flipflop[i];		/* take snapshot */
+
+		/*
+		 * Use a barrier instructions to ensure atomicity.  We AND
+		 * off the "in progress" bit to get the current generation
+		 * count.
+		 */
+		(void)hw_atomic_and(&stable.gen, ~(uint32_t)1);
+
+		/*
+		 * If an update _is_ in progress, the generation count will be
+		 * off by one, if it _was_ in progress, it will be off by two,
+		 * and if we caught it at a good time, it will be equal (and
+		 * our snapshot is threfore stable).
+		 */
+		if (flipflop[i].gen == stable.gen)
+			break;
+
+		/* Switch to the oher element of the flipflop, and try again. */
+		i ^= 1;
+	}
+
+	now = mach_absolute_time();
+
+	if (stable.calend.adjdelta < 0) {
+		uint32_t	t32;
+
+		if (now > stable.calend.adjstart) {
+			t32 = now - stable.calend.adjstart;
+
+			if (t32 > stable.calend.adjoffset)
+				now -= stable.calend.adjoffset;
+			else
+				now = stable.calend.adjstart;
+		}
+	}
+
+	now += stable.calend.offset;
+
+	absolutetime_to_microtime(now, secs, nanosecs);
+	*nanosecs *= NSEC_PER_USEC;
+
+	*secs += stable.calend.epoch;
+}
+
+static void 
+clock_track_calend_nowait(void)
+{
+	int i;
+
+	for (i = 0; i < 2; i++) {
+		struct clock_calend tmp = clock_calend;
+
+		/*
+		 * Set the low bit if the generation count; since we use a
+		 * barrier instruction to do this, we are guaranteed that this
+		 * will flag an update in progress to an async caller trying
+		 * to examine the contents.
+		 */
+		(void)hw_atomic_or(&flipflop[i].gen, 1);
+
+		flipflop[i].calend = tmp;
+
+		/*
+		 * Increment the generation count to clear the low bit to
+		 * signal completion.  If a caller compares the generation
+		 * count after taking a copy while in progress, the count
+		 * will be off by two.
+		 */
+		(void)hw_atomic_add(&flipflop[i].gen, 1);
+	}
+}
+#endif /* CONFIG_DTRACE */
diff --git a/osfmk/kern/clock.h b/osfmk/kern/clock.h
index ee52ad8f2..de8086406 100644
--- a/osfmk/kern/clock.h
+++ b/osfmk/kern/clock.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -111,17 +117,16 @@ extern void			machine_delay_until(
 
 #include <stat_time.h>
 
+#if	STAT_TIME || GPROF
+
 extern void		hertz_tick(
-#if	STAT_TIME
 					natural_t		ticks,
-#endif	/* STAT_TIME */
 					boolean_t		usermode,	/* executing user code */
 					natural_t		pc);
 
-extern void		absolutetime_to_microtime(
-					uint64_t		abstime,
-					uint32_t		*secs,
-					uint32_t		*microsecs);
+#endif	/* STAT_TIME */
+
+extern uint32_t		hz_tick_interval;
 
 extern void		absolutetime_to_nanotime(
 					uint64_t		abstime,
@@ -159,6 +164,11 @@ extern void			clock_get_boottime_nanotime(
 						uint32_t			*secs,
 						uint32_t			*nanosecs);
 
+extern void			absolutetime_to_microtime(
+						uint64_t		abstime,
+						uint32_t		*secs,
+						uint32_t		*microsecs);
+
 extern void			clock_deadline_for_periodic_event(
 						uint64_t			interval,
 						uint64_t			abstime,
@@ -166,7 +176,6 @@ extern void			clock_deadline_for_periodic_event(
 
 #endif	/* XNU_KERNEL_PRIVATE */
 
-
 extern void			clock_get_calendar_microtime(
 						uint32_t			*secs,
 						uint32_t			*microsecs);
@@ -175,6 +184,17 @@ extern void			clock_get_calendar_nanotime(
 						uint32_t			*secs,
 						uint32_t			*nanosecs);
 
+/*
+ * Gah! This file is included everywhere. The other domains do not correctly
+ * include config_dtrace headers, so this isn't being defined. The last test
+ * I ran stopped with a build failure in pexpert/i386/kd.c
+ */
+#if CONFIG_DTRACE
+extern void			clock_get_calendar_nanotime_nowait(
+						uint32_t			*secs,
+						uint32_t			*nanosecs);
+#endif /* CONFIG_DTRACE */
+
 extern void			clock_get_system_microtime(
 						uint32_t			*secs,
 						uint32_t			*microsecs);
diff --git a/osfmk/kern/clock_oldops.c b/osfmk/kern/clock_oldops.c
index ac03a7d9e..b69e91938 100644
--- a/osfmk/kern/clock_oldops.c
+++ b/osfmk/kern/clock_oldops.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -149,7 +155,7 @@ kern_return_t	calend_getattr(
 	mach_msg_type_number_t	*count);
 
 struct clock_ops calend_ops = {
-	0,					0,
+	NULL, NULL,
 	calend_gettime,
 	calend_getattr,
 };
@@ -182,7 +188,7 @@ clock_oldconfig(void)
 		clock = &clock_list[i];
 		if (clock->cl_ops && clock->cl_ops->c_config) {
 			if ((*clock->cl_ops->c_config)() == 0)
-				clock->cl_ops = 0;
+				clock->cl_ops = NULL;
 		}
 	}
 
@@ -639,7 +645,7 @@ alarm_expire(void)
 		 * which issued the clock_sleep() call.
 		 */
 		if (alrm2->al_status == ALARM_SLEEP) {
-			alrm2->al_next = 0;
+			alrm2->al_next = NULL;
 			alrm2->al_status = ALARM_DONE;
 			alrm2->al_time = clock_time;
 			thread_wakeup((event_t)alrm2);
diff --git a/osfmk/kern/counters.c b/osfmk/kern/counters.c
index ff186fe15..bac65ebd7 100644
--- a/osfmk/kern/counters.c
+++ b/osfmk/kern/counters.c
@@ -1,37 +1,33 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
-/*
- * Revision 2.3  91/05/14  16:40:19  mrt
- * 	Correcting copyright
- * 
- * Revision 2.2  91/03/16  15:15:51  rpd
- * 	Created.
- * 	[91/03/13            rpd]
- * 
- */
-/* CMU_ENDHIST */
 /* 
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
@@ -70,23 +66,14 @@
  *	This makes them easier to examine with ddb.
  */
 
-mach_counter_t c_thread_invoke_csw = 0;
-mach_counter_t c_thread_invoke_hits = 0;
-mach_counter_t c_thread_invoke_misses = 0;
-mach_counter_t c_thread_invoke_same = 0;
-mach_counter_t c_thread_invoke_same_cont = 0;
-mach_counter_t c_incoming_interrupts = 0;
-mach_counter_t c_syscalls_unix = 0;
-mach_counter_t c_syscalls_mach = 0;
-
 #if	MACH_COUNTERS
 mach_counter_t c_action_thread_block = 0;
 mach_counter_t c_ast_taken_block = 0;
-mach_counter_t c_clock_ticks = 0;
 mach_counter_t c_dev_io_blocks = 0;
 mach_counter_t c_dev_io_tries = 0;
 mach_counter_t c_idle_thread_block = 0;
 mach_counter_t c_idle_thread_handoff = 0;
+mach_counter_t c_incoming_interrupts = 0;
 mach_counter_t c_io_done_thread_block = 0;
 mach_counter_t c_ipc_mqueue_receive_block_kernel = 0;
 mach_counter_t c_ipc_mqueue_receive_block_user = 0;
@@ -99,6 +86,13 @@ mach_counter_t c_stacks_max = 0;
 mach_counter_t c_stacks_min = 0;
 mach_counter_t c_swtch_block = 0;
 mach_counter_t c_swtch_pri_block = 0;
+mach_counter_t c_syscalls_unix = 0;
+mach_counter_t c_syscalls_mach = 0;
+mach_counter_t c_thread_invoke_csw = 0;
+mach_counter_t c_thread_invoke_hits = 0;
+mach_counter_t c_thread_invoke_misses = 0;
+mach_counter_t c_thread_invoke_same = 0;
+mach_counter_t c_thread_invoke_same_cont = 0;
 mach_counter_t c_thread_switch_block = 0;
 mach_counter_t c_thread_switch_handoff = 0;
 mach_counter_t c_vm_fault_page_block_backoff_kernel = 0;
diff --git a/osfmk/kern/counters.h b/osfmk/kern/counters.h
index 61b58852d..f42a0a703 100644
--- a/osfmk/kern/counters.h
+++ b/osfmk/kern/counters.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -83,23 +89,14 @@
 
 typedef unsigned int mach_counter_t;
 
-extern mach_counter_t c_thread_invoke_csw;
-extern mach_counter_t c_thread_invoke_same;
-extern mach_counter_t c_thread_invoke_same_cont;
-extern mach_counter_t c_thread_invoke_misses;
-extern mach_counter_t c_thread_invoke_hits;
-extern mach_counter_t c_incoming_interrupts;
-extern mach_counter_t c_syscalls_unix;
-extern mach_counter_t c_syscalls_mach;
-
 #if	MACH_COUNTERS
 extern mach_counter_t c_action_thread_block;
 extern mach_counter_t c_ast_taken_block;
-extern mach_counter_t c_clock_ticks;
 extern mach_counter_t c_dev_io_blocks;
 extern mach_counter_t c_dev_io_tries;
 extern mach_counter_t c_idle_thread_block;
 extern mach_counter_t c_idle_thread_handoff;
+extern mach_counter_t c_incoming_interrupts;
 extern mach_counter_t c_io_done_thread_block;
 extern mach_counter_t c_ipc_mqueue_receive_block_kernel;
 extern mach_counter_t c_ipc_mqueue_receive_block_user;
@@ -112,6 +109,13 @@ extern mach_counter_t c_stacks_max;
 extern mach_counter_t c_stacks_min;
 extern mach_counter_t c_swtch_block;
 extern mach_counter_t c_swtch_pri_block;
+extern mach_counter_t c_syscalls_unix;
+extern mach_counter_t c_syscalls_mach;
+extern mach_counter_t c_thread_invoke_csw;
+extern mach_counter_t c_thread_invoke_same;
+extern mach_counter_t c_thread_invoke_same_cont;
+extern mach_counter_t c_thread_invoke_misses;
+extern mach_counter_t c_thread_invoke_hits;
 extern mach_counter_t c_thread_switch_block;
 extern mach_counter_t c_thread_switch_handoff;
 extern mach_counter_t c_vm_fault_page_block_backoff_kernel;
diff --git a/osfmk/kern/cpu_data.h b/osfmk/kern/cpu_data.h
index 25ea59254..373297d0e 100644
--- a/osfmk/kern/cpu_data.h
+++ b/osfmk/kern/cpu_data.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/cpu_number.h b/osfmk/kern/cpu_number.h
index 7c8fbdb87..9a85e2514 100644
--- a/osfmk/kern/cpu_number.h
+++ b/osfmk/kern/cpu_number.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/debug.c b/osfmk/kern/debug.c
index c5f6cf3d9..c7e01a134 100644
--- a/osfmk/kern/debug.c
+++ b/osfmk/kern/debug.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,21 +69,31 @@
 #include <vm/vm_kern.h>
 #include <vm/pmap.h>
 #include <stdarg.h>
+#if !MACH_KDP
+#include <kdp/kdp_udp.h>
+#endif
 
 #ifdef	__ppc__
 #include <ppc/Firmware.h>
 #include <ppc/low_trace.h>
 #endif
 
+#ifdef __i386__
+#include <i386/cpu_threads.h>
+#include <i386/pmCPU.h>
+#endif
+
+#include <IOKit/IOPlatformExpert.h>
+
 unsigned int	halt_in_debugger = 0;
 unsigned int	switch_debugger = 0;
 unsigned int	current_debugger = 0;
 unsigned int	active_debugger = 0;
 unsigned int	debug_mode=0;
-unsigned int 	disableDebugOuput = TRUE;
+unsigned int 	disable_debug_output = TRUE;
 unsigned int 	systemLogDiags = FALSE;
-unsigned int    logPanicDataToScreen = FALSE;
 unsigned int 	panicDebugging = FALSE;
+unsigned int	logPanicDataToScreen = FALSE;
 
 int mach_assert = 1;
 
@@ -93,19 +109,46 @@ unsigned long		panic_caller;
 char *debug_buf;
 ppnum_t debug_buf_page;
 char *debug_buf_ptr;
-unsigned int debug_buf_size = 0;
+unsigned int debug_buf_size;
+
+static char model_name[64];
+
+struct pasc {
+  unsigned a: 7;
+  unsigned b: 7;
+  unsigned c: 7;
+  unsigned d: 7;
+  unsigned e: 7;
+  unsigned f: 7;
+  unsigned g: 7;
+  unsigned h: 7;
+}  __attribute__((packed));
+
+typedef struct pasc pasc_t;
 
 void
 Assert(
 	const char	*file,
 	int		line,
-	const char	*expression)
+#if CONFIG_NO_PANIC_STRINGS
+	__unused const char	*expression
+#else
+	const char	*expression
+#endif
+      )
 {
+	int saved_return_on_panic;
+
 	if (!mach_assert) {
 		return;
 	}
-	panic("{%d} Assertion failed: file \"%s\", line %d: %s\n", 
-	       cpu_number(), file, line, expression);
+
+	saved_return_on_panic = return_on_panic;
+	return_on_panic = 1;
+
+	panic_plain("%s:%d Assertion failed: %s", file, line, expression);
+
+	return_on_panic = saved_return_on_panic;
 }
 
 /*
@@ -135,6 +178,31 @@ panic_init(void)
 	panic_caller = 0;
 }
 
+void
+debug_log_init(void)
+{
+	if (debug_buf_size != 0)
+		return;
+	if (kmem_alloc(kernel_map, (vm_offset_t *) &debug_buf, PAGE_SIZE)
+			!= KERN_SUCCESS)
+		panic("cannot allocate debug_buf\n");
+	debug_buf_ptr = debug_buf;
+	debug_buf_size = PAGE_SIZE;
+        debug_buf_page = pmap_find_phys(kernel_pmap,
+					(addr64_t)(uintptr_t)debug_buf_ptr);
+}
+
+#if __i386__
+#define panic_stop()	pmCPUHalt(PM_HALT_PANIC)
+#define panic_safe()	pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE)
+#define panic_normal()	pmSafeMode(x86_lcpu(), PM_SAFE_FL_NORMAL)
+#else
+#define panic_stop()	{ while (1) ; }
+#define panic_safe()
+#define panic_normal()
+#endif
+
+#undef panic(...) 
 void
 panic(const char *str, ...)
 {
@@ -146,22 +214,24 @@ panic(const char *str, ...)
 	s = splhigh();
 	disable_preemption();
 
+	panic_safe();
+
 #ifdef	__ppc__
 	lastTrace = LLTraceSet(0);		/* Disable low-level tracing */
 #endif
 
 	thread = current_thread();		/* Get failing thread */
 	wq = thread->wait_queue;		/* Save the old value */
-	thread->wait_queue = 0;			/* Clear the wait so we do not get double panics when we try locks */
+	thread->wait_queue = NULL;		/* Clear the wait so we do not get double panics when we try locks */
 
 	if( logPanicDataToScreen )
-		disableDebugOuput = FALSE;
+		disable_debug_output = FALSE;
 		
 	debug_mode = TRUE;
 
 	/* panic_caller is initialized to 0.  If set, don't change it */
 	if ( ! panic_caller )
-		panic_caller = (unsigned long) __builtin_return_address(0);
+		panic_caller = (unsigned long)(char *)__builtin_return_address(0);
 	
 restart:
 	PANIC_LOCK();
@@ -180,7 +250,7 @@ restart:
 			PANIC_UNLOCK();
 			Debugger("double panic");
 			printf("double panic:  We are hanging here...\n");
-			while(1);
+			panic_stop();
 			/* NOTREACHED */
 		}
 	}
@@ -189,10 +259,12 @@ restart:
 	panicwait = 1;
 
 	PANIC_UNLOCK();
-	kdb_printf("panic(cpu %d caller 0x%08X): ", (unsigned) paniccpu, panic_caller);
-	va_start(listp, str);
-	_doprnt(str, &listp, consdebug_putc, 0);
-	va_end(listp);
+	kdb_printf("panic(cpu %d caller 0x%08lX): ", (unsigned) paniccpu, panic_caller);
+	if (str) {
+		va_start(listp, str);
+		_doprnt(str, &listp, consdebug_putc, 0);
+		va_end(listp);
+	}
 	kdb_printf("\n");
 
 	/*
@@ -207,18 +279,21 @@ restart:
 	panicstr = (char *)0;
 	PANIC_UNLOCK();
 	thread->wait_queue = wq; 	/* Restore the wait queue */
+
 	if (return_on_panic) {
+		panic_normal();
 		enable_preemption();
 		splx(s);
 		return;
 	}
+
 	kdb_printf("panic: We are hanging here...\n");
-	while(1);
+	panic_stop();
 	/* NOTREACHED */
 }
 
 void
-log(int level, char *fmt, ...)
+log(__unused int level, char *fmt, ...)
 {
 	va_list	listp;
 
@@ -234,23 +309,181 @@ log(int level, char *fmt, ...)
 #endif
 }
 
-void
-debug_log_init(void)
-{
-	if (debug_buf_size != 0)
-		return;
-	if (kmem_alloc(kernel_map, (vm_offset_t *) &debug_buf, PAGE_SIZE) != KERN_SUCCESS)
-		panic("cannot allocate debug_buf \n");
-	debug_buf_ptr = debug_buf;
-	debug_buf_size = PAGE_SIZE;
-        debug_buf_page = pmap_find_phys(kernel_pmap, (addr64_t)(uintptr_t)debug_buf_ptr);
-}
-
 void
 debug_putc(char c)
 {
-	if ((debug_buf_size != 0) && ((debug_buf_ptr-debug_buf) < debug_buf_size)) {
+	if ((debug_buf_size != 0) &&
+		((debug_buf_ptr-debug_buf) < (int)debug_buf_size)) {
 		*debug_buf_ptr=c;
 		debug_buf_ptr++;
 	}
 }
+
+/* In-place packing routines -- inefficient, but they're called at most once.
+ */
+
+int packA(char *inbuf, uint32_t length, uint32_t buflen)
+{
+  unsigned int i, j = 0;
+  pasc_t pack;
+  
+  length = MIN(((length & ~7) +8), buflen);
+
+  for (i = 0; i < length; i+=8)
+    {
+      pack.a = inbuf[i];
+      pack.b = inbuf[i+1];
+      pack.c = inbuf[i+2];
+      pack.d = inbuf[i+3];
+      pack.e = inbuf[i+4];
+      pack.f = inbuf[i+5];
+      pack.g = inbuf[i+6];
+      pack.h = inbuf[i+7];
+      bcopy ((char *) &pack, inbuf + j, 7);
+      j += 7;
+    }
+  return ((length * 7)/8);
+}
+
+void unpackA(char *inbuf, uint32_t length)
+{
+	pasc_t packs;
+	unsigned i = 0;
+	length = (length * 8)/7;
+
+	while (i < length) {
+	  packs = *(pasc_t *)&inbuf[i];
+	  bcopy(&inbuf[i+7], &inbuf[i+8], MAX(0, (int) (length - i - 8)));
+	  inbuf[i++] = packs.a;
+	  inbuf[i++] = packs.b;
+	  inbuf[i++] = packs.c;
+	  inbuf[i++] = packs.d;
+	  inbuf[i++] = packs.e;
+	  inbuf[i++] = packs.f;
+	  inbuf[i++] = packs.g;
+	  inbuf[i++] = packs.h;
+	}
+}
+
+extern void *proc_name_address(void *p);
+
+static void
+panic_display_process_name(void) {
+	char proc_name[32] = "Unknown";
+	task_t ctask = 0;
+	void *cbsd_info = 0;
+
+	if (ml_nofault_copy((vm_offset_t)&current_thread()->task, (vm_offset_t) &ctask, sizeof(task_t)) == sizeof(task_t))
+		if(ml_nofault_copy((vm_offset_t)&ctask->bsd_info, (vm_offset_t)&cbsd_info, sizeof(&ctask->bsd_info)) == sizeof(&ctask->bsd_info))
+			if (cbsd_info && (ml_nofault_copy((vm_offset_t) proc_name_address(cbsd_info), (vm_offset_t) &proc_name, sizeof(proc_name)) > 0))
+				proc_name[sizeof(proc_name) - 1] = '\0';
+	kdb_printf("\nBSD process name corresponding to current thread: %s\n", proc_name);
+}
+
+unsigned	panic_active(void) {
+	return ((panicstr != (char *) 0));
+}
+
+void populate_model_name(char *model_string) {
+	strlcpy(model_name, model_string, sizeof(model_name));
+}
+
+static void panic_display_model_name(void) {
+	char tmp_model_name[sizeof(model_name)];
+
+	if (ml_nofault_copy((vm_offset_t) &model_name, (vm_offset_t) &tmp_model_name, sizeof(model_name)) != sizeof(model_name))
+		return;
+
+	model_name[sizeof(model_name) - 1] = '\0';
+
+	if (model_name[0] != 0)
+		kdb_printf("System model name: %s\n", model_name);
+}
+
+extern const char version[];
+extern char osversion[];
+
+__private_extern__ void panic_display_system_configuration(void) {
+	static boolean_t config_displayed = FALSE;
+
+	panic_display_process_name();
+	if (config_displayed == FALSE) {
+		kdb_printf("\nMac OS version:\n%s\n",
+		    (osversion[0] != 0) ? osversion : "Not yet set");
+		kdb_printf("\nKernel version:\n%s\n",version);
+		panic_display_model_name();
+		config_displayed = TRUE;
+	}
+}
+
+#if !MACH_KDP
+static struct ether_addr kdp_current_mac_address = {{0, 0, 0, 0, 0, 0}};
+unsigned int not_in_kdp = 1;
+
+/* XXX ugly forward declares to stop warnings */
+void *kdp_get_interface(void);
+void kdp_set_ip_and_mac_addresses(struct in_addr *, struct ether_addr *);
+void kdp_set_gateway_mac(void *);
+void kdp_set_interface(void *);
+void kdp_register_send_receive(void *, void *);
+void kdp_unregister_send_receive(void *, void *);
+void kdp_snapshot_preflight(int, void *, uint32_t, uint32_t);
+int kdp_stack_snapshot_geterror(void);
+int kdp_stack_snapshot_bytes_traced(void);
+
+void *
+kdp_get_interface( void)
+{
+        return(void *)0;
+}
+
+unsigned int
+kdp_get_ip_address(void )
+{ return 0; }
+
+struct ether_addr
+kdp_get_mac_addr(void)
+{       
+        return kdp_current_mac_address;
+}
+
+void
+kdp_set_ip_and_mac_addresses(   
+        __unused struct in_addr          *ipaddr,
+        __unused struct ether_addr       *macaddr)
+{}
+
+void
+kdp_set_gateway_mac(__unused void *gatewaymac)
+{}
+
+void
+kdp_set_interface(__unused void *ifp)
+{}
+
+void
+kdp_register_send_receive(__unused void *send, __unused void *receive)
+{}
+
+void
+kdp_unregister_send_receive(__unused void *send, __unused void *receive)
+{}
+
+void
+kdp_snapshot_preflight(__unused int pid, __unused void * tracebuf,
+		__unused uint32_t tracebuf_size, __unused uint32_t options)
+{}
+
+int
+kdp_stack_snapshot_geterror(void)
+{       
+        return -1;
+}
+
+int
+kdp_stack_snapshot_bytes_traced(void)
+{       
+        return 0;
+}
+
+#endif
diff --git a/osfmk/kern/debug.h b/osfmk/kern/debug.h
index d546fbfe9..4f2ab7f87 100644
--- a/osfmk/kern/debug.h
+++ b/osfmk/kern/debug.h
@@ -1,29 +1,36 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_KERN_DEBUG_H_
 #define _KERN_DEBUG_H_
 
 #include <sys/cdefs.h>
+#include <stdint.h>
 
 #ifdef	KERNEL_PRIVATE
 
@@ -42,7 +49,7 @@ extern unsigned int     current_debugger;
 
 extern unsigned int     active_debugger;
 extern unsigned int 	debug_mode; 
-extern unsigned int	disableDebugOuput;
+extern unsigned int	disable_debug_output;
 
 extern unsigned int     panicDebugging;
 extern unsigned int	logPanicDataToScreen;
@@ -62,6 +69,7 @@ extern int db_run_mode;
 
 extern const char		*panicstr;
 extern volatile unsigned int	nestedpanic;
+extern int unsigned long panic_caller;
 
 extern char *debug_buf;
 extern char *debug_buf_ptr;
@@ -72,6 +80,11 @@ extern void	debug_putc(char);
 
 extern void	panic_init(void);
 
+int	packA(char *inbuf, uint32_t length, uint32_t buflen);
+void	unpackA(char *inbuf, uint32_t length);
+
+void	panic_display_system_configuration(void);
+
 #endif /* MACH_KERNEL_PRIVATE */
 
 #define DB_HALT		0x1
@@ -94,8 +107,22 @@ extern void	panic_init(void);
 
 __BEGIN_DECLS
 
-extern void	panic(const char *string, ...) __dead2;
-
+extern void panic(const char *string, ...) __printflike(1,2);
+#if CONFIG_NO_PANIC_STRINGS
+#define panic_plain(...) (panic)((char *)0)
+#define panic(...)  (panic)((char *)0)
+#else /* CONFIGS_NO_PANIC_STRINGS */
+#define panic_plain(ex, ...) \
+	(panic)(ex, ## __VA_ARGS__)
+#define __STRINGIFY(x) #x
+#define LINE_NUMBER(x) __STRINGIFY(x)
+#define PANIC_LOCATION __FILE__ ":" LINE_NUMBER(__LINE__)
+#define panic(ex, ...) \
+	(panic)(# ex "@" PANIC_LOCATION, ## __VA_ARGS__)
+#endif /* CONFIGS_NO_PANIC_STRINGS */
+
+void 		populate_model_name(char *);
+unsigned	panic_active(void);
 __END_DECLS
 
 #endif	/* _KERN_DEBUG_H_ */
diff --git a/osfmk/kern/etimer.h b/osfmk/kern/etimer.h
index 3b5c4bf51..84674a990 100644
--- a/osfmk/kern/etimer.h
+++ b/osfmk/kern/etimer.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -49,8 +55,6 @@ extern void etimer_resync_deadlines(void);
 
 extern uint32_t rtclock_tick_interval;
 
-extern uint64_t tsc_to_nanoseconds(uint64_t abstime);
-
 #if 0 /* this is currently still MD */
 #pragma pack(push,4)
 struct rtclock_timer_t  {
diff --git a/osfmk/kern/exception.c b/osfmk/kern/exception.c
index 67abd9fbf..0122c44b1 100644
--- a/osfmk/kern/exception.c
+++ b/osfmk/kern/exception.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -61,6 +67,8 @@
 #include <mach/task.h>
 #include <mach/thread_status.h>
 #include <mach/exception_types.h>
+#include <mach/exc.h>
+#include <mach/mach_exc.h>
 #include <ipc/port.h>
 #include <ipc/ipc_entry.h>
 #include <ipc/ipc_object.h>
@@ -78,7 +86,6 @@
 #include <kern/host.h>
 #include <kern/misc_protos.h>
 #include <string.h>
-#include <mach/exc.h>
 
 #if	MACH_KDB
 #include <ddb/db_trap.h>
@@ -104,9 +111,10 @@ unsigned long c_tsk_exc_raise_state = 0;
 unsigned long c_tsk_exc_raise_state_id = 0;
 
 /* forward declarations */
-void exception_deliver(
+kern_return_t exception_deliver(
+	thread_t 		thread,
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t  codeCnt,
 	struct exception_action *excp,
 	mutex_t			*mutex);
@@ -114,7 +122,7 @@ void exception_deliver(
 #ifdef MACH_BSD
 kern_return_t bsd_exception(
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t  codeCnt);
 #endif /* MACH_BSD */
 
@@ -128,18 +136,20 @@ kern_return_t bsd_exception(
  *		thread_exception_return and thread_kdb_return
  *		are possible.
  *	Returns:
- *		If the exception was not handled by this handler
+ *		KERN_SUCCESS if the exception was handled
  */
-void
+kern_return_t 
 exception_deliver(
+	thread_t		thread,
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t  codeCnt,
 	struct exception_action *excp,
 	mutex_t			*mutex)
 {
-	thread_t		self = current_thread();
 	ipc_port_t		exc_port;
+	exception_data_type_t	small_code[EXCEPTION_CODE_MAX];
+	int			code64;
 	int			behavior;
 	int			flavor;
 	kern_return_t		kr;
@@ -148,8 +158,8 @@ exception_deliver(
 	 *  Save work if we are terminating.
 	 *  Just go back to our AST handler.
 	 */
-	if (!self->active)
-		thread_exception_return();
+	if (!thread->active)
+		return KERN_SUCCESS;
 
 	/*
 	 * Snapshot the exception action data under lock for consistency.
@@ -162,13 +172,13 @@ exception_deliver(
 	exc_port = excp->port;
 	if (!IP_VALID(exc_port)) {
 		mutex_unlock(mutex);
-		return;
+		return KERN_FAILURE;
 	}
 	ip_lock(exc_port);
 	if (!ip_active(exc_port)) {
 		ip_unlock(exc_port);
 		mutex_unlock(mutex);
-		return;
+		return KERN_FAILURE;
 	}
 	ip_reference(exc_port);	
 	exc_port->ip_srights++;
@@ -178,6 +188,15 @@ exception_deliver(
 	behavior = excp->behavior;
 	mutex_unlock(mutex);
 
+	code64 = (behavior & MACH_EXCEPTION_CODES);
+	behavior &= ~MACH_EXCEPTION_CODES;
+
+	if (!code64) {
+		small_code[0] = CAST_DOWN(exception_data_type_t, code[0]);
+		small_code[1] = CAST_DOWN(exception_data_type_t, code[1]);
+	}
+
+
 	switch (behavior) {
 	case EXCEPTION_STATE: {
 		mach_msg_type_number_t state_cnt;
@@ -185,39 +204,54 @@ exception_deliver(
 
 		c_thr_exc_raise_state++;
 		state_cnt = _MachineStateCount[flavor];
-		kr = thread_getstatus(self, flavor, 
+		kr = thread_getstatus(thread, flavor, 
 				      (thread_state_t)state,
 				      &state_cnt);
 		if (kr == KERN_SUCCESS) {
-			kr = exception_raise_state(exc_port, exception,
-						   code, codeCnt,
-						   &flavor,
-						   state, state_cnt,
-						   state, &state_cnt);
+			if (code64) {
+				kr = mach_exception_raise_state(exc_port, 
+						exception,
+						code, 
+						codeCnt,
+						&flavor,
+						state, state_cnt,
+						state, &state_cnt);
+			} else {
+				kr = exception_raise_state(exc_port, exception,
+						small_code, 
+						codeCnt,
+						&flavor,
+						state, state_cnt,
+						state, &state_cnt);
+			}
 			if (kr == MACH_MSG_SUCCESS)
-				kr = thread_setstatus(self, flavor, 
-						      (thread_state_t)state,
-						      state_cnt);
+				kr = thread_setstatus(thread, flavor, 
+						(thread_state_t)state,
+						state_cnt);
 		}
 
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			thread_exception_return();
-			/*NOTREACHED*/
-		return;
+		return kr;
 	}
 
 	case EXCEPTION_DEFAULT:
 		c_thr_exc_raise++;
-		kr = exception_raise(exc_port,
-				retrieve_thread_self_fast(self),
-				retrieve_task_self_fast(self->task),
-				exception,
-				code, codeCnt);
+		if (code64) {
+			kr = mach_exception_raise(exc_port,
+					retrieve_thread_self_fast(thread),
+					retrieve_task_self_fast(thread->task),
+					exception,
+					code, 
+					codeCnt);
+		} else {
+			kr = exception_raise(exc_port,
+					retrieve_thread_self_fast(thread),
+					retrieve_task_self_fast(thread->task),
+					exception,
+					small_code, 
+					codeCnt);
+		}
 
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			thread_exception_return();
-			/*NOTREACHED*/
-		return;
+		return kr;
 
 	case EXCEPTION_STATE_IDENTITY: {
 		mach_msg_type_number_t state_cnt;
@@ -225,32 +259,44 @@ exception_deliver(
 
 		c_thr_exc_raise_state_id++;
 		state_cnt = _MachineStateCount[flavor];
-		kr = thread_getstatus(self, flavor,
+		kr = thread_getstatus(thread, flavor,
 				      (thread_state_t)state,
 				      &state_cnt);
 		if (kr == KERN_SUCCESS) {
-		    kr = exception_raise_state_identity(exc_port,
-				retrieve_thread_self_fast(self),
-				retrieve_task_self_fast(self->task),
-				exception,
-				code, codeCnt,
-				&flavor,
-				state, state_cnt,
-				state, &state_cnt);
-		    if (kr == MACH_MSG_SUCCESS)
-			kr = thread_setstatus(self, flavor,
-					      (thread_state_t)state,
-					      state_cnt);
+			if (code64) {
+				kr = mach_exception_raise_state_identity(
+						exc_port,
+						retrieve_thread_self_fast(thread),
+						retrieve_task_self_fast(thread->task),
+						exception,
+						code, 
+						codeCnt,
+						&flavor,
+						state, state_cnt,
+						state, &state_cnt);
+			} else {
+				kr = exception_raise_state_identity(exc_port,
+						retrieve_thread_self_fast(thread),
+						retrieve_task_self_fast(thread->task),
+						exception,
+						small_code, 
+						codeCnt,
+						&flavor,
+						state, state_cnt,
+						state, &state_cnt);
+			}
+			if (kr == MACH_MSG_SUCCESS)
+				kr = thread_setstatus(thread, flavor,
+						(thread_state_t)state,
+						state_cnt);
 		}
 
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			thread_exception_return();
-			/*NOTREACHED*/
-		return;
+		return kr;
 	}
-	
+
 	default:
-		panic ("bad exception behavior!");
+	       panic ("bad exception behavior!");
+	       return KERN_FAILURE; 
 	}/* switch */
 }
 
@@ -270,7 +316,7 @@ exception_deliver(
 void
 exception_triage(
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t  codeCnt)
 {
 	thread_t		thread;
@@ -278,6 +324,7 @@ exception_triage(
 	host_priv_t		host_priv;
 	struct exception_action *excp;
 	mutex_t			*mutex;
+	kern_return_t		kr;
 
 	assert(exception != EXC_RPC_ALERT);
 
@@ -290,7 +337,9 @@ exception_triage(
 	thread = current_thread();
 	mutex = mutex_addr(thread->mutex);
 	excp = &thread->exc_actions[exception];
-	exception_deliver(exception, code, codeCnt, excp, mutex);
+	kr = exception_deliver(thread, exception, code, codeCnt, excp, mutex);
+	if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
+		goto out;
 
 	/*
 	 * Maybe the task level will handle it.
@@ -298,7 +347,9 @@ exception_triage(
 	task = current_task();
 	mutex = mutex_addr(task->lock);
 	excp = &task->exc_actions[exception];
-	exception_deliver(exception, code, codeCnt, excp, mutex);
+	kr = exception_deliver(thread, exception, code, codeCnt, excp, mutex);
+	if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
+		goto out;
 
 	/*
 	 * How about at the host level?
@@ -306,7 +357,9 @@ exception_triage(
 	host_priv = host_priv_self();
 	mutex = mutex_addr(host_priv->lock);
 	excp = &host_priv->exc_actions[exception];
-	exception_deliver(exception, code, codeCnt, excp, mutex);
+	kr = exception_deliver(thread, exception, code, codeCnt, excp, mutex);
+	if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
+		goto out;
 
 	/*
 	 * Nobody handled it, terminate the task.
@@ -325,23 +378,23 @@ exception_triage(
 #endif	/* MACH_KDB */
 
 	(void) task_terminate(task);
-	thread_exception_return();
-	/*NOTREACHED*/
+
+out:
+	if (exception != EXC_CRASH)
+		thread_exception_return();
+	return;
 }
 
 kern_return_t
 bsd_exception(
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t  codeCnt)
 {
 	task_t			task;
 	struct exception_action *excp;
 	mutex_t			*mutex;
 	thread_t		self = current_thread();
-	ipc_port_t		exc_port;
-	int			behavior;
-	int			flavor;
 	kern_return_t		kr;
 
 	/*
@@ -351,174 +404,73 @@ bsd_exception(
 	mutex = mutex_addr(task->lock);
 	excp = &task->exc_actions[exception];
 
-	/*
-	 *  Save work if we are terminating.
-	 *  Just go back to our AST handler.
-	 */
-	if (!self->active) {
-		return(KERN_FAILURE);
-	}
-
-	/*
-	 * Snapshot the exception action data under lock for consistency.
-	 * Hold a reference to the port over the exception_raise_* calls
-	 * so it can't be destroyed.  This seems like overkill, but keeps
-	 * the port from disappearing between now and when
-	 * ipc_object_copyin_from_kernel is finally called.
-	 */
-	mutex_lock(mutex);
-	exc_port = excp->port;
-	if (!IP_VALID(exc_port)) {
-		mutex_unlock(mutex);
-		return(KERN_FAILURE);
-	}
-	ip_lock(exc_port);
-	if (!ip_active(exc_port)) {
-		ip_unlock(exc_port);
-		mutex_unlock(mutex);
-		return(KERN_FAILURE);
-	}
-	ip_reference(exc_port);	
-	exc_port->ip_srights++;
-	ip_unlock(exc_port);
-
-	flavor = excp->flavor;
-	behavior = excp->behavior;
-	mutex_unlock(mutex);
-
-	switch (behavior) {
-	case EXCEPTION_STATE: {
-		mach_msg_type_number_t state_cnt;
-		thread_state_data_t state;
+	kr = exception_deliver(self, exception, code, codeCnt, excp, mutex);
 
-		c_thr_exc_raise_state++;
-		state_cnt = _MachineStateCount[flavor];
-		kr = thread_getstatus(self, flavor, 
-				      (thread_state_t)state,
-				      &state_cnt);
-		if (kr == KERN_SUCCESS) {
-			kr = exception_raise_state(exc_port, exception,
-						   code, codeCnt,
-						   &flavor,
-						   state, state_cnt,
-						   state, &state_cnt);
-			if (kr == MACH_MSG_SUCCESS)
-				kr = thread_setstatus(self, flavor, 
-						      (thread_state_t)state,
-						      state_cnt);
-		}
-
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			return(KERN_SUCCESS);
-
-		return(KERN_FAILURE);
-	}
-
-	case EXCEPTION_DEFAULT:
-		c_thr_exc_raise++;
-		kr = exception_raise(exc_port,
-				retrieve_thread_self_fast(self),
-				retrieve_task_self_fast(self->task),
-				exception,
-				code, codeCnt);
+	if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
+		return(KERN_SUCCESS);
+	return(KERN_FAILURE);
+}
 
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			return(KERN_SUCCESS);
-		return(KERN_FAILURE);
 
-	case EXCEPTION_STATE_IDENTITY: {
-		mach_msg_type_number_t state_cnt;
-		thread_state_data_t state;
+/*
+ * Raise an EXC_CRASH exception on the dying task.
+ * This should tell launchd to launch Crash Reporter for this task.
+ */
+kern_return_t abnormal_exit_notify(mach_exception_data_type_t exccode, 
+		mach_exception_data_type_t excsubcode)
+{
+	mach_exception_data_type_t	code[EXCEPTION_CODE_MAX];
+	wait_interrupt_t		wsave;
 
-		c_thr_exc_raise_state_id++;
-		state_cnt = _MachineStateCount[flavor];
-		kr = thread_getstatus(self, flavor,
-				      (thread_state_t)state,
-				      &state_cnt);
-		if (kr == KERN_SUCCESS) {
-		    kr = exception_raise_state_identity(exc_port,
-				retrieve_thread_self_fast(self),
-				retrieve_task_self_fast(self->task),
-				exception,
-				code, codeCnt,
-				&flavor,
-				state, state_cnt,
-				state, &state_cnt);
-		    if (kr == MACH_MSG_SUCCESS)
-			kr = thread_setstatus(self, flavor,
-					      (thread_state_t)state,
-					      state_cnt);
-		}
+	code[0] = exccode;
+	code[1] = excsubcode;
 
-		if (kr == KERN_SUCCESS || kr == MACH_RCV_PORT_DIED)
-			return(KERN_SUCCESS);
-		return(KERN_FAILURE);
-	}
-	
-	default:
-		
-		return(KERN_FAILURE);
-	}/* switch */
-	return(KERN_FAILURE);
+	wsave = thread_interrupt_level(THREAD_UNINT);
+	exception_triage(EXC_CRASH, code, EXCEPTION_CODE_MAX);
+	(void) thread_interrupt_level(wsave);
+	return (KERN_SUCCESS);
 }
 
 
-
-
 /*
- *	Handle interface for special perfomance monitoring
+ *	Handle interface for special performance monitoring
  *	This is a special case of the host exception handler
  */
-
-kern_return_t sys_perf_notify(struct task *task,
-	exception_data_t code,
-	mach_msg_type_number_t codeCnt)
+kern_return_t sys_perf_notify(thread_t thread, int pid) 
 {
 	host_priv_t		hostp;
 	struct exception_action *excp;
-	thread_t		thread = current_thread();
 	ipc_port_t		xport;
-	kern_return_t	ret;
 	wait_interrupt_t	wsave;
+	kern_return_t		ret;
 
-	hostp = host_priv_self();				/* Get the host privileged ports */
-	excp = &hostp->exc_actions[EXC_RPC_ALERT];	/* Point to the RPC_ALERT action */
+	hostp = host_priv_self();	/* Get the host privileged ports */
+	mach_exception_data_type_t	code[EXCEPTION_CODE_MAX];
+	code[0] = 0xFF000001;		/* Set terminate code */
+	code[1] = pid;		/* Pass out the pid */
 
-	mutex_lock(&hostp->lock);				/* Lock the priv port */
-	xport = excp->port;						/* Get the port for this exception */
-	if (!IP_VALID(xport)) {					/* Is it valid? */
-		mutex_unlock(&hostp->lock);			/* Unlock */
-		return(KERN_FAILURE);				/* Go away... */
-	}
+	struct task *task = thread->task;
+	excp = &hostp->exc_actions[EXC_RPC_ALERT];	
+	xport = excp->port;	
 
-	ip_lock(xport);							/* Lock the exception port */
-	if (!ip_active(xport)) {				/* and is it active? */
-		ip_unlock(xport);					/* Nope, fail */
-		mutex_unlock(&hostp->lock);			/* Unlock */
-		return(KERN_FAILURE);				/* Go away... */
-	}
+	/* Make sure we're not catching our own exception */
+	if (!IP_VALID(xport) ||
+			!ip_active(xport) ||
+			task->itk_space == xport->data.receiver) {
 
-	if (task->itk_space == xport->data.receiver) {	/* Are we trying to send to ourselves? */
-		ip_unlock(xport);					/* Yes, fail */
-		mutex_unlock(&hostp->lock);			/* Unlock */
-		return(KERN_FAILURE);				/* Go away... */
+		return(KERN_FAILURE);	
 	}
-	
-	ip_reference(xport);					/* Bump reference so it doesn't go away */
-	xport->ip_srights++;					/* Bump send rights */
-	ip_unlock(xport);						/* We can unlock it now */
-
-	mutex_unlock(&hostp->lock);				/* All done with the lock */
-
-	wsave = thread_interrupt_level(THREAD_UNINT);	/* Make sure we aren't aborted here */
-	
-	ret = exception_raise(xport,			/* Send the exception to the perf handler */
-		retrieve_thread_self_fast(thread),		/* Not always the dying guy */
-		retrieve_task_self_fast(thread->task),	/* Not always the dying guy */
-		EXC_RPC_ALERT,						/* Unused exception type until now */
-		code, codeCnt);	
-		
-	(void)thread_interrupt_level(wsave);	/* Restore interrupt level */			
-
-	return(ret);							/* Tell caller how it went */
+
+	wsave = thread_interrupt_level(THREAD_UNINT);	
+	ret = exception_deliver(
+			thread,
+			EXC_RPC_ALERT, 
+			code, 
+			2, 
+			excp,
+			&hostp->lock);
+	(void)thread_interrupt_level(wsave);
+
+	return(ret);
 }
+
diff --git a/osfmk/kern/exception.h b/osfmk/kern/exception.h
index 8ab24a482..78e659a97 100644
--- a/osfmk/kern/exception.h
+++ b/osfmk/kern/exception.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -45,12 +51,14 @@ struct exception_action {
 /* Make an up-call to a thread's exception server */
 extern void exception_triage(
 	exception_type_t	exception,
-	exception_data_t	code,
+	mach_exception_data_t	code,
 	mach_msg_type_number_t	codeCnt);
 
 /* Notify system performance monitor */
-extern kern_return_t sys_perf_notify(struct task *task,
-	exception_data_t	code,
-	mach_msg_type_number_t  codeCnt);
+extern kern_return_t sys_perf_notify(thread_t thread, int pid);
+
+/* Notify crash reporter */
+extern kern_return_t abnormal_exit_notify(mach_exception_data_type_t code, 
+		mach_exception_data_type_t subcode);
 
 #endif	/* _KERN_EXCEPTION_H_ */
diff --git a/osfmk/kern/hibernate.c b/osfmk/kern/hibernate.c
index 4bb00c5dd..3decaefe2 100644
--- a/osfmk/kern/hibernate.c
+++ b/osfmk/kern/hibernate.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/kalloc.h>
@@ -30,11 +36,11 @@
 #include <mach/mach_types.h>
 #include <default_pager/default_pager_internal.h>
 #include <IOKit/IOPlatformExpert.h>
-#define KERNEL
 
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_purgeable_internal.h>
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -67,7 +73,7 @@ hibernate_page_list_zero(hibernate_page_list_t *list)
 static boolean_t 
 consider_discard(vm_page_t m)
 {
-    register vm_object_t object = 0;
+    vm_object_t object = NULL;
     int                  refmod_state;
     boolean_t            discard = FALSE;
 
@@ -101,6 +107,9 @@ consider_discard(vm_page_t m)
         if (m->cleaning)
             break;
 
+	if (m->laundry || m->list_req_pending)
+            break;
+
         if (!m->dirty)
         {
             refmod_state = pmap_get_refmod(m->phys_page);
@@ -112,9 +121,12 @@ consider_discard(vm_page_t m)
         }
    
         /*
-         * If it's clean we can discard the page on wakeup.
+         * If it's clean or purgeable we can discard the page on wakeup.
+	 * JMM - consider purgeable (volatile or empty) objects here as well.
          */
-        discard = !m->dirty;
+        discard = (!m->dirty) 
+		    || (VM_PURGABLE_VOLATILE == object->purgable)
+		    || (VM_PURGABLE_EMPTY    == m->object->purgable);
     }
     while (FALSE);
 
@@ -134,18 +146,11 @@ discard_page(vm_page_t m)
         */
         return;
 
-    if (!m->no_isync) 
+    if (m->pmapped == TRUE) 
     {
-        int refmod_state = pmap_disconnect(m->phys_page);
-
-        if (refmod_state & VM_MEM_REFERENCED)
-            m->reference = TRUE;
-        if (refmod_state & VM_MEM_MODIFIED)
-            m->dirty = TRUE;
+        __unused int refmod_state = pmap_disconnect(m->phys_page);
     }
 
-    if (m->dirty)
-        panic("discard_page(%p) dirty", m);
     if (m->laundry)
         panic("discard_page(%p) laundry", m);
     if (m->private)
@@ -153,6 +158,20 @@ discard_page(vm_page_t m)
     if (m->fictitious)
         panic("discard_page(%p) fictitious", m);
 
+    if (VM_PURGABLE_VOLATILE == m->object->purgable)
+	{
+		assert(m->object->objq.next != NULL && m->object->objq.prev != NULL); /* object should be on a queue */
+		purgeable_q_t old_queue=vm_purgeable_object_remove(m->object);
+		assert(old_queue);
+		/* No need to lock page queue for token delete, hibernate_vm_unlock() 
+			makes sure these locks are uncontended before sleep */
+		vm_purgeable_token_delete_first(old_queue);
+		m->object->purgable = VM_PURGABLE_EMPTY;
+	}
+	
+    if (m->tabled)
+	vm_page_remove(m);
+
     vm_page_free(m);
 }
 
@@ -161,7 +180,6 @@ discard_page(vm_page_t m)
  pages known to VM to not need saving are subtracted.
  Wired pages to be saved are present in page_list_wired, pageable in page_list.
 */
-extern vm_page_t vm_lopage_queue_free;
 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,
@@ -171,10 +189,16 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
     uint64_t start, end, nsec;
     vm_page_t m;
     uint32_t pages = page_list->page_count;
-    uint32_t count_zf = 0, count_inactive = 0, count_active = 0;
+    uint32_t count_zf = 0, count_throttled = 0, count_inactive = 0, count_active = 0;
     uint32_t count_wire = pages;
-    uint32_t count_discard_active = 0, count_discard_inactive = 0;
+    uint32_t count_discard_active    = 0;
+    uint32_t count_discard_inactive  = 0;
+    uint32_t count_discard_purgeable = 0;
     uint32_t i;
+    uint32_t             bank;
+    hibernate_bitmap_t * bitmap;
+    hibernate_bitmap_t * bitmap_wired;
+
 
     HIBLOG("hibernate_page_list_setall start\n");
 
@@ -193,24 +217,46 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
 	m = (vm_page_t) m->pageq.next;
     }
 
-    m = (vm_page_t) vm_page_queue_free;
-    while(m)
+    for( i = 0; i < vm_colors; i++ )
+    {
+	queue_iterate(&vm_page_queue_free[i],
+		      m,
+		      vm_page_t,
+		      pageq)
+	{
+	    pages--;
+	    count_wire--;
+	    hibernate_page_bitset(page_list,       TRUE, m->phys_page);
+	    hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
+	}
+    }
+
+    queue_iterate(&vm_lopage_queue_free,
+		  m,
+		  vm_page_t,
+		  pageq)
     {
 	pages--;
 	count_wire--;
 	hibernate_page_bitset(page_list,       TRUE, m->phys_page);
 	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-	m = (vm_page_t) m->pageq.next;
     }
 
-    m = (vm_page_t) vm_lopage_queue_free;
-    while(m)
+    queue_iterate( &vm_page_queue_throttled,
+                    m,
+                    vm_page_t,
+                    pageq )
     {
-	pages--;
+        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
+         && consider_discard(m))
+        {
+            hibernate_page_bitset(page_list, TRUE, m->phys_page);
+            count_discard_inactive++;
+        }
+        else
+            count_throttled++;
 	count_wire--;
-	hibernate_page_bitset(page_list,       TRUE, m->phys_page);
 	hibernate_page_bitset(page_list_wired, TRUE, m->phys_page);
-	m = (vm_page_t) m->pageq.next;
     }
 
     queue_iterate( &vm_page_queue_zf,
@@ -222,7 +268,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
-            count_discard_inactive++;
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_inactive++;
         }
         else
             count_zf++;
@@ -239,7 +288,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
-            count_discard_inactive++;
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_inactive++;
         }
         else
             count_inactive++;
@@ -256,7 +308,10 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
          && consider_discard(m))
         {
             hibernate_page_bitset(page_list, TRUE, m->phys_page);
-            count_discard_active++;
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_active++;
         }
         else
             count_active++;
@@ -266,10 +321,6 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
 
     // pull wired from hibernate_bitmap
 
-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-    hibernate_bitmap_t * bitmap_wired;
-
     bitmap = &page_list->bank_bitmap[0];
     bitmap_wired = &page_list_wired->bank_bitmap[0];
     for (bank = 0; bank < page_list->bank_count; bank++)
@@ -287,11 +338,11 @@ hibernate_page_list_setall(hibernate_page_list_t * page_list,
     absolutetime_to_nanoseconds(end - start, &nsec);
     HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
 
-    HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, could discard act %d inact %d\n", 
-                pages, count_wire, count_active, count_inactive, count_zf,
-                count_discard_active, count_discard_inactive);
+    HIBLOG("pages %d, wire %d, act %d, inact %d, zf %d, throt %d, could discard act %d inact %d purgeable %d\n", 
+                pages, count_wire, count_active, count_inactive, count_zf, count_throttled,
+                count_discard_active, count_discard_inactive, count_discard_purgeable);
 
-    *pagesOut = pages;
+    *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable;
 }
 
 void
@@ -300,7 +351,9 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
     uint64_t  start, end, nsec;
     vm_page_t m;
     vm_page_t next;
-    uint32_t  count_discard_active = 0, count_discard_inactive = 0;
+    uint32_t  count_discard_active    = 0;
+    uint32_t  count_discard_inactive  = 0;
+    uint32_t  count_discard_purgeable = 0;
 
     clock_get_uptime(&start);
 
@@ -310,8 +363,11 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_inactive++;
             discard_page(m);
-            count_discard_inactive++;
         }
         m = next;
     }
@@ -322,8 +378,11 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_inactive++;
             discard_page(m);
-            count_discard_inactive++;
         }
         m = next;
     }
@@ -334,17 +393,20 @@ hibernate_page_list_discard(hibernate_page_list_t * page_list)
         next = (vm_page_t) m->pageq.next;
         if (hibernate_page_bittst(page_list, m->phys_page))
         {
+	    if (m->dirty)
+		count_discard_purgeable++;
+	    else
+		count_discard_active++;
             discard_page(m);
-            count_discard_active++;
         }
         m = next;
     }
 
     clock_get_uptime(&end);
     absolutetime_to_nanoseconds(end - start, &nsec);
-    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d\n",
+    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d\n",
                 nsec / 1000000ULL,
-                count_discard_active, count_discard_inactive);
+                count_discard_active, count_discard_inactive, count_discard_purgeable);
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@@ -385,7 +447,7 @@ hibernate_setup(IOHibernateImageHeader * header,
 
     hibernate_processor_setup(header);
 
-    HIBLOG("hibernate_alloc_pages flags %08lx, gobbling %d pages\n", 
+    HIBLOG("hibernate_alloc_pages flags %08x, gobbling %d pages\n", 
 	    header->processorFlags, gobble_count);
 
     if (gobble_count)
diff --git a/osfmk/kern/host.c b/osfmk/kern/host.c
index d2c24126c..010ac2887 100644
--- a/osfmk/kern/host.c
+++ b/osfmk/kern/host.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -56,8 +62,6 @@
  *	Non-ipc host functions.
  */
 
-#include <mach_host.h>
-
 #include <mach/mach_types.h>
 #include <mach/boolean.h>
 #include <mach/host_info.h>
@@ -84,11 +88,6 @@
 
 #include <vm/vm_map.h>
 
-#if     DIPC
-#include <dipc/dipc_funcs.h>
-#include <dipc/special_ports.h>
-#endif
-
 host_data_t	realhost;
 
 kern_return_t
@@ -164,9 +163,9 @@ host_info(
 
 		basic_info = (host_basic_info_t) info;
 
-		basic_info->max_cpus = machine_info.max_cpus;
-		basic_info->avail_cpus = machine_info.avail_cpus;
 		basic_info->memory_size = machine_info.memory_size;
+		basic_info->max_cpus = machine_info.max_cpus;
+		basic_info->avail_cpus = processor_avail_count;
 		master_slot = PROCESSOR_DATA(master_processor, slot_num);
 		basic_info->cpu_type = slot_type(master_slot);
 		basic_info->cpu_subtype = slot_subtype(master_slot);
@@ -293,6 +292,7 @@ host_statistics(
 		register processor_t		processor;
 		register vm_statistics_t	stat;
 		vm_statistics_data_t		host_vm_stat;
+		mach_msg_type_number_t		original_count;
                 
 		if (*count < HOST_VM_INFO_REV0_COUNT)
 			return (KERN_FAILURE);
@@ -322,7 +322,7 @@ host_statistics(
 
 		stat = (vm_statistics_t) info;
 
-		stat->free_count = vm_page_free_count;
+		stat->free_count = vm_page_free_count + vm_page_speculative_count;
 		stat->active_count = vm_page_active_count;
 		stat->inactive_count = vm_page_inactive_count;
 		stat->wire_count = vm_page_wire_count;
@@ -335,13 +335,23 @@ host_statistics(
 		stat->lookups = host_vm_stat.lookups;
 		stat->hits = host_vm_stat.hits;
 
-		if (*count >= HOST_VM_INFO_COUNT) {
-			/* info that was not in revision 0 of that interface */
+		/*
+		 * Fill in extra info added in later revisions of the
+		 * vm_statistics data structure.  Fill in only what can fit
+		 * in the data structure the caller gave us !
+		 */
+		original_count = *count;
+		*count = HOST_VM_INFO_REV0_COUNT; /* rev0 already filled in */
+		if (original_count >= HOST_VM_INFO_REV1_COUNT) {
+			/* rev1 added "purgeable" info */
 			stat->purgeable_count = vm_page_purgeable_count;
 			stat->purges = vm_page_purged_count;
-			*count = HOST_VM_INFO_COUNT;
-		} else {
-			*count = HOST_VM_INFO_REV0_COUNT;
+			*count = HOST_VM_INFO_REV1_COUNT;
+		}
+		if (original_count >= HOST_VM_INFO_REV2_COUNT) {
+			/* rev2 added "speculative" info */
+			stat->speculative_count = vm_page_speculative_count;
+			*count = HOST_VM_INFO_REV2_COUNT;
 		}
 
 		return (KERN_SUCCESS);
@@ -351,43 +361,34 @@ host_statistics(
 	{
 		register processor_t	processor;
 		host_cpu_load_info_t	cpu_load_info;
-		unsigned long			ticks_value1, ticks_value2;
 
 		if (*count < HOST_CPU_LOAD_INFO_COUNT)
 			return (KERN_FAILURE);
 
-#define GET_TICKS_VALUE(processor, state)	 					\
-MACRO_BEGIN														\
-	do {														\
-		ticks_value1 = *(volatile integer_t *)					\
-			&PROCESSOR_DATA((processor), cpu_ticks[(state)]);	\
-		ticks_value2 = *(volatile integer_t *)					\
-			&PROCESSOR_DATA((processor), cpu_ticks[(state)]);	\
-	} while (ticks_value1 != ticks_value2);						\
-																\
-	cpu_load_info->cpu_ticks[(state)] += ticks_value1;			\
+#define GET_TICKS_VALUE(processor, state, timer)										\
+MACRO_BEGIN																				\
+	cpu_load_info->cpu_ticks[(state)] +=												\
+				timer_grab(&PROCESSOR_DATA(processor, timer)) / hz_tick_interval;		\
 MACRO_END
 
 		cpu_load_info = (host_cpu_load_info_t)info;
 		cpu_load_info->cpu_ticks[CPU_STATE_USER] = 0;
-		cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
 		cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] = 0;
 		cpu_load_info->cpu_ticks[CPU_STATE_IDLE] = 0;
+		cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
 
 		processor = processor_list;
-		GET_TICKS_VALUE(processor, CPU_STATE_USER);
-		GET_TICKS_VALUE(processor, CPU_STATE_NICE);
-		GET_TICKS_VALUE(processor, CPU_STATE_SYSTEM);
-		GET_TICKS_VALUE(processor, CPU_STATE_IDLE);
+		GET_TICKS_VALUE(processor, CPU_STATE_USER, user_state);
+		GET_TICKS_VALUE(processor, CPU_STATE_SYSTEM, system_state);
+		GET_TICKS_VALUE(processor, CPU_STATE_IDLE, idle_state);
 
 		if (processor_count > 1) {
 			simple_lock(&processor_list_lock);
 
 			while ((processor = processor->processor_list) != NULL) {
-				GET_TICKS_VALUE(processor, CPU_STATE_USER);
-				GET_TICKS_VALUE(processor, CPU_STATE_NICE);
-				GET_TICKS_VALUE(processor, CPU_STATE_SYSTEM);
-				GET_TICKS_VALUE(processor, CPU_STATE_IDLE);
+				GET_TICKS_VALUE(processor, CPU_STATE_USER, user_state);
+				GET_TICKS_VALUE(processor, CPU_STATE_SYSTEM, system_state);
+				GET_TICKS_VALUE(processor, CPU_STATE_IDLE, idle_state);
 			}
 
 			simple_unlock(&processor_list_lock);
@@ -417,7 +418,6 @@ host_priv_statistics(
 	return(host_statistics((host_t)host_priv, flavor, info, count));
 }
 
-
 kern_return_t
 host_page_size(
 	host_t		host,
@@ -465,7 +465,7 @@ host_processor_sets(
 	void *addr;
 
 	if (host_priv == HOST_PRIV_NULL)
-		return KERN_INVALID_ARGUMENT;
+		return (KERN_INVALID_ARGUMENT);
 
 	/*
 	 *	Allocate memory.  Can be pageable because it won't be
@@ -474,17 +474,15 @@ host_processor_sets(
 
 	addr = kalloc((vm_size_t) sizeof(mach_port_t));
 	if (addr == 0)
-		return KERN_RESOURCE_SHORTAGE;
+		return (KERN_RESOURCE_SHORTAGE);
 
-	/* take ref for convert_pset_name_to_port */
-	pset_reference(&default_pset);
 	/* do the conversion that Mig should handle */
-	*((ipc_port_t *) addr) = convert_pset_name_to_port(&default_pset);
+	*((ipc_port_t *) addr) = convert_pset_name_to_port(&pset0);
 
 	*pset_list = (processor_set_array_t)addr;
 	*count = 1;
 
-	return KERN_SUCCESS;
+	return (KERN_SUCCESS);
 }
 
 /*
@@ -498,14 +496,15 @@ host_processor_set_priv(
 	processor_set_t	pset_name,
 	processor_set_t	*pset)
 {
-    if ((host_priv == HOST_PRIV_NULL) || (pset_name == PROCESSOR_SET_NULL)) {
-	*pset = PROCESSOR_SET_NULL;
-	return(KERN_INVALID_ARGUMENT);
+    if (host_priv == HOST_PRIV_NULL || pset_name == PROCESSOR_SET_NULL) {
+		*pset = PROCESSOR_SET_NULL;
+
+		return (KERN_INVALID_ARGUMENT);
     }
 
     *pset = pset_name;
-    pset_reference(*pset);
-    return(KERN_SUCCESS);
+
+    return (KERN_SUCCESS);
 }
 
 /*
@@ -660,11 +659,6 @@ host_get_special_port(
 	    id == HOST_SECURITY_PORT || id > HOST_MAX_SPECIAL_PORT )
 		return KERN_INVALID_ARGUMENT;
 
-#if     DIPC
-	if (node != HOST_LOCAL_NODE)
-        	return norma_get_special_port(host_priv, node, id, portp);
-#endif
-
 	host_lock(host_priv);
 	port = realhost.special[id];
 	*portp = ipc_port_copy_send(port);
diff --git a/osfmk/kern/host.h b/osfmk/kern/host.h
index 623884ba0..8381a868b 100644
--- a/osfmk/kern/host.h
+++ b/osfmk/kern/host.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/host_notify.c b/osfmk/kern/host_notify.c
index 9497b7863..064aab94e 100644
--- a/osfmk/kern/host_notify.c
+++ b/osfmk/kern/host_notify.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2003 Apple Computer, Inc.  All rights reserved.
diff --git a/osfmk/kern/host_notify.h b/osfmk/kern/host_notify.h
index 5ed08bce8..180805c85 100644
--- a/osfmk/kern/host_notify.h
+++ b/osfmk/kern/host_notify.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2003 Apple Computer, Inc.  All rights reserved.
@@ -39,7 +45,7 @@ void	host_notify_port_destroy(
 
 void	host_notify_calendar_change(void);
 
-void	host_notify_init(void);
+void	host_notify_init(void) __attribute__((section("__TEXT, initcode")));
 
 #endif /* MACH_KERNEL_PRIVATE */
 
diff --git a/osfmk/kern/host_statistics.h b/osfmk/kern/host_statistics.h
index 2be83e115..582105edc 100644
--- a/osfmk/kern/host_statistics.h
+++ b/osfmk/kern/host_statistics.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -32,14 +38,14 @@
 #ifndef	_KERN_HOST_STATISTICS_H_
 #define _KERN_HOST_STATISTICS_H_
 
+#include <libkern/OSAtomic.h>
 #include <mach/vm_statistics.h>
 #include <kern/processor.h>
 
-#define	VM_STAT(event)									\
+
+#define	VM_STAT_INCR(event)									\
 MACRO_BEGIN 											\
-	disable_preemption();								\
-	PROCESSOR_DATA(current_processor(), vm_stat).event;	\
-	enable_preemption();								\
+	OSAddAtomic(1, (SInt32 *)(&(PROCESSOR_DATA(current_processor(), vm_stat).event)));	\
 MACRO_END
 
 #endif	/* _KERN_HOST_STATISTICS_H_ */
diff --git a/osfmk/kern/ipc_clock.c b/osfmk/kern/ipc_clock.c
index a8524dbe5..04cefb66e 100644
--- a/osfmk/kern/ipc_clock.c
+++ b/osfmk/kern/ipc_clock.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/ipc_host.c b/osfmk/kern/ipc_host.c
index 0a36ae65c..69d620e8c 100644
--- a/osfmk/kern/ipc_host.c
+++ b/osfmk/kern/ipc_host.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -134,8 +140,8 @@ void ipc_host_init(void)
 	/*
 	 *	Set up ipc for default processor set.
 	 */
-	ipc_pset_init(&default_pset);
-	ipc_pset_enable(&default_pset);
+	ipc_pset_init(&pset0);
+	ipc_pset_enable(&pset0);
 
 	/*
 	 *	And for master processor
@@ -277,51 +283,14 @@ void
 ipc_pset_enable(
 	processor_set_t		pset)
 {
-	pset_lock(pset);
-	if (pset->active) {
-		ipc_kobject_set(pset->pset_self,
-				(ipc_kobject_t) pset, IKOT_PSET);
-		ipc_kobject_set(pset->pset_name_self,
-				(ipc_kobject_t) pset, IKOT_PSET_NAME);
-		pset->ref_count += 2;
-	}
-	pset_unlock(pset);
+	ipc_kobject_set(pset->pset_self, (ipc_kobject_t) pset, IKOT_PSET);
+	ipc_kobject_set(pset->pset_name_self, (ipc_kobject_t) pset, IKOT_PSET_NAME);
 }
 
 /*
- *	ipc_pset_disable:
+ *	processor_set_default:
  *
- *	Disable ipc access to a processor set by clearing the port objects.
- *	Caller must hold pset lock and a reference to the pset.  Ok to
- *	just decrement pset reference count as a result.
- */
-void
-ipc_pset_disable(
-	processor_set_t		pset)
-{
-	ipc_kobject_set(pset->pset_self, IKO_NULL, IKOT_NONE);
-	ipc_kobject_set(pset->pset_name_self, IKO_NULL, IKOT_NONE);
-	pset->ref_count -= 2;
-}
-
-/*
- *	ipc_pset_terminate:
- *
- *	Processor set is dead.  Deallocate the ipc control structures.
- */
-void
-ipc_pset_terminate(
-	processor_set_t		pset)
-{
-	ipc_port_dealloc_kernel(pset->pset_self);
-	ipc_port_dealloc_kernel(pset->pset_name_self);
-}
-
-/*
- *	processor_set_default, processor_set_default_priv:
- *
- *	Return ports for manipulating default_processor set.  MiG code
- *	differentiates between these two routines.
+ *	Return ports for manipulating default_processor set.
  */
 kern_return_t
 processor_set_default(
@@ -331,9 +300,9 @@ processor_set_default(
 	if (host == HOST_NULL)
 		return(KERN_INVALID_ARGUMENT);
 
-	*pset = &default_pset;
-	pset_reference(*pset);
-	return(KERN_SUCCESS);
+	*pset = &pset0;
+
+	return (KERN_SUCCESS);
 }
 
 /*
@@ -477,18 +446,13 @@ ref_pset_port_locked(ipc_port_t port, boolean_t matchn, processor_set_t *ppset)
 	pset = PROCESSOR_SET_NULL;
 	if (ip_active(port) &&
 		((ip_kotype(port) == IKOT_PSET) ||
-		 (matchn && (ip_kotype(port) == IKOT_PSET_NAME)))) {
+			(matchn && (ip_kotype(port) == IKOT_PSET_NAME)))) {
 		pset = (processor_set_t) port->ip_kobject;
-		if (!pset_lock_try(pset)) {
-			ip_unlock(port);
-			mutex_pause();
-			return (FALSE);
-		}
-		pset->ref_count++;
-		pset_unlock(pset);
 	}
+
 	*ppset = pset;
 	ip_unlock(port);
+
 	return (TRUE);
 }
 
@@ -545,7 +509,7 @@ convert_processor_to_port(
  *	Routine:	convert_pset_to_port
  *	Purpose:
  *		Convert from a pset to a port.
- *		Consumes a pset ref; produces a naked send right
+ *		Produces a naked send right
  *		which may be invalid.
  *	Conditions:
  *		Nothing locked.
@@ -555,16 +519,11 @@ ipc_port_t
 convert_pset_to_port(
 	processor_set_t		pset)
 {
-	ipc_port_t port;
+	ipc_port_t port = pset->pset_self;
 
-	pset_lock(pset);
-	if (pset->active)
-		port = ipc_port_make_send(pset->pset_self);
-	else
-		port = IP_NULL;
-	pset_unlock(pset);
+	if (port != IP_NULL)
+		port = ipc_port_make_send(port);
 
-	pset_deallocate(pset);
 	return port;
 }
 
@@ -572,7 +531,7 @@ convert_pset_to_port(
  *	Routine:	convert_pset_name_to_port
  *	Purpose:
  *		Convert from a pset to a port.
- *		Consumes a pset ref; produces a naked send right
+ *		Produces a naked send right
  *		which may be invalid.
  *	Conditions:
  *		Nothing locked.
@@ -582,16 +541,11 @@ ipc_port_t
 convert_pset_name_to_port(
 	processor_set_name_t		pset)
 {
-	ipc_port_t port;
+	ipc_port_t port = pset->pset_name_self;
 
-	pset_lock(pset);
-	if (pset->active)
-		port = ipc_port_make_send(pset->pset_name_self);
-	else
-		port = IP_NULL;
-	pset_unlock(pset);
+	if (port != IP_NULL)
+		port = ipc_port_make_send(port);
 
-	pset_deallocate(pset);
 	return port;
 }
 
@@ -659,7 +613,7 @@ host_set_exception_ports(
 	}
 
 	if (IP_VALID(new_port)) {
-		switch (new_behavior) {
+		switch (new_behavior & ~MACH_EXCEPTION_CODES) {
 		case EXCEPTION_DEFAULT:
 		case EXCEPTION_STATE:
 		case EXCEPTION_STATE_IDENTITY:
diff --git a/osfmk/kern/ipc_host.h b/osfmk/kern/ipc_host.h
index 6b4a0916b..863be913f 100644
--- a/osfmk/kern/ipc_host.h
+++ b/osfmk/kern/ipc_host.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,7 +63,7 @@
 #include <kern/kern_types.h>
 
 /* Initialize IPC host services */
-extern void ipc_host_init(void);
+extern void ipc_host_init(void) __attribute__((section("__TEXT, initcode")));
 
 /* Initialize ipc access to processor by allocating a port */
 extern void ipc_processor_init(
@@ -75,14 +81,6 @@ extern void ipc_pset_init(
 extern void ipc_pset_enable(
 	processor_set_t		pset);
 
-/* Disable ipc access to a processor set */
-extern void ipc_pset_disable(
-	processor_set_t		pset);
-
-/* Deallocate the ipc control structures for a processor set */
-extern void ipc_pset_terminate(
-	processor_set_t		pset);
-
 /* Initialize ipc control of a clock */
 extern void ipc_clock_init(
 	clock_t		clock);
diff --git a/osfmk/kern/ipc_kobject.c b/osfmk/kern/ipc_kobject.c
index 62230a676..646788daa 100644
--- a/osfmk/kern/ipc_kobject.c
+++ b/osfmk/kern/ipc_kobject.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -113,11 +126,13 @@
 #include <kern/misc_protos.h>
 #include <ipc/ipc_kmsg.h>
 #include <ipc/ipc_port.h>
+#include <ipc/ipc_labelh.h>
 #include <kern/counters.h>
 
-#include <vm/vm_shared_memory_server.h>
 #include <vm/vm_protos.h>
 
+#include <security/mac_mach_internal.h>
+
 /*
  *	Routine:	ipc_kobject_notify
  *	Purpose:
@@ -148,6 +163,9 @@ mig_hash_t mig_buckets[MAX_MIG_ENTRIES];
 int mig_table_max_displ;
 mach_msg_size_t mig_reply_size;
 
+#if CONFIG_MACF
+#include <mach/security_server.h>
+#endif
 
 
 
@@ -181,6 +199,10 @@ const struct mig_subsystem *mig_e[] = {
 #if     MCMSG && iPSC860
 	(const struct mig_subsystem *)&mcmsg_info_subsystem,
 #endif  /* MCMSG && iPSC860 */
+
+#if CONFIG_MACF
+	(const struct mig_subsystem *)&security_subsystem,
+#endif
 };
 
 void
@@ -202,7 +224,7 @@ mig_init(void)
 	    nentry = j + mig_e[i]->start;	
 	    for (pos = MIG_HASH(nentry) % MAX_MIG_ENTRIES, howmany = 1;
 		 mig_buckets[pos].num;
-		 pos = ++pos % MAX_MIG_ENTRIES, howmany++) {
+		 pos++, pos = pos % MAX_MIG_ENTRIES, howmany++) {
 	         if (mig_buckets[pos].num == nentry) {
 		        printf("message id = %d\n", nentry);
 		 	panic("multiple entries with the same msgh_id");
@@ -427,6 +449,11 @@ ipc_kobject_set(
 {
 	ip_lock(port);
 	ipc_kobject_set_atomically(port, kobject, type);
+
+#if CONFIG_MACF_MACH
+	mac_port_label_update_kobject (&port->ip_label, type);
+#endif
+
 	ip_unlock(port);
 }
 
@@ -476,6 +503,12 @@ ipc_kobject_destroy(
 		host_notify_port_destroy(port);
 		break;
 
+#if CONFIG_MACF_MACH
+	case IKOT_LABELH:
+		labelh_destroy(port);
+		break;
+#endif
+
 	default:
 		break;
 	}
@@ -510,8 +543,7 @@ ipc_kobject_notify(
 				(mach_port_mscount_t) 
 				((mach_no_senders_notification_t *) 
 				 request_header)->not_count);
-			   (ipc_port_t)reply_header->msgh_remote_port
-				   = MACH_PORT_NULL;
+			   reply_header->msgh_remote_port = MACH_PORT_NULL;
 			   return TRUE;
 		   }
 				
@@ -585,7 +617,7 @@ kobjserver_stats(void)
 	    nentry = j + mig_e[i]->start;	
 	    for (pos = MIG_HASH(nentry) % MAX_MIG_ENTRIES, howmany = 1;
 		 mig_buckets[pos].num;
-		 pos = ++pos % MAX_MIG_ENTRIES, howmany++) {
+		 pos++, pos = pos % MAX_MIG_ENTRIES, howmany++) {
 		    if (mig_buckets[pos].num == nentry)
 			bucket_stats_print(&mig_buckets[pos]);
 	    }
diff --git a/osfmk/kern/ipc_kobject.h b/osfmk/kern/ipc_kobject.h
index 9d19732bd..b2e7d1340 100644
--- a/osfmk/kern/ipc_kobject.h
+++ b/osfmk/kern/ipc_kobject.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -108,11 +120,12 @@ typedef natural_t	ipc_kobject_type_t;
 #define IKOT_IOKIT_OBJECT		30
 #define IKOT_UPL				31
 #define IKOT_MEM_OBJ_CONTROL		32
+#define IKOT_LABELH			33
 /*
  * Add new entries here and adjust IKOT_UNKNOWN.
  * Please keep ipc/ipc_object.c:ikot_print_array up to date.
  */
-#define	IKOT_UNKNOWN			33	/* magic catchall	*/
+#define	IKOT_UNKNOWN			34	/* magic catchall	*/
 #define	IKOT_MAX_TYPE	(IKOT_UNKNOWN+1)	/* # of IKOT_ types	*/
 
 
@@ -147,3 +160,4 @@ extern void		ipc_kobject_destroy(
 #endif /* KERNEL_PRIVATE */
 
 #endif /* _KERN_IPC_KOBJECT_H_ */
+
diff --git a/osfmk/kern/ipc_mig.c b/osfmk/kern/ipc_mig.c
index dcd9e87f2..3b2ae194b 100644
--- a/osfmk/kern/ipc_mig.c
+++ b/osfmk/kern/ipc_mig.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -112,6 +118,32 @@ mach_msg_send_from_kernel(
 	return MACH_MSG_SUCCESS;
 }
 
+mach_msg_return_t
+mach_msg_send_from_kernel_with_options(
+	mach_msg_header_t	*msg,
+	mach_msg_size_t		send_size,
+	mach_msg_option_t	option,
+	mach_msg_timeout_t	timeout_val)
+{
+	ipc_kmsg_t kmsg;
+	mach_msg_return_t mr;
+
+	if (!MACH_PORT_VALID((mach_port_name_t)msg->msgh_remote_port))
+		return MACH_SEND_INVALID_DEST;
+
+	mr = ipc_kmsg_get_from_kernel(msg, send_size, &kmsg);
+	if (mr != MACH_MSG_SUCCESS)
+		return mr;
+
+	ipc_kmsg_copyin_from_kernel(kmsg);
+	mr = ipc_kmsg_send(kmsg, option, timeout_val);
+	if (mr != MACH_MSG_SUCCESS) {
+		ipc_kmsg_free(kmsg);
+	}
+	
+	return mr;
+}
+
 /*
  *	Routine:	mach_msg_rpc_from_kernel
  *	Purpose:
@@ -211,32 +243,40 @@ mach_msg_rpc_from_kernel(
 	}
 	ipc_port_release(reply);
 
-	/*
-	 * XXXXX  Set manually for now ...
-	 *	No, why even bother, since the effort is wasted?
-	 *
-	{ mach_msg_format_0_trailer_t *trailer = (mach_msg_format_0_trailer_t *)
-		((vm_offset_t)&kmsg->ikm_header + kmsg->ikm_header.msgh_size);
-	trailer->msgh_trailer_type = MACH_MSG_TRAILER_FORMAT_0;
-	trailer->msgh_trailer_size = MACH_MSG_TRAILER_MINIMUM_SIZE;
+	/* 
+	 * Check to see how much of the message/trailer can be received.
+	 * We chose the maximum trailer that will fit, since we don't
+	 * have options telling us which trailer elements the caller needed.
+	 */
+	if (rcv_size >= kmsg->ikm_header->msgh_size) {
+		mach_msg_format_0_trailer_t *trailer =  (mach_msg_format_0_trailer_t *)
+			((vm_offset_t)kmsg->ikm_header + kmsg->ikm_header->msgh_size);
+
+		if (rcv_size >= kmsg->ikm_header->msgh_size + MAX_TRAILER_SIZE) {
+			/* Enough room for a maximum trailer */
+			trailer->msgh_trailer_size = MAX_TRAILER_SIZE;
+		} 
+		else if (rcv_size < kmsg->ikm_header->msgh_size + 
+			   trailer->msgh_trailer_size) {
+			/* no room for even the basic (default) trailer */
+			trailer->msgh_trailer_size = 0;
+		}
+		assert(trailer->msgh_trailer_type == MACH_MSG_TRAILER_FORMAT_0);
+		rcv_size = kmsg->ikm_header->msgh_size + trailer->msgh_trailer_size;
+		mr = MACH_MSG_SUCCESS;
+	} else {
+		mr = MACH_RCV_TOO_LARGE;
 	}
-	 *****/
 
-	if (rcv_size < kmsg->ikm_header->msgh_size) {
-		ipc_kmsg_copyout_dest(kmsg, ipc_space_reply);
-		ipc_kmsg_put_to_kernel(msg, kmsg, kmsg->ikm_header->msgh_size);
-		return MACH_RCV_TOO_LARGE;
-	}
 
 	/*
 	 *	We want to preserve rights and memory in reply!
 	 *	We don't have to put them anywhere; just leave them
 	 *	as they are.
 	 */
-
 	ipc_kmsg_copyout_to_kernel(kmsg, ipc_space_reply);
-	ipc_kmsg_put_to_kernel(msg, kmsg, kmsg->ikm_header->msgh_size);
-	return MACH_MSG_SUCCESS;
+	ipc_kmsg_put_to_kernel(msg, kmsg, rcv_size);
+	return mr;
 }
 
 
diff --git a/osfmk/kern/ipc_mig.h b/osfmk/kern/ipc_mig.h
index e25e071ad..3ddfc945d 100644
--- a/osfmk/kern/ipc_mig.h
+++ b/osfmk/kern/ipc_mig.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -135,6 +141,12 @@ extern mach_msg_return_t mach_msg_rpc_from_kernel(
 	mach_msg_size_t		send_size,
 	mach_msg_size_t		rcv_size);
 
+extern mach_msg_return_t mach_msg_send_from_kernel_with_options(
+	mach_msg_header_t	*msg,
+	mach_msg_size_t		send_size,
+	mach_msg_option_t	option,
+	mach_msg_timeout_t	timeout_val);
+
 __END_DECLS
 
 #ifdef	MACH_KERNEL_PRIVATE
@@ -142,7 +154,7 @@ __END_DECLS
 extern void mach_msg_receive_continue(void);
 
 /* Initialize kernel server dispatch table */
-extern void		mig_init(void);
+extern void		mig_init(void) __attribute__((section("__TEXT, initcode")));
 
 /*
  * Kernel implementation of the MIG object base class
diff --git a/osfmk/kern/ipc_sync.c b/osfmk/kern/ipc_sync.c
index 34c8ffc00..b2774b520 100644
--- a/osfmk/kern/ipc_sync.c
+++ b/osfmk/kern/ipc_sync.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/ipc_sync.h b/osfmk/kern/ipc_sync.h
index 5d18af44d..0732b2b9d 100644
--- a/osfmk/kern/ipc_sync.h
+++ b/osfmk/kern/ipc_sync.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/ipc_tt.c b/osfmk/kern/ipc_tt.c
index 548a47297..e14a6fe85 100644
--- a/osfmk/kern/ipc_tt.c
+++ b/osfmk/kern/ipc_tt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 
@@ -82,9 +94,10 @@
 
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
-#include <vm/vm_shared_memory_server.h>
 #include <vm/vm_protos.h>
 
+#include <security/mac_mach_internal.h>
+
 /* forward declarations */
 task_t convert_port_to_locked_task(ipc_port_t port);
 
@@ -116,6 +129,7 @@ ipc_task_init(
 	if (kr != KERN_SUCCESS)
 		panic("ipc_task_init");
 
+	space->is_task = task;
 
 	kport = ipc_port_alloc_kernel();
 	if (kport == IP_NULL)
@@ -132,6 +146,14 @@ ipc_task_init(
 	task->itk_space = space;
 	space->is_fast = FALSE;
 
+#if CONFIG_MACF_MACH
+	if (parent)
+		mac_task_label_associate(parent, task, &parent->maclabel,
+		    &task->maclabel, &kport->ip_label);
+	else
+		mac_task_label_associate_kernel(task, &task->maclabel, &kport->ip_label);
+#endif
+
 	if (parent == TASK_NULL) {
 		ipc_port_t port;
 
@@ -144,6 +166,10 @@ ipc_task_init(
 		task->itk_host = port;
 
 		task->itk_bootstrap = IP_NULL;
+		task->itk_seatbelt = IP_NULL;
+		task->itk_gssd = IP_NULL;
+		task->itk_automountd = IP_NULL;
+		task->itk_task_access = IP_NULL;
 
 		for (i = 0; i < TASK_PORT_REGISTER_MAX; i++)
 			task->itk_registered[i] = IP_NULL;
@@ -175,6 +201,18 @@ ipc_task_init(
 		task->itk_bootstrap =
 			ipc_port_copy_send(parent->itk_bootstrap);
 
+		task->itk_seatbelt =
+			ipc_port_copy_send(parent->itk_seatbelt);
+
+		task->itk_gssd =
+			ipc_port_copy_send(parent->itk_gssd);
+
+		task->itk_automountd =
+			ipc_port_copy_send(parent->itk_automountd);
+
+		task->itk_task_access =
+			ipc_port_copy_send(parent->itk_task_access);
+
 		itk_unlock(parent);
 	}
 }
@@ -279,6 +317,18 @@ ipc_task_terminate(
 	if (IP_VALID(task->itk_bootstrap))
 		ipc_port_release_send(task->itk_bootstrap);
 
+	if (IP_VALID(task->itk_seatbelt))
+		ipc_port_release_send(task->itk_seatbelt);
+	
+	if (IP_VALID(task->itk_gssd))
+		ipc_port_release_send(task->itk_gssd);
+
+	if (IP_VALID(task->itk_automountd))
+		ipc_port_release_send(task->itk_automountd);
+
+	if (IP_VALID(task->itk_task_access))
+		ipc_port_release_send(task->itk_task_access);
+
 	for (i = 0; i < TASK_PORT_REGISTER_MAX; i++)
 		if (IP_VALID(task->itk_registered[i]))
 			ipc_port_release_send(task->itk_registered[i]);
@@ -813,15 +863,32 @@ task_get_special_port(
 		port = ipc_port_copy_send(task->itk_bootstrap);
 		break;
 
-            case TASK_WIRED_LEDGER_PORT:
+	    case TASK_WIRED_LEDGER_PORT:
 		port = ipc_port_copy_send(task->wired_ledger_port);
-                break;
+		break;
 
-            case TASK_PAGED_LEDGER_PORT:
+	    case TASK_PAGED_LEDGER_PORT:
 		port = ipc_port_copy_send(task->paged_ledger_port);
-                break;
+		break;
                     
+	    case TASK_SEATBELT_PORT:
+		port = ipc_port_copy_send(task->itk_seatbelt);
+		break;
+
+	    case TASK_GSSD_PORT:
+		port = ipc_port_copy_send(task->itk_gssd);
+		break;
+			
+	    case TASK_ACCESS_PORT:
+		port = ipc_port_copy_send(task->itk_task_access);
+		break;
+			
+	    case TASK_AUTOMOUNTD_PORT:
+		port = ipc_port_copy_send(task->itk_automountd);
+		break;
+
 	    default:
+               itk_unlock(task);
 		return KERN_INVALID_ARGUMENT;
 	}
 	itk_unlock(task);
@@ -843,6 +910,7 @@ task_get_special_port(
  *		KERN_INVALID_ARGUMENT	The task is null.
  *		KERN_FAILURE		The task/space is dead.
  *		KERN_INVALID_ARGUMENT	Invalid special port.
+ * 		KERN_NO_ACCESS		Attempted overwrite of seatbelt port.
  */
 
 kern_return_t
@@ -870,14 +938,30 @@ task_set_special_port(
 		whichp = &task->itk_bootstrap;
 		break;
 
-            case TASK_WIRED_LEDGER_PORT:
-                whichp = &task->wired_ledger_port;
-                break;
+	    case TASK_WIRED_LEDGER_PORT:
+		whichp = &task->wired_ledger_port;
+		break;
 
-            case TASK_PAGED_LEDGER_PORT:
-                whichp = &task->paged_ledger_port;
-                break;
+	    case TASK_PAGED_LEDGER_PORT:
+		whichp = &task->paged_ledger_port;
+		break;
                     
+	    case TASK_SEATBELT_PORT:
+		whichp = &task->itk_seatbelt;
+		break;
+
+	    case TASK_GSSD_PORT:
+		whichp = &task->itk_gssd;
+		break;
+		
+	    case TASK_ACCESS_PORT:
+		whichp = &task->itk_task_access;
+		break;
+		
+	    case TASK_AUTOMOUNTD_PORT:
+		whichp = &task->itk_automountd;
+		break;
+		
 	    default:
 		return KERN_INVALID_ARGUMENT;
 	}/* switch */
@@ -888,6 +972,20 @@ task_set_special_port(
 		return KERN_FAILURE;
 	}
 
+	/* do not allow overwrite of seatbelt or task access ports */
+	if ((TASK_SEATBELT_PORT == which  || TASK_ACCESS_PORT == which) 
+		&& IP_VALID(*whichp)) {
+			itk_unlock(task);
+			return KERN_NO_ACCESS;
+	}
+
+#if CONFIG_MACF_MACH
+       if (mac_task_check_service(current_task(), task, "set_special_port")) {
+               itk_unlock(task);
+               return KERN_NO_ACCESS;
+       }
+#endif
+
 	old = *whichp;
 	*whichp = port;
 	itk_unlock(task);
@@ -1047,6 +1145,8 @@ mach_ports_lookup(
 task_t
 convert_port_to_locked_task(ipc_port_t port)
 {
+        int try_failed_count = 0;
+
 	while (IP_VALID(port)) {
 		task_t task;
 
@@ -1066,9 +1166,10 @@ convert_port_to_locked_task(ipc_port_t port)
 			ip_unlock(port);
 			return(task);
 		}
+		try_failed_count++;
 
 		ip_unlock(port);
-		mutex_pause();
+		mutex_pause(try_failed_count);
 	}
 	return TASK_NULL;
 }
@@ -1427,7 +1528,7 @@ thread_set_exception_ports(
 		return (KERN_INVALID_ARGUMENT);
 
 	if (IP_VALID(new_port)) {
-		switch (new_behavior) {
+		switch (new_behavior & ~MACH_EXCEPTION_CODES) {
 
 		case EXCEPTION_DEFAULT:
 		case EXCEPTION_STATE:
@@ -1498,7 +1599,7 @@ task_set_exception_ports(
 		return (KERN_INVALID_ARGUMENT);
 
 	if (IP_VALID(new_port)) {
-		switch (new_behavior) {
+		switch (new_behavior & ~MACH_EXCEPTION_CODES) {
 
 		case EXCEPTION_DEFAULT:
 		case EXCEPTION_STATE:
@@ -1594,7 +1695,7 @@ thread_swap_exception_ports(
 		return (KERN_INVALID_ARGUMENT);
 
 	if (IP_VALID(new_port)) {
-		switch (new_behavior) {
+		switch (new_behavior & ~MACH_EXCEPTION_CODES) {
 
 		case EXCEPTION_DEFAULT:
 		case EXCEPTION_STATE:
@@ -1690,7 +1791,7 @@ task_swap_exception_ports(
 		return (KERN_INVALID_ARGUMENT);
 
 	if (IP_VALID(new_port)) {
-		switch (new_behavior) {
+		switch (new_behavior & ~MACH_EXCEPTION_CODES) {
 
 		case EXCEPTION_DEFAULT:
 		case EXCEPTION_STATE:
diff --git a/osfmk/kern/ipc_tt.h b/osfmk/kern/ipc_tt.h
index 4c2330b4c..4a3a9ac1c 100644
--- a/osfmk/kern/ipc_tt.h
+++ b/osfmk/kern/ipc_tt.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c
index d8da5ae99..3d9d42ca8 100644
--- a/osfmk/kern/kalloc.c
+++ b/osfmk/kern/kalloc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -76,8 +82,9 @@
 zone_t kalloc_zone(vm_size_t);
 #endif
 
+#define KALLOC_MAP_SIZE_MIN  (16 * 1024 * 1024)
+#define KALLOC_MAP_SIZE_MAX  (128 * 1024 * 1024)
 vm_map_t kalloc_map;
-vm_size_t kalloc_map_size = 16 * 1024 * 1024;
 vm_size_t kalloc_max;
 vm_size_t kalloc_max_prerounded;
 vm_size_t kalloc_kernmap_size;	/* size of kallocs that can come from kernel map */
@@ -173,9 +180,19 @@ kalloc_init(
 {
 	kern_return_t retval;
 	vm_offset_t min;
-	vm_size_t size;
+	vm_size_t size, kalloc_map_size;
 	register int i;
 
+	/* 
+	 * Scale the kalloc_map_size to physical memory size: stay below 
+	 * 1/8th the total zone map size, or 128 MB.
+	 */
+	kalloc_map_size = sane_size >> 5;
+	if (kalloc_map_size > KALLOC_MAP_SIZE_MAX)
+		kalloc_map_size = KALLOC_MAP_SIZE_MAX;
+	if (kalloc_map_size < KALLOC_MAP_SIZE_MIN)
+		kalloc_map_size = KALLOC_MAP_SIZE_MIN;
+
 	retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,
 			       FALSE, VM_FLAGS_ANYWHERE, &kalloc_map);
 
@@ -202,7 +219,7 @@ kalloc_init(
 	 */
 	for (i = 0, size = 1; size < kalloc_max; i++, size <<= 1) {
 		if (size < KALLOC_MINSIZE) {
-			k_zone[i] = 0;
+			k_zone[i] = NULL;
 			continue;
 		}
 		if (size == KALLOC_MINSIZE) {
@@ -234,11 +251,11 @@ kalloc_canblock(
 
 		/* kmem_alloc could block so we return if noblock */
 		if (!canblock) {
-		  return(0);
+		  return(NULL);
 		}
 
-		if (size >=  kalloc_kernmap_size) {
-			alloc_map = kernel_map;
+		if (size >= kalloc_kernmap_size) {
+		        alloc_map = kernel_map;
 
 			if (size > kalloc_largest_allocated)
 			        kalloc_largest_allocated = size;
@@ -246,7 +263,7 @@ kalloc_canblock(
 			alloc_map = kalloc_map;
 
 		if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) 
-			addr = 0;
+			addr = NULL;
 
 		if (addr) {
 		        kalloc_large_inuse++;
@@ -327,10 +344,8 @@ krealloc(
 
 			if (KERN_SUCCESS != kmem_realloc(alloc_map, 
 			    (vm_offset_t)*addrp, old_size,
-			    (vm_offset_t *)&naddr, new_size)) {
+			    (vm_offset_t *)&naddr, new_size))
 				panic("krealloc: kmem_realloc");
-				naddr = 0;
-			}
 
 			simple_lock(lock);
 			*addrp = (void *) naddr;
@@ -456,7 +471,7 @@ kfree(
 			if (size > kalloc_largest_allocated)
 			        /*
 				 * work around double FREEs of small MALLOCs
-				 * this used to end up being a nop
+				 * this use to end up being a nop
 				 * since the pointer being freed from an
 				 * alloc backed by the zalloc world could
 				 * never show up in the kalloc_map... however,
@@ -578,7 +593,7 @@ OSMalloc_Tagref(
 	if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) 
 		panic("OSMalloc_Tagref(): bad state 0x%08X\n",tag->OSMT_state);
 
-	(void)hw_atomic_add((uint32_t *)(&tag->OSMT_refcnt), 1);
+	(void)hw_atomic_add(&tag->OSMT_refcnt, 1);
 }
 
 void
@@ -588,7 +603,7 @@ OSMalloc_Tagrele(
 	if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID))
 		panic("OSMalloc_Tagref(): bad state 0x%08X\n",tag->OSMT_state);
 
-	if (hw_atomic_sub((uint32_t *)(&tag->OSMT_refcnt), 1) == 0) {
+	if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
 		if (hw_compare_and_store(OSMT_VALID|OSMT_RELEASED, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state)) {
 			simple_lock(&OSMalloc_tag_lock);
 			(void)remque((queue_entry_t)tag);
@@ -606,7 +621,7 @@ OSMalloc_Tagfree(
 	if (!hw_compare_and_store(OSMT_VALID, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state))
 		panic("OSMalloc_Tagfree(): bad state 0x%08X\n", tag->OSMT_state);
 
-	if (hw_atomic_sub((uint32_t *)(&tag->OSMT_refcnt), 1) == 0) {
+	if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
 		simple_lock(&OSMalloc_tag_lock);
 		(void)remque((queue_entry_t)tag);
 		simple_unlock(&OSMalloc_tag_lock);
@@ -627,10 +642,13 @@ OSMalloc(
 	    && (size & ~PAGE_MASK)) {
 
 		if ((kr = kmem_alloc_pageable(kernel_map, (vm_offset_t *)&addr, size)) != KERN_SUCCESS)
-			panic("OSMalloc(): kmem_alloc_pageable() failed 0x%08X\n", kr);
+			addr = NULL;
 	} else 
 		addr = kalloc((vm_size_t)size);
 
+	if (!addr)
+		OSMalloc_Tagrele(tag);
+
 	return(addr);
 }
 
diff --git a/osfmk/kern/kalloc.h b/osfmk/kern/kalloc.h
index 2e7ecc170..a260e4944 100644
--- a/osfmk/kern/kalloc.h
+++ b/osfmk/kern/kalloc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -75,7 +81,7 @@ __END_DECLS
 
 #define KALLOC_MINSIZE		16
 
-extern void		kalloc_init(void);
+extern void		kalloc_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern void		krealloc(void		**addrp,
 				 vm_size_t	old_size,
diff --git a/osfmk/kern/kern_print.h b/osfmk/kern/kern_print.h
index 2a5b7d062..836211062 100644
--- a/osfmk/kern/kern_print.h
+++ b/osfmk/kern/kern_print.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -48,19 +54,10 @@
 
 #include <ddb/db_expr.h>
 
-extern void 	db_show_all_slocks(void);
-
+extern void db_show_all_slocks(void);
 
-extern void	db_show_one_zone(
-			        db_expr_t       addr,
-			        int		have_addr,
-			        db_expr_t	count,
-			        char *          modif);
+extern void db_show_one_zone(db_expr_t, boolean_t, db_expr_t, char *);
 
-extern void	db_show_all_zones(
-			        db_expr_t	addr,
-			        int		have_addr,
-			        db_expr_t	count,
-			        char *		modif);
+extern void db_show_all_zones(db_expr_t, boolean_t, db_expr_t, char *);
 
 #endif	/* KERN_PRINT_H_ */
diff --git a/osfmk/kern/kern_types.h b/osfmk/kern/kern_types.h
index 6ae7d07f8..bfc5528ab 100644
--- a/osfmk/kern/kern_types.h
+++ b/osfmk/kern/kern_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -68,7 +74,6 @@ typedef int wait_result_t;
 #define THREAD_INTERRUPTED	2		/* aborted/interrupted */
 #define THREAD_RESTART		3		/* restart operation entirely */
 
-
 typedef	void (*thread_continue_t)(void *, wait_result_t);
 #define	THREAD_CONTINUE_NULL	((thread_continue_t) 0)
 
@@ -93,13 +98,16 @@ typedef struct mig_object		*mig_object_t;
 typedef struct mig_notify		*mig_notify_t;
 #define MIG_NOTIFY_NULL 		((mig_notify_t) 0)
 
-typedef boolean_t				(*thread_roust_t)(thread_t, wait_result_t);
-#define THREAD_ROUST_NULL	 	((thread_roust_t) 0)
+typedef struct pset_node		*pset_node_t;
+#define PSET_NODE_NULL			((pset_node_t) 0)
+
+typedef struct affinity_set		*affinity_set_t;
+#define AFFINITY_SET_NULL		((affinity_set_t) 0)
 
 #else	/* MACH_KERNEL_PRIVATE */
 
 struct wait_queue_set ;
-struct wait_queue_link ;
+struct _wait_queue_link ;
 
 #endif	/* MACH_KERNEL_PRIVATE */
 
@@ -107,7 +115,7 @@ typedef struct wait_queue_set	*wait_queue_set_t;
 #define WAIT_QUEUE_SET_NULL 	((wait_queue_set_t)0)
 #define SIZEOF_WAITQUEUE_SET	wait_queue_set_size()
 
-typedef struct wait_queue_link	*wait_queue_link_t;
+typedef struct _wait_queue_link	*wait_queue_link_t;
 #define WAIT_QUEUE_LINK_NULL	((wait_queue_link_t)0)
 #define SIZEOF_WAITQUEUE_LINK	wait_queue_link_size()
 
diff --git a/osfmk/kern/kmod.c b/osfmk/kern/kmod.c
index a79fd2b0b..fac911d11 100644
--- a/osfmk/kern/kmod.c
+++ b/osfmk/kern/kmod.c
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 /*
  * Copyright (c) 1999 Apple Computer, Inc.  All rights reserved. 
@@ -40,33 +52,44 @@
 #include <vm/vm_kern.h>
 
 #include <mach-o/mach_header.h>
-
-#include <mach_host.h>
+#include <mach-o/loader.h>
+#include <mach-o/nlist.h>
 
 /*
  * XXX headers for which prototypes should be in a common include file;
  * XXX see libsa/kext.cpp for why.
  */
-kern_return_t	kmod_create_internal(kmod_info_t *info, kmod_t *id);
-kern_return_t	kmod_destroy_internal(kmod_t id);
-kern_return_t	kmod_start_or_stop(kmod_t id, int start, kmod_args_t *data,
-			mach_msg_type_number_t *dataCount);
-kern_return_t	kmod_retain(kmod_t id);
-kern_return_t	kmod_release(kmod_t id);
-kern_return_t	kmod_queue_cmd(vm_address_t data, vm_size_t size);
-kern_return_t	kmod_get_info(host_t host, kmod_info_array_t *kmods,
-			mach_msg_type_number_t *kmodCount);
-extern void	kdb_printf(const char *fmt, ...);
-
-
+kern_return_t kmod_create_internal(kmod_info_t *info, kmod_t *id);
+kern_return_t kmod_destroy_internal(kmod_t id);
+kern_return_t kmod_start_or_stop(kmod_t id, int start, kmod_args_t *data,
+    mach_msg_type_number_t *dataCount);
+kern_return_t kmod_retain(kmod_t id);
+kern_return_t kmod_release(kmod_t id);
+kern_return_t kmod_queue_cmd(vm_address_t data, vm_size_t size);
+kern_return_t kmod_get_info(host_t host, kmod_info_array_t *kmods,
+    mach_msg_type_number_t *kmodCount);
+
+static kern_return_t kmod_get_symbol_data(kmod_args_t * data,
+    mach_msg_type_number_t * dataCount);
+static kern_return_t kmod_free_linkedit_data(void);
+static kern_return_t kmod_get_kext_uuid(
+    const char * kext_id,
+    kmod_args_t * data,
+    mach_msg_type_number_t * dataCount);
+
+extern int IODTGetLoaderInfo(const char * key, void ** infoAddr, vm_size_t * infoSize);
+extern void IODTFreeLoaderInfo(const char * key, void * infoAddr, vm_size_t infoSize);
+/* operates on 32 bit segments */
+extern void OSRuntimeUnloadCPPForSegment(struct segment_command * segment);
 
 #define WRITE_PROTECT_MODULE_TEXT   (0)
 
-kmod_info_t *kmod = 0;
+kmod_info_t *kmod;
 static int kmod_index = 1;
+static int kmod_load_disabled = 0;
 
-decl_simple_lock_data(,kmod_lock)
-decl_simple_lock_data(,kmod_queue_lock)
+mutex_t * kmod_lock = 0;
+static mutex_t * kmod_queue_lock = 0;
 
 typedef struct cmd_queue_entry {
     queue_chain_t    links;
@@ -79,15 +102,15 @@ queue_head_t kmod_cmd_queue;
 void
 kmod_init(void)
 {
-    simple_lock_init(&kmod_lock, 0);
-    simple_lock_init(&kmod_queue_lock, 0);
+    kmod_lock = mutex_alloc(0);
+    kmod_queue_lock = mutex_alloc(0);
     queue_init(&kmod_cmd_queue);
 }
 
 kmod_info_t *
 kmod_lookupbyid(kmod_t id)
 {
-    kmod_info_t *k = 0;
+    kmod_info_t *k = NULL;
 
     k = kmod;
     while (k) {
@@ -100,12 +123,55 @@ kmod_lookupbyid(kmod_t id)
 
 kmod_info_t *
 kmod_lookupbyname(const char * name)
+{
+    kmod_info_t *k = NULL;
+
+    k = kmod;
+    while (k) {
+        if (!strncmp(k->name, name, sizeof(k->name)))
+        break;
+        k = k->next;
+    }
+
+    return k;
+}
+
+// get the id of a kext in a given range, if the address is not in a kext
+// -1 is returned
+int kmod_lookupidbyaddress_locked(vm_address_t addr)
+{
+    kmod_info_t *k = 0;
+	
+    mutex_lock(kmod_queue_lock);
+    k = kmod;
+	if(NULL != k) {
+		while (k) {
+			if ((k->address <= addr) && ((k->address + k->size) > addr)) {
+				break;
+			}
+			k = k->next;
+		}
+		mutex_unlock(kmod_queue_lock);
+	} else {
+		mutex_unlock(kmod_queue_lock);
+		return -1;
+	}
+	
+	if(NULL == k) {
+		return -1;
+	} else {
+		return k->id;
+	}
+}
+
+kmod_info_t *
+kmod_lookupbyaddress(vm_address_t addr)
 {
     kmod_info_t *k = 0;
 
     k = kmod;
     while (k) {
-        if (!strcmp(k->name, name)) break;
+        if ((k->address <= addr) && ((k->address + k->size) > addr)) break;
         k = k->next;
     }
 
@@ -115,23 +181,23 @@ kmod_lookupbyname(const char * name)
 kmod_info_t *
 kmod_lookupbyid_locked(kmod_t id)
 {
-    kmod_info_t *k = 0;
-    kmod_info_t *kc = 0;
+    kmod_info_t *k = NULL;
+    kmod_info_t *kc = NULL;
 
     kc = (kmod_info_t *)kalloc(sizeof(kmod_info_t));
     if (!kc) return kc;
 
-    simple_lock(&kmod_queue_lock);
+    mutex_lock(kmod_lock);
     k = kmod_lookupbyid(id);
     if (k) {
         bcopy((char*)k, (char *)kc, sizeof(kmod_info_t));
     }
 
-    simple_unlock(&kmod_queue_lock);
+    mutex_unlock(kmod_lock);
 
     if (k == 0) {
         kfree(kc, sizeof(kmod_info_t));
-	kc = 0;
+    kc = NULL;
     }
     return kc;
 }
@@ -139,23 +205,23 @@ kmod_lookupbyid_locked(kmod_t id)
 kmod_info_t *
 kmod_lookupbyname_locked(const char * name)
 {
-    kmod_info_t *k = 0;
-    kmod_info_t *kc = 0;
+    kmod_info_t *k = NULL;
+    kmod_info_t *kc = NULL;
 
     kc = (kmod_info_t *)kalloc(sizeof(kmod_info_t));
     if (!kc) return kc;
 
-    simple_lock(&kmod_queue_lock);
+    mutex_lock(kmod_lock);
     k = kmod_lookupbyname(name);
     if (k) {
         bcopy((char *)k, (char *)kc, sizeof(kmod_info_t));
     }
 
-    simple_unlock(&kmod_queue_lock);
+    mutex_unlock(kmod_lock);
 
     if (k == 0) {
         kfree(kc, sizeof(kmod_info_t));
-	kc = 0;
+    kc = NULL;
     }
     return kc;
 }
@@ -177,9 +243,9 @@ kmod_queue_cmd(vm_address_t data, vm_size_t size)
     e->size = size;
     bcopy((void *)data, (void *)e->data, size);
 
-    simple_lock(&kmod_queue_lock);
+    mutex_lock(kmod_queue_lock);
     enqueue_tail(&kmod_cmd_queue, (queue_entry_t)e);
-    simple_unlock(&kmod_queue_lock);
+    mutex_unlock(kmod_queue_lock);
 
     thread_wakeup_one((event_t)&kmod_cmd_queue);
     
@@ -189,27 +255,31 @@ kmod_queue_cmd(vm_address_t data, vm_size_t size)
 kern_return_t
 kmod_load_extension(char *name)
 {
-    kmod_load_extension_cmd_t    *data;
-    vm_size_t            size;
+    kmod_load_extension_cmd_t data;
 
-    size = sizeof(kmod_load_extension_cmd_t);
-    data = (kmod_load_extension_cmd_t *)kalloc(size);
-    if (!data) return KERN_RESOURCE_SHORTAGE;
+    if (kmod_load_disabled) {
+        return KERN_NO_ACCESS;
+    }
 
-    data->type = KMOD_LOAD_EXTENSION_PACKET;
-    strncpy(data->name, name, KMOD_MAX_NAME);
+    data.type = KMOD_LOAD_EXTENSION_PACKET;
+    strncpy(data.name, name, sizeof(data.name));
 
-    return kmod_queue_cmd((vm_address_t)data, size);
+    return kmod_queue_cmd((vm_address_t)&data, sizeof(data));
 }
 
 kern_return_t
 kmod_load_extension_with_dependencies(char *name, char **dependencies)
 {
-    kmod_load_with_dependencies_cmd_t *data;
+    kern_return_t result;
+    kmod_load_with_dependencies_cmd_t * data;
     vm_size_t    size;
     char        **c;
     int         i, count = 0;
 
+    if (kmod_load_disabled) {
+        return KERN_NO_ACCESS;
+    }
+
     c = dependencies;
     if (c) {
         while (*c) {
@@ -230,24 +300,33 @@ kmod_load_extension_with_dependencies(char *name, char **dependencies)
     }
     data->dependencies[count][0] = 0;
 
-    return kmod_queue_cmd((vm_address_t)data, size);
+    result = kmod_queue_cmd((vm_address_t)data, size);
+    kfree(data, size);
+    return result;
 }
 kern_return_t
 kmod_send_generic(int type, void *generic_data, int size)
 {
-    kmod_generic_cmd_t    *data;
+    kern_return_t result;
+    kmod_generic_cmd_t * data;
+    vm_size_t cmd_size;
 
-    data = (kmod_generic_cmd_t *)kalloc(size + sizeof(int));
+    // add sizeof(int) for the type field
+    cmd_size = size + sizeof(int);
+    data = (kmod_generic_cmd_t *)kalloc(cmd_size);
     if (!data) return KERN_RESOURCE_SHORTAGE;
 
     data->type = type;
     bcopy(data->data, generic_data, size);
 
-    return kmod_queue_cmd((vm_address_t)data, size + sizeof(int));
+    result = kmod_queue_cmd((vm_address_t)data, cmd_size);
+    kfree(data, cmd_size);
+    return result;
 }
 
 extern vm_offset_t sectPRELINKB;
 extern int sectSizePRELINK;
+extern int kth_started;
 
 /*
  * Operates only on 32 bit mach keaders on behalf of kernel module loader
@@ -267,35 +346,36 @@ kmod_create_internal(kmod_info_t *info, kmod_t *id)
     }
 
     isPrelink = ((info->address >= sectPRELINKB) && (info->address < (sectPRELINKB + sectSizePRELINK)));
-    if (!isPrelink) {
-	rc = vm_map_wire(kernel_map, info->address + info->hdr_size, 
-		info->address + info->size, VM_PROT_DEFAULT, FALSE);
-	if (rc != KERN_SUCCESS) {
-	    return rc;
-	}
+    if (!isPrelink && kth_started) {
+        rc = vm_map_wire(kernel_map, info->address + info->hdr_size, 
+            info->address + info->size, VM_PROT_DEFAULT, FALSE);
+        if (rc != KERN_SUCCESS) {
+            return rc;
+        }
     }
 #if WRITE_PROTECT_MODULE_TEXT
-     {
-	struct section * sect = getsectbynamefromheader(
-	    (struct mach_header*) info->address, "__TEXT", "__text");
-	
-	if(sect) {
-	    (void) vm_map_protect(kernel_map, round_page(sect->addr), trunc_page(sect->addr + sect->size),
-				    VM_PROT_READ|VM_PROT_EXECUTE, TRUE);
-	}
+    {
+        struct section * sect = getsectbynamefromheader(
+            (struct mach_header*) info->address, "__TEXT", "__text");
+        
+        if(sect) {
+            (void) vm_map_protect(kernel_map, round_page(sect->addr),
+                trunc_page(sect->addr + sect->size),
+                VM_PROT_READ|VM_PROT_EXECUTE, TRUE);
+        }
     }
 #endif /* WRITE_PROTECT_MODULE_TEXT */
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
 
     // check to see if already loaded
     if (kmod_lookupbyname(info->name)) {
-        simple_unlock(&kmod_lock);
-	if (!isPrelink) {
-	    rc = vm_map_unwire(kernel_map, info->address + info->hdr_size, 
-		info->address + info->size, FALSE);
-	    assert(rc == KERN_SUCCESS);
-	}
+        mutex_unlock(kmod_lock);
+        if (!isPrelink) {
+            rc = vm_map_unwire(kernel_map, info->address + info->hdr_size, 
+            info->address + info->size, FALSE);
+            assert(rc == KERN_SUCCESS);
+        }
         return KERN_INVALID_ARGUMENT;
     }
 
@@ -307,7 +387,7 @@ kmod_create_internal(kmod_info_t *info, kmod_t *id)
 
     *id = info->id;
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
 #if DEBUG
     printf("kmod_create: %s (id %d), %d pages loaded at 0x%x, header size 0x%x\n", 
@@ -323,10 +403,20 @@ kmod_create(host_priv_t host_priv,
         vm_address_t addr,
         kmod_t *id)
 {
-    kmod_info_t *info = (kmod_info_t *)addr;
+#ifdef SECURE_KERNEL
+    return KERN_NOT_SUPPORTED;
+#else
+    kmod_info_t *info;
  
+    if (kmod_load_disabled) {
+        return KERN_NO_ACCESS;
+    }
+
+    info = (kmod_info_t *)addr;
+
     if (host_priv == HOST_PRIV_NULL) return KERN_INVALID_HOST;
     return kmod_create_internal(info, id);
+#endif
 }
 
 kern_return_t
@@ -353,17 +443,18 @@ kmod_create_fake_with_address(const char *name, const char *version,
     bcopy(name, info->name, 1 + strlen(name));
     bcopy(version, info->version, 1 + strlen(version));  //NIK fixed this part
     info->reference_count = 1;    // keep it from unloading, starting, stopping
-    info->reference_list = 0;
+    info->reference_list = NULL;
     info->address = address;
     info->size = size;
     info->hdr_size = 0;
-    info->start = info->stop = 0;
+    info->start = info->stop = NULL;
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
 
     // check to see if already "loaded"
     if (kmod_lookupbyname(info->name)) {
-        simple_unlock(&kmod_lock);
+        mutex_unlock(kmod_lock);
+        kfree(info, sizeof(kmod_info_t));
         return KERN_INVALID_ARGUMENT;
     }
 
@@ -374,7 +465,7 @@ kmod_create_fake_with_address(const char *name, const char *version,
     info->next = kmod;
     kmod = info;
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
     return KERN_SUCCESS;
 }
@@ -393,7 +484,7 @@ _kmod_destroy_internal(kmod_t id, boolean_t fake)
     kmod_info_t *k;
     kmod_info_t *p;
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
 
     k = p = kmod;
     while (k) {
@@ -401,7 +492,7 @@ _kmod_destroy_internal(kmod_t id, boolean_t fake)
             kmod_reference_t *r, *t;
 
             if (!fake && (k->reference_count != 0)) {
-                simple_unlock(&kmod_lock);
+                mutex_unlock(kmod_lock);
                 return KERN_INVALID_ARGUMENT;
             }
                 
@@ -410,7 +501,7 @@ _kmod_destroy_internal(kmod_t id, boolean_t fake)
             } else {
                 p->next = k->next;
             }
-            simple_unlock(&kmod_lock);
+            mutex_unlock(kmod_lock);
 
             r = k->reference_list;
             while (r) {
@@ -451,7 +542,7 @@ _kmod_destroy_internal(kmod_t id, boolean_t fake)
         k = k->next;
     }
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
     return KERN_INVALID_ARGUMENT;
 }
@@ -484,15 +575,19 @@ kmod_start_or_stop(
     mach_msg_type_number_t *dataCount)
 {
     kern_return_t rc = KERN_SUCCESS;
-    void * user_data = 0;
+    void * user_data = NULL;
     kern_return_t (*func)(kmod_info_t *, void *);
     kmod_info_t *k;
 
-    simple_lock(&kmod_lock);
+    if (start && kmod_load_disabled) {
+        return KERN_NO_ACCESS;
+    }
+
+    mutex_lock(kmod_lock);
 
     k = kmod_lookupbyid(id);
     if (!k || k->reference_count) {
-        simple_unlock(&kmod_lock);
+        mutex_unlock(kmod_lock);
         rc = KERN_INVALID_ARGUMENT;
         goto finish;
     }
@@ -503,15 +598,15 @@ kmod_start_or_stop(
         func = (void *)k->stop;
     }
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
     //
     // call kmod entry point
     //
     if (data && dataCount && *data && *dataCount) {
-    	vm_map_offset_t map_addr;
+        vm_map_offset_t map_addr;
         vm_map_copyout(kernel_map, &map_addr, (vm_map_copy_t)*data);
-	user_data = CAST_DOWN(void *, map_addr);
+    user_data = CAST_DOWN(void *, map_addr);
     }
 
     rc = (*func)(k, user_data);
@@ -521,7 +616,7 @@ finish:
     if (user_data) {
         (void) vm_deallocate(kernel_map, (vm_offset_t)user_data, *dataCount);
     }
-    if (data) *data = 0;
+    if (data) *data = NULL;
     if (dataCount) *dataCount = 0;
 
     return rc;
@@ -542,7 +637,7 @@ kmod_retain(kmod_t id)
 
     kmod_info_t *t;    // reference to
     kmod_info_t *f;    // reference from
-    kmod_reference_t *r = 0;
+    kmod_reference_t *r = NULL;
 
     r = (kmod_reference_t *)kalloc(sizeof(struct kmod_reference));
     if (!r) {
@@ -550,12 +645,12 @@ kmod_retain(kmod_t id)
         goto finish;
     }
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
 
     t = kmod_lookupbyid(KMOD_UNPACK_TO_ID(id));
     f = kmod_lookupbyid(KMOD_UNPACK_FROM_ID(id));
     if (!t || !f) {
-        simple_unlock(&kmod_lock);
+        mutex_unlock(kmod_lock);
         if (r) kfree(r, sizeof(struct kmod_reference));
         rc = KERN_INVALID_ARGUMENT;
         goto finish;
@@ -566,7 +661,7 @@ kmod_retain(kmod_t id)
     f->reference_list = r;
     t->reference_count++;
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
 finish:
 
@@ -581,10 +676,10 @@ kmod_release(kmod_t id)
 
     kmod_info_t *t;    // reference to
     kmod_info_t *f;    // reference from
-    kmod_reference_t *r = 0;
+    kmod_reference_t *r = NULL;
     kmod_reference_t * p;
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
 
     t = kmod_lookupbyid(KMOD_UNPACK_TO_ID(id));
     f = kmod_lookupbyid(KMOD_UNPACK_FROM_ID(id));
@@ -603,7 +698,7 @@ kmod_release(kmod_t id)
             }
             r->info->reference_count--;
 
-        simple_unlock(&kmod_lock);
+        mutex_unlock(kmod_lock);
             kfree(r, sizeof(struct kmod_reference));
         rc = KERN_SUCCESS;
             goto finish;
@@ -612,7 +707,7 @@ kmod_release(kmod_t id)
         r = r->next;
     }
 
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
 
 finish:
 
@@ -629,7 +724,13 @@ kmod_control(host_priv_t host_priv,
 {
     kern_return_t rc = KERN_SUCCESS;
 
-    if (host_priv == HOST_PRIV_NULL) return KERN_INVALID_HOST;
+   /* Only allow non-root access to retrieve kernel symbols or UUID.
+    */
+    if (flavor != KMOD_CNTL_GET_KERNEL_SYMBOLS &&
+        flavor != KMOD_CNTL_GET_UUID) {
+
+        if (host_priv == HOST_PRIV_NULL) return KERN_INVALID_HOST;
+    }
 
     switch (flavor) {
 
@@ -658,44 +759,43 @@ kmod_control(host_priv_t host_priv,
 
             cmd_queue_entry_t *e;
 
-            /*
-             * Throw away any data the user may have sent in error.
-             * We must do this, because we are likely to return to
-             * some data for these commands (thus causing a leak of
-             * whatever data the user sent us in error).
-             */
+           /* Throw away any data the user may have sent in error.
+            * We must do this, because we are likely to return to
+            * some data for these commands (thus causing a leak of
+            * whatever data the user sent us in error).
+            */
             if (*data && *dataCount) {
                 vm_map_copy_discard(*data);
-                *data = 0;
+                *data = NULL;
                 *dataCount = 0;
             }
             
-            simple_lock(&kmod_queue_lock);
+            mutex_lock(kmod_queue_lock);
 
             if (queue_empty(&kmod_cmd_queue)) {
                 wait_result_t res;
 
-                res = thread_sleep_simple_lock((event_t)&kmod_cmd_queue,
-                                   &kmod_queue_lock,
-                                   THREAD_ABORTSAFE);
+                res = thread_sleep_mutex((event_t)&kmod_cmd_queue,
+                    kmod_queue_lock,
+                    THREAD_ABORTSAFE);
                 if (queue_empty(&kmod_cmd_queue)) {
                     // we must have been interrupted!
-                    simple_unlock(&kmod_queue_lock);
+                    mutex_unlock(kmod_queue_lock);
                     assert(res == THREAD_INTERRUPTED);
                     return KERN_ABORTED;
                 }
             }
             e = (cmd_queue_entry_t *)dequeue_head(&kmod_cmd_queue);
 
-            simple_unlock(&kmod_queue_lock);
+            mutex_unlock(kmod_queue_lock);
 
             rc = vm_map_copyin(kernel_map, (vm_map_address_t)e->data,
-			       (vm_map_size_t)e->size, TRUE, (vm_map_copy_t *)data);
+                   (vm_map_size_t)e->size, TRUE, (vm_map_copy_t *)data);
             if (rc) {
-                simple_lock(&kmod_queue_lock);
+                mutex_lock(kmod_queue_lock);
                 enqueue_head(&kmod_cmd_queue, (queue_entry_t)e);
-                simple_unlock(&kmod_queue_lock);
-                *data = 0;
+                mutex_unlock(kmod_queue_lock);
+                *data = NULL;
                 *dataCount = 0;
                 return rc;
             }
@@ -706,6 +806,72 @@ kmod_control(host_priv_t host_priv,
             break;
         }
 
+      case KMOD_CNTL_GET_KERNEL_SYMBOLS:
+        {
+           /* Throw away any data the user may have sent in error.
+            * We must do this, because we are likely to return to
+            * some data for these commands (thus causing a leak of
+            * whatever data the user sent us in error).
+            */
+            if (*data && *dataCount) {
+                vm_map_copy_discard(*data);
+                *data = NULL;
+                *dataCount = 0;
+            }
+            
+            return kmod_get_symbol_data(data, dataCount);
+            break;
+        }
+
+      case KMOD_CNTL_FREE_LINKEDIT_DATA:
+        {
+            return kmod_free_linkedit_data();
+            break;
+        }
+
+      case KMOD_CNTL_GET_UUID:
+        {
+            uint32_t id_length = *dataCount;
+            char * kext_id = NULL;
+            vm_map_offset_t map_addr;
+            void * user_data;
+            kern_return_t result;
+
+           /* Get the bundle id, if provided, and discard the buffer sent down.
+            */
+            if (*data && *dataCount) {
+                (char *)(kmem_alloc(kernel_map, (vm_offset_t *)&kext_id, id_length));
+                if (!kext_id) {
+                    return KERN_FAILURE;
+                }
+                
+                vm_map_copyout(kernel_map, &map_addr, (vm_map_copy_t)*data);
+                user_data = CAST_DOWN(void *, map_addr);
+
+                memcpy(kext_id, user_data, id_length);
+                kext_id[id_length-1] = '\0';
+                if (user_data) {
+                    (void)vm_deallocate(kernel_map, (vm_offset_t)user_data, *dataCount);
+                }
+                *data = NULL;
+                *dataCount = 0;
+            }
+            
+            result = kmod_get_kext_uuid(kext_id, data, dataCount);
+            if (kext_id) {
+                kmem_free(kernel_map, (vm_offset_t)kext_id, id_length);
+            }
+            return result;
+            break;
+        }
+
+      case KMOD_CNTL_DISABLE_LOAD:
+        {
+            kmod_load_disabled = 1;
+            rc = KERN_SUCCESS;
+            break;
+        }
+
       default:
         rc = KERN_INVALID_ARGUMENT;
     }
@@ -713,6 +879,443 @@ kmod_control(host_priv_t host_priv,
     return rc;
 };
 
+/*******************************************************************************
+* This function creates a dummy symbol file for the running kernel based on data
+* in the run-time image. This allows us to correctly link other executables
+* (drivers, etc) against the kernel when the kernel image on the root filesystem
+* does not match the live kernel, as c can occur during net-booting where the
+* actual kernel image is obtained from the network via tftp rather than the root
+* device.
+*
+* If a symbol table is available, then a link-suitable Mach-O file image is
+* created containing a Mach Header and an LC_SYMTAB load command followed by the
+* the symbol table data for mach_kernel. A UUID load command is also present for
+* identification, so we don't link against the wrong kernel.
+*
+* NOTE: This file supports only 32 bit kernels; adding support for 64 bit
+* kernels is possible, but is not necessary yet.
+*******************************************************************************/
+extern struct mach_header _mh_execute_header;
+static int                _linkedit_segment_freed = 0;
+
+static kern_return_t
+kmod_get_symbol_data(
+    kmod_args_t * symbol_data,
+    mach_msg_type_number_t * data_size)
+{
+    kern_return_t            result = KERN_FAILURE;
+
+    struct load_command    * load_cmd;
+    struct mach_header     * orig_header = &_mh_execute_header;
+    struct segment_command * orig_text = NULL;
+    struct segment_command * orig_data = NULL;
+    struct segment_command * orig_linkedit = NULL;
+    struct uuid_command    * orig_uuid = NULL;
+    struct symtab_command  * orig_symtab = NULL;
+    struct section         * sect;
+    struct section         * const_text = NULL;
+
+    vm_size_t                header_size = 0;
+    vm_offset_t              symtab_size;
+    vm_offset_t              total_size;  // copied out to 'data_size'
+    char                   * buffer = 0;  // copied out to 'symbol_data'
+
+    struct mach_header     * header;
+    struct segment_command * seg_cmd = NULL;
+    struct symtab_command  * symtab;
+
+    unsigned int             i;
+    caddr_t                  addr;
+    vm_offset_t              offset;
+
+    // only want to do these 1st call
+    static int               syms_marked = 0;
+
+    mutex_lock(kmod_lock);
+
+   /*****
+    * Check for empty out parameter pointers, and zero them if ok.
+    */
+    if (!symbol_data || !data_size) {
+        result = KERN_INVALID_ARGUMENT;
+        goto finish;
+    }
+
+    *symbol_data = NULL;
+    *data_size = 0;
+
+    if (_linkedit_segment_freed) {
+        result = KERN_MEMORY_FAILURE;
+        goto finish;
+    }
+    
+   /*****
+    * Scan the in-memory kernel's mach header for the parts we need to copy:
+    * TEXT (for basic file info + const section), DATA (for basic file info),
+    * LINKEDIT (for the symbol table entries), SYMTAB (for the symbol table
+    * overall).
+    */
+    load_cmd = (struct load_command *)&orig_header[1];
+    for (i = 0; i < orig_header->ncmds; i++) {
+        if (load_cmd->cmd == LC_SEGMENT) {
+            struct segment_command * orig_seg_cmd =
+                (struct segment_command *)load_cmd;
+
+            if (!strncmp(SEG_TEXT, orig_seg_cmd->segname, strlen(SEG_TEXT))) {
+                orig_text = orig_seg_cmd;
+            } else if (!strncmp(SEG_DATA, orig_seg_cmd->segname,
+                strlen(SEG_DATA))) {
+
+                orig_data = orig_seg_cmd;
+            } else if (!strncmp(SEG_LINKEDIT, orig_seg_cmd->segname,
+                strlen(SEG_LINKEDIT))) {
+
+                orig_linkedit = orig_seg_cmd;
+            }
+        } else if (load_cmd->cmd == LC_UUID) {
+            orig_uuid = (struct uuid_command *)load_cmd;
+        } else if (load_cmd->cmd == LC_SYMTAB) {
+            orig_symtab = (struct symtab_command *)load_cmd;
+        }
+
+        load_cmd = (struct load_command *)((caddr_t)load_cmd + load_cmd->cmdsize);
+    }
+
+   /* Bail if any wasn't found.
+    */
+    if (!orig_text || !orig_data || !orig_linkedit || !orig_uuid || !orig_symtab) {
+        goto finish;
+    }
+
+   /* Now seek out the const section of the TEXT segment, bailing if not found.
+    */
+    sect = (struct section *)&orig_text[1];
+    for (i = 0; i < orig_text->nsects; i++, sect++) {
+        if (!strncmp("__const", sect->sectname, sizeof("__const"))) {
+            const_text = sect;
+            break;
+        }
+    }
+    if (!const_text) {
+        goto finish;
+    }
+
+   /*****
+    * Calculate the total size needed and allocate the buffer. In summing the
+    * total size, every size before the last must be rounded to a
+    * page-size increment.
+    */
+    header_size = sizeof(struct mach_header) +
+        orig_text->cmdsize + orig_data->cmdsize +
+        orig_uuid->cmdsize + orig_symtab->cmdsize;
+    symtab_size = (orig_symtab->nsyms * sizeof(struct nlist)) +
+        orig_symtab->strsize;
+    total_size = round_page(header_size) + round_page(const_text->size) +
+        symtab_size;
+
+    (void)kmem_alloc(kernel_map, (vm_offset_t *)&buffer, total_size);
+    if (!buffer) {
+        goto finish;
+    }
+    bzero((void *)buffer, total_size);
+
+   /*****
+    * Set up the Mach-O header in the buffer.
+    */
+    header = (struct mach_header *)buffer;
+    header->magic      = orig_header->magic;
+    header->cputype    = orig_header->cputype;
+    header->cpusubtype = orig_header->cpusubtype;
+    header->filetype   = orig_header->filetype;
+    header->ncmds      = 4;  // TEXT, DATA, UUID, SYMTAB
+    header->sizeofcmds = header_size - sizeof(struct mach_header);
+    header->flags      = orig_header->flags;
+
+   /*****
+    * Initialize the current file offset and addr; updated as we go through,
+    * but only for fields that need proper info.
+    */
+    offset = round_page(header_size);
+    addr   = (caddr_t)const_text->addr;
+
+   /*****
+    * Construct a TEXT segment load command. The only content of the TEXT
+    * segment that we actually copy is the __TEXT,__const, which contains the
+    * kernel vtables.  The other sections are just filled with unincremented
+    * addr/offset and zero size and number fields.
+    */
+    seg_cmd = (struct segment_command *)&header[1]; // just past mach header
+    memcpy(seg_cmd, orig_text, orig_text->cmdsize);
+    seg_cmd->vmaddr   = (unsigned long)addr;
+    seg_cmd->vmsize   = const_text->size;
+    seg_cmd->fileoff  = 0;
+    seg_cmd->filesize = const_text->size + round_page(header_size);
+    seg_cmd->maxprot  = 0;
+    seg_cmd->initprot = 0;
+    seg_cmd->flags    = 0;
+    sect = (struct section *)(seg_cmd + 1);
+    for (i = 0; i < seg_cmd->nsects; i++, sect++) {
+        sect->addr  = (unsigned long)addr; // only valid for __TEXT,__const
+        sect->size  = 0;
+        sect->offset = offset;
+        sect->nreloc = 0;
+        if (0 == strncmp("__const", sect->sectname, sizeof("__const"))) {
+            sect->size = const_text->size;
+            addr      += const_text->size;
+            offset    += const_text->size;
+            const_text = sect;  // retarget to constructed section
+        }
+    }
+    offset = round_page(offset);
+
+   /*****
+    * Now copy the __DATA segment load command, but none of its content.
+    */
+    seg_cmd = (struct segment_command *)((int)seg_cmd + seg_cmd->cmdsize);
+    memcpy(seg_cmd, orig_data, orig_data->cmdsize);
+
+    seg_cmd->vmaddr   = (unsigned long)addr;
+    seg_cmd->vmsize   = 0x1000;    // Why not just zero? DATA seg is empty.
+    seg_cmd->fileoff  = offset;
+    seg_cmd->filesize = 0;
+    seg_cmd->maxprot  = 0;
+    seg_cmd->initprot = 0;
+    seg_cmd->flags    = 0;
+    sect = (struct section *)(seg_cmd+1);
+    for (i = 0; i < seg_cmd->nsects; i++, sect++) {
+        sect->addr   = (unsigned long)addr;
+        sect->size   = 0;
+        sect->offset = offset;
+        sect->nreloc = 0;
+    }
+    offset = round_page(offset);
+
+   /* Set up LC_UUID command
+    */
+    seg_cmd = (struct segment_command *)((int)seg_cmd + seg_cmd->cmdsize);
+    memcpy(seg_cmd, orig_uuid, orig_uuid->cmdsize);
+
+   /* Set up LC_SYMTAB command
+    */
+    symtab          = (struct symtab_command *)((int)seg_cmd + seg_cmd->cmdsize);
+    symtab->cmd     = LC_SYMTAB;
+    symtab->cmdsize = sizeof(struct symtab_command);
+    symtab->symoff  = offset;
+    symtab->nsyms   = orig_symtab->nsyms;
+    symtab->strsize = orig_symtab->strsize;
+    symtab->stroff  = offset + symtab->nsyms * sizeof(struct nlist);    
+
+   /* Convert the symbol table in place (yes, in the running kernel)
+    * from section references to absolute references.
+    */
+    if (!syms_marked) {
+        struct nlist * sym = (struct nlist *) orig_linkedit->vmaddr;
+        for (i = 0; i < orig_symtab->nsyms; i++, sym++) {
+            if ((sym->n_type & N_TYPE) == N_SECT) {
+                sym->n_sect = NO_SECT;
+                sym->n_type = (sym->n_type & ~N_TYPE) | N_ABS;
+            }
+        }
+        syms_marked = 1;
+    }
+    
+   /*****
+    * Copy the contents of the __TEXT,__const section and the linkedit symbol
+    * data into the constructed object file buffer. The header has already been
+    * filled in.
+    */
+    memcpy(buffer + const_text->offset, (void *)const_text->addr, const_text->size);
+    memcpy(buffer + symtab->symoff, (void *)orig_linkedit->vmaddr, symtab_size);
+
+    result = vm_map_copyin(kernel_map,
+        (vm_offset_t)buffer,
+        (vm_map_size_t)total_size,
+        /* src_destroy */ TRUE,
+        (vm_map_copy_t *)symbol_data);
+    if (result != KERN_SUCCESS) {
+        kmem_free(kernel_map, (vm_offset_t)buffer, total_size);
+        *symbol_data = NULL;
+        *data_size   = 0;
+        goto finish;
+    } else {
+        *data_size = total_size;
+    }
+
+finish:
+    mutex_unlock(kmod_lock);
+    return result;
+}
+
+/*******************************************************************************
+* Drop the LINKEDIT segment from the running kernel to recover wired memory.
+* This is invoked by kextd after it has successfully determined a file is
+* available in the root filesystem to link against (either a symbol file it
+* wrote, or /mach_kernel).
+*******************************************************************************/
+// in IOCatalogue.cpp
+extern int kernelLinkerPresent;
+
+static kern_return_t
+kmod_free_linkedit_data(void)
+{
+    kern_return_t result = KERN_FAILURE;
+
+    const char * dt_kernel_header_name = "Kernel-__HEADER";
+    const char * dt_kernel_symtab_name = "Kernel-__SYMTAB";
+    struct mach_header_t * dt_mach_header = NULL;
+    vm_size_t dt_mach_header_size = 0;
+    struct symtab_command *dt_symtab = NULL;
+    vm_size_t dt_symtab_size = 0;
+    int dt_result;
+
+    struct segment_command * segmentLE;
+    boolean_t    keepsyms = FALSE;
+    const char * segment_name = "__LINKEDIT";
+#if __ppc__ || __arm__
+    const char * devtree_segment_name = "Kernel-__LINKEDIT";
+    void       * segment_paddress;
+    vm_size_t    segment_size;
+#endif
+
+    mutex_lock(kmod_lock);
+
+   /* The semantic is "make sure the linkedit segment is freed", so if we
+    * previously did it, it's a success.
+    */
+    if (_linkedit_segment_freed) {
+        result = KERN_SUCCESS;
+        goto finish;
+    } else if (kernelLinkerPresent) {
+        // The in-kernel linker requires the linkedit segment to function.
+        // Refuse to dump if it's still around.
+        // XXX: We need a dedicated error return code for this.
+        printf("can't remove kernel __LINKEDIT segment - in-kernel linker needs it\n");
+        result = KERN_MEMORY_FAILURE;
+        goto finish;
+    }
+
+   /* Dispose of unnecessary stuff that the booter didn't need to load.
+    */
+    dt_result = IODTGetLoaderInfo(dt_kernel_header_name,
+        (void **)&dt_mach_header, &dt_mach_header_size);
+    if (dt_result == 0 && dt_mach_header) {
+        IODTFreeLoaderInfo(dt_kernel_header_name, (void *)dt_mach_header,
+            round_page_32(dt_mach_header_size));
+    }
+    dt_result = IODTGetLoaderInfo(dt_kernel_symtab_name,
+        (void **)&dt_symtab, &dt_symtab_size);
+    if (dt_result == 0 && dt_symtab) {
+        IODTFreeLoaderInfo(dt_kernel_symtab_name, (void *)dt_symtab,
+            round_page_32(dt_symtab_size));
+    }
+
+    PE_parse_boot_arg("keepsyms", &keepsyms);
+
+    segmentLE = getsegbyname(segment_name);
+    if (!segmentLE) {
+        printf("error removing kernel __LINKEDIT segment\n");
+        goto finish;
+    }
+    OSRuntimeUnloadCPPForSegment(segmentLE);
+#if __ppc__ || __arm__
+    if (!keepsyms && 0 == IODTGetLoaderInfo(devtree_segment_name,
+        &segment_paddress, &segment_size)) {
+
+        IODTFreeLoaderInfo(devtree_segment_name, (void *)segment_paddress,
+            (int)segment_size);
+    }
+#elif __i386__
+    if (!keepsyms && segmentLE->vmaddr && segmentLE->vmsize) {
+        ml_static_mfree(segmentLE->vmaddr, segmentLE->vmsize);
+    }
+#else
+#error arch
+#endif
+    result = KERN_SUCCESS;
+
+finish:
+    if (!keepsyms && result == KERN_SUCCESS) {
+        _linkedit_segment_freed = 1;
+    }
+    mutex_unlock(kmod_lock);
+    return result;
+}
+
+/*******************************************************************************
+* Retrieve the UUID load command payload from the running kernel.
+*******************************************************************************/
+static kern_return_t
+kmod_get_kext_uuid(
+    const char * kext_id,
+    kmod_args_t * data,
+    mach_msg_type_number_t * dataCount)
+{
+    kern_return_t result = KERN_FAILURE;
+    kmod_info_t * kmod_info = NULL;
+    unsigned int i;
+    char * uuid_data = 0;
+    struct mach_header  * header = &_mh_execute_header;
+    struct load_command * load_cmd = (struct load_command *)&header[1];
+    struct uuid_command * uuid_cmd;
+
+   /* If given no kext ID, retrieve the kernel UUID.
+    */
+    if (!kext_id) {
+        header = &_mh_execute_header;
+    } else {
+        kmod_info = kmod_lookupbyname_locked(kext_id);
+        if (!kmod_info) {
+            result = KERN_INVALID_ARGUMENT;
+            goto finish;
+        }
+        
+       /* If the kmod is build-in, it's part of the kernel, so retrieve the
+        * kernel UUID.
+        */
+        if (!kmod_info->address) {
+            header = &_mh_execute_header;
+        } else {
+            header = (struct mach_header *)kmod_info->address;
+        }
+    }
+
+    load_cmd = (struct load_command *)&header[1];
+
+    for (i = 0; i < header->ncmds; i++) {
+        if (load_cmd->cmd == LC_UUID) {
+            uuid_cmd = (struct uuid_command *)load_cmd;
+
+           /* kmem_alloc() a local buffer that's on a boundary known to work
+            * with vm_map_copyin().
+            */
+            result = kmem_alloc(kernel_map, (vm_offset_t *)&uuid_data,
+                sizeof(uuid_cmd->uuid));
+            if (result != KERN_SUCCESS) {
+                result = KERN_RESOURCE_SHORTAGE;
+                goto finish;
+            }
+            
+            memcpy(uuid_data, uuid_cmd->uuid, sizeof(uuid_cmd->uuid));
+            
+            result = vm_map_copyin(kernel_map, (vm_offset_t)uuid_data,
+                sizeof(uuid_cmd->uuid), /* src_destroy */ TRUE,
+                (vm_map_copy_t *)data);
+            if (result == KERN_SUCCESS) {
+                *dataCount = sizeof(uuid_cmd->uuid);
+            } else {
+                result = KERN_RESOURCE_SHORTAGE;
+                kmem_free(kernel_map, (vm_offset_t)uuid_data,
+                    sizeof(uuid_cmd->uuid));
+            }
+            goto finish;
+        }
+        
+        load_cmd = (struct load_command *)((caddr_t)load_cmd + load_cmd->cmdsize);
+    }
+
+finish:
+    return result;
+}
 
 kern_return_t
 kmod_get_info(__unused host_t host,
@@ -730,7 +1333,7 @@ kmod_get_info(__unused host_t host,
     *kmodCount = 0;
 
 retry:
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
     size = 0;
     k = kmod;
     while (k) {
@@ -742,7 +1345,7 @@ retry:
         }
         k = k->next;
     }
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
     if (!size) return KERN_SUCCESS;
 
     rc = kmem_alloc(kernel_map, &data, size);
@@ -754,11 +1357,11 @@ retry:
     // change the k->reference into a count, tack the references on
     // the end of the data packet in the order they are found
 
-    simple_lock(&kmod_lock);
+    mutex_lock(kmod_lock);
     k = kmod; p1 = (kmod_info_t *)data;
     while (k) {
         if ((p1 + 1) > (kmod_info_t *)(data + size)) {
-            simple_unlock(&kmod_lock);
+            mutex_unlock(kmod_lock);
             kmem_free(kernel_map, data, size);
             goto retry;
         }
@@ -774,7 +1377,7 @@ retry:
         r = k->reference_list; ref_count = 0;
         while (r) {
             if ((p2 + 1) > (kmod_reference_t *)(data + size)) {
-                simple_unlock(&kmod_lock);
+                mutex_unlock(kmod_lock);
                 kmem_free(kernel_map, data, size);
                 goto retry;
             }
@@ -787,12 +1390,12 @@ retry:
         p1->reference_list = (kmod_reference_t *)ref_count;
         p1++; k = k->next;
     }
-    simple_unlock(&kmod_lock);
+    mutex_unlock(kmod_lock);
     
     rc = vm_map_copyin(kernel_map, data, size, TRUE, (vm_map_copy_t *)kmods);
     if (rc) {
         kmem_free(kernel_map, data, size);
-        *kmods = 0;
+        *kmods = NULL;
         *kmodCount = 0;
         return rc;
     }
@@ -858,14 +1461,14 @@ kmod_default_stop(__unused struct kmod_info *ki, __unused void *data)
 
 static void
 kmod_dump_to(vm_offset_t *addr, unsigned int cnt,
-	void (*printf_func)(const char *fmt, ...))
+    int (*printf_func)(const char *fmt, ...))
 {
-    vm_offset_t * kscan_addr = 0;
+    vm_offset_t * kscan_addr = NULL;
     kmod_info_t * k;
     kmod_reference_t * r;
     unsigned int i;
     int found_kmod = 0;
-    kmod_info_t * stop_kmod = 0;
+    kmod_info_t * stop_kmod = NULL;
 
     for (k = kmod; k; k = k->next) {
         if (pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)k)) == 0) {
@@ -885,8 +1488,8 @@ kmod_dump_to(vm_offset_t *addr, unsigned int cnt,
                         "(with dependencies):\n");
                 }
                 found_kmod = 1;
-                (*printf_func)("         %s(%s)@0x%x\n",
-                    k->name, k->version, k->address);
+                (*printf_func)("         %s(%s)@0x%x->0x%x\n",
+                    k->name, k->version, k->address, k->address + k->size - 1);
 
                 for (r = k->reference_list; r; r = r->next) {
                     kmod_info_t * rinfo;
@@ -927,6 +1530,8 @@ kmod_dump(vm_offset_t *addr, unsigned int cnt)
     kmod_dump_to(addr, cnt, &kdb_printf);
 }
 
+void kmod_dump_log(vm_offset_t *, unsigned); /* gcc 4 warn fix */
+
 void
 kmod_dump_log(vm_offset_t *addr, unsigned int cnt)
 {
diff --git a/osfmk/kern/ledger.c b/osfmk/kern/ledger.c
index dbca9b48f..30e11cabd 100644
--- a/osfmk/kern/ledger.c
+++ b/osfmk/kern/ledger.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -66,7 +72,7 @@ ledger_enter(
 		if (ledger->ledger_limit != LEDGER_ITEM_INFINITY &&
 		    ledger->ledger_balance + amount > ledger->ledger_limit) {
 			/* XXX this is where you do BAD things */
-			printf("Ledger limit exceeded ! ledger=%x lim=%d balance=%d\n",
+			printf("Ledger limit exceeded ! ledger=%p lim=%d balance=%d\n",
 			       ledger, ledger->ledger_limit,
 			       ledger->ledger_balance);
 			ledger_unlock(ledger);
@@ -102,8 +108,10 @@ ledger_allocate(
 		return(LEDGER_NULL);
 
 	ledger->ledger_self = ipc_port_alloc_kernel();
-	if (ledger->ledger_self == IP_NULL)
+	if (ledger->ledger_self == IP_NULL) {
+		kfree(ledger, sizeof(ledger_data_t));
 		return(LEDGER_NULL);
+	}
 
 	ledger_lock_init(ledger);
 	ledger->ledger_limit = limit;
diff --git a/osfmk/kern/ledger.h b/osfmk/kern/ledger.h
index 0d6cbf87e..fe0fa2c04 100644
--- a/osfmk/kern/ledger.h
+++ b/osfmk/kern/ledger.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,7 +63,7 @@ extern ledger_t	root_paged_ledger;
 #define root_wired_ledger_port root_wired_ledger->ledger_self
 #define root_paged_ledger_port root_paged_ledger->ledger_self
 
-extern void ledger_init(void);
+extern void ledger_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern ipc_port_t ledger_copy(ledger_t);
 
diff --git a/osfmk/kern/lock.h b/osfmk/kern/lock.h
index 4cb10d325..eb5ed024e 100644
--- a/osfmk/kern/lock.h
+++ b/osfmk/kern/lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -79,6 +85,21 @@ extern void			mutex_init(
 						mutex_t		*mutex,
 						unsigned short	tag);
 
+#ifdef i386
+extern void			mutex_try_spin(
+						mutex_t		*mutex);
+
+extern void			mutex_lock_spin(
+						mutex_t		*mutex);
+
+extern void			mutex_convert_spin(
+						mutex_t		*mutex);
+#else
+#define	mutex_try_spin(l)	mutex_try(l)
+#define	mutex_lock_spin(l)	mutex_lock(l)
+#define	mutex_convert_spin(l)	do {} while (0)
+#endif
+
 #endif	/* MACH_KERNEL_PRIVATE */
 
 extern mutex_t		*mutex_alloc(
@@ -96,7 +117,8 @@ extern void			mutex_unlock(
 extern boolean_t	mutex_try(
 						mutex_t		*mutex);
 
-extern void			mutex_pause(void);
+extern void			mutex_pause(uint32_t);
+extern void			mutex_yield(mutex_t *);
 
 #define MA_OWNED        0x01
 #define MA_NOTOWNED     0x02
diff --git a/osfmk/kern/locks.c b/osfmk/kern/locks.c
index db2db8142..064a58252 100644
--- a/osfmk/kern/locks.c
+++ b/osfmk/kern/locks.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -67,6 +73,16 @@
 
 #include <sys/kdebug.h>
 
+#if	CONFIG_DTRACE
+/*
+ * We need only enough declarations from the BSD-side to be able to
+ * test if our probe is active, and to call __dtrace_probe().  Setting
+ * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
+ */
+#define NEED_DTRACE_DEFS
+#include <../bsd/sys/lockstat.h>
+#endif
+
 #define	LCK_MTX_SLEEP_CODE		0
 #define	LCK_MTX_SLEEP_DEADLINE_CODE	1
 #define	LCK_MTX_LCK_WAIT_CODE		2
@@ -138,7 +154,7 @@ void
 lck_grp_attr_setstat(
 	lck_grp_attr_t	*attr)
 {
-	(void)hw_atomic_or((uint32_t *)&attr->grp_attr_val, LCK_GRP_ATTR_STAT);
+	(void)hw_atomic_or(&attr->grp_attr_val, LCK_GRP_ATTR_STAT);
 }
 
 
@@ -227,7 +243,7 @@ void
 lck_grp_reference(
 	lck_grp_t	*grp)
 {
-	(void)hw_atomic_add((uint32_t *)(&grp->lck_grp_refcnt), 1);
+	(void)hw_atomic_add(&grp->lck_grp_refcnt, 1);
 }
 
 
@@ -239,7 +255,7 @@ void
 lck_grp_deallocate(
 	lck_grp_t	*grp)
 {
-	if (hw_atomic_sub((uint32_t *)(&grp->lck_grp_refcnt), 1) == 0)
+	if (hw_atomic_sub(&grp->lck_grp_refcnt, 1) == 0)
 	 	kfree(grp, sizeof(lck_grp_t));
 }
 
@@ -268,7 +284,7 @@ lck_grp_lckcnt_incr(
 		return panic("lck_grp_lckcnt_incr(): invalid lock type: %d\n", lck_type);
 	}
 
-	(void)hw_atomic_add((uint32_t *)lckcnt, 1);
+	(void)hw_atomic_add(lckcnt, 1);
 }
 
 /*
@@ -296,7 +312,7 @@ lck_grp_lckcnt_decr(
 		return panic("lck_grp_lckcnt_decr(): invalid lock type: %d\n", lck_type);
 	}
 
-	(void)hw_atomic_sub((uint32_t *)lckcnt, 1);
+	(void)hw_atomic_sub(lckcnt, 1);
 }
 
 /*
@@ -343,7 +359,17 @@ void
 lck_attr_setdebug(
 	lck_attr_t	*attr)
 {
-	(void)hw_atomic_or((uint32_t *)&attr->lck_attr_val, LCK_ATTR_DEBUG);
+	(void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_DEBUG);
+}
+
+/*
+ * Routine:	lck_attr_setdebug
+ */
+void
+lck_attr_cleardebug(
+	lck_attr_t	*attr)
+{
+	(void)hw_atomic_and(&attr->lck_attr_val, ~LCK_ATTR_DEBUG);
 }
 
 
@@ -354,7 +380,7 @@ void
 lck_attr_rw_shared_priority(
 	lck_attr_t	*attr)
 {
-	(void)hw_atomic_or((uint32_t *)&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
+	(void)hw_atomic_or(&attr->lck_attr_val, LCK_ATTR_RW_SHARED_PRIORITY);
 }
 
 
@@ -517,6 +543,13 @@ lck_mtx_lock_wait (
 	lck_mtx_t		*mutex;
 	integer_t		priority;
 	spl_t			s = splsched();
+#if	CONFIG_DTRACE
+	uint64_t		sleep_start = 0;
+
+	if (lockstat_probemap[LS_LCK_MTX_LOCK_BLOCK] || lockstat_probemap[LS_LCK_MTX_EXT_LOCK_BLOCK]) {
+		sleep_start = mach_absolute_time();
+	}
+#endif
 
 	if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT)
 		mutex = lck;
@@ -569,6 +602,21 @@ lck_mtx_lock_wait (
 	thread_block(THREAD_CONTINUE_NULL);
 
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_LCK_WAIT_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
+#if	CONFIG_DTRACE
+	/*
+	 * Record the Dtrace lockstat probe for blocking, block time
+	 * measured from when we were entered.
+	 */
+	if (sleep_start) {
+		if (lck->lck_mtx_tag != LCK_MTX_TAG_INDIRECT) {
+			LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_BLOCK, lck,
+			    mach_absolute_time() - sleep_start);
+		} else {
+			LOCKSTAT_RECORD(LS_LCK_MTX_EXT_LOCK_BLOCK, lck,
+			    mach_absolute_time() - sleep_start);
+		}
+	}
+#endif
 }
 
 /*
@@ -649,7 +697,7 @@ lck_mtx_unlock_wakeup (
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_START, (int)lck, (int)holder, 0, 0, 0);
 
 	if (thread != holder)
-		panic("lck_mtx_unlock_wakeup: mutex %x holder %x\n", mutex, holder);
+		panic("lck_mtx_unlock_wakeup: mutex %p holder %p\n", mutex, holder);
 
 	if (thread->promotions > 0) {
 		spl_t		s = splsched();
@@ -686,24 +734,89 @@ lck_mtx_unlock_wakeup (
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_END, 0, 0, 0, 0, 0);
 }
 
+void
+lck_mtx_unlockspin_wakeup (
+	lck_mtx_t			*lck)
+{
+	assert(lck->lck_mtx_waiters > 0);
+	thread_wakeup_one((event_t)(((unsigned int*)lck)+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)));
+
+	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_UNLCK_WAKEUP_CODE) | DBG_FUNC_NONE, (int)lck, 0, 0, 1, 0);
+#if CONFIG_DTRACE
+	/*
+	 * When there are waiters, we skip the hot-patch spot in the
+	 * fastpath, so we record it here.
+	 */
+	LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE, lck, 0);
+#endif
+}
+
+
 /*
  * Routine: 	mutex_pause
  *
  * Called by former callers of simple_lock_pause().
  */
+#define MAX_COLLISION_COUNTS	32
+#define MAX_COLLISION 	8
+
+unsigned int max_collision_count[MAX_COLLISION_COUNTS];
+
+uint32_t collision_backoffs[MAX_COLLISION] = {
+        10, 50, 100, 200, 400, 600, 800, 1000
+};
+
 
 void
-mutex_pause(void)
+mutex_pause(uint32_t collisions)
 {
 	wait_result_t wait_result;
+	uint32_t	back_off;
 
-	wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, 1, 1000*NSEC_PER_USEC);
+	if (collisions >= MAX_COLLISION_COUNTS)
+	        collisions = MAX_COLLISION_COUNTS - 1;
+	max_collision_count[collisions]++;
+
+	if (collisions >= MAX_COLLISION)
+	        collisions = MAX_COLLISION - 1;
+	back_off = collision_backoffs[collisions];
+
+	wait_result = assert_wait_timeout((event_t)mutex_pause, THREAD_UNINT, back_off, NSEC_PER_USEC);
 	assert(wait_result == THREAD_WAITING);
 
 	wait_result = thread_block(THREAD_CONTINUE_NULL);
 	assert(wait_result == THREAD_TIMED_OUT);
 }
 
+
+unsigned int mutex_yield_wait = 0;
+unsigned int mutex_yield_no_wait = 0;
+
+void
+mutex_yield(
+	    mutex_t	*mutex)
+{
+        lck_mtx_t	*lck;
+
+#if DEBUG
+	_mutex_assert(mutex, MA_OWNED);
+#endif /* DEBUG */
+
+	lck = (lck_mtx_t *) mutex;
+	if (lck->lck_mtx_tag == LCK_MTX_TAG_INDIRECT)
+	        lck = &lck->lck_mtx_ptr->lck_mtx;
+
+	if (! lck->lck_mtx_waiters) {
+	        mutex_yield_no_wait++;
+	} else {
+	        mutex_yield_wait++;
+		mutex_unlock(mutex);
+		mutex_pause(0);
+		mutex_lock(mutex);
+	}
+}
+
+
 /*
  * Routine:	lck_rw_sleep
  */
diff --git a/osfmk/kern/locks.h b/osfmk/kern/locks.h
index feca9487d..b936f226a 100644
--- a/osfmk/kern/locks.h
+++ b/osfmk/kern/locks.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _KERN_LOCKS_H_
@@ -34,13 +40,13 @@
 #include	<kern/queue.h>
 
 extern void				lck_mod_init(
-								void);
+								void) __attribute__((section("__TEXT, initcode")));
 
 typedef	unsigned int	lck_type_t;
 
 #define	LCK_TYPE_SPIN	1
 #define	LCK_TYPE_MTX	2
-#define	LCK_TYPE_RW		3
+#define	LCK_TYPE_RW	3
 
 #endif
 
@@ -66,9 +72,11 @@ typedef struct {
 
 typedef struct {
 	uint64_t			lck_grp_mtx_util_cnt;
+	/* On x86, this is used as the "direct wait" count */
 	uint64_t			lck_grp_mtx_held_cnt;
 	uint64_t			lck_grp_mtx_miss_cnt;
 	uint64_t			lck_grp_mtx_wait_cnt;
+	/* Rest currently unused */
 	uint64_t			lck_grp_mtx_held_max;
 	uint64_t			lck_grp_mtx_held_cum;
 	uint64_t			lck_grp_mtx_wait_max;
@@ -101,7 +109,7 @@ typedef	struct _lck_grp_ {
 	unsigned int		lck_grp_mtxcnt;
 	unsigned int		lck_grp_rwcnt;
 	unsigned int		lck_grp_attr;
-	char				lck_grp_name[LCK_GRP_MAX_NAME];
+	char			lck_grp_name[LCK_GRP_MAX_NAME];
 	lck_grp_stat_t		lck_grp_stat;
 } lck_grp_t;
 
@@ -203,6 +211,9 @@ extern	void			lck_attr_setdefault(
 extern	void			lck_attr_setdebug(
 									lck_attr_t		*attr);
 
+extern	void			lck_attr_cleardebug(
+									lck_attr_t		*attr);
+
 #ifdef	XNU_KERNEL_PRIVATE
 extern	void			lck_attr_rw_shared_priority(
 									lck_attr_t		*attr);
@@ -300,6 +311,21 @@ extern wait_result_t	lck_mtx_sleep_deadline(
 extern boolean_t		lck_mtx_try_lock(
 									lck_mtx_t		*lck);
 
+#ifdef i386
+extern boolean_t		lck_mtx_try_lock_spin(
+									lck_mtx_t		*lck);
+
+extern void			lck_mtx_lock_spin(
+									lck_mtx_t		*lck);
+
+extern void			lck_mtx_convert_spin(
+									lck_mtx_t		*lck);
+#else
+#define lck_mtx_try_lock_spin(l)	lck_mtx_try_lock(l)
+#define	lck_mtx_lock_spin(l)		lck_mtx_lock(l)
+#define	lck_mtx_convert_spin(l)		do {} while (0)
+#endif
+
 #endif	/* KERNEL_PRIVATE */
 
 extern void				lck_mtx_assert(
@@ -322,9 +348,15 @@ extern int				lck_mtx_lock_acquire(
 extern void				lck_mtx_unlock_wakeup(
 									lck_mtx_t		*lck,
 									thread_t		holder);
+extern void				lck_mtx_unlockspin_wakeup(
+							                lck_mtx_t		*lck);
 
 extern boolean_t		lck_mtx_ilk_unlock(
 									lck_mtx_t		*lck);
+
+struct _lck_mtx_ext_;
+extern void lck_mtx_init_ext(lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext,
+    lck_grp_t *grp, lck_attr_t *attr);
 #endif
 
 #define decl_lck_rw_data(class,name)     class lck_rw_t name;
@@ -334,6 +366,12 @@ typedef unsigned int	 lck_rw_type_t;
 #define	LCK_RW_TYPE_SHARED			0x01
 #define	LCK_RW_TYPE_EXCLUSIVE		0x02
 
+#ifdef XNU_KERNEL_PRIVATE
+#define LCK_RW_ASSERT_SHARED	0x01
+#define LCK_RW_ASSERT_EXCLUSIVE	0x02
+#define LCK_RW_ASSERT_HELD	(LCK_RW_ASSERT_SHARED | LCK_RW_ASSERT_EXCLUSIVE)
+#endif
+
 __BEGIN_DECLS
 
 extern lck_rw_t			*lck_rw_alloc_init(
@@ -365,6 +403,17 @@ extern void				lck_rw_lock_exclusive(
 extern void				lck_rw_unlock_exclusive(
 									lck_rw_t		*lck);
 
+#ifdef	XNU_KERNEL_PRIVATE
+/*
+ * CAUTION
+ * read-write locks do not have a concept of ownership, so lck_rw_assert()
+ * merely asserts that someone is holding the lock, not necessarily the caller.
+ */
+extern void				lck_rw_assert(
+									lck_rw_t		*lck,
+									unsigned int		type);
+#endif
+
 #ifdef	KERNEL_PRIVATE
 
 extern lck_rw_type_t	lck_rw_done(
@@ -392,8 +441,6 @@ extern wait_result_t	lck_rw_sleep_deadline(
 									wait_interrupt_t	interruptible,
 									uint64_t			deadline);
 
-#ifdef	KERNEL_PRIVATE
-
 extern boolean_t		lck_rw_lock_shared_to_exclusive(
 									lck_rw_t		*lck);
 
@@ -404,6 +451,8 @@ extern boolean_t		lck_rw_try_lock(
 									lck_rw_t		*lck,
 									lck_rw_type_t	lck_rw_type);
 
+#ifdef	KERNEL_PRIVATE
+
 extern boolean_t		lck_rw_try_lock_shared(
 									lck_rw_t		*lck);
 
diff --git a/osfmk/kern/mach_clock.c b/osfmk/kern/mach_clock.c
index fb4392290..2c7b7f254 100644
--- a/osfmk/kern/mach_clock.c
+++ b/osfmk/kern/mach_clock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -56,7 +62,6 @@
  *
  *	Clock primitives.
  */
-#include <mach_prof.h>
 #include <gprof.h>
 
 #include <mach/boolean.h>
@@ -65,125 +70,92 @@
 #include <mach/vm_param.h>
 #include <mach/vm_prot.h>
 #include <kern/clock.h>
-#include <kern/counters.h>
 #include <kern/cpu_number.h>
 #include <kern/host.h>
 #include <kern/lock.h>
 #include <kern/mach_param.h>
 #include <kern/misc_protos.h>
 #include <kern/processor.h>
-#include <kern/profile.h>
 #include <kern/sched.h>
 #include <kern/sched_prim.h>
-#include <kern/spl.h>
 #include <kern/thread.h>
-#include <vm/vm_kern.h>					/* kernel_map */
-
-#include <mach/clock_server.h>
-#include <mach/clock_priv_server.h>
-#include <mach/mach_host_server.h>
 
 #include <profiling/profile-mk.h>
 
-boolean_t	profile_kernel_services = TRUE;	/* Indicates wether or not we
-											 * account kernel services 
-
-											 * samples for user task */
-#ifdef MACH_BSD
-extern void	bsd_hardclock(
-			boolean_t	usermode,
-			natural_t	pc,
-			int		numticks);
-#endif /* MACH_BSD */
+#if GPROF
+static void prof_tick(boolean_t usermode, natural_t pc);
+#endif
 
+#if STAT_TIME || GPROF
 /*
- * Hertz rate clock interrupt servicing. Primarily used to
- * update CPU statistics, recompute thread priority, and to
- * do profiling
+ * Hertz rate clock interrupt servicing. Used to update processor
+ * statistics and perform kernel profiling.
  */
 void
 hertz_tick(
-#if	STAT_TIME
+#if GPROF
+	__unused natural_t	ticks,
+#else
 	natural_t		ticks,
-#endif	/* STAT_TIME */
+#endif
 	boolean_t		usermode,
+#if GPROF
 	natural_t		pc)
+#else
+	__unused natural_t		pc)
+#endif
 {
 	processor_t		processor = current_processor();
+#if !GPROF
 	thread_t		thread = current_thread();
-	int				state;
-#if		MACH_PROF
-#ifdef	__MACHO__
-#define	ETEXT		etext
-	extern long		etext;
-#else
-#define	ETEXT		&etext
-	extern char		etext;
-#endif
-	boolean_t		inkernel;
-#endif	/* MACH_PROF */
-#if GPROF
-	struct profile_vars	*pv;
-	prof_uptrint_t		s;
-#endif
-
-#ifdef	lint
-	pc++;
-#endif	/* lint */
-
-	/*
-	 *	The system startup sequence initializes the clock
-	 *	before kicking off threads.   So it's possible,
-	 *	especially when debugging, to wind up here with
-	 *	no thread to bill against.  So ignore the tick.
-	 */
-	if (thread == THREAD_NULL)
-		return;
-
-#if		MACH_PROF
-	inkernel = !usermode && (pc < (unsigned int)ETEXT);
-#endif	/* MACH_PROF */
-
-	/*
-	 * Hertz processing performed by all processors
-	 * includes statistics gathering, state tracking,
-	 * and quantum updating.
-	 */
-	counter(c_clock_ticks++);
-
-#if     GPROF
-	pv = PROFILE_VARS(cpu_number());
 #endif
+	timer_t			state;
 
 	if (usermode) {
 		TIMER_BUMP(&thread->user_timer, ticks);
-		if (thread->priority < BASEPRI_DEFAULT)
-			state = CPU_STATE_NICE;
-		else
-			state = CPU_STATE_USER;
-#if GPROF
-			if (pv->active)
-			    PROF_CNT_INC(pv->stats.user_ticks);
-#endif
+
+		state = &PROCESSOR_DATA(processor, user_state);
 	}
 	else {
 		TIMER_BUMP(&thread->system_timer, ticks);
 
-		state = processor->state;
-		if (	state == PROCESSOR_IDLE			||
-				state == PROCESSOR_DISPATCHING)
-			state = CPU_STATE_IDLE;
-		else
-		if (thread->options & TH_OPT_DELAYIDLE)
-			state = CPU_STATE_IDLE;
+		if (processor->state == PROCESSOR_IDLE)
+			state = &PROCESSOR_DATA(processor, idle_state);
 		else
-			state = CPU_STATE_SYSTEM;
+			state = &PROCESSOR_DATA(processor, system_state);
+	}
+
+	TIMER_BUMP(state, ticks);
+
 #if GPROF
+	prof_tick(usermode, pc);
+#endif	/* GPROF */
+}
+
+#endif	/* STAT_TIME */
+
+#if GPROF
+
+static void
+prof_tick(
+	boolean_t	usermode,
+	natural_t	pc)
+{
+	struct profile_vars	*pv;
+	prof_uptrint_t		s;
+
+	pv = PROFILE_VARS(cpu_number());
+
+	if (usermode) {
+		if (pv->active)
+			PROF_CNT_INC(pv->stats.user_ticks);
+	}
+	else {
 		if (pv->active) {
-			if (state == CPU_STATE_SYSTEM)
-				PROF_CNT_INC(pv->stats.kernel_ticks);
-			else
+			if (current_processor()->state == CPU_STATE_IDLE)
 				PROF_CNT_INC(pv->stats.idle_ticks);
+			else
+				PROF_CNT_INC(pv->stats.kernel_ticks);
 
 			if ((prof_uptrint_t)pc < _profile_vars.profil_info.lowpc)
 				PROF_CNT_INC(pv->stats.too_low);
@@ -197,46 +169,7 @@ hertz_tick(
 					PROF_CNT_INC(pv->stats.too_high);
 			}
 		}
-#endif
-	}
-
-	PROCESSOR_DATA(processor, cpu_ticks[state]++);
-
-#ifdef MACH_BSD
-	/*XXX*/
-	if (processor == master_processor) {
-		bsd_hardclock(usermode, pc, 1);
-	}
-	/*XXX*/
-#endif /* MACH_BSD */
-
-#if	MACH_PROF
-	if (thread->act_profiled) {
-		if (inkernel && thread->map != kernel_map) {
-			/* 
-			 * Non-kernel thread running in kernel
-			 * Register user pc (mach_msg, vm_allocate ...)
-			 */
-		  	if (profile_kernel_services)
-		  		profile(user_pc(thread), thread->profil_buffer);
-		}
-		else
-			/*
-			 * User thread and user mode or
-			 * user (server) thread in kernel-loaded server or
-			 * kernel thread and kernel mode
-			 * register interrupted pc
-			 */
-			profile(pc, thread->profil_buffer);
-	}
-	if (kernel_task->task_profiled) {
-		if (inkernel && thread->map != kernel_map)
-		  	/*
-			 * User thread not profiled in kernel mode,
-			 * kernel task profiled, register kernel pc
-			 * for kernel task
-			 */
-			profile(pc, kernel_task->profil_buffer);
 	}
-#endif	/* MACH_PROF */
 }
+
+#endif	/* GPROF */
diff --git a/osfmk/kern/mach_param.h b/osfmk/kern/mach_param.h
index 4e45a7396..6294d5884 100644
--- a/osfmk/kern/mach_param.h
+++ b/osfmk/kern/mach_param.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,11 +69,11 @@
 #ifndef	_KERN_MACH_PARAM_H_
 #define _KERN_MACH_PARAM_H_
 
-#define THREAD_MAX		2560	/* Max number of threads */
-#define THREAD_CHUNK	64		/* Allocation chunk */
+#define THREAD_MAX		CONFIG_THREAD_MAX	/* Max number of threads */
+#define TASK_MAX		CONFIG_TASK_MAX		/* Max number of tasks */
 
-#define TASK_MAX	1024		/* Max number of tasks */
-#define TASK_CHUNK	64			/* Allocation chunk */
+#define THREAD_CHUNK	64		/* Allocation chunk */
+#define TASK_CHUNK		64		/* Allocation chunk */
 
 #define PORT_MAX	((TASK_MAX * 3 + THREAD_MAX)	/* kernel */ \
 				+ (THREAD_MAX * 2)	/* user */ \
diff --git a/osfmk/kern/machine.c b/osfmk/kern/machine.c
index 8a331ca87..1e7d3e96f 100644
--- a/osfmk/kern/machine.c
+++ b/osfmk/kern/machine.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -82,7 +88,9 @@
 #include <kern/task.h>
 #include <kern/thread.h>
 
+#if HIBERNATION
 #include <IOKit/IOHibernatePrivate.h>
+#endif
 #include <IOKit/IOPlatformExpert.h>
 
 /*
@@ -103,22 +111,21 @@ void			processor_doshutdown(
  */
 void
 processor_up(
-	processor_t		processor)
+	processor_t			processor)
 {
-	processor_set_t		pset = &default_pset;
+	processor_set_t		pset;
 	spl_t				s;
 
 	s = splsched();
-	processor_lock(processor);
 	init_ast_check(processor);
-	simple_lock(&pset->sched_lock);
-	pset_add_processor(pset, processor);
-	enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
+	pset = processor->processor_set;
+	pset_lock(pset);
+	pset->processor_count++;
+	enqueue_head(&pset->active_queue, (queue_entry_t)processor);
 	processor->state = PROCESSOR_RUNNING;
-	simple_unlock(&pset->sched_lock);
-	hw_atomic_add(&machine_info.avail_cpus, 1);
+	(void)hw_atomic_add(&processor_avail_count, 1);
+	pset_unlock(pset);
 	ml_cpu_up();
-	processor_unlock(processor);
 	splx(s);
 }
 
@@ -164,12 +171,13 @@ processor_shutdown(
 	spl_t				s;
 
 	s = splsched();
-	processor_lock(processor);
+	pset = processor->processor_set;
+	pset_lock(pset);
 	if (processor->state == PROCESSOR_OFF_LINE) {
 		/*
 		 * Success if already shutdown.
 		 */
-		processor_unlock(processor);
+		pset_unlock(pset);
 		splx(s);
 
 		return (KERN_SUCCESS);
@@ -179,45 +187,26 @@ processor_shutdown(
 		/*
 		 * Failure if currently being started.
 		 */
-		processor_unlock(processor);
+		pset_unlock(pset);
 		splx(s);
 
 		return (KERN_FAILURE);
 	}
 
 	/*
-	 * Must lock the scheduling lock
-	 * to get at the processor state.
+	 * If the processor is dispatching, let it finish.
 	 */
-	pset = processor->processor_set;
-	if (pset != PROCESSOR_SET_NULL) {
-		simple_lock(&pset->sched_lock);
-
-		/*
-		 * If the processor is dispatching, let it finish.
-		 */
-		while (processor->state == PROCESSOR_DISPATCHING) {
-			simple_unlock(&pset->sched_lock);
-			delay(1);
-			simple_lock(&pset->sched_lock);
-		}
-
-		/*
-		 * Success if already being shutdown.
-		 */
-		if (processor->state == PROCESSOR_SHUTDOWN) {
-			simple_unlock(&pset->sched_lock);
-			processor_unlock(processor);
-			splx(s);
-
-			return (KERN_SUCCESS);
-		}
+	while (processor->state == PROCESSOR_DISPATCHING) {
+		pset_unlock(pset);
+		delay(1);
+		pset_lock(pset);
 	}
-	else {
-		/*
-		 * Success, already being shutdown.
-		 */
-		processor_unlock(processor);
+
+	/*
+	 * Success if already being shutdown.
+	 */
+	if (processor->state == PROCESSOR_SHUTDOWN) {
+		pset_unlock(pset);
 		splx(s);
 
 		return (KERN_SUCCESS);
@@ -235,9 +224,7 @@ processor_shutdown(
 
 	processor->state = PROCESSOR_SHUTDOWN;
 
-	simple_unlock(&pset->sched_lock);
-
-	processor_unlock(processor);
+	pset_unlock(pset);
 
 	processor_doshutdown(processor);
 	splx(s);
@@ -255,59 +242,50 @@ processor_doshutdown(
 	processor_t			processor)
 {
 	thread_t			old_thread, self = current_thread();
-	processor_set_t		pset;
 	processor_t			prev;
-	int					pcount;
 
 	/*
 	 *	Get onto the processor to shutdown
 	 */
-	prev = thread_bind(self, processor);
+	prev = thread_bind(processor);
 	thread_block(THREAD_CONTINUE_NULL);
 
-	processor_lock(processor);
-	pset = processor->processor_set;
-	simple_lock(&pset->sched_lock);
-
-	if ((pcount = pset->processor_count) == 1) {
-		simple_unlock(&pset->sched_lock);
-		processor_unlock(processor);
-
+#if HIBERNATION
+	if (processor_avail_count < 2)
 		hibernate_vm_lock();
-
-		processor_lock(processor);
-		simple_lock(&pset->sched_lock);
-	}
+#endif
 
 	assert(processor->state == PROCESSOR_SHUTDOWN);
 
-	pset_remove_processor(pset, processor);
-	simple_unlock(&pset->sched_lock);
-	processor_unlock(processor);
-
-	if (pcount == 1)
+#if HIBERNATION
+	if (processor_avail_count < 2)
 		hibernate_vm_unlock();
+#endif
 
 	/*
 	 *	Continue processor shutdown in shutdown context.
 	 */
-	thread_bind(self, prev);
+	thread_bind(prev);
 	old_thread = machine_processor_shutdown(self, processor_offline, processor);
 
-	thread_begin(self, self->last_processor);
-
-	thread_dispatch(old_thread);
+	thread_dispatch(old_thread, self);
 
 	/*
-	 * If we just shutdown another processor, move the
-	 * timer call outs to the current processor.
+	 * If we just shutdown another processor, move any
+	 * threads and timer call outs to the current processor.
 	 */
 	if (processor != current_processor()) {
-		processor_lock(processor);
-		if (	processor->state == PROCESSOR_OFF_LINE	||
-				processor->state == PROCESSOR_SHUTDOWN	)
+		processor_set_t		pset = processor->processor_set;
+
+		pset_lock(pset);
+
+		if (processor->state == PROCESSOR_OFF_LINE || processor->state == PROCESSOR_SHUTDOWN) {
 			timer_call_shutdown(processor);
-		processor_unlock(processor);
+			processor_queue_shutdown(processor);
+			return;
+		}
+
+		pset_unlock(pset);
 	}
 }
 
@@ -318,33 +296,35 @@ processor_doshutdown(
  */
 void
 processor_offline(
-	processor_t		processor)
+	processor_t			processor)
 {
-	thread_t		thread, old_thread = processor->active_thread;
+	thread_t			new_thread, old_thread = processor->active_thread;
+	processor_set_t		pset;
 
-	thread = processor->idle_thread;
-	processor->active_thread = thread;
+	new_thread = processor->idle_thread;
+	processor->active_thread = new_thread;
 	processor->current_pri = IDLEPRI;
+	processor->deadline = UINT64_MAX;
+	new_thread->last_processor = processor;
 
 	processor->last_dispatch = mach_absolute_time();
-	timer_switch((uint32_t)processor->last_dispatch,
-							&PROCESSOR_DATA(processor, offline_timer));
-
-	thread_done(old_thread, thread, processor);
+	timer_stop(PROCESSOR_DATA(processor, thread_timer), processor->last_dispatch);
 
-	machine_set_current_thread(thread);
+	machine_set_current_thread(new_thread);
 
-	thread_begin(thread, processor);
-
-	thread_dispatch(old_thread);
+	thread_dispatch(old_thread, new_thread);
 
 	PMAP_DEACTIVATE_KERNEL(PROCESSOR_DATA(processor, slot_num));
 
-	processor_lock(processor);
+	pset = processor->processor_set;
+	pset_lock(pset);
+	pset->processor_count--;
 	processor->state = PROCESSOR_OFF_LINE;
-	hw_atomic_sub(&machine_info.avail_cpus, 1);
+	if (processor == pset->low_hint)
+		pset->low_hint = PROCESSOR_NULL;
+	(void)hw_atomic_sub(&processor_avail_count, 1);
+	pset_unlock(pset);
 	ml_cpu_down();
-	processor_unlock(processor);
 
 	cpu_sleep();
 	panic("zombie processor");
diff --git a/osfmk/kern/machine.h b/osfmk/kern/machine.h
index ce77bd868..1d3e4108d 100644
--- a/osfmk/kern/machine.h
+++ b/osfmk/kern/machine.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/macro_help.h b/osfmk/kern/macro_help.h
index eeaf1ae97..159b43237 100644
--- a/osfmk/kern/macro_help.h
+++ b/osfmk/kern/macro_help.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/misc_protos.h b/osfmk/kern/misc_protos.h
index 05b8acd9b..84721990d 100644
--- a/osfmk/kern/misc_protos.h
+++ b/osfmk/kern/misc_protos.h
@@ -1,27 +1,39 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef	_MISC_PROTOS_H_
 #define	_MISC_PROTOS_H_
@@ -35,6 +47,13 @@
 #include <ipc/ipc_types.h>
 #include <kern/debug.h>
 
+#ifndef MIN
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#endif /* MIN */
+#ifndef MAX
+#define MAX(a,b) (((a)>(b))?(a):(b))
+#endif  /* MAX */
+
 /* Set a bit in a bit array */
 extern void setbit(
 	int		which,
@@ -79,7 +98,7 @@ extern int copyinmsg(
 
 /* Move arbitrarily-aligned data from a kernel space to user space */
 extern int copyout(
-	const char      *kernel_addr,
+	const void      *kernel_addr,
 	user_addr_t     user_addr,
 	vm_size_t       nbytes);
 
@@ -90,18 +109,18 @@ extern int copyoutmsg(
 	mach_msg_size_t nbytes);
 
 /* Invalidate copy window(s) cache */
-extern void	inval_copy_windows(thread_t);
+extern void    inval_copy_windows(thread_t);
 
+extern int sscanf(const char *input, const char *fmt, ...) __scanflike(2,3);
 
-extern int sscanf(const char *input, const char *fmt, ...);
+/* sprintf() is being deprecated. Please use snprintf() instead. */ 
+extern integer_t sprintf(char *buf, const char *fmt, ...) __deprecated;
 
-extern integer_t sprintf(char *buf, const char *fmt, ...);
+extern int printf(const char *format, ...) __printflike(1,2);
 
-extern void printf(const char *format, ...);
+extern void dbugprintf(const char *format, ...) __printflike(1,2);
 
-extern void dbugprintf(const char *format, ...);
-
-extern void kdb_printf(const char *format, ...);
+extern int kdb_printf(const char *format, ...) __printflike(1,2);
 
 extern void printf_init(void);
 
@@ -165,4 +184,19 @@ extern kern_return_t	kernel_set_special_port(
 		int 		which,
 		ipc_port_t	port);
 
+user_addr_t get_useraddr(void);
+
+/* symbol lookup */
+struct kmod_info_t;
+
+extern int syms_formataddr(
+		vm_offset_t	addr,
+		char		*out,
+		vm_offset_t	outsize);
+
+extern const char *syms_nameforaddr(
+		vm_offset_t	addr,
+		vm_offset_t	*ofs,
+		kmod_info_t	**kmod);
+
 #endif	/* _MISC_PROTOS_H_ */
diff --git a/osfmk/kern/mk_sp.c b/osfmk/kern/mk_sp.c
index 565681e39..78a6371fd 100644
--- a/osfmk/kern/mk_sp.c
+++ b/osfmk/kern/mk_sp.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,6 +76,9 @@ thread_policy_common(
 			invalid_policy(policy)		)
 		return(KERN_INVALID_ARGUMENT);
 
+	if (thread->static_param)
+		return (KERN_SUCCESS);
+
 	s = splsched();
 	thread_lock(thread);
 
@@ -81,15 +90,15 @@ thread_policy_common(
 			if (policy == POLICY_TIMESHARE && !oldmode) {
 				thread->sched_mode |= TH_MODE_TIMESHARE;
 
-				if (thread->state & TH_RUN)
-					pset_share_incr(thread->processor_set);
+				if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+					sched_share_incr();
 			}
 			else
 			if (policy != POLICY_TIMESHARE && oldmode) {
 				thread->sched_mode &= ~TH_MODE_TIMESHARE;
 
-				if (thread->state & TH_RUN)
-					pset_share_decr(thread->processor_set);
+				if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+					sched_share_decr();
 			}
 		}
 		else {
@@ -150,17 +159,11 @@ thread_set_policy(
 	kern_return_t			result = KERN_SUCCESS;
 
 	if (	thread == THREAD_NULL			||
-			pset == PROCESSOR_SET_NULL		)
+			pset == PROCESSOR_SET_NULL || pset != &pset0)
 		return (KERN_INVALID_ARGUMENT);
 
 	thread_mtx_lock(thread);
 
-	if (pset != thread->processor_set) {
-		thread_mtx_unlock(thread);
-
-		return (KERN_FAILURE);
-	}
-
 	switch (policy) {
 
 	case POLICY_RR:
@@ -261,9 +264,9 @@ thread_policy(
 	boolean_t				set_limit)
 {
 	kern_return_t			result = KERN_SUCCESS;
-	processor_set_t			pset;
-	policy_limit_t			limit;
-	int						limcount;
+	processor_set_t			pset = &pset0;
+	policy_limit_t			limit = NULL;
+	int						limcount = 0;
 	policy_rr_limit_data_t			rr_limit;
 	policy_fifo_limit_data_t		fifo_limit;
 	policy_timeshare_limit_data_t	ts_limit;
@@ -273,13 +276,6 @@ thread_policy(
 
 	thread_mtx_lock(thread);
 
-	pset = thread->processor_set;
-	if (pset == PROCESSOR_SET_NULL) {
-		thread_mtx_unlock(thread);
-
-		return (KERN_INVALID_ARGUMENT);
-	}
-
 	if (	invalid_policy(policy)											||
 			((POLICY_TIMESHARE | POLICY_RR | POLICY_FIFO) & policy) == 0	) {
 		thread_mtx_unlock(thread);
diff --git a/osfmk/kern/mk_timer.c b/osfmk/kern/mk_timer.c
index 6b6d15d23..2fac290ec 100644
--- a/osfmk/kern/mk_timer.c
+++ b/osfmk/kern/mk_timer.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2000 Apple Computer, Inc.  All rights reserved.
diff --git a/osfmk/kern/mk_timer.h b/osfmk/kern/mk_timer.h
index d47e02bb2..c67d74ba9 100644
--- a/osfmk/kern/mk_timer.h
+++ b/osfmk/kern/mk_timer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2000 Apple Computer, Inc.  All rights reserved.
@@ -50,7 +56,7 @@ typedef struct mk_timer		*mk_timer_t, mk_timer_data_t;
 void		mk_timer_port_destroy(
 				ipc_port_t				port);
 
-void		mk_timer_init(void);
+void		mk_timer_init(void) __attribute__((section("__TEXT, initcode")));
 
 #endif /* MACH_KERNEL_PRIVATE */
 
diff --git a/osfmk/kern/norma_protos.h b/osfmk/kern/norma_protos.h
index 2f45824a9..1071f40cc 100644
--- a/osfmk/kern/norma_protos.h
+++ b/osfmk/kern/norma_protos.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/page_decrypt.c b/osfmk/kern/page_decrypt.c
index c397ab1e5..f416dc105 100644
--- a/osfmk/kern/page_decrypt.c
+++ b/osfmk/kern/page_decrypt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 
@@ -27,8 +33,6 @@
 
 /*#include <sys/kernel.h> */
 extern int hz;			/* system clock's frequency */
-extern void*   dsmos_blobs[];
-extern int     dsmos_blob_count;
 
 /* #include <sys/proc.h> */
 extern int	tsleep(void *chan, int pri, const char *wmesg, int timo);
@@ -44,27 +48,23 @@ int
 _dsmos_wait_for_callback(const void* from, void *to)
 {
 /*	printf("%s\n", __FUNCTION__); */
-	while ( (dsmos_hook == NULL) || (dsmos_hook == _dsmos_wait_for_callback) )
+	while (dsmos_hook == NULL || dsmos_hook == _dsmos_wait_for_callback)
 		tsleep(&dsmos_hook, PZERO, "dsmos", hz / 10);
 
 	return (*dsmos_hook) (from, to);
 }
 
 void
-dsmos_page_transform_hook(dsmos_page_transform_hook_t hook,
-			  void (*commpage_setup_dsmos_blob)(void**, int))
+dsmos_page_transform_hook(dsmos_page_transform_hook_t hook)
 {
-#ifdef i386
-	/* finish initializing the commpage here */
-	(*commpage_setup_dsmos_blob)(dsmos_blobs, dsmos_blob_count);
-#endif
+/*	printf("%s\n", __FUNCTION__); */
 
 	/* set the hook now - new callers will run with it */
 	dsmos_hook = hook;
 }
 
 int
-dsmos_page_transform(const void* from, void* to)
+dsmos_page_transform(const void* from, void *to)
 {
 /*	printf("%s\n", __FUNCTION__); */
 	if (dsmos_hook == NULL)
diff --git a/osfmk/kern/page_decrypt.h b/osfmk/kern/page_decrypt.h
index 7e0e56135..a0517a01f 100644
--- a/osfmk/kern/page_decrypt.h
+++ b/osfmk/kern/page_decrypt.h
@@ -1,32 +1,40 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _KERN_PAGE_DECRYPT_H
 #define _KERN_PAGE_DECRYPT_H
 
-typedef	int	  (*dsmos_page_transform_hook_t) (const void*,void*);
-extern	void	dsmos_page_transform_hook(dsmos_page_transform_hook_t hook,
-					  void (*commpage_setup_dsmos_blob)(void**, int));	/* exported */
+typedef	int	  (*dsmos_page_transform_hook_t) (const void *,void*);
+extern	void	dsmos_page_transform_hook(dsmos_page_transform_hook_t hook);	/* exported */
 
-extern	int		dsmos_page_transform(const void*,void*);
+extern	int		dsmos_page_transform(const void *,void*);
 
 #endif	/* _KERN_PAGE_DECRYPT_H */
+
+
+
diff --git a/osfmk/kern/pms.h b/osfmk/kern/pms.h
index 72f601057..dceb5bbe5 100644
--- a/osfmk/kern/pms.h
+++ b/osfmk/kern/pms.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifdef KERNEL_PRIVATE
@@ -152,8 +158,11 @@ extern pmsQueryFunc_t pmsQueryFunc;
 extern uint32_t pmsPlatformData;
 
 #ifdef __ppc__
+# ifdef XNU_KERNEL_PRIVATE
+# include <ppc/savearea.h>
+# endif /* XNU_KERNEL_PRIVATE */
 extern int pmsCntrl(struct savearea *save);
-#endif
+#endif /* __ppc__ */
 extern kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize);
 extern void pmsInit(void);
 extern void pmsStep(int timer);
diff --git a/osfmk/kern/printf.c b/osfmk/kern/printf.c
index 90bda90e0..6faceb7aa 100644
--- a/osfmk/kern/printf.c
+++ b/osfmk/kern/printf.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -166,6 +172,7 @@
 #ifdef  MACH_BSD
 #include <sys/msgbuf.h>
 #endif
+#include <console/serial_protos.h>
 
 #ifdef __ppc__
 #include <ppc/Firmware.h>
@@ -177,15 +184,20 @@
 #define MAXBUF (sizeof(long long int) * 8)	/* enough for binary */
 static char digs[] = "0123456789abcdef";
 
+
+#if CONFIG_NO_PRINTF_STRINGS
+#undef printf(x, ...)
+#endif
+
 static int
 printnum(
-	register unsigned long long int	u,	/* number to print */
-	register int		base,
+	unsigned long long int	u,	/* number to print */
+	int		base,
 	void			(*putc)(int, void *),
 	void                    *arg)
 {
 	char	buf[MAXBUF];	/* build number here */
-	register char *	p = &buf[MAXBUF-1];
+	char *	p = &buf[MAXBUF-1];
 	int nprinted = 0;
 
 	do {
@@ -205,7 +217,7 @@ boolean_t	_doprnt_truncates = FALSE;
 
 int
 __doprnt(
-	register const char	*fmt,
+	const char	*fmt,
 	va_list			*argp,
 						/* character output routine */
 	void			(*putc)(int, void *arg),
@@ -222,7 +234,7 @@ __doprnt(
 	int		sign_char;
 	boolean_t	altfmt, truncate;
 	int		base;
-	register char	c;
+	char	c;
 	int		capitals;
 	int		long_long;
 	int             nprinted = 0;
@@ -394,15 +406,15 @@ __doprnt(
 
 		case 's':
 		{
-		    register char *p;
-		    register char *p2;
+		    register const char *p;
+		    register const char *p2;
 
 		    if (prec == -1)
 			prec = 0x7fffffff;	/* MAXINT */
 
 		    p = va_arg(*argp, char *);
 
-		    if (p == (char *)0)
+		    if (p == NULL)
 			p = "";
 
 		    if (length > 0 && !ladjust) {
@@ -545,7 +557,7 @@ __doprnt(
 		    char	buf[MAXBUF];	/* build number here */
 		    register char *	p = &buf[MAXBUF-1];
 		    static char digits[] = "0123456789abcdef0123456789ABCDEF";
-		    char *prefix = 0;
+		    const char *prefix = NULL;
 
 		    if (truncate) u = (long long)((int)(u));
 
@@ -566,7 +578,7 @@ __doprnt(
 		    if (sign_char)
 			length--;
 		    if (prefix)
-			length -= strlen((const char *) prefix);
+			length -= strlen(prefix);
 
 		    if (padc == ' ' && !ladjust) {
 			/* blank padding goes before prefix */
@@ -646,7 +658,6 @@ boolean_t	new_printf_cpu_number = FALSE;
 
 decl_simple_lock_data(,printf_lock)
 decl_simple_lock_data(,bsd_log_spinlock)
-decl_mutex_data(,sprintf_lock)
 extern void bsd_log_init(void);
 void bsd_log_lock(void);
 void bsd_log_unlock(void);
@@ -660,17 +671,16 @@ printf_init(void)
 	simple_lock_init(&printf_lock, 0);
 	simple_lock_init(&bsd_log_spinlock, 0);
 	bsd_log_init();
-	mutex_init(&sprintf_lock, 0);
 }
 
 void
-bsd_log_lock()
+bsd_log_lock(void)
 {
 	simple_lock(&bsd_log_spinlock);
 }
 
 void
-bsd_log_unlock()
+bsd_log_unlock(void)
 {
 	simple_unlock(&bsd_log_spinlock);
 }
@@ -724,13 +734,13 @@ safe_gets(
 	}
 }
 
+extern int disableConsoleOutput;
+
 void
 conslog_putc(
 	char c)
 {
-	extern unsigned int debug_mode, disableDebugOuput, disableConsoleOutput;
-
-	if ((debug_mode && !disableDebugOuput) || !disableConsoleOutput)
+	if ((debug_mode && !disable_debug_output) || !disableConsoleOutput)
 		cnputc(c);
 
 #ifdef	MACH_BSD
@@ -738,13 +748,15 @@ conslog_putc(
 #endif
 }
 
+#if	MACH_KDB
+extern void db_putchar(char c);
+#endif
+
 void
-dbugprintf(const char *fmt, ...)
+dbugprintf(__unused const char *fmt, ...)
 {
 
 #if	MACH_KDB
-
-	extern void db_putchar(char c);
 	va_list	listp;
 
 	va_start(listp, fmt);
@@ -754,37 +766,35 @@ dbugprintf(const char *fmt, ...)
 	return;
 }
 
-void
+int
 printf(const char *fmt, ...)
 {
 	va_list	listp;
 
-	disable_preemption();
-	va_start(listp, fmt);
-	_doprnt(fmt, &listp, conslog_putc, 16);
-	va_end(listp);
-	enable_preemption();
+	if (fmt) {
+		disable_preemption();
+		va_start(listp, fmt);
+		_doprnt(fmt, &listp, conslog_putc, 16);
+		va_end(listp);
+		enable_preemption();
+	}
+	return 0;
 }
 
-extern unsigned int disableSerialOuput;
-
 void
-consdebug_putc(
-	char c)
+consdebug_putc(char c)
 {
-	extern unsigned int debug_mode, disableDebugOuput, disableConsoleOutput;
-
-	if ((debug_mode && !disableDebugOuput) || !disableConsoleOutput)
+	if ((debug_mode && !disable_debug_output) || !disableConsoleOutput)
 		cnputc(c);
 
 	debug_putc(c);
 
 	if (!console_is_serial())
-		if (!disableSerialOuput)
+		if (!disable_serial_output)
 			PE_kputc(c);
 }
 
-void
+int
 kdb_printf(const char *fmt, ...)
 {
 	va_list	listp;
@@ -792,28 +802,37 @@ kdb_printf(const char *fmt, ...)
 	va_start(listp, fmt);
 	_doprnt(fmt, &listp, consdebug_putc, 16);
 	va_end(listp);
+	return 0;
 }
 
-static char *copybyte_str;
-
 static void
-copybyte(
-        char byte)
+copybyte(int c, void *arg)
 {
-  *copybyte_str++ = byte;
-  *copybyte_str = '\0';
+	/*
+	 * arg is a pointer (outside pointer) to the pointer
+	 * (inside pointer) which points to the character.
+	 * We pass a double pointer, so that we can increment
+	 * the inside pointer.
+	 */
+	char** p = arg;	/* cast outside pointer */
+	**p = c;	/* store character */
+	(*p)++;		/* increment inside pointer */
 }
 
+/*
+ * Deprecation Warning:
+ *	sprintf() is being deprecated. Please use snprintf() instead.
+ */
 int
 sprintf(char *buf, const char *fmt, ...)
 {
         va_list listp;
+	char *copybyte_str;
 
         va_start(listp, fmt);
-	mutex_lock(&sprintf_lock);
         copybyte_str = buf;
-        _doprnt(fmt, &listp, copybyte, 16);
-	mutex_unlock(&sprintf_lock);
+        __doprnt(fmt, &listp, copybyte, &copybyte_str, 16);
         va_end(listp);
+	*copybyte_str = '\0';
         return strlen(buf);
 }
diff --git a/osfmk/kern/priority.c b/osfmk/kern/priority.c
index 9ff146229..4f08fd378 100644
--- a/osfmk/kern/priority.c
+++ b/osfmk/kern/priority.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -72,6 +78,8 @@
  *	thread_quantum_expire:
  *
  *	Recalculate the quantum and priority for a thread.
+ *
+ *	Called at splsched.
  */
 
 void
@@ -79,11 +87,10 @@ thread_quantum_expire(
 	timer_call_param_t	p0,
 	timer_call_param_t	p1)
 {
-	register processor_t		myprocessor = p0;
-	register thread_t			thread = p1;
-	spl_t						s;
+	processor_t			processor = p0;
+	thread_t			thread = p1;
+	ast_t				preempt;
 
-	s = splsched();
 	thread_lock(thread);
 
 	/*
@@ -92,7 +99,7 @@ thread_quantum_expire(
 	if (!(thread->sched_mode & TH_MODE_TIMESHARE)) {
 		uint64_t			new_computation;
 
-		new_computation = myprocessor->quantum_end;
+		new_computation = processor->quantum_end;
 		new_computation -= thread->computation_epoch;
 		if (new_computation + thread->computation_metered >
 											max_unsafe_computation) {
@@ -104,7 +111,7 @@ thread_quantum_expire(
 				thread->sched_mode &= ~TH_MODE_REALTIME;
 			}
 
-			pset_share_incr(thread->processor_set);
+			sched_share_incr();
 
 			thread->safe_release = sched_tick + sched_safe_duration;
 			thread->sched_mode |= (TH_MODE_FAILSAFE|TH_MODE_TIMESHARE);
@@ -143,25 +150,36 @@ thread_quantum_expire(
 			compute_my_priority(thread);
 	}
 
+	processor->current_pri = thread->sched_pri;
+
 	/*
 	 *	This quantum is up, give this thread another.
 	 */
-	if (first_timeslice(myprocessor))
-		myprocessor->timeslice--;
+	if (first_timeslice(processor))
+		processor->timeslice--;
 
 	thread_quantum_init(thread);
-	myprocessor->quantum_end += thread->current_quantum;
-	timer_call_enter1(&myprocessor->quantum_timer,
-							thread, myprocessor->quantum_end);
-
-	thread_unlock(thread);
+	processor->quantum_end += thread->current_quantum;
+	timer_call_enter1(&processor->quantum_timer,
+							thread, processor->quantum_end);
 
 	/*
-	 * Check for and schedule ast if needed.
+	 *	Context switch check.
 	 */
-	ast_check(myprocessor);
+	if ((preempt = csw_check(thread, processor)) != AST_NONE)
+		ast_on(preempt);
+	else {
+		processor_set_t		pset = processor->processor_set;
 
-	splx(s);
+		pset_lock(pset);
+
+		pset_hint_low(pset, processor);
+		pset_hint_high(pset, processor);
+
+		pset_unlock(pset);
+	}
+
+	thread_unlock(thread);
 }
 
 /*
@@ -263,7 +281,7 @@ compute_my_priority(
 	register int		priority;
 
 	do_priority_computation(thread, priority);
-	assert(thread->runq == RUN_QUEUE_NULL);
+	assert(thread->runq == PROCESSOR_NULL);
 	thread->sched_pri = priority;
 }
 
@@ -284,7 +302,7 @@ update_priority(
 	ticks = sched_tick - thread->sched_stamp;
 	assert(ticks != 0);
 	thread->sched_stamp += ticks;
-	thread->pri_shift = thread->processor_set->pri_shift;
+	thread->pri_shift = sched_pri_shift;
 
 	/*
 	 *	Gather cpu usage data.
@@ -341,8 +359,8 @@ update_priority(
 
 			thread->sched_mode &= ~TH_MODE_TIMESHARE;
 
-			if (thread->state & TH_RUN)
-				pset_share_decr(thread->processor_set);
+			if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+				sched_share_decr();
 
 			if (!(thread->sched_mode & TH_MODE_ISDEPRESSED))
 				set_sched_pri(thread, thread->priority);
@@ -362,11 +380,10 @@ update_priority(
 
 		do_priority_computation(thread, new_pri);
 		if (new_pri != thread->sched_pri) {
-			run_queue_t		runq;
+			boolean_t		removed = run_queue_remove(thread);
 
-			runq = run_queue_remove(thread);
 			thread->sched_pri = new_pri;
-			if (runq != RUN_QUEUE_NULL)
+			if (removed)
 				thread_setrun(thread, SCHED_TAILQ);
 		}
 	}
diff --git a/osfmk/kern/processor.c b/osfmk/kern/processor.c
index 6f93f5ef4..9ef40cd53 100644
--- a/osfmk/kern/processor.c
+++ b/osfmk/kern/processor.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -78,32 +84,27 @@
 #include <mach/mach_host_server.h>
 #include <mach/processor_set_server.h>
 
-/*
- *	Exported variables.
- */
-struct processor_set	default_pset;
+struct processor_set	pset0;
+struct pset_node		pset_node0;
+decl_simple_lock_data(static,pset_node_lock)
+
+queue_head_t			tasks;
+int						tasks_count;
+queue_head_t			threads;
+int						threads_count;
+decl_mutex_data(,tasks_threads_lock)
 
 processor_t				processor_list;
 unsigned int			processor_count;
 static processor_t		processor_list_tail;
 decl_simple_lock_data(,processor_list_lock)
 
+uint32_t				processor_avail_count;
+
 processor_t	master_processor;
 int 		master_cpu = 0;
 
 /* Forwards */
-kern_return_t	processor_set_base(
-		processor_set_t 	pset,
-		policy_t             	policy,
-	        policy_base_t           base,
-		boolean_t       	change);
-
-kern_return_t	processor_set_limit(
-		processor_set_t 	pset,
-		policy_t		policy,
-	        policy_limit_t    	limit,
-		boolean_t       	change);
-
 kern_return_t	processor_set_things(
 		processor_set_t		pset,
 		mach_port_t		**thing_list,
@@ -113,84 +114,47 @@ kern_return_t	processor_set_things(
 void
 processor_bootstrap(void)
 {
-	simple_lock_init(&processor_list_lock, 0);
+	pset_init(&pset0, &pset_node0);
+	pset_node0.psets = &pset0;
 
-	master_processor = cpu_to_processor(master_cpu);
+	simple_lock_init(&pset_node_lock, 0);
 
-	processor_init(master_processor, master_cpu);
-}
+	mutex_init(&tasks_threads_lock, 0);
+	queue_init(&tasks);
+	queue_init(&threads);
 
-/*
- *	Initialize the given processor_set structure.
- */
-
-void
-pset_init(
-	register processor_set_t	pset)
-{
-	register int	i;
+	simple_lock_init(&processor_list_lock, 0);
 
-	/* setup run queue */
-	pset->runq.highq = IDLEPRI;
-	for (i = 0; i < NRQBM; i++)
-	    pset->runq.bitmap[i] = 0;
-	setbit(MAXPRI - IDLEPRI, pset->runq.bitmap); 
-	pset->runq.urgency = pset->runq.count = 0;
-	for (i = 0; i < NRQS; i++)
-	    queue_init(&pset->runq.queues[i]);
+	master_processor = cpu_to_processor(master_cpu);
 
-	queue_init(&pset->idle_queue);
-	pset->idle_count = 0;
-	queue_init(&pset->active_queue);
-	simple_lock_init(&pset->sched_lock, 0);
-	pset->run_count = pset->share_count = 0;
-	pset->mach_factor = pset->load_average = 0;
-	pset->pri_shift = INT8_MAX;
-	queue_init(&pset->processors);
-	pset->processor_count = 0;
-	queue_init(&pset->tasks);
-	pset->task_count = 0;
-	queue_init(&pset->threads);
-	pset->thread_count = 0;
-	pset->ref_count = 1;
-	pset->active = TRUE;
-	mutex_init(&pset->lock, 0);
-	pset->pset_self = IP_NULL;
-	pset->pset_name_self = IP_NULL;
-	pset->timeshare_quanta = 1;
+	processor_init(master_processor, master_cpu, &pset0);
 }
 
 /*
- *	Initialize the given processor structure for the processor in
- *	the slot specified by slot_num.
+ *	Initialize the given processor for the cpu
+ *	indicated by slot_num, and assign to the
+ *	specified processor set.
  */
 void
 processor_init(
-	register processor_t	p,
-	int						slot_num)
+	processor_t		p,
+	int				slot_num,
+	processor_set_t	pset)
 {
-	register int	i;
-
-	/* setup run queue */
-	p->runq.highq = IDLEPRI;
-	for (i = 0; i < NRQBM; i++)
-	    p->runq.bitmap[i] = 0;
-	setbit(MAXPRI - IDLEPRI, p->runq.bitmap); 
-	p->runq.urgency = p->runq.count = 0;
-	for (i = 0; i < NRQS; i++)
-	    queue_init(&p->runq.queues[i]);
+	run_queue_init(&p->runq);
 
 	p->state = PROCESSOR_OFF_LINE;
 	p->active_thread = p->next_thread = p->idle_thread = THREAD_NULL;
-	p->processor_set = PROCESSOR_SET_NULL;
+	p->processor_set = pset;
 	p->current_pri = MINPRI;
-	p->deadline = UINT64_MAX;
 	timer_call_setup(&p->quantum_timer, thread_quantum_expire, p);
+	p->deadline = UINT64_MAX;
 	p->timeslice = 0;
-	simple_lock_init(&p->lock, 0);
 	p->processor_self = IP_NULL;
+	simple_lock_init(&p->lock, 0);
 	processor_data_init(p);
 	PROCESSOR_DATA(p, slot_num) = slot_num;
+	p->processor_list = NULL;
 
 	simple_lock(&processor_list_lock);
 	if (processor_list == NULL)
@@ -199,167 +163,66 @@ processor_init(
 		processor_list_tail->processor_list = p;
 	processor_list_tail = p;
 	processor_count++;
-	p->processor_list = NULL;
 	simple_unlock(&processor_list_lock);
 }
 
-/*
- *	pset_deallocate:
- *
- *	Remove one reference to the processor set.  Destroy processor_set
- *	if this was the last reference.
- */
-void
-pset_deallocate(
-	processor_set_t	pset)
+processor_set_t
+processor_pset(
+	processor_t	processor)
 {
-	if (pset == PROCESSOR_SET_NULL)
-		return;
-
-	assert(pset == &default_pset);
-	return;
+	return (processor->processor_set);
 }
 
-/*
- *	pset_reference:
- *
- *	Add one reference to the processor set.
- */
-void
-pset_reference(
-	processor_set_t	pset)
+pset_node_t
+pset_node_root(void)
 {
-	if (pset == PROCESSOR_SET_NULL)
-  		return;
-
-	assert(pset == &default_pset);
+	return &pset_node0;
 }
 
-#define pset_reference_locked(pset) assert(pset == &default_pset)
-
-/*
- *	pset_remove_processor() removes a processor from a processor_set.
- *	It can only be called on the current processor.  Caller must
- *	hold lock on current processor and processor set.
- */
-void
-pset_remove_processor(
-	processor_set_t	pset,
-	processor_t	processor)
+processor_set_t
+pset_create(
+	pset_node_t			node)
 {
-	if (pset != processor->processor_set)
-		panic("pset_remove_processor: wrong pset");
+	processor_set_t		*prev, pset = kalloc(sizeof (*pset));
 
-	queue_remove(&pset->processors, processor, processor_t, processors);
-	processor->processor_set = PROCESSOR_SET_NULL;
-	pset->processor_count--;
-	timeshare_quanta_update(pset);
-}
+	if (pset != PROCESSOR_SET_NULL) {
+		pset_init(pset, node);
 
-/*
- *	pset_add_processor() adds a  processor to a processor_set.
- *	It can only be called on the current processor.  Caller must
- *	hold lock on curent processor and on pset.  No reference counting on
- *	processors.  Processor reference to pset is implicit.
- */
-void
-pset_add_processor(
-	processor_set_t	pset,
-	processor_t	processor)
-{
-	queue_enter(&pset->processors, processor, processor_t, processors);
-	processor->processor_set = pset;
-	pset->processor_count++;
-	timeshare_quanta_update(pset);
-}
+		simple_lock(&pset_node_lock);
 
-/*
- *	pset_remove_task() removes a task from a processor_set.
- *	Caller must hold locks on pset and task (unless task has
- *	no references left, in which case just the pset lock is
- *	needed).  Pset reference count is not decremented;
- *	caller must explicitly pset_deallocate.
- */
-void
-pset_remove_task(
-	processor_set_t	pset,
-	task_t		task)
-{
-	if (pset != task->processor_set)
-		return;
+		prev = &node->psets;
+		while (*prev != PROCESSOR_SET_NULL)
+			prev = &(*prev)->pset_list;
 
-	queue_remove(&pset->tasks, task, task_t, pset_tasks);
-	pset->task_count--;
-}
+		*prev = pset;
 
-/*
- *	pset_add_task() adds a  task to a processor_set.
- *	Caller must hold locks on pset and task.  Pset references to
- *	tasks are implicit.
- */
-void
-pset_add_task(
-	processor_set_t	pset,
-	task_t		task)
-{
-	queue_enter(&pset->tasks, task, task_t, pset_tasks);
-	task->processor_set = pset;
-	pset->task_count++;
-	pset_reference_locked(pset);
-}
+		simple_unlock(&pset_node_lock);
+	}
 
-/*
- *	pset_remove_thread() removes a thread from a processor_set.
- *	Caller must hold locks on pset and thread (but only if thread
- *  has outstanding references that could be used to lookup the pset).
- *  The pset reference count is not decremented; caller must explicitly
- *  pset_deallocate.
- */
-void
-pset_remove_thread(
-	processor_set_t	pset,
-	thread_t	thread)
-{
-	queue_remove(&pset->threads, thread, thread_t, pset_threads);
-	pset->thread_count--;
+	return (pset);
 }
 
 /*
- *	pset_add_thread() adds a  thread to a processor_set.
- *	Caller must hold locks on pset and thread.  Pset references to
- *	threads are implicit.
+ *	Initialize the given processor_set structure.
  */
 void
-pset_add_thread(
-	processor_set_t	pset,
-	thread_t	thread)
+pset_init(
+	processor_set_t		pset,
+	pset_node_t			node)
 {
-	queue_enter(&pset->threads, thread, thread_t, pset_threads);
-	thread->processor_set = pset;
-	pset->thread_count++;
-	pset_reference_locked(pset);
+	queue_init(&pset->active_queue);
+	queue_init(&pset->idle_queue);
+	pset->idle_count = 0;
+	pset->processor_count = 0;
+	pset->high_hint = PROCESSOR_NULL;
+	pset->low_hint = PROCESSOR_NULL;
+	pset_lock_init(pset);
+	pset->pset_self = IP_NULL;
+	pset->pset_name_self = IP_NULL;
+	pset->pset_list = PROCESSOR_SET_NULL;
+	pset->node = node;
 }
 
-/*
- *	thread_change_psets() changes the pset of a thread.  Caller must
- *	hold locks on both psets and thread.  The old pset must be
- *	explicitly pset_deallocat()'ed by caller.
- */
-void
-thread_change_psets(
-	thread_t	thread,
-	processor_set_t old_pset,
-	processor_set_t new_pset)
-{
-	queue_remove(&old_pset->threads, thread, thread_t, pset_threads);
-	old_pset->thread_count--;
-	queue_enter(&new_pset->threads, thread, thread_t, pset_threads);
-	thread->processor_set = new_pset;
-	new_pset->thread_count++;
-	pset_reference_locked(new_pset);
-}	
-
-
 kern_return_t
 processor_info_count(
 	processor_flavor_t		flavor,
@@ -391,7 +254,7 @@ processor_info(
 	processor_info_t		info,
 	mach_msg_type_number_t	*count)
 {
-	register int	i, slot_num, state;
+	register int	slot_num, state;
 	kern_return_t	result;
 
 	if (processor == PROCESSOR_NULL)
@@ -431,15 +294,18 @@ processor_info(
 	case PROCESSOR_CPU_LOAD_INFO:
 	{
 		register processor_cpu_load_info_t	cpu_load_info;
-		register integer_t					*cpu_ticks;
 
 	    if (*count < PROCESSOR_CPU_LOAD_INFO_COUNT)
 			return (KERN_FAILURE);
 
 	    cpu_load_info = (processor_cpu_load_info_t) info;
-		cpu_ticks = PROCESSOR_DATA(processor, cpu_ticks);
-	    for (i=0; i < CPU_STATE_MAX; i++)
-			cpu_load_info->cpu_ticks[i] = cpu_ticks[i];
+		cpu_load_info->cpu_ticks[CPU_STATE_USER] =
+							timer_grab(&PROCESSOR_DATA(processor, user_state)) / hz_tick_interval;
+		cpu_load_info->cpu_ticks[CPU_STATE_SYSTEM] =
+							timer_grab(&PROCESSOR_DATA(processor, system_state)) / hz_tick_interval;
+		cpu_load_info->cpu_ticks[CPU_STATE_IDLE] =
+							timer_grab(&PROCESSOR_DATA(processor, idle_state)) / hz_tick_interval;
+		cpu_load_info->cpu_ticks[CPU_STATE_NICE] = 0;
 
 	    *count = PROCESSOR_CPU_LOAD_INFO_COUNT;
 	    *host = &realhost;
@@ -458,40 +324,41 @@ processor_info(
 
 kern_return_t
 processor_start(
-	processor_t	processor)
+	processor_t			processor)
 {
-	kern_return_t	result;
-	thread_t		thread;   
-	spl_t			s;
+	processor_set_t		pset;
+	thread_t			thread;   
+	kern_return_t		result;
+	spl_t				s;
 
-	if (processor == PROCESSOR_NULL)
+	if (processor == PROCESSOR_NULL || processor->processor_set == PROCESSOR_SET_NULL)
 		return (KERN_INVALID_ARGUMENT);
 
 	if (processor == master_processor) {
-		thread_t		self = current_thread();
 		processor_t		prev;
 
-		prev = thread_bind(self, processor);
+		prev = thread_bind(processor);
 		thread_block(THREAD_CONTINUE_NULL);
 
 		result = cpu_start(PROCESSOR_DATA(processor, slot_num));
 
-		thread_bind(self, prev);
+		thread_bind(prev);
 
 		return (result);
 	}
 
 	s = splsched();
-	processor_lock(processor);
+	pset = processor->processor_set;
+	pset_lock(pset);
 	if (processor->state != PROCESSOR_OFF_LINE) {
-		processor_unlock(processor);
+		pset_unlock(pset);
 		splx(s);
 
 		return (KERN_FAILURE);
 	}
 
 	processor->state = PROCESSOR_START;
-	processor_unlock(processor);
+	pset_unlock(pset);
 	splx(s);
 
 	/*
@@ -501,9 +368,9 @@ processor_start(
 		result = idle_thread_create(processor);
 		if (result != KERN_SUCCESS) {
 			s = splsched();
-			processor_lock(processor);
+			pset_lock(pset);
 			processor->state = PROCESSOR_OFF_LINE;
-			processor_unlock(processor);
+			pset_unlock(pset);
 			splx(s);
 
 			return (result);
@@ -520,9 +387,9 @@ processor_start(
 		result = kernel_thread_create((thread_continue_t)processor_start_thread, NULL, MAXPRI_KERNEL, &thread);
 		if (result != KERN_SUCCESS) {
 			s = splsched();
-			processor_lock(processor);
+			pset_lock(pset);
 			processor->state = PROCESSOR_OFF_LINE;
-			processor_unlock(processor);
+			pset_unlock(pset);
 			splx(s);
 
 			return (result);
@@ -545,10 +412,10 @@ processor_start(
 	result = cpu_start(PROCESSOR_DATA(processor, slot_num));
 	if (result != KERN_SUCCESS) {
 		s = splsched();
-		processor_lock(processor);
+		pset_lock(pset);
 		processor->state = PROCESSOR_OFF_LINE;
 		timer_call_shutdown(processor);
-		processor_unlock(processor);
+		pset_unlock(pset);
 		splx(s);
 
 		return (result);
@@ -580,28 +447,6 @@ processor_control(
 
 	return(cpu_control(PROCESSOR_DATA(processor, slot_num), info, count));
 }
-
-/*
- *	Calculate the appropriate timesharing quanta based on set load.
- */
-
-void
-timeshare_quanta_update(
-	processor_set_t		pset)
-{
-	int		pcount = pset->processor_count;
-	int		i = pset->runq.count;
-
-	if (i >= pcount)
-		i = 1;
-	else
-	if (i <= 1)
-		i = pcount;
-	else
-		i = (pcount + (i / 2)) / i;
-
-	pset->timeshare_quanta = i;
-}
 	    
 kern_return_t
 processor_set_create(
@@ -624,14 +469,14 @@ processor_get_assignment(
 	processor_t	processor,
 	processor_set_t	*pset)
 {
-    	int state;
+	int state;
 
 	state = processor->state;
 	if (state == PROCESSOR_SHUTDOWN || state == PROCESSOR_OFF_LINE)
 		return(KERN_FAILURE);
 
-	*pset = processor->processor_set;
-	pset_reference(*pset);
+	*pset = &pset0;
+
 	return(KERN_SUCCESS);
 }
 
@@ -653,7 +498,7 @@ processor_set_info(
 			return(KERN_FAILURE);
 
 		basic_info = (processor_set_basic_info_t) info;
-		basic_info->processor_count = pset->processor_count;
+		basic_info->processor_count = processor_avail_count;
 		basic_info->default_policy = POLICY_TIMESHARE;
 
 		*count = PROCESSOR_SET_BASIC_INFO_COUNT;
@@ -770,29 +615,28 @@ processor_set_statistics(
 	processor_set_info_t    info,
 	mach_msg_type_number_t	*count)
 {
-        if (pset == PROCESSOR_SET_NULL)
-                return (KERN_INVALID_PROCESSOR_SET);
+	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
+		return (KERN_INVALID_PROCESSOR_SET);
+
+	if (flavor == PROCESSOR_SET_LOAD_INFO) {
+		register processor_set_load_info_t     load_info;
 
-        if (flavor == PROCESSOR_SET_LOAD_INFO) {
-                register processor_set_load_info_t     load_info;
+		if (*count < PROCESSOR_SET_LOAD_INFO_COUNT)
+			return(KERN_FAILURE);
 
-                if (*count < PROCESSOR_SET_LOAD_INFO_COUNT)
-                        return(KERN_FAILURE);
+		load_info = (processor_set_load_info_t) info;
 
-                load_info = (processor_set_load_info_t) info;
+		load_info->mach_factor = sched_mach_factor;
+		load_info->load_average = sched_load_average;
 
-                pset_lock(pset);
-                load_info->task_count = pset->task_count;
-                load_info->thread_count = pset->thread_count;
-                load_info->mach_factor = pset->mach_factor;
-                load_info->load_average = pset->load_average;
-                pset_unlock(pset);
+		load_info->task_count = tasks_count;
+		load_info->thread_count = threads_count;
 
-                *count = PROCESSOR_SET_LOAD_INFO_COUNT;
-                return(KERN_SUCCESS);
-        }
+		*count = PROCESSOR_SET_LOAD_INFO_COUNT;
+		return(KERN_SUCCESS);
+	}
 
-        return(KERN_INVALID_ARGUMENT);
+	return(KERN_INVALID_ARGUMENT);
 }
 
 /*
@@ -862,23 +706,19 @@ processor_set_things(
 	vm_size_t size, size_needed;
 	void  *addr;
 
-	if (pset == PROCESSOR_SET_NULL)
+	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
 		return (KERN_INVALID_ARGUMENT);
 
-	size = 0; addr = 0;
+	size = 0;
+	addr = NULL;
 
 	for (;;) {
-		pset_lock(pset);
-		if (!pset->active) {
-			pset_unlock(pset);
-
-			return (KERN_FAILURE);
-		}
+		mutex_lock(&tasks_threads_lock);
 
 		if (type == THING_TASK)
-			maxthings = pset->task_count;
+			maxthings = tasks_count;
 		else
-			maxthings = pset->thread_count;
+			maxthings = threads_count;
 
 		/* do we have the memory we need? */
 
@@ -886,8 +726,8 @@ processor_set_things(
 		if (size_needed <= size)
 			break;
 
-		/* unlock the pset and allocate more memory */
-		pset_unlock(pset);
+		/* unlock and allocate more memory */
+		mutex_unlock(&tasks_threads_lock);
 
 		if (size != 0)
 			kfree(addr, size);
@@ -900,48 +740,53 @@ processor_set_things(
 			return (KERN_RESOURCE_SHORTAGE);
 	}
 
-	/* OK, have memory and the processor_set is locked & active */
+	/* OK, have memory and the list locked */
 
 	actual = 0;
 	switch (type) {
 
-	case THING_TASK:
-	{
-		task_t		task, *tasks = (task_t *)addr;
-
-		for (task = (task_t)queue_first(&pset->tasks);
-				!queue_end(&pset->tasks, (queue_entry_t)task);
-					task = (task_t)queue_next(&task->pset_tasks)) {
-			task_reference_internal(task);
-			tasks[actual++] = task;
+	case THING_TASK: {
+		task_t		task, *task_list = (task_t *)addr;
+
+		for (task = (task_t)queue_first(&tasks);
+						!queue_end(&tasks, (queue_entry_t)task);
+								task = (task_t)queue_next(&task->tasks)) {
+#if defined(SECURE_KERNEL)
+			if (task != kernel_task) {
+#endif
+				task_reference_internal(task);
+				task_list[actual++] = task;
+#if defined(SECURE_KERNEL)
+			}
+#endif
 		}
 
 		break;
 	}
 
-	case THING_THREAD:
-	{
-		thread_t	thread, *threads = (thread_t *)addr;
+	case THING_THREAD: {
+		thread_t	thread, *thread_list = (thread_t *)addr;
 
-		for (i = 0, thread = (thread_t)queue_first(&pset->threads);
-				!queue_end(&pset->threads, (queue_entry_t)thread);
-					thread = (thread_t)queue_next(&thread->pset_threads)) {
+		for (thread = (thread_t)queue_first(&threads);
+						!queue_end(&threads, (queue_entry_t)thread);
+								thread = (thread_t)queue_next(&thread->threads)) {
 			thread_reference_internal(thread);
-			threads[actual++] = thread;
+			thread_list[actual++] = thread;
 		}
 
 		break;
 	}
+
 	}
 		
-	pset_unlock(pset);
+	mutex_unlock(&tasks_threads_lock);
 
 	if (actual < maxthings)
 		size_needed = actual * sizeof (mach_port_t);
 
 	if (actual == 0) {
 		/* no things, so return null pointer and deallocate memory */
-		*thing_list = 0;
+		*thing_list = NULL;
 		*count = 0;
 
 		if (size != 0)
@@ -957,23 +802,22 @@ processor_set_things(
 			if (newaddr == 0) {
 				switch (type) {
 
-				case THING_TASK:
-				{
-					task_t		*tasks = (task_t *)addr;
+				case THING_TASK: {
+					task_t		*task_list = (task_t *)addr;
 
 					for (i = 0; i < actual; i++)
-						task_deallocate(tasks[i]);
+						task_deallocate(task_list[i]);
 					break;
 				}
 
-				case THING_THREAD:
-				{
-					thread_t	*threads = (thread_t *)addr;
+				case THING_THREAD: {
+					thread_t	*thread_list = (thread_t *)addr;
 
 					for (i = 0; i < actual; i++)
-						thread_deallocate(threads[i]);
+						thread_deallocate(thread_list[i]);
 					break;
 				}
+
 				}
 
 				kfree(addr, size);
@@ -992,23 +836,22 @@ processor_set_things(
 
 		switch (type) {
 
-		case THING_TASK:
-		{
-			task_t		*tasks = (task_t *)addr;
+		case THING_TASK: {
+			task_t		*task_list = (task_t *)addr;
 
 			for (i = 0; i < actual; i++)
-				(*thing_list)[i] = convert_task_to_port(tasks[i]);
+				(*thing_list)[i] = convert_task_to_port(task_list[i]);
 			break;
 		}
 
-		case THING_THREAD:
-		{
-			thread_t	*threads = (thread_t *)addr;
+		case THING_THREAD: {
+			thread_t	*thread_list = (thread_t *)addr;
 
 			for (i = 0; i < actual; i++)
-			  	(*thing_list)[i] = convert_thread_to_port(threads[i]);
+			  	(*thing_list)[i] = convert_thread_to_port(thread_list[i]);
 			break;
 		}
+
 		}
 	}
 
@@ -1035,6 +878,16 @@ processor_set_tasks(
  *
  *	List all threads in the processor set.
  */
+#if defined(SECURE_KERNEL)
+kern_return_t
+processor_set_threads(
+	__unused processor_set_t		pset,
+	__unused thread_array_t		*thread_list,
+	__unused mach_msg_type_number_t	*count)
+{
+    return KERN_FAILURE;
+}
+#else
 kern_return_t
 processor_set_threads(
 	processor_set_t		pset,
@@ -1043,40 +896,7 @@ processor_set_threads(
 {
     return(processor_set_things(pset, (mach_port_t **)thread_list, count, THING_THREAD));
 }
-
-/*
- *      processor_set_base:
- *
- *      Specify per-policy base priority for a processor set.  Set processor
- *	set default policy to the given policy. This affects newly created
- *      and assigned threads.  Optionally change existing ones.
- */
-kern_return_t
-processor_set_base(
-	__unused processor_set_t 	pset,
-	__unused policy_t		policy,
-	__unused policy_base_t	base,
-	__unused boolean_t       	change)
-{
-	return (KERN_INVALID_ARGUMENT);
-}
-
-/*
- *      processor_set_limit:
- *
- *      Specify per-policy limits for a processor set.  This affects
- *      newly created and assigned threads.  Optionally change existing
- *      ones.
- */
-kern_return_t
-processor_set_limit(
-	__unused processor_set_t 	pset,
-	__unused policy_t		policy,
-	__unused policy_limit_t    	limit,
-	__unused boolean_t       	change)
-{
-	return (KERN_POLICY_LIMIT);
-}
+#endif
 
 /*
  *	processor_set_policy_control
@@ -1095,3 +915,21 @@ processor_set_policy_control(
 {
 	return (KERN_INVALID_ARGUMENT);
 }
+
+#undef pset_deallocate
+void pset_deallocate(processor_set_t pset);
+void
+pset_deallocate(
+__unused processor_set_t	pset)
+{
+	return;
+}
+
+#undef pset_reference
+void pset_reference(processor_set_t pset);
+void
+pset_reference(
+__unused processor_set_t	pset)
+{
+	return;
+}
diff --git a/osfmk/kern/processor.h b/osfmk/kern/processor.h
index 64c7315dd..50e7ae7f6 100644
--- a/osfmk/kern/processor.h
+++ b/osfmk/kern/processor.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -76,39 +82,40 @@
 #include <machine/ast_types.h>
 
 struct processor_set {
-	queue_head_t		idle_queue;		/* idle processors */
-	int					idle_count;		/* how many ? */
 	queue_head_t		active_queue;	/* active processors */
+	queue_head_t		idle_queue;		/* idle processors */
+	int					idle_count;
 
-	queue_head_t		processors;		/* all processors here */
-	int					processor_count;/* how many ? */
-	decl_simple_lock_data(,sched_lock)	/* lock for runq and above */
-
-	struct	run_queue	runq;			/* runq for this set */
+	processor_t			low_hint;
+	processor_t			high_hint;
 
-	queue_head_t		tasks;			/* tasks assigned */
-	int					task_count;		/* how many */
-	queue_head_t		threads;		/* threads in this set */
-	int					thread_count;	/* how many */
-	int					ref_count;		/* structure ref count */
-	boolean_t			active;			/* is pset in use */
-	decl_mutex_data(,	lock)			/* lock for above */
+	int					processor_count;
 
-	int					timeshare_quanta;	/* timeshare quantum factor */
+	decl_simple_lock_data(,sched_lock)	/* lock for above */
 
 	struct ipc_port	*	pset_self;		/* port for operations */
 	struct ipc_port *	pset_name_self;	/* port for information */
 
-	uint32_t			run_count;		/* threads running in set */
-	uint32_t			share_count;	/* timeshare threads running in set */
+	processor_set_t		pset_list;		/* chain of associated psets */
+	pset_node_t			node;
+};
+
+extern struct processor_set	pset0;
 
-	integer_t			mach_factor;	/* mach_factor */
-	integer_t			load_average;	/* load_average */
+struct pset_node {
+	processor_set_t		psets;			/* list of associated psets */
 
-	uint32_t			pri_shift;		/* timeshare usage -> priority */
+	pset_node_t			nodes;			/* list of associated subnodes */
+	pset_node_t			node_list;		/* chain of associated nodes */
+
+	pset_node_t			parent;
 };
 
-extern struct processor_set	default_pset;
+extern struct pset_node	pset_node0;
+
+extern queue_head_t		tasks, threads;
+extern int				tasks_count, threads_count;
+decl_mutex_data(extern,tasks_threads_lock)
 
 struct processor {
 	queue_chain_t		processor_queue;/* idle/active queue link,
@@ -119,7 +126,7 @@ struct processor {
 						*next_thread,	/* next thread when dispatched */
 						*idle_thread;	/* this processor's idle thread. */
 
-	processor_set_t		processor_set;	/* current membership */
+	processor_set_t		processor_set;	/* assigned set */
 
 	int					current_pri;	/* priority of current thread */
 
@@ -127,14 +134,14 @@ struct processor {
 	uint64_t			quantum_end;	/* time when current quantum ends */
 	uint64_t			last_dispatch;	/* time of last dispatch */
 
-	int					timeslice;		/* quanta before timeslice ends */
 	uint64_t			deadline;		/* current deadline */
+	int					timeslice;		/* quanta before timeslice ends */
 
-	struct run_queue	runq;			/* local runq for this processor */
+	struct run_queue	runq;			/* runq for this processor */
 
-	queue_chain_t		processors;		/* processors in set */
-	decl_simple_lock_data(,lock)
 	struct ipc_port *	processor_self;	/* port for operations */
+	decl_simple_lock_data(,lock)
+
 	processor_t			processor_list;	/* all existing processors */
 	processor_data_t	processor_data;	/* per-processor data */
 };
@@ -146,112 +153,90 @@ decl_simple_lock_data(extern,processor_list_lock)
 extern processor_t	master_processor;
 
 /*
- *	NOTE: The processor->processor_set link is needed in one of the
- *	scheduler's critical paths.  [Figure out where to look for another
- *	thread to run on this processor.]  It is accessed without locking.
- *	The following access protocol controls this field.
- *
- *	Read from own processor - just read.
- *	Read from another processor - lock processor structure during read.
- *	Write from own processor - lock processor structure during write.
- *	Write from another processor - NOT PERMITTED.
- *
+ *	Processor state is accessed by locking the scheduling lock
+ *	for the assigned processor set.
  */
-
-/*
- *	Processor state locking:
- *
- *	Values for the processor state are defined below.  If the processor
- *	is off-line or being shutdown, then it is only necessary to lock
- *	the processor to change its state.  Otherwise it is only necessary
- *	to lock its processor set's sched_lock.  Scheduler code will
- *	typically lock only the sched_lock, but processor manipulation code
- *	will often lock both.
- */
-
 #define PROCESSOR_OFF_LINE		0	/* Not available */
-#define	PROCESSOR_RUNNING		1	/* Normal execution */
-#define	PROCESSOR_IDLE			2	/* Idle */
-#define PROCESSOR_DISPATCHING	3	/* Dispatching (idle -> running) */
-#define PROCESSOR_SHUTDOWN		4	/* Going off-line */
-#define PROCESSOR_START			5	/* Being started */
+#define PROCESSOR_SHUTDOWN		1	/* Going off-line */
+#define PROCESSOR_START			2	/* Being started */
+#define	PROCESSOR_IDLE			3	/* Idle */
+#define PROCESSOR_DISPATCHING	4	/* Dispatching (idle -> running) */
+#define	PROCESSOR_RUNNING		5	/* Normal execution */
 
 extern processor_t	current_processor(void);
 
 extern processor_t	cpu_to_processor(
 						int			cpu);
 
-/* Useful lock macros */
-
-#define	pset_lock(pset)		mutex_lock(&(pset)->lock)
-#define	pset_lock_try(pset)	mutex_try(&(pset)->lock)
-#define pset_unlock(pset)	mutex_unlock(&(pset)->lock)
-
-#define processor_lock(pr)	simple_lock(&(pr)->lock)
-#define processor_unlock(pr)	simple_unlock(&(pr)->lock)
-
-extern void		processor_bootstrap(void);
+/* Lock macros */
+
+#define pset_lock(p)			simple_lock(&(p)->sched_lock)
+#define pset_unlock(p)			simple_unlock(&(p)->sched_lock)
+#define pset_lock_init(p)		simple_lock_init(&(p)->sched_lock, 0)
+
+#define processor_lock(p)		simple_lock(&(p)->lock)
+#define processor_unlock(p)		simple_unlock(&(p)->lock)
+#define processor_lock_init(p)	simple_lock_init(&(p)->lock, 0)
+
+/* Update hints */
+
+#define pset_hint_low(ps, p)	\
+MACRO_BEGIN														\
+	if ((ps)->low_hint != PROCESSOR_NULL) {						\
+		if ((p) != (ps)->low_hint) {							\
+			if ((p)->runq.count < (ps)->low_hint->runq.count)	\
+				(ps)->low_hint = (p);							\
+		}														\
+	}															\
+	else														\
+		(ps)->low_hint = (p);									\
+MACRO_END
+
+#define pset_hint_high(ps, p)	\
+MACRO_BEGIN														\
+	if ((ps)->high_hint != PROCESSOR_NULL) {					\
+		if ((p) != (ps)->high_hint) {							\
+			if ((p)->runq.count > (ps)->high_hint->runq.count)	\
+				(ps)->high_hint = (p);							\
+		}														\
+	}															\
+	else														\
+		(ps)->high_hint = (p);									\
+MACRO_END
+
+extern void		processor_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 extern void		processor_init(
 					processor_t		processor,
-					int				slot_num);
-
-extern void		timeshare_quanta_update(
-					processor_set_t		pset);
-
-extern void		pset_init(
-					processor_set_t		pset);
-
-#define pset_run_incr(pset)					\
-	hw_atomic_add(&(pset)->run_count, 1)
-
-#define pset_run_decr(pset)					\
-	hw_atomic_sub(&(pset)->run_count, 1)
-
-#define pset_share_incr(pset)				\
-	hw_atomic_add(&(pset)->share_count, 1)
-
-#define pset_share_decr(pset)				\
-	hw_atomic_sub(&(pset)->share_count, 1)
+					int				slot_num,
+					processor_set_t	processor_set) __attribute__((section("__TEXT, initcode")));
 
 extern kern_return_t	processor_shutdown(
 							processor_t		processor);
 
-extern void		pset_remove_processor(
-					processor_set_t		pset,
-					processor_t			processor);
+extern void		processor_queue_shutdown(
+					processor_t		processor);
 
-extern void		pset_add_processor(
-					processor_set_t		pset,
-					processor_t			processor);
+extern processor_set_t	processor_pset(
+							processor_t		processor);
 
-extern void		pset_remove_task(
-					processor_set_t		pset,
-					task_t				task);
+extern pset_node_t		pset_node_root(void);
 
-extern void		pset_add_task(
-					processor_set_t		pset,
-					task_t				task);
+extern processor_set_t	pset_create(
+							pset_node_t		node);
 
-extern void		pset_remove_thread(
-					processor_set_t		pset,
-					thread_t			thread);
-
-extern void		pset_add_thread(
+extern void		pset_init(
 					processor_set_t		pset,
-					thread_t			thread);
-
-extern void		thread_change_psets(
-					thread_t			thread,
-					processor_set_t		old_pset,
-					processor_set_t		new_pset);
-
+					pset_node_t			node) __attribute__((section("__TEXT, initcode")));
 
 extern kern_return_t	processor_info_count(
 							processor_flavor_t		flavor,
 							mach_msg_type_number_t	*count);
 
-#endif	/* MACH_KERNEL_PRIVATE */
+#define pset_deallocate(x)
+#define pset_reference(x)
+
+#else	/* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS
 
@@ -263,4 +248,11 @@ extern void		pset_reference(
 
 __END_DECLS
 
+#endif	/* MACH_KERNEL_PRIVATE */
+
+#ifdef	XNU_KERNEL_PRIVATE
+
+extern uint32_t		processor_avail_count;
+
+#endif
 #endif	/* _KERN_PROCESSOR_H_ */
diff --git a/osfmk/kern/processor_data.c b/osfmk/kern/processor_data.c
index 801cf3bdd..41031a8fe 100644
--- a/osfmk/kern/processor_data.c
+++ b/osfmk/kern/processor_data.c
@@ -1,36 +1,38 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Machine independent per processor data.
- *
- * HISTORY
- *
- * 16 October 2003 (debo)
- *	Created.
  */
 
 #include <mach/mach_types.h>
 
 #include <kern/processor.h>
+#include <kern/timer.h>
 
 void
 processor_data_init(
@@ -38,8 +40,9 @@ processor_data_init(
 {
 	(void)memset(&processor->processor_data, 0, sizeof (processor_data_t));
 
+	timer_init(&PROCESSOR_DATA(processor, idle_state));
+	timer_init(&PROCESSOR_DATA(processor, system_state));
+	timer_init(&PROCESSOR_DATA(processor, user_state));
+
 	queue_init(&PROCESSOR_DATA(processor, timer_call_queue));
-#if	!STAT_TIME
-	timer_init(&PROCESSOR_DATA(processor, offline_timer));
-#endif	/* STAT_TIME */
 }
diff --git a/osfmk/kern/processor_data.h b/osfmk/kern/processor_data.h
index 3aa431248..4debe720d 100644
--- a/osfmk/kern/processor_data.h
+++ b/osfmk/kern/processor_data.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Machine independent per processor data.
@@ -37,13 +43,15 @@
 
 struct processor_data {
 	/* Processor state statistics */
-	integer_t				cpu_ticks[CPU_STATE_MAX];
+	timer_data_t			idle_state;
+	timer_data_t			system_state;
+	timer_data_t			user_state;
+
+	timer_t					current_state;
 
-#if !STAT_TIME
-	/* Current execution timer */
-	timer_t					current_timer;
-	timer_data_t			offline_timer;
-#endif	/* STAT_TIME */
+	/* Thread execution timers */
+	timer_t					thread_timer;
+	timer_t					kernel_timer;
 
 	/* Kernel stack cache */
 	struct stack_cache {
@@ -65,6 +73,10 @@ struct processor_data {
 	}						ikm_cache;
 
 	int						slot_num;
+
+	unsigned long			page_grab_count;
+	int						start_color;
+	void					*free_pages;
 };
 
 typedef struct processor_data	processor_data_t;
diff --git a/osfmk/kern/profile.c b/osfmk/kern/profile.c
deleted file mode 100644
index ee8e346b3..000000000
--- a/osfmk/kern/profile.c
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-
-/*
- */
-#include	<mach_prof.h>
-
-#include	<mach/task_server.h>
-#include	<mach/thread_act_server.h>
-
-#if MACH_PROF
-#include	<kern/thread.h>
-#include	<kern/queue.h>
-#include	<kern/profile.h>
-#include	<kern/sched_prim.h>
-#include	<kern/spl.h>
-#include	<kern/misc_protos.h>
-#include	<ipc/ipc_space.h>
-#include        <machine/machparam.h>
-#include	<mach/prof.h>
-
-thread_t profile_thread_id = THREAD_NULL;
-int profile_sample_count = 0;	/* Provided for looking at from kdb. */
-extern kern_return_t task_suspend(task_t task);	/* ack */
-
-/* Forwards */
-prof_data_t	pbuf_alloc(void);
-void		pbuf_free(
-			prof_data_t	pbuf);
-void		profile_thread(void);
-void		send_last_sample_buf(
-			prof_data_t	pbuf);
-
-/*
- *****************************************************************************
- * profile_thread is the profile/trace kernel support thread. It is started
- * by a server/user request through task_sample, or thread_sample. The profile
- * thread dequeues messages and sends them to the receive_prof thread, in the
- * server, via the send_samples and send_notices mig interface functions. If
- * there are no messages in the queue profile thread blocks until wakened by
- * profile (called in from mach_clock), or last_sample (called by thread/task_
- * sample).
-*/
-
-void
-profile_thread(void)
-{
-    spl_t	    s;
-    buffer_t	    buf_entry;
-    queue_entry_t   prof_queue_entry;
-    prof_data_t	    pbuf;
-    kern_return_t   kr;
-    int		    j;
-
-    /* Initialise the queue header for the prof_queue */
-    mpqueue_init(&prof_queue);
-
-    while (TRUE) {
-
-	/* Dequeue the first buffer. */
-	s = splsched();
-	mpdequeue_head(&prof_queue, &prof_queue_entry);
-	splx(s);
-
-	if ((buf_entry = (buffer_t) prof_queue_entry) == NULLPBUF) { 
-	    assert_wait((event_t) profile_thread, THREAD_UNINT);
-	    thread_block(THREAD_CONTINUE_NULL);
-	    if (current_thread()->wait_result != THREAD_AWAKENED)
-		break;
-	} else 
-	{
-	    int		    dropped;
-
-	    pbuf = buf_entry->p_prof;
-	    kr = send_samples(pbuf->prof_port, (void *)buf_entry->p_zone,
-			(mach_msg_type_number_t)buf_entry->p_index);
-	    profile_sample_count += buf_entry->p_index;
-	    if (kr != KERN_SUCCESS)
-	      printf("send_samples(%x, %x, %d) error %x\n",
-			pbuf->prof_port, buf_entry->p_zone, buf_entry->p_index, kr); 
-	    dropped = buf_entry->p_dropped;
-	    if (dropped > 0) {
-		printf("kernel: profile dropped %d sample%s\n", dropped,
-		       dropped == 1 ? "" : "s");
-		buf_entry->p_dropped = 0;
-	    }
-
-	    /* Indicate you've finished the dirty job */
-	    buf_entry->p_full = FALSE;
-	    if (buf_entry->p_wakeme)
-	      thread_wakeup((event_t) &buf_entry->p_wakeme);
-	}
-
-    }
-    /* The profile thread has been signalled to exit.  Any threads waiting
-       for the last buffer of samples to be acknowledged should be woken
-       up now.  */
-    profile_thread_id = THREAD_NULL;
-    while (1) {
-	s = splsched();
-	mpdequeue_head(&prof_queue, &prof_queue_entry);
-	splx(s);
-	if ((buf_entry = (buffer_t) prof_queue_entry) == NULLPBUF)
-	    break;
-	if (buf_entry->p_wakeme)
-	    thread_wakeup((event_t) &buf_entry->p_wakeme);
-    }
-#if 0	/* XXXXX */
-    thread_halt_self();
-#else
-	panic("profile_thread(): halt_self");
-#endif	/* XXXXX */
-}
-
-/*
- *****************************************************************************
- * send_last_sample is the drain mechanism to allow partial profiled buffers
- * to be sent to the receive_prof thread in the server.
- *****************************************************************************
-*/
-
-void
-send_last_sample_buf(prof_data_t pbuf)
-{
-    spl_t    s;
-    buffer_t buf_entry;
-
-    if (pbuf == NULLPROFDATA)
-	return;
-
-    /* Ask for the sending of the last PC buffer.
-     * Make a request to the profile_thread by inserting
-     * the buffer in the send queue, and wake it up. 
-     * The last buffer must be inserted at the head of the
-     * send queue, so the profile_thread handles it immediatly. 
-     */ 
-    buf_entry = pbuf->prof_area + pbuf->prof_index;
-    buf_entry->p_prof = pbuf;
-
-    /* 
-       Watch out in case profile thread exits while we are about to
-       queue data for it.
-     */
-    s = splsched();
-    if (profile_thread_id == THREAD_NULL)
-	splx(s);
-    else {
-	buf_entry->p_wakeme = 1;
-	mpenqueue_tail(&prof_queue, &buf_entry->p_list);
-	thread_wakeup((event_t) profile_thread);
-	assert_wait((event_t) &buf_entry->p_wakeme, THREAD_ABORTSAFE);
-	splx(s); 
-	thread_block(THREAD_CONTINUE_NULL);
-    }
-}
-
-
-/*
- *****************************************************************************
- * add clock tick parameters to profile/trace buffers. Called from the mach_
- * clock heritz_tick function. DCI version stores thread, sp, and pc values
- * into the profile/trace buffers. MACH_PROF version just stores pc values.
- *****************************************************************************
- */
-
-void
-profile(natural_t	pc,
-	prof_data_t	pbuf)
-{
-    natural_t inout_val = pc; 
-    buffer_t buf_entry;
-
-    if (pbuf == NULLPROFDATA)
-	return;
-    
-    /* Inserts the PC value in the buffer of the thread */
-    set_pbuf_value(pbuf, &inout_val); 
-    switch((int)inout_val) {
-    case 0: 
-	if (profile_thread_id == THREAD_NULL) {
-	  reset_pbuf_area(pbuf);
-	}
-	break;
-    case 1: 
-	/* Normal case, value successfully inserted */
-	break;
-    case 2 : 
-	/*
-	 * The value we have just inserted caused the
-	 * buffer to be full, and ready to be sent.
-	 * If profile_thread_id is null, the profile
-	 * thread has been killed.  Since this generally
-	 * happens only when the O/S server task of which
-	 * it is a part is killed, it is not a great loss
-	 * to throw away the data.
-	 */
-	if (profile_thread_id == THREAD_NULL) {
-	  reset_pbuf_area(pbuf);
-	  break;
-	}
-
-	buf_entry = (buffer_t) &pbuf->prof_area[pbuf->prof_index];
-	buf_entry->p_prof = pbuf;
-	mpenqueue_tail(&prof_queue, &buf_entry->p_list);
-	
-	/* Switch to another buffer */
-	reset_pbuf_area(pbuf);
-	
-	/* Wake up the profile thread */
-	if (profile_thread_id != THREAD_NULL)
-	  thread_wakeup((event_t) profile_thread);
-	break;
-      
-      default: 
-	printf("profile : unexpected case\n"); 
-    }
-}
-
-/*
- *****************************************************************************
- * pbuf_alloc creates a profile/trace buffer and assoc. zones for storing
- * profiled items.
- *****************************************************************************
- */
-
-prof_data_t
-pbuf_alloc(void)
-{
-	register prof_data_t pbuf;
-	register int i;
-	register natural_t *zone;
-
-	pbuf = (prof_data_t)kalloc(sizeof(struct prof_data));
-	if (!pbuf)
-		return(NULLPROFDATA);
-	pbuf->prof_port = MACH_PORT_NULL;
-	for (i=0; i< NB_PROF_BUFFER; i++) {
-		zone = (natural_t *)kalloc(SIZE_PROF_BUFFER*sizeof(natural_t));
-		if (!zone) {
-			i--;
-			while (i--)
-				kfree((vm_offset_t)pbuf->prof_area[i].p_zone,
-				      SIZE_PROF_BUFFER*sizeof(natural_t));
-			kfree((vm_offset_t)pbuf, sizeof(struct prof_data));
-			return(NULLPROFDATA);
-		}
-		pbuf->prof_area[i].p_zone = zone;
-		pbuf->prof_area[i].p_full = FALSE;
-	}
-	pbuf->prof_port = MACH_PORT_NULL;
-	return(pbuf);
-}
-
-/*
- *****************************************************************************
- * pbuf_free free memory allocated for storing profile/trace items. Called
- * when a task is no longer profiled/traced. Pbuf_free tears down the memory
- * alloced in pbuf_alloc. It does not check to see if the structures are valid
- * since it is only called by functions in this file.
- *****************************************************************************
- */
-void
-pbuf_free(
-	prof_data_t	pbuf)
-{
-	register int i;
-
-	if (pbuf->prof_port)
-		ipc_port_release_send(pbuf->prof_port);
-	
-	for(i=0; i < NB_PROF_BUFFER ; i++)
-		kfree((vm_offset_t)pbuf->prof_area[i].p_zone,
-		      SIZE_PROF_BUFFER*sizeof(natural_t));
-	kfree((vm_offset_t)pbuf, sizeof(struct prof_data));
-}
-
-#endif /* MACH_PROF */
-
-/*
- *****************************************************************************
- * Thread_sample is used by MACH_PROF to profile a single thread, and is only
- * stub in DCI.
- *****************************************************************************
- */
-
-#if !MACH_PROF
-kern_return_t
-thread_sample(
-	__unused thread_t	thread,
-	__unused ipc_port_t		reply)
-{
-    return KERN_FAILURE;
-}
-#else 
-kern_return_t
-thread_sample(
-	thread_t	thread,
-	ipc_port_t	reply)
-{
-    /* 
-     * This routine is called every time that a new thread has made
-     * a request for the sampling service. We must keep track of the 
-     * correspondance between its identity (thread) and the port
-     * we are going to use as a reply port to send out the samples resulting 
-     * from its execution. 
-     */
-    prof_data_t	    pbuf;
-    vm_offset_t	    vmpbuf;
-
-    if (reply != MACH_PORT_NULL) {
-	if (thread->profiled) 	/* yuck! */
-		return KERN_INVALID_ARGUMENT;
-	/* Start profiling this activation, do the initialization. */
-	pbuf = pbuf_alloc();
-	if ((thread->profil_buffer = pbuf) == NULLPROFDATA) {
-	    printf("thread_sample: cannot allocate pbuf\n");
-	    return KERN_RESOURCE_SHORTAGE;
-	} 
-	else {
-	    if (!set_pbuf_nb(pbuf, NB_PROF_BUFFER-1)) {
-		printf("mach_sample_thread: cannot set pbuf_nb\n");
-		return KERN_FAILURE;
-	    }
-	    reset_pbuf_area(pbuf);
-	}
-	pbuf->prof_port = reply;
-	thread->profiled = TRUE;
-	thread->profiled_own = TRUE;
-	if (profile_thread_id == THREAD_NULL)
-	    profile_thread_id = kernel_thread(kernel_task, profile_thread);
-    } else {
-	if (!thread->profiled)
-		return(KERN_INVALID_ARGUMENT);
-
-	thread->profiled = FALSE;
-	/* do not stop sampling if thread is not profiled by its own */
-
-	if (!thread->profiled_own)
-	    return KERN_SUCCESS;
-	else
-	    thread->profiled_own = FALSE;
-
-	send_last_sample_buf(thread->profil_buffer);
-	pbuf_free(thread->profil_buffer);
-	thread->profil_buffer = NULLPROFDATA;
-    }
-    return KERN_SUCCESS;
-}
-#endif /* MACH_PROF */
-
-/*
- *****************************************************************************
- * Task_sample is used to profile/trace tasks - all thread within a task using
- * a common profile buffer to collect items generated by the hertz_tick. For
- * each task profiled a profile buffer is created that associates a reply port
- * (used to send the data to a server thread), task (used for throttling), and
- * a zone area (used to store profiled/traced items).
- *****************************************************************************
- */
-
-#if !MACH_PROF
-kern_return_t
-task_sample(
-	__unused task_t	task,
-	__unused ipc_port_t	reply)
-{
-    return KERN_FAILURE;
-}
-#else
-kern_return_t
-task_sample(
-	task_t		task,
-	ipc_port_t	reply)
-{
-    prof_data_t	    pbuf=task->profil_buffer;
-    vm_offset_t	    vmpbuf;
-    boolean_t	    turnon = (reply != MACH_PORT_NULL);
-
-    if (task == TASK_NULL)
-	    return KERN_INVALID_ARGUMENT;
-    if (turnon)			/* Do we want to profile this task? */
-    {
-	pbuf = pbuf_alloc();	/* allocate a profile buffer */
-	task_lock(task);
-	if (task->task_profiled) { /* if it is already profiled return so */
-		task_unlock(task);
-		if (pbuf != NULLPROFDATA)
-		    pbuf_free(pbuf);
-		return(KERN_INVALID_ARGUMENT);
-	}
-	if (pbuf == NULLPROFDATA) {
-	    task_unlock(task);
-	    return KERN_RESOURCE_SHORTAGE; /* can't allocate a buffer, quit */
-	}
-	task->profil_buffer = pbuf;
-	
-	if (!set_pbuf_nb(pbuf, NB_PROF_BUFFER-1)) {
-	    pbuf_free(pbuf);
-	    task_unlock(task);
-	    return KERN_FAILURE;
-	}
-	reset_pbuf_area(pbuf);
-	pbuf->prof_port = reply; /* assoc. buffer with reply port */
-    } else {			/* We want to stop profiling/tracing */
-	task_lock(task);
-	if (!task->task_profiled) { /* but this task is not being profiled */
-	    task_unlock(task);
-	    return(KERN_INVALID_ARGUMENT);
-	}
-    }
-
-    /*
-     * turnon = FALSE && task_profile = TRUE ||
-     * turnon = TRUE  && task_profile = FALSE
-     */
-
-    if (turnon != task->task_profiled) {
-	int actual, i;
-	thread_t thread;
-
-	if (turnon && profile_thread_id == THREAD_NULL)	/* 1st time thru? */
-	    profile_thread_id =	/* then start profile thread. */
-		kernel_thread(kernel_task, profile_thread);
-	task->task_profiled = turnon;  
-	actual = task->thread_count; 
-	for (i = 0, thread = (thread_t)queue_first(&task->threads);
-	     i < actual;
-	     i++, thread = (thread_t)queue_next(&thr_act->task_threads)) {
-		  if (!thread->profiled_own) {
-		    threadt->profiled = turnon;
-		    if (turnon) {
-			threadt->profil_buffer = task->profil_buffer;
-		  	thread->profiled = TRUE;
-		    } else {
-			thread->profiled = FALSE;
-		        thread->profil_buffer = NULLPROFDATA;
-		    }
-		  }
-	}
-	if (!turnon) {		/* drain buffers and clean-up */
-	    send_last_sample_buf(task->profil_buffer);
-	    pbuf_free(task->profil_buffer);
-	    task->profil_buffer = NULLPROFDATA;
-	}
-    }
-
-    task_unlock(task);
-    return KERN_SUCCESS;
-}
-#endif /* MACH_PROF */
-
diff --git a/osfmk/kern/profile.h b/osfmk/kern/profile.h
deleted file mode 100644
index 10488e8ef..000000000
--- a/osfmk/kern/profile.h
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- */
-/* 
- * Mach Operating System
- * Copyright (c) 1991,1990,1989 Carnegie Mellon University
- * All Rights Reserved.
- * 
- * Permission to use, copy, modify and distribute this software and its
- * documentation is hereby granted, provided that both the copyright
- * notice and this permission notice appear in all copies of the
- * software, derivative works or modified versions, and any portions
- * thereof, and that both notices appear in supporting documentation.
- * 
- * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
- * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
- * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- * 
- * Carnegie Mellon requests users of this software to return to
- * 
- *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
- *  School of Computer Science
- *  Carnegie Mellon University
- *  Pittsburgh PA 15213-3890
- * 
- * any improvements or extensions that they make and grant Carnegie Mellon
- * the rights to redistribute these changes.
- */
-
-/*
- */
-
-#ifndef _KERN_PROFILE_H
-#define _KERN_PROFILE_H
-
-#include <mach/boolean.h>
-#include <vm/vm_kern.h> 
-
-#define	NB_PROF_BUFFER		4	/* number of buffers servicing a */
-#define	SIZE_PROF_BUFFER	200	/* size of a profil buffer (in natural_t) */
-					/* -> at most 1 packet every 2 secs */
-					/*	profiled thread */
-
-struct	prof_data {
-	struct ipc_port *prof_port;	/* where to send a full buffer */
-
-	struct buffer {
-	    queue_chain_t p_list;
-	    natural_t	*p_zone;	/* points to the actual storage area */
-	    int p_index;		/* next slot to be filled */
-	    int p_dropped;		/* # dropped samples when full */
-	    boolean_t p_full;		/* is the current buffer full ? */ 
-	    struct prof_data *p_prof;	/* base to get prof_port */
-	    char p_wakeme;		/* do wakeup when sent */
-	} prof_area[NB_PROF_BUFFER];
-
-	int		prof_index;	/* index of the buffer structure */
-					/*   currently in use */
-
-};
-
-typedef struct prof_data	*prof_data_t;
-#define NULLPROFDATA ((prof_data_t) 0)
-typedef struct buffer		*buffer_t;
-#define NULLPBUF ((buffer_t) 0)
-
-/* Macros */
-
-#define	set_pbuf_nb(pbuf, nb) \
-         (((nb) >= 0 && (nb) < NB_PROF_BUFFER) \
-	 ? (pbuf)->prof_index = (nb), 1 \
-	 : 0)
-
-
-#define	get_pbuf_nb(pbuf) \
-	(pbuf)->prof_index
-
-
-/* MACRO set_pbuf_value 
-** 
-** enters the value 'val' in the buffer 'pbuf' and returns the following
-** indications:     0: means that a fatal error occurred: the buffer was full
-**                       (it hasn't been sent yet)
-**                  1: means that a value has been inserted successfully
-**		    2: means that we'v just entered the last value causing 
-**			the current buffer to be full.(must switch to 
-** 			another buffer and signal the sender to send it)
-*/ 
-
-#if	MACH_PROF
-
-#define set_pbuf_value(pbuf, val) \
-	 { \
-	  register buffer_t a = &((pbuf)->prof_area[(pbuf)->prof_index]); \
-	  register int i ;\
-	  register boolean_t f = a->p_full; \
-			  \
-	  if (f == TRUE ) {\
-	     a->p_dropped++; \
-             *(val) = 0L; \
-	  } else { \
-	    i = a->p_index++; \
-	    a->p_zone[i] = *(val); \
-	    if (i == SIZE_PROF_BUFFER-1) { \
-               a->p_full = TRUE; \
-               *(val) = 2; \
-            } \
-            else \
-		*(val) = 1; \
-          } \
-	}
-         
-#define	reset_pbuf_area(pbuf) \
-	{ \
-	 int i; \
-	 (pbuf)->prof_index = ((pbuf)->prof_index + 1) % NB_PROF_BUFFER; \
-	 i = (pbuf)->prof_index; \
-	 (pbuf)->prof_area[i].p_index = 0; \
-	 (pbuf)->prof_area[i].p_dropped = 0; \
-	}
-
-#endif	/* MACH_PROF */
-
-/*
-** Global variable: the head of the queue of buffers to send 
-** It is a queue with locks (uses macros from queue.h) and it
-** is shared by hardclock() and the sender_thread() 
-*/
-
-mpqueue_head_t prof_queue; 
-
-extern void	profile(
-			natural_t	pc,         /* program counter */
-			prof_data_t	pbuf);      /* trace/prof data area */
-
-#if MACH_PROF
-
-#define task_prof_init(task) \
-	task->task_profiled = FALSE; \
-     	task->profil_buffer = NULLPROFDATA;
-
-#define thread_prof_init(thread, task) \
-	thread->profiled = task->profiled;	\
-	thread->profil_buffer = task->profil_buffer;
-
-#define task_prof_deallocate(task) \
-	if (task->profil_buffer) \
-		task_sample(task, MACH_PORT_NULL); \
-
-#define thread_prof_deallocate(thread) \
-	if (thread->profiled_own && thread->profil_buffer)  \
-		thread_sample(thread, MACH_PORT_NULL); \
-
-extern kern_return_t thread_sample(thread_t, ipc_port_t);
-extern kern_return_t task_sample(task_t, ipc_port_t);
-
-#else /* !MACH_PROT */
-
-#define task_prof_init(task)
-#define thread_prof_init(thread, task)
-#define task_prof_deallocate(task)
-#define thread_prof_deallocate(thread)
-		
-#endif	/* !MACH_PROF */
-
-#endif	/* _KERN_PROFILE_H */
diff --git a/osfmk/kern/queue.c b/osfmk/kern/queue.c
index 2ada9236d..0de366e14 100644
--- a/osfmk/kern/queue.c
+++ b/osfmk/kern/queue.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/queue.h b/osfmk/kern/queue.h
index 98effaac4..158150719 100644
--- a/osfmk/kern/queue.h
+++ b/osfmk/kern/queue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -482,6 +488,9 @@ MACRO_BEGIN							\
 		(head)->next = __next;				\
 	else							\
 		((type)__prev)->field.next = __next;		\
+								\
+	(elt)->field.next = NULL;				\
+	(elt)->field.prev = NULL;				\
 MACRO_END
 
 /*
@@ -505,6 +514,9 @@ MACRO_BEGIN							\
 	else							\
 		((type)(__next))->field.prev = (head);		\
 	(head)->next = __next;					\
+								\
+	(entry)->field.next = NULL;				\
+	(entry)->field.prev = NULL;				\
 MACRO_END
 
 /*
@@ -528,6 +540,9 @@ MACRO_BEGIN							\
 	else							\
 		((type)(__prev))->field.next = (head);		\
 	(head)->prev = __prev;					\
+								\
+	(entry)->field.next = NULL;				\
+	(entry)->field.prev = NULL;				\
 MACRO_END
 
 /*
diff --git a/osfmk/kern/sched.h b/osfmk/kern/sched.h
index 4c16001bd..2e7018ef9 100644
--- a/osfmk/kern/sched.h
+++ b/osfmk/kern/sched.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -172,13 +178,6 @@ typedef struct run_queue	*run_queue_t;
 
 #define first_timeslice(processor)		((processor)->timeslice > 0)
 
-#define	processor_timeslice_setup(processor, thread)			\
-MACRO_BEGIN														\
-	(processor)->timeslice =									\
-		((thread)->sched_mode & TH_MODE_TIMESHARE)?				\
-			(processor)->processor_set->timeshare_quanta: 1;	\
-MACRO_END
-
 #define thread_quantum_init(thread)							\
 MACRO_BEGIN													\
 	(thread)->current_quantum = 							\
@@ -191,16 +190,18 @@ MACRO_END
 	((thread)->state & TH_SUSP)										||		\
 	(first_timeslice(processor)?											\
 	 ((processor)->runq.highq > (thread)->sched_pri				||			\
-	  (processor)->processor_set->runq.highq > (thread)->sched_pri) :		\
+	  rt_runq.highq > (thread)->sched_pri) :								\
 	 ((processor)->runq.highq >= (thread)->sched_pri			||			\
-	  (processor)->processor_set->runq.highq >= (thread)->sched_pri))	)
+	  rt_runq.highq >= (thread)->sched_pri))								)
+
+extern struct run_queue		rt_runq;
 
 /*
  *	Scheduler routines.
  */
 
 /* Remove thread from its run queue */
-extern run_queue_t	run_queue_remove(
+extern boolean_t	run_queue_remove(
 						thread_t	thread);
 
 /* Handle quantum expiration for an executing thread */
@@ -242,20 +243,31 @@ extern void		compute_stack_target(
  *	to priority.
  */
 extern uint32_t		sched_pri_shift;
-
-/*
- *	Scaling factor for usage
- *	based on load.
- */
+extern uint32_t		sched_fixed_shift;
 extern int8_t		sched_load_shifts[NRQS];
 
 extern int32_t		sched_poll_yield_shift;
 extern uint32_t		sched_safe_duration;
 
+extern uint32_t		sched_run_count, sched_share_count;
+extern uint32_t		sched_load_average, sched_mach_factor;
+
+extern uint32_t		avenrun[3], mach_factor[3];
+
 extern uint64_t		max_unsafe_computation;
 extern uint64_t		max_poll_computation;
 
-extern uint32_t		avenrun[3], mach_factor[3];
+#define sched_run_incr()			\
+	(void)hw_atomic_add(&sched_run_count, 1)
+
+#define sched_run_decr()			\
+	(void)hw_atomic_sub(&sched_run_count, 1)
+
+#define sched_share_incr()			\
+	(void)hw_atomic_add(&sched_share_count, 1)
+
+#define sched_share_decr()			\
+	(void)hw_atomic_sub(&sched_share_count, 1)
 
 /*
  *	thread_timer_delta macro takes care of both thread timers.
diff --git a/osfmk/kern/sched_average.c b/osfmk/kern/sched_average.c
index 61084906d..e1c02d7a0 100644
--- a/osfmk/kern/sched_average.c
+++ b/osfmk/kern/sched_average.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -104,72 +110,62 @@ typedef struct sched_average	*sched_average_t;
 void
 compute_averages(void)
 {
-	register processor_set_t	pset = &default_pset;
-	register int				ncpus;
-	register int				nthreads, nshared;
-	sched_average_t				avg;
-	register uint32_t			factor_now = 0;
-	register uint32_t			average_now = 0;
-	register uint32_t			load_now = 0;
-
-	if ((ncpus = pset->processor_count) > 0) {
-		/*
-		 *	Retrieve counts, ignoring
-		 *	the current thread.
-		 */
-		nthreads = pset->run_count - 1;
-		nshared = pset->share_count;
-
-		/*
-		 *	Load average and mach factor calculations for
-		 *	those which ask about these things.
-		 */
-		average_now = nthreads * LOAD_SCALE;
-
-		if (nthreads > ncpus)
-			factor_now = (ncpus * LOAD_SCALE) / (nthreads + 1);
+	int					ncpus, nthreads, nshared;
+	uint32_t			factor_now, average_now, load_now = 0;
+	sched_average_t		avg;
+
+	/*
+	 *	Retrieve counts, ignoring
+	 *	the current thread.
+	 */
+	ncpus = processor_avail_count;
+	nthreads = sched_run_count - 1;
+	nshared = sched_share_count;
+
+	/*
+	 *	Load average and mach factor calculations for
+	 *	those which ask about these things.
+	 */
+	average_now = nthreads * LOAD_SCALE;
+
+	if (nthreads > ncpus)
+		factor_now = (ncpus * LOAD_SCALE) / (nthreads + 1);
+	else
+		factor_now = (ncpus - nthreads) * LOAD_SCALE;
+
+	sched_mach_factor =	((sched_mach_factor << 2) + factor_now) / 5;
+	sched_load_average = ((sched_load_average << 2) + average_now) / 5;
+
+	/*
+	 *	Compute the timeshare priority
+	 *	conversion factor based on loading.
+	 */
+	if (nshared > nthreads)
+		nshared = nthreads;
+
+	if (nshared > ncpus) {
+		if (ncpus > 1)
+			load_now = nshared / ncpus;
 		else
-			factor_now = (ncpus - nthreads) * LOAD_SCALE;
-
-		pset->mach_factor =	((pset->mach_factor << 2) + factor_now) / 5;
-		pset->load_average = ((pset->load_average << 2) + average_now) / 5;
-
-		/*
-		 *	Compute the timeshare priority
-		 *	conversion factor based on loading.
-		 */
-		if (nshared > nthreads)
-			nshared = nthreads;
-
-		if (nshared > ncpus) {
-			if (ncpus > 1)
-				load_now = nshared / ncpus;
-			else
-				load_now = nshared;
-
-			if (load_now > NRQS - 1)
-				load_now = NRQS - 1;
-		}
+			load_now = nshared;
 
-		/*
-		 *	The conversion factor consists of
-		 *	two components: a fixed value based
-		 *	on the absolute time unit, and a
-		 *	dynamic portion based on loading.
-		 *
-		 *	Zero loading results in a out of range
-		 *	shift count.  Accumulated usage is ignored
-		 *	during conversion and new usage deltas
-		 *	are discarded.
-		 */
-		pset->pri_shift = sched_pri_shift - sched_load_shifts[load_now];
-	}
-	else {
-		pset->mach_factor = pset->load_average = 0;
-		pset->pri_shift = INT8_MAX;
-		nthreads = pset->run_count;
+		if (load_now > NRQS - 1)
+			load_now = NRQS - 1;
 	}
 
+	/*
+	 *	The conversion factor consists of
+	 *	two components: a fixed value based
+	 *	on the absolute time unit, and a
+	 *	dynamic portion based on loading.
+	 *
+	 *	Zero loading results in a out of range
+	 *	shift count.  Accumulated usage is ignored
+	 *	during conversion and new usage deltas
+	 *	are discarded.
+	 */
+	sched_pri_shift = sched_fixed_shift - sched_load_shifts[load_now];
+
 	/*
 	 *	Sample total running threads.
 	 */
diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c
index 56bb97555..0cbde484c 100644
--- a/osfmk/kern/sched_prim.c
+++ b/osfmk/kern/sched_prim.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -99,6 +105,10 @@
 
 #include <kern/pms.h>
 
+struct run_queue	rt_runq;
+#define RT_RUNQ		((processor_t)-1)
+decl_simple_lock_data(static,rt_lock);
+
 #define		DEFAULT_PREEMPTION_RATE		100		/* (1/s) */
 int			default_preemption_rate = DEFAULT_PREEMPTION_RATE;
 
@@ -125,27 +135,45 @@ uint32_t	min_rt_quantum;
 
 uint32_t	sched_cswtime;
 
-static uint32_t		delay_idle_limit, delay_idle_spin;
-static processor_t	delay_idle(
-						processor_t		processor,
-						thread_t		self);
-
 unsigned	sched_tick;
 uint32_t	sched_tick_interval;
 
-uint32_t	sched_pri_shift;
+uint32_t	sched_pri_shift = INT8_MAX;
+uint32_t	sched_fixed_shift;
+
+uint32_t	sched_run_count, sched_share_count;
+uint32_t	sched_load_average, sched_mach_factor;
+
+void		(*pm_tick_callout)(void)	= NULL;
 
 /* Forwards */
-void		wait_queues_init(void);
+void wait_queues_init(void) __attribute__((section("__TEXT, initcode")));
+
+static void load_shift_init(void) __attribute__((section("__TEXT, initcode")));
+
+static thread_t	thread_select_idle(
+					thread_t			thread,
+					processor_t			processor);
 
-static void		load_shift_init(void);
+static thread_t	processor_idle(
+					thread_t			thread,
+					processor_t			processor);
 
 static thread_t	choose_thread(
-					processor_set_t		pset,
+					processor_t			processor);
+
+static thread_t	steal_thread(
 					processor_t			processor);
 
 static void		thread_update_scan(void);
 
+#if	DEBUG
+extern int debug_task;
+#define TLOG(a, fmt, args...) if(debug_task & a) kprintf(fmt, ## args)
+#else
+#define TLOG(a, fmt, args...) do {} while (0)
+#endif
+
 #if	DEBUG
 static
 boolean_t	thread_runnable(
@@ -154,6 +182,7 @@ boolean_t	thread_runnable(
 #endif	/*DEBUG*/
 
 
+
 /*
  *	State machine
  *
@@ -233,7 +262,8 @@ sched_init(void)
 
 	wait_queues_init();
 	load_shift_init();
-	pset_init(&default_pset);
+	simple_lock_init(&rt_lock, 0);
+	run_queue_init(&rt_runq);
 	sched_tick = 0;
 	ast_init();
 }
@@ -279,19 +309,10 @@ sched_timebase_init(void)
 	abstime = (abstime * 5) / 3;
 	for (shift = 0; abstime > BASEPRI_DEFAULT; ++shift)
 		abstime >>= 1;
-	sched_pri_shift = shift;
+	sched_fixed_shift = shift;
 
 	max_unsafe_computation = max_unsafe_quanta * std_quantum;
 	max_poll_computation = max_poll_quanta * std_quantum;
-
-	/* delay idle constant(s) (60, 1 us) */
-	clock_interval_to_absolutetime_interval(60, NSEC_PER_USEC, &abstime);
-	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	delay_idle_limit = abstime;
-
-	clock_interval_to_absolutetime_interval(1, NSEC_PER_USEC, &abstime);
-	assert((abstime >> 32) == 0 && (uint32_t)abstime != 0);
-	delay_idle_spin = abstime;
 }
 
 void
@@ -425,12 +446,12 @@ thread_unblock(
 	boolean_t		result = FALSE;
 
 	/*
-	 * Set wait_result.
+	 *	Set wait_result.
 	 */
 	thread->wait_result = wresult;
 
 	/*
-	 * Cancel pending wait timer.
+	 *	Cancel pending wait timer.
 	 */
 	if (thread->wait_timer_is_set) {
 		if (timer_call_cancel(&thread->wait_timer))
@@ -439,28 +460,36 @@ thread_unblock(
 	}
 
 	/*
-	 * Update scheduling state.
+	 *	Update scheduling state: not waiting,
+	 *	set running.
 	 */
 	thread->state &= ~(TH_WAIT|TH_UNINT);
 
 	if (!(thread->state & TH_RUN)) {
 		thread->state |= TH_RUN;
 
-		/*
-		 * Mark unblocked if call out.
-		 */
-		if (thread->options & TH_OPT_CALLOUT)
-			call_thread_unblock();
+		(*thread->sched_call)(SCHED_CALL_UNBLOCK, thread);
 
 		/*
-		 * Update pset run counts.
+		 *	Update run counts.
 		 */
-		pset_run_incr(thread->processor_set);
+		sched_run_incr();
 		if (thread->sched_mode & TH_MODE_TIMESHARE)
-			pset_share_incr(thread->processor_set);
+			sched_share_incr();
 	}
-	else
+	else {
+		/*
+		 *	Signal if idling on another processor.
+		 */
+		if (thread->state & TH_IDLE) {
+			processor_t		processor = thread->last_processor;
+
+			if (processor != current_processor())
+				machine_signal_idle(processor);
+		}
+
 		result = TRUE;
+	}
 
 	/*
 	 * Calculate deadline for real-time threads.
@@ -543,16 +572,16 @@ thread_mark_wait_locked(
 	at_safe_point = (interruptible == THREAD_ABORTSAFE);
 
 	if (	interruptible == THREAD_UNINT			||
-			!(thread->state & TH_ABORT)				||
+			!(thread->sched_mode & TH_MODE_ABORT)	||
 			(!at_safe_point &&
-			 (thread->state & TH_ABORT_SAFELY))		) {
+				(thread->sched_mode & TH_MODE_ABORTSAFELY))) {
 		thread->state |= (interruptible) ? TH_WAIT : (TH_WAIT | TH_UNINT);
 		thread->at_safe_point = at_safe_point;
 		return (thread->wait_result = THREAD_WAITING);
 	}
 	else
-	if (thread->state & TH_ABORT_SAFELY)
-		thread->state &= ~(TH_ABORT|TH_ABORT_SAFELY);
+	if (thread->sched_mode & TH_MODE_ABORTSAFELY)
+		thread->sched_mode &= ~TH_MODE_ISABORTED;
 
 	return (thread->wait_result = THREAD_INTERRUPTED);
 }
@@ -832,13 +861,15 @@ thread_stop(
 	thread_t		thread)
 {
 	wait_result_t	wresult;
-	spl_t			s;
+	spl_t			s = splsched();
 
-	s = splsched();
 	wake_lock(thread);
+	thread_lock(thread);
 
 	while (thread->state & TH_SUSP) {
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_ABORTSAFE);
 		wake_unlock(thread);
 		splx(s);
@@ -851,21 +882,20 @@ thread_stop(
 
 		s = splsched();
 		wake_lock(thread);
+		thread_lock(thread);
 	}
 
-	thread_lock(thread);
 	thread->state |= TH_SUSP;
 
 	while (thread->state & TH_RUN) {
 		processor_t		processor = thread->last_processor;
 
-		if (	processor != PROCESSOR_NULL					&&
-				processor->state == PROCESSOR_RUNNING		&&
-				processor->active_thread == thread			)
+		if (processor != PROCESSOR_NULL && processor->active_thread == thread)
 			cause_ast_check(processor);
-		thread_unlock(thread);
 
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_ABORTSAFE);
 		wake_unlock(thread);
 		splx(s);
@@ -920,10 +950,11 @@ thread_unstop(
 		if (thread->wake_active) {
 			thread->wake_active = FALSE;
 			thread_unlock(thread);
+
+			thread_wakeup(&thread->wake_active);
 			wake_unlock(thread);
 			splx(s);
 
-			thread_wakeup(&thread->wake_active);
 			return;
 		}
 	}
@@ -952,13 +983,12 @@ thread_wait(
 	while (thread->state & TH_RUN) {
 		processor_t		processor = thread->last_processor;
 
-		if (	processor != PROCESSOR_NULL					&&
-				processor->state == PROCESSOR_RUNNING		&&
-				processor->active_thread == thread			)
+		if (processor != PROCESSOR_NULL && processor->active_thread == thread)
 			cause_ast_check(processor);
-		thread_unlock(thread);
 
 		thread->wake_active = TRUE;
+		thread_unlock(thread);
+
 		wresult = assert_wait(&thread->wake_active, THREAD_UNINT);
 		wake_unlock(thread);
 		splx(s);
@@ -1024,7 +1054,7 @@ clear_wait_internal(
 		return (thread_go(thread, wresult));
 	} while (--i > 0);
 
-	panic("clear_wait_internal: deadlock: thread=0x%x, wq=0x%x, cpu=%d\n",
+	panic("clear_wait_internal: deadlock: thread=%p, wq=%p, cpu=%d\n",
 		  thread, wq, cpu_number());
 
 	return (KERN_FAILURE);
@@ -1085,7 +1115,7 @@ thread_wakeup_prim(
 /*
  *	thread_bind:
  *
- *	Force a thread to execute on the specified processor.
+ *	Force the current thread to execute on the specified processor.
  *
  *	Returns the previous binding.  PROCESSOR_NULL means
  *	not bound.
@@ -1094,148 +1124,246 @@ thread_wakeup_prim(
  */
 processor_t
 thread_bind(
-	register thread_t	thread,
-	processor_t			processor)
+	processor_t		processor)
 {
+	thread_t		self = current_thread();
 	processor_t		prev;
-	run_queue_t		runq = RUN_QUEUE_NULL;
 	spl_t			s;
 
 	s = splsched();
-	thread_lock(thread);
-	prev = thread->bound_processor;
-	if (prev != PROCESSOR_NULL)
-		runq = run_queue_remove(thread);
+	thread_lock(self);
 
-	thread->bound_processor = processor;
+	prev = self->bound_processor;
+	self->bound_processor = processor;
 
-	if (runq != RUN_QUEUE_NULL)
-		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-	thread_unlock(thread);
+	thread_unlock(self);
 	splx(s);
 
 	return (prev);
 }
 
-struct {
-	uint32_t	idle_pset_last,
-				idle_pset_any,
-				idle_bound;
-
-	uint32_t	pset_self,
-				pset_last,
-				pset_other,
-				bound_self,
-				bound_other;
-
-	uint32_t	realtime_self,
-				realtime_last,
-				realtime_other;
-
-	uint32_t	missed_realtime,
-				missed_other;
-} dispatch_counts;
-
 /*
- *	Select a thread for the current processor to run.
+ *	thread_select:
+ *
+ *	Select a new thread for the current processor to execute.
  *
  *	May select the current thread, which must be locked.
  */
-thread_t
+static thread_t
 thread_select(
-	register processor_t	processor)
+	thread_t			thread,
+	processor_t			processor)
 {
-	register thread_t		thread;
-	processor_set_t			pset;
-	boolean_t				other_runnable;
+	processor_set_t		pset = processor->processor_set;
+	thread_t			new_thread;
+	boolean_t			other_runnable;
 
-	/*
-	 *	Check for other non-idle runnable threads.
-	 */
-	pset = processor->processor_set;
-	thread = processor->active_thread;
+	do {
+		/*
+		 *	Update the priority.
+		 */
+		if (thread->sched_stamp != sched_tick)
+			update_priority(thread);
 
-	/* Update the thread's priority */
-	if (thread->sched_stamp != sched_tick)
-		update_priority(thread);
+		processor->current_pri = thread->sched_pri;
 
-	processor->current_pri = thread->sched_pri;
+		pset_lock(pset);
+
+		simple_lock(&rt_lock);
+
+		/*
+		 *	Check for other runnable threads.
+		 */
+		other_runnable = processor->runq.count > 0 || rt_runq.count > 0;
 
-	simple_lock(&pset->sched_lock);
-
-	other_runnable = processor->runq.count > 0 || pset->runq.count > 0;
-
-	if (	thread->state == TH_RUN							&&
-			thread->processor_set == pset					&&
-			(thread->bound_processor == PROCESSOR_NULL	||
-			 thread->bound_processor == processor)				) {
-		if (	thread->sched_pri >= BASEPRI_RTQUEUES	&&
-						first_timeslice(processor)			) {
-			if (pset->runq.highq >= BASEPRI_RTQUEUES) {
-				register run_queue_t	runq = &pset->runq;
-				register queue_t		q;
-
-				q = runq->queues + runq->highq;
-				if (((thread_t)q->next)->realtime.deadline <
-												processor->deadline) {
-					thread = (thread_t)q->next;
-					((queue_entry_t)thread)->next->prev = q;
-					q->next = ((queue_entry_t)thread)->next;
-					thread->runq = RUN_QUEUE_NULL;
-					assert(thread->sched_mode & TH_MODE_PREEMPT);
-					runq->count--; runq->urgency--;
-					if (queue_empty(q)) {
-						if (runq->highq != IDLEPRI)
-							clrbit(MAXPRI - runq->highq, runq->bitmap);
-						runq->highq = MAXPRI - ffsbit(runq->bitmap);
+		/*
+		 *	Test to see if the current thread should continue
+		 *	to run on this processor.  Must be runnable, and not
+		 *	bound to a different processor, nor be in the wrong
+		 *	processor set.
+		 */
+		if (	thread->state == TH_RUN							&&
+				(thread->bound_processor == PROCESSOR_NULL	||
+				 thread->bound_processor == processor)			&&
+				(thread->affinity_set == AFFINITY_SET_NULL	||
+				 thread->affinity_set->aset_pset == pset)			) {
+			if (	thread->sched_pri >= BASEPRI_RTQUEUES	&&
+						first_timeslice(processor)				) {
+				if (rt_runq.highq >= BASEPRI_RTQUEUES) {
+					register run_queue_t	runq = &rt_runq;
+					register queue_t		q;
+
+					q = runq->queues + runq->highq;
+					if (((thread_t)q->next)->realtime.deadline <
+													processor->deadline) {
+						thread = (thread_t)q->next;
+						((queue_entry_t)thread)->next->prev = q;
+						q->next = ((queue_entry_t)thread)->next;
+						thread->runq = PROCESSOR_NULL;
+						assert(thread->sched_mode & TH_MODE_PREEMPT);
+						runq->count--; runq->urgency--;
+						if (queue_empty(q)) {
+							if (runq->highq != IDLEPRI)
+								clrbit(MAXPRI - runq->highq, runq->bitmap);
+							runq->highq = MAXPRI - ffsbit(runq->bitmap);
+						}
 					}
 				}
+
+				simple_unlock(&rt_lock);
+
+				processor->deadline = thread->realtime.deadline;
+
+				pset_unlock(pset);
+
+				return (thread);
 			}
 
-			processor->deadline = thread->realtime.deadline;
+			if (	(!other_runnable							||
+					 (processor->runq.highq < thread->sched_pri		&&
+					  rt_runq.highq < thread->sched_pri))				) {
 
-			simple_unlock(&pset->sched_lock);
+				simple_unlock(&rt_lock);
 
-			return (thread);
-		}
+				/* I am the highest priority runnable (non-idle) thread */
 
-		if (	(!other_runnable							||
-				 (processor->runq.highq < thread->sched_pri		&&
-				  pset->runq.highq < thread->sched_pri))			) {
+				pset_hint_low(pset, processor);
+				pset_hint_high(pset, processor);
 
-			/* I am the highest priority runnable (non-idle) thread */
+				processor->deadline = UINT64_MAX;
 
-			processor->deadline = UINT64_MAX;
+				pset_unlock(pset);
 
-			simple_unlock(&pset->sched_lock);
+				return (thread);
+			}
+		}
+
+		if (other_runnable)
+			return choose_thread(processor);
+
+		simple_unlock(&rt_lock);
 
-			return (thread);
+		/*
+		 *	No runnable threads, attempt to steal
+		 *	from other processors.
+		 */
+		if (pset->high_hint != PROCESSOR_NULL && pset->high_hint->runq.count > 0) {
+			new_thread = steal_thread(pset->high_hint);
+			if (new_thread != THREAD_NULL) {
+				pset_unlock(pset);
+
+				return (new_thread);
+			}
 		}
-	}
 
-	if (other_runnable)
-		thread = choose_thread(pset, processor);
-	else {
 		/*
 		 *	Nothing is runnable, so set this processor idle if it
-		 *	was running.  Return its idle thread.
+		 *	was running.
 		 */
 		if (processor->state == PROCESSOR_RUNNING) {
 			remqueue(&pset->active_queue, (queue_entry_t)processor);
 			processor->state = PROCESSOR_IDLE;
 
-			enqueue_tail(&pset->idle_queue, (queue_entry_t)processor);
+			enqueue_head(&pset->idle_queue, (queue_entry_t)processor);
+			pset->low_hint = processor;
 			pset->idle_count++;
 		}
 
 		processor->deadline = UINT64_MAX;
 
-		thread = processor->idle_thread;
+		pset_unlock(pset);
+
+		/*
+		 *	Choose idle thread if fast idle is not possible.
+		 */
+		if ((thread->state & (TH_IDLE|TH_TERMINATE|TH_SUSP)) || !(thread->state & TH_WAIT) || thread->wake_active)
+			return (processor->idle_thread);
+
+		/*
+		 *	Perform idling activities directly without a
+		 *	context switch.  Return dispatched thread,
+		 *	else check again for a runnable thread.
+		 */
+		new_thread = thread_select_idle(thread, processor);
+
+	} while (new_thread == THREAD_NULL);
+
+	return (new_thread);
+}
+
+/*
+ *	thread_select_idle:
+ *
+ *	Idle the processor using the current thread context.
+ *
+ *	Called with thread locked, then dropped and relocked.
+ */
+static thread_t
+thread_select_idle(
+	thread_t		thread,
+	processor_t		processor)
+{
+	thread_t		new_thread;
+
+	if (thread->sched_mode & TH_MODE_TIMESHARE)
+		sched_share_decr();
+	sched_run_decr();
+
+	thread->state |= TH_IDLE;
+	processor->current_pri = IDLEPRI;
+
+	thread_unlock(thread);
+
+	/*
+	 *	Switch execution timing to processor idle thread.
+	 */
+	processor->last_dispatch = mach_absolute_time();
+	thread_timer_event(processor->last_dispatch, &processor->idle_thread->system_timer);
+	PROCESSOR_DATA(processor, kernel_timer) = &processor->idle_thread->system_timer;
+
+	/*
+	 *	Cancel the quantum timer while idling.
+	 */
+	timer_call_cancel(&processor->quantum_timer);
+	processor->timeslice = 0;
+
+	(*thread->sched_call)(SCHED_CALL_BLOCK, thread);
+
+	/*
+	 *	Enable interrupts and perform idling activities.  No
+	 *	preemption due to TH_IDLE being set.
+	 */
+	spllo(); new_thread = processor_idle(thread, processor);
+
+	(*thread->sched_call)(SCHED_CALL_UNBLOCK, thread);
+
+	thread_lock(thread);
+
+	/*
+	 *	If awakened, switch to thread timer and start a new quantum.
+	 *	Otherwise skip; we will context switch to another thread or return here.
+	 */
+	if (!(thread->state & TH_WAIT)) {
+		processor->last_dispatch = mach_absolute_time();
+		thread_timer_event(processor->last_dispatch, &thread->system_timer);
+		PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+
+		thread_quantum_init(thread);
+
+		processor->quantum_end = processor->last_dispatch + thread->current_quantum;
+		timer_call_enter1(&processor->quantum_timer, thread, processor->quantum_end);
+		processor->timeslice = 1;
+
+		thread->computation_epoch = processor->last_dispatch;
 	}
 
-	simple_unlock(&pset->sched_lock);
+	thread->state &= ~TH_IDLE;
 
-	return (thread);
+	sched_run_incr();
+	if (thread->sched_mode & TH_MODE_TIMESHARE)
+		sched_share_incr();
+
+	return (new_thread);
 }
 
 /*
@@ -1272,124 +1400,125 @@ MACRO_BEGIN													\
 	}														\
 MACRO_END
 
-boolean_t
+static boolean_t
 thread_invoke(
-	register thread_t	old_thread,
-	register thread_t	new_thread,
+	register thread_t	self,
+	register thread_t	thread,
 	ast_t				reason)
 {
-	thread_continue_t	new_cont, continuation = old_thread->continuation;
-	void				*new_param, *parameter = old_thread->parameter;
+	thread_continue_t	continuation = self->continuation;
+	void				*parameter = self->parameter;
 	processor_t			processor;
-	thread_t			prev_thread;
 
 	if (get_preemption_level() != 0)
 		panic("thread_invoke: preemption_level %d\n",
-								get_preemption_level());
+				get_preemption_level());
 
-	assert(old_thread == current_thread());
+	assert(self == current_thread());
 
 	/*
 	 * Mark thread interruptible.
 	 */
-	thread_lock(new_thread);
-	new_thread->state &= ~TH_UNINT;
+	thread_lock(thread);
+	thread->state &= ~TH_UNINT;
 
-	assert(thread_runnable(new_thread));
+#if DEBUG
+	assert(thread_runnable(thread));
+#endif
 
 	/*
 	 * Allow time constraint threads to hang onto
 	 * a stack.
 	 */
-	if (	(old_thread->sched_mode & TH_MODE_REALTIME)		&&
-					!old_thread->reserved_stack				) {
-		old_thread->reserved_stack = old_thread->kernel_stack;
-	}
+	if ((self->sched_mode & TH_MODE_REALTIME) && !self->reserved_stack)
+		self->reserved_stack = self->kernel_stack;
 
 	if (continuation != NULL) {
-		if (!new_thread->kernel_stack) {
+		if (!thread->kernel_stack) {
 			/*
-			 * If the old thread is using a privileged stack,
+			 * If we are using a privileged stack,
 			 * check to see whether we can exchange it with
-			 * that of the new thread.
+			 * that of the other thread.
 			 */
-			if (	old_thread->kernel_stack == old_thread->reserved_stack	&&
-							!new_thread->reserved_stack)
+			if (self->kernel_stack == self->reserved_stack && !thread->reserved_stack)
 				goto need_stack;
 
 			/*
 			 * Context switch by performing a stack handoff.
 			 */
-			new_cont = new_thread->continuation;
-			new_thread->continuation = NULL;
-			new_param = new_thread->parameter;
-			new_thread->parameter = NULL;
+			continuation = thread->continuation;
+			parameter = thread->parameter;
 
 			processor = current_processor();
-			processor->active_thread = new_thread;
-			processor->current_pri = new_thread->sched_pri;
-			new_thread->last_processor = processor;
-			ast_context(new_thread);
-			thread_unlock(new_thread);
-		
-			current_task()->csw++;
+			processor->active_thread = thread;
+			processor->current_pri = thread->sched_pri;
+			if (thread->last_processor != processor && thread->last_processor != NULL) {
+				if (thread->last_processor->processor_set != processor->processor_set)
+					thread->ps_switch++;
+				thread->p_switch++;
+			}
+			thread->last_processor = processor;
+			thread->c_switch++;
+			ast_context(thread);
+			thread_unlock(thread);
 
-			old_thread->reason = reason;
+			self->reason = reason;
 
 			processor->last_dispatch = mach_absolute_time();
-			timer_event((uint32_t)processor->last_dispatch,
-										&new_thread->system_timer);
-	   
-			thread_done(old_thread, new_thread, processor);
+			thread_timer_event(processor->last_dispatch, &thread->system_timer);
+			PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+	
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_HANDOFF)|DBG_FUNC_NONE,
+										self->reason, (int)thread, self->sched_pri, thread->sched_pri, 0);
 
-			machine_stack_handoff(old_thread, new_thread);
+TLOG(1, "thread_invoke: calling machine_stack_handoff\n");
+			machine_stack_handoff(self, thread);
 
-			thread_begin(new_thread, processor);
+			thread_dispatch(self, thread);
 
-			/*
-			 * Now dispatch the old thread.
-			 */
-			thread_dispatch(old_thread);
+			thread->continuation = thread->parameter = NULL;
 
-			counter_always(c_thread_invoke_hits++);
+			counter(c_thread_invoke_hits++);
 
-			funnel_refunnel_check(new_thread, 2);
+			funnel_refunnel_check(thread, 2);
 			(void) spllo();
 
-			assert(new_cont);
-			call_continuation(new_cont, new_param, new_thread->wait_result);
+			assert(continuation);
+			call_continuation(continuation, parameter, thread->wait_result);
 			/*NOTREACHED*/
 		}
-		else
-		if (new_thread == old_thread) {
+		else if (thread == self) {
 			/* same thread but with continuation */
+			ast_context(self);
 			counter(++c_thread_invoke_same);
-			thread_unlock(new_thread);
+			thread_unlock(self);
 
-			funnel_refunnel_check(new_thread, 3);
+			self->continuation = self->parameter = NULL;
+
+			funnel_refunnel_check(self, 3);
 			(void) spllo();
 
-			call_continuation(continuation, parameter, new_thread->wait_result);
+			call_continuation(continuation, parameter, self->wait_result);
 			/*NOTREACHED*/
 		}
 	}
 	else {
 		/*
-		 * Check that the new thread has a stack
+		 * Check that the other thread has a stack
 		 */
-		if (!new_thread->kernel_stack) {
+		if (!thread->kernel_stack) {
 need_stack:
-			if (!stack_alloc_try(new_thread)) {
-				counter_always(c_thread_invoke_misses++);
-				thread_unlock(new_thread);
-				thread_stack_enqueue(new_thread);
+			if (!stack_alloc_try(thread)) {
+				counter(c_thread_invoke_misses++);
+				thread_unlock(thread);
+				thread_stack_enqueue(thread);
 				return (FALSE);
 			}
 		}
-		else
-		if (new_thread == old_thread) {
+		else if (thread == self) {
+			ast_context(self);
 			counter(++c_thread_invoke_same);
-			thread_unlock(new_thread);
+			thread_unlock(self);
 			return (TRUE);
 		}
 	}
@@ -1398,47 +1527,50 @@ need_stack:
 	 * Context switch by full context save.
 	 */
 	processor = current_processor();
-	processor->active_thread = new_thread;
-	processor->current_pri = new_thread->sched_pri;
-	new_thread->last_processor = processor;
-	ast_context(new_thread);
-	assert(thread_runnable(new_thread));
-	thread_unlock(new_thread);
+	processor->active_thread = thread;
+	processor->current_pri = thread->sched_pri;
+	if (thread->last_processor != processor && thread->last_processor != NULL) {
+		if (thread->last_processor->processor_set != processor->processor_set)
+			thread->ps_switch++;
+		thread->p_switch++;
+	}
+	thread->last_processor = processor;
+	thread->c_switch++;
+	ast_context(thread);
+	thread_unlock(thread);
 
-	counter_always(c_thread_invoke_csw++);
-	current_task()->csw++;
+	counter(c_thread_invoke_csw++);
 
-	assert(old_thread->runq == RUN_QUEUE_NULL);
-	old_thread->reason = reason;
+	assert(self->runq == PROCESSOR_NULL);
+	self->reason = reason;
 
 	processor->last_dispatch = mach_absolute_time();
-	timer_event((uint32_t)processor->last_dispatch, &new_thread->system_timer);
+	thread_timer_event(processor->last_dispatch, &thread->system_timer);
+	PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
 
-	thread_done(old_thread, new_thread, processor);
+	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
+							(int)self->reason, (int)thread, self->sched_pri, thread->sched_pri, 0);
 
 	/*
 	 * This is where we actually switch register context,
-	 * and address space if required.  Control will not
-	 * return here immediately.
-	 */
-	prev_thread = machine_switch_context(old_thread, continuation, new_thread);
-
-	/*
-	 * We are still old_thread, possibly on a different processor,
-	 * and new_thread is now stale.
+	 * and address space if required.  We will next run
+	 * as a result of a subsequent context switch.
 	 */
-	thread_begin(old_thread, old_thread->last_processor);
+	thread = machine_switch_context(self, continuation, thread);
+TLOG(1,"thread_invoke: returning machine_switch_context: self %p continuation %p thread %p\n", self, continuation, thread);
 
 	/*
-	 * Now dispatch the thread which resumed us.
+	 * We have been resumed and are set to run.
 	 */
-	thread_dispatch(prev_thread);
+	thread_dispatch(thread, self);
 
 	if (continuation) {
-		funnel_refunnel_check(old_thread, 3);
+		self->continuation = self->parameter = NULL;
+
+		funnel_refunnel_check(self, 3);
 		(void) spllo();
 
-		call_continuation(continuation, parameter, old_thread->wait_result);
+		call_continuation(continuation, parameter, self->wait_result);
 		/*NOTREACHED*/
 	}
 
@@ -1446,235 +1578,183 @@ need_stack:
 }
 
 /*
- *	thread_done:
+ *	thread_dispatch:
  *
- *	Perform calculations for thread
- *	finishing execution on the current processor.
+ *	Handle threads at context switch.  Re-dispatch other thread
+ *	if still running, otherwise update run state and perform
+ *	special actions.  Update quantum for other thread and begin
+ *	the quantum for ourselves.
  *
  *	Called at splsched.
  */
 void
-thread_done(
-	thread_t			old_thread,
-	thread_t			new_thread,
-	processor_t			processor)
+thread_dispatch(
+	thread_t		thread,
+	thread_t		self)
 {
-	if (!(old_thread->state & TH_IDLE)) {
+	processor_t		processor = self->last_processor;
+
+	if (thread != THREAD_NULL) {
 		/*
-		 * Compute remainder of current quantum.
+		 *	If blocked at a continuation, discard
+		 *	the stack.
 		 */
-		if (	first_timeslice(processor)							&&
-				processor->quantum_end > processor->last_dispatch		)
-			old_thread->current_quantum =
-					(processor->quantum_end - processor->last_dispatch);
-		else
-			old_thread->current_quantum = 0;
+		if (thread->continuation != NULL && thread->kernel_stack != 0)
+			stack_free(thread);
+
+		if (!(thread->state & TH_IDLE)) {
+			wake_lock(thread);
+			thread_lock(thread);
 
-		if (old_thread->sched_mode & TH_MODE_REALTIME) {
 			/*
-			 * Cancel the deadline if the thread has
-			 * consumed the entire quantum.
+			 *	Compute remainder of current quantum.
 			 */
-			if (old_thread->current_quantum == 0) {
-				old_thread->realtime.deadline = UINT64_MAX;
-				old_thread->reason |= AST_QUANTUM;
+			if (	first_timeslice(processor)							&&
+					processor->quantum_end > processor->last_dispatch		)
+				thread->current_quantum = (processor->quantum_end - processor->last_dispatch);
+			else
+				thread->current_quantum = 0;
+
+			if (thread->sched_mode & TH_MODE_REALTIME) {
+				/*
+				 *	Cancel the deadline if the thread has
+				 *	consumed the entire quantum.
+				 */
+				if (thread->current_quantum == 0) {
+					thread->realtime.deadline = UINT64_MAX;
+					thread->reason |= AST_QUANTUM;
+				}
 			}
-		}
-		else {
+			else {
+				/*
+				 *	For non-realtime threads treat a tiny
+				 *	remaining quantum as an expired quantum
+				 *	but include what's left next time.
+				 */
+				if (thread->current_quantum < min_std_quantum) {
+					thread->reason |= AST_QUANTUM;
+					thread->current_quantum += std_quantum;
+				}
+			}
+
 			/*
-			 * For non-realtime threads treat a tiny
-			 * remaining quantum as an expired quantum
-			 * but include what's left next time.
+			 *	If we are doing a direct handoff then
+			 *	take the remainder of the quantum.
 			 */
-			if (old_thread->current_quantum < min_std_quantum) {
-				old_thread->reason |= AST_QUANTUM;
-				old_thread->current_quantum += std_quantum;
+			if ((thread->reason & (AST_HANDOFF|AST_QUANTUM)) == AST_HANDOFF) {
+				self->current_quantum = thread->current_quantum;
+				thread->reason |= AST_QUANTUM;
+				thread->current_quantum = 0;
 			}
-		}
 
-		/*
-		 * If we are doing a direct handoff then
-		 * give the remainder of our quantum to
-		 * the next thread.
-		 */
-		if ((old_thread->reason & (AST_HANDOFF|AST_QUANTUM)) == AST_HANDOFF) {
-			new_thread->current_quantum = old_thread->current_quantum;
-			old_thread->reason |= AST_QUANTUM;
-			old_thread->current_quantum = 0;
-		}
+			thread->last_switch = processor->last_dispatch;
+
+			thread->computation_metered += (thread->last_switch - thread->computation_epoch);
+
+			if (!(thread->state & TH_WAIT)) {
+				/*
+				 *	Still running.
+				 */
+				if (thread->reason & AST_QUANTUM)
+					thread_setrun(thread, SCHED_TAILQ);
+				else
+				if (thread->reason & AST_PREEMPT)
+					thread_setrun(thread, SCHED_HEADQ);
+				else
+					thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
+
+				thread->reason = AST_NONE;
+
+				thread_unlock(thread);
+				wake_unlock(thread);
+			}
+			else {
+				/*
+				 *	Waiting.
+				 */
+				thread->state &= ~TH_RUN;
+
+				if (thread->sched_mode & TH_MODE_TIMESHARE)
+					sched_share_decr();
+				sched_run_decr();
+
+				if (thread->wake_active) {
+					thread->wake_active = FALSE;
+					thread_unlock(thread);
+
+					thread_wakeup(&thread->wake_active);
+				}
+				else
+					thread_unlock(thread);
 
-		old_thread->last_switch = processor->last_dispatch;
+				wake_unlock(thread);
 
-		old_thread->computation_metered +=
-				(old_thread->last_switch - old_thread->computation_epoch);
+				(*thread->sched_call)(SCHED_CALL_BLOCK, thread);
+
+				if (thread->state & TH_TERMINATE)
+					thread_terminate_enqueue(thread);
+			}
+		}
 	}
-}
 
-/*
- *	thread_begin:
- *
- *	Set up for thread beginning execution on
- *	the current processor.
- *
- *	Called at splsched.
- */
-void
-thread_begin(
-	thread_t			thread,
-	processor_t			processor)
-{
-	if (!(thread->state & TH_IDLE)) {
+	if (!(self->state & TH_IDLE)) {
 		/*
-		 * Give the thread a new quantum
-		 * if none remaining.
+		 *	Get a new quantum if none remaining.
 		 */
-		if (thread->current_quantum == 0)
-			thread_quantum_init(thread);
+		if (self->current_quantum == 0)
+			thread_quantum_init(self);
 
 		/*
-		 * Set up quantum timer and timeslice.
+		 *	Set up quantum timer and timeslice.
 		 */
-		processor->quantum_end =
-				(processor->last_dispatch + thread->current_quantum);
-		timer_call_enter1(&processor->quantum_timer,
-								thread, processor->quantum_end);
+		processor->quantum_end = (processor->last_dispatch + self->current_quantum);
+		timer_call_enter1(&processor->quantum_timer, self, processor->quantum_end);
 
-		processor_timeslice_setup(processor, thread);
+		processor->timeslice = 1;
 
-		thread->last_switch = processor->last_dispatch;
+		self->last_switch = processor->last_dispatch;
 
-		thread->computation_epoch = thread->last_switch;
+		self->computation_epoch = self->last_switch;
 	}
 	else {
 		timer_call_cancel(&processor->quantum_timer);
-		processor->timeslice = 1;
+		processor->timeslice = 0;
 	}
 }
 
 /*
- *	thread_dispatch:
+ *	thread_block_reason:
  *
- *	Handle previous thread at context switch.  Re-dispatch
- *	if still running, otherwise update run state and perform
- *	special actions.
+ *	Forces a reschedule, blocking the caller if a wait
+ *	has been asserted.
  *
- *	Called at splsched.
+ *	If a continuation is specified, then thread_invoke will
+ *	attempt to discard the thread's kernel stack.  When the
+ *	thread resumes, it will execute the continuation function
+ *	on a new kernel stack.
  */
-void
-thread_dispatch(
-	register thread_t	thread)
+counter(mach_counter_t  c_thread_block_calls = 0;)
+ 
+wait_result_t
+thread_block_reason(
+	thread_continue_t	continuation,
+	void				*parameter,
+	ast_t				reason)
 {
-	/*
-	 *	If blocked at a continuation, discard
-	 *	the stack.
-	 */
-    if (thread->continuation != NULL && thread->kernel_stack)
-		stack_free(thread);
-
-	if (!(thread->state & TH_IDLE)) {
-		wake_lock(thread);
-		thread_lock(thread);
-
-		if (!(thread->state & TH_WAIT)) {
-			/*
-			 *	Still running.
-			 */
-			if (thread->reason & AST_QUANTUM)
-				thread_setrun(thread, SCHED_TAILQ);
-			else
-			if (thread->reason & AST_PREEMPT)
-				thread_setrun(thread, SCHED_HEADQ);
-			else
-				thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
-
-			thread->reason = AST_NONE;
-
-			thread_unlock(thread);
-			wake_unlock(thread);
-		}
-		else {
-			boolean_t		wake;
-
-			/*
-			 *	Waiting.
-			 */
-			thread->state &= ~TH_RUN;
-
-			wake = thread->wake_active;
-			thread->wake_active = FALSE;
-
-			if (thread->sched_mode & TH_MODE_TIMESHARE)
-				pset_share_decr(thread->processor_set);
-			pset_run_decr(thread->processor_set);
-
-			thread_unlock(thread);
-			wake_unlock(thread);
-
-			if (thread->options & TH_OPT_CALLOUT)
-				call_thread_block();
-
-			if (wake)
-				thread_wakeup((event_t)&thread->wake_active);
-
-			if (thread->state & TH_TERMINATE)
-				thread_terminate_enqueue(thread);
-		}
-	}
-}
-
-/*
- *	thread_block_reason:
- *
- *	Forces a reschedule, blocking the caller if a wait
- *	has been asserted.
- *
- *	If a continuation is specified, then thread_invoke will
- *	attempt to discard the thread's kernel stack.  When the
- *	thread resumes, it will execute the continuation function
- *	on a new kernel stack.
- */
-counter(mach_counter_t  c_thread_block_calls = 0;)
- 
-wait_result_t
-thread_block_reason(
-	thread_continue_t	continuation,
-	void				*parameter,
-	ast_t				reason)
-{
-	register thread_t		self = current_thread();
-	register processor_t	processor;
-	register thread_t		new_thread;
-	spl_t					s;
+	register thread_t		self = current_thread();
+	register processor_t	processor;
+	register thread_t		new_thread;
+	spl_t					s;
 
 	counter(++c_thread_block_calls);
 
 	s = splsched();
 
-#if 0
-#if	MACH_KDB
-	{
-		extern void db_chkpmgr(void);
-		db_chkpmgr();						/* (BRINGUP) See if pm config changed */
-
-	}
-#endif
-#endif
-
 	if (!(reason & AST_PREEMPT))
 		funnel_release_check(self, 2);
 
 	processor = current_processor();
 
-	/*
-	 * Delay switching to the idle thread under certain conditions.
-	 */
-	if (s != FALSE && (self->state & (TH_IDLE|TH_TERMINATE|TH_WAIT)) == TH_WAIT) {
-		if (	processor->processor_set->processor_count > 1	&&
-				processor->processor_set->runq.count == 0		&&
-				processor->runq.count == 0						)
-			processor = delay_idle(processor, self);
-	}
-
 	/* If we're explicitly yielding, force a subsequent quantum */
 	if (reason & AST_YIELD)
 		processor->timeslice = 0;
@@ -1685,16 +1765,11 @@ thread_block_reason(
 	self->continuation = continuation;
 	self->parameter = parameter;
 
-	thread_lock(self);
-	new_thread = thread_select(processor);
-	assert(new_thread && thread_runnable(new_thread));
-	thread_unlock(self);
-	while (!thread_invoke(self, new_thread, reason)) {
+	do {
 		thread_lock(self);
-		new_thread = thread_select(processor);
-		assert(new_thread && thread_runnable(new_thread));
+		new_thread = thread_select(self, processor);
 		thread_unlock(self);
-	}
+	} while (!thread_invoke(self, new_thread, reason));
 
 	funnel_refunnel_check(self, 5);
 	splx(s);
@@ -1747,10 +1822,10 @@ thread_run(
 	self->parameter = parameter;
 
 	while (!thread_invoke(self, new_thread, handoff)) {
-		register processor_t		processor = current_processor();
+		processor_t		processor = current_processor();
 
 		thread_lock(self);
-		new_thread = thread_select(processor);
+		new_thread = thread_select(self, processor);
 		thread_unlock(self);
 		handoff = AST_NONE;
 	}
@@ -1768,101 +1843,113 @@ thread_run(
  */
 void
 thread_continue(
-	register thread_t	old_thread)
+	register thread_t	thread)
 {
 	register thread_t			self = current_thread();
 	register thread_continue_t	continuation;
 	register void				*parameter;
 	
 	continuation = self->continuation;
-	self->continuation = NULL;
 	parameter = self->parameter;
-	self->parameter = NULL;
 
-	thread_begin(self, self->last_processor);
+	thread_dispatch(thread, self);
 
-	if (old_thread != THREAD_NULL)
-		thread_dispatch(old_thread);
+	self->continuation = self->parameter = NULL;
 
 	funnel_refunnel_check(self, 4);
 
-	if (old_thread != THREAD_NULL)
+	if (thread != THREAD_NULL)
 		(void)spllo();
 
+ TLOG(1, "thread_continue: calling call_continuation \n");
 	call_continuation(continuation, parameter, self->wait_result);
 	/*NOTREACHED*/
 }
 
 /*
- *	Enqueue thread on run queue.  Thread must be locked,
- *	and not already be on a run queue.  Returns TRUE
- *	if a preemption is indicated based on the state
- *	of the run queue.
+ *	run_queue_init:
  *
- *	Run queue must be locked, see run_queue_remove()
- *	for more info.
+ *	Initialize a run queue before first use.
  */
-static boolean_t
-run_queue_enqueue(
-	register run_queue_t	rq,
-	register thread_t		thread,
-	integer_t				options)
-{
-	register int			whichq = thread->sched_pri;
-	register queue_t		queue = &rq->queues[whichq];
-	boolean_t				result = FALSE;
-	
-	assert(whichq >= MINPRI && whichq <= MAXPRI);
+void
+run_queue_init(
+	run_queue_t		rq)
+{
+	int				i;
+
+	rq->highq = IDLEPRI;
+	for (i = 0; i < NRQBM; i++)
+		rq->bitmap[i] = 0;
+	setbit(MAXPRI - IDLEPRI, rq->bitmap);
+	rq->urgency = rq->count = 0;
+	for (i = 0; i < NRQS; i++)
+		queue_init(&rq->queues[i]);
+}
 
-	assert(thread->runq == RUN_QUEUE_NULL);
-	if (queue_empty(queue)) {
-		enqueue_tail(queue, (queue_entry_t)thread);
+/*
+ *	run_queue_dequeue:
+ *
+ *	Perform a dequeue operation on a run queue,
+ *	and return the resulting thread.
+ *
+ *	The run queue must be locked (see run_queue_remove()
+ *	for more info), and not empty.
+ */
+static thread_t
+run_queue_dequeue(
+	run_queue_t		rq,
+	integer_t		options)
+{
+	thread_t		thread;
+	queue_t			queue = rq->queues + rq->highq;
 
-		setbit(MAXPRI - whichq, rq->bitmap);
-		if (whichq > rq->highq) {
-			rq->highq = whichq;
-			result = TRUE;
-		}
+	if (options & SCHED_HEADQ) {
+		thread = (thread_t)queue->next;
+		((queue_entry_t)thread)->next->prev = queue;
+		queue->next = ((queue_entry_t)thread)->next;
+	}
+	else {
+		thread = (thread_t)queue->prev;
+		((queue_entry_t)thread)->prev->next = queue;
+		queue->prev = ((queue_entry_t)thread)->prev;
 	}
-	else
-	if (options & SCHED_HEADQ)
-		enqueue_head(queue, (queue_entry_t)thread);
-	else
-		enqueue_tail(queue, (queue_entry_t)thread);
 
-	thread->runq = rq;
+	thread->runq = PROCESSOR_NULL;
+	rq->count--;
 	if (thread->sched_mode & TH_MODE_PREEMPT)
-		rq->urgency++;
-	rq->count++;
+		rq->urgency--;
+	if (queue_empty(queue)) {
+		if (rq->highq != IDLEPRI)
+			clrbit(MAXPRI - rq->highq, rq->bitmap);
+		rq->highq = MAXPRI - ffsbit(rq->bitmap);
+	}
 
-	return (result);
+	return (thread);
 }
 
 /*
- *	Enqueue a thread for realtime execution, similar
- *	to above.  Handles preemption directly.
+ *	realtime_queue_insert:
+ *
+ *	Enqueue a thread for realtime execution.
  */
-static void
-realtime_schedule_insert(
-	register processor_set_t	pset,
-	register thread_t			thread)
+static boolean_t
+realtime_queue_insert(
+	thread_t			thread)
 {
-	register run_queue_t	rq = &pset->runq;
-	register int			whichq = thread->sched_pri;
-	register queue_t		queue = &rq->queues[whichq];
-	uint64_t				deadline = thread->realtime.deadline;
-	boolean_t				try_preempt = FALSE;
+	run_queue_t			rq = &rt_runq;
+	queue_t				queue = rq->queues + thread->sched_pri;
+	uint64_t			deadline = thread->realtime.deadline;
+	boolean_t			preempt = FALSE;
 
-	assert(whichq >= BASEPRI_REALTIME && whichq <= MAXPRI);
+	simple_lock(&rt_lock);
 
-	assert(thread->runq == RUN_QUEUE_NULL);
 	if (queue_empty(queue)) {
 		enqueue_tail(queue, (queue_entry_t)thread);
 
-		setbit(MAXPRI - whichq, rq->bitmap);
-		if (whichq > rq->highq)
-			rq->highq = whichq;
-		try_preempt = TRUE;
+		setbit(MAXPRI - thread->sched_pri, rq->bitmap);
+		if (thread->sched_pri > rq->highq)
+			rq->highq = thread->sched_pri;
+		preempt = TRUE;
 	}
 	else {
 		register thread_t	entry = (thread_t)queue_first(queue);
@@ -1878,302 +1965,484 @@ realtime_schedule_insert(
 		}
 
 		if ((queue_entry_t)entry == queue)
-			try_preempt = TRUE;
+			preempt = TRUE;
 
 		insque((queue_entry_t)thread, (queue_entry_t)entry);
 	}
 
-	thread->runq = rq;
+	thread->runq = RT_RUNQ;
 	assert(thread->sched_mode & TH_MODE_PREEMPT);
 	rq->count++; rq->urgency++;
 
-	if (try_preempt) {
-		register processor_t	processor;
+	simple_unlock(&rt_lock);
 
-		processor = current_processor();
-		if (	pset == processor->processor_set				&&
-				(thread->sched_pri > processor->current_pri	||
-					deadline < processor->deadline			)		) {
-			dispatch_counts.realtime_self++;
-			simple_unlock(&pset->sched_lock);
+	return (preempt);
+}
 
-			ast_on(AST_PREEMPT | AST_URGENT);
-			return;
-		}
+/*
+ *	realtime_setrun:
+ *
+ *	Dispatch a thread for realtime execution.
+ *
+ *	Thread must be locked.  Associated pset must
+ *	be locked, and is returned unlocked.
+ */
+static void
+realtime_setrun(
+	processor_t			processor,
+	thread_t			thread)
+{
+	processor_set_t		pset = processor->processor_set;
 
-		if (	pset->processor_count > 1			||
-				pset != processor->processor_set	) {
-			processor_t		myprocessor, lastprocessor;
-			queue_entry_t	next;
+	/*
+	 *	Dispatch directly onto idle processor.
+	 */
+	if (processor->state == PROCESSOR_IDLE) {
+		remqueue(&pset->idle_queue, (queue_entry_t)processor);
+		pset->idle_count--;
+		enqueue_head(&pset->active_queue, (queue_entry_t)processor);
 
-			myprocessor = processor;
-			processor = thread->last_processor;
-			if (	processor != myprocessor						&&
-					processor != PROCESSOR_NULL						&&
-					processor->processor_set == pset				&&
-					processor->state == PROCESSOR_RUNNING			&&
-					(thread->sched_pri > processor->current_pri	||
-						deadline < processor->deadline			)		) {
-				dispatch_counts.realtime_last++;
-				cause_ast_check(processor);
-				simple_unlock(&pset->sched_lock);
-				return;
-			}
+		processor->next_thread = thread;
+		processor->deadline = thread->realtime.deadline;
+		processor->state = PROCESSOR_DISPATCHING;
+		pset_unlock(pset);
 
-			lastprocessor = processor;
-			queue = &pset->active_queue;
-			processor = (processor_t)queue_first(queue);
-			while (!queue_end(queue, (queue_entry_t)processor)) {
-				next = queue_next((queue_entry_t)processor);
-
-				if (	processor != myprocessor						&&
-						processor != lastprocessor						&&
-						(thread->sched_pri > processor->current_pri	||
-							deadline < processor->deadline			)		) {
-					if (!queue_end(queue, next)) {
-						remqueue(queue, (queue_entry_t)processor);
-						enqueue_tail(queue, (queue_entry_t)processor);
-					}
-					dispatch_counts.realtime_other++;
-					cause_ast_check(processor);
-					simple_unlock(&pset->sched_lock);
-					return;
-				}
+		if (processor != current_processor())
+			machine_signal_idle(processor);
+		return;
+	}
 
-				processor = (processor_t)next;
-			}
+	if (realtime_queue_insert(thread)) {
+		if (processor == current_processor())
+			ast_on(AST_PREEMPT | AST_URGENT);
+		else
+			cause_ast_check(processor);
+	}
+
+	pset_unlock(pset);
+}
+
+/*
+ *	processor_enqueue:
+ *
+ *	Enqueue thread on a processor run queue.  Thread must be locked,
+ *	and not already be on a run queue.
+ *
+ *	Returns TRUE if a preemption is indicated based on the state
+ *	of the run queue.
+ *
+ *	The run queue must be locked (see run_queue_remove()
+ *	for more info).
+ */
+static boolean_t
+processor_enqueue(
+	processor_t		processor,
+	thread_t		thread,
+	integer_t		options)
+{
+	run_queue_t		rq = &processor->runq;
+	queue_t			queue = rq->queues + thread->sched_pri;
+	boolean_t		result = FALSE;
+	
+	if (queue_empty(queue)) {
+		enqueue_tail(queue, (queue_entry_t)thread);
+
+		setbit(MAXPRI - thread->sched_pri, rq->bitmap);
+		if (thread->sched_pri > rq->highq) {
+			rq->highq = thread->sched_pri;
+			result = TRUE;
 		}
 	}
+	else
+	if (options & SCHED_TAILQ)
+		enqueue_tail(queue, (queue_entry_t)thread);
+	else
+		enqueue_head(queue, (queue_entry_t)thread);
 
-	simple_unlock(&pset->sched_lock);
+	thread->runq = processor;
+	if (thread->sched_mode & TH_MODE_PREEMPT)
+		rq->urgency++;
+	rq->count++;
+
+	return (result);
 }
 
 /*
- *	thread_setrun:
+ *	processor_setrun:
  *
- *	Dispatch thread for execution, directly onto an idle
- *	processor if possible.  Else put on appropriate run
- *	queue. (local if bound, else processor set)
+ *	Dispatch a thread for execution on a
+ *	processor.
  *
- *	Thread must be locked.
+ *	Thread must be locked.  Associated pset must
+ *	be locked, and is returned unlocked.
  */
-void
-thread_setrun(
-	register thread_t			new_thread,
-	integer_t					options)
+static void
+processor_setrun(
+	processor_t			processor,
+	thread_t			thread,
+	integer_t			options)
 {
-	register processor_t		processor;
-	register processor_set_t	pset;
-	register thread_t			thread;
-	ast_t						preempt = (options & SCHED_PREEMPT)?
-													AST_PREEMPT: AST_NONE;
+	processor_set_t		pset = processor->processor_set;
+	ast_t				preempt;
 
-	assert(thread_runnable(new_thread));
-	
 	/*
-	 *	Update priority if needed.
+	 *	Dispatch directly onto idle processor.
 	 */
-	if (new_thread->sched_stamp != sched_tick)
-		update_priority(new_thread);
+	if (processor->state == PROCESSOR_IDLE) {
+		remqueue(&pset->idle_queue, (queue_entry_t)processor);
+		pset->idle_count--;
+		enqueue_head(&pset->active_queue, (queue_entry_t)processor);
+
+		processor->next_thread = thread;
+		processor->deadline = UINT64_MAX;
+		processor->state = PROCESSOR_DISPATCHING;
+		pset_unlock(pset);
+
+		if (processor != current_processor())
+			machine_signal_idle(processor);
+		return;
+	}
 
 	/*
-	 *	Check for urgent preemption.
+	 *	Set preemption mode.
 	 */
-	if (new_thread->sched_mode & TH_MODE_PREEMPT)
+	if (thread->sched_mode & TH_MODE_PREEMPT)
 		preempt = (AST_PREEMPT | AST_URGENT);
+	else
+	if (thread->sched_mode & TH_MODE_TIMESHARE && thread->priority < BASEPRI_BACKGROUND)
+		preempt = AST_NONE;
+	else
+		preempt = (options & SCHED_PREEMPT)? AST_PREEMPT: AST_NONE;
 
-	assert(new_thread->runq == RUN_QUEUE_NULL);
+	if (!processor_enqueue(processor, thread, options))
+		preempt = AST_NONE;
 
-	if ((processor = new_thread->bound_processor) == PROCESSOR_NULL) {
-	    /*
-	     *	First try to dispatch on
-		 *	the last processor.
-	     */
-	    pset = new_thread->processor_set;
-		processor = new_thread->last_processor;
-		if (	pset->processor_count > 1				&&
-				processor != PROCESSOR_NULL				&&
-				processor->state == PROCESSOR_IDLE		) {
-			processor_lock(processor);
-			simple_lock(&pset->sched_lock);
-			if (	processor->processor_set == pset		&&
-					processor->state == PROCESSOR_IDLE		) {
-				remqueue(&pset->idle_queue, (queue_entry_t)processor);
-				pset->idle_count--;
-				processor->next_thread = new_thread;
-				if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-					processor->deadline = new_thread->realtime.deadline;
-				else
-					processor->deadline = UINT64_MAX;
-				processor->state = PROCESSOR_DISPATCHING;
-				dispatch_counts.idle_pset_last++;
-				simple_unlock(&pset->sched_lock);
-				processor_unlock(processor);
-				if (processor != current_processor())
-					machine_signal_idle(processor);
-				return;
-			}
-			processor_unlock(processor);
+	pset_hint_high(pset, processor);
+
+	if (preempt != AST_NONE) {
+		if (processor == current_processor()) {
+			thread_t	self = processor->active_thread;
+
+			if (csw_needed(self, processor))
+				ast_on(preempt);
 		}
 		else
-		simple_lock(&pset->sched_lock);
+		if (	(processor->state == PROCESSOR_RUNNING		||
+				 processor->state == PROCESSOR_SHUTDOWN)		&&
+				thread->sched_pri >= processor->current_pri		) {
+			cause_ast_check(processor);
+		}
+	}
+	else
+	if (	processor->state == PROCESSOR_SHUTDOWN		&&
+			thread->sched_pri >= processor->current_pri	) {
+		cause_ast_check(processor);
+	}
+
+	pset_unlock(pset);
+}
 
+#define next_pset(p)	(((p)->pset_list != PROCESSOR_SET_NULL)? (p)->pset_list: (p)->node->psets)
+
+/*
+ *	choose_next_pset:
+ *
+ *	Return the next sibling pset containing
+ *	available processors.
+ *
+ *	Returns the original pset if none other is
+ *	suitable.
+ */
+static processor_set_t
+choose_next_pset(
+	processor_set_t		pset)
+{
+	processor_set_t		nset = pset;
+
+	do {
+		nset = next_pset(nset);
+	} while (nset->processor_count < 1 && nset != pset);
+
+	return ((nset != pset)? nset: pset);
+}
+
+/*
+ *	choose_processor:
+ *
+ *	Choose a processor for the thread, beginning at
+ *	the pset.
+ *
+ *	Returns a processor, possibly from a different pset.
+ *
+ *	The thread must be locked.  The pset must be locked,
+ *	and the resulting pset is locked on return.
+ */
+static processor_t
+choose_processor(
+	processor_set_t		pset,
+	thread_t			thread)
+{
+	processor_set_t		nset, cset = pset;
+	processor_t			processor;
+
+	/*
+	 *	Iterate through the processor sets to locate
+	 *	an appropriate processor.
+	 */
+	do {
 		/*
-		 *	Next pick any idle processor
-		 *	in the processor set.
+		 *	Choose an idle processor.
 		 */
-		if (pset->idle_count > 0) {
-			processor = (processor_t)dequeue_head(&pset->idle_queue);
-			pset->idle_count--;
-			processor->next_thread = new_thread;
-			if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-				processor->deadline = new_thread->realtime.deadline;
-			else
-				processor->deadline = UINT64_MAX;
-			processor->state = PROCESSOR_DISPATCHING;
-			dispatch_counts.idle_pset_any++;
-			simple_unlock(&pset->sched_lock);
-			if (processor != current_processor())	
-				machine_signal_idle(processor);
-			return;
-		}
+		if (!queue_empty(&cset->idle_queue))
+			return ((processor_t)queue_first(&cset->idle_queue));
 
-		if (new_thread->sched_pri >= BASEPRI_RTQUEUES)
-			realtime_schedule_insert(pset, new_thread);
+		if (thread->sched_pri >= BASEPRI_RTQUEUES) {
+			/*
+			 *	For an RT thread, iterate through active processors, first fit.
+			 */
+			processor = (processor_t)queue_first(&cset->active_queue);
+			while (!queue_end(&cset->active_queue, (queue_entry_t)processor)) {
+				if (thread->sched_pri > processor->current_pri ||
+						thread->realtime.deadline < processor->deadline)
+					return (processor);
+
+				processor = (processor_t)queue_next((queue_entry_t)processor);
+			}
+		}
 		else {
-			if (!run_queue_enqueue(&pset->runq, new_thread, options))
-				preempt = AST_NONE;
+			/*
+			 *	Choose the low hint processor in the processor set if available.
+			 */
+			processor = cset->low_hint;
+			if (processor != PROCESSOR_NULL &&
+						processor->state != PROCESSOR_SHUTDOWN && processor->state != PROCESSOR_OFF_LINE)
+				return (processor);
 
 			/*
-			 *	Update the timesharing quanta.
+			 *	Choose any active processor if the hint was invalid.
 			 */
-			timeshare_quanta_update(pset);
+			processor = (processor_t)dequeue_head(&cset->active_queue);
+			if (processor != PROCESSOR_NULL) {
+				enqueue_tail(&cset->active_queue, (queue_entry_t)processor);
+				return (processor);
+			}
+		}
+
+		/*
+		 *	Move onto the next processor set.
+		 */
+		nset = next_pset(cset);
+
+		if (nset != pset) {
+			pset_unlock(cset);
+
+			cset = nset;
+			pset_lock(cset);
+		}
+	} while (nset != pset);
+
+	/*
+	 *	If all else fails choose the current processor,
+	 *	this routine must return a running processor.
+	 */
+	processor = current_processor();
+	if (cset != processor->processor_set) {
+		pset_unlock(cset);
+
+		cset = processor->processor_set;
+		pset_lock(cset);
+	}
+
+	return (processor);
+}
+
+/*
+ *	thread_setrun:
+ *
+ *	Dispatch thread for execution, onto an idle
+ *	processor or run queue, and signal a preemption
+ *	as appropriate.
+ *
+ *	Thread must be locked.
+ */
+void
+thread_setrun(
+	thread_t			thread,
+	integer_t			options)
+{
+	processor_t			processor;
+	processor_set_t		pset;
+
+#if DEBUG
+	assert(thread_runnable(thread));
+#endif
 	
+	/*
+	 *	Update priority if needed.
+	 */
+	if (thread->sched_stamp != sched_tick)
+		update_priority(thread);
+
+	assert(thread->runq == PROCESSOR_NULL);
+
+	if (thread->bound_processor == PROCESSOR_NULL) {
+		/*
+		 *	Unbound case.
+		 */
+		if (thread->affinity_set != AFFINITY_SET_NULL) {
+			/*
+			 * Use affinity set policy hint.
+			 */
+			pset = thread->affinity_set->aset_pset;
+			pset_lock(pset);
+
+			processor = choose_processor(pset, thread);
+		}
+		else
+		if (thread->last_processor != PROCESSOR_NULL) {
+			/*
+			 *	Simple (last processor) affinity case.
+			 */
+			processor = thread->last_processor;
+			pset = processor->processor_set;
+			pset_lock(pset);
+
 			/*
-			 *	Preempt check.
+			 *	Choose a different processor in certain cases.
 			 */
-			if (preempt != AST_NONE) {
+			if (processor->state == PROCESSOR_SHUTDOWN || processor->state == PROCESSOR_OFF_LINE)
+				processor = choose_processor(pset, thread);
+			else
+			if (thread->sched_pri >= BASEPRI_RTQUEUES) {
 				/*
-				 * First try the current processor
-				 * if it is a member of the correct
-				 * processor set.
+				 *	If the processor is executing an RT thread with
+				 *	an earlier deadline, choose another.
 				 */
-				processor = current_processor();
-				thread = processor->active_thread;
-				if (	pset == processor->processor_set	&&
-						csw_needed(thread, processor)		) {
-					dispatch_counts.pset_self++;
-					simple_unlock(&pset->sched_lock);
-
-					ast_on(preempt);
-					return;
-				}
+				if (thread->sched_pri <= processor->current_pri ||
+						thread->realtime.deadline >= processor->deadline)
+					processor = choose_processor(pset, thread);
+			}
+			else
+			if (processor->state != PROCESSOR_IDLE && pset->idle_count > 0) {
+				processor = choose_processor(pset, thread);
+			}
+			else {
+				processor_set_t		nset = choose_next_pset(pset);
 
 				/*
-				 * If that failed and we have other
-				 * processors available keep trying.
+				 *	Bump into a lesser loaded processor set if appropriate.
 				 */
-				if (	pset->processor_count > 1			||
-						pset != processor->processor_set	) {
-					queue_t			queue = &pset->active_queue;
-					processor_t		myprocessor, lastprocessor;
-					queue_entry_t	next;
-
-					/*
-					 * Next try the last processor
-					 * dispatched on.
-					 */
-					myprocessor = processor;
-					processor = new_thread->last_processor;
-					if (	processor != myprocessor						&&
-							processor != PROCESSOR_NULL						&&
-							processor->processor_set == pset				&&
-							processor->state == PROCESSOR_RUNNING			&&
-							new_thread->sched_pri > processor->current_pri	) {
-						dispatch_counts.pset_last++;
-						cause_ast_check(processor);
-						simple_unlock(&pset->sched_lock);
-						return;
-					}
+				if (pset != nset && (nset->low_hint == PROCESSOR_NULL ||
+												(pset->idle_count == 0 && nset->idle_count > 0) ||
+														processor->runq.count > nset->low_hint->runq.count)) {
+					pset_unlock(pset);
 
-					/*
-					 * Lastly, pick any other
-					 * available processor.
-					 */
-					lastprocessor = processor;
-					processor = (processor_t)queue_first(queue);
-					while (!queue_end(queue, (queue_entry_t)processor)) {
-						next = queue_next((queue_entry_t)processor);
-
-						if (	processor != myprocessor			&&
-								processor != lastprocessor			&&
-								new_thread->sched_pri >
-											processor->current_pri		) {
-							if (!queue_end(queue, next)) {
-								remqueue(queue, (queue_entry_t)processor);
-								enqueue_tail(queue, (queue_entry_t)processor);
-							}
-							dispatch_counts.pset_other++;
-							cause_ast_check(processor);
-							simple_unlock(&pset->sched_lock);
-							return;
-						}
+					pset = nset;
+					pset_lock(pset);
 
-						processor = (processor_t)next;
-					}
+					processor = choose_processor(pset, thread);
 				}
 			}
+		}
+		else {
+			/*
+			 *	No Affinity case:
+			 *
+			 *	Choose a processor from the current processor set.
+			 */
+			processor = current_processor();
+			pset = processor->processor_set;
+			pset_lock(pset);
 
-			simple_unlock(&pset->sched_lock);
+			processor = choose_processor(pset, thread);
 		}
 	}
 	else {
-	    /*
-	     *	Bound, can only run on bound processor.  Have to lock
-	     *  processor here because it may not be the current one.
-	     */
-		processor_lock(processor);
+		/*
+		 *	Bound case:
+		 *
+		 *	Unconditionally dispatch on the processor.
+		 */
+		processor = thread->bound_processor;
 		pset = processor->processor_set;
-		if (pset != PROCESSOR_SET_NULL) {
-			simple_lock(&pset->sched_lock);
-			if (processor->state == PROCESSOR_IDLE) {
-				remqueue(&pset->idle_queue, (queue_entry_t)processor);
-				pset->idle_count--;
-				processor->next_thread = new_thread;
-				processor->deadline = UINT64_MAX;
-				processor->state = PROCESSOR_DISPATCHING;
-				dispatch_counts.idle_bound++;
-				simple_unlock(&pset->sched_lock);
-				processor_unlock(processor);
-				if (processor != current_processor())	
-					machine_signal_idle(processor);
-				return;
-			}
-		}
-	  
-		if (!run_queue_enqueue(&processor->runq, new_thread, options))
-			preempt = AST_NONE;
-
-		if (preempt != AST_NONE) {
-			if (processor == current_processor()) {
-				thread = processor->active_thread;
-				if (csw_needed(thread, processor)) {
-					dispatch_counts.bound_self++;
-					ast_on(preempt);
+		pset_lock(pset);
+	}
+
+	/*
+	 *	Dispatch the thread on the choosen processor.
+	 */
+	if (thread->sched_pri >= BASEPRI_RTQUEUES)
+		realtime_setrun(processor, thread);
+	else
+		processor_setrun(processor, thread, options);
+}
+
+/*
+ *	processor_queue_shutdown:
+ *
+ *	Shutdown a processor run queue by moving
+ *	non-bound threads to the current processor.
+ *
+ *	Associated pset must be locked, and is
+ *	returned unlocked.
+ */
+void
+processor_queue_shutdown(
+	processor_t			processor)
+{
+	processor_set_t		pset = processor->processor_set;
+	run_queue_t			rq = &processor->runq;
+	queue_t				queue = rq->queues + rq->highq;
+	int					pri = rq->highq, count = rq->count;
+	thread_t			next, thread;
+	queue_head_t		tqueue;
+
+	queue_init(&tqueue);
+	
+	while (count > 0) {
+		thread = (thread_t)queue_first(queue);
+		while (!queue_end(queue, (queue_entry_t)thread)) {
+			next = (thread_t)queue_next((queue_entry_t)thread);
+
+			if (thread->bound_processor != processor) {
+				remqueue(queue, (queue_entry_t)thread);
+
+				thread->runq = PROCESSOR_NULL;
+				rq->count--;
+				if (thread->sched_mode & TH_MODE_PREEMPT)
+					rq->urgency--;
+				if (queue_empty(queue)) {
+					if (pri != IDLEPRI)
+						clrbit(MAXPRI - pri, rq->bitmap);
+					rq->highq = MAXPRI - ffsbit(rq->bitmap);
 				}
+
+				enqueue_tail(&tqueue, (queue_entry_t)thread);
 			}
-			else
-			if (	processor->state == PROCESSOR_RUNNING			&&
-					new_thread->sched_pri > processor->current_pri	) {
-				dispatch_counts.bound_other++;
-				cause_ast_check(processor);
-			}
+			count--;
+
+			thread = next;
 		}
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_unlock(&pset->sched_lock);
+		queue--; pri--;
+	}
+
+	pset_unlock(pset);
+
+	processor = current_processor();
+	pset = processor->processor_set;
+
+	while ((thread = (thread_t)dequeue_head(&tqueue)) != THREAD_NULL) {
+		thread_lock(thread);
+		thread->last_processor = PROCESSOR_NULL;
 
-		processor_unlock(processor);
+		pset_lock(pset);
+
+		processor_enqueue(processor, thread, SCHED_TAILQ);
+
+		pset_unlock(pset);
+
+		thread_unlock(thread);
 	}
 }
 
@@ -2193,7 +2462,7 @@ csw_check(
 	run_queue_t		runq;
 
 	if (first_timeslice(processor)) {
-		runq = &processor->processor_set->runq;
+		runq = &rt_runq;
 		if (runq->highq >= BASEPRI_RTQUEUES)
 			return (AST_PREEMPT | AST_URGENT);
 
@@ -2213,7 +2482,7 @@ csw_check(
 		}
 	}
 	else {
-		runq = &processor->processor_set->runq;
+		runq = &rt_runq;
 		if (runq->highq >= current_pri) {
 			if (runq->urgency > 0)
 				return (AST_PREEMPT | AST_URGENT);
@@ -2250,10 +2519,10 @@ csw_check(
  */
 void
 set_sched_pri(
-	thread_t			thread,
-	int					priority)
+	thread_t		thread,
+	int				priority)
 {
-	register struct run_queue	*rq = run_queue_remove(thread);
+	boolean_t		removed = run_queue_remove(thread);
 
 	if (	!(thread->sched_mode & TH_MODE_TIMESHARE)				&&
 			(priority >= BASEPRI_PREEMPT						||
@@ -2265,7 +2534,7 @@ set_sched_pri(
 		thread->sched_mode &= ~TH_MODE_PREEMPT;
 
 	thread->sched_pri = priority;
-	if (rq != RUN_QUEUE_NULL)
+	if (removed)
 		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
 	else
 	if (thread->state & TH_RUN) {
@@ -2318,41 +2587,43 @@ run_queue_check(
 /*
  *	run_queue_remove:
  *
- *	Remove a thread from its current run queue and
- *	return the run queue if successful.
+ *	Remove a thread from a current run queue and
+ *	return TRUE if successful.
  *
  *	Thread must be locked.
  */
-run_queue_t
+boolean_t
 run_queue_remove(
-	thread_t			thread)
+	thread_t		thread)
 {
-	register run_queue_t	rq = thread->runq;
+	processor_t		processor = thread->runq;
 
 	/*
-	 *	If rq is RUN_QUEUE_NULL, the thread will stay out of the
+	 *	If processor is PROCESSOR_NULL, the thread will stay out of the
 	 *	run queues because the caller locked the thread.  Otherwise
 	 *	the thread is on a run queue, but could be chosen for dispatch
 	 *	and removed.
 	 */
-	if (rq != RUN_QUEUE_NULL) {
-		processor_set_t		pset = thread->processor_set;
-		processor_t			processor = thread->bound_processor;
+	if (processor != PROCESSOR_NULL) {
+		void *			rqlock;
+		run_queue_t		rq;
 
 		/*
-		 *	The run queues are locked by the pset scheduling
-		 *	lock, except when a processor is off-line the
-		 *	local run queue is locked by the processor lock.
+		 *	The processor run queues are locked by the
+		 *	processor set.  Real-time priorities use a
+		 *	global queue with a dedicated lock.
 		 */
-		if (processor != PROCESSOR_NULL) {
-			processor_lock(processor);
-			pset = processor->processor_set;
+		if (thread->sched_pri < BASEPRI_RTQUEUES) {
+			rqlock = &processor->processor_set->sched_lock;
+			rq = &processor->runq;
+		}
+		else {
+			rqlock = &rt_lock; rq = &rt_runq;
 		}
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_lock(&pset->sched_lock);
+		simple_lock(rqlock);
 
-		if (rq == thread->runq) {
+		if (processor == thread->runq) {
 			/*
 			 *	Thread is on a run queue and we have a lock on
 			 *	that run queue.
@@ -2370,292 +2641,218 @@ run_queue_remove(
 				rq->highq = MAXPRI - ffsbit(rq->bitmap);
 			}
 
-			thread->runq = RUN_QUEUE_NULL;
+			thread->runq = PROCESSOR_NULL;
 		}
 		else {
 			/*
 			 *	The thread left the run queue before we could
 			 * 	lock the run queue.
 			 */
-			assert(thread->runq == RUN_QUEUE_NULL);
-			rq = RUN_QUEUE_NULL;
+			assert(thread->runq == PROCESSOR_NULL);
+			processor = PROCESSOR_NULL;
 		}
 
-		if (pset != PROCESSOR_SET_NULL)
-			simple_unlock(&pset->sched_lock);
-
-		if (processor != PROCESSOR_NULL)
-			processor_unlock(processor);
+		simple_unlock(rqlock);
 	}
 
-	return (rq);
+	return (processor != PROCESSOR_NULL);
 }
 
 /*
  *	choose_thread:
  *
- *	Remove a thread to execute from the run queues
- *	and return it.
+ *	Choose a thread to execute from the run queues
+ *	and return it.  May steal a thread from another
+ *	processor.
  *
- *	Called with pset scheduling lock held.
+ *	Called with pset scheduling lock and rt lock held,
+ *	released on return.
  */
 static thread_t
 choose_thread(
-	processor_set_t		pset,
 	processor_t			processor)
 {
-	register run_queue_t	runq;
-	register thread_t		thread;
-	register queue_t		q;
+	processor_set_t		pset = processor->processor_set;
+	thread_t			thread;
 
-	runq = &processor->runq;
-
-	if (runq->count > 0 && runq->highq >= pset->runq.highq) {
-		q = runq->queues + runq->highq;
-
-		thread = (thread_t)q->next;
-		((queue_entry_t)thread)->next->prev = q;
-		q->next = ((queue_entry_t)thread)->next;
-		thread->runq = RUN_QUEUE_NULL;
-		runq->count--;
-		if (thread->sched_mode & TH_MODE_PREEMPT)
-			runq->urgency--;
-		assert(runq->urgency >= 0);
-		if (queue_empty(q)) {
-			if (runq->highq != IDLEPRI)
-				clrbit(MAXPRI - runq->highq, runq->bitmap);
-			runq->highq = MAXPRI - ffsbit(runq->bitmap);
-		}
+	if (processor->runq.count > 0 && processor->runq.highq >= rt_runq.highq) {
+		simple_unlock(&rt_lock);
 
-		processor->deadline = UINT64_MAX;
+		pset_hint_low(pset, processor);
 
-		return (thread);
-	}
+		if (pset->high_hint != PROCESSOR_NULL) {
+			if (processor != pset->high_hint) {
+				if (processor->runq.count >= pset->high_hint->runq.count)
+					pset->high_hint = processor;
+				else
+				if (pset->high_hint->runq.highq > processor->runq.highq) {
+					thread = steal_thread(pset->high_hint);
+					if (thread != THREAD_NULL) {
+						processor->deadline = UINT64_MAX;
+						pset_unlock(pset);
 
-	runq = &pset->runq;
+						return (thread);
+					}
+				}
+			}
+		}
+		else
+			pset->high_hint = processor;
 
-	assert(runq->count > 0);
-	q = runq->queues + runq->highq;
+		thread = run_queue_dequeue(&processor->runq, SCHED_HEADQ);
 
-	thread = (thread_t)q->next;
-	((queue_entry_t)thread)->next->prev = q;
-	q->next = ((queue_entry_t)thread)->next;
-	thread->runq = RUN_QUEUE_NULL;
-	runq->count--;
-	if (runq->highq >= BASEPRI_RTQUEUES)
-		processor->deadline = thread->realtime.deadline;
-	else
 		processor->deadline = UINT64_MAX;
-	if (thread->sched_mode & TH_MODE_PREEMPT)
-		runq->urgency--;
-	assert(runq->urgency >= 0);
-	if (queue_empty(q)) {
-		if (runq->highq != IDLEPRI)
-			clrbit(MAXPRI - runq->highq, runq->bitmap);
-		runq->highq = MAXPRI - ffsbit(runq->bitmap);
+		pset_unlock(pset);
+
+		return (thread);
 	}
 
-	timeshare_quanta_update(pset);
+	thread = run_queue_dequeue(&rt_runq, SCHED_HEADQ);
+	simple_unlock(&rt_lock);
+
+	processor->deadline = thread->realtime.deadline;
+	pset_unlock(pset);
 
 	return (thread);
 }
 
-static processor_t
-delay_idle(
-	processor_t		processor,
-	thread_t		self)
+/*
+ *	steal_thread:
+ *
+ *	Steal a thread from a processor and return it.
+ *
+ *	Associated pset must be locked.  Returns THREAD_NULL
+ *	on failure.
+ */
+static thread_t
+steal_thread(
+	processor_t		processor)
 {
-	int				*gcount, *lcount;
-	uint64_t		abstime, spin, limit;
-
-	lcount = &processor->runq.count;
-	gcount = &processor->processor_set->runq.count;
-
-	abstime = mach_absolute_time();
-	limit = abstime + delay_idle_limit;
-	spin = abstime + delay_idle_spin;
-
-	timer_event((uint32_t)abstime, &processor->idle_thread->system_timer);
-
-	self->options |= TH_OPT_DELAYIDLE;
-
-	while (		*gcount == 0 && *lcount == 0	&&
-				(self->state & TH_WAIT)	!= 0	&&
-					abstime < limit				) {
-		if (abstime >= spin) {
-			(void)spllo();
-
-			(void)splsched();
-			processor = current_processor();
-			lcount = &processor->runq.count;
-			gcount = &processor->processor_set->runq.count;
+	run_queue_t		rq = &processor->runq;
+	queue_t			queue = rq->queues + rq->highq;
+	int				pri = rq->highq, count = rq->count;
+	thread_t		thread = THREAD_NULL;
+
+	while (count > 0) {
+		thread = (thread_t)queue_first(queue);
+		while (!queue_end(queue, (queue_entry_t)thread)) {
+			if (thread->bound_processor != processor) {
+				remqueue(queue, (queue_entry_t)thread);
+
+				thread->runq = PROCESSOR_NULL;
+				rq->count--;
+				if (thread->sched_mode & TH_MODE_PREEMPT)
+					rq->urgency--;
+				if (queue_empty(queue)) {
+					if (pri != IDLEPRI)
+						clrbit(MAXPRI - pri, rq->bitmap);
+					rq->highq = MAXPRI - ffsbit(rq->bitmap);
+				}
 
-			abstime = mach_absolute_time();
-			spin = abstime + delay_idle_spin;
+				return (thread);
+			}
+			count--;
 
-			timer_event((uint32_t)abstime, &processor->idle_thread->system_timer);
+			thread = (thread_t)queue_next((queue_entry_t)thread);
 		}
-		else {
-			cpu_pause();
-			abstime = mach_absolute_time();
-		}
-	}
-
-	timer_event((uint32_t)abstime, &self->system_timer);
 
-	self->options &= ~TH_OPT_DELAYIDLE;
+		queue--; pri--;
+	}
 
-	return (processor);
+	return (THREAD_NULL);
 }
 
 /*
- *	no_dispatch_count counts number of times processors go non-idle
- *	without being dispatched.  This should be very rare.
- */
-int	no_dispatch_count = 0;
-
-/*
- *	This is the idle processor thread, which just looks for other threads
- *	to execute.
+ *	This is the processor idle loop, which just looks for other threads
+ *	to execute.  Processor idle threads invoke this without supplying a
+ *	current thread to idle without an asserted wait state.
+ *
+ *	Returns a the next thread to execute if dispatched directly.
  */
-void
-idle_thread(void)
+static thread_t
+processor_idle(
+	thread_t			thread,
+	processor_t			processor)
 {
-	register processor_t		processor;
-	register thread_t			*threadp;
-	register int				*gcount;
-	register int				*lcount;
-	register thread_t			new_thread;
-	register int				state;
-	register processor_set_t 	pset;
-	ast_t						*myast = ast_pending();
-
-	processor = current_processor();
+	processor_set_t		pset = processor->processor_set;
+	thread_t			new_thread;
+	int					state;
 
-	threadp = &processor->next_thread;
-	lcount = &processor->runq.count;
-	gcount = &processor->processor_set->runq.count;
+	(void)splsched();
 
+#ifdef __ppc__
+	pmsDown();					/* Step power down */
+#endif
 
-	(void)splsched();			/* Turn interruptions off */
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_START, (int)thread, 0, 0, 0, 0);
 
-	pmsDown();					/* Step power down.  Note: interruptions must be disabled for this call */
+	timer_switch(&PROCESSOR_DATA(processor, system_state),
+									mach_absolute_time(), &PROCESSOR_DATA(processor, idle_state));
+	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, idle_state);
 
-	while (	(*threadp == THREAD_NULL)				&&
-				(*gcount == 0) && (*lcount == 0)	) {
+	while (processor->next_thread == THREAD_NULL && processor->runq.count == 0 &&
+				(thread == THREAD_NULL || ((thread->state & (TH_WAIT|TH_SUSP)) == TH_WAIT && !thread->wake_active))) {
+		volatile processor_t	hint;
 
-		/* check for ASTs while we wait */
-		if (*myast &~ (AST_SCHEDULING | AST_BSD)) {
-			/* no ASTs for us */
-			*myast &= AST_NONE;
-			(void)spllo();
-		}
-		else
-			machine_idle();
+		machine_idle();
 
 		(void)splsched();
+
+		if (pset->low_hint == PROCESSOR_NULL)
+			break;
+
+		hint = pset->high_hint;
+		if (hint != PROCESSOR_NULL && hint->runq.count > 0)
+			break;
 	}
 
-	/*
-	 *	This is not a switch statement to avoid the
-	 *	bounds checking code in the common case.
-	 */
-	pset = processor->processor_set;
-	simple_lock(&pset->sched_lock);
+	KERNEL_DEBUG_CONSTANT(
+		MACHDBG_CODE(DBG_MACH_SCHED,MACH_IDLE) | DBG_FUNC_END, (int)thread, 0, 0, 0, 0);
+
+	timer_switch(&PROCESSOR_DATA(processor, idle_state),
+									mach_absolute_time(), &PROCESSOR_DATA(processor, system_state));
+	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
 
-	pmsStep(0);					/* Step up out of idle power, may start timer for next step */
+	pset_lock(pset);
+
+#ifdef __ppc__
+	pmsStep(0);					/* Step up out of idle power */
+#endif
 
 	state = processor->state;
 	if (state == PROCESSOR_DISPATCHING) {
 		/*
 		 *	Commmon case -- cpu dispatched.
 		 */
-		new_thread = *threadp;
-		*threadp = (volatile thread_t) THREAD_NULL;
+		new_thread = processor->next_thread;
+		processor->next_thread = THREAD_NULL;
 		processor->state = PROCESSOR_RUNNING;
-		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
-
-		if (	pset->runq.highq >= BASEPRI_RTQUEUES			&&
-				new_thread->sched_pri >= BASEPRI_RTQUEUES		) {
-			register run_queue_t	runq = &pset->runq;
-			register queue_t		q;
-
-			q = runq->queues + runq->highq;
-			if (((thread_t)q->next)->realtime.deadline <
-											processor->deadline) {
-				thread_t	thread = new_thread;
-
-				new_thread = (thread_t)q->next;
-				((queue_entry_t)new_thread)->next->prev = q;
-				q->next = ((queue_entry_t)new_thread)->next;
-				new_thread->runq = RUN_QUEUE_NULL;
-				processor->deadline = new_thread->realtime.deadline;
-				assert(new_thread->sched_mode & TH_MODE_PREEMPT);
-				runq->count--; runq->urgency--;
-				if (queue_empty(q)) {
-					if (runq->highq != IDLEPRI)
-						clrbit(MAXPRI - runq->highq, runq->bitmap);
-					runq->highq = MAXPRI - ffsbit(runq->bitmap);
-				}
-				dispatch_counts.missed_realtime++;
-				simple_unlock(&pset->sched_lock);
-
-				thread_lock(thread);
-				thread_setrun(thread, SCHED_HEADQ);
-				thread_unlock(thread);
-
-				counter(c_idle_thread_handoff++);
-				thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-				/*NOTREACHED*/
-			}
-			simple_unlock(&pset->sched_lock);
 
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/*NOTREACHED*/
-		}
-
-		if (	processor->runq.highq > new_thread->sched_pri		||
-				pset->runq.highq > new_thread->sched_pri				) {
-			thread_t	thread = new_thread;
+		if (	processor->runq.highq > new_thread->sched_pri	||
+				rt_runq.highq >= new_thread->sched_pri			) {
+			processor->deadline = UINT64_MAX;
 
-			new_thread = choose_thread(pset, processor);
-			dispatch_counts.missed_other++;
-			simple_unlock(&pset->sched_lock);
+			pset_unlock(pset);
 
-			thread_lock(thread);
-			thread_setrun(thread, SCHED_HEADQ);
-			thread_unlock(thread);
+			thread_lock(new_thread);
+			thread_setrun(new_thread, SCHED_HEADQ);
+			thread_unlock(new_thread);
 
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/* NOTREACHED */
+			return (THREAD_NULL);
 		}
-		else {
-			simple_unlock(&pset->sched_lock);
 
-			counter(c_idle_thread_handoff++);
-			thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
-			/* NOTREACHED */
-		}
+		pset_unlock(pset);
+
+		return (new_thread);
 	}
 	else
 	if (state == PROCESSOR_IDLE) {
-		/*
-		 *	Processor was not dispatched (Rare).
-		 *	Set it running again and force a
-		 *	reschedule.
-		 */
-		no_dispatch_count++;
-		pset->idle_count--;
 		remqueue(&pset->idle_queue, (queue_entry_t)processor);
-		processor->state = PROCESSOR_RUNNING;
-		enqueue_tail(&pset->active_queue, (queue_entry_t)processor);
-		simple_unlock(&pset->sched_lock);
+		pset->idle_count--;
 
-		counter(c_idle_thread_block++);
-		thread_block((thread_continue_t)idle_thread);
-		/* NOTREACHED */
+		processor->state = PROCESSOR_RUNNING;
+		enqueue_head(&pset->active_queue, (queue_entry_t)processor);
 	}
 	else
 	if (state == PROCESSOR_SHUTDOWN) {
@@ -2663,26 +2860,38 @@ idle_thread(void)
 		 *	Going off-line.  Force a
 		 *	reschedule.
 		 */
-		if ((new_thread = (thread_t)*threadp) != THREAD_NULL) {
-			*threadp = (volatile thread_t) THREAD_NULL;
+		if ((new_thread = processor->next_thread) != THREAD_NULL) {
+			processor->next_thread = THREAD_NULL;
 			processor->deadline = UINT64_MAX;
-			simple_unlock(&pset->sched_lock);
+
+			pset_unlock(pset);
 
 			thread_lock(new_thread);
 			thread_setrun(new_thread, SCHED_HEADQ);
 			thread_unlock(new_thread);
-		}
-		else
-			simple_unlock(&pset->sched_lock);
 
-		counter(c_idle_thread_block++);
-		thread_block((thread_continue_t)idle_thread);
-		/* NOTREACHED */
+			return (THREAD_NULL);
+		}
 	}
 
-	simple_unlock(&pset->sched_lock);
+	pset_unlock(pset);
+
+	return (THREAD_NULL);
+}
+
+void
+idle_thread(void)
+{
+	processor_t		processor = current_processor();
+	thread_t		new_thread;
+
+	new_thread = processor_idle(THREAD_NULL, processor);
+	if (new_thread != THREAD_NULL) {
+		thread_run(processor->idle_thread, (thread_continue_t)idle_thread, NULL, new_thread);
+		/*NOTREACHED*/
+	}
 
-	panic("idle_thread: state %d\n", processor->state);
+	thread_block((thread_continue_t)idle_thread);
 	/*NOTREACHED*/
 }
 
@@ -2773,6 +2982,9 @@ sched_tick_continue(void)
 	 */
 	thread_update_scan();
 
+	if (pm_tick_callout != NULL)
+	    (*pm_tick_callout)();
+
 	clock_deadline_for_periodic_event(sched_tick_interval, abstime,
 														&sched_tick_deadline);
 
@@ -2886,41 +3098,38 @@ runq_scan(
 static void
 thread_update_scan(void)
 {
-	register boolean_t			restart_needed;
-	register processor_set_t	pset = &default_pset;
-	register processor_t		processor;
-	register thread_t			thread;
-	spl_t						s;
+	boolean_t			restart_needed = FALSE;
+	processor_t			processor = processor_list;
+	processor_set_t		pset;
+	thread_t			thread;
+	spl_t				s;
 
 	do {
-		s = splsched();
-		simple_lock(&pset->sched_lock);
-	    restart_needed = runq_scan(&pset->runq);
-		simple_unlock(&pset->sched_lock);
-
-		if (!restart_needed) {
-			simple_lock(&pset->sched_lock);
-			processor = (processor_t)queue_first(&pset->processors);
-			while (!queue_end(&pset->processors, (queue_entry_t)processor)) {
-				if ((restart_needed = runq_scan(&processor->runq)) != 0)
-					break;
+		do {
+			pset = processor->processor_set;
 
-				thread = processor->idle_thread;
-				if (thread->sched_stamp != sched_tick) {
-					if (thread_update_count == THREAD_UPDATE_SIZE) {
-						restart_needed = TRUE;
-						break;
-					}
+			s = splsched();
+			pset_lock(pset);
 
-					thread_update_array[thread_update_count++] = thread;
-					thread_reference_internal(thread);
+			restart_needed = runq_scan(&processor->runq);
+
+			pset_unlock(pset);
+			splx(s);
+
+			if (restart_needed)
+				break;
+
+			thread = processor->idle_thread;
+			if (thread != THREAD_NULL && thread->sched_stamp != sched_tick) {
+				if (thread_update_count == THREAD_UPDATE_SIZE) {
+					restart_needed = TRUE;
+					break;
 				}
 
-				processor = (processor_t)queue_next(&processor->processors);
+				thread_update_array[thread_update_count++] = thread;
+				thread_reference_internal(thread);
 			}
-			simple_unlock(&pset->sched_lock);
-		}
-		splx(s);
+		} while ((processor = processor->processor_list) != NULL);
 
 	    /*
 	     *	Ok, we now have a collection of candidates -- fix them.
@@ -2987,9 +3196,9 @@ db_sched(void)
 #if	MACH_COUNTERS
 	iprintf("Thread block:  calls %d\n",
 		c_thread_block_calls);
-	iprintf("Idle thread:\n\thandoff %d block %d no_dispatch %d\n",
+	iprintf("Idle thread:\n\thandoff %d block %d\n",
 		c_idle_thread_handoff,
-		c_idle_thread_block, no_dispatch_count);
+		c_idle_thread_block);
 	iprintf("Sched thread blocks:  %d\n", c_sched_thread_block);
 #endif	/* MACH_COUNTERS */
 	db_indent -= 2;
diff --git a/osfmk/kern/sched_prim.h b/osfmk/kern/sched_prim.h
index 929795787..6cdde8933 100644
--- a/osfmk/kern/sched_prim.h
+++ b/osfmk/kern/sched_prim.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -71,7 +77,7 @@
 #ifdef	MACH_KERNEL_PRIVATE
 
 /* Initialization */
-extern void		sched_init(void);
+extern void		sched_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern void		sched_startup(void);
 
@@ -89,10 +95,6 @@ extern void			thread_unstop(
 extern void			thread_wait(
 						thread_t	thread);
 
-/* Select a thread to run */
-extern thread_t		thread_select(
-						processor_t		myprocessor);
-
 /* Unblock thread on wake up */
 extern boolean_t	thread_unblock(
 						thread_t		thread,
@@ -103,26 +105,10 @@ extern kern_return_t	thread_go(
 						 	thread_t		thread,
 							wait_result_t	wresult);
 
-/* Context switch primitive */
-extern boolean_t	thread_invoke(
-						thread_t			old_thread,
-						thread_t			new_thread,
-						ast_t				reason);
-
-/* Perform calculations for thread finishing execution */
-extern void			thread_done(
-						thread_t		old_thread,
-						thread_t		new_thread,
-						processor_t		processor);
-
-/* Set up for thread beginning execution */
-extern void			thread_begin(
-						thread_t		thread,
-						processor_t		processor);
-
-/* Handle previous thread at context switch */
+/* Handle threads at context switch */
 extern void			thread_dispatch(
-						thread_t		thread);
+						thread_t		old_thread,
+						thread_t		new_thread);
 
 /* Switch directly to a particular thread */
 extern int			thread_run(
@@ -191,15 +177,17 @@ extern void		thread_setrun(
 					thread_t	thread,
 					integer_t	options);
 
-#define SCHED_TAILQ		0
-#define SCHED_HEADQ		1
-#define SCHED_PREEMPT	2
+#define SCHED_TAILQ		1
+#define SCHED_HEADQ		2
+#define SCHED_PREEMPT	4
 
-/* Bind thread to a particular processor */
+/* Bind the current thread to a particular processor */
 extern processor_t		thread_bind(
-							thread_t		thread,
 							processor_t		processor);
 
+extern void		run_queue_init(
+					run_queue_t		runq);
+
 extern void		thread_timer_expire(
 					void			*thread,
 					void			*p1);
@@ -235,7 +223,7 @@ extern kern_return_t clear_wait(
 						wait_result_t	result);
 
 /* Return from exception (BSD-visible interface) */
-extern void		thread_exception_return(void);
+extern void		thread_exception_return(void) __dead2;
 
 #endif	/* XNU_KERNEL_PRIVATE */
 
diff --git a/osfmk/kern/security.c b/osfmk/kern/security.c
new file mode 100644
index 000000000..b638479e9
--- /dev/null
+++ b/osfmk/kern/security.c
@@ -0,0 +1,425 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2005-2007 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <mach/message.h>
+#include <kern/kern_types.h>
+#include <kern/ipc_kobject.h>
+#include <ipc/ipc_object.h>
+#include <ipc/ipc_right.h>
+#include <ipc/ipc_labelh.h>
+#include <kern/task.h>
+#include <security/mac_mach_internal.h> 
+#include <mach/security.h> 
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mach_get_task_label(
+	ipc_space_t	 space,
+	mach_port_name_t *outlabel)
+{
+	kern_return_t	kr;
+	ipc_labelh_t	lh;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	lh = space->is_task->label;
+	ip_lock(lh->lh_port);
+	lh->lh_port->ip_mscount++;
+	lh->lh_port->ip_srights++;
+	ip_reference(lh->lh_port);
+	ip_unlock(lh->lh_port);
+	kr = ipc_object_copyout(space, (ipc_object_t) lh->lh_port,
+	    MACH_MSG_TYPE_PORT_SEND, 0, outlabel);
+	if (kr != KERN_SUCCESS) {
+		ip_lock(lh->lh_port);
+		ip_release(lh->lh_port);
+		ip_check_unlock(lh->lh_port);
+		*outlabel = MACH_PORT_NULL;
+	}
+  
+	return (KERN_SUCCESS);
+}
+#else
+kern_return_t
+mach_get_task_label(
+	ipc_space_t	 space __unused,
+	mach_port_name_t *outlabel __unused)
+{
+	return KERN_FAILURE;
+}
+#endif
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mach_get_task_label_text(
+	ipc_space_t	space,
+	labelstr_t	policies,
+	labelstr_t	outl)
+{
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	tasklabel_lock(space->is_task);
+	mac_task_label_externalize(&space->is_task->maclabel, policies, outl,
+	    512, 0);
+	tasklabel_unlock(space->is_task);
+  
+	return KERN_SUCCESS;
+}
+#else
+kern_return_t
+mach_get_task_label_text(
+	ipc_space_t	space __unused,
+	labelstr_t	policies __unused,
+	labelstr_t	outl __unused)
+{
+	return KERN_FAILURE;
+}
+#endif
+
+#if CONFIG_MACF_MACH
+int
+mac_task_check_service(
+	task_t       self,
+	task_t       obj,
+	const char * perm)
+{
+	tasklabel_lock2(self, obj);
+
+	int rc = mac_port_check_service(
+		&self->maclabel, &obj->maclabel, 
+		"mach_task", perm);
+
+	tasklabel_unlock2(self, obj);
+
+	return rc;
+}
+#else
+int
+mac_task_check_service(
+	task_t       self __unused,
+	task_t       obj __unused,
+	const char * perm __unused)
+{
+	return KERN_SUCCESS;
+}
+#endif
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mac_check_service(
+	__unused ipc_space_t space,
+	labelstr_t  subj,
+	labelstr_t  obj,
+	labelstr_t  serv,
+	labelstr_t  perm)
+{
+	struct label subjl, objl;
+
+	mac_task_label_init(&subjl);
+	int rc = mac_port_label_internalize(&subjl, subj);
+	if (rc) {
+		mac_task_label_destroy(&subjl);
+		return KERN_INVALID_ARGUMENT;
+	}
+	mac_task_label_init(&objl);
+	rc = mac_port_label_internalize(&objl, obj);
+	if (rc) {
+		mac_task_label_destroy(&subjl);
+		mac_task_label_destroy(&objl);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	rc = mac_port_check_service(&subjl, &objl, serv, perm);
+	mac_task_label_destroy(&subjl);
+	mac_task_label_destroy(&objl);
+
+	return rc ? KERN_NO_ACCESS : KERN_SUCCESS;
+}
+#else
+kern_return_t
+mac_check_service(
+	__unused ipc_space_t space,
+	__unused labelstr_t  subj,
+	__unused labelstr_t  obj,
+	__unused labelstr_t  serv,
+	__unused labelstr_t  perm)
+{
+	return KERN_FAILURE;
+}
+#endif 
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mac_port_check_service_obj(
+	ipc_space_t      space,
+	labelstr_t       subj,
+	mach_port_name_t obj,
+	labelstr_t       serv,
+	labelstr_t       perm)
+{
+	struct label  subjl;
+	ipc_entry_t   entry;
+	ipc_object_t  objp;
+	kern_return_t kr;
+	struct label  *objl;
+	int	      dead;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	if (!MACH_PORT_VALID(obj))
+		return KERN_INVALID_NAME;
+
+	mac_task_label_init(&subjl);
+	int rc = mac_port_label_internalize(&subjl, subj);
+	if (rc) {
+		mac_task_label_destroy(&subjl);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	kr = ipc_right_lookup_write(space, obj, &entry);
+	if (kr != KERN_SUCCESS) {
+		mac_task_label_destroy(&subjl);
+		return kr;
+	}
+
+	dead = ipc_right_check(space, (ipc_port_t) entry->ie_object, obj, entry);
+	if (dead) {
+		is_write_unlock(space);
+		mac_task_label_destroy(&subjl);
+		return KERN_INVALID_RIGHT;
+	}
+
+	objp = entry->ie_object;
+	io_lock (objp);
+	is_write_unlock (space);
+
+	objl = io_getlabel(objp);
+	if (objl == NULL) {
+		io_unlock(objp);
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	rc = mac_port_check_service(&subjl, objl, serv, perm);
+	io_unlocklabel(objp);
+	io_unlock (objp);
+
+	mac_task_label_destroy(&subjl);
+	return rc ? KERN_NO_ACCESS : KERN_SUCCESS;
+}
+#else
+kern_return_t
+mac_port_check_service_obj(
+	__unused ipc_space_t      space,
+	__unused labelstr_t       subj,
+	__unused mach_port_name_t obj,
+	__unused labelstr_t       serv,
+	__unused labelstr_t       perm)
+{
+	return KERN_FAILURE;
+}
+#endif
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mac_port_check_access(
+	ipc_space_t      space,
+	mach_port_name_t sub,
+	mach_port_name_t obj,
+	labelstr_t       serv,
+	labelstr_t       perm)
+{
+	ipc_entry_t    subi, obji;
+	ipc_object_t   subp, objp;
+	kern_return_t  kr;
+	struct label  *objl, *subl;
+	int            rc;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	if (!MACH_PORT_VALID(obj) || !MACH_PORT_VALID(sub))
+		return KERN_INVALID_NAME;
+
+	kr = ipc_right_lookup_two_write(space, obj, &obji, sub, &subi);
+	if (kr != KERN_SUCCESS)
+		return kr;
+
+	objp = obji->ie_object;
+	subp = subi->ie_object;
+
+	ipc_port_multiple_lock(); /* serialize (not necessary for LH, but simpler) */
+	io_lock(objp);
+	io_lock(subp);
+	is_write_unlock (space);
+
+	objl = io_getlabel(objp);
+	if (objl == NULL)
+		goto errout;
+	subl = io_getlabel(subp);
+	if (subl == NULL)
+		goto errout;
+
+	rc = mac_port_check_service(subl, objl, serv, perm);
+	io_unlocklabel(subp);
+	io_unlock(subp);
+	io_unlocklabel(objp);
+	io_unlock(objp);
+	ipc_port_multiple_unlock();
+
+	return rc ? KERN_NO_ACCESS : KERN_SUCCESS;
+
+errout:
+	io_unlocklabel(subp);
+	io_unlock(subp);
+	io_unlocklabel(objp);
+	io_unlock(objp);
+	ipc_port_multiple_unlock();
+	return KERN_INVALID_ARGUMENT;
+}
+#else
+kern_return_t
+mac_port_check_access(
+	__unused ipc_space_t      space,
+	__unused mach_port_name_t sub,
+	__unused mach_port_name_t obj,
+	__unused labelstr_t       serv,
+	__unused labelstr_t       perm)
+{
+	return KERN_FAILURE;
+}
+#endif
+
+#if CONFIG_MACF_MACH
+kern_return_t
+mac_request_label(
+	ipc_space_t      space,
+	mach_port_name_t sub,
+	mach_port_name_t obj,
+	labelstr_t       serv,
+	mach_port_name_t *outlabel)
+{
+	ipc_entry_t    subi, obji;
+	ipc_object_t   subp, objp;
+	kern_return_t  kr;
+	struct label  *objl, *subl, outl;
+	int            rc;
+
+	if (space == IS_NULL || space->is_task == NULL)
+		return KERN_INVALID_TASK;
+
+	if (!MACH_PORT_VALID(obj) || !MACH_PORT_VALID(sub))
+		return KERN_INVALID_NAME;
+
+	kr = ipc_right_lookup_two_write(space, obj, &obji, sub, &subi);
+	if (kr != KERN_SUCCESS)
+		return kr;
+
+	objp = obji->ie_object;
+	subp = subi->ie_object;
+
+	ipc_port_multiple_lock(); /* serialize (not necessary for LH, but simpler) */
+	io_lock(objp);
+	io_lock(subp);
+	is_write_unlock (space);
+
+	objl = io_getlabel(objp);
+	if (objl == NULL)
+		goto errout;
+	subl = io_getlabel(subp);
+	if (subl == NULL)
+		goto errout;
+
+	mac_port_label_init(&outl);
+	rc = mac_port_label_compute(subl, objl, serv, &outl);
+	io_unlocklabel(subp);
+	io_unlock(subp);
+	io_unlocklabel(objp);
+	io_unlock(objp);
+	ipc_port_multiple_unlock();
+
+	if (rc == 0)
+		kr = labelh_new_user(space, &outl, outlabel);
+	else
+		kr = KERN_NO_ACCESS;
+
+	if (kr != KERN_SUCCESS)
+		mac_port_label_destroy(&outl);
+
+	return kr;
+
+errout:
+	io_unlocklabel(subp);
+	io_unlock(subp);
+	io_unlocklabel(objp);
+	io_unlock(objp);
+	ipc_port_multiple_unlock();
+	return KERN_INVALID_ARGUMENT;
+}
+#else /* !MAC_MACH */
+
+kern_return_t
+mac_request_label(
+	__unused ipc_space_t      space,
+	__unused mach_port_name_t sub,
+	__unused mach_port_name_t obj,
+	__unused labelstr_t       serv,
+	__unused mach_port_name_t *outlabel)
+{
+	return KERN_FAILURE;
+}
+
+#endif /* MAC_MACH */
diff --git a/osfmk/kern/simple_lock.h b/osfmk/kern/simple_lock.h
index ad451134e..576212175 100644
--- a/osfmk/kern/simple_lock.h
+++ b/osfmk/kern/simple_lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Copyright (C) 1998 Apple Computer
@@ -96,25 +102,41 @@ extern unsigned int		hw_lock_held(
 __BEGIN_DECLS
 
 extern uint32_t			hw_atomic_add(
-					uint32_t	*dest,
+					volatile uint32_t	*dest,
 					uint32_t	delt);
 
 extern uint32_t			hw_atomic_sub(
-					uint32_t	*dest,
+					volatile uint32_t	*dest,
 					uint32_t	delt);
 
 extern uint32_t			hw_atomic_or(
-					uint32_t	*dest,
+					volatile uint32_t	*dest,
 					uint32_t	mask);
 
 extern uint32_t			hw_atomic_and(
-					uint32_t	*dest,
+					volatile uint32_t	*dest,
+					uint32_t	mask);
+
+/*
+ * Variant of hw_atomic_or which doesn't return a value; potentially
+ * more efficient on some platforms.
+ */
+extern void			hw_atomic_or_noret(
+					volatile uint32_t	*dest,
+					uint32_t	mask);
+/*
+ * Variant of hw_atomic_and which doesn't return a value; potentially
+ * more efficient on some platforms.
+ */
+
+extern void			hw_atomic_and_noret(
+					volatile uint32_t	*dest,
 					uint32_t	mask);
 
 extern uint32_t			hw_compare_and_store(
 					uint32_t	oldval,
 					uint32_t	newval,
-					uint32_t	*dest);
+					volatile uint32_t	*dest);
 
 extern void			hw_queue_atomic(
 					unsigned int *anchor,
diff --git a/osfmk/kern/spl.c b/osfmk/kern/spl.c
index 15d7cf72e..9036a5d83 100644
--- a/osfmk/kern/spl.c
+++ b/osfmk/kern/spl.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <kern/thread.h>
 #include <kern/lock.h>
diff --git a/osfmk/kern/spl.h b/osfmk/kern/spl.h
index f19a3fa59..ac615ff05 100644
--- a/osfmk/kern/spl.h
+++ b/osfmk/kern/spl.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef	_KERN_SPL_H_
 #define	_KERN_SPL_H_
diff --git a/osfmk/kern/stack.c b/osfmk/kern/stack.c
index ee1200b10..68cdcabc0 100644
--- a/osfmk/kern/stack.c
+++ b/osfmk/kern/stack.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	Kernel stack management routines.
@@ -52,7 +58,6 @@ decl_simple_lock_data(static,stack_lock_data)
 
 #define STACK_CACHE_SIZE	2
 
-static vm_map_t			stack_map;
 static vm_offset_t		stack_free_list;
 
 static unsigned int		stack_free_count, stack_free_hiwat;		/* free list count */
@@ -75,27 +80,12 @@ static vm_offset_t		stack_addr_mask;
 void
 stack_init(void)
 {
-	vm_offset_t			stacks, boundary;
-	vm_map_offset_t		map_addr;
-
 	simple_lock_init(&stack_lock_data, 0);
 	
 	if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE))
 		panic("stack_init: stack size %d not a multiple of page size %d\n",	KERNEL_STACK_SIZE, PAGE_SIZE);
 	
-	for (boundary = PAGE_SIZE; boundary <= KERNEL_STACK_SIZE; )
-		boundary <<= 1;
-
-	stack_addr_mask = boundary - 1;
-
-	if (kmem_suballoc(kernel_map, &stacks, (boundary * (2 * THREAD_MAX + 64)),
-								FALSE, VM_FLAGS_ANYWHERE, &stack_map) != KERN_SUCCESS)
-		panic("stack_init: kmem_suballoc");
-
-	map_addr = vm_map_min(stack_map);
-	if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, (VM_MAKE_TAG(VM_MEMORY_STACK) | VM_FLAGS_FIXED),
-						VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_DEFAULT) != KERN_SUCCESS)
-		panic("stack_init: vm_map_enter");
+	stack_addr_mask = KERNEL_STACK_SIZE - 1;
 }
 
 /*
@@ -110,6 +100,7 @@ stack_alloc(
 {
 	vm_offset_t		stack;
 	spl_t			s;
+	int			guard_flags;
 
 	assert(thread->kernel_stack == 0);
 
@@ -130,8 +121,27 @@ stack_alloc(
 	splx(s);
 		
 	if (stack == 0) {
-		if (kernel_memory_allocate(stack_map, &stack, KERNEL_STACK_SIZE, stack_addr_mask, KMA_KOBJECT) != KERN_SUCCESS)
+
+		/*
+		 * Request guard pages on either side of the stack.  Ask
+		 * kernel_memory_allocate() for two extra pages to account
+		 * for these.
+		 */
+
+		guard_flags = KMA_GUARD_FIRST | KMA_GUARD_LAST;
+		if (kernel_memory_allocate(kernel_map, &stack,
+					   KERNEL_STACK_SIZE + (2*PAGE_SIZE),
+					   stack_addr_mask,
+					   KMA_KOBJECT | guard_flags)
+		    != KERN_SUCCESS)
 			panic("stack_alloc: kernel_memory_allocate");
+
+		/*
+		 * The stack address that comes back is the address of the lower
+		 * guard page.  Skip past it to get the actual stack base address.
+		 */
+
+		stack += PAGE_SIZE;
 	}
 
 	machine_stack_attach(thread, stack);
@@ -149,28 +159,8 @@ stack_free(
     vm_offset_t		stack = machine_stack_detach(thread);
 
 	assert(stack);
-	if (stack != thread->reserved_stack) {
-		struct stack_cache	*cache;
-		spl_t				s;
-
-		s = splsched();
-		cache = &PROCESSOR_DATA(current_processor(), stack_cache);
-		if (cache->count < STACK_CACHE_SIZE) {
-			stack_next(stack) = cache->free;
-			cache->free = stack;
-			cache->count++;
-		}
-		else {
-			stack_lock();
-			stack_next(stack) = stack_free_list;
-			stack_free_list = stack;
-			if (++stack_free_count > stack_free_hiwat)
-				stack_free_hiwat = stack_free_count;
-			stack_free_delta++;
-			stack_unlock();
-		}
-		splx(s);
-	}
+	if (stack != thread->reserved_stack)
+		stack_free_stack(stack);
 }
 
 void
@@ -272,9 +262,24 @@ stack_collect(void)
 			stack_unlock();
 			splx(s);
 
-			if (vm_map_remove(stack_map, vm_map_trunc_page(stack),
-								vm_map_round_page(stack + KERNEL_STACK_SIZE), VM_MAP_REMOVE_KUNWIRE) != KERN_SUCCESS)
+			/*
+			 * Get the stack base address, then decrement by one page
+			 * to account for the lower guard page.  Add two extra pages
+			 * to the size to account for the guard pages on both ends
+			 * that were originally requested when the stack was allocated
+			 * back in stack_alloc().
+			 */
+
+			stack = vm_map_trunc_page(stack);
+			stack -= PAGE_SIZE;
+			if (vm_map_remove(
+				    kernel_map,
+				    stack,
+				    stack + KERNEL_STACK_SIZE+(2*PAGE_SIZE),
+				    VM_MAP_REMOVE_KUNWIRE)
+			    != KERN_SUCCESS)
 				panic("stack_collect: vm_map_remove");
+			stack = 0;
 
 			s = splsched();
 			stack_lock();
@@ -377,7 +382,7 @@ processor_set_stack_usage(
 	vm_size_t maxusage;
 	vm_offset_t maxstack;
 
-	register thread_t *threads;
+	register thread_t *thread_list;
 	register thread_t thread;
 
 	unsigned int actual;	/* this many things */
@@ -386,19 +391,16 @@ processor_set_stack_usage(
 	vm_size_t size, size_needed;
 	void *addr;
 
-	if (pset == PROCESSOR_SET_NULL)
+	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
 		return KERN_INVALID_ARGUMENT;
 
-	size = 0; addr = 0;
+	size = 0;
+	addr = NULL;
 
 	for (;;) {
-		pset_lock(pset);
-		if (!pset->active) {
-			pset_unlock(pset);
-			return KERN_INVALID_ARGUMENT;
-		}
+		mutex_lock(&tasks_threads_lock);
 
-		actual = pset->thread_count;
+		actual = threads_count;
 
 		/* do we have the memory we need? */
 
@@ -406,8 +408,7 @@ processor_set_stack_usage(
 		if (size_needed <= size)
 			break;
 
-		/* unlock the pset and allocate more memory */
-		pset_unlock(pset);
+		mutex_unlock(&tasks_threads_lock);
 
 		if (size != 0)
 			kfree(addr, size);
@@ -420,18 +421,17 @@ processor_set_stack_usage(
 			return KERN_RESOURCE_SHORTAGE;
 	}
 
-	/* OK, have memory and the processor_set is locked & active */
-	threads = (thread_t *) addr;
-	for (i = 0, thread = (thread_t) queue_first(&pset->threads);
-					!queue_end(&pset->threads, (queue_entry_t) thread);
-					thread = (thread_t) queue_next(&thread->pset_threads)) {
+	/* OK, have memory and list is locked */
+	thread_list = (thread_t *) addr;
+	for (i = 0, thread = (thread_t) queue_first(&threads);
+					!queue_end(&threads, (queue_entry_t) thread);
+					thread = (thread_t) queue_next(&thread->threads)) {
 		thread_reference_internal(thread);
-		threads[i++] = thread;
+		thread_list[i++] = thread;
 	}
 	assert(i <= actual);
 
-	/* can unlock processor set now that we have the thread refs */
-	pset_unlock(pset);
+	mutex_unlock(&tasks_threads_lock);
 
 	/* calculate maxusage and free thread references */
 
@@ -439,7 +439,7 @@ processor_set_stack_usage(
 	maxusage = 0;
 	maxstack = 0;
 	while (i > 0) {
-		thread_t threadref = threads[--i];
+		thread_t threadref = thread_list[--i];
 
 		if (threadref->kernel_stack != 0)
 			total++;
@@ -461,10 +461,10 @@ processor_set_stack_usage(
 
 vm_offset_t min_valid_stack_address(void)
 {
-	return vm_map_min(stack_map);
+	return vm_map_min(kernel_map);
 }
 
 vm_offset_t max_valid_stack_address(void)
 {
-	return vm_map_max(stack_map);
+	return vm_map_max(kernel_map);
 }
diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c
index dc9a6a92a..3c132bb74 100644
--- a/osfmk/kern/startup.c
+++ b/osfmk/kern/startup.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 
@@ -57,7 +69,6 @@
 #include <debug.h>
 #include <xpr_debug.h>
 #include <mach_kdp.h>
-#include <mach_host.h>
 #include <norma_vm.h>
 
 #include <mach/boolean.h>
@@ -82,17 +93,21 @@
 #include <kern/zalloc.h>
 #include <kern/locks.h>
 #include <console/serial_protos.h>
-#include <vm/vm_shared_memory_server.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_init.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_shared_region.h>
 #include <machine/pmap.h>
 #include <machine/commpage.h>
 #include <libkern/version.h>
 
+#if CONFIG_MACF
+#include <security/mac_mach_internal.h>
+#endif
+
 #ifdef __ppc__
 #include <ppc/Firmware.h>
 #include <ppc/mappings.h>
@@ -102,11 +117,14 @@ static void		kernel_bootstrap_thread(void);
 
 static void		load_context(
 					thread_t	thread);
-
 #ifdef i386
-extern void	cpu_window_init(int);
+extern void cpu_userwindow_init(int);
+extern void cpu_physwindow_init(int);
 #endif
 
+#ifdef CONFIG_JETTISON_KERNEL_LINKER
+extern void	jettison_kernel_linker(void);
+#endif
 
 /*
  *	Running in virtual memory, on the interrupt stack.
@@ -118,23 +136,47 @@ kernel_bootstrap(void)
 	kern_return_t	result;
 	thread_t		thread;
 
+	printf("%s\n", version); /* log kernel version */
+
+#define kernel_bootstrap_kprintf(x...) /* kprintf("kernel_bootstrap: " x) */
+
+	kernel_bootstrap_kprintf("calling lck_mod_init\n");
 	lck_mod_init();
+
+	kernel_bootstrap_kprintf("calling sched_init\n");
 	sched_init();
+
+	kernel_bootstrap_kprintf("calling vm_mem_bootstrap\n");
 	vm_mem_bootstrap();
+
+	kernel_bootstrap_kprintf("calling ipc_bootstrap\n");
 	ipc_bootstrap();
+
+	kernel_bootstrap_kprintf("calling vm_mem_init\n");
 	vm_mem_init();
+
+	kernel_bootstrap_kprintf("calling kmod_init\n");
+	kmod_init();
+#if CONFIG_MACF
+	mac_policy_init();
+#endif
+	kernel_bootstrap_kprintf("calling ipc_init\n");
 	ipc_init();
 
 	/*
 	 * As soon as the virtual memory system is up, we record
 	 * that this CPU is using the kernel pmap.
 	 */
+	kernel_bootstrap_kprintf("calling PMAP_ACTIVATE_KERNEL\n");
 	PMAP_ACTIVATE_KERNEL(master_cpu);
 
+	kernel_bootstrap_kprintf("calling mapping_free_prime\n");
 	mapping_free_prime();						/* Load up with temporary mapping blocks */
 
+	kernel_bootstrap_kprintf("calling machine_init\n");
 	machine_init();
-	kmod_init();
+
+	kernel_bootstrap_kprintf("calling clock_init\n");
 	clock_init();
 
 	machine_info.memory_size = mem_size;
@@ -145,13 +187,19 @@ kernel_bootstrap(void)
 	/*
 	 *	Initialize the IPC, task, and thread subsystems.
 	 */
+	kernel_bootstrap_kprintf("calling ledger_init\n");
 	ledger_init();
+
+	kernel_bootstrap_kprintf("calling task_init\n");
 	task_init();
+
+	kernel_bootstrap_kprintf("calling thread_init\n");
 	thread_init();
 	
 	/*
 	 *	Create a kernel thread to execute the kernel bootstrap.
 	 */
+	kernel_bootstrap_kprintf("calling kernel_thread_create\n");
 	result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread);
 
 	if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n", result);
@@ -159,10 +207,13 @@ kernel_bootstrap(void)
 	thread->state = TH_RUN;
 	thread_deallocate(thread);
 
+	kernel_bootstrap_kprintf("calling load_context - done\n");
 	load_context(thread);
 	/*NOTREACHED*/
 }
 
+int kth_started = 0;
+
 /*
  * Now running in a thread.  Kick off other services,
  * invoke user bootstrap, enter pageout loop.
@@ -171,8 +222,9 @@ static void
 kernel_bootstrap_thread(void)
 {
 	processor_t		processor = current_processor();
-	thread_t		self = current_thread();
 
+#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
+	kernel_bootstrap_thread_kprintf("calling idle_thread_create\n");
 	/*
 	 * Create the idle processor thread.
 	 */
@@ -184,22 +236,26 @@ kernel_bootstrap_thread(void)
 	 *
 	 * Start up the scheduler services.
 	 */
+	kernel_bootstrap_thread_kprintf("calling sched_startup\n");
 	sched_startup();
 
 	/*
 	 * Remain on current processor as
 	 * additional processors come online.
 	 */
-	thread_bind(self, processor);
+	kernel_bootstrap_thread_kprintf("calling thread_bind\n");
+	thread_bind(processor);
 
 	/*
 	 * Kick off memory mapping adjustments.
 	 */
+	kernel_bootstrap_thread_kprintf("calling mapping_adjust\n");
 	mapping_adjust();
 
 	/*
 	 *	Create the clock service.
 	 */
+	kernel_bootstrap_thread_kprintf("calling clock_service_create\n");
 	clock_service_create();
 
 	/*
@@ -207,41 +263,58 @@ kernel_bootstrap_thread(void)
 	 */
 	device_service_create();
 
-	shared_file_boot_time_init(ENV_DEFAULT_ROOT, cpu_type());
+	kth_started = 1;
+
+#ifdef i386
+	/*
+	 * Create and initialize the physical copy window for processor 0
+	 * This is required before starting kicking off  IOKit.
+	 */
+	cpu_physwindow_init(0);
+#endif
 
 #ifdef	IOKIT
-	{
-		PE_init_iokit();
-	}
+	PE_init_iokit();
 #endif
 	
 	(void) spllo();		/* Allow interruptions */
 
+#ifdef i386
 	/*
-	 *	Fill in the comm area (mapped into every task address space.)
+	 * Create and initialize the copy window for processor 0
+	 * This also allocates window space for all other processors.
+	 * However, this is dependent on the number of processors - so this call
+	 * must be after IOKit has been started because IOKit performs processor
+	 * discovery.
 	 */
-	commpage_populate();
+	cpu_userwindow_init(0);
+#endif
 
-#ifdef i386
 	/*
-	 *	create and initialize a copy window
-	 * 	for processor 0
+	 *	Initialize the shared region module.
 	 */
-	cpu_window_init(0);
+	vm_shared_region_init();
+	vm_commpage_init();
+
+#if CONFIG_MACF
+	mac_policy_initmach();
 #endif
 
 	/*
 	 *	Start the user bootstrap.
 	 */
 #ifdef	MACH_BSD
-	{ 
-		bsd_init();
-	}
+	bsd_init();
+#endif
+
+#ifdef CONFIG_JETTISON_KERNEL_LINKER
+	/* We do not run kextd, so get rid of the kernel linker now */
+	jettison_kernel_linker();
 #endif
 
 	serial_keyboard_init();		/* Start serial keyboard if wanted */
 
-	thread_bind(self, PROCESSOR_NULL);
+	thread_bind(PROCESSOR_NULL);
 
 	/*
 	 *	Become the pageout daemon.
@@ -316,7 +389,13 @@ load_context(
 {
 	processor_t		processor = current_processor();
 
+
+#define load_context_kprintf(x...) /* kprintf("load_context: " x) */
+
+	load_context_kprintf("calling machine_set_current_thread\n");
 	machine_set_current_thread(thread);
+
+	load_context_kprintf("calling processor_up\n");
 	processor_up(processor);
 
 	PMAP_ACTIVATE_KERNEL(PROCESSOR_DATA(processor, slot_num));
@@ -326,7 +405,10 @@ load_context(
 	 * should never occur since the thread is expected
 	 * to have reserved stack.
 	 */
+	load_context_kprintf("stack %x, stackptr %x\n", 
+			      thread->kernel_stack, thread->machine.kstackptr);
 	if (!thread->kernel_stack) {
+		load_context_kprintf("calling stack_alloc_try\n");
 		if (!stack_alloc_try(thread))
 			panic("load_context");
 	}
@@ -336,7 +418,7 @@ load_context(
 	 * running for load calculations.
 	 */
 	if (!(thread->state & TH_IDLE))
-		pset_run_incr(thread->processor_set);
+		sched_run_incr();
 
 	processor->active_thread = thread;
 	processor->current_pri = thread->sched_pri;
@@ -344,11 +426,15 @@ load_context(
 	thread->last_processor = processor;
 
 	processor->last_dispatch = mach_absolute_time();
-	timer_switch((uint32_t)processor->last_dispatch,
-							&PROCESSOR_DATA(processor, offline_timer));
+	timer_start(&thread->system_timer, processor->last_dispatch);
+	PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
+
+	timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch);
+	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
 
 	PMAP_ACTIVATE_USER(thread, PROCESSOR_DATA(processor, slot_num));
 
+	load_context_kprintf("calling machine_load_context\n");
 	machine_load_context(thread);
 	/*NOTREACHED*/
 }
diff --git a/osfmk/kern/startup.h b/osfmk/kern/startup.h
index 45689973e..475160ce8 100644
--- a/osfmk/kern/startup.h
+++ b/osfmk/kern/startup.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -36,7 +42,7 @@ __BEGIN_DECLS
  */
 
 /* Initialize kernel */
-extern void		kernel_bootstrap(void);
+extern void kernel_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 /* Initialize machine dependent stuff */
 extern void	machine_init(void);
diff --git a/osfmk/kern/symbols.c b/osfmk/kern/symbols.c
new file mode 100644
index 000000000..0d1b3065b
--- /dev/null
+++ b/osfmk/kern/symbols.c
@@ -0,0 +1,228 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 2004 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <mach/mach_types.h>
+#include <mach-o/mach_header.h>
+#include <mach-o/nlist.h>
+#include <mach/kmod.h>
+#include <libsa/stdlib.h>
+#include <kern/misc_protos.h>
+
+static const struct nlist *
+syms_find(const struct nlist *syms, int nsyms, vm_offset_t addr,
+    vm_offset_t *ofs)
+{
+	const struct nlist *best = 0;
+	int                 i;
+
+	for (i = 0; i < nsyms; i++) {
+		int st = syms[i].n_type & N_TYPE;
+
+		if (st == N_SECT || st == N_ABS) {
+			if (syms[i].n_value == addr) {
+				*ofs = 0;
+				return (syms+i);
+			}
+			else if (syms[i].n_value < addr &&
+				 (best == 0 ||
+				  (syms[i].n_value > best->n_value))) {
+				*ofs = addr - syms[i].n_value;
+				best = syms+i;
+			}
+		}
+	}
+
+	return (best);
+}
+
+static const char *
+syms_getname(const struct symtab_command *sc, const char *ss,
+	     const struct nlist *sp)
+{
+	if (sp->n_un.n_strx == 0)
+		return ("");
+	else if ((unsigned)sp->n_un.n_strx > sc->strsize)
+		return ("*bad string*");
+	else
+		return (ss + sp->n_un.n_strx);
+}
+
+/* Search for a symbol in the given object file, which must either
+ * have a LINKEDIT segment or have been read directly into memory
+ * and isload passed as 1.
+
+ * If mh has a LINKEDIT segment, an object file loaded in the normal
+ * way will have the symbol table available at the load address.
+ */
+
+static const char *
+syms_nameforaddr1(const struct mach_header *mh, int isload,
+    vm_offset_t addr, vm_offset_t *ofs)
+{
+	const struct symtab_command  *sc = NULL;
+	const struct segment_command *le = NULL;
+	const struct segment_command *p;
+	const struct segment_command *sym = NULL;
+	const struct nlist           *syms;
+	const char                   *strings;
+	unsigned int                  i;
+
+	p = (const struct segment_command *) (&mh[1]);
+
+	for (i = 0; i < mh->ncmds; i++) {
+		if (p->cmd == LC_SYMTAB)
+			sc = (const struct symtab_command *) p;
+		else if (p->cmd == LC_SEGMENT &&
+		    !strncmp(p->segname, "__LINKEDIT", sizeof(p->segname)))
+			le = p;
+
+		/* only try to find a name for an address that came from
+		 * a text section.
+		 */
+		if (p->cmd == LC_SEGMENT &&
+		    addr >= p->vmaddr && addr < p->vmaddr + p->vmsize) {
+			unsigned int j;
+
+			const struct section *sp = (const struct section *)
+			    (((const char *) p) + sizeof(struct segment_command));
+
+			for (j = 0; j < p->nsects; j++) {
+				if (addr >= sp[j].addr &&
+				    addr <  sp[j].addr + sp[j].size &&
+				    !strncmp (sp[j].sectname, "__text",
+					    	sizeof(sp[j].sectname))) {
+					sym = p;
+					break;
+				}
+			}
+		}
+		p = (const struct segment_command *)
+			(((const char *) p) + p->cmdsize);
+	}
+
+	if (sc == 0 || sym == NULL)
+		return (NULL);
+
+	if (!isload) {
+		syms = (const struct nlist *) (((const char *) mh) + sc->symoff);
+		strings = ((const char *) mh) + sc->stroff;
+	}
+	else if (le) {
+		syms = (const struct nlist *) le->vmaddr;
+		strings = (const char *)
+			(le->vmaddr + sc->nsyms * sizeof(struct nlist));
+	} else
+		return (NULL);
+
+	const struct nlist *sp = syms_find(syms, sc->nsyms, addr, ofs);
+	if (sp)
+		return syms_getname(sc, strings, sp);
+
+	return (NULL);
+}
+
+extern struct mach_header  _mh_execute_header;
+extern kmod_info_t        *kmod;
+
+/* Search for a symbol and return the name, offset, and module in which the
+ * address was found. A null module means the kernel itself.
+ */
+
+const char *
+syms_nameforaddr(vm_offset_t addr, vm_offset_t *ofs, kmod_info_t **km)
+{
+	const char  *name = NULL;
+
+	name = syms_nameforaddr1(&_mh_execute_header, 1, addr, ofs);
+	if (name) {
+		*km = NULL;
+		return (name);
+	}
+
+	return (NULL);
+}
+
+int     snprintf(char *, size_t, const char *, ...);
+
+/* Format the results of calling syms_nameforaddr into a single string.
+ * The buffer must be at least 13 bytes long; 80 is recommended.
+ */
+
+int
+syms_formataddr(vm_offset_t addr, char *out, vm_offset_t outsize)
+{
+	vm_offset_t  ofs;
+	kmod_info_t *k = NULL;
+	const char  *name;
+
+	name = syms_nameforaddr(addr, &ofs, &k);
+
+	if (ofs > 0x100000)
+		name = NULL;
+
+	if (name != NULL) {
+		if (k != NULL)
+			snprintf(out, outsize, "0x%08X <%s:%s + %d>", addr, 
+			    k->name, name, ofs);
+		else
+			snprintf(out, outsize, "0x%08X <%s + %d>", addr, name,
+			    ofs);
+
+		return (1);
+	}
+	else {
+		snprintf(out, outsize, "0x%08X", addr);
+		return (0);
+	}
+}
diff --git a/osfmk/kern/sync_lock.c b/osfmk/kern/sync_lock.c
index 6dc0ec281..af059424f 100644
--- a/osfmk/kern/sync_lock.c
+++ b/osfmk/kern/sync_lock.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -140,7 +146,7 @@ lock_set_create (
 	if (task == TASK_NULL || n_ulocks <= 0 || policy > SYNC_POLICY_MAX)
 		return KERN_INVALID_ARGUMENT;
 
-	if (VM_MAX_ADDRESS/sizeof(struct ulock) - sizeof(struct lock_set) < (unsigned)n_ulocks)
+	if ((VM_MAX_ADDRESS - sizeof(struct lock_set))/sizeof(struct ulock) < (unsigned)n_ulocks)
 		return KERN_RESOURCE_SHORTAGE;
 
 	size = sizeof(struct lock_set) + (sizeof(struct ulock) * (n_ulocks-1));
diff --git a/osfmk/kern/sync_lock.h b/osfmk/kern/sync_lock.h
index a746d58a9..f4f499f1e 100644
--- a/osfmk/kern/sync_lock.h
+++ b/osfmk/kern/sync_lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/sync_sema.c b/osfmk/kern/sync_sema.c
index dc22aaa4e..9ddb7e909 100644
--- a/osfmk/kern/sync_sema.c
+++ b/osfmk/kern/sync_sema.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -87,6 +93,8 @@ semaphore_timedwait_signal_trap_internal(
 				clock_res_t nsec,
 				void (*caller_cont)(kern_return_t));
 
+kern_return_t
+semaphore_signal_internal_trap(mach_port_name_t sema_name);
 
 kern_return_t
 semaphore_signal_internal(
@@ -446,6 +454,13 @@ semaphore_signal_trap(
 	struct semaphore_signal_trap_args *args)
 {
 	mach_port_name_t sema_name = args->signal_name;
+
+	return (semaphore_signal_internal_trap(sema_name));
+}
+
+kern_return_t
+semaphore_signal_internal_trap(mach_port_name_t sema_name)
+{
 	semaphore_t	semaphore;
 	kern_return_t kr;
 
diff --git a/osfmk/kern/sync_sema.h b/osfmk/kern/sync_sema.h
index 8d50bcfa7..43bedc091 100644
--- a/osfmk/kern/sync_sema.h
+++ b/osfmk/kern/sync_sema.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -55,7 +61,7 @@ typedef struct semaphore {
 #define semaphore_lock(semaphore)   wait_queue_lock(&(semaphore)->wait_queue)
 #define semaphore_unlock(semaphore) wait_queue_unlock(&(semaphore)->wait_queue)
 
-extern void semaphore_init(void);
+extern void semaphore_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern	void		semaphore_reference	(semaphore_t semaphore);
 extern	void		semaphore_dereference	(semaphore_t semaphore);
diff --git a/osfmk/kern/syscall_emulation.c b/osfmk/kern/syscall_emulation.c
index 54f50e76b..c067cb0db 100644
--- a/osfmk/kern/syscall_emulation.c
+++ b/osfmk/kern/syscall_emulation.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -75,10 +81,10 @@
  */
 kern_return_t
 task_set_emulation_vector(
-	task_t 			task,
-	int			vector_start,
-	emulation_vector_t	emulation_vector,
-	mach_msg_type_number_t	emulation_vector_count)
+	__unused task_t 		task,
+	__unused int			vector_start,
+	__unused emulation_vector_t	emulation_vector,
+	__unused mach_msg_type_number_t	emulation_vector_count)
 {
 	return KERN_NOT_SUPPORTED;
 }
@@ -91,10 +97,10 @@ task_set_emulation_vector(
  */
 kern_return_t
 task_get_emulation_vector(
-	task_t			task,
-	int			*vector_start,			/* out */
-	emulation_vector_t	*emulation_vector,		/* out */
-	mach_msg_type_number_t	*emulation_vector_count)	/* out */
+	__unused task_t			task,
+	__unused int			*vector_start,		/* out */
+	__unused emulation_vector_t	*emulation_vector,	/* out */
+	__unused mach_msg_type_number_t	*emulation_vector_count)/* out */
 {
 	return KERN_NOT_SUPPORTED;
 }
@@ -105,9 +111,9 @@ task_get_emulation_vector(
  */
 kern_return_t
 task_set_emulation(
-	task_t		task,
-	vm_offset_t 	routine_entry_pt,
-	int		routine_number)
+	__unused task_t		task,
+	__unused vm_offset_t 	routine_entry_pt,
+	__unused int		routine_number)
 {
 	return KERN_NOT_SUPPORTED;
 }
diff --git a/osfmk/kern/syscall_subr.c b/osfmk/kern/syscall_subr.c
index b40d4a63d..f7855e114 100644
--- a/osfmk/kern/syscall_subr.c
+++ b/osfmk/kern/syscall_subr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -90,8 +96,7 @@ swtch_continue(void)
 
     disable_preemption();
 	myprocessor = current_processor();
-	result =		myprocessor->runq.count > 0					||
-				myprocessor->processor_set->runq.count > 0;
+	result = myprocessor->runq.count > 0 || rt_runq.count > 0;
 	enable_preemption();
 
 	thread_syscall_return(result);
@@ -107,8 +112,7 @@ swtch(
 
 	disable_preemption();
 	myprocessor = current_processor();
-	if (		myprocessor->runq.count == 0				&&
-			myprocessor->processor_set->runq.count == 0			) {
+	if (myprocessor->runq.count == 0 &&	rt_runq.count == 0) {
 		mp_enable_preemption();
 
 		return (FALSE);
@@ -121,8 +125,7 @@ swtch(
 
 	disable_preemption();
 	myprocessor = current_processor();
-	result =		myprocessor->runq.count > 0					||
-				myprocessor->processor_set->runq.count > 0;
+	result = myprocessor->runq.count > 0 || rt_runq.count > 0;
 	enable_preemption();
 
 	return (result);
@@ -138,8 +141,7 @@ swtch_pri_continue(void)
 
     disable_preemption();
 	myprocessor = current_processor();
-	result =		myprocessor->runq.count > 0					||
-				myprocessor->processor_set->runq.count > 0;
+	result = myprocessor->runq.count > 0 || rt_runq.count > 0;
 	mp_enable_preemption();
 
 	thread_syscall_return(result);
@@ -155,8 +157,7 @@ __unused	struct swtch_pri_args *args)
 
 	disable_preemption();
 	myprocessor = current_processor();
-	if (	myprocessor->runq.count == 0					&&
-			myprocessor->processor_set->runq.count == 0			) {
+	if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
 		mp_enable_preemption();
 
 		return (FALSE);
@@ -173,8 +174,7 @@ __unused	struct swtch_pri_args *args)
 
 	disable_preemption();
 	myprocessor = current_processor();
-	result =	myprocessor->runq.count > 0						||
-				myprocessor->processor_set->runq.count > 0;
+	result = myprocessor->runq.count > 0 || rt_runq.count > 0;
 	enable_preemption();
 
 	return (result);
@@ -236,7 +236,7 @@ thread_switch(
 			ipc_port_release(port);
 
 			if (thread == self) {
-				thread_deallocate_internal(thread);
+				(void)thread_deallocate_internal(thread);
 				thread = THREAD_NULL;
 			}
 		}
@@ -257,9 +257,9 @@ thread_switch(
 		thread_lock(thread);
 
 		/*
-		 *	Check if the thread is in the right pset,
-		 *	is not bound to a different processor,
-		 *	and that realtime is not involved.
+		 *	Check that the thread is not bound
+		 *	to a different processor, and that realtime
+		 *	is not involved.
 		 *
 		 *	Next, pull it off its run queue.  If it
 		 *	doesn't come, it's not eligible.
@@ -267,16 +267,15 @@ thread_switch(
 		processor = current_processor();
 		if (processor->current_pri < BASEPRI_RTQUEUES			&&
 			thread->sched_pri < BASEPRI_RTQUEUES				&&
-			thread->processor_set == processor->processor_set	&&
 			(thread->bound_processor == PROCESSOR_NULL	||
 			 thread->bound_processor == processor)				&&
-			run_queue_remove(thread) != RUN_QUEUE_NULL			) {
+				run_queue_remove(thread)							) {
 			/*
 			 *	Hah, got it!!
 			 */
 			thread_unlock(thread);
 
-			thread_deallocate_internal(thread);
+			(void)thread_deallocate_internal(thread);
 
 			if (option == SWITCH_OPTION_WAIT)
 				assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE,
@@ -445,3 +444,27 @@ thread_poll_yield(
 	}
 	splx(s);
 }
+
+
+void
+thread_yield_internal(
+	mach_msg_timeout_t	ms)
+{
+	processor_t	myprocessor;
+
+	disable_preemption();
+	myprocessor = current_processor();
+	if (myprocessor->runq.count == 0 && rt_runq.count == 0) {
+		mp_enable_preemption();
+
+		return;
+	}
+	enable_preemption();
+
+	thread_depress_ms(ms);
+
+	thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD);
+
+	thread_depress_abort_internal(current_thread());
+}
+
diff --git a/osfmk/kern/syscall_subr.h b/osfmk/kern/syscall_subr.h
index 92b39cc7e..5a68b926b 100644
--- a/osfmk/kern/syscall_subr.h
+++ b/osfmk/kern/syscall_subr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/syscall_sw.c b/osfmk/kern/syscall_sw.c
index 61fc5685d..4a8dd5e30 100644
--- a/osfmk/kern/syscall_sw.c
+++ b/osfmk/kern/syscall_sw.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/syscall_sw.h b/osfmk/kern/syscall_sw.h
index 72a8afad0..4aab31696 100644
--- a/osfmk/kern/syscall_sw.h
+++ b/osfmk/kern/syscall_sw.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -67,12 +73,16 @@ typedef struct {
 	int			(*mach_trap_function)(void);
 #if defined(__i386__)
 	boolean_t		mach_trap_stack;
-#else 
+#elif defined(__arm__)
+	/* no space */
+#else
 	mach_munge_t		*mach_trap_arg_munge32; /* system call arguments for 32-bit */
 	mach_munge_t		*mach_trap_arg_munge64; /* system call arguments for 64-bit */
 #endif
 #if	!MACH_ASSERT
+#if !defined(__arm__)
 	int			mach_trap_unused;
+#endif
 #else
 	const char*		mach_trap_name;
 #endif /* !MACH_ASSERT */
@@ -92,6 +102,14 @@ extern int			mach_trap_count;
 #define MACH_TRAP(name, arg_count, munge32, munge64)		\
 		{ (arg_count), (int (*)(void)) (name), FALSE, #name }
 #endif /* !MACH_ASSERT */
+#elif defined(__arm__)
+#if	!MACH_ASSERT
+#define	MACH_TRAP(name, arg_count, munge32, munge64)	\
+		{ (arg_count), (int (*)(void)) (name) }
+#else
+#define MACH_TRAP(name, arg_count, munge32, munge64)		\
+		{ (arg_count), (int (*)(void)) (name), #name }
+#endif /* !MACH_ASSERT */
 #else  /* !defined(__i386__) */
 #if	!MACH_ASSERT
 #define	MACH_TRAP(name, arg_count, munge32, munge64)	\
diff --git a/osfmk/kern/task.c b/osfmk/kern/task.c
index 5c288e37b..2a2fde2d8 100644
--- a/osfmk/kern/task.c
+++ b/osfmk/kern/task.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -72,10 +78,15 @@
  * improvements that they make and grant CSL redistribution rights.
  *
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 
 #include <mach_kdb.h>
-#include <mach_host.h>
-#include <mach_prof.h>
 #include <fast_tas.h>
 #include <platforms.h>
 
@@ -106,15 +117,15 @@
 #include <kern/host.h>
 #include <kern/clock.h>
 #include <kern/timer.h>
-#include <kern/profile.h>
 #include <kern/assert.h>
 #include <kern/sync_lock.h>
+#include <kern/affinity.h>
 
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>		/* for kernel_map, ipc_kernel_map */
 #include <vm/vm_pageout.h>
-#include <vm/vm_protos.h>	/* for vm_map_remove_commpage */
+#include <vm/vm_protos.h>
 
 #if	MACH_KDB
 #include <ddb/db_sym.h>
@@ -133,9 +144,13 @@
 #include <mach/mach_host_server.h>
 #include <mach/host_security_server.h>
 #include <mach/mach_port_server.h>
+#include <mach/security_server.h>
 
-#include <vm/task_working_set.h>
-#include <vm/vm_shared_memory_server.h>
+#include <vm/vm_shared_region.h>
+
+#if CONFIG_MACF_MACH
+#include <security/mac_mach_internal.h>
+#endif
 
 task_t	kernel_task;
 zone_t	task_zone;
@@ -168,51 +183,53 @@ task_backing_store_privileged(
 	return;
 }
 
-void
-task_working_set_disable(task_t task)
-{
-	struct tws_hash *ws;
-
-	task_lock(task);
-	ws = task->dynamic_working_set;
-	task->dynamic_working_set = NULL;
-	task_unlock(task);
-	if (ws) {
-		tws_hash_ws_flush(ws);
-		tws_hash_destroy(ws);
-	}
-}
 
 void
 task_set_64bit(
 		task_t task,
 		boolean_t is64bit)
 {
-        thread_t thread;
+#ifdef __i386__
+	thread_t thread;
+#endif /* __i386__ */
+	int	vm_flags = 0;
 
 	if (is64bit) {
-	        if (task_has_64BitAddr(task))
-		        return;
+		if (task_has_64BitAddr(task))
+			return;
 
-		/* LP64todo - no task working set for 64-bit */
 		task_set_64BitAddr(task);
-		task_working_set_disable(task);
 	} else {
-	        if ( !task_has_64BitAddr(task))
-		        return;
+		if ( !task_has_64BitAddr(task))
+			return;
 
 		/*
 		 * Deallocate all memory previously allocated
 		 * above the 32-bit address space, since it won't
 		 * be accessible anymore.
 		 */
+		/* remove regular VM map entries & pmap mappings */
+		(void) vm_map_remove(task->map,
+				     (vm_map_offset_t) VM_MAX_ADDRESS,
+				     MACH_VM_MAX_ADDRESS,
+				     0);
+#ifdef __ppc__
 		/* LP64todo - make this clean */
-		vm_map_remove_commpage(task->map);
+		/*
+		 * PPC51: ppc64 is limited to 51-bit addresses.
+		 * Memory mapped above that limit is handled specially
+		 * at the pmap level, so let pmap clean the commpage mapping
+		 * explicitly...
+		 */
 		pmap_unmap_sharedpage(task->map->pmap);	/* Unmap commpage */
+		/* ... and avoid regular pmap cleanup */
+		vm_flags |= VM_MAP_REMOVE_NO_PMAP_CLEANUP;
+#endif /* __ppc__ */
+		/* remove the higher VM mappings */
 		(void) vm_map_remove(task->map,
-				     (vm_map_offset_t) VM_MAX_ADDRESS,
 				     MACH_VM_MAX_ADDRESS,
-				     VM_MAP_NO_FLAGS);
+				     0xFFFFFFFFFFFFF000ULL,
+				     vm_flags);
 		task_clear_64BitAddr(task);
 	}
 	/* FIXME: On x86, the thread save state flavor can diverge from the
@@ -223,9 +240,9 @@ task_set_64bit(
 	 */
 #ifdef __i386__
 	queue_iterate(&task->threads, thread, thread_t, task_threads) {
-	        machine_thread_switch_addrmode(thread, !is64bit);
+		machine_thread_switch_addrmode(thread);
 	}
-#endif
+#endif /* __i386__ */
 }
 
 void
@@ -247,55 +264,6 @@ task_init(void)
 	kernel_task->map = kernel_map;
 }
 
-#if	MACH_HOST
-
-#if 0
-static void
-task_freeze(
-	task_t task)
-{
-	task_lock(task);
-	/*
-	 *	If may_assign is false, task is already being assigned,
-	 *	wait for that to finish.
-	 */
-	while (task->may_assign == FALSE) {
-		wait_result_t res;
-
-		task->assign_active = TRUE;
-		res = thread_sleep_mutex((event_t) &task->assign_active,
-					 &task->lock, THREAD_UNINT);
-		assert(res == THREAD_AWAKENED);
-	}
-	task->may_assign = FALSE;
-	task_unlock(task);
-	return;
-}
-#else
-#define thread_freeze(thread)	assert(task->processor_set == &default_pset)
-#endif
-
-#if 0
-static void
-task_unfreeze(
-	task_t task)
-{
-	task_lock(task);
-	assert(task->may_assign == FALSE);
-	task->may_assign = TRUE;
-	if (task->assign_active == TRUE) {
-		task->assign_active = FALSE;
-		thread_wakeup((event_t)&task->assign_active);
-	}
-	task_unlock(task);
-	return;
-}
-#else
-#define thread_unfreeze(thread)	assert(task->processor_set == &default_pset)
-#endif
-
-#endif	/* MACH_HOST */
-
 /*
  * Create a task running in the kernel address space.  It may
  * have its own map of size mem_size and may have ipc privileges.
@@ -312,55 +280,44 @@ kernel_task_create(
 
 kern_return_t
 task_create(
-	task_t					parent_task,
+	task_t				parent_task,
 	__unused ledger_port_array_t	ledger_ports,
-	__unused  mach_msg_type_number_t	num_ledger_ports,
-	boolean_t				inherit_memory,
-	task_t					*child_task)	/* OUT */
+	__unused mach_msg_type_number_t	num_ledger_ports,
+	__unused boolean_t		inherit_memory,
+	__unused task_t			*child_task)	/* OUT */
 {
 	if (parent_task == TASK_NULL)
 		return(KERN_INVALID_ARGUMENT);
 
-	return task_create_internal(
-	    		parent_task, inherit_memory, task_has_64BitAddr(parent_task), child_task);
+	/*
+	 * No longer supported: too many calls assume that a task has a valid
+	 * process attached.
+	 */
+	return(KERN_FAILURE);
 }
 
 kern_return_t
 host_security_create_task_token(
 	host_security_t			host_security,
-	task_t					parent_task,
-	security_token_t			sec_token,
-	audit_token_t				audit_token,
-	host_priv_t				host_priv,
+	task_t				parent_task,
+	__unused security_token_t	sec_token,
+	__unused audit_token_t		audit_token,
+	__unused host_priv_t		host_priv,
 	__unused ledger_port_array_t	ledger_ports,
 	__unused mach_msg_type_number_t	num_ledger_ports,
-	boolean_t				inherit_memory,
-	task_t					*child_task)	/* OUT */
+	__unused boolean_t		inherit_memory,
+	__unused task_t			*child_task)	/* OUT */
 {
-        kern_return_t		result;
-        
 	if (parent_task == TASK_NULL)
 		return(KERN_INVALID_ARGUMENT);
 
 	if (host_security == HOST_NULL)
 		return(KERN_INVALID_SECURITY);
 
-	result = task_create_internal(
-			parent_task, inherit_memory, task_has_64BitAddr(parent_task), child_task);
-
-        if (result != KERN_SUCCESS)
-                return(result);
-
-	result = host_security_set_task_token(host_security,
-					      *child_task,
-					      sec_token,
-					      audit_token,
-					      host_priv);
-
-	if (result != KERN_SUCCESS)
-		return(result);
-
-	return(result);
+	/*
+	 * No longer supported.
+	 */
+	return(KERN_FAILURE);
 }
 
 kern_return_t
@@ -370,8 +327,8 @@ task_create_internal(
 	boolean_t	is_64bit,
 	task_t		*child_task)		/* OUT */
 {
-	task_t		new_task;
-	processor_set_t	pset;
+	task_t			new_task;
+	vm_shared_region_t	shared_region;
 
 	new_task = (task_t) zalloc(task_zone);
 
@@ -388,6 +345,10 @@ task_create_internal(
 					(vm_map_offset_t)(VM_MIN_ADDRESS),
 					(vm_map_offset_t)(VM_MAX_ADDRESS), TRUE);
 
+	/* Inherit memlock limit from parent */
+	if (parent_task)
+		vm_map_set_user_wire_limit(new_task->map, parent_task->map->user_wire_limit);
+
 	mutex_init(&new_task->lock, 0);
 	queue_init(&new_task->threads);
 	new_task->suspend_count = 0;
@@ -396,7 +357,7 @@ task_create_internal(
 	new_task->user_stop_count = 0;
 	new_task->role = TASK_UNSPECIFIED;
 	new_task->active = TRUE;
-	new_task->user_data = 0;
+	new_task->user_data = NULL;
 	new_task->faults = 0;
 	new_task->cow_faults = 0;
 	new_task->pageins = 0;
@@ -405,16 +366,12 @@ task_create_internal(
 	new_task->syscalls_mach = 0;
 	new_task->priv_flags = 0;
 	new_task->syscalls_unix=0;
-	new_task->csw=0;
+	new_task->c_switch = new_task->p_switch = new_task->ps_switch = 0;
 	new_task->taskFeatures[0] = 0;				/* Init task features */
 	new_task->taskFeatures[1] = 0;				/* Init task features */
-	new_task->dynamic_working_set = 0;
-
-	task_working_set_create(new_task, TWS_SMALL_HASH_LINE_COUNT, 
-				0, TWS_HASH_STYLE_DEFAULT);
 
 #ifdef MACH_BSD
-	new_task->bsd_info = 0;
+	new_task->bsd_info = NULL;
 #endif /* MACH_BSD */
 
 #ifdef __i386__
@@ -430,35 +387,30 @@ task_create_internal(
 	new_task->semaphores_owned = 0;
 	new_task->lock_sets_owned = 0;
 
-#if	MACH_HOST
-	new_task->may_assign = TRUE;
-	new_task->assign_active = FALSE;
-#endif	/* MACH_HOST */
+#if CONFIG_MACF_MACH
+	/*mutex_init(&new_task->labellock, ETAP_NO_TRACE);*/
+	new_task->label = labelh_new(1);
+	mac_task_label_init (&new_task->maclabel);
+#endif
 
 	ipc_task_init(new_task, parent_task);
 
 	new_task->total_user_time = 0;
 	new_task->total_system_time = 0;
 
-	task_prof_init(new_task);
+	new_task->vtimers = 0;
 
-	if (parent_task != TASK_NULL) {
-#if	MACH_HOST
-		/*
-		 * Freeze the parent, so that parent_task->processor_set
-		 * cannot change.
-		 */
-		task_freeze(parent_task);
-#endif	/* MACH_HOST */
-		pset = parent_task->processor_set;
-		if (!pset->active)
-			pset = &default_pset;
+	new_task->shared_region = NULL;
+
+	new_task->affinity_space = NULL;
 
+	if (parent_task != TASK_NULL) {
 		new_task->sec_token = parent_task->sec_token;
 		new_task->audit_token = parent_task->audit_token;
 
-		shared_region_mapping_ref(parent_task->system_shared_region);
-		new_task->system_shared_region = parent_task->system_shared_region;
+		/* inherit the parent's shared region */
+		shared_region = vm_shared_region_get(parent_task);
+		vm_shared_region_set(new_task, shared_region);
 
 		new_task->wired_ledger_port = ledger_copy(
 			convert_port_to_ledger(parent_task->wired_ledger_port));
@@ -471,10 +423,10 @@ task_create_internal(
 		if (inherit_memory && parent_task->i386_ldt)
 			new_task->i386_ldt = user_ldt_copy(parent_task->i386_ldt);
 #endif
+		if (inherit_memory && parent_task->affinity_space)
+			task_affinity_create(parent_task, new_task);
 	}
 	else {
-		pset = &default_pset;
-
 		new_task->sec_token = KERNEL_SECURITY_TOKEN;
 		new_task->audit_token = KERNEL_AUDIT_TOKEN;
 		new_task->wired_ledger_port = ledger_copy(root_wired_ledger);
@@ -489,14 +441,11 @@ task_create_internal(
 		new_task->priority = BASEPRI_DEFAULT;
 		new_task->max_priority = MAXPRI_USER;
 	}
-
-	pset_lock(pset);
-	pset_add_task(pset, new_task);
-	pset_unlock(pset);
-#if	MACH_HOST
-	if (parent_task != TASK_NULL)
-		task_unfreeze(parent_task);
-#endif	/* MACH_HOST */
+	
+	mutex_lock(&tasks_threads_lock);
+	queue_enter(&tasks, new_task, task_t, tasks);
+	tasks_count++;
+	mutex_unlock(&tasks_threads_lock);
 
 	if (vm_backing_store_low && parent_task != NULL)
 		new_task->priv_flags |= (parent_task->priv_flags&VM_BACKING_STORE_PRIV);
@@ -516,26 +465,23 @@ void
 task_deallocate(
 	task_t		task)
 {
-	processor_set_t		pset;
-
 	if (task == TASK_NULL)
 	    return;
 
 	if (task_deallocate_internal(task) > 0)
 		return;
 
-	pset = task->processor_set;
-	pset_deallocate(pset);
-
-	if(task->dynamic_working_set)
-		tws_hash_destroy(task->dynamic_working_set);
-
 	ipc_task_terminate(task);
 
+	if (task->affinity_space)
+		task_affinity_deallocate(task);
+
 	vm_map_deallocate(task->map);
 	is_release(task->itk_space);
 
-	task_prof_deallocate(task);
+#if CONFIG_MACF_MACH
+	labelh_release(task->label);
+#endif
 	zfree(task_zone, task);
 }
 
@@ -576,7 +522,6 @@ kern_return_t
 task_terminate_internal(
 	task_t			task)
 {
-	processor_set_t		pset;
 	thread_t			thread, self;
 	task_t				self_task;
 	boolean_t			interrupt_save;
@@ -664,9 +609,13 @@ task_terminate_internal(
 	 */
 	ipc_space_destroy(task->itk_space);
 
-/* LP64todo - make this clean */
-	vm_map_remove_commpage(task->map);
+#ifdef __ppc__
+	/* LP64todo - make this clean */
+	/*
+	 * PPC51: ppc64 is limited to 51-bit addresses.
+	 */
 	pmap_unmap_sharedpage(task->map->pmap);		/* Unmap commpage */
+#endif /* __ppc__ */
 
 	if (vm_map_has_4GB_pagezero(task->map))
 		vm_map_clear_4GB_pagezero(task->map);
@@ -679,21 +628,18 @@ task_terminate_internal(
 	 * expense of removing the address space regions
 	 * at reap time, we do it explictly here.
 	 */
-	vm_map_remove(task->map, task->map->min_offset,
-								task->map->max_offset, VM_MAP_NO_FLAGS);
+	vm_map_remove(task->map,
+		      task->map->min_offset,
+		      task->map->max_offset,
+		      VM_MAP_NO_FLAGS);
 
-	shared_region_mapping_dealloc(task->system_shared_region);
+	/* release our shared region */
+	vm_shared_region_set(task, NULL);
 
-	/*
-	 * Flush working set here to avoid I/O in reaper thread
-	 */
-	if (task->dynamic_working_set)
-		tws_hash_ws_flush(task->dynamic_working_set);
-
-	pset = task->processor_set;
-	pset_lock(pset);
-	pset_remove_task(pset,task);
-	pset_unlock(pset);
+	mutex_lock(&tasks_threads_lock);
+	queue_remove(&tasks, task, task_t, tasks);
+	tasks_count--;
+	mutex_unlock(&tasks_threads_lock);
 
 	/*
 	 * We no longer need to guard against being aborted, so restore
@@ -953,7 +899,7 @@ task_threads(
 	mach_msg_type_number_t	*count)
 {
 	mach_msg_type_number_t	actual;
-	thread_t				*threads;
+	thread_t				*thread_list;
 	thread_t				thread;
 	vm_size_t				size, size_needed;
 	void					*addr;
@@ -962,7 +908,7 @@ task_threads(
 	if (task == TASK_NULL)
 		return (KERN_INVALID_ARGUMENT);
 
-	size = 0; addr = 0;
+	size = 0; addr = NULL;
 
 	for (;;) {
 		task_lock(task);
@@ -997,14 +943,14 @@ task_threads(
 	}
 
 	/* OK, have memory and the task is locked & active */
-	threads = (thread_t *)addr;
+	thread_list = (thread_t *)addr;
 
 	i = j = 0;
 
 	for (thread = (thread_t)queue_first(&task->threads); i < actual;
 				++i, thread = (thread_t)queue_next(&thread->task_threads)) {
 		thread_reference_internal(thread);
-		threads[j++] = thread;
+		thread_list[j++] = thread;
 	}
 
 	assert(queue_end(&task->threads, (queue_entry_t)thread));
@@ -1018,7 +964,7 @@ task_threads(
 	if (actual == 0) {
 		/* no threads, so return null pointer and deallocate memory */
 
-		*threads_out = 0;
+		*threads_out = NULL;
 		*count = 0;
 
 		if (size != 0)
@@ -1033,23 +979,23 @@ task_threads(
 			newaddr = kalloc(size_needed);
 			if (newaddr == 0) {
 				for (i = 0; i < actual; ++i)
-					thread_deallocate(threads[i]);
+					thread_deallocate(thread_list[i]);
 				kfree(addr, size);
 				return (KERN_RESOURCE_SHORTAGE);
 			}
 
 			bcopy(addr, newaddr, size_needed);
 			kfree(addr, size);
-			threads = (thread_t *)newaddr;
+			thread_list = (thread_t *)newaddr;
 		}
 
-		*threads_out = threads;
+		*threads_out = thread_list;
 		*count = actual;
 
 		/* do the conversion that Mig should handle */
 
 		for (i = 0; i < actual; ++i)
-			((ipc_port_t *) threads)[i] = convert_thread_to_port(threads[i]);
+			((ipc_port_t *) thread_list)[i] = convert_thread_to_port(thread_list[i]);
 	}
 
 	return (KERN_SUCCESS);
@@ -1243,6 +1189,7 @@ task_info(
 	switch (flavor) {
 
 	case TASK_BASIC_INFO_32:
+	case TASK_BASIC2_INFO_32:
 	{
 		task_basic_info_32_t	basic_info;
 		vm_map_t			map;
@@ -1254,22 +1201,28 @@ task_info(
 
 		map = (task == kernel_task)? kernel_map: task->map;
 		basic_info->virtual_size  = CAST_DOWN(vm_offset_t,map->size);
-		basic_info->resident_size = pmap_resident_count(map->pmap)
-						   * PAGE_SIZE;
+		if (flavor == TASK_BASIC2_INFO_32) {
+			/*
+			 * The "BASIC2" flavor gets the maximum resident
+			 * size instead of the current resident size...
+			 */
+			basic_info->resident_size = pmap_resident_max(map->pmap);
+		} else {
+			basic_info->resident_size = pmap_resident_count(map->pmap);
+		}
+		basic_info->resident_size *= PAGE_SIZE;
 
 		task_lock(task);
 		basic_info->policy = ((task != kernel_task)?
 										  POLICY_TIMESHARE: POLICY_RR);
 		basic_info->suspend_count = task->user_stop_count;
 
-		absolutetime_to_microtime(
-						task->total_user_time,
-								&basic_info->user_time.seconds,
-								&basic_info->user_time.microseconds);
-		absolutetime_to_microtime(
-						task->total_system_time,
-								&basic_info->system_time.seconds,
-								&basic_info->system_time.microseconds);
+		absolutetime_to_microtime(task->total_user_time,
+					  (unsigned *)&basic_info->user_time.seconds,
+					  (unsigned *)&basic_info->user_time.microseconds);
+		absolutetime_to_microtime(task->total_system_time,
+					  (unsigned *)&basic_info->system_time.seconds,
+					  (unsigned *)&basic_info->system_time.microseconds);
 		task_unlock(task);
 
 		*task_info_count = TASK_BASIC_INFO_32_COUNT;
@@ -1288,22 +1241,21 @@ task_info(
 
 		map = (task == kernel_task)? kernel_map: task->map;
 		basic_info->virtual_size  = map->size;
-		basic_info->resident_size = (mach_vm_size_t)(pmap_resident_count(map->pmap)
-						   * PAGE_SIZE);
+		basic_info->resident_size =
+			(mach_vm_size_t)(pmap_resident_count(map->pmap))
+			* PAGE_SIZE_64;
 
 		task_lock(task);
 		basic_info->policy = ((task != kernel_task)?
 										  POLICY_TIMESHARE: POLICY_RR);
 		basic_info->suspend_count = task->user_stop_count;
 
-		absolutetime_to_microtime(
-						task->total_user_time,
-								&basic_info->user_time.seconds,
-								&basic_info->user_time.microseconds);
-		absolutetime_to_microtime(
-						task->total_system_time,
-								&basic_info->system_time.seconds,
-								&basic_info->system_time.microseconds);
+		absolutetime_to_microtime(task->total_user_time,
+					  (unsigned *)&basic_info->user_time.seconds,
+					  (unsigned *)&basic_info->user_time.microseconds);
+		absolutetime_to_microtime(task->total_system_time,
+					  (unsigned *)&basic_info->system_time.seconds,
+					  (unsigned *)&basic_info->system_time.microseconds);
 		task_unlock(task);
 
 		*task_info_count = TASK_BASIC_INFO_64_COUNT;
@@ -1473,6 +1425,7 @@ task_info(
 	case TASK_EVENTS_INFO:
 	{
 		register task_events_info_t	events_info;
+		register thread_t			thread;
 
 		if (*task_info_count < TASK_EVENTS_INFO_COUNT)
 		    return (KERN_INVALID_ARGUMENT);
@@ -1480,6 +1433,7 @@ task_info(
 		events_info = (task_events_info_t) task_info_out;
 
 		task_lock(task);
+
 		events_info->faults = task->faults;
 		events_info->pageins = task->pageins;
 		events_info->cow_faults = task->cow_faults;
@@ -1487,12 +1441,25 @@ task_info(
 		events_info->messages_received = task->messages_received;
 		events_info->syscalls_mach = task->syscalls_mach;
 		events_info->syscalls_unix = task->syscalls_unix;
-		events_info->csw = task->csw;
+
+		events_info->csw = task->c_switch;
+
+		queue_iterate(&task->threads, thread, thread_t, task_threads) {
+			events_info->csw += thread->c_switch;
+		}
+
 		task_unlock(task);
 
 		*task_info_count = TASK_EVENTS_INFO_COUNT;
 		break;
 	}
+	case TASK_AFFINITY_TAG_INFO:
+	{
+		if (*task_info_count < TASK_AFFINITY_TAG_INFO_COUNT)
+		    return (KERN_INVALID_ARGUMENT);
+
+		return task_affinity_info(task, task_info_out, task_info_count);
+	}
 
 	default:
 		return (KERN_INVALID_ARGUMENT);
@@ -1501,6 +1468,101 @@ task_info(
 	return (KERN_SUCCESS);
 }
 
+void
+task_vtimer_set(
+	task_t		task,
+	integer_t	which)
+{
+	thread_t	thread;
+
+	/* assert(task == current_task()); */ /* bogus assert 4803227 4807483 */
+
+	task_lock(task);
+
+	task->vtimers |= which;
+
+	switch (which) {
+
+	case TASK_VTIMER_USER:
+		queue_iterate(&task->threads, thread, thread_t, task_threads) {
+			thread->vtimer_user_save = timer_grab(&thread->user_timer);
+		}
+		break;
+
+	case TASK_VTIMER_PROF:
+		queue_iterate(&task->threads, thread, thread_t, task_threads) {
+			thread->vtimer_prof_save = timer_grab(&thread->user_timer);
+			thread->vtimer_prof_save += timer_grab(&thread->system_timer);
+		}
+		break;
+
+	case TASK_VTIMER_RLIM:
+		queue_iterate(&task->threads, thread, thread_t, task_threads) {
+			thread->vtimer_rlim_save = timer_grab(&thread->user_timer);
+			thread->vtimer_rlim_save += timer_grab(&thread->system_timer);
+		}
+		break;
+	}
+
+	task_unlock(task);
+}
+
+void
+task_vtimer_clear(
+	task_t		task,
+	integer_t	which)
+{
+	assert(task == current_task());
+
+	task_lock(task);
+
+	task->vtimers &= ~which;
+
+	task_unlock(task);
+}
+
+void
+task_vtimer_update(
+__unused
+	task_t		task,
+	integer_t	which,
+	uint32_t	*microsecs)
+{
+	thread_t	thread = current_thread();
+	uint32_t	tdelt, secs;
+	uint64_t	tsum;
+
+	assert(task == current_task());
+
+	assert(task->vtimers & which);
+
+	tdelt = secs = 0;
+
+	switch (which) {
+
+	case TASK_VTIMER_USER:
+		tdelt = timer_delta(&thread->user_timer,
+								&thread->vtimer_user_save);
+		break;
+
+	case TASK_VTIMER_PROF:
+		tsum = timer_grab(&thread->user_timer);
+		tsum += timer_grab(&thread->system_timer);
+		tdelt = tsum - thread->vtimer_prof_save;
+		thread->vtimer_prof_save = tsum;
+		break;
+
+	case TASK_VTIMER_RLIM:
+		tsum = timer_grab(&thread->user_timer);
+		tsum += timer_grab(&thread->system_timer);
+		tdelt = tsum - thread->vtimer_rlim_save;
+		thread->vtimer_rlim_save = tsum;
+		break;
+	}
+
+	absolutetime_to_microtime(tdelt, &secs, microsecs);
+}
+
 /*
  *	task_assign:
  *
@@ -1525,7 +1587,7 @@ task_assign_default(
 	task_t		task,
 	boolean_t	assign_threads)
 {
-    return (task_assign(task, &default_pset, assign_threads));
+    return (task_assign(task, &pset0, assign_threads));
 }
 
 /*
@@ -1541,9 +1603,9 @@ task_get_assignment(
 	if (!task->active)
 		return(KERN_FAILURE);
 
-	*pset = task->processor_set;
-	pset_reference(*pset);
-	return(KERN_SUCCESS);
+	*pset = &pset0;
+
+	return (KERN_SUCCESS);
 }
 
 
@@ -1672,3 +1734,64 @@ task_reference(
 	if (task != TASK_NULL)
 		task_reference_internal(task);
 }
+
+#if CONFIG_MACF_MACH
+/*
+ * Protect 2 task labels against modification by adding a reference on
+ * both label handles. The locks do not actually have to be held while
+ * using the labels as only labels with one reference can be modified
+ * in place.
+ */
+
+void
+tasklabel_lock2(
+	task_t a,
+	task_t b)
+{
+	labelh_reference(a->label);
+	labelh_reference(b->label);
+}
+
+void
+tasklabel_unlock2(
+	task_t a,
+	task_t b)
+{
+	labelh_release(a->label);
+	labelh_release(b->label);
+}
+
+void
+mac_task_label_update_internal(
+	struct label	*pl,
+	struct task	*task)
+{
+
+	tasklabel_lock(task);
+	task->label = labelh_modify(task->label);
+	mac_task_label_update(pl, &task->maclabel);
+	tasklabel_unlock(task);
+	ip_lock(task->itk_self);
+	mac_port_label_update_cred(pl, &task->itk_self->ip_label);
+	ip_unlock(task->itk_self);
+}
+
+void
+mac_task_label_modify(
+	struct task	*task,
+	void		*arg,
+	void (*f)	(struct label *l, void *arg))
+{
+
+	tasklabel_lock(task);
+	task->label = labelh_modify(task->label);
+	(*f)(&task->maclabel, arg);
+	tasklabel_unlock(task);
+}
+
+struct label *
+mac_task_get_label(struct task *task)
+{
+	return (&task->maclabel);
+}
+#endif
diff --git a/osfmk/kern/task.h b/osfmk/kern/task.h
index 5e3948343..21b65530b 100644
--- a/osfmk/kern/task.h
+++ b/osfmk/kern/task.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -74,6 +80,13 @@
  * improvements that they make and grant CSL redistribution rights.
  *
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 
 #ifndef	_KERN_TASK_H_
 #define _KERN_TASK_H_
@@ -98,32 +111,26 @@
 #include <kern/exception.h>
 #include <kern/lock.h>
 #include <kern/thread.h>
-
-#include <mach_host.h>
-#include <mach_prof.h>
+#include <security/_label.h>
+#include <ipc/ipc_labelh.h>
 
 struct task {
 	/* Synchronization/destruction information */
 	decl_mutex_data(,lock)		/* Task's lock */
-	int		ref_count;	/* Number of references to me */
+	uint32_t	ref_count;	/* Number of references to me */
 	boolean_t	active;		/* Task has not been terminated */
 
 	/* Miscellaneous */
 	vm_map_t	map;		/* Address space description */
-	queue_chain_t	pset_tasks;	/* list of tasks assigned to pset */
+	queue_chain_t	tasks;	/* global list of tasks */
 	void		*user_data;	/* Arbitrary data settable via IPC */
 	int		suspend_count;	/* Internal scheduling only */
 
 	/* Threads in this task */
-	queue_head_t	threads;
-	int				thread_count;
-	int				active_thread_count;
-
-	processor_set_t	processor_set;	/* processor set for new threads */
-#if	MACH_HOST
-	boolean_t	may_assign;	/* can assigned pset be changed? */
-	boolean_t	assign_active;	/* waiting for may_assign */
-#endif	/* MACH_HOST */
+	queue_head_t		threads;
+	int			thread_count;
+	uint32_t		active_thread_count;
+	struct affinity_space	*affinity_space;
 
 	/* User-visible scheduling information */
 	integer_t		user_stop_count;	/* outstanding stops */
@@ -141,10 +148,8 @@ struct task {
 	uint64_t		total_user_time;	/* terminated threads only */
 	uint64_t		total_system_time;
 
-#if	MACH_PROF
-	boolean_t	task_profiled;  /* is task being profiled ? */
-	struct prof_data *profil_buffer;/* profile struct if so */
-#endif	/* MACH_PROF */
+	/* Virtual timers */
+	uint32_t		vtimers;
 
 	/* IPC structures */
 	decl_mutex_data(,itk_lock_data)
@@ -155,6 +160,10 @@ struct task {
 		 			/* a send right each valid element  */
 	struct ipc_port *itk_host;	/* a send right */
 	struct ipc_port *itk_bootstrap;	/* a send right */
+	struct ipc_port *itk_seatbelt;	/* a send right */
+	struct ipc_port *itk_gssd;	/* yet another send right */
+	struct ipc_port *itk_task_access; /* and another send right */ 
+	struct ipc_port *itk_automountd;/* a send right */
 	struct ipc_port *itk_registered[TASK_PORT_REGISTER_MAX];
 					/* all send rights */
 
@@ -181,12 +190,13 @@ struct task {
         integer_t messages_received;   /* messages received counter */
         integer_t syscalls_mach;       /* mach system call counter */
         integer_t syscalls_unix;       /* unix system call counter */
-        integer_t csw;                 /* context switch counter */
+		uint32_t  c_switch;			   /* total context switches */
+		uint32_t  p_switch;			   /* total processor switches */
+		uint32_t  ps_switch;		   /* total pset switches */
 #ifdef  MACH_BSD 
 	void *bsd_info;
 #endif  
-	struct shared_region_mapping	*system_shared_region;
-	struct tws_hash 		*dynamic_working_set;
+	struct vm_shared_region		*shared_region;
 	uint32_t taskFeatures[2];		/* Special feature for this task */
 #define tf64BitAddr	0x80000000		/* Task has 64-bit addressing */
 #define tf64BitData	0x40000000		/* Task has 64-bit data registers */
@@ -197,18 +207,32 @@ struct task {
 #define task_clear_64BitAddr(task)	\
 	 ((task)->taskFeatures[0] &= ~tf64BitAddr)
 
+#if CONFIG_MACF_MACH
+	ipc_labelh_t label;
+#endif
+
 };
 
 #define task_lock(task)		mutex_lock(&(task)->lock)
 #define task_lock_try(task)	mutex_try(&(task)->lock)
 #define task_unlock(task)	mutex_unlock(&(task)->lock)
 
+#if CONFIG_MACF_MACH
+#define maclabel label->lh_label
+
+#define tasklabel_lock(task)	lh_lock((task)->label)
+#define tasklabel_unlock(task)	lh_unlock((task)->label)
+
+extern void tasklabel_lock2(task_t a, task_t b);
+extern void tasklabel_unlock2(task_t a, task_t b);
+#endif /* MAC_MACH */
+
 #define	itk_lock_init(task)	mutex_init(&(task)->itk_lock_data, 0)
 #define	itk_lock(task)		mutex_lock(&(task)->itk_lock_data)
 #define	itk_unlock(task)	mutex_unlock(&(task)->itk_lock_data)
 
 #define task_reference_internal(task)		\
-			hw_atomic_add(&(task)->ref_count, 1)
+			(void)hw_atomic_add(&(task)->ref_count, 1)
 
 #define task_deallocate_internal(task)		\
 			hw_atomic_sub(&(task)->ref_count, 1)
@@ -226,7 +250,7 @@ extern kern_return_t	kernel_task_create(
 							task_t 			*child);
 
 /* Initialize task module */
-extern void		task_init(void);
+extern void		task_init(void) __attribute__((section("__TEXT, initcode")));
 
 #define	current_task_fast()	(current_thread()->task)
 #define current_task()		current_task_fast()
@@ -272,6 +296,23 @@ extern kern_return_t	task_importance(
 							task_t			task,
 							integer_t		importance);
 
+extern void		task_vtimer_set(
+					task_t		task,
+					integer_t	which);
+
+extern void		task_vtimer_clear(
+					task_t		task,
+					integer_t	which);
+
+extern void		task_vtimer_update(
+					task_t		task,
+					integer_t	which,
+					uint32_t	*microsecs);
+
+#define	TASK_VTIMER_USER		0x01
+#define	TASK_VTIMER_PROF		0x02
+#define	TASK_VTIMER_RLIM		0x04
+
 extern void		task_set_64bit(
 					task_t		task,
 					boolean_t	is64bit);
@@ -279,10 +320,6 @@ extern void		task_set_64bit(
 extern void		task_backing_store_privileged(
 					task_t		task);
 
-extern void		task_working_set_disable(
-					task_t		task);
-
-
 /* Get number of activations in a task */
 extern int		get_task_numacts(
 					task_t		task);
@@ -296,11 +333,14 @@ extern pmap_t	get_task_pmap(task_t);
 
 extern boolean_t	is_kerneltask(task_t task);
 
+extern kern_return_t check_actforsig(task_t task, thread_t thread, int setast);
+
 #endif	/* XNU_KERNEL_PRIVATE */
 
 #ifdef	KERNEL_PRIVATE
 
 extern void 	*get_bsdtask_info(task_t);
+extern void	*get_bsdthreadtask_info(thread_t);
 extern vm_map_t get_task_map(task_t);
 
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/kern/task_policy.c b/osfmk/kern/task_policy.c
index a0029e1df..0f027820d 100644
--- a/osfmk/kern/task_policy.c
+++ b/osfmk/kern/task_policy.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
diff --git a/osfmk/kern/task_swap.c b/osfmk/kern/task_swap.c
index e7767de9b..6f9f462cb 100644
--- a/osfmk/kern/task_swap.c
+++ b/osfmk/kern/task_swap.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/task_swap.h b/osfmk/kern/task_swap.h
index 279caefe6..e3b50d7fd 100644
--- a/osfmk/kern/task_swap.h
+++ b/osfmk/kern/task_swap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index 2d8cbc4be..9e4360a73 100644
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -75,9 +81,6 @@
  *
  */
 
-#include <mach_host.h>
-#include <mach_prof.h>
-
 #include <mach/mach_types.h>
 #include <mach/boolean.h>
 #include <mach/policy.h>
@@ -108,7 +111,6 @@
 #include <kern/thread.h>
 #include <kern/host.h>
 #include <kern/zalloc.h>
-#include <kern/profile.h>
 #include <kern/assert.h>
 
 #include <ipc/ipc_kmsg.h>
@@ -119,6 +121,8 @@
 
 #include <sys/kdebug.h>
 
+#include <mach/sdt.h>
+
 /*
  * Exported interfaces
  */
@@ -137,9 +141,13 @@ static queue_head_t		thread_terminate_queue;
 
 static struct thread	thread_template, init_thread;
 
+static void		sched_call_null(
+					int			type,
+					thread_t	thread);
 #ifdef MACH_BSD
 extern void proc_exit(void *);
 #endif /* MACH_BSD */
+extern int debug_task;
 
 void
 thread_bootstrap(void)
@@ -148,7 +156,7 @@ thread_bootstrap(void)
 	 *	Fill in a template thread for fast initialization.
 	 */
 
-	thread_template.runq = RUN_QUEUE_NULL;
+	thread_template.runq = PROCESSOR_NULL;
 
 	thread_template.ref_count = 2;
 
@@ -188,34 +196,46 @@ thread_bootstrap(void)
 	thread_template.sched_usage = 0;
 	thread_template.pri_shift = INT8_MAX;
 	thread_template.cpu_usage = thread_template.cpu_delta = 0;
+	thread_template.c_switch = thread_template.p_switch = thread_template.ps_switch = 0;
 
 	thread_template.bound_processor = PROCESSOR_NULL;
 	thread_template.last_processor = PROCESSOR_NULL;
 	thread_template.last_switch = 0;
 
+	thread_template.sched_call = sched_call_null;
+
 	timer_init(&thread_template.user_timer);
 	timer_init(&thread_template.system_timer);
 	thread_template.user_timer_save = 0;
 	thread_template.system_timer_save = 0;
+	thread_template.vtimer_user_save = 0;
+	thread_template.vtimer_prof_save = 0;
+	thread_template.vtimer_rlim_save = 0;
 
 	thread_template.wait_timer_is_set = FALSE;
 	thread_template.wait_timer_active = 0;
 
 	thread_template.depress_timer_active = 0;
 
-	thread_template.processor_set = PROCESSOR_SET_NULL;
-
 	thread_template.special_handler.handler = special_handler;
-	thread_template.special_handler.next = 0;
+	thread_template.special_handler.next = NULL;
 
-#if	MACH_HOST
-	thread_template.may_assign = TRUE;
-	thread_template.assign_active = FALSE;
-#endif	/* MACH_HOST */
 	thread_template.funnel_lock = THR_FUNNEL_NULL;
 	thread_template.funnel_state = 0;
 	thread_template.recover = (vm_offset_t)NULL;
+	
+	thread_template.map = VM_MAP_NULL;
+
+#if CONFIG_DTRACE
+	thread_template.t_dtrace_predcache = 0;
+	thread_template.t_dtrace_vtime = 0;
+	thread_template.t_dtrace_tracing = 0;
+#endif /* CONFIG_DTRACE */
+	
+	thread_template.t_chud = 0;
 
+	thread_template.affinity_set = NULL;
+	
 	init_thread = thread_template;
 	machine_set_current_thread(&init_thread);
 }
@@ -254,13 +274,22 @@ thread_terminate_self(void)
 	thread_t		thread = current_thread();
 	task_t			task;
 	spl_t			s;
+	int lastthread = 0;
+
+	thread_mtx_lock(thread);
+
+	ulock_release_all(thread);
+
+	ipc_thread_disable(thread);
+	
+	thread_mtx_unlock(thread);
 
 	s = splsched();
 	thread_lock(thread);
 
 	/*
-	 *	Cancel priority depression, reset scheduling parameters,
-	 *	and wait for concurrent expirations on other processors.
+	 *	Cancel priority depression, wait for concurrent expirations
+	 *	on other processors.
 	 */
 	if (thread->sched_mode & TH_MODE_ISDEPRESSED) {
 		thread->sched_mode &= ~TH_MODE_ISDEPRESSED;
@@ -269,8 +298,6 @@ thread_terminate_self(void)
 			thread->depress_timer_active--;
 	}
 
-	thread_policy_reset(thread);
-
 	while (thread->depress_timer_active > 0) {
 		thread_unlock(thread);
 		splx(s);
@@ -284,23 +311,24 @@ thread_terminate_self(void)
 	thread_unlock(thread);
 	splx(s);
 
-	thread_mtx_lock(thread);
-
-	ulock_release_all(thread);
-
-	ipc_thread_disable(thread);
-	
-	thread_mtx_unlock(thread);
+	thread_policy_reset(thread);
 
 	/*
 	 * If we are the last thread to terminate and the task is
 	 * associated with a BSD process, perform BSD process exit.
 	 */
 	task = thread->task;
-	if (	hw_atomic_sub(&task->active_thread_count, 1) == 0	&&
-					task->bsd_info != NULL						)
+	uthread_cleanup(task, thread->uthread, task->bsd_info);
+	if (hw_atomic_sub(&task->active_thread_count, 1) == 0	&&
+					task->bsd_info != NULL) {
+		lastthread = 1;
+	} 
+	
+	if (lastthread != 0)
 		proc_exit(task->bsd_info);
 
+	uthread_cred_free(thread->uthread);
+
 	s = splsched();
 	thread_lock(thread);
 
@@ -351,7 +379,6 @@ void
 thread_deallocate(
 	thread_t			thread)
 {
-	processor_set_t		pset;
 	task_t				task;
 
 	if (thread == THREAD_NULL)
@@ -369,15 +396,12 @@ thread_deallocate(
 		void *ut = thread->uthread;
 
 		thread->uthread = NULL;
-		uthread_free(task, ut, task->bsd_info);
+		uthread_zone_free(ut);
 	}
 #endif  /* MACH_BSD */   
 
 	task_deallocate(task);
 
-	pset = thread->processor_set;
-	pset_deallocate(pset);
-
 	if (thread->kernel_stack != 0)
 		stack_free(thread);
 
@@ -396,7 +420,6 @@ thread_terminate_daemon(void)
 {
 	thread_t			thread;
 	task_t				task;
-	processor_set_t		pset;
 
 	(void)splsched();
 	simple_lock(&thread_terminate_lock);
@@ -411,15 +434,18 @@ thread_terminate_daemon(void)
 		task->total_user_time += timer_grab(&thread->user_timer);
 		task->total_system_time += timer_grab(&thread->system_timer);
 
+		task->c_switch += thread->c_switch;
+		task->p_switch += thread->p_switch;
+		task->ps_switch += thread->ps_switch;
+
 		queue_remove(&task->threads, thread, thread_t, task_threads);
 		task->thread_count--;
 		task_unlock(task);
 
-		pset = thread->processor_set;
-
-		pset_lock(pset);
-		pset_remove_thread(pset, thread);
-		pset_unlock(pset);
+		mutex_lock(&tasks_threads_lock);
+		queue_remove(&threads, thread, thread_t, threads);
+		threads_count--;
+		mutex_unlock(&tasks_threads_lock);
 
 		thread_deallocate(thread);
 
@@ -464,27 +490,24 @@ thread_stack_daemon(void)
 {
 	thread_t		thread;
 
-	(void)splsched();
 	simple_lock(&thread_stack_lock);
 
 	while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
 		simple_unlock(&thread_stack_lock);
-		/* splsched */
 
 		stack_alloc(thread);
-
+		
+		(void)splsched();
 		thread_lock(thread);
 		thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
 		thread_unlock(thread);
 		(void)spllo();
 
-		(void)splsched();
 		simple_lock(&thread_stack_lock);
 	}
 
 	assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
 	simple_unlock(&thread_stack_lock);
-	/* splsched */
 
 	thread_block((thread_continue_t)thread_stack_daemon);
 	/*NOTREACHED*/
@@ -545,7 +568,6 @@ thread_create_internal(
 	thread_t				*out_thread)
 {
 	thread_t				new_thread;
-	processor_set_t			pset;
 	static thread_t			first_thread;
 
 	/*
@@ -577,7 +599,10 @@ thread_create_internal(
 			void *ut = new_thread->uthread;
 
 			new_thread->uthread = NULL;
-			uthread_free(parent_task, ut, parent_task->bsd_info);
+			/* cred free may not be necessary */
+			uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+			uthread_cred_free(ut);
+			uthread_zone_free(ut);
 		}
 #endif  /* MACH_BSD */
 		zfree(thread_zone, new_thread);
@@ -593,29 +618,27 @@ thread_create_internal(
 
 	ipc_thread_init(new_thread);
 	queue_init(&new_thread->held_ulocks);
-	thread_prof_init(new_thread, parent_task);
 
 	new_thread->continuation = continuation;
 
-	pset = parent_task->processor_set;
-	assert(pset == &default_pset);
-	pset_lock(pset);
-
+	mutex_lock(&tasks_threads_lock);
 	task_lock(parent_task);
-	assert(parent_task->processor_set == pset);
 
 	if (	!parent_task->active							||
 			(parent_task->thread_count >= THREAD_MAX	&&
 			 parent_task != kernel_task)) {
 		task_unlock(parent_task);
-		pset_unlock(pset);
+		mutex_unlock(&tasks_threads_lock);
 
 #ifdef MACH_BSD
 		{
 			void *ut = new_thread->uthread;
 
 			new_thread->uthread = NULL;
-			uthread_free(parent_task, ut, parent_task->bsd_info);
+			uthread_cleanup(parent_task, ut, parent_task->bsd_info);
+			/* cred free may not be necessary */
+			uthread_cred_free(ut);
+			uthread_zone_free(ut);
 		}
 #endif  /* MACH_BSD */
 		ipc_thread_disable(new_thread);
@@ -637,8 +660,8 @@ thread_create_internal(
 	/* So terminating threads don't need to take the task lock to decrement */
 	hw_atomic_add(&parent_task->active_thread_count, 1);
 
-	/* Associate the thread with the processor set */
-	pset_add_thread(pset, new_thread);
+	queue_enter(&threads, new_thread, thread_t, threads);
+	threads_count++;
 
 	timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread);
 	timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread);
@@ -654,7 +677,7 @@ thread_create_internal(
 	new_thread->importance =
 					new_thread->priority - new_thread->task_priority;
 	new_thread->sched_stamp = sched_tick;
-	new_thread->pri_shift = new_thread->processor_set->pri_shift;
+	new_thread->pri_shift = sched_pri_shift;
 	compute_priority(new_thread, FALSE);
 
 	new_thread->active = TRUE;
@@ -678,6 +701,8 @@ thread_create_internal(
 							dbg_arg1, dbg_arg2, dbg_arg3, dbg_arg4, 0);
 	}
 
+	DTRACE_PROC1(lwp__create, thread_t, *out_thread);
+
 	return (KERN_SUCCESS);
 }
 
@@ -701,8 +726,8 @@ thread_create(
 	if (task->suspend_count > 0)
 		thread_hold(thread);
 
-	pset_unlock(task->processor_set);
 	task_unlock(task);
+	mutex_unlock(&tasks_threads_lock);
 	
 	*new_thread = thread;
 
@@ -730,8 +755,8 @@ thread_create_running(
 	result = machine_thread_set_state(
 						thread, flavor, new_state, new_state_count);
 	if (result != KERN_SUCCESS) {
-		pset_unlock(task->processor_set);
 		task_unlock(task);
+		mutex_unlock(&tasks_threads_lock);
 
 		thread_terminate(thread);
 		thread_deallocate(thread);
@@ -739,11 +764,11 @@ thread_create_running(
 	}
 
 	thread_mtx_lock(thread);
-	clear_wait(thread, THREAD_AWAKENED);
-	thread->started = TRUE;
+	thread_start_internal(thread);
 	thread_mtx_unlock(thread);
-	pset_unlock(task->processor_set);
+
 	task_unlock(task);
+	mutex_unlock(&tasks_threads_lock);
 
 	*new_thread = thread;
 
@@ -771,15 +796,20 @@ kernel_thread_create(
 	if (result != KERN_SUCCESS)
 		return (result);
 
-	pset_unlock(task->processor_set);
 	task_unlock(task);
+	mutex_unlock(&tasks_threads_lock);
 
 	stack_alloc(thread);
 	assert(thread->kernel_stack != 0);
+#if CONFIG_EMBEDDED
+	if (priority > BASEPRI_KERNEL)
+#endif
 	thread->reserved_stack = thread->kernel_stack;
 
 	thread->parameter = parameter;
 
+if(debug_task & 1)
+	kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation);
 	*new_thread = thread;
 
 	return (result);
@@ -800,8 +830,7 @@ kernel_thread_start_priority(
 		return (result);
 
 	thread_mtx_lock(thread);
-	clear_wait(thread, THREAD_AWAKENED);
-	thread->started = TRUE;
+	thread_start_internal(thread);
 	thread_mtx_unlock(thread);
 
 	*new_thread = thread;
@@ -891,7 +920,7 @@ thread_info_internal(
 												POLICY_TIMESHARE: POLICY_RR);
 
 	    flags = 0;
-		if (thread->state & TH_IDLE)
+		if (thread->bound_processor != PROCESSOR_NULL && thread->bound_processor->idle_thread == thread)
 			flags |= TH_FLAGS_IDLE;
 
 	    if (!thread->kernel_stack)
@@ -1020,13 +1049,13 @@ thread_read_times(
 	time_value_t	*user_time,
 	time_value_t	*system_time)
 {
-	absolutetime_to_microtime(
-			timer_grab(&thread->user_timer),
-							&user_time->seconds, &user_time->microseconds);
+	absolutetime_to_microtime(timer_grab(&thread->user_timer),
+				  (unsigned *)&user_time->seconds,
+				  (unsigned *)&user_time->microseconds);
 
-	absolutetime_to_microtime(
-			timer_grab(&thread->system_timer),
-							&system_time->seconds, &system_time->microseconds);
+	absolutetime_to_microtime(timer_grab(&thread->system_timer),
+				  (unsigned *)&system_time->seconds,
+				  (unsigned *)&system_time->microseconds);
 }
 
 kern_return_t
@@ -1047,7 +1076,7 @@ kern_return_t
 thread_assign_default(
 	thread_t		thread)
 {
-	return (thread_assign(thread, &default_pset));
+	return (thread_assign(thread, &pset0));
 }
 
 /*
@@ -1063,8 +1092,8 @@ thread_get_assignment(
 	if (thread == NULL)
 		return (KERN_INVALID_ARGUMENT);
 
-	*pset = thread->processor_set;
-	pset_reference(*pset);
+	*pset = &pset0;
+
 	return (KERN_SUCCESS);
 }
 
@@ -1172,6 +1201,7 @@ funnel_unlock(
 	funnel_t * fnl)
 {
 	lck_mtx_unlock(fnl->fnl_mutex);
+	fnl->fnl_mtxholder = NULL;
 	fnl->fnl_mtxrelease = current_thread();
 }
 
@@ -1204,7 +1234,7 @@ thread_funnel_set(
 
 		if (funneled == TRUE) {
 			if (cur_thread->funnel_lock)
-				panic("Funnel lock called when holding one %x", cur_thread->funnel_lock);
+				panic("Funnel lock called when holding one %p", cur_thread->funnel_lock);
 			KERNEL_DEBUG(0x6032428 | DBG_FUNC_NONE,
 											fnl, 1, 0, 0, 0);
 			funnel_lock(fnl);
@@ -1234,6 +1264,31 @@ thread_funnel_set(
 	return(funnel_state_prev);
 }
 
+static void
+sched_call_null(
+__unused	int			type,
+__unused	thread_t	thread)
+{
+	return;
+}
+
+void
+thread_sched_call(
+	thread_t		thread,
+	sched_call_t	call)
+{
+	thread->sched_call = (call != NULL)? call: sched_call_null;
+}
+
+void
+thread_static_param(
+	thread_t		thread,
+	boolean_t		state)
+{
+	thread_mtx_lock(thread);
+	thread->static_param = state;
+	thread_mtx_unlock(thread);
+}
 
 /*
  * Export routines to other components for things that are done as macros
@@ -1258,3 +1313,107 @@ thread_should_halt(
 {
 	return (thread_should_halt_fast(th));
 }
+
+#if CONFIG_DTRACE
+uint32_t dtrace_get_thread_predcache(thread_t thread)
+{
+	if (thread != THREAD_NULL)
+		return thread->t_dtrace_predcache;
+	else
+		return 0;
+}
+
+int64_t dtrace_get_thread_vtime(thread_t thread)
+{
+	if (thread != THREAD_NULL)
+		return thread->t_dtrace_vtime;
+	else
+		return 0;
+}
+
+int64_t dtrace_get_thread_tracing(thread_t thread)
+{
+	if (thread != THREAD_NULL)
+		return thread->t_dtrace_tracing;
+	else
+		return 0;
+}
+
+boolean_t dtrace_get_thread_reentering(thread_t thread)
+{
+	if (thread != THREAD_NULL)
+		return (thread->options & TH_OPT_DTRACE) ? TRUE : FALSE;
+	else
+		return 0;
+}
+
+vm_offset_t dtrace_get_kernel_stack(thread_t thread)
+{
+	if (thread != THREAD_NULL)
+		return thread->kernel_stack;
+	else
+		return 0;
+}
+
+int64_t dtrace_calc_thread_recent_vtime(thread_t thread)
+{
+#if STAT_TIME
+	if (thread != THREAD_NULL) {
+		return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer));
+	} else
+		return 0;
+#else
+	if (thread != THREAD_NULL) {
+		processor_t             processor = current_processor();
+		uint64_t 				abstime = mach_absolute_time();
+		timer_t					timer;
+
+		timer = PROCESSOR_DATA(processor, thread_timer);
+
+		return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) +
+				(abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */
+	} else
+		return 0;
+#endif
+}
+
+void dtrace_set_thread_predcache(thread_t thread, uint32_t predcache)
+{
+	if (thread != THREAD_NULL)
+		thread->t_dtrace_predcache = predcache;
+}
+
+void dtrace_set_thread_vtime(thread_t thread, int64_t vtime)
+{
+	if (thread != THREAD_NULL)
+		thread->t_dtrace_vtime = vtime;
+}
+
+void dtrace_set_thread_tracing(thread_t thread, int64_t accum)
+{
+	if (thread != THREAD_NULL)
+		thread->t_dtrace_tracing = accum;
+}
+
+void dtrace_set_thread_reentering(thread_t thread, boolean_t vbool)
+{
+	if (thread != THREAD_NULL) {
+		if (vbool)
+			thread->options |= TH_OPT_DTRACE;
+		else
+			thread->options &= (~TH_OPT_DTRACE);
+	}
+}
+
+vm_offset_t dtrace_set_thread_recover(thread_t thread, vm_offset_t recover)
+{
+	vm_offset_t prev = 0;
+
+	if (thread != THREAD_NULL) {
+		prev = thread->recover;
+		thread->recover = recover;
+	}
+	return prev;
+}
+
+#endif /* CONFIG_DTRACE */
diff --git a/osfmk/kern/thread.h b/osfmk/kern/thread.h
index bca2bc841..8387019b8 100644
--- a/osfmk/kern/thread.h
+++ b/osfmk/kern/thread.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -96,8 +102,6 @@
 #include <cputypes.h>
 
 #include <mach_assert.h>
-#include <mach_host.h>
-#include <mach_prof.h>
 #include <mach_ldebug.h>
 
 #include <ipc/ipc_types.h>
@@ -114,6 +118,7 @@
 #include <kern/timer_call.h>
 #include <kern/task.h>
 #include <kern/exception.h>
+#include <kern/affinity.h>
 
 #include <ipc/ipc_kmsg.h>
 
@@ -123,9 +128,9 @@
 struct thread {
 	/*
 	 *	NOTE:	The runq field in the thread structure has an unusual
-	 *	locking protocol.  If its value is RUN_QUEUE_NULL, then it is
+	 *	locking protocol.  If its value is PROCESSOR_NULL, then it is
 	 *	locked by the thread_lock, but if its value is something else
-	 *	(i.e. a run_queue) then it is locked by that run_queue's lock.
+	 *	then it is locked by the associated run queue lock.
 	 *
 	 *	When the thread is on a wait queue, these first three fields
 	 *	are treated as an unofficial union with a wait_queue_element.
@@ -134,19 +139,18 @@ struct thread {
 	 */
   	/* Items examined often, modified infrequently */
 	queue_chain_t	links;				/* run/wait queue links */
-	run_queue_t		runq;				/* run queue thread is on SEE BELOW */
+	processor_t		runq;				/* run queue assignment */
 	wait_queue_t	wait_queue;			/* wait queue we are currently on */
 	event64_t		wait_event;			/* wait queue event */
 	integer_t		options;			/* options set by thread itself */
 #define TH_OPT_INTMASK		0x03		/* interrupt / abort level */
 #define TH_OPT_VMPRIV		0x04		/* may allocate reserved memory */
-#define TH_OPT_DELAYIDLE	0x08		/* performing delayed idle */
-#define TH_OPT_CALLOUT		0x10		/* executing as callout */
+#define TH_OPT_DTRACE		0x08		/* executing under dtrace_probe */
 
 	/* Data updated during assert_wait/thread_wakeup */
 	decl_simple_lock_data(,sched_lock)	/* scheduling lock (thread_lock()) */
-	decl_simple_lock_data(,wake_lock)	/* covers wake_active (wake_lock())*/
-	boolean_t			wake_active;	/* Someone is waiting for this */
+	decl_simple_lock_data(,wake_lock)	/* for thread stop / wait (wake_lock()) */
+	boolean_t			wake_active;	/* wake event on stop */
 	int					at_safe_point;	/* thread_abort_safely allowed */
 	ast_t				reason;			/* why we blocked */
 	wait_result_t		wait_result;	/* outcome of wait -
@@ -175,12 +179,7 @@ struct thread {
 #define TH_UNINT		0x08			/* waiting uninteruptibly */
 #define	TH_TERMINATE	0x10			/* halted at termination */
 
-#define TH_ABORT		0x20			/* abort interruptible waits */
-#define TH_ABORT_SAFELY	0x40			/* ... but only those at safe point */
-
-#define TH_IDLE			0x80			/* processor idle thread */
-
-#define	TH_SCHED_STATE	(TH_WAIT|TH_SUSP|TH_RUN|TH_UNINT)
+#define TH_IDLE			0x80			/* idling processor */
 
 	/* Scheduling information */
 	integer_t			sched_mode;			/* scheduling mode bits */
@@ -189,8 +188,11 @@ struct thread {
 #define TH_MODE_PREEMPT			0x0004		/* can preempt kernel contexts */
 #define TH_MODE_FAILSAFE		0x0008		/* fail-safe has tripped */
 #define	TH_MODE_PROMOTED		0x0010		/* sched pri has been promoted */
-#define	TH_MODE_DEPRESS			0x0020		/* normal depress yield */
-#define TH_MODE_POLLDEPRESS		0x0040		/* polled depress yield */
+#define TH_MODE_ABORT			0x0020		/* abort interruptible waits */
+#define TH_MODE_ABORTSAFELY		0x0040		/* ... but only those at safe point */
+#define TH_MODE_ISABORTED		(TH_MODE_ABORT | TH_MODE_ABORTSAFELY)
+#define	TH_MODE_DEPRESS			0x0080		/* normal depress yield */
+#define TH_MODE_POLLDEPRESS		0x0100		/* polled depress yield */
 #define TH_MODE_ISDEPRESSED		(TH_MODE_DEPRESS | TH_MODE_POLLDEPRESS)
 
 	integer_t			sched_pri;			/* scheduled (current) priority */
@@ -218,7 +220,6 @@ struct thread {
 
   /* Data used during setrun/dispatch */
 	timer_data_t		system_timer;		/* system mode timer */
-	processor_set_t		processor_set;		/* assigned processor set */
 	processor_t			bound_processor;	/* bound to a processor? */
 	processor_t			last_processor;		/* processor last dispatched on */
 	uint64_t			last_switch;		/* time of last context switch */
@@ -229,17 +230,28 @@ struct thread {
 	integer_t			safe_mode;		/* saved mode during fail-safe */
 	natural_t			safe_release;	/* when to release fail-safe */
 
+	/* Call out from scheduler */
+	void				(*sched_call)(
+							int			type,
+							thread_t	thread);
+
 	/* Statistics and timesharing calculations */
 	natural_t			sched_stamp;	/* last scheduler tick */
 	natural_t			sched_usage;	/* timesharing cpu usage [sched] */
 	natural_t			pri_shift;		/* usage -> priority from pset */
 	natural_t			cpu_usage;		/* instrumented cpu usage [%cpu] */
 	natural_t			cpu_delta;		/* accumulated cpu_usage delta */
+	uint32_t			c_switch;		/* total context switches */
+	uint32_t			p_switch;		/* total processor switches */
+	uint32_t			ps_switch;		/* total pset switches */
 
 	/* Timing data structures */
 	timer_data_t		user_timer;			/* user mode timer */
-	uint64_t			system_timer_save;	/* saved system timer value */
 	uint64_t			user_timer_save;	/* saved user timer value */
+	uint64_t			system_timer_save;	/* saved system timer value */
+	uint64_t			vtimer_user_save;	/* saved values for vtimers */
+	uint64_t			vtimer_prof_save;
+	uint64_t			vtimer_rlim_save;
 
 	/* Timed wait expiration */
 	timer_call_data_t	wait_timer;
@@ -250,6 +262,13 @@ struct thread {
 	timer_call_data_t	depress_timer;
 	integer_t			depress_timer_active;
 
+	/*
+	 * Processor/cache affinity
+	 * - affinity_threads links task threads with the same affinity set
+	 */
+	affinity_set_t			affinity_set;
+	queue_chain_t			affinity_threads;
+
 	/* Various bits of stashed state */
 	union {
 		struct {
@@ -282,14 +301,9 @@ struct thread {
 
 	/* Ast/Halt data structures */
 	vm_offset_t			recover;		/* page fault recover(copyin/out) */
-	int					ref_count;		/* number of references to me */
+	uint32_t					ref_count;		/* number of references to me */
 
-	/* Processor set info */
-	queue_chain_t		pset_threads;	/* list of all threads in pset */
-#if	MACH_HOST
-	boolean_t			may_assign;		/* may assignment change? */
-	boolean_t			assign_active;	/* waiting for may_assign */
-#endif	/* MACH_HOST */
+	queue_chain_t		threads;		/* global list of all threads */
 
 	/* Activation */
 		queue_chain_t			task_threads;
@@ -314,12 +328,10 @@ struct thread {
 
 		/* Miscellaneous bits guarded by mutex */
 		uint32_t
-		/* Indicates that the thread has not been terminated */
-						active:1,
-
-	   /* Indicates that the thread has been started after creation */
-						started:1,
-						:0;
+			active:1,				/* Thread is active and has not been terminated */
+			started:1,				/* Thread has been started after creation */
+			static_param:1,			/* Disallow policy parameter changes */
+			:0;
 
 		/* Return Handers */
 		struct ReturnHandler {
@@ -337,16 +349,16 @@ struct thread {
 		/* Owned ulocks (a lock set element) */
 		queue_head_t			held_ulocks;
 
-#if	MACH_PROF
-		/* Profiling */
-		boolean_t				profiled;
-		boolean_t				profiled_own;
-		struct prof_data		*profil_buffer;
-#endif	/* MACH_PROF */
-
 #ifdef	MACH_BSD
 		void					*uthread;
 #endif
+
+#if CONFIG_DTRACE
+		uint32_t t_dtrace_predcache;/* DTrace per thread predicate value hint */
+		int64_t t_dtrace_tracing;       /* Thread time under dtrace_probe() */
+		int64_t t_dtrace_vtime;
+#endif
+		uint32_t t_chud;	/* CHUD flags, used for Shark */
 };
 
 #define ith_state		saved.receive.state
@@ -365,14 +377,14 @@ struct thread {
 #define sth_result		saved.sema.result
 #define sth_continuation	saved.sema.continuation
 
-extern void			thread_bootstrap(void);
+extern void			thread_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
-extern void			thread_init(void);
+extern void			thread_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern void			thread_daemon_init(void);
 
 #define	thread_reference_internal(thread)	\
-			hw_atomic_add(&(thread)->ref_count, 1)
+			(void)hw_atomic_add(&(thread)->ref_count, 1)
 
 #define thread_deallocate_internal(thread)	\
 			hw_atomic_sub(&(thread)->ref_count, 1)
@@ -380,7 +392,7 @@ extern void			thread_daemon_init(void);
 #define thread_reference(thread)					\
 MACRO_BEGIN											\
 	if ((thread) != THREAD_NULL)					\
-		thread_reference_internal(thread);			\
+		thread_reference_internal(thread);		\
 MACRO_END
 
 extern void			thread_deallocate(
@@ -391,6 +403,9 @@ extern void			thread_terminate_self(void);
 extern kern_return_t	thread_terminate_internal(
 							thread_t		thread);
 
+extern void			thread_start_internal(
+							thread_t			thread) __attribute__ ((noinline));
+
 extern void			thread_terminate_enqueue(
 						thread_t		thread);
 
@@ -406,14 +421,12 @@ extern void			thread_release(
 #define	thread_lock_init(th)	simple_lock_init(&(th)->sched_lock, 0)
 #define thread_lock(th)			simple_lock(&(th)->sched_lock)
 #define thread_unlock(th)		simple_unlock(&(th)->sched_lock)
-#define thread_lock_try(th)		simple_lock_try(&(th)->sched_lock)
-
-#define thread_should_halt_fast(thread)		(!(thread)->active)
 
 #define wake_lock_init(th)		simple_lock_init(&(th)->wake_lock, 0)
 #define wake_lock(th)			simple_lock(&(th)->wake_lock)
 #define wake_unlock(th)			simple_unlock(&(th)->wake_lock)
-#define wake_lock_try(th)		simple_lock_try(&(th)->wake_lock)
+
+#define thread_should_halt_fast(thread)		(!(thread)->active)
 
 extern void				stack_alloc(
 							thread_t		thread);
@@ -429,7 +442,7 @@ extern boolean_t		stack_alloc_try(
 
 extern void				stack_collect(void);
 
-extern void				stack_init(void);
+extern void				stack_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern kern_return_t    thread_state_initialize(
 							thread_t				thread);
@@ -516,8 +529,7 @@ extern kern_return_t	machine_thread_create(
 							thread_t		thread,
 							task_t			task);
 extern void		machine_thread_switch_addrmode(
-						       thread_t			thread,
-						       int			oldmode_is64bit);
+						       thread_t			thread);
 
 extern void 		    machine_thread_destroy(
 							thread_t		thread);
@@ -561,6 +573,8 @@ extern void			special_handler(
 						ReturnHandler	*rh,
 						thread_t		thread);
 
+void act_machine_sv_free(thread_t, int);
+
 #else	/* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS
@@ -614,6 +628,9 @@ __BEGIN_DECLS
 
 #ifdef	XNU_KERNEL_PRIVATE
 
+extern	void	thread_yield_internal(
+	mach_msg_timeout_t	interval);
+
 /*
  * XXX Funnel locks XXX
  */
@@ -643,7 +660,7 @@ extern void			thread_setentrypoint(
 						thread_t		thread,
 						mach_vm_offset_t	entry);
 
-extern void		thread_setsinglestep(
+extern kern_return_t	thread_setsinglestep(
 						thread_t		thread,
 						int			on);
 
@@ -653,12 +670,23 @@ extern kern_return_t	thread_wire_internal(
 							boolean_t		wired,
 							boolean_t		*prev_state);
 
-/* JMM - These are only temporary */
-extern boolean_t	is_thread_running(thread_t); /* True is TH_RUN */
-extern boolean_t	is_thread_idle(thread_t); /* True is TH_IDLE */
-
 extern kern_return_t	thread_dup(thread_t);
 
+typedef void	(*sched_call_t)(
+					int				type,
+					thread_t		thread);
+
+#define SCHED_CALL_BLOCK		0x1
+#define SCHED_CALL_UNBLOCK		0x2
+
+extern void		thread_sched_call(
+					thread_t		thread,
+					sched_call_t	call);
+
+extern void		thread_static_param(
+					thread_t		thread,
+					boolean_t		state);
+
 extern task_t	get_threadtask(thread_t);
 #define thread_is_64bit(thd)	\
 	task_has_64BitAddr(get_threadtask(thd))
@@ -667,11 +695,39 @@ extern task_t	get_threadtask(thread_t);
 extern void		*get_bsdthread_info(thread_t);
 extern void		set_bsdthread_info(thread_t, void *);
 extern void		*uthread_alloc(task_t, thread_t);
-extern void		uthread_free(task_t, void *, void *); 
+extern void		uthread_cleanup(task_t, void *, void *); 
+extern void		uthread_zone_free(void *); 
+extern void		uthread_cred_free(void *); 
 
 extern boolean_t	thread_should_halt(
 						thread_t		thread);
 
+extern int is_64signalregset(void);
+
+void act_set_apc(thread_t);
+
+extern uint32_t dtrace_get_thread_predcache(thread_t);
+extern int64_t dtrace_get_thread_vtime(thread_t);
+extern int64_t dtrace_get_thread_tracing(thread_t);
+extern boolean_t dtrace_get_thread_reentering(thread_t);
+extern vm_offset_t dtrace_get_kernel_stack(thread_t);
+extern void dtrace_set_thread_predcache(thread_t, uint32_t);
+extern void dtrace_set_thread_vtime(thread_t, int64_t);
+extern void dtrace_set_thread_tracing(thread_t, int64_t);
+extern void dtrace_set_thread_reentering(thread_t, boolean_t);
+extern vm_offset_t dtrace_set_thread_recover(thread_t, vm_offset_t);
+
+extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
+
+
+extern void		thread_set_wq_state32(
+					      thread_t          thread,
+					      thread_state_t    tstate);
+
+extern void		thread_set_wq_state64(
+					      thread_t          thread,
+					      thread_state_t    tstate);
+
 #endif	/* XNU_KERNEL_PRIVATE */
 
 extern kern_return_t	kernel_thread_start(
diff --git a/osfmk/kern/thread_act.c b/osfmk/kern/thread_act.c
index 872fc21dc..da6510382 100644
--- a/osfmk/kern/thread_act.c
+++ b/osfmk/kern/thread_act.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -46,7 +52,6 @@
 #include <mach/mach_types.h>
 #include <mach/kern_return.h>
 #include <mach/alert.h>
-#include <mach_prof.h>
 #include <mach/rpc.h>
 #include <mach/thread_act_server.h>
 
@@ -62,21 +67,39 @@
 #include <kern/exception.h>
 #include <kern/ipc_mig.h>
 #include <kern/ipc_tt.h>
-#include <kern/profile.h>
 #include <kern/machine.h>
 #include <kern/spl.h>
 #include <kern/syscall_subr.h>
 #include <kern/sync_lock.h>
 #include <kern/processor.h>
 #include <kern/timer.h>
-#include <mach_prof.h>
+#include <kern/affinity.h>
+
 #include <mach/rpc.h>
 
+#include <mach/sdt.h>
+
 void			act_abort(thread_t);
-void			act_set_apc(thread_t);
 void			install_special_handler_locked(thread_t);
 void			special_handler_continue(void);
 
+/*
+ * Internal routine to mark a thread as started.
+ * Always called with the thread locked.
+ *
+ * Note: function intentionall declared with the noinline attribute to
+ * prevent multiple declaration of probe symbols in this file; we would
+ * prefer "#pragma noinline", but gcc does not support it.
+ */
+void
+thread_start_internal(
+	thread_t			thread)
+{
+	clear_wait(thread, THREAD_AWAKENED);
+	thread->started = TRUE;
+	DTRACE_PROC1(lwp__start, thread_t, thread);
+}
+
 /*
  * Internal routine to terminate a thread.
  * Sometimes called with task already locked.
@@ -87,6 +110,8 @@ thread_terminate_internal(
 {
 	kern_return_t		result = KERN_SUCCESS;
 
+	DTRACE_PROC(lwp__exit);
+
 	thread_mtx_lock(thread);
 
 	if (thread->active) {
@@ -97,13 +122,15 @@ thread_terminate_internal(
 		if (thread->started)
 			clear_wait(thread, THREAD_INTERRUPTED);
 		else {
-			clear_wait(thread, THREAD_AWAKENED);
-			thread->started = TRUE;
+			thread_start_internal(thread);
 		}
 	}
 	else
 		result = KERN_TERMINATED;
 
+	if (thread->affinity_set != NULL)
+		thread_affinity_terminate(thread);
+
 	thread_mtx_unlock(thread);
 
 	if (thread != current_thread() && result == KERN_SUCCESS)
@@ -178,8 +205,7 @@ thread_release(
 		if (thread->started)
 			thread_wakeup_one(&thread->suspend_count);
 		else {
-			clear_wait(thread, THREAD_AWAKENED);
-			thread->started = TRUE;
+			thread_start_internal(thread);
 		}
 	}
 }
@@ -233,8 +259,7 @@ thread_resume(
 				if (thread->started)
 					thread_wakeup_one(&thread->suspend_count);
 				else {
-					clear_wait(thread, THREAD_AWAKENED);
-					thread->started = TRUE;
+					thread_start_internal(thread);
 				}
 			}
 		}
@@ -290,12 +315,12 @@ act_abort(
 
 	thread_lock(thread);
 
-	if (!(thread->state & TH_ABORT)) {
-		thread->state |= TH_ABORT;
+	if (!(thread->sched_mode & TH_MODE_ABORT)) {
+		thread->sched_mode |= TH_MODE_ABORT;
 		install_special_handler_locked(thread);
 	}
 	else
-		thread->state &= ~TH_ABORT_SAFELY;
+		thread->sched_mode &= ~TH_MODE_ABORTSAFELY;
 
 	thread_unlock(thread);
 	splx(s);
@@ -340,9 +365,9 @@ thread_abort_safely(
 
 		thread_lock(thread);
 		if (!thread->at_safe_point ||
-			clear_wait_internal(thread, THREAD_INTERRUPTED) != KERN_SUCCESS) {
-			if (!(thread->state & TH_ABORT)) {
-				thread->state |= (TH_ABORT|TH_ABORT_SAFELY);
+				clear_wait_internal(thread, THREAD_INTERRUPTED) != KERN_SUCCESS) {
+			if (!(thread->sched_mode & TH_MODE_ABORT)) {
+				thread->sched_mode |= TH_MODE_ISABORTED;
 				install_special_handler_locked(thread);
 			}
 		}
@@ -550,6 +575,8 @@ thread_dup(
 		if (thread_stop(target)) {
 			thread_mtx_lock(target);
 			result = machine_thread_dup(self, target);
+			if (self->affinity_set != AFFINITY_SET_NULL)
+				thread_affinity_dup(self, target);
 			thread_unstop(target);
 		}
 		else {
@@ -759,7 +786,7 @@ special_handler(
 
 	s = splsched();
 	thread_lock(thread);
-	thread->state &= ~(TH_ABORT|TH_ABORT_SAFELY);	/* clear any aborts */
+	thread->sched_mode &= ~TH_MODE_ISABORTED;
 	thread_unlock(thread);
 	splx(s);
 
diff --git a/osfmk/kern/thread_call.c b/osfmk/kern/thread_call.c
index 789e1d52f..d10c7b4bb 100644
--- a/osfmk/kern/thread_call.c
+++ b/osfmk/kern/thread_call.c
@@ -1,24 +1,29 @@
 /*
- * Copyright (c) 1993-1995, 1999-2005 Apple Computer, Inc.
- * All rights reserved.
+ * Copyright (c) 1993-1995, 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #include <mach/mach_types.h>
@@ -121,7 +126,7 @@ _remove_from_delayed_queue(
 	boolean_t			remove_all
 );
 
-static __inline__ void
+static inline void
 	_call_thread_wake(void);
 
 static void
@@ -512,7 +517,7 @@ thread_call_func(
 		call = _internal_call_allocate();
 		call->func			= func;
 		call->param0		= param;
-		call->param1		= 0;
+		call->param1		= NULL;
 	
 		_pending_call_enqueue(call);
 		
@@ -916,8 +921,7 @@ thread_call_is_delayed(
  * Postconditions:	None.
  */
 
-static __inline__
-void
+static inline void
 _call_thread_wake(void)
 {
 	if (wait_queue_wakeup_one(&call_thread_waitqueue, NULL, THREAD_AWAKENED) == KERN_SUCCESS) {
@@ -934,49 +938,34 @@ _call_thread_wake(void)
 }
 
 /*
- * Routine:	call_thread_block [private]
+ *	sched_call_thread:
  *
- * Purpose:	Hook via thread dispatch on
- *		the occasion of a callout blocking.
- *
- * Preconditions:	splsched.
- *
- * Postconditions:	None.
+ *	Call out invoked by the scheduler.
  */
 
-void
-call_thread_block(void)
+static void
+sched_call_thread(
+			int			type,
+__unused	thread_t	thread)
 {
 	simple_lock(&thread_call_lock);
 
-	if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
-		thread_call_vars.active_lowat = thread_call_vars.active_num;
+	switch (type) {
 
-	if (	thread_call_vars.active_num <= 0	&&
-			thread_call_vars.pending_num > 0		)
-		_call_thread_wake();
-
-	simple_unlock(&thread_call_lock);
-}
+	case SCHED_CALL_BLOCK:
+		if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
+			thread_call_vars.active_lowat = thread_call_vars.active_num;
 
-/*
- * Routine:	call_thread_unblock [private]
- *
- * Purpose:	Hook via thread wakeup on
- *		the occasion of a callout unblocking.
- *
- * Preconditions:	splsched.
- *
- * Postconditions:	None.
- */
-
-void
-call_thread_unblock(void)
-{
-	simple_lock(&thread_call_lock);
+		if (	thread_call_vars.active_num <= 0	&&
+				thread_call_vars.pending_num > 0		)
+			_call_thread_wake();
+		break;
 
-	if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
-		thread_call_vars.active_hiwat = thread_call_vars.active_num;
+	case SCHED_CALL_UNBLOCK:
+		if (++thread_call_vars.active_num > thread_call_vars.active_hiwat)
+			thread_call_vars.active_hiwat = thread_call_vars.active_num;
+		break;
+	}
 
 	simple_unlock(&thread_call_lock);
 }
@@ -1000,7 +989,7 @@ _call_thread_continue(void)
     (void) splsched();
     simple_lock(&thread_call_lock);
 
-	self->options |= TH_OPT_CALLOUT;
+	thread_sched_call(self, sched_call_thread);
 
     while (thread_call_vars.pending_num > 0) {
 		thread_call_t			call;
@@ -1033,7 +1022,7 @@ _call_thread_continue(void)
 		simple_lock(&thread_call_lock);
     }
 
-	self->options &= ~TH_OPT_CALLOUT;
+	thread_sched_call(self, NULL);
 
 	if (--thread_call_vars.active_num < thread_call_vars.active_lowat)
 		thread_call_vars.active_lowat = thread_call_vars.active_num;
diff --git a/osfmk/kern/thread_call.h b/osfmk/kern/thread_call.h
index 446bb1ce4..296c45079 100644
--- a/osfmk/kern/thread_call.h
+++ b/osfmk/kern/thread_call.h
@@ -1,24 +1,29 @@
 /*
- * Copyright (c) 1993-1995, 1999-2005 Apple Computer, Inc.
- * All rights reserved.
+ * Copyright (c) 1993-1995, 1999-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Declarations for thread-based callouts.
@@ -98,10 +103,6 @@ thread_call_setup(
 	thread_call_param_t		param0
 );
 
-void
-call_thread_block(void),
-call_thread_unblock(void);
-
 #endif	/* MACH_KERNEL_PRIVATE */
 
 #ifdef	KERNEL_PRIVATE
diff --git a/osfmk/kern/thread_policy.c b/osfmk/kern/thread_policy.c
index c9e75fc1e..45cd6ef24 100644
--- a/osfmk/kern/thread_policy.c
+++ b/osfmk/kern/thread_policy.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -26,6 +32,7 @@
 #include <kern/kern_types.h>
 #include <kern/processor.h>
 #include <kern/thread.h>
+#include <kern/affinity.h>
 
 static void
 thread_recompute_priority(
@@ -51,6 +58,12 @@ thread_policy_set(
 		return (KERN_TERMINATED);
 	}
 
+	if (thread->static_param) {
+		thread_mtx_unlock(thread);
+
+		return (KERN_SUCCESS);
+	}
+
 	switch (flavor) {
 
 	case THREAD_EXTENDED_POLICY:
@@ -75,15 +88,15 @@ thread_policy_set(
 			if (timeshare && !oldmode) {
 				thread->sched_mode |= TH_MODE_TIMESHARE;
 
-				if (thread->state & TH_RUN)
-					pset_share_incr(thread->processor_set);
+				if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+					sched_share_incr();
 			}
 			else
 			if (!timeshare && oldmode) {
 				thread->sched_mode &= ~TH_MODE_TIMESHARE;
 
-				if (thread->state & TH_RUN)
-					pset_share_decr(thread->processor_set);
+				if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+					sched_share_decr();
 			}
 
 			thread_recompute_priority(thread);
@@ -132,8 +145,8 @@ thread_policy_set(
 			if (thread->sched_mode & TH_MODE_TIMESHARE) {
 				thread->sched_mode &= ~TH_MODE_TIMESHARE;
 
-				if (thread->state & TH_RUN)
-					pset_share_decr(thread->processor_set);
+				if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+					sched_share_decr();
 			}
 			thread->sched_mode |= TH_MODE_REALTIME;
 			thread_recompute_priority(thread);
@@ -173,6 +186,29 @@ thread_policy_set(
 		break;
 	}
 
+	case THREAD_AFFINITY_POLICY:
+	{
+		thread_affinity_policy_t	info;
+
+		if (!thread_affinity_is_supported()) {
+			result = KERN_NOT_SUPPORTED;
+			break;
+		}
+		if (count < THREAD_AFFINITY_POLICY_COUNT) {
+			result = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		info = (thread_affinity_policy_t) policy_info;
+		/*
+		 * Unlock the thread mutex here and
+		 * return directly after calling thread_affinity_set().
+		 * This is necessary for correct lock ordering because
+		 * thread_affinity_set() takes the task lock.
+		 */
+		thread_mtx_unlock(thread);
+		return thread_affinity_set(thread, info->affinity_tag);
+	}
 	default:
 		result = KERN_INVALID_ARGUMENT;
 		break;
@@ -238,14 +274,19 @@ void
 thread_policy_reset(
 	thread_t		thread)
 {
+	spl_t		s;
+
+	s = splsched();
+	thread_lock(thread);
+
 	if (!(thread->sched_mode & TH_MODE_FAILSAFE)) {
 		thread->sched_mode &= ~TH_MODE_REALTIME;
 
 		if (!(thread->sched_mode & TH_MODE_TIMESHARE)) {
 			thread->sched_mode |= TH_MODE_TIMESHARE;
 
-			if (thread->state & TH_RUN)
-				pset_share_incr(thread->processor_set);
+			if ((thread->state & (TH_RUN|TH_IDLE)) == TH_RUN)
+				sched_share_incr();
 		}
 	}
 	else {
@@ -256,6 +297,9 @@ thread_policy_reset(
 	thread->importance = 0;
 
 	thread_recompute_priority(thread);
+
+	thread_unlock(thread);
+	splx(s);
 }
 
 kern_return_t
@@ -378,6 +422,29 @@ thread_policy_get(
 		break;
 	}
 
+	case THREAD_AFFINITY_POLICY:
+	{
+		thread_affinity_policy_t		info;
+
+		if (!thread_affinity_is_supported()) {
+			result = KERN_NOT_SUPPORTED;
+			break;
+		}
+		if (*count < THREAD_AFFINITY_POLICY_COUNT) {
+			result = KERN_INVALID_ARGUMENT;
+			break;
+		}
+
+		info = (thread_affinity_policy_t)policy_info;
+
+		if (!(*get_default))
+			info->affinity_tag = thread_affinity_get(thread);
+		else
+			info->affinity_tag = THREAD_AFFINITY_TAG_NULL;
+
+		break;
+	}
+
 	default:
 		result = KERN_INVALID_ARGUMENT;
 		break;
diff --git a/osfmk/kern/timer.c b/osfmk/kern/timer.c
index 4dc36f6b5..b53419857 100644
--- a/osfmk/kern/timer.c
+++ b/osfmk/kern/timer.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -68,12 +74,12 @@ void
 timer_init(
 	timer_t		timer)
 {
-	timer->low_bits = 0;
-	timer->high_bits = 0;
-	timer->high_bits_check = 0;
 #if	!STAT_TIME
 	timer->tstamp = 0;
 #endif	/* STAT_TIME */
+	timer->low_bits = 0;
+	timer->high_bits = 0;
+	timer->high_bits_check = 0;
 }
 
 /*
@@ -92,43 +98,49 @@ timer_delta(
 	return (new - old);
 }
 
+void
+timer_advance(
+	timer_t		timer,
+	uint64_t	delta)
+{
+	uint64_t	low;
+
+	low = delta + timer->low_bits;
+	if (low >> 32)
+		timer_update(timer, timer->high_bits + (low >> 32), low);
+	else
+		timer->low_bits = low;
+}
+
 #if	!STAT_TIME
 
+void
+timer_start(
+	timer_t		timer,
+	uint64_t	tstamp)
+{
+	timer->tstamp = tstamp;
+}
+
+void
+timer_stop(
+	timer_t		timer,
+	uint64_t	tstamp)
+{
+	timer_advance(timer, tstamp - timer->tstamp);
+}
+
 /*
- *	Update the current timer (if any)
- *	and start the new timer, which
- *	could be either the same or NULL.
- *
- *	Called with interrupts disabled.
+ *	Update the timer and start a new one.
  */
 void
 timer_switch(
-	uint32_t		tstamp,
+	timer_t			timer,
+	uint64_t		tstamp,
 	timer_t			new_timer)
 {
-	processor_t		processor = current_processor();
-	timer_t			timer;
-	uint32_t		old_low, low;
-
-	/*
-	 *	Update current timer.
-	 */
-	timer = PROCESSOR_DATA(processor, current_timer);
-	if (timer != NULL) {
-		old_low = timer->low_bits;
-		low = old_low + tstamp - timer->tstamp;
-		if (low < old_low)
-			timer_update(timer, timer->high_bits + 1, low);
-		else
-			timer->low_bits = low;
-	}
-
-	/*
-	 *	Start new timer.
-	 */
-	PROCESSOR_DATA(processor, current_timer) = new_timer;
-	if (new_timer != NULL)
-		new_timer->tstamp = tstamp;
+	timer_advance(timer, tstamp - timer->tstamp);
+	new_timer->tstamp = tstamp;
 }
 
 #if	MACHINE_TIMER_ROUTINES
@@ -140,36 +152,30 @@ timer_switch(
 #else	/* MACHINE_TIMER_ROUTINES */
 
 /*
- *	Update the current timer and start
- *	the new timer.  Requires a current
+ *	Update the current thread timer and
+ *	start the new timer.  Requires a current
  *	and new timer.
  *
  *	Called with interrupts disabled.
  */
 void
-timer_event(
-	uint32_t		tstamp,
+thread_timer_event(
+	uint64_t		tstamp,
 	timer_t			new_timer)
 {
 	processor_t		processor = current_processor();
 	timer_t			timer;
-	uint32_t		old_low, low;
 
 	/*
 	 *	Update current timer.
 	 */
-	timer = PROCESSOR_DATA(processor, current_timer);
-	old_low = timer->low_bits;
-	low = old_low + tstamp - timer->tstamp;
-	if (low < old_low)
-		timer_update(timer, timer->high_bits + 1, low);
-	else
-		timer->low_bits = low;
+	timer = PROCESSOR_DATA(processor, thread_timer);
+	timer_advance(timer, tstamp - timer->tstamp);
 
 	/*
 	 *	Start new timer.
 	 */
-	PROCESSOR_DATA(processor, current_timer) = new_timer;
+	PROCESSOR_DATA(processor, thread_timer) = new_timer;
 	new_timer->tstamp = tstamp;
 }
 
diff --git a/osfmk/kern/timer.h b/osfmk/kern/timer.h
index 1c202efd3..9c0fae288 100644
--- a/osfmk/kern/timer.h
+++ b/osfmk/kern/timer.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,12 +69,12 @@
  */
 
 struct timer {
+#if	!STAT_TIME
+	uint64_t	tstamp;
+#endif	/* STAT_TIME */
 	uint32_t	low_bits;
 	uint32_t	high_bits;
 	uint32_t	high_bits_check;
-#if	!STAT_TIME
-	uint32_t	tstamp;
-#endif	/* STAT_TIME */
 };
 
 typedef struct timer	timer_data_t, *timer_t;
@@ -81,7 +87,7 @@ typedef struct timer	timer_data_t, *timer_t;
 
 #include <kern/macro_help.h>
 
-/* Advance a timer by the specified amount */
+/* Advance a timer by a 32 bit value */
 #define TIMER_BUMP(timer, ticks)								\
 MACRO_BEGIN														\
 	uint32_t	old_low, low;									\
@@ -94,17 +100,35 @@ MACRO_BEGIN														\
 		(timer)->low_bits = low;								\
 MACRO_END
 
-#define timer_switch(tstamp, new_timer)
-#define timer_event(tstamp, new_timer)
+#define timer_start(timer, tstamp)
+#define timer_stop(timer, tstamp)
+#define timer_switch(timer, tstamp, new_timer)
+#define thread_timer_event(tstamp, new_timer)
 
 #else	/* STAT_TIME */
 
-/* Update the current timer and start a new one */
+#define	TIMER_BUMP(timer, ticks)
+
+/* Start a timer by setting the timestamp */
+extern void		timer_start(
+					timer_t		timer,
+					uint64_t	tstamp);
+
+/* Stop a timer by updating from the timestamp */
+extern void		timer_stop(
+					timer_t		timer,
+					uint64_t	tstamp);
+
+/* Update the timer and start a new one */
 extern void		timer_switch(
-					uint32_t	tstamp,
+					timer_t		timer,
+					uint64_t	tstamp,
 					timer_t		new_timer);
 
-#define	TIMER_BUMP(timer, ticks)
+/* Update the thread timer at an event */
+extern void		thread_timer_event(
+					uint64_t	tstamp,
+					timer_t		new_timer);
 
 #endif	/* STAT_TIME */
 
@@ -117,6 +141,11 @@ extern uint64_t	timer_delta(
 					timer_t		timer,
 					uint64_t	*save);
 
+/* Advance a timer by a 64 bit value */
+extern void		timer_advance(
+					timer_t		timer,
+					uint64_t	delta);
+
 /*
  *	Exported hardware interface to timers
  */
@@ -131,13 +160,4 @@ extern void		timer_update(
 					uint32_t	new_high,
 					uint32_t	new_low);
 
-#if	!STAT_TIME
-
-/* Update the current timer at an event */
-extern void		timer_event(
-					uint32_t	tstamp,
-					timer_t		new_timer);
-
-#endif	/* STAT_TIME */
-
 #endif	/* _KERN_TIMER_H_ */
diff --git a/osfmk/kern/timer_call.c b/osfmk/kern/timer_call.c
index 59b0a96b7..941061c3d 100644
--- a/osfmk/kern/timer_call.c
+++ b/osfmk/kern/timer_call.c
@@ -1,50 +1,49 @@
 /*
- * Copyright (c) 1993-1995, 1999-2004 Apple Computer, Inc.
- * All rights reserved.
+ * Copyright (c) 1993-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Timer interrupt callout module.
- *
- * HISTORY
- *
- * 20 December 2000 (debo)
- *	Created.
  */
 
 #include <mach/mach_types.h>
 
 #include <kern/clock.h>
 #include <kern/processor.h>
-
+#include <kern/etimer.h>
 #include <kern/timer_call.h>
 #include <kern/call_entry.h>
 
 #include <sys/kdebug.h>
 
-decl_simple_lock_data(static,timer_call_lock)
+#if CONFIG_DTRACE && (DEVELOPMENT || DEBUG )
+#include <mach/sdt.h>
+#endif
 
-static struct {
-	int		delayed_num,
-			delayed_hiwat;
-} timer_call_vars;
+decl_simple_lock_data(static,timer_call_lock)
 
 static void
 timer_call_interrupt(
@@ -99,8 +98,6 @@ _delayed_call_enqueue(
 	}
 
 	insque(qe(call), qe(current));
-	if (++timer_call_vars.delayed_num > timer_call_vars.delayed_hiwat)
-		timer_call_vars.delayed_hiwat = timer_call_vars.delayed_num;
 
 	call->state = DELAYED;
 }
@@ -111,7 +108,6 @@ _delayed_call_dequeue(
 	timer_call_t			call)
 {
 	(void)remque(qe(call));
-	timer_call_vars.delayed_num--;
 
 	call->state = IDLE;
 }
@@ -141,7 +137,7 @@ timer_call_enter(
 	else
 		result = FALSE;
 
-	call->param1	= 0;
+	call->param1	= NULL;
 	call->deadline	= deadline;
 
 	queue = &PROCESSOR_DATA(current_processor(), timer_call_queue);
@@ -201,8 +197,18 @@ timer_call_cancel(
 	s = splclock();
 	simple_lock(&timer_call_lock);
 
-	if (call->state == DELAYED)
-		_delayed_call_dequeue(call);
+	if (call->state == DELAYED) {
+		queue_t			queue = &PROCESSOR_DATA(current_processor(), timer_call_queue);
+
+		if (queue_first(queue) == qe(call)) {
+			_delayed_call_dequeue(call);
+
+			if (!queue_empty(queue))
+				_set_delayed_call_timer((timer_call_t)queue_first(queue));
+		}
+		else
+			_delayed_call_dequeue(call);
+	}
 	else
 		result = FALSE;
 
@@ -271,10 +277,8 @@ timer_call_shutdown(
 	simple_unlock(&timer_call_lock);
 }
 
-static
-void
-timer_call_interrupt(
-	uint64_t				timestamp)
+static void
+timer_call_interrupt(uint64_t timestamp)
 {
 	timer_call_t		call;
 	queue_t				queue;
@@ -286,7 +290,6 @@ timer_call_interrupt(
 	call = TC(queue_first(queue));
 
 	while (!queue_end(queue, qe(call))) {
-
 		if (call->deadline <= timestamp) {
 			timer_call_func_t		func;
 			timer_call_param_t		param0, param1;
@@ -299,15 +302,36 @@ timer_call_interrupt(
 
 			simple_unlock(&timer_call_lock);
 
-			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 2) | DBG_FUNC_START, (int)func, param0, param1, 0, 0);
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI,
+							   2)
+							| DBG_FUNC_START,
+					      (unsigned int)func,
+					      (unsigned int)param0,
+					      (unsigned int)param1, 0, 0);
+
+#if CONFIG_DTRACE && (DEVELOPMENT || DEBUG )
+			DTRACE_TMR3(callout__start, timer_call_func_t, func, 
+										timer_call_param_t, param0, 
+										timer_call_param_t, param1);
+#endif
 
 			(*func)(param0, param1);
 
-			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 2) | DBG_FUNC_END, (int)func, param0, param1, 0, 0);
+#if CONFIG_DTRACE && (DEVELOPMENT || DEBUG )
+			DTRACE_TMR3(callout__end, timer_call_func_t, func, 
+										timer_call_param_t, param0, 
+										timer_call_param_t, param1);
+#endif
+
+			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI,
+							   2)
+							| DBG_FUNC_END,
+					      (unsigned int)func,
+					      (unsigned int)param0,
+					      (unsigned int)param1, 0, 0);
 
 			simple_lock(&timer_call_lock);
-		}
-		else
+		} else
 			break;
 
 		call = TC(queue_first(queue));
@@ -317,5 +341,4 @@ timer_call_interrupt(
 		_set_delayed_call_timer(call);
 
 	simple_unlock(&timer_call_lock);
-
 }
diff --git a/osfmk/kern/timer_call.h b/osfmk/kern/timer_call.h
index bc5ef3773..d3beccfc7 100644
--- a/osfmk/kern/timer_call.h
+++ b/osfmk/kern/timer_call.h
@@ -2,23 +2,29 @@
  * Copyright (c) 1993-1995, 1999-2000 Apple Computer, Inc.
  * All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Declarations for timer interrupt callouts.
diff --git a/osfmk/kern/wait_queue.c b/osfmk/kern/wait_queue.c
index 361cf7961..187ac8b13 100644
--- a/osfmk/kern/wait_queue.c
+++ b/osfmk/kern/wait_queue.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -472,13 +478,13 @@ wait_queue_link(
 	wait_queue_link_t wql;
 	kern_return_t ret;
 
-	wql = (wait_queue_link_t) kalloc(sizeof(struct wait_queue_link));
+	wql = (wait_queue_link_t) kalloc(sizeof(struct _wait_queue_link));
 	if (wql == WAIT_QUEUE_LINK_NULL)
 		return KERN_RESOURCE_SHORTAGE;
 
 	ret = wait_queue_link_noalloc(wq, wq_set, wql);
 	if (ret != KERN_SUCCESS)
-		kfree(wql, sizeof(struct wait_queue_link));
+		kfree(wql, sizeof(struct _wait_queue_link));
 
 	return ret;
 }	
@@ -545,7 +551,7 @@ wait_queue_unlink(
 				wqs_unlock(wq_set);
 				wait_queue_unlock(wq);
 				splx(s);
-				kfree(wql, sizeof(struct wait_queue_link));
+				kfree(wql, sizeof(struct _wait_queue_link));
 				return KERN_SUCCESS;
 			}
 		}
@@ -665,7 +671,7 @@ wait_queue_unlink_all(
 
 	while(!queue_empty(links)) {
 		wql = (wait_queue_link_t) dequeue(links);
-		kfree(wql, sizeof(struct wait_queue_link));
+		kfree(wql, sizeof(struct _wait_queue_link));
 	}
 
 	return(KERN_SUCCESS);
@@ -782,7 +788,7 @@ retry:
 
 	while (!queue_empty (links)) {
 		wql = (wait_queue_link_t) dequeue(links);
-		kfree(wql, sizeof(struct wait_queue_link));
+		kfree(wql, sizeof(struct _wait_queue_link));
 	}
 	return(KERN_SUCCESS);
 }	
@@ -821,7 +827,7 @@ wait_queue_unlink_one(
 			wqs_unlock(wq_set);
 			wait_queue_unlock(wq);
 			splx(s);
-			kfree(wql,sizeof(struct wait_queue_link));
+			kfree(wql,sizeof(struct _wait_queue_link));
 			*wq_setp = wq_set;
 			return;
 		}
@@ -1060,9 +1066,9 @@ wait_queue_wakeup64_all_locked(
 	kern_return_t res;
 
 //	assert(wait_queue_held(wq));
-	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
-		panic("wait_queue_wakeup64_all_locked: lock not held on %08X\n", wq);	/* (BRINGUP) */
-	}
+//	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
+//		panic("wait_queue_wakeup64_all_locked: lock not held on %p\n", wq);	/* (BRINGUP) */
+//	}
 
 	queue_init(q);
 
@@ -1115,9 +1121,9 @@ wait_queue_wakeup_all(
 
 	s = splsched();
 	wait_queue_lock(wq);
-	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
-		panic("wait_queue_wakeup_all: we did not get the lock on %08X\n", wq);	/* (BRINGUP) */
-	}
+//	if(!wq->wq_interlock.lock_data) {		/* (BRINGUP */
+//		panic("wait_queue_wakeup_all: we did not get the lock on %p\n", wq);	/* (BRINGUP) */
+//	}
 	ret = wait_queue_wakeup64_all_locked(
 				wq, (event64_t)((uint32_t)event),
 				result, TRUE);
@@ -1235,94 +1241,12 @@ _wait_queue_select64_one(
 	return THREAD_NULL;
 }
 
-/*
- *	Routine:	wait_queue_peek64_locked
- *	Purpose:
- *		Select the best thread from a wait queue that meet the
- *		supplied criteria, but leave it on the queue it was
- *		found on.  The thread, and the actual wait_queue the
- *		thread was found on are identified.
- * 	Conditions:
- *		at splsched
- *		wait queue locked
- *		possibly recursive
- * 	Returns:
- *		a locked thread - if one found
- *		a locked waitq - the one the thread was found on
- *	Note:
- *		Both the waitq the thread was actually found on, and
- *		the supplied wait queue, are locked after this.
- */
-__private_extern__ void
-wait_queue_peek64_locked(
-	wait_queue_t wq,
-	event64_t event,
-	thread_t *tp,
-	wait_queue_t *wqp)
-{
-	wait_queue_element_t wq_element;
-	wait_queue_element_t wqe_next;
-	queue_t q;
-
-	assert(wq->wq_fifo);
-
-	*tp = THREAD_NULL;
-
-	q = &wq->wq_queue;
-
-	wq_element = (wait_queue_element_t) queue_first(q);
-	while (!queue_end(q, (queue_entry_t)wq_element)) {
-		WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
-		wqe_next = (wait_queue_element_t)
-			       queue_next((queue_t) wq_element);
-
-		/*
-		 * We may have to recurse if this is a compound wait queue.
-		 */
-		if (wq_element->wqe_type == WAIT_QUEUE_LINK) {
-			wait_queue_link_t wql = (wait_queue_link_t)wq_element;
-			wait_queue_t set_queue;
-
-			/*
-			 * We have to check the set wait queue.
-			 */
-			set_queue = (wait_queue_t)wql->wql_setqueue;
-			wait_queue_lock(set_queue);
-			if (! wait_queue_empty(set_queue)) {
-				wait_queue_peek64_locked(set_queue, event, tp, wqp);
-			}
-			if (*tp != THREAD_NULL) {
-				if (*wqp != set_queue)
-					wait_queue_unlock(set_queue);
-				return;  /* thread and its waitq locked */
-			}
-
-			wait_queue_unlock(set_queue);
-		} else {
-			
-			/*
-			 * Otherwise, its a thread.  If it is waiting on
-			 * the event we are posting to this queue, return
-			 * it locked, but leave it on the queue.
-			 */
-			thread_t t = (thread_t)wq_element;
-
-			if (t->wait_event == event) {
-				thread_lock(t);
-				*tp = t;
-				*wqp = wq;
-				return;
-			}
-		}
-		wq_element = wqe_next;
-	}
-}
 
 /*
  *	Routine:	wait_queue_pull_thread_locked
  *	Purpose:
- *		Pull a thread that was previously "peeked" off the wait
- *		queue and (possibly) unlock the waitq.
+ *		Pull a thread off its wait queue and (possibly) unlock 
+ *		the waitq.
  * 	Conditions:
  *		at splsched
  *		wait queue locked
@@ -1441,7 +1365,6 @@ wait_queue_wakeup64_identity_locked(
 
 	assert(wait_queue_held(wq));
 
-
 	thread = _wait_queue_select64_one(wq, event);
 	if (unlock)
 		wait_queue_unlock(wq);
diff --git a/osfmk/kern/wait_queue.h b/osfmk/kern/wait_queue.h
index 0af646ff1..5a1be3f35 100644
--- a/osfmk/kern/wait_queue.h
+++ b/osfmk/kern/wait_queue.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifdef	KERNEL_PRIVATE
@@ -117,7 +123,7 @@ typedef WaitQueueElement *wait_queue_element_t;
  *	with that port may wake up any thread from any of those portsets,
  *	or one that was waiting locally on the port itself.
  */
-typedef struct wait_queue_link {
+typedef struct _wait_queue_link {
 	WaitQueueElement		wql_element;	/* element on master */
 	queue_chain_t			wql_setlinks;	/* element on set */
     wait_queue_set_t		wql_setqueue;	/* set queue */
@@ -140,9 +146,16 @@ typedef struct wait_queue_link {
 	(((wq)->wq_type & ~1) == _WAIT_QUEUE_inited)
 
 #define wait_queue_empty(wq)	(queue_empty(&(wq)->wq_queue))
+
 #define wait_queue_held(wq)		(hw_lock_held(&(wq)->wq_interlock))
 #define wait_queue_lock_try(wq) (hw_lock_try(&(wq)->wq_interlock))
 
+/* For x86, the hardware timeout is in TSC units. */
+#if defined(i386)
+#define	hwLockTimeOut LockTimeOutTSC
+#else
+#define	hwLockTimeOut LockTimeOut
+#endif
 /*
  * Double the standard lock timeout, because wait queues tend
  * to iterate over a number of threads - locking each.  If there is
@@ -151,20 +164,21 @@ typedef struct wait_queue_link {
  */
 
 static inline void wait_queue_lock(wait_queue_t wq) {
-	if (!hw_lock_to(&(wq)->wq_interlock, LockTimeOut * 2))
-		panic("wait queue deadlock - wq=0x%x, cpu=%d\n", wq, cpu_number());
+	if (!hw_lock_to(&(wq)->wq_interlock, hwLockTimeOut * 2))
+		panic("wait queue deadlock - wq=%p, cpu=%d\n", wq, cpu_number(
+));
 }
-
+ 
 static inline void wait_queue_unlock(wait_queue_t wq) {
 	assert(wait_queue_held(wq));
 #if defined(__i386__)
-	/* On certain x86 systems, this spinlock is susceptible to
+	/* DRK: On certain x86 systems, this spinlock is susceptible to
 	 * lock starvation. Hence use an unlock variant which performs
 	 * a cacheline flush to minimize cache affinity on acquisition.
 	 */
 	i386_lock_unlock_with_flush(&(wq)->wq_interlock);
 #else
-        hw_lock_unlock(&(wq)->wq_interlock);
+	hw_lock_unlock(&(wq)->wq_interlock);
 #endif
 }
 
@@ -185,14 +199,7 @@ __private_extern__ wait_result_t wait_queue_assert_wait64_locked(
 			uint64_t deadline,
 			thread_t thread);
 
-/* peek to see which thread would be chosen for a wakeup - but keep on queue */
-__private_extern__ void wait_queue_peek64_locked(
-			wait_queue_t wait_queue,
-			event64_t event,
-			thread_t *thread,
-			wait_queue_t *found_queue);
-
-/* peek to see which thread would be chosen for a wakeup - but keep on queue */
+/* pull a thread from its wait queue */
 __private_extern__ void wait_queue_pull_thread_locked(
 			wait_queue_t wait_queue,
 			thread_t thread,
diff --git a/osfmk/kern/xpr.c b/osfmk/kern/xpr.c
index d6088c4c1..763259873 100644
--- a/osfmk/kern/xpr.c
+++ b/osfmk/kern/xpr.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -189,17 +195,22 @@ extern jmp_buf_t *db_recover;
  *	Called with arguments, it can dump xpr buffers in user tasks,
  *	assuming they use the same format as the kernel.
  */
+static spl_t xpr_dump_spl;
+static struct xprbuf *base;
+static int nbufs;
 void
 xpr_dump(
-	struct xprbuf	*base,
-	int		nbufs)
+	struct xprbuf	*_base,
+	int		_nbufs)
 {
 	jmp_buf_t db_jmpbuf;
 	jmp_buf_t *prev;
 	struct xprbuf *last, *ptr;
 	register struct xprbuf *x;
 	int i;
-	spl_t s;
+
+	base = _base;
+	nbufs = _nbufs;
 
 	if (base == 0) {
 		base = xprbase;
@@ -210,7 +221,7 @@ xpr_dump(
 		return;
 
 	if (base == xprbase) {
-		s = splhigh();
+		xpr_dump_spl = splhigh();
 		simple_lock(&xprlock);
 	}
 
@@ -233,7 +244,7 @@ xpr_dump(
 
 	if (base == xprbase) {
 		simple_unlock(&xprlock);
-		splx(s);
+		splx(xpr_dump_spl);
 	}
 }
 
diff --git a/osfmk/kern/xpr.h b/osfmk/kern/xpr.h
index 39430bf29..3a2a92d9b 100644
--- a/osfmk/kern/xpr.h
+++ b/osfmk/kern/xpr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/zalloc.c b/osfmk/kern/zalloc.c
index 0661d36a3..bdb373604 100644
--- a/osfmk/kern/zalloc.c
+++ b/osfmk/kern/zalloc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,6 +63,7 @@
  *	data blocks for which quick allocation/deallocation is possible.
  */
 #include <zone_debug.h>
+#include <zone_alias_addr.h>
 #include <norma_vm.h>
 #include <mach_kdb.h>
 
@@ -86,12 +93,17 @@
 
 #include <machine/machparam.h>
 
+#include <libkern/OSDebug.h>
+#include <sys/kdebug.h>
+
 #if defined(__ppc__)
 /* for fake zone stat routines */
 #include <ppc/savearea.h>
 #include <ppc/mappings.h>
 #endif
 
+int check_freed_element = 0;
+
 #if	MACH_ASSERT
 /* Detect use of zone elt after freeing it by two methods:
  * (1) Range-check the free-list "next" ptr for sanity.
@@ -120,7 +132,7 @@ boolean_t zfree_clear = FALSE;
 #define ADD_TO_ZONE(zone, element)					\
 MACRO_BEGIN								\
 		if (zfree_clear)					\
-		{   unsigned int i;						\
+		{   unsigned int i;					\
 		    for (i=1;						\
 			 i < zone->elem_size/sizeof(vm_offset_t) - 1;	\
 			 i++)						\
@@ -147,6 +159,11 @@ MACRO_END
 #define ADD_TO_ZONE(zone, element)					\
 MACRO_BEGIN								\
 		*((vm_offset_t *)(element)) = (zone)->free_elements;	\
+		if (check_freed_element) {  \
+			if ((zone)->elem_size >= (2 * sizeof(vm_offset_t)))	\
+				((vm_offset_t *)(element))[((zone)->elem_size/sizeof(vm_offset_t))-1] = \
+					(zone)->free_elements;				\
+		}	\
 		(zone)->free_elements = (vm_offset_t) (element);	\
 		(zone)->count--;					\
 MACRO_END
@@ -155,6 +172,12 @@ MACRO_END
 MACRO_BEGIN								\
 	(ret) = (type) (zone)->free_elements;				\
 	if ((ret) != (type) 0) {					\
+		if (check_freed_element) {		\
+	        if ((zone)->elem_size >= (2 * sizeof(vm_offset_t)) &&		\
+		    ((vm_offset_t *)(ret))[((zone)->elem_size/sizeof(vm_offset_t))-1] != \
+		    ((vm_offset_t *)(ret))[0])				\
+		        panic("a freed zone element has been modified");\
+		}		\
 		(zone)->count++;					\
 		(zone)->free_elements = *((vm_offset_t *)(ret));	\
 	}								\
@@ -230,26 +253,32 @@ vm_size_t	zdata_size;
 
 #define lock_zone(zone)					\
 MACRO_BEGIN						\
-	mutex_lock(&(zone)->lock);			\
+	lck_mtx_lock(&(zone)->lock);			\
 MACRO_END
 
 #define unlock_zone(zone)				\
 MACRO_BEGIN						\
-	mutex_unlock(&(zone)->lock);			\
+	lck_mtx_unlock(&(zone)->lock);			\
 MACRO_END
 
 #define zone_wakeup(zone) thread_wakeup((event_t)(zone))
 #define zone_sleep(zone)				\
-	thread_sleep_mutex((event_t)(zone),	\
-				&(zone)->lock,		\
-				THREAD_UNINT)
+	(void) lck_mtx_sleep(&(zone)->lock, 0, (event_t)(zone), THREAD_UNINT);
+
+extern int snprintf(char *, size_t, const char *, ...) __printflike(3,4);
 
 #define lock_zone_init(zone)				\
 MACRO_BEGIN						\
-	mutex_init(&zone->lock, 0);	\
+	char _name[32];					\
+	(void) snprintf(_name, sizeof (_name), "zone.%s", (zone)->zone_name); \
+	lck_grp_attr_setdefault(&(zone)->lock_grp_attr);		\
+	lck_grp_init(&(zone)->lock_grp, _name, &(zone)->lock_grp_attr);	\
+	lck_attr_setdefault(&(zone)->lock_attr);			\
+	lck_mtx_init_ext(&(zone)->lock, &(zone)->lock_ext,		\
+	    &(zone)->lock_grp, &(zone)->lock_attr);			\
 MACRO_END
 
-#define lock_try_zone(zone)	mutex_try(&zone->lock)
+#define lock_try_zone(zone)	lck_mtx_try_lock(&zone->lock)
 
 kern_return_t		zget_space(
 				vm_offset_t size,
@@ -273,9 +302,15 @@ unsigned int			zone_pages;
  */
 decl_mutex_data(,		zone_gc_lock)
 
+#if	!ZONE_ALIAS_ADDR
 #define from_zone_map(addr, size) \
 	((vm_offset_t)(addr) >= zone_map_min_address && \
 	 ((vm_offset_t)(addr) + size -1) <  zone_map_max_address)
+#else
+#define from_zone_map(addr, size) \
+	((vm_offset_t)(zone_virtual_addr((vm_map_address_t)addr)) >= zone_map_min_address && \
+	 ((vm_offset_t)(zone_virtual_addr((vm_map_address_t)addr)) + size -1) <  zone_map_max_address)
+#endif
 
 #define	ZONE_PAGE_USED  0
 #define ZONE_PAGE_UNUSED -1
@@ -338,6 +373,11 @@ zinit(
 	 * This size will be used unless
 	 * the user suggestion is larger AND has less fragmentation
 	 */
+#if	ZONE_ALIAS_ADDR
+	if ((size < PAGE_SIZE) && (PAGE_SIZE % size <= PAGE_SIZE / 10))
+		alloc = PAGE_SIZE;
+	else
+#endif
 	{	vm_size_t best, waste; unsigned int i;
 		best  = PAGE_SIZE;
 		waste = best % size;
@@ -379,7 +419,7 @@ use_this_allocation:
 	z->async_pending = FALSE;
 
 #if	ZONE_DEBUG
-	z->active_zones.next = z->active_zones.prev = 0;	
+	z->active_zones.next = z->active_zones.prev = NULL;	
 	zone_debug_enable(z);
 #endif	/* ZONE_DEBUG */
 	lock_zone_init(z);
@@ -473,6 +513,10 @@ zget_space(
 				space_to_add, 0, KMA_KOBJECT|KMA_NOPAGEWAIT);
 			if (retval != KERN_SUCCESS)
 				return(retval);
+#if	ZONE_ALIAS_ADDR
+		 	if (space_to_add == PAGE_SIZE)
+				new_space = zone_alias_addr(new_space);
+#endif
 			zone_page_init(new_space, space_to_add,
 							ZONE_PAGE_USED);
 			simple_lock(&zget_space_lock);
@@ -568,6 +612,12 @@ zone_bootstrap(void)
 {
 	vm_size_t zone_zone_size;
 	vm_offset_t zone_zone_space;
+	char temp_buf[16];
+
+	/* see if we want freed zone element checking */
+	if (PE_parse_boot_arg("-zc", temp_buf)) {
+		check_freed_element = 1;
+	}
 
 	simple_lock_init(&all_zones_lock, 0);
 
@@ -686,11 +736,11 @@ zalloc_canblock(
 			if (zone->collectable) {
 				vm_offset_t space;
 				vm_size_t alloc_size;
-				boolean_t retry = FALSE;
+				int retry = 0;
 
 				for (;;) {
 
-				        if (vm_pool_low() || retry == TRUE)
+				        if (vm_pool_low() || retry >= 1)
 					        alloc_size = 
 						  round_page(zone->elem_size);
 					else
@@ -700,16 +750,24 @@ zalloc_canblock(
 									&space, alloc_size, 0,
 									KMA_KOBJECT|KMA_NOPAGEWAIT);
 					if (retval == KERN_SUCCESS) {
+#if	ZONE_ALIAS_ADDR
+						if (alloc_size == PAGE_SIZE)
+							space = zone_alias_addr(space);
+#endif
 					        zone_page_init(space, alloc_size,
 							       ZONE_PAGE_USED);
 						zcram(zone, (void *)space, alloc_size);
 
 						break;
 					} else if (retval != KERN_RESOURCE_SHORTAGE) {
-					        /* would like to cause a zone_gc() */
-					        if (retry == TRUE)
+						retry++;
+
+						if (retry == 2) {
+							zone_gc();
+							printf("zalloc did gc\n");
+						}
+					        if (retry == 3)
 						        panic("zalloc: \"%s\" (%d elements) retry fail %d", zone->zone_name, zone->count, retval);
-						retry = TRUE;
 					} else {
 					        break;
 					}
@@ -752,7 +810,8 @@ zalloc_canblock(
 					if (zone_debug_enabled(zone))
 						space += ZONE_DEBUG_OFFSET;
 #endif
-					return((void *)space);
+					addr = space;
+					goto success;
 				}
 				if (retval == KERN_RESOURCE_SHORTAGE) {
 					unlock_zone(zone);
@@ -768,7 +827,7 @@ zalloc_canblock(
 			REMOVE_FROM_ZONE(zone, addr, vm_offset_t);
 	}
 
-	if ((addr == 0) && !canblock && (zone->async_pending == FALSE) && (!vm_pool_low())) {
+	if ((addr == 0) && !canblock && (zone->async_pending == FALSE) && (zone->exhaustible == FALSE) && (!vm_pool_low())) {
 		zone->async_pending = TRUE;
 		unlock_zone(zone);
 		thread_call_enter(&zone->call_async_alloc);
@@ -785,6 +844,9 @@ zalloc_canblock(
 
 	unlock_zone(zone);
 
+success:
+	TRACE_MACHLEAKS(ZALLOC_CODE, ZALLOC_CODE_2, zone->elem_size, addr);
+
 	return((void *)addr);
 }
 
@@ -871,6 +933,8 @@ zfree(
 		panic("zfree: freeing to zone_zone breaks zone_gc!");
 #endif
 
+	TRACE_MACHLEAKS(ZFREE_CODE, ZFREE_CODE_2, zone->elem_size, (int)addr);
+
 	if (zone->collectable && !zone->allows_foreign &&
 	    !from_zone_map(elem, zone->elem_size)) {
 #if MACH_ASSERT
@@ -958,7 +1022,6 @@ zone_change(
 			/* break; */
 #endif
 	}
-	lock_zone_init(zone);
 }
 
 /*
@@ -1013,6 +1076,9 @@ zone_page_collectable(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_collectable");
@@ -1036,6 +1102,9 @@ zone_page_keep(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_keep");
@@ -1056,6 +1125,9 @@ zone_page_collect(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_collect");
@@ -1077,6 +1149,9 @@ zone_page_init(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_init");
@@ -1099,6 +1174,9 @@ zone_page_alloc(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_alloc");
@@ -1128,6 +1206,9 @@ zone_page_free_element(
 	struct zone_page_table_entry	*zp;
 	natural_t i, j;
 
+#if	ZONE_ALIAS_ADDR
+	addr = zone_virtual_addr(addr);
+#endif
 #if MACH_ASSERT
 	if (!from_zone_map(addr, size))
 		panic("zone_page_free_element");
@@ -1156,6 +1237,36 @@ struct zone_free_element {
 	struct zone_free_element * next;
 };
 
+/*
+ * Add a linked list of pages starting at base back into the zone
+ * free list. Tail points to the last element on the list.
+ */
+
+#define ADD_LIST_TO_ZONE(zone, base, tail)				\
+MACRO_BEGIN								\
+	(tail)->next = (void *)((zone)->free_elements);			\
+	if (check_freed_element) {					\
+		if ((zone)->elem_size >= (2 * sizeof(vm_offset_t)))	\
+			((vm_offset_t *)(tail))[((zone)->elem_size/sizeof(vm_offset_t))-1] = \
+                                        (zone)->free_elements;		\
+	}								\
+	(zone)->free_elements = (unsigned long)(base);			\
+MACRO_END
+
+/*
+ * Add an element to the chain pointed to by prev.
+ */
+
+#define ADD_ELEMENT(zone, prev, elem)						\
+MACRO_BEGIN								\
+	(prev)->next = (elem);						\
+	if (check_freed_element) {					\
+		if ((zone)->elem_size >= (2 * sizeof(vm_offset_t)))     \
+			((vm_offset_t *)(prev))[((zone)->elem_size/sizeof(vm_offset_t))-1] = \
+					(vm_offset_t)(elem); 		\
+        }								\
+MACRO_END
+
 struct {
 	uint32_t	pgs_freed;
 
@@ -1213,9 +1324,11 @@ zone_gc(void)
 		 * (i.e we need a whole allocation block's worth of free
 		 * elements before we can garbage collect) and
 		 * the zone has more than 10 percent of it's elements free
+		 * or the element size is a multiple of the PAGE_SIZE 
 		 */
-		if (((z->cur_size - z->count * elt_size) <= (2 * z->alloc_size)) ||
-		    ((z->cur_size - z->count * elt_size) <= (z->cur_size / 10))) {
+		if ((elt_size & PAGE_MASK) && 
+		     (((z->cur_size - z->count * elt_size) <= (2 * z->alloc_size)) ||
+		      ((z->cur_size - z->count * elt_size) <= (z->cur_size / 10)))) {
 			unlock_zone(z);		
 			continue;
 		}
@@ -1254,11 +1367,14 @@ zone_gc(void)
 			else {
 				if (keep == NULL)
 					keep = tail = elt;
-				else
-					tail = tail->next = elt;
+				else {
+					ADD_ELEMENT(z, tail, elt);
+					tail = elt;
+				}
 
-				elt = prev->next = elt->next;
-				tail->next = NULL;
+				ADD_ELEMENT(z, prev, elt->next);
+				elt = elt->next;
+				ADD_ELEMENT(z, tail, NULL);
 			}
 
 			/*
@@ -1270,8 +1386,7 @@ zone_gc(void)
 					lock_zone(z);
 
 					if (keep != NULL) {
-						tail->next = (void *)z->free_elements;
-						z->free_elements = (vm_offset_t) keep;
+						ADD_LIST_TO_ZONE(z, keep, tail);
 						tail = keep = NULL;
 					} else {
 						m =0;
@@ -1282,9 +1397,8 @@ zone_gc(void)
 							elt = elt->next;
 						}
 						if (m !=0 ) {
-							prev->next = (void *)z->free_elements;
-							z->free_elements = (vm_offset_t) base_elt;
-							base_prev->next = elt;
+							ADD_LIST_TO_ZONE(z, base_elt, prev);
+							ADD_ELEMENT(z, base_prev, elt);
 							prev = base_prev;
 						}
 					}
@@ -1307,8 +1421,7 @@ zone_gc(void)
 		if (keep != NULL) {
 			lock_zone(z);
 
-			tail->next = (void *)z->free_elements;
-			z->free_elements = (vm_offset_t) keep;
+			ADD_LIST_TO_ZONE(z, keep, tail);
 
 			unlock_zone(z);
 		}
@@ -1321,7 +1434,6 @@ zone_gc(void)
 		 */
 
 		size_freed = 0;
-		prev = (void *)&scan;
 		elt = scan;
 		n = 0; tail = keep = NULL;
 		while (elt != NULL) {
@@ -1330,7 +1442,7 @@ zone_gc(void)
 				zone_page_free_element(&zone_free_pages,
 										(vm_offset_t)elt, elt_size);
 
-				elt = prev->next = elt->next;
+				elt = elt->next;
 
 				++zgc_stats.elems_freed;
 			}
@@ -1339,11 +1451,13 @@ zone_gc(void)
 
 				if (keep == NULL)
 					keep = tail = elt;
-				else
-					tail = tail->next = elt;
+				else {
+					ADD_ELEMENT(z, tail, elt);
+					tail = elt;
+				}
 
-				elt = prev->next = elt->next;
-				tail->next = NULL;
+				elt = elt->next;
+				ADD_ELEMENT(z, tail, NULL);
 
 				++zgc_stats.elems_kept;
 			}
@@ -1360,8 +1474,7 @@ zone_gc(void)
 				size_freed = 0;
 
 				if (keep != NULL) {
-					tail->next = (void *)z->free_elements;
-					z->free_elements = (vm_offset_t) keep;
+					ADD_LIST_TO_ZONE(z, keep, tail);
 				}
 
 				if (z->waiting) {
@@ -1387,8 +1500,7 @@ zone_gc(void)
 			z->cur_size -= size_freed;
 
 			if (keep != NULL) {
-				tail->next = (void *)z->free_elements;
-				z->free_elements = (vm_offset_t) keep;
+				ADD_LIST_TO_ZONE(z, keep, tail);
 			}
 
 		}
@@ -1407,6 +1519,9 @@ zone_gc(void)
 
 	while ((zp = zone_free_pages) != NULL) {
 		zone_free_pages = zp->link;
+#if	ZONE_ALIAS_ADDR
+		z = zone_virtual_addr((vm_map_address_t)z);
+#endif
 		kmem_free(zone_map, zone_map_min_address + PAGE_SIZE *
 										(zp - zone_page_table), PAGE_SIZE);
 		++zgc_stats.pgs_freed;
@@ -1441,6 +1556,38 @@ consider_zone_gc(void)
 	}
 }
 
+struct fake_zone_info {
+	const char* name;
+	void (*func)(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *,
+		    int *, int *);
+};
+
+static struct fake_zone_info fake_zones[] = {
+	{
+		.name = "kernel_stacks",
+		.func = stack_fake_zone_info,
+	},
+#ifdef ppc
+	{
+		.name = "save_areas",
+		.func = save_fake_zone_info,
+	},
+	{
+		.name = "pmap_mappings",
+		.func = mapping_fake_zone_info,
+	},
+#endif /* ppc */
+#ifdef i386
+	{
+		.name = "page_tables",
+		.func = pt_fake_zone_info,
+	},
+#endif /* i386 */
+	{
+		.name = "kalloc.large",
+		.func = kalloc_fake_zone_info,
+	},
+};
 
 kern_return_t
 host_zone_info(
@@ -1461,21 +1608,20 @@ host_zone_info(
 	zone_name_t    *zn;
 	zone_info_t    *zi;
 	kern_return_t	kr;
+	size_t		num_fake_zones;
 
 	if (host == HOST_NULL)
 		return KERN_INVALID_HOST;
 
+	num_fake_zones = sizeof fake_zones / sizeof fake_zones[0];
+
 	/*
 	 *	We assume that zones aren't freed once allocated.
 	 *	We won't pick up any zones that are allocated later.
 	 */
 
 	simple_lock(&all_zones_lock);
-#ifdef ppc
-	max_zones = num_zones + 4;
-#else
-	max_zones = num_zones + 3; /* ATN: count the number below!! */
-#endif
+	max_zones = num_zones + num_fake_zones;
 	z = first_zone;
 	simple_unlock(&all_zones_lock);
 
@@ -1528,6 +1674,7 @@ host_zone_info(
 		/* assuming here the name data is static */
 		(void) strncpy(zn->zn_name, zcopy.zone_name,
 			       sizeof zn->zn_name);
+		zn->zn_name[sizeof zn->zn_name - 1] = '\0';
 
 		zi->zi_count = zcopy.count;
 		zi->zi_cur_size = zcopy.cur_size;
@@ -1540,36 +1687,21 @@ host_zone_info(
 		zn++;
 		zi++;
 	}
-	strcpy(zn->zn_name, "kernel_stacks");
-	stack_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
-			     &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
-	zn++;
-	zi++;
-#ifdef ppc
-	strcpy(zn->zn_name, "save_areas");
-	save_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
-			    &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
-	zn++;
-	zi++;
-
-	strcpy(zn->zn_name, "pmap_mappings");
-	mapping_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
-			       &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
-	zn++;
-	zi++;
-#endif
 
-#ifdef i386
-	strcpy(zn->zn_name, "page_tables");
-	pt_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
-			  &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
-	zn++;
-	zi++;
-#endif
-
-	strcpy(zn->zn_name, "kalloc.large");
-	kalloc_fake_zone_info(&zi->zi_count, &zi->zi_cur_size, &zi->zi_max_size, &zi->zi_elem_size,
-			       &zi->zi_alloc_size, &zi->zi_collectable, &zi->zi_exhaustible);
+	/*
+	 * loop through the fake zones and fill them using the specialized
+	 * functions
+	 */
+	for (i = 0; i < num_fake_zones; i++) {
+		strncpy(zn->zn_name, fake_zones[i].name, sizeof zn->zn_name);
+		zn->zn_name[sizeof zn->zn_name - 1] = '\0';
+		fake_zones[i].func(&zi->zi_count, &zi->zi_cur_size,
+				   &zi->zi_max_size, &zi->zi_elem_size,
+				   &zi->zi_alloc_size, &zi->zi_collectable,
+				   &zi->zi_exhaustible);
+		zn++;
+		zi++;
+	}
 
 	if (names != *namesp) {
 		vm_size_t used;
@@ -1651,11 +1783,8 @@ db_print_zone(
 
 /*ARGSUSED*/
 void
-db_show_one_zone(
-        db_expr_t       	addr,
-        int			have_addr,
-        __unused db_expr_t	count,
-        __unused char *	modif)
+db_show_one_zone(db_expr_t addr, boolean_t have_addr,
+		 __unused db_expr_t count, __unused char *modif)
 {
 	struct zone *z = (zone_t)((char *)0 + addr);
 
@@ -1670,11 +1799,8 @@ db_show_one_zone(
 
 /*ARGSUSED*/
 void
-db_show_all_zones(
-        __unused db_expr_t	addr,
-        int			have_addr,
-        db_expr_t		count,
-        __unused char *	modif)
+db_show_all_zones(__unused db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		  __unused char *modif)
 {
 	zone_t		z;
 	unsigned total = 0;
@@ -1723,13 +1849,13 @@ db_zone_check_active(
 	while (count < zone->count) {
 		count++;
 		if (tmp_elem == 0) {
-			printf("unexpected zero element, zone=0x%x, count=%d\n",
+			printf("unexpected zero element, zone=%p, count=%d\n",
 				zone, count);
 			assert(FALSE);
 			break;
 		}
 		if (queue_end(tmp_elem, &zone->active_zones)) {
-			printf("unexpected queue_end, zone=0x%x, count=%d\n",
+			printf("unexpected queue_end, zone=%p, count=%d\n",
 				zone, count);
 			assert(FALSE);
 			break;
@@ -1737,7 +1863,7 @@ db_zone_check_active(
 		tmp_elem = queue_next(tmp_elem);
 	}
 	if (!queue_end(tmp_elem, &zone->active_zones)) {
-		printf("not at queue_end, zone=0x%x, tmp_elem=0x%x\n",
+		printf("not at queue_end, zone=%p, tmp_elem=%p\n",
 			zone, tmp_elem);
 		assert(FALSE);
 	}
@@ -1751,7 +1877,7 @@ db_zone_print_active(
 	queue_t	tmp_elem;
 
 	if (!zone_debug_enabled(zone)) {
-		printf("zone 0x%x debug not enabled\n", zone);
+		printf("zone %p debug not enabled\n", zone);
 		return;
 	}
 	if (!zone_check) {
@@ -1759,11 +1885,11 @@ db_zone_print_active(
 		return;
 	}
 
-	printf("zone 0x%x, active elements %d\n", zone, zone->count);
+	printf("zone %p, active elements %d\n", zone, zone->count);
 	printf("active list:\n");
 	tmp_elem = queue_first(&zone->active_zones);
 	while (count < zone->count) {
-		printf("  0x%x", tmp_elem);
+		printf("  %p", tmp_elem);
 		count++;
 		if ((count % 6) == 0)
 			printf("\n");
@@ -1778,7 +1904,7 @@ db_zone_print_active(
 		tmp_elem = queue_next(tmp_elem);
 	}
 	if (!queue_end(tmp_elem, &zone->active_zones))
-		printf("\nnot at queue_end, tmp_elem=0x%x\n", tmp_elem);
+		printf("\nnot at queue_end, tmp_elem=%p\n", tmp_elem);
 	else
 		printf("\n");
 }
@@ -1793,7 +1919,7 @@ db_zone_print_free(
 	vm_offset_t elem;
 
 	freecount = zone_free_count(zone);
-	printf("zone 0x%x, free elements %d\n", zone, freecount);
+	printf("zone %p, free elements %d\n", zone, freecount);
 	printf("free list:\n");
 	elem = zone->free_elements;
 	while (count < freecount) {
@@ -1829,11 +1955,11 @@ next_element(
 	char		*elt = (char *)prev;
 
 	if (!zone_debug_enabled(z))
-		return(0);
+		return(NULL);
 	elt -= ZONE_DEBUG_OFFSET;
 	elt = (char *) queue_next((queue_t) elt);
 	if ((queue_t) elt == &z->active_zones)
-		return(0);
+		return(NULL);
 	elt += ZONE_DEBUG_OFFSET;
 	return(elt);
 }
@@ -1845,9 +1971,9 @@ first_element(
 	char 		*elt;
 
 	if (!zone_debug_enabled(z))
-		return(0);
+		return(NULL);
 	if (queue_empty(&z->active_zones))
-		return(0);
+		return(NULL);
 	elt = (char *)queue_first(&z->active_zones);
 	elt += ZONE_DEBUG_OFFSET;
 	return(elt);
@@ -1903,6 +2029,6 @@ zone_debug_disable(
 	if (!zone_debug_enabled(z) || zone_in_use(z))
 		return;
 	z->elem_size -= ZONE_DEBUG_OFFSET;
-	z->active_zones.next = z->active_zones.prev = 0;	
+	z->active_zones.next = z->active_zones.prev = NULL;
 }
 #endif	/* ZONE_DEBUG */
diff --git a/osfmk/kern/zalloc.h b/osfmk/kern/zalloc.h
index f03900066..e8a1ee29d 100644
--- a/osfmk/kern/zalloc.h
+++ b/osfmk/kern/zalloc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,6 +76,7 @@
 #include <zone_debug.h>
 #include <mach_kdb.h>
 #include <kern/lock.h>
+#include <kern/locks.h>
 #include <kern/queue.h>
 #include <kern/call_entry.h>
 
@@ -84,7 +91,11 @@
 struct zone {
 	int		count;		/* Number of elements used now */
 	vm_offset_t	free_elements;
-	decl_mutex_data(,lock)		/* generic lock */
+	decl_lck_mtx_data(,lock)	/* zone lock */
+	lck_mtx_ext_t   lock_ext;	/* placeholder for indirect mutex */
+	lck_attr_t      lock_attr;	/* zone lock attribute */
+	lck_grp_t       lock_grp;	/* zone lock group */
+	lck_grp_attr_t  lock_grp_attr;	/* zone lock group attribute */
 	vm_size_t	cur_size;	/* current memory utilization */
 	vm_size_t	max_size;	/* how large can this zone grow */
 	vm_size_t	elem_size;	/* size of an element */
@@ -113,11 +124,11 @@ extern void		consider_zone_gc(void);
 extern void		zone_steal_memory(void);
 
 /* Bootstrap zone module (create zone zone) */
-extern void		zone_bootstrap(void);
+extern void		zone_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 /* Init zone module */
 extern void		zone_init(
-					vm_size_t	map_size);
+					vm_size_t	map_size) __attribute__((section("__TEXT, initcode")));
 
 /* Stack use statistics */
 extern void		stack_fake_zone_info(
diff --git a/osfmk/libsa/ctype.h b/osfmk/libsa/ctype.h
index 152f2ab4c..73d90e9ab 100644
--- a/osfmk/libsa/ctype.h
+++ b/osfmk/libsa/ctype.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/errno.h b/osfmk/libsa/errno.h
index 36b9df2c6..e40d14efb 100644
--- a/osfmk/libsa/errno.h
+++ b/osfmk/libsa/errno.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/float.h b/osfmk/libsa/float.h
index a1da5a179..1dab99631 100644
--- a/osfmk/libsa/float.h
+++ b/osfmk/libsa/float.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/i386/float.h b/osfmk/libsa/i386/float.h
index e3d1c1ef1..084245798 100644
--- a/osfmk/libsa/i386/float.h
+++ b/osfmk/libsa/i386/float.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/i386/math.h b/osfmk/libsa/i386/math.h
index 21728795a..7e061390c 100644
--- a/osfmk/libsa/i386/math.h
+++ b/osfmk/libsa/i386/math.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/i386/stdarg.h b/osfmk/libsa/i386/stdarg.h
deleted file mode 100644
index 7a9022fa6..000000000
--- a/osfmk/libsa/i386/stdarg.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- * 
- */
-/*
- * HISTORY
- * 
- * Revision 1.2  1998/09/30 21:21:00  wsanchez
- * Merged in IntelMerge1 (mburg: Intel support)
- *
- * Revision 1.1.2.1  1998/09/30 18:19:49  mburg
- * Changes for Intel port
- *
- * Revision 1.1.1.1  1998/03/07 02:25:36  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.2.1  1996/09/17  16:56:26  bruel
- * 	created from standalone mach servers
- * 	[1996/09/17  16:18:07  bruel]
- *
- * $EndLog$
- */
-
-#ifndef _MACHINE_STDARG_H
-#define _MACHINE_STDARG_H
-
-#include <machine/va_list.h>
-
-/* Amount of space required in an argument list for an arg of type TYPE.
-   TYPE may alternatively be an expression whose type is used.  */
-
-#define __va_rounded_size(TYPE)  \
-  (((sizeof (TYPE) + sizeof (int) - 1) / sizeof (int)) * sizeof (int))
-
-#define va_start(AP, LASTARG) 						\
- (AP = ((char *) &(LASTARG) + __va_rounded_size (LASTARG)))
-
-void va_end (va_list);		/* Defined in gnulib */
-#define va_end(AP)
-
-#define va_arg(AP, mode)					\
-		  (AP += __va_rounded_size (mode),		\
-		   *((mode *) (AP - __va_rounded_size (mode))))
-
-#endif /* _MACHINE_STDARG_H */
diff --git a/osfmk/libsa/i386/types.h b/osfmk/libsa/i386/types.h
index 650acf6a3..b7cbda01b 100644
--- a/osfmk/libsa/i386/types.h
+++ b/osfmk/libsa/i386/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/i386/va_list.h b/osfmk/libsa/i386/va_list.h
deleted file mode 100644
index aa14743d2..000000000
--- a/osfmk/libsa/i386/va_list.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- * 
- */
-/*
- * HISTORY
- * 
- * Revision 1.2  1998/09/30 21:21:00  wsanchez
- * Merged in IntelMerge1 (mburg: Intel support)
- *
- * Revision 1.1.2.1  1998/09/30 18:19:50  mburg
- * Changes for Intel port
- *
- * Revision 1.1.1.1  1998/03/07 02:25:36  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.2.1  1996/09/17  16:56:30  bruel
- * 	created from standalone mach servers
- * 	[1996/09/17  16:18:09  bruel]
- *
- * $EndLog$
- */
-
-#ifndef _MACHINE_VALIST_H
-#define _MACHINE_VALIST_H
-
-/*
- * Four possible situations:
- * 	- We are being included by {var,std}args.h (or anyone) before stdio.h.
- * 	  define real type.
- *
- * 	- We are being included by stdio.h before {var,std}args.h.
- * 	  define hidden type for prototypes in stdio, don't pollute namespace.
- * 
- * 	- We are being included by {var,std}args.h after stdio.h.
- * 	  define real type to match hidden type.  no longer use hidden type.
- * 
- * 	- We are being included again after defining the real va_list.
- * 	  do nothing.
- * 
- */
-
-#if	!defined(_HIDDEN_VA_LIST) && !defined(_VA_LIST)
-#define _VA_LIST
-typedef	char *va_list;
-
-#elif	defined(_HIDDEN_VA_LIST) && !defined(_VA_LIST)
-#define _VA_LIST
-typedef char *__va_list;
-
-#elif	defined(_HIDDEN_VA_LIST) && defined(_VA_LIST)
-#undef _HIDDEN_VA_LIST
-typedef __va_list va_list;
-
-#endif
-
-#endif /* _MACHINE_VALIST_H */
-
diff --git a/osfmk/libsa/ieeefloat.h b/osfmk/libsa/ieeefloat.h
index c42f141c6..2fcda29a7 100644
--- a/osfmk/libsa/ieeefloat.h
+++ b/osfmk/libsa/ieeefloat.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/machine/stdarg.h b/osfmk/libsa/machine/stdarg.h
deleted file mode 100644
index 424b40983..000000000
--- a/osfmk/libsa/machine/stdarg.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _MACH_MACHINE_STDARG_H
-#define _MACH_MACHINE_STDARG_H
-
-
-#if defined (__ppc__)
-#include "ppc/stdarg.h"
-#elif defined (__i386__)
-#include "i386/stdarg.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _MACH_MACHINE_STDARG_H */
diff --git a/osfmk/libsa/machine/stdarg_apple.h b/osfmk/libsa/machine/stdarg_apple.h
index fde11df56..c48e1cbdb 100644
--- a/osfmk/libsa/machine/stdarg_apple.h
+++ b/osfmk/libsa/machine/stdarg_apple.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACH_MACHINE_STDARG_APPLE_H
 #define _MACH_MACHINE_STDARG_APPLE_H
diff --git a/osfmk/libsa/machine/types.h b/osfmk/libsa/machine/types.h
index 710311b83..4a6b16e71 100644
--- a/osfmk/libsa/machine/types.h
+++ b/osfmk/libsa/machine/types.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACH_MACHINE_TYPES_H
 #define _MACH_MACHINE_TYPES_H
 
-
 #if defined (__ppc__)
 #include "ppc/types.h"
 #elif defined (__i386__)
 #include "i386/types.h"
+#elif defined (__arm__)
+#include "arm/types.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACH_MACHINE_TYPES_H */
diff --git a/osfmk/libsa/machine/va_list.h b/osfmk/libsa/machine/va_list.h
deleted file mode 100644
index 95929edb7..000000000
--- a/osfmk/libsa/machine/va_list.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#ifndef _MACH_MACHINE_VA_LIST_H
-#define _MACH_MACHINE_VA_LIST_H
-
-
-#if defined (__ppc__)
-#include "ppc/va_list.h"
-#elif defined (__i386__)
-#include "i386/va_list.h"
-#else
-#error architecture not supported
-#endif
-
-
-#endif /* _MACH_MACHINE_VA_LIST_H */
diff --git a/osfmk/libsa/math.h b/osfmk/libsa/math.h
index 49cffc7d0..024862d76 100644
--- a/osfmk/libsa/math.h
+++ b/osfmk/libsa/math.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/ppc/float.h b/osfmk/libsa/ppc/float.h
index f2608b5ee..ba074cdbb 100644
--- a/osfmk/libsa/ppc/float.h
+++ b/osfmk/libsa/ppc/float.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/ppc/math.h b/osfmk/libsa/ppc/math.h
index b094a20cb..a942324b0 100644
--- a/osfmk/libsa/ppc/math.h
+++ b/osfmk/libsa/ppc/math.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/ppc/stdarg.h b/osfmk/libsa/ppc/stdarg.h
deleted file mode 100644
index dcb67693b..000000000
--- a/osfmk/libsa/ppc/stdarg.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#if defined(APPLE) || defined(NeXT)
-#include <machine/stdarg_apple.h>
-#else
-#endif
diff --git a/osfmk/libsa/ppc/stdarg_apple.h b/osfmk/libsa/ppc/stdarg_apple.h
index c1bcea725..770e7d2f1 100644
--- a/osfmk/libsa/ppc/stdarg_apple.h
+++ b/osfmk/libsa/ppc/stdarg_apple.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* stdarg.h for GNU.
    Note that the type used in va_arg is supposed to match the
diff --git a/osfmk/libsa/ppc/types.h b/osfmk/libsa/ppc/types.h
index 939b82d4f..859f94b92 100644
--- a/osfmk/libsa/ppc/types.h
+++ b/osfmk/libsa/ppc/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/ppc/va_list.h b/osfmk/libsa/ppc/va_list.h
deleted file mode 100644
index 1c42bff89..000000000
--- a/osfmk/libsa/ppc/va_list.h
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#if !defined(APPLE) && !defined(NeXT)
-/*
- * @OSF_COPYRIGHT@
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:51  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:36  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.2.1  1996/12/09  16:59:07  stephen
- * 	nmklinux_1.0b3_shared into pmk1.1
- * 	[1996/12/09  11:18:59  stephen]
- *
- * Revision 1.1.4.1  1996/04/11  14:37:05  emcmanus
- * 	Copied from mainline.ppc.
- * 	[1996/04/11  14:36:22  emcmanus]
- * 
- * Revision 1.1.2.1  1995/12/28  16:37:24  barbou
- * 	Self-Contained Mach Distribution:
- * 	created.
- * 	[95/12/28            barbou]
- * 
- * $EndLog$
- */
-
-/*
- * Four possible situations:
- * 	- We are being included by {var,std}args.h (or anyone) before stdio.h.
- * 	  define real type.
- *
- * 	- We are being included by stdio.h before {var,std}args.h.
- * 	  define hidden type for prototypes in stdio, don't pollute namespace.
- * 
- * 	- We are being included by {var,std}args.h after stdio.h.
- * 	  define real type to match hidden type.  no longer use hidden type.
- * 
- * 	- We are being included again after defining the real va_list.
- * 	  do nothing.
- * 
- */
-
-#if	!defined(_HIDDEN_VA_LIST) && !defined(_VA_LIST)
-
-/* Define __gnuc_va_list. */
-
-#ifndef __GNUC_VA_LIST
-/*
- * If this is for internal libc use, don't define
- * anything but __gnuc_va_list.
- */
-#define __GNUC_VA_LIST
-typedef struct {
-  char gpr;			/* index into the array of 8 GPRs stored in the
-				   register save area gpr=0 corresponds to r3,
-				   gpr=1 to r4, etc. */
-  char fpr;			/* index into the array of 8 FPRs stored in the
-				   register save area fpr=0 corresponds to f1,
-				   fpr=1 to f2, etc. */
-  char *overflow_arg_area;	/* location on stack that holds the next
-				   overflow argument */
-  char *reg_save_area;		/* where r3:r10 and f1:f8, if saved are stored */
-} __gnuc_va_list[1];
-
-#endif /* not __GNUC_VA_LIST */
-
-#define _VA_LIST
-typedef struct {
-  char gpr;			/* index into the array of 8 GPRs stored in the
-				   register save area gpr=0 corresponds to r3,
-				   gpr=1 to r4, etc. */
-  char fpr;			/* index into the array of 8 FPRs stored in the
-				   register save area fpr=0 corresponds to f1,
-				   fpr=1 to f2, etc. */
-  char *overflow_arg_area;	/* location on stack that holds the next
-				   overflow argument */
-  char *reg_save_area;		/* where r3:r10 and f1:f8, if saved are stored */
-} va_list[1];
-
-#elif	defined(_HIDDEN_VA_LIST) && !defined(_VA_LIST)
-
-#define _VA_LIST
-typedef struct {
-  char gpr;			/* index into the array of 8 GPRs stored in the
-				   register save area gpr=0 corresponds to r3,
-				   gpr=1 to r4, etc. */
-  char fpr;			/* index into the array of 8 FPRs stored in the
-				   register save area fpr=0 corresponds to f1,
-				   fpr=1 to f2, etc. */
-  char *overflow_arg_area;	/* location on stack that holds the next
-				   overflow argument */
-  char *reg_save_area;		/* where r3:r10 and f1:f8, if saved are stored */
-} __va_list[1];
-
-#elif	defined(_HIDDEN_VA_LIST) && defined(_VA_LIST)
-
-#undef _HIDDEN_VA_LIST
-typedef __va_list va_list;
-
-#endif
-
-#endif
diff --git a/osfmk/libsa/stdarg.h b/osfmk/libsa/stdarg.h
deleted file mode 100644
index 962196ed7..000000000
--- a/osfmk/libsa/stdarg.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- * 
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:51  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:35  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.2.1  1996/09/17  16:56:23  bruel
- * 	created from standalone mach servers.
- * 	[96/09/17            bruel]
- *
- * $EndLog$
- */
-
-#include <machine/va_list.h>
-#include <machine/stdarg.h>
diff --git a/osfmk/libsa/stdio.h b/osfmk/libsa/stdio.h
index 37ecb60a5..4deea51e9 100644
--- a/osfmk/libsa/stdio.h
+++ b/osfmk/libsa/stdio.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -58,10 +64,13 @@
 #define NULL ((void *) 0)
 #endif
 
-extern int	sprintf(char *, const char *, ...); 
+/* sprintf() is being deprecated. Please use snprintf() instead. */ 
+extern int	sprintf(char *, const char *, ...) __deprecated; 
 extern int	printf(const char *, ...); 
 extern int	vprintf(const char *, va_list );
-extern int	vsprintf(char *, const char *, va_list );
+
+/* vsprintf() is being deprecated. Please use vsnprintf() instead. */
+extern int	vsprintf(char *, const char *, va_list ) __deprecated;
 
 extern int 	getchar(void);
 
diff --git a/osfmk/libsa/stdlib.h b/osfmk/libsa/stdlib.h
index 655f8779d..35ccc599d 100644
--- a/osfmk/libsa/stdlib.h
+++ b/osfmk/libsa/stdlib.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -61,6 +67,8 @@
 #endif
 
 extern int     atoi(const char *);
+extern int     atoi_term(char *, char **);
+extern char    *itoa(int, char *);
 
 extern void	free(void *);
 extern void	*malloc(size_t);
diff --git a/osfmk/libsa/string.h b/osfmk/libsa/string.h
index 49e7b463e..c94f56dd7 100644
--- a/osfmk/libsa/string.h
+++ b/osfmk/libsa/string.h
@@ -1,25 +1,38 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 /*
+   * HISTORY
  * @OSF_COPYRIGHT@
  */
 #ifndef	_STRING_H_
@@ -36,7 +49,11 @@ extern "C" {
 #endif
 
 #ifndef	NULL
-#define NULL	0
+#if defined (__cplusplus)
+#define NULL 0
+#else
+#define NULL ((void *)0)
+#endif
 #endif
 
 extern void	*memcpy(void *, const void *, size_t);
@@ -45,16 +62,28 @@ extern void	*memmove(void *, const void *, size_t);
 extern void	*memset(void *, int, size_t);
 
 extern size_t	strlen(const char *);
-extern size_t   strnlen(const char *, size_t);
-extern char	*strcpy(char *, const char *);
+extern size_t	strnlen(const char *, size_t);
+
+/* strcpy() is being deprecated. Please use strlcpy() instead. */
+extern char	*strcpy(char *, const char *) __deprecated;
 extern char	*strncpy(char *, const char *, size_t);
-extern char	*strcat(char *, const char *);
+
+extern size_t	strlcat(char *, const char *, size_t);
+extern size_t	strlcpy(char *, const char *, size_t);
+
+/* strcat() is being deprecated. Please use strlcat() instead. */
+extern char	*strcat(char *, const char *) __deprecated;
 extern char	*strncat(char *, const char *, size_t);
+
+/* strcmp() is being deprecated. Please use strncmp() instead. */
 extern int	strcmp(const char *, const char *);
 extern int	strncmp(const char *,const char *, size_t);
+
 extern int	strcasecmp(const char *s1, const char *s2);
 extern int	strncasecmp(const char *s1, const char *s2, size_t n);
 extern char	*strchr(const char *s, int c);
+extern char	*STRDUP(const char *, int);
+extern int	strprefix(const char *s1, const char *s2);
 
 extern int	bcmp(const void *, const void *, size_t);
 extern void	bcopy(const void *, void *, size_t);
diff --git a/osfmk/libsa/sys/timers.h b/osfmk/libsa/sys/timers.h
index 645aba560..e03d7737c 100644
--- a/osfmk/libsa/sys/timers.h
+++ b/osfmk/libsa/sys/timers.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/types.h b/osfmk/libsa/types.h
index 0a90a2d6d..e9f8ad7bf 100644
--- a/osfmk/libsa/types.h
+++ b/osfmk/libsa/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/libsa/va_list.h b/osfmk/libsa/va_list.h
deleted file mode 100644
index 4034a0ce7..000000000
--- a/osfmk/libsa/va_list.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- * @OSF_COPYRIGHT@
- * 
- */
-/*
- * HISTORY
- * 
- * Revision 1.1.1.1  1998/09/22 21:05:51  wsanchez
- * Import of Mac OS X kernel (~semeria)
- *
- * Revision 1.1.1.1  1998/03/07 02:25:35  wsanchez
- * Import of OSF Mach kernel (~mburg)
- *
- * Revision 1.1.2.1  1996/09/17  16:56:17  bruel
- * 	created from standalone mach servers.
- * 	[96/09/17            bruel]
- *
- * $EndLog$
- */
-
-#include <machine/va_list.h>
-
diff --git a/osfmk/lockd/Makefile b/osfmk/lockd/Makefile
new file mode 100644
index 000000000..45820f497
--- /dev/null
+++ b/osfmk/lockd/Makefile
@@ -0,0 +1,65 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+INSTINC_SUBDIRS =
+
+INSTINC_SUBDIRS_PPC = 
+
+INSTINC_SUBDIRS_I386 =
+
+EXPINC_SUBDIRS =
+
+EXPINC_SUBDIRS_PPC =
+
+EXPINC_SUBDIRS_I386 =
+
+MIG_DEFS = lockd_mach.defs
+
+DATAFILES = lockd_mach_types.h ${MIG_DEFS}
+
+INSTALL_MI_LIST =
+
+INSTALL_MI_LCL_LIST	= ${DATAFILES} 
+
+INSTALL_MI_GEN_LIST = 
+
+INSTALL_MI_DIR = lockd
+
+EXPORT_MI_LIST	= \
+	${DATAFILES}
+
+EXPORT_MI_GEN_LIST = lockd_mach.h
+
+EXPORT_MI_DIR = lockd
+
+#
+# Build path
+#       
+INCFLAGS_MAKEFILE= -I..
+
+MIGKUFLAGS	= -DMACH_KERNEL_PRIVATE -DKERNEL_USER=1 -maxonstack 1024
+
+MIG_KUHDRS = lockd_mach.h
+
+MIG_KUSRC = lockd_mach.c lockd_mach.h
+
+COMP_FILES = ${MIG_KUSRC}
+
+${COMP_FILES} : lockd_mach.defs
+
+${MIG_KUSRC} : lockd_mach.defs
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKUFLAGS}	\
+		-user    $*.c			\
+		-header  $*.h			\
+		-server  /dev/null		\
+		-sheader /dev/null		\
+		$<
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/osfmk/lockd/lockd_mach.defs b/osfmk/lockd/lockd_mach.defs
new file mode 100644
index 000000000..b5168bfd7
--- /dev/null
+++ b/osfmk/lockd/lockd_mach.defs
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+
+#include <mach/std_types.defs>
+#include <mach/mach_types.defs>
+
+#ifdef KERNEL
+import <lockd/lockd_mach_types.h>;
+#else
+import "lockd_mach_types.h";
+#endif
+
+type sock_storage = array [32] of uint32_t;
+type xcred = array [19] of uint32_t;
+type nfs_handle = array [64] of uint8_t;
+
+subsystem
+#ifdef KERNEL_USER
+KernelUser
+#endif
+lockd_mach 1023;
+
+serverprefix svc_;
+
+simpleroutine lockd_request(
+	server		: mach_port_t;
+	vers		: uint32_t;
+	flags		: uint32_t;
+	xid		: uint64_t;
+	flk_start	: int64_t;
+	flk_len		: int64_t;
+	flk_pid		: int32_t;
+	flk_type		: int32_t;
+	flk_whence	: int32_t;
+	sock_address	: sock_storage;
+	cred		: xcred;
+	fh_len		: uint32_t;
+	fh		: nfs_handle
+);
+
+/* Ping lockd, have launchd start it if its not running */
+routine lockd_ping(
+	server		: mach_port_t
+);
+
+/* Shutdown lockd */
+simpleroutine lockd_shutdown(
+	server		: mach_port_t
+);
+
diff --git a/osfmk/lockd/lockd_mach_types.h b/osfmk/lockd/lockd_mach_types.h
new file mode 100644
index 000000000..1e59569e9
--- /dev/null
+++ b/osfmk/lockd/lockd_mach_types.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+
+#ifndef _LOCKD_MACH_TYPES_H_
+#define _LOCKD_MACH_TYPES_H_
+
+/*
+ * XXX NFSV3_MAX_FH_SIZE is defined in sys/mount.h, but we can't include 
+ * that here. Osfmk includes libsa/types.h which causes massive conflicts
+ * with sys/types.h that get indirectly included with sys/mount.h. In user
+ * land below will work on a build that does not yet have the new macro
+ * definition.
+ */
+ 
+#ifndef NFSV3_MAX_FH_SIZE
+#define NFSV3_MAX_FH_SIZE 64
+#endif
+
+typedef uint32_t sock_storage[32];
+typedef uint32_t xcred[19];
+typedef uint8_t nfs_handle[NFSV3_MAX_FH_SIZE];
+
+#endif
diff --git a/osfmk/mach-o/loader.h b/osfmk/mach-o/loader.h
index 0422c582e..dfbf7ee8b 100644
--- a/osfmk/mach-o/loader.h
+++ b/osfmk/mach-o/loader.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHO_LOADER_H_
 #define _MACHO_LOADER_H_
@@ -152,6 +158,8 @@ struct load_command {
 #define	LC_PREBOUND_DYLIB 0x10	/* modules prebound for a dynamicly */
 				/*  linked shared library */
 
+#define LC_UUID		0x1b	/* the uuid */
+
 /*
  * A variable length string in a load command is represented by an lc_str
  * union.  The strings are stored just after the load command structure and
@@ -683,6 +691,16 @@ struct dylib_reference {
     		  flags:8;	/* flags to indicate the type of reference */
 };
 
+/*
+ * The uuid load command contains a single 128-bit unique random number that
+ * identifies an object produced by the static link editor.
+ */
+struct uuid_command {
+    unsigned long	cmd;		/* LC_UUID */
+    unsigned long	cmdsize;	/* sizeof(struct uuid_command) */
+    unsigned char	uuid[16];	/* the 128-bit uuid */
+};
+
 /*
  * The symseg_command contains the offset and size of the GNU style
  * symbol table information as described in the header file <symseg.h>.
diff --git a/osfmk/mach-o/mach_header.c b/osfmk/mach-o/mach_header.c
index dc4261337..fdf75d1aa 100644
--- a/osfmk/mach-o/mach_header.c
+++ b/osfmk/mach-o/mach_header.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.c
@@ -384,7 +390,8 @@ getsegbyname(const char *seg_name)
 	 * For the kernel's header add on the faked segment for the
 	 * USER boot code identified by a FVMFILE_COMMAND in the mach header.
 	 */
-	if (!this && strcmp(seg_name, fvm_seg->segname) == 0)
+	if (!this && strncmp(seg_name, fvm_seg->segname,
+				sizeof(fvm_seg->segname)) == 0)
 		this = fvm_seg;
 
 	return this;
@@ -494,12 +501,12 @@ getfakefvmseg(void)
 	sgp->vmaddr = fvp->header_addr;
 	sgp->vmsize = getsizeofmacho((struct mach_header *)(sgp->vmaddr));
 
-	strcpy(sp->sectname, fvp->name.ptr);
+	strlcpy(sp->sectname, fvp->name.ptr, sizeof(sp->sectname) /* 16 */);
 	sp->addr = sgp->vmaddr;
 	sp->size = sgp->vmsize;
 
 #if	DEBUG
-	printf("fake fvm seg __USER/\"%s\" at 0x%x, size 0x%x\n",
+	printf("fake fvm seg __USER/\"%s\" at 0x%lx, size 0x%lx\n",
 		sp->sectname, sp->addr, sp->size);
 #endif	/*DEBUG*/
 	return sgp;
@@ -564,7 +571,7 @@ boolean_t getsymtab(struct mach_header *header,
 	seglink_cmd = NULL;
 	
 	if(header->magic != MH_MAGIC) {						/* Check if this is a valid header format */
-		printf("Attempt to use invalid header (magic = %08X) to find symbol table\n", 
+		printf("Attempt to use invalid header (magic = %08lX) to find symbol table\n", 
 			header->magic);								/* Tell them what's wrong */
 		return (FALSE);									/* Bye y'all... */
 	}
diff --git a/osfmk/mach-o/mach_header.h b/osfmk/mach-o/mach_header.h
index 25f83b459..eda433213 100644
--- a/osfmk/mach-o/mach_header.h
+++ b/osfmk/mach-o/mach_header.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File: kern/mach_header.h
diff --git a/osfmk/mach/Makefile b/osfmk/mach/Makefile
index 89a374c11..e2f2f2de7 100644
--- a/osfmk/mach/Makefile
+++ b/osfmk/mach/Makefile
@@ -13,6 +13,9 @@ INSTINC_SUBDIRS = \
 INSTINC_SUBDIRS_PPC = \
 	ppc
 
+INSTINC_SUBDIRS_ARM = \
+	arm
+
 INSTINC_SUBDIRS_I386 = \
 	i386
 
@@ -25,6 +28,9 @@ EXPINC_SUBDIRS_PPC = \
 EXPINC_SUBDIRS_I386 = \
 	i386
 
+EXPINC_SUBDIRS_ARM = \
+	arm
+
 MIG_TYPES = \
 	clock_types.defs \
 	mach_types.defs \
@@ -41,16 +47,20 @@ MIG_DEFS =	\
 	host_security.defs \
 	ledger.defs \
 	lock_set.defs \
+	mach_exc.defs \
 	mach_host.defs \
 	mach_port.defs \
 	mach_vm.defs \
 	notify.defs \
 	processor.defs \
 	processor_set.defs \
+	security.defs \
 	task.defs \
+	task_access.defs \
 	thread_act.defs \
 	vm_map.defs
 
+
 MACH_PRIVATE_DEFS = \
 	mach_notify.defs \
 	memory_object.defs \
@@ -67,9 +77,11 @@ MIG_USHDRS = \
 	audit_triggers_server.h \
 	clock_reply_server.h \
 	exc_server.h \
+	mach_exc_server.h \
 	memory_object_server.h \
 	memory_object_default_server.h \
-	notify_server.h
+	notify_server.h \
+	task_access_server.h
 
 MIG_UUHDRS = \
 	clock.h \
@@ -85,7 +97,9 @@ MIG_UUHDRS = \
 	memory_object_name.h \
 	processor.h \
 	processor_set.h \
+	security.h \
 	task.h \
+	task_access.h \
 	thread_act.h \
 	upl.h \
 	vm_map.h
@@ -120,8 +134,10 @@ DATAFILES = \
 	port.h \
         processor_info.h \
 	rpc.h \
+	sdt.h \
         semaphore.h \
 	shared_memory_server.h \
+	shared_region.h \
 	std_types.h \
 	sync_policy.h \
         syscall_sw.h \
@@ -181,7 +197,8 @@ ${MIGINCLUDES} : ${MIG_TYPES}
 
 ${MIG_UUHDRS} : \
 	%.h : %.defs
-	$(MIG) $(MIGFLAGS)		\
+	@echo MIG $@
+	$(_v)$(MIG) $(MIGFLAGS)		\
 		-server /dev/null	\
 		-user /dev/null		\
 		-header $@ 		\
@@ -191,7 +208,8 @@ ${MIG_UUHDRS} : \
 
 ${MIG_USHDRS} : \
 	%_server.h : %.defs
-	$(MIG) $(MIGFLAGS)		\
+	@echo MIG $@
+	$(_v)$(MIG) $(MIGFLAGS)		\
 		-server /dev/null	\
 		-user /dev/null		\
 		-header /dev/null	\
@@ -217,10 +235,12 @@ MIG_KUHDRS = \
 	clock_reply.h \
 	exc.h \
 	host_notify_reply.h \
+	mach_exc.h \
 	mach_notify.h \
 	memory_object.h \
 	memory_object_control.h \
 	memory_object_default.h \
+	task_access.h \
 	upl.h \
 	vm_map.h
 
@@ -229,10 +249,12 @@ MIG_KUSRC = \
 	clock_reply_user.c \
 	exc_user.c \
 	host_notify_reply_user.c \
+	mach_exc_user.c \
 	mach_notify_user.c \
 	memory_object_user.c \
 	memory_object_control_user.c \
 	memory_object_default_user.c \
+	task_access_user.c \
 	upl_user.c \
 	vm_map_user.c
 
@@ -244,6 +266,7 @@ MIG_KSHDRS = \
 	host_security_server.h \
 	ledger_server.h \
 	lock_set_server.h \
+	mach_exc_server.h \
 	mach_host_server.h \
 	mach_notify_server.h \
 	mach_port_server.h \
@@ -254,6 +277,7 @@ MIG_KSHDRS = \
 	memory_object_name_server.h \
 	processor_server.h \
 	processor_set_server.h \
+	security_server.h \
 	semaphore_server.h \
 	task_server.h \
 	thread_act_server.h \
@@ -268,6 +292,7 @@ MIG_KSSRC = \
 	host_security_server.c \
 	ledger_server.c \
 	lock_set_server.c \
+	mach_exc_server.c \
 	mach_host_server.c \
 	mach_notify_server.c \
 	mach_port_server.c \
@@ -278,6 +303,7 @@ MIG_KSSRC = \
 	memory_object_name_server.c \
 	processor_server.c \
 	processor_set_server.c \
+	security_server.c \
 	semaphore_server.c \
 	task_server.c \
 	thread_act_server.c \
@@ -308,7 +334,8 @@ ${COMP_FILES} : ${MIG_TYPES}
 
 ${MIG_KUSRC} : \
 	%_user.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKUFLAGS}        \
 		-user    $*_user.c              \
 		-header  $*.h              \
 		-server  /dev/null		\
@@ -319,7 +346,8 @@ ${MIG_KUSRC} : \
         
 ${MIG_KSSRC}: \
 	%_server.c : %.defs
-	${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
+	@echo MIG $@
+	$(_v)${MIG} ${MIGFLAGS} ${MIGKSFLAGS}        \
 		-user    /dev/null              \
 		-header  /dev/null              \
 		-server  $*_server.c		\
diff --git a/osfmk/mach/Makefile.template b/osfmk/mach/Makefile.template
index 044b05036..8de510721 100644
--- a/osfmk/mach/Makefile.template
+++ b/osfmk/mach/Makefile.template
@@ -16,6 +16,8 @@ MACH_PORT_FILES =mach_port_server.h mach_port_server.c
 
 EXC_FILES = exc_user.h exc_user.c  exc_server.c
 
+MACH_EXC_FILES = mach_exc_user.h mach_exc_user.c  mach_exc_server.c
+
 MEMORY_OBJECT_FILES = memory_object_user.h memory_object_user.c 
 
 MEMORY_OBJECT_DEFAULT_FILES = memory_object_default.h memory_object_default_user.c
@@ -37,7 +39,7 @@ SYNC_FILES = sync_server.h sync_server.c
 MACH_USER_FILES = mach_user.h mach_user.c
  
 OTHERS	= ${MACH_FILES} ${MACH_PORT_FILES} \
-	${EXC_FILES} \
+	${EXC_FILES} ${MACH_EXC_FILES} \
 	${MEMORY_OBJECT_FILES} ${MEMORY_OBJECT_DEFAULT_FILES} \
 	${PROF_FILES} ${MACH_HOST_FILES} ${LEDGER_FILES} \
 	${CLOCK_FILES} ${CLOCK_REPLY_FILES} ${BOOTSTRAP_FILES} \
@@ -73,6 +75,15 @@ ${EXC_FILES}: mach/exc.defs
 		-server exc_server.c                    \
 		$<
 
+.ORDER: ${MACH_EXC_FILES}               
+${MACH_EXC_FILES}: mach/mach_exc.defs
+	${_MIG_} ${_MIGFLAGS_} ${MIGKUFLAGS}            \
+		-header mach_exc_user.h                      \
+		-user mach_exc_user.c                        \
+		-sheader mach_exc_server.h                   \
+		-server mach_exc_server.c                    \
+		$<
+
 .ORDER: ${MEMORY_OBJECT_FILES}
         
 ${MEMORY_OBJECT_FILES}: mach/memory_object.defs
diff --git a/osfmk/mach/alert.h b/osfmk/mach/alert.h
index 801aeb6bb..4365af96a 100644
--- a/osfmk/mach/alert.h
+++ b/osfmk/mach/alert.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/mach/audit_triggers.defs b/osfmk/mach/audit_triggers.defs
index 3988e586a..c7e0fcc64 100644
--- a/osfmk/mach/audit_triggers.defs
+++ b/osfmk/mach/audit_triggers.defs
@@ -1,15 +1,18 @@
 /*
- * @APPLE_LICENSE_HEADER_START@
- *
- * Copyright (c) 2004 Apple Computer, Inc.  All Rights Reserved.
- *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
@@ -17,8 +20,8 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -38,3 +41,4 @@ simpleroutine	audit_triggers(
 		audit_port	: mach_port_t; 
 	in	flags		: int);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/boolean.h b/osfmk/mach/boolean.h
index 83b25e31d..641c3962d 100644
--- a/osfmk/mach/boolean.h
+++ b/osfmk/mach/boolean.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/bootstrap.h b/osfmk/mach/bootstrap.h
index 045645938..67667c308 100644
--- a/osfmk/mach/bootstrap.h
+++ b/osfmk/mach/bootstrap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Mach bootstrap interfaces (obsolete: header included only for compatibility)
diff --git a/osfmk/mach/clock.defs b/osfmk/mach/clock.defs
index 5c94e007e..440042c86 100644
--- a/osfmk/mach/clock.defs
+++ b/osfmk/mach/clock.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -71,3 +77,5 @@ routine clock_alarm(
 		alarm_time	: mach_timespec_t;
                 alarm_port      : clock_reply_t =
 			MACH_MSG_TYPE_MAKE_SEND_ONCE|polymorphic);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/clock_priv.defs b/osfmk/mach/clock_priv.defs
index e9be2636f..b3854a4ec 100644
--- a/osfmk/mach/clock_priv.defs
+++ b/osfmk/mach/clock_priv.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -60,3 +66,4 @@ routine	clock_set_attributes(
 	in	flavor		: clock_flavor_t;
 	in	clock_attr	: clock_attr_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/clock_reply.defs b/osfmk/mach/clock_reply.defs
index c96663319..9646f90da 100644
--- a/osfmk/mach/clock_reply.defs
+++ b/osfmk/mach/clock_reply.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -46,3 +52,5 @@ simpleroutine clock_alarm_reply(
 		alarm_code	: kern_return_t;
 		alarm_type	: alarm_type_t;
 		alarm_time	: mach_timespec_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/clock_types.defs b/osfmk/mach/clock_types.defs
index 0b4c68257..4dbacfcf1 100644
--- a/osfmk/mach/clock_types.defs
+++ b/osfmk/mach/clock_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -63,3 +69,5 @@ type clock_id_t = int;
 import <mach/mach_types.h>;
 
 #endif	/* _MACH_CLOCK_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/clock_types.h b/osfmk/mach/clock_types.h
index 5fd3f1243..12035da3c 100644
--- a/osfmk/mach/clock_types.h
+++ b/osfmk/mach/clock_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/error.h b/osfmk/mach/error.h
index 6335bdfac..a32eaee53 100644
--- a/osfmk/mach/error.h
+++ b/osfmk/mach/error.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/events_info.h b/osfmk/mach/events_info.h
index ab5b2eca6..c48772069 100644
--- a/osfmk/mach/events_info.h
+++ b/osfmk/mach/events_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/mach/exc.defs b/osfmk/mach/exc.defs
index 8236168a8..de6aee7b9 100644
--- a/osfmk/mach/exc.defs
+++ b/osfmk/mach/exc.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -110,3 +116,4 @@ routine		exception_raise_state_identity(
 			old_state       : thread_state_t;
 		    out new_state       : thread_state_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/exception.h b/osfmk/mach/exception.h
index b4a74bc14..e7d5d236c 100644
--- a/osfmk/mach/exception.h
+++ b/osfmk/mach/exception.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/exception_types.h b/osfmk/mach/exception_types.h
index 421f3fa16..3f3b487d7 100644
--- a/osfmk/mach/exception_types.h
+++ b/osfmk/mach/exception_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -86,6 +92,8 @@
 #define EXC_MACH_SYSCALL	8	/* Mach system calls. */
 
 #define EXC_RPC_ALERT		9	/* RPC alert */
+ 
+#define EXC_CRASH		10	/* Abnormal process exit */
 
 /*
  *	Machine-independent exception behaviors
@@ -105,6 +113,9 @@
  *	the thread identity and state.
  */
 
+#define MACH_EXCEPTION_CODES		0x80000000
+/*	Send 64-bit code and subcode in the exception header */
+
 /*
  * Masks for exception definitions, above
  * bit zero is unused, therefore 1 word = 31 exception types
@@ -119,6 +130,7 @@
 #define EXC_MASK_SYSCALL		(1 << EXC_SYSCALL)
 #define EXC_MASK_MACH_SYSCALL		(1 << EXC_MACH_SYSCALL)
 #define EXC_MASK_RPC_ALERT		(1 << EXC_RPC_ALERT)
+#define EXC_MASK_CRASH			(1 << EXC_CRASH)
 
 #define EXC_MASK_ALL	(EXC_MASK_BAD_ACCESS |			\
 			 EXC_MASK_BAD_INSTRUCTION |		\
@@ -129,6 +141,7 @@
 			 EXC_MASK_SYSCALL |			\
 			 EXC_MASK_MACH_SYSCALL |		\
 			 EXC_MASK_RPC_ALERT |			\
+			 EXC_MASK_CRASH |			\
 			 EXC_MASK_MACHINE)
 
 
@@ -150,15 +163,19 @@
  * Exported types
  */
 
-typedef int			exception_type_t;
-typedef	integer_t		exception_data_type_t;
-typedef int			exception_behavior_t;
-typedef integer_t		*exception_data_t;
-typedef	unsigned int		exception_mask_t;
-typedef	exception_mask_t	*exception_mask_array_t;
-typedef exception_behavior_t	*exception_behavior_array_t;
-typedef thread_state_flavor_t	*exception_flavor_array_t;
-typedef mach_port_t		*exception_port_array_t;
+typedef int				exception_type_t;
+typedef	integer_t			exception_data_type_t;
+typedef	int64_t				mach_exception_data_type_t;
+typedef int				exception_behavior_t;
+typedef exception_data_type_t		*exception_data_t;
+typedef mach_exception_data_type_t	*mach_exception_data_t;
+typedef	unsigned int			exception_mask_t;
+typedef	exception_mask_t		*exception_mask_array_t;
+typedef exception_behavior_t		*exception_behavior_array_t;
+typedef thread_state_flavor_t		*exception_flavor_array_t;
+typedef mach_port_t			*exception_port_array_t;
+typedef	mach_exception_data_type_t	mach_exception_code_t;
+typedef	mach_exception_data_type_t	mach_exception_subcode_t;
 
 #endif	/* ASSEMBLER */
 
diff --git a/osfmk/mach/flipc_cb.h b/osfmk/mach/flipc_cb.h
index 43d811757..4c7f969b9 100644
--- a/osfmk/mach/flipc_cb.h
+++ b/osfmk/mach/flipc_cb.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/flipc_debug.h b/osfmk/mach/flipc_debug.h
index 693d695c5..969deda3a 100644
--- a/osfmk/mach/flipc_debug.h
+++ b/osfmk/mach/flipc_debug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/flipc_device.h b/osfmk/mach/flipc_device.h
index 413a1e1cb..e76e52047 100644
--- a/osfmk/mach/flipc_device.h
+++ b/osfmk/mach/flipc_device.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/flipc_locks.h b/osfmk/mach/flipc_locks.h
index 222bb0b79..94d9a6ee5 100644
--- a/osfmk/mach/flipc_locks.h
+++ b/osfmk/mach/flipc_locks.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/flipc_types.h b/osfmk/mach/flipc_types.h
index 1019cfbbc..1f7f9782b 100644
--- a/osfmk/mach/flipc_types.h
+++ b/osfmk/mach/flipc_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/host_info.h b/osfmk/mach/host_info.h
index eb5367490..003f75101 100644
--- a/osfmk/mach/host_info.h
+++ b/osfmk/mach/host_info.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -96,7 +102,7 @@ typedef	integer_t	host_flavor_t;
 #ifdef MACH_KERNEL_PRIVATE
 struct host_basic_info_old {
 	integer_t	max_cpus;	/* max number of cpus possible */
-	integer_t	avail_cpus;	/* number of cpus now available */
+	uint32_t	avail_cpus;	/* number of cpus now available */
 	natural_t	memory_size;	/* size of memory in bytes */
 	cpu_type_t	cpu_type;	/* cpu type */
 	cpu_subtype_t	cpu_subtype;	/* cpu subtype */
@@ -188,8 +194,17 @@ typedef struct host_load_info	*host_load_info_t;
 /* in <mach/vm_statistics.h> */
 #define	HOST_VM_INFO_COUNT ((mach_msg_type_number_t) \
 		(sizeof(vm_statistics_data_t)/sizeof(integer_t)))
-#define	HOST_VM_INFO_REV0_COUNT ((mach_msg_type_number_t) \
-		(sizeof(vm_statistics_rev0_data_t)/sizeof(integer_t)))
+
+/* size of the latest version of the structure */
+#define HOST_VM_INFO_LATEST_COUNT HOST_VM_INFO_COUNT
+#define HOST_VM_INFO_REV2_COUNT	HOST_VM_INFO_LATEST_COUNT
+/* previous versions: adjust the size according to what was added each time */
+#define	HOST_VM_INFO_REV1_COUNT /* added "speculative_count" (1 int) */	\
+	((mach_msg_type_number_t) \
+	 (HOST_VM_INFO_REV2_COUNT - 1))
+#define	HOST_VM_INFO_REV0_COUNT /* added "purgable" info (2 ints) */ 	\
+	((mach_msg_type_number_t) \
+	 (HOST_VM_INFO_REV1_COUNT - 2))
 
 struct host_cpu_load_info {		/* number of ticks while running... */
 	natural_t	cpu_ticks[CPU_STATE_MAX]; /* ... in the given mode */
diff --git a/osfmk/mach/host_notify.h b/osfmk/mach/host_notify.h
index fc165bd44..6c0ca74b9 100644
--- a/osfmk/mach/host_notify.h
+++ b/osfmk/mach/host_notify.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_MACH_HOST_NOTIFY_H_
diff --git a/osfmk/mach/host_notify_reply.defs b/osfmk/mach/host_notify_reply.defs
index dac4b9e97..85437f821 100644
--- a/osfmk/mach/host_notify_reply.defs
+++ b/osfmk/mach/host_notify_reply.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 subsystem
@@ -30,3 +36,5 @@ subsystem
 
 simpleroutine host_calendar_changed(
 		notify_port	: mach_port_move_send_once_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/host_priv.defs b/osfmk/mach/host_priv.defs
index 1bc09c837..6250870a8 100644
--- a/osfmk/mach/host_priv.defs
+++ b/osfmk/mach/host_priv.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -352,3 +358,5 @@ routine host_set_UNDServer(
 routine host_get_UNDServer(    
 		host		: host_priv_t;
 	out	server		: UNDServerRef);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/host_reboot.h b/osfmk/mach/host_reboot.h
index 076e28f05..65609dd7f 100644
--- a/osfmk/mach/host_reboot.h
+++ b/osfmk/mach/host_reboot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/host_security.defs b/osfmk/mach/host_security.defs
index 812618127..4098df44e 100644
--- a/osfmk/mach/host_security.defs
+++ b/osfmk/mach/host_security.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -92,3 +98,4 @@ routine host_security_set_task_token(
 		audit_token	: audit_token_t;
 		host		: host_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/host_special_ports.h b/osfmk/mach/host_special_ports.h
index 27d00aa99..00e9580ea 100644
--- a/osfmk/mach/host_special_ports.h
+++ b/osfmk/mach/host_special_ports.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -77,6 +83,8 @@
 #define HOST_DYNAMIC_PAGER_PORT         (1 + HOST_MAX_SPECIAL_KERNEL_PORT)
 #define HOST_AUDIT_CONTROL_PORT         (2 + HOST_MAX_SPECIAL_KERNEL_PORT)
 #define HOST_USER_NOTIFICATION_PORT     (3 + HOST_MAX_SPECIAL_KERNEL_PORT)
+#define HOST_LOCKD_PORT                 (5 + HOST_MAX_SPECIAL_KERNEL_PORT)
+#define HOST_SEATBELT_PORT              (7 + HOST_MAX_SPECIAL_KERNEL_PORT)
 #define HOST_MAX_SPECIAL_PORT           (8 + HOST_MAX_SPECIAL_KERNEL_PORT)
                                         /* room to grow here as well */
 
@@ -128,4 +136,10 @@
 #define host_set_user_notification_port(host, port)	\
 	(host_set_special_port((host), HOST_USER_NOTIFICATION_PORT, (port)))
 
+#define host_get_lockd_port(host, port)	\
+	(host_get_special_port((host), 			\
+	HOST_LOCAL_NODE, HOST_LOCKD_PORT, (port)))
+#define host_set_lockd_port(host, port)	\
+	(host_set_special_port((host), HOST_LOCKD_PORT, (port)))
+
 #endif	/* _MACH_HOST_SPECIAL_PORTS_H_ */
diff --git a/osfmk/mach/i386/Makefile b/osfmk/mach/i386/Makefile
index a177315eb..7f99e1758 100644
--- a/osfmk/mach/i386/Makefile
+++ b/osfmk/mach/i386/Makefile
@@ -13,7 +13,7 @@ DATAFILES = \
 	processor_info.h kern_return.h ndr_def.h syscall_sw.h \
 	thread_status.h thread_state.h vm_param.h \
 	vm_types.h rpc.h \
-	machine_types.defs 
+	machine_types.defs  _structs.h sdt_isa.h
 
 INSTALL_MD_LIST = ${DATAFILES}
 
diff --git a/osfmk/mach/i386/_structs.h b/osfmk/mach/i386/_structs.h
new file mode 100644
index 000000000..927d12f63
--- /dev/null
+++ b/osfmk/mach/i386/_structs.h
@@ -0,0 +1,561 @@
+/*
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+#ifndef	_MACH_I386__STRUCTS_H_
+#define	_MACH_I386__STRUCTS_H_
+
+/*
+ * i386 is the structure that is exported to user threads for 
+ * use in status/mutate calls.  This structure should never change.
+ *
+ */
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_X86_THREAD_STATE32	struct __darwin_i386_thread_state
+_STRUCT_X86_THREAD_STATE32
+{
+    unsigned int	__eax;
+    unsigned int	__ebx;
+    unsigned int	__ecx;
+    unsigned int	__edx;
+    unsigned int	__edi;
+    unsigned int	__esi;
+    unsigned int	__ebp;
+    unsigned int	__esp;
+    unsigned int	__ss;
+    unsigned int	__eflags;
+    unsigned int	__eip;
+    unsigned int	__cs;
+    unsigned int	__ds;
+    unsigned int	__es;
+    unsigned int	__fs;
+    unsigned int	__gs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_X86_THREAD_STATE32	struct i386_thread_state
+_STRUCT_X86_THREAD_STATE32
+{
+    unsigned int	eax;
+    unsigned int	ebx;
+    unsigned int	ecx;
+    unsigned int	edx;
+    unsigned int	edi;
+    unsigned int	esi;
+    unsigned int	ebp;
+    unsigned int	esp;
+    unsigned int	ss;
+    unsigned int	eflags;
+    unsigned int	eip;
+    unsigned int	cs;
+    unsigned int	ds;
+    unsigned int	es;
+    unsigned int	fs;
+    unsigned int	gs;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+/* This structure should be double-word aligned for performance */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_FP_CONTROL	struct __darwin_fp_control
+_STRUCT_FP_CONTROL
+{
+    unsigned short		__invalid	:1,
+    				__denorm	:1,
+				__zdiv		:1,
+				__ovrfl		:1,
+				__undfl		:1,
+				__precis	:1,
+						:2,
+				__pc		:2,
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define FP_PREC_24B		0
+#define	FP_PREC_53B		2
+#define FP_PREC_64B		3
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
+				__rc		:2,
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define FP_RND_NEAR		0
+#define FP_RND_DOWN		1
+#define FP_RND_UP		2
+#define FP_CHOP			3
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
+					/*inf*/	:1,
+						:3;
+};
+typedef _STRUCT_FP_CONTROL	__darwin_fp_control_t;
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_FP_CONTROL	struct fp_control
+_STRUCT_FP_CONTROL
+{
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+					:2,
+				pc	:2,
+#define FP_PREC_24B		0
+#define	FP_PREC_53B		2
+#define FP_PREC_64B		3
+				rc	:2,
+#define FP_RND_NEAR		0
+#define FP_RND_DOWN		1
+#define FP_RND_UP		2
+#define FP_CHOP			3
+				/*inf*/	:1,
+					:3;
+};
+typedef _STRUCT_FP_CONTROL	fp_control_t;
+#endif /* !__DARWIN_UNIX03 */
+
+/*
+ * Status word.
+ */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_FP_STATUS	struct __darwin_fp_status
+_STRUCT_FP_STATUS
+{
+    unsigned short		__invalid	:1,
+    				__denorm	:1,
+				__zdiv		:1,
+				__ovrfl		:1,
+				__undfl		:1,
+				__precis	:1,
+				__stkflt	:1,
+				__errsumm	:1,
+				__c0		:1,
+				__c1		:1,
+				__c2		:1,
+				__tos		:3,
+				__c3		:1,
+				__busy		:1;
+};
+typedef _STRUCT_FP_STATUS	__darwin_fp_status_t;
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_FP_STATUS	struct fp_status
+_STRUCT_FP_STATUS
+{
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+				stkflt	:1,
+				errsumm	:1,
+				c0	:1,
+				c1	:1,
+				c2	:1,
+				tos	:3,
+				c3	:1,
+				busy	:1;
+};
+typedef _STRUCT_FP_STATUS	fp_status_t;
+#endif /* !__DARWIN_UNIX03 */
+				
+/* defn of 80bit x87 FPU or MMX register  */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_MMST_REG	struct __darwin_mmst_reg
+_STRUCT_MMST_REG
+{
+	char	__mmst_reg[10];
+	char	__mmst_rsrv[6];
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_MMST_REG	struct mmst_reg
+_STRUCT_MMST_REG
+{
+	char	mmst_reg[10];
+	char	mmst_rsrv[6];
+};
+#endif /* !__DARWIN_UNIX03 */
+
+
+/* defn of 128 bit XMM regs */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_XMM_REG		struct __darwin_xmm_reg
+_STRUCT_XMM_REG
+{
+	char		__xmm_reg[16];
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_XMM_REG		struct xmm_reg
+_STRUCT_XMM_REG
+{
+	char		xmm_reg[16];
+};
+#endif /* !__DARWIN_UNIX03 */
+
+/* 
+ * Floating point state.
+ */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#define FP_STATE_BYTES		512	/* number of chars worth of data from fpu_fcw */
+#endif /* !_POSIX_C_SOURCE || _DARWIN_C_SOURCE */
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_X86_FLOAT_STATE32	struct __darwin_i386_float_state
+_STRUCT_X86_FLOAT_STATE32
+{
+	int 			__fpu_reserved[2];
+	_STRUCT_FP_CONTROL	__fpu_fcw;		/* x87 FPU control word */
+	_STRUCT_FP_STATUS	__fpu_fsw;		/* x87 FPU status word */
+	__uint8_t		__fpu_ftw;		/* x87 FPU tag word */
+	__uint8_t		__fpu_rsrv1;		/* reserved */ 
+	__uint16_t		__fpu_fop;		/* x87 FPU Opcode */
+	__uint32_t		__fpu_ip;		/* x87 FPU Instruction Pointer offset */
+	__uint16_t		__fpu_cs;		/* x87 FPU Instruction Pointer Selector */
+	__uint16_t		__fpu_rsrv2;		/* reserved */
+	__uint32_t		__fpu_dp;		/* x87 FPU Instruction Operand(Data) Pointer offset */
+	__uint16_t		__fpu_ds;		/* x87 FPU Instruction Operand(Data) Pointer Selector */
+	__uint16_t		__fpu_rsrv3;		/* reserved */
+	__uint32_t		__fpu_mxcsr;		/* MXCSR Register state */
+	__uint32_t		__fpu_mxcsrmask;	/* MXCSR mask */
+	_STRUCT_MMST_REG	__fpu_stmm0;		/* ST0/MM0   */
+	_STRUCT_MMST_REG	__fpu_stmm1;		/* ST1/MM1  */
+	_STRUCT_MMST_REG	__fpu_stmm2;		/* ST2/MM2  */
+	_STRUCT_MMST_REG	__fpu_stmm3;		/* ST3/MM3  */
+	_STRUCT_MMST_REG	__fpu_stmm4;		/* ST4/MM4  */
+	_STRUCT_MMST_REG	__fpu_stmm5;		/* ST5/MM5  */
+	_STRUCT_MMST_REG	__fpu_stmm6;		/* ST6/MM6  */
+	_STRUCT_MMST_REG	__fpu_stmm7;		/* ST7/MM7  */
+	_STRUCT_XMM_REG		__fpu_xmm0;		/* XMM 0  */
+	_STRUCT_XMM_REG		__fpu_xmm1;		/* XMM 1  */
+	_STRUCT_XMM_REG		__fpu_xmm2;		/* XMM 2  */
+	_STRUCT_XMM_REG		__fpu_xmm3;		/* XMM 3  */
+	_STRUCT_XMM_REG		__fpu_xmm4;		/* XMM 4  */
+	_STRUCT_XMM_REG		__fpu_xmm5;		/* XMM 5  */
+	_STRUCT_XMM_REG		__fpu_xmm6;		/* XMM 6  */
+	_STRUCT_XMM_REG		__fpu_xmm7;		/* XMM 7  */
+	char			__fpu_rsrv4[14*16];	/* reserved */
+	int 			__fpu_reserved1;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_X86_FLOAT_STATE32	struct i386_float_state
+_STRUCT_X86_FLOAT_STATE32
+{
+	int 			fpu_reserved[2];
+	_STRUCT_FP_CONTROL	fpu_fcw;		/* x87 FPU control word */
+	_STRUCT_FP_STATUS	fpu_fsw;		/* x87 FPU status word */
+	__uint8_t		fpu_ftw;		/* x87 FPU tag word */
+	__uint8_t		fpu_rsrv1;		/* reserved */ 
+	__uint16_t		fpu_fop;		/* x87 FPU Opcode */
+	__uint32_t		fpu_ip;			/* x87 FPU Instruction Pointer offset */
+	__uint16_t		fpu_cs;			/* x87 FPU Instruction Pointer Selector */
+	__uint16_t		fpu_rsrv2;		/* reserved */
+	__uint32_t		fpu_dp;			/* x87 FPU Instruction Operand(Data) Pointer offset */
+	__uint16_t		fpu_ds;			/* x87 FPU Instruction Operand(Data) Pointer Selector */
+	__uint16_t		fpu_rsrv3;		/* reserved */
+	__uint32_t		fpu_mxcsr;		/* MXCSR Register state */
+	__uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
+	_STRUCT_MMST_REG	fpu_stmm0;		/* ST0/MM0   */
+	_STRUCT_MMST_REG	fpu_stmm1;		/* ST1/MM1  */
+	_STRUCT_MMST_REG	fpu_stmm2;		/* ST2/MM2  */
+	_STRUCT_MMST_REG	fpu_stmm3;		/* ST3/MM3  */
+	_STRUCT_MMST_REG	fpu_stmm4;		/* ST4/MM4  */
+	_STRUCT_MMST_REG	fpu_stmm5;		/* ST5/MM5  */
+	_STRUCT_MMST_REG	fpu_stmm6;		/* ST6/MM6  */
+	_STRUCT_MMST_REG	fpu_stmm7;		/* ST7/MM7  */
+	_STRUCT_XMM_REG		fpu_xmm0;		/* XMM 0  */
+	_STRUCT_XMM_REG		fpu_xmm1;		/* XMM 1  */
+	_STRUCT_XMM_REG		fpu_xmm2;		/* XMM 2  */
+	_STRUCT_XMM_REG		fpu_xmm3;		/* XMM 3  */
+	_STRUCT_XMM_REG		fpu_xmm4;		/* XMM 4  */
+	_STRUCT_XMM_REG		fpu_xmm5;		/* XMM 5  */
+	_STRUCT_XMM_REG		fpu_xmm6;		/* XMM 6  */
+	_STRUCT_XMM_REG		fpu_xmm7;		/* XMM 7  */
+	char			fpu_rsrv4[14*16];	/* reserved */
+	int 			fpu_reserved1;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_X86_EXCEPTION_STATE32	struct __darwin_i386_exception_state
+_STRUCT_X86_EXCEPTION_STATE32
+{
+    unsigned int	__trapno;
+    unsigned int	__err;
+    unsigned int	__faultvaddr;
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_X86_EXCEPTION_STATE32	struct i386_exception_state
+_STRUCT_X86_EXCEPTION_STATE32
+{
+    unsigned int	trapno;
+    unsigned int	err;
+    unsigned int	faultvaddr;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_X86_DEBUG_STATE32	struct __darwin_x86_debug_state32
+_STRUCT_X86_DEBUG_STATE32
+{
+	unsigned int	__dr0;
+	unsigned int	__dr1;
+	unsigned int	__dr2;
+	unsigned int	__dr3;
+	unsigned int	__dr4;
+	unsigned int	__dr5;
+	unsigned int	__dr6;
+	unsigned int	__dr7;
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_X86_DEBUG_STATE32	struct x86_debug_state32
+_STRUCT_X86_DEBUG_STATE32
+{
+	unsigned int	dr0;
+	unsigned int	dr1;
+	unsigned int	dr2;
+	unsigned int	dr3;
+	unsigned int	dr4;
+	unsigned int	dr5;
+	unsigned int	dr6;
+	unsigned int	dr7;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+/*
+ * 64 bit versions of the above
+ */
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_X86_THREAD_STATE64	struct __darwin_x86_thread_state64
+_STRUCT_X86_THREAD_STATE64
+{
+	__uint64_t	__rax;
+	__uint64_t	__rbx;
+	__uint64_t	__rcx;
+	__uint64_t	__rdx;
+	__uint64_t	__rdi;
+	__uint64_t	__rsi;
+	__uint64_t	__rbp;
+	__uint64_t	__rsp;
+	__uint64_t	__r8;
+	__uint64_t	__r9;
+	__uint64_t	__r10;
+	__uint64_t	__r11;
+	__uint64_t	__r12;
+	__uint64_t	__r13;
+	__uint64_t	__r14;
+	__uint64_t	__r15;
+	__uint64_t	__rip;
+	__uint64_t	__rflags;
+	__uint64_t	__cs;
+	__uint64_t	__fs;
+	__uint64_t	__gs;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_X86_THREAD_STATE64	struct x86_thread_state64
+_STRUCT_X86_THREAD_STATE64
+{
+	__uint64_t	rax;
+	__uint64_t	rbx;
+	__uint64_t	rcx;
+	__uint64_t	rdx;
+	__uint64_t	rdi;
+	__uint64_t	rsi;
+	__uint64_t	rbp;
+	__uint64_t	rsp;
+	__uint64_t	r8;
+	__uint64_t	r9;
+	__uint64_t	r10;
+	__uint64_t	r11;
+	__uint64_t	r12;
+	__uint64_t	r13;
+	__uint64_t	r14;
+	__uint64_t	r15;
+	__uint64_t	rip;
+	__uint64_t	rflags;
+	__uint64_t	cs;
+	__uint64_t	fs;
+	__uint64_t	gs;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_X86_FLOAT_STATE64	struct __darwin_x86_float_state64
+_STRUCT_X86_FLOAT_STATE64
+{
+	int 			__fpu_reserved[2];
+	_STRUCT_FP_CONTROL	__fpu_fcw;		/* x87 FPU control word */
+	_STRUCT_FP_STATUS	__fpu_fsw;		/* x87 FPU status word */
+	__uint8_t		__fpu_ftw;		/* x87 FPU tag word */
+	__uint8_t		__fpu_rsrv1;		/* reserved */ 
+	__uint16_t		__fpu_fop;		/* x87 FPU Opcode */
+
+	/* x87 FPU Instruction Pointer */
+	__uint32_t		__fpu_ip;		/* offset */
+	__uint16_t		__fpu_cs;		/* Selector */
+
+	__uint16_t		__fpu_rsrv2;		/* reserved */
+
+	/* x87 FPU Instruction Operand(Data) Pointer */
+	__uint32_t		__fpu_dp;		/* offset */
+	__uint16_t		__fpu_ds;		/* Selector */
+
+	__uint16_t		__fpu_rsrv3;		/* reserved */
+	__uint32_t		__fpu_mxcsr;		/* MXCSR Register state */
+	__uint32_t		__fpu_mxcsrmask;	/* MXCSR mask */
+	_STRUCT_MMST_REG	__fpu_stmm0;		/* ST0/MM0   */
+	_STRUCT_MMST_REG	__fpu_stmm1;		/* ST1/MM1  */
+	_STRUCT_MMST_REG	__fpu_stmm2;		/* ST2/MM2  */
+	_STRUCT_MMST_REG	__fpu_stmm3;		/* ST3/MM3  */
+	_STRUCT_MMST_REG	__fpu_stmm4;		/* ST4/MM4  */
+	_STRUCT_MMST_REG	__fpu_stmm5;		/* ST5/MM5  */
+	_STRUCT_MMST_REG	__fpu_stmm6;		/* ST6/MM6  */
+	_STRUCT_MMST_REG	__fpu_stmm7;		/* ST7/MM7  */
+	_STRUCT_XMM_REG		__fpu_xmm0;		/* XMM 0  */
+	_STRUCT_XMM_REG		__fpu_xmm1;		/* XMM 1  */
+	_STRUCT_XMM_REG		__fpu_xmm2;		/* XMM 2  */
+	_STRUCT_XMM_REG		__fpu_xmm3;		/* XMM 3  */
+	_STRUCT_XMM_REG		__fpu_xmm4;		/* XMM 4  */
+	_STRUCT_XMM_REG		__fpu_xmm5;		/* XMM 5  */
+	_STRUCT_XMM_REG		__fpu_xmm6;		/* XMM 6  */
+	_STRUCT_XMM_REG		__fpu_xmm7;		/* XMM 7  */
+	_STRUCT_XMM_REG		__fpu_xmm8;		/* XMM 8  */
+	_STRUCT_XMM_REG		__fpu_xmm9;		/* XMM 9  */
+	_STRUCT_XMM_REG		__fpu_xmm10;		/* XMM 10  */
+	_STRUCT_XMM_REG		__fpu_xmm11;		/* XMM 11 */
+	_STRUCT_XMM_REG		__fpu_xmm12;		/* XMM 12  */
+	_STRUCT_XMM_REG		__fpu_xmm13;		/* XMM 13  */
+	_STRUCT_XMM_REG		__fpu_xmm14;		/* XMM 14  */
+	_STRUCT_XMM_REG		__fpu_xmm15;		/* XMM 15  */
+	char			__fpu_rsrv4[6*16];	/* reserved */
+	int 			__fpu_reserved1;
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_X86_FLOAT_STATE64	struct x86_float_state64
+_STRUCT_X86_FLOAT_STATE64
+{
+	int 			fpu_reserved[2];
+	_STRUCT_FP_CONTROL	fpu_fcw;		/* x87 FPU control word */
+	_STRUCT_FP_STATUS	fpu_fsw;		/* x87 FPU status word */
+	__uint8_t		fpu_ftw;		/* x87 FPU tag word */
+	__uint8_t		fpu_rsrv1;		/* reserved */ 
+	__uint16_t		fpu_fop;		/* x87 FPU Opcode */
+
+	/* x87 FPU Instruction Pointer */
+	__uint32_t		fpu_ip;			/* offset */
+	__uint16_t		fpu_cs;			/* Selector */
+
+	__uint16_t		fpu_rsrv2;		/* reserved */
+
+	/* x87 FPU Instruction Operand(Data) Pointer */
+	__uint32_t		fpu_dp;			/* offset */
+	__uint16_t		fpu_ds;			/* Selector */
+
+	__uint16_t		fpu_rsrv3;		/* reserved */
+	__uint32_t		fpu_mxcsr;		/* MXCSR Register state */
+	__uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
+	_STRUCT_MMST_REG	fpu_stmm0;		/* ST0/MM0   */
+	_STRUCT_MMST_REG	fpu_stmm1;		/* ST1/MM1  */
+	_STRUCT_MMST_REG	fpu_stmm2;		/* ST2/MM2  */
+	_STRUCT_MMST_REG	fpu_stmm3;		/* ST3/MM3  */
+	_STRUCT_MMST_REG	fpu_stmm4;		/* ST4/MM4  */
+	_STRUCT_MMST_REG	fpu_stmm5;		/* ST5/MM5  */
+	_STRUCT_MMST_REG	fpu_stmm6;		/* ST6/MM6  */
+	_STRUCT_MMST_REG	fpu_stmm7;		/* ST7/MM7  */
+	_STRUCT_XMM_REG		fpu_xmm0;		/* XMM 0  */
+	_STRUCT_XMM_REG		fpu_xmm1;		/* XMM 1  */
+	_STRUCT_XMM_REG		fpu_xmm2;		/* XMM 2  */
+	_STRUCT_XMM_REG		fpu_xmm3;		/* XMM 3  */
+	_STRUCT_XMM_REG		fpu_xmm4;		/* XMM 4  */
+	_STRUCT_XMM_REG		fpu_xmm5;		/* XMM 5  */
+	_STRUCT_XMM_REG		fpu_xmm6;		/* XMM 6  */
+	_STRUCT_XMM_REG		fpu_xmm7;		/* XMM 7  */
+	_STRUCT_XMM_REG		fpu_xmm8;		/* XMM 8  */
+	_STRUCT_XMM_REG		fpu_xmm9;		/* XMM 9  */
+	_STRUCT_XMM_REG		fpu_xmm10;		/* XMM 10  */
+	_STRUCT_XMM_REG		fpu_xmm11;		/* XMM 11 */
+	_STRUCT_XMM_REG		fpu_xmm12;		/* XMM 12  */
+	_STRUCT_XMM_REG		fpu_xmm13;		/* XMM 13  */
+	_STRUCT_XMM_REG		fpu_xmm14;		/* XMM 14  */
+	_STRUCT_XMM_REG		fpu_xmm15;		/* XMM 15  */
+	char			fpu_rsrv4[6*16];	/* reserved */
+	int 			fpu_reserved1;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_X86_EXCEPTION_STATE64	struct __darwin_x86_exception_state64
+_STRUCT_X86_EXCEPTION_STATE64
+{
+    unsigned int	__trapno;
+    unsigned int	__err;
+    __uint64_t		__faultvaddr;
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_X86_EXCEPTION_STATE64	struct x86_exception_state64
+_STRUCT_X86_EXCEPTION_STATE64
+{
+    unsigned int	trapno;
+    unsigned int	err;
+    __uint64_t		faultvaddr;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_X86_DEBUG_STATE64	struct __darwin_x86_debug_state64
+_STRUCT_X86_DEBUG_STATE64
+{
+	__uint64_t	__dr0;
+	__uint64_t	__dr1;
+	__uint64_t	__dr2;
+	__uint64_t	__dr3;
+	__uint64_t	__dr4;
+	__uint64_t	__dr5;
+	__uint64_t	__dr6;
+	__uint64_t	__dr7;
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_X86_DEBUG_STATE64	struct x86_debug_state64
+_STRUCT_X86_DEBUG_STATE64
+{
+	__uint64_t	dr0;
+	__uint64_t	dr1;
+	__uint64_t	dr2;
+	__uint64_t	dr3;
+	__uint64_t	dr4;
+	__uint64_t	dr5;
+	__uint64_t	dr6;
+	__uint64_t	dr7;
+};
+#endif /* !__DARWIN_UNIX03 */
+
+#endif /* _MACH_I386__STRUCTS_H_ */
diff --git a/osfmk/mach/i386/_types.h b/osfmk/mach/i386/_types.h
new file mode 100644
index 000000000..5679b84e2
--- /dev/null
+++ b/osfmk/mach/i386/_types.h
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+#ifndef	_MACH_I386__TYPES_H_
+#define	_MACH_I386__TYPES_H_
+
+/*
+ * i386_thread_state is the structure that is exported to user threads for 
+ * use in status/mutate calls.  This structure should never change.
+ *
+ */
+
+#if !__DARWIN_UNIX03
+struct i386_thread_state
+#else /* __DARWIN_UNIX03 */
+struct __darwin_i386_thread_state
+#endif /* __DARWIN_UNIX03 */
+{
+    unsigned int	eax;
+    unsigned int	ebx;
+    unsigned int	ecx;
+    unsigned int	edx;
+    unsigned int	edi;
+    unsigned int	esi;
+    unsigned int	ebp;
+    unsigned int	esp;
+    unsigned int	ss;
+    unsigned int	eflags;
+    unsigned int	eip;
+    unsigned int	cs;
+    unsigned int	ds;
+    unsigned int	es;
+    unsigned int	fs;
+    unsigned int	gs;
+};
+
+#if !__DARWIN_UNIX03
+struct x86_thread_state64
+#else /* __DARWIN_UNIX03 */
+struct __darwin_x86_thread_state64
+#endif /* __DARWIN_UNIX03 */
+{
+	uint64_t	rax;
+	uint64_t	rbx;
+	uint64_t	rcx;
+	uint64_t	rdx;
+	uint64_t	rdi;
+	uint64_t	rsi;
+	uint64_t	rbp;
+	uint64_t	rsp;
+	uint64_t	r8;
+	uint64_t	r9;
+	uint64_t	r10;
+	uint64_t	r11;
+	uint64_t	r12;
+	uint64_t	r13;
+	uint64_t	r14;
+	uint64_t	r15;
+	uint64_t	rip;
+	uint64_t	rflags;
+	uint64_t	cs;
+	uint64_t	fs;
+	uint64_t	gs;
+};
+
+
+typedef struct fp_control {
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+					:2,
+				pc	:2,
+#define FP_PREC_24B		0
+#define	FP_PREC_53B		2
+#define FP_PREC_64B		3
+				rc	:2,
+#define FP_RND_NEAR		0
+#define FP_RND_DOWN		1
+#define FP_RND_UP		2
+#define FP_CHOP			3
+				/*inf*/	:1,
+					:3;
+} fp_control_t;
+/*
+ * Status word.
+ */
+
+typedef struct fp_status {
+    unsigned short		invalid	:1,
+    				denorm	:1,
+				zdiv	:1,
+				ovrfl	:1,
+				undfl	:1,
+				precis	:1,
+				stkflt	:1,
+				errsumm	:1,
+				c0	:1,
+				c1	:1,
+				c2	:1,
+				tos	:3,
+				c3	:1,
+				busy	:1;
+} fp_status_t;
+				
+/* defn of 80bit x87 FPU or MMX register  */
+struct mmst_reg {
+	char	mmst_reg[10];
+	char	mmst_rsrv[6];
+};
+
+
+/* defn of 128 bit XMM regs */
+struct xmm_reg {
+	char		xmm_reg[16];
+};
+
+/* 
+ * Floating point state.
+ */
+
+#define FP_STATE_BYTES		512	/* number of chars worth of data from fpu_fcw */
+#if !__DARWIN_UNIX03
+struct i386_float_state
+#else /* __DARWIN_UNIX03 */
+struct __darwin_i386_float_state
+#endif /* __DARWIN_UNIX03 */
+{
+	int 			fpu_reserved[2];
+	fp_control_t	fpu_fcw;			/* x87 FPU control word */
+	fp_status_t		fpu_fsw;			/* x87 FPU status word */
+	uint8_t			fpu_ftw;			/* x87 FPU tag word */
+	uint8_t			fpu_rsrv1;			/* reserved */ 
+	uint16_t		fpu_fop;			/* x87 FPU Opcode */
+	uint32_t		fpu_ip;				/* x87 FPU Instruction Pointer offset */
+	uint16_t		fpu_cs;				/* x87 FPU Instruction Pointer Selector */
+	uint16_t		fpu_rsrv2;			/* reserved */
+	uint32_t		fpu_dp;				/* x87 FPU Instruction Operand(Data) Pointer offset */
+	uint16_t		fpu_ds;				/* x87 FPU Instruction Operand(Data) Pointer Selector */
+	uint16_t		fpu_rsrv3;			/* reserved */
+	uint32_t		fpu_mxcsr;			/* MXCSR Register state */
+	uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
+	struct mmst_reg	fpu_stmm0;		/* ST0/MM0   */
+	struct mmst_reg	fpu_stmm1;		/* ST1/MM1  */
+	struct mmst_reg	fpu_stmm2;		/* ST2/MM2  */
+	struct mmst_reg	fpu_stmm3;		/* ST3/MM3  */
+	struct mmst_reg	fpu_stmm4;		/* ST4/MM4  */
+	struct mmst_reg	fpu_stmm5;		/* ST5/MM5  */
+	struct mmst_reg	fpu_stmm6;		/* ST6/MM6  */
+	struct mmst_reg	fpu_stmm7;		/* ST7/MM7  */
+	struct xmm_reg	fpu_xmm0;		/* XMM 0  */
+	struct xmm_reg	fpu_xmm1;		/* XMM 1  */
+	struct xmm_reg	fpu_xmm2;		/* XMM 2  */
+	struct xmm_reg	fpu_xmm3;		/* XMM 3  */
+	struct xmm_reg	fpu_xmm4;		/* XMM 4  */
+	struct xmm_reg	fpu_xmm5;		/* XMM 5  */
+	struct xmm_reg	fpu_xmm6;		/* XMM 6  */
+	struct xmm_reg	fpu_xmm7;		/* XMM 7  */
+	char			fpu_rsrv4[14*16];	/* reserved */
+	int 			fpu_reserved1;
+};
+
+
+#if !__DARWIN_UNIX03
+struct i386_exception_state
+#else /* __DARWIN_UNIX03 */
+struct __darwin_i386_exception_state
+#endif /* __DARWIN_UNIX03 */
+{
+    unsigned int	trapno;
+    unsigned int	err;
+    unsigned int	faultvaddr;
+};
+
+#if !__DARWIN_UNIX03
+struct x86_debug_state
+#else /* __DARWIN_UNIX03 */
+struct __darwin_x86_debug_state
+#endif /* __DARWIN_UNIX03 */
+{
+	unsigned int	dr0;
+	unsigned int	dr1;
+	unsigned int	dr2;
+	unsigned int	dr3;
+	unsigned int	dr4;
+	unsigned int	dr5;
+	unsigned int	dr6;
+	unsigned int	dr7;
+};
+
+#endif /* _MACH_I386__TYPES_H_ */
diff --git a/osfmk/mach/i386/boolean.h b/osfmk/mach/i386/boolean.h
index 6dfd2d8fa..cfb425c54 100644
--- a/osfmk/mach/i386/boolean.h
+++ b/osfmk/mach/i386/boolean.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/exception.h b/osfmk/mach/i386/exception.h
index f39d6e6e7..77fe4e404 100644
--- a/osfmk/mach/i386/exception.h
+++ b/osfmk/mach/i386/exception.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,7 +63,7 @@
  * No machine dependent types for the 80386
  */
 
-#define	EXC_TYPES_COUNT	10	/* incl. illegal exception 0 */
+#define	EXC_TYPES_COUNT	11	/* incl. illegal exception 0 */
 
 /*
  *	Codes and subcodes for 80386 exceptions.
diff --git a/osfmk/mach/i386/flipc_dep.h b/osfmk/mach/i386/flipc_dep.h
index a8be8ed52..e04a83aaa 100644
--- a/osfmk/mach/i386/flipc_dep.h
+++ b/osfmk/mach/i386/flipc_dep.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/fp_reg.h b/osfmk/mach/i386/fp_reg.h
index 3ab06f2b0..47a26a99d 100644
--- a/osfmk/mach/i386/fp_reg.h
+++ b/osfmk/mach/i386/fp_reg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -69,7 +75,7 @@ struct x86_fx_save {
         unsigned short  fx_bbz3;	/* better be zero when calling fxrtstor */
         unsigned int  	fx_MXCSR;
         unsigned int  	fx_MXCSR_MASK;
-        unsigned short  fx_reg_word[8][8];	/* STx/MMx registers */
+        unsigned short  fx_reg_word[8][8];      /* STx/MMx registers */
         unsigned short  fx_XMM_reg[8][16];	/* XMM0-XMM15 on 64 bit processors */
                                                 /* XMM0-XMM7  on 32 bit processors... unused storage reserved */
         unsigned char 	fx_reserved[16*6];	/* reserved by intel for future expansion */
diff --git a/osfmk/mach/i386/kern_return.h b/osfmk/mach/i386/kern_return.h
index 6a19c7463..a9df708de 100644
--- a/osfmk/mach/i386/kern_return.h
+++ b/osfmk/mach/i386/kern_return.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/machine_types.defs b/osfmk/mach/i386/machine_types.defs
index 4132ef6e3..f490f14c2 100644
--- a/osfmk/mach/i386/machine_types.defs
+++ b/osfmk/mach/i386/machine_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -107,3 +113,5 @@ type mach_vm_size_t = uint64_t;
 #endif	/* MACH_IPC_COMPAT */
 
 #endif	/* _MACHINE_VM_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/i386/ndr_def.h b/osfmk/mach/i386/ndr_def.h
index f6af3476a..5163f639c 100644
--- a/osfmk/mach/i386/ndr_def.h
+++ b/osfmk/mach/i386/ndr_def.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/processor_info.h b/osfmk/mach/i386/processor_info.h
index 8ca068dac..8272c6d82 100644
--- a/osfmk/mach/i386/processor_info.h
+++ b/osfmk/mach/i386/processor_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File:	mach/i386/processor_info.h
diff --git a/osfmk/mach/i386/rpc.h b/osfmk/mach/i386/rpc.h
index e558e6fa7..55561f9eb 100644
--- a/osfmk/mach/i386/rpc.h
+++ b/osfmk/mach/i386/rpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/sdt_isa.h b/osfmk/mach/i386/sdt_isa.h
new file mode 100644
index 000000000..03947431e
--- /dev/null
+++ b/osfmk/mach/i386/sdt_isa.h
@@ -0,0 +1,465 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _MACH_I386_SDT_ISA_H
+#define	_MACH_I386_SDT_ISA_H
+
+/*
+ * Only define when testing.  This makes the calls into actual calls to
+ * test functions.
+ */
+/* #define DTRACE_CALL_TEST */
+
+#define DTRACE_STRINGIFY(s) #s
+#define DTRACE_TOSTRING(s) DTRACE_STRINGIFY(s)
+#if defined(KERNEL)
+/*
+ * For the kernel, set an explicit global label do the symbol can be located
+ */
+#define DTRACE_LAB(p, n)		\
+   "__dtrace_probe$" DTRACE_TOSTRING(__LINE__) DTRACE_STRINGIFY(_##p##___##n)
+
+#define DTRACE_LABEL(p, n)		\
+      ".section __DATA, __data\n\t"	\
+      ".globl " DTRACE_LAB(p, n) "\n\t"	\
+       DTRACE_LAB(p, n) ":" ".long 1f""\n\t"	\
+       ".text" "\n\t"			\
+	"1:"
+#else	/* !KERNEL */
+#define DTRACE_LABEL(p, n)									\
+	"__dtrace_probe$" DTRACE_TOSTRING(__LINE__) DTRACE_STRINGIFY(_##p##___##n) ":"	"\n\t"
+#endif	/* !KERNEL */
+
+#ifdef DTRACE_CALL_TEST
+
+#define DTRACE_CALL(p,n)	\
+	DTRACE_LABEL(p,n)	\
+	DTRACE_CALL_INSN(p,n)
+
+#else
+
+#define DTRACE_CALL(p,n)	\
+	DTRACE_LABEL(p,n)	\
+	DTRACE_NOPS
+
+#endif
+
+#ifdef __x86_64__
+
+#define DTRACE_NOPS			\
+	"nop"			"\n\t"	\
+	"leal 0(%%esi), %%esi"	"\n\t"	
+
+#define DTRACE_CALL_INSN(p,n)						\
+	"call _dtracetest" DTRACE_STRINGIFY(_##p##_##n)	"\n\t"
+
+#define ARG1_EXTENT	1
+#define ARGS2_EXTENT	2
+#define ARGS3_EXTENT	3
+#define ARGS4_EXTENT	4
+#define ARGS5_EXTENT	5
+#define ARGS6_EXTENT	6
+#define ARGS7_EXTENT	7
+#define ARGS8_EXTENT	8
+#define ARGS9_EXTENT	10	
+#define ARGS10_EXTENT	10	
+
+#define DTRACE_CALL0ARGS(provider, name)							\
+	asm volatile (										\
+		      DTRACE_CALL(provider, name)						\
+	);
+
+#define DTRACE_CALL1ARG(provider, name)								\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi"									\
+	);
+
+#define DTRACE_CALL2ARGS(provider, name)							\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi"								\
+	);
+
+#define DTRACE_CALL3ARGS(provider, name)							\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx"							\
+	);
+
+#define DTRACE_CALL4ARGS(provider, name)							\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx"						\
+	);
+
+#define DTRACE_CALL5ARGS(provider, name)							\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+	              "movq\t0x20(%0),%%r8"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx", "r8"					\
+	);
+
+#define DTRACE_CALL6ARGS(provider, name)							\
+	asm volatile ("movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+	              "movq\t0x20(%0),%%r8"						"\n\t"	\
+	              "movq\t0x28(%0),%%r9"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx", "r8", "r9"					\
+	);
+
+#define DTRACE_CALL7ARGS(provider, name)							\
+	asm volatile ("subq\t$0x8,%%rsp"						"\n\t"	\
+	              "movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+	              "movq\t0x20(%0),%%r8"						"\n\t"	\
+	              "movq\t0x28(%0),%%r9"						"\n\t"	\
+	              "movq\t0x30(%0),%%rax"						"\n\t"	\
+	              "movq\t%%rax,0x0(%%rsp)"						"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              "addq\t$0x8,%%rsp"						"\n\t"	\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx", "r8", "r9", "rax"				\
+	);
+
+#define DTRACE_CALL8ARGS(provider, name)							\
+	asm volatile ("subq\t$0x10,%%rsp"						"\n\t"	\
+	              "movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+	              "movq\t0x20(%0),%%r8"						"\n\t"	\
+	              "movq\t0x28(%0),%%r9"						"\n\t"	\
+	              "movdqa\t0x30(%0),%%xmm1"						"\n\t"	\
+	              "movdqa\t%%xmm1,0x0(%%rsp)"					"\n\t"	\
+		      DTRACE_CALL(provider, name)						\
+	              "addq\t$0x10,%%rsp"						"\n\t"	\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx", "r8", "r9", "xmm1"				\
+	);
+
+#define DTRACE_CALL9ARGS(provider, name)							\
+	DTRACE_CALL10ARGS(provider, name)
+
+#define DTRACE_CALL10ARGS(provider, name)							\
+	asm volatile ("subq\t$0x20,%%rsp"						"\n\t"	\
+	              "movq\t0x0(%0),%%rdi"						"\n\t"	\
+	              "movq\t0x8(%0),%%rsi"						"\n\t"	\
+	              "movq\t0x10(%0),%%rdx"						"\n\t"	\
+	              "movq\t0x18(%0),%%rcx"						"\n\t"	\
+	              "movq\t0x20(%0),%%r8"						"\n\t"	\
+	              "movq\t0x28(%0),%%r9"						"\n\t"	\
+	              "movdqa\t0x30(%0),%%xmm1"						"\n\t"	\
+	              "movdqa\t0x40(%0),%%xmm2"						"\n\t"	\
+	              "movdqa\t%%xmm1,0x0(%%rsp)"					"\n\t"	\
+	              "movdqa\t%%xmm2,0x10(%%rsp)"					"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addq\t$0x20,%%rsp"						"\n\t"	\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "rdi", "rsi", "rdx", "rcx", "r8", "r9", "xmm1", "xmm2"			\
+	);
+
+#endif // __x86_64__
+
+#ifdef __i386__
+
+#define DTRACE_NOPS			\
+	"nop"			"\n\t"	\
+	"leal 0(%%esi), %%esi"	"\n\t"	
+
+#define DTRACE_CALL_INSN(p,n)						\
+	"call _dtracetest" DTRACE_STRINGIFY(_##p##_##n)	"\n\t"
+
+#define ARG1_EXTENT	1
+#define ARGS2_EXTENT	2
+#define ARGS3_EXTENT	4
+#define ARGS4_EXTENT	4
+#define ARGS5_EXTENT	8
+#define ARGS6_EXTENT	8
+#define ARGS7_EXTENT	8
+#define ARGS8_EXTENT	8
+#define ARGS9_EXTENT	12	
+#define ARGS10_EXTENT	12	
+
+/*
+ * Because this code is used in the kernel, we must not touch any floating point
+ * or specialized registers. This leaves the following registers:
+ *
+ * eax ; volatile, safe to use
+ * ebx ; PIC register, gcc error when used
+ * ecx ; volatile, safe to use
+ * edx ; volatile, safe to use
+ * esi ; non-volatile, otherwise safe to use
+ * edi ; non-volatile, otherwise safe to use
+ *
+ * Using any of the non volatile register causes a spill to stack which is almost
+ * certainly a net performance loss. Also, note that the array ref (__dtrace_args)
+ * consumes one free register. If all three of the volatile regs are used for load/store,
+ * the compiler will spill a register to hold the array ref.
+ *
+ * The end result is that we only pipeline two loads/stores at a time. Blech.
+ */
+
+#define DTRACE_CALL0ARGS(provider, name)							\
+	asm volatile (										\
+	              DTRACE_CALL(provider, name)						\
+	              "# eat trailing nl +tabfrom DTRACE_CALL"					\
+	              :										\
+	              :										\
+	);
+
+#define DTRACE_CALL1ARG(provider, name)								\
+	asm volatile ("subl\t$0x10,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x10,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)		       					\
+	              : "memory", "eax"									\
+	);
+
+#define DTRACE_CALL2ARGS(provider, name)							\
+	asm volatile ("subl\t$0x10,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x10,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"								\
+	);
+
+#define DTRACE_CALL3ARGS(provider, name)							\
+	asm volatile ("subl\t$0x10,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x10,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"								\
+	);
+
+#define DTRACE_CALL4ARGS(provider, name)							\
+	asm volatile ("subl\t$0x10,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x10,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL5ARGS(provider, name)							\
+	asm volatile ("subl\t$0x20,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x20,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL6ARGS(provider, name)							\
+	asm volatile ("subl\t$0x20,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+		      "movl\t0x14(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x14(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x20,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL7ARGS(provider, name)							\
+	asm volatile ("subl\t$0x20,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+		      "movl\t0x14(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x14(%%esp)"						"\n\t"	\
+	              "movl\t0x18(%0),%%eax"						"\n\t"	\
+	              "movl\t%%eax,0x18(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x20,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL8ARGS(provider, name)							\
+	asm volatile ("subl\t$0x20,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+		      "movl\t0x14(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x14(%%esp)"						"\n\t"	\
+	              "movl\t0x18(%0),%%eax"						"\n\t"	\
+		      "movl\t0x1C(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x18(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x1C(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x20,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL9ARGS(provider, name)							\
+	asm volatile ("subl\t$0x30,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+		      "movl\t0x14(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x14(%%esp)"						"\n\t"	\
+	              "movl\t0x18(%0),%%eax"						"\n\t"	\
+		      "movl\t0x1C(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x18(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x1C(%%esp)"						"\n\t"	\
+	              "movl\t0x20(%0),%%eax"						"\n\t"	\
+	              "movl\t%%eax,0x20(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x30,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#define DTRACE_CALL10ARGS(provider, name)							\
+	asm volatile ("subl\t$0x30,%%esp"						"\n\t"	\
+	              "movl\t0x0(%0),%%eax"						"\n\t"	\
+		      "movl\t0x4(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x0(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x4(%%esp)"						"\n\t"	\
+	              "movl\t0x8(%0),%%eax"						"\n\t"	\
+		      "movl\t0xC(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x8(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0xC(%%esp)"						"\n\t"	\
+	              "movl\t0x10(%0),%%eax"						"\n\t"	\
+		      "movl\t0x14(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x10(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x14(%%esp)"						"\n\t"	\
+	              "movl\t0x18(%0),%%eax"						"\n\t"	\
+		      "movl\t0x1C(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x18(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x1C(%%esp)"						"\n\t"	\
+	              "movl\t0x20(%0),%%eax"						"\n\t"	\
+		      "movl\t0x24(%0),%%edx"						"\n\t"	\
+	              "movl\t%%eax,0x20(%%esp)"						"\n\t"	\
+	              "movl\t%%edx,0x24(%%esp)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addl\t$0x30,%%esp"							\
+	              :										\
+	              : "r" (__dtrace_args)							\
+	              : "memory", "eax", "edx"	       							\
+	);
+
+#endif // __i386__
+
+#endif	/* _MACH_I386_SDT_ISA_H */
diff --git a/osfmk/mach/i386/syscall_sw.h b/osfmk/mach/i386/syscall_sw.h
index b0a4affcc..5b34b8bb3 100644
--- a/osfmk/mach/i386/syscall_sw.h
+++ b/osfmk/mach/i386/syscall_sw.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,9 +53,6 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
-/*
- */
-
 #ifdef	PRIVATE
 
 #ifndef	_MACH_I386_SYSCALL_SW_H_
@@ -69,24 +72,20 @@
 #ifndef KERNEL
 /*
  * Syscall entry macros for use in libc:
- * [Note that the nop padding is temporary during 4/4 transition.]
  */
-#define SYSENTER_PAD	nop;nop;
-#define SYSCALL_PAD	nop;nop;nop;nop;nop;
 #define UNIX_SYSCALL_TRAP	\
-	SYSCALL_PAD		\
 	int $(UNIX_INT)
 #define MACHDEP_SYSCALL_TRAP	\
-	SYSCALL_PAD		\
 	int $(MACHDEP_INT)
 
 /*
  * Macro to generate Mach call stubs in libc:
  */
+
 #define kernel_trap(trap_name,trap_number,number_args) \
 LEAF(_##trap_name,0) ;\
-	movl	$##trap_number,%eax	;\
-	int		$(MACH_INT)		;\
+	movl	$##trap_number, %eax	;\
+	call	__sysenter_trap		;\
 END(_##trap_name)
 
 #endif
@@ -134,6 +133,14 @@ END(_##trap_name)
 #define SYSCALL_CLASS_MASK	(0xFF << SYSCALL_CLASS_SHIFT)
 #define SYSCALL_NUMBER_MASK	(~SYSCALL_CLASS_MASK)
 
+#define	I386_SYSCALL_CLASS_MASK		SYSCALL_CLASS_MASK
+#define	I386_SYSCALL_ARG_BYTES_SHIFT	(16)
+#define	I386_SYSCALL_ARG_DWORDS_SHIFT	(I386_SYSCALL_ARG_BYTES_SHIFT + 2)
+#define	I386_SYSCALL_ARG_BYTES_NUM	(64) /* Must be <= sizeof(uu_arg) */
+#define	I386_SYSCALL_ARG_DWORDS_MASK	((I386_SYSCALL_ARG_BYTES_NUM >> 2) -1)
+#define	I386_SYSCALL_ARG_BYTES_MASK	(((I386_SYSCALL_ARG_BYTES_NUM -1)&~0x3) << I386_SYSCALL_ARG_BYTES_SHIFT)
+#define	I386_SYSCALL_NUMBER_MASK	(0xFFFF)
+
 #define SYSCALL_CLASS_NONE	0	/* Invalid */
 #define SYSCALL_CLASS_MACH	1	/* Mach */	
 #define SYSCALL_CLASS_UNIX	2	/* Unix/BSD */
diff --git a/osfmk/mach/i386/thread_state.h b/osfmk/mach/i386/thread_state.h
index 2eb5259d1..dba05b947 100644
--- a/osfmk/mach/i386/thread_state.h
+++ b/osfmk/mach/i386/thread_state.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/i386/thread_status.h b/osfmk/mach/i386/thread_status.h
index 499d0cb48..d8c38843f 100644
--- a/osfmk/mach/i386/thread_status.h
+++ b/osfmk/mach/i386/thread_status.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -61,13 +67,12 @@
 #ifndef	_MACH_I386_THREAD_STATUS_H_
 #define _MACH_I386_THREAD_STATUS_H_
 
+#include <mach/i386/_structs.h>
 #include <mach/message.h>
 #include <mach/i386/fp_reg.h>
 #include <mach/i386/thread_state.h>
 #include <i386/eflags.h>
 
-
-
 /*
  * the i386_xxxx form is kept for legacy purposes since these types
  * are externally known... eventually they should be deprecated.
@@ -79,8 +84,6 @@
  *	via a self-describing mechanism
  */
 
-
-
 /*
  * these are the legacy names which should be deprecated in the future
  * they are externally known which is the only reason we don't just get
@@ -90,7 +93,6 @@
 #define i386_FLOAT_STATE		2
 #define i386_EXCEPTION_STATE		3
 
-
 /*
  * THREAD_STATE_FLAVOR_LIST 0
  * 	these are the supported flavors
@@ -109,15 +111,12 @@
 #define x86_DEBUG_STATE			12
 #define THREAD_STATE_NONE		13
 
-
-
 /*
  * Largest state on this machine:
  * (be sure mach/machine/thread_state.h matches!)
  */
 #define THREAD_MACHINE_STATE_MAX	THREAD_STATE_MAX
 
-
 /*
  * VALID_THREAD_STATE_FLAVOR is a platform specific macro that when passed
  * an exception flavor will return if that is a defined flavor for that
@@ -125,116 +124,26 @@
  * exception flavors as defined above.
  */
 #define VALID_THREAD_STATE_FLAVOR(x)       \
-	((x == x86_THREAD_STATE32)	|| \
-	 (x == x86_FLOAT_STATE32)	|| \
-	 (x == x86_EXCEPTION_STATE32)	|| \
-	 (x == x86_DEBUG_STATE32)	|| \
-	 (x == x86_THREAD_STATE64)	|| \
-	 (x == x86_FLOAT_STATE64)	|| \
-	 (x == x86_EXCEPTION_STATE64)	|| \
-	 (x == x86_DEBUG_STATE64)	|| \
-	 (x == x86_THREAD_STATE)	|| \
-	 (x == x86_FLOAT_STATE)		|| \
-	 (x == x86_EXCEPTION_STATE)	|| \
-	 (x == x86_DEBUG_STATE)		|| \
-	 (x == THREAD_STATE_NONE))
-
-
+	 ((x == x86_THREAD_STATE32)	|| \
+	  (x == x86_FLOAT_STATE32)	|| \
+	  (x == x86_EXCEPTION_STATE32)	|| \
+	  (x == x86_DEBUG_STATE32)	|| \
+	  (x == x86_THREAD_STATE64)	|| \
+	  (x == x86_FLOAT_STATE64)	|| \
+	  (x == x86_EXCEPTION_STATE64)	|| \
+	  (x == x86_DEBUG_STATE64)	|| \
+	  (x == x86_THREAD_STATE)	|| \
+	  (x == x86_FLOAT_STATE)	|| \
+	  (x == x86_EXCEPTION_STATE)	|| \
+	  (x == x86_DEBUG_STATE)	|| \
+	  (x == THREAD_STATE_NONE))
 
 struct x86_state_hdr {
-    int 	flavor;
-    int		count;
+	int	flavor;
+	int	count;
 };
 typedef struct x86_state_hdr x86_state_hdr_t;
 
-
-/*
- * Main thread state consists of
- * general registers, segment registers,
- * eip and eflags.
- */
-
-struct i386_thread_state {
-    unsigned int	eax;
-    unsigned int	ebx;
-    unsigned int	ecx;
-    unsigned int	edx;
-    unsigned int	edi;
-    unsigned int	esi;
-    unsigned int	ebp;
-    unsigned int	esp;
-    unsigned int	ss;
-    unsigned int	eflags;
-    unsigned int	eip;
-    unsigned int	cs;
-    unsigned int	ds;
-    unsigned int	es;
-    unsigned int	fs;
-    unsigned int	gs;
-} ;
-
-/*
- * to be depecrated in the future
- */
-typedef struct i386_thread_state i386_thread_state_t;
-#define i386_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (i386_thread_state_t) / sizeof (int) ))
-
-
-typedef struct i386_thread_state x86_thread_state32_t;
-#define x86_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
-    ( sizeof (x86_thread_state32_t) / sizeof (int) ))
-
-
-
-
-struct x86_thread_state64 {
-    uint64_t		rax;
-    uint64_t		rbx;
-    uint64_t		rcx;
-    uint64_t		rdx;
-    uint64_t		rdi;
-    uint64_t		rsi;
-    uint64_t		rbp;
-    uint64_t		rsp;
-    uint64_t		r8;
-    uint64_t		r9;
-    uint64_t		r10;
-    uint64_t		r11;
-    uint64_t		r12;
-    uint64_t		r13;
-    uint64_t		r14;
-    uint64_t		r15;
-    uint64_t		rip;
-    uint64_t		rflags;
-    uint64_t		cs;
-    uint64_t		fs;
-    uint64_t		gs;
-} ;
-
-
-typedef struct x86_thread_state64 x86_thread_state64_t;
-#define x86_THREAD_STATE64_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (x86_thread_state64_t) / sizeof (int) ))
-
-
-
-
-struct x86_thread_state {
-    x86_state_hdr_t		tsh;
-    union {
-        x86_thread_state32_t	ts32;
-        x86_thread_state64_t	ts64;
-    } uts;
-} ;
-
-
-typedef struct x86_thread_state x86_thread_state_t;
-#define x86_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (x86_thread_state_t) / sizeof (int) ))
-
-
-
 /*
  * Default segment register values.
  */
@@ -244,263 +153,93 @@ typedef struct x86_thread_state x86_thread_state_t;
 #define KERN_CODE_SELECTOR	0x0008
 #define KERN_DATA_SELECTOR	0x0010
 
-typedef struct fp_control {
-    unsigned short		invalid	:1,
-    				denorm	:1,
-				zdiv	:1,
-				ovrfl	:1,
-				undfl	:1,
-				precis	:1,
-					:2,
-				pc	:2,
-#define FP_PREC_24B		0
-#define	FP_PREC_53B		2
-#define FP_PREC_64B		3
-				rc	:2,
-#define FP_RND_NEAR		0
-#define FP_RND_DOWN		1
-#define FP_RND_UP		2
-#define FP_CHOP			3
-				/*inf*/	:1,
-					:3;
-} fp_control_t;
 /*
- * Status word.
+ * to be deprecated in the future
  */
+typedef _STRUCT_X86_THREAD_STATE32 i386_thread_state_t;
+#define i386_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
+    ( sizeof (i386_thread_state_t) / sizeof (int) ))
 
-typedef struct fp_status {
-    unsigned short		invalid	:1,
-    				denorm	:1,
-				zdiv	:1,
-				ovrfl	:1,
-				undfl	:1,
-				precis	:1,
-				stkflt	:1,
-				errsumm	:1,
-				c0	:1,
-				c1	:1,
-				c2	:1,
-				tos	:3,
-				c3	:1,
-				busy	:1;
-} fp_status_t;
-				
-/* defn of 80bit x87 FPU or MMX register  */
-struct mmst_reg {
-	char	mmst_reg[10];
-	char	mmst_rsrv[6];
-};
-
-
-/* defn of 128 bit XMM regs */
-struct xmm_reg {
-	char		xmm_reg[16];
-};
-
-/* 
- * Floating point state.
- */
-
-#define FP_STATE_BYTES		512	/* number of chars worth of data from fpu_fcw */
-
-/* For legacy reasons we need to leave the hw_state as char bytes */
-struct i386_float_state {
-	int 			fpu_reserved[2];
-	fp_control_t		fpu_fcw;			/* x87 FPU control word */
-	fp_status_t		fpu_fsw;			/* x87 FPU status word */
-	uint8_t			fpu_ftw;			/* x87 FPU tag word */
-	uint8_t			fpu_rsrv1;			/* reserved */ 
-	uint16_t		fpu_fop;			/* x87 FPU Opcode */
-	uint32_t		fpu_ip;				/* x87 FPU Instruction Pointer offset */
-	uint16_t		fpu_cs;				/* x87 FPU Instruction Pointer Selector */
-	uint16_t		fpu_rsrv2;			/* reserved */
-	uint32_t		fpu_dp;				/* x87 FPU Instruction Operand(Data) Pointer offset */
-	uint16_t		fpu_ds;				/* x87 FPU Instruction Operand(Data) Pointer Selector */
-	uint16_t		fpu_rsrv3;			/* reserved */
-	uint32_t		fpu_mxcsr;			/* MXCSR Register state */
-	uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
-	struct mmst_reg	fpu_stmm0;		/* ST0/MM0   */
-	struct mmst_reg	fpu_stmm1;		/* ST1/MM1  */
-	struct mmst_reg	fpu_stmm2;		/* ST2/MM2  */
-	struct mmst_reg	fpu_stmm3;		/* ST3/MM3  */
-	struct mmst_reg	fpu_stmm4;		/* ST4/MM4  */
-	struct mmst_reg	fpu_stmm5;		/* ST5/MM5  */
-	struct mmst_reg	fpu_stmm6;		/* ST6/MM6  */
-	struct mmst_reg	fpu_stmm7;		/* ST7/MM7  */
-	struct xmm_reg	fpu_xmm0;		/* XMM 0  */
-	struct xmm_reg	fpu_xmm1;		/* XMM 1  */
-	struct xmm_reg	fpu_xmm2;		/* XMM 2  */
-	struct xmm_reg	fpu_xmm3;		/* XMM 3  */
-	struct xmm_reg	fpu_xmm4;		/* XMM 4  */
-	struct xmm_reg	fpu_xmm5;		/* XMM 5  */
-	struct xmm_reg	fpu_xmm6;		/* XMM 6  */
-	struct xmm_reg	fpu_xmm7;		/* XMM 7  */
-	char			fpu_rsrv4[14*16];	/* reserved */
-	int 			fpu_reserved1;
-};
-
+typedef _STRUCT_X86_THREAD_STATE32 x86_thread_state32_t;
+#define x86_THREAD_STATE32_COUNT	((mach_msg_type_number_t) \
+    ( sizeof (x86_thread_state32_t) / sizeof (int) ))
 
 /*
- * to be depecrated in the future
+ * to be deprecated in the future
  */
-typedef struct i386_float_state i386_float_state_t;
+typedef _STRUCT_X86_FLOAT_STATE32 i386_float_state_t;
 #define i386_FLOAT_STATE_COUNT ((mach_msg_type_number_t) \
 		(sizeof(i386_float_state_t)/sizeof(unsigned int)))
-	 
-typedef struct i386_float_state x86_float_state32_t;
+
+typedef _STRUCT_X86_FLOAT_STATE32 x86_float_state32_t;
 #define x86_FLOAT_STATE32_COUNT ((mach_msg_type_number_t) \
 		(sizeof(x86_float_state32_t)/sizeof(unsigned int)))
-	 
-
-struct x86_float_state64 {
-	int 			fpu_reserved[2];
-	fp_control_t		fpu_fcw;			/* x87 FPU control word */
-	fp_status_t		fpu_fsw;			/* x87 FPU status word */
-	uint8_t			fpu_ftw;			/* x87 FPU tag word */
-	uint8_t			fpu_rsrv1;			/* reserved */ 
-	uint16_t		fpu_fop;			/* x87 FPU Opcode */
-	uint32_t		fpu_ip;				/* x87 FPU Instruction Pointer offset */
-	uint16_t		fpu_cs;				/* x87 FPU Instruction Pointer Selector */
-	uint16_t		fpu_rsrv2;			/* reserved */
-	uint32_t		fpu_dp;				/* x87 FPU Instruction Operand(Data) Pointer offset */
-	uint16_t		fpu_ds;				/* x87 FPU Instruction Operand(Data) Pointer Selector */
-	uint16_t		fpu_rsrv3;			/* reserved */
-	uint32_t		fpu_mxcsr;			/* MXCSR Register state */
-	uint32_t		fpu_mxcsrmask;		/* MXCSR mask */
-	struct mmst_reg	fpu_stmm0;		/* ST0/MM0   */
-	struct mmst_reg	fpu_stmm1;		/* ST1/MM1  */
-	struct mmst_reg	fpu_stmm2;		/* ST2/MM2  */
-	struct mmst_reg	fpu_stmm3;		/* ST3/MM3  */
-	struct mmst_reg	fpu_stmm4;		/* ST4/MM4  */
-	struct mmst_reg	fpu_stmm5;		/* ST5/MM5  */
-	struct mmst_reg	fpu_stmm6;		/* ST6/MM6  */
-	struct mmst_reg	fpu_stmm7;		/* ST7/MM7  */
-	struct xmm_reg	fpu_xmm0;		/* XMM 0  */
-	struct xmm_reg	fpu_xmm1;		/* XMM 1  */
-	struct xmm_reg	fpu_xmm2;		/* XMM 2  */
-	struct xmm_reg	fpu_xmm3;		/* XMM 3  */
-	struct xmm_reg	fpu_xmm4;		/* XMM 4  */
-	struct xmm_reg	fpu_xmm5;		/* XMM 5  */
-	struct xmm_reg	fpu_xmm6;		/* XMM 6  */
-	struct xmm_reg	fpu_xmm7;		/* XMM 7  */
-	struct xmm_reg	fpu_xmm8;		/* XMM 8  */
-	struct xmm_reg	fpu_xmm9;		/* XMM 9  */
-	struct xmm_reg	fpu_xmm10;		/* XMM 10  */
-	struct xmm_reg	fpu_xmm11;		/* XMM 11 */
-	struct xmm_reg	fpu_xmm12;		/* XMM 12  */
-	struct xmm_reg	fpu_xmm13;		/* XMM 13  */
-	struct xmm_reg	fpu_xmm14;		/* XMM 14  */
-	struct xmm_reg	fpu_xmm15;		/* XMM 15  */
-	char			fpu_rsrv4[6*16];	/* reserved */
-	int 			fpu_reserved1;
-};
-
-typedef struct x86_float_state64 x86_float_state64_t;
-#define x86_FLOAT_STATE64_COUNT ((mach_msg_type_number_t) \
-		(sizeof(x86_float_state64_t)/sizeof(unsigned int)))
-		
-	 
-
-
-struct x86_float_state {
-    x86_state_hdr_t		fsh;
-    union {
-        x86_float_state32_t	fs32;
-        x86_float_state64_t	fs64;
-    } ufs;
-} ;
-
-
-typedef struct x86_float_state x86_float_state_t;
-#define x86_FLOAT_STATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (x86_float_state_t) / sizeof (int) ))
-
-
 
 /*
- * Extra state that may be
- * useful to exception handlers.
+ * to be deprecated in the future
  */
-
-struct i386_exception_state {
-    unsigned int	trapno;
-    unsigned int	err;
-    unsigned int	faultvaddr;
-};
-
-/*
- * to be depecrated in the future
- */
-typedef struct i386_exception_state i386_exception_state_t;
+typedef _STRUCT_X86_EXCEPTION_STATE32 i386_exception_state_t;
 #define i386_EXCEPTION_STATE_COUNT	((mach_msg_type_number_t) \
     ( sizeof (i386_exception_state_t) / sizeof (int) ))
 
-#define I386_EXCEPTION_STATE_COUNT i386_EXCEPTION_STATE_COUNT
-
-typedef struct i386_exception_state x86_exception_state32_t;
+typedef _STRUCT_X86_EXCEPTION_STATE32 x86_exception_state32_t;
 #define x86_EXCEPTION_STATE32_COUNT	((mach_msg_type_number_t) \
     ( sizeof (x86_exception_state32_t) / sizeof (int) ))
 
-struct x86_debug_state32 {
-	unsigned int dr0;
-	unsigned int dr1;
-	unsigned int dr2;
-	unsigned int dr3;
-	unsigned int dr4;
-	unsigned int dr5;
-	unsigned int dr6;
-	unsigned int dr7;
-};
+#define I386_EXCEPTION_STATE_COUNT i386_EXCEPTION_STATE_COUNT
 
-typedef struct x86_debug_state32 x86_debug_state32_t;
+typedef _STRUCT_X86_DEBUG_STATE32 x86_debug_state32_t;
 #define x86_DEBUG_STATE32_COUNT       ((mach_msg_type_number_t) \
 	( sizeof (x86_debug_state32_t) / sizeof (int) ))
-#define X86_DEBUG_STATE32_COUNT x86_DEBUG_STATE32_COUNT
 
+#define X86_DEBUG_STATE32_COUNT x86_DEBUG_STATE32_COUNT
 
-struct x86_exception_state64 {
-    unsigned int	trapno;
-    unsigned int	err;
-    uint64_t		faultvaddr;
-};
+typedef _STRUCT_X86_THREAD_STATE64 x86_thread_state64_t;
+#define x86_THREAD_STATE64_COUNT	((mach_msg_type_number_t) \
+    ( sizeof (x86_thread_state64_t) / sizeof (int) ))
 
-typedef struct x86_exception_state64 x86_exception_state64_t;
+typedef _STRUCT_X86_FLOAT_STATE64 x86_float_state64_t;
+#define x86_FLOAT_STATE64_COUNT ((mach_msg_type_number_t) \
+		(sizeof(x86_float_state64_t)/sizeof(unsigned int)))
+		
+typedef _STRUCT_X86_EXCEPTION_STATE64 x86_exception_state64_t;
 #define x86_EXCEPTION_STATE64_COUNT	((mach_msg_type_number_t) \
     ( sizeof (x86_exception_state64_t) / sizeof (int) ))
 
+#define X86_EXCEPTION_STATE64_COUNT x86_EXCEPTION_STATE64_COUNT
 
-struct x86_debug_state64 {
-	uint64_t dr0;
-	uint64_t dr1;
-	uint64_t dr2;
-	uint64_t dr3;
-	uint64_t dr4;
-	uint64_t dr5;
-	uint64_t dr6;
-	uint64_t dr7;
-};
-
-
-typedef struct x86_debug_state64 x86_debug_state64_t;
+typedef _STRUCT_X86_DEBUG_STATE64 x86_debug_state64_t;
 #define x86_DEBUG_STATE64_COUNT	((mach_msg_type_number_t) \
     ( sizeof (x86_debug_state64_t) / sizeof (int) ))
 
 #define X86_DEBUG_STATE64_COUNT x86_DEBUG_STATE64_COUNT
 
+/*
+ * Combined thread, float and exception states
+ */
+struct x86_thread_state {
+	x86_state_hdr_t			tsh;
+	union {
+	    x86_thread_state32_t	ts32;
+	    x86_thread_state64_t	ts64;
+	} uts;
+};
 
+struct x86_float_state {
+	x86_state_hdr_t			fsh;
+	union {
+		x86_float_state32_t	fs32;
+		x86_float_state64_t	fs64;
+	} ufs;
+};
 
 struct x86_exception_state {
-    x86_state_hdr_t		esh;
-    union {
-        x86_exception_state32_t	es32;
-        x86_exception_state64_t	es64;
-    } ues;
-} ;
-
-
-typedef struct x86_exception_state x86_exception_state_t;
-#define x86_EXCEPTION_STATE_COUNT	((mach_msg_type_number_t) \
-    ( sizeof (x86_exception_state_t) / sizeof (int) ))
+	x86_state_hdr_t			esh;
+	union {
+		x86_exception_state32_t	es32;
+		x86_exception_state64_t	es64;
+	} ues;
+};
 
 struct x86_debug_state {
 	x86_state_hdr_t			dsh;
@@ -510,7 +249,17 @@ struct x86_debug_state {
 	} uds;
 };
 
+typedef struct x86_thread_state x86_thread_state_t;
+#define x86_THREAD_STATE_COUNT	((mach_msg_type_number_t) \
+		( sizeof (x86_thread_state_t) / sizeof (int) ))
 
+typedef struct x86_float_state x86_float_state_t;
+#define x86_FLOAT_STATE_COUNT ((mach_msg_type_number_t) \
+		(sizeof(x86_float_state_t)/sizeof(unsigned int)))
+
+typedef struct x86_exception_state x86_exception_state_t;
+#define x86_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
+		(sizeof(x86_exception_state_t)/sizeof(unsigned int)))
 
 typedef struct x86_debug_state x86_debug_state_t;
 #define x86_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
@@ -523,15 +272,6 @@ typedef struct x86_debug_state x86_debug_state_t;
 #define MACHINE_THREAD_STATE		x86_THREAD_STATE
 #define MACHINE_THREAD_STATE_COUNT	x86_THREAD_STATE_COUNT
 
-
-#ifdef XNU_KERNEL_PRIVATE
-
-#define x86_SAVED_STATE32		THREAD_STATE_NONE + 1
-#define x86_SAVED_STATE64		THREAD_STATE_NONE + 2
-
-#define OLD_i386_THREAD_STATE		-1
-
-
 /*
  * when reloading the segment registers on
  * a return out of the kernel, we may take
@@ -546,13 +286,17 @@ typedef struct x86_debug_state x86_debug_state_t;
  * enough stack
  */
 struct x86_seg_load_fault32 {
-	unsigned int	trapno;
-	unsigned int	err;
-	unsigned int	eip;
-	unsigned int	cs;
-	unsigned int	efl;
+	unsigned int    trapno;
+	unsigned int    err;
+	unsigned int    eip;
+	unsigned int    cs;
+	unsigned int    efl;
 };
 
+#ifdef XNU_KERNEL_PRIVATE
+
+#define x86_SAVED_STATE32		THREAD_STATE_NONE + 1
+#define x86_SAVED_STATE64		THREAD_STATE_NONE + 2
 
 /*
  * Subset of saved state stored by processor on kernel-to-kernel
@@ -584,7 +328,6 @@ struct x86_saved_state32_from_kernel {
  * state flavor is most efficient for exception RPC's to kernel-loaded
  * servers, because copying can be avoided:
  */
- 
 struct x86_saved_state32 {
 	unsigned int	gs;
 	unsigned int	fs;
@@ -618,7 +361,7 @@ struct x86_saved_state32_tagged {
 typedef struct x86_saved_state32_tagged x86_saved_state32_tagged_t;
 
 struct x86_sframe32 {
-        /*
+/*
 	 * in case we throw a fault reloading
 	 * segment registers on a return out of
 	 * the kernel... the 'slf' state is only kept
@@ -632,8 +375,6 @@ struct x86_sframe32 {
 };
 typedef struct x86_sframe32 x86_sframe32_t;
 
-
-
 /*
  * This is the state pushed onto the 64-bit interrupt stack
  * on any exception/trap/interrupt.
@@ -662,7 +403,6 @@ struct x86_saved_state_compat32 {
 };
 typedef struct x86_saved_state_compat32 x86_saved_state_compat32_t;
 
-
 struct x86_sframe_compat32 {
         struct x86_64_intr_stack_frame  slf;
 	uint32_t			pad_for_16byte_alignment[2];
@@ -671,15 +411,12 @@ struct x86_sframe_compat32 {
 };
 typedef struct x86_sframe_compat32 x86_sframe_compat32_t;
 
-
-
 /*
  * thread state format for task running in 64bit long mode
  * in long mode, the same hardware frame is always pushed regardless
  * of whether there was a change in privlege level... therefore, there
  * is no need for an x86_saved_state64_from_kernel variant
  */
-
 struct x86_saved_state64 {
         /*
 	 * saved state organized to reflect the
@@ -737,6 +474,7 @@ struct x86_sframe64 {
 typedef struct x86_sframe64 x86_sframe64_t;
 
 extern uint32_t get_eflags_exportmask(void);
+
 /*
  * Unified, tagged saved state:
  */
diff --git a/osfmk/mach/i386/vm_param.h b/osfmk/mach/i386/vm_param.h
index 4d1a7767f..f16790f1b 100644
--- a/osfmk/mach/i386/vm_param.h
+++ b/osfmk/mach/i386/vm_param.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -133,7 +139,6 @@
 #define VM_MAX_USER_PAGE_ADDRESS ((user_addr_t)0x00007FFFFFFFF000ULL)
 
 
-
 /* system-wide values */
 #define MACH_VM_MIN_ADDRESS		((mach_vm_offset_t) 0)
 #define MACH_VM_MAX_ADDRESS		((mach_vm_offset_t) VM_MAX_PAGE_ADDRESS)
@@ -144,8 +149,6 @@
 #define VM_MAX_ADDRESS		((vm_offset_t) 0xFFE00000)
 
 
-
-
 #ifdef	KERNEL_PRIVATE 
 
 /* Kernel-wide values */
@@ -158,7 +161,7 @@
  * We can't let VM allocate memory from there.
  */
 
-#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xFE7FFFFF)
+#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xFE7FFFFFU)
 #define KERNEL_STACK_SIZE		(I386_PGBYTES*4)
 
 #define VM_MAP_MIN_ADDRESS	MACH_VM_MIN_ADDRESS
@@ -179,7 +182,7 @@
 #define VM_MIN_KERNEL_LOADED_ADDRESS	((vm_offset_t) 0x00000000U)
 #define VM_MAX_KERNEL_LOADED_ADDRESS	((vm_offset_t) 0x1fffffffU)
 
-#define NCOPY_WINDOWS	4
+#define NCOPY_WINDOWS 4
 
 /*
  *	Conversion between 80386 pages and VM pages
@@ -193,8 +196,7 @@
 	MACRO_BEGIN					\
 	pmap_t __pmap = (pmap);				\
 	vm_page_t __page = (page);			\
-	vm_prot_t __prot__ =				\
-		(protection) & ~(page)->page_lock;	\
+	vm_prot_t __prot__ =  (protection);    		\
 							\
 	if (__pmap == kernel_pmap) {			\
 		__prot__ |= VM_PROT_WRITE;		\
diff --git a/osfmk/mach/i386/vm_types.h b/osfmk/mach/i386/vm_types.h
index afdf682c9..49c23c831 100644
--- a/osfmk/mach/i386/vm_types.h
+++ b/osfmk/mach/i386/vm_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -118,16 +124,9 @@ typedef uint64_t		mach_vm_address_t;
 typedef uint64_t		mach_vm_offset_t;
 typedef uint64_t		mach_vm_size_t;
 
-/* LP64todo - convert these over for good */
-#if 1 
 typedef uint64_t		vm_map_offset_t;
 typedef uint64_t		vm_map_address_t;
 typedef uint64_t		vm_map_size_t;
-#else
-typedef uint32_t		vm_map_offset_t;
-typedef uint32_t		vm_map_address_t;
-typedef uint32_t		vm_map_size_t;
-#endif
 
 #ifdef	MACH_KERNEL_PRIVATE
 
diff --git a/osfmk/mach/kern_return.h b/osfmk/mach/kern_return.h
index f2b1fcc70..5e7502be9 100644
--- a/osfmk/mach/kern_return.h
+++ b/osfmk/mach/kern_return.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/kmod.h b/osfmk/mach/kmod.h
index f63827898..4c6f34c05 100644
--- a/osfmk/mach/kmod.h
+++ b/osfmk/mach/kmod.h
@@ -1,23 +1,35 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
  */
 
 #ifndef	_MACH_KMOD_H_
@@ -27,12 +39,24 @@
 
 #include <sys/cdefs.h>
 
+/***********************************************************************
+* kmod_control() commands. 1-5 are long-established. 6-8 are new in
+* Leopard and used to reliably get and verify symbol information needed
+* to link kexts against the running kernel, or to disable kmod loading
+* if such symbol information cannot be found.
+***********************************************************************/
 #define KMOD_CNTL_START		1	// call kmod's start routine
 #define KMOD_CNTL_STOP		2	// call kmod's stop routine
 #define KMOD_CNTL_RETAIN	3	// increase a kmod's reference count
 #define KMOD_CNTL_RELEASE	4	// decrease a kmod's reference count
 #define KMOD_CNTL_GET_CMD	5	// get kmod load cmd from kernel
 
+#define KMOD_CNTL_GET_KERNEL_SYMBOLS  6  // get symfile as data buffer
+#define KMOD_CNTL_FREE_LINKEDIT_DATA  7  // refuse to create new kmods
+#define KMOD_CNTL_GET_KERNEL_UUID     8  // LC_UUID load command payload
+#define KMOD_CNTL_GET_UUID            8  // LC_UUID load command payload
+#define KMOD_CNTL_DISABLE_LOAD        9  // refuse to create new kmods
+
 #define KMOD_PACK_IDS(from, to)	(((unsigned long)from << 16) | (unsigned long)to)
 #define KMOD_UNPACK_FROM_ID(i)	((unsigned long)i >> 16)
 #define KMOD_UNPACK_TO_ID(i)	((unsigned long)i & 0xffff)
@@ -141,6 +165,8 @@ typedef struct kmod_generic_cmd {
 
 extern kmod_info_t *kmod_lookupbyname(const char * name);
 extern kmod_info_t *kmod_lookupbyid(kmod_t id);
+extern kmod_info_t *kmod_lookupbyaddress(vm_address_t address);
+extern int kmod_lookupidbyaddress_locked(vm_address_t address);
 
 extern kmod_info_t *kmod_lookupbyname_locked(const char * name);
 extern kmod_info_t *kmod_lookupbyid_locked(kmod_t id);
@@ -148,7 +174,7 @@ extern kmod_start_func_t kmod_default_start;
 extern kmod_stop_func_t  kmod_default_stop;
 
 __BEGIN_DECLS
-extern void kmod_init(void);
+extern void kmod_init(void) __attribute__((section("__TEXT, initcode")));
 
 extern kern_return_t kmod_create_fake(const char *name, const char *version);
 extern kern_return_t kmod_create_fake_with_address(const char *name, const char *version, 
diff --git a/osfmk/mach/ledger.defs b/osfmk/mach/ledger.defs
index ec1b94f76..76367a990 100644
--- a/osfmk/mach/ledger.defs
+++ b/osfmk/mach/ledger.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,3 +76,4 @@ routine ledger_read(
 	out	balance			: ledger_item_t;
 	out	limit			: ledger_item_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/lock_set.defs b/osfmk/mach/lock_set.defs
index da19ca388..c7ee5845b 100644
--- a/osfmk/mach/lock_set.defs
+++ b/osfmk/mach/lock_set.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,3 +76,4 @@ routine lock_handoff_accept(
 		lock_set	: lock_set_t;
 		lock_id		: int);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach.h b/osfmk/mach/mach.h
index c0e4100ec..df61ab89a 100644
--- a/osfmk/mach/mach.h
+++ b/osfmk/mach/mach.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
diff --git a/osfmk/mach/mach_exc.defs b/osfmk/mach/mach_exc.defs
new file mode 100644
index 000000000..00fae2853
--- /dev/null
+++ b/osfmk/mach/mach_exc.defs
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+/* 
+ * Mach Operating System
+ * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
+ * All Rights Reserved.
+ * 
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ * 
+ * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
+ * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
+ * 
+ * Carnegie Mellon requests users of this software to return to
+ * 
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ * 
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+/*
+ */
+/*
+ * Abstract:
+ *	MiG definitions file for Mach exception interface.
+ */
+
+subsystem
+#if	KERNEL_USER
+	  KernelUser
+#endif
+		     mach_exc 2405;
+
+#include <mach/std_types.defs>
+#include <mach/mach_types.defs>
+
+ServerPrefix catch_;
+
+type mach_exception_data_t	= array[*:2] of int64_t;
+type exception_type_t		= int;
+
+routine		mach_exception_raise(
+#if	KERNEL_USER
+			exception_port	: mach_port_move_send_t;
+			thread		: mach_port_move_send_t;
+			task		: mach_port_move_send_t;
+#else	/* KERNEL_USER */
+			exception_port	: mach_port_t;
+			thread		: mach_port_t;
+			task		: mach_port_t;
+#endif	/* KERNEL_USER */
+			exception	: exception_type_t;
+			code		: mach_exception_data_t
+			);
+
+routine		mach_exception_raise_state(
+#if	KERNEL_USER
+			exception_port	: mach_port_move_send_t;
+#else	/* KERNEL_USER */
+			exception_port	: mach_port_t;
+#endif	/* KERNEL_USER */
+			exception	: exception_type_t;
+			code		: mach_exception_data_t, const;
+		  inout flavor		: int;
+			old_state	: thread_state_t, const;
+		    out new_state	: thread_state_t);
+
+routine		mach_exception_raise_state_identity(
+#if	KERNEL_USER
+			exception_port  : mach_port_move_send_t;
+			thread		: mach_port_move_send_t;
+			task		: mach_port_move_send_t;
+#else	/* KERNEL_USER */
+			exception_port  : mach_port_t;
+			thread          : mach_port_t;
+			task            : mach_port_t;
+#endif	/* KERNEL_USER */
+			exception       : exception_type_t;
+			code            : mach_exception_data_t;
+		  inout flavor          : int;
+			old_state       : thread_state_t;
+		    out new_state       : thread_state_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_host.defs b/osfmk/mach/mach_host.defs
index fb062cde1..6d3e15b45 100644
--- a/osfmk/mach/mach_host.defs
+++ b/osfmk/mach/mach_host.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -59,7 +65,6 @@
  */
 
 #ifdef	MACH_KERNEL
-#include <mach_prof.h>
 #include <advisory_pageout.h>
 #endif	/* MACH_KERNEL */
 
@@ -100,8 +105,14 @@ routine	host_kernel_version(
 
 /*
  *      Get host page size
+ *	(compatibility for running old libraries on new kernels -
+ *	host_page_size() is now a library routine based on constants)
  */
+#if KERNEL
 routine host_page_size(
+#else
+routine _host_page_size(
+#endif
 		host		: host_t;
 	out	out_page_size	: vm_size_t);
 
@@ -244,3 +255,4 @@ routine host_lockgroup_info(
 	out	lockgroup_info	: lockgroup_info_array_t,
 					Dealloc);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_interface.h b/osfmk/mach/mach_interface.h
index 91b4b2bac..576f4119d 100644
--- a/osfmk/mach/mach_interface.h
+++ b/osfmk/mach/mach_interface.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_MACH_INTERFACE_H_
@@ -31,6 +37,7 @@
 #include <mach/host_security.h>
 #include <mach/ledger.h>
 #include <mach/lock_set.h>
+#include <mach/mach_exc_server.h>
 #include <mach/mach_host.h>
 #include <mach/mach_port.h>
 #include <mach/notify_server.h>
diff --git a/osfmk/mach/mach_norma.defs b/osfmk/mach/mach_norma.defs
index 8e5ff7f09..816599fdd 100644
--- a/osfmk/mach/mach_norma.defs
+++ b/osfmk/mach/mach_norma.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -251,3 +257,5 @@ routine norma_port_location_hint(
 		task		: task_t;
 		port		: mach_port_t;
 	out	node		: int);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_notify.defs b/osfmk/mach/mach_notify.defs
index 597b4fad9..b4d03a6c6 100644
--- a/osfmk/mach/mach_notify.defs
+++ b/osfmk/mach/mach_notify.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * The mach/notify.h file is hand-crafted.
@@ -27,4 +33,6 @@
  * those routines a new home (mach/mach_notify.h and mach_notify_user.c)
  * until the real notify.h becomes "MIG-safe."
  */
-#include <mach/notify.defs>
\ No newline at end of file
+#include <mach/notify.defs>
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_param.h b/osfmk/mach/mach_param.h
index c68789a1e..e4ead673e 100644
--- a/osfmk/mach/mach_param.h
+++ b/osfmk/mach/mach_param.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/mach_port.defs b/osfmk/mach/mach_port.defs
index 5da6fc755..88156a847 100644
--- a/osfmk/mach/mach_port.defs
+++ b/osfmk/mach/mach_port.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -218,9 +224,10 @@ routine mach_port_get_set_status(
 
 /*
  *	Puts the member port (the task must have receive rights)
- *	into the after port set.  (Or removes it from any port set
- *	if after is MACH_PORT_NULL.)  If the port is already in
- *	a set, does an atomic move.
+ *	into the after port set.  If the port is already a member
+ *	of any set(s), it is atomically removed from those sets as
+ *	part of this operation.  (If after is MACH_PORT_NULL, the
+ *	port is still removed from all current sets).
  */
 
 routine mach_port_move_member(
@@ -379,10 +386,8 @@ routine	mach_port_get_srights(
 routine mach_port_space_info(
 		task		: ipc_space_t;
 	out	space_info	: ipc_info_space_t;
-	out	table_info	: ipc_info_name_array_t,
-					Dealloc;
-	out	tree_info	: ipc_info_tree_name_array_t,
-					Dealloc);
+	out	table_info	: ipc_info_name_array_t;
+	out	tree_info	: ipc_info_tree_name_array_t);
 
 /*
  *	Returns information about the dead-name requests
@@ -448,3 +453,4 @@ routine mach_port_extract_member(
 		name		: mach_port_name_t;
 		pset		: mach_port_name_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_syscalls.h b/osfmk/mach/mach_syscalls.h
index dcf9dc2da..0e04e4f31 100644
--- a/osfmk/mach/mach_syscalls.h
+++ b/osfmk/mach/mach_syscalls.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/mach_time.h b/osfmk/mach/mach_time.h
index e1bb2d17c..de93d518d 100644
--- a/osfmk/mach/mach_time.h
+++ b/osfmk/mach/mach_time.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2001-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_MACH_MACH_TIME_H_
diff --git a/osfmk/mach/mach_traps.h b/osfmk/mach/mach_traps.h
index 17514527b..87c4df1f6 100644
--- a/osfmk/mach/mach_traps.h
+++ b/osfmk/mach/mach_traps.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -224,6 +230,7 @@ extern kern_return_t map_fd(
 /* Syscall data translations routines */
 #define	PAD_(t)	(sizeof(uint64_t) <= sizeof(t) \
  		? 0 : sizeof(uint64_t) - sizeof(t))
+#define PAD_ARG_8
 
 #if BYTE_ORDER == LITTLE_ENDIAN
 #define	PADL_(t)	0
@@ -325,6 +332,7 @@ struct mach_msg_overwrite_trap_args {
 	PAD_ARG_(mach_port_name_t, rcv_name);
 	PAD_ARG_(mach_msg_timeout_t, timeout);
 	PAD_ARG_(mach_port_name_t, notify);
+	PAD_ARG_8
 	PAD_ARG_(mach_vm_address_t, rcv_msg);  /* Unused on mach_msg_trap */
 };
 extern mach_msg_return_t mach_msg_trap(
@@ -536,6 +544,7 @@ struct iokit_user_client_trap_args {
 	PAD_ARG_(void *, p3);
 	PAD_ARG_(void *, p4);
 	PAD_ARG_(void *, p5);
+    	PAD_ARG_8
 	PAD_ARG_(void *, p6);
 };
 kern_return_t iokit_user_client_trap(
@@ -545,6 +554,7 @@ kern_return_t iokit_user_client_trap(
 #undef PADL_
 #undef PADR_
 #undef PAD_ARG_
+#undef PAD_ARG_8
 
 #endif	/* XNU_KERNEL_PRIVATE */
 
diff --git a/osfmk/mach/mach_types.defs b/osfmk/mach/mach_types.defs
index dffdb8e2a..7f7dfc470 100644
--- a/osfmk/mach/mach_types.defs
+++ b/osfmk/mach/mach_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -62,6 +74,7 @@
 type memory_object_offset_t 	= uint64_t;
 type memory_object_size_t 	= uint64_t;
 type memory_object_cluster_size_t = uint32_t;
+type memory_object_fault_info_t = array[16] of integer_t;
 
 #ifdef KERNEL_PRIVATE
 
@@ -171,7 +184,7 @@ type vm_inherit_t = int;
 type vm_purgable_t = int;
 type xxx_vm_statistics_data_t = struct[13] of integer_t;
 type vm_behavior_t = int;
-type vm_statistics_data_t = struct[14] of integer_t;
+type vm_statistics_data_t = struct[15] of integer_t;
 type vm_machine_attribute_t = int;
 type vm_machine_attribute_val_t = int;
 type vm_sync_t = int;
@@ -283,12 +296,12 @@ type host_security_t = mach_port_t
 		 * host_sched_info_t (2 ints)
 		 * kernel_resource_sizes_t (5 ints)
 		 * host_load_info_t (6 ints)
-		 * vm_statistics_t (14 ints)
+		 * vm_statistics_t (15 ints)
 		 * If other host_info flavors are added, this definition may
 		 * need to be changed. (See mach/{host_info,vm_statistics}.h)
 		 */
 type host_flavor_t		= int;
-type host_info_t 		= array[*:14] of integer_t;
+type host_info_t 		= array[*:15] of integer_t;
 
 type processor_t = mach_port_t
 #if	KERNEL_SERVER
@@ -381,6 +394,8 @@ type ledger_item_t		= integer_t;
 type security_token_t		= struct[2] of uint32_t;
 type audit_token_t		= struct[8] of uint32_t;
 
+type msg_labels_t = mach_port_t;
+
 		/* memory_object_info_t: variable-size inline array:
 		 * memory_object_attr_info_t (5 ints)
 		 * XXX actually it's 6 ints temporarily (object_ready!)
@@ -478,3 +493,5 @@ import <mach/mig.h>;
 import <mach/mach_types.h>;
 
 #endif	/* _MACH_MACH_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/mach_types.h b/osfmk/mach/mach_types.h
index e75e24303..5f9ddf14f 100644
--- a/osfmk/mach/mach_types.h
+++ b/osfmk/mach/mach_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -49,6 +55,12 @@
  */
 /*
  */
+/*
+ * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  *	File:	mach/mach_types.h
  *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
@@ -253,6 +265,7 @@ typedef natural_t 		ledger_item_t;
 typedef mach_vm_offset_t	*emulation_vector_t;
 typedef char			*user_subsystem_t;
 
+typedef char			*labelstr_t;
 /*
  *	Backwards compatibility, for those programs written
  *	before mach/{std,mach}_types.{defs,h} were set up.
diff --git a/osfmk/mach/mach_vm.defs b/osfmk/mach/mach_vm.defs
index 9e39042d7..dda18fc4c 100644
--- a/osfmk/mach/mach_vm.defs
+++ b/osfmk/mach/mach_vm.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -395,8 +401,12 @@ routine vm_region_64(
  *	THIS INTERFACE IS STILL EVOLVING.
  */
 #if !defined(_MACH_VM_PUBLISH_AS_LOCAL_)
+#if !defined(__LP64__)
 routine _mach_make_memory_entry(
 #else
+routine mach_make_memory_entry(
+#endif
+#else
 routine mach_make_memory_entry_64(
 #endif
 		target_task	:vm_map_t;
@@ -406,6 +416,23 @@ routine mach_make_memory_entry_64(
 	out	object_handle	:mem_entry_name_port_move_send_t;
 		parent_handle	:mem_entry_name_port_t);
 
+/*
+ *	Control behavior and investigate state of a "purgable" object in
+ *	the virtual address space of the target task.  A purgable object is
+ *	created via a call to mach_vm_allocate() with VM_FLAGS_PURGABLE
+ *	specified.  See the routine implementation for a complete
+ *	definition of the routine.
+ */
+#if !defined(_MACH_VM_PUBLISH_AS_LOCAL_)
+routine mach_vm_purgable_control(
+#else
+routine vm_purgable_control(
+#endif
+		target_task	: vm_map_t;
+		address		: mach_vm_address_t;
+		control		: vm_purgable_t;
+	inout	state		: int);
+
 
 /****************************** Legacy section ***************************/
 /*  The following definitions are exist to provide compatibility with    */
@@ -439,3 +466,5 @@ routine mach_make_memory_entry_64(
  *	mach_make_memory_entry() -
  *		use mach_vm_make_memory_entry() or mach_make_memory_entry_64()
  */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/machine.h b/osfmk/mach/machine.h
index 7c88e5836..8a73ba9d8 100644
--- a/osfmk/mach/machine.h
+++ b/osfmk/mach/machine.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 
  * Mach Operating System
@@ -87,12 +93,11 @@ struct machine_info {
 	integer_t	major_version;		/* kernel major version id */
 	integer_t	minor_version;		/* kernel minor version id */
 	integer_t	max_cpus;			/* max number of CPUs possible */
-	integer_t	avail_cpus;			/* number of CPUs now available */
 	uint32_t	memory_size;		/* size of memory in bytes, capped at 2 GB */
 	uint64_t	max_mem;			/* actual size of physical memory */
-	integer_t	physical_cpu;		/* number of physical CPUs now available */
+	uint32_t	physical_cpu;		/* number of physical CPUs now available */
 	integer_t	physical_cpu_max;	/* max number of physical CPUs possible */
-	integer_t	logical_cpu;		/* number of logical cpu now available */
+	uint32_t	logical_cpu;		/* number of logical cpu now available */
 	integer_t	logical_cpu_max;	/* max number of physical CPUs possible */
 };
 
@@ -142,7 +147,7 @@ __END_DECLS
 /* skip 			((cpu_type_t) 9)	*/
 #define CPU_TYPE_MC98000	((cpu_type_t) 10)
 #define CPU_TYPE_HPPA           ((cpu_type_t) 11)
-/* skip CPU_TYPE_ARM		((cpu_type_t) 12)	*/
+#define CPU_TYPE_ARM		((cpu_type_t) 12)
 #define CPU_TYPE_MC88000	((cpu_type_t) 13)
 #define CPU_TYPE_SPARC		((cpu_type_t) 14)
 #define CPU_TYPE_I860		((cpu_type_t) 15)
@@ -157,6 +162,13 @@ __END_DECLS
  *	regardless of where is it compiled).
  */
 
+/*
+ * Capability bits used in the definition of cpu_subtype.
+ */
+#define CPU_SUBTYPE_MASK	0xff000000	/* mask for feature flags */
+#define CPU_SUBTYPE_LIB64	0x80000000	/* 64 bit libraries */
+
+
 /*
  *	Object files that are hand-crafted to run on any
  *	implementation of an architecture are tagged with
@@ -328,8 +340,17 @@ __END_DECLS
 #define CPU_SUBTYPE_POWERPC_970		((cpu_subtype_t) 100)
 
 /*
- *      CPU families (sysctl hw.cpufamily)
+ *	ARM subtypes
+ */
+#define CPU_SUBTYPE_ARM_ALL             ((cpu_subtype_t) 0)
+#define CPU_SUBTYPE_ARM_V4T             ((cpu_subtype_t) 5)
+#define CPU_SUBTYPE_ARM_V6              ((cpu_subtype_t) 6)
+
+/*
+ *	CPU families (sysctl hw.cpufamily)
  *
+ * These are meant to identify the CPU's marketing name - an
+ * application can map these to (possibly) localized strings.
  * NB: the encodings of the CPU families are intentionally arbitrary.
  * There is no ordering, and you should never try to deduce whether
  * or not some feature is available based on the family.
@@ -340,7 +361,20 @@ __END_DECLS
 #define CPUFAMILY_POWERPC_G3 0xcee41549
 #define CPUFAMILY_POWERPC_G4 0x77c184ae
 #define CPUFAMILY_POWERPC_G5 0xed76d8aa
-#define CPUFAMILY_INTEL_6_14 0x73d67300  /* Intel Core Solo and Intel Core Duo (32-bit Pentium-M with SSE3) */
-#define CPUFAMILY_INTEL_6_15 0x426f69ef  /* Intel Core 2 */
+#define CPUFAMILY_INTEL_6_13 0xaa33392b
+#define CPUFAMILY_INTEL_6_14 0x73d67300  /* "Intel Core Solo" and "Intel Core Duo" (32-bit Pentium-M with SSE3) */
+#define CPUFAMILY_INTEL_6_15 0x426f69ef  /* "Intel Core 2 Duo" */
+#define CPUFAMILY_INTEL_6_23 0x78ea4fbc  /* Penryn */
+#define CPUFAMILY_INTEL_6_26 0x6b5a4cd2  /* Nehalem */
+#define CPUFAMILY_ARM_9      0xe73283ae
+#define CPUFAMILY_ARM_11     0x8ff620d8
+
+#define CPUFAMILY_INTEL_YONAH	CPUFAMILY_INTEL_6_14
+#define CPUFAMILY_INTEL_MEROM	CPUFAMILY_INTEL_6_15
+#define CPUFAMILY_INTEL_PENRYN	CPUFAMILY_INTEL_6_23
+#define CPUFAMILY_INTEL_NEHALEM	CPUFAMILY_INTEL_6_26
+
+#define CPUFAMILY_INTEL_CORE	CPUFAMILY_INTEL_6_14
+#define CPUFAMILY_INTEL_CORE2	CPUFAMILY_INTEL_6_15
 
 #endif	/* _MACH_MACHINE_H_ */
diff --git a/osfmk/mach/machine/Makefile b/osfmk/mach/machine/Makefile
index 830dd11e0..6ff836179 100644
--- a/osfmk/mach/machine/Makefile
+++ b/osfmk/mach/machine/Makefile
@@ -12,7 +12,7 @@ DATAFILES = \
 	asm.h boolean.h exception.h kern_return.h ndr_def.h rpc.h \
 	processor_info.h thread_state.h thread_status.h \
 	vm_param.h vm_types.h machine_types.defs \
-	syscall_sw.h
+	syscall_sw.h sdt.h sdt_isa.h
 
 INSTALL_MI_LIST = ${DATAFILES}
 
diff --git a/osfmk/mach/machine/asm.h b/osfmk/mach/machine/asm.h
index 76c330464..59e1da299 100644
--- a/osfmk/mach/machine/asm.h
+++ b/osfmk/mach/machine/asm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_ASM_H
@@ -27,6 +33,8 @@
 #include "mach/ppc/asm.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/asm.h"
+#elif defined (__arm__)
+#include "mach/arm/asm.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/boolean.h b/osfmk/mach/machine/boolean.h
index 3d8aba6a1..1bec00d4b 100644
--- a/osfmk/mach/machine/boolean.h
+++ b/osfmk/mach/machine/boolean.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_BOOLEAN_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/boolean.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/boolean.h"
+#elif defined (__arm__)
+#include "mach/arm/boolean.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/exception.h b/osfmk/mach/machine/exception.h
index a1f3270e1..5ce83f254 100644
--- a/osfmk/mach/machine/exception.h
+++ b/osfmk/mach/machine/exception.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_EXCEPTION_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/exception.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/exception.h"
+#elif defined (__arm__)
+#include "mach/arm/exception.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/kern_return.h b/osfmk/mach/machine/kern_return.h
index f5db5df57..df3739699 100644
--- a/osfmk/mach/machine/kern_return.h
+++ b/osfmk/mach/machine/kern_return.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_KERN_RETURN_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/kern_return.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/kern_return.h"
+#elif defined (__arm__)
+#include "mach/arm/kern_return.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/machine_types.defs b/osfmk/mach/machine/machine_types.defs
index b75229ee8..418d16bcc 100644
--- a/osfmk/mach/machine/machine_types.defs
+++ b/osfmk/mach/machine/machine_types.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_MACHINE_TYPES_DEFS
@@ -27,8 +33,12 @@
 #include "mach/ppc/machine_types.defs"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/machine_types.defs"
+#elif defined (__arm__)
+#include "mach/arm/machine_types.defs"
 #else
 #error architecture not supported
 #endif
 
 #endif /* _MACH_MACHINE_THREAD_STATUS_H_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/machine/ndr_def.h b/osfmk/mach/machine/ndr_def.h
index 160c75027..a7b5a9bdd 100644
--- a/osfmk/mach/machine/ndr_def.h
+++ b/osfmk/mach/machine/ndr_def.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_NDR_DEF_H
@@ -27,6 +33,8 @@
 #include "mach/ppc/ndr_def.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/ndr_def.h"
+#elif defined (__arm__)
+#include "mach/arm/ndr_def.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/processor_info.h b/osfmk/mach/machine/processor_info.h
index 0ca7074e3..ca9636795 100644
--- a/osfmk/mach/machine/processor_info.h
+++ b/osfmk/mach/machine/processor_info.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_PROCESSOR_INFO_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/processor_info.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/processor_info.h"
+#elif defined (__arm__)
+#include "mach/arm/processor_info.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/rpc.h b/osfmk/mach/machine/rpc.h
index 0bdf02379..b969bd4d3 100644
--- a/osfmk/mach/machine/rpc.h
+++ b/osfmk/mach/machine/rpc.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_RPC_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/rpc.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/rpc.h"
+#elif defined (__arm__)
+#include "mach/arm/rpc.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/sdt.h b/osfmk/mach/machine/sdt.h
new file mode 100644
index 000000000..6041fc3f5
--- /dev/null
+++ b/osfmk/mach/machine/sdt.h
@@ -0,0 +1,257 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ */
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _MACH_MACHINE_SYS_SDT_H
+#define	_MACH_MACHINE_SYS_SDT_H
+
+#include <mach/machine/sdt_isa.h>
+
+/*
+ * The following macros are used to create static probes. The argument types
+ * should be no greater than uintptr_t in size each. The behavior of larger
+ * types is undefined.
+ */
+
+#define DTRACE_PROBE(provider, name) {								\
+	DTRACE_CALL0ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE1(provider, name, arg0) {							\
+	uintptr_t __dtrace_args[ARG1_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	DTRACE_CALL1ARG(provider, name)								\
+}
+
+#define DTRACE_PROBE2(provider, name, arg0, arg1) {						\
+	uintptr_t __dtrace_args[ARGS2_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	DTRACE_CALL2ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE3(provider, name, arg0, arg1, arg2) {					\
+	uintptr_t __dtrace_args[ARGS3_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	DTRACE_CALL3ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE4(provider, name, arg0, arg1, arg2, arg3) {					\
+	uintptr_t __dtrace_args[ARGS4_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	DTRACE_CALL4ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE5(provider, name, arg0, arg1, arg2, arg3, arg4) {				\
+	uintptr_t __dtrace_args[ARGS5_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	__dtrace_args[4] = (uintptr_t)arg4;								\
+	DTRACE_CALL5ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE6(provider, name, arg0, arg1, arg2, arg3, arg4, arg5) {			\
+	uintptr_t __dtrace_args[ARGS6_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	__dtrace_args[4] = (uintptr_t)arg4;								\
+	__dtrace_args[5] = (uintptr_t)arg5;								\
+	DTRACE_CALL6ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE7(provider, name, arg0, arg1, arg2, arg3, arg4, arg5, arg6) {		\
+	uintptr_t __dtrace_args[ARGS7_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	__dtrace_args[4] = (uintptr_t)arg4;								\
+	__dtrace_args[5] = (uintptr_t)arg5;								\
+	__dtrace_args[6] = (uintptr_t)arg6;								\
+	DTRACE_CALL7ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE8(provider, name, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) {		\
+	uintptr_t __dtrace_args[ARGS8_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	__dtrace_args[4] = (uintptr_t)arg4;								\
+	__dtrace_args[5] = (uintptr_t)arg5;								\
+	__dtrace_args[6] = (uintptr_t)arg6;								\
+	__dtrace_args[7] = (uintptr_t)arg7;								\
+	DTRACE_CALL8ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE9(provider, name, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) {	\
+	uintptr_t __dtrace_args[ARGS9_EXTENT] __attribute__ ((aligned (16)));			\
+	__dtrace_args[0] = (uintptr_t)arg0;								\
+	__dtrace_args[1] = (uintptr_t)arg1;								\
+	__dtrace_args[2] = (uintptr_t)arg2;								\
+	__dtrace_args[3] = (uintptr_t)arg3;								\
+	__dtrace_args[4] = (uintptr_t)arg4;								\
+	__dtrace_args[5] = (uintptr_t)arg5;								\
+	__dtrace_args[6] = (uintptr_t)arg6;								\
+	__dtrace_args[7] = (uintptr_t)arg7;								\
+	__dtrace_args[8] = (uintptr_t)arg8;								\
+	DTRACE_CALL9ARGS(provider, name)							\
+}
+
+#define DTRACE_PROBE10(provider, name, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) {	\
+	uintptr_t __dtrace_args[ARGS10_EXTENT] __attribute__ ((aligned (16)));				\
+	__dtrace_args[0] = (uintptr_t)arg0;									\
+	__dtrace_args[1] = (uintptr_t)arg1;									\
+	__dtrace_args[2] = (uintptr_t)arg2;									\
+	__dtrace_args[3] = (uintptr_t)arg3;									\
+	__dtrace_args[4] = (uintptr_t)arg4;									\
+	__dtrace_args[5] = (uintptr_t)arg5;									\
+	__dtrace_args[6] = (uintptr_t)arg6;									\
+	__dtrace_args[7] = (uintptr_t)arg7;									\
+	__dtrace_args[8] = (uintptr_t)arg8;									\
+	__dtrace_args[9] = (uintptr_t)arg9;									\
+	DTRACE_CALL10ARGS(provider, name)								\
+}
+
+
+#ifdef KERNEL
+
+#if CONFIG_DTRACE
+
+#define	DTRACE_SCHED(name)						\
+	DTRACE_PROBE(__sched_, name);
+
+#define	DTRACE_SCHED1(name, type1, arg1)				\
+	DTRACE_PROBE1(__sched_, name, arg1);
+
+#define	DTRACE_SCHED2(name, type1, arg1, type2, arg2)			\
+	DTRACE_PROBE2(__sched_, name, arg1, arg2);
+
+#define	DTRACE_SCHED3(name, type1, arg1, type2, arg2, type3, arg3)	\
+	DTRACE_PROBE3(__sched_, name, arg1, arg2, arg3);
+
+#define	DTRACE_SCHED4(name, type1, arg1, type2, arg2, 			\
+    type3, arg3, type4, arg4)						\
+	DTRACE_PROBE4(__sched_, name, arg1, arg2, arg3, arg4);
+
+#define	DTRACE_PROC(name)						\
+	DTRACE_PROBE(__proc_, name);
+
+#define	DTRACE_PROC1(name, type1, arg1)					\
+	DTRACE_PROBE1(__proc_, name, arg1);
+
+#define	DTRACE_PROC2(name, type1, arg1, type2, arg2)			\
+	DTRACE_PROBE2(__proc_, name, arg1, arg2);
+
+#define	DTRACE_PROC3(name, type1, arg1, type2, arg2, type3, arg3)	\
+	DTRACE_PROBE3(__proc_, name, arg1, arg2, arg3);
+
+#define	DTRACE_PROC4(name, type1, arg1, type2, arg2, 			\
+    type3, arg3, type4, arg4)						\
+	DTRACE_PROBE4(__proc_, name, arg1, arg2, arg3, arg4);
+
+#define	DTRACE_IO(name)							\
+	DTRACE_PROBE(__io_, name);
+
+#define	DTRACE_IO1(name, type1, arg1)					\
+	DTRACE_PROBE1(__io_, name, arg1);
+
+#define	DTRACE_IO2(name, type1, arg1, type2, arg2)			\
+	DTRACE_PROBE2(__io_, name, type1, arg1, type2, arg2);
+
+#define	DTRACE_IO3(name, type1, arg1, type2, arg2, type3, arg3)		\
+	DTRACE_PROBE3(__io_, name, arg1, arg2, arg3);
+
+#define	DTRACE_IO4(name, type1, arg1, type2, arg2, 			\
+    type3, arg3, type4, arg4)						\
+	DTRACE_PROBE4(__io_, name, arg1, arg2, arg3, arg4);
+
+#define	DTRACE_INT5(name, type1, arg1, type2, arg2, 			\
+    type3, arg3, type4, arg4, type5, arg5)						\
+	DTRACE_PROBE5(__sdt_, name, arg1, arg2, arg3, arg4, arg5);
+
+#define	DTRACE_TMR3(name, type1, arg1, type2, arg2, type3, arg3)		\
+	DTRACE_PROBE3(__sdt_, name, arg1, arg2, arg3);
+
+#define	DTRACE_VM(name)							\
+	DTRACE_PROBE(__vminfo_, name)
+
+#define	DTRACE_VM1(name, type1, arg1)					\
+	DTRACE_PROBE1(__vminfo_, name, arg1)
+
+#define	DTRACE_VM2(name, type1, arg1, type2, arg2)			\
+	DTRACE_PROBE2(__vminfo_, name, arg1, arg2)
+
+#define	DTRACE_VM3(name, type1, arg1, type2, arg2, type3, arg3)		\
+	DTRACE_PROBE3(__vminfo_, name, arg1, arg2, arg3)
+
+#define	DTRACE_VM4(name, type1, arg1, type2, arg2, 			\
+    type3, arg3, type4, arg4)						\
+	DTRACE_PROBE4(__vminfo_, name, arg1, arg2, arg3, arg4)
+
+#else /* CONFIG_DTRACE */
+
+#define	DTRACE_SCHED(name) do {} while (0)
+#define	DTRACE_SCHED1(name, type1, arg1) do {} while (0)
+#define	DTRACE_SCHED2(name, type1, arg1, type2, arg2) do {} while (0)
+#define	DTRACE_SCHED3(name, type1, arg1, type2, arg2, type3, arg3) do {} while (0)
+#define	DTRACE_SCHED4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) do {} while (0)
+
+#define DTRACE_PROC(name) do {} while(0)
+#define DTRACE_PROC1(name, type1, arg1) do {} while(0)
+#define	DTRACE_PROC2(name, type1, arg1, type2, arg2) do {} while (0)
+#define	DTRACE_PROC3(name, type1, arg1, type2, arg2, type3, arg3) do {} while (0)
+#define	DTRACE_PROC4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) do {} while(0)
+#define	DTRACE_IO(name) do {} while(0)
+#define	DTRACE_IO1(name, type1, arg1) do {} while(0)
+#define	DTRACE_IO2(name, type1, arg1, type2, arg2) do {} while(0)
+#define	DTRACE_IO3(name, type1, arg1, type2, arg2, type3, arg3) do {} while(0)
+#define	DTRACE_IO4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) do {} while(0)
+#define	DTRACE_INT5(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4, type5, arg5) do {} while(0)
+#define	DTRACE_TMR3(name, type1, arg1, type2, arg2, type3, arg3) do {} while(0)
+
+#define DTRACE_VM(name) do {} while(0)
+#define DTRACE_VM1(name, type1, arg1) do {} while(0)
+#define DTRACE_VM2(name, type1, arg1, type2, arg2) do {} while(0)
+#define DTRACE_VM3(name, type1, arg1, type2, arg2, type3, arg3) do {} while(0)
+#define DTRACE_VM4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) do {} while(0)
+
+#endif /* CONFIG_DTRACE */
+
+#endif /* KERNEL */
+
+#endif	/* _MACH_MACHINE_SYS_SDT_H */
diff --git a/osfmk/mach/machine/sdt_isa.h b/osfmk/mach/machine/sdt_isa.h
new file mode 100644
index 000000000..a8e1379cd
--- /dev/null
+++ b/osfmk/mach/machine/sdt_isa.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifndef _MACH_MACHINE_SDT_ISA_H_
+#define _MACH_MACHINE_SDT_ISA_H_
+
+#if defined (__ppc__) || defined (__ppc64__)
+#include <mach/ppc/sdt_isa.h>
+#elif defined (__i386__) || defined(__x86_64__)
+#include <mach/i386/sdt_isa.h>
+#elif defined (__arm__)
+#include <mach/arm/sdt_isa.h>
+#else
+#error architecture not supported
+#endif
+
+#endif /* _BSD_MACHINE_SDT_ISA_H_ */
diff --git a/osfmk/mach/machine/syscall_sw.h b/osfmk/mach/machine/syscall_sw.h
index 2626d4c7a..e5aaa900a 100644
--- a/osfmk/mach/machine/syscall_sw.h
+++ b/osfmk/mach/machine/syscall_sw.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifdef	PRIVATE
@@ -29,6 +35,8 @@
 #include "mach/ppc/syscall_sw.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/syscall_sw.h"
+#elif defined (__arm__)
+#include "mach/arm/syscall_sw.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/thread_state.h b/osfmk/mach/machine/thread_state.h
index da012bcb1..18ffca4e5 100644
--- a/osfmk/mach/machine/thread_state.h
+++ b/osfmk/mach/machine/thread_state.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_THREAD_STATE_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/thread_state.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/thread_state.h"
+#elif defined (__arm__)
+#include "mach/arm/thread_state.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/thread_status.h b/osfmk/mach/machine/thread_status.h
index e13af4f42..ad5eff08f 100644
--- a/osfmk/mach/machine/thread_status.h
+++ b/osfmk/mach/machine/thread_status.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_THREAD_STATUS_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/thread_status.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/thread_status.h"
+#elif defined (__arm__)
+#include "mach/arm/thread_status.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/vm_param.h b/osfmk/mach/machine/vm_param.h
index 936fb3772..c71121b92 100644
--- a/osfmk/mach/machine/vm_param.h
+++ b/osfmk/mach/machine/vm_param.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_VM_PARAM_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/vm_param.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/vm_param.h"
+#elif defined (__arm__)
+#include "mach/arm/vm_param.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/machine/vm_types.h b/osfmk/mach/machine/vm_types.h
index 31621f173..b36609214 100644
--- a/osfmk/mach/machine/vm_types.h
+++ b/osfmk/mach/machine/vm_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_MACHINE_VM_TYPES_H_
@@ -27,6 +33,8 @@
 #include "mach/ppc/vm_types.h"
 #elif defined (__i386__) || defined(__x86_64__)
 #include "mach/i386/vm_types.h"
+#elif defined (__arm__)
+#include "mach/arm/vm_types.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/mach/memory_object.defs b/osfmk/mach/memory_object.defs
index 920397d64..9fd7664f4 100644
--- a/osfmk/mach/memory_object.defs
+++ b/osfmk/mach/memory_object.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -117,7 +123,8 @@ routine	memory_object_data_request(
 		memory_object		: memory_object_t;
 		offset			: memory_object_offset_t;
 		length			: memory_object_cluster_size_t;
-		desired_access		: vm_prot_t);
+		desired_access		: vm_prot_t;
+		fault_info		: memory_object_fault_info_t);
 
 /*
  *	Return data to manager.  This call is used in place of data_write
@@ -193,4 +200,4 @@ routine memory_object_synchronize(
 routine memory_object_unmap(
 		memory_object           : memory_object_t);
 
-
+/* vim: set ft=c : */
diff --git a/osfmk/mach/memory_object.h b/osfmk/mach/memory_object.h
index e7949d9c8..68223b8fa 100644
--- a/osfmk/mach/memory_object.h
+++ b/osfmk/mach/memory_object.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/memory_object_control.defs b/osfmk/mach/memory_object_control.defs
index 22a6518c0..fef553a3b 100644
--- a/osfmk/mach/memory_object_control.defs
+++ b/osfmk/mach/memory_object_control.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -156,6 +162,13 @@ routine memory_object_super_upl_request(
 	out	page_list	: upl_page_info_array_t, CountInOut;
 	in	cntrl_flags	: integer_t);
 
+
+routine memory_object_cluster_size(
+		control		: memory_object_control_t;
+	out	start		: memory_object_offset_t;
+	out	length		: vm_size_t;
+	in	fault_info	: memory_object_fault_info_t);
+
 /*
  * This functions allows a single page to be manipulated with less overhead
  * than creating a UPL.
@@ -182,4 +195,4 @@ routine memory_object_range_op(
 	in      ops             : integer_t;
 	out     range		: integer_t);
 
-
+/* vim: set ft=c : */
diff --git a/osfmk/mach/memory_object_default.defs b/osfmk/mach/memory_object_default.defs
index 315977690..d168ebffe 100644
--- a/osfmk/mach/memory_object_default.defs
+++ b/osfmk/mach/memory_object_default.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -85,3 +91,4 @@ routine	memory_object_create(
 		new_memory_object_size	: vm_size_t;
 	out	new_memory_object	: memory_object_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/memory_object_name.defs b/osfmk/mach/memory_object_name.defs
index 7825b9ecf..c4d4b77df 100644
--- a/osfmk/mach/memory_object_name.defs
+++ b/osfmk/mach/memory_object_name.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -72,3 +78,5 @@ subsystem
  * These are used simply to compare one mapping againsts another
  * and have no methods.
  */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/memory_object_types.h b/osfmk/mach/memory_object_types.h
index 1b6254a05..65c57e540 100644
--- a/osfmk/mach/memory_object_types.h
+++ b/osfmk/mach/memory_object_types.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -78,6 +84,7 @@
 typedef unsigned long long	memory_object_offset_t;
 typedef unsigned long long	memory_object_size_t;
 typedef natural_t		memory_object_cluster_size_t;
+typedef natural_t *		memory_object_fault_info_t;
 
 
 /*
@@ -112,7 +119,8 @@ typedef const struct memory_object_pager_ops {
 		memory_object_t mem_obj,
 		memory_object_offset_t offset,
 		memory_object_cluster_size_t length,
-		vm_prot_t desired_access);
+		vm_prot_t desired_access,
+		memory_object_fault_info_t fault_info);
 	kern_return_t (*memory_object_data_return)(
 		memory_object_t mem_obj,
 		memory_object_offset_t offset,
@@ -278,7 +286,7 @@ extern void memory_object_default_deallocate(memory_object_default_t);
 
 extern void memory_object_control_reference(memory_object_control_t control);
 extern void memory_object_control_deallocate(memory_object_control_t control);
-extern int  memory_object_control_uiomove(memory_object_control_t, memory_object_offset_t, void *, int, int, int);
+extern int  memory_object_control_uiomove(memory_object_control_t, memory_object_offset_t, void *, int, int, int, int);
 __END_DECLS
 
 #endif  /* KERNEL */
@@ -382,6 +390,7 @@ struct upl_page_info {
                         dirty:1,        /* Page must be cleaned (O) */
 			precious:1,     /* must be cleaned, we have only copy */
 			device:1,	/* no page data, mapped dev memory */
+	                speculative:1,  /* page is valid, but not yet accessed */
                         :0;		/* force to long boundary */
 #else
 			opaque;		/* use upl_page_xxx() accessor funcs */
@@ -441,13 +450,14 @@ typedef uint32_t	upl_size_t;	/* page-aligned byte size */
 
 
 /* upl abort error flags */
-#define UPL_ABORT_RESTART	0x1
+#define UPL_ABORT_RESTART		0x1
 #define UPL_ABORT_UNAVAILABLE	0x2
 #define UPL_ABORT_ERROR		0x4
 #define UPL_ABORT_FREE_ON_EMPTY	0x8  /* only implemented in wrappers */
 #define UPL_ABORT_DUMP_PAGES	0x10
 #define UPL_ABORT_NOTIFY_EMPTY	0x20
 #define UPL_ABORT_ALLOW_ACCESS	0x40
+#define UPL_ABORT_REFERENCE	0x80
 
 /* upl pages check flags */
 #define UPL_CHECK_DIRTY         0x1
@@ -501,6 +511,17 @@ typedef uint32_t	upl_size_t;	/* page-aligned byte size */
  */
 #define UPL_KEEPCACHED		0x40
 
+/*
+ * this pageout originated from within cluster_io to deal
+ * with a dirty page that hasn't yet been seen by the FS
+ * that backs it... tag it so that the FS can take the
+ * appropriate action w/r to its locking model since the
+ * pageout will reenter the FS for the same file currently
+ * being handled in this context.
+ */
+
+#define UPL_NESTED_PAGEOUT	0x80
+
 
 
 /* upl commit flags */
@@ -557,7 +578,7 @@ typedef uint32_t	upl_size_t;	/* page-aligned byte size */
 /* access macros for upl_t */
 
 #define UPL_DEVICE_PAGE(upl) \
-	(((upl)[(index)].phys_addr != 0) ? (!((upl)[0].device)) : FALSE)
+	(((upl)[0].phys_addr != 0) ? ((upl)[0].device) : FALSE)
 
 #define UPL_PAGE_PRESENT(upl, index) \
 	((upl)[(index)].phys_addr != 0)
@@ -565,6 +586,9 @@ typedef uint32_t	upl_size_t;	/* page-aligned byte size */
 #define UPL_PHYS_PAGE(upl, index) \
 	((upl)[(index)].phys_addr)
 
+#define UPL_SPECULATIVE_PAGE(upl, index) \
+	(((upl)[(index)].phys_addr != 0) ? ((upl)[(index)].speculative) : FALSE)
+
 #define UPL_DIRTY_PAGE(upl, index) \
 	(((upl)[(index)].phys_addr != 0) ? ((upl)[(index)].dirty) : FALSE)
 
@@ -601,6 +625,8 @@ extern vm_size_t 	upl_get_internal_pagelist_offset(void);
 __BEGIN_DECLS
 
 extern ppnum_t	upl_phys_page(upl_page_info_t *upl, int index);
+extern boolean_t	upl_device_page(upl_page_info_t *upl);
+extern boolean_t	upl_speculative_page(upl_page_info_t *upl, int index);
 extern void	upl_clear_dirty(upl_t upl, boolean_t value);
 
 __END_DECLS
diff --git a/osfmk/mach/message.h b/osfmk/mach/message.h
index 77f71a4e1..37c9aa3be 100644
--- a/osfmk/mach/message.h
+++ b/osfmk/mach/message.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,13 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ * Copyright (c) 2005 SPARTA, Inc.
+ */
 /*
  */
 /*
@@ -373,6 +386,27 @@ typedef struct
   audit_token_t			msgh_audit;
 } mach_msg_audit_trailer_t;
 
+typedef struct
+{
+  mach_port_name_t sender;
+} msg_labels_t;
+
+/* 
+   Trailer type to pass MAC policy label info as a mach message trailer.
+   
+*/
+
+typedef struct
+{
+  mach_msg_trailer_type_t       msgh_trailer_type;
+  mach_msg_trailer_size_t       msgh_trailer_size;
+  mach_port_seqno_t             msgh_seqno;
+  security_token_t              msgh_sender;
+  audit_token_t			msgh_audit;
+  msg_labels_t                  msgh_labels;
+  int				msgh_ad;
+} mach_msg_mac_trailer_t;
+
 #define MACH_MSG_TRAILER_MINIMUM_SIZE  sizeof(mach_msg_trailer_t)
 
 /*
@@ -384,7 +418,7 @@ typedef struct
  * another module may exceed the local modules notion of
  * MAX_TRAILER_SIZE.
  */
-typedef mach_msg_audit_trailer_t mach_msg_max_trailer_t;
+typedef mach_msg_mac_trailer_t mach_msg_max_trailer_t;
 #define MAX_TRAILER_SIZE sizeof(mach_msg_max_trailer_t)
 
 /*
@@ -395,6 +429,10 @@ typedef mach_msg_audit_trailer_t mach_msg_max_trailer_t;
  * REQUESTED_TRAILER_SIZE.
  */
 typedef mach_msg_security_trailer_t mach_msg_format_0_trailer_t;
+
+/*typedef mach_msg_mac_trailer_t mach_msg_format_0_trailer_t;
+*/
+
 #define MACH_MSG_TRAILER_FORMAT_0_SIZE sizeof(mach_msg_format_0_trailer_t)
 
 #define   KERNEL_SECURITY_TOKEN_VALUE  { {0, 1} }
@@ -535,17 +573,33 @@ typedef integer_t mach_msg_option_t;
  * NOTE: a 0x00------ RCV mask implies to ask for
  * a MACH_MSG_TRAILER_FORMAT_0 with 0 Elements, 
  * which is equivalent to a mach_msg_trailer_t.
+ *
+ * XXXMAC: unlike the rest of the MACH_RCV_* flags, MACH_RCV_TRAILER_LABELS
+ * and MACH_RCV_TRAILER_AV need their own private bit since we only calculate
+ * their fields when absolutely required.  This will cause us problems if
+ * Apple adds new trailers.
  */
 #define MACH_RCV_TRAILER_NULL   0
 #define MACH_RCV_TRAILER_SEQNO  1
 #define MACH_RCV_TRAILER_SENDER 2
 #define MACH_RCV_TRAILER_AUDIT  3
+#define MACH_RCV_TRAILER_LABELS 4
+#define MACH_RCV_TRAILER_AV     8
 
 #define MACH_RCV_TRAILER_TYPE(x)     (((x) & 0xf) << 28) 
 #define MACH_RCV_TRAILER_ELEMENTS(x) (((x) & 0xf) << 24)  
 #define MACH_RCV_TRAILER_MASK 	     ((0xff << 24))
 
 #define GET_RCV_ELEMENTS(y) (((y) >> 24) & 0xf)
+
+/* 
+ * XXXMAC: note that in the case of MACH_RCV_TRAILER_AV and
+ * MACH_RCV_TRAILER_LABELS, we just fall through to mach_msg_max_trailer_t.
+ * This is correct behavior since mach_msg_max_trailer_t is defined as
+ * mac_msg_mac_trailer_t which is used for the LABELS and AV trailers.
+ * It also makes things work properly if MACH_RCV_TRAILER_AV or
+ * MACH_RCV_TRAILER_LABELS are ORed with one of the other options.
+ */
 #define REQUESTED_TRAILER_SIZE(y) 				\
 	((mach_msg_trailer_size_t)				\
 	 ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_NULL) ?	\
@@ -554,7 +608,10 @@ typedef integer_t mach_msg_option_t;
 	   sizeof(mach_msg_seqno_trailer_t) :			\
 	  ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_SENDER) ?	\
 	   sizeof(mach_msg_security_trailer_t) :		\
-	   sizeof(mach_msg_audit_trailer_t)))))
+	   ((GET_RCV_ELEMENTS(y) == MACH_RCV_TRAILER_AUDIT) ?	\
+	    sizeof(mach_msg_audit_trailer_t) :      		\
+	    sizeof(mach_msg_max_trailer_t))))))
+
 /*
  *  Much code assumes that mach_msg_return_t == kern_return_t.
  *  This definition is useful for descriptive purposes.
@@ -703,3 +760,4 @@ extern mach_msg_return_t	mach_msg(
 __END_DECLS
 
 #endif	/* _MACH_MESSAGE_H_ */
+
diff --git a/osfmk/mach/mig.h b/osfmk/mach/mig.h
index bdb9ce2eb..31a454a60 100644
--- a/osfmk/mach/mig.h
+++ b/osfmk/mach/mig.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -39,8 +45,13 @@
 
 #if defined(MACH_KERNEL)
 
+#if defined(BSMALL_LATER)
+/* Really small configurations don't need type checking */
+#define __MigTypeCheck 0
+#else
 /* Turn MIG type checking on by default for kernel */
 #define __MigTypeCheck 1
+#endif
 #define __MigKernelSpecificCode 1
 #define _MIG_KERNEL_SPECIFIC_CODE_ 1
 
@@ -48,8 +59,17 @@
 #elif defined(TypeCheck)  
 
 #define __MigTypeCheck TypeCheck
- 
-#endif /* defined(TypeCheck) */
+
+#elif !defined(__MigTypeCheck)
+
+/* otherwise, default MIG type checking on - except in small configurations */
+#if defined(BSMALL)
+#define __MigTypeCheck 0
+#else
+#define __MigTypeCheck 1
+#endif
+
+#endif /* !defined(__MigTypeCheck) */
 
 /*
  * Pack MIG message structs.
diff --git a/osfmk/mach/mig_errors.h b/osfmk/mach/mig_errors.h
index 8e588251a..f1234bc7b 100644
--- a/osfmk/mach/mig_errors.h
+++ b/osfmk/mach/mig_errors.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -100,20 +106,18 @@ typedef struct {
 
 __BEGIN_DECLS
 
+#if !defined(__NDR_convert__mig_reply_error_t__defined)
 #define __NDR_convert__mig_reply_error_t__defined
-#if mig_internal
-mig_internal
-#else
-static
-#endif
-__inline__ void
-__NDR_convert__mig_reply_error_t(mig_reply_error_t *x)
+
+static __inline__ void
+__NDR_convert__mig_reply_error_t(__unused mig_reply_error_t *x)
 {
 #if defined(__NDR_convert__int_rep__kern_return_t__defined)
 	if (x->NDR.int_rep != NDR_record.int_rep)
 		__NDR_convert__int_rep__kern_return_t(&x->RetCode, x->NDR.int_rep);
 #endif /* __NDR_convert__int_rep__kern_return_t__defined */
 }
+#endif /* !defined(__NDR_convert__mig_reply_error_t__defined) */
 
 __END_DECLS
 
diff --git a/osfmk/mach/mig_log.h b/osfmk/mach/mig_log.h
index c8f1e7190..44ce00002 100644
--- a/osfmk/mach/mig_log.h
+++ b/osfmk/mach/mig_log.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/mk_timer.h b/osfmk/mach/mk_timer.h
index 1d2db1822..396c1a163 100644
--- a/osfmk/mach/mk_timer.h
+++ b/osfmk/mach/mk_timer.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2000 Apple Computer, Inc.  All rights reserved.
diff --git a/osfmk/mach/mk_traps.h b/osfmk/mach/mk_traps.h
index c5159b834..68e71faa9 100644
--- a/osfmk/mach/mk_traps.h
+++ b/osfmk/mach/mk_traps.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (c) 2000 Apple Computer, Inc.  All rights reserved.
diff --git a/osfmk/mach/msg_type.h b/osfmk/mach/msg_type.h
index 00d56bbb2..fb280331b 100644
--- a/osfmk/mach/msg_type.h
+++ b/osfmk/mach/msg_type.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ndr.h b/osfmk/mach/ndr.h
index 9b3d18037..cb64d3fc3 100644
--- a/osfmk/mach/ndr.h
+++ b/osfmk/mach/ndr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -59,6 +65,10 @@ typedef struct {
 
 extern NDR_record_t NDR_record;
 
+#if defined(BSMALL)
+#define __NDR_convert__ 0
+#define __NDR_convert__int_rep__ 0
+#else
 #ifndef __NDR_convert__
 #define __NDR_convert__ 1
 #endif /* __NDR_convert__ */
@@ -67,6 +77,8 @@ extern NDR_record_t NDR_record;
 #define __NDR_convert__int_rep__ 1
 #endif /* __NDR_convert__int_rep__ */
 
+#endif /* defined(BSMALL) */
+
 #ifndef __NDR_convert__char_rep__
 #define __NDR_convert__char_rep__ 0
 #endif /* __NDR_convert__char_rep__ */
diff --git a/osfmk/mach/norma_special_ports.h b/osfmk/mach/norma_special_ports.h
index 22056b1c3..e4da723f6 100644
--- a/osfmk/mach/norma_special_ports.h
+++ b/osfmk/mach/norma_special_ports.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/notify.defs b/osfmk/mach/notify.defs
index 47888bb9e..6f7f81d2d 100644
--- a/osfmk/mach/notify.defs
+++ b/osfmk/mach/notify.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -114,3 +120,5 @@ simpleroutine mach_notify_dead_name(
 	msgseqno seqno	: mach_port_seqno_t;
 #endif	/* SEQNOS */
 		name	: mach_port_name_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/notify.h b/osfmk/mach/notify.h
index 89515ec57..768a865cd 100644
--- a/osfmk/mach/notify.h
+++ b/osfmk/mach/notify.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/policy.h b/osfmk/mach/policy.h
index d82b9dda1..b6ec795a2 100644
--- a/osfmk/mach/policy.h
+++ b/osfmk/mach/policy.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/port.h b/osfmk/mach/port.h
index 8803a516c..ff3360297 100644
--- a/osfmk/mach/port.h
+++ b/osfmk/mach/port.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,6 +53,12 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 /*
  */
 /*
@@ -74,6 +86,7 @@
 #ifndef	_MACH_PORT_H_
 #define _MACH_PORT_H_
 
+#include <sys/cdefs.h>
 #include <stdint.h>
 #include <mach/boolean.h>
 #include <mach/machine/vm_types.h>
@@ -224,7 +237,8 @@ typedef natural_t mach_port_right_t;
 #define MACH_PORT_RIGHT_SEND_ONCE	((mach_port_right_t) 2)
 #define MACH_PORT_RIGHT_PORT_SET	((mach_port_right_t) 3)
 #define MACH_PORT_RIGHT_DEAD_NAME	((mach_port_right_t) 4)
-#define MACH_PORT_RIGHT_NUMBER		((mach_port_right_t) 5)
+#define MACH_PORT_RIGHT_LABELH	        ((mach_port_right_t) 5)
+#define MACH_PORT_RIGHT_NUMBER		((mach_port_right_t) 6)
 
 typedef natural_t mach_port_type_t;
 typedef mach_port_type_t *mach_port_type_array_t;
@@ -238,6 +252,7 @@ typedef mach_port_type_t *mach_port_type_array_t;
 #define MACH_PORT_TYPE_SEND_ONCE    MACH_PORT_TYPE(MACH_PORT_RIGHT_SEND_ONCE)
 #define MACH_PORT_TYPE_PORT_SET	    MACH_PORT_TYPE(MACH_PORT_RIGHT_PORT_SET)
 #define MACH_PORT_TYPE_DEAD_NAME    MACH_PORT_TYPE(MACH_PORT_RIGHT_DEAD_NAME)
+#define MACH_PORT_TYPE_LABELH       MACH_PORT_TYPE(MACH_PORT_RIGHT_LABELH)
 
 /* Convenient combinations. */
 
@@ -276,7 +291,7 @@ typedef natural_t mach_port_rights_t;		/* number of rights */
 typedef unsigned int mach_port_srights_t;	/* status of send rights */
 
 typedef struct mach_port_status {
-	mach_port_name_t	mps_pset;	/* containing port set */
+	mach_port_rights_t	mps_pset;	/* count of containing port sets */
 	mach_port_seqno_t	mps_seqno;	/* sequence number */
 	mach_port_mscount_t	mps_mscount;	/* make-send count */
 	mach_port_msgcount_t	mps_qlimit;	/* queue limit */
@@ -288,8 +303,14 @@ typedef struct mach_port_status {
 	natural_t		mps_flags;		/* port flags */
 } mach_port_status_t;
 
-#define MACH_PORT_QLIMIT_DEFAULT	((mach_port_msgcount_t) 5)
-#define MACH_PORT_QLIMIT_MAX	((mach_port_msgcount_t) 16)
+/* System-wide values for setting queue limits on a port */
+#define MACH_PORT_QLIMIT_ZERO		((mach_port_msgcount_t) 0)
+#define MACH_PORT_QLIMIT_BASIC		((mach_port_msgcount_t) 5)
+#define MACH_PORT_QLIMIT_SMALL		((mach_port_msgcount_t) 16)
+#define MACH_PORT_QLIMIT_LARGE		((mach_port_msgcount_t) 1024)
+#define MACH_PORT_QLIMIT_MIN		MACH_PORT_QLIMIT_ZERO
+#define MACH_PORT_QLIMIT_DEFAULT	MACH_PORT_QLIMIT_BASIC
+#define MACH_PORT_QLIMIT_MAX		MACH_PORT_QLIMIT_LARGE
 
 typedef struct mach_port_limits {
 	mach_port_msgcount_t	mpl_qlimit;	/* number of msgs */
@@ -314,13 +335,13 @@ typedef int	mach_port_flavor_t;
  * Must be padded to 64-bits total length.
  */
 typedef struct mach_port_qos {
-	boolean_t		name:1;		/* name given */
-	boolean_t		prealloc:1;	/* prealloced message */
+	unsigned int		name:1;		/* name given */
+	unsigned int 		prealloc:1;	/* prealloced message */
 	boolean_t		pad1:30;
 	natural_t		len;
 } mach_port_qos_t;
 
-#if	!defined(_POSIX_C_SOURCE) && !defined(_NO_PORT_T_FROM_MACH)
+#if	!__DARWIN_UNIX03 && !defined(_NO_PORT_T_FROM_MACH)
 /*
  *  Mach 3.0 renamed everything to have mach_ in front of it.
  *  These types and macros are provided for backward compatibility
@@ -335,6 +356,6 @@ typedef mach_port_name_t	*port_name_array_t;
 #define PORT_VALID(name) \
 		((port_t)(name) != PORT_NULL && (port_t)(name) != PORT_DEAD)
 
-#endif	/* !_POSIX_C_SOURCE && !_NO_PORT_T_FROM_MACH */
+#endif	/* !__DARWIN_UNIX03 && !_NO_PORT_T_FROM_MACH */
 
 #endif	/* _MACH_PORT_H_ */
diff --git a/osfmk/mach/ppc/Makefile b/osfmk/mach/ppc/Makefile
index bdfa7ed0a..83f21cec6 100644
--- a/osfmk/mach/ppc/Makefile
+++ b/osfmk/mach/ppc/Makefile
@@ -13,7 +13,7 @@ DATAFILES = \
 	boolean.h exception.h kern_return.h ndr_def.h \
 	processor_info.h rpc.h thread_state.h thread_status.h \
 	vm_param.h vm_types.h machine_types.defs \
-	syscall_sw.h _types.h
+	syscall_sw.h _structs.h sdt_isa.h
 
 INSTALL_MD_LIST = ${DATAFILES}
 
diff --git a/osfmk/mach/ppc/_structs.h b/osfmk/mach/ppc/_structs.h
new file mode 100644
index 000000000..f2c78cda1
--- /dev/null
+++ b/osfmk/mach/ppc/_structs.h
@@ -0,0 +1,392 @@
+/*
+ * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+#ifndef	_MACH_PPC__STRUCTS_H_
+#define	_MACH_PPC__STRUCTS_H_
+
+#include <sys/cdefs.h>
+
+/*
+ * ppc_thread_state is the structure that is exported to user threads for 
+ * use in status/mutate calls.  This structure should never change.
+ *
+ */
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_PPC_THREAD_STATE	struct __darwin_ppc_thread_state
+_STRUCT_PPC_THREAD_STATE
+{
+	unsigned int __srr0;	/* Instruction address register (PC) */
+	unsigned int __srr1;	/* Machine state register (supervisor) */
+	unsigned int __r0;
+	unsigned int __r1;
+	unsigned int __r2;
+	unsigned int __r3;
+	unsigned int __r4;
+	unsigned int __r5;
+	unsigned int __r6;
+	unsigned int __r7;
+	unsigned int __r8;
+	unsigned int __r9;
+	unsigned int __r10;
+	unsigned int __r11;
+	unsigned int __r12;
+	unsigned int __r13;
+	unsigned int __r14;
+	unsigned int __r15;
+	unsigned int __r16;
+	unsigned int __r17;
+	unsigned int __r18;
+	unsigned int __r19;
+	unsigned int __r20;
+	unsigned int __r21;
+	unsigned int __r22;
+	unsigned int __r23;
+	unsigned int __r24;
+	unsigned int __r25;
+	unsigned int __r26;
+	unsigned int __r27;
+	unsigned int __r28;
+	unsigned int __r29;
+	unsigned int __r30;
+	unsigned int __r31;
+
+	unsigned int __cr;	/* Condition register */
+	unsigned int __xer;	/* User's integer exception register */
+	unsigned int __lr;	/* Link register */
+	unsigned int __ctr;	/* Count register */
+	unsigned int __mq;	/* MQ register (601 only) */
+
+	unsigned int __vrsave;	/* Vector Save Register */
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_PPC_THREAD_STATE	struct ppc_thread_state
+_STRUCT_PPC_THREAD_STATE
+{
+	unsigned int srr0;	/* Instruction address register (PC) */
+	unsigned int srr1;	/* Machine state register (supervisor) */
+	unsigned int r0;
+	unsigned int r1;
+	unsigned int r2;
+	unsigned int r3;
+	unsigned int r4;
+	unsigned int r5;
+	unsigned int r6;
+	unsigned int r7;
+	unsigned int r8;
+	unsigned int r9;
+	unsigned int r10;
+	unsigned int r11;
+	unsigned int r12;
+	unsigned int r13;
+	unsigned int r14;
+	unsigned int r15;
+	unsigned int r16;
+	unsigned int r17;
+	unsigned int r18;
+	unsigned int r19;
+	unsigned int r20;
+	unsigned int r21;
+	unsigned int r22;
+	unsigned int r23;
+	unsigned int r24;
+	unsigned int r25;
+	unsigned int r26;
+	unsigned int r27;
+	unsigned int r28;
+	unsigned int r29;
+	unsigned int r30;
+	unsigned int r31;
+
+	unsigned int cr;	/* Condition register */
+	unsigned int xer;	/* User's integer exception register */
+	unsigned int lr;	/* Link register */
+	unsigned int ctr;	/* Count register */
+	unsigned int mq;	/* MQ register (601 only) */
+
+	unsigned int vrsave;	/* Vector Save Register */
+};
+#endif /* __DARWIN_UNIX03 */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+
+#pragma pack(4)			/* Make sure the structure stays as we defined it */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_PPC_THREAD_STATE64	struct __darwin_ppc_thread_state64
+_STRUCT_PPC_THREAD_STATE64
+{
+	unsigned long long __srr0;	/* Instruction address register (PC) */
+	unsigned long long __srr1;	/* Machine state register (supervisor) */
+	unsigned long long __r0;
+	unsigned long long __r1;
+	unsigned long long __r2;
+	unsigned long long __r3;
+	unsigned long long __r4;
+	unsigned long long __r5;
+	unsigned long long __r6;
+	unsigned long long __r7;
+	unsigned long long __r8;
+	unsigned long long __r9;
+	unsigned long long __r10;
+	unsigned long long __r11;
+	unsigned long long __r12;
+	unsigned long long __r13;
+	unsigned long long __r14;
+	unsigned long long __r15;
+	unsigned long long __r16;
+	unsigned long long __r17;
+	unsigned long long __r18;
+	unsigned long long __r19;
+	unsigned long long __r20;
+	unsigned long long __r21;
+	unsigned long long __r22;
+	unsigned long long __r23;
+	unsigned long long __r24;
+	unsigned long long __r25;
+	unsigned long long __r26;
+	unsigned long long __r27;
+	unsigned long long __r28;
+	unsigned long long __r29;
+	unsigned long long __r30;
+	unsigned long long __r31;
+
+	unsigned int __cr;		/* Condition register */
+	unsigned long long __xer;	/* User's integer exception register */
+	unsigned long long __lr;	/* Link register */
+	unsigned long long __ctr;	/* Count register */
+
+	unsigned int __vrsave;		/* Vector Save Register */
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_PPC_THREAD_STATE64	struct ppc_thread_state64
+_STRUCT_PPC_THREAD_STATE64
+{
+	unsigned long long srr0;	/* Instruction address register (PC) */
+	unsigned long long srr1;	/* Machine state register (supervisor) */
+	unsigned long long r0;
+	unsigned long long r1;
+	unsigned long long r2;
+	unsigned long long r3;
+	unsigned long long r4;
+	unsigned long long r5;
+	unsigned long long r6;
+	unsigned long long r7;
+	unsigned long long r8;
+	unsigned long long r9;
+	unsigned long long r10;
+	unsigned long long r11;
+	unsigned long long r12;
+	unsigned long long r13;
+	unsigned long long r14;
+	unsigned long long r15;
+	unsigned long long r16;
+	unsigned long long r17;
+	unsigned long long r18;
+	unsigned long long r19;
+	unsigned long long r20;
+	unsigned long long r21;
+	unsigned long long r22;
+	unsigned long long r23;
+	unsigned long long r24;
+	unsigned long long r25;
+	unsigned long long r26;
+	unsigned long long r27;
+	unsigned long long r28;
+	unsigned long long r29;
+	unsigned long long r30;
+	unsigned long long r31;
+
+	unsigned int cr;		/* Condition register */
+	unsigned long long xer;		/* User's integer exception register */
+	unsigned long long lr;		/* Link register */
+	unsigned long long ctr;		/* Count register */
+
+	unsigned int vrsave;		/* Vector Save Register */
+};
+#endif /* __DARWIN_UNIX03 */
+
+#pragma pack()
+
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+
+/* This structure should be double-word aligned for performance */
+
+#if __DARWIN_UNIX03
+#define	_STRUCT_PPC_FLOAT_STATE	struct __darwin_ppc_float_state
+_STRUCT_PPC_FLOAT_STATE
+{
+	double  __fpregs[32];
+
+	unsigned int __fpscr_pad; /* fpscr is 64 bits, 32 bits of rubbish */
+	unsigned int __fpscr;	/* floating point status register */
+};
+#else /* !__DARWIN_UNIX03 */
+#define	_STRUCT_PPC_FLOAT_STATE	struct ppc_float_state
+_STRUCT_PPC_FLOAT_STATE
+{
+	double  fpregs[32];
+
+	unsigned int fpscr_pad; /* fpscr is 64 bits, 32 bits of rubbish */
+	unsigned int fpscr;	/* floating point status register */
+};
+#endif /* __DARWIN_UNIX03 */
+
+#pragma pack(4)		/* Make sure the structure stays as we defined it */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_PPC_VECTOR_STATE	struct __darwin_ppc_vector_state
+_STRUCT_PPC_VECTOR_STATE
+{
+#if defined(__LP64__)
+	unsigned int	__save_vr[32][4];
+	unsigned int	__save_vscr[4];
+#else
+	unsigned long	__save_vr[32][4];
+	unsigned long	__save_vscr[4];
+#endif
+	unsigned int	__save_pad5[4];
+	unsigned int	__save_vrvalid;			/* VRs that have been saved */
+	unsigned int	__save_pad6[7];
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_PPC_VECTOR_STATE	struct ppc_vector_state
+_STRUCT_PPC_VECTOR_STATE
+{
+#if defined(__LP64__)
+	unsigned int	save_vr[32][4];
+	unsigned int	save_vscr[4];
+#else
+	unsigned long	save_vr[32][4];
+	unsigned long	save_vscr[4];
+#endif
+	unsigned int	save_pad5[4];
+	unsigned int	save_vrvalid;			/* VRs that have been saved */
+	unsigned int	save_pad6[7];
+};
+#endif /* __DARWIN_UNIX03 */
+
+#pragma pack()
+
+/*
+ * ppc_exception_state
+ *
+ * This structure corresponds to some additional state of the user
+ * registers as saved in the PCB upon kernel entry. They are only
+ * available if an exception is passed out of the kernel, and even
+ * then not all are guaranteed to be updated.
+ *
+ * Some padding is included in this structure which allows space for
+ * servers to store temporary values if need be, to maintain binary
+ * compatiblity.
+ */
+
+/* Exception state for 32-bit thread (on 32-bit processor) */
+/* Still available on 64-bit processors, but may fall short */
+/* of covering the full potential state (hi half available). */
+
+#pragma pack(4)	/* Make sure the structure stays as we defined it */
+
+#if __DARWIN_UNIX03
+#define _STRUCT_PPC_EXCEPTION_STATE	struct __darwin_ppc_exception_state
+_STRUCT_PPC_EXCEPTION_STATE
+{
+#if defined(__LP64__)
+	unsigned int __dar;		/* Fault registers for coredump */
+	unsigned int __dsisr;
+	unsigned int __exception;	/* number of powerpc exception taken */
+	unsigned int __pad0;		/* align to 16 bytes */
+	unsigned int __pad1[4];		/* space in PCB "just in case" */
+#else
+	unsigned long __dar;		/* Fault registers for coredump */
+	unsigned long __dsisr;
+	unsigned long __exception;	/* number of powerpc exception taken */
+	unsigned long __pad0;		/* align to 16 bytes */
+	unsigned long __pad1[4];	/* space in PCB "just in case" */
+#endif
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_PPC_EXCEPTION_STATE	struct ppc_exception_state
+_STRUCT_PPC_EXCEPTION_STATE
+{
+#if defined(__LP64__)
+	unsigned int dar;		/* Fault registers for coredump */
+	unsigned int dsisr;
+	unsigned int exception;		/* number of powerpc exception taken */
+	unsigned int pad0;		/* align to 16 bytes */
+	unsigned int pad1[4];		/* space in PCB "just in case" */
+#else
+	unsigned long dar;		/* Fault registers for coredump */
+	unsigned long dsisr;
+	unsigned long exception;	/* number of powerpc exception taken */
+	unsigned long pad0;		/* align to 16 bytes */
+	unsigned long pad1[4];		/* space in PCB "just in case" */
+#endif
+};
+#endif /* __DARWIN_UNIX03 */
+
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+#if __DARWIN_UNIX03
+#define _STRUCT_PPC_EXCEPTION_STATE64	struct __darwin_ppc_exception_state64
+_STRUCT_PPC_EXCEPTION_STATE64
+{
+	unsigned long long __dar;	/* Fault registers for coredump */
+#if defined(__LP64__)
+	unsigned int  __dsisr;
+	unsigned int  __exception;	/* number of powerpc exception taken */
+	unsigned int  __pad1[4];	/* space in PCB "just in case" */
+#else
+	unsigned long __dsisr;
+	unsigned long __exception;	/* number of powerpc exception taken */
+	unsigned long __pad1[4];	/* space in PCB "just in case" */
+#endif
+};
+#else /* !__DARWIN_UNIX03 */
+#define _STRUCT_PPC_EXCEPTION_STATE64	struct ppc_exception_state64
+_STRUCT_PPC_EXCEPTION_STATE64
+{
+	unsigned long long dar;		/* Fault registers for coredump */
+#if defined(__LP64__)
+	unsigned int  dsisr;
+	unsigned int  exception;	/* number of powerpc exception taken */
+	unsigned int  pad1[4];		/* space in PCB "just in case" */
+#else
+	unsigned long dsisr;
+	unsigned long exception;	/* number of powerpc exception taken */
+	unsigned long pad1[4];		/* space in PCB "just in case" */
+#endif
+};
+#endif /* __DARWIN_UNIX03 */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+
+#pragma pack()
+
+#endif /* _MACH_PPC__STRUCTS_H_ */
diff --git a/osfmk/mach/ppc/_types.h b/osfmk/mach/ppc/_types.h
index 098c1670a..fd3cb8f19 100644
--- a/osfmk/mach/ppc/_types.h
+++ b/osfmk/mach/ppc/_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,17 +32,19 @@
 #ifndef	_MACH_PPC__TYPES_H_
 #define	_MACH_PPC__TYPES_H_
 
+#include <sys/cdefs.h>
+
 /*
  * ppc_thread_state is the structure that is exported to user threads for 
  * use in status/mutate calls.  This structure should never change.
  *
  */
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct ppc_thread_state
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_ppc_thread_state
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 {
 	unsigned int srr0;      /* Instruction address register (PC) */
 	unsigned int srr1;	/* Machine state register (supervisor) */
@@ -82,7 +90,7 @@ struct __darwin_ppc_thread_state
 	unsigned int vrsave;	/* Vector Save Register */
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 #pragma pack(4)							/* Make sure the structure stays as we defined it */
 struct ppc_thread_state64 {
 	unsigned long long srr0;	/* Instruction address register (PC) */
@@ -129,15 +137,15 @@ struct ppc_thread_state64 {
 };
 
 #pragma pack()
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /* This structure should be double-word aligned for performance */
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct ppc_float_state
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_ppc_float_state
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 {
 	double  fpregs[32];
 
@@ -147,11 +155,11 @@ struct __darwin_ppc_float_state
 
 #pragma pack(4)		/* Make sure the structure stays as we defined it */
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct ppc_vector_state
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_ppc_vector_state
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 {
 #if defined(__LP64__)
 	unsigned int	save_vr[32][4];
@@ -185,11 +193,11 @@ struct __darwin_ppc_vector_state
 
 #pragma pack(4)	/* Make sure the structure stays as we defined it */
 
-#ifndef _POSIX_C_SOURCE
+#if !__DARWIN_UNIX03
 struct ppc_exception_state
-#else /* _POSIX_C_SOURCE */
+#else /* __DARWIN_UNIX03 */
 struct __darwin_ppc_exception_state
-#endif /* _POSIX_C_SOURCE */
+#endif /* __DARWIN_UNIX03 */
 {
 #if defined(__LP64__)
 	unsigned int dar;			/* Fault registers for coredump */
@@ -206,7 +214,7 @@ struct __darwin_ppc_exception_state
 #endif
 };
 
-#ifndef _POSIX_C_SOURCE
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
 struct ppc_exception_state64 {
 	unsigned long long dar;		/* Fault registers for coredump */
 #if defined(__LP64__)
@@ -219,7 +227,7 @@ struct ppc_exception_state64 {
 	unsigned long pad1[4];		/* space in PCB "just in case" */
 #endif
 };
-#endif /* _POSIX_C_SOURCE */
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 #pragma pack()
 
diff --git a/osfmk/mach/ppc/boolean.h b/osfmk/mach/ppc/boolean.h
index 3767a2115..aa3769c9f 100644
--- a/osfmk/mach/ppc/boolean.h
+++ b/osfmk/mach/ppc/boolean.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/exception.h b/osfmk/mach/ppc/exception.h
index 47232ac7a..da4e7cb6b 100644
--- a/osfmk/mach/ppc/exception.h
+++ b/osfmk/mach/ppc/exception.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -48,7 +54,7 @@
 #ifndef	_MACH_PPC_EXCEPTION_H_
 #define _MACH_PPC_EXCEPTION_H_
 
-#define	EXC_TYPES_COUNT	10	/* incl. illegal exception 0 */
+#define	EXC_TYPES_COUNT	11	/* incl. illegal exception 0 */
 
 #define EXCEPTION_CODE_MAX 2    /* elements in vector (code+subcode) */
 /*
diff --git a/osfmk/mach/ppc/kern_return.h b/osfmk/mach/ppc/kern_return.h
index 5ddff5a56..2c79023f2 100644
--- a/osfmk/mach/ppc/kern_return.h
+++ b/osfmk/mach/ppc/kern_return.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/machine_types.defs b/osfmk/mach/ppc/machine_types.defs
index 60eca8e0b..018f7809e 100644
--- a/osfmk/mach/ppc/machine_types.defs
+++ b/osfmk/mach/ppc/machine_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -106,3 +112,5 @@ type mach_vm_size_t = uint64_t;
 #endif	/* MACH_IPC_COMPAT */
 
 #endif	/* _PPC_VM_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/ppc/ndr_def.h b/osfmk/mach/ppc/ndr_def.h
index 818f9906c..cb012b2f4 100644
--- a/osfmk/mach/ppc/ndr_def.h
+++ b/osfmk/mach/ppc/ndr_def.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/processor_info.h b/osfmk/mach/ppc/processor_info.h
index 3195c2c23..168cb195d 100644
--- a/osfmk/mach/ppc/processor_info.h
+++ b/osfmk/mach/ppc/processor_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/mach/ppc/rpc.h b/osfmk/mach/ppc/rpc.h
index a5b6e1752..b3a274a2d 100644
--- a/osfmk/mach/ppc/rpc.h
+++ b/osfmk/mach/ppc/rpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/sdt_isa.h b/osfmk/mach/ppc/sdt_isa.h
new file mode 100644
index 000000000..c10e3e604
--- /dev/null
+++ b/osfmk/mach/ppc/sdt_isa.h
@@ -0,0 +1,427 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _MACH_PPC_SDT_ISA_H
+#define	_MACH_PPC_SDT_ISA_H
+
+/* #pragma ident	"@(#)sdt.h	1.7	05/06/08 SMI" */
+
+/*
+ * Only define when testing.  This makes the calls into actual calls to
+ * test functions.
+ */
+/* #define DTRACE_CALL_TEST */
+
+#define DTRACE_STRINGIFY(s) #s
+#define DTRACE_TOSTRING(s) DTRACE_STRINGIFY(s)
+
+#if defined(KERNEL)
+/*
+ * For the kernel, set an explicit global label do the symbol can be located
+ */
+#define DTRACE_LAB(p, n)                                                                        \
+    "__dtrace_probe$" DTRACE_TOSTRING(__LINE__) DTRACE_STRINGIFY(_##p##___##n)
+#define DTRACE_LABEL(p, n)		\
+      ".section __DATA, __data\n\t"	\
+      ".globl " DTRACE_LAB(p, n) "\n\t"	\
+       DTRACE_LAB(p, n) ":" ".long 1f""\n\t"	\
+       ".text" "\n\t"			\
+	"1:"
+#else	/* !KERNEL */
+#define DTRACE_LABEL(p, n)									\
+	"__dtrace_probe$" DTRACE_TOSTRING(__LINE__) DTRACE_STRINGIFY(_##p##___##n) ":"	"\n\t"
+#endif	/* !KERNEL */
+
+#ifdef DTRACE_CALL_TEST
+
+#define DTRACE_CALL(p,n)	\
+	DTRACE_LABEL(p,n)	\
+	DTRACE_CALL_INSN(p,n)
+
+#else	/* !DTRACE_CALL_TEST */
+
+#define DTRACE_CALL(p,n)	\
+	DTRACE_LABEL(p,n)	\
+	DTRACE_NOPS
+
+#endif	/* !DTRACE_CALL_TEST */
+
+#ifdef __ppc__
+
+#define DTRACE_NOPS			\
+	"nop"			"\n\t"
+
+#define DTRACE_CALL_INSN(p,n)						\
+	"bl _dtracetest" DTRACE_STRINGIFY(_##p##_##n)	"\n\t"
+
+#define ARG1_EXTENT	1
+#define ARGS2_EXTENT	2
+#define ARGS3_EXTENT	3
+#define ARGS4_EXTENT	4
+#define ARGS5_EXTENT	5
+#define ARGS6_EXTENT	6
+#define ARGS7_EXTENT	7
+#define ARGS8_EXTENT	8
+#define ARGS9_EXTENT	9
+#define ARGS10_EXTENT	10	
+
+#define DTRACE_CALL0ARGS(provider, name)							\
+	asm volatile (										\
+	              DTRACE_CALL(provider, name)						\
+	              "# eat trailing nl+tab from DTRACE_CALL"					\
+	              :										\
+	              :										\
+	);
+
+#define DTRACE_CALL1ARG(provider, name)								\
+	asm volatile ("subi r1,r1,0x20"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addi r1,r1,0x20"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3"									\
+	);
+
+#define DTRACE_CALL2ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x20"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x20"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4"								\
+	);
+
+#define DTRACE_CALL3ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x30"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x30"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5"							\
+	);
+
+#define DTRACE_CALL4ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x30"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x30"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6"							\
+	);
+
+#define DTRACE_CALL5ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x30"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x30"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7"						\
+	);
+
+#define DTRACE_CALL6ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x30"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              "lwz r8,0x14(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x30"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8"					\
+	);
+
+#define DTRACE_CALL7ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x40"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              "lwz r8,0x14(%0)"							"\n\t"	\
+	              "lwz r9,0x18(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x40"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9"				\
+	);
+
+#define DTRACE_CALL8ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x40"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              "lwz r8,0x14(%0)"							"\n\t"	\
+	              "lwz r9,0x18(%0)"							"\n\t"	\
+	              "lwz r10,0x1c(%0)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x40"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10"				\
+	);
+
+#define DTRACE_CALL9ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x40"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              "lwz r8,0x14(%0)"							"\n\t"	\
+	              "lwz r9,0x18(%0)"							"\n\t"	\
+	              "lwz r10,0x1c(%0)"						"\n\t"	\
+	              "lwz r11,0x20(%0)"						"\n\t"	\
+	              "stw r11,0x38(r1)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x40"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11"			\
+	);
+
+#define DTRACE_CALL10ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x40"							"\n\t"	\
+	              "lwz r3,0x0(%0)"							"\n\t"	\
+	              "lwz r4,0x4(%0)"							"\n\t"	\
+	              "lwz r5,0x8(%0)"							"\n\t"	\
+	              "lwz r6,0xc(%0)"							"\n\t"	\
+	              "lwz r7,0x10(%0)"							"\n\t"	\
+	              "lwz r8,0x14(%0)"							"\n\t"	\
+	              "lwz r9,0x18(%0)"							"\n\t"	\
+	              "lwz r10,0x1c(%0)"						"\n\t"	\
+	              "lwz r11,0x20(%0)"						"\n\t"	\
+	              "lwz r12,0x24(%0)"						"\n\t"	\
+	              "stw r11,0x38(r1)"						"\n\t"	\
+	              "stw r12,0x3c(r1)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x40"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12"		\
+	);
+
+#endif // __ppc__
+
+#ifdef __ppc64__
+
+#define DTRACE_NOPS			\
+	"nop"			"\n\t"
+
+#define DTRACE_CALL_INSN(p,n)						\
+	"bl _dtracetest" DTRACE_STRINGIFY(_##p##_##n)	"\n\t"
+
+#define ARG1_EXTENT	1
+#define ARGS2_EXTENT	2
+#define ARGS3_EXTENT	3
+#define ARGS4_EXTENT	4
+#define ARGS5_EXTENT	5
+#define ARGS6_EXTENT	6
+#define ARGS7_EXTENT	7
+#define ARGS8_EXTENT	8
+#define ARGS9_EXTENT	9
+#define ARGS10_EXTENT	10	
+
+#define DTRACE_CALL0ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x30"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x30"								\
+	              :										\
+	              :										\
+	              :										\
+	);
+
+#define DTRACE_CALL1ARG(provider, name)								\
+	asm volatile ("ld r3,0x0(%0)"							"\n\t"	\
+	              "subi r1,r1,0x38"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+                      "addi r1,r1,0x38"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3"									\
+	);
+
+#define DTRACE_CALL2ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x40"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x40"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4"								\
+	);
+
+#define DTRACE_CALL3ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x48"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x48"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5"							\
+	);
+
+#define DTRACE_CALL4ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x50"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x50"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6"							\
+	);
+
+#define DTRACE_CALL5ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x58"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x58"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7"						\
+	);
+
+#define DTRACE_CALL6ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x60"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              "ld r8,0x28(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x60"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8"					\
+	);
+
+#define DTRACE_CALL7ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x68"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              "ld r8,0x28(%0)"							"\n\t"	\
+	              "ld r9,0x30(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x68"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9"				\
+	);
+
+#define DTRACE_CALL8ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x70"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              "ld r8,0x28(%0)"							"\n\t"	\
+	              "ld r9,0x30(%0)"							"\n\t"	\
+	              "ld r10,0x38(%0)"							"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x70"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10"				\
+	);
+
+#define DTRACE_CALL9ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x78"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              "ld r8,0x28(%0)"							"\n\t"	\
+	              "ld r9,0x30(%0)"							"\n\t"	\
+	              "ld r10,0x38(%0)"							"\n\t"	\
+	              "ld r11,0x40(%0)"							"\n\t"	\
+	              "std r11,0x70(r1)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x78"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11"			\
+	);
+
+#define DTRACE_CALL10ARGS(provider, name)							\
+	asm volatile ("subi r1,r1,0x80"							"\n\t"	\
+	              "ld r3,0x0(%0)"							"\n\t"	\
+	              "ld r4,0x8(%0)"							"\n\t"	\
+	              "ld r5,0x10(%0)"							"\n\t"	\
+	              "ld r6,0x18(%0)"							"\n\t"	\
+	              "ld r7,0x20(%0)"							"\n\t"	\
+	              "ld r8,0x28(%0)"							"\n\t"	\
+	              "ld r9,0x30(%0)"							"\n\t"	\
+	              "ld r10,0x38(%0)"							"\n\t"	\
+	              "ld r11,0x40(%0)"							"\n\t"	\
+	              "ld r12,0x48(%0)"							"\n\t"	\
+	              "std r11,0x70(r1)"						"\n\t"	\
+	              "std r12,0x78(r1)"						"\n\t"	\
+	              DTRACE_CALL(provider, name)						\
+	              "addi r1,r1,0x80"								\
+	              :										\
+	              : "b" (__dtrace_args)							\
+	              : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12"		\
+	);
+
+#endif // __ppc64__
+
+#endif	/* _MACH_PPC_SDT_ISA_H */
diff --git a/osfmk/mach/ppc/syscall_sw.h b/osfmk/mach/ppc/syscall_sw.h
index 9e4c01ab8..335ff9e21 100644
--- a/osfmk/mach/ppc/syscall_sw.h
+++ b/osfmk/mach/ppc/syscall_sw.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/thread_state.h b/osfmk/mach/ppc/thread_state.h
index 668e0f8fa..3ab7baa1f 100644
--- a/osfmk/mach/ppc/thread_state.h
+++ b/osfmk/mach/ppc/thread_state.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/ppc/thread_status.h b/osfmk/mach/ppc/thread_status.h
index ef389380e..ba077f74d 100644
--- a/osfmk/mach/ppc/thread_status.h
+++ b/osfmk/mach/ppc/thread_status.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,7 +32,7 @@
 #ifndef	_MACH_PPC_THREAD_STATUS_H_
 #define _MACH_PPC_THREAD_STATUS_H_
 
-#include <mach/ppc/_types.h>
+#include <mach/ppc/_structs.h>
 #include <mach/message.h>
 
 /*
@@ -59,27 +65,12 @@
          (x == PPC_EXCEPTION_STATE64)   || \
          (x == THREAD_STATE_NONE))
 
-#ifndef _POSIX_C_SOURCE
-typedef struct ppc_thread_state			ppc_thread_state_t;
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ppc_thread_state	ppc_thread_state_t;
-#endif /* _POSIX_C_SOURCE */
-
-#ifndef _POSIX_C_SOURCE
-typedef struct ppc_thread_state64		ppc_thread_state64_t;
-#endif /* _POSIX_C_SOURCE */
-
-#ifndef _POSIX_C_SOURCE
-typedef struct ppc_float_state			ppc_float_state_t;
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ppc_float_state		ppc_float_state_t;
-#endif /* _POSIX_C_SOURCE */
-
-#ifndef _POSIX_C_SOURCE
-typedef struct ppc_vector_state			ppc_vector_state_t;
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ppc_vector_state	ppc_vector_state_t;
-#endif /* _POSIX_C_SOURCE */
+typedef _STRUCT_PPC_THREAD_STATE	ppc_thread_state_t;
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+typedef _STRUCT_PPC_THREAD_STATE64	ppc_thread_state64_t;
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
+typedef _STRUCT_PPC_FLOAT_STATE		ppc_float_state_t;
+typedef _STRUCT_PPC_VECTOR_STATE	ppc_vector_state_t;
 
 /*
  * saved state structure
@@ -117,12 +108,10 @@ typedef struct ppc_thread_state			ppc_saved_state_t;
 /* Still available on 64-bit processors, but may fall short */
 /* of covering the full potential state (hi half available). */
 
-#ifndef _POSIX_C_SOURCE
-typedef struct ppc_exception_state		ppc_exception_state_t;
-typedef struct ppc_exception_state64		ppc_exception_state64_t;
-#else /* _POSIX_C_SOURCE */
-typedef struct __darwin_ppc_exception_state	ppc_exception_state_t;
-#endif /* _POSIX_C_SOURCE */
+typedef _STRUCT_PPC_EXCEPTION_STATE	ppc_exception_state_t;
+#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE)
+typedef _STRUCT_PPC_EXCEPTION_STATE64	ppc_exception_state64_t;
+#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */
 
 /*
  * Save State Flags
diff --git a/osfmk/mach/ppc/vm_param.h b/osfmk/mach/ppc/vm_param.h
index 7a8a1ca84..ec18cb693 100644
--- a/osfmk/mach/ppc/vm_param.h
+++ b/osfmk/mach/ppc/vm_param.h
@@ -1,24 +1,29 @@
-
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -72,8 +77,8 @@
 #ifdef	KERNEL_PRIVATE
 
 /* Kernel-wide values */
-#define VM_MIN_KERNEL_ADDRESS	((vm_offset_t) 0x00001000)
-#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xDFFFFFFF)
+#define VM_MIN_KERNEL_ADDRESS	((vm_offset_t) 0x00001000U)
+#define VM_MAX_KERNEL_ADDRESS	((vm_offset_t) 0xDFFFFFFFU)
 #define KERNEL_STACK_SIZE		(4 * PPC_PGBYTES)
 #define INTSTACK_SIZE		(5 * PPC_PGBYTES)
 
diff --git a/osfmk/mach/ppc/vm_types.h b/osfmk/mach/ppc/vm_types.h
index b5b0bd8a4..8a3137bd2 100644
--- a/osfmk/mach/ppc/vm_types.h
+++ b/osfmk/mach/ppc/vm_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -121,16 +127,9 @@ typedef uint64_t		mach_vm_address_t;
 typedef uint64_t		mach_vm_offset_t;
 typedef uint64_t		mach_vm_size_t;
 
-/* LP64todo - convert these over for good */
-#if 1 
 typedef uint64_t		vm_map_offset_t;
 typedef uint64_t		vm_map_address_t;
 typedef uint64_t		vm_map_size_t;
-#else
-typedef uint32_t		vm_map_offset_t;
-typedef uint32_t		vm_map_address_t;
-typedef uint32_t		vm_map_size_t;
-#endif
 
 #ifdef	MACH_KERNEL_PRIVATE
 
diff --git a/osfmk/mach/processor.defs b/osfmk/mach/processor.defs
index a04091c76..f590633f1 100644
--- a/osfmk/mach/processor.defs
+++ b/osfmk/mach/processor.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -118,3 +124,5 @@ routine processor_assign(
 routine processor_get_assignment(
 		processor	: processor_t;
 	out	assigned_set	: processor_set_name_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/processor_info.h b/osfmk/mach/processor_info.h
index c9dd97fec..bd1f74056 100644
--- a/osfmk/mach/processor_info.h
+++ b/osfmk/mach/processor_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -103,7 +109,7 @@ typedef struct processor_basic_info	*processor_basic_info_t;
 		(sizeof(processor_basic_info_data_t)/sizeof(natural_t)))
 
 struct processor_cpu_load_info {             /* number of ticks while running... */
-        unsigned long   cpu_ticks[CPU_STATE_MAX]; /* ... in the given mode */
+        unsigned int	cpu_ticks[CPU_STATE_MAX]; /* ... in the given mode */
 }; 
 
 typedef	struct processor_cpu_load_info	processor_cpu_load_info_data_t;
diff --git a/osfmk/mach/processor_set.defs b/osfmk/mach/processor_set.defs
index 796f989bb..dc0f40774 100644
--- a/osfmk/mach/processor_set.defs
+++ b/osfmk/mach/processor_set.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -137,7 +143,7 @@ routine processor_set_policy_control(
  */
 routine processor_set_stack_usage(
 		pset		: processor_set_t;
-	out	total		: unsigned;
+	out	ltotal		: unsigned;
 	out	space		: vm_size_t;
 	out	resident	: vm_size_t;
 	out	maxusage	: vm_size_t;
@@ -152,3 +158,4 @@ routine processor_set_info(
         out     host            : host_t;
         out     info_out        : processor_set_info_t, CountInOut);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/prof.defs b/osfmk/mach/prof.defs
index 9c140a6b8..a334d6c79 100644
--- a/osfmk/mach/prof.defs
+++ b/osfmk/mach/prof.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -126,6 +132,4 @@ simpleroutine		notices(
 			samples		: sample_array_t;
 	msgoption	options		: mach_msg_options_t);
 
-
-
-
+/* vim: set ft=c : */
diff --git a/osfmk/mach/prof_types.h b/osfmk/mach/prof_types.h
index e7096ddc4..a41037212 100644
--- a/osfmk/mach/prof_types.h
+++ b/osfmk/mach/prof_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/mach/rpc.h b/osfmk/mach/rpc.h
index 079e92705..565d7ac6d 100644
--- a/osfmk/mach/rpc.h
+++ b/osfmk/mach/rpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/sdt.h b/osfmk/mach/sdt.h
new file mode 100644
index 000000000..3268551b2
--- /dev/null
+++ b/osfmk/mach/sdt.h
@@ -0,0 +1,32 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#ifndef _MACH_SDT_H
+#define	_MACH_SDT_H
+
+#include <mach/machine/sdt.h>
+
+#endif	/* _MACH_SDT_H */
diff --git a/osfmk/mach/security.defs b/osfmk/mach/security.defs
new file mode 100644
index 000000000..734aa90eb
--- /dev/null
+++ b/osfmk/mach/security.defs
@@ -0,0 +1,208 @@
+
+/* -*- C++ -*- */
+
+subsystem
+#if	KERNEL_SERVER
+	  KernelServer
+#endif	KERNEL_SERVER
+	  security 5200;
+
+#include <mach/std_types.defs>
+#include <mach/mach_types.defs>
+
+type labelstr_t = c_string[*:512];
+
+/**
+  @brief Retrieve a task label as a label handle
+  @param task Target's task port
+  @param label Returned label handle
+
+  This call retrieves the label handle of the specified task.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mach_get_task_label(task      : ipc_space_t;
+			    out label : mach_port_name_t);
+
+/**
+  @brief Retrieve a task label in textual form
+  @param task Target's task port
+  @param policies Comma-delimited list of policies to query
+  @param label Returned label text
+
+  This call retrieves an externalized task label for the
+  specified task, with respect to the specified policies.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mach_get_task_label_text(task      : ipc_space_t;
+				 policies  : labelstr_t;
+				 out label : labelstr_t);
+
+/**
+  @brief Retrieve a port label as a label handle
+  @param task Issuer's task port
+  @param port Port to query label from
+  @param label Returned label handle
+
+  This call retrieves a new label handle for the specified port.
+  If the port represents a label handle, KERN_INVALID_ARGUMENT is
+  returned.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mach_get_label(task      : ipc_space_t;
+		       port      : mach_port_name_t;
+		       out label : mach_port_name_t);
+
+/**
+  @brief Retrieve a port label in textual form
+  @param task Issuer's task port
+  @param name Port to query label from
+  @param policies Comma-delimited list of policies to query
+  @param label Returned label text
+
+  This call retrieves an externalized port label for the specified port,
+  with respect to the specified policies. If the port represents a label
+  handle, the returned label text refers to the stored label and not the
+  access control label.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mach_get_label_text(task      : ipc_space_t;
+			    name      : mach_port_name_t;
+			    policies  : labelstr_t;
+			    out label : labelstr_t);
+
+/**
+  @brief Relabel a port
+  @param task Task containing specified ports
+  @param name Port to relabel
+  @param label String representation of new label
+
+  This call attempts to relabel the specified port to the
+  label specified. For label handles, it changes the access control
+  label and not the stored label.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mach_set_port_label(task      : ipc_space_t;
+			    name      : mach_port_name_t;
+			    label     : labelstr_t);
+
+/**
+  @brief Generic access control check
+  @param task Any task port
+  @param subj subject label in textual form
+  @param obj object label in textual form
+  @param serv Service or object class name
+  @param perm Permission, or method, within the specified service
+
+  This function provides a general way for a user process to query
+  an arbitrary access control decision from the system's security policies.
+  Currently, there are no standards for the format of the service and
+  permission names. 
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mac_check_service(task    : ipc_space_t;
+			  subject : labelstr_t;
+			  object  : labelstr_t;
+			  service : labelstr_t;
+			  perm    : labelstr_t);
+
+/**
+  @brief Generic access control check
+  @param task Task containing specified ports (usually caller's)
+  @param subj subject label in textual form
+  @param obj port containing object label
+  @param serv Service or object class name
+  @param perm Permission, or method, within the specified service
+
+  This function provides a general way for a user process to query
+  an arbitrary access control decision from the system's security policies.
+  Currently, there are no standards for the format of the service and
+  permission names. If the port is a label handle, the stored label is
+  used. Otherwise, its access control label is used.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mac_port_check_service_obj(task    : ipc_space_t;
+				   subject : labelstr_t;
+				   object  : mach_port_name_t;
+				   service : labelstr_t;
+				   perm    : labelstr_t);
+
+/**
+  @brief Generic access control check
+  @param task Task containing specified ports (usually caller's)
+  @param subj port containing subject label
+  @param obj port containing object label
+  @param serv Service or object class name
+  @param perm Permission, or method, within the specified service
+
+  This function provides a general way for a user process to query
+  an arbitrary access control decision from the system's security policies.
+  Currently, there are no standards for the format of the service and
+  permission names. If any ports are label handles, the stored label is
+  used. Otherwise, the access control labels are used.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mac_port_check_access(task    : ipc_space_t;
+			      subject : mach_port_name_t;
+			      object  : mach_port_name_t;
+			      service : labelstr_t;
+			      perm    : labelstr_t);
+
+/**
+  @brief Create a new label handle
+  @param task Task to receive new ports (usually caller's)
+  @param name Returned label handle port
+  @param label String representation of new label
+
+  Creates a new label handle, with the stored label defined by the
+  given text. Any task may create a label handle with any valid label,
+  not necessarily one that it has permission to access. A port right
+  for the new label handle is inserted into the specified task.
+  Posession of label handles should not imply any security properties.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mac_label_new(task     : ipc_space_t;
+		      out name : mach_port_name_t;
+		      label    : labelstr_t);
+
+/**
+  @brief Request a computed label
+  @param task subj,obj,newlabel relative to this task (usually caller's)
+  @param subj port containing subject label
+  @param obj port containing object label
+  @param serv Service or object class name
+  @param newlabel Returned label handle port
+  
+  Ask the loaded policies to compute a label based on the two input labels
+  and the service name. There is currently no standard for the service name,
+  or even what the input labels represent (Subject and parent object are only
+  a suggestion). If any ports are label handles, the stored label is
+  used. Otherwise, the access control labels are used. A new label handle
+  is created to contain the computed label, and is stored into the
+  specified task.
+
+  @return Standard MiG return values (0 for success)
+*/
+
+routine mac_request_label(task         : ipc_space_t;
+			  subject      : mach_port_name_t;
+			  object       : mach_port_name_t;
+			  service      : labelstr_t;
+			  out newlabel : mach_port_name_t);
diff --git a/osfmk/mach/semaphore.defs b/osfmk/mach/semaphore.defs
index 468b8c94f..bbbdbedb1 100644
--- a/osfmk/mach/semaphore.defs
+++ b/osfmk/mach/semaphore.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -73,3 +79,5 @@ routine semaphore_timedwait_signal(
 		wait_semaphore	: semaphore_t;
 		signal_semaphore: semaphore_t;
 		wait_time	: mach_timespec_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/semaphore.h b/osfmk/mach/semaphore.h
index e5241b51f..f90bbcd24 100644
--- a/osfmk/mach/semaphore.h
+++ b/osfmk/mach/semaphore.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_SEMAPHORE_H_
diff --git a/osfmk/mach/shared_memory_server.h b/osfmk/mach/shared_memory_server.h
index af2aea45c..d2ba99b45 100644
--- a/osfmk/mach/shared_memory_server.h
+++ b/osfmk/mach/shared_memory_server.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
@@ -26,6 +32,16 @@
  * 	protos and struct definitions for shared library
  *	server and interface
  */
+
+/*
+ * XXX
+ * 
+ * NOTE: this file is deprecated and will be removed in the near future.
+ * Any project that includes this file should be changed to:
+ * 1. use <mach/shared_region.h> instead of this file,
+ * 2. handle the new shared regions, now available on more platforms
+ */
+
 #ifndef _MACH_SHARED_MEMORY_SERVER_H_
 #define _MACH_SHARED_MEMORY_SERVER_H_
 
@@ -37,16 +53,17 @@
 #define VM_PROT_COW  0x8  /* must not interfere with normal prot assignments */
 #define VM_PROT_ZF  0x10  /* must not interfere with normal prot assignments */
 
+#define GLOBAL_SHARED_TEXT_SEGMENT	0x90000000U
+#define GLOBAL_SHARED_DATA_SEGMENT	0xA0000000U
+#define GLOBAL_SHARED_SEGMENT_MASK	0xF0000000U
+
+#define	SHARED_TEXT_REGION_SIZE		0x10000000
+#define	SHARED_DATA_REGION_SIZE		0x10000000
 
 #if !defined(__LP64__)
 
 #define	SHARED_LIBRARY_SERVER_SUPPORTED
-#define GLOBAL_SHARED_TEXT_SEGMENT	0x90000000
-#define GLOBAL_SHARED_DATA_SEGMENT	0xA0000000
-#define GLOBAL_SHARED_SEGMENT_MASK	0xF0000000
 
-#define	SHARED_TEXT_REGION_SIZE		0x10000000
-#define	SHARED_DATA_REGION_SIZE		0x10000000
 #define	SHARED_ALTERNATE_LOAD_BASE	0x09000000
 
 /* 
@@ -58,8 +75,6 @@
 #define	SHARED_TEXT_REGION_MASK		0x0FFFFFFF
 #define	SHARED_DATA_REGION_MASK		0x0FFFFFFF
 
-#define SHARED_LIB_ALIAS  0x10
-
 
 /* flags field aliases for copyin_shared_file and load_shared_file */
 
@@ -82,16 +97,6 @@ struct sf_mapping {
 };
 typedef struct sf_mapping sf_mapping_t;
 
-#ifndef KERNEL
-/* load_shared_file and friends is deprecated */
-__BEGIN_DECLS
-int	load_shared_file(char *, caddr_t, u_long,
-		caddr_t *, int, sf_mapping_t *, int *);
-int	reset_shared_file(caddr_t *, int, sf_mapping_t *);
-int	new_system_shared_regions(void);
-__END_DECLS
-#endif /* !KERNEL */
-
 #endif  /* !defined(__LP64__) */
 
 /* 
diff --git a/osfmk/mach/shared_region.h b/osfmk/mach/shared_region.h
new file mode 100644
index 000000000..ab5a722e1
--- /dev/null
+++ b/osfmk/mach/shared_region.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+/*
+ *
+ *	File: mach/shared_region.h
+ *
+ * 	protos and struct definitions for shared region
+ */
+
+#ifndef _MACH_SHARED_REGION_H_
+#define _MACH_SHARED_REGION_H_
+
+#include <sys/cdefs.h>
+#include <mach/vm_prot.h>
+#include <mach/vm_types.h>
+#include <mach/mach_types.h>
+
+#define SHARED_REGION_BASE_I386			0x90000000ULL
+#define SHARED_REGION_SIZE_I386			0x20000000ULL
+#define SHARED_REGION_NESTING_BASE_I386		0x90000000ULL
+#define SHARED_REGION_NESTING_SIZE_I386		0x20000000ULL
+#define SHARED_REGION_NESTING_MIN_I386		0x00020000ULL
+#define SHARED_REGION_NESTING_MAX_I386		0xFFFE0000ULL
+
+#define SHARED_REGION_BASE_X86_64		0x00007FFF60000000ULL
+#define SHARED_REGION_SIZE_X86_64		0x000000009FE00000ULL
+#define SHARED_REGION_NESTING_BASE_X86_64	0x00007FFF60000000ULL
+#define SHARED_REGION_NESTING_SIZE_X86_64	0x000000009FE00000ULL
+#define SHARED_REGION_NESTING_MIN_X86_64	0x0000000000020000ULL
+#define SHARED_REGION_NESTING_MAX_X86_64	0xFFFFFFFFFFFE0000ULL
+
+#define SHARED_REGION_BASE_PPC			0x90000000ULL
+#define SHARED_REGION_SIZE_PPC			0x20000000ULL
+#define SHARED_REGION_NESTING_BASE_PPC		0x90000000ULL
+#define SHARED_REGION_NESTING_SIZE_PPC		0x10000000ULL
+#define SHARED_REGION_NESTING_MIN_PPC		0x10000000ULL
+#define SHARED_REGION_NESTING_MAX_PPC		0x10000000ULL
+
+#define SHARED_REGION_BASE_PPC64		0x00007FFF60000000ULL
+#define SHARED_REGION_SIZE_PPC64		0x00000000A0000000ULL
+#define SHARED_REGION_NESTING_BASE_PPC64	0x00007FFF60000000ULL
+#define SHARED_REGION_NESTING_SIZE_PPC64	0x00000000A0000000ULL
+#define SHARED_REGION_NESTING_MIN_PPC64		0x0000000010000000ULL
+#define SHARED_REGION_NESTING_MAX_PPC64		0x0000000010000000ULL
+
+#define SHARED_REGION_BASE_ARM			0x30000000ULL
+#define SHARED_REGION_SIZE_ARM			0x10000000ULL
+#define SHARED_REGION_NESTING_BASE_ARM		0x30000000ULL
+#define SHARED_REGION_NESTING_SIZE_ARM		0x08000000ULL
+#define SHARED_REGION_NESTING_MIN_ARM		?
+#define SHARED_REGION_NESTING_MAX_ARM		?
+
+/* 
+ * All shared_region_* declarations are a private interface
+ * between dyld and the kernel.
+ *
+ */
+struct shared_file_mapping_np {
+	mach_vm_address_t	sfm_address;
+	mach_vm_size_t		sfm_size;
+	mach_vm_offset_t	sfm_file_offset;
+	vm_prot_t		sfm_max_prot;
+	vm_prot_t		sfm_init_prot;
+};
+#define VM_PROT_COW  0x8  /* must not interfere with normal prot assignments */
+#define VM_PROT_ZF  0x10  /* must not interfere with normal prot assignments */
+
+#ifndef KERNEL
+
+__BEGIN_DECLS
+int	shared_region_check_np(uint64_t	*startaddress);
+int	shared_region_map_np(int fd,
+			     uint32_t mappingCount,
+			     const struct shared_file_mapping_np *mappings);
+__END_DECLS
+
+#endif /* !KERNEL */
+
+#endif /* _MACH_SHARED_REGION_H_ */
diff --git a/osfmk/mach/std_types.defs b/osfmk/mach/std_types.defs
index 339810085..0b483836f 100644
--- a/osfmk/mach/std_types.defs
+++ b/osfmk/mach/std_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -140,3 +146,5 @@ import <mach/std_types.h>;
 import <mach/mig.h>;
 
 #endif	/* _MACH_STD_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/std_types.h b/osfmk/mach/std_types.h
index d28084437..f5b838ab2 100644
--- a/osfmk/mach/std_types.h
+++ b/osfmk/mach/std_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/sync.defs b/osfmk/mach/sync.defs
index fb011c404..92ce0eb14 100644
--- a/osfmk/mach/sync.defs
+++ b/osfmk/mach/sync.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -135,3 +141,5 @@ routine semaphore_signal_thread(
 routine semaphore_timedwait(
 		semaphore	: semaphore_t;
 		wait_time	: mach_timespec_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/sync_policy.h b/osfmk/mach/sync_policy.h
index 132154756..50e0b2142 100644
--- a/osfmk/mach/sync_policy.h
+++ b/osfmk/mach/sync_policy.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/syscall_sw.h b/osfmk/mach/syscall_sw.h
index a3c028995..aa7508d76 100644
--- a/osfmk/mach/syscall_sw.h
+++ b/osfmk/mach/syscall_sw.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -131,10 +137,10 @@ kernel_trap(mk_timer_arm,-93,3)
 
 kernel_trap(mk_timer_cancel,-94,2)
 
-#if		!defined(__LP64__)
-kernel_trap(MKGetTimeBaseInfo,-95,5)
-#endif	/* __LP64__ */
-
+/*
+ * N.B: Trap #-100 is in use by IOTrap.s in the IOKit Framework
+ * (iokit_user_client_trap)
+ */
 #endif	/* _MACH_SYSCALL_SW_H_ */
 
 #endif	/* PRIVATE */
diff --git a/osfmk/mach/task.defs b/osfmk/mach/task.defs
index f1743eb1a..d8f69f47c 100644
--- a/osfmk/mach/task.defs
+++ b/osfmk/mach/task.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -264,11 +270,15 @@ out	policy_info		: task_policy_t, CountInOut;
 inout	get_default		: boolean_t);
 
 /*
- * Task profiling.
+ *	Removed from the kernel.
  */
+#if KERNEL_SERVER
+skip;
+#else
 routine task_sample(
 		task		: task_t;
 		reply		: mach_port_make_send_t);
+#endif
 
 /*
  * JMM - Everything from here down is likely to go away soon
@@ -361,3 +371,4 @@ routine task_set_policy(
 	limit			: policy_limit_t;
 	change			: boolean_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/task_access.defs b/osfmk/mach/task_access.defs
new file mode 100644
index 000000000..7035dbece
--- /dev/null
+++ b/osfmk/mach/task_access.defs
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * @OSF_COPYRIGHT@
+ */
+
+subsystem
+#if	KERNEL_USER
+		KernelUser
+#endif	/* KERN_USER */
+			task_access 27000;
+
+#include <mach/std_types.defs>
+#include <mach/mach_types.defs>
+
+/*
+ *	Verify task_for_pid access for the given pid
+ *	Access granted by return value (success/failure)
+ */
+routine check_task_access(
+		task_access_port			: mach_port_t;
+		calling_pid				: int32_t;
+		calling_gid				: uint32_t;
+		target_pid				: int32_t;
+		ServerAuditToken	caller_cred 	: audit_token_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/task_info.h b/osfmk/mach/task_info.h
index 398892373..1a1b1ec95 100644
--- a/osfmk/mach/task_info.h
+++ b/osfmk/mach/task_info.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,8 +53,6 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
-/*
- */
 /*
  *	Machine-independent task information structures and definitions.
  *
@@ -84,6 +88,7 @@ typedef	integer_t	task_info_data_t[TASK_INFO_MAX];
 #pragma pack(4)
 
 #define TASK_BASIC_INFO_32      4       /* basic information */
+#define TASK_BASIC2_INFO_32      6
 
 struct task_basic_info_32 {
         integer_t       suspend_count;  /* suspend count for task */
@@ -198,6 +203,20 @@ typedef struct task_absolutetime_info	*task_absolutetime_info_t;
 #define TASK_AUDIT_TOKEN_COUNT	\
 		(sizeof(audit_token_t) / sizeof(natural_t))
 
+
+#define TASK_AFFINITY_TAG_INFO		16 	/* This is experimental. */
+
+struct task_affinity_tag_info {
+	integer_t		set_count;
+	integer_t		min;
+	integer_t		max;
+	integer_t		task_count;
+};
+typedef struct task_affinity_tag_info	task_affinity_tag_info_data_t;
+typedef struct task_affinity_tag_info	*task_affinity_tag_info_t;
+#define TASK_AFFINITY_TAG_INFO_COUNT	\
+		(sizeof(task_affinity_tag_info_data_t) / sizeof(natural_t))
+
 #pragma pack()
 
 
diff --git a/osfmk/mach/task_ledger.h b/osfmk/mach/task_ledger.h
index 6076266a8..5d4bc2585 100644
--- a/osfmk/mach/task_ledger.h
+++ b/osfmk/mach/task_ledger.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/task_policy.h b/osfmk/mach/task_policy.h
index f4d108896..a07f4c98f 100644
--- a/osfmk/mach/task_policy.h
+++ b/osfmk/mach/task_policy.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_TASK_POLICY_H_
diff --git a/osfmk/mach/task_special_ports.h b/osfmk/mach/task_special_ports.h
index 9596805d8..99cd63450 100644
--- a/osfmk/mach/task_special_ports.h
+++ b/osfmk/mach/task_special_ports.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -80,6 +86,14 @@ typedef	int	task_special_port_t;
 
 #define TASK_PAGED_LEDGER_PORT	6	/* Paged resource ledger for task. */
 
+#define TASK_SEATBELT_PORT	7	/* Seatbelt compiler/DEM port for task. */
+
+#define TASK_GSSD_PORT		8	/* GSSD port for security context */
+
+#define TASK_ACCESS_PORT	9	/* Permission check for task_for_pid. */
+
+#define TASK_AUTOMOUNTD_PORT	10	/* Port to talk to the automounter */
+
 #define task_get_wired_ledger_port(task, port)	\
 		(task_get_special_port((task), TASK_WIRED_LEDGER_PORT, (port)))
 
@@ -114,4 +128,22 @@ typedef	int	task_special_port_t;
 #define task_set_bootstrap_port(task, port)	\
 		(task_set_special_port((task), TASK_BOOTSTRAP_PORT, (port)))
 
+#define task_get_gssd_port(task, port)	\
+		(task_get_special_port((task), TASK_GSSD_PORT, (port)))
+
+#define task_set_gssd_port(task, port)	\
+		(task_set_special_port((task), TASK_GSSD_PORT, (port)))
+
+#define task_get_task_access_port(task, port)	\
+		(task_get_special_port((task), TASK_ACCESS_PORT, (port)))
+
+#define task_set_task_access_port(task, port)	\
+		(task_set_special_port((task), TASK_ACCESS_PORT, (port)))
+
+#define task_get_automountd_port(task, port)  \
+                (task_get_special_port((task), TASK_AUTOMOUNTD_PORT, (port)))
+
+#define task_set_automountd_port(task, port)  \
+                (task_set_special_port((task), TASK_AUTOMOUNTD_PORT, (port)))
+
 #endif	/* _MACH_TASK_SPECIAL_PORTS_H_ */
diff --git a/osfmk/mach/thread_act.defs b/osfmk/mach/thread_act.defs
index 0478b1cfd..47a21a9e6 100644
--- a/osfmk/mach/thread_act.defs
+++ b/osfmk/mach/thread_act.defs
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -262,13 +268,15 @@ out	policy_info		: thread_policy_t, CountInOut;
 inout	get_default		: boolean_t);
 
 /*
- * Thread profiling
- * This call is only valid for MACH_PROF compiled kernels.
- * Otherwise, KERN_FAILURE is returned.
+ *	Removed from the kernel.
  */
+#if KERNEL_SERVER
+skip;
+#else
 routine thread_sample(
 		thread		: thread_act_t;
 		reply		: mach_port_make_send_t);
+#endif
 
 /*
  *	ETAP has been removed from the kernel.
@@ -311,3 +319,5 @@ routine thread_set_policy(
 	policy			: policy_t;
 	base			: policy_base_t;
 	limit			: policy_limit_t);
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach/thread_info.h b/osfmk/mach/thread_info.h
index e324aa5fc..e59b3d71a 100644
--- a/osfmk/mach/thread_info.h
+++ b/osfmk/mach/thread_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/thread_policy.h b/osfmk/mach/thread_policy.h
index 057d26831..d9530b776 100644
--- a/osfmk/mach/thread_policy.h
+++ b/osfmk/mach/thread_policy.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACH_THREAD_POLICY_H_
@@ -177,4 +183,36 @@ typedef struct thread_precedence_policy		*thread_precedence_policy_t;
 #define THREAD_PRECEDENCE_POLICY_COUNT	((mach_msg_type_number_t) \
 	(sizeof (thread_precedence_policy_data_t) / sizeof (integer_t)))
 
+/*
+ * THREAD_AFFINITY_POLICY:
+ *
+ * This policy is experimental.
+ * This may be used to express affinity relationships 
+ * between threads in the task. Threads with the same affinity tag will
+ * be scheduled to share an L2 cache if possible. That is, affinity tags
+ * are a hint to the scheduler for thread placement. 
+ *
+ * The namespace of affinity tags is generally local to one task. However,
+ * a child task created after the assignment of affinity tags by its parent
+ * will share that namespace. In particular, a family of forked processes
+ * may be created with a shared affinity namespace.
+ *
+ * Parameters:
+ * tag: The affinity set identifier.
+ */
+
+#define THREAD_AFFINITY_POLICY		4
+
+struct thread_affinity_policy {
+	integer_t	affinity_tag;
+};
+
+#define THREAD_AFFINITY_TAG_NULL		0
+
+typedef struct thread_affinity_policy		thread_affinity_policy_data_t;
+typedef struct thread_affinity_policy		*thread_affinity_policy_t;
+
+#define THREAD_AFFINITY_POLICY_COUNT	((mach_msg_type_number_t) \
+	(sizeof (thread_affinity_policy_data_t) / sizeof (integer_t)))
+
 #endif	/* _MACH_THREAD_POLICY_H_ */
diff --git a/osfmk/mach/thread_special_ports.h b/osfmk/mach/thread_special_ports.h
index a7e7a3193..1d44c02ed 100644
--- a/osfmk/mach/thread_special_ports.h
+++ b/osfmk/mach/thread_special_ports.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/thread_status.h b/osfmk/mach/thread_status.h
index b350ed303..aead09bf9 100644
--- a/osfmk/mach/thread_status.h
+++ b/osfmk/mach/thread_status.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/thread_switch.h b/osfmk/mach/thread_switch.h
index 522250329..f5558946a 100644
--- a/osfmk/mach/thread_switch.h
+++ b/osfmk/mach/thread_switch.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/time_value.h b/osfmk/mach/time_value.h
index c1969f281..e4f912d9e 100644
--- a/osfmk/mach/time_value.h
+++ b/osfmk/mach/time_value.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/upl.defs b/osfmk/mach/upl.defs
index 18d1737ec..e4f89e153 100644
--- a/osfmk/mach/upl.defs
+++ b/osfmk/mach/upl.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -95,3 +101,4 @@ routine upl_commit_range(
 	in	page_list	: upl_page_info_array_t;
 	out	empty		: boolean_t);
 
+/* vim: set ft=c : */
diff --git a/osfmk/mach/vm_attributes.h b/osfmk/mach/vm_attributes.h
index 61017334a..2be32337b 100644
--- a/osfmk/mach/vm_attributes.h
+++ b/osfmk/mach/vm_attributes.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/vm_behavior.h b/osfmk/mach/vm_behavior.h
index 6b2eee6d6..0d12ceef6 100644
--- a/osfmk/mach/vm_behavior.h
+++ b/osfmk/mach/vm_behavior.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/vm_inherit.h b/osfmk/mach/vm_inherit.h
index 3828478f1..d6440e441 100644
--- a/osfmk/mach/vm_inherit.h
+++ b/osfmk/mach/vm_inherit.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/vm_map.defs b/osfmk/mach/vm_map.defs
index 889d71416..439c0145a 100644
--- a/osfmk/mach/vm_map.defs
+++ b/osfmk/mach/vm_map.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -432,7 +438,7 @@ routine vm_map_64(
  */
 routine vm_map_get_upl(
 		target_task	: vm_map_t;
-		address		: vm_address_t;
+		address		: vm_map_offset_t;
 	inout	size		: vm_size_t;
 	out	upl		: upl_t;
 	out	page_info	: upl_page_info_array_t, CountInOut;
@@ -466,4 +472,4 @@ routine vm_purgable_control(
 		control		: vm_purgable_t;
 	inout	state		: int);
 
-
+/* vim: set ft=c : */
diff --git a/osfmk/mach/vm_param.h b/osfmk/mach/vm_param.h
index c23cf07f7..13078d58e 100644
--- a/osfmk/mach/vm_param.h
+++ b/osfmk/mach/vm_param.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -135,8 +141,8 @@
 
 #define round_page_32(x) (((uint32_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
 #define trunc_page_32(x) ((uint32_t)(x) & ~((signed)PAGE_MASK))
-#define round_page_64(x) (((uint64_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
-#define trunc_page_64(x) ((uint64_t)(x) & ~((signed)PAGE_MASK))
+#define round_page_64(x) (((uint64_t)(x) + PAGE_MASK_64) & ~((signed)PAGE_MASK_64))
+#define trunc_page_64(x) ((uint64_t)(x) & ~((signed)PAGE_MASK_64))
 
 
 /*
diff --git a/osfmk/mach/vm_prot.h b/osfmk/mach/vm_prot.h
index 71eaf4877..eed510771 100644
--- a/osfmk/mach/vm_prot.h
+++ b/osfmk/mach/vm_prot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -78,12 +84,6 @@ typedef int		vm_prot_t;
 #define VM_PROT_WRITE	((vm_prot_t) 0x02)	/* write permission */
 #define VM_PROT_EXECUTE	((vm_prot_t) 0x04)	/* execute permission */
 
-/*
- *	For now we can only support no execute on stacks... too many
- *	entries are marked w/o VM_PROT_EXECUTE that actually have code
- */
-#define STACK_ONLY_NX	1
-
 /*
  *	The default protection for newly-created virtual memory
  */
diff --git a/osfmk/mach/vm_purgable.h b/osfmk/mach/vm_purgable.h
index e5d919170..32463755b 100644
--- a/osfmk/mach/vm_purgable.h
+++ b/osfmk/mach/vm_purgable.h
@@ -1,28 +1,44 @@
 /*
- * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
- *	Virtual memory map purgable object definitions.
- *
+ * Virtual memory map purgeable object definitions.
+ * Objects that will be needed in the future (forward cached objects) should be queued LIFO. 
+ * Objects that have been used and are cached for reuse (backward cached) should be queued FIFO.
+ * Every user of purgeable memory is entitled to using the highest volatile group (7).
+ * Only if a client wants some of its objects to definitely be purged earlier, it can put those in
+ * another group. This could be used to make all FIFO objects (in the lower group) go away before 
+ * any LIFO objects (in the higher group) go away.
+ * Objects that should not get any chance to stay around can be marked as "obsolete". They will
+ * be emptied before any other objects or pages are reclaimed. Obsolete objects are not emptied
+ * in any particular order.
+ * 'purgeable' is recognized as the correct spelling. For historical reasons, definitions 
+ * in this file are spelled 'purgable'.
  */
 
 #ifndef	_MACH_VM_PURGABLE_H_
@@ -31,7 +47,7 @@
 /*
  *	Types defined:
  *
- *	vm_purgable_t	purgable object control codes.
+ *	vm_purgable_t	purgeable object control codes.
  */
 
 typedef int	vm_purgable_t;
@@ -39,17 +55,68 @@ typedef int	vm_purgable_t;
 /*
  *	Enumeration of valid values for vm_purgable_t.
  */
-#define VM_PURGABLE_SET_STATE	((vm_purgable_t) 0)	/* set state of purgable object */
-#define VM_PURGABLE_GET_STATE	((vm_purgable_t) 1)	/* get state of purgable object */
+#define VM_PURGABLE_SET_STATE	((vm_purgable_t) 0)	/* set state of purgeable object */
+#define VM_PURGABLE_GET_STATE	((vm_purgable_t) 1)	/* get state of purgeable object */
+
+/*
+ * Volatile memory ordering groups (group zero objects are purged before group 1, etc...
+ * It is implementation dependent as to whether these groups are global or per-address space.
+ * (for the moment, they are global).
+ */
+#define VM_VOLATILE_GROUP_SHIFT		8
+#define VM_VOLATILE_GROUP_MASK		(7 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_DEFAULT   VM_VOLATILE_GROUP_7
+
+#define VM_VOLATILE_GROUP_0			(0 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_1			(1 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_2			(2 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_3			(3 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_4			(4 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_5			(5 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_6			(6 << VM_VOLATILE_GROUP_SHIFT)
+#define VM_VOLATILE_GROUP_7			(7 << VM_VOLATILE_GROUP_SHIFT)
+
+/*
+ * Purgeable behavior
+ * Within the same group, FIFO objects will be emptied before objects that are added later.
+ * LIFO objects will be emptied after objects that are added later.
+ * - Input only, not returned on state queries.
+ */
+#define VM_PURGABLE_BEHAVIOR_SHIFT  6
+#define VM_PURGABLE_BEHAVIOR_MASK   (1 << VM_PURGABLE_BEHAVIOR_SHIFT)
+#define VM_PURGABLE_BEHAVIOR_FIFO   (0 << VM_PURGABLE_BEHAVIOR_SHIFT)
+#define VM_PURGABLE_BEHAVIOR_LIFO   (1 << VM_PURGABLE_BEHAVIOR_SHIFT)
+
+/*
+ * Obsolete object.
+ * Disregard volatile group, and put object into obsolete queue instead, so it is the next object
+ * to be purged.
+ * - Input only, not returned on state queries.
+ */
+#define VM_PURGABLE_ORDERING_SHIFT		5
+#define VM_PURGABLE_ORDERING_MASK		(1 << VM_PURGABLE_ORDERING_SHIFT)
+#define VM_PURGABLE_ORDERING_OBSOLETE	(1 << VM_PURGABLE_ORDERING_SHIFT)
+#define VM_PURGABLE_ORDERING_NORMAL		(0 << VM_PURGABLE_ORDERING_SHIFT)
+
+
+/*
+ * Obsolete parameter - do not use
+ */
+#define VM_VOLATILE_ORDER_SHIFT			4
+#define VM_VOLATILE_ORDER_MASK			(1 << VM_VOLATILE_ORDER_SHIFT)
+#define VM_VOLATILE_MAKE_FIRST_IN_GROUP	(1 << VM_VOLATILE_ORDER_SHIFT)
+#define VM_VOLATILE_MAKE_LAST_IN_GROUP	(0 << VM_VOLATILE_ORDER_SHIFT)
 
 /*
- * Valid states of a purgable object.
+ * Valid states of a purgeable object.
  */
-#define VM_PURGABLE_STATE_MIN	0			/* minimum purgable object state value */
-#define VM_PURGABLE_STATE_MAX	2			/* maximum purgable object state value */
+#define VM_PURGABLE_STATE_MIN	0		/* minimum purgeable object state value */
+#define VM_PURGABLE_STATE_MAX	3		/* maximum purgeable object state value */
+#define VM_PURGABLE_STATE_MASK	3		/* mask to separate state from group */
 
-#define VM_PURGABLE_NONVOLATILE	0			/* purgable object is non-volatile */
-#define VM_PURGABLE_VOLATILE	1			/* purgable object is volatile */
-#define VM_PURGABLE_EMPTY	2			/* purgable object is volatile and empty */
+#define VM_PURGABLE_NONVOLATILE	0		/* purgeable object is non-volatile */
+#define VM_PURGABLE_VOLATILE	1		/* purgeable object is volatile */
+#define VM_PURGABLE_EMPTY		2		/* purgeable object is volatile and empty */
+#define VM_PURGABLE_DENY		3		/* (mark) object not purgeable */
 
 #endif	/* _MACH_VM_PURGABLE_H_ */
diff --git a/osfmk/mach/vm_region.h b/osfmk/mach/vm_region.h
index 987ea01ae..f8b8ec650 100644
--- a/osfmk/mach/vm_region.h
+++ b/osfmk/mach/vm_region.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -234,6 +240,29 @@ typedef struct vm_region_submap_info_64		 vm_region_submap_info_data_64_t;
 #define VM_REGION_SUBMAP_INFO_COUNT_64		((mach_msg_type_number_t) \
 	(sizeof(vm_region_submap_info_data_64_t)/sizeof(int)))
 
+struct vm_region_submap_short_info_64 {
+	vm_prot_t		protection;     /* present access protection */
+	vm_prot_t		max_protection; /* max avail through vm_prot */
+	vm_inherit_t		inheritance;/* behavior of map/obj on fork */
+	memory_object_offset_t	offset;		/* offset into object/map */
+        unsigned int            user_tag;	/* user tag on map entry */
+        unsigned int            ref_count;	 /* obj/map mappers, etc */
+        unsigned short          shadow_depth; 	/* only for obj */
+        unsigned char           external_pager;  /* only for obj */
+        unsigned char           share_mode;	/* see enumeration */
+	boolean_t		is_submap;	/* submap vs obj */
+	vm_behavior_t		behavior;	/* access behavior hint */
+	vm_offset_t		object_id;	/* obj/map name, not a handle */
+	unsigned short		user_wired_count; 
+};
+
+typedef struct vm_region_submap_short_info_64	*vm_region_submap_short_info_64_t;
+typedef struct vm_region_submap_short_info_64	 vm_region_submap_short_info_data_64_t;
+
+#define VM_REGION_SUBMAP_SHORT_INFO_COUNT_64	((mach_msg_type_number_t) \
+	(sizeof(vm_region_submap_short_info_data_64_t)/sizeof(int)))
+
+
 struct mach_vm_read_entry {
 	mach_vm_address_t address;
 	mach_vm_size_t size;
diff --git a/osfmk/mach/vm_statistics.h b/osfmk/mach/vm_statistics.h
index 544c121ce..aab39fd3c 100644
--- a/osfmk/mach/vm_statistics.h
+++ b/osfmk/mach/vm_statistics.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -76,30 +82,23 @@ struct vm_statistics {
 	natural_t	lookups;		/* object cache lookups */
 	natural_t	hits;			/* object cache hits */
 
+	/* added for rev1 */
 	natural_t	purgeable_count;	/* # of pages purgeable */
 	natural_t	purges;			/* # of pages purged */
+
+	/* added for rev2 */
+	/*
+	 * NB: speculative pages are already accounted for in "free_count",
+	 * so "speculative_count" is the number of "free" pages that are
+	 * used to hold data that was read speculatively from disk but
+	 * haven't actually been used by anyone so far.
+	 */
+	natural_t	speculative_count;	/* # of pages speculative */
 };
 
 typedef struct vm_statistics	*vm_statistics_t;
 typedef struct vm_statistics	vm_statistics_data_t;
 
-struct vm_statistics_rev0 {
-	natural_t	free_count;		/* # of pages free */
-	natural_t	active_count;		/* # of pages active */
-	natural_t	inactive_count;		/* # of pages inactive */
-	natural_t	wire_count;		/* # of pages wired down */
-	natural_t	zero_fill_count;	/* # of zero fill pages */
-	natural_t	reactivations;		/* # of pages reactivated */
-	natural_t	pageins;		/* # of pageins */
-	natural_t	pageouts;		/* # of pageouts */
-	natural_t	faults;			/* # of faults */
-	natural_t	cow_faults;		/* # of copy-on-writes */
-	natural_t	lookups;		/* object cache lookups */
-	natural_t	hits;			/* object cache hits */
-};
-
-typedef struct vm_statistics_rev0	*vm_statistics_rev0_t;
-typedef struct vm_statistics_rev0	vm_statistics_rev0_data_t;
 
 /* included for the vm_map_page_query call */
 
@@ -107,6 +106,9 @@ typedef struct vm_statistics_rev0	vm_statistics_rev0_data_t;
 #define VM_PAGE_QUERY_PAGE_FICTITIOUS   0x2
 #define VM_PAGE_QUERY_PAGE_REF          0x4
 #define VM_PAGE_QUERY_PAGE_DIRTY        0x8
+#define VM_PAGE_QUERY_PAGE_PAGED_OUT    0x10
+#define VM_PAGE_QUERY_PAGE_COPIED       0x20
+#define VM_PAGE_QUERY_PAGE_SPECULATIVE	0x40
 
 #ifdef	MACH_KERNEL_PRIVATE
 
@@ -119,6 +121,7 @@ typedef struct vm_statistics_rev0	vm_statistics_rev0_data_t;
 
 struct pmap_statistics {
 	integer_t	resident_count;	/* # of pages mapped (total)*/
+	integer_t	resident_max;	/* # of pages mapped (peak) */
 	integer_t	wired_count;	/* # of pages wired */
 };
 
@@ -148,14 +151,26 @@ typedef struct pmap_statistics	*pmap_statistics_t;
  * VM_FLAGS_OVERWRITE
  *	The new VM region can replace existing VM regions if necessary
  *	(to be used in combination with VM_FLAGS_FIXED).
+ *
+ * VM_FLAGS_NO_CACHE
+ *	Pages brought in to this VM region are placed on the speculative
+ *	queue instead of the active queue.  In other words, they are not
+ *	cached so that they will be stolen first if memory runs low.
  */
 #define VM_FLAGS_FIXED		0x0000
 #define VM_FLAGS_ANYWHERE	0x0001
 #define VM_FLAGS_PURGABLE	0x0002
+#define VM_FLAGS_NO_CACHE	0x0010
+
 #ifdef KERNEL_PRIVATE
-#define VM_FLAGS_NO_PMAP_CHECK	0x0004
+#define VM_FLAGS_NO_PMAP_CHECK	0x8000	/* do not check that pmap is empty */
+#define VM_FLAGS_OVERWRITE	0x4000	/* delete any existing mappings first */
+#define VM_FLAGS_BEYOND_MAX	0x2000	/* map beyond the map's limits */
+#define VM_FLAGS_ALREADY	0x1000	/* OK if same mapping already exists */
+#define VM_FLAGS_SUBMAP		0x0800	/* mapping a VM submap */
 #endif /* KERNEL_PRIVATE */
-#define VM_FLAGS_OVERWRITE	0x0008
+#define VM_FLAGS_GUARD_BEFORE	0x0010
+#define VM_FLAGS_GUARD_AFTER	0x0020
 
 #define VM_FLAGS_ALIAS_MASK	0xFF000000
 #define VM_GET_FLAGS_ALIAS(flags, alias)			\
@@ -164,6 +179,14 @@ typedef struct pmap_statistics	*pmap_statistics_t;
 		(flags) = (((flags) & ~VM_FLAGS_ALIAS_MASK) |	\
 		(((alias) & ~VM_FLAGS_ALIAS_MASK) << 24))
 
+/* These are the flags that we accept from user-space */
+#define VM_FLAGS_USER_ALLOCATE	(VM_FLAGS_FIXED |		\
+				 VM_FLAGS_ANYWHERE |		\
+				 VM_FLAGS_PURGABLE |		\
+				 VM_FLAGS_NO_CACHE |		\
+				 VM_FLAGS_ALIAS_MASK)
+#define VM_FLAGS_USER_MAP	VM_FLAGS_USER_ALLOCATE
+
 #define VM_MEMORY_MALLOC 1
 #define VM_MEMORY_MALLOC_SMALL 2
 #define VM_MEMORY_MALLOC_LARGE 3
@@ -190,6 +213,24 @@ typedef struct pmap_statistics	*pmap_statistics_t;
 #define VM_MEMORY_CARBON 43
 #define VM_MEMORY_JAVA 44
 #define VM_MEMORY_ATS 50
+#define VM_MEMORY_LAYERKIT 51
+#define VM_MEMORY_CGIMAGE 52
+#define VM_MEMORY_TCMALLOC 53
+
+/* private raster data (i.e. layers, some images, QGL allocator) */
+#define	VM_MEMORY_COREGRAPHICS_DATA	54
+
+/* shared image and font caches */
+#define VM_MEMORY_COREGRAPHICS_SHARED	55
+
+/* Memory used for virtual framebuffers, shadowing buffers, etc... */
+#define	VM_MEMORY_COREGRAPHICS_FRAMEBUFFERS	56
+
+/* Window backing stores, custom shadow data, and compressed backing stores */
+#define VM_MEMORY_COREGRAPHICS_BACKINGSTORES	57
+
+/* catch-all for other uses, such as the read-only shared data page */
+#define VM_MEMORY_COREGRAPHICS_MISC VM_MEMORY_COREGRAPHICS
 
 /* memory allocated by the dynamic loader for itself */
 #define VM_MEMORY_DYLD 60
@@ -200,6 +241,6 @@ typedef struct pmap_statistics	*pmap_statistics_t;
 #define VM_MEMORY_APPLICATION_SPECIFIC_1 240
 #define VM_MEMORY_APPLICATION_SPECIFIC_16 255
 
-#define VM_MAKE_TAG(tag) (tag<<24)
+#define VM_MAKE_TAG(tag) ((tag) << 24)
 
 #endif	/* _MACH_VM_STATISTICS_H_ */
diff --git a/osfmk/mach/vm_sync.h b/osfmk/mach/vm_sync.h
index 4e38c4fa9..a0e3542cc 100644
--- a/osfmk/mach/vm_sync.h
+++ b/osfmk/mach/vm_sync.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach/vm_types.h b/osfmk/mach/vm_types.h
index 17ac4187f..f586d876c 100644
--- a/osfmk/mach/vm_types.h
+++ b/osfmk/mach/vm_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -77,7 +83,7 @@ typedef uint32_t ppnum_t;		/* Physical page number */
 __BEGIN_DECLS
 
 struct pmap ;
-struct vm_map ;
+struct _vm_map ;
 struct vm_object ;
 
 __END_DECLS
@@ -85,8 +91,9 @@ __END_DECLS
 #endif	/* MACH_KERNEL_PRIVATE */
 
 typedef struct pmap		*pmap_t;
-typedef struct vm_map		*vm_map_t;
+typedef struct _vm_map		*vm_map_t;
 typedef struct vm_object 	*vm_object_t;
+typedef struct vm_object_fault_info 	*vm_object_fault_info_t;
 
 #define PMAP_NULL		((pmap_t) 0)
 #define VM_OBJECT_NULL	((vm_object_t) 0)
diff --git a/osfmk/mach_debug/hash_info.h b/osfmk/mach_debug/hash_info.h
index 3629b02b7..ba626944b 100644
--- a/osfmk/mach_debug/hash_info.h
+++ b/osfmk/mach_debug/hash_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach_debug/ipc_info.h b/osfmk/mach_debug/ipc_info.h
index c358c9d78..94ef8d6b1 100644
--- a/osfmk/mach_debug/ipc_info.h
+++ b/osfmk/mach_debug/ipc_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach_debug/lockgroup_info.h b/osfmk/mach_debug/lockgroup_info.h
index d356d6b9f..99c908875 100644
--- a/osfmk/mach_debug/lockgroup_info.h
+++ b/osfmk/mach_debug/lockgroup_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	File:	mach/lockgroup_info.h
diff --git a/osfmk/mach_debug/mach_debug.h b/osfmk/mach_debug/mach_debug.h
index 8f1a6f095..ca02a8d8d 100644
--- a/osfmk/mach_debug/mach_debug.h
+++ b/osfmk/mach_debug/mach_debug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) Apple Computer 1998
diff --git a/osfmk/mach_debug/mach_debug_types.defs b/osfmk/mach_debug/mach_debug_types.defs
index 2d9090e5e..8e63dbb65 100644
--- a/osfmk/mach_debug/mach_debug_types.defs
+++ b/osfmk/mach_debug/mach_debug_types.defs
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -92,3 +98,5 @@ type lockgroup_info_array_t = array[] of lockgroup_info_t;
 import <mach_debug/mach_debug_types.h>;
 
 #endif	/* _MACH_DEBUG_MACH_DEBUG_TYPES_DEFS_ */
+
+/* vim: set ft=c : */
diff --git a/osfmk/mach_debug/mach_debug_types.h b/osfmk/mach_debug/mach_debug_types.h
index 66b5e4c8b..32754acad 100644
--- a/osfmk/mach_debug/mach_debug_types.h
+++ b/osfmk/mach_debug/mach_debug_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach_debug/page_info.h b/osfmk/mach_debug/page_info.h
index 3a95d9a8e..296c613b6 100644
--- a/osfmk/mach_debug/page_info.h
+++ b/osfmk/mach_debug/page_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/mach_debug/vm_info.h b/osfmk/mach_debug/vm_info.h
index a02bba1f0..91786c8ba 100644
--- a/osfmk/mach_debug/vm_info.h
+++ b/osfmk/mach_debug/vm_info.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,8 +53,6 @@
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
-/*
- */
 /*
  *	File:	mach_debug/vm_info.h
  *	Author:	Rich Draves
diff --git a/osfmk/mach_debug/zone_info.h b/osfmk/mach_debug/zone_info.h
index 81f945ae5..88957239d 100644
--- a/osfmk/mach_debug/zone_info.h
+++ b/osfmk/mach_debug/zone_info.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/machine/asm.h b/osfmk/machine/asm.h
index b0ace5272..2f8346ffc 100644
--- a/osfmk/machine/asm.h
+++ b/osfmk/machine/asm.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_ASM_H
 #define _MACHINE_ASM_H
 
-
 #if defined (__ppc__)
 #include "ppc/asm.h"
 #elif defined (__i386__)
 #include "i386/asm.h"
+#elif defined (__arm__)
+#include "arm/asm.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_ASM_H */
diff --git a/osfmk/machine/ast.h b/osfmk/machine/ast.h
index 5ec6ba6eb..722602d0e 100644
--- a/osfmk/machine/ast.h
+++ b/osfmk/machine/ast.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_AST_H
 #define _MACHINE_AST_H
 
-
 #if defined (__ppc__)
 #include "ppc/ast.h"
 #elif defined (__i386__)
 #include "i386/ast.h"
+#elif defined (__arm__)
+#include "arm/ast.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_AST_H */
diff --git a/osfmk/machine/ast_types.h b/osfmk/machine/ast_types.h
index b04ed6317..97c932d87 100644
--- a/osfmk/machine/ast_types.h
+++ b/osfmk/machine/ast_types.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_AST_TYPES_H
 #define _MACHINE_AST_TYPES_H
 
-
 #if defined (__ppc__)
 #include "ppc/ast_types.h"
 #elif defined (__i386__)
 #include "i386/ast_types.h"
+#elif defined (__arm__)
+#include "arm/ast_types.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_AST_TYPES_H */
diff --git a/osfmk/machine/commpage.h b/osfmk/machine/commpage.h
index 9a8736507..80c9e23db 100644
--- a/osfmk/machine/commpage.h
+++ b/osfmk/machine/commpage.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _MACHINE_COMMPAGE_H
@@ -27,6 +33,8 @@
 #include "ppc/commpage/commpage.h"
 #elif defined (__i386__)
 #include "i386/commpage/commpage.h"
+#elif defined (__arm__)
+#include "arm/commpage/commpage.h"
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/machine/cpu_affinity.h b/osfmk/machine/cpu_affinity.h
new file mode 100644
index 000000000..7d61bb445
--- /dev/null
+++ b/osfmk/machine/cpu_affinity.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef	KERNEL_PRIVATE
+
+#ifndef _MACHINE_CPU_AFFINITY_H
+#define _MACHINE_CPU_AFFINITY_H
+
+#if defined (__ppc__)
+#include "ppc/cpu_affinity.h"
+#elif defined (__i386__)
+#include "i386/cpu_affinity.h"
+#elif defined (__arm__)
+#include "arm/cpu_affinity.h"
+#else
+#error architecture not supported
+#endif
+
+#endif /* _MACHINE_CPU_AFFINITY_H */
+
+#endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/machine/cpu_capabilities.h b/osfmk/machine/cpu_capabilities.h
index 2d441d502..bfc586bbd 100644
--- a/osfmk/machine/cpu_capabilities.h
+++ b/osfmk/machine/cpu_capabilities.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	PRIVATE
 
@@ -29,6 +35,8 @@
 #include "ppc/cpu_capabilities.h"
 #elif defined (__i386__)
 #include "i386/cpu_capabilities.h"
+#elif defined (__arm__)
+#include "arm/cpu_capabilities.h"
 #else
 #error architecture not supported
 #endif
@@ -38,6 +46,8 @@
 #include <System/ppc/cpu_capabilities.h>
 #elif defined (__i386__) || defined(__x86_64__)
 #include <System/i386/cpu_capabilities.h>
+#elif defined (__arm__)
+#include <System/arm/cpu_capabilities.h>
 #else
 #error architecture not supported
 #endif
diff --git a/osfmk/machine/cpu_data.h b/osfmk/machine/cpu_data.h
index b10bac948..ae3f3b616 100644
--- a/osfmk/machine/cpu_data.h
+++ b/osfmk/machine/cpu_data.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_CPU_DATA_H
 #define _MACHINE_CPU_DATA_H
 
-
 #if defined (__ppc__)
 #include "ppc/cpu_data.h"
 #elif defined (__i386__)
 #include "i386/cpu_data.h"
+#elif defined (__arm__)
+#include "arm/cpu_data.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_CPU_DATA_H */
diff --git a/osfmk/machine/cpu_number.h b/osfmk/machine/cpu_number.h
index 57f865255..0097d4562 100644
--- a/osfmk/machine/cpu_number.h
+++ b/osfmk/machine/cpu_number.h
@@ -1,39 +1,45 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	KERNEL_PRIVATE
 
 #ifndef _MACHINE_CPU_NUMBER_H
 #define _MACHINE_CPU_NUMBER_H
 
-
 #if defined (__ppc__)
 #include "ppc/cpu_number.h"
 #elif defined (__i386__)
 #include "i386/cpu_number.h"
+#elif defined (__arm__)
+#include "arm/cpu_number.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_CPU_NUMBER_H */
 
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/machine/db_machdep.h b/osfmk/machine/db_machdep.h
index bec9b4715..dd6f51752 100644
--- a/osfmk/machine/db_machdep.h
+++ b/osfmk/machine/db_machdep.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_DB_MACHDEP_H
 #define _MACHINE_DB_MACHDEP_H
 
-
 #if defined (__ppc__)
 #include "ppc/db_machdep.h"
 #elif defined (__i386__)
 #include "i386/db_machdep.h"
+#elif defined (__arm__)
+#include "arm/db_machdep.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_DB_MACHDEP_H */
diff --git a/osfmk/machine/endian.h b/osfmk/machine/endian.h
index f0c8b29d3..e292ec80e 100644
--- a/osfmk/machine/endian.h
+++ b/osfmk/machine/endian.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_ENDIAN_H
 #define _MACHINE_ENDIAN_H
 
-
 #if defined (__ppc__)
 #include "ppc/endian.h"
 #elif defined (__i386__)
 #include "i386/endian.h"
+#elif defined (__arm__)
+#include "arm/endian.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_ENDIAN_H */
diff --git a/osfmk/machine/io_map_entries.h b/osfmk/machine/io_map_entries.h
index e946c123c..cc084f515 100644
--- a/osfmk/machine/io_map_entries.h
+++ b/osfmk/machine/io_map_entries.h
@@ -1,39 +1,45 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	KERNEL_PRIVATE
 
 #ifndef _MACHINE_IO_MAP_ENTRIES_H_
 #define _MACHINE_IO_MAP_ENTRIES_H_
 
-
 #if defined (__ppc__)
 #include "ppc/io_map_entries.h"
 #elif defined (__i386__)
 #include "i386/io_map_entries.h"
+#elif defined (__arm__)
+#include "arm/io_map_entries.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_IO_MAP_ENTRIES_H_ */
 
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/machine/lock.h b/osfmk/machine/lock.h
index 1fb0484c3..4cf25284c 100644
--- a/osfmk/machine/lock.h
+++ b/osfmk/machine/lock.h
@@ -1,39 +1,45 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	KERNEL_PRIVATE
 
 #ifndef _MACHINE_LOCK_H_
 #define _MACHINE_LOCK_H_
 
-
 #if defined (__ppc__)
 #include "ppc/lock.h"
 #elif defined (__i386__)
 #include "i386/lock.h"
+#elif defined (__arm__)
+#include "arm/lock.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_LOCK_H_ */
 
 #endif
diff --git a/osfmk/machine/locks.h b/osfmk/machine/locks.h
index 5ba5ab982..eff32fb17 100644
--- a/osfmk/machine/locks.h
+++ b/osfmk/machine/locks.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_LOCKS_H_
 #define _MACHINE_LOCKS_H_
 
-
 #if defined (__ppc__)
 #include "ppc/locks.h"
 #elif defined (__i386__)
 #include "i386/locks.h"
+#elif defined (__arm__)
+#include "arm/locks.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_LOCKS_H_ */
diff --git a/osfmk/machine/machine_cpu.h b/osfmk/machine/machine_cpu.h
index a9235e191..059d0eaf1 100644
--- a/osfmk/machine/machine_cpu.h
+++ b/osfmk/machine/machine_cpu.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_MACHINE_CPU_H
 #define _MACHINE_MACHINE_CPU_H
 
-
 #if defined (__ppc__)
 #include "ppc/machine_cpu.h"
 #elif defined (__i386__)
 #include "i386/machine_cpu.h"
+#elif defined (__arm__)
+#include "arm/machine_cpu.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_MACHINE_CPU_H */
diff --git a/osfmk/machine/machine_routines.h b/osfmk/machine/machine_routines.h
index 57481db5a..00dfc3216 100644
--- a/osfmk/machine/machine_routines.h
+++ b/osfmk/machine/machine_routines.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_MACHINE_ROUTINES_H
 #define _MACHINE_MACHINE_ROUTINES_H
 
-
 #if defined (__ppc__)
 #include "ppc/machine_routines.h"
 #elif defined (__i386__)
 #include "i386/machine_routines.h"
+#elif defined (__arm__)
+#include "arm/machine_routines.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_MACHINE_ROUTINES_H */
diff --git a/osfmk/machine/machine_rpc.h b/osfmk/machine/machine_rpc.h
index c582f112c..9307c9191 100644
--- a/osfmk/machine/machine_rpc.h
+++ b/osfmk/machine/machine_rpc.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_MACHINE_RPC_H
 #define _MACHINE_MACHINE_RPC_H
 
-
 #if defined (__ppc__)
 #include "ppc/machine_rpc.h"
 #elif defined (__i386__)
 #include "i386/machine_rpc.h"
+#elif defined (__arm__)
+#include "arm/machine_rpc.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_MACHINE_RPC_H */
diff --git a/osfmk/machine/machlimits.h b/osfmk/machine/machlimits.h
index 2add98192..5b970101c 100644
--- a/osfmk/machine/machlimits.h
+++ b/osfmk/machine/machlimits.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_MACHLIMITS_H
 #define _MACHINE_MACHLIMITS_H
 
-
 #if defined (__ppc__)
 #include "ppc/machlimits.h"
 #elif defined (__i386__)
 #include "i386/machlimits.h"
+#elif defined (__arm__)
+#include "arm/machlimits.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_MACHLIMITS_H */
diff --git a/osfmk/machine/machparam.h b/osfmk/machine/machparam.h
index f29685ffc..02ddd8c34 100644
--- a/osfmk/machine/machparam.h
+++ b/osfmk/machine/machparam.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_MACHPARAM_H
 #define _MACHINE_MACHPARAM_H
 
-
 #if defined (__ppc__)
 #include "ppc/machparam.h"
 #elif defined (__i386__)
 #include "i386/machparam.h"
+#elif defined (__arm__)
+#include "arm/machparam.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_MACHPARAM_H */
diff --git a/osfmk/machine/pmap.h b/osfmk/machine/pmap.h
index 198bdd167..336336309 100644
--- a/osfmk/machine/pmap.h
+++ b/osfmk/machine/pmap.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_PMAP_H
 #define _MACHINE_PMAP_H
 
-
 #if defined (__ppc__)
 #include "ppc/pmap.h"
 #elif defined (__i386__)
 #include "i386/pmap.h"
+#elif defined (__arm__)
+#include "arm/pmap.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_PMAP_H */
diff --git a/osfmk/machine/sched_param.h b/osfmk/machine/sched_param.h
index c32ea5f61..082690f0a 100644
--- a/osfmk/machine/sched_param.h
+++ b/osfmk/machine/sched_param.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_SCHED_PARAM_H
 #define _MACHINE_SCHED_PARAM_H
 
-
 #if defined (__ppc__)
 #include "ppc/sched_param.h"
 #elif defined (__i386__)
 #include "i386/sched_param.h"
+#elif defined (__arm__)
+#include "arm/sched_param.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_SCHED_PARAM_H */
diff --git a/osfmk/machine/setjmp.h b/osfmk/machine/setjmp.h
index 1f4e5ef2c..08a73f43a 100644
--- a/osfmk/machine/setjmp.h
+++ b/osfmk/machine/setjmp.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_SETJMP_H
 #define _MACHINE_SETJMP_H
 
-
 #if defined (__ppc__)
 #include "ppc/setjmp.h"
 #elif defined (__i386__)
 #include "i386/setjmp.h"
+#elif defined (__arm__)
+#include "arm/setjmp.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_SETJMP_H */
diff --git a/osfmk/machine/simple_lock.h b/osfmk/machine/simple_lock.h
index 1a28cbe75..67456c61d 100644
--- a/osfmk/machine/simple_lock.h
+++ b/osfmk/machine/simple_lock.h
@@ -1,39 +1,45 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	KERNEL_PRIVATE
 
 #ifndef _MACHINE_SIMPLE_LOCK_H_
 #define _MACHINE_SIMPLE_LOCK_H_
 
-
 #if defined (__ppc__)
 #include "ppc/simple_lock.h"
 #elif defined (__i386__)
 #include "i386/simple_lock.h"
+#elif defined (__arm__)
+#include "arm/simple_lock.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_SIMPLE_LOCK_H_ */
 
 #endif
diff --git a/osfmk/machine/task.h b/osfmk/machine/task.h
index 56f48ae72..f39e0adbe 100644
--- a/osfmk/machine/task.h
+++ b/osfmk/machine/task.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_TASK_H
 #define _MACHINE_TASK_H
 
-
 #if defined (__ppc__)
 #include "ppc/task.h"
 #elif defined (__i386__)
 #include "i386/task.h"
+#elif defined (__arm__)
+#include "arm/task.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_TASK_H */
diff --git a/osfmk/machine/thread.h b/osfmk/machine/thread.h
index 11ace465a..2b6153bd4 100644
--- a/osfmk/machine/thread.h
+++ b/osfmk/machine/thread.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_THREAD_H
 #define _MACHINE_THREAD_H
 
-
 #if defined (__ppc__)
 #include "ppc/thread.h"
 #elif defined (__i386__)
 #include "i386/thread.h"
+#elif defined (__arm__)
+#include "arm/thread.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_THREADx_H */
diff --git a/osfmk/machine/timer.h b/osfmk/machine/timer.h
index eafb03fb9..d39778ed9 100644
--- a/osfmk/machine/timer.h
+++ b/osfmk/machine/timer.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_TIMER_H
 #define _MACHINE_TIMER_H
 
-
 #if defined (__ppc__)
 #include "ppc/timer.h"
 #elif defined (__i386__)
 #include "i386/timer.h"
+#elif defined (__arm__)
+#include "arm/timer.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_TIMER_H */
diff --git a/osfmk/machine/trap.h b/osfmk/machine/trap.h
index ef021722c..737682fec 100644
--- a/osfmk/machine/trap.h
+++ b/osfmk/machine/trap.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_TRAP_H
 #define _MACHINE_TRAP_H
 
-
 #if defined (__ppc__)
 #include "ppc/trap.h"
 #elif defined (__i386__)
 #include "i386/trap.h"
+#elif defined (__arm__)
+#include "arm/trap.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_TRAP_H */
diff --git a/osfmk/machine/vm_tuning.h b/osfmk/machine/vm_tuning.h
index d74da3049..00c50d283 100644
--- a/osfmk/machine/vm_tuning.h
+++ b/osfmk/machine/vm_tuning.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_VM_TUNING_H
 #define _MACHINE_VM_TUNING_H
 
-
 #if defined (__ppc__)
 #include "ppc/vm_tuning.h"
 #elif defined (__i386__)
 #include "i386/vm_tuning.h"
+#elif defined (__arm__)
+#include "arm/vm_tuning.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_VM_TUNING_H */
diff --git a/osfmk/machine/xpr.h b/osfmk/machine/xpr.h
index 9507298b8..a5b64812d 100644
--- a/osfmk/machine/xpr.h
+++ b/osfmk/machine/xpr.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACHINE_XPR_H
 #define _MACHINE_XPR_H
 
-
 #if defined (__ppc__)
 #include "ppc/xpr.h"
 #elif defined (__i386__)
 #include "i386/xpr.h"
+#elif defined (__arm__)
+#include "arm/xpr.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACHINE_XPR_H */
diff --git a/osfmk/man/mach_port_status.html b/osfmk/man/mach_port_status.html
index 1e9b8d74e..99e5a8cd1 100755
--- a/osfmk/man/mach_port_status.html
+++ b/osfmk/man/mach_port_status.html
@@ -6,13 +6,13 @@
 <pre>
 <strong>struct mach_port_status</strong>
 <strong>{</strong>
-       <strong>natural_t</strong>                <var>mps_pset_count</var><strong>;</strong>
+       <strong>mach_port_rights_t</strong>         <var>mps_pset</var><strong>;</strong>
        <strong>mach_port_seqno_t</strong>         <var>mps_seqno</var><strong>;</strong>
        <strong>mach_port_mscount_t</strong>     <var>mps_mscount</var><strong>;</strong>
        <strong>mach_port_msgcount_t</strong>     <var>mps_qlimit</var><strong>;</strong>
        <strong>mach_port_msgcount_t</strong>   <var>mps_msgcount</var><strong>;</strong>
        <strong>mach_port_rights_t</strong>     <var>mps_sorights</var><strong>;</strong>
-       <strong>mach_port_srights_t</strong>     <var>mps_srights</var><strong>;</strong>
+       <strong>boolean_t</strong>               <var>mps_srights</var><strong>;</strong>
        <strong>boolean_t</strong>             <var>mps_pdrequest</var><strong>;</strong>
        <strong>boolean_t</strong>             <var>mps_nsrequest</var><strong>;</strong>
        <strong>unsigned int</strong>              <var>mps_flags</var><strong>;</strong>
@@ -24,7 +24,7 @@
 <dl>
 <dt> <var>mps_pset</var>
 <dd>
-Containing port set.
+Count of containing psets.
      <p>
 <dt> <var>mps_seqno</var>
 <dd>
@@ -36,36 +36,20 @@ Current sequence number for the port.
 	       <p>
 <dt> <var>mps_msgcount</var>
 <dd>
-Upper limit for the number of messages that may be queued to the port
+     Upper limit for the number of messages that may be queued to the port
      before the system blocks send operations.
      <p>
-     <dt> <var>mps_msgcount</var>
+<dt> <var>mps_msgcount</var>
 <dd>
      Number of messages currently queued on the port.
      <p>
 <dt> <var>mps_sorights</var>
 <dd>
-How many send-once rights.
+     How many send-once rights.
      <p>
 <dt> <var>mps_srights</var>
 <dd>
-Specifies whether or not send rights exist.  Valid values are as follows:
-     <dl>
-       <p>
-       <dt> MPS_FALSE
-	    <dd>
-		 No send rights currently in existence.
-		        <p>
-       <dt> MPS_TRUE
-	    <dd>
-		 Send rights exist and no-more-senders notification is enabled.
-		        <p>
-       <dt> MPS_UNKNOWN
-	    <dd>
-		 The port does not permit no-more-senders notification requests;
-		 consequently, the system does not know whether or not send rights
-		 exist.
-		 </dl>
+     Specifies whether or not send rights exist for the port (in this or any other task).
      <p>
 <dt> <var>mps_pdrequest</var>
 <dd>
@@ -73,8 +57,8 @@ Specifies whether or not send rights exist.  Valid values are as follows:
      <p>
 <dt> <var>mps_nsrequest</var>
 <dd>
-True if no-senders requested.
-          <p>
+     True if no-senders notification requested.
+     <p>
 <dt> <var>mps_flags</var>
 <dd>
      Flags associated with the port.
@@ -84,11 +68,6 @@ True if no-senders requested.
 The <strong>mach_port_status</strong> structure is used to provide
 information about a port in response to an invocation of the
 <strong>mach_port_get_attributes</strong> interface.
-<h3>NOTES</h3>
-<p>
-This structure is machine word length sensitive due
-to the presence of the port 
-set name.
 <h3>RELATED INFORMATION</h3>
 <p>
 Functions:
diff --git a/osfmk/ppc/AltiAssist.s b/osfmk/ppc/AltiAssist.s
index 6bf24f4aa..6ff23acba 100644
--- a/osfmk/ppc/AltiAssist.s
+++ b/osfmk/ppc/AltiAssist.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 																							
  	AltiAssist.s 
@@ -83,6 +89,3 @@ LEXT(AltivecAssist)
 			li			r11,T_IN_VAIN				; We are all done
 			b			EXT(EmulExit)				; We are done, no tracing on...
 
-
-
-
diff --git a/osfmk/ppc/Diagnostics.c b/osfmk/ppc/Diagnostics.c
index 0e8587034..5ac1d203a 100644
--- a/osfmk/ppc/Diagnostics.c
+++ b/osfmk/ppc/Diagnostics.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -49,6 +55,7 @@
 #include <ipc/ipc_space.h>
 #include <ipc/ipc_object.h>
 #include <ipc/ipc_port.h>
+#include <ipc/ipc_right.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
@@ -84,9 +91,9 @@ int diagCall(struct savearea *save) {
 	natural_t tbu, tbu2, tbl;
 	struct per_proc_info *per_proc;					/* Area for my per_proc address */
 	int cpu, ret, subc;
-	unsigned int tstrt, tend, temp, temp2, *baddr, oldwar;
+	unsigned int temp, temp2, *baddr, oldwar;
 	addr64_t src, snk;
-	uint64_t scom, hid1, hid4, srrwrk, stat;
+	uint64_t srrwrk;
 	scomcomm sarea;
 	ipc_port_t port;
 	ipc_entry_t ientry;
@@ -231,7 +238,7 @@ int diagCall(struct savearea *save) {
 		case dgtest:
 		
 			kprintf("Trying to hang\n");
-			baddr = (unsigned int)&baddr | 1;		/* Make an odd address */
+			baddr = (unsigned *)((unsigned)&baddr | 1); /* Make an odd address */
 			__asm__ volatile("lwarx r2,0,%0" : : "r" (baddr));
 			kprintf("Didn't hang\n");
 
@@ -380,7 +387,7 @@ int diagCall(struct savearea *save) {
 				}
 				else {									/* Otherwise, tell the other processor */
 					(void)cpu_signal(sarea.scomcpu, SIGPcpureq, CPRQscom ,(unsigned int)&sarea);	/* Ask him to do this */
-					(void)hw_cpu_sync((unsigned long)&sarea.scomstat, LockTimeOut);	/* Wait for the other processor to get its temperature */
+					(void)hw_cpu_sync((unsigned int*)&sarea.scomstat, LockTimeOut);	/* Wait for the other processor to get its temperature */
 				}
 			}
 
@@ -396,7 +403,7 @@ int diagCall(struct savearea *save) {
 		case dgBind:
 
 			if(save->save_r4 == 0) {				/* Are we unbinding? */
-				thread_bind(current_thread(), PROCESSOR_NULL);	/* Unbind us */
+				thread_bind(PROCESSOR_NULL);		/* Unbind us */
 				save->save_r3 = KERN_SUCCESS;		/* Set success */
 				return -1;							/* Return and check asts */
 			}
@@ -430,7 +437,7 @@ int diagCall(struct savearea *save) {
 				return -1;							/* Return and check asts */
 			}
 		
-			thread_bind(current_thread(), prssr);	/* Bind us to the processor */
+			thread_bind(prssr);						/* Bind us to the processor */
 			thread_block(THREAD_CONTINUE_NULL);		/* Make it so */
 	
 			save->save_r3 = KERN_SUCCESS;			/* Set success */
@@ -461,7 +468,7 @@ int diagCall(struct savearea *save) {
 			prssr = (processor_t)port->ip_kobject;	/* Extract the processor */
 			is_write_unlock(current_space());		/* All done with the space now, unlock it */
 			
-			save->save_r3 = (uint64_t)PerProcTable[prssr->processor_data.slot_num].ppe_vaddr;	/* Pass back ther per proc */
+			save->save_r3 = (uint64_t)(uint32_t)PerProcTable[prssr->processor_data.slot_num].ppe_vaddr;	/* Pass back ther per proc */
 			return -1;								/* Return and check asts */
 
 /*
@@ -478,7 +485,7 @@ int diagCall(struct savearea *save) {
 			addrs = 0;								/* Clear just in case */
 			
 			ret = kmem_alloc_contig(kernel_map, &addrs, (vm_size_t)save->save_r4,
-				PAGE_MASK, 0);						/* That which does not make us stronger, kills us... */
+						PAGE_MASK, 0, 0);						/* That which does not make us stronger, kills us... */
 			if(ret != KERN_SUCCESS) addrs = 0;		/* Pass 0 if error */
 		
 			save->save_r3 = (uint64_t)addrs;		/* Pass back whatever */
@@ -550,15 +557,15 @@ int diagCall(struct savearea *save) {
 
 };
 
-kern_return_t testPerfTrap(int trapno, struct savearea *ss, 
-	unsigned int dsisr, addr64_t dar) {
+kern_return_t
+testPerfTrap(int trapno, struct savearea *ss, unsigned int dsisr, addr64_t dar)
+{
 
 	if(trapno != T_ALIGNMENT) return KERN_FAILURE;
 
-	kprintf("alignment exception at %08X, srr1 = %08X, dsisr = %08X, dar = %08X\n", ss->save_srr0,
-		ss->save_srr1, dsisr, dar);
+	kprintf("alignment exception at %08llX, srr1 = %08llX, dsisr = %08X, dar = %08llX\n",
+			ss->save_srr0, ss->save_srr1, dsisr, dar);
 		
 	return KERN_SUCCESS;
-
 }
 
diff --git a/osfmk/ppc/Diagnostics.h b/osfmk/ppc/Diagnostics.h
index ad7767ca1..17e31a323 100644
--- a/osfmk/ppc/Diagnostics.h
+++ b/osfmk/ppc/Diagnostics.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/Emulate.s b/osfmk/ppc/Emulate.s
index 7a4ef3ab9..76ea4eb1c 100644
--- a/osfmk/ppc/Emulate.s
+++ b/osfmk/ppc/Emulate.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* 																							
  	Emulate.s 
@@ -145,8 +151,7 @@ eExit:		b		EXT(EmulExit)					; Just return for now...
 
 ;
 ;			Fetch the failing instruction.
-;			Image returned in R10 if CR0_EQ is false, otherwise, an ISI should be generated/
-;			The cr bit kernAccess is set if this was a kernel access.
+;			Image returned in R10 if CR0_EQ is false, otherwise, an ISI should be generated.
 ;			R1 has the DSISR if access failed.
 ;
 
@@ -155,15 +160,15 @@ eExit:		b		EXT(EmulExit)					; Just return for now...
 eIFetch:	lwz		r23,savesrr1+4(r13)				; Get old MSR
 			mflr	r28								; Save return
 
-			rlwinm.	r22,r23,0,MSR_PR_BIT,MSR_PR_BIT	; Within kernel?
-
+			rlwinm	r3,r23,32-MSR_DR_BIT+MSR_IR_BIT,MSR_DR_BIT,MSR_DR_BIT	; Move IR to DR for ifetch
 			mfmsr	r30								; Save the MSR for now
+			rlwimi	r3,r23,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	; Move DR to RI for ifetch
+			
 			lwz		r23,savesrr0+4(r13)				; Get instruction address
+			or		r3,r23,r3						; Turn on the DR and RI bit if translation was on
 			
-			ori		r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))		; Set RI and DR onto access MSR
-
 			crset	cr0_eq							; Set this to see if we failed
-			mtmsr	r22								; Flip DR, RI, and maybe PR on
+			mtmsr	r3								; Flip RI and, if IR was set, DR
 			isync
 			
 			lwz		r10,0(r23)						; Fetch the instruction
@@ -183,7 +188,7 @@ eRedriveAsISI:
 			lwz		r6,savesrr1+4(r13)				; Get the srr1 value
 			lwz		r4,SAVflags(r13)				; Pick up the flags
 			li		r11,T_INSTRUCTION_ACCESS		; Set failing instruction fetch code
-			rlwimi	r6,r1,0,0,4						; Move the DSISR bits to the SRR1
+			rlwimi	r6,r1,0,1,4						; Move the DSISR bits to the SRR1
 			oris	r4,r4,hi16(SAVredrive)			; Set the redrive bit
 			stw		r11,saveexception(r13)			; Set the replacement code
 			stw		r4,SAVflags(r13)				; Set redrive request
@@ -243,9 +248,11 @@ aan64:		lwz		r20,savedsisr(r13)				; Get the DSISR
 			dcbz	r29,r31							; Clear and allocate a cache line for us to work in
 			rlwinm	r24,r20,3,24,28					; Get displacement to register to update if update form
 			rlwimi	r20,r20,24,28,28				; Move load/store indication to the bottom of index
-			ori		r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))		; Set RI onto access MSR
+			rlwinm	r22,r22,0,MSR_DR_BIT,MSR_DR_BIT	; Move rupt DR to DR for ifetch
 			rlwimi	r20,r20,26,27,27				; Move single/double indication to just above the bottom
+			rlwimi	r22,r22,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	; Move DR to RI for i-fetch
 			lis		r29,hi16(EXT(aaFPopTable))		; High part of FP branch table
+			or		r22,r30,r22						; Set the DR and RI bits if translation was on
 			bf-		iFloat,aaNotFloat				; This is not a floating point instruction...
 			ori		r29,r29,lo16(EXT(aaFPopTable))	; Low part of FP branch table
 			
@@ -1012,9 +1019,10 @@ aaPassAlong:
 			.align	5
 			
 aaComExitrd:
+			lis		r11,hi16(srr1clr)				; Get the bits we need to clear
 			oris	r9,r9,hi16(SAVredrive)			; Set the redrive bit
+			andc	r12,r12,r11						; Clear what needs to be cleared
 			li		r11,T_TRACE						; Set trace interrupt
-			rlwinm	r12,r12,0,16,31					; Clear top half of SRR1
 			stw		r9,SAVflags(r13)				; Set the flags
 			stw		r11,saveexception(r13)			; Set the exception code
 			b		EXT(EmulExit)					; Exit and do trace interrupt...
diff --git a/osfmk/ppc/Emulate64.s b/osfmk/ppc/Emulate64.s
index 924caaf88..2e7854d3f 100644
--- a/osfmk/ppc/Emulate64.s
+++ b/osfmk/ppc/Emulate64.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2002-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */ 																							
 
 /* Emulate64.s
@@ -223,8 +229,8 @@ a64AlignAssistJoin:										// join here from program interrupt handler
 //		r29 = SRR1 (ie, MSR at interrupt)
 // Note that EE and IR are always off, and SF is always on in this code.
 
-		rlwinm	r3,r29,0,MSR_DR_BIT,MSR_DR_BIT			// was translation on at fault?
-        rlwimi	r3,r3,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	// if DR was set, set RI too
+		rlwinm	r3,r29,31,MSR_DR_BIT,MSR_DR_BIT			// Move instruction translate bit to DR
+        rlwimi	r3,r3,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	// if DR is now set, set RI too
         or		r25,r26,r3								// assemble MSR to use accessing target space
         
 
@@ -249,6 +255,10 @@ a64AlignAssistJoin:										// join here from program interrupt handler
         rlwinm.	r0,r19,0,enaNotifyEMb,enaNotifyEMb			// Should we notify?
         crnot	kNotify,cr0_eq        
 
+		rlwinm	r3,r29,0,MSR_DR_BIT,MSR_DR_BIT			// was data translation on at fault?
+        rlwimi	r3,r3,32-MSR_RI_BIT+MSR_DR_BIT,MSR_RI_BIT,MSR_RI_BIT	// if DR is now set, set RI too
+        or		r25,r26,r3								// assemble MSR to use accessing target space
+
 
 // Hash the intruction into a 5-bit value "AAAAB" used to index the branch table, and a
 // 1-bit kUpdate flag, as follows:
@@ -888,7 +898,7 @@ a64RedriveAsISI:						// this DSI happened fetching the opcode (r1==DSISR  r4==D
         mtmsr	r26						// turn DR back off
         isync							// wait for it to happen
         li		r30,T_INSTRUCTION_ACCESS
-        rlwimi	r29,r1,0,0,4			// insert the fault type from DSI's DSISR
+        rlwimi	r29,r1,0,1,4			// insert the fault type from DSI's DSISR
         std		r29,savesrr1(r13)		// update SRR1 to look like an ISI
         b		a64Redrive
 
diff --git a/osfmk/ppc/Firmware.h b/osfmk/ppc/Firmware.h
index c724b2e1d..c0a57f6f8 100644
--- a/osfmk/ppc/Firmware.h
+++ b/osfmk/ppc/Firmware.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/Firmware.s b/osfmk/ppc/Firmware.s
index d13104750..d5f687f34 100644
--- a/osfmk/ppc/Firmware.s
+++ b/osfmk/ppc/Firmware.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/FirmwareC.c b/osfmk/ppc/FirmwareC.c
index 654671ce1..5ddc41c71 100644
--- a/osfmk/ppc/FirmwareC.c
+++ b/osfmk/ppc/FirmwareC.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	This file contains firmware code.
@@ -44,238 +50,289 @@
 #include <pexpert/pexpert.h>
 #include <ddb/db_output.h>
 
-extern Boot_Video	boot_video_info;
 Boot_Video dgVideo;
-extern GDWorkArea	GratefulDebWork[];
+extern GDWorkArea GratefulDebWork[];
 
-typedef struct RuptCtr {								/* Counts hardware interrupts */
-	struct GDpos {										/* Screen position for Grateful Deb display */
-		unsigned short col;								/* Column  (-1 means no display) */
-		unsigned short row;								/* Row */
+struct RuptCtr {		/* Counts hardware interrupts */
+	struct GDpos {		/* Screen position for Grateful Deb display */
+		unsigned short col;	/* Column  (-1 means no display) */
+		unsigned short row;	/* Row */
 	} GDpos;
-	unsigned int count;									/* Count of interrupt */
-	unsigned int timed;									/* If set, count updates at timed rate  */
-	unsigned int lasttime;								/* Low of timebase when last updated */
-} RuptCtr;
-
-/*							Window layout for Grateful Deb:
- *							
- *							0				9
+	unsigned int count;	/* Count of interrupt */
+	unsigned int timed;	/* If set, count updates at timed rate  */
+	unsigned int lasttime;	/* Low of timebase when last updated */
+};
+
+/* Window layout for Grateful Deb:
  *
- *						0	Total			Decrimenter
- *						1	DSI				ISI
- *						2	System call		External
- *						3	SIGP			Floating point
- *						4	Program			Alignment
+ *	0				9
+ *
+ *	0	Total			Decrimenter
+ *	1	DSI				ISI
+ *	2	System call		External
+ *	3	SIGP			Floating point
+ *	4	Program			Alignment
  */
 
+struct RuptCtr RuptCtrs[96] = {
+	{	/* Total interruptions */
+		.GDpos = {
+			.col = 0,
+			.row = 0,
+		},
+		.count = 0,
+		.timed = 1,
+	},
+	{	/* Reset */
+		.GDpos = {
+			.col = -1,
+			.row = -1,
+		},
+		.count = 0,
+		.timed = 0,
+	},
+	{	/* Machine check */
+		.GDpos = {
+			.col = -1,
+			.row = -1,
+		},
+		.count = 0,
+		.timed = 0,
+	},
+	{	/* DSIs */
+		.GDpos = {
+			.col = 0,
+			.row = 1,
+		},
+		.count = 0,
+		.timed = 1},
+	{	/* ISIs */
+		.GDpos = {
+			.col = 1,
+			.row = 1,
+		},
+		.count = 0,
+		.timed = 1,
+	},
+	{	/* Externals */
+		.GDpos = {
+			.col = 1,
+			.row = 2,
+		},
+		.count = 0,
+		.timed = 1,
+	},
+	{	/* Alignment */
+		.GDpos = {
+			.col = 1,
+			.row = 4,
+		},
+		.count = 0,
+		.timed = 0,
+	},
+	{.GDpos = {.col = 0,.row = 4},.count = 0,.timed = 0},	/* Program */
+	{.GDpos = {.col = 1,.row = 3},.count = 0,.timed = 0},	/* Floating point */
+	{.GDpos = {.col = 1,.row = 0},.count = 0,.timed = 1},	/* Decrementer */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* I/O error */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = 0,.row = 2},.count = 0,.timed = 1},	/* System call */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Trace */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Floating point assist */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Performance monitor */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* VMX */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Instruction breakpoint */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* System management */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Trace */
+	{.GDpos = {.col = 0,.row = 3},.count = 0,.timed = 0},	/* SIGP */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Preemption */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Context switch */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Special, update frequency controls */
 
+	/*Start of second processor counts */
 
+	{.GDpos = {.col = 0,.row = 0},.count = 0,.timed = 1},	/* Total interruptions */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reset */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Machine check */
+	{.GDpos = {.col = 0,.row = 1},.count = 0,.timed = 1},	/* DSIs */
+	{.GDpos = {.col = 1,.row = 1},.count = 0,.timed = 1},	/* ISIs */
+	{.GDpos = {.col = 1,.row = 2},.count = 0,.timed = 1},	/* Externals */
+	{.GDpos = {.col = 1,.row = 4},.count = 0,.timed = 0},	/* Alignment */
+	{.GDpos = {.col = 0,.row = 4},.count = 0,.timed = 0},	/* Program */
+	{.GDpos = {.col = 1,.row = 3},.count = 0,.timed = 0},	/* Floating point */
+	{.GDpos = {.col = 1,.row = 0},.count = 0,.timed = 1},	/* Decrementer */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* I/O error */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = 0,.row = 2},.count = 0,.timed = 1},	/* System call */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Trace */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Floating point assist */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Performance monitor */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* VMX */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Instruction breakpoint */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* System management */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Trace */
+	{.GDpos = {.col = 0,.row = 3},.count = 0,.timed = 0},	/* SIGP */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Preemption */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Context switch */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Reserved */
+	{.GDpos = {.col = -1,.row = -1},.count = 0,.timed = 0},	/* Special, update frequency controls */
+};
 
-
-RuptCtr RuptCtrs[96] = {
-							{ { 0,  0}, 0, 1 }, 		/* Total interruptions */
-							{ {-1, -1}, 0, 0 }, 		/* Reset */
-							{ {-1, -1}, 0, 0 }, 		/* Machine check */
-							{ { 0,  1}, 0, 1 }, 		/* DSIs */
-							{ { 1,  1}, 0, 1 }, 		/* ISIs */
-							{ { 1,  2}, 0, 1 }, 		/* Externals */
-							{ { 1,  4}, 0, 0 }, 		/* Alignment */
-							{ { 0,  4}, 0, 0 }, 		/* Program */
-							{ { 1,  3}, 0, 0 }, 		/* Floating point */
-							{ { 1,  0}, 0, 1 }, 		/* Decrementer */
-							{ {-1, -1}, 0, 0 }, 		/* I/O error */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ { 0,  2}, 0, 1 }, 		/* System call */
-							{ {-1, -1}, 0, 0 }, 		/* Trace */
-							{ {-1, -1}, 0, 0 }, 		/* Floating point assist */
-							{ {-1, -1}, 0, 0 }, 		/* Performance monitor */
-							{ {-1, -1}, 0, 0 }, 		/* VMX */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Instruction breakpoint */
-							{ {-1, -1}, 0, 0 }, 		/* System management */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Trace */
-							{ { 0,  3}, 0, 0 }, 		/* SIGP */
-							{ {-1, -1}, 0, 0 }, 		/* Preemption */
-							{ {-1, -1}, 0, 0 }, 		/* Context switch */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Special, update frequency controls */
-
-/*							Start of second processor counts */
-							
-							{ { 0,  0}, 0, 1 }, 		/* Total interruptions */
-							{ {-1, -1}, 0, 0 }, 		/* Reset */
-							{ {-1, -1}, 0, 0 }, 		/* Machine check */
-							{ { 0,  1}, 0, 1 }, 		/* DSIs */
-							{ { 1,  1}, 0, 1 }, 		/* ISIs */
-							{ { 1,  2}, 0, 1 }, 		/* Externals */
-							{ { 1,  4}, 0, 0 }, 		/* Alignment */
-							{ { 0,  4}, 0, 0 }, 		/* Program */
-							{ { 1,  3}, 0, 0 }, 		/* Floating point */
-							{ { 1,  0}, 0, 1 }, 		/* Decrementer */
-							{ {-1, -1}, 0, 0 }, 		/* I/O error */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ { 0,  2}, 0, 1 }, 		/* System call */
-							{ {-1, -1}, 0, 0 }, 		/* Trace */
-							{ {-1, -1}, 0, 0 }, 		/* Floating point assist */
-							{ {-1, -1}, 0, 0 }, 		/* Performance monitor */
-							{ {-1, -1}, 0, 0 }, 		/* VMX */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Instruction breakpoint */
-							{ {-1, -1}, 0, 0 }, 		/* System management */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Trace */
-							{ { 0,  3}, 0, 0 }, 		/* SIGP */
-							{ {-1, -1}, 0, 0 }, 		/* Preemption */
-							{ {-1, -1}, 0, 0 }, 		/* Context switch */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 }, 		/* Reserved */
-							{ {-1, -1}, 0, 0 } 			/* Special, update frequency controls */
-						};
-
-void GratefulDebInit(bootBumbleC *boot_video_info) {	/* Initialize the video debugger */
+void
+GratefulDebInit(bootBumbleC *boot_video_info)
+{				/* Initialize the video debugger */
 
 	unsigned int fillframe[256];
-	unsigned int startpos, startbyte, windowleft, newwidth, i, j, startword, oldwidth, nrmlgn;
+	unsigned int startpos, startbyte, windowleft, newwidth, i, j, startword,
+	    oldwidth, nrmlgn;
 	unsigned int nwords, *byteleft, lstlgn, pixlgn, bytelgn;
-	
-	if(!boot_video_info) {								/* Are we disabling it?
-		GratefulDebWork[0].GDready = 0;					/* Disable output */
+
+	if (!boot_video_info) {	/* Are we disabling it? */
+		GratefulDebWork[0].GDready = 0;	/* Disable output */
 		return;
 	}
-	
+
 	nrmlgn = (9 * GDfontsize) * (boot_video_info->v_depth / 8);	/* Get the normal column size in bytes */
 	lstlgn = (((8 * GDfontsize) + (GDfontsize >> 1)) * boot_video_info->v_depth) / 8;	/* Same as normal, but with 1/2 character space */
-	nrmlgn = (nrmlgn + 31) & -32;						/* Round to a line */
-	
-	bytelgn = (nrmlgn * (GDdispcols - 1)) + lstlgn;		/* Length in bytes */
+	nrmlgn = (nrmlgn + 31) & -32;	/* Round to a line */
+
+	bytelgn = (nrmlgn * (GDdispcols - 1)) + lstlgn;	/* Length in bytes */
 	pixlgn = bytelgn / (boot_video_info->v_depth / 8);	/* Number of pixels wide */
-	
+
 	startbyte = (boot_video_info->v_width * (boot_video_info->v_depth / 8)) - bytelgn;	/* Get the starting byte unaligned */
-	startpos = boot_video_info->v_width - pixlgn;		/* Starting pixel position */
-	
+	startpos = boot_video_info->v_width - pixlgn;	/* Starting pixel position */
+
 	startbyte += (unsigned int)boot_video_info->v_baseAddr & 31;	/* Add the extra to cache boundary in frame buffer */
-	startbyte &= -32;									/* Make sure it's on a cache line for speed */
+	startbyte &= -32;	/* Make sure it's on a cache line for speed */
 	startbyte += (unsigned int)boot_video_info->v_baseAddr & 31;	/* Subtract the extra to cache boundary in frame buffer */
 
 	windowleft = startbyte - (((GDfontsize / 2) * boot_video_info->v_depth) / 8);	/* Back up a half character */
-	windowleft &= -4;									/* Make sure it is on a word boundary */
+	windowleft &= -4;	/* Make sure it is on a word boundary */
 	newwidth = windowleft / (boot_video_info->v_depth / 8);	/* Get the new pixel width of screen */
-	
-	oldwidth = boot_video_info->v_width;					/* Save the old width */
-//	boot_video_info->v_width = newwidth;					/* Set the new width */
-	
-	nwords = oldwidth - newwidth;						/* See how much to fill in pixels */
+
+	oldwidth = boot_video_info->v_width;	/* Save the old width */
+//      boot_video_info->v_width = newwidth;                                    /* Set the new width */
+
+	nwords = oldwidth - newwidth;	/* See how much to fill in pixels */
 	nwords = nwords / (32 / boot_video_info->v_depth);	/* Get that in bytes */
-	
-	startword = (newwidth + 3) / 4;						/* Where does it start? */
-	
-	
+
+	startword = (newwidth + 3) / 4;	/* Where does it start? */
+
 	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft);	/* Starting place */
-	for (i=0; i < nwords; i++) byteleft[i] = 0;		/* Set the row to all black */
-	
+	for (i = 0; i < nwords; i++)
+		byteleft[i] = 0;	/* Set the row to all black */
+
 	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + (boot_video_info->v_rowBytes * 1));	/* Starting place */
-	for (i=0; i < nwords; i++) byteleft[i] = 0;		/* Set the row to all black */
-	
-	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + 
-		(boot_video_info->v_rowBytes * (boot_video_info->v_height - 2)));	/* Starting place */
-	for (i=0; i < nwords; i++) byteleft[i] = 0;		/* Set the row to all black */
-	
-	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + 
-		(boot_video_info->v_rowBytes * (boot_video_info->v_height - 1)));	/* Starting place */
-	for (i=0; i < nwords; i++) byteleft[i] = 0;		/* Set the row to all black */
-	
-	for (i=0; i < nwords; i++) fillframe[i] = 0xFFFFFFFF;	/* Set the row to all white */
-	
-	if(boot_video_info->v_depth == 8) {				/* See if 8 bits a pixel */
-		fillframe[0] = 0x0000FFFF;					/* Make left border */
-		fillframe[nwords - 1] = 0xFFFF0000;			/* Make right border */
-	}
-	else if(boot_video_info->v_depth == 16) {		/* See if 16 bits a pixel */
-		fillframe[0] = 0x00000000;					/* Make left border */
-		fillframe[nwords - 1] = 0x00000000;			/* Make right border */
-	}
-	else {
-		fillframe[0] = 0x00000000;					/* Make left border */
-		fillframe[1] = 0x00000000;					/* Make left border */
-		fillframe[nwords - 1] = 0x00000000;			/* Make right border */
-		fillframe[nwords - 2] = 0x00000000;			/* Make right border */
+	for (i = 0; i < nwords; i++)
+		byteleft[i] = 0;	/* Set the row to all black */
+
+	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + (boot_video_info->v_rowBytes * (boot_video_info->v_height - 2)));	/* Starting place */
+	for (i = 0; i < nwords; i++)
+		byteleft[i] = 0;	/* Set the row to all black */
+
+	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + (boot_video_info->v_rowBytes * (boot_video_info->v_height - 1)));	/* Starting place */
+	for (i = 0; i < nwords; i++)
+		byteleft[i] = 0;	/* Set the row to all black */
+
+	for (i = 0; i < nwords; i++)
+		fillframe[i] = 0xFFFFFFFF;	/* Set the row to all white */
+
+	if (boot_video_info->v_depth == 8) {	/* See if 8 bits a pixel */
+		fillframe[0] = 0x0000FFFF;	/* Make left border */
+		fillframe[nwords - 1] = 0xFFFF0000;	/* Make right border */
+	} else if (boot_video_info->v_depth == 16) {	/* See if 16 bits a pixel */
+		fillframe[0] = 0x00000000;	/* Make left border */
+		fillframe[nwords - 1] = 0x00000000;	/* Make right border */
+	} else {
+		fillframe[0] = 0x00000000;	/* Make left border */
+		fillframe[1] = 0x00000000;	/* Make left border */
+		fillframe[nwords - 1] = 0x00000000;	/* Make right border */
+		fillframe[nwords - 2] = 0x00000000;	/* Make right border */
 	}
-	
+
 	byteleft = (unsigned int *)(boot_video_info->v_baseAddr + windowleft + (boot_video_info->v_rowBytes * 2));	/* Place to start filling */
-		
-	for(i=2; i < (boot_video_info->v_height - 2); i++) {	/* Fill the rest */
-		for(j=0; j<nwords; j++) byteleft[j] = fillframe[j];	/* Fill the row */
+
+	for (i = 2; i < (boot_video_info->v_height - 2); i++) {	/* Fill the rest */
+		for (j = 0; j < nwords; j++)
+			byteleft[j] = fillframe[j];	/* Fill the row */
 		byteleft = (unsigned int *)((unsigned int)byteleft + boot_video_info->v_rowBytes);	/* Next row */
 	}
 
-	for(i=0; i<2; i++) {							/* Initialize both (for now) processor areas */
-		
-		GratefulDebWork[i].GDtop = 2 + (GDfontsize / 2) + (i * 18 * GDfontsize);
+	for (i = 0; i < 2; i++) {	/* Initialize both (for now) processor areas */
+
+		GratefulDebWork[i].GDtop =
+		    2 + (GDfontsize / 2) + (i * 18 * GDfontsize);
 		GratefulDebWork[i].GDleft = 2 + startpos + (GDfontsize / 2);
-		GratefulDebWork[i].GDtopleft = boot_video_info->v_baseAddr + startbyte + 
-			(GratefulDebWork[i].GDtop * boot_video_info->v_rowBytes);
+		GratefulDebWork[i].GDtopleft =
+		    boot_video_info->v_baseAddr + startbyte +
+		    (GratefulDebWork[i].GDtop * boot_video_info->v_rowBytes);
 		GratefulDebWork[i].GDrowbytes = boot_video_info->v_rowBytes;
-		GratefulDebWork[i].GDrowchar = boot_video_info->v_rowBytes * (GDfontsize + (GDfontsize / 4));
+		GratefulDebWork[i].GDrowchar =
+		    boot_video_info->v_rowBytes * (GDfontsize +
+						   (GDfontsize / 4));
 		GratefulDebWork[i].GDdepth = boot_video_info->v_depth;
 		GratefulDebWork[i].GDcollgn = nrmlgn;
-	
-//		RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 4;	/* (Update every 16th of a second (16 fps) */
-		RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 3;	/* (Update every 8th of a second (8 fps) */
-//		RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 2;	/* (Update every 4th of a second (4 fps) */
-//		RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 1;	/* (Update every 2th of a second (2 fps) */
-//		RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 0;	/* (Update every 1 second (1 fps) */
-		
+
+//              RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 4;   /* (Update every 16th of a second (16 fps) */
+		RuptCtrs[(48 * i) + 47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 3;	/* (Update every 8th of a second (8 fps) */
+//              RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 2;   /* (Update every 4th of a second (4 fps) */
+//              RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 1;   /* (Update every 2th of a second (2 fps) */
+//              RuptCtrs[(48*i)+47].timed = gPEClockFrequencyInfo.timebase_frequency_hz >> 0;   /* (Update every 1 second (1 fps) */
+
 		sync();
-		
-		GratefulDebWork[i].GDready = 1;				/* This one's all ready */
-	}
 
+		GratefulDebWork[i].GDready = 1;	/* This one's all ready */
+	}
 }
 
 void debugNoop(void);
-void debugNoop(void) {								/* This does absolutely nothing */
-	return;
+void
+debugNoop(void)
+{				/* This does absolutely nothing */
 }
diff --git a/osfmk/ppc/FirmwareCalls.h b/osfmk/ppc/FirmwareCalls.h
index 248f76986..ec25ea30c 100644
--- a/osfmk/ppc/FirmwareCalls.h
+++ b/osfmk/ppc/FirmwareCalls.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/Makefile b/osfmk/ppc/Makefile
index 03891582a..b978cc676 100644
--- a/osfmk/ppc/Makefile
+++ b/osfmk/ppc/Makefile
@@ -16,7 +16,6 @@ EXPORT_ONLY_FILES = 		\
 	io_map_entries.h	\
 	lock.h			\
 	locks.h			\
-	pms.h			\
 	proc_reg.h		\
 	machine_routines.h	\
 	mappings.h		\
diff --git a/osfmk/ppc/PPCcalls.c b/osfmk/ppc/PPCcalls.c
index 2131832a1..39203ec5d 100644
--- a/osfmk/ppc/PPCcalls.c
+++ b/osfmk/ppc/PPCcalls.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/task.h>
diff --git a/osfmk/ppc/PPCcalls.h b/osfmk/ppc/PPCcalls.h
index adc5badf3..262fe2e91 100644
--- a/osfmk/ppc/PPCcalls.h
+++ b/osfmk/ppc/PPCcalls.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
diff --git a/osfmk/ppc/Performance.h b/osfmk/ppc/Performance.h
index b06f1f018..4442d603e 100644
--- a/osfmk/ppc/Performance.h
+++ b/osfmk/ppc/Performance.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *		Keep special performance related stuff in here
diff --git a/osfmk/ppc/Performance.s b/osfmk/ppc/Performance.s
index 7d6d438db..440c39678 100644
--- a/osfmk/ppc/Performance.s
+++ b/osfmk/ppc/Performance.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT_INTERNAL_USE_ONLY@
diff --git a/osfmk/ppc/PseudoKernel.c b/osfmk/ppc/PseudoKernel.c
index 2ef1a5986..66dd94e2e 100644
--- a/osfmk/ppc/PseudoKernel.c
+++ b/osfmk/ppc/PseudoKernel.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  	File:		PseudoKernel.c
@@ -31,6 +37,7 @@
 */
 
 #include <mach/mach_types.h>
+#include <mach/mach_host.h>
 #include <mach/kern_return.h>
 
 #include <kern/kalloc.h>
@@ -47,6 +54,9 @@
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
 
+extern int is_suser(void);
+extern void tbeproc(void *proc);
+
 void bbSetRupt(ReturnHandler *rh, thread_t ct);
 
 /*
@@ -61,10 +71,10 @@ void bbSetRupt(ReturnHandler *rh, thread_t ct);
 ** Notes:
 **
 */
-kern_return_t syscall_notify_interrupt ( void ) {
-  
-    UInt32			interruptState; 
-    task_t			task;
+kern_return_t
+syscall_notify_interrupt(void)
+{
+	task_t			task;
 	thread_t 		act, fact;
 	bbRupt			*bbr;
 	BTTD_t			*bttd;
@@ -75,7 +85,7 @@ kern_return_t syscall_notify_interrupt ( void ) {
 	task_lock(task);						/* Lock our task */
 	
 	fact = (thread_t)task->threads.next;		/* Get the first activation on task */
-	act = 0;										/* Pretend we didn't find it yet */
+	act = NULL;										/* Pretend we didn't find it yet */
 	
 	for(i = 0; i < task->thread_count; i++) {		/* Scan the whole list */
 		if(fact->machine.bbDescAddr) {					/* Is this a Blue thread? */
@@ -104,7 +114,7 @@ kern_return_t syscall_notify_interrupt ( void ) {
 	 * setting up the asynchronous task by setting a pending interrupt.
 	 */
 	
-	if ( (unsigned int)act == (unsigned int)current_thread() ) {		
+	if (act == current_thread()) {		
 		bttd->InterruptControlWord = bttd->InterruptControlWord | 
 			((bttd->postIntMask >> kCR2ToBackupShift) & kBackupCR2Mask);
 				
@@ -146,7 +156,7 @@ kern_return_t syscall_notify_interrupt ( void ) {
 
 void bbSetRupt(ReturnHandler *rh, thread_t act) {
 
-	savearea 	*sv;
+	struct savearea	*sv;
 	BTTD_t		*bttd;
 	bbRupt		*bbr;
 	UInt32		interruptState;
@@ -214,14 +224,20 @@ void bbSetRupt(ReturnHandler *rh, thread_t act) {
  * The assist code can be called from two types of threads.  The blue thread, which handles 
  * traps, system calls and interrupts and preemptive threads that only issue system calls.
  *
+ * Parameters:	host			.
+ * 		_taskID			opaque task ID
+ * 		_TWI_TableStart		Start of TWI table
+ * 		_Desc_TableStart	Start of descriptor table
  */ 
 
-kern_return_t enable_bluebox(
-      host_t host,
-	  void *taskID,								/* opaque task ID */
-	  void *TWI_TableStart,						/* Start of TWI table */
-	  char *Desc_TableStart						/* Start of descriptor table */
-	 ) {
+kern_return_t
+enable_bluebox(host_t host, unsigned _taskID, unsigned _TWI_TableStart,
+	       unsigned _Desc_TableStart)
+{
+	/* XXX mig funness */
+	void *taskID = (void *)_taskID;
+	void *TWI_TableStart = (void *)_TWI_TableStart;
+	char *Desc_TableStart = (char *)_Desc_TableStart;
 	
 	thread_t 		th;
 	vm_offset_t		kerndescaddr, origdescoffset;
@@ -254,7 +270,7 @@ kern_return_t enable_bluebox(
 	}
 		
 	physdescpage = 											/* Get the physical page number of the page */
-		pmap_find_phys(th->map->pmap, (addr64_t)Desc_TableStart);
+		pmap_find_phys(th->map->pmap, CAST_USER_ADDR_T(Desc_TableStart));
 
 	ret =  kmem_alloc_pageable(kernel_map, &kerndescaddr, PAGE_SIZE);	/* Find a virtual address to use */
 	if(ret != KERN_SUCCESS) {								/* Could we get an address? */
@@ -294,7 +310,6 @@ kern_return_t enable_bluebox(
 		
 	{
 		/* mark the proc to indicate that this is a TBE proc */
-		extern void tbeproc(void *proc);
 
 		tbeproc(th->task->bsd_info);
 	}
@@ -351,7 +366,10 @@ int bb_enable_bluebox( struct savearea *save )
 {
 	kern_return_t rc;
 
-	rc = enable_bluebox( (host_t)0xFFFFFFFF, (void *)save->save_r3, (void *)save->save_r4, (char *)save->save_r5 );
+	rc = enable_bluebox((host_t)0xFFFFFFFF,
+			    CAST_DOWN(unsigned, save->save_r3),
+			    CAST_DOWN(unsigned, save->save_r4),
+			    CAST_DOWN(unsigned, save->save_r5));
 	save->save_r3 = rc;
 	return 1;										/* Return with normal AST checking */
 }
@@ -389,7 +407,7 @@ int bb_settaskenv( struct savearea *save )
 
 	task_lock(task);								/* Lock our task */
 	fact = (thread_t)task->threads.next;		/* Get the first activation on task */
-	act = 0;										/* Pretend we didn't find it yet */
+	act = NULL;										/* Pretend we didn't find it yet */
 	
 	for(i = 0; i < task->thread_count; i++) {		/* Scan the whole list */
 		if(fact->machine.bbDescAddr) {					/* Is this a Blue thread? */
diff --git a/osfmk/ppc/PseudoKernel.h b/osfmk/ppc/PseudoKernel.h
index ca3722b2c..31b83af7e 100644
--- a/osfmk/ppc/PseudoKernel.h
+++ b/osfmk/ppc/PseudoKernel.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
 	File:		PseudoKernelPriv.h
@@ -40,6 +46,8 @@ extern int bb_enable_bluebox(struct savearea *);
 extern int bb_disable_bluebox(struct savearea *);
 extern int bb_settaskenv(struct savearea *);
 
+kern_return_t syscall_notify_interrupt(void);
+
 struct BlueExceptionDataArea {
 	UInt32				srr0;					// OUT PC at time of exception, IN return address
 	UInt32				srr1;					// OUT/IN msr FE0, BE, SE and FE1 bits to restore on exit
diff --git a/osfmk/ppc/_setjmp.s b/osfmk/ppc/_setjmp.s
index cfe3d05cf..534fc3536 100644
--- a/osfmk/ppc/_setjmp.s
+++ b/osfmk/ppc/_setjmp.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/aligned_data.s b/osfmk/ppc/aligned_data.s
index 4f703ebba..1777b577e 100644
--- a/osfmk/ppc/aligned_data.s
+++ b/osfmk/ppc/aligned_data.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *		This module only exists because I don't know how to get the silly C compiler
diff --git a/osfmk/ppc/asm.h b/osfmk/ppc/asm.h
index 320af7d1f..2535a8491 100644
--- a/osfmk/ppc/asm.h
+++ b/osfmk/ppc/asm.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -593,6 +599,23 @@
 #define ictce		31
 #define ictcem		0x00000001
 
+#define slbESID	36
+#define slbKey	52
+#define slbIndex 52
+#define slbV	36
+#define slbVm	0x08000000
+#define slbCnt	64
+
+/*
+ * Macros to access high and low word values of an address
+ */
+
+#define	HIGH_CADDR(x)	ha16(x)
+#define	HIGH_ADDR(x)	hi16(x)
+#define	LOW_ADDR(x)	lo16(x)
+
+#endif	/* ASSEMBLER */
+
 #define cr0_lt	0
 #define cr0_gt	1
 #define cr0_eq	2
@@ -634,23 +657,6 @@
 #define cr7_so	31
 #define cr7_un	31
 
-#define slbESID	36
-#define slbKey	52
-#define slbIndex 52
-#define slbV	36
-#define slbVm	0x08000000
-#define slbCnt	64
-
-/*
- * Macros to access high and low word values of an address
- */
-
-#define	HIGH_CADDR(x)	ha16(x)
-#define	HIGH_ADDR(x)	hi16(x)
-#define	LOW_ADDR(x)	lo16(x)
-
-#endif	/* ASSEMBLER */
-
 /*	GUS Mode Register */
 #define GUSModeReg 0x0430
 #define GUSMdmapen 0x00008000
@@ -662,6 +668,18 @@
 #define PowerTuneControlReg	0x0AA001
 #define PowerTuneStatusReg	0x408001
 
+/* Code inject */
+//	The following bits are always on in the MSR when injected code is executing
+#define ijemon  0x00000010
+//	The following bits are always off in the MSR when injected code it executing
+#define ijemoff 0x0000C620
+#define ijemtrap ijemon|1
+//	The following is the inject exit trap
+#define ijtrap 0x0FFFC9C9
+
+/* Misc */
+#define srr1clr 0x783F0000
+
 /* Tags are placed before Immediately Following Code (IFC) for the debugger
  * to be able to deduce where to find various registers when backtracing
  * 
diff --git a/osfmk/ppc/ast.h b/osfmk/ppc/ast.h
index 274a8d84a..a24933948 100644
--- a/osfmk/ppc/ast.h
+++ b/osfmk/ppc/ast.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/ast_types.h b/osfmk/ppc/ast_types.h
index 4db875d69..a32dd6f9d 100644
--- a/osfmk/ppc/ast_types.h
+++ b/osfmk/ppc/ast_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/atomic_switch.h b/osfmk/ppc/atomic_switch.h
index 7b2eed2ff..f31743cc5 100644
--- a/osfmk/ppc/atomic_switch.h
+++ b/osfmk/ppc/atomic_switch.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 typedef unsigned char	UInt8;
 typedef unsigned short	UInt16;
diff --git a/osfmk/ppc/atomic_switch.s b/osfmk/ppc/atomic_switch.s
index effa74de1..ef1edd940 100644
--- a/osfmk/ppc/atomic_switch.s
+++ b/osfmk/ppc/atomic_switch.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include	<ppc/asm.h>
 #include	<ppc/proc_reg.h>
diff --git a/osfmk/ppc/bat_init.c b/osfmk/ppc/bat_init.c
index 5d36e6fa4..7434a4f02 100644
--- a/osfmk/ppc/bat_init.c
+++ b/osfmk/ppc/bat_init.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/std_types.h>
 #include <ppc/proc_reg.h>
diff --git a/osfmk/ppc/bcopy.s b/osfmk/ppc/bcopy.s
index 12edce704..bc05940f2 100644
--- a/osfmk/ppc/bcopy.s
+++ b/osfmk/ppc/bcopy.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 ;
 ;			Copy bytes of data around. Handles overlapped data.
diff --git a/osfmk/ppc/bcopytest.c b/osfmk/ppc/bcopytest.c
index dc994332f..5903c43c4 100644
--- a/osfmk/ppc/bcopytest.c
+++ b/osfmk/ppc/bcopytest.c
@@ -48,14 +48,10 @@ void bcopytest(void);
 void bcopytest(void) {
 
 	void *srcptr, *snkptr, *asrc, *asnk;
-	int bsrc, bsnk, size, iterations, i, ret, n; 
-	unsigned int makerand, timein, timeout, ticks;
+	int bsrc, bsnk, size, i, ret, n; 
 	volatile int dbg = 0;
 	unsigned int *sink, *source;
 	
-	long long tottime, totbytes;
-	long long tottime2, totbytes2;
-	
 	kern_return_t retr;
 	
 	db_printf("bcopy test\n");	
@@ -451,8 +447,9 @@ void clrarea(unsigned int *source, unsigned int *sink) {
 	return;
 }
 
-void clrarea2(unsigned int *source, unsigned int *sink) {
-
+void
+clrarea2(unsigned int *source, __unused unsigned int *sink)
+{
 	unsigned int i;
 	unsigned char *ss;
 	
@@ -558,7 +555,7 @@ int tstcopy4(void *src, void *snk, unsigned int lgn) {
 
 int tstcopy5(void *src, void *snk, unsigned int lgn) {
 
-	unsigned int i, crap;
+	unsigned int i = 0, crap;
 	unsigned char ic, ec, oic, pc;
 	
 	oic = ((unsigned char *)snk)[0];				/* Original first sink character */
@@ -599,7 +596,7 @@ int tstcopy5(void *src, void *snk, unsigned int lgn) {
 }
 
 int dumbcopy(void *src, void *snk, unsigned int lgn) {
-	int i;
+	unsigned int i;
 	char *p = (char *)snk;
 	char *q = (char *)src;
 	
diff --git a/osfmk/ppc/bits.s b/osfmk/ppc/bits.s
index a01cf81fe..d8d5960d5 100644
--- a/osfmk/ppc/bits.s
+++ b/osfmk/ppc/bits.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/boot.h b/osfmk/ppc/boot.h
index e37a7d660..9d3e885ee 100644
--- a/osfmk/ppc/boot.h
+++ b/osfmk/ppc/boot.h
@@ -1,22 +1,28 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include	<pexpert/ppc/boot.h>
diff --git a/osfmk/ppc/bzero.s b/osfmk/ppc/bzero.s
index 4bb69a38e..0dbd810b4 100644
--- a/osfmk/ppc/bzero.s
+++ b/osfmk/ppc/bzero.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <ppc/asm.h>
@@ -30,7 +36,28 @@
         .globl	_bzero
         .globl	_bzero_nc
         .globl	_bzero_phys
+        .globl	_bzero_phys_nc
+
 
+// *****************************
+// * B Z E R O _ P H Y S _ N C *
+// *****************************
+//
+// void bzero_phys_nc(addr64_t phys_addr, uint32_t length);
+//
+// Takes a phys addr in (r3,r4), and length in r5.  NO CACHING
+
+        .align	5
+LEXT(bzero_phys_nc)
+        mflr	r12				// save return address
+        rlwinm	r3,r3,0,1,0		// coallesce long-long in (r3,r4) into reg64_t in r3
+        rlwimi	r3,r4,0,0,31
+        mr		r4,r5			// put length where bzero() expects it
+        bl		EXT(ml_set_physical_get_ffs)	// turn DR off, SF on, features in cr6, old MSR in r11
+        bl		EXT(bzero_nc)		// use normal bzero() routine
+        mtlr	r12				// restore return
+        b		EXT(ml_restore)		// restore MSR, turning DR on and SF off
+        
 
 // ***********************
 // * B Z E R O _ P H Y S *
diff --git a/osfmk/ppc/cache.s b/osfmk/ppc/cache.s
index 5318f83f6..94aa0aeeb 100644
--- a/osfmk/ppc/cache.s
+++ b/osfmk/ppc/cache.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/commpage/atomic.s b/osfmk/ppc/commpage/atomic.s
index 38a82f839..a53e61fe3 100644
--- a/osfmk/ppc/commpage/atomic.s
+++ b/osfmk/ppc/commpage/atomic.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/ppc/commpage/bcopy_64.s b/osfmk/ppc/commpage/bcopy_64.s
index e336c2ea6..4d0b2c9bd 100644
--- a/osfmk/ppc/commpage/bcopy_64.s
+++ b/osfmk/ppc/commpage/bcopy_64.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* =======================================
  * BCOPY, MEMCPY, and MEMMOVE for Mac OS X
@@ -61,7 +67,6 @@
 #define	w7	r0
 #define	w8	r2
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/bcopy_970.s b/osfmk/ppc/commpage/bcopy_970.s
index 8417fde74..61916abf2 100644
--- a/osfmk/ppc/commpage/bcopy_970.s
+++ b/osfmk/ppc/commpage/bcopy_970.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* =======================================
  * BCOPY, MEMCPY, and MEMMOVE for Mac OS X
@@ -66,7 +72,6 @@
 #define	vy	v11
 #define	vz	v12
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/bcopy_g3.s b/osfmk/ppc/commpage/bcopy_g3.s
index fa7d8dd24..f0900963e 100644
--- a/osfmk/ppc/commpage/bcopy_g3.s
+++ b/osfmk/ppc/commpage/bcopy_g3.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* =======================================
  * BCOPY, MEMCPY, and MEMMOVE for Mac OS X
@@ -55,7 +61,6 @@
 #define	w7	r0
 #define	w8	r2
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/bcopy_g4.s b/osfmk/ppc/commpage/bcopy_g4.s
index 1fd142e82..0d901ab20 100644
--- a/osfmk/ppc/commpage/bcopy_g4.s
+++ b/osfmk/ppc/commpage/bcopy_g4.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* =======================================
  * BCOPY, MEMCPY, and MEMMOVE for Mac OS X
@@ -79,7 +85,6 @@
 #define	vx	v6
 #define	vy	v7
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/bigcopy_970.s b/osfmk/ppc/commpage/bigcopy_970.s
index e8aad5f77..add093ea3 100644
--- a/osfmk/ppc/commpage/bigcopy_970.s
+++ b/osfmk/ppc/commpage/bigcopy_970.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /* ====================================
  * Very Long Operand BCOPY for Mac OS X
diff --git a/osfmk/ppc/commpage/bzero_128.s b/osfmk/ppc/commpage/bzero_128.s
index 055ef9968..f22198478 100644
--- a/osfmk/ppc/commpage/bzero_128.s
+++ b/osfmk/ppc/commpage/bzero_128.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/bzero_32.s b/osfmk/ppc/commpage/bzero_32.s
index 9bee8a55a..fe7653d6d 100644
--- a/osfmk/ppc/commpage/bzero_32.s
+++ b/osfmk/ppc/commpage/bzero_32.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/cacheflush.s b/osfmk/ppc/commpage/cacheflush.s
index 0b8cb48ae..43d7452ea 100644
--- a/osfmk/ppc/commpage/cacheflush.s
+++ b/osfmk/ppc/commpage/cacheflush.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>					// EXT, LEXT
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/commpage.c b/osfmk/ppc/commpage/commpage.c
index d7eca5030..6b0227322 100644
--- a/osfmk/ppc/commpage/commpage.c
+++ b/osfmk/ppc/commpage/commpage.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -51,8 +57,8 @@
 #include <vm/vm_map.h>
 #include <ipc/ipc_port.h>
 
-extern	vm_map_t	com_region_map32;   // the 32-bit shared submap, set up in vm init
-extern  vm_map_t	com_region_map64;   // the 64-bit shared submap
+extern	vm_map_t	commpage32_map;   // the 32-bit shared submap, set up in vm init
+extern  vm_map_t	commpage64_map;   // the 64-bit shared submap
 
 char	*commPagePtr32 = NULL;			// virtual address of 32-bit comm page in kernel map
 char	*commPagePtr64 = NULL;			// and 64-bit commpage
@@ -187,9 +193,9 @@ static	commpage_descriptor	*routines[] = {
  */
 static	void*
 commpage_allocate( 
-	vm_map_t			submap )					// com_region_map32 or com_region_map64
+	vm_map_t			submap )					// commpage32_map or commpage64_map
 {
-    vm_offset_t			kernel_addr;				// address of commpage in kernel map
+    vm_offset_t			kernel_addr = 0;		// address of commpage in kernel map
     vm_offset_t			zero = 0;
     vm_size_t			size = _COMM_PAGE_AREA_USED;	// size actually populated
     vm_map_entry_t		entry;
@@ -198,7 +204,7 @@ commpage_allocate(
     if (submap == NULL)
         panic("commpage submap is null");
     
-    if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_USED,VM_FLAGS_ANYWHERE))
+    if (vm_map(kernel_map,&kernel_addr,_COMM_PAGE_AREA_USED,0,VM_FLAGS_ANYWHERE,NULL,0,FALSE,VM_PROT_ALL,VM_PROT_ALL,VM_INHERIT_NONE))
         panic("cannot allocate commpage");
         
     if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_USED,VM_PROT_DEFAULT,FALSE))
@@ -219,7 +225,7 @@ commpage_allocate(
     if (mach_make_memory_entry(	kernel_map,			// target map
                                 &size,				// size
                                 kernel_addr,		// offset (address in kernel map)
-                                VM_PROT_DEFAULT,	// map it RW
+                                VM_PROT_ALL,		// map it RWX
                                 &handle,			// this is the object handle we get
                                 NULL ))				// parent_entry
         panic("cannot make entry for commpage");
@@ -232,8 +238,8 @@ commpage_allocate(
                     handle,							// port is the memory entry we just made
                     0,								// offset (map 1st page in memory entry)
                     FALSE,							// copy
-                    VM_PROT_READ,					// cur_protection (R-only in user map)
-                    VM_PROT_READ,					// max_protection
+                    VM_PROT_READ|VM_PROT_EXECUTE,				// cur_protection (R-only in user map)
+                    VM_PROT_READ|VM_PROT_EXECUTE,				// max_protection
                     VM_INHERIT_SHARE ))				// inheritance
         panic("cannot map commpage");
         
@@ -313,7 +319,7 @@ commpage_stuff(
     char	*dest = commpage_addr_of(address);
     
     if (dest < next)
-        panic("commpage overlap: %08 - %08X", dest, next);
+        panic("commpage overlap: %p - %p", dest, next);
     
     bcopy((const char*)source,dest,length);
     
@@ -664,9 +670,9 @@ void
 commpage_populate( void )
 {
     commpage_init_cpu_capabilities();
-	commpage_populate_one( com_region_map32, &commPagePtr32, kCommPage32, "commpage 32-bit");
+	commpage_populate_one( commpage32_map, &commPagePtr32, kCommPage32, "commpage 32-bit");
 	if (_cpu_capabilities & k64Bit) {
-		commpage_populate_one( com_region_map64, &commPagePtr64, kCommPage64, "commpage 64-bit");
+		commpage_populate_one( commpage64_map, &commPagePtr64, kCommPage64, "commpage 64-bit");
 		pmap_init_sharedpage((vm_offset_t)commPagePtr64);			// Do the 64-bit version        
 	}
         
diff --git a/osfmk/ppc/commpage/commpage.h b/osfmk/ppc/commpage/commpage.h
index 486e34242..ec176ec4e 100644
--- a/osfmk/ppc/commpage/commpage.h
+++ b/osfmk/ppc/commpage/commpage.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _PPC_COMMPAGE_H
@@ -74,9 +80,11 @@ typedef	struct	commpage_descriptor	{
 extern	char	*commPagePtr32;				// virt address of 32-bit commpage in kernel map
 extern	char	*commPagePtr64;				// virt address of 64-bit commpage in kernel map
 
-
 extern	void	commpage_set_timestamp(uint64_t tbr, uint64_t secs, uint32_t ticks_per_sec);
-extern	int		commpage_time_dcba( void );
+
+#define	commpage_disable_timestamp() commpage_set_timestamp( 0, 0, 0 )
+
+extern	int	commpage_time_dcba( void );
 
 #endif	/* __ASSEMBLER__ */
 
diff --git a/osfmk/ppc/commpage/commpage_asm.s b/osfmk/ppc/commpage/commpage_asm.s
index c3770a8fb..d3ea83c24 100644
--- a/osfmk/ppc/commpage/commpage_asm.s
+++ b/osfmk/ppc/commpage/commpage_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/ppc/commpage/gettimeofday.s b/osfmk/ppc/commpage/gettimeofday.s
index ada3d1481..e9645ee37 100644
--- a/osfmk/ppc/commpage/gettimeofday.s
+++ b/osfmk/ppc/commpage/gettimeofday.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>					// EXT, LEXT
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/mach_absolute_time.s b/osfmk/ppc/commpage/mach_absolute_time.s
index e2441bde5..be9345dad 100644
--- a/osfmk/ppc/commpage/mach_absolute_time.s
+++ b/osfmk/ppc/commpage/mach_absolute_time.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>					// EXT, LEXT
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/memset_64.s b/osfmk/ppc/commpage/memset_64.s
index 938159ac3..187e742b6 100644
--- a/osfmk/ppc/commpage/memset_64.s
+++ b/osfmk/ppc/commpage/memset_64.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/memset_g3.s b/osfmk/ppc/commpage/memset_g3.s
index cf9ae1d42..469627f85 100644
--- a/osfmk/ppc/commpage/memset_g3.s
+++ b/osfmk/ppc/commpage/memset_g3.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/memset_g4.s b/osfmk/ppc/commpage/memset_g4.s
index da47f785f..9e33f45f2 100644
--- a/osfmk/ppc/commpage/memset_g4.s
+++ b/osfmk/ppc/commpage/memset_g4.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/memset_g5.s b/osfmk/ppc/commpage/memset_g5.s
index 048ba75a4..6acf98579 100644
--- a/osfmk/ppc/commpage/memset_g5.s
+++ b/osfmk/ppc/commpage/memset_g5.s
@@ -1,26 +1,31 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
-#define	ASSEMBLER
 #include <sys/appleapiopts.h>
 #include <ppc/asm.h>
 #include <machine/cpu_capabilities.h>
diff --git a/osfmk/ppc/commpage/pthread.s b/osfmk/ppc/commpage/pthread.s
index 24c785121..58dd6c4aa 100644
--- a/osfmk/ppc/commpage/pthread.s
+++ b/osfmk/ppc/commpage/pthread.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/ppc/commpage/spinlocks.s b/osfmk/ppc/commpage/spinlocks.s
index cda60a78d..480f49050 100644
--- a/osfmk/ppc/commpage/spinlocks.s
+++ b/osfmk/ppc/commpage/spinlocks.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/appleapiopts.h>
diff --git a/osfmk/ppc/conf.c b/osfmk/ppc/conf.c
index db8796ddb..adeb60ea7 100644
--- a/osfmk/ppc/conf.c
+++ b/osfmk/ppc/conf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -71,10 +77,10 @@ extern	struct clock_ops	sysclk_ops, calend_ops;
 struct	clock	clock_list[] = {
 
 	/* SYSTEM_CLOCK */
-	{ &sysclk_ops, 0, 0 },
+	{ &sysclk_ops, NULL, NULL },
 
 	/* CALENDAR_CLOCK */
-	{ &calend_ops, 0, 0 },
+	{ &calend_ops, NULL, NULL },
 };
 int	clock_count = sizeof(clock_list) / sizeof(clock_list[0]);
 
diff --git a/osfmk/ppc/console_feed.c b/osfmk/ppc/console_feed.c
index 66b2e9217..8f029d49d 100644
--- a/osfmk/ppc/console_feed.c
+++ b/osfmk/ppc/console_feed.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/console_feed_entries.h b/osfmk/ppc/console_feed_entries.h
index 312734a44..729955043 100644
--- a/osfmk/ppc/console_feed_entries.h
+++ b/osfmk/ppc/console_feed_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/cpu.c b/osfmk/ppc/cpu.c
index 5326e5dcf..3a77bccb6 100644
--- a/osfmk/ppc/cpu.c
+++ b/osfmk/ppc/cpu.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -38,6 +44,7 @@
 
 #include <ppc/proc_reg.h>
 #include <ppc/misc_protos.h>
+#include <ppc/fpu_protos.h>
 #include <ppc/machine_routines.h>
 #include <ppc/cpu_internal.h>
 #include <ppc/exception.h>
@@ -71,9 +78,9 @@ struct SIGtimebase {
 	uint64_t	abstime;
 };
 
-perfCallback	   	perfCpuSigHook = 0;			/* Pointer to CHUD cpu signal hook routine */
+perfCallback	   	perfCpuSigHook;			/* Pointer to CHUD cpu signal hook routine */
 
-extern int 			debugger_sync;
+extern uint32_t			debugger_sync;
 
 /*
  * Forward definitions
@@ -187,7 +194,7 @@ cpu_machine_init(
 	proc_info->cpu_flags |= BootDone|SignalReady;
 	if (proc_info != mproc_info) {
 		if (proc_info->ppXFlags & SignalReadyWait) {
-			hw_atomic_and(&proc_info->ppXFlags, ~SignalReadyWait);
+			(void)hw_atomic_and(&proc_info->ppXFlags, ~SignalReadyWait);
 			thread_wakeup(&proc_info->cpu_flags);
 		}
 		simple_unlock(&SignalReadyLock);
@@ -204,9 +211,9 @@ struct per_proc_info *
 cpu_per_proc_alloc(
 		void)
 {
-	struct per_proc_info	*proc_info=0;
-	void			*interrupt_stack=0;
-	void			*debugger_stack=0;
+	struct per_proc_info	*proc_info = NULL;
+	void			*interrupt_stack = NULL;
+	void			*debugger_stack = NULL;
 
 	if ((proc_info = (struct per_proc_info*)kalloc(sizeof(struct per_proc_info))) == (struct per_proc_info*)0)
 		return (struct per_proc_info *)NULL;
@@ -223,7 +230,10 @@ cpu_per_proc_alloc(
 
 	bzero((void *)proc_info, sizeof(struct per_proc_info));
 
-	proc_info->pp2ndPage = (addr64_t)pmap_find_phys(kernel_pmap, (addr64_t)proc_info + 0x1000) << PAGE_SHIFT;	/* Set physical address of the second page */
+	/* Set physical address of the second page */
+	proc_info->pp2ndPage = (addr64_t)pmap_find_phys(kernel_pmap,
+				((addr64_t)(unsigned int)proc_info) + 0x1000)
+			       << PAGE_SHIFT;
 	proc_info->next_savearea = (uint64_t)save_get_init();
 	proc_info->pf = BootProcInfo.pf;
 	proc_info->istackptr = (vm_offset_t)interrupt_stack + INTSTACK_SIZE - FM_SIZE;
@@ -272,7 +282,7 @@ cpu_per_proc_register(
 	cpu = real_ncpus;
 	proc_info->cpu_number = cpu;
 	PerProcTable[cpu].ppe_vaddr = proc_info;
-	PerProcTable[cpu].ppe_paddr = (addr64_t)pmap_find_phys(kernel_pmap, (addr64_t)proc_info) << PAGE_SHIFT;
+	PerProcTable[cpu].ppe_paddr = (addr64_t)pmap_find_phys(kernel_pmap, (addr64_t)(unsigned int)proc_info) << PAGE_SHIFT;
 	eieio();
 	real_ncpus++;
 	mutex_unlock(&ppt_lock);
@@ -306,8 +316,8 @@ cpu_start(
 		proc_info->pending_ast = AST_NONE;
 		proc_info->istackptr = proc_info->intstack_top_ss;
 		proc_info->rtcPop = EndOfAllTime;
-		proc_info->FPU_owner = 0;
-		proc_info->VMX_owner = 0;
+		proc_info->FPU_owner = NULL;
+		proc_info->VMX_owner = NULL;
 		proc_info->pms.pmsStamp = 0;									/* Dummy transition time */
 		proc_info->pms.pmsPop = EndOfAllTime;							/* Set the pop way into the future */
 		proc_info->pms.pmsState = pmsParked;							/* Park the stepper */
@@ -359,7 +369,7 @@ cpu_start(
 		} else {
 			simple_lock(&SignalReadyLock);
 			if (!((*(volatile short *)&proc_info->cpu_flags) & SignalReady)) {
-				hw_atomic_or(&proc_info->ppXFlags, SignalReadyWait);
+				(void)hw_atomic_or(&proc_info->ppXFlags, SignalReadyWait);
 				thread_sleep_simple_lock((event_t)&proc_info->cpu_flags,
 				                          &SignalReadyLock, THREAD_UNINT);
 			}
@@ -418,12 +428,13 @@ cpu_sleep(
 	proc_info->running = FALSE;
 
 	fowner = proc_info->FPU_owner;					/* Cache this */
-	if(fowner) fpu_save(fowner);					/* If anyone owns FPU, save it */
-	proc_info->FPU_owner = 0;						/* Set no fpu owner now */
+	if(fowner) /* If anyone owns FPU, save it */
+		fpu_save(fowner);
+	proc_info->FPU_owner = NULL;						/* Set no fpu owner now */
 
 	fowner = proc_info->VMX_owner;					/* Cache this */
 	if(fowner) vec_save(fowner);					/* If anyone owns vectors, save it */
-	proc_info->VMX_owner = 0;						/* Set no vector owner now */
+	proc_info->VMX_owner = NULL;						/* Set no vector owner now */
 
 	if (proc_info->cpu_number == master_cpu)  {
 		proc_info->cpu_flags &= BootDone;
@@ -559,11 +570,9 @@ cpu_signal(
  *	the lock and signal the other guy.
  */
 void 
-cpu_signal_handler(
-	void)
+cpu_signal_handler(void)
 {
-
-	unsigned int holdStat, holdParm0, holdParm1, holdParm2, mtype;
+	unsigned int holdStat, holdParm0, holdParm1, holdParm2;
 	unsigned int *parmAddr;
 	struct per_proc_info	*proc_info;
 	int cpu;
@@ -655,7 +664,7 @@ cpu_signal_handler(
 
 					proc_info->hwCtr.numSIGPdebug++;	/* Count this one */
 					proc_info->debugger_is_slave++;		/* Bump up the count to show we're here */
-					hw_atomic_sub(&debugger_sync, 1);	/* Show we've received the 'rupt */
+					(void)hw_atomic_sub(&debugger_sync, 1);	/* Show we've received the 'rupt */
 					__asm__ volatile("tw 4,r3,r3");	/* Enter the debugger */
 					return;							/* All done now... */
 					
@@ -665,7 +674,7 @@ cpu_signal_handler(
 					
 				case SIGPcall:						/* Call function on CPU */
 					proc_info->hwCtr.numSIGPcall++;	/* Count this one */
-					xfunc = holdParm1;				/* Do this since I can't seem to figure C out */
+					xfunc = (broadcastFunc)holdParm1;				/* Do this since I can't seem to figure C out */
 					xfunc(holdParm2);				/* Call the passed function */
 					return;							/* Done... */
 					
@@ -1116,34 +1125,56 @@ cpu_threadtype(void)
  *	It is not passed to the other processor and must be known by the called function.
  *	The called function must do a thread_wakeup on the synch if it decrements the
  *	synch count to 0.
+ *
+ *	We start by initializing the synchronizer to the number of possible cpus.
+ *	The we signal each popssible processor.
+ *	If the signal fails, we count it.  We also skip our own.
+ *	When we are finished signaling, we adjust the syncronizer count down buy the number of failed signals.
+ *	Because the signaled processors are also decrementing the synchronizer count, the adjustment may result in a 0
+ *	If this happens, all other processors are finished with the function.
+ *	If so, we clear the wait and continue
+ *	Otherwise, we block waiting for the other processor(s) to finish.
+ *
+ *	Meanwhile, the other processors are decrementing the synchronizer when they are done
+ *	If it goes to zero, thread_wakeup is called to run the broadcaster
+ *
+ *	Note that because we account for the broadcaster in the synchronization count, we will not get any
+ *	premature wakeup calls.
+ *
+ *	Also note that when we do the adjustment of the synchronization count, it the result is 0, it means that
+ *	all of the other processors are finished.  Otherwise, we know that there is at least one more. 
+ *	When that thread decrements the synchronizer to zero, it will do a thread_wake.
+ *	
  */
 
-
-int32_t cpu_broadcast(uint32_t *synch, broadcastFunc func, uint32_t parm) {
-
-	int sigproc, cpu, ocpu;
-
-	cpu = cpu_number();									/* Who are we? */
-	sigproc = 0;										/* Clear called processor count */
-
-	if(real_ncpus > 1) {								/* Are we just a uni? */
+int32_t
+cpu_broadcast(uint32_t *synch, broadcastFunc func, uint32_t parm)
+{
+	int failsig;
+	unsigned int cpu, ocpu;
 	
+	cpu = cpu_number();						/* Who are we? */
+	failsig = 0;							/* Clear called processor count */
+	
+	if(real_ncpus > 1) {						/* Are we just a uni? */
+		
+		*synch = real_ncpus;					/* Set how many we are going to try */
 		assert_wait((event_t)synch, THREAD_UNINT);		/* If more than one processor, we may have to wait */
-
+		
 		for(ocpu = 0; ocpu < real_ncpus; ocpu++) {		/* Tell everyone to call */
-			if(ocpu == cpu) continue;					/* If we talk to ourselves, people will wonder... */
-			hw_atomic_add(synch, 1);					/* Tentatively bump synchronizer  */
-			sigproc++;									/* Tentatively bump signal sent count */
+			
+			if(ocpu == cpu)	continue;			/* If we talk to ourselves, people will wonder... */
+			
 			if(KERN_SUCCESS != cpu_signal(ocpu, SIGPcall, (uint32_t)func, parm)) {	/* Call the function on the other processor */
-				hw_atomic_sub(synch, 1);				/* Other guy isn't really there, ignore it  */
-				sigproc--;								/* and don't count it */
+				failsig++;				/* Count failed signals */
 			}
 		}
-
-		if(!sigproc) clear_wait(current_thread(), THREAD_AWAKENED);	/* Clear wait if we never signalled */
-		else thread_block(THREAD_CONTINUE_NULL);		/* Wait for everyone to get into step... */
+		
+		if (hw_atomic_sub(synch, failsig + 1) == 0)
+			clear_wait(current_thread(), THREAD_AWAKENED);	/* Clear wait if we never signalled or all of the others finished */
+		else
+			thread_block(THREAD_CONTINUE_NULL);		/* Wait for everyone to get into step... */
 	}
-
-	return sigproc;										/* Return the number of guys actually signalled */
-
+	
+	return (real_ncpus - failsig - 1);				/* Return the number of guys actually signalled... */
 }
diff --git a/osfmk/ppc/cpu_affinity.h b/osfmk/ppc/cpu_affinity.h
new file mode 100644
index 000000000..2e0ae7ce4
--- /dev/null
+++ b/osfmk/ppc/cpu_affinity.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#ifdef KERNEL_PRIVATE
+#ifndef _PPC_CPU_AFFINITY_H_
+#define _PPC_CPU_AFFINITY_H_
+
+/*
+ * Just one hardware affinity set - the whole machine.
+ * This allows us to give the pretense that PPC supports the affinity policy
+ * SPI. The kernel will accept affinity hints but effectively ignore them. 
+ * Hence Universal Apps can use platform-independent code.
+ */
+static inline int ml_get_max_affinity_sets(void)
+{
+	return 1;
+}
+
+/*
+ * Return the single processor set.
+ */
+static inline processor_set_t ml_affinity_to_pset(__unused int affinity_num)
+{
+	return processor_pset(master_processor);
+}
+
+#endif /* _I386_CPU_AFFINITY_H_ */
+#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/ppc/cpu_capabilities.h b/osfmk/ppc/cpu_capabilities.h
index ddaf69688..268666cd2 100644
--- a/osfmk/ppc/cpu_capabilities.h
+++ b/osfmk/ppc/cpu_capabilities.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	PRIVATE
 
@@ -105,6 +111,37 @@ static __inline__ int _NumCPUs( void ) { return (_cpu_capabilities & kNumCPUs) >
 /* The Objective-C runtime fixed address page to optimize message dispatch */
 #define _OBJC_PAGE_BASE_ADDRESS			(-20*4096)						// start at page -20, ie 0xFFFEC000
  
+/*
+ * Objective-C needs an "absolute" area all the way up to the top of the
+ * address space.
+ * For a ppc32 task, that area gets allocated at runtime from user space.
+ * For a ppc64 task, that area is not within the user-accessible address range,
+ * so we pre-allocate it at exec time (see vm_map_exec()) along with the
+ * comm page.
+ * 
+ * NOTE: that means we can't "nest" the 64-bit comm page...
+ */
+#define _COMM_PAGE32_OBJC_SIZE	0ULL
+#define _COMM_PAGE32_OBJC_BASE	0ULL
+#if 0
+#define _COMM_PAGE64_OBJC_SIZE	(4 * 4096)
+#define _COMM_PAGE64_OBJC_BASE	(_OBJC_PAGE_BASE_ADDRESS)
+#else
+/*
+ * PPC51: ppc64 is limited to 51-bit addresses.
+ * PPC64 has a 51-bit address space limit, so we can't just go and
+ * map the Obj-C area up there.  We would have to create a nested pmap
+ * and make a special mapping that redirects the large virtual addresses to
+ * that other address space with lower addresses that fit within the 51-bit
+ * limit.
+ * VM would then have to handle this redirection when we fault one
+ * of these pages in but it doesn't do that at this point, so no
+ * Obj-C area for ppc64 for now :-(
+ */
+#define _COMM_PAGE64_OBJC_SIZE	0ULL
+#define _COMM_PAGE64_OBJC_BASE	0ULL
+#endif
+
 /* data in the comm page */
  
 #define _COMM_PAGE_SIGNATURE			(_COMM_PAGE_BASE_ADDRESS+0x000)	// first few bytes are a signature
diff --git a/osfmk/ppc/cpu_data.h b/osfmk/ppc/cpu_data.h
index c5021b134..3a5fa9190 100644
--- a/osfmk/ppc/cpu_data.h
+++ b/osfmk/ppc/cpu_data.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/cpu_internal.h b/osfmk/ppc/cpu_internal.h
index 034f5167b..0c876a1e9 100644
--- a/osfmk/ppc/cpu_internal.h
+++ b/osfmk/ppc/cpu_internal.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -37,15 +43,6 @@ extern void						cpu_bootstrap(
 extern void						cpu_init(
 										void);
 
-extern	void					cpu_machine_init(
-										void);
-
-extern void						cpu_doshutdown(
-										void);
-
-extern void						cpu_signal_handler(
-										void);
-
 extern kern_return_t			cpu_signal(
 										int				target,
 										int				signal,
diff --git a/osfmk/ppc/cpu_number.h b/osfmk/ppc/cpu_number.h
index ab47c79e9..cd38aa6d6 100644
--- a/osfmk/ppc/cpu_number.h
+++ b/osfmk/ppc/cpu_number.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/cswtch.s b/osfmk/ppc/cswtch.s
index 96f051e16..54a17af4e 100644
--- a/osfmk/ppc/cswtch.s
+++ b/osfmk/ppc/cswtch.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/db_asm.s b/osfmk/ppc/db_asm.s
index 64e448a4b..626fa1822 100644
--- a/osfmk/ppc/db_asm.s
+++ b/osfmk/ppc/db_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/db_disasm.c b/osfmk/ppc/db_disasm.c
index 87c35fe6e..6410471ef 100644
--- a/osfmk/ppc/db_disasm.c
+++ b/osfmk/ppc/db_disasm.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -48,10 +54,7 @@ boolean_t	db_disasm_print_symaddr;
  * next instruction.
  */
 db_addr_t
-db_disasm(
-	db_addr_t	loc,
-	boolean_t	altfmt,
-	task_t		task)
+db_disasm(db_addr_t loc, __unused boolean_t altfmt, task_t task)
 {
 	int inst;
 	char *p;
@@ -77,8 +80,7 @@ db_disasm(
  * memory.
  */
 int
-db_inst_load(
-	unsigned long insw)
+db_inst_load(__unused unsigned long insw)
 {
 #if 1
 	db_printf("db_inst_load: coming soon in a debugger near you!\n");
@@ -123,8 +125,7 @@ db_inst_load(
  * memory.
  */
 int
-db_inst_store(
-	unsigned long insw)
+db_inst_store(__unused unsigned long insw)
 {
 #if 1
 	db_printf("db_inst_store: coming soon in a debugger near you!\n");
@@ -203,8 +204,7 @@ brdispl(
 }
 
 char *
-mbz(
-	bits n)
+mbz(bits n)
 {
 	return n ? "[reserved bits not zero]" : "";
 }
diff --git a/osfmk/ppc/db_interface.c b/osfmk/ppc/db_interface.c
index cfa40aa2c..f6f5bca5b 100644
--- a/osfmk/ppc/db_interface.c
+++ b/osfmk/ppc/db_interface.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -46,6 +52,7 @@
 #include <ppc/db_machdep.h>
 #include <ppc/mappings.h>
 #include <ppc/Firmware.h>
+#include <ppc/serial_io.h> /* for switch_to_serial_console */
 
 #include <mach/vm_param.h>
 #include <mach/machine/vm_types.h>
@@ -53,6 +60,7 @@
 #include <kern/thread.h>
 #include <kern/task.h>
 #include <kern/debug.h>
+#include <kern/machine.h> /* for halt_all_cpus() */
 #include <pexpert/pexpert.h>
 #include <IOKit/IOPlatformExpert.h>
 
@@ -89,7 +97,6 @@ struct int_regs {
 	struct ppc_interrupt_state *is;
 };
 
-extern char *	trap_type[];
 extern int	TRAP_TYPES;
 
 /*
@@ -212,7 +219,6 @@ kdb_trap(
 	                      == BKPT_INST))
 	    regs->save_srr0 += BKPT_SIZE;
 
-kdb_exit:
 	getPerProc()->db_saved_state = 0;
 	switch_to_old_console(previous_console_device);
 
@@ -243,9 +249,8 @@ kdbprinttrap(
 /*
  *
  */
-addr64_t db_vtophys(
-	pmap_t pmap,
-	vm_offset_t va)
+static addr64_t
+db_vtophys(pmap_t pmap, vm_offset_t va)
 {
 	ppnum_t pp;
 	addr64_t pa;
@@ -380,7 +385,6 @@ db_check_access(
 	task_t		task)
 {
 	register int	n;
-	unsigned int	kern_addr;
 
 	if (task == kernel_task || task == TASK_NULL) {
 	    if (kernel_task == TASK_NULL)  return(TRUE);
@@ -428,32 +432,25 @@ db_phys_eq(
 #define DB_USER_STACK_ADDR		(0xc0000000)
 #define DB_NAME_SEARCH_LIMIT		(DB_USER_STACK_ADDR-(PPC_PGBYTES*3))
 
-boolean_t	db_phys_cmp(
-				vm_offset_t a1, 
-				vm_offset_t a2, 
-				vm_size_t s1) {
-
+boolean_t
+db_phys_cmp(__unused vm_offset_t a1, __unused vm_offset_t a2,
+	    __unused vm_size_t s1)
+{
 	db_printf("db_phys_cmp: not implemented\n");
 	return 0;
 }
 
 
 int
-db_search_null(
-	task_t		task,
-	unsigned	*svaddr,
-	unsigned	evaddr,
-	unsigned	*skaddr,
-	int		flag)
+db_search_null(__unused task_t task, __unused unsigned *svaddr,
+	       __unused unsigned evaddr, __unused unsigned *skaddr,
+	       __unused int flag)
 {
-	register unsigned vaddr;
-	register unsigned *kaddr;
-
 	db_printf("db_search_null: not implemented\n");
-
 	return(-1);
 }
 
+struct proc;
 unsigned char *getProcName(struct proc *proc);
 
 void
@@ -461,8 +458,6 @@ db_task_name(
 	task_t		task)
 {
 	register unsigned char *p;
-	register int n;
-	unsigned int vaddr, kaddr;
 	unsigned char tname[33];
 	int i;
 
@@ -482,20 +477,22 @@ db_task_name(
 	else db_printf("no name");
 }
 
+extern int kdb_flag;  
 void
-db_machdep_init(void) {
+db_machdep_init(void)
+{
 #define KDB_READY       0x1
-	extern int     kdb_flag;  
-
 	kdb_flag |= KDB_READY;
 }
 
 
 #ifdef	__STDC__
-#define KDB_SAVE(type, name) extern type name; type name##_save = name
+//#define KDB_SAVE(type, name) extern type name; type name##_save = name
+#define KDB_SAVE(type, name) type name##_save = name
 #define KDB_RESTORE(name) name = name##_save
 #else	/* __STDC__ */
-#define KDB_SAVE(type, name) extern type name; type name/**/_save = name
+#define KDB_SAVE(type, name) type name/**/_save = name
+//#define KDB_SAVE(type, name) extern type name; type name/**/_save = name
 #define KDB_RESTORE(name) name = name/**/_save
 #endif	/* __STDC__ */
 
@@ -533,6 +530,16 @@ db_machdep_init(void) {
 	KDB_RESTORE(db_next); \
 	KDB_RESTORE(ddb_regs); 
 
+extern boolean_t db_sstep_print;
+extern int db_loop_count;
+extern int db_call_depth;
+extern int db_inst_count;
+extern int db_last_inst_count;
+extern int db_load_count;
+extern int db_store_count;
+extern boolean_t db_cmd_loop_done;
+extern void unlock_debugger(void);
+extern void lock_debugger(void);
 /*
  * switch to another cpu
  */
@@ -541,7 +548,7 @@ kdb_on(
 	int		cpu)
 {
 	KDB_SAVE_CTXT();
-	if (cpu < 0 || cpu >= real_ncpus || !PerProcTable[cpu].ppe_vaddr->debugger_active)
+	if (cpu < 0 || cpu >= (int)real_ncpus || !PerProcTable[cpu].ppe_vaddr->debugger_active)
 		return;
 	db_set_breakpoints();
 	db_set_watchpoints();
@@ -553,7 +560,7 @@ kdb_on(
 	KDB_RESTORE_CTXT();
 	if (debugger_cpu == -1)  {/* someone continued */
 		debugger_cpu = cpu_number();
-		db_continue_cmd(0, 0, 0, "");
+		db_continue_cmd(0, 0, 0, NULL);
 	}
 }
 
@@ -561,13 +568,9 @@ kdb_on(
  * system reboot
  */
 
-extern int (*PE_halt_restart)(unsigned int type);
-
-void db_reboot(
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		*modif)
+void
+db_reboot(__unused db_expr_t addr, __unused boolean_t have_addr,
+	  __unused db_expr_t count, char *modif)
 {
 	boolean_t	reboot = TRUE;
 	char		*cp, c;
@@ -581,7 +584,8 @@ void db_reboot(
 	}
 	if(!reboot) halt_all_cpus(FALSE);	/* If no reboot, try to be clean about it */
 
-	if (PE_halt_restart) return (*PE_halt_restart)(kPERestartCPU);
+	if (PE_halt_restart)
+		(*PE_halt_restart)(kPERestartCPU);
 	db_printf("Sorry, system can't reboot automatically yet...  You need to do it by hand...\n");
 
 }
diff --git a/osfmk/ppc/db_low_trace.c b/osfmk/ppc/db_low_trace.c
index 72454fed5..e081b2643 100644
--- a/osfmk/ppc/db_low_trace.c
+++ b/osfmk/ppc/db_low_trace.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -77,8 +83,9 @@ db_addr_t	db_low_trace_prev = 0;
  *		If entaddr is omitted, it starts with the most current
  *		If entaddr = 0, it starts with the most current and does the whole table
  */
-void db_low_trace(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_low_trace(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif)
+{
 	int		c, i;
 	unsigned int tempx, cnt;
 	unsigned int xTraceCurr, xTraceStart, xTraceEnd, cxltr;
@@ -147,7 +154,6 @@ void db_low_trace(db_expr_t addr, int have_addr, db_expr_t count, char * modif)
 	
 	}
 	db_next = (db_expr_t)(xxltr);
-	return;
 }
 
 
@@ -157,8 +163,10 @@ void db_low_trace(db_expr_t addr, int have_addr, db_expr_t count, char * modif)
  *		
  *		dl [entaddr]
  */
-void db_display_long(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_long(db_expr_t addr, __unused boolean_t have_addr,
+		db_expr_t count, char * modif)
+{
 	int				i;
 
 	for(i=0; i<8; i++) {									/* Print 256 bytes */
@@ -198,7 +206,10 @@ unsigned char xtran[256] = {
  *		
  *		dc [entaddr]
  */
-void db_display_char(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
+void
+db_display_char(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char * modif)
+{
 
 	int				i, j, k;
 	unsigned char xlt[256], *xaddr;
@@ -234,8 +245,10 @@ void db_display_char(db_expr_t addr, int have_addr, db_expr_t count, char * modi
  *		
  *		dr [entaddr]
  */
-void db_display_real(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_real(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	int				i;
 	unsigned int xbuf[8];
 
@@ -257,8 +270,10 @@ unsigned int	dvspace = 0;
  *		
  *		dm vaddr [space] (defaults to last entered) 
  */
-void db_display_mappings(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_mappings(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		    char *modif)
+{
 	db_expr_t	xspace;
 	pmap_t			pmap;
 	addr64_t		lnextva;
@@ -296,9 +311,7 @@ void db_display_mappings(db_expr_t addr, int have_addr, db_expr_t count, char *
 
 	db_dumpmapping(mp);										/* Dump it all out */
 
-	return;													/* Tell them we did it */
-
-
+	/* Tell them we did it */
 }
 
 /*
@@ -307,8 +320,10 @@ void db_display_mappings(db_expr_t addr, int have_addr, db_expr_t count, char *
  *		
  *		dh vaddr [space] (defaults to last entered) 
  */
-void db_display_hash(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_hash(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	db_expr_t		xspace;
 	unsigned int	seg, vsid, ptegindex, htsize;
 	pmap_t			pmap;
@@ -345,9 +360,7 @@ void db_display_hash(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 	ptegindex = llva & (htsize - 1);						/* Get the index to the pteg and pca */
 	db_dumppca(ptegindex);									/* dump the info */
 	
-	return;													/* Tell them we did it */
-
-
+	/* Tell them we did it */
 }
 
 /*
@@ -355,8 +368,10 @@ void db_display_hash(db_expr_t addr, int have_addr, db_expr_t count, char * modi
  *
   *		dp
  */
-void db_display_pmap(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_pmap(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	pmap_t			pmap;
 	int i;
 	unsigned int v0, v1, st0, st1;
@@ -397,7 +412,6 @@ void db_display_pmap(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 		pmap = (pmap_t)pmap->pmap_link.next;				/* Skip to the next */
 		if(pmap == kernel_pmap) break;						/* We've wrapped, we're done */
 	}
-	return;
 }
 
 
@@ -407,8 +421,10 @@ void db_display_pmap(db_expr_t addr, int have_addr, db_expr_t count, char * modi
  *		
  *		cp pmap
  */
-void db_check_pmaps(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_check_pmaps(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+	       char *modif)
+{
 	int				i;
 	unsigned int ret;
 	uint64_t dumpa[32];
@@ -431,9 +447,6 @@ void db_check_pmaps(db_expr_t addr, int have_addr, db_expr_t count, char * modif
 		pmap = (pmap_t)pmap->pmap_link.next;				/* Skip to the next */
 		if(pmap == kernel_pmap) break;						/* We've wrapped, we're done */
 	}
-	
-	return;
-
 }
 
 
@@ -445,11 +458,11 @@ void db_check_pmaps(db_expr_t addr, int have_addr, db_expr_t count, char * modif
 
 void db_piokjunk(void);
 
-void db_display_iokit(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_iokit(__unused db_expr_t addr, __unused boolean_t have_addr,
+		 __unused db_expr_t count, __unused char *modif)
+{
 	db_piokjunk();
-
-	return;
 }
 
 /*
@@ -476,8 +489,6 @@ void db_dumpmapping(struct mapping *mp) { 					/* Dump out a mapping */
 	for(i = 1; i < (mp->mpFlags & mpLists); i++) {			/* Dump out secondary physical skip lists */
 		db_printf("             mpList%02d: %016llX\n", i, mp->mpList[i - 1]);     
 	}
-	            
-	return;
 }
 
 /*
@@ -544,8 +555,6 @@ void db_dumppca(unsigned int ptegindex) {
 			db_printf("va = %016llX\n", llva);
 		}
 	}
-
-	return;
 }
 
 
@@ -556,7 +565,10 @@ void db_dumppca(unsigned int ptegindex) {
  *		dv [entaddr] [space]
  *		address must be on 32-byte boundary.  It will be rounded down if not
  */
-void db_display_virtual(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
+void
+db_display_virtual(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		   char *modif)
+{
 
 	int			i, size, lines, rlines;
 	unsigned int 	xbuf[8];
@@ -646,26 +658,27 @@ void db_display_virtual(db_expr_t addr, int have_addr, db_expr_t count, char * m
 
 #define chainmax 32
 
-void db_display_save(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_save(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	int				i, j, totsaves, tottasks, taskact, chainsize, vmid, didvmhead;
-	processor_set_t	pset = &default_pset;
 	task_t			task;
 	thread_act_t	act;
-	savearea		*save;
+	struct savearea		*save;
 	vmmCntrlTable	*CTable;
 	
 	tottasks = 0;
 	totsaves = 0;
 	
-	for(task = (task_t)pset->tasks.next; task != (task_t)&pset->tasks.next; task = (task_t)task->pset_tasks.next) {	/* Go through the tasks */
+	for(task = (task_t)tasks.next; task != (task_t)&tasks.next; task = (task_t)task->tasks.next) {	/* Go through the tasks */
 		taskact = 0;								/* Reset activation count */
 		db_printf("\nTask %4d @%08X:\n", tottasks, task);	/* Show where we're at */
 		for(act = (thread_act_t)task->threads.next; act != (thread_act_t)&task->threads; act = (thread_act_t)act->task_threads.next) {	/* Go through activations */
 			db_printf("   Act %4d @%08X - p: %08X  current context: %08X\n",
 					  taskact, act, act->machine.pcb, act->machine.curctx);					
 					
-			save = (savearea *)act->machine.pcb; 		/* Set the start of the normal chain */
+			save = (struct savearea *)act->machine.pcb; 		/* Set the start of the normal chain */
 			chainsize = 0;
 			
 			db_printf("      General context - fp: %08X  fl: %08X  fc: %d  vp: %08X  vl: %08X  vp: %d\n",
@@ -675,31 +688,31 @@ void db_display_save(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 			while(save) {							/* Do them all */
 				totsaves++;							/* Count savearea */
 				db_printf("         Norm %08X: %016llX %016llX - tot = %d\n", save, save->save_srr0, save->save_srr1, totsaves);
-				save = (savearea *)save->save_hdr.save_prev;	/* Next one */
+				save = (struct savearea *)save->save_hdr.save_prev;	/* Next one */
 				if(chainsize++ > chainmax) {		/* See if we might be in a loop */
 					db_printf("         Chain terminated by count (%d) before %08X\n", chainmax, save);
 					break;
 				}
 			}
 			
-			save = (savearea *)act->machine.facctx.FPUsave; 	/* Set the start of the floating point chain */
+			save = (struct savearea *)act->machine.facctx.FPUsave; 	/* Set the start of the floating point chain */
 			chainsize = 0;
 			while(save) {							/* Do them all */
 				totsaves++;							/* Count savearea */
 				db_printf("         FPU  %08X: %08X - tot = %d\n", save, save->save_hdr.save_level, totsaves);
-				save = (savearea *)save->save_hdr.save_prev;	/* Next one */
+				save = (struct savearea *)save->save_hdr.save_prev;	/* Next one */
 				if(chainsize++ > chainmax) {		/* See if we might be in a loop */
 					db_printf("         Chain terminated by count (%d) before %08X\n", chainmax, save);
 					break;
 				}
 			}
 			
-			save = (savearea *)act->machine.facctx.VMXsave; 	/* Set the start of the floating point chain */
+			save = (struct savearea *)act->machine.facctx.VMXsave; 	/* Set the start of the floating point chain */
 			chainsize = 0;
 			while(save) {							/* Do them all */
 				totsaves++;							/* Count savearea */
 				db_printf("         Vec  %08X: %08X - tot = %d\n", save, save->save_hdr.save_level, totsaves);
-				save = (savearea *)save->save_hdr.save_prev;	/* Next one */
+				save = (struct savearea *)save->save_hdr.save_prev;	/* Next one */
 				if(chainsize++ > chainmax) {		/* See if we might be in a loop */
 					db_printf("         Chain terminated by count (%d) before %08X\n", chainmax, save);
 					break;
@@ -719,24 +732,24 @@ void db_display_save(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 						CTable->vmmc[vmid].vmmFacCtx.VMXsave, CTable->vmmc[vmid].vmmFacCtx.VMXlevel, CTable->vmmc[vmid].vmmFacCtx.VMXcpu
 					);
 					
-					save = (savearea *)CTable->vmmc[vmid].vmmFacCtx.FPUsave; 	/* Set the start of the floating point chain */
+					save = (struct savearea *)CTable->vmmc[vmid].vmmFacCtx.FPUsave; 	/* Set the start of the floating point chain */
 					chainsize = 0;
 					while(save) {						/* Do them all */
 						totsaves++;						/* Count savearea */
 						db_printf("         FPU  %08X: %08X - tot = %d\n", save, save->save_hdr.save_level, totsaves);
-						save = (savearea *)save->save_hdr.save_prev;	/* Next one */
+						save = (struct savearea *)save->save_hdr.save_prev;	/* Next one */
 						if(chainsize++ > chainmax) {	/* See if we might be in a loop */
 							db_printf("         Chain terminated by count (%d) before %08X\n", chainmax, save);
 							break;
 						}
 					}
 					
-					save = (savearea *)CTable->vmmc[vmid].vmmFacCtx.VMXsave; 	/* Set the start of the floating point chain */
+					save = (struct savearea *)CTable->vmmc[vmid].vmmFacCtx.VMXsave; 	/* Set the start of the floating point chain */
 					chainsize = 0;
 					while(save) {						/* Do them all */
 						totsaves++;						/* Count savearea */
 						db_printf("         Vec  %08X: %08X - tot = %d\n", save, save->save_hdr.save_level, totsaves);
-						save = (savearea *)save->save_hdr.save_prev;	/* Next one */
+						save = (struct savearea *)save->save_hdr.save_prev;	/* Next one */
 						if(chainsize++ > chainmax) {	/* See if we might be in a loop */
 							db_printf("         Chain terminated by count (%d) before %08X\n", chainmax, save);
 							break;
@@ -750,7 +763,6 @@ void db_display_save(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 	}
 	
 	db_printf("Total saveareas accounted for: %d\n", totsaves);
-	return;
 }
 
 /*
@@ -764,8 +776,10 @@ extern unsigned int dbfloats[33][2];
 extern unsigned int dbvecs[33][4];
 extern unsigned int dbspecrs[336];
 
-void db_display_xregs(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_xregs(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		 char *modif)
+{
 	int				i, j, pents;
 
 	stSpecrs(dbspecrs);										/* Save special registers */
@@ -834,9 +848,7 @@ void db_display_xregs(db_expr_t addr, int have_addr, db_expr_t count, char * mod
 	}
 	db_printf("VCR: %08X %08X %08X %08X\n", dbvecs[32][0], dbvecs[32][1], dbvecs[32][2], dbvecs[32][3]);	/* Print VSCR */
 
-	return;													/* Tell them we did it */
-
-
+	/* Tell them we did it */
 }
 
 /*
@@ -844,8 +856,10 @@ void db_display_xregs(db_expr_t addr, int have_addr, db_expr_t count, char * mod
  *
   *		cm
  */
-void db_check_mappings(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_check_mappings(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		  char *modif)
+{
 	addr64_t  pteg, pca, llva, lnextva;	
 	unsigned int xpteg[32], xpca[8], space, hash, pva, seg, api, va, free, free2, xauto, PTEGcnt, wimgkk, wimgxx, slotoff;
 	int i, j, fnderr, slot, slot2, k, s4bit;
@@ -1027,8 +1041,6 @@ dcmout:
 
 
 	}
-
-	return;
 }
 
 /*
@@ -1036,8 +1048,10 @@ dcmout:
  *
   *		dp
  */
-void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_display_kmod(db_expr_t addr, boolean_t have_addr, db_expr_t count,
+		char *modif)
+{
 	kmod_info_t	*kmd;
 	unsigned int strt, end;
 	
@@ -1052,8 +1066,6 @@ void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modi
 			kmd->name, kmd->version);
 		kmd = kmd->next;				/* Step to it */
 	}
-
-	return;
 }
 
 /*
@@ -1063,8 +1075,9 @@ void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modi
  */
 unsigned char xxgpo[36] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
 
-void db_gsnoop(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
-
+void
+db_gsnoop(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif)
+{
 	int i, j;
 	unsigned char *gp, gpn[36];
 #define ngpr 34
@@ -1085,8 +1098,6 @@ void db_gsnoop(db_expr_t addr, int have_addr, db_expr_t count, char * modif) {
 	db_printf("\n");
 	
 	for(i = 0; i < ngpr; i++) xxgpo[i] = gpn[i];	/* Save 'em */
-
-	return;
 }
 
 
diff --git a/osfmk/ppc/db_low_trace.h b/osfmk/ppc/db_low_trace.h
index f88bd330a..efc3faedb 100644
--- a/osfmk/ppc/db_low_trace.h
+++ b/osfmk/ppc/db_low_trace.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -32,54 +38,25 @@
 #include <machine/db_machdep.h>
 #include <kern/task.h>
 
-/* Prototypes for functions exported by this module.
+/*
+ * Prototypes for functions exported by this module.
  */
 
-void db_list_pmap(
-	db_expr_t	addr,
-	int			have_addr,
-	db_expr_t	count,
-	char 		*modif
-);
-
-void db_low_trace(
-	db_expr_t	addr,
-	int			have_addr,
-	db_expr_t	count,
-	char 		*modif
-);
-
-void db_display_long(
-	db_expr_t	addr,
-	int			have_addr,
-	db_expr_t	count,
-	char 		*modif
-);
-
-void db_display_char(
-	db_expr_t	addr,
-	int			have_addr,
-	db_expr_t	count,
-	char 		*modif
-);
-
-void db_display_real(
-	db_expr_t	addr,
-	int			have_addr,
-	db_expr_t	count,
-	char 		*modif
-);
-
-void db_display_virtual(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_mappings(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_hash(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_pmap(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_iokit(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_save(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_xregs(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_display_kmod(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_gsnoop(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_check_mappings(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
-void db_check_pmaps(db_expr_t addr, int have_addr, db_expr_t count, char * modif);
+void db_list_pmap(db_expr_t, boolean_t, db_expr_t, char *);
+void db_low_trace(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_long(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_char(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_real(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_virtual(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_mappings(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_hash(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_pmap(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_iokit(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_save(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_xregs(db_expr_t, boolean_t, db_expr_t, char *);
+void db_display_kmod(db_expr_t, boolean_t, db_expr_t, char *);
+void db_gsnoop(db_expr_t, boolean_t, db_expr_t count, char *);
+void db_check_mappings(db_expr_t, boolean_t, db_expr_t, char *);
+void db_check_pmaps(db_expr_t, boolean_t, db_expr_t, char *);
 
 #endif	/* !_DDB_DB_LTR_H_ */
diff --git a/osfmk/ppc/db_machdep.h b/osfmk/ppc/db_machdep.h
index 6037cc731..20a2f1169 100644
--- a/osfmk/ppc/db_machdep.h
+++ b/osfmk/ppc/db_machdep.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -147,12 +153,6 @@ extern void		db_reboot(
 				boolean_t	have_addr,
 				db_expr_t	count,
 				char		*modif);
-extern void		db_low_trace(
-				db_expr_t	addr,
-				int		have_addr,
-				db_expr_t	count,
-				char 		*modif);
-
 
 /* macros for printing OS server dependent task name */
 
diff --git a/osfmk/ppc/db_trace.c b/osfmk/ppc/db_trace.c
index 531f9b6bb..601378162 100644
--- a/osfmk/ppc/db_trace.c
+++ b/osfmk/ppc/db_trace.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,17 +32,20 @@
 #include <string.h>
 
 #include <mach/boolean.h>
+#include <mach/machine.h>
+
 #include <vm/vm_map.h>
+
 #include <kern/thread.h>
 #include <kern/processor.h>
 #include <kern/task.h>
 
 #include <ppc/cpu_internal.h>
 #include <ppc/exception.h>
+
 #include <machine/asm.h>
 #include <machine/db_machdep.h>
 #include <machine/setjmp.h>
-#include <mach/machine.h>
 
 #include <ddb/db_access.h>
 #include <ddb/db_sym.h>
@@ -53,7 +62,6 @@ extern vm_offset_t vm_min_inks_addr;	/* set by db_clone_symtabXXX */
 
 #define DB_NUMARGS_MAX	5
 
-
 #define	INFIXEDSTACK(va)	0							\
 
 #define INKERNELSTACK(va, th) 1
@@ -79,75 +87,424 @@ db_addr_t	db_return_to_iret_symbol_value = 0;
 db_addr_t	db_syscall_symbol_value = 0;
 boolean_t	db_trace_symbols_found = FALSE;
 
-extern int	db_ppc_reg_value(
+static int db_ppc_reg_value(
 			struct db_variable	* vp,
 			db_expr_t		* val,
 			int			flag,
 			db_var_aux_param_t	ap);
-extern void	db_find_trace_symbols(void);
-extern int	db_numargs(
+static void db_find_trace_symbols(void);
+static int db_numargs(
 			struct db_ppc_frame	*fp,
 			task_t			task);
-extern boolean_t db_find_arg(
+static boolean_t db_find_arg(
 			struct db_ppc_frame	*frame,
 			db_addr_t		calleepc,
 			task_t			task,
 			int			narg,
 			db_addr_t		*arg);
-extern void	db_nextframe(
+static void db_nextframe(
 			struct db_ppc_frame	**lfp,
 			struct db_ppc_frame	**fp,
 			db_addr_t		*ip,
 			int			frame_type,
 			thread_act_t		thr_act,
 			db_addr_t		linkpc);
-extern int	_setjmp(
-			jmp_buf_t		* jb);
 
 /*
  * Machine register set.
  */
 struct db_variable db_regs[] = {
 	/* XXX "pc" is an alias to "srr0"... */
-  { "pc",	&ddb_regs.save_srr0,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "srr0",	&ddb_regs.save_srr0,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "srr1",	&ddb_regs.save_srr1,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r0",	&ddb_regs.save_r0,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r1",	&ddb_regs.save_r1,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r2",	&ddb_regs.save_r2,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r3",	&ddb_regs.save_r3,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r4",	&ddb_regs.save_r4,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r5",	&ddb_regs.save_r5,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r6",	&ddb_regs.save_r6,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r7",	&ddb_regs.save_r7,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r8",	&ddb_regs.save_r8,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r9",	&ddb_regs.save_r9,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r10",	&ddb_regs.save_r10,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r11",	&ddb_regs.save_r11,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r12",	&ddb_regs.save_r12,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r13",	&ddb_regs.save_r13,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r14",	&ddb_regs.save_r14,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r15",	&ddb_regs.save_r15,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r16",	&ddb_regs.save_r16,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r17",	&ddb_regs.save_r17,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r18",	&ddb_regs.save_r18,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r19",	&ddb_regs.save_r19,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r20",	&ddb_regs.save_r20,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r21",	&ddb_regs.save_r21,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r22",	&ddb_regs.save_r22,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r23",	&ddb_regs.save_r23,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r24",	&ddb_regs.save_r24,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r25",	&ddb_regs.save_r25,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r26",	&ddb_regs.save_r26,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r27",	&ddb_regs.save_r27,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r28",	&ddb_regs.save_r28,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r29",	&ddb_regs.save_r29,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r30",	&ddb_regs.save_r30,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "r31",	&ddb_regs.save_r31,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "cr",	&ddb_regs.save_cr,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "xer",	&ddb_regs.save_xer,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "lr",	&ddb_regs.save_lr,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
-  { "ctr",	&ddb_regs.save_ctr,	db_ppc_reg_value, 0, 0, 0, 0, TRUE },
+	{
+		.name = "pc",
+		.valuep = &ddb_regs.save_srr0,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "srr0",
+		.valuep = &ddb_regs.save_srr0,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "srr1",
+		.valuep = &ddb_regs.save_srr1,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r0",
+		.valuep = &ddb_regs.save_r0,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r1",
+		.valuep = &ddb_regs.save_r1,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r2",
+		.valuep = &ddb_regs.save_r2,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r3",
+		.valuep = &ddb_regs.save_r3,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r4",
+		.valuep = &ddb_regs.save_r4,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r5",
+		.valuep = &ddb_regs.save_r5,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r6",
+		.valuep = &ddb_regs.save_r6,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r7",
+		.valuep = &ddb_regs.save_r7,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r8",
+		.valuep = &ddb_regs.save_r8,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r9",
+		.valuep = &ddb_regs.save_r9,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r10",
+		.valuep = &ddb_regs.save_r10,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r11",
+		.valuep = &ddb_regs.save_r11,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r12",
+		.valuep = &ddb_regs.save_r12,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r13",
+		.valuep = &ddb_regs.save_r13,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r14",
+		.valuep = &ddb_regs.save_r14,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r15",
+		.valuep = &ddb_regs.save_r15,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r16",
+		.valuep = &ddb_regs.save_r16,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r17",
+		.valuep = &ddb_regs.save_r17,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r18",
+		.valuep = &ddb_regs.save_r18,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r19",
+		.valuep = &ddb_regs.save_r19,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r20",
+		.valuep = &ddb_regs.save_r20,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r21",
+		.valuep = &ddb_regs.save_r21,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r22",
+		.valuep = &ddb_regs.save_r22,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r23",
+		.valuep = &ddb_regs.save_r23,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r24",
+		.valuep = &ddb_regs.save_r24,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r25",
+		.valuep = &ddb_regs.save_r25,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r26",
+		.valuep = &ddb_regs.save_r26,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r27",
+		.valuep = &ddb_regs.save_r27,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r28",
+		.valuep = &ddb_regs.save_r28,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r29",
+		.valuep = &ddb_regs.save_r29,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r30",
+		.valuep = &ddb_regs.save_r30,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "r31",
+		.valuep = &ddb_regs.save_r31,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "cr",
+		.valuep = (db_expr_t *)&ddb_regs.save_cr,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "xer",
+		.valuep = &ddb_regs.save_xer,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "lr",
+		.valuep = &ddb_regs.save_lr,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
+	{
+		.name = "ctr",
+		.valuep = &ddb_regs.save_ctr,
+		.fcn = db_ppc_reg_value,
+		.min_level = 0,
+		.max_level = 0,
+		.low = 0,
+		.high = 0,
+		.hidden_level = TRUE,
+	},
 };
 struct db_variable *db_eregs = db_regs + sizeof(db_regs)/sizeof(db_regs[0]);
 
@@ -161,20 +518,19 @@ db_ppc_reg_value(
 	db_expr_t *dp = 0;
 	db_expr_t null_reg = 0;
 	uint32_t *dp32;
-	
-	register thread_act_t	thr_act = ap->thr_act;
-	int			cpu;
+	thread_act_t thr_act = ap->thr_act;
+	unsigned int cpu;
 
 	if (db_option(ap->modif, 'u')) {
-	    if (thr_act == THR_ACT_NULL) {
-		if ((thr_act = current_thread()) == THR_ACT_NULL)
-		    db_error("no user registers\n");
-	    }
-	    if (thr_act == current_thread()) {
+		if (thr_act == THR_ACT_NULL) {
+			if ((thr_act = current_thread()) == THR_ACT_NULL)
+				db_error("no user registers\n");
+		}
+		if (thr_act == current_thread()) {
 			if (IS_USER_TRAP((&ddb_regs))) dp = vp->valuep;
 			else if (INFIXEDSTACK(ddb_regs.save_r1))
 				db_error("cannot get/set user registers in nested interrupt\n");
-	    }
+		}
 	} 
 	else {
 		if (thr_act == THR_ACT_NULL || thr_act == current_thread()) {
@@ -182,64 +538,64 @@ db_ppc_reg_value(
 		} 
 		else {
 			if (thr_act->kernel_stack) {
-				
-				int cpu;
-
 				for (cpu = 0; cpu < real_ncpus; cpu++) {
 					if (cpu_to_processor(cpu)->state == PROCESSOR_RUNNING &&
-						cpu_to_processor(cpu)->active_thread == thr_act &&
-					    PerProcTable[cpu].ppe_vaddr->db_saved_state) {
-						
+							cpu_to_processor(cpu)->active_thread == thr_act &&
+							PerProcTable[cpu].ppe_vaddr->db_saved_state) {
+
 						dp = (db_expr_t)(((uint32_t)(PerProcTable[cpu].ppe_vaddr->db_saved_state)) +
-								  (((uint32_t) vp->valuep) -
-								   (uint32_t) &ddb_regs));
+								(((uint32_t) vp->valuep) -
+								 (uint32_t) &ddb_regs));
 						break;
 					}
 				}
 
-				if (dp == 0) dp = &null_reg;
+				if (dp == 0)
+					dp = &null_reg;
 			} 
 			else {
 				/* only PC is valid */
-				if (vp->valuep == (int *) &ddb_regs.save_srr0) {
-					dp = (int *)(&thr_act->continuation);
-				} 
-				else {
+				if (vp->valuep == &ddb_regs.save_srr0)
+					dp = (db_expr_t *)&thr_act->continuation;
+				else
 					dp = &null_reg;
-				}
 			}
-	    }
+		}
 	}
 	if (dp == 0) {
-
-	    if (!db_option(ap->modif, 'u')) {
+		if (!db_option(ap->modif, 'u')) {
 			for (cpu = 0; cpu < real_ncpus; cpu++) {
-			    if (cpu_to_processor(cpu)->state == PROCESSOR_RUNNING &&
-			    	cpu_to_processor(cpu)->active_thread == thr_act &&
-				    PerProcTable[cpu].ppe_vaddr->db_saved_state) {
-			    	    dp = (int *) (((int)(PerProcTable[cpu].ppe_vaddr->db_saved_state)) +
-						  (((int) vp->valuep) - (int) &ddb_regs));
+				if (cpu_to_processor(cpu)->state == PROCESSOR_RUNNING &&
+						cpu_to_processor(cpu)->active_thread == thr_act &&
+						PerProcTable[cpu].ppe_vaddr->db_saved_state) {
+					dp = (int *) (((int)(PerProcTable[cpu].ppe_vaddr->db_saved_state)) +
+							(((int) vp->valuep) - (int) &ddb_regs));
 					break;
 				}
 			}
-	    }
-	    if (dp == 0) {
-			if (!thr_act || thr_act->machine.pcb == 0) db_error("no pcb\n");
+		}
+		if (dp == 0) {
+			if (!thr_act || thr_act->machine.pcb == 0)
+				db_error("no pcb\n");
 			dp = (int *)((int)thr_act->machine.pcb + ((int)vp->valuep - (int)&ddb_regs));
-	    }
+		}
 	}
 
-	if(vp->valuep == (int *) &ddb_regs.save_cr) {	/* Is this the CR we are doing? */
+	if(vp->valuep == (db_expr_t *)&ddb_regs.save_cr) {	/* Is this the CR we are doing? */
 		dp32 = (uint32_t *)dp;						/* Make this easier */
-		if (flag == DB_VAR_SET) *dp32 = *valuep;
-		else *valuep = *dp32;
+		if (flag == DB_VAR_SET)
+			*dp32 = *valuep;
+		else
+			*valuep = *dp32;
 	}
 	else {											/* Normal 64-bit registers */
-		if (flag == DB_VAR_SET) *dp = *valuep;
-		else *valuep = *(unsigned long long *)dp;
+		if (flag == DB_VAR_SET)
+			*dp = *valuep;
+		else
+			*valuep = *(unsigned long long *)dp;
 	}
-	
-	return(0);
+
+	return 0;
 }
 
 
@@ -251,29 +607,29 @@ db_find_trace_symbols(void)
 
 	found_some = FALSE;
 	if (db_value_of_name(CC_SYM_PREFIX "thandler", &value)) {
-	    db_user_trap_symbol_value = (db_addr_t) value;
-	    found_some = TRUE;
+		db_user_trap_symbol_value = (db_addr_t) value;
+		found_some = TRUE;
 	}
 	if (db_value_of_name(CC_SYM_PREFIX "thandler", &value)) {
-	    db_kernel_trap_symbol_value = (db_addr_t) value;
-	    found_some = TRUE;
+		db_kernel_trap_symbol_value = (db_addr_t) value;
+		found_some = TRUE;
 	}
 	if (db_value_of_name(CC_SYM_PREFIX "ihandler", &value)) {
-	    db_interrupt_symbol_value = (db_addr_t) value;
-	    found_some = TRUE;
+		db_interrupt_symbol_value = (db_addr_t) value;
+		found_some = TRUE;
 	}
 #if 0
 	if (db_value_of_name(CC_SYM_PREFIX "return_to_iret", &value)) {
-	    db_return_to_iret_symbol_value = (db_addr_t) value;
-	    found_some = TRUE;
+		db_return_to_iret_symbol_value = (db_addr_t) value;
+		found_some = TRUE;
 	}
 #endif
 	if (db_value_of_name(CC_SYM_PREFIX "thandler", &value)) {
-	    db_syscall_symbol_value = (db_addr_t) value;
-	    found_some = TRUE;
+		db_syscall_symbol_value = (db_addr_t) value;
+		found_some = TRUE;
 	}
 	if (found_some) 
-	    db_trace_symbols_found = TRUE;
+		db_trace_symbols_found = TRUE;
 }
 
 int
@@ -281,7 +637,7 @@ db_numargs(
 	struct db_ppc_frame	*fp,
 	task_t			task)
 {
-	return (DB_NUMARGS_MAX);
+	return DB_NUMARGS_MAX;
 }
 
 boolean_t
@@ -309,7 +665,7 @@ db_find_arg(
 		}
 		inst = db_get_task_value(calleep, 4, FALSE, task);
 		if ((inst & 0xffff0000) == (0x907f0000 + (narg << 21)) ||
-		    (inst & 0xffff0000) == (0x90610000 + (narg << 21))) {
+				(inst & 0xffff0000) == (0x90610000 + (narg << 21))) {
 			argp = (db_addr_t) &(fp->f_arg[narg]);
 			*arg = argp;
 			return TRUE;
@@ -319,6 +675,7 @@ db_find_arg(
 	return FALSE;
 }
 
+extern int	TRAP_TYPES;
 /* 
  * Figure out the next frame up in the call stack.  
  * For trap(), we print the address of the faulting instruction and 
@@ -338,56 +695,51 @@ db_nextframe(
 	thread_act_t		thr_act,
 	db_addr_t		linkpc)		/* in */
 {
-	extern char *	trap_type[];
-	extern int	TRAP_TYPES;
-
 	struct savearea *saved_regs;
 
 	task_t task = (thr_act != THR_ACT_NULL)? thr_act->task: TASK_NULL;
 
 	switch(frame_type) {
 	case TRAP:
-
-	    db_printf(">>>>> trap <<<<<\n");
-	    goto miss_frame;
-	    break;
+		db_printf(">>>>> trap <<<<<\n");
+		goto miss_frame;
+		break;
 	case INTERRUPT:
-	    if (*lfp == 0) {
+		if (*lfp == 0) {
+			db_printf(">>>>> interrupt <<<<<\n");
+			goto miss_frame;
+		}
 		db_printf(">>>>> interrupt <<<<<\n");
 		goto miss_frame;
-	    }
-	    db_printf(">>>>> interrupt <<<<<\n");
-	    goto miss_frame;
-	    break;
-	case SYSCALL:
-	    if (thr_act != THR_ACT_NULL && thr_act->machine.pcb) {
-		*ip = (db_addr_t) thr_act->machine.pcb->save_srr0;
-		*fp = (struct db_ppc_frame *) (thr_act->machine.pcb->save_r1);
 		break;
-	    }
-	    /* falling down for unknown case */
+	case SYSCALL:
+		if (thr_act != THR_ACT_NULL && thr_act->machine.pcb) {
+			*ip = (db_addr_t) thr_act->machine.pcb->save_srr0;
+			*fp = (struct db_ppc_frame *) (thr_act->machine.pcb->save_r1);
+			break;
+		}
+		/* falling down for unknown case */
 	default:
-	miss_frame:
-		
+miss_frame:
 		if(!pmap_find_phys(kernel_pmap, (addr64_t)*fp)) {	/* Check if this is valid */
 			db_printf("Frame not mapped %08X\n",*fp);		/* Say not found */
 			*fp = 0;										/* Show not found */
 			break;											/* Out of here */
 		}
-		
+
 		if ((*fp)->f_frame)
-		    *ip = (db_addr_t)
-			    db_get_task_value((int)&(*fp)->f_frame->f_retaddr,
-					      4, FALSE, task);
+			*ip = (db_addr_t)
+				db_get_task_value((int)&(*fp)->f_frame->f_retaddr,
+						4, FALSE, task);
 		else
 			*ip = (db_addr_t) 
-                            db_get_task_value((int)&(*fp)->f_retaddr,
-                                              4, FALSE, task);
+				db_get_task_value((int)&(*fp)->f_retaddr,
+						4, FALSE, task);
 
-	    *lfp = *fp;
-	    *fp = (struct db_ppc_frame *)
-		db_get_task_value((int)&(*fp)->f_frame, 4, FALSE, task);
-	    break;
+		*lfp = *fp;
+		*fp = (struct db_ppc_frame *)
+			db_get_task_value((int)&(*fp)->f_frame, 4, FALSE, task);
+		break;
 	}
 }
 
@@ -416,10 +768,10 @@ db_stack_trace_cmd(
 	queue_entry_t	act_list;
 
 	if (!db_trace_symbols_found)
-	    db_find_trace_symbols();
+		db_find_trace_symbols();
 	{
-	    register char *cp = modif;
-	    register char c;
+		char *cp = modif;
+		char c;
 
 		while ((c = *cp++) != 0) {
 			if (c == 't')
@@ -434,12 +786,12 @@ db_stack_trace_cmd(
 	}
 
 	if (trace_all_threads) {
-	    if (!have_addr && !trace_thread) {
+		if (!have_addr && !trace_thread) {
 			have_addr = TRUE;
 			trace_thread = TRUE;
 			act_list = &(current_task()->threads);
 			addr = (db_expr_t) queue_first(act_list);
-	    } 
+		} 
 		else if (trace_thread) {
 			if (have_addr) {
 				if (!db_check_act_address_valid((thread_act_t)addr)) {
@@ -451,7 +803,7 @@ db_stack_trace_cmd(
 				else {
 					act_list = &(((thread_act_t)addr)->task->threads);
 					thcount = db_lookup_task_act(((thread_act_t)addr)->task,
-									(thread_act_t)addr);
+							(thread_act_t)addr);
 				}
 			} 
 			else {
@@ -466,11 +818,11 @@ db_stack_trace_cmd(
 				act_list = &th->task->threads;
 				addr = (db_expr_t) queue_first(act_list);
 			}
-	    }
+		}
 	}
 
 	if (count == -1)
-	    count = 65535;
+		count = 65535;
 
 next_thread:
 	top_act = THR_ACT_NULL;
@@ -479,53 +831,51 @@ next_thread:
 	frame_count = count;
 
 	if (!have_addr && !trace_thread) {
-	    frame = (struct db_ppc_frame *)(ddb_regs.save_r1);
-	    callpc = (db_addr_t)ddb_regs.save_srr0;
-	    linkpc = (db_addr_t)ddb_regs.save_lr;
-	    th = current_thread();
-	    task = (th != THR_ACT_NULL)? th->task: TASK_NULL;
+		frame = (struct db_ppc_frame *)(ddb_regs.save_r1);
+		callpc = (db_addr_t)ddb_regs.save_srr0;
+		linkpc = (db_addr_t)ddb_regs.save_lr;
+		th = current_thread();
+		task = (th != THR_ACT_NULL)? th->task: TASK_NULL;
 	} 
 	else if (trace_thread) {
-	    if (have_addr) {
+		if (have_addr) {
 			th = (thread_act_t) addr;
 			if (!db_check_act_address_valid(th))
-		   		return;
-	    } 
+				return;
+		} 
 		else {
 			th = db_default_act;
 			if (th == THR_ACT_NULL)
-			   th = current_thread();
+				th = current_thread();
 			if (th == THR_ACT_NULL) {
-			   db_printf("no active thread\n");
-			   return;
+				db_printf("no active thread\n");
+				return;
 			}
-	    }
-	    if (trace_all_threads)
-		db_printf("---------- Thread 0x%x (#%d of %d) ----------\n",
-			  addr, thcount, th->task->thread_count);
+		}
+		if (trace_all_threads)
+			db_printf("---------- Thread 0x%x (#%d of %d) ----------\n",
+					addr, thcount, th->task->thread_count);
 
 next_activation:
+		user_frame = 0;
 
-	    user_frame = 0;
-
-	    task = th->task;
-	    if (th == current_thread()) {
-	        frame = (struct db_ppc_frame *)(ddb_regs.save_r1);
-	        callpc = (db_addr_t)ddb_regs.save_srr0;
+		task = th->task;
+		if (th == current_thread()) {
+			frame = (struct db_ppc_frame *)(ddb_regs.save_r1);
+			callpc = (db_addr_t)ddb_regs.save_srr0;
 			linkpc = (db_addr_t)ddb_regs.save_lr;
-	    } 
+		} 
 		else {
 			if (th->machine.pcb == 0) {
-		    	db_printf("thread has no pcb\n");
+				db_printf("thread has no pcb\n");
 				goto thread_done;
 			}
 			if (th->kernel_stack == 0) {
-				register struct savearea *pss =
-							th->machine.pcb;
-	
+				struct savearea *pss = th->machine.pcb;
+
 				db_printf("Continuation ");
 				db_task_printsym((db_expr_t)th->continuation,
-								DB_STGY_PROC, task);
+						DB_STGY_PROC, task);
 				db_printf("\n");
 				frame = (struct db_ppc_frame *) (pss->save_r1);
 				callpc = (db_addr_t) (pss->save_srr0);
@@ -533,11 +883,11 @@ next_activation:
 			} 
 			else {
 				int cpu;
-	
+
 				for (cpu = 0; cpu < real_ncpus; cpu++) {
 					if (cpu_to_processor(cpu)->state == PROCESSOR_RUNNING &&
-						cpu_to_processor(cpu)->active_thread == th &&
-						PerProcTable[cpu].ppe_vaddr->db_saved_state) {
+							cpu_to_processor(cpu)->active_thread == th &&
+							PerProcTable[cpu].ppe_vaddr->db_saved_state) {
 						break;
 					}
 				}
@@ -548,68 +898,68 @@ next_activation:
 					 * use the activation's pcb.
 					 */
 					struct savearea *pss;
-	
+
 					pss = th->machine.pcb;
 					frame = (struct db_ppc_frame *) (pss->save_r1);
 					callpc = (db_addr_t) (pss->save_srr0);
 					linkpc = (db_addr_t) (pss->save_lr);
-					} else {
-						if (cpu == real_ncpus) {
-							register struct savearea *iks;
-							int r;
-			
-							iks = th->machine.pcb;
-							prev = db_recover;
-							if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
-								frame = (struct db_ppc_frame *) (iks->save_r1);
-								callpc = (db_addr_t) (iks->save_lr);
-								linkpc = 0;
-							} else {
-								/*
-								 * The kernel stack has probably been
-								 * paged out (swapped out activation).
-								 */
-								db_recover = prev;
-								if (r == 2)	/* 'q' from db_more() */
-									db_error(0);
-								db_printf("<kernel stack (0x%x) error "
-									  "(probably swapped out)>\n",
-									  iks);
-								goto next_act;
-							}
-							db_recover = prev;
+				} else {
+					if (cpu == real_ncpus) {
+						struct savearea *iks;
+						int r;
+
+						iks = th->machine.pcb;
+						prev = db_recover;
+						if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
+							frame = (struct db_ppc_frame *) (iks->save_r1);
+							callpc = (db_addr_t) (iks->save_lr);
+							linkpc = 0;
 						} else {
-							db_printf(">>>>> active on cpu %d <<<<<\n",
-								  cpu);
-							frame = (struct db_ppc_frame *)
-							(PerProcTable[cpu].ppe_vaddr->db_saved_state->save_r1);
-							callpc = (db_addr_t) PerProcTable[cpu].ppe_vaddr->db_saved_state->save_srr0;
-							linkpc = (db_addr_t) PerProcTable[cpu].ppe_vaddr->db_saved_state->save_lr;
+							/*
+							 * The kernel stack has probably been
+							 * paged out (swapped out activation).
+							 */
+							db_recover = prev;
+							if (r == 2)	/* 'q' from db_more() */
+								db_error(0);
+							db_printf("<kernel stack (0x%x) error "
+									"(probably swapped out)>\n",
+									iks);
+							goto next_act;
 						}
+						db_recover = prev;
+					} else {
+						db_printf(">>>>> active on cpu %d <<<<<\n",
+								cpu);
+						frame = (struct db_ppc_frame *)
+							(PerProcTable[cpu].ppe_vaddr->db_saved_state->save_r1);
+						callpc = (db_addr_t) PerProcTable[cpu].ppe_vaddr->db_saved_state->save_srr0;
+						linkpc = (db_addr_t) PerProcTable[cpu].ppe_vaddr->db_saved_state->save_lr;
 					}
 				}
-	    }
+			}
+		}
 	} else {
-	    frame = (struct db_ppc_frame *)addr;
-	    th = (db_default_act)? db_default_act: current_thread();
-	    task = (th != THR_ACT_NULL)? th->task: TASK_NULL;
-	    if (frame->f_frame) {
-	      callpc = (db_addr_t)db_get_task_value
+		frame = (struct db_ppc_frame *)addr;
+		th = (db_default_act)? db_default_act: current_thread();
+		task = (th != THR_ACT_NULL)? th->task: TASK_NULL;
+		if (frame->f_frame) {
+			callpc = (db_addr_t)db_get_task_value
 				((int)&frame->f_frame->f_retaddr,
-				4, FALSE, (user_frame) ? task : 0);
-	      callpc = callpc-sizeof(callpc);
-	    } else
-	      callpc =0;
-	    linkpc = 0;
+				 4, FALSE, (user_frame) ? task : 0);
+			callpc = callpc-sizeof(callpc);
+		} else
+			callpc =0;
+		linkpc = 0;
 	}
 
 	if (!INKERNELSTACK((unsigned)frame, th)) {
-	    db_printf(">>>>> user space <<<<<\n");
-	    if (kernel_only)
-		goto thread_done;
-	    user_frame++;
+		db_printf(">>>>> user space <<<<<\n");
+		if (kernel_only)
+			goto thread_done;
+		user_frame++;
 	}
-	
+
 	lastframe = 0;
 	lastcallpc = (db_addr_t) 0;
 	while (frame_count-- && frame != 0) {
@@ -622,51 +972,51 @@ next_activation:
 		db_addr_t	off;
 
 		db_symbol_values(NULL,
-			db_search_task_symbol_and_line(
-				callpc, DB_STGY_XTRN, &offset, &filename,
-				&linenum, (user_frame) ? task : 0, &narg),
-			&name, (db_expr_t *)&call_func);
+				db_search_task_symbol_and_line(
+					callpc, DB_STGY_XTRN, &offset, &filename,
+					&linenum, (user_frame) ? task : 0, &narg),
+				&name, (db_expr_t *)&call_func);
 		if ( name == NULL) {
 			db_find_task_sym_and_offset(callpc, 
-				&name, &off, (user_frame) ? task : 0);
+					&name, &off, (user_frame) ? task : 0);
 			offset = (db_expr_t) off;
 		}
 
 		if (user_frame == 0) {
 			if (call_func &&
-				(call_func == db_user_trap_symbol_value ||
-				call_func == db_kernel_trap_symbol_value)) {
-			frame_type = TRAP;
-			narg = 1;
+					(call_func == db_user_trap_symbol_value ||
+					 call_func == db_kernel_trap_symbol_value)) {
+				frame_type = TRAP;
+				narg = 1;
 			} else if (call_func &&
-				call_func == db_interrupt_symbol_value) {
+					call_func == db_interrupt_symbol_value) {
 				frame_type = INTERRUPT;
 				goto next_frame;
 			} else if (call_func &&
-				call_func == db_syscall_symbol_value) {
+					call_func == db_syscall_symbol_value) {
 				frame_type = SYSCALL;
 				goto next_frame;
 			} else {
 				frame_type = 0;
 				prev = db_recover;
 				if ((r = _setjmp(db_recover = &db_jmp_buf)) 
-								== 0) {
-			 		if (narg < 0)
+						== 0) {
+					if (narg < 0)
 						narg = db_numargs(frame,
-						    (user_frame) ? task : 0);
+								(user_frame) ? task : 0);
 					db_recover = prev;
 				} else {
 					db_recover = prev;
 					goto next_act;
 				}
 			}
-	    } else {
+		} else {
 			frame_type = 0;
 			prev = db_recover;
 			if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
 				if (narg < 0)
 					narg = db_numargs(frame,
-						(user_frame) ? task : 0);
+							(user_frame) ? task : 0);
 				db_recover = prev;
 			} else {
 				db_recover = prev;
@@ -674,99 +1024,99 @@ next_activation:
 			}
 		}
 
-	    if (name == 0 || offset > db_maxoff) {
+		if (name == 0 || offset > db_maxoff) {
 			db_printf("[%08X]0x%08X(", frame, callpc);
-	    } else {
-	        db_printf("[%08X]%s", frame, name);
+		} else {
+			db_printf("[%08X]%s", frame, name);
 			if (offset)
 				db_printf("+%llx", offset);
-	        db_printf("(");
-	   };
+			db_printf("(");
+		};
 
-	narg = db_numargs(frame, (user_frame) ? task : 0);
+		narg = db_numargs(frame, (user_frame) ? task : 0);
 
-	for (arg =0; arg < narg; arg++) {
-		db_addr_t	argp;
-		int value;
-		boolean_t found;
+		for (arg = 0; arg < narg; arg++) {
+			db_addr_t	argp;
+			int value;
+			boolean_t found;
+
+			prev = db_recover;
+			if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
+				found = FALSE;
+				if (lastframe) 
+					found = db_find_arg(frame, lastframe->f_retaddr,
+							(user_frame) ? task : 0, arg, &argp);
+				if (found)
+					value = db_get_task_value(argp, 4, FALSE,
+							(user_frame) ? task : 0);
+			} else {
+				db_recover = prev;
+				if (r == 2)	/* 'q' from db_more() */
+					db_error(0);
+				db_printf("... <stack error>)");
+				db_printf("\n");
+				goto next_act;
+			}
+			db_recover = prev;
+			if (found)
+				db_printf("%08X", value);
+			else
+				db_printf("??");	
+			argp = argp + sizeof(argp);
+			if (arg < narg-1)
+				db_printf(",");
+		}
+		if (arg != narg)
+			db_printf("...");
+		db_printf(")");
+		db_printf("\n");
 
+next_frame:
+		lastcallpc = callpc;
 		prev = db_recover;
 		if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
-			found = FALSE;
-			if (lastframe) 
-				found = db_find_arg(frame, lastframe->f_retaddr,
-					(user_frame) ? task : 0, arg, &argp);
-			if (found)
-				value = db_get_task_value(argp, 4, FALSE,
-					(user_frame) ? task : 0);
+			db_nextframe(&lastframe, &frame, &callpc, frame_type,
+					(user_frame) ? th : THR_ACT_NULL, linkpc);
+			callpc = callpc-sizeof(callpc);
+			db_recover = prev;
 		} else {
 			db_recover = prev;
-			if (r == 2)	/* 'q' from db_more() */
-				db_error(0);
-		    	db_printf("... <stack error>)");
-			db_printf("\n");
-			goto next_act;
+			frame = 0;
 		}
-		db_recover = prev;
-		if (found)
-			db_printf("%08X", value);
-		else
-			db_printf("??");	
-		argp = argp + sizeof(argp);
-		if (arg < narg-1)
-			db_printf(",");
-	    }
-	    if (arg != narg)
-		db_printf("...");
-	    db_printf(")");
-	    db_printf("\n");
-
-	next_frame:
-	    lastcallpc = callpc;
-	    prev = db_recover;
-	    if ((r = _setjmp(db_recover = &db_jmp_buf)) == 0) {
-		    db_nextframe(&lastframe, &frame, &callpc, frame_type,
-				 (user_frame) ? th : THR_ACT_NULL, linkpc);
-		    callpc = callpc-sizeof(callpc);
-		    db_recover = prev;
-	    } else {
-		    db_recover = prev;
-		    frame = 0;
-	    }
-	    linkpc = 0;
-
-	    if (frame == 0) {
-	next_act:
-		/* end of chain */
-		break;
-	    }
-	    if (!INKERNELSTACK(lastframe, th) ||
-		!INKERNELSTACK((unsigned)frame, th))
-		user_frame++;
-	    if (user_frame == 1) {
-		db_printf(">>>>> user space <<<<<\n");
-		if (kernel_only)
-		    break;
-	    }
-		
+		linkpc = 0;
+
+		if (frame == 0) {
+next_act:
+			/* end of chain */
+			break;
+		}
+		if (!INKERNELSTACK(lastframe, th) ||
+				!INKERNELSTACK((unsigned)frame, th))
+			user_frame++;
+		if (user_frame == 1) {
+			db_printf(">>>>> user space <<<<<\n");
+			if (kernel_only)
+				break;
+		}
+
 		if (frame <= lastframe) {
-		if ((INKERNELSTACK(lastframe, th) && !INKERNELSTACK(frame, th))) continue;
-		db_printf("Bad frame pointer: 0x%x\n", frame);
-		break;
-	    }
+			if ((INKERNELSTACK(lastframe, th) && !INKERNELSTACK(frame, th)))
+				continue;
+			db_printf("Bad frame pointer: 0x%x\n", frame);
+			break;
+		}
 	}
 
-    thread_done:
+thread_done:
 	if (trace_all_threads) {
-	    if (top_act != THR_ACT_NULL)
-		th = top_act;
-	    th = (thread_act_t) queue_next(&th->task_threads);
-	    if (! queue_end(act_list, (queue_entry_t) th)) {
-		db_printf("\n");
-		addr = (db_expr_t) th;
-		thcount++;
-		goto next_thread;
-
-	    }
+		if (top_act != THR_ACT_NULL)
+			th = top_act;
+		th = (thread_act_t) queue_next(&th->task_threads);
+		if (! queue_end(act_list, (queue_entry_t) th)) {
+			db_printf("\n");
+			addr = (db_expr_t) th;
+			thcount++;
+			goto next_thread;
+		}
 	}
 }
diff --git a/osfmk/ppc/endian.h b/osfmk/ppc/endian.h
index b162f0f38..397b09de8 100644
--- a/osfmk/ppc/endian.h
+++ b/osfmk/ppc/endian.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/kern/etimer.c b/osfmk/ppc/etimer.c
similarity index 57%
rename from osfmk/kern/etimer.c
rename to osfmk/ppc/etimer.c
index 67b99dbaf..4d3bb1a5f 100644
--- a/osfmk/kern/etimer.c
+++ b/osfmk/ppc/etimer.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -45,28 +51,11 @@
 #include <machine/machine_routines.h>
 
 #include <sys/kdebug.h>
-
-#ifdef __ppc__
 #include <ppc/exception.h>
-#else
-#include <i386/cpu_data.h>
-#endif
-
-#include <sys/kdebug.h>
-
 
 /* XXX from <arch>/rtclock.c */
-uint32_t		 	rtclock_tick_interval;
 clock_timer_func_t		rtclock_timer_expire;
 
-#ifdef __ppc__
-# define PER_PROC_INFO		struct per_proc_info
-# define GET_PER_PROC_INFO()	getPerProc()
-#else
-# define PER_PROC_INFO 		cpu_data_t
-# define GET_PER_PROC_INFO()	current_cpu_datap()
-#endif
-
 /*
  * 	Event timer interrupt.
  *
@@ -75,13 +64,16 @@ clock_timer_func_t		rtclock_timer_expire;
  *
  * XXX a better implementation would use a set of generic callouts and iterate over them
  */
-void etimer_intr(int inuser, uint64_t iaddr) {
-	
+void
+etimer_intr(
+__unused int inuser,
+__unused uint64_t iaddr)
+{
 	uint64_t		abstime;
 	rtclock_timer_t		*mytimer;
-	PER_PROC_INFO		*pp;
+	struct per_proc_info	*pp;
 
-	pp = GET_PER_PROC_INFO();
+	pp = getPerProc();
 
 	mytimer = &pp->rtclock_timer;				/* Point to the event timer */
 
@@ -89,7 +81,6 @@ void etimer_intr(int inuser, uint64_t iaddr) {
 
 	/* is it time for power management state change? */	
 	if (pp->pms.pmsPop <= abstime) {
-
 	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_START, 0, 0, 0, 0, 0);
 		pmsStep(1);					/* Yes, advance step */
 	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 3) | DBG_FUNC_END, 0, 0, 0, 0, 0);
@@ -97,25 +88,6 @@ void etimer_intr(int inuser, uint64_t iaddr) {
 		abstime = mach_absolute_time();			/* Get the time again since we ran a bit */
 	}
 
-	/* have we passed the rtclock pop time? */
-	if (pp->rtclock_intr_deadline <= abstime) {
-
-	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 4) | DBG_FUNC_START, (int)rtclock_tick_interval, 0, 0, 0, 0);
-
-		clock_deadline_for_periodic_event(rtclock_tick_interval,
-		    abstime,
-		    &pp->rtclock_intr_deadline);
-
-	        KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 4) | DBG_FUNC_END, 0, 0, 0, 0, 0);
-#if STAT_TIME
-		hertz_tick(NSEC_PER_HZ, inuser, iaddr);		/* Accumulate hertz */
-#else
-		hertz_tick(inuser, iaddr);			/* Accumulate hertz */
-#endif
-
-		abstime = mach_absolute_time();			/* Refresh the current time since we went away */
-	}
-
 	/* has a pending clock timer expired? */
 	if (mytimer->deadline <= abstime) {			/* Have we expired the deadline? */
 		mytimer->has_expired = TRUE;			/* Remember that we popped */
@@ -136,10 +108,10 @@ void etimer_set_deadline(uint64_t deadline)
 {
 	rtclock_timer_t		*mytimer;
 	spl_t			s;
-	PER_PROC_INFO		*pp;
+	struct per_proc_info	*pp;
 
 	s = splclock();					/* no interruptions */
-	pp = GET_PER_PROC_INFO();
+	pp = getPerProc();
 
 	mytimer = &pp->rtclock_timer;			/* Point to the timer itself */
 	mytimer->deadline = deadline;			/* Set the new expiration time */
@@ -149,6 +121,7 @@ void etimer_set_deadline(uint64_t deadline)
 	splx(s);
 }
 
+
 /*
  * Re-evaluate the outstanding deadlines and select the most proximate.
  *
@@ -160,34 +133,34 @@ etimer_resync_deadlines(void)
 	uint64_t		deadline;
 	rtclock_timer_t		*mytimer;
 	spl_t			s = splclock();		/* No interruptions please */
-	PER_PROC_INFO		*pp;
+	struct per_proc_info	*pp;
 
-	pp = GET_PER_PROC_INFO();
+	pp = getPerProc();
 
-	deadline = 0;
-	
-	/* next rtclock interrupt? */
-	if (pp->rtclock_intr_deadline > 0)
-		deadline = pp->rtclock_intr_deadline;
+	deadline = ~0ULL;
 
 	/* if we have a clock timer set sooner, pop on that */
 	mytimer = &pp->rtclock_timer;			/* Point to the timer itself */
-	if ((!mytimer->has_expired) && (mytimer->deadline > 0) && (mytimer->deadline < deadline))
+	if (!mytimer->has_expired && mytimer->deadline > 0)
 		deadline = mytimer->deadline;
 
 	/* if we have a power management event coming up, how about that? */
-	if ((pp->pms.pmsPop > 0) && (pp->pms.pmsPop < deadline))
+	if (pp->pms.pmsPop > 0 && pp->pms.pmsPop < deadline)
 		deadline = pp->pms.pmsPop;
 	
-#ifdef __ppc__
-#endif
 
-        if ((deadline > 0) && (deadline < pp->rtcPop)) {
+	if (deadline > 0 && deadline <= pp->rtcPop) {
 		int     decr;
+		uint64_t now;
 
-		pp->rtcPop = deadline;
+		now = mach_absolute_time();
 		decr = setPop(deadline);
 
+		if (deadline < now)
+		        pp->rtcPop = now + decr;
+		else
+		        pp->rtcPop = deadline;
+
 		KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, decr, 2, 0, 0, 0);
 	}
 	splx(s);
diff --git a/osfmk/ppc/exception.h b/osfmk/ppc/exception.h
index 9c4ecc0ef..75f7e2e28 100644
--- a/osfmk/ppc/exception.h
+++ b/osfmk/ppc/exception.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -350,12 +356,12 @@ struct per_proc_info {
 	/* PPC cache line boundary here - 160 */
 	struct savearea *	db_saved_state;
 	time_base_enable_t	time_base_enable;
-	int				ppXFlags;
+	uint32_t		ppXFlags;
 	int				running;
 	int				debugger_is_slave;
 	int				debugger_active;
 	int				debugger_pending;
-	int				debugger_holdoff;
+	uint32_t		debugger_holdoff;
 	
 	/* PPC cache line boundary here - 180 */
     uint64_t        Uassist;            /* User Assist DoubleWord */
@@ -513,10 +519,15 @@ struct per_proc_info {
 	hwCtrs			hwCtr;					/* Hardware exception counters */
 /*								   - A00 */
 	addr64_t		pp2ndPage;				/* Physical address of the second page of the per_proc */
-	uint32_t		pprsvd0A08[6];
+	addr64_t		ijsave;					/* Pointer to original savearea for injected code */
+	uint32_t		pprsvd0A10[4];
 /*								   - A20 */
 	pmsd			pms;					/* Power Management Stepper control */
-	unsigned int	pprsvd0A40[368];		/* Reserved out to next page boundary */
+	unsigned int	pprsvd0A40[16];			/* Reserved */
+/*								   - A80 */
+	uint32_t		pprsvd0A80[16];			/* Reserved */
+	
+	unsigned int	pprsvd0AC0[336];		/* Reserved out to next page boundary */
 /*								   - 1000 */
 
 /*
@@ -560,7 +571,7 @@ struct per_proc_entry {
 extern	struct per_proc_entry PerProcTable[MAX_CPUS-1];
 
 
-extern char *trap_type[];
+extern const char *trap_type[];
 
 #endif /* ndef ASSEMBLER */					/* with this savearea should be redriven */
 
@@ -568,8 +579,6 @@ extern char *trap_type[];
 #define SIGPactive	0x8000
 #define needSRload	0x4000
 #define turnEEon	0x2000
-#define traceBE     0x1000					/* user mode BE tracing in enabled */
-#define traceBEb    3						/* bit number for traceBE */
 #define SleepState	0x0800
 #define SleepStateb	4
 #define mcountOff	0x0400
@@ -634,6 +643,8 @@ extern char *trap_type[];
 #define T_INSTRUMENTATION		(0x2B * T_VECTOR_SIZE)
 #define T_ARCHDEP0				(0x2C * T_VECTOR_SIZE)
 #define T_HDEC					(0x2D * T_VECTOR_SIZE)
+#define T_INJECT_EXIT			(0x2E * T_VECTOR_SIZE)
+#define T_DTRACE_RET			T_INJECT_EXIT
 
 #define T_AST					(0x100 * T_VECTOR_SIZE) 
 #define T_MAX					T_CHOKE		 /* Maximum exception no */
diff --git a/osfmk/ppc/fpu_protos.h b/osfmk/ppc/fpu_protos.h
index 99ec6d953..7ceed096a 100644
--- a/osfmk/ppc/fpu_protos.h
+++ b/osfmk/ppc/fpu_protos.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/genassym.c b/osfmk/ppc/genassym.c
index a30f7da5e..8207aeb55 100644
--- a/osfmk/ppc/genassym.c
+++ b/osfmk/ppc/genassym.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -37,7 +43,6 @@
  * the values, but we cannot run anything on the target machine.
  */
 
-#include <va_list.h>
 #include <types.h>
 
 #include <kern/task.h>
@@ -68,6 +73,11 @@
 #include <ppc/boot.h>
 #include <ppc/lowglobals.h>
 
+#if	CONFIG_DTRACE
+#define NEED_DTRACE_DEFS
+#include <../bsd/sys/lockstat.h>
+#endif
+
 /* Undefine standard offsetof because it is different than the one here */
 #undef offsetof
 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE)0)->MEMBER)
@@ -435,6 +445,8 @@ int main(int argc, char *argv[])
 	DECLARE("hwMckIEratPE",			offsetof(struct per_proc_info *, hwCtr.hwMckIEratPE));
 	DECLARE("hwMckDEratPE",			offsetof(struct per_proc_info *, hwCtr.hwMckDEratPE));
 
+	DECLARE("ijsave", 				offsetof(struct per_proc_info *, ijsave));
+
 	DECLARE("napStamp", 			offsetof(struct per_proc_info *, hwCtr.napStamp));
 	DECLARE("napTotal", 			offsetof(struct per_proc_info *, hwCtr.napTotal));
 	DECLARE("PP_PROCESSOR",			offsetof(struct per_proc_info *, processor[0]));
@@ -464,8 +476,8 @@ int main(int argc, char *argv[])
 #define IKSBASE (u_int)STACK_IKS(0)
 
 	/* values from kern/thread.h */
-	DECLARE("THREAD_OPTIONS",		offsetof(thread_t, options));
-	DECLARE("TH_OPT_DELAYIDLE", 	TH_OPT_DELAYIDLE);
+	DECLARE("THREAD_STATE",		offsetof(thread_t, state));
+	DECLARE("TH_IDLE",				TH_IDLE);
 	DECLARE("THREAD_KERNEL_STACK",	offsetof(thread_t, kernel_stack));
 	DECLARE("THREAD_RECOVER",		offsetof(thread_t, recover));
 	DECLARE("THREAD_FUNNEL_LOCK",
@@ -746,13 +758,13 @@ int main(int argc, char *argv[])
 	DECLARE("vmm64Bit",				vmm64Bit);
 
 	/* values from kern/task.h */
-	DECLARE("TASK_SYSCALLS_MACH",
-		offsetof(struct task *, syscalls_mach));
-	DECLARE("TASK_SYSCALLS_UNIX",
-		offsetof(struct task *, syscalls_unix));
+	DECLARE("TASK_SYSCALLS_MACH",	offsetof(struct task *, syscalls_mach));
+	DECLARE("TASK_SYSCALLS_UNIX",	offsetof(struct task *, syscalls_unix));
+
+	DECLARE("TASK_VTIMERS",			offsetof(struct task *, vtimers));
 
 	/* values from vm/vm_map.h */
-	DECLARE("VMMAP_PMAP",	offsetof(struct vm_map *, pmap));
+	DECLARE("VMMAP_PMAP",	offsetof(struct _vm_map *, pmap));
 
 	/* values from machine/pmap.h */
 	DECLARE("pmapSpace",			offsetof(struct pmap *, space));
@@ -796,6 +808,7 @@ int main(int argc, char *argv[])
 	DECLARE("pmapVAddr",			offsetof(struct pmapTransTab *, pmapVAddr));
 	DECLARE("pmapTransSize",		sizeof(pmapTransTab));
 	DECLARE("pmapResidentCnt",		offsetof(struct pmap *, stats.resident_count));
+	DECLARE("pmapResidentMax",		offsetof(struct pmap *, stats.resident_max));
 
 	DECLARE("maxAdrSp",				maxAdrSp);
 	DECLARE("maxAdrSpb",			maxAdrSpb);
@@ -852,9 +865,13 @@ int main(int argc, char *argv[])
 	DECLARE("TIMER_HIGHCHK",	offsetof(struct timer *, high_bits_check));
 	DECLARE("TIMER_TSTAMP",		offsetof(struct timer *, tstamp));
 
-	DECLARE("CURRENT_TIMER",	offsetof(struct processor *, processor_data.current_timer));
+	DECLARE("THREAD_TIMER",		offsetof(struct processor *, processor_data.thread_timer));
+	DECLARE("KERNEL_TIMER",		offsetof(struct processor *, processor_data.kernel_timer));
 	DECLARE("SYSTEM_TIMER",		offsetof(struct thread *, system_timer));
 	DECLARE("USER_TIMER",		offsetof(struct thread *, user_timer));
+	DECLARE("SYSTEM_STATE",		offsetof(struct processor *, processor_data.system_state));
+	DECLARE("USER_STATE",		offsetof(struct processor *, processor_data.user_state));
+	DECLARE("CURRENT_STATE",	offsetof(struct processor *, processor_data.current_state));
 
 	/* Constants from pmap.h */
 	DECLARE("PPC_SID_KERNEL", PPC_SID_KERNEL);
@@ -876,6 +893,7 @@ int main(int argc, char *argv[])
 	/* Misc values used by assembler */
 	DECLARE("AST_ALL", AST_ALL);
 	DECLARE("AST_URGENT", AST_URGENT);
+	DECLARE("AST_BSD", AST_BSD);
 
 	/* Spin Lock structure */
 	DECLARE("SLOCK_ILK",	offsetof(lck_spin_t *, interlock));
@@ -1411,5 +1429,10 @@ int main(int argc, char *argv[])
 	DECLARE("scomstat",				offsetof(struct scomcomm *, scomstat));
 	DECLARE("scomdata",				offsetof(struct scomcomm *, scomdata));
 
+#if	CONFIG_DTRACE
+	DECLARE("LS_LCK_MTX_UNLOCK_RELEASE", LS_LCK_MTX_UNLOCK_RELEASE);
+	DECLARE("LS_LCK_MTX_LOCK_ACQUIRE", LS_LCK_MTX_LOCK_ACQUIRE);
+#endif
+
 	return(0);  /* For ANSI C :-) */
 }
diff --git a/osfmk/ppc/hardclock_entries.h b/osfmk/ppc/hardclock_entries.h
index ab499201d..3a804ea9e 100644
--- a/osfmk/ppc/hardclock_entries.h
+++ b/osfmk/ppc/hardclock_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/hexfont.h b/osfmk/ppc/hexfont.h
index 91b3caf3e..38035a0c9 100644
--- a/osfmk/ppc/hexfont.h
+++ b/osfmk/ppc/hexfont.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*					  0123456789ABCDEF */
 
diff --git a/osfmk/ppc/hibernate_ppc.c b/osfmk/ppc/hibernate_ppc.c
index 3dbf4a9d7..1adc102f7 100644
--- a/osfmk/ppc/hibernate_ppc.c
+++ b/osfmk/ppc/hibernate_ppc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/kern_types.h>
@@ -43,8 +49,6 @@
 #include <ppc/trap.h>
 #include <ppc/mem.h>
 #include <IOKit/IOPlatformExpert.h>
-#define KERNEL
-
 #include <IOKit/IOHibernatePrivate.h>
 #include <vm/vm_page.h>
 
@@ -115,9 +119,9 @@ hibernate_page_list_setall_machine(hibernate_page_list_t * page_list,
 
 // mark pages not to be saved and not for scratch usage during restore
 void
-hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
-				  hibernate_page_list_t * page_list_wired,
-				  uint32_t * pagesOut)
+hibernate_page_list_set_volatile(__unused hibernate_page_list_t *page_list,
+				 __unused hibernate_page_list_t *page_list_wired,
+				 __unused uint32_t *pagesOut)
 {
 }
 
@@ -199,6 +203,8 @@ void ml_ppc_sleep(void)
 }
 
 void
-hibernate_newruntime_map(void * map, vm_size_t map_size, uint32_t runtime_offset)
+hibernate_newruntime_map(__unused void * map,
+			 __unused vm_size_t map_size,
+			 __unused uint32_t runtime_offset)
 {
 }
diff --git a/osfmk/ppc/hibernate_restore.s b/osfmk/ppc/hibernate_restore.s
index 1d46fe2a6..9025e6589 100644
--- a/osfmk/ppc/hibernate_restore.s
+++ b/osfmk/ppc/hibernate_restore.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <ppc/asm.h>
diff --git a/osfmk/ppc/hw_counters.h b/osfmk/ppc/hw_counters.h
index 2a3dd6d9e..99b984a50 100644
--- a/osfmk/ppc/hw_counters.h
+++ b/osfmk/ppc/hw_counters.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *
diff --git a/osfmk/ppc/hw_exception.s b/osfmk/ppc/hw_exception.s
index 4c66797de..ab77e1774 100644
--- a/osfmk/ppc/hw_exception.s
+++ b/osfmk/ppc/hw_exception.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -155,9 +161,26 @@ tvecoff:	stw		r26,FM_BACKPTR(r1)				; Link back to the previous frame
 #endif /* DEBUG */
 
 			mr		r30,r4
-			lwz		r3,SAVtime+4(r4)
-			addi	r4,r13,SYSTEM_TIMER
-			bl		EXT(timer_event)
+			lwz		r3,SAVtime(r4)
+			lwz		r4,SAVtime+4(r4)
+			addi	r5,r13,SYSTEM_TIMER
+			bl		EXT(thread_timer_event)
+			addi	r5,r25,SYSTEM_STATE
+			bl		EXT(state_event)
+
+			lwz		r7,ACT_TASK(r13)
+			lwz		r8,TASK_VTIMERS(r7)
+			cmpwi	r8,0
+			beq++	0f
+
+			lwz		r7,ACT_PER_PROC(r13)
+			li		r4,AST_BSD
+			lwz		r8,PP_PENDING_AST(r7)
+			or		r8,r8,r4
+			stw		r8,PP_PENDING_AST(r7)
+			addi	r3,r13,ACT_AST
+			bl		EXT(hw_atomic_or)
+0:
 
 /* call trap handler proper, with
  *   ARG0 = type
@@ -358,9 +381,26 @@ noassist:	cmplwi	r15,0x7000						; Do we have a fast path trap?
 			stw		r0,ACT_MACT_KSP(r13)			; Mark stack as busy with 0 val 
 			stw		r15,FM_BACKPTR(r1)				; Link stack frame backwards
 
-			lwz		r3,SAVtime+4(r30)
-			addi	r4,r13,SYSTEM_TIMER
-			bl		EXT(timer_event)
+			lwz		r3,SAVtime(r30)
+			lwz		r4,SAVtime+4(r30)
+			addi	r5,r13,SYSTEM_TIMER
+			bl		EXT(thread_timer_event)
+			addi	r5,r25,SYSTEM_STATE
+			bl		EXT(state_event)
+		
+			lwz		r7,ACT_TASK(r13)
+			lwz		r8,TASK_VTIMERS(r7)
+			cmpwi	r8,0
+			beq++	0f
+
+			lwz		r7,ACT_PER_PROC(r13)
+			li		r4,AST_BSD
+			lwz		r8,PP_PENDING_AST(r7)
+			or		r8,r8,r4
+			stw		r8,PP_PENDING_AST(r7)
+			addi	r3,r13,ACT_AST
+			bl		EXT(hw_atomic_or)
+0:
 		
 #if	DEBUG
 /* If debugging, we need two frames, the first being a dummy
@@ -560,11 +600,11 @@ ksystrace:
 			ori		r8,r8,lo16(EXT(kdebug_enable))	; Get bottom of kdebug_enable 
 			lwz		r8,0(r8)						; Get kdebug_enable 
 
-			lwz		r7,TASK_SYSCALLS_MACH(r10)		; Get the current count
-			neg		r31,r0							; Make this positive
-			mr 		r3,r31							; save it
-			slwi	r27,r3,4						; multiply by 16
-			slwi	r3,r3,2							; and the original by 4
+			lwz		r7,TASK_SYSCALLS_MACH(r10)	; Get the current count
+			neg		r31,r0						; Make this positive
+			mr 		r3,r31						; save it
+			slwi	r27,r3,4					; multiply by 16
+			slwi	r3,r3,2						; and the original by 4
 			ori		r28,r28,lo16(EXT(mach_trap_table))	; Get address of table
 			add		r27,r27,r3						; for a total of 20x (5 words/entry)
 			addi	r7,r7,1							; Bump TASK_SYSCALLS_MACH count
@@ -586,10 +626,10 @@ ksystrace:
 .L_kernel_syscall_munge:
 			cmplwi	r0,0							; test for null munger
 			mtctr	r0								; Set the function call address
-			addi	r3,r30,saver3					; Pointer to args from save area
-			addi	r4,r1,FM_ARG0+ARG_SIZE			; Pointer for munged args
-			beq--	.L_kernel_syscall_trapcall		; null munger - skip to trap call
-			bctrl									; Call the munge function
+			addi	r3,r30,saver3						; Pointer to args from save area
+			addi	r4,r1,FM_ARG0+ARG_SIZE				; Pointer for munged args
+			beq--	.L_kernel_syscall_trapcall		;   null munger - skip to trap call
+			bctrl								; Call the munge function
 
 .L_kernel_syscall_trapcall:		
 			lwz		r0,MACH_TRAP_FUNCTION(r31)		; Pick up the function address
@@ -949,29 +989,46 @@ ihbootnover:										; (TEST/DEBUG)
 			mr		r31,r3
 			mr		r30,r4
 
-			lwz		r3,SAVtime+4(r4)
-			addi	r4,r13,SYSTEM_TIMER
-			bl		EXT(timer_event)
+			lwz		r3,SAVtime(r4)
+			lwz		r4,SAVtime+4(r4)
+			addi	r5,r25,PP_PROCESSOR
+			lwz		r5,KERNEL_TIMER(r5)
+			bl		EXT(thread_timer_event)
+			addi	r6,r25,PP_PROCESSOR
+			lwz		r5,CURRENT_STATE(r6)
+			addi	r7,r6,USER_STATE
+			cmplw	r5,r7
+			bne		0f
+			addi	r5,r6,SYSTEM_STATE
+			bl		EXT(state_event)
+0:
+
+			lwz		r7,ACT_TASK(r13)
+			lwz		r8,TASK_VTIMERS(r7)
+			cmpwi	r8,0
+			beq++	0f
+
+			lwz		r7,ACT_PER_PROC(r13)
+			li		r4,AST_BSD
+			lwz		r8,PP_PENDING_AST(r7)
+			or		r8,r8,r4
+			stw		r8,PP_PENDING_AST(r7)
+			addi	r3,r13,ACT_AST
+			bl		EXT(hw_atomic_or)
+0:
 
 			mr		r3,r31
 			mr		r4,r30
 			lwz		r5,savedsisr(r30)				; Get the DSISR
 			lwz		r6,savedar+4(r30)				; Get the DAR 
-			
+
 #if FPFLOOD
 			stfd	f31,emfp31(r25)					; (TEST/DEBUG)
 #endif
 
 			bl	EXT(interrupt)
 
-
-/* interrupt() returns a pointer to the saved state in r3
- *
- * Ok, back from C. Disable interrupts while we restore things
- */
-			.globl EXT(ihandler_ret)
-
-LEXT(ihandler_ret)									; Marks our return point from debugger entry
+/* interrupt() returns a pointer to the saved state in r3 */
 
 			lis		r10,hi16(MASK(MSR_VEC))			; Get the vector enable
 			mfmsr	r0								; Get our MSR
@@ -1423,13 +1480,19 @@ segtb:		mftbu	r20								; Get the upper time base
 			
 			mtdec	r13								; Set our value
 
-chkifuser:	addi	r4,r28,SYSTEM_TIMER
-			mftb	r3
+chkifuser:	bl		EXT(mach_absolute_time)
+			lwz		r5,ACT_PER_PROC(r28)
+			addi	r6,r5,PP_PROCESSOR
+			lwz		r5,KERNEL_TIMER(r6)
+			lwz		r29,CURRENT_STATE(r6)
 			beq--	cr7,chkifuser1					; Skip this if we are going to kernel...
 			stw		r18,umwSpace(r28)				; Half-invalidate to force MapUserAddressWindow to reload SRs
-			addi	r4,r28,USER_TIMER
+			addi	r5,r28,USER_TIMER
+			addi	r29,r6,USER_STATE
 
-chkifuser1:	bl		EXT(timer_event)
+chkifuser1:	bl		EXT(thread_timer_event)
+			mr		r5,r29
+			bl		EXT(state_event)
 
 chkenax:	
 
diff --git a/osfmk/ppc/hw_lock.s b/osfmk/ppc/hw_lock.s
index e56f3f94c..4fae9fe6e 100644
--- a/osfmk/ppc/hw_lock.s
+++ b/osfmk/ppc/hw_lock.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach_assert.h>
@@ -26,6 +32,37 @@
 #include <ppc/proc_reg.h>
 #include <assym.s>
 
+
+#include <config_dtrace.h>
+#if	CONFIG_DTRACE
+	#define	LOCKSTAT_LABEL(lab) \
+	.data		__ASMNL__	\
+	.globl	lab	__ASMNL__	\
+	lab:		__ASMNL__	\
+	.long 9f	__ASMNL__	\
+	.text		__ASMNL__	\
+	9:		__ASMNL__	\
+
+	.globl	_dtrace_probe, _lockstat_probemap
+#define		LOCKSTAT_RECORD(id)			\
+			lis	r6,hi16(_lockstat_probemap)		__ASMNL__	\
+			ori	r6,r6,lo16(_lockstat_probemap)		__ASMNL__	\
+			lwz	r5,4*id(r6)				__ASMNL__	\
+			mr.	r5,r5					__ASMNL__	\
+			beqlr--						__ASMNL__	\
+			mr	r4,r3					__ASMNL__	\
+			mr	r3,r5					__ASMNL__	\
+			li	r5,0					__ASMNL__	\
+			li	r6,0					__ASMNL__	\
+			li	r7,0					__ASMNL__	\
+			li	r8,0					__ASMNL__	\
+			PROLOG(0)					__ASMNL__	\
+			bl	_dtrace_probe				__ASMNL__	\
+			EPILOG
+#endif
+			
+
+
 #define	STRING	ascii
 
 #define	ILK_LOCKED		0x01
@@ -579,9 +616,9 @@ subtry:		lwarx	r3,0,r6							; Grab the area value
  */
 			.align	5
 			.globl	EXT(hw_atomic_or)
-
 LEXT(hw_atomic_or)
-
+			.globl	EXT(hw_atomic_or_noret)
+LEXT(hw_atomic_or_noret)
 			mr		r6,r3							; Save the area 		
 
 ortry:		lwarx	r3,0,r6							; Grab the area value
@@ -600,9 +637,9 @@ ortry:		lwarx	r3,0,r6							; Grab the area value
  */
 			.align	5
 			.globl	EXT(hw_atomic_and)
-
 LEXT(hw_atomic_and)
-
+			.globl	EXT(hw_atomic_and_noret)
+LEXT(hw_atomic_and_noret)
 			mr		r6,r3							; Save the area 		
 
 andtry:		lwarx	r3,0,r6							; Grab the area value
@@ -971,6 +1008,8 @@ mlcktry:
 LEXT(mlckPatch_isync)
 			isync									; stop prefeteching
 			blr
+; Need to debug making blr above a patch point and record:
+;			LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE)
 
 mlckspin00:
 			cmpli	cr0,r5,MUTEX_IND				; Is it a mutex indirect 
@@ -1041,11 +1080,11 @@ mlStatSkip:
 			lwz		r2,ACT_MACT_SPF(r10)			; Get the special flags
 			rlwinm. r2,r2,0,OnProcbit,OnProcbit 	; Is OnProcbit set?
 			beq		mlckslow0						; Lock owner isn't running
-			lis		r2,hi16(TH_OPT_DELAYIDLE)		; Get DelayedIdle Option
-			ori		r2,r2,lo16(TH_OPT_DELAYIDLE)	; Get DelayedIdle Option
-			lwz		r10,THREAD_OPTIONS(r10)			; Get the thread options
-			and.	r10,r10,r2						; Is DelayedIdle set?
-			bne		mlckslow0						; Lock owner is in delay idle
+			lis		r2,hi16(TH_IDLE)				; Get thread idle state
+			ori		r2,r2,lo16(TH_IDLE)				; Get thread idle state
+			lwz		r10,THREAD_STATE(r10)			; Get the thread state
+			and.	r10,r10,r2						; Is idle set?
+			bne		mlckslow0						; Lock owner is idling
 
 			mftb	r10								; Time stamp us now
 			sub		r10,r10,r8						; Get the elapsed time
@@ -1473,8 +1512,16 @@ mluLoop:
 			bne--	mluSlowX
 			stwcx.	r5,MUTEX_DATA,r3
 			bne--	mluLoop
+#if	CONFIG_DTRACE
+/* lock released - LS_LCK_MTX_UNLOCK_RELEASE */
+			LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
 			blr
 
+			LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE)
+#endif
+			blr
+
+
 mluSlow0:
 			cmpli	cr0,r5,MUTEX_IND				; Is it a mutex indirect 
 			bne--	L_mutex_unlock_slow				; No, go handle contention 
@@ -1892,108 +1939,6 @@ LEXT(ppc_usimple_unlock_rwmb)
 
 			b		epStart							; Go enable preemption...
 
-/*
- *		void enter_funnel_section(funnel_t *)
- *
- */
-			.align	5
-			.globl	EXT(enter_funnel_section)
-
-LEXT(enter_funnel_section)
-
-#if	!MACH_LDEBUG
-			lis		r10,hi16(EXT(kdebug_enable))
-			ori		r10,r10,lo16(EXT(kdebug_enable))
-			lwz		r10,0(r10)
-			lis		r11,hi16(EXT(split_funnel_off))
-			ori		r11,r11,lo16(EXT(split_funnel_off))
-			lwz		r11,0(r11)
-			or.		r10,r11,r10						; Check kdebug_enable or split_funnel_off
-			bne-	L_enter_funnel_section_slow		; If set, call the slow path
-			mfsprg	r6,1							; Get the current activation
-			lwz		r7,LOCK_FNL_MUTEX(r3)
-
-			lwz		r5,0(r7)						; Get lock quickly
-			mr.		r5,r5							; Locked?
-			bne--	L_enter_funnel_section_slow		; Yup...
-
-L_enter_funnel_section_loop:
-			lwarx	r5,0,r7							; Load the mutex lock
-			mr.		r5,r5
-			bne--	L_enter_funnel_section_slowX	; Go to the slow path
-			stwcx.	r6,0,r7							; Grab the lock
-			bne--	L_enter_funnel_section_loop		; Loop back if failed
-			.globl	EXT(entfsectPatch_isync)     
-LEXT(entfsectPatch_isync)
-			isync									; Stop prefeteching
-			li		r7,TH_FN_OWNED
-			stw		r3,THREAD_FUNNEL_LOCK(r6)		; Set the funnel lock reference
-			stw		r7,THREAD_FUNNEL_STATE(r6)		; Set the funnel state
-			blr
-
-L_enter_funnel_section_slowX:
-			li		r4,lgKillResv					; Killing field
-			stwcx.	r4,0,r4							; Kill reservation
-
-L_enter_funnel_section_slow:
-#endif
-			li		r4,TRUE
-			b		EXT(thread_funnel_set)
-
-/*
- *		void exit_funnel_section(void)
- *
- */
-			.align	5
-			.globl	EXT(exit_funnel_section)
-
-LEXT(exit_funnel_section)
-
-			mfsprg	r6,1							; Get the current activation
-			lwz		r3,THREAD_FUNNEL_LOCK(r6)		; Get the funnel lock
-			mr.		r3,r3							; Check on funnel held
-			beq-	L_exit_funnel_section_ret		; 
-#if	!MACH_LDEBUG
-			lis		r10,hi16(EXT(kdebug_enable))
-			ori		r10,r10,lo16(EXT(kdebug_enable))
-			lwz		r10,0(r10)
-			mr.		r10,r10
-			bne-	L_exit_funnel_section_slow		; If set, call the slow path
-			lwz		r7,LOCK_FNL_MUTEX(r3)			; Get the funnel mutex lock
-			.globl	EXT(retfsectPatch_isync)     
-LEXT(retfsectPatch_isync)
-			isync
-			.globl	EXT(retfsectPatch_eieio)     
-LEXT(retfsectPatch_eieio)
-			eieio
-
-			lwz		r5,0(r7)						; Get lock
-			rlwinm.	r4,r5,0,30,31					; Quick check for bail if pending waiter or interlock set 
-			bne--	L_exit_funnel_section_slow		; No can get...
-
-L_exit_funnel_section_loop:
-			lwarx	r5,0,r7
-			rlwinm.	r4,r5,0,30,31					; Bail if pending waiter or interlock set 
-			li		r5,0							; Clear the mutexlock 
-			bne--	L_exit_funnel_section_slowX
-			stwcx.	r5,0,r7							; Release the funnel mutexlock
-			bne--	L_exit_funnel_section_loop
-			li		r7,0
-			stw		r7,THREAD_FUNNEL_STATE(r6)		; Clear the funnel state
-			stw		r7,THREAD_FUNNEL_LOCK(r6)		; Clear the funnel lock reference
-			blr										; Return
-
-L_exit_funnel_section_slowX:
-			li		r4,lgKillResv					; Killing field
-			stwcx.	r4,0,r4							; Kill it
-
-L_exit_funnel_section_slow:
-#endif
-			li		r4,FALSE
-			b		EXT(thread_funnel_set)
-L_exit_funnel_section_ret:
-			blr
-
 /*
  *		void lck_rw_lock_exclusive(lck_rw_t*)
  *
@@ -2090,7 +2035,7 @@ rwlseloop:	lwarx	r5,RW_DATA,r3					; Grab the lock value
 			.globl  EXT(rwlsePatch_isync)
 LEXT(rwlsePatch_isync)
 			isync
-			li		r3,0							; Succeed, return FALSE...
+			li		r3,1							; Succeed, return TRUE...
 			blr
 rwlsespin:
 			li		r4,lgKillResv					; Killing field
diff --git a/osfmk/ppc/hw_lock_types.h b/osfmk/ppc/hw_lock_types.h
index f7478fbdc..be6e5568c 100644
--- a/osfmk/ppc/hw_lock_types.h
+++ b/osfmk/ppc/hw_lock_types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1998 Apple Computer
diff --git a/osfmk/ppc/hw_perfmon.c b/osfmk/ppc/hw_perfmon.c
index 609a68ec3..64d38d1e0 100644
--- a/osfmk/ppc/hw_perfmon.c
+++ b/osfmk/ppc/hw_perfmon.c
@@ -1,30 +1,39 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <kern/thread.h>
+#include <kern/ipc_tt.h>
 #include <ppc/exception.h>
 #include <ppc/savearea.h>
 #include <ppc/hw_perfmon.h>
 #include <ppc/hw_perfmon_mmcr.h>
+#include <ppc/trap.h>
+#include <mach/thread_act.h>
 
 decl_simple_lock_data(,hw_perfmon_lock)
 static task_t hw_perfmon_owner = TASK_NULL;
@@ -131,10 +140,10 @@ int perfmon_release_facility(task_t task)
 	return retval;
 }
 
-int perfmon_enable(thread_t thread)
+static int
+perfmon_enable(thread_t thread)
 {
 	struct savearea *sv = thread->machine.pcb;
-	kern_return_t kr;
 	kern_return_t retval = KERN_SUCCESS;
 	int curPMC;
   
@@ -233,7 +242,8 @@ int perfmon_disable(thread_t thread)
 	return KERN_SUCCESS;
 }
 
-int perfmon_clear_counters(thread_t thread)
+static int
+perfmon_clear_counters(thread_t thread)
 {
 	struct savearea *sv = thread->machine.pcb;
 	int curPMC;
@@ -251,7 +261,8 @@ int perfmon_clear_counters(thread_t thread)
 	return KERN_SUCCESS;
 }
 
-int perfmon_write_counters(thread_t thread, uint64_t *pmcs)
+static int
+perfmon_write_counters(thread_t thread, uint64_t *pmcs)
 {
 	struct savearea *sv = thread->machine.pcb;
 	int curPMC;
@@ -269,7 +280,8 @@ int perfmon_write_counters(thread_t thread, uint64_t *pmcs)
 	return KERN_SUCCESS;
 }
 
-int perfmon_read_counters(thread_t thread, uint64_t *pmcs)
+static int
+perfmon_read_counters(thread_t thread, uint64_t *pmcs)
 {
 	struct savearea *sv = thread->machine.pcb;
 	int curPMC;
@@ -300,7 +312,8 @@ int perfmon_read_counters(thread_t thread, uint64_t *pmcs)
 	return KERN_SUCCESS;
 }
 
-int perfmon_start_counters(thread_t thread)
+static int
+perfmon_start_counters(thread_t thread)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -356,7 +369,8 @@ int perfmon_start_counters(thread_t thread)
 	return retval;
 }
 
-int perfmon_stop_counters(thread_t thread)
+static int
+perfmon_stop_counters(thread_t thread)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -392,7 +406,8 @@ int perfmon_stop_counters(thread_t thread)
 	return retval;
 }
 
-int perfmon_set_event(thread_t thread, int pmc, int event)
+static int
+perfmon_set_event(thread_t thread, int pmc, int event)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -532,7 +547,8 @@ int perfmon_set_event(thread_t thread, int pmc, int event)
 	return retval;
 }
 
-int perfmon_set_event_func(thread_t thread, uint32_t f)
+static int
+perfmon_set_event_func(thread_t thread, uint32_t f)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -589,7 +605,8 @@ int perfmon_set_event_func(thread_t thread, uint32_t f)
 	return retval;
 }
 
-int perfmon_set_threshold(thread_t thread, int threshold)
+static int
+perfmon_set_threshold(thread_t thread, int threshold)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -686,7 +703,8 @@ int perfmon_set_threshold(thread_t thread, int threshold)
 	return retval;
 }
 
-int perfmon_set_tbsel(thread_t thread, int tbsel)
+static int
+perfmon_set_tbsel(thread_t thread, int tbsel)
 {
 	struct savearea *sv = thread->machine.pcb;
 	kern_return_t retval = KERN_SUCCESS;
@@ -744,7 +762,7 @@ int perfmon_set_tbsel(thread_t thread, int tbsel)
 
 int perfmon_control(struct savearea *ssp)
 {
-	mach_port_t thr_port = CAST_DOWN(mach_port_t, ssp->save_r3); 
+	mach_port_name_t thr_port = CAST_DOWN(mach_port_name_t, ssp->save_r3); 
 	int action = (int)ssp->save_r4;
 	int pmc = (int)ssp->save_r5;
 	int val = (int)ssp->save_r6;
@@ -755,7 +773,7 @@ int perfmon_control(struct savearea *ssp)
 	int error;  
 	boolean_t oldlevel;
 
-	thread = (thread_t) port_name_to_thread(thr_port); // convert user space thread port name to a thread_t
+	thread = port_name_to_thread(thr_port); // convert user space thread port name to a thread_t
 	if(!thread) {
 		ssp->save_r3 = KERN_INVALID_ARGUMENT;
 		return 1;  /* Return and check for ASTs... */
@@ -826,7 +844,7 @@ int perfmon_control(struct savearea *ssp)
 				}
 			}
 			if(action & PPC_PERFMON_WRITE_COUNTERS) {
-				if(error = copyin(CAST_USER_ADDR_T(usr_pmcs_p), (void *)kern_pmcs, MAX_CPUPMC_COUNT*sizeof(uint64_t))) {
+				if((error = copyin(CAST_USER_ADDR_T(usr_pmcs_p), (void *)kern_pmcs, MAX_CPUPMC_COUNT*sizeof(uint64_t)))) {
 					retval = error;
 					goto perfmon_return;
 				}
@@ -842,7 +860,7 @@ int perfmon_control(struct savearea *ssp)
 					retval = error;
 					goto perfmon_return;
 				}
-				if(error = copyout((void *)kern_pmcs, CAST_USER_ADDR_T(usr_pmcs_p), MAX_CPUPMC_COUNT*sizeof(uint64_t))) {
+				if((error = copyout((void *)kern_pmcs, CAST_USER_ADDR_T(usr_pmcs_p), MAX_CPUPMC_COUNT*sizeof(uint64_t)))) {
 					retval = error;
 					goto perfmon_return;
 				}
diff --git a/osfmk/ppc/hw_perfmon.h b/osfmk/ppc/hw_perfmon.h
index eb21a4d00..c091d0b7b 100644
--- a/osfmk/ppc/hw_perfmon.h
+++ b/osfmk/ppc/hw_perfmon.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _HW_PERFMON_H_
 #define _HW_PERFMON_H_
diff --git a/osfmk/ppc/hw_perfmon_mmcr.h b/osfmk/ppc/hw_perfmon_mmcr.h
index 6d4bc7a0b..6dd894d94 100644
--- a/osfmk/ppc/hw_perfmon_mmcr.h
+++ b/osfmk/ppc/hw_perfmon_mmcr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  
 #ifndef _HW_PERFMON_MMCR_H_
diff --git a/osfmk/ppc/hw_vm.s b/osfmk/ppc/hw_vm.s
index 27e805ae9..bcad7dad2 100644
--- a/osfmk/ppc/hw_vm.s
+++ b/osfmk/ppc/hw_vm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <assym.s>
 #include <debug.h>
@@ -264,9 +270,16 @@ hamGotX:	mr		r3,r28						; Get the pmap to insert into
 			lhz		r8,mpSpace(r31)				; Get the address space
 			lwz		r11,lgpPcfg(r11)			; Get the page config
 			mfsdr1	r7							; Get the hash table base/bounds
+
 			lwz		r4,pmapResidentCnt(r28)		; Get the mapped page count 
+			lwz	r12,pmapResidentMax(r28)		; r12 = pmap->stats.resident_max
+			addi	r4,r4,1						; Bump up the mapped page count
+			stw		r4,pmapResidentCnt(r28)		; Set the mapped page count
+			cmplw	r12,r4					; if pmap->stats.resident_max >= pmap->stats.resident_count
+			bge+	hamSkipMax				;	goto hamSkipResMax
+			stw	r4,pmapResidentMax(r28)			; pmap->stats.resident_max = pmap->stats.resident_count
 			
-			andi.	r0,r24,mpType				; Is this a normal mapping?
+hamSkipMax:		andi.	r0,r24,mpType				; Is this a normal mapping?
 
 			rlwimi	r8,r8,14,4,17				; Double address space
 			rlwinm	r9,r30,0,4,31				; Clear segment
@@ -274,10 +287,9 @@ hamGotX:	mr		r3,r28						; Get the pmap to insert into
 			rlwimi	r8,r8,28,0,3				; Get the last nybble of the hash
 			rlwimi	r10,r29,18,0,13				; Shift EA[18:31] down to VSID (31-bit math works because of max hash table size)			
 			rlwinm	r7,r7,0,16,31				; Isolate length mask (or count)
-			addi	r4,r4,1						; Bump up the mapped page count
 			srw		r9,r9,r11					; Isolate just the page index
 			xor		r10,r10,r8					; Calculate the low 32 bits of the VSID
-			stw		r4,pmapResidentCnt(r28)		; Set the mapped page count 
+
 			xor		r9,r9,r10					; Get the hash to the PTEG
 			
 			bne--	hamDoneNP					; Not a normal mapping, therefore, no physent...
@@ -5458,7 +5470,7 @@ isInv1:		lwz		r4,pmapCCtl(r3)				; Get the segment cache control
 ;				We are in the exception vectors
 ;				pf64Bitb is set up
 ;				R3 contains the MSR we going to
-;				We can not use R4, R13, R20, R21, R29
+;				We can not use R4, R13, R20, R21, R25, R26, R29
 ;				R13 is the savearea
 ;				R29 has the per_proc
 ;
diff --git a/osfmk/ppc/instrumentation.h b/osfmk/ppc/instrumentation.h
index 3ed622055..536d8aa59 100644
--- a/osfmk/ppc/instrumentation.h
+++ b/osfmk/ppc/instrumentation.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @APPLE_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/interrupt.c b/osfmk/ppc/interrupt.c
index b7bd02bbd..5ae0bc8f7 100644
--- a/osfmk/ppc/interrupt.c
+++ b/osfmk/ppc/interrupt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -25,6 +31,7 @@
 /*
  * @APPLE_FREE_COPYRIGHT@
  */
+
 #include <kern/misc_protos.h>
 #include <kern/assert.h>
 #include <kern/thread.h>
@@ -36,10 +43,28 @@
 #include <ppc/proc_reg.h>
 #include <ppc/exception.h>
 #include <ppc/savearea.h>
+#include <ppc/vmachmon.h>
+#include <ppc/machine_cpu.h>
 #include <pexpert/pexpert.h>
 #include <sys/kdebug.h>
 
-perfCallback perfIntHook = 0;						/* Pointer to CHUD trap hook routine */
+perfCallback perfIntHook;						/* Pointer to CHUD trap hook routine */
+
+#if CONFIG_DTRACE
+#if (DEVELOPMENT || DEBUG )
+#include <mach/sdt.h>
+#endif
+
+extern vm_offset_t dtrace_get_cpu_int_stack_top(void);
+
+vm_offset_t dtrace_get_cpu_int_stack_top(void)
+{
+	return getPerProc()->intstack_top_ss;
+}
+
+/* See <rdar://problem/4613924> */
+perfCallback tempDTraceIntHook; /* Pointer to DTrace fbt int handler */
+#endif
 
 void unresolved_kernel_trap(int trapno,
 				   struct savearea *ssp,
@@ -47,6 +72,8 @@ void unresolved_kernel_trap(int trapno,
 				   addr64_t dar,
 				   const char *message);
 
+unsigned int isync_mfdec(void);
+
 struct savearea * interrupt(
         int type,
         struct savearea *ssp,
@@ -64,6 +91,12 @@ struct savearea * interrupt(
 		if(perfIntHook(type, ssp, dsisr, dar) == KERN_SUCCESS) return ssp;	/* If it succeeds, we are done... */
 	}
 	
+#if CONFIG_DTRACE
+	if(tempDTraceIntHook) {							/* Is there a hook? */
+		if(tempDTraceIntHook(type, ssp, dsisr, dar) == KERN_SUCCESS) return ssp;	/* If it succeeds, we are done... */
+	}
+#endif
+
 #if 0
 	{
 		extern void fctx_text(void);
@@ -81,13 +114,6 @@ struct savearea * interrupt(
 			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE,
 				  isync_mfdec(), (unsigned int)ssp->save_srr0, 0, 0, 0);
 	
-#if 0
-			if (pcsample_enable) {
-				if (find_user_regs(current_thread()))
-				  add_pcsamples (user_pc(current_thread()));
-			}
-#endif
-
 			now = mach_absolute_time();				/* Find out what time it is */
 			
 			if(now >= proc_info->pms.pmsPop) {		/* Is it time for power management state change? */
@@ -110,17 +136,29 @@ struct savearea * interrupt(
 	
 		case T_INTERRUPT:
 			/* Call the platform interrupt routine */
-			counter_always(c_incoming_interrupts++);
+			counter(c_incoming_interrupts++);
 	
 			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
 			   current_cpu, (unsigned int)ssp->save_srr0, 0, 0, 0);
 	
+#if CONFIG_DTRACE && (DEVELOPMENT || DEBUG )
+			DTRACE_INT5(interrupt__start, void *, proc_info->interrupt_nub, int, proc_info->interrupt_source, 
+						void *, proc_info->interrupt_target, IOInterruptHandler, proc_info->interrupt_handler,
+						void *, proc_info->interrupt_refCon);
+#endif
+
 			proc_info->interrupt_handler(
 				proc_info->interrupt_target, 
 				proc_info->interrupt_refCon,
 				proc_info->interrupt_nub, 
 				proc_info->interrupt_source);
 	
+#if CONFIG_DTRACE && (DEVELOPMENT || DEBUG )
+			DTRACE_INT5(interrupt__complete, void *, proc_info->interrupt_nub, int, proc_info->interrupt_source, 
+						void *, proc_info->interrupt_target, IOInterruptHandler, proc_info->interrupt_handler,
+						void *, proc_info->interrupt_refCon);
+#endif
+
 			KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
 			   0, 0, 0, 0, 0);
 	
diff --git a/osfmk/ppc/io_map.c b/osfmk/ppc/io_map.c
index 373c3965f..25607c1cc 100644
--- a/osfmk/ppc/io_map.c
+++ b/osfmk/ppc/io_map.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -49,8 +55,8 @@ vm_offset_t
 io_map(vm_offset_t phys_addr, vm_size_t size, unsigned int flags)
 {
 	vm_offset_t	start;
-	int		i;
-	unsigned int    j, mflags;
+	vm_size_t	i;
+	unsigned int    mflags;
 	vm_page_t 	m;
 
 	mflags = mmFlgBlock | mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);	/* Convert to our mapping_make flags */
diff --git a/osfmk/ppc/io_map_entries.h b/osfmk/ppc/io_map_entries.h
index 45b25db4e..eb0adf48a 100644
--- a/osfmk/ppc/io_map_entries.h
+++ b/osfmk/ppc/io_map_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/lock.h b/osfmk/ppc/lock.h
index dde4dbbcc..376ff4991 100644
--- a/osfmk/ppc/lock.h
+++ b/osfmk/ppc/lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * Copyright (C) 1998 Apple Computer
diff --git a/osfmk/ppc/locks.h b/osfmk/ppc/locks.h
index d8fa8a14a..a13399a0b 100644
--- a/osfmk/ppc/locks.h
+++ b/osfmk/ppc/locks.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef	_PPC_LOCKS_H_
@@ -107,10 +113,11 @@ typedef struct {
 } lck_mtx_stat_t;
 
 typedef struct _lck_mtx_ext_ {
-	lck_mtx_t			lck_mtx;
+	lck_mtx_t		lck_mtx;
 	struct _lck_grp_	*lck_mtx_grp;
 	unsigned int		lck_mtx_attr;
 	lck_mtx_deb_t		lck_mtx_deb;
+	/* Unused on PowerPC */
 	lck_mtx_stat_t		lck_mtx_stat;
 } lck_mtx_ext_t;
 
@@ -176,7 +183,7 @@ typedef struct {
 } lck_rw_stat_t;
 
 typedef struct _lck_rw_ext_ {
-	lck_rw_t			lck_rw;
+	lck_rw_t		lck_rw;
 	struct _lck_grp_	*lck_rw_grp;
 	unsigned int		lck_rw_attr;
 	lck_rw_deb_t		lck_rw_deb;
diff --git a/osfmk/ppc/locks_ppc.c b/osfmk/ppc/locks_ppc.c
index 2fad8c38d..c747215f2 100644
--- a/osfmk/ppc/locks_ppc.c
+++ b/osfmk/ppc/locks_ppc.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -82,6 +88,19 @@
 
 #include <sys/kdebug.h>
 
+/*
+ * We need only enough declarations from the BSD-side to be able to
+ * test if our probe is active, and to call __dtrace_probe().  Setting
+ * NEED_DTRACE_DEFS gets a local copy of those definitions pulled in.
+ *
+ * Note that if CONFIG_DTRACE is off, the include file below stubs out
+ * the code hooks here.
+ */
+#if	CONFIG_DTRACE
+#define NEED_DTRACE_DEFS
+#include <../bsd/sys/lockstat.h>
+#endif
+
 #define	LCK_RW_LCK_EXCLUSIVE_CODE	0x100
 #define	LCK_RW_LCK_EXCLUSIVE1_CODE	0x101
 #define	LCK_RW_LCK_SHARED_CODE		0x102
@@ -145,7 +164,7 @@ void		usld_lock_post(usimple_lock_t, pc_t);
 void		usld_unlock(usimple_lock_t, pc_t);
 void		usld_lock_try_pre(usimple_lock_t, pc_t);
 void		usld_lock_try_post(usimple_lock_t, pc_t);
-int		usld_lock_common_checks(usimple_lock_t, char *);
+int		usld_lock_common_checks(usimple_lock_t, const char *);
 #else	/* USLOCK_DEBUG */
 #define	USLDBG(stmt)
 #endif	/* USLOCK_DEBUG */
@@ -235,17 +254,13 @@ usimple_lock(
 	usimple_lock_t	l)
 {
 #ifndef	MACHINE_SIMPLE_LOCK
-	int i;
 	pc_t		pc;
-#if	USLOCK_DEBUG
-	int		count = 0;
-#endif 	/* USLOCK_DEBUG */
 
 	OBTAIN_PC(pc, l);
 	USLDBG(usld_lock_pre(l, pc));
 
 	if(!hw_lock_to(&l->interlock, LockTimeOut))	/* Try to get the lock with a timeout */ 
-		panic("simple lock deadlock detection - l=0x%08X, cpu=%d, ret=0x%08X", l, cpu_number(), pc);
+		panic("simple lock deadlock detection - l=%p, cpu=%d, ret=%p", l, cpu_number(), pc);
 
 	USLDBG(usld_lock_post(l, pc));
 #else
@@ -300,9 +315,9 @@ usimple_lock_try(
 
 	OBTAIN_PC(pc, l);
 	USLDBG(usld_lock_try_pre(l, pc));
-	if (success = hw_lock_try(&l->interlock)) {
+	success = hw_lock_try(&l->interlock);
+	if (success)
 		USLDBG(usld_lock_try_post(l, pc));
-	}
 	return success;
 #else
 	return(simple_lock_try((simple_lock_t)l));
@@ -334,7 +349,7 @@ void	usl_trace(usimple_lock_t, int, pc_t, const char *);
 void
 usld_lock_init(
 	usimple_lock_t	l,
-	unsigned short	tag)
+	__unused unsigned short	tag)
 {
 	if (l == USIMPLE_LOCK_NULL)
 		panic("lock initialization:  null lock pointer");
@@ -355,9 +370,7 @@ usld_lock_init(
  *	those with USLOCK_CHECKED turned on.
  */
 int
-usld_lock_common_checks(
-	usimple_lock_t	l,
-	char		*caller)
+usld_lock_common_checks(usimple_lock_t l, const char *caller)
 {
 	if (l == USIMPLE_LOCK_NULL)
 		panic("%s:  null lock pointer", caller);
@@ -380,8 +393,7 @@ usld_lock_pre(
 	usimple_lock_t	l,
 	pc_t		pc)
 {
-	char		*caller = "usimple_lock";
-
+	const char *caller = "usimple_lock";
 
 	if (!usld_lock_common_checks(l, caller))
 		return;
@@ -396,11 +408,11 @@ usld_lock_pre(
 
 	if ((l->debug.state & USLOCK_TAKEN) && l->debug.lock_thread &&
 	    l->debug.lock_thread == (void *) current_thread()) {
-		printf("%s:  lock 0x%x already locked (at 0x%x) by",
+		printf("%s:  lock 0x%x already locked (at %p) by",
 		      caller, (integer_t) l, l->debug.lock_pc);
-		printf(" current thread 0x%x (new attempt at pc 0x%x)\n",
+		printf(" current thread %p (new attempt at pc %p)\n",
 		       l->debug.lock_thread, pc);
-		panic(caller);
+		panic("%s", caller);
 	}
 	mp_disable_preemption();
 	usl_trace(l, cpu_number(), pc, caller);
@@ -419,8 +431,8 @@ usld_lock_post(
 	usimple_lock_t	l,
 	pc_t		pc)
 {
-	register int	mycpu;
-	char		*caller = "successful usimple_lock";
+	int mycpu;
+	const char *caller = "successful usimple_lock";
 
 
 	if (!usld_lock_common_checks(l, caller))
@@ -456,8 +468,8 @@ usld_unlock(
 	usimple_lock_t	l,
 	pc_t		pc)
 {
-	register int	mycpu;
-	char		*caller = "usimple_unlock";
+	int mycpu;
+	const char *caller = "usimple_unlock";
 
 
 	if (!usld_lock_common_checks(l, caller))
@@ -469,13 +481,13 @@ usld_unlock(
 		panic("%s:  lock 0x%x hasn't been taken",
 		      caller, (integer_t) l);
 	if (l->debug.lock_thread != (void *) current_thread())
-		panic("%s:  unlocking lock 0x%x, owned by thread 0x%x",
+		panic("%s:  unlocking lock 0x%x, owned by thread %p",
 		      caller, (integer_t) l, l->debug.lock_thread);
 	if (l->debug.lock_cpu != mycpu) {
 		printf("%s:  unlocking lock 0x%x on cpu 0x%x",
 		       caller, (integer_t) l, mycpu);
 		printf(" (acquired on cpu 0x%x)\n", l->debug.lock_cpu);
-		panic(caller);
+		panic("%s", caller);
 	}
 	usl_trace(l, mycpu, pc, caller);
 
@@ -498,7 +510,7 @@ usld_lock_try_pre(
 	usimple_lock_t	l,
 	pc_t		pc)
 {
-	char		*caller = "usimple_lock_try";
+	const char *caller = "usimple_lock_try";
 
 	if (!usld_lock_common_checks(l, caller))
 		return;
@@ -521,8 +533,8 @@ usld_lock_try_post(
 	usimple_lock_t	l,
 	pc_t		pc)
 {
-	register int	mycpu;
-	char		*caller = "successful usimple_lock_try";
+	int mycpu;
+	const char *caller = "successful usimple_lock_try";
 
 	if (!usld_lock_common_checks(l, caller))
 		return;
@@ -660,6 +672,12 @@ lck_rw_check_type(
 	lck_rw_ext_t	*lck,
 	lck_rw_t	*rlck);
 
+void
+lck_rw_assert_ext(
+	lck_rw_ext_t	*lck,
+	lck_rw_t	*rlck,
+	unsigned int	type);
+
 /*
  *	Routine:	lock_alloc
  *	Function:
@@ -963,10 +981,17 @@ lck_rw_lock_exclusive_gen(
 	lck_rw_t	*lck)
 {
 	int	   i;
-	boolean_t		lock_miss = FALSE;
 	wait_result_t	res;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	lck_rw_ilk_lock(lck);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw_shared_cnt;
+#endif
 
 	/*
 	 *	Try to acquire the lck_rw_want_excl bit.
@@ -974,9 +999,13 @@ lck_rw_lock_exclusive_gen(
 	while (lck->lck_rw_want_excl) {
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_START, (int)lck, 0, 0, 0, 0);
 
-		if (!lock_miss) {
-			lock_miss = TRUE;
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = -1;
 		}
+#endif
 
 		i = lock_wait_time[1];
 		if (i != 0) {
@@ -992,6 +1021,9 @@ lck_rw_lock_exclusive_gen(
 			if (res == THREAD_WAITING) {
 				lck_rw_ilk_unlock(lck);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(lck);
 			}
 		}
@@ -1002,14 +1034,18 @@ lck_rw_lock_exclusive_gen(
 	/* Wait for readers (and upgrades) to finish */
 
 	while ((lck->lck_rw_shared_cnt != 0) || lck->lck_rw_want_upgrade) {
-		if (!lock_miss) {
-			lock_miss = TRUE;
-		}
 
 		i = lock_wait_time[1];
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_START,
 			     (int)lck, lck->lck_rw_shared_cnt, lck->lck_rw_want_upgrade, i, 0);
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (i != 0) {
 			lck_rw_ilk_unlock(lck);
@@ -1025,6 +1061,9 @@ lck_rw_lock_exclusive_gen(
 			if (res == THREAD_WAITING) {
 				lck_rw_ilk_unlock(lck);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(lck);
 			}
 		}
@@ -1033,6 +1072,32 @@ lck_rw_lock_exclusive_gen(
 	}
 
 	lck_rw_ilk_unlock(lck);
+#if	CONFIG_DTRACE
+	/*
+	 * Decide what latencies we suffered that are Dtrace events.
+	 * If we have set wait_interval, then we either spun or slept.
+	 * At least we get out from under the interlock before we record
+	 * which is the best we can do here to minimize the impact
+	 * of the tracing.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lck,
+			    mach_absolute_time() - wait_interval, 1);
+		} else {
+			/*
+			 * For the blocking case, we also record if when we blocked
+			 * it was held for read or write, and how many readers.
+			 * Notice that above we recorded this before we dropped
+			 * the interlock so the count is accurate.
+			 */
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 1,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lck, 1);
+#endif
 }
 
 
@@ -1078,6 +1143,7 @@ lck_rw_done_gen(
 
 	if (do_wakeup)
 		thread_wakeup((event_t)(((unsigned int*)lck)+((sizeof(lck_rw_t)-1)/sizeof(unsigned int))));
+	LOCKSTAT_RECORD(LS_LCK_RW_DONE_RELEASE, lck, lck_rw_type);
 	return(lck_rw_type);
 }
 
@@ -1091,8 +1157,16 @@ lck_rw_lock_shared_gen(
 {
 	int		i;
 	wait_result_t      res;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	lck_rw_ilk_lock(lck);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw_shared_cnt;
+#endif
 
 	while ((lck->lck_rw_want_excl || lck->lck_rw_want_upgrade) &&
 	        ((lck->lck_rw_shared_cnt == 0) || (lck->lck_rw_priv_excl))) {
@@ -1100,6 +1174,13 @@ lck_rw_lock_shared_gen(
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
 			     (int)lck, lck->lck_rw_want_excl, lck->lck_rw_want_upgrade, i, 0);
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (i != 0) {
 			lck_rw_ilk_unlock(lck);
@@ -1117,6 +1198,9 @@ lck_rw_lock_shared_gen(
 			if (res == THREAD_WAITING) {
 				lck_rw_ilk_unlock(lck);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(lck);
 			}
 		}
@@ -1127,6 +1211,18 @@ lck_rw_lock_shared_gen(
 	lck->lck_rw_shared_cnt++;
 
 	lck_rw_ilk_unlock(lck);
+#if	CONFIG_DTRACE
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 0,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0);
+#endif
 }
 
 
@@ -1138,7 +1234,7 @@ lck_rw_lock_shared_gen(
  *		already requested an upgrade to a write lock,
  *		no lock is held upon return.
  *
- *		Returns TRUE if the upgrade *failed*.
+ *		Returns FALSE if the upgrade *failed*.
  */
 
 boolean_t
@@ -1148,6 +1244,11 @@ lck_rw_lock_shared_to_exclusive_gen(
 	int	    i;
 	boolean_t	    do_wakeup = FALSE;
 	wait_result_t      res;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep = 0;
+#endif
 
 	lck_rw_ilk_lock(lck);
 
@@ -1175,7 +1276,7 @@ lck_rw_lock_shared_to_exclusive_gen(
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_END,
 			     (int)lck, lck->lck_rw_shared_cnt, lck->lck_rw_want_upgrade, 0, 0);
 
-		return (TRUE);
+		return (FALSE);
 	}
 
 	lck->lck_rw_want_upgrade = TRUE;
@@ -1186,6 +1287,14 @@ lck_rw_lock_shared_to_exclusive_gen(
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | DBG_FUNC_START,
 			     (int)lck, lck->lck_rw_shared_cnt, i, 0, 0);
 
+#if	CONFIG_DTRACE
+		readers_at_sleep = lck->lck_rw_shared_cnt;
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 		if (i != 0) {
 			lck_rw_ilk_unlock(lck);
 			while (--i != 0 && lck->lck_rw_shared_cnt != 0)
@@ -1199,6 +1308,9 @@ lck_rw_lock_shared_to_exclusive_gen(
 			if (res == THREAD_WAITING) {
 				lck_rw_ilk_unlock(lck);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(lck);
 			}
 		}
@@ -1208,7 +1320,24 @@ lck_rw_lock_shared_to_exclusive_gen(
 
 	lck_rw_ilk_unlock(lck);
 
-	return (FALSE);
+#if	CONFIG_DTRACE
+	/*
+	 * We infer if we took a sleep or spin path by whether readers_at_sleep
+	 * was set.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1 && readers_at_sleep) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 1,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+#endif
+
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lck, 1);
+	return (TRUE);
 }
 
 /*
@@ -1238,6 +1367,7 @@ lck_rw_lock_exclusive_to_shared_gen(
 	if (do_wakeup)
 		thread_wakeup((event_t)(((unsigned int*)lck)+((sizeof(lck_rw_t)-1)/sizeof(unsigned int))));
 
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, lck, 0);
 }
 
 
@@ -1271,6 +1401,7 @@ lck_rw_try_lock_exclusive_gen(
 
 	lck_rw_ilk_unlock(lck);
 
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lck, 1);
 	return(TRUE);
 }
 
@@ -1298,6 +1429,7 @@ lck_rw_try_lock_shared_gen(
 
 	lck_rw_ilk_unlock(lck);
 
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lck, 0);
 	return(TRUE);
 }
 
@@ -1342,14 +1474,22 @@ lck_rw_lock_exclusive_ext(
 	boolean_t		lock_miss = FALSE;
 	boolean_t		lock_wait = FALSE;
 	boolean_t		lock_stat;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	lck_rw_check_type(lck, rlck);
 
 	if ( ((lck->lck_rw_attr & (LCK_RW_ATTR_DEBUG|LCK_RW_ATTR_DIS_MYLOCK)) == LCK_RW_ATTR_DEBUG) 
 	     && (lck->lck_rw_deb.thread == current_thread()))
-		panic("rw lock (0x%08X) recursive lock attempt\n", rlck);
+		panic("rw lock (%p) recursive lock attempt\n", rlck);
 
 	lck_rw_ilk_lock(&lck->lck_rw);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw.lck_rw_shared_cnt;
+#endif
 
 	lock_stat = (lck->lck_rw_attr & LCK_RW_ATTR_STAT) ? TRUE : FALSE;
 
@@ -1366,6 +1506,13 @@ lck_rw_lock_exclusive_ext(
 			lock_miss = TRUE;
 			lck->lck_rw_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt++;
 		}
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		i = lock_wait_time[1];
 		if (i != 0) {
@@ -1385,6 +1532,9 @@ lck_rw_lock_exclusive_ext(
 				}
 				lck_rw_ilk_unlock(&lck->lck_rw);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(&lck->lck_rw);
 			}
 		}
@@ -1399,6 +1549,13 @@ lck_rw_lock_exclusive_ext(
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_START,
 			     (int)rlck, lck->lck_rw.lck_rw_shared_cnt, lck->lck_rw.lck_rw_want_upgrade, i, 0);
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (lock_stat && !lock_miss) {
 			lock_miss = TRUE;
@@ -1423,6 +1580,9 @@ lck_rw_lock_exclusive_ext(
 				}
 				lck_rw_ilk_unlock(&lck->lck_rw);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(&lck->lck_rw);
 			}
 		}
@@ -1436,6 +1596,32 @@ lck_rw_lock_exclusive_ext(
 	lck->lck_rw_deb.thread = current_thread();
 
 	lck_rw_ilk_unlock(&lck->lck_rw);
+#if	CONFIG_DTRACE
+	/*
+	 * Decide what latencies we suffered that are Dtrace events.
+	 * If we have set wait_interval, then we either spun or slept.
+	 * At least we get out from under the interlock before we record
+	 * which is the best we can do here to minimize the impact
+	 * of the tracing.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lck,
+			    mach_absolute_time() - wait_interval, 1);
+		} else {
+			/*
+			 * For the blocking case, we also record if when we blocked
+			 * it was held for read or write, and how many readers.
+			 * Notice that above we recorded this before we dropped
+			 * the interlock so the count is accurate.
+			 */
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 1,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lck, 1);
+#endif
 }
 
 
@@ -1466,14 +1652,14 @@ lck_rw_done_ext(
 		else if (lck->lck_rw.lck_rw_want_excl)
 			lck->lck_rw.lck_rw_want_excl = FALSE;
 		else
-			panic("rw lock (0x%08X) bad state (0x%08X) on attempt to release a shared or exlusive right\n",
-				  rlck, lck->lck_rw);
+			panic("rw lock (%p) bad state (0x%08X) on attempt to release a shared or exlusive right\n",
+				  rlck, lck->lck_rw.lck_rw_tag);
 		if (lck->lck_rw_deb.thread == THREAD_NULL)
-			panic("rw lock (0x%08X) not held\n",
+			panic("rw lock (%p) not held\n",
 			      rlck);
 		else if ( ((lck->lck_rw_attr & (LCK_RW_ATTR_DEBUG|LCK_RW_ATTR_DIS_THREAD)) == LCK_RW_ATTR_DEBUG) 
 			 && (lck->lck_rw_deb.thread != current_thread()))
-			panic("rw lock (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n",
+			panic("rw lock (%p) unlocked by non-owner(%p), current owner(%p)\n",
 				  rlck, current_thread(), lck->lck_rw_deb.thread);
 		lck->lck_rw_deb.thread = THREAD_NULL;
 	}
@@ -1498,6 +1684,7 @@ lck_rw_done_ext(
 
 	if (do_wakeup)
 		thread_wakeup((event_t)(((unsigned int*)rlck)+((sizeof(lck_rw_t)-1)/sizeof(unsigned int))));
+	LOCKSTAT_RECORD(LS_LCK_RW_DONE_RELEASE, lck, lck_rw_type);
 	return(lck_rw_type);
 }
 
@@ -1515,10 +1702,18 @@ lck_rw_lock_shared_ext(
 	boolean_t		lock_miss = FALSE;
 	boolean_t		lock_wait = FALSE;
 	boolean_t		lock_stat;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+	int readers_at_sleep;
+#endif
 
 	lck_rw_check_type(lck, rlck);
 
 	lck_rw_ilk_lock(&lck->lck_rw);
+#if	CONFIG_DTRACE
+	readers_at_sleep = lck->lck_rw.lck_rw_shared_cnt;
+#endif
 
 	lock_stat = (lck->lck_rw_attr & LCK_RW_ATTR_STAT) ? TRUE : FALSE;
 
@@ -1531,6 +1726,13 @@ lck_rw_lock_shared_ext(
 
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
 			     (int)rlck, lck->lck_rw.lck_rw_want_excl, lck->lck_rw.lck_rw_want_upgrade, i, 0);
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (lock_stat && !lock_miss) {
 			lock_miss = TRUE;
@@ -1557,6 +1759,9 @@ lck_rw_lock_shared_ext(
 				}
 				lck_rw_ilk_unlock(&lck->lck_rw);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(&lck->lck_rw);
 			}
 		}
@@ -1567,6 +1772,18 @@ lck_rw_lock_shared_ext(
 	lck->lck_rw.lck_rw_shared_cnt++;
 
 	lck_rw_ilk_unlock(&lck->lck_rw);
+#if	CONFIG_DTRACE
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck,
+			    mach_absolute_time() - wait_interval, 0,
+			    (readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
+		}
+	}
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0);
+#endif
 }
 
 
@@ -1578,7 +1795,7 @@ lck_rw_lock_shared_ext(
  *		already requested an upgrade to a write lock,
  *		no lock is held upon return.
  *
- *		Returns TRUE if the upgrade *failed*.
+ *		Returns FALSE if the upgrade *failed*.
  */
 
 boolean_t
@@ -1592,11 +1809,15 @@ lck_rw_lock_shared_to_exclusive_ext(
 	boolean_t		lock_miss = FALSE;
 	boolean_t		lock_wait = FALSE;
 	boolean_t		lock_stat;
+#if	CONFIG_DTRACE
+	uint64_t wait_interval = 0;
+	int slept = 0;
+#endif
 
 	lck_rw_check_type(lck, rlck);
 
 	if (lck->lck_rw_deb.thread == current_thread())
-		panic("rw lock (0x%08X) recursive lock attempt\n", rlck);
+		panic("rw lock (%p) recursive lock attempt\n", rlck);
 
 	lck_rw_ilk_lock(&lck->lck_rw);
 
@@ -1629,7 +1850,7 @@ lck_rw_lock_shared_to_exclusive_ext(
 		KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_END,
 			     (int)rlck, lck->lck_rw.lck_rw_shared_cnt, lck->lck_rw.lck_rw_want_upgrade, 0, 0);
 
-		return (TRUE);
+		return (FALSE);
 	}
 
 	lck->lck_rw.lck_rw_want_upgrade = TRUE;
@@ -1644,6 +1865,13 @@ lck_rw_lock_shared_to_exclusive_ext(
 			lock_miss = TRUE;
 			lck->lck_rw_grp->lck_grp_stat.lck_grp_rw_stat.lck_grp_rw_miss_cnt++;
 		}
+#if	CONFIG_DTRACE
+		if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK]) && wait_interval == 0) {
+			wait_interval = mach_absolute_time();
+		} else {
+			wait_interval = (unsigned) -1;
+		}
+#endif
 
 		if (i != 0) {
 			lck_rw_ilk_unlock(&lck->lck_rw);
@@ -1662,6 +1890,9 @@ lck_rw_lock_shared_to_exclusive_ext(
 				}
 				lck_rw_ilk_unlock(&lck->lck_rw);
 				res = thread_block(THREAD_CONTINUE_NULL);
+#if	CONFIG_DTRACE
+				slept = 1;
+#endif
 				lck_rw_ilk_lock(&lck->lck_rw);
 			}
 		}
@@ -1676,7 +1907,23 @@ lck_rw_lock_shared_to_exclusive_ext(
 
 	lck_rw_ilk_unlock(&lck->lck_rw);
 
-	return (FALSE);
+#if	CONFIG_DTRACE
+	/*
+	 * If we've travelled a path with no spin or sleep, then wait_interval
+	 * is still zero.
+	 */
+	if (wait_interval != 0 && wait_interval != (unsigned) -1) {
+		if (slept == 0) {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lck, mach_absolute_time() - wait_interval, 0);
+		} else {
+			LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lck, mach_absolute_time() - wait_interval, 0);
+		}
+	}
+#endif
+
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lck, 1);
+
+	return (TRUE);
 }
 
 /*
@@ -1702,14 +1949,14 @@ lck_rw_lock_exclusive_to_shared_ext(
 	else if (lck->lck_rw.lck_rw_want_excl)
 	 	lck->lck_rw.lck_rw_want_excl = FALSE;
 	else
-		panic("rw lock (0x%08X) bad state (0x%08X) on attempt to release a shared or exlusive right\n",
-			  rlck, lck->lck_rw);
+		panic("rw lock (%p) bad state (0x%08X) on attempt to release a shared or exlusive right\n",
+			  rlck, lck->lck_rw.lck_rw_tag);
 	if (lck->lck_rw_deb.thread == THREAD_NULL)
-		panic("rw lock (0x%08X) not held\n",
+		panic("rw lock (%p) not held\n",
 		      rlck);
 	else if ( ((lck->lck_rw_attr & (LCK_RW_ATTR_DEBUG|LCK_RW_ATTR_DIS_THREAD)) == LCK_RW_ATTR_DEBUG) 
 		  && (lck->lck_rw_deb.thread != current_thread()))
-		panic("rw lock (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n",
+		panic("rw lock (%p) unlocked by non-owner(%p), current owner(%p)\n",
 			  rlck, current_thread(), lck->lck_rw_deb.thread);
 
 	lck->lck_rw_deb.thread = THREAD_NULL;
@@ -1727,6 +1974,7 @@ lck_rw_lock_exclusive_to_shared_ext(
 	KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EX_TO_SH_CODE) | DBG_FUNC_END,
 			     (int)rlck, lck->lck_rw.lck_rw_want_excl, lck->lck_rw.lck_rw_want_upgrade, lck->lck_rw.lck_rw_shared_cnt, 0);
 
+	LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE, lck, 0);
 }
 
 
@@ -1777,6 +2025,8 @@ lck_rw_try_lock_exclusive_ext(
 
 	lck_rw_ilk_unlock(&lck->lck_rw);
 
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE, lck, 1);
+
 	return(TRUE);
 }
 
@@ -1817,6 +2067,8 @@ lck_rw_try_lock_shared_ext(
 
 	lck_rw_ilk_unlock(&lck->lck_rw);
 
+	LOCKSTAT_RECORD(LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE, lck, 0);
+
 	return(TRUE);
 }
 
@@ -1826,7 +2078,79 @@ lck_rw_check_type(
 	lck_rw_t		*rlck)
 {
 	if (lck->lck_rw_deb.type != RW_TAG)
-		panic("rw lock (0x%08X) not a rw lock type (0x%08X)\n",rlck, lck->lck_rw_deb.type);
+		panic("rw lock (%p) not a rw lock type (0x%08X)\n",rlck, lck->lck_rw_deb.type);
+}
+
+void
+lck_rw_assert_ext(
+	lck_rw_ext_t	*lck,
+	lck_rw_t	*rlck,
+	unsigned int	type)
+{
+	lck_rw_check_type(lck, rlck);
+
+	switch (type) {
+	case LCK_RW_ASSERT_SHARED:
+		if (lck->lck_rw.lck_rw_shared_cnt != 0) {
+			return;
+		}
+		break;
+	case LCK_RW_ASSERT_EXCLUSIVE:
+		if ((lck->lck_rw.lck_rw_want_excl ||
+		     lck->lck_rw.lck_rw_want_upgrade) &&
+		    lck->lck_rw.lck_rw_shared_cnt == 0) {
+			return;
+		}
+		break;
+	case LCK_RW_ASSERT_HELD:
+		if (lck->lck_rw.lck_rw_want_excl ||
+		    lck->lck_rw.lck_rw_want_upgrade ||
+		    lck->lck_rw.lck_rw_shared_cnt != 0) {
+			return;
+		}
+		break;
+	default:
+		break;
+	}
+
+	panic("rw lock (%p -> %p) not held (mode=%u)\n", rlck, lck, type);
+}
+
+void
+lck_rw_assert(
+	lck_rw_t	*lck,
+	unsigned int	type)
+{
+	if (lck->lck_rw_tag != LCK_RW_TAG_INDIRECT) {
+		switch (type) {
+		case LCK_RW_ASSERT_SHARED:
+			if (lck->lck_rw_shared_cnt != 0) {
+				return;
+			}
+			break;
+		case LCK_RW_ASSERT_EXCLUSIVE:
+			if (lck->lck_rw_shared_cnt == 0 &&
+			    (lck->lck_rw_want_excl ||
+			     lck->lck_rw_want_upgrade)) {
+				return;
+			}
+			break;
+		case LCK_RW_ASSERT_HELD:
+			if (lck->lck_rw_shared_cnt != 0 ||
+			    lck->lck_rw_want_excl ||
+			    lck->lck_rw_want_upgrade) {
+				return;
+			}
+			break;
+		default:
+			break;
+		}
+		panic("rw lock (%p) not held (mode=%u)\n", lck, type);
+	} else {
+		lck_rw_assert_ext((lck_rw_ext_t *)lck->lck_rw_ptr,
+				  (lck_rw_t *)lck,
+				  type);
+	}
 }
 
 /*
@@ -1930,6 +2254,36 @@ lck_mtx_init(
 	lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX);
 }
 
+/*
+ *      Routine:        lck_mtx_init_ext
+ */
+void
+lck_mtx_init_ext(
+	lck_mtx_t	*lck,
+	lck_mtx_ext_t	*lck_ext,
+	lck_grp_t	*grp,
+	lck_attr_t	*attr)
+{
+	lck_attr_t	*lck_attr;
+
+	if (attr != LCK_ATTR_NULL)
+		lck_attr = attr;
+	else
+		lck_attr = &LockDefaultLckAttr;
+
+	if ((lck_attr->lck_attr_val) & LCK_ATTR_DEBUG) {
+		lck_mtx_ext_init(lck_ext, grp, lck_attr);
+		lck->lck_mtx_tag = LCK_MTX_TAG_INDIRECT;
+		lck->lck_mtx_ptr = lck_ext;
+	} else {
+		lck->lck_mtx_data = 0;
+		lck->lck_mtx_waiters = 0;
+		lck->lck_mtx_pri = 0;
+	}
+	lck_grp_reference(grp);
+	lck_grp_lckcnt_incr(grp, LCK_TYPE_MTX);
+}
+
 /*
  *      Routine:        lck_mtx_ext_init
  */
@@ -1980,8 +2334,8 @@ lck_mtx_destroy(
  * fashion.
  */
 
-char *simple_lock_labels =	"ENTRY    ILK THREAD   DURATION CALLER";
-char *mutex_labels =		"ENTRY    LOCKED WAITERS   THREAD CALLER";
+const char *simple_lock_labels = "ENTRY    ILK THREAD   DURATION CALLER";
+const char *mutex_labels = "ENTRY    LOCKED WAITERS   THREAD CALLER";
 
 void	db_print_simple_lock(
 			simple_lock_t	addr);
@@ -1990,13 +2344,11 @@ void	db_print_mutex(
 			mutex_t		* addr);
 
 void
-db_show_one_simple_lock (
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		* modif)
+db_show_one_simple_lock (db_expr_t addr, boolean_t have_addr,
+			 __unused db_expr_t count,
+			 __unused char *modif)
 {
-	simple_lock_t	saddr = (simple_lock_t)addr;
+	simple_lock_t	saddr = (simple_lock_t)(unsigned long)addr;
 
 	if (saddr == (simple_lock_t)0 || !have_addr) {
 		db_error ("No simple_lock\n");
@@ -2025,13 +2377,11 @@ db_print_simple_lock (
 }
 
 void
-db_show_one_mutex (
-	db_expr_t	addr,
-	boolean_t	have_addr,
-	db_expr_t	count,
-	char		* modif)
+db_show_one_mutex (db_expr_t addr, boolean_t have_addr,
+		   __unused db_expr_t count,
+		   __unused char *modif)
 {
-	mutex_t		* maddr = (mutex_t *)addr;
+	mutex_t		* maddr = (mutex_t *)(unsigned long)addr;
 
 	if (maddr == (mutex_t *)0 || !have_addr)
 		db_error ("No mutex\n");
diff --git a/osfmk/ppc/low_trace.h b/osfmk/ppc/low_trace.h
index 1fd7b0bdd..e5a81d7a9 100644
--- a/osfmk/ppc/low_trace.h
+++ b/osfmk/ppc/low_trace.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/lowglobals.h b/osfmk/ppc/lowglobals.h
index 861d9034a..1a18cd1c5 100644
--- a/osfmk/ppc/lowglobals.h
+++ b/osfmk/ppc/lowglobals.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *		Header files for the Low Memory Globals (lg) 
@@ -85,8 +91,9 @@ typedef struct lowglo {
 	uint32_t		lgReadIO;		/* 0x5488 Pointer to kdp_read_io, debugging aid */
 	uint32_t		lgDevSlot1;		/* 0x548C For developer use */
 	uint32_t		lgDevSlot2;		/* 0x5490 For developer use */
-	uint32_t		lgRsv494[731];		/* 0x5494 reserved - push to 1 page */
-
+	uint32_t		lgOSVersion;		/* 0x5494 Pointer to OS version string */
+	uint32_t		lgRebootFlag;		/* 0x5498 Pointer to debugger reboot trigger */
+	uint32_t		lgRsv49C[729];		/* 0x549C Reserved - push to 1 page */
 } lowglo;
 
 extern lowglo lowGlo;
diff --git a/osfmk/ppc/lowmem_vectors.s b/osfmk/ppc/lowmem_vectors.s
index 3ee46a474..4b6b06c04 100644
--- a/osfmk/ppc/lowmem_vectors.s
+++ b/osfmk/ppc/lowmem_vectors.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -90,7 +96,7 @@ EXT(ResetHandler):
 			mtlr	r4
 			blr
 
-resetexc:	cmplwi	r13,RESET_HANDLER_BUPOR				; Special bring up POR sequence?
+resetexc:	cmplwi	r13,RESET_HANDLER_BUPOR			; Special bring up POR sequence?
 			bne		resetexc2						; No...
 			lis		r4,hi16(EXT(resetPOR))			; Get POR code
 			ori		r4,r4,lo16(EXT(resetPOR))		; The rest
@@ -130,7 +136,7 @@ rxCont:		mtcr	r11
 			li		r11,RESET_HANDLER_IGNORE		; Get set to ignore
 			stw		r11,lo16(EXT(ResetHandler)-EXT(ExceptionVectorsStart)+RESETHANDLER_TYPE)(br0)	; Start ignoring these
 			mfsprg	r13,1							/* Get the exception save area */
-			li		r11,T_RESET						/* Set interrupt code */
+			li		r11,T_RESET						/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -223,7 +229,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handler300:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_DATA_ACCESS				/* Set interrupt code */
+			li		r11,T_DATA_ACCESS				/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -279,7 +285,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handler600:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_ALIGNMENT|T_FAM			/* Set interrupt code */
+			li		r11,T_ALIGNMENT|T_FAM			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -288,23 +294,10 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 
 			. = 0x700
 .L_handler700:
-			mtsprg	2,r13							/* Save R13 */
-			mtsprg	3,r11							/* Save R11 */
-			
-#if 0
-			mfsrr1	r13								; (BRINGUP)
-			mfcr	r11								; (BRINGUP)
-			rlwinm.	r13,r13,0,12,12					; (BRINGUP)	
-			crmove	cr1_eq,cr0_eq					; (BRINGUP)
-			mfsrr1	r13								; (BRINGUP)
-			rlwinm.	r13,r13,0,MSR_PR_BIT,MSR_PR_BIT	; (BRINGUP)	
-			crorc	cr0_eq,cr1_eq,cr0_eq			; (BRINGUP)
-			bf--	cr0_eq,.						; (BRINGUP)
-			mtcrf	255,r11							; (BRINGUP)
-#endif
-		
-			li		r11,T_PROGRAM|T_FAM				/* Set interrupt code */
-			b		.L_exception_entry				/* Join common... */
+			mtsprg	2,r13							; Save R13
+			mtsprg	3,r11							; Save R11			
+			li		r11,T_PROGRAM|T_FAM				; Set program interruption code
+			b		.L_exception_entry				; Join common...
 
 /*
  * 			Floating point disabled
@@ -314,7 +307,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handler800:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_FP_UNAVAILABLE			/* Set interrupt code */
+			li		r11,T_FP_UNAVAILABLE			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -326,7 +319,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handler900:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_DECREMENTER				/* Set interrupt code */
+			li		r11,T_DECREMENTER				/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -337,7 +330,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handlerA00:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_IO_ERROR					/* Set interrupt code */
+			li		r11,T_IO_ERROR					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -348,7 +341,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handlerB00:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_RESERVED					/* Set interrupt code */
+			li		r11,T_RESERVED					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -369,26 +362,26 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 .L_handlerC00:
 			mtsprg	3,r11							; Save R11
 			mtsprg	2,r13							; Save R13
-            rlwinm  r11,r0,0,0xFFFFFFF8             ; mask off low 3 bits of syscall number
-            xori    r13,r11,0x7FF0                  ; start to check for the 0x7FFx traps
-            addi    r11,r11,8                       ; make a 0 iff this is a 0xFFFFFFF8 trap
-            cntlzw  r13,r13                         ; set bit 0x20 iff a 0x7FFx trap
-            cntlzw  r11,r11                         ; set bit 0x20 iff a 0xFFFFFFF8 trap
+			rlwinm	r11,r0,0,0xFFFFFFF8				; mask off low 3 bits of syscall number
+			xori	r13,r11,0x7FF0					; start to check for the 0x7FFx traps
+			addi	r11,r11,8						; make a 0 iff this is a 0xFFFFFFF8 trap
+			cntlzw	r13,r13							; set bit 0x20 iff a 0x7FFx trap
+			cntlzw	r11,r11							; set bit 0x20 iff a 0xFFFFFFF8 trap
 			xoris	r0,r0,0x8000					; Flip bit to make 0 iff 0x80000000
-            rlwimi  r11,r13,31,0x10                 ; move 0x7FFx bit into position
+			rlwimi	r11,r13,31,0x10					; move 0x7FFx bit into position
 			cntlzw	r13,r0							; Set bit 0x20 iff 0x80000000
 			xoris	r0,r0,0x8000					; Flip bit to restore R0
 			rlwimi	r11,r13,2,0x80					; Set bit 0x80 iff CutTrace
-            xori    r13,r0,0x6004                   ; start to check for 0x6004
-            rlwimi  r11,r0,1,0xE                    ; move in low 3 bits of syscall number
-            cntlzw  r13,r13                         ; set bit 0x20 iff 0x6004
-            rlwinm  r11,r11,0,0,30                  ; clear out bit 31
-            rlwimi  r11,r13,1,0x40                  ; move 0x6004 bit into position
-            lhz     r11,lo16(scTable)(r11)          ; get branch address from sc table
-            mfctr   r13                             ; save callers ctr in r13
-            mtctr   r11                             ; set up branch to syscall handler
-            mfsprg  r11,0                           ; get per_proc, which most UFTs use
-            bctr                                    ; dispatch (r11 in sprg3, r13 in sprg2, ctr in r13, per_proc in r11)
+			xori	r13,r0,0x6004					; start to check for 0x6004
+			rlwimi	r11,r0,1,0xE					; move in low 3 bits of syscall number
+			cntlzw	r13,r13							; set bit 0x20 iff 0x6004
+			rlwinm	r11,r11,0,0,30					; clear out bit 31
+			rlwimi	r11,r13,1,0x40					; move 0x6004 bit into position
+			lhz		r11,lo16(scTable)(r11)			; get branch address from sc table
+			mfctr	r13								; save callers ctr in r13
+			mtctr	r11								; set up branch to syscall handler
+			mfsprg	r11,0							; get per_proc, which most UFTs use
+			bctr									; dispatch (r11 in sprg3, r13 in sprg2, ctr in r13, per_proc in r11)
 
 /*
  * 			Trace - generated by single stepping
@@ -398,9 +391,7 @@ notDCache:	mtcrf	255,r13							; Restore CRs
  *				only executed when (a) a single step or branch exception is
  *				hit, (b) in the single step debugger case there is so much
  *				overhead already the few extra instructions for testing for BE
- *				are not even noticable, (c) the BE logging code is *only* run
- *				when it is enabled by the tool which will not happen during
- *				normal system usage
+ *				are not even noticable
  *
  *			Note that this trace is available only to user state so we do not 
  *			need to set sprg2 before returning.
@@ -411,67 +402,10 @@ notDCache:	mtcrf	255,r13							; Restore CRs
 			mtsprg	3,r11							; Save R11
 			mfsprg	r11,2							; Get the feature flags
 			mtsprg	2,r13							; Save R13
-			rlwinm	r11,r11,pf64Bitb-4,4,4			; Get the 64-bit flag
-			mfcr	r13								; Get the CR
-			mtcrf	0x40,r11						; Set the CR
-			mfsrr1	r11								; Get the old MSR
-			rlwinm.	r11,r11,0,MSR_PR_BIT,MSR_PR_BIT	; Are we in supervisor state?
-			
-			mfsprg	r11,0							; Get the per_proc
-			lhz		r11,PP_CPU_FLAGS(r11)			; Get the flags
-			crmove	cr1_eq,cr0_eq					; Remember if we are in supervisor state
-			rlwinm.	r11,r11,0,traceBEb+16,traceBEb+16	; Special trace enabled?
-			cror	cr0_eq,cr0_eq,cr1_eq			; Is trace off or supervisor state?
-			bf--	cr0_eq,specbrtr					; No, we need to trace...
 
-notspectr:	mtcr	r13								; Restore CR
 			li		r11,T_TRACE|T_FAM				; Set interrupt code
 			b		.L_exception_entry				; Join common...
 
-			.align	5
-
-;
-;			We are doing the special branch trace
-;
-
-specbrtr:	mfsprg	r11,0							; Get the per_proc area
-			bt++	4,sbxx64a						; Jump if 64-bit...
-			
-			stw		r1,tempr0+4(r11)				; Save in a scratch area
-			stw		r2,tempr1+4(r11)				; Save in a scratch area
-			stw		r3,tempr2+4(r11)				; Save in a scratch area
-			b		sbxx64b							; Skip...
-			
-sbxx64a:	std		r1,tempr0(r11)					; Save in a scratch area
-			std		r2,tempr1(r11)					; Save in a scratch area
-			std		r3,tempr2(r11)					; Save in a scratch area
-			
-sbxx64b:	lis		r2,hi16(EXT(pc_trace_buf))		; Get the top of the buffer
-			lwz		r3,spcTRp(r11)					; Pick up buffer position			
-			ori		r2,r2,lo16(EXT(pc_trace_buf))	; Get the bottom of the buffer
-			cmplwi	cr2,r3,4092						; Set cr1_eq if we should take exception			
-			mfsrr0	r1								; Get the pc
-			stwx	r1,r2,r3						; Save it in the buffer
-			addi	r3,r3,4							; Point to the next slot
-			rlwinm	r3,r3,0,20,31					; Wrap the slot at one page
-			stw		r3,spcTRp(r11)					; Save the new slot
-
-			bt++	4,sbxx64c						; Jump if 64-bit...
-
-			lwz		r1,tempr0+4(r11)				; Restore work register
-			lwz		r2,tempr1+4(r11)				; Restore work register
-			lwz		r3,tempr2+4(r11)				; Restore work register
-			beq		cr2,notspectr					; Buffer filled, make a rupt...
-			mtcr	r13								; Restore CR
-			b		uftRFI							; Go restore and leave...
-
-sbxx64c:	ld		r1,tempr0(r11)					; Restore work register
-			ld		r2,tempr1(r11)					; Restore work register
-			ld		r3,tempr2(r11)					; Restore work register
-			beq		cr2,notspectr					; Buffer filled, make a rupt...
-			mtcr	r13								; Restore CR
-			b		uftRFI							; Go restore and leave...
-
 /*
  * 			Floating point assist
  */
@@ -480,7 +414,7 @@ sbxx64c:	ld		r1,tempr0(r11)					; Restore work register
 .L_handlerE00:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_FP_ASSIST					/* Set interrupt code */
+			li		r11,T_FP_ASSIST					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -492,7 +426,7 @@ sbxx64c:	ld		r1,tempr0(r11)					; Restore work register
 PMIhandler:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_PERF_MON					/* Set interrupt code */
+			li		r11,T_PERF_MON					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 	
 
@@ -504,7 +438,7 @@ PMIhandler:
 VMXhandler:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_VMX						/* Set interrupt code */
+			li		r11,T_VMX						/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 	
@@ -555,7 +489,7 @@ VMXhandler:
 .L_handler1300:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_INSTRUCTION_BKPT			/* Set interrupt code */
+			li		r11,T_INSTRUCTION_BKPT			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -566,7 +500,7 @@ VMXhandler:
 .L_handler1400:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_SYSTEM_MANAGEMENT			/* Set interrupt code */
+			li		r11,T_SYSTEM_MANAGEMENT			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -578,7 +512,7 @@ VMXhandler:
 .L_handler1500:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_SOFT_PATCH				/* Set interrupt code */
+			li		r11,T_SOFT_PATCH				/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 ;
@@ -589,7 +523,7 @@ VMXhandler:
 .L_handler1600:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_ALTIVEC_ASSIST			/* Set interrupt code */
+			li		r11,T_ALTIVEC_ASSIST			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 ;
@@ -600,7 +534,7 @@ VMXhandler:
 .L_handler1700:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_THERMAL					/* Set interrupt code */
+			li		r11,T_THERMAL					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 ;
@@ -611,7 +545,7 @@ VMXhandler:
 .L_handler1800:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_ARCHDEP0					/* Set interrupt code */
+			li		r11,T_ARCHDEP0					/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 /*
@@ -626,7 +560,7 @@ VMXhandler:
 .L_handler2000:
 			mtsprg	2,r13							/* Save R13 */
 			mtsprg	3,r11							/* Save R11 */
-			li		r11,T_INSTRUMENTATION			/* Set interrupt code */
+			li		r11,T_INSTRUMENTATION			/* Set 'rupt code */
 			b		.L_exception_entry				/* Join common... */
 
 
@@ -845,89 +779,89 @@ scTable:                                            ; ABCD E
  * except the following:
  *
  *        r11 = per_proc ptr (ie, sprg0)
- *        r13 = holds callers ctr register
- *      sprg2 = holds callers r13
- *      sprg3 = holds callers r11
+ *        r13 = holds caller's ctr register
+ *      sprg2 = holds caller's r13
+ *      sprg3 = holds caller's r11
  */
 
-;           Handle "vmm_dispatch" (0x6004), of which only some selectors are UFTs.
+;			Handle "vmm_dispatch" (0x6004), of which only some selectors are UFTs.
 
 uftVMM:
-            mtctr   r13                             ; restore callers ctr
-            lwz     r11,spcFlags(r11)               ; get the special flags word from per_proc
-            mfcr    r13                             ; save callers entire cr (we use all fields below)
+			mtctr	r13								; restore callers ctr
+			lwz		r11,spcFlags(r11)				; get the special flags word from per_proc
+			mfcr	r13								; save callers entire cr (we use all fields below)
 			rlwinm	r11,r11,16,16,31				; Extract spcFlags upper bits
 			andi.	r11,r11,hi16(runningVM|FamVMena|FamVMmode)
 			cmpwi	cr0,r11,hi16(runningVM|FamVMena|FamVMmode)	; Test in VM FAM
-			bne--	uftNormal80                     ; not eligible for FAM UFTs
+			bne--	uftNormal80						; not eligible for FAM UFTs
 			cmpwi	cr5,r3,kvmmResumeGuest			; Compare r3 with kvmmResumeGuest
 			cmpwi	cr2,r3,kvmmSetGuestRegister		; Compare r3 with kvmmSetGuestRegister
 			cror	cr1_eq,cr5_lt,cr2_gt			; Set true if out of VMM Fast syscall range
-			bt--    cr1_eq,uftNormalFF              ; Exit if out of range (the others are not UFTs)
+			bt--	cr1_eq,uftNormalFF				; Exit if out of range (the others are not UFTs)
 			b		EXT(vmm_ufp)					; handle UFT range of vmm_dispatch syscall
 
-            
-;           Handle blue box UFTs (syscalls -1 and -2).
+			
+;			Handle blue box UFTs (syscalls -1 and -2).
 
 uftIsPreemptiveTask:
 uftIsPreemptiveTaskEnv:
-            mtctr   r13                             ; restore callers ctr
-            lwz     r11,spcFlags(r11)               ; get the special flags word from per_proc
-            mfcr    r13,0x80                        ; save callers cr0 so we can use it
-            andi.	r11,r11,bbNoMachSC|bbPreemptive	; Clear what we do not need
+			mtctr	r13								; restore callers ctr
+			lwz		r11,spcFlags(r11)				; get the special flags word from per_proc
+			mfcr	r13,0x80						; save callers cr0 so we can use it
+			andi.	r11,r11,bbNoMachSC|bbPreemptive ; Clear what we do not need
 			cmplwi	r11,bbNoMachSC					; See if we are trapping syscalls
-			blt--	uftNormal80                     ; No...
-            cmpwi   r0,-2                           ; is this call IsPreemptiveTaskEnv?
+			blt--	uftNormal80						; No...
+			cmpwi	r0,-2							; is this call IsPreemptiveTaskEnv?
 			rlwimi	r13,r11,bbPreemptivebit-cr0_eq,cr0_eq,cr0_eq	; Copy preemptive task flag into user cr0_eq
 			mfsprg	r11,0							; Get the per proc once more
-            bne++   uftRestoreThenRFI               ; do not load r0 if IsPreemptiveTask
+			bne++	uftRestoreThenRFI				; do not load r0 if IsPreemptiveTask
 			lwz		r0,ppbbTaskEnv(r11)				; Get the shadowed taskEnv (only difference)
-            b       uftRestoreThenRFI               ; restore modified cr0 and return
+			b		uftRestoreThenRFI				; restore modified cr0 and return
 
 
-;           Handle "Thread Info" UFT (0x7FF2)
+;			Handle "Thread Info" UFT (0x7FF2)
 
-            .globl  EXT(uft_uaw_nop_if_32bit)
+			.globl	EXT(uft_uaw_nop_if_32bit)
 uftThreadInfo:
-            lwz     r3,UAW+4(r11)                   ; get user assist word, assuming a 32-bit processor
+			lwz		r3,UAW+4(r11)					; get user assist word, assuming a 32-bit processor
 LEXT(uft_uaw_nop_if_32bit)
-            ld      r3,UAW(r11)                     ; get the whole doubleword if 64-bit (patched to nop if 32-bit)
-            mtctr   r13                             ; restore callers ctr
-            b       uftRFI                          ; done
+			ld		r3,UAW(r11)						; get the whole doubleword if 64-bit (patched to nop if 32-bit)
+			mtctr	r13								; restore callers ctr
+			b		uftRFI							; done
 
 
-;           Handle "Facility Status" UFT (0x7FF3)
+;			Handle "Facility Status" UFT (0x7FF3)
 
 uftFacilityStatus:
-            lwz     r3,spcFlags(r11)                ; get "special flags" word from per_proc
-            mtctr   r13                             ; restore callers ctr
-            b       uftRFI                          ; done
+			lwz		r3,spcFlags(r11)				; get "special flags" word from per_proc
+			mtctr	r13								; restore callers ctr
+			b		uftRFI							; done
 
 
-;           Handle "Load MSR" UFT (0x7FF4).  This is not used on 64-bit processors, though it would work.
+;			Handle "Load MSR" UFT (0x7FF4).	 This is not used on 64-bit processors, though it would work.
 
 uftLoadMSR:
-            mfsrr1  r11                             ; get callers MSR
-            mtctr   r13                             ; restore callers ctr
-            mfcr    r13,0x80                        ; save callers cr0 so we can test PR
-            rlwinm. r11,r11,0,MSR_PR_BIT,MSR_PR_BIT ; really in the kernel?
-            bne-    uftNormal80                     ; do not permit from user mode
-            mfsprg  r11,0                           ; restore per_proc
+			mfsrr1	r11								; get callers MSR
+			mtctr	r13								; restore callers ctr
+			mfcr	r13,0x80						; save callers cr0 so we can test PR
+			rlwinm. r11,r11,0,MSR_PR_BIT,MSR_PR_BIT ; really in the kernel?
+			bne-	uftNormal80						; do not permit from user mode
+			mfsprg	r11,0							; restore per_proc
 			mtsrr1	r3								; Set new MSR
 
 
-;           Return to caller after UFT.  When called:
-;               r11 = per_proc ptr
-;               r13 = callers cr0 in upper nibble (if uftRestoreThenRFI called)
-;               sprg2 = callers r13
-;               sprg3 = callers r11
+;			Return to caller after UFT.	 When called:
+;				r11 = per_proc ptr
+;				r13 = callers cr0 in upper nibble (if uftRestoreThenRFI called)
+;				sprg2 = callers r13
+;				sprg3 = callers r11
 
-uftRestoreThenRFI:                                  ; WARNING: can drop down to here
-            mtcrf   0x80,r13                        ; restore callers cr0
+uftRestoreThenRFI:									; WARNING: can drop down to here
+			mtcrf	0x80,r13						; restore callers cr0
 uftRFI:
-            .globl  EXT(uft_nop_if_32bit)
+			.globl	EXT(uft_nop_if_32bit)
 LEXT(uft_nop_if_32bit)
-            b       uftX64                          ; patched to NOP if 32-bit processor
+			b		uftX64							; patched to NOP if 32-bit processor
 			
 uftX32:		lwz		r11,pfAvailable(r11)			; Get the feature flags
 			mfsprg	r13,2							; Restore R13
@@ -1217,20 +1151,20 @@ ctbail64:	mtcrf	0x80,r25						; Restore the used condition register field
 
 
 
-;           Handle a system call that is not a UFT and which thus goes upstairs.
+;			Handle a system call that is not a UFT and which thus goes upstairs.
 
-uftNormalFF:                                        ; here with entire cr in r13
-            mtcr    r13                             ; restore all 8 fields
+uftNormalFF:										; here with entire cr in r13
+			mtcr	r13								; restore all 8 fields
 			b		uftNormalSyscall1				; Join common...
-            
-uftNormal80:                                        ; here with callers cr0 in r13
-            mtcrf   0x80,r13                        ; restore cr0
+			
+uftNormal80:										; here with callers cr0 in r13
+			mtcrf	0x80,r13						; restore cr0
 			b		uftNormalSyscall1				; Join common...
-            
-uftNormalSyscall:                                   ; r13 = callers ctr
-            mtctr   r13                             ; restore ctr
+			
+uftNormalSyscall:									; r13 = callers ctr
+			mtctr	r13								; restore ctr
 uftNormalSyscall1:
-            li      r11,T_SYSTEM_CALL|T_FAM         ; this is a system call (and fall through)
+			li		r11,T_SYSTEM_CALL|T_FAM			; this is a system call (and fall through)
 
 
 /*<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>*/
@@ -1241,8 +1175,8 @@ uftNormalSyscall1:
  * set up:
  *
  * ENTRY:	interrupts off, VM off, in 64-bit mode if supported
- *          Callers r13 saved in sprg2.
- *          Callers r11 saved in sprg3.
+ *          Caller's r13 saved in sprg2.
+ *          Caller's r11 saved in sprg3.
  *          Exception code (ie, T_SYSTEM_CALL etc) in r11.
  *          All other registers are live.
  *
@@ -1253,11 +1187,11 @@ uftNormalSyscall1:
 /*
  *
  *	Here we will save off a mess of registers, the special ones and R0-R12.  We use the DCBZ
- *	instruction to clear and allcoate a line in the cache.  This way we will not take any cache
- *	misses, so these stores will not take all that long. Except the first line that is because
- *	we can not do a DCBZ if the L1 D-cache is off.  The rest we will skip if they are
+ *	instruction to clear and allcoate a line in the cache.  This way we won't take any cache
+ *	misses, so these stores won't take all that long. Except the first line that is because
+ *	we can't do a DCBZ if the L1 D-cache is off.  The rest we will skip if they are
  *	off also.
- *
+ * 
  *	Note that if we are attempting to sleep (as opposed to nap or doze) all interruptions
  *	are ignored.
  */
@@ -1309,19 +1243,8 @@ notsleep:	stw		r2,saver2+4(r13)				; Save this one
 			bf		doze,notspdo					; Skip the next if we are not napping/dozing...
 			rlwinm	r2,r1,0,nap+1,doze-1			; Clear any possible nap and doze bits
 			mtspr	hid0,r2							; Clear the nap/doze bits
-notspdo:
-
-#if INSTRUMENT
-			mfspr	r2,pmc1							; INSTRUMENT - saveinstr[0] - Take earliest possible stamp
-			stw		r2,0x6100+(0x00*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r2,pmc2							; INSTRUMENT - Get stamp
-			stw		r2,0x6100+(0x00*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r2,pmc3							; INSTRUMENT - Get stamp
-			stw		r2,0x6100+(0x00*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r2,pmc4							; INSTRUMENT - Get stamp
-			stw		r2,0x6100+(0x00*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
 
+notspdo:
 			la		r1,saver4(r13)					; Point to the next line in case we need it
 			crmove	wasNapping,doze					; Remember if we were napping
 			mfsprg	r2,0							; Get the per_proc area
@@ -1332,8 +1255,8 @@ notspdo:
 ;
 			andi.	r1,r11,T_FAM					; Check FAM bit
 	
-			stw		r3,saver3+4(r13)					; Save this one
-			stw		r4,saver4+4(r13)					; Save this one
+			stw		r3,saver3+4(r13)				; Save this one
+			stw		r4,saver4+4(r13)				; Save this one
 			andc	r11,r11,r1						; Clear FAM bit
 			beq+	noFAM							; Is it FAM intercept
 			mfsrr1	r3								; Load srr1
@@ -1344,7 +1267,7 @@ notspdo:
 			cmpwi	cr0,r1,2						; Check FamVMena set without FamVMmode
 			bne+	noFAM							; Can this context be FAM intercept
 			lwz		r4,FAMintercept(r2)				; Load exceptions mask to intercept
-			srwi	r1,r11,2						; divide r11 by 4
+			srwi	r1,r11,2						; Divide r11 by 4
 			lis		r3,0x8000						; Set r3 to 0x80000000
 			srw		r1,r3,r1						; Set bit for current exception
 			and.	r1,r1,r4						; And current exception with the intercept mask
@@ -1369,16 +1292,11 @@ noFAM:
             la		r3,saver16(r13)					; point to next line
 			dcbz	0,r8							; allocate 32-byte line with SRR0, SRR1, CR, XER, and LR
 			stw		r7,saver7+4(r13)				; Save this one
-			lhz		r8,PP_CPU_FLAGS(r2)				; Get the flags
 			mfsrr1	r7								; Get the interrupt SRR1
-			rlwinm	r8,r8,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Set BE bit if special trace is on
 			stw		r6,savesrr0+4(r13)				; Save the SRR0 
-			rlwinm	r6,r7,(((31-MSR_BE_BIT)+(MSR_PR_BIT+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Move PR bit to BE bit
 			stw		r5,saver5+4(r13)				; Save this one 
-			and		r8,r6,r8						; Remove BE bit only if problem state and special tracing on
 			mfsprg	r6,2							; Get interrupt time R13
 			mtsprg	2,r1							; Set the feature flags
-			andc	r7,r7,r8						; Clear BE bit if special trace is on and PR is set
 			mfsprg	r8,3							; Get rupt time R11
 			stw		r7,savesrr1+4(r13)				; Save SRR1 
 			stw		r8,saver11+4(r13)				; Save rupt time R11
@@ -1392,17 +1310,6 @@ getTB:		mftbu	r6								; Get the upper timebase
 			cmplw	r6,r8							; Did the top tick?
 			bne-	getTB							; Yeah, need to get it again...
 
-#if INSTRUMENT
-			mfspr	r6,pmc1							; INSTRUMENT - saveinstr[1] - Save halfway context save stamp
-			stw		r6,0x6100+(0x01*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r6,pmc2							; INSTRUMENT - Get stamp
-			stw		r6,0x6100+(0x01*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r6,pmc3							; INSTRUMENT - Get stamp
-			stw		r6,0x6100+(0x01*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r6,pmc4							; INSTRUMENT - Get stamp
-			stw		r6,0x6100+(0x01*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
-
 			stw		r8,ruptStamp(r2)				; Save the top of time stamp
 			stw		r8,SAVtime(r13)					; Save the top of time stamp
 			stw		r7,ruptStamp+4(r2)				; Save the bottom of time stamp
@@ -1562,17 +1469,6 @@ noPerfMonSave32:
 ;			At this point, we can take another exception and lose nothing.
 ;
 
-#if INSTRUMENT
-			mfspr	r26,pmc1						; INSTRUMENT - saveinstr[2] - Take stamp after save is done
-			stw		r26,0x6100+(0x02*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r26,pmc2						; INSTRUMENT - Get stamp
-			stw		r26,0x6100+(0x02*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r26,pmc3						; INSTRUMENT - Get stamp
-			stw		r26,0x6100+(0x02*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r26,pmc4						; INSTRUMENT - Get stamp
-			stw		r26,0x6100+(0x02*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
-
 			bne+	cr5,xcp32xit					; Skip all of this if no tracing here...
 
 ;
@@ -1683,16 +1579,6 @@ xcp32xit:	mr		r14,r11							; Save the interrupt code across the call
 			dcbz	r10,r2							; Clear for speed
 			stw		r3,next_savearea+4(r2)			; Store the savearea for the next rupt
 
-#if INSTRUMENT
-			mfspr	r4,pmc1							; INSTRUMENT - saveinstr[3] - Take stamp after next savearea
-			stw		r4,0x6100+(0x03*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r4,pmc2							; INSTRUMENT - Get stamp
-			stw		r4,0x6100+(0x03*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r4,pmc3							; INSTRUMENT - Get stamp
-			stw		r4,0x6100+(0x03*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r4,pmc4							; INSTRUMENT - Get stamp
-			stw		r4,0x6100+(0x03*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
 			b		xcpCommon						; Go join the common interrupt processing...
 
 ;
@@ -1784,16 +1670,11 @@ eEnoFAM:	lwz		r1,pfAvailable(r2)				; Get the CPU features flags
 			std		r8,saver8(r13)					; Save this one
 			mtcrf	0x40,r1							; Put the features flags (that we care about) in the CR
 			mfsrr0	r6								; Get the interruption SRR0 
-			lhz		r8,PP_CPU_FLAGS(r2)				; Get the flags
 			mtcrf	0x20,r1							; Put the features flags (that we care about) in the CR
 			mfsrr1	r7								; Get the interrupt SRR1
-			rlwinm	r8,r8,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Set BE bit if special trace is on
 			std		r6,savesrr0(r13)				; Save the SRR0 
 			mtcrf	0x02,r1							; Put the features flags (that we care about) in the CR
-			rlwinm	r6,r7,(((31-MSR_BE_BIT)+(MSR_PR_BIT+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Move PR bit to BE bit
-			and		r8,r6,r8						; Remove BE bit only if problem state and special tracing on
 			std		r9,saver9(r13)					; Save this one
-			andc	r7,r7,r8						; Clear BE bit if special trace is on and PR is set
 			crmove	featAltivec,pfAltivecb			; Set the Altivec flag
 			std		r7,savesrr1(r13)				; Save SRR1 
 			mfsprg	r9,3							; Get rupt time R11
@@ -2070,18 +1951,6 @@ noavec:		stw		r19,savevrsave(r13)				; Save the vector register usage flags
 ;
 
 Redrive:	
-
-
-#if INSTRUMENT
-			mfspr	r20,pmc1						; INSTRUMENT - saveinstr[4] - Take stamp before exception filter
-			stw		r20,0x6100+(0x04*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r20,pmc2						; INSTRUMENT - Get stamp
-			stw		r20,0x6100+(0x04*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r20,pmc3						; INSTRUMENT - Get stamp
-			stw		r20,0x6100+(0x04*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r20,pmc4						; INSTRUMENT - Get stamp
-			stw		r20,0x6100+(0x04*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
 			lwz		r22,SAVflags(r13)				; Pick up the flags
 			lwz		r0,saver0+4(r13)				; Get back interrupt time syscall number
 			mfsprg	r2,0							; Restore per_proc
@@ -2120,7 +1989,7 @@ xcpTable:
 			.long	EXT(handlePF)					; T_INSTRUCTION_ACCESS
 			.long	PassUpRupt						; T_INTERRUPT		
 			.long	EXT(AlignAssist)				; T_ALIGNMENT			
-			.long	EXT(Emulate)					; T_PROGRAM			
+			.long	ProgramChk						; T_PROGRAM
 			.long	PassUpFPU						; T_FP_UNAVAILABLE		
 			.long	PassUpRupt						; T_DECREMENTER		
 			.long	PassUpTrap						; T_IO_ERROR			
@@ -2968,11 +2837,59 @@ ueMck:		li		r0,0							; Set the unrecovered flag before passing up
 			stw		r0,savemisc3(r13)				; Set it
 			b		PassUpTrap						; Go up and log error and probably panic
 			
+;
+;			We come here to handle program exceptions
+;
+;			When the program check is a trap instruction and it happens when
+;			we are executing injected code, we need to check if it is an exit trap.
+;			If it is, we need to populate the current savearea with some of the context from 
+;			the saved pre-inject savearea.  This is needed because the current savearea will be
+;			tossed as part of the pass up code.  Additionally, because we will not be nullifying
+;			the emulated instruction as we do with any other exception.
+;
+			
+			.align	5
+
+ProgramChk:	lwz		r5,savesrr1+4(r13)				; Get the interrupt SRR1
+			lwz		r3,ijsave(r2)					; Get the inject savearea top
+			lwz		r4,ijsave+4(r2)					; And get the bottom of the inject savearea pointer
+			rlwimi	r5,r5,15,31,31					; Scoot trap flag down to a spare bit
+			rlwinm	r3,r3,0,1,0						; Copy low 32 bits of to top 32
+			li		r0,0x0023						; Get bits that match scooted trap flag, IR, and RI
+			and		r0,r5,r0						; Clear any extra SRR1 bits
+			rlwimi.	r3,r4,0,0,31					; Insert low part of 64-bit address in bottom 32 bits and see if ijsave is 0		
+			cmplwi	cr1,r0,1						; Make sure we were IR off, RI off, and got a trap exception
+			crandc	cr0_eq,cr1_eq,cr0_eq			; If we are injecting, ijsave will be non-zero and we had the trap bit set
+			mfsrr0	r4								; Get the PC
+			bne++	cr0,mustem						; This is not an injection exit...
+
+			lwz		r4,0(r4)						; Get the trap instruction
+			lis		r5,hi16(ijtrap)					; Get high half of inject exit trap
+			ori		r5,r5,lo16(ijtrap)				; And the low half
+			cmplw	r4,r5							; Correct trap instruction?
+			bne		mustem							; No, not inject exit...
+
+			lwz		r4,savesrr0(r3)					; Get the original SRR0
+			lwz		r5,savesrr0+4(r3)				; And the rest of it
+			lwz		r6,savesrr1(r3)					; Get the original SRR1
+			stw		r4,savesrr0(r13)				; Set the new SRR0 to the original
+			lwz		r4,savesrr1+4(r13)				; Get the bottom of the new SRR1
+			lwz		r7,savesrr1+4(r3)				; Get the bottom of the original SRR1
+			li		r11,T_INJECT_EXIT				; Set an inject exit exception
+			stw		r5,savesrr0+4(r13)				; Set the new bottom of SRR0 to the original
+			rlwimi	r7,r4,0,MSR_FP_BIT,MSR_FP_BIT	; Make sure we retain the current floating point enable bit
+			stw		r6,savesrr1(r13)				; Save the top half of the original SRR1
+			sth		r7,savesrr1+6(r13)				; And the last bottom
+			stw		r11,saveexception(r13)			; Set the new the exception code
+			b		PassUpTrap						; Go pass it on up...
+
+mustem:		b		EXT(Emulate)					; Go try to emulate this one...
+
 
 /*
- *			Here is where we come back from some instruction emulator.  If we come back with
+ *			Here's where we come back from some instruction emulator.  If we come back with
  *			T_IN_VAIN, the emulation is done and we should just reload state and directly
- *			go back to the interrupted code. Otherwise, we will check to see if
+ *			go back to the interrupted code. Otherwise, we'll check to see if
  *			we need to redrive with a different interrupt, i.e., DSI.
  *			Note that this we are actually not redriving the rupt, rather changing it
  *			into a different one.  Thus we clear the redrive bit.
@@ -3017,7 +2934,9 @@ PassUpRupt:	lis		r20,hi16(EXT(ihandler))			; Get ihandler address
 PassUpFPU:	lis		r20,hi16(EXT(fpu_switch))		; Get FPU switcher address
 			ori		r20,r20,lo16(EXT(fpu_switch))	; Get FPU switcher address
 			b		PassUp							; Go pass it up...
-	
+
+			.align	5
+
 PassUpVMX:	lis		r20,hi16(EXT(vec_switch))		; Get VMX switcher address
 			ori		r20,r20,lo16(EXT(vec_switch))	; Get VMX switcher address
 			bt++	featAltivec,PassUp				; We have VMX on this CPU...
@@ -3038,18 +2957,41 @@ PassUpAbend:
 			.align	5
 
 PassUp:		
-#if INSTRUMENT
-			mfspr	r29,pmc1						; INSTRUMENT - saveinstr[11] - Take stamp at passup or eatrupt
-			stw		r29,0x6100+(11*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r29,pmc2						; INSTRUMENT - Get stamp
-			stw		r29,0x6100+(11*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r29,pmc3						; INSTRUMENT - Get stamp
-			stw		r29,0x6100+(11*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r29,pmc4						; INSTRUMENT - Get stamp
-			stw		r29,0x6100+(11*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
+			mfsprg	r29,0							; Get the per_proc block back
+			
+			cmplwi	cr1,r11,T_INJECT_EXIT			; Are we exiting from an injection?
+			lwz		r3,ijsave(r29)					; Get the inject savearea top
+			lwz		r4,ijsave+4(r29)				; And get the bottom of the inject savearea pointer
+			rlwinm	r3,r3,0,1,0						; Copy low 32 bits to top 32
+			rlwimi.	r3,r4,0,0,31					; Insert low part of 64-bit address in bottom 32 bits and see if ijsave is 0		
+			beq++	notaninjct						; Skip tossing savearea if no injection...
+
+			beq--	cr1,nonullify					; Have not finished the instruction, go nullify it...
+			
+			lwz		r4,savesrr1+4(r3)				; Get the interrupt modifiers from the original SRR1
+			lwz		r5,savesrr1+4(r13)				; Get the interrupt modifiers from the new SRR1
+			lwz		r6,savedar(r13)					; Get the top of the DAR
+			rlwimi	r4,r5,0,0,15					; copy the new top to the original SRR1
+			lwz		r7,savedar+4(r13)				; Get the bottom of the DAR
+			rlwimi	r4,r5,0,MSR_FP_BIT,MSR_FP_BIT	; Copy the new FP enable bit into the old SRR1
+			stw		r4,savesrr1+4(r3)				; Save the updated SRR1
+			lwz		r5,savedsisr(r13)				; Grab the new DSISR
+			
+			mr		r4,r13							; Save the new savearea pointer
+			mr		r13,r3							; Point to the old savearea we are keeping
+			stw		r6,savedar(r13)					; Save top of new DAR
+			stw		r7,savedar+4(r13)				; Save bottom of new DAR
+			stw		r5,savedsisr(r13)				; Set the new DSISR
+			stw		r11,saveexception(r13)			; Set the new exception code
+			mr		r3,r4							; Point to the new savearea in order to toss it
+			
+nonullify:	li		r0,0							; Get a zero
+			stw		r0,ijsave(r29)					; Clear the pointer to the saved savearea
+			stw		r0,ijsave+4(r29)				; Clear the pointer to the saved savearea
+			
+			bl		EXT(save_ret_phys)				; Dump that pesky extra savearea			
 			
-			lwz		r10,SAVflags(r13)				; Pick up the flags
+notaninjct:	lwz		r10,SAVflags(r13)				; Pick up the flags
 
 			li		r0,0xFFF						; Get a page mask
 			li		r2,MASK(MSR_BE)|MASK(MSR_SE)	; Get the mask to save trace bits
@@ -3063,22 +3005,11 @@ PassUp:
 			stw		r10,SAVflags(r13)				; Set the flags with the cleared redrive flag
 
 			xor		r4,r13,r5						; Pass up the virtual address of context savearea
-			mfsprg	r29,0							; Get the per_proc block back
 			rlwinm	r4,r4,0,0,31					; Clean top half of virtual savearea if 64-bit
 
 			mr		r3,r21							; Pass in the MSR we will go to
 			bl		EXT(switchSegs)					; Go handle the segment registers/STB
 
-#if INSTRUMENT
-			mfspr	r30,pmc1						; INSTRUMENT - saveinstr[7] - Take stamp afer switchsegs
-			stw		r30,0x6100+(7*16)+0x0(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc2						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(7*16)+0x4(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc3						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(7*16)+0x8(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc4						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(7*16)+0xC(0)			; INSTRUMENT - Save it
-#endif			
 			lwz		r3,saveexception(r13)			; Recall the exception code
 			
 			mtsrr0	r20								; Set up the handler address
@@ -3143,17 +3074,8 @@ erchkfret:	mr.		r3,r30							; Any savearea to quickly release?
 			.align	5
 			
 ernoqfret:	
-#if INSTRUMENT
-			mfspr	r30,pmc1						; INSTRUMENT - saveinstr[5] - Take stamp at saveareas released
-			stw		r30,0x6100+(5*16)+0x0(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc2						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(5*16)+0x4(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc3						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(5*16)+0x8(0)			; INSTRUMENT - Save it
-			mfspr	r30,pmc4						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(5*16)+0xC(0)			; INSTRUMENT - Save it
-#endif			
-
+			lwz		r30,SAVflags(r31)				; Pick up the flags
+			lis		r0,hi16(SAVinject)				; Get inject flag
 			dcbt	0,r21							; Touch in the first thing we need
 			
 ;
@@ -3167,47 +3089,70 @@ ernoqfret:
 ;			savearea to the head of the local list.  Then, if it needs to trim, it will
 ;			start with the SECOND savearea, leaving ours intact.
 ;
+;			If we are going to inject code here, we must not toss the savearea because
+;			we will continue to use it.  The code stream to inject is in it and we 
+;			use it to hold the pre-inject context so that we can merge that with the
+;			post-inject context.  The field ijsave in the per-proc is used to point to the savearea.
+;
+;			Note that we will NEVER pass an interrupt up without first dealing with this savearea.
+;			
+;			All permanent interruptions (i.e., not denorm, alignment, or handled page and segment faults)
+;			will nullify any injected code and pass the interrupt up in the original savearea.  A normal
+;			inject completion will merge the original context into the new savearea and pass that up.
+;			
+;			Note that the following code which sets up the injection will only be executed when
+;			SAVinject is set.  That means that if will not run if we are returning from an alignment
+;			or denorm exception, or from a handled page or segment fault.
 ;
 
+			andc	r0,r30,r0						; Clear the inject flag
+			cmplw	cr4,r0,r30						; Remember if we need to inject
 			mr		r3,r31							; Get the exiting savearea in parm register
-			bl		EXT(save_ret_phys)				; Put it on the free list			
-#if INSTRUMENT
-			mfspr	r3,pmc1							; INSTRUMENT - saveinstr[6] - Take stamp afer savearea released
-			stw		r3,0x6100+(6*16)+0x0(0)			; INSTRUMENT - Save it
-			mfspr	r3,pmc2							; INSTRUMENT - Get stamp
-			stw		r3,0x6100+(6*16)+0x4(0)			; INSTRUMENT - Save it
-			mfspr	r3,pmc3							; INSTRUMENT - Get stamp
-			stw		r3,0x6100+(6*16)+0x8(0)			; INSTRUMENT - Save it
-			mfspr	r3,pmc4							; INSTRUMENT - Get stamp
-			stw		r3,0x6100+(6*16)+0xC(0)			; INSTRUMENT - Save it
-#endif			
-
-			lwz		r3,savesrr1+4(r31)				; Pass in the MSR we are going to
+			beq+	cr4,noinject					; No, we are not going to inject instructions...	
+			
+			stw		r0,SAVflags(r31)				; Yes we are, clear the request...
+			
+			lhz		r26,PP_CPU_NUMBER(r29)			; Get the cpu number
+			lwz		r25,saveinstr(r31)				; Get the instruction count
+			la		r3,saveinstr+4(r31)				; Point to the instruction stream
+			slwi	r26,r26,6						; Get offset to the inject code stream for this processor
+			li		r5,0							; Get the current instruction offset
+			ori		r26,r26,lo16(EXT(ijcode))		; Get the base of the inject buffer for this processor (always < 64K)
+			slwi	r25,r25,2						; Multiply by 4
+			
+injctit:	lwzx	r6,r5,r3						; Pick up the instruction
+			stwx	r6,r5,r26						; Inject into code buffer
+			addi	r5,r5,4							; Bump offset
+			cmplw	r5,r25							; Have we hit the end?
+			blt-	injctit							; Continue until we have copied all...
+			
+			lis		r3,0x0FFF						; Build our magic trap
+			ori		r3,r3,0xC9C9					; Build our magic trap
+			stw		r31,ijsave+4(r29)				; Save the original savearea for injection
+			stwx	r3,r5,r26						; Save the magic trap
+
+			li		r3,32							; Get cache line size
+			dcbf	0,r26							; Flush first line
+			dcbf	r3,r26							; And the second
+			sync									; Hang on until it's done
+			
+			icbi	0,r26							; Flush instructions in the first line
+			icbi	r3,r26							; And the second
+			isync									; Throw anything stale away
+			sync									; Hang on until it's done
+			b		injected						; Skip the savearea release...
+			
+noinject:	bl		EXT(save_ret_phys)				; Put old savearea on the free list			
+
+injected:	lwz		r3,savesrr1+4(r31)				; Pass in the MSR we are going to
 			bl		EXT(switchSegs)					; Go handle the segment registers/STB
-#if INSTRUMENT
-			mfspr	r30,pmc1						; INSTRUMENT - saveinstr[10] - Take stamp afer switchsegs
-			stw		r30,0x6100+(10*16)+0x0(0)		; INSTRUMENT - Save it
-			mfspr	r30,pmc2						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(10*16)+0x4(0)		; INSTRUMENT - Save it
-			mfspr	r30,pmc3						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(10*16)+0x8(0)		; INSTRUMENT - Save it
-			mfspr	r30,pmc4						; INSTRUMENT - Get stamp
-			stw		r30,0x6100+(10*16)+0xC(0)		; INSTRUMENT - Save it
-#endif			
-			li		r3,savesrr1+4					; Get offset to the srr1 value
 
-			lhz		r9,PP_CPU_FLAGS(r29)			; Get the processor flags
-			lwarx	r26,r3,r31						; Get destination MSR and take reservation along the way (just so we can blow it away)
-			
-			rlwinm	r25,r26,27,22,22				; Move PR bit to BE
-			
+			li		r3,savesrr1+4					; Get offset to the srr1 value
+			lwarx	r8,r3,r31						; Get destination MSR and take reservation along the way (just so we can blow it away)
 			cmplw	cr3,r14,r14						; Set that we do not need to stop streams
 
-			rlwinm	r9,r9,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Set BE bit if special trace is on
 			li		r21,emfp0						; Point to the fp savearea
-			and		r9,r9,r25						; Clear BE if supervisor state
-			or		r26,r26,r9						; Flip on the BE bit for special trace if needed
-			stwcx.	r26,r3,r31						; Blow away any reservations we hold (and set BE)
+			stwcx.	r8,r3,r31						; Blow away any reservations we hold
 
 			lwz		r25,savesrr0+4(r31)				; Get the SRR0 to use
 			
@@ -3216,7 +3161,20 @@ ernoqfret:
 			lwz		r0,saver0+4(r31)				; Restore R0			
 			dcbt	0,r28							; Touch in r4-r7 
 			lwz		r1,saver1+4(r31)				; Restore R1	
-			lwz		r2,saver2+4(r31)				; Restore R2	
+			
+			beq+	cr4,noinject2					; No code injection here...
+			
+;
+;			If we are injecting, we need to stay in supervisor state with instruction
+;			address translation off.  We also need to have as few potential interruptions as
+;			possible.  Therefore, we turn off external interruptions and tracing (which doesn't
+;			make much sense anyway).
+;
+			ori		r8,r8,lo16(ijemoff)				; Force the need-to-be-off bits on
+			mr		r25,r26							; Get the injected code address
+			xori	r8,r8,lo16(ijemoff)				; Turn off all of the need-to-be-off bits
+			
+noinject2:	lwz		r2,saver2+4(r31)				; Restore R2	
 			la		r28,saver8(r31)					; Point to the 32-byte line with r8-r11
 			lwz		r3,saver3+4(r31)				; Restore R3
             andis.	r6,r27,hi16(pfAltivec)			; Do we have altivec on the machine?
@@ -3225,7 +3183,7 @@ ernoqfret:
             la		r28,saver12(r31)				; Point to the 32-byte line with r12-r15
 			mtsrr0	r25								; Restore the SRR0 now
 			lwz		r5,saver5+4(r31)				; Restore R5
-			mtsrr1	r26								; Restore the SRR1 now 
+			mtsrr1	r8								; Restore the SRR1 now 
 			lwz		r6,saver6+4(r31)				; Restore R6			
 			
 			dcbt	0,r28							; touch in r12-r15
@@ -3388,7 +3346,9 @@ erchkfre64:	mr.		r3,r30							; Any savearea to quickly release?
 
 			.align	7
 			
-ernoqfre64:	dcbt	0,r21							; Touch in the first thing we need
+ernoqfre64:	lwz		r30,SAVflags(r31)				; Pick up the flags
+			lis		r0,hi16(SAVinject)				; Get inject flag
+			dcbt	0,r21							; Touch in the first thing we need
 			
 ;
 ;			Here we release the savearea.
@@ -3401,26 +3361,67 @@ ernoqfre64:	dcbt	0,r21							; Touch in the first thing we need
 ;			savearea to the head of the local list.  Then, if it needs to trim, it will
 ;			start with the SECOND savearea, leaving ours intact.
 ;
+;			If we are going to inject code here, we must not toss the savearea because
+;			we will continue to use it.  The code stream to inject is in it and we 
+;			use it to hold the pre-inject context so that we can merge that with the
+;			post-inject context.  The field ijsave in the per-proc is used to point to the savearea.
 ;
+;			Note that we will NEVER pass an interrupt up without first dealing with this savearea.
+;			
+;			All permanent interruptions (i.e., not denorm, alignment, or handled page and segment faults)
+;			will nullify any injected code and pass the interrupt up in the original savearea.  A normal
+;			inject completion will merge the original context into the new savearea and pass that up.
+;			
+;			Note that the following code which sets up the injection will only be executed when
+;			SAVinject is set.  That means that if will not run if we are returning from an alignment
+;			or denorm exception, or from a handled page or segment fault.
+;
+
 
 			li		r3,lgKillResv					; Get spot to kill reservation
+			andc	r0,r30,r0						; Clear the inject flag
 			stdcx.	r3,0,r3							; Blow away any reservations we hold
-			
+			cmplw	cr4,r0,r30						; Remember if we need to inject
 			mr		r3,r31							; Get the exiting savearea in parm register
-			bl		EXT(save_ret_phys)				; Put it on the free list			
+			beq++	cr4,noinject3					; No, we are not going to inject instructions...	
+			
+			stw		r0,SAVflags(r31)				; Yes we are, clear the request...
 
-			lwz		r3,savesrr1+4(r31)				; Pass in the MSR we will be going to
+			lhz		r26,PP_CPU_NUMBER(r29)			; Get the cpu number
+			lwz		r25,saveinstr(r31)				; Get the instruction count
+			la		r3,saveinstr+4(r31)				; Point to the instruction stream
+			slwi	r26,r26,6						; Get offset to the inject code stream for this processor
+			li		r5,0							; Get the current instruction offset
+			ori		r26,r26,lo16(EXT(ijcode))		; Get the base of the inject buffer for this processor (always < 64K)
+			slwi	r25,r25,2						; Multiply by 4
+			
+injctit2:	lwzx	r6,r5,r3						; Pick up the instruction
+			stwx	r6,r5,r26						; Inject into code buffer
+			addi	r5,r5,4							; Bump offset
+			cmplw	r5,r25							; Have we hit the end?
+			blt--	injctit2						; Continue until we have copied all...
+			
+			lis		r3,0x0FFF						; Build our magic trap
+			ori		r3,r3,0xC9C9					; Build our magic trap
+			std		r31,ijsave(r29)					; Save the original savearea for injection
+			stwx	r3,r5,r26						; Save the magic trap
+
+			dcbf	0,r26							; Flush the line
+			sync									; Hang on until it's done
+			
+			icbi	0,r26							; Flush instructions in the line
+			isync									; Throw anything stale away
+			sync									; Hang on until it's done
+			b		injected2						; Skip the savearea release...
+			
+noinject3:	bl		EXT(save_ret_phys)				; Put it on the free list			
+
+injected2:	lwz		r3,savesrr1+4(r31)				; Pass in the MSR we will be going to
 			bl		EXT(switchSegs)					; Go handle the segment registers/STB
 
-			lhz		r9,PP_CPU_FLAGS(r29)			; Get the processor flags
-			ld		r26,savesrr1(r31)				; Get destination MSR
+			ld		r8,savesrr1(r31)				; Get destination MSR
 			cmplw	cr3,r14,r14						; Set that we do not need to stop streams
-			rlwinm	r25,r26,27,22,22				; Move PR bit to BE
-
-			rlwinm	r9,r9,(((31-MSR_BE_BIT)+(traceBEb+16+1))&31),MSR_BE_BIT,MSR_BE_BIT	; Set BE bit if special trace is on
 			li		r21,emfp0						; Point to a workarea
-			and		r9,r9,r25						; Clear BE if supervisor state
-			or		r26,r26,r9						; Flip on the BE bit for special trace if needed
 
 			ld		r25,savesrr0(r31)				; Get the SRR0 to use
 			la		r28,saver16(r31)				; Point to the 128-byte line with r16-r31
@@ -3428,13 +3429,26 @@ ernoqfre64:	dcbt	0,r21							; Touch in the first thing we need
 			ld		r0,saver0(r31)					; Restore R0			
 			dcbt	0,r28							; Touch in r16-r31 
 			ld		r1,saver1(r31)					; Restore R1	
-			ld		r2,saver2(r31)					; Restore R2	
+			
+			beq++	cr4,noinject4					; No code injection here...
+			
+;
+;			If we are injecting, we need to stay in supervisor state with instruction
+;			address translation off.  We also need to have as few potential interruptions as
+;			possible.  Therefore, we turn off external interruptions and tracing (which doesn't
+;			make much sense anyway).
+;
+			ori		r8,r8,lo16(ijemoff)				; Force the need-to-be-off bits on
+			mr		r25,r26							; Point pc to injection code buffer
+			xori	r8,r8,lo16(ijemoff)				; Turn off all of the need-to-be-off bits
+			
+noinject4:	ld		r2,saver2(r31)					; Restore R2	
 			ld		r3,saver3(r31)					; Restore R3
 			mtcrf	0x80,r27						; Get facility availability flags (do not touch CR1-7)
 			ld		r4,saver4(r31)					; Restore R4
 			mtsrr0	r25								; Restore the SRR0 now
 			ld		r5,saver5(r31)					; Restore R5
-			mtsrr1	r26								; Restore the SRR1 now 
+			mtsrr1	r8								; Restore the SRR1 now 
 			ld		r6,saver6(r31)					; Restore R6			
 						
 			ld		r7,saver7(r31)					; Restore R7
@@ -3599,18 +3613,6 @@ eeNoMSRx:	xor		r31,r31,r4						; Convert the savearea to physical addressing
 			lwz		r4,SAVflags(r31)				; Pick up the flags
 			mr		r13,r31							; Put savearea here also
 
-#if INSTRUMENT
-			mfspr	r5,pmc1							; INSTRUMENT - saveinstr[8] - stamp exception exit
-			stw		r5,0x6100+(8*16)+0x0(0)			; INSTRUMENT - Save it
-			mfspr	r5,pmc2							; INSTRUMENT - Get stamp
-			stw		r5,0x6100+(8*16)+0x4(0)			; INSTRUMENT - Save it
-			mfspr	r5,pmc3							; INSTRUMENT - Get stamp
-			stw		r5,0x6100+(8*16)+0x8(0)			; INSTRUMENT - Save it
-			mfspr	r5,pmc4							; INSTRUMENT - Get stamp
-			stw		r5,0x6100+(8*16)+0xC(0)			; INSTRUMENT - Save it
-#endif
-
-
 			and.	r0,r4,r1						; Check if redrive requested
 			
 			dcbt	br0,r2							; We will need this in just a sec
@@ -3960,29 +3962,42 @@ EXT(killresv):
 			.long	0								; 5474 reserved			
 			.long	0								; 5478 reserved			
 			.long	0								; 547C reserved
-			.long	EXT(kmod)							; 5480 Pointer to kmod, debugging aid
-			.long	EXT(kdp_trans_off)						; 5484 Pointer to kdp_trans_off, debugging aid
-			.long	EXT(kdp_read_io)						; 5488 Pointer to kdp_read_io, debugging aid
+			.long	EXT(kmod)						; 5480 Pointer to kmod, debugging aid
+			.long	EXT(kdp_trans_off)				; 5484 Pointer to kdp_trans_off, debugging aid
+			.long	EXT(kdp_read_io)				; 5488 Pointer to kdp_read_io, debugging aid
 			.long	0								; 548C Reserved for developer use
 			.long	0								; 5490 Reserved for developer use
+			.long	EXT(osversion)					; 5494	Pointer to osversion string, debugging aid
+			.long	EXT(flag_kdp_trigger_reboot)					; 5498	Pointer to KDP reboot trigger, debugging aid
+
 ;
-;	The "shared page" is used for low-level debugging
+;	The "shared page" is used for low-level debugging and is actually 1/2 page long
 ;
 
 			. = 0x6000
 			.globl	EXT(sharedPage)
 
-EXT(sharedPage):									; Per processor data area
-		.long	0xC24BC195							; Comm Area validity value 
-		.long	0x87859393							; Comm Area validity value 
-		.long	0xE681A2C8							; Comm Area validity value 
-		.long	0x8599855A							; Comm Area validity value 
-		.long	0xD74BD296							; Comm Area validity value 
-		.long	0x8388E681							; Comm Area validity value 
-		.long	0xA2C88599							; Comm Area validity value 
-		.short	0x855A								; Comm Area validity value 
-		.short	1									; Comm Area version number
-		.fill	1016*4,1,0							; (filled with 0s)
+EXT(sharedPage):									; This is a debugging page shared by all processors
+			.long	0xC24BC195						; Comm Area validity value 
+			.long	0x87859393						; Comm Area validity value 
+			.long	0xE681A2C8						; Comm Area validity value 
+			.long	0x8599855A						; Comm Area validity value 
+			.long	0xD74BD296						; Comm Area validity value 
+			.long	0x8388E681						; Comm Area validity value 
+			.long	0xA2C88599						; Comm Area validity value 
+			.short	0x855A							; Comm Area validity value 
+			.short	1								; Comm Area version number
+			.fill	504*4,1,0						; (filled with 0s)
+
+;
+;	The ijcode area is used for code injection.  It is 1/2 page long and will allow 32 processors to inject
+;	16 instructions each concurrently.
+;
+
+			.globl	EXT(ijcode)
+
+EXT(ijcode):										; Code injection area
+			.fill	512*4,1,0						; 6800 32x64 slots for code injection streams
 
 	.data
 	.align	ALIGN
diff --git a/osfmk/ppc/machine_cpu.h b/osfmk/ppc/machine_cpu.h
index 13967afec..88fe14def 100644
--- a/osfmk/ppc/machine_cpu.h
+++ b/osfmk/ppc/machine_cpu.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PPC_MACHINE_CPU_H_
 #define _PPC_MACHINE_CPU_H_
diff --git a/osfmk/ppc/machine_routines.c b/osfmk/ppc/machine_routines.c
index bcfaf79e2..7f127ee81 100644
--- a/osfmk/ppc/machine_routines.c
+++ b/osfmk/ppc/machine_routines.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <mach/mach_types.h>
@@ -33,6 +39,8 @@
 #include <ppc/mem.h>
 #include <ppc/new_screen.h>
 #include <ppc/proc_reg.h>
+#include <ppc/machine_cpu.h> /* for cpu_signal_handler() */
+#include <ppc/fpu_protos.h>
 #include <kern/kern_types.h>
 #include <kern/processor.h>
 #include <kern/machine.h>
@@ -82,11 +90,6 @@ extern unsigned int mulckePatch_eieio;
 extern unsigned int sulckPatch_eieio;
 extern unsigned int rwlesPatch_eieio;
 extern unsigned int rwldPatch_eieio;
-#if     !MACH_LDEBUG
-extern unsigned int entfsectPatch_isync;
-extern unsigned int retfsectPatch_isync;
-extern unsigned int retfsectPatch_eieio;
-#endif
 
 struct patch_up {
         unsigned int    *addr;
@@ -125,11 +128,6 @@ patch_up_t patch_up_table[] = {
 	{&sulckPatch_eieio,		0x60000000},
 	{&rwlesPatch_eieio,		0x60000000},
 	{&rwldPatch_eieio,		0x60000000},
-#if     !MACH_LDEBUG
-	{&entfsectPatch_isync,		0x60000000},
-	{&retfsectPatch_isync,		0x60000000},
-	{&retfsectPatch_eieio,		0x60000000},
-#endif
 	{NULL,				0x00000000}
 };
 
@@ -249,7 +247,48 @@ void ml_install_interrupt_handler(
 	proc_info->interrupts_enabled = TRUE;  
 	(void) ml_set_interrupts_enabled(current_state);
 
-	initialize_screen(0, kPEAcquireScreen);
+	initialize_screen(NULL, kPEAcquireScreen);
+}
+
+/*
+ *	Routine:        ml_nofault_copy
+ *	Function:	Perform a physical mode copy if the source and
+ *			destination have valid translations in the kernel pmap.
+ *			If translations are present, they are assumed to
+ *			be wired; i.e. no attempt is made to guarantee that the
+ *			translations obtained remained valid for
+ *			the duration of their use.
+ */
+
+vm_size_t ml_nofault_copy(
+	vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size)
+{
+	addr64_t cur_phys_dst, cur_phys_src;
+	uint32_t count, pindex, nbytes = 0;
+
+	while (size > 0) {
+		if (!(cur_phys_src = kvtophys(virtsrc)))
+			break;
+		if (!(cur_phys_dst = kvtophys(virtdst)))
+			break;
+		if (!mapping_phys_lookup((cur_phys_src>>12), &pindex) ||
+		    !mapping_phys_lookup((cur_phys_dst>>12), &pindex))
+			break;
+		count = PAGE_SIZE - (cur_phys_src & PAGE_MASK);
+		if (count > (PAGE_SIZE - (cur_phys_dst & PAGE_MASK)))
+			count = PAGE_SIZE - (cur_phys_dst & PAGE_MASK);
+		if (count > size)
+			count = size;
+
+		bcopy_phys(cur_phys_src, cur_phys_dst, count);
+
+		nbytes += count;
+		virtsrc += count;
+		virtdst += count;
+		size -= count;
+	}
+
+	return nbytes;
 }
 
 /*
@@ -305,14 +344,9 @@ void ml_cause_interrupt(void)
  */
 void ml_thread_policy(
 	thread_t thread,
-	unsigned policy_id,
+__unused	unsigned policy_id,
 	unsigned policy_info)
 {
-
-	if ((policy_id == MACHINE_GROUP) &&
-		((PerProcTable[master_cpu].ppe_vaddr->pf.Available) & pfSMPcap))
-			thread_bind(thread, master_processor);
-
 	if (policy_info & MACHINE_NETWORK_WORKLOOP) {
 		spl_t		s = splsched();
 
@@ -406,8 +440,9 @@ ml_processor_register(
 	}
 
 	if (!boot_processor) {
-		(void)hw_atomic_add((uint32_t *)&saveanchor.savetarget, FreeListMin);   /* saveareas for this processor */
-		processor_init((struct processor *)proc_info->processor, proc_info->cpu_number);
+		(void)hw_atomic_add(&saveanchor.savetarget, FreeListMin);   /* saveareas for this processor */
+		processor_init((struct processor *)proc_info->processor,
+								proc_info->cpu_number, processor_pset(master_processor));
 	}
 
 	*processor_out = (struct processor *)proc_info->processor;
@@ -516,8 +551,8 @@ ml_get_max_cpus(void)
 		mcpus_state |= MAX_CPUS_WAIT;
 		thread_sleep_mutex((event_t)&mcpus_state,
 					 &mcpus_lock, THREAD_UNINT);
-	} else
-		mutex_unlock(&mcpus_lock);
+	}
+	mutex_unlock(&mcpus_lock);
 	return(machine_info.max_cpus);
 }
 
@@ -528,8 +563,8 @@ ml_get_max_cpus(void)
 void
 ml_cpu_up(void)
 {
-	hw_atomic_add(&machine_info.physical_cpu, 1);
-	hw_atomic_add(&machine_info.logical_cpu, 1);
+	(void)hw_atomic_add(&machine_info.physical_cpu, 1);
+	(void)hw_atomic_add(&machine_info.logical_cpu, 1);
 }
 
 /*
@@ -539,8 +574,8 @@ ml_cpu_up(void)
 void
 ml_cpu_down(void)
 {
-	hw_atomic_sub(&machine_info.physical_cpu, 1);
-	hw_atomic_sub(&machine_info.logical_cpu, 1);
+	(void)hw_atomic_sub(&machine_info.physical_cpu, 1);
+	(void)hw_atomic_sub(&machine_info.logical_cpu, 1);
 }
 
 /*
@@ -624,8 +659,6 @@ ml_enable_cache_level(int cache_level, int enable)
 }
 
 
-decl_simple_lock_data(, spsLock);
-
 /*
  *      Routine:        ml_set_processor_speed
  *      Function:
@@ -656,7 +689,7 @@ ml_set_processor_speed(unsigned long speed)
 				 */
 				for (i=200; i>0; i--) {
 					current_state = ml_set_interrupts_enabled(FALSE);
-					if (cpu != cpu_number()) {
+					if (cpu != (unsigned)cpu_number()) {
 							if (PerProcTable[cpu].ppe_vaddr->cpu_flags & SignalReady)
 							/*
 							 * Target cpu is off-line, skip
@@ -771,49 +804,6 @@ machine_processor_shutdown(
 	return((thread_t)(getPerProc()->old_thread));
 }
 
-/*
- *	Routine:        set_be_bit
- *	Function:
- */
-int
-set_be_bit(
-	void)
-{
-	boolean_t current_state;
-
-	current_state = ml_set_interrupts_enabled(FALSE);
-	getPerProc()->cpu_flags |= traceBE;
-	(void) ml_set_interrupts_enabled(current_state);
-	return(1);
-}
-
-/*
- *	Routine:        clr_be_bit
- *	Function:
- */
-int
-clr_be_bit(
-	void)
-{
-	boolean_t current_state;
-
-	current_state = ml_set_interrupts_enabled(FALSE);
-	getPerProc()->cpu_flags &= ~traceBE;
-	(void) ml_set_interrupts_enabled(current_state);
-	return(1);
-}
-
-/*
- *	Routine:        be_tracing
- *	Function:
- */
-int
-be_tracing(
-	void)
-{
-  return(getPerProc()->cpu_flags & traceBE);
-}
-
 
 void ml_mem_backoff(void) {
 
diff --git a/osfmk/ppc/machine_routines.h b/osfmk/ppc/machine_routines.h
index 46c834146..08cb102e7 100644
--- a/osfmk/ppc/machine_routines.h
+++ b/osfmk/ppc/machine_routines.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -90,6 +96,9 @@ extern vm_offset_t		ml_static_ptovirt(
 extern vm_offset_t		ml_vtophys(
 							vm_offset_t			vaddr);
 
+vm_size_t ml_nofault_copy(
+	vm_offset_t virtsrc, vm_offset_t virtdst, vm_size_t size);
+
 /* PCI config cycle probing */
 extern boolean_t		ml_probe_read(
 							vm_offset_t			paddr,
@@ -202,6 +211,11 @@ extern void				bzero_phys(
 							addr64_t			phys_address,
 							uint32_t			length);
 
+/* Zero bytes starting at a physical address that's uncacheable */
+extern void				bzero_phys_nc(
+							addr64_t			phys_address,
+							uint32_t			length);
+
 #endif /* KERNEL_PRIVATE */
 
 #ifdef	XNU_KERNEL_PRIVATE
@@ -223,18 +237,6 @@ extern vm_offset_t		ml_static_malloc(
 
 #endif /* PEXPERT_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */
 
-#if	defined(BSD_KERNEL_PRIVATE) || defined(MACH_KERNEL_PRIVATE)
-
-extern int				set_be_bit(
-							void);
-
-extern int				clr_be_bit(
-							void);
-
-extern int				be_tracing(
-							void);
-
-#endif /* BSD_KERNEL_PRIVATE || MACH_KERNEL_PRIVATE */
 
 #ifdef	MACH_KERNEL_PRIVATE
 extern void				ml_init_interrupt(
@@ -318,6 +320,8 @@ extern unsigned int		ml_scom_read(
 
 extern uint32_t 		ml_hdec_ratio(void);
 
+extern int boffSettingsInit;
+
 #endif /* KERNEL_PRIVATE */
 
 #endif /* _PPC_MACHINE_ROUTINES_H_ */
diff --git a/osfmk/ppc/machine_routines_asm.s b/osfmk/ppc/machine_routines_asm.s
index e7c5916ba..45bfb5b5e 100644
--- a/osfmk/ppc/machine_routines_asm.s
+++ b/osfmk/ppc/machine_routines_asm.s
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <ppc/asm.h>
 #include <ppc/proc_reg.h>
@@ -756,34 +762,72 @@ LEXT(timer_grab)
 
 ;			Force a line boundry here
 			.align  5
-			.globl  EXT(timer_event)
+			.globl  EXT(thread_timer_event)
+ 
+LEXT(thread_timer_event)
+			mfsprg	r10,1							; Get the current activation
+			lwz		r10,ACT_PER_PROC(r10)			; Get the per_proc block
+			addi	r10,r10,PP_PROCESSOR
+			lwz		r11,THREAD_TIMER(r10)
+
+			lwz		r9,TIMER_LOW(r11)
+			lwz		r7,TIMER_TSTAMP(r11)
+			lwz		r8,TIMER_TSTAMP+4(r11)
+			subfc	r8,r8,r4
+			subfe	r7,r7,r3
+			addc	r8,r8,r9
+			addze.	r7,r7
+			beq++	0f
+
+			lwz		r6,TIMER_HIGH(r11)
+			add		r7,r7,r6
+			stw		r7,TIMER_HIGHCHK(r11)
+			eieio
+			stw		r8,TIMER_LOW(r11)
+			eieio
+	 		stw		r7,TIMER_HIGH(r11)
+			b		1f
+
+0:			stw		r8,TIMER_LOW(r11)
+
+1:			stw		r5,THREAD_TIMER(r10)
+			stw		r3,TIMER_TSTAMP(r5)
+			stw		r4,TIMER_TSTAMP+4(r5)
+			blr
+
+;			Force a line boundry here
+			.align  5
+			.globl  EXT(state_event)
  
-LEXT(timer_event)
+LEXT(state_event)
 			mfsprg	r10,1							; Get the current activation
 			lwz		r10,ACT_PER_PROC(r10)			; Get the per_proc block
 			addi	r10,r10,PP_PROCESSOR
-			lwz		r11,CURRENT_TIMER(r10)
+			lwz		r11,CURRENT_STATE(r10)
 
 			lwz		r9,TIMER_LOW(r11)
-			lwz		r2,TIMER_TSTAMP(r11)
-			add		r0,r9,r3
-			subf	r5,r2,r0
-			cmplw	r5,r9
-			bge++	0f
+			lwz		r7,TIMER_TSTAMP(r11)
+			lwz		r8,TIMER_TSTAMP+4(r11)
+			subfc	r8,r8,r4
+			subfe	r7,r7,r3
+			addc	r8,r8,r9
+			addze.	r7,r7
+			beq++	0f
 
 			lwz		r6,TIMER_HIGH(r11)
-			addi	r6,r6,1
-			stw		r6,TIMER_HIGHCHK(r11)
+			add		r7,r7,r6
+			stw		r7,TIMER_HIGHCHK(r11)
 			eieio
-			stw		r5,TIMER_LOW(r11)
+			stw		r8,TIMER_LOW(r11)
 			eieio
-	 		stw		r6,TIMER_HIGH(r11)
+	 		stw		r7,TIMER_HIGH(r11)
 			b		1f
 
-0:			stw		r5,TIMER_LOW(r11)
+0:			stw		r8,TIMER_LOW(r11)
 
-1:			stw		r4,CURRENT_TIMER(r10)
-			stw		r3,TIMER_TSTAMP(r4)
+1:			stw		r5,CURRENT_STATE(r10)
+			stw		r3,TIMER_TSTAMP(r5)
+			stw		r4,TIMER_TSTAMP+4(r5)
 			blr
 
 /*  Set machine into idle power-saving mode. 
diff --git a/osfmk/ppc/machine_rpc.h b/osfmk/ppc/machine_rpc.h
index 087c562d5..ffbf6c762 100644
--- a/osfmk/ppc/machine_rpc.h
+++ b/osfmk/ppc/machine_rpc.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002,2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/machlimits.h b/osfmk/ppc/machlimits.h
index 453c62f15..b43f64958 100644
--- a/osfmk/ppc/machlimits.h
+++ b/osfmk/ppc/machlimits.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/machparam.h b/osfmk/ppc/machparam.h
index 7df7e9afc..b5f5374cd 100644
--- a/osfmk/ppc/machparam.h
+++ b/osfmk/ppc/machparam.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/mappings.c b/osfmk/ppc/mappings.c
index e92ce2469..b6430b79b 100644
--- a/osfmk/ppc/mappings.c
+++ b/osfmk/ppc/mappings.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	This file is used to maintain the virtual to real mappings for a PowerPC machine.
@@ -77,7 +83,9 @@ extern unsigned int DebugWork;						/* (BRINGUP) */
 void mapping_verify(void);
 void mapping_phys_unused(ppnum_t pa);
 
-int nx_enabled = 0;			/* enable no-execute protection */
+int nx_enabled = 1;			/* enable no-execute protection */
+int allow_data_exec  = VM_ABI_32;	/* 32-bit apps may execute data by default, 64-bit apps may not */
+int allow_stack_exec = VM_ABI_32;	/* 32-bit apps may execute from the stack by default, 64-bit apps may not */
 
 /*
  *  ppc_prot translates Mach's representation of protections to that of the PPC hardware.
@@ -229,7 +237,7 @@ addr64_t mapping_remove(pmap_t pmap, addr64_t va) {		/* Remove a single mapping
 		case mapRtNotFnd:
 			return (nextva | 1);						/* Nothing found to unmap */
 		default:
-			panic("mapping_remove: hw_rem_map failed - pmap = %08X, va = %016llX, code = %08X\n",
+			panic("mapping_remove: hw_rem_map failed - pmap = %p, va = %016llX, code = %p\n",
 				pmap, va, mp);
 			break;
 	}
@@ -255,7 +263,7 @@ addr64_t mapping_remove(pmap_t pmap, addr64_t va) {		/* Remove a single mapping
 				case mapRtEmpty:						/* No guest mappings left to scrub */
 					break;
 				default:
-					panic("mapping_remove: hw_scrub_guest failed - physent = %08X, code = %08X\n",
+					panic("mapping_remove: hw_scrub_guest failed - physent = %p, code = %p\n",
 						physent, mp);					/* Cry havoc, cry wrack,
 															at least we die with harness on our backs */
 					break;
@@ -316,17 +324,17 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags,
 
 	pcf = (flags & mmFlgPcfg) >> 24;							/* Get the physical page config index */
 	if(!(pPcfg[pcf].pcfFlags)) {								/* Validate requested physical page configuration */
-		panic("mapping_make: invalid physical page configuration request - pmap = %08X, va = %016llX, cfg = %d\n",
+		panic("mapping_make: invalid physical page configuration request - pmap = %p, va = %016llX, cfg = %d\n",
 			pmap, va, pcf);
 	}
 	
 	psmask = (1ULL << pPcfg[pcf].pcfPSize) - 1;					/* Mask to isolate any offset into a page */
 	if(va & psmask) {											/* Make sure we are page aligned on virtual */
-		panic("mapping_make: attempt to map unaligned vaddr - pmap = %08X, va = %016llX, cfg = %d\n",
+		panic("mapping_make: attempt to map unaligned vaddr - pmap = %p, va = %016llX, cfg = %d\n",
 			pmap, va, pcf);
 	}
 	if(((addr64_t)pa << 12) & psmask) {							/* Make sure we are page aligned on physical */
-		panic("mapping_make: attempt to map unaligned paddr - pmap = %08X, pa = %016llX, cfg = %d\n",
+		panic("mapping_make: attempt to map unaligned paddr - pmap = %p, pa = %08X, cfg = %d\n",
 			pmap, pa, pcf);
 	}
 	
@@ -407,7 +415,7 @@ addr64_t mapping_make(pmap_t pmap, addr64_t va, ppnum_t pa, unsigned int flags,
 				return (colladdr | mapRtSmash);					/* Return colliding address, with some dirt added to avoid
 																   confusion if effective address is 0 */
 			default:
-				panic("mapping_make: hw_add_map failed - collision addr = %016llX, code = %02X, pmap = %08X, va = %016llX, mapping = %08X\n",
+				panic("mapping_make: hw_add_map failed - collision addr = %016llX, code = %02X, pmap = %p, va = %016llX, mapping = %p\n",
 					colladdr, rc, pmap, va, mp);				/* Die dead */
 		}
 		
@@ -449,19 +457,19 @@ mapping_t *mapping_find(pmap_t pmap, addr64_t va, addr64_t *nextva, int full) {
 
 		mp = hw_find_map(curpmap, curva, nextva);				/* Find the mapping for this address */
 		if((unsigned int)mp == mapRtBadLk) {					/* Did we lock up ok? */
-			panic("mapping_find: pmap lock failure - rc = %08X, pmap = %08X\n", mp, curpmap);	/* Die... */
+			panic("mapping_find: pmap lock failure - rc = %p, pmap = %p\n", mp, curpmap);	/* Die... */
 		}
 		
 		if(!mp || ((mp->mpFlags & mpType) < mpMinSpecial) || !full) break;		/* Are we done looking? */
 
 		if((mp->mpFlags & mpType) != mpNest) {					/* Don't chain through anything other than a nested pmap */
 			mapping_drop_busy(mp);								/* We have everything we need from the mapping */
-			mp = 0;												/* Set not found */
+			mp = NULL;												/* Set not found */
 			break;
 		}
 
 		if(nestdepth++ > 64) {									/* Have we nested too far down? */
-			panic("mapping_find: too many nested pmaps - va = %016llX, curva = %016llX, pmap = %08X, curpmap = %08X\n",
+			panic("mapping_find: too many nested pmaps - va = %016llX, curva = %016llX, pmap = %p, curpmap = %p\n",
 				va, curva, pmap, curpmap);
 		}
 		
@@ -507,7 +515,7 @@ mapping_protect(pmap_t pmap, addr64_t va, vm_prot_t prot, addr64_t *nextva) {	/*
 			break;
 			
 		default:
-			panic("mapping_protect: hw_protect failed - rc = %d, pmap = %08X, va = %016llX\n", ret, pmap, va);
+			panic("mapping_protect: hw_protect failed - rc = %d, pmap = %p, va = %016llX\n", ret, pmap, va);
 		
 	}
 
@@ -742,9 +750,10 @@ void mapping_clr_refmod(ppnum_t pa, unsigned int mask) {		/* Clears the referenc
  *		the reference bit. 
  */
 
-phys_entry_t *mapping_phys_lookup(ppnum_t pp, unsigned int *pindex) {	/* Finds the physical entry for the page */
-
-	int i;
+phys_entry_t *
+mapping_phys_lookup(ppnum_t pp, unsigned int *pindex)
+{	/* Finds the physical entry for the page */
+	unsigned int i;
 	
 	for(i = 0; i < pmap_mem_regions_count; i++) {				/* Walk through the list */
 		if(!(unsigned int)pmap_mem_regions[i].mrPhysTab) continue;	/* Skip any empty lists */
@@ -857,7 +866,7 @@ void mapping_adjust(void) {										/* Adjust free mappings */
 	}
 
 	mbn = mapCtl.mapcrel;										/* Get first pending release block */
-	mapCtl.mapcrel = 0;											/* Dequeue them */
+	mapCtl.mapcrel = NULL;											/* Dequeue them */
 	mapCtl.mapcreln = 0;										/* Set count to 0 */
 
 	hw_lock_unlock((hw_lock_t)&mapCtl.mapclock);				/* Unlock our stuff */
@@ -955,13 +964,13 @@ void mapping_free(struct mapping *mp) {							/* Release a mapping */
 
 			if(mapCtl.mapcnext == mb) {							/* Are we first on the list? */
 				mapCtl.mapcnext = mb->nextblok;					/* Unchain us */
-				if(!((unsigned int)mapCtl.mapcnext)) mapCtl.mapclast = 0;	/* If last, remove last */
+				if(!((unsigned int)mapCtl.mapcnext)) mapCtl.mapclast = NULL;	/* If last, remove last */
 			}
 			else {												/* We're not first */
 				for(mbn = mapCtl.mapcnext; mbn != 0; mbn = mbn->nextblok) {	/* Search for our block */
 					if(mbn->nextblok == mb) break;				/* Is the next one our's? */
 				}
-				if(!mbn) panic("mapping_free: attempt to release mapping block (%08X) not on list\n", mp);
+				if(!mbn) panic("mapping_free: attempt to release mapping block (%p) not on list\n", mp);
 				mbn->nextblok = mb->nextblok;					/* Dequeue us */
 				if(mapCtl.mapclast == mb) mapCtl.mapclast = mbn;	/* If last, make our predecessor last */
 			}
@@ -1087,15 +1096,16 @@ mapping_alloc(int lists) {								/* Obtain a mapping */
 							case mapRtNotFnd:
 								break;
 							default:
-								panic("mapping_alloc: hw_purge_map failed - pmap = %08X, va = %16llX, code = %08X\n", ckpmap, va, mp);
+								panic("mapping_alloc: hw_purge_map failed - pmap = %p, va = %16llX, code = %p\n", ckpmap, va, mp);
 								break;
 						}
 
-						if (mapRtNotFnd == ((unsigned int)mp & mapRetCode)) 
+						if (mapRtNotFnd == ((unsigned int)mp & mapRetCode)) {
 							if (do_rescan)
 								do_rescan = FALSE;
 							else
 								break;
+						}
 
 						va = nextva;
 					}
@@ -1159,13 +1169,13 @@ rescued:
     
     if ( !big ) {												/* if we need a small (64-byte) mapping */
         if(!(mindx = mapalc1(mb))) 								/* Allocate a 1-bit slot */
-            panic("mapping_alloc - empty mapping block detected at %08X\n", mb);
+            panic("mapping_alloc - empty mapping block detected at %p\n", mb);
     }
 	
 	if(mindx < 0) {												/* Did we just take the last one */
 		mindx = -mindx;											/* Make positive */
 		mapCtl.mapcnext = mb->nextblok;							/* Remove us from the list */
-		if(!((unsigned int)mapCtl.mapcnext)) mapCtl.mapclast = 0;	/* Removed the last one */
+		if(!((unsigned int)mapCtl.mapcnext)) mapCtl.mapclast = NULL;	/* Removed the last one */
 	}
 	
 	mapCtl.mapcfree--;											/* Decrement free count */
@@ -1315,7 +1325,7 @@ void mapping_free_init(vm_offset_t mbl, int perm, boolean_t locked) {
 	}
 	else {													/* Add to the free list */
 		
-		mb->nextblok = 0;									/* We always add to the end */
+		mb->nextblok = NULL;									/* We always add to the end */
 		mapCtl.mapcfree += MAPPERBLOK;						/* Bump count */
 		
 		if(!((unsigned int)mapCtl.mapcnext)) {				/* First entry on list? */
@@ -1524,6 +1534,21 @@ addr64_t kvtophys(vm_offset_t va) {
 
 }
 
+/*
+ *	kvtophys64(addr)
+ *
+ *	Convert a kernel virtual address to a 64-bit physical address
+ */
+vm_map_offset_t kvtophys64(vm_map_offset_t va) {
+
+	ppnum_t pa = pmap_find_phys(kernel_pmap, (addr64_t)va);
+
+	if (!pa)
+		return 0;
+	return (((vm_map_offset_t)pa) << 12) | (va & 0xfff);
+
+}
+
 /*
  *		void ignore_zero_fault(boolean_t) - Sets up to ignore or honor any fault on 
  *		page 0 access for the current thread.
@@ -1542,9 +1567,9 @@ void ignore_zero_fault(boolean_t type) {				/* Sets up to ignore or honor any fa
 }
 
 /*
- * nop in current ppc implementation
+ * no-op in current ppc implementation
  */
-void inval_copy_windows(__unused thread_t t)
+void inval_copy_windows(__unused thread_t th)
 {
 }
 
@@ -1566,18 +1591,19 @@ void inval_copy_windows(__unused thread_t t)
  *
  */
  
-kern_return_t hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, int which) {
- 
+kern_return_t
+hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, int which)
+{
 	vm_map_t map;
 	kern_return_t ret;
-	addr64_t nextva, vaddr, paddr;
-	register mapping_t *mp;
+	addr64_t nextva, vaddr = 0, paddr;
+	mapping_t *mp = NULL;
 	spl_t s;
 	unsigned int lop, csize;
 	int needtran, bothphys;
 	unsigned int pindex;
 	phys_entry_t *physent;
-	vm_prot_t prot;
+	vm_prot_t prot = 0;
 	int orig_which;
 
 	orig_which = which;
@@ -1615,7 +1641,7 @@ kern_return_t hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, in
 				mp = mapping_find(map->pmap, vaddr, &nextva, 1);	/* Find and busy the mapping */
 				if(!mp) {							/* Was it there? */
 					if(getPerProc()->istackptr == 0)
-						panic("copypv: No vaild mapping on memory %s %x", "RD", vaddr);
+						panic("copypv: No vaild mapping on memory %s %16llx", "RD", vaddr);
 
 					splx(s);						/* Restore the interrupt level */
 					ret = vm_fault(map, vm_map_trunc_page(vaddr), prot, FALSE, THREAD_UNINT, NULL, 0);	/* Didn't find it, try to fault it in... */
@@ -1641,7 +1667,7 @@ kern_return_t hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, in
 			
 				mapping_drop_busy(mp);				/* Go ahead and release the mapping for now */
 				if(getPerProc()->istackptr == 0)
-					panic("copypv: No vaild mapping on memory %s %x", "RDWR", vaddr);
+					panic("copypv: No vaild mapping on memory %s %16llx", "RDWR", vaddr);
 				splx(s);							/* Restore the interrupt level */
 				
 				ret = vm_fault(map, vm_map_trunc_page(vaddr), VM_PROT_READ | VM_PROT_WRITE, FALSE, THREAD_UNINT, NULL, 0);	/* check for a COW area */
@@ -1709,16 +1735,16 @@ void mapping_verify(void) {
 	
 	s = splhigh();											/* Don't bother from now on */
 
-	mbn = 0;												/* Start with none */
+	mbn = NULL;												/* Start with none */
 	for(mb = mapCtl.mapcnext; mb; mb = mb->nextblok) {		/* Walk the free chain */
 		if((mappingblok_t *)(mb->mapblokflags & 0x7FFFFFFF) != mb) {	/* Is tag ok? */
-			panic("mapping_verify: flags tag bad, free chain; mb = %08X, tag = %08X\n", mb, mb->mapblokflags);
+			panic("mapping_verify: flags tag bad, free chain; mb = %p, tag = %08X\n", mb, mb->mapblokflags);
 		}
 		mbn = mb;											/* Remember the last one */
 	}
 	
 	if(mapCtl.mapcnext && (mapCtl.mapclast != mbn)) {		/* Do we point to the last one? */
-		panic("mapping_verify: last pointer bad; mb = %08X, mapclast = %08X\n", mb, mapCtl.mapclast);
+		panic("mapping_verify: last pointer bad; mb = %p, mapclast = %p\n", mb, mapCtl.mapclast);
 	}
 	
 	relncnt = 0;											/* Clear count */
@@ -1746,14 +1772,14 @@ void mapping_phys_unused(ppnum_t pa) {
 
 	if(!(physent->ppLink & ~(ppLock | ppFlags))) return;	/* No one else is here */
 	
-	panic("mapping_phys_unused: physical page (%08X) in use, physent = %08X\n", pa, physent);
+	panic("mapping_phys_unused: physical page (%08X) in use, physent = %p\n", pa, physent);
 	
 }
 	
-void mapping_hibernate_flush(void)
+void
+mapping_hibernate_flush(void)
 {
-    int bank;
-    unsigned int page;
+    unsigned int page, bank;
     struct phys_entry * entry;
 
     for (bank = 0; bank < pmap_mem_regions_count; bank++)
diff --git a/osfmk/ppc/mappings.h b/osfmk/ppc/mappings.h
index 69cbe756d..0777f1b6f 100644
--- a/osfmk/ppc/mappings.h
+++ b/osfmk/ppc/mappings.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *		Header files for the hardware virtual memory mapping stuff 
@@ -98,7 +104,7 @@ typedef struct mem_region {
 #define PMAP_MEM_REGION_MAX 11
 
 extern mem_region_t pmap_mem_regions[PMAP_MEM_REGION_MAX + 1];
-extern int          pmap_mem_regions_count;
+extern unsigned int pmap_mem_regions_count;
 
 /* Prototypes */
 
@@ -464,7 +470,6 @@ extern void 		invalidateSegs(pmap_t pmap);				/* Invalidate the segment cache */
 extern struct phys_entry *pmap_find_physentry(ppnum_t pa);
 extern void			mapLog(unsigned int laddr, unsigned int type, addr64_t va);
 extern unsigned int	mapSkipListVerifyC(pmap_t pmap, unsigned long long *dumpa);
-extern void			fillPage(ppnum_t pa, unsigned int fill);
 extern kern_return_t hw_copypv_32(addr64_t source, addr64_t sink, unsigned int size, int which);
 
 extern void			hw_rem_all_gv(pmap_t pmap);					/* Remove all of a guest's mappings */
diff --git a/osfmk/ppc/mcount.s b/osfmk/ppc/mcount.s
index 38a4d9153..fd2518567 100644
--- a/osfmk/ppc/mcount.s
+++ b/osfmk/ppc/mcount.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <ppc/asm.h>
 #include <ppc/proc_reg.h>
diff --git a/osfmk/ppc/mem.h b/osfmk/ppc/mem.h
index 39609648c..e4ee80c0e 100644
--- a/osfmk/ppc/mem.h
+++ b/osfmk/ppc/mem.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/misc.c b/osfmk/ppc/misc.c
index b3a4ee934..807c03512 100644
--- a/osfmk/ppc/misc.c
+++ b/osfmk/ppc/misc.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/misc_asm.s b/osfmk/ppc/misc_asm.s
index 8ca2624a2..32d4b58df 100644
--- a/osfmk/ppc/misc_asm.s
+++ b/osfmk/ppc/misc_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/misc_protos.h b/osfmk/ppc/misc_protos.h
index edbf0fe19..27ad339bb 100644
--- a/osfmk/ppc/misc_protos.h
+++ b/osfmk/ppc/misc_protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -47,10 +53,7 @@ extern void		bzero_nc(
 					int					size);
 
 /* uncached-safe */
-extern void		bcopy_nc(
-					char				*from,
-					char				*to,
-					int					size);
+void bcopy_nc(const char *, char *, int);
 
 /* Physical to physical copy (ints must be disabled) */
 extern void		bcopy_phys(
@@ -72,8 +75,7 @@ extern void		phys_copy(
 extern void		machine_conf(
 					void);
 
-extern void		machine_startup(
-					boot_args			*args);
+extern void		machine_startup(void);
 
 extern void		ppc_vm_init(
 					uint64_t			ppc_mem_size,
@@ -97,13 +99,7 @@ extern void		hid5set64(
 extern void		Load_context(
 					thread_t			th);
 
-extern thread_t	Switch_context(
-					thread_t			old,
-					void				(*cont)(void),
-					thread_t			new);
-
-extern void		fpu_save(
-					struct facility_context *fpu_fc);
+thread_t Switch_context(thread_t, thread_continue_t, thread_t);
 
 extern void		vec_save(
 					struct facility_context *vec_fc);
@@ -114,14 +110,14 @@ extern void		toss_live_fpu(
 extern void		toss_live_vec(
 					struct facility_context *vec_fc);
 
-extern struct	savearea *enterDebugger(
-					unsigned int		trap,
-					struct savearea		*state,
-					unsigned int		dsisr);
+extern struct savearea *enterDebugger(unsigned int, struct savearea *,
+		unsigned int);
 
 extern void		draw_panic_dialog(
 					void);
 
+extern void		commit_paniclog(
+					void);
 #ifdef	DEBUG
 #define DPRINTF(x) { printf("%s : ",__FUNCTION__);printf x; }
 #endif	/* DEBUG */
@@ -129,6 +125,14 @@ extern void		draw_panic_dialog(
 #if MACH_ASSERT
 extern void		dump_thread(
 					thread_t			th);
-#endif 
+#endif /* MACH_ASSERT */
+
+/* XXX move to osfmk/ppc/debug.h or some other debug header */
+void print_backtrace(struct savearea *);
+int Call_Debugger(int, struct savearea *);
+int kdp_dump_trap(int, struct savearea *);
+void SysChoked(unsigned int, struct savearea *);
+int Call_DebuggerC(unsigned int, struct savearea *);
+void kdp_trap(unsigned int, struct savearea *);
 
 #endif /* _PPC_MISC_PROTOS_H_ */
diff --git a/osfmk/ppc/model_dep.c b/osfmk/ppc/model_dep.c
index b88e4af64..97f5276f2 100644
--- a/osfmk/ppc/model_dep.c
+++ b/osfmk/ppc/model_dep.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -52,6 +58,12 @@
  *
  * 	Utah $Hdr: model_dep.c 1.34 94/12/14$
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #include <debug.h>
 #include <mach_kdb.h>
@@ -83,18 +95,26 @@
 #include <ppc/exception.h>
 #include <ppc/hw_perfmon.h>
 #include <ppc/lowglobals.h>
+#include <ppc/machine_cpu.h>
+#include <ppc/db_machdep.h>
 
 #include <kern/clock.h>
 #include <kern/debug.h>
 #include <machine/trap.h>
 #include <kern/spl.h>
 #include <pexpert/pexpert.h>
+#include <kern/sched.h>
+#include <kern/task.h>
+#include <kern/machine.h>
+#include <vm/vm_map.h>
 
 #include <IOKit/IOPlatformExpert.h>
 
 #include <mach/vm_prot.h>
 #include <vm/pmap.h>
 #include <mach/time_value.h>
+#include <mach/mach_types.h>
+#include <mach/mach_vm.h>
 #include <machine/machparam.h>	/* for btop */
 
 #if	MACH_KDB
@@ -127,7 +147,7 @@ char env_buf[256];
 hw_lock_data_t debugger_lock;	/* debugger lock */
 hw_lock_data_t pbtlock;		/* backtrace print lock */
 
-int			debugger_cpu = -1;			/* current cpu running debugger	*/
+unsigned int debugger_cpu = (unsigned)-1; /* current cpu running debugger	*/
 int			debugger_debug = 0;			/* Debug debugger */
 int 		db_run_mode;				/* Debugger run mode */
 unsigned int debugger_sync = 0;			/* Cross processor debugger entry sync */
@@ -145,14 +165,15 @@ volatile unsigned int	cpus_holding_bkpts;	/* counter for number of cpus holding
 											   insert back breakpoints) */
 void unlock_debugger(void);
 void lock_debugger(void);
-void dump_backtrace(savearea *sv, unsigned int stackptr, unsigned int fence);
-void dump_savearea(savearea *sv, unsigned int fence);
-
-int packAsc (unsigned char *inbuf, unsigned int length);
+void dump_backtrace(struct savearea *sv,
+		    unsigned int stackptr,
+		    unsigned int fence);
+void dump_savearea(struct savearea *sv,
+		   unsigned int fence);
 
 #if !MACH_KDB
 boolean_t	db_breakpoints_inserted = TRUE;
-jmp_buf_t *db_recover = 0;
+jmp_buf_t *db_recover;
 #endif
 
 #if	MACH_KDB
@@ -168,11 +189,10 @@ extern int 	kdp_flag;
 #define	KDP_READY	0x1
 #endif
 
-boolean_t db_im_stepping = 0xFFFFFFFF;	/* Remember if we were stepping */
-
+unsigned int db_im_stepping = 0xFFFFFFFF; /* Remember if we were stepping */
 
-char *failNames[] = {	
 
+const char *failNames[] = {	
 	"Debugging trap",			/* failDebug */
 	"Corrupt stack",			/* failStack */
 	"Corrupt mapping tables",	/* failMapping */
@@ -187,31 +207,27 @@ char *failNames[] = {
 	"Unknown failure code"		/* Unknown failure code - must always be last */
 };
 
-char *invxcption = "Unknown code";
+const char *invxcption = "Unknown code";
 
-extern const char version[];
-extern char *trap_type[];
+static unsigned	commit_paniclog_to_nvram;
 
 #if !MACH_KDB
-void kdb_trap(int type, struct savearea *regs);
-void kdb_trap(int type, struct savearea *regs) {
-	return;
-}
-#endif
+void kdb_trap(__unused int type, __unused struct savearea *regs) {}
+#endif /* !MACH_KDB */
 
 #if !MACH_KDP
-void kdp_trap(int type, struct savearea *regs);
-void kdp_trap(int type, struct savearea *regs) {
-	return;
-}
-#endif
+void kdp_trap(__unused int type, __unused struct savearea *regs) {}
+#endif /* !MACH_KDP */
+
+extern int default_preemption_rate;
+extern int max_unsafe_quanta;
+extern int max_poll_quanta;
 
 void
-machine_startup(boot_args *args)
+machine_startup(void)
 {
 	int	boot_arg;
 	unsigned int wncpu;
-	unsigned int vmm_arg;
 
 	if (PE_parse_boot_arg("cpus", &wncpu)) {
 		if ((wncpu > 0) && (wncpu < MAX_CPUS))
@@ -223,12 +239,15 @@ machine_startup(boot_args *args)
 
 	if (PE_parse_boot_arg("debug", &boot_arg)) {
 		if (boot_arg & DB_HALT) halt_in_debugger=1;
-		if (boot_arg & DB_PRT) disableDebugOuput=FALSE; 
+		if (boot_arg & DB_PRT) disable_debug_output=FALSE; 
 		if (boot_arg & DB_SLOG) systemLogDiags=TRUE; 
 		if (boot_arg & DB_NMI) panicDebugging=TRUE; 
 		if (boot_arg & DB_LOG_PI_SCRN) logPanicDataToScreen=TRUE; 
 	}
 	
+	if (!PE_parse_boot_arg("nvram_paniclog", &commit_paniclog_to_nvram))
+		commit_paniclog_to_nvram = 1;
+
 	PE_parse_boot_arg("vmmforce", &lowGlo.lgVMMforcedFeats);
 
 	hw_lock_init(&debugger_lock);				/* initialize debugger lock */
@@ -258,23 +277,15 @@ machine_startup(boot_args *args)
 	}
 #endif /* MACH_KDB */
 	if (PE_parse_boot_arg("preempt", &boot_arg)) {
-		extern int default_preemption_rate;
-
 		default_preemption_rate = boot_arg;
 	}
 	if (PE_parse_boot_arg("unsafe", &boot_arg)) {
-		extern int max_unsafe_quanta;
-
 		max_unsafe_quanta = boot_arg;
 	}
 	if (PE_parse_boot_arg("poll", &boot_arg)) {
-		extern int max_poll_quanta;
-
 		max_poll_quanta = boot_arg;
 	}
 	if (PE_parse_boot_arg("yield", &boot_arg)) {
-		extern int sched_poll_yield_shift;
-
 		sched_poll_yield_shift = boot_arg;
 	}
 
@@ -288,9 +299,7 @@ machine_startup(boot_args *args)
 }
 
 char *
-machine_boot_info(
-	char *buf, 
-	vm_size_t size)
+machine_boot_info(__unused char *buf, __unused vm_size_t size)
 {
 	return(PE_boot_args());
 }
@@ -304,6 +313,7 @@ machine_conf(void)
 void
 machine_init(void)
 {
+	debug_log_init();
 	clock_config();
 /*	Note that we must initialize the stepper tables AFTER the clock is configured!!!!! */
 	if(pmsExperimental & 1) pmsCPUConf();	/* (EXPERIMENTAL) Initialize the stepper tables */
@@ -345,24 +355,18 @@ halt_cpu(void)
  * Machine-dependent routine to fill in an array with up to callstack_max
  * levels of return pc information.
  */
-void machine_callstack(
-	natural_t	*buf,
-	vm_size_t	callstack_max)
+void
+machine_callstack(__unused natural_t *buf, __unused vm_size_t callstack_max)
 {
 }
 #endif	/* MACH_ASSERT */
 
-
 void
 print_backtrace(struct savearea *ssp)
 {
-	unsigned int stackptr, *raddr, *rstack, trans, fence;
-	int i, frames_cnt, skip_top_frames, frames_max;
-	unsigned int store[8];			/* Buffer for real storage reads */
-	vm_offset_t backtrace_entries[32];
-	savearea *sv, *svssp;
-	int cpu;
-	savearea *psv;
+	unsigned int stackptr, fence;
+	struct savearea *sv, *svssp, *psv;
+	unsigned int cpu;
 
 /*
  *	We need this lock to make sure we don't hang up when we double panic on an MP.
@@ -370,14 +374,15 @@ print_backtrace(struct savearea *ssp)
 
 	cpu  = cpu_number();					/* Just who are we anyways? */
 	if(pbtcpu != cpu) {						/* Allow recursion */
-		hw_atomic_add((uint32_t *)&pbtcnt, 1); /* Remember we are trying */
+		(void)hw_atomic_add(&pbtcnt, 1); /* Remember we are trying */
 		while(!hw_lock_try(&pbtlock));		/* Spin here until we can get in. If we never do, well, we're crashing anyhow... */	
 		pbtcpu = cpu;						/* Mark it as us */	
 	}	
 
-	svssp = (savearea *)ssp;				/* Make this easier */
-	sv = 0;
-	if(current_thread()) sv = (savearea *)current_thread()->machine.pcb;	/* Find most current savearea if system has started */
+	svssp = (struct savearea *)ssp;				/* Make this easier */
+	sv = NULL;
+	if(current_thread())
+		sv = (struct savearea *)current_thread()->machine.pcb;	/* Find most current savearea if system has started */
 
 	fence = 0xFFFFFFFF;						/* Show we go all the way */
 	if(sv) fence = (unsigned int)sv->save_r1;	/* Stop at previous exception point */
@@ -385,31 +390,29 @@ print_backtrace(struct savearea *ssp)
 	if(!svssp) {							/* Should we start from stack? */
 		kdb_printf("Latest stack backtrace for cpu %d:\n", cpu_number());
 		__asm__ volatile("mr %0,r1" : "=r" (stackptr));	/* Get current stack */
-		dump_backtrace((savearea *)0,stackptr, fence);	/* Dump the backtrace */
+		dump_backtrace((struct savearea *)0,stackptr, fence);	/* Dump the backtrace */
 		if(!sv) {							/* Leave if no saveareas */
-			kdb_printf("\nKernel version:\n%s\n",version);	/* Print kernel version */
 			hw_lock_unlock(&pbtlock);		/* Allow another back trace to happen */
-			return;	
+			goto pbt_exit;
 		}
 	}
 	else {									/* Were we passed an exception? */
 		fence = 0xFFFFFFFF;					/* Show we go all the way */
 		if(svssp->save_hdr.save_prev) {
 			if((svssp->save_hdr.save_prev <= vm_last_addr) && ((unsigned int)pmap_find_phys(kernel_pmap, (addr64_t)svssp->save_hdr.save_prev))) {	/* Valid address? */	
-				psv = (savearea *)((unsigned int)svssp->save_hdr.save_prev);	/* Get the 64-bit back chain converted to a regualr pointer */
+				psv = (struct savearea *)((unsigned int)svssp->save_hdr.save_prev);	/* Get the 64-bit back chain converted to a regualr pointer */
 				fence = (unsigned int)psv->save_r1;	/* Stop at previous exception point */
 			}
 		}
 	
 		kdb_printf("Latest crash info for cpu %d:\n", cpu_number());
-		kdb_printf("   Exception state (sv=0x%08X)\n", sv);
+		kdb_printf("   Exception state (sv=%p)\n", svssp);
 		dump_savearea(svssp, fence);		/* Dump this savearea */	
 	}
 
 	if(!sv) {								/* Leave if no saveareas */
-		kdb_printf("\nKernel version:\n%s\n",version);	/* Print kernel version */
 		hw_lock_unlock(&pbtlock);			/* Allow another back trace to happen */
-		return;	
+		goto pbt_exit;
 	}
 	
 	kdb_printf("Proceeding back via exception chain:\n");
@@ -417,11 +420,11 @@ print_backtrace(struct savearea *ssp)
 	while(sv) {								/* Do them all... */
 		if(!(((addr64_t)((uintptr_t)sv) <= vm_last_addr) && 
 			(unsigned int)pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)sv)))) {	/* Valid address? */	
-			kdb_printf("   Exception state (sv=0x%08X) Not mapped or invalid. stopping...\n", sv);
+			kdb_printf("   Exception state (sv=%p) Not mapped or invalid. stopping...\n", sv);
 			break;
 		}
 		
-		kdb_printf("   Exception state (sv=0x%08X)\n", sv);
+		kdb_printf("   Exception state (sv=%p)\n", sv);
 		if(sv == svssp) {					/* Did we dump it already? */
 			kdb_printf("      previously dumped as \"Latest\" state. skipping...\n");
 		}
@@ -429,33 +432,37 @@ print_backtrace(struct savearea *ssp)
 			fence = 0xFFFFFFFF;				/* Show we go all the way */
 			if(sv->save_hdr.save_prev) {
 				if((sv->save_hdr.save_prev <= vm_last_addr) && ((unsigned int)pmap_find_phys(kernel_pmap, (addr64_t)sv->save_hdr.save_prev))) {	/* Valid address? */	
-					psv = (savearea *)((unsigned int)sv->save_hdr.save_prev);	/* Get the 64-bit back chain converted to a regualr pointer */
+					psv = (struct savearea *)((unsigned int)sv->save_hdr.save_prev);	/* Get the 64-bit back chain converted to a regualr pointer */
 					fence = (unsigned int)psv->save_r1;	/* Stop at previous exception point */
 				}
 			}
 			dump_savearea(sv, fence);		/* Dump this savearea */	
 		}	
 		
-		sv = CAST_DOWN(savearea *, sv->save_hdr.save_prev);	/* Back chain */ 
+		sv = CAST_DOWN(struct savearea *, sv->save_hdr.save_prev);	/* Back chain */ 
 	}
 	
-	kdb_printf("\nKernel version:\n%s\n",version);	/* Print kernel version */
 
 	pbtcpu = -1;							/* Mark as unowned */
 	hw_lock_unlock(&pbtlock);				/* Allow another back trace to happen */
-	hw_atomic_sub((uint32_t *) &pbtcnt, 1);  /* Show we are done */
+	(void)hw_atomic_sub(&pbtcnt, 1);  /* Show we are done */
 
 	while(pbtcnt);							/* Wait for completion */
+pbt_exit:
+	panic_display_system_configuration();
 
 	return;
 }
 
-void dump_savearea(savearea *sv, unsigned int fence) {
-
-	char *xcode;
+void
+dump_savearea(struct savearea *sv, unsigned int fence)
+{
+	const char *xcode;
 	
-	if(sv->save_exception > T_MAX) xcode = invxcption;	/* Too big for table */
-	else xcode = trap_type[sv->save_exception / 4];		/* Point to the type */
+	if(sv->save_exception > T_MAX)
+		xcode = invxcption;	/* Too big for table */
+	else
+		xcode = trap_type[sv->save_exception / 4];		/* Point to the type */
 	
 	kdb_printf("      PC=0x%08X; MSR=0x%08X; DAR=0x%08X; DSISR=0x%08X; LR=0x%08X; R1=0x%08X; XCP=0x%08X (%s)\n",
 		(unsigned int)sv->save_srr0, (unsigned int)sv->save_srr1, (unsigned int)sv->save_dar, sv->save_dsisr,
@@ -468,19 +475,18 @@ void dump_savearea(savearea *sv, unsigned int fence) {
 	return;
 }
 
-
-
 #define DUMPFRAMES 34
 #define LRindex 2
 
-void dump_backtrace(savearea *sv, unsigned int stackptr, unsigned int fence) {
+void dump_backtrace(struct savearea *sv, unsigned int stackptr, unsigned int fence) {
 
 	unsigned int bframes[DUMPFRAMES];
 	unsigned int  sframe[8], raddr, dumbo;
 	int i, index=0;
+//	char syminfo[80];
 	
 	kdb_printf("      Backtrace:\n");
-	if (sv != (savearea *)0) {
+	if (sv != (struct savearea *)0) {
 		bframes[0] = (unsigned int)sv->save_srr0;
 		bframes[1] = (unsigned int)sv->save_lr;
 		index = 2;
@@ -509,6 +515,8 @@ void dump_backtrace(savearea *sv, unsigned int stackptr, unsigned int fence) {
 
 		bframes[i] = sframe[LRindex];				/* Save the link register */
 		
+//		syms_formataddr((vm_offset_t)bframes[i], syminfo, sizeof (syminfo));
+//		kdb_printf("        %s\n", syminfo);
 		if(!i) kdb_printf("         ");				/* Indent first time */
 		else if(!(i & 7)) kdb_printf("\n         ");	/* Skip to new line every 8 */
 		kdb_printf("0x%08X ", bframes[i]);			/* Dump the link register */
@@ -521,14 +529,56 @@ void dump_backtrace(savearea *sv, unsigned int stackptr, unsigned int fence) {
 	
 }
 	
+void commit_paniclog(void) {
+	unsigned long pi_size = 0;
 
+	if (debug_buf_size > 0)	{
+		if (commit_paniclog_to_nvram) {
+			unsigned int bufpos;
+			
+			/* XXX Consider using the WKdm compressor in the
+			 * future, rather than just packing - would need to
+			 * be co-ordinated with crashreporter, which decodes
+			 * this post-restart. The compressor should be
+			 * capable of in-place compression.
+			 */
+			bufpos = packA(debug_buf, (unsigned) (debug_buf_ptr - debug_buf), debug_buf_size);
+			/* If compression was successful,
+			 * use the compressed length
+			 */
+			pi_size = bufpos ? bufpos : (unsigned) (debug_buf_ptr - debug_buf);
+
+			/* Truncate if the buffer is larger than a
+			 * certain magic size - this really ought to
+			 * be some appropriate fraction of the NVRAM
+			 * image buffer, and is best done in the
+			 * savePanicInfo() or PESavePanicInfo() calls
+			 * This call must save data synchronously,
+			 * since we can subsequently halt the system.
+			 */
+			kprintf("Attempting to commit panic log to NVRAM\n");
+			/* N.B.: This routine (currently an IOKit wrapper that
+			 * calls through to the appropriate platform NVRAM
+			 * driver, must be panic context safe, i.e.
+			 * acquire no locks or require kernel services.
+			 * This does not appear to be the case currently
+			 * on some platforms, unfortunately (the driver
+			 * on command gate serialization).
+			 */
+			pi_size = PESavePanicInfo((unsigned char *)debug_buf,
+			    ((pi_size > 2040) ? 2040 : pi_size));
+			/* Uncompress in-place, to allow debuggers to examine
+			 * the panic log.
+			 */
+			if (bufpos) 
+				unpackA(debug_buf, bufpos);
+		}
+	}
+}
 
 void 
 Debugger(const char	*message) {
 
-	int i;
-	unsigned int store[8];
-	unsigned long pi_size = 0;
 	spl_t spl;
 	
 	spl = splhigh();								/* No interruptions from here on */
@@ -557,54 +607,24 @@ Debugger(const char	*message) {
  */
 	if ( panicstr != (char *)0 )
 	{
-		/* diable kernel preemptions */
 		disable_preemption();
-	
-		/* everything should be printed now so copy to NVRAM
-		*/
-		if( debug_buf_size > 0)
-
-		  {
-		    /* Do not compress the panic log unless kernel debugging 
-		     * is disabled - the panic log isn't synced to NVRAM if 
-		     * debugging is enabled, and the panic log is valuable 
-		     * whilst debugging
-		     */
-		    if (!panicDebugging)
-		      {
-			unsigned int bufpos;
-			
-			/* Now call the compressor */
-			bufpos = packAsc (debug_buf, (unsigned int) (debug_buf_ptr - debug_buf) );
-			/* If compression was successful, use the compressed length 	           */
-			if (bufpos)
-			  {
-			    debug_buf_ptr = debug_buf + bufpos;
-			  }
-		      }
-		    /* Truncate if the buffer is larger than a certain magic 
-		     * size - this really ought to be some appropriate fraction
-		     * of the NVRAM image buffer, and is best done in the 
-		     * savePanicInfo() or PESavePanicInfo() calls 
-		     */
-		    pi_size = debug_buf_ptr - debug_buf;
-		    pi_size = PESavePanicInfo( debug_buf, ((pi_size > 2040) ? 2040 : pi_size));
-		  }
-			
-		if( !panicDebugging && (pi_size != 0) ) {
-			int	my_cpu;
-			int	tcpu;
+		/* Commit the panic log buffer to NVRAM, unless otherwise
+		 * specified via a boot-arg.
+		 */
+		commit_paniclog();
+		if(!panicDebugging) {
+			unsigned int my_cpu, tcpu;
 
 			my_cpu = cpu_number();
 			debugger_cpu = my_cpu;
 
-			hw_atomic_add(&debug_mode, 1);
+			(void)hw_atomic_add(&debug_mode, 1);
 			PerProcTable[my_cpu].ppe_vaddr->debugger_active++;
 			lock_debugger();
 
 			for(tcpu = 0; tcpu < real_ncpus; tcpu++) {
 				if(tcpu == my_cpu) continue;
-				hw_atomic_add(&debugger_sync, 1);
+				(void)hw_atomic_add(&debugger_sync, 1);
 				(void)cpu_signal(tcpu, SIGPdebug, 0 ,0);
 			}
 			(void)hw_cpu_sync(&debugger_sync, LockTimeOut);
@@ -613,13 +633,17 @@ Debugger(const char	*message) {
 
 		draw_panic_dialog();
 		
-		if( !panicDebugging && (pi_size != 0))
+		if(!panicDebugging) {
+#if CONFIG_EMBEDDED
+					PEHaltRestart(kPEPanicRestartCPU);
+#else
 					PEHaltRestart( kPEHangCPU );
+#endif
+		}
 
 		enable_preemption();
 	}
 
-
 	if ((current_debugger != NO_CUR_DB)) {			/* If there is a debugger configured, enter it */
 		printf("Debugger(%s)\n", message);
 		TRAP_DEBUGGER;
@@ -645,25 +669,38 @@ Debugger(const char	*message) {
  *		save_r3 contains the failure reason code.
  */
 
-void SysChoked(int type, savearea *sv) {			/* The system is bad dead */
-
+void
+SysChoked(unsigned int type, struct savearea *sv)
+{
 	unsigned int failcode;
-	
+	const char * const pmsg = "System Failure: cpu=%d; code=%08X (%s)\n";
 	mp_disable_preemption();
-	disableDebugOuput = FALSE;
+	disable_debug_output = FALSE;
 	debug_mode = TRUE;
 
 	failcode = (unsigned int)sv->save_r3;			/* Get the failure code */
 	if(failcode > failUnknown) failcode = failUnknown;	/* Set unknown code code */
 	
-	kprintf("System Failure: cpu=%d; code=%08X (%s)\n", cpu_number(), (unsigned int)sv->save_r3, failNames[failcode]);
-	kdb_printf("System Failure: cpu=%d; code=%08X (%s)\n", cpu_number(), (unsigned int)sv->save_r3, failNames[failcode]);
+	kprintf(pmsg, cpu_number(), (unsigned int)sv->save_r3, failNames[failcode]);
+	kdb_printf(pmsg, cpu_number(), (unsigned int)sv->save_r3, failNames[failcode]);
 
 	print_backtrace(sv);							/* Attempt to print backtrace */
-	Call_DebuggerC(type, sv);						/* Attempt to get into debugger */
 
-	if ((current_debugger != NO_CUR_DB)) Call_DebuggerC(type, sv);	/* Attempt to get into debugger */
+	/* Commit the panic log buffer to NVRAM, unless otherwise
+	 * specified via a boot-arg. For certain types of panics
+	 * which result in a "choke" exception, this may well
+	 * be inadvisable, and setting the nvram_paniclog=0
+	 * boot-arg may be useful.
+	 */
+
+	if (panicDebugging)
+		commit_paniclog();
+
+	Call_DebuggerC(type, sv);						/* Attempt to get into debugger */
 
+	if ((current_debugger != NO_CUR_DB))
+		Call_DebuggerC(type, sv);	/* Attempt to get into debugger */
+	panic_plain(pmsg, cpu_number(), (unsigned int)sv->save_r3, failNames[failcode]);
 }
 
 
@@ -673,17 +710,14 @@ void SysChoked(int type, savearea *sv) {			/* The system is bad dead */
  *	Never, ever, ever, ever enable interruptions from here on
  */
 
-int Call_DebuggerC(
-        int	type,
-        struct savearea *saved_state)
+int
+Call_DebuggerC(unsigned int type, struct savearea *saved_state)
 {
 	int				directcall, wait;
-	addr64_t		instr_ptr;
+	addr64_t		instr_ptr = 0ULL;
 	ppnum_t			instr_pp;
-	unsigned int 	instr;
-	int 			my_cpu, tcpu, wasdebugger;
-	struct per_proc_info *pp;
-	uint64_t nowtime, poptime;
+	unsigned int 	instr, tcpu, my_cpu;
+	int 			wasdebugger;
 
 	my_cpu = cpu_number();								/* Get our CPU */
 
@@ -696,7 +730,7 @@ int Call_DebuggerC(
 	}
 #endif
 	
-	hw_atomic_add(&debug_mode, 1);						/* Indicate we are in debugger */
+	(void)hw_atomic_add(&debug_mode, 1); /* Indicate we are in debugger */
 	PerProcTable[my_cpu].ppe_vaddr->debugger_active++;	/* Show active on our CPU */
 	
 	lock_debugger();									/* Insure that only one CPU is in debugger */
@@ -708,9 +742,9 @@ int Call_DebuggerC(
 
 	if (debugger_debug) {
 #if 0
-		kprintf("Call_DebuggerC(%d): %08X %08X, debact = %d\n", my_cpu, type, saved_state, debug_mode);	/* (TEST/DEBUG) */
+		kprintf("Call_DebuggerC(%d): %08X %08X, debact = %d\n", my_cpu, type, (uint32_t)saved_state, debug_mode);	/* (TEST/DEBUG) */
 #endif
-		printf("Call_Debugger: enter - cpu %d, is_slave %d, debugger_cpu %d, pc %08X\n",
+		printf("Call_Debugger: enter - cpu %d, is_slave %d, debugger_cpu %d, pc %08llX\n",
 		   my_cpu, PerProcTable[my_cpu].ppe_vaddr->debugger_is_slave, debugger_cpu, saved_state->save_srr0);
 	}
 	
@@ -727,7 +761,8 @@ int Call_DebuggerC(
 #endif
 
 	if (db_breakpoints_inserted) cpus_holding_bkpts++;	/* Bump up the holding count */
-	if (debugger_cpu == -1 && !PerProcTable[my_cpu].ppe_vaddr->debugger_is_slave) {
+	if ((debugger_cpu == (unsigned)-1) &&
+		!PerProcTable[my_cpu].ppe_vaddr->debugger_is_slave) {
 #if 0
 		if (debugger_debug) kprintf("Call_DebuggerC(%d): lasttrace = %08X\n", my_cpu, lastTrace);	/* (TEST/DEBUG) */
 #endif
@@ -738,7 +773,7 @@ int Call_DebuggerC(
 
 		for(tcpu = 0; tcpu < real_ncpus; tcpu++) {		/* Stop all the other guys */
 			if(tcpu == my_cpu) continue;				/* Don't diddle ourselves */
-			hw_atomic_add(&debugger_sync, 1);			/* Count signal sent */
+			(void)hw_atomic_add(&debugger_sync, 1); /* Count signal sent */
 			(void)cpu_signal(tcpu, SIGPdebug, 0 ,0);	/* Tell 'em to enter debugger */
 		}
 		(void)hw_cpu_sync(&debugger_sync, LockTimeOut);	/* Wait for the other processors to enter debug */
@@ -748,7 +783,7 @@ int Call_DebuggerC(
 	
 
 	if (instr == TRAP_DIRECT_INST) {
-		disableDebugOuput = FALSE;
+		disable_debug_output = FALSE;
 		print_backtrace(saved_state);
 	}
 
@@ -872,29 +907,31 @@ debugger_exit:
 	if (wait) while(cpus_holding_bkpts);				/* Wait for breakpoints to clear */
 
 
-	hw_atomic_sub(&debug_mode, 1);						/* Set out of debug now */
+	(void)hw_atomic_sub(&debug_mode, 1); /* Set out of debug now */
 
 	return(1);											/* Exit debugger normally */
 
 debugger_error:
 	if(db_run_mode != STEP_ONCE) enable_preemption_no_check();	/* Enable preemption, but don't preempt here */
-	hw_atomic_sub(&debug_mode, 1);						/* Set out of debug now */
+	(void)hw_atomic_sub(&debug_mode, 1); /* Set out of debug now */
 	return(0);											/* Return in shame... */
 
 }
 
-void lock_debugger(void) {
-	int		my_cpu;
-	register int	i;
+void
+lock_debugger(void)
+{
+	unsigned int my_cpu;
 
 	my_cpu = cpu_number();								/* Get our CPU number */
 
-	while(1) {											/* Check until we get it */
-
-		if (debugger_cpu != -1 && debugger_cpu != my_cpu) continue;	/* Someone, not us, is debugger... */
-		if (hw_lock_try(&debugger_lock)) {				/* Get the debug lock */			
-			if (debugger_cpu == -1 || debugger_cpu == my_cpu) break;	/* Is it us? */
-			hw_lock_unlock(&debugger_lock);				/* Not us, release lock */
+	while(1) { /* Check until we get it */
+		if (debugger_cpu != (unsigned)-1 && debugger_cpu != my_cpu)
+			continue;	/* Someone, not us, is debugger... */
+		if (hw_lock_try(&debugger_lock)) { /* Get the debug lock */			
+			if (debugger_cpu == (unsigned)-1 || debugger_cpu == my_cpu)
+				break;	/* Is it us? */
+			hw_lock_unlock(&debugger_lock); /* Not us, release lock */
 		}
 	} 
 }
@@ -905,38 +942,102 @@ void unlock_debugger(void) {
 
 }
 
-struct pasc {
-  unsigned a: 7;
-  unsigned b: 7;
-  unsigned c: 7;
-  unsigned d: 7;
-  unsigned e: 7;
-  unsigned f: 7;
-  unsigned g: 7;
-  unsigned h: 7;
-}  __attribute__((packed));
+int patchInst(task_t task, addr64_t vaddr, uint32_t inst);
+int patchInst(task_t task, addr64_t vaddr, uint32_t inst)
+{
+	vm_map_t map;
+	addr64_t paddr;
+	uint32_t instr, nestingDepth;
+	kern_return_t ret;
+	vm_region_submap_short_info_data_64_t info;
+	mach_msg_type_number_t count;
+	mach_vm_address_t address;
+	mach_vm_size_t sizeOfRegion;
+	vm_prot_t reprotect;
+
+	if(task == TASK_NULL) return -1;		/* Leave if task is bogus... */
+
+	task_lock(task);						/* Make sure the task doesn't go anywhaere */
+	if (!task->active) {					/* Is is alive? */
+		task_unlock(task);					/* Nope, unlock */
+		return -1;							/* Not a active task, fail... */
+	}
+	map = task->map;						/* Get his map */
+	vm_map_reference_swap(map);				/* Don't let it go away */
+	task_unlock(task);						/* Unleash the task */
 
-typedef struct pasc pasc_t;
+	/* Find the memory permissions. */
+	nestingDepth=999999;					/* Limit recursion */
+	
+	count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
+	address = (mach_vm_address_t)vaddr;
+	sizeOfRegion = (mach_vm_size_t)4;
+
+	ret = mach_vm_region_recurse(map, &address, &sizeOfRegion, &nestingDepth, (vm_region_recurse_info_t)&info, &count);
+	if (ret != KERN_SUCCESS) {				/* Leave if it didn't work */
+		vm_map_deallocate(map);				/* Drop reference on map */
+		return (-1);			
+	}
 
-int packAsc (unsigned char *inbuf, unsigned int length)
-{
-  unsigned int i, j = 0;
-  pasc_t pack;
-
-  for (i = 0; i < length; i+=8)
-    {
-      pack.a = inbuf[i];
-      pack.b = inbuf[i+1];
-      pack.c = inbuf[i+2];
-      pack.d = inbuf[i+3];
-      pack.e = inbuf[i+4];
-      pack.f = inbuf[i+5];
-      pack.g = inbuf[i+6];
-      pack.h = inbuf[i+7];
-      bcopy ((char *) &pack, inbuf + j, 7);
-      j += 7;
-    }
-  if (0 != (i - length))
-    inbuf[j - (i - length)] &= 0xFF << (8-(i - length));
-  return j-(((i-length) == 7) ? 6 : (i - length));
+/*
+ *	We need to check if there could be a problem if the dtrace probes are being removed and the code is being
+ *	executed at the same time.  This sequence may leave us with no-execute turned on temporarily when we execute
+ *	through it.
+ */
+ 
+	if (!(info.protection & VM_PROT_WRITE)) {
+		/* Save the original protection values for restoration later */
+		reprotect = info.protection;
+
+		if (info.max_protection & VM_PROT_WRITE) {
+			/* The memory is not currently writable, but can be made writable. */
+			ret = mach_vm_protect(map, (mach_vm_offset_t)vaddr, (mach_vm_size_t)4, 0, reprotect | VM_PROT_WRITE);
+		} 
+		else {
+			/*
+			 * The memory is not currently writable, and cannot be made writable. We need to COW this memory.
+			 *
+			 * Strange, we can't just say "reprotect | VM_PROT_COPY", that fails.
+			 */
+			ret = mach_vm_protect(map, (mach_vm_offset_t)vaddr, (mach_vm_size_t)4, 0, VM_PROT_COPY | VM_PROT_READ | VM_PROT_WRITE);
+		}
+
+		if (ret != KERN_SUCCESS) {
+			vm_map_deallocate(map);			/* Drop reference on map */
+			return (-1);		
+		}
+		
+	} 
+	else {
+		/* The memory was already writable. */
+		reprotect = VM_PROT_NONE;
+	}
+
+	instr = inst;							/* Place instruction in local memory */
+	ret = vm_map_write_user(map, &inst, (vm_map_address_t)vaddr, (vm_size_t)4);	/* Write the instruction */
+	if (ret != KERN_SUCCESS) {				/* Leave if it didn't work */
+	
+		if (reprotect != VM_PROT_NONE) {
+			ret = mach_vm_protect (map, (mach_vm_offset_t)vaddr, (mach_vm_size_t)4, 0, reprotect);
+		}
+
+		vm_map_deallocate(map);				/* Drop reference on map */
+		return (-1);			
+	}
+
+	paddr = (addr64_t)pmap_find_phys(map->pmap, vaddr) << 12;	/* Find the physical address of the patched address */
+	if(!paddr) {							/* Is address mapped now? */
+		vm_map_deallocate(map);				/* Drop reference on map */
+		return 0;							/* Leave... */
+	}
+	paddr = paddr | (vaddr & 4095);			/* Construct physical address */
+	invalidate_icache64(paddr, 4, 1);		/* Flush out the instruction cache here */
+
+	if (reprotect != VM_PROT_NONE) {
+		ret = mach_vm_protect(map, (mach_vm_offset_t)vaddr, (mach_vm_size_t)4, 0, reprotect);
+	}
+
+	vm_map_deallocate(map);
+
+	return (0);
 }
diff --git a/osfmk/ppc/movc.s b/osfmk/ppc/movc.s
index b7ec62939..1e111ec0c 100644
--- a/osfmk/ppc/movc.s
+++ b/osfmk/ppc/movc.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/mp.h b/osfmk/ppc/mp.h
index 2c7da0d11..4b187fb69 100644
--- a/osfmk/ppc/mp.h
+++ b/osfmk/ppc/mp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/new_screen.h b/osfmk/ppc/new_screen.h
index b2e6189eb..ba84184ef 100644
--- a/osfmk/ppc/new_screen.h
+++ b/osfmk/ppc/new_screen.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -38,8 +44,5 @@ extern void clear_RGB16(int color);
 extern void adj_position(unsigned char C);
 extern void put_cursor(int color);
 extern void screen_put_char(unsigned char C);
-extern void initialize_screen(
-		Boot_Video * boot_vinfo,
-		unsigned int op);
 
 #endif /* _NEW_SCREEN_H_ */
diff --git a/osfmk/ppc/pcb.c b/osfmk/ppc/pcb.c
index 653b9b3f3..38569fc94 100644
--- a/osfmk/ppc/pcb.c
+++ b/osfmk/ppc/pcb.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,9 +63,11 @@
 #include <kern/misc_protos.h>
 #include <kern/mach_param.h>
 #include <kern/spl.h>
+#include <kern/machine.h>
 
 #include <vm/vm_map.h>
 #include <vm/vm_kern.h>
+#include <vm/vm_protos.h>
 
 #include <ppc/misc_protos.h>
 #include <ppc/cpu_internal.h>
@@ -75,6 +83,7 @@
 #include <ppc/vmachmon.h>
 #include <ppc/low_trace.h>
 #include <ppc/lowglobals.h>
+#include <ppc/fpu_protos.h>
 
 #include <sys/kdebug.h>
 
@@ -97,16 +106,15 @@ int   vec_switch_count = 0;
  * consider_machine_collect: try to collect machine-dependent pages
  */
 void
-consider_machine_collect()
+consider_machine_collect(void)
 {
     /*
-     * none currently available
+     * XXX none currently available
      */
-	return;
 }
 
 void
-consider_machine_adjust()
+consider_machine_adjust(void)
 {
         consider_mapping_adjust();
 }
@@ -133,7 +141,6 @@ machine_switch_context(
 	ppinfo = getPerProc();								/* Get our processor block */
 
 	ppinfo->old_thread = (unsigned int)old;
-	ppinfo->cpu_flags &= ~traceBE; 						 /* disable branch tracing if on */
 	       
 	/* Our context might wake up on another processor, so we must
 	 * not keep hot state in our FPU, it must go back to the pcb
@@ -191,17 +198,9 @@ machine_switch_context(
 		hw_blow_seg(lowGlo.lgUMWvaddr + 0x10000000ULL);	/* Blow off the second segment */
 	}
 
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_SCHED) | DBG_FUNC_NONE,
-		     old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
-
 	retval = Switch_context(old, continuation, new);
 	assert(retval != NULL);
 
-	if (branch_tracing_enabled()) {
-		ppinfo = getPerProc();							/* Get our processor block */
-	  	ppinfo->cpu_flags |= traceBE;  					/* restore branch tracing */
-	}
-
 	/* We've returned from having switched context, so we should be
 	 * back in the original context.
 	 */
@@ -217,16 +216,15 @@ machine_thread_create(
 	thread_t		thread,
 	task_t			task)
 {
-	savearea		*sv;									/* Pointer to newly allocated savearea */
-	unsigned int	*CIsTooLimited, i;
+	struct savearea		*sv;									/* Pointer to newly allocated savearea */
 
-	hw_atomic_add((uint32_t *)&saveanchor.savetarget, 4);	/* Account for the number of saveareas we think we "need"
+	(void)hw_atomic_add(&saveanchor.savetarget, 4);	/* Account for the number of saveareas we think we "need"
 															   for this activation */
-	assert(thread->machine.pcb == (savearea *)0);				/* Make sure there was no previous savearea */
+	assert(thread->machine.pcb == (struct savearea *)0);				/* Make sure there was no previous savearea */
 	
 	sv = save_alloc();										/* Go get us a savearea */
 		
-	bzero((char *)((unsigned int)sv + sizeof(savearea_comm)), (sizeof(savearea) - sizeof(savearea_comm)));	/* Clear it */
+	bzero((char *)((unsigned int)sv + sizeof(savearea_comm)), (sizeof(struct savearea) - sizeof(savearea_comm)));	/* Clear it */
 		
 	sv->save_hdr.save_prev = 0;								/* Clear the back pointer */
 	sv->save_hdr.save_flags = (sv->save_hdr.save_flags & ~SAVtype) | (SAVgeneral << SAVtypeshft);	/* Mark as in use */
@@ -264,11 +262,9 @@ void
 machine_thread_destroy(
 	thread_t		thread)
 {
-	register savearea *pcb, *ppsv;
-	register savearea_vec *vsv, *vpsv;
-	register savearea_fpu *fsv, *fpsv;
-	register savearea *svp;
-	register int i;
+	struct savearea *local_pcb, *ppsv;
+	savearea_vec *vsv, *vpsv;
+	savearea_fpu *fsv, *fpsv;
  	boolean_t intr;
 
 /*
@@ -291,11 +287,12 @@ machine_thread_destroy(
 	
 	while(vsv) {											/* Any VMX saved state? */
 		vpsv = vsv;											/* Remember so we can toss this */
+		/* XXX save_prev should be a void * 4425537 */
 		vsv = CAST_DOWN(savearea_vec *, vsv->save_hdr.save_prev);  /* Get one underneath our's */
-		save_release((savearea *)vpsv);						/* Release it */
+		save_release((struct savearea *)vpsv);						/* Release it */
 	}
 	
-	thread->machine.curctx->VMXsave = 0;					/* Kill chain */
+	thread->machine.curctx->VMXsave = NULL;					/* Kill chain */
  
  	toss_live_fpu(thread->machine.curctx);					/* Dump live float */
 
@@ -303,25 +300,27 @@ machine_thread_destroy(
 	
 	while(fsv) {											/* Any float saved state? */
 		fpsv = fsv;											/* Remember so we can toss this */
+		/* XXX save_prev should be a void * 4425537 */
 		fsv = CAST_DOWN(savearea_fpu *, fsv->save_hdr.save_prev);   /* Get one underneath our's */
-		save_release((savearea *)fpsv);						/* Release it */
+		save_release((struct savearea *)fpsv);						/* Release it */
 	}
 	
-	thread->machine.curctx->FPUsave = 0;					/* Kill chain */
+	thread->machine.curctx->FPUsave = NULL;					/* Kill chain */
 
 /*
  * free all regular saveareas.
  */
 
-	pcb = thread->machine.pcb;								/* Get the general savearea */
+	local_pcb = thread->machine.pcb;								/* Get the general savearea */
 	
-	while(pcb) {											/* Any float saved state? */
-		ppsv = pcb;											/* Remember so we can toss this */
-		pcb = CAST_DOWN(savearea *, pcb->save_hdr.save_prev);  /* Get one underneath our's */ 
+	while(local_pcb) {											/* Any float saved state? */
+		ppsv = local_pcb;											/* Remember so we can toss this */
+		/* XXX save_prev should be a void * 4425537 */
+		local_pcb = CAST_DOWN(struct savearea *, local_pcb->save_hdr.save_prev);  /* Get one underneath our's */ 
 		save_release(ppsv);									/* Release it */
 	}
 	
-	hw_atomic_sub((uint32_t *)&saveanchor.savetarget, 4);	/* Unaccount for the number of saveareas we think we "need" */
+	(void)hw_atomic_sub(&saveanchor.savetarget, 4);	/* Unaccount for the number of saveareas we think we "need" */
 
 	(void) ml_set_interrupts_enabled(intr);					/* Restore interrupts if enabled */
 
@@ -329,21 +328,20 @@ machine_thread_destroy(
 
 /*
  * act_machine_sv_free
- * release saveareas associated with an act. if flag is true, release
+ * release saveareas associated with a thread. if flag is true, release
  * user level savearea(s) too, else don't
  *
- * This code must run with interruptions disabled because an interrupt handler could use
- * floating point and/or vectors.  If this happens and the thread we are blowing off owns
- * the facility, we can deadlock.
+ * This code must run with interruptions disabled because an interrupt handler
+ * could use floating point and/or vectors.  If this happens and the thread we
+ * are blowing off owns the facility, we can deadlock.
  */
 void
-act_machine_sv_free(thread_t act)
+act_machine_sv_free(thread_t act,  __unused int flag)
 {
-	register savearea *pcb, *userpcb;
+	struct savearea *local_pcb, *userpcb;
 	register savearea_vec *vsv, *vpst, *vsvt;
 	register savearea_fpu *fsv, *fpst, *fsvt;
-	register savearea *svp;
-	register int i;
+	struct savearea *svp;
  	boolean_t intr;
 
 /*
@@ -376,18 +374,22 @@ act_machine_sv_free(thread_t act)
 		}
 	
 		vsv = act->machine.curctx->VMXsave;					/* Get the top vector savearea */
-		while(vsv && vsv->save_hdr.save_level) vsv = (savearea_vec *)vsv->save_hdr.save_prev;	/* Find user context if any */
+		while(vsv && vsv->save_hdr.save_level)	/* Find user context if any */
+			/* XXX save_prev should be a void * 4425537 */
+			vsv = CAST_DOWN(savearea_vec *,
+					vsv->save_hdr.save_prev);
 		
 		vsvt = act->machine.curctx->VMXsave;				/* Get the top of the chain */
 		act->machine.curctx->VMXsave = vsv;					/* Point to the user context */
-		act->machine.curctx->VMXlevel = 0;					/* Set the level to user */
+		act->machine.curctx->VMXlevel = NULL;					/* Set the level to user */
 		hw_lock_unlock((hw_lock_t)&act->machine.curctx->VMXsync);	/* Unlock */
 		
 		while(vsvt) {										/* Clear any VMX saved state */
 			if (vsvt == vsv) break;   						/* Done when hit user if any */
 			vpst = vsvt;									/* Remember so we can toss this */
-			vsvt = (savearea_vec *)vsvt->save_hdr.save_prev;	/* Get one underneath our's */		
-			save_ret((savearea *)vpst);						/* Release it */
+			/* XXX save_prev should be a void * 4425537 */
+			vsvt = CAST_DOWN(savearea_vec *, vsvt->save_hdr.save_prev);	/* Get one underneath our's */		
+			save_ret((struct savearea *)vpst);						/* Release it */
 		}
 		
 	}
@@ -401,18 +403,21 @@ act_machine_sv_free(thread_t act)
 		}
 		
 		fsv = act->machine.curctx->FPUsave;					/* Get the top floats savearea */
-		while(fsv && fsv->save_hdr.save_level) fsv = (savearea_fpu *)fsv->save_hdr.save_prev;	/* Find user context if any */
+		while(fsv && fsv->save_hdr.save_level)	/* Find user context if any */
+			/* XXX save_prev should be a void * */
+			fsv = CAST_DOWN(savearea_fpu *, fsv->save_hdr.save_prev);
 		
 		fsvt = act->machine.curctx->FPUsave;				/* Get the top of the chain */
 		act->machine.curctx->FPUsave = fsv;					/* Point to the user context */
-		act->machine.curctx->FPUlevel = 0;					/* Set the level to user */
+		act->machine.curctx->FPUlevel = NULL;					/* Set the level to user */
 		hw_lock_unlock((hw_lock_t)&act->machine.curctx->FPUsync);	/* Unlock */
 		
 		while(fsvt) {										/* Clear any VMX saved state */
 			if (fsvt == fsv) break;   						/* Done when hit user if any */
 			fpst = fsvt;									/* Remember so we can toss this */
-			fsvt = (savearea_fpu *)fsvt->save_hdr.save_prev;	/* Get one underneath our's */		
-			save_ret((savearea *)fpst);						/* Release it */
+			/* XXX save_prev should be a void * 4425537 */
+			fsvt = CAST_DOWN(savearea_fpu *, fsvt->save_hdr.save_prev);	/* Get one underneath our's */		
+			save_ret((struct savearea *)fpst);						/* Release it */
 		}
 		
 	}
@@ -421,16 +426,17 @@ act_machine_sv_free(thread_t act)
  * free all regular saveareas except a user savearea, if any
  */
 
-	pcb = act->machine.pcb;									/* Get the general savearea */
-	userpcb = 0;											/* Assume no user context for now */
+	local_pcb = act->machine.pcb;									/* Get the general savearea */
+	userpcb = NULL;											/* Assume no user context for now */
 	
-	while(pcb) {											/* Any float saved state? */
-		if (pcb->save_srr1 & MASK(MSR_PR)) {				/* Is this a user savearea? */
-			userpcb = pcb;									/* Remember so we can toss this */
+	while(local_pcb) {											/* Any float saved state? */
+		if (local_pcb->save_srr1 & MASK(MSR_PR)) {				/* Is this a user savearea? */
+			userpcb = local_pcb;									/* Remember so we can toss this */
 			break;
 		}
-		svp = pcb;											/* Remember this */
-		pcb = CAST_DOWN(savearea *, pcb->save_hdr.save_prev);  /* Get one underneath our's */ 
+		svp = local_pcb;											/* Remember this */
+		/* XXX save_prev should be a void * 4425537 */
+		local_pcb = CAST_DOWN(struct savearea *, local_pcb->save_hdr.save_prev);  /* Get one underneath our's */ 
 		save_ret(svp);										/* Release it */
 	}
 	
@@ -469,33 +475,15 @@ machine_thread_init(void)
 }
 
 #if MACH_ASSERT
-
 void
 dump_thread(thread_t th)
 {
-	printf(" thread @ 0x%x:\n", th);
-}
-
-int
-    dump_act(thread_t thr_act)
-{
-    if (!thr_act)
-	return(0);
-
-    printf("thread(0x%x)(%d): task=%x(%d)\n",
-	   thr_act, thr_act->ref_count,
-	   thr_act->task,   thr_act->task   ? thr_act->task->ref_count : 0);
-
-    printf("\tsusp=%x active=%x\n",
-	   thr_act->suspend_count, thr_act->active);
-
-    return((int)thr_act);
+	printf(" thread @ %p:\n", th);
 }
-
-#endif
+#endif /* MACH_ASSERT */
 
 user_addr_t 
-get_useraddr()
+get_useraddr(void)
 {
 	return(current_thread()->machine.upcb->save_srr0);
 }
@@ -514,7 +502,7 @@ machine_stack_detach(
 											thread, thread->priority,
 											thread->sched_pri, 0, 0);
 
-  act_machine_sv_free(thread);
+  act_machine_sv_free(thread, 0);	/* XXX flag == 0 OK? */
 
   stack = thread->kernel_stack;
   thread->kernel_stack = 0;
@@ -601,8 +589,6 @@ machine_stack_handoff(
 
 	ppinfo = getPerProc();								/* Get our processor block */
 
-	ppinfo->cpu_flags &= ~traceBE;						/* Turn off special branch trace */
-
 	if(real_ncpus > 1) {								/* This is potentially slow, so only do when actually SMP */
 		fowner = ppinfo->FPU_owner;						/* Cache this because it may change */
 		if(fowner) {									/* Is there any live context? */
@@ -629,10 +615,6 @@ machine_stack_handoff(
 	old->machine.specFlags &= ~OnProc;
 	new->machine.specFlags |= OnProc;
 
-	KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_STACK_HANDOFF) | DBG_FUNC_NONE,
-		     old->reason, (int)new, old->sched_pri, new->sched_pri, 0);
-
-
 	if(new->machine.specFlags & runningVM) {	/* Is the new guy running a VM? */
 		pmap_switch(new->machine.vmmCEntry->vmmPmap);	/* Switch to the VM's pmap */
 		ppinfo->VMMareaPhys = new->machine.vmmCEntry->vmmContextPhys;
@@ -656,14 +638,13 @@ machine_stack_handoff(
 	mp = (mapping_t *)&ppinfo->ppUMWmp;
 	mp->mpSpace = invalSpace;						/* Since we can't handoff in the middle of copy in/out, just invalidate */
 
-	if (branch_tracing_enabled()) 
-		ppinfo->cpu_flags |= traceBE;
-    
 	if(trcWork.traceMask) dbgTrace(0x9903, (unsigned int)old, (unsigned int)new, 0, 0);	/* Cut trace entry if tracing */    
     
   return;
 }
 
+void Call_continuation(thread_continue_t, void *, wait_result_t, vm_offset_t);
+
 /*
  * clean and initialize the current kernel stack and go to
  * the given continuation routine
diff --git a/osfmk/ppc/pmap.c b/osfmk/ppc/pmap.c
index ce05767b9..4301c37dd 100644
--- a/osfmk/ppc/pmap.c
+++ b/osfmk/ppc/pmap.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -117,6 +123,7 @@
 #include <ppc/exception.h>
 #include <ppc/low_trace.h>
 #include <ppc/lowglobals.h>
+#include <ppc/limits.h>
 #include <ddb/db_output.h>
 #include <machine/cpu_capabilities.h>
 
@@ -127,7 +134,7 @@ extern unsigned int	avail_remaining;
 unsigned int 	debugbackpocket;							/* (TEST/DEBUG) */
 
 vm_offset_t		first_free_virt;
-int          	current_free_region;						/* Used in pmap_next_page */
+unsigned int current_free_region;						/* Used in pmap_next_page */
 
 pmapTransTab *pmapTrans;									/* Point to the hash to pmap translations */
 struct phys_entry *phys_table;
@@ -187,7 +194,7 @@ struct phys_entry *pmap_find_physentry(ppnum_t pa)
 		return (struct phys_entry *)entry;
 	}
 //	kprintf("DEBUG - pmap_find_physentry: page 0x%08X not found\n", pa);
-	return 0;
+	return NULL;
 }
 
 /*
@@ -235,12 +242,11 @@ pmap_map(
 	unsigned int flags)
 {
 	unsigned int mflags;
+	addr64_t colladr;
 	mflags = 0;										/* Make sure this is initialized to nothing special */
 	if(!(flags & VM_WIMG_USE_DEFAULT)) {			/* Are they supplying the attributes? */
 		mflags = mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1);	/* Convert to our mapping_make flags */
 	}
-
-	addr64_t colladr;
 	
 	if (spa == epa) return(va);
 
@@ -263,7 +269,7 @@ pmap_map(
  *
  */
 void
-pmap_map_physical()
+pmap_map_physical(void)
 {
 	unsigned region;
 	uint64_t msize, size;
@@ -284,7 +290,7 @@ pmap_map_physical()
 				(mmFlgBlock | mmFlgPerm), (msize >> 12),
 				(VM_PROT_READ | VM_PROT_WRITE));
 			if (colladdr) {
-				panic ("pmap_map_physical: mapping failure - va = %016llX, pa = %08X, size = %08X, collision = %016llX\n",
+				panic ("pmap_map_physical: mapping failure - va = %016llX, pa = %016llX, size = %016llX, collision = %016llX\n",
 					   vaddr, (paddr >> 12), (msize >> 12), colladdr);
 			}
 
@@ -306,7 +312,6 @@ pmap_map_iohole(addr64_t paddr, addr64_t size)
 {
 
 	addr64_t vaddr, colladdr, msize;
-	uint32_t psize;
 
 	vaddr = paddr + lowGlo.lgPMWvaddr;						/* Get starting virtual address */		
 
@@ -318,7 +323,7 @@ pmap_map_iohole(addr64_t paddr, addr64_t size)
 			(mmFlgBlock | mmFlgPerm | mmFlgGuarded | mmFlgCInhib), (msize >> 12),
 			(VM_PROT_READ | VM_PROT_WRITE));
 		if (colladdr) {
-			panic ("pmap_map_iohole: mapping failed - va = %016llX, pa = %08X, size = %08X, collision = %016llX\n",
+			panic ("pmap_map_iohole: mapping failed - va = %016llX, pa = %016llX, size = %016llX, collision = %016llX\n",
 				   vaddr, (paddr >> 12), (msize >> 12), colladdr);
 		}
 
@@ -510,7 +515,7 @@ pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize)
 
 	phys_entry = (struct phys_entry *) addr;				/* Get pointer to physical table */
 
-	for (bank = 0; bank < pmap_mem_regions_count; bank++) {	/* Set pointer and initialize all banks of ram */
+	for (bank = 0; (unsigned)bank < pmap_mem_regions_count; bank++) {	/* Set pointer and initialize all banks of ram */
 		
 		pmap_mem_regions[bank].mrPhysTab = phys_entry;		/* Set pointer to the physical table for this bank */
 		
@@ -556,7 +561,7 @@ pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize)
 
 	current_free_region = 0;								/* Set that we will start allocating in bank 0 */
 	avail_remaining = 0;									/* Clear free page count */
-	for(bank = 0; bank < pmap_mem_regions_count; bank++) {	/* Total up all of the pages in the system that are available */
+	for(bank = 0; (unsigned)bank < pmap_mem_regions_count; bank++) {	/* Total up all of the pages in the system that are available */
 		avail_remaining += (pmap_mem_regions[bank].mrAEnd - pmap_mem_regions[bank].mrAStart) + 1;	/* Add in allocatable pages in this bank */
 	}
 
@@ -589,7 +594,7 @@ pmap_init(void)
 	/*
 	 *	Initialize list of freed up pmaps
 	 */
-	free_pmap_list = 0;					/* Set that there are no free pmaps */
+	free_pmap_list = NULL;					/* Set that there are no free pmaps */
 	free_pmap_count = 0;
 	simple_lock_init(&free_pmap_lock, 0);
 	
@@ -609,16 +614,17 @@ unsigned int pmap_free_pages(void)
  * If there are no more free entries, too bad. 
  */
 
-boolean_t pmap_next_page(ppnum_t *addrp)
+boolean_t
+pmap_next_page(ppnum_t *addrp)
 {
-		int i;
+	unsigned int i;
 
 	if(current_free_region >= pmap_mem_regions_count) return FALSE;	/* Return failure if we have used everything... */
-	
+
 	for(i = current_free_region; i < pmap_mem_regions_count; i++) {	/* Find the next bank with free pages */
 		if(pmap_mem_regions[i].mrAStart <= pmap_mem_regions[i].mrAEnd) break;	/* Found one */
 	}
-	
+
 	current_free_region = i;										/* Set our current bank */
 	if(i >= pmap_mem_regions_count) return FALSE;					/* Couldn't find a free page */
 
@@ -730,7 +736,7 @@ pmap_create(vm_map_size_t size, __unused boolean_t is_64bit)
 		pmapTrans[pmap->space].pmapVAddr = CAST_DOWN(unsigned int, pmap);	/* Set translate table virtual to point to us */
 	}
 
-	pmap->pmapVmmExt = 0;							/* Clear VMM extension block vaddr */
+	pmap->pmapVmmExt = NULL;						/* Clear VMM extension block vaddr */
 	pmap->pmapVmmExtPhys = 0;						/*  and the paddr, too */
 	pmap->pmapFlags = pmapKeyDef;					/* Set default key */
 	pmap->pmapCCtl = pmapCCtlVal;					/* Initialize cache control */
@@ -755,18 +761,18 @@ pmap_create(vm_map_size_t size, __unused boolean_t is_64bit)
 void
 pmap_destroy(pmap_t pmap)
 {
-	int ref_count;
+	uint32_t ref_count;
 	spl_t s;
 	pmap_t fore, aft;
 
 	if (pmap == PMAP_NULL)
 		return;
 
-	ref_count=hw_atomic_sub(&pmap->ref_count, 1);			/* Back off the count */
-	if(ref_count>0) return;									/* Still more users, leave now... */
-
-	if(ref_count < 0)										/* Did we go too far? */
+	if ((ref_count = hw_atomic_sub(&pmap->ref_count, 1)) == UINT_MAX) /* underflow */
 		panic("pmap_destroy(): ref_count < 0");
+	
+	if (ref_count > 0)
+		return; /* Still more users, leave now... */
 
 	if (!(pmap->pmapFlags & pmapVMgsaa)) {					/* Don't try this for a shadow assist guest */
 		pmap_unmap_sharedpage(pmap);						/* Remove any mapping of page -1 */
@@ -819,7 +825,8 @@ pmap_destroy(pmap_t pmap)
 void
 pmap_reference(pmap_t pmap)
 {
-	if (pmap != PMAP_NULL) hw_atomic_add(&pmap->ref_count, 1);	/* Bump the count */
+	if (pmap != PMAP_NULL)
+		(void)hw_atomic_add(&pmap->ref_count, 1); /* Bump the count */
 }
 
 /*
@@ -857,7 +864,7 @@ void pmap_remove_some_phys(
 				case mapRtNotFnd:				
 					break;						/* Mapping disappeared on us, retry */	
 				default:
-					panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %08X, pmap = %08X, code = %08X\n",
+					panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %p, pmap = %p, code = %p\n",
 							pp, pmap, mp);		/* Handle failure with our usual lack of tact */
 			}
 		} else { 
@@ -871,7 +878,7 @@ void pmap_remove_some_phys(
 				case mapRtNotFnd:				
 					break;						/* Mapping disappeared on us, retry */	
 				default:
-					panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %08X, pmap = %08X, code = %08X\n",
+					panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %p, pmap = %p, code = %p\n",
 							pp, pmap, mp);		/* Handle failure with our usual lack of tact */
 			}
 		}
@@ -879,7 +886,7 @@ void pmap_remove_some_phys(
 
 #if DEBUG	
 	if ((pmap->pmapFlags & pmapVMhost) && !pmap_verify_free(pa)) 
-		panic("pmap_remove_some_phys: cruft left behind - pa = %08X, pmap = %08X\n", pa, pmap);
+		panic("pmap_remove_some_phys: cruft left behind - pa = %08X, pmap = %p\n", pa, pmap);
 #endif
 
 	return;										/* Leave... */
@@ -974,7 +981,7 @@ pmap_page_protect(
 							break;				/* Mapping disappeared on us, retry */
 				case mapRtEmpty:
 							break;				/* Physent chain empty, we're done */
-				default:	panic("pmap_page_protect: hw_purge_phys failed - pp = %08X, code = %08X\n",
+				default:	panic("pmap_page_protect: hw_purge_phys failed - pp = %p, code = %p\n",
 								  pp, mp);		/* Handle failure with our usual lack of tact */
 			}
 		} while (mapRtEmpty != ((unsigned int)mp & mapRetCode));
@@ -1026,7 +1033,7 @@ unsigned int pmap_disconnect(
 						break;					/* Mapping disappeared on us, retry */
 			case mapRtEmpty:
 						break;					/* Physent chain empty, we're done */
-			default:	panic("hw_purge_phys: hw_purge_phys failed - pp = %08X, code = %08X\n",
+			default:	panic("hw_purge_phys: hw_purge_phys failed - pp = %p, code = %p\n",
 							  pp, mp);			/* Handle failure with our usual lack of tact */
 		}
 	} while (mapRtEmpty != ((unsigned int)mp & mapRetCode));
@@ -1157,7 +1164,7 @@ void pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot
 	colva = mapping_make(pmap, va, pa, mflags, size, prot);	/* Enter the mapping into the pmap */
 	
 	if(colva) {										/* If there was a collision, panic */
-		panic("pmap_map_block: mapping error %d, pmap = %08X, va = %016llX\n", (uint32_t)(colva & mapRetCode), pmap, va);
+		panic("pmap_map_block: mapping error %d, pmap = %p, va = %016llX\n", (uint32_t)(colva & mapRetCode), pmap, va);
 	}
 	
 	return;											/* Return */
@@ -1366,6 +1373,7 @@ pmap_sync_page_attributes_phys(ppnum_t pa)
 	pmap_sync_page_data_phys(pa);
 }
 
+#ifdef CURRENTLY_UNUSED_AND_UNTESTED
 /*
  * pmap_collect
  * 
@@ -1377,6 +1385,7 @@ pmap_collect(__unused pmap_t pmap)
 {
 	return;
 }
+#endif
 
 /*
  *	Routine:	pmap_activate
@@ -1446,60 +1455,6 @@ pmap_change_wiring(
 	return;												/* This is not used... */
 }
 
-/*
- * pmap_modify_pages(pmap, s, e)
- *	sets the modified bit on all virtual addresses v in the 
- *	virtual address range determined by [s, e] and pmap,
- *	s and e must be on machine independent page boundaries and
- *	s must be less than or equal to e.
- *
- *  Note that this function will not descend nested pmaps.
- */
-void
-pmap_modify_pages(
-	     pmap_t pmap,
-	     vm_map_offset_t sva, 
-	     vm_map_offset_t eva)
-{
-	spl_t		spl;
-	mapping_t	*mp;
-	ppnum_t		pa;
-	addr64_t		va, endva;
-	unsigned int	savetype;
-
-	if (pmap == PMAP_NULL) return;					/* If no pmap, can't do it... */
-	
-	va = sva & -4096;								/* Round to page */
-	endva = eva & -4096;							/* Round to page */
-
-	while (va < endva) {							/* Walk through all pages */
-
-		spl = splhigh();							/* We can't allow any loss of control here */
-	
-		mp = mapping_find(pmap, (addr64_t)va, &va, 0);	/* Find the mapping for this address */
-		
-		if(!mp) {									/* Is the page mapped? */
-			splx(spl);								/* Page not mapped, restore interruptions */
-			if((va == 0) || (va >= endva)) break;	/* We are done if there are no more or we hit the end... */
-			continue;								/* We are not done and there is more to check... */
-		}
-		
-		savetype = mp->mpFlags & mpType;			/* Remember the type */
-		pa = mp->mpPAddr;							/* Remember ppage because mapping may vanish after drop call */
-	
-		mapping_drop_busy(mp);						/* We have everything we need from the mapping */
-	
-		splx(spl);									/* Restore 'rupts */
-	
-		if(savetype != mpNormal) continue;			/* Can't mess around with these guys... */	
-		
-		mapping_set_mod(pa);						/* Set the modfied bit for this page */
-		
-		if(va == 0) break;							/* We hit the end of the pmap, might as well leave now... */
-	}
-	return;											/* Leave... */
-}
-
 /*
  * pmap_clear_modify(phys)
  *	clears the hardware modified ("dirty") bit for one
@@ -1617,8 +1572,8 @@ pmap_copy_part_page(
 {
 	addr64_t fsrc, fdst;
 
-	assert(((dst <<12) & PAGE_MASK+dst_offset+len) <= PAGE_SIZE);
-	assert(((src <<12) & PAGE_MASK+src_offset+len) <= PAGE_SIZE);
+	assert((((dst << 12) & PAGE_MASK) + dst_offset + len) <= PAGE_SIZE);
+	assert((((src << 12) & PAGE_MASK) + src_offset + len) <= PAGE_SIZE);
 
 	fsrc = ((addr64_t)src << 12) + src_offset;
 	fdst = ((addr64_t)dst << 12) + dst_offset;
@@ -1666,6 +1621,13 @@ void pmap_switch(pmap_t map)
 	return;										/* Bye, bye, butterfly... */
 }
 
+
+/*
+ * The PPC pmap can only nest segments of 256MB, aligned on a 256MB boundary.
+ */
+uint64_t pmap_nesting_size_min = 0x10000000ULL;
+uint64_t pmap_nesting_size_max = 0x10000000ULL;
+
 /*
  *	kern_return_t pmap_nest(grand, subord, vstart, size)
  *
@@ -1722,7 +1684,7 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t n
 	
 	if(colladdr) {										/* Did it collide? */
 		vend = vstart + size - 4096;					/* Point to the last page we would cover in nest */	
-		panic("pmap_nest: attempt to nest into a non-empty range - pmap = %08X, start = %016llX, end = %016llX\n",
+		panic("pmap_nest: attempt to nest into a non-empty range - pmap = %p, start = %016llX, end = %016llX\n",
 			grand, vstart, vend);
 	}
 	
@@ -1730,23 +1692,31 @@ kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t n
 }
 
 /*
- *	kern_return_t pmap_unnest(grand, vaddr)
+ *	kern_return_t pmap_unnest(grand, vaddr, size)
  *
  *	grand  = the pmap that we will nest subord into
  *	vaddr  = start of range in pmap to be unnested
+ *	size   = size of range in pmap to be unnested
  *
  *	Removes a pmap from another.  This is used to implement shared segments.
  *	On the current PPC processors, this is limited to segment (256MB) aligned
  *	segment sized ranges.
  */
 
-kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr) {
+kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) {
 			
 	unsigned int tstamp, i, mycpu;
 	addr64_t nextva;
 	spl_t s;
 	mapping_t *mp;
 		
+	if (size != pmap_nesting_size_min ||
+	    (vaddr & (pmap_nesting_size_min-1))) {
+		panic("pmap_unnest(vaddr=0x%016llx, size=0x016%llx): "
+		      "must be 256MB and aligned\n",
+		      vaddr, size);
+	}
+
 	s = splhigh();										/* Make sure interruptions are disabled */
 
 	mp = mapping_find(grand, vaddr, &nextva, 0);		/* Find the nested map */
@@ -1949,6 +1919,29 @@ addr64_t MapUserMemoryWindow(
 	return ((va & 0x0FFFFFFFULL) | lowGlo.lgUMWvaddr);	/* Pass back the kernel address we are to use */
 }
 
+#if CONFIG_DTRACE
+/*
+ * Constrain DTrace copyin/copyout actions
+ */
+extern kern_return_t dtrace_copyio_preflight(addr64_t);
+extern kern_return_t dtrace_copyio_postflight(addr64_t);
+
+kern_return_t dtrace_copyio_preflight(__unused addr64_t va)
+{
+	if (current_map() == kernel_map)
+		return KERN_FAILURE;
+	else
+		return KERN_SUCCESS;
+}
+ 
+kern_return_t dtrace_copyio_postflight(__unused addr64_t va)
+{
+	thread_t thread = current_thread();
+
+	thread->machine.umwSpace |= umwSwitchAway;
+	return KERN_SUCCESS;
+}
+#endif /* CONFIG_DTRACE */
 
 /*
  *	kern_return_t pmap_boot_map(size)
@@ -1996,7 +1989,7 @@ void pmap_init_sharedpage(vm_offset_t cpg){
 	
 		cpphys = pmap_find_phys(kernel_pmap, (addr64_t)cpg + cpoff);
 		if(!cpphys) {
-			panic("pmap_init_sharedpage: compage %08X not mapped in kernel\n", cpg + cpoff);
+			panic("pmap_init_sharedpage: compage %016llX not mapped in kernel\n", cpg + cpoff);
 		}
 		
 		cva = mapping_make(sharedPmap, (addr64_t)((uint32_t)_COMM_PAGE_BASE_ADDRESS) + cpoff,
@@ -2055,7 +2048,7 @@ void pmap_unmap_sharedpage(pmap_t pmap){
 		inter  = ml_set_interrupts_enabled(FALSE);	/* Disable interruptions for now */
 		mp = hw_find_map(pmap, 0xFFFFFFFFF0000000ULL, &nextva);	/* Find the mapping for this address */
 		if((unsigned int)mp == mapRtBadLk) {		/* Did we lock up ok? */
-			panic("pmap_unmap_sharedpage: mapping lock failure - rc = %08X, pmap = %08X\n", mp, pmap);	/* Die... */
+			panic("pmap_unmap_sharedpage: mapping lock failure - rc = %p, pmap = %p\n", mp, pmap);	/* Die... */
 		}
 		
 		gotnest = 0;								/* Assume nothing here */
@@ -2068,7 +2061,7 @@ void pmap_unmap_sharedpage(pmap_t pmap){
 		
 		if(!gotnest) return;						/* Leave if there isn't any nesting here */
 		
-		ret = pmap_unnest(pmap, 0xFFFFFFFFF0000000ULL);	/* Unnest the max 64-bit page */
+		ret = pmap_unnest(pmap, 0xFFFFFFFFF0000000ULL, 0x0000000010000000ULL);	/* Unnest the max 64-bit page */
 		
 		if(ret != KERN_SUCCESS) {					/* Did it work? */
 			panic("pmap_unmap_sharedpage: couldn't unnest shared page - ret = %08X\n", ret);
@@ -2097,3 +2090,4 @@ void pmap_disable_NX(pmap_t pmap) {
   
         pmap->pmapFlags |= pmapNXdisabled;
 }
+
diff --git a/osfmk/ppc/pmap.h b/osfmk/ppc/pmap.h
index 76ff6cd34..3edad1d4b 100644
--- a/osfmk/ppc/pmap.h
+++ b/osfmk/ppc/pmap.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -154,7 +160,7 @@ struct pmap {
 	shexlock		pmapSXlk;			/* Shared/Exclusive lock for mapping changes */
 	unsigned int	space;				/* space for this pmap */
 #define invalSpace 0x00000001			/* Predefined always invalid space */
-	int				ref_count;			/* reference count */
+	uint32_t	ref_count;			/* reference count */
 	unsigned int	pmapFlags;			/* Flags */
 #define pmapKeys	0x00000007			/* Keys and no execute bit to use with this pmap */
 #define pmapKeyDef	0x00000006			/* Default keys - Sup = 1, user = 1, no ex = 0 */
@@ -255,6 +261,7 @@ extern pmapTransTab *pmapTrans;			/* Space to pmap translate table */
 
 #define pmap_kernel()			(kernel_pmap)
 #define	pmap_resident_count(pmap)	((pmap)->stats.resident_count)
+#define	pmap_resident_max(pmap)		((pmap)->stats.resident_max)
 #define pmap_remove_attributes(pmap,start,end)
 #define pmap_copy(dpmap,spmap,da,len,sa)
 #define	pmap_update()
@@ -277,7 +284,8 @@ extern pmapTransTab *pmapTrans;			/* Space to pmap translate table */
 /* 
  * prototypes.
  */
-extern addr64_t	   	kvtophys(vm_offset_t va);					/* Get physical address from kernel virtual */
+extern addr64_t	   	kvtophys(vm_offset_t va);				/* Get physical address from kernel virtual */
+extern vm_map_offset_t kvtophys64(vm_map_offset_t va);				/* Get 64-bit physical address from kernel virtual */
 extern vm_offset_t	pmap_map(vm_offset_t va,
 				 vm_offset_t spa,
 				 vm_offset_t epa,
@@ -302,13 +310,9 @@ extern void invalidate_dcache(vm_offset_t va, unsigned length, boolean_t phys);
 extern void invalidate_dcache64(addr64_t va, unsigned length, boolean_t phys);
 extern void invalidate_icache(vm_offset_t va, unsigned length, boolean_t phys);
 extern void invalidate_icache64(addr64_t va, unsigned length, boolean_t phys);
-extern void pmap_sync_page_data_phys(ppnum_t pa);
-extern void pmap_sync_page_attributes_phys(ppnum_t pa);
 extern void pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
 extern int pmap_map_block_rc(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags);
 
-extern kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size);
-extern kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr);
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 extern void MapUserMemoryWindowInit(void);
 extern addr64_t MapUserMemoryWindow(vm_map_t map, addr64_t va);
@@ -318,14 +322,10 @@ extern int pmap_list_resident_pages(
 	vm_offset_t	*listp,
 	int		space);
 extern void pmap_init_sharedpage(vm_offset_t cpg);
-extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
-extern void pmap_unmap_sharedpage(pmap_t pmap);
 extern void pmap_disable_NX(pmap_t pmap);
 /* Not required for ppc: */
 static inline void pmap_set_4GB_pagezero(__unused pmap_t pmap) {}
 static inline void pmap_clear_4GB_pagezero(__unused pmap_t pmap) {}
 
-
-
 #endif /* _PPC_PMAP_H_ */
 
diff --git a/osfmk/kern/pms.c b/osfmk/ppc/pms.c
similarity index 86%
rename from osfmk/kern/pms.c
rename to osfmk/ppc/pms.c
index b706faa64..fb69f7618 100644
--- a/osfmk/kern/pms.c
+++ b/osfmk/ppc/pms.c
@@ -1,29 +1,36 @@
 /*
- * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <machine/machine_routines.h>
 #include <machine/machine_cpu.h>
 #ifdef __ppc__
 # include <ppc/exception.h>
 # include <ppc/misc_protos.h>
+# include <ppc/cpu_internal.h>
 #else
 # include <i386/cpu_data.h>
 # include <i386/misc_protos.h>
@@ -34,6 +41,8 @@
 #include <kern/kalloc.h>
 #include <vm/vm_protos.h>
 
+extern int is_suser(void);
+
 static uint32_t pmsSyncrolator = 0;					/* Only one control operation at a time please */
 uint32_t pmsBroadcastWait = 0;						/* Number of outstanding broadcasts */
 
@@ -41,7 +50,7 @@ int pmsInstalled = 0;								/* Power Management Stepper can run and has table i
 int pmsExperimental = 0;							/* Power Management Stepper in experimental mode */
 decl_simple_lock_data(,pmsBuildLock)				/* Make sure only one guy can replace table  at the same time */
 
-static pmsDef *altDpmsTab = 0;						/* Alternate step definition table */
+static pmsDef *altDpmsTab;						/* Alternate step definition table */
 static uint32_t altDpmsTabSize = 0;					/* Size of alternate step definition table */
 
 pmsDef pmsDummy = {									/* This is the dummy step for initialization.  All it does is to park */
@@ -53,18 +62,14 @@ pmsDef pmsDummy = {									/* This is the dummy step for initialization.  All i
 	.pmsNext = pmsPrepSleep							/* We always park */
 };
 
-#ifdef __ppc__
 pmsStat pmsStatsd[4][pmsMaxStates];					/* Generate enough statistics blocks for 4 processors */
-#else
-pmsStat pmsStatsd[8][pmsMaxStates];					/* Generate enough statistics blocks for 8 processors */
-#endif
 
 pmsCtl pmsCtls = {									/* Power Management Stepper control */
-	.pmsStats = &pmsStatsd
+	.pmsStats = pmsStatsd,
 };
 
-pmsSetFunc_t pmsFuncTab[pmsSetFuncMax] = {0};		/* This is the function index table */
-pmsQueryFunc_t pmsQueryFunc = 0;					/* Pointer to pmsQuery function */
+pmsSetFunc_t pmsFuncTab[pmsSetFuncMax] = {NULL};		/* This is the function index table */
+pmsQueryFunc_t pmsQueryFunc;					/* Pointer to pmsQuery function */
 uint32_t pmsPlatformData = 0;						/* Data provided by and passed to platform functions */
 
 #ifdef __ppc__
@@ -81,16 +86,15 @@ uint32_t pmsPlatformData = 0;						/* Data provided by and passed to platform fu
  *	Do any initialization needed
  */
  
-void pmsInit(void) {
-
+void
+pmsInit(void)
+{
 	int i;
 	
 	simple_lock_init(&pmsBuildLock, 0);				/* Initialize the build lock */
 	for(i = 0; i < pmsMaxStates; i++) pmsCtls.pmsDefs[i] = &pmsDummy;	/* Initialize the table to dummy steps */
 
 	pmsCPUMachineInit();
-
-	return;
 }
 
 
@@ -102,18 +106,17 @@ void pmsInit(void) {
  *
  */
  
- void pmsStart(void) {
-
+ void
+ pmsStart(void)
+{
  	boolean_t	intr;
 
-	if(!pmsInstalled) return;						/* We can't do this if no table installed */
+	if(!pmsInstalled)
+		return;						/* We can't do this if no table installed */
 
 	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
 	pmsRun(pmsStartUp);								/* Start running the stepper everywhere */
 	(void)ml_set_interrupts_enabled(intr);			/* Restore interruptions */
-
-	return;
- 
  }
  
 
@@ -127,18 +130,17 @@ void pmsInit(void) {
  *
  */
  
-void pmsPark(void) {
-
+void
+pmsPark(void)
+{
 	boolean_t	intr;
 
-	if(!pmsInstalled) return;						/* We can't do this if no table installed */
+	if(!pmsInstalled)
+		return;						/* We can't do this if no table installed */
 
 	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
 	pmsSetStep(pmsParked, 0);						/* Park the stepper */
 	(void)ml_set_interrupts_enabled(intr);			/* Restore interruptions */
-	
-	return;
-
 }
  
 
@@ -147,18 +149,19 @@ void pmsPark(void) {
  *	Interrupts must be off...
  */
 
-void pmsDown(void) {
-
+void
+pmsDown(void)
+{
 	PER_PROC_INFO *pp;
 	uint32_t nstate;
 	
 	pp = GET_PER_PROC_INFO();								/* Get our per_proc */
 	
-	if(!pmsInstalled || pp->pms.pmsState == pmsParked) return;		/* No stepping if parked or not installed */
+	if(!pmsInstalled || pp->pms.pmsState == pmsParked)
+		return;		/* No stepping if parked or not installed */
 	
 	nstate = pmsCtls.pmsDefs[pp->pms.pmsState]->pmsDown;	/* Get the downward step */
 	pmsSetStep(nstate, 0);							/* Step to it */
-	return;
 }
 
 
@@ -172,8 +175,9 @@ void pmsDown(void) {
 int pmsStepIdleSneaks;
 int pmsStepIdleTries;
  
-void pmsStep(int timer) {
-
+void
+pmsStep(int timer)
+{
 	PER_PROC_INFO	*pp;
 	uint32_t	nstate;
 	uint32_t	tstate;
@@ -245,7 +249,6 @@ void pmsStep(int timer) {
 	}
 
 	pmsSetStep(nstate, dir);
-	return;
 }
 
 
@@ -257,8 +260,9 @@ void pmsStep(int timer) {
  *
  */
 
-void pmsSetStep(uint32_t nstep, int dir) {
-
+void
+pmsSetStep(uint32_t nstep, int dir)
+{
 	PER_PROC_INFO *pp;
 	uint32_t pstate, nCSetCmd, mCSetCmd;
 	pmsDef *pnstate, *pcstate;
@@ -331,9 +335,6 @@ void pmsSetStep(uint32_t nstep, int dir) {
 		
 		nstep = pnstate->pmsNext;					/* Chain on to the next */
 	}
-
-	return;
-
 }
 
 /*
@@ -341,14 +342,16 @@ void pmsSetStep(uint32_t nstep, int dir) {
  *
  */
  
-void pmsRunLocal(uint32_t nstep) {
-
+void
+pmsRunLocal(uint32_t nstep)
+{
 	PER_PROC_INFO *pp;
 	uint32_t lastState;
 	int cpu, i;
 	boolean_t	intr;
 
-	if(!pmsInstalled) return;						/* Ignore this if no step programs installed... */
+	if(!pmsInstalled) /* Ignore this if no step programs installed... */
+		return;
 
 	intr = ml_set_interrupts_enabled(FALSE);		/* No interruptions in here */
 
@@ -374,9 +377,6 @@ void pmsRunLocal(uint32_t nstep) {
 	}
 
 	(void)ml_set_interrupts_enabled(intr);			/* Restore interruptions */
-
-	return;
-
 }
 
 /*
@@ -385,11 +385,12 @@ void pmsRunLocal(uint32_t nstep) {
  *	Interruptions disabled.
  *
  */
-kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize) {
-
-	uint32_t nstep, result, presult;
+kern_return_t
+pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize)
+{
+	uint32_t nstep = 0, result, presult;
 	int ret, cpu;
-	kern_return_t kret;
+	kern_return_t kret = KERN_SUCCESS;
 	pmsDef *ndefs;
 	PER_PROC_INFO *pp;
 
@@ -397,53 +398,61 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
 	cpu = cpu_number();								/* Get our processor */
 	
 	if(!is_suser()) {								/* We are better than most, */
-		return KERN_FAILURE;				/* so we will only talk to the superuser. */
+		kret = KERN_FAILURE;
+		goto out;
 	}
 
 	if(request >= pmsCFree) {					/* Can we understand the request? */
-		return KERN_INVALID_ARGUMENT;		/* What language are these guys talking in, anyway? */
+		kret = KERN_INVALID_ARGUMENT;
+		goto out;
 	}
 	
 	if(request == pmsCQuery) {						/* Are we just checking? */
 		result = pmsCPUQuery() & pmsCPU;			/* Get the processor data and make sure there is no slop */
 		presult = 0;								/* Assume nothing */
-		if((uint32_t)pmsQueryFunc) presult = pmsQueryFunc(cpu, pmsPlatformData);	/* Go get the platform state */
-		result = result | (presult & (pmsXClk | pmsVoltage | pmsPowerID));	/* Merge the platform state with no slop */
-		return result;						/* Tell 'em... */
+		if((uint32_t)pmsQueryFunc)
+			presult = pmsQueryFunc(cpu, pmsPlatformData);	/* Go get the platform state */
+		kret = result | (presult & (pmsXClk | pmsVoltage | pmsPowerID));	/* Merge the platform state with no slop */
+		goto out;
 	}
 	
 	if(request == pmsCExperimental) {				/* Enter experimental mode? */
 	
 		if(pmsInstalled || (pmsExperimental & 1)) {	/* Are we already running or in experimental? */
-			return KERN_FAILURE;			/* Fail, since we are already running */
+			kret = KERN_FAILURE;
+			goto out;
 		}
 		
 		pmsExperimental |= 1;						/* Flip us into experimental but don't change other flags */
 		
 		pmsCPUConf();								/* Configure for this machine */
 		pmsStart();									/* Start stepping */
-		return KERN_SUCCESS;				/* We are victorious... */
-	
+		goto out;
 	}
 
 	if(request == pmsCCnfg) {						/* Do some up-front checking before we commit to doing this */
 		if((reqsize > (pmsMaxStates * sizeof(pmsDef))) || (reqsize < (pmsFree * sizeof(pmsDef)))) {	/* Check that the size is reasonable */
-			return KERN_NO_SPACE;			/* Tell them that they messed up */
+			kret = KERN_NO_SPACE;
+			goto out;
 		}
 	}
 
 	if (request == pmsGCtls) {
-		if (reqsize != sizeof(pmsCtls))
-			return(KERN_FAILURE);
+		if (reqsize != sizeof(pmsCtls)) {
+			kret = KERN_FAILURE;
+			goto out;
+		}
 		ret = copyout(&pmsCtls, reqaddr, reqsize);
-		return kret;
+		goto out;
 	}
 			
 	if (request == pmsGStats) {
-		if (reqsize != sizeof(pmsStatsd))		/* request size is fixed */
-			return KERN_FAILURE;
+		if (reqsize != sizeof(pmsStatsd)) { /* request size is fixed */
+			kret = KERN_FAILURE;
+			goto out;
+		}
 		ret = copyout(&pmsStatsd, reqaddr, reqsize);
-		return kret;							/* All done... */
+		goto out;
 	}
 
 /*
@@ -455,7 +464,9 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
  */
  		
 	if(!hw_compare_and_store(0, 1, &pmsSyncrolator)) {	/* Are we already doing this? */
-		return KERN_RESOURCE_SHORTAGE;		/* Tell them that we are already busy and to try again */
+		/* Tell them that we are already busy and to try again */
+		kret = KERN_RESOURCE_SHORTAGE;
+		goto out;
 	}
 
 //	NOTE:  We will block in the following code until everyone has finished the prepare
@@ -464,7 +475,7 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
 	
 	if(request == pmsCPark) {						/* Is all we're supposed to do park? */
 		pmsSyncrolator = 0;							/* Free us up */
-		return KERN_SUCCESS;									/* Well, then we're done... */
+		goto out;
 	}
 	
 	switch(request) {								/* Select the routine */
@@ -485,21 +496,23 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
 			
 			if(!(ndefs = (pmsDef *)kalloc(reqsize))) {	/* Get memory for the whole thing */
 				pmsSyncrolator = 0;					/* Free us up */
-				return KERN_INVALID_ADDRESS;			/* All done... */
+				kret = KERN_INVALID_ADDRESS;
+				goto out;
 			}
 			
 			ret = copyin(reqaddr, (void *)ndefs, reqsize);	/* Get the new config table */
 			if(ret) {								/* Hmmm, something went wrong with the copyin */
-				kfree((vm_offset_t)ndefs, reqsize);	/* Free up the copied in data */
+				kfree(ndefs, reqsize);	/* Free up the copied in data */
 				pmsSyncrolator = 0;					/* Free us up */
-				return KERN_INVALID_ADDRESS;				/* All done... */
+				kret = KERN_INVALID_ADDRESS;
+				goto out;
 			}
 
-			kret = pmsBuild(ndefs, reqsize, 0, 0, 0);	/* Go build and replace the tables.  Make sure we keep the old platform stuff */
+			kret = pmsBuild(ndefs, reqsize, NULL, 0, NULL);	/* Go build and replace the tables.  Make sure we keep the old platform stuff */
 			if(kret) {								/* Hmmm, something went wrong with the compilation */
-				kfree((vm_offset_t)ndefs, reqsize);	/* Free up the copied in data */
+				kfree(ndefs, reqsize);	/* Free up the copied in data */
 				pmsSyncrolator = 0;					/* Free us up */
-				return kret;							/* All done... */
+				goto out;
 			}
 
 			nstep = pmsNormHigh;					/* Set the request */
@@ -512,7 +525,8 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
 
 	pmsRun(nstep);									/* Get everyone into step */
 	pmsSyncrolator = 0;								/* Free us up */
-	return KERN_SUCCESS;										/* All done... */
+out:
+	return kret;
 
 }
 
@@ -522,8 +536,9 @@ kern_return_t pmsControl(uint32_t request, user_addr_t reqaddr, uint32_t reqsize
  *	Interruptions are disabled.
  */
  
-void pmsRun(uint32_t nstep) {
-
+void
+pmsRun(uint32_t nstep)
+{
 	pmsCPURun(nstep);
 }
 
@@ -556,15 +571,16 @@ void pmsRun(uint32_t nstep) {
  
 kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint32_t platformData, pmsQueryFunc_t queryFunc) {
 
-	int steps, newsize, i, cstp, nstps, oldAltSize, xdsply;
-	uint32_t setf;
+	int newsize, cstp, oldAltSize, xdsply;
+	uint32_t setf, steps, i, nstps;
 	uint64_t nlimit;
 	pmsDef *newpd, *oldAlt;
 	boolean_t intr;
 
 	xdsply = (pmsExperimental & 3) != 0;			/* Turn on kprintfs if requested or in experimental mode */
 
-	if(pdsize % sizeof(pmsDef)) return KERN_INVALID_ARGUMENT;	/* Length not multiple of definition size */
+	if(pdsize % sizeof(pmsDef))
+		return KERN_INVALID_ARGUMENT;	/* Length not multiple of definition size */
 	
 	steps = pdsize / sizeof(pmsDef);				/* Get the number of steps supplied */
 
@@ -718,7 +734,8 @@ kern_return_t pmsBuild(pmsDef *pd, uint32_t pdsize, pmsSetFunc_t *functab, uint3
 	simple_unlock(&pmsBuildLock);					/* Free play! */
 	(void)ml_set_interrupts_enabled(intr);			/* Interrupts back the way there were */
 
-	if((uint32_t)oldAlt) kfree(oldAlt, oldAltSize);	/* If we already had an alternate, free it */
+	if((uint32_t)oldAlt) /* If we already had an alternate, free it */
+		kfree(oldAlt, oldAltSize);
 
 	if(xdsply) kprintf("Stepper table installed\n");
 	
diff --git a/osfmk/ppc/pms.h b/osfmk/ppc/pms.h
deleted file mode 100644
index d978becea..000000000
--- a/osfmk/ppc/pms.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#ifdef KERNEL_PRIVATE
-
-#ifndef _PPC_PMS_H_
-#define _PPC_PMS_H_
-#include <kern/pms.h>
-#endif /* _PPC_PMS_H_ */
-#endif /* KERNEL_PRIVATE */
diff --git a/osfmk/ppc/pmsCPU.c b/osfmk/ppc/pmsCPU.c
index 86d179fed..0b12f2d31 100644
--- a/osfmk/ppc/pmsCPU.c
+++ b/osfmk/ppc/pmsCPU.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <ppc/machine_routines.h>
 #include <ppc/machine_cpu.h>
@@ -126,8 +132,7 @@ pmsDef pmsDefault[] = {
  */
 
 void pmsCPUSet(uint32_t sel) {
-	int nvoltage, nfreq;
-	uint32_t oldaack;
+	int nfreq;
 	struct per_proc_info *pp;
 
 	pp = getPerProc();									/* Get our per_proc */
@@ -168,12 +173,11 @@ void pmsCPUConf(void) {
 	kern_return_t ret;
 	pmsSetFunc_t pmsDfltFunc[pmsSetFuncMax];			/* List of functions for the external power control to use */
 
-	for(i = 0; i < pmsSetFuncMax; i++) pmsDfltFunc[i] = 0;	/* Clear this */
+	for(i = 0; i < pmsSetFuncMax; i++) pmsDfltFunc[i] = NULL;	/* Clear this */
 
 
 	ret = pmsBuild((pmsDef *)&pmsDefault, sizeof(pmsDefault), pmsDfltFunc, 0, (pmsQueryFunc_t)0);	/* Configure the default stepper */
 
-pCCfinish:
 	if(ret != KERN_SUCCESS) {							/* Some screw up? */
 		panic("pmsCPUConf: initial stepper table build failed, ret = %08X\n", ret);	/* Squeal */
 	}
@@ -209,12 +213,13 @@ void pmsCPUInit(void) {
 	pmsPark();											/* Then park */
 
 	kprintf("************ Stepper hardware initialized, cpu %d ******************\n", cpu);	/* (BRINGUP) */
-
-	return;
 }
 
-uint32_t pmsCPUQuery(void) {
+extern uint32_t hid1get(void);
 
+uint32_t
+pmsCPUQuery(void)
+{
 	uint32_t result;
 	struct per_proc_info *pp;
 	uint64_t scdata;
diff --git a/osfmk/ppc/ppc_init.c b/osfmk/ppc/ppc_init.c
index 2af1e82df..781eb38a1 100644
--- a/osfmk/ppc/ppc_init.c
+++ b/osfmk/ppc/ppc_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,9 +63,6 @@ extern unsigned int mckFlags;
 extern vm_offset_t	intstack;
 extern vm_offset_t	debstack;  
 
-int pc_trace_buf[1024] = {0};
-int pc_trace_cnt = 1024;
-
 extern unsigned int extPatchMCK;
 extern unsigned int extPatch32;
 extern unsigned int hwulckPatch_isync;
@@ -76,8 +79,6 @@ extern unsigned int rwlesPatch_isync;
 extern unsigned int rwlesPatch_eieio;
 extern unsigned int rwldPatch_isync;
 extern unsigned int rwldPatch_eieio;
-extern unsigned int retfsectPatch_eieio;
-extern unsigned int retfsectPatch_isync;
 extern unsigned int bcopy_nop_if_32bit;
 extern unsigned int bcopy_nc_nop_if_32bit;
 extern unsigned int memcpy_nop_if_32bit;
@@ -89,6 +90,8 @@ extern unsigned int uft_cuttrace;
 int forcenap = 0;
 int wcte = 0;				/* Non-cache gather timer disabled */
 
+int debug_task;
+
 patch_entry_t patch_table[] = {
 	{&extPatch32,			0x60000000, PATCH_FEATURE,		PatchExt32},
 	{&extPatchMCK,			0x60000000, PATCH_PROCESSOR,	CPU_SUBTYPE_POWERPC_970},
@@ -109,10 +112,6 @@ patch_entry_t patch_table[] = {
 	{&bcopy_nop_if_32bit,	0x60000000, PATCH_FEATURE,		PatchExt32},
 	{&bcopy_nc_nop_if_32bit,0x60000000, PATCH_FEATURE,		PatchExt32},
 	{&memcpy_nop_if_32bit,	0x60000000, PATCH_FEATURE,		PatchExt32},
-#if	!MACH_LDEBUG
-	{&retfsectPatch_isync,	0x60000000, PATCH_FEATURE,		PatchLwsync},
-	{&retfsectPatch_eieio,	0x7c2004ac, PATCH_FEATURE,		PatchLwsync},
-#endif
 	{&xsum_nop_if_32bit,	0x60000000,	PATCH_FEATURE,		PatchExt32},
 	{&uft_nop_if_32bit,		0x60000000,	PATCH_FEATURE,		PatchExt32},
 	{&uft_uaw_nop_if_32bit,	0x60000000,	PATCH_FEATURE,		PatchExt32},
@@ -130,6 +129,7 @@ void	ppc_init(
 void	ppc_init_cpu(
 			struct per_proc_info *proc_info);
 
+	
 /*
  *		Routine:		ppc_init
  *		Function:
@@ -149,7 +149,6 @@ ppc_init(
 	uint64_t 			scdata;
 
 
-
 	/*
 	 * Setup per_proc info for first cpu.
 	 */
@@ -162,8 +161,8 @@ ppc_init(
 	BootProcInfo.debstackptr = BootProcInfo.debstack_top_ss;
 	BootProcInfo.interrupts_enabled = 0;
 	BootProcInfo.pending_ast = AST_NONE;
-	BootProcInfo.FPU_owner = 0;
-	BootProcInfo.VMX_owner = 0;
+	BootProcInfo.FPU_owner = NULL;
+	BootProcInfo.VMX_owner = NULL;
 	BootProcInfo.pp_cbfr = console_per_proc_alloc(TRUE);
 	BootProcInfo.rtcPop = EndOfAllTime;
 	BootProcInfo.pp2ndPage = (addr64_t)&BootProcInfo;	/* Initial physical address of the second page */
@@ -194,7 +193,9 @@ ppc_init(
 	master_cpu = 0;
 	processor_bootstrap();
 
-	timer_switch((uint32_t)mach_absolute_time(), &thread->system_timer);
+	timer_start(&thread->system_timer, mach_absolute_time());
+	PROCESSOR_DATA(master_processor, kernel_timer) =
+				PROCESSOR_DATA(master_processor, thread_timer) = &thread->system_timer;
 
 	static_memory_end = round_page(args->topOfKernelData);;
       
@@ -272,7 +273,7 @@ ppc_init(
 		}
 	}
 		
-	machine_startup(args);
+	machine_startup();
 }
 
 /*
diff --git a/osfmk/ppc/ppc_vm_init.c b/osfmk/ppc/ppc_vm_init.c
index 18b9ce46f..af1598686 100644
--- a/osfmk/ppc/ppc_vm_init.c
+++ b/osfmk/ppc/ppc_vm_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -49,6 +55,7 @@
 #include <ppc/mappings.h>
 #include <ppc/exception.h>
 #include <ppc/lowglobals.h>
+#include <ppc/serial_io.h>
 
 #include <mach-o/mach_header.h>
 
@@ -80,7 +87,7 @@ uint64_t	sane_size;					/* Memory size to use for defaults calculations */
 						  
 
 mem_region_t pmap_mem_regions[PMAP_MEM_REGION_MAX + 1];
-int	 pmap_mem_regions_count = 0;		/* Assume no non-contiguous memory regions */
+unsigned int  pmap_mem_regions_count;		/* Assume no non-contiguous memory regions */
 
 unsigned int avail_remaining = 0;
 vm_offset_t first_avail;
@@ -201,21 +208,22 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 
 	first_avail = static_memory_end;
 
-/*
- * Now retrieve addresses for end, edata, and etext 
- * from MACH-O headers for the currently running 32 bit kernel.
- */
-	sectTEXTB = (vm_offset_t)getsegdatafromheader(
+	/*
+	 * Now retrieve addresses for end, edata, and etext 
+	 * from MACH-O headers for the currently running 32 bit kernel.
+	 */
+	/* XXX fix double casts for 64 bit kernel */
+	sectTEXTB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__TEXT", &sectSizeTEXT);
-	sectDATAB = (vm_offset_t)getsegdatafromheader(
+	sectDATAB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__DATA", &sectSizeDATA);
-	sectLINKB = (vm_offset_t)getsegdatafromheader(
+	sectLINKB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__LINKEDIT", &sectSizeLINK);
-	sectKLDB = (vm_offset_t)getsegdatafromheader(
+	sectKLDB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__KLD", &sectSizeKLD);
-	sectHIBB = (vm_offset_t)getsegdatafromheader(
+	sectHIBB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__HIB", &sectSizeHIB);
-	sectPRELINKB = (vm_offset_t)getsegdatafromheader(
+	sectPRELINKB = (vm_offset_t)(uint32_t *)getsegdatafromheader(
 		&_mh_execute_header, "__PRELINK", &sectSizePRELINK);
 
 	etext = (vm_offset_t) sectTEXTB + sectSizeTEXT;
@@ -369,7 +377,7 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 	taproot_size = PE_init_taproot(&taproot_addr);	/* (BRINGUP) See if there is a taproot */
 	if(taproot_size) {					/* (BRINGUP) */
 		kprintf("TapRoot card configured to use vaddr = %08X, size = %08X\n", taproot_addr, taproot_size);
-		bcopy_nc((void *)version, (void *)(taproot_addr + 16), strlen(version));	/* (BRINGUP) Pass it our kernel version */
+		bcopy_nc(version, (void *)(taproot_addr + 16), strlen(version));	/* (BRINGUP) Pass it our kernel version */
 		__asm__ volatile("eieio");		/* (BRINGUP) */
 		xtaproot = (unsigned int *)taproot_addr;	/* (BRINGUP) */
 		xtaproot[0] = 1;				/* (BRINGUP) */
@@ -387,15 +395,12 @@ void ppc_vm_init(uint64_t mem_limit, boot_args *args)
 
 
 	/* Processor version information */
-	{       
-		unsigned int pvr;
-		__asm__ ("mfpvr %0" : "=r" (pvr));
-		printf("processor version register : %08X\n", pvr);
-	}
+	__asm__ ("mfpvr %0" : "=r" (pvr));
+	printf("processor version register : %08X\n", pvr);
 
-	kprintf("Args at %08X\n", args);
+	kprintf("Args at %p\n", args);
 	for (i = 0; i < pmap_mem_regions_count; i++) {
-			printf("DRAM at %08X size %08X\n",
+			printf("DRAM at %08lX size %08lX\n",
 			       args->PhysicalDRAM[i].base,
 			       args->PhysicalDRAM[i].size);
 	}
diff --git a/osfmk/ppc/proc_reg.h b/osfmk/ppc/proc_reg.h
index a7e9ebfa9..6fb49c613 100644
--- a/osfmk/ppc/proc_reg.h
+++ b/osfmk/ppc/proc_reg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/rtclock.c b/osfmk/ppc/rtclock.c
index 6f0e68c40..90a5754ae 100644
--- a/osfmk/ppc/rtclock.c
+++ b/osfmk/ppc/rtclock.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -38,12 +44,12 @@
 #include <kern/processor.h>
 #include <kern/macro_help.h>
 #include <kern/spl.h>
+#include <kern/pms.h>
 
 #include <machine/commpage.h>
 #include <machine/machine_routines.h>
 #include <ppc/exception.h>
 #include <ppc/proc_reg.h>
-#include <ppc/pms.h>
 #include <ppc/rtclock.h>
 
 #include <sys/kdebug.h>
@@ -85,12 +91,11 @@ timebase_callback(
 	struct timebase_freq_t	*freq)
 {
 	uint32_t	numer, denom;
-	uint64_t	abstime;
 	spl_t		s;
 
 	if (	freq->timebase_den < 1 || freq->timebase_den > 4	||
 			freq->timebase_num < freq->timebase_den				)			
-		panic("rtclock timebase_callback: invalid constant %d / %d",
+		panic("rtclock timebase_callback: invalid constant %lu / %lu",
 					freq->timebase_num, freq->timebase_den);
 
 	denom = freq->timebase_num;
@@ -104,9 +109,6 @@ timebase_callback(
 		rtclock_timebase_const.denom = denom;
 		rtclock_sec_divisor = freq->timebase_num / freq->timebase_den;
 
-		nanoseconds_to_absolutetime(NSEC_PER_HZ, &abstime);
-		rtclock_tick_interval = abstime;
-
 		ml_init_lock_timeout();
 	}
 	else {
@@ -140,14 +142,6 @@ rtclock_config(void)
 int
 rtclock_init(void)
 {
-	uint64_t				abstime;
-	struct per_proc_info	*pp;
-
-	pp = getPerProc();
-
-	abstime = mach_absolute_time();
-	pp->rtclock_intr_deadline = abstime + rtclock_tick_interval;	/* Get the time we need to pop */
-	
 	etimer_resync_deadlines();			/* Start the timers going */
 
 	return (1);
diff --git a/osfmk/ppc/rtclock.h b/osfmk/ppc/rtclock.h
index 3e5c04262..ed0cbb333 100644
--- a/osfmk/ppc/rtclock.h
+++ b/osfmk/ppc/rtclock.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/savearea.c b/osfmk/ppc/savearea.c
index fcda2ab4c..a86f6d3cc 100644
--- a/osfmk/ppc/savearea.c
+++ b/osfmk/ppc/savearea.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *	This file is used to maintain the exception save areas
@@ -50,7 +56,6 @@
 #include <ddb/db_output.h>
 
 
-extern struct	Saveanchor saveanchor;							/* Aliged savearea anchor */
 struct Saveanchor backpocket;									/* Emergency saveareas */
 unsigned int	debsave0 = 0;									/* Debug flag */
 unsigned int	backchain = 0;									/* Debug flag */
@@ -131,7 +136,7 @@ void savearea_init(vm_offset_t addr) {
 
 	savearea_comm	*savec;
 	vm_offset_t	save;
-	int i;
+	unsigned int i;
 
 	
 	saveanchor.savetarget	= InitialSaveTarget;		/* Initial target value */
@@ -203,12 +208,12 @@ void savearea_init(vm_offset_t addr) {
 /*
  * This will populate the local list  and get the first one for the system
  */ 	
-	getPerProc()->next_savearea = (vm_offset_t)save_get();
+	/* XXX next_savearea should be a void * 4425541 */
+	getPerProc()->next_savearea = (unsigned long)(void *)save_get();
 
 /*
  *	The system is now able to take interruptions
  */
-	return;
 }
 
 
@@ -229,23 +234,22 @@ struct savearea	*save_alloc(void) {						/* Reserve a save area */
 
 
 /*
- *		This routine releases a save area to the free queue.  If after that, we have more than our maximum target,
- *		we start releasing what we can until we hit the normal target. 
+ * This routine releases a save area to the free queue.  If after that,
+ * we have more than our maximum target, we start releasing what we can
+ * until we hit the normal target. 
  */
 
+void
+save_release(struct savearea *save)
+{
+	/* Return a savearea to the free list */
+	save_ret(save);
 
-
-void save_release(struct savearea *save) {				/* Release a save area */
-	
-	save_ret(save);										/* Return a savearea to the free list */
-	
-	if(saveanchor.saveadjust) save_adjust();			/* Adjust the savearea free list and pool size if needed */
-	
-	return;
-	
+	/* Adjust the savearea free list and pool size if needed */
+	if(saveanchor.saveadjust)
+		save_adjust(); 
 }
 
-
 /*
  *		Adjusts the size of the free list.  Can either release or allocate full pages
  *		of kernel memory.  This can block.
@@ -265,7 +269,6 @@ void save_adjust(void) {
 	
 	savearea_comm	*sctl, *sctlnext, *freepage;
 	kern_return_t ret;
-	uint64_t vtopmask;
 	ppnum_t physpage;
 
 	if(saveanchor.saveadjust < 0) 					{	/* Do we need to adjust down? */
@@ -291,7 +294,7 @@ void save_adjust(void) {
 			
 			physpage = pmap_find_phys(kernel_pmap, (vm_offset_t)freepage);	/* Find physical page */
 			if(!physpage) {								/* See if we actually have this mapped*/
-				panic("save_adjust: wired page not mapped - va = %08X\n", freepage);	/* Die */
+				panic("save_adjust: wired page not mapped - va = %p\n", freepage);	/* Die */
 			}
 			
 			bzero((void *)freepage, PAGE_SIZE);			/* Clear it all to zeros */
@@ -315,7 +318,7 @@ save_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_siz
 	*count      = saveanchor.saveinuse;
 	*cur_size   = (saveanchor.savefreecnt + saveanchor.saveinuse) * (PAGE_SIZE / sac_cnt);
 	*max_size   = saveanchor.savemaxcount * (PAGE_SIZE / sac_cnt);
-	*elem_size  = sizeof(savearea);
+	*elem_size  = sizeof(struct savearea);
 	*alloc_size = PAGE_SIZE;
 	*collectable = 1;
 	*exhaustable = 0;
diff --git a/osfmk/ppc/savearea.h b/osfmk/ppc/savearea.h
index 492b0dda3..496591cb6 100644
--- a/osfmk/ppc/savearea.h
+++ b/osfmk/ppc/savearea.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifdef	XNU_KERNEL_PRIVATE
 
@@ -30,7 +36,7 @@
 
 #ifdef __APPLE_API_PRIVATE
 
-#ifdef	MACH_KERNEL_PRIVATE
+#if	defined(MACH_KERNEL_PRIVATE) || defined(BSD_KERNEL_PRIVATE)
 #include <stdint.h>
 #include <mach/vm_types.h>
 
@@ -76,15 +82,7 @@ typedef struct savearea_comm {
 												/* offset 0x0060 */
 } savearea_comm;
 #pragma pack()
-#endif
 
-#ifdef BSD_KERNEL_PRIVATE
-typedef struct savearea_comm {
-	unsigned int	save_000[24];
-} savearea_comm;
-#endif
-
-#if	defined(MACH_KERNEL_PRIVATE) || defined(BSD_KERNEL_PRIVATE)
 /*
  *	This type of savearea contains all of the general context.
  */
@@ -165,9 +163,9 @@ typedef struct savearea {
 
 	unsigned int	save_238[2];
 												/* offset 0x240 */
-	unsigned int	save_instr[16];				/* Instrumentation */
+	unsigned int	save_instr[16];				/* Instrumentation or emulation. Note: save_instr[0] is number of instructions */
 												/* offset 0x280 */
-} savearea;
+} savearea_t;
 #pragma pack()
 
 
@@ -320,7 +318,7 @@ struct Saveanchor {
 
 extern struct Saveanchor	saveanchor;			/* Aliged savearea anchor */
 
-#define sac_cnt		(4096 / sizeof(savearea))	/* Number of saveareas per page */
+#define sac_cnt		(4096 / sizeof(struct savearea))	/* Number of saveareas per page */
 #define sac_empty	(0xFFFFFFFF << (32 - sac_cnt))	/* Mask with all entries empty */
 #define sac_perm	0x40000000				/* Page permanently assigned */
 #define sac_permb	1						/* Page permanently assigned - bit position */
@@ -363,6 +361,7 @@ void 			save_fake_zone_info(		/* report savearea usage statistics as fake zone i
 
 void			save_snapshot(void);
 void			save_snapshot_restore(void);
+void save_release(struct savearea *);
 
 #endif /* MACH_KERNEL_PRIVATE */
 #endif /* __APPLE_API_PRIVATE */
@@ -378,6 +377,8 @@ void			save_snapshot_restore(void);
 #define	SAVinstrumentb 12					/* Indicates that we should return instrumentation data */
 #define	SAVeat 		0x00100000				/* Indicates that interruption should be ignored */
 #define	SAVeatb 	11						/* Indicates that interruption should be ignored */
+#define	SAVinject 	0x00200000				/* Indicates that save_instr contains code to inject */
+#define	SAVinjectb 	10						/* Indicates that save_instr contains code to inject */
 #define SAVtype		0x0000FF00				/* Shows type of savearea */
 #define SAVtypeshft	8						/* Shift to position type */
 #define SAVempty	0x86					/* Savearea is on free list */
diff --git a/osfmk/ppc/savearea_asm.s b/osfmk/ppc/savearea_asm.s
index ee8ed0f11..6b42c7dd3 100644
--- a/osfmk/ppc/savearea_asm.s
+++ b/osfmk/ppc/savearea_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #define FPVECDBG 0
diff --git a/osfmk/ppc/scc_8530.h b/osfmk/ppc/scc_8530.h
index 868e649df..2fcdfeb80 100644
--- a/osfmk/ppc/scc_8530.h
+++ b/osfmk/ppc/scc_8530.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/sched_param.h b/osfmk/ppc/sched_param.h
index aed63bcf2..eefe3303f 100644
--- a/osfmk/ppc/sched_param.h
+++ b/osfmk/ppc/sched_param.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/screen_switch.h b/osfmk/ppc/screen_switch.h
index cbf92e7cf..956d1ac84 100644
--- a/osfmk/ppc/screen_switch.h
+++ b/osfmk/ppc/screen_switch.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/serial_defs.h b/osfmk/ppc/serial_defs.h
index 508165fab..e18994c35 100644
--- a/osfmk/ppc/serial_defs.h
+++ b/osfmk/ppc/serial_defs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/serial_io.c b/osfmk/ppc/serial_io.c
index f944d2b4a..2f03aa110 100644
--- a/osfmk/ppc/serial_io.c
+++ b/osfmk/ppc/serial_io.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -96,8 +102,6 @@ struct scc_tty scc_tty[NSCC_LINE];
 #define scc_dev_no(chan) ((chan)^0x01)
 #define scc_chan(dev_no) ((dev_no)^0x01)
 
-extern unsigned int disableSerialOuput;
-
 int	serial_initted = 0;
 unsigned int scc_parm_done = 0;
 
@@ -106,27 +110,27 @@ static struct scc_byte {
 	unsigned char	val;
 } scc_init_hw[] = {
 	
-	9, 0x80,
-	4, 0x44,
-	3, 0xC0,
-	5, 0xE2,
-	2, 0x00,
-	10, 0x00,
-	11, 0x50,
-	12, 0x0A,
-	13, 0x00,
-	3, 0xC1,
-	5, 0xEA,
-	14, 0x01,
-	15, 0x00,
-	0, 0x10,
-	0, 0x10,
+	{9, 0x80},
+	{4, 0x44},
+	{3, 0xC0},
+	{5, 0xE2},
+	{2, 0x00},
+	{10, 0x00},
+	{11, 0x50},
+	{12, 0x0A},
+	{13, 0x00},
+	{3, 0xC1},
+	{5, 0xEA},
+	{14, 0x01},
+	{15, 0x00},
+	{0, 0x10},
+	{0, 0x10},
 #if 0
-	1, 0x12,			/* int or Rx, Tx int enable */
+	{1, 0x12},			/* int or Rx, Tx int enable */
 #else
-	1, 0x10,			/* int or Rx,  no Tx int enable */
+	{1, 0x10},			/* int or Rx,  no Tx int enable */
 #endif
-	9, 0x0A
+	{9, 0x0A}
 };
 
 static int	scc_init_hw_count = sizeof(scc_init_hw)/sizeof(scc_init_hw[0]);
@@ -252,8 +256,8 @@ int
 scc_probe(int32_t serial_baud)
 {
 	scc_softc_t     scc;
-	register int	val, i;
-	register scc_regmap_t	regs;
+	int i;
+	scc_regmap_t regs;
 	spl_t	s;
 	DECL_FUNNEL_VARS
 
@@ -316,12 +320,11 @@ scc_probe(int32_t serial_baud)
  */
 
 int
-scc_getc(int unit, int line, boolean_t wait, boolean_t raw)
+scc_getc(__unused int unit, int line, boolean_t wait, __unused boolean_t raw)
 {
-	register scc_regmap_t	regs;
+	scc_regmap_t	regs;
 	unsigned char   c, value;
-	int             rcvalue, from_line;
-	uint32_t 	fcrmunge;
+	int             rcvalue;
 	spl_t		s = splhigh();
 	DECL_FUNNEL_VARS
 
@@ -413,18 +416,17 @@ int _serial_getc(int unit, int line, boolean_t wait, boolean_t raw) {
  * use splhigh since we might be doing a printf in high spl'd code
  */
 
-int
-scc_putc(int unit, int line, int c)
+void
+scc_putc(__unused int unit, int line, int c)
 {
 	scc_regmap_t	regs;
 	spl_t            s;
 	unsigned char	 value;
-	uint32_t fcrmunge;
 	DECL_FUNNEL_VARS
 
 
-	if (disableSerialOuput)
-		return 0;
+	if (disable_serial_output)
+		return;
 
 	s = splhigh();
 	FUNNEL_ENTER(&SCC_FUNNEL);
@@ -453,7 +455,6 @@ scc_putc(int unit, int line, int c)
 	splx(s);
 
 	FUNNEL_EXIT(&SCC_FUNNEL);
-	return 0;
 }
 
 
@@ -508,7 +509,7 @@ scc_param(struct scc_tty *tp)
 	
 	/* Do a quick check to see if the hardware needs to change */
 	if ((sr->flags & (TF_ODDP|TF_EVENP)) == (tp->t_flags & (TF_ODDP|TF_EVENP))
-	    && sr->speed == tp->t_ispeed) {
+	    && sr->speed == (unsigned long)tp->t_ispeed) {
 		assert(FUNNEL_IN_USE(&SCC_FUNNEL));
 		simple_unlock(&scc_stomp);
 		splx(s);
@@ -655,6 +656,4 @@ scc_param(struct scc_tty *tp)
 	return 0;
 
 }
-
-
 #endif	/* NSCC > 0 */
diff --git a/osfmk/ppc/serial_io.h b/osfmk/ppc/serial_io.h
index 6fe428724..a280fa1a4 100644
--- a/osfmk/ppc/serial_io.h
+++ b/osfmk/ppc/serial_io.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,7 +32,10 @@
  * @APPLE_FREE_COPYRIGHT@
  */
 
+#ifndef _PPC_SERIAL_IO_H_
+#define _PPC_SERIAL_IO_H_
 
+#include <console/serial_protos.h>
 #include <device/device_types.h>
 #include <mach_kdp.h>
 
@@ -101,7 +110,7 @@ extern boolean_t	scc_portdeath(
 
 #endif /* 0 */
 
-extern int	 	scc_putc(
+extern void	 	scc_putc(
 				int			unit,
 				int			line,
 				int			c);
@@ -112,8 +121,6 @@ extern int		scc_getc(
 				boolean_t		wait,
 				boolean_t		raw);
 
-
-
 /*
  * JMM - We are not really going to support this driver in SMP (barely
  * support it now - so just pick up the stubbed out versions.
@@ -139,3 +146,5 @@ extern int		scc_getc(
 					   are time delays	XXX */
 #define	TF_ECHO		0x00000080	/* device wants user to echo input */
 #define	TS_MIN		0x00004000	/* buffer input chars, if possible */
+
+#endif /* _PPC_SERIAL_IO_H_ */
diff --git a/osfmk/ppc/setjmp.h b/osfmk/ppc/setjmp.h
index a1a0b34c4..2c7b1b9fc 100644
--- a/osfmk/ppc/setjmp.h
+++ b/osfmk/ppc/setjmp.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/simple_lock.h b/osfmk/ppc/simple_lock.h
index b7478bcb5..80be1e6ff 100644
--- a/osfmk/ppc/simple_lock.h
+++ b/osfmk/ppc/simple_lock.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/skiplists.s b/osfmk/ppc/skiplists.s
index f0cb982ca..69a9dccbb 100644
--- a/osfmk/ppc/skiplists.s
+++ b/osfmk/ppc/skiplists.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2002-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /* skiplists.s
diff --git a/osfmk/ppc/spec_reg.h b/osfmk/ppc/spec_reg.h
index ca8812f9e..46a13bdf6 100644
--- a/osfmk/ppc/spec_reg.h
+++ b/osfmk/ppc/spec_reg.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/start.s b/osfmk/ppc/start.s
index 8222e4c39..34ebea8ac 100644
--- a/osfmk/ppc/start.s
+++ b/osfmk/ppc/start.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -296,6 +302,7 @@ start64:	lis		r5,hi16(startcommon)				; Get top of address of continue point
 			ori		r5,r5,lo16(startcommon)				; Get low of address of continue point
 			lis		r9,hi16(MASK(MSR_HV)|MASK(MSR_SF))	; ?
 			lis		r20,hi16(dozem|napm|sleepm)			; Get mask of power saving features	
+			ori		r20,r20,lo16(1)						; Disable the attn instruction
 			li		r7,MSR_VM_OFF						; Get real mode MSR
 			sldi	r9,r9,32							; Slide into position
 			sldi	r20,r20,32							; Slide power stuff into position
diff --git a/osfmk/ppc/status.c b/osfmk/ppc/status.c
index e3e975a8d..50fee6015 100644
--- a/osfmk/ppc/status.c
+++ b/osfmk/ppc/status.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -31,11 +37,11 @@
 #include <ppc/cpu_internal.h>
 #include <ppc/exception.h>
 #include <ppc/misc_protos.h>
+#include <ppc/fpu_protos.h>
 #include <ppc/savearea.h>
 #include <ppc/thread.h>
 #include <ppc/Firmware.h>
 
-//#include <sys/time.h>
 typedef	unsigned int fixpt_t;	/* XXX <sys/resource.h> not self contained */
 #include <ppc/vmparam.h>	/* USRSTACK, etc. */
 
@@ -45,8 +51,6 @@ extern unsigned int killprint;
 extern double FloatInit;
 extern unsigned long QNaNbarbarian[4];
 
-#define       USRSTACK        0xc0000000
-
 kern_return_t
 thread_userstack(
     thread_t,
@@ -72,12 +76,12 @@ unsigned int get_msr_rbits(void);
 void ppc_checkthreadstate(void *, int);
 void thread_set_child(thread_t child, int pid);
 void thread_set_parent(thread_t parent, int pid);
-void save_release(struct savearea *save);
+void thread_set_cthreadself(thread_t thread, uint64_t pself, int isLP64);
 		
 /*
  * Maps state flavor to number of words in the state:
  */
-__private_extern__
+/* __private_extern__ */
 unsigned int _MachineStateCount[] = {
 	/* FLAVOR_LIST */ 0,
 	PPC_THREAD_STATE_COUNT,
@@ -105,7 +109,7 @@ machine_thread_get_state(
 	register struct savearea *sv;						/* Pointer to the context savearea */
 	register savearea_fpu *fsv;
 	register savearea_vec *vsv;
-	savearea *genuser;
+	struct savearea *genuser;
 	int i, j;
 	unsigned int vrvalidwrk;
 
@@ -429,7 +433,7 @@ machine_thread_get_kern_state(
 {
 	
 	register struct savearea *sv;						/* Pointer to the context savearea */
-	savearea *genkern;
+	struct savearea *genkern;
 	int i;
 
 	register struct ppc_thread_state *ts;
@@ -662,7 +666,7 @@ machine_thread_set_state(
 	mach_msg_type_number_t	count)
 {
   
-  	savearea		*genuser;
+  	struct savearea		*genuser;
   	savearea_fpu	*fsv, *fsvn, *fsvo;
   	savearea_vec	*vsv, *vsvn, *vsvo;
 	unsigned int	i;
@@ -955,6 +959,38 @@ machine_thread_set_state(
 }
 
 
+void
+thread_set_wq_state64(thread_t thread, thread_state_t tstate)
+{
+        struct ppc_thread_state64 *ts;
+  	struct savearea		*genuser;
+	thread_t curth = current_thread();
+
+	genuser = get_user_regs(thread);					/* Find or allocate and initialize one */
+	ts = (struct ppc_thread_state64 *)tstate;
+
+	if (curth != thread)
+	        thread_lock(thread);
+
+	genuser->save_r1	= ts->r1;
+	genuser->save_r3	= ts->r3;
+	genuser->save_r4	= ts->r4;
+	genuser->save_r5	= ts->r5;
+	genuser->save_r6	= ts->r6;
+	genuser->save_r7	= ts->r7;
+	genuser->save_r8	= ts->r8;
+	genuser->save_srr0	= ts->srr0;
+
+        genuser->save_srr1 = (uint64_t)MSR_EXPORT_MASK_SET;
+
+        if (task_has_64BitAddr(thread->task))
+	        genuser->save_srr1 |= (uint64_t)MASK32(MSR_SF) << 32;		/* If 64-bit task, force 64-bit mode */
+
+	if (curth != thread)
+	        thread_unlock(thread);
+}
+
+
 /*
  * This is where registers that are not normally specified by the mach-o
  * file on an execve should be nullified, perhaps to avoid a covert channel.
@@ -966,7 +1002,7 @@ kern_return_t
 machine_thread_state_initialize(
 	thread_t thread)
 {
-  	savearea		*sv;
+  	struct savearea		*sv;
 	
 	sv = get_user_regs(thread);						/* Find or allocate and initialize one */
 
@@ -995,7 +1031,7 @@ machine_thread_dup(
 	thread_t		self,
 	thread_t		target)
 {
-  	savearea		*sv, *osv; 
+  	struct savearea		*sv, *osv; 
   	savearea_fpu	*fsv, *fsvn;
   	savearea_vec	*vsv, *vsvn;
 	
@@ -1016,7 +1052,7 @@ machine_thread_dup(
 
 	fsv = find_user_fpu(self);						/* Get any user floating point */
 	
-	target->machine.curctx->FPUsave = 0;					/* Assume no floating point */
+	target->machine.curctx->FPUsave = NULL;					/* Assume no floating point */
 
 	if(fsv) {										/* Did we find one? */
 		fsvn = (savearea_fpu *)save_alloc();		/* If we still don't have one, get a new one */
@@ -1034,7 +1070,7 @@ machine_thread_dup(
 
 	vsv = find_user_vec(self);						/* Get any user vector */
 	
-	target->machine.curctx->VMXsave = 0;					/* Assume no vector */
+	target->machine.curctx->VMXsave = NULL;					/* Assume no vector */
 
 	if(vsv) {										/* Did we find one? */
 		vsvn = (savearea_vec *)save_alloc();		/* If we still don't have one, get a new one */
@@ -1060,22 +1096,22 @@ machine_thread_dup(
  *		We only set initial values if there was no context found.
  */
 
-savearea *
+struct savearea *
 get_user_regs(
 	thread_t	 thread)
 {
-  	savearea		*sv, *osv;
+  	struct savearea		*sv, *osv;
 	unsigned int	i;
 
 	if (thread->machine.upcb)
 		return	thread->machine.upcb;
 
 	sv = thread->machine.pcb;								/* Get the top savearea on the stack */
-	osv = 0;										/* Set no user savearea yet */	
+	osv = NULL;										/* Set no user savearea yet */	
 	
 	while(sv) {										/* Find the user context */
 		osv = sv;									/* Save the last one */
-		sv = CAST_DOWN(savearea *, sv->save_hdr.save_prev);	/* Get the previous context */ 
+		sv = CAST_DOWN(struct savearea *, sv->save_hdr.save_prev);	/* Get the previous context */ 
 	}
 
 	sv = save_alloc();								/* Get one */
@@ -1121,7 +1157,7 @@ get_user_regs(
  *		we just return a 0.
  */
 
-savearea *
+struct savearea *
 find_user_regs(
 	thread_t	thread)
 {
@@ -1131,7 +1167,7 @@ find_user_regs(
 /* The name of this call is something of a misnomer since the mact.pcb can 
  * contain chained saveareas, but it will do for now..
  */
-savearea *
+struct savearea *
 find_kern_regs(
 	thread_t	thread)
 {
@@ -1296,7 +1332,7 @@ thread_userstack(
 void
 thread_setuserstack(thread_t thread, mach_vm_address_t user_stack)
 {
-	savearea *sv;
+	struct savearea *sv;
 	
 	sv = get_user_regs(thread);	/* Get the user state registers */
 	
@@ -1305,6 +1341,22 @@ thread_setuserstack(thread_t thread, mach_vm_address_t user_stack)
 	return;
 }    
 
+void 
+thread_set_cthreadself(thread_t thread, uint64_t pself, int isLP64)
+{
+	struct savearea *sv;
+	
+	if (isLP64 == 0) {
+		thread->machine.cthread_self = pself;
+	} else {
+		sv = get_user_regs(thread);	/* Get the user state registers */
+
+		thread->machine.cthread_self = pself;
+		sv->save_r13 = pself;
+	}
+}
+
+
 /*
  * thread_adjuserstack:
  *
@@ -1314,7 +1366,7 @@ thread_setuserstack(thread_t thread, mach_vm_address_t user_stack)
 uint64_t
 thread_adjuserstack(thread_t thread, int adjust)
 {
-	savearea *sv;
+	struct savearea *sv;
 	
 	sv = get_user_regs(thread);	/* Get the user state registers */
 	
@@ -1324,10 +1376,10 @@ thread_adjuserstack(thread_t thread, int adjust)
 	
 }    
 
-void
+kern_return_t
 thread_setsinglestep(thread_t thread, int on)
 {
-	savearea *sv;
+	struct savearea *sv;
 	
 	sv = get_user_regs(thread);	/* Get the user state registers */
 	
@@ -1335,6 +1387,8 @@ thread_setsinglestep(thread_t thread, int on)
 	        sv->save_srr1 |= MASK(MSR_SE);
 	else
 	        sv->save_srr1 &= ~MASK(MSR_SE);
+	
+	return (KERN_SUCCESS);
 }    
 
 /*
@@ -1347,13 +1401,11 @@ thread_setsinglestep(thread_t thread, int on)
 void
 thread_setentrypoint(thread_t thread, uint64_t entry)
 {
-	savearea *sv;
+	struct savearea *sv;
 	
 	sv = get_user_regs(thread);	/* Get the user state registers */
 	
 	sv->save_srr0 = entry;
-	
-	return;
 }    
 
 kern_return_t
@@ -1491,7 +1543,7 @@ thread_set_parent(
 
 void *act_thread_csave(void) {
 
-  	savearea		*sv, *osv;
+  	struct savearea		*sv, *osv;
   	savearea_fpu	*fsv, *ofsv;
   	savearea_vec	*vsv, *ovsv;
 	
@@ -1505,7 +1557,7 @@ void *act_thread_csave(void) {
 	osv = find_user_regs(thread);						/* Get our savearea */
 
 	if(!osv) {
-		panic("act_thread_csave: attempting to preserve the context of an activation with none (%08X)\n", thread);
+		panic("act_thread_csave: attempting to preserve the context of an activation with none (%p)\n", thread);
 	}
 	
 	sv = save_alloc();								/* Get a fresh save area to save into */
@@ -1583,32 +1635,32 @@ void *act_thread_csave(void) {
 
 void act_thread_catt(void *ctx) {
 
-  	savearea		*sv, *osv, *psv;
+  	struct savearea		*sv, *osv, *psv;
   	savearea_fpu	*fsv, *ofsv, *pfsv;
   	savearea_vec	*vsv, *ovsv, *pvsv;
 	unsigned int	spc;
 	thread_t thread;	
 	
-	sv = (savearea *)ctx;							/* Make this easier for C */
+	sv = (struct savearea *)ctx;							/* Make this easier for C */
 	
 	fsv = CAST_DOWN(savearea_fpu *, sv->save_hdr.save_misc0);	/* Get a possible floating point savearea */ 
 	vsv = CAST_DOWN(savearea_vec *, sv->save_hdr.save_misc1);	/* Get a possible vector savearea */ 
 	
 	if((sv->save_hdr.save_misc2 != 0xDEBB1ED0) || (sv->save_hdr.save_misc3 != 0xE5DA11A5)) {	/* See if valid savearea */
-		panic("act_thread_catt: attempt to attach invalid general context savearea - %08X\n", sv);	/* Die */
+		panic("act_thread_catt: attempt to attach invalid general context savearea - %p\n", sv);	/* Die */
 	}
 
 	if(fsv && ((fsv->save_hdr.save_misc2 != 0xDEBB1ED0) || (fsv->save_hdr.save_misc3 != 0xE5DA11A5))) {	/* See if valid savearea */
-		panic("act_thread_catt: attempt to attach invalid float context savearea - %08X\n", fsv);	/* Die */
+		panic("act_thread_catt: attempt to attach invalid float context savearea - %p\n", fsv);	/* Die */
 	}
 
 	if(vsv && ((vsv->save_hdr.save_misc2 != 0xDEBB1ED0) || (vsv->save_hdr.save_misc3 != 0xE5DA11A5))) {	/* See if valid savearea */
-		panic("act_thread_catt: attempt to attach invalid vector context savearea - %08X\n", vsv);	/* Die */
+		panic("act_thread_catt: attempt to attach invalid vector context savearea - %p\n", vsv);	/* Die */
 	}
 
 	thread = current_thread();
 
-	act_machine_sv_free(thread);					/* Blow away any current kernel FP or vector.
+	act_machine_sv_free(thread, 0);					/* Blow away any current kernel FP or vector.
 													   We do not support those across a vfork */
 	toss_live_fpu(thread->machine.curctx);			/* Toss my floating point if live anywhere */
 	toss_live_vec(thread->machine.curctx);			/* Toss my vector if live anywhere */
@@ -1620,16 +1672,16 @@ void act_thread_catt(void *ctx) {
 	spc = (unsigned int)thread->map->pmap->space;	/* Get the space we're in */
 	
 	osv = thread->machine.pcb;						/* Get the top general savearea */
-	psv = 0;
+	psv = NULL;
 	while(osv) {									/* Any saved state? */
 		if(osv->save_srr1 & MASK(MSR_PR)) break;	/* Leave if this is user state */
 		psv = osv;									/* Save previous savearea address */
-		osv = CAST_DOWN(savearea *, osv->save_hdr.save_prev);	/* Get one underneath our's */
+		osv = CAST_DOWN(struct savearea *, osv->save_hdr.save_prev);	/* Get one underneath our's */
 	}
 	
 	if(osv) {										/* Did we find one? */
 		if(psv) psv->save_hdr.save_prev = 0;		/* Yes, clear pointer to it (it should always be last) or */	
-		else thread->machine.pcb = 0;						/* to the start if the only one */
+		else thread->machine.pcb = NULL;					/* to the start if the only one */
 
 		save_release(osv);							/* Nope, release it */
 		
@@ -1641,7 +1693,7 @@ void act_thread_catt(void *ctx) {
 	
 	ovsv = thread->machine.curctx->VMXsave;				/* Get the top vector savearea */
 	
-	pvsv = 0;
+	pvsv = NULL;
 	while(ovsv) {									/* Any VMX saved state? */
 		if(!(ovsv->save_hdr.save_level)) break;		/* Leave if this is user state */
 		pvsv = ovsv;								/* Save previous savearea address */
@@ -1650,16 +1702,16 @@ void act_thread_catt(void *ctx) {
 	
 	if(ovsv) {										/* Did we find one? */
 		if(pvsv) pvsv->save_hdr.save_prev = 0;		/* Yes, clear pointer to it (it should always be last) or */	
-		else thread->machine.curctx->VMXsave = 0;	/* to the start if the only one */
+		else thread->machine.curctx->VMXsave = NULL;	/* to the start if the only one */
 
-		save_release((savearea *)ovsv);				/* Nope, release it */
+		save_release((struct savearea *)ovsv);				/* Nope, release it */
 	}
 	
 	if(vsv) {										/* Are we sticking any vector on this one? */
 		if(pvsv) pvsv->save_hdr.save_prev = (addr64_t)((uintptr_t)vsv);	/* Yes, chain us to the end or */
 		else {
 			thread->machine.curctx->VMXsave = vsv;	/* to the start if the only one */
-			thread->machine.curctx->VMXlevel = 0;	/* Insure that we don't have a leftover level */
+			thread->machine.curctx->VMXlevel = NULL;	/* Insure that we don't have a leftover level */
 		}
 
 		vsv->save_hdr.save_misc2 = 0;				/* Eye catcher for debug */
@@ -1669,7 +1721,7 @@ void act_thread_catt(void *ctx) {
 	
 	ofsv = thread->machine.curctx->FPUsave;			/* Get the top float savearea */
 	
-	pfsv = 0;
+	pfsv = NULL;
 	while(ofsv) {									/* Any float saved state? */
 		if(!(ofsv->save_hdr.save_level)) break;		/* Leave if this is user state */
 		pfsv = ofsv;								/* Save previous savearea address */
@@ -1678,16 +1730,16 @@ void act_thread_catt(void *ctx) {
 	
 	if(ofsv) {										/* Did we find one? */
 		if(pfsv) pfsv->save_hdr.save_prev = 0;		/* Yes, clear pointer to it (it should always be last) or */	
-		else thread->machine.curctx->FPUsave = 0;	/* to the start if the only one */
+		else thread->machine.curctx->FPUsave = NULL;	/* to the start if the only one */
 
-		save_release((savearea *)ofsv);				/* Nope, release it */
+		save_release((struct savearea *)ofsv);				/* Nope, release it */
 	}
 	
 	if(fsv) {										/* Are we sticking any vector on this one? */
 		if(pfsv) pfsv->save_hdr.save_prev = (addr64_t)((uintptr_t)fsv);	/* Yes, chain us to the end or */
 		else {
 			thread->machine.curctx->FPUsave = fsv;	/* to the start if the only one */
-			thread->machine.curctx->FPUlevel = 0;	/* Insure that we don't have a leftover level */
+			thread->machine.curctx->FPUlevel = NULL;	/* Insure that we don't have a leftover level */
 		}
 
 		fsv->save_hdr.save_misc2 = 0;				/* Eye catcher for debug */
@@ -1709,35 +1761,35 @@ void
 act_thread_cfree(void *ctx)
 {
 
-  	savearea	*sv;
+  	struct savearea	*sv;
   	savearea_fpu	*fsv;
   	savearea_vec	*vsv;
 
-	sv = (savearea *)ctx;							/* Make this easier for C */
+	sv = (struct savearea *)ctx;							/* Make this easier for C */
 	
 	fsv = CAST_DOWN(savearea_fpu *, sv->save_hdr.save_misc0);	/* Get a possible floating point savearea */ 
 	vsv = CAST_DOWN(savearea_vec *, sv->save_hdr.save_misc1);	/* Get a possible vector savearea */ 
 	
 	if((sv->save_hdr.save_misc2 != 0xDEBB1ED0) || (sv->save_hdr.save_misc3 != 0xE5DA11A5)) {	/* See if valid savearea */
-		panic("act_thread_cfree: attempt to detatch invalid general context savearea - %08X\n", sv);	/* Die */
+		panic("act_thread_cfree: attempt to detatch invalid general context savearea - %p\n", sv);	/* Die */
 	}
 	
 	save_release(sv);								/* Toss the general savearea */
 
 	if(fsv) {										/* See if there is any saved floating point */ 
 		if((fsv->save_hdr.save_misc2 != 0xDEBB1ED0) || (fsv->save_hdr.save_misc3 != 0xE5DA11A5)) {	/* See if valid savearea */
-			panic("act_thread_cfree: attempt to detatch invalid float context savearea - %08X\n", fsv);	/* Die */
+			panic("act_thread_cfree: attempt to detatch invalid float context savearea - %p\n", fsv);	/* Die */
 		}
 		
-		save_release((savearea *)fsv);				/* Toss saved context */
+		save_release((struct savearea *)fsv);				/* Toss saved context */
 	}
 
 	if(vsv) {										/* See if there is any saved floating point */ 
 		if((vsv->save_hdr.save_misc2 != 0xDEBB1ED0) || (vsv->save_hdr.save_misc3 != 0xE5DA11A5)) {	/* See if valid savearea */
-			panic("act_thread_cfree: attempt to detatch invalid vector context savearea - %08X\n", vsv);	/* Die */
+			panic("act_thread_cfree: attempt to detatch invalid vector context savearea - %p\n", vsv);	/* Die */
 		}
 		
-		save_release((savearea *)vsv);				/* Toss saved context */
+		save_release((struct savearea *)vsv);				/* Toss saved context */
 	}
 	
 	return;
@@ -1753,7 +1805,7 @@ int thread_enable_fpe(
 	thread_t		thread,
 	int				onoff)
 {
-        savearea *sv;
+        struct savearea *sv;
         uint64_t oldmsr;
 
         sv = find_user_regs(thread);										/* Find the user registers */
diff --git a/osfmk/ppc/task.h b/osfmk/ppc/task.h
index 3d5259fac..3c9ad4164 100644
--- a/osfmk/ppc/task.h
+++ b/osfmk/ppc/task.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/testjump.c b/osfmk/ppc/testjump.c
index 5debfb6dc..be2ae5afa 100644
--- a/osfmk/ppc/testjump.c
+++ b/osfmk/ppc/testjump.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
diff --git a/osfmk/ppc/thread.h b/osfmk/ppc/thread.h
index 3356d44c7..d3e4b1109 100644
--- a/osfmk/ppc/thread.h
+++ b/osfmk/ppc/thread.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -61,11 +67,11 @@ typedef struct savearea *pcb_t;
 struct facility_context {
 
 	savearea_fpu	*FPUsave;		/* The floating point savearea */
-	savearea		*FPUlevel;		/* The floating point context level */
+	struct savearea		*FPUlevel;		/* The floating point context level */
 	unsigned int	FPUcpu;			/* The last processor to enable floating point */
 	unsigned int	FPUsync;		/* Sync lock */
 	savearea_vec	*VMXsave;		/* The VMX savearea */
-	savearea		*VMXlevel;		/* The VMX context level */
+	struct savearea		*VMXlevel;		/* The VMX context level */
 	unsigned int	VMXcpu;			/* The last processor to enable vector */
 	unsigned int	VMXsync;		/* Sync lock */
 	struct thread	*facAct;
@@ -90,8 +96,8 @@ struct machine_thread {
 	 * one for each active facility context.  They may point to the
 	 * same saveareas.
 	 */
-	savearea		*pcb;			/* The "normal" savearea */
-	savearea		*upcb;			/* The "normal" user savearea */
+	struct savearea		*pcb;			/* The "normal" savearea */
+	struct savearea		*upcb;			/* The "normal" user savearea */
 	facility_context *curctx;		/* Current facility context */
 	facility_context *deferctx;		/* Deferred facility context */
 	facility_context facctx;		/* "Normal" facility context */
diff --git a/osfmk/ppc/trap.c b/osfmk/ppc/trap.c
index 774ab23da..0843dcd44 100644
--- a/osfmk/ppc/trap.c
+++ b/osfmk/ppc/trap.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -52,11 +58,21 @@
 #include <ppc/low_trace.h>
 #include <ppc/Diagnostics.h>
 #include <ppc/hw_perfmon.h>
+#include <ppc/fpu_protos.h>
 
 #include <sys/kdebug.h>
 
-perfCallback perfTrapHook = 0; /* Pointer to CHUD trap hook routine */
-perfCallback perfASTHook = 0;  /* Pointer to CHUD AST hook routine */
+perfCallback perfTrapHook; /* Pointer to CHUD trap hook routine */
+perfCallback perfASTHook;  /* Pointer to CHUD AST hook routine */
+
+#if CONFIG_DTRACE
+extern kern_return_t dtrace_user_probe(ppc_saved_state_t *sv);
+
+/* See <rdar://problem/4613924> */
+perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */
+
+extern boolean_t dtrace_tally_fault(user_addr_t);
+#endif
 
 #if	MACH_KDB
 #include <ddb/db_watch.h>
@@ -113,15 +129,14 @@ struct savearea *trap(int trapno,
 			     addr64_t dar)
 {
 	int exception;
-	int code;
-	int subcode;
+	mach_exception_code_t code = 0;
+	mach_exception_subcode_t subcode = 0;
 	vm_map_t map;
-    unsigned int sp;
-	unsigned int space, space2;
 	vm_map_offset_t offset;
 	thread_t thread = current_thread();
 	boolean_t intr;
 	ast_t *myast;
+	int ret;
 	
 #ifdef MACH_BSD
 	time_value_t tv;
@@ -140,6 +155,12 @@ struct savearea *trap(int trapno,
 		if(perfTrapHook(trapno, ssp, dsisr, (unsigned int)dar) == KERN_SUCCESS) return ssp;	/* If it succeeds, we are done... */
 	}
 
+#if CONFIG_DTRACE
+	if(tempDTraceTrapHook) {							/* Is there a hook? */
+		if(tempDTraceTrapHook(trapno, ssp, dsisr, (unsigned int)dar) == KERN_SUCCESS) return ssp;	/* If it succeeds, we are done... */
+	}
+#endif
+
 #if 0
 	{
 		extern void fctx_text(void);
@@ -163,7 +184,7 @@ struct savearea *trap(int trapno,
 		/*
 		 * Trap came from kernel
 		 */
-	      	switch (trapno) {
+		switch (trapno) {
 
 		case T_PREEMPT:			/* Handle a preempt trap */
 			ast_taken(AST_PREEMPTION, FALSE);
@@ -305,6 +326,17 @@ struct savearea *trap(int trapno,
 					break;
 				}
 
+#if CONFIG_DTRACE
+				if (thread->options & TH_OPT_DTRACE) {	/* Executing under dtrace_probe? */
+					if (dtrace_tally_fault(dar)) { /* Should a fault under dtrace be ignored? */
+						ssp->save_srr0 += 4;                /* Point to next instruction */
+						break;
+					} else {
+						unresolved_kernel_trap(trapno, ssp, dsisr, dar, "Unexpected page fault under dtrace_probe");
+					}
+				}
+#endif
+
 				code = vm_fault(map, vm_map_trunc_page(offset),
 						dsisr & MASK(DSISR_WRITE) ? PROT_RW : PROT_RO,
 						FALSE, THREAD_UNINT, NULL, vm_map_trunc_page(0));
@@ -321,6 +353,22 @@ struct savearea *trap(int trapno,
 
 			/* If we get here, the fault was due to a user memory window access */
 
+#if CONFIG_DTRACE
+			if (thread->options & TH_OPT_DTRACE) {	/* Executing under dtrace_probe? */
+				if (dtrace_tally_fault(dar)) { /* Should a user memory window access fault under dtrace be ignored? */
+					if (thread->recover) {
+						ssp->save_srr0 = thread->recover;
+						thread->recover = (vm_offset_t)NULL;
+					} else {
+						unresolved_kernel_trap(trapno, ssp, dsisr, dar, "copyin/out has no recovery point");
+					}
+					break;
+				} else {
+					unresolved_kernel_trap(trapno, ssp, dsisr, dar, "Unexpected UMW page fault under dtrace_probe");
+				}
+			}
+#endif
+
 			map = thread->map;
 			
 			offset = (vm_map_offset_t)(thread->machine.umwRelo + dar);	/* Compute the user space address */
@@ -414,209 +462,222 @@ struct savearea *trap(int trapno,
 		 * Trap came from user task
 		 */
 
-	      	switch (trapno) {
-
-		case T_PREEMPT:
-			unresolved_kernel_trap(trapno, ssp, dsisr, dar, NULL);
-			break;	
-
-		case T_PERF_MON:
-			perfmon_handle_pmi(ssp);
-			break;
-
-			/*
-			 * These trap types should never be seen by trap()
-			 * Some are interrupts that should be seen by
-			 * interrupt() others just don't happen because they
-			 * are handled elsewhere.
-			 */
-		case T_DECREMENTER:
-		case T_IN_VAIN:								/* Shouldn't ever see this, lowmem_vectors eats it */
-		case T_INTERRUPT:
-		case T_FP_UNAVAILABLE:
-		case T_SYSTEM_MANAGEMENT:
-		case T_RESERVED:
-		case T_IO_ERROR:
-			
-		default:
-
-			ml_set_interrupts_enabled(FALSE);					/* Turn off interruptions */
-
-			panic("Unexpected user state trap(cpu %d): 0x%08X DSISR=0x%08X DAR=0x%016llX PC=0x%016llX, MSR=0x%016llX\n",
-			       cpu_number(), trapno, dsisr, dar, ssp->save_srr0, ssp->save_srr1);
-			break;
-
-
-/*
- *			Here we handle a machine check in user state
- */
-
-		case T_MACHINE_CHECK:
-			handleMck(ssp);						/* Common to both user and kernel */
-			break;
-
-		case T_RESET:
-			ml_set_interrupts_enabled(FALSE);					/* Turn off interruptions */
-			if (!Call_Debugger(trapno, ssp))
-				panic("Unexpected Reset exception: srr0 = %016llx, srr1 = %016llx\n",
-					ssp->save_srr0, ssp->save_srr1);
-			break;						/* We just ignore these */
-
-		case T_ALIGNMENT:
-/*
-*			If enaNotifyEMb is set, we get here, and
-*			we have actually already emulated the unaligned access.
-*			All that we want to do here is to ignore the interrupt. This is to allow logging or
-*			tracing of unaligned accesses.  
-*/
-			
-			KERNEL_DEBUG_CONSTANT(
-				MACHDBG_CODE(DBG_MACH_EXCP_ALNG, 0) | DBG_FUNC_NONE,
-				(int)ssp->save_srr0 - 4, (int)dar, (int)dsisr, (int)ssp->save_lr, 0);
-			
-			if(ssp->save_hdr.save_misc3) {				/* Was it a handled exception? */
-				exception = EXC_BAD_ACCESS;				/* Yes, throw exception */
-				code = EXC_PPC_UNALIGNED;
-				subcode = (unsigned int)dar;
-			}
-			break;
-
-		case T_EMULATE:
-/*
-*			If enaNotifyEMb is set we get here, and
-*			we have actually already emulated the instruction.
-*			All that we want to do here is to ignore the interrupt. This is to allow logging or
-*			tracing of emulated instructions.  
-*/
-
-			KERNEL_DEBUG_CONSTANT(
-				MACHDBG_CODE(DBG_MACH_EXCP_EMUL, 0) | DBG_FUNC_NONE,
-				(int)ssp->save_srr0 - 4, (int)((savearea_comm *)ssp)->save_misc2, (int)dsisr, (int)ssp->save_lr, 0);
-			break;
-
-		case T_TRACE:			/* Real PPC chips */
-		  if (be_tracing()) {
-		    add_pcbuffer();
-		    return ssp;
-		  }
-		  /* fall through */
-
-		case T_INSTRUCTION_BKPT:
-			exception = EXC_BREAKPOINT;
-			code = EXC_PPC_TRACE;
-			subcode = (unsigned int)ssp->save_srr0;
-			break;
-
-		case T_PROGRAM:
-			if (ssp->save_srr1 & MASK(SRR1_PRG_FE)) {
-				fpu_save(thread->machine.curctx);
-				UPDATE_PPC_EXCEPTION_STATE;
-				exception = EXC_ARITHMETIC;
-				code = EXC_ARITHMETIC;
-			
-				mp_disable_preemption();
-				subcode = ssp->save_fpscr;
-				mp_enable_preemption();
-			} 	
-			else if (ssp->save_srr1 & MASK(SRR1_PRG_ILL_INS)) {
+		switch (trapno) {
+	
+			case T_PREEMPT:
+				unresolved_kernel_trap(trapno, ssp, dsisr, dar, NULL);
+				break;	
+	
+			case T_PERF_MON:
+				perfmon_handle_pmi(ssp);
+				break;
+	
+				/*
+				 * These trap types should never be seen by trap()
+				 * Some are interrupts that should be seen by
+				 * interrupt() others just don't happen because they
+				 * are handled elsewhere.
+				 */
+			case T_DECREMENTER:
+			case T_IN_VAIN:								/* Shouldn't ever see this, lowmem_vectors eats it */
+			case T_INTERRUPT:
+			case T_FP_UNAVAILABLE:
+			case T_SYSTEM_MANAGEMENT:
+			case T_RESERVED:
+			case T_IO_ERROR:
 				
-				UPDATE_PPC_EXCEPTION_STATE
-				exception = EXC_BAD_INSTRUCTION;
-				code = EXC_PPC_UNIPL_INST;
-				subcode = (unsigned int)ssp->save_srr0;
-			} else if ((unsigned int)ssp->save_srr1 & MASK(SRR1_PRG_PRV_INS)) {
-
-				UPDATE_PPC_EXCEPTION_STATE;
-				exception = EXC_BAD_INSTRUCTION;
-				code = EXC_PPC_PRIVINST;
-				subcode = (unsigned int)ssp->save_srr0;
-			} else if (ssp->save_srr1 & MASK(SRR1_PRG_TRAP)) {
-				unsigned int inst;
-				//char *iaddr;
+			default:
+	
+				ml_set_interrupts_enabled(FALSE);		/* Turn off interruptions */
+	
+				panic("Unexpected user state trap(cpu %d): 0x%08X DSISR=0x%08X DAR=0x%016llX PC=0x%016llX, MSR=0x%016llX\n",
+					   cpu_number(), trapno, dsisr, dar, ssp->save_srr0, ssp->save_srr1);
+				break;
+	
+	
+	/*
+	 *			Here we handle a machine check in user state
+	 */
+	
+			case T_MACHINE_CHECK:
+				handleMck(ssp);							/* Common to both user and kernel */
+				break;
+	
+			case T_RESET:
+				ml_set_interrupts_enabled(FALSE);		/* Turn off interruptions */
+				if (!Call_Debugger(trapno, ssp))
+					panic("Unexpected Reset exception: srr0 = %016llx, srr1 = %016llx\n",
+						ssp->save_srr0, ssp->save_srr1);
+				break;									/* We just ignore these */
+	
+			case T_ALIGNMENT:
+	/*
+	*			If enaNotifyEMb is set, we get here, and
+	*			we have actually already emulated the unaligned access.
+	*			All that we want to do here is to ignore the interrupt. This is to allow logging or
+	*			tracing of unaligned accesses.  
+	*/
 				
-				//iaddr = CAST_DOWN(char *, ssp->save_srr0);		/* Trim from long long and make a char pointer */ 
-				if (copyin(ssp->save_srr0, (char *) &inst, 4 )) panic("copyin failed\n");
+				KERNEL_DEBUG_CONSTANT(
+					MACHDBG_CODE(DBG_MACH_EXCP_ALNG, 0) | DBG_FUNC_NONE,
+					(int)ssp->save_srr0 - 4, (int)dar, (int)dsisr, (int)ssp->save_lr, 0);
 				
-				if(dgWork.dgFlags & enaDiagTrap) {	/* Is the diagnostic trap enabled? */
-					if((inst & 0xFFFFFFF0) == 0x0FFFFFF0) {	/* Is this a TWI 31,R31,0xFFFx? */
-						if(diagTrap(ssp, inst & 0xF)) {	/* Call the trap code */
-							ssp->save_srr0 += 4ULL;	/* If we eat the trap, bump pc */
-							exception = 0;			/* Clear exception */
-							break;					/* All done here */
+				if(ssp->save_hdr.save_misc3) {			/* Was it a handled exception? */
+					exception = EXC_BAD_ACCESS;			/* Yes, throw exception */
+					code = EXC_PPC_UNALIGNED;
+					subcode = dar;
+				}
+				break;
+	
+			case T_EMULATE:
+	/*
+	*			If enaNotifyEMb is set we get here, and
+	*			we have actually already emulated the instruction.
+	*			All that we want to do here is to ignore the interrupt. This is to allow logging or
+	*			tracing of emulated instructions.  
+	*/
+	
+				KERNEL_DEBUG_CONSTANT(
+					MACHDBG_CODE(DBG_MACH_EXCP_EMUL, 0) | DBG_FUNC_NONE,
+					(int)ssp->save_srr0 - 4, (int)((savearea_comm *)ssp)->save_misc2, (int)dsisr, (int)ssp->save_lr, 0);
+				break;
+	
+			case T_TRACE:			/* Real PPC chips */
+			case T_INSTRUCTION_BKPT:
+				exception = EXC_BREAKPOINT;
+				code = EXC_PPC_TRACE;
+				subcode = ssp->save_srr0;
+				break;
+	
+			case T_PROGRAM:
+				if (ssp->save_srr1 & MASK(SRR1_PRG_FE)) {
+					fpu_save(thread->machine.curctx);
+					UPDATE_PPC_EXCEPTION_STATE;
+					exception = EXC_ARITHMETIC;
+					code = EXC_ARITHMETIC;
+				
+					mp_disable_preemption();
+					subcode = ssp->save_fpscr;
+					mp_enable_preemption();
+				} 	
+				else if (ssp->save_srr1 & MASK(SRR1_PRG_ILL_INS)) {
+					
+					UPDATE_PPC_EXCEPTION_STATE
+					exception = EXC_BAD_INSTRUCTION;
+					code = EXC_PPC_UNIPL_INST;
+					subcode = ssp->save_srr0;
+				} else if ((unsigned int)ssp->save_srr1 & MASK(SRR1_PRG_PRV_INS)) {
+	
+					UPDATE_PPC_EXCEPTION_STATE;
+					exception = EXC_BAD_INSTRUCTION;
+					code = EXC_PPC_PRIVINST;
+					subcode = ssp->save_srr0;
+				} else if (ssp->save_srr1 & MASK(SRR1_PRG_TRAP)) {
+					unsigned int inst;
+	
+					if (copyin(ssp->save_srr0, (char *) &inst, 4 )) panic("copyin failed\n");
+					
+					if(dgWork.dgFlags & enaDiagTrap) {	/* Is the diagnostic trap enabled? */
+						if((inst & 0xFFFFFFF0) == 0x0FFFFFF0) {	/* Is this a TWI 31,R31,0xFFFx? */
+							if(diagTrap(ssp, inst & 0xF)) {	/* Call the trap code */
+								ssp->save_srr0 += 4ULL;		/* If we eat the trap, bump pc */
+								exception = 0;				/* Clear exception */
+								break;						/* All done here */
+							}
+						}
+					}
+					
+#if CONFIG_DTRACE
+					if(inst == 0x0FFFDDDD) {				/* Is this the dtrace trap? */
+						ret = dtrace_user_probe((ppc_saved_state_t *)ssp);	/* Go check if it is for real and process if so... */
+						if(ret == KERN_SUCCESS) {			/* Was it really? */
+							exception = 0;					/* Clear the exception */
+							break;							/* Go flow through and out... */
 						}
 					}
+#endif				
+					
+					UPDATE_PPC_EXCEPTION_STATE;
+					
+					if (inst == 0x7FE00008) {
+						exception = EXC_BREAKPOINT;
+						code = EXC_PPC_BREAKPOINT;
+					} else {
+						exception = EXC_SOFTWARE;
+						code = EXC_PPC_TRAP;
+					}
+					subcode = ssp->save_srr0;
 				}
+				break;
+
+#if CONFIG_DTRACE
+			case T_DTRACE_RET:								/* Are we returning from a dtrace injection? */	
+				ret = dtrace_user_probe((ppc_saved_state_t *)ssp);	/* Call the probe function if so... */
+				if(ret == KERN_SUCCESS) {					/* Did this actually work? */
+					exception = 0;							/* Clear the exception */
+					break;									/* Go flow through and out... */
+				}
+				break;
+#endif				
 				
+			case T_ALTIVEC_ASSIST:
 				UPDATE_PPC_EXCEPTION_STATE;
+				exception = EXC_ARITHMETIC;
+				code = EXC_PPC_ALTIVECASSIST;
+				subcode = ssp->save_srr0;
+				break;
+	
+			case T_DATA_ACCESS:
+				map = thread->map;
+	
+				if(ssp->save_dsisr & dsiInvMode) {			/* Did someone try to reserve cache inhibited? */
+					UPDATE_PPC_EXCEPTION_STATE;				/* Don't even bother VM with this one */
+					exception = EXC_BAD_ACCESS;
+					subcode = dar;
+					break;
+				}
 				
-				if (inst == 0x7FE00008) {
-					exception = EXC_BREAKPOINT;
-					code = EXC_PPC_BREAKPOINT;
-				} else {
-					exception = EXC_SOFTWARE;
-					code = EXC_PPC_TRAP;
+				code = vm_fault(map, vm_map_trunc_page(dar),
+					 dsisr & MASK(DSISR_WRITE) ? PROT_RW : PROT_RO,
+					 FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
+	
+				if ((code != KERN_SUCCESS) && (code != KERN_ABORTED)) {
+					UPDATE_PPC_EXCEPTION_STATE;
+					exception = EXC_BAD_ACCESS;
+					subcode = dar;
+				} else { 
+					ssp->save_hdr.save_flags |= SAVredrive;	/* Tell low-level to retry fault */
+					ssp->save_dsisr = (ssp->save_dsisr & 
+						~((MASK(DSISR_NOEX) | MASK(DSISR_PROT)))) | MASK(DSISR_HASH);	/* Make sure this is marked as a miss */
 				}
-				subcode = (unsigned int)ssp->save_srr0;
-			}
-			break;
-			
-		case T_ALTIVEC_ASSIST:
-			UPDATE_PPC_EXCEPTION_STATE;
-			exception = EXC_ARITHMETIC;
-			code = EXC_PPC_ALTIVECASSIST;
-			subcode = (unsigned int)ssp->save_srr0;
-			break;
-
-		case T_DATA_ACCESS:
-			map = thread->map;
-
-			if(ssp->save_dsisr & dsiInvMode) {			/* Did someone try to reserve cache inhibited? */
-				UPDATE_PPC_EXCEPTION_STATE;				/* Don't even bother VM with this one */
-				exception = EXC_BAD_ACCESS;
-				subcode = (unsigned int)dar;
 				break;
-			}
-			
-			code = vm_fault(map, vm_map_trunc_page(dar),
-				 dsisr & MASK(DSISR_WRITE) ? PROT_RW : PROT_RO,
-				 FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
-
-			if ((code != KERN_SUCCESS) && (code != KERN_ABORTED)) {
-				UPDATE_PPC_EXCEPTION_STATE;
-				exception = EXC_BAD_ACCESS;
-				subcode = (unsigned int)dar;
-			} else { 
-				ssp->save_hdr.save_flags |= SAVredrive;	/* Tell low-level to re-try fault */
-				ssp->save_dsisr = (ssp->save_dsisr & 
-					~((MASK(DSISR_NOEX) | MASK(DSISR_PROT)))) | MASK(DSISR_HASH);	/* Make sure this is marked as a miss */
-			}
-			break;
-			
-		case T_INSTRUCTION_ACCESS:
-			/* Same as for data access, except fault type
-			 * is PROT_EXEC and addr comes from srr0
-			 */
-			map = thread->map;
-			
-			code = vm_fault(map, vm_map_trunc_page(ssp->save_srr0),
-					(PROT_EXEC | PROT_RO), FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
-
-			if ((code != KERN_SUCCESS) && (code != KERN_ABORTED)) {
-				UPDATE_PPC_EXCEPTION_STATE;
-				exception = EXC_BAD_ACCESS;
-				subcode = (unsigned int)ssp->save_srr0;
-			} else { 
-				ssp->save_hdr.save_flags |= SAVredrive;	/* Tell low-level to re-try fault */
-				ssp->save_srr1 = (ssp->save_srr1 & 
-					~((unsigned long long)(MASK(DSISR_NOEX) | MASK(DSISR_PROT)))) | MASK(DSISR_HASH);		/* Make sure this is marked as a miss */
-			}
-			break;
-
-		case T_AST:
-			/* AST delivery is done below */
-			break;
+				
+			case T_INSTRUCTION_ACCESS:
+				/* Same as for data access, except fault type
+				 * is PROT_EXEC and addr comes from srr0
+				 */
+				map = thread->map;
+				
+				code = vm_fault(map, vm_map_trunc_page(ssp->save_srr0),
+						(PROT_EXEC | PROT_RO), FALSE, THREAD_ABORTSAFE, NULL, vm_map_trunc_page(0));
+	
+				if ((code != KERN_SUCCESS) && (code != KERN_ABORTED)) {
+					UPDATE_PPC_EXCEPTION_STATE;
+					exception = EXC_BAD_ACCESS;
+					subcode = ssp->save_srr0;
+				} else { 
+					ssp->save_hdr.save_flags |= SAVredrive;	/* Tell low-level to re-try fault */
+					ssp->save_srr1 = (ssp->save_srr1 & 
+						~((unsigned long long)(MASK(DSISR_NOEX) | MASK(DSISR_PROT)))) | MASK(DSISR_HASH);		/* Make sure this is marked as a miss */
+				}
+				break;
+	
+			case T_AST:
+				/* AST delivery is done below */
+				break;
 			
 		}
+		
 #ifdef MACH_BSD
 		{
 		bsd_uprofil(&tv, ssp->save_srr0);
@@ -749,8 +810,8 @@ int syscall_trace_end(int retval, struct savearea *ssp)
 
 int syscall_error(
 	int exception,
-	int code,
-	int subcode,
+	mach_exception_code_t code,
+	mach_exception_subcode_t subcode,
 	struct savearea *ssp)
 {
 	register thread_t thread;
@@ -772,17 +833,17 @@ int syscall_error(
 void
 doexception(
 	    int exc,
-	    int code,
-	    int sub)
+	    mach_exception_code_t code,
+	    mach_exception_subcode_t sub)
 {
-	exception_data_type_t   codes[EXCEPTION_CODE_MAX];
+	mach_exception_data_type_t   codes[EXCEPTION_CODE_MAX];
 
 	codes[0] = code;	
 	codes[1] = sub;
 	exception_triage(exc, codes, 2);
 }
 
-char *trap_type[] = {
+const char *trap_type[] = {
 	"Unknown",
 	"0x100 - System reset",
 	"0x200 - Machine check",
@@ -827,20 +888,37 @@ int TRAP_TYPES = sizeof (trap_type) / sizeof (trap_type[0]);
 
 void unresolved_kernel_trap(int trapno,
 			    struct savearea *ssp,
-			    unsigned int dsisr,
+			    __unused unsigned int dsisr,
 			    addr64_t dar,
 			    const char *message)
 {
-	char *trap_name;
-	extern void print_backtrace(struct savearea *);
-	extern unsigned int debug_mode, disableDebugOuput;
-	extern unsigned long panic_caller;
+	const char *trap_name;
 
 	ml_set_interrupts_enabled(FALSE);					/* Turn off interruptions */
 	lastTrace = LLTraceSet(0);							/* Disable low-level tracing */
+	
+#if 0
+	{
+		struct per_proc_info *pp;
+		kprintf("  srr0: %016llX\n", ssp->save_srr0);	/* (TEST/DEBUG) */
+		kprintf("  srr1: %016llX\n", ssp->save_srr1);	/* (TEST/DEBUG) */
+		kprintf("   dar: %016llX\n", ssp->save_dar);	/* (TEST/DEBUG) */
+		kprintf("   xcp: %08X\n", ssp->save_exception);	/* (TEST/DEBUG) */
+		kprintf("  ins0: %08X\n", ssp->save_instr[0]);	/* (TEST/DEBUG) */
+		kprintf("  ins1: %08X\n", ssp->save_instr[1]);	/* (TEST/DEBUG) */
+		kprintf("  ins2: %08X\n", ssp->save_instr[2]);	/* (TEST/DEBUG) */
+		kprintf("  ins3: %08X\n", ssp->save_instr[3]);	/* (TEST/DEBUG) */
+		kprintf("  ins4: %08X\n", ssp->save_instr[4]);	/* (TEST/DEBUG) */
+		kprintf("  ins5: %08X\n", ssp->save_instr[5]);	/* (TEST/DEBUG) */
+		kprintf("  ins6: %08X\n", ssp->save_instr[6]);	/* (TEST/DEBUG) */
+		kprintf("  ins7: %08X\n", ssp->save_instr[7]);	/* (TEST/DEBUG) */
+		pp = getPerProc();								/* (TEST/DEBUG) */
+		kprintf("ijsave: %016llX\n", pp->ijsave);		/* (TEST/DEBUG) */
+	}
+#endif
 
 	if( logPanicDataToScreen )
-		disableDebugOuput = FALSE;
+		disable_debug_output = FALSE;
 
 	debug_mode++;
 	if ((unsigned)trapno <= T_MAX)
@@ -856,11 +934,23 @@ void unresolved_kernel_trap(int trapno,
 	print_backtrace(ssp);
 
 	panic_caller = (0xFFFF0000 | (trapno / T_VECTOR_SIZE) );
+	/* Commit the panic log buffer to NVRAM, unless otherwise
+	 * specified via a boot-arg.
+	 */
+	if (panicDebugging)
+		commit_paniclog();
+
 	draw_panic_dialog();
-		
+	/* XXX: This is yet another codepath into the debugger, which should
+	 * be reworked to enter the primary panic codepath instead.
+	 * The idea appears to be to enter the debugger (performing a
+	 * stack switch) as soon as possible, but we do have a 
+	 * savearea encapsulating state (accessible by walking the savearea
+	 * chain), so that's superfluous.
+	 */
 	if( panicDebugging )
-		(void *)Call_Debugger(trapno, ssp);
-	panic(message);
+		(void)Call_Debugger(trapno, ssp);
+	panic_plain(message);
 }
 
 const char *corr[2] = {"uncorrected", "corrected  "};
diff --git a/osfmk/ppc/trap.h b/osfmk/ppc/trap.h
index bb758d7d5..372526236 100644
--- a/osfmk/ppc/trap.h
+++ b/osfmk/ppc/trap.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -68,7 +74,8 @@
 #include <mach/kern_return.h>
 #include <ppc/thread.h>
 
-extern void			doexception(int exc, int code, int sub);
+extern void			doexception(int exc, mach_exception_code_t code,
+					mach_exception_subcode_t sub);
 
 extern struct savearea*	trap(int trapno,
 				     struct savearea *ss,
@@ -88,8 +95,8 @@ extern struct savearea* interrupt(int intno,
 					 unsigned int dar);
 
 extern int			syscall_error(int exception,
-					      int code,
-					      int subcode,
+					      int64_t code,
+					      int64_t subcode,
 					      struct savearea *ss);
 
 
diff --git a/osfmk/ppc/vm_tuning.h b/osfmk/ppc/vm_tuning.h
index a0254bc7c..6cf00baeb 100644
--- a/osfmk/ppc/vm_tuning.h
+++ b/osfmk/ppc/vm_tuning.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/ppc/vmachmon.c b/osfmk/ppc/vmachmon.c
index 3eeee08a0..ec7ab941b 100644
--- a/osfmk/ppc/vmachmon.c
+++ b/osfmk/ppc/vmachmon.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-----------------------------------------------------------------------
 ** vmachmon.c
@@ -37,6 +43,9 @@
 #include <ppc/exception.h>
 #include <ppc/mappings.h>
 #include <ppc/thread.h>
+#include <ppc/savearea.h>
+#include <ppc/misc_protos.h>
+#include <ppc/fpu_protos.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_fault.h>
 
@@ -125,6 +134,7 @@ static pmap_vmm_ext *vmm_build_shadow_hash(pmap_t pmap)
 	uint32_t		pages = GV_HPAGES;						/* Number of pages in the hash table */
 	vm_offset_t		free = VMX_HPIDX_OFFSET;				/* Offset into extension page of free area (128-byte aligned) */
 	uint32_t		freeSize  = PAGE_SIZE - free;			/* Number of free bytes in the extension page */
+	uint32_t	idx;
 															
 	if ((pages * sizeof(addr64_t)) + (pages * sizeof(vm_offset_t)) > freeSize) {
 		panic("vmm_build_shadow_hash: too little pmap_vmm_ext free space\n");
@@ -138,7 +148,7 @@ static pmap_vmm_ext *vmm_build_shadow_hash(pmap_t pmap)
 	extPP = pmap_find_phys(kernel_pmap, (vm_offset_t)ext);
 															/* Get extension block's physical page number */
 	if (!extPP) {											/* This should not fail, but then again... */
-		panic("vmm_build_shadow_hash: could not translate pmap_vmm_ext vaddr %08X\n", ext);
+		panic("vmm_build_shadow_hash: could not translate pmap_vmm_ext vaddr %p\n", ext);
 	}
 	
 	ext->vmxSalt	     = (addr64_t)(vm_offset_t)ext ^ ptoa_64(extPP);
@@ -156,8 +166,9 @@ static pmap_vmm_ext *vmm_build_shadow_hash(pmap_t pmap)
 	/* The hash table is typically larger than a single page, but we don't require it to be in a
 	   contiguous virtual or physical chunk. So, we allocate it page by page, noting the effective and
 	   physical address of each page in vmxHashPgList and vmxHashPgIdx, respectively. */
-	uint32_t	idx;
 	for (idx = 0; idx < pages; idx++) {
+		mapping_t *map;
+		uint32_t mapIdx;
 		ret = kmem_alloc_wired(kernel_map, &ext->vmxHashPgList[idx], PAGE_SIZE);
 															/* Allocate a hash-table page */
 		if (ret != KERN_SUCCESS) goto fail;					/* Allocation failed, exit through cleanup */
@@ -167,8 +178,7 @@ static pmap_vmm_ext *vmm_build_shadow_hash(pmap_t pmap)
 		if (!ext->vmxHashPgIdx[idx]) {						/* Hash-table page's LRA failed */
 			panic("vmm_build_shadow_hash: could not translate hash-table vaddr %08X\n", ext->vmxHashPgList[idx]);
 		}
-		mapping_t *map = (mapping_t *)ext->vmxHashPgList[idx];
-		uint32_t mapIdx;
+		map = (mapping_t *)ext->vmxHashPgList[idx];
 		for (mapIdx = 0; mapIdx < GV_SLTS_PPG; mapIdx++) {	/* Iterate over mappings in this page */
 			map->mpFlags = (mpGuest | mpgFree);				/* Mark guest type and free */
 			map = (mapping_t *)((char *)map + GV_SLOT_SZ);	/* Next slot-sized mapping */
@@ -216,22 +226,25 @@ static kern_return_t vmm_activate_gsa(
 	vmm_thread_index_t	index)
 {
 	vmmCntrlTable	*CTable = act->machine.vmmControl;		/* Get VMM control table */
+	vmmCntrlEntry *CEntry;
+	pmap_t hpmap;
+	pmap_t gpmap;
 	if (!CTable) {											/* Caller guarantees that this will work */
-		panic("vmm_activate_gsa: VMM control table not present; act = %08X, idx = %d\n",
+		panic("vmm_activate_gsa: VMM control table not present; act = %p, idx = %lu\n",
 			act, index);
 		return KERN_FAILURE;
 	}
-	vmmCntrlEntry	*CEntry = vmm_get_entry(act, index);	/* Get context from index */
+	CEntry = vmm_get_entry(act, index);	/* Get context from index */
 	if (!CEntry) {											/* Caller guarantees that this will work */
-		panic("vmm_activate_gsa: Unexpected failure of vmm_get_entry; act = %08X, idx = %d\n",
+		panic("vmm_activate_gsa: Unexpected failure of vmm_get_entry; act = %p, idx = %lu\n",
 			act, index);
 		return KERN_FAILURE;
 	}
 
-	pmap_t	hpmap = act->map->pmap;							/* Get host pmap */
-	pmap_t	gpmap = vmm_get_adsp(act, index);				/* Get guest pmap */
+	hpmap = act->map->pmap;							/* Get host pmap */
+	gpmap = vmm_get_adsp(act, index);				/* Get guest pmap */
 	if (!gpmap) {											/* Caller guarantees that this will work */
-		panic("vmm_activate_gsa: Unexpected failure of vmm_get_adsp; act = %08X, idx = %d\n",
+		panic("vmm_activate_gsa: Unexpected failure of vmm_get_adsp; act = %p, idx = %lu\n",
 			act, index);
 		return KERN_FAILURE;
 	}
@@ -260,22 +273,22 @@ static kern_return_t vmm_activate_gsa(
 ** Deactivate guest shadow assist
 **
 -----------------------------------------------------------------------*/
-static void vmm_deactivate_gsa(
+static void
+vmm_deactivate_gsa(
 	thread_t			act,
 	vmm_thread_index_t	index)
 {
 	vmmCntrlEntry	*CEntry = vmm_get_entry(act, index);	/* Get context from index */
+	pmap_t	gpmap;
 	if (!CEntry) {											/* Caller guarantees that this will work */
-		panic("vmm_deactivate_gsa: Unexpected failure of vmm_get_entry; act = %08X, idx = %d\n",
+		panic("vmm_deactivate_gsa: Unexpected failure of vmm_get_entry; act = %p, idx = %lu\n",
 			act, index);
-		return KERN_FAILURE;
 	}
 
-	pmap_t	gpmap = vmm_get_adsp(act, index);				/* Get guest pmap */
+	gpmap = vmm_get_adsp(act, index);				/* Get guest pmap */
 	if (!gpmap) {											/* Caller guarantees that this will work */
-		panic("vmm_deactivate_gsa: Unexpected failure of vmm_get_adsp; act = %08X, idx = %d\n",
+		panic("vmm_deactivate_gsa: Unexpected failure of vmm_get_adsp; act = %p, idx = %lu\n",
 			act, index);
-		return KERN_FAILURE;
 	}
 	
 	gpmap->pmapFlags &= ~pmapVMgsaa;						/* Deactivate GSA for this guest */
@@ -301,19 +314,19 @@ static void vmm_flush_context(
 
 	CEntry = vmm_get_entry(act, index);				/* Convert index to entry */
 	if (!CEntry) {									/* Caller guarantees that this will work */
-		panic("vmm_flush_context: Unexpected failure of vmm_get_entry; act = %08X, idx = %d\n",
+		panic("vmm_flush_context: Unexpected failure of vmm_get_entry; act = %p, idx = %lu\n",
 			act, index);
 		return;
 	}
 
 	if(CEntry->vmmFacCtx.FPUsave) {					/* Is there any floating point context? */
 		toss_live_fpu(&CEntry->vmmFacCtx);			/* Get rid of any live context here */
-		save_release((savearea *)CEntry->vmmFacCtx.FPUsave);	/* Release it */
+		save_release((struct savearea *)CEntry->vmmFacCtx.FPUsave);	/* Release it */
 	}
 
 	if(CEntry->vmmFacCtx.VMXsave) {					/* Is there any vector context? */
 		toss_live_vec(&CEntry->vmmFacCtx);			/* Get rid of any live context here */
-		save_release((savearea *)CEntry->vmmFacCtx.VMXsave);	/* Release it */
+		save_release((struct savearea *)CEntry->vmmFacCtx.VMXsave);	/* Release it */
 	}
 	
 	vmm_unmap_all_pages(act, index);				/* Blow away all mappings for this context */
@@ -322,11 +335,11 @@ static void vmm_flush_context(
 	CTable->vmmGFlags = CTable->vmmGFlags & ~vmmLastAdSp;	/* Make sure we don't try to automap into this */
 	
 	CEntry->vmmFlags &= vmmInUse;					/* Clear out all of the flags for this entry except in use */
-	CEntry->vmmFacCtx.FPUsave = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.FPUlevel = 0;					/* Clear facility context control */
+	CEntry->vmmFacCtx.FPUsave = NULL;					/* Clear facility context control */
+	CEntry->vmmFacCtx.FPUlevel = NULL;					/* Clear facility context control */
 	CEntry->vmmFacCtx.FPUcpu = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.VMXsave = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.VMXlevel = 0;					/* Clear facility context control */
+	CEntry->vmmFacCtx.VMXsave = NULL;					/* Clear facility context control */
+	CEntry->vmmFacCtx.VMXlevel = NULL;					/* Clear facility context control */
 	CEntry->vmmFacCtx.VMXcpu = 0;					/* Clear facility context control */
 	
 	vks = CEntry->vmmContextKern;					/* Get address of the context page */
@@ -336,7 +349,7 @@ static void vmm_flush_context(
 	vks->interface_version = version;				/* Set our version code */
 	vks->thread_index = index % vmmTInum;			/* Tell the user the index for this virtual machine */
 		
-	return;											/* Context is now flushed */
+	/* Context is now flushed */
 }
 
 
@@ -408,7 +421,8 @@ int vmm_get_features(struct savearea *save)
 **		Returns max address
 -----------------------------------------------------------------------*/
 
-addr64_t vmm_max_addr(thread_t act) 
+addr64_t
+vmm_max_addr(__unused thread_t act) 
 {
 	return vm_max_address;							/* Return the maximum address */
 }
@@ -479,6 +493,8 @@ int vmm_init_context(struct savearea *save)
 	int					cvi, i;
     task_t				task;
     thread_t			fact, gact;
+	pmap_t hpmap;
+	pmap_t gpmap;
 
 	vmm_user_state = CAST_DOWN(vmm_state_page_t *, save->save_r4);  /* Get the user address of the comm area */
 	if ((unsigned int)vmm_user_state & (PAGE_SIZE - 1)) {	/* Make sure the comm area is page aligned */
@@ -507,7 +523,7 @@ int vmm_init_context(struct savearea *save)
 	task_lock(task);							/* Lock our task */
 
 	fact = (thread_t)task->threads.next;	/* Get the first activation on task */
-	gact = 0;									/* Pretend we didn't find it yet */
+	gact = NULL;									/* Pretend we didn't find it yet */
 
 	for(i = 0; i < task->thread_count; i++) {	/* All of the activations */
 		if(fact->machine.vmmControl) {				/* Is this a virtual machine monitor? */
@@ -541,7 +557,7 @@ int vmm_init_context(struct savearea *save)
 	CTable = act->machine.vmmControl;				/* Get the control table address */
 	if ((unsigned int)CTable == 1) {			/* If we are marked, try to allocate a new table, otherwise we have one */
 		if(!(CTable = (vmmCntrlTable *)kalloc(sizeof(vmmCntrlTable)))) {	/* Get a fresh emulation control table */
-			act->machine.vmmControl = 0;			/* Unmark us as vmm 'cause we failed */
+			act->machine.vmmControl = NULL;			/* Unmark us as vmm 'cause we failed */
 			ml_set_interrupts_enabled(FALSE);	/* Set back interruptions */
 			save->save_r3 = KERN_RESOURCE_SHORTAGE;		/* No storage... */
 			return 1;
@@ -605,18 +621,18 @@ int vmm_init_context(struct savearea *save)
 	CTable->vmmc[cvi].vmmContextPhys = conphys;	/* Remember the state page physical addr */
 	CTable->vmmc[cvi].vmmContextUser = vmm_user_state;		/* Remember user address of comm area */
 	
-	CTable->vmmc[cvi].vmmFacCtx.FPUsave = 0;	/* Clear facility context control */
-	CTable->vmmc[cvi].vmmFacCtx.FPUlevel = 0;	/* Clear facility context control */
+	CTable->vmmc[cvi].vmmFacCtx.FPUsave = NULL;	/* Clear facility context control */
+	CTable->vmmc[cvi].vmmFacCtx.FPUlevel = NULL;	/* Clear facility context control */
 	CTable->vmmc[cvi].vmmFacCtx.FPUcpu = 0;		/* Clear facility context control */
-	CTable->vmmc[cvi].vmmFacCtx.VMXsave = 0;	/* Clear facility context control */
-	CTable->vmmc[cvi].vmmFacCtx.VMXlevel = 0;	/* Clear facility context control */
+	CTable->vmmc[cvi].vmmFacCtx.VMXsave = NULL;	/* Clear facility context control */
+	CTable->vmmc[cvi].vmmFacCtx.VMXlevel = NULL;	/* Clear facility context control */
 	CTable->vmmc[cvi].vmmFacCtx.VMXcpu = 0;		/* Clear facility context control */
 	CTable->vmmc[cvi].vmmFacCtx.facAct = act;	/* Point back to the activation */
 
-	hw_atomic_add((int *)&saveanchor.savetarget, 2);	/* Account for the number of extra saveareas we think we might "need" */
+	(void)hw_atomic_add(&saveanchor.savetarget, 2);	/* Account for the number of extra saveareas we think we might "need" */
 
-	pmap_t hpmap = act->map->pmap;						/* Get host pmap */
-	pmap_t gpmap = pmap_create(0, FALSE);					/* Make a fresh guest pmap */
+	hpmap = act->map->pmap;						/* Get host pmap */
+	gpmap = pmap_create(0, FALSE);					/* Make a fresh guest pmap */
 	if (gpmap) {										/* Did we succeed ? */
 		CTable->vmmAdsp[cvi] = gpmap;					/* Remember guest pmap for new context */
 		if (lowGlo.lgVMMforcedFeats & vmmGSA) {			/* Forcing on guest shadow assist ? */
@@ -637,7 +653,7 @@ int vmm_init_context(struct savearea *save)
 
 return_in_shame:
 	if(!gact) kfree(CTable, sizeof(vmmCntrlTable));	/* Toss the table if we just allocated it */
-	act->machine.vmmControl = 0;					/* Unmark us as vmm 'cause we failed */
+	act->machine.vmmControl = NULL;					/* Unmark us as vmm 'cause we failed */
 	ml_set_interrupts_enabled(FALSE);			/* Set back interruptions */
 	save->save_r3 = ret;						/* Pass back return code... */	
 	return 1;
@@ -672,27 +688,28 @@ kern_return_t vmm_tear_down_context(
 	vmmCntrlEntry 		*CEntry;
 	vmmCntrlTable		*CTable;
 	int					cvi;
-	register savearea 	*sv;
+	pmap_t gpmap;
+	pmap_t pmap;
 
 	CEntry = vmm_get_entry(act, index);					/* Convert index to entry */		
 	if (CEntry == NULL) return KERN_FAILURE;			/* Either this isn't vmm thread or the index is bogus */
 
 	ml_set_interrupts_enabled(TRUE);					/* This can take a bit of time so pass interruptions */
 
-	hw_atomic_sub((int *)&saveanchor.savetarget, 2);	/* We don't need these extra saveareas anymore */
+	(void)hw_atomic_sub(&saveanchor.savetarget, 2);	/* We don't need these extra saveareas anymore */
 
 	if(CEntry->vmmFacCtx.FPUsave) {						/* Is there any floating point context? */
 		toss_live_fpu(&CEntry->vmmFacCtx);				/* Get rid of any live context here */
-		save_release((savearea *)CEntry->vmmFacCtx.FPUsave);	/* Release it */
+		save_release((struct savearea *)CEntry->vmmFacCtx.FPUsave);	/* Release it */
 	}
 
 	if(CEntry->vmmFacCtx.VMXsave) {						/* Is there any vector context? */
 		toss_live_vec(&CEntry->vmmFacCtx);				/* Get rid of any live context here */
-		save_release((savearea *)CEntry->vmmFacCtx.VMXsave);	/* Release it */
+		save_release((struct savearea *)CEntry->vmmFacCtx.VMXsave);	/* Release it */
 	}
 	
-	CEntry->vmmPmap = 0;								/* Remove this trace */
-	pmap_t gpmap = act->machine.vmmControl->vmmAdsp[index - 1];
+	CEntry->vmmPmap = NULL;								/* Remove this trace */
+	gpmap = act->machine.vmmControl->vmmAdsp[index - 1];
 														/* Get context's guest pmap (if any) */
 	if (gpmap) {										/* Check if there is an address space assigned here */
 		if (gpmap->pmapFlags & pmapVMgsaa) {			/* Handle guest shadow assist case specially */
@@ -717,16 +734,16 @@ kern_return_t vmm_tear_down_context(
 	CTable->vmmGFlags = CTable->vmmGFlags & ~vmmLastAdSp;	/* Make sure we don't try to automap into this */
 
 	CEntry->vmmFlags = 0;							/* Clear out all of the flags for this entry including in use */
-	CEntry->vmmContextKern = 0;						/* Clear the kernel address of comm area */
-	CEntry->vmmContextUser = 0;						/* Clear the user address of comm area */
+	CEntry->vmmContextKern = NULL;						/* Clear the kernel address of comm area */
+	CEntry->vmmContextUser = NULL;						/* Clear the user address of comm area */
 	
-	CEntry->vmmFacCtx.FPUsave = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.FPUlevel = 0;					/* Clear facility context control */
+	CEntry->vmmFacCtx.FPUsave = NULL;					/* Clear facility context control */
+	CEntry->vmmFacCtx.FPUlevel = NULL;					/* Clear facility context control */
 	CEntry->vmmFacCtx.FPUcpu = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.VMXsave = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.VMXlevel = 0;					/* Clear facility context control */
+	CEntry->vmmFacCtx.VMXsave = NULL;					/* Clear facility context control */
+	CEntry->vmmFacCtx.VMXlevel = NULL;					/* Clear facility context control */
 	CEntry->vmmFacCtx.VMXcpu = 0;					/* Clear facility context control */
-	CEntry->vmmFacCtx.facAct = 0;					/* Clear facility context control */
+	CEntry->vmmFacCtx.facAct = NULL;					/* Clear facility context control */
 	
 	for(cvi = 0; cvi < kVmmMaxContexts; cvi++) {	/* Search to find a free slot */
 		if(CTable->vmmc[cvi].vmmFlags & vmmInUse) {	/* Return if there are still some in use */
@@ -745,19 +762,19 @@ kern_return_t vmm_tear_down_context(
 		mapping_remove(act->machine.vmmControl->vmmAdsp[index - 1], 0xFFFFFFFFFFFFF000LL);	/* Remove final page explicitly because we might have mapped it */	
 		pmap_remove(act->machine.vmmControl->vmmAdsp[index - 1], 0, 0xFFFFFFFFFFFFF000LL);	/* Remove all entries from this map */
 		pmap_destroy(act->machine.vmmControl->vmmAdsp[index - 1]);	/* Toss the pmap for this context */
-		act->machine.vmmControl->vmmAdsp[index - 1] = 0;	/* Clear just in case */
+		act->machine.vmmControl->vmmAdsp[index - 1] = NULL;	/* Clear just in case */
 	}
 	
-	pmap_t pmap = act->map->pmap;					/* Get our pmap */
+	pmap = act->map->pmap;					/* Get our pmap */
 	if (pmap->pmapVmmExt) {							/* Release any VMM pmap extension block and shadow hash table */
 		vmm_release_shadow_hash(pmap->pmapVmmExt);	/* Release extension block and shadow hash table */
-		pmap->pmapVmmExt     = 0;					/* Forget extension block */
+		pmap->pmapVmmExt     = NULL;					/* Forget extension block */
 		pmap->pmapVmmExtPhys = 0;					/* Forget extension block's physical address, too */
 	}
 	pmap->pmapFlags &= ~pmapVMhost;					/* We're no longer hosting */
 
 	kfree(CTable, sizeof(vmmCntrlTable));	/* Toss the table because to tossed the last context */
-	act->machine.vmmControl = 0;						/* Unmark us as vmm */
+	act->machine.vmmControl = NULL;						/* Unmark us as vmm */
 
 	ml_set_interrupts_enabled(FALSE);				/* No more interruptions */
 	
@@ -877,7 +894,7 @@ void vmm_tear_down_all(thread_t act) {
 	vmmCntrlTable		*CTable;
 	int					cvi;
 	kern_return_t		ret;
-	savearea			*save;
+	struct savearea			*save;
 	spl_t				s;
 	
 	if(act->machine.specFlags & runningVM) {			/* Are we actually in a context right now? */
@@ -893,14 +910,13 @@ void vmm_tear_down_all(thread_t act) {
 		splx(s);									/* Restore interrupts */
 	}
 	
-	if(CTable = act->machine.vmmControl) {				/* Do we have a vmm control block? */
-
-
+	if(act->machine.vmmControl) { /* Do we have a vmm control block? */
+		CTable = act->machine.vmmControl;
 		for(cvi = 1; cvi <= kVmmMaxContexts; cvi++) {	/* Look at all slots */
 			if(CTable->vmmc[cvi - 1].vmmFlags & vmmInUse) {	/* Is this one in use */
 				ret = vmm_tear_down_context(act, cvi);	/* Take down the found context */
 				if(ret != KERN_SUCCESS) {			/* Did it go away? */
-					panic("vmm_tear_down_all: vmm_tear_down_context failed; ret=%08X, act = %08X, cvi = %d\n",
+					panic("vmm_tear_down_all: vmm_tear_down_context failed; ret=%08X, act = %p, cvi = %d\n",
 					  ret, act, cvi);
 				}
 			}
@@ -913,8 +929,6 @@ void vmm_tear_down_all(thread_t act) {
 			panic("vmm_tear_down_all: control table did not get deallocated\n");	/* Table did not go away */
 		}
 	}
-
-	return;
 }
 
 /*-----------------------------------------------------------------------
@@ -978,6 +992,12 @@ kern_return_t vmm_map_page(
 		ret = hw_res_map_gv(map->pmap, pmap, cva, ava, getProtPPC(prot, TRUE));
 												/* Attempt to resume an existing gv->phys mapping */
 		if (mapRtOK != ret) {					/* Nothing to resume, construct a new mapping */
+			unsigned int  pindex;
+			phys_entry_t *physent;
+			unsigned int pattr;
+			unsigned int wimg;
+			unsigned int mflags;
+			addr64_t gva;
 			
 			while (1) {							/* Find host mapping or fail */
 				mp = mapping_find(map->pmap, cva, &nextva, 0);
@@ -1006,18 +1026,17 @@ kern_return_t vmm_map_page(
 			}
 	
 												/* Partially construct gv->phys mapping */
-			unsigned int  pindex;
-			phys_entry_t *physent = mapping_phys_lookup(mp->mpPAddr, &pindex);
+			physent = mapping_phys_lookup(mp->mpPAddr, &pindex);
 			if (!physent) {
 				mapping_drop_busy(mp);
 				return KERN_FAILURE;
 			}
-			unsigned int pattr = ((physent->ppLink & (ppI | ppG)) >> 60);
-			unsigned int wimg = 0x2;
+			pattr = ((physent->ppLink & (ppI | ppG)) >> 60);
+			wimg = 0x2;
 			if (pattr & mmFlgCInhib)  wimg |= 0x4;
 			if (pattr & mmFlgGuarded) wimg |= 0x1;
-			unsigned int mflags = (pindex << 16) | mpGuest;
-			addr64_t	 gva = ((ava & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot, TRUE));
+			mflags = (pindex << 16) | mpGuest;
+			gva = ((ava & ~mpHWFlags) | (wimg << 3) | getProtPPC(prot, TRUE));
 			
 			hw_add_map_gv(map->pmap, pmap, gva, mflags, mp->mpPAddr);
 												/* Construct new guest->phys mapping */
@@ -1178,14 +1197,16 @@ kern_return_t vmm_map_list(
 
 		prot = ava & vmmlProt;						/* Extract the protection bits */	
 		adsp = (ava & vmmlAdID) >> 4;				/* Extract an explicit address space request */	
-		if(!adsp) adsp = index - 1;					/* If no explicit, use supplied default */
-		ava = ava &= 0xFFFFFFFFFFFFF000ULL;			/* Clean up the address */
+		if(!adsp) /* If no explicit, use supplied default */
+			adsp = index - 1;
+		ava &= 0xFFFFFFFFFFFFF000ULL; /* Clean up the address */
 		
 		ret = vmm_map_page(act, index, cva, ava, prot);	/* Go try to map the page on in */
-		if(ret != KERN_SUCCESS) return ret;			/* Bail if any error */
+		if(ret != KERN_SUCCESS) /* Bail if any error */
+			return ret;
 	}
 	
-	return KERN_SUCCESS	;							/* Return... */
+	return KERN_SUCCESS;
 }
 
 /*-----------------------------------------------------------------------
@@ -1266,7 +1287,6 @@ kern_return_t vmm_unmap_page(
 	vmm_adsp_id_t	 	index,
 	addr64_t	 		va)
 {
-	vmmCntrlEntry 		*CEntry;
 	addr64_t			nadd;
 	pmap_t				pmap;
 
@@ -1308,9 +1328,8 @@ kern_return_t vmm_unmap_list(
 	unsigned int		flavor)
 {
 	vmmCntrlEntry 		*CEntry;
-	boolean_t			ret;
 	kern_return_t		kern_result = KERN_SUCCESS;
-	unsigned int		*pgaddr, i;
+	unsigned int i;
 	addr64_t			gva;
 	vmmUMList			*lst;
 	vmmUMList64			*lstx;
@@ -1318,12 +1337,22 @@ kern_return_t vmm_unmap_list(
 	int					adsp;
 
 	CEntry = vmm_get_entry(act, index);				/* Convert index to entry */		
-	if (CEntry == NULL) return KERN_FAILURE;		/* Either this isn't a vmm or the index is bogus */
+	if (CEntry == NULL) { /* Either this isn't a vmm or the index is bogus */
+		kern_result = KERN_FAILURE;
+		goto out;
+	}
 	
-	if(cnt > kVmmMaxUnmapPages) return KERN_FAILURE;	/* They tried to unmap too many */
-	if(!cnt) return KERN_SUCCESS;					/* If they said none, we're done... */
+	if(cnt > kVmmMaxUnmapPages) { /* They tried to unmap too many */
+		kern_result = KERN_FAILURE;
+		goto out;
+	}
+	if(!cnt) { /* If they said none, we're done... */
+		kern_result = KERN_SUCCESS;
+		goto out;
+	}
 	
-	lst = (vmmUMList *)lstx = (vmmUMList64 *) &((vmm_comm_page_t *)CEntry->vmmContextKern)->vmcpComm[0];	/* Point to the first entry */
+	lstx = (vmmUMList64 *) &((vmm_comm_page_t *)CEntry->vmmContextKern)->vmcpComm[0];	/* Point to the first entry */
+	lst = (vmmUMList *)lstx;
 	
 	for(i = 0; i < cnt; i++) {						/* Step and release all pages in list */
 		if(flavor) {								/* Check if 32- or 64-bit addresses */
@@ -1334,11 +1363,13 @@ kern_return_t vmm_unmap_list(
 		}
 
 		adsp = (gva & vmmlAdID) >> 4;				/* Extract an explicit address space request */	
-		if(!adsp) adsp = index - 1;					/* If no explicit, use supplied default */
+		if(!adsp) /* If no explicit, use supplied default */
+			adsp = index - 1;
 		pmap = act->machine.vmmControl->vmmAdsp[adsp];	/* Get the pmap for this request */
-		if(!pmap) continue;							/* Ain't nuthin' mapped here, no durn map... */
+		if(!pmap)
+			continue;							/* Ain't nuthin' mapped here, no durn map... */
 
-		gva = gva &= 0xFFFFFFFFFFFFF000ULL;			/* Clean up the address */	
+		gva &= 0xFFFFFFFFFFFFF000ULL; /* Clean up the address */	
 		if (pmap->pmapFlags & pmapVMgsaa) {			/* Handle guest shadow assist specially */
 			hw_susp_map_gv(act->map->pmap, pmap, gva);
 													/* Suspend the mapping */
@@ -1347,7 +1378,8 @@ kern_return_t vmm_unmap_list(
 		}
 	}
 	
-	return KERN_SUCCESS	;							/* Return... */
+out:
+	return kern_result;
 }
 
 /*-----------------------------------------------------------------------
@@ -1371,7 +1403,6 @@ void vmm_unmap_all_pages(
 	thread_t	 		act,
 	vmm_adsp_id_t	 	index)
 {
-	vmmCntrlEntry 		*CEntry;
 	pmap_t				pmap;
 
 	pmap = vmm_get_adsp(act, index);						/* Convert index to entry */		
@@ -1386,7 +1417,6 @@ void vmm_unmap_all_pages(
 		mapping_remove(pmap, 0xFFFFFFFFFFFFF000LL);			/* Remove final page explicitly because we might have mapped it */	
 		pmap_remove(pmap, 0, 0xFFFFFFFFFFFFF000LL);			/* Remove all entries from this map */
 	}
-	return;
 }
 
 
@@ -1417,8 +1447,6 @@ boolean_t vmm_get_page_dirty_flag(
 	addr64_t	 		va,
 	unsigned int		reset)
 {
-	vmmCntrlEntry 		*CEntry;
-	register mapping_t 	*mpv, *mp;
 	unsigned int		RC;
 	pmap_t				pmap;
 
@@ -1442,7 +1470,7 @@ boolean_t vmm_get_page_dirty_flag(
 			break;
 			
 		default:
-			panic("vmm_get_page_dirty_flag: hw_test_rc failed - rc = %d, pmap = %08X, va = %016llX\n", RC, pmap, va);
+			panic("vmm_get_page_dirty_flag: hw_test_rc failed - rc = %d, pmap = %p, va = %016llX\n", RC, pmap, va);
 		
 	}
 
@@ -1474,7 +1502,6 @@ kern_return_t vmm_protect_page(
 	addr64_t	 		va,
 	vm_prot_t			prot)
 {
-	vmmCntrlEntry 		*CEntry;
 	addr64_t			nextva;
 	int	ret;
 	pmap_t				pmap;
@@ -1498,7 +1525,7 @@ kern_return_t vmm_protect_page(
 			break;
 			
 		default:
-			panic("vmm_protect_page: hw_protect failed - rc = %d, pmap = %08X, va = %016llX\n", ret, pmap, (addr64_t)va);
+			panic("vmm_protect_page: hw_protect failed - rc = %d, pmap = %p, va = %016llX\n", ret, pmap, (addr64_t)va);
 		
 	}
 
@@ -1582,7 +1609,6 @@ kern_return_t vmm_get_float_state(
 	vmm_thread_index_t 	index)
 {
 	vmmCntrlEntry 		*CEntry;
-	vmmCntrlTable		*CTable;
 	int					i;
 	register struct savearea_fpu *sv;
 
@@ -1594,7 +1620,8 @@ kern_return_t vmm_get_float_state(
 
 	fpu_save(&CEntry->vmmFacCtx);					/* Save context if live */
 
-	if(sv = CEntry->vmmFacCtx.FPUsave) {			/* Is there context yet? */
+	if(CEntry->vmmFacCtx.FPUsave) { /* Is there context yet? */
+		sv = CEntry->vmmFacCtx.FPUsave;
 		bcopy((char *)&sv->save_fp0, (char *)&(CEntry->vmmContextKern->vmm_proc_state.ppcFPRs), 32 * 8); /* 32 registers */
 		return KERN_SUCCESS;
 	}
@@ -1627,7 +1654,6 @@ kern_return_t vmm_get_vector_state(
 	vmm_thread_index_t 	index)
 {
 	vmmCntrlEntry 		*CEntry;
-	vmmCntrlTable		*CTable;
 	int					i, j;
 	unsigned int 		vrvalidwrk;
 	register struct savearea_vec *sv;
@@ -1640,8 +1666,8 @@ kern_return_t vmm_get_vector_state(
 	act->machine.specFlags &= ~vectorCng;				/* Clear the special flag */
 	CEntry->vmmContextKern->vmmStat &= ~vmmVectCngd;	/* Clear the change indication */
 	
-	if(sv = CEntry->vmmFacCtx.VMXsave) {			/* Is there context yet? */
-
+	if(CEntry->vmmFacCtx.VMXsave) { /* Is there context yet? */
+		sv = CEntry->vmmFacCtx.VMXsave;
 		vrvalidwrk = sv->save_vrvalid;				/* Get the valid flags */
 
 		for(i = 0; i < 32; i++) {					/* Copy the saved registers and invalidate the others */
@@ -1738,7 +1764,6 @@ kern_return_t vmm_get_timer(
 	vmm_thread_index_t 	index)
 {
 	vmmCntrlEntry 		*CEntry;
-	vmmCntrlTable		*CTable;
 
 	CEntry = vmm_get_entry(act, index);				/* Convert index to entry */		
 	if (CEntry == NULL) return KERN_FAILURE;		/* Either this isn't vmm thread or the index is bogus */
@@ -1775,14 +1800,13 @@ kern_return_t vmm_get_timer(
 void vmm_timer_pop(
 	thread_t	 		act)
 {
-	vmmCntrlEntry 		*CEntry;
 	vmmCntrlTable		*CTable;
 	int					cvi, any;
 	uint64_t			now, soonest;
-	savearea			*sv;
+	struct savearea			*sv;
 		
 	if(!((unsigned int)act->machine.vmmControl & 0xFFFFFFFE)) {	/* Are there any virtual machines? */
-		panic("vmm_timer_pop: No virtual machines defined; act = %08X\n", act);
+		panic("vmm_timer_pop: No virtual machines defined; act = %p\n", act);
 	}
 
 	soonest = 0xFFFFFFFFFFFFFFFFULL;				/* Max time */
@@ -1808,7 +1832,7 @@ void vmm_timer_pop(
 			if((unsigned int)&CTable->vmmc[cvi] == (unsigned int)act->machine.vmmCEntry) {	/* Is this the running VM? */
 				sv = find_user_regs(act);			/* Get the user state registers */
 				if(!sv) {							/* Did we find something? */
-					panic("vmm_timer_pop: no user context; act = %08X\n", act);
+					panic("vmm_timer_pop: no user context; act = %p\n", act);
 				}
 				sv->save_exception = kVmmReturnNull*4;	/* Indicate that this is a null exception */
 				vmm_force_exit(act, sv);			/* Intercept a running VM */
@@ -1830,8 +1854,6 @@ void vmm_timer_pop(
 		if (act->machine.qactTimer == 0 || soonest <= act->machine.qactTimer)
 			act->machine.qactTimer = soonest;	/* Set lowest timer */
 	}
-
-	return;
 }
 
 
@@ -1876,7 +1898,7 @@ int vmm_stop_vm(struct savearea *save)
 	task_lock(task);							/* Lock our task */
 
 	fact = (thread_t)task->threads.next;	/* Get the first activation on task */
-	act = 0;									/* Pretend we didn't find it yet */
+	act = NULL;									/* Pretend we didn't find it yet */
 
 	for(i = 0; i < task->thread_count; i++) {	/* All of the activations */
 		if(fact->machine.vmmControl) {				/* Is this a virtual machine monitor? */
@@ -1917,7 +1939,7 @@ int vmm_stop_vm(struct savearea *save)
 
 	for(cvi = 0; cvi < kVmmMaxContexts; cvi++) {	/* Search slots */
 		if((0x80000000 & vmmask) && (CTable->vmmc[cvi].vmmFlags & vmmInUse)) {	/* See if we need to stop and if it is in use */
-			hw_atomic_or(&CTable->vmmc[cvi].vmmFlags, vmmXStop);	/* Set this one to stop */
+			(void)hw_atomic_or(&CTable->vmmc[cvi].vmmFlags, vmmXStop);	/* Set this one to stop */
 		}
 		vmmask = vmmask << 1;					/* Slide mask over */
 	}
@@ -1976,7 +1998,7 @@ int vmm_stop_vm(struct savearea *save)
 void vmm_interrupt(ReturnHandler *rh, thread_t act) {
 
 	vmmCntrlTable		*CTable;
-	savearea			*sv;
+	struct savearea			*sv;
 	boolean_t			inter;
 
 
@@ -1993,12 +2015,10 @@ void vmm_interrupt(ReturnHandler *rh, thread_t act) {
 	if(act->machine.vmmCEntry && (act->machine.vmmCEntry->vmmFlags & vmmXStop)) {	/* Do we need to stop the running guy? */
 		sv = find_user_regs(act);				/* Get the user state registers */
 		if(!sv) {								/* Did we find something? */
-			panic("vmm_interrupt: no user context; act = %08X\n", act);
+			panic("vmm_interrupt: no user context; act = %p\n", act);
 		}
 		sv->save_exception = kVmmStopped*4;		/* Set a "stopped" exception */
 		vmm_force_exit(act, sv);				/* Intercept a running VM */
 	}
 	ml_set_interrupts_enabled(inter);			/* Put interrupts back to what they were */
-
-	return;
 }
diff --git a/osfmk/ppc/vmachmon.h b/osfmk/ppc/vmachmon.h
index b2801eb9b..91626cfa2 100644
--- a/osfmk/ppc/vmachmon.h
+++ b/osfmk/ppc/vmachmon.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*-----------------------------------------------------------------------
 ** vmachmon.h
diff --git a/osfmk/ppc/vmachmon_asm.s b/osfmk/ppc/vmachmon_asm.s
index 538553085..3f8cac654 100644
--- a/osfmk/ppc/vmachmon_asm.s
+++ b/osfmk/ppc/vmachmon_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <assym.s>
 #include <debug.h>
diff --git a/osfmk/ppc/xpr.h b/osfmk/ppc/xpr.h
index 133cb3a1b..c81865886 100644
--- a/osfmk/ppc/xpr.h
+++ b/osfmk/ppc/xpr.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/Makefile b/osfmk/profiling/Makefile
index 878005ec7..410bab236 100644
--- a/osfmk/profiling/Makefile
+++ b/osfmk/profiling/Makefile
@@ -16,6 +16,9 @@ INSTINC_SUBDIRS_PPC = \
 INSTINC_SUBDIRS_I386 = \
 	i386
 
+INSTINC_SUBDIRS_ARM = \
+	arm
+
 EXPINC_SUBDIRS = \
 	machine
 
@@ -25,6 +28,9 @@ EXPINC_SUBDIRS_PPC = \
 EXPINC_SUBDIRS_I386 = \
 	i386
 
+EXPINC_SUBDIRS_ARM = \
+	arm
+
 MIG_DEFS =	\
 
 MIG_HDRS = \
diff --git a/osfmk/profiling/i386/profile-asm.s b/osfmk/profiling/i386/profile-asm.s
index a47e9fcd7..2386d46bf 100644
--- a/osfmk/profiling/i386/profile-asm.s
+++ b/osfmk/profiling/i386/profile-asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/i386/profile-md.c b/osfmk/profiling/i386/profile-md.c
index 5b7d5480c..c1390d7fd 100644
--- a/osfmk/profiling/i386/profile-md.c
+++ b/osfmk/profiling/i386/profile-md.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/i386/profile-md.h b/osfmk/profiling/i386/profile-md.h
index 6b60d5ee2..bbc8d2df8 100644
--- a/osfmk/profiling/i386/profile-md.h
+++ b/osfmk/profiling/i386/profile-md.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/machine/profile-md.h b/osfmk/profiling/machine/profile-md.h
index c66d76ffc..ec48f8f8d 100644
--- a/osfmk/profiling/machine/profile-md.h
+++ b/osfmk/profiling/machine/profile-md.h
@@ -1,35 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _MACH_MACHINE_PROFILE_MD_H
 #define _MACH_MACHINE_PROFILE_MD_H_
 
-
 #if defined (__ppc__)
 #include "profiling/ppc/profile-md.h"
 #elif defined (__i386__)
 #include "profiling/i386/profile-md.h"
+#elif defined (__arm__)
+#include "profiling/arm/profile-md.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _MACH_MACHINE_PROFILE_MD_H_ */
diff --git a/osfmk/profiling/ppc/profile-md.h b/osfmk/profiling/ppc/profile-md.h
index d6b4fa4fd..d8d83698e 100644
--- a/osfmk/profiling/ppc/profile-md.h
+++ b/osfmk/profiling/ppc/profile-md.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/profile-internal.h b/osfmk/profiling/profile-internal.h
index 242d290e8..2667419e0 100644
--- a/osfmk/profiling/profile-internal.h
+++ b/osfmk/profiling/profile-internal.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/profile-kgmon.c b/osfmk/profiling/profile-kgmon.c
index e2eb6272a..c00ff59f6 100644
--- a/osfmk/profiling/profile-kgmon.c
+++ b/osfmk/profiling/profile-kgmon.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -74,9 +80,6 @@
 #define PROFILE_VARS(cpu) (&_profile_vars)
 #endif
 
-extern int printf(const char *, ...);
-
-
 /*
  * Kgmon interface.  This returns the count of bytes moved if everything was ok,
  * or -1 if there were errors.
diff --git a/osfmk/profiling/profile-mk.c b/osfmk/profiling/profile-mk.c
index 84174f599..6a912bb69 100644
--- a/osfmk/profiling/profile-mk.c
+++ b/osfmk/profiling/profile-mk.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/profiling/profile-mk.h b/osfmk/profiling/profile-mk.h
index 89bf6681c..700e030e5 100644
--- a/osfmk/profiling/profile-mk.h
+++ b/osfmk/profiling/profile-mk.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/scsi.h b/osfmk/sys/scsi.h
index 319b0e2e2..545b08c78 100644
--- a/osfmk/sys/scsi.h
+++ b/osfmk/sys/scsi.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/sdi.h b/osfmk/sys/sdi.h
index 13d487ea6..6a92a8eb1 100644
--- a/osfmk/sys/sdi.h
+++ b/osfmk/sys/sdi.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/sdi_edt.h b/osfmk/sys/sdi_edt.h
index ac6576c05..1b73c9650 100644
--- a/osfmk/sys/sdi_edt.h
+++ b/osfmk/sys/sdi_edt.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/syslog.h b/osfmk/sys/syslog.h
index aed42097f..472111935 100644
--- a/osfmk/sys/syslog.h
+++ b/osfmk/sys/syslog.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/time.h b/osfmk/sys/time.h
index f2fa500d9..a48017b9e 100644
--- a/osfmk/sys/time.h
+++ b/osfmk/sys/time.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/tm.h b/osfmk/sys/tm.h
index db34b6f84..357ebd283 100644
--- a/osfmk/sys/tm.h
+++ b/osfmk/sys/tm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/types.h b/osfmk/sys/types.h
index 2f61d0065..5395efe2b 100644
--- a/osfmk/sys/types.h
+++ b/osfmk/sys/types.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/sys/varargs.h b/osfmk/sys/varargs.h
index 19c3b94b9..828d75e4b 100644
--- a/osfmk/sys/varargs.h
+++ b/osfmk/sys/varargs.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/vm/Makefile b/osfmk/vm/Makefile
index f5bc46fa4..e7e6726e0 100644
--- a/osfmk/vm/Makefile
+++ b/osfmk/vm/Makefile
@@ -11,13 +11,12 @@ DATAFILES =
 
 EXPORT_ONLY_FILES = \
 	pmap.h \
-	task_working_set.h \
 	vm_fault.h \
 	vm_kern.h \
 	vm_map.h \
 	vm_pageout.h \
 	vm_protos.h \
-	vm_shared_memory_server.h
+	vm_shared_region.h
 
 INSTALL_MI_LIST = ${DATAFILES}
 
diff --git a/osfmk/vm/bsd_vm.c b/osfmk/vm/bsd_vm.c
index ef9254714..6721d47bf 100644
--- a/osfmk/vm/bsd_vm.c
+++ b/osfmk/vm/bsd_vm.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/errno.h>
@@ -33,6 +39,7 @@
 #include <mach/upl.h>
 #include <mach/thread_act.h>
 
+#include <kern/assert.h>
 #include <kern/host.h>
 #include <kern/thread.h>
 
@@ -47,6 +54,8 @@
 #include <vm/memory_object.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>
+#include <vm/vm_purgeable_internal.h>
+
 
 /* BSD VM COMPONENT INTERFACES */
 int
@@ -213,10 +222,10 @@ macx_triggers(
 		return EINVAL;
 	}
 
-       if (default_pager_init_flag == 0) {
+	if (default_pager_init_flag == 0) {
                start_def_pager(NULL);
                default_pager_init_flag = 1;
-       }
+	}
 
 	if (flags & SWAP_ENCRYPT_ON) {
 		/* ENCRYPTED SWAP: tell default_pager to encrypt */
@@ -319,7 +328,8 @@ memory_object_control_uiomove(
 	void		*	uio,
 	int			start_offset,
 	int			io_requested,
-	int			mark_dirty)
+	int			mark_dirty,
+	int			take_reference)
 {
 	vm_object_t		object;
 	vm_page_t		dst_page;
@@ -328,9 +338,10 @@ memory_object_control_uiomove(
 	int			cur_run;
 	int			cur_needed;
 	int			i;
+	int			orig_offset;
+	boolean_t		make_lru = FALSE;
 	vm_page_t		page_run[MAX_RUN];
 
-
 	object = memory_object_control_to_vm_object(control);
 	if (object == VM_OBJECT_NULL) {
 		return (0);
@@ -349,6 +360,7 @@ memory_object_control_uiomove(
 		vm_object_unlock(object);
 		return 0;
 	}
+	orig_offset = start_offset;
 	    
 	while (io_requested && retval == 0) {
 
@@ -416,11 +428,42 @@ memory_object_control_uiomove(
 		}
 		vm_object_lock(object);
 
+		/*
+		 * if we have more than 1 page to work on
+		 * in the current run, or the original request
+		 * started at offset 0 of the page, or we're
+		 * processing multiple batches, we will move
+		 * the pages to the tail of the inactive queue
+		 * to implement an LRU for read/write accesses
+		 *
+		 * the check for orig_offset == 0 is there to 
+		 * mitigate the cost of small (< page_size) requests
+		 * to the same page (this way we only move it once)
+		 */
+		if (take_reference && (cur_run > 1 || orig_offset == 0)) {
+			vm_page_lockspin_queues();
+			make_lru = TRUE;
+		}
 		for (i = 0; i < cur_run; i++) {
 		        dst_page = page_run[i];
 
+			/*
+			 * someone is explicitly referencing this page...
+			 * update clustered and speculative state
+			 * 
+			 */
+			VM_PAGE_CONSUME_CLUSTERED(dst_page);
+
+			if (make_lru == TRUE)
+				vm_page_lru(dst_page);
+
 			PAGE_WAKEUP_DONE(dst_page);
 		}
+		if (make_lru == TRUE) {
+			vm_page_unlock_queues();
+			make_lru = FALSE;
+		}
+		orig_offset = 0;
 	}
 	vm_object_unlock(object);
 
@@ -588,7 +631,7 @@ vnode_pager_get_object_pathname(
 kern_return_t
 vnode_pager_get_object_filename(
 	memory_object_t	mem_obj,
-	char		**filename)
+	const char	**filename)
 {
 	vnode_pager_t	vnode_object;
 
@@ -602,6 +645,24 @@ vnode_pager_get_object_filename(
 					filename);
 }
 
+kern_return_t
+vnode_pager_get_object_cs_blobs(
+	memory_object_t	mem_obj,
+	void		**blobs)
+{
+	vnode_pager_t	vnode_object;
+
+	if (mem_obj == MEMORY_OBJECT_NULL ||
+	    mem_obj->mo_pager_ops != &vnode_pager_ops) {
+		return KERN_INVALID_ARGUMENT;
+	}
+
+	vnode_object = vnode_pager_lookup(mem_obj);
+
+	return vnode_pager_get_cs_blobs(vnode_object->vnode_handle,
+					blobs);
+}
+
 /*
  *
  */
@@ -609,21 +670,27 @@ kern_return_t
 vnode_pager_data_request(
 	memory_object_t		mem_obj,
 	memory_object_offset_t	offset,
-	vm_size_t		length,
-#if !DEBUG
-	__unused
-#endif
-vm_prot_t		protection_required)
+	__unused vm_size_t	length,
+	__unused vm_prot_t	desired_access,
+	memory_object_fault_info_t	fault_info)
 {
 	register vnode_pager_t	vnode_object;
-
-	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_data_request: %x, %x, %x, %x\n", mem_obj, offset, length, protection_required));
+	vm_size_t		size;
+#if MACH_ASSERT
+	memory_object_offset_t	original_offset = offset;
+#endif /* MACH_ASSERT */
 
 	vnode_object = vnode_pager_lookup(mem_obj);
 
-	PAGER_DEBUG(PAGER_PAGEIN, ("vnode_pager_data_request: %x, %x, %x, %x, vnode_object %x\n", mem_obj, offset, length, protection_required, vnode_object));
-		
-	return vnode_pager_cluster_read(vnode_object, offset, length);
+	size = MAX_UPL_TRANSFER * PAGE_SIZE;
+
+	if (memory_object_cluster_size(vnode_object->control_handle, &offset, &size, fault_info) != KERN_SUCCESS)
+	        size = PAGE_SIZE;
+
+	assert(original_offset >= offset &&
+	       original_offset < offset + size);
+
+	return vnode_pager_cluster_read(vnode_object, offset, size);
 }
 
 /*
@@ -650,7 +717,7 @@ vnode_pager_deallocate(
 {
 	register vnode_pager_t	vnode_object;
 
-	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_deallocate: %p\n", mem_obj));
 
 	vnode_object = vnode_pager_lookup(mem_obj);
 
@@ -673,7 +740,7 @@ vnode_pager_terminate(
 #endif
 	memory_object_t	mem_obj)
 {
-	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_terminate: %p\n", mem_obj));
 
 	return(KERN_SUCCESS);
 }
@@ -690,7 +757,7 @@ vnode_pager_synchronize(
 {
 	register vnode_pager_t	vnode_object;
 
-	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_synchronize: %p\n", mem_obj));
 
 	vnode_object = vnode_pager_lookup(mem_obj);
 
@@ -708,7 +775,7 @@ vnode_pager_unmap(
 {
 	register vnode_pager_t	vnode_object;
 
-	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("vnode_pager_unmap: %p\n", mem_obj));
 
 	vnode_object = vnode_pager_lookup(mem_obj);
 
@@ -771,8 +838,6 @@ vnode_pager_cluster_write(
 	        vm_object_offset_t      vnode_size;
 	        vm_object_offset_t	base_offset;
 		vm_object_t             object;
-		vm_page_t               target_page;
-		int                     ticket;
 
 	        /*
 		 * this is the pageout path
@@ -811,20 +876,6 @@ vnode_pager_cluster_write(
 		                UPL_RET_ONLY_DIRTY | UPL_COPYOUT_FROM |
 		                UPL_SET_INTERNAL | UPL_SET_LITE;
 
-		vm_object_lock(object);
-
-		if ((target_page = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
-		        /*
-			 * only pick up pages whose ticket number matches
-			 * the ticket number of the page orginally targeted
-			 * for pageout
-			 */
-			ticket = target_page->page_ticket;
-
-			request_flags |= ((ticket << UPL_PAGE_TICKET_SHIFT) & UPL_PAGE_TICKET_MASK);
-		}
-		vm_object_unlock(object);
-
 		vm_object_upl_request(object, base_offset, size,
 				      &upl, NULL, NULL, request_flags);
 		if (upl == NULL)
diff --git a/osfmk/vm/cpm.h b/osfmk/vm/cpm.h
index 8bbd672bf..08794ab7b 100644
--- a/osfmk/vm/cpm.h
+++ b/osfmk/vm/cpm.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -48,7 +54,7 @@
  *	These pages are all in "gobbled" state when .
  */
 extern kern_return_t
-	cpm_allocate(vm_size_t size, vm_page_t *list, boolean_t wire);
+cpm_allocate(vm_size_t size, vm_page_t *list, ppnum_t max_pnum, boolean_t wire);
 
 /*
  *	CPM-specific event counters.
diff --git a/osfmk/vm/device_vm.c b/osfmk/vm/device_vm.c
index 807d10b2d..015200a6a 100644
--- a/osfmk/vm/device_vm.c
+++ b/osfmk/vm/device_vm.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/errno.h>
@@ -298,7 +304,8 @@ device_pager_data_request(
 	memory_object_t		mem_obj,
 	memory_object_offset_t	offset,
 	vm_size_t		length,
-	__unused vm_prot_t		protection_required)
+	__unused vm_prot_t	protection_required,
+        __unused memory_object_fault_info_t 	fault_info)
 {
 	device_pager_t	device_object;
 
@@ -429,7 +436,7 @@ device_pager_unmap(
  *
  */
 device_pager_t
-device_object_create()
+device_object_create(void)
 {
 	register device_pager_t  device_object;
 
diff --git a/osfmk/vm/memory_object.c b/osfmk/vm/memory_object.c
index f990e664d..c84e776c6 100644
--- a/osfmk/vm/memory_object.c
+++ b/osfmk/vm/memory_object.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -94,6 +100,7 @@
 #include <vm/pmap.h>		/* For pmap_clear_modify */
 #include <vm/vm_kern.h>		/* For kernel_map, vm_move */
 #include <vm/vm_map.h>		/* For vm_map_pageable */
+#include <vm/vm_purgeable_internal.h>	/* Needed by some vm_page.h macros */
 
 #if	MACH_PAGEMAP
 #include <vm/vm_external.h>
@@ -103,7 +110,6 @@
 
 
 memory_object_default_t	memory_manager_default = MEMORY_OBJECT_DEFAULT_NULL;
-vm_size_t		memory_manager_default_cluster = 0;
 decl_mutex_data(,	memory_manager_default_lock)
 
 
@@ -199,37 +205,20 @@ memory_object_lock_page(
 
 	assert(!m->fictitious);
 
-	if (m->wire_count != 0) {
-		/*
-		 *	If no change would take place
-		 *	anyway, return successfully.
-		 *
-		 *	No change means:
-		 *		Not flushing AND
-		 *		No change to page lock [2 checks]  AND
-		 *		Should not return page
-		 *
-		 * XXX	This doesn't handle sending a copy of a wired
-		 * XXX	page to the pager, but that will require some
-		 * XXX	significant surgery.
-		 */
-		if (!should_flush &&
-		    (m->page_lock == prot || prot == VM_PROT_NO_CHANGE) &&
-		    ! memory_object_should_return_page(m, should_return)) {
-
-			/*
-			 *	Restart page unlock requests,
-			 *	even though no change took place.
-			 *	[Memory managers may be expecting
-			 *	to see new requests.]
-			 */
-			m->unlock_request = VM_PROT_NONE;
-			PAGE_WAKEUP(m);
+	/*
+	 *	If the page is wired, just clean or return the page if needed.
+	 *	Wired pages don't get flushed or disconnected from the pmap.
+	 */
 
-			return(MEMORY_OBJECT_LOCK_RESULT_DONE);
+	if (m->wire_count != 0) {
+		if (memory_object_should_return_page(m, should_return)) {
+			if (m->dirty)
+				return(MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN);
+			else
+				return(MEMORY_OBJECT_LOCK_RESULT_MUST_RETURN);
 		}
 
-		return(MEMORY_OBJECT_LOCK_RESULT_MUST_BLOCK);
+		return(MEMORY_OBJECT_LOCK_RESULT_DONE);
 	}
 
 	/*
@@ -249,36 +238,14 @@ memory_object_lock_page(
 	 */
 
 	if (prot != VM_PROT_NO_CHANGE) {
-		if ((m->page_lock ^ prot) & prot) {
-			pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
-		}
-#if 0
-		/* code associated with the vestigial 
-		 * memory_object_data_unlock
-		 */
-		m->page_lock = prot;
-		m->lock_supplied = TRUE;
-		if (prot != VM_PROT_NONE)
-			m->unusual = TRUE;
-		else
-			m->unusual = FALSE;
-
-		/*
-		 *	Restart any past unlock requests, even if no
-		 *	change resulted.  If the manager explicitly
-		 *	requested no protection change, then it is assumed
-		 *	to be remembering past requests.
-		 */
+	        pmap_page_protect(m->phys_page, VM_PROT_ALL & ~prot);
 
-		m->unlock_request = VM_PROT_NONE;
-#endif /* 0 */
 		PAGE_WAKEUP(m);
 	}
 
 	/*
 	 *	Handle page returning.
 	 */
-
 	if (memory_object_should_return_page(m, should_return)) {
 
 		/*
@@ -290,7 +257,7 @@ memory_object_lock_page(
 		 *	maps now.
 		 */
 		
-		vm_page_lock_queues();
+		vm_page_lockspin_queues();
 		VM_PAGE_QUEUES_REMOVE(m);
 		vm_page_unlock_queues();
 
@@ -306,7 +273,6 @@ memory_object_lock_page(
 	/*
 	 *	Handle flushing
 	 */
-
 	if (should_flush) {
 		VM_PAGE_FREE(m);
 	} else {
@@ -333,24 +299,29 @@ memory_object_lock_page(
 MACRO_BEGIN								\
 									\
         register int            upl_flags;                              \
+	memory_object_t		pager;					\
 				                   			\
-	vm_object_unlock(object);					\
+	if ((pager = (object)->pager) != MEMORY_OBJECT_NULL) {		\
+		vm_object_paging_begin(object);				\
+		vm_object_unlock(object);				\
 									\
-                if (iosync)                                             \
-                        upl_flags = UPL_MSYNC | UPL_IOSYNC;             \
-                else                                                    \
-                        upl_flags = UPL_MSYNC;                          \
+                if (iosync)                                     	\
+                        upl_flags = UPL_MSYNC | UPL_IOSYNC;     	\
+                else                                            	\
+                        upl_flags = UPL_MSYNC;                  	\
 				                   			\
-	   	(void) memory_object_data_return(object->pager,		\
-		po,							\
-		data_cnt,						\
-                ro,                                                     \
-                ioerr,                                                  \
-		(action == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN),	\
-		!should_flush,                                          \
-		upl_flags);                                 	        \
+	   	(void) memory_object_data_return(pager,			\
+			po,						\
+			data_cnt,					\
+	                ro,                                             \
+	                ioerr,                                          \
+			(action) == MEMORY_OBJECT_LOCK_RESULT_MUST_CLEAN,\
+			!should_flush,                                  \
+			upl_flags);                                 	\
 									\
-	vm_object_lock(object);						\
+		vm_object_lock(object);					\
+		vm_object_paging_end(object);				\
+	}								\
 MACRO_END
 
 /*
@@ -658,7 +629,7 @@ vm_object_update_extent(
 			    data_cnt += PAGE_SIZE;
 			    last_offset = offset + PAGE_SIZE_64;
 
-			    vm_page_lock_queues();
+			    vm_page_lockspin_queues();
 			    /*
 			     * Clean
 			     */
@@ -713,7 +684,7 @@ vm_object_update(
 	int				flags,
 	vm_prot_t			protection)
 {
-	vm_object_t		copy_object;
+        vm_object_t		copy_object = VM_OBJECT_NULL;
 	boolean_t		data_returned = FALSE;
 	boolean_t		update_cow;
 	boolean_t		should_flush = (flags & MEMORY_OBJECT_DATA_FLUSH) ? TRUE : FALSE;
@@ -751,9 +722,39 @@ vm_object_update(
 					!(flags & MEMORY_OBJECT_DATA_PURGE)))
 				|| (flags & MEMORY_OBJECT_COPY_SYNC);
 			
+	if (update_cow || (flags & (MEMORY_OBJECT_DATA_PURGE | MEMORY_OBJECT_DATA_SYNC))) {
+	        int collisions = 0;
+
+	        while ((copy_object = object->copy) != VM_OBJECT_NULL) {
+		        /*
+			 * need to do a try here since we're swimming upstream
+			 * against the normal lock ordering... however, we need
+			 * to hold the object stable until we gain control of the
+			 * copy object so we have to be careful how we approach this
+			 */
+		        if (vm_object_lock_try(copy_object)) {
+			       /*
+				* we 'won' the lock on the copy object...
+				* no need to hold the object lock any longer...
+				* take a real reference on the copy object because
+				* we're going to call vm_fault_page on it which may
+				* under certain conditions drop the lock and the paging
+				* reference we're about to take... the reference
+				* will keep the copy object from going away if that happens
+				*/
+			       vm_object_unlock(object);
+			       vm_object_reference_locked(copy_object);
+			       break;
+			}
+			vm_object_unlock(object);
+
+			collisions++;
+			mutex_pause(collisions);
 
-	if((((copy_object = object->copy) != NULL) && update_cow) ||
-					(flags & MEMORY_OBJECT_DATA_SYNC)) {
+			vm_object_lock(object);
+		}
+	}
+	if ((copy_object != VM_OBJECT_NULL && update_cow) || (flags & MEMORY_OBJECT_DATA_SYNC)) {
 		vm_map_size_t		i;
 		vm_map_size_t		copy_size;
 		vm_map_offset_t		copy_offset;
@@ -761,81 +762,83 @@ vm_object_update(
 		vm_page_t		page;
 		vm_page_t		top_page;
 		kern_return_t		error = 0;
+		struct vm_object_fault_info fault_info;
+
+		if (copy_object != VM_OBJECT_NULL) {
+		        /*
+			 * translate offset with respect to shadow's offset
+			 */
+		        copy_offset = (offset >= copy_object->shadow_offset) ?
+			  (vm_map_offset_t)(offset - copy_object->shadow_offset) :
+			  (vm_map_offset_t) 0;
+
+			if (copy_offset > copy_object->size)
+			        copy_offset = copy_object->size;
+
+			/*
+			 * clip size with respect to shadow offset
+			 */
+			if (offset >= copy_object->shadow_offset) {
+			        copy_size = size;
+			} else if (size >= copy_object->shadow_offset - offset) {
+			        copy_size = size - (copy_object->shadow_offset - offset);
+			} else {
+			        copy_size = 0;
+			}
+			
+			if (copy_offset + copy_size > copy_object->size) {
+			        if (copy_object->size >= copy_offset) {
+				        copy_size = copy_object->size - copy_offset;
+				} else {
+				        copy_size = 0;
+				}
+			}
+			copy_size+=copy_offset;
 
-		if(copy_object != NULL) {
-		   /* translate offset with respect to shadow's offset */
-		   copy_offset = (offset >= copy_object->shadow_offset)?
-		   	(vm_map_offset_t)(offset - copy_object->shadow_offset) :
-			(vm_map_offset_t) 0;
-		   if(copy_offset > copy_object->size)
-			copy_offset = copy_object->size;
-
-		   /* clip size with respect to shadow offset */
-		   if (offset >= copy_object->shadow_offset) {
-			   copy_size = size;
-		   } else if (size >= copy_object->shadow_offset - offset) {
-			   copy_size = size -
-				   (copy_object->shadow_offset - offset);
-		   } else {
-			   copy_size = 0;
-		   }
-
-		   if (copy_offset + copy_size > copy_object->size) {
-			   if (copy_object->size >= copy_offset) {
-				   copy_size = copy_object->size - copy_offset;
-			   } else {
-				   copy_size = 0;
-			   }
-		   }
-
-		   copy_size+=copy_offset;
-
-		   vm_object_unlock(object);
-		   vm_object_lock(copy_object);
 		} else {
 			copy_object = object;
 
 			copy_size   = offset + size;
 			copy_offset = offset;
 		}
+		fault_info.interruptible = THREAD_UNINT;
+		fault_info.behavior  = VM_BEHAVIOR_SEQUENTIAL;
+		fault_info.user_tag  = 0;
+		fault_info.lo_offset = copy_offset;
+		fault_info.hi_offset = copy_size;
+		fault_info.no_cache   = FALSE;
 
 		vm_object_paging_begin(copy_object);
-		for (i=copy_offset; i<copy_size; i+=PAGE_SIZE) {
+
+		for (i = copy_offset; i < copy_size; i += PAGE_SIZE) {
 	RETRY_COW_OF_LOCK_REQUEST:
+			fault_info.cluster_size = copy_size - i;
+
 			prot = 	VM_PROT_WRITE|VM_PROT_READ;
 			switch (vm_fault_page(copy_object, i, 
 				VM_PROT_WRITE|VM_PROT_READ,
 				FALSE,
-				THREAD_UNINT,
-				copy_offset,
-				copy_offset+copy_size,
-				VM_BEHAVIOR_SEQUENTIAL,
 				&prot,
 				&page,
 				&top_page,
 				(int *)0,
 				&error,
 			        FALSE,
-				FALSE, NULL, 0)) {
+		                FALSE, &fault_info)) {
 
 			case VM_FAULT_SUCCESS:
-				if(top_page) {
+				if (top_page) {
 					vm_fault_cleanup(
 						page->object, top_page);
-					PAGE_WAKEUP_DONE(page);
-					vm_page_lock_queues();
-					if (!page->active && !page->inactive)
-						vm_page_activate(page);
-					vm_page_unlock_queues();
 					vm_object_lock(copy_object);
 					vm_object_paging_begin(copy_object);
-				} else {
-					PAGE_WAKEUP_DONE(page);
-					vm_page_lock_queues();
-					if (!page->active && !page->inactive)
-						vm_page_activate(page);
-					vm_page_unlock_queues();
 				}
+				vm_page_lock_queues();
+				if (!page->active && !page->inactive)
+					vm_page_deactivate(page);
+				vm_page_unlock_queues();
+
+				PAGE_WAKEUP_DONE(page);
 				break;
 			case VM_FAULT_RETRY:
 				prot = 	VM_PROT_WRITE|VM_PROT_READ;
@@ -860,27 +863,36 @@ vm_object_update(
 				vm_object_paging_begin(copy_object);
 				goto RETRY_COW_OF_LOCK_REQUEST;
 			case VM_FAULT_MEMORY_ERROR:
+			        if (object != copy_object)
+				        vm_object_deallocate(copy_object);
 				vm_object_lock(object);
 				goto BYPASS_COW_COPYIN;
 			}
 
 		}
 		vm_object_paging_end(copy_object);
-		if(copy_object != object) {
+	}
+	if ((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
+	        if (copy_object != VM_OBJECT_NULL && copy_object != object) {
 			vm_object_unlock(copy_object);
+		        vm_object_deallocate(copy_object);
 			vm_object_lock(object);
 		}
+		return KERN_SUCCESS;
 	}
-	if((flags & (MEMORY_OBJECT_DATA_SYNC | MEMORY_OBJECT_COPY_SYNC))) {
-			return KERN_SUCCESS;
-	}
-	if(((copy_object = object->copy) != NULL) && 
-					(flags & MEMORY_OBJECT_DATA_PURGE)) {
-		copy_object->shadow_severed = TRUE;
-		copy_object->shadowed = FALSE;
-		copy_object->shadow = NULL;
-		/* delete the ref the COW was holding on the target object */
-		vm_object_deallocate(object);
+	if (copy_object != VM_OBJECT_NULL && copy_object != object) {
+	        if ((flags & MEMORY_OBJECT_DATA_PURGE)) {
+		        copy_object->shadow_severed = TRUE;
+			copy_object->shadowed = FALSE;
+			copy_object->shadow = NULL;
+			/*
+			 * delete the ref the COW was holding on the target object
+			 */
+			vm_object_deallocate(object);
+		}
+		vm_object_unlock(copy_object);
+	        vm_object_deallocate(copy_object);
+		vm_object_lock(object);
 	}
 BYPASS_COW_COPYIN:
 
@@ -1076,7 +1088,6 @@ vm_object_set_attributes_common(
 	boolean_t	may_cache,
 	memory_object_copy_strategy_t copy_strategy,
 	boolean_t	temporary,
-	memory_object_cluster_size_t	cluster_size,
         boolean_t	silent_overwrite,
 	boolean_t	advisory_pageout)
 {
@@ -1110,17 +1121,6 @@ vm_object_set_attributes_common(
 		may_cache = TRUE;
 	if (temporary)
 		temporary = TRUE;
-	if (cluster_size != 0) {
-		int	pages_per_cluster;
-		pages_per_cluster = atop_32(cluster_size);
-		/*
-		 * Cluster size must be integral multiple of page size,
-		 * and be a power of 2 number of pages.
-		 */
-		if ((cluster_size & (PAGE_SIZE-1)) ||
-		    ((pages_per_cluster-1) & pages_per_cluster))
-			return KERN_INVALID_ARGUMENT;
-	}
 
 	vm_object_lock(object);
 
@@ -1134,12 +1134,6 @@ vm_object_set_attributes_common(
 	object->temporary = temporary;
 	object->silent_overwrite = silent_overwrite;
 	object->advisory_pageout = advisory_pageout;
-	if (cluster_size == 0)
-		cluster_size = PAGE_SIZE;
-	object->cluster_size = cluster_size;
-
-	assert(cluster_size >= PAGE_SIZE &&
-	       cluster_size % PAGE_SIZE == 0);
 
 	/*
 	 *	Wake up anyone waiting for the ready attribute
@@ -1176,7 +1170,6 @@ memory_object_change_attributes(
 	boolean_t       		temporary;
 	boolean_t       		may_cache;
 	boolean_t       		invalidate;
-	memory_object_cluster_size_t	cluster_size;
 	memory_object_copy_strategy_t	copy_strategy;
 	boolean_t       		silent_overwrite;
 	boolean_t			advisory_pageout;
@@ -1195,7 +1188,6 @@ memory_object_change_attributes(
 #if notyet
 	invalidate = object->invalidate;
 #endif
-	cluster_size = object->cluster_size;
 	vm_object_unlock(object);	
 
 	switch (flavor) {
@@ -1248,7 +1240,6 @@ memory_object_change_attributes(
                 perf = (memory_object_perf_info_t) attributes;
 
 		may_cache = perf->may_cache;
-		cluster_size = round_page_32(perf->cluster_size);
 
 		break;
 	    }
@@ -1266,7 +1257,6 @@ memory_object_change_attributes(
 
                 may_cache = attr->may_cache;
                 copy_strategy = attr->copy_strategy;
-		cluster_size = page_size;
 
 		break;
 	    }
@@ -1284,7 +1274,6 @@ memory_object_change_attributes(
 
 		copy_strategy = attr->copy_strategy;
                 may_cache = attr->may_cache_object;
-		cluster_size = attr->cluster_size;
 		temporary = attr->temporary;
 
 		break;
@@ -1313,7 +1302,6 @@ memory_object_change_attributes(
 						     may_cache,
 						     copy_strategy,
 						     temporary,
-						     cluster_size,
 						     silent_overwrite,
 						     advisory_pageout));
 }
@@ -1390,7 +1378,7 @@ memory_object_get_attributes(
 		}
 
 		perf = (memory_object_perf_info_t) attributes;
-		perf->cluster_size = object->cluster_size;
+		perf->cluster_size = PAGE_SIZE;
 		perf->may_cache = object->can_persist;
 
 		*count = MEMORY_OBJECT_PERF_INFO_COUNT;
@@ -1425,7 +1413,7 @@ memory_object_get_attributes(
 
                 attr = (memory_object_attr_info_t) attributes;
         	attr->copy_strategy = object->copy_strategy;
-		attr->cluster_size = object->cluster_size;
+		attr->cluster_size = PAGE_SIZE;
         	attr->may_cache_object = object->can_persist;
 		attr->temporary = object->temporary;
 
@@ -1651,6 +1639,27 @@ memory_object_super_upl_request(
 					   cntrl_flags);
 }
 
+kern_return_t
+memory_object_cluster_size(memory_object_control_t control, memory_object_offset_t *start,
+			   vm_size_t *length, memory_object_fault_info_t fault_info)
+{
+	vm_object_t		object;
+
+	object = memory_object_control_to_vm_object(control);
+
+	if (object == VM_OBJECT_NULL || object->paging_offset > *start)
+		return (KERN_INVALID_ARGUMENT);
+
+	*start -= object->paging_offset;
+
+	vm_object_cluster_size(object, (vm_object_offset_t *)start, length, (vm_object_fault_info_t)fault_info);
+
+	*start += object->paging_offset;
+
+	return (KERN_SUCCESS);
+}
+
+
 int vm_stat_discard_cleared_reply = 0;
 int vm_stat_discard_cleared_unset = 0;
 int vm_stat_discard_cleared_too_late = 0;
@@ -1669,11 +1678,12 @@ kern_return_t
 host_default_memory_manager(
 	host_priv_t		host_priv,
 	memory_object_default_t	*default_manager,
-	memory_object_cluster_size_t cluster_size)
+	__unused memory_object_cluster_size_t cluster_size)
 {
 	memory_object_default_t current_manager;
 	memory_object_default_t new_manager;
 	memory_object_default_t returned_manager;
+	kern_return_t result = KERN_SUCCESS;
 
 	if (host_priv == HOST_PRIV_NULL)
 		return(KERN_INVALID_HOST);
@@ -1683,14 +1693,26 @@ host_default_memory_manager(
 	new_manager = *default_manager;
 	mutex_lock(&memory_manager_default_lock);
 	current_manager = memory_manager_default;
+	returned_manager = MEMORY_OBJECT_DEFAULT_NULL;
 
 	if (new_manager == MEMORY_OBJECT_DEFAULT_NULL) {
 		/*
 		 *	Retrieve the current value.
 		 */
-		memory_object_default_reference(current_manager);
 		returned_manager = current_manager;
+		memory_object_default_reference(returned_manager);
 	} else {
+
+		/*
+		 *	If this is the first non-null manager, start
+		 *	up the internal pager support.
+		 */
+		if (current_manager == MEMORY_OBJECT_DEFAULT_NULL) {
+			result = vm_pageout_internal_start();
+			if (result != KERN_SUCCESS)
+				goto out;
+		}
+
 		/*
 		 *	Retrieve the current value,
 		 *	and replace it with the supplied value.
@@ -1698,21 +1720,10 @@ host_default_memory_manager(
 		 *	but we have to take a reference on the new
 		 *	one.
 		 */
-
 		returned_manager = current_manager;
 		memory_manager_default = new_manager;
 		memory_object_default_reference(new_manager);
 
-		if (cluster_size % PAGE_SIZE != 0) {
-#if 0
-			mutex_unlock(&memory_manager_default_lock);
-			return KERN_INVALID_ARGUMENT;
-#else
-			cluster_size = round_page_32(cluster_size);
-#endif
-		}
-		memory_manager_default_cluster = cluster_size;
-
 		/*
 		 *	In case anyone's been waiting for a memory
 		 *	manager to be established, wake them up.
@@ -1720,11 +1731,11 @@ host_default_memory_manager(
 
 		thread_wakeup((event_t) &memory_manager_default);
 	}
-
+ out:
 	mutex_unlock(&memory_manager_default_lock);
 
 	*default_manager = returned_manager;
-	return(KERN_SUCCESS);
+	return(result);
 }
 
 /*
@@ -1736,8 +1747,7 @@ host_default_memory_manager(
  */
 
 __private_extern__ memory_object_default_t
-memory_manager_default_reference(
-	memory_object_cluster_size_t *cluster_size)
+memory_manager_default_reference(void)
 {
 	memory_object_default_t current_manager;
 
@@ -1753,7 +1763,6 @@ memory_manager_default_reference(
 		current_manager = memory_manager_default;
 	}
 	memory_object_default_reference(current_manager);
-	*cluster_size = memory_manager_default_cluster;
 	mutex_unlock(&memory_manager_default_lock);
 
 	return current_manager;
@@ -1872,6 +1881,23 @@ memory_object_pages_resident(
 	return (KERN_SUCCESS);
 }
 
+kern_return_t
+memory_object_signed(
+	memory_object_control_t	control,
+	boolean_t		is_signed)
+{
+	vm_object_t	object;
+
+	object = memory_object_control_to_vm_object(control);
+	if (object == VM_OBJECT_NULL)
+		return KERN_INVALID_ARGUMENT;
+
+	vm_object_lock(object);
+	object->code_signed = is_signed;
+	vm_object_unlock(object);
+
+	return KERN_SUCCESS;
+}
 
 static zone_t mem_obj_control_zone;
 
@@ -2039,14 +2065,16 @@ kern_return_t memory_object_data_request
 	memory_object_t memory_object,
 	memory_object_offset_t offset,
 	memory_object_cluster_size_t length,
-	vm_prot_t desired_access
+	vm_prot_t desired_access,
+	memory_object_fault_info_t fault_info
 )
 {
 	return (memory_object->mo_pager_ops->memory_object_data_request)(
 		memory_object,
 		offset, 
 		length,
-		desired_access);
+		desired_access,
+		fault_info);
 }
 
 /* Routine memory_object_data_return */
diff --git a/osfmk/vm/memory_object.h b/osfmk/vm/memory_object.h
index 8b915a089..e9b974174 100644
--- a/osfmk/vm/memory_object.h
+++ b/osfmk/vm/memory_object.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -60,18 +66,19 @@
 #include <mach/memory_object_types.h>
 #include <ipc/ipc_types.h>
 
+extern memory_object_default_t memory_manager_default;
+
 __private_extern__
-memory_object_default_t	memory_manager_default_reference(
-				vm_size_t		*cluster_size);
+memory_object_default_t	memory_manager_default_reference(void);
 
 __private_extern__
 kern_return_t		memory_manager_default_check(void);
 
 __private_extern__
-void			memory_manager_default_init(void);
+void			memory_manager_default_init(void) __attribute__((section("__TEXT, initcode")));
 
 __private_extern__
-void			memory_object_control_bootstrap(void);
+void			memory_object_control_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 __private_extern__
 memory_object_control_t memory_object_control_allocate(
 				vm_object_t		object);
@@ -130,4 +137,8 @@ extern kern_return_t	memory_object_pages_resident(
 	memory_object_control_t		control,
 	boolean_t			*		has_pages_resident);
 
+extern kern_return_t	memory_object_signed(
+	memory_object_control_t		control,
+	boolean_t			is_signed);
+
 #endif	/* _VM_MEMORY_OBJECT_H_ */
diff --git a/osfmk/vm/pmap.h b/osfmk/vm/pmap.h
index 7638a13a3..d30299778 100644
--- a/osfmk/vm/pmap.h
+++ b/osfmk/vm/pmap.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -135,7 +141,8 @@ extern void		pmap_startup(
 						 * use remaining physical pages
 						 * to allocate page frames.
 						 */
-extern void		pmap_init(void);	/* Initialization,
+extern void		pmap_init(void) __attribute__((section("__TEXT, initcode")));
+						/* Initialization,
 						 * after kernel runs
 						 * in virtual memory.
 						 */
@@ -180,7 +187,11 @@ extern void		pmap_virtual_space(
  */
 extern pmap_t		pmap_create(	/* Create a pmap_t. */
 				vm_map_size_t	size,
+#ifdef __i386__
 				boolean_t	is_64bit);
+#else
+				__unused boolean_t	is_64bit);
+#endif
 extern pmap_t		(pmap_kernel)(void);	/* Return the kernel's pmap */
 extern void		pmap_reference(pmap_t pmap);	/* Gain a reference. */
 extern void		pmap_destroy(pmap_t pmap); /* Release a reference. */
@@ -253,6 +264,9 @@ extern kern_return_t	(pmap_attribute_cache_sync)(  /* Flush appropriate
 extern unsigned int	(pmap_cache_attributes)(
 				ppnum_t		pn);
 
+extern void pmap_sync_page_data_phys(ppnum_t pa);
+extern void pmap_sync_page_attributes_phys(ppnum_t pa);
+
 /*
  * debug/assertions. pmap_verify_free returns true iff
  * the given physical page is mapped into no pmap.
@@ -263,13 +277,15 @@ extern boolean_t	pmap_verify_free(ppnum_t pn);
  *	Statistics routines
  */
 extern int		(pmap_resident_count)(pmap_t pmap);
+extern int		(pmap_resident_max)(pmap_t pmap);
 
 /*
  *	Sundry required (internal) routines
  */
+#ifdef CURRENTLY_UNUSED_AND_UNTESTED
 extern void		pmap_collect(pmap_t pmap);/* Perform garbage
 						 * collection, if any */
-
+#endif
 /*
  *	Optional routines
  */
@@ -350,10 +366,14 @@ extern kern_return_t	(pmap_attribute)(	/* Get/Set special memory
 	if (__pmap != kernel_pmap) {					\
 		ASSERT_PAGE_DECRYPTED(__page);				\
 	}								\
+	if (__page->error) {						\
+		panic("VM page %p should not have an error\n",		\
+			__page);					\
+	}								\
 	pmap_enter(__pmap,						\
 		   (virtual_address),					\
 		   __page->phys_page,					\
-		   (protection) & ~__page->page_lock,			\
+		   (protection),					\
 		   (flags),						\
 		   (wired));						\
 	MACRO_END
@@ -396,6 +416,19 @@ extern void		(pmap_pageable)(
 				vm_map_offset_t	end,
 				boolean_t	pageable);
 
+#ifndef NO_NESTED_PMAP
+extern uint64_t pmap_nesting_size_min;
+extern uint64_t pmap_nesting_size_max;
+extern kern_return_t pmap_nest(pmap_t grand,
+			       pmap_t subord,
+			       addr64_t vstart,
+			       addr64_t nstart,
+			       uint64_t size);
+extern kern_return_t pmap_unnest(pmap_t grand,
+				 addr64_t vaddr,
+				 uint64_t size);
+#endif /* NO_NESTED_PMAP */
+
 #endif	/* MACH_KERNEL_PRIVATE */
 
 /*
@@ -417,11 +450,6 @@ extern pmap_t	kernel_pmap;			/* The kernel's map */
 #define VM_WIMG_MASK		0xFF
 #define VM_WIMG_USE_DEFAULT	0x80000000
 
-extern void		pmap_modify_pages(	/* Set modify bit for pages */
-				pmap_t		map,
-				vm_map_offset_t	s,
-				vm_map_offset_t	e);
-
 extern vm_offset_t	pmap_extract(pmap_t pmap,
 				vm_map_offset_t va);
 
@@ -433,9 +461,13 @@ extern void		pmap_change_wiring(	/* Specify pageability */
 /* LP64todo - switch to vm_map_offset_t when it grows */
 extern void		pmap_remove(	/* Remove mappings. */
 				pmap_t		map,
-				addr64_t	s,
-				addr64_t	e);
+				vm_map_offset_t	s,
+				vm_map_offset_t	e);
+
+extern void		fillPage(ppnum_t pa, unsigned int fill);
 
+extern void pmap_map_sharedpage(task_t task, pmap_t pmap);
+extern void pmap_unmap_sharedpage(pmap_t pmap);
 
 #endif  /* KERNEL_PRIVATE */
 
diff --git a/osfmk/vm/task_working_set.c b/osfmk/vm/task_working_set.c
deleted file mode 100644
index 5a9fe42eb..000000000
--- a/osfmk/vm/task_working_set.c
+++ /dev/null
@@ -1,2305 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- */
-/*
- *	File:	vm/task_working_set.c
- *	Author:	Chris Youngworth
- *	Date:	2001
- *
- *	Working set detection and maintainence module
- */
-
-
-#include <mach/mach_types.h>
-#include <mach/memory_object.h>
-#include <mach/shared_memory_server.h>
-#include <vm/task_working_set.h>
-#include <vm/vm_kern.h>
-#include <vm/vm_map.h>
-#include <vm/vm_page.h>
-#include <vm/vm_pageout.h>
-#include <kern/sched.h>
-#include <kern/kalloc.h>
-
-#include <vm/vm_protos.h>
-
-/*
- * LP64todo - Task Working Set Support is for 32-bit only
- */
-extern zone_t lsf_zone;
-
-/* declarations for internal use only routines */
-int	startup_miss = 0;
-
-tws_hash_t
-tws_hash_create(
-	unsigned int lines,
-	unsigned int rows,
-	unsigned int style);
-
-kern_return_t
-tws_write_startup_file(
-	task_t		task,
-	int		fid,
-	int		mod,
-	char		*name,
-	unsigned int	string_length);
-
-kern_return_t
-tws_read_startup_file(
-	task_t		task,
-	tws_startup_t	startup,
-	vm_offset_t	cache_size);
-
-tws_startup_t
-tws_create_startup_list(
-	tws_hash_t	tws);
-
-unsigned int
-tws_startup_list_lookup(
-        tws_startup_t   startup,
-        vm_offset_t     addr);
-
-kern_return_t
-tws_internal_startup_send(
-	tws_hash_t	tws);
-
-void
-tws_hash_line_clear(
-	tws_hash_t	tws,
-	tws_hash_line_t hash_line,
-	vm_object_t	object,
-	boolean_t live);
-
-void
-tws_hash_clear(
-	tws_hash_t	tws);
-
-void
-tws_traverse_address_hash_list (
-	tws_hash_t 		tws, 
-	unsigned int		index,
-	vm_offset_t		page_addr,
-	vm_object_t		object,
-	vm_object_offset_t	offset,
-	vm_map_t		map,
-	tws_hash_ptr_t		*target_ele,
-	tws_hash_ptr_t		**previous_ptr,
-	tws_hash_ptr_t		**free_list,
-	unsigned int		exclusive_addr);
-
-void
-tws_traverse_object_hash_list (
-	tws_hash_t 		tws, 
-	unsigned int		index,
-	vm_object_t		object,
-	vm_object_offset_t	offset,
-	unsigned int		pagemask,
-	tws_hash_ptr_t		*target_ele,
-	tws_hash_ptr_t		**previous_ptr,
-	tws_hash_ptr_t		**free_list);
-
-tws_hash_ptr_t
-new_addr_hash(
-	tws_hash_t tws, 
-	unsigned int set, 
-	unsigned int index);
-
-tws_hash_ptr_t
-new_obj_hash(
-	tws_hash_t tws, 
-	unsigned int set, 
-	unsigned int index);
-
-int tws_test_for_community(
-		tws_hash_t		tws, 
-		vm_object_t		object, 
-		vm_object_offset_t	offset, 
-		unsigned int		threshold, 
-		unsigned int		*pagemask);
-
-kern_return_t
-tws_internal_lookup(
-	tws_hash_t		tws,	
-	vm_object_offset_t	offset, 
-	vm_object_t		object,
-	tws_hash_line_t		 *line);
-
-/* Note:  all of the routines below depend on the associated map lock for */
-/* synchronization, the map lock will be on when the routines are called  */
-/* and on when they return */
-
-tws_hash_t
-tws_hash_create(
-	unsigned int lines, 
-	unsigned int rows,
-	unsigned int style)
-{
-	tws_hash_t	tws;
-	
-	if ((style != TWS_HASH_STYLE_BASIC) &&
-			(style != TWS_HASH_STYLE_BASIC)) {
-		return((tws_hash_t)NULL);
-	}
-
-	
-	tws = (tws_hash_t)(kalloc(sizeof(struct tws_hash)));
-	if(tws == (tws_hash_t)NULL)
-		return tws;
-
-	if((tws->table[0] = (tws_hash_ptr_t *)
-			kalloc(sizeof(tws_hash_ptr_t) * lines * rows)) 
-								== NULL) {
-		kfree(tws, sizeof(struct tws_hash));
-		return (tws_hash_t)NULL;
-	}
-	if((tws->table_ele[0] = (tws_hash_ptr_t)
-			kalloc(sizeof(struct tws_hash_ptr) * lines * rows)) 
-								== NULL) {
-		kfree(tws->table[0], sizeof(tws_hash_ptr_t) * lines * rows);
-		kfree(tws, sizeof(struct tws_hash));
-		return (tws_hash_t)NULL;
-	}
-	if((tws->alt_ele[0] = (tws_hash_ptr_t)
-			kalloc(sizeof(struct tws_hash_ptr) * lines * rows))
-								== NULL) {
-		kfree(tws->table[0], sizeof(tws_hash_ptr_t) * lines * rows);
-		kfree(tws->table_ele[0],
-		      sizeof(struct tws_hash_ptr) * lines * rows);
-		kfree(tws, sizeof(struct tws_hash));
-		return (tws_hash_t)NULL;
-	}
-	if((tws->cache[0] = (struct tws_hash_line *)
-			kalloc(sizeof(struct tws_hash_line) * lines))
-								== NULL) {
-		kfree(tws->table[0], sizeof(tws_hash_ptr_t) * lines * rows);
-		kfree(tws->table_ele[0],
-		      sizeof(struct tws_hash_ptr) * lines * rows);
-		kfree(tws->alt_ele[0],
-		      sizeof(struct tws_hash_ptr) * lines * rows);
-		kfree(tws, sizeof(struct tws_hash));
-		return (tws_hash_t)NULL;
-	}
-	tws->free_hash_ele[0] = (tws_hash_ptr_t)0;
-	tws->obj_free_count[0] = 0;
-	tws->addr_free_count[0] = 0;
-
-	/* most defaults are such that a bzero will initialize */
-	bzero((char *)tws->table[0],sizeof(tws_hash_ptr_t)
-                                	* lines * rows);
-	bzero((char *)tws->table_ele[0],sizeof(struct tws_hash_ptr)
-                                	* lines * rows);
-	bzero((char *)tws->alt_ele[0],sizeof(struct tws_hash_ptr)
-                                	* lines * rows);
-	bzero((char *)tws->cache[0], sizeof(struct tws_hash_line) 
-					* lines);
-
-	mutex_init(&tws->lock, 0);
-	tws->style = style;
-	tws->current_line = 0;
-	tws->pageout_count = 0;
-	tws->line_count = 0;
-	tws->startup_cache = NULL;
-	tws->startup_name = NULL;
-	tws->number_of_lines = lines;
-	tws->number_of_elements = rows;
-	tws->expansion_count = 1;
-	tws->lookup_count = 0;
-	tws->insert_count = 0;
-	tws->time_of_creation = sched_tick;
-
-	return tws;
-}
-
-
-extern vm_page_t
-vm_page_lookup_nohint(vm_object_t object, vm_object_offset_t offset);
-
-
-void
-tws_hash_line_clear(
-	tws_hash_t	tws,
-	tws_hash_line_t hash_line,
-	__unused vm_object_t	object,
-	boolean_t live)
-{
-	struct tws_hash_ele	*hash_ele;
-	struct tws_hash_ptr	**trailer;
-	struct tws_hash_ptr	**free_list;
-	tws_hash_ele_t		addr_ele;
-	int			index;
-	unsigned int		i, j;
-	int			dump_pmap;
-	int			hash_loop;
-
-
-	if(tws->line_count < tws->number_of_lines) {
-		tws->line_count++;
-		dump_pmap = 1;
-	} else {
-		if(tws->pageout_count != vm_pageout_scan_event_counter) {
-			tws->pageout_count = 
-				vm_pageout_scan_event_counter;
-			tws->line_count = 0;
-			dump_pmap = 1;
-		} else {
-			dump_pmap = 0;
-		}
-	}
-	hash_line->ele_count = 0;
-
-	for (i=0; i<tws->number_of_elements; i++) {
-		hash_loop = 0;
-		hash_ele = &(hash_line->list[i]);
-		if(hash_ele->object != 0) {
-	
-			vm_object_offset_t local_off = 0;
-			tws_hash_ptr_t	cache_ele;
-	
-			index = alt_tws_hash(
-				hash_ele->page_addr & TWS_ADDR_OFF_MASK,
-				tws->number_of_elements, 
-				tws->number_of_lines);
-	
-			tws_traverse_address_hash_list(tws, index, 
-				hash_ele->page_addr, hash_ele->object,
-				hash_ele->offset, hash_ele->map,
-				&cache_ele, &trailer, &free_list, 0);
-			if(cache_ele != NULL) {
-				addr_ele = (tws_hash_ele_t)((unsigned int)
-				   (cache_ele->element) & ~TWS_ADDR_HASH);
-				if(addr_ele != hash_ele)
-					panic("tws_hash_line_clear:"
-						" out of sync\n");
-				cache_ele->element = 0;
-				*trailer = cache_ele->next;
-				cache_ele->next = *free_list;
-				*free_list = cache_ele;
-			}
-	
-			index = alt_tws_hash(
-				(hash_ele->page_addr - 0x1f000) 
-						& TWS_ADDR_OFF_MASK,
-				tws->number_of_elements, 
-				tws->number_of_lines);
-
-			tws_traverse_address_hash_list(tws, index, 
-				hash_ele->page_addr, hash_ele->object, 
-				hash_ele->offset, hash_ele->map,
-				&cache_ele, &trailer, &free_list, 0);
-
-			if(cache_ele != NULL) {
-				addr_ele = (tws_hash_ele_t)((unsigned int)
-				   (cache_ele->element) & ~TWS_ADDR_HASH);
-				if(addr_ele != hash_ele)
-					panic("tws_hash_line_clear: "
-						"out of sync\n");
-				cache_ele->element = 0;
-				*trailer = cache_ele->next;
-				cache_ele->next = *free_list;
-				*free_list = cache_ele;
-			}
-	
-	
-			if((hash_ele->map != NULL) && (live)) {
-				vm_page_t	p;
-				
-#if 0
-				if (object != hash_ele->object) {
-				    if (object)
-				        vm_object_unlock(object);
-				    vm_object_lock(hash_ele->object);
-				}
-#endif
-				if (dump_pmap == 1) {
-				    for (j = 0x1; j != 0; j = j<<1) {
-				        if(j & hash_ele->page_cache) {
-					    p = vm_page_lookup_nohint(hash_ele->object, 
-								      hash_ele->offset + local_off);
-					    if((p != NULL) && (p->wire_count == 0)) {
-					        pmap_remove_some_phys((pmap_t)vm_map_pmap(current_map()),
-								      p->phys_page);
-					    }
-					}
-					local_off += PAGE_SIZE_64;
-				    }
-				}
-#if 0
-				if (object != hash_ele->object) {
-				    vm_object_unlock(hash_ele->object);
-				    if (object)
-				        vm_object_lock(object);
-				}
-#endif
-			}
-	
-			if(tws->style == TWS_HASH_STYLE_SIGNAL) {
-				vm_object_deallocate(hash_ele->object);
-				vm_map_deallocate(hash_ele->map);
-			}
-
-			index = do_tws_hash(hash_ele->object, hash_ele->offset,
-					tws->number_of_elements, 
-					tws->number_of_lines);
-
-			tws_traverse_object_hash_list(tws,
-				index, hash_ele->object, hash_ele->offset,
-				0xFFFFFFFF, &cache_ele, &trailer, &free_list);
-			if((cache_ele != NULL) && (cache_ele->element == hash_ele)) {
-				cache_ele->element = 0;
-				*trailer = cache_ele->next;
-				cache_ele->next = *free_list;
-				*free_list = cache_ele;
-			}
-			hash_ele->object = 0;
-		}
-	}
-}
-
-kern_return_t
-tws_internal_lookup(
-	tws_hash_t		tws,	
-	vm_object_offset_t	offset, 
-	vm_object_t		object,
-	tws_hash_line_t		 *line) 
-{
-	int			index;
-	int			loop;
-	int		  	set; 
-	int			ele_line;
-	vm_offset_t		pagenum;
-	tws_hash_ptr_t		cache_ele;
-	tws_hash_ptr_t		*trailer;
-	tws_hash_ptr_t		*free_list;
-
-	/* don't cache private objects */
-	if(object->private)
-		return KERN_SUCCESS;
-
-	index = do_tws_hash(object, offset, 
-			tws->number_of_elements, tws->number_of_lines);
-	loop = 0;
-
-	tws->lookup_count++; 
-	if(tws->lookup_count == 0)
-		tws->insert_count = 0;
-	if(tws->startup_name != NULL) {
-		int age_of_cache;
-		age_of_cache = ((sched_tick 
-			- tws->time_of_creation) >> SCHED_TICK_SHIFT);
-       		if (age_of_cache > 45) {
-			return KERN_OPERATION_TIMED_OUT;
-		}
-	}
-
-	if(tws->lookup_count > (4 * tws->expansion_count 
-			* tws->number_of_elements * tws->number_of_lines) &&
-			(tws->lookup_count > (2 * tws->insert_count))) {
-		if(tws->startup_cache) {
-			int age_of_cache;
-			age_of_cache = ((sched_tick 
-				- tws->time_of_creation) >> SCHED_TICK_SHIFT);
-	        	if (age_of_cache > 45) {
-				return KERN_OPERATION_TIMED_OUT;
-			}
-      		}
-	}
-
-	pagenum = (vm_offset_t)(offset & TWS_INDEX_MASK);
-	pagenum = pagenum >> 12;
-	pagenum = 1 << pagenum;  /* get the appropriate page in 32 page block */
-	tws_traverse_object_hash_list(tws, index, object, offset, pagenum,
-		&cache_ele, &trailer, &free_list);
-	if(cache_ele != NULL) {
-		set = cache_ele->element->line/tws->number_of_lines;
-		ele_line = cache_ele->element->line - set;
-		*line = &tws->cache[set][ele_line];
-		return KERN_SUCCESS;
-	}
-
-	return KERN_FAILURE;
-
-
-}
-
-kern_return_t
-tws_lookup(
-	tws_hash_t		tws,	
-	vm_object_offset_t	offset, 
-	vm_object_t		object,
-	tws_hash_line_t		 *line) 
-{
-	kern_return_t kr;
-
-	if(!tws_lock_try(tws)) {
-		return KERN_FAILURE;
-	}
-	kr = tws_internal_lookup(tws,
-		offset, object, line);
-	tws_unlock(tws);
-	return kr;
-}
-
-kern_return_t
-tws_expand_working_set(
-	tws_hash_t	old_tws, 
-	unsigned int	line_count,
-	boolean_t	dump_data)
-{
-	tws_hash_t		new_tws;
-	unsigned int		i,j,k;
-	struct	tws_hash	temp;
-
-	/* Note we do an elaborate dance to preserve the header that */
-	/* task is pointing to.  In this way we can avoid taking a task */
-	/* lock every time we want to access the tws */
-
-	if (old_tws->number_of_lines >= line_count) {
-		return KERN_FAILURE;
-	}
-	if((new_tws = tws_hash_create(line_count, 
-			old_tws->number_of_elements, old_tws->style)) == 0) {
-		return(KERN_NO_SPACE);
-	}
-	tws_lock(old_tws);
-	
-	if(!dump_data) {
-	   for(i = 0; i<old_tws->number_of_lines; i++) {
-	      for(j = 0; j<old_tws->number_of_elements; j++) {
-	         for(k = 0; k<old_tws->expansion_count; k++) {
-		    tws_hash_ele_t		entry;
-		    vm_object_offset_t	paddr;
-		    unsigned int		page_index;
-		    entry = &old_tws->cache[k][i].list[j];
-		    if(entry->object != 0) {
-			paddr = 0;
-			for(page_index = 1; page_index != 0; 
-					page_index = page_index << 1) {
-				if (entry->page_cache & page_index) {
-					tws_insert(new_tws, 
-						entry->offset+paddr,
-						entry->object, 
-						entry->page_addr+paddr, 
-						entry->map);
-				}
-				paddr+=PAGE_SIZE;
-			}
-		
-		    }
-	        }
-	      }
-	   }
-	}
-
-	temp.style = new_tws->style;
-	temp.current_line = new_tws->current_line;
-	temp.pageout_count = new_tws->pageout_count;
-	temp.line_count = new_tws->line_count;
-	temp.number_of_lines = new_tws->number_of_lines;
-	temp.number_of_elements = new_tws->number_of_elements;
-	temp.expansion_count = new_tws->expansion_count;
-	temp.lookup_count = new_tws->lookup_count;
-	temp.insert_count = new_tws->insert_count;
-	for(i = 0; i<new_tws->expansion_count; i++) {
-		temp.obj_free_count[i] = new_tws->obj_free_count[i];
-		temp.addr_free_count[i] = new_tws->addr_free_count[i];
-		temp.free_hash_ele[i] = new_tws->free_hash_ele[i];
-		temp.table[i] = new_tws->table[i];
-		temp.table_ele[i] = new_tws->table_ele[i];
-		temp.alt_ele[i] = new_tws->alt_ele[i];
-		temp.cache[i] = new_tws->cache[i];
-	}
-
-	new_tws->style = old_tws->style;
-	new_tws->current_line = old_tws->current_line;
-	new_tws->pageout_count = old_tws->pageout_count;
-	new_tws->line_count = old_tws->line_count;
-	new_tws->number_of_lines = old_tws->number_of_lines;
-	new_tws->number_of_elements = old_tws->number_of_elements;
-	new_tws->expansion_count = old_tws->expansion_count;
-	new_tws->lookup_count = old_tws->lookup_count;
-	new_tws->insert_count = old_tws->insert_count;
-	for(i = 0; i<old_tws->expansion_count; i++) {
-		new_tws->obj_free_count[i] = old_tws->obj_free_count[i];
-		new_tws->addr_free_count[i] = old_tws->addr_free_count[i];
-		new_tws->free_hash_ele[i] = old_tws->free_hash_ele[i];
-		new_tws->table[i] = old_tws->table[i];
-		new_tws->table_ele[i] = old_tws->table_ele[i];
-		new_tws->alt_ele[i] = old_tws->alt_ele[i];
-		new_tws->cache[i] = old_tws->cache[i];
-	}
-	
-	old_tws->style = temp.style;
-	old_tws->current_line = temp.current_line;
-	old_tws->pageout_count = temp.pageout_count;
-	old_tws->line_count = temp.line_count;
-	old_tws->number_of_lines = temp.number_of_lines;
-	old_tws->number_of_elements = temp.number_of_elements;
-	old_tws->expansion_count = temp.expansion_count;
-	old_tws->lookup_count = temp.lookup_count;
-	old_tws->insert_count = temp.insert_count;
-	for(i = 0; i<temp.expansion_count; i++) {
-		old_tws->obj_free_count[i] = temp.obj_free_count[i];;
-		old_tws->addr_free_count[i] = temp.addr_free_count[i];;
-		old_tws->free_hash_ele[i] = NULL;
-		old_tws->table[i] = temp.table[i];
-		old_tws->table_ele[i] = temp.table_ele[i];
-		old_tws->alt_ele[i] = temp.alt_ele[i];
-		old_tws->cache[i] = temp.cache[i];
-	}
-
-	tws_hash_destroy(new_tws);
-	tws_unlock(old_tws);
-	return	KERN_SUCCESS;
-}
-
-tws_hash_t	test_tws = 0;
-
-kern_return_t
-tws_insert(
-	tws_hash_t		tws, 
-	vm_object_offset_t	offset,
-	vm_object_t		object,
-	vm_offset_t		page_addr,
-	vm_map_t		map)
-{
-	unsigned int		index;
-	unsigned int		alt_index;
-	unsigned int		index_enum[2];
-	unsigned int		ele_index;
-	tws_hash_ptr_t		cache_ele;
-	tws_hash_ptr_t		obj_ele = NULL;
-	tws_hash_ptr_t		addr_ele = NULL;
-	tws_hash_ptr_t		*trailer;
-	tws_hash_ptr_t		*free_list;
-	tws_hash_ele_t		target_element = NULL;
-	unsigned int		i;
-	unsigned int		current_line;
-	unsigned int		set;
-	int			ctr;
-	unsigned int		startup_cache_line;
-        int			age_of_cache = 0;
-
-	if(!tws_lock_try(tws)) {
-		return KERN_FAILURE;
-	}
-	tws->insert_count++;
-	current_line = 0xFFFFFFFF;
-	set = 0;
-
-	startup_cache_line = 0;
-
-	if (tws->startup_cache) {
-	        vm_offset_t	startup_page_addr;
-
-	        startup_page_addr = page_addr - (offset - (offset & TWS_HASH_OFF_MASK));
-
-		age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT);
-
-		startup_cache_line = tws_startup_list_lookup(tws->startup_cache, startup_page_addr);
-	}
-	/* This next bit of code, the and alternate hash */
-	/* are all made necessary because of IPC COW     */
-
-	/* Note: the use of page_addr modified by delta from offset */
-	/* frame base means we may miss some previous entries.  However */
-	/* we will not miss the present entry.  This is most important */
-	/* in avoiding duplication of entries against long lived non-cow */
-	/* objects */
-	index_enum[0] = alt_tws_hash(
-			page_addr & TWS_ADDR_OFF_MASK,
-			tws->number_of_elements, tws->number_of_lines);
-
-	index_enum[1] = alt_tws_hash(
-			(page_addr - 0x1f000) & TWS_ADDR_OFF_MASK,
-			tws->number_of_elements, tws->number_of_lines);
-
-	for(ctr = 0; ctr < 2;) {
-		tws_hash_ele_t	resident;
-		tws_traverse_address_hash_list(tws, 
-				index_enum[ctr], page_addr, NULL,
-				0, NULL,
-				&cache_ele, &trailer, &free_list, 1);
-		if(cache_ele != NULL) {
-			/* found one */
-			resident = (tws_hash_ele_t)((unsigned int)
-					cache_ele->element & ~TWS_ADDR_HASH);
-	   		if((object == resident->object) &&
-				resident->offset == 
-					(offset & TWS_HASH_OFF_MASK)) {
-				/* This is our object/offset */
-				resident->page_cache 
-						|= startup_cache_line;
-	      			resident->page_cache |= 
-				   (1<<(((vm_offset_t)
-				   (offset & TWS_INDEX_MASK))>>12));
-				tws_unlock(tws);
-				if (age_of_cache > 45)
-					return KERN_OPERATION_TIMED_OUT;
-				return KERN_SUCCESS;
-			}
-	   		if((object->shadow ==
-				resident->object) && 
-				((resident->offset 
-				+ object->shadow_offset) 
-				== (offset & TWS_HASH_OFF_MASK))) {
-				/* if we just shadowed, inherit */
-				/* access pattern from  parent */
-				startup_cache_line |= 
-				   resident->page_cache;
-				/* thow out old entry */
-				resident->page_cache = 0;
-				break;
-			} else {
-	      			resident->page_cache &= 
-				   ~(1<<(((vm_offset_t)(page_addr 
-					- resident->page_addr))
-				   	>>12));
-			}
-			/* Throw out old entry if there are no */
-			/* more pages in cache */
-			if(resident->page_cache == 0) {
-				/* delete addr hash entry */
-				cache_ele->element = 0;
-				*trailer = cache_ele->next;
-				cache_ele->next = *free_list;
-				*free_list = cache_ele;
-				/* go after object hash */
-				index = do_tws_hash(
-					resident->object, 
-					resident->offset,
-					tws->number_of_elements, 
-					tws->number_of_lines);
-				tws_traverse_object_hash_list(tws,
-					index, resident->object, 
-					resident->offset,
-					0xFFFFFFFF, &cache_ele,
-					&trailer, &free_list);
-				if(cache_ele != NULL) {
-					if(tws->style == 
-					   TWS_HASH_STYLE_SIGNAL) {
-						vm_object_deallocate(
-						   cache_ele->element->object);
-						vm_map_deallocate(
-						   cache_ele->element->map);
-					}
-					current_line = 
-					   cache_ele->element->line;
-					set = current_line
-						/tws->number_of_lines;
-					current_line -= set *
-						tws->number_of_lines;
-					if(cache_ele->element->object != 0) {
-						cache_ele->element->object = 0;
-						tws->cache[set]
-							[current_line].ele_count--;
-					}
-					cache_ele->element = 0;
-					*trailer = cache_ele->next;
-					cache_ele->next = *free_list;
-					*free_list = cache_ele;
-				}
-			}
-			continue;
-		}
-		ctr+=1;
-	}
-
-	/* 
-	 * We may or may not have a current line setting coming out of
-	 * the code above.  If we have a current line it means we can
-	 * choose to back-fill the spot vacated by a previous entry.
-	 * We have yet to do a definitive check using the original obj/off
-	 * We will do that now and override the current line if we
-	 * find an element
-	 */
-
-	index = do_tws_hash(object, offset, 
-			tws->number_of_elements, tws->number_of_lines);
-
-	alt_index = index_enum[0];
-
-	tws_traverse_object_hash_list(tws, index, object, offset,
-			0xFFFFFFFF, &cache_ele, &trailer, &free_list);
-	if(cache_ele != NULL) {
-		obj_ele = cache_ele;
-		current_line = cache_ele->element->line;
-		set = current_line/tws->number_of_lines;
-		current_line -= set * tws->number_of_lines;
-		target_element = cache_ele->element;
-
-		/* Now check to see if we have a hash addr for it */
-		tws_traverse_address_hash_list(tws, 
-				alt_index, obj_ele->element->page_addr,
-				obj_ele->element->object,
-				obj_ele->element->offset,
-				obj_ele->element->map,
-				&cache_ele, &trailer, &free_list, 0);
-		if(cache_ele != NULL) {
-			addr_ele = cache_ele;
-		} else  {
-			addr_ele = new_addr_hash(tws, set, alt_index);
-			/* if cannot allocate just do without */
-			/* we'll get it next time around */
-		}
-	}
-
-
-
-	if(tws->style == TWS_HASH_STYLE_SIGNAL) {
-		vm_object_reference(object);
-		vm_map_reference(map);
-	}
-
-	if(current_line == 0xFFFFFFFF) {
-		current_line = tws->current_line;
-		set = current_line/tws->number_of_lines;
-		current_line = current_line - (set * tws->number_of_lines);
-
-#ifdef notdef
-		if(cache_full) {
-			tws->current_line = tws->number_of_lines - 1;
-		}
-#endif
-		if(tws->cache[set][current_line].ele_count 
-				>= tws->number_of_elements) {
-			current_line++;
-			tws->current_line++;
-			if(current_line == tws->number_of_lines) {
-				set++;
-				current_line = 0;
-				if (set == tws->expansion_count) {
-				   if((tws->lookup_count <
-				      (2 * tws->insert_count)) &&
-				      (set<TWS_HASH_EXPANSION_MAX)) {
-				      tws->lookup_count = 0;
-				      tws->insert_count = 0;
-				      if(tws->number_of_lines 
-						< TWS_HASH_LINE_COUNT) {
-					tws->current_line--;
-					tws_unlock(tws);
-					return KERN_NO_SPACE;
-				      }
-				      /* object persistence is guaranteed by */
-				      /* an elevated paging or object        */
-				      /* reference count in the caller. */
-				      vm_object_unlock(object);
-				      if((tws->table[set] = (tws_hash_ptr_t *)
-				         kalloc(sizeof(tws_hash_ptr_t) 
-						* tws->number_of_lines 
-						* tws->number_of_elements)) 
-								== NULL) {
-				         set = 0;
-				      } else if((tws->table_ele[set] = 
-					(tws_hash_ptr_t)
-				         kalloc(sizeof(struct tws_hash_ptr) 
-						* tws->number_of_lines 
-						* tws->number_of_elements)) 
-								== NULL) {
-				         kfree(tws->table[set], 
-					       sizeof(tws_hash_ptr_t) 
-					       * tws->number_of_lines 
-					       * tws->number_of_elements);
-				         set = 0;
-				      } else if((tws->alt_ele[set] = 
-						(tws_hash_ptr_t)
-			 		 kalloc(sizeof(struct tws_hash_ptr)
-						* tws->number_of_lines 
-						* tws->number_of_elements)) 
-								== NULL) {
-				         kfree(tws->table_ele[set], 
-					       sizeof(struct tws_hash_ptr) 
-					       * tws->number_of_lines 
-					       * tws->number_of_elements);
-				         kfree(tws->table[set], 
-					       sizeof(tws_hash_ptr_t) 
-				       		* tws->number_of_lines 
-						* tws->number_of_elements);
-				         tws->table[set] = NULL;
-				         set = 0;
-
-				      } else if((tws->cache[set] = 
-						(struct tws_hash_line *)
-						kalloc(sizeof
-						(struct tws_hash_line) 
-						* tws->number_of_lines)) 
-								== NULL) {
-				         kfree(tws->alt_ele[set], 
-					       sizeof(struct tws_hash_ptr) 
-					       * tws->number_of_lines 
-					       * tws->number_of_elements);
-					 kfree(tws->table_ele[set], 
-					       sizeof(struct tws_hash_ptr) 
-					       * tws->number_of_lines 
-					       * tws->number_of_elements);
-				         kfree(tws->table[set], 
-					       sizeof(tws_hash_ptr_t) 
-					       * tws->number_of_lines 
-					       * tws->number_of_elements);
-				         tws->table[set] = NULL;
-				         set = 0;
-							
-				      } else {
-					 tws->free_hash_ele[set] = 
-							(tws_hash_ptr_t)0;
-					 tws->obj_free_count[set] = 0;
-					 tws->addr_free_count[set] = 0;
-				         bzero((char *)tws->table[set], 
-						sizeof(tws_hash_ptr_t) 
-						* tws->number_of_lines 
-						* tws->number_of_elements);
-				         bzero((char *)tws->table_ele[set], 
-						sizeof(struct tws_hash_ptr) 
-						* tws->number_of_lines 
-						* tws->number_of_elements);
-				         bzero((char *)tws->alt_ele[set], 
-						sizeof(struct tws_hash_ptr) 
-						* tws->number_of_lines 
-						* tws->number_of_elements);
-				         bzero((char *)tws->cache[set], 
-						sizeof(struct tws_hash_line) 
-						* tws->number_of_lines);
-				      }
-				      vm_object_lock(object);
-				   } else {
-				      if (tws->startup_name != NULL) {
-						tws->current_line--;
-
-						age_of_cache = ((sched_tick - tws->time_of_creation) >> SCHED_TICK_SHIFT);
-
-						tws_unlock(tws);
-
-						if (age_of_cache > 45)
-						        return KERN_OPERATION_TIMED_OUT;
-
-						return KERN_FAILURE;
-				      }
-				      tws->lookup_count = 0;
-				      tws->insert_count = 0;
-				      set = 0;
-				   }
-				}
-				tws->current_line = set * tws->number_of_lines;
-			}
-			if(set < tws->expansion_count) {
-			   tws_hash_line_clear(tws, 
-				&(tws->cache[set][current_line]), object, TRUE);
-			   if(tws->cache[set][current_line].ele_count 
-						>= tws->number_of_elements) {
-				if(tws->style == TWS_HASH_STYLE_SIGNAL) {
-					vm_object_deallocate(object);
-					vm_map_deallocate(map);
-				}
-				tws_unlock(tws);
-				return KERN_FAILURE;
-			   }
-			} else {
-				tws->expansion_count++;
-			}
-		}
-	}
-
-
-	/* set object hash element */
-	if(obj_ele == NULL) {
-		obj_ele = new_obj_hash(tws, set, index);
-		if(obj_ele == NULL) {
-			tws->cache[set][current_line].ele_count 
-				= tws->number_of_elements;
-			tws_unlock(tws);
-			return KERN_FAILURE;
-		}
-	}
-
-	/* set address hash element */
-	if(addr_ele == NULL) {
-		addr_ele = new_addr_hash(tws, set, alt_index);
-	}
-
-	if(target_element == NULL) {
-		ele_index = 0;
-		for(i = 0; i<tws->number_of_elements; i++) {
-			if(tws->cache[set][current_line].
-				list[ele_index].object == 0) {
-				break;
-			}
-			ele_index++;
-			if(ele_index >= tws->number_of_elements)
-				ele_index = 0;
-		
-		}
-
-		if(i == tws->number_of_elements) 
-			panic("tws_insert: no free elements");
-
-		target_element = 
-			&(tws->cache[set][current_line].list[ele_index]);
-
-		tws->cache[set][current_line].ele_count++;
-	}
-
-	obj_ele->element = target_element;
-	if(addr_ele) {
-		addr_ele->element = (tws_hash_ele_t)
-			(((unsigned int)target_element) | TWS_ADDR_HASH);
-	}
-	target_element->object = object;
-	target_element->offset = offset & TWS_HASH_OFF_MASK;
-	target_element->page_addr = 
-			page_addr - (offset - (offset & TWS_HASH_OFF_MASK));
-	target_element->map = map;
-	target_element->line = 
-			current_line + (set * tws->number_of_lines);
-
-	target_element->page_cache |= startup_cache_line;
-	target_element->page_cache |= 1<<(((vm_offset_t)(offset & TWS_INDEX_MASK))>>12);
-
-	tws_unlock(tws);
-
-	if (age_of_cache > 45)
-		return KERN_OPERATION_TIMED_OUT;
-
-	return KERN_SUCCESS;
-}
-
-/* 
- * tws_build_cluster
- * lengthen the cluster of pages by the number of pages encountered in the 
- * working set up to the limit requested by the caller.  The object needs
- * to be locked on entry.  The map does not because the tws_lookup function
- * is used only to find if their is an entry in the cache.  No transient
- * data from the cache is de-referenced.
- *
- */
-#if	MACH_PAGEMAP
-/*
- * MACH page map - an optional optimization where a bit map is maintained
- * by the VM subsystem for internal objects to indicate which pages of
- * the object currently reside on backing store.  This existence map
- * duplicates information maintained by the vnode pager.  It is 
- * created at the time of the first pageout against the object, i.e. 
- * at the same time pager for the object is created.  The optimization
- * is designed to eliminate pager interaction overhead, if it is 
- * 'known' that the page does not exist on backing store.
- *
- * LOOK_FOR() evaluates to TRUE if the page specified by object/offset is 
- * either marked as paged out in the existence map for the object or no 
- * existence map exists for the object.  LOOK_FOR() is one of the
- * criteria in the decision to invoke the pager.   It is also used as one
- * of the criteria to terminate the scan for adjacent pages in a clustered
- * pagein operation.  Note that LOOK_FOR() always evaluates to TRUE for
- * permanent objects.  Note also that if the pager for an internal object 
- * has not been created, the pager is not invoked regardless of the value 
- * of LOOK_FOR() and that clustered pagein scans are only done on an object
- * for which a pager has been created.
- *
- * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
- * is marked as paged out in the existence map for the object.  PAGED_OUT()
- * PAGED_OUT() is used to determine if a page has already been pushed
- * into a copy object in order to avoid a redundant page out operation.
- */
-#define LOOK_FOR(o, f) (vm_external_state_get((o)->existence_map, (f)) \
-			!= VM_EXTERNAL_STATE_ABSENT)
-#define PAGED_OUT(o, f) (vm_external_state_get((o)->existence_map, (f)) \
-			== VM_EXTERNAL_STATE_EXISTS)
-#else /* MACH_PAGEMAP */
-/*
- * If the MACH page map optimization is not enabled,
- * LOOK_FOR() always evaluates to TRUE.  The pager will always be 
- * invoked to resolve missing pages in an object, assuming the pager 
- * has been created for the object.  In a clustered page operation, the 
- * absence of a page on backing backing store cannot be used to terminate 
- * a scan for adjacent pages since that information is available only in 
- * the pager.  Hence pages that may not be paged out are potentially 
- * included in a clustered request.  The vnode pager is coded to deal 
- * with any combination of absent/present pages in a clustered 
- * pagein request.  PAGED_OUT() always evaluates to FALSE, i.e. the pager
- * will always be invoked to push a dirty page into a copy object assuming
- * a pager has been created.  If the page has already been pushed, the
- * pager will ingore the new request.
- */
-#define LOOK_FOR(o, f) TRUE
-#define PAGED_OUT(o, f) FALSE
-#endif /* MACH_PAGEMAP */
-
-
-void
-tws_build_cluster(
-		tws_hash_t		tws,
-		vm_object_t		object,
-		vm_object_offset_t	*start,
-		vm_object_offset_t	*end,
-		vm_size_t		max_length)
-{
-	tws_hash_line_t		line;
-	vm_object_offset_t	before = *start;
-	vm_object_offset_t	after =  *end;
-	vm_object_offset_t	original_start = *start;
-	vm_object_offset_t	original_end = *end;
-	vm_size_t		length =  (vm_size_t)(*end - *start);
-	vm_page_t		m;
-	kern_return_t		kret;
-	vm_object_offset_t	object_size;
-	int			age_of_cache;
-	vm_size_t		pre_heat_size;
-	unsigned int		ele_cache;
-	unsigned int		end_cache = 0;
-	unsigned int		start_cache = 0;
-	unsigned int		memory_scarce = 0;
-
-	if((object->private) || !(object->pager))
-		return;
-
-	if (!object->internal) {
-		/* XXX FBDP !internal doesn't mean vnode pager */
-		kret = vnode_pager_get_object_size(
-       		 	object->pager,
-			&object_size);
-		if (kret != KERN_SUCCESS) {
-			object_size = object->size;
-		}
-	} else {
-		object_size = object->size;
-	}
-
-	if((!tws) || (!tws_lock_try(tws))) {
-		return;
-	}
-	age_of_cache = ((sched_tick 
-			- tws->time_of_creation) >> SCHED_TICK_SHIFT);
-
-	if (vm_page_free_count < (2 * vm_page_free_target))
-	        memory_scarce = 1;
-
-	/* When pre-heat files are not available, resort to speculation */
-	/* based on size of file */
-
-	if (tws->startup_cache || object->internal || age_of_cache > 45) {
-		pre_heat_size = 0;
-	} else {
-		if (object_size > (vm_object_offset_t)(1024 * 1024))
-			pre_heat_size = 8 * PAGE_SIZE;
-		else if (object_size > (vm_object_offset_t)(128 * 1024))
-			pre_heat_size = 4 * PAGE_SIZE;
-		else
-			pre_heat_size = 2 * PAGE_SIZE;
-	}
-
-       	if (tws->startup_cache) {
-	        int target_page_count;
-
-		if (memory_scarce)
-		        target_page_count = 16;
-		else
-		        target_page_count = 4;
-
-	        if (tws_test_for_community(tws, object, *start, target_page_count, &ele_cache))
-		{
-	                start_cache = ele_cache;
-			*start = *start & TWS_HASH_OFF_MASK;
-			*end = *start + (32 * PAGE_SIZE_64);
-
-			if (*end > object_size) {
-				*end = round_page_64(object_size);
-				max_length = 0;
-			} else
-			        end_cache = ele_cache;
-
-			while (max_length > ((*end - *start) + (32 * PAGE_SIZE))) {
-			        int	expanded;
-
-				expanded = 0;
-				after = *end;
-
-				if ((after + (32 * PAGE_SIZE_64)) <= object_size &&
-				    (tws_test_for_community(tws, object, after, 8, &ele_cache))) {
-
-					*end = after + (32 * PAGE_SIZE_64);
-					end_cache = ele_cache;
-					expanded = 1;
-				}
-				if (max_length < ((*end - *start) + (32 * PAGE_SIZE_64))) {
-					break;
-				}
-				if (*start) {
-				        before = (*start - PAGE_SIZE_64) & TWS_HASH_OFF_MASK;
-
-					if (tws_test_for_community(tws, object, before, 8, &ele_cache)) {
-
-					        *start = before;
-						start_cache = ele_cache;
-						expanded = 1;
-					}
-				}
-				if (expanded == 0)
-					break;
-			}
-			if (end_cache)
-			        *end -= PAGE_SIZE_64;
-
-			if (start_cache != 0) {
-				unsigned int mask;
-
-				for (mask = 1; mask != 0; mask = mask << 1) {
-			        	if (*start == original_start)
-				        	break;
-					if (!(start_cache & mask))
-						*start += PAGE_SIZE_64;
-					else
-						break;
-				}
-			}
-			if (end_cache != 0) {
-				unsigned int mask;
-
-				for (mask = 0x80000000; 
-						mask != 0; mask = mask >> 1) {
-			        	if (*end == original_end)
-				       		 break;
-					if (!(end_cache & mask))
-						*end -= PAGE_SIZE_64;
-					else
-						break;
-				}
-			}
-			tws_unlock(tws);
-
-			if (*end < original_end)
-			        *end = original_end;
-			return;
-		}
-	}
-
-	while ((length < max_length) &&
-		(object_size >= 
-			(after + PAGE_SIZE_64))) {
-		if(length >= pre_heat_size) {
-		   if(tws_internal_lookup(tws, after, object,
-				&line) != KERN_SUCCESS) {
-			vm_object_offset_t	extend;
-				
-			extend = after + PAGE_SIZE_64;
-			if(tws_internal_lookup(tws, extend, object,
-						&line) != KERN_SUCCESS) {
-				break;
-			}
-		   }
-		}
-
-	        if ((object->existence_map != NULL)
-					&& (!LOOK_FOR(object, after))) {
-			break;
-		} 
-
-		if (vm_page_lookup(object, after) != VM_PAGE_NULL) {
-			/*
-			 * don't bridge resident pages
-			 */
-		        break;
-		}
-
-		if (object->internal) {
-			/*
-			 * need to acquire a real page in
-			 * advance because this acts as
-			 * a throttling mechanism for
-			 * data_requests to the default
-			 * pager.  If this fails, give up
-			 * trying to find any more pages
-			 * in the cluster and send off the
-			 * request for what we already have.
-			 */
-			if ((m = vm_page_grab()) == VM_PAGE_NULL) {
-				break;
-			}
-	        } else if ((m = vm_page_grab_fictitious())
-			       	        == VM_PAGE_NULL) {
-			break;
-		}
-		m->absent = TRUE;
-		m->unusual = TRUE;
-	        m->clustered = TRUE;
-		m->list_req_pending = TRUE;
-
-	        vm_page_insert(m, object, after);
-		object->absent_count++;
-		after += PAGE_SIZE_64;
-		length += PAGE_SIZE;
-	}
-	*end = after;
-	while (length < max_length) {
-		if (before == 0) 
-			break;
-		before -= PAGE_SIZE_64;
-
-		if(length >= pre_heat_size) {
-		   if(tws_internal_lookup(tws, before, object,
-				&line) != KERN_SUCCESS) {
-			vm_object_offset_t	extend;
-				
-			extend = before;
-			if (extend == 0)
-				break;
-			extend -= PAGE_SIZE_64;
-			if(tws_internal_lookup(tws, extend, object,
-					&line) != KERN_SUCCESS) {
-				break;
-			}
-		   }
-		}
-		if ((object->existence_map != NULL) 
-					&& (!LOOK_FOR(object, before))) {
-			break;
-		}
-
-		if (vm_page_lookup(object, before) != VM_PAGE_NULL) {
-			/*
-			 * don't bridge resident pages
-			 */
-		        break;
-		}
-
-		if (object->internal) {
-			/*
-			 * need to acquire a real page in
-			 * advance because this acts as
-			 * a throttling mechanism for
-			 * data_requests to the default
-			 * pager.  If this fails, give up
-			 * trying to find any more pages
-			 * in the cluster and send off the
-			 * request for what we already have.
-			 */
-			if ((m = vm_page_grab()) == VM_PAGE_NULL) {
-				break;
-			}
-	        } else if ((m = vm_page_grab_fictitious())
-			       	        == VM_PAGE_NULL) {
-			break;
-	        }
-	        m->absent = TRUE;
-		m->unusual = TRUE;
-	        m->clustered = TRUE;
-		m->list_req_pending = TRUE;
-
-	        vm_page_insert(m, object, before);
-		object->absent_count++;
-		*start -= PAGE_SIZE_64;
-		length += PAGE_SIZE;
-	}
-	tws_unlock(tws);
-}
-
-void
-tws_line_signal(
-	tws_hash_t	tws,
-	vm_map_t	map,
-	tws_hash_line_t hash_line,
-	vm_offset_t	target_page)
-{
-	unsigned int		i,j;
-	vm_object_t		object;
-	vm_object_offset_t	offset;
-	vm_object_offset_t	before;
-	vm_object_offset_t	after;
-	struct tws_hash_ele	*element;
-	vm_page_t		m,p;
-	kern_return_t		rc;
-
-	if(tws->style != TWS_HASH_STYLE_SIGNAL)
-		return;
-
-	vm_map_lock(map);
-	for (i=0; i<tws->number_of_elements; i++) {
-
-		vm_object_offset_t local_off = 0;
-
-		if(hash_line->list[i].object == 0)
-			continue;
-
-		element = &hash_line->list[i];
-
-		if (element->page_addr == target_page)
-			continue;
-
-		j = 1;
-		while (j != 0) {
-			if(j & element->page_cache)
-				break;
-			j <<= 1;
-			local_off += PAGE_SIZE_64;
-		}
-		object = element->object;
-		offset = element->offset + local_off;
-
-		/* first try a fast test to speed up no-op signal */
-		if (((p = vm_page_lookup(object, offset)) != NULL) 
-			|| (object->pager == NULL) 
-			|| (object->shadow_severed)) {
-			continue;
-		}	
-
-		if((!object->alive) || 
-			(!object->pager_created) || (!object->pager_ready))
-			continue;
-
-                if (object->internal) { 
-			if (object->existence_map == NULL) {
-				if (object->shadow)
-					continue;
-			} else {
-				if(!LOOK_FOR(object, offset))
-					continue;
-			}
-		}
-
-		vm_object_reference(object);
-		vm_map_unlock(map);
-			
-		if(object->internal) {
-			m = vm_page_grab();
-		} else {
-			m = vm_page_grab_fictitious();
-		}
-
-		if(m == NULL) {
-			vm_object_deallocate(object);
-			vm_map_lock(map);
-			continue;
-		}
-
-		vm_object_lock(object);
-		if (((p = vm_page_lookup(object, offset)) != NULL) 
-			|| (object->pager == NULL) 
-			|| (object->shadow_severed)) {
-			VM_PAGE_FREE(m);
-			vm_object_unlock(object);
-			vm_object_deallocate(object);
-			vm_map_lock(map);
-			continue;
-		}
-
-		vm_page_insert(m, object, offset);
-
-		if (object->absent_count > vm_object_absent_max) {
-			VM_PAGE_FREE(m);
-			vm_object_unlock(object);
-			vm_object_deallocate(object);
-			vm_map_lock(map);
-			break;
-		}
-		m->list_req_pending = TRUE; 
-		m->absent = TRUE; 
-		m->unusual = TRUE; 
-		object->absent_count++;
-
-		before = offset;
-		after = offset + PAGE_SIZE_64;
-		tws_build_cluster(tws, object, &before, &after, 0x16000);
-		vm_object_unlock(object);
-
-		rc = memory_object_data_request(object->pager, 
-				before + object->paging_offset, 
-				(vm_size_t)(after - before), VM_PROT_READ);
-		if (rc != KERN_SUCCESS) {
-			offset = before;
-			vm_object_lock(object);
-			while (offset < after) {
-                        	m = vm_page_lookup(object, offset);
-				if(m && m->absent && m->busy)
-					VM_PAGE_FREE(m);
-				offset += PAGE_SIZE;
-			}
-			vm_object_unlock(object);
-			vm_object_deallocate(object);
-		} else {
-			vm_object_deallocate(object);
-		}
-		vm_map_lock(map);
-		continue;
-	}
-	vm_map_unlock(map);
-}
-
-/* tws locked on entry */
-
-tws_startup_t
-tws_create_startup_list(
-	tws_hash_t	tws)
-{
-
-	tws_startup_t		startup;
-	unsigned int		i,j,k;
-	unsigned int		total_elements;
-	unsigned int		startup_size;
-	unsigned int		sindex;
-	unsigned int		hash_index;
-	tws_startup_ptr_t	element;
-
-	total_elements = tws->expansion_count *
-			(tws->number_of_lines * tws->number_of_elements);
-
-	startup_size = sizeof(struct tws_startup) 
-			+ (total_elements * sizeof(tws_startup_ptr_t *))
-			+ (total_elements * sizeof(struct tws_startup_ptr))
-			+ (total_elements * sizeof(struct tws_startup_ele));
-	startup = (tws_startup_t)(kalloc(startup_size));
-
-	if(startup == NULL)
-		return startup;
-
-	bzero((char *) startup, startup_size);
-
-	startup->table = (tws_startup_ptr_t *)
-		(((int)startup) + (sizeof(struct tws_startup)));
-	startup->ele = (struct tws_startup_ptr *)
-			(((vm_offset_t)startup->table) + 
-			(total_elements * sizeof(tws_startup_ptr_t)));
-
-	startup->array = (struct tws_startup_ele *)
-			(((vm_offset_t)startup->ele) + 
-			(total_elements * sizeof(struct tws_startup_ptr)));
-
-	startup->tws_hash_size = startup_size;
-	startup->ele_count = 0;  /* burn first hash ele, else we can't tell from zero */
-	startup->array_size = total_elements;
-	startup->hash_count = 1;
-
-	sindex = 0;
-	
-	
-	for(i = 0; i<tws->number_of_lines; i++) {
-	   for(j = 0; j<tws->number_of_elements; j++) {
-	      for(k = 0; k<tws->expansion_count; k++) {
-		tws_hash_ele_t		entry;
-		unsigned int		hash_retry;
-		vm_offset_t		addr;
-
-		entry = &tws->cache[k][i].list[j];
-		addr = entry->page_addr;
-		hash_retry = 0;
-		if(entry->object != 0) {
-			/* get a hash element */
-			hash_index = do_startup_hash(addr, 
-						startup->array_size);
-
-			if(startup->hash_count < total_elements) {
-				element = &(startup->ele[startup->hash_count]);
-				startup->hash_count += 1;
-			} else {
-				/* exit we're out of elements */
-				break;
-			}
-			/* place the hash element */
-			element->next = startup->table[hash_index];
-			startup->table[hash_index] = (tws_startup_ptr_t)
-				((int)element - (int)&startup->ele[0]);
-
-			/* set entry OFFSET in hash element */
-			element->element =  (tws_startup_ele_t)
-				((int)&startup->array[sindex] - 
-						(int)&startup->array[0]);
-		
-			startup->array[sindex].page_addr = entry->page_addr;
-			startup->array[sindex].page_cache = entry->page_cache;
-			startup->ele_count++;
-			sindex++;
-		
-		}
-	     }
-	   }
-	}
-
-	return startup;
-}
-
-
-/*
- * Returns an entire cache line.  The line is deleted from the startup
- * cache on return. The caller can check startup->ele_count for an empty
- * list. Access synchronization is the responsibility of the caller.
- */
-
-unsigned int
-tws_startup_list_lookup(
-	tws_startup_t	startup,
-	vm_offset_t	addr)
-{
-	unsigned int		hash_index;
-	unsigned int		page_cache_bits;
-	unsigned int		startup_shift;
-	tws_startup_ele_t	entry;
-	vm_offset_t		next_addr;
-	tws_startup_ptr_t	element;
-	tws_startup_ptr_t	base_ele;
-	tws_startup_ptr_t	*previous_ptr;
-
-	page_cache_bits = 0;
-
-	hash_index = do_startup_hash(addr, startup->array_size);
-
-	if(((unsigned int)&(startup->table[hash_index])) >= ((unsigned int)startup + startup->tws_hash_size)) {
-		return page_cache_bits = 0;
-	}
-	element = (tws_startup_ptr_t)((int)startup->table[hash_index] +
-			(int)&startup->ele[0]);
-	base_ele = element;
-	previous_ptr = &(startup->table[hash_index]);
-	while(element > &startup->ele[0]) {
-		if (((int)element + sizeof(struct tws_startup_ptr))
-			> ((int)startup + startup->tws_hash_size))  {
-			return page_cache_bits;
-		}
-		entry = (tws_startup_ele_t)
-			((int)element->element 
-				+ (int)&startup->array[0]);
-		if((((int)entry + sizeof(struct tws_startup_ele))
-			> ((int)startup + startup->tws_hash_size)) 
-				|| ((int)entry < (int)startup))  {
-			return page_cache_bits;
-		}
-		if ((addr >= entry->page_addr) && 
-			(addr <= (entry->page_addr + 0x1F000))) {
-			startup_shift = (addr - entry->page_addr)>>12;
-			page_cache_bits |= entry->page_cache >> startup_shift;
-			/* don't dump the pages, unless the addresses */
-			/* line up perfectly.  The cache may be used */
-			/* by other mappings */
-			entry->page_cache &= (1 << startup_shift) - 1;
-			if(addr == entry->page_addr) {
-				if(base_ele == element) {
-					base_ele = (tws_startup_ptr_t)
-						((int)element->next 
-						+ (int)&startup->ele[0]);
-					startup->table[hash_index] = element->next;
-					element = base_ele;
-				} else {
-					*previous_ptr = element->next;
-					element = (tws_startup_ptr_t)
-						((int)*previous_ptr 
-						+ (int)&startup->ele[0]);
-				}
-				entry->page_addr = 0;
-				startup->ele_count--;
-				continue;
-			}
-		}
-		next_addr = addr + 0x1F000;
-		if ((next_addr >= entry->page_addr) && 
-			(next_addr <= (entry->page_addr + 0x1F000))) {
-			startup_shift = (next_addr - entry->page_addr)>>12;
-			page_cache_bits |= entry->page_cache << (0x1F - startup_shift);
-			entry->page_cache &= ~((1 << (startup_shift + 1)) - 1);
-			if(entry->page_cache == 0) {
-				if(base_ele == element) {
-					base_ele = (tws_startup_ptr_t)
-						((int)element->next 
-						+ (int)&startup->ele[0]);
-					startup->table[hash_index] = element->next;
-					element = base_ele;
-				} else {
-					*previous_ptr = element->next;
-					element = (tws_startup_ptr_t)
-						((int)*previous_ptr 
-						+ (int)&startup->ele[0]);
-				}
-				entry->page_addr = 0;
-				startup->ele_count--;
-				continue;
-			}
-		}
-		previous_ptr = &(element->next);
-		element = (tws_startup_ptr_t) 
-			((int) element->next + (int) &startup->ele[0]);
-	}
-
-	return page_cache_bits;
-}
-
-kern_return_t
-tws_send_startup_info(
-		task_t		task)
-{
-
-	tws_hash_t		tws;
-
-	task_lock(task);
-	tws = task->dynamic_working_set;
-	task_unlock(task);
-	if(tws == NULL) {
-		return KERN_FAILURE;
-	}
-	return tws_internal_startup_send(tws);
-}
-
-
-kern_return_t
-tws_internal_startup_send(
-		tws_hash_t	tws)
-{
-
-	tws_startup_t 		scache;
-
-	if(tws == NULL) {
-		return KERN_FAILURE;
-	}
-	tws_lock(tws);
-	/* used to signal write or release depending on state of tws */
-	if(tws->startup_cache) {
-		vm_offset_t	startup_buf;
-		vm_size_t	size;
-		startup_buf = (vm_offset_t)tws->startup_cache;
-		size = tws->startup_cache->tws_hash_size;
-		tws->startup_cache = 0;
-		tws_unlock(tws);
-		kmem_free(kernel_map, startup_buf, size);
-		return KERN_SUCCESS;
-	}
-   	if(tws->startup_name == NULL) {
-		tws_unlock(tws);
-		return KERN_FAILURE;
-	}
-	scache = tws_create_startup_list(tws);
-	if(scache == NULL)
-		return KERN_FAILURE;
-	bsd_write_page_cache_file(tws->uid, tws->startup_name, 
-				  (caddr_t) scache, scache->tws_hash_size, 
-				  tws->mod, tws->fid);
-	kfree(scache, scache->tws_hash_size);
-	kfree(tws->startup_name, tws->startup_name_length);
-	tws->startup_name = NULL;
-	tws_unlock(tws);
-	return KERN_SUCCESS;
-}
-
-kern_return_t
-tws_handle_startup_file(
-		task_t			task,
-		unsigned int		uid,
-		char			*app_name,
-		void			*app_vp,
-		boolean_t		*new_info)
-
-{
-		tws_startup_t	startup;
-		vm_offset_t	cache_size;	
-		kern_return_t	error;
-		int		fid;
-		int		mod;
-		
-		*new_info = FALSE;
-		/* don't pre-heat kernel task */
-		if(task == kernel_task)
-			return KERN_SUCCESS;
-		error = bsd_read_page_cache_file(uid, &fid, 
-					&mod, app_name, 
-					app_vp,
-					(vm_offset_t *) &startup, 
-					&cache_size);
-		if(error) {
-			return KERN_FAILURE;
-		}
-		if(startup == NULL) {
-			/* Entry for app does not exist, make */
-			/* one */
-			/* we will want our own copy of the shared */
-			/* regions to pick up a true picture of all */
-			/* the pages we will touch. */
-			if((lsf_zone->count * lsf_zone->elem_size) 
-						> (lsf_zone->max_size >> 1)) {
-				/* We don't want to run out of shared memory */
-				/* map entries by starting too many private versions */
-				/* of the shared library structures */
-				return KERN_SUCCESS;
-			}
-			*new_info = TRUE;
-
-			error = tws_write_startup_file(task, 
-					fid, mod, app_name, uid);
-			if(error)
-				return error;
-
-		} else {
-			error = tws_read_startup_file(task, 
-						(tws_startup_t)startup, 
-						cache_size);
-			if(error) {
-				kmem_free(kernel_map, 
-					(vm_offset_t)startup, cache_size); 
-				return error;
-			}
-		}
-		return KERN_SUCCESS;
-}
-
-kern_return_t
-tws_write_startup_file(
-		task_t			task,
-		int			fid,
-		int			mod,
-		char			*name,
-		unsigned int		uid)
-{
-		tws_hash_t		tws;
-		unsigned int		string_length;
-
-		string_length = strlen(name);
-
-restart:
-		task_lock(task);
-		tws = task->dynamic_working_set;
-		task_unlock(task);
-
-		if(tws == NULL) {
-			kern_return_t error;
-
-			/* create a dynamic working set of normal size */
-			if((error = task_working_set_create(task, 0, 0, TWS_HASH_STYLE_DEFAULT)) != KERN_SUCCESS)
-				return error;
-			/* we need to reset tws and relock */
-			goto restart;
-		}
-		tws_lock(tws);
-
-		if(tws->startup_name != NULL) {
-			tws_unlock(tws);
-			return KERN_FAILURE;
-		}
-
-		tws->startup_name = (char *)
-			kalloc((string_length + 1) * (sizeof(char)));
-		if(tws->startup_name == NULL) {
-			tws_unlock(tws);
-			return KERN_FAILURE;
-		}
-
-		bcopy(name, (char *)tws->startup_name, string_length + 1);
-		tws->startup_name_length = (string_length + 1) * sizeof(char);
-		tws->uid = uid;
-		tws->fid = fid;
-		tws->mod = mod;
-
-		tws_unlock(tws);
-		return KERN_SUCCESS;
-}
-
-unsigned long tws_read_startup_file_rejects = 0;
-
-kern_return_t
-tws_read_startup_file(
-		task_t			task,
-		tws_startup_t		startup,
-		vm_offset_t		cache_size)
-{
-		tws_hash_t		tws;
-		int			lines;
-		int			old_exp_count;
-		unsigned int		ele_count;
-
-restart:
-		task_lock(task);
-		tws = task->dynamic_working_set;
-
-		/* create a dynamic working set to match file size */
-
-		/* start with total size of the data we got from app_profile */
-		ele_count = cache_size;
-		/* skip the startup header */
-		ele_count -= sizeof(struct tws_startup); 
-		/*
-		 * For each startup cache entry, we have one of these:
-		 *	tws_startup_ptr_t startup->table[];
-		 * 	struct tws_startup_ptr startup->ele[];
-		 *	struct tws_startup_ele startup->array[];
-		 */
-		ele_count /= (sizeof (tws_startup_ptr_t) +
-			      sizeof (struct tws_startup_ptr) +
-			      sizeof (struct tws_startup_ele));
-
-		/*
-		 * Sanity check: make sure the value for startup->array_size
-		 * that we read from the app_profile file matches the size
-		 * of the data we read from disk.  If it doesn't match, we
-		 * can't trust the data and we just drop it all.
-		 */
-		if (cache_size < sizeof(struct tws_startup) ||
-		    startup->array_size != ele_count) {
-			tws_read_startup_file_rejects++;
-			task_unlock(task);
-			kmem_free(kernel_map, (vm_offset_t)startup, cache_size); 
-			return(KERN_SUCCESS);
-		}
-
-		/*
-		 * We'll create the task working set with the default row size
-		 * (TWS_ARRAY_SIZE), so this will give us the number of lines
-		 * we need to store all the data from the app_profile startup
-		 * cache.
-		 */
-		lines = ele_count / TWS_ARRAY_SIZE;
-
-		if(lines <= TWS_SMALL_HASH_LINE_COUNT) {
-			lines = TWS_SMALL_HASH_LINE_COUNT;
-			task_unlock(task);
-			kmem_free(kernel_map, (vm_offset_t)startup, cache_size); 
-			return(KERN_SUCCESS);
-		} else {
-			old_exp_count = lines/TWS_HASH_LINE_COUNT;
-			if((old_exp_count * TWS_HASH_LINE_COUNT) != lines) {
-				lines = (old_exp_count + 1) 
-						* TWS_HASH_LINE_COUNT;
-			}
-			if(tws == NULL) {
-				kern_return_t error;
-
-				task_unlock(task);
-				if ((error = task_working_set_create(task, lines, 0, TWS_HASH_STYLE_DEFAULT)) != KERN_SUCCESS)
-					return error;
-				/* we need to reset tws and relock */
-				goto restart;
-			} else {
-				task_unlock(task);
-				tws_expand_working_set(
-					(void *)tws, lines, TRUE);
-			}
-		}
-		
-		tws_lock(tws);
-		
-		if(tws->startup_cache != NULL) {
-			tws_unlock(tws);
-			return KERN_FAILURE;
-		}
-
-
-		/* now need to fix up internal table pointers */
-		startup->table = (tws_startup_ptr_t *)
-			(((int)startup) + (sizeof(struct tws_startup)));
-		startup->ele = (struct tws_startup_ptr *)
-			(((vm_offset_t)startup->table) + 
-			(startup->array_size * sizeof(tws_startup_ptr_t)));
-		startup->array = (struct tws_startup_ele *)
-			(((vm_offset_t)startup->ele) + 
-			(startup->array_size * sizeof(struct tws_startup_ptr)));
-		/* the allocation size and file size should be the same */
-		/* just in case their not, make sure we dealloc correctly  */
-		startup->tws_hash_size = cache_size;
-
-		tws->startup_cache = startup;
-		tws_unlock(tws);
-		return KERN_SUCCESS;
-}
-
-
-void
-tws_hash_ws_flush(tws_hash_t tws) {
-	tws_startup_t 		scache;
-	if(tws == NULL) {
-		return;
-	}
-	tws_lock(tws);
-	if(tws->startup_name != NULL) {
-		scache = tws_create_startup_list(tws);
-		if(scache == NULL) {
-			/* dump the name cache, we'll */
-			/* get it next time */
-			kfree(tws->startup_name, tws->startup_name_length);
-			tws->startup_name = NULL;
-			tws_unlock(tws);
-			return;
-		}
-		bsd_write_page_cache_file(tws->uid, tws->startup_name, 
-				(caddr_t) scache, scache->tws_hash_size, 
-				tws->mod, tws->fid);
-		kfree(scache, scache->tws_hash_size);
-		kfree(tws->startup_name, tws->startup_name_length);
-		tws->startup_name = NULL;
-	}
-	tws_unlock(tws);
-	return;
-}
-
-void
-tws_hash_destroy(tws_hash_t	tws)
-{
-	unsigned int	i,k;
-
-	if(tws->startup_cache != NULL) {
-		kmem_free(kernel_map,
-			(vm_offset_t)tws->startup_cache, 
-			tws->startup_cache->tws_hash_size);
-		tws->startup_cache = NULL;
-	}
-	if(tws->startup_name != NULL) {
-		tws_internal_startup_send(tws);
-	}
-	for (i=0; i<tws->number_of_lines; i++) {
-		for(k=0; k<tws->expansion_count; k++) {
-			/* clear the object refs */
-			tws_hash_line_clear(tws, &(tws->cache[k][i]), NULL, FALSE);
-		}
-	}
-	i = 0;
-	while (i < tws->expansion_count) {
-		
-		kfree(tws->table[i],
-		      sizeof(tws_hash_ptr_t) 
-		      * tws->number_of_lines 
-		      * tws->number_of_elements);
-		kfree(tws->table_ele[i], 
-		      sizeof(struct tws_hash_ptr) 
-		      * tws->number_of_lines 
-		      * tws->number_of_elements);
-		kfree(tws->alt_ele[i], 
-		      sizeof(struct tws_hash_ptr) 
-		      * tws->number_of_lines 
-		      * tws->number_of_elements);
-		kfree(tws->cache[i],
-		      sizeof(struct tws_hash_line) * tws->number_of_lines);
-		i++;
-	}
-	if(tws->startup_name != NULL) {
-		kfree(tws->startup_name, tws->startup_name_length);
-	}
-	kfree(tws, sizeof(struct tws_hash));
-}
-
-void
-tws_hash_clear(tws_hash_t	tws)
-{
-	unsigned int	i, k;
-
-	for (i=0; i<tws->number_of_lines; i++) {
-		for(k=0; k<tws->expansion_count; k++) {
-			/* clear the object refs */
-			tws_hash_line_clear(tws, &(tws->cache[k][i]), NULL, FALSE);
-		}
-	}
-}
-
-kern_return_t
-task_working_set_create(
-	task_t	task, 
-	unsigned int lines, 
-	unsigned int rows,
-	unsigned int style)
-{
-
-	if (lines == 0) {
-		lines = TWS_HASH_LINE_COUNT;
-	}
-	if (rows == 0) {
-		rows = TWS_ARRAY_SIZE;
-	}
-	if (style == TWS_HASH_STYLE_DEFAULT) {
-		style = TWS_HASH_STYLE_BASIC;
-	}
-	task_lock(task);
-	if(task->dynamic_working_set != 0) {
-		task_unlock(task);
-		return(KERN_FAILURE);
-	} else if((task->dynamic_working_set =
-		tws_hash_create(lines, rows, style)) == 0) {
-		task_unlock(task);
-		return(KERN_NO_SPACE);
-	}
-	task_unlock(task);
-	return KERN_SUCCESS;
-}
-
-
-/* Internal use only routines */
-
-
-/* 
- * internal sub-function for address space lookup
- * returns the target element and the address of the
- * previous pointer  The previous pointer is the address 
- * of the pointer pointing to the target element.
- * TWS must be locked
- */
-
-void
-tws_traverse_address_hash_list (
-	tws_hash_t 		tws, 
-	unsigned int		index,
-	vm_offset_t		page_addr,
-	vm_object_t		object,
-	vm_object_offset_t	offset,
-	vm_map_t		map,
-	tws_hash_ptr_t		*target_ele,
-	tws_hash_ptr_t		**previous_ptr,
-	tws_hash_ptr_t		**free_list,
-	unsigned int		exclusive_addr)
-{
-	unsigned int	k;
-	tws_hash_ptr_t	cache_ele;
-	tws_hash_ptr_t	base_ele;
-
-	*target_ele = NULL;
-	*previous_ptr = NULL;
-
-	for(k=0; k<tws->expansion_count; k++) {
-		tws_hash_ele_t ele;
-		cache_ele = tws->table[k][index];
-		base_ele = cache_ele;
-		*previous_ptr = (tws_hash_ptr_t *)&(tws->table[k][index]);
-		while(cache_ele != NULL) {
-			if(((unsigned int)
-				cache_ele->element & TWS_ADDR_HASH) == 0) {
-				*previous_ptr = (tws_hash_ptr_t *)&(cache_ele->next);
-				cache_ele = cache_ele->next;
-				continue;
-			}
-			ele = (tws_hash_ele_t)((unsigned int)
-				cache_ele->element & ~TWS_ADDR_HASH);
-			if ((ele == 0) || (ele->object == 0)) {
-				/* A little clean-up of empty elements */
-				cache_ele->element = 0;
-				if(base_ele == cache_ele) {
-					base_ele = cache_ele->next;
-					tws->table[k][index] = cache_ele->next;
-					cache_ele->next = tws->free_hash_ele[k];
-					tws->free_hash_ele[k] = cache_ele;
-					cache_ele = base_ele;
-				} else {
-					**previous_ptr = cache_ele->next;
-					cache_ele->next = tws->free_hash_ele[k];
-					tws->free_hash_ele[k] = cache_ele;
-					cache_ele = **previous_ptr;
-				}
-				continue;
-			}
-	
-			if ((ele->page_addr <= page_addr) 
-				&& (page_addr <= (ele->page_addr + 
-				(vm_offset_t)TWS_INDEX_MASK))
-				&& ((object == NULL) 
-					|| ((object == ele->object)
-					&& (offset == ele->offset)
-					&& (map == ele->map)))) {
-				if(exclusive_addr) {
-					int delta;
-					delta = ((page_addr - ele->page_addr)
-					 >> 12);
-					if((1 << delta) & ele->page_cache) {
-						/* We've found a match */
-						*target_ele = cache_ele;
-						*free_list = 
-						   (tws_hash_ptr_t *)
-						   &(tws->free_hash_ele[k]);
-						return;
-					}
-				} else {
-					/* We've found a match */
-					*target_ele = cache_ele;
-					*free_list = (tws_hash_ptr_t *)
-						&(tws->free_hash_ele[k]);
-					return;
-				}
-			}
-			*previous_ptr = (tws_hash_ptr_t *)&(cache_ele->next);
-			cache_ele = cache_ele->next;
-		}
-	}
-}
-
-
-/* 
- * internal sub-function for object space lookup
- * returns the target element and the address of the
- * previous pointer  The previous pointer is the address 
- * of the pointer pointing to the target element.
- * TWS must be locked
- */
-
-
-void
-tws_traverse_object_hash_list (
-	tws_hash_t 		tws, 
-	unsigned int		index,
-	vm_object_t		object,
-	vm_object_offset_t	offset,
-	unsigned int		pagemask,
-	tws_hash_ptr_t		*target_ele,
-	tws_hash_ptr_t		**previous_ptr,
-	tws_hash_ptr_t		**free_list)
-{
-	unsigned int	k;
-	tws_hash_ptr_t	cache_ele;
-	tws_hash_ptr_t	base_ele;
-
-	*target_ele = NULL;
-	*previous_ptr = NULL;
-
-	for(k=0; k<tws->expansion_count; k++) {
-		cache_ele = tws->table[k][index];
-		base_ele = cache_ele;
-		*previous_ptr = &(tws->table[k][index]);
-		while(cache_ele != NULL) {
-			if((((unsigned int)cache_ele->element) 
-						& TWS_ADDR_HASH) != 0) {
-				*previous_ptr = &(cache_ele->next);
-				cache_ele = cache_ele->next;
-				continue;
-			}
-			if ((cache_ele->element == 0) || 
-				(cache_ele->element->object == 0)) {
-				/* A little clean-up of empty elements */
-				cache_ele->element = 0;
-				if(base_ele == cache_ele) {
-					base_ele = cache_ele->next;
-					tws->table[k][index] = cache_ele->next;
-					cache_ele->next = tws->free_hash_ele[k];
-					tws->free_hash_ele[k] = cache_ele;
-					cache_ele = tws->table[k][index];
-				} else {
-					**previous_ptr = cache_ele->next;
-					cache_ele->next = tws->free_hash_ele[k];
-					tws->free_hash_ele[k] = cache_ele;
-					cache_ele = **previous_ptr;
-				}
-				continue;
-			}
-			if ((cache_ele->element->object == object)
-				&& (cache_ele->element->offset == 
-				(offset - (offset & ~TWS_HASH_OFF_MASK)))) {
-				if((cache_ele->element->page_cache & pagemask) 
-					|| (pagemask == 0xFFFFFFFF)) {
-					/* We've found a match */
-					*target_ele = cache_ele;
-					*free_list = &(tws->free_hash_ele[k]);
-					return;
-				}
-			}
-			*previous_ptr = (tws_hash_ptr_t *)&(cache_ele->next);
-			cache_ele = cache_ele->next;
-		}
-	}
-}
-
-
-/*
- * For a given object/offset, discover whether the indexed 32 page frame 
- * containing the object/offset exists and if their are at least threshold
- * pages present.  Returns true if population meets threshold.
- */
-int 
-tws_test_for_community(
-		tws_hash_t		tws, 
-		vm_object_t		object, 
-		vm_object_offset_t	offset, 
-		unsigned int		threshold, 
-		unsigned int		*pagemask) 
-{
-	int	index;
-	tws_hash_ptr_t	cache_ele;
-	tws_hash_ptr_t	*trailer;
-	tws_hash_ptr_t	*free_list;
-	int		community = 0;
-
-	index = do_tws_hash(object, offset, 
-			tws->number_of_elements, tws->number_of_lines);
-	tws_traverse_object_hash_list(tws, index, object, offset, 0xFFFFFFFF,
-		&cache_ele, &trailer, &free_list);
-
-	if(cache_ele != NULL) {
-		int i;
-		unsigned int ctr;
-		ctr = 0;
-		for(i=1; i!=0; i=i<<1) {
-			if(i & cache_ele->element->page_cache)
-				ctr++;
-			if(ctr == threshold) {
-				community = 1;
-				*pagemask = cache_ele->element->page_cache;
-				break;
-			}
-		}
-	}
-
-	return community;
-	
-}
-
-
-/* 
- * Gets new hash element for object hash from free pools
- * TWS must be locked
- */
-
-tws_hash_ptr_t
-new_obj_hash(
-	tws_hash_t tws, 
-	unsigned int set, 
-	unsigned int index)
-{
-	tws_hash_ptr_t element;
-
-	if(tws->obj_free_count[set] < tws->number_of_lines * tws->number_of_elements) {
-		element = &(tws->table_ele[set][tws->obj_free_count[set]]);
-		tws->obj_free_count[set]+=1;
-	} else if(tws->free_hash_ele[set] == NULL) {
-		return NULL;
-	} else {
-		element = tws->free_hash_ele[set];
-		if(element == NULL)
-			return element;
-		tws->free_hash_ele[set] = tws->free_hash_ele[set]->next;
-	}
-	element->element = 0;
-	element->next = tws->table[set][index];
-	tws->table[set][index] = element;
-	return element;
-}
-
-/* 
- * Gets new hash element for addr hash from free pools
- * TWS must be locked
- */
-
-tws_hash_ptr_t
-new_addr_hash(
-	tws_hash_t tws, 
-	unsigned int set, 
-	unsigned int index)
-{
-	tws_hash_ptr_t element;
-
-	if(tws->addr_free_count[set] 
-			< tws->number_of_lines * tws->number_of_elements) {
-		element = &(tws->alt_ele[set][tws->addr_free_count[set]]);
-		tws->addr_free_count[set]+=1;
-	} else if(tws->free_hash_ele[set] == NULL) {
-		return NULL;
-	} else {
-		element = tws->free_hash_ele[set];
-		if(element == NULL)
-			return element;
-		tws->free_hash_ele[set] = tws->free_hash_ele[set]->next;
-	}
-	element->element = (tws_hash_ele_t)TWS_ADDR_HASH;
-	element->next = tws->table[set][index];
-	tws->table[set][index] = element;
-	return element;
-}
diff --git a/osfmk/vm/task_working_set.h b/osfmk/vm/task_working_set.h
deleted file mode 100644
index 366c5ffce..000000000
--- a/osfmk/vm/task_working_set.h
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- */
-
-/*
- *	File:	vm/task_working_set.h
- *	Author:	Chris Youngworth
- *	Date:	2001
- *
- *	Working set detection and maintainence module
- *
- */
-
-#ifndef	_VM_TASK_WORKING_SET_H_
-#define _VM_TASK_WORKING_SET_H_
-
-#include <mach/mach_types.h>
-
-#ifdef	KERNEL_PRIVATE
-
-#ifdef	MACH_KERNEL_PRIVATE
-
-#include <kern/queue.h>
-#include <vm/vm_object.h>
-
-/* task working set */
-
-#define tws_lock(tws)		mutex_lock(&(tws)->lock)
-#define tws_lock_try(tws)	mutex_try(&(tws)->lock)
-#define tws_unlock(tws)		mutex_unlock(&(tws)->lock)
-
-
-#define	TWS_ARRAY_SIZE	8
-#define TWS_HASH_LINE_COUNT 32
-/* start out size to allow monitoring of working set without excessive use */
-/* of wired memory resource. */
-#define TWS_SMALL_HASH_LINE_COUNT 4 
-
-/*
- * do not think of changing this hash unless you understand the implications
- * for the hash element page_cache field 
- */
-#define do_tws_hash(object,offset, rows, lines) \
-		((((((natural_t)(object)) +  \
-			(((natural_t)(object)) >> 6) +  \
-			(((natural_t)(object)) >> 12) +  \
-			(((natural_t)(object)) >> 18) +  \
-			(((natural_t)(object)) >> 24)) << 5) +  \
-			((natural_t)(((vm_object_offset_t)(offset)) >> 17))) & \
-			((rows * lines) -1))
-
-
-#define alt_tws_hash(addr, rows, lines) \
-		((((natural_t)(addr)) >> 17) & \
-		((rows * lines) -1))
-
-
-/* Long term startup data structures for initial cache filling */
-
-#define	TWS_STARTUP_MAX_HASH_RETRY	3
-
-/* 87 is the wrap skew, its based on  RETRY times the RETRY offset of 29 */
-/*
-#define do_startup_hash(addr, hash_size) \
-		((((((natural_t)(addr)) >> 17) & \
-		((2 * (hash_size)) -1)) + \
-		(87 * (((addr) & TWS_ADDR_OFF_MASK)/(2 * (hash_size))))) & \
-		((2 * (hash_size)) -1))
-*/
-#define do_startup_hash(addr, hash_size) \
-		(((((natural_t)(addr)) >> 17) * 3) & \
-		(hash_size -1))
-
-
-
-struct tws_startup_ele {
-	unsigned int		page_cache;
-	vm_offset_t		page_addr;
-};
-
-typedef struct tws_startup_ele *tws_startup_ele_t;
-
-
-struct tws_startup_ptr {
-	tws_startup_ele_t	element;
-	struct tws_startup_ptr	*next;
-};
-
-typedef struct tws_startup_ptr	*tws_startup_ptr_t;
-
-struct tws_startup {
-	unsigned int	tws_hash_size;	/* total size of struct in bytes */
-	unsigned int	ele_count;
-	unsigned int	array_size;	/* lines * rows * expansion_count */
-	unsigned int	hash_count;
-	
-	tws_startup_ptr_t	*table; /* hash table */
-	struct tws_startup_ptr	*ele;   /* hash elements */
-	struct	tws_startup_ele	*array;
-};
-
-typedef struct tws_startup	*tws_startup_t;
-
-
-/* Dynamic cache data structures for working set */
-
-struct tws_hash_ele {
-	vm_object_t		object;
-	vm_object_offset_t	offset;
-	unsigned int		page_cache;
-	vm_offset_t		page_addr;
-	int			line;
-	vm_map_t		map;
-};
-typedef struct tws_hash_ele *tws_hash_ele_t;
-
-#define TWS_HASH_OFF_MASK ((vm_object_offset_t)0xFFFFFFFFFFFE0000ULL)
-#define TWS_ADDR_OFF_MASK ((vm_offset_t)0xFFFE0000)
-#define TWS_INDEX_MASK ((vm_object_offset_t)0x000000000001F000ULL)
-
-struct tws_hash_ptr {
-	tws_hash_ele_t		element;
-	struct tws_hash_ptr	*next;
-};
-typedef struct tws_hash_ptr *tws_hash_ptr_t;
-
-struct tws_hash_line {
-	unsigned int		ele_count;
-	struct tws_hash_ele	list[TWS_ARRAY_SIZE];
-};
-typedef struct tws_hash_line *tws_hash_line_t;
-
-#define TWS_HASH_STYLE_DEFAULT	0x0
-#define TWS_HASH_STYLE_BASIC	0x1
-#define TWS_HASH_STYLE_SIGNAL	0x2
-
-
-#define TWS_ADDR_HASH 1
-#define TWS_HASH_EXPANSION_MAX	10
-#define TWS_MAX_REHASH 3
-
-
-struct tws_hash {
-	decl_mutex_data(,lock)          /* tws_hash's lock */
-	int		style;
-
-	unsigned int	current_line;
-	unsigned int	pageout_count;
-	unsigned int	line_count;
-
-	unsigned int	number_of_lines;
-	unsigned int	number_of_elements;
-	unsigned int	expansion_count;
-	unsigned int	time_of_creation;
-
-	unsigned int	lookup_count;
-	unsigned int	insert_count;
-
-	tws_startup_t	startup_cache;
-	char		*startup_name;
-	int		startup_name_length;
-	unsigned int	uid;
-	int		mod;
-	int		fid;
-
-	unsigned int    obj_free_count[TWS_HASH_EXPANSION_MAX];
-	unsigned int    addr_free_count[TWS_HASH_EXPANSION_MAX];
-	tws_hash_ptr_t	free_hash_ele[TWS_HASH_EXPANSION_MAX];
-	tws_hash_ptr_t	*table[TWS_HASH_EXPANSION_MAX];
-	tws_hash_ptr_t	table_ele[TWS_HASH_EXPANSION_MAX];
-	tws_hash_ptr_t	alt_ele[TWS_HASH_EXPANSION_MAX];
-	struct tws_hash_line	*cache[TWS_HASH_EXPANSION_MAX];
-}; 
-
-typedef struct tws_hash *tws_hash_t;
-
-
-extern kern_return_t tws_lookup(
-			tws_hash_t		tws,	
-			vm_object_offset_t	offset, 
-			vm_object_t		object,
-			tws_hash_line_t		 *line);
-
-extern kern_return_t tws_insert(
-			tws_hash_t		tws, 
-			vm_object_offset_t	offset,
-			vm_object_t		object,
-			vm_offset_t		page_addr,
-			vm_map_t		map);
-
-extern void tws_build_cluster(
-			tws_hash_t		tws,
-			vm_object_t		object,
-			vm_object_offset_t	*start,
-			vm_object_offset_t	*end,
-			vm_size_t		max_length);
-
-extern void tws_line_signal(
-			tws_hash_t		tws,
-			vm_map_t		map,
-			tws_hash_line_t 	hash_line,
-			vm_offset_t		target_page);
-
-extern void tws_hash_destroy(
-			tws_hash_t		tws);
-
-extern void tws_hash_ws_flush(
-			tws_hash_t		tws);
-
-extern kern_return_t tws_expand_working_set(
-			tws_hash_t		old_tws,
-			unsigned int		line_count,
-			boolean_t		dump_data);
-
-extern kern_return_t task_working_set_create(                
-			task_t  		task,
-			unsigned int 		lines,
-			unsigned int 		rows,
-			unsigned int 		style);
-
-#endif	/* MACH_KERNEL_PRIVATE */
-
-extern kern_return_t tws_handle_startup_file(
-			task_t			task,
-			unsigned int		uid,
-			char			*app_name,
-			void			*app_vp,
-			boolean_t		*new_info);
-
-extern kern_return_t tws_send_startup_info(
-			task_t			task);
-
-#endif	/* KERNEL_PRIVATE */
-
-#endif  /* _VM_TASK_WORKING_SET_H_ */
diff --git a/osfmk/vm/vm_apple_protect.c b/osfmk/vm/vm_apple_protect.c
index 8fcbaab43..5045c1b15 100644
--- a/osfmk/vm/vm_apple_protect.c
+++ b/osfmk/vm/vm_apple_protect.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <sys/errno.h>
@@ -46,6 +52,7 @@
 #include <default_pager/default_pager_types.h>
 #include <default_pager/default_pager_object_server.h>
 
+#include <vm/vm_fault.h>
 #include <vm/vm_map.h>
 #include <vm/vm_pageout.h>
 #include <vm/memory_object.h>
@@ -82,7 +89,8 @@ kern_return_t apple_protect_pager_terminate(memory_object_t mem_obj);
 kern_return_t apple_protect_pager_data_request(memory_object_t mem_obj,
 					       memory_object_offset_t offset,
 					       vm_size_t length,
-					       vm_prot_t protection_required);
+					       vm_prot_t protection_required,
+					       memory_object_fault_info_t fault_info);
 kern_return_t apple_protect_pager_data_return(memory_object_t mem_obj,
 					      memory_object_offset_t offset,
 					      vm_size_t	data_cnt,
@@ -299,89 +307,40 @@ apple_protect_pager_data_request(
 #if !DEBUG
 	__unused
 #endif
-	vm_prot_t		protection_required)
+	vm_prot_t		protection_required,
+	memory_object_fault_info_t mo_fault_info)
 {
 	apple_protect_pager_t	pager;
 	memory_object_control_t	mo_control;
-	upl_t			upl = NULL;
+	upl_t			upl;
 	int			upl_flags;
 	upl_size_t		upl_size;
 	upl_page_info_t		*upl_pl;
 	vm_object_t		src_object, dst_object;
 	kern_return_t		kr, retval;
-	vm_map_offset_t		src_mapping = 0, dst_mapping = 0;
+	vm_map_offset_t		kernel_mapping;
 	vm_offset_t		src_vaddr, dst_vaddr;
 	vm_offset_t		cur_offset;
-	boolean_t		src_map_page_by_page;
 	vm_map_entry_t		map_entry;
+	kern_return_t		error_code;
+	vm_prot_t		prot;
+	vm_page_t		src_page, top_page;
+	int			interruptible;
+	vm_object_fault_info_t	fault_info;
+
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %p, %llx, %x, %x\n", mem_obj, offset, length, protection_required));
 
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_data_request: %x, %llx, %llxx, %x\n", mem_obj, offset, length, protection_required));
+	src_object = VM_OBJECT_NULL;
+	kernel_mapping = 0;
+	upl = NULL;
+	fault_info = (vm_object_fault_info_t) mo_fault_info;
+	interruptible = fault_info->interruptible;
 
 	pager = apple_protect_pager_lookup(mem_obj);
 	assert(pager->is_ready);
 	assert(pager->ref_count > 1); /* pager is alive and mapped */
 
-	PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %x, %llx, %llx, %x, pager %x\n", mem_obj, offset, length, protection_required, pager));
-
-	/*
-	 * Map the encrypted data in the kernel address space from the 
-	 * backing VM object (itself backed by the encrypted file via
-	 * the vnode pager).
-	 */
-	src_object = pager->backing_object;
-	assert(src_object != VM_OBJECT_NULL);
-	vm_object_reference(src_object); /* ref. for the mapping */
-	src_mapping = 0;
-	kr = vm_map_enter(kernel_map,
-			  &src_mapping,
-			  length,
-			  0,
-			  VM_FLAGS_ANYWHERE,
-			  src_object,
-			  offset,
-			  FALSE,
-			  VM_PROT_READ,
-			  VM_PROT_READ,
-			  VM_INHERIT_NONE);
-	switch (kr) {
-	case KERN_SUCCESS:
-		/* wire the memory to make sure it is available */
-		kr = vm_map_wire(kernel_map,
-				 src_mapping,
-				 src_mapping + length,
-				 VM_PROT_READ,
-				 FALSE);
-		if (kr != KERN_SUCCESS) {
-			/*
-			 * Wiring failed, so unmap source and fall back
-			 * to page by page mapping of the source.
-			 */
-			kr = vm_map_remove(kernel_map,
-					   src_mapping,
-					   src_mapping + length,
-					   VM_MAP_NO_FLAGS);
-			assert(kr == KERN_SUCCESS);
-			src_mapping = 0;
-			src_vaddr = 0;
-			src_map_page_by_page = TRUE;
-			break;
-		}
-		/* source region is now fully mapped and wired */
-		src_map_page_by_page = FALSE;
-		src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
-		break;
-	case KERN_NO_SPACE:
-		/* we couldn't map the entire source, so map it page by page */
-		src_map_page_by_page = TRUE;
-		/* release the reference for the failed mapping */
-		vm_object_deallocate(src_object);
-		break;
-	default:
-		vm_object_deallocate(src_object);
-		retval = kr;
-		goto done;
-	}
-
+	PAGER_DEBUG(PAGER_PAGEIN, ("apple_protect_pager_data_request: %p, %llx, %x, %x, pager %p\n", mem_obj, offset, length, protection_required, pager));
 
 	/*
 	 * Gather in a UPL all the VM pages requested by VM.
@@ -402,18 +361,19 @@ apple_protect_pager_data_request(
 		retval = kr;
 		goto done;
 	}
+	dst_object = mo_control->moc_object;
+	assert(dst_object != VM_OBJECT_NULL);
+
 
 	/*
-	 * Reserve a virtual page in the kernel address space to map each
-	 * destination physical page when it's its turn to be filled.
+	 * Reserve 2 virtual pages in the kernel address space to map each
+	 * source and destination physical pages when it's their turn to
+	 * be processed.
 	 */
-	dst_object = mo_control->moc_object;
-	assert(dst_object != VM_OBJECT_NULL);
-	dst_mapping = 0;
 	vm_object_reference(kernel_object);	/* ref. for mapping */
 	kr = vm_map_find_space(kernel_map,
-			       &dst_mapping,
-			       PAGE_SIZE_64,
+			       &kernel_mapping,
+			       2 * PAGE_SIZE_64,
 			       0,
 			       0,
 			       &map_entry);
@@ -423,9 +383,19 @@ apple_protect_pager_data_request(
 		goto done;
 	}
 	map_entry->object.vm_object = kernel_object;
-	map_entry->offset = dst_mapping - VM_MIN_KERNEL_ADDRESS;
+	map_entry->offset = kernel_mapping - VM_MIN_KERNEL_ADDRESS;
 	vm_map_unlock(kernel_map);
-	dst_vaddr = CAST_DOWN(vm_offset_t, dst_mapping);
+	src_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping);
+	dst_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping + PAGE_SIZE_64);
+
+	/*
+	 * We'll map the encrypted data in the kernel address space from the 
+	 * backing VM object (itself backed by the encrypted file via
+	 * the vnode pager).
+	 */
+	src_object = pager->backing_object;
+	assert(src_object != VM_OBJECT_NULL);
+	vm_object_reference(src_object); /* to keep the source object alive */
 
 	/*
 	 * Fill in the contents of the pages requested by VM.
@@ -442,52 +412,63 @@ apple_protect_pager_data_request(
 		/*
 		 * Map the source (encrypted) page in the kernel's
 		 * virtual address space.
+		 * We already hold a reference on the src_object.
 		 */
-		if (src_map_page_by_page) {
-			vm_object_reference(src_object); /* ref. for mapping */
-			kr = vm_map_enter(kernel_map,
-					  &src_mapping,
-					  PAGE_SIZE_64,
-					  0,
-					  VM_FLAGS_ANYWHERE,
-					  src_object,
-					  offset + cur_offset,
-					  FALSE,
-					  VM_PROT_READ,
-					  VM_PROT_READ,
-					  VM_INHERIT_NONE);
-			if (kr != KERN_SUCCESS) {
-				vm_object_deallocate(src_object);
-				retval = kr;
-				goto done;
+	retry_src_fault:
+		vm_object_lock(src_object);
+		vm_object_paging_begin(src_object);
+		error_code = 0;
+		prot = VM_PROT_READ;
+		kr = vm_fault_page(src_object,
+				   offset + cur_offset,
+				   VM_PROT_READ,
+				   FALSE,
+				   &prot,
+				   &src_page,
+				   &top_page,
+				   NULL,
+				   &error_code,
+				   FALSE,
+				   FALSE,
+				   fault_info);
+		switch (kr) {
+		case VM_FAULT_SUCCESS:
+			break;
+		case VM_FAULT_RETRY:
+			goto retry_src_fault;
+		case VM_FAULT_MEMORY_SHORTAGE:
+			if (vm_page_wait(interruptible)) {
+				goto retry_src_fault;
 			}
-			kr = vm_map_wire(kernel_map,
-					 src_mapping,
-					 src_mapping + PAGE_SIZE_64,
-					 VM_PROT_READ,
-					 FALSE);
-			if (kr != KERN_SUCCESS) {
-				retval = kr;
-				kr = vm_map_remove(kernel_map,
-						   src_mapping,
-						   src_mapping + PAGE_SIZE_64,
-						   VM_MAP_NO_FLAGS);
-				assert(kr == KERN_SUCCESS);
-				src_mapping = 0;
-				src_vaddr = 0;
-				printf("apple_protect_pager_data_request: "
-				       "failed to resolve page fault for src "
-				       "object %p offset 0x%llx "
-				       "preempt %d error 0x%x\n",
-				       src_object, offset + cur_offset,
-				       get_preemption_level(), retval);
-				goto done;
+			/* fall thru */
+		case VM_FAULT_INTERRUPTED:
+			retval = MACH_SEND_INTERRUPTED;
+			goto done;
+		case VM_FAULT_MEMORY_ERROR:
+			/* the page is not there ! */
+			if (error_code) {
+				retval = error_code;
+			} else {
+				retval = KERN_MEMORY_ERROR;
 			}
-			src_vaddr = CAST_DOWN(vm_offset_t, src_mapping);
-		} else {
-			src_vaddr = src_mapping + cur_offset;
+			goto done;
+		default:
+			retval = KERN_FAILURE;
+			goto done;
 		}
-
+		assert(src_page != VM_PAGE_NULL);
+		assert(src_page->busy);
+		
+		/*
+		 * Establish an explicit mapping of the source
+		 * physical page.
+		 */
+		pmap_enter(kernel_pmap,
+			   kernel_mapping,
+			   src_page->phys_page,
+			   VM_PROT_READ,
+			   src_object->wimg_bits & VM_WIMG_MASK,
+			   TRUE);
 		/*
 		 * Establish an explicit pmap mapping of the destination
 		 * physical page.
@@ -497,10 +478,12 @@ apple_protect_pager_data_request(
 		dst_pnum = (addr64_t)
 			upl_phys_page(upl_pl, cur_offset / PAGE_SIZE);
 		assert(dst_pnum != 0);
-		pmap_enter(kernel_pmap, dst_mapping, dst_pnum,
+		pmap_enter(kernel_pmap,
+			   kernel_mapping + PAGE_SIZE_64,
+			   dst_pnum,
 			   VM_PROT_READ | VM_PROT_WRITE,
 			   dst_object->wimg_bits & VM_WIMG_MASK,
-			   FALSE);
+			   TRUE);
 
 		/*
 		 * Decrypt the encrypted contents of the source page
@@ -510,41 +493,32 @@ apple_protect_pager_data_request(
 				     (void *) dst_vaddr);
 
 		/*
-		 * Remove the pmap mapping of the destination page
+		 * Remove the pmap mapping of the source and destination pages
 		 * in the kernel.
 		 */
 		pmap_remove(kernel_pmap,
-			    (addr64_t) dst_mapping,
-			    (addr64_t) (dst_mapping + PAGE_SIZE_64));
-
-		if (src_map_page_by_page) {
-			/*
-			 * Remove the wired kernel mapping of the source page.
-			 * This releases the extra reference we took on
-			 * src_object.
-			 */
-			kr = vm_map_remove(kernel_map,
-					   src_mapping,
-					   src_mapping + PAGE_SIZE_64,
-					   VM_MAP_REMOVE_KUNWIRE);
-			assert(kr == KERN_SUCCESS);
-			src_mapping = 0;
-			src_vaddr = 0;
+			    (addr64_t) kernel_mapping,
+			    (addr64_t) (kernel_mapping + (2 * PAGE_SIZE_64)));
+
+		/*
+		 * Cleanup the result of vm_fault_page() of the source page.
+		 */
+		PAGE_WAKEUP_DONE(src_page);
+		vm_object_paging_end(src_page->object);
+		vm_object_unlock(src_page->object);
+		if (top_page != VM_PAGE_NULL) {
+			vm_object_t top_object;
+
+			top_object = top_page->object;
+			vm_object_lock(top_object);
+			VM_PAGE_FREE(top_page);
+			vm_object_paging_end(top_object);
+			vm_object_unlock(top_object);
 		}
 	}
 
 	retval = KERN_SUCCESS;
 done:
-	if (src_mapping != 0) {
-		/* remove the wired mapping of the source pages */
-		kr = vm_map_remove(kernel_map,
-				   src_mapping,
-				   src_mapping + length,
-				   VM_MAP_REMOVE_KUNWIRE);
-		assert(kr == KERN_SUCCESS);
-		src_mapping = 0;
-		src_vaddr = 0;
-	}
 	if (upl != NULL) {
 		/* clean up the UPL */
 
@@ -568,16 +542,20 @@ done:
 		upl_deallocate(upl);
 		upl = NULL;
 	}
-	if (dst_mapping != 0) {
-		/* clean up the mapping of the destination pages */
+	if (kernel_mapping != 0) {
+		/* clean up the mapping of the source and destination pages */
 		kr = vm_map_remove(kernel_map,
-				   dst_mapping,
-				   dst_mapping + PAGE_SIZE_64,
+				   kernel_mapping,
+				   kernel_mapping + (2 * PAGE_SIZE_64),
 				   VM_MAP_NO_FLAGS);
 		assert(kr == KERN_SUCCESS);
-		dst_mapping = 0;
+		kernel_mapping = 0;
+		src_vaddr = 0;
 		dst_vaddr = 0;
 	}
+	if (src_object != VM_OBJECT_NULL) {
+		vm_object_deallocate(src_object);
+	}
 
 	return retval;
 }
@@ -691,7 +669,7 @@ apple_protect_pager_deallocate_internal(
 	if (pager->ref_count == 1) {
 		/*
 		 * Only the "named" reference is left, which means that
-		 * no one is realy holding on to this pager anymore.
+		 * no one is really holding on to this pager anymore.
 		 * Terminate it.
 		 */
 		apple_protect_pager_dequeue(pager);
@@ -734,7 +712,7 @@ apple_protect_pager_deallocate(
 {
 	apple_protect_pager_t	pager;
 
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_deallocate: %p\n", mem_obj));
 	pager = apple_protect_pager_lookup(mem_obj);
 	apple_protect_pager_deallocate_internal(pager, FALSE);
 }
@@ -749,7 +727,7 @@ apple_protect_pager_terminate(
 #endif
 	memory_object_t	mem_obj)
 {
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_terminate: %p\n", mem_obj));
 
 	return KERN_SUCCESS;
 }
@@ -766,7 +744,7 @@ apple_protect_pager_synchronize(
 {
 	apple_protect_pager_t	pager;
 
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_synchronize: %p\n", mem_obj));
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
@@ -790,7 +768,7 @@ apple_protect_pager_map(
 {
 	apple_protect_pager_t	pager;
 
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_map: %p\n", mem_obj));
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
@@ -822,7 +800,7 @@ apple_protect_pager_unmap(
 	apple_protect_pager_t	pager;
 	int			count_unmapped;
 
-	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_unmap: %x\n", mem_obj));
+	PAGER_DEBUG(PAGER_ALL, ("apple_protect_pager_unmap: %p\n", mem_obj));
 
 	pager = apple_protect_pager_lookup(mem_obj);
 
diff --git a/osfmk/vm/vm_debug.c b/osfmk/vm/vm_debug.c
index d786d081a..702ce6a42 100644
--- a/osfmk/vm/vm_debug.c
+++ b/osfmk/vm/vm_debug.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -201,8 +207,6 @@ mach_vm_region_info(
 					cobject->ref_count;
 				vio->vio_resident_page_count =
 					cobject->resident_page_count;
-				vio->vio_absent_count =
-					cobject->absent_count;
 				vio->vio_copy =
 					(vm_offset_t) cobject->copy;
 				vio->vio_shadow =
@@ -232,10 +236,10 @@ mach_vm_region_info(
 				vio->vio_alive =
 					cobject->alive;
 				vio->vio_purgable =
-					(cobject->purgable != VM_OBJECT_NONPURGABLE);
+					(cobject->purgable != VM_PURGABLE_DENY);
 				vio->vio_purgable_volatile =
-					(cobject->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-					 cobject->purgable == VM_OBJECT_PURGABLE_EMPTY);
+					(cobject->purgable == VM_PURGABLE_VOLATILE ||
+					 cobject->purgable == VM_PURGABLE_EMPTY);
 			}
 
 			used++;
@@ -404,8 +408,6 @@ mach_vm_region_info_64(
 					cobject->ref_count;
 				vio->vio_resident_page_count =
 					cobject->resident_page_count;
-				vio->vio_absent_count =
-					cobject->absent_count;
 				vio->vio_copy =
 					(vm_offset_t) cobject->copy;
 				vio->vio_shadow =
@@ -435,10 +437,10 @@ mach_vm_region_info_64(
 				vio->vio_alive =
 					cobject->alive;
 				vio->vio_purgable =
-					(cobject->purgable != VM_OBJECT_NONPURGABLE);
+					(cobject->purgable != VM_PURGABLE_DENY);
 				vio->vio_purgable_volatile =
-					(cobject->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-					 cobject->purgable == VM_OBJECT_PURGABLE_EMPTY);
+					(cobject->purgable == VM_PURGABLE_VOLATILE ||
+					 cobject->purgable == VM_PURGABLE_EMPTY);
 			}
 
 			used++;
diff --git a/osfmk/vm/vm_debug.h b/osfmk/vm/vm_debug.h
index db8c2c8b2..b2dbd6fec 100644
--- a/osfmk/vm/vm_debug.h
+++ b/osfmk/vm/vm_debug.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/vm/vm_external.c b/osfmk/vm/vm_external.c
index 5afba3ca0..85712c9a6 100644
--- a/osfmk/vm/vm_external.c
+++ b/osfmk/vm/vm_external.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/vm/vm_external.h b/osfmk/vm/vm_external.h
index e2354d963..7f46331ae 100644
--- a/osfmk/vm/vm_external.h
+++ b/osfmk/vm/vm_external.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/osfmk/vm/vm_fault.c b/osfmk/vm/vm_fault.c
index ef81f0886..87ffc3ee7 100644
--- a/osfmk/vm/vm_fault.c
+++ b/osfmk/vm/vm_fault.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -59,6 +65,7 @@
 #include <mach_cluster_stats.h>
 #include <mach_pagemap.h>
 #include <mach_kdb.h>
+#include <libkern/OSAtomic.h>
 
 #include <mach/mach_types.h>
 #include <mach/kern_return.h>
@@ -67,6 +74,7 @@
 #include <mach/vm_behavior.h>
 #include <mach/memory_object.h>
 				/* For memory_object_data_{request,unlock} */
+#include <mach/sdt.h>
 
 #include <kern/kern_types.h>
 #include <kern/host_statistics.h>
@@ -84,7 +92,6 @@
 #include <ppc/proc_reg.h>
 
 #include <vm/vm_fault.h>
-#include <vm/task_working_set.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
@@ -92,23 +99,25 @@
 #include <vm/pmap.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>
+#include <vm/vm_external.h>
+#include <vm/memory_object.h>
+#include <vm/vm_purgeable_internal.h>	/* Needed by some vm_page.h macros */
 
 #include <sys/kdebug.h>
 
 #define VM_FAULT_CLASSIFY	0
-#define VM_FAULT_STATIC_CONFIG	1
 
-#define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */
+/* Zero-filled pages are marked "m->zero_fill" and put on the
+ * special zero-fill inactive queue  only if they belong to
+ * an object at least this big.
+ */
+#define	VM_ZF_OBJECT_SIZE_THRESHOLD	(0x200000)
 
-unsigned int	vm_object_absent_max = 50;
+#define TRACEFAULTPAGE 0 /* (TEST/DEBUG) */
 
-int		vm_fault_debug = 0;
+int	vm_object_pagein_throttle = 16;
 
-#if	!VM_FAULT_STATIC_CONFIG
-boolean_t	vm_fault_dirty_handling = FALSE;
-boolean_t	vm_fault_interruptible = FALSE;
-boolean_t	software_reference_bits = TRUE;
-#endif
+extern int cs_debug;
 
 #if	MACH_KDB
 extern struct db_watchpoint *db_watchpoint_list;
@@ -173,11 +182,12 @@ vm_fault_cleanup(
 	vm_object_unlock(object);
 
 	if (top_page != VM_PAGE_NULL) {
-	    object = top_page->object;
-	    vm_object_lock(object);
-	    VM_PAGE_FREE(top_page);
-	    vm_object_paging_end(object);
-	    vm_object_unlock(object);
+	        object = top_page->object;
+
+		vm_object_lock(object);
+		VM_PAGE_FREE(top_page);
+		vm_object_paging_end(object);
+		vm_object_unlock(object);
 	}
 }
 
@@ -199,29 +209,142 @@ struct {
 #define CLUSTER_STAT(clause)
 #endif	/* MACH_CLUSTER_STATS */
 
-/* XXX - temporary */
-boolean_t vm_allow_clustered_pagein = FALSE;
-int vm_pagein_cluster_used = 0;
-
 #define ALIGNED(x) (((x) & (PAGE_SIZE_64 - 1)) == 0)
 
 
 boolean_t	vm_page_deactivate_behind = TRUE;
 /* 
- * Prepage default sizes given VM_BEHAVIOR_DEFAULT reference behavior 
+ * default sizes given VM_BEHAVIOR_DEFAULT reference behavior 
  */
 int vm_default_ahead = 0;
 int vm_default_behind = MAX_UPL_TRANSFER;
 
+#define MAX_SEQUENTIAL_RUN	(1024 * 1024 * 1024)
+
+/*
+ * vm_page_is_sequential
+ *
+ * Determine if sequential access is in progress
+ * in accordance with the behavior specified.
+ * Update state to indicate current access pattern.
+ *
+ * object must have at least the shared lock held
+ */
+static
+void
+vm_fault_is_sequential(
+	vm_object_t		object,
+	vm_object_offset_t	offset,
+	vm_behavior_t		behavior)
+{
+        vm_object_offset_t	last_alloc;
+	int			sequential;
+	int			orig_sequential;
+
+        last_alloc = object->last_alloc;
+	sequential = object->sequential;
+	orig_sequential = sequential;
+
+	switch (behavior) {
+	case VM_BEHAVIOR_RANDOM:
+	        /*
+		 * reset indicator of sequential behavior
+		 */
+	        sequential = 0;
+	        break;
+
+	case VM_BEHAVIOR_SEQUENTIAL:
+	        if (offset && last_alloc == offset - PAGE_SIZE_64) {
+		        /*
+			 * advance indicator of sequential behavior
+			 */
+		        if (sequential < MAX_SEQUENTIAL_RUN)
+			        sequential += PAGE_SIZE;
+		} else {
+		        /*
+			 * reset indicator of sequential behavior
+			 */
+		        sequential = 0;
+		}
+	        break;
+
+	case VM_BEHAVIOR_RSEQNTL:
+	        if (last_alloc && last_alloc == offset + PAGE_SIZE_64) {
+		        /*
+			 * advance indicator of sequential behavior
+			 */
+		        if (sequential > -MAX_SEQUENTIAL_RUN)
+			        sequential -= PAGE_SIZE;
+		} else {
+		        /*
+			 * reset indicator of sequential behavior
+			 */
+		        sequential = 0;
+		}
+	        break;
+
+	case VM_BEHAVIOR_DEFAULT:
+	default:
+	        if (offset && last_alloc == (offset - PAGE_SIZE_64)) {
+		        /*
+			 * advance indicator of sequential behavior
+			 */
+		        if (sequential < 0)
+			        sequential = 0;
+		        if (sequential < MAX_SEQUENTIAL_RUN)
+			        sequential += PAGE_SIZE;
+
+		} else if (last_alloc && last_alloc == (offset + PAGE_SIZE_64)) {
+		        /*
+			 * advance indicator of sequential behavior
+			 */
+		        if (sequential > 0)
+			        sequential = 0;
+		        if (sequential > -MAX_SEQUENTIAL_RUN)
+			        sequential -= PAGE_SIZE;
+		} else {
+		        /*
+			 * reset indicator of sequential behavior
+			 */
+		        sequential = 0;
+		}
+	        break;
+	}
+	if (sequential != orig_sequential) {
+	        if (!OSCompareAndSwap(orig_sequential, sequential, (UInt32 *)&object->sequential)) {
+		        /*
+			 * if someone else has already updated object->sequential
+			 * don't bother trying to update it or object->last_alloc
+			 */
+		        return;
+		}
+	}
+	/*
+	 * I'd like to do this with a OSCompareAndSwap64, but that
+	 * doesn't exist for PPC...  however, it shouldn't matter
+	 * that much... last_alloc is maintained so that we can determine
+	 * if a sequential access pattern is taking place... if only
+	 * one thread is banging on this object, no problem with the unprotected
+	 * update... if 2 or more threads are banging away, we run the risk of
+	 * someone seeing a mangled update... however, in the face of multiple
+	 * accesses, no sequential access pattern can develop anyway, so we
+	 * haven't lost any real info.
+	 */
+	object->last_alloc = offset;
+}
+
+
 /*
- *	vm_page_deactivate_behind
+ * vm_page_deactivate_behind
+ *
+ * Determine if sequential access is in progress
+ * in accordance with the behavior specified.  If
+ * so, compute a potential page to deactivate and
+ * deactivate it.
  *
- *	Determine if sequential access is in progress
- *	in accordance with the behavior specified.  If
- *	so, compute a potential page to deactive and
- *	deactivate it.
+ * object must be locked.
  *
- *	The object must be locked.
+ * return TRUE if we actually deactivate a page
  */
 static
 boolean_t
@@ -230,89 +353,201 @@ vm_fault_deactivate_behind(
 	vm_object_offset_t	offset,
 	vm_behavior_t		behavior)
 {
-	vm_page_t m;
+	vm_page_t	m = NULL;
+	int		sequential_run;
+	int		sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
 
 #if TRACEFAULTPAGE
 	dbgTrace(0xBEEF0018, (unsigned int) object, (unsigned int) vm_fault_deactivate_behind);	/* (TEST/DEBUG) */
 #endif
 
-	if (object == kernel_object) {
+	if (object == kernel_object || vm_page_deactivate_behind == FALSE) {
 		/*
 		 * Do not deactivate pages from the kernel object: they
 		 * are not intended to become pageable.
+		 * or we've disabled the deactivate behind mechanism
 		 */
 		return FALSE;
 	}
-
+	if ((sequential_run = object->sequential)) {
+		  if (sequential_run < 0) {
+		          sequential_behavior = VM_BEHAVIOR_RSEQNTL;
+			  sequential_run = 0 - sequential_run;
+		  } else {
+		          sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
+		  }
+	}
 	switch (behavior) {
 	case VM_BEHAVIOR_RANDOM:
-		object->sequential = PAGE_SIZE_64;
-		m = VM_PAGE_NULL;
 		break;
 	case VM_BEHAVIOR_SEQUENTIAL:
-		if (offset &&
-			object->last_alloc == offset - PAGE_SIZE_64) {
-			object->sequential += PAGE_SIZE_64;
+	        if (sequential_run >= (int)PAGE_SIZE)
 			m = vm_page_lookup(object, offset - PAGE_SIZE_64);
-		} else {
-			object->sequential = PAGE_SIZE_64; /* reset */
-			m = VM_PAGE_NULL;
-		}
 		break;
 	case VM_BEHAVIOR_RSEQNTL:
-		if (object->last_alloc &&
-			object->last_alloc == offset + PAGE_SIZE_64) {
-			object->sequential += PAGE_SIZE_64;
+	        if (sequential_run >= (int)PAGE_SIZE)
 			m = vm_page_lookup(object, offset + PAGE_SIZE_64);
-		} else {
-			object->sequential = PAGE_SIZE_64; /* reset */
-			m = VM_PAGE_NULL;
-		}
 		break;
 	case VM_BEHAVIOR_DEFAULT:
 	default:
-		if (offset && 
-			object->last_alloc == offset - PAGE_SIZE_64) {
-			vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
-
-			object->sequential += PAGE_SIZE_64;
-			m = (offset >= behind &&
-				object->sequential >= behind) ?
-				vm_page_lookup(object, offset - behind) :
-				VM_PAGE_NULL;
-		} else if (object->last_alloc &&
-			object->last_alloc == offset + PAGE_SIZE_64) {
-			vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
-
-			object->sequential += PAGE_SIZE_64;
-			m = (offset < -behind &&
-				object->sequential >= behind) ?
-				vm_page_lookup(object, offset + behind) :
-				VM_PAGE_NULL;
-		} else {
-			object->sequential = PAGE_SIZE_64;
-			m = VM_PAGE_NULL;
+	{	vm_object_offset_t behind = vm_default_behind * PAGE_SIZE_64;
+
+	        /*
+		 * determine if the run of sequential accesss has been
+		 * long enough on an object with default access behavior
+		 * to consider it for deactivation
+		 */
+		if ((uint64_t)sequential_run >= behind) {
+		        if (sequential_behavior == VM_BEHAVIOR_SEQUENTIAL) {
+			        if (offset >= behind)
+				        m = vm_page_lookup(object, offset - behind);
+			} else {
+			        if (offset < -behind)
+				        m = vm_page_lookup(object, offset + behind);
+			}
 		}
 		break;
 	}
-
-	object->last_alloc = offset;
-
+	}
 	if (m) {
-		if (!m->busy) {
-			vm_page_lock_queues();
-			vm_page_deactivate(m);
-			vm_page_unlock_queues();
+		if (!m->busy && !m->no_cache && !m->throttled && !m->fictitious && !m->absent) {
+		        pmap_clear_reference(m->phys_page);
+			m->deactivated = TRUE;
 #if TRACEFAULTPAGE
 			dbgTrace(0xBEEF0019, (unsigned int) object, (unsigned int) m);	/* (TEST/DEBUG) */
 #endif
+			return TRUE;
 		}
-		return TRUE;
 	}
 	return FALSE;
 }
 
 
+/*
+ * check for various conditions that would
+ * prevent us from creating a ZF page...
+ * cleanup is based on being called from vm_fault_page
+ *
+ * object must be locked
+ * object == m->object
+ */
+static vm_fault_return_t
+vm_fault_check(vm_object_t object, vm_page_t m, vm_page_t first_m, boolean_t interruptible_state)
+{
+        if (object->shadow_severed) {
+	        /*
+		 * the shadow chain was severed
+		 * just have to return an error at this point
+		 */
+	        if (m != VM_PAGE_NULL)
+		        VM_PAGE_FREE(m);
+		vm_fault_cleanup(object, first_m);
+
+		thread_interrupt_level(interruptible_state);
+
+		return (VM_FAULT_MEMORY_ERROR);
+	}
+	if (vm_backing_store_low) {
+	        /*
+		 * are we protecting the system from
+		 * backing store exhaustion.  If so
+		 * sleep unless we are privileged.
+		 */
+	        if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
+
+			if (m != VM_PAGE_NULL)
+			        VM_PAGE_FREE(m);
+			vm_fault_cleanup(object, first_m);
+
+		        assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
+
+			thread_block(THREAD_CONTINUE_NULL);
+			thread_interrupt_level(interruptible_state);
+
+			return (VM_FAULT_RETRY);
+		}
+	}
+	if (VM_PAGE_ZFILL_THROTTLED()) {
+	        /*
+		 * we're throttling zero-fills...
+		 * treat this as if we couldn't grab a page
+		 */
+	        if (m != VM_PAGE_NULL)
+		        VM_PAGE_FREE(m);
+		vm_fault_cleanup(object, first_m);
+
+		thread_interrupt_level(interruptible_state);
+
+		return (VM_FAULT_MEMORY_SHORTAGE);
+	}
+	return (VM_FAULT_SUCCESS);
+}
+
+
+/*
+ * do the work to zero fill a page and
+ * inject it into the correct paging queue
+ *
+ * m->object must be locked
+ * page queue lock must NOT be held
+ */
+static int
+vm_fault_zero_page(vm_page_t m, boolean_t no_zero_fill)
+{
+        int my_fault = DBG_ZERO_FILL_FAULT;
+
+	/*
+	 * This is is a zero-fill page fault...
+	 *
+	 * Checking the page lock is a waste of
+	 * time;  this page was absent, so
+	 * it can't be page locked by a pager.
+	 *
+	 * we also consider it undefined
+	 * with respect to instruction
+	 * execution.  i.e. it is the responsibility
+	 * of higher layers to call for an instruction
+	 * sync after changing the contents and before
+	 * sending a program into this area.  We 
+	 * choose this approach for performance
+	 */
+	m->pmapped = TRUE;
+
+	m->cs_validated = FALSE;
+	m->cs_tainted = FALSE;
+
+	if (no_zero_fill == TRUE)
+	        my_fault = DBG_NZF_PAGE_FAULT;
+	else {
+		vm_page_zero_fill(m);
+
+		VM_STAT_INCR(zero_fill_count);
+		DTRACE_VM2(zfod, int, 1, (uint64_t *), NULL);
+	}
+	assert(!m->laundry);
+	assert(m->object != kernel_object);
+	//assert(m->pageq.next == NULL && m->pageq.prev == NULL);
+
+	if (!IP_VALID(memory_manager_default) &&
+		(m->object->purgable == VM_PURGABLE_DENY ||
+		 m->object->purgable == VM_PURGABLE_NONVOLATILE)) {
+		vm_page_lock_queues();
+
+                queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq);
+                m->throttled = TRUE;
+                vm_page_throttled_count++;
+
+		vm_page_unlock_queues();
+	} else {
+		if (m->object->size > VM_ZF_OBJECT_SIZE_THRESHOLD) {
+			m->zero_fill = TRUE;
+			OSAddAtomic(1, (SInt32 *)&vm_zf_count);
+		}
+	}
+	return (my_fault);
+}
+
+
 /*
  *	Routine:	vm_fault_page
  *	Purpose:
@@ -322,12 +557,10 @@ vm_fault_deactivate_behind(
  *	Additional arguments:
  *		The required permissions for the page is given
  *		in "fault_type".  Desired permissions are included
- *		in "protection".  The minimum and maximum valid offsets
- *		within the object for the relevant map entry are
- *		passed in "lo_offset" and "hi_offset" respectively and
- *		the expected page reference pattern is passed in "behavior".
- *		These three parameters are used to determine pagein cluster 
- *		limits.
+ *		in "protection".
+ *		fault_info is passed along to determine pagein cluster 
+ *		limits... it contains the expected reference pattern,
+ *		cluster size if available, etc...
  *
  *		If the desired page is known to be resident (for
  *		example, because it was previously wired down), asserting
@@ -365,10 +598,6 @@ vm_fault_page(
 	vm_object_offset_t first_offset,	/* Offset into object */
 	vm_prot_t	fault_type,	/* What access is requested */
 	boolean_t	must_be_resident,/* Must page be resident? */
-	int		interruptible,	/* how may fault be interrupted? */
-	vm_map_offset_t lo_offset,	/* Map entry start */
-	vm_map_offset_t hi_offset,	/* Map entry end */
-	vm_behavior_t	behavior,	/* Page reference behavior */
 	/* Modifies in place: */
 	vm_prot_t	*protection,	/* Protection for mapping */
 	/* Returns: */
@@ -380,17 +609,17 @@ vm_fault_page(
 	/* More arguments: */
 	kern_return_t	*error_code,	/* code if page is in error */
 	boolean_t	no_zero_fill,	/* don't zero fill absent pages */
+#if MACH_PAGEMAP
 	boolean_t	data_supply,	/* treat as data_supply if 
 					 * it is a write fault and a full
 					 * page is provided */
-	vm_map_t	map,
-	__unused vm_map_offset_t	vaddr)
+#else
+	__unused boolean_t data_supply,
+#endif
+	vm_object_fault_info_t fault_info)
 {
-	register
 	vm_page_t		m;
-	register
 	vm_object_t		object;
-	register
 	vm_object_offset_t	offset;
 	vm_page_t		first_m;
 	vm_object_t		next_object;
@@ -398,16 +627,16 @@ vm_fault_page(
 	boolean_t		look_for_page;
 	vm_prot_t		access_required = fault_type;
 	vm_prot_t		wants_copy_flag;
-	vm_object_size_t	length;
-	vm_object_offset_t	cluster_start, cluster_end;
 	CLUSTER_STAT(int pages_at_higher_offsets;)
 	CLUSTER_STAT(int pages_at_lower_offsets;)
-	kern_return_t	wait_result;
+	kern_return_t		wait_result;
 	boolean_t		interruptible_state;
-	boolean_t               bumped_pagein = FALSE;
+	vm_fault_return_t	error;
+	int			my_fault;
+	uint32_t		try_failed_count;
+	int			interruptible; /* how may fault be interrupted? */
+	memory_object_t		pager;
 
-
-#if	MACH_PAGEMAP
 /*
  * MACH page map - an optional optimization where a bit map is maintained
  * by the VM subsystem for internal objects to indicate which pages of
@@ -418,15 +647,15 @@ vm_fault_page(
  * is designed to eliminate pager interaction overhead, if it is 
  * 'known' that the page does not exist on backing store.
  *
- * LOOK_FOR() evaluates to TRUE if the page specified by object/offset is 
+ * MUST_ASK_PAGER() evaluates to TRUE if the page specified by object/offset is 
  * either marked as paged out in the existence map for the object or no 
- * existence map exists for the object.  LOOK_FOR() is one of the
+ * existence map exists for the object.  MUST_ASK_PAGER() is one of the
  * criteria in the decision to invoke the pager.   It is also used as one
  * of the criteria to terminate the scan for adjacent pages in a clustered
- * pagein operation.  Note that LOOK_FOR() always evaluates to TRUE for
+ * pagein operation.  Note that MUST_ASK_PAGER() always evaluates to TRUE for
  * permanent objects.  Note also that if the pager for an internal object 
  * has not been created, the pager is not invoked regardless of the value 
- * of LOOK_FOR() and that clustered pagein scans are only done on an object
+ * of MUST_ASK_PAGER() and that clustered pagein scans are only done on an object
  * for which a pager has been created.
  *
  * PAGED_OUT() evaluates to TRUE if the page specified by the object/offset
@@ -434,29 +663,15 @@ vm_fault_page(
  * PAGED_OUT() is used to determine if a page has already been pushed
  * into a copy object in order to avoid a redundant page out operation.
  */
-#define LOOK_FOR(o, f) (vm_external_state_get((o)->existence_map, (f)) \
+#if MACH_PAGEMAP
+#define MUST_ASK_PAGER(o, f) (vm_external_state_get((o)->existence_map, (f)) \
 			!= VM_EXTERNAL_STATE_ABSENT)
 #define PAGED_OUT(o, f) (vm_external_state_get((o)->existence_map, (f)) \
 			== VM_EXTERNAL_STATE_EXISTS)
-#else /* MACH_PAGEMAP */
-/*
- * If the MACH page map optimization is not enabled,
- * LOOK_FOR() always evaluates to TRUE.  The pager will always be 
- * invoked to resolve missing pages in an object, assuming the pager 
- * has been created for the object.  In a clustered page operation, the 
- * absence of a page on backing backing store cannot be used to terminate 
- * a scan for adjacent pages since that information is available only in 
- * the pager.  Hence pages that may not be paged out are potentially 
- * included in a clustered request.  The vnode pager is coded to deal 
- * with any combination of absent/present pages in a clustered 
- * pagein request.  PAGED_OUT() always evaluates to FALSE, i.e. the pager
- * will always be invoked to push a dirty page into a copy object assuming
- * a pager has been created.  If the page has already been pushed, the
- * pager will ingore the new request.
- */
-#define LOOK_FOR(o, f) TRUE
-#define PAGED_OUT(o, f) FALSE
-#endif /* MACH_PAGEMAP */
+#else
+#define MUST_ASK_PAGER(o, f) (TRUE)
+#define PAGED_OUT(o, f) (FALSE)
+#endif
 
 /*
  *	Recovery actions
@@ -469,7 +684,7 @@ vm_fault_page(
 #define DO_RELEASE_PAGE(m)				\
 	MACRO_BEGIN					\
 	PAGE_WAKEUP_DONE(m);				\
-	if (!m->active && !m->inactive)			\
+	if (!m->active && !m->inactive && !m->throttled)\
 		vm_page_activate(m);			\
 	vm_page_unlock_queues();			\
 	MACRO_END
@@ -485,31 +700,6 @@ vm_fault_page(
 #endif
 
 
-
-#if	!VM_FAULT_STATIC_CONFIG
-	if (vm_fault_dirty_handling
-#if	MACH_KDB
-		/*
-		 *	If there are watchpoints set, then
-		 *	we don't want to give away write permission
-		 *	on a read fault.  Make the task write fault,
-		 *	so that the watchpoint code notices the access.
-		 */
-	    || db_watchpoint_list
-#endif	/* MACH_KDB */
-	    ) {
-		/*
-		 *	If we aren't asking for write permission,
-		 *	then don't give it away.  We're using write
-		 *	faults to set the dirty bit.
-		 */
-		if (!(fault_type & VM_PROT_WRITE))
-			*protection &= ~VM_PROT_WRITE;
-	}
-
-	if (!vm_fault_interruptible)
-		interruptible = THREAD_UNINT;
-#else	/* STATIC_CONFIG */
 #if	MACH_KDB
 		/*
 		 *	If there are watchpoints set, then
@@ -526,10 +716,9 @@ vm_fault_page(
 		if (!(fault_type & VM_PROT_WRITE))
 			*protection &= ~VM_PROT_WRITE;
 	}
-
 #endif	/* MACH_KDB */
-#endif	/* STATIC_CONFIG */
 
+	interruptible = fault_info->interruptible;
 	interruptible_state = thread_interrupt_level(interruptible);
  
 	/*
@@ -544,83 +733,69 @@ vm_fault_page(
 	 *		pager access or when waiting for memory, so
 	 *		we use a busy page then.
 	 *
-	 *		Note also that we aren't as concerned about more than
-	 *		one thread attempting to memory_object_data_unlock
-	 *		the same page at once, so we don't hold the page
-	 *		as busy then, but do record the highest unlock
-	 *		value so far.  [Unlock requests may also be delivered
-	 *		out of order.]
-	 *
 	 *	2)	To prevent another thread from racing us down the
 	 *		shadow chain and entering a new page in the top
 	 *		object before we do, we must keep a busy page in
 	 *		the top object while following the shadow chain.
 	 *
 	 *	3)	We must increment paging_in_progress on any object
-	 *		for which we have a busy page
+	 *		for which we have a busy page before dropping
+	 *		the object lock
 	 *
 	 *	4)	We leave busy pages on the pageout queues.
 	 *		If the pageout daemon comes across a busy page,
 	 *		it will remove the page from the pageout queues.
 	 */
 
-	/*
-	 *	Search for the page at object/offset.
-	 */
-
 	object = first_object;
 	offset = first_offset;
 	first_m = VM_PAGE_NULL;
 	access_required = fault_type;
 
+
 	XPR(XPR_VM_FAULT,
 		"vm_f_page: obj 0x%X, offset 0x%X, type %d, prot %d\n",
 		(integer_t)object, offset, fault_type, *protection, 0);
 
 	/*
-	 *	See whether this page is resident
+	 * default type of fault
 	 */
+	my_fault = DBG_CACHE_HIT_FAULT;
 
 	while (TRUE) {
 #if TRACEFAULTPAGE
 		dbgTrace(0xBEEF0003, (unsigned int) 0, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
 		if (!object->alive) {
+		        /*
+			 * object is no longer valid
+			 * clean up and return error
+			 */
 			vm_fault_cleanup(object, first_m);
 			thread_interrupt_level(interruptible_state);
-			return(VM_FAULT_MEMORY_ERROR);
+
+			return (VM_FAULT_MEMORY_ERROR);
 		}
+
+		/*
+		 * See whether the page at 'offset' is resident
+		 */
 		m = vm_page_lookup(object, offset);
 #if TRACEFAULTPAGE
 		dbgTrace(0xBEEF0004, (unsigned int) m, (unsigned int) object);	/* (TEST/DEBUG) */
 #endif
 		if (m != VM_PAGE_NULL) {
-			/*
-			 *	If the page was pre-paged as part of a
-			 *	cluster, record the fact.
-			 *      If we were passed a valid pointer for
-			 *      "type_of_fault", than we came from
-			 * 	vm_fault... we'll let it deal with
-			 *      this condition, since it
-			 *      needs to see m->clustered to correctly
-			 *      account the pageins.
-			 */
-			if (type_of_fault == NULL && m->clustered) {
-				vm_pagein_cluster_used++;
-				m->clustered = FALSE;
-			}
-
-			/*
-			 *	If the page is being brought in,
-			 *	wait for it and then retry.
-			 *
-			 *	A possible optimization: if the page
-			 *	is known to be resident, we can ignore
-			 *	pages that are absent (regardless of
-			 *	whether they're busy).
-			 */
 
 			if (m->busy) {
+			        /*
+				 * The page is being brought in,
+				 * wait for it and then retry.
+				 *
+				 * A possible optimization: if the page
+				 * is known to be resident, we can ignore
+				 * pages that are absent (regardless of
+				 * whether they're busy).
+				 */
 #if TRACEFAULTPAGE
 				dbgTrace(0xBEEF0005, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
@@ -634,82 +809,82 @@ vm_fault_page(
 				if (wait_result != THREAD_AWAKENED) {
 					vm_fault_cleanup(object, first_m);
 					thread_interrupt_level(interruptible_state);
+
 					if (wait_result == THREAD_RESTART)
-					  {
-						return(VM_FAULT_RETRY);
-					  }
+					        return (VM_FAULT_RETRY);
 					else
-					  {
-						return(VM_FAULT_INTERRUPTED);
-					  }
+						return (VM_FAULT_INTERRUPTED);
 				}
 				continue;
 			}
 
-			if (m->encrypted) {
-				/*
-				 * ENCRYPTED SWAP:
-				 * the user needs access to a page that we
-				 * encrypted before paging it out.
-				 * Decrypt the page now.
-				 * Keep it busy to prevent anyone from
-				 * accessing it during the decryption.
-				 */
-				m->busy = TRUE;
-				vm_page_decrypt(m, 0);
-				assert(object == m->object);
-				assert(m->busy);
-				PAGE_WAKEUP_DONE(m);
-
+			if (m->phys_page == vm_page_guard_addr) {
 				/*
-				 * Retry from the top, in case
-				 * something changed while we were
-				 * decrypting.
+				 * Guard page: off limits !
 				 */
-				continue;
+				if (fault_type == VM_PROT_NONE) {
+					/*
+					 * The fault is not requesting any
+					 * access to the guard page, so it must
+					 * be just to wire or unwire it.
+					 * Let's pretend it succeeded...
+					 */
+					m->busy = TRUE;
+					*result_page = m;
+					assert(first_m == VM_PAGE_NULL);
+					*top_page = first_m;
+					if (type_of_fault)
+						*type_of_fault = DBG_GUARD_FAULT;
+					return VM_FAULT_SUCCESS;
+				} else {
+					/*
+					 * The fault requests access to the
+					 * guard page: let's deny that !
+					 */
+					vm_fault_cleanup(object, first_m);
+					thread_interrupt_level(interruptible_state);
+					return VM_FAULT_MEMORY_ERROR;
+				}
 			}
-			ASSERT_PAGE_DECRYPTED(m);
-
-			/*
-			 *	If the page is in error, give up now.
-			 */
 
 			if (m->error) {
+			        /*
+				 * The page is in error, give up now.
+				 */
 #if TRACEFAULTPAGE
 				dbgTrace(0xBEEF0006, (unsigned int) m, (unsigned int) error_code);	/* (TEST/DEBUG) */
 #endif
 				if (error_code)
-					*error_code = m->page_error;
+				        *error_code = KERN_MEMORY_ERROR;
 				VM_PAGE_FREE(m);
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_MEMORY_ERROR);
-			}
-
-			/*
-			 *	If the pager wants us to restart
-			 *	at the top of the chain,
-			 *	typically because it has moved the
-			 *	page to another pager, then do so.
-			 */
 
+				return (VM_FAULT_MEMORY_ERROR);
+			}
 			if (m->restart) {
+			        /*
+				 * The pager wants us to restart
+				 * at the top of the chain,
+				 * typically because it has moved the
+				 * page to another pager, then do so.
+				 */
 #if TRACEFAULTPAGE
 				dbgTrace(0xBEEF0007, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
 				VM_PAGE_FREE(m);
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_RETRY);
-			}
-
-			/*
-			 *	If the page isn't busy, but is absent,
-			 *	then it was deemed "unavailable".
-			 */
 
+				return (VM_FAULT_RETRY);
+			}
 			if (m->absent) {
-				/*
+			        /*
+				 * The page isn't busy, but is absent,
+				 * therefore it's deemed "unavailable".
+				 *
 				 * Remove the non-existent page (unless it's
 				 * in the top object) and move on down to the
 				 * next object (if there is one).
@@ -717,151 +892,86 @@ vm_fault_page(
 #if TRACEFAULTPAGE
 				dbgTrace(0xBEEF0008, (unsigned int) m, (unsigned int) object->shadow);	/* (TEST/DEBUG) */
 #endif
-
 				next_object = object->shadow;
-				if (next_object == VM_OBJECT_NULL) {
-					vm_page_t real_m;
-
-					assert(!must_be_resident);
-
-					if (object->shadow_severed) {
-						vm_fault_cleanup(
-							object, first_m);
-						thread_interrupt_level(interruptible_state);
-						return VM_FAULT_MEMORY_ERROR;
-					}
 
+				if (next_object == VM_OBJECT_NULL) {
 					/*
 					 * Absent page at bottom of shadow
 					 * chain; zero fill the page we left
-					 * busy in the first object, and flush
-					 * the absent page.  But first we
-					 * need to allocate a real page.
+					 * busy in the first object, and free
+					 * the absent page.
 					 */
-					if (VM_PAGE_THROTTLED() ||
-					    (real_m = vm_page_grab()) 
-							== VM_PAGE_NULL) {
-						vm_fault_cleanup(
-							object, first_m);
-						thread_interrupt_level(
-							interruptible_state);
-						return(
-						   VM_FAULT_MEMORY_SHORTAGE);
-					}
+					assert(!must_be_resident);
 
 					/*
-					 * are we protecting the system from
-					 * backing store exhaustion.  If so
-					 * sleep unless we are privileged.
+					 * check for any conditions that prevent
+					 * us from creating a new zero-fill page
+					 * vm_fault_check will do all of the 
+					 * fault cleanup in the case of an error condition
+					 * including resetting the thread_interrupt_level
 					 */
+					error = vm_fault_check(object, m, first_m, interruptible_state);
 
-					if(vm_backing_store_low) {
-					   if(!(current_task()->priv_flags 
-						& VM_BACKING_STORE_PRIV)) {
-						assert_wait((event_t) 
-							&vm_backing_store_low, 
-							THREAD_UNINT);
-						vm_fault_cleanup(object, 
-								    first_m);
-						thread_block(THREAD_CONTINUE_NULL);
-						thread_interrupt_level(
-							interruptible_state);
-						return(VM_FAULT_RETRY);
-					   }
-					}
-
+					if (error != VM_FAULT_SUCCESS)
+					        return (error);
 
 					XPR(XPR_VM_FAULT,
-	      "vm_f_page: zero obj 0x%X, off 0x%X, page 0x%X, first_obj 0x%X\n",
+					    "vm_f_page: zero obj 0x%X, off 0x%X, page 0x%X, first_obj 0x%X\n",
 						(integer_t)object, offset,
 						(integer_t)m,
 						(integer_t)first_object, 0);
+
 					if (object != first_object) {
+					        /*
+						 * free the absent page we just found
+						 */
 						VM_PAGE_FREE(m);
+
+						/*
+						 * drop reference and lock on current object
+						 */
 						vm_object_paging_end(object);
 						vm_object_unlock(object);
-						object = first_object;
-						offset = first_offset;
+
+						/*
+						 * grab the original page we 
+						 * 'soldered' in place and
+						 * retake lock on 'first_object'
+						 */
 						m = first_m;
 						first_m = VM_PAGE_NULL;
-						vm_object_lock(object);
-					}
 
-					VM_PAGE_FREE(m);
-					assert(real_m->busy);
-					vm_page_insert(real_m, object, offset);
-					m = real_m;
-
-					/*
-					 *  Drop the lock while zero filling
-					 *  page.  Then break because this
-					 *  is the page we wanted.  Checking
-					 *  the page lock is a waste of time;
-					 *  this page was either absent or
-					 *  newly allocated -- in both cases
-					 *  it can't be page locked by a pager.
-					 */
-					m->no_isync = FALSE;
+						object = first_object;
+						offset = first_offset;
 
-					if (!no_zero_fill) {
-						vm_object_unlock(object);
-						vm_page_zero_fill(m);
 						vm_object_lock(object);
-
-						if (type_of_fault)
-						        *type_of_fault = DBG_ZERO_FILL_FAULT;
-						VM_STAT(zero_fill_count++);
-					}
-					if (bumped_pagein == TRUE) {
-					        VM_STAT(pageins--);
-						current_task()->pageins--;
-					}
-					vm_page_lock_queues();
-					VM_PAGE_QUEUES_REMOVE(m);
-					m->page_ticket = vm_page_ticket;
-					assert(!m->laundry);
-					assert(m->object != kernel_object);
-					assert(m->pageq.next == NULL &&
-					       m->pageq.prev == NULL);
-					if(m->object->size > 0x200000) {
-						m->zero_fill = TRUE;
-						/* depends on the queues lock */
-						vm_zf_count += 1;
-						queue_enter(&vm_page_queue_zf, 
-							m, vm_page_t, pageq);
 					} else {
-						queue_enter(
-							&vm_page_queue_inactive, 
-							m, vm_page_t, pageq);
-					}
-					vm_page_ticket_roll++;
-					if(vm_page_ticket_roll == 
-						VM_PAGE_TICKETS_IN_ROLL) {
-						vm_page_ticket_roll = 0;
-						if(vm_page_ticket == 
-						     VM_PAGE_TICKET_ROLL_IDS)
-							vm_page_ticket= 0;
-						else
-							vm_page_ticket++;
+					        /*
+						 * we're going to use the absent page we just found
+						 * so convert it to a 'busy' page
+						 */
+					        m->absent = FALSE;
+						m->busy = TRUE;
 					}
-					m->inactive = TRUE;
-					vm_page_inactive_count++;
-					vm_page_unlock_queues();
+					/*
+					 * zero-fill the page and put it on
+					 * the correct paging queue
+					 */
+					my_fault = vm_fault_zero_page(m, no_zero_fill);
+
 					break;
 				} else {
-					if (must_be_resident) {
+					if (must_be_resident)
 						vm_object_paging_end(object);
-					} else if (object != first_object) {
+					else if (object != first_object) {
 						vm_object_paging_end(object);
 						VM_PAGE_FREE(m);
 					} else {
 						first_m = m;
 						m->absent = FALSE;
-						m->unusual = FALSE;
-						vm_object_absent_release(object);
 						m->busy = TRUE;
 
-						vm_page_lock_queues();
+						vm_page_lockspin_queues();
 						VM_PAGE_QUEUES_REMOVE(m);
 						vm_page_unlock_queues();
 					}
@@ -870,22 +980,28 @@ vm_fault_page(
 						(integer_t)object, offset,
 						(integer_t)next_object,
 						offset+object->shadow_offset,0);
+
 					offset += object->shadow_offset;
-					hi_offset += object->shadow_offset;
-					lo_offset += object->shadow_offset;
+					fault_info->lo_offset += object->shadow_offset;
+					fault_info->hi_offset += object->shadow_offset;
 					access_required = VM_PROT_READ;
+
 					vm_object_lock(next_object);
 					vm_object_unlock(object);
 					object = next_object;
 					vm_object_paging_begin(object);
+					
+					/*
+					 * reset to default type of fault
+					 */
+					my_fault = DBG_CACHE_HIT_FAULT;
+
 					continue;
 				}
 			}
-
 			if ((m->cleaning)
-				&& ((object != first_object) ||
-				    (object->copy != VM_OBJECT_NULL))
-				&& (fault_type & VM_PROT_WRITE)) {
+			    && ((object != first_object) || (object->copy != VM_OBJECT_NULL))
+			    && (fault_type & VM_PROT_WRITE)) {
 				/*
 				 * This is a copy-on-write fault that will
 				 * cause us to revoke access to this page, but
@@ -903,188 +1019,140 @@ vm_fault_page(
 				    "vm_f_page: cleaning obj 0x%X, offset 0x%X, page 0x%X\n",
 					(integer_t)object, offset,
 					(integer_t)m, 0, 0);
-				/* take an extra ref so that object won't die */
-				assert(object->ref_count > 0);
-				object->ref_count++;
-				vm_object_res_reference(object);
+				/*
+				 * take an extra ref so that object won't die
+				 */
+				vm_object_reference_locked(object);
+
 				vm_fault_cleanup(object, first_m);
+				
 				counter(c_vm_fault_page_block_backoff_kernel++);
 				vm_object_lock(object);
 				assert(object->ref_count > 0);
+
 				m = vm_page_lookup(object, offset);
+
 				if (m != VM_PAGE_NULL && m->cleaning) {
 					PAGE_ASSERT_WAIT(m, interruptible);
+
 					vm_object_unlock(object);
 					wait_result = thread_block(THREAD_CONTINUE_NULL);
 					vm_object_deallocate(object);
+
 					goto backoff;
 				} else {
 					vm_object_unlock(object);
+
 					vm_object_deallocate(object);
 					thread_interrupt_level(interruptible_state);
-					return VM_FAULT_RETRY;
+
+					return (VM_FAULT_RETRY);
 				}
 			}
+			if (type_of_fault == NULL && m->speculative) {
+			        /*
+				 * If we were passed a non-NULL pointer for
+				 * "type_of_fault", than we came from
+				 * vm_fault... we'll let it deal with
+				 * this condition, since it
+				 * needs to see m->speculative to correctly
+				 * account the pageins, otherwise...
+				 * take it off the speculative queue, we'll
+				 * let the caller of vm_fault_page deal
+				 * with getting it onto the correct queue
+				 */
+			        vm_page_lockspin_queues();
+			        VM_PAGE_QUEUES_REMOVE(m);
+			        vm_page_unlock_queues();
+			}
 
-			/*
-			 *	If the desired access to this page has
-			 *	been locked out, request that it be unlocked.
-			 */
-
-			if (access_required & m->page_lock) {
-				if ((access_required & m->unlock_request) != access_required) {
-					vm_prot_t	new_unlock_request;
-					kern_return_t	rc;
-					
-#if TRACEFAULTPAGE
-					dbgTrace(0xBEEF000A, (unsigned int) m, (unsigned int) object->pager_ready);	/* (TEST/DEBUG) */
-#endif
-					if (!object->pager_ready) {
-					XPR(XPR_VM_FAULT,
-					    "vm_f_page: ready wait acc_req %d, obj 0x%X, offset 0x%X, page 0x%X\n",
-						access_required,
-						(integer_t)object, offset,
-						(integer_t)m, 0);
-						/* take an extra ref */
-						assert(object->ref_count > 0);
-						object->ref_count++;
-						vm_object_res_reference(object);
-						vm_fault_cleanup(object,
-								 first_m);
-						counter(c_vm_fault_page_block_backoff_kernel++);
-						vm_object_lock(object);
-						assert(object->ref_count > 0);
-						if (!object->pager_ready) {
-							wait_result = vm_object_assert_wait(
-								object,
-								VM_OBJECT_EVENT_PAGER_READY,
-								interruptible);
-							vm_object_unlock(object);
-							if (wait_result == THREAD_WAITING)
-								wait_result = thread_block(THREAD_CONTINUE_NULL);
-							vm_object_deallocate(object);
-							goto backoff;
-						} else {
-							vm_object_unlock(object);
-							vm_object_deallocate(object);
-							thread_interrupt_level(interruptible_state);
-							return VM_FAULT_RETRY;
-						}
-					}
+			if (m->encrypted) {
+				/*
+				 * ENCRYPTED SWAP:
+				 * the user needs access to a page that we
+				 * encrypted before paging it out.
+				 * Decrypt the page now.
+				 * Keep it busy to prevent anyone from
+				 * accessing it during the decryption.
+				 */
+				m->busy = TRUE;
+				vm_page_decrypt(m, 0);
+				assert(object == m->object);
+				assert(m->busy);
+				PAGE_WAKEUP_DONE(m);
 
-					new_unlock_request = m->unlock_request =
-						(access_required | m->unlock_request);
-					vm_object_unlock(object);
-					XPR(XPR_VM_FAULT,
-					    "vm_f_page: unlock obj 0x%X, offset 0x%X, page 0x%X, unl_req %d\n",
-					(integer_t)object, offset,
-					(integer_t)m, new_unlock_request, 0);
-					if ((rc = memory_object_data_unlock(
-						object->pager,
-						offset + object->paging_offset,
-						PAGE_SIZE,
-						new_unlock_request))
-					     != KERN_SUCCESS) {
-						if (vm_fault_debug)
-					     	    printf("vm_fault: memory_object_data_unlock failed\n");
-						vm_object_lock(object);
-						vm_fault_cleanup(object, first_m);
-						thread_interrupt_level(interruptible_state);
-						return((rc == MACH_SEND_INTERRUPTED) ?
-							VM_FAULT_INTERRUPTED :
-							VM_FAULT_MEMORY_ERROR);
-					}
-					vm_object_lock(object);
-					continue;
-				}
+				/*
+				 * Retry from the top, in case
+				 * something changed while we were
+				 * decrypting.
+				 */
+				continue;
+			}
+			ASSERT_PAGE_DECRYPTED(m);
 
-				XPR(XPR_VM_FAULT,
-	"vm_f_page: access wait acc_req %d, obj 0x%X, offset 0x%X, page 0x%X\n",
-					access_required, (integer_t)object,
-					offset, (integer_t)m, 0);
-				/* take an extra ref so object won't die */
-				assert(object->ref_count > 0);
-				object->ref_count++;
-				vm_object_res_reference(object);
-				vm_fault_cleanup(object, first_m);
-				counter(c_vm_fault_page_block_backoff_kernel++);
-				vm_object_lock(object);
-				assert(object->ref_count > 0);
-				m = vm_page_lookup(object, offset);
-				if (m != VM_PAGE_NULL && 
-				    (access_required & m->page_lock) &&
-				    !((access_required & m->unlock_request) != access_required)) {
-					PAGE_ASSERT_WAIT(m, interruptible);
-					vm_object_unlock(object);
-					wait_result = thread_block(THREAD_CONTINUE_NULL);
-					vm_object_deallocate(object);
-					goto backoff;
-				} else {
-					vm_object_unlock(object);
-					vm_object_deallocate(object);
-					thread_interrupt_level(interruptible_state);
-					return VM_FAULT_RETRY;
-				}
+			if (m->object->code_signed) {
+				/*
+				 * CODE SIGNING:
+				 * We just paged in a page from a signed
+				 * memory object but we don't need to
+				 * validate it now.  We'll validate it if
+				 * when it gets mapped into a user address
+				 * space for the first time or when the page
+				 * gets copied to another object as a result
+				 * of a copy-on-write.
+				 */
 			}
+
 			/*
-			 *	We mark the page busy and leave it on
-			 *	the pageout queues.  If the pageout
-			 *	deamon comes across it, then it will
-			 *	remove the page.
+			 * We mark the page busy and leave it on
+			 * the pageout queues.  If the pageout
+			 * deamon comes across it, then it will
+			 * remove the page from the queue, but not the object
 			 */
-
 #if TRACEFAULTPAGE
 			dbgTrace(0xBEEF000B, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
-
-#if	!VM_FAULT_STATIC_CONFIG
-			if (!software_reference_bits) {
-				vm_page_lock_queues();
-				if (m->inactive)
-					vm_stat.reactivations++;
-
-				VM_PAGE_QUEUES_REMOVE(m);
-				vm_page_unlock_queues();
-			}
-#endif
 			XPR(XPR_VM_FAULT,
 			    "vm_f_page: found page obj 0x%X, offset 0x%X, page 0x%X\n",
 				(integer_t)object, offset, (integer_t)m, 0, 0);
 			assert(!m->busy);
-			m->busy = TRUE;
 			assert(!m->absent);
+
+			m->busy = TRUE;
 			break;
 		}
+		
 
-		look_for_page =
-			(object->pager_created) &&
-			  LOOK_FOR(object, offset) &&
-			    (!data_supply);
-
+		/*
+		 * we get here when there is no page present in the object at
+		 * the offset we're interested in... we'll allocate a page
+		 * at this point if the pager associated with
+		 * this object can provide the data or we're the top object...
+		 * object is locked;  m == NULL
+		 */
+		look_for_page =	(object->pager_created && (MUST_ASK_PAGER(object, offset) == TRUE) && !data_supply);
+		
 #if TRACEFAULTPAGE
 		dbgTrace(0xBEEF000C, (unsigned int) look_for_page, (unsigned int) object);	/* (TEST/DEBUG) */
 #endif
-		if ((look_for_page || (object == first_object))
-			 	&& !must_be_resident 
-				&& !(object->phys_contiguous))  {
+		if ((look_for_page || (object == first_object)) && !must_be_resident && !object->phys_contiguous) {
 			/*
-			 *	Allocate a new page for this object/offset
-			 *	pair.
+			 * Allocate a new page for this object/offset pair
 			 */
-
-			m = vm_page_grab_fictitious();
+			m = vm_page_grab();
 #if TRACEFAULTPAGE
 			dbgTrace(0xBEEF000D, (unsigned int) m, (unsigned int) object);	/* (TEST/DEBUG) */
 #endif
 			if (m == VM_PAGE_NULL) {
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_FICTITIOUS_SHORTAGE);
+
+				return (VM_FAULT_MEMORY_SHORTAGE);
 			}
 			vm_page_insert(m, object, offset);
 		}
-
-		if ((look_for_page && !must_be_resident)) {
+		if (look_for_page && !must_be_resident) {
 			kern_return_t	rc;
 
 			/*
@@ -1095,231 +1163,166 @@ vm_fault_page(
 #if TRACEFAULTPAGE
 				dbgTrace(0xBEEF000E, (unsigned int) 0, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
-				if(m != VM_PAGE_NULL)
-					VM_PAGE_FREE(m);
+				if (m != VM_PAGE_NULL)
+				        VM_PAGE_FREE(m);
+
 				XPR(XPR_VM_FAULT,
 				"vm_f_page: ready wait obj 0x%X, offset 0x%X\n",
 					(integer_t)object, offset, 0, 0, 0);
-				/* take an extra ref so object won't die */
-				assert(object->ref_count > 0);
-				object->ref_count++;
-				vm_object_res_reference(object);
+
+				/*
+				 * take an extra ref so object won't die
+				 */
+				vm_object_reference_locked(object);
 				vm_fault_cleanup(object, first_m);
 				counter(c_vm_fault_page_block_backoff_kernel++);
+
 				vm_object_lock(object);
 				assert(object->ref_count > 0);
+
 				if (!object->pager_ready) {
-					wait_result = vm_object_assert_wait(object,
-							      VM_OBJECT_EVENT_PAGER_READY,
-							      interruptible);
+					wait_result = vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGER_READY, interruptible);
+
 					vm_object_unlock(object);
 					if (wait_result == THREAD_WAITING)
 						wait_result = thread_block(THREAD_CONTINUE_NULL);
 					vm_object_deallocate(object);
+
 					goto backoff;
 				} else {
 					vm_object_unlock(object);
 					vm_object_deallocate(object);
 					thread_interrupt_level(interruptible_state);
-					return VM_FAULT_RETRY;
-				}
-			}
 
-			if(object->phys_contiguous) {
-				if(m != VM_PAGE_NULL) {
-					VM_PAGE_FREE(m);
-					m = VM_PAGE_NULL;
+					return (VM_FAULT_RETRY);
 				}
-				goto no_clustering;
 			}
-			if (object->internal) {
+			if (!object->internal && !object->phys_contiguous && object->paging_in_progress > vm_object_pagein_throttle) {
 				/*
-				 *	Requests to the default pager
-				 *	must reserve a real page in advance,
-				 *	because the pager's data-provided
-				 *	won't block for pages.  IMPORTANT:
-				 *	this acts as a throttling mechanism
-				 *	for data_requests to the default
-				 *	pager.
+				 * If there are too many outstanding page
+				 * requests pending on this external object, we
+				 * wait for them to be resolved now.
 				 */
-
 #if TRACEFAULTPAGE
-				dbgTrace(0xBEEF000F, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
+				dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
-				if (m->fictitious && !vm_page_convert(m)) {
+				if (m != VM_PAGE_NULL)
 					VM_PAGE_FREE(m);
-					vm_fault_cleanup(object, first_m);
-					thread_interrupt_level(interruptible_state);
-					return(VM_FAULT_MEMORY_SHORTAGE);
-				}
-			} else if (object->absent_count >
-						vm_object_absent_max) {
 				/*
-				 *	If there are too many outstanding page
-				 *	requests pending on this object, we
-				 *	wait for them to be resolved now.
+				 * take an extra ref so object won't die
 				 */
+				vm_object_reference_locked(object);
 
-#if TRACEFAULTPAGE
-				dbgTrace(0xBEEF0010, (unsigned int) m, (unsigned int) 0);	/* (TEST/DEBUG) */
-#endif
-				if(m != VM_PAGE_NULL)
-					VM_PAGE_FREE(m);
-				/* take an extra ref so object won't die */
-				assert(object->ref_count > 0);
-				object->ref_count++;
-				vm_object_res_reference(object);
 				vm_fault_cleanup(object, first_m);
+
 				counter(c_vm_fault_page_block_backoff_kernel++);
+
 				vm_object_lock(object);
 				assert(object->ref_count > 0);
-				if (object->absent_count > vm_object_absent_max) {
-					vm_object_absent_assert_wait(object,
-								     interruptible);
+
+				if (object->paging_in_progress > vm_object_pagein_throttle) {
+				        vm_object_assert_wait(object, VM_OBJECT_EVENT_PAGING_IN_PROGRESS, interruptible);
+
 					vm_object_unlock(object);
 					wait_result = thread_block(THREAD_CONTINUE_NULL);
 					vm_object_deallocate(object);
+
 					goto backoff;
 				} else {
 					vm_object_unlock(object);
 					vm_object_deallocate(object);
 					thread_interrupt_level(interruptible_state);
-					return VM_FAULT_RETRY;
+
+					return (VM_FAULT_RETRY);
 				}
 			}
-
-			/*
-			 *	Indicate that the page is waiting for data
-			 *	from the memory manager.
-			 */
-
-			if(m != VM_PAGE_NULL) {
-
-				m->list_req_pending = TRUE;
+			if (m != VM_PAGE_NULL) {
+			        /*
+				 * Indicate that the page is waiting for data
+				 * from the memory manager.
+				 */
+			        m->list_req_pending = TRUE;
 				m->absent = TRUE;
-				m->unusual = TRUE;
-				object->absent_count++;
-
 			}
 
-no_clustering:
-			cluster_start = offset;
-			length = PAGE_SIZE;
-
-			/* 
-			 * lengthen the cluster by the pages in the working set
-			 */
-			if((map != NULL) && 
-				(current_task()->dynamic_working_set != 0)) {
-				cluster_end = cluster_start + length;
-				/* tws values for start and end are just a 
-				 * suggestions.  Therefore, as long as
-				 * build_cluster does not use pointers or
-				 * take action based on values that
-				 * could be affected by re-entrance we
-				 * do not need to take the map lock.
-				 */
-				cluster_end = offset + PAGE_SIZE_64;
-				tws_build_cluster(
-					current_task()->dynamic_working_set,
-					object, &cluster_start,
-					&cluster_end, 0x40000);
-				length = cluster_end - cluster_start;
-			}
 #if TRACEFAULTPAGE
 			dbgTrace(0xBEEF0012, (unsigned int) object, (unsigned int) 0);	/* (TEST/DEBUG) */
 #endif
+
 			/*
-			 *	We have a busy page, so we can
-			 *	release the object lock.
+			 * It's possible someone called vm_object_destroy while we weren't
+			 * holding the object lock.  If that has happened, then bail out 
+			 * here.
 			 */
-			vm_object_unlock(object);
+
+			pager = object->pager;
+
+			if (pager == MEMORY_OBJECT_NULL) {
+				vm_fault_cleanup(object, first_m);
+				thread_interrupt_level(interruptible_state);
+				return VM_FAULT_MEMORY_ERROR;
+			}
 
 			/*
-			 *	Call the memory manager to retrieve the data.
+			 * We have an absent page in place for the faulting offset,
+			 * so we can release the object lock.
 			 */
 
-			if (type_of_fault)
-				*type_of_fault = ((int)length << 8) | DBG_PAGEIN_FAULT;
-			VM_STAT(pageins++);
-			current_task()->pageins++;
-			bumped_pagein = TRUE;
+			vm_object_unlock(object);
 
 			/*
-			 *	If this object uses a copy_call strategy,
-			 *	and we are interested in a copy of this object
-			 *	(having gotten here only by following a
-			 *	shadow chain), then tell the memory manager
-			 *	via a flag added to the desired_access
-			 *	parameter, so that it can detect a race
-			 *	between our walking down the shadow chain
-			 *	and its pushing pages up into a copy of
-			 *	the object that it manages.
+			 * If this object uses a copy_call strategy,
+			 * and we are interested in a copy of this object
+			 * (having gotten here only by following a
+			 * shadow chain), then tell the memory manager
+			 * via a flag added to the desired_access
+			 * parameter, so that it can detect a race
+			 * between our walking down the shadow chain
+			 * and its pushing pages up into a copy of
+			 * the object that it manages.
 			 */
-
-			if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL &&
-			    object != first_object) {
+			if (object->copy_strategy == MEMORY_OBJECT_COPY_CALL && object != first_object)
 				wants_copy_flag = VM_PROT_WANTS_COPY;
-			} else {
+			else
 				wants_copy_flag = VM_PROT_NONE;
-			}
 
 			XPR(XPR_VM_FAULT,
 			    "vm_f_page: data_req obj 0x%X, offset 0x%X, page 0x%X, acc %d\n",
 				(integer_t)object, offset, (integer_t)m,
 				access_required | wants_copy_flag, 0);
 
-			rc = memory_object_data_request(object->pager, 
-					cluster_start +	object->paging_offset, 
-					length,
-					access_required | wants_copy_flag);
-
+			/*
+			 * Call the memory manager to retrieve the data.
+			 */
+			rc = memory_object_data_request(
+				pager,
+				offset + object->paging_offset,
+				PAGE_SIZE,
+				access_required | wants_copy_flag,
+				(memory_object_fault_info_t)fault_info);
 
 #if TRACEFAULTPAGE
 			dbgTrace(0xBEEF0013, (unsigned int) object, (unsigned int) rc);	/* (TEST/DEBUG) */
 #endif
+			vm_object_lock(object);
+
 			if (rc != KERN_SUCCESS) {
-				if (rc != MACH_SEND_INTERRUPTED
-				    && vm_fault_debug)
-					printf("%s(0x%x, 0x%xll, 0x%xll, 0x%x) failed, rc=%d\n",
-						"memory_object_data_request",
-						object->pager,
-						cluster_start + object->paging_offset, 
-						length, access_required, rc);
-				/*
-				 *	Don't want to leave a busy page around,
-				 *	but the data request may have blocked,
-				 *	so check if it's still there and busy.
-				 */
-				if(!object->phys_contiguous) {
-				   vm_object_lock(object);
-				   for (; length; length -= PAGE_SIZE,
-				      cluster_start += PAGE_SIZE_64) {
-				      vm_page_t p;
-				      if ((p = vm_page_lookup(object,
-								cluster_start))
-				            && p->absent && p->busy
-				            && p != first_m) {
-				         VM_PAGE_FREE(p);
-				      }
-				   }
-				}
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return((rc == MACH_SEND_INTERRUPTED) ?
+
+				return ((rc == MACH_SEND_INTERRUPTED) ?
 					VM_FAULT_INTERRUPTED :
 					VM_FAULT_MEMORY_ERROR);
 			}
-			
-			vm_object_lock(object);
-			if ((interruptible != THREAD_UNINT) && 
-			    (current_thread()->state & TH_ABORT)) {
+			if ((interruptible != THREAD_UNINT) && (current_thread()->sched_mode & TH_MODE_ABORT)) {
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_INTERRUPTED);
+
+				return (VM_FAULT_INTERRUPTED);
 			}
-			if (m == VM_PAGE_NULL &&
-			    object->phys_contiguous) {
+			if (m == VM_PAGE_NULL && object->phys_contiguous) {
 				/*
 				 * No page here means that the object we
 				 * initially looked up was "physically 
@@ -1335,6 +1338,12 @@ no_clustering:
 				 */
 				break;
 			}
+			/*
+			 * potentially a pagein fault
+			 * if we make it through the state checks
+			 * above, than we'll count it as such
+			 */
+			my_fault = DBG_PAGEIN_FAULT;
 
 			/*
 			 * Retry with same object/offset, since new data may
@@ -1345,10 +1354,11 @@ no_clustering:
 		}
 
 		/*
-		 * The only case in which we get here is if
-		 * object has no pager (or unwiring).  If the pager doesn't
-		 * have the page this is handled in the m->absent case above
-		 * (and if you change things here you should look above).
+		 * We get here if the object has no pager, or an existence map 
+		 * exists and indicates the page isn't present on the pager
+		 * or we're unwiring a page.  If a pager exists, but there
+		 * is no existence map, then the m->absent case above handles
+		 * the ZF case when the pager can't provide the page
 		 */
 #if TRACEFAULTPAGE
 		dbgTrace(0xBEEF0014, (unsigned int) object, (unsigned int) m);	/* (TEST/DEBUG) */
@@ -1362,18 +1372,15 @@ no_clustering:
 		    "vm_f_page: no pager obj 0x%X, offset 0x%X, page 0x%X, next_obj 0x%X\n",
 			(integer_t)object, offset, (integer_t)m,
 			(integer_t)object->shadow, 0);
-		/*
-		 *	Move on to the next object.  Lock the next
-		 *	object before unlocking the current one.
-		 */
+
 		next_object = object->shadow;
+
 		if (next_object == VM_OBJECT_NULL) {
-			assert(!must_be_resident);
 			/*
-			 *	If there's no object left, fill the page
-			 *	in the top object with zeros.  But first we
-			 *	need to allocate a real page.
+			 * we've hit the bottom of the shadown chain,
+			 * fill the page in the top object with zeros.
 			 */
+			assert(!must_be_resident);
 
 			if (object != first_object) {
 				vm_object_paging_end(object);
@@ -1383,118 +1390,53 @@ no_clustering:
 				offset = first_offset;
 				vm_object_lock(object);
 			}
-
 			m = first_m;
 			assert(m->object == object);
 			first_m = VM_PAGE_NULL;
 
-			if(m == VM_PAGE_NULL) {
-				m = vm_page_grab();
-				if (m == VM_PAGE_NULL) {
-					vm_fault_cleanup(
-						object, VM_PAGE_NULL);
-					thread_interrupt_level(
-						interruptible_state);
-					return(VM_FAULT_MEMORY_SHORTAGE);
-				}
-				vm_page_insert(
-					m, object, offset);
-			}
-
-			if (object->shadow_severed) {
-				VM_PAGE_FREE(m);
-				vm_fault_cleanup(object, VM_PAGE_NULL);
-				thread_interrupt_level(interruptible_state);
-				return VM_FAULT_MEMORY_ERROR;
-			}
-
 			/*
-			 * are we protecting the system from
-			 * backing store exhaustion.  If so
-			 * sleep unless we are privileged.
+			 * check for any conditions that prevent
+			 * us from creating a new zero-fill page
+			 * vm_fault_check will do all of the 
+			 * fault cleanup in the case of an error condition
+			 * including resetting the thread_interrupt_level
 			 */
+			error = vm_fault_check(object, m, first_m, interruptible_state);
 
-			if(vm_backing_store_low) {
-				if(!(current_task()->priv_flags 
-						& VM_BACKING_STORE_PRIV)) {
-					assert_wait((event_t) 
-						&vm_backing_store_low, 
-						THREAD_UNINT);
-					VM_PAGE_FREE(m);
-					vm_fault_cleanup(object, VM_PAGE_NULL);
-					thread_block(THREAD_CONTINUE_NULL);
-					thread_interrupt_level(
-						interruptible_state);
-					return(VM_FAULT_RETRY);
-				}
-			}
+			if (error != VM_FAULT_SUCCESS)
+			        return (error);
 
-			if (VM_PAGE_THROTTLED() ||
-			    (m->fictitious && !vm_page_convert(m))) {
-				VM_PAGE_FREE(m);
-				vm_fault_cleanup(object, VM_PAGE_NULL);
-				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_MEMORY_SHORTAGE);
-			}
-			m->no_isync = FALSE;
+			if (m == VM_PAGE_NULL) {
+				m = vm_page_grab();
 
-			if (!no_zero_fill) {
-				vm_object_unlock(object);
-				vm_page_zero_fill(m);
-				vm_object_lock(object);
+				if (m == VM_PAGE_NULL) {
+					vm_fault_cleanup(object, VM_PAGE_NULL);
+					thread_interrupt_level(interruptible_state);
 
-				if (type_of_fault)
-				        *type_of_fault = DBG_ZERO_FILL_FAULT;
-				VM_STAT(zero_fill_count++);
-			}
-			if (bumped_pagein == TRUE) {
-			        VM_STAT(pageins--);
-				current_task()->pageins--;
-			}
-			vm_page_lock_queues();
-			VM_PAGE_QUEUES_REMOVE(m);
-			assert(!m->laundry);
-			assert(m->object != kernel_object);
-			assert(m->pageq.next == NULL &&
-			       m->pageq.prev == NULL);
-			if(m->object->size > 0x200000) {
-				m->zero_fill = TRUE;
-				/* depends on the queues lock */
-				vm_zf_count += 1;
-				queue_enter(&vm_page_queue_zf, 
-					m, vm_page_t, pageq);
-			} else {
-				queue_enter(
-					&vm_page_queue_inactive, 
-					m, vm_page_t, pageq);
-			}
-			m->page_ticket = vm_page_ticket;
-			vm_page_ticket_roll++;
-			if(vm_page_ticket_roll == VM_PAGE_TICKETS_IN_ROLL) {
-				vm_page_ticket_roll = 0;
-				if(vm_page_ticket == 
-					VM_PAGE_TICKET_ROLL_IDS)
-					vm_page_ticket= 0;
-				else
-					vm_page_ticket++;
+					return (VM_FAULT_MEMORY_SHORTAGE);
+				}
+				vm_page_insert(m, object, offset);
 			}
-			m->inactive = TRUE;
-			vm_page_inactive_count++;
-			vm_page_unlock_queues();
-#if 0
-			pmap_clear_modify(m->phys_page);
-#endif
+			my_fault = vm_fault_zero_page(m, no_zero_fill);
+
 			break;
-		}
-		else {
+
+		} else {
+		        /*
+			 * Move on to the next object.  Lock the next
+			 * object before unlocking the current one.
+			 */
 			if ((object != first_object) || must_be_resident)
 				vm_object_paging_end(object);
+
 			offset += object->shadow_offset;
-			hi_offset += object->shadow_offset;
-			lo_offset += object->shadow_offset;
+			fault_info->lo_offset += object->shadow_offset;
+			fault_info->hi_offset += object->shadow_offset;
 			access_required = VM_PROT_READ;
+
 			vm_object_lock(next_object);
 			vm_object_unlock(object);
+
 			object = next_object;
 			vm_object_paging_begin(object);
 		}
@@ -1522,7 +1464,7 @@ no_clustering:
 	dbgTrace(0xBEEF0015, (unsigned int) object, (unsigned int) m);	/* (TEST/DEBUG) */
 #endif
 #if	EXTRA_ASSERTIONS
-	if(m != VM_PAGE_NULL) {
+	if (m != VM_PAGE_NULL) {
 		assert(m->busy && !m->absent);
 		assert((first_m == VM_PAGE_NULL) ||
 			(first_m->busy && !first_m->absent &&
@@ -1540,28 +1482,28 @@ no_clustering:
 	}
 
 	XPR(XPR_VM_FAULT,
-       "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n",
+	    "vm_f_page: FOUND obj 0x%X, off 0x%X, page 0x%X, 1_obj 0x%X, 1_m 0x%X\n",
 		(integer_t)object, offset, (integer_t)m,
 		(integer_t)first_object, (integer_t)first_m);
+
 	/*
-	 *	If the page is being written, but isn't
-	 *	already owned by the top-level object,
-	 *	we have to copy it into a new page owned
-	 *	by the top-level object.
+	 * If the page is being written, but isn't
+	 * already owned by the top-level object,
+	 * we have to copy it into a new page owned
+	 * by the top-level object.
 	 */
-
 	if ((object != first_object) && (m != VM_PAGE_NULL)) {
-	    	/*
-		 *	We only really need to copy if we
-		 *	want to write it.
-		 */
 
 #if TRACEFAULTPAGE
-			dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type);	/* (TEST/DEBUG) */
+		dbgTrace(0xBEEF0016, (unsigned int) object, (unsigned int) fault_type);	/* (TEST/DEBUG) */
 #endif
 	    	if (fault_type & VM_PROT_WRITE) {
 			vm_page_t copy_m;
 
+			/*
+			 * We only really need to copy if we
+			 * want to write it.
+			 */
 			assert(!must_be_resident);
 
 			/*
@@ -1569,131 +1511,128 @@ no_clustering:
 			 * backing store exhaustion.  If so
 			 * sleep unless we are privileged.
 			 */
+			if (vm_backing_store_low) {
+				if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
 
-			if(vm_backing_store_low) {
-				if(!(current_task()->priv_flags 
-						& VM_BACKING_STORE_PRIV)) {
-					assert_wait((event_t) 
-						&vm_backing_store_low, 
-						THREAD_UNINT);
 					RELEASE_PAGE(m);
 					vm_fault_cleanup(object, first_m);
+
+					assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
+
 					thread_block(THREAD_CONTINUE_NULL);
-					thread_interrupt_level(
-						interruptible_state);
-					return(VM_FAULT_RETRY);
+					thread_interrupt_level(interruptible_state);
+
+					return (VM_FAULT_RETRY);
 				}
 			}
-
 			/*
-			 *	If we try to collapse first_object at this
-			 *	point, we may deadlock when we try to get
-			 *	the lock on an intermediate object (since we
-			 *	have the bottom object locked).  We can't
-			 *	unlock the bottom object, because the page
-			 *	we found may move (by collapse) if we do.
+			 * If we try to collapse first_object at this
+			 * point, we may deadlock when we try to get
+			 * the lock on an intermediate object (since we
+			 * have the bottom object locked).  We can't
+			 * unlock the bottom object, because the page
+			 * we found may move (by collapse) if we do.
 			 *
-			 *	Instead, we first copy the page.  Then, when
-			 *	we have no more use for the bottom object,
-			 *	we unlock it and try to collapse.
+			 * Instead, we first copy the page.  Then, when
+			 * we have no more use for the bottom object,
+			 * we unlock it and try to collapse.
 			 *
-			 *	Note that we copy the page even if we didn't
-			 *	need to... that's the breaks.
+			 * Note that we copy the page even if we didn't
+			 * need to... that's the breaks.
 			 */
 
 			/*
-			 *	Allocate a page for the copy
+			 * Allocate a page for the copy
 			 */
 			copy_m = vm_page_grab();
+
 			if (copy_m == VM_PAGE_NULL) {
 				RELEASE_PAGE(m);
+
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_MEMORY_SHORTAGE);
-			}
-
 
+				return (VM_FAULT_MEMORY_SHORTAGE);
+			}
 			XPR(XPR_VM_FAULT,
 			    "vm_f_page: page_copy obj 0x%X, offset 0x%X, m 0x%X, copy_m 0x%X\n",
 				(integer_t)object, offset,
 				(integer_t)m, (integer_t)copy_m, 0);
+
 			vm_page_copy(m, copy_m);
 
 			/*
-			 *	If another map is truly sharing this
-			 *	page with us, we have to flush all
-			 *	uses of the original page, since we
-			 *	can't distinguish those which want the
-			 *	original from those which need the
-			 *	new copy.
+			 * If another map is truly sharing this
+			 * page with us, we have to flush all
+			 * uses of the original page, since we
+			 * can't distinguish those which want the
+			 * original from those which need the
+			 * new copy.
 			 *
-			 *	XXXO If we know that only one map has
-			 *	access to this page, then we could
-			 *	avoid the pmap_disconnect() call.
+			 * XXXO If we know that only one map has
+			 * access to this page, then we could
+			 * avoid the pmap_disconnect() call.
 			 */
+			if (m->pmapped)
+			        pmap_disconnect(m->phys_page);
 
-			vm_page_lock_queues();
 			assert(!m->cleaning);
-			pmap_disconnect(m->phys_page);
-			vm_page_deactivate(m);
-			copy_m->dirty = TRUE;
-			/*
-			 * Setting reference here prevents this fault from
-			 * being counted as a (per-thread) reactivate as well
-			 * as a copy-on-write.
-			 */
-			first_m->reference = TRUE;
-			vm_page_unlock_queues();
 
 			/*
-			 *	We no longer need the old page or object.
+			 * We no longer need the old page or object.
 			 */
-
 			PAGE_WAKEUP_DONE(m);
 			vm_object_paging_end(object);
 			vm_object_unlock(object);
 
-			if (type_of_fault)
-			        *type_of_fault = DBG_COW_FAULT;
-			VM_STAT(cow_faults++);
+			my_fault = DBG_COW_FAULT;
+			VM_STAT_INCR(cow_faults);
+			DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
 			current_task()->cow_faults++;
+
 			object = first_object;
 			offset = first_offset;
 
 			vm_object_lock(object);
+			/*
+			 * get rid of the place holder
+			 * page that we soldered in earlier
+			 */
 			VM_PAGE_FREE(first_m);
 			first_m = VM_PAGE_NULL;
+			
+			/*
+			 * and replace it with the
+			 * page we just copied into
+			 */
 			assert(copy_m->busy);
 			vm_page_insert(copy_m, object, offset);
-			m = copy_m;
+			copy_m->dirty = TRUE;
 
+			m = copy_m;
 			/*
-			 *      Now that we've gotten the copy out of the
-			 *      way, let's try to collapse the top object.
-			 *      But we have to play ugly games with
-			 *      paging_in_progress to do that...
+			 * Now that we've gotten the copy out of the
+			 * way, let's try to collapse the top object.
+			 * But we have to play ugly games with
+			 * paging_in_progress to do that...
 			 */     
-
 			vm_object_paging_end(object); 
 			vm_object_collapse(object, offset, TRUE);
 			vm_object_paging_begin(object);
 
-		}
-		else {
+		} else
 		    	*protection &= (~VM_PROT_WRITE);
-		}
 	}
-
 	/*
-	 *	Now check whether the page needs to be pushed into the
-	 *	copy object.  The use of asymmetric copy on write for
-	 *	shared temporary objects means that we may do two copies to
-	 *	satisfy the fault; one above to get the page from a
-	 *	shadowed object, and one here to push it into the copy.
+	 * Now check whether the page needs to be pushed into the
+	 * copy object.  The use of asymmetric copy on write for
+	 * shared temporary objects means that we may do two copies to
+	 * satisfy the fault; one above to get the page from a
+	 * shadowed object, and one here to push it into the copy.
 	 */
+	try_failed_count = 0;
 
-	while ((copy_object = first_object->copy) != VM_OBJECT_NULL &&
-		   (m!= VM_PAGE_NULL)) {
+	while ((copy_object = first_object->copy) != VM_OBJECT_NULL && (m != VM_PAGE_NULL)) {
 		vm_object_offset_t	copy_offset;
 		vm_page_t		copy_m;
 
@@ -1701,72 +1640,76 @@ no_clustering:
 		dbgTrace(0xBEEF0017, (unsigned int) copy_object, (unsigned int) fault_type);	/* (TEST/DEBUG) */
 #endif
 		/*
-		 *	If the page is being written, but hasn't been
-		 *	copied to the copy-object, we have to copy it there.
+		 * If the page is being written, but hasn't been
+		 * copied to the copy-object, we have to copy it there.
 		 */
-
 		if ((fault_type & VM_PROT_WRITE) == 0) {
 			*protection &= ~VM_PROT_WRITE;
 			break;
 		}
 
 		/*
-		 *	If the page was guaranteed to be resident,
-		 *	we must have already performed the copy.
+		 * If the page was guaranteed to be resident,
+		 * we must have already performed the copy.
 		 */
-
 		if (must_be_resident)
 			break;
 
 		/*
-		 *	Try to get the lock on the copy_object.
+		 * Try to get the lock on the copy_object.
 		 */
 		if (!vm_object_lock_try(copy_object)) {
-			vm_object_unlock(object);
 
-			mutex_pause();	/* wait a bit */
+			vm_object_unlock(object);
+			try_failed_count++;
 
+			mutex_pause(try_failed_count);	/* wait a bit */
 			vm_object_lock(object);
+
 			continue;
 		}
+		try_failed_count = 0;
 
 		/*
-		 *	Make another reference to the copy-object,
-		 *	to keep it from disappearing during the
-		 *	copy.
+		 * Make another reference to the copy-object,
+		 * to keep it from disappearing during the
+		 * copy.
 		 */
-		assert(copy_object->ref_count > 0);
-		copy_object->ref_count++;
-		VM_OBJ_RES_INCR(copy_object);
+		vm_object_reference_locked(copy_object);
 
 		/*
-		 *	Does the page exist in the copy?
+		 * Does the page exist in the copy?
 		 */
 		copy_offset = first_offset - copy_object->shadow_offset;
+
 		if (copy_object->size <= copy_offset)
 			/*
 			 * Copy object doesn't cover this page -- do nothing.
 			 */
 			;
-		else if ((copy_m =
-			vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) {
-			/* Page currently exists in the copy object */
+		else if ((copy_m = vm_page_lookup(copy_object, copy_offset)) != VM_PAGE_NULL) {
+			/*
+			 * Page currently exists in the copy object
+			 */
 			if (copy_m->busy) {
 				/*
-				 *	If the page is being brought
-				 *	in, wait for it and then retry.
+				 * If the page is being brought
+				 * in, wait for it and then retry.
 				 */
 				RELEASE_PAGE(m);
-				/* take an extra ref so object won't die */
-				assert(copy_object->ref_count > 0);
-				copy_object->ref_count++;
-				vm_object_res_reference(copy_object);
+
+				/*
+				 * take an extra ref so object won't die
+				 */
+				vm_object_reference_locked(copy_object);
 				vm_object_unlock(copy_object);
 				vm_fault_cleanup(object, first_m);
 				counter(c_vm_fault_page_block_backoff_kernel++);
+
 				vm_object_lock(copy_object);
 				assert(copy_object->ref_count > 0);
 				VM_OBJ_RES_DECR(copy_object);
+				vm_object_lock_assert_exclusive(copy_object);
 				copy_object->ref_count--;
 				assert(copy_object->ref_count > 0);
 				copy_m = vm_page_lookup(copy_object, copy_offset);
@@ -1778,15 +1721,18 @@ no_clustering:
 				 */
 				if (copy_m != VM_PAGE_NULL && copy_m->busy) {
 					PAGE_ASSERT_WAIT(copy_m, interruptible);
+
 					vm_object_unlock(copy_object);
 					wait_result = thread_block(THREAD_CONTINUE_NULL);
 					vm_object_deallocate(copy_object);
+
 					goto backoff;
 				} else {
 					vm_object_unlock(copy_object);
 					vm_object_deallocate(copy_object);
 					thread_interrupt_level(interruptible_state);
-					return VM_FAULT_RETRY;
+
+					return (VM_FAULT_RETRY);
 				}
 			}
 		}
@@ -1801,301 +1747,410 @@ no_clustering:
 			 * We must copy the page to the copy object.
 			 */
 
-			/*
-			 * are we protecting the system from
-			 * backing store exhaustion.  If so
-			 * sleep unless we are privileged.
-			 */
+			if (vm_backing_store_low) {
+			        /*
+				 * we are protecting the system from
+				 * backing store exhaustion.  If so
+				 * sleep unless we are privileged.
+				 */
+				if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV)) {
+					assert_wait((event_t)&vm_backing_store_low, THREAD_UNINT);
 
-			if(vm_backing_store_low) {
-				if(!(current_task()->priv_flags 
-						& VM_BACKING_STORE_PRIV)) {
-					assert_wait((event_t) 
-						&vm_backing_store_low, 
-						THREAD_UNINT);
 					RELEASE_PAGE(m);
 					VM_OBJ_RES_DECR(copy_object);
+					vm_object_lock_assert_exclusive(copy_object);
 					copy_object->ref_count--;
 					assert(copy_object->ref_count > 0);
+
 					vm_object_unlock(copy_object);
 					vm_fault_cleanup(object, first_m);
 					thread_block(THREAD_CONTINUE_NULL);
-					thread_interrupt_level(
-						interruptible_state);
-					return(VM_FAULT_RETRY);
+					thread_interrupt_level(interruptible_state);
+
+					return (VM_FAULT_RETRY);
 				}
 			}
-
 			/*
-			 *	Allocate a page for the copy
+			 * Allocate a page for the copy
 			 */
 			copy_m = vm_page_alloc(copy_object, copy_offset);
+
 			if (copy_m == VM_PAGE_NULL) {
 				RELEASE_PAGE(m);
+
 				VM_OBJ_RES_DECR(copy_object);
+				vm_object_lock_assert_exclusive(copy_object);
 				copy_object->ref_count--;
 				assert(copy_object->ref_count > 0);
+
 				vm_object_unlock(copy_object);
 				vm_fault_cleanup(object, first_m);
 				thread_interrupt_level(interruptible_state);
-				return(VM_FAULT_MEMORY_SHORTAGE);
-			}
 
+				return (VM_FAULT_MEMORY_SHORTAGE);
+			}
 			/*
-			 *	Must copy page into copy-object.
+			 * Must copy page into copy-object.
 			 */
-
 			vm_page_copy(m, copy_m);
 			
 			/*
-			 *	If the old page was in use by any users
-			 *	of the copy-object, it must be removed
-			 *	from all pmaps.  (We can't know which
-			 *	pmaps use it.)
+			 * If the old page was in use by any users
+			 * of the copy-object, it must be removed
+			 * from all pmaps.  (We can't know which
+			 * pmaps use it.)
 			 */
-
-			vm_page_lock_queues();
-			assert(!m->cleaning);
-			pmap_disconnect(m->phys_page);
-			copy_m->dirty = TRUE;
-			vm_page_unlock_queues();
+			if (m->pmapped)
+			        pmap_disconnect(m->phys_page);
 
 			/*
-			 *	If there's a pager, then immediately
-			 *	page out this page, using the "initialize"
-			 *	option.  Else, we use the copy.
+			 * If there's a pager, then immediately
+			 * page out this page, using the "initialize"
+			 * option.  Else, we use the copy.
 			 */
-
-		 	if 
-#if	MACH_PAGEMAP
-			  ((!copy_object->pager_created) ||
-				vm_external_state_get(
-					copy_object->existence_map, copy_offset)
-				== VM_EXTERNAL_STATE_ABSENT)
-#else
-			  (!copy_object->pager_created)
+		 	if ((!copy_object->pager_created)
+#if MACH_PAGEMAP
+			    || vm_external_state_get(copy_object->existence_map, copy_offset) == VM_EXTERNAL_STATE_ABSENT
 #endif
-				{
-				vm_page_lock_queues();
+			    ) {
+
+				vm_page_lockspin_queues();
+				assert(!m->cleaning);
 				vm_page_activate(copy_m);
 				vm_page_unlock_queues();
+
+				copy_m->dirty = TRUE;
 				PAGE_WAKEUP_DONE(copy_m);
 			} 
 			else {
 				assert(copy_m->busy == TRUE);
+				assert(!m->cleaning);
 
 				/*
-				 *	The page is already ready for pageout:
-				 *	not on pageout queues and busy.
-				 *	Unlock everything except the
-				 *	copy_object itself.
+				 * dirty is protected by the object lock
 				 */
+				copy_m->dirty = TRUE;
 
+				/*
+				 * The page is already ready for pageout:
+				 * not on pageout queues and busy.
+				 * Unlock everything except the
+				 * copy_object itself.
+				 */
 				vm_object_unlock(object);
 
 				/*
-				 *	Write the page to the copy-object,
-				 *	flushing it from the kernel.
+				 * Write the page to the copy-object,
+				 * flushing it from the kernel.
 				 */
-
 				vm_pageout_initialize_page(copy_m);
 
 				/*
-				 *	Since the pageout may have
-				 *	temporarily dropped the
-				 *	copy_object's lock, we
-				 *	check whether we'll have
-				 *	to deallocate the hard way.
+				 * Since the pageout may have
+				 * temporarily dropped the
+				 * copy_object's lock, we
+				 * check whether we'll have
+				 * to deallocate the hard way.
 				 */
-
-				if ((copy_object->shadow != object) ||
-				    (copy_object->ref_count == 1)) {
+				if ((copy_object->shadow != object) || (copy_object->ref_count == 1)) {
 					vm_object_unlock(copy_object);
 					vm_object_deallocate(copy_object);
 					vm_object_lock(object);
+
 					continue;
 				}
-
 				/*
-				 *	Pick back up the old object's
-				 *	lock.  [It is safe to do so,
-				 *	since it must be deeper in the
-				 *	object tree.]
+				 * Pick back up the old object's
+				 * lock.  [It is safe to do so,
+				 * since it must be deeper in the
+				 * object tree.]
 				 */
-
 				vm_object_lock(object);
 			}
-
 			/*
-			 *	Because we're pushing a page upward
-			 *	in the object tree, we must restart
-			 *	any faults that are waiting here.
-			 *	[Note that this is an expansion of
-			 *	PAGE_WAKEUP that uses the THREAD_RESTART
-			 *	wait result].  Can't turn off the page's
-			 *	busy bit because we're not done with it.
+			 * Because we're pushing a page upward
+			 * in the object tree, we must restart
+			 * any faults that are waiting here.
+			 * [Note that this is an expansion of
+			 * PAGE_WAKEUP that uses the THREAD_RESTART
+			 * wait result].  Can't turn off the page's
+			 * busy bit because we're not done with it.
 			 */
-			 
 			if (m->wanted) {
 				m->wanted = FALSE;
-				thread_wakeup_with_result((event_t) m,
-					THREAD_RESTART);
+				thread_wakeup_with_result((event_t) m, THREAD_RESTART);
 			}
 		}
-
 		/*
-		 *	The reference count on copy_object must be
-		 *	at least 2: one for our extra reference,
-		 *	and at least one from the outside world
-		 *	(we checked that when we last locked
-		 *	copy_object).
+		 * The reference count on copy_object must be
+		 * at least 2: one for our extra reference,
+		 * and at least one from the outside world
+		 * (we checked that when we last locked
+		 * copy_object).
 		 */
+		vm_object_lock_assert_exclusive(copy_object);
 		copy_object->ref_count--;
 		assert(copy_object->ref_count > 0);
+
 		VM_OBJ_RES_DECR(copy_object);	
 		vm_object_unlock(copy_object);
 
 		break;
 	}
-
 	*result_page = m;
 	*top_page = first_m;
 
 	XPR(XPR_VM_FAULT,
 		"vm_f_page: DONE obj 0x%X, offset 0x%X, m 0x%X, first_m 0x%X\n",
 		(integer_t)object, offset, (integer_t)m, (integer_t)first_m, 0);
-	/*
-	 *	If the page can be written, assume that it will be.
-	 *	[Earlier, we restrict the permission to allow write
-	 *	access only if the fault so required, so we don't
-	 *	mark read-only data as dirty.]
-	 */
 
+	if (m != VM_PAGE_NULL) {
+		if (my_fault == DBG_PAGEIN_FAULT) {
 
-	if(m != VM_PAGE_NULL) {
-#if	!VM_FAULT_STATIC_CONFIG
-		if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
-			m->dirty = TRUE;
-#endif
-		if (vm_page_deactivate_behind)
-			vm_fault_deactivate_behind(object, offset, behavior);
-	} else {
+			VM_STAT_INCR(pageins);
+			DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL);
+			DTRACE_VM2(maj_fault, int, 1, (uint64_t *), NULL);
+			current_task()->pageins++;
+
+			if (m->object->internal) {
+				DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL);
+			} else {
+				DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL);
+			}
+
+		        /*
+			 * evaluate access pattern and update state
+			 * vm_fault_deactivate_behind depends on the
+			 * state being up to date
+			 */
+		        vm_fault_is_sequential(object, offset, fault_info->behavior);
+
+			vm_fault_deactivate_behind(object, offset, fault_info->behavior);
+		}
+		if (type_of_fault)
+		        *type_of_fault = my_fault;
+	} else
 		vm_object_unlock(object);
-	}
+
 	thread_interrupt_level(interruptible_state);
 
 #if TRACEFAULTPAGE
 	dbgTrace(0xBEEF001A, (unsigned int) VM_FAULT_SUCCESS, 0);	/* (TEST/DEBUG) */
 #endif
-	return(VM_FAULT_SUCCESS);
+	return (VM_FAULT_SUCCESS);
 
-#if 0
-    block_and_backoff:
-	vm_fault_cleanup(object, first_m);
-
-	counter(c_vm_fault_page_block_backoff_kernel++);
-	thread_block(THREAD_CONTINUE_NULL);
-#endif
-
-    backoff:
+backoff:
 	thread_interrupt_level(interruptible_state);
+
 	if (wait_result == THREAD_INTERRUPTED)
-		return VM_FAULT_INTERRUPTED;
-	return VM_FAULT_RETRY;
+		return (VM_FAULT_INTERRUPTED);
+	return (VM_FAULT_RETRY);
 
 #undef	RELEASE_PAGE
 }
 
+
+
 /*
- *	Routine:	vm_fault_tws_insert
- *	Purpose:
- *		Add fault information to the task working set.
- *	Implementation:
- *		We always insert the base object/offset pair
- *		rather the actual object/offset.
- *	Assumptions:
- *		Map and real_map locked.
- *		Object locked and referenced.
- *	Returns:
- *		TRUE if startup file should be written.
- *		With object locked and still referenced.
- *		But we may drop the object lock temporarily.
+ * page queue lock must NOT be held
+ * m->object must be locked
+ *
+ * NOTE: m->object could be locked "shared" only if we are called
+ * from vm_fault() as part of a soft fault.  If so, we must be
+ * careful not to modify the VM object in any way that is not
+ * legal under a shared lock...
  */
-static boolean_t
-vm_fault_tws_insert(
-	vm_map_t map,
-	vm_map_t real_map,
-	vm_map_offset_t vaddr,
-	vm_object_t object,
-	vm_object_offset_t offset)
+unsigned long cs_enter_tainted_rejected = 0;
+unsigned long cs_enter_tainted_accepted = 0;
+kern_return_t
+vm_fault_enter(vm_page_t m,
+	       pmap_t pmap,
+	       vm_map_offset_t vaddr,
+	       vm_prot_t prot,
+	       boolean_t wired,
+	       boolean_t change_wiring,
+	       boolean_t no_cache,
+	       int *type_of_fault)
 {
-	tws_hash_line_t	line;
-	task_t		task;
+	unsigned int	cache_attr;
 	kern_return_t	kr;
-	boolean_t	result = FALSE;
-
-	/* Avoid possible map lock deadlock issues */
-	if (map == kernel_map || map == kalloc_map ||
-	    real_map == kernel_map || real_map == kalloc_map)
-		return result;
-
-	task = current_task();
-	if (task->dynamic_working_set != 0) {
-		vm_object_t	base_object;
-		vm_object_t	base_shadow;
-		vm_object_offset_t base_offset;
-		base_object = object;
-		base_offset = offset;
-		while ((base_shadow = base_object->shadow)) {
-			vm_object_lock(base_shadow);
-			vm_object_unlock(base_object);
-			base_offset +=
-				base_object->shadow_offset;
-			base_object = base_shadow;
-		}
-		kr = tws_lookup(
-			task->dynamic_working_set,
-			base_offset, base_object, 
-			&line);
-		if (kr == KERN_OPERATION_TIMED_OUT){
-			result = TRUE;
-			if (base_object != object) {
-				vm_object_unlock(base_object);
-				vm_object_lock(object);
+	boolean_t	previously_pmapped = m->pmapped;
+
+	vm_object_lock_assert_held(m->object);
+#if DEBUG
+	mutex_assert(&vm_page_queue_lock, MA_NOTOWNED);
+#endif /* DEBUG */
+
+	if (m->phys_page == vm_page_guard_addr) {
+		assert(m->fictitious);
+		return KERN_SUCCESS;
+	}
+
+        cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
+
+	if (m->object->code_signed && !m->cs_validated &&
+	    pmap != kernel_pmap) {
+		/*
+		 * CODE SIGNING:
+		 * This page comes from a VM object backed by a
+		 * signed memory object and it hasn't been validated yet.
+		 * We're about to enter it into a process address space,
+		 * so we need to validate its signature now.
+		 */
+		vm_object_lock_assert_exclusive(m->object);
+
+		/* VM map still locked, so 1 ref will remain on VM object */
+
+		vm_page_validate_cs(m);
+	}
+
+	if (m->pmapped == FALSE) {
+		/*
+		 * This is the first time this page is being
+		 * mapped in an address space (pmapped == FALSE).
+		 *
+		 * Part of that page may still be in the data cache
+		 * and not flushed to memory.  In case we end up
+		 * accessing that page via the instruction cache,
+		 * we need to ensure that the 2 caches are in sync.
+		 */
+		pmap_sync_page_data_phys(m->phys_page);
+
+		if ((*type_of_fault == DBG_CACHE_HIT_FAULT) && m->clustered) {
+		        /*
+			 * found it in the cache, but this
+			 * is the first fault-in of the page (m->pmapped == FALSE)
+			 * so it must have come in as part of
+			 * a cluster... account 1 pagein against it
+			 */
+		        VM_STAT_INCR(pageins);
+			DTRACE_VM2(pgin, int, 1, (uint64_t *), NULL);
+
+			if (m->object->internal) {
+				DTRACE_VM2(anonpgin, int, 1, (uint64_t *), NULL);
+			} else {
+				DTRACE_VM2(fspgin, int, 1, (uint64_t *), NULL);
 			}
-		} else if (kr != KERN_SUCCESS) {
-			if(base_object != object)
-				vm_object_reference_locked(base_object);
-		   	kr = tws_insert(
-				   task->dynamic_working_set,
-				   base_offset, base_object,
-				   vaddr, real_map);
-			if(base_object != object) {
-				vm_object_unlock(base_object);
-				vm_object_deallocate(base_object);
+
+			current_task()->pageins++;
+
+			*type_of_fault = DBG_PAGEIN_FAULT;
+		}
+		VM_PAGE_CONSUME_CLUSTERED(m);
+
+	} else if (cache_attr != VM_WIMG_DEFAULT)
+	        pmap_sync_page_attributes_phys(m->phys_page);
+
+	if (*type_of_fault != DBG_COW_FAULT) {
+		DTRACE_VM2(as_fault, int, 1, (uint64_t *), NULL);
+
+		if (pmap == kernel_pmap) {
+			DTRACE_VM2(kernel_asflt, int, 1, (uint64_t *), NULL);
+		}
+	}
+
+	if (m->cs_tainted) {
+		/*
+		 * CODE SIGNING:
+		 * This page has been tainted and can not be trusted.
+		 * Let's notify the current process and let it take any
+		 * necessary precautions before we enter the tainted page
+		 * into its address space.
+		 */
+		if (cs_invalid_page()) {
+			/* reject the tainted page: abort the page fault */
+			kr = KERN_MEMORY_ERROR;
+			cs_enter_tainted_rejected++;
+		} else {
+			/* proceed with the tainted page */
+			kr = KERN_SUCCESS;
+			cs_enter_tainted_accepted++;
+		}
+		if (cs_debug || kr != KERN_SUCCESS) {
+			printf("CODESIGNING: vm_fault_enter(0x%llx): "
+			       "page %p obj %p off 0x%llx *** TAINTED ***\n",
+			       (long long)vaddr, m, m->object, m->offset);
+		}
+	} else {
+		/* proceed with the valid page */
+		kr = KERN_SUCCESS;
+	}
+
+	if (kr == KERN_SUCCESS) {
+	        /*
+		 * NOTE: we may only hold the vm_object lock SHARED
+		 * at this point, but the update of pmapped is ok
+		 * since this is the ONLY bit updated behind the SHARED
+		 * lock... however, we need to figure out how to do an atomic
+		 * update on a bit field to make this less fragile... right
+		 * now I don'w know how to coerce 'C' to give me the offset info
+		 * that's needed for an AtomicCompareAndSwap
+		 */
+		m->pmapped = TRUE;
+
+		PMAP_ENTER(pmap, vaddr, m, prot, cache_attr, wired);
+	}
+
+	/*
+	 * Hold queues lock to manipulate
+	 * the page queues.  Change wiring
+	 * case is obvious.
+	 */
+	if (change_wiring) {
+	        vm_page_lockspin_queues();
+
+		if (wired) {
+			if (kr == KERN_SUCCESS) {
+				vm_page_wire(m);
 			}
-		   	if(kr == KERN_NO_SPACE) {
-				if (base_object == object)
-					vm_object_unlock(object);
-				tws_expand_working_set(
-			 	   task->dynamic_working_set, 
-				   TWS_HASH_LINE_COUNT,
-				   FALSE);
-				if (base_object == object)
-					vm_object_lock(object);
-			} else if(kr == KERN_OPERATION_TIMED_OUT) {
-				result = TRUE;
+		} else {
+		        vm_page_unwire(m);
+		}
+		vm_page_unlock_queues();
+
+	} else {
+	        if (kr != KERN_SUCCESS) {
+		        vm_page_lock_queues();
+		        vm_page_deactivate(m);
+		        vm_page_unlock_queues();
+		} else {
+		        if (((!m->active && !m->inactive) || no_cache) && !m->wire_count && !m->throttled) {
+			        vm_page_lockspin_queues();
+				/*
+				 * test again now that we hold the page queue lock
+				 */
+				if (((!m->active && !m->inactive) || no_cache) && !m->wire_count) {
+
+					/*
+					 * If this is a no_cache mapping and the page has never been
+					 * mapped before or was previously a no_cache page, then we
+					 * want to leave pages in the speculative state so that they
+					 * can be readily recycled if free memory runs low.  Otherwise
+					 * the page is activated as normal. 
+					 */
+
+					if (no_cache && (!previously_pmapped || m->no_cache)) {
+						m->no_cache = TRUE;
+
+						if (m->active || m->inactive)
+							VM_PAGE_QUEUES_REMOVE(m);
+
+						if (!m->speculative) 
+							vm_page_speculate(m, TRUE);
+
+					} else if (!m->active && !m->inactive)
+						vm_page_activate(m);
+
+				}
+
+				vm_page_unlock_queues();
 			}
-			if(base_object != object)
-				vm_object_lock(object);
-		} else if (base_object != object) {
-			vm_object_unlock(base_object);
-			vm_object_lock(object);
 		}
 	}
-	return result;
+	return kr;
 }
 
+
 /*
  *	Routine:	vm_fault
  *	Purpose:
@@ -2112,6 +2167,9 @@ vm_fault_tws_insert(
 
 extern int _map_enter_debug;
 
+unsigned long vm_fault_collapse_total = 0;
+unsigned long vm_fault_collapse_skipped = 0;
+
 kern_return_t
 vm_fault(
 	vm_map_t	map,
@@ -2127,53 +2185,60 @@ vm_fault(
 	vm_object_t		object;		/* Top-level object */
 	vm_object_offset_t	offset;		/* Top-level offset */
 	vm_prot_t		prot;		/* Protection for mapping */
-	vm_behavior_t		behavior;	/* Expected paging behavior */
-	vm_map_offset_t		lo_offset, hi_offset;
 	vm_object_t		old_copy_object; /* Saved copy object */
 	vm_page_t		result_page;	/* Result of vm_fault_page */
 	vm_page_t		top_page;	/* Placeholder page */
 	kern_return_t		kr;
 
-	register
 	vm_page_t		m;	/* Fast access to result_page */
-	kern_return_t		error_code = 0;	/* page error reasons */
-	register
+	kern_return_t		error_code;
 	vm_object_t		cur_object;
-	register
 	vm_object_offset_t	cur_offset;
 	vm_page_t		cur_m;
 	vm_object_t		new_object;
 	int                     type_of_fault;
+	pmap_t			pmap;
+	boolean_t		interruptible_state;
 	vm_map_t		real_map = map;
 	vm_map_t		original_map = map;
-	pmap_t			pmap = NULL;
-	boolean_t		interruptible_state;
-	unsigned int		cache_attr;
-	int			write_startup_file = 0;
-	boolean_t		need_activation;
 	vm_prot_t		original_fault_type;
+	struct vm_object_fault_info fault_info;
+	boolean_t		need_collapse = FALSE;
+	int			object_lock_type = 0;
+	int			cur_object_lock_type;
 
 
-	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_START,
-			      vaddr,
-			      0,
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_START,
+			      (int)((uint64_t)vaddr >> 32),
+			      (int)vaddr,
 			      0,
 			      0,
 			      0);
 
 	if (get_preemption_level() != 0) {
-	        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END,
-				      vaddr,
-				      0,
+	        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
+				      (int)((uint64_t)vaddr >> 32),
+				      (int)vaddr,
 				      KERN_FAILURE,
 				      0,
 				      0);
 
 		return (KERN_FAILURE);
 	}
-
 	interruptible_state = thread_interrupt_level(interruptible);
 
+	VM_STAT_INCR(faults);
+	current_task()->faults++;
+	original_fault_type = fault_type;
+
+	if (fault_type & VM_PROT_WRITE)
+	        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+	else
+	        object_lock_type = OBJECT_LOCK_SHARED;
+
+	cur_object_lock_type = OBJECT_LOCK_SHARED;
+
+RetryFault:
 	/*
 	 * assume we will hit a page in the cache
 	 * otherwise, explicitly override with
@@ -2181,13 +2246,6 @@ vm_fault(
 	 */
 	type_of_fault = DBG_CACHE_HIT_FAULT;
 
-	VM_STAT(faults++);
-	current_task()->faults++;
-
-	original_fault_type = fault_type;
-
-    RetryFault: ;
-
 	/*
 	 *	Find the backing store object and offset into
 	 *	it to begin the search.
@@ -2195,27 +2253,44 @@ vm_fault(
 	fault_type = original_fault_type;
 	map = original_map;
 	vm_map_lock_read(map);
-	kr = vm_map_lookup_locked(&map, vaddr, fault_type, &version,
-				&object, &offset,
-				&prot, &wired,
-				&behavior, &lo_offset, &hi_offset, &real_map);
-
-//if (_map_enter_debug)printf("vm_map_lookup_locked(map=0x%x, addr=0x%llx, prot=%d wired=%d) = %d\n", map, vaddr, prot, wired, kr);
 
-	pmap = real_map->pmap;
+	kr = vm_map_lookup_locked(&map, vaddr, fault_type,
+				  object_lock_type, &version,
+				  &object, &offset, &prot, &wired,
+				  &fault_info,
+				  &real_map);
 
 	if (kr != KERN_SUCCESS) {
 		vm_map_unlock_read(map);
 		goto done;
 	}
+	pmap = real_map->pmap;
+	fault_info.interruptible = interruptible;
 
 	/*
-	 *	If the page is wired, we must fault for the current protection
-	 *	value, to avoid further faults.
+	 * If the page is wired, we must fault for the current protection
+	 * value, to avoid further faults.
 	 */
-
-	if (wired)
+	if (wired) {
 		fault_type = prot | VM_PROT_WRITE;
+	
+		/*
+		 * since we're treating this fault as a 'write'
+		 * we must hold the top object lock exclusively
+		 */
+		if (object_lock_type == OBJECT_LOCK_SHARED) {
+
+		        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+			if (vm_object_lock_upgrade(object) == FALSE) {
+			        /*
+				 * couldn't upgrade, so explictly
+				 * take the lock exclusively
+				 */
+			        vm_object_lock(object);
+			}
+		}
+	}
 
 #if	VM_FAULT_CLASSIFY
 	/*
@@ -2257,33 +2332,73 @@ vm_fault(
 
 
 	/*
-	 *	If this page is to be inserted in a copy delay object
-	 *	for writing, and if the object has a copy, then the
-	 *	copy delay strategy is implemented in the slow fault page.
+	 * If this page is to be inserted in a copy delay object
+	 * for writing, and if the object has a copy, then the
+	 * copy delay strategy is implemented in the slow fault page.
 	 */
-	if (object->copy_strategy != MEMORY_OBJECT_COPY_DELAY ||
-	    object->copy == VM_OBJECT_NULL ||
-	    (fault_type & VM_PROT_WRITE) == 0) {
+	if (object->copy_strategy == MEMORY_OBJECT_COPY_DELAY &&
+	    object->copy != VM_OBJECT_NULL && (fault_type & VM_PROT_WRITE))
+	        goto handle_copy_delay;
+
 	cur_object = object;
 	cur_offset = offset;
 
 	while (TRUE) {
 		m = vm_page_lookup(cur_object, cur_offset);
+
 		if (m != VM_PAGE_NULL) {
 			if (m->busy) {
 			        wait_result_t	result;
 
-				if (object != cur_object)
+				/*
+				 * in order to do the PAGE_ASSERT_WAIT, we must
+				 * have object that 'm' belongs to locked exclusively
+				 */
+				if (object != cur_object) {
 				        vm_object_unlock(object);
 
+					if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
+
+					        cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+						if (vm_object_lock_upgrade(cur_object) == FALSE) {
+						        /*
+							 * couldn't upgrade so go do a full retry
+							 * immediately since we've already dropped
+							 * the top object lock associated with this page
+							 * and the current one got dropped due to the
+							 * failed upgrade... the state is no longer valid
+							 */
+						        vm_map_unlock_read(map);
+							if (real_map != map)
+							        vm_map_unlock(real_map);
+
+							goto RetryFault;
+						}
+					}
+				} else if (object_lock_type == OBJECT_LOCK_SHARED) {
+
+				        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+					if (vm_object_lock_upgrade(object) == FALSE) {
+					        /*
+						 * couldn't upgrade, so explictly take the lock
+						 * exclusively and go relookup the page since we
+						 * will have dropped the object lock and
+						 * a different thread could have inserted
+						 * a page at this offset
+						 * no need for a full retry since we're
+						 * at the top level of the object chain
+						 */
+					        vm_object_lock(object);
+
+						continue;
+					}
+				}
 				vm_map_unlock_read(map);
 				if (real_map != map)
 				        vm_map_unlock(real_map);
 
-#if	!VM_FAULT_STATIC_CONFIG
-				if (!vm_fault_interruptible)
-				        interruptible = THREAD_UNINT;
-#endif
 				result = PAGE_ASSERT_WAIT(m, interruptible);
 
 				vm_object_unlock(cur_object);
@@ -2299,29 +2414,74 @@ vm_fault(
 				kr = KERN_ABORTED;
 				goto done;
 			}
-			if (m->unusual && (m->error || m->restart || m->private
-			    || m->absent || (fault_type & m->page_lock))) {
-
+			if (m->phys_page == vm_page_guard_addr) {
+				/*
+				 * Guard page: let the slow path deal with it
+				 */
+				break;
+			}
+			if (m->unusual && (m->error || m->restart || m->private || m->absent)) {
 			        /*
-				 *	Unusual case. Give up.
+				 * Unusual case... let the slow path deal with it
 				 */
 				break;
 			}
-
 			if (m->encrypted) {
 				/*
 				 * ENCRYPTED SWAP:
 				 * We've soft-faulted (because it's not in the page
 				 * table) on an encrypted page.
-				 * Keep the page "busy" so that noone messes with
+				 * Keep the page "busy" so that no one messes with
 				 * it during the decryption.
 				 * Release the extra locks we're holding, keep only
 				 * the page's VM object lock.
+				 *
+				 * in order to set 'busy' on 'm', we must
+				 * have object that 'm' belongs to locked exclusively
 				 */
-				m->busy = TRUE;
-				if (object != cur_object) {
+			        if (object != cur_object) {
 					vm_object_unlock(object);
+
+					if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
+
+					        cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+						if (vm_object_lock_upgrade(cur_object) == FALSE) {
+						        /*
+							 * couldn't upgrade so go do a full retry
+							 * immediately since we've already dropped
+							 * the top object lock associated with this page
+							 * and the current one got dropped due to the
+							 * failed upgrade... the state is no longer valid
+							 */
+						        vm_map_unlock_read(map);
+							if (real_map != map)
+							        vm_map_unlock(real_map);
+
+							goto RetryFault;
+						}
+					}
+				} else if (object_lock_type == OBJECT_LOCK_SHARED) {
+
+				        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+					if (vm_object_lock_upgrade(object) == FALSE) {
+					        /*
+						 * couldn't upgrade, so explictly take the lock
+						 * exclusively and go relookup the page since we
+						 * will have dropped the object lock and
+						 * a different thread could have inserted
+						 * a page at this offset
+						 * no need for a full retry since we're
+						 * at the top level of the object chain
+						 */
+					        vm_object_lock(object);
+
+						continue;
+					}
 				}
+				m->busy = TRUE;
+
 				vm_map_unlock_read(map);
 				if (real_map != map) 
 					vm_map_unlock(real_map);
@@ -2330,8 +2490,8 @@ vm_fault(
 
 				assert(m->busy);
 				PAGE_WAKEUP_DONE(m);
-				vm_object_unlock(m->object);
 
+				vm_object_unlock(cur_object);
 				/*
 				 * Retry from the top, in case anything
 				 * changed while we were decrypting...
@@ -2340,6 +2500,46 @@ vm_fault(
 			}
 			ASSERT_PAGE_DECRYPTED(m);
 
+			if (m->object->code_signed && !m->cs_validated) {
+				/*
+				 * We will need to validate this page
+				 * against its code signature, so we
+				 * want to hold the VM object exclusively.
+				 */
+			        if (object != cur_object) {
+					if (cur_object_lock_type == OBJECT_LOCK_SHARED) {
+						vm_object_unlock(object);
+						vm_object_unlock(cur_object);
+
+					        cur_object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+						vm_map_unlock_read(map);
+						if (real_map != map)
+							vm_map_unlock(real_map);
+
+						goto RetryFault;
+					}
+
+				} else if (object_lock_type == OBJECT_LOCK_SHARED) {
+
+				        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+					if (vm_object_lock_upgrade(object) == FALSE) {
+					        /*
+						 * couldn't upgrade, so explictly take the lock
+						 * exclusively and go relookup the page since we
+						 * will have dropped the object lock and
+						 * a different thread could have inserted
+						 * a page at this offset
+						 * no need for a full retry since we're
+						 * at the top level of the object chain
+						 */
+					        vm_object_lock(object);
+
+						continue;
+					}
+				}
+			}
 			/*
 			 *	Two cases of map in faults:
 			 *	    - At top level w/o copy object.
@@ -2347,648 +2547,621 @@ vm_fault(
 			 *		--> must disallow write.
 			 */
 
-			if (object == cur_object &&
-			    object->copy == VM_OBJECT_NULL)
-				goto FastMapInFault;
+			if (object == cur_object && object->copy == VM_OBJECT_NULL)
+				goto FastPmapEnter;
 
 			if ((fault_type & VM_PROT_WRITE) == 0) {
-				boolean_t sequential;
 
 				prot &= ~VM_PROT_WRITE;
 
 				/*
-				 *	Set up to map the page ...
-				 *	mark the page busy, drop
-				 *	locks and take a paging reference
-				 *	on the object with the page.
+				 * Set up to map the page...
+				 * mark the page busy, drop
+				 * unneeded object lock
 				 */	
-
 			  	if (object != cur_object) {
+				        /*	
+					 * don't need the original object anymore
+					 */
 					vm_object_unlock(object);
+
+					/*
+					 * switch to the object that has the new page
+					 */
 					object = cur_object;
+					object_lock_type = cur_object_lock_type;
 				}
-FastMapInFault:
-				m->busy = TRUE;
-
 FastPmapEnter:
 				/*
-				 *	Check a couple of global reasons to
-				 *	be conservative about write access.
-				 *	Then do the pmap_enter.
+				 * prepare for the pmap_enter...
+				 * object and map are both locked
+				 * m contains valid data
+				 * object == m->object
+				 * cur_object == NULL or it's been unlocked
+				 * no paging references on either object or cur_object
 				 */
-#if	!VM_FAULT_STATIC_CONFIG
-				if (vm_fault_dirty_handling
-#if	MACH_KDB
-				    || db_watchpoint_list
-#endif
-				    && (fault_type & VM_PROT_WRITE) == 0)
-					prot &= ~VM_PROT_WRITE;
-#else	/* STATIC_CONFIG */
 #if	MACH_KDB
-				if (db_watchpoint_list
-				    && (fault_type & VM_PROT_WRITE) == 0)
+				if (db_watchpoint_list && (fault_type & VM_PROT_WRITE) == 0)
 					prot &= ~VM_PROT_WRITE;
-#endif	/* MACH_KDB */
-#endif	/* STATIC_CONFIG */
-				cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
-
-				sequential = FALSE;
-				need_activation = FALSE;
-
-				if (m->no_isync == TRUE) {
-					m->no_isync = FALSE;
-					pmap_sync_page_data_phys(m->phys_page);
-
-					if ((type_of_fault == DBG_CACHE_HIT_FAULT) && m->clustered) {
-						/*
-						 * found it in the cache, but this
-						 * is the first fault-in of the page (no_isync == TRUE)
-						 * so it must have come in as part of
-						 * a cluster... account 1 pagein against it
-						 */
-						VM_STAT(pageins++);
-						current_task()->pageins++;
-						type_of_fault = DBG_PAGEIN_FAULT;
-						sequential = TRUE;
-					}
-					if (m->clustered)
-						need_activation = TRUE;
-
-				} else if (cache_attr != VM_WIMG_DEFAULT) {
-					pmap_sync_page_attributes_phys(m->phys_page);
-				}
-
-				if(caller_pmap) {
-					PMAP_ENTER(caller_pmap, 
-						caller_pmap_addr, m, 
-						prot, cache_attr, wired);
+#endif
+				if (caller_pmap) {
+				        kr = vm_fault_enter(m,
+							    caller_pmap,
+							    caller_pmap_addr,
+							    prot,
+							    wired,
+							    change_wiring,
+							    fault_info.no_cache,
+							    &type_of_fault);
 				} else {
-					PMAP_ENTER(pmap, vaddr, m, 
-						prot, cache_attr, wired);
+				        kr = vm_fault_enter(m,
+							    pmap,
+							    vaddr,
+							    prot,
+							    wired,
+							    change_wiring,
+							    fault_info.no_cache,
+							    &type_of_fault);
 				}
 
-				/*
-				 *	Hold queues lock to manipulate
-				 *	the page queues.  Change wiring
-				 *	case is obvious.  In soft ref bits
-				 *	case activate page only if it fell
-				 *	off paging queues, otherwise just
-				 *	activate it if it's inactive.
-				 *
-				 *	NOTE: original vm_fault code will
-				 *	move active page to back of active
-				 *	queue.  This code doesn't.
-				 */
-				if (m->clustered) {
-				        vm_pagein_cluster_used++;
-					m->clustered = FALSE;
-				}
-				if (change_wiring) {
-				        vm_page_lock_queues();
+				if (need_collapse == TRUE)
+				        vm_object_collapse(object, offset, TRUE);
 
-					if (wired)
-						vm_page_wire(m);
-					else
-						vm_page_unwire(m);
+				if (type_of_fault == DBG_PAGEIN_FAULT) {
+				        /*
+					 * evaluate access pattern and update state
+					 * vm_fault_deactivate_behind depends on the
+					 * state being up to date
+					 */
+				        vm_fault_is_sequential(object, cur_offset, fault_info.behavior);
 
-					vm_page_unlock_queues();
+					vm_fault_deactivate_behind(object, cur_offset, fault_info.behavior);
 				}
-				else {
-				        if ((!m->active && !m->inactive) || ((need_activation == TRUE) && !m->active)) {
-					        vm_page_lock_queues();
-					        vm_page_activate(m);
-						vm_page_unlock_queues();
-					}
-				}
-
 				/*
-				 *	That's it, clean up and return.
+				 * That's it, clean up and return.
 				 */
-				PAGE_WAKEUP_DONE(m);
-
-				sequential = (sequential && vm_page_deactivate_behind) ?
-					vm_fault_deactivate_behind(object, cur_offset, behavior) :
-					FALSE;
-
-				/* 
-				 * Add non-sequential pages to the working set.
-				 * The sequential pages will be brought in through
-				 * normal clustering behavior.
-				 */
-				if (!sequential && !object->private) {
-				        vm_object_paging_begin(object);
-
-					write_startup_file = 
-						vm_fault_tws_insert(map, real_map, vaddr, 
-					                        object, cur_offset);
+				if (m->busy)
+				        PAGE_WAKEUP_DONE(m);
 
-					vm_object_paging_end(object);
-				}
 				vm_object_unlock(object);
 
 				vm_map_unlock_read(map);
-				if(real_map != map)
+				if (real_map != map)
 					vm_map_unlock(real_map);
 
-				if(write_startup_file)
-					tws_send_startup_info(current_task());
-
-				thread_interrupt_level(interruptible_state);
-
-
-				KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END,
-						      vaddr,
-						      type_of_fault & 0xff,
-						      KERN_SUCCESS,
-						      type_of_fault >> 8,
-						      0);
-
-				return KERN_SUCCESS;
+				goto done;
 			}
-
 			/*
-			 *	Copy on write fault.  If objects match, then
-			 *	object->copy must not be NULL (else control
-			 *	would be in previous code block), and we
-			 *	have a potential push into the copy object
-			 *	with which we won't cope here.
+			 * COPY ON WRITE FAULT
+			 *
+			 * If objects match, then
+			 * object->copy must not be NULL (else control
+			 * would be in previous code block), and we
+			 * have a potential push into the copy object
+			 * with which we can't cope with here.
 			 */
-
-			if (cur_object == object)
+			if (cur_object == object) {
+			        /*
+				 * must take the slow path to
+				 * deal with the copy push
+				 */
 				break;
+			}
+			assert(object_lock_type == OBJECT_LOCK_EXCLUSIVE);
+
 			/*
-			 *	This is now a shadow based copy on write
-			 *	fault -- it requires a copy up the shadow
-			 *	chain.
+			 * This is now a shadow based copy on write
+			 * fault -- it requires a copy up the shadow
+			 * chain.
+			 *
+			 * Allocate a page in the original top level
+			 * object. Give up if allocate fails.  Also
+			 * need to remember current page, as it's the
+			 * source of the copy.
 			 *
-			 *	Allocate a page in the original top level
-			 *	object. Give up if allocate fails.  Also
-			 *	need to remember current page, as it's the
-			 *	source of the copy.
+			 * at this point we hold locks on both 
+			 * object and cur_object... no need to take
+			 * paging refs or mark pages BUSY since
+			 * we don't drop either object lock until
+			 * the page has been copied and inserted
 			 */
 			cur_m = m;
 			m = vm_page_grab();
+
 			if (m == VM_PAGE_NULL) {
+			        /*
+				 * no free page currently available...
+				 * must take the slow path
+				 */
 				break;
 			}
 			/*
-			 *	Now do the copy.  Mark the source busy
-			 *	and take out paging references on both
-			 *	objects.
+			 * Now do the copy.  Mark the source page busy...
 			 *
 			 *	NOTE: This code holds the map lock across
 			 *	the page copy.
 			 */
-
-			cur_m->busy = TRUE;
 			vm_page_copy(cur_m, m);
 			vm_page_insert(m, object, offset);
-
-			vm_object_paging_begin(cur_object);
-			vm_object_paging_begin(object);
-
-			type_of_fault = DBG_COW_FAULT;
-			VM_STAT(cow_faults++);
-			current_task()->cow_faults++;
+			m->dirty = TRUE;
 
 			/*
-			 *	Now cope with the source page and object
-			 *	If the top object has a ref count of 1
-			 *	then no other map can access it, and hence
-			 *	it's not necessary to do the pmap_disconnect.
+			 * Now cope with the source page and object
 			 */
+			if (object->ref_count > 1 && cur_m->pmapped)
+			        pmap_disconnect(cur_m->phys_page);
 
-			vm_page_lock_queues();
-			vm_page_deactivate(cur_m);
-			m->dirty = TRUE;
-			pmap_disconnect(cur_m->phys_page);
-			vm_page_unlock_queues();
+			need_collapse = TRUE;
 
-			PAGE_WAKEUP_DONE(cur_m);
-			vm_object_paging_end(cur_object);
+			if (!cur_object->internal &&
+			    cur_object->copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
+			        /*
+				 * The object from which we've just
+				 * copied a page is most probably backed
+				 * by a vnode.  We don't want to waste too
+				 * much time trying to collapse the VM objects
+				 * and create a bottleneck when several tasks
+				 * map the same file.
+				 */
+			        if (cur_object->copy == object) {
+				        /*
+					 * Shared mapping or no COW yet.
+					 * We can never collapse a copy
+					 * object into its backing object.
+					 */
+				        need_collapse = FALSE;
+				} else if (cur_object->copy == object->shadow &&
+					   object->shadow->resident_page_count == 0) {
+				        /*
+					 * Shared mapping after a COW occurred.
+					 */
+				        need_collapse = FALSE;
+				}
+			}
 			vm_object_unlock(cur_object);
 
-			/*      
-			 *      Slight hack to call vm_object collapse
-			 *      and then reuse common map in code. 
-			 *      note that the object lock was taken above.
-			 */     
- 
-			vm_object_paging_end(object); 
-			vm_object_collapse(object, offset, TRUE);
+			if (need_collapse == FALSE)
+			        vm_fault_collapse_skipped++;
+			vm_fault_collapse_total++;
+
+			type_of_fault = DBG_COW_FAULT;
+			VM_STAT_INCR(cow_faults);
+			DTRACE_VM2(cow_fault, int, 1, (uint64_t *), NULL);
+			current_task()->cow_faults++;
 
 			goto FastPmapEnter;
-		}
-		else {
 
+		} else {
 			/*
-			 *	No page at cur_object, cur_offset
+			 * No page at cur_object, cur_offset... m == NULL
 			 */
-
 			if (cur_object->pager_created) {
-
+			        if (MUST_ASK_PAGER(cur_object, cur_offset) == TRUE) {
+				        /*
+					 * May have to talk to a pager...
+					 * take the slow path.
+					 */
+				        break;
+				}
 				/*
-				 *	Have to talk to the pager.  Give up.
+				 * existence map present and indicates
+				 * that the pager doesn't have this page
 				 */
-				break;
 			}
-
-
 			if (cur_object->shadow == VM_OBJECT_NULL) {
-
+				/*
+				 * Zero fill fault.  Page gets
+				 * inserted into the original object.
+				 */
 				if (cur_object->shadow_severed) {
-					vm_object_paging_end(object);
+
+					if (object != cur_object)
+					        vm_object_unlock(cur_object);
 					vm_object_unlock(object);
+
 					vm_map_unlock_read(map);
-					if(real_map != map)
+					if (real_map != map)
 						vm_map_unlock(real_map);
 
-					if(write_startup_file)
-						tws_send_startup_info(
-								current_task());
+					kr = KERN_MEMORY_ERROR;
+					goto done;
+				}
+				if (VM_PAGE_ZFILL_THROTTLED()) {
+					/*
+					 * drop all of our locks...
+					 * wait until the free queue is
+					 * pumped back up and then
+					 * redrive the fault
+					 */
+					if (object != cur_object)
+						vm_object_unlock(cur_object);
+					vm_object_unlock(object);
+					vm_map_unlock_read(map);
+					if (real_map != map)
+						vm_map_unlock(real_map);
 
-					thread_interrupt_level(interruptible_state);
+					if (vm_page_wait((change_wiring) ? 
+							 THREAD_UNINT :
+							 THREAD_ABORTSAFE))
+						goto RetryFault;
 
-					return KERN_MEMORY_ERROR;
+					kr = KERN_ABORTED;
+					goto done;
+				}
+				if (vm_backing_store_low) {
+				        /*
+					 * we are protecting the system from
+					 * backing store exhaustion... 
+					 * must take the slow path if we're
+					 * not privileged
+					 */
+					if (!(current_task()->priv_flags & VM_BACKING_STORE_PRIV))
+					        break;
 				}
+			  	if (cur_object != object) {
+					vm_object_unlock(cur_object);
 
-				/*
-				 *	Zero fill fault.  Page gets
-				 *	filled in top object. Insert
-				 *	page, then drop any lower lock.
-				 *	Give up if no page.
-				 */
-				if (VM_PAGE_THROTTLED()) {
-					break;
+					cur_object = object;
 				}
+				if (object_lock_type == OBJECT_LOCK_SHARED) {
 
-				/*
-			 	 * are we protecting the system from
-			 	 * backing store exhaustion.  If so
-			 	 * sleep unless we are privileged.
-			 	 */
-				if(vm_backing_store_low) {
-					if(!(current_task()->priv_flags 
-						& VM_BACKING_STORE_PRIV))
-					break;
+				        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+					if (vm_object_lock_upgrade(object) == FALSE) {
+					        /*
+						 * couldn't upgrade so do a full retry on the fault
+						 * since we dropped the object lock which
+						 * could allow another thread to insert
+						 * a page at this offset
+						 */
+					        vm_map_unlock_read(map);
+						if (real_map != map)
+						        vm_map_unlock(real_map);
+
+						goto RetryFault;
+					}
 				}
 				m = vm_page_alloc(object, offset);
+
 				if (m == VM_PAGE_NULL) {
+				        /*
+					 * no free page currently available...
+					 * must take the slow path
+					 */
 					break;
 				}
-				/*
-				 * This is a zero-fill or initial fill
-				 * page fault.  As such, we consider it
-				 * undefined with respect to instruction
-				 * execution.  i.e. it is the responsibility
-				 * of higher layers to call for an instruction
-				 * sync after changing the contents and before
-				 * sending a program into this area.  We 
-				 * choose this approach for performance
-				 */
-
-				m->no_isync = FALSE;
-
-			  	if (cur_object != object)
-					vm_object_unlock(cur_object);
 
 				/*
-				 *	Now zero fill page and map it.
-				 *	the page is probably going to 
-				 *	be written soon, so don't bother
-				 *      to clear the modified bit
+				 * Now zero fill page...
+				 * the page is probably going to 
+				 * be written soon, so don't bother
+				 * to clear the modified bit
 				 *
-				 *	NOTE: This code holds the map
-				 *	lock across the zero fill.
+				 *   NOTE: This code holds the map
+				 *   lock across the zero fill.
 				 */
-
-				if (!map->no_zero_fill) {
-					vm_page_zero_fill(m);
-					type_of_fault = DBG_ZERO_FILL_FAULT;
-					VM_STAT(zero_fill_count++);
-				}
-				vm_page_lock_queues();
-				VM_PAGE_QUEUES_REMOVE(m);
-
-				m->page_ticket = vm_page_ticket;
-				assert(!m->laundry);
-				assert(m->object != kernel_object);
-				assert(m->pageq.next == NULL &&
-				       m->pageq.prev == NULL);
-				if(m->object->size > 0x200000) {
-					m->zero_fill = TRUE;
-					/* depends on the queues lock */
-					vm_zf_count += 1;
-					queue_enter(&vm_page_queue_zf, 
-						m, vm_page_t, pageq);
-				} else {
-					queue_enter(
-						&vm_page_queue_inactive, 
-						m, vm_page_t, pageq);
-				}
-				vm_page_ticket_roll++;
-				if(vm_page_ticket_roll == 
-						VM_PAGE_TICKETS_IN_ROLL) {
-					vm_page_ticket_roll = 0;
-					if(vm_page_ticket == 
-						VM_PAGE_TICKET_ROLL_IDS)
-						vm_page_ticket= 0;
-					else
-						vm_page_ticket++;
-				}
-
-				m->inactive = TRUE;
-				vm_page_inactive_count++;
-				vm_page_unlock_queues();
+				type_of_fault = vm_fault_zero_page(m, map->no_zero_fill);
 
 				goto FastPmapEnter;
 		        }
-
 			/*
-			 *	On to the next level
+			 * On to the next level in the shadow chain
 			 */
-
 			cur_offset += cur_object->shadow_offset;
 			new_object = cur_object->shadow;
-			vm_object_lock(new_object);
+
+			/*
+			 * take the new_object's lock with the indicated state
+			 */
+			if (cur_object_lock_type == OBJECT_LOCK_SHARED)
+			        vm_object_lock_shared(new_object);
+			else
+			        vm_object_lock(new_object);
+
 			if (cur_object != object)
 				vm_object_unlock(cur_object);
+
 			cur_object = new_object;
 
 			continue;
 		}
 	}
-
 	/*
-	 *	Cleanup from fast fault failure.  Drop any object
-	 *	lock other than original and drop map lock.
+	 * Cleanup from fast fault failure.  Drop any object
+	 * lock other than original and drop map lock.
 	 */
-
 	if (object != cur_object)
 		vm_object_unlock(cur_object);
+
+	/*
+	 * must own the object lock exclusively at this point
+	 */
+	if (object_lock_type == OBJECT_LOCK_SHARED) {
+	        object_lock_type = OBJECT_LOCK_EXCLUSIVE;
+
+		if (vm_object_lock_upgrade(object) == FALSE) {
+		        /*
+			 * couldn't upgrade, so explictly
+			 * take the lock exclusively
+			 * no need to retry the fault at this
+			 * point since "vm_fault_page" will
+			 * completely re-evaluate the state
+			 */
+		        vm_object_lock(object);
+		}
 	}
-	vm_map_unlock_read(map);
 
-	if(real_map != map)
+handle_copy_delay:
+	vm_map_unlock_read(map);
+	if (real_map != map)
 		vm_map_unlock(real_map);
 
    	/*
-	 *	Make a reference to this object to
-	 *	prevent its disposal while we are messing with
-	 *	it.  Once we have the reference, the map is free
-	 *	to be diddled.  Since objects reference their
-	 *	shadows (and copies), they will stay around as well.
+	 * Make a reference to this object to
+	 * prevent its disposal while we are messing with
+	 * it.  Once we have the reference, the map is free
+	 * to be diddled.  Since objects reference their
+	 * shadows (and copies), they will stay around as well.
 	 */
-
-	assert(object->ref_count > 0);
-	object->ref_count++;
-	vm_object_res_reference(object);
+	vm_object_reference_locked(object);
 	vm_object_paging_begin(object);
 
 	XPR(XPR_VM_FAULT,"vm_fault -> vm_fault_page\n",0,0,0,0,0);
 
-	if (!object->private) {
-		write_startup_file = 
-			vm_fault_tws_insert(map, real_map, vaddr, object, offset);
-	}
+	error_code = 0;
 
 	kr = vm_fault_page(object, offset, fault_type,
 			   (change_wiring && !wired),
-			   interruptible,
-			   lo_offset, hi_offset, behavior,
 			   &prot, &result_page, &top_page,
 			   &type_of_fault,
-			   &error_code, map->no_zero_fill, FALSE, map, vaddr);
+			   &error_code, map->no_zero_fill,
+			   FALSE, &fault_info);
 
 	/*
-	 *	If we didn't succeed, lose the object reference immediately.
+	 * if kr != VM_FAULT_SUCCESS, then the paging reference
+	 * has been dropped and the object unlocked... the ref_count
+	 * is still held
+	 *
+	 * if kr == VM_FAULT_SUCCESS, then the paging reference
+	 * is still held along with the ref_count on the original object
+	 *
+	 *	if m != NULL, then the object it belongs to 
+	 *	is returned locked with a paging reference
+	 *
+	 *	if top_page != NULL, then it's BUSY and the 
+	 *	object it belongs to has a paging reference
+	 *	but is returned unlocked
 	 */
-
-	if (kr != VM_FAULT_SUCCESS)
+	if (kr != VM_FAULT_SUCCESS) {
+	        /*
+		 * we didn't succeed, lose the object reference immediately.
+		 */
 		vm_object_deallocate(object);
 
-	/*
-	 *	See why we failed, and take corrective action.
-	 */
-
-	switch (kr) {
-		case VM_FAULT_SUCCESS:
-			break;
+		/*
+		 * See why we failed, and take corrective action.
+		 */
+		switch (kr) {
 		case VM_FAULT_MEMORY_SHORTAGE:
 			if (vm_page_wait((change_wiring) ? 
 					 THREAD_UNINT :
 					 THREAD_ABORTSAFE))
 				goto RetryFault;
-			/* fall thru */
+			/*
+			 * fall thru
+			 */
 		case VM_FAULT_INTERRUPTED:
 			kr = KERN_ABORTED;
 			goto done;
 		case VM_FAULT_RETRY:
 			goto RetryFault;
-		case VM_FAULT_FICTITIOUS_SHORTAGE:
-			vm_page_more_fictitious();
-			goto RetryFault;
 		case VM_FAULT_MEMORY_ERROR:
 			if (error_code)
 				kr = error_code;
 			else
 				kr = KERN_MEMORY_ERROR;
 			goto done;
+		}
 	}
-
 	m = result_page;
 
-	if(m != VM_PAGE_NULL) {
+	if (m != VM_PAGE_NULL) {
 		assert((change_wiring && !wired) ?
 	   	    (top_page == VM_PAGE_NULL) :
 	   	    ((top_page == VM_PAGE_NULL) == (m->object == object)));
 	}
 
 	/*
-	 *	How to clean up the result of vm_fault_page.  This
-	 *	happens whether the mapping is entered or not.
+	 * What to do with the resulting page from vm_fault_page
+	 * if it doesn't get entered into the physical map:
 	 */
-
-#define UNLOCK_AND_DEALLOCATE				\
-	MACRO_BEGIN					\
-	vm_fault_cleanup(m->object, top_page);		\
-	vm_object_deallocate(object);			\
-	MACRO_END
-
-	/*
-	 *	What to do with the resulting page from vm_fault_page
-	 *	if it doesn't get entered into the physical map:
-	 */
-
 #define RELEASE_PAGE(m)					\
 	MACRO_BEGIN					\
 	PAGE_WAKEUP_DONE(m);				\
-	vm_page_lock_queues();				\
-	if (!m->active && !m->inactive)			\
+	vm_page_lockspin_queues();			\
+	if (!m->active && !m->inactive && !m->throttled)\
 		vm_page_activate(m);			\
 	vm_page_unlock_queues();			\
 	MACRO_END
 
 	/*
-	 *	We must verify that the maps have not changed
-	 *	since our last lookup.
+	 * We must verify that the maps have not changed
+	 * since our last lookup.
 	 */
-
-	if(m != VM_PAGE_NULL) {
+	if (m != VM_PAGE_NULL) {
 		old_copy_object = m->object->copy;
 		vm_object_unlock(m->object);
-	} else {
+	} else
 		old_copy_object = VM_OBJECT_NULL;
-	}
+
+	/*
+	 * no object locks are held at this point
+	 */
 	if ((map != original_map) || !vm_map_verify(map, &version)) {
 		vm_object_t		retry_object;
 		vm_object_offset_t	retry_offset;
 		vm_prot_t		retry_prot;
 
 		/*
-		 *	To avoid trying to write_lock the map while another
-		 *	thread has it read_locked (in vm_map_pageable), we
-		 *	do not try for write permission.  If the page is
-		 *	still writable, we will get write permission.  If it
-		 *	is not, or has been marked needs_copy, we enter the
-		 *	mapping without write permission, and will merely
-		 *	take another fault.
+		 * To avoid trying to write_lock the map while another
+		 * thread has it read_locked (in vm_map_pageable), we
+		 * do not try for write permission.  If the page is
+		 * still writable, we will get write permission.  If it
+		 * is not, or has been marked needs_copy, we enter the
+		 * mapping without write permission, and will merely
+		 * take another fault.
 		 */
 		map = original_map;
 		vm_map_lock_read(map);
+
 		kr = vm_map_lookup_locked(&map, vaddr,
-				   fault_type & ~VM_PROT_WRITE, &version,
-				   &retry_object, &retry_offset, &retry_prot,
-				   &wired, &behavior, &lo_offset, &hi_offset,
-				   &real_map);
+					  fault_type & ~VM_PROT_WRITE,
+					  OBJECT_LOCK_EXCLUSIVE, &version,
+					  &retry_object, &retry_offset, &retry_prot,
+					  &wired,
+					  &fault_info,
+					  &real_map);
 		pmap = real_map->pmap;
 
 		if (kr != KERN_SUCCESS) {
 			vm_map_unlock_read(map);
-			if(m != VM_PAGE_NULL) {
+
+			if (m != VM_PAGE_NULL) {
+			        /*
+				 * retake the lock so that
+				 * we can drop the paging reference
+				 * in vm_fault_cleanup and do the
+				 * PAGE_WAKEUP_DONE in RELEASE_PAGE
+				 */
 				vm_object_lock(m->object);
+
 				RELEASE_PAGE(m);
-				UNLOCK_AND_DEALLOCATE;
+
+				vm_fault_cleanup(m->object, top_page);
 			} else {
-				vm_object_deallocate(object);
+			        /*
+				 * retake the lock so that
+				 * we can drop the paging reference
+				 * in vm_fault_cleanup
+				 */
+			        vm_object_lock(object);
+
+			        vm_fault_cleanup(object, top_page);
 			}
+			vm_object_deallocate(object);
+
 			goto done;
 		}
-
 		vm_object_unlock(retry_object);
-		if(m != VM_PAGE_NULL) {
-			vm_object_lock(m->object);
-		} else {
-			vm_object_lock(object);
-		}
 
-		if ((retry_object != object) ||
-		    (retry_offset != offset)) {
+		if ((retry_object != object) || (retry_offset != offset)) {
+
 			vm_map_unlock_read(map);
-			if(real_map != map)
+			if (real_map != map)
 				vm_map_unlock(real_map);
-			if(m != VM_PAGE_NULL) {
+
+			if (m != VM_PAGE_NULL) {
+			        /*
+				 * retake the lock so that
+				 * we can drop the paging reference
+				 * in vm_fault_cleanup and do the
+				 * PAGE_WAKEUP_DONE in RELEASE_PAGE
+				 */
+			        vm_object_lock(m->object);
+
 				RELEASE_PAGE(m);
-				UNLOCK_AND_DEALLOCATE;
+
+				vm_fault_cleanup(m->object, top_page);
 			} else {
-				vm_object_deallocate(object);
+			        /*
+				 * retake the lock so that
+				 * we can drop the paging reference
+				 * in vm_fault_cleanup
+				 */
+			        vm_object_lock(object);
+
+			        vm_fault_cleanup(object, top_page);
 			}
+			vm_object_deallocate(object);
+
 			goto RetryFault;
 		}
-
 		/*
-		 *	Check whether the protection has changed or the object
-		 *	has been copied while we left the map unlocked.
+		 * Check whether the protection has changed or the object
+		 * has been copied while we left the map unlocked.
 		 */
 		prot &= retry_prot;
-		if(m != VM_PAGE_NULL) {
-			vm_object_unlock(m->object);
-		} else {
-			vm_object_unlock(object);
-		}
 	}
-	if(m != VM_PAGE_NULL) {
+	if (m != VM_PAGE_NULL) {
 		vm_object_lock(m->object);
-	} else {
-		vm_object_lock(object);
-	}
-
-	/*
-	 *	If the copy object changed while the top-level object
-	 *	was unlocked, then we must take away write permission.
-	 */
 
-	if(m != VM_PAGE_NULL) {
-		if (m->object->copy != old_copy_object)
+		if (m->object->copy != old_copy_object) {
+		        /*
+			 * The copy object changed while the top-level object
+			 * was unlocked, so take away write permission.
+			 */
 			prot &= ~VM_PROT_WRITE;
-	}
+		}
+	} else
+		vm_object_lock(object);
 
 	/*
-	 *	If we want to wire down this page, but no longer have
-	 *	adequate permissions, we must start all over.
+	 * If we want to wire down this page, but no longer have
+	 * adequate permissions, we must start all over.
 	 */
+	if (wired && (fault_type != (prot | VM_PROT_WRITE))) {
 
-	if (wired && (fault_type != (prot|VM_PROT_WRITE))) {
 		vm_map_verify_done(map, &version);
-		if(real_map != map)
+		if (real_map != map)
 			vm_map_unlock(real_map);
-		if(m != VM_PAGE_NULL) {
-			RELEASE_PAGE(m);
-			UNLOCK_AND_DEALLOCATE;
-		} else {
-			vm_object_deallocate(object);
-		}
-		goto RetryFault;
-	}
 
-	/*
-	 *	Put this page into the physical map.
-	 *	We had to do the unlock above because pmap_enter
-	 *	may cause other faults.  The page may be on
-	 *	the pageout queues.  If the pageout daemon comes
-	 *	across the page, it will remove it from the queues.
-	 */
-	need_activation = FALSE;
+		if (m != VM_PAGE_NULL) {
+			RELEASE_PAGE(m);
 
-	if (m != VM_PAGE_NULL) {
-		if (m->no_isync == TRUE) {
-		        pmap_sync_page_data_phys(m->phys_page);
-
-                        if ((type_of_fault == DBG_CACHE_HIT_FAULT) && m->clustered) {
-                                /*
-                                 * found it in the cache, but this
-                                 * is the first fault-in of the page (no_isync == TRUE)
-                                 * so it must have come in as part of
-                                 * a cluster... account 1 pagein against it
-                                 */
-                                 VM_STAT(pageins++);
-                                 current_task()->pageins++;
-
-                                 type_of_fault = DBG_PAGEIN_FAULT;
-                        }
-			if (m->clustered) {
-			        need_activation = TRUE;
-			}
-		        m->no_isync = FALSE;
-		}
-		cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
+			vm_fault_cleanup(m->object, top_page);
+		} else
+		        vm_fault_cleanup(object, top_page);
 
-		if(caller_pmap) {
-			PMAP_ENTER(caller_pmap, 
-					caller_pmap_addr, m, 
-					prot, cache_attr, wired);
-		} else {
-			PMAP_ENTER(pmap, vaddr, m, 
-					prot, cache_attr, wired);
-		}
+		vm_object_deallocate(object);
 
+		goto RetryFault;
+	}
+	if (m != VM_PAGE_NULL) {
 		/*
-		 * Add working set information for private objects here.
+		 * Put this page into the physical map.
+		 * We had to do the unlock above because pmap_enter
+		 * may cause other faults.  The page may be on
+		 * the pageout queues.  If the pageout daemon comes
+		 * across the page, it will remove it from the queues.
 		 */
-		if (m->object->private) {
-			write_startup_file =
-				vm_fault_tws_insert(map, real_map, vaddr, 
-		                            m->object, m->offset);
+		if (caller_pmap) {
+			kr = vm_fault_enter(m,
+					    caller_pmap,
+					    caller_pmap_addr,
+					    prot,
+					    wired,
+					    change_wiring,
+					    fault_info.no_cache,
+					    &type_of_fault);
+		} else {
+			kr = vm_fault_enter(m,
+					    pmap,
+					    vaddr,
+					    prot,
+					    wired,
+					    change_wiring,
+					    fault_info.no_cache,
+					    &type_of_fault);
+		}
+		if (kr != KERN_SUCCESS) {
+			/* abort this page fault */
+			vm_map_verify_done(map, &version);
+			if (real_map != map)
+				vm_map_unlock(real_map);
+			PAGE_WAKEUP_DONE(m);
+			vm_fault_cleanup(m->object, top_page);
+			vm_object_deallocate(object);
+			goto done;
 		}
 	} else {
 
@@ -3001,29 +3174,31 @@ FastPmapEnter:
 		 * in the object 
 		 */
 
-#ifndef i386
+#ifdef ppc
 		/* While we do not worry about execution protection in   */
 		/* general, certian pages may have instruction execution */
 		/* disallowed.  We will check here, and if not allowed   */
 		/* to execute, we return with a protection failure.      */
 
-		if((fault_type & VM_PROT_EXECUTE) &&
-			(!pmap_eligible_for_execute((ppnum_t)
-				(object->shadow_offset >> 12)))) {
+		if ((fault_type & VM_PROT_EXECUTE) &&
+			(!pmap_eligible_for_execute((ppnum_t)(object->shadow_offset >> 12)))) {
 
 			vm_map_verify_done(map, &version);
-			if(real_map != map)
+
+			if (real_map != map)
 				vm_map_unlock(real_map);
+
 			vm_fault_cleanup(object, top_page);
 			vm_object_deallocate(object);
+
 			kr = KERN_PROTECTION_FAILURE;
 			goto done;
 		}
-#endif	/* !i386 */
+#endif	/* ppc */
 
-		if(real_map != map) {
+		if (real_map != map)
 			vm_map_unlock(real_map);
-		}
+
 		if (original_map != map) {
 			vm_map_unlock_read(map);
 			vm_map_lock_read(original_map);
@@ -3035,20 +3210,20 @@ FastPmapEnter:
 		hdelta = 0xFFFFF000;
 		ldelta = 0xFFFFF000;
 
-
-		while(vm_map_lookup_entry(map, laddr, &entry)) {
-			if(ldelta > (laddr - entry->vme_start))
+		while (vm_map_lookup_entry(map, laddr, &entry)) {
+			if (ldelta > (laddr - entry->vme_start))
 				ldelta = laddr - entry->vme_start;
-			if(hdelta > (entry->vme_end - laddr))
+			if (hdelta > (entry->vme_end - laddr))
 				hdelta = entry->vme_end - laddr;
-			if(entry->is_sub_map) {
+			if (entry->is_sub_map) {
 				
 				laddr = (laddr - entry->vme_start) 
 							+ entry->offset;
 				vm_map_lock_read(entry->object.sub_map);
-				if(map != real_map)
+
+				if (map != real_map)
 					vm_map_unlock_read(map);
-				if(entry->use_pmap) {
+				if (entry->use_pmap) {
 					vm_map_unlock_read(real_map);
 					real_map = entry->object.sub_map;
 				}
@@ -3059,112 +3234,64 @@ FastPmapEnter:
 			}
 		}
 
-		if(vm_map_lookup_entry(map, laddr, &entry) && 
-		   (entry->object.vm_object != NULL) &&
-		   (entry->object.vm_object == object)) {
-
-			vm_map_offset_t	phys_offset;
-
-			phys_offset = (entry->object.vm_object->shadow_offset
-				       + entry->offset
-				       + laddr
-				       - entry->vme_start);
-			phys_offset -= ldelta;
-			if(caller_pmap) {
-				/* Set up a block mapped area */
-				pmap_map_block(
-					caller_pmap, 
-					(addr64_t)(caller_pmap_addr - ldelta), 
-					phys_offset >> 12,
-					(ldelta + hdelta) >> 12,
-					prot,
-					(VM_WIMG_MASK &	(int)object->wimg_bits),
-					0);
+		if (vm_map_lookup_entry(map, laddr, &entry) && 
+					(entry->object.vm_object != NULL) &&
+					(entry->object.vm_object == object)) {
+
+			if (caller_pmap) {
+				/*
+				 * Set up a block mapped area
+				 */
+				pmap_map_block(caller_pmap, 
+					       (addr64_t)(caller_pmap_addr - ldelta), 
+					       (((vm_map_offset_t) (entry->object.vm_object->shadow_offset)) +
+						entry->offset + (laddr - entry->vme_start) - ldelta) >> 12,
+					       ((ldelta + hdelta) >> 12), prot, 
+					       (VM_WIMG_MASK & (int)object->wimg_bits), 0);
 			} else { 
-				/* Set up a block mapped area */
-				pmap_map_block(
-					real_map->pmap, 
-					(addr64_t)(vaddr - ldelta), 
-					phys_offset >> 12,
-					(ldelta + hdelta) >> 12,
-					prot,
-					(VM_WIMG_MASK & (int)object->wimg_bits),
-					0);
+				/*
+				 * Set up a block mapped area
+				 */
+				pmap_map_block(real_map->pmap, 
+					       (addr64_t)(vaddr - ldelta), 
+					       (((vm_map_offset_t)(entry->object.vm_object->shadow_offset)) +
+						entry->offset + (laddr - entry->vme_start) - ldelta) >> 12,
+					       ((ldelta + hdelta) >> 12), prot, 
+					       (VM_WIMG_MASK & (int)object->wimg_bits), 0);
 			}
 		}
-
-	}
-
-	/*
-	 *	If the page is not wired down and isn't already
-	 *	on a pageout queue, then put it where the
-	 *	pageout daemon can find it.
-	 */
-	if(m != VM_PAGE_NULL) {
-		vm_page_lock_queues();
-
-		if (m->clustered) {
-		        vm_pagein_cluster_used++;
-			m->clustered = FALSE;
-		}
-		m->reference = TRUE;
-
-		if (change_wiring) {
-			if (wired)
-				vm_page_wire(m);
-			else
-				vm_page_unwire(m);
-		}
-#if	VM_FAULT_STATIC_CONFIG
-		else {
-		        if ((!m->active && !m->inactive) || ((need_activation == TRUE) && !m->active))
-				vm_page_activate(m);
-		}
-#else
-		else if (software_reference_bits) {
-			if (!m->active && !m->inactive)
-				vm_page_activate(m);
-			m->reference = TRUE;
-		} else {
-			vm_page_activate(m);
-		}
-#endif
-		vm_page_unlock_queues();
 	}
 
 	/*
-	 *	Unlock everything, and return
+	 * Unlock everything, and return
 	 */
-
 	vm_map_verify_done(map, &version);
-	if(real_map != map)
+	if (real_map != map)
 		vm_map_unlock(real_map);
-	if(m != VM_PAGE_NULL) {
+
+	if (m != VM_PAGE_NULL) {
 		PAGE_WAKEUP_DONE(m);
-		UNLOCK_AND_DEALLOCATE;
-	} else {
-		vm_fault_cleanup(object, top_page);
-		vm_object_deallocate(object);
-	}
-	kr = KERN_SUCCESS;
 
-#undef	UNLOCK_AND_DEALLOCATE
-#undef	RELEASE_PAGE
+		vm_fault_cleanup(m->object, top_page);
+	} else
+	        vm_fault_cleanup(object, top_page);
 
-    done:
-	if(write_startup_file)
-		tws_send_startup_info(current_task());
+	vm_object_deallocate(object);
+
+#undef	RELEASE_PAGE
 
+	kr = KERN_SUCCESS;
+done:
 	thread_interrupt_level(interruptible_state);
 
-	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 0)) | DBG_FUNC_END,
-			      vaddr,
-			      type_of_fault & 0xff,
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, 2)) | DBG_FUNC_END,
+			      (int)((uint64_t)vaddr >> 32),
+			      (int)vaddr,
 			      kr,
-			      type_of_fault >> 8,
+			      type_of_fault,
 			      0);
 
-	return(kr);
+	return (kr);
 }
 
 /*
@@ -3215,6 +3342,7 @@ vm_fault_wire(
 			  	(pmap == kernel_pmap) ? 
 					THREAD_UNINT : THREAD_ABORTSAFE, 
 				pmap, pmap_addr + (va - entry->vme_start));
+			DTRACE_VM2(softlock, int, 1, (uint64_t *), NULL);
 		}
 
 		if (rc != KERN_SUCCESS) {
@@ -3247,24 +3375,41 @@ vm_fault_unwire(
 	register vm_map_offset_t	va;
 	register vm_map_offset_t	end_addr = entry->vme_end;
 	vm_object_t		object;
+	struct vm_object_fault_info fault_info;
 
 	object = (entry->is_sub_map)
 			? VM_OBJECT_NULL : entry->object.vm_object;
 
+	/*
+	 * If it's marked phys_contiguous, then vm_fault_wire() didn't actually
+	 * do anything since such memory is wired by default.  So we don't have
+	 * anything to undo here.
+	 */
+
+	if (object != VM_OBJECT_NULL && object->phys_contiguous)
+		return;
+
+	fault_info.interruptible = THREAD_UNINT;
+	fault_info.behavior = entry->behavior;
+	fault_info.user_tag = entry->alias;
+	fault_info.lo_offset = entry->offset;
+	fault_info.hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;
+	fault_info.no_cache = entry->no_cache;
+
 	/*
 	 *	Since the pages are wired down, we must be able to
 	 *	get their mappings from the physical map system.
 	 */
 
 	for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
-		pmap_change_wiring(pmap, 
-			pmap_addr + (va - entry->vme_start), FALSE);
 
+		if (pmap) {
+			pmap_change_wiring(pmap, 
+					   pmap_addr + (va - entry->vme_start), FALSE);
+		}
 		if (object == VM_OBJECT_NULL) {
 			(void) vm_fault(map, va, VM_PROT_NONE, 
 					TRUE, THREAD_UNINT, pmap, pmap_addr);
-		} else if (object->phys_contiguous) {
-			continue;
 		} else {
 		 	vm_prot_t	prot;
 			vm_page_t	result_page;
@@ -3272,6 +3417,8 @@ vm_fault_unwire(
 			vm_object_t	result_object;
 			vm_fault_return_t result;
 
+			fault_info.cluster_size = end_addr - va;
+
 			do {
 				prot = VM_PROT_NONE;
 
@@ -3280,39 +3427,45 @@ vm_fault_unwire(
 				XPR(XPR_VM_FAULT,
 					"vm_fault_unwire -> vm_fault_page\n",
 					0,0,0,0,0);
-			 	result = vm_fault_page(object,
-						entry->offset +
-						  (va - entry->vme_start),
-						VM_PROT_NONE, TRUE,
-						THREAD_UNINT,
-					        entry->offset,
-						entry->offset +
-						       (entry->vme_end
-							- entry->vme_start),
-						entry->behavior,
-						&prot,
-						&result_page,
-						&top_page,
-						(int *)0,
-						0, map->no_zero_fill, 
-						FALSE, NULL, 0);
+			 	result = vm_fault_page(
+					object,
+					entry->offset + (va - entry->vme_start),
+					VM_PROT_NONE, TRUE,
+					&prot, &result_page, &top_page,
+					(int *)0,
+					NULL, map->no_zero_fill, 
+					FALSE, &fault_info);
 			} while (result == VM_FAULT_RETRY);
 
+			/*
+			 * If this was a mapping to a file on a device that has been forcibly
+			 * unmounted, then we won't get a page back from vm_fault_page().  Just
+			 * move on to the next one in case the remaining pages are mapped from
+			 * different objects.  During a forced unmount, the object is terminated
+			 * so the alive flag will be false if this happens.  A forced unmount will
+			 * will occur when an external disk is unplugged before the user does an 
+			 * eject, so we don't want to panic in that situation.
+			 */
+
+			if (result == VM_FAULT_MEMORY_ERROR && !object->alive)
+				continue;
+
 			if (result != VM_FAULT_SUCCESS)
 				panic("vm_fault_unwire: failure");
 
 			result_object = result_page->object;
+
 			if (deallocate) {
-				assert(!result_page->fictitious);
+				assert(result_page->phys_page !=
+				       vm_page_fictitious_addr);
 				pmap_disconnect(result_page->phys_page);
 				VM_PAGE_FREE(result_page);
 			} else {
-				vm_page_lock_queues();
+				vm_page_lockspin_queues();
 				vm_page_unwire(result_page);
 				vm_page_unlock_queues();
 				PAGE_WAKEUP_DONE(result_page);
 			}
-
 			vm_fault_cleanup(result_object, top_page);
 		}
 	}
@@ -3361,9 +3514,10 @@ vm_fault_wire_fast(
 	register vm_page_t	m;
 	vm_prot_t		prot;
 	thread_t           	thread = current_thread();
-	unsigned int		cache_attr;
+	int			type_of_fault;
+	kern_return_t		kr;
 
-	VM_STAT(faults++);
+	VM_STAT_INCR(faults);
 
 	if (thread != THREAD_NULL && thread->task != TASK_NULL)
 	  thread->task->faults++;
@@ -3375,7 +3529,7 @@ vm_fault_wire_fast(
 #undef	RELEASE_PAGE
 #define RELEASE_PAGE(m)	{				\
 	PAGE_WAKEUP_DONE(m);				\
-	vm_page_lock_queues();				\
+	vm_page_lockspin_queues();			\
 	vm_page_unwire(m);				\
 	vm_page_unlock_queues();			\
 }
@@ -3422,9 +3576,7 @@ vm_fault_wire_fast(
 	 */
 
 	vm_object_lock(object);
-	assert(object->ref_count > 0);
-	object->ref_count++;
-	vm_object_res_reference(object);
+	vm_object_reference_locked(object);
 	vm_object_paging_begin(object);
 
 	/*
@@ -3449,19 +3601,28 @@ vm_fault_wire_fast(
 	 */
 	m = vm_page_lookup(object, offset);
 	if ((m == VM_PAGE_NULL) || (m->busy) || (m->encrypted) ||
-	    (m->unusual && ( m->error || m->restart || m->absent ||
-				prot & m->page_lock))) {
+	    (m->unusual && ( m->error || m->restart || m->absent))) {
 
 		GIVE_UP;
 	}
 	ASSERT_PAGE_DECRYPTED(m);
 
+	if (m->fictitious &&
+	    m->phys_page == vm_page_guard_addr) {
+		/*
+		 * Guard pages are fictitious pages and are never
+		 * entered into a pmap, so let's say it's been wired...
+		 */
+		kr = KERN_SUCCESS;
+		goto done;
+	}
+
 	/*
 	 *	Wire the page down now.  All bail outs beyond this
 	 *	point must unwire the page.  
 	 */
 
-	vm_page_lock_queues();
+	vm_page_lockspin_queues();
 	vm_page_wire(m);
 	vm_page_unlock_queues();
 
@@ -3482,19 +3643,18 @@ vm_fault_wire_fast(
 
 	/*
 	 *	Put this page into the physical map.
-	 *	We have to unlock the object because pmap_enter
-	 *	may cause other faults.   
 	 */
-	if (m->no_isync == TRUE) {
-	        pmap_sync_page_data_phys(m->phys_page);
-
-		m->no_isync = FALSE;
-	}
-
-	cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
-
-	PMAP_ENTER(pmap, pmap_addr, m, prot, cache_attr, TRUE);
-
+	type_of_fault = DBG_CACHE_HIT_FAULT;
+	kr = vm_fault_enter(m,
+			    pmap,
+			    pmap_addr,
+			    prot,
+			    TRUE,
+			    FALSE,
+			    FALSE,
+			    &type_of_fault);
+
+done:
 	/*
 	 *	Unlock everything, and return
 	 */
@@ -3502,7 +3662,7 @@ vm_fault_wire_fast(
 	PAGE_WAKEUP_DONE(m);
 	UNLOCK_AND_DEALLOCATE;
 
-	return(KERN_SUCCESS);
+	return kr;
 
 }
 
@@ -3521,8 +3681,8 @@ vm_fault_copy_cleanup(
 
 	vm_object_lock(object);
 	PAGE_WAKEUP_DONE(page);
-	vm_page_lock_queues();
-	if (!page->active && !page->inactive)
+	vm_page_lockspin_queues();
+	if (!page->active && !page->inactive && !page->throttled)
 		vm_page_activate(page);
 	vm_page_unlock_queues();
 	vm_fault_cleanup(object, top_page);
@@ -3537,7 +3697,7 @@ vm_fault_copy_dst_cleanup(
 	if (page != VM_PAGE_NULL) {
 		object = page->object;
 		vm_object_lock(object);
-		vm_page_lock_queues();
+		vm_page_lockspin_queues();
 		vm_page_unwire(page);
 		vm_page_unlock_queues();
 		vm_object_paging_end(object);	
@@ -3598,15 +3758,13 @@ vm_fault_copy(
 	kern_return_t		error = 0;
 
 	vm_map_size_t		part_size;
+	struct vm_object_fault_info fault_info_src;
+	struct vm_object_fault_info fault_info_dst;
 
 	/*
 	 * In order not to confuse the clustered pageins, align
 	 * the different offsets on a page boundary.
 	 */
-	vm_object_offset_t	src_lo_offset = vm_object_trunc_page(src_offset);
-	vm_object_offset_t	dst_lo_offset = vm_object_trunc_page(dst_offset);
-	vm_object_offset_t	src_hi_offset = vm_object_round_page(src_offset + *copy_size);
-	vm_object_offset_t	dst_hi_offset = vm_object_round_page(dst_offset + *copy_size);
 
 #define	RETURN(x)					\
 	MACRO_BEGIN					\
@@ -3615,6 +3773,21 @@ vm_fault_copy(
 	MACRO_END
 
 	amount_left = *copy_size;
+
+	fault_info_src.interruptible = interruptible;
+	fault_info_src.behavior = VM_BEHAVIOR_SEQUENTIAL;
+	fault_info_src.user_tag  = 0;
+	fault_info_src.lo_offset = vm_object_trunc_page(src_offset);
+	fault_info_src.hi_offset = fault_info_src.lo_offset + amount_left;
+	fault_info_src.no_cache   = FALSE;
+
+	fault_info_dst.interruptible = interruptible;
+	fault_info_dst.behavior = VM_BEHAVIOR_SEQUENTIAL;
+	fault_info_dst.user_tag  = 0;
+	fault_info_dst.lo_offset = vm_object_trunc_page(dst_offset);
+	fault_info_dst.hi_offset = fault_info_dst.lo_offset + amount_left;
+	fault_info_dst.no_cache   = FALSE;
+
 	do { /* while (amount_left > 0) */
 		/*
 		 * There may be a deadlock if both source and destination
@@ -3630,22 +3803,18 @@ vm_fault_copy(
 		vm_object_lock(dst_object);
 		vm_object_paging_begin(dst_object);
 
+		fault_info_dst.cluster_size = amount_left;
+
 		XPR(XPR_VM_FAULT,"vm_fault_copy -> vm_fault_page\n",0,0,0,0,0);
 		switch (vm_fault_page(dst_object,
 				      vm_object_trunc_page(dst_offset),
 				      VM_PROT_WRITE|VM_PROT_READ,
 				      FALSE,
-				      interruptible,
-				      dst_lo_offset,
-				      dst_hi_offset,
-				      VM_BEHAVIOR_SEQUENTIAL,
-				      &dst_prot,
-				      &dst_page,
-				      &dst_top_page,
+				      &dst_prot, &dst_page, &dst_top_page,
 				      (int *)0,
 				      &error,
 				      dst_map->no_zero_fill,
-				      FALSE, NULL, 0)) {
+				      FALSE, &fault_info_dst)) {
 		case VM_FAULT_SUCCESS:
 			break;
 		case VM_FAULT_RETRY:
@@ -3656,9 +3825,6 @@ vm_fault_copy(
 			/* fall thru */
 		case VM_FAULT_INTERRUPTED:
 			RETURN(MACH_SEND_INTERRUPTED);
-		case VM_FAULT_FICTITIOUS_SHORTAGE:
-			vm_page_more_fictitious();
-			goto RetryDestinationFault;
 		case VM_FAULT_MEMORY_ERROR:
 			if (error)
 				return (error);
@@ -3681,7 +3847,7 @@ vm_fault_copy(
 		 * holding the dest page so it doesn't go away.
 		 */
 
-		vm_page_lock_queues();
+		vm_page_lockspin_queues();
 		vm_page_wire(dst_page);
 		vm_page_unlock_queues();
 		PAGE_WAKEUP_DONE(dst_page);
@@ -3714,24 +3880,19 @@ vm_fault_copy(
 				src_prot = VM_PROT_READ;
 				vm_object_paging_begin(src_object);
 
+				fault_info_src.cluster_size = amount_left;
+
 				XPR(XPR_VM_FAULT,
 					"vm_fault_copy(2) -> vm_fault_page\n",
 					0,0,0,0,0);
-				switch (vm_fault_page(src_object, 
-						      vm_object_trunc_page(src_offset),
-						      VM_PROT_READ, 
-						      FALSE, 
-						      interruptible,
-						      src_lo_offset,
-						      src_hi_offset,
-						      VM_BEHAVIOR_SEQUENTIAL,
-						      &src_prot, 
-						      &result_page,
-						      &src_top_page,
-						      (int *)0,
-						      &error,
-						      FALSE,
-						      FALSE, NULL, 0)) {
+				switch (vm_fault_page(
+						src_object, 
+						vm_object_trunc_page(src_offset),
+						VM_PROT_READ, FALSE,
+						&src_prot, 
+						&result_page, &src_top_page,
+						(int *)0, &error, FALSE,
+						FALSE, &fault_info_src)) {
 
 				case VM_FAULT_SUCCESS:
 					break;
@@ -3744,9 +3905,6 @@ vm_fault_copy(
 				case VM_FAULT_INTERRUPTED:
 					vm_fault_copy_dst_cleanup(dst_page);
 					RETURN(MACH_SEND_INTERRUPTED);
-				case VM_FAULT_FICTITIOUS_SHORTAGE:
-					vm_page_more_fictitious();
-					goto RetrySourceFault;
 				case VM_FAULT_MEMORY_ERROR:
 					vm_fault_copy_dst_cleanup(dst_page);
 					if (error)
@@ -3856,165 +4014,6 @@ vm_fault_copy(
 	/*NOTREACHED*/	
 }
 
-#ifdef	notdef
-
-/*
- *	Routine:	vm_fault_page_overwrite
- *
- *	Description:
- *		A form of vm_fault_page that assumes that the
- *		resulting page will be overwritten in its entirety,
- *		making it unnecessary to obtain the correct *contents*
- *		of the page.
- *
- *	Implementation:
- *		XXX Untested.  Also unused.  Eventually, this technology
- *		could be used in vm_fault_copy() to advantage.
- */
-vm_fault_return_t
-vm_fault_page_overwrite(
-	register
-	vm_object_t		dst_object,
-	vm_object_offset_t	dst_offset,
-	vm_page_t		*result_page)	/* OUT */
-{
-	register
-	vm_page_t	dst_page;
-	kern_return_t	wait_result;
-
-#define	interruptible	THREAD_UNINT	/* XXX */
-
-	while (TRUE) {
-		/*
-		 *	Look for a page at this offset
-		 */
-
-		while ((dst_page = vm_page_lookup(dst_object, dst_offset))
-				 == VM_PAGE_NULL) {
-			/*
-			 *	No page, no problem... just allocate one.
-			 */
-
-			dst_page = vm_page_alloc(dst_object, dst_offset);
-			if (dst_page == VM_PAGE_NULL) {
-				vm_object_unlock(dst_object);
-				VM_PAGE_WAIT();
-				vm_object_lock(dst_object);
-				continue;
-			}
-
-			/*
-			 *	Pretend that the memory manager
-			 *	write-protected the page.
-			 *
-			 *	Note that we will be asking for write
-			 *	permission without asking for the data
-			 *	first.
-			 */
-
-			dst_page->overwriting = TRUE;
-			dst_page->page_lock = VM_PROT_WRITE;
-			dst_page->absent = TRUE;
-			dst_page->unusual = TRUE;
-			dst_object->absent_count++;
-
-			break;
-
-			/*
-			 *	When we bail out, we might have to throw
-			 *	away the page created here.
-			 */
-
-#define	DISCARD_PAGE						\
-	MACRO_BEGIN						\
-	vm_object_lock(dst_object);				\
-	dst_page = vm_page_lookup(dst_object, dst_offset);	\
-	if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
-	   	VM_PAGE_FREE(dst_page);				\
-	vm_object_unlock(dst_object);				\
-	MACRO_END
-		}
-
-		/*
-		 *	If the page is write-protected...
-		 */
-
-		if (dst_page->page_lock & VM_PROT_WRITE) {
-			/*
-			 *	... and an unlock request hasn't been sent
-			 */
-
-			if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
-				vm_prot_t	u;
-				kern_return_t	rc;
-
-				/*
-				 *	... then send one now.
-				 */
-
-				if (!dst_object->pager_ready) {
-					wait_result = vm_object_assert_wait(dst_object,
-								VM_OBJECT_EVENT_PAGER_READY,
-								interruptible);
-					vm_object_unlock(dst_object);
-					if (wait_result == THREAD_WAITING)
-						wait_result = thread_block(THREAD_CONTINUE_NULL);
-					if (wait_result != THREAD_AWAKENED) {
-						DISCARD_PAGE;
-						return(VM_FAULT_INTERRUPTED);
-					}
-					continue;
-				}
-
-				u = dst_page->unlock_request |= VM_PROT_WRITE;
-				vm_object_unlock(dst_object);
-
-				if ((rc = memory_object_data_unlock(
-						dst_object->pager,
-						dst_offset + dst_object->paging_offset,
-						PAGE_SIZE,
-						u)) != KERN_SUCCESS) {
-					if (vm_fault_debug)
-				     	    printf("vm_object_overwrite: memory_object_data_unlock failed\n");
-					DISCARD_PAGE;
-					return((rc == MACH_SEND_INTERRUPTED) ?
-						VM_FAULT_INTERRUPTED :
-						VM_FAULT_MEMORY_ERROR);
-				}
-				vm_object_lock(dst_object);
-				continue;
-			}
-
-			/* ... fall through to wait below */
-		} else {
-			/*
-			 *	If the page isn't being used for other
-			 *	purposes, then we're done.
-			 */
-			if ( ! (dst_page->busy || dst_page->absent ||
-				dst_page->error || dst_page->restart) )
-				break;
-		}
-
-		wait_result = PAGE_ASSERT_WAIT(dst_page, interruptible);
-		vm_object_unlock(dst_object);
-		if (wait_result == THREAD_WAITING)
-			wait_result = thread_block(THREAD_CONTINUE_NULL);
-		if (wait_result != THREAD_AWAKENED) {
-			DISCARD_PAGE;
-			return(VM_FAULT_INTERRUPTED);
-		}
-	}
-
-	*result_page = dst_page;
-	return(VM_FAULT_SUCCESS);
-
-#undef	interruptible
-#undef	DISCARD_PAGE
-}
-
-#endif	/* notdef */
-
 #if	VM_FAULT_CLASSIFY
 /*
  *	Temporary statistics gathering support.
@@ -4046,8 +4045,7 @@ vm_fault_classify(vm_object_t		object,
 	while (TRUE) {
 		m = vm_page_lookup(object, offset);
 		if (m != VM_PAGE_NULL) {		
-			if (m->busy || m->error || m->restart || m->absent ||
-			    fault_type & m->page_lock) {
+		        if (m->busy || m->error || m->restart || m->absent) {
 				type = VM_FAULT_TYPE_OTHER;
 				break;
 			}
@@ -4100,3 +4098,113 @@ vm_fault_classify_init(void)
 	return;
 }
 #endif	/* VM_FAULT_CLASSIFY */
+
+
+extern int cs_validation;
+
+void
+vm_page_validate_cs(
+	vm_page_t	page)
+{
+	vm_object_t		object;
+	vm_object_offset_t	offset;
+	vm_map_offset_t		koffset;
+	vm_map_size_t		ksize;
+	vm_offset_t		kaddr;
+	kern_return_t		kr;
+	memory_object_t		pager;
+	void			*blobs;
+	boolean_t		validated, tainted;
+	boolean_t		busy_page;
+
+	vm_object_lock_assert_exclusive(page->object);
+	assert(!page->cs_validated);
+
+	if (!cs_validation) {
+		return;
+	}
+
+	object = page->object;
+	assert(object->code_signed);
+	offset = page->offset;
+
+	busy_page = page->busy;
+	if (!busy_page) {
+		/* keep page busy while we map (and unlock) the VM object */
+		page->busy = TRUE;
+	}
+	
+	/*
+	 * Take a paging reference on the VM object
+	 * to protect it from collapse or bypass,
+	 * and keep it from disappearing too.
+	 */
+	vm_object_paging_begin(object);
+
+	/* map the page in the kernel address space */
+	koffset = 0;
+	ksize = PAGE_SIZE_64;
+	kr = vm_paging_map_object(&koffset,
+				  page,
+				  object,
+				  offset,
+				  &ksize,
+				  FALSE); /* can't unlock object ! */
+	if (kr != KERN_SUCCESS) {
+		panic("vm_page_validate_cs: could not map page: 0x%x\n", kr);
+	}
+	kaddr = CAST_DOWN(vm_offset_t, koffset);
+
+	/*
+	 * Since we get here to validate a page that was brought in by
+	 * the pager, we know that this pager is all setup and ready
+	 * by now.
+	 */
+	assert(!object->internal);
+	assert(object->pager != NULL);
+	assert(object->pager_ready);
+
+	if (!object->alive || object->terminating || object->pager == NULL) {
+		/*
+		 * The object is terminating and we don't have its pager
+		 * so we can't validate the data...
+		 */
+		goto out;
+	}
+
+	pager = object->pager;
+	assert(pager != NULL);
+
+	kr = vnode_pager_get_object_cs_blobs(pager, &blobs);
+	if (kr != KERN_SUCCESS) {
+		blobs = NULL;
+	}
+
+	/* verify the SHA1 hash for this page */
+	validated = cs_validate_page(blobs,
+				     offset + object->paging_offset,
+				     (const void *)kaddr,
+				     &tainted);
+
+	assert(page->busy);
+	assert(object == page->object);
+	vm_object_lock_assert_exclusive(object);
+
+	page->cs_validated = validated;
+	if (validated) {
+		page->cs_tainted = tainted;
+	}
+
+out:
+	if (!busy_page) {
+		PAGE_WAKEUP_DONE(page);
+	}
+	if (koffset != 0) {
+		/* unmap the map from the kernel address space */
+		vm_paging_unmap_object(object, koffset, koffset + ksize);
+		koffset = 0;
+		ksize = 0;
+		kaddr = 0;
+	}
+	vm_object_paging_end(object);
+}
diff --git a/osfmk/vm/vm_fault.h b/osfmk/vm/vm_fault.h
index 4941efe69..439af2540 100644
--- a/osfmk/vm/vm_fault.h
+++ b/osfmk/vm/vm_fault.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -107,10 +113,6 @@ extern vm_fault_return_t vm_fault_page(
 		vm_object_offset_t first_offset,/* Offset into object */
 		vm_prot_t	fault_type,	/* What access is requested */
 		boolean_t	must_be_resident,/* Must page be resident? */
-		int		interruptible,/* how may fault be interrupted */
-		vm_map_offset_t lo_offset,	/* Map entry start */
-		vm_map_offset_t hi_offset,	/* Map entry end */
-		vm_behavior_t	behavior,	/* Expected paging behavior */
 		/* Modifies in place: */
 		vm_prot_t	*protection,	/* Protection for mapping */
 		/* Returns: */
@@ -123,8 +125,7 @@ extern vm_fault_return_t vm_fault_page(
 		kern_return_t	*error_code,	/* code if page is in error */
 		boolean_t	no_zero_fill,	/* don't fill absent pages */
 		boolean_t	data_supply,	/* treat as data_supply */
-		vm_map_t	map,
-		vm_map_offset_t	vaddr);
+	        vm_object_fault_info_t fault_info);
 
 extern void vm_fault_cleanup(
 		vm_object_t	object,
@@ -153,6 +154,16 @@ extern kern_return_t	vm_fault_copy(
 		vm_map_version_t	 *dst_version,
 		int			interruptible);
 
+extern kern_return_t vm_fault_enter(
+	vm_page_t m,
+	pmap_t pmap,
+	vm_map_offset_t vaddr,
+	vm_prot_t prot,
+	boolean_t wired,
+	boolean_t change_wiring,
+	boolean_t no_cache,
+	int *type_of_fault);
+
 #endif	/* MACH_KERNEL_PRIVATE */
 
 #endif	/* KERNEL_PRIVATE */
diff --git a/osfmk/vm/vm_init.c b/osfmk/vm/vm_init.c
index 0b283b4b7..11927893c 100644
--- a/osfmk/vm/vm_init.c
+++ b/osfmk/vm/vm_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -72,13 +78,16 @@
 
 #include <vm/vm_protos.h>
 
-#define ZONE_MAP_MIN (12 * 1024 * 1024) 
-/* Maximum Zone size is 1G */
-#define ZONE_MAP_MAX (1024 * 1024 * 1024) 
+#define ZONE_MAP_MIN CONFIG_ZONE_MAP_MIN
+
+/* Maximum zone size is 1.5G */
+#define ZONE_MAP_MAX (1024 * 1024 * 1536) 
 
 const vm_offset_t vm_min_kernel_address = VM_MIN_KERNEL_ADDRESS;
 const vm_offset_t vm_max_kernel_address = VM_MAX_KERNEL_ADDRESS;
 
+boolean_t vm_kernel_ready = FALSE;
+
 /*
  *	vm_mem_bootstrap initializes the virtual memory system.
  *	This is done only by the first cpu up.
@@ -96,17 +105,30 @@ vm_mem_bootstrap(void)
 	 *	From here on, all physical memory is accounted for,
 	 *	and we use only virtual addresses.
 	 */
+#define vm_mem_bootstrap_kprintf(x)
 
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling vm_page_bootstrap\n"));
 	vm_page_bootstrap(&start, &end);
 
 	/*
 	 *	Initialize other VM packages
 	 */
 
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling zone_bootstrap\n"));
 	zone_bootstrap();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling vm_object_bootstrap\n"));
 	vm_object_bootstrap();
+
+	vm_kernel_ready = TRUE;
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling vm_map_int\n"));
 	vm_map_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling kmem_init\n"));
 	kmem_init(start, end);
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling pmap_init\n"));
 	pmap_init();
 	
 	if (PE_parse_boot_arg("zsize", &zsizearg))
@@ -117,14 +139,31 @@ vm_mem_bootstrap(void)
 
 	if(zsize < ZONE_MAP_MIN) zsize = ZONE_MAP_MIN;	/* Clamp to min */
 	if(zsize > ZONE_MAP_MAX) zsize = ZONE_MAP_MAX;	/* Clamp to max */
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling zone_init\n"));
 	zone_init(zsize);						/* Allocate address space for zones */
 	
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling kalloc_init\n"));
 	kalloc_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling vm_fault_init\n"));
 	vm_fault_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling vm_page_module_init\n"));
 	vm_page_module_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling memory_manager_default_init\n"));
 	memory_manager_default_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling meory_object_control_bootstrap\n"));
 	memory_object_control_bootstrap();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: calling device_pager_bootstrap\n"));
 	device_pager_bootstrap();
+
+	vm_paging_map_init();
+
+	vm_mem_bootstrap_kprintf(("vm_mem_bootstrap: done\n"));
 }
 
 void
diff --git a/osfmk/vm/vm_init.h b/osfmk/vm/vm_init.h
index 98733d845..06beca0ee 100644
--- a/osfmk/vm/vm_init.h
+++ b/osfmk/vm/vm_init.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -26,8 +32,8 @@
 #ifndef VM_INIT_H
 #define VM_INIT_H
 
-extern void vm_mem_bootstrap(void);
-extern void vm_mem_init(void);
+extern void vm_mem_bootstrap(void) __attribute__((section("__TEXT, initcode")));
+extern void vm_mem_init(void) __attribute__((section("__TEXT, initcode")));
 extern void vm_map_steal_memory(void);
 
 #endif /* VM_INIT_H */
diff --git a/osfmk/vm/vm_kern.c b/osfmk/vm/vm_kern.c
index 14a276d00..f8b306855 100644
--- a/osfmk/vm/vm_kern.c
+++ b/osfmk/vm/vm_kern.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -71,6 +77,10 @@
 #include <vm/cpm.h>
 
 #include <string.h>
+
+#include <libkern/OSDebug.h>
+#include <sys/kdebug.h>
+
 /*
  *	Variables exported by this module.
  */
@@ -78,6 +88,8 @@
 vm_map_t	kernel_map;
 vm_map_t	kernel_pageable_map;
 
+extern boolean_t vm_kernel_ready;
+
 /*
  * Forward declarations for internal functions.
  */
@@ -99,6 +111,7 @@ kmem_alloc_contig(
 	vm_offset_t		*addrp,
 	vm_size_t		size,
 	vm_offset_t 		mask,
+	ppnum_t			max_pnum,
 	int 			flags)
 {
 	vm_object_t		object;
@@ -147,7 +160,7 @@ kmem_alloc_contig(
 	vm_object_reference(object);
 	vm_map_unlock(map);
 
-	kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, FALSE);
+	kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, FALSE);
 
 	if (kr != KERN_SUCCESS) {
 		vm_map_remove(map, vm_map_trunc_page(map_addr),
@@ -217,11 +230,17 @@ kernel_memory_allocate(
 	vm_object_t 		object;
 	vm_object_offset_t 	offset;
 	vm_map_entry_t 		entry;
-	vm_map_offset_t 	map_addr;
+	vm_map_offset_t 	map_addr, fill_start;
 	vm_map_offset_t		map_mask;
-	vm_map_size_t		map_size;
+	vm_map_size_t		map_size, fill_size;
 	vm_map_size_t		i;
 	kern_return_t 		kr;
+	vm_page_t		mem;
+	int			vm_alloc_flags;
+
+	if (! vm_kernel_ready) {
+		panic("kernel_memory_allocate: VM is not ready");
+	}
 
 	if (size == 0) {
 		*addrp = 0;
@@ -236,6 +255,44 @@ kernel_memory_allocate(
 
 	map_size = vm_map_round_page(size);
 	map_mask = (vm_map_offset_t) mask;
+	vm_alloc_flags = 0;
+
+	/*
+	 * Guard pages:
+	 *
+	 * Guard pages are implemented as ficticious pages.  By placing guard pages
+	 * on either end of a stack, they can help detect cases where a thread walks
+	 * off either end of its stack.  They are allocated and set up here and attempts
+	 * to access those pages are trapped in vm_fault_page().
+	 *
+	 * The map_size we were passed may include extra space for
+	 * guard pages.  If those were requested, then back it out of fill_size
+	 * since vm_map_find_space() takes just the actual size not including
+	 * guard pages.  Similarly, fill_start indicates where the actual pages
+	 * will begin in the range.
+	 */
+
+	fill_start = 0;
+	fill_size = map_size;
+	if (flags & KMA_GUARD_FIRST) {
+		vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE;
+		fill_start += PAGE_SIZE_64;
+		fill_size -= PAGE_SIZE_64;
+		if (map_size < fill_start + fill_size) {
+			/* no space for a guard page */
+			*addrp = 0;
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+	if (flags & KMA_GUARD_LAST) {
+		vm_alloc_flags |= VM_FLAGS_GUARD_AFTER;
+		fill_size -= PAGE_SIZE_64;
+		if (map_size <= fill_start + fill_size) {
+			/* no space for a guard page */
+			*addrp = 0;
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
 
 	/*
 	 *	Allocate a new object (if necessary).  We must do this before
@@ -248,11 +305,14 @@ kernel_memory_allocate(
 		object = vm_object_allocate(map_size);
 	}
 
-	kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
+	kr = vm_map_find_space(map, &map_addr,
+			       fill_size, map_mask,
+			       vm_alloc_flags, &entry);
 	if (KERN_SUCCESS != kr) {
 		vm_object_deallocate(object);
 		return kr;
 	}
+
 	entry->object.vm_object = object;
 	entry->offset = offset = (object == kernel_object) ? 
 		        map_addr - VM_MIN_KERNEL_ADDRESS : 0;
@@ -261,9 +321,35 @@ kernel_memory_allocate(
 	vm_map_unlock(map);
 
 	vm_object_lock(object);
-	for (i = 0; i < map_size; i += PAGE_SIZE) {
-		vm_page_t	mem;
 
+	/*
+	 * Allocate the lower guard page if one was requested.  The guard
+	 * page extends up to fill_start which is where the real memory
+	 * begins.
+	 */
+
+	for (i = 0; i < fill_start; i += PAGE_SIZE) {
+		for (;;) {
+			mem = vm_page_alloc_guard(object, offset + i);
+			if (mem != VM_PAGE_NULL)
+				break;
+			if (flags & KMA_NOPAGEWAIT) {
+				kr = KERN_RESOURCE_SHORTAGE;
+				goto nopage;
+			}
+			vm_object_unlock(object);
+			vm_page_more_fictitious();
+			vm_object_lock(object);
+		}
+		mem->busy = FALSE;
+	}
+
+	/*
+	 * Allocate the real memory here.  This extends from offset fill_start
+	 * for fill_size bytes.
+	 */
+
+	for (i = fill_start; i < fill_start + fill_size; i += PAGE_SIZE) {
 		for (;;) {
 		        if (flags & KMA_LOMEM)
 			        mem = vm_page_alloclo(object, offset + i);
@@ -274,12 +360,8 @@ kernel_memory_allocate(
 			        break;
 
 			if (flags & KMA_NOPAGEWAIT) {
-				if (object == kernel_object)
-					vm_object_page_remove(object, offset, offset + i);
-				vm_object_unlock(object);
-				vm_map_remove(map, map_addr, map_addr + map_size, 0);
-				vm_object_deallocate(object);
-				return KERN_RESOURCE_SHORTAGE;
+				kr = KERN_RESOURCE_SHORTAGE;
+				goto nopage;
 			}
 			vm_object_unlock(object);
 			VM_PAGE_WAIT();
@@ -287,19 +369,37 @@ kernel_memory_allocate(
 		}
 		mem->busy = FALSE;
 	}
-	vm_object_unlock(object);
 
-	if ((kr = vm_map_wire(map, map_addr, map_addr + map_size, VM_PROT_DEFAULT, FALSE)) 
-		!= KERN_SUCCESS) {
-		if (object == kernel_object) {
-			vm_object_lock(object);
-			vm_object_page_remove(object, offset, offset + map_size);
+	/*
+	 * Lastly, allocate the ending guard page if requested.  This starts at the ending
+	 * address from the loop above up to the map_size that was originaly 
+	 * requested.
+	 */
+
+	for (i = fill_start + fill_size; i < map_size; i += PAGE_SIZE) {
+		for (;;) {
+			mem = vm_page_alloc_guard(object, offset + i);
+			if (mem != VM_PAGE_NULL)
+				break;
+			if (flags & KMA_NOPAGEWAIT) {
+				kr = KERN_RESOURCE_SHORTAGE;
+				goto nopage;
+			}
 			vm_object_unlock(object);
+			vm_page_more_fictitious();
+			vm_object_lock(object);
 		}
-		vm_map_remove(map, map_addr, map_addr + map_size, 0);
-		vm_object_deallocate(object);
-		return (kr);
+		mem->busy = FALSE;
+	}
+	vm_object_unlock(object);
+
+	kr = vm_map_wire(map, map_addr, map_addr + map_size,
+			 VM_PROT_DEFAULT, FALSE);
+	if (kr != KERN_SUCCESS) {
+		vm_object_lock(object);
+		goto nopage;
 	}
+
 	/* now that the page is wired, we no longer have to fear coalesce */
 	vm_object_deallocate(object);
 	if (object == kernel_object)
@@ -310,6 +410,14 @@ kernel_memory_allocate(
 	 */
 	*addrp = CAST_DOWN(vm_offset_t, map_addr);
 	return KERN_SUCCESS;
+
+nopage:
+	if (object == kernel_object)
+		vm_object_page_remove(object, offset, offset + i);
+	vm_object_unlock(object);
+	vm_map_remove(map, map_addr, map_addr + map_size, 0);
+	vm_object_deallocate(object);
+	return KERN_RESOURCE_SHORTAGE;
 }
 
 /*
@@ -325,7 +433,9 @@ kmem_alloc(
 	vm_offset_t	*addrp,
 	vm_size_t	size)
 {
-	return kernel_memory_allocate(map, addrp, size, 0, 0);
+	kern_return_t kr = kernel_memory_allocate(map, addrp, size, 0, 0);
+	TRACE_MACHLEAKS(KMEM_ALLOC_CODE, KMEM_ALLOC_CODE_2, size, *addrp);
+	return kr;
 }
 
 /*
@@ -537,6 +647,8 @@ kmem_free(
 {
 	kern_return_t kr;
 
+	TRACE_MACHLEAKS(KMEM_FREE_CODE, KMEM_FREE_CODE_2, size, addr);
+
 	kr = vm_map_remove(map, vm_map_trunc_page(addr),
 				vm_map_round_page(addr + size), 
 				VM_MAP_REMOVE_KUNWIRE);
@@ -619,7 +731,7 @@ kmem_remap_pages(
 	    /*
 	     *	Wire it down (again)
 	     */
-	    vm_page_lock_queues();
+	    vm_page_lockspin_queues();
 	    vm_page_wire(mem);
 	    vm_page_unlock_queues();
 	    vm_object_unlock(object);
@@ -738,7 +850,6 @@ kmem_init(
 	/*
 	 *	Reserve virtual memory allocated up to this time.
 	 */
-
 	if (start != VM_MIN_KERNEL_ADDRESS) {
 		vm_map_offset_t map_addr;
  
@@ -754,6 +865,7 @@ kmem_init(
 			    VM_INHERIT_DEFAULT);
 	}
 
+
         /*
          * Account for kernel memory (text, data, bss, vm shenanigans).
          * This may include inaccessible "holes" as determined by what
@@ -762,6 +874,23 @@ kmem_init(
         vm_page_wire_count = (atop_64(max_mem) - (vm_page_free_count
                                                 + vm_page_active_count
                                                 + vm_page_inactive_count));
+
+	/*
+	 * Set the default global user wire limit which limits the amount of
+	 * memory that can be locked via mlock().  We set this to the total number of
+	 * pages that are potentially usable by a user app (max_mem) minus
+	 * 1000 pages.  This keeps 4MB in reserve for the kernel which will hopefully be
+	 * enough to avoid memory deadlocks. If for some reason the system has less than
+	 * 2000 pages of memory at this point, then we'll allow users to lock up to 80%
+	 * of that.  This can be overridden via a sysctl.
+	 */
+
+	if (max_mem > 2000)
+		vm_global_user_wire_limit = max_mem - 1000;
+	else
+		vm_global_user_wire_limit = max_mem * 100 / 80;
+	
+	vm_user_wire_limit = vm_global_user_wire_limit;		/* the default per user limit is the same as the global limit */
 }
 
 
diff --git a/osfmk/vm/vm_kern.h b/osfmk/vm/vm_kern.h
index a8bc5ec11..3e22b0633 100644
--- a/osfmk/vm/vm_kern.h
+++ b/osfmk/vm/vm_kern.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -78,12 +84,15 @@ extern kern_return_t	kernel_memory_allocate(
 #define KMA_NOPAGEWAIT	0x02
 #define KMA_KOBJECT	0x04
 #define KMA_LOMEM	0x08
+#define KMA_GUARD_FIRST	0x10
+#define KMA_GUARD_LAST	0x20
 
 extern kern_return_t kmem_alloc_contig(
 				vm_map_t	map,
 				vm_offset_t	*addrp,
 				vm_size_t	size,
 				vm_offset_t 	mask,
+				ppnum_t		max_pnum,
 				int 		flags);
 
 extern kern_return_t	kmem_alloc(
@@ -113,17 +122,6 @@ extern void		kmem_free(
 				vm_offset_t	addr,
 				vm_size_t	size);
 
-#ifdef	MACH_KERNEL_PRIVATE
-
-extern void		kmem_init(
-					vm_offset_t	start,
-					vm_offset_t	end);
-
-extern kern_return_t	kmem_alloc_wired(
-				vm_map_t	map,
-				vm_offset_t	*addrp,
-				vm_size_t	size);
-
 extern kern_return_t	kmem_suballoc(
 				vm_map_t	parent,
 				vm_offset_t	*addr,
@@ -132,6 +130,21 @@ extern kern_return_t	kmem_suballoc(
 				boolean_t	anywhere,
 				vm_map_t	*new_map);
 
+
+#ifdef XNU_KERNEL_PRIVATE
+extern kern_return_t	kmem_alloc_wired(
+				vm_map_t	map,
+				vm_offset_t	*addrp,
+				vm_size_t	size);
+#endif
+
+#ifdef	MACH_KERNEL_PRIVATE
+
+extern void		kmem_init(
+					vm_offset_t	start,
+					vm_offset_t	end) __attribute__((section("__TEXT, initcode")));
+
+
 extern kern_return_t	copyinmap(
 				vm_map_t	map,
 				vm_map_offset_t	fromaddr,
diff --git a/osfmk/vm/vm_map.c b/osfmk/vm/vm_map.c
index 3f0d8bf31..f20b587c1 100644
--- a/osfmk/vm/vm_map.c
+++ b/osfmk/vm/vm_map.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,6 +76,7 @@
 #include <mach/memory_object.h>
 #include <mach/mach_vm.h>
 #include <machine/cpu_capabilities.h>
+#include <mach/sdt.h>
 
 #include <kern/assert.h>
 #include <kern/counters.h>
@@ -92,173 +99,174 @@
 
 #include <mach/vm_map_server.h>
 #include <mach/mach_host_server.h>
-#include <vm/vm_shared_memory_server.h>
-#include <vm/vm_protos.h>       // for vm_map_commpage64 and vm_map_remove_compage64
+#include <vm/vm_protos.h>
 
 #ifdef ppc
 #include <ppc/mappings.h>
 #endif /* ppc */
 
 #include <vm/vm_protos.h>
+#include <vm/vm_shared_region.h>
 
 /* Internal prototypes
  */
- 
+
 static void vm_map_simplify_range(
 	vm_map_t	map,
 	vm_map_offset_t	start,
 	vm_map_offset_t	end);	/* forward */
 
 static boolean_t	vm_map_range_check(
-				vm_map_t	map,
-				vm_map_offset_t	start,
-				vm_map_offset_t	end,
-				vm_map_entry_t	*entry);
+	vm_map_t	map,
+	vm_map_offset_t	start,
+	vm_map_offset_t	end,
+	vm_map_entry_t	*entry);
 
 static vm_map_entry_t	_vm_map_entry_create(
-				struct vm_map_header	*map_header);
+	struct vm_map_header	*map_header);
 
 static void		_vm_map_entry_dispose(
-				struct vm_map_header	*map_header,
-				vm_map_entry_t		entry);
+	struct vm_map_header	*map_header,
+	vm_map_entry_t		entry);
 
 static void		vm_map_pmap_enter(
-				vm_map_t		map,
-				vm_map_offset_t 	addr,
-				vm_map_offset_t		end_addr,
-				vm_object_t 		object,
-				vm_object_offset_t	offset,
-				vm_prot_t		protection);
+	vm_map_t		map,
+	vm_map_offset_t 	addr,
+	vm_map_offset_t		end_addr,
+	vm_object_t 		object,
+	vm_object_offset_t	offset,
+	vm_prot_t		protection);
 
 static void		_vm_map_clip_end(
-				struct vm_map_header	*map_header,
-				vm_map_entry_t		entry,
-				vm_map_offset_t		end);
+	struct vm_map_header	*map_header,
+	vm_map_entry_t		entry,
+	vm_map_offset_t		end);
 
 static void		_vm_map_clip_start(
-				struct vm_map_header	*map_header,
-				vm_map_entry_t		entry,
-				vm_map_offset_t		start);
+	struct vm_map_header	*map_header,
+	vm_map_entry_t		entry,
+	vm_map_offset_t		start);
 
 static void		vm_map_entry_delete(
-				vm_map_t	map,
-				vm_map_entry_t	entry);
+	vm_map_t	map,
+	vm_map_entry_t	entry);
 
 static kern_return_t	vm_map_delete(
-				vm_map_t	map,
-				vm_map_offset_t	start,
-				vm_map_offset_t	end,
-				int		flags,
-				vm_map_t	zap_map);
+	vm_map_t	map,
+	vm_map_offset_t	start,
+	vm_map_offset_t	end,
+	int		flags,
+	vm_map_t	zap_map);
 
 static kern_return_t	vm_map_copy_overwrite_unaligned(
-				vm_map_t	dst_map,
-				vm_map_entry_t	entry,
-				vm_map_copy_t	copy,
-				vm_map_address_t start);
+	vm_map_t	dst_map,
+	vm_map_entry_t	entry,
+	vm_map_copy_t	copy,
+	vm_map_address_t start);
 
 static kern_return_t	vm_map_copy_overwrite_aligned(
-				vm_map_t	dst_map,
-				vm_map_entry_t	tmp_entry,
-				vm_map_copy_t	copy,
-				vm_map_offset_t start,
-				pmap_t		pmap);
+	vm_map_t	dst_map,
+	vm_map_entry_t	tmp_entry,
+	vm_map_copy_t	copy,
+	vm_map_offset_t start,
+	pmap_t		pmap);
 
 static kern_return_t	vm_map_copyin_kernel_buffer(
-				vm_map_t	src_map,
-				vm_map_address_t src_addr,
-				vm_map_size_t	len,
-				boolean_t	src_destroy,
-				vm_map_copy_t	*copy_result);  /* OUT */
+	vm_map_t	src_map,
+	vm_map_address_t src_addr,
+	vm_map_size_t	len,
+	boolean_t	src_destroy,
+	vm_map_copy_t	*copy_result);  /* OUT */
 
 static kern_return_t	vm_map_copyout_kernel_buffer(
-				vm_map_t	map,
-				vm_map_address_t *addr,	/* IN/OUT */
-				vm_map_copy_t	copy,
-				boolean_t	overwrite);
+	vm_map_t	map,
+	vm_map_address_t *addr,	/* IN/OUT */
+	vm_map_copy_t	copy,
+	boolean_t	overwrite);
 
 static void		vm_map_fork_share(
-				vm_map_t	old_map,
-				vm_map_entry_t	old_entry,
-				vm_map_t	new_map);
+	vm_map_t	old_map,
+	vm_map_entry_t	old_entry,
+	vm_map_t	new_map);
 
 static boolean_t	vm_map_fork_copy(
-				vm_map_t	old_map,
-				vm_map_entry_t	*old_entry_p,
-				vm_map_t	new_map);
+	vm_map_t	old_map,
+	vm_map_entry_t	*old_entry_p,
+	vm_map_t	new_map);
 
 void		vm_map_region_top_walk(
-        			vm_map_entry_t		   entry,
-				vm_region_top_info_t       top);
+	vm_map_entry_t		   entry,
+	vm_region_top_info_t       top);
 
 void		vm_map_region_walk(
-				vm_map_t		   map,
-				vm_map_offset_t		   va,
-        			vm_map_entry_t		   entry,
-				vm_object_offset_t	   offset,
-				vm_object_size_t	   range,
-				vm_region_extended_info_t  extended);
+	vm_map_t		   map,
+	vm_map_offset_t		   va,
+	vm_map_entry_t		   entry,
+	vm_object_offset_t	   offset,
+	vm_object_size_t	   range,
+	vm_region_extended_info_t  extended,
+	boolean_t		   look_for_pages);
 
 static kern_return_t	vm_map_wire_nested(
-				vm_map_t		   map,
-				vm_map_offset_t		   start,
-				vm_map_offset_t		   end,
-				vm_prot_t		   access_type,
-				boolean_t		   user_wire,
-				pmap_t			   map_pmap, 
-				vm_map_offset_t		   pmap_addr);
+	vm_map_t		   map,
+	vm_map_offset_t		   start,
+	vm_map_offset_t		   end,
+	vm_prot_t		   access_type,
+	boolean_t		   user_wire,
+	pmap_t			   map_pmap, 
+	vm_map_offset_t		   pmap_addr);
 
 static kern_return_t	vm_map_unwire_nested(
-				vm_map_t		   map,
-				vm_map_offset_t		   start,
-				vm_map_offset_t		   end,
-				boolean_t		   user_wire,
-				pmap_t			   map_pmap,
-				vm_map_offset_t		   pmap_addr);
+	vm_map_t		   map,
+	vm_map_offset_t		   start,
+	vm_map_offset_t		   end,
+	boolean_t		   user_wire,
+	pmap_t			   map_pmap,
+	vm_map_offset_t		   pmap_addr);
 
 static kern_return_t	vm_map_overwrite_submap_recurse(
-				vm_map_t		   dst_map,
-				vm_map_offset_t		   dst_addr,
-				vm_map_size_t		   dst_size);
+	vm_map_t		   dst_map,
+	vm_map_offset_t		   dst_addr,
+	vm_map_size_t		   dst_size);
 
 static kern_return_t	vm_map_copy_overwrite_nested(
-				vm_map_t		   dst_map,
-				vm_map_offset_t		   dst_addr,
-				vm_map_copy_t		   copy,
-				boolean_t		   interruptible,
-				pmap_t			   pmap);
+	vm_map_t		   dst_map,
+	vm_map_offset_t		   dst_addr,
+	vm_map_copy_t		   copy,
+	boolean_t		   interruptible,
+	pmap_t			   pmap);
 
 static kern_return_t	vm_map_remap_extract(
-				vm_map_t		map,
-				vm_map_offset_t		addr,
-				vm_map_size_t		size,
-				boolean_t		copy,
-				struct vm_map_header 	*map_header,
-				vm_prot_t		*cur_protection,
-				vm_prot_t		*max_protection,
-				vm_inherit_t		inheritance,
-				boolean_t		pageable);
+	vm_map_t		map,
+	vm_map_offset_t		addr,
+	vm_map_size_t		size,
+	boolean_t		copy,
+	struct vm_map_header 	*map_header,
+	vm_prot_t		*cur_protection,
+	vm_prot_t		*max_protection,
+	vm_inherit_t		inheritance,
+	boolean_t		pageable);
 
 static kern_return_t	vm_map_remap_range_allocate(
-				vm_map_t		map,
-				vm_map_address_t	*address,
-				vm_map_size_t		size,
-				vm_map_offset_t		mask,
-				boolean_t		anywhere,
-				vm_map_entry_t		*map_entry);
+	vm_map_t		map,
+	vm_map_address_t	*address,
+	vm_map_size_t		size,
+	vm_map_offset_t		mask,
+	boolean_t		anywhere,
+	vm_map_entry_t		*map_entry);
 
 static void		vm_map_region_look_for_page(
-				vm_map_t		   map,
-				vm_map_offset_t            va,
-				vm_object_t		   object,
-				vm_object_offset_t	   offset,
-				int                        max_refcnt,
-				int                        depth,
-				vm_region_extended_info_t  extended);
+	vm_map_t		   map,
+	vm_map_offset_t            va,
+	vm_object_t		   object,
+	vm_object_offset_t	   offset,
+	int                        max_refcnt,
+	int                        depth,
+	vm_region_extended_info_t  extended);
 
 static int		vm_map_region_count_obj_refs(
-				vm_map_entry_t    	   entry,
-				vm_object_t       	   object);
+	vm_map_entry_t    	   entry,
+	vm_object_t       	   object);
 
 /*
  * Macros to copy a vm_map_entry. We must be careful to correctly
@@ -271,16 +279,69 @@ static int		vm_map_region_count_obj_refs(
  */
 #define vm_map_entry_copy(NEW,OLD) \
 MACRO_BEGIN                                     \
-                *(NEW) = *(OLD);                \
-                (NEW)->is_shared = FALSE;	\
-                (NEW)->needs_wakeup = FALSE;    \
-                (NEW)->in_transition = FALSE;   \
-                (NEW)->wired_count = 0;         \
-                (NEW)->user_wired_count = 0;    \
+	*(NEW) = *(OLD);                \
+	(NEW)->is_shared = FALSE;	\
+	(NEW)->needs_wakeup = FALSE;    \
+	(NEW)->in_transition = FALSE;   \
+	(NEW)->wired_count = 0;         \
+	(NEW)->user_wired_count = 0;    \
 MACRO_END
 
 #define vm_map_entry_copy_full(NEW,OLD)        (*(NEW) = *(OLD))
 
+/*
+ *	Decide if we want to allow processes to execute from their data or stack areas.
+ *	override_nx() returns true if we do.  Data/stack execution can be enabled independently 
+ *	for 32 and 64 bit processes.  Set the VM_ABI_32 or VM_ABI_64 flags in allow_data_exec
+ *	or allow_stack_exec to enable data execution for that type of data area for that particular
+ *	ABI (or both by or'ing the flags together).  These are initialized in the architecture
+ *	specific pmap files since the default behavior varies according to architecture.  The 
+ *	main reason it varies is because of the need to provide binary compatibility with old 
+ *	applications that were written before these restrictions came into being.  In the old 
+ *	days, an app could execute anything it could read, but this has slowly been tightened 
+ *	up over time.  The default behavior is:
+ *
+ *	32-bit PPC apps		may execute from both stack and data areas
+ *	32-bit Intel apps	may exeucte from data areas but not stack
+ *	64-bit PPC/Intel apps	may not execute from either data or stack
+ *
+ *	An application on any architecture may override these defaults by explicitly
+ *	adding PROT_EXEC permission to the page in question with the mprotect(2) 
+ *	system call.  This code here just determines what happens when an app tries to
+ * 	execute from a page that lacks execute permission.
+ *
+ *	Note that allow_data_exec or allow_stack_exec may also be modified by sysctl to change the
+ *	default behavior for both 32 and 64 bit apps on a system-wide basis.
+ */
+
+extern int allow_data_exec, allow_stack_exec;
+
+int
+override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */
+{
+	int current_abi;
+
+	/*
+	 * Determine if the app is running in 32 or 64 bit mode.
+	 */
+
+	if (vm_map_is_64bit(map))
+		current_abi = VM_ABI_64;
+	else
+		current_abi = VM_ABI_32;
+
+	/*
+	 * Determine if we should allow the execution based on whether it's a 
+	 * stack or data area and the current architecture.
+	 */
+
+	if (user_tag == VM_MEMORY_STACK)
+		return allow_stack_exec & current_abi;
+
+	return allow_data_exec & current_abi;
+}
+
+
 /*
  *	Virtual memory maps provide for the mapping, protection,
  *	and sharing of virtual memory objects.  In addition,
@@ -368,11 +429,6 @@ static int		kentry_count = 2048;		/* to init kentry_data_size */
 
 #define         NO_COALESCE_LIMIT  (1024 * 128)
 
-/*
- *	Threshold for aggressive (eager) page map entering for vm copyout
- *	operations.  Any copyout larger will NOT be aggressively entered.
- */
-static vm_map_size_t vm_map_aggressive_enter_max;		/* set by bootstrap */
 
 /* Skip acquiring locks if we're in the midst of a kernel core dump */
 extern unsigned int not_in_kdp;
@@ -430,17 +486,18 @@ vm_map_apple_protected(
 
 	/* map this memory object in place of the current one */
 	map_addr = start;
-	kr = mach_vm_map(map,
-			 &map_addr,
-			 end - start,
-			 (mach_vm_offset_t) 0,
-			 VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
-			 (ipc_port_t) protected_mem_obj,
-			 map_entry->offset + (start - map_entry->vme_start),
-			 TRUE,
-			 map_entry->protection,
-			 map_entry->max_protection,
-			 map_entry->inheritance);
+	kr = vm_map_enter_mem_object(map,
+				     &map_addr,
+				     end - start,
+				     (mach_vm_offset_t) 0,
+				     VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
+				     (ipc_port_t) protected_mem_obj,
+				     (map_entry->offset +
+				      (start - map_entry->vme_start)),
+				     TRUE,
+				     map_entry->protection,
+				     map_entry->max_protection,
+				     map_entry->inheritance);
 	assert(map_addr == start);
 	if (kr == KERN_SUCCESS) {
 		/* let the pager know that this mem_obj is mapped */
@@ -466,19 +523,19 @@ void
 vm_map_init(
 	void)
 {
-	vm_map_zone = zinit((vm_map_size_t) sizeof(struct vm_map), 40*1024,
-					PAGE_SIZE, "maps");
+	vm_map_zone = zinit((vm_map_size_t) sizeof(struct _vm_map), 40*1024,
+			    PAGE_SIZE, "maps");
 
 	vm_map_entry_zone = zinit((vm_map_size_t) sizeof(struct vm_map_entry),
-					1024*1024, PAGE_SIZE*5,
-					"non-kernel map entries");
+				  1024*1024, PAGE_SIZE*5,
+				  "non-kernel map entries");
 
 	vm_map_kentry_zone = zinit((vm_map_size_t) sizeof(struct vm_map_entry),
-					kentry_data_size, kentry_data_size,
-					"kernel map entries");
+				   kentry_data_size, kentry_data_size,
+				   "kernel map entries");
 
 	vm_map_copy_zone = zinit((vm_map_size_t) sizeof(struct vm_map_copy),
-					16*1024, PAGE_SIZE, "map copies");
+				 16*1024, PAGE_SIZE, "map copies");
 
 	/*
 	 *	Cram the map and kentry zones with initial data.
@@ -495,7 +552,7 @@ void
 vm_map_steal_memory(
 	void)
 {
-	map_data_size = vm_map_round_page(10 * sizeof(struct vm_map));
+	map_data_size = vm_map_round_page(10 * sizeof(struct _vm_map));
 	map_data = pmap_steal_memory(map_data_size);
 
 #if 0
@@ -529,6 +586,7 @@ vm_map_create(
 	vm_map_offset_t	max,
 	boolean_t		pageable)
 {
+	static int		color_seed = 0;
 	register vm_map_t	result;
 
 	result = (vm_map_t) zalloc(vm_map_zone);
@@ -541,6 +599,8 @@ vm_map_create(
 	result->hdr.entries_pageable = pageable;
 
 	result->size = 0;
+	result->user_wire_limit = MACH_VM_MAX_ADDRESS;	/* default limit is unlimited */
+	result->user_wire_size  = 0;
 	result->ref_count = 1;
 #if	TASK_SWAPPER
 	result->res_count = 1;
@@ -555,6 +615,7 @@ vm_map_create(
 	result->wait_for_space = FALSE;
 	result->first_free = vm_map_to_entry(result);
 	result->hint = vm_map_to_entry(result);
+	result->color_rr = (color_seed++) & vm_color_mask;
 	vm_map_lock_init(result);
 	mutex_init(&result->s_lock, 0);
 
@@ -568,10 +629,10 @@ vm_map_create(
  *	given map (or map copy).  No fields are filled.
  */
 #define	vm_map_entry_create(map) \
-	    _vm_map_entry_create(&(map)->hdr)
+	_vm_map_entry_create(&(map)->hdr)
 
 #define	vm_map_copy_entry_create(copy) \
-	    _vm_map_entry_create(&(copy)->cpy_hdr)
+	_vm_map_entry_create(&(copy)->cpy_hdr)
 
 static vm_map_entry_t
 _vm_map_entry_create(
@@ -581,9 +642,9 @@ _vm_map_entry_create(
 	register vm_map_entry_t	entry;
 
 	if (map_header->entries_pageable)
-	    zone = vm_map_entry_zone;
+		zone = vm_map_entry_zone;
 	else
-	    zone = vm_map_kentry_zone;
+		zone = vm_map_kentry_zone;
 
 	entry = (vm_map_entry_t) zalloc(zone);
 	if (entry == VM_MAP_ENTRY_NULL)
@@ -596,15 +657,19 @@ _vm_map_entry_create(
  *	vm_map_entry_dispose:	[ internal use only ]
  *
  *	Inverse of vm_map_entry_create.
+ *
+ * 	write map lock held so no need to
+ *	do anything special to insure correctness
+ * 	of the stores
  */
 #define	vm_map_entry_dispose(map, entry)			\
-MACRO_BEGIN							\
+	MACRO_BEGIN						\
 	if((entry) == (map)->first_free)			\
 		(map)->first_free = vm_map_to_entry(map);	\
 	if((entry) == (map)->hint)				\
 		(map)->hint = vm_map_to_entry(map);		\
 	_vm_map_entry_dispose(&(map)->hdr, (entry));		\
-MACRO_END
+	MACRO_END
 
 #define	vm_map_copy_entry_dispose(map, entry) \
 	_vm_map_entry_dispose(&(copy)->cpy_hdr, (entry))
@@ -617,9 +682,9 @@ _vm_map_entry_dispose(
 	register zone_t		zone;
 
 	if (map_header->entries_pageable)
-	    zone = vm_map_entry_zone;
+		zone = vm_map_entry_zone;
 	else
-	    zone = vm_map_kentry_zone;
+		zone = vm_map_kentry_zone;
 
 	zfree(zone, entry);
 }
@@ -635,7 +700,7 @@ first_free_is_valid(
 
 	if (!first_free_check)
 		return TRUE;
-		
+	
 	entry = vm_map_to_entry(map);
 	next = entry->vme_next;
 	while (vm_map_trunc_page(next->vme_start) == vm_map_trunc_page(entry->vme_end) ||
@@ -647,7 +712,7 @@ first_free_is_valid(
 			break;
 	}
 	if (map->first_free != entry) {
-		printf("Bad first_free for map 0x%x: 0x%x should be 0x%x\n",
+		printf("Bad first_free for map %p: %p should be %p\n",
 		       map, map->first_free, entry);
 		return FALSE;
 	}
@@ -663,7 +728,7 @@ first_free_is_valid(
  * 	The map should be locked.
  */
 #define UPDATE_FIRST_FREE(map, new_first_free) 				\
-MACRO_BEGIN 								\
+	MACRO_BEGIN							\
 	vm_map_t	UFF_map; 					\
 	vm_map_entry_t	UFF_first_free; 				\
 	vm_map_entry_t	UFF_next_entry; 				\
@@ -682,7 +747,7 @@ MACRO_BEGIN 								\
 	} 								\
 	UFF_map->first_free = UFF_first_free; 				\
 	assert(first_free_is_valid(UFF_map));				\
-MACRO_END
+	MACRO_END
 
 /*
  *	vm_map_entry_{un,}link:
@@ -690,14 +755,14 @@ MACRO_END
  *	Insert/remove entries from maps (or map copies).
  */
 #define vm_map_entry_link(map, after_where, entry)			\
-MACRO_BEGIN 								\
+	MACRO_BEGIN							\
 	vm_map_t VMEL_map; 						\
 	vm_map_entry_t VMEL_entry; 					\
 	VMEL_map = (map);						\
 	VMEL_entry = (entry); 						\
 	_vm_map_entry_link(&VMEL_map->hdr, after_where, VMEL_entry); 	\
 	UPDATE_FIRST_FREE(VMEL_map, VMEL_map->first_free); 		\
-MACRO_END
+	MACRO_END
 
 
 #define vm_map_copy_entry_link(copy, after_where, entry)		\
@@ -712,7 +777,7 @@ MACRO_END
 	MACRO_END
 
 #define vm_map_entry_unlink(map, entry)					\
-MACRO_BEGIN 								\
+	MACRO_BEGIN							\
 	vm_map_t VMEU_map; 						\
 	vm_map_entry_t VMEU_entry; 					\
 	vm_map_entry_t VMEU_first_free;					\
@@ -724,7 +789,7 @@ MACRO_BEGIN 								\
 		VMEU_first_free = VMEU_map->first_free;			\
 	_vm_map_entry_unlink(&VMEU_map->hdr, VMEU_entry); 		\
 	UPDATE_FIRST_FREE(VMEU_map, VMEU_first_free);			\
-MACRO_END
+	MACRO_END
 
 #define vm_map_copy_entry_unlink(copy, entry)				\
 	_vm_map_entry_unlink(&(copy)->cpy_hdr, (entry))
@@ -812,22 +877,35 @@ void vm_map_res_deallocate(register vm_map_t map)
  */
 void
 vm_map_destroy(
-	register vm_map_t	map)
+	vm_map_t	map,
+	int		flags)
 {	
 	vm_map_lock(map);
-	(void) vm_map_delete(map, map->min_offset,
-			     map->max_offset, VM_MAP_NO_FLAGS,
-			     VM_MAP_NULL);
+
+	/* clean up regular map entries */
+	(void) vm_map_delete(map, map->min_offset, map->max_offset,
+			     flags, VM_MAP_NULL);
+	/* clean up leftover special mappings (commpage, etc...) */
+#ifdef __ppc__
+	/*
+	 * PPC51: ppc64 is limited to 51-bit addresses.
+	 * Memory beyond this 51-bit limit is mapped specially at the
+	 * pmap level, so do not interfere.
+	 * On PPC64, the commpage is mapped beyond the addressable range
+	 * via a special pmap hack, so ask pmap to clean it explicitly...
+	 */
+	if (map->pmap) {
+		pmap_unmap_sharedpage(map->pmap);
+	}
+	/* ... and do not let regular pmap cleanup apply here */
+	flags |= VM_MAP_REMOVE_NO_PMAP_CLEANUP;
+#endif /* __ppc__ */
+	(void) vm_map_delete(map, 0x0, 0xFFFFFFFFFFFFF000ULL,
+			     flags, VM_MAP_NULL);
 	vm_map_unlock(map);
-	
-	if (map->hdr.nentries!=0)
-		vm_map_remove_commpage(map);
-	
-//	assert(map->hdr.nentries==0);
-//	if(map->hdr.nentries) {				/* (BRINGUP) */
-//		panic("vm_map_destroy: hdr.nentries is not 0 (%d) in map %08X\n", map->hdr.nentries, map);
-//	}
 
+	assert(map->hdr.nentries == 0);
+	
 	if(map->pmap)
 		pmap_destroy(map->pmap);
 
@@ -887,7 +965,7 @@ int vm_map_swap_enable = 1;
 void vm_map_swapin (vm_map_t map)
 {
 	register vm_map_entry_t entry;
-	
+
 	if (!vm_map_swap_enable)	/* debug */
 		return;
 
@@ -1028,9 +1106,9 @@ void vm_map_swapout(vm_map_t map)
  *	and someone else wins and stores their 'hint'
  */
 #define	SAVE_HINT_MAP_READ(map,value) \
-MACRO_BEGIN \
-                OSCompareAndSwap((UInt32)((map)->hint), (UInt32)value, (UInt32 *)(&(map)->hint)); \
-MACRO_END
+	MACRO_BEGIN							\
+	OSCompareAndSwap((UInt32)((map)->hint), (UInt32)value, (UInt32 *)(&(map)->hint)); \
+	MACRO_END
 
 
 /*
@@ -1043,9 +1121,9 @@ MACRO_END
  * 	to just do an assignment
  */
 #define	SAVE_HINT_MAP_WRITE(map,value) \
-MACRO_BEGIN \
-		(map)->hint = (value); \
-MACRO_END
+	MACRO_BEGIN		       \
+	(map)->hint = (value);	       \
+	MACRO_END
 
 /*
  *	vm_map_lookup_entry:	[ internal use only ]
@@ -1076,7 +1154,7 @@ vm_map_lookup_entry(
 		cur = cur->vme_next;
 
 	if (address >= cur->vme_start) {
-	    	/*
+		/*
 		 *	Go from hint to end of list.
 		 *
 		 *	But first, make a quick check to see if
@@ -1094,7 +1172,7 @@ vm_map_lookup_entry(
 		}
 	}
 	else {
-	    	/*
+		/*
 		 *	Go from start to hint, *inclusively*
 		 */
 		last = cur->vme_next;
@@ -1108,7 +1186,7 @@ vm_map_lookup_entry(
 	while (cur != last) {
 		if (cur->vme_end > address) {
 			if (address >= cur->vme_start) {
-			    	/*
+				/*
 				 *	Save this lookup for future
 				 *	hints, and return
 				 */
@@ -1159,6 +1237,11 @@ vm_map_find_space(
 		return KERN_INVALID_ARGUMENT;
 	}
 
+	if (flags & VM_FLAGS_GUARD_AFTER) {
+		/* account for the back guard page in the size */
+		size += PAGE_SIZE_64;
+	}
+
 	new_entry = vm_map_entry_create(map);
 
 	/*
@@ -1188,7 +1271,12 @@ vm_map_find_space(
 		 *	wrap around the address.
 		 */
 
+		if (flags & VM_FLAGS_GUARD_BEFORE) {
+			/* reserve space for the front guard page */
+			start += PAGE_SIZE_64;
+		}
 		end = ((start + mask) & ~mask);
+			
 		if (end < start) {
 			vm_map_entry_dispose(map, new_entry);
 			vm_map_unlock(map);
@@ -1237,6 +1325,10 @@ vm_map_find_space(
 	 *		the map should be locked.
 	 */
 
+	if (flags & VM_FLAGS_GUARD_BEFORE) {
+		/* go back for the front guard page */
+		start -= PAGE_SIZE_64;
+	}
 	*address = start;
 
 	new_entry->vme_start = start;
@@ -1261,6 +1353,9 @@ vm_map_find_space(
 
 	new_entry->in_transition = FALSE;
 	new_entry->needs_wakeup = FALSE;
+	new_entry->no_cache = FALSE;
+
+	new_entry->alias = 0;
 
 	VM_GET_FLAGS_ALIAS(flags, new_entry->alias);
 
@@ -1307,7 +1402,8 @@ vm_map_pmap_enter(
 	vm_object_offset_t	offset,
 	vm_prot_t		protection)
 {
-	unsigned int		cache_attr;
+	int			type_of_fault;
+	kern_return_t		kr;
 
 	if(map->pmap == 0)
 		return;
@@ -1316,7 +1412,6 @@ vm_map_pmap_enter(
 		register vm_page_t	m;
 
 		vm_object_lock(object);
-		vm_object_paging_begin(object);
 
 		m = vm_page_lookup(object, offset);
 		/*
@@ -1325,42 +1420,22 @@ vm_map_pmap_enter(
 		 * enter an encrypted page in the page table.
 		 */
 		if (m == VM_PAGE_NULL || m->busy || m->encrypted ||
-		    (m->unusual && ( m->error || m->restart || m->absent ||
-				    protection & m->page_lock))) {
-
-			vm_object_paging_end(object);
+		    m->fictitious ||
+		    (m->unusual && ( m->error || m->restart || m->absent))) {
 			vm_object_unlock(object);
 			return;
 		}
 
-		assert(!m->fictitious);	/* XXX is this possible ??? */
-
 		if (vm_map_pmap_enter_print) {
 			printf("vm_map_pmap_enter:");
-			printf("map: %x, addr: %llx, object: %x, offset: %llx\n",
-				map, (unsigned long long)addr, object, (unsigned long long)offset);
+			printf("map: %p, addr: %llx, object: %p, offset: %llx\n",
+			       map, (unsigned long long)addr, object, (unsigned long long)offset);
 		}
-		m->busy = TRUE;
-
-		if (m->no_isync == TRUE) {
-		        pmap_sync_page_data_phys(m->phys_page);
-			m->no_isync = FALSE;
-		}
-
-		cache_attr = ((unsigned int)object->wimg_bits) & VM_WIMG_MASK;
-		vm_object_unlock(object);
+		type_of_fault = DBG_CACHE_HIT_FAULT;
+		kr = vm_fault_enter(m, map->pmap, addr, protection,
+				    m->wire_count != 0, FALSE, FALSE,
+				    &type_of_fault);
 
-		PMAP_ENTER(map->pmap, addr, m, 
-				protection, cache_attr, FALSE);
-
-		vm_object_lock(object);
-
-		PAGE_WAKEUP_DONE(m);
-		vm_page_lock_queues();
-		if (!m->active && !m->inactive)
-		    vm_page_activate(m);
-		vm_page_unlock_queues();
-		vm_object_paging_end(object);
 		vm_object_unlock(object);
 
 		offset += PAGE_SIZE_64;
@@ -1377,12 +1452,16 @@ boolean_t vm_map_pmap_is_empty(
 	vm_map_offset_t	start,
 	vm_map_offset_t	end)
 {
+#ifdef MACHINE_PMAP_IS_EMPTY
+	return pmap_is_empty(map->pmap, start, end);
+#else 	/* MACHINE_PMAP_IS_EMPTY */
 	vm_map_offset_t	offset;
 	ppnum_t		phys_page;
 
 	if (map->pmap == NULL) {
 		return TRUE;
 	}
+
 	for (offset = start;
 	     offset < end;
 	     offset += PAGE_SIZE) {
@@ -1390,11 +1469,13 @@ boolean_t vm_map_pmap_is_empty(
 		if (phys_page) {
 			kprintf("vm_map_pmap_is_empty(%p,0x%llx,0x%llx): "
 				"page %d at 0x%llx\n",
-				map, start, end, phys_page, offset);
+				map, (long long)start, (long long)end,
+				phys_page, (long long)offset);
 			return FALSE;
 		}
 	}
 	return TRUE;
+#endif	/* MACHINE_PMAP_IS_EMPTY */
 }
 
 /*
@@ -1425,7 +1506,7 @@ vm_map_enter(
 	vm_inherit_t		inheritance)
 {
 	vm_map_entry_t		entry, new_entry;
-	vm_map_offset_t		start, tmp_start;
+	vm_map_offset_t		start, tmp_start, tmp_offset;
 	vm_map_offset_t		end, tmp_end;
 	kern_return_t		result = KERN_SUCCESS;
 	vm_map_t		zap_old_map = VM_MAP_NULL;
@@ -1436,15 +1517,56 @@ vm_map_enter(
 	boolean_t		anywhere = ((flags & VM_FLAGS_ANYWHERE) != 0);
 	boolean_t		purgable = ((flags & VM_FLAGS_PURGABLE) != 0);
 	boolean_t		overwrite = ((flags & VM_FLAGS_OVERWRITE) != 0);
+	boolean_t		no_cache = ((flags & VM_FLAGS_NO_CACHE) != 0);
+	boolean_t		is_submap = ((flags & VM_FLAGS_SUBMAP) != 0);
 	char			alias;
+	vm_map_offset_t		effective_min_offset, effective_max_offset;
 
-	if (size == 0) {
+	if (is_submap) {
+		if (purgable) {
+			/* submaps can not be purgeable */
+			return KERN_INVALID_ARGUMENT;
+		}
+		if (object == VM_OBJECT_NULL) {
+			/* submaps can not be created lazily */
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+	if (flags & VM_FLAGS_ALREADY) {
+		/*
+		 * VM_FLAGS_ALREADY says that it's OK if the same mapping
+		 * is already present.  For it to be meaningul, the requested
+		 * mapping has to be at a fixed address (!VM_FLAGS_ANYWHERE) and
+		 * we shouldn't try and remove what was mapped there first
+		 * (!VM_FLAGS_OVERWRITE).
+		 */
+		if ((flags & VM_FLAGS_ANYWHERE) ||
+		    (flags & VM_FLAGS_OVERWRITE)) {
+			return KERN_INVALID_ARGUMENT;
+		}
+	}
+
+	effective_min_offset = map->min_offset;
+	if (flags & VM_FLAGS_BEYOND_MAX) {
+		/*
+		 * Allow an insertion beyond the map's official top boundary.
+		 */
+		if (vm_map_is_64bit(map))
+			effective_max_offset = 0xFFFFFFFFFFFFF000ULL;
+		else
+			effective_max_offset = 0x00000000FFFFF000ULL;
+	} else {
+		effective_max_offset = map->max_offset;
+	}
+
+	if (size == 0 ||
+	    (offset & PAGE_MASK_64) != 0) {
 		*address = 0;
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	VM_GET_FLAGS_ALIAS(flags, alias);
-	
+
 #define	RETURN(value)	{ result = value; goto BailOut; }
 
 	assert(page_aligned(*address));
@@ -1458,7 +1580,7 @@ vm_map_enter(
 	    (offset != 0 ||
 	     (object != VM_OBJECT_NULL &&
 	      (object->size != size ||
-	       object->purgable == VM_OBJECT_NONPURGABLE))
+	       object->purgable == VM_PURGABLE_DENY))
 	     || size > VM_MAX_ADDRESS)) /* LP64todo: remove when dp capable */
 		return KERN_INVALID_ARGUMENT;
 
@@ -1478,7 +1600,7 @@ vm_map_enter(
 					    TRUE);
 	}
 
- StartAgain: ;
+StartAgain: ;
 
 	start = *address;
 
@@ -1490,9 +1612,9 @@ vm_map_enter(
 		 *	Calculate the first possible address.
 		 */
 
-		if (start < map->min_offset)
-			start = map->min_offset;
-		if (start > map->max_offset)
+		if (start < effective_min_offset)
+			start = effective_min_offset;
+		if (start > effective_max_offset)
 			RETURN(KERN_NO_SPACE);
 
 		/*
@@ -1502,7 +1624,7 @@ vm_map_enter(
 		 */
 
 		assert(first_free_is_valid(map));
-		if (start == map->min_offset) {
+		if (start == effective_min_offset) {
 			if ((entry = map->first_free) != vm_map_to_entry(map))
 				start = entry->vme_end;
 		} else {
@@ -1521,7 +1643,7 @@ vm_map_enter(
 		while (TRUE) {
 			register vm_map_entry_t	next;
 
-		    	/*
+			/*
 			 *	Find the end of the proposed new region.
 			 *	Be sure we didn't go beyond the end, or
 			 *	wrap around the address.
@@ -1533,10 +1655,10 @@ vm_map_enter(
 			start = end;
 			end += size;
 
-			if ((end > map->max_offset) || (end < start)) {
+			if ((end > effective_max_offset) || (end < start)) {
 				if (map->wait_for_space) {
-					if (size <= (map->max_offset -
-						     map->min_offset)) {
+					if (size <= (effective_max_offset -
+						     effective_min_offset)) {
 						assert_wait((event_t)map,
 							    THREAD_ABORTSAFE);
 						vm_map_unlock(map);
@@ -1573,8 +1695,6 @@ vm_map_enter(
 		}
 		*address = start;
 	} else {
-		vm_map_entry_t		temp_entry;
-
 		/*
 		 *	Verify that:
 		 *		the address doesn't itself violate
@@ -1592,8 +1712,8 @@ vm_map_enter(
 
 		end = start + size;
 
-		if ((start < map->min_offset) ||
-		    (end > map->max_offset) ||
+		if ((start < effective_min_offset) ||
+		    (end > effective_max_offset) ||
 		    (start >= end)) {
 			RETURN(KERN_INVALID_ADDRESS);
 		}
@@ -1613,10 +1733,45 @@ vm_map_enter(
 		 *	...	the starting address isn't allocated
 		 */
 
-		if (vm_map_lookup_entry(map, start, &temp_entry))
-			RETURN(KERN_NO_SPACE);
-
-		entry = temp_entry;
+		if (vm_map_lookup_entry(map, start, &entry)) {
+			if (! (flags & VM_FLAGS_ALREADY)) {
+				RETURN(KERN_NO_SPACE);
+			}
+			/*
+			 * Check if what's already there is what we want.
+			 */
+			tmp_start = start;
+			tmp_offset = offset;
+			if (entry->vme_start < start) {
+				tmp_start -= start - entry->vme_start;
+				tmp_offset -= start - entry->vme_start;
+				
+			}
+			for (; entry->vme_start < end;
+			     entry = entry->vme_next) {
+				if (entry == vm_map_to_entry(map) ||
+				    entry->vme_start != tmp_start ||
+				    entry->is_sub_map != is_submap ||
+				    entry->object.vm_object != object ||
+				    entry->offset != tmp_offset ||
+				    entry->needs_copy != needs_copy ||
+				    entry->protection != cur_protection ||
+				    entry->max_protection != max_protection ||
+				    entry->inheritance != inheritance ||
+				    entry->alias != alias) {
+					/* not the same mapping ! */
+					RETURN(KERN_NO_SPACE);
+				}
+				tmp_offset += entry->vme_end - entry->vme_start;
+				tmp_start += entry->vme_end - entry->vme_start;
+				if (entry->vme_end >= end) {
+					/* reached the end of our mapping */
+					break;
+				}
+			}
+			/* it all matches:  let's use what's already there ! */
+			RETURN(KERN_MEMORY_PRESENT);
+		}
 
 		/*
 		 *	...	the next region doesn't overlap the
@@ -1651,28 +1806,31 @@ vm_map_enter(
 		if (object == VM_OBJECT_NULL) {
 			object = vm_object_allocate(size);
 			object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
-			object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
+			object->purgable = VM_PURGABLE_NONVOLATILE;
 			offset = (vm_object_offset_t)0;
 		}
-	} else if ((object == VM_OBJECT_NULL) &&
-	    (entry != vm_map_to_entry(map)) &&
-	    (entry->vme_end == start) &&
-	    (!entry->is_shared) &&
-	    (!entry->is_sub_map) &&
-	    (entry->alias == alias) &&
-	    (entry->inheritance == inheritance) &&
-	    (entry->protection == cur_protection) &&
-	    (entry->max_protection == max_protection) &&
-	    (entry->behavior == VM_BEHAVIOR_DEFAULT) &&
-	    (entry->in_transition == 0) &&
-	    ((alias == VM_MEMORY_REALLOC) || ((entry->vme_end - entry->vme_start) + size < NO_COALESCE_LIMIT)) &&
-	    (entry->wired_count == 0)) { /* implies user_wired_count == 0 */
+	} else if ((is_submap == FALSE) &&
+		   (object == VM_OBJECT_NULL) &&
+		   (entry != vm_map_to_entry(map)) &&
+		   (entry->vme_end == start) &&
+		   (!entry->is_shared) &&
+		   (!entry->is_sub_map) &&
+		   (entry->alias == alias) &&
+		   (entry->inheritance == inheritance) &&
+		   (entry->protection == cur_protection) &&
+		   (entry->max_protection == max_protection) &&
+		   (entry->behavior == VM_BEHAVIOR_DEFAULT) &&
+		   (entry->in_transition == 0) &&
+		   (entry->no_cache == no_cache) &&
+		   ((alias == VM_MEMORY_REALLOC) ||
+		    ((entry->vme_end - entry->vme_start) + size < NO_COALESCE_LIMIT)) &&
+		   (entry->wired_count == 0)) { /* implies user_wired_count == 0 */
 		if (vm_object_coalesce(entry->object.vm_object,
-				VM_OBJECT_NULL,
-				entry->offset,
-				(vm_object_offset_t) 0,
-				(vm_map_size_t)(entry->vme_end - entry->vme_start),
-				(vm_map_size_t)(end - entry->vme_end))) {
+				       VM_OBJECT_NULL,
+				       entry->offset,
+				       (vm_object_offset_t) 0,
+				       (vm_map_size_t)(entry->vme_end - entry->vme_start),
+				       (vm_map_size_t)(end - entry->vme_end))) {
 
 			/*
 			 *	Coalesced the two objects - can extend
@@ -1710,10 +1868,52 @@ vm_map_enter(
 		tmp_end = end;
 	do {
 		new_entry = vm_map_entry_insert(map, entry, tmp_start, tmp_end,
-					object,	offset, needs_copy, FALSE, FALSE,
-					cur_protection, max_protection,
-					VM_BEHAVIOR_DEFAULT, inheritance, 0);
+						object,	offset, needs_copy,
+						FALSE, FALSE,
+						cur_protection, max_protection,
+						VM_BEHAVIOR_DEFAULT,
+						inheritance, 0, no_cache);
 		new_entry->alias = alias;
+		if (is_submap) {
+			vm_map_t	submap;
+			boolean_t	submap_is_64bit;
+			boolean_t	use_pmap;
+
+			new_entry->is_sub_map = TRUE;
+			submap = (vm_map_t) object;
+			submap_is_64bit = vm_map_is_64bit(submap);
+			use_pmap = (alias == VM_MEMORY_SHARED_PMAP);
+#ifndef NO_NESTED_PMAP 
+			if (use_pmap && submap->pmap == NULL) {
+				/* we need a sub pmap to nest... */
+				submap->pmap = pmap_create(0, submap_is_64bit);
+				if (submap->pmap == NULL) {
+					/* let's proceed without nesting... */
+				}
+			}
+			if (use_pmap && submap->pmap != NULL) {
+				kern_return_t kr;
+
+				kr = pmap_nest(map->pmap,
+					       submap->pmap,
+					       tmp_start,
+					       tmp_start,
+					       tmp_end - tmp_start);
+				if (kr != KERN_SUCCESS) {
+					printf("vm_map_enter: "
+					       "pmap_nest(0x%llx,0x%llx) "
+					       "error 0x%x\n",
+					       (long long)tmp_start,
+					       (long long)tmp_end,
+					       kr);
+				} else {
+					/* we're now nested ! */
+					new_entry->use_pmap = TRUE;
+					pmap_empty = FALSE;
+				}
+			}
+#endif /* NO_NESTED_PMAP */
+		}
 		entry = new_entry;
 	} while (tmp_end != end && 
 		 (tmp_start = tmp_end) &&
@@ -1742,15 +1942,14 @@ vm_map_enter(
 	    (size < (128*1024))) {
 		pmap_empty = FALSE; /* pmap won't be empty */
 
-#ifdef STACK_ONLY_NX
-		if (alias != VM_MEMORY_STACK && cur_protection)
+		if (override_nx(map, alias) && cur_protection)
 		        cur_protection |= VM_PROT_EXECUTE;
-#endif
+
 		vm_map_pmap_enter(map, start, end, 
 				  object, offset, cur_protection);
 	}
 
- BailOut: ;
+BailOut: ;
 	if (result == KERN_SUCCESS &&
 	    pmap_empty &&
 	    !(flags & VM_FLAGS_NO_PMAP_CHECK)) {
@@ -1811,9 +2010,15 @@ vm_map_enter(
 				for (entry2 = vm_map_first_entry(zap_old_map);
 				     entry2 != vm_map_to_entry(zap_old_map);
 				     entry2 = vm_map_first_entry(zap_old_map)) {
+					vm_map_size_t entry_size;
+
+					entry_size = (entry2->vme_end -
+						      entry2->vme_start);
 					vm_map_entry_unlink(zap_old_map,
 							    entry2);
+					zap_old_map->size -= entry_size;
 					vm_map_entry_link(map, entry1, entry2);
+					map->size += entry_size;
 					entry1 = entry2;
 				}
 				if (map->wiring_required) {
@@ -1836,11 +2041,11 @@ vm_map_enter(
 	 * they may still contain.
 	 */
 	if (zap_old_map != VM_MAP_NULL) {
-		vm_map_destroy(zap_old_map);
+		vm_map_destroy(zap_old_map, VM_MAP_REMOVE_NO_PMAP_CLEANUP);
 		zap_old_map = VM_MAP_NULL;
 	}
 	if (zap_new_map != VM_MAP_NULL) {
-		vm_map_destroy(zap_new_map);
+		vm_map_destroy(zap_new_map, VM_MAP_REMOVE_NO_PMAP_CLEANUP);
 		zap_new_map = VM_MAP_NULL;
 	}
 
@@ -1849,74 +2054,366 @@ vm_map_enter(
 #undef	RETURN
 }
 
-
-#if	VM_CPM
-
-#ifdef MACH_ASSERT
-extern pmap_paddr_t	avail_start, avail_end;
-#endif
-
-/*
- *	Allocate memory in the specified map, with the caveat that
- *	the memory is physically contiguous.  This call may fail
- *	if the system can't find sufficient contiguous memory.
- *	This call may cause or lead to heart-stopping amounts of
- *	paging activity.
- *
- *	Memory obtained from this call should be freed in the
- *	normal way, viz., via vm_deallocate.
- */
 kern_return_t
-vm_map_enter_cpm(
-	vm_map_t		map,
-	vm_map_offset_t	*addr,
-	vm_map_size_t		size,
-	int			flags)
+vm_map_enter_mem_object(
+	vm_map_t		target_map,
+	vm_map_offset_t		*address,
+	vm_map_size_t		initial_size,
+	vm_map_offset_t		mask,
+	int			flags,
+	ipc_port_t		port,
+	vm_object_offset_t	offset,
+	boolean_t		copy,
+	vm_prot_t		cur_protection,
+	vm_prot_t		max_protection,
+	vm_inherit_t		inheritance)
 {
-	vm_object_t		cpm_obj;
-	pmap_t			pmap;
-	vm_page_t		m, pages;
-	kern_return_t		kr;
-	vm_map_offset_t		va, start, end, offset;
-#if	MACH_ASSERT
-	vm_map_offset_t		prev_addr;
-#endif	/* MACH_ASSERT */
-
-	boolean_t		anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
-
-	if (!vm_allocate_cpm_enabled)
-		return KERN_FAILURE;
-
-	if (size == 0) {
-		*addr = 0;
-		return KERN_SUCCESS;
-	}
-
-	if (anywhere)
-		*addr = vm_map_min(map);
-	else
-		*addr = vm_map_trunc_page(*addr);
-	size = vm_map_round_page(size);
+	vm_map_address_t	map_addr;
+	vm_map_size_t		map_size;
+	vm_object_t		object;
+	vm_object_size_t	size;
+	kern_return_t		result;
 
 	/*
-	 * LP64todo - cpm_allocate should probably allow
-	 * allocations of >4GB, but not with the current
-	 * algorithm, so just cast down the size for now.
+	 * Check arguments for validity
 	 */
-	if (size > VM_MAX_ADDRESS)
-		return KERN_RESOURCE_SHORTAGE;
-	if ((kr = cpm_allocate(CAST_DOWN(vm_size_t, size),
-			       &pages, TRUE)) != KERN_SUCCESS)
-		return kr;
-
-	cpm_obj = vm_object_allocate((vm_object_size_t)size);
-	assert(cpm_obj != VM_OBJECT_NULL);
-	assert(cpm_obj->internal);
-	assert(cpm_obj->size == (vm_object_size_t)size);
-	assert(cpm_obj->can_persist == FALSE);
-	assert(cpm_obj->pager_created == FALSE);
-	assert(cpm_obj->pageout == FALSE);
-	assert(cpm_obj->shadow == VM_OBJECT_NULL);
+	if ((target_map == VM_MAP_NULL) ||
+	    (cur_protection & ~VM_PROT_ALL) ||
+	    (max_protection & ~VM_PROT_ALL) ||
+	    (inheritance > VM_INHERIT_LAST_VALID) ||
+	    initial_size == 0)
+		return KERN_INVALID_ARGUMENT;
+
+	map_addr = vm_map_trunc_page(*address);
+	map_size = vm_map_round_page(initial_size);
+	size = vm_object_round_page(initial_size);	
+	
+	/*
+	 * Find the vm object (if any) corresponding to this port.
+	 */
+	if (!IP_VALID(port)) {
+		object = VM_OBJECT_NULL;
+		offset = 0;
+		copy = FALSE;
+	} else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
+		vm_named_entry_t	named_entry;
+
+		named_entry = (vm_named_entry_t) port->ip_kobject;
+		/* a few checks to make sure user is obeying rules */
+		if (size == 0) {
+			if (offset >= named_entry->size)
+				return KERN_INVALID_RIGHT;
+			size = named_entry->size - offset;
+		}
+		if ((named_entry->protection & max_protection) !=
+		    max_protection)
+			return KERN_INVALID_RIGHT;
+		if ((named_entry->protection & cur_protection) !=
+		    cur_protection)
+			return KERN_INVALID_RIGHT;
+		if (named_entry->size < (offset + size))
+			return KERN_INVALID_ARGUMENT;
+
+		/* the callers parameter offset is defined to be the */
+		/* offset from beginning of named entry offset in object */
+		offset = offset + named_entry->offset;
+		
+		named_entry_lock(named_entry);
+		if (named_entry->is_sub_map) {
+			vm_map_t		submap;
+
+			submap = named_entry->backing.map;
+			vm_map_lock(submap);
+			vm_map_reference(submap);
+			vm_map_unlock(submap);
+			named_entry_unlock(named_entry);
+
+			result = vm_map_enter(target_map,
+					      &map_addr,
+					      map_size,
+					      mask,
+					      flags | VM_FLAGS_SUBMAP,
+					      (vm_object_t) submap,
+					      offset,
+					      copy,
+					      cur_protection,
+					      max_protection,
+					      inheritance);
+			if (result != KERN_SUCCESS) {
+				vm_map_deallocate(submap);
+			} else {
+				/*
+				 * No need to lock "submap" just to check its
+				 * "mapped" flag: that flag is never reset
+				 * once it's been set and if we race, we'll
+				 * just end up setting it twice, which is OK.
+				 */
+				if (submap->mapped == FALSE) {
+					/*
+					 * This submap has never been mapped.
+					 * Set its "mapped" flag now that it
+					 * has been mapped.
+					 * This happens only for the first ever
+					 * mapping of a "submap".
+					 */
+					vm_map_lock(submap);
+					submap->mapped = TRUE;
+					vm_map_unlock(submap);
+				}
+				*address = map_addr;
+			}
+			return result;
+
+		} else if (named_entry->is_pager) {
+			unsigned int	access;
+			vm_prot_t	protections;
+			unsigned int	wimg_mode;
+			boolean_t	cache_attr;
+
+			protections = named_entry->protection & VM_PROT_ALL;
+			access = GET_MAP_MEM(named_entry->protection);
+
+			object = vm_object_enter(named_entry->backing.pager, 
+						 named_entry->size, 
+						 named_entry->internal, 
+						 FALSE,
+						 FALSE);
+			if (object == VM_OBJECT_NULL) {
+				named_entry_unlock(named_entry);
+				return KERN_INVALID_OBJECT;
+			}
+
+			/* JMM - drop reference on pager here */
+
+			/* create an extra ref for the named entry */
+			vm_object_lock(object);
+			vm_object_reference_locked(object);
+			named_entry->backing.object = object;
+			named_entry->is_pager = FALSE;
+			named_entry_unlock(named_entry);
+
+			wimg_mode = object->wimg_bits;
+			if (access == MAP_MEM_IO) {
+				wimg_mode = VM_WIMG_IO;
+			} else if (access == MAP_MEM_COPYBACK) {
+				wimg_mode = VM_WIMG_USE_DEFAULT;
+			} else if (access == MAP_MEM_WTHRU) {
+				wimg_mode = VM_WIMG_WTHRU;
+			} else if (access == MAP_MEM_WCOMB) {
+				wimg_mode = VM_WIMG_WCOMB;
+			}
+			if (wimg_mode == VM_WIMG_IO ||
+			    wimg_mode == VM_WIMG_WCOMB)
+				cache_attr = TRUE;
+			else 
+				cache_attr = FALSE;
+
+			/* wait for object (if any) to be ready */
+			if (!named_entry->internal) {
+				while (!object->pager_ready) {
+					vm_object_wait(
+						object,
+						VM_OBJECT_EVENT_PAGER_READY,
+						THREAD_UNINT);
+					vm_object_lock(object);
+				}
+			}
+
+			if (object->wimg_bits != wimg_mode) {
+				vm_page_t p;
+
+				vm_object_paging_wait(object, THREAD_UNINT);
+
+				object->wimg_bits = wimg_mode;
+				queue_iterate(&object->memq, p, vm_page_t, listq) {
+					if (!p->fictitious) {
+					        if (p->pmapped)
+						        pmap_disconnect(p->phys_page);
+						if (cache_attr)
+						        pmap_sync_page_attributes_phys(p->phys_page);
+					}
+				}
+			}
+			object->true_share = TRUE;
+			if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
+				object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+			vm_object_unlock(object);
+		} else {
+			/* This is the case where we are going to map */
+			/* an already mapped object.  If the object is */
+			/* not ready it is internal.  An external     */
+			/* object cannot be mapped until it is ready  */
+			/* we can therefore avoid the ready check     */
+			/* in this case.  */
+			object = named_entry->backing.object;
+			assert(object != VM_OBJECT_NULL);
+			named_entry_unlock(named_entry);
+			vm_object_reference(object);
+		}
+	} else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
+		/*
+		 * JMM - This is temporary until we unify named entries
+		 * and raw memory objects.
+		 *
+		 * Detected fake ip_kotype for a memory object.  In
+		 * this case, the port isn't really a port at all, but
+		 * instead is just a raw memory object.
+		 */
+		 
+		object = vm_object_enter((memory_object_t)port,
+					 size, FALSE, FALSE, FALSE);
+		if (object == VM_OBJECT_NULL)
+			return KERN_INVALID_OBJECT;
+
+		/* wait for object (if any) to be ready */
+		if (object != VM_OBJECT_NULL) {
+			if (object == kernel_object) {
+				printf("Warning: Attempt to map kernel object"
+					" by a non-private kernel entity\n");
+				return KERN_INVALID_OBJECT;
+			}
+			vm_object_lock(object);
+			while (!object->pager_ready) {
+				vm_object_wait(object,
+					       VM_OBJECT_EVENT_PAGER_READY,
+					       THREAD_UNINT);
+				vm_object_lock(object);
+			}
+			vm_object_unlock(object);
+		}
+	} else {
+		return KERN_INVALID_OBJECT;
+	}
+
+	/*
+	 *	Perform the copy if requested
+	 */
+
+	if (copy) {
+		vm_object_t		new_object;
+		vm_object_offset_t	new_offset;
+
+		result = vm_object_copy_strategically(object, offset, size,
+						      &new_object, &new_offset,
+						      &copy);
+
+
+		if (result == KERN_MEMORY_RESTART_COPY) {
+			boolean_t success;
+			boolean_t src_needs_copy;
+
+			/*
+			 * XXX
+			 * We currently ignore src_needs_copy.
+			 * This really is the issue of how to make
+			 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
+			 * non-kernel users to use. Solution forthcoming.
+			 * In the meantime, since we don't allow non-kernel
+			 * memory managers to specify symmetric copy,
+			 * we won't run into problems here.
+			 */
+			new_object = object;
+			new_offset = offset;
+			success = vm_object_copy_quickly(&new_object,
+							 new_offset, size,
+							 &src_needs_copy,
+							 &copy);
+			assert(success);
+			result = KERN_SUCCESS;
+		}
+		/*
+		 *	Throw away the reference to the
+		 *	original object, as it won't be mapped.
+		 */
+
+		vm_object_deallocate(object);
+
+		if (result != KERN_SUCCESS)
+			return result;
+
+		object = new_object;
+		offset = new_offset;
+	}
+
+	result = vm_map_enter(target_map,
+			      &map_addr, map_size,
+			      (vm_map_offset_t)mask,
+			      flags,
+			      object, offset,
+			      copy,
+			      cur_protection, max_protection, inheritance);
+	if (result != KERN_SUCCESS)
+		vm_object_deallocate(object);
+	*address = map_addr;
+	return result;
+}
+
+#if	VM_CPM
+
+#ifdef MACH_ASSERT
+extern pmap_paddr_t	avail_start, avail_end;
+#endif
+
+/*
+ *	Allocate memory in the specified map, with the caveat that
+ *	the memory is physically contiguous.  This call may fail
+ *	if the system can't find sufficient contiguous memory.
+ *	This call may cause or lead to heart-stopping amounts of
+ *	paging activity.
+ *
+ *	Memory obtained from this call should be freed in the
+ *	normal way, viz., via vm_deallocate.
+ */
+kern_return_t
+vm_map_enter_cpm(
+	vm_map_t		map,
+	vm_map_offset_t	*addr,
+	vm_map_size_t		size,
+	int			flags)
+{
+	vm_object_t		cpm_obj;
+	pmap_t			pmap;
+	vm_page_t		m, pages;
+	kern_return_t		kr;
+	vm_map_offset_t		va, start, end, offset;
+#if	MACH_ASSERT
+	vm_map_offset_t		prev_addr;
+#endif	/* MACH_ASSERT */
+
+	boolean_t		anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
+
+	if (!vm_allocate_cpm_enabled)
+		return KERN_FAILURE;
+
+	if (size == 0) {
+		*addr = 0;
+		return KERN_SUCCESS;
+	}
+	if (anywhere)
+		*addr = vm_map_min(map);
+	else
+		*addr = vm_map_trunc_page(*addr);
+	size = vm_map_round_page(size);
+
+	/*
+	 * LP64todo - cpm_allocate should probably allow
+	 * allocations of >4GB, but not with the current
+	 * algorithm, so just cast down the size for now.
+	 */
+	if (size > VM_MAX_ADDRESS)
+		return KERN_RESOURCE_SHORTAGE;
+	if ((kr = cpm_allocate(CAST_DOWN(vm_size_t, size),
+			       &pages, 0, TRUE)) != KERN_SUCCESS)
+		return kr;
+
+	cpm_obj = vm_object_allocate((vm_object_size_t)size);
+	assert(cpm_obj != VM_OBJECT_NULL);
+	assert(cpm_obj->internal);
+	assert(cpm_obj->size == (vm_object_size_t)size);
+	assert(cpm_obj->can_persist == FALSE);
+	assert(cpm_obj->pager_created == FALSE);
+	assert(cpm_obj->pageout == FALSE);
+	assert(cpm_obj->shadow == VM_OBJECT_NULL);
 
 	/*
 	 *	Insert pages into object.
@@ -1932,6 +2429,7 @@ vm_map_enter_cpm(
 		assert(!m->wanted);
 		assert(!m->pageout);
 		assert(!m->tabled);
+		assert(m->wire_count);
 		/*
 		 * ENCRYPTED SWAP:
 		 * "m" is not supposed to be pageable, so it
@@ -1966,17 +2464,17 @@ vm_map_enter_cpm(
 	 */
 
 	kr = vm_map_enter(
-			  map,
-			  addr,
-			  size,
-			  (vm_map_offset_t)0,
-			  flags,
-			  cpm_obj,
-			  (vm_object_offset_t)0,
-			  FALSE,
-			  VM_PROT_ALL,
-			  VM_PROT_ALL,
-			  VM_INHERIT_DEFAULT);
+		map,
+		addr,
+		size,
+		(vm_map_offset_t)0,
+		flags,
+		cpm_obj,
+		(vm_object_offset_t)0,
+		FALSE,
+		VM_PROT_ALL,
+		VM_PROT_ALL,
+		VM_INHERIT_DEFAULT);
 
 	if (kr != KERN_SUCCESS) {
 		/*
@@ -2010,13 +2508,21 @@ vm_map_enter_cpm(
 	 */
 	for (offset = 0, va = start; offset < size;
 	     va += PAGE_SIZE, offset += PAGE_SIZE) {
+	        int type_of_fault;
+
 		vm_object_lock(cpm_obj);
 		m = vm_page_lookup(cpm_obj, (vm_object_offset_t)offset);
-		vm_object_unlock(cpm_obj);
 		assert(m != VM_PAGE_NULL);
-		PMAP_ENTER(pmap, va, m, VM_PROT_ALL, 	
-			((unsigned int)(m->object->wimg_bits)) & VM_WIMG_MASK,
-			TRUE);
+
+		vm_page_zero_fill(m);
+
+		type_of_fault = DBG_ZERO_FILL_FAULT;
+
+		vm_fault_enter(m, pmap, va, VM_PROT_ALL,
+			       m->wire_count != 0, FALSE, FALSE,
+			       &type_of_fault);
+
+		vm_object_unlock(cpm_obj);
 	}
 
 #if	MACH_ASSERT
@@ -2079,6 +2585,53 @@ vm_map_enter_cpm(
 }
 #endif /* VM_CPM */
 
+/*
+ * Clip and unnest a portion of a nested submap mapping.
+ */
+static void
+vm_map_clip_unnest(
+	vm_map_t	map,
+	vm_map_entry_t	entry,
+	vm_map_offset_t	start_unnest,
+	vm_map_offset_t	end_unnest)
+{
+	assert(entry->is_sub_map);
+	assert(entry->object.sub_map != NULL);
+
+	if (entry->vme_start > start_unnest ||
+	    entry->vme_end < end_unnest) {
+		panic("vm_map_clip_unnest(0x%llx,0x%llx): "
+		      "bad nested entry: start=0x%llx end=0x%llx\n",
+		      (long long)start_unnest, (long long)end_unnest,
+		      (long long)entry->vme_start, (long long)entry->vme_end);
+	}
+	if (start_unnest > entry->vme_start) {
+		_vm_map_clip_start(&map->hdr,
+				   entry,
+				   start_unnest);
+		UPDATE_FIRST_FREE(map, map->first_free);
+	}
+	if (entry->vme_end > end_unnest) {
+		_vm_map_clip_end(&map->hdr,
+				 entry,
+				 end_unnest);
+		UPDATE_FIRST_FREE(map, map->first_free);
+	}
+
+	pmap_unnest(map->pmap,
+		    entry->vme_start,
+		    entry->vme_end - entry->vme_start);
+	if ((map->mapped) && (map->ref_count)) {
+		/* clean up parent map/maps */
+		vm_map_submap_pmap_clean(
+			map, entry->vme_start,
+			entry->vme_end,
+			entry->object.sub_map,
+			entry->offset);
+	}
+	entry->use_pmap = FALSE;
+}
+
 /*
  *	vm_map_clip_start:	[ internal use only ]
  *
@@ -2086,48 +2639,40 @@ vm_map_enter_cpm(
  *	the specified address; if necessary,
  *	it splits the entry into two.
  */
+static void
+vm_map_clip_start(
+	vm_map_t	map,
+	vm_map_entry_t	entry,
+	vm_map_offset_t	startaddr)
+{
 #ifndef NO_NESTED_PMAP
-#define vm_map_clip_start(map, entry, startaddr) 			\
-MACRO_BEGIN 								\
-	vm_map_t VMCS_map;						\
-	vm_map_entry_t VMCS_entry;					\
-	vm_map_offset_t VMCS_startaddr;					\
-	VMCS_map = (map);						\
-	VMCS_entry = (entry);						\
-	VMCS_startaddr = (startaddr);					\
-	if (VMCS_startaddr > VMCS_entry->vme_start) { 			\
-		if(entry->use_pmap) {					\
-	   		vm_map_offset_t	pmap_base_addr;			\
-									\
-	   		pmap_base_addr = 0xF0000000 & entry->vme_start;	\
-   	   		pmap_unnest(map->pmap, (addr64_t)pmap_base_addr); \
-			entry->use_pmap = FALSE;			\
-		} else if(entry->object.vm_object			\
-			&& !entry->is_sub_map				\
-			&& entry->object.vm_object->phys_contiguous) {	\
-			pmap_remove(map->pmap,				\
-				(addr64_t)(entry->vme_start),		\
-				(addr64_t)(entry->vme_end));		\
-		}							\
-		_vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
-	}								\
-	UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);		\
-MACRO_END
-#else /* NO_NESTED_PMAP */
-#define vm_map_clip_start(map, entry, startaddr) 			\
-MACRO_BEGIN 								\
-	vm_map_t VMCS_map;						\
-	vm_map_entry_t VMCS_entry;					\
-	vm_map_offset_t VMCS_startaddr;					\
-	VMCS_map = (map);						\
-	VMCS_entry = (entry);						\
-	VMCS_startaddr = (startaddr);					\
-	if (VMCS_startaddr > VMCS_entry->vme_start) { 			\
-		_vm_map_clip_start(&VMCS_map->hdr,VMCS_entry,VMCS_startaddr);\
-	}								\
-	UPDATE_FIRST_FREE(VMCS_map, VMCS_map->first_free);		\
-MACRO_END
-#endif	/* NO_NESTED_PMAP */
+	if (entry->use_pmap &&
+	    startaddr >= entry->vme_start) {
+		vm_map_offset_t	start_unnest, end_unnest;
+
+		/*
+		 * Make sure "startaddr" is no longer in a nested range
+		 * before we clip.  Unnest only the minimum range the platform
+		 * can handle.
+		 */
+		start_unnest = startaddr & ~(pmap_nesting_size_min - 1);
+		end_unnest = start_unnest + pmap_nesting_size_min;
+		vm_map_clip_unnest(map, entry, start_unnest, end_unnest);
+	}
+#endif /* NO_NESTED_PMAP */
+	if (startaddr > entry->vme_start) {
+		if (entry->object.vm_object &&
+		    !entry->is_sub_map &&
+		    entry->object.vm_object->phys_contiguous) {
+			pmap_remove(map->pmap,
+				    (addr64_t)(entry->vme_start),
+				    (addr64_t)(entry->vme_end));
+		}
+		_vm_map_clip_start(&map->hdr, entry, startaddr);
+		UPDATE_FIRST_FREE(map, map->first_free);
+	}
+}
+
 
 #define vm_map_copy_clip_start(copy, entry, startaddr) \
 	MACRO_BEGIN \
@@ -2165,7 +2710,7 @@ _vm_map_clip_start(
 	_vm_map_entry_link(map_header, entry->vme_prev, new_entry);
 
 	if (entry->is_sub_map)
-	 	vm_map_reference(new_entry->object.sub_map);
+		vm_map_reference(new_entry->object.sub_map);
 	else
 		vm_object_reference(new_entry->object.vm_object);
 }
@@ -2178,48 +2723,47 @@ _vm_map_clip_start(
  *	the specified address; if necessary,
  *	it splits the entry into two.
  */
-#ifndef NO_NESTED_PMAP 
-#define vm_map_clip_end(map, entry, endaddr) 				\
-MACRO_BEGIN 								\
-	vm_map_t VMCE_map;						\
-	vm_map_entry_t VMCE_entry;					\
-	vm_map_offset_t VMCE_endaddr;					\
-	VMCE_map = (map);						\
-	VMCE_entry = (entry);						\
-	VMCE_endaddr = (endaddr);					\
-	if (VMCE_endaddr < VMCE_entry->vme_end) { 			\
-		if(entry->use_pmap) {					\
-	   		vm_map_offset_t	pmap_base_addr;			\
-									\
-	   		pmap_base_addr = 0xF0000000 & entry->vme_start;	\
- 	   		pmap_unnest(map->pmap, (addr64_t)pmap_base_addr); \
-			entry->use_pmap = FALSE;			\
-		} else if(entry->object.vm_object			\
-			&& !entry->is_sub_map				\
-			&& entry->object.vm_object->phys_contiguous) {	\
-			pmap_remove(map->pmap,				\
-				(addr64_t)(entry->vme_start),		\
-				(addr64_t)(entry->vme_end));		\
-		}							\
-		_vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
-	}								\
-	UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);		\
-MACRO_END
-#else	/* NO_NESTED_PMAP */
-#define vm_map_clip_end(map, entry, endaddr) 				\
-MACRO_BEGIN 								\
-	vm_map_t VMCE_map;						\
-	vm_map_entry_t VMCE_entry;					\
-	vm_map_offset_t VMCE_endaddr;					\
-	VMCE_map = (map);						\
-	VMCE_entry = (entry);						\
-	VMCE_endaddr = (endaddr);					\
-	if (VMCE_endaddr < VMCE_entry->vme_end) { 			\
-		_vm_map_clip_end(&VMCE_map->hdr,VMCE_entry,VMCE_endaddr); \
-	}								\
-	UPDATE_FIRST_FREE(VMCE_map, VMCE_map->first_free);		\
-MACRO_END
-#endif	/* NO_NESTED_PMAP */
+static void
+vm_map_clip_end(
+	vm_map_t	map,
+	vm_map_entry_t	entry,
+	vm_map_offset_t	endaddr)
+{
+	if (endaddr > entry->vme_end) {
+		/*
+		 * Within the scope of this clipping, limit "endaddr" to
+		 * the end of this map entry...
+		 */
+		endaddr = entry->vme_end;
+	}
+#ifndef NO_NESTED_PMAP
+	if (entry->use_pmap) {
+		vm_map_offset_t	start_unnest, end_unnest;
+
+		/*
+		 * Make sure the range between the start of this entry and
+		 * the new "endaddr" is no longer nested before we clip.
+		 * Unnest only the minimum range the platform can handle.
+		 */
+		start_unnest = entry->vme_start;
+		end_unnest =
+			(endaddr + pmap_nesting_size_min - 1) &
+			~(pmap_nesting_size_min - 1);
+		vm_map_clip_unnest(map, entry, start_unnest, end_unnest);
+	}
+#endif /* NO_NESTED_PMAP */
+	if (endaddr < entry->vme_end) {
+		if (entry->object.vm_object &&
+		    !entry->is_sub_map &&
+		    entry->object.vm_object->phys_contiguous) {
+			pmap_remove(map->pmap,
+				    (addr64_t)(entry->vme_start),
+				    (addr64_t)(entry->vme_end));
+		}
+		_vm_map_clip_end(&map->hdr, entry, endaddr);
+		UPDATE_FIRST_FREE(map, map->first_free);
+	}
+}
 
 
 #define vm_map_copy_clip_end(copy, entry, endaddr) \
@@ -2236,7 +2780,7 @@ static void
 _vm_map_clip_end(
 	register struct vm_map_header	*map_header,
 	register vm_map_entry_t		entry,
-	register vm_map_offset_t		end)
+	register vm_map_offset_t	end)
 {
 	register vm_map_entry_t	new_entry;
 
@@ -2254,7 +2798,7 @@ _vm_map_clip_end(
 	_vm_map_entry_link(map_header, entry, new_entry);
 
 	if (entry->is_sub_map)
-	 	vm_map_reference(new_entry->object.sub_map);
+		vm_map_reference(new_entry->object.sub_map);
 	else
 		vm_object_reference(new_entry->object.vm_object);
 }
@@ -2266,15 +2810,15 @@ _vm_map_clip_end(
  *	Asserts that the starting and ending region
  *	addresses fall within the valid range of the map.
  */
-#define	VM_MAP_RANGE_CHECK(map, start, end)		\
-		{					\
-		if (start < vm_map_min(map))		\
-			start = vm_map_min(map);	\
-		if (end > vm_map_max(map))		\
-			end = vm_map_max(map);		\
-		if (start > end)			\
-			start = end;			\
-		}
+#define	VM_MAP_RANGE_CHECK(map, start, end)	\
+	MACRO_BEGIN				\
+	if (start < vm_map_min(map))		\
+		start = vm_map_min(map);	\
+	if (end > vm_map_max(map))		\
+		end = vm_map_max(map);		\
+	if (start > end)			\
+		start = end;			\
+	MACRO_END
 
 /*
  *	vm_map_range_check:	[ internal use only ]
@@ -2373,21 +2917,18 @@ vm_map_submap(
 
 	vm_map_lock(map);
 
-	submap->mapped = TRUE;
-
-	VM_MAP_RANGE_CHECK(map, start, end);
-
-	if (vm_map_lookup_entry(map, start, &entry)) {
-		vm_map_clip_start(map, entry, start);
-	}
-	else
+	if (! vm_map_lookup_entry(map, start, &entry)) {
 		entry = entry->vme_next;
+	}
 
-	if(entry == vm_map_to_entry(map)) {
+	if (entry == vm_map_to_entry(map) ||
+	    entry->is_sub_map) {
 		vm_map_unlock(map);
 		return KERN_INVALID_ARGUMENT;
 	}
 
+	assert(!entry->use_pmap); /* we don't want to unnest anything here */
+	vm_map_clip_start(map, entry, start);
 	vm_map_clip_end(map, entry, end);
 
 	if ((entry->vme_start == start) && (entry->vme_end == end) &&
@@ -2397,34 +2938,37 @@ vm_map_submap(
 	    (object->copy == VM_OBJECT_NULL) &&
 	    (object->shadow == VM_OBJECT_NULL) &&
 	    (!object->pager_created)) {
-			entry->offset = (vm_object_offset_t)offset;
-			entry->object.vm_object = VM_OBJECT_NULL;
-			vm_object_deallocate(object);
-			entry->is_sub_map = TRUE;
-			entry->object.sub_map = submap;
-			vm_map_reference(submap);
+		entry->offset = (vm_object_offset_t)offset;
+		entry->object.vm_object = VM_OBJECT_NULL;
+		vm_object_deallocate(object);
+		entry->is_sub_map = TRUE;
+		entry->object.sub_map = submap;
+		vm_map_reference(submap);
+		submap->mapped = TRUE;
+
 #ifndef NO_NESTED_PMAP
-			if ((use_pmap) && (offset == 0)) {
-				/* nest if platform code will allow */
-				if(submap->pmap == NULL) {
-					submap->pmap = pmap_create((vm_map_size_t) 0, FALSE);
-					if(submap->pmap == PMAP_NULL) {
-						vm_map_unlock(map);
-						return(KERN_NO_SPACE);
-					}
+		if (use_pmap) {
+			/* nest if platform code will allow */
+			if(submap->pmap == NULL) {
+				submap->pmap = pmap_create((vm_map_size_t) 0, FALSE);
+				if(submap->pmap == PMAP_NULL) {
+					vm_map_unlock(map);
+					return(KERN_NO_SPACE);
 				}
-				result = pmap_nest(map->pmap, (entry->object.sub_map)->pmap, 
-						   (addr64_t)start,
-						   (addr64_t)start,
-						   (uint64_t)(end - start));
-				if(result)
-					panic("vm_map_submap: pmap_nest failed, rc = %08X\n", result);
-				entry->use_pmap = TRUE;
 			}
+			result = pmap_nest(map->pmap,
+					   (entry->object.sub_map)->pmap, 
+					   (addr64_t)start,
+					   (addr64_t)start,
+					   (uint64_t)(end - start));
+			if(result)
+				panic("vm_map_submap: pmap_nest failed, rc = %08X\n", result);
+			entry->use_pmap = TRUE;
+		}
 #else	/* NO_NESTED_PMAP */
-			pmap_remove(map->pmap, (addr64_t)start, (addr64_t)end);
+		pmap_remove(map->pmap, (addr64_t)start, (addr64_t)end);
 #endif	/* NO_NESTED_PMAP */
-			result = KERN_SUCCESS;
+		result = KERN_SUCCESS;
 	}
 	vm_map_unlock(map);
 
@@ -2448,14 +2992,13 @@ vm_map_protect(
 	register boolean_t	set_max)
 {
 	register vm_map_entry_t		current;
-	register vm_map_offset_t		prev;
+	register vm_map_offset_t	prev;
 	vm_map_entry_t			entry;
 	vm_prot_t			new_max;
-	boolean_t			clip;
 
 	XPR(XPR_VM_MAP,
-		"vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
-		(integer_t)map, start, end, new_prot, set_max);
+	    "vm_map_protect, 0x%X start 0x%X end 0x%X, new 0x%X %d",
+	    (integer_t)map, start, end, new_prot, set_max);
 
 	vm_map_lock(map);
 
@@ -2470,13 +3013,9 @@ vm_map_protect(
 
 	/*
 	 * 	Lookup the entry.  If it doesn't start in a valid
-	 *	entry, return an error.  Remember if we need to
-	 *	clip the entry.  We don't do it here because we don't
-	 *	want to make any changes until we've scanned the 
-	 *	entire range below for address and protection
-	 *	violations.
+	 *	entry, return an error.
 	 */
-	if (!(clip = vm_map_lookup_entry(map, start, &entry))) {
+	if (! vm_map_lookup_entry(map, start, &entry)) {
 		vm_map_unlock(map);
 		return(KERN_INVALID_ADDRESS);
 	}
@@ -2528,9 +3067,11 @@ vm_map_protect(
 	 */
 
 	current = entry;
-	if (clip) {
-		vm_map_clip_start(map, entry, start);
+	if (current != vm_map_to_entry(map)) {
+		/* clip and unnest if necessary */
+		vm_map_clip_start(map, current, start);
 	}
+
 	while ((current != vm_map_to_entry(map)) &&
 	       (current->vme_start < end)) {
 
@@ -2538,6 +3079,8 @@ vm_map_protect(
 
 		vm_map_clip_end(map, current, end);
 
+		assert(!current->use_pmap); /* clipping did unnest if needed */
+
 		old_prot = current->protection;
 
 		if(new_prot & VM_PROT_COPY) {
@@ -2553,8 +3096,8 @@ vm_map_protect(
 		if (set_max)
 			current->protection =
 				(current->max_protection = 
-					new_prot & ~VM_PROT_COPY) &
-					old_prot;
+				 new_prot & ~VM_PROT_COPY) &
+				old_prot;
 		else
 			current->protection = new_prot & ~VM_PROT_COPY;
 
@@ -2567,54 +3110,29 @@ vm_map_protect(
 		 *	write-protect fault occurred, it will be fixed up 
 		 *	properly, COW or not.
 		 */
-	 	/* the 256M hack for existing hardware limitations */
 		if (current->protection != old_prot) {
-	 	   if(current->is_sub_map && current->use_pmap) {
-			vm_map_offset_t	pmap_base_addr;
-			vm_map_offset_t	pmap_end_addr;
-#ifdef NO_NESTED_PMAP
-			__unused
-#endif	/* NO_NESTED_PMAP */
-			vm_map_entry_t	local_entry;
-
-
-			pmap_base_addr = 0xF0000000 & current->vme_start;
-			pmap_end_addr = (pmap_base_addr + 0x10000000) - 1;
-#ifndef NO_NESTED_PMAP
-			if(!vm_map_lookup_entry(map, 
-					pmap_base_addr, &local_entry))
-			   panic("vm_map_protect: nested pmap area is missing");
-			   while ((local_entry != vm_map_to_entry(map)) &&
-	       	  	          (local_entry->vme_start < pmap_end_addr)) {
-				local_entry->use_pmap = FALSE;
-				local_entry = local_entry->vme_next;
-			   }
-			   pmap_unnest(map->pmap, (addr64_t)pmap_base_addr);
-#endif	/* NO_NESTED_PMAP */
-		   }
-		   if (!(current->protection & VM_PROT_WRITE)) {
 			/* Look one level in we support nested pmaps */
 			/* from mapped submaps which are direct entries */
 			/* in our map */
 
-		        vm_prot_t prot;
+			vm_prot_t prot;
 
-			prot = current->protection;
-#ifdef STACK_ONLY_NX
-			if (current->alias != VM_MEMORY_STACK && prot)
+			prot = current->protection & ~VM_PROT_WRITE;
+
+			if (override_nx(map, current->alias) && prot)
 			        prot |= VM_PROT_EXECUTE;
-#endif
+
 			if (current->is_sub_map && current->use_pmap) {
 				pmap_protect(current->object.sub_map->pmap, 
-					current->vme_start,
-					current->vme_end,
-					prot);
+					     current->vme_start,
+					     current->vme_end,
+					     prot);
 			} else {
-				pmap_protect(map->pmap, current->vme_start,
-					current->vme_end,
-					prot);
+				pmap_protect(map->pmap,
+					     current->vme_start,
+					     current->vme_end,
+					     prot);
 			}
-		   }
 		}
 		current = current->vme_next;
 	}
@@ -2654,7 +3172,6 @@ vm_map_inherit(
 
 	if (vm_map_lookup_entry(map, start, &temp_entry)) {
 		entry = temp_entry;
-		vm_map_clip_start(map, entry, start);
 	}
 	else {
 		temp_entry = temp_entry->vme_next;
@@ -2675,9 +3192,14 @@ vm_map_inherit(
 	}
 
 	entry = temp_entry;
+	if (entry != vm_map_to_entry(map)) {
+		/* clip and unnest if necessary */
+		vm_map_clip_start(map, entry, start);
+	}
 
 	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
 		vm_map_clip_end(map, entry, end);
+		assert(!entry->use_pmap); /* clip did unnest if needed */
 
 		entry->inheritance = new_inheritance;
 
@@ -2688,6 +3210,116 @@ vm_map_inherit(
 	return(KERN_SUCCESS);
 }
 
+/*
+ * Update the accounting for the amount of wired memory in this map.  If the user has
+ * exceeded the defined limits, then we fail.  Wiring on behalf of the kernel never fails.
+ */
+
+static kern_return_t
+add_wire_counts(
+	vm_map_t	map,
+	vm_map_entry_t	entry, 
+	boolean_t	user_wire)
+{ 
+	vm_map_size_t	size;
+
+	if (user_wire) {
+
+		/*
+		 * We're wiring memory at the request of the user.  Check if this is the first time the user is wiring
+		 * this map entry.
+		 */
+
+		if (entry->user_wired_count == 0) {
+			size = entry->vme_end - entry->vme_start;
+ 
+			/*
+			 * Since this is the first time the user is wiring this map entry, check to see if we're
+			 * exceeding the user wire limits.  There is a per map limit which is the smaller of either
+			 * the process's rlimit or the global vm_user_wire_limit which caps this value.  There is also
+			 * a system-wide limit on the amount of memory all users can wire.  If the user is over either
+			 * limit, then we fail.
+			 */
+
+			if(size + map->user_wire_size > MIN(map->user_wire_limit, vm_user_wire_limit) ||
+		    	   size + ptoa_64(vm_page_wire_count) > vm_global_user_wire_limit)
+				return KERN_RESOURCE_SHORTAGE;
+
+			/*
+			 * The first time the user wires an entry, we also increment the wired_count and add this to
+			 * the total that has been wired in the map.
+			 */
+
+			if (entry->wired_count >= MAX_WIRE_COUNT)
+				return KERN_FAILURE;
+
+			entry->wired_count++;
+			map->user_wire_size += size;
+		}
+
+		if (entry->user_wired_count >= MAX_WIRE_COUNT)
+			return KERN_FAILURE;
+
+		entry->user_wired_count++;
+
+	} else {
+
+		/*
+		 * The kernel's wiring the memory.  Just bump the count and continue.
+		 */
+
+		if (entry->wired_count >= MAX_WIRE_COUNT)
+			panic("vm_map_wire: too many wirings");
+
+		entry->wired_count++;
+	}
+
+	return KERN_SUCCESS;
+}
+
+/*
+ * Update the memory wiring accounting now that the given map entry is being unwired.
+ */
+
+static void
+subtract_wire_counts(
+	vm_map_t	map,
+	vm_map_entry_t	entry, 
+	boolean_t	user_wire)
+{ 
+
+	if (user_wire) {
+
+		/*
+		 * We're unwiring memory at the request of the user.  See if we're removing the last user wire reference.
+		 */
+
+		if (entry->user_wired_count == 1) {
+
+			/*
+			 * We're removing the last user wire reference.  Decrement the wired_count and the total
+			 * user wired memory for this map.
+			 */
+
+			assert(entry->wired_count >= 1);
+			entry->wired_count--;
+			map->user_wire_size -= entry->vme_end - entry->vme_start;
+		}
+
+		assert(entry->user_wired_count >= 1);
+		entry->user_wired_count--;
+
+	} else {
+
+		/*
+		 * The kernel is unwiring the memory.   Just update the count.
+		 */
+
+		assert(entry->wired_count >= 1);
+		entry->wired_count--;
+	}
+}
+
 /*
  *	vm_map_wire:
  *
@@ -2737,19 +3369,38 @@ vm_map_wire_nested(
 		return KERN_SUCCESS;
 	}
 
-	if (vm_map_lookup_entry(map, start, &first_entry)) {
+	need_wakeup = FALSE;
+	cur_thread = current_thread();
+
+	s = start;
+	rc = KERN_SUCCESS;
+
+	if (vm_map_lookup_entry(map, s, &first_entry)) {
 		entry = first_entry;
-		/* vm_map_clip_start will be done later. */
+		/*
+		 * vm_map_clip_start will be done later.
+		 * We don't want to unnest any nested submaps here !
+		 */
 	} else {
 		/* Start address is not in map */
-		vm_map_unlock(map);
-		return(KERN_INVALID_ADDRESS);
+		rc = KERN_INVALID_ADDRESS;
+		goto done;
 	}
 
-	s=start;
-	need_wakeup = FALSE;
-	cur_thread = current_thread();
-	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
+	while ((entry != vm_map_to_entry(map)) && (s < end)) {
+		/*
+		 * At this point, we have wired from "start" to "s".
+		 * We still need to wire from "s" to "end".
+		 *
+		 * "entry" hasn't been clipped, so it could start before "s"
+		 * and/or end after "end".
+		 */
+
+		/* "e" is how far we want to wire in this entry */
+		e = entry->vme_end;
+		if (e > end)
+			e = end;
+
 		/*
 		 * If another thread is wiring/unwiring this entry then
 		 * block after informing other thread to wake us up.
@@ -2761,8 +3412,10 @@ vm_map_wire_nested(
 			 * We have not clipped the entry.  Make sure that
 			 * the start address is in range so that the lookup
 			 * below will succeed.
+			 * "s" is the current starting point: we've already
+			 * wired from "start" to "s" and we still have
+			 * to wire from "s" to "end".
 			 */
-			s = entry->vme_start < start? start: entry->vme_start;
 
 			entry->needs_wakeup = TRUE;
 
@@ -2778,8 +3431,8 @@ vm_map_wire_nested(
 			 * User wiring is interruptible
 			 */
 			wait_result = vm_map_entry_wait(map, 
-					  (user_wire) ? THREAD_ABORTSAFE :
-					                THREAD_UNINT);
+							(user_wire) ? THREAD_ABORTSAFE :
+							THREAD_UNINT);
 			if (user_wire && wait_result ==	THREAD_INTERRUPTED) {
 				/*
 				 * undo the wirings we have done so far
@@ -2787,9 +3440,8 @@ vm_map_wire_nested(
 				 * because we cannot tell if we were the
 				 * only one waiting.
 				 */
-				vm_map_unlock(map);
-				vm_map_unwire(map, start, s, user_wire);
-				return(KERN_FAILURE);
+				rc = KERN_FAILURE;
+				goto done;
 			}
 
 			/*
@@ -2810,30 +3462,37 @@ vm_map_wire_nested(
 				 * entry.  let vm_map_unwire worry about
 				 * checking the validity of the range.
 				 */
-				vm_map_unlock(map);
-				vm_map_unwire(map, start, s, user_wire);
-				return(KERN_FAILURE);
+				rc = KERN_FAILURE;
+				goto done;
 			}
 			entry = first_entry;
 			continue;
 		}
-		
-		if(entry->is_sub_map) {
+	
+		if (entry->is_sub_map) {
 			vm_map_offset_t	sub_start;
 			vm_map_offset_t	sub_end;
 			vm_map_offset_t	local_start;
 			vm_map_offset_t	local_end;
 			pmap_t		pmap;
-			
-			vm_map_clip_start(map, entry, start);
+
+			vm_map_clip_start(map, entry, s);
 			vm_map_clip_end(map, entry, end);
 
 			sub_start = entry->offset;
-			sub_end = entry->vme_end - entry->vme_start;
-			sub_end += entry->offset;
-			
+			sub_end = entry->vme_end;
+			sub_end += entry->offset - entry->vme_start;
+		
 			local_end = entry->vme_end;
 			if(map_pmap == NULL) {
+				vm_object_t		object;
+				vm_object_offset_t	offset;
+				vm_prot_t		prot;
+				boolean_t		wired;
+				vm_map_entry_t		local_entry;
+				vm_map_version_t	 version;
+				vm_map_t		lookup_map;
+
 				if(entry->use_pmap) {
 					pmap = entry->object.sub_map->pmap;
 					/* ppc implementation requires that */
@@ -2842,114 +3501,95 @@ vm_map_wire_nested(
 #ifdef notdef
 					pmap_addr = sub_start;
 #endif
-					pmap_addr = start;
+					pmap_addr = s;
 				} else {
 					pmap = map->pmap;
-					pmap_addr = start;
+					pmap_addr = s;
 				}
+
 				if (entry->wired_count) {
-					if (entry->wired_count 
-							>= MAX_WIRE_COUNT)
-					panic("vm_map_wire: too many wirings");
-
-					if (user_wire &&
-			    			entry->user_wired_count 
-							>= MAX_WIRE_COUNT) {
-				   	   vm_map_unlock(map);
-				   	   vm_map_unwire(map, start,
-						entry->vme_start, user_wire);
-				   	   return(KERN_FAILURE);
-					}
-					if(user_wire)
-						entry->user_wired_count++;
-					if((!user_wire) ||
-					     (entry->user_wired_count == 0))
-						entry->wired_count++;
+					if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS)
+						goto done;
+
+					/*
+					 * The map was not unlocked:
+					 * no need to goto re-lookup.
+					 * Just go directly to next entry.
+					 */
 					entry = entry->vme_next;
+					s = entry->vme_start;
 					continue;
 
-				} else {
-					vm_object_t		object;
-					vm_map_offset_t		offset_hi;
-					vm_map_offset_t		offset_lo;
-					vm_object_offset_t	offset;
-					vm_prot_t		prot;
-					boolean_t		wired;
-					vm_behavior_t		behavior;
-					vm_map_entry_t		local_entry;
-					vm_map_version_t	 version;
-					vm_map_t		lookup_map;
-
-					/* call vm_map_lookup_locked to */
-					/* cause any needs copy to be   */
-					/* evaluated */
-					local_start = entry->vme_start;
-					lookup_map = map;
-					vm_map_lock_write_to_read(map);
-					if(vm_map_lookup_locked(
-						&lookup_map, local_start, 
-						access_type,
-						&version, &object,
-						&offset, &prot, &wired,
-						&behavior, &offset_lo,
-						&offset_hi, &real_map)) {
-						
-						vm_map_unlock_read(lookup_map);
-				   	   	vm_map_unwire(map, start,
-						   entry->vme_start, user_wire);
-				   	   	return(KERN_FAILURE);
-					}
-					if(real_map != lookup_map)
-						vm_map_unlock(real_map);
-					vm_map_unlock_read(lookup_map);
-					vm_map_lock(map);
-					vm_object_unlock(object);
+				}
 
-					if (!vm_map_lookup_entry(map, 
-						local_start, &local_entry)) {
-						vm_map_unlock(map);
-				   	   	vm_map_unwire(map, start,
-						   entry->vme_start, user_wire);
-				   	   	return(KERN_FAILURE);
-					}
-					/* did we have a change of type? */
-					if (!local_entry->is_sub_map) {
-						last_timestamp = map->timestamp;
-						continue;
-					}
-					entry = local_entry;
-					if (user_wire)
-						entry->user_wired_count++;
-					if((!user_wire) || 
-						(entry->user_wired_count == 1))
-					entry->wired_count++;
+				/* call vm_map_lookup_locked to */
+				/* cause any needs copy to be   */
+				/* evaluated */
+				local_start = entry->vme_start;
+				lookup_map = map;
+				vm_map_lock_write_to_read(map);
+				if(vm_map_lookup_locked(
+					   &lookup_map, local_start, 
+					   access_type,
+					   OBJECT_LOCK_EXCLUSIVE,
+					   &version, &object,
+					   &offset, &prot, &wired,
+					   NULL,
+					   &real_map)) {
 
-					entry->in_transition = TRUE;
+					vm_map_unlock_read(lookup_map);
+					vm_map_unwire(map, start,
+						      s, user_wire);
+					return(KERN_FAILURE);
+				}
+				if(real_map != lookup_map)
+					vm_map_unlock(real_map);
+				vm_map_unlock_read(lookup_map);
+				vm_map_lock(map);
+				vm_object_unlock(object);
 
-					vm_map_unlock(map);
-					rc = vm_map_wire_nested(
-						entry->object.sub_map, 
-						sub_start, sub_end,
-						access_type, 
-						user_wire, pmap, pmap_addr);
-					vm_map_lock(map);
+				/* we unlocked, so must re-lookup */
+				if (!vm_map_lookup_entry(map, 
+							 local_start,
+							 &local_entry)) {
+					rc = KERN_FAILURE;
+					goto done;
+				}
+
+				/*
+				 * entry could have been "simplified",
+				 * so re-clip
+				 */
+				entry = local_entry;
+				assert(s == local_start);
+				vm_map_clip_start(map, entry, s);
+				vm_map_clip_end(map, entry, end);
+				/* re-compute "e" */
+				e = entry->vme_end;
+				if (e > end)
+					e = end;
+
+				/* did we have a change of type? */
+				if (!entry->is_sub_map) {
+					last_timestamp = map->timestamp;
+					continue;
 				}
 			} else {
 				local_start = entry->vme_start;
-				if (user_wire)
-					entry->user_wired_count++;
-				if((!user_wire) || 
-					(entry->user_wired_count == 1))
-					entry->wired_count++;
-				vm_map_unlock(map);
-				rc = vm_map_wire_nested(entry->object.sub_map, 
+				pmap = map_pmap;
+			}
+
+			if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS)
+				goto done;
+
+			entry->in_transition = TRUE;
+
+			vm_map_unlock(map);
+			rc = vm_map_wire_nested(entry->object.sub_map, 
 						sub_start, sub_end,
 						access_type, 
-						user_wire, map_pmap, pmap_addr);
-				vm_map_lock(map);
-			}
-			s = entry->vme_start;
-			e = entry->vme_end;
+						user_wire, pmap, pmap_addr);
+			vm_map_lock(map);
 
 			/*
 			 * Find the entry again.  It could have been clipped
@@ -2960,9 +3600,15 @@ vm_map_wire_nested(
 				panic("vm_map_wire: re-lookup failed");
 			entry = first_entry;
 
+			assert(local_start == s);
+			/* re-compute "e" */
+			e = entry->vme_end;
+			if (e > end)
+				e = end;
+
 			last_timestamp = map->timestamp;
 			while ((entry != vm_map_to_entry(map)) &&
-		       		     (entry->vme_start < e)) {
+			       (entry->vme_start < e)) {
 				assert(entry->in_transition);
 				entry->in_transition = FALSE;
 				if (entry->needs_wakeup) {
@@ -2970,24 +3616,16 @@ vm_map_wire_nested(
 					need_wakeup = TRUE;
 				}
 				if (rc != KERN_SUCCESS) {/* from vm_*_wire */
-					if (user_wire)
-						entry->user_wired_count--;
-					if ((!user_wire) || 
-						(entry->user_wired_count == 0))
-						entry->wired_count--;
+					subtract_wire_counts(map, entry, user_wire);
 				}
 				entry = entry->vme_next;
 			}
 			if (rc != KERN_SUCCESS) {	/* from vm_*_wire */
-				vm_map_unlock(map);
-				if (need_wakeup)
-					vm_map_entry_wakeup(map);
-				/*
-				 * undo everything upto the previous entry.
-				 */
-				(void)vm_map_unwire(map, start, s, user_wire);
-				return rc;
+				goto done;
 			}
+
+			/* no need to relookup again */
+			s = entry->vme_start;
 			continue;
 		}
 
@@ -2996,29 +3634,19 @@ vm_map_wire_nested(
 		 * the appropriate wire reference count.
 		 */
 		if (entry->wired_count) {
-			/* sanity check: wired_count is a short */
-			if (entry->wired_count >= MAX_WIRE_COUNT)
-				panic("vm_map_wire: too many wirings");
-
-			if (user_wire &&
-			    entry->user_wired_count >= MAX_WIRE_COUNT) {
-				vm_map_unlock(map);
-				vm_map_unwire(map, start,
-						entry->vme_start, user_wire);
-				return(KERN_FAILURE);
-			}
 			/*
 			 * entry is already wired down, get our reference
 			 * after clipping to our range.
 			 */
-			vm_map_clip_start(map, entry, start);
+			vm_map_clip_start(map, entry, s);
 			vm_map_clip_end(map, entry, end);
-			if (user_wire)
-				entry->user_wired_count++;
-			if ((!user_wire) || (entry->user_wired_count == 1))
-				entry->wired_count++;
 
+			if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS)
+				goto done;
+
+			/* map was not unlocked: no need to relookup */
 			entry = entry->vme_next;
+			s = entry->vme_start;
 			continue;
 		}
 
@@ -3048,11 +3676,13 @@ vm_map_wire_nested(
 			entry->offset = (vm_object_offset_t)0;
 		}
 
-		vm_map_clip_start(map, entry, start);
+		vm_map_clip_start(map, entry, s);
 		vm_map_clip_end(map, entry, end);
 
-		s = entry->vme_start;
+		/* re-compute "e" */
 		e = entry->vme_end;
+		if (e > end)
+			e = end;
 
 		/*
 		 * Check for holes and protection mismatch.
@@ -3061,30 +3691,23 @@ vm_map_wire_nested(
 		 * Protection: Access requested must be allowed, unless
 		 *	wiring is by protection class
 		 */
-		if ((((entry->vme_end < end) &&
-		     ((entry->vme_next == vm_map_to_entry(map)) ||
-		      (entry->vme_next->vme_start > entry->vme_end))) ||
-		     ((entry->protection & access_type) != access_type))) {
-			/*
-			 * Found a hole or protection problem.
-			 * Unwire the region we wired so far.
-			 */
-			if (start != entry->vme_start) {
-				vm_map_unlock(map);
-				vm_map_unwire(map, start, s, user_wire);
-			} else {
-				vm_map_unlock(map);
-			}
-			return((entry->protection&access_type) != access_type?
-				KERN_PROTECTION_FAILURE: KERN_INVALID_ADDRESS);
+		if ((entry->vme_end < end) &&
+		    ((entry->vme_next == vm_map_to_entry(map)) ||
+		     (entry->vme_next->vme_start > entry->vme_end))) {
+			/* found a hole */
+			rc = KERN_INVALID_ADDRESS;
+			goto done;
+		}
+		if ((entry->protection & access_type) != access_type) {
+			/* found a protection problem */
+			rc = KERN_PROTECTION_FAILURE;
+			goto done;
 		}
 
 		assert(entry->wired_count == 0 && entry->user_wired_count == 0);
 
-		if (user_wire)
-			entry->user_wired_count++;
-		if ((!user_wire) || (entry->user_wired_count == 1))
-			entry->wired_count++;
+		if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS)
+			goto done;
 
 		entry->in_transition = TRUE;
 
@@ -3113,11 +3736,11 @@ vm_map_wire_nested(
 
 		if(map_pmap)
 			rc = vm_fault_wire(map, 
-					&tmp_entry, map_pmap, pmap_addr);
+					   &tmp_entry, map_pmap, pmap_addr);
 		else
 			rc = vm_fault_wire(map, 
-					&tmp_entry, map->pmap, 
-					tmp_entry.vme_start);
+					   &tmp_entry, map->pmap, 
+					   tmp_entry.vme_start);
 
 		if (!user_wire && cur_thread != THREAD_NULL)
 			thread_interrupt_level(interruptible_state);
@@ -3130,7 +3753,7 @@ vm_map_wire_nested(
 			 * after we unlocked the map.
 			 */
 			if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
-								&first_entry))
+						 &first_entry))
 				panic("vm_map_wire: re-lookup failed");
 
 			entry = first_entry;
@@ -3147,26 +3770,24 @@ vm_map_wire_nested(
 				need_wakeup = TRUE;
 			}
 			if (rc != KERN_SUCCESS) {	/* from vm_*_wire */
-				if (user_wire)
-					entry->user_wired_count--;
-				if ((!user_wire) || 
-						(entry->user_wired_count == 0))
-					entry->wired_count--;
+				subtract_wire_counts(map, entry, user_wire);
 			}
 			entry = entry->vme_next;
 		}
 
 		if (rc != KERN_SUCCESS) {		/* from vm_*_wire */
-			vm_map_unlock(map);
-			if (need_wakeup)
-				vm_map_entry_wakeup(map);
-			/*
-			 * undo everything upto the previous entry.
-			 */
-			(void)vm_map_unwire(map, start, s, user_wire);
-			return rc;
+			goto done;
 		}
+
+		s = entry->vme_start;
 	} /* end while loop through map entries */
+
+done:
+	if (rc == KERN_SUCCESS) {
+		/* repair any damage we may have made to the VM map */
+		vm_map_simplify_range(map, start, end);
+	}
+
 	vm_map_unlock(map);
 
 	/*
@@ -3175,7 +3796,12 @@ vm_map_wire_nested(
 	if (need_wakeup)
 		vm_map_entry_wakeup(map);
 
-	return(KERN_SUCCESS);
+	if (rc != KERN_SUCCESS) {
+		/* undo what has been wired so far */
+		vm_map_unwire(map, start, s, user_wire);
+	}
+
+	return rc;
 
 }
 
@@ -3201,10 +3827,10 @@ vm_map_wire(
 	 * existing mappings
 	 */
 	VM_MAP_RANGE_CHECK(map, start, end);
-        mapping_prealloc(end - start);
+	mapping_prealloc(end - start);
 #endif
 	kret = vm_map_wire_nested(map, start, end, access_type, 
-						user_wire, (pmap_t)NULL, 0);
+				  user_wire, (pmap_t)NULL, 0);
 #ifdef ppc
 	mapping_relpre();
 #endif
@@ -3248,11 +3874,23 @@ vm_map_unwire_nested(
 	assert(page_aligned(start));
 	assert(page_aligned(end));
 
+	if (start == end) {
+		/* We unwired what the caller asked for: zero pages */
+		vm_map_unlock(map);
+		return KERN_SUCCESS;
+	}
+
 	if (vm_map_lookup_entry(map, start, &first_entry)) {
 		entry = first_entry;
-		/*	vm_map_clip_start will be done later. */
+		/*
+		 * vm_map_clip_start will be done later.
+		 * We don't want to unnest any nested sub maps here !
+		 */
 	}
 	else {
+		if (!user_wire) {
+			panic("vm_map_unwire: start not found");
+		}
 		/*	Start address is not in map. */
 		vm_map_unlock(map);
 		return(KERN_INVALID_ADDRESS);
@@ -3274,20 +3912,41 @@ vm_map_unwire_nested(
 			 * have a reference to it, because if we did, this
 			 * entry will not be getting unwired now.
 			 */
-			if (!user_wire)
+			if (!user_wire) {
+				/*
+				 * XXX FBDP
+				 * This could happen:  there could be some
+				 * overlapping vslock/vsunlock operations
+				 * going on.
+				 * We should probably just wait and retry,
+				 * but then we have to be careful that this
+				 * entry could get "simplified" after 
+				 * "in_transition" gets unset and before
+				 * we re-lookup the entry, so we would
+				 * have to re-clip the entry to avoid
+				 * re-unwiring what we have already unwired...
+				 * See vm_map_wire_nested().
+				 *
+				 * Or we could just ignore "in_transition"
+				 * here and proceed to decement the wired
+				 * count(s) on this entry.  That should be fine
+				 * as long as "wired_count" doesn't drop all
+				 * the way to 0 (and we should panic if THAT
+				 * happens).
+				 */
 				panic("vm_map_unwire: in_transition entry");
+			}
 
 			entry = entry->vme_next;
 			continue;
 		}
 
-		if(entry->is_sub_map) {
+		if (entry->is_sub_map) {
 			vm_map_offset_t	sub_start;
 			vm_map_offset_t	sub_end;
 			vm_map_offset_t	local_end;
 			pmap_t		pmap;
-			
-
+		
 			vm_map_clip_start(map, entry, start);
 			vm_map_clip_end(map, entry, end);
 
@@ -3296,127 +3955,126 @@ vm_map_unwire_nested(
 			sub_end += entry->offset;
 			local_end = entry->vme_end;
 			if(map_pmap == NULL) {
-			   if(entry->use_pmap) {
+				if(entry->use_pmap) {
 					pmap = entry->object.sub_map->pmap;
 					pmap_addr = sub_start;
-			   } else {
+				} else {
 					pmap = map->pmap;
 					pmap_addr = start;
-			   }
-			   if (entry->wired_count == 0 ||
-		   	       (user_wire && entry->user_wired_count == 0)) {
-				if (!user_wire)
-				   panic("vm_map_unwire: entry is unwired");
-			      entry = entry->vme_next;
-			      continue;
-			   }
-
-			   /*
-		 	    * Check for holes
-		 	    * Holes: Next entry should be contiguous unless
-		 	    * this is the end of the region.
-		 	    */
-			   if (((entry->vme_end < end) && 
-		    		((entry->vme_next == vm_map_to_entry(map)) ||
-		     		(entry->vme_next->vme_start 
-						> entry->vme_end)))) {
-				if (!user_wire)
-				  panic("vm_map_unwire: non-contiguous region");
+				}
+				if (entry->wired_count == 0 ||
+				    (user_wire && entry->user_wired_count == 0)) {
+					if (!user_wire)
+						panic("vm_map_unwire: entry is unwired");
+					entry = entry->vme_next;
+					continue;
+				}
+
+				/*
+				 * Check for holes
+				 * Holes: Next entry should be contiguous unless
+				 * this is the end of the region.
+				 */
+				if (((entry->vme_end < end) && 
+				     ((entry->vme_next == vm_map_to_entry(map)) ||
+				      (entry->vme_next->vme_start 
+				       > entry->vme_end)))) {
+					if (!user_wire)
+						panic("vm_map_unwire: non-contiguous region");
 /*
-				entry = entry->vme_next;
-				continue;
+					entry = entry->vme_next;
+					continue;
 */
-			   }
+				}
 
-			   if (!user_wire || (--entry->user_wired_count == 0))
-				entry->wired_count--;
+				subtract_wire_counts(map, entry, user_wire);
 
-			   if (entry->wired_count != 0) {
-				entry = entry->vme_next;
-				continue;
-			   }
+				if (entry->wired_count != 0) {
+					entry = entry->vme_next;
+					continue;
+				}
 
-			   entry->in_transition = TRUE;
-			   tmp_entry = *entry;/* see comment in vm_map_wire() */
+				entry->in_transition = TRUE;
+				tmp_entry = *entry;/* see comment in vm_map_wire() */
+
+				/*
+				 * We can unlock the map now. The in_transition state
+				 * guarantees existance of the entry.
+				 */
+				vm_map_unlock(map);
+				vm_map_unwire_nested(entry->object.sub_map, 
+						     sub_start, sub_end, user_wire, pmap, pmap_addr);
+				vm_map_lock(map);
 
-			   /*
-		 	    * We can unlock the map now. The in_transition state
-		 	    * guarantees existance of the entry.
-		 	    */
-			   vm_map_unlock(map);
-			   vm_map_unwire_nested(entry->object.sub_map, 
-				sub_start, sub_end, user_wire, pmap, pmap_addr);
-			   vm_map_lock(map);
+				if (last_timestamp+1 != map->timestamp) {
+					/*
+					 * Find the entry again.  It could have been 
+					 * clipped or deleted after we unlocked the map.
+					 */
+					if (!vm_map_lookup_entry(map, 
+								 tmp_entry.vme_start,
+								 &first_entry)) {
+						if (!user_wire)
+							panic("vm_map_unwire: re-lookup failed");
+						entry = first_entry->vme_next;
+					} else
+						entry = first_entry;
+				}
+				last_timestamp = map->timestamp;
 
-			   if (last_timestamp+1 != map->timestamp) {
 				/*
-				 * Find the entry again.  It could have been 
-				 * clipped or deleted after we unlocked the map.
-			 	 */
-				if (!vm_map_lookup_entry(map, 
-						tmp_entry.vme_start,
-							&first_entry)) {
-					if (!user_wire)
-				          panic("vm_map_unwire: re-lookup failed");
-					entry = first_entry->vme_next;
-				} else
-					entry = first_entry;
-			   }
-			   last_timestamp = map->timestamp;
-
-			   /*
-		 	    * clear transition bit for all constituent entries
-		 	    * that were in the original entry (saved in 
-			    * tmp_entry).  Also check for waiters.
-		 	    */
-			   while ((entry != vm_map_to_entry(map)) &&
-		       		(entry->vme_start < tmp_entry.vme_end)) {
-				assert(entry->in_transition);
-				entry->in_transition = FALSE;
-				if (entry->needs_wakeup) {
-					entry->needs_wakeup = FALSE;
-					need_wakeup = TRUE;
+				 * clear transition bit for all constituent entries
+				 * that were in the original entry (saved in 
+				 * tmp_entry).  Also check for waiters.
+				 */
+				while ((entry != vm_map_to_entry(map)) &&
+				       (entry->vme_start < tmp_entry.vme_end)) {
+					assert(entry->in_transition);
+					entry->in_transition = FALSE;
+					if (entry->needs_wakeup) {
+						entry->needs_wakeup = FALSE;
+						need_wakeup = TRUE;
+					}
+					entry = entry->vme_next;
 				}
-				entry = entry->vme_next;
-			   }
-			   continue;
+				continue;
 			} else {
-			   vm_map_unlock(map);
-			   vm_map_unwire_nested(entry->object.sub_map,
-			   	sub_start, sub_end, user_wire, map_pmap,
-				pmap_addr);
-			   vm_map_lock(map);
+				vm_map_unlock(map);
+				vm_map_unwire_nested(entry->object.sub_map,
+						     sub_start, sub_end, user_wire, map_pmap,
+						     pmap_addr);
+				vm_map_lock(map);
 
-			   if (last_timestamp+1 != map->timestamp) {
-				/*
-				 * Find the entry again.  It could have been 
-				 * clipped or deleted after we unlocked the map.
-			 	 */
-				if (!vm_map_lookup_entry(map, 
-						tmp_entry.vme_start,
-							&first_entry)) {
-					if (!user_wire)
-				          panic("vm_map_unwire: re-lookup failed");
-					entry = first_entry->vme_next;
-				} else
-					entry = first_entry;
-			   }
-			   last_timestamp = map->timestamp;
+				if (last_timestamp+1 != map->timestamp) {
+					/*
+					 * Find the entry again.  It could have been 
+					 * clipped or deleted after we unlocked the map.
+					 */
+					if (!vm_map_lookup_entry(map, 
+								 tmp_entry.vme_start,
+								 &first_entry)) {
+						if (!user_wire)
+							panic("vm_map_unwire: re-lookup failed");
+						entry = first_entry->vme_next;
+					} else
+						entry = first_entry;
+				}
+				last_timestamp = map->timestamp;
 			}
 		}
 
 
 		if ((entry->wired_count == 0) ||
-		   (user_wire && entry->user_wired_count == 0)) {
+		    (user_wire && entry->user_wired_count == 0)) {
 			if (!user_wire)
 				panic("vm_map_unwire: entry is unwired");
 
 			entry = entry->vme_next;
 			continue;
 		}
-		
+	
 		assert(entry->wired_count > 0 &&
-			(!user_wire || entry->user_wired_count > 0));
+		       (!user_wire || entry->user_wired_count > 0));
 
 		vm_map_clip_start(map, entry, start);
 		vm_map_clip_end(map, entry, end);
@@ -3427,8 +4085,8 @@ vm_map_unwire_nested(
 		 *	  this is the end of the region.
 		 */
 		if (((entry->vme_end < end) && 
-		    ((entry->vme_next == vm_map_to_entry(map)) ||
-		     (entry->vme_next->vme_start > entry->vme_end)))) {
+		     ((entry->vme_next == vm_map_to_entry(map)) ||
+		      (entry->vme_next->vme_start > entry->vme_end)))) {
 
 			if (!user_wire)
 				panic("vm_map_unwire: non-contiguous region");
@@ -3436,8 +4094,7 @@ vm_map_unwire_nested(
 			continue;
 		}
 
-		if (!user_wire || (--entry->user_wired_count == 0))
-			entry->wired_count--;
+		subtract_wire_counts(map, entry, user_wire);
 
 		if (entry->wired_count != 0) {
 			entry = entry->vme_next;
@@ -3454,11 +4111,11 @@ vm_map_unwire_nested(
 		vm_map_unlock(map);
 		if(map_pmap) {
 			vm_fault_unwire(map, 
-				&tmp_entry, FALSE, map_pmap, pmap_addr);
+					&tmp_entry, FALSE, map_pmap, pmap_addr);
 		} else {
 			vm_fault_unwire(map, 
-				&tmp_entry, FALSE, map->pmap, 
-				tmp_entry.vme_start);
+					&tmp_entry, FALSE, map->pmap, 
+					tmp_entry.vme_start);
 		}
 		vm_map_lock(map);
 
@@ -3468,9 +4125,9 @@ vm_map_unwire_nested(
 			 * or deleted after we unlocked the map.
 			 */
 			if (!vm_map_lookup_entry(map, tmp_entry.vme_start,
-								&first_entry)) {
+						 &first_entry)) {
 				if (!user_wire)
-				       panic("vm_map_unwire: re-lookup failed");
+					panic("vm_map_unwire: re-lookup failed");
 				entry = first_entry->vme_next;
 			} else
 				entry = first_entry;
@@ -3522,7 +4179,7 @@ vm_map_unwire(
 	boolean_t		user_wire)
 {
 	return vm_map_unwire_nested(map, start, end, 
-					user_wire, (pmap_t)NULL, 0);
+				    user_wire, (pmap_t)NULL, 0);
 }
 
 
@@ -3568,11 +4225,10 @@ vm_map_entry_delete(
 	if (submap)
 		vm_map_deallocate(submap);
 	else
-	 	vm_object_deallocate(object);
+		vm_object_deallocate(object);
 
 }
 
-
 void
 vm_map_submap_pmap_clean(
 	vm_map_t	map,
@@ -3589,11 +4245,11 @@ vm_map_submap_pmap_clean(
 	submap_end = offset + (end - start);
 	submap_start = offset;
 	if(vm_map_lookup_entry(sub_map, offset, &entry)) {
-			
+		
 		remove_size = (entry->vme_end - entry->vme_start);
 		if(offset > entry->vme_start)
 			remove_size -= offset - entry->vme_start;
-			
+		
 
 		if(submap_end < entry->vme_end) {
 			remove_size -=
@@ -3609,7 +4265,7 @@ vm_map_submap_pmap_clean(
 		} else {
 
 			if((map->mapped) && (map->ref_count)
-				&& (entry->object.vm_object != NULL)) {
+			   && (entry->object.vm_object != NULL)) {
 				vm_object_pmap_protect(
 					entry->object.vm_object,
 					entry->offset,
@@ -3619,16 +4275,16 @@ vm_map_submap_pmap_clean(
 					VM_PROT_NONE);
 			} else {
 				pmap_remove(map->pmap, 
-					(addr64_t)start, 
-					(addr64_t)(start + remove_size));
+					    (addr64_t)start, 
+					    (addr64_t)(start + remove_size));
 			}
 		}
 	}
 
 	entry = entry->vme_next;
-	
+
 	while((entry != vm_map_to_entry(sub_map)) 
-			&& (entry->vme_start < submap_end)) {
+	      && (entry->vme_start < submap_end)) {
 		remove_size = (entry->vme_end - entry->vme_start); 
 		if(submap_end < entry->vme_end) {
 			remove_size -= entry->vme_end - submap_end;
@@ -3642,7 +4298,7 @@ vm_map_submap_pmap_clean(
 				entry->offset);
 		} else {
 			if((map->mapped) && (map->ref_count)
-				&& (entry->object.vm_object != NULL)) {
+			   && (entry->object.vm_object != NULL)) {
 				vm_object_pmap_protect(
 					entry->object.vm_object,
 					entry->offset,
@@ -3652,10 +4308,10 @@ vm_map_submap_pmap_clean(
 					VM_PROT_NONE);
 			} else {
 				pmap_remove(map->pmap, 
-				   (addr64_t)((start + entry->vme_start) 
-								- offset),
-				   (addr64_t)(((start + entry->vme_start) 
-					- offset) + remove_size));
+					    (addr64_t)((start + entry->vme_start) 
+						       - offset),
+					    (addr64_t)(((start + entry->vme_start) 
+							- offset) + remove_size));
 			}
 		}
 		entry = entry->vme_next;
@@ -3684,14 +4340,14 @@ vm_map_delete(
 {
 	vm_map_entry_t		entry, next;
 	struct	 vm_map_entry	*first_entry, tmp_entry;
-	register vm_map_offset_t	s, e;
+	register vm_map_offset_t s;
 	register vm_object_t	object;
 	boolean_t		need_wakeup;
 	unsigned int		last_timestamp = ~0; /* unlikely value */
 	int			interruptible;
 
 	interruptible = (flags & VM_MAP_REMOVE_INTERRUPTIBLE) ? 
-	  		THREAD_ABORTSAFE : THREAD_UNINT;
+		THREAD_ABORTSAFE : THREAD_UNINT;
 
 	/*
 	 * All our DMA I/O operations in IOKit are currently done by
@@ -3711,7 +4367,14 @@ vm_map_delete(
 	 */
 	if (vm_map_lookup_entry(map, start, &first_entry)) {
 		entry = first_entry;
-		vm_map_clip_start(map, entry, start);
+		if (start == entry->vme_start) {
+			/*
+			 * No need to clip.  We don't want to cause
+			 * any unnecessary unnesting in this case...
+			 */
+		} else {
+			vm_map_clip_start(map, entry, start);
+		}
 
 		/*
 		 *	Fix the lookup hint now, rather than each
@@ -3726,9 +4389,41 @@ vm_map_delete(
 	/*
 	 *	Step through all entries in this region
 	 */
-	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
+	s = entry->vme_start;
+	while ((entry != vm_map_to_entry(map)) && (s < end)) {
+		/*
+		 * At this point, we have deleted all the memory entries
+		 * between "start" and "s".  We still need to delete
+		 * all memory entries between "s" and "end".
+		 * While we were blocked and the map was unlocked, some
+		 * new memory entries could have been re-allocated between
+		 * "start" and "s" and we don't want to mess with those.
+		 * Some of those entries could even have been re-assembled
+		 * with an entry after "s" (in vm_map_simplify_entry()), so
+		 * we may have to vm_map_clip_start() again.
+		 */
 
-		vm_map_clip_end(map, entry, end);
+		if (entry->vme_start >= s) {
+			/*
+			 * This entry starts on or after "s"
+			 * so no need to clip its start.
+			 */
+		} else {
+			/*
+			 * This entry has been re-assembled by a
+			 * vm_map_simplify_entry().  We need to
+			 * re-clip its start.
+			 */
+			vm_map_clip_start(map, entry, s);
+		}
+		if (entry->vme_end <= end) {
+			/*
+			 * This entry is going away completely, so no need
+			 * to clip and possibly cause an unnecessary unnesting.
+			 */
+		} else {
+			vm_map_clip_end(map, entry, end);
+		}
 		if (entry->in_transition) {
 			wait_result_t wait_result;
 
@@ -3736,7 +4431,7 @@ vm_map_delete(
 			 * Another thread is wiring/unwiring this entry.
 			 * Let the other thread know we are waiting.
 			 */
-			s = entry->vme_start;
+			assert(s == entry->vme_start);
 			entry->needs_wakeup = TRUE;
 
 			/*
@@ -3771,6 +4466,7 @@ vm_map_delete(
 				 * User: use the next entry
 				 */
 				entry = first_entry->vme_next;
+				s = entry->vme_start;
 			} else {
 				entry = first_entry;
 				SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
@@ -3780,20 +4476,23 @@ vm_map_delete(
 		} /* end in_transition */
 
 		if (entry->wired_count) {
+			boolean_t	user_wire;
+
+			user_wire = entry->user_wired_count > 0;
+
 			/*
 			 * 	Remove a kernel wiring if requested or if
 			 *	there are user wirings.
 			 */
 			if ((flags & VM_MAP_REMOVE_KUNWIRE) || 
-			   (entry->user_wired_count > 0))
+			    (entry->user_wired_count > 0))
 				entry->wired_count--;
 
 			/* remove all user wire references */
 			entry->user_wired_count = 0;
 
 			if (entry->wired_count != 0) {
-				assert((map != kernel_map) && 
-				       (!entry->is_sub_map));
+				assert(map != kernel_map);
 				/*
 				 * Cannot continue.  Typical case is when
 				 * a user thread has physical io pending on
@@ -3804,36 +4503,36 @@ vm_map_delete(
 				if (flags & VM_MAP_REMOVE_WAIT_FOR_KWIRE) {
 					wait_result_t wait_result;
 
-					s = entry->vme_start;
+					assert(s == entry->vme_start);
 					entry->needs_wakeup = TRUE;
 					wait_result = vm_map_entry_wait(map,
-								interruptible);
+									interruptible);
 
 					if (interruptible &&
-			   		    wait_result == THREAD_INTERRUPTED) {
+					    wait_result == THREAD_INTERRUPTED) {
 						/*
-				 	 	 * We do not clear the 
+						 * We do not clear the 
 						 * needs_wakeup flag, since we 
 						 * cannot tell if we were the 
 						 * only one.
-				 	 	 */
+						 */
 						vm_map_unlock(map);
 						return KERN_ABORTED;
 					}
 
 					/*
-			 		 * The entry could have been clipped or
+					 * The entry could have been clipped or
 					 * it may not exist anymore.  Look it
 					 * up again.
-			 		 */
+					 */
 					if (!vm_map_lookup_entry(map, s, 
-								&first_entry)) {
-						assert((map != kernel_map) && 
-				       		(!entry->is_sub_map));
+								 &first_entry)) {
+						assert(map != kernel_map);
 						/*
-				 		 * User: use the next entry
-				 		 */
+						 * User: use the next entry
+						 */
 						entry = first_entry->vme_next;
+						s = entry->vme_start;
 					} else {
 						entry = first_entry;
 						SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
@@ -3851,17 +4550,43 @@ vm_map_delete(
 			 * copy current entry.  see comment in vm_map_wire()
 			 */
 			tmp_entry = *entry;
-			s = entry->vme_start;
-			e = entry->vme_end;
+			assert(s == entry->vme_start);
 
 			/*
 			 * We can unlock the map now. The in_transition
 			 * state guarentees existance of the entry.
 			 */
 			vm_map_unlock(map);
-			vm_fault_unwire(map, &tmp_entry,
-				tmp_entry.object.vm_object == kernel_object,
-				map->pmap, tmp_entry.vme_start);
+
+			if (tmp_entry.is_sub_map) {
+				vm_map_t sub_map;
+				vm_map_offset_t sub_start, sub_end;
+				pmap_t pmap;
+				vm_map_offset_t pmap_addr;
+				
+
+				sub_map = tmp_entry.object.sub_map;
+				sub_start = tmp_entry.offset;
+				sub_end = sub_start + (tmp_entry.vme_end -
+						       tmp_entry.vme_start);
+				if (tmp_entry.use_pmap) {
+					pmap = sub_map->pmap;
+					pmap_addr = tmp_entry.vme_start;
+				} else {
+					pmap = map->pmap;
+					pmap_addr = tmp_entry.vme_start;
+				}
+				(void) vm_map_unwire_nested(sub_map,
+							    sub_start, sub_end,
+							    user_wire,
+							    pmap, pmap_addr);
+			} else {
+
+				vm_fault_unwire(map, &tmp_entry,
+						tmp_entry.object.vm_object == kernel_object,
+						map->pmap, tmp_entry.vme_start);
+			}
+
 			vm_map_lock(map);
 
 			if (last_timestamp+1 != map->timestamp) {
@@ -3871,8 +4596,9 @@ vm_map_delete(
 				 */
 				if (!vm_map_lookup_entry(map, s, &first_entry)){
 					assert((map != kernel_map) && 
-				       	       (!entry->is_sub_map));
+					       (!entry->is_sub_map));
 					first_entry = first_entry->vme_next;
+					s = first_entry->vme_start;
 				} else {
 					SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
 				}
@@ -3906,16 +4632,27 @@ vm_map_delete(
 		assert(entry->wired_count == 0);
 		assert(entry->user_wired_count == 0);
 
-		if ((!entry->is_sub_map &&
-		    entry->object.vm_object != kernel_object) ||
-		    entry->is_sub_map) {
-			if(entry->is_sub_map) {
-			   if(entry->use_pmap) {
+		assert(s == entry->vme_start);
+
+		if (flags & VM_MAP_REMOVE_NO_PMAP_CLEANUP) {
+			/*
+			 * XXX with the VM_MAP_REMOVE_SAVE_ENTRIES flag to
+			 * vm_map_delete(), some map entries might have been
+			 * transferred to a "zap_map", which doesn't have a
+			 * pmap.  The original pmap has already been flushed
+			 * in the vm_map_delete() call targeting the original
+			 * map, but when we get to destroying the "zap_map",
+			 * we don't have any pmap to flush, so let's just skip
+			 * all this.
+			 */
+		} else if (entry->is_sub_map) {
+			if (entry->use_pmap) {
 #ifndef NO_NESTED_PMAP
 				pmap_unnest(map->pmap,
-					    (addr64_t)entry->vme_start);
+					    (addr64_t)entry->vme_start,
+					    entry->vme_end - entry->vme_start);
 #endif	/* NO_NESTED_PMAP */
-				if((map->mapped) && (map->ref_count)) {
+				if ((map->mapped) && (map->ref_count)) {
 					/* clean up parent map/maps */
 					vm_map_submap_pmap_clean(
 						map, entry->vme_start,
@@ -3923,26 +4660,25 @@ vm_map_delete(
 						entry->object.sub_map,
 						entry->offset);
 				}
-			   } else {
+			} else {
 				vm_map_submap_pmap_clean(
 					map, entry->vme_start, entry->vme_end,
 					entry->object.sub_map,
 					entry->offset);
-			   }
-			} else {
-			   object = entry->object.vm_object;
-			   if((map->mapped) && (map->ref_count)) {
-			      vm_object_pmap_protect(
+			}
+		} else if (entry->object.vm_object != kernel_object) {
+			object = entry->object.vm_object;
+			if((map->mapped) && (map->ref_count)) {
+				vm_object_pmap_protect(
 					object, entry->offset,
 					entry->vme_end - entry->vme_start,
 					PMAP_NULL,
 					entry->vme_start,
 					VM_PROT_NONE);
-			   } else {
-				   pmap_remove(map->pmap,
-					       (addr64_t)entry->vme_start,
-					       (addr64_t)entry->vme_end);
-			   }
+			} else {
+				pmap_remove(map->pmap,
+					    (addr64_t)entry->vme_start,
+					    (addr64_t)entry->vme_end);
 			}
 		}
 
@@ -3960,6 +4696,7 @@ vm_map_delete(
 
 		if ((flags & VM_MAP_REMOVE_SAVE_ENTRIES) &&
 		    zap_map != VM_MAP_NULL) {
+			vm_map_size_t entry_size;
 			/*
 			 * The caller wants to save the affected VM map entries
 			 * into the "zap_map".  The caller will take care of
@@ -3971,6 +4708,11 @@ vm_map_delete(
 			vm_map_entry_link(zap_map,
 					  vm_map_last_entry(zap_map),
 					  entry);
+			entry_size = entry->vme_end - entry->vme_start;
+			map->size -= entry_size;
+			zap_map->size += entry_size;
+			/* we didn't unlock the map, so no timestamp increase */
+			last_timestamp--;
 		} else {
 			vm_map_entry_delete(map, entry);
 			/* vm_map_entry_delete unlocks the map */
@@ -3993,8 +4735,9 @@ vm_map_delete(
 			 */
         		if (!vm_map_lookup_entry(map, s, &entry)){
 	               		entry = entry->vme_next;
+				s = entry->vme_start;
         		} else {
-				 SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
+				SAVE_HINT_MAP_WRITE(map, entry->vme_prev);
        		 	}
 			/* 
 			 * others can not only allocate behind us, we can 
@@ -4003,7 +4746,6 @@ vm_map_delete(
 			if(entry == vm_map_to_entry(map)) {
 				break;
 			}
-			vm_map_clip_start(map, entry, s);
 		}
 		last_timestamp = map->timestamp;
 	}
@@ -4019,7 +4761,6 @@ vm_map_delete(
 	return KERN_SUCCESS;
 }
 
-
 /*
  *	vm_map_remove:
  *
@@ -4065,7 +4806,7 @@ vm_map_copy_discard(
 	switch (copy->type) {
 	case VM_MAP_COPY_ENTRY_LIST:
 		while (vm_map_copy_first_entry(copy) !=
-					vm_map_copy_to_entry(copy)) {
+		       vm_map_copy_to_entry(copy)) {
 			vm_map_entry_t	entry = vm_map_copy_first_entry(copy);
 
 			vm_map_copy_entry_unlink(copy, entry);
@@ -4178,6 +4919,7 @@ start_pass_1:
 	}
 
 	vm_map_clip_start(dst_map, tmp_entry, vm_map_trunc_page(dst_addr));
+	assert(!tmp_entry->use_pmap); /* clipping did unnest if needed */
 
 	for (entry = tmp_entry;;) {
 		vm_map_entry_t	next;
@@ -4189,9 +4931,9 @@ start_pass_1:
 			vm_map_offset_t	local_end;
 
 			if (entry->in_transition) {
-                        /*
-                         * Say that we are waiting, and wait for entry.
-                         */
+				/*
+				 * Say that we are waiting, and wait for entry.
+				 */
                         	entry->needs_wakeup = TRUE;
                         	vm_map_entry_wait(dst_map, THREAD_UNINT);
 
@@ -4211,9 +4953,9 @@ start_pass_1:
 			vm_map_unlock(dst_map);
 			
 			result = vm_map_overwrite_submap_recurse(
-					entry->object.sub_map,
-					sub_start,
-					sub_end - sub_start);
+				entry->object.sub_map,
+				sub_start,
+				sub_end - sub_start);
 
 			if(result != KERN_SUCCESS)
 				return result;
@@ -4270,8 +5012,8 @@ start_pass_1:
 		 *	Check for permanent objects in the destination.
 		 */
 		if ((entry->object.vm_object != VM_OBJECT_NULL) &&
-			   ((!entry->object.vm_object->internal) ||
-			   (entry->object.vm_object->true_share))) {
+		    ((!entry->object.vm_object->internal) ||
+		     (entry->object.vm_object->true_share))) {
 			if(encountered_sub_map) {
 				vm_map_unlock(dst_map);
 				return(KERN_FAILURE);
@@ -4370,8 +5112,8 @@ vm_map_copy_overwrite_nested(
 
 	if (copy->type == VM_MAP_COPY_KERNEL_BUFFER) {
 		return(vm_map_copyout_kernel_buffer(
-						dst_map, &dst_addr, 
-					    	copy, TRUE));
+			       dst_map, &dst_addr, 
+			       copy, TRUE));
 	}
 
 	/*
@@ -4394,8 +5136,8 @@ vm_map_copy_overwrite_nested(
 	 */
 
 	if (!page_aligned(copy->size) ||
-		!page_aligned (copy->offset) ||
-		!page_aligned (dst_addr))
+	    !page_aligned (copy->offset) ||
+	    !page_aligned (dst_addr))
 	{
 		aligned = FALSE;
 		dst_end = vm_map_round_page(dst_addr + copy->size);
@@ -4430,9 +5172,9 @@ start_pass_1:
 
                 	if (entry->in_transition) {
 
-                        /*
-                         * Say that we are waiting, and wait for entry.
-                         */
+				/*
+				 * Say that we are waiting, and wait for entry.
+				 */
                         	entry->needs_wakeup = TRUE;
                         	vm_map_entry_wait(dst_map, THREAD_UNINT);
 
@@ -4515,8 +5257,8 @@ start_pass_1:
 		 *	Check for permanent objects in the destination.
 		 */
 		if ((entry->object.vm_object != VM_OBJECT_NULL) &&
-			   ((!entry->object.vm_object->internal) ||
-			   (entry->object.vm_object->true_share))) {
+		    ((!entry->object.vm_object->internal) ||
+		     (entry->object.vm_object->true_share))) {
 			contains_permanent_objects = TRUE;
 		}
 
@@ -4589,7 +5331,7 @@ start_overwrite:
                        		vm_map_entry_wait(dst_map, THREAD_UNINT);
 
 				if(!vm_map_lookup_entry(dst_map, base_addr, 
-								&tmp_entry)) {
+							&tmp_entry)) {
 					vm_map_unlock(dst_map);
 					return(KERN_INVALID_ADDRESS);
 				}
@@ -4618,6 +5360,7 @@ start_overwrite:
 						dst_map, entry, sub_end);
 					vm_map_clip_start(
 						dst_map, entry, sub_start);
+					assert(!entry->use_pmap);
 					entry->is_sub_map = FALSE;
 					vm_map_deallocate(
 						entry->object.sub_map);
@@ -4625,13 +5368,21 @@ start_overwrite:
 					entry->is_shared = FALSE;
 					entry->needs_copy = FALSE;
 					entry->offset = 0;
+					/*
+					 * XXX FBDP
+					 * We should propagate the protections
+					 * of the submap entry here instead
+					 * of forcing them to VM_PROT_ALL...
+					 * Or better yet, we should inherit
+					 * the protection of the copy_entry.
+					 */
 					entry->protection = VM_PROT_ALL;
 					entry->max_protection = VM_PROT_ALL;
 					entry->wired_count = 0;
 					entry->user_wired_count = 0;
 					if(entry->inheritance 
-							== VM_INHERIT_SHARE) 
-					   entry->inheritance = VM_INHERIT_COPY;
+					   == VM_INHERIT_SHARE) 
+						entry->inheritance = VM_INHERIT_COPY;
 					continue;
 				}
 				/* first take care of any non-sub_map */
@@ -4659,45 +5410,45 @@ start_overwrite:
 					vm_map_size_t	local_size = 0;
 					vm_map_size_t	entry_size;
 
-				   nentries = 1;
-				   new_offset = copy->offset;
-				   copy_entry = vm_map_copy_first_entry(copy);
-				   while(copy_entry != 
-					         vm_map_copy_to_entry(copy)){
-				       entry_size = copy_entry->vme_end - 
-						      copy_entry->vme_start;
-				       if((local_size < copy_size) &&
-				       		((local_size + entry_size) 
-					         >= copy_size)) {
-				          vm_map_copy_clip_end(copy, 
-					         copy_entry, 
-					         copy_entry->vme_start +
-					         (copy_size - local_size));
-				          entry_size = copy_entry->vme_end - 
-				  		         copy_entry->vme_start;
-					  local_size += entry_size;
-					  new_offset += entry_size;
-				       }
-				       if(local_size >= copy_size) {
-				          next_copy = copy_entry->vme_next;
-					  copy_entry->vme_next = 
-					            vm_map_copy_to_entry(copy);
-				          previous_prev = 
-						   copy->cpy_hdr.links.prev;
-					  copy->cpy_hdr.links.prev = copy_entry;
-				          copy->size = copy_size;
-					  remaining_entries = 
-						        copy->cpy_hdr.nentries;
-					  remaining_entries -= nentries;
-					  copy->cpy_hdr.nentries = nentries;
-				          break;
-				       } else {
-				          local_size += entry_size;
-					  new_offset += entry_size;
-					  nentries++;
-				       }
-				       copy_entry = copy_entry->vme_next;
-				   }
+					nentries = 1;
+					new_offset = copy->offset;
+					copy_entry = vm_map_copy_first_entry(copy);
+					while(copy_entry != 
+					      vm_map_copy_to_entry(copy)){
+						entry_size = copy_entry->vme_end - 
+							copy_entry->vme_start;
+						if((local_size < copy_size) &&
+						   ((local_size + entry_size) 
+						    >= copy_size)) {
+							vm_map_copy_clip_end(copy, 
+									     copy_entry, 
+									     copy_entry->vme_start +
+									     (copy_size - local_size));
+							entry_size = copy_entry->vme_end - 
+								copy_entry->vme_start;
+							local_size += entry_size;
+							new_offset += entry_size;
+						}
+						if(local_size >= copy_size) {
+							next_copy = copy_entry->vme_next;
+							copy_entry->vme_next = 
+								vm_map_copy_to_entry(copy);
+							previous_prev = 
+								copy->cpy_hdr.links.prev;
+							copy->cpy_hdr.links.prev = copy_entry;
+							copy->size = copy_size;
+							remaining_entries = 
+								copy->cpy_hdr.nentries;
+							remaining_entries -= nentries;
+							copy->cpy_hdr.nentries = nentries;
+							break;
+						} else {
+							local_size += entry_size;
+							new_offset += entry_size;
+							nentries++;
+						}
+						copy_entry = copy_entry->vme_next;
+					}
 				}
 			
 				if((entry->use_pmap) && (pmap == NULL)) {
@@ -4723,13 +5474,13 @@ start_overwrite:
 				}
 				if(kr != KERN_SUCCESS) {
 					if(next_copy != NULL) {
-					   copy->cpy_hdr.nentries += 
-							   remaining_entries;
-				           copy->cpy_hdr.links.prev->vme_next = 
-							   next_copy;
-				           copy->cpy_hdr.links.prev 
-							   = previous_prev;
-					   copy->size = total_size;
+						copy->cpy_hdr.nentries += 
+							remaining_entries;
+						copy->cpy_hdr.links.prev->vme_next = 
+							next_copy;
+						copy->cpy_hdr.links.prev 
+							= previous_prev;
+						copy->size = total_size;
 					}
 					return kr;
 				}
@@ -4739,10 +5490,10 @@ start_overwrite:
 				/* otherwise copy no longer exists, it was */
 				/* destroyed after successful copy_overwrite */
 			        copy = (vm_map_copy_t) 
-						zalloc(vm_map_copy_zone);
+					zalloc(vm_map_copy_zone);
 				vm_map_copy_first_entry(copy) =
-				   vm_map_copy_last_entry(copy) =
-			           vm_map_copy_to_entry(copy);
+					vm_map_copy_last_entry(copy) =
+					vm_map_copy_to_entry(copy);
 				copy->type = VM_MAP_COPY_ENTRY_LIST;
 				copy->offset = new_offset;
 
@@ -4750,18 +5501,18 @@ start_overwrite:
 				copy_size = 0;
 				/* put back remainder of copy in container */
 				if(next_copy != NULL) {
-				   copy->cpy_hdr.nentries = remaining_entries;
-				   copy->cpy_hdr.links.next = next_copy;
-			           copy->cpy_hdr.links.prev = previous_prev;
-				   copy->size = total_size;
-				   next_copy->vme_prev = 
-					         vm_map_copy_to_entry(copy);
-				   next_copy = NULL;
+					copy->cpy_hdr.nentries = remaining_entries;
+					copy->cpy_hdr.links.next = next_copy;
+					copy->cpy_hdr.links.prev = previous_prev;
+					copy->size = total_size;
+					next_copy->vme_prev = 
+						vm_map_copy_to_entry(copy);
+					next_copy = NULL;
 				}
 				base_addr = local_end;
 				vm_map_lock(dst_map);
 				if(!vm_map_lookup_entry(dst_map, 
-						local_end, &tmp_entry)) {
+							local_end, &tmp_entry)) {
 					vm_map_unlock(dst_map);
 					return(KERN_INVALID_ADDRESS);
 				}
@@ -4774,7 +5525,7 @@ start_overwrite:
 			}
 
 			if ((next == vm_map_to_entry(dst_map)) ||
-				    (next->vme_start != entry->vme_end)) {
+			    (next->vme_start != entry->vme_end)) {
 				vm_map_unlock(dst_map);
 				return(KERN_INVALID_ADDRESS);
 			}
@@ -4794,15 +5545,15 @@ start_overwrite:
 			copy_entry = vm_map_copy_first_entry(copy);
 			while(copy_entry != vm_map_copy_to_entry(copy)) {
 				entry_size = copy_entry->vme_end - 
-						copy_entry->vme_start;
+					copy_entry->vme_start;
 				if((local_size < copy_size) &&
-						((local_size + entry_size) 
-						>= copy_size)) {
+				   ((local_size + entry_size) 
+				    >= copy_size)) {
 					vm_map_copy_clip_end(copy, copy_entry, 
-						copy_entry->vme_start +
-						(copy_size - local_size));
+							     copy_entry->vme_start +
+							     (copy_size - local_size));
 					entry_size = copy_entry->vme_end - 
-				  		   copy_entry->vme_start;
+						copy_entry->vme_start;
 					local_size += entry_size;
 					new_offset += entry_size;
 				}
@@ -4837,39 +5588,39 @@ start_overwrite:
 				local_pmap = dst_map->pmap;
 
 			if ((kr =  vm_map_copy_overwrite_aligned( 
-				dst_map, tmp_entry, copy,
-				base_addr, local_pmap)) != KERN_SUCCESS) {
+				     dst_map, tmp_entry, copy,
+				     base_addr, local_pmap)) != KERN_SUCCESS) {
 				if(next_copy != NULL) {
 					copy->cpy_hdr.nentries += 
-							   remaining_entries;
+						remaining_entries;
 				        copy->cpy_hdr.links.prev->vme_next = 
-							   next_copy;
+						next_copy;
 			       		copy->cpy_hdr.links.prev = 
-							previous_prev;
+						previous_prev;
 					copy->size += copy_size;
 				}
 				return kr;
 			}
 			vm_map_unlock(dst_map);
 		} else {
-		/*
-		 * Performance gain:
-		 *
-		 * if the copy and dst address are misaligned but the same
-		 * offset within the page we can copy_not_aligned the
-		 * misaligned parts and copy aligned the rest.  If they are
-		 * aligned but len is unaligned we simply need to copy
-		 * the end bit unaligned.  We'll need to split the misaligned
-		 * bits of the region in this case !
-		 */
-		/* ALWAYS UNLOCKS THE dst_map MAP */
+			/*
+			 * Performance gain:
+			 *
+			 * if the copy and dst address are misaligned but the same
+			 * offset within the page we can copy_not_aligned the
+			 * misaligned parts and copy aligned the rest.  If they are
+			 * aligned but len is unaligned we simply need to copy
+			 * the end bit unaligned.  We'll need to split the misaligned
+			 * bits of the region in this case !
+			 */
+			/* ALWAYS UNLOCKS THE dst_map MAP */
 			if ((kr =  vm_map_copy_overwrite_unaligned( dst_map,
-				tmp_entry, copy, base_addr)) != KERN_SUCCESS) {
+								    tmp_entry, copy, base_addr)) != KERN_SUCCESS) {
 				if(next_copy != NULL) {
 					copy->cpy_hdr.nentries +=
-							     remaining_entries;
+						remaining_entries;
 			       		copy->cpy_hdr.links.prev->vme_next = 
-							     next_copy;
+						next_copy;
 			       		copy->cpy_hdr.links.prev = 
 						previous_prev;
 					copy->size += copy_size;
@@ -4893,7 +5644,7 @@ start_overwrite:
 		vm_map_lock(dst_map);
 		while(TRUE) {
 			if (!vm_map_lookup_entry(dst_map, 
-						base_addr, &tmp_entry)) {
+						 base_addr, &tmp_entry)) {
 				vm_map_unlock(dst_map);
 				return(KERN_INVALID_ADDRESS);
 			}
@@ -4925,7 +5676,7 @@ vm_map_copy_overwrite(
 	boolean_t	interruptible)
 {
 	return vm_map_copy_overwrite_nested(
-			dst_map, dst_addr, copy, interruptible, (pmap_t) NULL);
+		dst_map, dst_addr, copy, interruptible, (pmap_t) NULL);
 }
 
 
@@ -5019,16 +5770,16 @@ vm_map_copy_overwrite_unaligned(
  *		Copy on write region.
  */
 		if (entry->needs_copy &&
-			 ((entry->protection & VM_PROT_WRITE) != 0))
+		    ((entry->protection & VM_PROT_WRITE) != 0))
 		{
 			if (vm_map_lock_read_to_write(dst_map)) {
 				vm_map_lock_read(dst_map);
 				goto RetryLookup;
 			}
 			vm_object_shadow(&entry->object.vm_object,
-					&entry->offset,
-					(vm_map_size_t)(entry->vme_end
-						- entry->vme_start));
+					 &entry->offset,
+					 (vm_map_size_t)(entry->vme_end
+							 - entry->vme_start));
 			entry->needs_copy = FALSE;
 			vm_map_lock_write_to_read(dst_map);
 		}
@@ -5043,7 +5794,7 @@ vm_map_copy_overwrite_unaligned(
 				goto RetryLookup;
 			}
 			dst_object = vm_object_allocate((vm_map_size_t)
-					entry->vme_end - entry->vme_start);
+							entry->vme_end - entry->vme_start);
 			entry->object.vm_object = dst_object;
 			entry->offset = 0;
 			vm_map_lock_write_to_read(dst_map);
@@ -5084,7 +5835,7 @@ vm_map_copy_overwrite_unaligned(
 			return kr;
 
 		if ((copy_entry->vme_start + src_offset) == copy_entry->vme_end
-			|| amount_left == 0)
+		    || amount_left == 0)
 		{
 /*
  *			all done with this copy entry, dispose.
@@ -5094,7 +5845,7 @@ vm_map_copy_overwrite_unaligned(
 			vm_map_copy_entry_dispose(copy, copy_entry);
 
 			if ((copy_entry = vm_map_copy_first_entry(copy))
-				== vm_map_copy_to_entry(copy) && amount_left) {
+			    == vm_map_copy_to_entry(copy) && amount_left) {
 /*
  *				not finished copying but run out of source
  */
@@ -5130,7 +5881,7 @@ vm_map_copy_overwrite_unaligned(
  *			we must lookup the entry because somebody
  *			might have changed the map behind our backs.
  */
-RetryLookup:
+		RetryLookup:
 			if (!vm_map_lookup_entry(dst_map, start, &entry))
 			{
 				vm_map_unlock_read(dst_map);
@@ -5167,10 +5918,7 @@ vm_map_copy_overwrite_aligned(
 	vm_map_entry_t	tmp_entry,
 	vm_map_copy_t	copy,
 	vm_map_offset_t	start,
-#if !BAD_OPTIMIZATION
-	__unused
-#endif /* !BAD_OPTIMIZATION */
-	pmap_t		pmap)
+	__unused pmap_t	pmap)
 {
 	vm_object_t	object;
 	vm_map_entry_t	copy_entry;
@@ -5179,11 +5927,12 @@ vm_map_copy_overwrite_aligned(
 	vm_map_entry_t	entry;
 		
 	while ((copy_entry = vm_map_copy_first_entry(copy))
-		!= vm_map_copy_to_entry(copy))
+	       != vm_map_copy_to_entry(copy))
 	{
 		copy_size = (copy_entry->vme_end - copy_entry->vme_start);
 		
 		entry = tmp_entry;
+		assert(!entry->use_pmap); /* unnested when clipped earlier */
 		if (entry == vm_map_to_entry(dst_map)) {
 			vm_map_unlock(dst_map);
 			return KERN_INVALID_ADDRESS;
@@ -5197,7 +5946,7 @@ vm_map_copy_overwrite_aligned(
 		 */
 
 		if ((entry->vme_start != start) || ((entry->is_sub_map)
-				&& !entry->needs_copy)) {
+						    && !entry->needs_copy)) {
 			vm_map_unlock(dst_map);
 			return(KERN_INVALID_ADDRESS);
 		}
@@ -5227,7 +5976,7 @@ vm_map_copy_overwrite_aligned(
 
 		if (size < copy_size) {
 			vm_map_copy_clip_end(copy, copy_entry,
-				copy_entry->vme_start + size);
+					     copy_entry->vme_start + size);
 			copy_size = size;
 		}
 
@@ -5243,8 +5992,8 @@ vm_map_copy_overwrite_aligned(
 
 		object = entry->object.vm_object;
 		if ((!entry->is_shared && 
-		    ((object == VM_OBJECT_NULL) || 
-		    (object->internal && !object->true_share))) ||
+		     ((object == VM_OBJECT_NULL) || 
+		      (object->internal && !object->true_share))) ||
 		    entry->needs_copy) {
 			vm_object_t	old_object = entry->object.vm_object;
 			vm_object_offset_t	old_offset = entry->offset;
@@ -5272,16 +6021,17 @@ vm_map_copy_overwrite_aligned(
 					if(entry->use_pmap) {
 #ifndef NO_NESTED_PMAP
 						pmap_unnest(dst_map->pmap, 
-							    (addr64_t)entry->vme_start);
+							    (addr64_t)entry->vme_start,
+							    entry->vme_end - entry->vme_start);
 #endif	/* NO_NESTED_PMAP */
 						if(dst_map->mapped) {
 							/* clean up parent */
 							/* map/maps */
-						   vm_map_submap_pmap_clean(
-							dst_map, entry->vme_start,
-							entry->vme_end,
-							entry->object.sub_map,
-							entry->offset);
+							vm_map_submap_pmap_clean(
+								dst_map, entry->vme_start,
+								entry->vme_end,
+								entry->object.sub_map,
+								entry->offset);
 						}
 					} else {
 						vm_map_submap_pmap_clean(
@@ -5298,14 +6048,14 @@ vm_map_copy_overwrite_aligned(
 							entry->object.vm_object,
 							entry->offset,
 							entry->vme_end 
-								- entry->vme_start,
+							- entry->vme_start,
 							PMAP_NULL,
 							entry->vme_start,
 							VM_PROT_NONE);
 					} else {
-					   pmap_remove(dst_map->pmap, 
-					     (addr64_t)(entry->vme_start), 
-					     (addr64_t)(entry->vme_end));
+						pmap_remove(dst_map->pmap, 
+							    (addr64_t)(entry->vme_start), 
+							    (addr64_t)(entry->vme_end));
 					}
 					vm_object_deallocate(old_object);
 			   	}
@@ -5318,92 +6068,28 @@ vm_map_copy_overwrite_aligned(
 			entry->wired_count = 0;
 			entry->user_wired_count = 0;
 			offset = entry->offset = copy_entry->offset;
+			/*
+			 * XXX FBDP
+			 * We should propagate the submap entry's protections
+			 * here instead of forcing VM_PROT_ALL.
+			 * Or better yet, we should inherit the protection
+			 * of the copy_entry.
+			 */
+			entry->protection = VM_PROT_ALL;
+			entry->max_protection = VM_PROT_ALL;
 
 			vm_map_copy_entry_unlink(copy, copy_entry);
 			vm_map_copy_entry_dispose(copy, copy_entry);
-#if BAD_OPTIMIZATION
+
 			/*
-			 * if we turn this optimization back on
-			 * we need to revisit our use of pmap mappings
-			 * large copies will cause us to run out and panic
+			 * we could try to push pages into the pmap at this point, BUT
 			 * this optimization only saved on average 2 us per page if ALL
 			 * the pages in the source were currently mapped
 			 * and ALL the pages in the dest were touched, if there were fewer
 			 * than 2/3 of the pages touched, this optimization actually cost more cycles
+			 * it also puts a lot of pressure on the pmap layer w/r to mapping structures
 			 */
 
-			/*
-			 * Try to aggressively enter physical mappings
-			 * (but avoid uninstantiated objects)
-			 */
-			if (object != VM_OBJECT_NULL) {
-			    vm_map_offset_t	va = entry->vme_start;
-
-			    while (va < entry->vme_end) {
-				register vm_page_t	m;
-				vm_prot_t		prot;
-
-				/*
-				 * Look for the page in the top object
-				 */
-				prot = entry->protection;
-				vm_object_lock(object);
-				vm_object_paging_begin(object);
-
-				/*
-				 * ENCRYPTED SWAP:
-				 * If the page is encrypted, skip it:
-				 * we can't let the user see the encrypted
-				 * contents.  The page will get decrypted
-				 * on demand when the user generates a
-				 * soft-fault when trying to access it.
-				 */
-				if ((m = vm_page_lookup(object,offset)) !=
-				    VM_PAGE_NULL && !m->busy &&
-				    !m->fictitious && !m->encrypted &&
-				    (!m->unusual || (!m->error &&
-					!m->restart && !m->absent &&
-					 (prot & m->page_lock) == 0))) {
-					
-					m->busy = TRUE;
-					vm_object_unlock(object);
-					
-					/* 
-					 * Honor COW obligations
-					 */
-					if (entry->needs_copy)
-						prot &= ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-					if (entry->alias != VM_MEMORY_STACK && prot)
-					        prot |= VM_PROT_EXECUTE;
-#endif
-					/* It is our policy to require */
-					/* explicit sync from anyone   */
-					/* writing code and then       */
-					/* a pc to execute it.         */
-					/* No isync here */
-
-					PMAP_ENTER(pmap, va, m, prot, 
-						((unsigned int)
-							(m->object->wimg_bits)) 
-							& VM_WIMG_MASK,
-						FALSE);
-		
-					vm_object_lock(object);
-					vm_page_lock_queues();
-					if (!m->active && !m->inactive)
-						vm_page_activate(m);
-					vm_page_unlock_queues();
-					PAGE_WAKEUP_DONE(m);
-				}
-				vm_object_paging_end(object);
-				vm_object_unlock(object);
-
-				offset += PAGE_SIZE_64;
-				va += PAGE_SIZE;
-			    } /* end while (va < entry->vme_end) */
-			} /* end if (object) */
-#endif
 			/*
 			 *	Set up for the next iteration.  The map
 			 *	has not been unlocked, so the next
@@ -5439,14 +6125,14 @@ vm_map_copy_overwrite_aligned(
 
 			copy_size = size;
 			r = vm_fault_copy(
-					copy_entry->object.vm_object,
-					copy_entry->offset,
-					&copy_size,
-					dst_object,
-					dst_offset,
-					dst_map,
-					&version,
-					THREAD_UNINT );
+				copy_entry->object.vm_object,
+				copy_entry->offset,
+				&copy_size,
+				dst_object,
+				dst_offset,
+				dst_map,
+				&version,
+				THREAD_UNINT );
 
 			/*
 			 *	Release the object reference
@@ -5467,7 +6153,7 @@ vm_map_copy_overwrite_aligned(
 				 */
 
 				vm_map_copy_clip_end(copy, copy_entry,
-					copy_entry->vme_start + copy_size);
+						     copy_entry->vme_start + copy_size);
 				vm_map_copy_entry_unlink(copy, copy_entry);
 				vm_object_deallocate(copy_entry->object.vm_object);
 				vm_map_copy_entry_dispose(copy, copy_entry);
@@ -5541,11 +6227,11 @@ vm_map_copyin_kernel_buffer(
 	}
 	if (src_destroy) {
 		(void) vm_map_remove(src_map, vm_map_trunc_page(src_addr), 
-				vm_map_round_page(src_addr + len),
-				VM_MAP_REMOVE_INTERRUPTIBLE |
-				VM_MAP_REMOVE_WAIT_FOR_KWIRE |
-				(src_map == kernel_map) ?
-					VM_MAP_REMOVE_KUNWIRE : 0);
+				     vm_map_round_page(src_addr + len),
+				     VM_MAP_REMOVE_INTERRUPTIBLE |
+				     VM_MAP_REMOVE_WAIT_FOR_KWIRE |
+				     (src_map == kernel_map) ?
+				     VM_MAP_REMOVE_KUNWIRE : 0);
 	}
 	*copy_result = copy;
 	return KERN_SUCCESS;
@@ -5721,7 +6407,7 @@ vm_map_copyout(
 
 		offset = vm_object_trunc_page(copy->offset);
 		size = vm_map_round_page(copy->size + 
-				(vm_map_size_t)(copy->offset - offset));
+					 (vm_map_size_t)(copy->offset - offset));
 		*dst_addr = 0;
 		kr = vm_map_enter(dst_map, dst_addr, size,
 				  (vm_map_offset_t) 0, VM_FLAGS_ANYWHERE,
@@ -5752,9 +6438,9 @@ vm_map_copyout(
 
 	vm_copy_start = vm_object_trunc_page(copy->offset);
 	size =	vm_map_round_page((vm_map_size_t)copy->offset + copy->size) 
-							- vm_copy_start;
+		- vm_copy_start;
 
- StartAgain: ;
+StartAgain: ;
 
 	vm_map_lock(dst_map);
 	assert(first_free_is_valid(dst_map));
@@ -5794,45 +6480,45 @@ vm_map_copyout(
 	 */
 
 	if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {
-	    /*
-	     * Mismatches occur when dealing with the default
-	     * pager.
-	     */
-	    zone_t		old_zone;
-	    vm_map_entry_t	next, new;
-
-	    /*
-	     * Find the zone that the copies were allocated from
-	     */
-	    old_zone = (copy->cpy_hdr.entries_pageable)
+		/*
+		 * Mismatches occur when dealing with the default
+		 * pager.
+		 */
+		zone_t		old_zone;
+		vm_map_entry_t	next, new;
+
+		/*
+		 * Find the zone that the copies were allocated from
+		 */
+		old_zone = (copy->cpy_hdr.entries_pageable)
 			? vm_map_entry_zone
 			: vm_map_kentry_zone;
-	    entry = vm_map_copy_first_entry(copy);
-
-	    /*
-	     * Reinitialize the copy so that vm_map_copy_entry_link
-	     * will work.
-	     */
-	    copy->cpy_hdr.nentries = 0;
-	    copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
-	    vm_map_copy_first_entry(copy) =
-	     vm_map_copy_last_entry(copy) =
-		vm_map_copy_to_entry(copy);
-
-	    /*
-	     * Copy each entry.
-	     */
-	    while (entry != vm_map_copy_to_entry(copy)) {
-		new = vm_map_copy_entry_create(copy);
-		vm_map_entry_copy_full(new, entry);
-		new->use_pmap = FALSE;	/* clr address space specifics */
-		vm_map_copy_entry_link(copy,
-				vm_map_copy_last_entry(copy),
-				new);
-		next = entry->vme_next;
-		zfree(old_zone, entry);
-		entry = next;
-	    }
+		entry = vm_map_copy_first_entry(copy);
+
+		/*
+		 * Reinitialize the copy so that vm_map_copy_entry_link
+		 * will work.
+		 */
+		copy->cpy_hdr.nentries = 0;
+		copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;
+		vm_map_copy_first_entry(copy) =
+			vm_map_copy_last_entry(copy) =
+			vm_map_copy_to_entry(copy);
+
+		/*
+		 * Copy each entry.
+		 */
+		while (entry != vm_map_copy_to_entry(copy)) {
+			new = vm_map_copy_entry_create(copy);
+			vm_map_entry_copy_full(new, entry);
+			new->use_pmap = FALSE;	/* clr address space specifics */
+			vm_map_copy_entry_link(copy,
+					       vm_map_copy_last_entry(copy),
+					       new);
+			next = entry->vme_next;
+			zfree(old_zone, entry);
+			entry = next;
+		}
 	}
 
 	/*
@@ -5857,150 +6543,73 @@ vm_map_copyout(
 		 * map the pages into the destination map.
 		 */
 		if (entry->wired_count != 0) {
-		    register vm_map_offset_t va;
-		    vm_object_offset_t	 offset;
-		    register vm_object_t object;
-		    vm_prot_t prot;
-
-		    object = entry->object.vm_object;
-		    offset = entry->offset;
-		    va = entry->vme_start;
-
-		    pmap_pageable(dst_map->pmap,
-				  entry->vme_start,
-				  entry->vme_end,
-				  TRUE);
+			register vm_map_offset_t va;
+			vm_object_offset_t	 offset;
+			register vm_object_t object;
+			vm_prot_t prot;
+			int	type_of_fault;
 
-		    while (va < entry->vme_end) {
-			register vm_page_t	m;
+			object = entry->object.vm_object;
+			offset = entry->offset;
+			va = entry->vme_start;
 
-			/*
-			 * Look up the page in the object.
-			 * Assert that the page will be found in the
-			 * top object:
-			 * either
-			 *	the object was newly created by
-			 *	vm_object_copy_slowly, and has
-			 *	copies of all of the pages from
-			 *	the source object
-			 * or
-			 *	the object was moved from the old
-			 *	map entry; because the old map
-			 *	entry was wired, all of the pages
-			 *	were in the top-level object.
-			 *	(XXX not true if we wire pages for
-			 *	 reading)
-			 */
-			vm_object_lock(object);
-			vm_object_paging_begin(object);
+			pmap_pageable(dst_map->pmap,
+				      entry->vme_start,
+				      entry->vme_end,
+				      TRUE);
 
-			m = vm_page_lookup(object, offset);
-			if (m == VM_PAGE_NULL || m->wire_count == 0 ||
-			    m->absent)
-			    panic("vm_map_copyout: wiring 0x%x", m);
+			while (va < entry->vme_end) {
+				register vm_page_t	m;
 
-			/*
-			 * ENCRYPTED SWAP:
-			 * The page is assumed to be wired here, so it
-			 * shouldn't be encrypted.  Otherwise, we
-			 * couldn't enter it in the page table, since
-			 * we don't want the user to see the encrypted
-			 * data.
-			 */
-			ASSERT_PAGE_DECRYPTED(m);
+				/*
+				 * Look up the page in the object.
+				 * Assert that the page will be found in the
+				 * top object:
+				 * either
+				 *	the object was newly created by
+				 *	vm_object_copy_slowly, and has
+				 *	copies of all of the pages from
+				 *	the source object
+				 * or
+				 *	the object was moved from the old
+				 *	map entry; because the old map
+				 *	entry was wired, all of the pages
+				 *	were in the top-level object.
+				 *	(XXX not true if we wire pages for
+				 *	 reading)
+				 */
+				vm_object_lock(object);
 
-			m->busy = TRUE;
-			vm_object_unlock(object);
-			prot = entry->protection;
-#ifdef STACK_ONLY_NX
-			if (entry->alias != VM_MEMORY_STACK && prot)
-			        prot |= VM_PROT_EXECUTE;
-#endif
-			PMAP_ENTER(dst_map->pmap, va, m, prot, 
-						((unsigned int)
-							(m->object->wimg_bits)) 
-							& VM_WIMG_MASK,
-						TRUE);
+				m = vm_page_lookup(object, offset);
+				if (m == VM_PAGE_NULL || m->wire_count == 0 ||
+				    m->absent)
+					panic("vm_map_copyout: wiring %p", m);
 
-			vm_object_lock(object);
-			PAGE_WAKEUP_DONE(m);
-			/* the page is wired, so we don't have to activate */
-			vm_object_paging_end(object);
-			vm_object_unlock(object);
+				/*
+				 * ENCRYPTED SWAP:
+				 * The page is assumed to be wired here, so it
+				 * shouldn't be encrypted.  Otherwise, we
+				 * couldn't enter it in the page table, since
+				 * we don't want the user to see the encrypted
+				 * data.
+				 */
+				ASSERT_PAGE_DECRYPTED(m);
 
-			offset += PAGE_SIZE_64;
-			va += PAGE_SIZE;
-		    }
-		}
-		else if (size <= vm_map_aggressive_enter_max) {
+				prot = entry->protection;
 
-			register vm_map_offset_t	va;
-			vm_object_offset_t	offset;
-			register vm_object_t	object;
-			vm_prot_t		prot;
+				if (override_nx(dst_map, entry->alias) && prot)
+				        prot |= VM_PROT_EXECUTE;
 
-			object = entry->object.vm_object;
-			if (object != VM_OBJECT_NULL) {
+				type_of_fault = DBG_CACHE_HIT_FAULT;
 
-				offset = entry->offset;
-				va = entry->vme_start;
-				while (va < entry->vme_end) {
-					register vm_page_t	m;
-				    
-					/*
-					 * Look up the page in the object.
-					 * Assert that the page will be found
-					 * in the top object if at all...
-					 */
-					vm_object_lock(object);
-					vm_object_paging_begin(object);
+				vm_fault_enter(m, dst_map->pmap, va, prot,
+					       m->wire_count != 0, FALSE, FALSE,
+					       &type_of_fault);
 
-					/*
-					 * ENCRYPTED SWAP:
-					 * If the page is encrypted, skip it:
-					 * we can't let the user see the
-					 * encrypted contents.  The page will
-					 * get decrypted on demand when the
-					 * user generates a soft-fault when
-					 * trying to access it.
-					 */
-					if (((m = vm_page_lookup(object,
-								 offset))
-					     != VM_PAGE_NULL) &&
-					    !m->busy && !m->fictitious &&
-					    !m->encrypted &&
-					    !m->absent && !m->error) {
-						m->busy = TRUE;
-						vm_object_unlock(object);
-
-						/* honor cow obligations */
-						prot = entry->protection;
-						if (entry->needs_copy)
-							prot &= ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-						if (entry->alias != VM_MEMORY_STACK && prot)
-						        prot |= VM_PROT_EXECUTE;
-#endif
-						PMAP_ENTER(dst_map->pmap, va, 
-							m, prot, 
-							((unsigned int)
-						        (m->object->wimg_bits)) 
-							   & VM_WIMG_MASK,
-							FALSE);
-
-						vm_object_lock(object);
-						vm_page_lock_queues();
-						if (!m->active && !m->inactive)
-							vm_page_activate(m);
-						vm_page_unlock_queues();
-						PAGE_WAKEUP_DONE(m);
-					}
-					vm_object_paging_end(object);
-					vm_object_unlock(object);
+				vm_object_unlock(object);
 
-					offset += PAGE_SIZE_64;
-					va += PAGE_SIZE;
-				}
+				offset += PAGE_SIZE_64;
+				va += PAGE_SIZE;
 			}
 		}
 	}
@@ -6038,6 +6647,28 @@ vm_map_copyout(
  *	Routine:	vm_map_copyin
  *
  *	Description:
+ *		see vm_map_copyin_common.  Exported via Unsupported.exports.
+ *
+ */
+
+#undef vm_map_copyin
+
+kern_return_t
+vm_map_copyin(
+	vm_map_t			src_map,
+	vm_map_address_t	src_addr,
+	vm_map_size_t		len,
+	boolean_t			src_destroy,
+	vm_map_copy_t		*copy_result)	/* OUT */
+{
+	return(vm_map_copyin_common(src_map, src_addr, len, src_destroy,
+					FALSE, copy_result, FALSE));
+}
+
+/*
+ *	Routine:	vm_map_copyin_common
+ *
+ *	Description:
  *		Copy the specified region (src_addr, len) from the
  *		source address space (src_map), possibly removing
  *		the region from the source address space (src_destroy).
@@ -6057,6 +6688,7 @@ typedef struct submap_map {
 	vm_map_t	parent_map;
 	vm_map_offset_t	base_start;
 	vm_map_offset_t	base_end;
+	vm_map_size_t	base_len;
 	struct submap_map *next;
 } submap_map_t;
 
@@ -6083,6 +6715,7 @@ vm_map_copyin_common(
 					 */
 	vm_map_offset_t	src_end;	/* End of entire region to be
 					 * copied */
+	vm_map_offset_t src_base;
 	vm_map_t	base_map = src_map;
 	boolean_t	map_share=FALSE;
 	submap_map_t	*parent_maps = NULL;
@@ -6114,8 +6747,8 @@ vm_map_copyin_common(
 	 * for small regions.
 	 */
 	if ((len < msg_ool_size_small) && !use_maxprot)
-	  return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
-					     src_destroy, copy_result);
+		return vm_map_copyin_kernel_buffer(src_map, src_addr, len,
+						   src_destroy, copy_result);
 
 	/*
 	 *	Compute (page aligned) start and end of region
@@ -6134,7 +6767,7 @@ vm_map_copyin_common(
 
 	copy = (vm_map_copy_t) zalloc(vm_map_copy_zone);
 	vm_map_copy_first_entry(copy) =
-	 vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
+		vm_map_copy_last_entry(copy) = vm_map_copy_to_entry(copy);
 	copy->type = VM_MAP_COPY_ENTRY_LIST;
 	copy->cpy_hdr.nentries = 0;
 	copy->cpy_hdr.entries_pageable = TRUE;
@@ -6222,7 +6855,7 @@ vm_map_copyin_common(
 			submap_len = tmp_entry->vme_end - src_start;
 			if(submap_len > (src_end-src_start))
 				submap_len = src_end-src_start;
-			ptr->base_start += submap_len;
+			ptr->base_len = submap_len;
 	
 			src_start -= tmp_entry->vme_start;
 			src_start += tmp_entry->offset;
@@ -6234,13 +6867,15 @@ vm_map_copyin_common(
 			vm_map_reference(src_map);
 			vm_map_unlock(ptr->parent_map);
 			if (!vm_map_lookup_entry(
-					src_map, src_start, &tmp_entry))
+				    src_map, src_start, &tmp_entry))
 				RETURN(KERN_INVALID_ADDRESS);
 			map_share = TRUE;
 			if(!tmp_entry->is_sub_map)
-			   vm_map_clip_start(src_map, tmp_entry, src_start);
+				vm_map_clip_start(src_map, tmp_entry, src_start);
 			src_entry = tmp_entry;
 		}
+		/* we are now in the lowest level submap... */
+
 		if ((tmp_entry->object.vm_object != VM_OBJECT_NULL) && 
 		    (tmp_entry->object.vm_object->phys_contiguous)) {
 			/* This is not, supported for now.In future */
@@ -6259,27 +6894,28 @@ vm_map_copyin_common(
 		 *	to allocate a map entry.
 		 */
 		if (new_entry == VM_MAP_ENTRY_NULL) {
-		    version.main_timestamp = src_map->timestamp;
-		    vm_map_unlock(src_map);
+			version.main_timestamp = src_map->timestamp;
+			vm_map_unlock(src_map);
 
-		    new_entry = vm_map_copy_entry_create(copy);
+			new_entry = vm_map_copy_entry_create(copy);
 
-		    vm_map_lock(src_map);
-		    if ((version.main_timestamp + 1) != src_map->timestamp) {
-			if (!vm_map_lookup_entry(src_map, src_start,
-					&tmp_entry)) {
-				RETURN(KERN_INVALID_ADDRESS);
+			vm_map_lock(src_map);
+			if ((version.main_timestamp + 1) != src_map->timestamp) {
+				if (!vm_map_lookup_entry(src_map, src_start,
+							 &tmp_entry)) {
+					RETURN(KERN_INVALID_ADDRESS);
+				}
+				if (!tmp_entry->is_sub_map)
+					vm_map_clip_start(src_map, tmp_entry, src_start);
+				continue; /* restart w/ new tmp_entry */
 			}
-			vm_map_clip_start(src_map, tmp_entry, src_start);
-			continue; /* restart w/ new tmp_entry */
-		    }
 		}
 
 		/*
 		 *	Verify that the region can be read.
 		 */
 		if (((src_entry->protection & VM_PROT_READ) == VM_PROT_NONE &&
-			!use_maxprot) ||
+		     !use_maxprot) ||
 		    (src_entry->max_protection & VM_PROT_READ) == 0)
 			RETURN(KERN_PROTECTION_FAILURE);
 
@@ -6303,39 +6939,39 @@ vm_map_copyin_common(
 
 		if (src_destroy && 
 		    (src_object == VM_OBJECT_NULL || 
-		    (src_object->internal && !src_object->true_share
-		    && !map_share))) {
-		    /*
-		     * If we are destroying the source, and the object
-		     * is internal, we can move the object reference
-		     * from the source to the copy.  The copy is
-		     * copy-on-write only if the source is.
-		     * We make another reference to the object, because
-		     * destroying the source entry will deallocate it.
-		     */
-		    vm_object_reference(src_object);
+		     (src_object->internal && !src_object->true_share
+		      && !map_share))) {
+			/*
+			 * If we are destroying the source, and the object
+			 * is internal, we can move the object reference
+			 * from the source to the copy.  The copy is
+			 * copy-on-write only if the source is.
+			 * We make another reference to the object, because
+			 * destroying the source entry will deallocate it.
+			 */
+			vm_object_reference(src_object);
 
-		    /*
-		     * Copy is always unwired.  vm_map_copy_entry
-		     * set its wired count to zero.
-		     */
+			/*
+			 * Copy is always unwired.  vm_map_copy_entry
+			 * set its wired count to zero.
+			 */
 
-		    goto CopySuccessful;
+			goto CopySuccessful;
 		}
 
 
-RestartCopy:
+	RestartCopy:
 		XPR(XPR_VM_MAP, "vm_map_copyin_common src_obj 0x%x ent 0x%x obj 0x%x was_wired %d\n",
 		    src_object, new_entry, new_entry->object.vm_object,
 		    was_wired, 0);
 		if ((src_object == VM_OBJECT_NULL ||
-			(!was_wired && !map_share && !tmp_entry->is_shared)) &&
-			vm_object_copy_quickly(
-				&new_entry->object.vm_object,
-				src_offset,
-				src_size,
-				&src_needs_copy,
-				&new_entry_needs_copy)) {
+		     (!was_wired && !map_share && !tmp_entry->is_shared)) &&
+		    vm_object_copy_quickly(
+			    &new_entry->object.vm_object,
+			    src_offset,
+			    src_size,
+			    &src_needs_copy,
+			    &new_entry_needs_copy)) {
 
 			new_entry->needs_copy = new_entry_needs_copy;
 
@@ -6347,17 +6983,17 @@ RestartCopy:
 			        vm_prot_t prot;
 
 				prot = src_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-				if (src_entry->alias != VM_MEMORY_STACK && prot)
+
+				if (override_nx(src_map, src_entry->alias) && prot)
 				        prot |= VM_PROT_EXECUTE;
-#endif
+
 				vm_object_pmap_protect(
 					src_object,
 					src_offset,
 					src_size,
 			      		(src_entry->is_shared ? 
-						PMAP_NULL
-						: src_map->pmap),
+					 PMAP_NULL
+					 : src_map->pmap),
 					src_entry->vme_start,
 					prot);
 
@@ -6397,24 +7033,25 @@ RestartCopy:
 		CopySlowly:
 			vm_object_lock(src_object);
 			result = vm_object_copy_slowly(
-					src_object,
-					src_offset,
-					src_size,
-					THREAD_UNINT,
-					&new_entry->object.vm_object);
+				src_object,
+				src_offset,
+				src_size,
+				THREAD_UNINT,
+				&new_entry->object.vm_object);
 			new_entry->offset = 0;
 			new_entry->needs_copy = FALSE;
 
 		}
 		else if (src_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC &&
-			(tmp_entry->is_shared  || map_share)) {
+			 (tmp_entry->is_shared  || map_share)) {
 		  	vm_object_t new_object;
 
-			vm_object_lock(src_object);
+			vm_object_lock_shared(src_object);
 			new_object = vm_object_copy_delayed(
-					src_object,
-					src_offset,	
-					src_size);
+				src_object,
+				src_offset,	
+				src_size,
+				TRUE);
 			if (new_object == VM_OBJECT_NULL)
 			  	goto CopySlowly;
 
@@ -6424,11 +7061,11 @@ RestartCopy:
 
 		} else {
 			result = vm_object_copy_strategically(src_object,
-				src_offset,
-				src_size,
-				&new_entry->object.vm_object,
-				&new_entry->offset,
-				&new_entry_needs_copy);
+							      src_offset,
+							      src_size,
+							      &new_entry->object.vm_object,
+							      &new_entry->offset,
+							      &new_entry_needs_copy);
 
 			new_entry->needs_copy = new_entry_needs_copy;
 		}
@@ -6491,7 +7128,7 @@ RestartCopy:
 			 *	Start over with this top-level entry.
 			 */
 
-		 VerificationFailed: ;
+		VerificationFailed: ;
 
 			vm_object_deallocate(new_entry->object.vm_object);
 			tmp_entry = src_entry;
@@ -6502,7 +7139,7 @@ RestartCopy:
 		 *	Verification succeeded.
 		 */
 
-	 VerificationSuccessful: ;
+	VerificationSuccessful: ;
 
 		if (result == KERN_MEMORY_RESTART_COPY)
 			goto RestartCopy;
@@ -6511,7 +7148,7 @@ RestartCopy:
 		 *	Copy succeeded.
 		 */
 
-	 CopySuccessful: ;
+	CopySuccessful: ;
 
 		/*
 		 *	Link in the new copy entry.
@@ -6524,6 +7161,7 @@ RestartCopy:
 		 *	Determine whether the entire region
 		 *	has been copied.
 		 */
+		src_base = src_start;
 		src_start = new_entry->vme_end;
 		new_entry = VM_MAP_ENTRY_NULL;
 		while ((src_start >= src_end) && (src_end != 0)) {
@@ -6533,15 +7171,22 @@ RestartCopy:
 				ptr = parent_maps;
 				assert(ptr != NULL);
 				parent_maps = parent_maps->next;
+
+				/* fix up the damage we did in that submap */
+				vm_map_simplify_range(src_map,
+						      src_base,
+						      src_end);
+
 				vm_map_unlock(src_map);
 				vm_map_deallocate(src_map);
 				vm_map_lock(ptr->parent_map);
 				src_map = ptr->parent_map;
-				src_start = ptr->base_start;
+				src_base = ptr->base_start;
+				src_start = ptr->base_start + ptr->base_len;
 				src_end = ptr->base_end;
 				if ((src_end > src_start) &&
-					      !vm_map_lookup_entry(
-					      src_map, src_start, &tmp_entry))
+				    !vm_map_lookup_entry(
+					    src_map, src_start, &tmp_entry))
 					RETURN(KERN_INVALID_ADDRESS);
 				kfree(ptr, sizeof(submap_map_t));
 				if(parent_maps == NULL)
@@ -6559,7 +7204,7 @@ RestartCopy:
 
 		tmp_entry = src_entry->vme_next;
 		if ((tmp_entry->vme_start != src_start) || 
-				(tmp_entry == vm_map_to_entry(src_map)))
+		    (tmp_entry == vm_map_to_entry(src_map)))
 			RETURN(KERN_INVALID_ADDRESS);
 	}
 
@@ -6572,9 +7217,14 @@ RestartCopy:
 				     vm_map_trunc_page(src_addr),
 				     src_end,
 				     (src_map == kernel_map) ?
-					VM_MAP_REMOVE_KUNWIRE :
-				        VM_MAP_NO_FLAGS,
+				     VM_MAP_REMOVE_KUNWIRE :
+				     VM_MAP_NO_FLAGS,
 				     VM_MAP_NULL);
+	} else {
+		/* fix up the damage we did in the base map */
+		vm_map_simplify_range(src_map,
+				      vm_map_trunc_page(src_addr), 
+				      vm_map_round_page(src_end));
 	}
 
 	vm_map_unlock(src_map);
@@ -6656,22 +7306,22 @@ vm_map_fork_share(
 			kern_return_t	result;
 
 			result = pmap_nest(new_map->pmap, 
-				(old_entry->object.sub_map)->pmap, 
-				(addr64_t)old_entry->vme_start,
-				(addr64_t)old_entry->vme_start,
-				(uint64_t)(old_entry->vme_end - old_entry->vme_start));
+					   (old_entry->object.sub_map)->pmap, 
+					   (addr64_t)old_entry->vme_start,
+					   (addr64_t)old_entry->vme_start,
+					   (uint64_t)(old_entry->vme_end - old_entry->vme_start));
 			if(result)
 				panic("vm_map_fork_share: pmap_nest failed!");
 		}
 #endif	/* NO_NESTED_PMAP */
 	} else if (object == VM_OBJECT_NULL) {
 		object = vm_object_allocate((vm_map_size_t)(old_entry->vme_end -
-							old_entry->vme_start));
+							    old_entry->vme_start));
 		old_entry->offset = 0;
 		old_entry->object.vm_object = object;
 		assert(!old_entry->needs_copy);
 	} else if (object->copy_strategy !=
-		 MEMORY_OBJECT_COPY_SYMMETRIC) {
+		   MEMORY_OBJECT_COPY_SYMMETRIC) {
 		
 		/*
 		 *	We are already using an asymmetric
@@ -6684,10 +7334,10 @@ vm_map_fork_share(
 	else if (old_entry->needs_copy ||	/* case 1 */
 		 object->shadowed ||		/* case 2 */
 		 (!object->true_share && 	/* case 3 */
-		 !old_entry->is_shared &&
-		 (object->size >
-		  (vm_map_size_t)(old_entry->vme_end -
-			      old_entry->vme_start)))) {
+		  !old_entry->is_shared &&
+		  (object->size >
+		   (vm_map_size_t)(old_entry->vme_end -
+				   old_entry->vme_start)))) {
 		
 		/*
 		 *	We need to create a shadow.
@@ -6766,11 +7416,10 @@ vm_map_fork_share(
 		 *	case 2.)
 		 */
 		
-		assert(!(object->shadowed && old_entry->is_shared));
 		vm_object_shadow(&old_entry->object.vm_object,
 				 &old_entry->offset,
 				 (vm_map_size_t) (old_entry->vme_end -
-					      old_entry->vme_start));
+						  old_entry->vme_start));
 		
 		/*
 		 *	If we're making a shadow for other than
@@ -6783,24 +7432,24 @@ vm_map_fork_share(
 		        vm_prot_t prot;
 
 			prot = old_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-			if (old_entry->alias != VM_MEMORY_STACK && prot)
+
+			if (override_nx(old_map, old_entry->alias) && prot)
 			        prot |= VM_PROT_EXECUTE;
-#endif
+
 			if (old_map->mapped) {
 				vm_object_pmap_protect(
 					old_entry->object.vm_object,
 					old_entry->offset,
 					(old_entry->vme_end -
-							old_entry->vme_start),
+					 old_entry->vme_start),
 					PMAP_NULL,
 					old_entry->vme_start,
 					prot);
 			} else {
 				pmap_protect(old_map->pmap,
-				     old_entry->vme_start,
-				     old_entry->vme_end,
-				     prot);
+					     old_entry->vme_start,
+					     old_entry->vme_end,
+					     prot);
 			}
 		}
 		
@@ -6823,8 +7472,7 @@ vm_map_fork_share(
 		vm_map_unlock(old_entry->object.sub_map);
 	} else {
 		vm_object_lock(object);
-		object->ref_count++;
-		vm_object_res_reference(object);
+		vm_object_reference_locked(object);
 		if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
 			object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
 		}
@@ -6857,8 +7505,8 @@ vm_map_fork_share(
 		/* Bill Angell pmap support goes here */
 	} else {
 		pmap_copy(new_map->pmap, old_map->pmap, new_entry->vme_start,
-		  old_entry->vme_end - old_entry->vme_start,
-		  old_entry->vme_start);
+			  old_entry->vme_end - old_entry->vme_start,
+			  old_entry->vme_start);
 	}
 }
 
@@ -6922,7 +7570,15 @@ vm_map_fork_copy(
 	if (! vm_map_lookup_entry(old_map, start, &last)) {
 		last = last->vme_next;
 	} else {
-		vm_map_clip_start(old_map, last, start);
+		if (last->vme_start == start) {
+			/*
+			 * No need to clip here and we don't
+			 * want to cause any unnecessary
+			 * unnesting...
+			 */
+		} else {
+			vm_map_clip_start(old_map, last, start);
+		}
 	}
 	*old_entry_p = last;
 
@@ -6942,9 +7598,7 @@ vm_map_t
 vm_map_fork(
 	vm_map_t	old_map)
 {
-	pmap_t		new_pmap = pmap_create(
-					(vm_map_size_t) 0,
-					task_has_64BitAddr(current_task()));
+	pmap_t		new_pmap;
 	vm_map_t	new_map;
 	vm_map_entry_t	old_entry;
 	vm_map_size_t	new_size = 0, entry_size;
@@ -6952,18 +7606,27 @@ vm_map_fork(
 	boolean_t	src_needs_copy;
 	boolean_t	new_entry_needs_copy;
 
+#ifdef __i386__
+	new_pmap = pmap_create((vm_map_size_t) 0,
+			       old_map->pmap->pm_task_map != TASK_MAP_32BIT);
+	if (old_map->pmap->pm_task_map == TASK_MAP_64BIT_SHARED)
+		pmap_set_4GB_pagezero(new_pmap);
+#else
+	new_pmap = pmap_create((vm_map_size_t) 0, 0);
+#endif
+
 	vm_map_reference_swap(old_map);
 	vm_map_lock(old_map);
 
 	new_map = vm_map_create(new_pmap,
-			old_map->min_offset,
-			old_map->max_offset,
-			old_map->hdr.entries_pageable);
+				old_map->min_offset,
+				old_map->max_offset,
+				old_map->hdr.entries_pageable);
 
 	for (
-	    old_entry = vm_map_first_entry(old_map);
-	    old_entry != vm_map_to_entry(old_map);
-	    ) {
+		old_entry = vm_map_first_entry(old_map);
+		old_entry != vm_map_to_entry(old_map);
+		) {
 
 		entry_size = old_entry->vme_end - old_entry->vme_start;
 
@@ -6987,8 +7650,8 @@ vm_map_fork(
 			if(old_entry->is_sub_map)
 				break;
 			if ((old_entry->wired_count != 0) ||
-				((old_entry->object.vm_object != NULL) &&
-				(old_entry->object.vm_object->true_share))) {
+			    ((old_entry->object.vm_object != NULL) &&
+			     (old_entry->object.vm_object->true_share))) {
 				goto slow_vm_map_fork_copy;
 			}
 
@@ -6998,12 +7661,12 @@ vm_map_fork(
 			new_entry->use_pmap = FALSE;
 
 			if (! vm_object_copy_quickly(
-						&new_entry->object.vm_object,
-						old_entry->offset,
-						(old_entry->vme_end -
-							old_entry->vme_start),
-						&src_needs_copy,
-						&new_entry_needs_copy)) {
+				    &new_entry->object.vm_object,
+				    old_entry->offset,
+				    (old_entry->vme_end -
+				     old_entry->vme_start),
+				    &src_needs_copy,
+				    &new_entry_needs_copy)) {
 				vm_map_entry_dispose(new_map, new_entry);
 				goto slow_vm_map_fork_copy;
 			}
@@ -7016,19 +7679,19 @@ vm_map_fork(
 			        vm_prot_t prot;
 
 				prot = old_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-				if (old_entry->alias != VM_MEMORY_STACK && prot)
+
+				if (override_nx(old_map, old_entry->alias) && prot)
 				        prot |= VM_PROT_EXECUTE;
-#endif
+
 				vm_object_pmap_protect(
 					old_entry->object.vm_object,
 					old_entry->offset,
 					(old_entry->vme_end -
-							old_entry->vme_start),
+					 old_entry->vme_start),
 					((old_entry->is_shared 
-						|| old_map->mapped)
-							? PMAP_NULL :
-							old_map->pmap),
+					  || old_map->mapped)
+					 ? PMAP_NULL :
+					 old_map->pmap),
 					old_entry->vme_start,
 					prot);
 
@@ -7062,6 +7725,30 @@ vm_map_fork(
 	return(new_map);
 }
 
+/*
+ * vm_map_exec:
+ *
+ * 	Setup the "new_map" with the proper execution environment according
+ *	to the type of executable (platform, 64bit, chroot environment).
+ *	Map the comm page and shared region, etc...
+ */
+kern_return_t
+vm_map_exec(
+	vm_map_t	new_map,
+	task_t		task,
+	void		*fsroot,
+	cpu_type_t	cpu)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: task %p: vm_map_exec(%p,%p,%p,0x%x): ->\n",
+		 current_task(), new_map, task, fsroot, cpu));
+	(void) vm_commpage_enter(new_map, task);
+	(void) vm_shared_region_enter(new_map, task, fsroot, cpu);
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: task %p: vm_map_exec(%p,%p,%p,0x%x): <-\n",
+		 current_task(), new_map, task, fsroot, cpu));
+	return KERN_SUCCESS;
+}
 
 /*
  *	vm_map_lookup_locked:
@@ -7090,16 +7777,15 @@ vm_map_fork(
 kern_return_t
 vm_map_lookup_locked(
 	vm_map_t		*var_map,	/* IN/OUT */
-	vm_map_offset_t	vaddr,
+	vm_map_offset_t		vaddr,
 	vm_prot_t		fault_type,
+	int			object_lock_type,
 	vm_map_version_t	*out_version,	/* OUT */
 	vm_object_t		*object,	/* OUT */
 	vm_object_offset_t	*offset,	/* OUT */
 	vm_prot_t		*out_prot,	/* OUT */
 	boolean_t		*wired,		/* OUT */
-	int			*behavior,	/* OUT */
-	vm_map_offset_t	*lo_offset,	/* OUT */
-	vm_map_offset_t	*hi_offset,	/* OUT */
+	vm_object_fault_info_t	fault_info,	/* OUT */
 	vm_map_t		*real_map)
 {
 	vm_map_entry_t			entry;
@@ -7112,7 +7798,7 @@ vm_map_lookup_locked(
 	register vm_prot_t		prot;
 
 	*real_map = map;
-	RetryLookup: ;
+RetryLookup: ;
 
 	/*
 	 *	If the map has an interesting hint, try it before calling
@@ -7132,7 +7818,7 @@ vm_map_lookup_locked(
 			if((cow_sub_map_parent) && (cow_sub_map_parent != map))
 				vm_map_unlock(cow_sub_map_parent);
 			if((*real_map != map) 
-					&& (*real_map != cow_sub_map_parent))
+			   && (*real_map != cow_sub_map_parent))
 				vm_map_unlock(*real_map);
 			return KERN_INVALID_ADDRESS;
 		}
@@ -7159,18 +7845,19 @@ submap_recurse:
 
 		local_vaddr = vaddr;
 
-		if ((!entry->needs_copy) && (entry->use_pmap)) {
+		if ((entry->use_pmap && !(fault_type & VM_PROT_WRITE))) {
 			/* if real_map equals map we unlock below */
 			if ((*real_map != map) && 
-					(*real_map != cow_sub_map_parent))
+			    (*real_map != cow_sub_map_parent))
 				vm_map_unlock(*real_map);
 			*real_map = entry->object.sub_map;
 		}
 
-		if(entry->needs_copy) {
+		if(entry->needs_copy && (fault_type & VM_PROT_WRITE)) {
 			if (!mapped_needs_copy) {
 				if (vm_map_lock_read_to_write(map)) {
 					vm_map_lock_read(map);
+					/* XXX FBDP: entry still valid ? */
 					if(*real_map == entry->object.sub_map)
 						*real_map = map;
 					goto RetryLookup;
@@ -7187,7 +7874,7 @@ submap_recurse:
 			} else {
 				vm_map_lock_read(entry->object.sub_map);
 				if((cow_sub_map_parent != map) &&
-							(*real_map != map))
+				   (*real_map != map))
 					vm_map_unlock(map);
 			}
 		} else {
@@ -7201,23 +7888,25 @@ submap_recurse:
 				vm_map_unlock_read(map);
 		}
 
+		/* XXX FBDP: map has been unlocked, what protects "entry" !? */
 		*var_map = map = entry->object.sub_map;
 
 		/* calculate the offset in the submap for vaddr */
 		local_vaddr = (local_vaddr - entry->vme_start) + entry->offset;
 
-RetrySubMap:
+	RetrySubMap:
 		if(!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) {
 			if((cow_sub_map_parent) && (cow_sub_map_parent != map)){
 				vm_map_unlock(cow_sub_map_parent);
 			}
 			if((*real_map != map) 
-					&& (*real_map != cow_sub_map_parent)) {
+			   && (*real_map != cow_sub_map_parent)) {
 				vm_map_unlock(*real_map);
 			}
 			*real_map = map;
 			return KERN_INVALID_ADDRESS;
 		}
+
 		/* find the attenuated shadow of the underlying object */
 		/* on our target map */
 
@@ -7229,14 +7918,14 @@ RetrySubMap:
 		/* to be as big as the portion of the underlying entry    */
 		/* which is mapped */
 		start_delta = submap_entry->vme_start > entry->offset ?
-	  	            submap_entry->vme_start - entry->offset : 0;
+			submap_entry->vme_start - entry->offset : 0;
 
 		end_delta = 
-		   (entry->offset + start_delta + (old_end - old_start)) <=
+			(entry->offset + start_delta + (old_end - old_start)) <=
 			submap_entry->vme_end ?
-				0 : (entry->offset + 
-					(old_end - old_start))
-					- submap_entry->vme_end; 
+			0 : (entry->offset + 
+			     (old_end - old_start))
+			- submap_entry->vme_end; 
 
 		old_start += start_delta;
 		old_end -= end_delta;
@@ -7249,7 +7938,8 @@ RetrySubMap:
 
 		if(((fault_type & VM_PROT_WRITE) && cow_sub_map_parent)) {
 
-			vm_object_t	copy_object;
+			vm_object_t	sub_object, copy_object;
+			vm_object_offset_t copy_offset;
 			vm_map_offset_t	local_start;
 			vm_map_offset_t	local_end;
 			boolean_t		copied_slowly = FALSE;
@@ -7262,20 +7952,24 @@ RetrySubMap:
 			}
 
 
-			if (submap_entry->object.vm_object == VM_OBJECT_NULL) {
-				submap_entry->object.vm_object = 
+			sub_object = submap_entry->object.vm_object;
+			if (sub_object == VM_OBJECT_NULL) {
+				sub_object =
 					vm_object_allocate(
 						(vm_map_size_t)
-						(submap_entry->vme_end 
-						- submap_entry->vme_start));
+						(submap_entry->vme_end -
+						 submap_entry->vme_start));
+				submap_entry->object.vm_object = sub_object;
 				submap_entry->offset = 0;
 			}
 			local_start =  local_vaddr - 
-					(cow_parent_vaddr - old_start);
+				(cow_parent_vaddr - old_start);
 			local_end = local_vaddr + 
-					(old_end - cow_parent_vaddr);
+				(old_end - cow_parent_vaddr);
 			vm_map_clip_start(map, submap_entry, local_start);
 			vm_map_clip_end(map, submap_entry, local_end);
+			/* unnesting was done in vm_map_clip_start/end() */
+			assert(!submap_entry->use_pmap);
 
 			/* This is the COW case, lets connect */
 			/* an entry in our space to the underlying */
@@ -7283,41 +7977,48 @@ RetrySubMap:
 			/* submap. */
 
 
-			if(submap_entry->wired_count != 0) {
-					vm_object_lock(
-					     submap_entry->object.vm_object);
-					vm_object_copy_slowly(
-						submap_entry->object.vm_object,
-						submap_entry->offset,
-						submap_entry->vme_end -
-							submap_entry->vme_start,
-						FALSE,
-						&copy_object);
-					copied_slowly = TRUE;
+			if(submap_entry->wired_count != 0 ||
+			   (sub_object->copy_strategy !=
+			    MEMORY_OBJECT_COPY_SYMMETRIC)) {
+				vm_object_lock(sub_object);
+				vm_object_copy_slowly(sub_object,
+						      submap_entry->offset,
+						      (submap_entry->vme_end -
+						       submap_entry->vme_start),
+						      FALSE,
+						      &copy_object);
+				copied_slowly = TRUE;
 			} else {
+				
 				/* set up shadow object */
-				copy_object = submap_entry->object.vm_object;
+				copy_object = sub_object;
 				vm_object_reference(copy_object);
-				submap_entry->object.vm_object->shadowed = TRUE;
+				sub_object->shadowed = TRUE;
 				submap_entry->needs_copy = TRUE;
 
 				prot = submap_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-				if (submap_entry->alias != VM_MEMORY_STACK && prot)
+
+				if (override_nx(map, submap_entry->alias) && prot)
 				        prot |= VM_PROT_EXECUTE;
-#endif
+
 				vm_object_pmap_protect(
-					submap_entry->object.vm_object,
+					sub_object,
 					submap_entry->offset,
 					submap_entry->vme_end - 
-						submap_entry->vme_start,
+					submap_entry->vme_start,
 					(submap_entry->is_shared 
-						|| map->mapped) ?
-						PMAP_NULL : map->pmap,
+					 || map->mapped) ?
+					PMAP_NULL : map->pmap,
 					submap_entry->vme_start,
 					prot);
 			}
 			
+			/*
+			 * Adjust the fault offset to the submap entry.
+			 */
+			copy_offset = (local_vaddr -
+				       submap_entry->vme_start +
+				       submap_entry->offset);
 
 			/* This works diffently than the   */
 			/* normal submap case. We go back  */
@@ -7334,21 +8035,45 @@ RetrySubMap:
 			vaddr = cow_parent_vaddr;
 			cow_sub_map_parent = NULL;
 
-			if(!vm_map_lookup_entry(map, 
-					vaddr, &entry)) {
-				        vm_object_deallocate(
-							copy_object);
-					vm_map_lock_write_to_read(map);
-					return KERN_INVALID_ADDRESS;
-			}
-					
-			/* clip out the portion of space */
-			/* mapped by the sub map which   */
-			/* corresponds to the underlying */
-			/* object */
+			if(!vm_map_lookup_entry(map, 
+						vaddr, &entry)) {
+				vm_object_deallocate(
+					copy_object);
+				vm_map_lock_write_to_read(map);
+				return KERN_INVALID_ADDRESS;
+			}
+					
+			/* clip out the portion of space */
+			/* mapped by the sub map which   */
+			/* corresponds to the underlying */
+			/* object */
+
+			/*
+			 * Clip (and unnest) the smallest nested chunk
+			 * possible around the faulting address...
+			 */
+			local_start = vaddr & ~(pmap_nesting_size_min - 1);
+			local_end = local_start + pmap_nesting_size_min;
+			/*
+			 * ... but don't go beyond the "old_start" to "old_end"
+			 * range, to avoid spanning over another VM region
+			 * with a possibly different VM object and/or offset.
+			 */
+			if (local_start < old_start) {
+				local_start = old_start;
+			}
+			if (local_end > old_end) {
+				local_end = old_end;
+			}
+			/*
+			 * Adjust copy_offset to the start of the range.
+			 */
+			copy_offset -= (vaddr - local_start);
+
 			vm_map_clip_start(map, entry, local_start);
 			vm_map_clip_end(map, entry, local_end);
-
+			/* unnesting was done in vm_map_clip_start/end() */
+			assert(!entry->use_pmap);
 
 			/* substitute copy object for */
 			/* shared map entry           */
@@ -7356,14 +8081,16 @@ RetrySubMap:
 			entry->is_sub_map = FALSE;
 			entry->object.vm_object = copy_object;
 
-			entry->protection |= VM_PROT_WRITE;
-			entry->max_protection |= VM_PROT_WRITE;
+			/* propagate the submap entry's protections */
+			entry->protection |= submap_entry->protection;
+			entry->max_protection |= submap_entry->max_protection;
+
 			if(copied_slowly) {
 				entry->offset = 0;
 				entry->needs_copy = FALSE;
 				entry->is_shared = FALSE;
 			} else {
-				entry->offset = submap_entry->offset;
+				entry->offset = copy_offset;
 				entry->needs_copy = TRUE;
 				if(entry->inheritance == VM_INHERIT_SHARE) 
 					entry->inheritance = VM_INHERIT_COPY;
@@ -7376,8 +8103,8 @@ RetrySubMap:
 			vm_map_lock_write_to_read(map);
 		} else {
 			if((cow_sub_map_parent)
-					&& (cow_sub_map_parent != *real_map)
-					&& (cow_sub_map_parent != map)) {
+			   && (cow_sub_map_parent != *real_map)
+			   && (cow_sub_map_parent != map)) {
 				vm_map_unlock(cow_sub_map_parent);
 			}
 			entry = submap_entry;
@@ -7389,24 +8116,26 @@ RetrySubMap:
 	 *	Check whether this task is allowed to have
 	 *	this page.
 	 */
+
 	prot = entry->protection;
 
-#ifdef STACK_ONLY_NX
-	if (entry->alias != VM_MEMORY_STACK && prot)
+	if (override_nx(map, entry->alias) && prot) {
 	        /*
-		 * HACK -- if not a stack, than allow execution
+		 * HACK -- if not a stack, then allow execution
 		 */
 	        prot |= VM_PROT_EXECUTE;
-#endif
+	}
+
 	if ((fault_type & (prot)) != fault_type) {
-	        if (*real_map != map) {
-		        vm_map_unlock(*real_map);
+		if (*real_map != map) {
+			vm_map_unlock(*real_map);
 		}
 		*real_map = map;
 
 		if ((fault_type & VM_PROT_EXECUTE) && prot)
-		        log_nx_failure((addr64_t)vaddr, prot);
+		        log_stack_execution_failure((addr64_t)vaddr, prot);
 
+		DTRACE_VM2(prot_fault, int, 1, (uint64_t *), NULL);
 		return KERN_PROTECTION_FAILURE;
 	}
 
@@ -7447,7 +8176,7 @@ RetrySubMap:
 			vm_object_shadow(&entry->object.vm_object,
 					 &entry->offset,
 					 (vm_map_size_t) (entry->vme_end -
-						      entry->vme_start));
+							  entry->vme_start));
 
 			entry->object.vm_object->shadowed = TRUE;
 			entry->needs_copy = FALSE;
@@ -7488,16 +8217,26 @@ RetrySubMap:
         *offset = (vaddr - entry->vme_start) + entry->offset;
         *object = entry->object.vm_object;
 	*out_prot = prot;
-	*behavior = entry->behavior;
-	*lo_offset = entry->offset;
-	*hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;
+
+	if (fault_info) {
+		fault_info->interruptible = THREAD_UNINT; /* for now... */
+		/* ... the caller will change "interruptible" if needed */
+	        fault_info->cluster_size = 0;
+		fault_info->user_tag = entry->alias;
+	        fault_info->behavior = entry->behavior;
+		fault_info->lo_offset = entry->offset;
+		fault_info->hi_offset = (entry->vme_end - entry->vme_start) + entry->offset;
+		fault_info->no_cache  = entry->no_cache;
+	}
 
 	/*
 	 *	Lock the object to prevent it from disappearing
 	 */
-
-	vm_object_lock(*object);
-
+	if (object_lock_type == OBJECT_LOCK_EXCLUSIVE)
+	        vm_object_lock(*object);
+	else
+	        vm_object_lock_shared(*object);
+	
 	/*
 	 *	Save the version number
 	 */
@@ -7595,17 +8334,33 @@ vm_map_region_recurse_64(
 	unsigned int			next_depth;
 	vm_map_offset_t			next_max_offset;
 
+	boolean_t			look_for_pages;
+	vm_region_submap_short_info_64_t short_info;
+
 	if (map == VM_MAP_NULL) {
 		/* no address space to work on */
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	if (*count < VM_REGION_SUBMAP_INFO_COUNT_64) {
-		/* "info" structure is not big enough and would overflow */
-		return KERN_INVALID_ARGUMENT;
+		if (*count < VM_REGION_SUBMAP_SHORT_INFO_COUNT_64) {
+			/*
+			 * "info" structure is not big enough and
+			 * would overflow
+			 */
+			return KERN_INVALID_ARGUMENT;
+		} else {
+			look_for_pages = FALSE;
+			*count = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64;
+			short_info = (vm_region_submap_short_info_64_t) submap_info;
+			submap_info = NULL;
+		}
+	} else {
+		look_for_pages = TRUE;
+		*count = VM_REGION_SUBMAP_INFO_COUNT_64;
+		short_info = NULL;
 	}
 
-	*count = VM_REGION_SUBMAP_INFO_COUNT_64;
 
 	user_address = *address;
 	user_max_depth = *nesting_depth;
@@ -7776,15 +8531,27 @@ vm_map_region_recurse_64(
 	*size = curr_entry->vme_end - curr_entry->vme_start;
 	*address = curr_entry->vme_start + curr_offset;
 
-	submap_info->user_tag = curr_entry->alias;
-	submap_info->offset = curr_entry->offset; 
-	submap_info->protection = curr_entry->protection;
-	submap_info->inheritance = curr_entry->inheritance;
-	submap_info->max_protection = curr_entry->max_protection;
-	submap_info->behavior = curr_entry->behavior;
-	submap_info->user_wired_count = curr_entry->user_wired_count;
-	submap_info->is_submap = curr_entry->is_sub_map;
-	submap_info->object_id = (uint32_t) curr_entry->object.vm_object;
+	if (look_for_pages) {
+		submap_info->user_tag = curr_entry->alias;
+		submap_info->offset = curr_entry->offset; 
+		submap_info->protection = curr_entry->protection;
+		submap_info->inheritance = curr_entry->inheritance;
+		submap_info->max_protection = curr_entry->max_protection;
+		submap_info->behavior = curr_entry->behavior;
+		submap_info->user_wired_count = curr_entry->user_wired_count;
+		submap_info->is_submap = curr_entry->is_sub_map;
+		submap_info->object_id = (uint32_t) curr_entry->object.vm_object;
+	} else {
+		short_info->user_tag = curr_entry->alias;
+		short_info->offset = curr_entry->offset; 
+		short_info->protection = curr_entry->protection;
+		short_info->inheritance = curr_entry->inheritance;
+		short_info->max_protection = curr_entry->max_protection;
+		short_info->behavior = curr_entry->behavior;
+		short_info->user_wired_count = curr_entry->user_wired_count;
+		short_info->is_submap = curr_entry->is_sub_map;
+		short_info->object_id = (uint32_t) curr_entry->object.vm_object;
+	}
 
 	extended.pages_resident = 0;
 	extended.pages_swapped_out = 0;
@@ -7801,32 +8568,40 @@ vm_map_region_recurse_64(
 					   curr_entry->offset,
 					   (curr_entry->vme_end -
 					    curr_entry->vme_start),
-					   &extended);
-			submap_info->share_mode = extended.share_mode;
+					   &extended,
+					   look_for_pages);
 			if (extended.external_pager &&
 			    extended.ref_count == 2 &&
 			    extended.share_mode == SM_SHARED) {
-				submap_info->share_mode = SM_PRIVATE;
+				extended.share_mode = SM_PRIVATE;
 			}
-			submap_info->ref_count = extended.ref_count;
 		} else {
 			if (curr_entry->use_pmap) {
-				submap_info->share_mode = SM_TRUESHARED;
+				extended.share_mode = SM_TRUESHARED;
 			} else {
-				submap_info->share_mode = SM_PRIVATE;
+				extended.share_mode = SM_PRIVATE;
 			}
-			submap_info->ref_count =
+			extended.ref_count =
 				curr_entry->object.sub_map->ref_count;
 		}
 	}
 
-	submap_info->pages_resident = extended.pages_resident;
-	submap_info->pages_swapped_out = extended.pages_swapped_out;
-	submap_info->pages_shared_now_private =
-		extended.pages_shared_now_private;
-	submap_info->pages_dirtied = extended.pages_dirtied;
-	submap_info->external_pager = extended.external_pager;
-	submap_info->shadow_depth = extended.shadow_depth;
+	if (look_for_pages) {
+		submap_info->pages_resident = extended.pages_resident;
+		submap_info->pages_swapped_out = extended.pages_swapped_out;
+		submap_info->pages_shared_now_private =
+			extended.pages_shared_now_private;
+		submap_info->pages_dirtied = extended.pages_dirtied;
+		submap_info->external_pager = extended.external_pager;
+		submap_info->shadow_depth = extended.shadow_depth;
+		submap_info->share_mode = extended.share_mode;
+		submap_info->ref_count = extended.ref_count;
+	} else {
+		short_info->external_pager = extended.external_pager;
+		short_info->shadow_depth = extended.shadow_depth;
+		short_info->share_mode = extended.share_mode;
+		short_info->ref_count = extended.ref_count;
+	}
 
 	if (not_in_kdp) {
 		vm_map_unlock_read(curr_map);
@@ -7867,253 +8642,261 @@ vm_map_region(
 	switch (flavor) {
 
 	case VM_REGION_BASIC_INFO:
-	/* legacy for old 32-bit objects info */
+		/* legacy for old 32-bit objects info */
 	{
-	    vm_region_basic_info_t	basic;
+		vm_region_basic_info_t	basic;
 
-	    if (*count < VM_REGION_BASIC_INFO_COUNT)
-		return(KERN_INVALID_ARGUMENT);
+		if (*count < VM_REGION_BASIC_INFO_COUNT)
+			return(KERN_INVALID_ARGUMENT);
 
-	    basic = (vm_region_basic_info_t) info;
-	    *count = VM_REGION_BASIC_INFO_COUNT;
+		basic = (vm_region_basic_info_t) info;
+		*count = VM_REGION_BASIC_INFO_COUNT;
 
-	    vm_map_lock_read(map);
+		vm_map_lock_read(map);
 
-	    start = *address;
-	    if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
-		if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
-			vm_map_unlock_read(map);
-		   	return(KERN_INVALID_ADDRESS);
+		start = *address;
+		if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+				vm_map_unlock_read(map);
+				return(KERN_INVALID_ADDRESS);
+			}
+		} else {
+			entry = tmp_entry;
 		}
-	    } else {
-		entry = tmp_entry;
-	    }
 
-	    start = entry->vme_start;
+		start = entry->vme_start;
 
-	    basic->offset = (uint32_t)entry->offset;
-	    basic->protection = entry->protection;
-	    basic->inheritance = entry->inheritance;
-	    basic->max_protection = entry->max_protection;
-	    basic->behavior = entry->behavior;
-	    basic->user_wired_count = entry->user_wired_count;
-	    basic->reserved = entry->is_sub_map;
-	    *address = start;
-	    *size = (entry->vme_end - start);
+		basic->offset = (uint32_t)entry->offset;
+		basic->protection = entry->protection;
+		basic->inheritance = entry->inheritance;
+		basic->max_protection = entry->max_protection;
+		basic->behavior = entry->behavior;
+		basic->user_wired_count = entry->user_wired_count;
+		basic->reserved = entry->is_sub_map;
+		*address = start;
+		*size = (entry->vme_end - start);
 
-	    if (object_name) *object_name = IP_NULL;
-	    if (entry->is_sub_map) {
-	        basic->shared = FALSE;
-	    } else {
-	        basic->shared = entry->is_shared;
-	    }
+		if (object_name) *object_name = IP_NULL;
+		if (entry->is_sub_map) {
+			basic->shared = FALSE;
+		} else {
+			basic->shared = entry->is_shared;
+		}
 
-	    vm_map_unlock_read(map);
-	    return(KERN_SUCCESS);
+		vm_map_unlock_read(map);
+		return(KERN_SUCCESS);
 	}
 
 	case VM_REGION_BASIC_INFO_64:
 	{
-	    vm_region_basic_info_64_t	basic;
+		vm_region_basic_info_64_t	basic;
 
-	    if (*count < VM_REGION_BASIC_INFO_COUNT_64)
-		return(KERN_INVALID_ARGUMENT);
+		if (*count < VM_REGION_BASIC_INFO_COUNT_64)
+			return(KERN_INVALID_ARGUMENT);
+
+		basic = (vm_region_basic_info_64_t) info;
+		*count = VM_REGION_BASIC_INFO_COUNT_64;
+
+		vm_map_lock_read(map);
+
+		start = *address;
+		if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+				vm_map_unlock_read(map);
+				return(KERN_INVALID_ADDRESS);
+			}
+		} else {
+			entry = tmp_entry;
+		}
 
-	    basic = (vm_region_basic_info_64_t) info;
-	    *count = VM_REGION_BASIC_INFO_COUNT_64;
+		start = entry->vme_start;
 
-	    vm_map_lock_read(map);
+		basic->offset = entry->offset;
+		basic->protection = entry->protection;
+		basic->inheritance = entry->inheritance;
+		basic->max_protection = entry->max_protection;
+		basic->behavior = entry->behavior;
+		basic->user_wired_count = entry->user_wired_count;
+		basic->reserved = entry->is_sub_map;
+		*address = start;
+		*size = (entry->vme_end - start);
 
-	    start = *address;
-	    if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
-		if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
-			vm_map_unlock_read(map);
-		   	return(KERN_INVALID_ADDRESS);
+		if (object_name) *object_name = IP_NULL;
+		if (entry->is_sub_map) {
+			basic->shared = FALSE;
+		} else {
+			basic->shared = entry->is_shared;
 		}
-	    } else {
-		entry = tmp_entry;
-	    }
-
-	    start = entry->vme_start;
-
-	    basic->offset = entry->offset;
-	    basic->protection = entry->protection;
-	    basic->inheritance = entry->inheritance;
-	    basic->max_protection = entry->max_protection;
-	    basic->behavior = entry->behavior;
-	    basic->user_wired_count = entry->user_wired_count;
-	    basic->reserved = entry->is_sub_map;
-	    *address = start;
-	    *size = (entry->vme_end - start);
-
-	    if (object_name) *object_name = IP_NULL;
-	    if (entry->is_sub_map) {
-	        basic->shared = FALSE;
-	    } else {
-	        basic->shared = entry->is_shared;
-	    }
-
-	    vm_map_unlock_read(map);
-	    return(KERN_SUCCESS);
+
+		vm_map_unlock_read(map);
+		return(KERN_SUCCESS);
 	}
 	case VM_REGION_EXTENDED_INFO:
 	{
-	    vm_region_extended_info_t	extended;
+		vm_region_extended_info_t	extended;
 
-	    if (*count < VM_REGION_EXTENDED_INFO_COUNT)
-		return(KERN_INVALID_ARGUMENT);
+		if (*count < VM_REGION_EXTENDED_INFO_COUNT)
+			return(KERN_INVALID_ARGUMENT);
 
-	    extended = (vm_region_extended_info_t) info;
-	    *count = VM_REGION_EXTENDED_INFO_COUNT;
+		extended = (vm_region_extended_info_t) info;
+		*count = VM_REGION_EXTENDED_INFO_COUNT;
 
-	    vm_map_lock_read(map);
+		vm_map_lock_read(map);
 
-	    start = *address;
-	    if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
-		if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
-			vm_map_unlock_read(map);
-		   	return(KERN_INVALID_ADDRESS);
+		start = *address;
+		if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+				vm_map_unlock_read(map);
+				return(KERN_INVALID_ADDRESS);
+			}
+		} else {
+			entry = tmp_entry;
 		}
-	    } else {
-		entry = tmp_entry;
-	    }
-	    start = entry->vme_start;
+		start = entry->vme_start;
 
-	    extended->protection = entry->protection;
-	    extended->user_tag = entry->alias;
-	    extended->pages_resident = 0;
-	    extended->pages_swapped_out = 0;
-	    extended->pages_shared_now_private = 0;
-	    extended->pages_dirtied = 0;
-	    extended->external_pager = 0;
-	    extended->shadow_depth = 0;
+		extended->protection = entry->protection;
+		extended->user_tag = entry->alias;
+		extended->pages_resident = 0;
+		extended->pages_swapped_out = 0;
+		extended->pages_shared_now_private = 0;
+		extended->pages_dirtied = 0;
+		extended->external_pager = 0;
+		extended->shadow_depth = 0;
 
-	    vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, extended);
+		vm_map_region_walk(map, start, entry, entry->offset, entry->vme_end - start, extended, TRUE);
 
-	    if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
-	            extended->share_mode = SM_PRIVATE;
+		if (extended->external_pager && extended->ref_count == 2 && extended->share_mode == SM_SHARED)
+			extended->share_mode = SM_PRIVATE;
 
-	    if (object_name)
-	        *object_name = IP_NULL;
-	    *address = start;
-	    *size = (entry->vme_end - start);
+		if (object_name)
+			*object_name = IP_NULL;
+		*address = start;
+		*size = (entry->vme_end - start);
 
-	    vm_map_unlock_read(map);
-	    return(KERN_SUCCESS);
+		vm_map_unlock_read(map);
+		return(KERN_SUCCESS);
 	}
 	case VM_REGION_TOP_INFO:
 	{   
-	    vm_region_top_info_t	top;
+		vm_region_top_info_t	top;
 
-	    if (*count < VM_REGION_TOP_INFO_COUNT)
-		return(KERN_INVALID_ARGUMENT);
+		if (*count < VM_REGION_TOP_INFO_COUNT)
+			return(KERN_INVALID_ARGUMENT);
 
-	    top = (vm_region_top_info_t) info;
-	    *count = VM_REGION_TOP_INFO_COUNT;
+		top = (vm_region_top_info_t) info;
+		*count = VM_REGION_TOP_INFO_COUNT;
 
-	    vm_map_lock_read(map);
+		vm_map_lock_read(map);
 
-	    start = *address;
-	    if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
-		if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
-			vm_map_unlock_read(map);
-		   	return(KERN_INVALID_ADDRESS);
-		}
-	    } else {
-		entry = tmp_entry;
+		start = *address;
+		if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
+			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {
+				vm_map_unlock_read(map);
+				return(KERN_INVALID_ADDRESS);
+			}
+		} else {
+			entry = tmp_entry;
 
-	    }
-	    start = entry->vme_start;
+		}
+		start = entry->vme_start;
 
-	    top->private_pages_resident = 0;
-	    top->shared_pages_resident = 0;
+		top->private_pages_resident = 0;
+		top->shared_pages_resident = 0;
 
-	    vm_map_region_top_walk(entry, top);
+		vm_map_region_top_walk(entry, top);
 
-	    if (object_name)
-	        *object_name = IP_NULL;
-	    *address = start;
-	    *size = (entry->vme_end - start);
+		if (object_name)
+			*object_name = IP_NULL;
+		*address = start;
+		*size = (entry->vme_end - start);
 
-	    vm_map_unlock_read(map);
-	    return(KERN_SUCCESS);
+		vm_map_unlock_read(map);
+		return(KERN_SUCCESS);
 	}
 	default:
-	    return(KERN_INVALID_ARGUMENT);
+		return(KERN_INVALID_ARGUMENT);
 	}
 }
 
+#define min(a, b)  (((a) < (b)) ? (a) : (b))
+
 void
 vm_map_region_top_walk(
         vm_map_entry_t		   entry,
 	vm_region_top_info_t       top)
 {
-        register struct vm_object *obj, *tmp_obj;
-	register int    ref_count;
 
 	if (entry->object.vm_object == 0 || entry->is_sub_map) {
-	    top->share_mode = SM_EMPTY;
-	    top->ref_count = 0;
-	    top->obj_id = 0;
-	    return;
+		top->share_mode = SM_EMPTY;
+		top->ref_count = 0;
+		top->obj_id = 0;
+		return;
 	}
+
 	{
-	    obj = entry->object.vm_object;
+	        struct	vm_object *obj, *tmp_obj;
+		int		ref_count;
+		uint32_t	entry_size;
 
-	    vm_object_lock(obj);
+		entry_size = (entry->vme_end - entry->vme_start) / PAGE_SIZE;
 
-	    if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
-	        ref_count--;
+		obj = entry->object.vm_object;
 
-	    if (obj->shadow) {
-		if (ref_count == 1)
-		    top->private_pages_resident = obj->resident_page_count;
-		else
-		    top->shared_pages_resident = obj->resident_page_count;
-		top->ref_count  = ref_count;
-	        top->share_mode = SM_COW;
+		vm_object_lock(obj);
+
+		if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+			ref_count--;
+
+		if (obj->shadow) {
+			if (ref_count == 1)
+				top->private_pages_resident = min(obj->resident_page_count, entry_size);
+			else
+				top->shared_pages_resident = min(obj->resident_page_count, entry_size);
+			top->ref_count  = ref_count;
+			top->share_mode = SM_COW;
 	    
-	        while ((tmp_obj = obj->shadow)) {
-		    vm_object_lock(tmp_obj);
-		    vm_object_unlock(obj);
-		    obj = tmp_obj;
+			while ((tmp_obj = obj->shadow)) {
+				vm_object_lock(tmp_obj);
+				vm_object_unlock(obj);
+				obj = tmp_obj;
 
-		    if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
-		        ref_count--;
+				if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+					ref_count--;
 
-		    top->shared_pages_resident += obj->resident_page_count;
-		    top->ref_count += ref_count - 1;
-		}
-	    } else {
-	        if (entry->needs_copy) {
-		    top->share_mode = SM_COW;
-		    top->shared_pages_resident = obj->resident_page_count;
+				top->shared_pages_resident += min(obj->resident_page_count, entry_size);
+				top->ref_count += ref_count - 1;
+			}
 		} else {
-		    if (ref_count == 1 ||
-		       (ref_count == 2 && !(obj->pager_trusted) && !(obj->internal))) {
-		        top->share_mode = SM_PRIVATE;
-			top->private_pages_resident = obj->resident_page_count;
-		    } else {
-		        top->share_mode = SM_SHARED;
-			top->shared_pages_resident = obj->resident_page_count;
-		    }
+			if (entry->needs_copy) {
+				top->share_mode = SM_COW;
+				top->shared_pages_resident = min(obj->resident_page_count, entry_size);
+			} else {
+				if (ref_count == 1 ||
+				    (ref_count == 2 && !(obj->pager_trusted) && !(obj->internal))) {
+					top->share_mode = SM_PRIVATE;
+					top->private_pages_resident = min(obj->resident_page_count, entry_size);
+				} else {
+					top->share_mode = SM_SHARED;
+					top->shared_pages_resident = min(obj->resident_page_count, entry_size);
+				}
+			}
+			top->ref_count = ref_count;
 		}
-		top->ref_count = ref_count;
-	    }
-	    top->obj_id = (int)obj;
+		top->obj_id = (int)obj;
 
-	    vm_object_unlock(obj);
+		vm_object_unlock(obj);
 	}
 }
 
 void
 vm_map_region_walk(
 	vm_map_t		   	map,
-	vm_map_offset_t		va,
-	vm_map_entry_t		entry,
+	vm_map_offset_t			va,
+	vm_map_entry_t			entry,
 	vm_object_offset_t		offset,
 	vm_object_size_t		range,
-	vm_region_extended_info_t	extended)
+	vm_region_extended_info_t	extended,
+	boolean_t			look_for_pages)
 {
         register struct vm_object *obj, *tmp_obj;
 	register vm_map_offset_t       last_offset;
@@ -8123,89 +8906,95 @@ vm_map_region_walk(
 	int			shadow_depth;
 
 	if ((entry->object.vm_object == 0) ||
-	        (entry->is_sub_map) ||
-		(entry->object.vm_object->phys_contiguous)) {
-	    extended->share_mode = SM_EMPTY;
-	    extended->ref_count = 0;
-	    return;
+	    (entry->is_sub_map) ||
+	    (entry->object.vm_object->phys_contiguous)) {
+		extended->share_mode = SM_EMPTY;
+		extended->ref_count = 0;
+		return;
 	}
 	{
-	    obj = entry->object.vm_object;
-
-	    vm_object_lock(obj);
-
-	    if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
-	        ref_count--;
-
-	    for (last_offset = offset + range; offset < last_offset; offset += PAGE_SIZE_64, va += PAGE_SIZE)
-		    vm_map_region_look_for_page(map, va, obj, offset, ref_count, 0, extended);
-
-	    shadow_object = obj->shadow;
-	    shadow_depth = 0;
-	    if (shadow_object != VM_OBJECT_NULL) {
-		    vm_object_lock(shadow_object);
-		    for (;
-			 shadow_object != VM_OBJECT_NULL;
-			 shadow_depth++) {
-			    vm_object_t	next_shadow;
-
-			    next_shadow = shadow_object->shadow;
-			    if (next_shadow) {
-				    vm_object_lock(next_shadow);
-			    }
-			    vm_object_unlock(shadow_object);
-			    shadow_object = next_shadow;
-		    }
-	    }
-	    extended->shadow_depth = shadow_depth;
-
-	    if (extended->shadow_depth || entry->needs_copy)
-	        extended->share_mode = SM_COW;
-	    else {
-	        if (ref_count == 1)
-		    extended->share_mode = SM_PRIVATE;
+		obj = entry->object.vm_object;
+
+		vm_object_lock(obj);
+
+		if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+			ref_count--;
+
+		if (look_for_pages) {
+			for (last_offset = offset + range;
+			     offset < last_offset;
+			     offset += PAGE_SIZE_64, va += PAGE_SIZE)
+				vm_map_region_look_for_page(map, va, obj,
+							    offset, ref_count,
+							    0, extended);
+		}
+
+		shadow_object = obj->shadow;
+		shadow_depth = 0;
+		if (shadow_object != VM_OBJECT_NULL) {
+			vm_object_lock(shadow_object);
+			for (;
+			     shadow_object != VM_OBJECT_NULL;
+			     shadow_depth++) {
+				vm_object_t	next_shadow;
+
+				next_shadow = shadow_object->shadow;
+				if (next_shadow) {
+					vm_object_lock(next_shadow);
+				}
+				vm_object_unlock(shadow_object);
+				shadow_object = next_shadow;
+			}
+		}
+		extended->shadow_depth = shadow_depth;
+
+		if (extended->shadow_depth || entry->needs_copy)
+			extended->share_mode = SM_COW;
 		else {
-	            if (obj->true_share)
-		        extended->share_mode = SM_TRUESHARED;
-		    else
-		        extended->share_mode = SM_SHARED;
+			if (ref_count == 1)
+				extended->share_mode = SM_PRIVATE;
+			else {
+				if (obj->true_share)
+					extended->share_mode = SM_TRUESHARED;
+				else
+					extended->share_mode = SM_SHARED;
+			}
 		}
-	    }
-	    extended->ref_count = ref_count - extended->shadow_depth;
+		extended->ref_count = ref_count - extended->shadow_depth;
 	    
-	    for (i = 0; i < extended->shadow_depth; i++) {
-	        if ((tmp_obj = obj->shadow) == 0)
-		    break;
-		vm_object_lock(tmp_obj);
-		vm_object_unlock(obj);
+		for (i = 0; i < extended->shadow_depth; i++) {
+			if ((tmp_obj = obj->shadow) == 0)
+				break;
+			vm_object_lock(tmp_obj);
+			vm_object_unlock(obj);
 
-		if ((ref_count = tmp_obj->ref_count) > 1 && tmp_obj->paging_in_progress)
-		    ref_count--;
+			if ((ref_count = tmp_obj->ref_count) > 1 && tmp_obj->paging_in_progress)
+				ref_count--;
 
-		extended->ref_count += ref_count;
-		obj = tmp_obj;
-	    }
-	    vm_object_unlock(obj);
+			extended->ref_count += ref_count;
+			obj = tmp_obj;
+		}
+		vm_object_unlock(obj);
 
-	    if (extended->share_mode == SM_SHARED) {
-	        register vm_map_entry_t	     cur;
-	        register vm_map_entry_t	     last;
-		int      my_refs;
+		if (extended->share_mode == SM_SHARED) {
+			register vm_map_entry_t	     cur;
+			register vm_map_entry_t	     last;
+			int      my_refs;
 
-	        obj = entry->object.vm_object;
-		last = vm_map_to_entry(map);
-		my_refs = 0;
+			obj = entry->object.vm_object;
+			last = vm_map_to_entry(map);
+			my_refs = 0;
 
-		if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
-		        ref_count--;
-		for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
-		    my_refs += vm_map_region_count_obj_refs(cur, obj);
+			if ((ref_count = obj->ref_count) > 1 && obj->paging_in_progress)
+				ref_count--;
+			for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next)
+				my_refs += vm_map_region_count_obj_refs(cur, obj);
 
-		if (my_refs == ref_count)
-		    extended->share_mode = SM_PRIVATE_ALIASED;
-		else if (my_refs > 1)
-		    extended->share_mode = SM_SHARED_ALIASED;
-	    }
+			if (my_refs == ref_count)
+				extended->share_mode = SM_PRIVATE_ALIASED;
+			else if (my_refs > 1)
+				extended->share_mode = SM_SHARED_ALIASED;
+		}
 	}
 }
 
@@ -8216,18 +9005,20 @@ vm_map_region_walk(
 static void
 vm_map_region_look_for_page(
 	__unused vm_map_t		map,
-	__unused vm_map_offset_t    	va,
-	vm_object_t		   	object,
-	vm_object_offset_t   	offset,
+	__unused vm_map_offset_t	va,
+	vm_object_t			object,
+	vm_object_offset_t		offset,
 	int				max_refcnt,
 	int				depth,
 	vm_region_extended_info_t	extended)
 {
-        register vm_page_t	   p;
-        register vm_object_t	   shadow;
-	register int               ref_count;
-	vm_object_t		   caller_object;
-        
+        register vm_page_t	p;
+        register vm_object_t	shadow;
+	register int            ref_count;
+	vm_object_t		caller_object;
+#if	MACH_PAGEMAP
+	kern_return_t		kr;
+#endif
 	shadow = object->shadow;
 	caller_object = object;
 
@@ -8235,7 +9026,7 @@ vm_map_region_look_for_page(
 	while (TRUE) {
 
 		if ( !(object->pager_trusted) && !(object->internal))
-			    extended->external_pager = 1;
+			extended->external_pager = 1;
 
 		if ((p = vm_page_lookup(object, offset)) != VM_PAGE_NULL) {
 	        	if (shadow && (max_refcnt == 1))
@@ -8248,23 +9039,54 @@ vm_map_region_look_for_page(
 	        	extended->pages_resident++;
 
 			if(object != caller_object)
-			     vm_object_unlock(object);
+				vm_object_unlock(object);
 
 			return;
 		}
+#if	MACH_PAGEMAP
 		if (object->existence_map) {
 	    		if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_EXISTS) {
 
 	        		extended->pages_swapped_out++;
 
 				if(object != caller_object)
-				     vm_object_unlock(object);
+					vm_object_unlock(object);
 
 				return;
 	    		}
+		} else if (object->internal &&
+			   object->alive &&
+			   !object->terminating &&
+			   object->pager_ready) {
+
+			memory_object_t pager;
+
+			vm_object_paging_begin(object);
+			pager = object->pager;
+			vm_object_unlock(object);
+
+			kr = memory_object_data_request(
+				pager,
+				offset + object->paging_offset,
+				0, /* just poke the pager */
+				VM_PROT_READ,
+				NULL);
+
+			vm_object_lock(object);
+			vm_object_paging_end(object);
+
+			if (kr == KERN_SUCCESS) {
+				/* the pager has that page */
+				extended->pages_swapped_out++;
+				if (object != caller_object)
+					vm_object_unlock(object);
+				return;
+			}
 		}
+#endif /* MACH_PAGEMAP */
+
 		if (shadow) {
-		    vm_object_lock(shadow);
+			vm_object_lock(shadow);
 
 			if ((ref_count = shadow->ref_count) > 1 && shadow->paging_in_progress)
 			        ref_count--;
@@ -8276,7 +9098,7 @@ vm_map_region_look_for_page(
 	        		max_refcnt = ref_count;
 			
 			if(object != caller_object)
-			     vm_object_unlock(object);
+				vm_object_unlock(object);
 
 			offset = offset + object->shadow_offset;
 			object = shadow;
@@ -8284,7 +9106,7 @@ vm_map_region_look_for_page(
 			continue;
 		}
 		if(object != caller_object)
-		     vm_object_unlock(object);
+			vm_object_unlock(object);
 		break;
 	}
 }
@@ -8299,26 +9121,26 @@ vm_map_region_count_obj_refs(
 	register vm_object_t tmp_obj;
 
 	if (entry->object.vm_object == 0)
-	    return(0);
+		return(0);
 
         if (entry->is_sub_map)
-	    return(0);
+		return(0);
 	else {
-	    ref_count = 0;
+		ref_count = 0;
 
-	    chk_obj = entry->object.vm_object;
-	    vm_object_lock(chk_obj);
+		chk_obj = entry->object.vm_object;
+		vm_object_lock(chk_obj);
 
-	    while (chk_obj) {
-	        if (chk_obj == object)
-		    ref_count++;
-		tmp_obj = chk_obj->shadow;
-		if (tmp_obj)
-		   vm_object_lock(tmp_obj);
-		vm_object_unlock(chk_obj);
+		while (chk_obj) {
+			if (chk_obj == object)
+				ref_count++;
+			tmp_obj = chk_obj->shadow;
+			if (tmp_obj)
+				vm_object_lock(tmp_obj);
+			vm_object_unlock(chk_obj);
 
-		chk_obj = tmp_obj;
-	    }
+			chk_obj = tmp_obj;
+		}
 	}
 	return(ref_count);
 }
@@ -8349,12 +9171,11 @@ vm_map_simplify_entry(
 	prev_entry = this_entry->vme_prev;
 
 	if ((this_entry != vm_map_to_entry(map)) &&
-	     (prev_entry != vm_map_to_entry(map)) &&
+	    (prev_entry != vm_map_to_entry(map)) &&
 
 	    (prev_entry->vme_end == this_entry->vme_start) &&
 
-	    (prev_entry->is_sub_map == FALSE) &&
-	    (this_entry->is_sub_map == FALSE) &&
+	    (prev_entry->is_sub_map == this_entry->is_sub_map) &&
 
 	    (prev_entry->object.vm_object == this_entry->object.vm_object) &&
 	    ((prev_entry->offset + (prev_entry->vme_end -
@@ -8366,6 +9187,7 @@ vm_map_simplify_entry(
 	    (prev_entry->max_protection == this_entry->max_protection) &&
 	    (prev_entry->behavior == this_entry->behavior) &&
 	    (prev_entry->alias == this_entry->alias) &&
+	    (prev_entry->no_cache == this_entry->no_cache) &&
 	    (prev_entry->wired_count == this_entry->wired_count) &&
 	    (prev_entry->user_wired_count == this_entry->user_wired_count) &&
 
@@ -8379,11 +9201,15 @@ vm_map_simplify_entry(
 	    (this_entry->needs_wakeup == FALSE) &&
 	    (prev_entry->is_shared == FALSE) &&
 	    (this_entry->is_shared == FALSE)
-		    ) {
+		) {
 		_vm_map_entry_unlink(&map->hdr, prev_entry);
 		this_entry->vme_start = prev_entry->vme_start;
 		this_entry->offset = prev_entry->offset;
-	 	vm_object_deallocate(prev_entry->object.vm_object);
+		if (prev_entry->is_sub_map) {
+			vm_map_deallocate(prev_entry->object.sub_map);
+		} else {
+			vm_object_deallocate(prev_entry->object.vm_object);
+		}
 		vm_map_entry_dispose(map, prev_entry);
 		SAVE_HINT_MAP_WRITE(map, this_entry);
 		counter(c_vm_map_simplified++);
@@ -8423,6 +9249,9 @@ vm_map_simplify_range(
 		return;
 	}
 
+	start = vm_map_trunc_page(start);
+	end = vm_map_round_page(end);
+
 	if (!vm_map_lookup_entry(map, start, &entry)) {
 		/* "start" is not mapped and "entry" ends before "start" */
 		if (entry == vm_map_to_entry(map)) {
@@ -8496,14 +9325,14 @@ vm_map_machine_attribute(
 				sync_size = 0;
 			} else {
 				sub_size = entry->vme_end - start;
-					sync_size -= sub_size;
+				sync_size -= sub_size;
 			}
 			if(entry->is_sub_map) {
 				vm_map_offset_t sub_start;
 				vm_map_offset_t sub_end;
 
 				sub_start = (start - entry->vme_start) 
-					    + entry->offset;
+					+ entry->offset;
 				sub_end = sub_start + sub_size;
 				vm_map_machine_attribute(
 					entry->object.sub_map, 
@@ -8521,7 +9350,7 @@ vm_map_machine_attribute(
 					vm_map_size_t		range;
 					range = sub_size;
 					offset = (start - entry->vme_start)
-							+ entry->offset;
+						+ entry->offset;
 					base_offset = offset;
 					object = entry->object.vm_object;
 					base_object = object;
@@ -8535,10 +9364,10 @@ vm_map_machine_attribute(
 
 						if (m && !m->fictitious) {
 						        ret = 
-							    pmap_attribute_cache_sync(
-										      m->phys_page, 	
-										      PAGE_SIZE, 
-										      attribute, value);
+								pmap_attribute_cache_sync(
+									m->phys_page, 	
+									PAGE_SIZE, 
+									attribute, value);
 							
 						} else if (object->shadow) {
 						        offset = offset + object->shadow_offset;
@@ -8594,8 +9423,8 @@ vm_map_behavior_set(
 	vm_map_entry_t	temp_entry;
 
 	XPR(XPR_VM_MAP,
-		"vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
-		(integer_t)map, start, end, new_behavior, 0);
+	    "vm_map_behavior_set, 0x%X start 0x%X end 0x%X behavior %d",
+	    (integer_t)map, start, end, new_behavior, 0);
 
 	switch (new_behavior) {
 	case VM_BEHAVIOR_DEFAULT:
@@ -8631,6 +9460,7 @@ vm_map_behavior_set(
 
 	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {
 		vm_map_clip_end(map, entry, end);
+		assert(!entry->use_pmap);
 
 		entry->behavior = new_behavior;
 
@@ -8653,19 +9483,19 @@ vm_map_behavior_set(
  * Forward declarations for internal functions.
  */
 extern void vm_map_links_print(
-		struct vm_map_links	*links);
+	struct vm_map_links	*links);
 
 extern void vm_map_header_print(
-		struct vm_map_header	*header);
+	struct vm_map_header	*header);
 
 extern void vm_map_entry_print(
-		vm_map_entry_t		entry);
+	vm_map_entry_t		entry);
 
 extern void vm_follow_entry(
-		vm_map_entry_t		entry);
+	vm_map_entry_t		entry);
 
 extern void vm_follow_map(
-		vm_map_t		map);
+	vm_map_t		map);
 
 /*
  *	vm_map_links_print:	[ debug ]
@@ -8748,8 +9578,8 @@ vm_map_entry_print(
 
 	if (entry->is_sub_map) {
 		iprintf("submap = %08X - offset = %016llX\n",
-		       entry->object.sub_map,
-		       (unsigned long long)entry->offset);
+			entry->object.sub_map,
+			(unsigned long long)entry->offset);
 	} else {
 		iprintf("object = %08X  offset = %016llX - ",
 			entry->object.vm_object,
@@ -8778,7 +9608,7 @@ vm_follow_map(
 	for (entry = vm_map_first_entry(map);
 	     entry && entry != vm_map_to_entry(map);
 	     entry = entry->vme_next) {
-	    vm_follow_entry(entry);
+		vm_follow_entry(entry);
 	}
 
 	db_indent -= 2;
@@ -8820,13 +9650,13 @@ vm_map_print(
 
 #if	TASK_SWAPPER
 	switch (map->sw_state) {
-	    case MAP_SW_IN:
+	case MAP_SW_IN:
 		swstate = "SW_IN";
 		break;
-	    case MAP_SW_OUT:
+	case MAP_SW_OUT:
 		swstate = "SW_OUT";
 		break;
-	    default:
+	default:
 		swstate = "????";
 		break;
 	}
@@ -8864,19 +9694,19 @@ vm_map_copy_print(
 
 	iprintf("type=%d", copy->type);
 	switch (copy->type) {
-		case VM_MAP_COPY_ENTRY_LIST:
+	case VM_MAP_COPY_ENTRY_LIST:
 		printf("[entry_list]");
 		break;
 		
-		case VM_MAP_COPY_OBJECT:
+	case VM_MAP_COPY_OBJECT:
 		printf("[object]");
 		break;
 		
-		case VM_MAP_COPY_KERNEL_BUFFER:
+	case VM_MAP_COPY_KERNEL_BUFFER:
 		printf("[kernel_buffer]");
 		break;
 
-		default:
+	default:
 		printf("[bad type]");
 		break;
 	}
@@ -8884,7 +9714,7 @@ vm_map_copy_print(
 	printf(", size=0x%x\n", copy->size);
 
 	switch (copy->type) {
-		case VM_MAP_COPY_ENTRY_LIST:
+	case VM_MAP_COPY_ENTRY_LIST:
 		vm_map_header_print(&copy->cpy_hdr);
 		for (entry = vm_map_copy_first_entry(copy);
 		     entry && entry != vm_map_copy_to_entry(copy);
@@ -8893,11 +9723,11 @@ vm_map_copy_print(
 		}
 		break;
 
-		case VM_MAP_COPY_OBJECT:
+	case VM_MAP_COPY_OBJECT:
 		iprintf("object=0x%x\n", copy->cpy_object);
 		break;
 
-		case VM_MAP_COPY_KERNEL_BUFFER:
+	case VM_MAP_COPY_KERNEL_BUFFER:
 		iprintf("kernel buffer=0x%x", copy->cpy_kdata);
 		printf(", kalloc_size=0x%x\n", copy->cpy_kalloc_size);
 		break;
@@ -8955,7 +9785,8 @@ vm_map_entry_insert(
 	vm_prot_t		max_protection,
 	vm_behavior_t		behavior,
 	vm_inherit_t		inheritance,
-	unsigned		wired_count)
+	unsigned		wired_count,
+	boolean_t		no_cache)
 {
 	vm_map_entry_t	new_entry;
 
@@ -8983,6 +9814,7 @@ vm_map_entry_insert(
 	new_entry->user_wired_count = 0;
 	new_entry->use_pmap = FALSE;
 	new_entry->alias = 0;
+	new_entry->no_cache = no_cache;
 
 	/*
 	 *	Insert the new entry into the list.
@@ -9090,7 +9922,7 @@ vm_map_remap_extract(
 			tmp_size -= (src_end - src_entry->vme_end);
 
 		entry_size = (vm_map_size_t)(src_entry->vme_end -
-					 src_entry->vme_start);
+					     src_entry->vme_start);
 
 		if(src_entry->is_sub_map) {
 			vm_map_reference(src_entry->object.sub_map);
@@ -9112,7 +9944,7 @@ vm_map_remap_extract(
 				assert(!src_entry->needs_copy);
 			} else if (src_entry->needs_copy || object->shadowed ||
 				   (object->internal && !object->true_share &&
-				   !src_entry->is_shared &&
+				    !src_entry->is_shared &&
 				    object->size > entry_size)) {
 
 				vm_object_shadow(&src_entry->object.vm_object,
@@ -9124,23 +9956,23 @@ vm_map_remap_extract(
 				        vm_prot_t prot;
 
 				        prot = src_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-					if (src_entry->alias != VM_MEMORY_STACK && prot)
+
+					if (override_nx(map, src_entry->alias) && prot)
 					        prot |= VM_PROT_EXECUTE;
-#endif
+
 					if(map->mapped) {
-					   vm_object_pmap_protect(
-						src_entry->object.vm_object,
-						src_entry->offset,
-						entry_size,
-						PMAP_NULL,
-						src_entry->vme_start,
-						prot);
-					} else {
-					   pmap_protect(vm_map_pmap(map),
+						vm_object_pmap_protect(
+							src_entry->object.vm_object,
+							src_entry->offset,
+							entry_size,
+							PMAP_NULL,
 							src_entry->vme_start,
-							src_entry->vme_end,
 							prot);
+					} else {
+						pmap_protect(vm_map_pmap(map),
+							     src_entry->vme_start,
+							     src_entry->vme_end,
+							     prot);
 					}
 				}
 
@@ -9150,10 +9982,9 @@ vm_map_remap_extract(
 
 
 			vm_object_lock(object);
-			object->ref_count++;	/* object ref. for new entry */
-			VM_OBJ_RES_INCR(object);
+			vm_object_reference_locked(object); /* object ref. for new entry */
 			if (object->copy_strategy == 
-					MEMORY_OBJECT_COPY_SYMMETRIC) {
+			    MEMORY_OBJECT_COPY_SYMMETRIC) {
 				object->copy_strategy = 
 					MEMORY_OBJECT_COPY_DELAY;
 			}
@@ -9186,12 +10017,12 @@ vm_map_remap_extract(
 			new_entry->needs_copy = TRUE;
 			object = VM_OBJECT_NULL;
 		} else if (src_entry->wired_count == 0 &&
-			 vm_object_copy_quickly(&new_entry->object.vm_object,
-						new_entry->offset,
-						(new_entry->vme_end -
-						    new_entry->vme_start),
-						&src_needs_copy,
-						&new_entry_needs_copy)) {
+			   vm_object_copy_quickly(&new_entry->object.vm_object,
+						  new_entry->offset,
+						  (new_entry->vme_end -
+						   new_entry->vme_start),
+						  &src_needs_copy,
+						  &new_entry_needs_copy)) {
 
 			new_entry->needs_copy = new_entry_needs_copy;
 			new_entry->is_shared = FALSE;
@@ -9203,15 +10034,15 @@ vm_map_remap_extract(
 			        vm_prot_t prot;
 
 				prot = src_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-				if (src_entry->alias != VM_MEMORY_STACK && prot)
+
+				if (override_nx(map, src_entry->alias) && prot)
 				        prot |= VM_PROT_EXECUTE;
-#endif
+
 				vm_object_pmap_protect(object,
 						       offset,
 						       entry_size,
 						       ((src_entry->is_shared 
-							  || map->mapped) ?
+							 || map->mapped) ?
 							PMAP_NULL : map->pmap),
 						       src_entry->vme_start,
 						       prot);
@@ -9242,22 +10073,22 @@ vm_map_remap_extract(
 			if (src_entry->wired_count > 0) {
 				vm_object_lock(object);
 				result = vm_object_copy_slowly(
-						object,
-						offset,
-						entry_size,
-						THREAD_UNINT,
-						&new_entry->object.vm_object);
+					object,
+					offset,
+					entry_size,
+					THREAD_UNINT,
+					&new_entry->object.vm_object);
 
 				new_entry->offset = 0;
 				new_entry->needs_copy = FALSE;
 			} else {
 				result = vm_object_copy_strategically(
-						object,
-						offset,
-						entry_size,
-						&new_entry->object.vm_object,
-						&new_entry->offset,
-						&new_entry_needs_copy);
+					object,
+					offset,
+					entry_size,
+					&new_entry->object.vm_object,
+					&new_entry->offset,
+					&new_entry_needs_copy);
 
 				new_entry->needs_copy = new_entry_needs_copy;
 			}
@@ -9364,25 +10195,25 @@ vm_map_remap(
 		return KERN_INVALID_ARGUMENT;
 
 	switch (inheritance) {
-	    case VM_INHERIT_NONE:
-	    case VM_INHERIT_COPY:
-	    case VM_INHERIT_SHARE:
+	case VM_INHERIT_NONE:
+	case VM_INHERIT_COPY:
+	case VM_INHERIT_SHARE:
 		if (size != 0 && src_map != VM_MAP_NULL)
 			break;
 		/*FALL THRU*/
-	    default:
+	default:
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	size = vm_map_round_page(size);
 
 	result = vm_map_remap_extract(src_map, memory_address,
-				size, copy, &map_header,
-				cur_protection,
-				max_protection,
-				inheritance,
-				target_map->hdr.
-				entries_pageable);
+				      size, copy, &map_header,
+				      cur_protection,
+				      max_protection,
+				      inheritance,
+				      target_map->hdr.
+				      entries_pageable);
 
 	if (result != KERN_SUCCESS) {
 		return result;
@@ -9395,7 +10226,7 @@ vm_map_remap(
 	*address = vm_map_trunc_page(*address);
 	vm_map_lock(target_map);
 	result = vm_map_remap_range_allocate(target_map, address, size,
-					 mask, anywhere, &insp_entry);
+					     mask, anywhere, &insp_entry);
 
 	for (entry = map_header.links.next;
 	     entry != (struct vm_map_entry *)&map_header.links;
@@ -9412,7 +10243,7 @@ vm_map_remap(
 				vm_object_deallocate(entry->object.vm_object);
 			} else {
 				vm_map_deallocate(entry->object.sub_map);
-			   }
+			}
 			_vm_map_entry_dispose(&map_header, entry);
 		}
 	}
@@ -9453,143 +10284,143 @@ vm_map_remap_range_allocate(
 	register vm_map_offset_t	start;
 	register vm_map_offset_t	end;
 
- StartAgain: ;
+StartAgain: ;
 
-    start = *address;
+	start = *address;
 
-    if (anywhere)
-    {
-	/*
-	 *	Calculate the first possible address.
-	 */
+	if (anywhere)
+	{
+		/*
+		 *	Calculate the first possible address.
+		 */
 
-	if (start < map->min_offset)
-	    start = map->min_offset;
-	if (start > map->max_offset)
-	    return(KERN_NO_SPACE);
+		if (start < map->min_offset)
+			start = map->min_offset;
+		if (start > map->max_offset)
+			return(KERN_NO_SPACE);
 		
-	/*
-	 *	Look for the first possible address;
-	 *	if there's already something at this
-	 *	address, we have to start after it.
-	 */
+		/*
+		 *	Look for the first possible address;
+		 *	if there's already something at this
+		 *	address, we have to start after it.
+		 */
 
-	assert(first_free_is_valid(map));
-	if (start == map->min_offset) {
-	    if ((entry = map->first_free) != vm_map_to_entry(map))
-		start = entry->vme_end;
-	} else {
-	    vm_map_entry_t	tmp_entry;
-	    if (vm_map_lookup_entry(map, start, &tmp_entry))
-		start = tmp_entry->vme_end;
-	    entry = tmp_entry;
-	}
+		assert(first_free_is_valid(map));
+		if (start == map->min_offset) {
+			if ((entry = map->first_free) != vm_map_to_entry(map))
+				start = entry->vme_end;
+		} else {
+			vm_map_entry_t	tmp_entry;
+			if (vm_map_lookup_entry(map, start, &tmp_entry))
+				start = tmp_entry->vme_end;
+			entry = tmp_entry;
+		}
 		
-	/*
-	 *	In any case, the "entry" always precedes
-	 *	the proposed new region throughout the
-	 *	loop:
-	 */
+		/*
+		 *	In any case, the "entry" always precedes
+		 *	the proposed new region throughout the
+		 *	loop:
+		 */
 
-	while (TRUE) {
-	    register vm_map_entry_t	next;
-
-	    /*
-	     *	Find the end of the proposed new region.
-	     *	Be sure we didn't go beyond the end, or
-	     *	wrap around the address.
-	     */
-
-	    end = ((start + mask) & ~mask);
-	    if (end < start)
-		    return(KERN_NO_SPACE);
-	    start = end;
-	    end += size;
-
-	    if ((end > map->max_offset) || (end < start)) {
-		if (map->wait_for_space) {
-		    if (size <= (map->max_offset -
-				 map->min_offset)) {
-			assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
-			vm_map_unlock(map);
-			thread_block(THREAD_CONTINUE_NULL);
-			vm_map_lock(map);
-			goto StartAgain;
-		    }
-		}
+		while (TRUE) {
+			register vm_map_entry_t	next;
+
+			/*
+			 *	Find the end of the proposed new region.
+			 *	Be sure we didn't go beyond the end, or
+			 *	wrap around the address.
+			 */
+
+			end = ((start + mask) & ~mask);
+			if (end < start)
+				return(KERN_NO_SPACE);
+			start = end;
+			end += size;
+
+			if ((end > map->max_offset) || (end < start)) {
+				if (map->wait_for_space) {
+					if (size <= (map->max_offset -
+						     map->min_offset)) {
+						assert_wait((event_t) map, THREAD_INTERRUPTIBLE);
+						vm_map_unlock(map);
+						thread_block(THREAD_CONTINUE_NULL);
+						vm_map_lock(map);
+						goto StartAgain;
+					}
+				}
 		
-		return(KERN_NO_SPACE);
-	    }
+				return(KERN_NO_SPACE);
+			}
 
-	    /*
-	     *	If there are no more entries, we must win.
-	     */
+			/*
+			 *	If there are no more entries, we must win.
+			 */
 
-	    next = entry->vme_next;
-	    if (next == vm_map_to_entry(map))
-		break;
+			next = entry->vme_next;
+			if (next == vm_map_to_entry(map))
+				break;
 
-	    /*
-	     *	If there is another entry, it must be
-	     *	after the end of the potential new region.
-	     */
+			/*
+			 *	If there is another entry, it must be
+			 *	after the end of the potential new region.
+			 */
 
-	    if (next->vme_start >= end)
-		break;
+			if (next->vme_start >= end)
+				break;
 
-	    /*
-	     *	Didn't fit -- move to the next entry.
-	     */
+			/*
+			 *	Didn't fit -- move to the next entry.
+			 */
 
-	    entry = next;
-	    start = entry->vme_end;
-	}
-	*address = start;
-    } else {
-	vm_map_entry_t		temp_entry;
+			entry = next;
+			start = entry->vme_end;
+		}
+		*address = start;
+	} else {
+		vm_map_entry_t		temp_entry;
 	
-	/*
-	 *	Verify that:
-	 *		the address doesn't itself violate
-	 *		the mask requirement.
-	 */
+		/*
+		 *	Verify that:
+		 *		the address doesn't itself violate
+		 *		the mask requirement.
+		 */
 
-	if ((start & mask) != 0)
-	    return(KERN_NO_SPACE);
+		if ((start & mask) != 0)
+			return(KERN_NO_SPACE);
 
 
-	/*
-	 *	...	the address is within bounds
-	 */
+		/*
+		 *	...	the address is within bounds
+		 */
 
-	end = start + size;
+		end = start + size;
 
-	if ((start < map->min_offset) ||
-	    (end > map->max_offset) ||
-	    (start >= end)) {
-	    return(KERN_INVALID_ADDRESS);
-	}
+		if ((start < map->min_offset) ||
+		    (end > map->max_offset) ||
+		    (start >= end)) {
+			return(KERN_INVALID_ADDRESS);
+		}
 
-	/*
-	 *	...	the starting address isn't allocated
-	 */
+		/*
+		 *	...	the starting address isn't allocated
+		 */
 
-	if (vm_map_lookup_entry(map, start, &temp_entry))
-	    return(KERN_NO_SPACE);
+		if (vm_map_lookup_entry(map, start, &temp_entry))
+			return(KERN_NO_SPACE);
 
-	entry = temp_entry;
+		entry = temp_entry;
 
-	/*
-	 *	...	the next region doesn't overlap the
-	 *		end point.
-	 */
+		/*
+		 *	...	the next region doesn't overlap the
+		 *		end point.
+		 */
 
-	if ((entry->vme_next != vm_map_to_entry(map)) &&
-	    (entry->vme_next->vme_start < end))
-	    return(KERN_NO_SPACE);
-    }
-    *map_entry = entry;
-    return(KERN_SUCCESS);
+		if ((entry->vme_next != vm_map_to_entry(map)) &&
+		    (entry->vme_next->vme_start < end))
+			return(KERN_NO_SPACE);
+	}
+	*map_entry = entry;
+	return(KERN_SUCCESS);
 }
 
 /*
@@ -9711,21 +10542,19 @@ vm_map_read_user(
  *	privilege on the entire address region given.
  *	The entire region must be allocated.
  */
-boolean_t vm_map_check_protection(map, start, end, protection)
-	register vm_map_t	map;
-	register vm_map_offset_t	start;
-	register vm_map_offset_t	end;
-	register vm_prot_t	protection;
+boolean_t
+vm_map_check_protection(vm_map_t map, vm_map_offset_t start,
+			vm_map_offset_t end, vm_prot_t protection)
 {
-	register vm_map_entry_t	entry;
-	vm_map_entry_t		tmp_entry;
+	vm_map_entry_t entry;
+	vm_map_entry_t tmp_entry;
 
 	vm_map_lock(map);
 
-    if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
+	if (start < vm_map_min(map) || end > vm_map_max(map) || start > end)
 	{
-			vm_map_unlock(map);
-			return (FALSE);
+		vm_map_unlock(map);
+		return (FALSE);
 	}
 
 	if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
@@ -9791,8 +10620,8 @@ vm_map_purgable_control(
 		return(KERN_INVALID_ARGUMENT);
 
 	if (control == VM_PURGABLE_SET_STATE &&
-	    (*state < VM_PURGABLE_STATE_MIN ||
-	     *state > VM_PURGABLE_STATE_MAX))
+	    (((*state & ~(VM_PURGABLE_STATE_MASK|VM_VOLATILE_ORDER_MASK|VM_PURGABLE_ORDERING_MASK|VM_PURGABLE_BEHAVIOR_MASK|VM_VOLATILE_GROUP_MASK)) != 0) ||
+	     ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
 		return(KERN_INVALID_ARGUMENT);
 
 	vm_map_lock(map);
@@ -9847,57 +10676,119 @@ vm_map_purgable_control(
 
 kern_return_t
 vm_map_page_info(
-	vm_map_t		target_map,
+	vm_map_t	target_map,
 	vm_map_offset_t	offset,
-	int			*disposition,
-	int			*ref_count)
+	int		*disposition,
+	int		*ref_count)
 {
 	vm_map_entry_t	map_entry;
 	vm_object_t	object;
 	vm_page_t	m;
+	kern_return_t	kr;
+	kern_return_t	retval = KERN_SUCCESS;
+	boolean_t	top_object = TRUE;
 
-restart_page_query:
 	*disposition = 0;
 	*ref_count = 0;
-	vm_map_lock(target_map);
-	if(!vm_map_lookup_entry(target_map, offset, &map_entry)) {
-		vm_map_unlock(target_map);
+
+	vm_map_lock_read(target_map);
+
+restart_page_query:
+	if (!vm_map_lookup_entry(target_map, offset, &map_entry)) {
+		vm_map_unlock_read(target_map);
 		return KERN_FAILURE;
 	}
 	offset -= map_entry->vme_start;  /* adjust to offset within entry */
 	offset += map_entry->offset;	 /* adjust to target object offset */
-	if(map_entry->object.vm_object != VM_OBJECT_NULL) {
-		if(!map_entry->is_sub_map) {
+
+	if (map_entry->object.vm_object != VM_OBJECT_NULL) {
+		if (!map_entry->is_sub_map) {
 			object = map_entry->object.vm_object;
 		} else {
-			vm_map_unlock(target_map);
-			target_map = map_entry->object.sub_map;
+		        vm_map_t sub_map;
+
+			sub_map = map_entry->object.sub_map;
+			vm_map_lock_read(sub_map);
+			vm_map_unlock_read(target_map);
+
+			target_map = sub_map;
 			goto restart_page_query;
 		}
 	} else {
-		vm_map_unlock(target_map);
-		return KERN_FAILURE;
+		vm_map_unlock_read(target_map);
+		return KERN_SUCCESS;
 	}
 	vm_object_lock(object);
-	vm_map_unlock(target_map);
-	while(TRUE) {
+	vm_map_unlock_read(target_map);
+
+	while (TRUE) {
 		m = vm_page_lookup(object, offset);
+
 		if (m != VM_PAGE_NULL) {
 			*disposition |= VM_PAGE_QUERY_PAGE_PRESENT;
 			break;
 		} else {
-			if(object->shadow) {
-				offset += object->shadow_offset;
+#if MACH_PAGEMAP
+			if (object->existence_map) {
+				if (vm_external_state_get(object->existence_map, offset)
+				    == VM_EXTERNAL_STATE_EXISTS) {
+					/*
+					 * this page has been paged out
+					 */
+				        *disposition |= VM_PAGE_QUERY_PAGE_PAGED_OUT;
+					break;
+				}
+			} else
+#endif
+				if (object->internal &&
+				   object->alive &&
+				   !object->terminating &&
+				   object->pager_ready) {
+
+				memory_object_t pager;
+
+				vm_object_paging_begin(object);
+				pager = object->pager;
 				vm_object_unlock(object);
-				object = object->shadow;
+
+				kr = memory_object_data_request(
+					pager,
+					offset + object->paging_offset,
+					0,	/* just poke the pager */
+					VM_PROT_READ,
+					NULL);
+
 				vm_object_lock(object);
-				continue;
+				vm_object_paging_end(object);
+
+				if (kr == KERN_SUCCESS) {
+					/*
+					 * the pager has this page
+					 */
+				        *disposition |= VM_PAGE_QUERY_PAGE_PAGED_OUT;
+					break;
+				}
+			}
+			if (object->shadow != VM_OBJECT_NULL) {
+			        vm_object_t shadow;
+
+				offset += object->shadow_offset;
+				shadow = object->shadow;
+				
+				vm_object_lock(shadow);
+				vm_object_unlock(object);
+
+				object = shadow;
+				top_object = FALSE;
+			} else {
+			        if (!object->internal)
+				        break;
+
+				retval = KERN_FAILURE;
+				goto page_query_done;
 			}
-			vm_object_unlock(object);
-			return KERN_FAILURE;
 		}
 	}
-
 	/* The ref_count is not strictly accurate, it measures the number   */
 	/* of entities holding a ref on the object, they may not be mapping */
 	/* the object or may not be mapping the section holding the         */
@@ -9910,131 +10801,29 @@ restart_page_query:
 
 	*ref_count = object->ref_count;
 
+	if (top_object == TRUE && object->shadow)
+	        *disposition |= VM_PAGE_QUERY_PAGE_COPIED;
+
+	if (m == VM_PAGE_NULL)
+	        goto page_query_done;
+
 	if (m->fictitious) {
 		*disposition |= VM_PAGE_QUERY_PAGE_FICTITIOUS;
-		vm_object_unlock(object);
-		return KERN_SUCCESS;
+		goto page_query_done;
 	}
-
-	if (m->dirty)
-		*disposition |= VM_PAGE_QUERY_PAGE_DIRTY;
-	else if(pmap_is_modified(m->phys_page))
+	if (m->dirty || pmap_is_modified(m->phys_page))
 		*disposition |= VM_PAGE_QUERY_PAGE_DIRTY;
 
-	if (m->reference)
-		*disposition |= VM_PAGE_QUERY_PAGE_REF;
-	else if(pmap_is_referenced(m->phys_page))
+	if (m->reference || pmap_is_referenced(m->phys_page))
 		*disposition |= VM_PAGE_QUERY_PAGE_REF;
 
-	vm_object_unlock(object);
-	return KERN_SUCCESS;
-	
-}
-
-
-/* For a given range, check all map entries.  If the entry coresponds to */
-/* the old vm_region/map provided on the call, replace it with the 	 */
-/* corresponding range in the new vm_region/map */
-kern_return_t vm_map_region_replace(
-	vm_map_t	target_map,
-	ipc_port_t	old_region, 
-	ipc_port_t	new_region,
-	vm_map_offset_t	start,
-	vm_map_offset_t	end)
-{
-	vm_named_entry_t	old_object;
-	vm_named_entry_t	new_object;
-	vm_map_t		old_submap;
-	vm_map_t		new_submap;
-	vm_map_offset_t		addr;
-	vm_map_entry_t		entry;
-	int			nested_pmap = 0;
-
+	if (m->speculative)
+		*disposition |= VM_PAGE_QUERY_PAGE_SPECULATIVE;
 
-	vm_map_lock(target_map);
-	old_object = (vm_named_entry_t)old_region->ip_kobject;
-	new_object = (vm_named_entry_t)new_region->ip_kobject;
-	if((!old_object->is_sub_map) || (!new_object->is_sub_map)) {
-		vm_map_unlock(target_map);
-		return KERN_INVALID_ARGUMENT;
-	}
-	old_submap = (vm_map_t)old_object->backing.map;
-	new_submap = (vm_map_t)new_object->backing.map;
-	vm_map_lock(old_submap);
-	if((old_submap->min_offset != new_submap->min_offset) || 
-			(old_submap->max_offset != new_submap->max_offset)) {
-		vm_map_unlock(old_submap);
-		vm_map_unlock(target_map);
-		return KERN_INVALID_ARGUMENT;
-	}
-	if(!vm_map_lookup_entry(target_map, start, &entry)) {
-		/* if the src is not contained, the entry preceeds */
-		/* our range */
-		addr = entry->vme_start;
-		if(entry == vm_map_to_entry(target_map)) {
-			vm_map_unlock(old_submap);
-			vm_map_unlock(target_map);
-			return KERN_SUCCESS;
-		}
-	}
-	if ((entry->use_pmap) && 
-			(new_submap->pmap == NULL)) {
-		new_submap->pmap = pmap_create((vm_map_size_t) 0, FALSE);
-		if(new_submap->pmap == PMAP_NULL) {
-			vm_map_unlock(old_submap);
-			vm_map_unlock(target_map);
-			return(KERN_NO_SPACE);
-		}
-	}
+page_query_done:
+	vm_object_unlock(object);
 
-	/*
-	 * Mark the new submap as "mapped", so that we get proper
-	 * cleanup of the sub-pmap when we unmap it.
-	 */
-	new_submap->mapped = TRUE;
-
-	addr = entry->vme_start;
-	vm_map_reference(old_submap);
-	while((entry != vm_map_to_entry(target_map)) && 
-					(entry->vme_start < end)) {
-		if((entry->is_sub_map) && 
-			(entry->object.sub_map == old_submap)) {
-			if(entry->use_pmap) {
-				if((start & 0x0fffffff) || 
-					((end - start) != 0x10000000)) {
-					vm_map_unlock(old_submap);
-					vm_map_deallocate(old_submap);
-					vm_map_unlock(target_map);
-					return  KERN_INVALID_ARGUMENT;
-				}
-				nested_pmap = 1;
-			}
-			entry->object.sub_map = new_submap;
-			vm_map_reference(new_submap);
-			vm_map_deallocate(old_submap);
-		}
-		entry = entry->vme_next;
-		addr = entry->vme_start;
-	}
-	if(nested_pmap) {
-#ifndef NO_NESTED_PMAP
-		pmap_unnest(target_map->pmap, (addr64_t)start);
-		if(target_map->mapped) {
-			vm_map_submap_pmap_clean(target_map,
-				start, end, old_submap, 0);
-		}
-		pmap_nest(target_map->pmap, new_submap->pmap, 
-				(addr64_t)start, (addr64_t)start, 
-				(uint64_t)(end - start));
-#endif	/* NO_NESTED_PMAP */
-	} else {
-		vm_map_submap_pmap_clean(target_map,
-				start, end, old_submap, 0);
-	}
-	vm_map_unlock(old_submap);
-	vm_map_deallocate(old_submap);
-	vm_map_unlock(target_map);
-	return KERN_SUCCESS;
+	return retval;
 }
 
 /*
@@ -10091,6 +10880,7 @@ vm_map_msync(
 	boolean_t		do_sync_req;
 	boolean_t		modifiable;
 	boolean_t		had_hole = FALSE;
+	memory_object_t		pager;
 	
 	if ((sync_flags & VM_SYNC_ASYNCHRONOUS) &&
 	    (sync_flags & VM_SYNC_SYNCHRONOUS))
@@ -10117,9 +10907,9 @@ vm_map_msync(
 
 		vm_map_lock(map);
 		if (!vm_map_lookup_entry(map,
-			vm_map_trunc_page(address), &entry)) {
+					 vm_map_trunc_page(address), &entry)) {
 
-			vm_size_t	skip;
+			vm_map_size_t	skip;
 
 			/*
 			 * hole in the address map.
@@ -10164,7 +10954,7 @@ vm_map_msync(
 		 */
 		if (amount_left + entry->vme_start + offset > entry->vme_end) {
 			flush_size = entry->vme_end -
-						 (entry->vme_start + offset);
+				(entry->vme_start + offset);
 		} else {
 			flush_size = amount_left;
 		}
@@ -10179,10 +10969,10 @@ vm_map_msync(
 			local_offset = entry->offset;
 			vm_map_unlock(map);
 			if (vm_map_msync(
-				local_map,
-				local_offset,
-				flush_size,
-				sync_flags) == KERN_INVALID_ADDRESS) {
+				    local_map,
+				    local_offset,
+				    flush_size,
+				    sync_flags) == KERN_INVALID_ADDRESS) {
 				had_hole = TRUE;
 			}
 			continue;
@@ -10199,7 +10989,7 @@ vm_map_msync(
 		}
 		offset += entry->offset;
 		modifiable = (entry->protection & VM_PROT_WRITE)
-				!= VM_PROT_NONE;
+			!= VM_PROT_NONE;
 
                 vm_object_lock(object);
 
@@ -10214,7 +11004,7 @@ vm_map_msync(
 			}
 			if (kill_pages != -1)
 			        vm_object_deactivate_pages(object, offset, 
-							       (vm_object_size_t)flush_size, kill_pages);
+							   (vm_object_size_t)flush_size, kill_pages);
 			vm_object_unlock(object);
 			vm_map_unlock(map);
 			continue;
@@ -10233,26 +11023,37 @@ vm_map_msync(
 		/*
 		 * keep reference on the object until syncing is done
 		 */
-		assert(object->ref_count > 0);
-		object->ref_count++;
-		vm_object_res_reference(object);
+		vm_object_reference_locked(object);
 		vm_object_unlock(object);
 
 		vm_map_unlock(map);
 
 		do_sync_req = vm_object_sync(object,
-					offset,
-					flush_size,
-					sync_flags & VM_SYNC_INVALIDATE,
-					(modifiable &&
-					(sync_flags & VM_SYNC_SYNCHRONOUS ||
-					 sync_flags & VM_SYNC_ASYNCHRONOUS)),
-					 sync_flags & VM_SYNC_SYNCHRONOUS);
+					     offset,
+					     flush_size,
+					     sync_flags & VM_SYNC_INVALIDATE,
+					     (modifiable &&
+					      (sync_flags & VM_SYNC_SYNCHRONOUS ||
+					       sync_flags & VM_SYNC_ASYNCHRONOUS)),
+					     sync_flags & VM_SYNC_SYNCHRONOUS);
 		/*
 		 * only send a m_o_s if we returned pages or if the entry
 		 * is writable (ie dirty pages may have already been sent back)
 		 */
 		if (!do_sync_req && !modifiable) {
+			if ((sync_flags & VM_SYNC_INVALIDATE) && object->resident_page_count == 0) {
+				/*
+				 * clear out the clustering and read-ahead hints
+				 */
+				vm_object_lock(object);
+
+				object->pages_created = 0;
+				object->pages_used = 0;
+				object->sequential = 0;
+				object->last_alloc = 0;
+
+				vm_object_unlock(object);
+			}
 			vm_object_deallocate(object);
 			continue;
 		}
@@ -10265,7 +11066,22 @@ vm_map_msync(
 		new_msr->length = flush_size;
 		new_msr->object = object;
 		new_msr->flag = VM_MSYNC_SYNCHRONIZING;
-re_iterate:
+	re_iterate:
+
+		/*
+		 * We can't sync this object if there isn't a pager.  The
+		 * pager can disappear anytime we're not holding the object
+		 * lock.  So this has to be checked anytime we goto re_iterate.
+		 */
+
+		pager = object->pager;
+
+		if (pager == MEMORY_OBJECT_NULL) {
+			vm_object_unlock(object);
+			vm_object_deallocate(object);
+			continue;
+		}
+
 		queue_iterate(&object->msr_q, msr, msync_req_t, msr_q) {
 			/*
 			 * need to check for overlapping entry, if found, wait
@@ -10289,15 +11105,21 @@ re_iterate:
 		}/* queue_iterate */
 
 		queue_enter(&object->msr_q, new_msr, msync_req_t, msr_q);
+
+		vm_object_paging_begin(object);
 		vm_object_unlock(object);
 
 		queue_enter(&req_q, new_msr, msync_req_t, req_q);
 
 		(void) memory_object_synchronize(
-				object->pager,
-				offset,
-				flush_size,
-				sync_flags & ~VM_SYNC_CONTIGUOUS);
+			pager,
+			offset,
+			flush_size,
+			sync_flags & ~VM_SYNC_CONTIGUOUS);
+
+		vm_object_lock(object);
+		vm_object_paging_end(object);
+		vm_object_unlock(object);
 	}/* while */
 
 	/*
@@ -10326,75 +11148,6 @@ re_iterate:
 	return(KERN_SUCCESS);
 }/* vm_msync */
 
-/* Takes existing source and destination sub-maps and clones the contents of */
-/* the source map */
-kern_return_t
-vm_region_clone(
-	ipc_port_t	src_region, 
-	ipc_port_t	dst_region)
-{
-	vm_named_entry_t	src_object;
-	vm_named_entry_t	dst_object;
-	vm_map_t		src_map;
-	vm_map_t		dst_map;
-	vm_map_offset_t		addr;
-	vm_map_offset_t		max_off;
-	vm_map_entry_t		entry;
-	vm_map_entry_t		new_entry;
-	vm_map_entry_t		insert_point;
-
-	src_object = (vm_named_entry_t)src_region->ip_kobject;
-	dst_object = (vm_named_entry_t)dst_region->ip_kobject;
-	if((!src_object->is_sub_map) || (!dst_object->is_sub_map)) {
-		return KERN_INVALID_ARGUMENT;
-	}
-	src_map = (vm_map_t)src_object->backing.map;
-	dst_map = (vm_map_t)dst_object->backing.map;
-	/* destination map is assumed to be unavailable to any other */
-	/* activity.  i.e. it is new */
-	vm_map_lock(src_map);
-	if((src_map->min_offset != dst_map->min_offset) 
-			|| (src_map->max_offset != dst_map->max_offset)) {
-		vm_map_unlock(src_map);
-		return KERN_INVALID_ARGUMENT;
-	}
-	addr = src_map->min_offset;
-	vm_map_lookup_entry(dst_map, addr, &entry);
-	if(entry == vm_map_to_entry(dst_map)) {
-		entry = entry->vme_next;
-	}
-	if(entry == vm_map_to_entry(dst_map)) {
-		max_off = src_map->max_offset;
-	} else {
-		max_off =  entry->vme_start;
-	}
-	vm_map_lookup_entry(src_map, addr, &entry);
-	if(entry == vm_map_to_entry(src_map)) {
-		entry = entry->vme_next;
-	}
-	vm_map_lookup_entry(dst_map, addr, &insert_point);
-	while((entry != vm_map_to_entry(src_map)) && 
-					(entry->vme_end <= max_off)) {
-		addr = entry->vme_start;
-		new_entry = vm_map_entry_create(dst_map);
-		vm_map_entry_copy(new_entry, entry);
-		vm_map_entry_link(dst_map, insert_point, new_entry);
-		insert_point = new_entry;
-		if (entry->object.vm_object != VM_OBJECT_NULL) {
-			if (new_entry->is_sub_map) {
-				vm_map_reference(new_entry->object.sub_map);
-			} else {
-				vm_object_reference(
-					new_entry->object.vm_object);
-			}
-		}
-		dst_map->size += new_entry->vme_end - new_entry->vme_start;
-		entry = entry->vme_next;
-	}
-	vm_map_unlock(src_map);
-	return KERN_SUCCESS;
-}
-
 /*
  *	Routine:	convert_port_entry_to_map
  *	Purpose:
@@ -10413,25 +11166,28 @@ convert_port_entry_to_map(
 {
 	vm_map_t map;
 	vm_named_entry_t	named_entry;
+	uint32_t	try_failed_count = 0;
 
 	if(IP_VALID(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) {
 		while(TRUE) {
 			ip_lock(port);
 			if(ip_active(port) && (ip_kotype(port) 
-						== IKOT_NAMED_ENTRY)) {
+					       == IKOT_NAMED_ENTRY)) {
 				named_entry =
-					 (vm_named_entry_t)port->ip_kobject;
+					(vm_named_entry_t)port->ip_kobject;
 				if (!(mutex_try(&(named_entry)->Lock))) {
                        			ip_unlock(port);
-                       			mutex_pause();
+
+					try_failed_count++;
+                       			mutex_pause(try_failed_count);
                        			continue;
                 		}
 				named_entry->ref_count++;
 				mutex_unlock(&(named_entry)->Lock);
 				ip_unlock(port);
 				if ((named_entry->is_sub_map) &&
-					(named_entry->protection 
-					& VM_PROT_WRITE)) {
+				    (named_entry->protection 
+				     & VM_PROT_WRITE)) {
 					map = named_entry->backing.map;
 				} else {
 					mach_destroy_memory_entry(port);
@@ -10468,26 +11224,29 @@ convert_port_entry_to_object(
 {
 	vm_object_t object;
 	vm_named_entry_t	named_entry;
+	uint32_t	try_failed_count = 0;
 
 	if(IP_VALID(port) && (ip_kotype(port) == IKOT_NAMED_ENTRY)) {
 		while(TRUE) {
 			ip_lock(port);
 			if(ip_active(port) && (ip_kotype(port) 
-						== IKOT_NAMED_ENTRY)) {
+					       == IKOT_NAMED_ENTRY)) {
 				named_entry =
-					 (vm_named_entry_t)port->ip_kobject;
+					(vm_named_entry_t)port->ip_kobject;
 				if (!(mutex_try(&(named_entry)->Lock))) {
                        			ip_unlock(port);
-                       			mutex_pause();
+
+					try_failed_count++;
+                       			mutex_pause(try_failed_count);
                        			continue;
                 		}
 				named_entry->ref_count++;
 				mutex_unlock(&(named_entry)->Lock);
 				ip_unlock(port);
 				if ((!named_entry->is_sub_map) &&
-					(!named_entry->is_pager) &&
-					(named_entry->protection 
-					& VM_PROT_WRITE)) {
+				    (!named_entry->is_pager) &&
+				    (named_entry->protection 
+				     & VM_PROT_WRITE)) {
 					object = named_entry->backing.object;
 				} else {
 					mach_destroy_memory_entry(port);
@@ -10578,114 +11337,9 @@ vm_map_deallocate(
 	 */
 #endif
 
-	vm_map_destroy(map);
-}
-
-
-/* LP64todo - this whole mechanism is temporary.  It should be redone when
- * the pmap layer can handle 64-bit address spaces.  Until then, we trump
- * up a map entry for the 64-bit commpage above the map's max_offset.
- */
-extern vm_map_t		com_region_map64;	/* the submap for 64-bit commpage */
-extern vm_map_t		com_region_map32;	/* the submap for 32-bit commpage */
-
- 
-static void
-vm_map_commpage(
-        vm_map_t        user_map,
-	vm_map_t	com_region_map,		/* com_region_map32 or com_region_map64 */
-	vm_map_offset_t	base_address,
-	vm_map_size_t	size)
-{
-        vm_map_entry_t  entry;
-	vm_object_t	object;
-        
-        vm_map_lock(user_map);
-        
-        /* The commpage is necessarily the last entry in the map.
-	 * See if one is already there (not sure if this can happen???)
-	 */
-	entry = vm_map_last_entry(user_map);
-	if (entry != vm_map_to_entry(user_map)) {
-		if (entry->vme_end >= base_address) {
-			vm_map_unlock(user_map);
-			return;
-		}
-	}
-	
-	entry = vm_map_first_entry(com_region_map);
-	object = entry->object.vm_object;
-	vm_object_reference(object);
-
-	/* We bypass vm_map_enter() because we are adding the entry past the
-	 * map's max_offset.
-	 */
-        entry = vm_map_entry_insert(
-                        user_map,
-                        vm_map_last_entry(user_map),	/* insert after last entry */
-                        base_address,
-                        base_address + size,
-                        object,
-                        0,				/* offset */
-                        FALSE,				/* needs_copy */
-			FALSE,				/* is_shared */
-                        FALSE,				/* in_transition */
-			VM_PROT_READ|VM_PROT_EXECUTE,
-			VM_PROT_READ|VM_PROT_EXECUTE,
-			VM_BEHAVIOR_DEFAULT,
-			VM_INHERIT_NONE,
-			1 );				/* wired_count */
-	
-	vm_map_unlock(user_map);
-}
-
-#ifdef __i386__
-void
-vm_map_commpage32(
-	vm_map_t	map)
-{
-	vm_map_commpage(map,
-			com_region_map32,
-			(vm_map_offset_t) (unsigned) _COMM_PAGE32_BASE_ADDRESS,
-			(vm_map_size_t) (unsigned) _COMM_PAGE32_AREA_USED);
-}
-#endif	/* __i386__ */
-
-
-
-void
-vm_map_commpage64(
-	vm_map_t	map)
-{
-
-	vm_map_commpage(map,
-			com_region_map64,
-			(vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS,
-			(vm_map_size_t) _COMM_PAGE64_AREA_USED);
+	vm_map_destroy(map, VM_MAP_NO_FLAGS);
 }
 
-void
-vm_map_remove_commpage(
-        vm_map_t        map )
-{
-        vm_map_entry_t  entry;
-	
-	while( 1 ) {
-		vm_map_lock(map);
-	
-		entry = vm_map_last_entry(map);
-
-		if ((entry == vm_map_to_entry(map)) || 
-			(entry->vme_start < map->max_offset))
-				break;
-				
-		/* clearing the wired count isn't strictly correct */
-		entry->wired_count = 0;
-		vm_map_entry_delete(map,entry);
-	}
-	
-	vm_map_unlock(map);
-}
 
 void
 vm_map_disable_NX(vm_map_t map)
@@ -10721,7 +11375,15 @@ vm_compute_max_offset(unsigned is64)
 }
 
 boolean_t
-vm_map_has_4GB_pagezero(vm_map_t map)
+vm_map_is_64bit(
+		vm_map_t map)
+{
+	return map->max_offset > ((vm_map_offset_t)VM_MAX_ADDRESS);
+}
+
+boolean_t
+vm_map_has_4GB_pagezero(
+		vm_map_t map)
 {
 	/*
 	 * XXX FBDP
@@ -10790,3 +11452,17 @@ vm_map_raise_min_offset(
 
 	return KERN_SUCCESS;
 }
+
+/*
+ * Set the limit on the maximum amount of user wired memory allowed for this map.
+ * This is basically a copy of the MEMLOCK rlimit value maintained by the BSD side of
+ * the kernel.  The limits are checked in the mach VM side, so we keep a copy so we
+ * don't have to reach over to the BSD data structures.
+ */
+
+void
+vm_map_set_user_wire_limit(vm_map_t 	map,
+			   vm_size_t	limit)
+{
+	map->user_wire_limit = limit;
+}
diff --git a/osfmk/vm/vm_map.h b/osfmk/vm/vm_map.h
index 02bf60eea..0b181f35b 100644
--- a/osfmk/vm/vm_map.h
+++ b/osfmk/vm/vm_map.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -83,6 +89,13 @@ __BEGIN_DECLS
 extern void	vm_map_reference(vm_map_t	map);
 extern vm_map_t current_map(void);
 
+/* Setup reserved areas in a new VM map */
+extern kern_return_t	vm_map_exec(
+				vm_map_t		new_map,
+				task_t			task,
+				void			*fsroot,
+				cpu_type_t		cpu);
+
 __END_DECLS
 
 #ifdef	MACH_KERNEL_PRIVATE
@@ -207,7 +220,8 @@ struct vm_map_entry {
 	/* vm_inherit_t */	inheritance:2,	/* inheritance */
 	/* boolean_t */		use_pmap:1,	/* nested pmaps */
 	/* unsigned char */	alias:8,	/* user alias */
-	/* unsigned char */	pad:8;		/* available bits */
+	/* boolean_t */		no_cache:1,	/* should new pages be cached? */
+	/* unsigned char */	pad:7;		/* available bits */
 	unsigned short		wired_count;	/* can be paged if = 0 */
 	unsigned short		user_wired_count; /* for vm_wire */
 };
@@ -248,19 +262,21 @@ struct vm_map_header {
  *		insertion, or removal.  Another hint is used to
  *		quickly find free space.
  */
-struct vm_map {
+struct _vm_map {
 	lock_t			lock;		/* uni- and smp-lock */
 	struct vm_map_header	hdr;		/* Map entry header */
 #define min_offset		hdr.links.start	/* start of range */
 #define max_offset		hdr.links.end	/* end of range */
 	pmap_t			pmap;		/* Physical map */
 	vm_map_size_t		size;		/* virtual size */
+	vm_map_size_t		user_wire_limit;/* rlimit on user locked memory */
+	vm_map_size_t		user_wire_size; /* current size of user locked memory in this map */
 	int			ref_count;	/* Reference count */
 #if	TASK_SWAPPER
 	int			res_count;	/* Residence count (swap) */
 	int			sw_state;	/* Swap state */
 #endif	/* TASK_SWAPPER */
-	decl_mutex_data(,	s_lock)		/* Lock ref, res, hint fields */
+	decl_mutex_data(,	s_lock)		/* Lock ref, res fields */
 	vm_map_entry_t		hint;		/* hint for quick lookups */
 	vm_map_entry_t		first_free;	/* First free space hint */
 	boolean_t		wait_for_space;	/* Should callers wait
@@ -269,6 +285,7 @@ struct vm_map {
 	boolean_t		no_zero_fill;	/* No zero fill absent pages */
 	boolean_t		mapped;		/* has this map been mapped */
 	unsigned int		timestamp;	/* Version number */
+	unsigned int		color_rr;	/* next color (not protected by a lock) */
 } ;
 
 #define vm_map_to_entry(map)	((struct vm_map_entry *) &(map)->hdr.links)
@@ -388,14 +405,17 @@ struct vm_map_copy {
 #define vm_map_unlock_read(map)		lock_read_done(&(map)->lock)
 #define vm_map_lock_write_to_read(map)					\
 		((map)->timestamp++ ,	lock_write_to_read(&(map)->lock))
-#define vm_map_lock_read_to_write(map)	lock_read_to_write(&(map)->lock)
+/* lock_read_to_write() returns FALSE on failure.  Macro evaluates to 
+ * zero on success and non-zero value on failure.
+ */
+#define vm_map_lock_read_to_write(map)	(lock_read_to_write(&(map)->lock) != TRUE)
 
 /*
  *	Exported procedures that operate on vm_map_t.
  */
 
 /* Initialize the module */
-extern void		vm_map_init(void);
+extern void		vm_map_init(void) __attribute__((section("__TEXT, initcode")));
 
 /* Allocate a range in the specified virtual address map and
  * return the entry allocated for that range. */
@@ -419,14 +439,13 @@ extern kern_return_t	vm_map_lookup_locked(
 				vm_map_t		*var_map,	/* IN/OUT */
 				vm_map_address_t	vaddr,
 				vm_prot_t		fault_type,
+				int			object_lock_type,
 				vm_map_version_t 	*out_version,	/* OUT */
 				vm_object_t		*object,	/* OUT */
 				vm_object_offset_t 	*offset,	/* OUT */
 				vm_prot_t		*out_prot,	/* OUT */
 				boolean_t		*wired,		/* OUT */
-				int			*behavior,	/* OUT */
-				vm_map_offset_t		*lo_offset,	/* OUT */
-				vm_map_offset_t		*hi_offset,	/* OUT */
+				vm_object_fault_info_t	fault_info,	/* OUT */
 				vm_map_t		*real_map);	/* OUT */
 
 /* Verifies that the map has not changed since the given version. */
@@ -448,7 +467,8 @@ extern vm_map_entry_t	vm_map_entry_insert(
 				vm_prot_t		max_protection,
 				vm_behavior_t		behavior,
 				vm_inherit_t		inheritance,
-				unsigned		wired_count);
+				unsigned		wired_count,
+				boolean_t		no_cache);
 
 
 /*
@@ -757,6 +777,13 @@ extern vm_object_t convert_port_entry_to_object(
 	ipc_port_t	port);
 
 
+/* definitions related to overriding the NX behavior */
+
+#define VM_ABI_32	0x1
+#define VM_ABI_64	0x2
+
+extern int override_nx(vm_map_t map, uint32_t user_tag);
+
 #endif /* MACH_KERNEL_PRIVATE */
 
 __BEGIN_DECLS
@@ -770,7 +797,9 @@ extern vm_map_t		vm_map_create(
 
 /* Get rid of a map */
 extern void		vm_map_destroy(
-				vm_map_t		map);
+				vm_map_t		map,
+				int			flags);
+
 /* Lose a reference */
 extern void		vm_map_deallocate(
 				vm_map_t		map);
@@ -808,6 +837,20 @@ extern kern_return_t	vm_map_unwire(
 				vm_map_offset_t		end,
 				boolean_t		user_wire);
 
+/* Enter a mapping of a memory object */
+extern kern_return_t	vm_map_enter_mem_object(
+				vm_map_t		map,
+				vm_map_offset_t		*address,
+				vm_map_size_t		size,
+				vm_map_offset_t		mask,
+				int			flags,
+				ipc_port_t		port,
+				vm_object_offset_t	offset,
+				boolean_t		needs_copy,
+				vm_prot_t		cur_protection,
+				vm_prot_t		max_protection,
+				vm_inherit_t		inheritance);
+
 /* Deallocate a region */
 extern kern_return_t	vm_map_remove(
 				vm_map_t		map,
@@ -832,6 +875,13 @@ extern kern_return_t	vm_map_copyout(
 				vm_map_address_t	*dst_addr,	/* OUT */
 				vm_map_copy_t		copy);
 
+extern kern_return_t	vm_map_copyin(
+				vm_map_t			src_map,
+				vm_map_address_t	src_addr,
+				vm_map_size_t		len,
+				boolean_t			src_destroy,
+				vm_map_copy_t		*copy_result);	/* OUT */
+
 extern kern_return_t	vm_map_copyin_common(
 				vm_map_t		src_map,
 				vm_map_address_t	src_addr,
@@ -850,6 +900,9 @@ extern void		vm_map_set_64bit(
 extern void		vm_map_set_32bit(
 			        vm_map_t		map);
 
+extern boolean_t	vm_map_is_64bit(
+			        vm_map_t		map);
+
 extern boolean_t	vm_map_has_4GB_pagezero(
 		       		vm_map_t		map);
 
@@ -866,6 +919,12 @@ extern kern_return_t	vm_map_raise_min_offset(
 extern vm_map_offset_t	vm_compute_max_offset(
 				unsigned		is64);
 
+extern void		vm_map_set_user_wire_limit(
+				vm_map_t		map,
+				vm_size_t		limit);
+
+#ifdef	MACH_KERNEL_PRIVATE
+
 /*
  *	Macros to invoke vm_map_copyin_common.  vm_map_copyin is the
  *	usual form; it handles a copyin based on the current protection
@@ -884,6 +943,8 @@ extern vm_map_offset_t	vm_compute_max_offset(
 		vm_map_copyin_common(src_map, src_addr, len, src_destroy, \
 					FALSE, copy_result, TRUE)
 
+#endif /* MACH_KERNEL_PRIVATE */
+
 /*
  * Macros for rounding and truncation of vm_map offsets and sizes
  */
@@ -898,18 +959,7 @@ extern vm_map_offset_t	vm_compute_max_offset(
 #define	VM_MAP_REMOVE_INTERRUPTIBLE  	0x2
 #define	VM_MAP_REMOVE_WAIT_FOR_KWIRE  	0x4
 #define VM_MAP_REMOVE_SAVE_ENTRIES	0x8
-
-/* Support for shared regions */
-extern kern_return_t vm_region_clone(
-				ipc_port_t		src_region,
-				ipc_port_t		dst_region);
-
-extern kern_return_t vm_map_region_replace(
-				vm_map_t		target_map,
-				ipc_port_t		old_region,
-				ipc_port_t		new_region,
-				vm_map_offset_t		start,  
-				vm_map_offset_t		end);
+#define VM_MAP_REMOVE_NO_PMAP_CLEANUP	0x10
 
 /* Support for UPLs from vm_maps */
 
diff --git a/osfmk/vm/vm_object.c b/osfmk/vm/vm_object.c
index 221565271..60a80d38a 100644
--- a/osfmk/vm/vm_object.c
+++ b/osfmk/vm/vm_object.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -56,6 +62,7 @@
  *	Virtual memory object module.
  */
 
+#include <debug.h>
 #include <mach_pagemap.h>
 #include <task_swapper.h>
 
@@ -86,6 +93,7 @@
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h>
+#include <vm/vm_purgeable_internal.h>
 
 /*
  *	Virtual memory objects maintain the actual data
@@ -199,6 +207,7 @@ static zone_t		vm_object_zone;		/* vm backing store zone */
 static struct vm_object			kernel_object_store;
 vm_object_t						kernel_object;
 
+
 /*
  *	The submap object is used as a placeholder for vm_map_submap
  *	operations.  The object is declared in vm_map.c because it
@@ -397,6 +406,12 @@ vm_object_allocate(
 	return object;
 }
 
+
+lck_grp_t	vm_object_lck_grp;
+lck_grp_attr_t	vm_object_lck_grp_attr;
+lck_attr_t	vm_object_lck_attr;
+lck_attr_t	kernel_object_lck_attr;
+
 /*
  *	vm_object_bootstrap:
  *
@@ -413,7 +428,6 @@ vm_object_bootstrap(void)
 				"vm objects");
 
 	queue_init(&vm_object_reaper_queue);
-
 	queue_init(&vm_object_cached_list);
 	mutex_init(&vm_object_cached_lock_data, 0);
 
@@ -426,11 +440,25 @@ vm_object_bootstrap(void)
 	for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
 		queue_init(&vm_object_hashtable[i]);
 
+	vm_object_init_lck_grp();
+
 	/*
 	 *	Fill in a template object, for quick initialization
 	 */
 
 	/* memq; Lock; init after allocation */
+	vm_object_template.memq.prev = NULL;
+	vm_object_template.memq.next = NULL;
+#if 0
+	/*
+	 * We can't call vm_object_lock_init() here because that will
+	 * allocate some memory and VM is not fully initialized yet.
+	 * The lock will be initialized for each allocate object in
+	 * _vm_object_allocate(), so we don't need to initialize it in
+	 * the vm_object_template.
+	 */
+	vm_object_lock_init(&vm_object_template);
+#endif
 	vm_object_template.size = 0;
 	vm_object_template.memq_hint = VM_PAGE_NULL;
 	vm_object_template.ref_count = 1;
@@ -441,16 +469,10 @@ vm_object_bootstrap(void)
 	vm_object_template.copy = VM_OBJECT_NULL;
 	vm_object_template.shadow = VM_OBJECT_NULL;
 	vm_object_template.shadow_offset = (vm_object_offset_t) 0;
-	vm_object_template.cow_hint = ~(vm_offset_t)0;
-	vm_object_template.true_share = FALSE;
-
 	vm_object_template.pager = MEMORY_OBJECT_NULL;
 	vm_object_template.paging_offset = 0;
 	vm_object_template.pager_control = MEMORY_OBJECT_CONTROL_NULL;
-	/* msr_q; init after allocation */
-
 	vm_object_template.copy_strategy = MEMORY_OBJECT_COPY_SYMMETRIC;
-	vm_object_template.absent_count = 0;
 	vm_object_template.paging_in_progress = 0;
 
 	/* Begin bitfields */
@@ -465,29 +487,53 @@ vm_object_bootstrap(void)
 	vm_object_template.private = FALSE;
 	vm_object_template.pageout = FALSE;
 	vm_object_template.alive = TRUE;
-	vm_object_template.purgable = VM_OBJECT_NONPURGABLE;
+	vm_object_template.purgable = VM_PURGABLE_DENY;
+	vm_object_template.shadowed = FALSE;
 	vm_object_template.silent_overwrite = FALSE;
 	vm_object_template.advisory_pageout = FALSE;
-	vm_object_template.shadowed = FALSE;
+	vm_object_template.true_share = FALSE;
 	vm_object_template.terminating = FALSE;
+	vm_object_template.named = FALSE;
 	vm_object_template.shadow_severed = FALSE;
 	vm_object_template.phys_contiguous = FALSE;
 	vm_object_template.nophyscache = FALSE;
 	/* End bitfields */
 
-	/* cache bitfields */
-	vm_object_template.wimg_bits = VM_WIMG_DEFAULT;
-
-	/* cached_list; init after allocation */
+	vm_object_template.cached_list.prev = NULL;
+	vm_object_template.cached_list.next = NULL;
+	vm_object_template.msr_q.prev = NULL;
+	vm_object_template.msr_q.next = NULL;
+	
 	vm_object_template.last_alloc = (vm_object_offset_t) 0;
-	vm_object_template.cluster_size = 0;
+	vm_object_template.sequential = (vm_object_offset_t) 0;
+	vm_object_template.pages_created = 0;
+	vm_object_template.pages_used = 0;
+
 #if	MACH_PAGEMAP
 	vm_object_template.existence_map = VM_EXTERNAL_NULL;
 #endif	/* MACH_PAGEMAP */
+	vm_object_template.cow_hint = ~(vm_offset_t)0;
 #if	MACH_ASSERT
 	vm_object_template.paging_object = VM_OBJECT_NULL;
 #endif	/* MACH_ASSERT */
 
+	/* cache bitfields */
+	vm_object_template.wimg_bits = VM_WIMG_DEFAULT;
+	vm_object_template.code_signed = FALSE;
+	vm_object_template.not_in_use = 0;
+#ifdef UPL_DEBUG
+	vm_object_template.uplq.prev = NULL;
+	vm_object_template.uplq.next = NULL;
+#endif /* UPL_DEBUG */
+#ifdef VM_PIP_DEBUG
+	bzero(&vm_object_template.pip_holders,
+	      sizeof (vm_object_template.pip_holders));
+#endif /* VM_PIP_DEBUG */
+
+	vm_object_template.objq.next=NULL;
+	vm_object_template.objq.prev=NULL;
+
+	
 	/*
 	 *	Initialize the "kernel object"
 	 */
@@ -547,7 +593,7 @@ vm_object_reaper_init(void)
 		BASEPRI_PREEMPT - 1,
 		&thread);
 	if (kr != KERN_SUCCESS) {
-		panic("failed to launch vm_object_reaper_thread kr=0x%x\n", kr);
+		panic("failed to launch vm_object_reaper_thread kr=0x%x", kr);
 	}
 	thread_deallocate(thread);
 }
@@ -560,13 +606,20 @@ vm_object_init(void)
 	 */
 }
 
-/* remove the typedef below when emergency work-around is taken out */
-typedef struct vnode_pager {
-	memory_object_t pager;
-	memory_object_t pager_handle;	/* pager */
-	memory_object_control_t 	control_handle;	/* memory object's control handle */
-	void	   	*vnode_handle;  /* vnode handle */
-} *vnode_pager_t;
+
+__private_extern__ void
+vm_object_init_lck_grp(void)
+{
+        /*
+	 * initialze the vm_object lock world
+	 */
+        lck_grp_attr_setdefault(&vm_object_lck_grp_attr);
+	lck_grp_init(&vm_object_lck_grp, "vm_object", &vm_object_lck_grp_attr);
+	lck_attr_setdefault(&vm_object_lck_attr);
+	lck_attr_setdefault(&kernel_object_lck_attr);
+	lck_attr_cleardebug(&kernel_object_lck_attr);
+}
+
 
 #define	MIGHT_NOT_CACHE_SHADOWS		1
 #if	MIGHT_NOT_CACHE_SHADOWS
@@ -584,16 +637,83 @@ static int cache_shadows = TRUE;
  *
  *	No object may be locked.
  */
+unsigned long vm_object_deallocate_shared_successes = 0;
+unsigned long vm_object_deallocate_shared_failures = 0;
+unsigned long vm_object_deallocate_shared_swap_failures = 0;
 __private_extern__ void
 vm_object_deallocate(
 	register vm_object_t	object)
 {
-	boolean_t retry_cache_trim = FALSE;
-	vm_object_t shadow = VM_OBJECT_NULL;
+	boolean_t	retry_cache_trim = FALSE;
+	vm_object_t	shadow = VM_OBJECT_NULL;
+	uint32_t	try_failed_count = 0;
 	
 //	if(object)dbgLog(object, object->ref_count, object->can_persist, 3);	/* (TEST/DEBUG) */
 //	else dbgLog(object, 0, 0, 3);	/* (TEST/DEBUG) */
 
+	if (object == VM_OBJECT_NULL)
+	        return;
+
+	if (object == kernel_object) {
+		vm_object_lock(kernel_object);
+		kernel_object->ref_count--;
+		if (kernel_object->ref_count == 0) {
+			panic("vm_object_deallocate: losing kernel_object\n");
+		}
+		vm_object_unlock(kernel_object);
+		return;
+	}
+
+	if (object->ref_count > 2 ||
+	    (!object->named && object->ref_count > 1)) {
+		UInt32		original_ref_count;
+		volatile UInt32	*ref_count_p;
+		Boolean		atomic_swap;
+
+		/*
+		 * The object currently looks like it is not being
+		 * kept alive solely by the reference we're about to release.
+		 * Let's try and release our reference without taking
+		 * all the locks we would need if we had to terminate the
+		 * object (cache lock + exclusive object lock).
+		 * Lock the object "shared" to make sure we don't race with
+		 * anyone holding it "exclusive".
+		 */
+	        vm_object_lock_shared(object);
+		ref_count_p = (volatile UInt32 *) &object->ref_count;
+		original_ref_count = object->ref_count;
+		/*
+		 * Test again as "ref_count" could have changed.
+		 * "named" shouldn't change.
+		 */
+		if (original_ref_count > 2 ||
+		    (!object->named && original_ref_count > 1)) {
+			atomic_swap = OSCompareAndSwap(
+				original_ref_count,
+				original_ref_count - 1,
+				(UInt32 *) &object->ref_count);
+			if (atomic_swap == FALSE) {
+				vm_object_deallocate_shared_swap_failures++;
+			}
+
+		} else {
+			atomic_swap = FALSE;
+		}
+		vm_object_unlock(object);
+
+		if (atomic_swap) {
+			/* ref_count was updated atomically ! */
+			vm_object_deallocate_shared_successes++;
+			return;
+		}
+
+		/*
+		 * Someone else updated the ref_count at the same
+		 * time and we lost the race.  Fall back to the usual
+		 * slow but safe path...
+		 */
+		vm_object_deallocate_shared_failures++;
+	}
 
 	while (object != VM_OBJECT_NULL) {
 
@@ -615,7 +735,9 @@ vm_object_deallocate(
 			if (vm_object_lock_try(object))
 			        break;
 		        vm_object_cache_unlock();
-			mutex_pause();  /* wait a bit */
+			try_failed_count++;
+
+			mutex_pause(try_failed_count);  /* wait a bit */
 		}
 		assert(object->ref_count > 0);
 
@@ -636,6 +758,7 @@ vm_object_deallocate(
 					
 				memory_object_unmap(pager);
 
+				try_failed_count = 0;
 				for (;;) {
 				        vm_object_cache_lock();
 
@@ -649,7 +772,9 @@ vm_object_deallocate(
 					if (vm_object_lock_try(object))
 					        break;
 					vm_object_cache_unlock();
-					mutex_pause();  /* wait a bit */
+					try_failed_count++;
+
+					mutex_pause(try_failed_count);  /* wait a bit */
 				}
 				assert(object->ref_count > 0);
 			}
@@ -668,6 +793,7 @@ vm_object_deallocate(
 		/* terminate again.  */
 
 		if ((object->ref_count > 1) || object->terminating) {
+			vm_object_lock_assert_exclusive(object);
 			object->ref_count--;
 			vm_object_res_deallocate(object);
 			vm_object_cache_unlock();
@@ -733,6 +859,7 @@ vm_object_deallocate(
 			 *	Now it is safe to decrement reference count,
 			 *	and to return if reference count is > 0.
 			 */
+			vm_object_lock_assert_exclusive(object);
 			if (--object->ref_count > 0) {
 				vm_object_res_deallocate(object);
 				vm_object_unlock(object);
@@ -903,6 +1030,7 @@ vm_object_cache_trim(
 
 		assert(object->pager_initialized);
 		assert(object->ref_count == 0);
+		vm_object_lock_assert_exclusive(object);
 		object->ref_count++;
 
 		/*
@@ -926,7 +1054,15 @@ vm_object_cache_trim(
 	}
 }
 
-boolean_t	vm_object_terminate_remove_all = FALSE;
+#define VM_OBJ_TERM_STATS DEBUG
+#if VM_OBJ_TERM_STATS
+uint32_t vm_object_terminate_pages_freed = 0;
+uint32_t vm_object_terminate_pages_removed = 0;
+uint32_t vm_object_terminate_batches = 0;
+uint32_t vm_object_terminate_biggest_batch = 0;
+#endif /* VM_OBJ_TERM_STATS */
+
+#define V_O_T_MAX_BATCH	256
 
 /*
  *	Routine:	vm_object_terminate
@@ -951,14 +1087,72 @@ vm_object_terminate(
 {
 	register vm_page_t	p;
 	vm_object_t		shadow_object;
+	vm_page_t		local_free_q;
+	int			loop_count;
+#if VM_OBJ_TERM_STATS
+	uint32_t		local_free_count;
+	uint32_t		pages_removed;
+#endif /* VM_OBJ_TERM_STATS */
+
+#if VM_OBJ_TERM_STATS
+#define VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count)	\
+	MACRO_BEGIN							\
+		if (_pages_removed) {					\
+			hw_atomic_add(&vm_object_terminate_batches, 1); \
+			hw_atomic_add(&vm_object_terminate_pages_removed, \
+				      _pages_removed);			\
+			hw_atomic_add(&vm_object_terminate_pages_freed, \
+				      _local_free_count);		\
+			if (_local_free_count >				\
+			    vm_object_terminate_biggest_batch) {	\
+				vm_object_terminate_biggest_batch =	\
+					_local_free_count;		\
+			}						\
+			_local_free_count = 0;				\
+		}							\
+	MACRO_END
+#else /* VM_OBJ_TERM_STATS */
+#define VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count)
+#endif /* VM_OBJ_TERM_STATS */
+
+#define VM_OBJ_TERM_FREELIST(_pages_removed, _local_free_count, _local_free_q) \
+	MACRO_BEGIN							\
+		VM_OBJ_TERM_FREELIST_DEBUG(_pages_removed, _local_free_count); \
+		if (_local_free_q) {					\
+			vm_page_free_list(_local_free_q);		\
+			_local_free_q = VM_PAGE_NULL;			\
+		}							\
+	MACRO_END
+
+
 
 	XPR(XPR_VM_OBJECT, "vm_object_terminate, object 0x%X ref %d\n",
 		(integer_t)object, object->ref_count, 0, 0, 0);
 
+	local_free_q = VM_PAGE_NULL;
+#if VM_OBJ_TERM_STATS
+	local_free_count = 0;
+	pages_removed = 0;
+#endif /* VM_OBJ_TERM_STATS */
+
 	if (!object->pageout && (!object->temporary || object->can_persist)
 			&& (object->pager != NULL || object->shadow_severed)) {
 	   vm_object_cache_unlock();
+	   loop_count = V_O_T_MAX_BATCH;
+	   vm_page_lock_queues();
 	   while (!queue_empty(&object->memq)) {
+		if (--loop_count == 0) {
+			/*
+			 * Free the pages we've reclaimed so far and
+			 * take a little break to avoid hogging
+			 * the page queues lock too long.
+			 */
+			VM_OBJ_TERM_FREELIST(pages_removed,
+					     local_free_count,
+					     local_free_q);
+			mutex_yield(&vm_page_queue_lock);
+			loop_count = V_O_T_MAX_BATCH;
+		}
 		/*
 		 * Clear pager_trusted bit so that the pages get yanked
 		 * out of the object instead of cleaned in place.  This
@@ -972,17 +1166,24 @@ vm_object_terminate(
 
 		if (p->busy || p->cleaning) {
 			if(p->cleaning || p->absent) {
+				/* free the pages reclaimed so far */
+				VM_OBJ_TERM_FREELIST(pages_removed,
+						     local_free_count,
+						     local_free_q);
+				vm_page_unlock_queues();
 				vm_object_paging_wait(object, THREAD_UNINT);
+				vm_page_lock_queues();
 				continue;
 			} else {
-			   panic("vm_object_terminate.3 0x%x 0x%x", object, p);
+				panic("vm_object_terminate.3 %p %p", object, p);
 			}
 		}
 
-		vm_page_lock_queues();
 		p->busy = TRUE;
 		VM_PAGE_QUEUES_REMOVE(p);
-		vm_page_unlock_queues();
+#if VM_OBJ_TERM_STATS
+		pages_removed++;
+#endif /* VM_OBJ_TERM_STATS */
 
 		if (p->absent || p->private) {
 
@@ -997,23 +1198,50 @@ vm_object_terminate(
 			goto free_page;
 		}
 
-		if (p->fictitious)
-			panic("vm_object_terminate.4 0x%x 0x%x", object, p);
+		if (p->fictitious) {
+			if (p->phys_page == vm_page_guard_addr) {
+				goto free_page;
+			}
+			panic("vm_object_terminate.4 %p %p", object, p);
+		}
 
-		if (!p->dirty)
+		if (!p->dirty && p->pmapped)
 			p->dirty = pmap_is_modified(p->phys_page);
 
 		if ((p->dirty || p->precious) && !p->error && object->alive) {
+			/* free the pages reclaimed so far */
+			VM_OBJ_TERM_FREELIST(pages_removed,
+					     local_free_count,
+					     local_free_q);
+			vm_page_unlock_queues();
 			vm_pageout_cluster(p); /* flush page */
 			vm_object_paging_wait(object, THREAD_UNINT);
 			XPR(XPR_VM_OBJECT,
 			    "vm_object_terminate restart, object 0x%X ref %d\n",
 			    (integer_t)object, object->ref_count, 0, 0, 0);
+			vm_page_lock_queues();
 		} else {
 		    free_page:
-		    	VM_PAGE_FREE(p);
+			/*
+			 * Add this page to our list of reclaimed pages,
+			 * to be freed later.
+			 */
+			vm_page_free_prepare(p);
+			p->pageq.next = (queue_entry_t) local_free_q;
+			local_free_q = p;
+#if VM_OBJ_TERM_STATS
+			local_free_count++;
+#endif /* VM_OBJ_TERM_STATS */
 		}
 	   }
+
+	   /*
+	    * Free the remaining reclaimed pages.
+	    */
+	   VM_OBJ_TERM_FREELIST(pages_removed,
+				local_free_count,
+				local_free_q);
+	   vm_page_unlock_queues();
 	   vm_object_unlock(object);
 	   vm_object_cache_lock();
 	   vm_object_lock(object);
@@ -1023,7 +1251,8 @@ vm_object_terminate(
 	 *	Make sure the object isn't already being terminated
 	 */
 	if(object->terminating) {
-		object->ref_count -= 1;
+		vm_object_lock_assert_exclusive(object);
+		object->ref_count--;
 		assert(object->ref_count > 0);
 		vm_object_cache_unlock();
 		vm_object_unlock(object);
@@ -1035,7 +1264,8 @@ vm_object_terminate(
 	 * cleaning it?
 	 */
 	if(object->ref_count != 1) {
-		object->ref_count -= 1;
+		vm_object_lock_assert_exclusive(object);
+		object->ref_count--;
 		assert(object->ref_count > 0);
 		vm_object_res_deallocate(object);
 		vm_object_cache_unlock();
@@ -1104,6 +1334,14 @@ vm_object_terminate(
 	object = VM_OBJECT_NULL;
 	/* cache lock and object lock were released by vm_object_reap() */
 
+	/*
+	 * KERN_SUCCESS means that this object has been terminated
+	 * and no longer needs its shadow object but still holds a
+	 * reference on it.
+	 * The caller is responsible for dropping that reference.
+	 * We can't call vm_object_deallocate() here because that
+	 * would create a recursion.
+	 */
 	return KERN_SUCCESS;
 }
 
@@ -1122,32 +1360,25 @@ vm_object_reap(
 {
 	memory_object_t		pager;
 	vm_page_t		p;
+	vm_page_t		local_free_q;
+	int			loop_count;
+#if VM_OBJ_TERM_STATS
+	uint32_t		local_free_count;
+#endif /* VM_OBJ_TERM_STATS */
 
 #if DEBUG
 	mutex_assert(&vm_object_cached_lock_data, MA_OWNED);
-	mutex_assert(&object->Lock, MA_OWNED);
 #endif /* DEBUG */
+	vm_object_lock_assert_exclusive(object);
+	assert(object->paging_in_progress == 0);
 
 	vm_object_reap_count++;
 
-	/*
-	 * The pageout daemon might be playing with our pages.
-	 * Now that the object is dead, it won't touch any more
-	 * pages, but some pages might already be on their way out.
-	 * Hence, we wait until the active paging activities have
-	 * ceased before we break the association with the pager
-	 * itself.
-	 */
-	while (object->paging_in_progress != 0) {
-		vm_object_cache_unlock();
-		vm_object_wait(object,
-			       VM_OBJECT_EVENT_PAGING_IN_PROGRESS,
-			       THREAD_UNINT);
-		vm_object_cache_lock();
-		vm_object_lock(object);
-	}
+	local_free_q = VM_PAGE_NULL;
+#if VM_OBJ_TERM_STATS
+	local_free_count = 0;
+#endif /* VM_OBJ_TERM_STATS */
 
-	assert(object->paging_in_progress == 0);
 	pager = object->pager;
 	object->pager = MEMORY_OBJECT_NULL;
 
@@ -1155,6 +1386,7 @@ vm_object_reap(
 		memory_object_control_disable(object->pager_control);
 	vm_object_cache_unlock();
 
+	vm_object_lock_assert_exclusive(object);
 	object->ref_count--;
 #if	TASK_SWAPPER
 	assert(object->res_count == 0);
@@ -1162,6 +1394,19 @@ vm_object_reap(
 
 	assert (object->ref_count == 0);
 
+	/* remove from purgeable queue if it's on */
+	if (object->objq.next || object->objq.prev) {
+	        purgeable_q_t queue = vm_purgeable_object_remove(object);
+		assert(queue);
+
+		/* Must take page lock for this - using it to protect token queue */
+		vm_page_lock_queues();
+		vm_purgeable_token_delete_first(queue);
+        
+		assert(queue->debug_count_objects>=0);
+		vm_page_unlock_queues();
+	}
+    
 	/*
 	 *	Clean or free the pages, as appropriate.
 	 *	It is possible for us to find busy/absent pages,
@@ -1174,12 +1419,39 @@ vm_object_reap(
 
 	} else if ((object->temporary && !object->can_persist) ||
 		   (pager == MEMORY_OBJECT_NULL)) {
+		loop_count = V_O_T_MAX_BATCH;
+		vm_page_lock_queues();
 		while (!queue_empty(&object->memq)) {
+			if (--loop_count == 0) {
+				/*
+				 * Free the pages we reclaimed so far
+				 * and take a little break to avoid
+				 * hogging the page queue lock too long
+				 */
+				VM_OBJ_TERM_FREELIST(local_free_count,
+						     local_free_count,
+						     local_free_q);
+				mutex_yield(&vm_page_queue_lock);
+				loop_count = V_O_T_MAX_BATCH;
+			}
 			p = (vm_page_t) queue_first(&object->memq);
 
-			VM_PAGE_CHECK(p);
-			VM_PAGE_FREE(p);
+			vm_page_free_prepare(p);
+
+			assert(p->pageq.next == NULL && p->pageq.prev == NULL);
+			p->pageq.next = (queue_entry_t) local_free_q;
+			local_free_q = p;
+#if VM_OBJ_TERM_STATS
+			local_free_count++;
+#endif /* VM_OBJ_TERM_STATS */
 		}
+		/*
+		 * Free the remaining reclaimed pages
+		 */
+		VM_OBJ_TERM_FREELIST(local_free_count,
+				     local_free_count,
+				     local_free_q);
+		vm_page_unlock_queues();
 	} else if (!queue_empty(&object->memq)) {
 		panic("vm_object_reap: queue just emptied isn't");
 	}
@@ -1210,6 +1482,7 @@ vm_object_reap(
 
 	object->shadow = VM_OBJECT_NULL;
 
+	vm_object_lock_destroy(object);
 	/*
 	 *	Free the space for the object.
 	 */
@@ -1223,8 +1496,8 @@ vm_object_reap_async(
 {
 #if DEBUG
 	mutex_assert(&vm_object_cached_lock_data, MA_OWNED);
-	mutex_assert(&object->Lock, MA_OWNED);
 #endif /* DEBUG */
+	vm_object_lock_assert_exclusive(object);
 
 	vm_object_reap_count_async++;
 
@@ -1250,6 +1523,23 @@ vm_object_reaper_thread(void)
 		vm_object_lock(object);
 		assert(object->terminating);
 		assert(!object->alive);
+		
+		/*
+		 * The pageout daemon might be playing with our pages.
+		 * Now that the object is dead, it won't touch any more
+		 * pages, but some pages might already be on their way out.
+		 * Hence, we wait until the active paging activities have
+		 * ceased before we break the association with the pager
+		 * itself.
+		 */
+		while (object->paging_in_progress != 0) {
+			vm_object_cache_unlock();
+			vm_object_wait(object,
+				       VM_OBJECT_EVENT_PAGING_IN_PROGRESS,
+				       THREAD_UNINT);
+			vm_object_cache_lock();
+			vm_object_lock(object);
+		}
 
 		shadow_object =
 			object->pageout ? VM_OBJECT_NULL : object->shadow;
@@ -1406,6 +1696,11 @@ vm_object_destroy(
 	return(KERN_SUCCESS);
 }
 
+#define VM_OBJ_DEACT_ALL_STATS DEBUG
+#if VM_OBJ_DEACT_ALL_STATS
+uint32_t vm_object_deactivate_all_pages_batches = 0;
+uint32_t vm_object_deactivate_all_pages_pages = 0;
+#endif /* VM_OBJ_DEACT_ALL_STATS */
 /*
  *	vm_object_deactivate_pages
  *
@@ -1419,13 +1714,45 @@ vm_object_deactivate_all_pages(
 	register vm_object_t	object)
 {
 	register vm_page_t	p;
-
+	int			loop_count;
+#if VM_OBJ_DEACT_ALL_STATS
+	int			pages_count;
+#endif /* VM_OBJ_DEACT_ALL_STATS */
+#define V_O_D_A_P_MAX_BATCH	256
+
+	loop_count = V_O_D_A_P_MAX_BATCH;
+#if VM_OBJ_DEACT_ALL_STATS
+	pages_count = 0;
+#endif /* VM_OBJ_DEACT_ALL_STATS */
+	vm_page_lock_queues();
 	queue_iterate(&object->memq, p, vm_page_t, listq) {
-		vm_page_lock_queues();
-		if (!p->busy)
+		if (--loop_count == 0) {
+#if VM_OBJ_DEACT_ALL_STATS
+			hw_atomic_add(&vm_object_deactivate_all_pages_batches,
+				      1);
+			hw_atomic_add(&vm_object_deactivate_all_pages_pages,
+				      pages_count);
+			pages_count = 0;
+#endif /* VM_OBJ_DEACT_ALL_STATS */
+			mutex_yield(&vm_page_queue_lock);
+			loop_count = V_O_D_A_P_MAX_BATCH;
+		}
+		if (!p->busy && !p->throttled) {
+#if VM_OBJ_DEACT_ALL_STATS
+			pages_count++;
+#endif /* VM_OBJ_DEACT_ALL_STATS */
 			vm_page_deactivate(p);
-		vm_page_unlock_queues();
+		}
+	}
+#if VM_OBJ_DEACT_ALL_STATS
+	if (pages_count) {
+		hw_atomic_add(&vm_object_deactivate_all_pages_batches, 1);
+		hw_atomic_add(&vm_object_deactivate_all_pages_pages,
+			      pages_count);
+		pages_count = 0;
 	}
+#endif /* VM_OBJ_DEACT_ALL_STATS */
+	vm_page_unlock_queues();
 }
 
 __private_extern__ void
@@ -1471,29 +1798,40 @@ vm_object_deactivate_pages(
 				        	m->precious = FALSE;
 					        m->dirty = FALSE;
 						pmap_clear_modify(m->phys_page);
+#if	MACH_PAGEMAP
 						vm_external_state_clr(object->existence_map, offset);
+#endif	/* MACH_PAGEMAP */
 					}
-					VM_PAGE_QUEUES_REMOVE(m);
 
-					assert(!m->laundry);
-					assert(m->object != kernel_object);
-					assert(m->pageq.next == NULL &&
-					       m->pageq.prev == NULL);
-					if(m->zero_fill) {
-						queue_enter_first(
+					if (!m->throttled) {
+						VM_PAGE_QUEUES_REMOVE(m);
+
+						assert(!m->laundry);
+						assert(m->object != kernel_object);
+						assert(m->pageq.next == NULL &&
+						       m->pageq.prev == NULL);
+					
+						if(m->zero_fill) {
+							queue_enter_first(
 							&vm_page_queue_zf, 
 							m, vm_page_t, pageq);
-					} else {
-						queue_enter_first(
-							&vm_page_queue_inactive, 
-							m, vm_page_t, pageq);
-					}
+							vm_zf_queue_count++;
+						} else {
+						        queue_enter_first(
+									  &vm_page_queue_inactive, 
+									  m, vm_page_t, pageq);
+						}
 
-					m->inactive = TRUE;
-					if (!m->fictitious)  
-					        vm_page_inactive_count++;
+						m->inactive = TRUE;
+						if (!m->fictitious) {
+							vm_page_inactive_count++;
+							token_new_pagecount++;
+						} else {
+							assert(m->phys_page == vm_page_fictitious_addr);
+						}
 
-					pages_moved++;
+						pages_moved++;
+					}
 				}
 			}
 		}
@@ -1560,6 +1898,28 @@ vm_object_pmap_protect(
 
 	vm_object_lock(object);
 
+	if (object->phys_contiguous) {
+		if (pmap != NULL) {
+			vm_object_unlock(object);
+			pmap_protect(pmap, pmap_start, pmap_start + size, prot);
+		} else {
+			vm_object_offset_t phys_start, phys_end, phys_addr;
+
+			phys_start = object->shadow_offset + offset;
+			phys_end = phys_start + size;
+			assert(phys_start <= phys_end);
+			assert(phys_end <= object->shadow_offset + object->size);
+			vm_object_unlock(object);
+
+			for (phys_addr = phys_start;
+			     phys_addr < phys_end;
+			     phys_addr += PAGE_SIZE_64) {
+				pmap_page_protect(phys_addr >> 12, prot);
+			}
+		}
+		return;
+	}
+
 	assert(object->internal);
 
 	while (TRUE) {
@@ -1593,8 +1953,7 @@ vm_object_pmap_protect(
 		    if (!p->fictitious &&
 			(offset <= p->offset) && (p->offset < end)) {
 
-			    pmap_page_protect(p->phys_page,
-					      prot & ~p->page_lock);
+		        pmap_page_protect(p->phys_page, prot);
 		    }
 		  }
 		}
@@ -1623,8 +1982,7 @@ vm_object_pmap_protect(
 				target_off < end; target_off += PAGE_SIZE) {
 				p = vm_page_lookup(object, target_off);
 				if (p != VM_PAGE_NULL) {
-		    			pmap_page_protect(p->phys_page,
-						      prot & ~p->page_lock);
+				        pmap_page_protect(p->phys_page, prot);
 				}
 		    	}
 		}
@@ -1704,8 +2062,7 @@ vm_object_copy_slowly(
 	vm_object_t		new_object;
 	vm_object_offset_t	new_offset;
 
-	vm_object_offset_t	src_lo_offset = src_offset;
-	vm_object_offset_t	src_hi_offset = src_offset + size;
+	struct vm_object_fault_info fault_info;
 
 	XPR(XPR_VM_OBJECT, "v_o_c_slowly obj 0x%x off 0x%x size 0x%x\n",
 	    src_object, src_offset, size, 0, 0);
@@ -1720,9 +2077,7 @@ vm_object_copy_slowly(
 	 *	Prevent destruction of the source object while we copy.
 	 */
 
-	assert(src_object->ref_count > 0);
-	src_object->ref_count++;
-	VM_OBJ_RES_INCR(src_object);
+	vm_object_reference_locked(src_object);
 	vm_object_unlock(src_object);
 
 	/*
@@ -1736,10 +2091,16 @@ vm_object_copy_slowly(
 
 	new_object = vm_object_allocate(size);
 	new_offset = 0;
-	vm_object_lock(new_object);
 
 	assert(size == trunc_page_64(size));	/* Will the loop terminate? */
 
+	fault_info.interruptible = interruptible;
+	fault_info.behavior  = VM_BEHAVIOR_SEQUENTIAL;
+	fault_info.user_tag  = 0;
+	fault_info.lo_offset = src_offset;
+	fault_info.hi_offset = src_offset + size;
+	fault_info.no_cache  = FALSE;
+
 	for ( ;
 	    size != 0 ;
 	    src_offset += PAGE_SIZE_64, 
@@ -1748,16 +2109,22 @@ vm_object_copy_slowly(
 		vm_page_t	new_page;
 		vm_fault_return_t result;
 
+		vm_object_lock(new_object);
+
 		while ((new_page = vm_page_alloc(new_object, new_offset))
 				== VM_PAGE_NULL) {
+
+			vm_object_unlock(new_object);
+
 			if (!vm_page_wait(interruptible)) {
-			        vm_object_unlock(new_object);
 				vm_object_deallocate(new_object);
 				vm_object_deallocate(src_object);
 				*_result_object = VM_OBJECT_NULL;
 				return(MACH_SEND_INTERRUPTED);
 			}
+			vm_object_lock(new_object);
 		}
+		vm_object_unlock(new_object);
 
 		do {
 			vm_prot_t	prot = VM_PROT_READ;
@@ -1770,14 +2137,14 @@ vm_object_copy_slowly(
 			vm_object_lock(src_object);
 			vm_object_paging_begin(src_object);
 
+			fault_info.cluster_size = size;
+
 			XPR(XPR_VM_FAULT,"vm_object_copy_slowly -> vm_fault_page",0,0,0,0,0);
 			result = vm_fault_page(src_object, src_offset,
-				VM_PROT_READ, FALSE, interruptible,
-				src_lo_offset, src_hi_offset,
-				VM_BEHAVIOR_SEQUENTIAL,
+				VM_PROT_READ, FALSE,
 				&prot, &_result_page, &top_page,
 			        (int *)0,
-				&error_code, FALSE, FALSE, NULL, 0);
+				&error_code, FALSE, FALSE, &fault_info);
 
 			switch(result) {
 				case VM_FAULT_SUCCESS:
@@ -1804,15 +2171,18 @@ vm_object_copy_slowly(
 					 *	Let go of both pages (make them
 					 *	not busy, perform wakeup, activate).
 					 */
-
-					new_page->busy = FALSE;
+					vm_object_lock(new_object);
 					new_page->dirty = TRUE;
+					PAGE_WAKEUP_DONE(new_page);
+					vm_object_unlock(new_object);
+
 					vm_object_lock(result_page->object);
 					PAGE_WAKEUP_DONE(result_page);
 
-					vm_page_lock_queues();
+					vm_page_lockspin_queues();
 					if (!result_page->active &&
-					    !result_page->inactive)
+					    !result_page->inactive &&
+					    !result_page->throttled)
 						vm_page_activate(result_page);
 					vm_page_activate(new_page);
 					vm_page_unlock_queues();
@@ -1841,7 +2211,6 @@ vm_object_copy_slowly(
 
 				case VM_FAULT_INTERRUPTED:
 					vm_page_free(new_page);
-					vm_object_unlock(new_object);
 					vm_object_deallocate(new_object);
 					vm_object_deallocate(src_object);
 					*_result_object = VM_OBJECT_NULL;
@@ -1859,7 +2228,7 @@ vm_object_copy_slowly(
 					vm_page_lock_queues();
 					vm_page_free(new_page);
 					vm_page_unlock_queues();
-					vm_object_unlock(new_object);
+
 					vm_object_deallocate(new_object);
 					vm_object_deallocate(src_object);
 					*_result_object = VM_OBJECT_NULL;
@@ -1872,8 +2241,6 @@ vm_object_copy_slowly(
 	/*
 	 *	Lose the extra reference, and return our object.
 	 */
-
-        vm_object_unlock(new_object);
 	vm_object_deallocate(src_object);
 	*_result_object = new_object;
 	return(KERN_SUCCESS);
@@ -1929,9 +2296,7 @@ vm_object_copy_quickly(
 		 *	Leave object/offset unchanged.
 		 */
 
-		assert(object->ref_count > 0);
-		object->ref_count++;
-		vm_object_res_reference(object);
+		vm_object_reference_locked(object);
 		object->shadowed = TRUE;
 		vm_object_unlock(object);
 
@@ -1989,6 +2354,7 @@ vm_object_copy_call(
 	kern_return_t	kr;
 	vm_object_t	copy;
 	boolean_t	check_ready = FALSE;
+	uint32_t	try_failed_count = 0;
 
 	/*
 	 *	If a copy is already in progress, wait and retry.
@@ -2048,7 +2414,10 @@ Retry:
 	copy = src_object->copy;
 	if (!vm_object_lock_try(copy)) {
 		vm_object_unlock(src_object);
-		mutex_pause();	/* wait a bit */
+
+		try_failed_count++;
+		mutex_pause(try_failed_count);	/* wait a bit */
+
 		vm_object_lock(src_object);
 		goto Retry;
 	}
@@ -2100,13 +2469,15 @@ __private_extern__ vm_object_t
 vm_object_copy_delayed(
 	vm_object_t		src_object,
 	vm_object_offset_t	src_offset,
-	vm_object_size_t	size)
+	vm_object_size_t	size,
+	boolean_t		src_object_shared)
 {
 	vm_object_t		new_copy = VM_OBJECT_NULL;
 	vm_object_t		old_copy;
 	vm_page_t		p;
 	vm_object_size_t	copy_size = src_offset + size;
 
+
 	int collisions = 0;
 	/*
 	 *	The user-level memory manager wants to see all of the changes
@@ -2152,9 +2523,15 @@ vm_object_copy_delayed(
 	/*
 	 * Wait for paging in progress.
 	 */
-	if (!src_object->true_share)
+	if (!src_object->true_share && src_object->paging_in_progress) {
+	        if (src_object_shared == TRUE) {
+		        vm_object_unlock(src_object);
+				
+			vm_object_lock(src_object);
+			src_object_shared = FALSE;
+		}
 		vm_object_paging_wait(src_object, THREAD_UNINT);
-
+	}
 	/*
 	 *	See whether we can reuse the result of a previous
 	 *	copy operation.
@@ -2162,22 +2539,34 @@ vm_object_copy_delayed(
 
 	old_copy = src_object->copy;
 	if (old_copy != VM_OBJECT_NULL) {
+	        int lock_granted;
+
 		/*
 		 *	Try to get the locks (out of order)
 		 */
-		if (!vm_object_lock_try(old_copy)) {
+		if (src_object_shared == TRUE)
+		        lock_granted = vm_object_lock_try_shared(old_copy);
+		else
+		        lock_granted = vm_object_lock_try(old_copy);
+
+		if (!lock_granted) {
 			vm_object_unlock(src_object);
-			mutex_pause();
 
-			/* Heisenberg Rules */
-			copy_delayed_lock_collisions++;
 			if (collisions++ == 0)
 				copy_delayed_lock_contention++;
+			mutex_pause(collisions);
+
+			/* Heisenberg Rules */
+			copy_delayed_lock_collisions++;
 
 			if (collisions > copy_delayed_max_collisions)
 				copy_delayed_max_collisions = collisions;
 
-			vm_object_lock(src_object);
+			if (src_object_shared == TRUE)
+			        vm_object_lock_shared(src_object);
+			else
+			        vm_object_lock(src_object);
+
 			goto Retry;
 		}
 
@@ -2198,6 +2587,14 @@ vm_object_copy_delayed(
 			 */
 
 			if (old_copy->size < copy_size) {
+			        if (src_object_shared == TRUE) {
+				        vm_object_unlock(old_copy);
+					vm_object_unlock(src_object);
+				
+					vm_object_lock(src_object);
+					src_object_shared = FALSE;
+					goto Retry;
+				}
 				/*
 				 * We can't perform a delayed copy if any of the
 				 * pages in the extended range are wired (because
@@ -2206,6 +2603,7 @@ vm_object_copy_delayed(
 				 * go ahead and protect them.
 				 */
 				copy_delayed_protect_iterate++;
+
 				queue_iterate(&src_object->memq, p, vm_page_t, listq) {
 					if (!p->fictitious && 
 					    p->offset >= old_copy->size && 
@@ -2222,15 +2620,16 @@ vm_object_copy_delayed(
 							return VM_OBJECT_NULL;
 						} else {
 							pmap_page_protect(p->phys_page, 
-								(VM_PROT_ALL & ~VM_PROT_WRITE &
-								 ~p->page_lock));
+									  (VM_PROT_ALL & ~VM_PROT_WRITE));
 						}
 					}
 				}
 				old_copy->size = copy_size;
 			}
-				
-			vm_object_reference_locked(old_copy);
+			if (src_object_shared == TRUE)
+			        vm_object_reference_shared(old_copy);
+			else
+			        vm_object_reference_locked(old_copy);
 			vm_object_unlock(old_copy);
 			vm_object_unlock(src_object);
 
@@ -2238,9 +2637,10 @@ vm_object_copy_delayed(
 				vm_object_unlock(new_copy);
 				vm_object_deallocate(new_copy);
 			}
-
 			return(old_copy);
 		}
+		
+		
 
 		/*
 		 * Adjust the size argument so that the newly-created 
@@ -2256,6 +2656,8 @@ vm_object_copy_delayed(
 			new_copy = vm_object_allocate(copy_size);
 			vm_object_lock(src_object);
 			vm_object_lock(new_copy);
+
+			src_object_shared = FALSE;
 			goto Retry;
 		}
 		new_copy->size = copy_size;	
@@ -2275,6 +2677,8 @@ vm_object_copy_delayed(
 		new_copy = vm_object_allocate(copy_size);
 		vm_object_lock(src_object);
 		vm_object_lock(new_copy);
+
+		src_object_shared = FALSE;
 		goto Retry;
 	}
 
@@ -2290,6 +2694,7 @@ vm_object_copy_delayed(
 	 * wired, then go ahead and protect them.
 	 */
 	copy_delayed_protect_iterate++;
+
 	queue_iterate(&src_object->memq, p, vm_page_t, listq) {
 		if (!p->fictitious && p->offset < copy_size) {
 			if (p->wire_count > 0) {
@@ -2301,12 +2706,10 @@ vm_object_copy_delayed(
 				return VM_OBJECT_NULL;
 			} else {
 				pmap_page_protect(p->phys_page, 
-					(VM_PROT_ALL & ~VM_PROT_WRITE &
-					 ~p->page_lock));
+						  (VM_PROT_ALL & ~VM_PROT_WRITE));
 			}
 		}
 	}
-
 	if (old_copy != VM_OBJECT_NULL) {
 		/*
 		 *	Make the old copy-object shadow the new one.
@@ -2314,9 +2717,13 @@ vm_object_copy_delayed(
 		 *	object.
 		 */
 
-		src_object->ref_count--;	/* remove ref. from old_copy */
+		/* remove ref. from old_copy */
+		vm_object_lock_assert_exclusive(src_object);
+		src_object->ref_count--;
 		assert(src_object->ref_count > 0);
+		vm_object_lock_assert_exclusive(old_copy);
 		old_copy->shadow = new_copy;
+		vm_object_lock_assert_exclusive(new_copy);
 		assert(new_copy->ref_count > 0);
 		new_copy->ref_count++;		/* for old_copy->shadow ref. */
 
@@ -2333,12 +2740,13 @@ vm_object_copy_delayed(
 	/*
 	 *	Point the new copy at the existing object.
 	 */
+	vm_object_lock_assert_exclusive(new_copy);
 	new_copy->shadow = src_object;
 	new_copy->shadow_offset = 0;
 	new_copy->shadowed = TRUE;	/* caller must set needs_copy */
-	assert(src_object->ref_count > 0);
-	src_object->ref_count++;
-	VM_OBJ_RES_INCR(src_object);
+
+	vm_object_lock_assert_exclusive(src_object);
+	vm_object_reference_locked(src_object);
 	src_object->copy = new_copy;
 	vm_object_unlock(src_object);
 	vm_object_unlock(new_copy);
@@ -2347,7 +2755,7 @@ vm_object_copy_delayed(
 		"vm_object_copy_delayed: used copy object %X for source %X\n",
 		(integer_t)new_copy, (integer_t)src_object, 0, 0, 0);
 
-	return(new_copy);
+	return new_copy;
 }
 
 /*
@@ -2369,11 +2777,18 @@ vm_object_copy_strategically(
 {
 	boolean_t	result;
 	boolean_t	interruptible = THREAD_ABORTSAFE; /* XXX */
+	boolean_t	object_lock_shared = FALSE;
 	memory_object_copy_strategy_t copy_strategy;
 
 	assert(src_object != VM_OBJECT_NULL);
 
-	vm_object_lock(src_object);
+	copy_strategy = src_object->copy_strategy;
+
+	if (copy_strategy == MEMORY_OBJECT_COPY_DELAY) {
+	        vm_object_lock_shared(src_object);
+		object_lock_shared = TRUE;
+	} else
+	        vm_object_lock(src_object);
 
 	/*
 	 *	The copy strategy is only valid if the memory manager
@@ -2383,6 +2798,12 @@ vm_object_copy_strategically(
 	while (!src_object->internal && !src_object->pager_ready) {
 		wait_result_t wait_result;
 
+		if (object_lock_shared == TRUE) {
+		        vm_object_unlock(src_object);
+			vm_object_lock(src_object);
+			object_lock_shared = FALSE;
+			continue;
+		}
 		wait_result = vm_object_sleep(	src_object,
 						VM_OBJECT_EVENT_PAGER_READY,
 						interruptible);
@@ -2395,8 +2816,6 @@ vm_object_copy_strategically(
 		}
 	}
 
-	copy_strategy = src_object->copy_strategy;
-
 	/*
 	 *	Use the appropriate copy strategy.
 	 */
@@ -2404,7 +2823,7 @@ vm_object_copy_strategically(
 	switch (copy_strategy) {
 	    case MEMORY_OBJECT_COPY_DELAY:
 		*dst_object = vm_object_copy_delayed(src_object,
-						     src_offset, size);
+						     src_offset, size, object_lock_shared);
 		if (*dst_object != VM_OBJECT_NULL) {
 			*dst_offset = src_offset;
 			*dst_needs_copy = TRUE;
@@ -2467,7 +2886,22 @@ vm_object_shadow(
 	register vm_object_t	result;
 
 	source = *object;
+#if 0
+	/*
+	 * XXX FBDP
+	 * This assertion is valid but it gets triggered by Rosetta for example
+	 * due to a combination of vm_remap() that changes a VM object's
+	 * copy_strategy from SYMMETRIC to DELAY and vm_protect(VM_PROT_COPY)
+	 * that then sets "needs_copy" on its map entry.  This creates a
+	 * mapping situation that VM should never see and doesn't know how to
+	 * handle.
+	 * It's not clear if this can create any real problem but we should
+	 * look into fixing this, probably by having vm_protect(VM_PROT_COPY)
+	 * do more than just set "needs_copy" to handle the copy-on-write...
+	 * In the meantime, let's disable the assertion.
+	 */
 	assert(source->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);
+#endif
 
 	/*
 	 *	Determine if we really need a shadow.
@@ -2579,175 +3013,19 @@ vm_object_shadow(
  *	Because the pager field may be cleared spontaneously, it
  *	cannot be used to determine whether a memory object has
  *	ever been associated with a particular vm_object.  [This
- *	knowledge is important to the shadow object mechanism.]
- *	For this reason, an additional "created" attribute is
- *	provided.
- *
- *	During various paging operations, the pager reference found in the
- *	vm_object must be valid.  To prevent this from being released,
- *	(other than being removed, i.e., made null), routines may use
- *	the vm_object_paging_begin/end routines [actually, macros].
- *	The implementation uses the "paging_in_progress" and "wanted" fields.
- *	[Operations that alter the validity of the pager values include the
- *	termination routines and vm_object_collapse.]
- */
-
-#if 0
-static void		vm_object_abort_activity(
-				vm_object_t	object);
-
-/*
- *	Routine:	vm_object_abort_activity [internal use only]
- *	Purpose:
- *		Abort paging requests pending on this object.
- *	In/out conditions:
- *		The object is locked on entry and exit.
- */
-static void
-vm_object_abort_activity(
-	vm_object_t	object)
-{
-	register
-	vm_page_t	p;
-	vm_page_t	next;
-
-	XPR(XPR_VM_OBJECT, "vm_object_abort_activity, object 0x%X\n",
-		(integer_t)object, 0, 0, 0, 0);
-
-	/*
-	 *	Abort all activity that would be waiting
-	 *	for a result on this memory object.
-	 *
-	 *	We could also choose to destroy all pages
-	 *	that we have in memory for this object, but
-	 *	we don't.
-	 */
-
-	p = (vm_page_t) queue_first(&object->memq);
-	while (!queue_end(&object->memq, (queue_entry_t) p)) {
-		next = (vm_page_t) queue_next(&p->listq);
-
-		/*
-		 *	If it's being paged in, destroy it.
-		 *	If an unlock has been requested, start it again.
-		 */
-
-		if (p->busy && p->absent) {
-			VM_PAGE_FREE(p);
-		}
-		 else {
-		 	if (p->unlock_request != VM_PROT_NONE)
-			 	p->unlock_request = VM_PROT_NONE;
-			PAGE_WAKEUP(p);
-		}
-		
-		p = next;
-	}
-
-	/*
-	 *	Wake up threads waiting for the memory object to
-	 *	become ready.
-	 */
-
-	object->pager_ready = TRUE;
-	vm_object_wakeup(object, VM_OBJECT_EVENT_PAGER_READY);
-}
-
-/*
- *	Routine:	vm_object_pager_dead
- *
- *	Purpose:
- *		A port is being destroy, and the IPC kobject code
- *		can't tell if it represents a pager port or not.
- *		So this function is called each time it sees a port
- *		die.
- *		THIS IS HORRIBLY INEFFICIENT.  We should only call
- *		this routine if we had requested a notification on
- *		the port.
- */
-
-__private_extern__ void
-vm_object_pager_dead(
-	ipc_port_t	pager)
-{
-	vm_object_t		object;
-	vm_object_hash_entry_t	entry;
-
-	/*
-	 *	Perform essentially the same operations as in vm_object_lookup,
-	 *	except that this time we look up based on the memory_object
-	 *	port, not the control port.
-	 */
-	vm_object_cache_lock();
-	entry = vm_object_hash_lookup(pager, FALSE);
-	if (entry == VM_OBJECT_HASH_ENTRY_NULL ||
-	    		entry->object == VM_OBJECT_NULL) {
-		vm_object_cache_unlock();
-		return;
-	}
-
-	object = entry->object;
-	entry->object = VM_OBJECT_NULL;
-
-	vm_object_lock(object);
-	if (object->ref_count == 0) {
-		XPR(XPR_VM_OBJECT_CACHE,
-		   "vm_object_destroy: removing %x from cache, head (%x, %x)\n",
-			(integer_t)object,
-			(integer_t)vm_object_cached_list.next,
-			(integer_t)vm_object_cached_list.prev, 0,0);
-
-		queue_remove(&vm_object_cached_list, object,
-				vm_object_t, cached_list);
-		vm_object_cached_count--;
-	}
-	object->ref_count++;
-	vm_object_res_reference(object);
-
-	object->can_persist = FALSE;
-
-	assert(object->pager == pager);
-
-	/*
-	 *	Remove the pager association.
-	 *
-	 *	Note that the memory_object itself is dead, so
-	 *	we don't bother with it.
-	 */
-
-	object->pager = MEMORY_OBJECT_NULL;
-
-	vm_object_unlock(object);
-	vm_object_cache_unlock();
-
-	vm_object_pager_wakeup(pager);
-
-	/*
-	 *	Release the pager reference.  Note that there's no
-	 *	point in trying the memory_object_terminate call
-	 *	because the memory_object itself is dead.  Also
-	 *	release the memory_object_control reference, since
-	 *	the pager didn't do that either.
-	 */
-
-	memory_object_deallocate(pager);
-	memory_object_control_deallocate(object->pager_request);
-	
-
-	/*
-	 *	Restart pending page requests
-	 */
-	vm_object_lock(object);
-	vm_object_abort_activity(object);
-	vm_object_unlock(object);
-
-	/*
-	 *	Lose the object reference.
-	 */
+ *	knowledge is important to the shadow object mechanism.]
+ *	For this reason, an additional "created" attribute is
+ *	provided.
+ *
+ *	During various paging operations, the pager reference found in the
+ *	vm_object must be valid.  To prevent this from being released,
+ *	(other than being removed, i.e., made null), routines may use
+ *	the vm_object_paging_begin/end routines [actually, macros].
+ *	The implementation uses the "paging_in_progress" and "wanted" fields.
+ *	[Operations that alter the validity of the pager values include the
+ *	termination routines and vm_object_collapse.]
+ */
 
-	vm_object_deallocate(object);
-}
-#endif
 
 /*
  *	Routine:	vm_object_enter
@@ -2768,6 +3046,7 @@ vm_object_enter(
 	vm_object_t		new_object;
 	boolean_t		must_init;
 	vm_object_hash_entry_t	entry, new_entry;
+	uint32_t        try_failed_count = 0;
 
 	if (pager == MEMORY_OBJECT_NULL)
 		return(vm_object_allocate(size));
@@ -2779,7 +3058,7 @@ vm_object_enter(
 	/*
 	 *	Look for an object associated with this port.
 	 */
-
+Retry:
 	vm_object_cache_lock();
 	do {
 		entry = vm_object_hash_lookup(pager, FALSE);
@@ -2826,7 +3105,15 @@ vm_object_enter(
 	assert(object != VM_OBJECT_NULL);
 
 	if (!must_init) {
-		vm_object_lock(object);
+	        if (!vm_object_lock_try(object)) {
+
+		        vm_object_cache_unlock();
+
+		        try_failed_count++;
+			mutex_pause(try_failed_count);  /* wait a bit */
+
+			goto Retry;
+		}
 		assert(!internal || object->internal);
 		if (named) {
 			assert(!object->named);
@@ -2842,15 +3129,16 @@ vm_object_enter(
 				     vm_object_t, cached_list);
 			vm_object_cached_count--;
 		}
+		vm_object_lock_assert_exclusive(object);
 		object->ref_count++;
 		vm_object_res_reference(object);
 		vm_object_unlock(object);
 
-		VM_STAT(hits++);
+		VM_STAT_INCR(hits);
 	} 
 	assert(object->ref_count > 0);
 
-	VM_STAT(lookups++);
+	VM_STAT_INCR(lookups);
 
 	vm_object_cache_unlock();
 
@@ -3026,13 +3314,10 @@ vm_object_pager_create(
 	 */
 	{
 		memory_object_default_t		dmm;
-		vm_size_t	cluster_size;
 
 		/* acquire a reference for the default memory manager */
-		dmm = memory_manager_default_reference(&cluster_size);
-		assert(cluster_size >= PAGE_SIZE);
+		dmm = memory_manager_default_reference();
 
-		object->cluster_size = cluster_size; /* XXX ??? */
 		assert(object->temporary);
 
 		/* create our new memory object */
@@ -3107,8 +3392,10 @@ static long	object_bypasses  = 0;
 static boolean_t	vm_object_collapse_allowed = TRUE;
 static boolean_t	vm_object_bypass_allowed = TRUE;
 
+#if MACH_PAGEMAP
 static int	vm_external_discarded;
 static int	vm_external_collapsed;
+#endif
 
 unsigned long vm_object_collapse_encrypted = 0;
 
@@ -3164,16 +3451,15 @@ vm_object_do_collapse(
 			/*
 			 * ENCRYPTED SWAP:
 			 * The encryption key includes the "pager" and the
-			 * "paging_offset".  These might not be the same in
-			 * the new object, so we can't just move an encrypted
-			 * page from one object to the other.  We can't just
-			 * decrypt the page here either, because that would drop
+			 * "paging_offset".  These will not change during the 
+			 * object collapse, so we can just move an encrypted
+			 * page from one object to the other in this case.
+			 * We can't decrypt the page here, since we can't drop
 			 * the object lock.
-			 * The caller should check for encrypted pages before
-			 * attempting to collapse.
 			 */
-			ASSERT_PAGE_DECRYPTED(p);
-
+			if (p->encrypted) {
+				vm_object_collapse_encrypted++;
+			}
 			pp = vm_page_lookup(object, new_offset);
 			if (pp == VM_PAGE_NULL) {
 
@@ -3182,7 +3468,7 @@ vm_object_do_collapse(
 				 *	Move the backing object's page up.
 				 */
 
-				vm_page_rename(p, object, new_offset);
+				vm_page_rename(p, object, new_offset, TRUE);
 #if	MACH_PAGEMAP
 			} else if (pp->absent) {
 
@@ -3199,7 +3485,7 @@ vm_object_do_collapse(
 				 */
 
 				VM_PAGE_FREE(pp);
-				vm_page_rename(p, object, new_offset);
+				vm_page_rename(p, object, new_offset, TRUE);
 #endif	/* MACH_PAGEMAP */
 			} else {
 				assert(! pp->absent);
@@ -3215,9 +3501,9 @@ vm_object_do_collapse(
 	}
 	
 #if	!MACH_PAGEMAP
-	assert(!object->pager_created && object->pager == MEMORY_OBJECT_NULL
+	assert((!object->pager_created && (object->pager == MEMORY_OBJECT_NULL))
 		|| (!backing_object->pager_created
-		&&  backing_object->pager == MEMORY_OBJECT_NULL));
+		&&  (backing_object->pager == MEMORY_OBJECT_NULL)));
 #else 
         assert(!object->pager_created && object->pager == MEMORY_OBJECT_NULL);
 #endif	/* !MACH_PAGEMAP */
@@ -3242,7 +3528,6 @@ vm_object_do_collapse(
 		object->pager_control = backing_object->pager_control;
 		object->pager_ready = backing_object->pager_ready;
 		object->pager_initialized = backing_object->pager_initialized;
-		object->cluster_size = backing_object->cluster_size;
 		object->paging_offset =
 		    backing_object->paging_offset + backing_offset;
 		if (object->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
@@ -3315,6 +3600,8 @@ vm_object_do_collapse(
 	XPR(XPR_VM_OBJECT, "vm_object_collapse, collapsed 0x%X\n",
 		(integer_t)backing_object, 0,0,0,0);
 
+	vm_object_lock_destroy(backing_object);
+
 	zfree(vm_object_zone, backing_object);
 	
 	object_collapses++;
@@ -3330,6 +3617,8 @@ vm_object_do_bypass(
 	 *	in the chain.
 	 */
 	
+	vm_object_lock_assert_exclusive(backing_object);
+
 #if	TASK_SWAPPER
 	/*
 	 *	Do object reference in-line to 
@@ -3340,6 +3629,7 @@ vm_object_do_bypass(
 	 */
 	if (backing_object->shadow != VM_OBJECT_NULL) {
 		vm_object_lock(backing_object->shadow);
+		vm_object_lock_assert_exclusive(backing_object->shadow);
 		backing_object->shadow->ref_count++;
 		if (object->res_count != 0)
 			vm_object_res_reference(backing_object->shadow);
@@ -3392,6 +3682,7 @@ vm_object_do_bypass(
 #endif
 	 */
 	if (backing_object->ref_count > 1) {
+		vm_object_lock_assert_exclusive(backing_object);
 		backing_object->ref_count--;
 #if	TASK_SWAPPER
 		if (object->res_count != 0)
@@ -3444,6 +3735,7 @@ static unsigned long vm_object_collapse_calls = 0;
 static unsigned long vm_object_collapse_objects = 0;
 static unsigned long vm_object_collapse_do_collapse = 0;
 static unsigned long vm_object_collapse_do_bypass = 0;
+static unsigned long vm_object_collapse_delays = 0;
 __private_extern__ void
 vm_object_collapse(
 	register vm_object_t			object,
@@ -3453,9 +3745,6 @@ vm_object_collapse(
 	register vm_object_t			backing_object;
 	register unsigned int			rcount;
 	register unsigned int			size;
-	vm_object_offset_t			collapse_min_offset;
-	vm_object_offset_t			collapse_max_offset;
-	vm_page_t				page;
 	vm_object_t				original_object;
 
 	vm_object_collapse_calls++;
@@ -3496,8 +3785,7 @@ vm_object_collapse(
 		 *	No pages in the object are currently
 		 *	being paged out, and
 		 */
-		if (object->paging_in_progress != 0 ||
-		    object->absent_count != 0) {
+		if (object->paging_in_progress != 0) {
 			/* try and collapse the rest of the shadow chain */
 			vm_object_lock(backing_object);
 			if (object != original_object) {
@@ -3594,39 +3882,6 @@ vm_object_collapse(
 				return;
 			}
 
-			/*
-			 * ENCRYPTED SWAP
-			 * We can't collapse the object if it contains
-			 * any encypted page, because the encryption key
-			 * includes the <object,offset> info.  We can't
-			 * drop the object lock in vm_object_do_collapse()
-			 * so we can't decrypt the page there either.
-			 */
-			if (vm_pages_encrypted) {
-				collapse_min_offset = object->shadow_offset;
-				collapse_max_offset =
-					object->shadow_offset + object->size;
-				queue_iterate(&backing_object->memq,
-					      page, vm_page_t, listq) {
-					if (page->encrypted &&
-					    (page->offset >=
-					     collapse_min_offset) &&
-					    (page->offset <
-					     collapse_max_offset)) {
-						/*
-						 * We found an encrypted page
-						 * in the backing object,
-						 * within the range covered 
-						 * by the parent object: we can
-						 * not collapse them.
-						 */
-						vm_object_collapse_encrypted++;
-						vm_object_cache_unlock();
-						goto try_bypass;
-					}
-				}
-			}
-		       
 			/*
 			 *	Collapse the object with its backing
 			 *	object, and try again with the object's
@@ -3638,7 +3893,6 @@ vm_object_collapse(
 			continue;
 		}
 
-	try_bypass:
 		/*
 		 *	Collapsing the backing object was not possible
 		 *	or permitted, so let's try bypassing it.
@@ -3719,10 +3973,15 @@ vm_object_collapse(
 			backing_offset = object->shadow_offset;
 			backing_rcount = backing_object->resident_page_count;
 
+#if	MACH_PAGEMAP
 #define EXISTS_IN_OBJECT(obj, off, rc) \
 	(vm_external_state_get((obj)->existence_map, \
 	 (vm_offset_t)(off)) == VM_EXTERNAL_STATE_EXISTS || \
 	 ((rc) && ++lookups && vm_page_lookup((obj), (off)) != VM_PAGE_NULL && (rc)--))
+#else
+#define EXISTS_IN_OBJECT(obj, off, rc) \
+	(((rc) && ++lookups && vm_page_lookup((obj), (off)) != VM_PAGE_NULL && (rc)--))
+#endif	/* MACH_PAGEMAP */
 
 			/*
 			 * Check the hint location first
@@ -3761,9 +4020,14 @@ vm_object_collapse(
 			 * double-decrement the rcount.  We also may or
 			 * may not have found the 
 			 */
-			if (backing_rcount && size >
-			    ((backing_object->existence_map) ?
-			     backing_rcount : (backing_rcount >> 1))) {
+			if (backing_rcount && 
+#if	MACH_PAGEMAP
+			    size > ((backing_object->existence_map) ?
+			     backing_rcount : (backing_rcount >> 1))
+#else
+			    size > (backing_rcount >> 1)
+#endif	/* MACH_PAGEMAP */
+				) {
 				unsigned int rc = rcount;
 				vm_page_t p;
 
@@ -3772,8 +4036,9 @@ vm_object_collapse(
 				do {
 					/* Until we get more than one lookup lock */
 					if (lookups > 256) {
+						vm_object_collapse_delays++;
 						lookups = 0;
-						delay(1);
+						mutex_pause(0);
 					}
 
 					offset = (p->offset - backing_offset);
@@ -3801,7 +4066,11 @@ vm_object_collapse(
 			 * Walk through the offsets looking for pages in the
 			 * backing object that show through to the object.
 			 */
+#if	MACH_PAGEMAP
 			if (backing_rcount || backing_object->existence_map) {
+#else
+			if (backing_rcount) {
+#endif	/* MACH_PAGEMAP */
 				offset = hint_offset;
 				
 				while((offset =
@@ -3810,8 +4079,9 @@ vm_object_collapse(
 
 					/* Until we get more than one lookup lock */
 					if (lookups > 256) {
+						vm_object_collapse_delays++;
 						lookups = 0;
-						delay(1);
+						mutex_pause(0);
 					}
 
 					if (EXISTS_IN_OBJECT(backing_object, offset +
@@ -3892,7 +4162,7 @@ vm_object_page_remove(
 			p = vm_page_lookup(object, start);
 			if (p != VM_PAGE_NULL) {
 				assert(!p->cleaning && !p->pageout);
-				if (!p->fictitious)
+				if (!p->fictitious && p->pmapped)
 				        pmap_disconnect(p->phys_page);
 				VM_PAGE_FREE(p);
 			}
@@ -3905,7 +4175,7 @@ vm_object_page_remove(
 			next = (vm_page_t) queue_next(&p->listq);
 			if ((start <= p->offset) && (p->offset < end)) {
 				assert(!p->cleaning && !p->pageout);
-				if (!p->fictitious)
+				if (!p->fictitious && p->pmapped)
 				        pmap_disconnect(p->phys_page);
 				VM_PAGE_FREE(p);
 			}
@@ -3981,7 +4251,7 @@ vm_object_coalesce(
 	 *	. paged out
 	 *	. shadows another object
 	 *	. has a copy elsewhere
-	 *	. is purgable
+	 *	. is purgeable
 	 *	. paging references (pages might be in page-list)
 	 */
 
@@ -3990,7 +4260,7 @@ vm_object_coalesce(
 	    (prev_object->shadow != VM_OBJECT_NULL) ||
 	    (prev_object->copy != VM_OBJECT_NULL) ||
 	    (prev_object->true_share != FALSE) ||
-	    (prev_object->purgable != VM_OBJECT_NONPURGABLE) ||
+	    (prev_object->purgable != VM_PURGABLE_DENY) ||
 	    (prev_object->paging_in_progress != 0)) {
 		vm_object_unlock(prev_object);
 		return(FALSE);
@@ -4188,11 +4458,8 @@ vm_follow_object(
  *	vm_object_print:	[ debug ]
  */
 void
-vm_object_print(
-	db_addr_t	db_addr,
-	__unused boolean_t	have_addr,
-	__unused int		arg_count,
-	__unused char		*modif)
+vm_object_print(db_expr_t db_addr, __unused boolean_t have_addr,
+		__unused db_expr_t arg_count, __unused char *modif)
 {
 	vm_object_t	object;
 	register vm_page_t p;
@@ -4209,7 +4476,6 @@ vm_object_print(
 	db_indent += 2;
 
 	iprintf("size=0x%x", object->size);
-	printf(", cluster=0x%x", object->cluster_size);
 	printf(", memq_hint=%p", object->memq_hint);
 	printf(", ref_count=%d\n", object->ref_count);
 	iprintf("");
@@ -4260,7 +4526,6 @@ vm_object_print(
 		printf("?");
 	}
 	printf("]");
-	printf(", absent_count=%d\n", object->absent_count);
 
 	iprintf("all_wanted=0x%x<", object->all_wanted);
 	s = "";
@@ -4276,10 +4541,6 @@ vm_object_print(
 		printf("%spaging", s);
 		s = ",";
 	}
-	if (vm_object_wanted(object, VM_OBJECT_EVENT_ABSENT_COUNT)) {
-		printf("%sabsent", s);
-		s = ",";
-	}
 	if (vm_object_wanted(object, VM_OBJECT_EVENT_LOCK_IN_PROGRESS)) {
 		printf("%slock", s);
 		s = ",";
@@ -4308,11 +4569,11 @@ vm_object_print(
 		(object->pageout ? "" : "!"),
 		(object->internal ? "internal" : "external"),
 		(object->temporary ? "temporary" : "permanent"));
-	iprintf("%salive, %spurgable, %spurgable_volatile, %spurgable_empty, %sshadowed, %scached, %sprivate\n",
+	iprintf("%salive, %spurgeable, %spurgeable_volatile, %spurgeable_empty, %sshadowed, %scached, %sprivate\n",
 		(object->alive ? "" : "!"),
-		((object->purgable != VM_OBJECT_NONPURGABLE) ? "" : "!"),
-		((object->purgable == VM_OBJECT_PURGABLE_VOLATILE) ? "" : "!"),
-		((object->purgable == VM_OBJECT_PURGABLE_EMPTY) ? "" : "!"),
+		((object->purgable != VM_PURGABLE_DENY) ? "" : "!"),
+		((object->purgable == VM_PURGABLE_VOLATILE) ? "" : "!"),
+		((object->purgable == VM_PURGABLE_EMPTY) ? "" : "!"),
 		(object->shadowed ? "" : "!"),
 		(vm_object_cached(object) ? "" : "!"),
 		(object->private ? "" : "!"));
@@ -4370,10 +4631,9 @@ vm_object_find(
         task_t task;
 	vm_map_t map;
 	vm_map_entry_t entry;
-	processor_set_t pset = &default_pset;
 	boolean_t found = FALSE;
 
-	queue_iterate(&pset->tasks, task, task_t, pset_tasks) {
+	queue_iterate(&tasks, task, task_t, tasks) {
 		map = task->map;
 		for (entry = vm_map_first_entry(map);
 			 entry && entry != vm_map_to_entry(map);
@@ -4443,22 +4703,28 @@ vm_object_populate_with_private(
 			m = vm_page_lookup(object, base_offset);
 			if(m != VM_PAGE_NULL) {
 				if(m->fictitious) {
-					vm_page_lock_queues();
-					m->fictitious = FALSE;
-					m->private = TRUE;
-					m->phys_page = base_page;
-					if(!m->busy) {
-						m->busy = TRUE;
-					}
-					if(!m->absent) {
-						m->absent = TRUE;
-						object->absent_count++;
+					if (m->phys_page !=
+					    vm_page_guard_addr) {
+						vm_page_lockspin_queues();
+						m->fictitious = FALSE;
+						m->private = TRUE;
+						m->phys_page = base_page;
+						if(!m->busy) {
+							m->busy = TRUE;
+						}
+						if(!m->absent) {
+							m->absent = TRUE;
+						}
+						m->list_req_pending = TRUE;
+						vm_page_unlock_queues();
 					}
-					m->list_req_pending = TRUE;
-					vm_page_unlock_queues();
 				} else if (m->phys_page != base_page) {
-					/* pmap call to clear old mapping */
-				        pmap_disconnect(m->phys_page);
+				        if (m->pmapped) {
+					        /*
+						 * pmap call to clear old mapping
+						 */
+					        pmap_disconnect(m->phys_page);
+					}
 					m->phys_page = base_page;
 				}
 
@@ -4477,14 +4743,13 @@ vm_object_populate_with_private(
 				while ((m = vm_page_grab_fictitious()) 
 							 == VM_PAGE_NULL)
                 			vm_page_more_fictitious();	
-				vm_page_lock_queues();
+				vm_page_lockspin_queues();
 				m->fictitious = FALSE;
 				m->private = TRUE;
 				m->phys_page = base_page;
 				m->list_req_pending = TRUE;
 				m->absent = TRUE;
 				m->unusual = TRUE;
-				object->absent_count++;
 				vm_page_unlock_queues();
 	    			vm_page_insert(m, object, base_offset);
 			}
@@ -4558,6 +4823,7 @@ memory_object_free_from_cache(
 
 			assert(object->pager_initialized);
 			assert(object->ref_count == 0);
+			vm_object_lock_assert_exclusive(object);
 			object->ref_count++;
 
 			/*
@@ -4693,6 +4959,7 @@ restart:
 	vm_object_cache_unlock();
 
 	object->named = TRUE;
+	vm_object_lock_assert_exclusive(object);
 	object->ref_count++;
 	vm_object_res_reference(object);
 	while (!object->pager_ready) {
@@ -4804,6 +5071,7 @@ vm_object_release_name(
 			}
 			return KERN_SUCCESS;
 		} else {
+			vm_object_lock_assert_exclusive(object);
 			object->ref_count--;
 			assert(object->ref_count > 0);
 			if(original_object)
@@ -4865,16 +5133,16 @@ vm_object_lock_request(
 }
 
 /*
- * Empty a purgable object by grabbing the physical pages assigned to it and
+ * Empty a purgeable object by grabbing the physical pages assigned to it and
  * putting them on the free queue without writing them to backing store, etc.
  * When the pages are next touched they will be demand zero-fill pages.  We
  * skip pages which are busy, being paged in/out, wired, etc.  We do _not_
  * skip referenced/dirty pages, pages on the active queue, etc.  We're more
- * than happy to grab these since this is a purgable object.  We mark the
+ * than happy to grab these since this is a purgeable object.  We mark the
  * object as "empty" after reaping its pages.
  *
  * On entry the object and page queues are locked, the object must be a
- * purgable object with no delayed copies pending.
+ * purgeable object with no delayed copies pending.
  */
 unsigned int
 vm_object_purge(vm_object_t object)
@@ -4890,10 +5158,11 @@ vm_object_purge(vm_object_t object)
 #define PURGE_LOOP_QUOTA	100
 
 	num_purged_pages = 0;
-	if (object->purgable == VM_OBJECT_NONPURGABLE)
+	if (object->purgable == VM_PURGABLE_DENY)
 		return num_purged_pages;
 
-	object->purgable = VM_OBJECT_PURGABLE_EMPTY;
+	assert(object->purgable != VM_PURGABLE_NONVOLATILE);
+	object->purgable = VM_PURGABLE_EMPTY;
 
 	assert(object->copy == VM_OBJECT_NULL);
 	assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
@@ -4926,9 +5195,7 @@ vm_object_purge(vm_object_t object)
 				local_freeq = VM_PAGE_NULL;
 				local_freed = 0;
 			}
-			vm_page_unlock_queues();
-			mutex_pause();
-			vm_page_lock_queues();
+			mutex_yield(&vm_page_queue_lock);
 
 			/* resume with the current page and a new quota */
 			purge_loop_quota = PURGE_LOOP_QUOTA;
@@ -4945,24 +5212,15 @@ vm_object_purge(vm_object_t object)
 			continue;
 		}
 
-		if (p->tabled) {
-			/* clean up the object/offset table */
-			vm_page_remove(p);
-		}
-		if (p->absent) {
-			/* update the object's count of absent pages */
-			vm_object_absent_release(object);
-		}
+		assert(!p->laundry);
+		assert(p->object != kernel_object);
 
 		/* we can discard this page */
 
 		/* advertize that this page is in a transition state */
 		p->busy = TRUE;
 
-		if (p->no_isync == TRUE) {
-			/* the page hasn't been mapped yet */
-			/* (optimization to delay the i-cache sync) */
-		} else {
+		if (p->pmapped == TRUE) {
 			/* unmap the page */
 			int refmod_state;
 
@@ -4978,12 +5236,9 @@ vm_object_purge(vm_object_t object)
 			vm_page_purged_count++;
 		}
 
-		/* remove page from active or inactive queue... */
-		VM_PAGE_QUEUES_REMOVE(p);
+		vm_page_free_prepare(p);
 
 		/* ... and put it on our queue of pages to free */
-		assert(!p->laundry);
-		assert(p->object != kernel_object);
 		assert(p->pageq.next == NULL &&
 		       p->pageq.prev == NULL);
 		p->pageq.next = (queue_entry_t) local_freeq;
@@ -5007,72 +5262,72 @@ vm_object_purge(vm_object_t object)
 }
 
 /*
- * vm_object_purgable_control() allows the caller to control and investigate the
- * state of a purgable object.  A purgable object is created via a call to
- * vm_allocate() with VM_FLAGS_PURGABLE specified.  A purgable object will
- * never be coalesced with any other object -- even other purgable objects --
- * and will thus always remain a distinct object.  A purgable object has
+ * vm_object_purgeable_control() allows the caller to control and investigate the
+ * state of a purgeable object.  A purgeable object is created via a call to
+ * vm_allocate() with VM_FLAGS_PURGABLE specified.  A purgeable object will
+ * never be coalesced with any other object -- even other purgeable objects --
+ * and will thus always remain a distinct object.  A purgeable object has
  * special semantics when its reference count is exactly 1.  If its reference
- * count is greater than 1, then a purgable object will behave like a normal
+ * count is greater than 1, then a purgeable object will behave like a normal
  * object and attempts to use this interface will result in an error return
  * of KERN_INVALID_ARGUMENT.
  *
- * A purgable object may be put into a "volatile" state which will make the
+ * A purgeable object may be put into a "volatile" state which will make the
  * object's pages elligable for being reclaimed without paging to backing
  * store if the system runs low on memory.  If the pages in a volatile
- * purgable object are reclaimed, the purgable object is said to have been
- * "emptied."  When a purgable object is emptied the system will reclaim as
+ * purgeable object are reclaimed, the purgeable object is said to have been
+ * "emptied."  When a purgeable object is emptied the system will reclaim as
  * many pages from the object as it can in a convenient manner (pages already
  * en route to backing store or busy for other reasons are left as is).  When
- * a purgable object is made volatile, its pages will generally be reclaimed
+ * a purgeable object is made volatile, its pages will generally be reclaimed
  * before other pages in the application's working set.  This semantic is
  * generally used by applications which can recreate the data in the object
  * faster than it can be paged in.  One such example might be media assets
  * which can be reread from a much faster RAID volume.
  *
- * A purgable object may be designated as "non-volatile" which means it will
+ * A purgeable object may be designated as "non-volatile" which means it will
  * behave like all other objects in the system with pages being written to and
  * read from backing store as needed to satisfy system memory needs.  If the
  * object was emptied before the object was made non-volatile, that fact will
- * be returned as the old state of the purgable object (see
+ * be returned as the old state of the purgeable object (see
  * VM_PURGABLE_SET_STATE below).  In this case, any pages of the object which
  * were reclaimed as part of emptying the object will be refaulted in as
  * zero-fill on demand.  It is up to the application to note that an object
  * was emptied and recreate the objects contents if necessary.  When a
- * purgable object is made non-volatile, its pages will generally not be paged
- * out to backing store in the immediate future.  A purgable object may also
+ * purgeable object is made non-volatile, its pages will generally not be paged
+ * out to backing store in the immediate future.  A purgeable object may also
  * be manually emptied.
  *
  * Finally, the current state (non-volatile, volatile, volatile & empty) of a
- * volatile purgable object may be queried at any time.  This information may
+ * volatile purgeable object may be queried at any time.  This information may
  * be used as a control input to let the application know when the system is
  * experiencing memory pressure and is reclaiming memory.
  *
- * The specified address may be any address within the purgable object.  If
+ * The specified address may be any address within the purgeable object.  If
  * the specified address does not represent any object in the target task's
  * virtual address space, then KERN_INVALID_ADDRESS will be returned.  If the
- * object containing the specified address is not a purgable object, then
+ * object containing the specified address is not a purgeable object, then
  * KERN_INVALID_ARGUMENT will be returned.  Otherwise, KERN_SUCCESS will be
  * returned.
  *
  * The control parameter may be any one of VM_PURGABLE_SET_STATE or
  * VM_PURGABLE_GET_STATE.  For VM_PURGABLE_SET_STATE, the in/out parameter
- * state is used to set the new state of the purgable object and return its
- * old state.  For VM_PURGABLE_GET_STATE, the current state of the purgable
+ * state is used to set the new state of the purgeable object and return its
+ * old state.  For VM_PURGABLE_GET_STATE, the current state of the purgeable
  * object is returned in the parameter state.
  *
  * The in/out parameter state may be one of VM_PURGABLE_NONVOLATILE,
  * VM_PURGABLE_VOLATILE or VM_PURGABLE_EMPTY.  These, respectively, represent
  * the non-volatile, volatile and volatile/empty states described above.
- * Setting the state of a purgable object to VM_PURGABLE_EMPTY will
+ * Setting the state of a purgeable object to VM_PURGABLE_EMPTY will
  * immediately reclaim as many pages in the object as can be conveniently
  * collected (some may have already been written to backing store or be
  * otherwise busy).
  *
- * The process of making a purgable object non-volatile and determining its
- * previous state is atomic.  Thus, if a purgable object is made
+ * The process of making a purgeable object non-volatile and determining its
+ * previous state is atomic.  Thus, if a purgeable object is made
  * VM_PURGABLE_NONVOLATILE and the old state is returned as
- * VM_PURGABLE_VOLATILE, then the purgable object's previous contents are
+ * VM_PURGABLE_VOLATILE, then the purgeable object's previous contents are
  * completely intact and will remain so until the object is made volatile
  * again.  If the old state is returned as VM_PURGABLE_EMPTY then the object
  * was reclaimed while it was in a volatile state and its previous contents
@@ -5088,42 +5343,23 @@ vm_object_purgable_control(
 	int		*state)
 {
 	int		old_state;
-	vm_page_t	p;
+	int		new_state;
 
 	if (object == VM_OBJECT_NULL) {
 		/*
-		 * Object must already be present or it can't be purgable.
+		 * Object must already be present or it can't be purgeable.
 		 */
 		return KERN_INVALID_ARGUMENT;
 	}
 
 	/*
-	 * Get current state of the purgable object.
+	 * Get current state of the purgeable object.
 	 */
-	switch (object->purgable) {
-	    case VM_OBJECT_NONPURGABLE:
+	old_state = object->purgable;
+	if (old_state == VM_PURGABLE_DENY)
 		return KERN_INVALID_ARGUMENT;
     
-	    case VM_OBJECT_PURGABLE_NONVOLATILE:
-		old_state = VM_PURGABLE_NONVOLATILE;
-		break;
-
-	    case VM_OBJECT_PURGABLE_VOLATILE:
-		old_state = VM_PURGABLE_VOLATILE;
-		break;
-
-	    case VM_OBJECT_PURGABLE_EMPTY:
-		old_state = VM_PURGABLE_EMPTY;
-		break;
-
-	    default:
-		old_state = VM_PURGABLE_NONVOLATILE;
-		panic("Bad state (%d) for purgable object!\n",
-		      object->purgable);
-		/*NOTREACHED*/
-	}
-
-	/* purgable cant have delayed copies - now or in the future */
+	/* purgeable cant have delayed copies - now or in the future */
 	assert(object->copy == VM_OBJECT_NULL); 
 	assert(object->copy_strategy == MEMORY_OBJECT_COPY_NONE);
 
@@ -5135,86 +5371,116 @@ vm_object_purgable_control(
 		return KERN_SUCCESS;
 	}
 
-	switch (*state) {
+	new_state = *state & VM_PURGABLE_STATE_MASK;
+	switch (new_state) {
+	case VM_PURGABLE_DENY:
 	case VM_PURGABLE_NONVOLATILE:
-		vm_page_lock_queues();
-		if (object->purgable != VM_OBJECT_PURGABLE_NONVOLATILE) {
+		object->purgable = new_state;
+
+		if (old_state != VM_PURGABLE_NONVOLATILE) {
+			vm_page_lock_queues();
 			assert(vm_page_purgeable_count >=
 			       object->resident_page_count);
 			vm_page_purgeable_count -= object->resident_page_count;
-		}
 
-		object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
+			if (old_state==VM_PURGABLE_VOLATILE) {
+			        assert(object->objq.next != NULL && object->objq.prev != NULL); /* object should be on a queue */
+				purgeable_q_t queue = vm_purgeable_object_remove(object);
+				assert(queue);
 
-		/*
-		 * If the object wasn't emptied, then mark all pages of the
-		 * object as referenced in order to give them a complete turn
-		 * of the virtual memory "clock" before becoming candidates
-		 * for paging out (if the system is suffering from memory
-		 * pressure).  We don't really need to set the pmap reference
-		 * bits (which would be expensive) since the software copies
-		 * are believed if they're set to true ...
-		 */
-		if (old_state != VM_PURGABLE_EMPTY) {
-			for (p = (vm_page_t)queue_first(&object->memq);
-			     !queue_end(&object->memq, (queue_entry_t)p);
-			     p = (vm_page_t)queue_next(&p->listq))
-				p->reference = TRUE;
+				vm_purgeable_token_delete_first(queue);
+				assert(queue->debug_count_objects>=0);
+			};
+			vm_page_unlock_queues();
 		}
-
-		vm_page_unlock_queues();
-
 		break;
 
 	case VM_PURGABLE_VOLATILE:
-		vm_page_lock_queues();
 
-		if (object->purgable != VM_OBJECT_PURGABLE_VOLATILE &&
-		    object->purgable != VM_OBJECT_PURGABLE_EMPTY) {
-			vm_page_purgeable_count += object->resident_page_count;
+		if ((old_state != VM_PURGABLE_NONVOLATILE) && (old_state != VM_PURGABLE_VOLATILE))
+			break;
+		purgeable_q_t queue;
+        
+		/* find the correct queue */
+		if ((*state&VM_PURGABLE_ORDERING_MASK) == VM_PURGABLE_ORDERING_OBSOLETE)
+		        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
+		else {
+		        if ((*state&VM_PURGABLE_BEHAVIOR_MASK) == VM_PURGABLE_BEHAVIOR_FIFO)
+			        queue = &purgeable_queues[PURGEABLE_Q_TYPE_FIFO];
+			else
+			        queue = &purgeable_queues[PURGEABLE_Q_TYPE_LIFO];
 		}
+        
+		if (old_state == VM_PURGABLE_NONVOLATILE) {
+		        /* try to add token... this can fail */
+		        vm_page_lock_queues();
 
-		object->purgable = VM_OBJECT_PURGABLE_VOLATILE;
-
-		/*
-		 * We want the newly volatile purgable object to be a
-		 * candidate for the pageout scan before other pages in the
-		 * application if the system is suffering from memory
-		 * pressure.  To do this, we move a page of the object from
-		 * the active queue onto the inactive queue in order to
-		 * promote the object for early reclaim.  We only need to move
-		 * a single page since the pageout scan will reap the entire
-		 * purgable object if it finds a single page in a volatile
-		 * state.  Obviously we don't do this if there are no pages
-		 * associated with the object or we find a page of the object
-		 * already on the inactive queue.
-		 */
-		for (p = (vm_page_t)queue_first(&object->memq);
-		     !queue_end(&object->memq, (queue_entry_t)p);
-		     p = (vm_page_t)queue_next(&p->listq)) {
-			if (p->inactive) {
-				/* already a page on the inactive queue */
-				break;
-			}
-			if (p->active && !p->busy) {
-				/* found one we can move */
-				vm_page_deactivate(p);
-				break;
+			kern_return_t result = vm_purgeable_token_add(queue);
+			if (result != KERN_SUCCESS) {
+			        vm_page_unlock_queues();
+				return result;
 			}
+			vm_page_purgeable_count += object->resident_page_count;
+
+			vm_page_unlock_queues();
+
+			object->purgable = new_state;
+
+			/* object should not be on a queue */
+			assert(object->objq.next == NULL && object->objq.prev == NULL);
 		}
-		vm_page_unlock_queues();
+		else if (old_state == VM_PURGABLE_VOLATILE) {
+		        /*
+			 * if reassigning priorities / purgeable groups, we don't change the
+			 * token queue. So moving priorities will not make pages stay around longer.
+			 * Reasoning is that the algorithm gives most priority to the most important
+			 * object. If a new token is added, the most important object' priority is boosted.
+			 * This biases the system already for purgeable queues that move a lot.
+			 * It doesn't seem more biasing is neccessary in this case, where no new object is added.
+			 */
+		        assert(object->objq.next != NULL && object->objq.prev != NULL); /* object should be on a queue */
+            
+			purgeable_q_t old_queue=vm_purgeable_object_remove(object);
+			assert(old_queue);
+            
+			if (old_queue != queue) {
+				kern_return_t result;
+
+			        /* Changing queue. Have to move token. */
+			        vm_page_lock_queues();
+				vm_purgeable_token_delete_first(old_queue);
+				result = vm_purgeable_token_add(queue);
+				vm_page_unlock_queues();
 
+				assert(result==KERN_SUCCESS);   /* this should never fail since we just freed a token */
+			}
+		};
+		vm_purgeable_object_add(object, queue, (*state&VM_VOLATILE_GROUP_MASK)>>VM_VOLATILE_GROUP_SHIFT );
+
+		assert(queue->debug_count_objects>=0);
+        
 		break;
 
 
 	case VM_PURGABLE_EMPTY:
-		vm_page_lock_queues();
-		if (object->purgable != VM_OBJECT_PURGABLE_VOLATILE &&
-		    object->purgable != VM_OBJECT_PURGABLE_EMPTY) {
-			vm_page_purgeable_count += object->resident_page_count;
+		if (old_state != new_state) 
+		{
+			assert(old_state==VM_PURGABLE_NONVOLATILE || old_state==VM_PURGABLE_VOLATILE);
+			if(old_state==VM_PURGABLE_VOLATILE) {
+				assert(object->objq.next != NULL && object->objq.prev != NULL); /* object should be on a queue */
+				purgeable_q_t old_queue=vm_purgeable_object_remove(object);
+				assert(old_queue);
+				vm_page_lock_queues();
+				vm_purgeable_token_delete_first(old_queue);
+			}
+
+			if (old_state==VM_PURGABLE_NONVOLATILE) {
+				vm_page_purgeable_count += object->resident_page_count;
+				vm_page_lock_queues();
+			}
+			(void) vm_object_purge(object);
+			vm_page_unlock_queues();
 		}
-		(void) vm_object_purge(object);
-		vm_page_unlock_queues();
 		break;
 
 	}
@@ -5389,8 +5655,9 @@ vm_object_transpose(
 
 	vm_object_lock(object1);
 	object1_locked = TRUE;
-	if (object1->copy || object1->shadow || object1->shadowed ||
-	    object1->purgable != VM_OBJECT_NONPURGABLE) {
+	if (!object1->alive || object1->terminating ||
+	    object1->copy || object1->shadow || object1->shadowed ||
+	    object1->purgable != VM_PURGABLE_DENY) {
 		/*
 		 * We don't deal with copy or shadow objects (yet).
 		 */
@@ -5415,8 +5682,9 @@ vm_object_transpose(
 	 */
 	vm_object_lock(object2);
 	object2_locked = TRUE;
-	if (object2->copy || object2->shadow || object2->shadowed ||
-	    object2->purgable != VM_OBJECT_NONPURGABLE) {
+	if (! object2->alive || object2->terminating ||
+	    object2->copy || object2->shadow || object2->shadowed ||
+	    object2->purgable != VM_PURGABLE_DENY) {
 		retval = KERN_INVALID_VALUE;
 		goto done;
 	}
@@ -5466,6 +5734,7 @@ vm_object_transpose(
 
 	/*
 	 * Transpose the lists of resident pages.
+	 * This also updates the resident_page_count and the memq_hint.
 	 */
 	if (object1->phys_contiguous || queue_empty(&object1->memq)) {
 		/*
@@ -5475,7 +5744,7 @@ vm_object_transpose(
 		 */
 		while (!queue_empty(&object2->memq)) {
 			page = (vm_page_t) queue_first(&object2->memq);
-			vm_page_rename(page, object1, page->offset);
+			vm_page_rename(page, object1, page->offset, FALSE);
 		}
 		assert(queue_empty(&object2->memq));
 	} else if (object2->phys_contiguous || queue_empty(&object2->memq)) {
@@ -5486,7 +5755,7 @@ vm_object_transpose(
 		 */
 		while (!queue_empty(&object1->memq)) {
 			page = (vm_page_t) queue_first(&object1->memq);
-			vm_page_rename(page, object2, page->offset);
+			vm_page_rename(page, object2, page->offset, FALSE);
 		}
 		assert(queue_empty(&object1->memq));
 	} else {
@@ -5504,7 +5773,7 @@ vm_object_transpose(
 		/* transfer object2's pages to object1 */
 		while (!queue_empty(&object2->memq)) {
 			page = (vm_page_t) queue_first(&object2->memq);
-			vm_page_rename(page, object1, page->offset);
+			vm_page_rename(page, object1, page->offset, FALSE);
 		}
 		assert(queue_empty(&object2->memq));
 		/* transfer tmp_object's pages to object1 */
@@ -5517,9 +5786,6 @@ vm_object_transpose(
 		assert(queue_empty(&tmp_object->memq));
 	}
 
-	/* no need to transpose the size: they should be identical */
-	assert(object1->size == object2->size);
-
 #define __TRANSPOSE_FIELD(field)				\
 MACRO_BEGIN							\
 	tmp_object->field = object1->field;			\
@@ -5527,16 +5793,23 @@ MACRO_BEGIN							\
 	object2->field = tmp_object->field;			\
 MACRO_END
 
+	/* "size" should be identical */
+	assert(object1->size == object2->size);
+	/* "Lock" refers to the object not its contents */
+	/* "ref_count" refers to the object not its contents */
+#if TASK_SWAPPER
+	/* "res_count" refers to the object not its contents */
+#endif
+	/* "resident_page_count" was updated above when transposing pages */
+	/* there should be no "copy" */
 	assert(!object1->copy);
 	assert(!object2->copy);
-
+	/* there should be no "shadow" */
 	assert(!object1->shadow);
 	assert(!object2->shadow);
-
 	__TRANSPOSE_FIELD(shadow_offset); /* used by phys_contiguous objects */
 	__TRANSPOSE_FIELD(pager);
 	__TRANSPOSE_FIELD(paging_offset);
-
 	__TRANSPOSE_FIELD(pager_control);
 	/* update the memory_objects' pointers back to the VM objects */
 	if (object1->pager_control != MEMORY_OBJECT_CONTROL_NULL) {
@@ -5547,29 +5820,62 @@ MACRO_END
 		memory_object_control_collapse(object2->pager_control,
 					       object2);
 	}
-		
-	__TRANSPOSE_FIELD(absent_count);
-
+	__TRANSPOSE_FIELD(copy_strategy);
+	/* "paging_in_progress" refers to the object not its contents */
 	assert(object1->paging_in_progress);
 	assert(object2->paging_in_progress);
-
+	/* "all_wanted" refers to the object not its contents */
 	__TRANSPOSE_FIELD(pager_created);
 	__TRANSPOSE_FIELD(pager_initialized);
 	__TRANSPOSE_FIELD(pager_ready);
 	__TRANSPOSE_FIELD(pager_trusted);
+	__TRANSPOSE_FIELD(can_persist);
 	__TRANSPOSE_FIELD(internal);
 	__TRANSPOSE_FIELD(temporary);
 	__TRANSPOSE_FIELD(private);
 	__TRANSPOSE_FIELD(pageout);
+	/* "alive" should be set */
+	assert(object1->alive);
+	assert(object2->alive);
+	/* "purgeable" should be non-purgeable */
+	assert(object1->purgable == VM_PURGABLE_DENY);
+	assert(object2->purgable == VM_PURGABLE_DENY);
+	/* "shadowed" refers to the the object not its contents */
+	__TRANSPOSE_FIELD(silent_overwrite);
+	__TRANSPOSE_FIELD(advisory_pageout);
 	__TRANSPOSE_FIELD(true_share);
+	/* "terminating" should not be set */
+	assert(!object1->terminating);
+	assert(!object2->terminating);
+	__TRANSPOSE_FIELD(named);
+	/* "shadow_severed" refers to the object not its contents */
 	__TRANSPOSE_FIELD(phys_contiguous);
 	__TRANSPOSE_FIELD(nophyscache);
+	/* "cached_list" should be NULL */
+	assert(object1->cached_list.prev == NULL);
+	assert(object1->cached_list.next == NULL);
+	assert(object2->cached_list.prev == NULL);
+	assert(object2->cached_list.next == NULL);
+	/* "msr_q" is linked to the object not its contents */
+	assert(queue_empty(&object1->msr_q));
+	assert(queue_empty(&object2->msr_q));
 	__TRANSPOSE_FIELD(last_alloc);
 	__TRANSPOSE_FIELD(sequential);
-	__TRANSPOSE_FIELD(cluster_size);
+	__TRANSPOSE_FIELD(pages_created);
+	__TRANSPOSE_FIELD(pages_used);
+#if MACH_PAGEMAP
 	__TRANSPOSE_FIELD(existence_map);
+#endif
 	__TRANSPOSE_FIELD(cow_hint);
+#if MACH_ASSERT
+	__TRANSPOSE_FIELD(paging_object);
+#endif
 	__TRANSPOSE_FIELD(wimg_bits);
+	__TRANSPOSE_FIELD(code_signed);
+	__TRANSPOSE_FIELD(not_in_use);
+#ifdef UPL_DEBUG
+	/* "uplq" refers to the object not its contents (see upl_transpose()) */
+#endif
 
 #undef __TRANSPOSE_FIELD
 
@@ -5616,8 +5922,262 @@ done:
 }
 
 
-/* Allow manipulation of individual page state.  This is actually part of */
-/* the UPL regimen but takes place on the VM object rather than on a UPL */
+/*
+ *      vm_object_build_cluster
+ *
+ *      Determine how big a cluster we should issue an I/O for...
+ *
+ *	Inputs:   *start == offset of page needed
+ *		  *length == maximum cluster pager can handle
+ *	Outputs:  *start == beginning offset of cluster
+ *		  *length == length of cluster to try
+ *
+ *	The original *start will be encompassed by the cluster
+ *
+ */
+extern int speculative_reads_disabled;
+
+uint32_t pre_heat_scaling[MAX_UPL_TRANSFER];
+uint32_t pre_heat_cluster[MAX_UPL_TRANSFER];
+
+#define PRE_HEAT_MULTIPLIER 4
+
+__private_extern__ void
+vm_object_cluster_size(vm_object_t object, vm_object_offset_t *start,
+		       vm_size_t *length, vm_object_fault_info_t fault_info)
+{
+	vm_size_t		pre_heat_size;
+	vm_size_t		tail_size;
+	vm_size_t		head_size;
+	vm_size_t		max_length;
+	vm_size_t		cluster_size;
+	vm_object_offset_t	object_size;
+	vm_object_offset_t	orig_start;
+	vm_object_offset_t	target_start;
+	vm_object_offset_t	offset;
+	vm_behavior_t		behavior;
+	boolean_t		look_behind = TRUE;
+	boolean_t		look_ahead  = TRUE;
+	int			sequential_run;
+	int			sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
+
+	assert( !(*length & PAGE_MASK));
+	assert( !(*start & PAGE_MASK_64));
+
+	if ( (max_length = *length) > (MAX_UPL_TRANSFER * PAGE_SIZE) ) 
+	        max_length = (MAX_UPL_TRANSFER * PAGE_SIZE);
+	/*
+	 * we'll always return a cluster size of at least
+	 * 1 page, since the original fault must always
+	 * be processed
+	 */
+	*length = PAGE_SIZE;
+
+	if (speculative_reads_disabled || fault_info == NULL || max_length == 0) {
+	        /*
+		 * no cluster... just fault the page in
+		 */
+	        return;
+	}
+	orig_start = *start;
+	target_start = orig_start;
+	cluster_size = round_page_32(fault_info->cluster_size);
+	behavior = fault_info->behavior;
+
+	vm_object_lock(object);
+
+	if (object->internal)
+	        object_size = object->size;
+	else if (object->pager != MEMORY_OBJECT_NULL)
+	        vnode_pager_get_object_size(object->pager, &object_size);
+	else
+		goto out;	/* pager is gone for this object, nothing more to do */
+
+	object_size = round_page_64(object_size);
+
+	if (orig_start >= object_size) {
+	        /*
+		 * fault occurred beyond the EOF...
+		 * we need to punt w/o changing the
+		 * starting offset
+		 */
+	        goto out;
+	}
+	if (object->pages_used > object->pages_created) {
+	        /*
+		 * must have wrapped our 32 bit counters
+		 * so reset
+		 */
+ 	        object->pages_used = object->pages_created = 0;
+	}
+	if ((sequential_run = object->sequential)) {
+		  if (sequential_run < 0) {
+		          sequential_behavior = VM_BEHAVIOR_RSEQNTL;
+			  sequential_run = 0 - sequential_run;
+		  } else {
+		          sequential_behavior = VM_BEHAVIOR_SEQUENTIAL;
+		  }
+	}
+	switch(behavior) {
+
+	default:
+	        behavior = VM_BEHAVIOR_DEFAULT;
+
+	case VM_BEHAVIOR_DEFAULT:
+	        if (object->internal && fault_info->user_tag == VM_MEMORY_STACK)
+		        goto out;
+
+	        if (sequential_run >= (3 * PAGE_SIZE)) {
+		        pre_heat_size = sequential_run + PAGE_SIZE;
+
+			if ((behavior = sequential_behavior) == VM_BEHAVIOR_SEQUENTIAL)
+			        look_behind = FALSE;
+			else
+			        look_ahead = FALSE;
+		} else {
+		        uint32_t	pages_unused;
+
+			if (object->pages_created < 32 * PRE_HEAT_MULTIPLIER) {
+			        /*
+				 * prime the pump
+				 */
+			        pre_heat_size = PAGE_SIZE * 8 * PRE_HEAT_MULTIPLIER;
+			        break;
+			}
+			pages_unused = object->pages_created - object->pages_used;
+
+			if (pages_unused < (object->pages_created / 8)) {
+			        pre_heat_size = PAGE_SIZE * 32 * PRE_HEAT_MULTIPLIER;
+			} else if (pages_unused < (object->pages_created / 4)) {
+			        pre_heat_size = PAGE_SIZE * 16 * PRE_HEAT_MULTIPLIER;
+			} else if (pages_unused < (object->pages_created / 2)) {
+			        pre_heat_size = PAGE_SIZE * 8 * PRE_HEAT_MULTIPLIER;
+			} else {
+			        pre_heat_size = PAGE_SIZE * 4 * PRE_HEAT_MULTIPLIER;
+			}
+		}
+		break;
+
+	case VM_BEHAVIOR_RANDOM:
+	        if ((pre_heat_size = cluster_size) <= PAGE_SIZE)
+		        goto out;
+	        break;
+
+	case VM_BEHAVIOR_SEQUENTIAL:
+	        if ((pre_heat_size = cluster_size) == 0)
+		        pre_heat_size = sequential_run + PAGE_SIZE;
+		look_behind = FALSE;
+
+	        break;
+
+	case VM_BEHAVIOR_RSEQNTL:
+	        if ((pre_heat_size = cluster_size) == 0)
+		        pre_heat_size = sequential_run + PAGE_SIZE;
+		look_ahead = FALSE;
+
+	        break;
+
+	}
+	if (pre_heat_size > max_length)
+	        pre_heat_size = max_length;
+
+	if (behavior == VM_BEHAVIOR_DEFAULT && vm_page_free_count < vm_page_free_target)
+	        pre_heat_size /= 2;
+
+	if (look_ahead == TRUE) {
+	        if (look_behind == TRUE)
+		        target_start &= ~(pre_heat_size - 1);
+
+	        if ((target_start + pre_heat_size) > object_size)
+		        pre_heat_size = (vm_size_t)(trunc_page_64(object_size - target_start));
+
+	        tail_size = pre_heat_size - (orig_start - target_start) - PAGE_SIZE;
+	} else {
+	        if (pre_heat_size > target_start)
+	                pre_heat_size = target_start;
+		tail_size = 0;
+	}
+	pre_heat_scaling[pre_heat_size / PAGE_SIZE]++;
+
+	if (pre_heat_size <= PAGE_SIZE)
+	        goto out;
+
+	if (look_behind == TRUE) {
+	        /*
+		 * take a look at the pages before the original
+		 * faulting offset 
+		 */
+	        head_size = pre_heat_size - tail_size - PAGE_SIZE;
+
+	        for (offset = orig_start - PAGE_SIZE_64; head_size; offset -= PAGE_SIZE_64, head_size -= PAGE_SIZE) {
+		        /*
+			 * don't poke below the lowest offset 
+			 */
+		        if (offset < fault_info->lo_offset)
+			        break;
+		        /*
+			 * for external objects and internal objects w/o an existence map
+			 * vm_externl_state_get will return VM_EXTERNAL_STATE_UNKNOWN
+			 */
+#if MACH_PAGEMAP
+		        if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_ABSENT) {
+			        /*
+				 * we know for a fact that the pager can't provide the page
+				 * so don't include it or any pages beyond it in this cluster
+				 */
+			        break;
+			}
+#endif
+			if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
+			        /*
+				 * don't bridge resident pages
+				 */
+			        break;
+			}
+			*start = offset;
+			*length += PAGE_SIZE;
+		}
+	}
+	if (look_ahead == TRUE) {
+	        for (offset = orig_start + PAGE_SIZE_64; tail_size; offset += PAGE_SIZE_64, tail_size -= PAGE_SIZE) {
+		        /*
+			 * don't poke above the highest offset 
+			 */
+		        if (offset >= fault_info->hi_offset)
+			        break;
+		        /*
+			 * for external objects and internal objects w/o an existence map
+			 * vm_externl_state_get will return VM_EXTERNAL_STATE_UNKNOWN
+			 */
+#if MACH_PAGEMAP
+		        if (vm_external_state_get(object->existence_map, offset) == VM_EXTERNAL_STATE_ABSENT) {
+			        /*
+				 * we know for a fact that the pager can't provide the page
+				 * so don't include it or any pages beyond it in this cluster
+				 */
+			        break;
+			}
+#endif
+			if (vm_page_lookup(object, offset) != VM_PAGE_NULL) {
+			        /*
+				 * don't bridge resident pages
+				 */
+			        break;
+			}
+			*length += PAGE_SIZE;
+		}
+	}
+out:
+	pre_heat_cluster[*length / PAGE_SIZE]++;
+
+	vm_object_unlock(object);
+}
+
+
+/*
+ * Allow manipulation of individual page state.  This is actually part of
+ * the UPL regimen but takes place on the VM object rather than on a UPL
+ */
 
 kern_return_t
 vm_object_page_op(
@@ -5666,13 +6226,13 @@ vm_object_page_op(
 		}
 
 		if (ops & UPL_POP_DUMP) {
-		        vm_page_lock_queues();
-
-			if (dst_page->no_isync == FALSE)
+			if (dst_page->pmapped == TRUE)
 			        pmap_disconnect(dst_page->phys_page);
-			vm_page_free(dst_page);
 
+		        vm_page_lock_queues();
+			vm_page_free(dst_page);
 			vm_page_unlock_queues();
+
 			break;
 		}
 
@@ -5807,7 +6367,7 @@ vm_object_range_op(
 	        return KERN_INVALID_OBJECT;
 	}
 	
-	offset = offset_beg;
+	offset = offset_beg & ~PAGE_MASK_64;
 
 	while (offset < offset_end) {
 		dst_page = vm_page_lookup(object, offset);
@@ -5818,8 +6378,7 @@ vm_object_range_op(
 					 * someone else is playing with the 
 					 * page, we will have to wait
 					 */
-				        PAGE_SLEEP(object,
-						dst_page, THREAD_UNINT);
+				        PAGE_SLEEP(object, dst_page, THREAD_UNINT);
 					/*
 					 * need to relook the page up since it's
 					 * state may have changed while we slept
@@ -5828,13 +6387,13 @@ vm_object_range_op(
 					 */
 					continue;
 				}
-				vm_page_lock_queues();
-
-				if (dst_page->no_isync == FALSE)
+				if (dst_page->pmapped == TRUE)
 				        pmap_disconnect(dst_page->phys_page);
-				vm_page_free(dst_page);
 
+				vm_page_lock_queues();
+				vm_page_free(dst_page);
 				vm_page_unlock_queues();
+
 			} else if (ops & UPL_ROP_ABSENT)
 			        break;
 		} else if (ops & UPL_ROP_PRESENT)
@@ -5844,8 +6403,53 @@ vm_object_range_op(
 	}
 	vm_object_unlock(object);
 
-	if (range)
+	if (range) {
+	        if (offset > offset_end)
+		        offset = offset_end;
 	        *range = offset - offset_beg;
-
+	}
 	return KERN_SUCCESS;
 }
+
+
+uint32_t scan_object_collision = 0;
+
+void
+vm_object_lock(vm_object_t object)
+{
+        if (object == vm_pageout_scan_wants_object) {
+	        scan_object_collision++;
+	        mutex_pause(2);
+	}
+        lck_rw_lock_exclusive(&object->Lock);
+}
+
+boolean_t
+vm_object_lock_try(vm_object_t object)
+{
+        if (object == vm_pageout_scan_wants_object) {
+	        scan_object_collision++;
+	        mutex_pause(2);
+	}
+	return (lck_rw_try_lock_exclusive(&object->Lock));
+}
+
+void
+vm_object_lock_shared(vm_object_t object)
+{
+        if (object == vm_pageout_scan_wants_object) {
+	        scan_object_collision++;
+	        mutex_pause(2);
+	}
+	lck_rw_lock_shared(&object->Lock);
+}
+
+boolean_t
+vm_object_lock_try_shared(vm_object_t object)
+{
+        if (object == vm_pageout_scan_wants_object) {
+	        scan_object_collision++;
+	        mutex_pause(2);
+	}
+	return (lck_rw_try_lock_shared(&object->Lock));
+}
diff --git a/osfmk/vm/vm_object.h b/osfmk/vm/vm_object.h
index 4df51903d..a093bf2f4 100644
--- a/osfmk/vm/vm_object.h
+++ b/osfmk/vm/vm_object.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -72,6 +78,7 @@
 #include <mach/machine/vm_types.h>
 #include <kern/queue.h>
 #include <kern/lock.h>
+#include <kern/locks.h>
 #include <kern/assert.h>
 #include <kern/misc_protos.h>
 #include <kern/macro_help.h>
@@ -88,11 +95,24 @@ struct vm_page;
  *	Types defined:
  *
  *	vm_object_t		Virtual memory object.
+ *	vm_object_fault_info_t	Used to determine cluster size.
  */
 
+struct vm_object_fault_info {
+	int		interruptible;
+        uint32_t	user_tag;
+        vm_size_t	cluster_size;
+        vm_behavior_t	behavior;
+        vm_map_offset_t	lo_offset;
+	vm_map_offset_t	hi_offset;
+	boolean_t	no_cache;
+};
+
+
+
 struct vm_object {
 	queue_head_t		memq;		/* Resident memory */
-	decl_mutex_data(,	Lock)		/* Synchronization */
+        lck_rw_t		Lock;		/* Synchronization */
 
 	vm_object_size_t	size;		/* Object size (only valid
 						 * if internal)
@@ -122,14 +142,6 @@ struct vm_object {
 	memory_object_copy_strategy_t
 				copy_strategy;	/* How to handle data copy */
 
-	unsigned int		absent_count;	/* The number of pages that
-						 * have been requested but
-						 * not filled.  That is, the
-						 * number of pages for which
-						 * the "absent" attribute is
-						 * asserted.
-						 */
-
 	int			paging_in_progress;
 						/* The memory object ports are
 						 * being used (e.g., for pagein
@@ -179,8 +191,7 @@ struct vm_object {
 	/* boolean_t */		alive:1,	/* Not yet terminated */
 
 	/* boolean_t */		purgable:2,	/* Purgable state.  See
-						 * VM_OBJECT_PURGABLE_*
-						 * items below.
+						 * VM_PURGABLE_* 
 						 */
 	/* boolean_t */		shadowed:1,	/* Shadow may exist */
 	/* boolean_t */		silent_overwrite:1,
@@ -250,9 +261,15 @@ struct vm_object {
 	queue_head_t		msr_q;		/* memory object synchronise
 						   request queue */
 
+  /*
+   * the following fields are not protected by any locks
+   * they are updated via atomic compare and swap
+   */
 	vm_object_offset_t	last_alloc;	/* last allocation offset */
-	vm_object_offset_t	sequential;	/* sequential access size */
-	vm_size_t		cluster_size;	/* size of paging cluster */
+	int			sequential;	/* sequential access size */
+
+        uint32_t		pages_created;
+        uint32_t		pages_used;
 #if	MACH_PAGEMAP
 	vm_external_map_t	existence_map;	/* bitmap of pages written to
 						 * backing storage */
@@ -265,12 +282,18 @@ struct vm_object {
 						 * put in current object
 						 */
 #endif
-					/* hold object lock when altering */
-	unsigned	int			/* cache WIMG bits         */		
-			wimg_bits:8,		/* wimg plus some expansion*/
-			not_in_use:24;
+	/* hold object lock when altering */
+	unsigned	int
+		wimg_bits:8,	        /* cache WIMG bits         */		
+		code_signed:1,		/* pages are signed and should be
+					   validated; the signatures are stored
+					   with the pager */
+		not_in_use:23;		/* for expansion */
+
 #ifdef	UPL_DEBUG
 	queue_head_t		uplq;		/* List of outstanding upls */
+#endif /* UPL_DEBUG */
+
 #ifdef	VM_PIP_DEBUG
 /*
  * Keep track of the stack traces for the first holders
@@ -282,7 +305,8 @@ struct vm_object {
 		void *pip_retaddr[VM_PIP_DEBUG_STACK_FRAMES];
 	} pip_holders[VM_PIP_DEBUG_MAX_REFS];
 #endif	/* VM_PIP_DEBUG  */
-#endif /* UPL_DEBUG */
+
+        queue_chain_t       objq;      /* object queue - currently used for purgable queues */
 };
 
 #define VM_PAGE_REMOVE(page)						\
@@ -357,10 +381,12 @@ typedef struct msync_req	*msync_req_t;
  *	Declare procedures that operate on VM objects.
  */
 
-__private_extern__ void		vm_object_bootstrap(void);
+__private_extern__ void		vm_object_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 __private_extern__ void		vm_object_init(void);
 
+__private_extern__ void		vm_object_init_lck_grp(void);
+
 __private_extern__ void		vm_object_reaper_init(void);
 
 __private_extern__ vm_object_t	vm_object_allocate(
@@ -388,12 +414,26 @@ __private_extern__ void	vm_object_res_deallocate(
 #endif	/* TASK_SWAPPER */
 
 #define vm_object_reference_locked(object)		\
-MACRO_BEGIN						\
-		vm_object_t RLObject = (object);	\
-		assert((RLObject)->ref_count > 0);	\
-		(RLObject)->ref_count++;		\
-		vm_object_res_reference(RLObject);	\
-MACRO_END
+	MACRO_BEGIN					\
+	vm_object_t RLObject = (object);		\
+	vm_object_lock_assert_exclusive(object);	\
+	assert((RLObject)->ref_count > 0);		\
+	(RLObject)->ref_count++;			\
+	assert((RLObject)->ref_count > 1);		\
+	vm_object_res_reference(RLObject);		\
+	MACRO_END
+
+
+#define vm_object_reference_shared(object)				\
+	MACRO_BEGIN							\
+	vm_object_t RLObject = (object);				\
+	vm_object_lock_assert_shared(object);				\
+	assert((RLObject)->ref_count > 0);				\
+	OSAddAtomic(1, (SInt32 *)&(RLObject)->ref_count);		\
+	assert((RLObject)->ref_count > 1);				\
+	/* XXX we would need an atomic version of the following ... */	\
+	vm_object_res_reference(RLObject);				\
+	MACRO_END
 
 
 __private_extern__ void		vm_object_reference(
@@ -490,7 +530,8 @@ __private_extern__ kern_return_t	vm_object_copy_slowly(
 __private_extern__ vm_object_t	vm_object_copy_delayed(
 				vm_object_t		src_object,
 				vm_object_offset_t	src_offset,
-				vm_object_size_t	size);
+				vm_object_size_t	size,
+				boolean_t		src_object_shared);
 
 
 
@@ -559,14 +600,11 @@ __private_extern__ vm_object_t	vm_object_enter(
 					boolean_t		check_named);
 
 
-/*
- * Purgable object state.
- */
-
-#define VM_OBJECT_NONPURGABLE		0	/* not a purgable object */
-#define VM_OBJECT_PURGABLE_NONVOLATILE	1	/* non-volatile purgable object */
-#define VM_OBJECT_PURGABLE_VOLATILE	2	/* volatile (but intact) purgable object */
-#define VM_OBJECT_PURGABLE_EMPTY	3	/* volatile purgable object that has been emptied */
+__private_extern__ void	vm_object_cluster_size(
+					vm_object_t		object,
+					vm_object_offset_t	*start,
+					vm_size_t		*length,
+					vm_object_fault_info_t  fault_info);
 
 __private_extern__ kern_return_t vm_object_populate_with_private(
 	vm_object_t		object,
@@ -599,7 +637,6 @@ extern kern_return_t vm_object_range_op(
 #define	VM_OBJECT_EVENT_INITIALIZED		0
 #define	VM_OBJECT_EVENT_PAGER_READY		1
 #define	VM_OBJECT_EVENT_PAGING_IN_PROGRESS	2
-#define	VM_OBJECT_EVENT_ABSENT_COUNT		3
 #define	VM_OBJECT_EVENT_LOCK_IN_PROGRESS	4
 #define	VM_OBJECT_EVENT_UNCACHING		5
 #define	VM_OBJECT_EVENT_COPY_CALL		6
@@ -615,7 +652,7 @@ extern kern_return_t vm_object_range_op(
 	thread_block(THREAD_CONTINUE_NULL))				\
 
 #define thread_sleep_vm_object(object, event, interruptible)		\
-	thread_sleep_mutex((event_t)(event), &(object)->Lock, (interruptible))
+        lck_rw_sleep(&(object)->Lock, LCK_SLEEP_DEFAULT, (event_t)(event), (interruptible))
 
 #define vm_object_sleep(object, event, interruptible)			\
 	(((object)->all_wanted |= 1 << (event)),			\
@@ -641,12 +678,12 @@ extern kern_return_t vm_object_range_op(
  *	Routines implemented as macros
  */
 #ifdef VM_PIP_DEBUG
-extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
+#include <libkern/OSDebug.h>
 #define VM_PIP_DEBUG_BEGIN(object)					\
 	MACRO_BEGIN							\
 	if ((object)->paging_in_progress < VM_PIP_DEBUG_MAX_REFS) {	\
 		int pip = (object)->paging_in_progress;			\
-		(void) OSBacktrace(&(object)->pip_holders[pip].retaddr[0], \
+		(void) OSBacktrace(&(object)->pip_holders[pip].pip_retaddr[0], \
 				   VM_PIP_DEBUG_STACK_FRAMES);		\
 	}								\
 	MACRO_END
@@ -656,6 +693,7 @@ extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
 
 #define		vm_object_paging_begin(object) 				\
 	MACRO_BEGIN							\
+	vm_object_lock_assert_exclusive((object));			\
 	assert((object)->paging_in_progress >= 0);			\
 	VM_PIP_DEBUG_BEGIN((object));					\
 	(object)->paging_in_progress++;					\
@@ -663,6 +701,7 @@ extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
 
 #define		vm_object_paging_end(object) 				\
 	MACRO_BEGIN							\
+	vm_object_lock_assert_exclusive((object));			\
 	assert((object)->paging_in_progress > 0);			\
 	if (--(object)->paging_in_progress == 0) {			\
 		vm_object_wakeup(object,				\
@@ -672,6 +711,7 @@ extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
 
 #define		vm_object_paging_wait(object, interruptible)		\
 	MACRO_BEGIN							\
+	vm_object_lock_assert_exclusive((object));			\
 	while ((object)->paging_in_progress != 0) {			\
 		wait_result_t  _wr;					\
 									\
@@ -684,29 +724,56 @@ extern unsigned OSBacktrace(void **bt, unsigned maxAddrs);
 	}								\
 	MACRO_END
 
-#define	vm_object_absent_assert_wait(object, interruptible)		\
-	MACRO_BEGIN							\
-	vm_object_assert_wait(	(object),				\
-			VM_OBJECT_EVENT_ABSENT_COUNT,			\
-			(interruptible));				\
-	MACRO_END
 
 
-#define	vm_object_absent_release(object)				\
-	MACRO_BEGIN							\
-	(object)->absent_count--;					\
-	vm_object_wakeup((object),					\
-			 VM_OBJECT_EVENT_ABSENT_COUNT);			\
-	MACRO_END
+#define OBJECT_LOCK_SHARED	0
+#define OBJECT_LOCK_EXCLUSIVE	1
+
+extern lck_grp_t	vm_object_lck_grp;
+extern lck_grp_attr_t	vm_object_lck_grp_attr;
+extern lck_attr_t	vm_object_lck_attr;
+extern lck_attr_t	kernel_object_lck_attr;
+
+extern vm_object_t	vm_pageout_scan_wants_object;
+
+extern void		vm_object_lock(vm_object_t);
+extern boolean_t	vm_object_lock_try(vm_object_t);
+extern void		vm_object_lock_shared(vm_object_t);
+extern boolean_t	vm_object_lock_try_shared(vm_object_t);
 
 /*
  *	Object locking macros
  */
 
-#define vm_object_lock_init(object)	mutex_init(&(object)->Lock, 0)
-#define vm_object_lock(object)		mutex_lock(&(object)->Lock)
-#define vm_object_unlock(object)	mutex_unlock(&(object)->Lock)
-#define vm_object_lock_try(object)	mutex_try(&(object)->Lock)
+#define vm_object_lock_init(object)					\
+	lck_rw_init(&(object)->Lock, &vm_object_lck_grp,		\
+		    (((object) == kernel_object ||			\
+		      (object) == vm_submap_object) ?			\
+		     &kernel_object_lck_attr :				\
+		     &vm_object_lck_attr))
+#define vm_object_lock_destroy(object)	lck_rw_destroy(&(object)->Lock, &vm_object_lck_grp)
+
+#define vm_object_unlock(object)	lck_rw_done(&(object)->Lock)
+#define vm_object_lock_upgrade(object)	lck_rw_lock_shared_to_exclusive(&(object)->Lock)
+#define vm_object_lock_try_scan(object)	lck_rw_try_lock_exclusive(&(object)->Lock)
+
+/*
+ * CAUTION: the following vm_object_lock_assert_held*() macros merely
+ * check if anyone is holding the lock, but the holder may not necessarily
+ * be the caller...
+ */
+#if DEBUG
+#define vm_object_lock_assert_held(object) \
+	lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_HELD)
+#define vm_object_lock_assert_shared(object)	\
+	lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_SHARED)
+#define vm_object_lock_assert_exclusive(object) \
+	lck_rw_assert(&(object)->Lock, LCK_RW_ASSERT_EXCLUSIVE)
+#else /* DEBUG */
+#define vm_object_lock_assert_held(object)
+#define vm_object_lock_assert_shared(object)
+#define vm_object_lock_assert_exclusive(object)
+#endif /* DEBUG */
 
 #define vm_object_round_page(x) (((vm_object_offset_t)(x) + PAGE_MASK) & ~((signed)PAGE_MASK))
 #define vm_object_trunc_page(x) ((vm_object_offset_t)(x) & ~((signed)PAGE_MASK))
diff --git a/osfmk/vm/vm_page.h b/osfmk/vm/vm_page.h
index 9b66d6210..53d137654 100644
--- a/osfmk/vm/vm_page.h
+++ b/osfmk/vm/vm_page.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -70,25 +76,60 @@
 #include <kern/lock.h>
 
 #include <kern/macro_help.h>
+#include <libkern/OSAtomic.h>
+
 
 /* 
- * Each page entered on the inactive queue obtains a ticket from a
- * particular ticket roll.  Pages granted tickets from a particular 
- * roll  generally flow through the queue as a group.  In this way when a
- * page with a ticket from a particular roll is pulled from the top of the
- * queue it is extremely likely that the pages near the top will have tickets
- * from the same or adjacent rolls.  In this way the proximity to the top
- * of the queue can be loosely ascertained by determining the identity of
- * the roll the pages ticket came from. 
+ * VM_PAGE_MIN_SPECULATIVE_AGE_Q through VM_PAGE_MAX_SPECULATIVE_AGE_Q
+ * represents a set of aging bins that are 'protected'...
+ *
+ * VM_PAGE_SPECULATIVE_AGED_Q is a list of the speculative pages that have
+ * not yet been 'claimed' but have been aged out of the protective bins
+ * this occurs in vm_page_speculate when it advances to the next bin 
+ * and discovers that it is still occupied... at that point, all of the
+ * pages in that bin are moved to the VM_PAGE_SPECULATIVE_AGED_Q.  the pages
+ * in that bin are all guaranteed to have reached at least the maximum age
+ * we allow for a protected page... they can be older if there is no
+ * memory pressure to pull them from the bin, or there are no new speculative pages
+ * being generated to push them out.
+ * this list is the one that vm_pageout_scan will prefer when looking 
+ * for pages to move to the underweight free list
+ * 
+ * VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS
+ * defines the amount of time a speculative page is normally
+ * allowed to live in the 'protected' state (i.e. not available
+ * to be stolen if vm_pageout_scan is running and looking for
+ * pages)...  however, if the total number of speculative pages
+ * in the protected state exceeds our limit (defined in vm_pageout.c)
+ * and there are none available in VM_PAGE_SPECULATIVE_AGED_Q, then
+ * vm_pageout_scan is allowed to steal pages from the protected
+ * bucket even if they are underage.
+ *
+ * vm_pageout_scan is also allowed to pull pages from a protected
+ * bin if the bin has reached the "age of consent" we've set
  */
+#define VM_PAGE_MAX_SPECULATIVE_AGE_Q	10
+#define VM_PAGE_MIN_SPECULATIVE_AGE_Q	1
+#define VM_PAGE_SPECULATIVE_AGED_Q	0
+
+#define VM_PAGE_SPECULATIVE_Q_AGE_MS	500
+
+
+struct vm_speculative_age_q {
+	/*
+	 * memory queue for speculative pages via clustered pageins
+	 */
+        queue_head_t	age_q;
+        mach_timespec_t	age_ts;
+};
 
 
-extern unsigned int	vm_page_ticket_roll;
-extern unsigned int	vm_page_ticket;
+extern
+struct vm_speculative_age_q	vm_page_queue_speculative[];
 
+extern int			speculative_steal_index;
+extern int			speculative_age_index;
 
-#define VM_PAGE_TICKETS_IN_ROLL  512
-#define VM_PAGE_TICKET_ROLL_IDS  16
 
 /*
  *	Management of resident (logical) pages.
@@ -130,11 +171,10 @@ struct vm_page {
 	 * by the "page queues" lock.
 	 */
 	unsigned int	wire_count:16,	/* how many wired down maps use me? (O&P) */
-			page_ticket:4,	/* age of the page on the       */
-					/* inactive queue.		    */
 	/* boolean_t */	inactive:1,	/* page is in inactive list (P) */
 			active:1,	/* page is in active list (P) */
 			pageout_queue:1,/* page is on queue for pageout (P) */
+			speculative:1,	/* page is on speculative list (P) */
 			laundry:1,	/* page is being cleaned now (P)*/
 			free:1,		/* page is on free list (P) */
 			reference:1,	/* page has been used (P) */
@@ -142,20 +182,20 @@ struct vm_page {
 			gobbled:1,      /* page used internally (P) */
 			private:1,	/* Page should not be returned to
 					 *  the free list (P) */
-			zero_fill:1,
-			:0;
+			throttled:1,	/* pager is not responding (P) */
+			__unused_pageq_bits:5;	/* 5 bits available here */
 
 	/*
 	 * The following word of flags is protected
 	 * by the "VM object" lock.
 	 */
 	unsigned int
-	                page_error:8,   /* error from I/O operations */
 	/* boolean_t */	busy:1,		/* page is in transit (O) */
 			wanted:1,	/* someone is waiting for page (O) */
 			tabled:1,	/* page is in VP table (O) */
 			fictitious:1,	/* Physical page doesn't exist (O) */
-			no_isync:1,     /* page has not been instruction synced */
+			pmapped:1,     	/* page has been entered at some
+					 * point into a pmap (O) */
 			absent:1,	/* Data has been requested, but is
 					 *  not yet available (O) */
 			error:1,	/* Data manager was unable to provide
@@ -171,21 +211,26 @@ struct vm_page {
 			restart:1,	/* Page was pushed higher in shadow
 					   chain by copy_call-related pagers;
 					   start again at top of chain */
-			lock_supplied:1,/* protection supplied by pager (O) */
-	/* vm_prot_t */	page_lock:3,	/* Uses prohibited by pager (O) */
-	/* vm_prot_t */	unlock_request:3,/* Outstanding unlock request (O) */
 			unusual:1,	/* Page is absent, error, restart or
 					   page locked */
 			encrypted:1,	/* encrypted for secure swap (O) */
+			encrypted_cleaning:1,	/* encrypting page */
 			list_req_pending:1, /* pagein/pageout alt mechanism */
 					    /* allows creation of list      */
 					    /* requests on pages that are   */
 					    /* actively being paged.        */
-			dump_cleaning:1;   /* set by the pageout daemon when */
+			dump_cleaning:1,   /* set by the pageout daemon when */
 					   /* a page being cleaned is       */
 					   /* encountered and targeted as   */
 					   /* a pageout candidate           */
-        /* we've used up all 32 bits */
+			cs_validated:1,    /* code-signing: page was checked */	
+			cs_tainted:1,	   /* code-signing: page is tainted */
+			no_cache:1,	   /* page is not to be cached and */
+					   /* should be reused ahead of    */
+					   /* other pages		   */
+	                deactivated:1,
+			zero_fill:1,
+			__unused_object_bits:9;  /* 9 bits available here */
 
 	ppnum_t		phys_page;	/* Physical address of page, passed
 					 *  to pmap_enter (read-only) */
@@ -220,18 +265,43 @@ typedef struct vm_page	*vm_page_t;
  *	some useful check on a page structure.
  */
 
-#define VM_PAGE_CHECK(mem)
+#define VM_PAGE_CHECK(mem) do {} while (0)
+
+/*     Page coloring:
+ *
+ *     The free page list is actually n lists, one per color,
+ *     where the number of colors is a function of the machine's
+ *     cache geometry set at system initialization.  To disable
+ *     coloring, set vm_colors to 1 and vm_color_mask to 0.
+ *     The boot-arg "colors" may be used to override vm_colors.
+ *     Note that there is little harm in having more colors than needed.
+ */
+ 
+#define MAX_COLORS      128
+#define	DEFAULT_COLORS	32
+
+extern
+unsigned int	vm_colors;		/* must be in range 1..MAX_COLORS */
+extern
+unsigned int	vm_color_mask;		/* must be (vm_colors-1) */
+extern
+unsigned int	vm_cache_geometry_colors; /* optimal #colors based on cache geometry */
 
 /*
  *	Each pageable resident page falls into one of three lists:
  *
  *	free	
- *		Available for allocation now.
+ *		Available for allocation now.  The free list is
+ *		actually an array of lists, one per color.
  *	inactive
  *		Not referenced in any map, but still has an
  *		object/offset-page mapping, and may be dirty.
  *		This is the list of pages that should be
- *		paged out next.
+ *		paged out next.  There are actually two
+ *		inactive lists, one for pages brought in from
+ *		disk or other backing store, and another
+ *		for "zero-filled" pages.  See vm_pageout_scan()
+ *		for the distinction and usage.
  *	active
  *		A list of pages which have been placed in
  *		at least one physical map.  This list is
@@ -239,14 +309,18 @@ typedef struct vm_page	*vm_page_t;
  */
 
 extern
-vm_page_t	vm_page_queue_free;	/* memory free queue */
+queue_head_t	vm_page_queue_free[MAX_COLORS];	/* memory free queue */
+extern
+queue_head_t	vm_lopage_queue_free;		/* low memory free queue */
 extern
 vm_page_t	vm_page_queue_fictitious;	/* fictitious free queue */
 extern
 queue_head_t	vm_page_queue_active;	/* active memory queue */
 extern
-queue_head_t	vm_page_queue_inactive;	/* inactive memory queue */
+queue_head_t	vm_page_queue_inactive;	/* inactive memory queue for normal pages */
+extern
 queue_head_t	vm_page_queue_zf;	/* inactive memory queue for zero fill */
+queue_head_t	vm_page_queue_throttled;	/* memory queue for throttled pageout pages */
 
 extern
 vm_offset_t	first_phys_addr;	/* physical address for first_page */
@@ -254,7 +328,7 @@ extern
 vm_offset_t	last_phys_addr;		/* physical address for last_page */
 
 extern
-unsigned int	vm_page_free_count;	/* How many pages are free? */
+unsigned int	vm_page_free_count;	/* How many pages are free? (sum of all colors) */
 extern
 unsigned int	vm_page_fictitious_count;/* How many fictitious pages are free? */
 extern
@@ -262,20 +336,34 @@ unsigned int	vm_page_active_count;	/* How many pages are active? */
 extern
 unsigned int	vm_page_inactive_count;	/* How many pages are inactive? */
 extern
+unsigned int	vm_page_throttled_count;/* How many inactives are throttled */
+extern
+unsigned int	vm_page_speculative_count;	/* How many speculative pages are unclaimed? */
+extern
 unsigned int	vm_page_wire_count;	/* How many pages are wired? */
 extern
+vm_map_size_t	vm_user_wire_limit;	/* How much memory can be locked by a user? */
+extern
+vm_map_size_t	vm_global_user_wire_limit;	/* How much memory can be locked system wide by users? */
+extern
 unsigned int	vm_page_free_target;	/* How many do we want free? */
 extern
 unsigned int	vm_page_free_min;	/* When to wakeup pageout */
 extern
 unsigned int	vm_page_inactive_target;/* How many do we want inactive? */
 extern
+unsigned int	vm_page_inactive_min;   /* When do wakeup pageout */
+extern
 unsigned int	vm_page_free_reserved;	/* How many pages reserved to do pageout */
 extern
-unsigned int	vm_page_throttled_count;/* Count of zero-fill allocations throttled */
+unsigned int	vm_page_zfill_throttle_count;/* Count of zero-fill allocations throttled */
 extern
 unsigned int	vm_page_gobble_count;
 
+extern
+unsigned int	vm_page_speculative_unused;
+extern
+unsigned int	vm_page_speculative_used;
 extern
 unsigned int	vm_page_purgeable_count;/* How many pages are purgeable now ? */
 extern
@@ -284,14 +372,21 @@ uint64_t	vm_page_purged_count;	/* How many pages got purged so far ? */
 decl_mutex_data(,vm_page_queue_lock)
 				/* lock on active and inactive page queues */
 decl_mutex_data(,vm_page_queue_free_lock)
-				/* lock on free page queue */
+				/* lock on free page queue array (ie, all colors) */
 
 extern unsigned int	vm_page_free_wanted;
 				/* how many threads are waiting for memory */
 
+extern unsigned int	vm_page_free_wanted_privileged;
+				/* how many VM privileged threads are waiting for memory */
+
 extern vm_offset_t	vm_page_fictitious_addr;
 				/* (fake) phys_addr of fictitious pages */
 
+extern vm_offset_t	vm_page_guard_addr;
+				/* (fake) phys_addr of guard pages */
+
+
 extern boolean_t	vm_page_deactivate_hint;
 
 // 0 = all pages avail, 1 = disable high mem, 2 = prefer himem
@@ -307,10 +402,10 @@ extern uint64_t		max_valid_dma_address;
  */
 extern void		vm_page_bootstrap(
 					vm_offset_t	*startp,
-					vm_offset_t	*endp);
-
-extern void		vm_page_module_init(void);
+					vm_offset_t	*endp) __attribute__((section("__TEXT, initcode")));
 
+extern void		vm_page_module_init(void) __attribute__((section("__TEXT, initcode")));
+					
 extern void		vm_page_create(
 					ppnum_t		start,
 					ppnum_t		end);
@@ -321,12 +416,11 @@ extern vm_page_t	vm_page_lookup(
 
 extern vm_page_t	vm_page_grab_fictitious(void);
 
+extern vm_page_t	vm_page_grab_guard(void);
+
 extern void		vm_page_release_fictitious(
 					vm_page_t page);
 
-extern boolean_t	vm_page_convert(
-					vm_page_t	page);
-
 extern void		vm_page_more_fictitious(void);
 
 extern int		vm_pool_low(void);
@@ -349,6 +443,10 @@ extern vm_page_t	vm_page_alloclo(
 					vm_object_t		object,
 					vm_object_offset_t	offset);
 
+extern vm_page_t	vm_page_alloc_guard(
+	vm_object_t		object,
+	vm_object_offset_t	offset);
+
 extern void		vm_page_init(
 					vm_page_t	page,
 					ppnum_t		phys_page);
@@ -356,16 +454,30 @@ extern void		vm_page_init(
 extern void		vm_page_free(
 					vm_page_t	page);
 
+extern void		vm_page_free_prepare(
+					vm_page_t	page);
+
 extern void		vm_page_activate(
 					vm_page_t	page);
 
 extern void		vm_page_deactivate(
 					vm_page_t	page);
 
+extern void		vm_page_lru(
+					vm_page_t	page);
+
+extern void		vm_page_speculate(
+					vm_page_t	page,
+					boolean_t	new);
+
+extern void		vm_page_speculate_ageit(
+					struct vm_speculative_age_q *aq);
+
 extern void		vm_page_rename(
 					vm_page_t		page,
 					vm_object_t		new_object,
-					vm_object_offset_t	new_offset);
+					vm_object_offset_t	new_offset,
+					boolean_t		encrypted_ok);
 
 extern void		vm_page_insert(
 					vm_page_t		page,
@@ -410,6 +522,8 @@ extern void		vm_set_page_size(void);
 extern void		vm_page_gobble(
 				        vm_page_t      page);
 
+extern void		vm_page_validate_cs(vm_page_t	page);
+
 /*
  *	Functions implemented as macros. m->wanted and m->busy are
  *	protected by the object lock.
@@ -453,46 +567,88 @@ extern void		vm_page_gobble(
 			vm_page_more_fictitious();			\
 		MACRO_END
 
-#define VM_PAGE_THROTTLED()						\
+#define VM_PAGE_ZFILL_THROTTLED()						\
 		(vm_page_free_count < vm_page_free_min &&		\
-		 !(current_thread()->options & TH_OPT_VMPRIV) && 			\
-		 ++vm_page_throttled_count)
+		 !(current_thread()->options & TH_OPT_VMPRIV) &&	\
+		 ++vm_page_zfill_throttle_count)
 
 #define	VM_PAGE_WAIT()		((void)vm_page_wait(THREAD_UNINT))
 
 #define vm_page_lock_queues()	mutex_lock(&vm_page_queue_lock)
 #define vm_page_unlock_queues()	mutex_unlock(&vm_page_queue_lock)
 
+#define vm_page_lockspin_queues()	mutex_lock_spin(&vm_page_queue_lock)
+
 #define VM_PAGE_QUEUES_REMOVE(mem)				\
 	MACRO_BEGIN						\
 	assert(!mem->laundry);					\
 	if (mem->active) {					\
 		assert(mem->object != kernel_object);		\
-		assert(!mem->inactive);				\
+		assert(!mem->inactive && !mem->speculative);	\
+		assert(!mem->throttled);			\
 		queue_remove(&vm_page_queue_active,		\
 			mem, vm_page_t, pageq);			\
-		mem->pageq.next = NULL;				\
-		mem->pageq.prev = NULL;			       	\
 		mem->active = FALSE;				\
-		if (!mem->fictitious)				\
+		if (!mem->fictitious) {				\
 			vm_page_active_count--;			\
+		} else {					\
+			assert(mem->phys_page ==		\
+			       vm_page_fictitious_addr);	\
+		}						\
 	}							\
 								\
-	if (mem->inactive) {					\
+	else if (mem->inactive) {				\
 		assert(mem->object != kernel_object);		\
-		assert(!mem->active);				\
+		assert(!mem->active && !mem->speculative);	\
+		assert(!mem->throttled);			\
 		if (mem->zero_fill) {				\
 			queue_remove(&vm_page_queue_zf,		\
 			mem, vm_page_t, pageq);			\
+			vm_zf_queue_count--;			\
 		} else {					\
 			queue_remove(&vm_page_queue_inactive,	\
 			mem, vm_page_t, pageq);			\
 		}						\
-		mem->pageq.next = NULL;				\
-		mem->pageq.prev = NULL;			       	\
 		mem->inactive = FALSE;				\
-		if (!mem->fictitious)				\
+		if (!mem->fictitious) {				\
 			vm_page_inactive_count--;		\
+			vm_purgeable_q_advance_all(1);		\
+		} else {					\
+			assert(mem->phys_page ==		\
+			       vm_page_fictitious_addr);	\
+		}						\
+	}							\
+								\
+	else if (mem->throttled) {				\
+		assert(!mem->active && !mem->inactive);		\
+		assert(!mem->speculative);			\
+		queue_remove(&vm_page_queue_throttled,		\
+			     mem, vm_page_t, pageq);		\
+		mem->throttled = FALSE;				\
+		if (!mem->fictitious)				\
+			vm_page_throttled_count--;		\
+	}							\
+								\
+	else if (mem->speculative) {				\
+		assert(!mem->active && !mem->inactive);		\
+		assert(!mem->throttled);			\
+		assert(!mem->fictitious);			\
+                remque(&mem->pageq);				\
+		mem->speculative = FALSE;			\
+		vm_page_speculative_count--;			\
+	}							\
+	mem->pageq.next = NULL;					\
+	mem->pageq.prev = NULL;					\
+	MACRO_END
+
+
+#define VM_PAGE_CONSUME_CLUSTERED(mem)				\
+	MACRO_BEGIN						\
+	if (mem->clustered) {					\
+	        assert(mem->object);				\
+	        mem->object->pages_used++;			\
+		mem->clustered = FALSE;				\
+		OSAddAtomic(1, (SInt32 *)&vm_page_speculative_used);	\
 	}							\
 	MACRO_END
 
diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c
index 97fda21af..7eeace1d0 100644
--- a/osfmk/vm/vm_pageout.c
+++ b/osfmk/vm/vm_pageout.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -74,6 +80,7 @@
 #include <mach/vm_map.h>
 #include <mach/vm_param.h>
 #include <mach/vm_statistics.h>
+#include <mach/sdt.h>
 
 #include <kern/kern_types.h>
 #include <kern/counters.h>
@@ -86,6 +93,10 @@
 
 #include <machine/vm_tuning.h>
 
+#if CONFIG_EMBEDDED
+#include <sys/kern_memorystatus.h>
+#endif
+
 #include <vm/pmap.h>
 #include <vm/vm_fault.h>
 #include <vm/vm_map.h>
@@ -93,21 +104,29 @@
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_protos.h> /* must be last */
+#include <vm/memory_object.h>
+#include <vm/vm_purgeable_internal.h>
 
 /*
  * ENCRYPTED SWAP:
  */
 #include <../bsd/crypto/aes/aes.h>
 
-extern ipc_port_t	memory_manager_default;
-
 
-#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE
-#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE  10000  /* maximum iterations of the active queue to move pages to inactive */
+#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE   /* maximum iterations of the active queue to move pages to inactive */
+#ifdef	CONFIG_EMBEDDED
+#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE  2048
+#else
+#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE  100
+#endif
 #endif
 
-#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE
-#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096  /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
+#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE  /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
+#ifdef	CONFIG_EMBEDDED
+#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
+#else
+#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
+#endif
 #endif
 
 #ifndef VM_PAGEOUT_DEADLOCK_RELIEF
@@ -138,6 +157,14 @@ extern ipc_port_t	memory_manager_default;
 #define VM_PAGEOUT_IDLE_WAIT	10	/* milliseconds */
 #endif	/* VM_PAGEOUT_IDLE_WAIT */
 
+#ifndef VM_PAGE_SPECULATIVE_TARGET
+#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / 20)
+#endif /* VM_PAGE_SPECULATIVE_TARGET */
+
+#ifndef VM_PAGE_INACTIVE_HEALTHY_LIMIT
+#define VM_PAGE_INACTIVE_HEALTHY_LIMIT(total) ((total) * 1 / 200)
+#endif /* VM_PAGE_INACTIVE_HEALTHY_LIMIT */
+
 
 /*
  *	To obtain a reasonable LRU approximation, the inactive queue
@@ -161,7 +188,11 @@ extern ipc_port_t	memory_manager_default;
  */
 
 #ifndef	VM_PAGE_FREE_TARGET
+#ifdef	CONFIG_EMBEDDED
+#define	VM_PAGE_FREE_TARGET(free)	(15 + (free) / 100)
+#else
 #define	VM_PAGE_FREE_TARGET(free)	(15 + (free) / 80)
+#endif
 #endif	/* VM_PAGE_FREE_TARGET */
 
 /*
@@ -170,9 +201,17 @@ extern ipc_port_t	memory_manager_default;
  */
 
 #ifndef	VM_PAGE_FREE_MIN
-#define	VM_PAGE_FREE_MIN(free)	(10 + (free) / 100)
+#ifdef	CONFIG_EMBEDDED
+#define	VM_PAGE_FREE_MIN(free)		(10 + (free) / 200)
+#else
+#define	VM_PAGE_FREE_MIN(free)		(10 + (free) / 100)
+#endif
 #endif	/* VM_PAGE_FREE_MIN */
 
+#define VM_PAGE_FREE_MIN_LIMIT		1500
+#define VM_PAGE_FREE_TARGET_LIMIT	2000
+
+
 /*
  *	When vm_page_free_count falls below vm_page_free_reserved,
  *	only vm-privileged threads can allocate pages.  vm-privilege
@@ -186,6 +225,24 @@ extern ipc_port_t	memory_manager_default;
 	((6 * VM_PAGE_LAUNDRY_MAX) + (n))
 #endif	/* VM_PAGE_FREE_RESERVED */
 
+/*
+ *	When we dequeue pages from the inactive list, they are
+ *	reactivated (ie, put back on the active queue) if referenced.
+ *	However, it is possible to starve the free list if other
+ *	processors are referencing pages faster than we can turn off
+ *	the referenced bit.  So we limit the number of reactivations
+ *	we will make per call of vm_pageout_scan().
+ */
+#define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
+#ifndef	VM_PAGE_REACTIVATE_LIMIT
+#ifdef	CONFIG_EMBEDDED
+#define	VM_PAGE_REACTIVATE_LIMIT(avail)	(VM_PAGE_INACTIVE_TARGET(avail) / 2)
+#else
+#define	VM_PAGE_REACTIVATE_LIMIT(avail)	(MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
+#endif
+#endif	/* VM_PAGE_REACTIVATE_LIMIT */
+#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM	100
+
 
 /*
  * must hold the page queues lock to
@@ -228,6 +285,9 @@ static void vm_pageout_queue_steal(vm_page_t);
 extern void vm_pageout_continue(void);
 extern void vm_pageout_scan(void);
 
+static thread_t	vm_pageout_external_iothread = THREAD_NULL;
+static thread_t	vm_pageout_internal_iothread = THREAD_NULL;
+
 unsigned int vm_pageout_reserved_internal = 0;
 unsigned int vm_pageout_reserved_really = 0;
 
@@ -245,10 +305,9 @@ unsigned int vm_pageout_burst_inactive_throttle = 0;
  *	from existing backing store and files
  */
 unsigned int vm_accellerate_zf_pageout_trigger = 400;
-unsigned int vm_zf_iterator;
-unsigned int vm_zf_iterator_count = 40;
-unsigned int last_page_zf;
+unsigned int zf_queue_min_count = 100;
 unsigned int vm_zf_count = 0;
+unsigned int vm_zf_queue_count = 0;
 
 /*
  *	These variables record the pageout daemon's actions:
@@ -273,6 +332,9 @@ unsigned int vm_stat_discard = 0;		/* debugging */
 unsigned int vm_stat_discard_sent = 0;		/* debugging */
 unsigned int vm_stat_discard_failure = 0;	/* debugging */
 unsigned int vm_stat_discard_throttle = 0;	/* debugging */
+unsigned int vm_pageout_reactivation_limit_exceeded = 0;	/* debugging */
+unsigned int vm_pageout_catch_ups = 0;				/* debugging */
+unsigned int vm_pageout_inactive_force_reclaim = 0;	/* debugging */
 
 unsigned int vm_pageout_scan_active_throttled = 0;
 unsigned int vm_pageout_scan_inactive_throttled = 0;
@@ -301,10 +363,13 @@ unsigned long vm_page_encrypt_abort_counter = 0;
 unsigned long vm_page_encrypt_already_encrypted_counter = 0;
 boolean_t vm_pages_encrypted = FALSE; /* are there encrypted pages ? */
 
-
 struct	vm_pageout_queue vm_pageout_queue_internal;
 struct	vm_pageout_queue vm_pageout_queue_external;
 
+unsigned int vm_page_speculative_target = 0;
+
+vm_object_t 	vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
 
 /*
  *	Routine:	vm_backing_store_disable
@@ -328,58 +393,6 @@ vm_backing_store_disable(
 }
 
 
-/*
- *	Routine:	vm_pageout_object_allocate
- *	Purpose:
- *		Allocate an object for use as out-of-line memory in a
- *		data_return/data_initialize message.
- *		The page must be in an unlocked object.
- *
- *		If the page belongs to a trusted pager, cleaning in place
- *		will be used, which utilizes a special "pageout object"
- *		containing private alias pages for the real page frames.
- *		Untrusted pagers use normal out-of-line memory.
- */
-vm_object_t
-vm_pageout_object_allocate(
-	vm_page_t		m,
-	vm_size_t		size,
-	vm_object_offset_t	offset)
-{
-	vm_object_t	object = m->object;
-	vm_object_t 	new_object;
-
-	assert(object->pager_ready);
-
-	new_object = vm_object_allocate(size);
-
-	if (object->pager_trusted) {
-		assert (offset < object->size);
-
-		vm_object_lock(new_object);
-		new_object->pageout = TRUE;
-		new_object->shadow = object;
-		new_object->can_persist = FALSE;
-		new_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
-		new_object->shadow_offset = offset;
-		vm_object_unlock(new_object);
-
-		/*
-		 * Take a paging reference on the object. This will be dropped
-		 * in vm_pageout_object_terminate()
-		 */
-		vm_object_lock(object);
-		vm_object_paging_begin(object);
-		vm_page_lock_queues();
-		vm_page_unlock_queues();
-		vm_object_unlock(object);
-
-		vm_pageout_in_place++;
-	} else
-		vm_pageout_out_of_line++;
-	return(new_object);
-}
-
 #if MACH_CLUSTER_STATS
 unsigned long vm_pageout_cluster_dirtied = 0;
 unsigned long vm_pageout_cluster_cleaned = 0;
@@ -397,8 +410,7 @@ unsigned long vm_pageout_target_page_freed = 0;
 /* 
  *	Routine:	vm_pageout_object_terminate
  *	Purpose:
- *		Destroy the pageout_object allocated by
- *		vm_pageout_object_allocate(), and perform all of the
+ *		Destroy the pageout_object, and perform all of the
  *		required cleanup actions.
  * 
  *	In/Out conditions:
@@ -409,7 +421,6 @@ vm_pageout_object_terminate(
 	vm_object_t	object)
 {
 	vm_object_t	shadow_object;
-	boolean_t	shadow_internal;
 
 	/*
 	 * Deal with the deallocation (last reference) of a pageout object
@@ -420,7 +431,6 @@ vm_pageout_object_terminate(
 	assert(object->pageout);
 	shadow_object = object->shadow;
 	vm_object_lock(shadow_object);
-	shadow_internal = shadow_object->internal;
 
 	while (!queue_empty(&object->memq)) {
 		vm_page_t 		p, m;
@@ -450,11 +460,6 @@ vm_pageout_object_terminate(
 		/* caller's page list indication */
 		m->dump_cleaning = FALSE;
 
-		/*
-		 * Account for the paging reference taken when
-		 * m->cleaning was set on this page.
-		 */
-		vm_object_paging_end(shadow_object);
 		assert((m->dirty) || (m->precious) ||
 				(m->busy && m->cleaning));
 
@@ -479,6 +484,7 @@ vm_pageout_object_terminate(
 			assert(m->busy);
 			assert(m->wire_count == 1);
 			m->cleaning = FALSE;
+			m->encrypted_cleaning = FALSE;
 			m->pageout = FALSE;
 #if MACH_CLUSTER_STATS
 			if (m->wanted) vm_pageout_target_collisions++;
@@ -501,7 +507,7 @@ vm_pageout_object_terminate(
 			if (m->dirty) {
 				CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
 				vm_page_unwire(m);/* reactivates */
-				VM_STAT(reactivations++);
+				VM_STAT_INCR(reactivations);
 				PAGE_WAKEUP_DONE(m);
 			} else {
 				CLUSTER_STAT(vm_pageout_target_page_freed++;)
@@ -516,7 +522,7 @@ vm_pageout_object_terminate(
 		 * If prep_pin_count is nonzero, then someone is using the
 		 * page, so make it active.
 		 */
-		if (!m->active && !m->inactive && !m->private) {
+		if (!m->active && !m->inactive && !m->throttled && !m->private) {
 			if (m->reference)
 				vm_page_activate(m);
 			else
@@ -535,13 +541,8 @@ vm_pageout_object_terminate(
 			/* will take care of resetting dirty. We clear the */
 			/* modify however for the Programmed I/O case.     */ 
 			pmap_clear_modify(m->phys_page);
-			if(m->absent) {
-				m->absent = FALSE;
-				if(shadow_object->absent_count == 1)
-					vm_object_absent_release(shadow_object);
-				else
-					shadow_object->absent_count--;
-			}
+
+			m->absent = FALSE;
 			m->overwriting = FALSE;
 		} else if (m->overwriting) {
 			/* alternate request page list, write to page_list */
@@ -571,6 +572,7 @@ vm_pageout_object_terminate(
 #endif
 		}
 		m->cleaning = FALSE;
+		m->encrypted_cleaning = FALSE;
 
 		/*
 		 * Wakeup any thread waiting for the page to be un-cleaning.
@@ -590,161 +592,6 @@ vm_pageout_object_terminate(
 	return;
 }
 
-/*
- *	Routine:	vm_pageout_setup
- *	Purpose:
- *		Set up a page for pageout (clean & flush).
- *
- *		Move the page to a new object, as part of which it will be
- *		sent to its memory manager in a memory_object_data_write or
- *		memory_object_initialize message.
- *
- *		The "new_object" and "new_offset" arguments
- *		indicate where the page should be moved.
- *
- *	In/Out conditions:
- *		The page in question must not be on any pageout queues,
- *		and must be busy.  The object to which it belongs
- *		must be unlocked, and the caller must hold a paging
- *		reference to it.  The new_object must not be locked.
- *
- *		This routine returns a pointer to a place-holder page,
- *		inserted at the same offset, to block out-of-order
- *		requests for the page.  The place-holder page must
- *		be freed after the data_write or initialize message
- *		has been sent.
- *
- *		The original page is put on a paging queue and marked
- *		not busy on exit.
- */
-vm_page_t
-vm_pageout_setup(
-	register vm_page_t	m,
-	register vm_object_t	new_object,
-	vm_object_offset_t	new_offset)
-{
-	register vm_object_t	old_object = m->object;
-	vm_object_offset_t	paging_offset;
-	vm_object_offset_t	offset;
-	register vm_page_t	holding_page;
-	register vm_page_t	new_m;
-	boolean_t		need_to_wire = FALSE;
-
-
-        XPR(XPR_VM_PAGEOUT,
-     "vm_pageout_setup, obj 0x%X off 0x%X page 0x%X new obj 0x%X offset 0x%X\n",
-                (integer_t)m->object, (integer_t)m->offset, 
-		(integer_t)m, (integer_t)new_object, 
-		(integer_t)new_offset);
-	assert(m && m->busy && !m->absent && !m->fictitious && !m->error &&
-		!m->restart);
-
-	assert(m->dirty || m->precious);
-
-	/*
-	 *	Create a place-holder page where the old one was, to prevent
-	 *	attempted pageins of this page while we're unlocked.
-	 */
-	VM_PAGE_GRAB_FICTITIOUS(holding_page);
-
-	vm_object_lock(old_object);
-
-	offset = m->offset;
-	paging_offset = offset + old_object->paging_offset;
-
-	if (old_object->pager_trusted) {
-		/*
-		 * This pager is trusted, so we can clean this page
-		 * in place. Leave it in the old object, and mark it
-		 * cleaning & pageout.
-		 */
-		new_m = holding_page;
-		holding_page = VM_PAGE_NULL;
-
-		/*
-		 * Set up new page to be private shadow of real page.
-		 */
-		new_m->phys_page = m->phys_page;
-		new_m->fictitious = FALSE;
-		new_m->pageout = TRUE;
-
-		/*
-		 * Mark real page as cleaning (indicating that we hold a
-		 * paging reference to be released via m_o_d_r_c) and
-		 * pageout (indicating that the page should be freed
-		 * when the pageout completes).
-		 */
-		pmap_clear_modify(m->phys_page);
-		vm_page_lock_queues();
-		new_m->private = TRUE;
-		vm_page_wire(new_m);
-		m->cleaning = TRUE;
-		m->pageout = TRUE;
-
-		vm_page_wire(m);
-		assert(m->wire_count == 1);
-		vm_page_unlock_queues();
-
-		m->dirty = TRUE;
-		m->precious = FALSE;
-		m->page_lock = VM_PROT_NONE;
-		m->unusual = FALSE;
-		m->unlock_request = VM_PROT_NONE;
-	} else {
-		/*
-		 * Cannot clean in place, so rip the old page out of the
-		 * object, and stick the holding page in. Set new_m to the
-		 * page in the new object.
-		 */
-		vm_page_lock_queues();
-		VM_PAGE_QUEUES_REMOVE(m);
-		vm_page_remove(m);
-
-		vm_page_insert(holding_page, old_object, offset);
-		vm_page_unlock_queues();
-
-		m->dirty = TRUE;
-		m->precious = FALSE;
-		new_m = m;
-		new_m->page_lock = VM_PROT_NONE;
-		new_m->unlock_request = VM_PROT_NONE;
-
-		if (old_object->internal)
-			need_to_wire = TRUE;
-	}
-	/*
-	 *	Record that this page has been written out
-	 */
-#if	MACH_PAGEMAP
-	vm_external_state_set(old_object->existence_map, offset);
-#endif	/* MACH_PAGEMAP */
-
-	vm_object_unlock(old_object);
-
-	vm_object_lock(new_object);
-
-	/*
-	 *	Put the page into the new object. If it is a not wired
-	 *	(if it's the real page) it will be activated.
-	 */
-
-	vm_page_lock_queues();
-	vm_page_insert(new_m, new_object, new_offset);
-	if (need_to_wire)
-		vm_page_wire(new_m);
-	else
-		vm_page_activate(new_m);
-	PAGE_WAKEUP_DONE(new_m);
-	vm_page_unlock_queues();
-
-	vm_object_unlock(new_object);
-
-	/*
-	 *	Return the placeholder page to simplify cleanup.
-	 */
-	return (holding_page);
-}
-
 /*
  * Routine:	vm_pageclean_setup
  *
@@ -763,24 +610,17 @@ vm_pageclean_setup(
 	vm_object_t		new_object,
 	vm_object_offset_t	new_offset)
 {
-	vm_object_t old_object = m->object;
 	assert(!m->busy);
+#if 0
 	assert(!m->cleaning);
+#endif
 
 	XPR(XPR_VM_PAGEOUT,
     "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
-		(integer_t)old_object, m->offset, (integer_t)m, 
+		(integer_t)m->object, m->offset, (integer_t)m, 
 		(integer_t)new_m, new_offset);
 
 	pmap_clear_modify(m->phys_page);
-	vm_object_paging_begin(old_object);
-
-	/*
-	 *	Record that this page has been written out
-	 */
-#if	MACH_PAGEMAP
-	vm_external_state_set(old_object->existence_map, m->offset);
-#endif	/*MACH_PAGEMAP*/
 
 	/*
 	 * Mark original page as cleaning in place.
@@ -794,6 +634,7 @@ vm_pageclean_setup(
 	 * the real page.
 	 */
 	assert(new_m->fictitious);
+	assert(new_m->phys_page == vm_page_fictitious_addr);
 	new_m->fictitious = FALSE;
 	new_m->private = TRUE;
 	new_m->pageout = TRUE;
@@ -805,53 +646,6 @@ vm_pageclean_setup(
 	new_m->busy = FALSE;
 }
 
-void
-vm_pageclean_copy(
-	vm_page_t		m,
-	vm_page_t		new_m,
-	vm_object_t		new_object,
-	vm_object_offset_t	new_offset)
-{
-	XPR(XPR_VM_PAGEOUT,
-	"vm_pageclean_copy, page 0x%X new_m 0x%X new_obj 0x%X offset 0x%X\n",
-		m, new_m, new_object, new_offset, 0);
-
-	assert((!m->busy) && (!m->cleaning));
-
-	assert(!new_m->private && !new_m->fictitious);
-
-	pmap_clear_modify(m->phys_page);
-
-	m->busy = TRUE;
-	vm_object_paging_begin(m->object);
-	vm_page_unlock_queues();
-	vm_object_unlock(m->object);
-
-	/*
-	 * Copy the original page to the new page.
-	 */
-	vm_page_copy(m, new_m);
-
-	/*
-	 * Mark the old page as clean. A request to pmap_is_modified
-	 * will get the right answer.
-	 */
-	vm_object_lock(m->object);
-	m->dirty = FALSE;
-
-	vm_object_paging_end(m->object);
-
-	vm_page_lock_queues();
-	if (!m->active && !m->inactive)
-		vm_page_activate(m);
-	PAGE_WAKEUP_DONE(m);
-
-	vm_page_insert(new_m, new_object, new_offset);
-	vm_page_activate(new_m);
-	new_m->busy = FALSE;	/* No other thread can be waiting */
-}
-
-
 /*
  *	Routine:	vm_pageout_initialize_page
  *	Purpose:
@@ -877,7 +671,7 @@ vm_pageout_initialize_page(
 	vm_object_t		object;
 	vm_object_offset_t	paging_offset;
 	vm_page_t		holding_page;
-
+	memory_object_t		pager;
 
 	XPR(XPR_VM_PAGEOUT,
 		"vm_pageout_initialize_page, page 0x%X\n",
@@ -896,16 +690,31 @@ vm_pageout_initialize_page(
 	 */
 	object = m->object;
 	paging_offset = m->offset + object->paging_offset;
-	vm_object_paging_begin(object);
-	if (m->absent || m->error || m->restart ||
-	    (!m->dirty && !m->precious)) {
+
+	if (m->absent || m->error || m->restart || (!m->dirty && !m->precious)) {
 		VM_PAGE_FREE(m);
 		panic("reservation without pageout?"); /* alan */
-	     vm_object_unlock(object);
+		vm_object_unlock(object);
+
+		return;
+	}
+
+	/*
+	 * If there's no pager, then we can't clean the page.  This should 
+	 * never happen since this should be a copy object and therefore not
+	 * an external object, so the pager should always be there.
+	 */
+
+	pager = object->pager;
+
+	if (pager == MEMORY_OBJECT_NULL) {
+		VM_PAGE_FREE(m);
+		panic("missing pager for copy object");
 		return;
 	}
 
 	/* set the page for future call to vm_fault_list_request */
+	vm_object_paging_begin(object);
 	holding_page = NULL;
 	vm_page_lock_queues();
 	pmap_clear_modify(m->phys_page);
@@ -926,11 +735,10 @@ vm_pageout_initialize_page(
 	 *	[The object reference from its allocation is donated
 	 *	to the eventual recipient.]
 	 */
-	memory_object_data_initialize(object->pager,
-					paging_offset,
-					PAGE_SIZE);
+	memory_object_data_initialize(pager, paging_offset, PAGE_SIZE);
 
 	vm_object_lock(object);
+	vm_object_paging_end(object);
 }
 
 #if	MACH_CLUSTER_STATS
@@ -942,7 +750,6 @@ struct {
 } cluster_stats[MAXCLUSTERPAGES];
 #endif	/* MACH_CLUSTER_STATS */
 
-boolean_t allow_clustered_pageouts = FALSE;
 
 /*
  * vm_pageout_cluster:
@@ -975,6 +782,7 @@ vm_pageout_cluster(vm_page_t m)
 	 */
 	assert(m->busy && (m->dirty || m->precious) && (m->wire_count == 0));
 	assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
+	assert(!m->throttled);
 
 	/*
 	 * protect the object from collapse - 
@@ -1049,7 +857,7 @@ vm_pageout_throttle_up(
  *	vm_page_free_wanted == 0.
  */
 
-#define DELAYED_UNLOCK_LIMIT  (3 * MAX_UPL_TRANSFER)
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT  (3 * MAX_UPL_TRANSFER)
 
 #define	FCS_IDLE		0
 #define FCS_DELAYED		1
@@ -1066,28 +874,52 @@ vm_pageout_scan(void)
 	unsigned int loop_count = 0;
 	unsigned int inactive_burst_count = 0;
 	unsigned int active_burst_count = 0;
-	vm_page_t   local_freeq = 0;
+	unsigned int reactivated_this_call;
+	unsigned int reactivate_limit;
+	vm_page_t   local_freeq = NULL;
 	int         local_freed = 0;
-	int         delayed_unlock = 0;
+	int         delayed_unlock;
 	int         need_internal_inactive = 0;
 	int	    refmod_state = 0;
         int	vm_pageout_deadlock_target = 0;
 	struct	vm_pageout_queue *iq;
 	struct	vm_pageout_queue *eq;
+        struct	vm_speculative_age_q *sq;
 	struct  flow_control	flow_control;
-        boolean_t active_throttled = FALSE;
         boolean_t inactive_throttled = FALSE;
+	boolean_t try_failed;
 	mach_timespec_t		ts;
 	unsigned int msecs = 0;
 	vm_object_t	object;
-	
+	vm_object_t	last_object_tried;
+	int	zf_ratio;
+	int	zf_run_count;
+	uint32_t	catch_up_count = 0;
+	uint32_t	inactive_reclaim_run;
+	boolean_t	forced_reclaim;
 
 	flow_control.state = FCS_IDLE;
 	iq = &vm_pageout_queue_internal;
 	eq = &vm_pageout_queue_external;
+	sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
 
         XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
 
+        
+	vm_page_lock_queues();
+	delayed_unlock = 1;	/* must be nonzero if Qs are locked, 0 if unlocked */
+
+	/*
+	 *	Calculate the max number of referenced pages on the inactive
+	 *	queue that we will reactivate.
+	 */
+	reactivated_this_call = 0;
+	reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
+						    vm_page_inactive_count);
+	inactive_reclaim_run = 0;
+
+
 /*???*/	/*
 	 *	We want to gradually dribble pages from the active queue
 	 *	to the inactive queue.  If we let the inactive queue get
@@ -1113,36 +945,87 @@ vm_pageout_scan(void)
 	 *	aren't vm-privileged.  If we kept sending dirty pages to them,
 	 *	we could exhaust the free list.
 	 */
-	vm_page_lock_queues();
-	delayed_unlock = 1;
 
 
 Restart:
+	assert(delayed_unlock!=0);
+	
+	/*
+	 *	A page is "zero-filled" if it was not paged in from somewhere,
+	 *	and it belongs to an object at least VM_ZF_OBJECT_SIZE_THRESHOLD big.
+	 *	Recalculate the zero-filled page ratio.  We use this to apportion
+	 *	victimized pages between the normal and zero-filled inactive
+	 *	queues according to their relative abundance in memory.  Thus if a task
+	 *	is flooding memory with zf pages, we begin to hunt them down.
+	 *	It would be better to throttle greedy tasks at a higher level,
+	 *	but at the moment mach vm cannot do this.
+	 */
+	{
+		uint32_t  total  = vm_page_active_count + vm_page_inactive_count;
+		uint32_t  normal = total - vm_zf_count;
+		
+		/* zf_ratio is the number of zf pages we victimize per normal page */
+		
+		if (vm_zf_count < vm_accellerate_zf_pageout_trigger)
+			zf_ratio = 0;
+		else if ((vm_zf_count <= normal) || (normal == 0))
+			zf_ratio = 1;
+		else 
+			zf_ratio = vm_zf_count / normal;
+			
+		zf_run_count = 0;
+	}
+        
 	/*
 	 *	Recalculate vm_page_inactivate_target.
 	 */
 	vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
-							  vm_page_inactive_count);
-	object = NULL;
+							  vm_page_inactive_count +
+							  vm_page_speculative_count);
+	/*
+	 * don't want to wake the pageout_scan thread up everytime we fall below
+	 * the targets... set a low water mark at 0.25% below the target
+	 */
+	vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400);
 
+	vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
+								vm_page_inactive_count);
+	object = NULL;
+	last_object_tried = NULL;
+	try_failed = FALSE;
+	
+	if ((vm_page_inactive_count + vm_page_speculative_count) < VM_PAGE_INACTIVE_HEALTHY_LIMIT(vm_page_active_count))
+	        catch_up_count = vm_page_inactive_count + vm_page_speculative_count;
+	else
+	        catch_up_count = 0;
+		    
 	for (;;) {
 		vm_page_t m;
 
-		if (delayed_unlock == 0)
-		        vm_page_lock_queues();
+		DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
 
-		active_burst_count = vm_page_active_count;
+		if (delayed_unlock == 0) {
+		        vm_page_lock_queues();
+			delayed_unlock = 1;
+		}
 
-		if (active_burst_count > vm_pageout_burst_active_throttle)
-		        active_burst_count = vm_pageout_burst_active_throttle;
+		/*
+		 *	Don't sweep through active queue more than the throttle
+		 *	which should be kept relatively low
+		 */
+		active_burst_count = vm_pageout_burst_active_throttle;
 
 		/*
 		 *	Move pages from active to inactive.
 		 */
-		while ((need_internal_inactive ||
-			   vm_page_inactive_count < vm_page_inactive_target) &&
-		       !queue_empty(&vm_page_queue_active) &&
-		       ((active_burst_count--) > 0)) {
+		if (need_internal_inactive == 0 && (vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
+		        goto done_moving_active_pages;
+
+		while (!queue_empty(&vm_page_queue_active) &&
+		       (need_internal_inactive || active_burst_count)) {
+
+		        if (active_burst_count)
+			       active_burst_count--;
 
 			vm_pageout_active++;
 
@@ -1151,6 +1034,9 @@ Restart:
 			assert(m->active && !m->inactive);
 			assert(!m->laundry);
 			assert(m->object != kernel_object);
+			assert(m->phys_page != vm_page_guard_addr);
+
+			DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
 
 			/*
 			 * Try to lock object; since we've already got the
@@ -1166,8 +1052,9 @@ Restart:
 			        if (object != NULL) {
 				        vm_object_unlock(object);
 					object = NULL;
+					vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 				}
-			        if (!vm_object_lock_try(m->object)) {
+			        if (!vm_object_lock_try_scan(m->object)) {
 				        /*
 					 * move page to end of active queue and continue
 					 */
@@ -1175,11 +1062,24 @@ Restart:
 						     vm_page_t, pageq);
 					queue_enter(&vm_page_queue_active, m,
 						    vm_page_t, pageq);
+
+					try_failed = TRUE;
 					
+					m = (vm_page_t) queue_first(&vm_page_queue_active);
+					/*
+					 * this is the next object we're going to be interested in
+					 * try to make sure its available after the mutex_yield
+					 * returns control
+					 */
+					vm_pageout_scan_wants_object = m->object;
+
 					goto done_with_activepage;
 				}
 				object = m->object;
+
+				try_failed = FALSE;
 			}
+
 			/*
 			 * if the page is BUSY, then we pull it
 			 * off the active queue and leave it alone.
@@ -1198,53 +1098,7 @@ Restart:
 
 				goto done_with_activepage;
 			}
-			if (need_internal_inactive) {
-			        /*
-				 * If we're unable to make forward progress
-				 * with the current set of pages on the 
-				 * inactive queue due to busy objects or
-				 * throttled pageout queues, then 
-				 * move a page that is already clean
-				 * or belongs to a pageout queue that
-				 * isn't currently throttled
-				 */
-			        active_throttled = FALSE;
-
-			        if (object->internal) {
-				        if ((VM_PAGE_Q_THROTTLED(iq) || !IP_VALID(memory_manager_default)))
-					        active_throttled = TRUE;
-				} else if (VM_PAGE_Q_THROTTLED(eq)) {
-				                active_throttled = TRUE;
-				}
-				if (active_throttled == TRUE) {
-				        if (!m->dirty) {
-					        refmod_state = pmap_get_refmod(m->phys_page);
-		  
-						if (refmod_state & VM_MEM_REFERENCED)
-						        m->reference = TRUE;
-						if (refmod_state & VM_MEM_MODIFIED)
-						        m->dirty = TRUE;
-					}
-					if (m->dirty || m->precious) {
-					        /*
-						 * page is dirty and targets a THROTTLED queue
-						 * so all we can do is move it back to the
-						 * end of the active queue to get it out
-						 * of the way
-						 */
-						queue_remove(&vm_page_queue_active, m,
-							     vm_page_t, pageq);
-						queue_enter(&vm_page_queue_active, m,
-							    vm_page_t, pageq);
 
-						vm_pageout_scan_active_throttled++;
-
-						goto done_with_activepage;
-					}
-				}
-				vm_pageout_scan_active_throttle_success++;
-				need_internal_inactive--;
-			}
 			/*
 			 *	Deactivate the page while holding the object
 			 *	locked, so we know the page is still not busy.
@@ -1254,30 +1108,34 @@ Restart:
 			 *	can handle that.
 			 */
 			vm_page_deactivate(m);
+
+			if (need_internal_inactive) {
+				vm_pageout_scan_active_throttle_success++;
+				need_internal_inactive--;
+			}
 done_with_activepage:
-			if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
+			if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
 
 			        if (object != NULL) {
 				        vm_object_unlock(object);
 					object = NULL;
+					vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 				}
 			        if (local_freeq) {
 				        vm_page_free_list(local_freeq);
 					
-					local_freeq = 0;
+					local_freeq = NULL;
 					local_freed = 0;
 				}
-			        delayed_unlock = 0;
-			        vm_page_unlock_queues();
+			        mutex_yield(&vm_page_queue_lock);
+
+				delayed_unlock = 1;
 
-				mutex_pause();
-				vm_page_lock_queues();
 				/*
 				 * continue the while loop processing
 				 * the active queue... need to hold
 				 * the page queues lock
 				 */
-				continue;
 			}
 		}
 
@@ -1289,7 +1147,7 @@ done_with_activepage:
 		 * and the inactive queue
 		 **********************************************************************/
 
-
+done_moving_active_pages:
 
 		/*
 		 *	We are done if we have met our target *and*
@@ -1300,25 +1158,90 @@ done_with_activepage:
 			        vm_object_unlock(object);
 				object = NULL;
 			}
+			vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
 			if (local_freeq) {
 			        vm_page_free_list(local_freeq);
 					
-				local_freeq = 0;
+				local_freeq = NULL;
 				local_freed = 0;
 			}
+			/*
+			 * inactive target still not met... keep going
+			 * until we get the queues balanced
+			 */
+			if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
+			    !queue_empty(&vm_page_queue_active))
+			        continue;
+
 		        mutex_lock(&vm_page_queue_free_lock);
 
 			if ((vm_page_free_count >= vm_page_free_target) &&
-			          (vm_page_free_wanted == 0)) {
+			    (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
 
 			        vm_page_unlock_queues();
 
 				thread_wakeup((event_t) &vm_pageout_garbage_collect);
+
+				assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+
 				return;
 			}
 			mutex_unlock(&vm_page_queue_free_lock);
 		}
+		/*
+		 * Before anything, we check if we have any ripe volatile objects around.
+		 * If so, purge the first and see what it gives us.
+		 */
+		assert (available_for_purge>=0);
+		if (available_for_purge)
+		{
+		        if (object != NULL) {
+			        vm_object_unlock(object);
+				object = NULL;
+			}
+			vm_purgeable_object_purge_one();
+			continue;
+		}
+        
+		if (queue_empty(&sq->age_q) && vm_page_speculative_count) {
+		        /*
+			 * try to pull pages from the aging bins
+			 * see vm_page.h for an explanation of how
+			 * this mechanism works
+			 */
+		        struct vm_speculative_age_q	*aq;
+			mach_timespec_t	ts_fully_aged;
+			boolean_t	can_steal = FALSE;
+		       
+			aq = &vm_page_queue_speculative[speculative_steal_index];
+
+			while (queue_empty(&aq->age_q)) {
+
+			        speculative_steal_index++;
+
+				if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
+				        speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+				
+				aq = &vm_page_queue_speculative[speculative_steal_index];
+			}
+			if (vm_page_speculative_count > vm_page_speculative_target)
+			        can_steal = TRUE;
+			else {
+			        ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) / 1000;
+				ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * VM_PAGE_SPECULATIVE_Q_AGE_MS) % 1000)
+				                      * 1000 * NSEC_PER_USEC;
+
+				ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
 
+			        clock_get_system_nanotime(&ts.tv_sec, (unsigned *)&ts.tv_nsec);
+
+				if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0)
+				        can_steal = TRUE;
+			}
+			if (can_steal == TRUE)
+			        vm_page_speculate_ageit(aq);
+		}
 
 		/*
 		 * Sometimes we have to pause:
@@ -1327,7 +1250,8 @@ done_with_activepage:
 		 *	3) Loop control - no acceptable pages found on the inactive queue
 		 *         within the last vm_pageout_burst_inactive_throttle iterations
 		 */
-		if ((queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_zf))) {
+		if (queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_zf) && queue_empty(&sq->age_q) &&
+		    (VM_PAGE_Q_THROTTLED(iq) || queue_empty(&vm_page_queue_throttled))) {
 		        vm_pageout_scan_empty_throttle++;
 			msecs = vm_pageout_empty_wait;
 			goto vm_pageout_scan_delay;
@@ -1337,7 +1261,7 @@ done_with_activepage:
 			msecs = vm_pageout_burst_wait;
 			goto vm_pageout_scan_delay;
 
-		} else if (VM_PAGE_Q_THROTTLED(iq)) {
+		} else if (VM_PAGE_Q_THROTTLED(iq) && IP_VALID(memory_manager_default)) {
 
 		        switch (flow_control.state) {
 
@@ -1345,9 +1269,8 @@ done_with_activepage:
 reset_deadlock_timer:
 			        ts.tv_sec = vm_pageout_deadlock_wait / 1000;
 				ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
-				clock_get_system_nanotime(
-					&flow_control.ts.tv_sec,
-					(uint32_t *) &flow_control.ts.tv_nsec);
+				clock_get_system_nanotime(&flow_control.ts.tv_sec,
+							  (unsigned *)&flow_control.ts.tv_nsec);
 				ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
 				
 				flow_control.state = FCS_DELAYED;
@@ -1356,9 +1279,8 @@ reset_deadlock_timer:
 				break;
 					
 			case FCS_DELAYED:
-			        clock_get_system_nanotime(
-					&ts.tv_sec,
-					(uint32_t *) &ts.tv_nsec);
+			        clock_get_system_nanotime(&ts.tv_sec,
+							  (unsigned *)&ts.tv_nsec);
 
 				if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
 				        /*
@@ -1374,10 +1296,10 @@ reset_deadlock_timer:
 					 * with a new timeout target since we have no way of knowing 
 					 * whether we've broken the deadlock except through observation
 					 * of the queue associated with the default pager... we need to
-					 * stop moving pagings and allow the system to run to see what
+					 * stop moving pages and allow the system to run to see what
 					 * state it settles into.
 					 */
-				        vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted;
+				        vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged;
 					vm_pageout_scan_deadlock_detected++;
 					flow_control.state = FCS_DEADLOCK_DETECTED;
 
@@ -1406,17 +1328,40 @@ vm_pageout_scan_delay:
 			        vm_object_unlock(object);
 				object = NULL;
 			}
+			vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
 			if (local_freeq) {
 			        vm_page_free_list(local_freeq);
 					
-				local_freeq = 0;
+				local_freeq = NULL;
 				local_freed = 0;
 			}
-			assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
+#if CONFIG_EMBEDDED
+			{
+			int percent_avail;
 
-			counter(c_vm_pageout_scan_block++);
+			/*
+			 * Decide if we need to send a memory status notification.
+			 */
+			percent_avail = 
+				(vm_page_active_count + vm_page_inactive_count + 
+				 vm_page_speculative_count + vm_page_free_count +
+				 vm_page_purgeable_count ) * 100 /
+				atop_64(max_mem);
+			if (percent_avail >= (kern_memorystatus_level + 5) || 
+			    percent_avail <= (kern_memorystatus_level - 5)) {
+				kern_memorystatus_level = percent_avail;
+				thread_wakeup((event_t)&kern_memorystatus_wakeup);
+			}
+			}
+#endif
+			assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
+
+			counter(c_vm_pageout_scan_block++);
 
 			vm_page_unlock_queues();
+
+			assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
 				
 			thread_block(THREAD_CONTINUE_NULL);
 
@@ -1425,17 +1370,8 @@ vm_pageout_scan_delay:
 
 			iq->pgo_throttled = FALSE;
 
-			if (loop_count >= vm_page_inactive_count) {
-			        if (VM_PAGE_Q_THROTTLED(eq) || VM_PAGE_Q_THROTTLED(iq)) {
-					/*
-					 * Make sure we move enough "appropriate"
-					 * pages to the inactive queue before trying
-					 * again.
-					 */
-					need_internal_inactive = vm_pageout_inactive_relief;
-				}
+			if (loop_count >= vm_page_inactive_count)
 				loop_count = 0;
-			}
 			inactive_burst_count = 0;
 
 			goto Restart;
@@ -1449,51 +1385,86 @@ consider_inactive:
 		inactive_burst_count++;
 		vm_pageout_inactive++;
 
-		if (!queue_empty(&vm_page_queue_inactive)) {
-		        m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+		/* Choose a victim. */
+		
+		while (1) {	
+			m = NULL;
 			
-			if (m->clustered && (m->no_isync == TRUE)) {
-			        goto use_this_page;
+			/*
+			 * the most eligible pages are ones that were throttled because the
+			 * pager wasn't ready at the time.  If a pager is ready now,
+			 * see if one of these is useful.
+			 */
+			if (!VM_PAGE_Q_THROTTLED(iq) && !queue_empty(&vm_page_queue_throttled)) {
+				m = (vm_page_t) queue_first(&vm_page_queue_throttled);
+				break;
 			}
-		}
-		if (vm_zf_count < vm_accellerate_zf_pageout_trigger) {
-			vm_zf_iterator = 0;
-		} else {
-			last_page_zf = 0;
-			if((vm_zf_iterator+=1) >= vm_zf_iterator_count) {
-					vm_zf_iterator = 0;
+
+			/*
+			 * The second most eligible pages are ones we paged in speculatively,
+			 * but which have not yet been touched.
+			 */
+			if ( !queue_empty(&sq->age_q) ) {
+			        m = (vm_page_t) queue_first(&sq->age_q);
+				break;
 			}
+			/*
+			 * Time for a zero-filled inactive page?
+			 */
+			if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) ||
+			     queue_empty(&vm_page_queue_inactive)) {
+				if ( !queue_empty(&vm_page_queue_zf) ) {
+					m = (vm_page_t) queue_first(&vm_page_queue_zf);
+					zf_run_count++;
+					break;
+				}
+			}
+			/*
+			 * It's either a normal inactive page or nothing.
+			 */
+                        if ( !queue_empty(&vm_page_queue_inactive) ) {
+                                m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+                                zf_run_count = 0;
+				break;
+                        }
+
+                        panic("vm_pageout: no victim");
 		}
-		if (queue_empty(&vm_page_queue_zf) ||
-				(((last_page_zf) || (vm_zf_iterator == 0)) &&
-				!queue_empty(&vm_page_queue_inactive))) {
-			m = (vm_page_t) queue_first(&vm_page_queue_inactive);
-			last_page_zf = 0;
-		} else {
-			m = (vm_page_t) queue_first(&vm_page_queue_zf);
-			last_page_zf = 1;
-		}
-use_this_page:
-		assert(!m->active && m->inactive);
+
+		assert(!m->active && (m->inactive || m->speculative || m->throttled));
 		assert(!m->laundry);
 		assert(m->object != kernel_object);
+		assert(m->phys_page != vm_page_guard_addr);
+
+		DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
 
 		/*
-		 * Try to lock object; since we've alread got the
-		 * page queues lock, we can only 'try' for this one.
-		 * if the 'try' fails, we need to do a mutex_pause
-		 * to allow the owner of the object lock a chance to
-		 * run... otherwise, we're likely to trip over this
-		 * object in the same state as we work our way through
-		 * the queue... clumps of pages associated with the same
-		 * object are fairly typical on the inactive and active queues
+		 * check to see if we currently are working
+		 * with the same object... if so, we've
+		 * already got the lock
 		 */
 		if (m->object != object) {
+		        /*
+			 * the object associated with candidate page is 
+			 * different from the one we were just working
+			 * with... dump the lock if we still own it
+			 */
 		        if (object != NULL) {
 			        vm_object_unlock(object);
 				object = NULL;
+				vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 			}
-			if (!vm_object_lock_try(m->object)) {
+			/*
+			 * Try to lock object; since we've alread got the
+			 * page queues lock, we can only 'try' for this one.
+			 * if the 'try' fails, we need to do a mutex_pause
+			 * to allow the owner of the object lock a chance to
+			 * run... otherwise, we're likely to trip over this
+			 * object in the same state as we work our way through
+			 * the queue... clumps of pages associated with the same
+			 * object are fairly typical on the inactive and active queues
+			 */
+			if (!vm_object_lock_try_scan(m->object)) {
 			        /*
 				 *	Move page to end and continue.
 				 * 	Don't re-issue ticket
@@ -1503,47 +1474,87 @@ use_this_page:
 						     vm_page_t, pageq);
 					queue_enter(&vm_page_queue_zf, m,
 						    vm_page_t, pageq);
+				} else if (m->speculative) {
+				        remque(&m->pageq);
+					m->speculative = FALSE;
+					vm_page_speculative_count--;
+					
+					/*
+					 * move to the tail of the inactive queue
+					 * to get it out of the way... the speculative
+					 * queue is generally too small to depend
+					 * on there being enough pages from other
+					 * objects to make cycling it back on the
+					 * same queue a winning proposition
+					 */
+					queue_enter(&vm_page_queue_inactive, m,
+						    vm_page_t, pageq);
+					m->inactive = TRUE;
+					vm_page_inactive_count++;
+					token_new_pagecount++;
+				}  else if (m->throttled) {
+					queue_remove(&vm_page_queue_throttled, m,
+						     vm_page_t, pageq);
+					m->throttled = FALSE;
+					vm_page_throttled_count--;
+
+					/*
+					 * not throttled any more, so can stick
+					 * it on the inactive queue.
+					 */
+					queue_enter(&vm_page_queue_inactive, m,
+						    vm_page_t, pageq);
+					m->inactive = TRUE;
+					vm_page_inactive_count++;
+					token_new_pagecount++;
 				} else {
 				        queue_remove(&vm_page_queue_inactive, m,
 						     vm_page_t, pageq);
+#if MACH_ASSERT
+					vm_page_inactive_count--;	/* balance for purgeable queue asserts */
+#endif
+					vm_purgeable_q_advance_all(1);
+
 					queue_enter(&vm_page_queue_inactive, m,
 						    vm_page_t, pageq);
+#if MACH_ASSERT
+					vm_page_inactive_count++;	/* balance for purgeable queue asserts */
+#endif
+					token_new_pagecount++;
 				}
+				pmap_clear_reference(m->phys_page);
+				m->reference = FALSE;
+
 				vm_pageout_inactive_nolock++;
 
+				if ( !queue_empty(&sq->age_q) )
+				        m = (vm_page_t) queue_first(&sq->age_q);
+				else if ( ((zf_run_count < zf_ratio) && vm_zf_queue_count >= zf_queue_min_count) ||
+					  queue_empty(&vm_page_queue_inactive)) {
+				        if ( !queue_empty(&vm_page_queue_zf) )
+					        m = (vm_page_t) queue_first(&vm_page_queue_zf);
+				} else if ( !queue_empty(&vm_page_queue_inactive) ) {
+				        m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+				}
+				/*
+				 * this is the next object we're going to be interested in
+				 * try to make sure its available after the mutex_yield
+				 * returns control
+				 */
+				vm_pageout_scan_wants_object = m->object;
+
 				/*
 				 * force us to dump any collected free pages
 				 * and to pause before moving on
 				 */
-				delayed_unlock = DELAYED_UNLOCK_LIMIT + 1;
+				try_failed = TRUE;
 
 				goto done_with_inactivepage;
 			}
 			object = m->object;
-		}
-		/*
-		 * If the page belongs to a purgable object with no pending copies
-		 * against it, then we reap all of the pages in the object
-		 * and note that the object has been "emptied".  It'll be up to the
-		 * application the discover this and recreate its contents if desired.
-		 */
-		if ((object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-		     object->purgable == VM_OBJECT_PURGABLE_EMPTY) &&
-		    object->copy == VM_OBJECT_NULL) {
-
-			(void) vm_object_purge(object);
-			vm_pageout_purged_objects++;
-			/*
-			 * we've just taken all of the pages from this object,
-			 * so drop the lock now since we're not going to find
-			 * any more pages belonging to it anytime soon
-			 */
-		        vm_object_unlock(object);
-			object = NULL;
+			vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 
-			inactive_burst_count = 0;
-
-			goto done_with_inactivepage;
+			try_failed = FALSE;
 		}
 
 		/*
@@ -1566,44 +1577,89 @@ use_this_page:
 			 *	pulled from the queue and paged out whenever
 			 *	one of its logically adjacent fellows is
 			 *	targeted.
+			 *
+			 *	Pages found on the speculative list can never be
+			 *	in this state... they always have a pager associated
+			 *	with them.
 			 */
+		        assert(!m->speculative);
+
 			if (m->zero_fill) {
 				queue_remove(&vm_page_queue_zf, m,
 					     vm_page_t, pageq);
 				queue_enter(&vm_page_queue_zf, m,
 					    vm_page_t, pageq);
-				last_page_zf = 1;
-				vm_zf_iterator = vm_zf_iterator_count - 1;
 			} else {
 				queue_remove(&vm_page_queue_inactive, m,
 					     vm_page_t, pageq);
+#if MACH_ASSERT
+				vm_page_inactive_count--;	/* balance for purgeable queue asserts */
+#endif
+				vm_purgeable_q_advance_all(1);
+
 				queue_enter(&vm_page_queue_inactive, m,
 					    vm_page_t, pageq);
-				last_page_zf = 0;
-				vm_zf_iterator = 1;
+#if MACH_ASSERT
+				vm_page_inactive_count++;	/* balance for purgeable queue asserts */
+#endif
+				token_new_pagecount++;
 			}
 			vm_pageout_inactive_avoid++;
 
 			goto done_with_inactivepage;
 		}
 		/*
-		 *	Remove the page from the inactive list.
+		 *	Remove the page from its list.
 		 */
-		if (m->zero_fill) {
-			queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
+		if (m->speculative) {
+			remque(&m->pageq);
+			m->speculative = FALSE;
+			vm_page_speculative_count--;
+		} else if (m->throttled) {
+			queue_remove(&vm_page_queue_throttled, m, vm_page_t, pageq);
+			m->throttled = FALSE;
+			vm_page_throttled_count--;
 		} else {
-			queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
+			if (m->zero_fill) {
+				queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
+				vm_zf_queue_count--;
+			} else {
+			        queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
+			}
+			m->inactive = FALSE;
+			if (!m->fictitious)
+				vm_page_inactive_count--;
+				vm_purgeable_q_advance_all(1);
+		}
+
+		if (object->copy == VM_OBJECT_NULL && 
+		    (object->purgable == VM_PURGABLE_EMPTY ||
+		     object->purgable == VM_PURGABLE_VOLATILE)) {
+		        assert(m->wire_count == 0);     /* if it's wired, we can't put it on our queue */
+			/* just stick it back on! */
+			goto reactivate_page;
 		}
 		m->pageq.next = NULL;
 		m->pageq.prev = NULL;
-		m->inactive = FALSE;
-		if (!m->fictitious)
-			vm_page_inactive_count--;
 
-		if (m->busy || !object->alive) {
+		if ( !m->fictitious && catch_up_count)
+		        catch_up_count--;
+
+		/*
+		 * ENCRYPTED SWAP:
+		 * if this page has already been picked up as part of a
+		 * page-out cluster, it will be busy because it is being
+		 * encrypted (see vm_object_upl_request()).  But we still
+		 * want to demote it from "clean-in-place" (aka "adjacent")
+		 * to "clean-and-free" (aka "target"), so let's ignore its
+		 * "busy" bit here and proceed to check for "cleaning" a
+		 * little bit below...
+		 */
+		if ( !m->encrypted_cleaning && (m->busy || !object->alive)) {
 			/*
 			 *	Somebody is already playing with this page.
 			 *	Leave it off the pageout queues.
+			 *
 			 */
 			vm_pageout_inactive_busy++;
 
@@ -1621,10 +1677,16 @@ reclaim_page:
 				vm_pageout_scan_inactive_throttle_success++;
 			        vm_pageout_deadlock_target--;
 			}
-			if (m->tabled)
-			        vm_page_remove(m);    /* clears tabled, object, offset */
-			if (m->absent)
-			        vm_object_absent_release(object);
+
+			DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
+
+			if (m->object->internal) {
+				DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
+			} else {
+				DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
+			}
+
+			vm_page_free_prepare(m);
 
 			assert(m->pageq.next == NULL &&
 			       m->pageq.prev == NULL);
@@ -1663,8 +1725,12 @@ reclaim_page:
 		/*
 		 *	If it's being used, reactivate.
 		 *	(Fictitious pages are either busy or absent.)
+		 *	First, update the reference and dirty bits
+		 *	to make sure the page is unreferenced.
 		 */
-		if ( (!m->reference) ) {
+		refmod_state = -1;
+
+		if (m->reference == FALSE && m->pmapped == TRUE) {
 		        refmod_state = pmap_get_refmod(m->phys_page);
 		  
 		        if (refmod_state & VM_MEM_REFERENCED)
@@ -1672,16 +1738,47 @@ reclaim_page:
 		        if (refmod_state & VM_MEM_MODIFIED)
 			        m->dirty = TRUE;
 		}
-		if (m->reference) {
-was_referenced:
-			vm_page_activate(m);
-			VM_STAT(reactivations++);
+		if (m->reference && !m->no_cache) {
+			/*
+			 * The page we pulled off the inactive list has
+			 * been referenced.  It is possible for other
+			 * processors to be touching pages faster than we
+			 * can clear the referenced bit and traverse the
+			 * inactive queue, so we limit the number of
+			 * reactivations.
+			 */
+			if (++reactivated_this_call >= reactivate_limit) {
+				vm_pageout_reactivation_limit_exceeded++;
+			} else if (catch_up_count) {
+				vm_pageout_catch_ups++;
+			} else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
+				vm_pageout_inactive_force_reclaim++;
+			} else {
+			        /*
+				 * The page was being used, so put back on active list.
+				 */
+reactivate_page:
+				vm_page_activate(m);
+				VM_STAT_INCR(reactivations);
 
-			vm_pageout_inactive_used++;
-			last_page_zf = 0;
-			inactive_burst_count = 0;
+				vm_pageout_inactive_used++;
+				inactive_burst_count = 0;
 
-			goto done_with_inactivepage;
+                                goto done_with_inactivepage;
+			}
+			/* 
+			 * Make sure we call pmap_get_refmod() if it
+			 * wasn't already called just above, to update
+			 * the dirty bit.
+			 */
+			if ((refmod_state == -1) && !m->dirty && m->pmapped) {
+				refmod_state = pmap_get_refmod(m->phys_page);
+				if (refmod_state & VM_MEM_MODIFIED)
+					m->dirty = TRUE;
+			}
+			forced_reclaim = TRUE;
+		} else {
+			forced_reclaim = FALSE;
 		}
 
                 XPR(XPR_VM_PAGEOUT,
@@ -1694,8 +1791,10 @@ was_referenced:
 		 * m->dirty is up to date courtesy of the
 		 * preceding check for m->reference... if 
 		 * we get here, then m->reference had to be
-		 * FALSE which means we did a pmap_get_refmod
-		 * and updated both m->reference and m->dirty
+		 * FALSE (or possibly "reactivate_limit" was
+                 * exceeded), but in either case we called
+                 * pmap_get_refmod() and updated both
+                 * m->reference and m->dirty
 		 *
 		 * if it's dirty or precious we need to
 		 * see if the target queue is throtttled
@@ -1706,28 +1805,40 @@ was_referenced:
 
 		if (m->dirty || m->precious) {
 		        if (object->internal) {
-			        if ((VM_PAGE_Q_THROTTLED(iq) || !IP_VALID(memory_manager_default)))
+				if (VM_PAGE_Q_THROTTLED(iq))
 				        inactive_throttled = TRUE;
 			} else if (VM_PAGE_Q_THROTTLED(eq)) {
-			                inactive_throttled = TRUE;
+				inactive_throttled = TRUE;
 			}
 		}
 		if (inactive_throttled == TRUE) {
-			if (m->zero_fill) {
-			        queue_enter(&vm_page_queue_zf, m,
+throttle_inactive:
+			if (!IP_VALID(memory_manager_default) &&
+				object->internal && 
+				(object->purgable == VM_PURGABLE_DENY ||
+				 object->purgable == VM_PURGABLE_NONVOLATILE)) {
+			        queue_enter(&vm_page_queue_throttled, m,
 					    vm_page_t, pageq);
+				m->throttled = TRUE;
+				vm_page_throttled_count++;
 			} else {
-			        queue_enter(&vm_page_queue_inactive, m,
-					    vm_page_t, pageq);
+			        if (m->zero_fill) {
+					queue_enter(&vm_page_queue_zf, m,
+						    vm_page_t, pageq);
+					vm_zf_queue_count++;
+				} else 
+					queue_enter(&vm_page_queue_inactive, m,
+						    vm_page_t, pageq);
+				m->inactive = TRUE;
+				if (!m->fictitious) {
+				        vm_page_inactive_count++;
+					token_new_pagecount++;
+				}
 			}
-			if (!m->fictitious)
-			        vm_page_inactive_count++;
-			m->inactive = TRUE;
-
 			vm_pageout_scan_inactive_throttled++;
-
 			goto done_with_inactivepage;
 		}
+
 		/*
 		 * we've got a page that we can steal...
 		 * eliminate all mappings and make sure
@@ -1749,25 +1860,40 @@ was_referenced:
 		 * since we already set m->busy = TRUE, before 
 		 * going off to reactivate it
 		 *
-		 * if we don't need the pmap_disconnect, then
-		 * m->dirty is up to date courtesy of the
-		 * earlier check for m->reference... if 
-		 * we get here, then m->reference had to be
-		 * FALSE which means we did a pmap_get_refmod
-		 * and updated both m->reference and m->dirty...
+		 * Note that if 'pmapped' is FALSE then the page is not
+		 * and has not been in any map, so there is no point calling
+		 * pmap_disconnect().  m->dirty and/or m->reference could
+		 * have been set in anticipation of likely usage of the page.
 		 */
-		if (m->no_isync == FALSE) {
+		if (m->pmapped == TRUE) {
 		        refmod_state = pmap_disconnect(m->phys_page);
 
 		        if (refmod_state & VM_MEM_MODIFIED)
 			        m->dirty = TRUE;
 		        if (refmod_state & VM_MEM_REFERENCED) {
-			        m->reference = TRUE;
-
-				PAGE_WAKEUP_DONE(m);
-				goto was_referenced;
+				
+				/* If m->reference is already set, this page must have
+				 * already failed the reactivate_limit test, so don't
+				 * bump the counts twice.
+				 */
+				if ( ! m->reference ) {
+					m->reference = TRUE;
+					if (forced_reclaim ||
+					    ++reactivated_this_call >= reactivate_limit)
+						vm_pageout_reactivation_limit_exceeded++;
+					else {
+						PAGE_WAKEUP_DONE(m);
+						goto reactivate_page;
+					}
+				}
 			}
 		}
+		/*
+		 * reset our count of pages that have been reclaimed 
+		 * since the last page was 'stolen'
+		 */
+		inactive_reclaim_run = 0;
+
 		/*
 		 *	If it's clean and not precious, we can free the page.
 		 */
@@ -1775,6 +1901,28 @@ was_referenced:
 			vm_pageout_inactive_clean++;
 			goto reclaim_page;
 		}
+
+		/*
+		 * The page may have been dirtied since the last check
+		 * for a throttled target queue (which may have been skipped
+		 * if the page was clean then).  With the dirty page
+		 * disconnected here, we can make one final check.
+		 */
+		{
+			boolean_t disconnect_throttled = FALSE;
+			if (object->internal) {
+				if (VM_PAGE_Q_THROTTLED(iq))
+					disconnect_throttled = TRUE;
+			} else if (VM_PAGE_Q_THROTTLED(eq)) {
+				disconnect_throttled = TRUE;
+			}
+
+			if (disconnect_throttled == TRUE) {
+				PAGE_WAKEUP_DONE(m);
+				goto throttle_inactive;
+			}
+		}
+
 		vm_pageout_cluster(m);
 
 		vm_pageout_inactive_dirty++;
@@ -1782,21 +1930,22 @@ was_referenced:
 		inactive_burst_count = 0;
 
 done_with_inactivepage:
-		if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
+		if (delayed_unlock++ > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT || try_failed == TRUE) {
 
 		        if (object != NULL) {
 			        vm_object_unlock(object);
 				object = NULL;
+				vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 			}
 		        if (local_freeq) {
 			        vm_page_free_list(local_freeq);
 				
-				local_freeq = 0;
+				local_freeq = NULL;
 				local_freed = 0;
 			}
-		        delayed_unlock = 0;
-			vm_page_unlock_queues();
-			mutex_pause();
+			mutex_yield(&vm_page_queue_lock);
+
+			delayed_unlock = 1;
 		}
 		/*
 		 * back to top of pageout scan loop
@@ -1820,11 +1969,18 @@ vm_page_free_reserve(
 	vm_page_free_min = vm_page_free_reserved +
 		VM_PAGE_FREE_MIN(free_after_reserve);
 
+	if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT)
+	        vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
+
 	vm_page_free_target = vm_page_free_reserved +
 		VM_PAGE_FREE_TARGET(free_after_reserve);
 
+	if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT)
+	        vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
+
 	if (vm_page_free_target < vm_page_free_min + 5)
 		vm_page_free_target = vm_page_free_min + 5;
+
 }
 
 /*
@@ -1834,10 +1990,12 @@ vm_page_free_reserve(
 void
 vm_pageout_continue(void)
 {
+	DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
 	vm_pageout_scan_event_counter++;
 	vm_pageout_scan();
 	/* we hold vm_page_queue_free_lock now */
 	assert(vm_page_free_wanted == 0);
+	assert(vm_page_free_wanted_privileged == 0);
 	assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
 	mutex_unlock(&vm_page_queue_free_lock);
 
@@ -1889,8 +2047,15 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 	vm_page_t	m = NULL;
 	vm_object_t	object;
 	boolean_t	need_wakeup;
+	memory_object_t	pager;
+	thread_t	self = current_thread();
 
-	vm_page_lock_queues();
+	if ((vm_pageout_internal_iothread != THREAD_NULL)
+	    && (self == vm_pageout_external_iothread )
+	    && (self->options & TH_OPT_VMPRIV))
+		self->options &= ~TH_OPT_VMPRIV;
+
+	vm_page_lockspin_queues();
 
         while ( !queue_empty(&q->pgo_pending) ) {
 
@@ -1922,8 +2087,9 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 #endif
 		   object = m->object;
 
+		   vm_object_lock(object);
+
 		   if (!object->pager_initialized) {
-		           vm_object_lock(object);
 
 			   /*
 			    *	If there is no memory object for the page, create
@@ -1947,11 +2113,10 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 			           m->list_req_pending = FALSE;
 				   m->cleaning = FALSE;
 				   m->pageout = FALSE;
-				   vm_page_unwire(m);
 
+			           vm_page_lockspin_queues();
+				   vm_page_unwire(m);
 				   vm_pageout_throttle_up(m);
-
-			           vm_page_lock_queues();
 				   vm_pageout_dirty_no_pager++;
 				   vm_page_activate(m);
 				   vm_page_unlock_queues();
@@ -1964,27 +2129,30 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 			           vm_object_paging_end(object);
 				   vm_object_unlock(object);
 
-				   vm_page_lock_queues();
+				   vm_page_lockspin_queues();
 				   continue;
-			   } else if (object->pager == MEMORY_OBJECT_NULL) {
-			           /*
-				    * This pager has been destroyed by either
-				    * memory_object_destroy or vm_object_destroy, and
-				    * so there is nowhere for the page to go.
-				    * Just free the page... VM_PAGE_FREE takes
-				    * care of cleaning up all the state...
-				    * including doing the vm_pageout_throttle_up
-				    */
-			           VM_PAGE_FREE(m);
+			   }
+		   }
+		   pager = object->pager;
+	           if (pager == MEMORY_OBJECT_NULL) {
+		           /*
+			    * This pager has been destroyed by either
+			    * memory_object_destroy or vm_object_destroy, and
+			    * so there is nowhere for the page to go.
+			    * Just free the page... VM_PAGE_FREE takes
+			    * care of cleaning up all the state...
+			    * including doing the vm_pageout_throttle_up
+			    */
 
-			           vm_object_paging_end(object);
-				   vm_object_unlock(object);
+		           VM_PAGE_FREE(m);
 
-				   vm_page_lock_queues();
-				   continue;
-			   }
+			   vm_object_paging_end(object);
 			   vm_object_unlock(object);
+
+			   vm_page_lockspin_queues();
+			   continue;
 		   }
+		   vm_object_unlock(object);
 		   /*
 		    * we expect the paging_in_progress reference to have
 		    * already been taken on the object before it was added
@@ -1997,7 +2165,7 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 		    * Send the data to the pager.
 		    * any pageout clustering happens there
 		    */
-		   memory_object_data_return(object->pager,
+		   memory_object_data_return(pager,
 					     m->offset + object->paging_offset,
 					     PAGE_SIZE,
 					     NULL,
@@ -2010,7 +2178,7 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 		   vm_object_paging_end(object);
 		   vm_object_unlock(object);
 
-		   vm_page_lock_queues();
+		   vm_page_lockspin_queues();
 	}
 	assert_wait((event_t) q, THREAD_UNINT);
 
@@ -2036,6 +2204,9 @@ vm_pageout_iothread_continue(struct vm_pageout_queue *q)
 static void
 vm_pageout_iothread_external(void)
 {
+	thread_t	self = current_thread();
+
+	self->options |= TH_OPT_VMPRIV;
 
 	vm_pageout_iothread_continue(&vm_pageout_queue_external);
 	/*NOTREACHED*/
@@ -2093,6 +2264,10 @@ vm_pageout(void)
 	self->priority = BASEPRI_PREEMPT - 1;
 	set_sched_pri(self, self->priority);
 	thread_unlock(self);
+
+	if (!self->reserved_stack)
+		self->reserved_stack = self->kernel_stack;
+
 	splx(s);
 
 	/*
@@ -2132,7 +2307,7 @@ vm_pageout(void)
 	task_unlock(kernel_task);
 
 	vm_page_free_count_init = vm_page_free_count;
-	vm_zf_iterator = 0;
+
 	/*
 	 * even if we've already called vm_page_free_reserve
 	 * call it again here to insure that the targets are
@@ -2154,28 +2329,27 @@ vm_pageout(void)
 	vm_pageout_queue_external.pgo_throttled = FALSE;
 
 	queue_init(&vm_pageout_queue_internal.pgo_pending);
-	vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
+	vm_pageout_queue_internal.pgo_maxlaundry = 0;
 	vm_pageout_queue_internal.pgo_laundry = 0;
 	vm_pageout_queue_internal.pgo_idle = FALSE;
 	vm_pageout_queue_internal.pgo_busy = FALSE;
 	vm_pageout_queue_internal.pgo_throttled = FALSE;
 
 
-	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, NULL, BASEPRI_PREEMPT - 1, &thread);
-	if (result != KERN_SUCCESS)
-		panic("vm_pageout_iothread_internal: create failed");
-
-	thread_deallocate(thread);
-
+	/* internal pageout thread started when default pager registered first time */
+	/* external pageout and garbage collection threads started here */
 
-	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, BASEPRI_PREEMPT - 1, &thread);
+	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, 
+					      BASEPRI_PREEMPT - 1, 
+					      &vm_pageout_external_iothread);
 	if (result != KERN_SUCCESS)
 		panic("vm_pageout_iothread_external: create failed");
 
-	thread_deallocate(thread);
-
+	thread_deallocate(vm_pageout_external_iothread);
 
-	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL, BASEPRI_PREEMPT - 2, &thread);
+	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,
+					      MINPRI_KERNEL, 
+					      &thread);
 	if (result != KERN_SUCCESS)
 		panic("vm_pageout_garbage_collect: create failed");
 
@@ -2183,32 +2357,72 @@ vm_pageout(void)
 
 	vm_object_reaper_init();
 
+
 	vm_pageout_continue();
+
+	/*
+	 * Unreached code!
+	 *
+	 * The vm_pageout_continue() call above never returns, so the code below is never
+	 * executed.  We take advantage of this to declare several DTrace VM related probe
+	 * points that our kernel doesn't have an analog for.  These are probe points that
+	 * exist in Solaris and are in the DTrace documentation, so people may have written
+	 * scripts that use them.  Declaring the probe points here means their scripts will
+	 * compile and execute which we want for portability of the scripts, but since this
+	 * section of code is never reached, the probe points will simply never fire.  Yes,
+	 * this is basically a hack.  The problem is the DTrace probe points were chosen with
+	 * Solaris specific VM events in mind, not portability to different VM implementations.
+	 */
+
+	DTRACE_VM2(execfree, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(execpgin, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(execpgout, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(pgswapin, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(pgswapout, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(swapin, int, 1, (uint64_t *), NULL);
+	DTRACE_VM2(swapout, int, 1, (uint64_t *), NULL);
 	/*NOTREACHED*/
 }
 
+kern_return_t
+vm_pageout_internal_start(void)
+{
+	kern_return_t result;
+
+	vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
+	result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, NULL, BASEPRI_PREEMPT - 1, &vm_pageout_internal_iothread);
+	if (result == KERN_SUCCESS)
+		thread_deallocate(vm_pageout_internal_iothread);
+	return result;
+}
+
+#define UPL_DELAYED_UNLOCK_LIMIT  (MAX_UPL_TRANSFER / 2)
 
 static upl_t
-upl_create(
-	int		   flags,
-	upl_size_t       size)
+upl_create(int type, int flags, upl_size_t size)
 {
 	upl_t	upl;
-	int	page_field_size;  /* bit field in word size buf */
+	int	page_field_size = 0;
+	int	upl_flags = 0;
+	int	upl_size  = sizeof(struct upl);
 
-	page_field_size = 0;
-	if (flags & UPL_CREATE_LITE) {
+	if (type & UPL_CREATE_LITE) {
 		page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
 		page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
+
+		upl_flags |= UPL_LITE;
 	}
-	if(flags & UPL_CREATE_INTERNAL) {
-		upl = (upl_t)kalloc(sizeof(struct upl)
-			+ (sizeof(struct upl_page_info)*(size/PAGE_SIZE))
-			+ page_field_size);
-	} else {
-		upl = (upl_t)kalloc(sizeof(struct upl) + page_field_size);
+	if (type & UPL_CREATE_INTERNAL) {
+		upl_size += sizeof(struct upl_page_info) * (size/PAGE_SIZE);
+
+		upl_flags |= UPL_INTERNAL;
 	}
-	upl->flags = 0;
+	upl = (upl_t)kalloc(upl_size + page_field_size);
+
+	if (page_field_size)
+	        bzero((char *)upl + upl_size, page_field_size);
+
+	upl->flags = upl_flags | flags;
 	upl->src_object = NULL;
 	upl->kaddr = (vm_offset_t)0;
 	upl->size = 0;
@@ -2224,45 +2438,46 @@ upl_create(
 }
 
 static void
-upl_destroy(
-	upl_t	upl)
+upl_destroy(upl_t upl)
 {
 	int	page_field_size;  /* bit field in word size buf */
+        int	size;
 
 #ifdef UPL_DEBUG
 	{
-		upl_t	upl_ele;
 		vm_object_t	object;
-		if (upl->map_object->pageout) {
+
+		if (upl->flags & UPL_SHADOWED) {
 			object = upl->map_object->shadow;
 		} else {
 			object = upl->map_object;
 		}
 		vm_object_lock(object);
-		queue_iterate(&object->uplq, upl_ele, upl_t, uplq) {
-			if(upl_ele == upl) {
-				queue_remove(&object->uplq, 
-						upl_ele, upl_t, uplq);
-				break;
-			}
-		}
+		queue_remove(&object->uplq, upl, upl_t, uplq);
 		vm_object_unlock(object);
 	}
 #endif /* UPL_DEBUG */
-	/* drop a reference on the map_object whether or */
-	/* not a pageout object is inserted */
-	if(upl->map_object->pageout)
+	/*
+	 * drop a reference on the map_object whether or
+	 * not a pageout object is inserted
+	 */
+	if (upl->flags & UPL_SHADOWED)
 		vm_object_deallocate(upl->map_object);
 
+        if (upl->flags & UPL_DEVICE_MEMORY)
+	        size = PAGE_SIZE;
+	else
+	        size = upl->size;
 	page_field_size = 0;
+
 	if (upl->flags & UPL_LITE) {
-		page_field_size = ((upl->size/PAGE_SIZE) + 7) >> 3;
+		page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
 		page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
 	}
-	if(upl->flags & UPL_INTERNAL) {
+	if (upl->flags & UPL_INTERNAL) {
 		kfree(upl,
 		      sizeof(struct upl) + 
-		      (sizeof(struct upl_page_info) * (upl->size/PAGE_SIZE))
+		      (sizeof(struct upl_page_info) * (size/PAGE_SIZE))
 		      + page_field_size);
 	} else {
 		kfree(upl, sizeof(struct upl) + page_field_size);
@@ -2271,24 +2486,17 @@ upl_destroy(
 
 void uc_upl_dealloc(upl_t upl);
 __private_extern__ void
-uc_upl_dealloc(
-	upl_t	upl)
+uc_upl_dealloc(upl_t upl)
 {
-	upl->ref_count -= 1;
-	if(upl->ref_count == 0) {
+	if (--upl->ref_count == 0)
 		upl_destroy(upl);
-	}
 }
 
 void
-upl_deallocate(
-	upl_t	upl)
+upl_deallocate(upl_t upl)
 {
-	
-	upl->ref_count -= 1;
-	if(upl->ref_count == 0) {
+	if (--upl->ref_count == 0)
 		upl_destroy(upl);
-	}
 }
 
 /*
@@ -2356,9 +2564,8 @@ vm_object_upl_request(
 	int			cntrl_flags)
 {
 	vm_page_t		dst_page = VM_PAGE_NULL;
-	vm_object_offset_t	dst_offset = offset;
-	upl_size_t		xfer_size = size;
-	boolean_t		do_m_lock = FALSE;
+	vm_object_offset_t	dst_offset;
+	upl_size_t		xfer_size;
 	boolean_t		dirty;
 	boolean_t		hw_dirty;
 	upl_t			upl = NULL;
@@ -2367,11 +2574,10 @@ vm_object_upl_request(
 	boolean_t		encountered_lrp = FALSE;
 #endif
 	vm_page_t		alias_page = NULL;
-	int			page_ticket; 
-        int			refmod_state;
+        int			refmod_state = 0;
 	wpl_array_t 		lite_list = NULL;
 	vm_object_t		last_copy_object;
-
+	int                     delayed_unlock = 0;
 
 	if (cntrl_flags & ~UPL_VALID_FLAGS) {
 		/*
@@ -2380,195 +2586,96 @@ vm_object_upl_request(
 		 */
 		return KERN_INVALID_VALUE;
 	}
+	if ( (!object->internal) && (object->paging_offset != 0) )
+		panic("vm_object_upl_request: external object with non-zero paging offset\n");
+	if (object->phys_contiguous)
+	        panic("vm_object_upl_request: contiguous object specified\n");
 
-	page_ticket = (cntrl_flags & UPL_PAGE_TICKET_MASK)
-					>> UPL_PAGE_TICKET_SHIFT;
 
-	if(((size/PAGE_SIZE) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
+	if ((size / PAGE_SIZE) > MAX_UPL_TRANSFER)
 		size = MAX_UPL_TRANSFER * PAGE_SIZE;
-	}
-
-	if(cntrl_flags & UPL_SET_INTERNAL)
-		if(page_list_count != NULL)
-			*page_list_count = MAX_UPL_TRANSFER;
 
-	if((!object->internal) && (object->paging_offset != 0))
-		panic("vm_object_upl_request: external object with non-zero paging offset\n");
+	if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL)
+	        *page_list_count = MAX_UPL_TRANSFER;
 
-	if((cntrl_flags & UPL_COPYOUT_FROM) && (upl_ptr == NULL)) {
-		return KERN_SUCCESS;
-	}
+	if (cntrl_flags & UPL_SET_INTERNAL) {
+	        if (cntrl_flags & UPL_SET_LITE) {
 
-	vm_object_lock(object);
-	vm_object_paging_begin(object);
-	vm_object_unlock(object);
+			upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, 0, size);
 
-	if(upl_ptr) {
-		if(cntrl_flags & UPL_SET_INTERNAL) {
-			if(cntrl_flags & UPL_SET_LITE) {
-				uintptr_t page_field_size;
-				upl = upl_create(
-					UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
-					size);
-				user_page_list = (upl_page_info_t *)
-				   (((uintptr_t)upl) + sizeof(struct upl));
-				lite_list = (wpl_array_t)
+			user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+			lite_list = (wpl_array_t)
 					(((uintptr_t)user_page_list) + 
-					((size/PAGE_SIZE) * 
-						sizeof(upl_page_info_t)));
-				page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
-				page_field_size = 
-					(page_field_size + 3) & 0xFFFFFFFC;
-				bzero((char *)lite_list, page_field_size);
-				upl->flags = 
-					UPL_LITE | UPL_INTERNAL;
-			} else {
-				upl = upl_create(UPL_CREATE_INTERNAL, size);
-				user_page_list = (upl_page_info_t *)
-					(((uintptr_t)upl) + sizeof(struct upl));
-				upl->flags = UPL_INTERNAL;
-			}
+					((size/PAGE_SIZE) * sizeof(upl_page_info_t)));
 		} else {
-			if(cntrl_flags & UPL_SET_LITE) {
-				uintptr_t page_field_size;
-				upl = upl_create(UPL_CREATE_LITE, size);
-				lite_list = (wpl_array_t)
-				   (((uintptr_t)upl) + sizeof(struct upl));
-				page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
-				page_field_size = 
-					(page_field_size + 3) & 0xFFFFFFFC;
-				bzero((char *)lite_list, page_field_size);
-				upl->flags = UPL_LITE;
-			} else {
-				upl = upl_create(UPL_CREATE_EXTERNAL, size);
-				upl->flags = 0;
-			}
-		}
-
-		if (object->phys_contiguous) {
-			if ((cntrl_flags & UPL_WILL_MODIFY) &&
-			    object->copy != VM_OBJECT_NULL) {
-				/* Honor copy-on-write obligations */
-
-				/*
-				 * XXX FBDP
-				 * We could still have a race...
-				 * A is here building the UPL for a write().
-				 * A pushes the pages to the current copy
-				 * object.
-				 * A returns the UPL to the caller.
-				 * B comes along and establishes another
-				 * private mapping on this object, inserting 
-				 * a new copy object between the original
-				 * object and the old copy object.
-				 * B reads a page and gets the original contents
-				 * from the original object.
-				 * A modifies the page in the original object.
-				 * B reads the page again and sees A's changes,
-				 * which is wrong...
-				 *
-				 * The problem is that the pages are not
-				 * marked "busy" in the original object, so
-				 * nothing prevents B from reading it before
-				 * before A's changes are completed.
-				 *
-				 * The "paging_in_progress" might protect us
-				 * from the insertion of a new copy object
-				 * though...  To be verified.
-				 */
-				vm_object_lock_request(object,
-						       offset,
-						       size,
-						       FALSE,
-						       MEMORY_OBJECT_COPY_SYNC,
-						       VM_PROT_NO_CHANGE);
-				upl_cow_contiguous++;
-				upl_cow_contiguous_pages += size >> PAGE_SHIFT;
-			}
-
-			upl->map_object = object;
-			/* don't need any shadow mappings for this one */
-			/* since it is already I/O memory */
-			upl->flags |= UPL_DEVICE_MEMORY;
+		        upl = upl_create(UPL_CREATE_INTERNAL, 0, size);
 
-
-			/* paging_in_progress protects paging_offset */
-			upl->offset = offset + object->paging_offset;
-			upl->size = size;
-			*upl_ptr = upl;
-			if(user_page_list) {
-				user_page_list[0].phys_addr = 
-				   (offset + object->shadow_offset)>>PAGE_SHIFT;
-				user_page_list[0].device = TRUE;
-			}
-			upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
-
-			if(page_list_count != NULL) {
-				if (upl->flags & UPL_INTERNAL) {
-					*page_list_count = 0;
-				} else {
-					*page_list_count = 1;
-				}
-			}
-
-			return KERN_SUCCESS;
+			user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
 		}
+	} else {
+	        if (cntrl_flags & UPL_SET_LITE) {
 
-		if(user_page_list)
-			user_page_list[0].device = FALSE;
+			upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE, 0, size);
 
-		if(cntrl_flags & UPL_SET_LITE) {
-			upl->map_object = object;
+			lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
 		} else {
-			upl->map_object = vm_object_allocate(size);
-			/*
-			 * No neeed to lock the new object: nobody else knows
-			 * about it yet, so it's all ours so far.
-			 */
-			upl->map_object->shadow = object;
-			upl->map_object->pageout = TRUE;
-			upl->map_object->can_persist = FALSE;
-			upl->map_object->copy_strategy = 
-					MEMORY_OBJECT_COPY_NONE;
-			upl->map_object->shadow_offset = offset;
-			upl->map_object->wimg_bits = object->wimg_bits;
+		        upl = upl_create(UPL_CREATE_EXTERNAL, 0, size);
 		}
-
-	}
-	if (!(cntrl_flags & UPL_SET_LITE)) {
-		VM_PAGE_GRAB_FICTITIOUS(alias_page);
 	}
+	*upl_ptr = upl;
+	
+	if (user_page_list)
+	        user_page_list[0].device = FALSE;
 
-	/*
+	if (cntrl_flags & UPL_SET_LITE) {
+	        upl->map_object = object;
+	} else {
+	        upl->map_object = vm_object_allocate(size);
+		/*
+		 * No neeed to lock the new object: nobody else knows
+		 * about it yet, so it's all ours so far.
+		 */
+		upl->map_object->shadow = object;
+		upl->map_object->pageout = TRUE;
+		upl->map_object->can_persist = FALSE;
+		upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+		upl->map_object->shadow_offset = offset;
+		upl->map_object->wimg_bits = object->wimg_bits;
+
+		VM_PAGE_GRAB_FICTITIOUS(alias_page);
+
+		upl->flags |= UPL_SHADOWED;
+	}
+	/*
 	 * ENCRYPTED SWAP:
 	 * Just mark the UPL as "encrypted" here.
 	 * We'll actually encrypt the pages later,
 	 * in upl_encrypt(), when the caller has
 	 * selected which pages need to go to swap.
 	 */
-	if (cntrl_flags & UPL_ENCRYPT) {
+	if (cntrl_flags & UPL_ENCRYPT)
 		upl->flags |= UPL_ENCRYPTED;
-	}
-	if (cntrl_flags & UPL_FOR_PAGEOUT) {
+
+	if (cntrl_flags & UPL_FOR_PAGEOUT)
 		upl->flags |= UPL_PAGEOUT;
-	}
+
 	vm_object_lock(object);
+	vm_object_paging_begin(object);
+
+	/*
+	 * we can lock in the paging_offset once paging_in_progress is set
+	 */
+	upl->size = size;
+	upl->offset = offset + object->paging_offset;
 
-	/* we can lock in the paging_offset once paging_in_progress is set */
-	if(upl_ptr) {
-		upl->size = size;
-		upl->offset = offset + object->paging_offset;
-		*upl_ptr = upl;
 #ifdef UPL_DEBUG
-		queue_enter(&object->uplq, upl, upl_t, uplq);
+	queue_enter(&object->uplq, upl, upl_t, uplq);
 #endif /* UPL_DEBUG */
-	}
-
-	if ((cntrl_flags & UPL_WILL_MODIFY) &&
-	    object->copy != VM_OBJECT_NULL) {
-		/* Honor copy-on-write obligations */
 
+	if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != VM_OBJECT_NULL) {
 		/*
+		 * Honor copy-on-write obligations
+		 *
 		 * The caller is gathering these pages and
 		 * might modify their contents.  We need to
 		 * make sure that the copy object has its own
@@ -2585,292 +2692,279 @@ vm_object_upl_request(
 				 VM_PROT_NO_CHANGE);
 		upl_cow++;
 		upl_cow_pages += size >> PAGE_SHIFT;
-		
 	}
-	/* remember which copy object we synchronized with */
+	/*
+	 * remember which copy object we synchronized with
+	 */
 	last_copy_object = object->copy;
-
 	entry = 0;
-	if(cntrl_flags & UPL_COPYOUT_FROM) {
-		upl->flags |= UPL_PAGE_SYNC_DONE;
 
-		while (xfer_size) {
-			if((alias_page == NULL) && 
-				!(cntrl_flags & UPL_SET_LITE)) {
-				vm_object_unlock(object);
-				VM_PAGE_GRAB_FICTITIOUS(alias_page);
-				vm_object_lock(object);
+	xfer_size = size;
+	dst_offset = offset;
+
+	while (xfer_size) {
+
+		if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
+		        if (delayed_unlock) {
+			        delayed_unlock = 0;
+				vm_page_unlock_queues();
 			}
+			vm_object_unlock(object);
+			VM_PAGE_GRAB_FICTITIOUS(alias_page);
+			vm_object_lock(object);
+		}
+		if (delayed_unlock == 0)
+		        vm_page_lock_queues();
+
+		if (cntrl_flags & UPL_COPYOUT_FROM) {
+		        upl->flags |= UPL_PAGE_SYNC_DONE;
+
 			if ( ((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
 				dst_page->fictitious ||
 				dst_page->absent ||
 				dst_page->error ||
-			       (dst_page->wire_count && !dst_page->pageout) ||
-
-			     ((!dst_page->inactive) && (cntrl_flags & UPL_FOR_PAGEOUT) &&
-			       (dst_page->page_ticket != page_ticket) && 
-			      ((dst_page->page_ticket+1) != page_ticket)) ) {
+			       (dst_page->wire_count && !dst_page->pageout && !dst_page->list_req_pending)) {
 
 				if (user_page_list)
 					user_page_list[entry].phys_addr = 0;
-			} else { 
+
+				goto delay_unlock_queues;
+			}
+			/*
+			 * grab this up front...
+			 * a high percentange of the time we're going to
+			 * need the hardware modification state a bit later
+			 * anyway... so we can eliminate an extra call into
+			 * the pmap layer by grabbing it here and recording it
+			 */
+			if (dst_page->pmapped)
+			        refmod_state = pmap_get_refmod(dst_page->phys_page);
+			else
+			        refmod_state = 0;
+
+			if ( (refmod_state & VM_MEM_REFERENCED) && dst_page->inactive ) {
 			        /*
-				 * grab this up front...
-				 * a high percentange of the time we're going to
-				 * need the hardware modification state a bit later
-				 * anyway... so we can eliminate an extra call into
-				 * the pmap layer by grabbing it here and recording it
+				 * page is on inactive list and referenced...
+				 * reactivate it now... this gets it out of the
+				 * way of vm_pageout_scan which would have to
+				 * reactivate it upon tripping over it
 				 */
-			        refmod_state = pmap_get_refmod(dst_page->phys_page);
-					
-			        if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
+			        vm_page_activate(dst_page);
+				VM_STAT_INCR(reactivations);
+			}
+			if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
+			        /*
+				 * we're only asking for DIRTY pages to be returned
+				 */
+			        if (dst_page->list_req_pending || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
 				        /*
-					 * we're only asking for DIRTY pages to be returned
-					 */
-
-				        if (dst_page->list_req_pending || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
-					        /*
-						 * if we were the page stolen by vm_pageout_scan to be
-						 * cleaned (as opposed to a buddy being clustered in 
-						 * or this request is not being driven by a PAGEOUT cluster
-						 * then we only need to check for the page being diry or
-						 * precious to decide whether to return it
-						 */
-					        if (dst_page->dirty || dst_page->precious ||
-						    (refmod_state & VM_MEM_MODIFIED)) {
-						        goto check_busy;
-						}
-					}
-					/*
-					 * this is a request for a PAGEOUT cluster and this page
-					 * is merely along for the ride as a 'buddy'... not only
-					 * does it have to be dirty to be returned, but it also
-					 * can't have been referenced recently... note that we've
-					 * already filtered above based on whether this page is
-					 * currently on the inactive queue or it meets the page
-					 * ticket (generation count) check
+					 * if we were the page stolen by vm_pageout_scan to be
+					 * cleaned (as opposed to a buddy being clustered in 
+					 * or this request is not being driven by a PAGEOUT cluster
+					 * then we only need to check for the page being dirty or
+					 * precious to decide whether to return it
 					 */
-					if ( !(refmod_state & VM_MEM_REFERENCED) && 
-					     ((refmod_state & VM_MEM_MODIFIED) ||
-					      dst_page->dirty || dst_page->precious) ) {
+				        if (dst_page->dirty || dst_page->precious || (refmod_state & VM_MEM_MODIFIED))
 					        goto check_busy;
-					}
-					/*
-					 * if we reach here, we're not to return
-					 * the page... go on to the next one
-					 */
-					if (user_page_list)
-					        user_page_list[entry].phys_addr = 0;
-					entry++;
-					dst_offset += PAGE_SIZE_64;
-					xfer_size -= PAGE_SIZE;
-					continue;
-				}
-check_busy:			
-				if(dst_page->busy && 
-					(!(dst_page->list_req_pending && 
-						dst_page->pageout))) {
-					if(cntrl_flags & UPL_NOBLOCK) {
-						if(user_page_list) {
-					   		user_page_list[entry].phys_addr = 0;
-						}
-						entry++;
-						dst_offset += PAGE_SIZE_64;
-						xfer_size -= PAGE_SIZE;
-						continue;
-					}
-					/*
-					 * someone else is playing with the
-					 * page.  We will have to wait.
-					 */
-					PAGE_SLEEP(object, dst_page, THREAD_UNINT);
-					continue;
+					goto dont_return;
 				}
-				/* Someone else already cleaning the page? */
-				if((dst_page->cleaning || dst_page->absent ||
-					dst_page->wire_count != 0) && 
-					!dst_page->list_req_pending) {
-				   if(user_page_list) {
-					   user_page_list[entry].phys_addr = 0;
-				   }
-				   entry++;
-				   dst_offset += PAGE_SIZE_64;
-				   xfer_size -= PAGE_SIZE;
-				   continue;
+				/*
+				 * this is a request for a PAGEOUT cluster and this page
+				 * is merely along for the ride as a 'buddy'... not only
+				 * does it have to be dirty to be returned, but it also
+				 * can't have been referenced recently... note that we've
+				 * already filtered above based on whether this page is
+				 * currently on the inactive queue or it meets the page
+				 * ticket (generation count) check
+				 */
+				if ( !(refmod_state & VM_MEM_REFERENCED) && 
+				     ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
+				        goto check_busy;
 				}
-				/* eliminate all mappings from the */
-				/* original object and its prodigy */
-				
-				vm_page_lock_queues();
+dont_return:
+				/*
+				 * if we reach here, we're not to return
+				 * the page... go on to the next one
+				 */
+				if (user_page_list)
+				        user_page_list[entry].phys_addr = 0;
 
-				if (dst_page->pageout_queue == TRUE)
-				        /*
-					 * we've buddied up a page for a clustered pageout
-					 * that has already been moved to the pageout
-					 * queue by pageout_scan... we need to remove
-					 * it from the queue and drop the laundry count
-					 * on that queue
-					 */
-				        vm_pageout_queue_steal(dst_page);
-#if MACH_CLUSTER_STATS
-				/* pageout statistics gathering.  count  */
-				/* all the pages we will page out that   */
-				/* were not counted in the initial       */
-				/* vm_pageout_scan work                  */
-				if(dst_page->list_req_pending)
-					encountered_lrp = TRUE;
-				if((dst_page->dirty ||
-					(dst_page->object->internal &&
-					dst_page->precious)) &&
-					(dst_page->list_req_pending 
-					== FALSE)) {
-					if(encountered_lrp) {
-						CLUSTER_STAT
-						(pages_at_higher_offsets++;)
-					} else {
-						CLUSTER_STAT
-						(pages_at_lower_offsets++;)
-					}
-				}
-#endif
-				/* Turn off busy indication on pending */
-				/* pageout.  Note: we can only get here */
-				/* in the request pending case.  */
-				dst_page->list_req_pending = FALSE;
-				dst_page->busy = FALSE;
-				dst_page->cleaning = FALSE;
-
-			        hw_dirty = refmod_state & VM_MEM_MODIFIED;
-				dirty = hw_dirty ? TRUE : dst_page->dirty;
-
-				if(cntrl_flags & UPL_SET_LITE) {
-					int	pg_num;
-					pg_num = (dst_offset-offset)/PAGE_SIZE;
-					lite_list[pg_num>>5] |= 
-							1 << (pg_num & 31);
-					if (hw_dirty)
-					        pmap_clear_modify(dst_page->phys_page);
-					/*
-					 * Record that this page has been 
-					 * written out
-					 */
-#if     MACH_PAGEMAP
-					vm_external_state_set(
-						object->existence_map, 
-						dst_page->offset);
-#endif  /*MACH_PAGEMAP*/
+				goto delay_unlock_queues;
+			}
+check_busy:			
+			if (dst_page->busy && (!(dst_page->list_req_pending && dst_page->pageout))) {
+			        if (cntrl_flags & UPL_NOBLOCK) {
+				        if (user_page_list)
+					        user_page_list[entry].phys_addr = 0;
 
-					/*
-					 * Mark original page as cleaning 
-					 * in place.
-					 */
-					dst_page->cleaning = TRUE;
-					dst_page->dirty = TRUE;
-					dst_page->precious = FALSE;
-				} else {
-					/* use pageclean setup, it is more */
-					/* convenient even for the pageout */
-					/* cases here */
-
-				        vm_object_lock(upl->map_object);
-					vm_pageclean_setup(dst_page, 
-						alias_page, upl->map_object, 
-						size - xfer_size);
-				        vm_object_unlock(upl->map_object);
-
-					alias_page->absent = FALSE;
-					alias_page = NULL;
-				}
-						
-				if(!dirty) {
-					dst_page->dirty = FALSE;
-					dst_page->precious = TRUE;
+					goto delay_unlock_queues;
 				}
+				/*
+				 * someone else is playing with the
+				 * page.  We will have to wait.
+				 */
+				delayed_unlock = 0;
+				vm_page_unlock_queues();
 
-				if(dst_page->pageout)
-					dst_page->busy = TRUE;
+				PAGE_SLEEP(object, dst_page, THREAD_UNINT);
 
-				if ( (cntrl_flags & UPL_ENCRYPT) ) {
-				        /*
-					 * ENCRYPTED SWAP:
-					 * We want to deny access to the target page
-					 * because its contents are about to be
-					 * encrypted and the user would be very
-					 * confused to see encrypted data instead
-					 * of their data.
-					 */
-					dst_page->busy = TRUE;
-				}
-				if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
-					/*
-					 * deny access to the target page
-					 * while it is being worked on
-					 */
-					if ((!dst_page->pageout) &&
-					    (dst_page->wire_count == 0)) {
-						dst_page->busy = TRUE;
-						dst_page->pageout = TRUE;
-						vm_page_wire(dst_page);
-					}
-				}
+				continue;
+			}
+			/*
+			 * Someone else already cleaning the page?
+			 */
+			if ((dst_page->cleaning || dst_page->absent || dst_page->wire_count != 0) && !dst_page->list_req_pending) {
+			        if (user_page_list)
+				        user_page_list[entry].phys_addr = 0;
 
-				if (dst_page->phys_page > upl->highest_page)
-					upl->highest_page = dst_page->phys_page;
-
-				if(user_page_list) {
-					user_page_list[entry].phys_addr
-						= dst_page->phys_page;
-					user_page_list[entry].dirty =   
-							dst_page->dirty;
-					user_page_list[entry].pageout =
-							dst_page->pageout;
-					user_page_list[entry].absent =
-							dst_page->absent;
-					user_page_list[entry].precious =
-							dst_page->precious;
-				}
-				vm_page_unlock_queues();
+				goto delay_unlock_queues;
+			}
+			/*
+			 * ENCRYPTED SWAP:
+			 * The caller is gathering this page and might
+			 * access its contents later on.  Decrypt the
+			 * page before adding it to the UPL, so that
+			 * the caller never sees encrypted data.
+			 */
+			if (! (cntrl_flags & UPL_ENCRYPT) && dst_page->encrypted) {
+			        int  was_busy;
 
+				delayed_unlock = 0;
+				vm_page_unlock_queues();
 				/*
-				 * ENCRYPTED SWAP:
-				 * The caller is gathering this page and might
-				 * access its contents later on.  Decrypt the
-				 * page before adding it to the UPL, so that
-				 * the caller never sees encrypted data.
+				 * save the current state of busy
+				 * mark page as busy while decrypt
+				 * is in progress since it will drop
+				 * the object lock...
 				 */
-				if (! (cntrl_flags & UPL_ENCRYPT) &&
-				    dst_page->encrypted) {
-					assert(dst_page->busy);
+				was_busy = dst_page->busy;
+				dst_page->busy = TRUE;
 
-					vm_page_decrypt(dst_page, 0);
-					vm_page_decrypt_for_upl_counter++;
+				vm_page_decrypt(dst_page, 0);
+				vm_page_decrypt_for_upl_counter++;
+				/*
+				 * restore to original busy state
+				 */
+				dst_page->busy = was_busy;
 
-					/*
-					 * Retry this page, since anything
-					 * could have changed while we were
-					 * decrypting.
-					 */
-					continue;
-				}
+				vm_page_lock_queues();
 			}
-			entry++;
-			dst_offset += PAGE_SIZE_64;
-			xfer_size -= PAGE_SIZE;
-		}
-	} else {
-		while (xfer_size) {
-			if((alias_page == NULL) && 
-				!(cntrl_flags & UPL_SET_LITE)) {
-				vm_object_unlock(object);
-				VM_PAGE_GRAB_FICTITIOUS(alias_page);
-				vm_object_lock(object);
+			if (dst_page->pageout_queue == TRUE)
+			        /*
+				 * we've buddied up a page for a clustered pageout
+				 * that has already been moved to the pageout
+				 * queue by pageout_scan... we need to remove
+				 * it from the queue and drop the laundry count
+				 * on that queue
+				 */
+			        vm_pageout_queue_steal(dst_page);
+#if MACH_CLUSTER_STATS
+			/*
+			 * pageout statistics gathering.  count
+			 * all the pages we will page out that
+			 * were not counted in the initial
+			 * vm_pageout_scan work
+			 */
+			if (dst_page->list_req_pending)
+			        encountered_lrp = TRUE;
+			if ((dst_page->dirty ||	(dst_page->object->internal && dst_page->precious)) && !dst_page->list_req_pending) {
+			        if (encountered_lrp)
+				        CLUSTER_STAT(pages_at_higher_offsets++;)
+				else
+				        CLUSTER_STAT(pages_at_lower_offsets++;)
+			}
+#endif
+			/*
+			 * Turn off busy indication on pending
+			 * pageout.  Note: we can only get here
+			 * in the request pending case.
+			 */
+			dst_page->list_req_pending = FALSE;
+			dst_page->busy = FALSE;
+
+			hw_dirty = refmod_state & VM_MEM_MODIFIED;
+			dirty = hw_dirty ? TRUE : dst_page->dirty;
+
+			if (dst_page->phys_page > upl->highest_page)
+			        upl->highest_page = dst_page->phys_page;
+
+			if (cntrl_flags & UPL_SET_LITE) {
+			        int	pg_num;
+
+				pg_num = (dst_offset-offset)/PAGE_SIZE;
+				lite_list[pg_num>>5] |= 1 << (pg_num & 31);
+
+				if (hw_dirty)
+				        pmap_clear_modify(dst_page->phys_page);
+
+				/*
+				 * Mark original page as cleaning 
+				 * in place.
+				 */
+				dst_page->cleaning = TRUE;
+				dst_page->precious = FALSE;
+			} else {
+			        /*
+				 * use pageclean setup, it is more
+				 * convenient even for the pageout
+				 * cases here
+				 */
+			        vm_object_lock(upl->map_object);
+				vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+				vm_object_unlock(upl->map_object);
+
+				alias_page->absent = FALSE;
+				alias_page = NULL;
 			}
+#if     MACH_PAGEMAP
+			/*
+			 * Record that this page has been 
+			 * written out
+			 */
+			vm_external_state_set(object->existence_map, dst_page->offset);
+#endif  /*MACH_PAGEMAP*/
+			dst_page->dirty = dirty;
 
-			if ((cntrl_flags & UPL_WILL_MODIFY) &&
-			    object->copy != last_copy_object) {
-				/* Honor copy-on-write obligations */
+			if (!dirty)
+				dst_page->precious = TRUE;
 
+			if (dst_page->pageout)
+			        dst_page->busy = TRUE;
+
+			if ( (cntrl_flags & UPL_ENCRYPT) ) {
+			        /*
+				 * ENCRYPTED SWAP:
+				 * We want to deny access to the target page
+				 * because its contents are about to be
+				 * encrypted and the user would be very
+				 * confused to see encrypted data instead
+				 * of their data.
+				 * We also set "encrypted_cleaning" to allow
+				 * vm_pageout_scan() to demote that page
+				 * from "adjacent/clean-in-place" to
+				 * "target/clean-and-free" if it bumps into
+				 * this page during its scanning while we're
+				 * still processing this cluster.
+				 */
+			        dst_page->busy = TRUE;
+				dst_page->encrypted_cleaning = TRUE;
+			}
+			if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
+			        /*
+				 * deny access to the target page
+				 * while it is being worked on
+				 */
+			        if ((!dst_page->pageout) && (dst_page->wire_count == 0)) {
+				        dst_page->busy = TRUE;
+					dst_page->pageout = TRUE;
+					vm_page_wire(dst_page);
+				}
+			}
+		} else {
+			if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
 				/*
+				 * Honor copy-on-write obligations
+				 *
 				 * The copy object has changed since we
 				 * last synchronized for copy-on-write.
 				 * Another copy object might have been
@@ -2890,6 +2984,9 @@ check_busy:
 				 * to see both the *before* and *after* pages.
 				 */
 				if (object->copy != VM_OBJECT_NULL) {
+				        delayed_unlock = 0;
+					vm_page_unlock_queues();
+
 					vm_object_update(
 						object,
 						dst_offset,/* current offset */
@@ -2899,113 +2996,128 @@ check_busy:
 						FALSE,	   /* should_return */
 						MEMORY_OBJECT_COPY_SYNC,
 						VM_PROT_NO_CHANGE);
+
 					upl_cow_again++;
-					upl_cow_again_pages +=
-						xfer_size >> PAGE_SHIFT;
+					upl_cow_again_pages += xfer_size >> PAGE_SHIFT;
+
+					vm_page_lock_queues();
 				}
-				/* remember the copy object we synced with */
+				/*
+				 * remember the copy object we synced with
+				 */
 				last_copy_object = object->copy;
 			}
-
 			dst_page = vm_page_lookup(object, dst_offset);
 			
-			if(dst_page != VM_PAGE_NULL) {
-			        if((cntrl_flags & UPL_RET_ONLY_ABSENT) &&
-				        !((dst_page->list_req_pending)
-					        && (dst_page->absent))) {
-				        /* we are doing extended range */
-				        /* requests.  we want to grab  */
-				        /* pages around some which are */
-				        /* already present.  */
-				        if(user_page_list) {
-					        user_page_list[entry].phys_addr = 0;
+			if (dst_page != VM_PAGE_NULL) {
+			        if ( !(dst_page->list_req_pending) ) {
+				        if ((cntrl_flags & UPL_RET_ONLY_ABSENT) && !dst_page->absent) {
+					        /*
+						 * skip over pages already present in the cache
+						 */
+					        if (user_page_list)
+						        user_page_list[entry].phys_addr = 0;
+
+						goto delay_unlock_queues;
 					}
-					entry++;
-					dst_offset += PAGE_SIZE_64;
-					xfer_size -= PAGE_SIZE;
-					continue;
-				}
-				if((dst_page->cleaning) && 
-				   !(dst_page->list_req_pending)) {
-					/*someone else is writing to the */
-					/* page.  We will have to wait.  */
-					PAGE_SLEEP(object,dst_page,THREAD_UNINT);
-					continue;
-				}
-				if ((dst_page->fictitious && 
-				     dst_page->list_req_pending)) {
-					/* dump the fictitious page */
-					dst_page->list_req_pending = FALSE;
-					dst_page->clustered = FALSE;
+					if (dst_page->cleaning) {
+					        /*
+						 * someone else is writing to the page... wait...
+						 */
+					        delayed_unlock = 0;
+						vm_page_unlock_queues();
 
-					vm_page_lock_queues();
-					vm_page_free(dst_page);
-					vm_page_unlock_queues();
+					        PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+						continue;
+					}
+				} else {
+				        if (dst_page->fictitious &&
+					    dst_page->phys_page == vm_page_fictitious_addr) {
+					        assert( !dst_page->speculative);
+					        /*
+						 * dump the fictitious page
+						 */
+					        dst_page->list_req_pending = FALSE;
 
-					dst_page = NULL;
-				} else if ((dst_page->absent && 
-					    dst_page->list_req_pending)) {
-					/* the default_pager case */
-					dst_page->list_req_pending = FALSE;
-					dst_page->busy = FALSE;
+						vm_page_free(dst_page);
+
+						dst_page = NULL;
+					} else if (dst_page->absent) {
+					        /*
+						 * the default_pager case
+						 */
+					        dst_page->list_req_pending = FALSE;
+						dst_page->busy = FALSE;
+					}
 				}
 			}
-			if(dst_page == VM_PAGE_NULL) {
-				if(object->private) {
+			if (dst_page == VM_PAGE_NULL) {
+				if (object->private) {
 					/* 
 					 * This is a nasty wrinkle for users 
 					 * of upl who encounter device or 
 					 * private memory however, it is 
 					 * unavoidable, only a fault can
-					 * reslove the actual backing
+					 * resolve the actual backing
 					 * physical page by asking the
 					 * backing device.
 					 */
-					if(user_page_list) {
+					if (user_page_list)
 						user_page_list[entry].phys_addr = 0;
-					}
-					entry++;
-					dst_offset += PAGE_SIZE_64;
-					xfer_size -= PAGE_SIZE;
-					continue;
+
+					goto delay_unlock_queues;
 				}
-				/* need to allocate a page */
+				/*
+				 * need to allocate a page
+				 * vm_page_alloc may grab the
+				 * queues lock for a purgeable object
+				 * so drop it
+				 */
+				delayed_unlock = 0;
+				vm_page_unlock_queues();
+
 		 		dst_page = vm_page_alloc(object, dst_offset);
+
 				if (dst_page == VM_PAGE_NULL) {
+				        if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
+					       /*
+						* we don't want to stall waiting for pages to come onto the free list
+						* while we're already holding absent pages in this UPL
+						* the caller will deal with the empty slots
+						*/
+					        if (user_page_list)
+						        user_page_list[entry].phys_addr = 0;
+
+						goto try_next_page;
+					}
+				        /*
+					 * no pages available... wait
+					 * then try again for the same
+					 * offset...
+					 */
 					vm_object_unlock(object);
 					VM_PAGE_WAIT();
 					vm_object_lock(object);
+
 					continue;
 				}
 				dst_page->busy = FALSE;
-#if 0
-				if(cntrl_flags & UPL_NO_SYNC) {
-					dst_page->page_lock = 0;
-					dst_page->unlock_request = 0;
-				}
-#endif
-				if(cntrl_flags & UPL_RET_ONLY_ABSENT) {
+				dst_page->absent = TRUE;
+
+				if (cntrl_flags & UPL_RET_ONLY_ABSENT) {
 				        /*
 					 * if UPL_RET_ONLY_ABSENT was specified,
 					 * than we're definitely setting up a
 					 * upl for a clustered read/pagein 
 					 * operation... mark the pages as clustered
-					 * so vm_fault can correctly attribute them
-					 * to the 'pagein' bucket the first time
-					 * a fault happens on them
+					 * so upl_commit_range can put them on the
+					 * speculative list
 					 */
 				        dst_page->clustered = TRUE;
 				}
-				dst_page->absent = TRUE;
-				object->absent_count++;
-			}
-#if 1
-			if(cntrl_flags & UPL_NO_SYNC) {
-				dst_page->page_lock = 0;
-				dst_page->unlock_request = 0;
+				vm_page_lock_queues();
 			}
-#endif /* 1 */
-
 			/*
 			 * ENCRYPTED SWAP:
 			 */
@@ -3022,202 +3134,166 @@ check_busy:
 				 */
 				dst_page->encrypted = FALSE;
 			}
-
 			dst_page->overwriting = TRUE;
-			if(dst_page->fictitious) {
+
+			if (dst_page->fictitious) {
 				panic("need corner case for fictitious page");
 			}
-			if(dst_page->page_lock) {
-				do_m_lock = TRUE;
-			}
-			if(upl_ptr) {
+			if (dst_page->busy) {
+				/*
+				 * someone else is playing with the
+				 * page.  We will have to wait.
+				 */
+			        delayed_unlock = 0;
+				vm_page_unlock_queues();
 
-				/* eliminate all mappings from the */
-				/* original object and its prodigy */
-				
-				if(dst_page->busy) {
-					/*someone else is playing with the */
-					/* page.  We will have to wait.    */
-					PAGE_SLEEP(object, dst_page, THREAD_UNINT);
-					continue;
-				}
-				vm_page_lock_queues();
+				PAGE_SLEEP(object, dst_page, THREAD_UNINT);
 
-				if( !(cntrl_flags & UPL_FILE_IO))
-				        hw_dirty = pmap_disconnect(dst_page->phys_page) & VM_MEM_MODIFIED;
-				else
-				        hw_dirty = pmap_get_refmod(dst_page->phys_page) & VM_MEM_MODIFIED;
-				dirty = hw_dirty ? TRUE : dst_page->dirty;
-
-				if(cntrl_flags & UPL_SET_LITE) {
-					int	pg_num;
-					pg_num = (dst_offset-offset)/PAGE_SIZE;
-					lite_list[pg_num>>5] |= 
-							1 << (pg_num & 31);
-					if (hw_dirty)
-					        pmap_clear_modify(dst_page->phys_page);
-					/*
-					 * Record that this page has been 
-					 * written out
+				continue;
+			}
+			if (dst_page->pmapped) {
+			        if ( !(cntrl_flags & UPL_FILE_IO))
+				        /*
+					 * eliminate all mappings from the
+					 * original object and its prodigy
 					 */
-#if     MACH_PAGEMAP
-					vm_external_state_set(
-						object->existence_map, 
-						dst_page->offset);
-#endif  /*MACH_PAGEMAP*/
+				        refmod_state = pmap_disconnect(dst_page->phys_page);
+				else
+				        refmod_state = pmap_get_refmod(dst_page->phys_page);
+			} else
+			        refmod_state = 0;
 
-					/*
-					 * Mark original page as cleaning 
-					 * in place.
-					 */
-					dst_page->cleaning = TRUE;
-					dst_page->dirty = TRUE;
-					dst_page->precious = FALSE;
-				} else {
-					/* use pageclean setup, it is more */
-					/* convenient even for the pageout */
-					/* cases here */
-				        vm_object_lock(upl->map_object);
-					vm_pageclean_setup(dst_page, 
-						alias_page, upl->map_object, 
-						size - xfer_size);
-				        vm_object_unlock(upl->map_object);
-
-					alias_page->absent = FALSE;
-					alias_page = NULL;
-				}
+			hw_dirty = refmod_state & VM_MEM_MODIFIED;
+			dirty = hw_dirty ? TRUE : dst_page->dirty;
 
-				if(cntrl_flags & UPL_CLEAN_IN_PLACE) {
-					/* clean in place for read implies   */
-					/* that a write will be done on all  */
-					/* the pages that are dirty before   */
-					/* a upl commit is done.  The caller */
-					/* is obligated to preserve the      */
-					/* contents of all pages marked      */
-					/* dirty. */
-					upl->flags |= UPL_CLEAR_DIRTY;
-				}
+			if (cntrl_flags & UPL_SET_LITE) {
+				int	pg_num;
 
-				if(!dirty) {
-					dst_page->dirty = FALSE;
-					dst_page->precious = TRUE;
-				}
-						
-				if (dst_page->wire_count == 0) {
-				   /* deny access to the target page while */
-				   /* it is being worked on */
-					dst_page->busy = TRUE;
-				} else {
-			 		vm_page_wire(dst_page);
-				}
-				if(cntrl_flags & UPL_RET_ONLY_ABSENT) {
-				        /*
-					 * expect the page not to be used
-					 * since it's coming in as part
-					 * of a cluster and could be 
-					 * speculative... pages that
-					 * are 'consumed' will get a
-					 * hardware reference
-					 */
-				        dst_page->reference = FALSE;
-				} else {
-				        /*
-					 * expect the page to be used
-					 */
-				        dst_page->reference = TRUE;
-				}
-				dst_page->precious = 
-					(cntrl_flags & UPL_PRECIOUS) 
-							? TRUE : FALSE;
-
-				if (dst_page->phys_page > upl->highest_page)
-					upl->highest_page = dst_page->phys_page;
-
-				if(user_page_list) {
-					user_page_list[entry].phys_addr
-						= dst_page->phys_page;
-					user_page_list[entry].dirty =
-							dst_page->dirty;
-					user_page_list[entry].pageout =
-				   			dst_page->pageout;
-					user_page_list[entry].absent =
-				   			dst_page->absent;
-					user_page_list[entry].precious =
-							dst_page->precious;
-				}
-				vm_page_unlock_queues();
-			}
-			entry++;
-			dst_offset += PAGE_SIZE_64;
-			xfer_size -= PAGE_SIZE;
-		}
-	}
+				pg_num = (dst_offset-offset)/PAGE_SIZE;
+				lite_list[pg_num>>5] |= 1 << (pg_num & 31);
 
-	if (upl->flags & UPL_INTERNAL) {
-		if(page_list_count != NULL)
-			*page_list_count = 0;
-	} else if (*page_list_count > entry) {
-		if(page_list_count != NULL)
-			*page_list_count = entry;
-	}
+				if (hw_dirty)
+				        pmap_clear_modify(dst_page->phys_page);
 
-	if(alias_page != NULL) {
-		vm_page_lock_queues();
-		vm_page_free(alias_page);
-		vm_page_unlock_queues();
-	}
+				/*
+				 * Mark original page as cleaning 
+				 * in place.
+				 */
+				dst_page->cleaning = TRUE;
+				dst_page->precious = FALSE;
+			} else {
+				/*
+				 * use pageclean setup, it is more
+				 * convenient even for the pageout
+				 * cases here
+				 */
+			        vm_object_lock(upl->map_object);
+				vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+			        vm_object_unlock(upl->map_object);
 
-	if(do_m_lock) {
-	   vm_prot_t	access_required;
-	   /* call back all associated pages from other users of the pager */
-	   /* all future updates will be on data which is based on the     */
-	   /* changes we are going to make here. Note: it is assumed that  */
-	   /* we already hold copies of the data so we will not be seeing  */
-	   /* an avalanche of incoming data from the pager */
-	   access_required = (cntrl_flags & UPL_COPYOUT_FROM) 
-					? VM_PROT_READ : VM_PROT_WRITE;
-	   while (TRUE) {
-		kern_return_t	rc;
-
-		if(!object->pager_ready) {
-		   wait_result_t wait_result;
-
-		   wait_result = vm_object_sleep(object, 
-						VM_OBJECT_EVENT_PAGER_READY,
-						THREAD_UNINT);
-		   if (wait_result !=  THREAD_AWAKENED) {
-		   	vm_object_unlock(object);
-		   	return KERN_FAILURE;
-		   }
-		   continue;
-		}
+				alias_page->absent = FALSE;
+				alias_page = NULL;
+			}
 
-		vm_object_unlock(object);
-		rc = memory_object_data_unlock(
-			object->pager,
-			dst_offset + object->paging_offset,
-			size,
-			access_required);
-		if (rc != KERN_SUCCESS && rc != MACH_SEND_INTERRUPTED)
-			return KERN_FAILURE;
-		vm_object_lock(object);
+			if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
+				/*
+				 * clean in place for read implies
+				 * that a write will be done on all
+				 * the pages that are dirty before
+				 * a upl commit is done.  The caller
+				 * is obligated to preserve the
+				 * contents of all pages marked dirty
+				 */
+				upl->flags |= UPL_CLEAR_DIRTY;
+			}
+			dst_page->dirty = dirty;
 
-		if (rc == KERN_SUCCESS)
-			break;
-	   }
-
-	   /* lets wait on the last page requested */
-	   /* NOTE: we will have to update lock completed routine to signal */
-	   if(dst_page != VM_PAGE_NULL && 
-		(access_required & dst_page->page_lock) != access_required) {
-	   	PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT);
-	   	vm_object_unlock(object);
-	   	thread_block(THREAD_CONTINUE_NULL);
-		return KERN_SUCCESS;
-	   }
+			if (!dirty)
+				dst_page->precious = TRUE;
+
+			if (dst_page->wire_count == 0) {
+			        /*
+				 * deny access to the target page while
+				 * it is being worked on
+				 */
+				dst_page->busy = TRUE;
+			} else
+		 		vm_page_wire(dst_page);
+
+			if (dst_page->clustered) {
+			        /*
+				 * expect the page not to be used
+				 * since it's coming in as part
+				 * of a speculative cluster... 
+				 * pages that are 'consumed' will
+				 * get a hardware reference
+				 */
+			        dst_page->reference = FALSE;
+			} else {
+			        /*
+				 * expect the page to be used
+				 */
+			        dst_page->reference = TRUE;
+			}
+			dst_page->precious = (cntrl_flags & UPL_PRECIOUS) ? TRUE : FALSE;
+		}
+		if (dst_page->phys_page > upl->highest_page)
+		        upl->highest_page = dst_page->phys_page;
+		if (user_page_list) {
+			user_page_list[entry].phys_addr = dst_page->phys_page;
+			user_page_list[entry].dirty	= dst_page->dirty;
+			user_page_list[entry].pageout	= dst_page->pageout;
+			user_page_list[entry].absent	= dst_page->absent;
+			user_page_list[entry].precious	= dst_page->precious;
+
+			if (dst_page->clustered == TRUE)
+			        user_page_list[entry].speculative = dst_page->speculative;
+			else
+			        user_page_list[entry].speculative = FALSE;
+		}
+	        /*
+		 * if UPL_RET_ONLY_ABSENT is set, then
+		 * we are working with a fresh page and we've
+		 * just set the clustered flag on it to
+		 * indicate that it was drug in as part of a
+		 * speculative cluster... so leave it alone
+		 */
+		if ( !(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+		        /*
+			 * someone is explicitly grabbing this page...
+			 * update clustered and speculative state
+			 * 
+			 */
+		        VM_PAGE_CONSUME_CLUSTERED(dst_page);
+		}
+delay_unlock_queues:
+		if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
+			mutex_yield(&vm_page_queue_lock);
+		        delayed_unlock = 1;
+		}
+try_next_page:
+		entry++;
+		dst_offset += PAGE_SIZE_64;
+		xfer_size -= PAGE_SIZE;
+	}
+	if (alias_page != NULL) {
+	        if (delayed_unlock == 0) {
+		        vm_page_lock_queues();
+			delayed_unlock++;
+		}
+		vm_page_free(alias_page);
 	}
+	if (delayed_unlock)
+	        vm_page_unlock_queues();
 
+	if (page_list_count != NULL) {
+	        if (upl->flags & UPL_INTERNAL)
+			*page_list_count = 0;
+		else if (*page_list_count > entry)
+			*page_list_count = entry;
+	}
 	vm_object_unlock(object);
+
 	return KERN_SUCCESS;
 }
 
@@ -3229,7 +3305,7 @@ vm_fault_list_request(			/* forward */
 	upl_size_t		size,
 	upl_t			*upl_ptr,
 	upl_page_info_t		**user_page_list_ptr,
-	int			page_list_count,
+	unsigned int		page_list_count,
 	int			cntrl_flags);
 kern_return_t
 vm_fault_list_request(
@@ -3238,7 +3314,7 @@ vm_fault_list_request(
 	upl_size_t		size,
 	upl_t			*upl_ptr,
 	upl_page_info_t		**user_page_list_ptr,
-	int			page_list_count,
+	unsigned int		page_list_count,
 	int			cntrl_flags)
 {
 	unsigned int		local_list_count;
@@ -3294,61 +3370,39 @@ vm_object_super_upl_request(
 	unsigned int		*page_list_count,
 	int			cntrl_flags)
 {
-	vm_page_t	target_page;
-	int		ticket;
-
-
-	if(object->paging_offset > offset)
+	if (object->paging_offset > offset)
 		return KERN_FAILURE;
 
 	assert(object->paging_in_progress);
 	offset = offset - object->paging_offset;
 
-	if(cntrl_flags & UPL_FOR_PAGEOUT) {
-	  
-	        vm_object_lock(object);
-
-		if((target_page = vm_page_lookup(object, offset))
-							!= VM_PAGE_NULL) {
-			ticket = target_page->page_ticket;
-			cntrl_flags = cntrl_flags & ~(int)UPL_PAGE_TICKET_MASK;
-			cntrl_flags = cntrl_flags | 
-				((ticket << UPL_PAGE_TICKET_SHIFT) 
-							& UPL_PAGE_TICKET_MASK);
-		}
-	        vm_object_unlock(object);
-	}
-
 	if (super_cluster > size) {
 
 		vm_object_offset_t	base_offset;
 		upl_size_t		super_size;
 
-		base_offset = (offset &  
-			~((vm_object_offset_t) super_cluster - 1));
-		super_size = (offset+size) > (base_offset + super_cluster) ?
-				super_cluster<<1 : super_cluster;
-		super_size = ((base_offset + super_size) > object->size) ? 
-				(object->size - base_offset) : super_size;
-		if(offset > (base_offset + super_size))
-		   panic("vm_object_super_upl_request: Missed target pageout"
-			 " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
-			 offset, base_offset, super_size, super_cluster,
-			 size, object->paging_offset);
+		base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
+		super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster;
+		super_size = ((base_offset + super_size) > object->size) ? (object->size - base_offset) : super_size;
+
+		if (offset > (base_offset + super_size)) {
+		        panic("vm_object_super_upl_request: Missed target pageout"
+			      " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
+			      offset, base_offset, super_size, super_cluster,
+			      size, object->paging_offset);
+		}
 		/*
 		 * apparently there is a case where the vm requests a
 		 * page to be written out who's offset is beyond the
 		 * object size
 		 */
-		if((offset + size) > (base_offset + super_size))
-		   super_size = (offset + size) - base_offset;
+		if ((offset + size) > (base_offset + super_size))
+		        super_size = (offset + size) - base_offset;
 
 		offset = base_offset;
 		size = super_size;
 	}
-	return vm_object_upl_request(object, offset, size,
-				     upl, user_page_list, page_list_count,
-				     cntrl_flags);
+	return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags);
 }
 
 				 
@@ -3380,46 +3434,43 @@ vm_map_create_upl(
 		 */
 		return KERN_INVALID_VALUE;
 	}
-
 	force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
 	sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
 
-	if(upl == NULL)
+	if (upl == NULL)
 		return KERN_INVALID_ARGUMENT;
 
-
 REDISCOVER_ENTRY:
 	vm_map_lock(map);
+
 	if (vm_map_lookup_entry(map, offset, &entry)) {
-		if (entry->object.vm_object == VM_OBJECT_NULL ||
-			!entry->object.vm_object->phys_contiguous) {
-        		if((*upl_size/page_size) > MAX_UPL_TRANSFER) {
-               			*upl_size = MAX_UPL_TRANSFER * page_size;
-			}
-		}
-		if((entry->vme_end - offset) < *upl_size) {
+
+		if ((entry->vme_end - offset) < *upl_size)
 			*upl_size = entry->vme_end - offset;
-		}
+
 		if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
-			if (entry->object.vm_object == VM_OBJECT_NULL) {
-				*flags = 0;
-			} else if (entry->object.vm_object->private) {
-				*flags = UPL_DEV_MEMORY;
-				if (entry->object.vm_object->phys_contiguous) {
+		        *flags = 0;
+
+			if (entry->object.vm_object != VM_OBJECT_NULL) {
+			        if (entry->object.vm_object->private)
+				        *flags = UPL_DEV_MEMORY;
+
+				if (entry->object.vm_object->phys_contiguous)
 					*flags |= UPL_PHYS_CONTIG;
-				}
-			} else  {
-				*flags = 0;
 			}
 			vm_map_unlock(map);
+
 			return KERN_SUCCESS;
 		}
+	        if (entry->object.vm_object == VM_OBJECT_NULL || !entry->object.vm_object->phys_contiguous) {
+        		if ((*upl_size/page_size) > MAX_UPL_TRANSFER)
+               			*upl_size = MAX_UPL_TRANSFER * page_size;
+		}
 		/*
 		 *      Create an object if necessary.
 		 */
 		if (entry->object.vm_object == VM_OBJECT_NULL) {
-			entry->object.vm_object = vm_object_allocate(
-				(vm_size_t)(entry->vme_end - entry->vme_start));
+			entry->object.vm_object = vm_object_allocate((vm_size_t)(entry->vme_end - entry->vme_start));
 			entry->offset = 0;
 		}
 		if (!(caller_flags & UPL_COPYOUT_FROM)) {
@@ -3430,29 +3481,27 @@ REDISCOVER_ENTRY:
 			if (entry->needs_copy)  {
 				vm_map_t		local_map;
 				vm_object_t		object;
-				vm_map_offset_t		offset_hi;
-				vm_map_offset_t		offset_lo;
 				vm_object_offset_t	new_offset;
 				vm_prot_t		prot;
 				boolean_t		wired;
-				vm_behavior_t		behavior;
 				vm_map_version_t	version;
 				vm_map_t		real_map;
 
 				local_map = map;
 				vm_map_lock_write_to_read(map);
-				if(vm_map_lookup_locked(&local_map,
-					offset, VM_PROT_WRITE,
-					&version, &object,
-					&new_offset, &prot, &wired,
-					&behavior, &offset_lo,
-					&offset_hi, &real_map)) {
-					vm_map_unlock(local_map);
+
+				if (vm_map_lookup_locked(&local_map,
+							 offset, VM_PROT_WRITE,
+							 OBJECT_LOCK_EXCLUSIVE,
+							 &version, &object,
+							 &new_offset, &prot, &wired,
+							 NULL,
+							 &real_map)) {
+				        vm_map_unlock(local_map);
 					return KERN_FAILURE;
 				}
-				if (real_map != map) {
+				if (real_map != map)
 					vm_map_unlock(real_map);
-				}
 				vm_object_unlock(object);
 				vm_map_unlock(local_map);
 
@@ -3465,110 +3514,93 @@ REDISCOVER_ENTRY:
 			submap = entry->object.sub_map;
 			local_start = entry->vme_start;
 			local_offset = entry->offset;
+
 			vm_map_reference(submap);
 			vm_map_unlock(map);
 
-			ret = (vm_map_create_upl(submap, 
-				local_offset + (offset - local_start), 
-				upl_size, upl, page_list, count, 
-				flags));
-
+			ret = vm_map_create_upl(submap, 
+						local_offset + (offset - local_start), 
+						upl_size, upl, page_list, count, flags);
 			vm_map_deallocate(submap);
+
 			return ret;
 		}
-					
 		if (sync_cow_data) {
-			if (entry->object.vm_object->shadow
-				    || entry->object.vm_object->copy) {
-
+			if (entry->object.vm_object->shadow || entry->object.vm_object->copy) {
 				local_object = entry->object.vm_object;
 				local_start = entry->vme_start;
 				local_offset = entry->offset;
+
 				vm_object_reference(local_object);
 				vm_map_unlock(map);
 
-				if (entry->object.vm_object->shadow && 
-					   entry->object.vm_object->copy) {
-				   vm_object_lock_request(
-					local_object->shadow,
-					(vm_object_offset_t)
-					((offset - local_start) +
-					 local_offset) +
-					local_object->shadow_offset,
-					*upl_size, FALSE, 
-					MEMORY_OBJECT_DATA_SYNC,
-					VM_PROT_NO_CHANGE);
+				if (entry->object.vm_object->shadow && entry->object.vm_object->copy) {
+				        vm_object_lock_request(
+							       local_object->shadow,
+							       (vm_object_offset_t)
+							       ((offset - local_start) +
+								local_offset) +
+							       local_object->shadow_offset,
+							       *upl_size, FALSE, 
+							       MEMORY_OBJECT_DATA_SYNC,
+							       VM_PROT_NO_CHANGE);
 				}
 				sync_cow_data = FALSE;
 				vm_object_deallocate(local_object);
+
 				goto REDISCOVER_ENTRY;
 			}
 		}
-
 		if (force_data_sync) {
-
 			local_object = entry->object.vm_object;
 			local_start = entry->vme_start;
 			local_offset = entry->offset;
+
 			vm_object_reference(local_object);
 		        vm_map_unlock(map);
 
 			vm_object_lock_request(
-				   local_object,
-				   (vm_object_offset_t)
-				   ((offset - local_start) + local_offset),
-				   (vm_object_size_t)*upl_size, FALSE, 
-				   MEMORY_OBJECT_DATA_SYNC,
-				   VM_PROT_NO_CHANGE);
+					       local_object,
+					       (vm_object_offset_t)
+					       ((offset - local_start) + local_offset),
+					       (vm_object_size_t)*upl_size, FALSE, 
+					       MEMORY_OBJECT_DATA_SYNC,
+					       VM_PROT_NO_CHANGE);
+
 			force_data_sync = FALSE;
 			vm_object_deallocate(local_object);
+
 			goto REDISCOVER_ENTRY;
 		}
+		if (entry->object.vm_object->private)
+		        *flags = UPL_DEV_MEMORY;
+		else
+		        *flags = 0;
+
+		if (entry->object.vm_object->phys_contiguous)
+		        *flags |= UPL_PHYS_CONTIG;
 
-		if(!(entry->object.vm_object->private)) {
-			if(*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
-				*upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
-			if(entry->object.vm_object->phys_contiguous) {
-				*flags = UPL_PHYS_CONTIG;
-			} else {
-				*flags = 0;
-			}
-		} else {
-			*flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
-		}
 		local_object = entry->object.vm_object;
 		local_offset = entry->offset;
 		local_start = entry->vme_start;
+
 		vm_object_reference(local_object);
 		vm_map_unlock(map);
-		if(caller_flags & UPL_SET_IO_WIRE) {
-			ret = (vm_object_iopl_request(local_object, 
-				(vm_object_offset_t)
-				   ((offset - local_start) 
-						+ local_offset),
-				*upl_size,
-				upl,
-				page_list,
-				count,
-				caller_flags));
-		} else {
-			ret = (vm_object_upl_request(local_object, 
-				(vm_object_offset_t)
-				   ((offset - local_start) 
-						+ local_offset),
-				*upl_size,
-				upl,
-				page_list,
-				count,
-				caller_flags));
-		}
+
+		ret = vm_object_iopl_request(local_object, 
+					      (vm_object_offset_t) ((offset - local_start) + local_offset),
+					      *upl_size,
+					      upl,
+					      page_list,
+					      count,
+					      caller_flags);
 		vm_object_deallocate(local_object);
+
 		return(ret);
 	} 
-
 	vm_map_unlock(map);
-	return(KERN_FAILURE);
 
+	return(KERN_FAILURE);
 }
 
 /*
@@ -3594,149 +3626,155 @@ vm_map_enter_upl(
 
 	upl_lock(upl);
 
-	/* check to see if already mapped */
-	if(UPL_PAGE_LIST_MAPPED & upl->flags) {
+	/*
+	 * check to see if already mapped
+	 */
+	if (UPL_PAGE_LIST_MAPPED & upl->flags) {
 		upl_unlock(upl);
 		return KERN_FAILURE;
 	}
 
-	if((!(upl->map_object->pageout)) && 	
-		!((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) ||
-					(upl->map_object->phys_contiguous))) {
+	if ((!(upl->flags & UPL_SHADOWED)) && !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) ||
+					       (upl->map_object->phys_contiguous))) {
 		vm_object_t 		object;
 		vm_page_t		alias_page;
 		vm_object_offset_t	new_offset;
 		int			pg_num;
 		wpl_array_t 		lite_list;
 
-		if(upl->flags & UPL_INTERNAL) {
+		if (upl->flags & UPL_INTERNAL) {
 			lite_list = (wpl_array_t) 
 				((((uintptr_t)upl) + sizeof(struct upl))
-				+ ((upl->size/PAGE_SIZE) 
-						* sizeof(upl_page_info_t)));
+				 + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
 		} else {
-			lite_list = (wpl_array_t)
-				(((uintptr_t)upl) + sizeof(struct upl));
+		        lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl));
 		}
 		object = upl->map_object;
 		upl->map_object = vm_object_allocate(upl->size);
+
 		vm_object_lock(upl->map_object);
+
 		upl->map_object->shadow = object;
 		upl->map_object->pageout = TRUE;
 		upl->map_object->can_persist = FALSE;
-		upl->map_object->copy_strategy = 
-				MEMORY_OBJECT_COPY_NONE;
-		upl->map_object->shadow_offset = 
-				upl->offset - object->paging_offset;
+		upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+		upl->map_object->shadow_offset = upl->offset - object->paging_offset;
 		upl->map_object->wimg_bits = object->wimg_bits;
 		offset = upl->map_object->shadow_offset;
 		new_offset = 0;
 		size = upl->size;
 
-		vm_object_lock(object);
+		upl->flags |= UPL_SHADOWED;
 
-		while(size) {
-		   pg_num = (new_offset)/PAGE_SIZE;
-		   if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
-			vm_object_unlock(object);
-			VM_PAGE_GRAB_FICTITIOUS(alias_page);
-			vm_object_lock(object);
-			m = vm_page_lookup(object, offset);
-			if (m == VM_PAGE_NULL) {
-				panic("vm_upl_map: page missing\n");
-			}
+		while (size) {
+		        pg_num = (new_offset)/PAGE_SIZE;
 
-			vm_object_paging_begin(object);
+			if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
 
-			/*
- 		 	* Convert the fictitious page to a private 
-			 * shadow of the real page.
-			 */
-			assert(alias_page->fictitious);
-			alias_page->fictitious = FALSE;
-			alias_page->private = TRUE;
-			alias_page->pageout = TRUE;
-			alias_page->phys_page = m->phys_page;
+				VM_PAGE_GRAB_FICTITIOUS(alias_page);
 
-		        vm_page_lock_queues();
-			vm_page_wire(alias_page);
-		        vm_page_unlock_queues();
+				vm_object_lock(object);
 
-			/*
-			 * ENCRYPTED SWAP:
-			 * The virtual page ("m") has to be wired in some way
-			 * here or its physical page ("m->phys_page") could
-			 * be recycled at any time.
-			 * Assuming this is enforced by the caller, we can't
-			 * get an encrypted page here.  Since the encryption
-			 * key depends on the VM page's "pager" object and
-			 * the "paging_offset", we couldn't handle 2 pageable
-			 * VM pages (with different pagers and paging_offsets)
-			 * sharing the same physical page:  we could end up
-			 * encrypting with one key (via one VM page) and
-			 * decrypting with another key (via the alias VM page).
-			 */
-			ASSERT_PAGE_DECRYPTED(m);
+				m = vm_page_lookup(object, offset);
+				if (m == VM_PAGE_NULL) {
+				        panic("vm_upl_map: page missing\n");
+				}
 
-			vm_page_insert(alias_page, 
-					upl->map_object, new_offset);
-			assert(!alias_page->wanted);
-			alias_page->busy = FALSE;
-			alias_page->absent = FALSE;
-		   }
+				/*
+				 * Convert the fictitious page to a private 
+				 * shadow of the real page.
+				 */
+				assert(alias_page->fictitious);
+				alias_page->fictitious = FALSE;
+				alias_page->private = TRUE;
+				alias_page->pageout = TRUE;
+				/*
+				 * since m is a page in the upl it must
+				 * already be wired or BUSY, so it's
+				 * safe to assign the underlying physical
+				 * page to the alias
+				 */
+				alias_page->phys_page = m->phys_page;
+
+			        vm_object_unlock(object);
+
+				vm_page_lockspin_queues();
+				vm_page_wire(alias_page);
+				vm_page_unlock_queues();
+				
+				/*
+				 * ENCRYPTED SWAP:
+				 * The virtual page ("m") has to be wired in some way
+				 * here or its physical page ("m->phys_page") could
+				 * be recycled at any time.
+				 * Assuming this is enforced by the caller, we can't
+				 * get an encrypted page here.  Since the encryption
+				 * key depends on the VM page's "pager" object and
+				 * the "paging_offset", we couldn't handle 2 pageable
+				 * VM pages (with different pagers and paging_offsets)
+				 * sharing the same physical page:  we could end up
+				 * encrypting with one key (via one VM page) and
+				 * decrypting with another key (via the alias VM page).
+				 */
+				ASSERT_PAGE_DECRYPTED(m);
 
-		   size -= PAGE_SIZE;
-		   offset += PAGE_SIZE_64;
-		   new_offset += PAGE_SIZE_64;
+				vm_page_insert(alias_page, upl->map_object, new_offset);
+
+				assert(!alias_page->wanted);
+				alias_page->busy = FALSE;
+				alias_page->absent = FALSE;
+			}
+			size -= PAGE_SIZE;
+			offset += PAGE_SIZE_64;
+			new_offset += PAGE_SIZE_64;
 		}
-		vm_object_unlock(object);
 		vm_object_unlock(upl->map_object);
 	}
 	if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || upl->map_object->phys_contiguous)
 	        offset = upl->offset - upl->map_object->paging_offset;
 	else
 	        offset = 0;
-
 	size = upl->size;
 	
-	vm_object_lock(upl->map_object);
-	upl->map_object->ref_count++;
-	vm_object_res_reference(upl->map_object);
-	vm_object_unlock(upl->map_object);
+	vm_object_reference(upl->map_object);
 
 	*dst_addr = 0;
-
-
-	/* NEED A UPL_MAP ALIAS */
+	/*
+	 * NEED A UPL_MAP ALIAS
+	 */
 	kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
-		VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
-		VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+			  VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
+			  VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
 
 	if (kr != KERN_SUCCESS) {
 		upl_unlock(upl);
 		return(kr);
 	}
-
 	vm_object_lock(upl->map_object);
 
-	for(addr=*dst_addr; size > 0; size-=PAGE_SIZE,addr+=PAGE_SIZE) {
+	for (addr = *dst_addr; size > 0; size -= PAGE_SIZE, addr += PAGE_SIZE) {
 		m = vm_page_lookup(upl->map_object, offset);
-		if(m) {
-		   unsigned int	cache_attr;
-		   cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
+
+		if (m) {
+		        unsigned int	cache_attr;
+			cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
+
+			m->pmapped = TRUE;
 	
-		   PMAP_ENTER(map->pmap, addr,
-				m, VM_PROT_ALL, 
-				cache_attr, TRUE);
+			PMAP_ENTER(map->pmap, addr, m, VM_PROT_ALL, cache_attr, TRUE);
 		}
-		offset+=PAGE_SIZE_64;
+		offset += PAGE_SIZE_64;
 	}
 	vm_object_unlock(upl->map_object);
 
-	upl->ref_count++;  /* hold a reference for the mapping */
+	/*
+	 * hold a reference for the mapping
+	 */
+	upl->ref_count++;
 	upl->flags |= UPL_PAGE_LIST_MAPPED;
 	upl->kaddr = *dst_addr;
 	upl_unlock(upl);
+
 	return KERN_SUCCESS;
 }
 	
@@ -3762,22 +3800,27 @@ vm_map_remove_upl(
 		return KERN_INVALID_ARGUMENT;
 
 	upl_lock(upl);
-	if(upl->flags & UPL_PAGE_LIST_MAPPED) {
+
+	if (upl->flags & UPL_PAGE_LIST_MAPPED) {
 		addr = upl->kaddr;
 		size = upl->size;
+
 		assert(upl->ref_count > 1);
 		upl->ref_count--;		/* removing mapping ref */
+
 		upl->flags &= ~UPL_PAGE_LIST_MAPPED;
 		upl->kaddr = (vm_offset_t) 0;
 		upl_unlock(upl);
 
-		vm_map_remove(  map,
-				vm_map_trunc_page(addr),
-				vm_map_round_page(addr + size),
-				VM_MAP_NO_FLAGS);
+		vm_map_remove(map,
+			      vm_map_trunc_page(addr),
+			      vm_map_round_page(addr + size),
+			      VM_MAP_NO_FLAGS);
+
 		return KERN_SUCCESS;
 	}
 	upl_unlock(upl);
+
 	return KERN_FAILURE;
 }
 
@@ -3791,31 +3834,31 @@ upl_commit_range(
 	mach_msg_type_number_t	count,
 	boolean_t		*empty) 
 {
-	upl_size_t		xfer_size = size;
+	upl_size_t		xfer_size;
 	vm_object_t		shadow_object;
-	vm_object_t		object = upl->map_object;
+	vm_object_t		object;
 	vm_object_offset_t	target_offset;
 	int			entry;
 	wpl_array_t 		lite_list;
 	int			occupied;
 	int                     delayed_unlock = 0;
 	int			clear_refmod = 0;
-	boolean_t		shadow_internal;
+	int			pgpgout_count = 0;
 
 	*empty = FALSE;
 
 	if (upl == UPL_NULL)
 		return KERN_INVALID_ARGUMENT;
 
-
 	if (count == 0)
 		page_list = NULL;
 
-	if (object->pageout) {
-		shadow_object = object->shadow;
-	} else {
-		shadow_object = object;
-	}
+	if (upl->flags & UPL_DEVICE_MEMORY)
+		xfer_size = 0;
+	else if ((offset + size) <= upl->size)
+	        xfer_size = size;
+	else
+		return KERN_FAILURE;
 
 	upl_lock(upl);
 
@@ -3827,37 +3870,33 @@ upl_commit_range(
 		 */
 		flags |= UPL_COMMIT_ALLOW_ACCESS;
 	}
-
 	if (upl->flags & UPL_CLEAR_DIRTY)
 	        flags |= UPL_COMMIT_CLEAR_DIRTY;
 
-	if (upl->flags & UPL_DEVICE_MEMORY) {
-		xfer_size = 0;
-	} else if ((offset + size) > upl->size) {
-		upl_unlock(upl);
-		return KERN_FAILURE;
-	}
+	if (upl->flags & UPL_INTERNAL)
+		lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl))
+					   + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+	else
+		lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
 
-	if (upl->flags & UPL_INTERNAL) {
-		lite_list = (wpl_array_t) 
-			((((uintptr_t)upl) + sizeof(struct upl))
-			+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+	object = upl->map_object;
+
+	if (upl->flags & UPL_SHADOWED) {
+	        vm_object_lock(object);
+		shadow_object = object->shadow;
 	} else {
-		lite_list = (wpl_array_t)
-			(((uintptr_t)upl) + sizeof(struct upl));
+		shadow_object = object;
 	}
-	if (object != shadow_object)
-	        vm_object_lock(object);
 	vm_object_lock(shadow_object);
 
-	shadow_internal = shadow_object->internal;
-
 	entry = offset/PAGE_SIZE;
 	target_offset = (vm_object_offset_t)offset;
 
 	while (xfer_size) {
-		vm_page_t	t,m;
-		upl_page_info_t *p;
+		vm_page_t	t, m;
+
+		if (delayed_unlock == 0)
+		        vm_page_lock_queues();
 
 		m = VM_PAGE_NULL;
 
@@ -3868,231 +3907,224 @@ upl_commit_range(
 
 			if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
 			        lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
-				m = vm_page_lookup(shadow_object,
-						   target_offset + (upl->offset - 
-								    shadow_object->paging_offset));
+
+				m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));
 			}
 		}
-		if (object->pageout) {
-			if ((t = vm_page_lookup(object, target_offset))	!= NULL) {
+		if (upl->flags & UPL_SHADOWED) {
+			if ((t = vm_page_lookup(object, target_offset))	!= VM_PAGE_NULL) {
+
 				t->pageout = FALSE;
 
-				if (delayed_unlock) {
-				        delayed_unlock = 0;
-					vm_page_unlock_queues();
-				}
-				VM_PAGE_FREE(t);
+				vm_page_free(t);
 
-				if (m == NULL) {
-					m = vm_page_lookup(
-					    shadow_object, 
-					    target_offset + 
-						object->shadow_offset);
-				}
-				if (m != VM_PAGE_NULL)
-					vm_object_paging_end(m->object);
+				if (m == VM_PAGE_NULL)
+					m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
 			}
 		}
 		if (m != VM_PAGE_NULL) {
 
-		   clear_refmod = 0;
+		        clear_refmod = 0;
+
+			if (upl->flags & UPL_IO_WIRE) {
 
-		   if (upl->flags & UPL_IO_WIRE) {
+				vm_page_unwire(m);
 
-		        if (delayed_unlock == 0)
-			        vm_page_lock_queues();
+				if (page_list)
+				        page_list[entry].phys_addr = 0;
 
-			vm_page_unwire(m);
+				if (flags & UPL_COMMIT_SET_DIRTY)
+				        m->dirty = TRUE;
+				else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+				        m->dirty = FALSE;
+					clear_refmod |= VM_MEM_MODIFIED;
+				}
+				if (flags & UPL_COMMIT_INACTIVATE)
+					vm_page_deactivate(m);
 
-		        if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
-			        delayed_unlock = 0;
-			        vm_page_unlock_queues();
-			}
-		   	if (page_list) {
-				page_list[entry].phys_addr = 0;
+				if (clear_refmod)
+				        pmap_clear_refmod(m->phys_page, clear_refmod);
+
+				if (flags & UPL_COMMIT_ALLOW_ACCESS) {
+				        /*
+					 * We blocked access to the pages in this UPL.
+					 * Clear the "busy" bit and wake up any waiter
+					 * for this page.
+					 */
+				        PAGE_WAKEUP_DONE(m);
+				}
+				goto commit_next_page;
 			}
-		   	if (flags & UPL_COMMIT_SET_DIRTY) {
-				m->dirty = TRUE;
-		   	} else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
-				m->dirty = FALSE;
+			/*
+			 * make sure to clear the hardware
+			 * modify or reference bits before
+			 * releasing the BUSY bit on this page
+			 * otherwise we risk losing a legitimate
+			 * change of state
+			 */
+			if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+			        m->dirty = FALSE;
 				clear_refmod |= VM_MEM_MODIFIED;
-		   	}
-		   	if (flags & UPL_COMMIT_INACTIVATE) {
-				m->reference = FALSE;
-				clear_refmod |= VM_MEM_REFERENCED;
-              			vm_page_deactivate(m);
 			}
 			if (clear_refmod)
 			        pmap_clear_refmod(m->phys_page, clear_refmod);
 
-			if (flags & UPL_COMMIT_ALLOW_ACCESS) {
-				/*
-				 * We blocked access to the pages in this UPL.
-				 * Clear the "busy" bit and wake up any waiter
-				 * for this page.
-				 */
-				PAGE_WAKEUP_DONE(m);
-			}
+			if (page_list) {
+			        upl_page_info_t *p;
 
-			target_offset += PAGE_SIZE_64;
-			xfer_size -= PAGE_SIZE;
-			entry++;
-			continue;
-		   }
-		   if (delayed_unlock == 0)
-		        vm_page_lock_queues();
-		   /*
-		    * make sure to clear the hardware
-		    * modify or reference bits before
-		    * releasing the BUSY bit on this page
-		    * otherwise we risk losing a legitimate
-		    * change of state
-		    */
-		   if (flags & UPL_COMMIT_CLEAR_DIRTY) {
-			m->dirty = FALSE;
-			clear_refmod |= VM_MEM_MODIFIED;
-		   }
-		   if (flags & UPL_COMMIT_INACTIVATE)
-			clear_refmod |= VM_MEM_REFERENCED;
-
-		   if (clear_refmod)
-		        pmap_clear_refmod(m->phys_page, clear_refmod);
-
-		   if (page_list) {
-			p = &(page_list[entry]);
-			if(p->phys_addr && p->pageout && !m->pageout) {
-				m->busy = TRUE;
-				m->pageout = TRUE;
-				vm_page_wire(m);
-			} else if (page_list[entry].phys_addr &&
-					!p->pageout && m->pageout &&
-					!m->dump_cleaning) {
-				m->pageout = FALSE;
-				m->absent = FALSE;
-				m->overwriting = FALSE;
-				vm_page_unwire(m);
-				PAGE_WAKEUP_DONE(m);
+			        p = &(page_list[entry]);
+
+				if (p->phys_addr && p->pageout && !m->pageout) {
+				        m->busy = TRUE;
+					m->pageout = TRUE;
+					vm_page_wire(m);
+				} else if (p->phys_addr &&
+					   !p->pageout && m->pageout &&
+					   !m->dump_cleaning) {
+				        m->pageout = FALSE;
+					m->absent = FALSE;
+					m->overwriting = FALSE;
+					vm_page_unwire(m);
+
+					PAGE_WAKEUP_DONE(m);
+				}
+				page_list[entry].phys_addr = 0;
 			}
-			page_list[entry].phys_addr = 0;
-		   }
-		   m->dump_cleaning = FALSE;
-		   if(m->laundry) {
-			   vm_pageout_throttle_up(m);
-		   }
-		   if(m->pageout) {
-		      m->cleaning = FALSE;
-		      m->pageout = FALSE;
+			m->dump_cleaning = FALSE;
+
+			if (m->laundry)
+			        vm_pageout_throttle_up(m);
+
+			if (m->pageout) {
+			        m->cleaning = FALSE;
+				m->encrypted_cleaning = FALSE;
+				m->pageout = FALSE;
 #if MACH_CLUSTER_STATS
-		      if (m->wanted) vm_pageout_target_collisions++;
+				if (m->wanted) vm_pageout_target_collisions++;
 #endif
-		      if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)
-			      m->dirty = TRUE;
-		      else
-			      m->dirty = FALSE;
+				m->dirty = FALSE;
 
-		      if(m->dirty) {
-                              vm_page_unwire(m);/* reactivates */
+				if (m->pmapped && (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED))
+				        m->dirty = TRUE;
+
+				if (m->dirty) {
+				       /*
+					* page was re-dirtied after we started
+					* the pageout... reactivate it since 
+					* we don't know whether the on-disk
+					* copy matches what is now in memory
+					*/
+				        vm_page_unwire(m);
+
+					if (upl->flags & UPL_PAGEOUT) {
+					        CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
+						VM_STAT_INCR(reactivations);
+						DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
+					}
+					PAGE_WAKEUP_DONE(m);
+				} else {
+				        /*
+					 * page has been successfully cleaned
+					 * go ahead and free it for other use
+					 */
+
+					if (m->object->internal) {
+						DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
+					} else {
+						DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
+					}
 
-			      if (upl->flags & UPL_PAGEOUT) {
-				      CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
-				      VM_STAT(reactivations++);
-			      }
-                              PAGE_WAKEUP_DONE(m);
-              	      } else {
-                            vm_page_free(m);/* clears busy, etc. */
+				        vm_page_free(m);
  
-			    if (upl->flags & UPL_PAGEOUT) {
-			            CLUSTER_STAT(vm_pageout_target_page_freed++;)
-
-			            if (page_list[entry].dirty)
-				            VM_STAT(pageouts++);
-			    }
-       		      }
-		      if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
-			    delayed_unlock = 0;
-			    vm_page_unlock_queues();
-		      }
-		      target_offset += PAGE_SIZE_64;
-		      xfer_size -= PAGE_SIZE;
-		      entry++;
-                      continue;
-		   }
+					if (upl->flags & UPL_PAGEOUT) {
+					        CLUSTER_STAT(vm_pageout_target_page_freed++;)
+
+						if (page_list[entry].dirty) {
+						        VM_STAT_INCR(pageouts);
+							DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
+							pgpgout_count++;
+						}
+					}
+				}
+				goto commit_next_page;
+			}
 #if MACH_CLUSTER_STATS
-                   m->dirty = pmap_is_modified(m->phys_page);
+			if (m->pmapped)
+			        m->dirty = pmap_is_modified(m->phys_page);
 
-                   if (m->dirty)   vm_pageout_cluster_dirtied++;
-                   else            vm_pageout_cluster_cleaned++;
-                   if (m->wanted)  vm_pageout_cluster_collisions++;
-#else
-                   m->dirty = 0;
+			if (m->dirty)   vm_pageout_cluster_dirtied++;
+			else            vm_pageout_cluster_cleaned++;
+			if (m->wanted)  vm_pageout_cluster_collisions++;
 #endif
+			m->dirty = FALSE;
 
-                   if((m->busy) && (m->cleaning)) {
-                   	/* the request_page_list case */
-			if(m->absent) {
-				m->absent = FALSE;
-				if(shadow_object->absent_count == 1)
-				      vm_object_absent_release(shadow_object);
-				else
-				      shadow_object->absent_count--;
+			if ((m->busy) && (m->cleaning)) {
+			        /*
+				 * the request_page_list case
+				 */
+			        m->absent = FALSE;
+				m->overwriting = FALSE;
+				m->busy = FALSE;
+			} else if (m->overwriting) {
+			        /*
+				 * alternate request page list, write to 
+				 * page_list case.  Occurs when the original
+				 * page was wired at the time of the list
+				 * request
+				 */
+			        assert(m->wire_count != 0);
+				vm_page_unwire(m);/* reactivates */
+				m->overwriting = FALSE;
 			}
-			m->overwriting = FALSE;
-                        m->busy = FALSE;
-                        m->dirty = FALSE;
-                   } else if (m->overwriting) {
-		         /* alternate request page list, write to 
-		          * page_list case.  Occurs when the original
-		          * page was wired at the time of the list
-		          * request */
-		         assert(m->wire_count != 0);
-		         vm_page_unwire(m);/* reactivates */
-		         m->overwriting = FALSE;
-		   }
-                   m->cleaning = FALSE;
-
-		   /* It is a part of the semantic of COPYOUT_FROM */
-		   /* UPLs that a commit implies cache sync 	      */
-		   /* between the vm page and the backing store    */
-		   /* this can be used to strip the precious bit   */
-		   /* as well as clean */
-		   if (upl->flags & UPL_PAGE_SYNC_DONE)
-		         m->precious = FALSE;
-
-		   if (flags & UPL_COMMIT_SET_DIRTY)
-			m->dirty = TRUE;
-
-		   if (flags & UPL_COMMIT_INACTIVATE) {
-			m->reference = FALSE;
-              		vm_page_deactivate(m);
-		   } else if (!m->active && !m->inactive) {
-                	if (m->reference)
-				vm_page_activate(m);
-			else
-				vm_page_deactivate(m);
-		   }
+			m->cleaning = FALSE;
+			m->encrypted_cleaning = FALSE;
 
-		   if (flags & UPL_COMMIT_ALLOW_ACCESS) {
-			   /*
-			    * We blocked access to the pages in this URL.
-			    * Clear the "busy" bit on this page before we
-			    * wake up any waiter.
-			    */
-                           m->busy = FALSE;
-		   }
+			/*
+			 * It is a part of the semantic of COPYOUT_FROM
+			 * UPLs that a commit implies cache sync
+			 * between the vm page and the backing store
+			 * this can be used to strip the precious bit
+			 * as well as clean
+			 */
+			if (upl->flags & UPL_PAGE_SYNC_DONE)
+			        m->precious = FALSE;
 
-                   /*
-                    * Wakeup any thread waiting for the page to be un-cleaning.
-                    */
-                   PAGE_WAKEUP(m);
+			if (flags & UPL_COMMIT_SET_DIRTY)
+			        m->dirty = TRUE;
 
-		   if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
-		         delayed_unlock = 0;
-			 vm_page_unlock_queues();
-		   }
+			if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && !m->speculative) {
+				vm_page_deactivate(m);
+			} else if (!m->active && !m->inactive && !m->speculative) {
+
+			        if (m->clustered)
+				        vm_page_speculate(m, TRUE);
+				else if (m->reference)
+				        vm_page_activate(m);
+				else
+				        vm_page_deactivate(m);
+			}
+			if (flags & UPL_COMMIT_ALLOW_ACCESS) {
+			        /*
+				 * We blocked access to the pages in this URL.
+				 * Clear the "busy" bit on this page before we
+				 * wake up any waiter.
+				 */
+			        m->busy = FALSE;
+			}
+			/*
+			 * Wakeup any thread waiting for the page to be un-cleaning.
+			 */
+			PAGE_WAKEUP(m);
 		}
+commit_next_page:
 		target_offset += PAGE_SIZE_64;
 		xfer_size -= PAGE_SIZE;
 		entry++;
+
+		if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
+			mutex_yield(&vm_page_queue_lock);
+		        delayed_unlock = 1;
+		}
 	}
 	if (delayed_unlock)
 	        vm_page_unlock_queues();
@@ -4104,33 +4136,50 @@ upl_commit_range(
 	} else if (upl->flags & UPL_LITE) {
 		int	pg_num;
 		int	i;
+
 		pg_num = upl->size/PAGE_SIZE;
 		pg_num = (pg_num + 31) >> 5;
 		occupied = 0;
-		for(i= 0; i<pg_num; i++) {
-			if(lite_list[i] != 0) {
+
+		for (i = 0; i < pg_num; i++) {
+			if (lite_list[i] != 0) {
 				occupied = 1;
 				break;
 			}
 		}
 	} else {
-		if(queue_empty(&upl->map_object->memq)) {
+		if (queue_empty(&upl->map_object->memq))
 			occupied = 0;
-		}
 	}
-
-	if(occupied == 0) {
-		if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
+	if (occupied == 0) {
+		if (upl->flags & UPL_COMMIT_NOTIFY_EMPTY)
 			*empty = TRUE;
-		}
-		if(object == shadow_object)
+
+		if (object == shadow_object) {
+		        /*
+			 * this is not a paging object
+			 * so we need to drop the paging reference
+			 * that was taken when we created the UPL
+			 * against this object
+			 */
 			vm_object_paging_end(shadow_object);
+		} else {
+		         /*
+			  * we dontated the paging reference to
+			  * the map object... vm_pageout_object_terminate
+			  * will drop this reference
+			  */
+		}
 	}
 	vm_object_unlock(shadow_object);
 	if (object != shadow_object)
 	        vm_object_unlock(object);
 	upl_unlock(upl);
 
+	if (pgpgout_count) {
+		DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL);
+	}
+
 	return KERN_SUCCESS;
 }
 
@@ -4142,46 +4191,33 @@ upl_abort_range(
 	int			error,
 	boolean_t		*empty) 
 {
-	upl_size_t		xfer_size = size;
+	upl_size_t		xfer_size;
 	vm_object_t		shadow_object;
-	vm_object_t		object = upl->map_object;
+	vm_object_t		object;
 	vm_object_offset_t	target_offset;
 	int			entry;
 	wpl_array_t 	 	lite_list;
 	int			occupied;
-	boolean_t		shadow_internal;
+	int			delayed_unlock = 0;
 
 	*empty = FALSE;
 
 	if (upl == UPL_NULL)
 		return KERN_INVALID_ARGUMENT;
 
-	if (upl->flags & UPL_IO_WIRE) {
-		return upl_commit_range(upl, 
-			offset, size, 0, 
-			NULL, 0, empty);
-	}
-
-	if(object->pageout) {
-		shadow_object = object->shadow;
-	} else {
-		shadow_object = object;
-	}
+	if ( (upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES) )
+		return upl_commit_range(upl, offset, size, 0, NULL, 0, empty);
 
-	upl_lock(upl);
-	if(upl->flags & UPL_DEVICE_MEMORY) {
+	if (upl->flags & UPL_DEVICE_MEMORY)
 		xfer_size = 0;
-	} else if ((offset + size) > upl->size) {
-		upl_unlock(upl);
+	else if ((offset + size) <= upl->size)
+	        xfer_size = size;
+	else
 		return KERN_FAILURE;
-	}
-	if (object != shadow_object)
-	        vm_object_lock(object);
-	vm_object_lock(shadow_object);
 
-	shadow_internal = shadow_object->internal;
+	upl_lock(upl);
 
-	if(upl->flags & UPL_INTERNAL) {
+	if (upl->flags & UPL_INTERNAL) {
 		lite_list = (wpl_array_t) 
 			((((uintptr_t)upl) + sizeof(struct upl))
 			+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
@@ -4189,63 +4225,74 @@ upl_abort_range(
 		lite_list = (wpl_array_t) 
 			(((uintptr_t)upl) + sizeof(struct upl));
 	}
+	object = upl->map_object;
+
+	if (upl->flags & UPL_SHADOWED) {
+	        vm_object_lock(object);
+		shadow_object = object->shadow;
+	} else
+		shadow_object = object;
+
+	vm_object_lock(shadow_object);
 
 	entry = offset/PAGE_SIZE;
 	target_offset = (vm_object_offset_t)offset;
-	while(xfer_size) {
-		vm_page_t	t,m;
+
+	while (xfer_size) {
+		vm_page_t	t, m;
+
+		if (delayed_unlock == 0)
+		        vm_page_lock_queues();
 
 		m = VM_PAGE_NULL;
-		if(upl->flags & UPL_LITE) {
+
+		if (upl->flags & UPL_LITE) {
 			int	pg_num;
 			pg_num = target_offset/PAGE_SIZE;
-			if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
+
+			if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
 				lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
-				m = vm_page_lookup(shadow_object,
-					target_offset + (upl->offset - 
-						shadow_object->paging_offset));
+
+				m = vm_page_lookup(shadow_object, target_offset +
+						   (upl->offset - shadow_object->paging_offset));
 			}
 		}
-		if(object->pageout) {
-			if ((t = vm_page_lookup(object, target_offset))
-								!= NULL) {
-				t->pageout = FALSE;
-				VM_PAGE_FREE(t);
-				if(m == NULL) {
-					m = vm_page_lookup(
-					    shadow_object, 
-					    target_offset + 
-						object->shadow_offset);
-				}
-				if(m != VM_PAGE_NULL)
-					vm_object_paging_end(m->object);
+		if (upl->flags & UPL_SHADOWED) {
+		        if ((t = vm_page_lookup(object, target_offset))	!= VM_PAGE_NULL) {
+			        t->pageout = FALSE;
+
+				vm_page_free(t);
+
+				if (m == VM_PAGE_NULL)
+					m = vm_page_lookup(shadow_object, target_offset + object->shadow_offset);
 			}
 		}
-		if(m != VM_PAGE_NULL) {
-			vm_page_lock_queues();
-			if(m->absent) {
+		if (m != VM_PAGE_NULL) {
+
+			if (m->absent) {
 			        boolean_t must_free = TRUE;
 
-				/* COPYOUT = FALSE case */
-				/* check for error conditions which must */
-				/* be passed back to the pages customer  */
-				if(error & UPL_ABORT_RESTART) {
+				m->clustered = FALSE;
+				/*
+				 * COPYOUT = FALSE case
+				 * check for error conditions which must
+				 * be passed back to the pages customer
+				 */
+				if (error & UPL_ABORT_RESTART) {
 					m->restart = TRUE;
 					m->absent = FALSE;
-					vm_object_absent_release(m->object);
-					m->page_error = KERN_MEMORY_ERROR;
 					m->error = TRUE;
+					m->unusual = TRUE;
 					must_free = FALSE;
-				} else if(error & UPL_ABORT_UNAVAILABLE) {
+				} else if (error & UPL_ABORT_UNAVAILABLE) {
 					m->restart = FALSE;
 					m->unusual = TRUE;
 					must_free = FALSE;
-				} else if(error & UPL_ABORT_ERROR) {
+				} else if (error & UPL_ABORT_ERROR) {
 					m->restart = FALSE;
 					m->absent = FALSE;
-					vm_object_absent_release(m->object);
-					m->page_error = KERN_MEMORY_ERROR;
 					m->error = TRUE;
+					m->unusual = TRUE;
 					must_free = FALSE;
 				}
 
@@ -4259,415 +4306,136 @@ upl_abort_range(
 				 */
 
 				m->cleaning = FALSE;
+				m->encrypted_cleaning = FALSE;
 				m->overwriting = FALSE;
 				PAGE_WAKEUP_DONE(m);
 
-				if (must_free == TRUE) {
+				if (must_free == TRUE)
 					vm_page_free(m);
-				} else {
+				else
 					vm_page_activate(m);
+			} else {
+			        /*                          
+				 * Handle the trusted pager throttle.
+				 */                     
+			        if (m->laundry)
+				        vm_pageout_throttle_up(m);
+
+				if (m->pageout) {
+				        assert(m->busy);
+					assert(m->wire_count == 1);
+					m->pageout = FALSE;
+					vm_page_unwire(m);
 				}
-				vm_page_unlock_queues();
-
-				target_offset += PAGE_SIZE_64;
-				xfer_size -= PAGE_SIZE;
-				entry++;
-				continue;
-			}
-			/*                          
-		 	* Handle the trusted pager throttle.
-		 	*/                     
-			if (m->laundry) {
-				vm_pageout_throttle_up(m);
-			}         
-			if(m->pageout) {
-				assert(m->busy);
-				assert(m->wire_count == 1);
-				m->pageout = FALSE;
-				vm_page_unwire(m);
-			}
-			m->dump_cleaning = FALSE;
-			m->cleaning = FALSE;
-			m->overwriting = FALSE;
+				m->dump_cleaning = FALSE;
+				m->cleaning = FALSE;
+				m->encrypted_cleaning = FALSE;
+				m->overwriting = FALSE;
 #if	MACH_PAGEMAP
-			vm_external_state_clr(
-				m->object->existence_map, m->offset);
+				vm_external_state_clr(m->object->existence_map, m->offset);
 #endif	/* MACH_PAGEMAP */
-			if(error & UPL_ABORT_DUMP_PAGES) {
-				vm_page_free(m);
-				pmap_disconnect(m->phys_page);
-			} else {
-				PAGE_WAKEUP_DONE(m);
+				if (error & UPL_ABORT_DUMP_PAGES) {
+					pmap_disconnect(m->phys_page);
+				        vm_page_free(m);
+				} else {
+				        if (error & UPL_ABORT_REFERENCE) {
+						/*
+						 * we've been told to explictly
+						 * reference this page... for 
+						 * file I/O, this is done by
+						 * implementing an LRU on the inactive q
+						 */
+						vm_page_lru(m);
+					}
+				        PAGE_WAKEUP_DONE(m);
+				}
 			}
-			vm_page_unlock_queues();
+		}
+		if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
+			mutex_yield(&vm_page_queue_lock);
+		        delayed_unlock = 1;
 		}
 		target_offset += PAGE_SIZE_64;
 		xfer_size -= PAGE_SIZE;
 		entry++;
 	}
+	if (delayed_unlock)
+	        vm_page_unlock_queues();
+
 	occupied = 1;
+
 	if (upl->flags & UPL_DEVICE_MEMORY)  {
 		occupied = 0;
 	} else if (upl->flags & UPL_LITE) {
 		int	pg_num;
 		int	i;
+
 		pg_num = upl->size/PAGE_SIZE;
 		pg_num = (pg_num + 31) >> 5;
 		occupied = 0;
-		for(i= 0; i<pg_num; i++) {
-			if(lite_list[i] != 0) {
+
+		for (i = 0; i < pg_num; i++) {
+			if (lite_list[i] != 0) {
 				occupied = 1;
 				break;
 			}
 		}
 	} else {
-		if(queue_empty(&upl->map_object->memq)) {
+		if (queue_empty(&upl->map_object->memq))
 			occupied = 0;
-		}
 	}
-
-	if(occupied == 0) {
-		if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
+	if (occupied == 0) {
+		if (upl->flags & UPL_COMMIT_NOTIFY_EMPTY)
 			*empty = TRUE;
-		}
-		if(object == shadow_object)
+
+		if (object == shadow_object) {
+		        /*
+			 * this is not a paging object
+			 * so we need to drop the paging reference
+			 * that was taken when we created the UPL
+			 * against this object
+			 */
 			vm_object_paging_end(shadow_object);
+		} else {
+		         /*
+			  * we dontated the paging reference to
+			  * the map object... vm_pageout_object_terminate
+			  * will drop this reference
+			  */
+		}
 	}
 	vm_object_unlock(shadow_object);
 	if (object != shadow_object)
 	        vm_object_unlock(object);
-
 	upl_unlock(upl);
 
 	return KERN_SUCCESS;
 }
 
+
 kern_return_t
 upl_abort(
 	upl_t	upl,
 	int	error)
-{
-	vm_object_t		object = NULL;
-	vm_object_t		shadow_object = NULL;
-	vm_object_offset_t	offset;
-	vm_object_offset_t	shadow_offset;
-	vm_object_offset_t	target_offset;
-	upl_size_t		i;
-	wpl_array_t		lite_list;
-	vm_page_t		t,m;
-	int			occupied;
-	boolean_t		shadow_internal;
-
-	if (upl == UPL_NULL)
-		return KERN_INVALID_ARGUMENT;
-
-	if (upl->flags & UPL_IO_WIRE) {
-		boolean_t	empty;
-		return upl_commit_range(upl, 
-			0, upl->size, 0, 
-			NULL, 0, &empty);
-	}
-
-	upl_lock(upl);
-	if(upl->flags & UPL_DEVICE_MEMORY) {
-		upl_unlock(upl);
-		return KERN_SUCCESS;
-	}
-
-	object = upl->map_object;
-
-	if (object == NULL) {
-		panic("upl_abort: upl object is not backed by an object");
-		upl_unlock(upl);
-		return KERN_INVALID_ARGUMENT;
-	}
-
-	if(object->pageout) {
-		shadow_object = object->shadow;
-		shadow_offset = object->shadow_offset;
-	} else {
-		shadow_object = object;
-		shadow_offset = upl->offset - object->paging_offset;
-	}
-
-	if(upl->flags & UPL_INTERNAL) {
-		lite_list = (wpl_array_t)
-			((((uintptr_t)upl) + sizeof(struct upl))
-			+ ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
-	} else {
-		lite_list = (wpl_array_t)
-			(((uintptr_t)upl) + sizeof(struct upl));
-	}
-	offset = 0;
-
-	if (object != shadow_object)
-	        vm_object_lock(object);
-	vm_object_lock(shadow_object);
-
-	shadow_internal = shadow_object->internal;
-
-	for(i = 0; i<(upl->size); i+=PAGE_SIZE, offset += PAGE_SIZE_64) {
-		m = VM_PAGE_NULL;
-		target_offset = offset + shadow_offset;
-		if(upl->flags & UPL_LITE) {
-			int	pg_num;
-			pg_num = offset/PAGE_SIZE;
-			if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
-				lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
-				m = vm_page_lookup(
-					shadow_object, target_offset);
-			}
-		}
-		if(object->pageout) {
-			if ((t = vm_page_lookup(object, offset)) != NULL) {
-				t->pageout = FALSE;
-				VM_PAGE_FREE(t);
-				if(m == NULL) {
-					m = vm_page_lookup(
-					    shadow_object, target_offset);
-				}
-				if(m != VM_PAGE_NULL)
-					vm_object_paging_end(m->object);
-			}
-		}
-		if(m != VM_PAGE_NULL) {
-			vm_page_lock_queues();
-			if(m->absent) {
-			        boolean_t must_free = TRUE;
-
-				/* COPYOUT = FALSE case */
-				/* check for error conditions which must */
-				/* be passed back to the pages customer  */
-				if(error & UPL_ABORT_RESTART) {
-					m->restart = TRUE;
-					m->absent = FALSE;
-					vm_object_absent_release(m->object);
-					m->page_error = KERN_MEMORY_ERROR;
-					m->error = TRUE;
-					must_free = FALSE;
-				} else if(error & UPL_ABORT_UNAVAILABLE) {
-					m->restart = FALSE;
-					m->unusual = TRUE;
-					must_free = FALSE;
-				} else if(error & UPL_ABORT_ERROR) {
-					m->restart = FALSE;
-					m->absent = FALSE;
-					vm_object_absent_release(m->object);
-					m->page_error = KERN_MEMORY_ERROR;
-					m->error = TRUE;
-					must_free = FALSE;
-				}
-
-				/*
-				 * ENCRYPTED SWAP:
-				 * If the page was already encrypted,
-				 * we don't really need to decrypt it
-				 * now.  It will get decrypted later,
-				 * on demand, as soon as someone needs
-				 * to access its contents.
-				 */
-
-				m->cleaning = FALSE;
-				m->overwriting = FALSE;
-				PAGE_WAKEUP_DONE(m);
-
-				if (must_free == TRUE) {
-					vm_page_free(m);
-				} else {
-					vm_page_activate(m);
-				}
-				vm_page_unlock_queues();
-				continue;
-			}
-			/*                          
-			 * Handle the trusted pager throttle.
-			 */                     
-			if (m->laundry) { 
-				vm_pageout_throttle_up(m);
-			}         
-			if(m->pageout) {
-				assert(m->busy);
-				assert(m->wire_count == 1);
-				m->pageout = FALSE;
-				vm_page_unwire(m);
-			}
-			m->dump_cleaning = FALSE;
-			m->cleaning = FALSE;
-			m->overwriting = FALSE;
-#if	MACH_PAGEMAP
-			vm_external_state_clr(
-				m->object->existence_map, m->offset);
-#endif	/* MACH_PAGEMAP */
-			if(error & UPL_ABORT_DUMP_PAGES) {
-				vm_page_free(m);
-				pmap_disconnect(m->phys_page);
-			} else {
-				PAGE_WAKEUP_DONE(m);
-			}
-			vm_page_unlock_queues();
-		}
-	}
-	occupied = 1;
-	if (upl->flags & UPL_DEVICE_MEMORY)  {
-		occupied = 0;
-	} else if (upl->flags & UPL_LITE) {
-		int	pg_num;
-		int	j;
-		pg_num = upl->size/PAGE_SIZE;
-		pg_num = (pg_num + 31) >> 5;
-		occupied = 0;
-		for(j= 0; j<pg_num; j++) {
-			if(lite_list[j] != 0) {
-				occupied = 1;
-				break;
-			}
-		}
-	} else {
-		if(queue_empty(&upl->map_object->memq)) {
-			occupied = 0;
-		}
-	}
-
-	if(occupied == 0) {
-		if(object == shadow_object)
-			vm_object_paging_end(shadow_object);
-	}
-	vm_object_unlock(shadow_object);
-	if (object != shadow_object)
-	        vm_object_unlock(object);
-
-	upl_unlock(upl);
-	return KERN_SUCCESS;
-}
-
-/* an option on commit should be wire */
-kern_return_t
-upl_commit(
-	upl_t			upl,
-	upl_page_info_t		*page_list,
-	mach_msg_type_number_t	count)
-{
-	if (upl == UPL_NULL)
-		return KERN_INVALID_ARGUMENT;
-
-	if(upl->flags & (UPL_LITE | UPL_IO_WIRE)) {
-		boolean_t	empty;
-		return upl_commit_range(upl, 0, upl->size, 0, 
-					page_list, count, &empty);
-	}
-
-	if (count == 0)
-		page_list = NULL;
-
-	upl_lock(upl);
-	if (upl->flags & UPL_DEVICE_MEMORY)
-		page_list = NULL;
-
-	if (upl->flags & UPL_ENCRYPTED) {
-		/*
-		 * ENCRYPTED SWAP:
-		 * This UPL was encrypted, but we don't need
-		 * to decrypt here.  We'll decrypt each page
-		 * later, on demand, as soon as someone needs
-		 * to access the page's contents.
-		 */
-	}
-
-	if ((upl->flags & UPL_CLEAR_DIRTY) ||
-		(upl->flags & UPL_PAGE_SYNC_DONE) || page_list) {
-		vm_object_t	shadow_object = upl->map_object->shadow;
-		vm_object_t	object = upl->map_object;
-		vm_object_offset_t target_offset;
-		upl_size_t	xfer_end;
-		int		entry;
-
-		vm_page_t	t, m;
-		upl_page_info_t	*p;
-
-		if (object != shadow_object)
-		        vm_object_lock(object);
-		vm_object_lock(shadow_object);
-
-		entry = 0;
-		target_offset = object->shadow_offset;
-		xfer_end = upl->size + object->shadow_offset;
-
-		while(target_offset < xfer_end) {
-
-			if ((t = vm_page_lookup(object, 
-				target_offset - object->shadow_offset))
-				== NULL) {
-				target_offset += PAGE_SIZE_64;
-				entry++;
-				continue;
-			}
-
-			m = vm_page_lookup(shadow_object, target_offset);
-			if(m != VM_PAGE_NULL) {
-			    /*
-			     * ENCRYPTED SWAP:
-			     * If this page was encrypted, we
-			     * don't need to decrypt it here.
-			     * We'll decrypt it later, on demand,
-			     * as soon as someone needs to access
-			     * its contents.
-			     */
-
-			    if (upl->flags & UPL_CLEAR_DIRTY) {
-				pmap_clear_modify(m->phys_page);
-				m->dirty = FALSE;
-			    }
-			    /* It is a part of the semantic of */
-			    /* COPYOUT_FROM UPLs that a commit */
-			    /* implies cache sync between the  */
-			    /* vm page and the backing store   */
-			    /* this can be used to strip the   */
-			    /* precious bit as well as clean   */
-			    if (upl->flags & UPL_PAGE_SYNC_DONE)
-				m->precious = FALSE;
-
-			   if(page_list) {
-			   	p = &(page_list[entry]);
-			   	if(page_list[entry].phys_addr &&
-						p->pageout && !m->pageout) {
-					vm_page_lock_queues();
-					m->busy = TRUE;
-					m->pageout = TRUE;
-					vm_page_wire(m);
-					vm_page_unlock_queues();
-			   	} else if (page_list[entry].phys_addr &&
-						!p->pageout && m->pageout &&
-						!m->dump_cleaning) {
-					vm_page_lock_queues();
-					m->pageout = FALSE;
-					m->absent = FALSE;
-					m->overwriting = FALSE;
-					vm_page_unwire(m);
-					PAGE_WAKEUP_DONE(m);
-					vm_page_unlock_queues();
-			   	}
-			   	page_list[entry].phys_addr = 0;
-			   }
-			}
-			target_offset += PAGE_SIZE_64;
-			entry++;
-		}
-		vm_object_unlock(shadow_object);
-		if (object != shadow_object)
-		        vm_object_unlock(object);
-
-	}
-	if (upl->flags & UPL_DEVICE_MEMORY)  {
-		vm_object_lock(upl->map_object->shadow);
-		if(upl->map_object == upl->map_object->shadow)
-			vm_object_paging_end(upl->map_object->shadow);
-		vm_object_unlock(upl->map_object->shadow);
-	}
-	upl_unlock(upl);
-	return KERN_SUCCESS;
+{
+	boolean_t	empty;
+
+	return upl_abort_range(upl, 0, upl->size, error, &empty);
 }
 
 
+/* an option on commit should be wire */
+kern_return_t
+upl_commit(
+	upl_t			upl,
+	upl_page_info_t		*page_list,
+	mach_msg_type_number_t	count)
+{
+	boolean_t	empty;
+
+	return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty);
+}
+
 
 kern_return_t
 vm_object_iopl_request(
@@ -4680,17 +4448,17 @@ vm_object_iopl_request(
 	int			cntrl_flags)
 {
 	vm_page_t		dst_page;
-	vm_object_offset_t	dst_offset = offset;
-	upl_size_t		xfer_size = size;
+	vm_object_offset_t	dst_offset;
+	upl_size_t		xfer_size;
 	upl_t			upl = NULL;
 	unsigned int		entry;
 	wpl_array_t 		lite_list = NULL;
-	int			page_field_size;
 	int                     delayed_unlock = 0;
 	int			no_zero_fill = FALSE;
-	vm_page_t		alias_page = NULL;
+	u_int32_t		psize;
 	kern_return_t		ret;
 	vm_prot_t		prot;
+	struct vm_object_fault_info fault_info;
 
 
 	if (cntrl_flags & ~UPL_VALID_FLAGS) {
@@ -4710,8 +4478,8 @@ vm_object_iopl_request(
 		if (object->phys_contiguous) {
 		        if ((offset + object->shadow_offset) >= (vm_object_offset_t)max_valid_dma_address)
 			        return KERN_INVALID_ADDRESS;
-			  
-		        if (((offset + object->shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
+	      
+			if (((offset + object->shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
 			        return KERN_INVALID_ADDRESS;
 		}
 	}
@@ -4725,7 +4493,6 @@ vm_object_iopl_request(
 		 */
 		assert(! (cntrl_flags & UPL_ENCRYPT));
 	}
-
 	if (cntrl_flags & UPL_NOZEROFILL)
 	        no_zero_fill = TRUE;
 
@@ -4734,135 +4501,89 @@ vm_object_iopl_request(
 	else
 		prot = VM_PROT_READ | VM_PROT_WRITE;
 
-	if(((size/page_size) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
+	if (((size/page_size) > MAX_UPL_TRANSFER) && !object->phys_contiguous)
 		size = MAX_UPL_TRANSFER * page_size;
-	}
 
-	if(cntrl_flags & UPL_SET_INTERNAL)
-		if(page_list_count != NULL)
+	if (cntrl_flags & UPL_SET_INTERNAL) {
+		if (page_list_count != NULL)
 			*page_list_count = MAX_UPL_TRANSFER;
-	if(((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
-	   ((page_list_count != NULL) && (*page_list_count != 0)
-				&& *page_list_count < (size/page_size)))
-		return KERN_INVALID_ARGUMENT;
+	}
+	if (((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
+	    ((page_list_count != NULL) && (*page_list_count != 0) && *page_list_count < (size/page_size)))
+	        return KERN_INVALID_ARGUMENT;
 
-	if((!object->internal) && (object->paging_offset != 0))
-		panic("vm_object_upl_request: external object with non-zero paging offset\n");
+	if ((!object->internal) && (object->paging_offset != 0))
+		panic("vm_object_iopl_request: external object with non-zero paging offset\n");
+
+
+	if (object->phys_contiguous)
+	        psize = PAGE_SIZE;
+	else
+	        psize = size;
+
+	if (cntrl_flags & UPL_SET_INTERNAL) {
+	        upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE, UPL_IO_WIRE, psize);
+
+		user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+		lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +
+					   ((psize / PAGE_SIZE) * sizeof(upl_page_info_t)));
+	} else {
+	        upl = upl_create(UPL_CREATE_LITE, UPL_IO_WIRE, psize);
 
-	if(object->phys_contiguous) {
-		/* No paging operations are possible against this memory */
-		/* and so no need for map object, ever */
-		cntrl_flags |= UPL_SET_LITE;
+		lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
 	}
+	if (user_page_list)
+	        user_page_list[0].device = FALSE;
+	*upl_ptr = upl;
 
-	if(upl_ptr) {
-		if(cntrl_flags & UPL_SET_INTERNAL) {
-			if(cntrl_flags & UPL_SET_LITE) {
-				upl = upl_create(
-					UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
-					size);
-				user_page_list = (upl_page_info_t *)
-				   (((uintptr_t)upl) + sizeof(struct upl));
-				lite_list = (wpl_array_t)
-					(((uintptr_t)user_page_list) + 
-					((size/PAGE_SIZE) * 
-						sizeof(upl_page_info_t)));
-				page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
-				page_field_size = 
-					(page_field_size + 3) & 0xFFFFFFFC;
-				bzero((char *)lite_list, page_field_size);
-				upl->flags = 
-					UPL_LITE | UPL_INTERNAL | UPL_IO_WIRE;
-			} else {
-				upl = upl_create(UPL_CREATE_INTERNAL, size);
-				user_page_list = (upl_page_info_t *)
-					(((uintptr_t)upl) 
-						+ sizeof(struct upl));
-				upl->flags = UPL_INTERNAL | UPL_IO_WIRE;
-			}
-		} else {
-			if(cntrl_flags & UPL_SET_LITE) {
-				upl = upl_create(UPL_CREATE_LITE, size);
-				lite_list = (wpl_array_t)
-				   (((uintptr_t)upl) + sizeof(struct upl));
-				page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
-				page_field_size = 
-					(page_field_size + 3) & 0xFFFFFFFC;
-				bzero((char *)lite_list, page_field_size);
-				upl->flags = UPL_LITE | UPL_IO_WIRE;
-			} else {
-				upl = upl_create(UPL_CREATE_EXTERNAL, size);
-				upl->flags = UPL_IO_WIRE;
-			}
-		}
+	upl->map_object = object;
+	upl->size = size;
+
+	vm_object_lock(object);
+	vm_object_paging_begin(object);
+	/*
+	 * paging in progress also protects the paging_offset
+	 */
+	upl->offset = offset + object->paging_offset;
 
-		if(object->phys_contiguous) {
-			upl->map_object = object;
-			/* don't need any shadow mappings for this one */
-			/* since it is already I/O memory */
-			upl->flags |= UPL_DEVICE_MEMORY;
+	if (object->phys_contiguous) {
+#ifdef UPL_DEBUG
+		queue_enter(&object->uplq, upl, upl_t, uplq);
+#endif /* UPL_DEBUG */
 
-			vm_object_lock(object);
-			vm_object_paging_begin(object);
-			vm_object_unlock(object);
+		vm_object_unlock(object);
 
-			/* paging in progress also protects the paging_offset */
-			upl->offset = offset + object->paging_offset;
-			upl->size = size;
-			*upl_ptr = upl;
-			if(user_page_list) {
-				user_page_list[0].phys_addr = 
-				  (offset + object->shadow_offset)>>PAGE_SHIFT;
-				user_page_list[0].device = TRUE;
-			}
-			upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
+		/*
+		 * don't need any shadow mappings for this one
+		 * since it is already I/O memory
+		 */
+		upl->flags |= UPL_DEVICE_MEMORY;
 
-			if(page_list_count != NULL) {
-				if (upl->flags & UPL_INTERNAL) {
-					*page_list_count = 0;
-				} else {
-					*page_list_count = 1;
-				}
-			}
-			return KERN_SUCCESS;
+		upl->highest_page = (offset + object->shadow_offset + size - 1)>>PAGE_SHIFT;
+
+		if (user_page_list) {
+		        user_page_list[0].phys_addr = (offset + object->shadow_offset)>>PAGE_SHIFT;
+			user_page_list[0].device = TRUE;
 		}
-		if(user_page_list)
-			user_page_list[0].device = FALSE;
-			
-		if(cntrl_flags & UPL_SET_LITE) {
-			upl->map_object = object;
-		} else {
-			upl->map_object = vm_object_allocate(size);
-			vm_object_lock(upl->map_object);
-			upl->map_object->shadow = object;
-			upl->map_object->pageout = TRUE;
-			upl->map_object->can_persist = FALSE;
-			upl->map_object->copy_strategy = 
-					MEMORY_OBJECT_COPY_NONE;
-			upl->map_object->shadow_offset = offset;
-			upl->map_object->wimg_bits = object->wimg_bits;
-			vm_object_unlock(upl->map_object);
+		if (page_list_count != NULL) {
+		        if (upl->flags & UPL_INTERNAL)
+			        *page_list_count = 0;
+			else
+			        *page_list_count = 1;
 		}
+		return KERN_SUCCESS;
 	}
-	vm_object_lock(object);
-	vm_object_paging_begin(object);
+	/*
+	 * Protect user space from future COW operations
+	 */
+	object->true_share = TRUE;
 
-	if (!object->phys_contiguous) {
-		/* Protect user space from future COW operations */
-		object->true_share = TRUE;
-		if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
-			object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
-	}
+	if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
+	        object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
 
-	/* we can lock the upl offset now that paging_in_progress is set */
-	if(upl_ptr) {
-		upl->size = size;
-		upl->offset = offset + object->paging_offset;
-		*upl_ptr = upl;
 #ifdef UPL_DEBUG
-		queue_enter(&object->uplq, upl, upl_t, uplq);
+	queue_enter(&object->uplq, upl, upl_t, uplq);
 #endif /* UPL_DEBUG */
-	}
 
 	if (cntrl_flags & UPL_BLOCK_ACCESS) {
 		/*
@@ -4871,18 +4592,21 @@ vm_object_iopl_request(
 		 */
 		upl->flags |= UPL_ACCESS_BLOCKED;
 	}
-
 	entry = 0;
+
+	xfer_size = size;
+	dst_offset = offset;
+
+	fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
+	fault_info.user_tag  = 0;
+	fault_info.lo_offset = offset;
+	fault_info.hi_offset = offset + xfer_size;
+	fault_info.no_cache  = FALSE;
+
 	while (xfer_size) {
-		if((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
-		        if (delayed_unlock) {
-			        delayed_unlock = 0;
-			        vm_page_unlock_queues();
-			}
-			vm_object_unlock(object);
-			VM_PAGE_GRAB_FICTITIOUS(alias_page);
-			vm_object_lock(object);
-		}
+	        vm_fault_return_t	result;
+	        int			pg_num;
+
 		dst_page = vm_page_lookup(object, dst_offset);
 
 		/*
@@ -4895,71 +4619,58 @@ vm_object_iopl_request(
 		    (dst_page->unusual && (dst_page->error || 
 					   dst_page->restart ||
 					   dst_page->absent ||
-					   dst_page->fictitious ||
-					   (prot & dst_page->page_lock)))) {
-			vm_fault_return_t	result;
+					   dst_page->fictitious))) {
+
 		   do {
 			vm_page_t	top_page;
 			kern_return_t	error_code;
 			int		interruptible;
 
-			vm_object_offset_t	lo_offset = offset;
-			vm_object_offset_t	hi_offset = offset + size;
-
-
 		        if (delayed_unlock) {
 			        delayed_unlock = 0;
 			        vm_page_unlock_queues();
 			}
-
-			if(cntrl_flags & UPL_SET_INTERRUPTIBLE) {
+			if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
 				interruptible = THREAD_ABORTSAFE;
-			} else {
+			else
 				interruptible = THREAD_UNINT;
-			}
+
+			fault_info.interruptible = interruptible;
+			fault_info.cluster_size = xfer_size;
 
 			result = vm_fault_page(object, dst_offset,
-				prot | VM_PROT_WRITE, FALSE, 
-				interruptible,
-				lo_offset, hi_offset,
-				VM_BEHAVIOR_SEQUENTIAL,
-				&prot, &dst_page, &top_page,
-			        (int *)0,
-				&error_code, no_zero_fill, FALSE, NULL, 0);
-
-			switch(result) {
+					       prot | VM_PROT_WRITE, FALSE, 
+					       &prot, &dst_page, &top_page,
+					       (int *)0,
+					       &error_code, no_zero_fill,
+					       FALSE, &fault_info);
+
+			switch (result) {
+
 			case VM_FAULT_SUCCESS:
 
 				PAGE_WAKEUP_DONE(dst_page);
-
 				/*
 				 *	Release paging references and
 				 *	top-level placeholder page, if any.
 				 */
-
-				if(top_page != VM_PAGE_NULL) {
+				if (top_page != VM_PAGE_NULL) {
 					vm_object_t local_object;
-					local_object = 
-						top_page->object;
-					if(top_page->object 
-						!= dst_page->object) {
-						vm_object_lock(
-							local_object);
+
+					local_object = top_page->object;
+
+					if (top_page->object != dst_page->object) {
+						vm_object_lock(local_object);
 						VM_PAGE_FREE(top_page);
-						vm_object_paging_end(
-							local_object);
-						vm_object_unlock(
-							local_object);
+						vm_object_paging_end(local_object);
+						vm_object_unlock(local_object);
 					} else {
 						VM_PAGE_FREE(top_page);
-						vm_object_paging_end(
-							local_object);
+						vm_object_paging_end(local_object);
 					}
 				}
-
 				break;
 			
-				
 			case VM_FAULT_RETRY:
 				vm_object_lock(object);
 				vm_object_paging_begin(object);
@@ -4967,6 +4678,7 @@ vm_object_iopl_request(
 
 			case VM_FAULT_FICTITIOUS_SHORTAGE:
 				vm_page_more_fictitious();
+
 				vm_object_lock(object);
 				vm_object_paging_begin(object);
 				break;
@@ -4982,14 +4694,13 @@ vm_object_iopl_request(
 			case VM_FAULT_INTERRUPTED:
 				error_code = MACH_SEND_INTERRUPTED;
 			case VM_FAULT_MEMORY_ERROR:
-				ret = (error_code ? error_code:
-					KERN_MEMORY_ERROR);
-				vm_object_lock(object);
+				ret = (error_code ? error_code:	KERN_MEMORY_ERROR);
 
+				vm_object_lock(object);
+				vm_object_paging_begin(object);
 				goto return_err;
 			}
-		   } while ((result != VM_FAULT_SUCCESS) 
-				|| (result == VM_FAULT_INTERRUPTED));
+		   } while (result != VM_FAULT_SUCCESS);
 		}
 
 		if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
@@ -5005,13 +4716,13 @@ vm_object_iopl_request(
 			 * we don't know whether that physical address has been
 			 * handed out to some other 64 bit capable DMA device to use
 			 */
-		        if (dst_page->wire_count) {
+			if (dst_page->wire_count) {
 			        ret = KERN_PROTECTION_FAILURE;
 				goto return_err;
 			}
-		        if (delayed_unlock) {
+			if (delayed_unlock) {
 			        delayed_unlock = 0;
-			        vm_page_unlock_queues();
+				vm_page_unlock_queues();
 			}
 			low_page = vm_page_grablo();
 
@@ -5026,10 +4737,12 @@ vm_object_iopl_request(
 			 * it after we disconnect it... we want the fault
 			 * to find the new page being substituted.
 			 */
-			refmod = pmap_disconnect(dst_page->phys_page);
-
+			if (dst_page->pmapped)
+			        refmod = pmap_disconnect(dst_page->phys_page);
+			else
+			        refmod = 0;
 			vm_page_copy(dst_page, low_page);
-			
+		  
 			low_page->reference = dst_page->reference;
 			low_page->dirty     = dst_page->dirty;
 
@@ -5038,7 +4751,7 @@ vm_object_iopl_request(
 			if (refmod & VM_MEM_MODIFIED)
 			        low_page->dirty = TRUE;
 
-		        vm_page_lock_queues();
+			vm_page_lock_queues();
 			vm_page_replace(low_page, object, dst_offset);
 			/*
 			 * keep the queue lock since we're going to 
@@ -5056,6 +4769,7 @@ vm_object_iopl_request(
 		}
 		if (delayed_unlock == 0)
 		        vm_page_lock_queues();
+
 		vm_page_wire(dst_page);
 
 		if (cntrl_flags & UPL_BLOCK_ACCESS) {
@@ -5067,57 +4781,43 @@ vm_object_iopl_request(
 			assert(!dst_page->fictitious);
 			dst_page->busy = TRUE;
 		}
+		pg_num = (dst_offset-offset)/PAGE_SIZE;
+		lite_list[pg_num>>5] |= 1 << (pg_num & 31);
 
-		if (upl_ptr) {
-			if (cntrl_flags & UPL_SET_LITE) {
-				int	pg_num;
-				pg_num = (dst_offset-offset)/PAGE_SIZE;
-				lite_list[pg_num>>5] |= 1 << (pg_num & 31);
-			} else {
-				/*
-	 			 * Convert the fictitious page to a 
-				 * private shadow of the real page.
-	 			 */
-				assert(alias_page->fictitious);
-				alias_page->fictitious = FALSE;
-				alias_page->private = TRUE;
-				alias_page->pageout = TRUE;
-				alias_page->phys_page = dst_page->phys_page;
-				vm_page_wire(alias_page);
+		/*
+		 * expect the page to be used
+		 * page queues lock must be held to set 'reference'
+		 */
+		dst_page->reference = TRUE;
 
-				vm_page_insert(alias_page, 
-					upl->map_object, size - xfer_size);
-				assert(!alias_page->wanted);
-				alias_page->busy = FALSE;
-				alias_page->absent = FALSE;
-			}
+   		if (!(cntrl_flags & UPL_COPYOUT_FROM))
+			dst_page->dirty = TRUE;
 
-			/* expect the page to be used */
-			dst_page->reference = TRUE;
+		if (dst_page->phys_page > upl->highest_page)
+		        upl->highest_page = dst_page->phys_page;
 
-	   		if (!(cntrl_flags & UPL_COPYOUT_FROM))
-				dst_page->dirty = TRUE;
-			alias_page = NULL;
+		if (user_page_list) {
+			user_page_list[entry].phys_addr	= dst_page->phys_page;
+			user_page_list[entry].dirty 	= dst_page->dirty;
+			user_page_list[entry].pageout	= dst_page->pageout;
+			user_page_list[entry].absent	= dst_page->absent;
+			user_page_list[entry].precious	= dst_page->precious;
 
-			if (dst_page->phys_page > upl->highest_page)
-				upl->highest_page = dst_page->phys_page;
-
-			if (user_page_list) {
-				user_page_list[entry].phys_addr
-					= dst_page->phys_page;
-				user_page_list[entry].dirty =
-						dst_page->dirty;
-				user_page_list[entry].pageout =
-			   			dst_page->pageout;
-				user_page_list[entry].absent =
-			   			dst_page->absent;
-				user_page_list[entry].precious =
-						dst_page->precious;
-			}
+			if (dst_page->clustered == TRUE)
+			        user_page_list[entry].speculative = dst_page->speculative;
+			else
+			        user_page_list[entry].speculative = FALSE;
 		}
-		if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
-		        delayed_unlock = 0;
-			vm_page_unlock_queues();
+		/*
+		 * someone is explicitly grabbing this page...
+		 * update clustered and speculative state
+		 * 
+		 */
+		VM_PAGE_CONSUME_CLUSTERED(dst_page);
+
+		if (delayed_unlock++ > UPL_DELAYED_UNLOCK_LIMIT) {
+			mutex_yield(&vm_page_queue_lock);
+		        delayed_unlock = 1;
 		}
 		entry++;
 		dst_offset += PAGE_SIZE_64;
@@ -5126,20 +4826,12 @@ vm_object_iopl_request(
 	if (delayed_unlock)
 	        vm_page_unlock_queues();
 
-	if (upl->flags & UPL_INTERNAL) {
-		if(page_list_count != NULL)
+	if (page_list_count != NULL) {
+	        if (upl->flags & UPL_INTERNAL)
 			*page_list_count = 0;
-	} else if (*page_list_count > entry) {
-		if(page_list_count != NULL)
+		else if (*page_list_count > entry)
 			*page_list_count = entry;
 	}
-
-	if (alias_page != NULL) {
-		vm_page_lock_queues();
-		vm_page_free(alias_page);
-		vm_page_unlock_queues();
-	}
-
 	vm_object_unlock(object);
 
 	if (cntrl_flags & UPL_BLOCK_ACCESS) {
@@ -5152,10 +4844,8 @@ vm_object_iopl_request(
 		vm_object_pmap_protect(object, offset, (vm_object_size_t)size,
 				       PMAP_NULL, 0, VM_PROT_NONE);
 	}
-
 	return KERN_SUCCESS;
 
-
 return_err:
 	if (delayed_unlock)
 	        vm_page_unlock_queues();
@@ -5165,10 +4855,12 @@ return_err:
 
 		if (dst_page == VM_PAGE_NULL)
 		        panic("vm_object_iopl_request: Wired pages missing. \n");
-		vm_page_lock_queues();
+
+		vm_page_lockspin_queues();
 		vm_page_unwire(dst_page);
 		vm_page_unlock_queues();
-		VM_STAT(reactivations++);
+
+		VM_STAT_INCR(reactivations);
 	}
 	vm_object_paging_end(object);
 	vm_object_unlock(object);
@@ -5177,7 +4869,6 @@ return_err:
 	return ret;
 }
 
-
 kern_return_t
 upl_transpose(
 	upl_t		upl1,
@@ -5232,8 +4923,16 @@ upl_transpose(
 		 * Make each UPL point to the correct VM object, i.e. the
 		 * object holding the pages that the UPL refers to...
 		 */
+#ifdef UPL_DEBUG
+		queue_remove(&object1->uplq, upl1, upl_t, uplq);
+		queue_remove(&object2->uplq, upl2, upl_t, uplq);
+#endif
 		upl1->map_object = object2;
 		upl2->map_object = object1;
+#ifdef UPL_DEBUG
+		queue_enter(&object1->uplq, upl2, upl_t, uplq);
+		queue_enter(&object2->uplq, upl1, upl_t, uplq);
+#endif
 	}
 
 done:
@@ -5297,12 +4996,47 @@ decl_simple_lock_data(,vm_paging_lock)
 vm_map_offset_t vm_paging_base_address = 0;
 boolean_t	vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
 int		vm_paging_max_index = 0;
+int		vm_paging_page_waiter = 0;
+int		vm_paging_page_waiter_total = 0;
 unsigned long	vm_paging_no_kernel_page = 0;
 unsigned long	vm_paging_objects_mapped = 0;
 unsigned long	vm_paging_pages_mapped = 0;
 unsigned long	vm_paging_objects_mapped_slow = 0;
 unsigned long	vm_paging_pages_mapped_slow = 0;
 
+void
+vm_paging_map_init(void)
+{
+	kern_return_t	kr;
+	vm_map_offset_t	page_map_offset;
+	vm_map_entry_t	map_entry;
+
+	assert(vm_paging_base_address == 0);
+
+	/*
+	 * Initialize our pool of pre-allocated kernel
+	 * virtual addresses.
+	 */
+	page_map_offset = 0;
+	kr = vm_map_find_space(kernel_map,
+			       &page_map_offset,
+			       VM_PAGING_NUM_PAGES * PAGE_SIZE,
+			       0,
+			       0,
+			       &map_entry);
+	if (kr != KERN_SUCCESS) {
+		panic("vm_paging_map_init: kernel_map full\n");
+	}
+	map_entry->object.vm_object = kernel_object;
+	map_entry->offset =
+		page_map_offset - VM_MIN_KERNEL_ADDRESS;
+	vm_object_reference(kernel_object);
+	vm_map_unlock(kernel_map);
+
+	assert(vm_paging_base_address == 0);
+	vm_paging_base_address = page_map_offset;
+}
+
 /*
  * ENCRYPTED SWAP:
  * vm_paging_map_object:
@@ -5311,7 +5045,12 @@ unsigned long	vm_paging_pages_mapped_slow = 0;
  *	kernel virtual addresses, if possible.
  * Context:
  * 	The VM object is locked.  This lock will get
- * 	dropped and re-acquired though.
+ * 	dropped and re-acquired though, so the caller
+ * 	must make sure the VM object is kept alive
+ *	(by holding a VM map that has a reference
+ * 	on it, for example, or taking an extra reference).
+ * 	The page should also be kept busy to prevent
+ *	it from being reclaimed.
  */
 kern_return_t
 vm_paging_map_object(
@@ -5319,76 +5058,62 @@ vm_paging_map_object(
 	vm_page_t		page,
 	vm_object_t		object,
 	vm_object_offset_t	offset,
-	vm_map_size_t		*size)
+	vm_map_size_t		*size,
+	boolean_t		can_unlock_object)
 {
 	kern_return_t		kr;
 	vm_map_offset_t		page_map_offset;
 	vm_map_size_t		map_size;
 	vm_object_offset_t	object_offset;
 	int			i;
-	vm_map_entry_t		map_entry;
 
 
 	if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
+		assert(page->busy);
 		/*
 		 * Use one of the pre-allocated kernel virtual addresses
 		 * and just enter the VM page in the kernel address space
 		 * at that virtual address.
 		 */
-		vm_object_unlock(object);
 		simple_lock(&vm_paging_lock);
 
-		if (vm_paging_base_address == 0) {
-			/*
-			 * Initialize our pool of pre-allocated kernel
-			 * virtual addresses.
-			 */
-			simple_unlock(&vm_paging_lock);
-			page_map_offset = 0;
-			kr = vm_map_find_space(kernel_map,
-					       &page_map_offset,
-					       VM_PAGING_NUM_PAGES * PAGE_SIZE,
-					       0,
-					       0,
-					       &map_entry);
-			if (kr != KERN_SUCCESS) {
-				panic("vm_paging_map_object: "
-				      "kernel_map full\n");
-			}
-			map_entry->object.vm_object = kernel_object;
-			map_entry->offset =
-				page_map_offset - VM_MIN_KERNEL_ADDRESS;
-			vm_object_reference(kernel_object);
-			vm_map_unlock(kernel_map);
-
-			simple_lock(&vm_paging_lock);
-			if (vm_paging_base_address != 0) {
-				/* someone raced us and won: undo */
-				simple_unlock(&vm_paging_lock);
-				kr = vm_map_remove(kernel_map,
-						   page_map_offset,
-						   page_map_offset + 
-						   (VM_PAGING_NUM_PAGES
-						    * PAGE_SIZE),
-						   VM_MAP_NO_FLAGS);
-				assert(kr == KERN_SUCCESS);
-				simple_lock(&vm_paging_lock);
-			} else {
-				vm_paging_base_address = page_map_offset;
-			}
-		}
-
 		/*
 		 * Try and find an available kernel virtual address
 		 * from our pre-allocated pool.
 		 */
 		page_map_offset = 0;
-		for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
-			if (vm_paging_page_inuse[i] == FALSE) {
-				page_map_offset = vm_paging_base_address +
-					(i * PAGE_SIZE);
+		for (;;) {
+			for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
+				if (vm_paging_page_inuse[i] == FALSE) {
+					page_map_offset =
+						vm_paging_base_address +
+						(i * PAGE_SIZE);
+					break;
+				}
+			}
+			if (page_map_offset != 0) {
+				/* found a space to map our page ! */
+				break;
+			}
+
+			if (can_unlock_object) {
+				/*
+				 * If we can afford to unlock the VM object,
+				 * let's take the slow path now...
+				 */
 				break;
 			}
+			/*
+			 * We can't afford to unlock the VM object, so
+			 * let's wait for a space to become available...
+			 */
+			vm_paging_page_waiter_total++;
+			vm_paging_page_waiter++;
+			thread_sleep_fast_usimple_lock(&vm_paging_page_waiter,
+						       &vm_paging_lock,
+						       THREAD_UNINT);
+			vm_paging_page_waiter--;
+			/* ... and try again */
 		}
 
 		if (page_map_offset != 0) {
@@ -5401,10 +5126,18 @@ vm_paging_map_object(
 			}
 			vm_paging_page_inuse[i] = TRUE;
 			simple_unlock(&vm_paging_lock);
-			if (page->no_isync == TRUE) {
+
+			if (page->pmapped == FALSE) {
 				pmap_sync_page_data_phys(page->phys_page);
 			}
-			assert(pmap_verify_free(page->phys_page));
+			page->pmapped = TRUE;
+
+			/*
+			 * Keep the VM object locked over the PMAP_ENTER
+			 * and the actual use of the page by the kernel,
+			 * or this pmap mapping might get undone by a 
+			 * vm_object_pmap_protect() call...
+			 */
 			PMAP_ENTER(kernel_pmap,
 				   page_map_offset,
 				   page,
@@ -5415,7 +5148,6 @@ vm_paging_map_object(
 			vm_paging_objects_mapped++;
 			vm_paging_pages_mapped++; 
 			*address = page_map_offset;
-			vm_object_lock(object);
 
 			/* all done and mapped, ready to use ! */
 			return KERN_SUCCESS;
@@ -5428,7 +5160,10 @@ vm_paging_map_object(
 		 */
 		vm_paging_no_kernel_page++;
 		simple_unlock(&vm_paging_lock);
-		vm_object_lock(object);
+	}
+
+	if (! can_unlock_object) {
+		return KERN_NOT_SUPPORTED;
 	}
 
 	object_offset = vm_object_trunc_page(offset);
@@ -5439,13 +5174,6 @@ vm_paging_map_object(
 	 * in the kernel_map
 	 */
 
-	/* don't go beyond the object's end... */
-	if (object_offset >= object->size) {
-		map_size = 0;
-	} else if (map_size > object->size - offset) {
-		map_size = object->size - offset;
-	}
-
 	vm_object_reference_locked(object);	/* for the map entry */
 	vm_object_unlock(object);
 
@@ -5464,6 +5192,7 @@ vm_paging_map_object(
 		*address = 0;
 		*size = 0;
 		vm_object_deallocate(object);	/* for the map entry */
+		vm_object_lock(object);
 		return kr;
 	}
 
@@ -5473,6 +5202,13 @@ vm_paging_map_object(
 	 * Enter the mapped pages in the page table now.
 	 */
 	vm_object_lock(object);
+	/*
+	 * VM object must be kept locked from before PMAP_ENTER()
+	 * until after the kernel is done accessing the page(s).
+	 * Otherwise, the pmap mappings in the kernel could be
+	 * undone by a call to vm_object_pmap_protect().
+	 */
+
 	for (page_map_offset = 0;
 	     map_size != 0;
 	     map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
@@ -5480,14 +5216,23 @@ vm_paging_map_object(
 
 		page = vm_page_lookup(object, offset + page_map_offset);
 		if (page == VM_PAGE_NULL) {
-			panic("vm_paging_map_object: no page !?");
+			printf("vm_paging_map_object: no page !?");
+			vm_object_unlock(object);
+			kr = vm_map_remove(kernel_map, *address, *size,
+					   VM_MAP_NO_FLAGS);
+			assert(kr == KERN_SUCCESS);
+			*address = 0;
+			*size = 0;
+			vm_object_lock(object);
+			return KERN_MEMORY_ERROR;
 		}
-		if (page->no_isync == TRUE) {
+		if (page->pmapped == FALSE) {
 			pmap_sync_page_data_phys(page->phys_page);
 		}
+		page->pmapped = TRUE;
 		cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK;
 
-		assert(pmap_verify_free(page->phys_page));
+		//assert(pmap_verify_free(page->phys_page));
 		PMAP_ENTER(kernel_pmap,
 			   *address + page_map_offset,
 			   page,
@@ -5523,7 +5268,7 @@ vm_paging_unmap_object(
 	if ((vm_paging_base_address == 0) ||
 	    (start < vm_paging_base_address) ||
 	    (end > (vm_paging_base_address
-		    + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
+		     + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
 		/*
 		 * We didn't use our pre-allocated pool of
 		 * kernel virtual address.  Deallocate the
@@ -5551,10 +5296,14 @@ vm_paging_unmap_object(
 
 		simple_lock(&vm_paging_lock);
 		vm_paging_page_inuse[i] = FALSE;
+		if (vm_paging_page_waiter) {
+			thread_wakeup(&vm_paging_page_waiter);
+		}
 		simple_unlock(&vm_paging_lock);
 	}
 }
 
+#if CRYPTO
 /*
  * Encryption data.
  * "iv" is the "initial vector".  Ideally, we want to
@@ -5678,10 +5427,7 @@ vm_page_encrypt(
 	vm_page_t	page,
 	vm_map_offset_t	kernel_mapping_offset)
 {
-        int			clear_refmod = 0;
 	kern_return_t		kr;
-	boolean_t		page_was_referenced;
-	boolean_t		page_was_modified;
 	vm_map_size_t		kernel_mapping_size;
 	vm_offset_t		kernel_vaddr;
 	union {
@@ -5709,15 +5455,11 @@ vm_page_encrypt(
 	ASSERT_PAGE_DECRYPTED(page);
 
 	/*
-	 * Gather the "reference" and "modified" status of the page.
-	 * We'll restore these values after the encryption, so that
-	 * the encryption is transparent to the rest of the system
-	 * and doesn't impact the VM's LRU logic.
+	 * Take a paging-in-progress reference to keep the object
+	 * alive even if we have to unlock it (in vm_paging_map_object()
+	 * for example)...
 	 */
-	page_was_referenced =
-		(page->reference || pmap_is_referenced(page->phys_page));
-	page_was_modified = 
-		(page->dirty || pmap_is_modified(page->phys_page));
+	vm_object_paging_begin(page->object);
 
 	if (kernel_mapping_offset == 0) {
 		/*
@@ -5730,7 +5472,8 @@ vm_page_encrypt(
 					  page,
 					  page->object,
 					  page->offset,
-					  &kernel_mapping_size);
+					  &kernel_mapping_size,
+					  FALSE);
 		if (kr != KERN_SUCCESS) {
 			panic("vm_page_encrypt: "
 			      "could not map page in kernel: 0x%x\n",
@@ -5758,8 +5501,6 @@ vm_page_encrypt(
 	encrypt_iv.vm.paging_offset =
 		page->object->paging_offset + page->offset;
 
-	vm_object_unlock(page->object);
-
 	/* encrypt the "initial vector" */
 	aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0],
 			swap_crypt_null_iv,
@@ -5778,8 +5519,6 @@ vm_page_encrypt(
 
 	vm_page_encrypt_counter++;
 
-	vm_object_lock(page->object);
-
 	/*
 	 * Unmap the page from the kernel's address space,
 	 * if we had to map it ourselves.  Otherwise, let
@@ -5792,22 +5531,20 @@ vm_page_encrypt(
 	}
 
 	/*
-	 * Restore the "reference" and "modified" bits.
+	 * Clear the "reference" and "modified" bits.
 	 * This should clean up any impact the encryption had
 	 * on them.
+	 * The page was kept busy and disconnected from all pmaps,
+	 * so it can't have been referenced or modified from user
+	 * space.
+	 * The software bits will be reset later after the I/O
+	 * has completed (in upl_commit_range()).
 	 */
-	if (! page_was_referenced) {
-		clear_refmod |= VM_MEM_REFERENCED;
-		page->reference = FALSE;
-	}
-	if (! page_was_modified) {
-		clear_refmod |= VM_MEM_MODIFIED;
-		page->dirty = FALSE;
-	}
-	if (clear_refmod)
-	        pmap_clear_refmod(page->phys_page, clear_refmod);
+	pmap_clear_refmod(page->phys_page, VM_MEM_REFERENCED | VM_MEM_MODIFIED);
 
 	page->encrypted = TRUE;
+
+	vm_object_paging_end(page->object);
 }
 
 /*
@@ -5827,11 +5564,9 @@ vm_page_decrypt(
 	vm_page_t	page,
 	vm_map_offset_t	kernel_mapping_offset)
 {
-        int			clear_refmod = 0;
 	kern_return_t		kr;
 	vm_map_size_t		kernel_mapping_size;
 	vm_offset_t		kernel_vaddr;
-	boolean_t		page_was_referenced;
 	union {
 		unsigned char	aes_iv[AES_BLOCK_SIZE];
 		struct {
@@ -5844,13 +5579,11 @@ vm_page_decrypt(
 	assert(page->encrypted);
 
 	/*
-	 * Gather the "reference" status of the page.
-	 * We'll restore its value after the decryption, so that
-	 * the decryption is transparent to the rest of the system
-	 * and doesn't impact the VM's LRU logic.
+	 * Take a paging-in-progress reference to keep the object
+	 * alive even if we have to unlock it (in vm_paging_map_object()
+	 * for example)...
 	 */
-	page_was_referenced =
-		(page->reference || pmap_is_referenced(page->phys_page));
+	vm_object_paging_begin(page->object);
 
 	if (kernel_mapping_offset == 0) {
 		/*
@@ -5863,10 +5596,12 @@ vm_page_decrypt(
 					  page,
 					  page->object,
 					  page->offset,
-					  &kernel_mapping_size);
+					  &kernel_mapping_size,
+					  FALSE);
 		if (kr != KERN_SUCCESS) {
 			panic("vm_page_decrypt: "
-			      "could not map page in kernel: 0x%x\n");
+			      "could not map page in kernel: 0x%x\n",
+			      kr);
 		}
 	} else {
 		kernel_mapping_size = 0;
@@ -5885,8 +5620,6 @@ vm_page_decrypt(
 	decrypt_iv.vm.paging_offset =
 		page->object->paging_offset + page->offset;
 
-	vm_object_unlock(page->object);
-
 	/* encrypt the "initial vector" */
 	aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0],
 			swap_crypt_null_iv,
@@ -5904,8 +5637,6 @@ vm_page_decrypt(
 			&swap_crypt_ctx.decrypt);
 	vm_page_decrypt_counter++;
 
-	vm_object_lock(page->object);
-
 	/*
 	 * Unmap the page from the kernel's address space,
 	 * if we had to map it ourselves.  Otherwise, let
@@ -5925,14 +5656,7 @@ vm_page_decrypt(
 	 * and the decryption doesn't count.
 	 */
 	page->dirty = FALSE;
-	clear_refmod = VM_MEM_MODIFIED;
-
-	/* restore the "reference" bit */
-	if (! page_was_referenced) {
-		page->reference = FALSE;
-		clear_refmod |= VM_MEM_REFERENCED;
-	}
-	pmap_clear_refmod(page->phys_page, clear_refmod);
+	pmap_clear_refmod(page->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
 
 	page->encrypted = FALSE;
 
@@ -5947,12 +5671,13 @@ vm_page_decrypt(
 	/*
 	 * Since the page is not mapped yet, some code might assume that it
 	 * doesn't need to invalidate the instruction cache when writing to
-	 * that page.  That code relies on "no_isync" being set, so that the
-	 * caches get syncrhonized when the page is first mapped.  So we need
-	 * to set "no_isync" here too, despite the fact that we just
-	 * synchronized the caches above...
+	 * that page.  That code relies on "pmapped" being FALSE, so that the
+	 * caches get synchronized when the page is first mapped.
 	 */
-	page->no_isync = TRUE;
+	assert(pmap_verify_free(page->phys_page));
+	page->pmapped = FALSE;
+
+	vm_object_paging_end(page->object);
 }
 
 unsigned long upl_encrypt_upls = 0;
@@ -5983,14 +5708,10 @@ upl_encrypt(
 	upl_encrypt_upls++;
 	upl_encrypt_pages += crypt_size / PAGE_SIZE;
 
-	upl_lock(upl);
-
 	upl_object = upl->map_object;
 	upl_offset = upl->offset;
 	upl_size = upl->size;
 
-	upl_unlock(upl);
-
 	vm_object_lock(upl_object);
 
 	/*
@@ -6014,32 +5735,16 @@ upl_encrypt(
 	paging_offset = shadow_object->paging_offset;
 	vm_object_paging_begin(shadow_object);
 
-	if (shadow_object != upl_object) {
-		vm_object_unlock(shadow_object);
-	}
-	vm_object_unlock(upl_object);
+	if (shadow_object != upl_object)
+	        vm_object_unlock(upl_object);
+
 
 	base_offset = shadow_offset;
 	base_offset += upl_offset;
 	base_offset += crypt_offset;
 	base_offset -= paging_offset;
-	/*
-	 * Unmap the pages, so that nobody can continue accessing them while
-	 * they're encrypted.  After that point, all accesses to these pages
-	 * will cause a page fault and block while the page is being encrypted
-	 * (busy).  After the encryption completes, any access will cause a
-	 * page fault and the page gets decrypted at that time.
-	 */
-	assert(crypt_offset + crypt_size <= upl_size);
-	vm_object_pmap_protect(shadow_object, 
-			       base_offset,
-			       (vm_object_size_t)crypt_size,
-			       PMAP_NULL,
-			       0,
-			       VM_PROT_NONE);
 
-	/* XXX FBDP could the object have changed significantly here ? */
-	vm_object_lock(shadow_object);
+	assert(crypt_offset + crypt_size <= upl_size);
 
 	for (upl_offset = 0;
 	     upl_offset < crypt_size;
@@ -6053,13 +5758,58 @@ upl_encrypt(
 			      base_offset,
 			      upl_offset);
 		}
+		/*
+		 * Disconnect the page from all pmaps, so that nobody can
+		 * access it while it's encrypted.  After that point, all
+		 * accesses to this page will cause a page fault and block
+		 * while the page is busy being encrypted.  After the
+		 * encryption completes, any access will cause a
+		 * page fault and the page gets decrypted at that time.
+		 */
+		pmap_disconnect(page->phys_page);
 		vm_page_encrypt(page, 0);
+
+		if (shadow_object == vm_pageout_scan_wants_object) {
+			/*
+			 * Give vm_pageout_scan() a chance to convert more
+			 * pages from "clean-in-place" to "clean-and-free",
+			 * if it's interested in the same pages we selected
+			 * in this cluster.
+			 */
+			vm_object_unlock(shadow_object);
+			vm_object_lock(shadow_object);
+		}
 	}
 
 	vm_object_paging_end(shadow_object);
 	vm_object_unlock(shadow_object);
 }
 
+#else /* CRYPTO */
+void
+upl_encrypt(
+	__unused upl_t			upl,
+	__unused upl_offset_t	crypt_offset,
+	__unused upl_size_t	crypt_size)
+{
+}
+
+void
+vm_page_encrypt(
+	__unused vm_page_t		page,
+	__unused vm_map_offset_t	kernel_mapping_offset)
+{
+} 
+
+void
+vm_page_decrypt(
+	__unused vm_page_t		page,
+	__unused vm_map_offset_t	kernel_mapping_offset)
+{
+}
+
+#endif /* CRYPTO */
+
 vm_size_t
 upl_get_internal_pagelist_offset(void)
 {
@@ -6081,10 +5831,18 @@ upl_clear_dirty(
 
 #ifdef MACH_BSD
 
+boolean_t  upl_device_page(upl_page_info_t *upl)
+{
+	return(UPL_DEVICE_PAGE(upl));
+}
 boolean_t  upl_page_present(upl_page_info_t *upl, int index)
 {
 	return(UPL_PAGE_PRESENT(upl, index));
 }
+boolean_t  upl_speculative_page(upl_page_info_t *upl, int index)
+{
+	return(UPL_SPECULATIVE_PAGE(upl, index));
+}
 boolean_t  upl_dirty_page(upl_page_info_t *upl, int index)
 {
 	return(UPL_DIRTY_PAGE(upl, index));
@@ -6098,6 +5856,7 @@ ppnum_t  upl_phys_page(upl_page_info_t *upl, int index)
 	return(UPL_PHYS_PAGE(upl, index));
 }
 
+
 void
 vm_countdirtypages(void)
 {
@@ -6127,6 +5886,21 @@ vm_countdirtypages(void)
 	} while (!queue_end(&vm_page_queue_inactive,(queue_entry_t) m));
 	vm_page_unlock_queues();
 
+	vm_page_lock_queues();
+	m = (vm_page_t) queue_first(&vm_page_queue_throttled);
+	do {
+		if (m ==(vm_page_t )0) break;
+
+		dpages++;
+		assert(m->dirty);
+		assert(!m->pageout);
+		assert(m->object != kernel_object);
+		m = (vm_page_t) queue_next(&m->pageq);
+		if (m ==(vm_page_t )0) break;
+
+	} while (!queue_end(&vm_page_queue_throttled,(queue_entry_t) m));
+	vm_page_unlock_queues();
+
 	vm_page_lock_queues();
 	m = (vm_page_t) queue_first(&vm_page_queue_zf);
 	do {
@@ -6171,9 +5945,9 @@ vm_countdirtypages(void)
 #endif /* MACH_BSD */
 
 ppnum_t upl_get_highest_page(
-	upl_t			upl)
+			     upl_t			upl)
 {
-	return upl->highest_page;
+        return upl->highest_page;
 }
 
 #ifdef UPL_DEBUG
diff --git a/osfmk/vm/vm_pageout.h b/osfmk/vm/vm_pageout.h
index 0c54ccc31..c7ab4ca8f 100644
--- a/osfmk/vm/vm_pageout.h
+++ b/osfmk/vm/vm_pageout.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -88,25 +94,18 @@ extern ppnum_t upl_get_highest_page(
 
 extern unsigned int	vm_pageout_scan_event_counter;
 extern unsigned int	vm_zf_count;
+extern unsigned int	vm_zf_queue_count;
 
 /*
  *	Routines exported to Mach.
  */
 extern void		vm_pageout(void);
 
-extern vm_object_t	vm_pageout_object_allocate(
-					vm_page_t		m,
-					vm_size_t		size,
-					vm_object_offset_t	offset);
+extern kern_return_t	vm_pageout_internal_start(void);
 
 extern void		vm_pageout_object_terminate(
 					vm_object_t	object);
 
-extern vm_page_t	vm_pageout_setup(
-					vm_page_t		m,
-					vm_object_t		new_object,
-					vm_object_offset_t	new_offset);
-
 extern void		vm_pageout_cluster(
 					vm_page_t	m);
 
@@ -119,12 +118,6 @@ extern void		vm_pageclean_setup(
 					vm_object_t		new_object,
 					vm_object_offset_t	new_offset);
 
-extern void		vm_pageclean_copy(
-					vm_page_t		m,
-					vm_page_t		new_m,
-					vm_object_t		new_object,
-					vm_object_offset_t	new_offset);
-
 /* UPL exported routines and structures */
 
 #define upl_lock_init(object)	mutex_init(&(object)->Lock, 0)
@@ -164,7 +157,7 @@ struct upl {
 #define UPL_IO_WIRE		0x200
 #define UPL_ACCESS_BLOCKED	0x400
 #define UPL_ENCRYPTED		0x800
-
+#define UPL_SHADOWED		0x1000
 
 /* flags for upl_create flags parameter */
 #define UPL_CREATE_EXTERNAL	0
@@ -242,7 +235,8 @@ extern kern_return_t vm_paging_map_object(
 	vm_page_t		page,
 	vm_object_t		object,
 	vm_object_offset_t	offset,
-	vm_map_size_t		*size);
+	vm_map_size_t		*size,
+	boolean_t		can_unlock_object);
 extern void vm_paging_unmap_object(
 	vm_object_t		object,
 	vm_map_offset_t		start,
diff --git a/osfmk/vm/vm_print.h b/osfmk/vm/vm_print.h
index a96b03a0c..b31deda03 100644
--- a/osfmk/vm/vm_print.h
+++ b/osfmk/vm/vm_print.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -40,11 +46,7 @@ extern void	vm_map_copy_print(
 extern int	vm_follow_object(
 			vm_object_t	object);
 
-extern void	vm_object_print(
-			db_addr_t	object,
-			boolean_t	have_addr,
-			int		arg_count,
-			char		*modif);
+extern void vm_object_print(db_expr_t, boolean_t, db_expr_t, char *);
 
 #include <vm/vm_page.h>
 
diff --git a/osfmk/vm/vm_protos.h b/osfmk/vm/vm_protos.h
index 2d6e13c36..f1d0f65af 100644
--- a/osfmk/vm/vm_protos.h
+++ b/osfmk/vm/vm_protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifdef	XNU_KERNEL_PRIVATE
@@ -66,6 +72,8 @@ extern int default_pager_init_flag;
 /* these should be exported cleanly from OSFMK since BSD needs them */
 extern ipc_port_t convert_task_to_port(
 	task_t		task);
+extern ipc_port_t convert_thread_to_port(
+	thread_t		thread);
 extern ipc_port_t convert_task_name_to_port(
 	task_name_t	task_name);
 #endif /* _KERN_IPC_TT_H_ */
@@ -136,13 +144,7 @@ extern mach_vm_offset_t mach_get_vm_end(vm_map_t);
 extern vm_offset_t get_vm_start(vm_map_t);
 extern vm_offset_t get_vm_end(vm_map_t);
 
-/*
- * LP64todo - map in the commpage cleanly and remove these.
- */
-extern void vm_map_commpage64( vm_map_t );
-extern void vm_map_remove_commpage( vm_map_t );
 #ifdef __i386__
-extern void vm_map_commpage32(vm_map_t);
 extern kern_return_t vm_map_apple_protected(
 	vm_map_t	map,
 	vm_map_offset_t	start,
@@ -157,24 +159,6 @@ extern void apple_protect_pager_map(memory_object_t mem_obj);
  * bsd
  */
 struct vnode;
-extern int is_suser(void);
-extern int bsd_read_page_cache_file(
-	unsigned int	user,
-	int		*fid,
-	int		*mod,
-	char		*app_name,
-	struct vnode	*app_vp,
-	vm_offset_t	*buffer,
-	vm_offset_t	*bufsize);
-extern int bsd_write_page_cache_file(
-	unsigned int	user,
-	char	 	*file_name,
-	caddr_t		buffer,
-	vm_size_t	size,
-	int		mod,
-	int		fid);
-extern int prepare_profile_database(
-	int	user);
 extern void vnode_pager_shutdown(void);
 extern void *upl_get_internal_page_list(
 	upl_t upl);
@@ -198,10 +182,13 @@ extern kern_return_t vnode_pager_get_pathname(
 	vm_size_t	*length_p);
 extern kern_return_t vnode_pager_get_filename(
 	struct vnode	*vp,
-	char		**filename);
+	const char	**filename);
+extern kern_return_t vnode_pager_get_cs_blobs(
+	struct vnode	*vp,
+	void		**blobs);
 	
 #endif /* _VNODE_PAGER_ */
-extern void vnode_pager_bootstrap(void);
+extern void vnode_pager_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 extern kern_return_t
 vnode_pager_data_unlock(
 	memory_object_t		mem_obj,
@@ -221,12 +208,16 @@ extern kern_return_t vnode_pager_get_object_pathname(
 	vm_size_t	*length_p);
 extern kern_return_t vnode_pager_get_object_filename(
 	memory_object_t	mem_obj,
-	char		**filename);
+	const char	**filename);
+extern kern_return_t vnode_pager_get_object_cs_blobs(
+	memory_object_t	mem_obj,
+	void		**blobs);
 extern kern_return_t vnode_pager_data_request( 
 	memory_object_t, 
-	memory_object_offset_t, 
-	vm_size_t, 
-	vm_prot_t);
+	memory_object_offset_t,
+	vm_size_t,
+	vm_prot_t,
+	memory_object_fault_info_t);
 extern kern_return_t vnode_pager_data_return(
 	memory_object_t,
 	memory_object_offset_t,
@@ -268,7 +259,10 @@ extern kern_return_t   dp_memory_object_init(memory_object_t,
 					     vm_size_t);
 extern	kern_return_t dp_memory_object_terminate(memory_object_t);
 extern	kern_return_t   dp_memory_object_data_request(memory_object_t, 
-			memory_object_offset_t, vm_size_t, vm_prot_t);
+						      memory_object_offset_t,
+						      vm_size_t,
+						      vm_prot_t,
+						      memory_object_fault_info_t);
 extern kern_return_t dp_memory_object_data_return(memory_object_t,
 						    memory_object_offset_t,
 						    vm_size_t,
@@ -306,7 +300,8 @@ extern	kern_return_t device_pager_terminate(memory_object_t);
 extern	kern_return_t   device_pager_data_request(memory_object_t, 
 						  memory_object_offset_t,
 						  vm_size_t,
-						  vm_prot_t);
+						  vm_prot_t,
+						  memory_object_fault_info_t);
 extern kern_return_t device_pager_data_return(memory_object_t,
 					      memory_object_offset_t,
 					      vm_size_t,
@@ -337,7 +332,7 @@ extern memory_object_t device_pager_setup(
 	int,
 	vm_size_t,
 	int);
-extern void device_pager_bootstrap(void);
+extern void device_pager_bootstrap(void) __attribute__((section("__TEXT, initcode")));
 
 extern kern_return_t memory_object_create_named(
 	memory_object_t	pager,
@@ -351,7 +346,39 @@ extern int macx_swapinfo(
 	vm_size_t		*pagesize_p,
 	boolean_t		*encrypted_p);
 
-extern void log_nx_failure(addr64_t vaddr, vm_prot_t prot);
+extern void log_stack_execution_failure(addr64_t vaddr, vm_prot_t prot);
+extern int cs_invalid_page(void);
+extern boolean_t cs_validate_page(void *blobs,
+				  memory_object_offset_t offset, 
+				  const void *data,
+				  boolean_t *tainted);
+
+extern kern_return_t mach_memory_entry_purgable_control(
+	ipc_port_t	entry_port,
+	vm_purgable_t	control,
+	int		*state);
+
+extern kern_return_t mach_memory_entry_page_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset,
+	int			ops,
+	ppnum_t			*phys_entry,
+	int			*flags);
+
+extern kern_return_t mach_memory_entry_range_op(
+	ipc_port_t		entry_port,
+	vm_object_offset_t	offset_beg,
+	vm_object_offset_t	offset_end,
+	int                     ops,
+	int                     *range);
+
+extern void mach_memory_entry_port_release(ipc_port_t port);
+extern void mach_destroy_memory_entry(ipc_port_t port);
+extern kern_return_t mach_memory_entry_allocate(
+	struct vm_named_entry **user_entry_p,
+	ipc_port_t *user_handle_p);
+
+extern void vm_paging_map_init(void);
 
 #endif	/* _VM_VM_PROTOS_H_ */
 
diff --git a/osfmk/vm/vm_purgeable.c b/osfmk/vm/vm_purgeable.c
new file mode 100644
index 000000000..df1f3f810
--- /dev/null
+++ b/osfmk/vm/vm_purgeable.c
@@ -0,0 +1,563 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+#include <mach/mach_types.h>
+#include <vm/vm_page.h>
+#include <vm/vm_purgeable_internal.h>
+#include <sys/kdebug.h>
+
+struct token {
+	token_cnt_t     count;
+	token_idx_t     next;
+};
+
+struct token    tokens[MAX_VOLATILE];
+
+token_idx_t     token_free_idx = 0;	/* head of free queue */
+token_cnt_t     token_init_count = 1;	/* token 0 is reserved!! */
+token_cnt_t     token_new_pagecount = 0;	/* count of pages that will
+						 * be added onto token queue */
+
+int             available_for_purge = 0;	/* increase when ripe token
+						 * added, decrease when ripe
+						 * token removed protect with
+						 * page_queue_lock */
+
+struct purgeable_q purgeable_queues[PURGEABLE_Q_TYPE_MAX];
+
+#define TOKEN_ADD           0x40/* 0x100 */
+#define TOKEN_DELETE        0x41/* 0x104 */
+#define TOKEN_QUEUE_ADVANCE 0x42/* 0x108 actually means "token ripened" */
+#define TOKEN_OBJECT_PURGED 0x43/* 0x10c */
+#define OBJECT_ADDED        0x50/* 0x140 */
+#define OBJECT_REMOVED      0x51/* 0x144 */
+
+static void     vm_purgeable_q_advance(uint32_t num_pages, purgeable_q_t queue);
+static token_idx_t vm_purgeable_token_remove_first(purgeable_q_t queue);
+
+#if MACH_ASSERT
+static void
+vm_purgeable_token_check_queue(purgeable_q_t queue)
+{
+	int             token_cnt = 0, page_cnt = 0;
+	token_idx_t     token = queue->token_q_head;
+	token_idx_t     unripe = 0;
+	int             our_inactive_count;
+
+	while (token) {
+		if (tokens[token].count != 0) {
+			assert(queue->token_q_unripe);
+			if (unripe == 0) {
+				assert(token == queue->token_q_unripe);
+				unripe = token;
+			}
+			page_cnt += tokens[token].count;
+		}
+		if (tokens[token].next == 0)
+			assert(queue->token_q_tail == token);
+
+		token_cnt++;
+		token = tokens[token].next;
+	}
+
+	if (unripe)
+		assert(queue->token_q_unripe == unripe);
+	assert(token_cnt == queue->debug_count_tokens);
+	our_inactive_count = page_cnt + queue->new_pages + token_new_pagecount;
+	assert(our_inactive_count >= 0);
+	assert((uint32_t) our_inactive_count == vm_page_inactive_count);
+}
+#endif
+
+kern_return_t
+vm_purgeable_token_add(purgeable_q_t queue)
+{
+	/* new token */
+	token_idx_t     token;
+	enum purgeable_q_type i;
+
+	if (token_init_count < MAX_VOLATILE) {	/* lazy token array init */
+		token = token_init_count;
+		token_init_count++;
+	} else if (token_free_idx) {
+		token = token_free_idx;
+		token_free_idx = tokens[token_free_idx].next;
+	} else {
+		return KERN_FAILURE;
+	}
+
+	/*
+	 * the new pagecount we got need to be applied to all queues except
+	 * obsolete
+	 */
+	for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
+		purgeable_queues[i].new_pages += token_new_pagecount;
+		assert(purgeable_queues[i].new_pages >= 0);
+		assert((uint64_t) (purgeable_queues[i].new_pages) <= TOKEN_COUNT_MAX);
+	}
+	token_new_pagecount = 0;
+
+	/* set token counter value */
+	if (queue->type != PURGEABLE_Q_TYPE_OBSOLETE)
+		tokens[token].count = queue->new_pages;
+	else
+		tokens[token].count = 0;	/* all obsolete items are
+						 * ripe immediately */
+	queue->new_pages = 0;
+
+	/* put token on token counter list */
+	tokens[token].next = 0;
+	if (queue->token_q_tail == 0) {
+		assert(queue->token_q_head == 0 && queue->token_q_unripe == 0);
+		queue->token_q_head = token;
+	} else {
+		tokens[queue->token_q_tail].next = token;
+	}
+	if (queue->token_q_unripe == 0) {	/* only ripe tokens (token
+						 * count == 0) in queue */
+		if (tokens[token].count > 0)
+			queue->token_q_unripe = token;	/* first unripe token */
+		else
+			available_for_purge++;	/* added a ripe token?
+						 * increase available count */
+	}
+	queue->token_q_tail = token;
+
+#if MACH_ASSERT
+	queue->debug_count_tokens++;
+	/* Check both queues, since we modified the new_pages count on each */
+	vm_purgeable_token_check_queue(&purgeable_queues[PURGEABLE_Q_TYPE_FIFO]);
+	vm_purgeable_token_check_queue(&purgeable_queues[PURGEABLE_Q_TYPE_LIFO]);
+
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_ADD)),
+			      queue->type,
+			      tokens[token].count,	/* num pages on token
+							 * (last token) */
+			      queue->debug_count_tokens,
+			      0,
+			      0);
+#endif
+
+	return KERN_SUCCESS;
+}
+
+/*
+ * Remove first token from queue and return its index. Add its count to the
+ * count of the next token.
+ */
+static token_idx_t 
+vm_purgeable_token_remove_first(purgeable_q_t queue)
+{
+	token_idx_t     token;
+	token = queue->token_q_head;
+
+	assert(token);
+
+	if (token) {
+		assert(queue->token_q_tail);
+		if (queue->token_q_head == queue->token_q_unripe) {
+			/* no ripe tokens... must move unripe pointer */
+			queue->token_q_unripe = tokens[token].next;
+		} else {
+			/* we're removing a ripe token. decrease count */
+			available_for_purge--;
+			assert(available_for_purge >= 0);
+		}
+
+		if (queue->token_q_tail == queue->token_q_head)
+			assert(tokens[token].next == 0);
+
+		queue->token_q_head = tokens[token].next;
+		if (queue->token_q_head) {
+			tokens[queue->token_q_head].count += tokens[token].count;
+		} else {
+			/* currently no other tokens in the queue */
+			/*
+			 * the page count must be added to the next newly
+			 * created token
+			 */
+			queue->new_pages += tokens[token].count;
+			/* if head is zero, tail is too */
+			queue->token_q_tail = 0;
+		}
+
+#if MACH_ASSERT
+		queue->debug_count_tokens--;
+		vm_purgeable_token_check_queue(queue);
+
+		KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_DELETE)),
+				      queue->type,
+				      tokens[queue->token_q_head].count,	/* num pages on new
+										 * first token */
+				      token_new_pagecount,	/* num pages waiting for
+								 * next token */
+				      available_for_purge,
+				      0);
+#endif
+	}
+	return token;
+}
+
+/* Delete first token from queue. Return token to token queue. */
+void
+vm_purgeable_token_delete_first(purgeable_q_t queue)
+{
+	token_idx_t     token = vm_purgeable_token_remove_first(queue);
+
+	if (token) {
+		/* stick removed token on free queue */
+		tokens[token].next = token_free_idx;
+		token_free_idx = token;
+	}
+}
+
+
+void
+vm_purgeable_q_advance_all(uint32_t num_pages)
+{
+	/*
+	 * don't need to advance obsolete queue - all items are ripe there,
+	 * always
+	 */
+	vm_purgeable_q_advance(num_pages, &purgeable_queues[PURGEABLE_Q_TYPE_FIFO]);
+	vm_purgeable_q_advance(num_pages, &purgeable_queues[PURGEABLE_Q_TYPE_LIFO]);
+}
+
+/*
+ * Decrements token counters. A token counter can be zero, this means the
+ * object is ripe to be purged. It is not purged immediately, because that
+ * could cause several objects to be purged even if purging one would satisfy
+ * the memory needs. Instead, the pageout thread purges one after the other
+ * by calling vm_purgeable_object_purge_one and then rechecking the memory
+ * balance.
+ */
+static void
+vm_purgeable_q_advance(uint32_t num_pages, purgeable_q_t queue)
+{
+	/* Iterate over tokens as long as there are unripe tokens. */
+	while (queue->token_q_unripe) {
+		int             min = (tokens[queue->token_q_unripe].count < num_pages) ?
+		tokens[queue->token_q_unripe].count : num_pages;
+		tokens[queue->token_q_unripe].count -= min;
+		num_pages -= min;
+
+		if (tokens[queue->token_q_unripe].count == 0) {
+			queue->token_q_unripe = tokens[queue->token_q_unripe].next;
+			available_for_purge++;
+			KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_QUEUE_ADVANCE)),
+					      queue->type,
+					  tokens[queue->token_q_head].count,	/* num pages on new
+										 * first token */
+					      0,
+					      available_for_purge,
+					      0);
+			continue;	/* One token ripened. Make sure to
+					 * check the next. */
+		}
+		if (num_pages == 0)
+			break;	/* Current token not ripe and no more pages.
+				 * Work done. */
+	}
+
+	/*
+	 * if there are no unripe tokens in the queue, decrement the
+	 * new_pages counter instead new_pages can be negative, but must be
+	 * canceled out by token_new_pagecount -- since inactive queue as a
+	 * whole always contains a nonnegative number of pages
+	 */
+	if (!queue->token_q_unripe) {
+		queue->new_pages -= num_pages;
+		assert((int32_t) token_new_pagecount + queue->new_pages >= 0);
+	}
+#if MACH_ASSERT
+	vm_purgeable_token_check_queue(queue);
+#endif
+}
+
+/*
+ * grab any ripe object and purge it obsolete queue first. then, go through
+ * each volatile group. Select a queue with a ripe token.
+ * Start with first group (0)
+ * 1. Look at queue. Is there an object?
+ *   Yes - purge it. Remove token.
+ *   No - check other queue. Is there an object?
+ *     No - increment group, then go to (1)
+ *     Yes - purge it. Remove token. If there is no ripe token, remove ripe
+ *      token from other queue and migrate unripe token from this
+ *      queue to other queue.
+ */
+static void
+vm_purgeable_token_remove_ripe(purgeable_q_t queue)
+{
+	assert(queue->token_q_head && tokens[queue->token_q_head].count == 0);
+	/* return token to free list. advance token list. */
+	token_idx_t     new_head = tokens[queue->token_q_head].next;
+	tokens[queue->token_q_head].next = token_free_idx;
+	token_free_idx = queue->token_q_head;
+	queue->token_q_head = new_head;
+	if (new_head == 0)
+		queue->token_q_tail = 0;
+
+#if MACH_ASSERT
+	queue->debug_count_tokens--;
+	vm_purgeable_token_check_queue(queue);
+#endif
+
+	available_for_purge--;
+	assert(available_for_purge >= 0);
+}
+
+/*
+ * Delete a ripe token from the given queue. If there are no ripe tokens on
+ * that queue, delete a ripe token from queue2, and migrate an unripe token
+ * from queue to queue2
+ */
+static void
+vm_purgeable_token_choose_and_delete_ripe(purgeable_q_t queue, purgeable_q_t queue2)
+{
+	assert(queue->token_q_head);
+
+	if (tokens[queue->token_q_head].count == 0) {
+		/* This queue has a ripe token. Remove. */
+		vm_purgeable_token_remove_ripe(queue);
+	} else {
+		assert(queue2);
+		/*
+		 * queue2 must have a ripe token. Remove, and migrate one
+		 * from queue to queue2.
+		 */
+		vm_purgeable_token_remove_ripe(queue2);
+		/* migrate unripe token */
+		token_idx_t     token;
+		token_cnt_t     count;
+
+		/* remove token from queue1 */
+		assert(queue->token_q_unripe == queue->token_q_head);	/* queue1 had no unripe
+									 * tokens, remember? */
+		token = vm_purgeable_token_remove_first(queue);
+		assert(token);
+
+		count = tokens[token].count;
+
+		/* migrate to queue2 */
+		/* go to migration target loc */
+		token_idx_t    *token_in_queue2 = &queue2->token_q_head;
+		while (*token_in_queue2 && count > tokens[*token_in_queue2].count) {
+			count -= tokens[*token_in_queue2].count;
+			token_in_queue2 = &tokens[*token_in_queue2].next;
+		}
+
+		if ((*token_in_queue2 == queue2->token_q_unripe) ||	/* becomes the first
+									 * unripe token */
+		    (queue2->token_q_unripe == 0))
+			queue2->token_q_unripe = token;	/* must update unripe
+							 * pointer */
+
+		/* insert token */
+		tokens[token].count = count;
+		tokens[token].next = *token_in_queue2;
+
+		/*
+		 * if inserting at end, reduce new_pages by that value if
+		 * inserting before token, reduce counter of that token
+		 */
+		if (*token_in_queue2 == 0) {	/* insertion at end of queue2 */
+			queue2->token_q_tail = token;	/* must update tail
+							 * pointer */
+			assert(queue2->new_pages >= (int32_t) count);
+			queue2->new_pages -= count;
+		} else {
+			assert(tokens[*token_in_queue2].count >= count);
+			tokens[*token_in_queue2].count -= count;
+		}
+		*token_in_queue2 = token;
+
+#if MACH_ASSERT
+		queue2->debug_count_tokens++;
+		vm_purgeable_token_check_queue(queue2);
+#endif
+	}
+}
+
+/* Find an object that can be locked. Returns locked object. */
+static          vm_object_t
+vm_purgeable_object_find_and_lock(purgeable_q_t queue, int group)
+{
+	/*
+	 * Usually we would pick the first element from a queue. However, we
+	 * might not be able to get a lock on it, in which case we try the
+	 * remaining elements in order.
+	 */
+
+	vm_object_t     object;
+	for (object = (vm_object_t) queue_first(&queue->objq[group]);
+	     !queue_end(&queue->objq[group], (queue_entry_t) object);
+	     object = (vm_object_t) queue_next(&object->objq)) {
+		if (vm_object_lock_try(object)) {
+			/* Locked. Great. We'll take it. Remove and return. */
+			queue_remove(&queue->objq[group], object,
+				     vm_object_t, objq);
+			object->objq.next = 0;
+			object->objq.prev = 0;
+#if MACH_ASSERT
+			queue->debug_count_objects--;
+#endif
+			return object;
+		}
+	}
+
+	return 0;
+}
+
+void
+vm_purgeable_object_purge_one(void)
+{
+	enum purgeable_q_type i;
+	int             group;
+	vm_object_t     object = 0;
+
+	mutex_lock(&vm_purgeable_queue_lock);
+	/* Cycle through all queues */
+	for (i = PURGEABLE_Q_TYPE_OBSOLETE; i < PURGEABLE_Q_TYPE_MAX; i++) {
+		purgeable_q_t   queue = &purgeable_queues[i];
+
+		/*
+		 * Are there any ripe tokens on this queue? If yes, we'll
+		 * find an object to purge there
+		 */
+		if (!(queue->token_q_head && tokens[queue->token_q_head].count == 0))
+			continue;	/* no token? Look at next purgeable
+					 * queue */
+
+		/*
+		 * Now look through all groups, starting from the lowest. If
+		 * we find an object in that group, try to lock it (this can
+		 * fail). If locking is successful, we can drop the queue
+		 * lock, remove a token and then purge the object.
+		 */
+		for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+			if (!queue_empty(&queue->objq[group]) && (object = vm_purgeable_object_find_and_lock(queue, group))) {
+				mutex_unlock(&vm_purgeable_queue_lock);
+				vm_purgeable_token_choose_and_delete_ripe(queue, 0);
+				goto purge_now;
+			} else {
+				assert(i != PURGEABLE_Q_TYPE_OBSOLETE);	/* obsolete queue must
+									 * have all objects in
+									 * group 0 */
+				purgeable_q_t   queue2 = &purgeable_queues[i != PURGEABLE_Q_TYPE_FIFO ? PURGEABLE_Q_TYPE_FIFO : PURGEABLE_Q_TYPE_LIFO];
+
+				if (!queue_empty(&queue2->objq[group]) && (object = vm_purgeable_object_find_and_lock(queue2, group))) {
+					mutex_unlock(&vm_purgeable_queue_lock);
+					vm_purgeable_token_choose_and_delete_ripe(queue2, queue);
+					goto purge_now;
+				}
+			}
+			assert(queue->debug_count_objects >= 0);
+		}
+	}
+	/*
+         * because we have to do a try_lock on the objects which could fail,
+         * we could end up with no object to purge at this time, even though
+         * we have objects in a purgeable state
+         */
+	mutex_unlock(&vm_purgeable_queue_lock);
+	return;
+
+purge_now:
+
+	assert(object);
+	(void) vm_object_purge(object);
+	vm_object_unlock(object);
+
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, TOKEN_OBJECT_PURGED)),
+			      (unsigned int) object,	/* purged object */
+			      0,
+			      available_for_purge,
+			      0,
+			      0);
+}
+
+void
+vm_purgeable_object_add(vm_object_t object, purgeable_q_t queue, int group)
+{
+	mutex_lock(&vm_purgeable_queue_lock);
+
+	if (queue->type == PURGEABLE_Q_TYPE_OBSOLETE)
+		group = 0;
+	if (queue->type != PURGEABLE_Q_TYPE_LIFO)	/* fifo and obsolete are
+							 * fifo-queued */
+		queue_enter(&queue->objq[group], object, vm_object_t, objq);	/* last to die */
+	else
+		queue_enter_first(&queue->objq[group], object, vm_object_t, objq);	/* first to die */
+
+#if MACH_ASSERT
+	queue->debug_count_objects++;
+	KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_ADDED)),
+			      0,
+			      tokens[queue->token_q_head].count,
+			      queue->type,
+			      group,
+			      0);
+#endif
+
+	mutex_unlock(&vm_purgeable_queue_lock);
+}
+
+/* Look for object. If found, remove from purgeable queue. */
+purgeable_q_t
+vm_purgeable_object_remove(vm_object_t object)
+{
+	enum purgeable_q_type i;
+	int             group;
+
+	mutex_lock(&vm_purgeable_queue_lock);
+	for (i = PURGEABLE_Q_TYPE_FIFO; i < PURGEABLE_Q_TYPE_MAX; i++) {
+		purgeable_q_t   queue = &purgeable_queues[i];
+		for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+			vm_object_t     o;
+			for (o = (vm_object_t) queue_first(&queue->objq[group]);
+			 !queue_end(&queue->objq[group], (queue_entry_t) o);
+			     o = (vm_object_t) queue_next(&o->objq)) {
+				if (o == object) {
+					queue_remove(&queue->objq[group], object,
+						     vm_object_t, objq);
+#if MACH_ASSERT
+					queue->debug_count_objects--;
+					KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, OBJECT_REMOVED)),
+							      0,
+					  tokens[queue->token_q_head].count,
+							      queue->type,
+							      group,
+							      0);
+#endif
+					mutex_unlock(&vm_purgeable_queue_lock);
+					object->objq.next = 0;
+					object->objq.prev = 0;
+					return &purgeable_queues[i];
+				}
+			}
+		}
+	}
+	mutex_unlock(&vm_purgeable_queue_lock);
+	return 0;
+}
diff --git a/osfmk/vm/vm_purgeable_internal.h b/osfmk/vm/vm_purgeable_internal.h
new file mode 100644
index 000000000..ab2db597e
--- /dev/null
+++ b/osfmk/vm/vm_purgeable_internal.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+/*
+ * Purgeable spelling rules
+ * It is believed that the correct spelling is 
+ * { 'p', 'u', 'r', 'g', 'e', 'a', 'b', 'l', 'e' }.
+ * However, there is one published API that likes to spell it without the
+ * first 'e', vm_purgable_control(). Since we can't change that API,
+ * here are the rules.
+ * All qualifiers defined in vm_purgable.h are spelled without the e.
+ * All other qualifiers are spelled with the e.
+ * Right now, there are remains of the wrong spelling throughout the code,
+ * vm_object_t.purgable for example. We expect to change these on occasion.
+ */
+
+#ifndef __VM_PURGEABLE_INTERNAL__
+#define __VM_PURGEABLE_INTERNAL__
+
+#include <kern/queue.h>
+
+enum purgeable_q_type {
+	PURGEABLE_Q_TYPE_OBSOLETE,
+	PURGEABLE_Q_TYPE_FIFO,
+	PURGEABLE_Q_TYPE_LIFO,
+	PURGEABLE_Q_TYPE_MAX
+};
+
+/* 
+ * It appears there's a 16 vs 32 size mismatch when using
+ * CONFIG_TOKEN_QUEUE_SMALL and the resulting math can lead to a large 
+ * negative value for new_pages in vm_purgeable.c.
+ */
+#if (CONFIG_TOKEN_QUEUE_SMALL == 1) && 0
+typedef uint16_t token_idx_t;
+typedef uint16_t token_cnt_t;
+#define MAX_VOLATILE 0x01000
+#define TOKEN_COUNT_MAX UINT16_MAX
+#else
+typedef uint32_t token_idx_t;
+typedef uint32_t token_cnt_t;
+#define MAX_VOLATILE 0x10000
+#define TOKEN_COUNT_MAX UINT32_MAX
+#endif
+
+#define NUM_VOLATILE_GROUPS 8
+struct purgeable_q {
+	token_idx_t token_q_head;    /* first token */
+	token_idx_t token_q_tail;    /* last token  */
+	token_idx_t token_q_unripe;  /* first token which is not ripe */
+	int32_t new_pages;
+	queue_head_t objq[NUM_VOLATILE_GROUPS];
+	enum purgeable_q_type type;
+#if MACH_ASSERT
+	int debug_count_tokens;
+	int debug_count_objects;
+#endif
+};
+
+typedef struct purgeable_q * purgeable_q_t;
+
+extern struct purgeable_q purgeable_queues[PURGEABLE_Q_TYPE_MAX];
+extern token_cnt_t token_new_pagecount;
+extern int available_for_purge;
+
+
+/*
+ * Locking:
+ * the token counters are protected by the vm_page_queue_lock, since they're
+ * mostly used in that context and we don't want to do a lot of extra locking
+ * the purgeable page queues are protected by a separate lock since they're
+ * mostly user on a user context and we don't want any contention with the
+ * pageout daemon.
+ */
+
+decl_mutex_data(,vm_purgeable_queue_lock)
+
+/* add a new token to queue. called by vm_object_purgeable_control */
+/* enter with page queue locked */
+kern_return_t vm_purgeable_token_add(purgeable_q_t queue);
+
+/* enter with page queue locked */
+void vm_purgeable_token_delete_first(purgeable_q_t queue);
+
+/*
+ * decrement token counters. the function will call the object purger if a
+ * token expires.
+ */
+/* enter with page queue locked */
+void vm_purgeable_q_advance_all(uint32_t num_pages);
+
+/* the object purger. purges the specified number of objects from memory. */
+void vm_purgeable_object_purge_one(void);
+
+/* insert purgeable object into queue */
+void vm_purgeable_object_add(vm_object_t object, purgeable_q_t queue, int group);
+
+/* Look for page belonging to object. If found, put on inactive queue. */
+purgeable_q_t vm_purgeable_object_remove(vm_object_t object);
+
+#endif /* __VM_PURGEABLE_INTERNAL__ */
diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c
index 2ba881ad4..5d4d80b47 100644
--- a/osfmk/vm/vm_resident.c
+++ b/osfmk/vm/vm_resident.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -57,10 +63,12 @@
  */
 
 #include <debug.h>
+#include <libkern/OSAtomic.h>
 
 #include <mach/clock_types.h>
 #include <mach/vm_prot.h>
 #include <mach/vm_statistics.h>
+#include <mach/sdt.h>
 #include <kern/counters.h>
 #include <kern/sched_prim.h>
 #include <kern/task.h>
@@ -80,13 +88,21 @@
 #include <pexpert/pexpert.h>	/* (BRINGUP) */
 
 #include <vm/vm_protos.h>
+#include <vm/memory_object.h>
+#include <vm/vm_purgeable_internal.h>
+
+#if CONFIG_EMBEDDED
+#include <sys/kern_memorystatus.h>
+#endif
+
+int			speculative_age_index = 0;
+int			speculative_steal_index = 0;
+
+struct vm_speculative_age_q vm_page_queue_speculative[VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1];
+
+static void vm_page_insert_internal(vm_page_t, vm_object_t, vm_object_offset_t, boolean_t);
 
-/*	Variables used to indicate the relative age of pages in the
- *	inactive list
- */
 
-unsigned int	vm_page_ticket_roll = 0;
-unsigned int	vm_page_ticket = 0;
 /*
  *	Associated with page of user-allocatable memory is a
  *	page structure.
@@ -122,12 +138,9 @@ vm_page_bucket_t *vm_page_buckets;		/* Array of buckets */
 unsigned int	vm_page_bucket_count = 0;	/* How big is array? */
 unsigned int	vm_page_hash_mask;		/* Mask for hash function */
 unsigned int	vm_page_hash_shift;		/* Shift for hash function */
-uint32_t		vm_page_bucket_hash;	/* Basic bucket hash */
+uint32_t	vm_page_bucket_hash;		/* Basic bucket hash */
 decl_simple_lock_data(,vm_page_bucket_lock)
 
-vm_page_t
-vm_page_lookup_nohint(vm_object_t object, vm_object_offset_t offset);
-
 
 #if	MACH_PAGE_HASH_STATS
 /* This routine is only for debug.  It is intended to be called by
@@ -177,7 +190,7 @@ hash_debug(void)
  */
 vm_size_t	page_size  = PAGE_SIZE;
 vm_size_t	page_mask  = PAGE_MASK;
-int			page_shift = PAGE_SHIFT;
+int		page_shift = PAGE_SHIFT;
 
 /*
  *	Resident page structures are initialized from
@@ -189,13 +202,21 @@ int			page_shift = PAGE_SHIFT;
  */
 struct vm_page	vm_page_template;
 
+vm_page_t	vm_pages = VM_PAGE_NULL;
+unsigned int	vm_pages_count = 0;
+
 /*
  *	Resident pages that represent real memory
- *	are allocated from a free list.
+ *	are allocated from a set of free lists,
+ *	one per color.
  */
-vm_page_t	vm_page_queue_free;
+unsigned int	vm_colors;
+unsigned int    vm_color_mask;			/* mask is == (vm_colors-1) */
+unsigned int	vm_cache_geometry_colors = 0;	/* set by hw dependent code during startup */
+queue_head_t	vm_page_queue_free[MAX_COLORS];
 vm_page_t       vm_page_queue_fictitious;
 unsigned int	vm_page_free_wanted;
+unsigned int	vm_page_free_wanted_privileged;
 unsigned int	vm_page_free_count;
 unsigned int	vm_page_fictitious_count;
 
@@ -222,6 +243,16 @@ unsigned int io_throttle_zero_fill;
  */
 vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
 
+/*
+ *	Guard pages are not accessible so they don't
+ * 	need a physical address, but we need to enter
+ *	one in the pmap.
+ *	Let's make it recognizable and make sure that
+ *	we don't use a real physical page with that
+ *	physical address.
+ */
+vm_offset_t vm_page_guard_addr = (vm_offset_t) -2;
+
 /*
  *	Resident page structures are also chained on
  *	queues that are used by the page replacement
@@ -234,8 +265,12 @@ vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1;
  */
 queue_head_t	vm_page_queue_active;
 queue_head_t	vm_page_queue_inactive;
+queue_head_t	vm_page_queue_zf;	/* inactive memory queue for zero fill */
+
 unsigned int	vm_page_active_count;
 unsigned int	vm_page_inactive_count;
+unsigned int	vm_page_throttled_count;
+unsigned int	vm_page_speculative_count;
 unsigned int	vm_page_wire_count;
 unsigned int	vm_page_gobble_count = 0;
 unsigned int	vm_page_wire_count_warning = 0;
@@ -244,6 +279,10 @@ unsigned int	vm_page_gobble_count_warning = 0;
 unsigned int	vm_page_purgeable_count = 0; /* # of pages purgeable now */
 uint64_t	vm_page_purged_count = 0;    /* total count of purged pages */
 
+unsigned int	vm_page_speculative_recreated = 0;
+unsigned int	vm_page_speculative_created = 0;
+unsigned int	vm_page_speculative_used = 0;
+
 ppnum_t		vm_lopage_poolstart = 0;
 ppnum_t		vm_lopage_poolend = 0;
 int		vm_lopage_poolsize = 0;
@@ -259,8 +298,9 @@ uint64_t	max_valid_dma_address = 0xffffffffffffffffULL;
 unsigned int	vm_page_free_target = 0;
 unsigned int	vm_page_free_min = 0;
 unsigned int	vm_page_inactive_target = 0;
+unsigned int	vm_page_inactive_min = 0;
 unsigned int	vm_page_free_reserved = 0;
-unsigned int	vm_page_throttled_count = 0;
+unsigned int	vm_page_zfill_throttle_count = 0;
 
 /*
  *	The VM system has a couple of heuristics for deciding
@@ -294,6 +334,34 @@ vm_set_page_size(void)
 			break;
 }
 
+
+/* Called once during statup, once the cache geometry is known.
+ */
+static void
+vm_page_set_colors( void )
+{
+	unsigned int	n, override;
+	
+	if ( PE_parse_boot_arg("colors", &override) )		/* colors specified as a boot-arg? */
+		n = override;	
+	else if ( vm_cache_geometry_colors )			/* do we know what the cache geometry is? */
+		n = vm_cache_geometry_colors;
+	else	n = DEFAULT_COLORS;				/* use default if all else fails */
+
+	if ( n == 0 )
+		n = 1;
+	if ( n > MAX_COLORS )
+		n = MAX_COLORS;
+		
+	/* the count must be a power of 2  */
+	if ( ( n & (n - 1)) !=0  )
+		panic("vm_page_set_colors");
+	
+	vm_colors = n;
+	vm_color_mask = n - 1;
+}
+
+
 /*
  *	vm_page_bootstrap:
  *
@@ -330,11 +398,14 @@ vm_page_bootstrap(
 	m->listq.next = NULL;
 	m->listq.prev = NULL;
 
+	m->speculative = FALSE;
+	m->throttled = FALSE;
 	m->inactive = FALSE;
 	m->active = FALSE;
+	m->no_cache = FALSE;
 	m->laundry = FALSE;
 	m->free = FALSE;
-	m->no_isync = TRUE;
+	m->pmapped = FALSE;
 	m->reference = FALSE;
 	m->pageout = FALSE;
 	m->dump_cleaning = FALSE;
@@ -351,18 +422,15 @@ vm_page_bootstrap(
 	m->cleaning = FALSE;
 	m->precious = FALSE;
 	m->clustered = FALSE;
-	m->lock_supplied = FALSE;
 	m->unusual = FALSE;
 	m->restart = FALSE;
 	m->zero_fill = FALSE;
 	m->encrypted = FALSE;
+	m->encrypted_cleaning = FALSE;
+	m->deactivated = FALSE;
 
 	m->phys_page = 0;		/* reset later */
 
-	m->page_lock = VM_PROT_NONE;
-	m->unlock_request = VM_PROT_NONE;
-	m->page_error = KERN_SUCCESS;
-
 	/*
 	 *	Initialize the page queues.
 	 */
@@ -370,13 +438,44 @@ vm_page_bootstrap(
 	mutex_init(&vm_page_queue_free_lock, 0);
 	mutex_init(&vm_page_queue_lock, 0);
 
-	vm_page_queue_free = VM_PAGE_NULL;
+	mutex_init(&vm_purgeable_queue_lock, 0);
+    
+	for (i = 0; i < PURGEABLE_Q_TYPE_MAX; i++) {
+		int group;
+
+		purgeable_queues[i].token_q_head = 0;
+		purgeable_queues[i].token_q_tail = 0;
+		for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
+		        queue_init(&purgeable_queues[i].objq[group]);
+
+		purgeable_queues[i].type = i;
+		purgeable_queues[i].new_pages = 0;
+#if MACH_ASSERT
+		purgeable_queues[i].debug_count_tokens = 0;
+		purgeable_queues[i].debug_count_objects = 0;
+#endif
+	};
+    
+	for (i = 0; i < MAX_COLORS; i++ )
+		queue_init(&vm_page_queue_free[i]);
+	queue_init(&vm_lopage_queue_free);
 	vm_page_queue_fictitious = VM_PAGE_NULL;
 	queue_init(&vm_page_queue_active);
 	queue_init(&vm_page_queue_inactive);
+	queue_init(&vm_page_queue_throttled);
 	queue_init(&vm_page_queue_zf);
 
+	for ( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ ) {
+	        queue_init(&vm_page_queue_speculative[i].age_q);
+
+		vm_page_queue_speculative[i].age_ts.tv_sec = 0;
+		vm_page_queue_speculative[i].age_ts.tv_nsec = 0;
+	}
 	vm_page_free_wanted = 0;
+	vm_page_free_wanted_privileged = 0;
+	
+	vm_page_set_colors();
+
 
 	/*
 	 *	Steal memory for the map and zone subsystems.
@@ -466,10 +565,11 @@ vm_page_bootstrap(
 	 *	all VM managed pages are "free", courtesy of pmap_startup.
 	 */
 	vm_page_wire_count = atop_64(max_mem) - vm_page_free_count;	/* initial value */
-
-	printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count);
 	vm_page_free_count_minimum = vm_page_free_count;
 
+	printf("vm_page_bootstrap: %d free pages and %d wired pages\n",
+	       vm_page_free_count, vm_page_wire_count);
+
 	simple_lock_init(&vm_paging_lock, 0);
 }
 
@@ -552,7 +652,6 @@ pmap_startup(
 	vm_offset_t *endp)
 {
 	unsigned int i, npages, pages_initialized, fill, fillval;
-	vm_page_t	pages;
 	ppnum_t		phys_page;
 	addr64_t	tmpaddr;
 	unsigned int	num_of_lopages = 0;
@@ -565,9 +664,9 @@ pmap_startup(
 
 	tmpaddr = (addr64_t)pmap_free_pages() * (addr64_t)PAGE_SIZE;	/* Get the amount of memory left */
 	tmpaddr = tmpaddr + (addr64_t)(round_page_32(virtual_space_start) - virtual_space_start);	/* Account for any slop */
-	npages = (unsigned int)(tmpaddr / (addr64_t)(PAGE_SIZE + sizeof(*pages)));	/* Figure size of all vm_page_ts, including enough to hold the vm_page_ts */
+	npages = (unsigned int)(tmpaddr / (addr64_t)(PAGE_SIZE + sizeof(*vm_pages)));	/* Figure size of all vm_page_ts, including enough to hold the vm_page_ts */
 
-	pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
+	vm_pages = (vm_page_t) pmap_steal_memory(npages * sizeof *vm_pages);
 
 	/*
 	 *	Initialize the page frames.
@@ -576,16 +675,18 @@ pmap_startup(
 		if (!pmap_next_page(&phys_page))
 			break;
 
-		vm_page_init(&pages[i], phys_page);
+		vm_page_init(&vm_pages[i], phys_page);
 		vm_page_pages++;
 		pages_initialized++;
 	}
+	vm_pages_count = pages_initialized;
 
 	/*
 	 * Check if we want to initialize pages to a known value
 	 */
 	fill = 0;								/* Assume no fill */
 	if (PE_parse_boot_arg("fill", &fillval)) fill = 1;			/* Set fill */
+	
 
 	/*
 	 * if vm_lopage_poolsize is non-zero, than we need to reserve
@@ -598,7 +699,7 @@ pmap_startup(
 	for (i = 0, num_of_lopages = vm_lopage_poolsize; num_of_lopages && i < pages_initialized; num_of_lopages--, i++) {
 	        vm_page_t m;
 
-		m = &pages[i];
+		m = &vm_pages[i];
 
 		if (m->phys_page >= (1 << (32 - PAGE_SHIFT)))
 		        panic("couldn't reserve the lopage pool: not enough lo pages\n");
@@ -626,8 +727,8 @@ pmap_startup(
 	if (2 == vm_himemory_mode) {
 		// free low -> high so high is preferred
 		for (i = last_index + 1; i <= pages_initialized; i++) {
-			if(fill) fillPage(pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
-			vm_page_release(&pages[i - 1]);
+			if(fill) fillPage(vm_pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
+			vm_page_release(&vm_pages[i - 1]);
 		}
 	}
 	else
@@ -640,35 +741,42 @@ pmap_startup(
 	 * they require several consecutive pages.
 	 */
 	for (i = pages_initialized; i > last_index; i--) {
-		if(fill) fillPage(pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
-		vm_page_release(&pages[i - 1]);
+		if(fill) fillPage(vm_pages[i - 1].phys_page, fillval);		/* Fill the page with a know value if requested at boot */			
+		vm_page_release(&vm_pages[i - 1]);
 	}
 
 #if 0
 	{
 		vm_page_t xx, xxo, xxl;
-		int j, k, l;
+		int i, j, k, l;
 	
 		j = 0;													/* (BRINGUP) */
 		xxl = 0;
 		
-		for(xx = vm_page_queue_free; xx; xxl = xx, xx = xx->pageq.next) {	/* (BRINGUP) */
-			j++;												/* (BRINGUP) */
-			if(j > vm_page_free_count) {						/* (BRINGUP) */
-				panic("pmap_startup: too many pages, xx = %08X, xxl = %08X\n", xx, xxl);
-			}
-			
-			l = vm_page_free_count - j;							/* (BRINGUP) */
-			k = 0;												/* (BRINGUP) */
-			
-			if(((j - 1) & 0xFFFF) == 0) kprintf("checking number %d of %d\n", j, vm_page_free_count);
-
-			for(xxo = xx->pageq.next; xxo; xxo = xxo->pageq.next) {	/* (BRINGUP) */
-				k++;
-				if(k > l) panic("pmap_startup: too many in secondary check %d %d\n", k, l);
-				if((xx->phys_page & 0xFFFFFFFF) == (xxo->phys_page & 0xFFFFFFFF)) {	/* (BRINGUP) */
-					panic("pmap_startup: duplicate physaddr, xx = %08X, xxo = %08X\n", xx, xxo);
+		for( i = 0; i < vm_colors; i++ ) {
+			queue_iterate(&vm_page_queue_free[i],
+				      xx,
+				      vm_page_t,
+				      pageq) {	/* BRINGUP */
+				j++;												/* (BRINGUP) */
+				if(j > vm_page_free_count) {						/* (BRINGUP) */
+					panic("pmap_startup: too many pages, xx = %08X, xxl = %08X\n", xx, xxl);
 				}
+				
+				l = vm_page_free_count - j;							/* (BRINGUP) */
+				k = 0;												/* (BRINGUP) */
+				
+				if(((j - 1) & 0xFFFF) == 0) kprintf("checking number %d of %d\n", j, vm_page_free_count);
+
+				for(xxo = xx->pageq.next; xxo != &vm_page_queue_free[i]; xxo = xxo->pageq.next) {	/* (BRINGUP) */
+					k++;
+					if(k > l) panic("pmap_startup: too many in secondary check %d %d\n", k, l);
+					if((xx->phys_page & 0xFFFFFFFF) == (xxo->phys_page & 0xFFFFFFFF)) {	/* (BRINGUP) */
+						panic("pmap_startup: duplicate physaddr, xx = %08X, xxo = %08X\n", xx, xxo);
+					}
+				}
+
+				xxl = xx;
 			}
 		}
 		
@@ -762,6 +870,7 @@ vm_page_create(
 	( (natural_t)((uint32_t)object * vm_page_bucket_hash) + ((uint32_t)atop_64(offset) ^ vm_page_bucket_hash))\
 	 & vm_page_hash_mask)
 
+
 /*
  *	vm_page_insert:		[ internal use only ]
  *
@@ -770,12 +879,22 @@ vm_page_create(
  *
  *	The object must be locked.
  */
-
 void
 vm_page_insert(
-	register vm_page_t		mem,
-	register vm_object_t		object,
-	register vm_object_offset_t	offset)
+	vm_page_t		mem,
+	vm_object_t		object,
+	vm_object_offset_t	offset)
+{
+	vm_page_insert_internal(mem, object, offset, FALSE);
+}
+
+
+static void
+vm_page_insert_internal(
+	vm_page_t		mem,
+	vm_object_t		object,
+	vm_object_offset_t	offset,
+	boolean_t	queues_lock_held)
 {
 	register vm_page_bucket_t *bucket;
 
@@ -784,9 +903,14 @@ vm_page_insert(
                 (integer_t)object, (integer_t)offset, (integer_t)mem, 0,0);
 
 	VM_PAGE_CHECK(mem);
-#if DEBUG
-	_mutex_assert(&object->Lock, MA_OWNED);
 
+	if (object == vm_submap_object) {
+		/* the vm_submap_object is only a placeholder for submaps */
+		panic("vm_page_insert(vm_submap_object,0x%llx)\n", offset);
+	}
+
+	vm_object_lock_assert_exclusive(object);
+#if DEBUG
 	if (mem->tabled || mem->object != VM_OBJECT_NULL)
 		panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) "
 		      "already in (obj=%p,off=0x%llx)",
@@ -834,11 +958,15 @@ vm_page_insert(
 
 	object->resident_page_count++;
 
-	if (object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-	    object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
-		vm_page_lock_queues();
+	if (object->purgable == VM_PURGABLE_VOLATILE ||
+	    object->purgable == VM_PURGABLE_EMPTY) {
+		if (queues_lock_held == FALSE)
+			vm_page_lockspin_queues();
+
 		vm_page_purgeable_count++;
-		vm_page_unlock_queues();
+
+		if (queues_lock_held == FALSE)
+			vm_page_unlock_queues();
 	}
 }
 
@@ -861,8 +989,8 @@ vm_page_replace(
 	vm_page_t	 found_m = VM_PAGE_NULL;
 
 	VM_PAGE_CHECK(mem);
+	vm_object_lock_assert_exclusive(object);
 #if DEBUG
-	_mutex_assert(&object->Lock, MA_OWNED);
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 
 	if (mem->tabled || mem->object != VM_OBJECT_NULL)
@@ -926,8 +1054,8 @@ vm_page_replace(
 		found_m->offset = (vm_object_offset_t) -1;
 		object->resident_page_count--;
 
-		if (object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-		    object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
+		if (object->purgable == VM_PURGABLE_VOLATILE ||
+		    object->purgable == VM_PURGABLE_EMPTY) {
 		        assert(vm_page_purgeable_count > 0);
 			vm_page_purgeable_count--;
 		}
@@ -952,8 +1080,8 @@ vm_page_replace(
 
 	object->resident_page_count++;
 
-	if (object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-	    object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
+	if (object->purgable == VM_PURGABLE_VOLATILE ||
+	    object->purgable == VM_PURGABLE_EMPTY) {
 		vm_page_purgeable_count++;
 	}
 }
@@ -980,8 +1108,8 @@ vm_page_remove(
 		(integer_t)mem, 0,0);
 #if DEBUG
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
-	_mutex_assert(&mem->object->Lock, MA_OWNED);
 #endif
+	vm_object_lock_assert_exclusive(mem->object);
 	assert(mem->tabled);
 	assert(!mem->cleaning);
 	VM_PAGE_CHECK(mem);
@@ -1024,12 +1152,11 @@ vm_page_remove(
 
 	mem->object->resident_page_count--;
 
-	if (mem->object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
-	    mem->object->purgable == VM_OBJECT_PURGABLE_EMPTY) {
+	if (mem->object->purgable == VM_PURGABLE_VOLATILE ||
+	    mem->object->purgable == VM_PURGABLE_EMPTY) {
 		assert(vm_page_purgeable_count > 0);
 		vm_page_purgeable_count--;
 	}
-
 	mem->tabled = FALSE;
 	mem->object = VM_OBJECT_NULL;
 	mem->offset = (vm_object_offset_t) -1;
@@ -1048,6 +1175,9 @@ unsigned long vm_page_lookup_hint = 0;
 unsigned long vm_page_lookup_hint_next = 0;
 unsigned long vm_page_lookup_hint_prev = 0;
 unsigned long vm_page_lookup_hint_miss = 0;
+unsigned long vm_page_lookup_bucket_NULL = 0;
+unsigned long vm_page_lookup_miss = 0;
+
 
 vm_page_t
 vm_page_lookup(
@@ -1057,23 +1187,25 @@ vm_page_lookup(
 	register vm_page_t	mem;
 	register vm_page_bucket_t *bucket;
 	queue_entry_t		qe;
-#if 0
-	_mutex_assert(&object->Lock, MA_OWNED);
-#endif
 
+	vm_object_lock_assert_held(object);
 	mem = object->memq_hint;
+
 	if (mem != VM_PAGE_NULL) {
 		assert(mem->object == object);
+
 		if (mem->offset == offset) {
 			vm_page_lookup_hint++;
 			return mem;
 		}
 		qe = queue_next(&mem->listq);
+
 		if (! queue_end(&object->memq, qe)) {
 			vm_page_t	next_page;
 
 			next_page = (vm_page_t) qe;
 			assert(next_page->object == object);
+
 			if (next_page->offset == offset) {
 				vm_page_lookup_hint_next++;
 				object->memq_hint = next_page; /* new hint */
@@ -1081,11 +1213,13 @@ vm_page_lookup(
 			}
 		}
 		qe = queue_prev(&mem->listq);
+
 		if (! queue_end(&object->memq, qe)) {
 			vm_page_t prev_page;
 
 			prev_page = (vm_page_t) qe;
 			assert(prev_page->object == object);
+
 			if (prev_page->offset == offset) {
 				vm_page_lookup_hint_prev++;
 				object->memq_hint = prev_page; /* new hint */
@@ -1093,24 +1227,24 @@ vm_page_lookup(
 			}
 		}
 	}
-
 	/*
-	 *	Search the hash table for this object/offset pair
+	 * Search the hash table for this object/offset pair
 	 */
-
 	bucket = &vm_page_buckets[vm_page_hash(object, offset)];
 
-        /*
-         * since we hold the object lock, we are guaranteed that no
-         * new pages can be inserted into this object... this in turn
-         * guarantess that the page we're looking for can't exist
-         * if the bucket it hashes to is currently NULL even when looked
-         * at outside the scope of the hash bucket lock... this is a
-         * really cheap optimiztion to avoid taking the lock
-         */
-        if (bucket->pages == VM_PAGE_NULL) {
-                return (VM_PAGE_NULL);
-        }
+	/*
+	 * since we hold the object lock, we are guaranteed that no
+	 * new pages can be inserted into this object... this in turn
+	 * guarantess that the page we're looking for can't exist
+	 * if the bucket it hashes to is currently NULL even when looked
+	 * at outside the scope of the hash bucket lock... this is a 
+	 * really cheap optimiztion to avoid taking the lock
+	 */
+	if (bucket->pages == VM_PAGE_NULL) {
+	        vm_page_lookup_bucket_NULL++;
+
+	        return (VM_PAGE_NULL);
+	}
 	simple_lock(&vm_page_bucket_lock);
 
 	for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
@@ -1126,40 +1260,13 @@ vm_page_lookup(
 		}
 		assert(mem->object == object);
 		object->memq_hint = mem;
-	}
+	} else
+	        vm_page_lookup_miss++;
 
 	return(mem);
 }
 
 
-vm_page_t
-vm_page_lookup_nohint(
-	vm_object_t		object,
-	vm_object_offset_t	offset)
-{
-	register vm_page_t	mem;
-	register vm_page_bucket_t *bucket;
-
-#if 0
-	_mutex_assert(&object->Lock, MA_OWNED);
-#endif
-	/*
-	 *	Search the hash table for this object/offset pair
-	 */
-
-	bucket = &vm_page_buckets[vm_page_hash(object, offset)];
-
-	simple_lock(&vm_page_bucket_lock);
-	for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) {
-		VM_PAGE_CHECK(mem);
-		if ((mem->object == object) && (mem->offset == offset))
-			break;
-	}
-	simple_unlock(&vm_page_bucket_lock);
-
-	return(mem);
-}
-
 /*
  *	vm_page_rename:
  *
@@ -1172,19 +1279,28 @@ void
 vm_page_rename(
 	register vm_page_t		mem,
 	register vm_object_t		new_object,
-	vm_object_offset_t		new_offset)
+	vm_object_offset_t		new_offset,
+	boolean_t			encrypted_ok)
 {
 	assert(mem->object != new_object);
+
 	/*
 	 * ENCRYPTED SWAP:
 	 * The encryption key is based on the page's memory object
 	 * (aka "pager") and paging offset.  Moving the page to
 	 * another VM object changes its "pager" and "paging_offset"
-	 * so it has to be decrypted first.
+	 * so it has to be decrypted first, or we would lose the key.
+	 *
+	 * One exception is VM object collapsing, where we transfer pages
+	 * from one backing object to its parent object.  This operation also
+	 * transfers the paging information, so the <pager,paging_offset> info
+	 * should remain consistent.  The caller (vm_object_do_collapse())
+	 * sets "encrypted_ok" in this case.
 	 */
-	if (mem->encrypted) {
+	if (!encrypted_ok && mem->encrypted) {
 		panic("vm_page_rename: page %p is encrypted\n", mem);
 	}
+
 	/*
 	 *	Changes to mem->object require the page lock because
 	 *	the pageout daemon uses that lock to get the object.
@@ -1195,7 +1311,7 @@ vm_page_rename(
                 (integer_t)new_object, (integer_t)new_offset, 
 		(integer_t)mem, 0,0);
 
-	vm_page_lock_queues();
+	vm_page_lockspin_queues();
     	vm_page_remove(mem);
 	vm_page_insert(mem, new_object, new_offset);
 	vm_page_unlock_queues();
@@ -1228,14 +1344,17 @@ int	c_vm_page_grab_fictitious = 0;
 int	c_vm_page_release_fictitious = 0;
 int	c_vm_page_more_fictitious = 0;
 
+extern vm_page_t vm_page_grab_fictitious_common(vm_offset_t phys_addr);
+
 vm_page_t
-vm_page_grab_fictitious(void)
+vm_page_grab_fictitious_common(
+	vm_offset_t phys_addr)
 {
 	register vm_page_t m;
 
 	m = (vm_page_t)zget(vm_page_zone);
 	if (m) {
-		vm_page_init(m, vm_page_fictitious_addr);
+		vm_page_init(m, phys_addr);
 		m->fictitious = TRUE;
 	}
 
@@ -1243,6 +1362,18 @@ vm_page_grab_fictitious(void)
 	return m;
 }
 
+vm_page_t
+vm_page_grab_fictitious(void)
+{
+	return vm_page_grab_fictitious_common(vm_page_fictitious_addr);
+}
+
+vm_page_t
+vm_page_grab_guard(void)
+{
+	return vm_page_grab_fictitious_common(vm_page_guard_addr);
+}
+
 /*
  *	vm_page_release_fictitious:
  *
@@ -1256,7 +1387,8 @@ vm_page_release_fictitious(
 	assert(!m->free);
 	assert(m->busy);
 	assert(m->fictitious);
-	assert(m->phys_page == vm_page_fictitious_addr);
+	assert(m->phys_page == vm_page_fictitious_addr ||
+	       m->phys_page == vm_page_guard_addr);
 
 	c_vm_page_release_fictitious++;
 #if DEBUG
@@ -1362,43 +1494,6 @@ void vm_page_more_fictitious(void)
 	mutex_unlock(&vm_page_alloc_lock);
 }
 
-/*
- *	vm_page_convert:
- *
- *	Attempt to convert a fictitious page into a real page.
- */
-
-boolean_t
-vm_page_convert(
-	register vm_page_t m)
-{
-	register vm_page_t real_m;
-
-	assert(m->busy);
-	assert(m->fictitious);
-	assert(!m->dirty);
-
-	real_m = vm_page_grab();
-	if (real_m == VM_PAGE_NULL)
-		return FALSE;
-
-	m->phys_page = real_m->phys_page;
-	m->fictitious = FALSE;
-	m->no_isync = TRUE;
-
-	vm_page_lock_queues();
-	if (m->active)
-		vm_page_active_count++;
-	else if (m->inactive)
-		vm_page_inactive_count++;
-	vm_page_unlock_queues();
-
-	real_m->phys_page = vm_page_fictitious_addr;
-	real_m->fictitious = TRUE;
-
-	vm_page_release_fictitious(real_m);
-	return TRUE;
-}
 
 /*
  *	vm_pool_low():
@@ -1430,7 +1525,7 @@ int		vm_himemory_mode = 0;
  */
 unsigned int	vm_lopage_free_count = 0;
 unsigned int	vm_lopage_max_count = 0;
-vm_page_t	vm_lopage_queue_free = VM_PAGE_NULL;
+queue_head_t	vm_lopage_queue_free;
 
 vm_page_t
 vm_page_grablo(void)
@@ -1443,18 +1538,25 @@ vm_page_grablo(void)
 
 	mutex_lock(&vm_page_queue_free_lock);
 
-	if ((mem = vm_lopage_queue_free) != VM_PAGE_NULL) {
+	if (! queue_empty(&vm_lopage_queue_free)) {
+		queue_remove_first(&vm_lopage_queue_free,
+				   mem,
+				   vm_page_t,
+				   pageq);
+		assert(mem->free);
+		assert(mem->busy);
+		assert(!mem->pmapped);
 
-	        vm_lopage_queue_free = (vm_page_t) mem->pageq.next;
 		mem->pageq.next = NULL;
 		mem->pageq.prev = NULL;
 		mem->free = FALSE;
-		mem->no_isync = TRUE;
 
 		vm_lopage_free_count--;
 		vm_lopage_alloc_count = (vm_lopage_poolend - vm_lopage_poolstart) - vm_lopage_free_count;
 		if (vm_lopage_alloc_count > vm_lopage_max_count)
 			vm_lopage_max_count = vm_lopage_alloc_count;
+	} else {
+		mem = VM_PAGE_NULL;
 	}
 	mutex_unlock(&vm_page_queue_free_lock);
 
@@ -1462,23 +1564,62 @@ vm_page_grablo(void)
 }
 
 
-
 /*
  *	vm_page_grab:
  *
- *	Remove a page from the free list.
- *	Returns VM_PAGE_NULL if the free list is too small.
+ *	first try to grab a page from the per-cpu free list...
+ *	this must be done while pre-emption is disabled... if
+ * 	a page is available, we're done... 
+ *	if no page is available, grab the vm_page_queue_free_lock
+ *	and see if current number of free pages would allow us
+ * 	to grab at least 1... if not, return VM_PAGE_NULL as before... 
+ *	if there are pages available, disable preemption and
+ * 	recheck the state of the per-cpu free list... we could
+ *	have been preempted and moved to a different cpu, or
+ * 	some other thread could have re-filled it... if still
+ *	empty, figure out how many pages we can steal from the
+ *	global free queue and move to the per-cpu queue...
+ *	return 1 of these pages when done... only wakeup the
+ * 	pageout_scan thread if we moved pages from the global
+ *	list... no need for the wakeup if we've satisfied the
+ *	request from the per-cpu queue.
  */
 
-unsigned long	vm_page_grab_count = 0;	/* measure demand */
+#define COLOR_GROUPS_TO_STEAL	4
+
 
 vm_page_t
-vm_page_grab(void)
+vm_page_grab( void )
 {
-	register vm_page_t	mem;
+	vm_page_t	mem;
+
+
+	disable_preemption();
+
+	if ((mem = PROCESSOR_DATA(current_processor(), free_pages))) {
+return_page_from_cpu_list:
+	        PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
+	        PROCESSOR_DATA(current_processor(), free_pages) = mem->pageq.next;
+		mem->pageq.next = NULL;
+
+	        enable_preemption();
+
+		assert(mem->listq.next == NULL && mem->listq.prev == NULL);
+		assert(mem->tabled == FALSE);
+		assert(mem->object == VM_OBJECT_NULL);
+		assert(!mem->laundry);
+		assert(!mem->free);
+		assert(pmap_verify_free(mem->phys_page));
+		assert(mem->busy);
+		assert(!mem->encrypted);
+		assert(!mem->pmapped);
+
+		return mem;
+	}
+	enable_preemption();
+
 
 	mutex_lock(&vm_page_queue_free_lock);
-	vm_page_grab_count++;
 
 	/*
 	 *	Optionally produce warnings if the wire or gobble
@@ -1501,36 +1642,102 @@ vm_page_grab(void)
 	 *	Only let privileged threads (involved in pageout)
 	 *	dip into the reserved pool.
 	 */
-
 	if ((vm_page_free_count < vm_page_free_reserved) &&
 	    !(current_thread()->options & TH_OPT_VMPRIV)) {
 		mutex_unlock(&vm_page_queue_free_lock);
 		mem = VM_PAGE_NULL;
-		goto wakeup_pageout;
 	}
+	else {
+	       vm_page_t	head;
+	       vm_page_t	tail;
+	       unsigned int	pages_to_steal;
+	       unsigned int	color;
 
-	while (vm_page_queue_free == VM_PAGE_NULL) {
-		mutex_unlock(&vm_page_queue_free_lock);
-		VM_PAGE_WAIT();
-		mutex_lock(&vm_page_queue_free_lock);
-	}
-
-	if (--vm_page_free_count < vm_page_free_count_minimum)
-		vm_page_free_count_minimum = vm_page_free_count;
-	mem = vm_page_queue_free;
-	vm_page_queue_free = (vm_page_t) mem->pageq.next;
-	mem->pageq.next = NULL;
-	mem->pageq.prev = NULL;
-	assert(mem->listq.next == NULL && mem->listq.prev == NULL);
-	assert(mem->tabled == FALSE);
-	assert(mem->object == VM_OBJECT_NULL);
-	assert(!mem->laundry);
-	mem->free = FALSE;
-	mem->no_isync = TRUE;
-	mutex_unlock(&vm_page_queue_free_lock);
+	       while ( vm_page_free_count == 0 ) {
 
-	assert(pmap_verify_free(mem->phys_page));
+			mutex_unlock(&vm_page_queue_free_lock);
+			/*
+			 * must be a privileged thread to be
+			 * in this state since a non-privileged 
+			 * thread would have bailed if we were
+			 * under the vm_page_free_reserved mark
+			 */
+			VM_PAGE_WAIT();
+			mutex_lock(&vm_page_queue_free_lock);
+		}
+
+		disable_preemption();
+
+		if ((mem = PROCESSOR_DATA(current_processor(), free_pages))) {
+			mutex_unlock(&vm_page_queue_free_lock);
+
+		        /*
+			 * we got preempted and moved to another processor
+			 * or we got preempted and someone else ran and filled the cache
+			 */
+			goto return_page_from_cpu_list;
+		}
+		if (vm_page_free_count <= vm_page_free_reserved)
+		        pages_to_steal = 1;
+		else {
+		        pages_to_steal = COLOR_GROUPS_TO_STEAL * vm_colors;
+		
+			if (pages_to_steal > (vm_page_free_count - vm_page_free_reserved))
+			        pages_to_steal = (vm_page_free_count - vm_page_free_reserved);
+		}
+		color = PROCESSOR_DATA(current_processor(), start_color);
+		head = tail = NULL;
+
+		while (pages_to_steal--) {
+		        if (--vm_page_free_count < vm_page_free_count_minimum)
+			        vm_page_free_count_minimum = vm_page_free_count;
+
+			while (queue_empty(&vm_page_queue_free[color]))
+			        color = (color + 1) & vm_color_mask;
+		
+			queue_remove_first(&vm_page_queue_free[color],
+					   mem,
+					   vm_page_t,
+					   pageq);
+			mem->pageq.next = NULL;
+			mem->pageq.prev = NULL;
+
+			color = (color + 1) & vm_color_mask;
+
+			if (head == NULL)
+				head = mem;
+			else
+			        tail->pageq.next = (queue_t)mem;
+		        tail = mem;
+
+			mem->pageq.prev = NULL;
+			assert(mem->listq.next == NULL && mem->listq.prev == NULL);
+			assert(mem->tabled == FALSE);
+			assert(mem->object == VM_OBJECT_NULL);
+			assert(!mem->laundry);
+			assert(mem->free);
+			mem->free = FALSE;
+
+			assert(pmap_verify_free(mem->phys_page));
+			assert(mem->busy);
+			assert(!mem->free);
+			assert(!mem->encrypted);
+			assert(!mem->pmapped);
+		}
+		PROCESSOR_DATA(current_processor(), free_pages) = head->pageq.next;
+		PROCESSOR_DATA(current_processor(), start_color) = color;
+
+		/*
+		 * satisfy this request
+		 */
+	        PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
+		mem = head;
+		mem->pageq.next = NULL;
 
+		mutex_unlock(&vm_page_queue_free_lock);
+
+		enable_preemption();
+	}
 	/*
 	 *	Decide if we should poke the pageout daemon.
 	 *	We do this if the free count is less than the low
@@ -1541,12 +1748,29 @@ vm_page_grab(void)
 	 *	We don't have the counts locked ... if they change a little,
 	 *	it doesn't really matter.
 	 */
-
-wakeup_pageout:
 	if ((vm_page_free_count < vm_page_free_min) ||
 	    ((vm_page_free_count < vm_page_free_target) &&
-	     (vm_page_inactive_count < vm_page_inactive_target)))
-		thread_wakeup((event_t) &vm_page_free_wanted);
+	     ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min)))
+	        thread_wakeup((event_t) &vm_page_free_wanted);
+
+#if CONFIG_EMBEDDED
+	{
+	int 	percent_avail;
+
+	/*
+	 * Decide if we need to poke the memorystatus notification thread.
+	 */
+	percent_avail = 
+		(vm_page_active_count + vm_page_inactive_count + 
+		 vm_page_speculative_count + vm_page_free_count +
+		 vm_page_purgeable_count ) * 100 /
+		atop_64(max_mem);
+	if (percent_avail <= (kern_memorystatus_level - 5)) {
+		kern_memorystatus_level = percent_avail;
+		thread_wakeup((event_t)&kern_memorystatus_wakeup);
+	}
+	}
+#endif
 
 //	dbgLog(mem->phys_page, vm_page_free_count, vm_page_wire_count, 4);	/* (TEST/DEBUG) */
 
@@ -1563,7 +1787,7 @@ void
 vm_page_release(
 	register vm_page_t	mem)
 {
-
+	unsigned int	color;
 #if 0
 	unsigned int pindex;
 	phys_entry *physent;
@@ -1584,23 +1808,32 @@ vm_page_release(
 		panic("vm_page_release");
 #endif
 	mem->free = TRUE;
+
+	assert(mem->busy);
 	assert(!mem->laundry);
 	assert(mem->object == VM_OBJECT_NULL);
 	assert(mem->pageq.next == NULL &&
 	       mem->pageq.prev == NULL);
-
+	assert(mem->listq.next == NULL &&
+	       mem->listq.prev == NULL);
+	
 	if (mem->phys_page <= vm_lopage_poolend && mem->phys_page >= vm_lopage_poolstart) {
 	        /*
 		 * this exists to support hardware controllers
 		 * incapable of generating DMAs with more than 32 bits
 		 * of address on platforms with physical memory > 4G...
 		 */
-	        mem->pageq.next = (queue_entry_t) vm_lopage_queue_free;
-		vm_lopage_queue_free = mem;
+		queue_enter_first(&vm_lopage_queue_free,
+				  mem,
+				  vm_page_t,
+				  pageq);
 		vm_lopage_free_count++;
 	} else {	  
-	        mem->pageq.next = (queue_entry_t) vm_page_queue_free;
-		vm_page_queue_free = mem;
+	        color = mem->phys_page & vm_color_mask;
+		queue_enter_first(&vm_page_queue_free[color],
+				  mem,
+				  vm_page_t,
+				  pageq);
 		vm_page_free_count++;
 		/*
 		 *	Check if we should wake up someone waiting for page.
@@ -1622,13 +1855,36 @@ vm_page_release(
 		 *	as long as vm_page_free_wanted is non-zero.
 		 */
 
-		if ((vm_page_free_wanted > 0) &&
-		    (vm_page_free_count >= vm_page_free_reserved)) {
+		if ((vm_page_free_wanted_privileged > 0) && vm_page_free_count) {
+		        vm_page_free_wanted_privileged--;
+			thread_wakeup_one((event_t) &vm_page_free_wanted_privileged);
+		} else if ((vm_page_free_wanted > 0) &&
+			   (vm_page_free_count >= vm_page_free_reserved)) {
 		        vm_page_free_wanted--;
 			thread_wakeup_one((event_t) &vm_page_free_count);
 		}
 	}
 	mutex_unlock(&vm_page_queue_free_lock);
+
+#if CONFIG_EMBEDDED
+	{
+	int	percent_avail;
+
+	/*
+	 * Decide if we need to poke the memorystatus notification thread.
+	 * Locking is not a big issue, as only a single thread delivers these.
+	 */
+	percent_avail = 
+		(vm_page_active_count + vm_page_inactive_count + 
+		 vm_page_speculative_count + vm_page_free_count +
+		 vm_page_purgeable_count ) * 100 /
+		atop_64(max_mem);
+	if (percent_avail >= (kern_memorystatus_level + 5)) {
+		kern_memorystatus_level = percent_avail;
+		thread_wakeup((event_t)&kern_memorystatus_wakeup);
+	}
+	}
+#endif
 }
 
 /*
@@ -1655,12 +1911,25 @@ vm_page_wait(
 	 */
 	kern_return_t	wait_result;
 	int          	need_wakeup = 0;
+	int		is_privileged = current_thread()->options & TH_OPT_VMPRIV;
 
 	mutex_lock(&vm_page_queue_free_lock);
+
+	if (is_privileged && vm_page_free_count) {
+		mutex_unlock(&vm_page_queue_free_lock);
+		return TRUE;
+	}
 	if (vm_page_free_count < vm_page_free_target) {
-		if (vm_page_free_wanted++ == 0)
-		        need_wakeup = 1;
-		wait_result = assert_wait((event_t)&vm_page_free_count, interruptible);
+
+	        if (is_privileged) {
+		        if (vm_page_free_wanted_privileged++ == 0)
+			        need_wakeup = 1;
+			wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, interruptible);
+		} else {
+		        if (vm_page_free_wanted++ == 0)
+			        need_wakeup = 1;
+			wait_result = assert_wait((event_t)&vm_page_free_count, interruptible);
+		}
 		mutex_unlock(&vm_page_queue_free_lock);
 		counter(c_vm_page_wait_block++);
 
@@ -1693,9 +1962,7 @@ vm_page_alloc(
 {
 	register vm_page_t	mem;
 
-#if DEBUG
-	_mutex_assert(&object->Lock, MA_OWNED);
-#endif
+	vm_object_lock_assert_exclusive(object);
 	mem = vm_page_grab();
 	if (mem == VM_PAGE_NULL)
 		return VM_PAGE_NULL;
@@ -1705,7 +1972,6 @@ vm_page_alloc(
 	return(mem);
 }
 
-
 vm_page_t
 vm_page_alloclo(
 	vm_object_t		object,
@@ -1713,9 +1979,7 @@ vm_page_alloclo(
 {
 	register vm_page_t	mem;
 
-#if DEBUG
-	_mutex_assert(&object->Lock, MA_OWNED);
-#endif
+	vm_object_lock_assert_exclusive(object);
 	mem = vm_page_grablo();
 	if (mem == VM_PAGE_NULL)
 		return VM_PAGE_NULL;
@@ -1726,9 +1990,34 @@ vm_page_alloclo(
 }
 
 
+/*
+ *	vm_page_alloc_guard:
+ *	
+ * 	Allocate a ficticious page which will be used
+ *	as a guard page.  The page will be inserted into
+ *	the object and returned to the caller.
+ */
+
+vm_page_t
+vm_page_alloc_guard(
+	vm_object_t		object,
+	vm_object_offset_t	offset)
+{
+	register vm_page_t	mem;
+
+	vm_object_lock_assert_exclusive(object);
+	mem = vm_page_grab_guard();
+	if (mem == VM_PAGE_NULL)
+		return VM_PAGE_NULL;
+
+	vm_page_insert(mem, object, offset);
+
+	return(mem);
+}
+
+
 counter(unsigned int c_laundry_pages_freed = 0;)
 
-int vm_pagein_cluster_unused = 0;
 boolean_t	vm_page_free_verify = TRUE;
 /*
  *	vm_page_free:
@@ -1739,34 +2028,42 @@ boolean_t	vm_page_free_verify = TRUE;
  *	Object and page queues must be locked prior to entry.
  */
 void
-vm_page_free(
+vm_page_free_prepare(
 	register vm_page_t	mem)
 {
-	vm_object_t	object = mem->object;
-
+	VM_PAGE_CHECK(mem);
 	assert(!mem->free);
 	assert(!mem->cleaning);
 	assert(!mem->pageout);
+
+#if DEBUG
 	if (vm_page_free_verify && !mem->fictitious && !mem->private) {
 		assert(pmap_verify_free(mem->phys_page));
 	}
-
-#if DEBUG
 	if (mem->object)
-	        _mutex_assert(&mem->object->Lock, MA_OWNED);
+	        vm_object_lock_assert_exclusive(mem->object);
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 
 	if (mem->free)
 	       panic("vm_page_free: freeing page on free list\n");
 #endif
+
+	if (mem->laundry) {
+		/*
+		 * We may have to free a page while it's being laundered
+		 * if we lost its pager (due to a forced unmount, for example).
+		 * We need to call vm_pageout_throttle_up() before removing
+		 * the page from its VM object, so that we can find out on
+		 * which pageout queue the page is.
+		 */
+		vm_pageout_throttle_up(mem);
+		counter(++c_laundry_pages_freed);
+	}
+
 	if (mem->tabled)
 		vm_page_remove(mem);	/* clears tabled, object, offset */
-	VM_PAGE_QUEUES_REMOVE(mem);	/* clears active or inactive */
 
-	if (mem->clustered) {
-		mem->clustered = FALSE;
-		vm_pagein_cluster_unused++;
-	}
+	VM_PAGE_QUEUES_REMOVE(mem);	/* clears active/inactive/throttled/speculative */
 
 	if (mem->wire_count) {
 		if (!mem->private && !mem->fictitious)
@@ -1780,19 +2077,9 @@ vm_page_free(
 	}
 	mem->gobbled = FALSE;
 
-	if (mem->laundry) {
-		vm_pageout_throttle_up(mem);
-		counter(++c_laundry_pages_freed);
-	}
-
 	PAGE_WAKEUP(mem);	/* clears wanted */
 
-	if (mem->absent)
-		vm_object_absent_release(object);
-
 	/* Some of these may be unnecessary */
-	mem->page_lock = 0;
-	mem->unlock_request = 0;
 	mem->busy = TRUE;
 	mem->absent = FALSE;
 	mem->error = FALSE;
@@ -1800,37 +2087,61 @@ vm_page_free(
 	mem->precious = FALSE;
 	mem->reference = FALSE;
 	mem->encrypted = FALSE;
-
-	mem->page_error = KERN_SUCCESS;
+	mem->encrypted_cleaning = FALSE;
+	mem->deactivated = FALSE;
+	mem->pmapped = FALSE;
 
 	if (mem->private) {
 		mem->private = FALSE;
 		mem->fictitious = TRUE;
 		mem->phys_page = vm_page_fictitious_addr;
 	}
-	if (mem->fictitious) {
-		vm_page_release_fictitious(mem);
-	} else {
-		/* depends on the queues lock */
-		if(mem->zero_fill) {
-			vm_zf_count-=1;
+	if (!mem->fictitious) {
+		if (mem->zero_fill == TRUE) {
 			mem->zero_fill = FALSE;
+		        OSAddAtomic(-1, (SInt32 *)&vm_zf_count);
 		}
 		vm_page_init(mem, mem->phys_page);
-		vm_page_release(mem);
 	}
 }
 
+void
+vm_page_free(
+	vm_page_t	mem)
+{
+	vm_page_free_prepare(mem);	
+	if (mem->fictitious) {
+		vm_page_release_fictitious(mem);
+	} else {
+		vm_page_release(mem);
+	}
+}
 
+/*
+ * Free a list of pages.  The list can be up to several hundred pages,
+ * as blocked up by vm_pageout_scan().
+ * The big win is not having to take the page q and free list locks once
+ * per page.  We sort the incoming pages into n lists, one for
+ * each color.
+ *
+ * The page queues must be locked, and are kept locked.
+ */
 void
 vm_page_free_list(
-	register vm_page_t	mem)
+	vm_page_t	mem)
 {
-        register vm_page_t	nxt;
-	register vm_page_t      first = NULL;
-	register vm_page_t      last = VM_PAGE_NULL;
-	register int            pg_count = 0;
+        vm_page_t	nxt;
+	int		pg_count = 0;
+	int		color;
+	int		inuse_list_head = -1;
+
+	queue_head_t	free_list[MAX_COLORS];
+	int		inuse[MAX_COLORS];
 
+	for (color = 0; color < (signed) vm_colors; color++) {
+		queue_init(&free_list[color]);
+	}
+	
 #if DEBUG
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 #endif
@@ -1838,63 +2149,84 @@ vm_page_free_list(
 #if DEBUG
 		if (mem->tabled || mem->object)
 		        panic("vm_page_free_list: freeing tabled page\n");
-		if (mem->inactive || mem->active || mem->free)
+		if (mem->inactive || mem->active || mem->throttled || mem->free)
 		        panic("vm_page_free_list: freeing page on list\n");
+		if (vm_page_free_verify && !mem->fictitious && !mem->private) {
+			assert(pmap_verify_free(mem->phys_page));
+		}
 #endif
 		assert(mem->pageq.prev == NULL);
+		assert(mem->busy);
+		assert(!mem->free);
 		nxt = (vm_page_t)(mem->pageq.next);
 
-	        if (mem->clustered)
-			vm_pagein_cluster_unused++;
-
-		if (mem->laundry) {
-			vm_pageout_throttle_up(mem);
-			counter(++c_laundry_pages_freed);
-		}
-		mem->busy = TRUE;
-
-		PAGE_WAKEUP(mem);	/* clears wanted */
-
-		if (mem->private)
-			mem->fictitious = TRUE;
-
 		if (!mem->fictitious) {
-		        /* depends on the queues lock */
-		        if (mem->zero_fill)
-			        vm_zf_count -= 1;
-			assert(!mem->laundry);
-			vm_page_init(mem, mem->phys_page);
-
 			mem->free = TRUE;
 
-			if (first == NULL)
-			        last = mem;
-			mem->pageq.next = (queue_t) first;
-			first = mem;
-
+			color = mem->phys_page & vm_color_mask;
+			if (queue_empty(&free_list[color])) {
+				inuse[color] = inuse_list_head;
+				inuse_list_head = color;
+			}
+			queue_enter_first(&free_list[color],
+					  mem,
+					  vm_page_t,
+					  pageq);
 			pg_count++;
 		} else {
-			mem->phys_page = vm_page_fictitious_addr;
+			assert(mem->phys_page == vm_page_fictitious_addr ||
+			       mem->phys_page == vm_page_guard_addr);
 		        vm_page_release_fictitious(mem);
 		}
 		mem = nxt;
 	}
-	if (first) {
-	      
+	if (pg_count) {
+	        unsigned int	avail_free_count;
+	  
 	        mutex_lock(&vm_page_queue_free_lock);
 
-		last->pageq.next = (queue_entry_t) vm_page_queue_free;
-		vm_page_queue_free = first;
+		color = inuse_list_head;
+		
+		while( color != -1 ) {
+			vm_page_t first, last;
+			vm_page_t first_free;
+
+			first = (vm_page_t) queue_first(&free_list[color]);
+			last = (vm_page_t) queue_last(&free_list[color]);
+			first_free = (vm_page_t) queue_first(&vm_page_queue_free[color]);
 
+			if (queue_empty(&vm_page_queue_free[color])) {
+				queue_last(&vm_page_queue_free[color]) =
+					(queue_entry_t) last;
+			} else {
+				queue_prev(&first_free->pageq) =
+					(queue_entry_t) last;
+			}
+			queue_first(&vm_page_queue_free[color]) =
+				(queue_entry_t) first;
+			queue_prev(&first->pageq) =
+				(queue_entry_t) &vm_page_queue_free[color];
+			queue_next(&last->pageq) =
+				(queue_entry_t) first_free;
+			color = inuse[color];
+		}
+		
 		vm_page_free_count += pg_count;
+		avail_free_count = vm_page_free_count;
+
+		while ((vm_page_free_wanted_privileged > 0) && avail_free_count) {
+		        vm_page_free_wanted_privileged--;
+			avail_free_count--;
+
+			thread_wakeup_one((event_t) &vm_page_free_wanted_privileged);
+		}
 
 		if ((vm_page_free_wanted > 0) &&
-		    (vm_page_free_count >= vm_page_free_reserved)) {
+		    (avail_free_count >= vm_page_free_reserved)) {
 		        unsigned int  available_pages;
 
-			if (vm_page_free_count >= vm_page_free_reserved) {
-				available_pages = (vm_page_free_count
-						   - vm_page_free_reserved);
+			if (avail_free_count >= vm_page_free_reserved) {
+				available_pages = (avail_free_count - vm_page_free_reserved);
 			} else {
 				available_pages = 0;
 			}
@@ -1910,6 +2242,25 @@ vm_page_free_list(
 			}
 		}
 		mutex_unlock(&vm_page_queue_free_lock);
+
+#if CONFIG_EMBEDDED
+		{
+		int percent_avail;
+
+		/*
+		 * Decide if we need to poke the memorystatus notification thread.
+		 */
+		percent_avail = 
+			(vm_page_active_count + vm_page_inactive_count + 
+			 vm_page_speculative_count + vm_page_free_count +
+			 vm_page_purgeable_count ) * 100 /
+			atop_64(max_mem);
+		if (percent_avail >= (kern_memorystatus_level + 5)) {
+			kern_memorystatus_level = percent_avail;
+			thread_wakeup((event_t)&kern_memorystatus_wakeup);
+		}
+		}
+#endif
 	}
 }
 
@@ -1933,7 +2284,7 @@ vm_page_wire(
 	VM_PAGE_CHECK(mem);
 #if DEBUG
 	if (mem->object)
-	        _mutex_assert(&mem->object->Lock, MA_OWNED);
+	        vm_object_lock_assert_exclusive(mem->object);
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 #endif
 	if (mem->wire_count == 0) {
@@ -1943,10 +2294,9 @@ vm_page_wire(
 		if (mem->gobbled)
 			vm_page_gobble_count--;
 		mem->gobbled = FALSE;
-		if(mem->zero_fill) {
-			/* depends on the queues lock */
-			vm_zf_count-=1;
+		if (mem->zero_fill == TRUE) {
 			mem->zero_fill = FALSE;
+		        OSAddAtomic(-1, (SInt32 *)&vm_zf_count);
 		}
 		/* 
 		 * ENCRYPTED SWAP:
@@ -1973,7 +2323,7 @@ void
 vm_page_gobble(
         register vm_page_t      mem)
 {
-        vm_page_lock_queues();
+        vm_page_lockspin_queues();
         VM_PAGE_CHECK(mem);
 
 	assert(!mem->gobbled);
@@ -2007,7 +2357,7 @@ vm_page_unwire(
 	assert(mem->wire_count > 0);
 #if DEBUG
 	if (mem->object)
-	        _mutex_assert(&mem->object->Lock, MA_OWNED);
+	        vm_object_lock_assert_exclusive(mem->object);
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 #endif
 	if (--mem->wire_count == 0) {
@@ -2016,13 +2366,23 @@ vm_page_unwire(
 		assert(!mem->laundry);
 		assert(mem->object != kernel_object);
 		assert(mem->pageq.next == NULL && mem->pageq.prev == NULL);
-		queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
-		vm_page_active_count++;
-		mem->active = TRUE;
+		if (!IP_VALID(memory_manager_default) && 
+			mem->dirty && mem->object->internal && 
+			(mem->object->purgable == VM_PURGABLE_DENY ||
+			 mem->object->purgable == VM_PURGABLE_NONVOLATILE)) {
+			queue_enter(&vm_page_queue_throttled, mem, vm_page_t, pageq);
+			vm_page_throttled_count++;
+			mem->throttled = TRUE;
+		} else {
+			queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq);
+			vm_page_active_count++;
+			mem->active = TRUE;
+		}
 		mem->reference = TRUE;
 	}
 }
 
+
 /*
  *	vm_page_deactivate:
  *
@@ -2036,8 +2396,11 @@ void
 vm_page_deactivate(
 	register vm_page_t	m)
 {
+        boolean_t rapid_age = FALSE;
+
 	VM_PAGE_CHECK(m);
 	assert(m->object != kernel_object);
+	assert(m->phys_page != vm_page_guard_addr);
 
 //	dbgLog(m->phys_page, vm_page_free_count, vm_page_wire_count, 6);	/* (TEST/DEBUG) */
 #if DEBUG
@@ -2052,6 +2415,7 @@ vm_page_deactivate(
 	 */
 	if (m->gobbled) {		/* can this happen? */
 		assert(m->wire_count == 0);
+
 		if (!m->private && !m->fictitious)
 			vm_page_wire_count--;
 		vm_page_gobble_count--;
@@ -2059,36 +2423,51 @@ vm_page_deactivate(
 	}
 	if (m->private || (m->wire_count != 0))
 		return;
-	if (m->active || (m->inactive && m->reference)) {
-		if (!m->fictitious && !m->absent)
-			pmap_clear_reference(m->phys_page);
-		m->reference = FALSE;
-		VM_PAGE_QUEUES_REMOVE(m);
+
+	if (m->active && m->deactivated == TRUE) {
+	        if (!pmap_is_referenced(m->phys_page))
+		        rapid_age = TRUE;
 	}
-	if (m->wire_count == 0 && !m->inactive) {
-		m->page_ticket = vm_page_ticket;
-		vm_page_ticket_roll++;
+	if (rapid_age == FALSE && !m->fictitious && !m->absent)
+		pmap_clear_reference(m->phys_page);
+
+	m->reference = FALSE;
+	m->deactivated = FALSE;
+	m->no_cache = FALSE;
+
+	if (!m->inactive) {
+		VM_PAGE_QUEUES_REMOVE(m);
 
-		if(vm_page_ticket_roll == VM_PAGE_TICKETS_IN_ROLL) {
-			vm_page_ticket_roll = 0;
-			if(vm_page_ticket == VM_PAGE_TICKET_ROLL_IDS)
-				vm_page_ticket= 0;
-			else
-				vm_page_ticket++;
-		}
-		
 		assert(!m->laundry);
 		assert(m->pageq.next == NULL && m->pageq.prev == NULL);
-		if(m->zero_fill) {
-			queue_enter(&vm_page_queue_zf, m, vm_page_t, pageq);
+
+		if (!IP_VALID(memory_manager_default) &&
+			m->dirty && m->object->internal &&
+			(m->object->purgable == VM_PURGABLE_DENY ||
+			 m->object->purgable == VM_PURGABLE_NONVOLATILE)) {
+			queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq);
+			m->throttled = TRUE;
+			vm_page_throttled_count++;
 		} else {
-			queue_enter(&vm_page_queue_inactive,
-							m, vm_page_t, pageq);
+			if (rapid_age == TRUE ||
+			    (!m->fictitious && m->object->named && m->object->ref_count == 1)) {
+			        vm_page_speculate(m, FALSE);
+				vm_page_speculative_recreated++;
+				return;
+			} else {
+				if (m->zero_fill) {
+					queue_enter(&vm_page_queue_zf, m, vm_page_t, pageq);
+					vm_zf_queue_count++;
+				} else {
+					queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
+				}
+			}
+			m->inactive = TRUE;
+			if (!m->fictitious) {
+			        vm_page_inactive_count++;
+				token_new_pagecount++;
+			}
 		}
-
-		m->inactive = TRUE;
-		if (!m->fictitious)
-			vm_page_inactive_count++;
 	}
 }
 
@@ -2105,7 +2484,10 @@ vm_page_activate(
 	register vm_page_t	m)
 {
 	VM_PAGE_CHECK(m);
+#ifdef	FIXME_4778297
 	assert(m->object != kernel_object);
+#endif
+	assert(m->phys_page != vm_page_guard_addr);
 #if DEBUG
 	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 #endif
@@ -2119,35 +2501,190 @@ vm_page_activate(
 	if (m->private)
 		return;
 
-	if (m->inactive) {
+#if DEBUG
+	if (m->active)
+	        panic("vm_page_activate: already active");
+#endif
+
+	if (m->speculative) {
+		DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
+		DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL);
+	}
+
+	VM_PAGE_QUEUES_REMOVE(m);
+
+	if (m->wire_count == 0) {
 		assert(!m->laundry);
-		if (m->zero_fill) {
-			queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
+		assert(m->pageq.next == NULL && m->pageq.prev == NULL);
+		if (!IP_VALID(memory_manager_default) && 
+			!m->fictitious && m->dirty && m->object->internal && 
+			(m->object->purgable == VM_PURGABLE_DENY ||
+			 m->object->purgable == VM_PURGABLE_NONVOLATILE)) {
+			queue_enter(&vm_page_queue_throttled, m, vm_page_t, pageq);
+			m->throttled = TRUE;
+			vm_page_throttled_count++;
 		} else {
-			queue_remove(&vm_page_queue_inactive, 
-						m, vm_page_t, pageq);
+			queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
+			m->active = TRUE;
+			if (!m->fictitious)
+				vm_page_active_count++;
 		}
-		m->pageq.next = NULL;
-		m->pageq.prev = NULL;
-		if (!m->fictitious)
-			vm_page_inactive_count--;
-		m->inactive = FALSE;
+		m->reference = TRUE;
+		m->no_cache = FALSE;
 	}
-	if (m->wire_count == 0) {
+}
+
+
+/*
+ *      vm_page_speculate:
+ *
+ *      Put the specified page on the speculative list (if appropriate).
+ *
+ *      The page queues must be locked.
+ */
+void
+vm_page_speculate(
+	vm_page_t	m,
+	boolean_t	new)
+{
+        struct vm_speculative_age_q	*aq;
+
+	VM_PAGE_CHECK(m);
+	assert(m->object != kernel_object);
+	assert(!m->speculative && !m->active && !m->inactive && !m->throttled);
+	assert(m->phys_page != vm_page_guard_addr);
+	assert(m->pageq.next == NULL && m->pageq.prev == NULL);
 #if DEBUG
-		if (m->active)
-			panic("vm_page_activate: already active");
+	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
 #endif
-		assert(!m->laundry);
-		assert(m->pageq.next == NULL && m->pageq.prev == NULL);
-		queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
-		m->active = TRUE;
-		m->reference = TRUE;
-		if (!m->fictitious)
-			vm_page_active_count++;
+	if (m->wire_count == 0) {
+	        mach_timespec_t		ts;
+
+		clock_get_system_nanotime(&ts.tv_sec, (unsigned *)&ts.tv_nsec);
+
+		if (vm_page_speculative_count == 0) {
+
+			speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+			speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+
+			aq = &vm_page_queue_speculative[speculative_age_index];
+
+		        /*
+			 * set the timer to begin a new group
+			 */
+			aq->age_ts.tv_sec = VM_PAGE_SPECULATIVE_Q_AGE_MS / 1000;
+			aq->age_ts.tv_nsec = (VM_PAGE_SPECULATIVE_Q_AGE_MS % 1000) * 1000 * NSEC_PER_USEC;
+
+			ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
+		} else {
+			aq = &vm_page_queue_speculative[speculative_age_index];
+
+			if (CMP_MACH_TIMESPEC(&ts, &aq->age_ts) >= 0) {
+
+			        speculative_age_index++;
+
+				if (speculative_age_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
+				        speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+				if (speculative_age_index == speculative_steal_index) {
+				        speculative_steal_index = speculative_age_index + 1;
+
+					if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
+					        speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+				}
+				aq = &vm_page_queue_speculative[speculative_age_index];
+
+				if (!queue_empty(&aq->age_q))
+				        vm_page_speculate_ageit(aq);
+
+				aq->age_ts.tv_sec = VM_PAGE_SPECULATIVE_Q_AGE_MS / 1000;
+				aq->age_ts.tv_nsec = (VM_PAGE_SPECULATIVE_Q_AGE_MS % 1000) * 1000 * NSEC_PER_USEC;
+
+				ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
+			}
+		}
+		enqueue_tail(&aq->age_q, &m->pageq);
+		m->speculative = TRUE;
+		vm_page_speculative_count++;
+
+		if (new == TRUE) {
+		        m->object->pages_created++;
+			vm_page_speculative_created++;
+		}
+	}
+}
+
+
+/*
+ * move pages from the specified aging bin to
+ * the speculative bin that pageout_scan claims from
+ *
+ *      The page queues must be locked.
+ */
+void
+vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
+{
+        struct vm_speculative_age_q	*sq;
+	vm_page_t	t;
+
+	sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
+	if (queue_empty(&sq->age_q)) {
+	        sq->age_q.next = aq->age_q.next;
+		sq->age_q.prev = aq->age_q.prev;
+		
+		t = (vm_page_t)sq->age_q.next;
+		t->pageq.prev = &sq->age_q;
+
+		t = (vm_page_t)sq->age_q.prev;
+		t->pageq.next = &sq->age_q;
+	} else {
+	        t = (vm_page_t)sq->age_q.prev;
+		t->pageq.next = aq->age_q.next;
+						
+		t = (vm_page_t)aq->age_q.next;
+		t->pageq.prev = sq->age_q.prev;
+
+		t = (vm_page_t)aq->age_q.prev;
+		t->pageq.next = &sq->age_q;
+
+		sq->age_q.prev = aq->age_q.prev;
 	}
+	queue_init(&aq->age_q);
+}
+
+
+void
+vm_page_lru(
+	vm_page_t	m)
+{
+	VM_PAGE_CHECK(m);
+	assert(m->object != kernel_object);
+	assert(m->phys_page != vm_page_guard_addr);
+
+#if DEBUG
+	_mutex_assert(&vm_page_queue_lock, MA_OWNED);
+#endif
+	if (m->active || m->reference)
+		return;
+
+	if (m->private || (m->wire_count != 0))
+		return;
+
+	m->no_cache = FALSE;
+
+	VM_PAGE_QUEUES_REMOVE(m);
+
+	assert(!m->laundry);
+	assert(m->pageq.next == NULL && m->pageq.prev == NULL);
+
+	queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq);
+	m->inactive = TRUE;
+
+        vm_page_inactive_count++;
+	token_new_pagecount++;
 }
 
+
 /*
  *	vm_page_part_zero_fill:
  *
@@ -2239,6 +2776,9 @@ vm_page_part_copy(
  * make sure the page is decrypted first, if necessary.
  */
 
+int vm_page_copy_cs_validations = 0;
+int vm_page_copy_cs_tainted = 0;
+
 void
 vm_page_copy(
 	vm_page_t	src_m,
@@ -2264,32 +2804,38 @@ vm_page_copy(
 	}
 	dest_m->encrypted = FALSE;
 
+	if (src_m->object != VM_OBJECT_NULL &&
+	    src_m->object->code_signed &&
+	    !src_m->cs_validated) {
+		/*
+		 * We're copying a not-yet-validated page from a
+		 * code-signed object.
+		 * Whoever ends up mapping the copy page might care about
+		 * the original page's integrity, so let's validate the
+		 * source page now.
+		 */
+		vm_page_copy_cs_validations++;
+		vm_page_validate_cs(src_m);
+	}
+	/*
+	 * Propagate the code-signing bits to the copy page.
+	 */
+	dest_m->cs_validated = src_m->cs_validated;
+	dest_m->cs_tainted = src_m->cs_tainted;
+	if (dest_m->cs_tainted) {
+		assert(dest_m->cs_validated);
+		vm_page_copy_cs_tainted++;
+	}
+
 	pmap_copy_page(src_m->phys_page, dest_m->phys_page);
 }
 
-/*
- *	Currently, this is a primitive allocator that grabs
- *	free pages from the system, sorts them by physical
- *	address, then searches for a region large enough to
- *	satisfy the user's request.
- *
- *	Additional levels of effort:
- *		+ steal clean active/inactive pages
- *		+ force pageouts of dirty pages
- *		+ maintain a map of available physical
- *		memory
- */
-
-#if	MACH_ASSERT
+#if MACH_ASSERT
 /*
  *	Check that the list of pages is ordered by
  *	ascending physical address and has no holes.
  */
-int	vm_page_verify_contiguous(
-	vm_page_t	pages,
-	unsigned int	npages);
-
-int
+static int
 vm_page_verify_contiguous(
 	vm_page_t	pages,
 	unsigned int	npages)
@@ -2302,16 +2848,16 @@ vm_page_verify_contiguous(
 	page_count = 1;
 	for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
 		if (m->phys_page != prev_addr + 1) {
-			printf("m 0x%x prev_addr 0x%x, current addr 0x%x\n",
+			printf("m %p prev_addr 0x%x, current addr 0x%x\n",
 			       m, prev_addr, m->phys_page);
-			printf("pages 0x%x page_count %d\n", pages, page_count);
+			printf("pages %p page_count %d\n", pages, page_count);
 			panic("vm_page_verify_contiguous:  not contiguous!");
 		}
 		prev_addr = m->phys_page;
 		++page_count;
 	}
 	if (page_count != npages) {
-		printf("pages 0x%x actual count 0x%x but requested 0x%x\n",
+		printf("pages %p actual count 0x%x but requested 0x%x\n",
 		       pages, page_count, npages);
 		panic("vm_page_verify_contiguous:  count error");
 	}
@@ -2320,213 +2866,522 @@ vm_page_verify_contiguous(
 #endif	/* MACH_ASSERT */
 
 
-cpm_counter(unsigned int	vpfls_pages_handled = 0;)
-cpm_counter(unsigned int	vpfls_head_insertions = 0;)
-cpm_counter(unsigned int	vpfls_tail_insertions = 0;)
-cpm_counter(unsigned int	vpfls_general_insertions = 0;)
-cpm_counter(unsigned int	vpfc_failed = 0;)
-cpm_counter(unsigned int	vpfc_satisfied = 0;)
+#if MACH_ASSERT
+/*
+ *	Check the free lists for proper length etc.
+ */
+static void
+vm_page_verify_free_lists( void )
+{
+	unsigned int	color, npages;
+	vm_page_t	m;
+	vm_page_t	prev_m;
+	
+	npages = 0;
+	
+	mutex_lock(&vm_page_queue_free_lock);
+
+	for( color = 0; color < vm_colors; color++ ) {
+		prev_m = (vm_page_t) &vm_page_queue_free[color];
+		queue_iterate(&vm_page_queue_free[color],
+			      m,
+			      vm_page_t,
+			      pageq) {
+			if ((vm_page_t) m->pageq.prev != prev_m)
+				panic("vm_page_verify_free_lists: corrupted prev ptr");
+			if ( ! m->free )
+				panic("vm_page_verify_free_lists: not free");
+			if ( ! m->busy )
+				panic("vm_page_verify_free_lists: not busy");
+			if ( (m->phys_page & vm_color_mask) != color)
+				panic("vm_page_verify_free_lists: wrong color");
+			++npages;
+			prev_m = m;
+		}
+	}
+	if (npages != vm_page_free_count)
+		panic("vm_page_verify_free_lists:  npages %u free_count %d",
+		      npages, vm_page_free_count);
+
+	mutex_unlock(&vm_page_queue_free_lock);
+}
+#endif	/* MACH_ASSERT */
+
+
 
 /*
- *	Find a region large enough to contain at least npages
+ *	CONTIGUOUS PAGE ALLOCATION
+ *	Additional levels of effort:
+ *		+ consider pages that are currently 'pmapped'
+ *		    this could be expensive since we'd have
+ * 		    to ask the pmap layer about there state
+ *		+ consider dirty pages
+ * 		    either clean them or
+ *		    copy them to other locations...
+ *
+ *	Find a region large enough to contain at least n pages
  *	of contiguous physical memory.
  *
+ *	This is done by traversing the vm_page_t array in a linear fashion
+ *	we assume that the vm_page_t array has the avaiable physical pages in an
+ *	ordered, ascending list... this is currently true of all our implementations
+ * 	and must remain so... there can be 'holes' in the array...  we also can
+ *	no longer tolerate the vm_page_t's in the list being 'freed' and reclaimed
+ * 	which use to happen via 'vm_page_convert'... that function was no longer
+ * 	being called and was removed...
+ *	
+ *	The basic flow consists of stabilizing some of the interesting state of 
+ *	a vm_page_t behind the vm_page_queue and vm_page_free locks... we start our
+ *	sweep at the beginning of the array looking for pages that meet our criterea
+ *	for a 'stealable' page... currently we are pretty conservative... if the page
+ *	meets this criterea and is physically contiguous to the previous page in the 'run'
+ * 	we keep developing it.  If we hit a page that doesn't fit, we reset our state
+ *	and start to develop a new run... if at this point we've already considered
+ * 	at least MAX_CONSIDERED_BEFORE_YIELD pages, we'll drop the 2 locks we hold,
+ *	and mutex_pause (which will yield the processor), to keep the latency low w/r 
+ *	to other threads trying to acquire free pages (or move pages from q to q),
+ *	and then continue from the spot we left off... we only make 1 pass through the
+ *	array.  Once we have a 'run' that is long enough, we'll go into the loop which
+ * 	which steals the pages from the queues they're currently on... pages on the free
+ *	queue can be stolen directly... pages that are on any of the other queues
+ *	must be removed from the object they are tabled on... this requires taking the
+ * 	object lock... we do this as a 'try' to prevent deadlocks... if the 'try' fails
+ *	or if the state of the page behind the vm_object lock is no longer viable, we'll
+ *	dump the pages we've currently stolen back to the free list, and pick up our
+ *	scan from the point where we aborted the 'current' run.
+ *
+ *
  *	Requirements:
- *		- Called while holding vm_page_queue_free_lock.
- *		- Doesn't respect vm_page_free_reserved; caller
- *		must not ask for more pages than are legal to grab.
+ *		- neither vm_page_queue nor vm_free_list lock can be held on entry
  *
- *	Returns a pointer to a list of gobbled pages or	VM_PAGE_NULL.
+ *	Returns a pointer to a list of gobbled/wired pages or VM_PAGE_NULL.
  *
  * Algorithm:
- *	Loop over the free list, extracting one page at a time and
- *	inserting those into a sorted sub-list.  We stop as soon as
- *	there's a contiguous range within the sorted list that can
- *	satisfy the contiguous memory request.  This contiguous sub-
- *	list is chopped out of the sorted sub-list and the remainder
- *	of the sorted sub-list is put back onto the beginning of the
- *	free list.
  */
+
+#define	MAX_CONSIDERED_BEFORE_YIELD	1000
+
+
+#define RESET_STATE_OF_RUN()	\
+	MACRO_BEGIN		\
+	prevcontaddr = -2;	\
+	free_considered = 0;	\
+	substitute_needed = 0;	\
+	npages = 0;		\
+	MACRO_END			
+
+
 static vm_page_t
 vm_page_find_contiguous(
-	unsigned int	contig_pages)
+	unsigned int	contig_pages,
+	ppnum_t		max_pnum,
+	boolean_t	wire)
 {
-	vm_page_t	sort_list;
-	vm_page_t	*contfirstprev, contlast;
-	vm_page_t	m, m1;
+	vm_page_t	m = NULL;
 	ppnum_t		prevcontaddr;
-	ppnum_t		nextcontaddr;
-	unsigned int	npages;
-
-	m = NULL;
-#if DEBUG
-	_mutex_assert(&vm_page_queue_free_lock, MA_OWNED);
+	unsigned int	npages, considered;
+	unsigned int	page_idx, start_idx;
+	int		free_considered, free_available;
+	int		substitute_needed;
+#if MACH_ASSERT
+	uint32_t	tv_start_sec, tv_start_usec, tv_end_sec, tv_end_usec;
+	int		yielded = 0;
+	int		dumped_run = 0;
+	int		stolen_pages = 0;
 #endif
-#if	MACH_ASSERT
-	/*
-	 *	Verify pages in the free list..
-	 */
-	npages = 0;
-	for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m))
-		++npages;
-	if (npages != vm_page_free_count)
-		panic("vm_sort_free_list:  prelim:  npages %u free_count %d",
-		      npages, vm_page_free_count);
-#endif	/* MACH_ASSERT */
 
-	if (contig_pages == 0 || vm_page_queue_free == VM_PAGE_NULL)
+	if (contig_pages == 0)
 		return VM_PAGE_NULL;
 
-#define PPNUM_PREV(x)	(((x) > 0) ? ((x) - 1) : 0)
-#define PPNUM_NEXT(x)	(((x) < PPNUM_MAX) ? ((x) + 1) : PPNUM_MAX)
-#define SET_NEXT_PAGE(m,n)	((m)->pageq.next = (struct queue_entry *) (n))
+#if MACH_ASSERT
+	vm_page_verify_free_lists();
+
+	clock_get_system_microtime(&tv_start_sec, &tv_start_usec);
+#endif
+	vm_page_lock_queues();
+	mutex_lock(&vm_page_queue_free_lock);
+
+	RESET_STATE_OF_RUN();
 
-	npages = 1;
-	contfirstprev = &sort_list;
-	contlast = sort_list = vm_page_queue_free;
-	vm_page_queue_free = NEXT_PAGE(sort_list);
-	SET_NEXT_PAGE(sort_list, VM_PAGE_NULL);
-	prevcontaddr = PPNUM_PREV(sort_list->phys_page);
-	nextcontaddr = PPNUM_NEXT(sort_list->phys_page);
+	considered = 0;
+	free_available = vm_page_free_count - vm_page_free_reserved;
 
- 	while (npages < contig_pages && 
-	       (m = vm_page_queue_free) != VM_PAGE_NULL)
-	{
-		cpm_counter(++vpfls_pages_handled);
+	for (page_idx = 0, start_idx = 0;
+	     npages < contig_pages && page_idx < vm_pages_count;
+	     page_idx++) {
+retry:		
+		m = &vm_pages[page_idx];
 
-		/* prepend to existing run? */
-		if (m->phys_page == prevcontaddr)
-		{
-			vm_page_queue_free = NEXT_PAGE(m);
-			cpm_counter(++vpfls_head_insertions);
-			prevcontaddr = PPNUM_PREV(prevcontaddr);
-			SET_NEXT_PAGE(m, *contfirstprev);
-			*contfirstprev = m;
-			npages++;
-			continue; /* no tail expansion check needed */
-		} 
-
-		/* append to tail of existing run? */
-		else if (m->phys_page == nextcontaddr)
-		{
-			vm_page_queue_free = NEXT_PAGE(m);
-			cpm_counter(++vpfls_tail_insertions);
-			nextcontaddr = PPNUM_NEXT(nextcontaddr);
-			SET_NEXT_PAGE(m, NEXT_PAGE(contlast));
-			SET_NEXT_PAGE(contlast, m);
-			contlast = m;
-			npages++;
+		if (max_pnum && m->phys_page > max_pnum) {
+			/* no more low pages... */
+			break;
 		}
+		if (m->phys_page <= vm_lopage_poolend &&
+		    m->phys_page >= vm_lopage_poolstart) {
+			/*
+			 * don't want to take pages from our
+			 * reserved pool of low memory
+			 * so don't consider it which
+			 * means starting a new run
+			 */
+			RESET_STATE_OF_RUN();
 
-		/* prepend to the very front of sorted list? */
-		else if (m->phys_page < sort_list->phys_page)
-		{
-			vm_page_queue_free = NEXT_PAGE(m);
-			cpm_counter(++vpfls_general_insertions);
-			prevcontaddr = PPNUM_PREV(m->phys_page);
-			nextcontaddr = PPNUM_NEXT(m->phys_page);
-			SET_NEXT_PAGE(m, sort_list);
-			contfirstprev = &sort_list;
-			contlast = sort_list = m;
-			npages = 1;
-		}
+		} else if (m->wire_count || m->gobbled ||
+			   m->encrypted || m->encrypted_cleaning || m->cs_validated || m->cs_tainted ||
+			   m->error || m->absent || m->pageout_queue || m->laundry || m->wanted || m->precious ||
+			   m->cleaning || m->overwriting || m->restart || m->unusual || m->list_req_pending) {
+			/*
+			 * page is in a transient state
+			 * or a state we don't want to deal
+			 * with, so don't consider it which
+			 * means starting a new run
+			 */
+			RESET_STATE_OF_RUN();
 
-		else /* get to proper place for insertion */
-		{
-			if (m->phys_page < nextcontaddr)
-			{
-				prevcontaddr = PPNUM_PREV(sort_list->phys_page);
-				nextcontaddr = PPNUM_NEXT(sort_list->phys_page);
-				contfirstprev = &sort_list;
-				contlast = sort_list;
+		} else if (!m->free && !m->active && !m->inactive && !m->speculative && !m->throttled) {
+			/*
+			 * page needs to be on one of our queues
+			 * in order for it to be stable behind the
+			 * locks we hold at this point...
+			 * if not, don't consider it which
+			 * means starting a new run
+			 */
+			RESET_STATE_OF_RUN();
+
+		} else if (!m->free && (!m->tabled || m->busy)) {
+			/*
+			 * pages on the free list are always 'busy'
+			 * so we couldn't test for 'busy' in the check
+			 * for the transient states... pages that are
+			 * 'free' are never 'tabled', so we also couldn't
+			 * test for 'tabled'.  So we check here to make
+			 * sure that a non-free page is not busy and is
+			 * tabled on an object... 
+			 * if not, don't consider it which
+			 * means starting a new run
+			 */
+			RESET_STATE_OF_RUN();
+
+		} else {
+			if (m->phys_page != prevcontaddr + 1) {
 				npages = 1;
+				start_idx = page_idx;
+			} else {
+				npages++;
 			}
-			for (m1 = NEXT_PAGE(contlast);
-			     npages < contig_pages &&
-			     m1 != VM_PAGE_NULL && m1->phys_page < m->phys_page;
-			     m1 = NEXT_PAGE(m1))
-			{
-				if (m1->phys_page != nextcontaddr) {
-					prevcontaddr = PPNUM_PREV(m1->phys_page);
-					contfirstprev = NEXT_PAGE_PTR(contlast);
-					npages = 1;
-				} else {
-					npages++;
+			prevcontaddr = m->phys_page;
+
+			if (m->pmapped || m->dirty)
+				substitute_needed++;
+
+			if (m->free) {
+				free_considered++;
+			}
+			if ((free_considered + substitute_needed) > free_available) {	
+				/*
+				 * if we let this run continue
+				 * we will end up dropping the vm_page_free_count
+				 * below the reserve limit... we need to abort
+				 * this run, but we can at least re-consider this
+				 * page... thus the jump back to 'retry'
+				 */
+				RESET_STATE_OF_RUN();
+
+				if (free_available && considered <= MAX_CONSIDERED_BEFORE_YIELD) {
+					considered++;
+					goto retry;
 				}
-				nextcontaddr = PPNUM_NEXT(m1->phys_page);
-				contlast = m1;
+				/*
+				 * free_available == 0
+				 * so can't consider any free pages... if
+				 * we went to retry in this case, we'd
+				 * get stuck looking at the same page
+				 * w/o making any forward progress
+				 * we also want to take this path if we've already
+				 * reached our limit that controls the lock latency
+				 */
 			}
+		}
+		if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) {
+			
+			mutex_unlock(&vm_page_queue_free_lock);
+			vm_page_unlock_queues();
 
+			mutex_pause(0);
+
+			vm_page_lock_queues();
+			mutex_lock(&vm_page_queue_free_lock);
+
+			RESET_STATE_OF_RUN();
 			/*
-			 * We may actually already have enough.
-			 * This could happen if a previous prepend
-			 * joined up two runs to meet our needs.
-			 * If so, bail before we take the current
-			 * page off the free queue.
+			 * reset our free page limit since we
+			 * dropped the lock protecting the vm_page_free_queue
 			 */
-			if (npages == contig_pages)
-				break;
+			free_available = vm_page_free_count - vm_page_free_reserved;
+			considered = 0;
+#if MACH_ASSERT
+			yielded++;
+#endif
+			goto retry;
+		}
+		considered++;
+	}
+	m = VM_PAGE_NULL;
+
+	if (npages != contig_pages)
+		mutex_unlock(&vm_page_queue_free_lock);
+	else {
+		vm_page_t	m1;
+		vm_page_t	m2;
+		unsigned int	cur_idx;
+		unsigned int	tmp_start_idx;
+		vm_object_t	locked_object = VM_OBJECT_NULL;
+		boolean_t	abort_run = FALSE;
+		
+		tmp_start_idx = start_idx;
+
+		/*
+		 * first pass through to pull the free pages
+		 * off of the free queue so that in case we
+		 * need substitute pages, we won't grab any 
+		 * of the free pages in the run... we'll clear
+		 * the 'free' bit in the 2nd pass, and even in
+		 * an abort_run case, we'll collect all of the
+		 * free pages in this run and return them to the free list
+		 */
+		while (start_idx < page_idx) {
+
+			m1 = &vm_pages[start_idx++];
+
+			if (m1->free) {
+				unsigned int color;
+
+				color = m1->phys_page & vm_color_mask;
+				queue_remove(&vm_page_queue_free[color],
+					     m1,
+					     vm_page_t,
+					     pageq);
+
+				vm_page_free_count--;
+			}
+		}
+		/*
+		 * adjust global freelist counts
+		 */
+		if (vm_page_free_count < vm_page_free_count_minimum)
+			vm_page_free_count_minimum = vm_page_free_count;
+
+		/*
+		 * we can drop the free queue lock at this point since
+		 * we've pulled any 'free' candidates off of the list
+		 * we need it dropped so that we can do a vm_page_grab
+		 * when substituing for pmapped/dirty pages
+		 */
+		mutex_unlock(&vm_page_queue_free_lock);
+
+		start_idx = tmp_start_idx;
+		cur_idx = page_idx - 1;
+
+		while (start_idx++ < page_idx) {
+			/*
+			 * must go through the list from back to front
+			 * so that the page list is created in the
+			 * correct order - low -> high phys addresses
+			 */
+			m1 = &vm_pages[cur_idx--];
+
+			if (m1->free) {
+				/*
+				 * pages have already been removed from
+				 * the free list in the 1st pass
+				 */
+				assert(m1->free);
+				assert(m1->busy);
+				assert(!m1->wanted);
+				assert(!m1->laundry);
+				m1->free = FALSE;
 
-			if (m->phys_page != nextcontaddr) 
-			{
-				contfirstprev = NEXT_PAGE_PTR(contlast);
-				prevcontaddr = PPNUM_PREV(m->phys_page);
-				nextcontaddr = PPNUM_NEXT(m->phys_page);
-				npages = 1;
 			} else {
-				nextcontaddr = PPNUM_NEXT(nextcontaddr);
-				npages++;
+				vm_object_t object;
+
+				if (abort_run == TRUE)
+					continue;
+
+				object = m1->object;
+
+				if (object != locked_object) {
+					if (locked_object) {
+						vm_object_unlock(locked_object);
+						locked_object = VM_OBJECT_NULL;
+					}
+					if (vm_object_lock_try(object))
+						locked_object = object;
+				}
+				if (locked_object == VM_OBJECT_NULL || 
+				    (m1->wire_count || m1->gobbled ||
+				     m1->encrypted || m1->encrypted_cleaning || m1->cs_validated || m1->cs_tainted ||
+				     m1->error || m1->absent || m1->pageout_queue || m1->laundry || m1->wanted || m1->precious ||
+				     m1->cleaning || m1->overwriting || m1->restart || m1->unusual || m1->list_req_pending || m1->busy)) {
+
+					if (locked_object) {
+						vm_object_unlock(locked_object);
+						locked_object = VM_OBJECT_NULL;
+					}
+					tmp_start_idx = cur_idx;
+					abort_run = TRUE;
+					continue;
+				}
+				if (m1->pmapped || m1->dirty) {
+					int refmod;
+					vm_object_offset_t offset;
+
+					m2 = vm_page_grab();
+
+					if (m2 == VM_PAGE_NULL) {
+						if (locked_object) {
+							vm_object_unlock(locked_object);
+							locked_object = VM_OBJECT_NULL;
+						}
+						tmp_start_idx = cur_idx;
+						abort_run = TRUE;
+						continue;
+					}
+					if (m1->pmapped)
+						refmod = pmap_disconnect(m1->phys_page);
+					else
+						refmod = 0;
+					vm_page_copy(m1, m2);
+		  
+					m2->reference = m1->reference;
+					m2->dirty     = m1->dirty;
+
+					if (refmod & VM_MEM_REFERENCED)
+						m2->reference = TRUE;
+					if (refmod & VM_MEM_MODIFIED)
+						m2->dirty = TRUE;
+					offset = m1->offset;
+
+					/*
+					 * completely cleans up the state
+					 * of the page so that it is ready
+					 * to be put onto the free list, or
+					 * for this purpose it looks like it
+					 * just came off of the free list
+					 */
+					vm_page_free_prepare(m1);
+
+					/*
+					 * make sure we clear the ref/mod state
+					 * from the pmap layer... else we risk
+					 * inheriting state from the last time
+					 * this page was used...
+					 */
+					pmap_clear_refmod(m2->phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
+					/*
+					 * now put the substitute page on the object
+					 */
+					vm_page_insert_internal(m2, locked_object, offset, TRUE);
+
+					if (m2->reference)
+						vm_page_activate(m2);
+					else
+						vm_page_deactivate(m2);
+
+					PAGE_WAKEUP_DONE(m2);
+
+				} else {
+					/*
+					 * completely cleans up the state
+					 * of the page so that it is ready
+					 * to be put onto the free list, or
+					 * for this purpose it looks like it
+					 * just came off of the free list
+					 */
+					vm_page_free_prepare(m1);
+				}
+#if MACH_ASSERT
+				stolen_pages++;
+#endif
 			}
-			vm_page_queue_free = NEXT_PAGE(m);
-			cpm_counter(++vpfls_general_insertions);
-			SET_NEXT_PAGE(m, NEXT_PAGE(contlast));
-			SET_NEXT_PAGE(contlast, m);
-			contlast = m;
+			m1->pageq.next = (queue_entry_t) m;
+			m1->pageq.prev = NULL;
+			m = m1;
 		}
-		
-		/* See how many pages are now contiguous after the insertion */
-		for (m1 = NEXT_PAGE(m);
-		     npages < contig_pages &&
-		     m1 != VM_PAGE_NULL && m1->phys_page == nextcontaddr;
-		     m1 = NEXT_PAGE(m1))
-		{
-			nextcontaddr = PPNUM_NEXT(nextcontaddr);
-			contlast = m1;
-			npages++;
+		if (locked_object) {
+			vm_object_unlock(locked_object);
+			locked_object = VM_OBJECT_NULL;
 		}
-	}
 
-	/* how did we do? */
-	if (npages == contig_pages)
-	{
-		cpm_counter(++vpfc_satisfied);
+		if (abort_run == TRUE) {
+			if (m != VM_PAGE_NULL) {
+				vm_page_free_list(m);
+			}
+#if MACH_ASSERT
+			dumped_run++;
+#endif
+			/*
+			 * want the index of the last
+			 * page in this run that was
+			 * successfully 'stolen', so back
+			 * it up 1 for the auto-decrement on use
+			 * and 1 more to bump back over this page
+			 */
+			page_idx = tmp_start_idx + 2;
 
-		/* remove the contiguous range from the sorted list */
-		m = *contfirstprev;
-		*contfirstprev = NEXT_PAGE(contlast);
-		SET_NEXT_PAGE(contlast, VM_PAGE_NULL);
-		assert(vm_page_verify_contiguous(m, npages));
+			if (page_idx >= vm_pages_count)
+				goto done_scanning;
+
+			mutex_lock(&vm_page_queue_free_lock);
+
+			RESET_STATE_OF_RUN();
+
+			/*
+			 * reset our free page limit since we
+			 * dropped the lock protecting the vm_page_free_queue
+			 */
+			free_available = vm_page_free_count - vm_page_free_reserved;
+			
+			goto retry;
+		}
 
-		/* inline vm_page_gobble() for each returned page */
 		for (m1 = m; m1 != VM_PAGE_NULL; m1 = NEXT_PAGE(m1)) {
-			assert(m1->free);
-			assert(!m1->wanted);
-			assert(!m1->laundry);
-			m1->free = FALSE;
-			m1->no_isync = TRUE;
-			m1->gobbled = TRUE;
+
+			if (wire == TRUE)
+				m1->wire_count++;
+			else
+				m1->gobbled = TRUE;
 		}
+		if (wire == FALSE)
+			vm_page_gobble_count += npages;
+
+		/*
+		 * gobbled pages are also counted as wired pages
+		 */
 		vm_page_wire_count += npages;
-		vm_page_gobble_count += npages;
-		vm_page_free_count -= npages;
 
-		/* stick free list at the tail of the sorted list  */
-		while ((m1 = *contfirstprev) != VM_PAGE_NULL)
-			contfirstprev = (vm_page_t *)&m1->pageq.next;
-		*contfirstprev = vm_page_queue_free;
+ 		assert(vm_page_verify_contiguous(m, npages));
+	}
+done_scanning:
+	vm_page_unlock_queues();
+
+#if MACH_ASSERT
+	clock_get_system_microtime(&tv_end_sec, &tv_end_usec);
+
+	tv_end_sec -= tv_start_sec;
+	if (tv_end_usec < tv_start_usec) {
+		tv_end_sec--;
+		tv_end_usec += 1000000;
 	}
+	tv_end_usec -= tv_start_usec;
+	if (tv_end_usec >= 1000000) {
+		tv_end_sec++;
+		tv_end_sec -= 1000000;
+	}
+	printf("vm_find_page_contiguous(num=%d,low=%d): found %d pages in %d.%06ds...  scanned %d pages...  yielded %d times...  dumped run %d times... stole %d pages\n",
+	       contig_pages, max_pnum, npages, tv_end_sec, tv_end_usec, page_idx, yielded, dumped_run, stolen_pages);
 
-	vm_page_queue_free = sort_list;
+	vm_page_verify_free_lists();
+#endif
 	return m;
 }
 
@@ -2537,78 +3392,52 @@ kern_return_t
 cpm_allocate(
 	vm_size_t	size,
 	vm_page_t	*list,
+	ppnum_t		max_pnum,
 	boolean_t	wire)
 {
-	register vm_page_t	m;
 	vm_page_t		pages;
 	unsigned int		npages;
-	unsigned int		vm_pages_available;
-	boolean_t		wakeup;
 
 	if (size % page_size != 0)
 		return KERN_INVALID_ARGUMENT;
 
-	vm_page_lock_queues();
-	mutex_lock(&vm_page_queue_free_lock);
-
-	/*
-	 *	Should also take active and inactive pages
-	 *	into account...  One day...
-	 */
 	npages = size / page_size;
-	vm_pages_available = vm_page_free_count - vm_page_free_reserved;
-
-	if (npages > vm_pages_available) {
-		mutex_unlock(&vm_page_queue_free_lock);
-		vm_page_unlock_queues();
-		return KERN_RESOURCE_SHORTAGE;
-	}
 
 	/*
 	 *	Obtain a pointer to a subset of the free
 	 *	list large enough to satisfy the request;
 	 *	the region will be physically contiguous.
 	 */
-	pages = vm_page_find_contiguous(npages);
+	pages = vm_page_find_contiguous(npages, max_pnum, wire);
 
-	/* adjust global freelist counts and determine need for wakeups */
-	if (vm_page_free_count < vm_page_free_count_minimum)
-		vm_page_free_count_minimum = vm_page_free_count;
-
-	wakeup = ((vm_page_free_count < vm_page_free_min) ||
-		  ((vm_page_free_count < vm_page_free_target) &&
-		   (vm_page_inactive_count < vm_page_inactive_target)));
-		
-	mutex_unlock(&vm_page_queue_free_lock);
-
-	if (pages == VM_PAGE_NULL) {
-		vm_page_unlock_queues();
+	if (pages == VM_PAGE_NULL)
 		return KERN_NO_SPACE;
-	}
-
 	/*
-	 *	Walk the returned list, wiring the pages.
+	 * determine need for wakeups
 	 */
-	if (wire == TRUE)
-		for (m = pages; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
-			/*
-			 *	Essentially inlined vm_page_wire.
-			 */
-			assert(!m->active);
-			assert(!m->inactive);
-			assert(!m->private);
-			assert(!m->fictitious);
-			assert(m->wire_count == 0);
-			assert(m->gobbled);
-			m->gobbled = FALSE;
-			m->wire_count++;
-			--vm_page_gobble_count;
-		}
-	vm_page_unlock_queues();
-
-	if (wakeup)
+	if ((vm_page_free_count < vm_page_free_min) ||
+	    ((vm_page_free_count < vm_page_free_target) &&
+	     ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_min)))
 		thread_wakeup((event_t) &vm_page_free_wanted);
+		
+#if CONFIG_EMBEDDED
+	{
+	int			percent_avail;
 
+	/*
+	 * Decide if we need to poke the memorystatus notification thread.
+	 */
+	percent_avail = 
+		(vm_page_active_count + vm_page_inactive_count + 
+		 vm_page_speculative_count + vm_page_free_count +
+		 vm_page_purgeable_count ) * 100 /
+		atop_64(max_mem);
+	if (percent_avail <= (kern_memorystatus_level - 5)) {
+		kern_memorystatus_level = percent_avail;
+		thread_wakeup((event_t)&kern_memorystatus_wakeup);
+	}
+	}
+#endif
 	/*
 	 *	The CPM pages should now be available and
 	 *	ordered by ascending physical address.
@@ -2618,7 +3447,7 @@ cpm_allocate(
 	*list = pages;
 	return KERN_SUCCESS;
 }
-
+	
 
 #include <mach_vm_debug.h>
 #if	MACH_VM_DEBUG
@@ -2691,9 +3520,10 @@ vm_page_print(
 	printf(", offset=0x%x", p->offset);
 	printf(", wire_count=%d", p->wire_count);
 
-	iprintf("%sinactive, %sactive, %sgobbled, %slaundry, %sfree, %sref, %sencrypted\n",
+	iprintf("%sinactive, %sactive, %sthrottled, %sgobbled, %slaundry, %sfree, %sref, %sencrypted\n",
 		(p->inactive ? "" : "!"),
 		(p->active ? "" : "!"),
+		(p->throttled ? "" : "!"),
 		(p->gobbled ? "" : "!"),
 		(p->laundry ? "" : "!"),
 		(p->free ? "" : "!"),
@@ -2713,16 +3543,12 @@ vm_page_print(
 		(p->cleaning ? "" : "!"),
 		(p->pageout ? "" : "!"),
 		(p->clustered ? "" : "!"));
-	iprintf("%slock_supplied, %soverwriting, %srestart, %sunusual\n",
-		(p->lock_supplied ? "" : "!"),
+	iprintf("%soverwriting, %srestart, %sunusual\n",
 		(p->overwriting ? "" : "!"),
 		(p->restart ? "" : "!"),
 		(p->unusual ? "" : "!"));
 
 	iprintf("phys_page=0x%x", p->phys_page);
-	printf(", page_error=0x%x", p->page_error);
-	printf(", page_lock=0x%x", p->page_lock);
-	printf(", unlock_request=%d\n", p->unlock_request);
 
 	db_indent -= 2;
 }
diff --git a/osfmk/vm/vm_shared_memory_server.c b/osfmk/vm/vm_shared_memory_server.c
deleted file mode 100644
index cc895e0e5..000000000
--- a/osfmk/vm/vm_shared_memory_server.c
+++ /dev/null
@@ -1,2581 +0,0 @@
-/*
- * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *
- *	File: vm/vm_shared_memory_server.c
- *	Author: Chris Youngworth
- *
- *      Support routines for an in-kernel shared memory allocator
- */
-
-#include <debug.h>
-
-#include <mach/mach_types.h>
-#include <mach/kern_return.h>
-#include <mach/vm_inherit.h>
-#include <mach/vm_map.h>
-#include <machine/cpu_capabilities.h>
-
-#include <kern/kern_types.h>
-#include <kern/ipc_kobject.h>
-#include <kern/thread.h>
-#include <kern/zalloc.h>
-#include <kern/kalloc.h>
-
-#include <ipc/ipc_types.h>
-#include <ipc/ipc_port.h>
-
-#include <vm/vm_kern.h>
-#include <vm/vm_map.h>
-#include <vm/vm_page.h>
-#include <vm/vm_protos.h>
-
-#include <mach/mach_vm.h>
-#include <mach/shared_memory_server.h>
-#include <vm/vm_shared_memory_server.h>
-
-int shared_region_trace_level = SHARED_REGION_TRACE_ERROR;
-
-#if DEBUG
-int lsf_debug = 0;
-int lsf_alloc_debug = 0;
-#define LSF_DEBUG(args)				\
-	MACRO_BEGIN				\
-	if (lsf_debug) {			\
-		kprintf args;			\
-	}					\
-	MACRO_END
-#define LSF_ALLOC_DEBUG(args)			\
-	MACRO_BEGIN				\
-	if (lsf_alloc_debug) {			\
-		kprintf args;			\
-	}					\
-	MACRO_END
-#else /* DEBUG */
-#define LSF_DEBUG(args)
-#define LSF_ALLOC_DEBUG(args)
-#endif /* DEBUG */
-
-/* forward declarations */
-static kern_return_t
-shared_region_object_create(
-	vm_size_t		size,
-	ipc_port_t		*object_handle);
-
-static kern_return_t
-shared_region_mapping_dealloc_lock(
-	shared_region_mapping_t	shared_region,
-	int need_sfh_lock,
-	int need_drl_lock);
-
-
-static kern_return_t           
-shared_file_init(               
-        ipc_port_t      *text_region_handle,
-        vm_size_t       text_region_size,
-        ipc_port_t      *data_region_handle,
-        vm_size_t       data_region_size, 
-        vm_offset_t     *file_mapping_array);
-
-static kern_return_t
-shared_file_header_init(
-	shared_file_info_t	*shared_file_header);
-
-static load_struct_t  *
-lsf_hash_lookup(   
-        queue_head_t    		*hash_table,
-        void    			*file_object,  
-        vm_offset_t                     recognizableOffset,
-        int     			size,
-	boolean_t			regular,
-	boolean_t			alternate,
-	shared_region_task_mappings_t	sm_info);
-
-static load_struct_t *
-lsf_hash_delete(
-	load_struct_t			*target_entry, /* optional */
-        void            		*file_object,
-	vm_offset_t			base_offset,
-	shared_region_task_mappings_t	sm_info);
-
-static void    
-lsf_hash_insert(
-        load_struct_t   *entry,
-	shared_region_task_mappings_t	sm_info);
-
-static kern_return_t
-lsf_slide(
-	unsigned int			map_cnt,
-	struct shared_file_mapping_np	*mappings,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		*base_offset_p);
-
-static kern_return_t
-lsf_map(
-	struct shared_file_mapping_np	*mappings,
-	int				map_cnt,
-	void				*file_control,
-	memory_object_size_t		file_size,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		base_offset,
-	mach_vm_offset_t		*slide_p);
-
-static void
-lsf_unload(
-        void     			*file_object,
-	vm_offset_t			base_offset,
-	shared_region_task_mappings_t	sm_info);
-
-static void
-lsf_deallocate(
-	load_struct_t			*target_entry,	/* optional */
-        void     			*file_object,
-	vm_offset_t			base_offset,
-	shared_region_task_mappings_t	sm_info,
-	boolean_t			unload);
-
-
-#define load_file_hash(file_object, size) \
-		((((natural_t)file_object) & 0xffffff) % size)
-
-/* Implementation */
-vm_offset_t		shared_file_mapping_array = 0;
-
-shared_region_mapping_t default_environment_shared_regions = NULL;
-static decl_mutex_data(,default_regions_list_lock_data)
-
-#define default_regions_list_lock()		\
-		mutex_lock(&default_regions_list_lock_data)
-#define default_regions_list_lock_try()	\
-		mutex_try(&default_regions_list_lock_data)
-#define default_regions_list_unlock()	\
-		mutex_unlock(&default_regions_list_lock_data)
-
-
-ipc_port_t		sfma_handle = NULL;
-zone_t          	lsf_zone;
-
-int		shared_file_available_hash_ele;
-
-/* com region support */
-ipc_port_t		com_region_handle32 = NULL;
-ipc_port_t		com_region_handle64 = NULL;
-vm_map_t		com_region_map32 = NULL;
-vm_map_t		com_region_map64 = NULL;
-vm_size_t		com_region_size32 = _COMM_PAGE32_AREA_LENGTH;
-vm_size_t		com_region_size64 = _COMM_PAGE64_AREA_LENGTH;
-shared_region_mapping_t	com_mapping_resource = NULL;
-
-
-#if DEBUG
-int shared_region_debug = 0;
-#endif /* DEBUG */
-
-
-kern_return_t
-vm_get_shared_region(
-	task_t	task,
-	shared_region_mapping_t	*shared_region)
-{
-	*shared_region = (shared_region_mapping_t) task->system_shared_region;
-	if (*shared_region) {
-		assert((*shared_region)->ref_count > 0);
-	}
-	SHARED_REGION_DEBUG(("vm_get_shared_region(task=%p) -> %p\n",
-			     task, *shared_region));
-	return KERN_SUCCESS;
-}
-
-kern_return_t
-vm_set_shared_region(
-	task_t	task,
-	shared_region_mapping_t	shared_region)
-{
-	shared_region_mapping_t old_region;
-
-	SHARED_REGION_DEBUG(("vm_set_shared_region(task=%p, "
-			     "shared_region=%p[%x,%x,%x])\n",
-			     task, shared_region,
-			     shared_region ? shared_region->fs_base : 0,
-			     shared_region ? shared_region->system : 0,
-			     shared_region ? shared_region->flags : 0));
-	if (shared_region) {
-		assert(shared_region->ref_count > 0);
-	}
-
-	old_region = task->system_shared_region;
-	SHARED_REGION_TRACE(
-		SHARED_REGION_TRACE_INFO,
-		("shared_region: %p set_region(task=%p)"
-		 "old=%p[%x,%x,%x], new=%p[%x,%x,%x]\n",
-		 current_thread(), task,
-		 old_region,
-		 old_region ? old_region->fs_base : 0,
-		 old_region ? old_region->system : 0,
-		 old_region ? old_region->flags : 0,
-		 shared_region,
-		 shared_region ? shared_region->fs_base : 0,
-		 shared_region ? shared_region->system : 0,
-		 shared_region ? shared_region->flags : 0));
-
-	task->system_shared_region = shared_region;
-	return KERN_SUCCESS;
-}
-
-/*
- * shared_region_object_chain_detach:
- *
- * Mark the shared region as being detached or standalone.  This means
- * that we won't keep track of which file is mapped and how, for this shared
- * region.  And we don't have a "shadow" shared region.
- * This is used when we clone a private shared region and we intend to remove
- * some mappings from it.  It won't need to maintain mappings info because it's
- * now private.  It can't have a "shadow" shared region because we don't want
- * to see the shadow of the mappings we're about to remove.
- */
-void
-shared_region_object_chain_detached(
-	shared_region_mapping_t		target_region)
-{
-	shared_region_mapping_lock(target_region);
-	target_region->flags |= SHARED_REGION_STANDALONE;
-	shared_region_mapping_unlock(target_region);
-}
-
-/*
- * shared_region_object_chain_attach:
- *
- * Link "target_region" to "object_chain_region".  "object_chain_region"
- * is treated as a shadow of "target_region" for the purpose of looking up
- * mappings.  Since the "target_region" preserves all the mappings of the
- * older "object_chain_region", we won't duplicate all the mappings info and
- * we'll just lookup the next region in the "object_chain" if we can't find
- * what we're looking for in the "target_region".  See lsf_hash_lookup().
- */
-kern_return_t
-shared_region_object_chain_attach(
-	shared_region_mapping_t		target_region,
-	shared_region_mapping_t		object_chain_region)
-{
-	shared_region_object_chain_t	object_ele;
-	
-	SHARED_REGION_DEBUG(("shared_region_object_chain_attach("
-			     "target_region=%p, object_chain_region=%p\n",
-			     target_region, object_chain_region));
-	assert(target_region->ref_count > 0);
-	assert(object_chain_region->ref_count > 0);
-	if(target_region->object_chain)
-		return KERN_FAILURE;
-	object_ele = (shared_region_object_chain_t)
-			kalloc(sizeof (struct shared_region_object_chain));
-	shared_region_mapping_lock(object_chain_region);
-	target_region->object_chain = object_ele;
-	object_ele->object_chain_region = object_chain_region;
-	object_ele->next = object_chain_region->object_chain;
-	object_ele->depth = object_chain_region->depth;
-	object_chain_region->depth++;
-	target_region->alternate_next = object_chain_region->alternate_next;
-	shared_region_mapping_unlock(object_chain_region);
-	return KERN_SUCCESS;
-}
-
-/* LP64todo - need 64-bit safe version */
-kern_return_t
-shared_region_mapping_create(
-	ipc_port_t		text_region,
-	vm_size_t		text_size,
-	ipc_port_t		data_region,
-	vm_size_t		data_size,
-	vm_offset_t		region_mappings,
-	vm_offset_t		client_base,
-	shared_region_mapping_t	*shared_region,
-	vm_offset_t		alt_base,
-	vm_offset_t		alt_next,
-	int			fs_base,
-	int			system)
-{
-	SHARED_REGION_DEBUG(("shared_region_mapping_create()\n"));
-	*shared_region = (shared_region_mapping_t) 
-			kalloc(sizeof (struct shared_region_mapping));
-	if(*shared_region == NULL) {
-		SHARED_REGION_DEBUG(("shared_region_mapping_create: "
-				     "failure\n"));
-		return KERN_FAILURE;
-	}
-	shared_region_mapping_lock_init((*shared_region));
-	(*shared_region)->text_region = text_region;
-	(*shared_region)->text_size = text_size;
-	(*shared_region)->fs_base = fs_base;
-	(*shared_region)->system = system;
-	(*shared_region)->data_region = data_region;
-	(*shared_region)->data_size = data_size;
-	(*shared_region)->region_mappings = region_mappings;
-	(*shared_region)->client_base = client_base;
-	(*shared_region)->ref_count = 1;
-	(*shared_region)->next = NULL;
-	(*shared_region)->object_chain = NULL;
-	(*shared_region)->self = *shared_region;
-	(*shared_region)->flags = 0;
-	(*shared_region)->depth = 0;
-	(*shared_region)->default_env_list = NULL;
-	(*shared_region)->alternate_base = alt_base;
-	(*shared_region)->alternate_next = alt_next;
-	SHARED_REGION_DEBUG(("shared_region_mapping_create -> %p\n",
-			     *shared_region));
-	return KERN_SUCCESS;
-}
-
-/* LP64todo - need 64-bit safe version */
-kern_return_t
-shared_region_mapping_info(
-	shared_region_mapping_t	shared_region,
-	ipc_port_t		*text_region,
-	vm_size_t		*text_size,
-	ipc_port_t		*data_region,
-	vm_size_t		*data_size,
-	vm_offset_t		*region_mappings,
-	vm_offset_t		*client_base,
-	vm_offset_t		*alt_base,
-	vm_offset_t		*alt_next,
-	unsigned int		*fs_base,
-	unsigned int		*system,
-	int			*flags,
-	shared_region_mapping_t	*next)
-{
-	shared_region_mapping_lock(shared_region);
-
-	SHARED_REGION_DEBUG(("shared_region_mapping_info(shared_region=%p)\n",
-			     shared_region));
-	assert(shared_region->ref_count > 0);
-	*text_region = shared_region->text_region;
-	*text_size = shared_region->text_size;
-	*data_region = shared_region->data_region;
-	*data_size = shared_region->data_size;
-	*region_mappings = shared_region->region_mappings;
-	*client_base = shared_region->client_base;
-	*alt_base = shared_region->alternate_base;
-	*alt_next = shared_region->alternate_next;
-	*flags = shared_region->flags;
-	*fs_base = shared_region->fs_base;
-	*system = shared_region->system;
-	*next = shared_region->next;
-
-	shared_region_mapping_unlock(shared_region);
-	
-	return KERN_SUCCESS;
-}
-
-kern_return_t
-shared_region_mapping_ref(
-	shared_region_mapping_t	shared_region)
-{
-	SHARED_REGION_DEBUG(("shared_region_mapping_ref(shared_region=%p): "
-			     "ref_count=%d + 1\n",
-			     shared_region,
-			     shared_region ? shared_region->ref_count : 0));
-	if(shared_region == NULL)
-		return KERN_SUCCESS;
-	assert(shared_region->ref_count > 0);
-	hw_atomic_add(&shared_region->ref_count, 1);
-	return KERN_SUCCESS;
-}
-
-static kern_return_t
-shared_region_mapping_dealloc_lock(
-	shared_region_mapping_t	shared_region,
-	int need_sfh_lock,
-	int need_drl_lock)
-{
-	struct shared_region_task_mappings sm_info;
-	shared_region_mapping_t next = NULL;
-	unsigned int ref_count;
-
-	SHARED_REGION_DEBUG(("shared_region_mapping_dealloc_lock"
-			     "(shared_region=%p,%d,%d) ref_count=%d\n",
-			     shared_region, need_sfh_lock, need_drl_lock,
-			     shared_region ? shared_region->ref_count : 0));
-	while (shared_region) {
-		SHARED_REGION_DEBUG(("shared_region_mapping_dealloc_lock(%p): "
-				     "ref_count=%d\n",
-				     shared_region, shared_region->ref_count));
-		assert(shared_region->ref_count > 0);
-		if ((ref_count = 
-			  hw_atomic_sub(&shared_region->ref_count, 1)) == 0) {
-			shared_region_mapping_lock(shared_region);
-
-			sm_info.text_region = shared_region->text_region;
-			sm_info.text_size = shared_region->text_size;
-			sm_info.data_region = shared_region->data_region;
-			sm_info.data_size = shared_region->data_size;
-			sm_info.region_mappings = shared_region->region_mappings;
-			sm_info.client_base = shared_region->client_base;
-			sm_info.alternate_base = shared_region->alternate_base;
-			sm_info.alternate_next = shared_region->alternate_next;
-			sm_info.flags = shared_region->flags;
-			sm_info.self = (vm_offset_t)shared_region;
-
-			if(shared_region->region_mappings) {
-				lsf_remove_regions_mappings_lock(shared_region, &sm_info, need_sfh_lock);
-			}
-			if(((vm_named_entry_t)
-				(shared_region->text_region->ip_kobject))
-                                                        ->backing.map->pmap) {
-			    pmap_remove(((vm_named_entry_t)
-				(shared_region->text_region->ip_kobject))
-							->backing.map->pmap, 
-				sm_info.client_base, 
-				sm_info.client_base + sm_info.text_size);
-			}
-			ipc_port_release_send(shared_region->text_region);
-			if(shared_region->data_region)
-				ipc_port_release_send(shared_region->data_region);
-			if (shared_region->object_chain) {
-				next = shared_region->object_chain->object_chain_region;
-				kfree(shared_region->object_chain,
-				      sizeof (struct shared_region_object_chain));
-			} else {
-				next = NULL;
-			}
-			shared_region_mapping_unlock(shared_region);
-			SHARED_REGION_DEBUG(
-				("shared_region_mapping_dealloc_lock(%p): "
-				 "freeing\n",
-				 shared_region));
-			bzero((void *)shared_region,
-			      sizeof (*shared_region)); /* FBDP debug */
-			kfree(shared_region,
-				sizeof (struct shared_region_mapping));
-			shared_region = next;
-		} else {
-			/* Stale indicates that a system region is no */
-			/* longer in the default environment list.    */
-			if((ref_count == 1) && 
-			  (shared_region->flags & SHARED_REGION_SYSTEM)
-			  && !(shared_region->flags & SHARED_REGION_STALE)) {
-				SHARED_REGION_DEBUG(
-					("shared_region_mapping_dealloc_lock"
-					 "(%p): removing stale\n",
-					 shared_region));
-				remove_default_shared_region_lock(shared_region,need_sfh_lock, need_drl_lock);
-			}
-			break;
-		}
-	}
-	SHARED_REGION_DEBUG(("shared_region_mapping_dealloc_lock(%p): done\n",
-			     shared_region));
-	return KERN_SUCCESS;
-}
-
-/*
- * Stub function; always indicates that the lock needs to be taken in the
- * call to lsf_remove_regions_mappings_lock().
- */
-kern_return_t
-shared_region_mapping_dealloc(
-	shared_region_mapping_t	shared_region)
-{
-	SHARED_REGION_DEBUG(("shared_region_mapping_dealloc"
-			     "(shared_region=%p)\n",
-			     shared_region));
-	if (shared_region) {
-		assert(shared_region->ref_count > 0);
-	}
-	return shared_region_mapping_dealloc_lock(shared_region, 1, 1);
-}
-
-static 
-kern_return_t
-shared_region_object_create(
-	vm_size_t		size,
-	ipc_port_t		*object_handle)
-{
-	vm_named_entry_t	user_entry;
-	ipc_port_t		user_handle;
-
-	ipc_port_t	previous;
-	vm_map_t	new_map;
-	
-	user_entry = (vm_named_entry_t) 
-			kalloc(sizeof (struct vm_named_entry));
-	if(user_entry == NULL) {
-		return KERN_FAILURE;
-	}
-	named_entry_lock_init(user_entry);
-	user_handle = ipc_port_alloc_kernel();
-
-
-	ip_lock(user_handle);
-
-	/* make a sonce right */
-	user_handle->ip_sorights++;
-	ip_reference(user_handle);
-
-	user_handle->ip_destination = IP_NULL;
-	user_handle->ip_receiver_name = MACH_PORT_NULL;
-	user_handle->ip_receiver = ipc_space_kernel;
-
-	/* make a send right */
-        user_handle->ip_mscount++;
-        user_handle->ip_srights++;
-        ip_reference(user_handle);
-
-	ipc_port_nsrequest(user_handle, 1, user_handle, &previous);
-	/* nsrequest unlocks user_handle */
-
-	/* Create a named object based on a submap of specified size */
-
-	new_map = vm_map_create(pmap_create(0, FALSE), 0, size, TRUE);
-	user_entry->backing.map = new_map;
-	user_entry->internal = TRUE;
-	user_entry->is_sub_map = TRUE;
-	user_entry->is_pager = FALSE;
-	user_entry->offset = 0;
-	user_entry->protection = VM_PROT_ALL;
-	user_entry->size = size;
-	user_entry->ref_count = 1;
-
-	ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
-							IKOT_NAMED_ENTRY);
-	*object_handle = user_handle;
-	return KERN_SUCCESS;
-}
-
-/* called for the non-default, private branch shared region support */
-/* system default fields for fs_base and system supported are not   */
-/* relevant as the system default flag is not set */
-kern_return_t
-shared_file_create_system_region(
-	shared_region_mapping_t	*shared_region,
-	int			fs_base,
-	int			system)
-{
-	ipc_port_t		text_handle;
-	ipc_port_t		data_handle;
-	long			text_size;
-	long			data_size;
-	vm_offset_t		mapping_array;
-	kern_return_t		kret;
-
-	SHARED_REGION_DEBUG(("shared_file_create_system_region()\n"));
-
-	text_size = 0x10000000;
-	data_size = 0x10000000;
-
-	kret = shared_file_init(&text_handle,
-			text_size, &data_handle, data_size, &mapping_array);
-	if(kret) {
-		SHARED_REGION_DEBUG(("shared_file_create_system_region: "
-				     "shared_file_init failed kret=0x%x\n",
-				     kret));
-		return kret;
-	}
-	kret = shared_region_mapping_create(text_handle, text_size,
-					    data_handle, data_size,
-					    mapping_array,
-					    GLOBAL_SHARED_TEXT_SEGMENT,
-					    shared_region, 
-					    SHARED_ALTERNATE_LOAD_BASE,
-					    SHARED_ALTERNATE_LOAD_BASE,
-					    fs_base,
-					    system);
-	if(kret) {
-		SHARED_REGION_DEBUG(("shared_file_create_system_region: "
-				     "shared_region_mapping_create failed "
-				     "kret=0x%x\n",
-				     kret));
-		return kret;
-	}
-	(*shared_region)->flags = 0;
-	if(com_mapping_resource) {
-        	shared_region_mapping_ref(com_mapping_resource);
-        	(*shared_region)->next = com_mapping_resource;
-	}
-
-	SHARED_REGION_DEBUG(("shared_file_create_system_region() "
-			     "-> shared_region=%p\n",
-			     *shared_region));
-	return KERN_SUCCESS;
-}
-
-/*
- * load a new default for a specified environment into the default share
- * regions list.  If a previous default exists for the envrionment specification
- * it is returned along with its reference.  It is expected that the new
- * sytem region structure passes a reference.
- */
-
-shared_region_mapping_t
-update_default_shared_region(
-		shared_region_mapping_t new_system_region)
-{
-	shared_region_mapping_t old_system_region;
-	unsigned int fs_base;
-	unsigned int system;
-
-	SHARED_REGION_DEBUG(("update_default_shared_region(new=%p)\n",
-			     new_system_region));
-	assert(new_system_region->ref_count > 0);
-	fs_base = new_system_region->fs_base;
-	system = new_system_region->system;
-	new_system_region->flags |= SHARED_REGION_SYSTEM;
-	default_regions_list_lock();
-	old_system_region = default_environment_shared_regions;
-
-	if((old_system_region != NULL) && 
-		(old_system_region->fs_base == fs_base) &&
-			(old_system_region->system == system)) {
-		new_system_region->default_env_list =
-			old_system_region->default_env_list;
-		old_system_region->default_env_list = NULL;
-		default_environment_shared_regions = new_system_region;
-		old_system_region->flags |= SHARED_REGION_STALE;
-		default_regions_list_unlock();
-		SHARED_REGION_DEBUG(("update_default_shared_region(%p): "
-				     "old=%p stale 1\n",
-				     new_system_region, old_system_region));
-		assert(old_system_region->ref_count > 0);
-		return old_system_region;
-	}
-	if (old_system_region) {
-	   while(old_system_region->default_env_list != NULL) {
-		if((old_system_region->default_env_list->fs_base == fs_base) &&
-		      (old_system_region->default_env_list->system == system)) {
-			shared_region_mapping_t tmp_system_region;
-
-			tmp_system_region =
-				old_system_region->default_env_list;
-			new_system_region->default_env_list =
-			   		tmp_system_region->default_env_list;
-			tmp_system_region->default_env_list = NULL;
-			old_system_region->default_env_list = 
-					new_system_region;
-			old_system_region = tmp_system_region;
-			old_system_region->flags |= SHARED_REGION_STALE;
-			default_regions_list_unlock();
-			SHARED_REGION_DEBUG(("update_default_shared_region(%p)"
-					     ": old=%p stale 2\n",
-					     new_system_region,
-					     old_system_region));
-			assert(old_system_region->ref_count > 0);
-			return old_system_region;
-		}
-		old_system_region = old_system_region->default_env_list;
-	   }
-	}
-	/* If we get here, we are at the end of the system list and we */
-	/* did not find a pre-existing entry */
-	if(old_system_region) {
-		SHARED_REGION_DEBUG(("update_default_system_region(%p): "
-				     "adding after old=%p\n",
-				     new_system_region, old_system_region));
-		assert(old_system_region->ref_count > 0);
-		old_system_region->default_env_list = new_system_region;
-	} else {
-		SHARED_REGION_DEBUG(("update_default_system_region(%p): "
-				     "new default\n",
-				     new_system_region));
-		default_environment_shared_regions = new_system_region;
-	}
-	assert(new_system_region->ref_count > 0);
-	default_regions_list_unlock();
-	return NULL;
-}
-
-/* 
- * lookup a system_shared_region for the environment specified.  If one is
- * found, it is returned along with a reference against the structure
- */
-
-shared_region_mapping_t
-lookup_default_shared_region(
-		unsigned int fs_base,
-		unsigned int system)
-{
-	shared_region_mapping_t	system_region;
-	default_regions_list_lock();
-	system_region = default_environment_shared_regions;
-
-	SHARED_REGION_DEBUG(("lookup_default_shared_region"
-			     "(base=0x%x, system=0x%x)\n",
-			     fs_base, system));
-	while(system_region != NULL) {
-		SHARED_REGION_DEBUG(("lookup_default_shared_region(0x%x, 0x%x)"
-				     ": system_region=%p base=0x%x system=0x%x"
-				     " ref_count=%d\n",
-				     fs_base, system, system_region,
-				     system_region->fs_base,
-				     system_region->system,
-				     system_region->ref_count));
-		assert(system_region->ref_count > 0);
-		if((system_region->fs_base == fs_base) &&
-		      	(system_region->system == system)) {
-			break;
-		}
-		system_region = system_region->default_env_list;
-	}
-	if(system_region)
-		shared_region_mapping_ref(system_region);
-	default_regions_list_unlock();
-	SHARED_REGION_DEBUG(("lookup_default_system_region(0x%x,0x%x) -> %p\n",
-			     system_region));
-	return system_region;
-}
-
-/*
- * remove a system_region default if it appears in the default regions list. 
- * Drop a reference on removal.
- */
-
-__private_extern__ void
-remove_default_shared_region_lock(
-		shared_region_mapping_t system_region,
-		int need_sfh_lock,
-		int need_drl_lock)
-{
-	shared_region_mapping_t old_system_region;
-
-	SHARED_REGION_DEBUG(("remove_default_shared_region_lock"
-			     "(system_region=%p, %d, %d)\n",
-			     system_region, need_sfh_lock, need_drl_lock));
-	if (need_drl_lock) {
-		default_regions_list_lock();
-	}
-	old_system_region = default_environment_shared_regions;
-
-	if(old_system_region == NULL) {
-		SHARED_REGION_DEBUG(("remove_default_shared_region_lock(%p)"
-				     "-> default_env=NULL\n",
-				     system_region));
-		if (need_drl_lock) {
-			default_regions_list_unlock();
-		}
-		return;
-	}
-
-	SHARED_REGION_DEBUG(("remove_default_shared_region_lock(%p): "
-			     "default_env=%p\n",
-			     system_region, old_system_region));
-	assert(old_system_region->ref_count > 0);
-	if (old_system_region == system_region) {
-		default_environment_shared_regions 
-			= old_system_region->default_env_list;
-		old_system_region->default_env_list = NULL;
-		old_system_region->flags |= SHARED_REGION_STALE;
-		SHARED_REGION_DEBUG(("remove_default_shared_region_lock(%p): "
-				     "old=%p ref_count=%d STALE\n",
-				     system_region, old_system_region,
-				     old_system_region->ref_count));
-               	shared_region_mapping_dealloc_lock(old_system_region,
-						   need_sfh_lock,
-						   0);
-		if (need_drl_lock) {
-			default_regions_list_unlock();
-		}
-		return;
-	}
-
-	while(old_system_region->default_env_list != NULL) {
-		SHARED_REGION_DEBUG(("remove_default_shared_region_lock(%p): "
-				     "old=%p->default_env=%p\n",
-				     system_region, old_system_region,
-				     old_system_region->default_env_list));
-		assert(old_system_region->default_env_list->ref_count > 0);
-		if(old_system_region->default_env_list == system_region) {
-			shared_region_mapping_t dead_region;
-			dead_region = old_system_region->default_env_list;
-			old_system_region->default_env_list = 
-				dead_region->default_env_list;
-			dead_region->default_env_list = NULL;
-			dead_region->flags |= SHARED_REGION_STALE;
-			SHARED_REGION_DEBUG(
-				("remove_default_shared_region_lock(%p): "
-				 "dead=%p ref_count=%d stale\n",
-				 system_region, dead_region,
-				 dead_region->ref_count));
-               		shared_region_mapping_dealloc_lock(dead_region,
-							   need_sfh_lock,
-							   0);
-			if (need_drl_lock) {
-				default_regions_list_unlock();
-			}
-			return;
-		}
-		old_system_region = old_system_region->default_env_list;
-	}
-	if (need_drl_lock) {
-		default_regions_list_unlock();
-	}
-}
-
-/*
- * Symbol compatability; we believe shared_region_mapping_dealloc_lock() is
- * the only caller.  Remove this stub function and the corresponding symbol
- * export for Merlot.
- */
-void
-remove_default_shared_region(
-		shared_region_mapping_t system_region)
-{
-	SHARED_REGION_DEBUG(("remove_default_shared_region(%p)\n",
-			     system_region));
-	if (system_region) {
-		assert(system_region->ref_count > 0);
-	}
-	remove_default_shared_region_lock(system_region, 1, 1);
-}
-
-void
-remove_all_shared_regions(void)
-{
-	shared_region_mapping_t system_region;
-	shared_region_mapping_t next_system_region;
-
-	SHARED_REGION_DEBUG(("***** REMOVE_ALL_SHARED_REGIONS()\n"));
-	LSF_ALLOC_DEBUG(("***** REMOVE_ALL_SHARED_REGIONS()\n"));
-	LSF_DEBUG(("***** REMOVE_ALL_SHARED_REGIONS()\n"));
-	default_regions_list_lock();
-	system_region = default_environment_shared_regions;
-
-	if(system_region == NULL) {
-		default_regions_list_unlock();
-		return;
-	}
-
-	while(system_region != NULL) {
-		next_system_region = system_region->default_env_list;
-		system_region->default_env_list = NULL;
-		system_region->flags |= SHARED_REGION_STALE;
-		SHARED_REGION_DEBUG(("remove_all_shared_regions(): "
-				     "%p ref_count=%d stale\n",
-				     system_region, system_region->ref_count));
-		assert(system_region->ref_count > 0);
-               	shared_region_mapping_dealloc_lock(system_region, 1, 0);
-		system_region = next_system_region;
-	}
-	default_environment_shared_regions = NULL;
-	default_regions_list_unlock();
-	SHARED_REGION_DEBUG(("***** remove_all_shared_regions() done\n"));
-	LSF_ALLOC_DEBUG(("***** remove_all_shared_regions() done\n"));
-	LSF_DEBUG(("***** remove_all_shared_regions() done\n"));
-}
-		
-/* shared_com_boot_time_init initializes the common page shared data and */
-/* text region.  This region is semi independent of the split libs       */
-/* and so its policies have to be handled differently by the code that   */
-/* manipulates the mapping of shared region environments.  However,      */
-/* the shared region delivery system supports both */
-void shared_com_boot_time_init(void);	/* forward */
-void
-shared_com_boot_time_init(void)
-{
-	kern_return_t		 kret;
-	vm_named_entry_t	named_entry;
-
-	SHARED_REGION_DEBUG(("shared_com_boot_time_init()\n"));
-	if(com_region_handle32) {
-		panic("shared_com_boot_time_init: "
-			"com_region_handle32 already set\n");
-	}
-	if(com_region_handle64) {
-		panic("shared_com_boot_time_init: "
-			"com_region_handle64 already set\n");
-	}
-
-	/* create com page regions, 1 each for 32 and 64-bit code  */
-	if((kret = shared_region_object_create(
-			com_region_size32, 
-			&com_region_handle32))) {
-		panic("shared_com_boot_time_init: "
-				"unable to create 32-bit comm page\n");
-		return;
-	}
-	if((kret = shared_region_object_create(
-			com_region_size64, 
-			&com_region_handle64))) {
-		panic("shared_com_boot_time_init: "
-				"unable to create 64-bit comm page\n");
-		return;
-	}
-	
-	/* now set export the underlying region/map */
-	named_entry = (vm_named_entry_t)com_region_handle32->ip_kobject;
-	com_region_map32 = named_entry->backing.map;
-	named_entry = (vm_named_entry_t)com_region_handle64->ip_kobject;
-	com_region_map64 = named_entry->backing.map;
-	
-	/* wrap the com region in its own shared file mapping structure */
-	/* 64-bit todo: call "shared_region_mapping_create" on com_region_handle64 */
-	kret = shared_region_mapping_create(com_region_handle32,
-					    com_region_size32,
-					    NULL, 0, 0,
-					    _COMM_PAGE_BASE_ADDRESS,
-					    &com_mapping_resource,
-					    0, 0,
-					    ENV_DEFAULT_ROOT, cpu_type());
-	if (kret) {
-	  panic("shared_region_mapping_create failed for commpage");
-	}
-}
-
-void
-shared_file_boot_time_init(
-		unsigned int fs_base, 
-		unsigned int system)
-{
-	mach_port_t		text_region_handle;
-	mach_port_t		data_region_handle;
-	long			text_region_size;
-	long			data_region_size;
-	shared_region_mapping_t	new_system_region;
-	shared_region_mapping_t	old_default_env;
-
-	SHARED_REGION_DEBUG(("shared_file_boot_time_init"
-			     "(base=0x%x,system=0x%x)\n",
-			     fs_base, system));
-	text_region_size = 0x10000000;
-	data_region_size = 0x10000000;
-	shared_file_init(&text_region_handle,
-			 text_region_size,
-			 &data_region_handle,
-			 data_region_size,
-			 &shared_file_mapping_array);
-	
-	shared_region_mapping_create(text_region_handle,
-				     text_region_size,
-				     data_region_handle,
-				     data_region_size,
-				     shared_file_mapping_array,
-				     GLOBAL_SHARED_TEXT_SEGMENT,
-				     &new_system_region,
-				     SHARED_ALTERNATE_LOAD_BASE,
-				     SHARED_ALTERNATE_LOAD_BASE,
-				     fs_base, system);
-
-	new_system_region->flags = SHARED_REGION_SYSTEM;
-
-	/* grab an extra reference for the caller */
-	/* remember to grab before call to update */
-	shared_region_mapping_ref(new_system_region);
-	old_default_env = update_default_shared_region(new_system_region);
-	/* hold an extra reference because these are the system */
-	/* shared regions. */
-	if(old_default_env)
-        	shared_region_mapping_dealloc(old_default_env);
-	if(com_mapping_resource == NULL) {
-		shared_com_boot_time_init();
-	}
-	shared_region_mapping_ref(com_mapping_resource);
-	new_system_region->next = com_mapping_resource;
-	vm_set_shared_region(current_task(), new_system_region);
-	SHARED_REGION_DEBUG(("shared_file_boot_time_init(0x%x,0x%x) done\n",
-			     fs_base, system));
-}
-
-
-/* called at boot time, allocates two regions, each 256 megs in size */
-/* these regions are later mapped into task spaces, allowing them to */
-/* share the contents of the regions.  shared_file_init is part of   */
-/* a shared_memory_server which not only allocates the backing maps  */
-/* but also coordinates requests for space.  */
-
-
-static kern_return_t
-shared_file_init(
-	ipc_port_t	*text_region_handle,
-	vm_size_t 	text_region_size, 
-	ipc_port_t	*data_region_handle,
-	vm_size_t 	data_region_size,
-	vm_offset_t	*file_mapping_array)
-{
-	shared_file_info_t	*sf_head;
-	vm_size_t		data_table_size;
-	int			hash_size;
-	kern_return_t		kret;
-
-	vm_object_t		buf_object;
-	vm_map_entry_t		entry;
-	vm_size_t		alloced;
-	vm_offset_t		b;
-	vm_page_t		p;
-
-	SHARED_REGION_DEBUG(("shared_file_init()\n"));
-	/* create text and data maps/regions */
-	kret = shared_region_object_create(
-				       text_region_size, 
-				       text_region_handle);
-	if (kret) {
-		return kret;
-	}
-	kret = shared_region_object_create(
-				       data_region_size, 
-				       data_region_handle);
-	if (kret) {
-		ipc_port_release_send(*text_region_handle);
-		return kret;
-	}
-
-	data_table_size = data_region_size >> 9;
-	hash_size = data_region_size >> 14;
-
-	if(shared_file_mapping_array == 0) {
-		vm_map_address_t map_addr;
-		buf_object = vm_object_allocate(data_table_size);
-
-		if(vm_map_find_space(kernel_map, &map_addr,
-				     data_table_size, 0, 0, &entry)
-		   != KERN_SUCCESS) {
-			panic("shared_file_init: no space");
-		}
-		shared_file_mapping_array = CAST_DOWN(vm_offset_t, map_addr);
-		*file_mapping_array = shared_file_mapping_array;
-		vm_map_unlock(kernel_map);
-		entry->object.vm_object = buf_object;
-		entry->offset = 0;
-
-		for (b = *file_mapping_array, alloced = 0; 
-			   alloced < (hash_size +
-				round_page(sizeof(struct sf_mapping)));
-			   alloced += PAGE_SIZE,  b += PAGE_SIZE) {
-			vm_object_lock(buf_object);
-			p = vm_page_alloc(buf_object, alloced);
-			if (p == VM_PAGE_NULL) {
-				panic("shared_file_init: no space");
-			} 	
-			p->busy = FALSE;
-			vm_object_unlock(buf_object);
-			pmap_enter(kernel_pmap, b, p->phys_page,
-				VM_PROT_READ | VM_PROT_WRITE, 
-				((unsigned int)(p->object->wimg_bits)) 
-							& VM_WIMG_MASK,
-				TRUE);
-		}
-
-
-		/* initialize loaded file array */
-		sf_head = (shared_file_info_t *)*file_mapping_array;
-		sf_head->hash = (queue_head_t *) 
-				(((int)*file_mapping_array) + 
-					sizeof(struct shared_file_info));
-		sf_head->hash_size = hash_size/sizeof(queue_head_t);
-		mutex_init(&(sf_head->lock), 0);
-		sf_head->hash_init = FALSE;
-
-
-		mach_make_memory_entry(kernel_map, &data_table_size, 
-			*file_mapping_array, VM_PROT_READ, &sfma_handle,
-			NULL);
-
-		if (vm_map_wire(kernel_map, 
-			vm_map_trunc_page(*file_mapping_array),
-			vm_map_round_page(*file_mapping_array + 
-					  hash_size + 
-					  round_page(sizeof(struct sf_mapping))),
-		   	VM_PROT_DEFAULT, FALSE) != KERN_SUCCESS) {
-			panic("shared_file_init: No memory for data table");
-		}
-
-		lsf_zone = zinit(sizeof(struct load_file_ele), 
-			data_table_size - 
-			   (hash_size + round_page_32(sizeof(struct sf_mapping))),
-			0, "load_file_server"); 
-
-		zone_change(lsf_zone, Z_EXHAUST, TRUE);
-		zone_change(lsf_zone, Z_COLLECT, FALSE);
-		zone_change(lsf_zone, Z_EXPAND, FALSE);
-		zone_change(lsf_zone, Z_FOREIGN, TRUE);
-
-		/* initialize the global default environment lock */
-		mutex_init(&default_regions_list_lock_data, 0);
-
-	} else {
-		*file_mapping_array = shared_file_mapping_array;
-	}
-
-	SHARED_REGION_DEBUG(("shared_file_init() done\n"));
-	return KERN_SUCCESS;
-}
-
-static kern_return_t
-shared_file_header_init(
-	shared_file_info_t		*shared_file_header)
-{
-	vm_size_t		hash_table_size;
-	vm_size_t		hash_table_offset;
-	int			i;
-	/* wire hash entry pool only as needed, since we are the only */
-	/* users, we take a few liberties with the population of our  */
-	/* zone. */
-	static int		allocable_hash_pages;
-	static vm_offset_t	hash_cram_address;
-	
-		
-	hash_table_size = shared_file_header->hash_size 
-		* sizeof (struct queue_entry);
-	hash_table_offset = hash_table_size + 
-		round_page(sizeof (struct sf_mapping));
-	for (i = 0; i < shared_file_header->hash_size; i++)
-		queue_init(&shared_file_header->hash[i]);
-
-	allocable_hash_pages = (((hash_table_size << 5) - hash_table_offset)
-				/ PAGE_SIZE);
-	hash_cram_address = ((vm_offset_t) shared_file_header)
-		+ hash_table_offset;
-	shared_file_available_hash_ele = 0;
-
-	shared_file_header->hash_init = TRUE;
-
-	if ((shared_file_available_hash_ele < 20) && (allocable_hash_pages)) {
-		int cram_pages, cram_size;
-
-		cram_pages = allocable_hash_pages > 3 ? 
-					3 : allocable_hash_pages;
-		cram_size = cram_pages * PAGE_SIZE;
-		if (vm_map_wire(kernel_map, hash_cram_address,
-				hash_cram_address + cram_size, 
-				VM_PROT_DEFAULT, FALSE) != KERN_SUCCESS) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: shared_file_header_init: "
-				 "No memory for data table\n"));
-			return KERN_NO_SPACE;
-		}
-		allocable_hash_pages -= cram_pages;
-		zcram(lsf_zone, (void *) hash_cram_address, cram_size);
-		shared_file_available_hash_ele 
-				+= cram_size/sizeof(struct load_file_ele);
-		hash_cram_address += cram_size;
-	}
-
-	return KERN_SUCCESS;
-}
-
-
-extern void shared_region_dump_file_entry(
-	int		trace_level,
-	load_struct_t	*entry);	/* forward */
-
-void shared_region_dump_file_entry(
-	int		trace_level,
-	load_struct_t	*entry)
-{
-	int			i;
-	loaded_mapping_t	*mapping;
-
-	if (trace_level > shared_region_trace_level) {
-		return;
-	}
-	printf("shared region: %p: "
-	       "file_entry %p  base_address=0x%x  file_offset=0x%x  "
-	       "%d mappings\n",
-	       current_thread(), entry,
-	       entry->base_address, entry->file_offset, entry->mapping_cnt);
-	mapping = entry->mappings;
-	for (i = 0; i < entry->mapping_cnt; i++) {
-		printf("shared region: %p:\t#%d: "
-		       "offset=0x%x size=0x%x file_offset=0x%x prot=%d\n",
-		       current_thread(),
-		       i,
-		       mapping->mapping_offset,
-		       mapping->size,
-		       mapping->file_offset,
-		       mapping->protection);
-		mapping = mapping->next;
-	}
-}
-
-extern void shared_region_dump_mappings(
-	int				trace_level,
-	struct shared_file_mapping_np	*mappings,
-	int				map_cnt,
-	mach_vm_offset_t		base_offset);	/* forward */
-
-void shared_region_dump_mappings(
-	int				trace_level,
-	struct shared_file_mapping_np	*mappings,
-	int				map_cnt,
-	mach_vm_offset_t		base_offset)
-{
-	int	i;
-
-	if (trace_level > shared_region_trace_level) {
-		return;
-	}
-
-	printf("shared region: %p: %d mappings  base_offset=0x%llx\n",
-	       current_thread(), map_cnt, (uint64_t) base_offset);
-	for (i = 0; i < map_cnt; i++) {
-		printf("shared region: %p:\t#%d: "
-		       "addr=0x%llx, size=0x%llx, file_offset=0x%llx, "
-		       "prot=(%d,%d)\n",
-		       current_thread(),
-		       i,
-		       (uint64_t) mappings[i].sfm_address,
-		       (uint64_t) mappings[i].sfm_size,
-		       (uint64_t) mappings[i].sfm_file_offset,
-		       mappings[i].sfm_max_prot,
-		       mappings[i].sfm_init_prot);
-	}
-}
-
-extern void shared_region_dump_conflict_info(
-	int		trace_level,
-	vm_map_t	map,
-	vm_map_offset_t	offset,
-	vm_map_size_t	size);	/* forward */
-
-void
-shared_region_dump_conflict_info(
-	int		trace_level,
-	vm_map_t	map,
-	vm_map_offset_t	offset,
-	vm_map_size_t	size)
-{
-	vm_map_entry_t	entry;
-	vm_object_t	object;
-	memory_object_t	mem_object;
-	kern_return_t	kr;
-	char		*filename;
-
-	if (trace_level > shared_region_trace_level) {
-		return;
-	}
-
-	object = VM_OBJECT_NULL;
-
-	vm_map_lock_read(map);
-	if (!vm_map_lookup_entry(map, offset, &entry)) {
-		entry = entry->vme_next;
-	}
-	
-	if (entry != vm_map_to_entry(map)) {
-		if (entry->is_sub_map) {
-			printf("shared region: %p: conflict with submap "
-			       "at 0x%llx size 0x%llx !?\n",
-			       current_thread(),
-			       (uint64_t) offset,
-			       (uint64_t) size);
-			goto done;
-		}
-
-		object = entry->object.vm_object;
-		if (object == VM_OBJECT_NULL) {
-			printf("shared region: %p: conflict with NULL object "
-			       "at 0x%llx size 0x%llx !?\n",
-			       current_thread(),
-			       (uint64_t) offset,
-			       (uint64_t) size);
-			object = VM_OBJECT_NULL;
-			goto done;
-		}
-
-		vm_object_lock(object);
-		while (object->shadow != VM_OBJECT_NULL) {
-			vm_object_t	shadow;
-
-			shadow = object->shadow;
-			vm_object_lock(shadow);
-			vm_object_unlock(object);
-			object = shadow;
-		}
-
-		if (object->internal) {
-			printf("shared region: %p: conflict with anonymous "
-			       "at 0x%llx size 0x%llx\n",
-			       current_thread(),
-			       (uint64_t) offset,
-			       (uint64_t) size);
-			goto done;
-		}
-		if (! object->pager_ready) {
-			printf("shared region: %p: conflict with uninitialized "
-			       "at 0x%llx size 0x%llx\n",
-			       current_thread(),
-			       (uint64_t) offset,
-			       (uint64_t) size);
-			goto done;
-		}
-
-		mem_object = object->pager;
-
-		/*
-		 * XXX FBDP: "!internal" doesn't mean it's a vnode pager...
-		 */
-		kr = vnode_pager_get_object_filename(mem_object,
-						     &filename);
-		if (kr != KERN_SUCCESS) {
-			filename = NULL;
-		}
-		printf("shared region: %p: conflict with '%s' "
-		       "at 0x%llx size 0x%llx\n",
-		       current_thread(),
-		       filename ? filename : "<unknown>",
-		       (uint64_t) offset,
-		       (uint64_t) size);
-	}
-done:
-	if (object != VM_OBJECT_NULL) {
-		vm_object_unlock(object);
-	}
-	vm_map_unlock_read(map);
-}
-
-/*
- * map_shared_file:
- *
- * Attempt to map a split library into the shared region.  Check if the mappings
- * are already in place.
- */
-kern_return_t
-map_shared_file(
-	int 				map_cnt,
-	struct shared_file_mapping_np 	*mappings,
-	memory_object_control_t		file_control,
-	memory_object_size_t		file_size,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		base_offset,
-	mach_vm_offset_t		*slide_p)
-{
-	vm_object_t		file_object;
-	shared_file_info_t	*shared_file_header;
-	load_struct_t		*file_entry;
-	loaded_mapping_t	*file_mapping;
-	int			i;
-	kern_return_t		ret;
-	mach_vm_offset_t	slide;
-
-	SHARED_REGION_DEBUG(("map_shared_file()\n"));
-
-	shared_file_header = (shared_file_info_t *)sm_info->region_mappings;
-
-	mutex_lock(&shared_file_header->lock);
-
-	/* If this is the first call to this routine, take the opportunity */
-	/* to initialize the hash table which will be used to look-up      */
-	/* mappings based on the file object */ 
-
-	if(shared_file_header->hash_init == FALSE) {
-		ret = shared_file_header_init(shared_file_header);
-		if (ret != KERN_SUCCESS) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: %p: map_shared_file: "
-				 "shared_file_header_init() failed kr=0x%x\n",
-				 current_thread(), ret));
-			mutex_unlock(&shared_file_header->lock);
-			return KERN_NO_SPACE;
-		}
-	}
-
-	
-	/* Find the entry in the map associated with the current mapping */
-	/* of the file object */
-	file_object = memory_object_control_to_vm_object(file_control);
-
-	file_entry = lsf_hash_lookup(shared_file_header->hash, 
-				     (void *) file_object,
-				     mappings[0].sfm_file_offset,
-				     shared_file_header->hash_size, 
-				     TRUE, TRUE, sm_info);
-	if (file_entry) {
-		/* File is loaded, check the load manifest for exact match */
-		/* we simplify by requiring that the elements be the same  */
-		/* size and in the same order rather than checking for     */
-		/* semantic equivalence. */
-
-		i = 0;
-		file_mapping = file_entry->mappings;
-		while(file_mapping != NULL) {
-			if(i>=map_cnt) {
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p: map_shared_file: "
-					 "already mapped with "
-					 "more than %d mappings\n",
-					 current_thread(), map_cnt));
-				shared_region_dump_file_entry(
-					SHARED_REGION_TRACE_INFO,
-					file_entry);
-				shared_region_dump_mappings(
-					SHARED_REGION_TRACE_INFO,
-					mappings, map_cnt, base_offset);
-
-				mutex_unlock(&shared_file_header->lock);
-				return KERN_INVALID_ARGUMENT;
-			}
-			if(((mappings[i].sfm_address)
-			    & SHARED_DATA_REGION_MASK) !=
-			   file_mapping->mapping_offset ||
-			   mappings[i].sfm_size != file_mapping->size ||	
-			   mappings[i].sfm_file_offset != file_mapping->file_offset ||	
-			   mappings[i].sfm_init_prot != file_mapping->protection) {
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p: "
-					 "mapping #%d differs\n",
-					 current_thread(), i));
-				shared_region_dump_file_entry(
-					SHARED_REGION_TRACE_INFO,
-					file_entry);
-				shared_region_dump_mappings(
-					SHARED_REGION_TRACE_INFO,
-					mappings, map_cnt, base_offset);
-
-				break;
-			}
-			file_mapping = file_mapping->next;
-			i++;
-		}
-		if(i!=map_cnt) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_CONFLICT,
-				("shared_region: %p: map_shared_file: "
-				 "already mapped with "
-				 "%d mappings instead of %d\n",
-				 current_thread(), i, map_cnt));
-			shared_region_dump_file_entry(
-				SHARED_REGION_TRACE_INFO,
-				file_entry);
-			shared_region_dump_mappings(
-				SHARED_REGION_TRACE_INFO,
-				mappings, map_cnt, base_offset);
-
-			mutex_unlock(&shared_file_header->lock);
-			return KERN_INVALID_ARGUMENT;
-		}
-
-		slide = file_entry->base_address - base_offset; 
-		if (slide_p != NULL) {
-			/*
-			 * File already mapped but at different address,
-			 * and the caller is OK with the sliding.
-			 */
-			*slide_p = slide;
-			ret = KERN_SUCCESS;
-		} else {
-			/*
-			 * The caller doesn't want any sliding.  The file needs
-			 * to be mapped at the requested address or not mapped.
-			 */
-			if (slide != 0) {
-				/*
-				 * The file is already mapped but at a different
-				 * address.
-				 * We fail.
-				 * XXX should we attempt to load at
-				 * requested address too ?
-				 */
-				ret = KERN_FAILURE;
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p: "
-					 "map_shared_file: already mapped, "
-					 "would need to slide 0x%llx\n",
-					 current_thread(),
-					 slide));
-			} else {
-				/*
-				 * The file is already mapped at the correct
-				 * address.
-				 * We're done !
-				 */
-				ret = KERN_SUCCESS;
-			}
-		}
-		mutex_unlock(&shared_file_header->lock);
-		return ret;
-	} else {
-		/* File is not loaded, lets attempt to load it */
-		ret = lsf_map(mappings, map_cnt, 
-			      (void *)file_control, 
-			      file_size,
-			      sm_info,
-			      base_offset,
-			      slide_p);
-		if(ret == KERN_NO_SPACE) {
-			shared_region_mapping_t	regions;
-			shared_region_mapping_t	system_region;
-			regions = (shared_region_mapping_t)sm_info->self;
-			regions->flags |= SHARED_REGION_FULL;
-			system_region = lookup_default_shared_region(
-				regions->fs_base, regions->system);
-			if (system_region == regions) {
-				shared_region_mapping_t	new_system_shared_region;
-				shared_file_boot_time_init(
-					regions->fs_base, regions->system);
-				/* current task must stay with its current */
-				/* regions, drop count on system_shared_region */
-				/* and put back our original set */
-				vm_get_shared_region(current_task(), 
-						&new_system_shared_region);
-                		shared_region_mapping_dealloc_lock(
-					new_system_shared_region, 0, 1);
-				vm_set_shared_region(current_task(), regions);
-			} else if (system_region != NULL) {
-                		shared_region_mapping_dealloc_lock(
-					system_region, 0, 1);
-			}
-		}
-		mutex_unlock(&shared_file_header->lock);
-		return ret;
-	}
-}
-
-/*
- * shared_region_cleanup:
- *
- * Deallocates all the mappings in the shared region, except those explicitly
- * specified in the "ranges" set of address ranges.
- */
-kern_return_t
-shared_region_cleanup(
-	unsigned int			range_count,
-	struct shared_region_range_np	*ranges,
-	shared_region_task_mappings_t	sm_info)
-{
-	kern_return_t		kr;
-	ipc_port_t		region_handle;
-	vm_named_entry_t	region_named_entry;
-	vm_map_t		text_submap, data_submap, submap, next_submap;
-	unsigned int		i_range;
-	vm_map_offset_t		range_start, range_end;
-	vm_map_offset_t		submap_base, submap_end, submap_offset;
-	vm_map_size_t		delete_size;
-
-	struct shared_region_range_np	tmp_range;
-	unsigned int			sort_index, sorted_index;
-	vm_map_offset_t			sort_min_address;
-	unsigned int			sort_min_index;
-
-	/*
-	 * Since we want to deallocate the holes between the "ranges",
-	 * sort the array by increasing addresses.
-	 */
-	for (sorted_index = 0;
-	     sorted_index < range_count;
-	     sorted_index++) {
-
-		/* first remaining entry is our new starting point */
-		sort_min_index = sorted_index;
-		sort_min_address = ranges[sort_min_index].srr_address;
-
-		/* find the lowest mapping_offset in the remaining entries */
-		for (sort_index = sorted_index + 1;
-		     sort_index < range_count;
-		     sort_index++) {
-			if (ranges[sort_index].srr_address < sort_min_address) {
-				/* lowest address so far... */
-				sort_min_index = sort_index;
-				sort_min_address =
-					ranges[sort_min_index].srr_address;
-			}
-		}
-
-		if (sort_min_index != sorted_index) {
-			/* swap entries */
-			tmp_range = ranges[sort_min_index];
-			ranges[sort_min_index] = ranges[sorted_index];
-			ranges[sorted_index] = tmp_range;
-		}
-	}
-
-	region_handle = (ipc_port_t) sm_info->text_region;
-	region_named_entry = (vm_named_entry_t) region_handle->ip_kobject;
-	text_submap = region_named_entry->backing.map;
-
-	region_handle = (ipc_port_t) sm_info->data_region;
-	region_named_entry = (vm_named_entry_t) region_handle->ip_kobject;
-	data_submap = region_named_entry->backing.map;
-
-	submap = text_submap;
-	next_submap = submap;
-	submap_base = sm_info->client_base;
-	submap_offset = 0;
-	submap_end = submap_base + sm_info->text_size;
-	for (i_range = 0;
-	     i_range < range_count;
-	     i_range++) {
-
-		/* get the next range of addresses to keep */
-		range_start = ranges[i_range].srr_address;
-		range_end = range_start + ranges[i_range].srr_size;
-		/* align them to page boundaries */
-		range_start = vm_map_trunc_page(range_start);
-		range_end = vm_map_round_page(range_end);
-
-		/* make sure we don't go beyond the submap's boundaries */
-		if (range_start < submap_base) {
-			range_start = submap_base;
-		} else if (range_start >= submap_end) {
-			range_start = submap_end;
-		}
-		if (range_end < submap_base) {
-			range_end = submap_base;
-		} else if (range_end >= submap_end) {
-			range_end = submap_end;
-		}
-
-		if (range_start > submap_base + submap_offset) {
-			/*
-			 * Deallocate everything between the last offset in the
-			 * submap and the start of this range.
-			 */
-			delete_size = range_start -
-				(submap_base + submap_offset);
-			(void) vm_deallocate(submap,
-					     submap_offset,
-					     delete_size);
-		} else {
-			delete_size = 0;
-		}
-
-		/* skip to the end of the range */
-		submap_offset += delete_size + (range_end - range_start);
-
-		if (submap_base + submap_offset >= submap_end) {
-			/* get to next submap */
-
-			if (submap == data_submap) {
-				/* no other submap after data: done ! */
-				break;
-			}
-
-			/* get original range again */
-			range_start = ranges[i_range].srr_address;
-			range_end = range_start + ranges[i_range].srr_size;
-			range_start = vm_map_trunc_page(range_start);
-			range_end = vm_map_round_page(range_end);
-
-			if (range_end > submap_end) {
-				/*
-				 * This last range overlaps with the next
-				 * submap.  We need to process it again
-				 * after switching submaps.  Otherwise, we'll
-				 * just continue with the next range.
-				 */
-				i_range--;
-			}
-
-			if (submap == text_submap) {
-				/*
-				 * Switch to the data submap.
-				 */
-				submap = data_submap;
-				submap_offset = 0;
-				submap_base = sm_info->client_base + 
-					sm_info->text_size;
-				submap_end = submap_base + sm_info->data_size;
-			}
-		}
-	}
-
-	if (submap_base + submap_offset < submap_end) {
-		/* delete remainder of this submap, from "offset" to the end */
-		(void) vm_deallocate(submap,
-				     submap_offset,
-				     submap_end - submap_base - submap_offset);
-		/* if nothing to keep in data submap, delete it all */
-		if (submap == text_submap) {
-			submap = data_submap;
-			submap_offset = 0;
-			submap_base = sm_info->client_base + sm_info->text_size;
-			submap_end = submap_base + sm_info->data_size;
-			(void) vm_deallocate(data_submap,
-					     0,
-					     submap_end - submap_base);
-		}
-	}
-
-	kr = KERN_SUCCESS;
-	return kr;
-}
-
-/* A hash lookup function for the list of loaded files in      */
-/* shared_memory_server space.  */
-
-static load_struct_t  *
-lsf_hash_lookup(
-	queue_head_t			*hash_table,
-	void				*file_object,
-	vm_offset_t			recognizableOffset,
-	int				size,
-	boolean_t			regular,
-	boolean_t			alternate,
-	shared_region_task_mappings_t	sm_info)
-{
-	register queue_t	bucket;
-	load_struct_t		*entry;
-	shared_region_mapping_t	target_region;
-	int			depth;
-	
-	LSF_DEBUG(("lsf_hash_lookup: table=%p, file=%p, offset=0x%x size=0x%x "
-		   "reg=%d alt=%d sm_info=%p\n",
-		   hash_table, file_object, recognizableOffset, size,
-		   regular, alternate, sm_info));
-
-	bucket = &(hash_table[load_file_hash((int)file_object, size)]);
-	for (entry = (load_struct_t *)queue_first(bucket);
-		!queue_end(bucket, &entry->links);
-		entry = (load_struct_t *)queue_next(&entry->links)) {
-
-		if ((entry->file_object == (int)file_object) &&
-                    (entry->file_offset == recognizableOffset)) {
-		   target_region = (shared_region_mapping_t)sm_info->self;
-		   depth = target_region->depth;
-		   while(target_region) {
-		      if((!(sm_info->self)) ||
-				((target_region == entry->regions_instance) &&
-				(target_region->depth >= entry->depth))) {
-			if(alternate &&
-			   entry->base_address >= sm_info->alternate_base) {
-				LSF_DEBUG(("lsf_hash_lookup: "
-					   "alt=%d found entry %p "
-					   "(base=0x%x "
-					   "alt_base=0x%x)\n",
-					   alternate, entry,
-					   entry->base_address,
-					   sm_info->alternate_base));
-				return entry;
-			}
-		        if (regular &&
-			    entry->base_address < sm_info->alternate_base) {
-				LSF_DEBUG(("lsf_hash_lookup: "
-					   "reg=%d found entry %p "
-					   "(base=0x%x "
-					   "alt_base=0x%x)\n",
-					   regular, entry,
-					   entry->base_address,
-					   sm_info->alternate_base));
-				return entry;
-			}
-		      }
-		      if(target_region->object_chain) {
-		         target_region = (shared_region_mapping_t)
-			    target_region->object_chain->object_chain_region;
-		         depth = target_region->object_chain->depth;
-		      } else {
-			target_region = NULL;
-		      }
-		   }
-		}
-	}
-
-	LSF_DEBUG(("lsf_hash_lookup: table=%p, file=%p, offset=0x%x size=0x%x "
-		   "reg=%d alt=%d sm_info=%p NOT FOUND\n",
-		   hash_table, file_object, recognizableOffset, size,
-		   regular, alternate, sm_info));
-	return (load_struct_t *)0;
-}
-
-__private_extern__ load_struct_t *
-lsf_remove_regions_mappings_lock(
-	shared_region_mapping_t	region,
-	shared_region_task_mappings_t	sm_info,
-	int need_sfh_lock)
-{
-	int			i;
-	register queue_t	bucket;
-	shared_file_info_t	*shared_file_header;
-	load_struct_t		*entry;
-	load_struct_t		*next_entry;
-
-	shared_file_header = (shared_file_info_t *)sm_info->region_mappings;
-
-	LSF_DEBUG(("lsf_remove_regions_mappings_lock(region=%p,sm_info=%p) "
-		   "sfh=%p\n",
-		   region, sm_info, shared_file_header));
-	if (need_sfh_lock)
-		mutex_lock(&shared_file_header->lock);
-	if(shared_file_header->hash_init == FALSE) {
-		if (need_sfh_lock)
-			mutex_unlock(&shared_file_header->lock);
-		LSF_DEBUG(("lsf_remove_regions_mappings_lock"
-			   "(region=%p,sm_info=%p): not inited\n",
-			   region, sm_info));
-		return NULL;
-	}
-	for(i = 0;  i<shared_file_header->hash_size; i++) {
-		bucket = &shared_file_header->hash[i];
-		for (entry = (load_struct_t *)queue_first(bucket);
-			!queue_end(bucket, &entry->links);) {
-		   next_entry = (load_struct_t *)queue_next(&entry->links);
-		   if(region == entry->regions_instance) {
-			   LSF_DEBUG(("lsf_remove_regions_mapping_lock: "
-				      "entry %p region %p: "
-				      "unloading\n",
-				      entry, region));
-			   lsf_unload((void *)entry->file_object, 
-					entry->base_address, sm_info);
-		   } else {
-			   LSF_DEBUG(("lsf_remove_regions_mapping_lock: "
-				      "entry %p region %p target region %p: "
-				      "not unloading\n",
-				      entry, entry->regions_instance, region));
-		   }
-			   
-		   entry = next_entry;
-		}
-	}
-	if (need_sfh_lock)
-		mutex_unlock(&shared_file_header->lock);
-	LSF_DEBUG(("lsf_removed_regions_mapping_lock done\n"));
-
-	return NULL;	/* XXX */
-}
-
-/*
- * Symbol compatability; we believe shared_region_mapping_dealloc() is the
- * only caller.  Remove this stub function and the corresponding symbol
- * export for Merlot.
- */
-load_struct_t *
-lsf_remove_regions_mappings(
-	shared_region_mapping_t	region,
-	shared_region_task_mappings_t	sm_info)
-{
-	return lsf_remove_regions_mappings_lock(region, sm_info, 1);
-}
-
-/* Removes a map_list, (list of loaded extents) for a file from     */
-/* the loaded file hash table.  */
-
-static load_struct_t *
-lsf_hash_delete(
-	load_struct_t	*target_entry,	/* optional:  NULL if not relevant */
-	void		*file_object,
-	vm_offset_t	base_offset,
-	shared_region_task_mappings_t	sm_info)
-{
-	register queue_t	bucket;
-	shared_file_info_t	*shared_file_header;
-	load_struct_t		*entry;
-
-	LSF_DEBUG(("lsf_hash_delete(target=%p,file=%p,base=0x%x,sm_info=%p)\n",
-		   target_entry, file_object, base_offset, sm_info));
-
-	shared_file_header = (shared_file_info_t *)sm_info->region_mappings;
-
-	bucket = &shared_file_header->hash
-	     [load_file_hash((int)file_object, shared_file_header->hash_size)];
-
-	for (entry = (load_struct_t *)queue_first(bucket);
-		!queue_end(bucket, &entry->links);
-		entry = (load_struct_t *)queue_next(&entry->links)) {
-		if((!(sm_info->self)) || ((shared_region_mapping_t)
-				sm_info->self == entry->regions_instance)) {
-			if ((target_entry == NULL ||
-			     entry == target_entry) &&
-			    (entry->file_object == (int) file_object)  &&
-			    (entry->base_address == base_offset)) {
-				queue_remove(bucket, entry, 
-						load_struct_ptr_t, links);
-				LSF_DEBUG(("lsf_hash_delete: found it\n"));
-				return entry;
-			}
-		}
-	}
-
-	LSF_DEBUG(("lsf_hash_delete; not found\n"));
-	return (load_struct_t *)0;
-}
-
-/* Inserts a new map_list, (list of loaded file extents), into the */
-/* server loaded file hash table. */
-
-static void
-lsf_hash_insert(
-	load_struct_t			*entry,
-	shared_region_task_mappings_t	sm_info)
-{
-	shared_file_info_t *shared_file_header;
-
-	LSF_DEBUG(("lsf_hash_insert(entry=%p,sm_info=%p): file=%p base=0x%x\n",
-		   entry, sm_info, entry->file_object, entry->base_address));
-
-	shared_file_header = (shared_file_info_t *)sm_info->region_mappings;
-	queue_enter(&shared_file_header->hash
-			[load_file_hash(entry->file_object, 
-					shared_file_header->hash_size)],
-			entry, load_struct_ptr_t, links);
-}
-	
-
-
-/*
- * lsf_slide:
- *
- * Look in the shared region, starting from the end, for a place to fit all the
- * mappings while respecting their relative offsets.
- */
-static kern_return_t
-lsf_slide(
-	unsigned int			map_cnt,
-	struct shared_file_mapping_np	*mappings_in,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		*base_offset_p)
-{
-	mach_vm_offset_t 		max_mapping_offset;
-	int 			       	i;
-	vm_map_entry_t			map_entry, prev_entry, next_entry;
-	mach_vm_offset_t		prev_hole_start, prev_hole_end;
-	mach_vm_offset_t		mapping_offset, mapping_end_offset;
-	mach_vm_offset_t		base_offset;
-	mach_vm_size_t			mapping_size;
-	mach_vm_offset_t		wiggle_room, wiggle;
-	vm_map_t			text_map, data_map, map;
-	vm_named_entry_t		region_entry;
-	ipc_port_t			region_handle;
-	kern_return_t			kr;
-
-	struct shared_file_mapping_np	*mappings, tmp_mapping;
-	unsigned int			sort_index, sorted_index;
-	vm_map_offset_t			sort_min_address;
-	unsigned int			sort_min_index;
-
-	/*
-	 * Sort the mappings array, so that we can try and fit them in
-	 * in the right order as we progress along the VM maps.
-	 *
-	 * We can't modify the original array (the original order is
-	 * important when doing lookups of the mappings), so copy it first.
-	 */
-
-	kr = kmem_alloc(kernel_map,
-			(vm_offset_t *) &mappings,
-			(vm_size_t) (map_cnt * sizeof (mappings[0])));
-	if (kr != KERN_SUCCESS) {
-		return KERN_NO_SPACE;
-	}
-
-	bcopy(mappings_in, mappings, map_cnt * sizeof (mappings[0]));
-
-	max_mapping_offset = 0;
-	for (sorted_index = 0;
-	     sorted_index < map_cnt;
-	     sorted_index++) {
-
-		/* first remaining entry is our new starting point */
-		sort_min_index = sorted_index;
-		mapping_end_offset = ((mappings[sort_min_index].sfm_address &
-				       SHARED_TEXT_REGION_MASK) +
-				      mappings[sort_min_index].sfm_size);
-		sort_min_address = mapping_end_offset;
-		/* compute the highest mapping_offset as well... */
-		if (mapping_end_offset > max_mapping_offset) {
-			max_mapping_offset = mapping_end_offset;
-		}
-		/* find the lowest mapping_offset in the remaining entries */
-		for (sort_index = sorted_index + 1;
-		     sort_index < map_cnt;
-		     sort_index++) {
-
-			mapping_end_offset =
-				((mappings[sort_index].sfm_address &
-				  SHARED_TEXT_REGION_MASK) +
-				 mappings[sort_index].sfm_size);
-
-			if (mapping_end_offset < sort_min_address) {
-				/* lowest mapping_offset so far... */
-				sort_min_index = sort_index;
-				sort_min_address = mapping_end_offset;
-			}
-		}
-		if (sort_min_index != sorted_index) {
-			/* swap entries */
-			tmp_mapping = mappings[sort_min_index];
-			mappings[sort_min_index] = mappings[sorted_index];
-			mappings[sorted_index] = tmp_mapping;
-		}
-
-	}
-
-	max_mapping_offset = vm_map_round_page(max_mapping_offset);
-
-	/* start from the end of the shared area */
-	base_offset = sm_info->text_size;
-
-	/* can all the mappings fit ? */
-	if (max_mapping_offset > base_offset) {
-		kmem_free(kernel_map,
-			  (vm_offset_t) mappings,
-			  map_cnt * sizeof (mappings[0]));
-		return KERN_FAILURE;
-	}
-
-	/*
-	 * Align the last mapping to the end of the submaps
-	 * and start from there.
-	 */
-	base_offset -= max_mapping_offset;
-
-	region_handle = (ipc_port_t) sm_info->text_region;
-	region_entry = (vm_named_entry_t) region_handle->ip_kobject;
-	text_map = region_entry->backing.map;
-
-	region_handle = (ipc_port_t) sm_info->data_region;
-	region_entry = (vm_named_entry_t) region_handle->ip_kobject;
-	data_map = region_entry->backing.map;
-
-	vm_map_lock_read(text_map);
-	vm_map_lock_read(data_map);
-
-start_over:
-	/*
-	 * At first, we can wiggle all the way from our starting point
-	 * (base_offset) towards the start of the map (0), if needed.
-	 */
-	wiggle_room = base_offset;
-
-	for (i = (signed) map_cnt - 1; i >= 0; i--) {
-		if (mappings[i].sfm_size == 0) {
-			/* nothing to map here... */
-			continue;
-		}
-		if (mappings[i].sfm_init_prot & VM_PROT_COW) {
-			/* copy-on-write mappings are in the data submap */
-			map = data_map;
-		} else {
-			/* other mappings are in the text submap */
-			map = text_map;
-		}
-		/* get the offset within the appropriate submap */
-		mapping_offset = (mappings[i].sfm_address &
-				  SHARED_TEXT_REGION_MASK);
-		mapping_size = mappings[i].sfm_size;
-		mapping_end_offset = mapping_offset + mapping_size;
-		mapping_offset = vm_map_trunc_page(mapping_offset);
-		mapping_end_offset = vm_map_round_page(mapping_end_offset);
-		mapping_size = mapping_end_offset - mapping_offset;
-
-		for (;;) {
-			if (vm_map_lookup_entry(map,
-						base_offset + mapping_offset,
-						&map_entry)) {
-				/*
-				 * The start address for that mapping
-				 * is already mapped: no fit.
-				 * Locate the hole immediately before this map
-				 * entry.
-				 */
-				prev_hole_end = map_entry->vme_start;
-				prev_entry = map_entry->vme_prev;
-				if (prev_entry == vm_map_to_entry(map)) {
-					/* no previous entry */
-					prev_hole_start = map->min_offset;
-				} else {
-					/* previous entry ends here */
-					prev_hole_start = prev_entry->vme_end;
-				}
-			} else {
-				/*
-				 * The start address for that mapping is not
-				 * mapped.
-				 * Locate the start and end of the hole
-				 * at that location.
-				 */
-				/* map_entry is the previous entry */
-				if (map_entry == vm_map_to_entry(map)) {
-					/* no previous entry */
-					prev_hole_start = map->min_offset;
-				} else {
-					/* previous entry ends there */
-					prev_hole_start = map_entry->vme_end;
-				}
-				next_entry = map_entry->vme_next;
-				if (next_entry == vm_map_to_entry(map)) {
-					/* no next entry */
-					prev_hole_end = map->max_offset;
-				} else {
-					prev_hole_end = next_entry->vme_start;
-				}
-			}
-
-			if (prev_hole_end <= base_offset + mapping_offset) {
-				/* hole is to our left: try and wiggle to fit */
-				wiggle = base_offset + mapping_offset - prev_hole_end + mapping_size;
-				if (wiggle > base_offset) {
-					/* we're getting out of the map */
-					kr = KERN_FAILURE;
-					goto done;
-				}
-				base_offset -= wiggle;
-				if (wiggle > wiggle_room) {
-					/* can't wiggle that much: start over */
-					goto start_over;
-				}
-				/* account for the wiggling done */
-				wiggle_room -= wiggle;
-			}
-
-			if (prev_hole_end >
-			    base_offset + mapping_offset + mapping_size) {
-				/*
-				 * The hole extends further to the right
-				 * than what we need.  Ignore the extra space.
-				 */
-				prev_hole_end =	(base_offset + mapping_offset +
-						 mapping_size);
-			}
-
-			if (prev_hole_end <
-			    base_offset + mapping_offset + mapping_size) {
-				/*
-				 * The hole is not big enough to establish
-				 * the mapping right there:  wiggle towards
-				 * the beginning of the hole so that the end
-				 * of our mapping fits in the hole...
-				 */
-				wiggle = base_offset + mapping_offset
-					+ mapping_size - prev_hole_end;
-				if (wiggle > base_offset) {
-					/* we're getting out of the map */
-					kr = KERN_FAILURE;
-					goto done;
-				}
-				base_offset -= wiggle;
-				if (wiggle > wiggle_room) {
-					/* can't wiggle that much: start over */
-					goto start_over;
-				}
-				/* account for the wiggling done */
-				wiggle_room -= wiggle;
-
-				/* keep searching from this new base */
-				continue;
-			}
-
-			if (prev_hole_start > base_offset + mapping_offset) {
-				/* no hole found: keep looking */
-				continue;
-			}
-
-			/* compute wiggling room at this hole */
-			wiggle = base_offset + mapping_offset - prev_hole_start;
-			if (wiggle < wiggle_room) {
-				/* less wiggle room than before... */
-				wiggle_room = wiggle;
-			}
-
-			/* found a hole that fits: skip to next mapping */
-			break;
-		} /* while we look for a hole */
-	} /* for each mapping */
-
-	*base_offset_p = base_offset;
-	kr = KERN_SUCCESS;
-
-done:
-	vm_map_unlock_read(text_map);
-	vm_map_unlock_read(data_map);
-
-	kmem_free(kernel_map,
-		  (vm_offset_t) mappings,
-		  map_cnt * sizeof (mappings[0]));
-
-	return kr;
-}
-
-/*
- * lsf_map:
- *
- * Attempt to establish the mappings for a split library into the shared region.
- */
-static kern_return_t
-lsf_map(
-	struct shared_file_mapping_np	*mappings,
-	int				map_cnt,
-	void				*file_control,
-	memory_object_offset_t		file_size,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		base_offset,
-	mach_vm_offset_t		*slide_p)
-{
-	load_struct_t		*entry;
-	loaded_mapping_t	*file_mapping;
-	loaded_mapping_t	**tptr;
-	ipc_port_t		region_handle;
-	vm_named_entry_t	region_entry;
-	mach_port_t		map_port;
-	vm_object_t		file_object;
-	kern_return_t		kr;
-	int			i;
-	mach_vm_offset_t	original_base_offset;
-	mach_vm_size_t		total_size;
-
-	/* get the VM object from the file's memory object handle */
-	file_object = memory_object_control_to_vm_object(file_control);
-
-	original_base_offset = base_offset;
-
-	LSF_DEBUG(("lsf_map"
-		   "(cnt=%d,file=%p,sm_info=%p)"
-		   "\n",
-		   map_cnt, file_object,
-		   sm_info));
-
-restart_after_slide:
-	/* get a new "load_struct_t" to described the mappings for that file */
-	entry = (load_struct_t *)zalloc(lsf_zone);
-	LSF_ALLOC_DEBUG(("lsf_map: entry=%p map_cnt=%d\n", entry, map_cnt));
-	LSF_DEBUG(("lsf_map"
-		   "(cnt=%d,file=%p,sm_info=%p) "
-		   "entry=%p\n",
-		   map_cnt, file_object,
-		   sm_info, entry));
-	if (entry == NULL) {
-		SHARED_REGION_TRACE(
-			SHARED_REGION_TRACE_ERROR,
-			("shared_region: %p: "
-			 "lsf_map: unable to allocate entry\n",
-			 current_thread()));
-		return KERN_NO_SPACE;
-	}
-	shared_file_available_hash_ele--;
-	entry->file_object = (int)file_object;
-	entry->mapping_cnt = map_cnt;
-	entry->mappings = NULL;
-	entry->links.prev = (queue_entry_t) 0;
-	entry->links.next = (queue_entry_t) 0;
-	entry->regions_instance = (shared_region_mapping_t)sm_info->self;
-	entry->depth=((shared_region_mapping_t)sm_info->self)->depth;
-        entry->file_offset = mappings[0].sfm_file_offset;
-
-	/* insert the new file entry in the hash table, for later lookups */
-	lsf_hash_insert(entry, sm_info);
-
-	/* where we should add the next mapping description for that file */
-	tptr = &(entry->mappings);
-
-	entry->base_address = base_offset;
-	total_size = 0;
-
-	/* establish each requested mapping */
-	for (i = 0; i < map_cnt; i++) {
-		mach_vm_offset_t	target_address;
-		mach_vm_offset_t	region_mask;
-
-		if (mappings[i].sfm_init_prot & VM_PROT_COW) {
-			region_handle = (ipc_port_t)sm_info->data_region;
-			region_mask = SHARED_DATA_REGION_MASK;
-			if ((((mappings[i].sfm_address + base_offset)
-			      & GLOBAL_SHARED_SEGMENT_MASK) != 0x10000000) ||
-			    (((mappings[i].sfm_address + base_offset +
-			       mappings[i].sfm_size - 1)
-			      & GLOBAL_SHARED_SEGMENT_MASK) != 0x10000000)) {
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_ERROR,
-					("shared_region: %p: lsf_map: "
-					 "RW mapping #%d not in segment",
-					 current_thread(), i));
-				shared_region_dump_mappings(
-					SHARED_REGION_TRACE_ERROR,
-					mappings, map_cnt, base_offset);
-
-				lsf_deallocate(entry,
-					       file_object, 
-					       entry->base_address,
-					       sm_info,
-					       TRUE);
-				return KERN_INVALID_ARGUMENT;
-			}
-		} else {
-			region_mask = SHARED_TEXT_REGION_MASK;
-			region_handle = (ipc_port_t)sm_info->text_region;
-			if (((mappings[i].sfm_address + base_offset)
-			     & GLOBAL_SHARED_SEGMENT_MASK) ||
-			    ((mappings[i].sfm_address + base_offset +
-			      mappings[i].sfm_size - 1)
-			     & GLOBAL_SHARED_SEGMENT_MASK)) {
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_ERROR,
-					("shared_region: %p: lsf_map: "
-					 "RO mapping #%d not in segment",
-					 current_thread(), i));
-				shared_region_dump_mappings(
-					SHARED_REGION_TRACE_ERROR,
-					mappings, map_cnt, base_offset);
-
-				lsf_deallocate(entry,
-					       file_object, 
-					       entry->base_address,
-					       sm_info,
-					       TRUE);
-				return KERN_INVALID_ARGUMENT;
-			}
-		}
-		if (!(mappings[i].sfm_init_prot & VM_PROT_ZF) &&
-		    ((mappings[i].sfm_file_offset + mappings[i].sfm_size) >
-		     (file_size))) {
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: %p: lsf_map: "
-				 "ZF mapping #%d beyond EOF",
-				 current_thread(), i));
-			shared_region_dump_mappings(SHARED_REGION_TRACE_ERROR,
-						    mappings, map_cnt,
-						    base_offset);
-
-
-			lsf_deallocate(entry,
-				       file_object,
-				       entry->base_address,
-				       sm_info,
-				       TRUE);
-			return KERN_INVALID_ARGUMENT;
-		}
-		target_address = entry->base_address +
-			((mappings[i].sfm_address) & region_mask);
-		if (mappings[i].sfm_init_prot & VM_PROT_ZF) {
-			map_port = MACH_PORT_NULL;
-		} else {
-			map_port = (ipc_port_t) file_object->pager;
-		}
-		region_entry = (vm_named_entry_t) region_handle->ip_kobject;
-
-		total_size += mappings[i].sfm_size;
-		if (mappings[i].sfm_size == 0) {
-			/* nothing to map... */
-			kr = KERN_SUCCESS;
-		} else {
-			kr = mach_vm_map(
-				region_entry->backing.map,
-				&target_address,
-				vm_map_round_page(mappings[i].sfm_size),
-				0,
-				VM_FLAGS_FIXED,
-				map_port,
-				mappings[i].sfm_file_offset,
-				TRUE,
-				(mappings[i].sfm_init_prot &
-				 (VM_PROT_READ|VM_PROT_EXECUTE)),
-				(mappings[i].sfm_max_prot &
-				 (VM_PROT_READ|VM_PROT_EXECUTE)),
-				VM_INHERIT_DEFAULT);
-		}
-		if (kr != KERN_SUCCESS) {
-			vm_offset_t old_base_address;
-
-			old_base_address = entry->base_address;
-			lsf_deallocate(entry,
-				       file_object,
-				       entry->base_address,
-				       sm_info,
-				       TRUE);
-			entry = NULL;
-
-			if (slide_p != NULL) {
-				/*
-				 * Requested mapping failed but the caller
-				 * is OK with sliding the library in the
-				 * shared region, so let's try and slide it...
-				 */
-
-				SHARED_REGION_TRACE(
-					SHARED_REGION_TRACE_CONFLICT,
-					("shared_region: %p: lsf_map: "
-					 "mapping #%d failed to map, "
-					 "kr=0x%x, sliding...\n",
-					 current_thread(), i, kr));
-				shared_region_dump_mappings(
-					SHARED_REGION_TRACE_INFO,
-					mappings, map_cnt, base_offset);
-				shared_region_dump_conflict_info(
-					SHARED_REGION_TRACE_CONFLICT,
-					region_entry->backing.map,
-					(old_base_address +
-					 ((mappings[i].sfm_address)
-					  & region_mask)),
-					vm_map_round_page(mappings[i].sfm_size));
-
-				/* lookup an appropriate spot */
-				kr = lsf_slide(map_cnt, mappings,
-					       sm_info, &base_offset);
-				if (kr == KERN_SUCCESS) {
-					/* try and map it there ... */
-					goto restart_after_slide;
-				}
-				/* couldn't slide ... */
-			}
-			       
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_CONFLICT,
-				("shared_region: %p: lsf_map: "
-				 "mapping #%d failed to map, "
-				 "kr=0x%x, no sliding\n",
-				 current_thread(), i, kr));
-			shared_region_dump_mappings(
-				SHARED_REGION_TRACE_INFO,
-				mappings, map_cnt, base_offset);
-			shared_region_dump_conflict_info(
-				SHARED_REGION_TRACE_CONFLICT,
-				region_entry->backing.map,
-				(old_base_address +
-				 ((mappings[i].sfm_address)
-				  & region_mask)),
-				vm_map_round_page(mappings[i].sfm_size));
-			return KERN_FAILURE;
-		}
-
-		/* record this mapping */
-		file_mapping = (loaded_mapping_t *)zalloc(lsf_zone);
-		if (file_mapping == NULL) {
-			lsf_deallocate(entry,
-				       file_object,
-				       entry->base_address,
-				       sm_info,
-				       TRUE);
-			SHARED_REGION_TRACE(
-				SHARED_REGION_TRACE_ERROR,
-				("shared_region: %p: "
-				 "lsf_map: unable to allocate mapping\n",
-				 current_thread()));
-			return KERN_NO_SPACE;
-		}
-		shared_file_available_hash_ele--;
-		file_mapping->mapping_offset = (mappings[i].sfm_address) 
-								& region_mask;
-		file_mapping->size = mappings[i].sfm_size;
-		file_mapping->file_offset = mappings[i].sfm_file_offset;
-		file_mapping->protection = mappings[i].sfm_init_prot;
-		file_mapping->next = NULL;
-		LSF_DEBUG(("lsf_map: file_mapping %p "
-			   "for offset=0x%x size=0x%x\n",
-			   file_mapping, file_mapping->mapping_offset,
-			   file_mapping->size));
-
-		/* and link it to the file entry */
-		*tptr = file_mapping;
-
-		/* where to put the next mapping's description */
-		tptr = &(file_mapping->next);
-	}
-
-	if (slide_p != NULL) {
-		*slide_p = base_offset - original_base_offset;
-	}
-
-	if ((sm_info->flags & SHARED_REGION_STANDALONE) ||
-	    (total_size == 0)) {
-		/*
-		 * Two cases:
-		 * 1. we have a standalone and private shared region, so we
-		 * don't really need to keep the information about each file
-		 * and each mapping.  Just deallocate it all.
-		 * 2. the total size of the mappings is 0, so nothing at all
-		 * was mapped.  Let's not waste kernel resources to describe
-		 * nothing.
-		 *
-		 * XXX we still have the hash table, though...
-		 */
-		lsf_deallocate(entry, file_object, entry->base_address, sm_info,
-			       FALSE);
-	}
-
-	LSF_DEBUG(("lsf_map: done\n"));
-	return KERN_SUCCESS;			
-}
-
-
-/* finds the file_object extent list in the shared memory hash table       */
-/* If one is found the associated extents in shared memory are deallocated */
-/* and the extent list is freed */
-
-static void
-lsf_unload(
-	void			*file_object,
-	vm_offset_t	        base_offset,
-	shared_region_task_mappings_t	sm_info)
-{
-	lsf_deallocate(NULL, file_object, base_offset, sm_info, TRUE);
-}
-
-/*
- * lsf_deallocate:
- *
- * Deallocates all the "shared region" internal data structures describing
- * the file and its mappings.
- * Also deallocate the actual file mappings if requested ("unload" arg).
- */
-static void
-lsf_deallocate(
-	load_struct_t		*target_entry,
-	void			*file_object,
-	vm_offset_t		base_offset,
-	shared_region_task_mappings_t	sm_info,
-	boolean_t		unload)
-{
-	load_struct_t		*entry;
-	loaded_mapping_t	*map_ele;
-	loaded_mapping_t	*back_ptr;
-	kern_return_t		kr;
-
-	LSF_DEBUG(("lsf_deallocate(target=%p,file=%p,base=0x%x,sm_info=%p,unload=%d)\n",
-		   target_entry, file_object, base_offset, sm_info, unload));
-	entry = lsf_hash_delete(target_entry,
-				file_object,
-				base_offset,
-				sm_info);
-	if (entry) {
-		map_ele = entry->mappings;
-		while(map_ele != NULL) {
-			if (unload) {
-				ipc_port_t		region_handle;
-				vm_named_entry_t	region_entry;
-
-				if(map_ele->protection & VM_PROT_COW) {
-					region_handle = (ipc_port_t)
-						sm_info->data_region;
-				} else {
-					region_handle = (ipc_port_t)
-						sm_info->text_region;
-				}
-				region_entry = (vm_named_entry_t)
-					region_handle->ip_kobject;
-				
-				kr = vm_deallocate(region_entry->backing.map,
-						   (entry->base_address + 
-						    map_ele->mapping_offset),
-						   map_ele->size);
-				assert(kr == KERN_SUCCESS);
-			}
-			back_ptr = map_ele;
-			map_ele = map_ele->next;
-			LSF_DEBUG(("lsf_deallocate: freeing mapping %p "
-				   "offset 0x%x size 0x%x\n",
-				   back_ptr, back_ptr->mapping_offset,
-				   back_ptr->size));
-			zfree(lsf_zone, back_ptr);
-			shared_file_available_hash_ele++;
-		}
-		LSF_DEBUG(("lsf_deallocate: freeing entry %p\n", entry));
-		LSF_ALLOC_DEBUG(("lsf_deallocate: entry=%p", entry));
-		zfree(lsf_zone, entry);
-	        shared_file_available_hash_ele++;
-	}
-	LSF_DEBUG(("lsf_deallocate: done\n"));
-}
-
-/* integer is from 1 to 100 and represents percent full */
-unsigned int
-lsf_mapping_pool_gauge(void)
-{
-	return ((lsf_zone->count * lsf_zone->elem_size) * 100)/lsf_zone->max_size;
-}
diff --git a/osfmk/vm/vm_shared_memory_server.h b/osfmk/vm/vm_shared_memory_server.h
deleted file mode 100644
index 3c107fade..000000000
--- a/osfmk/vm/vm_shared_memory_server.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-/*
- *
- *	File: vm/vm_shared_memory_server.h
- *
- * 	protos and struct definitions for shared library
- *	server and interface
- */
-
-#ifndef _VM_SHARED_MEMORY_SERVER_H_
-#define _VM_SHARED_MEMORY_SERVER_H_
-
-#ifdef	KERNEL_PRIVATE
-
-#include <mach/vm_prot.h>
-#include <mach/mach_types.h>
-#include <mach/shared_memory_server.h>
-
-#include <kern/kern_types.h>
-#include <kern/macro_help.h>
-
-#if DEBUG
-extern int shared_region_debug;
-#define SHARED_REGION_DEBUG(args)		\
-	MACRO_BEGIN				\
-	if (shared_region_debug) {		\
-		kprintf args;			\
-	}					\
-	MACRO_END
-#else /* DEBUG */
-#define SHARED_REGION_DEBUG(args)
-#endif /* DEBUG */
-
-extern int shared_region_trace_level;
-#define SHARED_REGION_TRACE_NONE	0	/* no trace */
-#define SHARED_REGION_TRACE_ERROR	1	/* trace abnormal events */
-#define SHARED_REGION_TRACE_CONFLICT	2	/* trace library conflicts */
-#define SHARED_REGION_TRACE_INFO	3	/* trace all events */
-#define SHARED_REGION_TRACE(level, args)		\
-	MACRO_BEGIN					\
-	if (level <= shared_region_trace_level) {	\
-		printf args;				\
-	}						\
-	MACRO_END
-
-struct shared_region_task_mappings {
-	mach_port_t		text_region;
-	vm_size_t		text_size;
-	mach_port_t		data_region;
-	vm_size_t		data_size;
-	vm_offset_t		region_mappings;
-	vm_offset_t		client_base;
-	vm_offset_t		alternate_base;
-	vm_offset_t		alternate_next;
-	unsigned int		fs_base;
-	unsigned int		system;
-	int			flags;
-	vm_offset_t		self;
-};
-
-#define SHARED_REGION_SYSTEM	0x1 // Default env for system and fs_root
-#define SHARED_REGION_FULL	0x2 // Shared regions are full
-#define SHARED_REGION_STALE 	0x4 // Indicates no longer in default list
-#define SHARED_REGION_STANDALONE 0x10 // Shared region is not shared !
-
-
-/* defines for default environment, and co-resident systems */
-
-#define ENV_DEFAULT_ROOT	0
-
-typedef	struct shared_region_task_mappings *shared_region_task_mappings_t;
-typedef struct shared_region_mapping *shared_region_mapping_t;
-
-#ifdef MACH_KERNEL_PRIVATE
-
-#include <kern/queue.h>
-#include <vm/vm_object.h>
-#include <vm/memory_object.h>
-
-extern vm_offset_t     shared_file_mapping_array;
-
-struct loaded_mapping {
-	vm_offset_t	mapping_offset;
-	vm_size_t	size;
-	vm_offset_t	file_offset;
-	vm_prot_t	protection;  /* read/write/execute/COW/ZF */
-	
-	struct loaded_mapping *next;
-};
-
-typedef struct loaded_mapping loaded_mapping_t;
-
-struct load_struct {
-	queue_chain_t   	links;  
-	shared_region_mapping_t	regions_instance;
-	int			depth;
-	int			file_object;
-	vm_offset_t		base_address;
-	int			mapping_cnt;
-	loaded_mapping_t	*mappings;
-        vm_offset_t             file_offset; // start of file we mapped in
-};
-
-typedef struct load_struct load_struct_t;
-typedef struct load_struct *load_struct_ptr_t;
-
-struct load_file_ele {
-	union {
-		sf_mapping_t	mapping;
-		load_struct_t	element;
-	} u;
-};
-
-struct shared_file_info {
-	mutex_t	    	lock;   /* lock for the structure */
-	queue_head_t	*hash;  /* for later perf enhance */
-	int		hash_size;
-	boolean_t	hash_init;
-};
-
-typedef struct shared_file_info shared_file_info_t;
-
-struct shared_region_object_chain {
-	shared_region_mapping_t	object_chain_region;
-	int			depth;
-	struct shared_region_object_chain *next;
-};
-typedef struct shared_region_object_chain *shared_region_object_chain_t;
-
-/* address space shared region descriptor */
-struct shared_region_mapping {
-        decl_mutex_data(,       Lock)   /* Synchronization */
-	unsigned int		ref_count;
-	unsigned int		fs_base;
-	unsigned int		system;
-	mach_port_t		text_region;
-	vm_size_t		text_size;
-	mach_port_t		data_region;
-	vm_size_t		data_size;
-	vm_offset_t		region_mappings;
-	vm_offset_t		client_base;
-	vm_offset_t		alternate_base;
-	vm_offset_t		alternate_next;
-	int			flags;
-	int			depth;
-	shared_region_mapping_t default_env_list;
-	shared_region_object_chain_t object_chain;
-	shared_region_mapping_t self;
-	shared_region_mapping_t next;
-};
-
-#define shared_region_mapping_lock_init(object)   \
-			mutex_init(&(object)->Lock, 0)
-#define shared_region_mapping_lock(object)        mutex_lock(&(object)->Lock)
-#define shared_region_mapping_unlock(object)      mutex_unlock(&(object)->Lock)
-
-#else  /* !MACH_KERNEL_PRIVATE */
-
-struct shared_region_mapping ;
-
-#endif /* MACH_KERNEL_PRIVATE */
-
-#define load_file_hash(file_object, size) \
-		((((natural_t)file_object) & 0xffffff) % size)
-
-extern kern_return_t map_shared_file(
-	int 				map_cnt,
-	struct shared_file_mapping_np 	*mappings,
-	memory_object_control_t		file_control,
-	memory_object_size_t		file_size,
-	shared_region_task_mappings_t	sm_info,
-	mach_vm_offset_t		base_offset,
-	mach_vm_offset_t		*slide_p);
-
-extern kern_return_t shared_region_cleanup(
-	unsigned int			range_count,
-	struct shared_region_range_np	*ranges,
-	shared_region_task_mappings_t	sm_info);
-
-extern kern_return_t shared_region_mapping_info(
-				shared_region_mapping_t	shared_region,
-				mach_port_t		*text_region,
-				vm_size_t		*text_size,
-				mach_port_t		*data_region,
-				vm_size_t		*data_size,
-				vm_offset_t		*region_mappings,
-				vm_offset_t		*client_base,
-				vm_offset_t		*alternate_base,
-				vm_offset_t		*alternate_next,
-				unsigned int		*fs_base,
-				unsigned int		*system,
-				int			*flags,
-				shared_region_mapping_t	*next);
-
-extern kern_return_t shared_region_mapping_create(
-				mach_port_t		text_region,
-				vm_size_t		text_size,
-				mach_port_t		data_region,
-				vm_size_t		data_size,
-				vm_offset_t		region_mappings,
-				vm_offset_t		client_base,
-				shared_region_mapping_t	*shared_region,
-				vm_offset_t		alt_base,
-				vm_offset_t		alt_next,
-				int			fs_base,
-				int			system);
-
-extern kern_return_t shared_region_mapping_ref(
-				shared_region_mapping_t	shared_region);
-
-extern kern_return_t shared_region_mapping_dealloc(
-				shared_region_mapping_t	shared_region);
-
-extern kern_return_t shared_region_object_chain_attach(
-				shared_region_mapping_t	target_region,
-				shared_region_mapping_t	object_chain);
-
-extern void shared_region_object_chain_detached(
-	shared_region_mapping_t	target_region);
-
-extern kern_return_t vm_get_shared_region(
-				task_t	task,
-				shared_region_mapping_t	*shared_region);
-
-extern kern_return_t vm_set_shared_region(
-				task_t	task,
-				shared_region_mapping_t	shared_region);
-
-extern shared_region_mapping_t update_default_shared_region(
-				shared_region_mapping_t new_system_region);
-
-extern shared_region_mapping_t lookup_default_shared_region(
-				unsigned int fs_base,
-				unsigned int system);
-
-extern void remove_default_shared_region(
-				shared_region_mapping_t system_region);
-
-__private_extern__ void remove_default_shared_region_lock(
-				shared_region_mapping_t system_region,
-				int need_sfh_lock,
-				int need_drl_lock);
-
-__private_extern__ struct load_struct *lsf_remove_regions_mappings_lock(
-				shared_region_mapping_t	region,
-				shared_region_task_mappings_t	sm_info,
-				int need_lock);
-
-extern unsigned int lsf_mapping_pool_gauge(void);
-
-extern kern_return_t shared_file_create_system_region(
-	shared_region_mapping_t	*shared_region,
-	int			fs_base,
-	int			system);
-
-extern void remove_all_shared_regions(void);
-
-extern void shared_file_boot_time_init(
-		unsigned int fs_base, 
-		unsigned int system);
-
-extern struct load_struct *lsf_remove_regions_mappings(
-	shared_region_mapping_t	region,
-	shared_region_task_mappings_t	sm_info);
-
-extern void mach_memory_entry_port_release(ipc_port_t port);
-extern void mach_destroy_memory_entry(ipc_port_t port);
-
-extern kern_return_t mach_memory_entry_purgable_control(
-	ipc_port_t	entry_port,
-	vm_purgable_t	control,
-	int		*state);
-
-extern kern_return_t mach_memory_entry_page_op(
-	ipc_port_t		entry_port,
-	vm_object_offset_t	offset,
-	int			ops,
-	ppnum_t			*phys_entry,
-	int			*flags);
-
-extern kern_return_t mach_memory_entry_range_op(
-	ipc_port_t		entry_port,
-	vm_object_offset_t	offset_beg,
-	vm_object_offset_t	offset_end,
-	int                     ops,
-	int                     *range);
-
-#endif /* KERNEL_PRIVATE */
-
-#endif	/* _VM_SHARED_MEMORY_SERVER_H_ */
diff --git a/osfmk/vm/vm_shared_region.c b/osfmk/vm/vm_shared_region.c
new file mode 100644
index 000000000..50632a9a0
--- /dev/null
+++ b/osfmk/vm/vm_shared_region.c
@@ -0,0 +1,1271 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+
+/*
+ * Shared region (... and comm page)
+ *
+ * This file handles the VM shared region and comm page.
+ *
+ */
+/*
+ * SHARED REGIONS
+ * --------------
+ *
+ * A shared region is a submap that contains the most common system shared
+ * libraries for a given environment.
+ * An environment is defined by (cpu-type, 64-bitness, root directory).
+ *
+ * The point of a shared region is to reduce the setup overhead when exec'ing
+ * a new process.
+ * A shared region uses a shared VM submap that gets mapped automatically
+ * at exec() time (see vm_map_exec()).  The first process of a given
+ * environment sets up the shared region and all further processes in that
+ * environment can re-use that shared region without having to re-create
+ * the same mappings in their VM map.  All they need is contained in the shared
+ * region.
+ * It can also shared a pmap (mostly for read-only parts but also for the
+ * initial version of some writable parts), which gets "nested" into the 
+ * process's pmap.  This reduces the number of soft faults:  once one process
+ * brings in a page in the shared region, all the other processes can access
+ * it without having to enter it in their own pmap.
+ *
+ *
+ * When a process is being exec'ed, vm_map_exec() calls vm_shared_region_enter()
+ * to map the appropriate shared region in the process's address space.
+ * We look up the appropriate shared region for the process's environment.
+ * If we can't find one, we create a new (empty) one and add it to the list.
+ * Otherwise, we just take an extra reference on the shared region we found.
+ *
+ * The "dyld" runtime (mapped into the process's address space at exec() time)
+ * will then use the shared_region_check_np() and shared_region_map_np()
+ * system call to validate and/or populate the shared region with the
+ * appropriate dyld_shared_cache file.
+ *
+ * The shared region is inherited on fork() and the child simply takes an
+ * extra reference on its parent's shared region.
+ *
+ * When the task terminates, we release a reference on its shared region.
+ * When the last reference is released, we destroy the shared region.
+ *
+ * After a chroot(), the calling process keeps using its original shared region,
+ * since that's what was mapped when it was started.  But its children
+ * will use a different shared region, because they need to use the shared
+ * cache that's relative to the new root directory.
+ */
+/*
+ * COMM PAGE
+ *
+ * A "comm page" is an area of memory that is populated by the kernel with
+ * the appropriate platform-specific version of some commonly used code.
+ * There is one "comm page" per platform (cpu-type, 64-bitness) but only
+ * for the native cpu-type.  No need to overly optimize translated code
+ * for hardware that is not really there !
+ *
+ * The comm pages are created and populated at boot time.
+ *
+ * The appropriate comm page is mapped into a process's address space
+ * at exec() time, in vm_map_exec().
+ * It is then inherited on fork().
+ *
+ * The comm page is shared between the kernel and all applications of
+ * a given platform.  Only the kernel can modify it.
+ *
+ * Applications just branch to fixed addresses in the comm page and find
+ * the right version of the code for the platform.  There is also some
+ * data provided and updated by the kernel for processes to retrieve easily
+ * without having to do a system call.
+ */
+
+#include <debug.h>
+
+#include <kern/ipc_tt.h>
+#include <kern/kalloc.h>
+
+#include <vm/vm_map.h>
+#include <vm/vm_shared_region.h>
+
+#include <vm/vm_protos.h>
+
+#include <machine/commpage.h>
+#include <machine/cpu_capabilities.h>
+
+/* "dyld" uses this to figure out what the kernel supports */
+int shared_region_version = 3;
+
+/* should local (non-chroot) shared regions persist when no task uses them ? */
+int shared_region_persistence = 1;	/* yes by default */
+
+/* trace level, output is sent to the system log file */
+int shared_region_trace_level = SHARED_REGION_TRACE_ERROR_LVL;
+
+/* this lock protects all the shared region data structures */
+lck_grp_t *vm_shared_region_lck_grp;
+lck_mtx_t vm_shared_region_lock;
+
+#define vm_shared_region_lock() lck_mtx_lock(&vm_shared_region_lock)
+#define vm_shared_region_unlock() lck_mtx_unlock(&vm_shared_region_lock)
+#define vm_shared_region_sleep(event, interruptible)			\
+	lck_mtx_sleep(&vm_shared_region_lock,				\
+		      LCK_SLEEP_DEFAULT,				\
+		      (event_t) (event),				\
+		      (interruptible))
+
+/* the list of currently available shared regions (one per environment) */
+queue_head_t	vm_shared_region_queue;
+
+static void vm_shared_region_reference_locked(vm_shared_region_t shared_region);
+static vm_shared_region_t vm_shared_region_create(
+	void			*root_dir,
+	cpu_type_t		cputype,
+	boolean_t		is_64bit);
+static void vm_shared_region_destroy(vm_shared_region_t shared_region);
+
+/*
+ * Initialize the module...
+ */
+void
+vm_shared_region_init(void)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> init\n"));
+
+	vm_shared_region_lck_grp = lck_grp_alloc_init("vm shared region",
+						      LCK_GRP_ATTR_NULL);
+	lck_mtx_init(&vm_shared_region_lock,
+		     vm_shared_region_lck_grp,
+		     LCK_ATTR_NULL);
+
+	queue_init(&vm_shared_region_queue);
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: <- init\n"));
+}
+
+/*
+ * Retrieve a task's shared region and grab an extra reference to 
+ * make sure it doesn't disappear while the caller is using it.	
+ * The caller is responsible for consuming that extra reference if
+ * necessary.
+ */
+vm_shared_region_t
+vm_shared_region_get(
+	task_t		task)
+{
+	vm_shared_region_t	shared_region;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> get(%p)\n",
+		 task));
+
+	task_lock(task);
+	vm_shared_region_lock();
+	shared_region = task->shared_region;
+	if (shared_region) {
+		assert(shared_region->sr_ref_count > 0);
+		vm_shared_region_reference_locked(shared_region);
+	}
+	vm_shared_region_unlock();
+	task_unlock(task);
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: get(%p) <- %p\n",
+		 task, shared_region));
+
+	return shared_region;
+}
+
+/*
+ * Get the base address of the shared region.
+ * That's the address at which it needs to be mapped in the process's address
+ * space.
+ * No need to lock since this data is set when the shared region is
+ * created and is never modified after that.  The caller must hold an extra
+ * reference on the shared region to prevent it from being destroyed.
+ */
+mach_vm_offset_t
+vm_shared_region_base_address(
+	vm_shared_region_t	shared_region)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> base_address(%p)\n",
+		 shared_region));
+	assert(shared_region->sr_ref_count > 1);
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: base_address(%p) <- 0x%llx\n",
+		 shared_region, (long long)shared_region->sr_base_address));
+	return shared_region->sr_base_address;
+}
+
+/*
+ * Get the size of the shared region.
+ * That's the size that needs to be mapped in the process's address
+ * space.
+ * No need to lock since this data is set when the shared region is
+ * created and is never modified after that.  The caller must hold an extra
+ * reference on the shared region to prevent it from being destroyed.
+ */
+mach_vm_size_t
+vm_shared_region_size(
+	vm_shared_region_t	shared_region)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> size(%p)\n",
+		 shared_region));
+	assert(shared_region->sr_ref_count > 1);
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: size(%p) <- 0x%llx\n",
+		 shared_region, (long long)shared_region->sr_size));
+	return shared_region->sr_size;
+}
+
+/*
+ * Get the memory entry of the shared region.
+ * That's the "memory object" that needs to be mapped in the process's address
+ * space.
+ * No need to lock since this data is set when the shared region is
+ * created and is never modified after that.  The caller must hold an extra
+ * reference on the shared region to prevent it from being destroyed.
+ */
+ipc_port_t
+vm_shared_region_mem_entry(
+	vm_shared_region_t	shared_region)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> mem_entry(%p)\n",
+		 shared_region));
+	assert(shared_region->sr_ref_count > 1);
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: mem_entry(%p) <- %p\n",
+		 shared_region, shared_region->sr_mem_entry));
+	return shared_region->sr_mem_entry;
+}
+
+/*
+ * Set the shared region the process should use.
+ * A NULL new shared region means that we just want to release the old
+ * shared region.
+ * The caller should already have an extra reference on the new shared region
+ * (if any).  We release a reference on the old shared region (if any).
+ */
+void
+vm_shared_region_set(
+	task_t			task,
+	vm_shared_region_t	new_shared_region)
+{
+	vm_shared_region_t	old_shared_region;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> set(%p, %p)\n",
+		 task, new_shared_region));
+
+	task_lock(task);
+	vm_shared_region_lock();
+
+	old_shared_region = task->shared_region;
+	if (new_shared_region) {
+		assert(new_shared_region->sr_ref_count > 0);
+	}
+
+	task->shared_region = new_shared_region;
+
+	vm_shared_region_unlock();
+	task_unlock(task);
+
+	if (old_shared_region) {
+		assert(old_shared_region->sr_ref_count > 0);
+		vm_shared_region_deallocate(old_shared_region);
+	}
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: set(%p) <- old=%p new=%p\n",
+		 task, old_shared_region, new_shared_region));
+}
+
+/*
+ * Lookup up the shared region for the desired environment.
+ * If none is found, create a new (empty) one.
+ * Grab an extra reference on the returned shared region, to make sure
+ * it doesn't get destroyed before the caller is done with it.  The caller
+ * is responsible for consuming that extra reference if necessary.
+ */
+vm_shared_region_t
+vm_shared_region_lookup(
+	void		*root_dir,
+	cpu_type_t	cputype,
+	boolean_t	is_64bit)
+{
+	vm_shared_region_t	shared_region;
+	vm_shared_region_t	new_shared_region;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> lookup(root=%p,cpu=%d,64bit=%d)\n",
+		 root_dir, cputype, is_64bit));
+
+	shared_region = NULL;
+	new_shared_region = NULL;
+
+	vm_shared_region_lock();
+	for (;;) {
+		queue_iterate(&vm_shared_region_queue,
+			      shared_region,
+			      vm_shared_region_t,
+			      sr_q) {
+			assert(shared_region->sr_ref_count > 0);
+			if (shared_region->sr_cpu_type == cputype &&
+			    shared_region->sr_root_dir == root_dir &&
+			    shared_region->sr_64bit == is_64bit) {
+				/* found a match ! */
+				vm_shared_region_reference_locked(shared_region);
+				goto done;
+			}
+		}
+		if (new_shared_region == NULL) {
+			/* no match: create a new one */
+			vm_shared_region_unlock();
+			new_shared_region = vm_shared_region_create(root_dir,
+								    cputype,
+								    is_64bit);
+			/* do the lookup again, in case we lost a race */
+			vm_shared_region_lock();
+			continue;
+		}
+		/* still no match: use our new one */
+		shared_region = new_shared_region;
+		new_shared_region = NULL;
+		queue_enter(&vm_shared_region_queue,
+			    shared_region,
+			    vm_shared_region_t,
+			    sr_q);
+		break;
+	}
+
+done:
+	vm_shared_region_unlock();
+
+	if (new_shared_region) {
+		/*
+		 * We lost a race with someone else to create a new shared
+		 * region for that environment.  Get rid of our unused one.
+		 */
+		assert(new_shared_region->sr_ref_count == 1);
+		new_shared_region->sr_ref_count--;
+		vm_shared_region_destroy(new_shared_region);
+		new_shared_region = NULL;
+	}
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: lookup(root=%p,cpu=%d,64bit=%d) <- %p\n",
+		 root_dir, cputype, is_64bit, shared_region));
+
+	assert(shared_region->sr_ref_count > 0);
+	return shared_region;
+}
+
+/*
+ * Take an extra reference on a shared region.
+ * The vm_shared_region_lock should already be held by the caller.
+ */
+static void
+vm_shared_region_reference_locked(
+	vm_shared_region_t	shared_region)
+{
+#if DEBUG
+	lck_mtx_assert(&vm_shared_region_lock, LCK_MTX_ASSERT_OWNED);
+#endif
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> reference_locked(%p)\n",
+		 shared_region));
+	assert(shared_region->sr_ref_count > 0);
+	shared_region->sr_ref_count++;
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: reference_locked(%p) <- %d\n",
+		 shared_region, shared_region->sr_ref_count));
+}
+
+/*
+ * Release a reference on the shared region.
+ * Destroy it if there are no references left.
+ */
+void
+vm_shared_region_deallocate(
+	vm_shared_region_t	shared_region)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> deallocate(%p)\n",
+		 shared_region));
+
+	vm_shared_region_lock();
+	
+	assert(shared_region->sr_ref_count > 0);
+
+	if (shared_region->sr_root_dir == NULL) {
+		/*
+		 * Local (i.e. based on the boot volume) shared regions
+		 * can persist or not based on the "shared_region_persistence"
+		 * sysctl.
+		 * Make sure that this one complies.
+		 */
+		if (shared_region_persistence &&
+		    !shared_region->sr_persists) {
+			/* make this one persistent */
+			shared_region->sr_ref_count++;
+			shared_region->sr_persists = TRUE;
+		} else if (!shared_region_persistence &&
+			   shared_region->sr_persists) {
+			/* make this one no longer persistent */
+			assert(shared_region->sr_ref_count > 1);
+			shared_region->sr_ref_count--;
+			shared_region->sr_persists = FALSE;
+		}
+	}
+
+	assert(shared_region->sr_ref_count > 0);
+	shared_region->sr_ref_count--;
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: deallocate(%p): ref now %d\n",
+		 shared_region, shared_region->sr_ref_count));
+
+	if (shared_region->sr_ref_count == 0) {
+		assert(! shared_region->sr_mapping_in_progress);
+		/* remove it from the queue first, so no one can find it... */
+		queue_remove(&vm_shared_region_queue,
+			     shared_region,
+			     vm_shared_region_t,
+			     sr_q);
+		vm_shared_region_unlock();
+		/* ... and destroy it */
+		vm_shared_region_destroy(shared_region);
+		shared_region = NULL;
+	} else {
+		vm_shared_region_unlock();
+	}
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: deallocate(%p) <-\n",
+		 shared_region));
+}
+
+/*
+ * Create a new (empty) shared region for a new environment.
+ */
+static vm_shared_region_t
+vm_shared_region_create(
+	void			*root_dir,
+	cpu_type_t		cputype,
+	boolean_t		is_64bit)
+{
+	kern_return_t		kr;
+	vm_named_entry_t	mem_entry;
+	ipc_port_t		mem_entry_port;
+	vm_shared_region_t	shared_region;
+	vm_map_t		sub_map;
+	mach_vm_offset_t	base_address, pmap_nesting_start;
+	mach_vm_size_t		size, pmap_nesting_size;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> create(root=%p,cpu=%d,64bit=%d)\n",
+		 root_dir, cputype, is_64bit));
+
+	base_address = 0;
+	size = 0;
+	mem_entry = NULL;
+	mem_entry_port = IPC_PORT_NULL;
+	sub_map = VM_MAP_NULL;
+
+	/* create a new shared region structure... */
+	shared_region = kalloc(sizeof (*shared_region));
+	if (shared_region == NULL) {
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: create: couldn't allocate\n"));
+		goto done;
+	}
+
+	/* figure out the correct settings for the desired environment */
+	if (is_64bit) {
+		switch (cputype) {
+		case CPU_TYPE_I386:
+			base_address = SHARED_REGION_BASE_X86_64;
+			size = SHARED_REGION_SIZE_X86_64;
+			pmap_nesting_start = SHARED_REGION_NESTING_BASE_X86_64;
+			pmap_nesting_size = SHARED_REGION_NESTING_SIZE_X86_64;
+			break;
+		case CPU_TYPE_POWERPC:
+			base_address = SHARED_REGION_BASE_PPC64;
+			size = SHARED_REGION_SIZE_PPC64;
+			pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC64;
+			pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC64;
+			break;
+		default:
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: create: unknown cpu type %d\n",
+				 cputype));
+			kfree(shared_region, sizeof (*shared_region));
+			shared_region = NULL;
+			goto done;
+		}
+	} else {
+		switch (cputype) {
+		case CPU_TYPE_I386:
+			base_address = SHARED_REGION_BASE_I386;
+			size = SHARED_REGION_SIZE_I386;
+			pmap_nesting_start = SHARED_REGION_NESTING_BASE_I386;
+			pmap_nesting_size = SHARED_REGION_NESTING_SIZE_I386;
+			break;
+		case CPU_TYPE_POWERPC:
+			base_address = SHARED_REGION_BASE_PPC;
+			size = SHARED_REGION_SIZE_PPC;
+			pmap_nesting_start = SHARED_REGION_NESTING_BASE_PPC;
+			pmap_nesting_size = SHARED_REGION_NESTING_SIZE_PPC;
+			break;
+#ifdef CPU_TYPE_ARM
+		case CPU_TYPE_ARM:
+			base_address = SHARED_REGION_BASE_ARM;
+			size = SHARED_REGION_SIZE_ARM;
+			pmap_nesting_start = SHARED_REGION_NESTING_BASE_ARM;
+			pmap_nesting_size = SHARED_REGION_NESTING_SIZE_ARM;
+			break;
+#endif /* CPU_TYPE_ARM */
+		default:
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: create: unknown cpu type %d\n",
+				 cputype));
+			kfree(shared_region, sizeof (*shared_region));
+			shared_region = NULL;
+			goto done;
+			
+		}
+	}
+
+	/* create a memory entry structure and a Mach port handle */
+	kr = mach_memory_entry_allocate(&mem_entry,
+					&mem_entry_port);
+	if (kr != KERN_SUCCESS) {
+		kfree(shared_region, sizeof (*shared_region));
+		shared_region = NULL;
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: create: "
+			 "couldn't allocate mem_entry\n"));
+		goto done;
+	}
+
+	/* create a VM sub map and its pmap */
+	sub_map = vm_map_create(pmap_create(0, is_64bit),
+				0, size,
+				TRUE);
+	if (sub_map == VM_MAP_NULL) {
+		ipc_port_release_send(mem_entry_port);
+		kfree(shared_region, sizeof (*shared_region));
+		shared_region = NULL;
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: create: "
+			 "couldn't allocate map\n"));
+		goto done;
+	}
+
+	/* make the memory entry point to the VM sub map */
+	mem_entry->is_sub_map = TRUE;
+	mem_entry->backing.map = sub_map;
+	mem_entry->size = size;
+	mem_entry->protection = VM_PROT_ALL;
+
+	/* make the shared region point at the memory entry */
+	shared_region->sr_mem_entry = mem_entry_port;
+
+	/* fill in the shared region's environment and settings */
+	shared_region->sr_base_address = base_address;
+	shared_region->sr_size = size;
+	shared_region->sr_pmap_nesting_start = pmap_nesting_start;
+	shared_region->sr_pmap_nesting_size = pmap_nesting_size;
+	shared_region->sr_cpu_type = cputype;
+	shared_region->sr_64bit = is_64bit;
+	shared_region->sr_root_dir = root_dir;
+
+	queue_init(&shared_region->sr_q);
+	shared_region->sr_mapping_in_progress = FALSE;
+	shared_region->sr_persists = FALSE;
+	shared_region->sr_first_mapping = (mach_vm_offset_t) -1;
+
+	/* grab a reference for the caller */
+	shared_region->sr_ref_count = 1;
+
+done:
+	if (shared_region) {
+		SHARED_REGION_TRACE_INFO(
+			("shared_region: create(root=%p,cpu=%d,64bit=%d,"
+			 "base=0x%llx,size=0x%llx) <- "
+			 "%p mem=(%p,%p) map=%p pmap=%p\n",
+			 root_dir, cputype, is_64bit, (long long)base_address,
+			 (long long)size, shared_region,
+			 mem_entry_port, mem_entry, sub_map, sub_map->pmap));
+	} else {
+		SHARED_REGION_TRACE_INFO(
+			("shared_region: create(root=%p,cpu=%d,64bit=%d,"
+			 "base=0x%llx,size=0x%llx) <- NULL",
+			 root_dir, cputype, is_64bit, (long long)base_address,
+			 (long long)size));
+	}
+	return shared_region;
+}
+
+/*
+ * Destroy a now-unused shared region.
+ * The shared region is no longer in the queue and can not be looked up.
+ */
+static void
+vm_shared_region_destroy(
+	vm_shared_region_t	shared_region)
+{
+	vm_named_entry_t	mem_entry;
+	vm_map_t		map;
+
+	SHARED_REGION_TRACE_INFO(
+		("shared_region: -> destroy(%p) (root=%p,cpu=%d,64bit=%d)\n",
+		 shared_region,
+		 shared_region->sr_root_dir,
+		 shared_region->sr_cpu_type,
+		 shared_region->sr_64bit));
+
+	assert(shared_region->sr_ref_count == 0);
+	assert(!shared_region->sr_persists);
+
+	mem_entry = (vm_named_entry_t) shared_region->sr_mem_entry->ip_kobject;
+	assert(mem_entry->is_sub_map);
+	assert(!mem_entry->internal);
+	assert(!mem_entry->is_pager);
+	map = mem_entry->backing.map;
+
+	/*
+	 * Clean up the pmap first.  The virtual addresses that were
+	 * entered in this possibly "nested" pmap may have different values
+	 * than the VM map's min and max offsets, if the VM sub map was
+	 * mapped at a non-zero offset in the processes' main VM maps, which
+	 * is usually the case, so the clean-up we do in vm_map_destroy() would
+	 * not be enough.
+	 */
+	if (map->pmap) {
+		pmap_remove(map->pmap,
+			    shared_region->sr_base_address,
+			    (shared_region->sr_base_address +
+			     shared_region->sr_size));
+	}
+
+	/*
+	 * Release our (one and only) handle on the memory entry.
+	 * This will generate a no-senders notification, which will be processed
+	 * by ipc_kobject_notify(), which will release the one and only
+	 * reference on the memory entry and cause it to be destroyed, along
+	 * with the VM sub map and its pmap.
+	 */
+	mach_memory_entry_port_release(shared_region->sr_mem_entry);
+	mem_entry = NULL;
+	shared_region->sr_mem_entry = IPC_PORT_NULL;
+
+	/* release the shared region structure... */
+	kfree(shared_region, sizeof (*shared_region));
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: destroy(%p) <-\n",
+		 shared_region));
+	shared_region = NULL;
+
+}
+
+/*
+ * Gets the address of the first (in time) mapping in the shared region.
+ */
+kern_return_t
+vm_shared_region_start_address(
+	vm_shared_region_t	shared_region,
+	mach_vm_offset_t	*start_address)
+{
+	kern_return_t		kr;
+	mach_vm_offset_t	sr_base_address;
+	mach_vm_offset_t	sr_first_mapping;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> start_address(%p)\n",
+		 shared_region));
+	assert(shared_region->sr_ref_count > 1);
+
+	vm_shared_region_lock();
+
+	/*
+	 * Wait if there's another thread establishing a mapping
+	 * in this shared region right when we're looking at it.
+	 * We want a consistent view of the map...
+	 */
+	while (shared_region->sr_mapping_in_progress) {
+		/* wait for our turn... */
+		assert(shared_region->sr_ref_count > 1);
+		vm_shared_region_sleep(&shared_region->sr_mapping_in_progress,
+				       THREAD_UNINT);
+	}
+	assert(! shared_region->sr_mapping_in_progress);
+	assert(shared_region->sr_ref_count > 1);
+	
+	sr_base_address = shared_region->sr_base_address;
+	sr_first_mapping = shared_region->sr_first_mapping;
+
+	if (sr_first_mapping == (mach_vm_offset_t) -1) {
+		/* shared region is empty */
+		kr = KERN_INVALID_ADDRESS;
+	} else {
+		kr = KERN_SUCCESS;
+		*start_address = sr_base_address + sr_first_mapping;
+	}
+
+	vm_shared_region_unlock();
+	
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: start_address(%p) <- 0x%llx\n",
+		 shared_region, (long long)shared_region->sr_base_address));
+
+	return kr;
+}
+/*
+ * Establish some mappings of a file in the shared region.
+ * This is used by "dyld" via the shared_region_map_np() system call
+ * to populate the shared region with the appropriate shared cache.
+ *
+ * One could also call it several times to incrementally load several
+ * libraries, as long as they do not overlap.  
+ * It will return KERN_SUCCESS if the mappings were successfully established
+ * or if they were already established identically by another process.
+ */
+kern_return_t
+vm_shared_region_map_file(
+	vm_shared_region_t		shared_region,
+	unsigned int			mappings_count,
+	struct shared_file_mapping_np	*mappings,
+	memory_object_control_t		file_control,
+	memory_object_size_t		file_size,
+	void				*root_dir)
+{
+	kern_return_t		kr;
+	vm_object_t		file_object;
+	ipc_port_t		sr_handle;
+	vm_named_entry_t	sr_mem_entry;
+	vm_map_t		sr_map;
+	mach_vm_offset_t	sr_base_address;
+	unsigned int		i;
+	mach_port_t		map_port;
+	mach_vm_offset_t	target_address;
+
+	kr = KERN_SUCCESS;
+
+	vm_shared_region_lock();
+	assert(shared_region->sr_ref_count > 1);
+
+	if (shared_region->sr_root_dir != root_dir) {
+		/*
+		 * This shared region doesn't match the current root
+		 * directory of this process.  Deny the mapping to
+		 * avoid tainting the shared region with something that	
+		 * doesn't quite belong into it.
+		 */
+		vm_shared_region_unlock();
+		kr = KERN_PROTECTION_FAILURE;
+		goto done;
+	}
+
+	/*
+	 * Make sure we handle only one mapping at a time in a given
+	 * shared region, to avoid race conditions.  This should not
+	 * happen frequently...
+	 */
+	while (shared_region->sr_mapping_in_progress) {
+		/* wait for our turn... */
+		vm_shared_region_sleep(&shared_region->sr_mapping_in_progress,
+				       THREAD_UNINT);
+	}
+	assert(! shared_region->sr_mapping_in_progress);
+	assert(shared_region->sr_ref_count > 1);
+	/* let others know we're working in this shared region */
+	shared_region->sr_mapping_in_progress = TRUE;
+
+	vm_shared_region_unlock();
+
+	/* no need to lock because this data is never modified... */
+	sr_handle = shared_region->sr_mem_entry;
+	sr_mem_entry = (vm_named_entry_t) sr_handle->ip_kobject;
+	sr_map = sr_mem_entry->backing.map;
+	sr_base_address = shared_region->sr_base_address;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> map(%p,%d,%p,%p,0x%llx)\n",
+		 shared_region, mappings_count, mappings,
+		 file_control, file_size));
+
+	/* get the VM object associated with the file to be mapped */
+	file_object = memory_object_control_to_vm_object(file_control);
+
+	/* establish the mappings */
+	for (i = 0; i < mappings_count; i++) {
+		SHARED_REGION_TRACE_INFO(
+			("shared_region: mapping[%d]: "
+			 "address:0x%016llx size:0x%016llx offset:0x%016llx "
+			 "maxprot:0x%x prot:0x%x\n",
+			 i,
+			 (long long)mappings[i].sfm_address,
+			 (long long)mappings[i].sfm_size,
+			 (long long)mappings[i].sfm_file_offset,
+			 mappings[i].sfm_max_prot,
+			 mappings[i].sfm_init_prot));
+
+		if (mappings[i].sfm_init_prot & VM_PROT_ZF) {
+			/* zero-filled memory */
+			map_port = MACH_PORT_NULL;
+		} else {
+			/* file-backed memory */
+			map_port = (ipc_port_t) file_object->pager;
+		}
+
+		/* mapping's address is relative to the shared region base */
+		target_address =
+			mappings[i].sfm_address - sr_base_address;
+
+		/* establish that mapping, OK if it's to "already" there */
+		kr = vm_map_enter_mem_object(
+			sr_map,
+			&target_address,
+			vm_map_round_page(mappings[i].sfm_size),
+			0,
+			VM_FLAGS_FIXED | VM_FLAGS_ALREADY,
+			map_port,
+			mappings[i].sfm_file_offset,
+			TRUE,
+			mappings[i].sfm_init_prot & VM_PROT_ALL,
+			mappings[i].sfm_max_prot & VM_PROT_ALL,
+			VM_INHERIT_DEFAULT);
+		if (kr == KERN_MEMORY_PRESENT) {
+			/* this exact mapping was already there: that's fine */
+			SHARED_REGION_TRACE_INFO(
+				("shared_region: mapping[%d]: "
+				 "address:0x%016llx size:0x%016llx "
+				 "offset:0x%016llx "
+				 "maxprot:0x%x prot:0x%x already mapped...\n",
+				 i,
+				 (long long)mappings[i].sfm_address,
+				 (long long)mappings[i].sfm_size,
+				 (long long)mappings[i].sfm_file_offset,
+				 mappings[i].sfm_max_prot,
+				 mappings[i].sfm_init_prot));
+			kr = KERN_SUCCESS;
+		} else if (kr != KERN_SUCCESS) {
+			/* this mapping failed ! */
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: mapping[%d]: "
+				 "address:0x%016llx size:0x%016llx "
+				 "offset:0x%016llx "
+				 "maxprot:0x%x prot:0x%x failed 0x%x\n",
+				 i,
+				 (long long)mappings[i].sfm_address,
+				 (long long)mappings[i].sfm_size,
+				 (long long)mappings[i].sfm_file_offset,
+				 mappings[i].sfm_max_prot,
+				 mappings[i].sfm_init_prot,
+				 kr));
+			break;
+		}
+
+		/* we're protected by "sr_mapping_in_progress" */
+		if (shared_region->sr_first_mapping == (mach_vm_offset_t) -1) {
+			shared_region->sr_first_mapping = target_address;
+		}
+	}
+
+	vm_shared_region_lock();
+	assert(shared_region->sr_ref_count > 1);
+	assert(shared_region->sr_mapping_in_progress);
+	/* we're done working on that shared region */
+	shared_region->sr_mapping_in_progress = FALSE;
+	thread_wakeup((event_t) &shared_region->sr_mapping_in_progress);
+	vm_shared_region_unlock();
+
+done:
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: map(%p,%d,%p,%p,0x%llx) <- 0x%x \n",
+		 shared_region, mappings_count, mappings,
+		 file_control, file_size, kr));
+	return kr;
+}
+
+/*
+ * Enter the appropriate shared region into "map" for "task".
+ * This involves looking up the shared region (and possibly creating a new
+ * one) for the desired environment, then mapping the VM sub map into the
+ * task's VM "map", with the appropriate level of pmap-nesting.
+ */
+kern_return_t
+vm_shared_region_enter(
+	struct _vm_map		*map,
+	struct task		*task,
+	void			*fsroot,
+	cpu_type_t		cpu)
+{
+	kern_return_t		kr;
+	vm_shared_region_t	shared_region;
+	vm_map_offset_t		sr_address, sr_offset, target_address;
+	vm_map_size_t		sr_size, mapping_size;
+	vm_map_offset_t		sr_pmap_nesting_start;
+	vm_map_size_t		sr_pmap_nesting_size;
+	ipc_port_t		sr_handle;
+	boolean_t		is_64bit;
+
+	is_64bit = task_has_64BitAddr(task);
+
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: -> "
+		 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d)\n",
+		 map, task, fsroot, cpu, is_64bit));
+
+	/* lookup (create if needed) the shared region for this environment */
+	shared_region = vm_shared_region_lookup(fsroot, cpu, is_64bit);
+	if (shared_region == NULL) {
+		/* this should not happen ! */
+		SHARED_REGION_TRACE_ERROR(
+			("shared_region: -> "
+			 "enter(map=%p,task=%p,root=%p,cpu=%d,64bit=%d): "
+			 "lookup failed !\n",
+			 map, task, fsroot, cpu, is_64bit));
+		//panic("shared_region_enter: lookup failed\n");
+		return KERN_FAILURE;
+	}
+	
+	/* let the task use that shared region */
+	vm_shared_region_set(task, shared_region);
+
+	kr = KERN_SUCCESS;
+	/* no need to lock since this data is never modified */
+	sr_address = shared_region->sr_base_address;
+	sr_size = shared_region->sr_size;
+	sr_handle = shared_region->sr_mem_entry;
+	sr_pmap_nesting_start = shared_region->sr_pmap_nesting_start;
+	sr_pmap_nesting_size = shared_region->sr_pmap_nesting_size;
+
+	/*
+	 * Start mapping the shared region's VM sub map into the task's VM map.
+	 */
+	sr_offset = 0;
+
+	if (sr_pmap_nesting_start > sr_address) {
+		/* we need to map a range without pmap-nesting first */
+		target_address = sr_address;
+		mapping_size = sr_pmap_nesting_start - sr_address;
+		kr = vm_map_enter_mem_object(
+			map,
+			&target_address,
+			mapping_size,
+			0,
+			VM_FLAGS_FIXED,
+			sr_handle,
+			sr_offset,
+			TRUE,
+			VM_PROT_READ,
+			VM_PROT_ALL,
+			VM_INHERIT_SHARE);
+		if (kr != KERN_SUCCESS) {
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: enter(%p,%p,%p,%d,%d): "
+				 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+				 map, task, fsroot, cpu, is_64bit,
+				 (long long)target_address,
+				 (long long)mapping_size, sr_handle, kr));
+			goto done;
+		}
+		SHARED_REGION_TRACE_DEBUG(
+			("shared_region: enter(%p,%p,%p,%d,%d): "
+			 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+			 map, task, fsroot, cpu, is_64bit,
+			 (long long)target_address, (long long)mapping_size,
+			 sr_handle, kr));
+		sr_offset += mapping_size;
+		sr_size -= mapping_size;
+	}
+	/*
+	 * We may need to map several pmap-nested portions, due to platform
+	 * specific restrictions on pmap nesting.
+	 * The pmap-nesting is triggered by the "VM_MEMORY_SHARED_PMAP" alias...
+	 */
+	for (;
+	     sr_pmap_nesting_size > 0;
+	     sr_offset += mapping_size,
+		     sr_size -= mapping_size,
+		     sr_pmap_nesting_size -= mapping_size) {
+		target_address = sr_address + sr_offset;
+		mapping_size = sr_pmap_nesting_size;
+		if (mapping_size > pmap_nesting_size_max) {
+			mapping_size = (vm_map_offset_t) pmap_nesting_size_max;
+		}
+		kr = vm_map_enter_mem_object(
+			map,
+			&target_address,
+			mapping_size,
+			0,
+			(VM_FLAGS_FIXED | VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP)),
+			sr_handle,
+			sr_offset,
+			TRUE,
+			VM_PROT_READ,
+			VM_PROT_ALL,
+			VM_INHERIT_SHARE);
+		if (kr != KERN_SUCCESS) {
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: enter(%p,%p,%p,%d,%d): "
+				 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+				 map, task, fsroot, cpu, is_64bit,
+				 (long long)target_address,
+				 (long long)mapping_size, sr_handle, kr));
+			goto done;
+		}
+		SHARED_REGION_TRACE_DEBUG(
+			("shared_region: enter(%p,%p,%p,%d,%d): "
+			 "nested vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+			 map, task, fsroot, cpu, is_64bit,
+			 (long long)target_address, (long long)mapping_size,
+			 sr_handle, kr));
+	}
+	if (sr_size > 0) {
+		/* and there's some left to be mapped without pmap-nesting */
+		target_address = sr_address + sr_offset;
+		mapping_size = sr_size;
+		kr = vm_map_enter_mem_object(
+			map,
+			&target_address,
+			mapping_size,
+			0,
+			VM_FLAGS_FIXED,
+			sr_handle,
+			sr_offset,
+			TRUE,
+			VM_PROT_READ,
+			VM_PROT_ALL,
+			VM_INHERIT_SHARE);
+		if (kr != KERN_SUCCESS) {
+			SHARED_REGION_TRACE_ERROR(
+				("shared_region: enter(%p,%p,%p,%d,%d): "
+				 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+				 map, task, fsroot, cpu, is_64bit,
+				 (long long)target_address,
+				 (long long)mapping_size, sr_handle, kr));
+			goto done;
+		}
+		SHARED_REGION_TRACE_DEBUG(
+			("shared_region: enter(%p,%p,%p,%d,%d): "
+			 "vm_map_enter(0x%llx,0x%llx,%p) error 0x%x\n",
+			 map, task, fsroot, cpu, is_64bit,
+			 (long long)target_address, (long long)mapping_size,
+			 sr_handle, kr));
+		sr_offset += mapping_size;
+		sr_size -= mapping_size;
+	}
+	assert(sr_size == 0);
+
+done:
+	SHARED_REGION_TRACE_DEBUG(
+		("shared_region: enter(%p,%p,%p,%d,%d) <- 0x%x\n",
+		 map, task, fsroot, cpu, is_64bit, kr));
+	return kr;
+}
+
+/******************************************************************************/
+/* Comm page support                                                          */
+/******************************************************************************/
+
+ipc_port_t commpage32_handle = IPC_PORT_NULL;
+ipc_port_t commpage64_handle = IPC_PORT_NULL;
+vm_named_entry_t commpage32_entry = NULL;
+vm_named_entry_t commpage64_entry = NULL;
+vm_map_t commpage32_map = VM_MAP_NULL;
+vm_map_t commpage64_map = VM_MAP_NULL;
+
+/*
+ * Create a memory entry, VM submap and pmap for one commpage.
+ */
+static void
+_vm_commpage_init(
+	ipc_port_t	*handlep,
+	vm_map_size_t	size)
+{
+	kern_return_t		kr;
+	vm_named_entry_t	mem_entry;
+	vm_map_t		new_map;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: -> _init(0x%llx)\n",
+		 (long long)size));
+
+	kr = mach_memory_entry_allocate(&mem_entry,
+					handlep);
+	if (kr != KERN_SUCCESS) {
+		panic("_vm_commpage_init: could not allocate mem_entry");
+	}
+	new_map = vm_map_create(pmap_create(0, FALSE), 0, size, TRUE);
+	if (new_map == VM_MAP_NULL) {
+		panic("_vm_commpage_init: could not allocate VM map");
+	}
+	mem_entry->backing.map = new_map;
+	mem_entry->internal = TRUE;
+	mem_entry->is_sub_map = TRUE;
+	mem_entry->offset = 0;
+	mem_entry->protection = VM_PROT_ALL;
+	mem_entry->size = size;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: _init(0x%llx) <- %p\n",
+		 (long long)size, *handlep));
+}
+
+/*
+ * Initialize the comm pages at boot time.
+ */
+void
+vm_commpage_init(void)
+{
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: -> init()\n"));
+
+	/* create the 32-bit comm page */
+	_vm_commpage_init(&commpage32_handle, _COMM_PAGE32_AREA_LENGTH);
+	commpage32_entry = (vm_named_entry_t) commpage32_handle->ip_kobject;
+	commpage32_map = commpage32_entry->backing.map;
+
+	/* XXX if (cpu_is_64bit_capable()) ? */
+	/* create the 64-bit comm page */
+	_vm_commpage_init(&commpage64_handle, _COMM_PAGE64_AREA_LENGTH);
+	commpage64_entry = (vm_named_entry_t) commpage64_handle->ip_kobject;
+	commpage64_map = commpage64_entry->backing.map;
+
+	/* populate them according to this specific platform */
+	commpage_populate();
+
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: init() <-\n"));
+}
+
+/*
+ * Enter the appropriate comm page into the task's address space.
+ * This is called at exec() time via vm_map_exec().
+ */
+kern_return_t
+vm_commpage_enter(
+	vm_map_t	map,
+	task_t		task)
+{
+	ipc_port_t		commpage_handle;
+	vm_map_offset_t		commpage_address, objc_address;
+	vm_map_size_t		commpage_size, objc_size;
+	int			vm_flags;
+	kern_return_t		kr;
+
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: -> enter(%p,%p)\n",
+		 map, task));
+
+	/* the comm page is likely to be beyond the actual end of the VM map */
+	vm_flags = VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX;
+
+	/* select the appropriate comm page for this task */
+	assert(! (task_has_64BitAddr(task) ^ vm_map_is_64bit(map)));
+	if (task_has_64BitAddr(task)) {
+#ifdef __ppc__
+		/*
+		 * PPC51: ppc64 is limited to 51-bit addresses.
+		 * Memory above that limit is handled specially at the
+		 * pmap level, so do not interfere.
+		 */
+		vm_flags |= VM_FLAGS_NO_PMAP_CHECK;
+#endif /* __ppc__ */
+		commpage_handle = commpage64_handle;
+		commpage_address = (vm_map_offset_t) _COMM_PAGE64_BASE_ADDRESS;
+		commpage_size = _COMM_PAGE64_AREA_LENGTH;
+		objc_size = _COMM_PAGE64_OBJC_SIZE;
+		objc_address = _COMM_PAGE64_OBJC_BASE;
+	} else {
+		commpage_handle = commpage32_handle;
+		commpage_address =
+			(vm_map_offset_t)(unsigned) _COMM_PAGE32_BASE_ADDRESS;
+		commpage_size = _COMM_PAGE32_AREA_LENGTH;
+		objc_size = _COMM_PAGE32_OBJC_SIZE;
+		objc_address = _COMM_PAGE32_OBJC_BASE;
+	}
+
+	if ((commpage_address & (pmap_nesting_size_min - 1)) == 0 &&
+	    (commpage_size & (pmap_nesting_size_min - 1)) == 0) {
+		/* the commpage is properly aligned or sized for pmap-nesting */
+		vm_flags |= VM_MAKE_TAG(VM_MEMORY_SHARED_PMAP);
+	}
+
+	/* map the comm page in the task's address space */
+	assert(commpage_handle != IPC_PORT_NULL);
+	kr = vm_map_enter_mem_object(
+		map,
+		&commpage_address,
+		commpage_size,
+		0,
+		vm_flags,
+		commpage_handle,
+		0,
+		FALSE,
+		VM_PROT_READ|VM_PROT_EXECUTE,
+		VM_PROT_READ|VM_PROT_EXECUTE,
+		VM_INHERIT_SHARE);
+	if (kr != KERN_SUCCESS) {
+		SHARED_REGION_TRACE_ERROR(
+			("commpage: enter(%p,0x%llx,0x%llx) "
+			 "commpage %p mapping failed 0x%x\n",
+			 map, (long long)commpage_address,
+			 (long long)commpage_size, commpage_handle, kr));
+	}
+
+	/*
+	 * Since we're here, we also pre-allocate some virtual space for the
+	 * Objective-C run-time, if needed...
+	 */
+	if (objc_size != 0) {
+		kr = vm_map_enter_mem_object(
+			map,
+			&objc_address,
+			objc_size,
+			0,
+			VM_FLAGS_FIXED | VM_FLAGS_BEYOND_MAX,
+			IPC_PORT_NULL,
+			0,
+			FALSE,
+			VM_PROT_ALL,
+			VM_PROT_ALL,
+			VM_INHERIT_DEFAULT);
+		if (kr != KERN_SUCCESS) {
+			SHARED_REGION_TRACE_ERROR(
+				("commpage: enter(%p,0x%llx,0x%llx) "
+				 "objc mapping failed 0x%x\n",
+				 map, (long long)objc_address,
+				 (long long)objc_size, kr));
+		}
+	}
+
+	SHARED_REGION_TRACE_DEBUG(
+		("commpage: enter(%p,%p) <- 0x%x\n",
+		 map, task, kr));
+	return kr;
+}
diff --git a/osfmk/vm/vm_shared_region.h b/osfmk/vm/vm_shared_region.h
new file mode 100644
index 000000000..17067253a
--- /dev/null
+++ b/osfmk/vm/vm_shared_region.h
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this
+ * file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_LICENSE_HEADER_END@
+ */
+/*
+ *
+ *	File: vm/vm_shared_region.h
+ *
+ * 	protos and struct definitions for shared region
+ */
+
+#ifndef _VM_SHARED_REGION_H_
+#define _VM_SHARED_REGION_H_
+
+#ifdef	KERNEL_PRIVATE
+
+#include <mach/vm_prot.h>
+#include <mach/mach_types.h>
+#include <mach/shared_region.h>
+
+#include <kern/kern_types.h>
+#include <kern/macro_help.h>
+
+#include <vm/vm_map.h>
+
+extern int shared_region_version;
+extern int shared_region_persistence;
+
+#if DEBUG
+extern int shared_region_debug;
+#define SHARED_REGION_DEBUG(args)		\
+	MACRO_BEGIN				\
+	if (shared_region_debug) {		\
+		kprintf args;			\
+	}					\
+	MACRO_END
+#else /* DEBUG */
+#define SHARED_REGION_DEBUG(args)
+#endif /* DEBUG */
+
+extern int shared_region_trace_level;
+#define SHARED_REGION_TRACE_NONE_LVL		0 /* no trace */
+#define SHARED_REGION_TRACE_ERROR_LVL		1 /* trace abnormal events */
+#define SHARED_REGION_TRACE_INFO_LVL		2 /* trace all events */
+#define SHARED_REGION_TRACE_DEBUG_LVL		3 /* extra traces for debug */
+#define SHARED_REGION_TRACE(level, args)		\
+	MACRO_BEGIN					\
+	if (shared_region_trace_level >= level) {	\
+		printf args;				\
+	}						\
+	MACRO_END
+#define SHARED_REGION_TRACE_NONE(args)
+#define SHARED_REGION_TRACE_ERROR(args)				\
+	MACRO_BEGIN						\
+	SHARED_REGION_TRACE(SHARED_REGION_TRACE_ERROR_LVL,	\
+			    args);				\
+	MACRO_END
+#define SHARED_REGION_TRACE_INFO(args)				\
+	MACRO_BEGIN						\
+	SHARED_REGION_TRACE(SHARED_REGION_TRACE_INFO_LVL,	\
+			    args);				\
+	MACRO_END
+#define SHARED_REGION_TRACE_DEBUG(args)				\
+	MACRO_BEGIN						\
+	SHARED_REGION_TRACE(SHARED_REGION_TRACE_DEBUG_LVL,	\
+			    args);				\
+	MACRO_END
+
+typedef struct vm_shared_region *vm_shared_region_t;
+
+#ifdef MACH_KERNEL_PRIVATE
+
+#include <kern/queue.h>
+#include <vm/vm_object.h>
+#include <vm/memory_object.h>
+
+/* address space shared region descriptor */
+struct vm_shared_region {
+	uint32_t		sr_ref_count;
+	void			*sr_root_dir;
+	cpu_type_t		sr_cpu_type;
+	boolean_t		sr_64bit;
+	boolean_t		sr_mapping_in_progress;
+	boolean_t		sr_persists;
+	ipc_port_t		sr_mem_entry;
+	mach_vm_offset_t	sr_first_mapping;
+	mach_vm_offset_t	sr_base_address;
+	mach_vm_size_t		sr_size;
+	mach_vm_offset_t	sr_pmap_nesting_start;
+	mach_vm_size_t		sr_pmap_nesting_size;
+	queue_chain_t		sr_q;
+};
+
+#else  /* !MACH_KERNEL_PRIVATE */
+
+struct vm_shared_region;
+
+#endif /* MACH_KERNEL_PRIVATE */
+
+extern void vm_shared_region_init(void);
+extern kern_return_t vm_shared_region_enter(
+	struct _vm_map		*map,
+	struct task		*task,
+	void			*fsroot,
+	cpu_type_t		cpu);
+extern kern_return_t vm_shared_region_remove(
+	struct _vm_map		*map,
+	struct task		*task);
+extern vm_shared_region_t vm_shared_region_get(
+	struct task		*task);
+extern void vm_shared_region_deallocate(
+	struct vm_shared_region	*shared_region);
+extern mach_vm_offset_t vm_shared_region_base_address(
+	struct vm_shared_region	*shared_region);
+extern mach_vm_size_t vm_shared_region_size(
+	struct vm_shared_region	*shared_region);
+extern ipc_port_t vm_shared_region_mem_entry(
+	struct vm_shared_region	*shared_region);
+extern void vm_shared_region_set(
+	struct task		*task,
+	struct vm_shared_region	*new_shared_region);
+extern vm_shared_region_t vm_shared_region_lookup(
+	void			*root_dir,
+	cpu_type_t		cpu,
+	boolean_t		is_64bit);
+extern kern_return_t vm_shared_region_start_address(
+	struct vm_shared_region	*shared_region,
+	mach_vm_offset_t	*start_address);
+extern kern_return_t vm_shared_region_map_file(
+	struct vm_shared_region	*shared_region,
+	unsigned int		mappings_count,
+	struct shared_file_mapping_np *mappings,
+	memory_object_control_t	file_control,
+	memory_object_size_t	file_size,
+	void			*root_dir);
+
+extern void vm_commpage_init(void);
+extern kern_return_t vm_commpage_enter(
+	struct _vm_map		*map,
+	struct task		*task);
+extern kern_return_t vm_commpage_remove(
+	struct _vm_map		*map,
+	struct task		*task);
+
+#endif /* KERNEL_PRIVATE */
+
+#endif	/* _VM_SHARED_REGION_H_ */
diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c
index d459c63e0..0ed0a84b2 100644
--- a/osfmk/vm/vm_user.c
+++ b/osfmk/vm/vm_user.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
@@ -72,9 +78,7 @@
 
 #include <mach/host_priv_server.h>
 #include <mach/mach_vm_server.h>
-#include <mach/shared_memory_server.h>
 #include <mach/vm_map_server.h>
-#include <vm/vm_shared_memory_server.h>
 
 #include <kern/host.h>
 #include <kern/kalloc.h>
@@ -110,7 +114,11 @@ mach_vm_allocate(
 	vm_map_offset_t map_addr;
 	vm_map_size_t	map_size;
 	kern_return_t	result;
-	boolean_t	anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
+	boolean_t	anywhere;
+
+	/* filter out any kernel-only flags */
+	if (flags & ~VM_FLAGS_USER_ALLOCATE)
+		return KERN_INVALID_ARGUMENT;
 
 	if (map == VM_MAP_NULL)
 		return(KERN_INVALID_ARGUMENT);
@@ -119,6 +127,7 @@ mach_vm_allocate(
 		return(KERN_SUCCESS);
 	}
 
+	anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
 	if (anywhere) {
 		/*
 		 * No specific address requested, so start candidate address
@@ -172,7 +181,11 @@ vm_allocate(
 	vm_map_offset_t map_addr;
 	vm_map_size_t	map_size;
 	kern_return_t	result;
-	boolean_t	anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
+	boolean_t	anywhere;
+
+	/* filter out any kernel-only flags */
+	if (flags & ~VM_FLAGS_USER_ALLOCATE)
+		return KERN_INVALID_ARGUMENT;
 
 	if (map == VM_MAP_NULL)
 		return(KERN_INVALID_ARGUMENT);
@@ -181,6 +194,7 @@ vm_allocate(
 		return(KERN_SUCCESS);
 	}
 
+	anywhere = ((VM_FLAGS_ANYWHERE & flags) != 0);
 	if (anywhere) {
 		/*
 		 * No specific address requested, so start candidate address
@@ -830,276 +844,21 @@ mach_vm_map(
 	vm_prot_t		max_protection,
 	vm_inherit_t		inheritance)
 {
-	vm_map_address_t	map_addr;
-	vm_map_size_t		map_size;
-	vm_object_t		object;
-	vm_object_size_t	size;
-	kern_return_t		result;
-
-	/*
-	 * Check arguments for validity
-	 */
-	if ((target_map == VM_MAP_NULL) ||
-		(cur_protection & ~VM_PROT_ALL) ||
-		(max_protection & ~VM_PROT_ALL) ||
-		(inheritance > VM_INHERIT_LAST_VALID) ||
-		initial_size == 0)
-		return(KERN_INVALID_ARGUMENT);
-
-	map_addr = vm_map_trunc_page(*address);
-	map_size = vm_map_round_page(initial_size);
-	size = vm_object_round_page(initial_size);	
-	
-	/*
-	 * Find the vm object (if any) corresponding to this port.
-	 */
-	if (!IP_VALID(port)) {
-		object = VM_OBJECT_NULL;
-		offset = 0;
-		copy = FALSE;
-	} else if (ip_kotype(port) == IKOT_NAMED_ENTRY) {
-		vm_named_entry_t	named_entry;
-
-		named_entry = (vm_named_entry_t)port->ip_kobject;
-		/* a few checks to make sure user is obeying rules */
-		if(size == 0) {
-			if(offset >= named_entry->size)
-				return(KERN_INVALID_RIGHT);
-			size = named_entry->size - offset;
-		}
-		if((named_entry->protection & max_protection) != max_protection)
-			return(KERN_INVALID_RIGHT);
-		if((named_entry->protection & cur_protection) != cur_protection)
-			return(KERN_INVALID_RIGHT);
-		if(named_entry->size < (offset + size))
-			return(KERN_INVALID_ARGUMENT);
-
-		/* the callers parameter offset is defined to be the */
-		/* offset from beginning of named entry offset in object */
-		offset = offset + named_entry->offset;
-		
-		named_entry_lock(named_entry);
-		if(named_entry->is_sub_map) {
-			vm_map_entry_t		map_entry;
-
-			named_entry_unlock(named_entry);
-			vm_object_reference(vm_submap_object);
-			if ((result = vm_map_enter(target_map,
-				&map_addr, map_size,
-				(vm_map_offset_t)mask, flags,
-				vm_submap_object, 0,
-				FALSE,
-				cur_protection, max_protection, inheritance
-				)) != KERN_SUCCESS) {
-					vm_object_deallocate(vm_submap_object);
-			} else {
-				char	alias;
-
-				VM_GET_FLAGS_ALIAS(flags, alias);
-				if ((alias == VM_MEMORY_SHARED_PMAP) &&
-					!copy) {
-					vm_map_submap(target_map, map_addr, 
-						map_addr + map_size, 
-						named_entry->backing.map,
-						(vm_map_offset_t)offset, TRUE);
-				} else {
-					vm_map_submap(target_map, map_addr, 
-						map_addr + map_size, 
-						named_entry->backing.map,
-						(vm_map_offset_t)offset, FALSE);
-				}
-				if(copy) {
-					if(vm_map_lookup_entry(
-					   target_map, map_addr, &map_entry)) {
-						map_entry->needs_copy = TRUE;
-					}
-				}
-				*address = map_addr;
-			}
-			return(result);
-
-		} else if (named_entry->is_pager) {
-			unsigned int		access;
-			vm_prot_t		protections;
-			unsigned int		wimg_mode;
-			boolean_t		cache_attr;
-
-			protections = named_entry->protection 
-							& VM_PROT_ALL;
-			access = GET_MAP_MEM(named_entry->protection);
-
-			object = vm_object_enter(
-				named_entry->backing.pager, 
-				named_entry->size, 
-				named_entry->internal, 
-				FALSE,
-				FALSE);
-			if (object == VM_OBJECT_NULL) {
-				named_entry_unlock(named_entry);
-				return(KERN_INVALID_OBJECT);
-			}
-
-			/* JMM - drop reference on pager here */
-
-			/* create an extra ref for the named entry */
-			vm_object_lock(object);
-			vm_object_reference_locked(object);
-			named_entry->backing.object = object;
-			named_entry->is_pager = FALSE;
-			named_entry_unlock(named_entry);
-
-			wimg_mode = object->wimg_bits;
-			if(access == MAP_MEM_IO) {
-				wimg_mode = VM_WIMG_IO;
-			} else if (access == MAP_MEM_COPYBACK) {
-				wimg_mode = VM_WIMG_USE_DEFAULT;
-			} else if (access == MAP_MEM_WTHRU) {
-				wimg_mode = VM_WIMG_WTHRU;
-			} else if (access == MAP_MEM_WCOMB) {
-				wimg_mode = VM_WIMG_WCOMB;
-			}
-			if ((wimg_mode == VM_WIMG_IO)
-				|| (wimg_mode == VM_WIMG_WCOMB))
-				cache_attr = TRUE;
-			else 
-				cache_attr = FALSE;
-
-			/* wait for object (if any) to be ready */
-			if (!named_entry->internal) {
-				while (!object->pager_ready) {
-					vm_object_wait(object,
-						       VM_OBJECT_EVENT_PAGER_READY,
-						       THREAD_UNINT);
-					vm_object_lock(object);
-				}
-			}
-
-			if(object->wimg_bits != wimg_mode) {
-				vm_page_t p;
-
-				vm_object_paging_wait(object, THREAD_UNINT);
-
-				object->wimg_bits = wimg_mode;
-				queue_iterate(&object->memq, p, vm_page_t, listq) {
-					if (!p->fictitious) {
-					        pmap_disconnect(p->phys_page);
-						if (cache_attr)
-						        pmap_sync_page_attributes_phys(p->phys_page);
-					}
-				}
-			}
-			object->true_share = TRUE;
-			if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
-				object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
-			vm_object_unlock(object);
-		} else {
-			/* This is the case where we are going to map */
-			/* an already mapped object.  If the object is */
-			/* not ready it is internal.  An external     */
-			/* object cannot be mapped until it is ready  */
-			/* we can therefore avoid the ready check     */
-			/* in this case.  */
-			object = named_entry->backing.object;
-			assert(object != VM_OBJECT_NULL);
-			named_entry_unlock(named_entry);
-			vm_object_reference(object);
-		}
-	} else if (ip_kotype(port) == IKOT_MEMORY_OBJECT) {
-		/*
-		 * JMM - This is temporary until we unify named entries
-		 * and raw memory objects.
-		 *
-		 * Detected fake ip_kotype for a memory object.  In
-		 * this case, the port isn't really a port at all, but
-		 * instead is just a raw memory object.
-		 */
-		 
-		if ((object = vm_object_enter((memory_object_t)port,
-					      size, FALSE, FALSE, FALSE))
-			== VM_OBJECT_NULL)
-			return(KERN_INVALID_OBJECT);
-
-		/* wait for object (if any) to be ready */
-		if (object != VM_OBJECT_NULL) {
-			if(object == kernel_object) {
-				printf("Warning: Attempt to map kernel object"
-					" by a non-private kernel entity\n");
-				return(KERN_INVALID_OBJECT);
-			}
-			vm_object_lock(object);
-			while (!object->pager_ready) {
-				vm_object_wait(object,
-					VM_OBJECT_EVENT_PAGER_READY,
-					THREAD_UNINT);
-				vm_object_lock(object);
-			}
-			vm_object_unlock(object);
-		}
-	} else {
-		return (KERN_INVALID_OBJECT);
-	}
-
-	/*
-	 *	Perform the copy if requested
-	 */
-
-	if (copy) {
-		vm_object_t		new_object;
-		vm_object_offset_t	new_offset;
-
-		result = vm_object_copy_strategically(object, offset, size,
-				&new_object, &new_offset,
-				&copy);
-
-
-		if (result == KERN_MEMORY_RESTART_COPY) {
-			boolean_t success;
-			boolean_t src_needs_copy;
-
-			/*
-			 * XXX
-			 * We currently ignore src_needs_copy.
-			 * This really is the issue of how to make
-			 * MEMORY_OBJECT_COPY_SYMMETRIC safe for
-			 * non-kernel users to use. Solution forthcoming.
-			 * In the meantime, since we don't allow non-kernel
-			 * memory managers to specify symmetric copy,
-			 * we won't run into problems here.
-			 */
-			new_object = object;
-			new_offset = offset;
-			success = vm_object_copy_quickly(&new_object,
-							 new_offset, size,
-							 &src_needs_copy,
-							 &copy);
-			assert(success);
-			result = KERN_SUCCESS;
-		}
-		/*
-		 *	Throw away the reference to the
-		 *	original object, as it won't be mapped.
-		 */
-
-		vm_object_deallocate(object);
-
-		if (result != KERN_SUCCESS)
-			return (result);
-
-		object = new_object;
-		offset = new_offset;
-	}
+	/* filter out any kernel-only flags */
+	if (flags & ~VM_FLAGS_USER_MAP)
+		return KERN_INVALID_ARGUMENT;
 
-	if ((result = vm_map_enter(target_map,
-				   &map_addr, map_size,
-				   (vm_map_offset_t)mask,
-				   flags,
-				   object, offset,
-				   copy,
-				   cur_protection, max_protection, inheritance
-				   )) != KERN_SUCCESS)
-		vm_object_deallocate(object);
-	*address = map_addr;
-	return(result);
+	return vm_map_enter_mem_object(target_map,
+				       address,
+				       initial_size,
+				       mask,
+				       flags,
+				       port,
+				       offset,
+				       copy,
+				       cur_protection,
+				       max_protection,
+				       inheritance);
 }
 
 
@@ -1760,6 +1519,22 @@ vm_region_recurse(
 	return kr;
 }
 
+kern_return_t
+mach_vm_purgable_control(
+	vm_map_t		map,
+	mach_vm_offset_t	address,
+	vm_purgable_t		control,
+	int			*state)
+{
+	if (VM_MAP_NULL == map)
+		return KERN_INVALID_ARGUMENT;
+
+	return vm_map_purgable_control(map,
+				       vm_map_trunc_page(address),
+				       control,
+				       state);
+}
+
 kern_return_t
 vm_purgable_control(
 	vm_map_t		map,
@@ -1920,12 +1695,6 @@ vm_map_get_upl(
 	return kr;
 }
 
-
-__private_extern__ kern_return_t
-mach_memory_entry_allocate(
-	vm_named_entry_t	*user_entry_p,
-	ipc_port_t		*user_handle_p);	/* forward */
-
 /*
  * mach_make_memory_entry_64
  *
@@ -1955,8 +1724,7 @@ mach_make_memory_entry_64(
 	boolean_t		wired;
 	vm_object_offset_t	obj_off;
 	vm_prot_t		prot;
-	vm_map_offset_t		lo_offset, hi_offset;
-	vm_behavior_t		behavior;
+	struct vm_object_fault_info	fault_info;
 	vm_object_t		object;
 	vm_object_t		shadow_object;
 
@@ -2049,7 +1817,8 @@ mach_make_memory_entry_64(
 			   queue_iterate(&object->memq, 
 						p, vm_page_t, listq) {
 				if (!p->fictitious) {
-				        pmap_disconnect(p->phys_page);
+				        if (p->pmapped)
+					        pmap_disconnect(p->phys_page);
 					if (cache_attr)
 					        pmap_sync_page_attributes_phys(p->phys_page);
 				}
@@ -2091,7 +1860,7 @@ mach_make_memory_entry_64(
 				kr = KERN_INVALID_ARGUMENT;
 				goto make_mem_done;
 			}
-			object->purgable = VM_OBJECT_PURGABLE_NONVOLATILE;
+			object->purgable = VM_PURGABLE_NONVOLATILE;
 		}
 
 		/*
@@ -2148,6 +1917,10 @@ mach_make_memory_entry_64(
 		/* Create a named object based on address range within the task map */
 		/* Go find the object at given address */
 
+		if (target_map == VM_MAP_NULL) {
+			return KERN_INVALID_TASK;
+		}
+
 redo_lookup:
 		vm_map_lock_read(target_map);
 
@@ -2156,9 +1929,10 @@ redo_lookup:
 		/* that requested by the caller */
 
 		kr = vm_map_lookup_locked(&target_map, map_offset, 
-				protections, &version,
-				&object, &obj_off, &prot, &wired, &behavior,
-				&lo_offset, &hi_offset, &real_map);
+			        protections, OBJECT_LOCK_EXCLUSIVE, &version,
+				&object, &obj_off, &prot, &wired,
+				&fault_info,
+				&real_map);
 		if (kr != KERN_SUCCESS) {
 			vm_map_unlock_read(target_map);
 			goto make_mem_done;
@@ -2259,7 +2033,7 @@ redo_lookup:
                          goto make_mem_done;
 		}
 
-		mappable_size  =  hi_offset - obj_off;
+		mappable_size = fault_info.hi_offset - obj_off;
 		total_size = map_entry->vme_end - map_entry->vme_start;
 		if(map_size > mappable_size) {
 			/* try to extend mappable size if the entries */
@@ -2345,10 +2119,10 @@ redo_lookup:
 				vm_object_unlock(object);
 
 				prot = map_entry->protection & ~VM_PROT_WRITE;
-#ifdef STACK_ONLY_NX
-				if (map_entry->alias != VM_MEMORY_STACK && prot)
+
+				if (override_nx(target_map, map_entry->alias) && prot)
 				        prot |= VM_PROT_EXECUTE;
-#endif
+
 				vm_object_pmap_protect(
 					object, map_entry->offset,
 					total_size,
@@ -2362,10 +2136,11 @@ redo_lookup:
 						- map_entry->vme_start);
 				next_entry = map_entry->vme_next;
 				map_entry->needs_copy = FALSE;
+
+				vm_object_lock(shadow_object);
 				while (total_size) {
 				   if(next_entry->object.vm_object == object) {
-					shadow_object->ref_count++; 
-					vm_object_res_reference(shadow_object);
+					vm_object_reference_locked(shadow_object);
 					next_entry->object.vm_object 
 							= shadow_object;
 					vm_object_deallocate(object);
@@ -2396,8 +2171,6 @@ redo_lookup:
 							 + map_entry->offset;
 
 				vm_map_lock_write_to_read(target_map);
-				vm_object_lock(object);
-
 	        	}
 	   	}
 
@@ -2453,7 +2226,8 @@ redo_lookup:
 			queue_iterate(&object->memq, 
 						p, vm_page_t, listq) {
 				if (!p->fictitious) {
-				        pmap_disconnect(p->phys_page);
+				        if (p->pmapped)
+					        pmap_disconnect(p->phys_page);
 					if (cache_attr)
 					        pmap_sync_page_attributes_phys(p->phys_page);
 				}
@@ -2605,10 +2379,10 @@ _mach_make_memory_entry(
 	ipc_port_t		*object_handle,
 	ipc_port_t		parent_entry)
 {
-	memory_object_offset_t 	mo_size;
+	memory_object_size_t 	mo_size;
 	kern_return_t		kr;
 	
-	mo_size = (memory_object_offset_t)*size;
+	mo_size = (memory_object_size_t)*size;
 	kr = mach_make_memory_entry_64(target_map, &mo_size, 
 			(memory_object_offset_t)offset, permission, object_handle,
 			parent_entry);
@@ -2625,10 +2399,10 @@ mach_make_memory_entry(
 	ipc_port_t		*object_handle,
 	ipc_port_t		parent_entry)
 {	
-	memory_object_offset_t 	mo_size;
+	memory_object_size_t 	mo_size;
 	kern_return_t		kr;
 	
-	mo_size = (memory_object_offset_t)*size;
+	mo_size = (memory_object_size_t)*size;
 	kr = mach_make_memory_entry_64(target_map, &mo_size, 
 			(memory_object_offset_t)offset, permission, object_handle,
 			parent_entry);
@@ -2701,8 +2475,10 @@ mach_memory_entry_allocate(
 	user_entry->backing.pager = NULL;
 	user_entry->is_sub_map = FALSE;
 	user_entry->is_pager = FALSE;
-	user_entry->size = 0;
 	user_entry->internal = FALSE;
+	user_entry->size = 0;
+	user_entry->offset = 0;
+	user_entry->protection = VM_PROT_NONE;
 	user_entry->ref_count = 1;
 
 	ipc_kobject_set(user_handle, (ipc_kobject_t) user_entry,
@@ -2786,6 +2562,14 @@ mach_memory_entry_purgable_control(
 	    ip_kotype(entry_port) != IKOT_NAMED_ENTRY) {
 		return KERN_INVALID_ARGUMENT;
 	}
+	if (control != VM_PURGABLE_SET_STATE &&
+	    control != VM_PURGABLE_GET_STATE)
+		return(KERN_INVALID_ARGUMENT);
+
+	if (control == VM_PURGABLE_SET_STATE &&
+	    (((*state & ~(VM_PURGABLE_STATE_MASK|VM_VOLATILE_ORDER_MASK|VM_PURGABLE_ORDERING_MASK|VM_PURGABLE_BEHAVIOR_MASK|VM_VOLATILE_GROUP_MASK)) != 0) ||
+	     ((*state & VM_PURGABLE_STATE_MASK) > VM_PURGABLE_STATE_MASK)))
+		return(KERN_INVALID_ARGUMENT);
 
 	mem_entry = (vm_named_entry_t) entry_port->ip_kobject;
 
diff --git a/pexpert/Makefile b/pexpert/Makefile
index 7497f281e..59300a74d 100644
--- a/pexpert/Makefile
+++ b/pexpert/Makefile
@@ -12,9 +12,10 @@ INSTINC_SUBDIRS = pexpert
 
 INSTINC_SUBDIRS_PPC = pexpert
 
-
 INSTINC_SUBDIRS_I386 = pexpert
 
+INSTINC_SUBDIRS_ARM = pexpert
+
 
 EXPINC_SUBDIRS = pexpert
 
@@ -24,6 +25,8 @@ EXPINC_SUBDIRS_PPC = pexpert
 
 EXPINC_SUBDIRS_I386 = pexpert
 
+EXPINC_SUBDIRS_ARM = pexpert
+
 
 SETUP_SUBDIRS = 	\
 	conf
diff --git a/pexpert/conf/MASTER b/pexpert/conf/MASTER
index 84438831d..1588c80c5 100644
--- a/pexpert/conf/MASTER
+++ b/pexpert/conf/MASTER
@@ -83,6 +83,17 @@
 ident		PEXPERT
 
 options		MACH_PE		# Objective-C support		# <mach_pe>
-options         MACH_KERNEL
-options         DEBUG
+options		MACH_KERNEL
+options		DEBUG
+options     CONFIG_DTRACE 	# dtrace support			# <config_dtrace>
 
+options		PANIC_INFO	# want kernel panic info	# <panic_info>
+
+options		CONFIG_NO_PANIC_STRINGS		# <no_panic_str>
+options		CONFIG_NO_PRINTF_STRINGS	# <no_printf_str>
+options		CONFIG_NO_KPRINTF_STRINGS	# <no_kprintf_str>
+
+# configurable kernel - general switch to say we are building for an
+# embedded device
+#
+options   CONFIG_EMBEDDED                       # <config_embedded>
diff --git a/pexpert/conf/MASTER.i386 b/pexpert/conf/MASTER.i386
index f276a2461..3bde93bca 100644
--- a/pexpert/conf/MASTER.i386
+++ b/pexpert/conf/MASTER.i386
@@ -1,15 +1,16 @@
-#
 ######################################################################
 #  
-#  Standard NeXT Research Configurations:
-#  -------- ---- -------- ---------------
+#  Standard Apple Mac OS Configurations:
+#  -------- ----- ------ ---------------
+#
+#  RELEASE	= [ i386 mach mach_pe panic_info config_dtrace ]
+#  PROFILE	= [ RELEASE profile ]
+#  DEBUG	= [ RELEASE debug ]
 #
-#  RELEASE = [i386 mach mach_pe]
-#  PROFILE = [i386 mach mach_pe profile]
-#  DEBUG = [i386 mach mach_pe debug]
+#  EMBEDDED	= [ i386 mach mach_pe panic_info ]
+#  DEVELOPMENT	= [ EMBEDDED config_dtrace ]
 #
 ######################################################################
 
 machine		"i386"					# <i386>
 cpu		"i386"					# <i386>
-
diff --git a/pexpert/conf/MASTER.ppc b/pexpert/conf/MASTER.ppc
index 81050216c..f36c6062d 100644
--- a/pexpert/conf/MASTER.ppc
+++ b/pexpert/conf/MASTER.ppc
@@ -4,14 +4,15 @@
 #  Standard NeXT Research Configurations:
 #  -------- ---- -------- ---------------
 #
-#  RELEASE = [ppc mach mach_pe]
+#  RELEASE = [ppc mach mach_pe panic_info config_dtrace]
+#  DEVELOPMENT = [ RELEASE ]
 #  RELEASE_TRACE = [RELEASE kdebug]
-#  PROFILE = [ppc mach mach_pe profile]
-#  DEBUG = [ppc mach mach_pe debug]
+#  PROFILE = [RELEASE profile]
+#  DEBUG = [RELEASE debug]
 #  DEBUG_TRACE = [DEBUG kdebug]
 #
 ######################################################################
 
 machine		"ppc"					# <ppc>
-cpu		"ppc"					# <ppc>
+cpu			"ppc"					# <ppc>
 
diff --git a/pexpert/conf/Makefile b/pexpert/conf/Makefile
index 68774ccc1..054208e35 100644
--- a/pexpert/conf/Makefile
+++ b/pexpert/conf/Makefile
@@ -18,10 +18,14 @@ ifndef PEXPERT_KERNEL_CONFIG
 export PEXPERT_KERNEL_CONFIG = $(KERNEL_CONFIG)
 endif
 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
 export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
 
-$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf:
-	make build_setup 
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
 
 $(COMPOBJROOT)/$(PEXPERT_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
@@ -29,36 +33,32 @@ $(COMPOBJROOT)/$(PEXPERT_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
 	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
 	$(SOURCE)/files \
 	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf
-	@echo "Running doconf for $(PEXPERT_KERNEL_CONFIG)";
-	(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
 	echo $${doconf_target};\
 	$(MKDIR) $${doconf_target}; \
 	cd $${doconf_target}; \
 	rm -f $(notdir $?); \
 	cp  $? $${doconf_target}; \
-	$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(PEXPERT_KERNEL_CONFIG) $(PEXPERT_KERNEL_CONFIG); \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(PEXPERT_KERNEL_CONFIG) $(PEXPERT_KERNEL_CONFIG); \
 	);
 
 .ORDER: $(COMPOBJROOT)/$(PEXPERT_KERNEL_CONFIG)/Makefile 
 
-do_setup_conf: $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/doconf \
+do_setup_conf: $(COMPOBJROOT)/doconf \
 		$(COMPOBJROOT)/$(PEXPERT_KERNEL_CONFIG)/Makefile 
 
 do_all: do_setup_conf
-	@echo "[ $(SOURCE) ] Starting do_all  $(COMPONENT) $(PEXPERT_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";	\
-	next_source=$(subst conf/,,$(SOURCE));			\
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
 	${MAKE} -C $(COMPOBJROOT)/$(PEXPERT_KERNEL_CONFIG)	\
 		MAKEFILES=$(TARGET)/$(PEXPERT_KERNEL_CONFIG)/Makefile	\
 		SOURCE=$${next_source}			\
 		TARGET=$(TARGET)					\
 		INCL_MAKEDEP=FALSE	\
 		KERNEL_CONFIG=$(PEXPERT_KERNEL_CONFIG)	\
-		build_all;	\
-	echo "[ $(SOURCE) ] Returning do_all $(COMPONENT) $(PEXPERT_KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
+		build_all;
 
 do_build_all: do_all 
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/pexpert/conf/Makefile.template b/pexpert/conf/Makefile.template
index 388c6f036..526b89513 100644
--- a/pexpert/conf/Makefile.template
+++ b/pexpert/conf/Makefile.template
@@ -28,9 +28,9 @@ include $(MakeInc_def)
 #
 # -D_KERNEL_BUILD -DKERNEL_BUILD -DARCH_PRIVATE -DBSD_BUILD -DMACH_KERNEL
 #
-CFLAGS+= -DPEXPERT_KERNEL_PRIVATE -DKERNEL -DDRIVER_PRIVATE 	\
+CFLAGS+= -imacros meta_features.h -DPEXPERT_KERNEL_PRIVATE -DKERNEL -DDRIVER_PRIVATE 	\
 	-Wall -Wno-four-char-constants -fno-common		\
-	-DRelease3CompatibilityBuild
+	-DRelease3CompatibilityBuild $(CFLAGS_INLINE_CONFIG)
 
 SFLAGS+= -DKERNEL
 
@@ -79,12 +79,11 @@ ${OBJS}: ${OBJSDEPS}
 LDOBJS = $(OBJS)
 
 $(COMPONENT).o: $(LDOBJS)
-	@echo "creating $(COMPONENT).o"
-	@echo [ updating $(COMPONENT).o ${PEXPERT_KERNEL_CONFIG} ]
-	$(LD) $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS}
+	@echo LD $(COMPONENT)
+	$(_v)$(LD) $(LDFLAGS_COMPONENT) -o $(COMPONENT).o ${LDOBJS}
 
 do_depend: do_all
-	${MD} -u Makedep -f -d `ls *.d`;
+	$(_v)${MD} -u Makedep -f -d `ls *.d`;
 	
 do_all: $(COMPONENT).o
 
@@ -94,4 +93,3 @@ do_build_all: do_depend
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
diff --git a/pexpert/conf/files b/pexpert/conf/files
index 8568b9940..3ad1855bc 100644
--- a/pexpert/conf/files
+++ b/pexpert/conf/files
@@ -1,6 +1,7 @@
 #
 OPTIONS/mach_kdb				optional mach_kdb
-
+OPTIONS/panic_info				optional panic_info
+OPTIONS/config_dtrace           optional config_dtrace
 
 #
 # pexpert generic
diff --git a/pexpert/conf/files.i386 b/pexpert/conf/files.i386
index 8ae285f05..9808c74ec 100644
--- a/pexpert/conf/files.i386
+++ b/pexpert/conf/files.i386
@@ -5,7 +5,6 @@ pexpert/i386/pe_bootargs.c            standard
 pexpert/i386/pe_identify_machine.c    standard
 pexpert/i386/pe_kprintf.c             standard
 pexpert/i386/pe_interrupt.c           standard
-pexpert/i386/fakePPCDeviceTree.c      standard
 pexpert/i386/pe_serial.c              standard
 
 
diff --git a/pexpert/conf/tools/doconf/Makefile b/pexpert/conf/tools/doconf/Makefile
index 2bf0b7a10..aa55a9419 100644
--- a/pexpert/conf/tools/doconf/Makefile
+++ b/pexpert/conf/tools/doconf/Makefile
@@ -16,7 +16,11 @@ INST_SUBDIRS = \
 # Who and where
 #
 BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
 DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
 PROGRAM= $(DSTDIR)doconf
 
 # 
@@ -25,25 +29,19 @@ PROGRAM= $(DSTDIR)doconf
 IFLAGS= -c -m 555
 
 $(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)";
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
-	sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
 		< $< >$(notdir $(PROGRAM)).VERS;
-	install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
-	-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
 
 do_build_setup: $(PROGRAM)
 
 do_build_all:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 setup_build_install:
-	@echo "[ $(SOURCE) ] make setup_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 do_build_install:
-	@echo "[ $(SOURCE) ] make do_build_all $(KERNEL_CONFIG) $(ARCH_CONFIG) $(TARGET)"
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
-
-
diff --git a/pexpert/conf/tools/doconf/doconf.csh b/pexpert/conf/tools/doconf/doconf.csh
index ae5ab908b..6fedb4786 100755
--- a/pexpert/conf/tools/doconf/doconf.csh
+++ b/pexpert/conf/tools/doconf/doconf.csh
@@ -175,7 +175,9 @@ if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
 if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
 
 if (! -d $OBJDIR) then
-    echo "[ creating $OBJDIR ]"
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
     mkdir -p $OBJDIR
 endif
 
@@ -264,7 +266,9 @@ part != 0 {\
 	rm -f $SYSCONF.new
     endif
     if (! -d $BLDDIR) then
-	echo "[ creating $BLDDIR ]"
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
 	mkdir -p $BLDDIR
     endif
 #
@@ -299,7 +303,9 @@ part != 0 {\
     rm -f $SYSCONF
     mv $SYSCONF.new $SYSCONF
     if ($?doconfig) then
-	echo "[ configuring $SYSID ]"
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
 	if ($?profile) then
 	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
 	else
@@ -307,7 +313,9 @@ part != 0 {\
 	endif
     endif
     if ($?domake) then
-        echo "[ making $SYSID ]"
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
         (cd $BLDDIR; make)
     endif
 end
diff --git a/pexpert/gen/bootargs.c b/pexpert/gen/bootargs.c
index 558175349..d7d0a3cd4 100644
--- a/pexpert/gen/bootargs.c
+++ b/pexpert/gen/bootargs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/pexpert.h>
 
@@ -25,6 +31,8 @@ extern boolean_t isargsep( char c);
 extern int argstrcpy(char *from, char *to);
 extern int getval(char *s, int *val);
 
+int argstrcpy2(char *from,char *to, unsigned maxlen);
+
 #define	NUM	0
 #define	STR	1
 
@@ -33,7 +41,14 @@ PE_parse_boot_arg(
 	const char  *arg_string,
 	void		*arg_ptr)
 {
-	return PE_parse_boot_argn(arg_string, arg_ptr, -1);
+	int max_len = -1;
+
+#if CONFIG_EMBEDDED
+	/* Limit arg size to 4 byte when no size is given */
+	max_len = 4;
+#endif
+
+	return PE_parse_boot_argn(arg_string, arg_ptr, max_len);
 }
 
 boolean_t
@@ -44,7 +59,7 @@ PE_parse_boot_argn(
 {
 	char *args;
 	char *cp, c;
-	int i;
+	unsigned int i;
 	int val;
 	boolean_t arg_boolean;
 	boolean_t arg_found;
@@ -88,8 +103,8 @@ PE_parse_boot_argn(
 			}
 			if ('_' == *arg_string) /* Force a string copy if the argument name begins with an underscore */
 			{
-				int hacklen = 16 > max_len ? 16 : max_len;
-				argstrcpy2 (++cp, (char *)arg_ptr, hacklen); /* Hack - terminate after 16 characters */
+				int hacklen = 17 > max_len ? 17 : max_len;
+				argstrcpy2 (++cp, (char *)arg_ptr, hacklen - 1); /* Hack - terminate after 16 characters */
 				arg_found = TRUE;
 				break;
 			}
@@ -101,7 +116,7 @@ PE_parse_boot_argn(
 					break;
 				case STR:
 					if(max_len > 0) //max_len of 0 performs no copy at all
-						argstrcpy2(++cp, (char *)arg_ptr, max_len);
+						argstrcpy2(++cp, (char *)arg_ptr, max_len - 1);
 					else if(max_len == -1)
 						argstrcpy(++cp, (char *)arg_ptr);
 					arg_found = TRUE;
@@ -150,7 +165,7 @@ argstrcpy2(
 	char *to,
 	unsigned maxlen)
 {
-	int i = 0;
+	unsigned int i = 0;
 
 	while (!isargsep(*from) && i < maxlen) {
 		i++;
@@ -166,7 +181,7 @@ getval(
 	int *val)
 {
 	unsigned int radix, intval;
-        char c;
+    unsigned char c;
 	int sign = 1;
 
 	if (*s == '=') {
diff --git a/pexpert/gen/device_tree.c b/pexpert/gen/device_tree.c
index fe8e6940c..0ea525699 100644
--- a/pexpert/gen/device_tree.c
+++ b/pexpert/gen/device_tree.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
@@ -33,9 +39,6 @@
 #include <kern/kalloc.h>
 
 #include <sys/types.h>
-#ifdef i386
-#include <i386/fakePPCStructs.h>
-#endif
 
 #ifndef NULL
 #define       NULL    ((void *) 0)
@@ -73,8 +76,6 @@ typedef struct OpaqueDTPropertyIterator {
 static int DTInitialized;
 static RealDTEntry DTRootNode;
 
-void DTInit(void *base);
-
 /*
  * Support Routines
  */
@@ -82,7 +83,7 @@ static RealDTEntry
 skipProperties(RealDTEntry entry)
 {
 	DeviceTreeNodeProperty *prop;
-	int k;
+	unsigned int k;
 
 	if (entry == NULL || entry->nProperties == 0) {
 		return NULL;
@@ -99,7 +100,7 @@ static RealDTEntry
 skipTree(RealDTEntry root)
 {
 	RealDTEntry entry;
-	int k;
+	unsigned int k;
 
 	entry = skipProperties(root);
 	if (entry == NULL) {
@@ -142,8 +143,8 @@ FindChild(RealDTEntry cur, char *buf)
 {
 	RealDTEntry	child;
 	unsigned long	index;
-	char *		str;
-	int		dummy;
+	char *			str;
+	unsigned int	dummy;
 
 	if (cur->nChildren == 0) {
 		return NULL;
@@ -200,7 +201,7 @@ int DTFindEntry(const char *propName, const char *propValue, DTEntry *entryH)
 int find_entry(const char *propName, const char *propValue, DTEntry *entryH)
 {
 	DeviceTreeNode *nodeP = (DeviceTreeNode *) startingP;
-	int k;
+	unsigned int k;
 
 	if (nodeP->nProperties == 0) return(kError);	// End of the list of nodes
 	startingP = (char *) (nodeP + 1);
@@ -212,7 +213,7 @@ int find_entry(const char *propName, const char *propValue, DTEntry *entryH)
 		startingP += sizeof (*propP) + ((propP->length + 3) & -4);
 
 		if (strcmp (propP->name, propName) == 0) {
-			if (strcmp( (char *)(propP + 1), propValue) == 0)
+			if (propValue == NULL || strcmp( (char *)(propP + 1), propValue) == 0)
 			{
 				*entryH = (DTEntry)nodeP;
 				return(kSuccess);
@@ -398,10 +399,10 @@ DTRestartEntryIteration(DTEntryIterator iterator)
 }
 
 int
-DTGetProperty(const DTEntry entry, const char *propertyName, void **propertyValue, int *propertySize)
+DTGetProperty(const DTEntry entry, const char *propertyName, void **propertyValue, unsigned int *propertySize)
 {
 	DeviceTreeNodeProperty *prop;
-	int k;
+	unsigned int k;
 
 	if (entry == NULL || entry->nProperties == 0) {
 		return kError;
diff --git a/pexpert/gen/pe_gen.c b/pexpert/gen/pe_gen.c
index bed890a89..32be95e27 100644
--- a/pexpert/gen/pe_gen.c
+++ b/pexpert/gen/pe_gen.c
@@ -1,29 +1,36 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
- *
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- *
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_FREE_COPYRIGHT@
  */
 
 #include <pexpert/pexpert.h>
+#include <pexpert/protos.h>
 #include <kern/debug.h>
 
 static int DEBUGFlag;
@@ -43,11 +50,10 @@ void PE_enter_debugger(char *cause)
 }
 
 /* extern references */
-extern void cnputc(char c);
 extern void vcattach(void);
 
 /* Globals */
-void (*PE_putc)(char c) = 0;
+void (*PE_putc)(char c);
 
 void PE_init_printf(boolean_t vm_initialized)
 {
diff --git a/pexpert/i386/boot_images.h b/pexpert/i386/boot_images.h
index 6f91ee881..71e0f2860 100644
--- a/pexpert/i386/boot_images.h
+++ b/pexpert/i386/boot_images.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 static const unsigned char bootClut[ 256 * 3 ] = 
diff --git a/pexpert/i386/fakePPCDeviceTree.c b/pexpert/i386/fakePPCDeviceTree.c
deleted file mode 100644
index 66480bc14..000000000
--- a/pexpert/i386/fakePPCDeviceTree.c
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-#include <pexpert/protos.h>
-
-#include "fakePPCStructs.h"
-
-boot_args fakePPCBootArgs = { .Version = kBootArgsVersion };
-
-void * createdt(dt_init * template, long * retSize)
-{
-    dt_init *    next;
-    size_t       size, allocSize;
-    vm_address_t out, saveout;
-    void *       source;
-    
-    // calc size of expanded data
-    for ( next = template, allocSize = 0;
-          next;
-          next++ )
-    {
-        if ( next->nodeInit.zero == 0 )
-        {
-            if( next->nodeInit.nProps == 0) break;
-            allocSize += 2 * sizeof( long);
-        }
-        else if ( next->dataInit.one == 1 )
-        {
-            allocSize += *(next->dataInit.length);
-        }
-        else if ( next->stringInit.two == 2 )
-        {
-            dt_data *dp = (dt_data *)(next->stringInit.data);
-            allocSize += (32 + 4 + 3 + dp->length) & (-4);
-        }
-        else
-        {
-            allocSize += (32 + 4 + 3 + next->propInit.length) & (-4);
-        }
-    }
-    saveout = out = (vm_address_t) kalloc(allocSize);
-    if ( out == 0 ) return 0;
-
-    // copy out
-    for ( next = template; next; next++)
-    {
-        if ( next->nodeInit.zero == 0 )
-        {
-            if ( next->nodeInit.nProps == 0 ) break;
-            source = &next->nodeInit.nProps;
-            size = 2 * sizeof( long);
-        }
-        else if ( next->dataInit.one == 1 )
-        {
-            *((long *)next->dataInit.address) = out;
-            source = 0;
-            size   = *(next->dataInit.length);
-        }
-        else if ( next->stringInit.two == 2 )
-        {
-            dt_data *dp = (dt_data *)next->stringInit.data;
-            bcopy( (void *)(uintptr_t)next->stringInit.name, (void *)out, 32);
-            out += 32;
-            size = dp->length;
-            *(long *)out = size;
-            out += sizeof(long);
-            source = (char *)dp->address;
-            size = (size + 3) & (-4);
-        }
-        else
-        {
-            bcopy( (void *)(uintptr_t)next->propInit.name, (void *)out, 32);
-            out += 32;
-            size = next->propInit.length;
-            *(long *)out = size;
-            out += sizeof(long);
-            if ( size == 4 )
-                source = &next->propInit.value;
-            else {
-                source = next->propInit.value;
-                size = (size + 3) & (-4);
-            }
-        }
-        if ( source )
-            bcopy(source, (void *)out, size);
-        else
-            bzero((void *)out, size);
-        out += size;
-    }
-
-    if( allocSize != (out - saveout))
-        printf("WARNING: DT corrupt (%x)\n", (out - saveout) - allocSize);
-
-    *retSize = allocSize;
-    return( (void *)saveout );
-}
-
-unsigned char *nptr;
-
-#define kPropNameLength 32
-
-typedef struct property_t {
-    char                name[kPropNameLength];  // NUL terminated property name
-    unsigned long       length;         // Length (bytes) of folloing prop value
-    unsigned long       *value;         // Variable length value of property
-} property_t;
-
-typedef struct node_t {
-    unsigned long       nProperties;    // Number of props[] elements (0 => end)
-    unsigned long       nChildren;      // Number of children[] elements
-    property_t *props;      // array size == nProperties
-    struct node_t   *children;     // array size == nChildren
-} node_t;
-
-
-unsigned int indent = 0;
-
-void printdt()
-{
-    node_t *nodeptr = (node_t *)nptr;
-    unsigned long num_props    = nodeptr->nProperties;
-    unsigned long len;
-    unsigned int i, j;
-    unsigned char *sptr;
-
-    nptr = (unsigned char *)&nodeptr->props;
-    for (i=0; i < num_props; i++)
-    {
-        for (j = 0; j < indent; j++)
-            printf("    ");
-        printf("'");
-        printf("%s", nptr);
-        nptr+=32;
-        len = *((unsigned long*)nptr);
-        nptr += 4;
-        printf("'\t\t(%ld)  '", len);
-        sptr = nptr;
-        for (j = 0; j < len; j++)
-            printf("%2.2x", *nptr++);
-        printf("'\t('%s')\n", sptr);
-        if (len % 4)
-            nptr += (4 - (len % 4));
-    }
-    for (i=0; i<nodeptr->nChildren; i++)
-    {
-        indent++;
-        printdt();
-        indent--;
-    }
-}
-
diff --git a/pexpert/i386/fakePPCDeviceTree.h b/pexpert/i386/fakePPCDeviceTree.h
deleted file mode 100644
index 5c234c08e..000000000
--- a/pexpert/i386/fakePPCDeviceTree.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-
-extern dt_data gMemoryMapNode;
-extern dt_data gDriversProp;
-extern dt_data gRootpathProp;
-extern dt_data gCompatibleProp;
-
-#define kDefaultPlatformName "ACPI"
-
-dt_init fakePPCDeviceTree[] =
-{
-    NODE( 2, 1 ),
-        PROP( "name", "device-tree"),
-        STRINGPROP( "compatible", gCompatibleProp ),
-        NODE( 2, 1 ),
-            PROP( "name", "chosen" ),
-            STRINGPROP( "rootpath", gRootpathProp ),
-            DATANODE( gMemoryMapNode ),
-                PROP( "name", "memory-map" ),
-                DATAPROP( gDriversProp ),
-    NODE( 0, 0),
-};
diff --git a/pexpert/i386/fakePPCStructs.h b/pexpert/i386/fakePPCStructs.h
deleted file mode 100644
index 53447b30a..000000000
--- a/pexpert/i386/fakePPCStructs.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
- * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
- * 
- * @APPLE_LICENSE_HEADER_END@
- */
-#include <pexpert/protos.h>
-#include <pexpert/boot.h>
-
-typedef struct {
-    const char *  name;
-    unsigned long length;
-    void *        value;
-} prop_init;
-
-typedef struct {
-    long   zero;
-    long   nProps;
-    long   nChildren;
-} node_init;
-
-typedef struct {
-    long   one;
-    long * length;
-    long * address;
-} data_init;
-
-typedef struct {
-    long         two;
-    const char * name;
-    void *       data;
-} string_init;
-
-typedef union {
-    prop_init propInit;
-    node_init nodeInit;
-    data_init dataInit;
-    string_init stringInit;
-} dt_init;
-
-typedef struct {
-    long   length;
-    void * address;
-} dt_data;
-
-extern boot_args fakePPCBootArgs;
-extern unsigned char *nptr;
-
-void   printdt(void);
-void * createdt(dt_init * template, long * retSize);
-
-#define NODE(props,children)  \
-        { .nodeInit = {0, props, children }}
-
-#define INTPROP(name,value)   \
-        { .propInit = {name, 4, (void *)(uintptr_t)value }}
-
-#define PROP(name,value)      \
-        { .propInit = {name, sizeof( value), (void *)(uintptr_t)value }}
-
-#define STRINGPROP(name,value) \
-        { .stringInit = { 2, name, (void *)&(value) }}
-
-#define NULLPROP(name)        \
-        { propInit = {name, 0, (void *)0 }}
-
-#define DATAPROP(data)    \
-	{ .dataInit = {1, &((data).length), (long *) &((data).address) }}
-        
-#define DATANODE(data)    \
-	{ .dataInit = {1, &((data).length), (long *)&((data).address) }}
diff --git a/pexpert/i386/kd.c b/pexpert/i386/kd.c
index 147c1376a..7424ed2e4 100644
--- a/pexpert/i386/kd.c
+++ b/pexpert/i386/kd.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
diff --git a/pexpert/i386/pe_bootargs.c b/pexpert/i386/pe_bootargs.c
index d5c1be3b5..7c2f531a6 100644
--- a/pexpert/i386/pe_bootargs.c
+++ b/pexpert/i386/pe_bootargs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/pexpert.h>
 #include <pexpert/boot.h>
diff --git a/pexpert/i386/pe_identify_machine.c b/pexpert/i386/pe_identify_machine.c
index fe013d819..2d59f1b66 100644
--- a/pexpert/i386/pe_identify_machine.c
+++ b/pexpert/i386/pe_identify_machine.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/pexpert.h>
 #include <pexpert/protos.h>
diff --git a/pexpert/i386/pe_init.c b/pexpert/i386/pe_init.c
index 82247037d..feb0a8843 100644
--- a/pexpert/i386/pe_init.c
+++ b/pexpert/i386/pe_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * file: pe_init.c
@@ -33,72 +39,48 @@
 #include <kern/sched_prim.h>
 #include <kern/debug.h>
 
-#include "fakePPCStructs.h"
-#include "fakePPCDeviceTree.h"
 #include "boot_images.h"
 
 /* extern references */
 extern void pe_identify_machine(void * args);
-extern void initialize_screen(void *, unsigned int);
 
 /* private globals */
 PE_state_t  PE_state;
-dt_data     gMemoryMapNode;
-dt_data     gDriversProp;
-dt_data     gRootpathProp;
-dt_data     gCompatibleProp;
 
 /* Clock Frequency Info */
 clock_frequency_info_t gPEClockFrequencyInfo;
 
-void *gPEEFISystemTable     = 0;
-void *gPEEFIRuntimeServices = 0;
+void *gPEEFISystemTable;
+void *gPEEFIRuntimeServices;
 
 int PE_initialize_console( PE_Video * info, int op )
 {
     static int   last_console = -1;
-    Boot_Video   bootInfo;
-    Boot_Video * bInfo;
 
-    /*
-     * Refuse changes from outside pexpert.
-     * The video mode setup by the booter cannot be changed.
-     */
-    if ( info )
-    {
-        bootInfo.v_baseAddr = info->v_baseAddr;
-        bootInfo.v_rowBytes	 = info->v_rowBytes;
-        bootInfo.v_width     = info->v_width;
-        bootInfo.v_height    = info->v_height;
-        bootInfo.v_depth     = info->v_depth;
-        bInfo = &bootInfo;
-        if (info == &PE_state.video) {
-            bootInfo.v_display   = info->v_display;
-        } else {
-            bootInfo.v_display  = GRAPHICS_MODE;
-        }
+    if (info) {
+	info->v_offset  = 0;
+	info->v_length  = 0;
+	info->v_display = GRAPHICS_MODE;
     }
-    else
-        bInfo = 0;
 
     switch ( op ) {
 
         case kPEDisableScreen:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
             kprintf("kPEDisableScreen %d\n", last_console);
 	    if (!console_is_serial())
 		last_console = switch_to_serial_console();
             break;
 
         case kPEEnableScreen:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
             kprintf("kPEEnableScreen %d\n", last_console);
             if( last_console != -1)
                 switch_to_old_console( last_console);
             break;
 	
         default:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
             break;
     }
 
@@ -118,8 +100,9 @@ void PE_init_iokit(void)
     boolean_t bootClutInitialized = FALSE;
     boolean_t norootInitialized = FALSE;
     DTEntry             entry;
-    int			size;
+    unsigned int	size;
     void **		map;
+	boot_progress_element *bootPict;
 
     PE_init_kprintf(TRUE);
     PE_init_printf(TRUE);
@@ -129,9 +112,8 @@ void PE_init_iokit(void)
     /*
      * Fetch the CLUT and the noroot image.
      */
-	boot_progress_element * bootPict;
 
-    if( kSuccess == DTLookupEntry(0, "/chosen/memory-map", &entry)) {
+    if( kSuccess == DTLookupEntry(NULL, "/chosen/memory-map", &entry)) {
 	if( kSuccess == DTGetProperty(entry, "BootCLUT", (void **) &map, &size)) {
 	    bcopy( map[0], appleClut8, sizeof(appleClut8) );
             bootClutInitialized = TRUE;
@@ -171,7 +153,7 @@ void PE_init_iokit(void)
     vc_progress_initialize( &default_progress, default_progress_data,
                             (unsigned char *) appleClut8 );
 
-    (void) StartIOKit( PE_state.deviceTreeHead, PE_state.bootArgs, gPEEFIRuntimeServices, 0);
+    (void) StartIOKit( PE_state.deviceTreeHead, PE_state.bootArgs, gPEEFIRuntimeServices, NULL);
 }
 
 void PE_init_platform(boolean_t vm_initialized, void * _args)
@@ -183,14 +165,15 @@ void PE_init_platform(boolean_t vm_initialized, void * _args)
 
         // New EFI-style
         PE_state.bootArgs           = _args;
-        PE_state.deviceTreeHead	    = args->deviceTreeP;
+        PE_state.deviceTreeHead	    = (void *) args->deviceTreeP;
         PE_state.video.v_baseAddr   = args->Video.v_baseAddr;
         PE_state.video.v_rowBytes   = args->Video.v_rowBytes;
         PE_state.video.v_width	    = args->Video.v_width;
         PE_state.video.v_height	    = args->Video.v_height;
         PE_state.video.v_depth	    = args->Video.v_depth;
         PE_state.video.v_display    = args->Video.v_display;
-        strcpy( PE_state.video.v_pixelFormat, "PPPPPPPP");
+        strlcpy(PE_state.video.v_pixelFormat, "PPPPPPPP",
+		sizeof(PE_state.video.v_pixelFormat));
     }
 
     if (!vm_initialized) {
@@ -204,10 +187,6 @@ void PE_init_platform(boolean_t vm_initialized, void * _args)
 
         pe_identify_machine(args);
     } else {
-        DTEntry entry;
-        void *ptr;
-        uint32_t size;
-
         pe_init_debug();
     }
 }
@@ -233,7 +212,8 @@ void PE_display_icon( __unused unsigned int flags, __unused const char * name )
         vc_display_icon( &default_noroot, default_noroot_data );
 }
 
-extern boolean_t PE_get_hotkey( __unused unsigned char key )
+boolean_t
+PE_get_hotkey(__unused unsigned char key)
 {
     return (FALSE);
 }
diff --git a/pexpert/i386/pe_interrupt.c b/pexpert/i386/pe_interrupt.c
index 49afab3d5..61b306419 100644
--- a/pexpert/i386/pe_interrupt.c
+++ b/pexpert/i386/pe_interrupt.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/pexpert.h>
 #include <pexpert/protos.h>
@@ -26,6 +32,10 @@
 #include <sys/kdebug.h>
 
 
+#if CONFIG_DTRACE && DEVELOPMENT
+#include <mach/sdt.h>
+#endif
+
 void PE_incoming_interrupt(x86_saved_state_t *);
 
 
@@ -58,11 +68,11 @@ PE_incoming_interrupt(x86_saved_state_t *state)
 		interrupt = state64->isf.trapno;
 		user_mode = TRUE;
 	} else {
-	        x86_saved_state32_t	*state32;
+		x86_saved_state32_t	*state32;
 
-	        state32 = saved_state32(state);
+		state32 = saved_state32(state);
 		if (state32->cs & 0x03)
-		        user_mode = TRUE;
+			user_mode = TRUE;
 		rip = state32->eip;
 		interrupt = state32->trapno;
 	}
@@ -73,13 +83,25 @@ PE_incoming_interrupt(x86_saved_state_t *state)
 
 	vector = &PE_interrupt_handler;
 
+#if CONFIG_DTRACE && DEVELOPMENT
+            DTRACE_INT5(interrupt_start, void *, vector->nub, int, 0, 
+                        void *, vector->target, IOInterruptHandler, vector->handler,
+                        void *, vector->refCon);
+#endif
+
 	if (!lapic_interrupt(interrupt, state)) {
 		vector->handler(vector->target, NULL, vector->nub, interrupt);
 	}
 
+#if CONFIG_DTRACE && DEVELOPMENT
+            DTRACE_INT5(interrupt_complete, void *, vector->nub, int, 0, 
+                        void *, vector->target, IOInterruptHandler, vector->handler,
+                        void *, vector->refCon);
+#endif
+
 	KERNEL_DEBUG_CONSTANT(
 		MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END,
-		0, 0, 0, 0, 0);
+	   0, 0, 0, 0, 0);
 }
 
 void PE_install_interrupt_handler(void *nub,
diff --git a/pexpert/i386/pe_kprintf.c b/pexpert/i386/pe_kprintf.c
index 5501f3ddb..7db76a179 100644
--- a/pexpert/i386/pe_kprintf.c
+++ b/pexpert/i386/pe_kprintf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * file: pe_kprintf.c
@@ -30,15 +36,10 @@
 #include <kern/simple_lock.h>
 #include <i386/mp.h>
 
-/* extern references */
-extern void cnputc(char c);
-extern int  serial_init(void);
-extern void serial_putc(char c);
-
 /* Globals */
-void (*PE_kputc)(char c) = 0;
+void (*PE_kputc)(char c);
 
-unsigned int disableSerialOuput = TRUE;
+unsigned int disable_serial_output = TRUE;
 
 decl_simple_lock_data(static, kprintf_lock)
 
@@ -49,18 +50,18 @@ void PE_init_kprintf(boolean_t vm_initialized)
 	if (PE_state.initialized == FALSE)
 		panic("Platform Expert not initialized");
 
-	if (!vm_initialized)
-	{
-        simple_lock_init(&kprintf_lock, 0);
+	if (!vm_initialized) {
+		simple_lock_init(&kprintf_lock, 0);
 
-	    if (PE_parse_boot_arg("debug", &boot_arg)) 
-	        if (boot_arg & DB_KPRT) disableSerialOuput = FALSE;
+		if (PE_parse_boot_arg("debug", &boot_arg))
+			if (boot_arg & DB_KPRT)
+				disable_serial_output = FALSE;
 
-        if (!disableSerialOuput && serial_init())
-            PE_kputc = serial_putc;
-        else
-            PE_kputc = cnputc;
-    }
+		if (!disable_serial_output && serial_init())
+			PE_kputc = serial_putc;
+		else
+			PE_kputc = cnputc;
+	}
 }
 
 #ifdef MP_DEBUG
@@ -81,33 +82,34 @@ static int cpu_last_locked = 0;
 void kprintf(const char *fmt, ...)
 {
 	va_list   listp;
-    boolean_t state;
-    
-    if (!disableSerialOuput) {
-
-	/*
-	 * Spin to get kprintf lock but re-enable interrupts while failing.
-	 * This allows interrupts to be handled while waiting but
-	 * interrupts are disabled once we have the lock.
-	 */
-	state = ml_set_interrupts_enabled(FALSE);
-	while (!simple_lock_try(&kprintf_lock)) {
-             ml_set_interrupts_enabled(state);
-             ml_set_interrupts_enabled(FALSE);
-	}
-
-	if (cpu_number() != cpu_last_locked) {
-	    MP_DEBUG_KPRINTF("[cpu%d...]\n", cpu_number());
-	    cpu_last_locked = cpu_number();
+	boolean_t state;
+
+	if (!disable_serial_output) {
+
+		/*
+		 * Spin to get kprintf lock but re-enable interrupts while
+		 * failing.
+		 * This allows interrupts to be handled while waiting but
+		 * interrupts are disabled once we have the lock.
+		 */
+		state = ml_set_interrupts_enabled(FALSE);
+		while (!simple_lock_try(&kprintf_lock)) {
+			ml_set_interrupts_enabled(state);
+			ml_set_interrupts_enabled(FALSE);
+		}
+
+		if (cpu_number() != cpu_last_locked) {
+			MP_DEBUG_KPRINTF("[cpu%d...]\n", cpu_number());
+			cpu_last_locked = cpu_number();
+		}
+
+		va_start(listp, fmt);
+		_doprnt(fmt, &listp, PE_kputc, 16);
+		va_end(listp);
+
+		simple_unlock(&kprintf_lock);
+		ml_set_interrupts_enabled(state);
 	}
-
-        va_start(listp, fmt);
-        _doprnt(fmt, &listp, PE_kputc, 16);
-        va_end(listp);
-
-        simple_unlock(&kprintf_lock);
-        ml_set_interrupts_enabled(state);
-    }
 }
 
 extern void kprintf_break_lock(void);
diff --git a/pexpert/i386/pe_misc.s b/pexpert/i386/pe_misc.s
index 0759794a4..71bc46014 100644
--- a/pexpert/i386/pe_misc.s
+++ b/pexpert/i386/pe_misc.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/i386/asm.h>
 
diff --git a/pexpert/i386/pe_serial.c b/pexpert/i386/pe_serial.c
index 2bce2811c..2080b6024 100644
--- a/pexpert/i386/pe_serial.c
+++ b/pexpert/i386/pe_serial.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 /*
@@ -28,10 +34,6 @@
 #include <pexpert/protos.h>
 #include <pexpert/pexpert.h>
 
-void	serial_putc(char);
-int	serial_getc(void);
-int	serial_init(void);
-
 /* standard port addresses */
 enum {
     COM1_PORT_ADDR = 0x3f8,
diff --git a/pexpert/i386/pe_spl.c b/pexpert/i386/pe_spl.c
index 3e247bc56..b72ae8bd9 100644
--- a/pexpert/i386/pe_spl.c
+++ b/pexpert/i386/pe_spl.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/protos.h>
 
diff --git a/pexpert/pexpert/AppleBoot.h b/pexpert/pexpert/AppleBoot.h
new file mode 100644
index 000000000..44b1fd4da
--- /dev/null
+++ b/pexpert/pexpert/AppleBoot.h
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *  appleboot.h - Apple logo shown durring boot
+ *
+ *  Copyright (c) 2002 Apple Computer, Inc.
+ *
+ *  DRI: Josh de Cesare
+ */
+
+#define kAppleBootWidth		(128)
+#define kAppleBootHeight	(128)
+#define kAppleBootOffset	(0)
+#define kAppleBootFrames	(1)
+#define kAppleBootFPS		(0)
+
+const unsigned char gAppleBootPict[] = {
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x17,0x34,0x47,0x50,0x50,0x50,0x13,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x03,0x20,0x40,0x50,0x50,0x50,0x50,0x50,0x50,0x13,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1a,0x42,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0xfa,0x0a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x07,0x34,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x43,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0e,0x43,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x36,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x14,0x4a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x20,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x12,0x4b,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x0b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0b,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0xf9,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x03,0x3e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x1f,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x2f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0x06,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x15,0x4f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x27,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x3f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x45,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x19,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x40,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2f,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x15,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3b,0x03,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x34,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3f,0x07,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf7,0x47,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0xf9,0x07,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x14,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x36,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x24,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x28,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x33,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3f,0x14,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x3a,0x50,0x50,0x50,0x50,0x50,0x50,0x41,0x1f,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x3a,0x50,0x50,0x50,0x48,0x35,0x17,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x15,0x33,0x27,0x15,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x08,0x0e,0x13,0x14,0x12,0x0d,0x06,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x04,0x0b,0x12,0x17,0x19,0x15,0x11,0x08,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x11,0x26,0x39,0x44,0x4c,0x4f,0x50,0x50,0x4f,0x4d,0x4a,0x3f,0x31,0x19,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0b,0x1c,0x2e,0xf9,0x46,0x4d,0x50,0x50,0x50,0x50,0x4f,0x4b,0x42,0x34,0x1f,0x0a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf7,0x21,0x3f,0x4f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x47,0x2d,0x0d,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0d,0x25,0x3d,0xfa,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4c,0x37,0x15,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x04,0x25,0x45,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x39,0x19,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x09,0x21,0xf9,0x4d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0xf9,0x15,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x17,0x42,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x42,0x27,0x0e,0x02,0x01,0x01,0x01,0x02,0x0b,0x1f,0x38,0x4b,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x33,0x06,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x03,0x2e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x43,0x3a,0x37,0x3a,0x42,0x4d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x42,0x0e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0a,0xf9,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0x13,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0d,0x45,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4b,0x12,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x10,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0x0b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0c,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3d,0x03,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x07,0x42,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x21,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x37,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x42,0x12,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x22,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x34,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0b,0x4b,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x36,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x29,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x11,0x4f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x36,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3a,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0b,0xfa,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x46,0x08,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x26,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x15,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x40,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x31,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0e,0x4e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x49,0x08,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x25,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x29,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x39,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4c,0x0a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf7,0x47,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x10,0x4f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x22,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x12,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x29,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4b,0x07,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x36,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x42,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf9,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x42,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x46,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0xf9,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3f,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x46,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0xf7,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x43,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x0e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x3f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x1c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x3a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x33,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x34,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x46,0x04,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x2c,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x19,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x15,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x11,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0a,0xfa,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x37,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x42,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4f,0x15,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x37,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x42,0x04,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x25,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x13,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x15,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf7,0x47,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0x0b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x37,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x44,0x0a,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x44,0x0d,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0a,0xfa,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x48,0x17,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x3a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4f,0x2c,0x04,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x41,0x19,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x07,0x4a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x44,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x33,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x36,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x14,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x1f,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x3f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x0b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x04,0x44,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x20,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x20,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4a,0x07,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x04,0x43,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x30,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1c,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x0e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x3e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x36,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x14,0x4f,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x0e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x32,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x33,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x07,0x47,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4b,0x0b,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1c,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x26,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x37,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x41,0x04,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x08,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x13,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x19,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2d,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x2e,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x3f,0x03,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x03,0xf9,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4a,0x0d,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x09,0x46,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4e,0x19,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x11,0x4b,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x25,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x17,0x4d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x1b,0x4d,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x31,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x17,0x4a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4d,0x44,0x3b,0x34,0x2e,0x2c,0x2e,0x35,0x3e,0x4a,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x2e,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0e,0x41,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4a,0x37,0x1f,0x0c,0x03,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x06,0x15,0x2d,0x41,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4a,0x1f,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0xf7,0x2e,0xfa,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x4f,0xf9,0x1e,0x07,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x02,0x14,0x31,0x48,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x50,0x37,0x0c,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x0d,0x2f,0x45,0x4f,0x50,0x50,0x50,0x50,0x50,0x4f,0x43,0x2e,0x11,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x07,0x1f,0x3c,0x4c,0x50,0x50,0x50,0x50,0x4f,0x46,0x33,0x13,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x04,0x12,0x1f,0x26,0x28,0x26,0x1e,0x10,0x02,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x09,0x16,0x1e,0x1f,0x1c,0x11,0x04,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,
+	0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01,0x01
+};
diff --git a/pexpert/pexpert/Clut.h b/pexpert/pexpert/Clut.h
new file mode 100644
index 000000000..9f38cd31b
--- /dev/null
+++ b/pexpert/pexpert/Clut.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2002 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *  clut.h - Custom CLUT for xnu console and boot graphics
+ *
+ *  Copyright (c) 1999-2002 Apple Computer, Inc.
+ *
+ *  DRI: Josh de Cesare
+ */
+
+const unsigned char gClut[] = {
+	0xff,0xff,0xff, 0xbf,0xbf,0xbf, 0xbe,0xbe,0xbe, 0xbd,0xbd,0xbd,
+	0xbc,0xbc,0xbc, 0xff,0xff,0x00, 0xba,0xba,0xba, 0xb9,0xb9,0xb9,
+	0xb8,0xb8,0xb8, 0xb7,0xb7,0xb7, 0xb6,0xb6,0xb6, 0xb5,0xb5,0xb5,
+	0xb4,0xb4,0xb4, 0xb3,0xb3,0xb3, 0xb2,0xb2,0xb2, 0x00,0x00,0x00,
+	0xb1,0xb1,0xb1, 0xb0,0xb0,0xb0, 0xaf,0xaf,0xaf, 0xae,0xae,0xae,
+	0xad,0xad,0xad, 0xac,0xac,0xac, 0xab,0xab,0xab, 0xaa,0xaa,0xaa,
+	0xff,0x00,0xff, 0xa9,0xa9,0xa9, 0xa8,0xa8,0xa8, 0xa7,0xa7,0xa7,
+	0xa6,0xa6,0xa6, 0xa5,0xa5,0xa5, 0xa4,0xa4,0xa4, 0xa3,0xa3,0xa3,
+	0xa2,0xa2,0xa2, 0xa1,0xa1,0xa1, 0xa0,0xa0,0xa0, 0xff,0x00,0x00,
+	0x9f,0x9f,0x9f, 0x9e,0x9e,0x9e, 0x9d,0x9d,0x9d, 0x9c,0x9c,0x9c,
+	0x9b,0x9b,0x9b, 0x9a,0x9a,0x9a, 0xcc,0xcc,0xff, 0xcc,0xcc,0xcc,
+	0x99,0x99,0x99, 0x98,0x98,0x98, 0x97,0x97,0x97, 0x96,0x96,0x96,
+	0x95,0x95,0x95, 0x94,0x94,0x94, 0x93,0x93,0x93, 0x92,0x92,0x92,
+	0x91,0x91,0x91, 0x90,0x90,0x90, 0x8f,0x8f,0x8f, 0x8e,0x8e,0x8e,
+	0x8d,0x8d,0x8d, 0x8c,0x8c,0x8c, 0x8b,0x8b,0x8b, 0x8a,0x8a,0x8a,
+	0x89,0x89,0x89, 0x87,0x87,0x87, 0x86,0x86,0x86, 0x85,0x85,0x85,
+	0x84,0x84,0x84, 0x83,0x83,0x83, 0x82,0x82,0x82, 0x81,0x81,0x81,
+	0x80,0x80,0x80, 0x7f,0x7f,0x7f, 0x7e,0x7e,0x7e, 0x7d,0x7d,0x7d,
+	0x7c,0x7c,0x7c, 0x7b,0x7b,0x7b, 0x7a,0x7a,0x7a, 0x79,0x79,0x79,
+	0x78,0x78,0x78, 0x76,0x76,0x76, 0x75,0x75,0x75, 0x74,0x74,0x74,
+	0x73,0x73,0x73, 0x72,0x72,0x72, 0x71,0x71,0x71, 0x70,0x70,0x70,
+	0x6f,0x6f,0x6f, 0x6e,0x6e,0x6e, 0x6d,0x6d,0x6d, 0x6c,0x6c,0x6c,
+	0x6b,0x6b,0x6b, 0x6a,0x6a,0x6a, 0x69,0x69,0x69, 0x68,0x68,0x68,
+	0x67,0x67,0x67, 0x66,0x66,0x66, 0x64,0x64,0x64, 0x63,0x63,0x63,
+	0x62,0x62,0x62, 0x61,0x61,0x61, 0x60,0x60,0x60, 0x5f,0x5f,0x5f,
+	0x5e,0x5e,0x5e, 0x5d,0x5d,0x5d, 0x5c,0x5c,0x5c, 0x5b,0x5b,0x5b,
+	0x5a,0x5a,0x5a, 0x59,0x59,0x59, 0x58,0x58,0x58, 0x57,0x57,0x57,
+	0x56,0x56,0x56, 0x54,0x54,0x54, 0x53,0x53,0x53, 0x52,0x52,0x52,
+	0x51,0x51,0x51, 0x50,0x50,0x50, 0x4f,0x4f,0x4f, 0x4e,0x4e,0x4e,
+	0x4d,0x4d,0x4d, 0x4c,0x4c,0x4c, 0x4b,0x4b,0x4b, 0x4a,0x4a,0x4a,
+	0x49,0x49,0x49, 0x48,0x48,0x48, 0x47,0x47,0x47, 0x46,0x46,0x46,
+	0x45,0x45,0x45, 0x43,0x43,0x43, 0x42,0x42,0x42, 0x41,0x41,0x41,
+	0x40,0x40,0x40, 0x3f,0x3f,0x3f, 0x3e,0x3e,0x3e, 0x3d,0x3d,0x3d,
+	0x3c,0x3c,0x3c, 0x3b,0x3b,0x3b, 0x3a,0x3a,0x3a, 0x39,0x39,0x39,
+	0x38,0x38,0x38, 0x37,0x37,0x37, 0x36,0x36,0x36, 0x35,0x35,0x35,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x00,0xff,0xff, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x00,0xff,0x00, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x00,0x00,0xff, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0xdd,0x00,0x00, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x00,0xbb,0x00,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65,
+	0x65,0x65,0x65, 0x65,0x65,0x65, 0x65,0x65,0x65, 0xbb,0xbb,0xbb,
+	0x65,0x65,0x65, 0x88,0x88,0x88, 0x77,0x77,0x77, 0x55,0x55,0x55,
+	0x44,0x44,0x44, 0x22,0x22,0x22, 0x65,0x65,0x65, 0x00,0x00,0x00
+};
diff --git a/pexpert/pexpert/Makefile b/pexpert/pexpert/Makefile
index 30ff9e4c1..3c1284231 100644
--- a/pexpert/pexpert/Makefile
+++ b/pexpert/pexpert/Makefile
@@ -16,12 +16,17 @@ INSTINC_SUBDIRS_PPC = \
 INSTINC_SUBDIRS_I386 = \
 	i386
 
+INSTINC_SUBDIRS_ARM = \
+	arm
+
 EXPINC_SUBDIRS = ${INSTINC_SUBDIRS}
 
 EXPINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS_PPC}
 
 EXPINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS_I386}
 
+EXPINC_SUBDIRS_ARM = ${INSTINC_SUBDIRS_ARM}
+
 DATAFILES = \
         boot.h \
 	protos.h \
diff --git a/pexpert/pexpert/boot.h b/pexpert/pexpert/boot.h
index d703fb7c6..f26ec4aad 100644
--- a/pexpert/pexpert/boot.h
+++ b/pexpert/pexpert/boot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_BOOT_H_
 #define _PEXPERT_BOOT_H_
diff --git a/pexpert/pexpert/device_tree.h b/pexpert/pexpert/device_tree.h
index 772e43084..cb5dda3b6 100644
--- a/pexpert/pexpert/device_tree.h
+++ b/pexpert/pexpert/device_tree.h
@@ -2,23 +2,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_DEVICE_TREE_H_
 #define _PEXPERT_DEVICE_TREE_H_
@@ -209,7 +215,7 @@ extern int DTRestartEntryIteration(DTEntryIterator iterator);
 
  Get Property
 */
-extern int DTGetProperty(const DTEntry entry, const char *propertyName, void **propertyValue, int *propertySize);
+extern int DTGetProperty(const DTEntry entry, const char *propertyName, void **propertyValue, unsigned int *propertySize);
 
 /*
 -------------------------------------------------------------------------------
diff --git a/pexpert/pexpert/i386/boot.h b/pexpert/pexpert/i386/boot.h
index 22169e861..73254f6e4 100644
--- a/pexpert/pexpert/i386/boot.h
+++ b/pexpert/pexpert/i386/boot.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_I386_BOOT_H
 #define _PEXPERT_I386_BOOT_H
diff --git a/pexpert/pexpert/i386/efi.h b/pexpert/pexpert/i386/efi.h
index c8fc91a67..c345f761b 100644
--- a/pexpert/pexpert/i386/efi.h
+++ b/pexpert/pexpert/i386/efi.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2005 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 2.0 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #ifndef _PEXPERT_I386_EFI_H
@@ -483,6 +489,11 @@ typedef struct {
   VOID      *VendorTable;
 } EFI_CONFIGURATION_TABLE;
 
+typedef struct {
+  EFI_GUID  VendorGuid;
+  EFI_PTR64 VendorTable;
+} __attribute__((aligned(8))) EFI_CONFIGURATION_TABLE_64;
+
 //
 // EFI System Table
 //
diff --git a/pexpert/pexpert/i386/fb_entries.h b/pexpert/pexpert/i386/fb_entries.h
index b4dc94132..d9c14341a 100644
--- a/pexpert/pexpert/i386/fb_entries.h
+++ b/pexpert/pexpert/i386/fb_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _I386_FD_ENTRIES_H_
 #define _I386_FD_ENTRIES_H_
diff --git a/pexpert/pexpert/i386/kd_entries.h b/pexpert/pexpert/i386/kd_entries.h
index f3b328a77..ce5d6ee84 100644
--- a/pexpert/pexpert/i386/kd_entries.h
+++ b/pexpert/pexpert/i386/kd_entries.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_I386_KD_ENTRIES_H_
 #define _PEXPERT_I386_KD_ENTRIES_H_
diff --git a/pexpert/pexpert/i386/kdsoft.h b/pexpert/pexpert/i386/kdsoft.h
index 2cdfd0aa9..399fa9e6e 100644
--- a/pexpert/pexpert/i386/kdsoft.h
+++ b/pexpert/pexpert/i386/kdsoft.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_I386_KDSOFT_H_
 #define _PEXPERT_I386_KDSOFT_H_
diff --git a/pexpert/pexpert/i386/protos.h b/pexpert/pexpert/i386/protos.h
index cfcbdd9f3..b97cad5a9 100644
--- a/pexpert/pexpert/i386/protos.h
+++ b/pexpert/pexpert/i386/protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_I386_PROTOS_H
 #define _PEXPERT_I386_PROTOS_H
@@ -79,4 +85,20 @@ extern void vc_progress_initialize( void * desc,
 
 extern void vc_display_icon( void * desc, const unsigned char * data );
 
+/* ------------------------------------------------------------------------
+ * from osfmk/i386/serial_io.h
+ */
+int switch_to_serial_console(void);
+void switch_to_old_console(int);
+boolean_t console_is_serial(void);
+int serial_init(void);
+void serial_putc(char);
+int serial_getc(void);
+
+/* ------------------------------------------------------------------------
+ * from osfmk/kern/misc_protos.h
+ */
+void cnputc(char);
+int cngetc(void);
+
 #endif /* _PEXPERT_I386_PROTOS_H */
diff --git a/pexpert/pexpert/machine/boot.h b/pexpert/pexpert/machine/boot.h
index e6e841f65..97edae887 100644
--- a/pexpert/pexpert/machine/boot.h
+++ b/pexpert/pexpert/machine/boot.h
@@ -1,36 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_MACHINE_BOOT_H
 #define _PEXPERT_MACHINE_BOOT_H
 
-
 #if defined (__ppc__)
 #include "pexpert/ppc/boot.h"
 #elif defined (__i386__)
 #include "pexpert/i386/boot.h"
+#elif defined (__arm__)
+#include "pexpert/arm/boot.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _PEXPERT_MACHINE_BOOT_H */
-
diff --git a/pexpert/pexpert/machine/protos.h b/pexpert/pexpert/machine/protos.h
index 520daef69..7412f19e8 100644
--- a/pexpert/pexpert/machine/protos.h
+++ b/pexpert/pexpert/machine/protos.h
@@ -1,36 +1,41 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_MACHINE_PROTOS_H
 #define _PEXPERT_MACHINE_PROTOS_H
 
-
 #if defined (__ppc__)
 #include "pexpert/ppc/protos.h"
 #elif defined (__i386__)
 #include "pexpert/i386/protos.h"
+#elif defined (__arm__)
+#include "pexpert/arm/protos.h"
 #else
 #error architecture not supported
 #endif
 
-
 #endif /* _PEXPERT_MACHINE_PROTOS_H */
-
diff --git a/pexpert/pexpert/pe_images.h b/pexpert/pexpert/pe_images.h
index 998e96eac..18b73d8d9 100644
--- a/pexpert/pexpert/pe_images.h
+++ b/pexpert/pexpert/pe_images.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
 #include <pexpert/GearImage.h>
@@ -50,7 +56,7 @@ typedef struct boot_progress_element boot_progress_element;
 
 
 static const unsigned char * default_progress_data = gGearPict;
-static const unsigned char * default_noroot_data = 0;
+static const unsigned char * default_noroot_data;
 
 static vc_progress_element default_progress = 
 	{   0, 4|1, 1000 / kGearFPS, kGearFrames, {0, 0, 0}, 
diff --git a/pexpert/pexpert/pexpert.h b/pexpert/pexpert/pexpert.h
index d69c8cbd1..5284266a8 100644
--- a/pexpert/pexpert/pexpert.h
+++ b/pexpert/pexpert/pexpert.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_PEXPERT_H_
 #define _PEXPERT_PEXPERT_H_
@@ -51,6 +57,7 @@ void PE_init_platform(
 	boolean_t vm_initialized, 
 	void *args);
 
+
 void PE_init_kprintf(
 	boolean_t vm_initialized);
 
@@ -87,6 +94,13 @@ struct clock_frequency_info_t {
   unsigned long long cpu_frequency_hz;
   unsigned long long cpu_frequency_min_hz;
   unsigned long long cpu_frequency_max_hz;
+  unsigned long long prf_frequency_hz;
+  unsigned long long prf_frequency_min_hz;
+  unsigned long long prf_frequency_max_hz;
+  unsigned long long mem_frequency_hz;
+  unsigned long long mem_frequency_min_hz;
+  unsigned long long mem_frequency_max_hz;
+  unsigned long long fix_frequency_hz;
 };
 
 typedef struct clock_frequency_info_t clock_frequency_info_t;
@@ -110,7 +124,11 @@ void PE_install_interrupt_handler(
 
 #ifndef _FN_KPRINTF
 #define	_FN_KPRINTF
-void kprintf(const char *fmt, ...);
+void kprintf(const char *fmt, ...) __printflike(1,2);
+#endif
+
+#if CONFIG_NO_KPRINTF_STRINGS
+#define kprintf(x, ...) do {} while (0)
 #endif
 
 void init_display_putc(unsigned char *baseaddr, int rowbytes, int height);
@@ -152,11 +170,15 @@ struct PE_Video {
         unsigned long   v_depth;        /* Pixel Depth */
         unsigned long   v_display;      /* Text or Graphics */
 	char		v_pixelFormat[64];
-	long		v_resv[ 4 ];
+	unsigned long	v_offset;	/* offset into video memory to start at */
+	unsigned long	v_length;	/* length of video memory (0 for v_rowBytes * v_height) */
+	long		v_resv[ 2 ];
 };
 
 typedef struct PE_Video       PE_Video;
 
+extern void initialize_screen(PE_Video *, unsigned int);
+
 extern int PE_current_console(
 	PE_Video *info);
 
@@ -183,7 +205,7 @@ typedef struct PE_state {
 	PE_Video	video;
 	void		*deviceTreeHead;
 	void		*bootArgs;
-#if __i386__
+#if defined(i386) || defined(arm)
 	void		*fakePPCBootArgs;
 #endif
 } PE_state_t;
@@ -226,13 +248,14 @@ extern void PE_cpu_signal(
 
 extern void PE_cpu_machine_init(
 	cpu_id_t target,
-	boolean_t boot);
+	boolean_t bootb);
 
 extern void PE_cpu_machine_quiesce(
 	cpu_id_t target);
 
 extern void pe_init_debug(void);
 
+
 __END_DECLS
 
 #endif /* _PEXPERT_PEXPERT_H_ */
diff --git a/pexpert/pexpert/ppc/boot.h b/pexpert/pexpert/ppc/boot.h
index 4f6d6ee6d..3ba51feb3 100644
--- a/pexpert/pexpert/ppc/boot.h
+++ b/pexpert/pexpert/ppc/boot.h
@@ -1,27 +1,39 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
+/*
+ * NOTICE: This file was modified by McAfee Research in 2004 to introduce
+ * support for mandatory and extensible security protections.  This notice
+ * is included in support of clause 2.2 (b) of the Apple Public License,
+ * Version 2.0.
+ */
 
 #ifndef _PEXPERT_PPC_BOOT_H_
 #define _PEXPERT_PPC_BOOT_H_
@@ -57,7 +69,7 @@ typedef struct DRAMBank DRAMBank;
 
 /* Boot argument structure - passed into Mach kernel at boot time.
  */
-#define kBootArgsRevision		1
+#define kBootArgsRevision		2
 #define kBootArgsVersion1		1
 #define kBootArgsVersion2		2
 
@@ -71,6 +83,8 @@ typedef struct boot_args {
     void		*deviceTreeP;	/* Base of flattened device tree */
     unsigned long	deviceTreeLength;/* Length of flattened tree */
     unsigned long	topOfKernelData;/* Highest address used in kernel data area */
+  void                  *exdata;
+  unsigned long         exdatalen;
 } boot_args;
 
 extern boot_args passed_args;
diff --git a/pexpert/pexpert/ppc/interrupts.h b/pexpert/pexpert/ppc/interrupts.h
index 7d2ee550f..eac70164c 100644
--- a/pexpert/pexpert/ppc/interrupts.h
+++ b/pexpert/pexpert/ppc/interrupts.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _POWERMAC_INTERRUPTS_H_
 #define _POWERMAC_INTERRUPTS_H_
diff --git a/pexpert/pexpert/ppc/powermac.h b/pexpert/pexpert/ppc/powermac.h
index aed753792..82a61fcba 100644
--- a/pexpert/pexpert/ppc/powermac.h
+++ b/pexpert/pexpert/ppc/powermac.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_PPC_POWERMAC_H_
 #define _PEXPERT_PPC_POWERMAC_H_
@@ -49,9 +55,6 @@ typedef volatile unsigned long  v_u_long;
 #define reg16(reg) (*(v_u_short *)reg)
 #define reg32(reg) (*(v_u_int *)reg)
 
-/* Non-cached version of bcopy */
-extern void	bcopy_nc(char *from, char *to, int size);
-
 #endif /* ASSEMBLER */
 
 #endif /* _PEXPERT_PPC_POWERMAC_H_ */
diff --git a/pexpert/pexpert/ppc/protos.h b/pexpert/pexpert/ppc/protos.h
index e10a52852..74ed2485b 100644
--- a/pexpert/pexpert/ppc/protos.h
+++ b/pexpert/pexpert/ppc/protos.h
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_PPC_PROTOS_H_
 #define _PEXPERT_PPC_PROTOS_H_
@@ -69,13 +75,14 @@ static __inline__ unsigned int byte_reverse_word(unsigned int word) {
 //------------------------------------------------------------------------
 // from ppc/serial_io.h
 extern void initialize_serial(void * scc_phys_base, uint32_t serial_baud);
+void serial_putc(char);
+int serial_getc(void);
+void cnputc(char);
 
 
 //------------------------------------------------------------------------
 // from osfmk/ppc/POWERMAC/video_console.c
 
-extern void initialize_screen(void *, unsigned int);
-
 extern void vc_progress_initialize( void * desc,
 				    const unsigned char * data,
 				    const unsigned char * clut );
@@ -87,9 +94,11 @@ extern void vc_display_icon( void * desc,
 // from osfmk/console/panic_dialog.c
 extern void panic_ui_initialize(const unsigned char * clut);
 
-// from osfmk/ppc/serial_console.c
-extern int  switch_to_serial_console(void);
-extern void switch_to_old_console(int old_console);
+/*
+ * from osfmk/ppc/serial_console.h
+ */
+int switch_to_serial_console(void);
+void switch_to_old_console(int);
 
 typedef unsigned spl_t;
 
diff --git a/pexpert/pexpert/protos.h b/pexpert/pexpert/protos.h
index d146f7a12..cd339325c 100644
--- a/pexpert/pexpert/protos.h
+++ b/pexpert/pexpert/protos.h
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #ifndef _PEXPERT_PROTOS_H_
 #define _PEXPERT_PROTOS_H_
@@ -41,7 +47,7 @@ extern void printf(const char *fmt, ...);
 extern void interrupt_enable(void);
 extern void interrupt_disable(void);
 #if __ppc__
-extern void bcopy_nc(char *from, char *to, int size); /* uncached-safe */
+extern void bcopy_nc(const char *from, char *to, int size); /* uncached-safe */
 #else
 #define bcopy_nc bcopy
 #endif 
diff --git a/pexpert/ppc/pe_bootargs.c b/pexpert/ppc/pe_bootargs.c
index cf67d7205..a0d2b2a08 100644
--- a/pexpert/ppc/pe_bootargs.c
+++ b/pexpert/ppc/pe_bootargs.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/pexpert.h>
 #include <pexpert/boot.h>
diff --git a/pexpert/ppc/pe_clock_speed.c b/pexpert/ppc/pe_clock_speed.c
index ce7d4c072..4a859e2b0 100644
--- a/pexpert/ppc/pe_clock_speed.c
+++ b/pexpert/ppc/pe_clock_speed.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *  pe_clock_speed.c - Determine the best guess for the processor and bus
@@ -37,6 +43,7 @@
 extern void pe_run_clock_test(void *tmp);
 void pe_do_clock_test(unsigned int via_addr,
 		      int num_speeds, unsigned long *speed_list);
+void PE_Determine_Clock_Speeds(unsigned int via_addr, int num_speeds, unsigned long *speed_list);
 
 // Threshold for bus speed matches.
 #define kMaxFreqDiff  (30000)
diff --git a/pexpert/ppc/pe_clock_speed_asm.s b/pexpert/ppc/pe_clock_speed_asm.s
index dfcd81a64..41e9a0fb6 100644
--- a/pexpert/ppc/pe_clock_speed_asm.s
+++ b/pexpert/ppc/pe_clock_speed_asm.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  *  clock_speed_asm.s - Uses the Via timer, decrementer, and counter
diff --git a/pexpert/ppc/pe_identify_machine.c b/pexpert/ppc/pe_identify_machine.c
index 7b20bcdd7..993b124f9 100644
--- a/pexpert/ppc/pe_identify_machine.c
+++ b/pexpert/ppc/pe_identify_machine.c
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <pexpert/protos.h>
 #include <pexpert/pexpert.h>
@@ -25,6 +31,12 @@
 #include <pexpert/device_tree.h>
 #include <vm/pmap.h>
 
+extern void panic(const char *str, ...);
+
+/* Local declarations */
+void pe_identify_machine(void);
+vm_offset_t get_io_base_addr(void);
+
 /* pe_identify_machine:
  *
  *   Sets up platform parameters.
@@ -34,7 +46,7 @@ void pe_identify_machine(void)
 {
   DTEntry       cpu, root;
   unsigned long *value;
-  int           size;
+  unsigned int  size;
 
   // Clear the gPEClockFrequencyInfo struct
   bzero((void *)&gPEClockFrequencyInfo, sizeof(clock_frequency_info_t));
@@ -115,7 +127,7 @@ vm_offset_t get_io_base_addr(void)
 {
   DTEntry     entryP;
   vm_offset_t *address;
-  int         size;
+  unsigned int size;
   
   if ((DTFindEntry("device_type", "dbdma", &entryP) == kSuccess)
       || (DTFindEntry("device_type", "mac-io", &entryP) == kSuccess))
diff --git a/pexpert/ppc/pe_init.c b/pexpert/ppc/pe_init.c
index 38ff042ca..11a7a2a1a 100644
--- a/pexpert/ppc/pe_init.c
+++ b/pexpert/ppc/pe_init.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * file: pe_init.c
@@ -52,15 +58,15 @@ static int PE_stub_read_write_time_of_day(unsigned int options, long * secs)
     return 0;
 }
 
-static int PE_stub_poll_input(unsigned int options, char * c)
+static int PE_stub_poll_input(__unused unsigned int options, char * c)
 {
     *c = 0xff;
 
     return 1;
 }
 
-static int PE_stub_write_IIC(unsigned char addr, unsigned char reg,
-				unsigned char data)
+static int PE_stub_write_IIC(__unused unsigned char addr, __unused unsigned char reg,
+				__unused unsigned char data)
 {
     return 1;
 }
@@ -78,37 +84,30 @@ int (*PE_write_IIC)(unsigned char addr, unsigned char reg,
 int PE_initialize_console( PE_Video * info, int op )
 {
     static int		last_console = -1;
-    Boot_Video		bootInfo;
-    Boot_Video	*	bInfo;
-
-    if( info) {
-        bootInfo.v_baseAddr		= info->v_baseAddr;
-        bootInfo.v_rowBytes		= info->v_rowBytes;
-        bootInfo.v_width		= info->v_width;
-        bootInfo.v_height		= info->v_height;
-        bootInfo.v_depth		= info->v_depth;
-        bootInfo.v_display		= 0;
-	bInfo = &bootInfo;
-    } else
-	bInfo = 0;
+
+    if (info) {
+	info->v_offset  = 0;
+	info->v_length  = 0;
+	info->v_display = 0;
+    }
 
     switch( op ) {
 
 	case kPEDisableScreen:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
             last_console = switch_to_serial_console();
             kprintf("kPEDisableScreen %d\n",last_console);
 	    break;
 
 	case kPEEnableScreen:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
             kprintf("kPEEnableScreen %d\n",last_console);
             if( last_console != -1)
                 switch_to_old_console( last_console);
 	    break;
 	
 	default:
-            initialize_screen((void *) bInfo, op);
+            initialize_screen(info, op);
 	    break;
     }
 
@@ -118,14 +117,14 @@ int PE_initialize_console( PE_Video * info, int op )
 void PE_init_iokit(void)
 {
     kern_return_t	ret;
-    DTEntry		entry;
-    int			size;
-    void **		map;
+    DTEntry			entry;
+    unsigned int	size;
+    void **			map;
 
     PE_init_kprintf(TRUE);
     PE_init_printf(TRUE);
 
-    if( kSuccess == DTLookupEntry(0, "/chosen/memory-map", &entry)) {
+    if( kSuccess == DTLookupEntry(NULL, "/chosen/memory-map", &entry)) {
 
 	boot_progress_element * bootPict;
 
@@ -152,7 +151,7 @@ void PE_init_platform(boolean_t vm_initialized, void *_args)
 {
 	DTEntry dsouth, dnorth, root, dcpu;
 	char *model;
-	int msize, size;
+	unsigned int msize, size;
 	uint32_t *south, *north, *pdata, *ddata;
 	int i;
 	
@@ -169,7 +168,8 @@ void PE_init_platform(boolean_t vm_initialized, void *_args)
 	  PE_state.video.v_height	= args->Video.v_height;
 	  PE_state.video.v_depth	= args->Video.v_depth;
 	  PE_state.video.v_display	= args->Video.v_display;
-	  strcpy( PE_state.video.v_pixelFormat, "PPPPPPPP");
+	  strlcpy(PE_state.video.v_pixelFormat, "PPPPPPPP",
+		  sizeof(PE_state.video.v_pixelFormat));
 	}
 
 	if (!vm_initialized)
@@ -202,25 +202,25 @@ void PE_create_console( void )
 int PE_current_console( PE_Video * info )
 {
     *info = PE_state.video;
-    info->v_baseAddr = 0;
+
     return( 0);
 }
 
-void PE_display_icon(	unsigned int flags,
-			const char * name )
+void PE_display_icon(	__unused unsigned int flags,
+			__unused const char * name )
 {
     if( default_noroot_data)
 	vc_display_icon( &default_noroot, default_noroot_data );
 }
 
-extern boolean_t PE_get_hotkey(
-	unsigned char	key)
+boolean_t
+PE_get_hotkey(unsigned char key)
 {
     unsigned char * adbKeymap;
-    int		size;
-    DTEntry	entry;
+    unsigned int	size;
+    DTEntry			entry;
 
-    if( (kSuccess != DTLookupEntry( 0, "/", &entry))
+    if( (kSuccess != DTLookupEntry(NULL, "/", &entry))
     ||  (kSuccess != DTGetProperty( entry, "AAPL,adb-keymap",
             (void **)&adbKeymap, &size))
     || (size != 16))
diff --git a/pexpert/ppc/pe_kprintf.c b/pexpert/ppc/pe_kprintf.c
index 58154df8a..9dffd0de0 100644
--- a/pexpert/ppc/pe_kprintf.c
+++ b/pexpert/ppc/pe_kprintf.c
@@ -1,23 +1,29 @@
 /*
- * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * file: pe_kprintf.c
@@ -34,27 +40,23 @@
 #include <vm/pmap.h>
 
 /* extern references */
-extern void init_display_putc(unsigned char*, int, int);
-extern void display_putc(char c);
-extern int scc_putc(int unit, int line, int c);
-extern void cnputc(char c);
-
-/* Internal routines -- eventually put this in serial driver */
-void serial_putc(char c);
+extern void scc_putc(int unit, int line, int c);
+extern long strtol(const char *, char **, int);
 
 /* Globals */
-void (*PE_kputc)(char c) = 0;
+void (*PE_kputc)(char c);
 
-unsigned int disableSerialOuput = TRUE;
+unsigned int disable_serial_output = TRUE;
 
 vm_offset_t	scc = 0;
 
 struct slock kprintf_lock;
 
-void PE_init_kprintf(boolean_t vm_initialized)
+void PE_init_kprintf(__unused boolean_t vm_initialized)
 {
 	unsigned int	boot_arg;
-	int32_t			cnt, size, serial_baud = -1;
+	int32_t			serial_baud = -1;
+	unsigned int	size;
 	DTEntry         options;
 	char            *str, baud[7];
 
@@ -62,28 +64,28 @@ void PE_init_kprintf(boolean_t vm_initialized)
 		panic("Platform Expert not initialized");
 
 	if (PE_parse_boot_arg("debug", &boot_arg))
-		if(boot_arg & DB_KPRT) disableSerialOuput = FALSE; 
+		if(boot_arg & DB_KPRT) disable_serial_output = FALSE; 
 
-	if (DTLookupEntry(0, "/options", &options) == kSuccess) {
-	  if (DTGetProperty(options, "input-device", &str, &size) == kSuccess) {
+	if (DTLookupEntry(NULL, "/options", &options) == kSuccess) {
+	  if (DTGetProperty(options, "input-device", (void **)&str, &size) == kSuccess) {
 		if ((size > 5) && !strncmp("scca:", str, 5)) {
 		  size -= 5;
 		  str += 5;
 		  if (size <= 6) {
 			strncpy(baud, str, size);
 			baud[size] = '\0';
-			gPESerialBaud = strtol(baud, 0, 0);
+			gPESerialBaud = strtol(baud, NULL, 0);
 		  }
 		}
 	  }
-	  if (DTGetProperty(options, "output-device", &str, &size) == kSuccess) {
+	  if (DTGetProperty(options, "output-device", (void **)&str, &size) == kSuccess) {
 		if ((size > 5) && !strncmp("scca:", str, 5)) {
 		  size -= 5;
 		  str += 5;
 		  if (size <= 6) {
 			strncpy(baud, str, size);
 			baud[size] = '\0';
-			gPESerialBaud = strtol(baud, 0, 0);
+			gPESerialBaud = strtol(baud, NULL, 0);
 		  }
 		}
 	  }	  
@@ -127,13 +129,9 @@ void PE_init_kprintf(boolean_t vm_initialized)
 
 void serial_putc(char c)
 {
-	(void) scc_putc(0, 1, c);
-	if (c == '\n') (void) scc_putc(0, 1, '\r');
-
-#if 0
-	(void) scc_putc(0, (int)PE_state.debug_video.v_baseAddr, c);
-	if (c == '\n') (void) scc_putc(0, (int)PE_state.debug_video.v_baseAddr, '\r');
-#endif
+	scc_putc(0, 1, c);
+	if (c == '\n')
+		scc_putc(0, 1, '\r');
 }
 
 void kprintf(const char *fmt, ...)
@@ -144,7 +142,7 @@ void kprintf(const char *fmt, ...)
 	state = ml_set_interrupts_enabled(FALSE);
 	simple_lock(&kprintf_lock);
 	
-	if (!disableSerialOuput) {	
+	if (!disable_serial_output) {	
         	va_start(listp, fmt);
         	_doprnt(fmt, &listp, PE_kputc, 16);
         	va_end(listp);
diff --git a/pexpert/ppc/pe_misc.s b/pexpert/ppc/pe_misc.s
index 8cda8649f..55c2d4cc1 100644
--- a/pexpert/ppc/pe_misc.s
+++ b/pexpert/ppc/pe_misc.s
@@ -1,23 +1,29 @@
 /*
  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
  *
- * @APPLE_LICENSE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * 
- * The contents of this file constitute Original Code as defined in and
- * are subject to the Apple Public Source License Version 1.1 (the
- * "License").  You may not use this file except in compliance with the
- * License.  Please obtain a copy of the License at
- * http://www.apple.com/publicsource and read it before using this file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
  * 
- * This Original Code and all software distributed under the License are
- * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
- * License for the specific language governing rights and limitations
- * under the License.
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
  * 
- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 #include <mach/ppc/asm.h>
 
diff --git a/security/Makefile b/security/Makefile
new file mode 100644
index 000000000..34ab8930c
--- /dev/null
+++ b/security/Makefile
@@ -0,0 +1,53 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+INSTINC_SUBDIRS = \
+
+INSTINC_SUBDIRS_PPC =  \
+
+INSTINC_SUBDIRS_I386 = \
+
+EXPINC_SUBDIRS = \
+
+EXPINC_SUBDIRS_PPC = \
+
+EXPINC_SUBDIRS_I386 = \
+
+DATAFILES = \
+	mac.h \
+	mac_policy.h
+
+PRIVATE_DATAFILES = \
+	_label.h \
+	mac_alloc.h \
+	mac_data.h \
+	mac_framework.h \
+	mac_mach_internal.h \
+	mac_internal.h
+
+# Installed in /usr/include/security/
+INSTALL_MI_LIST = ${DATAFILES} 
+
+INSTALL_MI_DIR = security
+
+EXPORT_MI_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
+
+EXPORT_MI_DIR = security
+
+# /System/Library/Frameworks/System.framework/PrivateHeaders
+INSTALL_MI_LCL_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
+
+# /System/Library/Frameworks/Kernel.framework/PrivateHeaders
+INSTALL_KF_MI_LCL_LIST = ${DATAFILES} ${PRIVATE_DATAFILES}
+
+SETUP_SUBDIRS = conf
+
+COMP_SUBDIRS = conf
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/security/_label.h b/security/_label.h
new file mode 100644
index 000000000..93b57b4f2
--- /dev/null
+++ b/security/_label.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001, 2002 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson for the TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/sys/sys/_label.h,v 1.4 2003/05/08 19:49:42 rwatson Exp $
+ */
+#ifndef _SECURITY_LABEL_H_
+#define	_SECURITY_LABEL_H_
+
+/*
+ * XXXMAC: This shouldn't be exported to userland, but is because of ucred.h
+ * and various other messes.
+ */
+
+#define	MAC_MAX_SLOTS	8
+
+#define	MAC_FLAG_INITIALIZED	0x0000001	/* Is initialized for use. */
+
+struct label {
+	int	l_flags;
+	union {
+		void	*l_ptr;
+		long	 l_long;
+	}	l_perpolicy[MAC_MAX_SLOTS];
+};
+
+#endif /* !_SECURITY_LABEL_H_ */
diff --git a/security/conf/MASTER b/security/conf/MASTER
new file mode 100644
index 000000000..d3fcf6d1f
--- /dev/null
+++ b/security/conf/MASTER
@@ -0,0 +1,66 @@
+#
+# Mach Operating System
+# Copyright (c) 1986 Carnegie-Mellon University
+# All rights reserved.  The CMU software License Agreement
+# specifies the terms and conditions for use and redistribution.
+#  
+#######################################################################
+#
+#	Master machine independent configuration file.
+#
+#	Specific configuration files are created based on this and
+#	the machine specific master file using the doconf script.
+#
+#	Any changes to the master configuration files will affect all
+#	other configuration files based upon it.
+#
+#######################################################################
+#
+#	To build a configuration, execute "doconf <configuration>."
+#	Configurations are specified in the "Configurations:" section
+#	of the MASTER and MASTER.* files as follows:
+#
+#	<configuration> = [ <attribute0> <attribute1> ... <attributeN> ]
+#
+#	Lines in the MASTER and MASTER.* files are selected based on
+#	the attribute selector list, found in a comment at the end of
+#	the line.  This is a list of attributes separated by commas.
+#	The "!" operator selects the line if none of the attributes are
+#	specified.
+#
+#	For example:
+#
+#	<foo,bar>	selects a line if "foo" or "bar" are specified.
+#	<!foo,bar>	selects a line if neither "foo" nor "bar" is
+#			specified.
+#
+#	Lines with no attributes specified are selected for all
+#	configurations.
+#
+#######################################################################
+#  
+#  Standard Mach Research Configurations:
+#  -------- ---- -------- ---------------
+#
+#  These are the default configurations that can be used by most sites.
+#  They are used internally by the Mach project.
+#
+#  LIBSA      = [debug]
+#
+#######################################################################
+#
+ident		SECURITY
+
+#
+# Note: MAC options must be set in both bsd/conf and security/conf MASTER files
+#
+options		KDEBUG		# kernel tracing		# <kdebug>
+options		AUDIT		# Security event auditing
+options		CONFIG_LCTX	# Login Context
+
+options		CONFIG_DTRACE 	# dtrace support	# <config_dtrace>
+
+options		CONFIG_NO_PANIC_STRINGS			# <no_panic_str>
+options		CONFIG_NO_PRINTF_STRINGS		# <no_printf_str>
+options		CONFIG_NO_KPRINTF_STRINGS		# <no_kprintf_str>
+
diff --git a/security/conf/MASTER.i386 b/security/conf/MASTER.i386
new file mode 100644
index 000000000..01b3a55d2
--- /dev/null
+++ b/security/conf/MASTER.i386
@@ -0,0 +1,24 @@
+######################################################################
+#
+#  RELEASE	= [ intel mach libkerncpp config_dtrace ]
+#  PROFILE	= [ RELEASE profile ]
+#  DEBUG	= [ RELEASE debug ]
+#
+#  EMBEDDED	= [ intel mach libkerncpp ]
+#  DEVELOPMENT	= [ EMBEDDED config_dtrace ]
+#
+######################################################################
+
+#
+# Note: MAC options must be set in both bsd/conf and security/conf MASTER files
+#
+options		CONFIG_MACF			# Mandatory Access Control Framework
+options		CONFIG_MACF_SOCKET_SUBSET	# MACF subset of socket support
+#options	CONFIG_MACF_SOCKET
+#options	CONFIG_MACF_NET
+#options	CONFIG_MACF_ALWAYS_LABEL_MBUF
+#options	CONFIG_MACF_DEBUG
+#options	CONFIG_MACF_MACH
+
+machine		"i386"					# <intel>
+cpu		"i386"					# <intel>
diff --git a/security/conf/MASTER.ppc b/security/conf/MASTER.ppc
new file mode 100644
index 000000000..177301b38
--- /dev/null
+++ b/security/conf/MASTER.ppc
@@ -0,0 +1,29 @@
+#
+######################################################################
+#  
+#  Standard Apple MacOS X Configurations:
+#  -------- ---- -------- ---------------
+#
+#  RELEASE = [ppc mach libkerncpp config_dtrace]
+#  DEVELOPMENT = [RELEASE]
+#  PROFILE = [RELEASE]
+#  DEBUG = [RELEASE debug]
+#  RELEASE_TRACE = [ RELEASE kdebug ]
+#  DEBUG_TRACE   = [ DEBUG kdebug ]
+#
+######################################################################
+
+#
+# Note: corresponding MACF options must be set in both security/conf
+#       bsd/conf and/or osfmk/conf MASTER files (depending upon the option)
+#
+options		CONFIG_MACF			# Mandatory Access Control Framework
+options		CONFIG_MACF_SOCKET_SUBSET	# MACF subset of socket support
+#options	CONFIG_MACF_SOCKET
+#options	CONFIG_MACF_NET
+#options	CONFIG_MACF_ALWAYS_LABEL_MBUF
+#options	CONFIG_MACF_DEBUG
+#options	CONFIG_MACF_MACH
+
+machine		"ppc"					# <ppc>
+cpu		"ppc"					# <ppc>
diff --git a/security/conf/Makefile b/security/conf/Makefile
new file mode 100644
index 000000000..f8934f813
--- /dev/null
+++ b/security/conf/Makefile
@@ -0,0 +1,64 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+SETUP_SUBDIRS = \
+	tools
+
+COMP_SUBDIRS = 
+
+INST_SUBDIRS = 
+
+ifndef SECURITY_KERNEL_CONFIG
+export SECURITY_KERNEL_CONFIG = $(KERNEL_CONFIG)
+endif
+
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)
+else
+export COMPOBJROOT=$(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)
+endif
+
+$(COMPOBJROOT)/doconf:
+	@make build_setup 
+
+$(COMPOBJROOT)/$(SECURITY_KERNEL_CONFIG)/Makefile :  $(SOURCE)/MASTER  \
+	$(SOURCE)/MASTER.$(ARCH_CONFIG_LC) \
+	$(SOURCE)/Makefile.template  \
+	$(SOURCE)/Makefile.$(ARCH_CONFIG_LC)  \
+	$(SOURCE)/files \
+	$(SOURCE)/files.$(ARCH_CONFIG_LC) \
+	$(COMPOBJROOT)/doconf
+	$(_v)(doconf_target=$(addsuffix /conf, $(TARGET)); \
+	echo $${doconf_target};\
+	$(MKDIR) $${doconf_target}; \
+	cd $${doconf_target}; \
+	rm -f $(notdir $?); \
+	cp  $? $${doconf_target}; \
+	$(COMPOBJROOT)/doconf -c -cpu $(ARCH_CONFIG_LC) -d  $(TARGET)/$(SECURITY_KERNEL_CONFIG) $(SECURITY_KERNEL_CONFIG); \
+	);
+
+.ORDER: $(COMPOBJROOT)/$(SECURITY_KERNEL_CONFIG)/Makefile
+
+do_setup_conf: $(COMPOBJROOT)/doconf \
+		$(COMPOBJROOT)/$(SECURITY_KERNEL_CONFIG)/Makefile
+
+do_all: do_setup_conf
+	$(_v)next_source=$(subst conf/,,$(SOURCE));			\
+	${MAKE} -C $(COMPOBJROOT)/$(SECURITY_KERNEL_CONFIG)	\
+		MAKEFILES=$(TARGET)/$(SECURITY_KERNEL_CONFIG)/Makefile	\
+		SOURCE=$${next_source}			\
+		TARGET=$(TARGET)					\
+		INCL_MAKEDEP=FALSE	\
+		KERNEL_CONFIG=$(SECURITY_KERNEL_CONFIG)	\
+		build_all;
+
+do_build_all: do_all 
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/security/conf/Makefile.i386 b/security/conf/Makefile.i386
new file mode 100644
index 000000000..b89fdd145
--- /dev/null
+++ b/security/conf/Makefile.i386
@@ -0,0 +1,8 @@
+######################################################################
+#BEGIN	Machine dependent Makefile fragment for i386
+######################################################################
+
+
+######################################################################
+#END	Machine dependent Makefile fragment for i386
+######################################################################
diff --git a/security/conf/Makefile.ppc b/security/conf/Makefile.ppc
new file mode 100644
index 000000000..2b438f2fa
--- /dev/null
+++ b/security/conf/Makefile.ppc
@@ -0,0 +1,8 @@
+######################################################################
+#BEGIN	Machine dependent Makefile fragment for ppc
+######################################################################
+
+
+######################################################################
+#END	Machine dependent Makefile fragment for ppc
+######################################################################
diff --git a/security/conf/Makefile.template b/security/conf/Makefile.template
new file mode 100644
index 000000000..b5a57e158
--- /dev/null
+++ b/security/conf/Makefile.template
@@ -0,0 +1,105 @@
+#
+# Mach Operating System
+# Copyright (c) 1986 Carnegie-Mellon University
+# All rights reserved.  The CMU software License Agreement specifies
+# the terms and conditions for use and redistribution.
+#
+
+#
+# Export IDENT for sub-makefiles
+#
+export IDENT
+
+#
+# INCFLAGS to include security prototypes
+#
+INCFLAGS_MAKEFILE= -I$(SOURCE) -I$(SOURCE)/..
+
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+#
+# XXX: CFLAGS
+#
+CFLAGS+= -DKERNEL -DBSD_KERNEL_PRIVATE \
+	-Wall -Wno-four-char-constants -fno-common
+
+#
+# Directories for mig generated files
+#
+COMP_SUBDIRS =
+
+
+#  Make sure we don't remove this by accident if interrupted at the wrong
+#  time.
+#
+.PRECIOUS: Makefile
+
+VERSION_FILES= \
+	$(SOURCE_DIR)/$(COMPONENT)/conf/version.major  \
+	$(SOURCE_DIR)/$(COMPONENT)/conf/version.minor   \
+	$(SOURCE_DIR)/$(COMPONENT)/conf/version.variant
+
+COPYRIGHT_FILES = \
+	$(SOURCE_DIR)/$(COMPONENT)/conf/copyright.nai
+
+#
+#  Theses macros are filled in by the config program depending on the
+#  current configuration.  The MACHDEP macro is replaced by the
+#  contents of the machine dependent makefile template and the others
+#  are replaced by the corresponding symbol definitions for the
+#  configuration.
+#
+
+%OBJS
+
+%CFILES
+
+%MFILES
+
+%SFILES
+
+%BFILES
+
+%ORDERED
+%MACHDEP
+
+#
+#  OBJSDEPS is the set of files (defined in the machine dependent
+#  template if necessary) which all objects depend on (such as an
+#  in-line assembler expansion filter)
+#
+${OBJS}: ${OBJSDEPS}
+
+
+%LOAD
+
+LDOBJS = $(OBJS)
+
+$(COMPONENT).o: $(LDOBJS)
+	$(_v)$(RM) $(RMFLAGS) vers.c
+	$(_v)$(COMPOBJROOT)/newvers \
+	`$(CAT) ${VERSION_FILES}` ${COPYRIGHT_FILES}
+	$(_v)${KCC} $(CFLAGS) $(INCLUDES) -c vers.c
+	@echo LD $(COMPONENT)
+	$(_v)$(LD)  $(LDFLAGS_COMPONENT)  -o $(COMPONENT).o ${LDOBJS} vers.o 
+
+do_all: $(COMPONENT).o
+
+do_depend: do_all
+	$(_v)${MD} -u Makedep -f -d `ls *.d`;
+	
+do_build_all: do_depend
+
+${SOBJS}: 
+
+
+%RULES
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/bsd/net/if_faith.h b/security/conf/copyright.nai
similarity index 60%
rename from bsd/net/if_faith.h
rename to security/conf/copyright.nai
index 953c9b081..60a27a56a 100644
--- a/bsd/net/if_faith.h
+++ b/security/conf/copyright.nai
@@ -1,10 +1,12 @@
-/*	$FreeBSD: src/sys/net/if_faith.h,v 1.1.2.2 2001/07/05 14:46:25 ume Exp $	*/
-/*	$KAME: if_faith.h,v 1.1 2000/07/26 05:49:21 itojun Exp $	*/
-
-/*
- * Copyright (C) 2000 WIDE Project.
+/*-
+ * Copyright (c) 2003 Networks Associates Technology, Inc.
  * All rights reserved.
  *
+ * This software was developed for the FreeBSD Project by and Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -13,14 +15,11 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the project nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -28,17 +27,6 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ * $FreeBSD$
  */
-
-#ifndef _NET_IF_FAITH_H_
-#define _NET_IF_FAITH_H_
-#include <sys/appleapiopts.h>
-
-#ifdef KERNEL_PRIVATE
-#if INET6
-struct in6_addr;
-int faithprefix(struct in6_addr *);
-#endif /* INET6 */
-#endif /* KERNEL_PRIVATE */
-
-#endif /* _NET_IF_FAITH_H_ */
diff --git a/security/conf/files b/security/conf/files
new file mode 100644
index 000000000..c0565103d
--- /dev/null
+++ b/security/conf/files
@@ -0,0 +1,28 @@
+# options
+
+# OPTIONS/kdebug                        optional kdebug
+# security
+
+security/mac_alloc.c					optional config_macf
+security/mac_audit.c					optional config_macf
+security/mac_base.c					standard
+security/mac_data.c					optional config_macf
+security/mac_label.c					optional config_macf
+security/mac_task.c					optional config_macf_mach
+security/mac_port.c					optional config_macf_mach
+security/mac_process.c					optional config_macf
+security/mac_vfs.c					optional config_macf
+security/mac_vfs_subr.c					optional config_macf
+security/mac_system.c					optional config_macf
+security/mac_sysv_sem.c					optional config_macf
+security/mac_sysv_shm.c					optional config_macf
+security/mac_sysv_msg.c					optional config_macf
+security/mac_posix_sem.c				optional config_macf
+security/mac_posix_shm.c				optional config_macf
+security/mac_socket.c					optional config_macf
+security/mac_stub.c					standard
+security/mac_net.c					optional config_macf_net
+security/mac_pipe.c					optional config_macf
+security/mac_iokit.c					optional config_macf
+security/mac_file.c					optional config_macf
+security/mac_inet.c					optional config_macf_net
diff --git a/security/conf/files.i386 b/security/conf/files.i386
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/security/conf/files.i386
@@ -0,0 +1 @@
+
diff --git a/security/conf/files.ppc b/security/conf/files.ppc
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/security/conf/files.ppc
@@ -0,0 +1 @@
+
diff --git a/security/conf/kernelversion.major b/security/conf/kernelversion.major
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/security/conf/kernelversion.major
@@ -0,0 +1 @@
+6
diff --git a/security/conf/kernelversion.minor b/security/conf/kernelversion.minor
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/security/conf/kernelversion.minor
@@ -0,0 +1 @@
+6
diff --git a/security/conf/kernelversion.variant b/security/conf/kernelversion.variant
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/security/conf/kernelversion.variant
@@ -0,0 +1 @@
+
diff --git a/iokit/include/Makefile b/security/conf/tools/Makefile
similarity index 54%
rename from iokit/include/Makefile
rename to security/conf/tools/Makefile
index a813d3ea6..a8111c252 100644
--- a/iokit/include/Makefile
+++ b/security/conf/tools/Makefile
@@ -7,22 +7,24 @@ export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
 include $(MakeInc_cmd)
 include $(MakeInc_def)
 
-INSTINC_SUBDIRS = bsddev drivers
-INSTINC_SUBDIRS_PPC = ${INSTINC_SUBDIRS}
-INSTINC_SUBDIRS_I386 = ${INSTINC_SUBDIRS}
+SETUP_SUBDIRS = \
+	doconf \
+	newvers
 
+COMP_SUBDIRS = \
+	doconf \
+	newvers
 
-EXPINC_SUBDIRS =
-EXPINC_SUBDIRS_PPC = ${EXPINC_SUBDIRS}
-EXPINC_SUBDIRS_I386 = ${EXPINC_SUBDIRS}
+INST_SUBDIRS = \
 
 
-SETUP_SUBDIRS =
+setup_build_all:
 
-COMP_SUBDIRS = 
+do_build_all:
 
-INST_SUBDIRS =
+setup_build_install:
 
+do_build_install:
 
 include $(MakeInc_rule)
 include $(MakeInc_dir)
diff --git a/security/conf/tools/doconf/Makefile b/security/conf/tools/doconf/Makefile
new file mode 100644
index 000000000..7794a4ceb
--- /dev/null
+++ b/security/conf/tools/doconf/Makefile
@@ -0,0 +1,49 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+COMP_SUBDIRS = \
+
+INST_SUBDIRS = \
+
+
+#
+# Who and where
+#
+BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
+PROGRAM= $(DSTDIR)doconf
+
+# 
+# How to install it
+#
+IFLAGS= -c -m 555
+
+$(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+		< $< >$(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+
+do_build_setup: $(PROGRAM)
+
+do_build_all:
+
+setup_build_install:
+
+do_build_install:
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
+
+
diff --git a/security/conf/tools/doconf/doconf.csh b/security/conf/tools/doconf/doconf.csh
new file mode 100644
index 000000000..6fedb4786
--- /dev/null
+++ b/security/conf/tools/doconf/doconf.csh
@@ -0,0 +1,321 @@
+#!/bin/csh -f
+set path = ($path .)
+######################################################################
+# HISTORY
+#  1-Dec-87  Michael Young (mwyoung) at Carnegie-Mellon University
+#	Added "-verbose" switch, so this script produces no output
+#	in the normal case.
+#
+# 10-Oct-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Flushed cmu_*.h and spin_locks.h
+#	[ V5.1(XF18) ]
+#
+#  6-Apr-87  Avadis Tevanian (avie) at Carnegie-Mellon University
+#	Use MASTER.local and MASTER.<machine>.local for generation of
+#	configuration files in addition to MASTER and MASTER.<machine>.
+#
+# 25-Mar-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Removed use of obsolete wb_*.h files when building the feature
+#	list;  modified to save the previous configuration file and
+#	display the differences between it and the new file.
+#	[ V5.1(F8) ]
+#
+# 25-Mar-87  Avadis Tevanian (avie) at Carnegie-Mellon University
+#	If there is no /etc/machine just print out a message telling
+#	user to use the -cpu option.  I thought this script was supposed
+#	to work even without a /etc/machine, but it doesn't... and this
+#	is the easiest way out.
+#
+# 13-Mar-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Added "romp_fpa.h" file to extra features for the RT.
+#	[ V5.1(F7) ]
+#
+# 11-Mar-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Updated to maintain the appropriate configuration features file
+#	in the "machine" directory whenever the corresponding
+#	configuration is generated.  This replaces the old mechanism of
+#	storing this directly in the <sys/features.h> file since it was
+#	machine dependent and also precluded building programs for more
+#	than one configuration from the same set of sources.
+#	[ V5.1(F6) ]
+#
+# 21-Feb-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Fixed to require wired-in cpu type names for only those
+#	machines where the kernel name differs from that provided by
+#	/etc/machine (i.e. IBMRT => ca and SUN => sun3);  updated to
+#	permit configuration descriptions in both machine indepedent
+#	and dependent master configuration files so that attributes can
+#	be grouped accordingly.
+#	[ V5.1(F3) ]
+#
+# 17-Jan-87  Mike Accetta (mja) at Carnegie-Mellon University
+#	Updated to work from any directory at the same level as
+#	"conf"; generate configuration from both MASTER and
+#	MASTER.<machine-type> files; added -cpu switch.
+#	[ V5.1(F1) ]
+#
+# 18-Aug-86  Mike Accetta (mja) at Carnegie-Mellon University
+#	Added -make switch and changed meaning of -config;  upgraded to
+#	allow multiple attributes per configuration and to define
+#	configurations in terms of these attributes within MASTER.
+#
+# 14-Apr-83  Mike Accetta (mja) at Carnegie-Mellon University
+#	Added -config switch to only run /etc/config without 
+#	"make depend" and "make".
+#
+######################################################################
+
+set prog=$0
+set prog=$prog:t
+set nonomatch
+set OBJDIR=../BUILD
+if ("`/usr/bin/uname`" == "Rhapsody" ) then
+set CONFIG_DIR=/usr/local/bin
+else
+set CONFIG_DIR=/usr/bin
+endif
+
+unset domake
+unset doconfig
+unset beverbose
+unset MACHINE
+unset profile
+
+while ($#argv >= 1)
+    if ("$argv[1]" =~ -*) then
+        switch ("$argv[1]")
+	case "-c":
+	case "-config":
+	    set doconfig
+	    breaksw
+	case "-m":
+	case "-make":
+	    set domake
+	    breaksw
+	case "-cpu":
+	    if ($#argv < 2) then
+		echo "${prog}: missing argument to ${argv[1]}"
+		exit 1
+	    endif
+	    set MACHINE="$argv[2]"
+	    shift
+	    breaksw
+	case "-d":
+	    if ($#argv < 2) then
+		echo "${prog}: missing argument to ${argv[1]}"
+		exit 1
+	    endif
+	    set OBJDIR="$argv[2]"
+	    shift
+	    breaksw
+	case "-verbose":
+	    set beverbose
+	    breaksw
+	case "-p":
+	case "-profile":
+	    set profile
+	    breaksw
+	default:
+	    echo "${prog}: ${argv[1]}: unknown switch"
+	    exit 1
+	    breaksw
+	endsw
+	shift
+    else
+	break
+    endif
+end
+
+if ($#argv == 0) set argv=(GENERIC)
+
+if (! $?MACHINE) then
+    if (-d /NextApps) then
+	set MACHINE=`hostinfo | awk '/MC680x0/ { printf("m68k") } /MC880x0/ { printf("m88k") }'`
+    endif
+endif
+
+if (! $?MACHINE) then
+    if (-f /etc/machine) then
+	    set MACHINE="`/etc/machine`"
+    else
+	    echo "${prog}: no /etc/machine, specify machine type with -cpu"
+	    echo "${prog}: e.g. ${prog} -cpu VAX CONFIGURATION"
+	    exit 1
+    endif
+endif
+
+set FEATURES_EXTRA=
+
+switch ("$MACHINE")
+    case IBMRT:
+	set cpu=ca
+	set ID=RT
+	set FEATURES_EXTRA="romp_dualcall.h romp_fpa.h"
+	breaksw
+    case SUN:
+	set cpu=sun3
+	set ID=SUN3
+	breaksw
+    default:
+	set cpu=`echo $MACHINE | tr A-Z a-z`
+	set ID=`echo $MACHINE | tr a-z A-Z`
+	breaksw
+endsw
+set FEATURES=../h/features.h
+set FEATURES_H=(cs_*.h mach_*.h net_*.h\
+	        cputypes.h cpus.h vice.h\
+	        $FEATURES_EXTRA)
+set MASTER_DIR=../conf
+set MASTER =   ${MASTER_DIR}/MASTER
+set MASTER_CPU=${MASTER}.${cpu}
+
+set MASTER_LOCAL = ${MASTER}.local
+set MASTER_CPU_LOCAL = ${MASTER_CPU}.local
+if (! -f $MASTER_LOCAL) set MASTER_LOCAL = ""
+if (! -f $MASTER_CPU_LOCAL) set MASTER_CPU_LOCAL = ""
+
+if (! -d $OBJDIR) then
+    if ($?beverbose) then
+        echo "[ creating $OBJDIR ]"
+    endif
+    mkdir -p $OBJDIR
+endif
+
+foreach SYS ($argv)
+    set SYSID=${SYS}_${ID}
+    set SYSCONF=$OBJDIR/config.$SYSID
+    set BLDDIR=$OBJDIR
+    if ($?beverbose) then
+	echo "[ generating $SYSID from $MASTER_DIR/MASTER{,.$cpu}{,.local} ]"
+    endif
+    echo +$SYS \
+    | \
+    cat $MASTER $MASTER_LOCAL $MASTER_CPU $MASTER_CPU_LOCAL - \
+        $MASTER $MASTER_LOCAL $MASTER_CPU $MASTER_CPU_LOCAL \
+    | \
+    sed -n \
+	-e "/^+/{" \
+	   -e "s;[-+];#&;gp" \
+	      -e 't loop' \
+	   -e ': loop' \
+           -e 'n' \
+	   -e '/^#/b loop' \
+	   -e '/^$/b loop' \
+	   -e 's;^\([^#]*\).*#[ 	]*<\(.*\)>[ 	]*$;\2#\1;' \
+	      -e 't not' \
+	   -e 's;\([^#]*\).*;#\1;' \
+	      -e 't not' \
+	   -e ': not' \
+	   -e 's;[ 	]*$;;' \
+	   -e 's;^\!\(.*\);\1#\!;' \
+	   -e 'p' \
+	      -e 't loop' \
+           -e 'b loop' \
+	-e '}' \
+	-e "/^[^#]/d" \
+	-e 's;	; ;g' \
+	-e "s;^# *\([^ ]*\)[ ]*=[ ]*\[\(.*\)\].*;\1#\2;p" \
+    | \
+    awk '-F#' '\
+part == 0 && $1 != "" {\
+	m[$1]=m[$1] " " $2;\
+	next;\
+}\
+part == 0 && $1 == "" {\
+	for (i=NF;i>1;i--){\
+		s=substr($i,2);\
+		c[++na]=substr($i,1,1);\
+		a[na]=s;\
+	}\
+	while (na > 0){\
+		s=a[na];\
+		d=c[na--];\
+		if (m[s] == "") {\
+			f[s]=d;\
+		} else {\
+			nx=split(m[s],x," ");\
+			for (j=nx;j>0;j--) {\
+				z=x[j];\
+				a[++na]=z;\
+				c[na]=d;\
+			}\
+		}\
+	}\
+	part=1;\
+	next;\
+}\
+part != 0 {\
+	if ($1 != "") {\
+		n=split($1,x,",");\
+		ok=0;\
+		for (i=1;i<=n;i++) {\
+			if (f[x[i]] == "+") {\
+				ok=1;\
+			}\
+		}\
+		if (NF > 2 && ok == 0 || NF <= 2 && ok != 0) {\
+			print $2; \
+		}\
+	} else { \
+		print $2; \
+	}\
+}\
+' >$SYSCONF.new
+    if (-z $SYSCONF.new) then
+	echo "${prog}: ${$SYSID}: no such configuration in $MASTER_DIR/MASTER{,.$cpu}"
+	rm -f $SYSCONF.new
+    endif
+    if (! -d $BLDDIR) then
+        if ($?beverbose) then
+	    echo "[ creating $BLDDIR ]"
+        endif
+	mkdir -p $BLDDIR
+    endif
+#
+# These paths are used by config.
+#
+# "builddir" is the name of the directory where kernel binaries
+# are put.  It is a single path element, never absolute, and is
+# always relative to "objectdir".  "builddir" is used by config
+# solely to determine where to put files created by "config" (e.g.
+# the created Makefile and *.h's.)
+#
+# "objectdir" is the name of the directory which will hold "builddir".
+# It is a path; if relative, it is relative to the current directory
+# where config is run.  It's sole use is to be prepended to "builddir"
+# to indicate where config-created files are to be placed (see above).
+#
+# "sourcedir" is the location of the sources used to build the kernel.
+# It is a path; if relative, it is relative to the directory specified
+# by the concatenation of "objectdir" and "builddir" (i.e. where the
+# kernel binaries are put).
+#
+    echo 'builddir	"."'			>> $SYSCONF.new
+    set OBJRELDIR=`$RELPATH $OBJROOT $OBJDIR`
+    echo 'objectdir	"'$OBJROOT'/'$OBJRELDIR'"'		>> $SYSCONF.new
+    set SRCDIR=`dirname $SOURCE`
+    echo 'sourcedir	"'$SRCROOT'"'		>> $SYSCONF.new
+    if (-f $SYSCONF) then
+	diff $SYSCONF $SYSCONF.new
+	rm -f $SYSCONF.old
+	mv $SYSCONF $SYSCONF.old
+    endif
+    rm -f $SYSCONF
+    mv $SYSCONF.new $SYSCONF
+    if ($?doconfig) then
+        if ($?beverbose) then
+	    echo "[ configuring $SYSID ]"
+        endif
+	if ($?profile) then
+	    $CONFIG_DIR/config -c $MASTER_DIR -p $SYSCONF
+	else
+	    $CONFIG_DIR/config -c $MASTER_DIR $SYSCONF
+	endif
+    endif
+    if ($?domake) then
+        if ($?beverbose) then
+            echo "[ making $SYSID ]"
+        endif
+        (cd $BLDDIR; make)
+    endif
+end
diff --git a/security/conf/tools/newvers/Makefile b/security/conf/tools/newvers/Makefile
new file mode 100644
index 000000000..a430a1fd5
--- /dev/null
+++ b/security/conf/tools/newvers/Makefile
@@ -0,0 +1,47 @@
+export MakeInc_cmd=${SRCROOT}/makedefs/MakeInc.cmd
+export MakeInc_def=${SRCROOT}/makedefs/MakeInc.def
+export MakeInc_rule=${SRCROOT}/makedefs/MakeInc.rule
+export MakeInc_dir=${SRCROOT}/makedefs/MakeInc.dir
+
+
+include $(MakeInc_cmd)
+include $(MakeInc_def)
+
+COMP_SUBDIRS = \
+
+INST_SUBDIRS = \
+
+
+#
+# Who and where
+#
+BINDIR= 
+ifneq ($(MACHINE_CONFIG), DEFAULT)
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)_$(MACHINE_CONFIG)/$(COMPONENT)/)
+else
+DSTDIR= $(strip $(OBJROOT)/$(KERNEL_CONFIG)_$(ARCH_CONFIG)/$(COMPONENT)/)
+endif
+PROGRAM= $(DSTDIR)newvers
+
+# 
+# How to install it
+#
+IFLAGS= -c -m 555
+
+$(PROGRAM): $(DSTDIR)% : $(SOURCE)%.csh
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS 
+	@sed -e "s/#PROGRAM.*/#`vers_string $(notdir $(PROGRAM))`/" \
+		< $< >$(notdir $(PROGRAM)).VERS;
+	@install $(IFLAGS) $(notdir $(PROGRAM)).VERS $(PROGRAM);
+	@-$(RM) $(RMFLAGS) $(notdir $(PROGRAM)).VERS;
+
+do_build_setup: $(PROGRAM)
+
+do_build_all:
+
+setup_build_install:
+
+do_build_install:
+
+include $(MakeInc_rule)
+include $(MakeInc_dir)
diff --git a/security/conf/tools/newvers/newvers.csh b/security/conf/tools/newvers/newvers.csh
new file mode 100644
index 000000000..a1035e209
--- /dev/null
+++ b/security/conf/tools/newvers/newvers.csh
@@ -0,0 +1,33 @@
+#!/bin/sh -
+#
+# Mach Operating System
+# Copyright (c) 1990 Carnegie-Mellon University
+# Copyright (c) 1989 Carnegie-Mellon University
+# All rights reserved.  The CMU software License Agreement specifies
+# the terms and conditions for use and redistribution.
+#
+
+#
+# newvers.sh	copyright major minor variant
+#
+
+major="$1"; minor="$2"; variant="$3"
+v="${major}.${minor}" d=`pwd` h="rcbuilder" t=`date` w=`whoami`
+if [ -z "$d" -o -z "$h" -o -z "$t" ]; then
+    exit 1
+fi
+CONFIG=`expr "$d" : '.*/\([^/]*\)$'`
+d=`expr "$d" : '.*/\([^/]*/[^/]*/[^/]*\)$'`
+(
+  /bin/echo "int  ${COMPONENT}_version_major      = ${major};" ;
+  /bin/echo "int  ${COMPONENT}_version_minor      = ${minor};" ;
+  /bin/echo "char ${COMPONENT}_version_variant[]  = \"${variant}\";" ;
+  /bin/echo "char ${COMPONENT}_version[] = \"Security Component Version ${v}:\\n${t}; $w($h):$d\\n\";" ;
+  /bin/echo "char ${COMPONENT}_osrelease[] = \"${major}.${minor}\";" ;
+  /bin/echo "char ${COMPONENT}_ostype[] = \"Security Modules\";" ;
+) > vers.c
+if [ -s vers.suffix -o ! -f vers.suffix ]; then
+    rm -f vers.suffix
+    echo ".${variant}.${CONFIG}" > vers.suffix
+fi
+exit 0
diff --git a/security/conf/version.major b/security/conf/version.major
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/security/conf/version.major
@@ -0,0 +1 @@
+6
diff --git a/security/conf/version.minor b/security/conf/version.minor
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/security/conf/version.minor
@@ -0,0 +1 @@
+6
diff --git a/security/conf/version.variant b/security/conf/version.variant
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/security/conf/version.variant
@@ -0,0 +1 @@
+
diff --git a/security/mac.h b/security/mac.h
new file mode 100644
index 000000000..936abc34a
--- /dev/null
+++ b/security/mac.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001-2005 Networks Associates Technology, Inc.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson for the TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/sys/sys/mac.h,v 1.40 2003/04/18 19:57:37 rwatson Exp $
+ */
+/*
+ * Userland interface for Mandatory Access Control.
+ *
+ * The POSIX.1e implementation page may be reached at:
+ * http://www.trustedbsd.org/
+ */
+
+#ifndef _SECURITY_MAC_H_
+#define	_SECURITY_MAC_H_
+
+#ifndef _POSIX_MAC
+#define	_POSIX_MAC
+#endif
+
+#include <sys/types.h>
+
+/*
+ * MAC framework-related constants and limits.
+ */
+#define	MAC_MAX_POLICY_NAME		32
+#define	MAC_MAX_LABEL_ELEMENT_NAME	32
+#define	MAC_MAX_LABEL_ELEMENT_DATA	4096
+#define	MAC_MAX_LABEL_BUF_LEN		8192
+#define MAC_MAX_MANAGED_NAMESPACES	4
+
+struct mac {
+	size_t		 m_buflen;
+	char		*m_string;
+};
+
+typedef struct mac	*mac_t;
+
+#ifdef KERNEL
+struct user_mac {
+	user_size_t	m_buflen;
+	user_addr_t	m_string;
+};
+#endif /* KERNEL */
+
+/*
+ * Flags to control which MAC subsystems are enforced
+ * on a per-process/thread/credential basis.
+ */
+#define MAC_SYSTEM_ENFORCE	0x0001	/* system management */
+#define MAC_PROC_ENFORCE	0x0002	/* process management */
+#define MAC_MACH_ENFORCE	0x0004	/* mach interfaces */
+#define MAC_VM_ENFORCE		0x0008	/* VM interfaces */
+#define MAC_FILE_ENFORCE	0x0010	/* file operations */
+#define MAC_SOCKET_ENFORCE	0x0020	/* socket operations */
+#define MAC_PIPE_ENFORCE	0x0040	/* pipes */
+#define MAC_VNODE_ENFORCE	0x0080	/* vnode operations */
+#define MAC_NET_ENFORCE		0x0100	/* network management */
+#define MAC_MBUF_ENFORCE	0x0200	/* network traffic */
+#define MAC_POSIXSEM_ENFORCE	0x0400	/* posix semaphores */
+#define MAC_POSIXSHM_ENFORCE	0x0800	/* posix shared memory */
+#define MAC_SYSVMSG_ENFORCE	0x1000	/* SysV message queues */
+#define MAC_SYSVSEM_ENFORCE	0x2000	/* SysV semaphores */
+#define MAC_SYSVSHM_ENFORCE	0x4000	/* SysV shared memory */
+#define MAC_ALL_ENFORCE		0x7fff	/* enforce everything */
+
+/*
+ * Device types for mac_iokit_check_device()
+ */
+#define MAC_DEVICE_USB		"USB"
+#define MAC_DEVICE_FIREWIRE	"FireWire"
+#define MAC_DEVICE_TYPE_KEY	"DeviceType"
+
+#ifndef KERNEL
+/*
+ * Location of the userland MAC framework configuration file.  mac.conf
+ * binds policy names to shared libraries that understand those policies,
+ * as well as setting defaults for MAC-aware applications.
+ */
+#define	MAC_CONFFILE	"/etc/mac.conf"
+
+/*
+ * Extended non-POSIX.1e interfaces that offer additional services
+ * available from the userland and kernel MAC frameworks.
+ */
+#ifdef __APPLE_API_PRIVATE
+__BEGIN_DECLS
+int	 __mac_execve(char *fname, char **argv, char **envv, mac_t _label);
+int	 __mac_get_fd(int _fd, mac_t _label);
+int	 __mac_get_file(const char *_path, mac_t _label);
+int	 __mac_get_lcid(pid_t _lcid, mac_t _label);
+int	 __mac_get_lctx(mac_t _label);
+int	 __mac_get_link(const char *_path, mac_t _label);
+int	 __mac_get_pid(pid_t _pid, mac_t _label);
+int	 __mac_get_proc(mac_t _label);
+int	 __mac_set_fd(int _fildes, const mac_t _label);
+int	 __mac_set_file(const char *_path, mac_t _label);
+int	 __mac_set_lctx(mac_t _label);
+int	 __mac_set_link(const char *_path, mac_t _label);
+int	 __mac_mount(const char *type, const char *path, int flags, void *data,
+    struct mac *label);
+int	 __mac_get_mount(const char *path, struct mac *label);
+int	 __mac_set_proc(const mac_t _label);
+int	 __mac_syscall(const char *_policyname, int _call, void *_arg);
+__END_DECLS
+#endif /*__APPLE_API_PRIVATE*/
+
+#endif
+
+#endif /* !_SECURITY_MAC_H_ */
diff --git a/security/mac_alloc.c b/security/mac_alloc.c
new file mode 100644
index 000000000..8cd41403d
--- /dev/null
+++ b/security/mac_alloc.c
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ *
+ */
+
+#include <kern/kalloc.h>
+#include <kern/zalloc.h>
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/systm.h>
+#include <sys/mbuf.h>
+
+#include <vm/vm_map.h>
+
+#include "mac_alloc.h"
+
+/*
+ * XXXMAC: We should probably make sure only registered policies can
+ * call these, otherwise we're effectively changing Apple's plan wrt
+ * exported allocators.
+ */
+
+/*
+ * Kernel allocator
+ */
+void *
+mac_kalloc(vm_size_t size, int how)
+{
+
+	if (how == M_WAITOK)
+		return kalloc(size);
+	else
+		return kalloc_noblock(size);
+}
+
+/*
+ * for temporary binary compatibility
+ */
+void *	mac_kalloc_noblock	(vm_size_t size);
+void *
+mac_kalloc_noblock(vm_size_t size)
+{
+	return kalloc_noblock(size);
+}
+
+void
+mac_kfree(void * data, vm_size_t size)
+{
+
+	return kfree(data, size);
+}
+
+/*
+ * MBuf tag allocator.
+ */
+
+void *
+mac_mbuf_alloc(int len, int wait)
+{
+	struct m_tag *t;
+
+	t = m_tag_alloc(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_MAC_POLICY_LABEL,
+			len, wait);
+	if (t == NULL)
+		return (NULL);
+
+	return ((void *)(t + 1));
+}
+
+void
+mac_mbuf_free(void *data)
+{
+	struct m_tag *t;
+
+	t = (struct m_tag *)((char *)data - sizeof(struct m_tag));
+	m_tag_free(t);
+}
+
+/*
+ * VM functions
+ */
+
+extern vm_map_t kalloc_map;
+
+int
+mac_wire(void *start, void *end)
+{
+
+	return (vm_map_wire(kalloc_map, CAST_USER_ADDR_T(start),
+		CAST_USER_ADDR_T(end), VM_PROT_READ|VM_PROT_WRITE, FALSE));
+}
+
+int
+mac_unwire(void *start, void *end)
+{
+
+	return (vm_map_unwire(kalloc_map, CAST_USER_ADDR_T(start),
+		CAST_USER_ADDR_T(end), FALSE));
+}
+
+/*
+ * Zone allocator
+ */
+zone_t
+mac_zinit(vm_size_t size, vm_size_t maxmem, vm_size_t alloc, const char *name)
+{
+
+	return zinit(size, maxmem, alloc, name);
+}
+
+void
+mac_zone_change(zone_t zone, unsigned int item, boolean_t value)
+{
+
+	zone_change(zone, item, value);
+}
+
+void *
+mac_zalloc(zone_t zone, int how)
+{
+
+	if (how == M_WAITOK)
+		return zalloc(zone);
+	else
+		return zalloc_noblock(zone);
+}
+
+void
+mac_zfree(zone_t zone, void *elem)
+{
+
+	zfree(zone, elem);
+}
diff --git a/security/mac_alloc.h b/security/mac_alloc.h
new file mode 100644
index 000000000..70e1baef0
--- /dev/null
+++ b/security/mac_alloc.h
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Memory allocation wrappers.
+ */
+
+#ifndef _SECURITY_MAC_ALLOC_H_
+#define	_SECURITY_MAC_ALLOC_H_
+
+#include <mach/machine/vm_types.h>
+#include <kern/kern_types.h>
+#include <sys/appleapiopts.h>
+
+/* JMM - should use OSMlloc.h interfaces */
+
+#ifdef __APPLE_API_EVOLVING
+/*
+ * Kernel Memory allocator
+ */
+void *	mac_kalloc	(vm_size_t size, int how);
+void	mac_kfree	(void *data, vm_size_t size);
+
+/*
+ * Mbuf allocator for mbuf labels.
+ */
+void *	mac_mbuf_alloc	(int len, int wait);
+void	mac_mbuf_free	(void *data);
+
+/*
+ * 
+ */
+int	mac_wire	(void *start, void *end);
+int	mac_unwire	(void *start, void *end);
+
+/*
+ * Zone allocator
+ */
+zone_t	mac_zinit	(vm_size_t size, vm_size_t maxmem,
+			 vm_size_t alloc, const char *name);
+void	mac_zone_change	(zone_t zone, unsigned int item, boolean_t value);
+void *	mac_zalloc	(zone_t zone, int how);
+void	mac_zfree	(zone_t zone, void *elem);
+
+/* Item definitions */
+#define Z_EXHAUST       1       /* Make zone exhaustible        */
+#define Z_COLLECT       2       /* Make zone collectable        */
+#define Z_EXPAND        3       /* Make zone expandable         */
+#define Z_FOREIGN       4       /* Allow collectable zone to contain foreign elements */
+
+#endif  /* __APPLE_API_EVOLVING */
+#endif	/* _SECURITY_MAC_ALLOC_H_ */
diff --git a/security/mac_audit.c b/security/mac_audit.c
new file mode 100644
index 000000000..cb61c1912
--- /dev/null
+++ b/security/mac_audit.c
@@ -0,0 +1,398 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+#include <sys/param.h>
+#include <sys/types.h>  
+#include <sys/vnode.h>  
+#include <sys/vnode_internal.h>
+#include <sys/kauth.h>
+#include <sys/queue.h>  
+#include <security/mac_internal.h>
+#include <bsd/bsm/audit.h>
+#include <bsd/bsm/audit_kernel.h>
+#include <bsd/sys/malloc.h>
+#include <vm/vm_kern.h>
+#include <kern/kalloc.h>
+#include <kern/zalloc.h>
+
+#ifdef AUDIT
+
+/* The zone allocator is initialized in mac_base.c. */
+zone_t mac_audit_data_zone;
+
+int
+mac_system_check_audit(struct ucred *cred, void *record, int length)
+{
+	int error;
+
+	MAC_CHECK(system_check_audit, cred, record, length);
+
+	return (error);
+}
+
+int
+mac_system_check_auditon(struct ucred *cred, int cmd)
+{
+	int error;
+
+	MAC_CHECK(system_check_auditon, cred, cmd);
+
+	return (error);
+}
+
+int
+mac_system_check_auditctl(struct ucred *cred, struct vnode *vp)
+{
+	int error;
+	struct label *vl = vp ? vp->v_label : NULL;
+
+	MAC_CHECK(system_check_auditctl, cred, vp, vl);
+
+	return (error);
+}
+
+int
+mac_proc_check_getauid(struct proc *curp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return 0;
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_getauid, cred);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_setauid(struct proc *curp, uid_t auid)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return 0;
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_setauid, cred, auid);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int 
+mac_proc_check_getaudit(struct proc *curp) 
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return 0;
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_getaudit, cred);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_setaudit(struct proc *curp, struct auditinfo *ai)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return 0;
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_setaudit, cred, ai);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+#if 0
+/*
+ * This is the framework entry point for MAC policies to use to add
+ * arbitrary data to the current audit record.
+ * (Currently not supported, as no existing audit viewers would 
+ * display this format)
+ * 
+ */
+int
+mac_audit_data(int len, u_char *data, mac_policy_handle_t handle)
+{
+	char *sanitized;
+
+	if ((len <= 0) || (len > MAC_AUDIT_DATA_LIMIT))
+		return (EINVAL);
+
+	sanitized = (char *)zalloc(mac_audit_data_zone);
+
+	bcopy(data, sanitized, len);
+	return (audit_mac_data(MAC_AUDIT_DATA_TYPE, len, sanitized));
+}
+#endif
+
+/*
+ * This is the entry point a MAC policy will call to add NULL-
+ * terminated ASCII text to the current audit record.
+ */
+int
+mac_audit_text(char *text, mac_policy_handle_t handle)
+{
+	char *sanitized;
+	const char *name;
+	int i, size, plen, len;
+
+	name = mac_get_mpc(handle)->mpc_name;
+	len = strlen(text);
+	plen = 2 + strlen(name);
+	if (plen + len >= MAC_AUDIT_DATA_LIMIT)
+		return (EINVAL);
+
+	/*
+	 * Make sure the text is only composed of only ASCII printable
+	 * characters.
+	 */
+	for (i=0; i < len; i++)
+		if (text[i] < (char) 32 || text[i] > (char) 126)
+			return (EINVAL);
+
+	size = len + plen + 1;
+ 	sanitized = (char *)zalloc(mac_audit_data_zone);
+
+	strlcpy(sanitized, name, MAC_AUDIT_DATA_LIMIT);
+	strncat(sanitized, ": ", MAC_AUDIT_DATA_LIMIT - plen + 2);
+	strncat(sanitized, text, MAC_AUDIT_DATA_LIMIT - plen);
+
+	return (audit_mac_data(MAC_AUDIT_TEXT_TYPE, size, (u_char *)sanitized));
+}
+
+int
+mac_audit_check_preselect(struct ucred *cred, unsigned short syscode, void *args)
+{
+	struct mac_policy_conf *mpc;
+	int ret, error;
+	u_int i;
+
+	ret = MAC_AUDIT_DEFAULT;
+	for (i = 0; i < mac_policy_list.staticmax; i++) {
+		mpc = mac_policy_list.entries[i].mpc;
+		if (mpc == NULL)
+			continue;
+
+		if (mpc->mpc_ops->mpo_audit_check_preselect != NULL) {
+			error = mpc->mpc_ops->mpo_audit_check_preselect(cred,
+			    syscode, args);
+			ret = (ret > error ? ret : error);
+		}
+	}
+	if (mac_policy_list_conditional_busy() != 0) {
+		for (; i <= mac_policy_list.maxindex; i++) {
+			mpc = mac_policy_list.entries[i].mpc;
+			if (mpc == NULL)
+				continue;
+
+			if (mpc->mpc_ops->mpo_audit_check_preselect != NULL) {
+				error = mpc->mpc_ops->mpo_audit_check_preselect(cred,
+				    syscode, args);
+				ret = (ret > error ? ret : error);
+			}
+		}
+		mac_policy_list_unbusy();
+	}
+
+	return (ret);
+}
+
+int
+mac_audit_check_postselect(struct ucred *cred, unsigned short syscode,
+    void *args, int error, int retval, int mac_forced)
+{
+	struct mac_policy_conf *mpc;
+	int ret, mac_error;
+	u_int i;
+
+	/*
+	 * If the audit was forced by a MAC policy by mac_audit_check_preselect(),
+	 * echo that.
+	 */
+	if (mac_forced)
+		return (MAC_AUDIT_YES);
+
+	ret = MAC_AUDIT_DEFAULT;
+	for (i = 0; i < mac_policy_list.staticmax; i++) {
+		mpc = mac_policy_list.entries[i].mpc;
+		if (mpc == NULL)
+			continue;
+
+		if (mpc->mpc_ops->mpo_audit_check_postselect != NULL) {
+			mac_error = mpc->mpc_ops->mpo_audit_check_postselect(cred,
+			    syscode, args, error, retval);
+			ret = (ret > mac_error ? ret : mac_error);
+		}
+	}
+	if (mac_policy_list_conditional_busy() != 0) {
+		for (; i <= mac_policy_list.maxindex; i++) {
+			mpc = mac_policy_list.entries[i].mpc;
+			if (mpc == NULL)
+				continue;
+
+			if (mpc->mpc_ops->mpo_audit_check_postselect != NULL) {
+				mac_error = mpc->mpc_ops->mpo_audit_check_postselect(cred,
+				    syscode, args, error, retval);
+				ret = (ret > mac_error ? ret : mac_error);
+			}
+		}
+		mac_policy_list_unbusy();
+	}
+
+	return (ret);
+}
+
+#else	/* AUDIT */
+
+/*
+ * Function stubs for when AUDIT isn't defined.
+ */
+
+int
+mac_system_check_audit(struct ucred *cred, void *record, int length)
+{
+
+	return (0);
+}
+
+int
+mac_system_check_auditon(struct ucred *cred, int cmd)
+{
+
+	return (0);
+}
+
+int
+mac_system_check_auditctl(struct ucred *cred, struct vnode *vp)
+{
+
+	return (0);
+}
+
+int
+mac_proc_check_getauid(__unused struct proc *curp)
+{
+
+	return (0);
+}
+
+int
+mac_proc_check_setauid(__unused struct proc *curp, __unused uid_t auid)
+{
+
+	return (0);
+}
+
+int
+mac_proc_check_getaudit(__unused struct proc *curp)
+{
+
+	return (0);
+}
+
+int
+mac_proc_check_setaudit(__unused struct proc *curp, struct auditinfo *ai)
+{
+
+	return (0);
+}
+
+int
+mac_audit_check_preselect(__unused struct ucred *cred, __unused unsigned short syscode,
+    __unused void *args)
+{
+
+	return (MAC_AUDIT_DEFAULT);
+}
+
+int
+mac_audit_check_postselect(__unused struct ucred *cred, __unused unsigned short syscode,
+    __unused void *args, __unused int error, __unused int retval, __unused int mac_forced)
+{
+
+	return (MAC_AUDIT_DEFAULT);
+}
+
+int
+mac_audit(int len, u_char *data)
+{
+
+	return (0);
+}
+#endif	/* !AUDIT */
diff --git a/security/mac_base.c b/security/mac_base.c
new file mode 100644
index 000000000..37c9d05af
--- /dev/null
+++ b/security/mac_base.c
@@ -0,0 +1,2279 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*-
+ * Framework for extensible kernel access control.  This file contains
+ * Kernel and userland interface to the framework, policy registration
+ * and composition.  Per-object interfaces, controls, and labeling may be
+ * found in src/sys/mac/.  Sample policies may be found in src/sys/mac*.
+ */
+
+#include <stdarg.h>
+#include <string.h>
+#include <security/mac_internal.h>
+#include <security/mac_mach_internal.h>
+#include <sys/param.h>
+#include <sys/vnode.h>
+#include <sys/vnode_internal.h>
+#include <sys/vfs_context.h>
+#include <sys/namei.h>
+#include <bsd/bsm/audit.h>
+#include <bsd/bsm/audit_kernel.h>
+#include <sys/file.h>
+#include <sys/file_internal.h>
+#include <sys/filedesc.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/sysproto.h>
+
+#include <mach/vm_types.h>
+#include <mach/vm_prot.h>
+
+#include <kern/zalloc.h>
+#include <kern/sched_prim.h>
+#include <osfmk/kern/task.h>
+#include <osfmk/kern/kalloc.h>
+#include <libsa/libsa/kext.h>
+
+#if CONFIG_MACF
+#include <security/mac.h>
+#include <security/mac_policy.h>
+#include <security/mac_framework.h>
+#include <security/mac_internal.h>
+#include <security/mac_mach_internal.h>
+#endif
+
+/* 
+ * define MB_DEBUG to display run-time debugging information
+ * #define MB_DEBUG 1
+ */
+
+#ifdef MB_DEBUG
+#define DPRINTF(x)	printf x
+#else
+#define MB_DEBUG
+#define DPRINTF(x)
+#endif
+
+#if CONFIG_MACF
+SYSCTL_NODE(, OID_AUTO, security, CTLFLAG_RW|CTLFLAG_LOCKED, 0, 
+    "Security Controls");
+SYSCTL_NODE(_security, OID_AUTO, mac, CTLFLAG_RW|CTLFLAG_LOCKED, 0,
+    "TrustedBSD MAC policy controls");
+
+
+
+/*
+ * Declare that the kernel provides MAC support, version 1.  This permits
+ * modules to refuse to be loaded if the necessary support isn't present,
+ * even if it's pre-boot.
+ */
+#if 0
+MODULE_VERSION(kernel_mac_support, 1);
+#endif
+
+#if MAC_MAX_SLOTS > 32
+#error "MAC_MAX_SLOTS too large"
+#endif
+
+static unsigned int mac_max_slots = MAC_MAX_SLOTS;
+static unsigned int mac_slot_offsets_free = (1 << MAC_MAX_SLOTS) - 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, max_slots, CTLFLAG_RD,
+    &mac_max_slots, 0, "");
+
+/*
+ * Has the kernel started generating labeled objects yet?  All read/write
+ * access to this variable is serialized during the boot process.  Following
+ * the end of serialization, we don't update this flag; no locking.
+ */
+int	mac_late = 0;
+
+/*
+ * Flag to indicate whether or not we should allocate label storage for
+ * new mbufs.  Since most dynamic policies we currently work with don't
+ * rely on mbuf labeling, try to avoid paying the cost of mtag allocation
+ * unless specifically notified of interest.  One result of this is
+ * that if a dynamically loaded policy requests mbuf labels, it must
+ * be able to deal with a NULL label being returned on any mbufs that
+ * were already in flight when the policy was loaded.  Since the policy
+ * already has to deal with uninitialized labels, this probably won't
+ * be a problem.  Note: currently no locking.  Will this be a problem?
+ */
+#if !defined(CONFIG_MACF_ALWAYS_LABEL_MBUF) && 0
+static int	mac_labelmbufs = 0;
+#endif
+
+unsigned int	mac_mmap_revocation = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, mmap_revocation, CTLFLAG_RW,
+    &mac_mmap_revocation, 0, "Revoke mmap access to files on subject "
+    "relabel");
+
+unsigned int	mac_mmap_revocation_via_cow = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, mmap_revocation_via_cow, CTLFLAG_RW,
+    &mac_mmap_revocation_via_cow, 0, "Revoke mmap access to files via "
+    "copy-on-write semantics, or by removing all write access");
+
+unsigned int mac_device_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, device_enforce, CTLFLAG_RW,
+	   &mac_device_enforce, 0, "Enforce MAC policy on device operations");
+
+unsigned int mac_file_enforce = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, file_enforce, CTLFLAG_RW,
+	&mac_file_enforce, 0, "Enforce MAC policy on file operations");
+
+unsigned int mac_iokit_enforce = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, iokit_enforce, CTLFLAG_RW,
+	&mac_file_enforce, 0, "Enforce MAC policy on IOKit operations");
+
+unsigned int	mac_pipe_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, pipe_enforce, CTLFLAG_RW,
+    &mac_pipe_enforce, 0, "Enforce MAC policy on pipe operations");
+
+unsigned int	mac_posixsem_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, posixsem_enforce, CTLFLAG_RW,
+    &mac_posixsem_enforce, 0, "Enforce MAC policy on POSIX semaphores");
+
+unsigned int mac_posixshm_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, posixshm_enforce, CTLFLAG_RW,
+    &mac_posixshm_enforce, 0, "Enforce MAC policy on Posix Shared Memory");
+
+unsigned int	mac_proc_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, proc_enforce, CTLFLAG_RW,
+	   &mac_proc_enforce, 0, "Enforce MAC policy on process operations");
+
+unsigned int mac_socket_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, socket_enforce, CTLFLAG_RW,
+	&mac_socket_enforce, 0, "Enforce MAC policy on socket operations");
+
+unsigned int	mac_system_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, system_enforce, CTLFLAG_RW,
+    &mac_system_enforce, 0, "Enforce MAC policy on system-wide interfaces");
+
+unsigned int	mac_sysvmsg_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, sysvmsg_enforce, CTLFLAG_RW,
+    &mac_sysvmsg_enforce, 0, "Enforce MAC policy on System V IPC message queues");
+
+unsigned int	mac_sysvsem_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, sysvsem_enforce, CTLFLAG_RW,
+    &mac_sysvsem_enforce, 0, "Enforce MAC policy on System V IPC semaphores");
+
+unsigned int	mac_sysvshm_enforce = 1;
+SYSCTL_INT(_security_mac, OID_AUTO, sysvshm_enforce, CTLFLAG_RW,
+    &mac_sysvshm_enforce, 0, "Enforce MAC policy on System V Shared Memory");
+
+unsigned int	mac_vm_enforce = 1;
+SYSCTL_INT(_security_mac, OID_AUTO, vm_enforce, CTLFLAG_RW,
+	   &mac_vm_enforce, 0, "Enforce MAC policy on VM operations");
+
+unsigned int	mac_vnode_enforce = 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, vnode_enforce, CTLFLAG_RW,
+	   &mac_vnode_enforce, 0, "Enforce MAC policy on vnode operations");
+
+
+#if CONFIG_MACF_MACH
+unsigned int	mac_port_enforce = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, port_enforce, CTLFLAG_RW,
+    &mac_port_enforce, 0, "Enforce MAC policy on Mach port operations");
+
+unsigned int	mac_task_enforce = 0;
+SYSCTL_UINT(_security_mac, OID_AUTO, task_enforce, CTLFLAG_RW,
+    &mac_task_enforce, 0, "Enforce MAC policy on Mach task operations");
+#endif
+
+#if CONFIG_MACF_NET
+unsigned int mac_label_mbufs	= 1;
+SYSCTL_UINT(_security_mac, OID_AUTO, label_mbufs, CTLFLAG_RW,
+	&mac_label_mbufs, 0, "Label all MBUFs");
+#endif
+
+/*
+ * mac_audit_data_zone is the zone used for data pushed into the audit
+ * record by policies. Using a zone simplifies memory management of this
+ * data, and allows tracking of the amount of data in flight.
+ */
+extern zone_t mac_audit_data_zone;
+
+/*
+ * mac_policy_list holds the list of policy modules.  Modules with a
+ * handle lower than staticmax are considered "static" and cannot be
+ * unloaded.  Such policies can be invoked without holding the busy count.
+ *
+ * Modules with a handle at or above the staticmax high water mark
+ * are considered to be "dynamic" policies.  A busy count is maintained
+ * for the list, stored in mac_policy_busy.  The busy count is protected
+ * by mac_policy_mtx; the list may be modified only while the busy
+ * count is 0, requiring that the lock be held to prevent new references
+ * to the list from being acquired.  For almost all operations,
+ * incrementing the busy count is sufficient to guarantee consistency,
+ * as the list cannot be modified while the busy count is elevated.
+ * For a few special operations involving a change to the list of
+ * active policies, the mtx itself must be held.
+ */
+static lck_mtx_t *mac_policy_mtx;
+
+/*
+ * Policy list array allocation chunk size. Trying to set this so that we
+ * allocate a page at a time.
+ */
+#define MAC_POLICY_LIST_CHUNKSIZE 512
+
+static int mac_policy_busy;
+
+mac_policy_list_t mac_policy_list;
+
+/*
+ * mac_label_element_list holds the master list of label namespaces for
+ * all the policies. When a policy is loaded, each of it's label namespace
+ * elements is added to the master list if not already present. When a
+ * policy is unloaded, the namespace elements are removed if no other 
+ * policy is interested in that namespace element.
+ */
+struct mac_label_element_list_t mac_label_element_list;
+struct mac_label_element_list_t mac_static_label_element_list;
+
+/*
+ * Journal of label operations that occur before policies are loaded.
+ */
+struct mac_label_journal_list_t mac_label_journal_list;
+
+int
+mac_label_journal_add (struct label *l, int type)
+{
+	struct mac_label_journal *mlj;
+
+	if (mac_label_journal_find(l))
+		return (0);
+
+	MALLOC(mlj, struct mac_label_journal *,
+		sizeof(struct mac_label_journal), M_MACTEMP, M_WAITOK);
+	mlj->l = l;
+	mlj->type = type;
+	TAILQ_INSERT_TAIL(&mac_label_journal_list, mlj, link);
+
+	return (0);
+}
+
+int
+mac_label_journal_remove (struct label *l)
+{
+	struct mac_label_journal *mlj;
+
+	mlj = mac_label_journal_find(l);
+	if (mlj == NULL)
+		return (-1);
+
+	TAILQ_REMOVE(&mac_label_journal_list, mlj, link);
+	FREE(mlj, M_MACTEMP);
+	return (0);
+}
+
+struct mac_label_journal *
+mac_label_journal_find (struct label *l)
+{
+	struct mac_label_journal *mlj;
+
+	TAILQ_FOREACH(mlj, &mac_label_journal_list, link) {
+		if (l == mlj->l)
+			return (mlj);
+	}
+
+	return (NULL);
+}
+
+int
+mac_label_journal (struct label *l, int op, ...)
+{
+	struct mac_label_journal *mlj;
+	va_list ap;
+
+	mlj = mac_label_journal_find(l);
+	if (mlj == NULL) {
+		printf("%s(): Label not in list!\n", __func__);
+		return (-1);
+	}
+
+	if (op == MLJ_PORT_OP_UPDATE) {
+		va_start(ap, op);
+		mlj->kotype = va_arg(ap, int);
+		va_end(ap);
+	}
+
+	mlj->ops |= op;
+	return (0);
+}
+
+/*
+ * The assumption during replay is that the system is totally
+ * serialized and no additional tasks/ports will be created.
+ */
+void
+mac_label_journal_replay (void)
+{
+	struct mac_label_journal *mlj;
+
+	TAILQ_FOREACH(mlj, &mac_label_journal_list, link) {
+		switch (mlj->type) {
+		case MLJ_TYPE_PORT:
+			if (mlj->ops & MLJ_PORT_OP_INIT)
+				MAC_PERFORM(port_label_init, mlj->l);
+			if (mlj->ops & MLJ_PORT_OP_CREATE_K)
+				MAC_PERFORM(port_label_associate_kernel, mlj->l, 0);
+			if (mlj->ops & MLJ_PORT_OP_UPDATE)
+				MAC_PERFORM(port_label_update_kobject, mlj->l,
+						mlj->kotype);
+			break;
+		case MLJ_TYPE_TASK:
+			if (mlj->ops & MLJ_TASK_OP_INIT)
+				MAC_PERFORM(task_label_init, mlj->l);
+#if 0
+			/* Not enough context to replay. */
+			if (mlj->ops & MLJ_TASK_OP_CREATE_K)
+				;
+#endif
+			break;
+		default:
+			break;
+		}
+	}
+
+	/* Free list */
+	while (!TAILQ_EMPTY(&mac_label_journal_list)) {
+		mlj = TAILQ_FIRST(&mac_label_journal_list);
+		TAILQ_REMOVE(&mac_label_journal_list, mlj, link);
+		FREE(mlj, M_MACTEMP);
+	}
+	return;
+}
+
+static __inline void
+mac_policy_grab_exclusive(void)
+{
+	lck_mtx_lock(mac_policy_mtx);
+	while (mac_policy_busy != 0) {
+		lck_mtx_sleep(mac_policy_mtx, LCK_SLEEP_UNLOCK,
+			      (event_t)&mac_policy_busy, THREAD_UNINT);
+		lck_mtx_lock(mac_policy_mtx);
+	}
+}
+
+static __inline void
+mac_policy_assert_exclusive(void)
+{
+	lck_mtx_assert(mac_policy_mtx, LCK_MTX_ASSERT_OWNED);
+	KASSERT(mac_policy_busy == 0,
+	    ("mac_policy_assert_exclusive(): not exclusive"));
+}
+
+static __inline void
+mac_policy_release_exclusive(void)
+{
+
+	KASSERT(mac_policy_busy == 0,
+	    ("mac_policy_release_exclusive(): not exclusive"));
+	lck_mtx_unlock(mac_policy_mtx);
+	thread_wakeup((event_t) &mac_policy_busy);
+}
+
+void
+mac_policy_list_busy(void)
+{
+        lck_mtx_lock(mac_policy_mtx);
+	mac_policy_busy++;
+	lck_mtx_unlock(mac_policy_mtx);
+}
+
+int
+mac_policy_list_conditional_busy(void)
+{
+	int ret;
+
+	if (mac_policy_list.numloaded <= mac_policy_list.staticmax)
+		return(0);
+
+	lck_mtx_lock(mac_policy_mtx);
+	if (mac_policy_list.numloaded > mac_policy_list.staticmax) {
+		mac_policy_busy++;
+		ret = 1;
+	} else
+		ret = 0;
+	lck_mtx_unlock(mac_policy_mtx);
+	return (ret);
+}
+
+void
+mac_policy_list_unbusy(void)
+{
+	lck_mtx_lock(mac_policy_mtx);
+	mac_policy_busy--;
+	KASSERT(mac_policy_busy >= 0, ("MAC_POLICY_LIST_LOCK"));
+	if (mac_policy_busy == 0)
+		thread_wakeup(&mac_policy_busy);
+	lck_mtx_unlock(mac_policy_mtx);
+}
+
+/*
+ * Early pre-malloc MAC initialization, including appropriate SMP locks.
+ */
+void
+mac_policy_init(void)
+{
+	lck_grp_attr_t *mac_lck_grp_attr;
+	lck_attr_t *mac_lck_attr;
+	lck_grp_t *mac_lck_grp;
+
+	mac_policy_list.numloaded = 0;
+	mac_policy_list.max = MAC_POLICY_LIST_CHUNKSIZE;
+	mac_policy_list.maxindex = 0;
+	mac_policy_list.staticmax = 0;
+	mac_policy_list.freehint = 0;
+	mac_policy_list.chunks = 1;
+
+	mac_policy_list.entries = kalloc(sizeof(struct mac_policy_list_element) * MAC_POLICY_LIST_CHUNKSIZE);
+	bzero(mac_policy_list.entries, sizeof(struct mac_policy_list_element) * MAC_POLICY_LIST_CHUNKSIZE); 
+
+	LIST_INIT(&mac_label_element_list);
+	LIST_INIT(&mac_static_label_element_list);
+	TAILQ_INIT(&mac_label_journal_list);
+
+	mac_lck_grp_attr = lck_grp_attr_alloc_init();
+	lck_grp_attr_setstat(mac_lck_grp_attr);
+	mac_lck_grp = lck_grp_alloc_init("MAC lock", mac_lck_grp_attr);
+	mac_lck_attr = lck_attr_alloc_init();
+	lck_attr_setdefault(mac_lck_attr);
+	mac_policy_mtx = lck_mtx_alloc_init(mac_lck_grp, mac_lck_attr);
+	lck_attr_free(mac_lck_attr);
+	lck_grp_attr_free(mac_lck_grp_attr);
+	lck_grp_free(mac_lck_grp);
+	
+	mac_labelzone_init();
+}
+
+/*
+ * Init after early Mach startup, but before BSD
+ */
+void
+mac_policy_initmach(void)
+{
+
+	/*
+	 * For the purposes of modules that want to know if they were
+	 * loaded "early", set the mac_late flag once we've processed
+	 * modules either linked into the kernel, or loaded before the
+	 * kernel startup.
+	 */
+
+	load_security_extensions();
+	mac_late = 1;
+#if CONFIG_MACF_MACH
+	mac_label_journal_replay();
+#endif
+}
+
+/*
+ * BSD startup.
+ */
+void
+mac_policy_initbsd(void)
+{
+	struct mac_policy_conf *mpc;
+	u_int i;
+
+	mac_audit_data_zone = zinit(MAC_AUDIT_DATA_LIMIT,
+				    AQ_HIWATER * MAC_AUDIT_DATA_LIMIT,
+				    8192, "mac_audit_data_zone");
+
+	printf("MAC Framework successfully initialized\n");
+
+	/* Call bsd init functions of already loaded policies */
+
+	/*
+	 * Using the exclusive lock means no other framework entry
+	 * points can proceed while initializations are running.
+	 * This may not be necessary.
+	 */
+	mac_policy_grab_exclusive();
+
+	for (i = 0; i <= mac_policy_list.maxindex; i++) {
+		mpc = mac_get_mpc(i);
+		if ((mpc != NULL) && (mpc->mpc_ops->mpo_policy_initbsd != NULL))
+			(*(mpc->mpc_ops->mpo_policy_initbsd))(mpc);
+	}
+
+	mac_policy_release_exclusive();
+}
+
+/*
+ * After a policy has been loaded, add the label namespaces managed by the
+ * policy to either the static or non-static label namespace list.  
+ * A namespace is added to the the list only if it is not already on one of 
+ * the lists.
+ */
+void
+mac_policy_addto_labellist(mac_policy_handle_t handle, int static_entry)
+{
+	struct mac_label_listener **new_mlls;
+	struct mac_label_element *mle, **new_mles;
+	struct mac_label_element_list_t *list;
+	struct mac_policy_conf *mpc;
+	const char *name, *name2;
+	u_int idx, mle_free, mll_free;
+
+	mpc = mac_get_mpc(handle);
+
+	if (mpc->mpc_labelnames == NULL)
+		return;
+
+	if (mpc->mpc_labelname_count == 0)
+		return;
+
+	if (static_entry)
+		list = &mac_static_label_element_list;
+	else
+		list = &mac_label_element_list;
+
+	/*
+	 * Before we grab the policy list lock, allocate enough memory
+	 * to contain the potential new elements so we don't have to 
+	 * give up the lock, or allocate with the lock held.
+	 */
+	MALLOC(new_mles, struct mac_label_element **,
+	    sizeof(struct mac_label_element *) *
+	    mpc->mpc_labelname_count, M_MACTEMP, M_WAITOK | M_ZERO);
+	for (idx = 0; idx < mpc->mpc_labelname_count; idx++)
+		MALLOC(new_mles[idx], struct mac_label_element *, 
+		    sizeof(struct mac_label_element),
+		    M_MACTEMP, M_WAITOK);
+	mle_free = 0;
+	MALLOC(new_mlls, struct mac_label_listener **,
+	    sizeof(struct mac_label_listener *) *
+	    mpc->mpc_labelname_count, M_MACTEMP, M_WAITOK);
+	for (idx = 0; idx < mpc->mpc_labelname_count; idx++)
+		MALLOC(new_mlls[idx], struct mac_label_listener *,
+		    sizeof(struct mac_label_listener), M_MACTEMP, M_WAITOK);
+	mll_free = 0;
+
+	if (mac_late)
+		mac_policy_grab_exclusive();
+	for (idx = 0; idx < mpc->mpc_labelname_count; idx++) {
+
+		if (*(name = mpc->mpc_labelnames[idx]) == '?')
+			name++;
+		/*
+		 * Check both label element lists and add to the 
+		 * appropriate list only if not already on a list.
+		 */
+		LIST_FOREACH(mle, &mac_static_label_element_list, mle_list) {
+			if (*(name2 = mle->mle_name) == '?')
+				name2++;
+			if (strcmp(name, name2) == 0)
+				break;
+		}
+		if (mle == NULL) {
+			LIST_FOREACH(mle, &mac_label_element_list, mle_list) {
+				if (*(name2 = mle->mle_name) == '?')
+					name2++;
+				if (strcmp(name, name2) == 0)
+					break;
+			}
+		}
+		if (mle == NULL) {
+			mle = new_mles[mle_free];
+			strlcpy(mle->mle_name, mpc->mpc_labelnames[idx],
+					MAC_MAX_LABEL_ELEMENT_NAME);
+			LIST_INIT(&mle->mle_listeners);
+			LIST_INSERT_HEAD(list, mle, mle_list);
+			mle_free++;
+		}
+		/* Add policy handler as a listener. */
+		new_mlls[mll_free]->mll_handle = handle;
+		LIST_INSERT_HEAD(&mle->mle_listeners, new_mlls[mll_free],
+		    mll_list);
+		mll_free++;
+	}
+	if (mac_late)
+		mac_policy_release_exclusive();
+
+	/* Free up any unused label elements and listeners */
+	for (idx = mle_free; idx < mpc->mpc_labelname_count; idx++)
+		FREE(new_mles[idx], M_MACTEMP);
+	FREE(new_mles, M_MACTEMP);
+	for (idx = mll_free; idx < mpc->mpc_labelname_count; idx++)
+		FREE(new_mlls[idx], M_MACTEMP);
+	FREE(new_mlls, M_MACTEMP);
+}
+
+/*
+ * After a policy has been unloaded, remove the label namespaces that the
+ * the policy manages from the non-static list of namespaces.
+ * The removal only takes place when no other policy is interested in the
+ * namespace.
+ *
+ * Must be called with the policy exclusive lock held.
+ */
+void
+mac_policy_removefrom_labellist(mac_policy_handle_t handle)
+{
+	struct mac_label_listener *mll;
+	struct mac_label_element *mle;
+	struct mac_policy_conf *mpc;
+
+	mpc = mac_get_mpc(handle);
+
+	if (mpc->mpc_labelnames == NULL)
+		return;
+
+	if (mpc->mpc_labelname_count == 0)
+		return;
+
+	/*
+	 * Unregister policy as being interested in any label
+	 * namespaces.  If no other policy is listening, remove
+	 * that label element from the list.  Note that we only
+	 * have to worry about the non-static list.
+	 */
+	LIST_FOREACH(mle, &mac_label_element_list, mle_list) {
+		LIST_FOREACH(mll, &mle->mle_listeners, mll_list) {
+			if (mll->mll_handle == handle) {
+				LIST_REMOVE(mll, mll_list);
+				FREE(mll, M_MACTEMP);
+				if (LIST_EMPTY(&mle->mle_listeners)) {
+					LIST_REMOVE(mle, mle_list);
+					FREE(mle, M_MACTEMP);
+				}
+				return;
+			}
+		}
+	}
+}
+
+/*
+ * After the policy list has changed, walk the list to update any global
+ * flags.
+ */
+static void
+mac_policy_updateflags(void)
+{
+#if !defined(CONFIG_MACF_ALWAYS_LABEL_MBUF) && 0 /* port to new list style */
+
+	struct mac_policy_conf *tmpc;
+	int labelmbufs;
+
+	mac_policy_assert_exclusive();
+
+	labelmbufs = 0;
+
+	/* XXX - convert to new list structure */
+	LIST_FOREACH(tmpc, &mac_static_policy_list, mpc_list) {
+		if (tmpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_LABELMBUFS)
+			labelmbufs++;
+	}
+	LIST_FOREACH(tmpc, &mac_policy_list, mpc_list) {
+		if (tmpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_LABELMBUFS)
+			labelmbufs++;
+	}
+	mac_labelmbufs = (labelmbufs != 0);
+#endif
+}
+
+static __inline void
+mac_policy_fixup_mmd_list(struct mac_module_data *new)
+{
+	struct mac_module_data *old;
+	struct mac_module_data_element *ele, *aele;
+	struct mac_module_data_list *arr, *dict;
+	unsigned int i, j, k;
+
+	old = new->base_addr;
+	DPRINTF(("fixup_mmd: old %p new %p\n", old, new));
+	for (i = 0; i < new->count; i++) {
+		ele = &(new->data[i]);
+		DPRINTF(("fixup_mmd: ele %p\n", ele));
+		DPRINTF(("   key %p value %p\n", ele->key, ele->value));
+		mmd_fixup_ele(old, new, ele); /* Fix up key/value ptrs.       */
+		DPRINTF(("   key %p value %p\n", ele->key, ele->value));
+		if (ele->value_type == MAC_DATA_TYPE_ARRAY) {
+			arr = (struct mac_module_data_list *)ele->value;
+			DPRINTF(("fixup_mmd: array @%p\n", arr));
+			for (j = 0; j < arr->count; j++) {
+				aele = &(arr->list[j]);
+				DPRINTF(("fixup_mmd: aele %p\n", aele));
+				DPRINTF(("   key %p value %p\n", aele->key, aele->value));
+				mmd_fixup_ele(old, new, aele);
+				DPRINTF(("   key %p value %p\n", aele->key, aele->value));
+				if (arr->type == MAC_DATA_TYPE_DICT) {
+					dict = (struct mac_module_data_list *)aele->value;
+					DPRINTF(("fixup_mmd: dict @%p\n", dict));
+					for (k = 0; k < dict->count; k++)
+						mmd_fixup_ele(old, new,
+						    &(dict->list[k]));
+				}
+			}
+		}
+	}
+	new->base_addr = new;
+}
+
+int
+mac_policy_register(struct mac_policy_conf *mpc, mac_policy_handle_t *handlep,
+    void *xd)
+{
+	struct mac_policy_list_element *tmac_policy_list_element;
+	int error, slot, static_entry = 0;
+	u_int i;
+
+	/*
+	 * Some preliminary checks to make sure the policy's conf structure
+	 * contains the required fields.
+	 */
+	if (mpc->mpc_name == NULL)
+		panic("policy's name is not set\n");
+
+	if (mpc->mpc_fullname == NULL)
+		panic("policy's full name is not set\n");
+
+	if (mpc->mpc_labelname_count > MAC_MAX_MANAGED_NAMESPACES)
+		panic("policy's managed label namespaces exceeds maximum\n");
+
+	if (mpc->mpc_ops == NULL)
+		panic("policy's OPs field is NULL\n");
+
+	error = 0;
+
+	if (mac_late) {
+		if (mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_NOTLATE) {
+			printf("Module %s does not support late loading.\n",
+			    mpc->mpc_name);
+			return (EPERM);
+		}
+		mac_policy_grab_exclusive();
+	}
+
+	if (mac_policy_list.numloaded >= mac_policy_list.max) {
+		/* allocate new policy list array, zero new chunk */
+		tmac_policy_list_element =
+		    kalloc((sizeof(struct mac_policy_list_element) *
+		    MAC_POLICY_LIST_CHUNKSIZE) * (mac_policy_list.chunks + 1));
+		bzero(&tmac_policy_list_element[mac_policy_list.max],
+		    sizeof(struct mac_policy_list_element) *
+		    MAC_POLICY_LIST_CHUNKSIZE);
+		
+		/* copy old entries into new list */
+		memcpy(tmac_policy_list_element, mac_policy_list.entries, 
+		   sizeof(struct mac_policy_list_element) *
+		   MAC_POLICY_LIST_CHUNKSIZE * mac_policy_list.chunks);
+	
+		/* free old array */
+		kfree(mac_policy_list.entries,
+		    sizeof(struct mac_policy_list_element) *
+		    MAC_POLICY_LIST_CHUNKSIZE * mac_policy_list.chunks);
+		
+		mac_policy_list.entries = tmac_policy_list_element;
+
+		/* Update maximums, etc */
+		mac_policy_list.max += MAC_POLICY_LIST_CHUNKSIZE;
+		mac_policy_list.chunks++;
+	}
+
+	/* Check for policy with same name already loaded */
+	for (i = 0; i <= mac_policy_list.maxindex; i++) {
+		if (mac_policy_list.entries[i].mpc == NULL) 
+			continue;	
+
+		if (strcmp(mac_policy_list.entries[i].mpc->mpc_name,
+		    mpc->mpc_name) == 0) {
+			error = EEXIST;
+			goto out;
+		}
+	}
+
+	if (mpc->mpc_field_off != NULL) {
+		slot = ffs(mac_slot_offsets_free);
+		if (slot == 0) {
+			error = ENOMEM;
+			goto out;
+		}
+		slot--;
+		mac_slot_offsets_free &= ~(1 << slot);
+		*mpc->mpc_field_off = slot;
+	}
+	mpc->mpc_runtime_flags |= MPC_RUNTIME_FLAG_REGISTERED;
+
+	if (xd) {
+		struct mac_module_data *mmd = xd; /* module data from plist */
+
+		/* Make a copy of the data. */
+		mpc->mpc_data = (void *)kalloc(mmd->size);
+		if (mpc->mpc_data != NULL) {
+			memcpy(mpc->mpc_data, mmd, mmd->size);
+
+			/* Fix up pointers after copy. */
+			mac_policy_fixup_mmd_list(mpc->mpc_data);
+		}
+	}
+
+	/* Find the first free handle in the list (using our hint). */
+	for (i = mac_policy_list.freehint; i < mac_policy_list.max; i++) {
+		if (mac_policy_list.entries[i].mpc == NULL) {
+			*handlep = i;
+			mac_policy_list.freehint = ++i;
+			break;
+		}
+	}
+
+	/*
+	 * If we are loading a MAC module before the framework has
+	 * finished initializing or the module is not unloadable and
+	 * we can place its handle adjacent to the last static entry,
+	 * bump the static policy high water mark.
+	 * Static policies can get by with weaker locking requirements.
+	 */
+	if (!mac_late ||
+	    ((mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_UNLOADOK) == 0 &&
+	    *handlep == mac_policy_list.staticmax)) {
+		static_entry = 1;
+		mac_policy_list.staticmax++;
+	}
+
+	mac_policy_list.entries[*handlep].mpc = mpc;
+
+	/* Update counters, etc */
+	if (*handlep > mac_policy_list.maxindex) 
+		mac_policy_list.maxindex = *handlep;
+	mac_policy_list.numloaded++;
+	
+	/* Per-policy initialization. */
+	printf ("calling mpo_policy_init for %s\n", mpc->mpc_name);
+	if (mpc->mpc_ops->mpo_policy_init != NULL)
+		(*(mpc->mpc_ops->mpo_policy_init))(mpc);
+
+	if (mac_late && mpc->mpc_ops->mpo_policy_initbsd != NULL) {
+		printf ("calling mpo_policy_initbsd for %s\n", mpc->mpc_name);
+		(*(mpc->mpc_ops->mpo_policy_initbsd))(mpc);
+	}
+
+	mac_policy_updateflags();
+
+	if (mac_late)
+		mac_policy_release_exclusive();
+
+	mac_policy_addto_labellist(*handlep, static_entry);
+
+	printf("Security policy loaded: %s (%s)\n", mpc->mpc_fullname,
+	    mpc->mpc_name);
+
+	return (0);
+
+out:
+	if (mac_late)
+		mac_policy_release_exclusive();
+
+	return (error);
+}
+
+int
+mac_policy_unregister(mac_policy_handle_t handle)
+{
+	struct mac_policy_conf *mpc;
+
+	/*
+	 * If we fail the load, we may get a request to unload.  Check
+	 * to see if we did the run-time registration, and if not,
+	 * silently succeed.
+	 */
+	mac_policy_grab_exclusive();
+	mpc = mac_get_mpc(handle);
+	if ((mpc->mpc_runtime_flags & MPC_RUNTIME_FLAG_REGISTERED) == 0) {
+		mac_policy_release_exclusive();
+		return (0);
+	}
+
+#if 0
+	/*
+	 * Don't allow unloading modules with private data.
+	 */
+	if (mpc->mpc_field_off != NULL) {
+		MAC_POLICY_LIST_UNLOCK();
+		return (EBUSY);
+	}
+#endif
+	/*
+	 * Only allow the unload to proceed if the module is unloadable
+	 * by its own definition.
+	 */
+	if ((mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_UNLOADOK) == 0) {
+		mac_policy_release_exclusive();
+		return (EBUSY);
+	}
+
+	mac_policy_removefrom_labellist(handle);
+
+	mac_get_mpc(handle) = NULL;
+	if (handle < mac_policy_list.freehint &&
+	    handle >= mac_policy_list.staticmax)
+		mac_policy_list.freehint = handle;
+
+	if (handle == mac_policy_list.maxindex)
+		mac_policy_list.maxindex--;
+
+	mac_policy_list.numloaded--; 
+	if (mpc->mpc_field_off != NULL) {
+		mac_slot_offsets_free |= (1 << *mpc->mpc_field_off);
+	}
+
+	if (mpc->mpc_ops->mpo_policy_destroy != NULL)
+		(*(mpc->mpc_ops->mpo_policy_destroy))(mpc);
+
+	mpc->mpc_runtime_flags &= ~MPC_RUNTIME_FLAG_REGISTERED;
+	mac_policy_updateflags();
+
+	mac_policy_release_exclusive();
+
+	if (mpc->mpc_data) {
+		struct mac_module_data *mmd = mpc->mpc_data;
+		kfree(mmd, mmd->size);
+		mpc->mpc_data = NULL;
+	}
+
+	printf("Security policy unload: %s (%s)\n", mpc->mpc_fullname,
+	    mpc->mpc_name);
+
+	return (0);
+}
+
+/*
+ * Define an error value precedence, and given two arguments, selects the
+ * value with the higher precedence.
+ */
+int
+mac_error_select(int error1, int error2)
+{
+
+	/* Certain decision-making errors take top priority. */
+	if (error1 == EDEADLK || error2 == EDEADLK)
+		return (EDEADLK);
+
+	/* Invalid arguments should be reported where possible. */
+	if (error1 == EINVAL || error2 == EINVAL)
+		return (EINVAL);
+
+	/* Precedence goes to "visibility", with both process and file. */
+	if (error1 == ESRCH || error2 == ESRCH)
+		return (ESRCH);
+
+	if (error1 == ENOENT || error2 == ENOENT)
+		return (ENOENT);
+
+	/* Precedence goes to DAC/MAC protections. */
+	if (error1 == EACCES || error2 == EACCES)
+		return (EACCES);
+
+	/* Precedence goes to privilege. */
+	if (error1 == EPERM || error2 == EPERM)
+		return (EPERM);
+
+	/* Precedence goes to error over success; otherwise, arbitrary. */
+	if (error1 != 0)
+		return (error1);
+	return (error2);
+}
+
+void
+mac_label_init(struct label *label)
+{
+
+	bzero(label, sizeof(*label));
+	label->l_flags = MAC_FLAG_INITIALIZED;
+}
+
+void
+mac_label_destroy(struct label *label)
+{
+
+	KASSERT(label->l_flags & MAC_FLAG_INITIALIZED,
+	    ("destroying uninitialized label"));
+
+	bzero(label, sizeof(*label));
+	/* implicit: label->l_flags &= ~MAC_FLAG_INITIALIZED; */
+}
+
+int
+mac_port_check_service (struct label *subj, struct label *obj,
+    const char *s, const char *p)
+{
+	int error;
+
+	MAC_CHECK(port_check_service, subj, obj, s, p);
+	return (error);
+}
+
+int
+mac_port_label_compute(struct label *subj, struct label *obj,
+    const char *s, struct label *out)
+{
+	int error;
+
+	MAC_CHECK(port_label_compute, subj, obj, s, out);
+	return error;
+}
+
+int
+mac_check_structmac_consistent(struct user_mac *mac)
+{
+
+	if (mac->m_buflen > MAC_MAX_LABEL_BUF_LEN || mac->m_buflen == 0)
+		return (EINVAL);
+
+	return (0);
+}
+
+/*
+ * Get the external forms of labels from all policies, for a single
+ * label namespace or "*" for all namespaces.  Returns ENOENT if no policy
+ * is registered for the namespace, unless the namespace begins with a '?'.
+ */
+static int
+mac_label_externalize(size_t mpo_externalize_off, struct label *label,
+    const char *element, struct sbuf *sb)
+{
+	struct mac_policy_conf *mpc;
+	struct mac_label_listener *mll;
+	struct mac_label_element *mle;
+	struct mac_label_element_list_t *element_list;
+	const char *name;
+	int (*mpo_externalize)(struct label *, char *, struct sbuf *);
+	int all_labels = 0, ignorenotfound = 0, error = 0, busy = FALSE;
+	unsigned int count = 0;
+
+	if (element[0] == '?') {
+		element++;
+		ignorenotfound = 1;
+	} else if (element[0] == '*' && element[1] == '\0')
+		all_labels = 1;
+
+	element_list = &mac_static_label_element_list;
+element_loop:
+	LIST_FOREACH(mle, element_list, mle_list) {
+		name = mle->mle_name;
+		if (all_labels) {
+			if (*name == '?')
+			    continue;
+		} else {
+			if (*name == '?')
+				name++;
+			if (strcmp(name, element) != 0)
+				continue;
+		}
+		LIST_FOREACH(mll, &mle->mle_listeners, mll_list) {
+			mpc = mac_policy_list.entries[mll->mll_handle].mpc;
+			if (mpc == NULL)
+				continue;
+			mpo_externalize = *(typeof(mpo_externalize) *)
+			    ((char *)mpc->mpc_ops + mpo_externalize_off);
+			if (mpo_externalize == NULL)
+				continue;
+			error = sbuf_printf(sb, "%s/", name);
+			if (error)
+				goto done;
+			error = mpo_externalize(label, mle->mle_name, sb);
+			if (error) {
+				if (error != ENOENT)
+					goto done;
+				/*
+				 * If a policy doesn't have a label to
+				 * externalize it returns ENOENT.  This
+				 * may occur for policies that support
+				 * multiple label elements for some
+				 * (but not all) object types.
+				 */
+				sbuf_setpos(sb, sbuf_len(sb) -
+				    (strlen(name) + 1));
+				error = 0;
+				continue;
+			}
+			error = sbuf_putc(sb, ',');
+			if (error)
+				goto done;
+			count++;
+		}
+	}
+	/* If there are dynamic policies present, check their elements too. */
+	if (!busy && mac_policy_list_conditional_busy() == 1) {
+		element_list = &mac_label_element_list;
+		busy = TRUE;
+		goto element_loop;
+	}
+done:
+	if (busy)
+		mac_policy_list_unbusy();
+	if (!error && count == 0) {
+		if (!all_labels && !ignorenotfound)
+			error = ENOENT;	/* XXX: ENOLABEL? */
+	}
+	return (error);
+}
+
+/*
+ * Get the external forms of labels from all policies, for all label
+ * namespaces contained in a list.
+ */
+int
+mac_externalize(size_t mpo_externalize_off, struct label *label,
+    const char *elementlist, char *outbuf, size_t outbuflen)
+{
+	char *element;
+	struct sbuf sb;
+	int error = 0, len;
+
+	sbuf_new(&sb, outbuf, outbuflen, SBUF_FIXEDLEN);
+	while ((element = strsep(&elementlist, ",")) != NULL) {
+		error = mac_label_externalize(mpo_externalize_off, label,
+		    element, &sb);
+		if (error)
+			break;
+	}
+	if ((len = sbuf_len(&sb)) > 0)
+		sbuf_setpos(&sb, len - 1);	/* trim trailing comma */
+	sbuf_finish(&sb);
+	return (error);
+}
+
+/*
+ * Have all policies set the internal form of a label, for a single
+ * label namespace.
+ */
+static int
+mac_label_internalize(size_t mpo_internalize_off, struct label *label,
+    char *element_name, char *element_data)
+{
+	struct mac_policy_conf *mpc;
+	struct mac_label_listener *mll;
+	struct mac_label_element *mle;
+	struct mac_label_element_list_t *element_list;
+	int (*mpo_internalize)(struct label *, char *, char *);
+	int error = 0, busy = FALSE;
+	unsigned int count = 0;
+	const char *name;
+
+	element_list = &mac_static_label_element_list;
+element_loop:
+	LIST_FOREACH(mle, element_list, mle_list) {
+		if (*(name = mle->mle_name) == '?')
+			name++;
+		if (strcmp(element_name, name) != 0)
+			continue;
+		LIST_FOREACH(mll, &mle->mle_listeners, mll_list) {
+			mpc = mac_policy_list.entries[mll->mll_handle].mpc;
+			if (mpc == NULL)
+				continue;
+			mpo_internalize = *(typeof(mpo_internalize) *)
+			    ((char *)mpc->mpc_ops + mpo_internalize_off);
+			if (mpo_internalize == NULL)
+				continue;
+			error = mpo_internalize(label, element_name,
+			    element_data);
+			if (error)
+				goto done;
+			count++;
+		}
+	}
+	/* If there are dynamic policies present, check their elements too. */
+	if (!busy && mac_policy_list_conditional_busy() == 1) {
+		element_list = &mac_label_element_list;
+		busy = TRUE;
+		goto element_loop;
+	}
+done:
+	if (busy)
+		mac_policy_list_unbusy();
+	if (!error && count == 0)
+		error = ENOPOLICY;
+	return (error);
+}
+
+int
+mac_internalize(size_t mpo_internalize_off, struct label *label,
+    char *textlabels)
+{
+	char *element_name, *element_data;
+	int error = 0;
+
+	while (!error && (element_name = strsep(&textlabels, ",")) != NULL) {
+		element_data = strchr(element_name, '/');
+		if (element_data == NULL) {
+			error = EINVAL;
+			break;
+		}
+		*element_data++ = '\0';
+		error = mac_label_internalize(mpo_internalize_off, label,
+		    element_name, element_data);
+	}
+	return (error);
+}
+
+/* system calls */
+
+int
+__mac_get_pid(struct proc *p, struct __mac_get_pid_args *uap, register_t *ret __unused)
+{
+	char *elements, *buffer;
+	struct user_mac mac;
+	struct proc *tproc;
+	struct ucred *tcred;
+	int error;
+	size_t ulen;
+
+	AUDIT_ARG(pid, uap->pid);
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	tproc = proc_find(uap->pid);
+	if (tproc == NULL)
+		return (ESRCH);
+	tcred = kauth_cred_proc_ref(tproc);
+	proc_rele(tproc);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		kauth_cred_unref(&tcred);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	error = mac_cred_label_externalize(tcred->cr_label, elements,
+	    buffer, mac.m_buflen, M_WAITOK);
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer)+1);
+
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+	kauth_cred_unref(&tcred);
+	return (error);
+}
+
+int
+__mac_get_proc(proc_t p, struct __mac_get_proc_args *uap, register_t *ret __unused)
+{
+	char *elements, *buffer;
+	struct user_mac mac;
+	kauth_cred_t cr;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	cr = kauth_cred_proc_ref(p);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	error = mac_cred_label_externalize(cr->cr_label,
+	    elements, buffer, mac.m_buflen, M_WAITOK);
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer)+1);
+
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+	kauth_cred_unref(&cr);
+	return (error);
+}
+
+/*
+ * MPSAFE
+ */
+
+int
+__mac_set_proc(proc_t p, struct __mac_set_proc_args *uap, register_t *ret __unused)
+{
+	kauth_cred_t newcred, oldcred;
+	struct label *intlabel;
+	struct user_mac mac;
+	char *buffer;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, buffer, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, buffer);
+
+	intlabel = mac_cred_label_alloc();
+	error = mac_cred_label_internalize(intlabel, buffer);
+	FREE(buffer, M_MACTEMP);
+	if (error)
+		goto out;
+
+	error = mac_cred_check_label_update(kauth_cred_get(), intlabel);
+	if (error) {
+		goto out;
+	}
+
+	error = kauth_proc_label_update(p, intlabel);
+	if (error)
+		goto out;
+
+	newcred = kauth_cred_proc_ref(p);
+	mac_task_label_update_cred(newcred, p->task);
+
+#if 0
+	if (mac_vm_enforce) {
+		mutex_lock(Giant);			/* XXX FUNNEL? */
+		mac_cred_mmapped_drop_perms(p, newcred);
+		mutex_unlock(Giant);			/* XXX FUNNEL? */
+	}
+#endif
+
+	kauth_cred_unref(&newcred);
+out:
+	mac_cred_label_free(intlabel);
+	return (error);
+}
+
+#if CONFIG_LCTX
+int
+__mac_get_lcid(proc_t p, struct __mac_get_lcid_args *uap, register_t *ret __unused)
+{
+	char *elements, *buffer;
+	struct user_mac mac;
+	struct lctx *l;
+	int error;
+	size_t ulen;
+
+	AUDIT_ARG(value, uap->lcid);
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	l = lcfind(uap->lcid);
+	if (l == NULL)
+		return (ESRCH);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		LCTX_UNLOCK(l);
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = mac_lctx_label_externalize(l->lc_label, elements,
+					   buffer, mac.m_buflen);
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer)+1);
+
+	LCTX_UNLOCK(l);
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+	return (error);
+}
+
+int
+__mac_get_lctx(proc_t p, struct __mac_get_lctx_args *uap, register_t *ret __unused)
+{
+	char *elements, *buffer;
+	struct user_mac mac;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+
+	proc_lock(p);
+	if (p->p_lctx == NULL) {
+		proc_unlock(p);
+		error = ENOENT;
+		goto out;
+	}
+
+	error = mac_lctx_label_externalize(p->p_lctx->lc_label,
+					   elements, buffer, mac.m_buflen);
+	proc_unlock(p);
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer)+1);
+
+out:
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+	return (error);
+}
+
+int
+__mac_set_lctx(proc_t p, struct __mac_set_lctx_args *uap, register_t *ret __unused)
+{
+	struct user_mac mac;
+	struct label *intlabel;
+	char *buffer;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, buffer, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, buffer);
+
+	intlabel = mac_lctx_label_alloc();
+	error = mac_lctx_label_internalize(intlabel, buffer);
+	FREE(buffer, M_MACTEMP);
+	if (error)
+		goto out;
+
+	proc_lock(p);
+	if (p->p_lctx == NULL) {
+		proc_unlock(p);
+		error = ENOENT;
+		goto out;
+	}
+
+	error = mac_lctx_check_label_update(p->p_lctx, intlabel);
+	if (error) {
+		proc_unlock(p);
+		goto out;
+	}
+	mac_lctx_label_update(p->p_lctx, intlabel);
+	proc_unlock(p);
+out:
+	mac_lctx_label_free(intlabel);
+	return (error);
+}
+
+#else	/* LCTX */
+
+int
+__mac_get_lcid(proc_t p __unused, struct __mac_get_lcid_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_lctx(proc_t p __unused, struct __mac_get_lctx_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_lctx(proc_t p __unused, struct __mac_set_lctx_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+#endif	/* !LCTX */
+
+int
+__mac_get_fd(proc_t p, struct __mac_get_fd_args *uap, register_t *ret __unused)
+{
+	struct fileproc *fp;
+	struct vnode *vp;
+	struct user_mac mac;
+	char *elements, *buffer;
+	int error;
+	size_t ulen;
+	kauth_cred_t my_cred;
+#if CONFIG_MACF_SOCKET
+	struct socket *so;
+#endif	/* MAC_SOCKET */
+	struct label *intlabel;
+
+	AUDIT_ARG(fd, uap->fd);
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+
+	if (error) 
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+			
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = fp_lookup(p, uap->fd, &fp, 0);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	
+	my_cred = kauth_cred_proc_ref(p);
+	error = mac_file_check_get(my_cred, fp->f_fglob, elements, mac.m_buflen);
+	kauth_cred_unref(&my_cred);
+	if (error) {
+		fp_drop(p, uap->fd, fp, 0);
+		FREE(buffer, M_MACTEMP);
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	
+	switch (fp->f_fglob->fg_type) {
+		case DTYPE_VNODE:
+
+			intlabel = mac_vnode_label_alloc();
+			vp = (struct vnode *)fp->f_fglob->fg_data;
+
+			error = vnode_getwithref(vp);
+			if (error == 0) {
+				mac_vnode_label_copy(vp->v_label, intlabel);
+				error = mac_vnode_label_externalize(intlabel,
+						elements, buffer,
+						mac.m_buflen, M_WAITOK);
+				vnode_put(vp);
+			}
+			mac_vnode_label_free(intlabel);
+			break;
+		case DTYPE_SOCKET:
+#if CONFIG_MACF_SOCKET
+			so = (struct socket *) fp->f_fglob->fg_data;
+			intlabel = mac_socket_label_alloc(MAC_WAITOK);
+			sock_lock(so, 1);
+			mac_socket_label_copy(so->so_label, intlabel);
+			sock_unlock(so, 1);
+			error = mac_socket_label_externalize(intlabel, elements, buffer, mac.m_buflen);
+			mac_socket_label_free(intlabel);
+			break;
+#endif
+		case DTYPE_PSXSHM:
+		case DTYPE_PSXSEM:
+		case DTYPE_PIPE:
+		case DTYPE_KQUEUE:
+		case DTYPE_FSEVENTS:
+		default:
+			error = ENOSYS;   // only sockets/vnodes so far
+			break;
+	}
+	fp_drop(p, uap->fd, fp, 0);
+	
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer)+1);
+		
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+	return (error);
+}
+
+/*
+ * MPSAFE
+ */
+static int
+mac_get_filelink(proc_t p, user_addr_t mac_p, user_addr_t path_p, int follow)
+{
+	struct vnode *vp;
+	vfs_context_t ctx;
+	char *elements, *buffer;
+	struct nameidata nd;
+	struct label *intlabel;
+	struct user_mac mac;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	ctx = vfs_context_current();
+
+	NDINIT(&nd, LOOKUP,
+		LOCKLEAF | (follow ? FOLLOW : NOFOLLOW) | AUDITVNPATH1,
+		UIO_USERSPACE, path_p, ctx);
+	error = namei(&nd);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	vp = nd.ni_vp;
+
+	nameidone(&nd);
+
+	intlabel = mac_vnode_label_alloc();
+	mac_vnode_label_copy(vp->v_label, intlabel);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	error = mac_vnode_label_externalize(intlabel, elements, buffer,
+					    mac.m_buflen, M_WAITOK);
+	FREE(elements, M_MACTEMP);
+
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer) + 1);
+	FREE(buffer, M_MACTEMP);
+
+	vnode_put(vp);
+	mac_vnode_label_free(intlabel);
+
+	return (error);
+}
+
+int
+__mac_get_file(proc_t p, struct __mac_get_file_args *uap,
+	       register_t *ret __unused)
+{
+
+	return (mac_get_filelink(p, uap->mac_p, uap->path_p, 1));
+}
+
+int
+__mac_get_link(proc_t p, struct __mac_get_link_args *uap,
+	       register_t *ret __unused)
+{
+
+	return (mac_get_filelink(p, uap->mac_p, uap->path_p, 0));
+}
+
+int
+__mac_set_fd(proc_t p, struct __mac_set_fd_args *uap, register_t *ret __unused)
+{
+
+	struct fileproc *fp;
+	struct user_mac mac;
+	struct vfs_context *ctx = vfs_context_current();
+	int error;
+	size_t ulen;
+	char *buffer;
+	struct label *intlabel;
+#if CONFIG_MACF_SOCKET
+	struct socket *so;
+#endif
+	struct vnode *vp;
+
+	AUDIT_ARG(fd, uap->fd);
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(uap->mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(uap->mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error) 
+		return (error);
+		
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+	
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, buffer, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, buffer);
+	
+	error = fp_lookup(p, uap->fd, &fp, 0);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	
+
+	error = mac_file_check_set(vfs_context_ucred(ctx), fp->f_fglob, buffer, mac.m_buflen);
+	if (error) {
+		fp_drop(p, uap->fd, fp, 0);
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	
+	switch (fp->f_fglob->fg_type) {
+
+		case DTYPE_VNODE:
+			intlabel = mac_vnode_label_alloc();
+
+			error = mac_vnode_label_internalize(intlabel, buffer);
+			if (error) {
+				mac_vnode_label_free(intlabel);
+				break;
+			}
+
+
+			vp = (struct vnode *)fp->f_fglob->fg_data;
+
+			error = vnode_getwithref(vp);
+			if (error == 0) {
+				error = vn_setlabel(vp, intlabel, ctx);
+				vnode_put(vp);
+			}
+			mac_vnode_label_free(intlabel);
+			break;
+
+		case DTYPE_SOCKET:
+#if CONFIG_MACF_SOCKET
+			intlabel = mac_socket_label_alloc(MAC_WAITOK);
+			error = mac_socket_label_internalize(intlabel, buffer);
+			if (error == 0) {
+				so = (struct socket *) fp->f_fglob->fg_data;
+				SOCK_LOCK(so);
+				error = mac_socket_label_update(vfs_context_ucred(ctx), so, intlabel);
+				SOCK_UNLOCK(so);
+			}
+			mac_socket_label_free(intlabel);
+			break;
+#endif
+		case DTYPE_PSXSHM:
+		case DTYPE_PSXSEM:
+		case DTYPE_PIPE:
+		case DTYPE_KQUEUE:
+		case DTYPE_FSEVENTS:
+		default:
+			error = ENOSYS;  // only sockets/vnodes so far
+			break;
+	}
+
+	fp_drop(p, uap->fd, fp, 0);
+	FREE(buffer, M_MACTEMP);
+	return (error);
+}
+
+/*
+ * MPSAFE
+ */
+static int
+mac_set_filelink(proc_t p, user_addr_t mac_p, user_addr_t path_p,
+		 int follow)
+{
+	register struct vnode *vp;
+	struct vfs_context *ctx = vfs_context_current();
+	struct label *intlabel;
+	struct nameidata nd;
+	struct user_mac mac;
+	char *buffer;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(p)) {
+		error = copyin(mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error) {
+		printf("mac_set_file: failed structure consistency check\n");
+		return (error);
+	}
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, buffer, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, buffer);
+
+	intlabel = mac_vnode_label_alloc();
+	error = mac_vnode_label_internalize(intlabel, buffer);
+	FREE(buffer, M_MACTEMP);
+	if (error) {
+		mac_vnode_label_free(intlabel);
+		return (error);
+	}
+
+	NDINIT(&nd, LOOKUP,
+		LOCKLEAF | (follow ? FOLLOW : NOFOLLOW) | AUDITVNPATH1,
+		UIO_USERSPACE, path_p, ctx);
+	error = namei(&nd);
+	if (error) {
+		mac_vnode_label_free(intlabel);
+		return (error);
+	}
+	vp = nd.ni_vp;
+
+	nameidone(&nd);
+
+	error = vn_setlabel(vp, intlabel, ctx);
+	vnode_put(vp);
+	mac_vnode_label_free(intlabel);
+
+	return (error);
+}
+
+int
+__mac_set_file(proc_t p, struct __mac_set_file_args *uap,
+	       register_t *ret __unused)
+{
+
+	return (mac_set_filelink(p, uap->mac_p, uap->path_p, 1));
+}
+
+int
+__mac_set_link(proc_t p, struct __mac_set_link_args *uap,
+	       register_t *ret __unused)
+{
+
+	return (mac_set_filelink(p, uap->mac_p, uap->path_p, 0));
+}
+
+/*
+ * MPSAFE
+ */
+
+int
+__mac_syscall(proc_t p, struct __mac_syscall_args *uap, register_t *retv __unused)
+{
+	struct mac_policy_conf *mpc;
+	char target[MAC_MAX_POLICY_NAME];
+	int error;
+	u_int i;
+	size_t ulen;
+
+	error = copyinstr(uap->policy, target, sizeof(target), &ulen);
+	if (error)
+		return (error);
+	AUDIT_ARG(value, uap->call);
+	AUDIT_ARG(mac_string, target);
+
+	error = ENOPOLICY;
+
+	for (i = 0; i < mac_policy_list.staticmax; i++) {
+		mpc = mac_policy_list.entries[i].mpc;
+		if (mpc == NULL)
+			continue;
+
+		if (strcmp(mpc->mpc_name, target) == 0 &&
+		    mpc->mpc_ops->mpo_policy_syscall != NULL) {
+			error = mpc->mpc_ops->mpo_policy_syscall(p,
+			    uap->call, uap->arg);
+			goto done;
+ 		}
+	}
+	if (mac_policy_list_conditional_busy() != 0) {
+		for (; i <= mac_policy_list.maxindex; i++) {
+			mpc = mac_policy_list.entries[i].mpc;
+			if (mpc == NULL)
+				continue;
+
+			if (strcmp(mpc->mpc_name, target) == 0 &&
+			    mpc->mpc_ops->mpo_policy_syscall != NULL) {
+				error = mpc->mpc_ops->mpo_policy_syscall(p,
+				    uap->call, uap->arg);
+				break;
+			}
+		}
+		mac_policy_list_unbusy();
+	}
+
+done:
+	return (error);
+}
+
+int
+mac_mount_label_get(struct mount *mp, user_addr_t mac_p)
+{
+	char *elements, *buffer;
+	struct label *label;
+	struct user_mac mac;
+	int error;
+	size_t ulen;
+
+	if (IS_64BIT_PROCESS(current_proc())) {
+		error = copyin(mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(mac.m_string, elements, mac.m_buflen, &ulen);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	label = mp->mnt_mntlabel;
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	error = mac_mount_label_externalize(label, elements, buffer,
+	    mac.m_buflen);
+	FREE(elements, M_MACTEMP);
+
+	if (error == 0)
+		error = copyout(buffer, mac.m_string, strlen(buffer) + 1);
+	FREE(buffer, M_MACTEMP);
+
+	return (error);
+}
+
+int
+__mac_get_mount(proc_t p __unused, struct __mac_get_mount_args *uap,
+    register_t *ret __unused)
+{
+	struct nameidata nd;
+	struct vfs_context *ctx = vfs_context_current();
+	struct mount *mp;
+	int error;
+
+	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNPATH1,
+		UIO_USERSPACE, uap->path, ctx);
+	error = namei(&nd);
+	if (error) {
+		return (error);
+	}
+	mp = nd.ni_vp->v_mount;
+	nameidone(&nd);
+
+	return mac_mount_label_get(mp, uap->mac_p);
+}
+
+#else /* MAC */
+
+int
+mac_policy_register(struct mac_policy_conf *mpc __unused, 
+	mac_policy_handle_t *handlep __unused, void *xd __unused)
+{
+
+	return (0);
+}
+
+int
+mac_policy_unregister(mac_policy_handle_t handle __unused)
+{
+
+	return (0);
+}
+
+int
+mac_audit_text(char *text __unused, mac_policy_handle_t handle __unused)
+{
+
+	return (0);
+}
+
+int
+mac_mount_label_get(struct mount *mp __unused, user_addr_t mac_p __unused)
+{
+	return (ENOSYS);
+}
+
+int
+mac_vnop_setxattr(struct vnode *vp __unused, const char *name __unused, char *buf __unused, size_t len __unused)
+{
+
+	return (ENOENT);
+}
+
+int
+mac_vnop_getxattr(struct vnode *vp __unused, const char *name __unused, 
+	char *buf __unused, size_t len __unused, size_t *attrlen __unused)
+{
+
+	return (ENOENT);
+}
+
+int
+mac_vnop_removexattr(struct vnode *vp __unused, const char *name __unused)
+{
+
+	return (ENOENT);
+}
+
+int
+__mac_get_pid(proc_t p __unused, struct __mac_get_pid_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_proc(proc_t p __unused, struct __mac_get_proc_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_proc(proc_t p __unused, struct __mac_set_proc_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_file(proc_t p __unused, struct __mac_get_file_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_link(proc_t p __unused, struct __mac_get_link_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_file(proc_t p __unused, struct __mac_set_file_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_link(proc_t p __unused, struct __mac_set_link_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_fd(proc_t p __unused, struct __mac_get_fd_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_fd(proc_t p __unused, struct __mac_set_fd_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_syscall(proc_t p __unused, struct __mac_syscall_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_lcid(proc_t p __unused, struct __mac_get_lcid_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_lctx(proc_t p __unused, struct __mac_get_lctx_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_set_lctx(proc_t p __unused, struct __mac_set_lctx_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+
+int
+__mac_get_mount(proc_t p __unused,
+    struct __mac_get_mount_args *uap __unused, register_t *ret __unused)
+{
+
+	return (ENOSYS);
+}
+#endif /* !MAC */
diff --git a/security/mac_data.c b/security/mac_data.c
new file mode 100644
index 000000000..ffe50237d
--- /dev/null
+++ b/security/mac_data.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/types.h> 
+
+#include "mac_internal.h"
+
+int
+mac_find_policy_data(const mac_policy_handle_t handle, const char *key,
+    void **valp, size_t *sizep)
+{
+	struct mac_policy_conf *mpc;
+	int error = ENOENT;
+
+	if ((mpc = mac_get_mpc(handle)) != NULL)
+		error = mac_find_module_data(mpc->mpc_data, key, valp, sizep);
+	return (error);
+}
+
+int
+mac_find_module_data(struct mac_module_data *mmd, const char *key,
+    void **valp, size_t *sizep)
+{
+	int error = ENOENT;
+	unsigned int i;
+
+	if (mmd != NULL) {
+		for (i = 0; i < mmd->count; i++) {
+			if (strcmp(key, mmd->data[i].key) == 0) {
+				*valp = mmd->data[i].value;
+				*sizep = (size_t)mmd->data[i].value_size;
+				error = 0;
+				break;
+			}
+		}
+	}
+
+	return (error);
+}
diff --git a/security/mac_data.h b/security/mac_data.h
new file mode 100644
index 000000000..f1b78dd54
--- /dev/null
+++ b/security/mac_data.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SECURITY_MAC_DATA_H_
+#define	_SECURITY_MAC_DATA_H_
+
+/**
+  @brief Mac policy module data
+
+  This structure specifies module data that is passed in to the
+  TrustedBSD MAC policy module by the kernel module loader.  The
+  data is made of up key/value pairs where the key is always a
+  string and the value is a string, binary data or array.  An array
+  may be a list of values (actually a similar set of key/value pairs,
+  but in this case the keys are always null), and may also consist of
+  a set of dictionaries, which in turn are made up of a list of key/value
+  pairs.
+
+  Module data may be specified in the MAC policy module's
+  Info.plist file as part of the OSModuleData dictionary.
+
+  E.g.
+
+  <key>OSModuleData</key>
+  <dict>
+	<key>foo</key>
+	<string>bar</string>
+	<key>Beers</key>
+	<array>
+	<dict>
+		<key>type</key>
+		<string>lager</string>
+		<key>Name</key>
+		<string>Anchor Steam</string>
+	</dict>
+	<dict>
+		<key>type</key>
+		<string>ale</string>
+		<key>Name</key>
+		<string>Sierra Nevada Pale Ale</string>
+	</dict>
+	</array>
+  </dict>
+
+*/
+struct mac_module_data_element {
+	unsigned int key_size;
+	unsigned int value_size;
+	unsigned int value_type;
+	char *key;
+	char *value;
+};
+struct mac_module_data_list {
+	unsigned int count;
+	unsigned int type;
+	struct mac_module_data_element list[1];
+};
+struct mac_module_data {
+	void *base_addr;		/* Orig base address, for ptr fixup.  */
+	unsigned int size;
+	unsigned int count;
+	struct mac_module_data_element data[1];	/* actually bigger */
+};
+
+#define MAC_DATA_TYPE_PRIMITIVE	0	/* Primitive type (int, string, etc.) */
+#define MAC_DATA_TYPE_ARRAY	1	/* Array type.                        */
+#define MAC_DATA_TYPE_DICT	2	/* Dictionary type.                   */
+
+#ifdef _SECURITY_MAC_POLICY_H_
+/* XXX mac_policy_handle_t is defined in mac_policy.h, move prototype there? */
+int mac_find_policy_data(const mac_policy_handle_t, const char *key,
+    void **valp, size_t *sizep);
+int mac_find_module_data(struct mac_module_data *mmd, const char *key,
+    void **valp, size_t *sizep);
+
+/*
+ * This is a routine to fix up pointers in a mac_module_data_element when the
+ * mac_module_data has been copied to a new area.  It depends on the pointers
+ * all being offset from base_addr.
+ */
+static __inline void
+mmd_fixup_ele(struct mac_module_data *oldbase,
+    struct mac_module_data *newbase, struct mac_module_data_element *ele)
+{
+	if (ele->key != NULL) {		/* Array elements have no keys.       */
+		ele->key -= (unsigned int)oldbase;
+		ele->key += (unsigned int)newbase;
+	}
+	ele->value -= (unsigned int)oldbase;
+	ele->value += (unsigned int)newbase;
+}
+
+#endif
+
+#endif /* !_SECURITY_MAC_DATA_H_ */
diff --git a/security/mac_file.c b/security/mac_file.c
new file mode 100644
index 000000000..626b3e0b8
--- /dev/null
+++ b/security/mac_file.c
@@ -0,0 +1,230 @@
+/*-
+ * Copyright (c) 2002, 2003 Networks Associates Technology, Inc.
+ * Copyright (c) 2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/vnode.h>
+#include <sys/vnode_internal.h>
+#include <sys/file.h>
+#include <sys/file_internal.h>
+
+#include <security/mac_internal.h>
+
+
+static struct label *
+mac_file_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(file_label_init, label);
+	return (label);
+}
+
+void
+mac_file_label_init(struct fileglob *fg)
+{
+
+	fg->fg_label = mac_file_label_alloc();
+}
+
+static void
+mac_file_label_free(struct label *label)
+{
+
+	MAC_PERFORM(file_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_file_label_associate(struct ucred *cred, struct fileglob *fg)
+{
+
+	MAC_PERFORM(file_label_associate, cred, fg, fg->fg_label);
+}
+
+void
+mac_file_label_destroy(struct fileglob *fg)
+{
+
+	mac_file_label_free(fg->fg_label);
+	fg->fg_label = NULL;
+}
+
+int
+mac_file_check_create(struct ucred *cred)
+{
+	int error;
+
+	MAC_CHECK(file_check_create, cred);
+	return (error);
+}
+
+int
+mac_file_check_dup(struct ucred *cred, struct fileglob *fg, int newfd)
+{
+	int error;
+
+	MAC_CHECK(file_check_dup, cred, fg, fg->fg_label, newfd);
+	return (error);
+}
+
+int
+mac_file_check_fcntl(struct ucred *cred, struct fileglob *fg, int cmd,
+    user_long_t arg)
+{
+	int error;
+
+	MAC_CHECK(file_check_fcntl, cred, fg, fg->fg_label, cmd, arg);
+	return (error);
+}
+
+int
+mac_file_check_ioctl(struct ucred *cred, struct fileglob *fg, u_int cmd)
+{
+	int error;
+
+	MAC_CHECK(file_check_ioctl, cred, fg, fg->fg_label, cmd);
+	return (error);
+}
+
+int
+mac_file_check_inherit(struct ucred *cred, struct fileglob *fg)
+{
+	int error;
+
+	MAC_CHECK(file_check_inherit, cred, fg, fg->fg_label);
+	return (error);
+}
+
+int
+mac_file_check_receive(struct ucred *cred, struct fileglob *fg)
+{
+	int error;
+
+	MAC_CHECK(file_check_receive, cred, fg, fg->fg_label);
+	return (error);
+}
+
+int
+mac_file_check_get_offset(struct ucred *cred, struct fileglob *fg)
+{
+	int error;
+
+	MAC_CHECK(file_check_get_offset, cred, fg, fg->fg_label);
+	return (error);
+}
+
+int
+mac_file_check_change_offset(struct ucred *cred, struct fileglob *fg)
+{
+	int error;
+
+	MAC_CHECK(file_check_change_offset, cred, fg, fg->fg_label);
+	return (error);
+}
+ 
+int
+mac_file_check_get(struct ucred *cred, struct fileglob *fg, char *elements,
+    int len)
+{
+	int error;
+	
+	MAC_CHECK(file_check_get, cred, fg, elements, len);
+	return (error);
+}
+
+int
+mac_file_check_set(struct ucred *cred, struct fileglob *fg, char *buf,
+    int buflen)
+{
+	int error;
+	
+	MAC_CHECK(file_check_set, cred, fg, buf, buflen);
+	return (error);
+}
+
+int
+mac_file_check_lock(struct ucred *cred, struct fileglob *fg, int op,
+    struct flock *fl)
+{
+	int error;
+	
+	MAC_CHECK(file_check_lock, cred, fg, fg->fg_label, op, fl);
+	return (error);
+}
+
+/*
+ * On some platforms, VM_PROT_READ implies VM_PROT_EXECUTE. If that is true,
+ * both prot and maxprot will have VM_PROT_EXECUTE set after file_check_mmap
+ * if VM_PROT_READ is set.
+ *
+ * The type of maxprot in file_check_mmap must be equivalent to vm_prot_t *
+ * (defined in <mach/vm_prot.h>). mac_policy.h does not include any header
+ * files, so cannot use the typedef itself.
+ */
+int
+mac_file_check_mmap(struct ucred *cred, struct fileglob *fg, int prot,
+    int flags, int *maxprot)
+{
+	int error;
+	int maxp;
+
+	maxp = *maxprot;
+	MAC_CHECK(file_check_mmap, cred, fg, fg->fg_label, prot, flags, &maxp);
+	if ((maxp | *maxprot) != *maxprot)
+		panic("file_check_mmap increased max protections");
+	*maxprot = maxp;
+	return (error);
+}
+
+void
+mac_file_check_mmap_downgrade(struct ucred *cred, struct fileglob *fg,
+    int *prot)
+{
+	int result = *prot;
+
+	MAC_PERFORM(file_check_mmap_downgrade, cred, fg, fg->fg_label,
+	    &result);
+
+	*prot = result;
+}
diff --git a/security/mac_framework.h b/security/mac_framework.h
new file mode 100644
index 000000000..4b9613d2f
--- /dev/null
+++ b/security/mac_framework.h
@@ -0,0 +1,526 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001-2005 Networks Associates Technology, Inc.
+ * Copyright (c) 2005-2007 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson for the TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/sys/sys/mac.h,v 1.40 2003/04/18 19:57:37 rwatson Exp $
+ *
+ */
+/*
+ * Kernel interface for Mandatory Access Control -- how kernel services
+ * interact with the TrustedBSD MAC Framework.
+ */
+
+#ifndef _SECURITY_MAC_FRAMEWORK_H_
+#define	_SECURITY_MAC_FRAMEWORK_H_
+
+#ifndef KERNEL
+#error "no user-serviceable parts inside"
+#endif
+
+#if CONFIG_MACF
+
+struct attrlist;
+struct auditinfo;
+struct bpf_d;
+struct componentname;
+struct devnode;
+struct flock;
+struct fdescnode;
+struct fileglob;
+struct fileproc;
+struct ifnet;
+struct ifreq;
+struct image_params;
+struct inpcb;
+struct ipq;
+struct knote;
+struct lctx;
+struct m_tag;
+struct mac;
+struct mac_module_data;
+struct mbuf;
+struct msg;
+struct msqid_kernel;
+struct mount;
+struct pipe;
+struct proc;
+struct pseminfo;
+struct pshminfo;
+struct semid_kernel;
+struct shmid_kernel;
+struct sockaddr;
+struct sockopt;
+struct socket;
+struct task;
+struct thread;
+struct timespec;
+struct ucred;
+struct uio;
+struct vfs_attr;
+struct vfs_context;
+struct vnode;
+struct vnode_attr;
+struct vop_setlabel_args;
+
+/*@ macros */
+#define	VNODE_LABEL_CREATE	1
+#define	VNODE_LABEL_NEEDREF	2
+
+#if CONFIG_MACF_MACH
+#define mac_task_label_update_cred(cred, task)				\
+        mac_task_label_update_internal(((cred)->cr_label), task)
+#else
+#define mac_task_label_update_cred(cred, task)
+#endif
+
+/*@ === */
+int	mac_audit_check_postselect(kauth_cred_t cred, unsigned short syscode,
+	    void *args, int error, int retval, int mac_forced);
+int	mac_audit_check_preselect(kauth_cred_t cred, unsigned short syscode,
+	    void *args);
+int	mac_bpfdesc_check_receive(struct bpf_d *bpf_d, struct ifnet *ifp);
+void	mac_bpfdesc_label_destroy(struct bpf_d *bpf_d);
+void	mac_bpfdesc_label_init(struct bpf_d *bpf_d);
+void	mac_bpfdesc_label_associate(kauth_cred_t cred, struct bpf_d *bpf_d);
+int	mac_cred_check_label_update(kauth_cred_t cred,
+	    struct label *newlabel);
+int	mac_cred_check_label_update_execve(vfs_context_t ctx,
+	    struct vnode *vp, struct label *scriptvnodelabel, 
+	    struct label *execlabel, proc_t proc);
+int	mac_cred_check_visible(kauth_cred_t u1, kauth_cred_t u2);
+struct label	*mac_cred_label_alloc(void);
+void	mac_cred_label_associate(kauth_cred_t cred_parent,
+	    kauth_cred_t cred_child);
+void	mac_cred_label_associate_fork(kauth_cred_t cred, proc_t child);
+void	mac_cred_label_associate_kernel(kauth_cred_t cred);
+void	mac_cred_label_associate_user(kauth_cred_t cred);
+void	mac_cred_label_destroy(kauth_cred_t cred);
+int	mac_cred_label_externalize_audit(proc_t p, struct mac *mac);
+void	mac_cred_label_free(struct label *label);
+void	mac_cred_label_init(kauth_cred_t cred);
+void	mac_cred_label_update(kauth_cred_t cred, struct label *newlabel);
+void	mac_cred_label_update_execve(vfs_context_t ctx, kauth_cred_t newcred,
+	    struct vnode *vp, struct label *scriptvnodelabel,
+	    struct label *execlabel);
+void	mac_devfs_label_associate_device(dev_t dev, struct devnode *de,
+	    const char *fullpath);
+void	mac_devfs_label_associate_directory(const char *dirname, int dirnamelen,
+	    struct devnode *de, const char *fullpath);
+void	mac_devfs_label_copy(struct label *, struct label *label);
+void	mac_devfs_label_destroy(struct devnode *de);
+void	mac_devfs_label_init(struct devnode *de);
+void	mac_devfs_label_update(struct mount *mp, struct devnode *de,
+	    struct vnode *vp);
+int	mac_execve_enter(user_addr_t mac_p, struct image_params *imgp);
+int	mac_file_check_change_offset(kauth_cred_t cred, struct fileglob *fg);
+int	mac_file_check_create(kauth_cred_t cred);
+int	mac_file_check_dup(kauth_cred_t cred, struct fileglob *fg, int newfd);
+int	mac_file_check_fcntl(kauth_cred_t cred, struct fileglob *fg, int cmd,
+	    user_long_t arg);
+int	mac_file_check_get(kauth_cred_t cred, struct fileglob *fg,
+	    char *elements, int len);
+int	mac_file_check_get_offset(kauth_cred_t cred, struct fileglob *fg);
+int	mac_file_check_inherit(kauth_cred_t cred, struct fileglob *fg);
+int	mac_file_check_ioctl(kauth_cred_t cred, struct fileglob *fg,
+	    unsigned int cmd);
+int	mac_file_check_lock(kauth_cred_t cred, struct fileglob *fg, int op,
+	    struct flock *fl);
+int	mac_file_check_mmap(kauth_cred_t cred, struct fileglob *fg,
+	    int prot, int flags, int *maxprot);
+void	mac_file_check_mmap_downgrade(kauth_cred_t cred, struct fileglob *fg,
+	    int *prot);
+int	mac_file_check_receive(kauth_cred_t cred, struct fileglob *fg);
+int	mac_file_check_set(kauth_cred_t cred, struct fileglob *fg,
+	    char *bufp, int buflen);
+void	mac_file_label_associate(kauth_cred_t cred, struct fileglob *fg);
+void	mac_file_label_destroy(struct fileglob *fg);
+void	mac_file_label_init(struct fileglob *fg);
+int	mac_ifnet_check_transmit(struct ifnet *ifp, struct mbuf *mbuf,
+	    int family, int type);
+void	mac_ifnet_label_associate(struct ifnet *ifp);
+void	mac_ifnet_label_destroy(struct ifnet *ifp);
+int	mac_ifnet_label_get(kauth_cred_t cred, struct ifreq *ifr,
+	    struct ifnet *ifp);
+void	mac_ifnet_label_init(struct ifnet *ifp);
+void	mac_ifnet_label_recycle(struct ifnet *ifp);
+int	mac_ifnet_label_set(kauth_cred_t cred, struct ifreq *ifr,
+	    struct ifnet *ifp);
+int	mac_inpcb_check_deliver(struct inpcb *inp, struct mbuf *mbuf,
+	    int family, int type);
+void	mac_inpcb_label_associate(struct socket *so, struct inpcb *inp);
+void	mac_inpcb_label_destroy(struct inpcb *inp);
+int	mac_inpcb_label_init(struct inpcb *inp, int flag);
+void	mac_inpcb_label_recycle(struct inpcb *inp);
+void	mac_inpcb_label_update(struct socket *so);
+int	mac_iokit_check_device(char *devtype, struct mac_module_data *mdata);
+void	mac_ipq_label_associate(struct mbuf *fragment, struct ipq *ipq);
+int	mac_ipq_label_compare(struct mbuf *fragment, struct ipq *ipq);
+void	mac_ipq_label_destroy(struct ipq *ipq);
+int	mac_ipq_label_init(struct ipq *ipq, int flag);
+void	mac_ipq_label_update(struct mbuf *fragment, struct ipq *ipq);
+struct label	*mac_lctx_label_alloc(void);
+void    mac_lctx_label_free(struct label *label);
+void	mac_lctx_label_update(struct lctx *l, struct label *newlabel);
+int	mac_lctx_check_label_update(struct lctx *l, struct label *newlabel);
+void	mac_lctx_notify_create(proc_t proc, struct lctx *l);
+void	mac_lctx_notify_join(proc_t proc, struct lctx *l);
+void	mac_lctx_notify_leave(proc_t proc, struct lctx *l);
+void	mac_mbuf_label_associate_bpfdesc(struct bpf_d *bpf_d, struct mbuf *m);
+void	mac_mbuf_label_associate_ifnet(struct ifnet *ifp, struct mbuf *m);
+void	mac_mbuf_label_associate_inpcb(struct inpcb *inp, struct mbuf *m);
+void	mac_mbuf_label_associate_ipq(struct ipq *ipq, struct mbuf *mbuf);
+void	mac_mbuf_label_associate_linklayer(struct ifnet *ifp, struct mbuf *m);
+void	mac_mbuf_label_associate_multicast_encap(struct mbuf *oldmbuf,
+	    struct ifnet *ifp, struct mbuf *newmbuf);
+void	mac_mbuf_label_associate_netlayer(struct mbuf *oldmbuf,
+	    struct mbuf *newmbuf);
+void	mac_mbuf_label_associate_socket(struct socket *so, struct mbuf *m);
+void	mac_mbuf_label_copy(struct mbuf *m_from, struct mbuf *m_to);
+void	mac_mbuf_label_destroy(struct mbuf *m);
+int	mac_mbuf_label_init(struct mbuf *m, int flag);
+void	mac_mbuf_tag_copy(struct m_tag *m, struct m_tag *mtag);
+void	mac_mbuf_tag_destroy(struct m_tag *mtag);
+int	mac_mbuf_tag_init(struct m_tag *, int how);
+int	mac_mount_check_fsctl(vfs_context_t ctx, struct mount *mp,
+	    unsigned int cmd);
+int	mac_mount_check_getattr(vfs_context_t ctx, struct mount *mp,
+	    struct vfs_attr *vfa);
+int	mac_mount_check_label_update(vfs_context_t ctx, struct mount *mp);
+int	mac_mount_check_mount(vfs_context_t ctx, struct vnode *vp,
+	    struct componentname *cnp, const char *vfc_name);
+int	mac_mount_check_remount(vfs_context_t ctx, struct mount *mp);
+int	mac_mount_check_setattr(vfs_context_t ctx, struct mount *mp,
+	    struct vfs_attr *vfa);
+int	mac_mount_check_stat(vfs_context_t ctx, struct mount *mp);
+int	mac_mount_check_umount(vfs_context_t ctx, struct mount *mp);
+void	mac_mount_label_associate(vfs_context_t ctx, struct mount *mp);
+void	mac_mount_label_destroy(struct mount *mp);
+int	mac_mount_label_externalize(struct label *label, char *elements,
+	    char *outbuf, size_t outbuflen);
+int	mac_mount_label_get(struct mount *mp, user_addr_t mac_p);
+void	mac_mount_label_init(struct mount *);
+int	mac_mount_label_internalize(struct label *, char *string);
+void	mac_netinet_fragment(struct mbuf *datagram, struct mbuf *fragment);
+void	mac_netinet_icmp_reply(struct mbuf *m);
+void	mac_netinet_tcp_reply(struct mbuf *m);
+int	mac_pipe_check_ioctl(kauth_cred_t cred, struct pipe *cpipe,
+	    unsigned int cmd);
+int	mac_pipe_check_kqfilter(kauth_cred_t cred, struct knote *kn,
+	    struct pipe *cpipe);
+int	mac_pipe_check_read(kauth_cred_t cred, struct pipe *cpipe);
+int	mac_pipe_check_select(kauth_cred_t cred, struct pipe *cpipe,
+	    int which);
+int	mac_pipe_check_stat(kauth_cred_t cred, struct pipe *cpipe);
+int	mac_pipe_check_write(kauth_cred_t cred, struct pipe *cpipe);
+struct label	*mac_pipe_label_alloc(void);
+void	mac_pipe_label_associate(kauth_cred_t cred, struct pipe *cpipe);
+void	mac_pipe_label_copy(struct label *src, struct label *dest);
+void	mac_pipe_label_destroy(struct pipe *cpipe);
+void	mac_pipe_label_free(struct label *label);
+void	mac_pipe_label_init(struct pipe *cpipe);
+int	mac_pipe_label_update(kauth_cred_t cred, struct pipe *cpipe,
+	    struct label *label);
+void    mac_policy_initbsd(void);
+int	mac_posixsem_check_create(kauth_cred_t cred, const char *name);
+int	mac_posixsem_check_open(kauth_cred_t cred, struct pseminfo *psem);
+int	mac_posixsem_check_post(kauth_cred_t cred, struct pseminfo *psem);
+int	mac_posixsem_check_unlink(kauth_cred_t cred, struct pseminfo *psem,
+	    const char *name);
+int	mac_posixsem_check_wait(kauth_cred_t cred, struct pseminfo *psem);
+void	mac_posixsem_vnode_label_associate(kauth_cred_t cred,
+	    struct pseminfo *psem, struct label *plabel,
+	    vnode_t vp, struct label *vlabel);
+void	mac_posixsem_label_associate(kauth_cred_t cred,
+	    struct pseminfo *psem, const char *name);
+void	mac_posixsem_label_destroy(struct pseminfo *psem);
+void	mac_posixsem_label_init(struct pseminfo *psem);
+int	mac_posixshm_check_create(kauth_cred_t cred, const char *name);
+int	mac_posixshm_check_mmap(kauth_cred_t cred, struct pshminfo *pshm,
+	    int prot, int flags);
+int	mac_posixshm_check_open(kauth_cred_t cred, struct pshminfo *pshm);
+int	mac_posixshm_check_stat(kauth_cred_t cred, struct pshminfo *pshm);
+int	mac_posixshm_check_truncate(kauth_cred_t cred, struct pshminfo *pshm,
+	    size_t s);
+int	mac_posixshm_check_unlink(kauth_cred_t cred, struct pshminfo *pshm,
+	    const char *name);
+void	mac_posixshm_vnode_label_associate(kauth_cred_t cred,
+	    struct pshminfo *pshm, struct label *plabel,
+	    vnode_t vp, struct label *vlabel);
+void	mac_posixshm_label_associate(kauth_cred_t cred,
+	    struct pshminfo *pshm, const char *name);
+void	mac_posixshm_label_destroy(struct pshminfo *pshm);
+void	mac_posixshm_label_init(struct pshminfo *pshm);
+int	mac_proc_check_debug(proc_t proc1, proc_t proc2);
+int	mac_proc_check_fork(proc_t proc);
+int	mac_proc_check_get_task_name(kauth_cred_t cred, struct proc *p);
+int	mac_proc_check_get_task(kauth_cred_t cred, struct proc *p);
+int	mac_proc_check_getaudit(proc_t proc);
+int	mac_proc_check_getauid(proc_t proc);
+int     mac_proc_check_getlcid(proc_t proc1, proc_t proc2,
+	    pid_t pid);
+int	mac_proc_check_mprotect(proc_t proc,
+	    user_addr_t addr, user_size_t size, int prot);
+int	mac_proc_check_sched(proc_t proc, proc_t proc2);
+int	mac_proc_check_setaudit(proc_t proc, struct auditinfo *ai);
+int	mac_proc_check_setauid(proc_t proc, uid_t auid);
+int     mac_proc_check_setlcid(proc_t proc1, proc_t proc2,
+	    pid_t pid1, pid_t pid2);
+int	mac_proc_check_signal(proc_t proc1, proc_t proc2,
+	    int signum);
+int	mac_proc_check_wait(proc_t proc1, proc_t proc2);
+void	mac_proc_set_enforce(proc_t p, int enforce_flags);
+int	mac_setsockopt_label(kauth_cred_t cred, struct socket *so,
+	    struct mac *extmac);
+int     mac_socket_check_accept(kauth_cred_t cred, struct socket *so);
+int     mac_socket_check_accepted(kauth_cred_t cred, struct socket *so);
+int	mac_socket_check_bind(kauth_cred_t cred, struct socket *so,
+	    struct sockaddr *addr);
+int	mac_socket_check_connect(kauth_cred_t cred, struct socket *so,
+	    struct sockaddr *addr);
+int	mac_socket_check_create(kauth_cred_t cred, int domain,
+	    int type, int protocol);
+int	mac_socket_check_deliver(struct socket *so, struct mbuf *m);
+int	mac_socket_check_kqfilter(kauth_cred_t cred, struct knote *kn,
+	    struct socket *so);
+int	mac_socket_check_listen(kauth_cred_t cred, struct socket *so);
+int	mac_socket_check_receive(kauth_cred_t cred, struct socket *so);
+int	mac_socket_check_received(kauth_cred_t cred, struct socket *so, 
+	    struct sockaddr *saddr);
+int     mac_socket_check_select(kauth_cred_t cred, struct socket *so,
+	    int which);
+int	mac_socket_check_send(kauth_cred_t cred, struct socket *so,
+	    struct sockaddr *addr);
+int	mac_socket_check_getsockopt(kauth_cred_t cred, struct socket *so,
+	    struct sockopt *sopt);
+int	mac_socket_check_setsockopt(kauth_cred_t cred, struct socket *so,
+	    struct sockopt *sopt);
+int	mac_socket_check_stat(kauth_cred_t cred, struct socket *so);
+void	mac_socket_label_associate(kauth_cred_t cred, struct socket *so);
+void	mac_socket_label_associate_accept(struct socket *oldsocket,
+	    struct socket *newsocket);
+void	mac_socket_label_copy(struct label *from, struct label *to);
+void	mac_socket_label_destroy(struct socket *);
+int	mac_socket_label_get(kauth_cred_t cred, struct socket *so,
+	    struct mac *extmac);
+int	mac_socket_label_init(struct socket *, int waitok);
+void	mac_socketpeer_label_associate_mbuf(struct mbuf *m, struct socket *so);
+void	mac_socketpeer_label_associate_socket(struct socket *peersocket,
+	    struct socket *socket_to_modify);
+int	mac_socketpeer_label_get(kauth_cred_t cred, struct socket *so,
+	    struct mac *extmac);
+int	mac_system_check_acct(kauth_cred_t cred, struct vnode *vp);
+int	mac_system_check_audit(kauth_cred_t cred, void *record, int length);
+int	mac_system_check_auditctl(kauth_cred_t cred, struct vnode *vp);
+int	mac_system_check_auditon(kauth_cred_t cred, int cmd);
+int	mac_system_check_host_priv(kauth_cred_t cred);
+int	mac_system_check_nfsd(kauth_cred_t cred);
+int	mac_system_check_reboot(kauth_cred_t cred, int howto);
+int	mac_system_check_settime(kauth_cred_t cred);
+int	mac_system_check_swapoff(kauth_cred_t cred, struct vnode *vp);
+int	mac_system_check_swapon(kauth_cred_t cred, struct vnode *vp);
+int	mac_system_check_sysctl(kauth_cred_t cred, int *name,
+	    u_int namelen, user_addr_t oldctl, user_addr_t oldlenp, int inkernel,
+	    user_addr_t newctl, size_t newlen);
+void	mac_sysvmsg_label_associate(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr, struct msg *msgptr);
+void	mac_sysvmsg_label_init(struct msg *msgptr);
+void	mac_sysvmsg_label_recycle(struct msg *msgptr);
+int	mac_sysvmsq_check_enqueue(kauth_cred_t cred, struct msg *msgptr,
+	    struct msqid_kernel *msqptr);
+int	mac_sysvmsq_check_msgrcv(kauth_cred_t cred, struct msg *msgptr);
+int	mac_sysvmsq_check_msgrmid(kauth_cred_t cred, struct msg *msgptr);
+int	mac_sysvmsq_check_msqctl(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr, int cmd);
+int	mac_sysvmsq_check_msqget(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr);
+int	mac_sysvmsq_check_msqrcv(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr);
+int	mac_sysvmsq_check_msqsnd(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr);
+void	mac_sysvmsq_label_associate(kauth_cred_t cred,
+	    struct msqid_kernel *msqptr);
+void 	mac_sysvmsq_label_init(struct msqid_kernel *msqptr);
+void 	mac_sysvmsq_label_recycle(struct msqid_kernel *msqptr);
+int	mac_sysvsem_check_semctl(kauth_cred_t cred,
+	    struct semid_kernel *semakptr, int cmd);
+int	mac_sysvsem_check_semget(kauth_cred_t cred,
+	    struct semid_kernel *semakptr);
+int	mac_sysvsem_check_semop(kauth_cred_t cred,
+	    struct semid_kernel *semakptr, size_t accesstype);
+void	mac_sysvsem_label_associate(kauth_cred_t cred,
+	    struct semid_kernel *semakptr);
+void	mac_sysvsem_label_destroy(struct semid_kernel *semakptr);
+void	mac_sysvsem_label_init(struct semid_kernel *semakptr);
+void	mac_sysvsem_label_recycle(struct semid_kernel *semakptr);
+int	mac_sysvshm_check_shmat(kauth_cred_t cred,
+	    struct shmid_kernel *shmsegptr, int shmflg);
+int	mac_sysvshm_check_shmctl(kauth_cred_t cred,
+	    struct shmid_kernel *shmsegptr, int cmd);
+int	mac_sysvshm_check_shmdt(kauth_cred_t cred,
+	    struct shmid_kernel *shmsegptr);
+int	mac_sysvshm_check_shmget(kauth_cred_t cred,
+	    struct shmid_kernel *shmsegptr, int shmflg);
+void	mac_sysvshm_label_associate(kauth_cred_t cred,
+	    struct shmid_kernel *shmsegptr);
+void	mac_sysvshm_label_destroy(struct shmid_kernel *shmsegptr);
+void	mac_sysvshm_label_init(struct shmid_kernel* shmsegptr);
+void	mac_sysvshm_label_recycle(struct shmid_kernel *shmsegptr);
+void	mac_thread_userret(int code, int error, struct thread *thread);
+int	mac_vnode_check_access(vfs_context_t ctx, struct vnode *vp,
+	    int acc_mode);
+int	mac_vnode_check_chdir(vfs_context_t ctx, struct vnode *dvp);
+int	mac_vnode_check_chroot(vfs_context_t ctx, struct vnode *dvp,
+	    struct componentname *cnp);
+int	mac_vnode_check_create(vfs_context_t ctx, struct vnode *dvp,
+	    struct componentname *cnp, struct vnode_attr *vap);
+int	mac_vnode_check_deleteextattr(vfs_context_t ctx, struct vnode *vp,
+	    const char *name);
+int	mac_vnode_check_exchangedata(vfs_context_t ctx, struct vnode *v1,
+	    struct vnode *v2);
+int	mac_vnode_check_exec(vfs_context_t ctx, struct vnode *vp,
+	    struct image_params *imgp);
+int     mac_vnode_check_getattrlist(vfs_context_t ctx, struct vnode *vp,
+	    struct attrlist *alist);
+int	mac_vnode_check_getextattr(vfs_context_t ctx, struct vnode *vp,
+	    const char *name, struct uio *uio);
+int	mac_vnode_check_ioctl(vfs_context_t ctx, struct vnode *vp,
+	    unsigned int cmd);
+int	mac_vnode_check_kqfilter(vfs_context_t ctx,
+	    kauth_cred_t file_cred, struct knote *kn, struct vnode *vp);
+int	mac_vnode_check_label_update(vfs_context_t ctx, struct vnode *vp,
+	    struct label *newlabel);
+int	mac_vnode_check_link(vfs_context_t ctx, struct vnode *dvp,
+	    struct vnode *vp, struct componentname *cnp);
+int	mac_vnode_check_listextattr(vfs_context_t ctx, struct vnode *vp);
+int	mac_vnode_check_lookup(vfs_context_t ctx, struct vnode *dvp,
+	    struct componentname *cnp);
+int	mac_vnode_check_open(vfs_context_t ctx, struct vnode *vp,
+	    int acc_mode);
+int	mac_vnode_check_read(vfs_context_t ctx,
+	    kauth_cred_t file_cred, struct vnode *vp);
+int	mac_vnode_check_readdir(vfs_context_t ctx, struct vnode *vp);
+int	mac_vnode_check_readlink(vfs_context_t ctx, struct vnode *vp);
+int	mac_vnode_check_rename_from(vfs_context_t ctx, struct vnode *dvp,
+	    struct vnode *vp, struct componentname *cnp);
+int	mac_vnode_check_rename_to(vfs_context_t ctx, struct vnode *dvp,
+	    struct vnode *vp, int samedir, struct componentname *cnp);
+int	mac_vnode_check_revoke(vfs_context_t ctx, struct vnode *vp);
+int     mac_vnode_check_select(vfs_context_t ctx, struct vnode *vp,
+	    int which);
+int     mac_vnode_check_setattrlist(vfs_context_t ctxd, struct vnode *vp,
+	    struct attrlist *alist);
+int	mac_vnode_check_setextattr(vfs_context_t ctx, struct vnode *vp,
+	    const char *name, struct uio *uio);
+int	mac_vnode_check_setflags(vfs_context_t ctx, struct vnode *vp,
+	    u_long flags);
+int	mac_vnode_check_setmode(vfs_context_t ctx, struct vnode *vp,
+	    mode_t mode);
+int	mac_vnode_check_setowner(vfs_context_t ctx, struct vnode *vp,
+	    uid_t uid, gid_t gid);
+int	mac_vnode_check_setutimes(vfs_context_t ctx, struct vnode *vp,
+	    struct timespec atime, struct timespec mtime);
+int	mac_vnode_check_stat(vfs_context_t ctx,
+	    kauth_cred_t file_cred, struct vnode *vp);
+int	mac_vnode_check_truncate(vfs_context_t ctx,
+	    kauth_cred_t file_cred, struct vnode *vp);
+int	mac_vnode_check_unlink(vfs_context_t ctx, struct vnode *dvp,
+	    struct vnode *vp, struct componentname *cnp);
+int	mac_vnode_check_write(vfs_context_t ctx,
+	    kauth_cred_t file_cred, struct vnode *vp);
+struct label	*mac_vnode_label_alloc(void);
+int	mac_vnode_label_associate(struct mount *mp, struct vnode *vp,
+	    vfs_context_t ctx);
+void	mac_vnode_label_associate_devfs(struct mount *mp, struct devnode *de,
+	    struct vnode *vp);
+int	mac_vnode_label_associate_extattr(struct mount *mp, struct vnode *vp);
+int	mac_vnode_label_associate_fdesc(struct mount *mp, struct fdescnode *fnp,
+	    struct vnode *vp, vfs_context_t ctx);
+void	mac_vnode_label_associate_singlelabel(struct mount *mp,
+	    struct vnode *vp);
+void	mac_vnode_label_copy(struct label *l1, struct label *l2);
+void	mac_vnode_label_destroy(struct vnode *vp);
+int	mac_vnode_label_externalize_audit(struct vnode *vp, struct mac *mac);
+void	mac_vnode_label_free(struct label *label);
+void	mac_vnode_label_init(struct vnode *vp);
+void	mac_vnode_label_recycle(struct vnode *vp);
+void	mac_vnode_label_update(vfs_context_t ctx, struct vnode *vp,
+	    struct label *newlabel);
+void	mac_vnode_label_update_extattr(struct mount *mp, struct vnode *vp,
+	    const char *name);
+int	mac_vnode_notify_create(vfs_context_t ctx, struct mount *mp,
+	    struct vnode *dvp, struct vnode *vp, struct componentname *cnp);
+int	vnode_label(struct mount *mp, struct vnode *dvp, struct vnode *vp,
+	    struct componentname *cnp, int flags, vfs_context_t ctx);
+void	vnode_relabel(struct vnode *vp);
+
+void psem_label_associate(struct fileproc *fp, struct vnode *vp, struct vfs_context *ctx);
+void pshm_label_associate(struct fileproc *fp, struct vnode *vp, struct vfs_context *ctx);
+
+#if CONFIG_MACF_NET
+struct label *mac_bpfdesc_label_get(struct bpf_d *d);
+void mac_bpfdesc_label_set(struct bpf_d *d, struct label *label);
+#endif
+
+#endif	/* CONFIG_MACF */
+
+#endif /* !_SECURITY_MAC_FRAMEWORK_H_ */
diff --git a/security/mac_inet.c b/security/mac_inet.c
new file mode 100644
index 000000000..2b823bab9
--- /dev/null
+++ b/security/mac_inet.c
@@ -0,0 +1,308 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001-2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2006-2007 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/route.h>
+
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet/ip_var.h>
+
+#include <security/mac_internal.h>
+
+static struct label *
+mac_inpcb_label_alloc(int flag)
+{
+	struct label *label;
+	int error;
+
+	label = mac_labelzone_alloc(flag);
+	if (label == NULL)
+		return (NULL);
+	MAC_CHECK(inpcb_label_init, label, flag);
+	if (error) {
+		MAC_PERFORM(inpcb_label_destroy, label);
+		mac_labelzone_free(label);
+		return (NULL);
+	}
+	return (label);
+}
+
+int
+mac_inpcb_label_init(struct inpcb *inp, int flag)
+{
+
+	inp->inp_label = mac_inpcb_label_alloc(flag);
+	if (inp->inp_label == NULL)
+		return (ENOMEM);
+	return (0);
+}
+
+static struct label *
+mac_ipq_label_alloc(int flag)
+{
+	struct label *label;
+	int error;
+
+	label = mac_labelzone_alloc(flag);
+	if (label == NULL)
+		return (NULL);
+
+	MAC_CHECK(ipq_label_init, label, flag);
+	if (error) {
+		MAC_PERFORM(ipq_label_destroy, label);
+		mac_labelzone_free(label);
+		return (NULL);
+	}
+	return (label);
+}
+
+int
+mac_ipq_label_init(struct ipq *ipq, int flag)
+{
+
+	ipq->ipq_label = mac_ipq_label_alloc(flag);
+	if (ipq->ipq_label == NULL)
+		return (ENOMEM);
+	return (0);
+}
+
+static void
+mac_inpcb_label_free(struct label *label)
+{
+
+	MAC_PERFORM(inpcb_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_inpcb_label_destroy(struct inpcb *inp)
+{
+
+	mac_inpcb_label_free(inp->inp_label);
+	inp->inp_label = NULL;
+}
+
+void
+mac_inpcb_label_recycle(struct inpcb *inp)
+{
+
+	MAC_PERFORM(inpcb_label_recycle, inp->inp_label);
+}
+
+static void
+mac_ipq_label_free(struct label *label)
+{
+
+	MAC_PERFORM(ipq_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_ipq_label_destroy(struct ipq *ipq)
+{
+
+	mac_ipq_label_free(ipq->ipq_label);
+	ipq->ipq_label = NULL;
+}
+
+void
+mac_inpcb_label_associate(struct socket *so, struct inpcb *inp)
+{
+
+	MAC_PERFORM(inpcb_label_associate, so, so->so_label, inp,
+	    inp->inp_label);
+}
+
+void
+mac_mbuf_label_associate_ipq(struct ipq *ipq, struct mbuf *m)
+{
+	struct label *label;
+
+	label = mac_mbuf_to_label(m);
+
+	MAC_PERFORM(mbuf_label_associate_ipq, ipq, ipq->ipq_label, m, label);
+}
+
+void
+mac_netinet_fragment(struct mbuf *datagram, struct mbuf *fragment)
+{
+	struct label *datagramlabel, *fragmentlabel;
+
+	datagramlabel = mac_mbuf_to_label(datagram);
+	fragmentlabel = mac_mbuf_to_label(fragment);
+
+	MAC_PERFORM(netinet_fragment, datagram, datagramlabel, fragment,
+	    fragmentlabel);
+}
+
+void
+mac_ipq_label_associate(struct mbuf *fragment, struct ipq *ipq)
+{
+	struct label *label;
+
+	label = mac_mbuf_to_label(fragment);
+
+	MAC_PERFORM(ipq_label_associate, fragment, label, ipq, ipq->ipq_label);
+}
+
+void
+mac_mbuf_label_associate_inpcb(struct inpcb *inp, struct mbuf *m)
+{
+	struct label *mlabel;
+
+	/* INP_LOCK_ASSERT(inp); */
+	mlabel = mac_mbuf_to_label(m);
+
+	MAC_PERFORM(mbuf_label_associate_inpcb, inp, inp->inp_label, m, mlabel);
+}
+
+int
+mac_ipq_label_compare(struct mbuf *fragment, struct ipq *ipq)
+{
+	struct label *label;
+	int result;
+
+	label = mac_mbuf_to_label(fragment);
+
+	result = 1;
+	MAC_BOOLEAN(ipq_label_compare, &&, fragment, label, ipq, ipq->ipq_label);
+
+	return (result);
+}
+
+void
+mac_netinet_icmp_reply(struct mbuf *m)
+{
+	struct label *label;
+
+	label = mac_mbuf_to_label(m);
+
+	MAC_PERFORM(netinet_icmp_reply, m, label);
+}
+
+void
+mac_netinet_tcp_reply(struct mbuf *m)
+{
+	struct label *label;
+
+	label = mac_mbuf_to_label(m);
+
+	MAC_PERFORM(netinet_tcp_reply, m, label);
+}
+
+void
+mac_ipq_label_update(struct mbuf *fragment, struct ipq *ipq)
+{
+	struct label *label;
+
+	label = mac_mbuf_to_label(fragment);
+
+	MAC_PERFORM(ipq_label_update, fragment, label, ipq, ipq->ipq_label);
+}
+
+int
+mac_inpcb_check_deliver(struct inpcb *inp, struct mbuf *m, int family, int type)
+{
+	struct label *label;
+	int error;
+
+	if ((m->m_flags & M_PKTHDR) == 0)
+		panic("%s: no mbuf packet header!", __func__);
+
+	label = mac_mbuf_to_label(m);
+
+	MAC_CHECK(inpcb_check_deliver, inp, inp->inp_label, m, label,
+	    family, type);
+
+	return (error);
+}
+
+/*
+ * Propagate a change in the socket label to the inpcb label.
+ */
+void
+mac_inpcb_label_update(struct socket *so)
+{
+	struct inpcb *inp;
+
+	/* XXX: assert socket lock. */
+	inp = sotoinpcb(so);	/* XXX: inp locking */
+
+	if (inp != NULL) {
+		/* INP_LOCK_ASSERT(inp); */
+		MAC_PERFORM(inpcb_label_update, so, so->so_label, inp,
+		    inp->inp_label);
+	}
+}
diff --git a/security/mac_internal.h b/security/mac_internal.h
new file mode 100644
index 000000000..e41c429af
--- /dev/null
+++ b/security/mac_internal.h
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef _SECURITY_MAC_INTERNAL_H_
+#define _SECURITY_MAC_INTERNAL_H_
+
+#include <string.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <security/mac.h>
+#include <security/mac_framework.h>
+#include <security/mac_policy.h>
+#include <security/mac_data.h>
+#include <sys/sysctl.h>
+#include <kern/wait_queue.h>
+#include <kern/locks.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/vnode.h>
+
+/*
+ * MAC Framework sysctl namespace.
+ */
+
+SYSCTL_DECL(_security);
+SYSCTL_DECL(_security_mac);
+
+extern int mac_late;
+
+struct mac_policy_list_element {
+        struct mac_policy_conf *mpc;
+};    
+
+struct mac_policy_list {
+	u_int				numloaded;
+	u_int 				max;
+	u_int				maxindex;
+	u_int				staticmax;
+	u_int				chunks;
+	u_int				freehint;
+	struct mac_policy_list_element	*entries;
+};
+
+typedef struct mac_policy_list mac_policy_list_t;
+
+
+/*
+ * Darwin functions not properly exported
+ */
+extern void kmod_load_early(void);	/* defined in libsa/kext.cpp */
+
+/*
+ * Policy that has registered with the framework for a specific
+ * label namespace name.
+ */
+struct mac_label_listener {
+	mac_policy_handle_t		mll_handle;
+	LIST_ENTRY(mac_label_listener)	mll_list;
+};
+
+LIST_HEAD(mac_label_listeners_t, mac_label_listener);
+
+/* 
+ * Type of list used to manage label namespace names.
+ */   
+struct mac_label_element {
+	char				mle_name[MAC_MAX_LABEL_ELEMENT_NAME];
+	struct mac_label_listeners_t	mle_listeners;
+	LIST_ENTRY(mac_label_element)	mle_list;
+};
+
+LIST_HEAD(mac_label_element_list_t, mac_label_element);
+
+/*
+ * Journal operations
+ */
+
+#define	MLJ_TYPE_PORT		1
+#define	MLJ_TYPE_TASK		2
+
+#define	MLJ_PORT_OP_INIT	0x0001
+#define	MLJ_PORT_OP_CREATE_K	0x0002
+#define	MLJ_PORT_OP_CREATE	0x0004
+#define	MLJ_PORT_OP_UPDATE	0x0008
+
+#define	MLJ_TASK_OP_INIT	0x0001
+#define	MLJ_TASK_OP_CREATE_K	0x0002
+
+struct mac_label_journal {
+	struct label *l;
+	int type;
+	int ops;
+
+	int kotype;	/* Kernel Port */
+
+	TAILQ_ENTRY(mac_label_journal) link;
+};
+TAILQ_HEAD(mac_label_journal_list_t, mac_label_journal);
+
+int	mac_label_journal_add	(struct label *, int);
+int	mac_label_journal_remove(struct label *);
+struct mac_label_journal *
+	mac_label_journal_find	(struct label *);
+int	mac_label_journal	(struct label *, int, ...);
+void	mac_label_journal_replay(void);
+
+
+/*
+ * MAC Framework global variables.
+ */
+
+extern struct mac_label_element_list_t mac_label_element_list;
+extern struct mac_label_element_list_t mac_static_label_element_list;
+extern struct mac_label_journal_list_t mac_label_journal_list;
+
+extern struct mac_policy_list mac_policy_list;
+
+/*
+ * global flags to control whether a MACF subsystem is configured
+ * at all in the system.
+ */
+extern unsigned int mac_device_enforce;
+extern unsigned int mac_file_enforce;
+extern unsigned int mac_iokit_enforce;
+extern unsigned int mac_pipe_enforce;
+extern unsigned int mac_posixsem_enforce;
+extern unsigned int mac_posixshm_enforce;
+extern unsigned int mac_proc_enforce;
+extern unsigned int mac_socket_enforce;
+extern unsigned int mac_system_enforce;
+extern unsigned int mac_sysvmsg_enforce;
+extern unsigned int mac_sysvsem_enforce;
+extern unsigned int mac_sysvshm_enforce;
+extern unsigned int mac_vm_enforce;
+extern unsigned int mac_vnode_enforce;
+
+#if CONFIG_MACF_MACH
+extern unsigned int mac_port_enforce;
+extern unsigned int mac_task_enforce;
+#endif
+
+#if CONFIG_MACF_NET
+extern unsigned int mac_label_mbufs;
+#endif
+
+static int mac_proc_check_enforce(proc_t p, int enforce_flag);
+
+static __inline__ int mac_proc_check_enforce(proc_t p, int enforce_flags)
+{
+#if CONFIG_MACF
+	return ((p->p_mac_enforce & enforce_flags) != 0);
+#else
+	return 0;
+#endif
+}
+
+static int mac_context_check_enforce(vfs_context_t ctx, int enforce_flags);
+static void mac_context_set_enforce(vfs_context_t ctx, int enforce_flags);
+
+static __inline__ int mac_context_check_enforce(vfs_context_t ctx, int enforce_flags)
+{
+	proc_t proc = vfs_context_proc(ctx);
+
+	if (proc == NULL)
+		return 0;
+
+	return (mac_proc_check_enforce(proc, enforce_flags));
+}
+
+static __inline__ void mac_context_set_enforce(vfs_context_t ctx, int enforce_flags)
+{
+	proc_t proc = vfs_context_proc(ctx);
+
+	if (proc == NULL)
+		return;
+
+	mac_proc_set_enforce(proc, enforce_flags);
+}
+
+
+/*
+ * MAC Framework infrastructure functions.
+ */
+
+int mac_error_select(int error1, int error2);
+
+void  mac_policy_list_busy(void);
+int   mac_policy_list_conditional_busy(void);
+void  mac_policy_list_unbusy(void);
+
+void           mac_labelzone_init(void);
+struct label  *mac_labelzone_alloc(int flags);
+void           mac_labelzone_free(struct label *label);
+
+void  mac_label_init(struct label *label);
+void  mac_label_destroy(struct label *label);
+#if KERNEL
+int   mac_check_structmac_consistent(struct user_mac *mac);
+#else
+int   mac_check_structmac_consistent(struct mac *mac);
+#endif
+	
+int mac_cred_label_externalize(struct label *, char *e, char *out, size_t olen, int flags);
+int mac_lctx_label_externalize(struct label *, char *e, char *out, size_t olen);
+#if CONFIG_MACF_SOCKET
+int mac_socket_label_externalize(struct label *, char *e, char *out, size_t olen);
+#endif /* CONFIG_MACF_SOCKET */
+int mac_vnode_label_externalize(struct label *, char *e, char *out, size_t olen, int flags);
+int mac_pipe_label_externalize(struct label *label, char *elements,
+ char *outbuf, size_t outbuflen);
+
+int mac_cred_label_internalize(struct label *label, char *string);
+int mac_lctx_label_internalize(struct label *label, char *string);
+#if CONFIG_MACF_SOCKET
+int mac_socket_label_internalize(struct label *label, char *string);
+#endif /* CONFIG_MACF_SOCKET */
+int mac_vnode_label_internalize(struct label *label, char *string);
+int mac_pipe_label_internalize(struct label *label, char *string);
+
+#if CONFIG_MACF_SOCKET
+/* internal socket label manipulation functions */
+struct  label *mac_socket_label_alloc(int flags);
+void    mac_socket_label_free(struct label *l);
+int     mac_socket_label_update(struct ucred *cred, struct socket *so, struct label *l);
+#endif /* MAC_SOCKET */
+
+#if CONFIG_MACF_NET
+struct label *mac_mbuf_to_label(struct mbuf *m);
+#else
+#define mac_mbuf_to_label(m) (NULL)
+#endif
+
+/*
+ * MAC_CHECK performs the designated check by walking the policy
+ * module list and checking with each as to how it feels about the
+ * request.  Note that it returns its value via 'error' in the scope
+ * of the caller.
+ */
+#define	MAC_CHECK(check, args...) do {					\
+	struct mac_policy_conf *mpc;					\
+	u_int i;                                               		\
+									\
+	error = 0;							\
+	for (i = 0; i < mac_policy_list.staticmax; i++) {		\
+		mpc = mac_policy_list.entries[i].mpc;              	\
+		if (mpc == NULL)                                	\
+			continue;                               	\
+									\
+		if (mpc->mpc_ops->mpo_ ## check != NULL)		\
+			error = mac_error_select(      			\
+			    mpc->mpc_ops->mpo_ ## check (args),		\
+			    error);					\
+	}								\
+	if (mac_policy_list_conditional_busy() != 0) {			\
+		for (; i <= mac_policy_list.maxindex; i++) {		\
+			mpc = mac_policy_list.entries[i].mpc;		\
+			if (mpc == NULL)                                \
+				continue;                               \
+                                                                        \
+			if (mpc->mpc_ops->mpo_ ## check != NULL)	\
+				error = mac_error_select(      		\
+				    mpc->mpc_ops->mpo_ ## check (args),	\
+				    error);				\
+		}							\
+		mac_policy_list_unbusy();				\
+	}								\
+} while (0)
+
+/*
+ * MAC_BOOLEAN performs the designated boolean composition by walking
+ * the module list, invoking each instance of the operation, and
+ * combining the results using the passed C operator.  Note that it
+ * returns its value via 'result' in the scope of the caller, which
+ * should be initialized by the caller in a meaningful way to get
+ * a meaningful result.
+ */
+#define	MAC_BOOLEAN(operation, composition, args...) do {		\
+	struct mac_policy_conf *mpc;					\
+	u_int i;							\
+									\
+	for (i = 0; i < mac_policy_list.staticmax; i++) {		\
+		mpc = mac_policy_list.entries[i].mpc;			\
+		if (mpc == NULL)                                	\
+			continue;                               	\
+									\
+		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
+			result = result composition			\
+			    mpc->mpc_ops->mpo_ ## operation		\
+			    (args);					\
+	}								\
+	if (mac_policy_list_conditional_busy() != 0) {			\
+		for (; i <= mac_policy_list.maxindex; i++) {		\
+			mpc = mac_policy_list.entries[i].mpc;		\
+			if (mpc == NULL)                                \
+				continue;                               \
+                                                                        \
+			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
+				result = result composition		\
+				    mpc->mpc_ops->mpo_ ## operation	\
+				    (args);				\
+		}							\
+		mac_policy_list_unbusy();				\
+	}								\
+} while (0)
+
+#define	MAC_INTERNALIZE(obj, label, instring)				\
+	mac_internalize(offsetof(struct mac_policy_ops, mpo_ ## obj ## _label_internalize), label, instring)
+
+#define MAC_EXTERNALIZE(obj, label, elementlist, outbuf, outbuflen)	\
+	mac_externalize(offsetof(struct mac_policy_ops, mpo_ ## obj ## _label_externalize), label, elementlist, outbuf, outbuflen)
+
+#define MAC_EXTERNALIZE_AUDIT(obj, label, outbuf, outbuflen)	\
+	mac_externalize(offsetof(struct mac_policy_ops, mpo_ ## obj ## _label_externalize_audit), label, "*", outbuf, outbuflen)
+
+/*
+ * MAC_PERFORM performs the designated operation by walking the policy
+ * module list and invoking that operation for each policy.
+ */
+#define	MAC_PERFORM(operation, args...) do {				\
+	struct mac_policy_conf *mpc;					\
+	u_int i;							\
+									\
+	for (i = 0; i < mac_policy_list.staticmax; i++) {		\
+		mpc = mac_policy_list.entries[i].mpc;			\
+		if (mpc == NULL)					\
+			continue;					\
+									\
+		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
+			mpc->mpc_ops->mpo_ ## operation (args);		\
+	}								\
+	if (mac_policy_list_conditional_busy() != 0) {			\
+		for (; i <= mac_policy_list.maxindex; i++) {		\
+			mpc = mac_policy_list.entries[i].mpc;		\
+			if (mpc == NULL)				\
+				continue;				\
+									\
+			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
+				mpc->mpc_ops->mpo_ ## operation (args);	\
+		}							\
+		mac_policy_list_unbusy();				\
+	}								\
+} while (0)
+
+struct __mac_get_pid_args;
+struct __mac_get_proc_args;
+struct __mac_set_proc_args;
+struct __mac_get_lcid_args;
+struct __mac_get_lctx_args;
+struct __mac_set_lctx_args;
+struct __mac_get_fd_args;
+struct __mac_get_file_args;
+struct __mac_get_link_args;
+struct __mac_set_fd_args;
+struct __mac_set_file_args;
+struct __mac_syscall_args;
+
+void mac_policy_addto_labellist(const mac_policy_handle_t, int);
+void mac_policy_removefrom_labellist(const mac_policy_handle_t);
+
+int mac_externalize(size_t mpo_externalize_off, struct label *label,
+    const char *elementlist, char *outbuf, size_t outbuflen);
+int mac_internalize(size_t mpo_internalize_off, struct label *label,
+    char *elementlist);
+#endif	/* !_SECURITY_MAC_INTERNAL_H_ */
diff --git a/security/mac_iokit.c b/security/mac_iokit.c
new file mode 100644
index 000000000..fd78646ec
--- /dev/null
+++ b/security/mac_iokit.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>  
+#include <sys/vnode.h>  
+#include <sys/vnode_internal.h>
+#include <sys/queue.h>  
+#include <bsd/bsm/audit.h>
+#include <bsd/bsm/audit_kernel.h>
+#include <bsd/sys/malloc.h>
+#include <vm/vm_kern.h>
+#include <kern/kalloc.h>
+
+#include <security/mac_framework.h>
+#include <security/mac_internal.h>
+
+int
+mac_iokit_check_device(char *devtype, struct mac_module_data *mdata)
+{
+	int error;
+
+	MAC_CHECK(iokit_check_device, devtype, mdata);
+	return (error);
+}
diff --git a/security/mac_label.c b/security/mac_label.c
new file mode 100644
index 000000000..ef2c09e74
--- /dev/null
+++ b/security/mac_label.c
@@ -0,0 +1,81 @@
+/*-
+ * Copyright (c) 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <kern/zalloc.h>
+#include <security/_label.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/systm.h>
+#include <security/mac_internal.h>
+
+static zone_t zone_label;
+
+void
+mac_labelzone_init(void)
+{
+
+	zone_label = zinit(sizeof(struct label),
+	    8192 * sizeof(struct label),
+	    sizeof(struct label), "MAC Labels");
+	zone_change(zone_label, Z_EXPAND, TRUE);
+	zone_change(zone_label, Z_EXHAUST, FALSE);
+}
+
+struct label *
+mac_labelzone_alloc(int flags)
+{
+	struct label *l;
+
+	if (flags & MAC_NOWAIT) 
+		l = (struct label *) zalloc_noblock(zone_label);
+	else
+		l = (struct label *) zalloc(zone_label);
+	if (l == NULL)
+		return (NULL);
+
+	bzero(l, sizeof(struct label));
+	l->l_flags = MAC_FLAG_INITIALIZED;
+	return (l);
+}
+
+void
+mac_labelzone_free(struct label *l)
+{
+
+	if (l == NULL)
+		panic("Free of NULL MAC label\n");
+
+	if ((l->l_flags & MAC_FLAG_INITIALIZED) == 0)
+		panic("Free of uninitialized label\n");
+	bzero(l, sizeof(struct label));
+	zfree(zone_label, l);
+}
diff --git a/security/mac_mach_internal.h b/security/mac_mach_internal.h
new file mode 100644
index 000000000..799f0fac0
--- /dev/null
+++ b/security/mac_mach_internal.h
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SECURITY_MAC_MACH_INTERNAL_H_
+#define _SECURITY_MAC_MACH_INTERNAL_H_
+
+int mac_task_check_service(task_t self, task_t obj, const char *perm);
+void mac_task_label_update_internal(struct label *pl, struct task *t);
+int mac_port_label_compute(struct label *subj, struct label *obj,
+    const char *serv, struct label *out);
+int mac_port_check_method(task_t task, struct label *sub, struct label *obj, int msgid);
+
+#if CONFIG_MACF
+void mac_policy_init(void);
+void mac_policy_initmach(void);
+
+/* tasks */
+void mac_task_label_init(struct label *);
+void mac_task_label_copy(struct label *src, struct label *dest);
+void mac_task_label_destroy(struct label *);
+void mac_task_label_associate(struct task *, struct task *, struct label *,
+    struct label *, struct label *);
+void mac_task_label_associate_kernel(struct task *, struct label *, struct label *);
+void mac_task_label_modify( struct task *pt, void *arg,
+    void (*f)(struct label *l, void *arg));
+struct label *mac_task_get_label(struct task *task);
+
+/* ports */
+void mac_port_label_init(struct label *l);
+void mac_port_label_destroy(struct label *l);
+void mac_port_label_associate(struct label *it, struct label *st, struct label *plabel);
+void mac_port_label_associate_kernel(struct label *plabel, int isreply);
+void mac_port_label_update_kobject(struct label *plabel, int kotype);
+void mac_port_label_copy(struct label *src, struct label *dest);
+void mac_port_label_update_cred(struct label *src, struct label *dest);
+int mac_port_check_label_update(struct label *task, struct label *oldl, struct label *newl);
+
+int mac_port_check_send(struct label *task, struct label *port);
+int mac_port_check_receive(struct label *task, struct label *sender);
+int mac_port_check_make_send(struct label *task, struct label *port);
+int mac_port_check_make_send_once(struct label *task, struct label *port);
+int mac_port_check_move_receive(struct label *task, struct label *port);
+int mac_port_check_copy_send(struct label *task, struct label *port);
+int mac_port_check_move_send(struct label *task, struct label *port);
+int mac_port_check_move_send_once(struct label *task, struct label *port);
+
+int mac_port_check_hold_send(struct label *task, struct label *port);
+int mac_port_check_hold_send_once(struct label *task, struct label *port);
+int mac_port_check_hold_receive(struct label *task, struct label *port);
+
+int mac_task_label_externalize(struct label *, char *e, char *out, size_t olen, int flags);
+int mac_task_label_internalize(struct label *label, char *string);
+int mac_port_label_externalize(struct label *, char *e, char *out, size_t olen, int flags);
+int mac_port_label_internalize(struct label *label, char *string);
+
+void	mac_task_label_update(struct label *cred, struct label *task);
+int	mac_port_check_service(struct label *subj, struct label *obj,
+	    const char *serv, const char *perm);
+#endif /* MAC */
+
+#endif	/* !_SECURITY_MAC_MACH_INTERNAL_H_ */
diff --git a/security/mac_net.c b/security/mac_net.c
new file mode 100644
index 000000000..cb847acb0
--- /dev/null
+++ b/security/mac_net.c
@@ -0,0 +1,532 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001-2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/mbuf.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+
+#include <bsd/bsm/audit.h>
+#include <bsd/bsm/audit_kernel.h>
+
+#include <security/mac_internal.h>
+
+struct label *
+mac_mbuf_to_label(struct mbuf *mbuf)
+{
+	struct m_tag *tag;
+	struct label *label;
+
+	if (mbuf == NULL)
+		return (NULL);
+
+	if ((mbuf->m_flags & M_PKTHDR) == 0) {
+		printf("%s() got non-header MBUF!\n", __func__);
+		return (NULL);
+	}
+
+	tag = m_tag_locate(mbuf, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_MACLABEL,
+			   NULL);
+	if (tag == NULL) {
+		printf("%s() m_tag_locate() returned NULL! (m->flags %04x)\n",
+			__func__, mbuf->m_flags);
+		return (NULL);
+	}
+	label = (struct label *)(tag+1);
+	return (label);
+}
+
+static struct label *
+mac_bpfdesc_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(M_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(bpfdesc_label_init, label);
+	return (label);
+}
+
+void
+mac_bpfdesc_label_init(struct bpf_d *bpf_d)
+{
+	struct label *label;
+
+	label = mac_bpfdesc_label_alloc();
+	mac_bpfdesc_label_set(bpf_d, label);
+}
+
+static struct label *
+mac_ifnet_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(M_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(ifnet_label_init, label);
+	return (label);
+}
+
+void
+mac_ifnet_label_init(struct ifnet *ifp)
+{
+
+	ifp->if_label = mac_ifnet_label_alloc();
+}
+
+/*
+ * On failure, caller should cleanup with m_tag_free().
+ */
+int
+mac_mbuf_tag_init(struct m_tag *tag, int flag)
+{
+	struct label *label;
+	int error;
+
+	label = (struct label *) (tag + 1);
+	mac_label_init(label);
+
+	MAC_CHECK(mbuf_label_init, label, flag);
+	if (error)
+		printf("%s(): mpo_mbuf_label_init() failed!\n", __func__);
+
+	return (error);
+}
+
+static void
+mac_bpfdesc_label_free(struct label *label)
+{
+
+	MAC_PERFORM(bpfdesc_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_bpfdesc_label_destroy(struct bpf_d *bpf_d)
+{
+	struct label *label;
+
+	label = mac_bpfdesc_label_get(bpf_d);
+	mac_bpfdesc_label_free(label);
+	mac_bpfdesc_label_set(bpf_d, NULL);
+}
+
+static void
+mac_ifnet_label_free(struct label *label)
+{
+
+	MAC_PERFORM(ifnet_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_ifnet_label_destroy(struct ifnet *ifp)
+{
+
+	mac_ifnet_label_free(ifp->if_label);
+	ifp->if_label = NULL;
+}
+
+void
+mac_ifnet_label_recycle(struct ifnet *ifp)
+{
+
+	MAC_PERFORM(ifnet_label_recycle, ifp->if_label);
+}
+
+void
+mac_mbuf_tag_destroy(struct m_tag *tag)
+{
+	struct label *label;
+
+	label = (struct label *)(tag + 1);
+	MAC_PERFORM(mbuf_label_destroy, label);
+	mac_label_destroy(label);
+
+	return;
+}
+
+void
+mac_mbuf_tag_copy(struct m_tag *src, struct m_tag *dest)
+{
+	struct label *src_label, *dest_label;
+
+	src_label = (struct label *)(src + 1);
+	dest_label = (struct label *)(dest + 1);
+
+	if (src_label == NULL || dest_label == NULL)
+		return;
+
+	/*
+	 * mac_mbuf_tag_init() is called on the target tag
+	 * in m_tag_copy(), so we don't need to call it here.
+	 */
+	MAC_PERFORM(mbuf_label_copy, src_label, dest_label);
+
+	return;
+}
+
+void
+mac_mbuf_label_copy(struct mbuf *m_from, struct mbuf *m_to)
+{
+	struct label *src_label, *dest_label;
+
+	src_label = mac_mbuf_to_label(m_from);
+	dest_label = mac_mbuf_to_label(m_to);
+
+	MAC_PERFORM(mbuf_label_copy, src_label, dest_label);
+}
+
+static void
+mac_ifnet_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(ifnet_label_copy, src, dest);
+}
+
+static int
+mac_ifnet_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+
+	return (MAC_EXTERNALIZE(ifnet, label, elements, outbuf, outbuflen));
+}
+
+static int
+mac_ifnet_label_internalize(struct label *label, char *string)
+{
+
+	return (MAC_INTERNALIZE(ifnet, label, string));
+}
+
+void
+mac_ifnet_label_associate(struct ifnet *ifp)
+{
+
+	MAC_PERFORM(ifnet_label_associate, ifp, ifp->if_label);
+}
+
+void
+mac_bpfdesc_label_associate(struct ucred *cred, struct bpf_d *bpf_d)
+{
+	struct label *label;
+
+	label = mac_bpfdesc_label_get(bpf_d);
+	MAC_PERFORM(bpfdesc_label_associate, cred, bpf_d, label);
+}
+
+int
+mac_bpfdesc_check_receive(struct bpf_d *bpf_d, struct ifnet *ifp)
+{
+	struct label *label;
+	int error;
+
+	label = mac_bpfdesc_label_get(bpf_d);
+	ifnet_lock_shared(ifp);
+	MAC_CHECK(bpfdesc_check_receive, bpf_d, label, ifp,
+	    ifp->if_label);
+	ifnet_lock_done(ifp);
+
+	return (error);
+}
+
+int
+mac_mbuf_label_init(struct mbuf *m, int flag)
+{
+	struct m_tag *tag;
+	int error;
+
+	if (mac_label_mbufs == 0)
+		return (0);
+
+	tag = m_tag_alloc(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_MACLABEL,
+			  sizeof(struct label), flag);
+	if (tag == NULL) {
+		printf("%s(): m_tag_alloc() failed!\n", __func__);
+		return (ENOBUFS);
+	}
+	error = mac_mbuf_tag_init(tag, flag);
+	if (error) {
+		printf("%s(): mac_mbuf_tag_init() failed!\n", __func__);
+		m_tag_free(tag);
+		return (error);
+	}
+	m_tag_prepend(m, tag);
+	return (0);
+}
+
+void
+mac_mbuf_label_associate_bpfdesc(struct bpf_d *bpf_d, struct mbuf *mbuf)
+{
+	struct label *m_label, *b_label;
+
+	/* bpf_d must be locked */
+
+	m_label = mac_mbuf_to_label(mbuf);
+	b_label = mac_bpfdesc_label_get(bpf_d);
+
+	MAC_PERFORM(mbuf_label_associate_bpfdesc, bpf_d, b_label, mbuf,
+	    m_label);
+}
+
+void
+mac_mbuf_label_associate_ifnet(struct ifnet *ifp, struct mbuf *mbuf)
+{
+	struct label *m_label;
+
+	/* ifp must be locked */
+
+	m_label = mac_mbuf_to_label(mbuf);
+
+	MAC_PERFORM(mbuf_label_associate_ifnet, ifp, ifp->if_label, mbuf,
+	    m_label);
+}
+
+void
+mac_mbuf_label_associate_linklayer(struct ifnet *ifp, struct mbuf *mbuf)
+{
+	struct label *m_label;
+
+	/* ifp must be locked */
+
+	m_label = mac_mbuf_to_label(mbuf);
+
+	MAC_PERFORM(mbuf_label_associate_linklayer, ifp, ifp->if_label, mbuf,
+	    m_label);
+}
+
+void
+mac_mbuf_label_associate_multicast_encap(struct mbuf *oldmbuf,
+    struct ifnet *ifp, struct mbuf *newmbuf)
+{
+	struct label *oldmbuflabel, *newmbuflabel;
+
+	oldmbuflabel = mac_mbuf_to_label(oldmbuf);
+	newmbuflabel = mac_mbuf_to_label(newmbuf);
+
+	/* ifp must be locked */
+
+	MAC_PERFORM(mbuf_label_associate_multicast_encap, oldmbuf, oldmbuflabel,
+	    ifp, ifp->if_label, newmbuf, newmbuflabel);
+}
+
+void
+mac_mbuf_label_associate_netlayer(struct mbuf *oldmbuf, struct mbuf *newmbuf)
+{
+	struct label *oldmbuflabel, *newmbuflabel;
+
+	oldmbuflabel = mac_mbuf_to_label(oldmbuf);
+	newmbuflabel = mac_mbuf_to_label(newmbuf);
+
+	MAC_PERFORM(mbuf_label_associate_netlayer, oldmbuf, oldmbuflabel,
+	    newmbuf, newmbuflabel);
+}
+
+void
+mac_mbuf_label_associate_socket(struct socket *socket, struct mbuf *mbuf)
+{
+	struct label *label;
+	struct xsocket xso;
+
+	/* socket must be locked */
+
+	label = mac_mbuf_to_label(mbuf);
+
+	sotoxsocket(socket, &xso);
+	MAC_PERFORM(mbuf_label_associate_socket, &xso, socket->so_label,
+		    mbuf, label);
+}
+
+int
+mac_ifnet_check_transmit(struct ifnet *ifp, struct mbuf *mbuf, int family,
+    int type)
+{
+	struct label *label;
+	int error;
+
+	label = mac_mbuf_to_label(mbuf);
+
+	ifnet_lock_shared(ifp);
+	MAC_CHECK(ifnet_check_transmit, ifp, ifp->if_label, mbuf, label,
+	    family, type);
+	ifnet_lock_done(ifp);
+
+	return (error);
+}
+
+int
+mac_ifnet_label_get(__unused struct ucred *cred, struct ifreq *ifr,
+    struct ifnet *ifp)
+{
+	char *elements, *buffer;
+	struct label *intlabel;
+	struct mac mac;
+	int error;
+	size_t len;
+
+	error = copyin(CAST_USER_ADDR_T(ifr->ifr_ifru.ifru_data),
+	    &mac, sizeof(mac));
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac.m_string), elements,
+	    mac.m_buflen, &len);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, elements);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	intlabel = mac_ifnet_label_alloc();
+	ifnet_lock_shared(ifp);
+	mac_ifnet_label_copy(ifp->if_label, intlabel);
+	ifnet_lock_done(ifp);
+	error = mac_ifnet_label_externalize(intlabel, elements,
+	    buffer, mac.m_buflen);
+	mac_ifnet_label_free(intlabel);
+	FREE(elements, M_MACTEMP);
+
+	if (error == 0)
+		error = copyout(buffer, CAST_USER_ADDR_T(mac.m_string),
+		    strlen(buffer) + 1);
+	FREE(buffer, M_MACTEMP);
+
+	return (error);
+}
+
+int
+mac_ifnet_label_set(struct ucred *cred, struct ifreq *ifr,
+    struct ifnet *ifp)
+{
+	struct label *intlabel;
+	struct mac mac;
+	char *buffer;
+	int error;
+	size_t len;
+
+	error = copyin(CAST_USER_ADDR_T(ifr->ifr_ifru.ifru_data),
+	    &mac, sizeof(mac));
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac.m_string), buffer,
+	    mac.m_buflen, &len);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+	AUDIT_ARG(mac_string, buffer);
+
+	intlabel = mac_ifnet_label_alloc();
+	error = mac_ifnet_label_internalize(intlabel, buffer);
+	FREE(buffer, M_MACTEMP);
+	if (error) {
+		mac_ifnet_label_free(intlabel);
+		return (error);
+	}
+
+	/*
+	 * XXX: Note that this is a redundant privilege check, since
+	 * policies impose this check themselves if required by the
+	 * policy.  Eventually, this should go away.
+	 */
+	error = suser(cred, NULL);
+	if (error) {
+		mac_ifnet_label_free(intlabel);
+		return (error);
+	}
+
+	ifnet_lock_exclusive(ifp);
+	MAC_CHECK(ifnet_check_label_update, cred, ifp, ifp->if_label,
+	    intlabel);
+	if (error) {
+		ifnet_lock_done(ifp);
+		mac_ifnet_label_free(intlabel);
+		return (error);
+	}
+
+	MAC_PERFORM(ifnet_label_update, cred, ifp, ifp->if_label, intlabel);
+	ifnet_lock_done(ifp);
+	mac_ifnet_label_free(intlabel);
+
+	return (0);
+}
diff --git a/security/mac_pipe.c b/security/mac_pipe.c
new file mode 100644
index 000000000..fce1303ac
--- /dev/null
+++ b/security/mac_pipe.c
@@ -0,0 +1,249 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2002, 2003 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/vnode.h>
+#include <sys/pipe.h>
+#include <sys/sysctl.h>
+
+#include <security/mac_internal.h>
+
+
+struct label *
+mac_pipe_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(pipe_label_init, label);
+	return (label);
+}
+
+void
+mac_pipe_label_init(struct pipe *cpipe)
+{
+
+	cpipe->pipe_label = mac_pipe_label_alloc();
+}
+
+void
+mac_pipe_label_free(struct label *label)
+{
+
+	MAC_PERFORM(pipe_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_pipe_label_destroy(struct pipe *cpipe)
+{
+
+	mac_pipe_label_free(cpipe->pipe_label);
+	cpipe->pipe_label = NULL;
+}
+
+void
+mac_pipe_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(pipe_label_copy, src, dest);
+}
+
+int
+mac_pipe_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(pipe, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_pipe_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(pipe, label, string);
+
+	return (error);
+}
+
+void
+mac_pipe_label_associate(kauth_cred_t cred, struct pipe *cpipe)
+{
+
+	MAC_PERFORM(pipe_label_associate, cred, cpipe, cpipe->pipe_label);
+}
+
+int
+mac_pipe_check_kqfilter(kauth_cred_t cred, struct knote *kn,
+    struct pipe *cpipe)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_kqfilter, cred, kn, cpipe, cpipe->pipe_label);
+	return (error);
+}
+int
+mac_pipe_check_ioctl(kauth_cred_t cred, struct pipe *cpipe, u_int cmd)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_ioctl, cred, cpipe, cpipe->pipe_label, cmd);
+
+	return (error);
+}
+
+int
+mac_pipe_check_read(kauth_cred_t cred, struct pipe *cpipe)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_read, cred, cpipe, cpipe->pipe_label);
+
+	return (error);
+}
+
+static int
+mac_pipe_check_label_update(kauth_cred_t cred, struct pipe *cpipe,
+    struct label *newlabel)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_label_update, cred, cpipe, cpipe->pipe_label, newlabel);
+
+	return (error);
+}
+
+int
+mac_pipe_check_select(kauth_cred_t cred, struct pipe *cpipe, int which)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_select, cred, cpipe, cpipe->pipe_label, which);
+
+	return (error);
+}
+
+int
+mac_pipe_check_stat(kauth_cred_t cred, struct pipe *cpipe)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_stat, cred, cpipe, cpipe->pipe_label);
+
+	return (error);
+}
+
+int
+mac_pipe_check_write(kauth_cred_t cred, struct pipe *cpipe)
+{
+	int error;
+
+	if (!mac_pipe_enforce)
+		return (0);
+
+	MAC_CHECK(pipe_check_write, cred, cpipe, cpipe->pipe_label);
+
+	return (error);
+}
+
+int
+mac_pipe_label_update(kauth_cred_t cred, struct pipe *cpipe,
+    struct label *label)
+{
+	int error;
+
+	error = mac_pipe_check_label_update(cred, cpipe, label);
+	if (error)
+		return (error);
+
+	MAC_PERFORM(pipe_label_update, cred, cpipe, cpipe->pipe_label, label);
+
+	return (0);
+}
diff --git a/security/mac_policy.h b/security/mac_policy.h
new file mode 100644
index 000000000..a12cacb30
--- /dev/null
+++ b/security/mac_policy.h
@@ -0,0 +1,6254 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001-2005 Networks Associates Technology, Inc.
+ * Copyright (c) 2005-2007 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson for the TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD: src/sys/sys/mac_policy.h,v 1.39 2003/04/18 19:57:37 rwatson Exp $
+ */
+
+/**
+  @file mac_policy.h
+  @brief Kernel Interfaces for MAC policy modules
+
+  This header defines the list of operations that are defined by the
+  TrustedBSD MAC Framwork on Darwin.  MAC Policy modules register
+  with the framework to declare interest in a specific set of
+  operations.  If interest in an entry point is not declared, then
+  the policy will be ignored when the Framework evaluates that entry
+  point.
+*/
+
+#ifndef _SECURITY_MAC_POLICY_H_
+#define _SECURITY_MAC_POLICY_H_
+
+#include <security/_label.h>
+
+struct attrlist;
+struct auditinfo;
+struct bpf_d;
+struct devnode;
+struct fileglob;
+struct ifnet;
+struct inpcb;
+struct ipq;
+struct label;
+struct lctx;
+struct mac_module_data;
+struct mac_policy_conf;
+struct mbuf;
+struct mount;
+struct pipe;
+struct pseminfo;
+struct pshminfo;
+struct sbuf;
+struct semid_kernel;
+struct shmid_kernel;
+struct task;
+struct thread;
+struct ucred;
+struct vnode;
+/** @struct dummy */
+
+
+
+#ifndef _KAUTH_CRED_T
+#define	_KAUTH_CRED_T
+typedef struct ucred *kauth_cred_t;
+#endif	/* !_KAUTH_CRED_T */
+
+
+/*-
+ * MAC entry points are generally named using the following template:
+ *
+ *   mpo_<object>_<operation>()
+ *
+ * or:
+ *
+ *   mpo_<object>_check_<operation>()
+ *
+ * Entry points are sorted by object type.
+ *
+ * It may be desirable also to consider some subsystems as "objects", such
+ * as system, iokit, etc.
+ */
+
+/**
+  @name Entry Points for Label Management
+
+  These are the entry points corresponding to the life cycle events for
+  kernel objects, such as initialization, creation, and destruction.
+
+  Most policies (that use labels) will initialize labels by allocating
+  space for policy-specific data.  In most cases, it is permitted to
+  sleep during label initialization operations; it will be noted when
+  it is not permitted.
+
+  Initialization usually will not require doing more than allocating a
+  generic label for the given object.  What follows initialization is
+  creation, where a label is made specific to the object it is associated
+  with.  Destruction occurs when the label is no longer needed, such as
+  when the corresponding object is destroyed.  All necessary cleanup should
+  be performed in label destroy operations.
+
+  Where possible, the label entry points have identical parameters.  If
+  the policy module does not require structure-specific label
+  information, the same function may be registered in the policy
+  operation vector.  Many policies will implement two such generic
+  allocation calls: one to handle sleepable requests, and one to handle
+  potentially non-sleepable requests.
+*/
+
+
+/**
+  @brief Audit event postselection
+  @param cred Subject credential
+  @param syscode Syscall number
+  @param args Syscall arguments
+  @param error Syscall errno
+  @param retval Syscall return value
+
+  This is the MAC Framework audit postselect, which is called before
+  exiting a syscall to determine if an audit event should be committed.
+  A return value of MAC_AUDIT_NO forces the audit record to be suppressed.
+  Any other return value results in the audit record being committed.
+
+  @warning The suppression behavior will probably go away in Apple's
+  future version of the audit implementation.
+
+  @return Return MAC_AUDIT_NO to force suppression of the audit record.
+  Any other value results in the audit record being committed.
+
+*/
+typedef int mpo_audit_check_postselect_t(
+	kauth_cred_t cred,
+	unsigned short syscode,
+	void *args,
+	int error,
+	int retval
+);
+/**
+  @brief Audit event preselection
+  @param cred Subject credential
+  @param syscode Syscall number
+  @param args Syscall arguments
+
+  This is the MAC Framework audit preselect, which is called before a
+  syscall is entered to determine if an audit event should be created.
+  If the MAC policy forces the syscall to be audited, MAC_AUDIT_YES should be
+  returned. A return value of MAC_AUDIT_NO causes the audit record to
+  be suppressed. Returning MAC_POLICY_DEFAULT indicates that the policy wants
+  to defer to the system's existing preselection mechanism.
+
+  When policies return different preferences, the Framework decides what action
+  to take based on the following policy.  If any policy returns MAC_AUDIT_YES,
+  then create an audit record, else if any policy returns MAC_AUDIT_NO, then
+  suppress the creations of an audit record, else defer to the system's
+  existing preselection mechanism.
+
+  @warning The audit implementation in Apple's current version is
+  incomplete, so the MAC policies have priority over the system's existing
+  mechanisms. This will probably change in the future version where
+  the audit implementation is more complete.
+
+  @return Return MAC_AUDIT_YES to force auditing of the syscall,
+  MAC_AUDIT_NO to force no auditing of the syscall, MAC_AUDIT_DEFAULT
+  to allow auditing mechanisms to determine if the syscall is audited.
+
+*/
+typedef int mpo_audit_check_preselect_t(
+	kauth_cred_t cred,
+	unsigned short syscode,
+	void *args
+);
+/**
+  @brief Initialize BPF descriptor label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated BPF descriptor.
+  Sleeping is permitted.
+*/
+typedef void mpo_bpfdesc_label_init_t(
+	struct label *label
+);
+/**
+  @brief Destroy BPF descriptor label
+  @param label The label to be destroyed
+
+  Destroy a BPF descriptor label.  Since the BPF descriptor
+  is going out of scope, policy modules should free any internal
+  storage associated with the label so that it may be destroyed.
+*/
+typedef void mpo_bpfdesc_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Associate a BPF descriptor with a label
+  @param cred User credential creating the BPF descriptor
+  @param bpf_d The BPF descriptor
+  @param bpflabel The new label
+
+  Set the label on a newly created BPF descriptor from the passed
+  subject credential. This call will be made when a BPF device node
+  is opened by a process with the passed subject credential.
+*/
+typedef void mpo_bpfdesc_label_associate_t(
+	kauth_cred_t cred,
+	struct bpf_d *bpf_d,
+	struct label *bpflabel
+);
+/**
+  @brief Check whether BPF can read from a network interface
+  @param bpf_d Subject; the BPF descriptor
+  @param bpflabel Policy label for bpf_d 
+  @param ifp Object; the network interface 
+  @param ifnetlabel Policy label for ifp
+
+  Determine whether the MAC framework should permit datagrams from
+  the passed network interface to be delivered to the buffers of
+  the passed BPF descriptor.  Return (0) for success, or an errno
+  value for failure.  Suggested failure: EACCES for label mismatches,
+  EPERM for lack of privilege.
+*/
+typedef int mpo_bpfdesc_check_receive_t(
+	struct bpf_d *bpf_d,
+	struct label *bpflabel,
+	struct ifnet *ifp,
+	struct label *ifnetlabel
+);
+/**
+  @brief Indicate desire to change the process label at exec time
+  @param old Existing subject credential
+  @param vp File being executed
+  @param vnodelabel Label corresponding to vp
+  @param scriptvnodelabel Script vnode label
+  @param execlabel Userspace provided execution label
+  @param proc Object process
+  @see mac_execve
+  @see mpo_cred_label_update_execve_t
+  @see mpo_vnode_check_exec_t
+
+  Indicate whether this policy intends to update the label of a newly
+  created credential from the existing subject credential (old).  This
+  call occurs when a process executes the passed vnode.  If a policy
+  returns success from this entry point, the mpo_cred_label_update_execve
+  entry point will later be called with the same parameters.  Access
+  has already been checked via the mpo_vnode_check_exec entry point,
+  this entry point is necessary to preserve kernel locking constraints
+  during program execution.
+
+  The supplied vnode and vnodelabel correspond with the file actually
+  being executed; in the case that the file is interpreted (for
+  example, a script), the label of the original exec-time vnode has
+  been preserved in scriptvnodelabel.
+
+  The final label, execlabel, corresponds to a label supplied by a
+  user space application through the use of the mac_execve system call.
+
+  The vnode lock is held during this operation.  No changes should be
+  made to the old credential structure.
+
+  @warning Even if a policy returns 0, it should behave correctly in
+  the presence of an invocation of mpo_cred_label_update_execve, as that
+  call may happen as a result of another policy requesting a transition.
+
+  @return Non-zero if a transition is required, 0 otherwise.
+*/
+typedef int mpo_cred_check_label_update_execve_t(
+	kauth_cred_t old,
+	struct vnode *vp,
+	struct label *vnodelabel,
+	struct label *scriptvnodelabel,
+	struct label *execlabel,
+	struct proc *proc
+);
+/**
+  @brief Access control check for relabelling processes
+  @param cred Subject credential
+  @param newlabel New label to apply to the user credential
+  @see mpo_cred_label_update_t
+  @see mac_set_proc
+
+  Determine whether the subject identified by the credential can relabel
+  itself to the supplied new label (newlabel).  This access control check
+  is called when the mac_set_proc system call is invoked.  A user space
+  application will supply a new value, the value will be internalized
+  and provided in newlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_cred_check_label_update_t(
+	kauth_cred_t cred,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for visibility of other subjects
+  @param u1 Subject credential
+  @param u2 Object credential
+
+  Determine whether the subject identified by the credential u1 can
+  "see" other subjects with the passed subject credential u2. This call
+  may be made in a number of situations, including inter-process status
+  sysctls used by ps, and in procfs lookups.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to hide visibility.
+*/
+typedef int mpo_cred_check_visible_t(
+	kauth_cred_t u1,
+	kauth_cred_t u2
+);
+/**
+  @brief Associate a credential with a new process at fork
+  @param cred credential to inherited by new process
+  @param proc the new process
+
+  Allow a process to associate the credential with a new
+  process for reference countng purposes.
+  NOTE: the credential can be dis-associated in ways other
+        than exit - so this strategy is flawed - should just
+	catch label destroy callback.
+*/
+typedef void mpo_cred_label_associate_fork_t(
+	kauth_cred_t cred,
+	proc_t proc
+);
+/**
+  @brief Create the first process
+  @param cred Subject credential to be labeled
+
+  Create the subject credential of process 0, the parent of all BSD
+  kernel processes.  Policies should update the label in the
+  previously initialized credential structure.
+*/
+typedef void mpo_cred_label_associate_kernel_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Create a credential label
+  @param parent_cred Parent credential
+  @param child_cred Child credential
+
+  Set the label of a newly created credential, most likely using the
+  information in the supplied parent credential.
+
+  @warning This call is made when crcopy or crdup is invoked on a
+  newly created struct ucred, and should not be confused with a
+  process fork or creation event.
+*/
+typedef void mpo_cred_label_associate_t(
+	kauth_cred_t parent_cred,
+	kauth_cred_t child_cred
+);
+/**
+  @brief Create the first process
+  @param cred Subject credential to be labeled
+
+  Create the subject credential of process 1, the parent of all BSD
+  user processes.  Policies should update the label in the previously
+  initialized credential structure.  This is the 'init' process.
+*/
+typedef void mpo_cred_label_associate_user_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Destroy credential label
+  @param label The label to be destroyed
+
+  Destroy a user credential label.  Since the user credential
+  is going out of scope, policy modules should free any internal
+  storage associated with the label so that it may be destroyed.
+*/
+typedef void mpo_cred_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a user credential label for auditing
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a user credential for
+  inclusion in an audit record.  An externalized label consists of a text
+  representation of the label contents that will be added to the audit record
+  as part of a text token.  Policy-agnostic user space tools will display
+  this externalized version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_cred_label_externalize_audit_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Externalize a user credential label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a user
+  credential.  An externalized label consists of a text representation
+  of the label contents that can be used with user applications.
+  Policy-agnostic user space tools will display this externalized
+  version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_cred_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize user credential label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated user credential.
+  Sleeping is permitted.
+*/
+typedef void mpo_cred_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a user credential label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a user credential label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_cred_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Update credential at exec time
+  @param old_cred Existing subject credential
+  @param new_cred New subject credential to be labeled
+  @param vp File being executed
+  @param vnodelabel Label corresponding to vp
+  @param scriptvnodelabel Script vnode label
+  @param execlabel Userspace provided execution label
+  @see mac_execve
+  @see mpo_cred_check_label_update_execve_t
+  @see mpo_vnode_check_exec_t
+
+  Update the label of a newly created credential (new) from the
+  existing subject credential (old).  This call occurs when a process
+  executes the passed vnode and one of the loaded policy modules has
+  returned success from the mpo_cred_check_label_update_execve entry point.
+  Access has already been checked via the mpo_vnode_check_exec entry
+  point, this entry point is only used to update any policy state.
+
+  The supplied vnode and vnodelabel correspond with the file actually
+  being executed; in the case that the file is interpreted (for
+  example, a script), the label of the original exec-time vnode has
+  been preserved in scriptvnodelabel.
+
+  The final label, execlabel, corresponds to a label supplied by a
+  user space application through the use of the mac_execve system call.
+
+  The vnode lock is held during this operation.  No changes should be
+  made to the old credential structure.
+*/
+typedef void mpo_cred_label_update_execve_t(
+	kauth_cred_t old_cred,
+	kauth_cred_t new_cred,
+	struct vnode *vp,
+	struct label *vnodelabel,
+	struct label *scriptvnodelabel,
+	struct label *execlabel
+);
+/**
+  @brief Update a credential label
+  @param cred The existing credential
+  @param newlabel A new label to apply to the credential
+  @see mpo_cred_check_label_update_t
+  @see mac_set_proc
+
+  Update the label on a user credential, using the supplied new label.
+  This is called as a result of a process relabel operation.  Access
+  control was already confirmed by mpo_cred_check_label_update.
+*/
+typedef void mpo_cred_label_update_t(
+	kauth_cred_t cred,
+	struct label *newlabel
+);
+/**
+  @brief Create a new devfs device
+  @param dev Major and minor numbers of special file
+  @param de "inode" of new device file
+  @param label Destination label
+  @param fullpath Path relative to mount (e.g. /dev) of new device file
+
+  This entry point labels a new devfs device. The label will likely be based
+  on the path to the device, or the major and minor numbers.
+  The policy should store an appropriate label into 'label'.
+*/
+typedef void mpo_devfs_label_associate_device_t(
+	dev_t dev,
+	struct devnode *de,
+	struct label *label,
+	const char *fullpath
+);
+/**
+  @brief Create a new devfs directory
+  @param dirname Name of new directory
+  @param dirnamelen Length of 'dirname'
+  @param de "inode" of new directory
+  @param label Destination label
+  @param fullpath Path relative to mount (e.g. /dev) of new directory
+
+  This entry point labels a new devfs directory. The label will likely be
+  based on the path of the new directory. The policy should store an appropriate
+  label into 'label'. The devfs root directory is labelled in this way.
+*/
+typedef void mpo_devfs_label_associate_directory_t(
+	const char *dirname,
+	int dirnamelen,
+	struct devnode *de,
+	struct label *label,
+	const char *fullpath
+);
+/**
+  @brief Copy a devfs label
+  @param src Source devfs label
+  @param dest Destination devfs label
+
+  Copy the label information from src to dest.  The devfs file system
+  often duplicates (splits) existing device nodes rather than creating
+  new ones.
+*/
+typedef void mpo_devfs_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy devfs label
+  @param label The label to be destroyed
+
+  Destroy a devfs entry label.  Since the object is going out
+  of scope, policy modules should free any internal storage associated
+  with the label so that it may be destroyed.
+*/
+typedef void mpo_devfs_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize devfs label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated devfs entry.  Sleeping
+  is permitted.
+*/
+typedef void mpo_devfs_label_init_t(
+	struct label *label
+);
+/**
+  @brief Update a devfs label after relabelling its vnode
+  @param mp Devfs mount point
+  @param de Affected devfs directory entry
+  @param delabel Label of devfs directory entry
+  @param vp Vnode associated with de
+  @param vnodelabel New label of vnode
+
+  Update a devfs label when its vnode is manually relabelled,
+  for example with setfmac(1). Typically, this will simply copy
+  the vnode label into the devfs label.
+*/
+typedef void mpo_devfs_label_update_t(
+	struct mount *mp,
+	struct devnode *de,
+	struct label *delabel,
+	struct vnode *vp,
+	struct label *vnodelabel
+);
+/**
+  @brief Access control for changing the offset of a file descriptor
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+
+  Determine whether the subject identified by the credential can
+  change the offset of the file represented by fg.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_change_offset_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label
+);
+/**
+  @brief Access control for creating a file descriptor
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential can
+  allocate a new file descriptor.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_create_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control for duplicating a file descriptor
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+  @param newfd New file descriptor number
+
+  Determine whether the subject identified by the credential can
+  duplicate the fileglob structure represented by fg and as file
+  descriptor number newfd.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_dup_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	int newfd
+);
+/**
+  @brief Access control check for fcntl
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+  @param cmd Control operation to be performed; see fcntl(2)
+  @param arg fcnt arguments; see fcntl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the file control operation indicated by cmd.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_file_check_fcntl_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	int cmd,
+	user_long_t arg
+);
+/**
+  @brief Access control check for mac_get_fd
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param elements Element buffer
+  @param len Length of buffer
+
+  Determine whether the subject identified by the credential should be allowed
+  to get an externalized version of the label on the object indicated by fd.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_file_check_get_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	char *elements,
+	int len
+);
+/**
+  @brief Access control for getting the offset of a file descriptor
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+
+  Determine whether the subject identified by the credential can
+  get the offset of the file represented by fg.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_get_offset_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label
+);
+/**
+  @brief Access control for inheriting a file descriptor
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+
+  Determine whether the subject identified by the credential can
+  inherit the fileglob structure represented by fg.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_inherit_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label
+);
+/**
+  @brief Access control check for file ioctl
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+  @param cmd The ioctl command; see ioctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the ioctl operation indicated by cmd.
+
+  @warning Since ioctl data is opaque from the standpoint of the MAC
+  framework, policies must exercise extreme care when implementing
+  access control checks.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_file_check_ioctl_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	unsigned int cmd
+);
+/**
+  @brief Access control check for file locking
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+  @param op The lock operation (F_GETLK, F_SETLK, F_UNLK)
+  @param fl The flock structure
+
+  Determine whether the subject identified by the credential can perform
+  the lock operation indicated by op and fl on the file represented by fg.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_file_check_lock_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	int op,
+	struct flock *fl
+);
+/**
+  @brief Access control check for mapping a file
+  @param cred Subject credential
+  @param fg fileglob representing file to map
+  @param label Policy label associated with vp
+  @param prot mmap protections; see mmap(2)
+  @param flags Type of mapped object; see mmap(2)
+  @param maxprot Maximum rights
+
+  Determine whether the subject identified by the credential should be
+  allowed to map the file represented by fg with the protections specified
+  in prot.  The maxprot field holds the maximum permissions on the new
+  mapping, a combination of VM_PROT_READ, VM_PROT_WRITE, and VM_PROT_EXECUTE.
+  To avoid overriding prior access control checks, a policy should only
+  remove flags from maxprot.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_file_check_mmap_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	int prot,
+	int flags,
+	int *maxprot
+);
+/**
+  @brief Downgrade the mmap protections
+  @param cred Subject credential
+  @param fg file to map
+  @param label Policy label associated with vp
+  @param prot mmap protections to be downgraded
+
+  Downgrade the mmap protections based on the subject and object labels.
+*/
+typedef void mpo_file_check_mmap_downgrade_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label,
+	int *prot
+);
+/**
+  @brief Access control for receiving a file descriptor
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+
+  Determine whether the subject identified by the credential can
+  receive the fileglob structure represented by fg.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_file_check_receive_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label
+);
+/**
+  @brief Access control check for mac_set_fd
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param elements Elements buffer
+  @param len Length of elements buffer
+
+  Determine whether the subject identified by the credential can
+  perform the mac_set_fd operation.  The mac_set_fd operation is used
+  to associate a MAC label with a file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_file_check_set_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	char *elements,
+	int len
+);
+/**
+  @brief Create file label
+  @param cred Subject credential
+  @param fg Fileglob structure
+  @param label Policy label for fg
+*/
+typedef void mpo_file_label_associate_t(
+	kauth_cred_t cred,
+	struct fileglob *fg,
+	struct label *label
+);
+/**
+ @brief Destroy file label
+ @param label The label to be destroyed
+
+ Destroy the label on a file descriptor.  In this entry point, a
+ policy module should free any internal storage associated with
+ label so that it may be destroyed.
+*/
+typedef void mpo_file_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize file label
+  @param label New label to initialize
+*/
+typedef void mpo_file_label_init_t(
+	struct label *label
+);
+/**
+  @brief Access control check for relabeling network interfaces
+  @param cred Subject credential
+  @param ifp network interface being relabeled
+  @param ifnetlabel Current label of the network interfaces
+  @param newlabel New label to apply to the network interfaces
+  @see mpo_ifnet_label_update_t
+
+  Determine whether the subject identified by the credential can
+  relabel the network interface represented by ifp to the supplied
+  new label (newlabel).
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_ifnet_check_label_update_t(
+	kauth_cred_t cred,
+	struct ifnet *ifp,
+	struct label *ifnetlabel,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for relabeling network interfaces
+  @param ifp Network interface mbuf will be transmitted through
+  @param ifnetlabel Label of the network interfaces
+  @param m The mbuf to be transmitted
+  @param mbuflabel Label of the mbuf to be transmitted
+  @param family Address Family, AF_*
+  @param type Type of socket, SOCK_{STREAM,DGRAM,RAW}
+
+  Determine whether the mbuf with label mbuflabel may be transmitted
+  through the network interface represented by ifp that has the
+  label ifnetlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_ifnet_check_transmit_t(
+	struct ifnet *ifp,
+	struct label *ifnetlabel,
+	struct mbuf *m,
+	struct label *mbuflabel,
+	int family,
+	int type
+);
+/**
+  @brief Create a network interface label
+  @param ifp Network interface labeled
+  @param ifnetlabel Label for the network interface
+
+  Set the label of a newly created network interface, most likely
+  using the information in the supplied network interface struct.
+*/
+typedef void mpo_ifnet_label_associate_t(
+	struct ifnet *ifp,
+	struct label *ifnetlabel
+);
+/**
+  @brief Copy an ifnet label
+  @param src Source ifnet label
+  @param dest Destination ifnet label
+
+  Copy the label information from src to dest.
+*/
+typedef void mpo_ifnet_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+ @brief Destroy ifnet label
+ @param label The label to be destroyed
+
+ Destroy the label on an ifnet label.  In this entry point, a
+ policy module should free any internal storage associated with
+ label so that it may be destroyed.
+*/
+typedef void mpo_ifnet_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize an ifnet label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on an interface.
+  An externalized label consists of a text representation of the
+  label contents that can be used with user applications.
+  Policy-agnostic user space tools will display this externalized
+  version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_ifnet_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize ifnet label
+  @param label New label to initialize
+*/
+typedef void mpo_ifnet_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize an interface label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce an interface label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_ifnet_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Recycle up a network interface label
+  @param label The label to be recycled
+
+  Recycle a network interface label.  Darwin caches the struct ifnet
+  of detached ifnets in a "free pool".  Before ifnets are returned
+  to the "free pool", policies can cleanup or overwrite any information
+  present in the label.
+*/
+typedef void mpo_ifnet_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Update a network interface label
+  @param cred Subject credential
+  @param ifp The network interface to be relabeled
+  @param ifnetlabel The current label of the network interface
+  @param newlabel A new label to apply to the network interface
+  @see mpo_ifnet_check_label_update_t
+
+  Update the label on a network interface, using the supplied new label.
+*/
+typedef void mpo_ifnet_label_update_t(
+	kauth_cred_t cred,
+	struct ifnet *ifp,
+	struct label *ifnetlabel,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for delivering a packet to a socket
+  @param inp inpcb the socket is associated with
+  @param inplabel Label of the inpcb
+  @param m The mbuf being received
+  @param mbuflabel Label of the mbuf being received
+  @param family Address family, AF_*
+  @param type Type of socket, SOCK_{STREAM,DGRAM,RAW}
+
+  Determine whether the mbuf with label mbuflabel may be received
+  by the socket associated with inpcb that has the label inplabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_inpcb_check_deliver_t(
+	struct inpcb *inp,
+	struct label *inplabel,
+	struct mbuf *m,
+	struct label *mbuflabel,
+	int family,
+	int type
+);
+/**
+  @brief Create an inpcb label
+  @param so Socket containing the inpcb to be labeled
+  @param solabel Label of the socket
+  @param inp inpcb to be labeled
+  @param inplabel Label for the inpcb
+
+  Set the label of a newly created inpcb, most likely
+  using the information in the socket and/or socket label.
+*/
+typedef void mpo_inpcb_label_associate_t(
+	struct socket *so,
+	struct label *solabel,
+	struct inpcb *inp,
+	struct label *inplabel
+);
+/**
+ @brief Destroy inpcb label
+ @param label The label to be destroyed
+
+ Destroy the label on an inpcb label.  In this entry point, a
+ policy module should free any internal storage associated with
+ label so that it may be destroyed.
+*/
+typedef void mpo_inpcb_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize inpcb label
+  @param label New label to initialize
+  @param flag M_WAITOK or M_NOWAIT
+*/
+typedef int mpo_inpcb_label_init_t(
+	struct label *label,
+	int flag
+);
+/**
+  @brief Recycle up an inpcb label
+  @param label The label to be recycled
+
+  Recycle an inpcb label.  Darwin allocates the inpcb as part of
+  the socket structure in some cases.  For this case we must recycle
+  rather than destroy the inpcb as it will be reused later.
+*/
+typedef void mpo_inpcb_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Update an inpcb label from a socket label
+  @param so Socket containing the inpcb to be relabeled
+  @param solabel New label of the socket
+  @param inp inpcb to be labeled
+  @param inplabel Label for the inpcb
+
+  Set the label of a newly created inpcb due to a change in the
+  underlying socket label.
+*/
+typedef void mpo_inpcb_label_update_t(
+	struct socket *so,
+	struct label *solabel,
+	struct inpcb *inp,
+	struct label *inplabel
+);
+/**
+  @brief Device hardware access control
+  @param devtype Type of device connected
+  @param properties XML-formatted property list
+  @param proplen Length of the property list
+
+  This is the MAC Framework device access control, which is called by the I/O
+  Kit when a new device is connected to the system to determine whether that
+  device should be trusted.  A list of properties associated with the device
+  is passed as an XML-formatted string.  The routine should examine these
+  properties to determine the trustworthiness of the device.  A return value
+  of EPERM forces the device to be claimed by a special device driver that
+  will prevent its operation.
+
+  @warning This is an experimental interface and may change in the future.
+
+  @return Return EPERM to indicate that the device is untrusted and should
+  not be allowed to operate.  Return zero to indicate that the device is
+  trusted and should be allowed to operate normally.
+
+*/
+typedef int mpo_iokit_check_device_t(
+	char *devtype,
+	struct mac_module_data *mdata
+);
+/**
+  @brief Create an IP reassembly queue label
+  @param fragment First received IP fragment
+  @param fragmentlabel Policy label for fragment
+  @param ipq IP reassembly queue to be labeled
+  @param ipqlabel Policy label to be filled in for ipq
+
+  Set the label on a newly created IP reassembly queue from
+  the mbuf header of the first received fragment.
+*/
+typedef void mpo_ipq_label_associate_t(
+	struct mbuf *fragment,
+	struct label *fragmentlabel,
+	struct ipq *ipq,
+	struct label *ipqlabel
+);
+/**
+  @brief Compare an mbuf header label to an ipq label
+  @param fragment IP datagram fragment
+  @param fragmentlabel Policy label for fragment
+  @param ipq IP fragment reassembly queue
+  @param ipqlabel Policy label for ipq
+
+  Compare the label of the mbuf header containing an IP datagram
+  (fragment) fragment with the label of the passed IP fragment
+  reassembly queue (ipq). Return (1) for a successful match, or (0)
+  for no match. This call is made when the IP stack attempts to
+  find an existing fragment reassembly queue for a newly received
+  fragment; if this fails, a new fragment reassembly queue may be
+  instantiated for the fragment. Policies may use this entry point
+  to prevent the reassembly of otherwise matching IP fragments if
+  policy does not permit them to be reassembled based on the label
+  or other information.
+*/
+typedef int mpo_ipq_label_compare_t(
+	struct mbuf *fragment,
+	struct label *fragmentlabel,
+	struct ipq *ipq,
+	struct label *ipqlabel
+);
+/**
+ @brief Destroy IP reassembly queue label
+ @param label The label to be destroyed
+
+ Destroy the label on an IP fragment queue.  In this entry point, a
+ policy module should free any internal storage associated with
+ label so that it may be destroyed.
+*/
+typedef void mpo_ipq_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize IP reassembly queue label
+  @param label New label to initialize
+  @param flag M_WAITOK or M_NOWAIT
+
+  Initialize the label on a newly instantiated IP fragment reassembly
+  queue.  The flag field may be one of M_WAITOK and M_NOWAIT, and
+  should be employed to avoid performing a sleeping malloc(9) during
+  this initialization call. IP fragment reassembly queue allocation
+  frequently occurs in performance sensitive environments, and the
+  implementation should be careful to avoid sleeping or long-lived
+  operations. This entry point is permitted to fail resulting in
+  the failure to allocate the IP fragment reassembly queue.
+*/
+typedef int mpo_ipq_label_init_t(
+	struct label *label,
+	int flag
+);
+/**
+  @brief Update the label on an IP fragment reassembly queue
+  @param fragment IP fragment
+  @param fragmentlabel Policy label for fragment
+  @param ipq IP fragment reassembly queue
+  @param ipqlabel Policy label to be updated for ipq
+
+  Update the label on an IP fragment reassembly queue (ipq) based
+  on the acceptance of the passed IP fragment mbuf header (fragment).
+*/
+typedef void mpo_ipq_label_update_t(
+	struct mbuf *fragment,
+	struct label *fragmentlabel,
+	struct ipq *ipq,
+	struct label *ipqlabel
+);
+/**
+  @brief Access control check for relabelling Login Context
+  @param l Subject credential
+  @param newlabel New label to apply to the Login Context
+  @see mpo_lctx_label_update_t
+  @see mac_set_lcid
+  @see mac_set_lctx
+
+  Determine whether the subject identified by the credential can relabel
+  itself to the supplied new label (newlabel).  This access control check
+  is called when the mac_set_lctx/lcid system call is invoked.  A user space
+  application will supply a new value, the value will be internalized
+  and provided in newlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_lctx_check_label_update_t(
+	struct lctx *l,
+	struct label *newlabel
+);
+/**
+ @brief Destroy Login Context label
+ @param label The label to be destroyed
+*/
+typedef void mpo_lctx_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a Login Context label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a Login Context.
+  An externalized label consists of a text representation
+  of the label contents that can be used with user applications.
+  Policy-agnostic user space tools will display this externalized
+  version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_lctx_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize Login Context label
+  @param label New label to initialize
+*/
+typedef void mpo_lctx_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a Login Context label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a Login Context label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_lctx_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Update a Login Context label
+  @param l
+  @param newlabel A new label to apply to the Login Context
+  @see mpo_lctx_check_label_update_t
+  @see mac_set_lcid
+  @see mac_set_lctx
+
+  Update the label on a login context, using the supplied new label.
+  This is called as a result of a login context relabel operation.  Access
+  control was already confirmed by mpo_lctx_check_label_update.
+*/
+typedef void mpo_lctx_label_update_t(
+	struct lctx *l,
+	struct label *newlabel
+);
+/**
+  @brief A process has created a login context
+  @param p Subject
+  @param l Login Context
+
+  When a process creates a login context (via setlcid()) this entrypoint
+  is called to notify the policy that the process 'p' has created login
+  context 'l'.
+*/
+typedef void mpo_lctx_notify_create_t(
+	struct proc *p,
+	struct lctx *l
+);
+/**
+  @brief A process has joined a login context
+  @param p Subject
+  @param l Login Context
+
+  When a process joins a login context, either via setlcid() or via
+  fork() this entrypoint is called to notify the policy that process
+  'p' is now a member of login context 'l'.
+*/
+typedef void mpo_lctx_notify_join_t(
+	struct proc *p,
+	struct lctx *l
+);
+/**
+  @brief A process has left a login context
+  @param p Subject
+  @param l Login Context
+
+  When a process leaves a login context either via setlcid() or as a
+  result of the process exiting this entrypoint is called to notify
+  the policy that the process 'p' is no longer a member of login context 'l'.
+*/
+typedef void mpo_lctx_notify_leave_t(
+	struct proc *p,
+	struct lctx *l
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param bpf_d BPF descriptor
+ @param b_label Policy label for bpf_d
+ @param m Object; mbuf
+ @param m_label Policy label to fill in for m
+
+ Set the label on the mbuf header of a newly created datagram
+ generated using the passed BPF descriptor. This call is made when
+ a write is performed to the BPF device associated with the passed
+ BPF descriptor.
+*/
+typedef void mpo_mbuf_label_associate_bpfdesc_t(
+	struct bpf_d *bpf_d,
+	struct label *b_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param ifp Interface descriptor
+ @param i_label Existing label of ifp
+ @param m Object; mbuf
+ @param m_label Policy label to fill in for m
+
+ Label an mbuf based on the interface from which it was received.
+*/
+typedef void mpo_mbuf_label_associate_ifnet_t(
+	struct ifnet *ifp,
+	struct label *i_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param inp inpcb structure
+ @param i_label Existing label of inp
+ @param m Object; mbuf
+ @param m_label Policy label to fill in for m
+
+ Label an mbuf based on the inpcb from which it was derived.
+*/
+typedef void mpo_mbuf_label_associate_inpcb_t(
+	struct inpcb *inp,
+	struct label *i_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+  @brief Set the label on a newly reassembled IP datagram
+  @param ipq IP fragment reassembly queue
+  @param ipqlabel Policy label for ipq
+  @param mbuf IP datagram to be labeled
+  @param mbuflabel Policy label to be filled in for mbuf
+
+  Set the label on a newly reassembled IP datagram (mbuf) from the IP
+  fragment reassembly queue (ipq) from which it was generated.
+*/
+typedef void mpo_mbuf_label_associate_ipq_t(
+	struct ipq *ipq,
+	struct label *ipqlabel,
+	struct mbuf *mbuf,
+	struct label *mbuflabel
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param ifp Subject; network interface
+ @param i_label Existing label of ifp
+ @param m Object; mbuf
+ @param m_label Policy label to fill in for m
+
+ Set the label on the mbuf header of a newly created datagram
+ generated for the purposes of a link layer response for the passed
+ interface. This call may be made in a number of situations, including
+ for ARP or ND6 responses in the IPv4 and IPv6 stacks.
+*/
+typedef void mpo_mbuf_label_associate_linklayer_t(
+	struct ifnet *ifp,
+	struct label *i_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param oldmbuf mbuf headerder for existing datagram for existing datagram
+ @param oldmbuflabel Policy label for oldmbuf
+ @param ifp Network interface
+ @param ifplabel Policy label for ifp
+ @param newmbuf mbuf header to be labeled for new datagram
+ @param newmbuflabel Policy label for newmbuf
+
+ Set the label on the mbuf header of a newly created datagram
+ generated from the existing passed datagram when it is processed
+ by the passed multicast encapsulation interface. This call is made
+ when an mbuf is to be delivered using the virtual interface.
+*/
+typedef void mpo_mbuf_label_associate_multicast_encap_t(
+	struct mbuf *oldmbuf,
+	struct label *oldmbuflabel,
+	struct ifnet *ifp,
+	struct label *ifplabel,
+	struct mbuf *newmbuf,
+	struct label *newmbuflabel
+);
+/**
+ @brief Assign a label to a new mbuf
+ @param oldmbuf Received datagram
+ @param oldmbuflabel Policy label for oldmbuf
+ @param newmbuf Newly created datagram
+ @param newmbuflabel Policy label for newmbuf
+
+ Set the label on the mbuf header of a newly created datagram generated
+ by the IP stack in response to an existing received datagram (oldmbuf).
+ This call may be made in a number of situations, including when responding
+ to ICMP request datagrams.
+*/
+typedef void mpo_mbuf_label_associate_netlayer_t(
+	struct mbuf *oldmbuf,
+	struct label *oldmbuflabel,
+	struct mbuf *newmbuf,
+	struct label *newmbuflabel
+);
+/**
+  @brief Assign a label to a new mbuf
+  @param so Socket to label
+  @param so_label Policy label for socket
+  @param m Object; mbuf
+  @param m_label Policy label to fill in for m
+
+  An mbuf structure is used to store network traffic in transit.
+  When an application sends data to a socket or a pipe, it is wrapped
+  in an mbuf first.  This function sets the label on a newly created mbuf header
+  based on the socket sending the data.  The contents of the label should be
+  suitable for performing an access check on the receiving side of the
+  communication.
+
+  Only labeled MBUFs will be presented to the policy via this entrypoint.
+*/
+typedef void mpo_mbuf_label_associate_socket_t(
+	socket_t so,
+	struct label *so_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+  @brief Copy a mbuf label
+  @param src Source label
+  @param dest Destination label
+
+  Copy the mbuf label information in src into dest.
+
+  Only called when both source and destination mbufs have labels.
+*/
+typedef void mpo_mbuf_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy mbuf label
+  @param label The label to be destroyed
+
+  Destroy a mbuf label.  Since the
+  object is going out of scope, policy modules should free any
+  internal storage associated with the label so that it may be
+  destroyed.
+*/
+typedef void mpo_mbuf_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize mbuf label
+  @param label New label to initialize
+  @param flag Malloc flags
+
+  Initialize the label for a newly instantiated mbuf.
+
+  @warning Since it is possible for the flags to be set to
+  M_NOWAIT, the malloc operation may fail.
+
+  @return On success, 0, otherwise, an appropriate errno return value.
+*/
+typedef int mpo_mbuf_label_init_t(
+	struct label *label,
+	int flag
+);
+/**
+  @brief Access control check for fsctl
+  @param cred Subject credential
+  @param mp The mount point
+  @param label Label associated with the mount point
+  @param com Filesystem-dependent request code; see fsctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the volume operation indicated by com.
+
+  @warning The fsctl() system call is directly analogous to ioctl(); since
+  the associated data is opaque from the standpoint of the MAC framework
+  and since these operations can affect many aspects of system operation,
+  policies must exercise extreme care when implementing access control checks.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_mount_check_fsctl_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *label,
+	unsigned int cmd
+);
+/**
+  @brief Access control check for the retrieval of file system attributes
+  @param cred Subject credential
+  @param mp The mount structure of the file system
+  @param vfa The attributes requested
+
+  This entry point determines whether given subject can get information
+  about the given file system.  This check happens during statfs() syscalls,
+  but is also used by other parts within the kernel such as the audit system.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+
+typedef int mpo_mount_check_getattr_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mp_label,
+	struct vfs_attr *vfa
+);
+/**
+  @brief Access control check for mount point relabeling
+  @param cred Subject credential
+  @param mp Object file system mount point
+  @param mntlabel Policy label for fle system mount point
+
+  Determine whether the subject identified by the credential can relabel
+  the mount point. This call is made when a file system mount is updated.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch
+  or EPERM for lack of privilege.
+*/
+typedef int mpo_mount_check_label_update_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mntlabel
+);
+/**
+  @brief Access control check for mounting a file system
+  @param cred Subject credential
+  @param vp Vnode that is to be the mount point
+  @param vlabel Label associated with the vnode
+  @param cnp Component name for vp
+  @param vfc_name Filesystem type name
+
+  Determine whether the subject identified by the credential can perform
+  the mount operation on the target vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_mount_check_mount_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel,
+	struct componentname *cnp,
+	const char *vfc_name
+);
+/**
+  @brief Access control check remounting a filesystem
+  @param cred Subject credential
+  @param mp The mount point
+  @param mlabel Label currently associated with the mount point
+
+  Determine whether the subject identified by the credential can perform
+  the remount operation on the target vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_mount_check_remount_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mlabel
+);
+/**
+  @brief Access control check for the settting of file system attributes
+  @param cred Subject credential
+  @param mp The mount structure of the file system
+  @param vfa The attributes requested
+
+  This entry point determines whether given subject can set information
+  about the given file system, for example the volume name.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+
+typedef int mpo_mount_check_setattr_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mp_label,
+	struct vfs_attr *vfa
+);
+/**
+  @brief Access control check for file system statistics
+  @param cred Subject credential
+  @param mp Object file system mount
+  @param mntlabel Policy label for mp
+
+  Determine whether the subject identified by the credential can see
+  the results of a statfs performed on the file system. This call may
+  be made in a number of situations, including during invocations of
+  statfs(2) and related calls, as well as to determine what file systems
+  to exclude from listings of file systems, such as when getfsstat(2)
+  is invoked.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch
+  or EPERM for lack of privilege.
+*/
+typedef int mpo_mount_check_stat_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mntlabel
+);
+/**
+  @brief Access control check for unmounting a filesystem
+  @param cred Subject credential
+  @param mp The mount point
+  @param mlabel Label associated with the mount point
+
+  Determine whether the subject identified by the credential can perform
+  the unmount operation on the target vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_mount_check_umount_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mlabel
+);
+/**
+  @brief Create mount labels
+  @param cred Subject credential
+  @param mp Mount point of file system being mounted
+  @param mntlabel Label to associate with the new mount point
+  @see mpo_mount_label_init_t
+
+  Fill out the labels on the mount point being created by the supplied
+  user credential.  This call is made when file systems are first mounted.
+*/
+typedef void mpo_mount_label_associate_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mntlabel
+);
+/**
+  @brief Destroy mount label
+  @param label The label to be destroyed
+
+  Destroy a file system mount label.  Since the
+  object is going out of scope, policy modules should free any
+  internal storage associated with the label so that it may be
+  destroyed.
+*/
+typedef void mpo_mount_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a mount point label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the mount point label.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will display this externalized version.
+
+  The policy's externalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_mount_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize mount point label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated mount structure.
+  This label is typically used to store a default label in the case
+  that the file system has been mounted singlelabel.  Since some
+  file systems do not support persistent labels (extended attributes)
+  or are read-only (such as CD-ROMs), it is often necessary to store
+  a default label separately from the label of the mount point
+  itself.  Sleeping is permitted.
+*/
+typedef void mpo_mount_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a mount point label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a mount point file system label from an external representation.
+  An externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_mount_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Set the label on an IPv4 datagram fragment
+  @param datagram Datagram being fragmented
+  @param datagramlabel Policy label for datagram
+  @param fragment New fragment
+  @param fragmentlabel Policy label for fragment
+
+  Called when an IPv4 datagram is fragmented into several smaller datagrams.
+  Policies implementing mbuf labels will typically copy the label from the
+  source datagram to the new fragment.
+*/
+typedef void mpo_netinet_fragment_t(
+	struct mbuf *datagram,
+	struct label *datagramlabel,
+	struct mbuf *fragment,
+	struct label *fragmentlabel
+);
+/**
+  @brief Set the label on an ICMP reply
+  @param m mbuf containing the ICMP reply
+  @param mlabel Policy label for m
+
+  A policy may wish to update the label of an mbuf that refers to
+  an ICMP packet being sent in response to an IP packet.  This may
+  be called in response to a bad packet or an ICMP request.
+*/
+typedef void mpo_netinet_icmp_reply_t(
+	struct mbuf *m,
+	struct label *mlabel
+);
+/**
+  @brief Set the label on a TCP reply
+  @param m mbuf containing the TCP reply
+  @param mlabel Policy label for m
+
+  Called for outgoing TCP packets not associated with an actual socket.
+*/
+typedef void mpo_netinet_tcp_reply_t(
+	struct mbuf *m,
+	struct label *mlabel
+);
+/**
+  @brief Access control check for pipe ioctl
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The label on the pipe
+  @param cmd The ioctl command; see ioctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the ioctl operation indicated by cmd.
+
+  @warning Since ioctl data is opaque from the standpoint of the MAC
+  framework, policies must exercise extreme care when implementing
+  access control checks.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_ioctl_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel,
+	unsigned int cmd
+);
+/**
+  @brief Access control check for pipe kqfilter
+  @param cred Subject credential
+  @param kn Object knote
+  @param cpipe Object to be accessed
+  @param pipelabel Policy label for the pipe
+
+  Determine whether the subject identified by the credential can
+  receive the knote on the passed pipe.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_pipe_check_kqfilter_t(
+	kauth_cred_t cred,
+	struct knote *kn,
+	struct pipe *cpipe,
+	struct label *pipelabel
+);
+/**
+  @brief Access control check for pipe relabel
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The current label on the pipe
+  @param newlabel The new label to be used
+
+  Determine whether the subject identified by the credential can
+  perform a relabel operation on the passed pipe.  The cred object holds
+  the credentials of the subject performing the operation.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_label_update_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for pipe read
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The label on the pipe
+
+  Determine whether the subject identified by the credential can
+  perform a read operation on the passed pipe.  The cred object holds
+  the credentials of the subject performing the operation.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_read_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel
+);
+/**
+  @brief Access control check for pipe select
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The label on the pipe
+  @param which The operation selected on: FREAD or FWRITE
+
+  Determine whether the subject identified by the credential can
+  perform a select operation on the passed pipe.  The cred object holds
+  the credentials of the subject performing the operation.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_select_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel,
+	int which
+);
+/**
+  @brief Access control check for pipe stat
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The label on the pipe
+
+  Determine whether the subject identified by the credential can
+  perform a stat operation on the passed pipe.  The cred object holds
+  the credentials of the subject performing the operation.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_stat_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel
+);
+/**
+  @brief Access control check for pipe write
+  @param cred Subject credential
+  @param cpipe Object to be accessed
+  @param pipelabel The label on the pipe
+
+  Determine whether the subject identified by the credential can
+  perform a write operation on the passed pipe.  The cred object holds
+  the credentials of the subject performing the operation.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+
+*/
+typedef int mpo_pipe_check_write_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel
+);
+/**
+  @brief Create a pipe label
+  @param cred Subject credential
+  @param cpipe object to be labeled
+  @param label Label for the pipe object
+
+  Create a label for the pipe object being created by the supplied
+  user credential. This call is made when the pipe is being created
+  XXXPIPE(for one or both sides of the pipe?).
+
+*/
+typedef void mpo_pipe_label_associate_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *pipelabel
+);
+/**
+  @brief Copy a pipe label
+  @param src Source pipe label
+  @param dest Destination pipe label
+
+  Copy the pipe label associated with src to dest.
+  XXXPIPE Describe when this is used: most likely during pipe creation to
+          copy from rpipe to wpipe.
+*/
+typedef void mpo_pipe_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy pipe label
+  @param label The label to be destroyed
+
+  Destroy a pipe label.  Since the object is going out of scope,
+  policy modules should free any internal storage associated with the
+  label so that it may be destroyed.
+*/
+typedef void mpo_pipe_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a pipe label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a pipe.
+  An externalized label consists of a text representation
+  of the label contents that can be used with user applications.
+  Policy-agnostic user space tools will display this externalized
+  version.
+
+  The policy's externalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_pipe_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize pipe label
+  @param label New label to initialize
+
+  Initialize label storage for use with a newly instantiated pipe object.
+  Sleeping is permitted.
+*/
+typedef void mpo_pipe_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a pipe label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a pipe label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_pipe_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Update a pipe label
+  @param cred Subject credential
+  @param cpipe Object to be labeled
+  @param oldlabel Existing pipe label
+  @param newlabel New label to replace existing label
+  @see mpo_pipe_check_label_update_t
+
+  The subject identified by the credential has previously requested
+  and was authorized to relabel the pipe; this entry point allows
+  policies to perform the actual relabel operation.  Policies should
+  update oldlabel using the label stored in the newlabel parameter.
+
+*/
+typedef void mpo_pipe_label_update_t(
+	kauth_cred_t cred,
+	struct pipe *cpipe,
+	struct label *oldlabel,
+	struct label *newlabel
+);
+/**
+  @brief Policy unload event
+  @param mpc MAC policy configuration
+
+  This is the MAC Framework policy unload event.  This entry point will
+  only be called if the module's policy configuration allows unload (if
+  the MPC_LOADTIME_FLAG_UNLOADOK is set).  Most security policies won't
+  want to be unloaded; they should set their flags to prevent this
+  entry point from being called.
+
+  @warning During this call, the mac policy list mutex is held, so
+  sleep operations cannot be performed, and calls out to other kernel
+  subsystems must be made with caution.
+
+  @see MPC_LOADTIME_FLAG_UNLOADOK
+*/
+typedef void mpo_policy_destroy_t(
+	struct mac_policy_conf *mpc
+);
+/**
+  @brief Policy initialization event
+  @param mpc MAC policy configuration
+  @see mac_policy_register
+  @see mpo_policy_initbsd_t
+
+  This is the MAC Framework policy initialization event.  This entry
+  point is called during mac_policy_register, when the policy module
+  is first registered with the MAC Framework.  This is often done very
+  early in the boot process, after the kernel Mach subsystem has been
+  initialized, but prior to the BSD subsystem being initialized.
+  Since the kernel BSD services are not yet available, it is possible
+  that some initialization must occur later, possibly in the
+  mpo_policy_initbsd_t policy entry point, such as registering BSD system
+  controls (sysctls).  Policy modules loaded at boot time will be
+  registered and initialized before labeled Mach objects are created.
+
+  @warning During this call, the mac policy list mutex is held, so
+  sleep operations cannot be performed, and calls out to other kernel
+  subsystems must be made with caution.
+*/
+typedef void mpo_policy_init_t(
+	struct mac_policy_conf *mpc
+);
+/**
+  @brief Policy BSD initialization event
+  @param mpc MAC policy configuration
+  @see mpo_policy_init_t
+
+  This entry point is called after the kernel BSD subsystem has been
+  initialized.  By this point, the module should already be loaded,
+  registered, and initialized.  Since policy modules are initialized
+  before kernel BSD services are available, this second initialization
+  phase is necessary.  At this point, BSD services (memory management,
+  synchronization primitives, vfs, etc.) are available, but the first
+  process has not yet been created.  Mach-related objects and tasks
+  will already be fully initialized and may be in use--policies requiring
+  ubiquitous labeling may also want to implement mpo_policy_init_t.
+
+  @warning During this call, the mac policy list mutex is held, so
+  sleep operations cannot be performed, and calls out to other kernel
+  subsystems must be made with caution.
+*/
+typedef void mpo_policy_initbsd_t(
+	struct mac_policy_conf *mpc
+);
+/**
+  @brief Policy extension service
+  @param p Calling process
+  @param call Policy-specific syscall number
+  @param arg Pointer to syscall arguments
+
+  This entry point provides a policy-multiplexed system call so that
+  policies may provide additional services to user processes without
+  registering specific system calls. The policy name provided during
+  registration is used to demux calls from userland, and the arguments
+  will be forwarded to this entry point.  When implementing new
+  services, security modules should be sure to invoke appropriate
+  access control checks from the MAC framework as needed.  For
+  example, if a policy implements an augmented signal functionality,
+  it should call the necessary signal access control checks to invoke
+  the MAC framework and other registered policies.
+
+  @warning Since the format and contents of the policy-specific
+  arguments are unknown to the MAC Framework, modules must perform the
+  required copyin() of the syscall data on their own.  No policy
+  mediation is performed, so policies must perform any necessary
+  access control checks themselves.  If multiple policies are loaded,
+  they will currently be unable to mediate calls to other policies.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_policy_syscall_t(
+	struct proc *p,
+	int call,
+	user_addr_t arg
+);
+/**
+  @brief Access control check for copying a send right to another task
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for copying send rights to the port from the
+  specified task. A complementary entry point, mpo_port_check_hold_send,
+  handles the receiving task. port_check_copy_send is called as part of
+  a group of policy invocations when messages with port rights are sent.
+  All access control checks made for a particular message must be successful
+  for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_copy_send_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for obtaining a receive right
+  @param task Label of the receiving task
+  @param port Label of the affected port
+
+  Access control check for a task obtaining receive rights to a
+  port. Usually, these are port rights that were obtained with a call
+  to mach_port_allocate.  This entry point is called as part of a
+  group of policy invocations when messages with port rights are
+  received.  All of these access control checks must succeed in order
+  to receive the message.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_hold_receive_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for obtaining a send once right
+  @param task Label of the receiving task
+  @param port Label of the affected port
+
+  Access control check for a task obtaining send once rights to a port. Usually,
+  these are port rights that were part of a message sent by another userspace
+  task. port_check_hold_send_once is called as part of a group of policy
+  invocations when messages with port rights are received. All of these access
+  control checks must succeed in order to receive the message.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_hold_send_once_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for obtaining a send right
+  @param task Label of the receiving task
+  @param port Label of the affected port
+
+  Access control check for a task obtaining send rights to a port. Usually,
+  these are port rights that were part of a message sent by another userspace
+  task. port_check_hold_send is called as part of a group of policy
+  invocations when messages with port rights are received. All of these access
+  control checks must succeed in order to receive the message.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_hold_send_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for relabelling ports
+  @param task Subject's task label
+  @param oldlabel Original label of port
+  @param newlabel New label for port
+
+  Access control check for relabelling ports. The policy should
+  indicate whether the subject is permitted to change the label
+  of a port from oldlabel to newlabel. The port is locked, but
+  the subject's task label is not locked.
+
+  @warning XXX In future releases, the task label lock will likely
+  also be held.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_label_update_t(
+	struct label *task,
+	struct label *oldlabel,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for producing a send once right from a receive right
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for obtaining send once rights from receive rights.
+  The new send once right may be destined for the calling task, or a different
+  task.  In either case the mpo_port_check_hold_send_once entry point handles
+  the receiving task. port_check_make_send_once may be called as part of a
+  group of policy invocations when messages with port rights are sent.
+  All access control checks made for a particular message must be successful
+  for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_make_send_once_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for producing a send right from a receive right
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for obtaining send rights from receive rights. The new
+  send right may be destined for the calling task, or a different task.
+  In either case the mpo_port_check_hold_send entry point
+  handles the receiving task. port_check_make_send may be called as part of
+  a group of policy invocations when messages with port rights are sent.
+  All access control checks made for a particular message must be successful
+  for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_make_send_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Compute access control check for a Mach message-based service
+  @param proc Sender's process structure (may be NULL)
+  @param task Sender's task label
+  @param port Destination port label
+  @param msgid Message id
+
+  Access control computation for message-based services. This entry point
+  computes permission to the service requested by the specified port and message
+  id, for example a single MiG server routine, and is unrelated to the access
+  check for sending messages to ports (but that check must succeed for the
+  message to be sent to the destination). The result of this access computation
+  is stored in the message trailer field msgh_ad (only if requested by the
+  recipient); it does not actually inhibit the message from being sent or
+  received.
+
+  @return 0 for access granted, nonzero for access denied.
+*/
+
+typedef int mpo_port_check_method_t(
+	struct proc *proc,
+	struct label *task,
+	struct label *port,
+	int msgid
+);
+/**
+  @brief Access control check for transferring a receive right
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for transferring the receive right to a port out
+  of the specified task. A complementary entry point,
+  mpo_port_check_hold_receive, handles the receiving task.
+  port_check_move_receive is called as part of
+  a group of policy invocations when messages with port rights are sent.
+  All access control checks made for a particular message must be successful
+  for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_move_receive_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for transferring a send once right
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for transferring a send once right from one task to
+  the task listening to the specified port. A complementary entry point,
+  mpo_port_check_hold_send_once, handles the receiving task.
+  port_check_move_send_once is called as part of a group of policy invocations
+  when messages with port rights are sent.  All access control checks made
+  for a particular message must be successful for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_move_send_once_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for transferring a send right
+  @param task Label of the sender task
+  @param port Label of the affected port
+
+  Access control check for transferring a send right from one task to the
+  task listening to the specified port. A complementary entry point,
+  mpo_port_check_hold_send, handles the receiving task.
+  port_check_move_send is called as part of a group of policy invocations
+  when messages with port rights are sent.  All access control checks made
+  for a particular message must be successful for the message to be sent.
+
+  The task label and the port are locked. Sleeping is permitted.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_move_send_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Access control check for receiving Mach messsages
+  @param task Label of the receiving task
+  @param sender Label of the sending task
+
+  Access control check for receiving messages. The two labels are locked.
+
+  @warning This entry point can be invoked from many places inside the
+  kernel, with arbitrary other locks held. The implementation of this
+  entry point must not cause page faults, as those are handled by mach
+  messages.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_receive_t(
+	struct label *task,
+	struct label *sender
+);
+/**
+  @brief Access control check for sending Mach messsages
+  @param task Label of the sender task
+  @param port Label of the destination port
+
+  Access control check for sending messages. The task label and the
+  port are locked.
+
+  @warning This entry point can be invoked from many places inside the
+  kernel, with arbitrary other locks held. The implementation of this
+  entry point must not cause page faults, as those are handled by mach
+  messages.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_send_t(
+	struct label *task,
+	struct label *port
+);
+/**
+  @brief Generic access control check
+  @param subj Caller-provided subject label
+  @param obj Caller-provided object label
+  @param serv Service or object class name
+  @param perm Permission, or method, within the specified service
+
+  This function provides a general way for a user process to query
+  an arbitrary access control decision from the system's security policies.
+  Currently, there are no standards for the format of the service and
+  permission names. Labels may be either cred or port labels; the policy
+  must accept either. The userspace interfaces to this entry point allow
+  label strings or label handles (ports) to be provided.
+
+  @return Return 0 if access is granted, non-zero otherwise.
+*/
+typedef int mpo_port_check_service_t(
+	struct label *subj,
+	struct label *obj,
+	const char *serv,
+	const char *perm
+);
+/**
+  @brief Assign a label to a new Mach port created by the kernel
+  @param portlabel Label for the new port
+  @param isreply True if the port is for a reply message from the kernel
+
+  Assign a label to a new port created by the kernel. If the port is being
+  used to reply to a message, isreply is 1 (0 otherwise). The port is locked.
+*/
+typedef void mpo_port_label_associate_kernel_t(
+	struct label *portlabel,
+	int isreply
+);
+/**
+  @brief Assign a label to a new Mach port
+  @param it Task label of issuer
+  @param st Task label of target
+  @param portlabel Label for the new port
+
+  Assign a label to a new port. The policy can base this label on
+  the label of the calling task, as well as the label of the target task.
+  The target task is the one which recieves the first right for this port.
+  Both task labels and the port are locked.
+*/
+typedef void mpo_port_label_associate_t(
+	struct label *it,
+	struct label *st,
+	struct label *portlabel
+);
+/**
+  @brief Request label for new (userspace) object
+  @param subj Subject label
+  @param obj Parent or existing object label
+  @param serv Name of service
+  @param out Computed label
+
+  Ask the loaded policies to compute a label based on the two input labels
+  and the service name. There is currently no standard for the service name,
+  or even what the input labels represent (Subject and parent object are only
+  a suggestion). If successful, the computed label is stored in out. All labels
+  must be port (or task) labels. The userspace interfaces to this entry point
+  allow label handles (ports) to be provided.
+
+  @return 0 on success, or an errno value for failure.
+*/
+typedef int mpo_port_label_compute_t(
+	struct label *subj,
+	struct label *obj,
+	const char *serv,
+	struct label *out
+);
+/**
+  @brief Copy a Mach port label
+  @param src Source port label
+  @param dest Destination port label
+
+  Copy the Mach port label information from src to dest.  This is used
+  to copy user-suplied labels into an existing port.
+*/
+typedef void mpo_port_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy Mach port label
+  @param label The label to be destroyed
+
+  Destroy a Mach port label.  Since the object is going out of
+  scope, policy modules should free any internal storage associated
+  with the label so that it may be destroyed.
+*/
+typedef void mpo_port_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize Mach port label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated Mach port.  Sleeping
+  is permitted.
+*/
+typedef void mpo_port_label_init_t(
+	struct label *label
+);
+/**
+  @brief Update a Mach task port label
+  @param cred User credential label to be used as the source
+  @param task Mach port label to be used as the destination
+  @see mpo_cred_label_update_t
+  @see mpo_cred_label_update_execve_t
+
+  Update the label on a Mach task port, using the supplied user
+  credential label. When a mac_cred_label_update_execve or a mac_cred_label_update
+  operation causes the label on a user credential to change, the Mach
+  task port label also needs to be updated to reflect the change.
+  Both labels are already valid (initialized and created).
+*/
+typedef void mpo_port_label_update_cred_t(
+	struct label *cred,
+	struct label *task
+);
+/**
+  @brief Assign a label to a Mach port connected to a kernel object
+  @param portlabel Label for the port
+  @param kotype Type of kernel object
+
+  Label a kernel port based on the type of object behind it. The
+  kotype parameter is one of the IKOT constants in
+  <kern/ipc_kobject.h>. The port already has a valid label from either
+  mpo_port_label_associate_kernel, or because it is a task port and has a label
+  derived from the process and task labels. The port is locked.
+*/
+typedef void mpo_port_label_update_kobject_t(
+	struct label *portlabel,
+	int kotype
+);
+/**
+  @brief Access control check for POSIX semaphore create
+  @param cred Subject credential
+  @param name String name of the semaphore
+
+  Determine whether the subject identified by the credential can create
+  a POSIX semaphore specified by name.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixsem_check_create_t(
+	kauth_cred_t cred,
+	const char *name
+);
+/**
+  @brief Access control check for POSIX semaphore open
+  @param cred Subject credential
+  @param ps Pointer to semaphore information structure
+  @param semlabel Label associated with the semaphore
+
+  Determine whether the subject identified by the credential can open
+  the named POSIX semaphore with label semlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixsem_check_open_t(
+	kauth_cred_t cred,
+	struct pseminfo *ps,
+	struct label *semlabel
+);
+/**
+  @brief Access control check for POSIX semaphore post
+  @param cred Subject credential
+  @param ps Pointer to semaphore information structure
+  @param semlabel Label associated with the semaphore
+
+  Determine whether the subject identified by the credential can unlock
+  the named POSIX semaphore with label semlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixsem_check_post_t(
+	kauth_cred_t cred,
+	struct pseminfo *ps,
+	struct label *semlabel
+);
+/**
+  @brief Access control check for POSIX semaphore unlink
+  @param cred Subject credential
+  @param ps Pointer to semaphore information structure
+  @param semlabel Label associated with the semaphore
+  @param name String name of the semaphore
+
+  Determine whether the subject identified by the credential can remove
+  the named POSIX semaphore with label semlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixsem_check_unlink_t(
+	kauth_cred_t cred,
+	struct pseminfo *ps,
+	struct label *semlabel,
+	const char *name
+);
+/**
+  @brief Access control check for POSIX semaphore wait
+  @param cred Subject credential
+  @param ps Pointer to semaphore information structure
+  @param semlabel Label associated with the semaphore
+
+  Determine whether the subject identified by the credential can lock
+  the named POSIX semaphore with label semlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixsem_check_wait_t(
+	kauth_cred_t cred,
+	struct pseminfo *ps,
+	struct label *semlabel
+);
+/**
+  @brief Create a POSIX semaphore label
+  @param cred Subject credential
+  @param ps Pointer to semaphore information structure
+  @param semlabel Label to associate with the new semaphore
+  @param name String name of the semaphore
+
+  Label a new POSIX semaphore.  The label was previously
+  initialized and associated with the semaphore.  At this time, an
+  appropriate initial label value should be assigned to the object and
+  stored in semalabel.
+*/
+typedef void mpo_posixsem_label_associate_t(
+	kauth_cred_t cred,
+	struct pseminfo *ps,
+	struct label *semlabel,
+	const char *name
+);
+/**
+  @brief Destroy POSIX semaphore label
+  @param label The label to be destroyed
+
+  Destroy a POSIX semaphore label.  Since the object is
+  going out of scope, policy modules should free any internal storage
+  associated with the label so that it may be destroyed.
+*/
+typedef void mpo_posixsem_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize POSIX semaphore label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated POSIX semaphore. Sleeping
+  is permitted.
+*/
+typedef void mpo_posixsem_label_init_t(
+	struct label *label
+);
+/**
+  @brief Access control check for POSIX shared memory region create
+  @param cred Subject credential
+  @param name String name of the shared memory region
+
+  Determine whether the subject identified by the credential can create
+  the POSIX shared memory region referenced by name.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_create_t(
+	kauth_cred_t cred,
+	const char *name
+);
+/**
+  @brief Access control check for mapping POSIX shared memory
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label associated with the shared memory region
+  @param prot mmap protections; see mmap(2)
+  @param flags shmat flags; see shmat(2)
+
+  Determine whether the subject identified by the credential can map
+  the POSIX shared memory segment associated with shmlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_mmap_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel,
+	int prot,
+	int flags
+);
+/**
+  @brief Access control check for POSIX shared memory region open
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label associated with the shared memory region
+
+  Determine whether the subject identified by the credential can open
+  the POSIX shared memory region.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_open_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel
+);
+/**
+  @brief Access control check for POSIX shared memory stat
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label associated with the shared memory region
+
+  Determine whether the subject identified by the credential can obtain
+  status for the POSIX shared memory segment associated with shmlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_stat_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel
+);
+/**
+  @brief Access control check for POSIX shared memory truncate
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label associated with the shared memory region
+  @param len Length to truncate or extend shared memory segment
+
+  Determine whether the subject identified by the credential can truncate
+  or extend (to len) the POSIX shared memory segment associated with shmlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_truncate_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel,
+	size_t len
+);
+/**
+  @brief Access control check for POSIX shared memory unlink
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label associated with the shared memory region
+  @param name String name of the shared memory region
+
+  Determine whether the subject identified by the credential can delete
+  the POSIX shared memory segment associated with shmlabel.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_posixshm_check_unlink_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel,
+	const char *name
+);
+/**
+  @brief Create a POSIX shared memory region label
+  @param cred Subject credential
+  @param ps Pointer to shared memory information structure
+  @param shmlabel Label to associate with the new shared memory region
+  @param name String name of the shared memory region
+
+  Label a new POSIX shared memory region.  The label was previously
+  initialized and associated with the shared memory region.  At this
+  time, an appropriate initial label value should be assigned to the
+  object and stored in shmlabel.
+*/
+typedef void mpo_posixshm_label_associate_t(
+	kauth_cred_t cred,
+	struct pshminfo *ps,
+	struct label *shmlabel,
+	const char *name
+);
+/**
+  @brief Destroy POSIX shared memory label
+  @param label The label to be destroyed
+
+  Destroy a POSIX shared memory region label.  Since the
+  object is going out of scope, policy modules should free any
+  internal storage associated with the label so that it may be
+  destroyed.
+*/
+typedef void mpo_posixshm_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize POSIX Shared Memory region label
+  @param label New label to initialize
+
+  Initialize the label for newly a instantiated POSIX Shared Memory
+  region. Sleeping is permitted.
+*/
+typedef void mpo_posixshm_label_init_t(
+	struct label *label
+);
+/**
+  @brief Access control check for debugging process
+  @param cred Subject credential
+  @param proc Object process
+
+  Determine whether the subject identified by the credential can debug
+  the passed process. This call may be made in a number of situations,
+  including use of the ptrace(2) and ktrace(2) APIs, as well as for some
+  types of procfs operations.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to hide visibility of the target.
+*/
+typedef int mpo_proc_check_debug_t(
+	kauth_cred_t cred,
+	struct proc *proc
+);
+/**
+  @brief Access control over fork
+  @param cred Subject credential
+  @param proc Subject process trying to fork
+
+  Determine whether the subject identified is allowed to fork.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_fork_t(
+	kauth_cred_t cred,
+	struct proc *proc
+);
+/**
+  @brief Access control check for retrieving audit information
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential can get
+  audit information such as the audit user ID, the preselection mask,
+  the terminal ID and the audit session ID, using the getaudit() system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_getaudit_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control check for retrieving audit user ID
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential can get
+  the user identity being used by the auditing system, using the getauid()
+  system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_getauid_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control check for retrieving Login Context ID
+  @param p0 Calling process
+  @param p Effected process
+  @param pid syscall PID argument
+
+  Determine if getlcid(2) system call is permitted.
+
+  Information returned by this system call is similar to that returned via
+  process listings etc.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_getlcid_t(
+	struct proc *p0,
+	struct proc *p,
+	pid_t pid
+);
+/**
+  @brief Access control check for setting memory protections
+  @param cred Subject credential
+  @param proc User process requesting the change
+  @param addr Start address of the memory range
+  @param size Length address of the memory range
+  @param prot Memory protections, see mmap(2)
+
+  Determine whether the subject identified by the credential should
+  be allowed to set the specified memory protections on memory mapped
+  in the process proc.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_mprotect_t(
+	kauth_cred_t cred,
+	struct proc *proc,
+	user_addr_t addr,
+	user_size_t size,
+	int prot
+);
+/**
+  @brief Access control check for changing scheduling parameters
+  @param cred Subject credential
+  @param proc Object process
+
+  Determine whether the subject identified by the credential can change
+  the scheduling parameters of the passed process.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to limit visibility.
+*/
+typedef int mpo_proc_check_sched_t(
+	kauth_cred_t cred,
+	struct proc *proc
+);
+/**
+  @brief Access control check for setting audit information
+  @param cred Subject credential
+  @param ai Audit information
+
+  Determine whether the subject identified by the credential can set
+  audit information such as the the preselection mask, the terminal ID
+  and the audit session ID, using the setaudit() system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_setaudit_t(
+	kauth_cred_t cred,
+	struct auditinfo *ai
+);
+/**
+  @brief Access control check for setting audit user ID
+  @param cred Subject credential
+  @param auid Audit user ID
+
+  Determine whether the subject identified by the credential can set
+  the user identity used by the auditing system, using the setauid()
+  system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_setauid_t(
+	kauth_cred_t cred,
+	uid_t auid
+);
+/**
+  @brief Access control check for setting the Login Context
+  @param p0 Calling process
+  @param p Effected process
+  @param pid syscall PID argument
+  @param lcid syscall LCID argument
+
+  Determine if setlcid(2) system call is permitted.
+
+  See xnu/bsd/kern/kern_prot.c:setlcid() implementation for example of
+  decoding syscall arguments to determine action desired by caller.
+
+  Five distinct actions are possible: CREATE JOIN LEAVE ADOPT ORPHAN
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_setlcid_t(
+	struct proc *p0,
+	struct proc *p,
+	pid_t pid,
+	pid_t lcid
+);
+/**
+  @brief Access control check for delivering signal
+  @param cred Subject credential
+  @param proc Object process
+  @param signum Signal number; see kill(2)
+
+  Determine whether the subject identified by the credential can deliver
+  the passed signal to the passed process.
+
+  @warning Programs typically expect to be able to send and receive
+  signals as part or their normal process lifecycle; caution should be
+  exercised when implementing access controls over signal events.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to limit visibility.
+*/
+typedef int mpo_proc_check_signal_t(
+	kauth_cred_t cred,
+	struct proc *proc,
+	int signum
+);
+/**
+  @brief Access control check for wait
+  @param cred Subject credential
+  @param proc Object process
+
+  Determine whether the subject identified by the credential can wait
+  for process termination.
+
+  @warning Caution should be exercised when implementing access
+  controls for wait, since programs often wait for child processes to
+  exit.  Failure to be notified of a child process terminating may
+  cause the parent process to hang, or may produce zombie processes.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_proc_check_wait_t(
+	kauth_cred_t cred,
+	struct proc *proc
+);
+/**
+  @brief Destroy process label
+  @param label The label to be destroyed
+
+  Destroy a process label.  Since the object is going
+  out of scope, policy modules should free any internal storage
+  associated with the label so that it may be destroyed.
+*/
+typedef void mpo_proc_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize process label
+  @param label New label to initialize
+  @see mpo_cred_label_init_t
+
+  Initialize the label for a newly instantiated BSD process structure.
+  Normally, security policies will store the process label in the user
+  credential rather than here in the process structure.  However,
+  there are some floating label policies that may need to temporarily
+  store a label in the process structure until it is safe to update
+  the user credential label.  Sleeping is permitted.
+*/
+typedef void mpo_proc_label_init_t(
+	struct label *label
+);
+/**
+  @brief Access control check for socket accept
+  @param cred Subject credential
+  @param socket Object socket
+  @param socklabel Policy label for socket
+
+  Determine whether the subject identified by the credential can accept()
+  a new connection on the socket from the host specified by addr.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_accept_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel
+);
+/**
+  @brief Access control check for a pending socket accept
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+  @param addr Address of the listening socket (coming soon)
+
+  Determine whether the subject identified by the credential can accept()
+  a pending connection on the socket from the host specified by addr.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_accepted_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockaddr *addr
+);
+/**
+  @brief Access control check for socket bind
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+  @param addr Name to assign to the socket
+
+  Determine whether the subject identified by the credential can bind()
+  the name (addr) to the socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_bind_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockaddr *addr
+);
+/**
+  @brief Access control check for socket connect
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+  @param addr Name to assign to the socket
+
+  Determine whether the subject identified by the credential can
+  connect() the passed socket to the remote host specified by addr.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_connect_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockaddr *addr
+);
+/**
+  @brief Access control check for socket() system call.
+  @param cred Subject credential
+  @param domain communication domain
+  @param type socket type
+  @param protocol socket protocol
+
+  Determine whether the subject identified by the credential can
+  make the socket() call.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_create_t(
+	kauth_cred_t cred,
+	int domain,
+	int type,
+	int protocol
+);
+/**
+  @brief Access control check for delivering data to a user's receieve queue
+  @param so The socket data is being delivered to
+  @param so_label The label of so
+  @param m The mbuf whose data will be deposited into the receive queue
+  @param m_label The label of the sender of the data.
+
+  A socket has a queue for receiving incoming data.  When a packet arrives
+  on the wire, it eventually gets deposited into this queue, which the
+  owner of the socket drains when they read from the socket's file descriptor.
+
+  This function determines whether the socket can receive data from
+  the sender specified by m_label.
+
+  @warning There is an outstanding design issue surrounding the placement
+  of this function.  The check must be placed either before or after the
+  TCP sequence and ACK counters are updated.  Placing the check before
+  the counters are updated causes the incoming packet to be resent by
+  the remote if the check rejects it.  Placing the check after the counters
+  are updated results in a completely silent drop.  As far as each TCP stack
+  is concerned the packet was received, however, the data will not be in the
+  socket's receive queue.  Another consideration is that the current design
+  requires using the "failed label" occasionally.  In that case, on rejection,
+  we want the remote TCP to resend the data.  Because of this, we chose to
+  place this check before the counters are updated, so rejected packets will be
+  resent by the remote host.
+
+  If a policy keeps rejecting the same packet, eventually the connection will
+  be dropped.  Policies have several options if this design causes problems.
+  For example, one options is to sanitize the mbuf such that it is acceptable,
+  then accept it.  That may require negotiation between policies as the
+  Framework will not know to re-check the packet.
+
+  The policy must handle NULL MBUF labels.  This will likely be the case
+  for non-local TCP sockets for example.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_deliver_t(
+	socket_t so,
+	struct label *so_label,
+	struct mbuf *m,
+	struct label *m_label
+);
+/**
+  @brief Access control check for socket kqfilter
+  @param cred Subject credential
+  @param kn Object knote
+  @param so Object socket
+  @param socklabel Policy label for socket
+
+  Determine whether the subject identified by the credential can
+  receive the knote on the passed socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_kqfilter_t(
+	kauth_cred_t cred,
+	struct knote *kn,
+	socket_t so,
+	struct label *socklabel
+);
+/**
+  @brief Access control check for socket relabel
+  @param cred Subject credential
+  @param so Object socket
+  @param so_label The current label of so
+  @param newlabel The label to be assigned to so
+
+  Determine whether the subject identified by the credential can
+  change the label on the socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_label_update_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *so_label,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for socket listen
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+
+  Determine whether the subject identified by the credential can
+  listen() on the passed socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_listen_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel
+);
+/**
+  @brief Access control check for socket receive
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+
+  Determine whether the subject identified by the credential can
+  receive data from the socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_receive_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel
+);
+
+/**                                                                                               
+  @brief Access control check for socket receive                                                  
+  @param cred Subject credential                                                                  
+  @param socket Object socket                                                                     
+  @param socklabel Policy label for socket                                                        
+  @param addr Name of the remote socket                                                           
+                                                                                                  
+  Determine whether the subject identified by the credential can                                  
+  receive data from the remote host specified by addr.                                            
+                                                                                                  
+  @return Return 0 if access if granted, otherwise an appropriate                                 
+  value for errno should be returned.                                                             
+*/
+typedef int mpo_socket_check_received_t(
+					kauth_cred_t cred,
+					struct socket *sock,
+					struct label *socklabel,
+					struct sockaddr *saddr
+					);
+
+
+/**
+  @brief Access control check for socket select
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+  @param which The operation selected on: FREAD or FWRITE
+
+  Determine whether the subject identified by the credential can use the
+  socket in a call to select().
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_select_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	int which
+);
+/**
+  @brief Access control check for socket send
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for socket
+  @param addr Address being sent to
+
+  Determine whether the subject identified by the credential can send
+  data to the socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_send_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockaddr *addr
+);
+/**
+  @brief Access control check for retrieving socket status
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for so
+
+  Determine whether the subject identified by the credential can
+  execute the stat() system call on the given socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_stat_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel
+);
+/**
+  @brief Access control check for setting socket options
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for so
+  @param sopt The options being set
+
+  Determine whether the subject identified by the credential can
+  execute the setsockopt system call on the given socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_setsockopt_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockopt *sopt
+);
+/**
+  @brief Access control check for getting socket options
+  @param cred Subject credential
+  @param so Object socket
+  @param socklabel Policy label for so
+  @param sopt The options to get
+
+  Determine whether the subject identified by the credential can
+  execute the getsockopt system call on the given socket.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_socket_check_getsockopt_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *socklabel,
+	struct sockopt *sopt
+);
+/**
+  @brief Label a socket
+  @param oldsock Listening socket
+  @param oldlabel Policy label associated with oldsock
+  @param newsock New socket
+  @param newlabel Policy label associated with newsock
+
+  A new socket is created when a connection is accept(2)ed.  This
+  function labels the new socket based on the existing listen(2)ing
+  socket.
+*/
+typedef void mpo_socket_label_associate_accept_t(
+	socket_t oldsock,
+	struct label *oldlabel,
+	socket_t newsock,
+	struct label *newlabel
+);
+/**
+  @brief Assign a label to a new socket
+  @param cred Credential of the owning process
+  @param so The socket being labeled
+  @param solabel The label
+  @warning cred can be NULL
+
+  Set the label on a newly created socket from the passed subject
+  credential.  This call is made when a socket is created.  The
+  credentials may be null if the socket is being created by the
+  kernel.
+*/
+typedef void mpo_socket_label_associate_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *solabel
+);
+/**
+  @brief Copy a socket label
+  @param src Source label
+  @param dest Destination label
+
+  Copy the socket label information in src into dest.
+*/
+typedef void mpo_socket_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy socket label
+  @param label The label to be destroyed
+
+  Destroy a socket label.  Since the object is going out of
+  scope, policy modules should free any internal storage associated
+  with the label so that it may be destroyed.
+*/
+typedef void mpo_socket_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a socket label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of label
+
+  Produce an externalized socket label based on the label structure passed.
+  An externalized label consists of a text representation of the label
+  contents that can be used with userland applications and read by the
+  user.  If element_name does not match a namespace managed by the policy,
+  simply return 0. Only return nonzero if an error occurs while externalizing
+  the label data.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_socket_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize socket label
+  @param label New label to initialize
+  @param waitok Malloc flags
+
+  Initialize the label of a newly instantiated socket.  The waitok
+  field may be one of M_WAITOK and M_NOWAIT, and should be employed to
+  avoid performing a sleeping malloc(9) during this initialization
+  call.  It it not always safe to sleep during this entry point.
+
+  @warning Since it is possible for the waitok flags to be set to
+  M_NOWAIT, the malloc operation may fail.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_socket_label_init_t(
+	struct label *label,
+	int waitok
+);
+/**
+  @brief Internalize a socket label
+  @param label Label to be filled in
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce an internal socket label structure based on externalized label
+  data in text format.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_socket_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Relabel socket
+  @param cred Subject credential
+  @param so Object; socket
+  @param so_label Current label of the socket
+  @param newlabel The label to be assigned to so
+
+  The subject identified by the credential has previously requested
+  and was authorized to relabel the socket; this entry point allows
+  policies to perform the actual label update operation.
+
+  @warning XXX This entry point will likely change in future versions.
+*/
+typedef void mpo_socket_label_update_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *so_label,
+	struct label *newlabel
+);
+/**
+  @brief Set the peer label on a socket from mbuf
+  @param m Mbuf chain received on socket so
+  @param m_label Label for m
+  @param so Current label for the socket
+  @param so_label Policy label to be filled out for the socket
+
+  Set the peer label of a socket based on the label of the sender of the
+  mbuf.
+
+  This is called for every TCP/IP packet received.  The first call for a given
+  socket operates on a newly initialized label, and subsequent calls operate
+  on existing label data.
+
+  @warning Because this can affect performance significantly, it has
+  different sematics than other 'set' operations.  Typically, 'set' operations
+  operate on newly initialzed labels and policies do not need to worry about
+  clobbering existing values.  In this case, it is too inefficient to
+  initialize and destroy a label every time data is received for the socket.
+  Instead, it is up to the policies to determine how to replace the label data.
+  Most policies should be able to replace the data inline.
+*/
+typedef void mpo_socketpeer_label_associate_mbuf_t(
+	struct mbuf *m,
+	struct label *m_label,
+	socket_t so,
+	struct label *so_label
+);
+/**
+  @brief Set the peer label on a socket from socket
+  @param source Local socket
+  @param sourcelabel Policy label for source
+  @param target Peer socket
+  @param targetlabel Policy label to fill in for target
+
+  Set the peer label on a stream UNIX domain socket from the passed
+  remote socket endpoint. This call will be made when the socket pair
+  is connected, and will be made for both endpoints.
+
+  Note that this call is only made on connection; it is currently not updated
+  during communication.
+*/
+typedef void mpo_socketpeer_label_associate_socket_t(
+	socket_t source,
+	struct label *sourcelabel,
+	socket_t target,
+	struct label *targetlabel
+);
+/**
+  @brief Destroy socket peer label
+  @param label The peer label to be destroyed
+
+  Destroy a socket peer label.  Since the object is going out of
+  scope, policy modules should free any internal storage associated
+  with the label so that it may be destroyed.
+*/
+typedef void mpo_socketpeer_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a socket peer label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of label
+
+  Produce an externalized socket peer label based on the label structure
+  passed. An externalized label consists of a text representation of the
+  label contents that can be used with userland applications and read by the
+  user.  If element_name does not match a namespace managed by the policy,
+  simply return 0. Only return nonzero if an error occurs while externalizing
+  the label data.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_socketpeer_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize socket peer label
+  @param label New label to initialize
+  @param waitok Malloc flags
+
+  Initialize the peer label of a newly instantiated socket.  The
+  waitok field may be one of M_WAITOK and M_NOWAIT, and should be
+  employed to avoid performing a sleeping malloc(9) during this
+  initialization call.  It it not always safe to sleep during this
+  entry point.
+
+  @warning Since it is possible for the waitok flags to be set to
+  M_NOWAIT, the malloc operation may fail.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_socketpeer_label_init_t(
+	struct label *label,
+	int waitok
+);
+/**
+  @brief Access control check for enabling accounting
+  @param cred Subject credential
+  @param vp Accounting file
+  @param vlabel Label associated with vp
+
+  Determine whether the subject should be allowed to enable accounting,
+  based on its label and the label of the accounting log file.  See
+  acct(5) for more information.
+
+  As accounting is disabled by passing NULL to the acct(2) system call,
+  the policy should be prepared for both 'vp' and 'vlabel' to be NULL.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_acct_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Access control check for audit
+  @param cred Subject credential
+  @param record Audit record
+  @param length Audit record length
+
+  Determine whether the subject identified by the credential can submit
+  an audit record for inclusion in the audit log via the audit() system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_audit_t(
+	kauth_cred_t cred,
+	void *record,
+	int length
+);
+/**
+  @brief Access control check for controlling audit
+  @param cred Subject credential
+  @param vp Audit file
+  @param vl Label associated with vp
+
+  Determine whether the subject should be allowed to enable auditing using
+  the auditctl() system call, based on its label and the label of the proposed
+  audit file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_auditctl_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vl
+);
+/**
+  @brief Access control check for manipulating auditing
+  @param cred Subject credential
+  @param cmd Audit control command
+
+  Determine whether the subject identified by the credential can perform
+  the audit subsystem control operation cmd via the auditon() system call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_auditon_t(
+	kauth_cred_t cred,
+	int cmd
+);
+/**
+  @brief Access control check for obtaining the host control port
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential can
+  obtain the host control port.
+
+  @return Return 0 if access is granted, or non-zero otherwise.
+*/
+typedef int mpo_system_check_host_priv_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control check for calling NFS services
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential should be
+  allowed to call nfssrv(2).
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_nfsd_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control check for reboot
+  @param cred Subject credential
+  @param howto howto parameter from reboot(2)
+
+  Determine whether the subject identified by the credential should be
+  allowed to reboot the system in the specified manner.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_reboot_t(
+	kauth_cred_t cred,
+	int howto
+);
+/**
+  @brief Access control check for setting system clock
+  @param cred Subject credential
+
+  Determine whether the subject identified by the credential should be
+  allowed to set the system clock.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_settime_t(
+	kauth_cred_t cred
+);
+/**
+  @brief Access control check for removing swap devices
+  @param cred Subject credential
+  @param vp Swap device
+  @param label Label associated with vp
+
+  Determine whether the subject identified by the credential should be
+  allowed to remove vp as a swap device.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_swapoff_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for adding swap devices
+  @param cred Subject credential
+  @param vp Swap device
+  @param label Label associated with vp
+
+  Determine whether the subject identified by the credential should be
+  allowed to add vp as a swap device.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_swapon_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for sysctl
+  @param cred Subject credential
+  @param name Integer name; see sysctl(3)
+  @param namelen Length of name array of integers; see sysctl(3)
+  @param old 0 or address where to store old value; see sysctl(3)
+  @param oldlenp Pointer to length of old buffer; see sysctl(3)
+  @param inkernel Boolean; 1 if called from kernel
+  @param newvalue 0 or address of new value; see sysctl(3)
+  @param newlen Length of new buffer; see sysctl(3)
+
+  Determine whether the subject identified by the credential should be
+  allowed to make the specified sysctl(3) transaction.
+
+  The sysctl(3) call specifies that if the old value is not desired,
+  oldp and oldlenp should be set to NULL.  Likewise, if a new value is
+  not to be set, newp should be set to NULL and newlen set to 0.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_system_check_sysctl_t(
+	kauth_cred_t cred,
+	int *name,
+	u_int namelen,
+	user_addr_t old,	/* NULLOK */
+	user_addr_t oldlenp,	/* NULLOK */
+	int inkernel,
+	user_addr_t newvalue,	/* NULLOK */
+	size_t newlen
+);
+/**
+  @brief Create a System V message label
+  @param cred Subject credential
+  @param msqkptr The message queue the message will be placed in
+  @param msqlabel The label of the message queue
+  @param msgptr The message
+  @param msglabel The label of the message
+
+  Label the message as its placed in the message queue.
+*/
+typedef void mpo_sysvmsg_label_associate_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel,
+	struct msg *msgptr,
+	struct label *msglabel
+);
+/**
+  @brief Destroy System V message label
+  @param label The label to be destroyed
+
+  Destroy a System V message label.  Since the object is
+  going out of scope, policy modules should free any internal storage
+  associated with the label so that it may be destroyed.
+*/
+typedef void mpo_sysvmsg_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize System V message label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated System V message.
+*/
+typedef void mpo_sysvmsg_label_init_t(
+	struct label *label
+);
+/**
+  @brief Clean up a System V message label
+  @param label The label to be destroyed
+
+  Clean up a System V message label.  Darwin pre-allocates
+  messages at system boot time and re-uses them rather than
+  allocating new ones.  Before messages are returned to the "free
+  pool", policies can cleanup or overwrite any information present in
+  the label.
+*/
+typedef void mpo_sysvmsg_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Access control check for System V message enqueuing
+  @param cred Subject credential
+  @param msgptr The message
+  @param msglabel The message's label
+  @param msqkptr The message queue
+  @param msqlabel The message queue's label
+
+  Determine whether the subject identified by the credential can add the
+  given message to the given message queue.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_enqueue_t(
+	kauth_cred_t cred,
+	struct msg *msgptr,
+	struct label *msglabel,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel
+);
+/**
+  @brief Access control check for System V message reception
+  @param cred The credential of the intended recipient
+  @param msgptr The message
+  @param msglabel The message's label
+
+  Determine whether the subject identified by the credential can receive
+  the given message.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msgrcv_t(
+	kauth_cred_t cred,
+	struct msg *msgptr,
+	struct label *msglabel
+);
+/**
+  @brief Access control check for System V message queue removal
+  @param cred The credential of the caller
+  @param msgptr The message
+  @param msglabel The message's label
+
+  System V message queues are removed using the msgctl() system call.
+  The system will iterate over each messsage in the queue, calling this
+  function for each, to determine whether the caller has the appropriate
+  credentials.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msgrmid_t(
+	kauth_cred_t cred,
+	struct msg *msgptr,
+	struct label *msglabel
+);
+/**
+  @brief Access control check for msgctl()
+  @param cred The credential of the caller
+  @param msqptr The message queue
+  @param msqlabel The message queue's label
+
+  This access check is performed to validate calls to msgctl().
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msqctl_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel,
+	int cmd
+);
+/**
+  @brief Access control check to get a System V message queue
+  @param cred The credential of the caller
+  @param msqptr The message queue requested
+  @param msqlabel The message queue's label
+
+  On a call to msgget(), if the queue requested already exists,
+  and it is a public queue, this check will be performed before the
+  queue's ID is returned to the user.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msqget_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel
+);
+/**
+  @brief Access control check to receive a System V message from the given queue
+  @param cred The credential of the caller
+  @param msqptr The message queue to receive from
+  @param msqlabel The message queue's label
+
+  On a call to msgrcv(), this check is performed to determine whether the
+  caller has receive rights on the given queue.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msqrcv_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel
+);
+/**
+  @brief Access control check to send a System V message to the given queue
+  @param cred The credential of the caller
+  @param msqptr The message queue to send to
+  @param msqlabel The message queue's label
+
+  On a call to msgsnd(), this check is performed to determine whether the
+  caller has send rights on the given queue.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvmsq_check_msqsnd_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel
+);
+/**
+  @brief Create a System V message queue label
+  @param cred Subject credential
+  @param msqkptr The message queue
+  @param msqlabel The label of the message queue
+
+*/
+typedef void mpo_sysvmsq_label_associate_t(
+	kauth_cred_t cred,
+	struct msqid_kernel *msqptr,
+	struct label *msqlabel
+);
+/**
+  @brief Destroy System V message queue label
+  @param label The label to be destroyed
+
+  Destroy a System V message queue label.  Since the object is
+  going out of scope, policy modules should free any internal storage
+  associated with the label so that it may be destroyed.
+*/
+typedef void mpo_sysvmsq_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize System V message queue label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated System V message queue.
+*/
+typedef void mpo_sysvmsq_label_init_t(
+	struct label *label
+);
+/**
+  @brief Clean up a System V message queue label
+  @param label The label to be destroyed
+
+  Clean up a System V message queue label.  Darwin pre-allocates
+  message queues at system boot time and re-uses them rather than
+  allocating new ones.  Before message queues are returned to the "free
+  pool", policies can cleanup or overwrite any information present in
+  the label.
+*/
+typedef void mpo_sysvmsq_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Access control check for System V semaphore control operation
+  @param cred Subject credential
+  @param semakptr Pointer to semaphore identifier
+  @param semaklabel Label associated with semaphore
+  @param cmd Control operation to be performed; see semctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the operation indicated by cmd on the System V semaphore semakptr.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvsem_check_semctl_t(
+	kauth_cred_t cred,
+	struct semid_kernel *semakptr,
+	struct label *semaklabel,
+	int cmd
+);
+/**
+  @brief Access control check for obtaining a System V semaphore
+  @param cred Subject credential
+  @param semakptr Pointer to semaphore identifier
+  @param semaklabel Label to associate with the semaphore
+
+  Determine whether the subject identified by the credential can
+  obtain a System V semaphore.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvsem_check_semget_t(
+	kauth_cred_t cred,
+	struct semid_kernel *semakptr,
+	struct label *semaklabel
+);
+/**
+  @brief Access control check for System V semaphore operations
+  @param cred Subject credential
+  @param semakptr Pointer to semaphore identifier
+  @param semaklabel Label associated with the semaphore
+  @param accesstype Flags to indicate access (read and/or write)
+
+  Determine whether the subject identified by the credential can
+  perform the operations on the System V semaphore indicated by
+  semakptr.  The accesstype flags hold the maximum set of permissions
+  from the sem_op array passed to the semop system call.  It may
+  contain SEM_R for read-only operations or SEM_A for read/write
+  operations.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvsem_check_semop_t(
+	kauth_cred_t cred,
+	struct semid_kernel *semakptr,
+	struct label *semaklabel,
+	size_t accesstype
+);
+/**
+  @brief Create a System V semaphore label
+  @param cred Subject credential
+  @param semakptr The semaphore being created
+  @param semalabel Label to associate with the new semaphore
+
+  Label a new System V semaphore.  The label was previously
+  initialized and associated with the semaphore.  At this time, an
+  appropriate initial label value should be assigned to the object and
+  stored in semalabel.
+*/
+typedef void mpo_sysvsem_label_associate_t(
+	kauth_cred_t cred,
+	struct semid_kernel *semakptr,
+	struct label *semalabel
+);
+/**
+  @brief Destroy System V semaphore label
+  @param label The label to be destroyed
+
+  Destroy a System V semaphore label.  Since the object is
+  going out of scope, policy modules should free any internal storage
+  associated with the label so that it may be destroyed.
+*/
+typedef void mpo_sysvsem_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize System V semaphore label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated System V semaphore.  Sleeping
+  is permitted.
+*/
+typedef void mpo_sysvsem_label_init_t(
+	struct label *label
+);
+/**
+  @brief Clean up a System V semaphore label
+  @param label The label to be cleaned
+
+  Clean up a System V semaphore label.  Darwin pre-allocates
+  semaphores at system boot time and re-uses them rather than
+  allocating new ones.  Before semaphores are returned to the "free
+  pool", policies can cleanup or overwrite any information present in
+  the label.
+*/
+typedef void mpo_sysvsem_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Access control check for mapping System V shared memory
+  @param cred Subject credential
+  @param shmsegptr Pointer to shared memory segment identifier
+  @param shmseglabel Label associated with the shared memory segment
+  @param shmflg shmat flags; see shmat(2)
+
+  Determine whether the subject identified by the credential can map
+  the System V shared memory segment associated with shmsegptr.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvshm_check_shmat_t(
+	kauth_cred_t cred,
+	struct shmid_kernel *shmsegptr,
+	struct label *shmseglabel,
+	int shmflg
+);
+/**
+  @brief Access control check for System V shared memory control operation
+  @param cred Subject credential
+  @param shmsegptr Pointer to shared memory segment identifier
+  @param shmseglabel Label associated with the shared memory segment
+  @param cmd Control operation to be performed; see shmctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the operation indicated by cmd on the System V shared memory segment
+  shmsegptr.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvshm_check_shmctl_t(
+	kauth_cred_t cred,
+	struct shmid_kernel *shmsegptr,
+	struct label *shmseglabel,
+	int cmd
+);
+/**
+  @brief Access control check for unmapping System V shared memory
+  @param cred Subject credential
+  @param shmsegptr Pointer to shared memory segment identifier
+  @param shmseglabel Label associated with the shared memory segment
+
+  Determine whether the subject identified by the credential can unmap
+  the System V shared memory segment associated with shmsegptr.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvshm_check_shmdt_t(
+	kauth_cred_t cred,
+	struct shmid_kernel *shmsegptr,
+	struct label *shmseglabel
+);
+/**
+  @brief Access control check obtaining System V shared memory identifier
+  @param cred Subject credential
+  @param shmsegptr Pointer to shared memory segment identifier
+  @param shmseglabel Label associated with the shared memory segment
+  @param shmflg shmget flags; see shmget(2)
+
+  Determine whether the subject identified by the credential can get
+  the System V shared memory segment address.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_sysvshm_check_shmget_t(
+	kauth_cred_t cred,
+	struct shmid_kernel *shmsegptr,
+	struct label *shmseglabel,
+	int shmflg
+);
+/**
+  @brief Create a System V shared memory region label
+  @param cred Subject credential
+  @param shmsegptr The shared memory region being created
+  @param shmlabel Label to associate with the new shared memory region
+
+  Label a new System V shared memory region.  The label was previously
+  initialized and associated with the shared memory region.  At this
+  time, an appropriate initial label value should be assigned to the
+  object and stored in shmlabel.
+*/
+typedef void mpo_sysvshm_label_associate_t(
+	kauth_cred_t cred,
+	struct shmid_kernel *shmsegptr,
+	struct label *shmlabel
+);
+/**
+  @brief Destroy System V shared memory label
+  @param label The label to be destroyed
+
+  Destroy a System V shared memory region label.  Since the
+  object is going out of scope, policy modules should free any
+  internal storage associated with the label so that it may be
+  destroyed.
+*/
+typedef void mpo_sysvshm_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Initialize System V Shared Memory region label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated System V Shared Memory
+  region.  Sleeping is permitted.
+*/
+typedef void mpo_sysvshm_label_init_t(
+	struct label *label
+);
+/**
+  @brief Clean up a System V Share Memory Region label
+  @param shmlabel The label to be cleaned
+
+  Clean up a System V Shared Memory Region label.  Darwin
+  pre-allocates these objects at system boot time and re-uses them
+  rather than allocating new ones.  Before the memory regions are
+  returned to the "free pool", policies can cleanup or overwrite any
+  information present in the label.
+*/
+typedef void mpo_sysvshm_label_recycle_t(
+	struct label *shmlabel
+);
+/**
+  @brief Access control check for getting a process's task name
+  @param cred Subject credential
+  @param proc Object process
+
+  Determine whether the subject identified by the credential can get
+  the passed process's task name port.
+  This call is used by the task_name_for_pid(2) API.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to hide visibility of the target.
+*/
+typedef int mpo_proc_check_get_task_name_t(
+	kauth_cred_t cred,
+	struct proc *p
+);
+/**
+  @brief Access control check for getting a process's task port
+  @param cred Subject credential
+  @param proc Object process
+
+  Determine whether the subject identified by the credential can get
+  the passed process's task control port.
+  This call is used by the task_for_pid(2) API.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch,
+  EPERM for lack of privilege, or ESRCH to hide visibility of the target.
+*/
+typedef int mpo_proc_check_get_task_t(
+	kauth_cred_t cred,
+	struct proc *p
+);
+/**
+  @brief Assign a label to a new kernelspace Mach task
+  @param kproc New task
+  @param tasklabel Label for new task
+  @param portlabel Label for new task port
+  @see mpo_cred_label_associate_kernel_t
+
+  Assign labels to a new kernel task and its task port. Both the task and
+  task port labels should be specified. Both new labels are initialized.
+  If there is an associated BSD process structure, it will be labelled
+  with calls to mpo_cred_label_associate_kernel.
+*/
+typedef void mpo_task_label_associate_kernel_t(
+	struct task *kproc,
+	struct label *tasklabel,
+	struct label *portlabel
+);
+/**
+  @brief Assign a label to a new (userspace) Mach task
+  @param parent Parent task
+  @param child New (child) task
+  @param parentlabel Label of parent task
+  @param childlabel Label for new task
+  @param childportlabel Label for new task's task port
+
+  Assign labels to a new task and its task port. Both the task and task port
+  labels should be specified. Both new labels are initialized.  If the task
+  will have an associated BSD process, that information will be made available
+  by the task_label_update and port_label_update_cred entry points.
+*/
+typedef void mpo_task_label_associate_t(
+	struct task *parent,
+	struct task *child,
+	struct label *parentlabel,
+	struct label *childlabel,
+	struct label *childportlabel
+);
+/**
+  @brief Copy a Mach task label
+  @param src Source task label
+  @param dest Destination task label
+
+  Copy the Mach task label information from src to dest.  This is used
+  when duplicating label handles to implement copy-on-write semantics.
+*/
+typedef void mpo_task_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy Mach task label
+  @param label The label to be destroyed
+
+  Destroy a Mach task label.  Since the object is going out of
+  scope, policy modules should free any internal storage associated
+  with the label so that it may be destroyed.
+*/
+typedef void mpo_task_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a task label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a task.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will display this externalized version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_task_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize Mach task label
+  @param label New label to initialize
+
+  Initialize the label for a newly instantiated Mach task.  Sleeping
+  is permitted.
+*/
+typedef void mpo_task_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a task label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a task label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+
+*/
+typedef int mpo_task_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Update a Mach task label
+  @param cred User credential label to be used as the source
+  @param task Mach task label to be used as the destination
+  @see mpo_cred_label_update_t
+  @see mpo_cred_label_update_execve_t
+
+  Update the label on a Mach task, using the supplied user credential
+  label. When a mac_cred_label_update_execve or a mac_cred_label_update operation
+  causes the label on a user credential to change, the Mach task label
+  also needs to be updated to reflect the change.  Both labels are
+  already valid (initialized and created).
+
+  @warning XXX We may change the name of this entry point in a future
+  version of the MAC framework.
+*/
+typedef void mpo_task_label_update_t(
+	struct label *cred,
+	struct label *task
+);
+/**
+  @brief Perform MAC-related events when a thread returns to user space
+  @param code The number of the syscall/trap that has finished
+  @param error The error code that will be returned to user space
+  @param thread Mach (not BSD) thread that is returning
+
+  This entry point permits policy modules to perform MAC-related
+  events when a thread returns to user space, via a system call
+  return, trap return, or otherwise.
+*/
+typedef void mpo_thread_userret_t(
+	int code,
+	int error,
+	struct thread *thread
+);
+/**
+  @brief Check vnode access
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Label for vp
+  @param acc_mode access(2) flags
+
+  Determine how invocations of access(2) and related calls by the
+  subject identified by the credential should return when performed
+  on the passed vnode using the passed access flags. This should
+  generally be implemented using the same semantics used in
+  mpo_vnode_check_open.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_access_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	int acc_mode
+);
+/**
+  @brief Access control check for changing working directory
+  @param cred Subject credential
+  @param dvp Object; vnode to chdir(2) into
+  @param dlabel Policy label for dvp
+
+  Determine whether the subject identified by the credential can change
+  the process working directory to the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_chdir_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel
+);
+/**
+  @brief Access control check for changing root directory
+  @param cred Subject credential
+  @param dvp Directory vnode
+  @param dlabel Policy label associated with dvp
+  @param cnp Component name for dvp
+
+  Determine whether the subject identified by the credential should be
+  allowed to chroot(2) into the specified directory (dvp).
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_vnode_check_chroot_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for creating vnode
+  @param cred Subject credential
+  @param dvp Directory vnode
+  @param dlabel Policy label for dvp
+  @param cnp Component name for dvp
+  @param vap vnode attributes for vap
+
+  Determine whether the subject identified by the credential can create
+  a vnode with the passed parent directory, passed name information,
+  and passed attribute information. This call may be made in a number of
+  situations, including as a result of calls to open(2) with O_CREAT,
+  mknod(2), mkfifo(2), and others.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_create_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct componentname *cnp,
+	struct vnode_attr *vap
+);
+/**
+  @brief Access control check for deleting extended attribute
+  @param cred Subject credential
+  @param vp Object vnode
+  @param vlabel Label associated with vp
+  @param name Extended attribute name
+
+  Determine whether the subject identified by the credential can delete
+  the extended attribute from the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_deleteextattr_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel,
+	const char *name
+);
+/**
+  @brief Access control check for exchanging file data
+  @param cred Subject credential
+  @param v1 vnode 1 to swap
+  @param vl1 Policy label for v1
+  @param v2 vnode 2 to swap
+  @param vl2 Policy label for v2
+
+  Determine whether the subject identified by the credential can swap the data
+  in the two supplied vnodes.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_exchangedata_t(
+	kauth_cred_t cred,
+	struct vnode *v1,
+	struct label *vl1,
+	struct vnode *v2,
+	struct label *vl2
+);
+/**
+  @brief Access control check for executing the vnode
+  @param cred Subject credential
+  @param vp Object vnode to execute
+  @param label Policy label for vp
+  @param execlabel Userspace provided execution label
+  @param cnp Component name for file being executed
+
+  Determine whether the subject identified by the credential can execute
+  the passed vnode. Determination of execute privilege is made separately
+  from decisions about any process label transitioning event.
+
+  The final label, execlabel, corresponds to a label supplied by a
+  user space application through the use of the mac_execve system call.
+  This label will be NULL if the user application uses the the vendor
+  execve(2) call instead of the MAC Framework mac_execve() call.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_exec_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	struct label *execlabel,	/* NULLOK */
+	struct componentname *cnp,
+	u_int *csflags
+);
+/**
+  @brief Access control check for retrieving file attributes
+  @param cred Subject credential
+  @param vp Object vnode
+  @param vlabel Policy label for vp
+  @param alist List of attributes to retrieve
+
+  Determine whether the subject identified by the credential can read
+  various attributes of the specified vnode, or the filesystem or volume on
+  which that vnode resides. See <sys/attr.h> for definitions of the
+  attributes.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege. Access control covers all attributes requested
+  with this call; the security policy is not permitted to change the set of
+  attributes requested.
+*/
+typedef int mpo_vnode_check_getattrlist_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel,
+	struct attrlist *alist
+);
+/**
+  @brief Access control check for retrieving an extended attribute
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param name Extended attribute name
+  @param uio I/O structure pointer
+
+  Determine whether the subject identified by the credential can retrieve
+  the extended attribute from the passed vnode.  The uio parameter
+  will be NULL when the getxattr(2) call has been made with a NULL data
+  value; this is done to request the size of the data only.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_getextattr_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,		/* NULLOK */
+	const char *name,
+	struct uio *uio			/* NULLOK */
+);
+/**
+  @brief Access control check for ioctl
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param com Device-dependent request code; see ioctl(2)
+
+  Determine whether the subject identified by the credential can perform
+  the ioctl operation indicated by com.
+
+  @warning Since ioctl data is opaque from the standpoint of the MAC
+  framework, and since ioctls can affect many aspects of system
+  operation, policies must exercise extreme care when implementing
+  access control checks.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_ioctl_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	unsigned int cmd
+);
+/**
+  @brief Access control check for vnode kqfilter
+  @param cred Subject credential
+  @param kn Object knote
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can
+  receive the knote on the passed vnode.
+
+  @return Return 0 if access if granted, otherwise an appropriate
+  value for errno should be returned.
+*/
+typedef int mpo_vnode_check_kqfilter_t(
+	kauth_cred_t active_cred,
+	kauth_cred_t file_cred,		/* NULLOK */
+	struct knote *kn,
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for relabel
+  @param cred Subject credential
+  @param vp Object vnode
+  @param vnodelabel Existing policy label for vp
+  @param newlabel Policy label update to later be applied to vp
+  @see mpo_relable_vnode_t
+
+  Determine whether the subject identified by the credential can relabel
+  the passed vnode to the passed label update.  If all policies permit
+  the label change, the actual relabel entry point (mpo_vnode_label_update)
+  will follow.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_label_update_t(
+	struct ucred *cred,
+	struct vnode *vp,
+	struct label *vnodelabel,
+	struct label *newlabel
+);
+/**
+  @brief Access control check for creating link
+  @param cred Subject credential
+  @param dvp Directory vnode
+  @param dlabel Policy label associated with dvp
+  @param vp Link destination vnode
+  @param label Policy label associated with vp
+  @param cnp Component name for the link being created
+
+  Determine whether the subject identified by the credential should be
+  allowed to create a link to the vnode vp with the name specified by cnp.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_link_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct vnode *vp,
+	struct label *label,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for listing extended attributes
+  @param cred Subject credential
+  @param vp Object vnode
+  @param vlabel Policy label associated with vp
+
+  Determine whether the subject identified by the credential can retrieve
+  a list of named extended attributes from a vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_listextattr_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Access control check for lookup
+  @param cred Subject credential
+  @param dvp Object vnode
+  @param dlabel Policy label for dvp
+  @param cnp Component name being looked up
+
+  Determine whether the subject identified by the credential can perform
+  a lookup in the passed directory vnode for the passed name (cnp).
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_lookup_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for open
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label associated with vp
+  @param acc_mode open(2) access mode
+
+  Determine whether the subject identified by the credential can perform
+  an open operation on the passed vnode with the passed access mode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_open_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	int acc_mode
+);
+/**
+  @brief Access control check for read
+  @param active_cred Subject credential
+  @param file_cred Credential associated with the struct fileproc
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can perform
+  a read operation on the passed vnode.  The active_cred hold the credentials
+  of the subject performing the operation, and file_cred holds the
+  credentials of the subject that originally opened the file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_read_t(
+	kauth_cred_t active_cred,	/* SUBJECT */
+	kauth_cred_t file_cred,	/* NULLOK */
+	struct vnode *vp,		/* OBJECT */
+	struct label *label		/* LABEL */
+);
+/**
+  @brief Access control check for read directory
+  @param cred Subject credential
+  @param dvp Object directory vnode
+  @param dlabel Policy label for dvp
+
+  Determine whether the subject identified by the credential can
+  perform a readdir operation on the passed directory vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_readdir_t(
+	kauth_cred_t cred,		/* SUBJECT */
+	struct vnode *dvp,		/* OBJECT */
+	struct label *dlabel		/* LABEL */
+);
+/**
+  @brief Access control check for read link
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can perform
+  a readlink operation on the passed symlink vnode.  This call can be made
+  in a number of situations, including an explicit readlink call by the
+  user process, or as a result of an implicit readlink during a name
+  lookup by the process.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_readlink_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for rename from
+  @param cred Subject credential
+  @param dvp Directory vnode
+  @param dlabel Policy label associated with dvp
+  @param vp vnode to be renamed
+  @param label Policy label associated with vp
+  @param cnp Component name for vp
+  @see mpo_vnode_check_rename_to_t
+
+  Determine whether the subject identified by the credential should be
+  allowed to rename the vnode vp to something else.
+
+  Due to VFS locking constraints (to make sure proper vnode locks are
+  held during this entry point), the vnode relabel checks had to be
+  split into two parts: relabel_from and relabel to.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_rename_from_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct vnode *vp,
+	struct label *label,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for rename to
+  @param cred Subject credential
+  @param dvp Directory vnode
+  @param dlabel Policy label associated with dvp
+  @param vp Overwritten vnode
+  @param label Policy label associated with vp
+  @param samedir Boolean; 1 if the source and destination directories are the same
+  @param cnp Destination component name
+  @see mpo_vnode_check_rename_from_t
+
+  Determine whether the subject identified by the credential should be
+  allowed to rename to the vnode vp, into the directory dvp, or to the
+  name represented by cnp. If there is no existing file to overwrite,
+  vp and label will be NULL.
+
+  Due to VFS locking constraints (to make sure proper vnode locks are
+  held during this entry point), the vnode relabel checks had to be
+  split into two parts: relabel_from and relabel to.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_rename_to_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct vnode *vp,			/* NULLOK */
+	struct label *label,			/* NULLOK */
+	int samedir,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for revoke
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can revoke
+  access to the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_revoke_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for select
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param which The operation selected on: FREAD or FWRITE
+
+  Determine whether the subject identified by the credential can select
+  the vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned.
+*/
+typedef int mpo_vnode_check_select_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	int which
+);
+/**
+  @brief Access control check for setting file attributes
+  @param cred Subject credential
+  @param vp Object vnode
+  @param vlabel Policy label for vp
+  @param alist List of attributes to set
+
+  Determine whether the subject identified by the credential can set
+  various attributes of the specified vnode, or the filesystem or volume on
+  which that vnode resides. See <sys/attr.h> for definitions of the
+  attributes.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege. Access control covers all attributes requested
+  with this call.
+*/
+typedef int mpo_vnode_check_setattrlist_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel,
+	struct attrlist *alist
+);
+/**
+  @brief Access control check for setting extended attribute
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param name Extended attribute name
+  @param uio I/O structure pointer
+
+  Determine whether the subject identified by the credential can set the
+  extended attribute of passed name and passed namespace on the passed
+  vnode. Policies implementing security labels backed into extended
+  attributes may want to provide additional protections for those
+  attributes. Additionally, policies should avoid making decisions based
+  on the data referenced from uio, as there is a potential race condition
+  between this check and the actual operation. The uio may also be NULL
+  if a delete operation is being performed.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_setextattr_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	const char *name,
+	struct uio *uio
+);
+/**
+  @brief Access control check for setting flags
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param flags File flags; see chflags(2)
+
+  Determine whether the subject identified by the credential can set
+  the passed flags on the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_setflags_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	u_long flags
+);
+/**
+  @brief Access control check for setting mode
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param mode File mode; see chmod(2)
+
+  Determine whether the subject identified by the credential can set
+  the passed mode on the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_setmode_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	mode_t mode
+);
+/**
+  @brief Access control check for setting uid and gid
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param uid User ID
+  @param gid Group ID
+
+  Determine whether the subject identified by the credential can set
+  the passed uid and passed gid as file uid and file gid on the passed
+  vnode. The IDs may be set to (-1) to request no update.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_setowner_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	uid_t uid,
+	gid_t gid
+);
+/**
+  @brief Access control check for setting timestamps
+  @param cred Subject credential
+  @param vp Object vnode
+  @param label Policy label for vp
+  @param atime Access time; see utimes(2)
+  @param mtime Modification time; see utimes(2)
+
+  Determine whether the subject identified by the credential can set
+  the passed access timestamps on the passed vnode.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_setutimes_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *label,
+	struct timespec atime,
+	struct timespec mtime
+);
+/**
+  @brief Access control check for stat
+  @param active_cred Subject credential
+  @param file_cred Credential associated with the struct fileproc
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can stat
+  the passed vnode. See stat(2) for more information.  The active_cred
+  hold the credentials of the subject performing the operation, and
+  file_cred holds the credentials of the subject that originally
+  opened the file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_stat_t(
+	struct ucred *active_cred,
+	struct ucred *file_cred,	/* NULLOK */
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for truncate/ftruncate
+  @param active_cred Subject credential
+  @param file_cred Credential associated with the struct fileproc
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can
+  perform a truncate operation on the passed vnode.  The active_cred hold
+  the credentials of the subject performing the operation, and
+  file_cred holds the credentials of the subject that originally
+  opened the file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_truncate_t(
+	kauth_cred_t active_cred,
+	kauth_cred_t file_cred,	/* NULLOK */
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Access control check for deleting vnode
+  @param cred Subject credential
+  @param dvp Parent directory vnode
+  @param dlabel Policy label for dvp
+  @param vp Object vnode to delete
+  @param label Policy label for vp
+  @param cnp Component name for vp
+  @see mpo_check_rename_to_t
+
+  Determine whether the subject identified by the credential can delete
+  a vnode from the passed parent directory and passed name information.
+  This call may be made in a number of situations, including as a
+  results of calls to unlink(2) and rmdir(2). Policies implementing
+  this entry point should also implement mpo_check_rename_to to
+  authorize deletion of objects as a result of being the target of a rename.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_unlink_t(
+	kauth_cred_t cred,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct vnode *vp,
+	struct label *label,
+	struct componentname *cnp
+);
+/**
+  @brief Access control check for write
+  @param active_cred Subject credential
+  @param file_cred Credential associated with the struct fileproc
+  @param vp Object vnode
+  @param label Policy label for vp
+
+  Determine whether the subject identified by the credential can
+  perform a write operation on the passed vnode.  The active_cred hold
+  the credentials of the subject performing the operation, and
+  file_cred holds the credentials of the subject that originally
+  opened the file.
+
+  @return Return 0 if access is granted, otherwise an appropriate value for
+  errno should be returned. Suggested failure: EACCES for label mismatch or
+  EPERM for lack of privilege.
+*/
+typedef int mpo_vnode_check_write_t(
+	kauth_cred_t active_cred,
+	kauth_cred_t file_cred,	/* NULLOK */
+	struct vnode *vp,
+	struct label *label
+);
+/**
+  @brief Associate a vnode with a devfs entry
+  @param mp Devfs mount point
+  @param mntlabel Devfs mount point label
+  @param de Devfs directory entry
+  @param delabel Label associated with de
+  @param vp vnode associated with de
+  @param vlabel Label associated with vp
+
+  Fill in the label (vlabel) for a newly created devfs vnode.  The
+  label is typically derived from the label on the devfs directory
+  entry or the label on the filesystem, supplied as parameters.
+*/
+typedef void mpo_vnode_label_associate_devfs_t(
+	struct mount *mp,
+	struct label *mntlabel,
+	struct devnode *de,
+	struct label *delabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a label with a vnode
+  @param mp File system mount point
+  @param mntlabel File system mount point label
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Attempt to retrieve label information for the vnode, vp, from the
+  file system extended attribute store.  The label should be stored in
+  the supplied vlabel parameter.  If a policy cannot retrieve an
+  extended attribute, sometimes it is acceptible to fallback to using
+  the mntlabel.
+
+  If the policy requires vnodes to have a valid label elsewhere it
+  MUST NOT return other than temporary errors, and must always provide
+  a valid label of some sort.  Returning an error will cause vnode
+  labeling to be retried at a later access.  Failure to handle policy
+  centric errors internally (corrupt labels etc.) will result in
+  inaccessible files.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+*/
+typedef int mpo_vnode_label_associate_extattr_t(
+	struct mount *mp,
+	struct label *mntlabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a file label with a vnode
+  @param cred User credential
+  @param mp Fdesc mount point
+  @param mntlabel Fdesc mount point label
+  @param fg Fileglob structure
+  @param label Policy label for fg
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Associate label information for the vnode, vp, with the label of
+  the open file descriptor described by fg.
+  The label should be stored in the supplied vlabel parameter.
+*/
+typedef void mpo_vnode_label_associate_file_t(
+	struct ucred *cred,
+	struct mount *mp,
+	struct label *mntlabel,
+	struct fileglob *fg,
+	struct label *label,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a pipe label with a vnode
+  @param cred User credential for the process that opened the pipe
+  @param cpipe Pipe structure
+  @param pipelabel Label associated with pipe
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Associate label information for the vnode, vp, with the label of
+  the pipe described by the pipe structure cpipe.
+  The label should be stored in the supplied vlabel parameter.
+*/
+typedef void mpo_vnode_label_associate_pipe_t(
+	struct ucred *cred,
+	struct pipe *cpipe,
+	struct label *pipelabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a POSIX semaphore label with a vnode
+  @param cred User credential for the process that create psem
+  @param psem POSIX semaphore structure
+  @param psemlabel Label associated with psem
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Associate label information for the vnode, vp, with the label of
+  the POSIX semaphore described by psem.
+  The label should be stored in the supplied vlabel parameter.
+*/
+typedef void mpo_vnode_label_associate_posixsem_t(
+	struct ucred *cred,
+	struct pseminfo *psem,
+	struct label *psemlabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a POSIX shared memory label with a vnode
+  @param cred User credential for the process that created pshm
+  @param pshm POSIX shared memory structure
+  @param pshmlabel Label associated with pshm
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Associate label information for the vnode, vp, with the label of
+  the POSIX shared memory region described by pshm.
+  The label should be stored in the supplied vlabel parameter.
+*/
+typedef void mpo_vnode_label_associate_posixshm_t(
+	struct ucred *cred,
+	struct pshminfo *pshm,
+	struct label *pshmlabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a label with a vnode
+  @param mp File system mount point
+  @param mntlabel File system mount point label
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  On non-multilabel file systems, set the label for a vnode.  The
+  label will most likely be based on the file system label.
+*/
+typedef void mpo_vnode_label_associate_singlelabel_t(
+	struct mount *mp,
+	struct label *mntlabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Associate a socket label with a vnode
+  @param cred User credential for the process that opened the socket
+  @param so Socket structure
+  @param solabel Label associated with so
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+
+  Associate label information for the vnode, vp, with the label of
+  the open socket described by the socket structure so.
+  The label should be stored in the supplied vlabel parameter.
+*/
+typedef void mpo_vnode_label_associate_socket_t(
+	kauth_cred_t cred,
+	socket_t so,
+	struct label *solabel,
+	struct vnode *vp,
+	struct label *vlabel
+);
+/**
+  @brief Copy a vnode label
+  @param src Source vnode label
+  @param dest Destination vnode label
+
+  Copy the vnode label information from src to dest.  On Darwin, this
+  is currently only necessary when executing interpreted scripts, but
+  will later be used if vnode label externalization cannot be an
+  atomic operation.
+*/
+typedef void mpo_vnode_label_copy_t(
+	struct label *src,
+	struct label *dest
+);
+/**
+  @brief Destroy vnode label
+  @param label The label to be destroyed
+
+  Destroy a vnode label.  Since the object is going out of scope,
+  policy modules should free any internal storage associated with the
+  label so that it may be destroyed.
+*/
+typedef void mpo_vnode_label_destroy_t(
+	struct label *label
+);
+/**
+  @brief Externalize a vnode label for auditing
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a vnode suitable for
+  inclusion in an audit record.  An externalized label consists of a text
+  representation of the label contents that will be added to the audit record
+  as part of a text token.  Policy-agnostic user space tools will display
+  this externalized version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_vnode_label_externalize_audit_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Externalize a vnode label
+  @param label Label to be externalized
+  @param element_name Name of the label namespace for which labels should be
+  externalized
+  @param sb String buffer to be filled with a text representation of the label
+
+  Produce an external representation of the label on a vnode.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will display this externalized version.
+
+  @return 0 on success, return non-zero if an error occurs while
+  externalizing the label data.
+
+*/
+typedef int mpo_vnode_label_externalize_t(
+	struct label *label,
+	char *element_name,
+	struct sbuf *sb
+);
+/**
+  @brief Initialize vnode label
+  @param label New label to initialize
+
+  Initialize label storage for use with a newly instantiated vnode, or
+  for temporary storage associated with the copying in or out of a
+  vnode label.  While it is necessary to allocate space for a
+  kernel-resident vnode label, it is not yet necessary to link this vnode
+  with persistent label storage facilities, such as extended attributes.
+  Sleeping is permitted.
+*/
+typedef void mpo_vnode_label_init_t(
+	struct label *label
+);
+/**
+  @brief Internalize a vnode label
+  @param label Label to be internalized
+  @param element_name Name of the label namespace for which the label should
+  be internalized
+  @param element_data Text data to be internalized
+
+  Produce a vnode label from an external representation.  An
+  externalized label consists of a text representation of the label
+  contents that can be used with user applications.  Policy-agnostic
+  user space tools will forward text version to the kernel for
+  processing by individual policy modules.
+
+  The policy's internalize entry points will be called only if the
+  policy has registered interest in the label namespace.
+
+  @return 0 on success, Otherwise, return non-zero if an error occurs
+  while internalizing the label data.
+*/
+typedef int mpo_vnode_label_internalize_t(
+	struct label *label,
+	char *element_name,
+	char *element_data
+);
+/**
+  @brief Clean up a vnode label
+  @param label The label to be cleaned for re-use
+
+  Clean up a vnode label.  Darwin (Tiger, 8.x) allocates vnodes on demand, but
+  typically never frees them.  Before vnodes are placed back on free lists for
+  re-use, policies can cleanup or overwrite any information present in the label.
+*/
+typedef void mpo_vnode_label_recycle_t(
+	struct label *label
+);
+/**
+  @brief Write a label to a extended attribute
+  @param cred Subject credential
+  @param vp The vnode for which the label is being stored
+  @param vlabel Label associated with vp
+  @param intlabel The new label to store
+
+  Store a new label in the extended attribute corresponding to the
+  supplied vnode.  The policy has already authorized the operation;
+  this call must be implemented in order to perform the actual
+  operation.
+
+  @return In the event of an error, an appropriate value for errno
+  should be returned, otherwise return 0 upon success.
+
+  @warning XXX After examining the extended attribute implementation on
+  Apple's future release, this entry point may be changed.
+*/
+typedef int mpo_vnode_label_store_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vlabel,
+	struct label *intlabel
+);
+/**
+  @brief Update vnode label from extended attributes
+  @param mp File system mount point
+  @param mntlabel Mount point label
+  @param vp Vnode to label
+  @param vlabel Label associated with vp
+  @param name Name of the xattr
+  @see mpo_vnode_check_setextattr_t
+
+  When an extended attribute is updated via the Vendor attribute management
+  functions, the MAC vnode label might also require an update.
+  Policies should first determine if 'name' matches their xattr label
+  name.  If it does, the kernel is has either replaced or removed the
+  named extended attribute that was previously associated with the
+  vnode.  Normally labels should only be modified via MAC Framework label
+  management calls, but sometimes the user space components will directly
+  modify extended attributes.  For example, 'cp', 'tar', etc. manage
+  extended attributes in userspace, not the kernel.
+
+  This entry point is called after the label update has occurred, so
+  it cannot return a failure.  However, the operation is preceded by
+  the mpo_vnode_check_setextattr() access control check.
+
+  If the vnode label needs to be updated the policy should return
+  a non-zero value.  The vnode label will be marked for re-association
+  by the framework.
+*/
+typedef int mpo_vnode_label_update_extattr_t(
+	struct mount *mp,
+	struct label *mntlabel,
+	struct vnode *vp,
+	struct label *vlabel,
+	const char *name
+);
+/**
+  @brief Update a vnode label
+  @param cred Subject credential
+  @param vp The vnode to relabel
+  @param vnodelabel Existing vnode label
+  @param label New label to replace existing label
+  @see mpo_vnode_check_label_update_t
+
+  The subject identified by the credential has previously requested
+  and was authorized to relabel the vnode; this entry point allows
+  policies to perform the actual relabel operation.  Policies should
+  update vnodelabel using the label stored in the label parameter.
+*/
+typedef void mpo_vnode_label_update_t(
+	kauth_cred_t cred,
+	struct vnode *vp,
+	struct label *vnodelabel,
+	struct label *label
+);
+/**
+  @brief Create a new vnode, backed by extended attributes
+  @param cred User credential for the creating process
+  @param mp File system mount point
+  @param mntlabel File system mount point label
+  @param dvp Parent directory vnode
+  @param dlabel Parent directory vnode label
+  @param vp Newly created vnode
+  @param vlabel Label to associate with the new vnode
+  @param cnp Component name for vp
+
+  Write out the label for the newly created vnode, most likely storing
+  the results in a file system extended attribute.  Most policies will
+  derive the new vnode label using information from a combination
+  of the subject (user) credential, the file system label, the parent
+  directory label, and potentially the path name component.
+
+  @return If the operation succeeds, store the new label in vlabel and
+  return 0.  Otherwise, return an appropriate errno value.
+*/
+typedef int mpo_vnode_notify_create_t(
+	kauth_cred_t cred,
+	struct mount *mp,
+	struct label *mntlabel,
+	struct vnode *dvp,
+	struct label *dlabel,
+	struct vnode *vp,
+	struct label *vlabel,
+	struct componentname *cnp
+);
+
+/*
+ * Placeholder for future events that may need mac hooks.
+ */
+typedef void mpo_reserved_hook_t(void);
+
+/*!
+  \struct mac_policy_ops
+*/
+struct mac_policy_ops {
+	mpo_audit_check_postselect_t		*mpo_audit_check_postselect;
+	mpo_audit_check_preselect_t		*mpo_audit_check_preselect;
+	mpo_bpfdesc_label_associate_t		*mpo_bpfdesc_label_associate;
+	mpo_bpfdesc_label_destroy_t		*mpo_bpfdesc_label_destroy;
+	mpo_bpfdesc_label_init_t		*mpo_bpfdesc_label_init;
+	mpo_bpfdesc_check_receive_t		*mpo_bpfdesc_check_receive;
+	mpo_cred_check_label_update_execve_t	*mpo_cred_check_label_update_execve;
+	mpo_cred_check_label_update_t		*mpo_cred_check_label_update;
+	mpo_cred_check_visible_t		*mpo_cred_check_visible;
+	mpo_cred_label_associate_fork_t		*mpo_cred_label_associate_fork;
+	mpo_cred_label_associate_kernel_t	*mpo_cred_label_associate_kernel;
+	mpo_cred_label_associate_t		*mpo_cred_label_associate;
+	mpo_cred_label_associate_user_t		*mpo_cred_label_associate_user;
+	mpo_cred_label_destroy_t		*mpo_cred_label_destroy;
+	mpo_cred_label_externalize_audit_t	*mpo_cred_label_externalize_audit;
+	mpo_cred_label_externalize_t		*mpo_cred_label_externalize;
+	mpo_cred_label_init_t			*mpo_cred_label_init;
+	mpo_cred_label_internalize_t		*mpo_cred_label_internalize;
+	mpo_cred_label_update_execve_t		*mpo_cred_label_update_execve;
+	mpo_cred_label_update_t			*mpo_cred_label_update;
+	mpo_devfs_label_associate_device_t	*mpo_devfs_label_associate_device;
+	mpo_devfs_label_associate_directory_t	*mpo_devfs_label_associate_directory;
+	mpo_devfs_label_copy_t			*mpo_devfs_label_copy;
+	mpo_devfs_label_destroy_t		*mpo_devfs_label_destroy;
+	mpo_devfs_label_init_t			*mpo_devfs_label_init;
+	mpo_devfs_label_update_t		*mpo_devfs_label_update;
+	mpo_file_check_change_offset_t		*mpo_file_check_change_offset;
+	mpo_file_check_create_t			*mpo_file_check_create;
+	mpo_file_check_dup_t			*mpo_file_check_dup;
+	mpo_file_check_fcntl_t			*mpo_file_check_fcntl;
+	mpo_file_check_get_offset_t		*mpo_file_check_get_offset;
+	mpo_file_check_get_t			*mpo_file_check_get;
+	mpo_file_check_inherit_t		*mpo_file_check_inherit;
+	mpo_file_check_ioctl_t			*mpo_file_check_ioctl;
+	mpo_file_check_lock_t			*mpo_file_check_lock;
+	mpo_file_check_mmap_downgrade_t		*mpo_file_check_mmap_downgrade;
+	mpo_file_check_mmap_t			*mpo_file_check_mmap;
+	mpo_file_check_receive_t		*mpo_file_check_receive;
+	mpo_file_check_set_t			*mpo_file_check_set;
+	mpo_file_label_init_t			*mpo_file_label_init;
+	mpo_file_label_destroy_t		*mpo_file_label_destroy;
+	mpo_file_label_associate_t		*mpo_file_label_associate;
+	mpo_ifnet_check_label_update_t		*mpo_ifnet_check_label_update;
+	mpo_ifnet_check_transmit_t		*mpo_ifnet_check_transmit;
+	mpo_ifnet_label_associate_t		*mpo_ifnet_label_associate;
+	mpo_ifnet_label_copy_t			*mpo_ifnet_label_copy;
+	mpo_ifnet_label_destroy_t		*mpo_ifnet_label_destroy;
+	mpo_ifnet_label_externalize_t		*mpo_ifnet_label_externalize;
+	mpo_ifnet_label_init_t			*mpo_ifnet_label_init;
+	mpo_ifnet_label_internalize_t		*mpo_ifnet_label_internalize;
+	mpo_ifnet_label_update_t		*mpo_ifnet_label_update;
+	mpo_ifnet_label_recycle_t		*mpo_ifnet_label_recycle;
+	mpo_inpcb_check_deliver_t		*mpo_inpcb_check_deliver;
+	mpo_inpcb_label_associate_t		*mpo_inpcb_label_associate;
+	mpo_inpcb_label_destroy_t		*mpo_inpcb_label_destroy;
+	mpo_inpcb_label_init_t			*mpo_inpcb_label_init;
+	mpo_inpcb_label_recycle_t		*mpo_inpcb_label_recycle;
+	mpo_inpcb_label_update_t		*mpo_inpcb_label_update;
+	mpo_iokit_check_device_t		*mpo_iokit_check_device;
+	mpo_ipq_label_associate_t		*mpo_ipq_label_associate;
+	mpo_ipq_label_compare_t			*mpo_ipq_label_compare;
+	mpo_ipq_label_destroy_t			*mpo_ipq_label_destroy;
+	mpo_ipq_label_init_t			*mpo_ipq_label_init;
+	mpo_ipq_label_update_t			*mpo_ipq_label_update;
+	mpo_lctx_check_label_update_t		*mpo_lctx_check_label_update;
+	mpo_lctx_label_destroy_t		*mpo_lctx_label_destroy;
+	mpo_lctx_label_externalize_t		*mpo_lctx_label_externalize;
+	mpo_lctx_label_init_t			*mpo_lctx_label_init;
+	mpo_lctx_label_internalize_t		*mpo_lctx_label_internalize;
+	mpo_lctx_label_update_t			*mpo_lctx_label_update;
+	mpo_lctx_notify_create_t		*mpo_lctx_notify_create;
+	mpo_lctx_notify_join_t			*mpo_lctx_notify_join;
+	mpo_lctx_notify_leave_t			*mpo_lctx_notify_leave;
+	mpo_mbuf_label_associate_bpfdesc_t	*mpo_mbuf_label_associate_bpfdesc;
+	mpo_mbuf_label_associate_ifnet_t	*mpo_mbuf_label_associate_ifnet;
+	mpo_mbuf_label_associate_inpcb_t	*mpo_mbuf_label_associate_inpcb;
+	mpo_mbuf_label_associate_ipq_t		*mpo_mbuf_label_associate_ipq;
+	mpo_mbuf_label_associate_linklayer_t	*mpo_mbuf_label_associate_linklayer;
+	mpo_mbuf_label_associate_multicast_encap_t *mpo_mbuf_label_associate_multicast_encap;
+	mpo_mbuf_label_associate_netlayer_t	*mpo_mbuf_label_associate_netlayer;
+	mpo_mbuf_label_associate_socket_t	*mpo_mbuf_label_associate_socket;
+	mpo_mbuf_label_copy_t			*mpo_mbuf_label_copy;
+	mpo_mbuf_label_destroy_t		*mpo_mbuf_label_destroy;
+	mpo_mbuf_label_init_t			*mpo_mbuf_label_init;
+	mpo_mount_check_fsctl_t			*mpo_mount_check_fsctl;
+	mpo_mount_check_getattr_t		*mpo_mount_check_getattr;
+	mpo_mount_check_label_update_t		*mpo_mount_check_label_update;
+	mpo_mount_check_mount_t			*mpo_mount_check_mount;
+	mpo_mount_check_remount_t		*mpo_mount_check_remount;
+	mpo_mount_check_setattr_t		*mpo_mount_check_setattr;
+	mpo_mount_check_stat_t			*mpo_mount_check_stat;
+	mpo_mount_check_umount_t		*mpo_mount_check_umount;
+	mpo_mount_label_associate_t		*mpo_mount_label_associate;
+	mpo_mount_label_destroy_t		*mpo_mount_label_destroy;
+	mpo_mount_label_externalize_t		*mpo_mount_label_externalize;
+	mpo_mount_label_init_t			*mpo_mount_label_init;
+	mpo_mount_label_internalize_t		*mpo_mount_label_internalize;
+	mpo_netinet_fragment_t			*mpo_netinet_fragment;
+	mpo_netinet_icmp_reply_t		*mpo_netinet_icmp_reply;
+	mpo_netinet_tcp_reply_t			*mpo_netinet_tcp_reply;
+	mpo_pipe_check_ioctl_t			*mpo_pipe_check_ioctl;
+	mpo_pipe_check_kqfilter_t		*mpo_pipe_check_kqfilter;
+	mpo_pipe_check_label_update_t		*mpo_pipe_check_label_update;
+	mpo_pipe_check_read_t			*mpo_pipe_check_read;
+	mpo_pipe_check_select_t			*mpo_pipe_check_select;
+	mpo_pipe_check_stat_t			*mpo_pipe_check_stat;
+	mpo_pipe_check_write_t			*mpo_pipe_check_write;
+	mpo_pipe_label_associate_t		*mpo_pipe_label_associate;
+	mpo_pipe_label_copy_t			*mpo_pipe_label_copy;
+	mpo_pipe_label_destroy_t		*mpo_pipe_label_destroy;
+	mpo_pipe_label_externalize_t		*mpo_pipe_label_externalize;
+	mpo_pipe_label_init_t			*mpo_pipe_label_init;
+	mpo_pipe_label_internalize_t		*mpo_pipe_label_internalize;
+	mpo_pipe_label_update_t			*mpo_pipe_label_update;
+	mpo_policy_destroy_t			*mpo_policy_destroy;
+	mpo_policy_init_t			*mpo_policy_init;
+	mpo_policy_initbsd_t			*mpo_policy_initbsd;
+	mpo_policy_syscall_t			*mpo_policy_syscall;
+	mpo_port_check_copy_send_t		*mpo_port_check_copy_send;
+	mpo_port_check_hold_receive_t		*mpo_port_check_hold_receive;
+	mpo_port_check_hold_send_once_t		*mpo_port_check_hold_send_once;
+	mpo_port_check_hold_send_t		*mpo_port_check_hold_send;
+	mpo_port_check_label_update_t		*mpo_port_check_label_update;
+	mpo_port_check_make_send_once_t		*mpo_port_check_make_send_once;
+	mpo_port_check_make_send_t		*mpo_port_check_make_send;
+	mpo_port_check_method_t			*mpo_port_check_method;
+	mpo_port_check_move_receive_t		*mpo_port_check_move_receive;
+	mpo_port_check_move_send_once_t		*mpo_port_check_move_send_once;
+	mpo_port_check_move_send_t		*mpo_port_check_move_send;
+	mpo_port_check_receive_t		*mpo_port_check_receive;
+	mpo_port_check_send_t			*mpo_port_check_send;
+	mpo_port_check_service_t		*mpo_port_check_service;
+	mpo_port_label_associate_kernel_t	*mpo_port_label_associate_kernel;
+	mpo_port_label_associate_t		*mpo_port_label_associate;
+	mpo_port_label_compute_t		*mpo_port_label_compute;
+	mpo_port_label_copy_t			*mpo_port_label_copy;
+	mpo_port_label_destroy_t		*mpo_port_label_destroy;
+	mpo_port_label_init_t			*mpo_port_label_init;
+	mpo_port_label_update_cred_t		*mpo_port_label_update_cred;
+	mpo_port_label_update_kobject_t		*mpo_port_label_update_kobject;
+	mpo_posixsem_check_create_t		*mpo_posixsem_check_create;
+	mpo_posixsem_check_open_t		*mpo_posixsem_check_open;
+	mpo_posixsem_check_post_t		*mpo_posixsem_check_post;
+	mpo_posixsem_check_unlink_t		*mpo_posixsem_check_unlink;
+	mpo_posixsem_check_wait_t		*mpo_posixsem_check_wait;
+	mpo_posixsem_label_associate_t		*mpo_posixsem_label_associate;
+	mpo_posixsem_label_destroy_t		*mpo_posixsem_label_destroy;
+	mpo_posixsem_label_init_t		*mpo_posixsem_label_init;
+	mpo_posixshm_check_create_t		*mpo_posixshm_check_create;
+	mpo_posixshm_check_mmap_t		*mpo_posixshm_check_mmap;
+	mpo_posixshm_check_open_t		*mpo_posixshm_check_open;
+	mpo_posixshm_check_stat_t		*mpo_posixshm_check_stat;
+	mpo_posixshm_check_truncate_t		*mpo_posixshm_check_truncate;
+	mpo_posixshm_check_unlink_t		*mpo_posixshm_check_unlink;
+	mpo_posixshm_label_associate_t		*mpo_posixshm_label_associate;
+	mpo_posixshm_label_destroy_t		*mpo_posixshm_label_destroy;
+	mpo_posixshm_label_init_t		*mpo_posixshm_label_init;
+	mpo_proc_check_debug_t			*mpo_proc_check_debug;
+	mpo_proc_check_fork_t			*mpo_proc_check_fork;
+	mpo_proc_check_get_task_name_t		*mpo_proc_check_get_task_name;
+	mpo_proc_check_get_task_t		*mpo_proc_check_get_task;
+	mpo_proc_check_getaudit_t		*mpo_proc_check_getaudit;
+	mpo_proc_check_getauid_t		*mpo_proc_check_getauid;
+	mpo_proc_check_getlcid_t		*mpo_proc_check_getlcid;
+	mpo_proc_check_mprotect_t		*mpo_proc_check_mprotect;
+	mpo_proc_check_sched_t			*mpo_proc_check_sched;
+	mpo_proc_check_setaudit_t		*mpo_proc_check_setaudit;
+	mpo_proc_check_setauid_t		*mpo_proc_check_setauid;
+	mpo_proc_check_setlcid_t		*mpo_proc_check_setlcid;
+	mpo_proc_check_signal_t			*mpo_proc_check_signal;
+	mpo_proc_check_wait_t			*mpo_proc_check_wait;
+	mpo_proc_label_destroy_t		*mpo_proc_label_destroy;
+	mpo_proc_label_init_t			*mpo_proc_label_init;
+	mpo_socket_check_accept_t		*mpo_socket_check_accept;
+	mpo_socket_check_accepted_t		*mpo_socket_check_accepted;
+	mpo_socket_check_bind_t			*mpo_socket_check_bind;
+	mpo_socket_check_connect_t		*mpo_socket_check_connect;
+	mpo_socket_check_create_t		*mpo_socket_check_create;
+	mpo_socket_check_deliver_t		*mpo_socket_check_deliver;
+	mpo_socket_check_kqfilter_t		*mpo_socket_check_kqfilter;
+	mpo_socket_check_label_update_t		*mpo_socket_check_label_update;
+	mpo_socket_check_listen_t		*mpo_socket_check_listen;
+	mpo_socket_check_receive_t		*mpo_socket_check_receive;
+	mpo_socket_check_received_t		*mpo_socket_check_received;
+	mpo_socket_check_select_t		*mpo_socket_check_select;
+	mpo_socket_check_send_t			*mpo_socket_check_send;
+	mpo_socket_check_stat_t			*mpo_socket_check_stat;
+	mpo_socket_check_setsockopt_t		*mpo_socket_check_setsockopt;
+	mpo_socket_check_getsockopt_t		*mpo_socket_check_getsockopt;
+	mpo_socket_label_associate_accept_t	*mpo_socket_label_associate_accept;
+	mpo_socket_label_associate_t		*mpo_socket_label_associate;
+	mpo_socket_label_copy_t			*mpo_socket_label_copy;
+	mpo_socket_label_destroy_t		*mpo_socket_label_destroy;
+	mpo_socket_label_externalize_t		*mpo_socket_label_externalize;
+	mpo_socket_label_init_t			*mpo_socket_label_init;
+	mpo_socket_label_internalize_t		*mpo_socket_label_internalize;
+	mpo_socket_label_update_t		*mpo_socket_label_update;
+	mpo_socketpeer_label_associate_mbuf_t	*mpo_socketpeer_label_associate_mbuf;
+	mpo_socketpeer_label_associate_socket_t	*mpo_socketpeer_label_associate_socket;
+	mpo_socketpeer_label_destroy_t		*mpo_socketpeer_label_destroy;
+	mpo_socketpeer_label_externalize_t	*mpo_socketpeer_label_externalize;
+	mpo_socketpeer_label_init_t		*mpo_socketpeer_label_init;
+	mpo_system_check_acct_t			*mpo_system_check_acct;
+	mpo_system_check_audit_t		*mpo_system_check_audit;
+	mpo_system_check_auditctl_t		*mpo_system_check_auditctl;
+	mpo_system_check_auditon_t		*mpo_system_check_auditon;
+	mpo_system_check_host_priv_t		*mpo_system_check_host_priv;
+	mpo_system_check_nfsd_t			*mpo_system_check_nfsd;
+	mpo_system_check_reboot_t		*mpo_system_check_reboot;
+	mpo_system_check_settime_t		*mpo_system_check_settime;
+	mpo_system_check_swapoff_t		*mpo_system_check_swapoff;
+	mpo_system_check_swapon_t		*mpo_system_check_swapon;
+	mpo_system_check_sysctl_t		*mpo_system_check_sysctl;
+	mpo_sysvmsg_label_associate_t		*mpo_sysvmsg_label_associate;
+	mpo_sysvmsg_label_destroy_t		*mpo_sysvmsg_label_destroy;
+	mpo_sysvmsg_label_init_t		*mpo_sysvmsg_label_init;
+	mpo_sysvmsg_label_recycle_t		*mpo_sysvmsg_label_recycle;
+	mpo_sysvmsq_check_enqueue_t		*mpo_sysvmsq_check_enqueue;
+	mpo_sysvmsq_check_msgrcv_t		*mpo_sysvmsq_check_msgrcv;
+	mpo_sysvmsq_check_msgrmid_t		*mpo_sysvmsq_check_msgrmid;
+	mpo_sysvmsq_check_msqctl_t		*mpo_sysvmsq_check_msqctl;
+	mpo_sysvmsq_check_msqget_t		*mpo_sysvmsq_check_msqget;
+	mpo_sysvmsq_check_msqrcv_t		*mpo_sysvmsq_check_msqrcv;
+	mpo_sysvmsq_check_msqsnd_t		*mpo_sysvmsq_check_msqsnd;
+	mpo_sysvmsq_label_associate_t		*mpo_sysvmsq_label_associate;
+	mpo_sysvmsq_label_destroy_t		*mpo_sysvmsq_label_destroy;
+	mpo_sysvmsq_label_init_t		*mpo_sysvmsq_label_init;
+	mpo_sysvmsq_label_recycle_t		*mpo_sysvmsq_label_recycle;
+	mpo_sysvsem_check_semctl_t		*mpo_sysvsem_check_semctl;
+	mpo_sysvsem_check_semget_t		*mpo_sysvsem_check_semget;
+	mpo_sysvsem_check_semop_t		*mpo_sysvsem_check_semop;
+	mpo_sysvsem_label_associate_t		*mpo_sysvsem_label_associate;
+	mpo_sysvsem_label_destroy_t		*mpo_sysvsem_label_destroy;
+	mpo_sysvsem_label_init_t		*mpo_sysvsem_label_init;
+	mpo_sysvsem_label_recycle_t		*mpo_sysvsem_label_recycle;
+	mpo_sysvshm_check_shmat_t		*mpo_sysvshm_check_shmat;
+	mpo_sysvshm_check_shmctl_t		*mpo_sysvshm_check_shmctl;
+	mpo_sysvshm_check_shmdt_t		*mpo_sysvshm_check_shmdt;
+	mpo_sysvshm_check_shmget_t		*mpo_sysvshm_check_shmget;
+	mpo_sysvshm_label_associate_t		*mpo_sysvshm_label_associate;
+	mpo_sysvshm_label_destroy_t		*mpo_sysvshm_label_destroy;
+	mpo_sysvshm_label_init_t		*mpo_sysvshm_label_init;
+	mpo_sysvshm_label_recycle_t		*mpo_sysvshm_label_recycle;
+	mpo_task_label_associate_kernel_t	*mpo_task_label_associate_kernel;
+	mpo_task_label_associate_t		*mpo_task_label_associate;
+	mpo_task_label_copy_t			*mpo_task_label_copy;
+	mpo_task_label_destroy_t		*mpo_task_label_destroy;
+	mpo_task_label_externalize_t		*mpo_task_label_externalize;
+	mpo_task_label_init_t			*mpo_task_label_init;
+	mpo_task_label_internalize_t		*mpo_task_label_internalize;
+	mpo_task_label_update_t			*mpo_task_label_update;
+	mpo_thread_userret_t			*mpo_thread_userret;
+	mpo_vnode_check_access_t		*mpo_vnode_check_access;
+	mpo_vnode_check_chdir_t			*mpo_vnode_check_chdir;
+	mpo_vnode_check_chroot_t		*mpo_vnode_check_chroot;
+	mpo_vnode_check_create_t		*mpo_vnode_check_create;
+	mpo_vnode_check_deleteextattr_t		*mpo_vnode_check_deleteextattr;
+	mpo_vnode_check_exchangedata_t		*mpo_vnode_check_exchangedata;
+	mpo_vnode_check_exec_t			*mpo_vnode_check_exec;
+	mpo_vnode_check_getattrlist_t		*mpo_vnode_check_getattrlist;
+	mpo_vnode_check_getextattr_t		*mpo_vnode_check_getextattr;
+	mpo_vnode_check_ioctl_t			*mpo_vnode_check_ioctl;
+	mpo_vnode_check_kqfilter_t		*mpo_vnode_check_kqfilter;
+	mpo_vnode_check_label_update_t		*mpo_vnode_check_label_update;
+	mpo_vnode_check_link_t			*mpo_vnode_check_link;
+	mpo_vnode_check_listextattr_t		*mpo_vnode_check_listextattr;
+	mpo_vnode_check_lookup_t		*mpo_vnode_check_lookup;
+	mpo_vnode_check_open_t			*mpo_vnode_check_open;
+	mpo_vnode_check_read_t			*mpo_vnode_check_read;
+	mpo_vnode_check_readdir_t		*mpo_vnode_check_readdir;
+	mpo_vnode_check_readlink_t		*mpo_vnode_check_readlink;
+	mpo_vnode_check_rename_from_t		*mpo_vnode_check_rename_from;
+	mpo_vnode_check_rename_to_t		*mpo_vnode_check_rename_to;
+	mpo_vnode_check_revoke_t		*mpo_vnode_check_revoke;
+	mpo_vnode_check_select_t		*mpo_vnode_check_select;
+	mpo_vnode_check_setattrlist_t		*mpo_vnode_check_setattrlist;
+	mpo_vnode_check_setextattr_t		*mpo_vnode_check_setextattr;
+	mpo_vnode_check_setflags_t		*mpo_vnode_check_setflags;
+	mpo_vnode_check_setmode_t		*mpo_vnode_check_setmode;
+	mpo_vnode_check_setowner_t		*mpo_vnode_check_setowner;
+	mpo_vnode_check_setutimes_t		*mpo_vnode_check_setutimes;
+	mpo_vnode_check_stat_t			*mpo_vnode_check_stat;
+	mpo_vnode_check_truncate_t		*mpo_vnode_check_truncate;
+	mpo_vnode_check_unlink_t		*mpo_vnode_check_unlink;
+	mpo_vnode_check_write_t			*mpo_vnode_check_write;
+	mpo_vnode_label_associate_devfs_t	*mpo_vnode_label_associate_devfs;
+	mpo_vnode_label_associate_extattr_t	*mpo_vnode_label_associate_extattr;
+	mpo_vnode_label_associate_file_t	*mpo_vnode_label_associate_file;
+	mpo_vnode_label_associate_pipe_t	*mpo_vnode_label_associate_pipe;
+	mpo_vnode_label_associate_posixsem_t	*mpo_vnode_label_associate_posixsem;
+	mpo_vnode_label_associate_posixshm_t	*mpo_vnode_label_associate_posixshm;
+	mpo_vnode_label_associate_singlelabel_t	*mpo_vnode_label_associate_singlelabel;
+	mpo_vnode_label_associate_socket_t	*mpo_vnode_label_associate_socket;
+	mpo_vnode_label_copy_t			*mpo_vnode_label_copy;
+	mpo_vnode_label_destroy_t		*mpo_vnode_label_destroy;
+	mpo_vnode_label_externalize_audit_t	*mpo_vnode_label_externalize_audit;
+	mpo_vnode_label_externalize_t		*mpo_vnode_label_externalize;
+	mpo_vnode_label_init_t			*mpo_vnode_label_init;
+	mpo_vnode_label_internalize_t		*mpo_vnode_label_internalize;
+	mpo_vnode_label_recycle_t		*mpo_vnode_label_recycle;
+	mpo_vnode_label_store_t			*mpo_vnode_label_store;
+	mpo_vnode_label_update_extattr_t	*mpo_vnode_label_update_extattr;
+	mpo_vnode_label_update_t		*mpo_vnode_label_update;
+	mpo_vnode_notify_create_t		*mpo_vnode_notify_create;
+	mpo_reserved_hook_t			*mpo_reserved0;
+	mpo_reserved_hook_t			*mpo_reserved1;
+	mpo_reserved_hook_t			*mpo_reserved2;
+	mpo_reserved_hook_t			*mpo_reserved3;
+	mpo_reserved_hook_t			*mpo_reserved4;
+	mpo_reserved_hook_t			*mpo_reserved5;
+	mpo_reserved_hook_t			*mpo_reserved6;
+	mpo_reserved_hook_t			*mpo_reserved7;
+	mpo_reserved_hook_t			*mpo_reserved8;
+	mpo_reserved_hook_t			*mpo_reserved9;
+};
+
+/**
+   @brief MAC policy handle type
+
+   The MAC handle is used to uniquely identify a loaded policy within
+   the MAC Framework.
+
+   A variable of this type is set by mac_policy_register().
+ */
+typedef unsigned int mac_policy_handle_t;
+
+#define mpc_t	struct mac_policy_conf *
+
+/**
+  @brief Mac policy configuration
+
+  This structure specifies the configuration information for a
+  MAC policy module.  A policy module developer must supply
+  a short unique policy name, a more descriptive full name, a list of label
+  namespaces and count, a pointer to the registered enty point operations,
+  any load time flags, and optionally, a pointer to a label slot identifier.
+
+  The Framework will update the runtime flags (mpc_runtime_flags) to
+  indicate that the module has been registered.
+
+  If the label slot identifier (mpc_field_off) is NULL, the Framework
+  will not provide label storage for the policy.  Otherwise, the
+  Framework will store the label location (slot) in this field.
+
+  The mpc_list field is used by the Framework and should not be
+  modified by policies.
+*/
+/* XXX - reorder these for better aligment on 64bit platforms */
+struct mac_policy_conf {
+	const char		*mpc_name;		/** policy name */
+	const char		*mpc_fullname;		/** full name */
+	const char		**mpc_labelnames;	/** managed label namespaces */
+	unsigned int		 mpc_labelname_count;	/** number of managed label namespaces */
+	struct mac_policy_ops	*mpc_ops;		/** operation vector */
+	int			 mpc_loadtime_flags;	/** load time flags */
+	int			*mpc_field_off;		/** label slot */
+	int			 mpc_runtime_flags;	/** run time flags */
+	mpc_t			 mpc_list;		/** List reference */
+	void			*mpc_data;		/** module data */
+};
+
+/**
+   @brief MAC policy module registration routine
+
+   This function is called to register a policy with the
+   MAC framework.  A policy module will typically call this from the
+   Darwin KEXT registration routine.
+ */
+int	mac_policy_register(struct mac_policy_conf *mpc,
+    mac_policy_handle_t *handlep, void *xd);
+
+/**
+   @brief MAC policy module de-registration routine
+
+   This function is called to de-register a policy with theD
+   MAC framework.  A policy module will typically call this from the
+   Darwin KEXT de-registration routine.
+ */
+int	mac_policy_unregister(mac_policy_handle_t handle);
+
+/*
+ * Framework entry points for the policies to add audit data.
+ */
+int	mac_audit_text(char *text, mac_policy_handle_t handle);
+
+/*
+ * Calls to assist with use of Apple XATTRs within policy modules.
+ */
+int	mac_vnop_setxattr(struct vnode *, const char *, char *, size_t);
+int	mac_vnop_getxattr(struct vnode *, const char *, char *, size_t,
+			  size_t *);
+int	mac_vnop_removexattr(struct vnode *, const char *);
+
+/*
+ * Arbitrary limit on how much data will be logged by the audit
+ * entry points above.
+ */
+#define	MAC_AUDIT_DATA_LIMIT	1024
+
+/*
+ * Values returned by mac_audit_{pre,post}select. To combine the responses
+ * of the security policies into a single decision,
+ * mac_audit_{pre,post}select() choose the greatest value returned.
+ */
+#define	MAC_AUDIT_DEFAULT	0	/* use system behavior */
+#define	MAC_AUDIT_NO		1	/* force not auditing this event */
+#define	MAC_AUDIT_YES		2	/* force auditing this event */
+
+//  \defgroup mpc_loadtime_flags Flags for the mpc_loadtime_flags field
+
+/**
+  @name Flags for the mpc_loadtime_flags field
+  @see mac_policy_conf
+
+  This is the complete list of flags that are supported by the
+  mpc_loadtime_flags field of the mac_policy_conf structure.  These
+  flags specify the load time behavior of MAC Framework policy
+  modules.
+*/
+
+/*@{*/
+
+/**
+  @brief Flag to indicate registration preference
+
+  This flag indicates that the policy module must be loaded and
+  initialized early in the boot process. If the flag is specified,
+  attempts to register the module following boot will be rejected. The
+  flag may be used by policies that require pervasive labeling of all
+  system objects, and cannot handle objects that have not been
+  properly initialized by the policy.
+ */
+#define	MPC_LOADTIME_FLAG_NOTLATE	0x00000001
+
+/**
+  @brief Flag to indicate unload preference
+
+  This flag indicates that the policy module may be unloaded. If this
+  flag is not set, then the policy framework will reject requests to
+  unload the module. This flag might be used by modules that allocate
+  label state and are unable to free that state at runtime, or for
+  modules that simply do not want to permit unload operations.
+*/
+#define	MPC_LOADTIME_FLAG_UNLOADOK	0x00000002
+
+/**
+  @brief Unsupported
+
+  XXX This flag is not yet supported.
+*/
+#define	MPC_LOADTIME_FLAG_LABELMBUFS	0x00000004
+
+/**
+  @brief Flag to indicate a base policy
+
+  This flag indicates that the policy module is a base policy. Only
+  one module can declare itself as base, otherwise the boot process
+  will be halted.
+ */
+#define	MPC_LOADTIME_BASE_POLICY	0x00000008
+
+/*@}*/
+
+/**
+  @brief Policy registration flag
+  @see mac_policy_conf
+
+  This flag indicates that the policy module has been successfully
+  registered with the TrustedBSD MAC Framework.  The Framework will
+  set this flag in the mpc_runtime_flags field of the policy's
+  mac_policy_conf structure after registering the policy.
+ */
+#define	MPC_RUNTIME_FLAG_REGISTERED	0x00000001
+
+/*
+ * Depends on POLICY_VER
+ */
+
+#ifndef POLICY_VER
+#define	POLICY_VER	1.0
+#endif
+
+#define	MAC_POLICY_SET(handle, mpops, mpname, mpfullname, lnames, lcount, slot, lflags, rflags) \
+	static struct mac_policy_conf mpname##_mac_policy_conf = {	\
+		.mpc_name		= #mpname,			\
+		.mpc_fullname		= mpfullname,			\
+		.mpc_labelnames		= lnames,			\
+		.mpc_labelname_count	= lcount,			\
+		.mpc_ops		= mpops,			\
+		.mpc_loadtime_flags	= lflags,			\
+		.mpc_field_off		= slot,				\
+		.mpc_runtime_flags	= rflags			\
+	};								\
+									\
+	static kern_return_t						\
+	kmod_start(kmod_info_t *ki, void *xd)				\
+	{								\
+		return mac_policy_register(&mpname##_mac_policy_conf,	\
+		    &handle, xd);					\
+	}								\
+									\
+	static kern_return_t						\
+	kmod_stop(kmod_info_t *ki, void *xd)				\
+	{								\
+		return mac_policy_unregister(handle);			\
+	}								\
+									\
+	extern kern_return_t _start(kmod_info_t *ki, void *data);	\
+	extern kern_return_t _stop(kmod_info_t *ki, void *data);	\
+									\
+	KMOD_EXPLICIT_DECL(security.mpname, POLICY_VER, _start, _stop)	\
+	kmod_start_func_t *_realmain = kmod_start;			\
+	kmod_stop_func_t *_antimain = kmod_stop;			\
+	int _kext_apple_cc = __APPLE_CC__
+
+
+#define	LABEL_TO_SLOT(l, s)	(l)->l_perpolicy[s]
+
+#define	mac_get_mpc(h)		(mac_policy_list.entries[h].mpc)
+
+/**
+  @name Flags for MAC allocator interfaces
+
+  These flags are passed to the Darwin kernel allocator routines to
+  indicate whether the allocation is permitted to block or not.
+  Caution should be taken; some operations are not permitted to sleep,
+  and some types of locks cannot be held when sleeping.
+ */
+
+/*@{*/
+
+/**
+    @brief Allocation operations may block
+
+    If memory is not immediately available, the allocation routine
+    will block (typically sleeping) until memory is available.
+
+    @warning Inappropriate use of this flag may cause kernel panics.
+ */
+#define MAC_WAITOK  0
+
+/**
+    @brief Allocation operations may not block
+
+    Rather than blocking, the allocator may return an error if memory
+    is not immediately available.  This type of allocation will not
+    sleep, preserving locking semantics.
+ */
+#define MAC_NOWAIT  1
+
+/*@}*/
+
+#endif /* !_SECURITY_MAC_POLICY_H_ */
diff --git a/security/mac_port.c b/security/mac_port.c
new file mode 100644
index 000000000..6dbeeb1e6
--- /dev/null
+++ b/security/mac_port.c
@@ -0,0 +1,282 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 2003, 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005-2006 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <security/mac_internal.h>
+#include <security/mac_mach_internal.h>
+#include <mach/message.h>
+#include <kern/task.h>
+
+void
+mac_port_label_init(struct label *l)
+{
+
+	mac_label_init(l);
+	if (mac_late == 0) {
+		mac_label_journal_add(l, MLJ_TYPE_PORT);
+		mac_label_journal(l, MLJ_PORT_OP_INIT);
+	}
+	MAC_PERFORM (port_label_init, l);
+}
+
+void
+mac_port_label_destroy(struct label *l)
+{
+
+	MAC_PERFORM (port_label_destroy, l);
+	if (mac_late == 0)
+		mac_label_journal_remove(l);
+	mac_label_destroy(l);
+}
+
+void
+mac_port_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(port_label_copy, src, dest);
+}
+
+void
+mac_port_label_update_cred(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(port_label_update_cred, src, dest);
+}
+
+void
+mac_port_label_associate(struct label *it, struct label *st, struct label *port)
+{
+
+	if (mac_late == 0)
+		mac_label_journal(port, MLJ_PORT_OP_CREATE);
+	MAC_PERFORM(port_label_associate, it, st, port);
+}
+
+void
+mac_port_label_associate_kernel(struct label *port, int isreply)
+{
+
+	if (mac_late == 0)
+		mac_label_journal(port, MLJ_PORT_OP_CREATE_K);
+	MAC_PERFORM(port_label_associate_kernel, port, isreply);
+}
+
+void
+mac_port_label_update_kobject(struct label *port, int kotype)
+{
+
+	if (mac_late == 0)
+		mac_label_journal(port, MLJ_PORT_OP_UPDATE, kotype);
+	MAC_PERFORM(port_label_update_kobject, port, kotype);
+}
+
+int
+mac_port_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	/* XXX - should have mpo_port_label_internalize */
+	error = MAC_INTERNALIZE(cred, label, string);
+
+	return (error);
+}
+
+int
+mac_port_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen, int flags __unused)
+{
+	int error;
+
+	/* XXX - should have mpo_port_label_externalize */
+	error = MAC_EXTERNALIZE(cred, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_port_check_label_update(struct label *task, struct label *old,
+    struct label *newlabel)
+{
+	int error;
+
+	MAC_CHECK(port_check_label_update, task, old, newlabel);
+
+	return (error);
+}
+
+int
+mac_port_check_send(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_send, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_receive(struct label *task, struct label *sender)
+{
+	int error;
+ 
+	MAC_CHECK(port_check_receive, task, sender);
+ 
+	return (error);
+}
+
+int
+mac_port_check_make_send(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_make_send, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_make_send_once(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_make_send_once, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_copy_send(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_copy_send, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_move_send(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_move_send, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_move_send_once(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_move_send_once, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_move_receive(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_move_receive, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_hold_send(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_hold_send, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_hold_send_once(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_hold_send_once, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_hold_receive(struct label *task, struct label *port)
+{
+	int error;
+
+	MAC_CHECK(port_check_hold_receive, task, port);
+
+	return (error);
+}
+
+int
+mac_port_check_method(task_t task, struct label *sub, struct label *obj, int msgid)
+{
+	int error;
+
+	MAC_CHECK(port_check_method, get_bsdtask_info(task), sub, obj, msgid);
+
+	return (error);
+}
diff --git a/security/mac_posix_sem.c b/security/mac_posix_sem.c
new file mode 100644
index 000000000..b9851ed0f
--- /dev/null
+++ b/security/mac_posix_sem.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2003-2005 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/sysctl.h>
+#include <security/mac_internal.h>
+#include <sys/posix_sem.h>
+
+
+static struct label *
+mac_posixsem_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(posixsem_label_init, label);
+	return (label);
+}
+
+void
+mac_posixsem_label_init(struct pseminfo *psem)
+{
+
+	psem->psem_label = mac_posixsem_label_alloc();
+}
+
+static void
+mac_posixsem_label_free(struct label *label)
+{
+
+	MAC_PERFORM(posixsem_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_posixsem_label_destroy(struct pseminfo *psem)
+{
+
+	mac_posixsem_label_free(psem->psem_label);
+	psem->psem_label = NULL;
+}
+
+void
+mac_posixsem_label_associate(kauth_cred_t cred, struct pseminfo *psem,
+    const char *name)
+{
+
+	MAC_PERFORM(posixsem_label_associate, cred, psem, psem->psem_label, name);
+}
+
+
+void
+mac_posixsem_vnode_label_associate(kauth_cred_t cred, 
+	struct pseminfo *psem,  struct label *plabel,
+	vnode_t vp, struct label *vlabel)
+{
+	MAC_PERFORM(vnode_label_associate_posixsem, cred,
+		    psem, plabel, vp, vlabel);
+}
+
+int
+mac_posixsem_check_create(kauth_cred_t cred, const char *name)
+{
+	int error;
+
+	if (!mac_posixsem_enforce)
+		return (0);
+
+	MAC_CHECK(posixsem_check_create, cred, name);
+
+	return (error);
+}
+
+int
+mac_posixsem_check_open(kauth_cred_t cred, struct pseminfo *psem)
+{
+	int error;
+
+	if (!mac_posixsem_enforce)
+		return (0);
+
+	MAC_CHECK(posixsem_check_open, cred, psem,
+	    psem->psem_label);
+
+	return (error);
+}
+
+int
+mac_posixsem_check_post(kauth_cred_t cred, struct pseminfo *psem)
+{
+	int error;
+
+	if (!mac_posixsem_enforce)
+		return (0);
+
+	MAC_CHECK(posixsem_check_post, cred, psem, psem->psem_label);
+
+	return (error);
+}
+
+int
+mac_posixsem_check_unlink(kauth_cred_t cred, struct pseminfo *psem,
+    const char *name)
+{
+	int error;
+
+	if (!mac_posixsem_enforce)
+		return (0);
+
+	MAC_CHECK(posixsem_check_unlink, cred, psem, psem->psem_label, name);
+
+	return (error);
+}
+
+int
+mac_posixsem_check_wait(kauth_cred_t cred, struct pseminfo *psem)
+{
+	int error;
+
+	if (!mac_posixsem_enforce)
+		return (0);
+
+	MAC_CHECK(posixsem_check_wait, cred, psem, psem->psem_label);
+
+	return (error);
+}
diff --git a/security/mac_posix_shm.c b/security/mac_posix_shm.c
new file mode 100644
index 000000000..c42cfbb46
--- /dev/null
+++ b/security/mac_posix_shm.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2003-2005 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/posix_shm.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/sysctl.h>
+#include <sys/vnode.h>
+
+#include <security/mac_internal.h>
+
+static struct label *
+mac_posixshm_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(posixshm_label_init, label);
+	return (label);
+}
+
+void
+mac_posixshm_label_init(struct pshminfo *pshm)
+{
+
+	pshm->pshm_label = mac_posixshm_label_alloc();
+}
+
+static void
+mac_posixshm_label_free(struct label *label)
+{
+
+	MAC_PERFORM(posixshm_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_posixshm_label_destroy(struct pshminfo *pshm)
+{
+
+	mac_posixshm_label_free(pshm->pshm_label);
+	pshm->pshm_label = NULL;
+}
+
+void
+mac_posixshm_vnode_label_associate(kauth_cred_t cred,
+	struct pshminfo *pshm, struct label *plabel,
+	vnode_t vp, struct label *vlabel)
+{
+	MAC_PERFORM(vnode_label_associate_posixshm, cred, 
+		    pshm, plabel, vp, vlabel);
+}
+
+void
+mac_posixshm_label_associate(kauth_cred_t cred, struct pshminfo *pshm,
+    const char *name)
+{
+
+	MAC_PERFORM(posixshm_label_associate, cred, pshm, pshm->pshm_label, name);
+}
+
+int
+mac_posixshm_check_create(kauth_cred_t cred, const char *name)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_create, cred, name);
+
+	return error;
+}
+
+int
+mac_posixshm_check_open(kauth_cred_t cred, struct pshminfo *shm)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_open, cred, shm, shm->pshm_label);
+
+	return (error);
+}
+
+int
+mac_posixshm_check_mmap(kauth_cred_t cred, struct pshminfo *shm,
+    int prot, int flags)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_mmap, cred, shm, shm->pshm_label,
+            prot, flags);
+
+	return (error);
+}
+
+int
+mac_posixshm_check_stat(kauth_cred_t cred, struct pshminfo *shm)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_stat, cred, shm, shm->pshm_label);
+
+	return (error);
+}
+
+int
+mac_posixshm_check_truncate(kauth_cred_t cred, struct pshminfo *shm,
+    size_t size)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_truncate, cred, shm, shm->pshm_label, size);
+
+	return (error);
+}
+
+int
+mac_posixshm_check_unlink(kauth_cred_t cred, struct pshminfo *shm,
+    const char *name)
+{
+	int error = 0;
+
+	if (!mac_posixshm_enforce)
+		return 0;
+
+	MAC_CHECK(posixshm_check_unlink, cred, shm, shm->pshm_label, name);
+
+	return (error);
+}
diff --git a/security/mac_process.c b/security/mac_process.c
new file mode 100644
index 000000000..20ca2fb64
--- /dev/null
+++ b/security/mac_process.c
@@ -0,0 +1,542 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <string.h>
+#include <sys/param.h>
+#include <sys/ucred.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/vnode.h>
+#include <sys/proc.h>
+#include <sys/proc_internal.h>
+#include <sys/kauth.h>
+#include <sys/imgact.h>
+
+#include <security/mac_internal.h>
+
+#include <bsd/bsm/audit_kernel.h>
+
+struct label *
+mac_cred_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(cred_label_init, label);
+	return (label);
+}
+
+void
+mac_cred_label_init(struct ucred *cred)
+{
+	cred->cr_label = mac_cred_label_alloc();
+}
+
+void
+mac_cred_label_free(struct label *label)
+{
+	MAC_PERFORM(cred_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+int
+mac_cred_label_externalize_audit(struct proc *p, struct mac *mac)
+{
+	kauth_cred_t cr;
+	int error;
+
+	cr = kauth_cred_proc_ref(p);
+
+	error = MAC_EXTERNALIZE_AUDIT(cred, cr->cr_label,
+	    mac->m_string, mac->m_buflen);
+
+	kauth_cred_unref(&cr);
+	return (error);
+}
+
+void
+mac_cred_label_destroy(kauth_cred_t cred)
+{
+
+	mac_cred_label_free(cred->cr_label);
+	cred->cr_label = NULL;
+}
+
+int
+mac_cred_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen, int flags __unused)
+{
+	int error = 0;
+
+	error = MAC_EXTERNALIZE(cred, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_cred_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(cred, label, string);
+
+	return (error);
+}
+
+/*
+ * By default, fork just adds a reference to the parent
+ * credential.  Policies may need to know about this reference
+ * if they are tracking exit calls to know when to free the
+ * label.
+ */
+void
+mac_cred_label_associate_fork(kauth_cred_t cred, proc_t proc)
+{
+	MAC_PERFORM(cred_label_associate_fork, cred, proc);
+}
+	
+/*
+ * Initialize MAC label for the first kernel process, from which other
+ * kernel processes and threads are spawned.
+ */
+void
+mac_cred_label_associate_kernel(kauth_cred_t cred)
+{
+
+	MAC_PERFORM(cred_label_associate_kernel, cred);
+}
+
+/*
+ * Initialize MAC label for the first userland process, from which other
+ * userland processes and threads are spawned.
+ */
+void
+mac_cred_label_associate_user(kauth_cred_t cred)
+{
+
+	MAC_PERFORM(cred_label_associate_user, cred);
+}
+
+/*
+ * When a new process is created, its label must be initialized.  Generally,
+ * this involves inheritence from the parent process, modulo possible
+ * deltas.  This function allows that processing to take place.
+ */
+void
+mac_cred_label_associate(struct ucred *parent_cred, struct ucred *child_cred)
+{
+
+	MAC_PERFORM(cred_label_associate, parent_cred, child_cred);
+}
+
+int
+mac_execve_enter(user_addr_t mac_p, struct image_params *imgp)
+{
+	struct user_mac mac;
+	struct label *execlabel;
+	char *buffer;
+	int error;
+	size_t ulen;
+
+	if (mac_p == USER_ADDR_NULL)
+		return (0);
+
+	if (IS_64BIT_PROCESS(current_proc())) {
+		error = copyin(mac_p, &mac, sizeof(mac));
+	} else {
+		struct mac mac32;
+		error = copyin(mac_p, &mac32, sizeof(mac32));
+		mac.m_buflen = mac32.m_buflen;
+		mac.m_string = CAST_USER_ADDR_T(mac32.m_string);
+	}
+	if (error)
+		return (error);
+
+	error = mac_check_structmac_consistent(&mac);
+	if (error)
+		return (error);
+
+	execlabel = mac_cred_label_alloc();
+	MALLOC(buffer, char *, mac.m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac.m_string), buffer, mac.m_buflen, &ulen);
+	if (error)
+		goto out;
+	AUDIT_ARG(mac_string, buffer);
+
+	error = mac_cred_label_internalize(execlabel, buffer);
+out:
+	if (error) {
+		mac_cred_label_free(execlabel);
+		execlabel = NULL;
+	}
+	imgp->ip_execlabelp = execlabel;
+	FREE(buffer, M_MACTEMP);
+	return (error);
+}
+
+/*
+ * When the subject's label changes, it may require revocation of privilege
+ * to mapped objects.  This can't be done on-the-fly later with a unified
+ * buffer cache.
+ */
+void
+mac_cred_label_update(kauth_cred_t cred, struct label *newlabel)
+{
+
+	/* force label to be part of "matching" for credential */
+	cred->cr_flags |= CRF_MAC_ENFORCE;
+
+	/* inform the policies of the update */
+	MAC_PERFORM(cred_label_update, cred, newlabel);
+}
+
+int
+mac_cred_check_label_update(kauth_cred_t cred, struct label *newlabel)
+{
+	int error;
+
+	if (!mac_proc_enforce)
+		return (0);
+
+	MAC_CHECK(cred_check_label_update, cred, newlabel);
+
+	return (error);
+}
+
+int
+mac_cred_check_visible(kauth_cred_t u1, kauth_cred_t u2)
+{
+	int error;
+
+
+
+	if (!mac_proc_enforce)
+		return (0);
+
+
+
+	MAC_CHECK(cred_check_visible, u1, u2);
+
+
+	return (error);
+}
+
+/*                                                                                                    
+ * called with process locked.                                                                        
+ */
+void mac_proc_set_enforce(proc_t p, int enforce_flags)
+{
+        p->p_mac_enforce |= enforce_flags;
+}
+
+int
+mac_proc_check_debug(proc_t curp, struct proc *proc)
+{
+	kauth_cred_t cred;
+	int error;
+
+
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_debug, cred, proc);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_fork(proc_t curp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_fork, cred, curp);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_get_task_name(struct ucred *cred, struct proc *p)
+{
+	int error;
+
+	MAC_CHECK(proc_check_get_task_name, cred, p);
+
+	return (error);
+}
+
+int
+mac_proc_check_get_task(struct ucred *cred, struct proc *p)
+{
+	int error;
+
+	MAC_CHECK(proc_check_get_task, cred, p);
+
+	return (error);
+}
+
+int
+mac_proc_check_mprotect(proc_t proc,
+    user_addr_t addr, user_size_t size, int prot)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vm_enforce ||
+	    !mac_proc_check_enforce(proc, MAC_VM_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(proc);
+	MAC_CHECK(proc_check_mprotect, cred, proc, addr, size, prot);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_sched(proc_t curp, struct proc *proc)
+{
+	kauth_cred_t cred;
+	int error;
+
+
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_sched, cred, proc);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_signal(proc_t curp, struct proc *proc, int signum)
+{
+	kauth_cred_t cred;
+	int error;
+
+
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_signal, cred, proc, signum);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+int
+mac_proc_check_wait(proc_t curp, struct proc *proc)
+{
+	kauth_cred_t cred;
+	int error;
+
+
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(curp, MAC_PROC_ENFORCE))
+		return (0);
+
+	cred = kauth_cred_proc_ref(curp);
+	MAC_CHECK(proc_check_wait, cred, proc);
+	kauth_cred_unref(&cred);
+
+	return (error);
+}
+
+#if CONFIG_LCTX
+/*
+ * Login Context
+ */
+
+int
+mac_proc_check_setlcid (struct proc *p0, struct proc *p,
+			pid_t pid, pid_t lcid)
+{
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(p0, MAC_PROC_ENFORCE))
+		return (0);
+
+	MAC_CHECK(proc_check_setlcid, p0, p, pid, lcid);
+	return (error);
+}
+
+int
+mac_proc_check_getlcid (struct proc *p0, struct proc *p, pid_t pid)
+{
+	int error;
+
+	if (!mac_proc_enforce ||
+	    !mac_proc_check_enforce(p0, MAC_PROC_ENFORCE))
+		return (0);
+
+	MAC_CHECK(proc_check_getlcid, p0, p, pid);
+	return (error);
+}
+
+void
+mac_lctx_notify_create (struct proc *p, struct lctx *l)
+{
+	MAC_PERFORM(lctx_notify_create, p, l);
+}
+
+void
+mac_lctx_notify_join (struct proc *p, struct lctx *l)
+{
+	MAC_PERFORM(lctx_notify_join, p, l);
+}
+
+void
+mac_lctx_notify_leave (struct proc *p, struct lctx *l)
+{
+	MAC_PERFORM(lctx_notify_leave, p, l);
+}
+
+struct label *
+mac_lctx_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(lctx_label_init, label);
+	return (label);
+}
+
+void
+mac_lctx_label_free(struct label *label)
+{
+
+	MAC_PERFORM(lctx_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+int
+mac_lctx_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(lctx, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_lctx_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(lctx, label, string);
+
+	return (error);
+}
+
+void
+mac_lctx_label_update(struct lctx *l, struct label *newlabel)
+{
+
+	MAC_PERFORM(lctx_label_update, l, newlabel);
+}
+
+int
+mac_lctx_check_label_update(struct lctx *l, struct label *newlabel)
+{
+	int error;
+
+	MAC_CHECK(lctx_check_label_update, l, newlabel);
+
+	return (error);
+}
+#endif	/* LCTX */
+
+
+void
+mac_thread_userret(int code, int error, struct thread *thread)
+{
+
+	if (mac_late)
+		MAC_PERFORM(thread_userret, code, error, thread);
+}
diff --git a/security/mac_socket.c b/security/mac_socket.c
new file mode 100644
index 000000000..bd35170ee
--- /dev/null
+++ b/security/mac_socket.c
@@ -0,0 +1,644 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999-2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001-2005 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by McAfee
+ * Research, the Technology Research Division of Network Associates, Inc.
+ * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
+ * DARPA CHATS research program.
+ * 
+ * This software was enhanced by SPARTA ISSO under SPAWAR contract
+ * N66001-04-C-6019 ("SEFOS").
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/namei.h>
+#include <sys/protosw.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/sysctl.h>
+#include <sys/kpi_socket.h>
+
+#include <net/if.h>
+#include <net/if_var.h>
+
+#include <netinet/in.h>
+#include <netinet/ip_var.h>
+
+#include <security/mac_internal.h>
+
+#if CONFIG_MACF_SOCKET
+struct label *
+mac_socket_label_alloc(int flag)
+{
+	struct label *label;
+	int error;
+
+	label = mac_labelzone_alloc(flag);
+	if (label == NULL)
+		return (NULL);
+
+	MAC_CHECK(socket_label_init, label, flag);
+	if (error) {
+		MAC_PERFORM(socket_label_destroy, label);
+		mac_labelzone_free(label);
+		return (NULL);
+	}
+
+	return (label);
+}
+
+static struct label *
+mac_socket_peer_label_alloc(int flag)
+{
+	struct label *label;
+	int error;
+
+	label = mac_labelzone_alloc(flag);
+	if (label == NULL)
+		return (NULL);
+
+	MAC_CHECK(socketpeer_label_init, label, flag);
+	if (error) {
+		MAC_PERFORM(socketpeer_label_destroy, label);
+		mac_labelzone_free(label);
+		return (NULL);
+	}
+
+	return (label);
+}
+
+int
+mac_socket_label_init(struct socket *so, int flag)
+{
+
+	so->so_label = mac_socket_label_alloc(flag);
+	if (so->so_label == NULL)
+		return (ENOMEM);
+	so->so_peerlabel = mac_socket_peer_label_alloc(flag);
+	if (so->so_peerlabel == NULL) {
+		mac_socket_label_free(so->so_label);
+		so->so_label = NULL;
+		return (ENOMEM);
+	}
+	return (0);
+}
+
+void
+mac_socket_label_free(struct label *label)
+{
+
+	MAC_PERFORM(socket_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+static void
+mac_socket_peer_label_free(struct label *label)
+{
+
+	MAC_PERFORM(socketpeer_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_socket_label_destroy(struct socket *so)
+{
+
+	if (so->so_label != NULL) {
+		mac_socket_label_free(so->so_label);
+		so->so_label = NULL;
+	}
+	if (so->so_peerlabel != NULL) {
+		mac_socket_peer_label_free(so->so_peerlabel);
+		so->so_peerlabel = NULL;
+	}
+}
+
+void
+mac_socket_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(socket_label_copy, src, dest);
+}
+
+int
+mac_socket_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(socket, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+static int
+mac_socketpeer_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(socketpeer, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_socket_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(socket, label, string);
+
+	return (error);
+}
+
+void
+mac_socket_label_associate(struct ucred *cred, struct socket *so)
+{
+	if (!mac_socket_enforce)
+		return;
+
+	MAC_PERFORM(socket_label_associate, cred, 
+		    (socket_t)so, so->so_label);
+}
+
+void
+mac_socket_label_associate_accept(struct socket *oldsocket,
+    struct socket *newsocket)
+{
+	if (!mac_socket_enforce)
+		return;
+
+	MAC_PERFORM(socket_label_associate_accept, 
+		    (socket_t)oldsocket, oldsocket->so_label,
+		    (socket_t)newsocket, newsocket->so_label);
+}
+
+#if CONFIG_MACF_SOCKET && CONFIG_MACF_NET
+void
+mac_socketpeer_label_associate_mbuf(struct mbuf *mbuf, struct socket *so)
+{
+	struct label *label;
+
+	if (!mac_socket_enforce && !mac_net_enforce)
+		return;
+
+	label = mac_mbuf_to_label(mbuf);
+
+	/* Policy must deal with NULL label (unlabeled mbufs) */
+	MAC_PERFORM(socketpeer_label_associate_mbuf, mbuf, label,
+		    (socket_t)so, so->so_peerlabel);
+}
+#else
+void
+mac_socketpeer_label_associate_mbuf(__unused struct mbuf *mbuf, 
+	__unused struct socket *so)
+{
+	return;
+}
+#endif
+
+void
+mac_socketpeer_label_associate_socket(struct socket *oldsocket,
+    struct socket *newsocket)
+{
+	if (!mac_socket_enforce)
+		return;
+
+	MAC_PERFORM(socketpeer_label_associate_socket,
+		    (socket_t)oldsocket, oldsocket->so_label,
+		    (socket_t)newsocket, newsocket->so_peerlabel);
+}
+
+int
+mac_socket_check_kqfilter(kauth_cred_t cred, struct knote *kn,
+			  struct socket *so)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_kqfilter, cred, kn, 
+		  (socket_t)so, so->so_label);
+	return (error);
+}
+
+static int
+mac_socket_check_label_update(kauth_cred_t cred, struct socket *so,
+    struct label *newlabel)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_label_update, cred,
+		  (socket_t)so, so->so_label,
+		  newlabel);
+	return (error);
+}
+
+int
+mac_socket_check_select(kauth_cred_t cred, struct socket *so, int which)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_select, cred,
+		  (socket_t)so, so->so_label, which);
+	return (error);
+}
+
+int
+mac_socket_check_stat(kauth_cred_t cred, struct socket *so)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_stat, cred,
+		  (socket_t)so, so->so_label);
+	return (error);
+}
+
+
+int
+mac_socket_label_update(kauth_cred_t cred, struct socket *so, struct label *label)
+{
+	int error;
+#if 0
+	if (!mac_socket_enforce)
+		return;
+#endif
+	error = mac_socket_check_label_update(cred, so, label);
+	if (error)
+		return (error);
+
+	MAC_PERFORM(socket_label_update, cred,
+		    (socket_t)so, so->so_label, label);
+
+#if CONFIG_MACF_NET
+	/*
+	 * If the protocol has expressed interest in socket layer changes,
+	 * such as if it needs to propagate changes to a cached pcb
+	 * label from the socket, notify it of the label change while
+	 * holding the socket lock.
+	 * XXXMAC - are there cases when we should not do this?
+	 */
+	mac_inpcb_label_update(so);
+#endif
+	return (0);
+}
+
+int
+mac_setsockopt_label(kauth_cred_t cred, struct socket *so, struct mac *mac)
+{
+	struct label *intlabel;
+	char *buffer;
+	int error;
+	size_t len;
+
+	error = mac_check_structmac_consistent(mac);
+	if (error)
+		return (error);
+
+	MALLOC(buffer, char *, mac->m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac->m_string), buffer,
+		mac->m_buflen, &len);
+	if (error) {
+		FREE(buffer, M_MACTEMP);
+		return (error);
+	}
+
+	intlabel = mac_socket_label_alloc(MAC_WAITOK);
+	error = mac_socket_label_internalize(intlabel, buffer);
+	FREE(buffer, M_MACTEMP);
+	if (error)
+		goto out;
+
+	error = mac_socket_label_update(cred, so, intlabel);
+out:
+	mac_socket_label_free(intlabel);
+	return (error);
+}
+
+int
+mac_socket_label_get(__unused kauth_cred_t cred, struct socket *so,
+    struct mac *mac)
+{
+	char *buffer, *elements;
+	struct label *intlabel;
+	int error;
+	size_t len;
+
+	error = mac_check_structmac_consistent(mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac->m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac->m_string), elements,
+		mac->m_buflen, &len);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+
+	MALLOC(buffer, char *, mac->m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	intlabel = mac_socket_label_alloc(MAC_WAITOK);
+	mac_socket_label_copy(so->so_label, intlabel);
+	error = mac_socket_label_externalize(intlabel, elements, buffer,
+	    mac->m_buflen);
+	mac_socket_label_free(intlabel);
+	if (error == 0)
+		error = copyout(buffer, CAST_USER_ADDR_T(mac->m_string),
+			strlen(buffer)+1);
+
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+
+	return (error);
+}
+
+int
+mac_socketpeer_label_get(__unused kauth_cred_t cred, struct socket *so,
+    struct mac *mac)
+{
+	char *elements, *buffer;
+	struct label *intlabel;
+	int error;
+	size_t len;
+
+	error = mac_check_structmac_consistent(mac);
+	if (error)
+		return (error);
+
+	MALLOC(elements, char *, mac->m_buflen, M_MACTEMP, M_WAITOK);
+	error = copyinstr(CAST_USER_ADDR_T(mac->m_string), elements,
+		mac->m_buflen, &len);
+	if (error) {
+		FREE(elements, M_MACTEMP);
+		return (error);
+	}
+
+	MALLOC(buffer, char *, mac->m_buflen, M_MACTEMP, M_WAITOK | M_ZERO);
+	intlabel = mac_socket_label_alloc(MAC_WAITOK);
+	mac_socket_label_copy(so->so_peerlabel, intlabel);
+	error = mac_socketpeer_label_externalize(intlabel, elements, buffer,
+	    mac->m_buflen);
+	mac_socket_label_free(intlabel);
+	if (error == 0)
+		error = copyout(buffer, CAST_USER_ADDR_T(mac->m_string),
+				strlen(buffer)+1);
+
+	FREE(buffer, M_MACTEMP);
+	FREE(elements, M_MACTEMP);
+
+	return (error);
+}
+#endif /* MAC_SOCKET */
+
+int
+mac_socket_check_accept(kauth_cred_t cred, struct socket *so)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_accept, cred,
+		  (socket_t)so, so->so_label);
+	return (error);
+}
+
+int
+mac_socket_check_accepted(kauth_cred_t cred, struct socket *so)
+{
+	struct sockaddr *sockaddr;
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	if (sock_getaddr((socket_t)so, &sockaddr, 1) != 0) {
+		error = ECONNABORTED;
+	} else {
+		MAC_CHECK(socket_check_accepted, cred,
+			  (socket_t)so, so->so_label, sockaddr);
+		sock_freeaddr(sockaddr);
+	}
+	return (error);
+}
+
+int
+mac_socket_check_bind(kauth_cred_t ucred, struct socket *so,
+    struct sockaddr *sockaddr)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_bind, ucred,
+		  (socket_t)so, so->so_label, sockaddr);
+	return (error);
+}
+
+int
+mac_socket_check_connect(kauth_cred_t cred, struct socket *so,
+    struct sockaddr *sockaddr)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_connect, cred,
+		  (socket_t)so, so->so_label,
+		  sockaddr);
+	return (error);
+}
+
+int
+mac_socket_check_create(kauth_cred_t cred, int domain, int type, int protocol)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_create, cred, domain, type, protocol);
+	return (error);
+}
+
+#if CONFIG_MACF_SOCKET && CONFIG_MACF_NET
+int
+mac_socket_check_deliver(struct socket *so, struct mbuf *mbuf)
+{
+	struct label *label;
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	label = mac_mbuf_to_label(mbuf);
+
+	/* Policy must deal with NULL label (unlabeled mbufs) */
+	MAC_CHECK(socket_check_deliver,
+		  (socket_t)so, so->so_label, mbuf, label);
+	return (error);
+}
+#else
+int
+mac_socket_check_deliver(__unused struct socket *so, __unused struct mbuf *mbuf)
+{
+	return (0);
+}
+#endif
+
+int
+mac_socket_check_listen(kauth_cred_t cred, struct socket *so)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_listen, cred,
+		  (socket_t)so, so->so_label);
+	return (error);
+}
+
+int
+mac_socket_check_receive(kauth_cred_t cred, struct socket *so)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_receive, cred,
+		  (socket_t)so, so->so_label);
+	return (error);
+}
+
+int
+mac_socket_check_received(kauth_cred_t cred, struct socket *so, struct sockaddr *saddr)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_received, cred,
+		  (socket_t)so, so->so_label, saddr);
+	return (error);
+}
+
+int
+mac_socket_check_send(kauth_cred_t cred, struct socket *so,
+		      struct sockaddr *sockaddr)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return 0;
+
+	MAC_CHECK(socket_check_send, cred,
+		  (socket_t)so, so->so_label, sockaddr);
+	return (error);
+}
+
+int
+mac_socket_check_setsockopt(kauth_cred_t cred, struct socket *so,
+			    struct sockopt *sopt)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return (0);
+
+	MAC_CHECK(socket_check_setsockopt, cred,
+		  (socket_t)so, so->so_label, sopt);
+	return (error);
+}
+
+int mac_socket_check_getsockopt(kauth_cred_t cred, struct socket *so,
+				struct sockopt *sopt)
+{
+	int error;
+
+	if (!mac_socket_enforce)
+		return (0);
+
+	MAC_CHECK(socket_check_getsockopt, cred,
+		  (socket_t)so, so->so_label, sopt);
+	return (error);
+}
diff --git a/security/mac_stub.c b/security/mac_stub.c
new file mode 100644
index 000000000..3a59c4374
--- /dev/null
+++ b/security/mac_stub.c
@@ -0,0 +1,679 @@
+/*
+ * Copyright (c) 2006-2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+#include <mac.h>
+
+#if !CONFIG_MACF
+/*
+ * XXX stubs until we fix <rdar://problem/4607887>
+ */
+int mac_check_ipc_method(void)
+{
+	return 0;
+}
+int mac_check_port_copy_send(void)
+{
+	return 0;
+}
+int mac_check_port_hold_receive(void)
+{
+	return 0;
+}
+int mac_check_port_hold_send(void)
+{
+	return 0;
+}
+int mac_check_port_make_send(void)
+{
+	return 0;
+}
+int mac_check_port_move_receive(void)
+{
+	return 0;
+}
+int mac_check_port_relabel(void)
+{
+	return 0;
+}
+int mac_check_port_send(void)
+{
+	return 0;
+}
+int mac_check_service_access(void)
+{
+	return 0;
+}
+int mac_copy_cred_to_task(void)
+{
+	return 0;
+}
+int mac_copy_port_label(void)
+{
+	return 0;
+}
+int mac_create_kernel_port(void)
+{
+	return 0;
+}
+int mac_create_kernel_task(void)
+{
+	return 0;
+}
+int mac_create_port(void)
+{
+	return 0;
+}
+int mac_create_task(void)
+{
+	return 0;
+}
+int mac_destroy_port_label(void)
+{
+	return 0;
+}
+int mac_destroy_task_label(void)
+{
+	return 0;
+}
+int mac_externalize_port_label(void)
+{
+	return 0;
+}
+int mac_externalize_task_label(void)
+{
+	return 0;
+}
+int mac_init(void)
+{
+	return 0;
+}
+int mac_init_mach(void)
+{
+	return 0;
+}
+int mac_init_port_label(void)
+{
+	return 0;
+}
+int mac_init_task_label(void)
+{
+	return 0;
+}
+int mac_internalize_port_label(void)
+{
+	return 0;
+}
+int mac_request_object_label(void)
+{
+	return 0;
+}
+int mac_update_port_from_cred_label(void)
+{
+	return 0;
+}
+int mac_update_port_kobject(void)
+{
+	return 0;
+}
+int mac_associate_vnode_devfs(void)
+{
+	return 0;
+}
+int mac_associate_vnode_extattr(void)
+{
+	return 0;
+}
+int mac_associate_vnode_singlelabel(void)
+{
+	return 0;
+}
+int mac_check_mount_getattr(void)
+{
+	return 0;
+}
+int mac_check_mount_setattr(void)
+{
+	return 0;
+}
+int mac_check_pipe_ioctl(void)
+{
+	return 0;
+}
+int mac_check_pipe_kqfilter(void)
+{
+	return 0;
+}
+int mac_check_pipe_read(void)
+{
+	return 0;
+}
+int mac_check_pipe_select(void)
+{
+	return 0;
+}
+int mac_check_pipe_stat(void)
+{
+	return 0;
+}
+int mac_check_pipe_write(void)
+{
+	return 0;
+}
+int mac_check_posix_sem_create(void)
+{
+	return 0;
+}
+int mac_check_posix_sem_open(void)
+{
+	return 0;
+}
+int mac_check_posix_sem_post(void)
+{
+	return 0;
+}
+int mac_check_posix_sem_unlink(void)
+{
+	return 0;
+}
+int mac_check_posix_sem_wait(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_create(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_mmap(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_open(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_stat(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_truncate(void)
+{
+	return 0;
+}
+int mac_check_posix_shm_unlink(void)
+{
+	return 0;
+}
+int mac_check_proc_getlcid(void)
+{
+	return 0;
+}
+int mac_check_proc_fork(void)
+{
+	return 0;
+}
+int mac_check_proc_sched(void)
+{
+	return 0;
+}
+int mac_check_proc_setlcid(void)
+{
+	return 0;
+}
+int mac_check_proc_signal(void)
+{
+	return 0;
+}
+int mac_check_socket_received(void)
+{
+	return 0;
+}
+int mac_check_proc_wait(void)
+{
+	return 0;
+}
+int mac_check_system_acct(void)
+{
+	return 0;
+}
+int mac_check_system_nfsd(void)
+{
+	return 0;
+}
+int mac_check_system_reboot(void)
+{
+	return 0;
+}
+int mac_check_system_settime(void)
+{
+	return 0;
+}
+int mac_check_system_swapoff(void)
+{
+	return 0;
+}
+int mac_check_system_swapon(void)
+{
+	return 0;
+}
+int mac_check_system_sysctl(void)
+{
+	return 0;
+}
+int mac_check_vnode_access(void)
+{
+	return 0;
+}
+int mac_check_vnode_chdir(void)
+{
+	return 0;
+}
+int mac_check_vnode_chroot(void)
+{
+	return 0;
+}
+int mac_check_vnode_create(void)
+{
+	return 0;
+}
+int mac_check_vnode_delete(void)
+{
+	return 0;
+}
+int mac_check_vnode_deleteextattr(void)
+{
+	return 0;
+}
+int mac_check_vnode_exchangedata(void)
+{
+	return 0;
+}
+int mac_check_vnode_exec(void)
+{
+	return 0;
+}
+int mac_check_vnode_getattrlist(void)
+{
+	return 0;
+}
+int mac_check_vnode_getextattr(void)
+{
+	return 0;
+}
+int mac_check_vnode_kqfilter(void)
+{
+	return 0;
+}
+int mac_check_vnode_link(void)
+{
+	return 0;
+}
+int mac_check_vnode_listextattr(void)
+{
+	return 0;
+}
+int mac_check_vnode_lookup(void)
+{
+	return 0;
+}
+int mac_check_vnode_mmap(void)
+{
+	return 0;
+}
+int mac_check_vnode_open(void)
+{
+	return 0;
+}
+int mac_check_vnode_read(void)
+{
+	return 0;
+}
+int mac_check_vnode_readdir(void)
+{
+	return 0;
+}
+int mac_check_vnode_readlink(void)
+{
+	return 0;
+}
+int mac_check_vnode_rename_from(void)
+{
+	return 0;
+}
+int mac_check_vnode_rename_to(void)
+{
+	return 0;
+}
+int mac_check_vnode_revoke(void)
+{
+	return 0;
+}
+int mac_check_vnode_select(void)
+{
+	return 0;
+}
+int mac_check_vnode_setattrlist(void)
+{
+	return 0;
+}
+int mac_check_vnode_setextattr(void)
+{
+	return 0;
+}
+int mac_check_vnode_setflags(void)
+{
+	return 0;
+}
+int mac_check_vnode_setmode(void)
+{
+	return 0;
+}
+int mac_check_vnode_setowner(void)
+{
+	return 0;
+}
+int mac_check_vnode_setutimes(void)
+{
+	return 0;
+}
+int mac_check_vnode_stat(void)
+{
+	return 0;
+}
+int mac_check_vnode_write(void)
+{
+	return 0;
+}
+int mac_cleanup_vnode(void)
+{
+	return 0;
+}
+int mac_copy_devfs_label(void)
+{
+	return 0;
+}
+int mac_copy_vnode_label(void)
+{
+	return 0;
+}
+int mac_create_cred(void)
+{
+	return 0;
+}
+int mac_create_devfs_device(void)
+{
+	return 0;
+}
+int mac_create_devfs_directory(void)
+{
+	return 0;
+}
+int mac_create_mount(void)
+{
+	return 0;
+}
+int mac_create_pipe(void)
+{
+	return 0;
+}
+int mac_create_posix_sem(void)
+{
+	return 0;
+}
+int mac_create_posix_shm(void)
+{
+	return 0;
+}
+int mac_create_proc0(void)
+{
+	return 0;
+}
+int mac_create_proc1(void)
+{
+	return 0;
+}
+int mac_create_vnode_extattr(void)
+{
+	return 0;
+}
+int mac_cred_label_alloc(void)
+{
+	return 0;
+}
+int mac_cred_label_free(void)
+{
+	return 0;
+}
+int mac_destroy_cred(void)
+{
+	return 0;
+}
+int mac_destroy_devfsdirent(void)
+{
+	return 0;
+}
+int mac_destroy_mount(void)
+{
+	return 0;
+}
+int mac_destroy_pipe(void)
+{
+	return 0;
+}
+int mac_destroy_posix_sem(void)
+{
+	return 0;
+}
+int mac_destroy_posix_shm(void)
+{
+	return 0;
+}
+int mac_destroy_proc(void)
+{
+	return 0;
+}
+int mac_execve_enter(void)
+{
+	return 0;
+}
+int mac_execve_transition(void)
+{
+	return 0;
+}
+int mac_execve_will_transition(void)
+{
+	return 0;
+}
+int mac_init_bsd(void)
+{
+	return 0;
+}
+int mac_init_cred(void)
+{
+	return 0;
+}
+int mac_init_devfsdirent(void)
+{
+	return 0;
+}
+int mac_init_mount(void)
+{
+	return 0;
+}
+int mac_init_pipe(void)
+{
+	return 0;
+}
+int mac_init_posix_sem(void)
+{
+	return 0;
+}
+int mac_init_posix_shm(void)
+{
+	return 0;
+}
+int mac_init_proc(void)
+{
+	return 0;
+}
+int mac_init_vnode(void)
+{
+	return 0;
+}
+int mac_lctx_label_alloc(void)
+{
+	return 0;
+}
+int mac_lctx_label_free(void)
+{
+	return 0;
+}
+int mac_proc_create_lctx(void)
+{
+	return 0;
+}
+int mac_proc_join_lctx(void)
+{
+	return 0;
+}
+int mac_proc_leave_lctx(void)
+{
+	return 0;
+}
+int mac_relabel_cred(void)
+{
+	return 0;
+}
+int mac_relabel_vnode(void)
+{
+	return 0;
+}
+int mac_update_devfsdirent(void)
+{
+	return 0;
+}
+int mac_update_vnode_extattr(void)
+{
+	return 0;
+}
+int mac_vnode_label_alloc(void)
+{
+	return 0;
+}
+int mac_vnode_label_free(void)
+{
+	return 0;
+}
+int vop_stdsetlabel_ea(void)
+{
+	return 0;
+}
+int kau_will_audit(void)
+{
+	return 0;
+}
+int mac_kalloc(void)
+{
+	return 0;
+}
+int mac_kalloc_noblock(void)
+{
+	return 0;
+}
+int mac_kfree(void)
+{
+	return 0;
+}
+int mac_mbuf_alloc(void)
+{
+	return 0;
+}
+int mac_mbuf_free(void)
+{
+	return 0;
+}
+int mac_unwire(void)
+{
+	return 0;
+}
+int mac_wire(void)
+{
+	return 0;
+}
+int sysctl__security_mac_children(void)
+{
+	return 0;
+}
+int mac_check_socket_accept(void)
+{
+	return 0;
+}
+int mac_check_socket_accepted(void)
+{
+	return 0;
+}
+int mac_check_socket_bind(void)
+{
+	return 0;
+}
+int mac_check_socket_connect(void)
+{
+	return 0;
+}
+int mac_check_socket_create(void)
+{
+	return 0;
+}
+int mac_check_socket_getsockopt(void)
+{
+	return 0;
+}
+int mac_check_socket_listen(void)
+{
+	return 0;
+}
+int mac_check_socket_receive(void)
+{
+	return 0;
+}
+int mac_check_socket_send(void)
+{
+	return 0;
+}
+int mac_check_socket_setsockopt(void)
+{
+	return 0;
+}
+int mac_fork_proc(void)
+{
+	return 0;
+}
+int mac_set_enforce_proc(void)
+{
+	return 0;
+}
+#endif /* CONFIG_MACF */
diff --git a/security/mac_system.c b/security/mac_system.c
new file mode 100644
index 000000000..410c71310
--- /dev/null
+++ b/security/mac_system.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/param.h>
+#include <sys/vnode.h>
+#include <sys/vnode_internal.h>
+
+#include <security/mac_internal.h>
+
+
+int
+mac_system_check_acct(kauth_cred_t cred, struct vnode *vp)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_acct, cred, vp,
+	    vp != NULL ? vp->v_label : NULL);
+
+	return (error);
+}
+
+int
+mac_system_check_host_priv(kauth_cred_t cred)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_host_priv, cred);
+
+	return (error);
+}
+
+int
+mac_system_check_nfsd(kauth_cred_t cred)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_nfsd, cred);
+
+	return (error);
+}
+
+int
+mac_system_check_reboot(kauth_cred_t cred, int howto)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_reboot, cred, howto);
+
+	return (error);
+}
+
+int
+mac_system_check_settime(kauth_cred_t cred)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_settime, cred);
+
+	return (error);
+}
+
+int
+mac_system_check_swapon(kauth_cred_t cred, struct vnode *vp)
+{
+	int error;
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_swapon, cred, vp, vp->v_label);
+	return (error);
+}
+
+int
+mac_system_check_swapoff(kauth_cred_t cred, struct vnode *vp)
+{
+	int error;
+
+
+
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_swapoff, cred, vp, vp->v_label);
+	return (error);
+}
+
+int
+mac_system_check_sysctl(kauth_cred_t cred, int *name, u_int namelen,
+    user_addr_t old, user_addr_t oldlenp, int inkernel, user_addr_t new, size_t newlen)
+{
+	int error;
+
+	/*
+	 * XXXMAC: We're very much like to assert the SYSCTL_LOCK here,
+	 * but since it's not exported from kern_sysctl.c, we can't.
+	 */
+	if (!mac_system_enforce)
+		return (0);
+
+	MAC_CHECK(system_check_sysctl, cred, name, namelen, old, oldlenp,
+	    inkernel, new, newlen);
+
+	return (error);
+}
diff --git a/security/mac_sysv_msg.c b/security/mac_sysv_msg.c
new file mode 100644
index 000000000..540f5b09b
--- /dev/null
+++ b/security/mac_sysv_msg.c
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2003-2005 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/vnode.h>
+#include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/namei.h>
+#include <sys/sysctl.h>
+#include <sys/msg.h>
+
+#include <security/mac_internal.h>
+
+static struct label *
+mac_sysv_msgmsg_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(sysvmsg_label_init, label);
+	return (label);
+}
+
+void
+mac_sysvmsg_label_init(struct msg *msgptr)
+{
+
+	msgptr->label = mac_sysv_msgmsg_label_alloc();
+}
+
+static struct label *
+mac_sysv_msgqueue_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(sysvmsq_label_init, label);
+	return (label);
+}
+
+void
+mac_sysvmsq_label_init(struct msqid_kernel *msqptr)
+{
+
+	msqptr->label = mac_sysv_msgqueue_label_alloc();
+}
+
+void
+mac_sysvmsg_label_associate(kauth_cred_t cred, struct msqid_kernel *msqptr, 
+    struct msg *msgptr)
+{
+				
+	MAC_PERFORM(sysvmsg_label_associate, cred, msqptr, msqptr->label, 
+		msgptr, msgptr->label);
+}
+
+void
+mac_sysvmsq_label_associate(kauth_cred_t cred, struct msqid_kernel *msqptr)
+{
+				
+	MAC_PERFORM(sysvmsq_label_associate, cred, msqptr, msqptr->label);
+}
+
+void
+mac_sysvmsg_label_recycle(struct msg *msgptr)
+{
+
+	MAC_PERFORM(sysvmsg_label_recycle, msgptr->label);
+}
+
+void
+mac_sysvmsq_label_recycle(struct msqid_kernel *msqptr)
+{
+				
+	MAC_PERFORM(sysvmsq_label_recycle, msqptr->label);
+}
+
+int
+mac_sysvmsq_check_enqueue(kauth_cred_t cred, struct msg *msgptr,
+	struct msqid_kernel *msqptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_enqueue, cred,  msgptr, msgptr->label, msqptr,
+	    msqptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msgrcv(kauth_cred_t cred, struct msg *msgptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msgrcv, cred, msgptr, msgptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msgrmid(kauth_cred_t cred, struct msg *msgptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msgrmid, cred,  msgptr, msgptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msqget(kauth_cred_t cred, struct msqid_kernel *msqptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msqget, cred, msqptr, msqptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msqsnd(kauth_cred_t cred, struct msqid_kernel *msqptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msqsnd, cred, msqptr, msqptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msqrcv(kauth_cred_t cred, struct msqid_kernel *msqptr)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msqrcv, cred, msqptr, msqptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvmsq_check_msqctl(kauth_cred_t cred, struct msqid_kernel *msqptr,
+    int cmd)
+{
+	int error;
+
+	if (!mac_sysvmsg_enforce)
+		return (0);
+
+	MAC_CHECK(sysvmsq_check_msqctl, cred, msqptr, msqptr->label, cmd);
+
+	return(error);
+}
diff --git a/security/mac_sysv_sem.c b/security/mac_sysv_sem.c
new file mode 100644
index 000000000..b433aa6b2
--- /dev/null
+++ b/security/mac_sysv_sem.c
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2003-2004 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/vnode.h>
+#include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/namei.h>
+#include <sys/sysctl.h>
+#include <sys/sem.h>
+#include <sys/sem_internal.h>
+
+#include <security/mac_internal.h>
+
+
+static struct label *
+mac_sysv_sem_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(sysvsem_label_init, label);
+	return (label);
+}
+
+void
+mac_sysvsem_label_init(struct semid_kernel *semakptr)
+{
+
+	semakptr->label = mac_sysv_sem_label_alloc();
+}
+
+static void
+mac_sysv_sem_label_free(struct label *label)
+{
+
+	MAC_PERFORM(sysvsem_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_sysvsem_label_destroy(struct semid_kernel *semakptr)
+{
+
+	mac_sysv_sem_label_free(semakptr->label);
+	semakptr->label = NULL;
+}
+
+void
+mac_sysvsem_label_associate(kauth_cred_t cred, struct semid_kernel *semakptr)
+{
+
+	MAC_PERFORM(sysvsem_label_associate, cred, semakptr, semakptr->label);
+}
+
+void
+mac_sysvsem_label_recycle(struct semid_kernel *semakptr)
+{
+
+	MAC_PERFORM(sysvsem_label_recycle, semakptr->label);
+}
+
+int
+mac_sysvsem_check_semctl(kauth_cred_t cred, struct semid_kernel *semakptr,
+    int cmd)
+{
+	int error;
+
+	if (!mac_sysvsem_enforce)
+		return (0);
+
+	MAC_CHECK(sysvsem_check_semctl, cred, semakptr, semakptr->label, cmd);
+
+	return(error);
+}
+
+int
+mac_sysvsem_check_semget(kauth_cred_t cred, struct semid_kernel *semakptr)
+{
+	int error;
+
+	if (!mac_sysvsem_enforce)
+		return (0);
+
+	MAC_CHECK(sysvsem_check_semget, cred, semakptr, semakptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvsem_check_semop(kauth_cred_t cred, struct semid_kernel *semakptr,
+    size_t accesstype)
+{
+	int error;
+
+	if (!mac_sysvsem_enforce)
+		return (0);
+
+	MAC_CHECK(sysvsem_check_semop, cred, semakptr, semakptr->label,
+	    accesstype);
+
+	return(error);
+}
diff --git a/security/mac_sysv_shm.c b/security/mac_sysv_shm.c
new file mode 100644
index 000000000..bcacb48a1
--- /dev/null
+++ b/security/mac_sysv_shm.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 2003-2004 Networks Associates Technology, Inc.
+ * All rights reserved.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+#include <sys/cdefs.h>
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/sbuf.h>
+#include <sys/systm.h>
+#include <sys/vnode.h>
+#include <sys/mount.h>
+#include <sys/file.h>
+#include <sys/namei.h>
+#include <sys/sysctl.h>
+#include <sys/shm.h>
+#include <sys/shm_internal.h>
+
+#include <security/mac_internal.h>
+
+
+static struct label *
+mac_sysv_shm_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(sysvshm_label_init, label);
+	return (label);
+}
+
+void
+mac_sysvshm_label_init(struct shmid_kernel *shmsegptr)
+{
+
+	shmsegptr->label = mac_sysv_shm_label_alloc();
+}
+
+static void
+mac_sysv_shm_label_free(struct label *label)
+{
+
+	MAC_PERFORM(sysvshm_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_sysvshm_label_destroy(struct shmid_kernel *shmsegptr)
+{
+
+	mac_sysv_shm_label_free(shmsegptr->label);
+	shmsegptr->label = NULL;
+}
+
+void
+mac_sysvshm_label_associate(struct ucred *cred, struct shmid_kernel *shmsegptr)
+{
+
+	MAC_PERFORM(sysvshm_label_associate, cred, shmsegptr, shmsegptr->label);
+}
+
+void
+mac_sysvshm_label_recycle(struct shmid_kernel *shmsegptr)
+{
+
+	MAC_PERFORM(sysvshm_label_recycle, shmsegptr->label);
+}
+
+int
+mac_sysvshm_check_shmat(struct ucred *cred, struct shmid_kernel *shmsegptr,
+    int shmflg)
+{
+	int error;
+
+	if (!mac_sysvshm_enforce)
+		return 0;
+
+	MAC_CHECK(sysvshm_check_shmat, cred, shmsegptr, shmsegptr->label,
+	    shmflg);
+
+	return(error);
+}
+
+int
+mac_sysvshm_check_shmctl(struct ucred *cred, struct shmid_kernel *shmsegptr,
+    int cmd)
+{
+	int error;
+
+	if (!mac_sysvshm_enforce)
+		return 0;
+
+	MAC_CHECK(sysvshm_check_shmctl, cred, shmsegptr, shmsegptr->label,
+	    cmd);
+
+	return(error);
+}
+
+int
+mac_sysvshm_check_shmdt(struct ucred *cred, struct shmid_kernel *shmsegptr)
+{
+	int error;
+
+	if (!mac_sysvshm_enforce)
+		return 0;
+
+	MAC_CHECK(sysvshm_check_shmdt, cred, shmsegptr, shmsegptr->label);
+
+	return(error);
+}
+
+int
+mac_sysvshm_check_shmget(struct ucred *cred, struct shmid_kernel *shmsegptr,
+    int shmflg)
+{
+	int error;
+
+	if (!mac_sysvshm_enforce)
+		return 0;
+
+	MAC_CHECK(sysvshm_check_shmget, cred, shmsegptr, shmsegptr->label,
+	    shmflg);
+
+	return(error);
+}
diff --git a/security/mac_task.c b/security/mac_task.c
new file mode 100644
index 000000000..917cdef48
--- /dev/null
+++ b/security/mac_task.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+
+/*-
+ * Copyright (c) 2003, 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <security/mac_internal.h>
+#include <security/mac_mach_internal.h>
+
+void
+mac_task_label_init(struct label *label)
+{
+
+	mac_label_init(label);
+	if (mac_late == 0) {
+		mac_label_journal_add(label, MLJ_TYPE_TASK);
+		mac_label_journal(label, MLJ_TASK_OP_INIT);
+	}
+	MAC_PERFORM(task_label_init, label);
+}
+
+void
+mac_task_label_update(struct label *cred, struct label *task)
+{
+
+	MAC_PERFORM(task_label_update, cred, task);
+}
+
+void
+mac_task_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(task_label_copy, src, dest);
+}
+
+void
+mac_task_label_destroy(struct label *label)
+{
+
+	MAC_PERFORM(task_label_destroy, label);
+	if (mac_late == 0)
+		mac_label_journal_remove(label);
+	mac_label_destroy(label);
+}
+
+void
+mac_task_label_associate(struct task *parent, struct task *child, struct label *pl,
+    struct label *chl, struct label *chportl)
+{
+
+	MAC_PERFORM(task_label_associate, parent, child, pl, chl, chportl);
+}
+
+void
+mac_task_label_associate_kernel(struct task *t, struct label *tl, struct label *tportl)
+{
+
+	if (mac_late == 0)
+		mac_label_journal(tl, MLJ_TASK_OP_CREATE_K);
+	MAC_PERFORM(task_label_associate_kernel, t, tl, tportl);
+}
+
+int
+mac_task_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen, int flags __unused)
+{
+	int error = 0;
+
+	error = MAC_EXTERNALIZE(task, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_task_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(task, label, string);
+
+	return (error);
+}
diff --git a/security/mac_vfs.c b/security/mac_vfs.c
new file mode 100644
index 000000000..4316e2d3b
--- /dev/null
+++ b/security/mac_vfs.c
@@ -0,0 +1,1395 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson
+ * Copyright (c) 2001 Ilmar S. Habibulin
+ * Copyright (c) 2001, 2002, 2003, 2004 Networks Associates Technology, Inc.
+ * Copyright (c) 2005 SPARTA, Inc.
+ *
+ * This software was developed by Robert Watson and Ilmar Habibulin for the
+ * TrustedBSD Project.
+ *
+ * This software was developed for the FreeBSD Project in part by Network
+ * Associates Laboratories, the Security Research Division of Network
+ * Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"),
+ * as part of the DARPA CHATS research program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/kauth.h>
+
+#include <sys/file_internal.h>
+#include <sys/imgact.h>
+#include <sys/namei.h>
+#include <sys/mount_internal.h>
+#include <sys/pipe.h>
+#include <sys/posix_sem.h>
+#include <sys/posix_shm.h>
+#include <sys/uio_internal.h>
+#include <sys/vnode_internal.h>
+
+#include <miscfs/devfs/devfsdefs.h>
+#include <miscfs/fdesc/fdesc.h>
+
+#include <security/mac_internal.h>
+
+/* convert {R,W,X}_OK values to V{READ,WRITE,EXEC} */
+#define	ACCESS_MODE_TO_VNODE_MASK(m)	(m << 6)
+
+static struct label *
+mac_devfsdirent_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(devfs_label_init, label);
+	return (label);
+}
+
+void
+mac_devfs_label_init(struct devnode *de)
+{
+
+	de->dn_label = mac_devfsdirent_label_alloc();
+}
+
+static struct label *
+mac_mount_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(mount_label_init, label);
+	return (label);
+}
+
+void
+mac_mount_label_init(struct mount *mp)
+{
+
+	mp->mnt_mntlabel = mac_mount_label_alloc();
+}
+
+struct label *
+mac_vnode_label_alloc(void)
+{
+	struct label *label;
+
+	label = mac_labelzone_alloc(MAC_WAITOK);
+	if (label == NULL)
+		return (NULL);
+	MAC_PERFORM(vnode_label_init, label);
+	return (label);
+}
+
+void
+mac_vnode_label_init(vnode_t vp)
+{
+
+	vp->v_label = mac_vnode_label_alloc();
+}
+
+/* 
+ * vnode labels are allocated at the same time as vnodes, but vnodes are never
+ * freed.  Instead, we want to remove any sensitive information before putting
+ * them on the free list for reuse.
+*/
+void
+mac_vnode_label_recycle(vnode_t vp)
+{
+
+	MAC_PERFORM(vnode_label_recycle, vp->v_label);
+}
+
+static void
+mac_devfs_label_free(struct label *label)
+{
+	MAC_PERFORM(devfs_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_devfs_label_destroy(struct devnode *de)
+{
+	if (de->dn_label != NULL) {
+		mac_devfs_label_free(de->dn_label);
+		de->dn_label = NULL;
+	}
+}
+
+static void
+mac_mount_label_free(struct label *label)
+{
+
+	MAC_PERFORM(mount_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+void
+mac_mount_label_destroy(struct mount *mp)
+{
+
+
+	if (mp->mnt_mntlabel != NULL) {
+		mac_mount_label_free(mp->mnt_mntlabel);
+		mp->mnt_mntlabel = NULL;
+	}
+}
+
+void
+mac_vnode_label_free(struct label *label)
+{
+
+	MAC_PERFORM(vnode_label_destroy, label);
+	mac_labelzone_free(label);
+}
+
+#ifndef __APPLE__
+void
+mac_vnode_label_destroy(struct vnode *vp)
+{
+
+	mac_vnode_label_free(vp->v_label);
+	vp->v_label = NULL;
+}
+#endif
+
+void
+mac_vnode_label_copy(struct label *src, struct label *dest)
+{
+
+	MAC_PERFORM(vnode_label_copy, src, dest);
+}
+
+int
+mac_vnode_label_externalize_audit(struct vnode *vp, struct mac *mac)
+{
+	int error;
+
+	/* It is assumed that any necessary vnode locking is done on entry */
+	error = MAC_EXTERNALIZE_AUDIT(vnode, vp->v_label,
+	    mac->m_string, mac->m_buflen);
+
+	return (error);
+}
+
+int
+mac_vnode_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen, int flags __unused)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(vnode, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+int
+mac_vnode_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(vnode, label, string);
+
+	return (error);
+}
+
+int
+mac_mount_label_internalize(struct label *label, char *string)
+{
+	int error;
+
+	error = MAC_INTERNALIZE(mount, label, string);
+
+	return (error);
+}
+
+int
+mac_mount_label_externalize(struct label *label, char *elements,
+    char *outbuf, size_t outbuflen)
+{
+	int error;
+
+	error = MAC_EXTERNALIZE(mount, label, elements, outbuf, outbuflen);
+
+	return (error);
+}
+
+void
+mac_devfs_label_copy(struct label *src, struct label *dest)
+{
+	if (!mac_device_enforce)
+		return;
+
+	MAC_PERFORM(devfs_label_copy, src, dest);
+}
+
+void
+mac_devfs_label_update(struct mount *mp, struct devnode *de,
+    struct vnode *vp)
+{
+
+	if (!mac_device_enforce)
+		return;
+
+	MAC_PERFORM(devfs_label_update, mp, de, de->dn_label, vp,
+	    vp->v_label);
+}
+
+int
+mac_vnode_label_associate(struct mount *mp, struct vnode *vp, vfs_context_t ctx)
+{
+	struct devnode *dnp;
+	struct fdescnode *fnp;
+	int error = 0;
+
+	if (!mac_vnode_enforce)
+		return (error);
+
+	/* XXX: should not inspect v_tag in kernel! */
+	switch (vp->v_tag) {
+	case VT_DEVFS:
+		dnp = VTODN(vp);
+		mac_vnode_label_associate_devfs(mp, dnp, vp);
+		break;
+	case VT_FDESC:
+		fnp = VTOFDESC(vp);
+		error = mac_vnode_label_associate_fdesc(mp, fnp, vp, ctx);
+		break;
+	default:
+		error = mac_vnode_label_associate_extattr(mp, vp);
+		break;
+	}
+
+	return (error);
+}
+
+void
+mac_vnode_label_associate_devfs(struct mount *mp, struct devnode *de,
+    struct vnode *vp)
+{
+	if (!mac_device_enforce)
+		return;
+
+	MAC_PERFORM(vnode_label_associate_devfs,
+	    mp, mp ? mp->mnt_mntlabel : NULL,
+	    de, de->dn_label,
+	    vp, vp->v_label);
+}
+
+int
+mac_vnode_label_associate_extattr(struct mount *mp, struct vnode *vp)
+{
+	int error;
+
+	MAC_CHECK(vnode_label_associate_extattr, mp, mp->mnt_mntlabel, vp,
+	    vp->v_label);
+
+	return (error);
+}
+
+void
+mac_vnode_label_associate_singlelabel(struct mount *mp, struct vnode *vp)
+{
+
+	if (!mac_vnode_enforce)
+		return;
+
+	MAC_PERFORM(vnode_label_associate_singlelabel, mp,
+	    mp ? mp->mnt_mntlabel : NULL, vp, vp->v_label);
+}
+
+int
+mac_vnode_notify_create(vfs_context_t ctx, struct mount *mp,
+    struct vnode *dvp, struct vnode *vp, struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_notify_create, cred, mp, mp->mnt_mntlabel,
+	    dvp, dvp->v_label, vp, vp->v_label, cnp);
+
+	return (error);
+}
+
+/*
+ * Extended attribute 'name' was updated via
+ * vn_setxattr() or vn_removexattr().  Allow the
+ * policy to update the vnode label.
+ */
+void
+mac_vnode_label_update_extattr(struct mount *mp, struct vnode *vp,
+    const char *name)
+{
+	int error = 0;
+
+	if (!mac_vnode_enforce)
+		return;
+
+	MAC_PERFORM(vnode_label_update_extattr, mp, mp->mnt_mntlabel, vp,
+		    vp->v_label, name);
+	if (error == 0)
+		return;
+
+	vnode_lock(vp);
+	vnode_relabel(vp);
+	vnode_unlock(vp);
+	return;
+}
+
+static int
+mac_vnode_label_store(vfs_context_t ctx, struct vnode *vp,
+    struct label *intlabel)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce &&
+	    !mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return 0;
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_label_store, cred, vp, vp->v_label, intlabel);
+
+	return (error);
+}
+
+void
+mac_cred_label_update_execve(vfs_context_t ctx, kauth_cred_t new, struct vnode *vp,
+    struct label *scriptvnodelabel, struct label *execl)
+{
+	kauth_cred_t cred;
+
+	if (!mac_proc_enforce && !mac_vnode_enforce)
+		return;	
+
+	/* mark the new cred to indicate "matching" includes the label */
+	new->cr_flags |= CRF_MAC_ENFORCE;
+
+	cred = vfs_context_ucred(ctx);
+	MAC_PERFORM(cred_label_update_execve, cred, new, vp, vp->v_label,
+	    scriptvnodelabel, execl);
+}
+
+int
+mac_cred_check_label_update_execve(vfs_context_t ctx, struct vnode *vp,
+    struct label *scriptvnodelabel, struct label *execlabel, struct proc *p)
+{
+	kauth_cred_t cred;
+	int result = 0;
+
+	if (!mac_proc_enforce && !mac_vnode_enforce)
+		return result;
+
+	cred = vfs_context_ucred(ctx);
+	MAC_BOOLEAN(cred_check_label_update_execve, ||, cred, vp, vp->v_label,
+		    scriptvnodelabel, execlabel, p);
+
+	return (result);
+}
+
+int
+mac_vnode_check_access(vfs_context_t ctx, struct vnode *vp,
+    int acc_mode)
+{
+	kauth_cred_t cred;
+	int error;
+	int mask;
+
+	if (!mac_vnode_enforce ||
+	    !mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return 0;
+
+	cred = vfs_context_ucred(ctx);
+	/* Convert {R,W,X}_OK values to V{READ,WRITE,EXEC} for entry points */
+	mask = ACCESS_MODE_TO_VNODE_MASK(acc_mode);
+	MAC_CHECK(vnode_check_access, cred, vp, vp->v_label, mask);
+	return (error);
+ }
+
+int
+mac_vnode_check_chdir(vfs_context_t ctx, struct vnode *dvp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+	    !mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_chdir, cred, dvp, dvp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_chroot(vfs_context_t ctx, struct vnode *dvp,
+    struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_chroot, cred, dvp, dvp->v_label, cnp);
+	return (error);
+}
+
+int
+mac_vnode_check_create(vfs_context_t ctx, struct vnode *dvp,
+    struct componentname *cnp, struct vnode_attr *vap)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_create, cred, dvp, dvp->v_label, cnp, vap);
+	return (error);
+}
+
+int
+mac_vnode_check_unlink(vfs_context_t ctx, struct vnode *dvp, struct vnode *vp,
+    struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_unlink, cred, dvp, dvp->v_label, vp,
+	    vp->v_label, cnp);
+	return (error);
+}
+#if 0
+int
+mac_vnode_check_deleteacl(vfs_context_t ctx, struct vnode *vp,
+    acl_type_t type)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_deleteacl, cred, vp, vp->v_label, type);
+	return (error);
+}
+#endif
+
+int
+mac_vnode_check_deleteextattr(vfs_context_t ctx, struct vnode *vp,
+    const char *name)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_deleteextattr, cred, vp, vp->v_label, name);
+	return (error);
+}
+int
+mac_vnode_check_exchangedata(vfs_context_t ctx,
+    struct vnode *v1, struct vnode *v2)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_exchangedata, cred, v1, v1->v_label, 
+	    v2, v2->v_label);
+
+	return (error);
+}
+
+#if 0
+int
+mac_vnode_check_getacl(vfs_context_t ctx, struct vnode *vp, acl_type_t type)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_getacl, cred, vp, vp->v_label, type);
+	return (error);
+}
+#endif
+
+int
+mac_vnode_check_getattrlist(vfs_context_t ctx, struct vnode *vp,
+    struct attrlist *alist)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_getattrlist, cred, vp, vp->v_label, alist);
+
+	/* Falsify results instead of returning error? */
+	return (error);
+}
+
+int
+mac_vnode_check_exec(vfs_context_t ctx, struct vnode *vp,
+    struct image_params *imgp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || !mac_proc_enforce)
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_exec, cred, vp, vp->v_label,
+		  (imgp != NULL) ? imgp->ip_execlabelp : NULL, 
+		  (imgp != NULL) ? &imgp->ip_ndp->ni_cnd : NULL,
+		  (imgp != NULL) ? &imgp->ip_csflags : NULL);
+	return (error);
+}
+
+#if 0
+int
+mac_vnode_check_getacl(vfs_context_t ctx, struct vnode *vp, acl_type_t type)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_getacl, cred, vp, vp->v_label, type);
+	return (error);
+}
+#endif
+
+int
+mac_vnode_check_getextattr(vfs_context_t ctx, struct vnode *vp,
+    const char *name, struct uio *uio)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_getextattr, cred, vp, vp->v_label,
+	    name, uio);
+	return (error);
+}
+
+int
+mac_vnode_check_ioctl(vfs_context_t ctx, struct vnode *vp, u_int cmd)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_ioctl, cred, vp, vp->v_label, cmd);
+	return (error);
+}
+
+int
+mac_vnode_check_kqfilter(vfs_context_t ctx, kauth_cred_t file_cred,
+    struct knote *kn, struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_kqfilter, cred, file_cred, kn, vp,
+	    vp->v_label);
+
+	return (error);
+}
+
+int
+mac_vnode_check_link(vfs_context_t ctx, struct vnode *dvp,
+    struct vnode *vp, struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_link, cred, dvp, dvp->v_label, vp,
+	    vp->v_label, cnp);
+	return (error);
+}
+
+int
+mac_vnode_check_listextattr(vfs_context_t ctx, struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_listextattr, cred, vp, vp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_lookup(vfs_context_t ctx, struct vnode *dvp,
+    struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_lookup, cred, dvp, dvp->v_label, cnp);
+	return (error);
+}
+
+int
+mac_vnode_check_open(vfs_context_t ctx, struct vnode *vp, int acc_mode)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_open, cred, vp, vp->v_label, acc_mode);
+	return (error);
+}
+
+int
+mac_vnode_check_read(vfs_context_t ctx, struct ucred *file_cred,
+    struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_read, cred, file_cred, vp,
+	    vp->v_label);
+
+	return (error);
+}
+
+int
+mac_vnode_check_readdir(vfs_context_t ctx, struct vnode *dvp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_readdir, cred, dvp, dvp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_readlink(vfs_context_t ctx, struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_readlink, cred, vp, vp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_label_update(vfs_context_t ctx, struct vnode *vp,
+    struct label *newlabel)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_label_update, cred, vp, vp->v_label, newlabel);
+
+	return (error);
+}
+
+int
+mac_vnode_check_rename_from(vfs_context_t ctx, struct vnode *dvp,
+    struct vnode *vp, struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_rename_from, cred, dvp, dvp->v_label, vp,
+	    vp->v_label, cnp);
+	return (error);
+}
+
+int
+mac_vnode_check_rename_to(vfs_context_t ctx, struct vnode *dvp,
+    struct vnode *vp, int samedir, struct componentname *cnp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_rename_to, cred, dvp, dvp->v_label, vp,
+	    vp != NULL ? vp->v_label : NULL, samedir, cnp);
+	return (error);
+}
+
+int
+mac_vnode_check_revoke(vfs_context_t ctx, struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_revoke, cred, vp, vp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_select(vfs_context_t ctx, struct vnode *vp, int which)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_select, cred, vp, vp->v_label, which);
+	return (error);
+}
+
+#if 0
+int
+mac_vnode_check_setacl(vfs_context_t ctx, struct vnode *vp, acl_type_t type,
+    struct acl *acl)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setacl, cred, vp, vp->v_label, type, acl);
+	return (error);
+}
+#endif
+
+int
+mac_vnode_check_setattrlist(vfs_context_t ctx, struct vnode *vp,
+    struct attrlist *alist)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setattrlist, cred, vp, vp->v_label, alist);
+	return (error);
+}
+
+int
+mac_vnode_check_setextattr(vfs_context_t ctx, struct vnode *vp,
+    const char *name, struct uio *uio)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setextattr, cred, vp, vp->v_label,
+	    name, uio);
+	return (error);
+}
+
+int
+mac_vnode_check_setflags(vfs_context_t ctx, struct vnode *vp, u_long flags)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setflags, cred, vp, vp->v_label, flags);
+	return (error);
+}
+
+int
+mac_vnode_check_setmode(vfs_context_t ctx, struct vnode *vp, mode_t mode)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setmode, cred, vp, vp->v_label, mode);
+	return (error);
+}
+
+int
+mac_vnode_check_setowner(vfs_context_t ctx, struct vnode *vp, uid_t uid,
+    gid_t gid)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setowner, cred, vp, vp->v_label, uid, gid);
+	return (error);
+}
+
+int
+mac_vnode_check_setutimes(vfs_context_t ctx, struct vnode *vp,
+    struct timespec atime, struct timespec mtime)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_setutimes, cred, vp, vp->v_label, atime,
+	    mtime);
+	return (error);
+}
+
+int
+mac_vnode_check_stat(vfs_context_t ctx, struct ucred *file_cred,
+    struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_stat, cred, file_cred, vp,
+	    vp->v_label);
+	return (error);
+}
+
+int
+mac_vnode_check_truncate(vfs_context_t ctx, struct ucred *file_cred,
+    struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_truncate, cred, file_cred, vp,
+	    vp->v_label);
+
+	return (error);
+}
+
+int
+mac_vnode_check_write(vfs_context_t ctx, struct ucred *file_cred,
+    struct vnode *vp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(vnode_check_write, cred, file_cred, vp, vp->v_label);
+
+	return (error);
+}
+
+void
+mac_vnode_label_update(vfs_context_t ctx, struct vnode *vp, struct label *newlabel)
+{
+	kauth_cred_t cred = vfs_context_ucred(ctx);
+
+	vnode_lock(vp);
+	MAC_PERFORM(vnode_label_update, cred, vp, vp->v_label, newlabel);
+	vnode_unlock(vp);
+}
+
+void
+mac_mount_label_associate(vfs_context_t ctx, struct mount *mp)
+{
+	kauth_cred_t cred = vfs_context_ucred(ctx);
+
+	/* XXX: eventually this logic may be handled by the policy? */
+
+	/* We desire MULTILABEL for the root filesystem. */
+	if ((mp->mnt_flag & MNT_ROOTFS) && 
+	    (strcmp(mp->mnt_vfsstat.f_fstypename, "hfs") == 0))
+		mp->mnt_flag |= MNT_MULTILABEL;
+
+	/* MULTILABEL on DEVFS. */
+	if (strcmp(mp->mnt_vfsstat.f_fstypename, "devfs") == 0)
+		mp->mnt_flag |= MNT_MULTILABEL;
+
+	/* MULTILABEL on FDESC pseudo-filesystem. */
+	if (strcmp(mp->mnt_vfsstat.f_fstypename, "fdesc") == 0)
+		mp->mnt_flag |= MNT_MULTILABEL;
+
+	/* MULTILABEL on all NFS filesystems. */
+	if (strcmp(mp->mnt_vfsstat.f_fstypename, "nfs") == 0)
+		mp->mnt_flag |= MNT_MULTILABEL;
+
+	/* MULTILABEL on all AFP filesystems. */
+	if (strcmp(mp->mnt_vfsstat.f_fstypename, "afpfs") == 0)
+		mp->mnt_flag |= MNT_MULTILABEL;
+
+	if (mp->mnt_vtable != NULL) {
+		/* Any filesystem that supports native XATTRs. */
+		if ((mp->mnt_vtable->vfc_vfsflags & VFC_VFSNATIVEXATTR))
+			mp->mnt_flag |= MNT_MULTILABEL;
+
+		/* Filesystem does not support multilabel. */
+		if ((mp->mnt_vtable->vfc_vfsflags & VFC_VFSNOMACLABEL) &&
+		    (mp->mnt_flag & MNT_MULTILABEL))
+			mp->mnt_flag &= ~MNT_MULTILABEL;
+	}
+
+	MAC_PERFORM(mount_label_associate, cred, mp, mp->mnt_mntlabel);
+#if MAC_DEBUG
+	printf("MAC Framework enabling %s support: %s -> %s (%s)\n",
+		mp->mnt_flag & MNT_MULTILABEL ? "multilabel" : "singlelabel", 
+		mp->mnt_vfsstat.f_mntfromname,
+		mp->mnt_vfsstat.f_mntonname,
+		mp->mnt_vfsstat.f_fstypename);
+#endif
+}
+
+int
+mac_mount_check_mount(vfs_context_t ctx, struct vnode *vp,
+    struct componentname *cnp, const char *vfc_name)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_mount, cred, vp, vp->v_label, cnp, vfc_name);
+
+	return (error);
+}
+
+int
+mac_mount_check_remount(vfs_context_t ctx, struct mount *mp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_remount, cred, mp, mp->mnt_mntlabel);
+
+	return (error);
+}
+
+int
+mac_mount_check_umount(vfs_context_t ctx, struct mount *mp)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_umount, cred, mp, mp->mnt_mntlabel);
+
+	return (error);
+}
+
+int
+mac_mount_check_getattr(vfs_context_t ctx, struct mount *mp, 
+    struct vfs_attr *vfa)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_getattr, cred, mp, mp->mnt_mntlabel, vfa);
+	return (error);
+}
+
+int
+mac_mount_check_setattr(vfs_context_t ctx, struct mount *mp, 
+    struct vfs_attr *vfa)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_setattr, cred, mp, mp->mnt_mntlabel, vfa);
+	return (error);
+}
+
+int
+mac_mount_check_stat(vfs_context_t ctx, struct mount *mount)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_stat, cred, mount, mount->mnt_mntlabel);
+
+	return (error);
+}
+
+int
+mac_mount_check_label_update(vfs_context_t ctx, struct mount *mount)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_label_update, cred, mount, mount->mnt_mntlabel);
+
+	return (error);
+}
+
+int
+mac_mount_check_fsctl(vfs_context_t ctx, struct mount *mp, u_int cmd)
+{
+	kauth_cred_t cred;
+	int error;
+
+	if (!mac_vnode_enforce || 
+		!mac_context_check_enforce(ctx, MAC_VNODE_ENFORCE))
+		return (0);
+
+	cred = vfs_context_ucred(ctx);
+	MAC_CHECK(mount_check_fsctl, cred, mp, mp->mnt_mntlabel, cmd);
+
+	return (error);
+}
+
+void
+mac_devfs_label_associate_device(dev_t dev, struct devnode *de,
+    const char *fullpath)
+{
+	if (!mac_device_enforce)
+		return;
+
+	MAC_PERFORM(devfs_label_associate_device, dev, de, de->dn_label,
+	    fullpath);
+}
+
+void
+mac_devfs_label_associate_directory(const char *dirname, int dirnamelen,
+    struct devnode *de, const char *fullpath)
+{
+	if (!mac_device_enforce)
+		return;
+
+	MAC_PERFORM(devfs_label_associate_directory, dirname, dirnamelen, de,
+	    de->dn_label, fullpath);
+}
+
+int
+vn_setlabel(struct vnode *vp, struct label *intlabel, vfs_context_t context)
+{
+	int error;
+
+	if (!mac_vnode_enforce)
+		return (0);
+
+	if (vp->v_mount == NULL) {
+		printf("vn_setlabel: null v_mount\n");
+		if (vp->v_type != VNON)
+			printf("vn_setlabel: null v_mount with non-VNON\n");
+		return (EBADF);
+	}
+
+	if ((vp->v_mount->mnt_flag & MNT_MULTILABEL) == 0)
+		return (ENOTSUP);
+
+	/*
+	 * Multi-phase commit.  First check the policies to confirm the
+	 * change is OK.  Then commit via the filesystem.  Finally,
+	 * update the actual vnode label.  Question: maybe the filesystem
+	 * should update the vnode at the end as part of VNOP_SETLABEL()?
+	 */
+	error = mac_vnode_check_label_update(context, vp, intlabel);
+	if (error)
+		return (error);
+
+	error = VNOP_SETLABEL(vp, intlabel, context);
+	if (error == ENOTSUP) {
+		error = mac_vnode_label_store(context, vp,
+						   intlabel);
+		if (error) {
+			printf("%s: mac_vnode_label_store failed %d\n",
+				__func__, error);
+			return (error);
+		}
+		mac_vnode_label_update(context, vp, intlabel);
+	} else
+	if (error) {
+		printf("vn_setlabel: vop setlabel failed %d\n", error);
+		return (error);
+	}
+
+	return (0);
+}
+
+int
+mac_vnode_label_associate_fdesc(struct mount *mp, struct fdescnode *fnp,
+    struct vnode *vp, vfs_context_t ctx)
+{
+	struct fileproc *fp;
+	struct socket *so;
+	struct pipe *cpipe;
+	struct vnode *fvp;
+	struct proc *p;
+	int error;
+
+	error = 0;
+
+	/*
+	 * If no backing file, let the policy choose which label to use.
+	 */
+	if (fnp->fd_fd == -1) {
+		MAC_PERFORM(vnode_label_associate_file, vfs_context_ucred(ctx),
+		    mp, mp->mnt_mntlabel, NULL, NULL, vp, vp->v_label);
+		return (0);
+	}
+
+	p = vfs_context_proc(ctx);
+	error = fp_lookup(p, fnp->fd_fd, &fp, 0);
+	if (error)
+		return (error);
+
+	if (fp->f_fglob == NULL) {
+		error = EBADF;
+		goto out;
+	}
+
+	switch (fp->f_fglob->fg_type) {
+	case DTYPE_VNODE:
+		fvp = (struct vnode *)fp->f_fglob->fg_data;
+		if ((error = vnode_getwithref(fvp)))
+			goto out;
+		MAC_PERFORM(vnode_label_copy, fvp->v_label, vp->v_label);
+		(void)vnode_put(fvp);
+		break;
+	case DTYPE_SOCKET:
+		so = (struct socket *)fp->f_fglob->fg_data;
+		socket_lock(so, 1);
+		MAC_PERFORM(vnode_label_associate_socket,
+			    vfs_context_ucred(ctx), (socket_t)so, so->so_label,
+		    vp, vp->v_label);
+		socket_unlock(so, 1);
+		break;
+	case DTYPE_PSXSHM:
+		pshm_label_associate(fp, vp, ctx);
+		break;
+	case DTYPE_PSXSEM:
+		psem_label_associate(fp, vp, ctx);
+		break;
+	case DTYPE_PIPE:
+		cpipe = (struct pipe *)fp->f_fglob->fg_data;
+		/* kern/sys_pipe.c:pipe_select() suggests this test. */
+		if (cpipe == (struct pipe *)-1) {
+			error = EINVAL;
+			goto out;
+		}
+		PIPE_LOCK(cpipe);
+		MAC_PERFORM(vnode_label_associate_pipe, vfs_context_ucred(ctx),
+		    cpipe, cpipe->pipe_label, vp, vp->v_label);
+		PIPE_UNLOCK(cpipe);
+		break;
+	case DTYPE_KQUEUE:
+	case DTYPE_FSEVENTS:
+	default:
+		MAC_PERFORM(vnode_label_associate_file, vfs_context_ucred(ctx),
+		    mp, mp->mnt_mntlabel, fp->f_fglob, fp->f_fglob->fg_label,
+		    vp, vp->v_label);
+		break;
+	}
+out:
+	fp_drop(p, fnp->fd_fd, fp, 0);
+	return (error);
+}
diff --git a/security/mac_vfs_subr.c b/security/mac_vfs_subr.c
new file mode 100644
index 000000000..e418342b0
--- /dev/null
+++ b/security/mac_vfs_subr.c
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2007 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+#include <sys/param.h>
+#include <sys/vnode.h>
+#include <sys/vnode_internal.h>
+#include <sys/kauth.h>
+#include <sys/namei.h>
+#include <sys/mount.h>
+#include <sys/mount_internal.h>
+#include <sys/uio_internal.h>
+#include <sys/xattr.h>
+
+#include <security/mac_internal.h>
+
+/*
+ * Caller holds reference or sets VNODE_LABEL_NEEDREF to non-zero.
+ *
+ * Function will drop lock and reference on return.
+ */
+int
+vnode_label(struct mount *mp, struct vnode *dvp, struct vnode *vp,
+	    struct componentname *cnp, int flags, vfs_context_t ctx)
+{
+	int error;
+
+	error = 0;
+
+	vnode_lock(vp);
+
+	if (vp->v_lflag & VL_LABELED) {
+		if (!(flags & VNODE_LABEL_NEEDREF))
+			vnode_put_locked(vp);
+		vnode_unlock(vp);
+		return (0);
+	}
+
+	if ((flags & VNODE_LABEL_NEEDREF) && vnode_get_locked(vp)) {
+		vnode_unlock(vp);
+		return (ENOENT);
+	}
+
+	if ((vp->v_lflag & VL_LABEL) == 0) {
+		vp->v_lflag |= VL_LABEL;
+
+		/* Could sleep on disk I/O, drop lock. */
+		vnode_unlock(vp);
+		if (flags & VNODE_LABEL_CREATE)
+			error = mac_vnode_notify_create(ctx,
+			    mp, dvp, vp, cnp);
+		else
+			error = mac_vnode_label_associate(mp, vp, ctx);
+		vnode_lock(vp);
+
+		if ((error == 0) && (vp->v_flag & VNCACHEABLE))
+			vp->v_lflag |= VL_LABELED;
+		vp->v_lflag &= ~VL_LABEL;
+
+		if (vp->v_lflag & VL_LABELWAIT) {
+			vp->v_lflag &= ~VL_LABELWAIT;
+			wakeup(vp->v_label);
+		}
+		vnode_put_locked(vp);
+		vnode_unlock(vp);
+	} else {
+		struct timespec ts;
+
+		ts.tv_sec = 10;
+		ts.tv_nsec = 0;
+
+		while (vp->v_lflag & VL_LABEL) {
+			vp->v_lflag |= VL_LABELWAIT;
+			error = msleep(vp->v_label, &vp->v_lock, PVFS|PDROP,
+			    "vnode_label", &ts);
+			vnode_lock(vp);
+			if (error == EWOULDBLOCK) {
+				vprint("vnode label timeout", vp);
+				break;
+			}
+		}
+		/* XXX: what should be done if labeling failed (above)? */
+		vnode_put_locked(vp);
+		vnode_unlock(vp);
+	}
+
+	return (error);
+}
+
+
+/*
+ * Clear the "labeled" flag on a VNODE.
+ * VNODE will have label re-associated upon
+ * next call to lookup().
+ *
+ * Caller verifies vfs_flags(vnode_mount(vp)) & MNT_MULTILABEL
+ * Caller holds vnode lock.
+ */
+void
+vnode_relabel(struct vnode *vp)
+{
+
+	/* Wait for any other labeling to complete. */
+	while (vp->v_lflag & VL_LABEL) {
+		vp->v_lflag |= VL_LABELWAIT;
+		(void)msleep(vp->v_label, &vp->v_lock, PVFS, "vnode_relabel", 0);
+	}
+
+	/* Clear labeled flag */
+	vp->v_lflag &= ~VL_LABELED;
+
+	return;
+}
+
+/*
+ * VFS XATTR helpers.
+ */
+
+int
+mac_vnop_setxattr (struct vnode *vp, const char *name, char *buf, size_t len)
+{
+	vfs_context_t ctx;
+	int options = XATTR_NOSECURITY;
+	char uio_buf[ UIO_SIZEOF(1) ];
+        uio_t auio;
+	int error;
+
+	if (vfs_isrdonly(vp->v_mount))
+		return (EROFS);
+
+	ctx = vfs_context_current();
+	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_WRITE,
+				    &uio_buf[0], sizeof(uio_buf));
+	uio_addiov(auio, CAST_USER_ADDR_T(buf), len);
+
+	error = vn_setxattr(vp, name, auio, options, ctx);
+
+	return (error);
+}
+
+int
+mac_vnop_getxattr (struct vnode *vp, const char *name, char *buf, size_t len,
+		   size_t *attrlen)
+{
+	vfs_context_t ctx = vfs_context_current();
+	int options = XATTR_NOSECURITY;
+	char uio_buf[ UIO_SIZEOF(1) ];
+        uio_t auio;
+	int error;
+
+	auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
+				    &uio_buf[0], sizeof(uio_buf));
+	uio_addiov(auio, CAST_USER_ADDR_T(buf), len);
+
+	error = vn_getxattr(vp, name, auio, attrlen, options, ctx);
+	*attrlen = len - uio_resid(auio);
+
+	return (error);
+}
+
+int
+mac_vnop_removexattr (struct vnode *vp, const char *name)
+{
+	vfs_context_t ctx = vfs_context_current();
+	int options = XATTR_NOSECURITY;
+	int error;
+
+	if (vfs_isrdonly(vp->v_mount))
+		return (EROFS);
+
+	error = vn_removexattr(vp, name, options, ctx);
+
+	return (error);
+}
diff --git a/tools/cred_dump_backtraces.c b/tools/cred_dump_backtraces.c
new file mode 100644
index 000000000..12daa525d
--- /dev/null
+++ b/tools/cred_dump_backtraces.c
@@ -0,0 +1,80 @@
+/* quick and dirty hack to grab credential backtrace info from kernel via sysctl.
+ * sysctl is only defined if xnu is built with DEBUG_CRED defined.
+ * The current version of this is used to target a specific credential and gather
+ * backtrace info on all references and unreferences.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <bsm/audit.h>
+
+/* bad!  this is replicated in kern_credential.c.  make sure they stay in sync!
+ * Or better yet have commone header file? 
+ */
+#define MAX_STACK_DEPTH 8
+struct cred_backtrace {
+	int				depth;
+	uint32_t		stack[ MAX_STACK_DEPTH ];
+};
+typedef struct cred_backtrace cred_backtrace;
+
+struct cred_debug_buffer {
+	int				next_slot;
+	cred_backtrace	stack_buffer[ 1 ];	
+};
+typedef struct cred_debug_buffer cred_debug_buffer;
+
+
+main( int argc, char *argv[] )
+{
+	int				err, i, j;
+	size_t			len;
+    char 			*my_bufferp = NULL;
+	cred_debug_buffer	*bt_buffp;
+	cred_backtrace		*btp;
+
+	/* get size of buffer we will need */
+	len = 0;
+    err = sysctlbyname( "kern.cred_bt", NULL, &len, NULL, 0 );
+	if ( err != 0 ) {
+		printf( "sysctl failed  \n" );
+		printf( "\terrno %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+	
+	/* get a buffer for our back traces */
+	my_bufferp = malloc( len );
+	if ( my_bufferp == NULL ) {
+		printf( "malloc error %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+    err = sysctlbyname( "kern.cred_bt", my_bufferp, &len, NULL, 0 );
+	if ( err != 0 ) {
+		printf( "sysctl 2 failed  \n" );
+		printf( "\terrno %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+
+	bt_buffp = (cred_debug_buffer *) my_bufferp;
+	btp = &bt_buffp->stack_buffer[ 0 ];
+	
+	printf("number of traces %d \n", bt_buffp->next_slot);
+	for ( i = 0; i < bt_buffp->next_slot; i++, btp++ ) {
+		printf("[%d] ", i);
+		for ( j = 0; j < btp->depth; j++ ) {
+			printf("%p ", btp->stack[ j ]);
+		}
+		printf("\n");
+	}
+	
+	return;
+}
+
diff --git a/tools/cred_dump_creds.c b/tools/cred_dump_creds.c
new file mode 100644
index 000000000..560040679
--- /dev/null
+++ b/tools/cred_dump_creds.c
@@ -0,0 +1,117 @@
+/* quick and dirty hack to grab all credentials in the cred hash table
+ * from kernel via sysctl.
+ * sysctl is only defined if xnu is built with DEBUG_CRED defined.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <bsm/audit.h>
+
+/* bad!  this is replicated in kern_credential.c.  make sure they stay in sync!
+ * Or better yet have commone header file? 
+ */
+struct debug_ucred {
+	uint32_t	credp;
+	uint32_t	cr_ref;				/* reference count */
+	uid_t		cr_uid;				/* effective user id */
+	uid_t		cr_ruid;			/* real user id */
+	uid_t		cr_svuid;			/* saved user id */
+	short		cr_ngroups;			/* number of groups in advisory list */
+	gid_t		cr_groups[NGROUPS];	/* advisory group list */
+	gid_t		cr_rgid;			/* real group id */
+	gid_t		cr_svgid;			/* saved group id */
+	uid_t		cr_gmuid;			/* UID for group membership purposes */
+	struct auditinfo cr_au;		/* user auditing data */
+	uint32_t	cr_label;			/* MACF label */
+	int			cr_flags;			/* flags on credential */
+};
+typedef struct debug_ucred debug_ucred;
+
+void dump_cred_hash_table( debug_ucred * credp, size_t buf_size );				
+void dump_cred( debug_ucred * credp );
+
+
+main( int argc, char *argv[] )
+{
+	int				err;
+	size_t			len;
+    char 			*my_bufferp = NULL;
+
+	/* get size of buffer we will need */
+	len = 0;
+    err = sysctlbyname( "kern.dump_creds", NULL, &len, NULL, 0 );
+	if ( err != 0 ) {
+		printf( "sysctl failed  \n" );
+		printf( "\terrno %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+	
+	/* get a buffer for our credentials.  need some spare room since table could have grown */
+	my_bufferp = malloc( len );
+	if ( my_bufferp == NULL ) {
+		printf( "malloc error %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+    err = sysctlbyname( "kern.dump_creds", my_bufferp, &len, NULL, 0 );
+	if ( err != 0 ) {
+		printf( "sysctl 2 failed  \n" );
+		printf( "\terrno %d - \"%s\" \n", errno, strerror( errno ) );
+		return;
+	}
+	dump_cred_hash_table( (debug_ucred *)my_bufferp, len );
+
+	return;
+}
+
+void dump_cred_hash_table( debug_ucred * credp, size_t buf_size )
+{
+	int		i, my_count = (buf_size / sizeof(debug_ucred));
+	
+	printf("\n\t dumping credential hash table - total creds %d \n", 
+			my_count);
+	for (i = 0; i < my_count; i++) {
+		printf("[%02d] ", i);
+		dump_cred( credp );
+		credp++;
+	}
+	return;
+}
+
+void dump_cred( debug_ucred * credp )
+{
+	int		i;
+	printf("%p ", credp->credp);
+	printf("%lu ", credp->cr_ref);
+	printf("%d ", credp->cr_uid);
+	printf("%d ", credp->cr_ruid);
+	printf("%d ", credp->cr_svuid);
+	printf("%d g[", credp->cr_ngroups);
+	for (i = 0; i < credp->cr_ngroups; i++) {
+		printf("%d", credp->cr_groups[i]);
+		if ( (i + 1) < credp->cr_ngroups ) {
+			printf(" ");
+		}
+	}
+	printf("] %d ", credp->cr_rgid);
+	printf("%d ", credp->cr_svgid);
+	printf("%d ", credp->cr_gmuid);
+	printf("a[%d ", credp->cr_au.ai_auid);
+	printf("%d ", credp->cr_au.ai_mask.am_success);
+	printf("%d ", credp->cr_au.ai_mask.am_failure);
+	printf("%d ", credp->cr_au.ai_termid.port);
+	printf("%d ", credp->cr_au.ai_termid.machine);
+	printf("%d ", credp->cr_au.ai_asid);
+	printf("] ");
+	printf("%p ", credp->cr_label);
+	printf("0x%08x \n", credp->cr_flags);
+	printf("\n");
+	return;
+}
diff --git a/tools/lockstat/Makefile b/tools/lockstat/Makefile
new file mode 100644
index 000000000..f67ee3ddf
--- /dev/null
+++ b/tools/lockstat/Makefile
@@ -0,0 +1,10 @@
+CFLAGS=-g -Os -arch ppc -arch i386
+
+TARGETS	= lockstat
+
+all:	$(TARGETS)
+
+lockstat: lockstat.c
+	${CC} ${CFLAGS} -o $@ $?
+clean:
+	rm -rf $(TARGETS)
diff --git a/tools/lockstat/lockstat.c b/tools/lockstat/lockstat.c
new file mode 100644
index 000000000..2ec7f324c
--- /dev/null
+++ b/tools/lockstat/lockstat.c
@@ -0,0 +1,457 @@
+#include <mach/mach.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <time.h>
+#include <mach/error.h>
+#include <mach/mach_error.h>
+#include <mach/mig_errors.h>
+#include <mach/machine.h>
+#include <mach/processor_info.h>
+#include <assert.h>
+#include <nlist.h>
+#include <fcntl.h>
+#include <string.h>
+#include <mach/mach.h>
+#include <mach/host_info.h>
+
+/*
+ *	lockstat.c
+ *
+ *	Utility to display kernel lock contention statistics.
+ *	Usage:
+ *	lockstat [all, spin, mutex, rw, <lock group name>] {<repeat interval>} {abs}
+ *
+ *	Argument 1 specifies the type of lock to display contention statistics
+ *	for; alternatively, a lock group (a logically grouped set of locks,
+ *	which can encompass multiple types of locks) can be specified by name.
+ *	When argument 1 is "all", statistics are displayed for all lock groups
+ *	which have statistics enabled.
+ *	Lock types include mutexes, reader-writer locks and spin locks.
+ *	Note that support for gathering contention statistics may not be present
+ *	for all types of locks on all platforms.
+ *
+ *	Argument 2 specifies a periodic interval. The program will display an
+ *	updated list of statistics every <repeat interval> seconds. This
+ *	argument is optional. The updates display the deltas from the previous
+ *	set of statistics, unless "abs" is specified as argument 3.
+ *
+ *	Argument 3, if "abs", causes the periodically refreshed lock statistics
+ *	to be displayed as absolute values rather than deltas from the previous
+ *	display.
+ *
+ *	Types of statistics:
+ *	Acquisitions: These can include both normal acquisitions, as well
+ *	as acquisition attempts. These are listed in the first column.
+ *	Examples include calls to lck_mtx_lock and lck_mtx_try_lock
+ *	Misses: Incremented if  a lock acquisition attempt failed, due to
+ *	contention.
+ *	Waits (Meaningful only for lock types that can block): Incremented
+ *	if a lock acquisition attempt proceeded to block.
+ *
+ *	Direct Waits (currently implemented only on i386): For adaptive
+ *	locks, such as mutexes, incremented if the owner of the mutex
+ *	wasn't active on another processor at the time of the lock
+ *	attempt. This indicates that no adaptive spin occurred.
+ */
+
+/*
+ * HISTORY
+ * 2005: Bernard Semeria
+ *		Created.
+ * 2006: Derek Kumar
+ *		Display i386 specific stats, fix incremental display, add
+ *		explanatory block comment.
+ */
+void usage(void);
+void print_spin_hdr(void);
+void print_spin(int requested, lockgroup_info_t *lockgroup);
+void print_all_spin(lockgroup_info_t *lockgroup);
+void print_mutex_hdr(void);
+void print_mutex(int requested, lockgroup_info_t *lockgroup);
+void print_all_mutex(lockgroup_info_t *lockgroup);
+void print_rw_hdr(void);
+void print_rw(int requested, lockgroup_info_t *lockgroup);
+void print_all_rw(lockgroup_info_t *lockgroup);
+void prime_lockgroup_deltas(void);
+void get_lockgroup_deltas(void);
+
+char *pgmname;
+mach_port_t host_control;
+
+lockgroup_info_t	*lockgroup_info, *lockgroup_start, *lockgroup_deltas;
+unsigned int		count;
+
+unsigned int		gDebug = 1;
+
+int
+main(int argc, char **argv)
+{
+	kern_return_t		kr;
+	int 			arg2;
+	unsigned int 		i;
+	int 			found;
+
+	setlinebuf(stdout);
+
+	pgmname = argv[0];
+	gDebug = (NULL != strstr(argv[0], "debug"));
+
+	host_control = mach_host_self();  
+
+	kr = host_lockgroup_info(host_control, &lockgroup_info, &count);
+
+	if (kr != KERN_SUCCESS)
+	{
+		mach_error("host_statistics", kr);
+		exit (EXIT_FAILURE);
+	}
+	if (gDebug) {
+		printf("count = %d\n", count);
+		for (i = 0; i < count; i++) {
+			printf("%s\n",lockgroup_info[i].lockgroup_name);
+		}
+	}
+
+	switch (argc) {
+	case 2:
+		if (strcmp(argv[1], "all") == 0) {
+			print_spin_hdr();
+			print_all_spin(lockgroup_info);
+			print_mutex_hdr();
+			print_all_mutex(lockgroup_info);
+			print_rw_hdr();
+			print_all_rw(lockgroup_info);
+		}
+		else if (strcmp(argv[1], "spin") == 0) {
+			print_spin_hdr();
+			print_all_spin(lockgroup_info);
+		}
+		else if (strcmp(argv[1], "mutex") == 0) {
+			print_mutex_hdr();
+			print_all_mutex(lockgroup_info);
+		}
+		else if (strcmp(argv[1], "rw") == 0) {
+			print_rw_hdr();
+			print_all_rw(lockgroup_info);
+		}
+		else {
+			found = 0;
+			for (i = 0;i < count;i++) {
+				if (strcmp(argv[1], lockgroup_info[i].lockgroup_name) == 0) {
+					found = 1;
+					print_spin_hdr();
+					print_spin(i, lockgroup_info);
+					print_mutex_hdr();
+					print_mutex(i, lockgroup_info);
+					print_rw_hdr();
+					print_rw(i, lockgroup_info);
+					break;
+				}
+			}
+			if (found == 0) 
+			{ usage(); }
+		}
+		break;	
+	case 3:
+		if (sscanf(argv[2], "%d", &arg2) != 1) {
+			usage();
+		}
+		if (arg2 < 0) {
+			usage();
+		}
+		prime_lockgroup_deltas();
+		if (strcmp(argv[1], "all") == 0) {
+
+			while (1) {
+				sleep(arg2);
+				get_lockgroup_deltas();
+				print_spin_hdr();
+				print_all_spin(lockgroup_deltas);
+				print_mutex_hdr();
+				print_all_mutex(lockgroup_deltas);
+				print_rw_hdr();
+				print_all_rw(lockgroup_deltas);
+			}
+		}
+		else if (strcmp(argv[1], "spin") == 0) {
+
+			while (1) {
+				sleep(arg2);
+				get_lockgroup_deltas();
+				print_spin_hdr();
+				print_all_spin(lockgroup_deltas);
+			}
+		}
+		else if (strcmp(argv[1], "mutex") == 0) {
+
+			while (1) {
+				sleep(arg2);
+				get_lockgroup_deltas();
+				print_mutex_hdr();
+				print_all_mutex(lockgroup_deltas);
+			}
+		}
+		else if (strcmp(argv[1], "rw") == 0) {
+
+			while (1) {
+				sleep(arg2);
+				get_lockgroup_deltas();
+				print_rw_hdr();
+				print_all_rw(lockgroup_deltas);
+			}
+		}
+		else {
+
+			found = 0;
+			for (i = 0;i < count;i++) {
+				if (strcmp(argv[1], lockgroup_info[i].lockgroup_name) == 0) {
+					found = 1;
+					while (1) {
+						sleep(arg2);
+						get_lockgroup_deltas();
+						print_spin_hdr();
+						print_spin(i, lockgroup_deltas);
+						print_mutex_hdr();
+						print_mutex(i, lockgroup_deltas);
+						print_rw_hdr();
+						print_rw(i, lockgroup_deltas);
+					}
+				}
+			}
+			if (found == 0)
+			{ usage(); }
+		}
+		break;
+	case 4:
+		if (strcmp(argv[3], "abs") != 0)
+		{ usage(); }
+		if (sscanf(argv[2], "%d", &arg2) != 1)
+		{ usage(); }
+		if (strcmp(argv[1], "all") == 0) {
+			while (1)
+			{
+				print_spin_hdr();
+				print_all_spin(lockgroup_info);
+				print_mutex_hdr();
+				print_all_mutex(lockgroup_info);
+				print_rw_hdr();
+				print_all_rw(lockgroup_info);
+				sleep(arg2);
+			}
+		}
+		else if (strcmp(argv[1], "spin") == 0) {
+			while (1)
+			{print_all_spin(lockgroup_info);
+				sleep(arg2);
+			}
+		}
+		else if (strcmp(argv[1], "mutex") == 0) {
+			print_mutex_hdr();
+			while (1)
+			{print_all_mutex(lockgroup_info);
+				sleep(arg2);
+			}
+		}
+		else if (strcmp(argv[1], "rw") == 0) {
+			print_rw_hdr();
+			while (1)
+			{print_all_rw(lockgroup_info);
+				sleep(arg2);
+			}
+		}
+		else {
+			found = 0;
+			for (i = 0;i < count;i++) {
+				if (strcmp(argv[1], lockgroup_info[i].lockgroup_name) == 0) {
+					found = 1;
+					while (1)
+					{
+						print_spin_hdr();
+						print_spin(i, lockgroup_info);
+						print_mutex_hdr();
+						print_mutex(i, lockgroup_info);
+						print_rw_hdr();
+						print_rw(i, lockgroup_info);
+						sleep(arg2);
+					}
+				}
+			}
+			if (found == 0)
+			{ usage(); }
+		}
+		break;
+	default:
+		usage();
+		break;
+	}	
+
+	exit(0);
+}
+ 
+void 
+usage()
+{
+	fprintf(stderr, "Usage: %s [all, spin, mutex, rw, <lock group name>] {<repeat interval>} {abs}\n", pgmname);
+	exit(EXIT_FAILURE);
+}
+
+void
+print_spin_hdr(void)
+{
+	printf("    Spinlock acquires           misses   Name\n");
+}
+
+void
+print_spin(int requested, lockgroup_info_t *lockgroup)
+{
+	lockgroup_info_t	*curptr = &lockgroup[requested];
+
+	if (curptr->lock_spin_cnt != 0 && curptr->lock_spin_util_cnt != 0) {
+		printf("%16lld ", curptr->lock_spin_util_cnt);
+		printf("%16lld   ", curptr->lock_spin_miss_cnt);
+		printf("%-14s\n", curptr->lockgroup_name);
+	}
+}
+
+void
+print_all_spin(lockgroup_info_t *lockgroup)
+{
+	unsigned int		i;
+
+	for (i = 0;i < count;i++)
+		print_spin(i, lockgroup);
+        printf("\n");
+}
+
+void
+print_mutex_hdr(void)
+{
+#if defined(__i386__)
+	printf("Mutex lock attempts  Misses      Waits Direct Waits Name\n");
+#else
+        printf("     mutex locks           misses            waits   name\n");
+#endif
+}
+
+void
+print_mutex(int requested, lockgroup_info_t *lockgroup)
+{
+	lockgroup_info_t	*curptr = &lockgroup[requested];
+
+	if (curptr->lock_mtx_cnt != 0 && curptr->lock_mtx_util_cnt != 0) {
+		printf("%16lld ", curptr->lock_mtx_util_cnt);
+#if defined(__i386__)
+		printf("%10lld %10lld %10lld   ", curptr->lock_mtx_miss_cnt,  curptr->lock_mtx_wait_cnt, curptr->lock_mtx_held_cnt);
+#else
+		printf("%16lld %16lld   ", curptr->lock_mtx_miss_cnt,  curptr->lock_mtx_wait_cnt);
+#endif
+		printf("%-14s\n", curptr->lockgroup_name);
+	}
+}
+
+void
+print_all_mutex(lockgroup_info_t *lockgroup)
+{
+	unsigned int		i;
+
+	for (i = 0;i < count;i++)
+		print_mutex(i, lockgroup);
+        printf("\n");
+
+}
+
+void
+print_rw_hdr(void)
+{
+	printf("        RW locks           Misses            Waits   Name\n");
+}
+
+void
+print_rw(int requested, lockgroup_info_t *lockgroup)
+{
+	lockgroup_info_t	*curptr = &lockgroup[requested];
+
+	if (curptr->lock_rw_cnt != 0 && curptr->lock_rw_util_cnt != 0) {
+		printf("%16lld ", curptr->lock_rw_util_cnt);
+		printf("%16lld %16lld   ", curptr->lock_rw_miss_cnt,  curptr->lock_rw_wait_cnt);
+		printf("%-14s\n", curptr->lockgroup_name);
+	}
+}
+
+void
+print_all_rw(lockgroup_info_t *lockgroup)
+{
+	unsigned int		i;
+
+	for (i = 0;i < count;i++)
+		print_rw(i, lockgroup);
+        printf("\n");
+
+}
+
+void
+prime_lockgroup_deltas(void)
+{
+	lockgroup_start = calloc(count, sizeof(lockgroup_info_t));
+	if (lockgroup_start == NULL) {
+		fprintf(stderr, "Can't allocate memory for lockgroup info\n");
+		exit (EXIT_FAILURE);
+	}
+	memcpy(lockgroup_start, lockgroup_info, count * sizeof(lockgroup_info_t));
+
+	lockgroup_deltas = calloc(count,  sizeof(lockgroup_info_t));
+	if (lockgroup_deltas == NULL) {
+		fprintf(stderr, "Can't allocate memory for lockgroup info\n");
+		exit (EXIT_FAILURE);
+	}
+}
+
+void
+get_lockgroup_deltas(void)
+{
+	kern_return_t 			kr;
+	unsigned int			i;
+
+	kr = host_lockgroup_info(host_control, &lockgroup_info, &count);
+
+	if (kr != KERN_SUCCESS)
+	{
+		mach_error("host_statistics", kr);
+		exit (EXIT_FAILURE);
+	}
+
+	memcpy(lockgroup_deltas, lockgroup_info, count * sizeof(lockgroup_info_t));
+	for (i = 0; i < count; i++) {
+		lockgroup_deltas[i].lock_spin_util_cnt =
+		    lockgroup_info[i].lock_spin_util_cnt -
+		    lockgroup_start[i].lock_spin_util_cnt;
+		lockgroup_deltas[i].lock_spin_miss_cnt =
+		    lockgroup_info[i].lock_spin_miss_cnt -
+		    lockgroup_start[i].lock_spin_miss_cnt;
+		lockgroup_deltas[i].lock_mtx_util_cnt =
+		    lockgroup_info[i].lock_mtx_util_cnt -
+		    lockgroup_start[i].lock_mtx_util_cnt;
+		lockgroup_deltas[i].lock_mtx_miss_cnt =
+		    lockgroup_info[i].lock_mtx_miss_cnt -
+		    lockgroup_start[i].lock_mtx_miss_cnt;
+		lockgroup_deltas[i].lock_mtx_wait_cnt =
+		    lockgroup_info[i].lock_mtx_wait_cnt -
+		    lockgroup_start[i].lock_mtx_wait_cnt;
+		lockgroup_deltas[i].lock_mtx_held_cnt =
+		    lockgroup_info[i].lock_mtx_held_cnt -
+		    lockgroup_start[i].lock_mtx_held_cnt;
+		lockgroup_deltas[i].lock_rw_util_cnt =
+		    lockgroup_info[i].lock_rw_util_cnt -
+		    lockgroup_start[i].lock_rw_util_cnt;
+		lockgroup_deltas[i].lock_rw_miss_cnt =
+		    lockgroup_info[i].lock_rw_miss_cnt -
+		    lockgroup_start[i].lock_rw_miss_cnt;
+		lockgroup_deltas[i].lock_rw_wait_cnt =
+		    lockgroup_info[i].lock_rw_wait_cnt -
+		    lockgroup_start[i].lock_rw_wait_cnt;
+	}
+	memcpy(lockgroup_start, lockgroup_info, count * sizeof(lockgroup_info_t));
+}
diff --git a/tools/tests/MMTest/MMtest.c b/tools/tests/MMTest/MMtest.c
new file mode 100644
index 000000000..b11b16fe8
--- /dev/null
+++ b/tools/tests/MMTest/MMtest.c
@@ -0,0 +1,516 @@
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <mach/mach.h>
+#include <mach/mach_error.h>
+#include <mach/notify.h>
+#include <servers/bootstrap.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/signal.h>
+
+#define MAX(A, B) ((A) < (B) ? (B) : (A))
+
+static __inline__ unsigned long long ReadTSR() {
+	union {
+		unsigned long long time64;
+		unsigned long word[2];
+	} now;
+#if defined(__i386__)
+	/* Read from Pentium and Pentium Pro 64-bit timestamp counter.
+	 * The counter is set to 0 at processor reset and increments on
+	 * every clock cycle. */
+	__asm__ volatile("rdtsc" : : : "eax", "edx");
+	__asm__ volatile("movl %%eax,%0" : "=m" (now.word[0]) : : "eax");
+	__asm__ volatile("movl %%edx,%0" : "=m" (now.word[1]) : : "edx");
+#elif defined(__ppc__)
+	/* Read from PowerPC 64-bit time base register. The increment
+	 * rate of the time base is implementation-dependent, but is
+	 * 1/4th the bus clock cycle on 603/604 processors. */
+	unsigned long t3;
+	do {
+		__asm__ volatile("mftbu %0" : "=r" (now.word[0]));
+		__asm__ volatile("mftb %0" : "=r" (now.word[1]));
+		__asm__ volatile("mftbu %0" : "=r" (t3));
+	} while (now.word[0] != t3);
+#else
+#warning Do not know how to read a time stamp register on this architecture
+	now.time64 = 0ULL;
+#endif
+	return now.time64;
+}
+
+typedef struct  {
+	unsigned int        msgt_name : 8,
+			    msgt_size : 8,
+			    msgt_number : 12,
+			    msgt_inline : 1,
+			    msgt_longform : 1,
+			    msgt_deallocate : 1,
+			    msgt_unused : 1;
+} mach_msg_type_t;
+
+typedef struct  {
+	mach_msg_type_t     msgtl_header;
+	unsigned short      msgtl_name;
+	unsigned short      msgtl_size;
+	natural_t           msgtl_number;
+} mach_msg_type_long_t;
+#define MACH_MSG_TYPE_INTEGER_32 0
+
+
+typedef struct {
+	mach_msg_header_t	header;
+	mach_msg_trailer_t	trailer;		// subtract this when sending
+} ipc_trivial_message;
+
+typedef struct {
+	mach_msg_header_t	header;
+	mach_msg_type_t		type;
+	u_int32_t		numbers[0];
+	mach_msg_trailer_t	trailer;		// subtract this when sending
+} ipc_inline_message;
+
+typedef struct {
+	mach_msg_header_t		header;
+	mach_msg_body_t			body;
+	mach_msg_ool_descriptor_t	descriptor;
+	mach_msg_trailer_t		trailer;	// subtract this when sending
+} ipc_complex_message;
+
+enum {
+	msg_type_trivial = 0,
+	msg_type_inline = 1,
+	msg_type_complex = 2
+};
+
+struct port_args {
+	int req_size;
+	mach_msg_header_t *req_msg;
+	int reply_size;
+	mach_msg_header_t *reply_msg;
+	mach_port_t port;
+};
+
+/* Global options */
+static int verbose;
+int oneway;
+int msg_type;
+int num_ints;
+int num_msgs;
+int num_clients;
+int client_delay;
+char *server_port_name;
+
+void signal_handler(int sig) {
+}
+
+void usage(const char *progname) {
+	fprintf(stderr, "usage: %s [options]\n", progname);
+	fprintf(stderr, "where options are:\n");
+	fprintf(stderr, "    -verbose\t\tbe verbose\n");
+	fprintf(stderr, "    -oneway\t\tdo not request return reply\n");
+	fprintf(stderr, "    -count num\t\tnumber of messages to send\n");
+	fprintf(stderr, "    -type trivial|inline|complex\ttype of messages to send\n");
+	fprintf(stderr, "    -numints num\tnumber of 32-bit ints to send in messages\n");
+	fprintf(stderr, "    -clients num\tnumber of client threads to run\n");
+	fprintf(stderr, "    -delay num\t\tmicroseconds to sleep clients between messages\n");
+	fprintf(stderr, "    -name portname\tname of port on which to communicate\n");
+	fprintf(stderr, "default values are:\n");
+	fprintf(stderr, "    . not verbose\n");
+	fprintf(stderr, "    . not oneway\n");
+	fprintf(stderr, "    . client sends 10000 messages\n");
+	fprintf(stderr, "    . inline message type\n");
+	fprintf(stderr, "    . 64 32-bit integers in inline/complex messages\n");
+	fprintf(stderr, "    . avail_cpus - 1 clients\n");
+	fprintf(stderr, "    . no delay\n");
+	fprintf(stderr, "    . port name 'TEST'\n");
+	exit(1);
+}
+
+void parse_args(int argc, char *argv[]) {
+	host_basic_info_data_t		info;
+	mach_msg_type_number_t		count;
+	kern_return_t			result;
+
+	/* Initialize defaults */
+	verbose = 0;
+	oneway = 0;
+	msg_type = msg_type_trivial;
+	num_ints = 64;
+	num_msgs = 10000;
+	client_delay = 0;
+	server_port_name = "TEST";
+
+	count = HOST_BASIC_INFO_COUNT;
+	result = host_info(mach_host_self(), HOST_BASIC_INFO, 
+			(host_info_t)&info, &count);
+	if (result == KERN_SUCCESS)
+		num_clients = MAX(1, info.avail_cpus - 1);
+	else
+		num_clients = 1;
+
+	const char *progname = argv[0];
+	argc--; argv++;
+	while (0 < argc) {
+		if (0 == strcmp("-verbose", argv[0])) {
+			verbose = 1;
+			argc--; argv++;
+		} else if (0 == strcmp("-oneway", argv[0])) {
+			oneway = 1;
+			argc--; argv++;
+		} else if (0 == strcmp("-type", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			if (0 == strcmp("trivial", argv[1])) {
+				msg_type = msg_type_trivial;
+			} else if (0 == strcmp("inline", argv[1])) {
+				msg_type = msg_type_inline;
+			} else if (0 == strcmp("complex", argv[1])) {
+				msg_type = msg_type_complex;
+			} else 
+				usage(progname);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-name", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			server_port_name = argv[1];
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-numints", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_ints = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-count", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_msgs = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		}  else if (0 == strcmp("-clients", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_clients = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-delay", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			client_delay = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else 
+			usage(progname);
+	}
+}
+
+void setup_server_ports(struct port_args *ports)
+{
+	kern_return_t ret = 0;
+	mach_port_t bsport;
+
+	ports->req_size = MAX(sizeof(ipc_inline_message) +  
+			sizeof(u_int32_t) * num_ints, 
+			sizeof(ipc_complex_message));
+	ports->reply_size = sizeof(ipc_trivial_message) - 
+		sizeof(mach_msg_trailer_t);
+	ports->req_msg = malloc(ports->req_size);
+	ports->reply_msg = malloc(ports->reply_size);
+
+	ret = mach_port_allocate(mach_task_self(), 
+			MACH_PORT_RIGHT_RECEIVE,  
+			&(ports->port));
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_allocate(): ", ret);
+		exit(1);
+	}
+
+	ret = mach_port_insert_right(mach_task_self(), 
+			ports->port, 
+			ports->port, 
+			MACH_MSG_TYPE_MAKE_SEND);
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_insert_right(): ", ret);
+		exit(1);
+	}
+
+	ret = task_get_bootstrap_port(mach_task_self(), &bsport);
+	if (KERN_SUCCESS != ret) {
+		mach_error("task_get_bootstrap_port(): ", ret);
+		exit(1);
+	}
+
+	ret = bootstrap_register(bsport, server_port_name, ports->port);
+	if (KERN_SUCCESS != ret) {
+		mach_error("bootstrap_register(): ", ret);
+		exit(1);
+	}
+	if (verbose) {
+		printf("server waiting for IPC messages from client on port '%s'.\n",
+				server_port_name);
+	}
+}
+
+void setup_client_ports(struct port_args *ports)
+{
+	kern_return_t ret = 0;
+	switch(msg_type) {
+		case msg_type_trivial:
+			ports->req_size = sizeof(ipc_trivial_message);
+			break;
+		case msg_type_inline:
+			ports->req_size = sizeof(ipc_inline_message) +  
+				sizeof(u_int32_t) * num_ints;
+			break;
+		case msg_type_complex:
+			ports->req_size = sizeof(ipc_complex_message);
+			break;
+	}
+	ports->req_size -= sizeof(mach_msg_trailer_t);
+	ports->reply_size = sizeof(ipc_trivial_message);
+	ports->req_msg = malloc(ports->req_size);
+	ports->reply_msg = malloc(ports->reply_size);
+
+	ret = mach_port_allocate(mach_task_self(), 
+			MACH_PORT_RIGHT_RECEIVE,  
+			&(ports->port));
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_allocate(): ", ret);
+		exit(1);
+	}
+	if (verbose) {
+		printf("Client sending %d %s IPC messages to port '%s' in %s mode.\n",
+				num_msgs, (msg_type == msg_type_inline) ? 
+				"inline" :  ((msg_type == msg_type_complex) ? 
+					"complex" : "trivial"),  
+				server_port_name, (oneway ? "oneway" : "rpc"));
+	}
+
+}
+
+void server(struct port_args *args) 
+{
+	int idx;
+	kern_return_t ret;
+	int totalmsg = num_msgs * num_clients;
+
+	unsigned long long starttsc, endtsc, deltatsc;
+	struct timeval starttv, endtv, deltatv;
+
+	/* Call gettimeofday() once and throw away result; some implementations
+	 * (like Mach's) cache some time zone info on first call.  Then, call
+	 * ReadTSR in case that helps warm things up, again discarding the 
+	 * results.
+	 */
+	gettimeofday(&starttv, NULL);
+	ReadTSR();
+
+	gettimeofday(&starttv, NULL);
+	starttsc = ReadTSR();
+
+	for (idx = 0; idx < totalmsg; idx++) {
+		if (verbose) 
+			printf("server awaiting message %d\n", idx);
+		args->req_msg->msgh_bits = 0;
+		args->req_msg->msgh_size = args->req_size;
+		args->req_msg->msgh_local_port = args->port;
+		ret = mach_msg(args->req_msg,  
+				MACH_RCV_MSG|MACH_RCV_INTERRUPT|MACH_RCV_LARGE, 
+				0, 
+				args->req_size,  
+				args->port, 
+				MACH_MSG_TIMEOUT_NONE, 
+				MACH_PORT_NULL);
+		if (MACH_MSG_SUCCESS != ret) {
+			mach_error("mach_msg (receive): ", ret);
+			exit(1);
+		}
+		if (verbose)
+			printf("server received message %d\n", idx);
+		if (args->req_msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
+			ret = vm_deallocate(mach_task_self(),  
+					(vm_address_t)((ipc_complex_message *)args->req_msg)->descriptor.address,  
+					((ipc_complex_message *)args->req_msg)->descriptor.size);
+		}
+
+		if (1 == args->req_msg->msgh_id) {
+			if (verbose) 
+				printf("server sending reply %d\n", idx);
+			args->reply_msg->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,  
+					MACH_MSG_TYPE_MAKE_SEND);
+			args->reply_msg->msgh_size = args->reply_size;
+			args->reply_msg->msgh_remote_port = args->req_msg->msgh_remote_port;
+			args->reply_msg->msgh_local_port = args->req_msg->msgh_local_port;
+			args->reply_msg->msgh_id = 2;
+			ret = mach_msg(args->reply_msg, 
+					MACH_SEND_MSG, 
+					args->reply_size, 
+					0, 
+					MACH_PORT_NULL, 
+					MACH_MSG_TIMEOUT_NONE,  
+					MACH_PORT_NULL);
+			if (MACH_MSG_SUCCESS != ret) {
+				mach_error("mach_msg (send): ", ret);
+				exit(1);
+			}
+		}
+	}
+
+	endtsc = ReadTSR();
+	gettimeofday(&endtv, NULL);
+
+	/* report results */
+	deltatsc = endtsc - starttsc;
+	deltatv.tv_sec = endtv.tv_sec - starttv.tv_sec;
+	deltatv.tv_usec = endtv.tv_usec - starttv.tv_usec;
+	if (endtv.tv_usec < starttv.tv_usec) {
+		deltatv.tv_sec--;
+		deltatv.tv_usec += 1000000;
+	}
+
+	double dsecs = (double) deltatv.tv_sec + 
+		1.0E-6 * (double) deltatv.tv_usec;
+
+	printf("\n%u messages during %qd time stamp ticks\n", 
+			totalmsg, deltatsc);
+	printf("%g time stamp ticks per message\n", 
+			(double) deltatsc / (double) totalmsg);
+	printf("\n%u messages during %u.%06u seconds\n",  
+			totalmsg, deltatv.tv_sec, deltatv.tv_usec);
+	printf("%g messages per second\n", (double)totalmsg /  dsecs);
+	printf("%g microseconds per message\n\n", 
+			dsecs * 1.0E6 / (double) totalmsg); 
+}
+
+void *client(void *threadarg) 
+{
+	struct port_args args;
+	int idx;
+	mach_msg_header_t *req, *reply; 
+	mach_port_t bsport, servport;
+	kern_return_t ret;
+	void *ints = malloc(sizeof(u_int32_t) * num_ints);
+
+	/* find server port */
+	ret = task_get_bootstrap_port(mach_task_self(), &bsport);
+	if (KERN_SUCCESS != ret) {
+		mach_error("task_get_bootstrap_port(): ", ret);
+		exit(1);
+	}
+	ret = bootstrap_look_up(bsport, server_port_name, &servport); 
+	if (KERN_SUCCESS != ret) {
+		mach_error("bootstrap_look_up(): ", ret);
+		exit(1);
+	}
+
+	setup_client_ports(&args);
+	
+	/* start message loop */
+	for (idx = 0; idx < num_msgs; idx++) {
+		req = args.req_msg;
+		reply = args.reply_msg;
+
+		req->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 
+				MACH_MSG_TYPE_MAKE_SEND);
+		req->msgh_size = args.req_size;
+		req->msgh_remote_port = servport;
+		req->msgh_local_port = args.port;
+		req->msgh_id = oneway ? 0 : 1;
+		switch (msg_type) {
+			case msg_type_trivial:
+				break;
+			case msg_type_inline:
+				((ipc_inline_message *)req)->type.msgt_name = MACH_MSG_TYPE_INTEGER_32;
+				((ipc_inline_message *)req)->type.msgt_size = 32;
+				((ipc_inline_message *)req)->type.msgt_number = num_ints;
+				((ipc_inline_message *)req)->type.msgt_inline = TRUE;
+				((ipc_inline_message *)req)->type.msgt_longform = FALSE;
+				((ipc_inline_message *)req)->type.msgt_deallocate = FALSE;
+				((ipc_inline_message *)req)->type.msgt_unused = 0;
+				break;
+			case msg_type_complex:
+				(req)->msgh_bits |=  MACH_MSGH_BITS_COMPLEX;
+				((ipc_complex_message *)req)->body.msgh_descriptor_count = 1;
+				((ipc_complex_message *)req)->descriptor.address = ints;
+				((ipc_complex_message *)req)->descriptor.size = 
+					num_ints * sizeof(u_int32_t);
+				((ipc_complex_message *)req)->descriptor.deallocate = FALSE;
+				((ipc_complex_message *)req)->descriptor.copy = MACH_MSG_VIRTUAL_COPY;
+				((ipc_complex_message *)req)->descriptor.type = MACH_MSG_OOL_DESCRIPTOR;
+				break;
+		}
+		if (verbose) 
+			printf("client sending message %d\n", idx);
+		ret = mach_msg(req,  
+				MACH_SEND_MSG, 
+				args.req_size, 
+				0, 
+				MACH_PORT_NULL,  
+				MACH_MSG_TIMEOUT_NONE, 
+				MACH_PORT_NULL);
+		if (MACH_MSG_SUCCESS != ret) {
+			mach_error("mach_msg (send): ", ret);
+			fprintf(stderr, "bailing after %u iterations\n", idx);
+			exit(1);
+			break;
+		}
+		if (!oneway) {
+			if (verbose) 
+				printf("client awaiting reply %d\n", idx);
+			reply->msgh_bits = 0;
+			reply->msgh_size = args.reply_size;
+			reply->msgh_local_port = args.port;
+			ret = mach_msg(args.reply_msg,  
+					MACH_RCV_MSG|MACH_RCV_INTERRUPT, 
+					0, 
+					args.reply_size, 
+					args.port,  
+					MACH_MSG_TIMEOUT_NONE, 
+					MACH_PORT_NULL);
+			if (MACH_MSG_SUCCESS != ret) {
+				mach_error("mach_msg (receive): ", ret);
+				fprintf(stderr, "bailing after %u iterations\n",
+						idx);
+				exit(1);
+			}
+			if (verbose) 
+				printf("client received reply %d\n", idx);
+		}
+
+		if (client_delay) {
+			usleep(client_delay);
+		}
+	}
+
+	free(ints);
+	return;
+}
+
+
+int main(int argc, char *argv[]) 
+{
+	struct port_args portargs;
+	int i;
+
+	signal(SIGINT, signal_handler);
+	parse_args(argc, argv);
+
+	setup_server_ports(&portargs);
+
+	if (fork() != 0) {
+		server(&portargs);
+		exit(0);
+	}
+
+	if (num_clients > 1) {
+		for (i = 1; i < num_clients; i++) {
+			if (fork() == 0) {
+				client(NULL);
+				exit(0);
+			}
+		}
+	}
+
+	client(NULL);
+
+	return (0);
+}
diff --git a/tools/tests/MMTest/MPMMtest.c b/tools/tests/MMTest/MPMMtest.c
new file mode 100644
index 000000000..ac555fa05
--- /dev/null
+++ b/tools/tests/MMTest/MPMMtest.c
@@ -0,0 +1,773 @@
+#include <AvailabilityMacros.h>
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+#include </System/Library/Frameworks/System.framework/PrivateHeaders/mach/thread_policy.h>
+#endif
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <err.h>
+
+#include <pthread.h>
+#include <mach/mach.h>
+#include <mach/mach_error.h>
+#include <mach/notify.h>
+#include <servers/bootstrap.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/signal.h>
+
+#define MAX(A, B) ((A) < (B) ? (B) : (A))
+
+typedef struct  {
+	unsigned int        msgt_name : 8,
+			    msgt_size : 8,
+			    msgt_number : 12,
+			    msgt_inline : 1,
+			    msgt_longform : 1,
+			    msgt_deallocate : 1,
+			    msgt_unused : 1;
+} mach_msg_type_t;
+
+typedef struct  {
+	mach_msg_type_t     msgtl_header;
+	unsigned short      msgtl_name;
+	unsigned short      msgtl_size;
+	natural_t           msgtl_number;
+} mach_msg_type_long_t;
+#define MACH_MSG_TYPE_INTEGER_32 0
+
+
+typedef struct {
+	mach_msg_header_t	header;
+	mach_msg_trailer_t	trailer;		// subtract this when sending
+} ipc_trivial_message;
+
+typedef struct {
+	mach_msg_header_t	header;
+	mach_msg_type_t		type;
+	u_int32_t		numbers[0];
+	mach_msg_trailer_t	trailer;		// subtract this when sending
+} ipc_inline_message;
+
+typedef struct {
+	mach_msg_header_t		header;
+	mach_msg_body_t			body;
+	mach_msg_ool_descriptor_t	descriptor;
+	mach_msg_trailer_t		trailer;	// subtract this when sending
+} ipc_complex_message;
+
+enum {
+	msg_type_trivial = 0,
+	msg_type_inline = 1,
+	msg_type_complex = 2
+};
+
+struct port_args {
+	int server_num;
+	int req_size;
+	mach_msg_header_t *req_msg;
+	int reply_size;
+	mach_msg_header_t *reply_msg;
+	mach_port_t port;
+};
+
+typedef union {
+	pid_t		pid;
+	pthread_t	tid;
+} thread_id_t;
+
+/* Global options */
+static boolean_t	verbose = FALSE;
+static boolean_t	affinity = FALSE;
+static boolean_t	timeshare = FALSE;
+static boolean_t	threaded = FALSE;
+static boolean_t	oneway = FALSE;
+int			msg_type;
+int			num_ints;
+int			num_msgs;
+int			num_clients;
+int			num_servers;
+int			client_delay;
+int			client_spin;
+int			client_pages;
+char			**server_port_name;
+
+void signal_handler(int sig) {
+}
+
+void usage(const char *progname) {
+	fprintf(stderr, "usage: %s [options]\n", progname);
+	fprintf(stderr, "where options are:\n");
+	fprintf(stderr, "    -affinity\t\tthreads use affinity\n");
+	fprintf(stderr, "    -timeshare\t\tthreads use timeshare\n");
+	fprintf(stderr, "    -threaded\t\tuse (p)threads\n");
+	fprintf(stderr, "    -verbose\t\tbe verbose\n");
+	fprintf(stderr, "    -oneway\t\tdo not request return reply\n");
+	fprintf(stderr, "    -count num\t\tnumber of messages to send\n");
+	fprintf(stderr, "    -type trivial|inline|complex\ttype of messages to send\n");
+	fprintf(stderr, "    -numints num\tnumber of 32-bit ints to send in messages\n");
+	fprintf(stderr, "    -servers num\tnumber of servers threads to run\n");
+	fprintf(stderr, "    -clients num\tnumber of clients per server\n");
+	fprintf(stderr, "    -delay num\t\tmicroseconds to sleep clients between messages\n");
+	fprintf(stderr, "    -work num\t\tmicroseconds of client work\n");
+	fprintf(stderr, "    -pages num\t\tpages of memory touched by client work\n");
+	fprintf(stderr, "default values are:\n");
+	fprintf(stderr, "    . no affinity\n");
+	fprintf(stderr, "    . not timeshare\n");
+	fprintf(stderr, "    . not verbose\n");
+	fprintf(stderr, "    . not oneway\n");
+	fprintf(stderr, "    . client sends 100000 messages\n");
+	fprintf(stderr, "    . inline message type\n");
+	fprintf(stderr, "    . 64 32-bit integers in inline/complex messages\n");
+	fprintf(stderr, "    . (num_available_processors+1)%%2 servers\n");
+	fprintf(stderr, "    . 4 clients per server\n");
+	fprintf(stderr, "    . no delay\n");
+	exit(1);
+}
+
+void parse_args(int argc, char *argv[]) {
+	host_basic_info_data_t		info;
+	mach_msg_type_number_t		count;
+	kern_return_t			result;
+
+	/* Initialize defaults */
+	msg_type = msg_type_trivial;
+	num_ints = 64;
+	num_msgs = 100000;
+	client_delay = 0;
+	num_clients = 4;
+
+	count = HOST_BASIC_INFO_COUNT;
+	result = host_info(mach_host_self(), HOST_BASIC_INFO, 
+			(host_info_t)&info, &count);
+	if (result == KERN_SUCCESS && info.avail_cpus > 1)
+		num_servers = info.avail_cpus / 2;
+	else
+		num_servers = 1;
+
+	const char *progname = argv[0];
+	argc--; argv++;
+	while (0 < argc) {
+		if (0 == strcmp("-verbose", argv[0])) {
+			verbose = TRUE;
+			argc--; argv++;
+		} else if (0 == strcmp("-affinity", argv[0])) {
+			affinity = TRUE;
+			argc--; argv++;
+		} else if (0 == strcmp("-timeshare", argv[0])) {
+			timeshare = TRUE;
+			argc--; argv++;
+		} else if (0 == strcmp("-threaded", argv[0])) {
+			threaded = TRUE;
+			argc--; argv++;
+		} else if (0 == strcmp("-oneway", argv[0])) {
+			oneway = TRUE;
+			argc--; argv++;
+		} else if (0 == strcmp("-type", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			if (0 == strcmp("trivial", argv[1])) {
+				msg_type = msg_type_trivial;
+			} else if (0 == strcmp("inline", argv[1])) {
+				msg_type = msg_type_inline;
+			} else if (0 == strcmp("complex", argv[1])) {
+				msg_type = msg_type_complex;
+			} else 
+				usage(progname);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-numints", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_ints = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-count", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_msgs = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		}  else if (0 == strcmp("-clients", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_clients = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		}  else if (0 == strcmp("-servers", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			num_servers = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-delay", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			client_delay = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-spin", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			client_spin = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else if (0 == strcmp("-pages", argv[0])) {
+			if (argc < 2) 
+				usage(progname);
+			client_pages = strtoul(argv[1], NULL, 0);
+			argc -= 2; argv += 2;
+		} else 
+			usage(progname);
+	}
+}
+
+void setup_server_ports(struct port_args *ports)
+{
+	kern_return_t ret = 0;
+	mach_port_t bsport;
+
+	ports->req_size = MAX(sizeof(ipc_inline_message) +  
+			sizeof(u_int32_t) * num_ints, 
+			sizeof(ipc_complex_message));
+	ports->reply_size = sizeof(ipc_trivial_message) - 
+		sizeof(mach_msg_trailer_t);
+	ports->req_msg = malloc(ports->req_size);
+	ports->reply_msg = malloc(ports->reply_size);
+
+	ret = mach_port_allocate(mach_task_self(), 
+			MACH_PORT_RIGHT_RECEIVE,  
+			&(ports->port));
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_allocate(): ", ret);
+		exit(1);
+	}
+
+	ret = mach_port_insert_right(mach_task_self(), 
+			ports->port, 
+			ports->port, 
+			MACH_MSG_TYPE_MAKE_SEND);
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_insert_right(): ", ret);
+		exit(1);
+	}
+
+	ret = task_get_bootstrap_port(mach_task_self(), &bsport);
+	if (KERN_SUCCESS != ret) {
+		mach_error("task_get_bootstrap_port(): ", ret);
+		exit(1);
+	}
+
+	if (verbose) {
+		printf("server waiting for IPC messages from client on port '%s'.\n",
+			server_port_name[ports->server_num]);
+	}
+	ret = bootstrap_register(bsport,
+				 server_port_name[ports->server_num],
+				 ports->port);
+	if (KERN_SUCCESS != ret) {
+		mach_error("bootstrap_register(): ", ret);
+		exit(1);
+	}
+}
+
+void setup_client_ports(struct port_args *ports)
+{
+	kern_return_t ret = 0;
+	switch(msg_type) {
+		case msg_type_trivial:
+			ports->req_size = sizeof(ipc_trivial_message);
+			break;
+		case msg_type_inline:
+			ports->req_size = sizeof(ipc_inline_message) +  
+				sizeof(u_int32_t) * num_ints;
+			break;
+		case msg_type_complex:
+			ports->req_size = sizeof(ipc_complex_message);
+			break;
+	}
+	ports->req_size -= sizeof(mach_msg_trailer_t);
+	ports->reply_size = sizeof(ipc_trivial_message);
+	ports->req_msg = malloc(ports->req_size);
+	ports->reply_msg = malloc(ports->reply_size);
+
+	ret = mach_port_allocate(mach_task_self(), 
+			MACH_PORT_RIGHT_RECEIVE,  
+			&(ports->port));
+	if (KERN_SUCCESS != ret) {
+		mach_error("mach_port_allocate(): ", ret);
+		exit(1);
+	}
+	if (verbose) {
+		printf("Client sending %d %s IPC messages to port '%s' in %s mode.\n",
+				num_msgs, (msg_type == msg_type_inline) ? 
+				"inline" :  ((msg_type == msg_type_complex) ? 
+					"complex" : "trivial"),  
+				server_port_name[ports->server_num],
+				(oneway ? "oneway" : "rpc"));
+	}
+
+}
+
+
+static void
+thread_setup(int tag) {
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+	kern_return_t			ret;
+        thread_extended_policy_data_t   epolicy;
+        thread_affinity_policy_data_t   policy;
+
+	if (!timeshare) {
+		epolicy.timeshare = FALSE;
+		ret = thread_policy_set(
+				mach_thread_self(), THREAD_EXTENDED_POLICY,
+				(thread_policy_t) &epolicy,
+				THREAD_EXTENDED_POLICY_COUNT);
+		if (ret != KERN_SUCCESS)
+			printf("thread_policy_set(THREAD_EXTENDED_POLICY) returned %d\n", ret);
+	}
+
+        if (affinity) {
+                policy.affinity_tag = tag;
+                ret = thread_policy_set(
+                                mach_thread_self(), THREAD_AFFINITY_POLICY,
+                                (thread_policy_t) &policy,
+                                THREAD_AFFINITY_POLICY_COUNT);
+                if (ret != KERN_SUCCESS)
+                        printf("thread_policy_set(THREAD_AFFINITY_POLICY) returned %d\n", ret);
+        }
+#endif
+}
+
+void *
+server(void *serverarg) 
+{
+	struct port_args args;
+	int idx;
+	kern_return_t ret;
+	int totalmsg = num_msgs * num_clients;
+
+	args.server_num = (int) (long) serverarg;
+	setup_server_ports(&args);
+
+	thread_setup(args.server_num + 1);
+
+	for (idx = 0; idx < totalmsg; idx++) {
+		if (verbose) 
+			printf("server awaiting message %d\n", idx);
+		args.req_msg->msgh_bits = 0;
+		args.req_msg->msgh_size = args.req_size;
+		args.req_msg->msgh_local_port = args.port;
+		ret = mach_msg(args.req_msg,  
+				MACH_RCV_MSG|MACH_RCV_INTERRUPT|MACH_RCV_LARGE, 
+				0, 
+				args.req_size,  
+				args.port, 
+				MACH_MSG_TIMEOUT_NONE, 
+				MACH_PORT_NULL);
+		if (MACH_RCV_INTERRUPTED == ret)
+			break;
+		if (MACH_MSG_SUCCESS != ret) {
+			if (verbose)
+				printf("mach_msg() ret=%d", ret);
+			mach_error("mach_msg (receive): ", ret);
+			exit(1);
+		}
+		if (verbose)
+			printf("server received message %d\n", idx);
+		if (args.req_msg->msgh_bits & MACH_MSGH_BITS_COMPLEX) {
+			ret = vm_deallocate(mach_task_self(),  
+					(vm_address_t)((ipc_complex_message *)args.req_msg)->descriptor.address,  
+					((ipc_complex_message *)args.req_msg)->descriptor.size);
+		}
+
+		if (1 == args.req_msg->msgh_id) {
+			if (verbose) 
+				printf("server sending reply %d\n", idx);
+			args.reply_msg->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,  
+					MACH_MSG_TYPE_MAKE_SEND);
+			args.reply_msg->msgh_size = args.reply_size;
+			args.reply_msg->msgh_remote_port = args.req_msg->msgh_remote_port;
+			args.reply_msg->msgh_local_port = args.req_msg->msgh_local_port;
+			args.reply_msg->msgh_id = 2;
+			ret = mach_msg(args.reply_msg, 
+					MACH_SEND_MSG, 
+					args.reply_size, 
+					0, 
+					MACH_PORT_NULL, 
+					MACH_MSG_TIMEOUT_NONE,  
+					MACH_PORT_NULL);
+			if (MACH_MSG_SUCCESS != ret) {
+				mach_error("mach_msg (send): ", ret);
+				exit(1);
+			}
+		}
+	}
+}
+
+static inline void
+client_spin_loop(unsigned count, void (fn)(void))
+{
+	while (count--)
+		fn();
+}
+
+static long	dummy_memory;
+static long	*client_memory = &dummy_memory;
+static void
+client_work_atom(void)
+{
+	static int	i;
+
+	if (++i > client_pages * PAGE_SIZE / sizeof(long))
+		i = 0;
+	client_memory[i] = 0;
+}
+
+static	int	calibration_count = 10000;
+static	int	calibration_usec;
+static void *
+calibrate_client_work(void)
+{
+	long		dummy;
+	struct timeval	nowtv;
+	struct timeval	warmuptv = { 0, 100 * 1000 }; /* 100ms */
+	struct timeval	starttv;
+	struct timeval	endtv;
+
+	if (client_spin) {
+		/* Warm-up the stepper first... */
+		gettimeofday(&nowtv, NULL);
+		timeradd(&nowtv, &warmuptv, &endtv);
+		do {
+			client_spin_loop(calibration_count, client_work_atom);
+			gettimeofday(&nowtv, NULL);
+		} while (timercmp(&nowtv, &endtv, < ));	
+	
+		/* Now do the calibration */
+		while (TRUE) {
+			gettimeofday(&starttv, NULL);
+			client_spin_loop(calibration_count, client_work_atom);
+			gettimeofday(&endtv, NULL);
+			if (endtv.tv_sec - starttv.tv_sec > 1) {
+				calibration_count /= 10;
+				continue;
+			}
+			calibration_usec = endtv.tv_usec - starttv.tv_usec;
+			if (endtv.tv_usec < starttv.tv_usec) {
+				calibration_usec += 1000000;
+			}
+			if (calibration_usec < 1000) {
+				calibration_count *= 10;
+				continue;
+			}
+			calibration_count /= calibration_usec;
+			break;
+		}
+		if (verbose)
+			printf("calibration_count=%d calibration_usec=%d\n",
+				calibration_count, calibration_usec);
+	}
+}
+
+static void *
+client_work(void)
+{
+
+	if (client_spin) {
+		client_spin_loop(calibration_count*client_spin,
+				 client_work_atom);
+	}
+	
+	if (client_delay) {
+		usleep(client_delay);
+	}
+}
+
+void *client(void *threadarg) 
+{
+	struct port_args args;
+	int idx;
+	mach_msg_header_t *req, *reply; 
+	mach_port_t bsport, servport;
+	kern_return_t ret;
+	long server_num = (long) threadarg;
+	void *ints = malloc(sizeof(u_int32_t) * num_ints);
+
+	if (verbose) 
+		printf("client(%d) started, server port name %s\n",
+			server_num, server_port_name[server_num]);
+
+	args.server_num = server_num;
+	thread_setup(server_num + 1);
+
+	/* find server port */
+	ret = task_get_bootstrap_port(mach_task_self(), &bsport);
+	if (KERN_SUCCESS != ret) {
+		mach_error("task_get_bootstrap_port(): ", ret);
+		exit(1);
+	}
+	ret = bootstrap_look_up(bsport,
+				server_port_name[server_num],
+				&servport); 
+	if (KERN_SUCCESS != ret) {
+		mach_error("bootstrap_look_up(): ", ret);
+		exit(1);
+	}
+
+	setup_client_ports(&args);
+
+	/* Allocate and touch memory */
+	if (client_pages) {
+		unsigned	i;
+		client_memory = (long *) malloc(client_pages * PAGE_SIZE);
+		for (i = 0; i < client_pages; i++)
+			client_memory[i * PAGE_SIZE / sizeof(long)] = 0;
+	}
+	
+	/* start message loop */
+	for (idx = 0; idx < num_msgs; idx++) {
+		req = args.req_msg;
+		reply = args.reply_msg;
+
+		req->msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND, 
+				MACH_MSG_TYPE_MAKE_SEND);
+		req->msgh_size = args.req_size;
+		req->msgh_remote_port = servport;
+		req->msgh_local_port = args.port;
+		req->msgh_id = oneway ? 0 : 1;
+		switch (msg_type) {
+			case msg_type_trivial:
+				break;
+			case msg_type_inline:
+				((ipc_inline_message *)req)->type.msgt_name = MACH_MSG_TYPE_INTEGER_32;
+				((ipc_inline_message *)req)->type.msgt_size = 32;
+				((ipc_inline_message *)req)->type.msgt_number = num_ints;
+				((ipc_inline_message *)req)->type.msgt_inline = TRUE;
+				((ipc_inline_message *)req)->type.msgt_longform = FALSE;
+				((ipc_inline_message *)req)->type.msgt_deallocate = FALSE;
+				((ipc_inline_message *)req)->type.msgt_unused = 0;
+				break;
+			case msg_type_complex:
+				(req)->msgh_bits |=  MACH_MSGH_BITS_COMPLEX;
+				((ipc_complex_message *)req)->body.msgh_descriptor_count = 1;
+				((ipc_complex_message *)req)->descriptor.address = ints;
+				((ipc_complex_message *)req)->descriptor.size = 
+					num_ints * sizeof(u_int32_t);
+				((ipc_complex_message *)req)->descriptor.deallocate = FALSE;
+				((ipc_complex_message *)req)->descriptor.copy = MACH_MSG_VIRTUAL_COPY;
+				((ipc_complex_message *)req)->descriptor.type = MACH_MSG_OOL_DESCRIPTOR;
+				break;
+		}
+		if (verbose) 
+			printf("client sending message %d\n", idx);
+		ret = mach_msg(req,  
+				MACH_SEND_MSG, 
+				args.req_size, 
+				0, 
+				MACH_PORT_NULL,  
+				MACH_MSG_TIMEOUT_NONE, 
+				MACH_PORT_NULL);
+		if (MACH_MSG_SUCCESS != ret) {
+			mach_error("mach_msg (send): ", ret);
+			fprintf(stderr, "bailing after %u iterations\n", idx);
+			exit(1);
+			break;
+		}
+		if (!oneway) {
+			if (verbose) 
+				printf("client awaiting reply %d\n", idx);
+			reply->msgh_bits = 0;
+			reply->msgh_size = args.reply_size;
+			reply->msgh_local_port = args.port;
+			ret = mach_msg(args.reply_msg,  
+					MACH_RCV_MSG|MACH_RCV_INTERRUPT, 
+					0, 
+					args.reply_size, 
+					args.port,  
+					MACH_MSG_TIMEOUT_NONE, 
+					MACH_PORT_NULL);
+			if (MACH_MSG_SUCCESS != ret) {
+				mach_error("mach_msg (receive): ", ret);
+				fprintf(stderr, "bailing after %u iterations\n",
+						idx);
+				exit(1);
+			}
+			if (verbose) 
+				printf("client received reply %d\n", idx);
+		}
+
+		client_work();
+	}
+
+	free(ints);
+	return;
+}
+
+static void
+thread_spawn(thread_id_t *thread, void *(fn)(void *), void *arg) {
+	if (threaded) {
+		kern_return_t	ret;
+		ret = pthread_create(
+				&thread->tid,
+				NULL,
+				fn,
+				arg);
+		if (ret != 0)
+			err(1, "pthread_create()");
+		if (verbose)
+			printf("created pthread 0x%x\n", thread->tid);
+	} else {
+		thread->pid = fork();
+		if (thread->pid == 0) {
+			if (verbose)
+				printf("calling 0x%x(0x%x)\n", fn, arg);
+			fn(arg);
+			exit(0);
+		}
+		if (verbose)
+			printf("forked pid %d\n", thread->pid);
+	}
+}
+
+static void
+thread_join(thread_id_t *thread) {
+	if (threaded) {
+		kern_return_t	ret;
+		if (verbose)
+			printf("joining thread 0x%x\n", thread->tid);
+		ret = pthread_join(thread->tid, NULL);
+		if (ret != KERN_SUCCESS)
+			err(1, "pthread_join(0x%x)", thread->tid);
+	} else {
+		int	stat;
+		if (verbose)
+			printf("waiting for pid %d\n", thread->pid);
+		waitpid(thread->pid, &stat, 0);
+	}
+}
+
+static void
+wait_for_servers(void)
+{
+	int		i;
+	int		retry_count = 10;
+	mach_port_t	bsport, servport;
+	kern_return_t	ret;
+
+	/* find server port */
+	ret = task_get_bootstrap_port(mach_task_self(), &bsport);
+	if (KERN_SUCCESS != ret) {
+		mach_error("task_get_bootstrap_port(): ", ret);
+		exit(1);
+	}
+
+	while (retry_count-- > 0) {
+		for (i = 0; i < num_servers; i++) {
+			ret = bootstrap_look_up(bsport,
+					server_port_name[i],
+					&servport); 
+			if (ret != KERN_SUCCESS) {
+				break;
+			}
+		}
+		if (ret == KERN_SUCCESS)
+			return;
+		usleep(100 * 1000);	/* 100ms */
+	}
+	fprintf(stderr, "Server(s) failed to register\n");
+	exit(1);
+}
+
+int main(int argc, char *argv[]) 
+{
+	int		i;
+	int		j;
+	thread_id_t	*client_id;
+	thread_id_t	*server_id;
+
+	signal(SIGINT, signal_handler);
+	parse_args(argc, argv);
+
+	calibrate_client_work();
+
+	/*
+	 * If we're using affinity create an empty namespace now
+	 * so this is shared by all our offspring.
+	 */
+	if (affinity)
+		thread_setup(0);
+
+	server_id = (thread_id_t *) malloc(num_servers * sizeof(thread_id_t));
+	server_port_name = (char **) malloc(num_servers * sizeof(char *));
+	if (verbose)
+		printf("creating %d servers\n", num_servers);
+	for (i = 0; i < num_servers; i++) {
+		server_port_name[i] = (char *) malloc(sizeof("PORT.pppppp.xx"));
+		/* PORT names include pid of main process for disambiguation */
+		sprintf(server_port_name[i], "PORT.%06d.%02d", getpid(), i);
+		thread_spawn(&server_id[i], server, (void *) (long) i);
+	}
+
+	int totalclients = num_servers * num_clients;
+	int totalmsg = num_msgs * totalclients;
+	struct timeval starttv, endtv, deltatv;
+
+	/*
+	 * Wait for all servers to have registered all ports before starting
+	 * the clients and the clock.
+	 */
+	wait_for_servers();
+	
+	printf("%d server%s, %d client%s per server (%d total) %u messages...", 
+			num_servers, (num_servers > 1)? "s" : "",
+			num_clients, (num_clients > 1)? "s" : "",
+			totalclients,
+			totalmsg);
+	fflush(stdout);
+
+	/* Call gettimeofday() once and throw away result; some implementations
+	 * (like Mach's) cache some time zone info on first call.
+	 */
+	gettimeofday(&starttv, NULL);
+	gettimeofday(&starttv, NULL);
+
+	client_id = (thread_id_t *) malloc(totalclients * sizeof(thread_id_t));
+	if (verbose)
+		printf("creating %d clients\n", totalclients);
+	for (i = 0; i < num_servers; i++) {
+		for (j = 0; j < num_clients; j++) {
+			thread_spawn(
+				&client_id[(i*num_clients) + j],
+				client,
+				(void *) (long) i);
+		}
+	}
+
+	/* Wait for servers to complete */
+	for (i = 0; i < num_servers; i++) {
+		thread_join(&server_id[i]);
+	}
+
+	gettimeofday(&endtv, NULL);
+
+	for (i = 0; i < totalclients; i++) {
+		thread_join(&client_id[i]);
+	}
+
+	/* report results */
+	deltatv.tv_sec = endtv.tv_sec - starttv.tv_sec;
+	deltatv.tv_usec = endtv.tv_usec - starttv.tv_usec;
+	if (endtv.tv_usec < starttv.tv_usec) {
+		deltatv.tv_sec--;
+		deltatv.tv_usec += 1000000;
+	}
+
+	double dsecs = (double) deltatv.tv_sec + 
+		1.0E-6 * (double) deltatv.tv_usec;
+
+	printf(" in %u.%03u seconds\n",  
+			deltatv.tv_sec, deltatv.tv_usec/1000);
+	printf("  throughput in messages/sec:     %g\n",
+			(double)totalmsg / dsecs);
+	printf("  average message latency (usec): %2.3g\n", 
+			dsecs * 1.0E6 / (double) totalmsg);
+
+	return (0);
+
+}
diff --git a/tools/tests/MMTest/Makefile b/tools/tests/MMTest/Makefile
new file mode 100644
index 000000000..c09cdc9e8
--- /dev/null
+++ b/tools/tests/MMTest/Makefile
@@ -0,0 +1,21 @@
+CFLAGS=-g -O2 -arch ppc -arch i386
+CFLAGS64=-g -O2 -arch ppc64 -arch x86_64
+
+TARGETS	= MMtest MMtest_64 MPMMtest MPMMtest_64
+
+all:	$(TARGETS)
+
+MMtest: MMtest.c
+	${CC} ${CFLAGS} -o $@ $?
+
+MMtest_64: MMtest.c
+	${CC} ${CFLAGS64} -o $@ $?
+
+MPMMtest: MPMMtest.c
+	${CC} ${CFLAGS} -o $@ $?
+
+MPMMtest_64: MPMMtest.c
+	${CC} ${CFLAGS64} -o $@ $?
+
+clean:
+	rm -rf $(TARGETS)
diff --git a/tools/tests/affinity/Makefile b/tools/tests/affinity/Makefile
new file mode 100644
index 000000000..9450d79c6
--- /dev/null
+++ b/tools/tests/affinity/Makefile
@@ -0,0 +1,18 @@
+CFLAGS	=-g -arch ppc -arch i386
+CFLAGS64=-g -arch ppc64 -arch x86_64
+
+TESTS =		\
+	sets	\
+	pool	\
+	tags	\
+
+TARGETS = $(TESTS) $(TESTS:=64)
+
+all:	$(TESTS)
+
+.c:	
+	gcc $(CFLAGS) $< -o $@		# 32-bit fat 
+	gcc $(CFLAGS64) $< -o $@64 		# 64-bit fat
+
+clean:
+	rm -rf $(TARGETS) 
diff --git a/tools/tests/affinity/pool.c b/tools/tests/affinity/pool.c
new file mode 100644
index 000000000..97626c5f8
--- /dev/null
+++ b/tools/tests/affinity/pool.c
@@ -0,0 +1,504 @@
+#include <AvailabilityMacros.h>
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+#include </System/Library/Frameworks/System.framework/PrivateHeaders/mach/thread_policy.h>
+#endif
+#include <mach/mach.h>
+#include <mach/mach_error.h>
+#include <mach/mach_time.h>
+#include <pthread.h>
+#include <sys/queue.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <err.h>
+
+/*
+ * Pool is another multithreaded test/benchmarking program to evaluate
+ * affinity set placement in Leopard.
+ *
+ * The basic picture is:
+ *
+ *                  -> producer --                 -> consumer --
+ *       free     /                \    work     /                \
+ *    -> queue --      ...          --> queue --                   --
+ *   |            \                /             \                /  |
+ *   |              -> producer --                 -> consumer --    |
+ *    ---------------------------------------------------------------
+ *
+ *       <---------- "stage" ---------> <---------- "stage" --------->
+ *
+ * There are a series of work stages. Each stage has an input and an output
+ * queue and multiple threads. The first stage is the producer and subsequent
+ * stages are consumers. By defuaut there are 2 stages. There are N producer
+ * and M consumer threads. The are B buffers per producer threads circulating
+ * through the system.
+ *
+ * When affinity is enabled, each producer thread is tagged with an affinity tag
+ * 1 .. N - so each runs on a different L2 cache. When a buffer is queued to
+ * the work queue it is tagged with this affinity. When a consumer dequeues a
+ * work item, it sets its affinity to this tag. Hence consumer threads migrate
+ * to the same affinity set where the data was produced.
+ *
+ * Buffer management uses pthread mutex/condition variables. A thread blocks
+ * when no buffer is available on a queue and it is signaled when a buffer
+ * is placed on an empty queue. Queues are tailq'a a la <sys/queue.h>.
+ * The queue management is centralized in a single routine: what queues to
+ * use as input and output and what function to call for processing is
+ * data-driven.
+ */
+  
+pthread_mutex_t funnel;
+pthread_cond_t	barrier;
+
+uint64_t	timer;
+int		threads;
+int		threads_ready = 0;
+
+int		iterations = 10000;
+boolean_t	affinity = FALSE;
+boolean_t	halting = FALSE;
+int		verbosity = 1;
+
+typedef struct work {
+	TAILQ_ENTRY(work)	link;
+	int			*data;
+	int			isize;
+	int			tag;
+	int			number;
+} work_t;
+
+/*
+ * A work queue, complete with pthread objects for its management
+ */
+typedef struct work_queue {
+	pthread_mutex_t		mtx;
+	pthread_cond_t		cnd;
+	TAILQ_HEAD(, work)	queue;
+	unsigned int		waiters;
+} work_queue_t;
+
+/* Worker functions take a integer array and size */
+typedef void (worker_fn_t)(int *, int); 
+
+/* This struct controls the function of a stage */
+#define WORKERS_MAX 10
+typedef struct {
+	int			stagenum;
+	char			*name;
+	worker_fn_t		*fn;
+	work_queue_t		*input;		
+	work_queue_t		*output;		
+	work_queue_t		bufq;
+	int			work_todo;
+} stage_info_t;
+
+/* This defines a worker thread */
+typedef struct worker_info {
+	int			setnum;
+	stage_info_t		*stage;
+	pthread_t		thread;
+} worker_info_t;
+
+#define DBG(x...) do {				\
+	if (verbosity > 1) {			\
+		pthread_mutex_lock(&funnel);	\
+		printf(x);			\
+		pthread_mutex_unlock(&funnel);	\
+	}					\
+} while (0)
+
+#define mutter(x...) do {			\
+	if (verbosity > 0) {			\
+		printf(x);			\
+	}					\
+} while (0)
+
+#define s_if_plural(x)	(((x) > 1) ? "s" : "")
+
+static void
+usage()
+{
+	fprintf(stderr,
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+		"usage: pool [-a]    Turn affinity on (off)\n"
+		"            [-b B]  Number of buffers per producer (2)\n"
+#else
+		"usage: pool [-b B]  Number of buffers per producer (2)\n"
+#endif
+		"            [-i I]  Number of buffers to produce (10000)\n"
+		"            [-s S]  Number of stages (2)\n"
+		"            [-p P]  Number of pages per buffer (256=1MB)]\n"
+		"            [-w]    Consumer writes data\n"
+		"            [-v V]  Verbosity level 0..2 (1)\n"
+		"            [N [M]] Number of producer and consumers (2)\n"
+	);
+	exit(1);
+}
+
+/* Trivial producer: write to each byte */
+void
+writer_fn(int *data, int isize)
+{
+	int 	i;
+
+	for (i = 0; i < isize; i++) {
+		data[i] = i;
+	}
+}
+
+/* Trivial consumer: read each byte */
+void
+reader_fn(int *data, int isize)
+{
+	int 	i;
+	int	datum;
+
+	for (i = 0; i < isize; i++) {
+		datum = data[i];
+	}
+}
+
+/* Consumer reading and writing the buffer */
+void
+reader_writer_fn(int *data, int isize)
+{
+	int 	i;
+
+	for (i = 0; i < isize; i++) {
+		data[i] += 1;
+	}
+}
+
+void
+affinity_set(int tag)
+{
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+	kern_return_t			ret;
+	thread_affinity_policy_data_t	policy;
+	if (affinity) {
+		policy.affinity_tag = tag;
+		ret = thread_policy_set(
+				mach_thread_self(), THREAD_AFFINITY_POLICY,
+				(thread_policy_t) &policy,
+				THREAD_AFFINITY_POLICY_COUNT);
+		if (ret != KERN_SUCCESS)
+			printf("thread_policy_set(THREAD_AFFINITY_POLICY) returned %d\n", ret);
+	}
+#endif
+}
+
+/*
+ * This is the central function for every thread.
+ * For each invocation, its role is ets by (a pointer to) a stage_info_t.
+ */
+void *
+manager_fn(void *arg)
+{
+	worker_info_t	*wp = (worker_info_t *) arg;
+	stage_info_t	*sp = wp->stage;
+	boolean_t	is_producer = (sp->stagenum == 0);
+	long		iteration = 0;
+	int		current_tag = 0;
+
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+	kern_return_t			ret;
+	thread_extended_policy_data_t	epolicy;
+	epolicy.timeshare = FALSE;
+	ret = thread_policy_set(
+			mach_thread_self(), THREAD_EXTENDED_POLICY,
+			(thread_policy_t) &epolicy,
+			THREAD_EXTENDED_POLICY_COUNT);
+	if (ret != KERN_SUCCESS)
+		printf("thread_policy_set(THREAD_EXTENDED_POLICY) returned %d\n", ret);
+	
+#endif
+	/*
+	 * If we're using affinity sets and we're a producer
+	 * set our tag to by our thread set number.
+	 */
+	if (affinity && is_producer) {
+		affinity_set(wp->setnum);
+		current_tag = wp->setnum;
+	}
+
+	DBG("Starting %s %d, stage: %d\n", sp->name, wp->setnum, sp->stagenum);
+
+	/*
+	 * Start barrier.
+	 * The tets thread to get here releases everyone and starts the timer.
+	 */
+	pthread_mutex_lock(&funnel);
+	threads_ready++;
+	if (threads_ready == threads) {
+		pthread_mutex_unlock(&funnel);
+		if (halting) {
+			printf("  all threads ready for process %d, "
+				"hit any key to start", getpid());
+			fflush(stdout);
+			(void) getchar();
+		}
+		pthread_cond_broadcast(&barrier);
+		timer = mach_absolute_time();
+	} else {
+		pthread_cond_wait(&barrier, &funnel);
+		pthread_mutex_unlock(&funnel);
+	}
+
+	do {
+		work_t		*workp;
+
+		/*
+		 * Get a buffer from the input queue.
+		 * Block if none.
+		 * Quit if all work done.
+		 */
+		pthread_mutex_lock(&sp->input->mtx);
+		while (1) {
+			if (sp->work_todo == 0) {
+				pthread_mutex_unlock(&sp->input->mtx);
+				goto out;
+			}
+			workp = TAILQ_FIRST(&(sp->input->queue));
+			if (workp != NULL)
+				break;
+			DBG("    %s[%d,%d] todo %d waiting for buffer\n",
+				sp->name, wp->setnum, sp->stagenum, sp->work_todo);
+			sp->input->waiters++;
+			pthread_cond_wait(&sp->input->cnd, &sp->input->mtx);
+			sp->input->waiters--;
+		}
+		TAILQ_REMOVE(&(sp->input->queue), workp, link);
+		iteration = sp->work_todo--;
+		pthread_mutex_unlock(&sp->input->mtx);
+
+		if (is_producer) {
+			workp->number = iteration;
+			workp->tag = wp->setnum;
+		} else {
+			if (affinity && current_tag != workp->tag) {
+				affinity_set(workp->tag);
+				current_tag = workp->tag;
+			}
+		}
+
+		DBG("  %s[%d,%d] todo %d work %p data %p\n",
+			sp->name, wp->setnum, sp->stagenum, iteration, workp, workp->data);
+
+		/* Do our stuff with the buffer */
+		(void) sp->fn(workp->data, workp->isize);
+
+		/*
+		 * Place the buffer on the input queue of the next stage.
+		 * Signal waiters if required.
+		 */
+		pthread_mutex_lock(&sp->output->mtx);
+		TAILQ_INSERT_TAIL(&(sp->output->queue), workp, link);
+		if (sp->output->waiters) {
+			DBG("    %s[%d,%d] todo %d signaling work\n",
+				sp->name, wp->setnum, sp->stagenum, iteration);
+			pthread_cond_signal(&sp->output->cnd);
+		}
+		pthread_mutex_unlock(&sp->output->mtx);
+
+	} while (1);
+
+out:
+	pthread_cond_broadcast(&sp->output->cnd);
+
+	DBG("Ending %s[%d,%d]\n", sp->name, wp->setnum, sp->stagenum);
+
+	return (void *) iteration;
+}
+
+void (*producer_fnp)(int *data, int isize) = &writer_fn;
+void (*consumer_fnp)(int *data, int isize) = &reader_fn;
+
+int
+main(int argc, char *argv[])
+{
+	int			i;
+	int			j;
+	int			k;
+	int			pages = 256; /* 1MB */
+	int			buffers = 2;
+	int			producers = 2;
+	int			consumers = 2;
+	int			stages = 2;
+	int			*status;
+	stage_info_t		*stage_info;
+	stage_info_t		*sp;
+	worker_info_t		*worker_info;
+	worker_info_t		*wp;
+	kern_return_t		ret;
+	int			c;
+
+	/* Do switch parsing: */
+	while ((c = getopt (argc, argv, "ab:i:p:s:twv:")) != -1) {
+		switch (c) {
+		case 'a':
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+			affinity = !affinity;
+			break;
+#else
+			usage();
+#endif
+		case 'b':
+			buffers = atoi(optarg);
+			break;
+		case 'i':
+			iterations = atoi(optarg);
+			break;
+		case 'p':
+			pages = atoi(optarg);
+			break;
+		case 's':
+			stages = atoi(optarg);
+			if (stages >= WORKERS_MAX)
+				usage();
+			break;
+		case 't':
+			halting = TRUE;
+			break;
+		case 'w':
+			consumer_fnp = &reader_writer_fn;
+			break;
+		case 'v':
+			verbosity = atoi(optarg);
+			break;
+		case 'h':
+		case '?':
+		default:
+			usage();
+		}
+	}
+	argc -= optind; argv += optind;
+	if (argc > 0)
+		producers = atoi(*argv);
+	argc--; argv++;
+	if (argc > 0)
+		consumers = atoi(*argv);
+	
+	pthread_mutex_init(&funnel, NULL);
+	pthread_cond_init(&barrier, NULL);
+
+	/*
+ 	 * Fire up the worker threads.
+	 */
+	threads = consumers * (stages - 1) + producers;
+	mutter("Launching %d producer%s with %d stage%s of %d consumer%s\n"
+		"  with %saffinity, consumer reads%s data\n",
+		producers, s_if_plural(producers),
+		stages - 1, s_if_plural(stages - 1),
+		consumers, s_if_plural(consumers),
+		affinity? "": "no ",
+		(consumer_fnp == &reader_writer_fn)? " and writes" : "");
+	if (pages < 256)
+		mutter("  %dkB bytes per buffer, ", pages * 4);
+	else
+		mutter("  %dMB bytes per buffer, ", pages / 256);
+	mutter("%d buffer%s per producer ",
+		buffers, s_if_plural(buffers));
+	if (buffers * pages < 256)
+		mutter("(total %dkB)\n", buffers * pages * 4);
+	else
+		mutter("(total %dMB)\n", buffers * pages / 256);
+	mutter("  processing %d buffer%s...\n",
+		iterations, s_if_plural(iterations));
+
+	stage_info = (stage_info_t *) malloc(stages * sizeof(stage_info_t));
+	worker_info = (worker_info_t *) malloc(threads * sizeof(worker_info_t));
+
+	/* Set up the queue for the workers of this thread set: */
+	for (i = 0; i < stages; i++) {
+		sp = &stage_info[i];
+		sp->stagenum = i;
+		pthread_mutex_init(&sp->bufq.mtx, NULL);
+		pthread_cond_init(&sp->bufq.cnd, NULL);
+		TAILQ_INIT(&sp->bufq.queue);
+		sp->bufq.waiters = 0;
+		if (i == 0) {
+			sp->fn = producer_fnp;
+			sp->name = "producer";
+		} else {
+			sp->fn = consumer_fnp;
+			sp->name = "consumer";
+		}
+		sp->input = &sp->bufq;
+		sp->output = &stage_info[(i + 1) % stages].bufq;
+		stage_info[i].work_todo = iterations;
+	}
+ 
+	/* Create the producers */
+	for (i = 0; i < producers; i++) {
+		work_t	*work_array;
+		int	*data;
+		int	isize;
+
+		isize = pages * 4096 / sizeof(int);
+		data = (int *) malloc(buffers * pages * 4096);
+
+		/* Set up the empty work buffers */
+		work_array = (work_t *)  malloc(buffers * sizeof(work_t));
+		for (j = 0; j < buffers; j++) {
+			work_array[j].data = data + (isize * j);	
+			work_array[j].isize = isize;
+			work_array[j].tag = 0;
+			TAILQ_INSERT_TAIL(&stage_info[0].bufq.queue, &work_array[j], link);
+			DBG("  empty work item %p for data %p\n",
+				&work_array[j], work_array[j].data);
+		}
+		wp = &worker_info[i];
+		wp->setnum = i + 1;
+		wp->stage = &stage_info[0];
+		if (ret = pthread_create(&wp->thread,
+					 NULL,
+					 &manager_fn,
+					 (void *) wp))
+			err(1, "pthread_create %d,%d", 0, i);
+	}
+
+	/* Create consumers */
+	for (i = 1; i < stages; i++) {
+		for (j = 0; j < consumers; j++) {
+			wp = &worker_info[producers + (consumers*(i-1)) + j];
+			wp->setnum = j + 1;
+			wp->stage = &stage_info[i];
+			if (ret = pthread_create(&wp->thread,
+						NULL,
+						&manager_fn,
+						(void *) wp))
+				err(1, "pthread_create %d,%d", i, j);
+		}
+	}
+
+	/*
+	 * We sit back anf wait for the slaves to finish.
+	 */
+	for (k = 0; k < threads; k++) {
+		int	i;
+		int	j;
+
+		wp = &worker_info[k];
+		if (k < producers) {
+			i = 0;
+			j = k;
+		} else {
+			i = (k - producers) / consumers;
+			j = (k - producers) % consumers;
+		}
+		if(ret = pthread_join(wp->thread, (void **)&status))
+		    err(1, "pthread_join %d,%d", i, j);
+		DBG("Thread %d,%d status %d\n", i, j, status);
+	}
+
+	/*
+	 * See how long the work took.
+	 */
+	timer = mach_absolute_time() - timer;
+	timer = timer / 1000000ULL;
+	printf("%d.%03d seconds elapsed.\n",
+		(int) (timer/1000ULL), (int) (timer % 1000ULL));
+
+	return 0;
+}
diff --git a/tools/tests/affinity/sets.c b/tools/tests/affinity/sets.c
new file mode 100644
index 000000000..a7115e56b
--- /dev/null
+++ b/tools/tests/affinity/sets.c
@@ -0,0 +1,496 @@
+#include <AvailabilityMacros.h>
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+#include </System/Library/Frameworks/System.framework/PrivateHeaders/mach/thread_policy.h>
+#endif
+#include <mach/mach.h>
+#include <mach/mach_error.h>
+#include <mach/mach_time.h>
+#include <pthread.h>
+#include <sys/queue.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <err.h>
+#include <errno.h>
+
+/*
+ * Sets is a multithreaded test/benchmarking program to evaluate
+ * affinity set placement in Leopard.
+ *
+ * The picture here, for each set, is:
+ *  
+ *       free                   work
+ *    -> queue --> producer --> queue --> consumer --
+ *   |                                               |
+ *    -----------------------------------------------
+ *
+ *       <------ "stage" -----> <------ "stage" ----->
+
+ * We spin off sets of production line threads (2 sets by default).
+ * All threads of each line sets the same affinity tag (unless disabled).
+ * By default there are 2 stage (worker) threads per production line.
+ * A worker thread removes a buffer from an input queue, processses it and
+ * queues it on an output queue.  By default the initial stage (producer)
+ * writes every byte in a buffer and the other (consumer) stages read every
+ * byte. By default the buffers are 1MB (256 pages) in size but this can be
+ * overidden.  By default there are 2 buffers per set (again overridable).
+ * Worker threads process (iterate over) 10000 buffers by default.
+ *
+ * With affinity enabled, each producer and consumer thread sets its affinity
+ * to the set number, 1 .. N. So the threads of each set share an L2 cache.
+ *
+ * Buffer management uses pthread mutex/condition variables. A thread blocks
+ * when no buffer is available on a queue and it is signaled when a buffer
+ * is placed on an empty queue. Queues are tailq'a a la <sys/queue.h>.
+ * The queue management is centralized in a single routine: what queues to
+ * use as input and output and what function to call for processing is
+ * data-driven.
+ */
+  
+pthread_mutex_t funnel;
+pthread_cond_t	barrier;
+
+uint64_t	timer;
+int		threads;
+int		threads_ready = 0;
+
+int		iterations = 10000;
+boolean_t	affinity = FALSE;
+boolean_t	halting = FALSE;
+boolean_t	cache_config = FALSE;
+int		verbosity = 1;
+
+typedef struct work {
+	TAILQ_ENTRY(work)	link;
+	int			*data;
+} work_t;
+
+/*
+ * A work queue, complete with pthread objects for its management
+ */
+typedef struct work_queue {
+	pthread_mutex_t		mtx;
+	pthread_cond_t		cnd;
+	TAILQ_HEAD(, work)	queue;
+	boolean_t		waiters;
+} work_queue_t;
+
+/* Worker functions take a integer array and size */
+typedef void (worker_fn_t)(int *, int); 
+
+/* This struct controls the function of a thread */
+typedef struct {
+	int			stagenum;
+	char			*name;
+	worker_fn_t		*fn;
+	work_queue_t		*input;		
+	work_queue_t		*output;		
+	struct line_info	*set;
+	pthread_t		thread;
+	work_queue_t		bufq;
+} stage_info_t;
+
+/* This defines a thread set */
+#define WORKERS_MAX 10
+typedef struct line_info {
+	int			setnum;
+	int			*data;
+	int			isize;
+	stage_info_t		*stage[WORKERS_MAX];
+} line_info_t;
+
+#define DBG(x...) do {				\
+	if (verbosity > 1) {			\
+		pthread_mutex_lock(&funnel);	\
+		printf(x);			\
+		pthread_mutex_unlock(&funnel);	\
+	}					\
+} while (0)
+
+#define mutter(x...) do {			\
+	if (verbosity > 0) {			\
+		printf(x);			\
+	}					\
+} while (0)
+
+#define s_if_plural(x)	(((x) > 1) ? "s" : "")
+
+static void
+usage()
+{
+	fprintf(stderr,
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+		"usage: sets [-a]   Turn affinity on (off)\n"
+		"            [-b B] Number of buffers per set/line (2)\n"
+#else
+		"usage: sets [-b B] Number of buffers per set/line (2)\n"
+#endif
+		"            [-c]   Configure for max cache performance\n"
+		"            [-h]   Print this\n"
+		"            [-i I] Number of items/buffers to process (1000)\n"
+		"            [-s S] Number of stages per set/line (2)\n"
+		"            [-t]   Halt for keyboard input to start\n"
+		"            [-p P] Number of pages per buffer (256=1MB)]\n"
+		"            [-w]   Consumer writes data\n"
+		"            [-v V] Level of verbosity 0..2 (1)\n"
+		"            [N]    Number of sets/lines (2)\n"
+	);
+	exit(1);
+}
+
+/* Trivial producer: write to each byte */
+void
+writer_fn(int *data, int isize)
+{
+	int 	i;
+
+	for (i = 0; i < isize; i++) {
+		data[i] = i;
+	}
+}
+
+/* Trivial consumer: read each byte */
+void
+reader_fn(int *data, int isize)
+{
+	int 	i;
+	int	datum;
+
+	for (i = 0; i < isize; i++) {
+		datum = data[i];
+	}
+}
+
+/* Consumer reading and writing the buffer */
+void
+reader_writer_fn(int *data, int isize)
+{
+	int 	i;
+
+	for (i = 0; i < isize; i++) {
+		data[i] += 1;
+	}
+}
+
+/*
+ * This is the central function for every thread.
+ * For each invocation, its role is ets by (a pointer to) a stage_info_t.
+ */
+void *
+manager_fn(void *arg)
+{
+	stage_info_t			*sp = (stage_info_t *) arg;
+	line_info_t			*lp = sp->set;
+	kern_return_t			ret;
+	long				iteration = 0;
+
+	/*
+	 * If we're using affinity sets (we are by default)
+	 * set our tag to by our thread set number.
+	 */
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+	thread_extended_policy_data_t	epolicy;
+	thread_affinity_policy_data_t	policy;
+
+	epolicy.timeshare = FALSE;
+	ret = thread_policy_set(
+			mach_thread_self(), THREAD_EXTENDED_POLICY,
+			(thread_policy_t) &epolicy,
+			THREAD_EXTENDED_POLICY_COUNT);
+	if (ret != KERN_SUCCESS)
+		printf("thread_policy_set(THREAD_EXTENDED_POLICY) returned %d\n", ret);
+	
+	if (affinity) {
+		policy.affinity_tag = lp->setnum;
+		ret = thread_policy_set(
+				mach_thread_self(), THREAD_AFFINITY_POLICY,
+				(thread_policy_t) &policy,
+				THREAD_AFFINITY_POLICY_COUNT);
+		if (ret != KERN_SUCCESS)
+			printf("thread_policy_set(THREAD_AFFINITY_POLICY) returned %d\n", ret);
+	}
+#endif
+
+	DBG("Starting %s set: %d stage: %d\n", sp->name, lp->setnum, sp->stagenum);
+
+	/*
+	 * Start barrier.
+	 * The tets thread to get here releases everyone and starts the timer.
+	 */
+	pthread_mutex_lock(&funnel);
+	threads_ready++;
+	if (threads_ready == threads) {
+		pthread_mutex_unlock(&funnel);
+		if (halting) {
+			printf("  all threads ready for process %d, "
+				"hit any key to start", getpid());
+			fflush(stdout);
+			(void) getchar();
+		}
+		pthread_cond_broadcast(&barrier);
+		timer = mach_absolute_time();
+	} else {
+		pthread_cond_wait(&barrier, &funnel);
+		pthread_mutex_unlock(&funnel);
+	}
+
+	do {
+		int		i;
+		work_t		*workp;
+
+		/*
+		 * Get a buffer from the input queue.
+		 * Block if none.
+		 */
+		pthread_mutex_lock(&sp->input->mtx);
+		while (1) {
+			workp = TAILQ_FIRST(&(sp->input->queue));
+			if (workp != NULL)
+				break;
+			DBG("    %s[%d,%d] iteration %d waiting for buffer\n",
+				sp->name, lp->setnum, sp->stagenum, iteration);
+			sp->input->waiters = TRUE;
+			pthread_cond_wait(&sp->input->cnd, &sp->input->mtx);
+			sp->input->waiters = FALSE;
+		}
+		TAILQ_REMOVE(&(sp->input->queue), workp, link);
+		pthread_mutex_unlock(&sp->input->mtx);
+
+		DBG("  %s[%d,%d] iteration %d work %p data %p\n",
+			sp->name, lp->setnum, sp->stagenum, iteration, workp, workp->data);
+
+		/* Do our stuff with the buffer */
+		(void) sp->fn(workp->data, lp->isize);
+
+		/*
+		 * Place the buffer on the input queue.
+		 * Signal  waiters if required.
+		 */
+		pthread_mutex_lock(&sp->output->mtx);
+		TAILQ_INSERT_TAIL(&(sp->output->queue), workp, link);
+		if (sp->output->waiters) {
+			DBG("    %s[%d,%d] iteration %d signaling work\n",
+				sp->name, lp->setnum, sp->stagenum, iteration);
+			pthread_cond_signal(&sp->output->cnd);
+		}
+		pthread_mutex_unlock(&sp->output->mtx);
+	} while (++iteration < iterations);
+
+	DBG("Ending %s[%d,%d]\n", sp->name, lp->setnum, sp->stagenum);
+
+	return (void *) iteration;
+}
+
+static void
+auto_config(int npages, int *nbufs, int *nsets)
+{
+	int	len;
+	int	ncpu;
+	int64_t	cacheconfig[10];
+	int64_t	cachesize[10];
+
+	mutter("Autoconfiguring...\n");
+
+	len = sizeof(cacheconfig);
+	if (sysctlbyname("hw.cacheconfig",
+			 &cacheconfig[0], &len, NULL, 0) != 0) {
+		printf("Unable to get hw.cacheconfig, %d\n", errno);
+		exit(1);
+	}
+	len = sizeof(cachesize);
+	if (sysctlbyname("hw.cachesize",
+			 &cachesize[0],  &len, NULL, 0) != 0) {
+		printf("Unable to get hw.cachesize, %d\n", errno);
+		exit(1);
+	}
+
+	/*
+	 * Calculate number of buffers of size pages*4096 bytes
+	 * fit into 90% of an L2 cache.
+	 */
+	*nbufs = cachesize[2] * 9 / (npages * 4096 * 10);
+	mutter("  L2 cache %qd bytes: "
+		"using %d buffers of size %d bytes\n",
+		cachesize[2], *nbufs, (npages * 4096));
+
+	/* 
+	 * Calcalute how many sets:
+	 */
+	*nsets = cacheconfig[0]/cacheconfig[2];
+	mutter("  %qd cpus; %qd cpus per L2 cache: using %d sets\n",
+		cacheconfig[0], cacheconfig[2], *nsets);
+}
+
+void (*producer_fnp)(int *data, int isize) = &writer_fn;
+void (*consumer_fnp)(int *data, int isize) = &reader_fn;
+
+int
+main(int argc, char *argv[])
+{
+	int			i;
+	int			j;
+	int			pages = 256; /* 1MB */
+	int			buffers = 2;
+	int			sets = 2;
+	int			stages = 2;
+	int			*status;
+	line_info_t		*line_info;
+	line_info_t		*lp;
+	stage_info_t		*stage_info;
+	stage_info_t		*sp;
+	kern_return_t		ret;
+	int			c;
+
+	/* Do switch parsing: */
+	while ((c = getopt (argc, argv, "ab:chi:p:s:twv:")) != -1) {
+		switch (c) {
+		case 'a':
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+			affinity = !affinity;
+			break;
+#else
+			usage();
+#endif
+		case 'b':
+			buffers = atoi(optarg);
+			break;
+		case 'c':
+			cache_config = TRUE;
+			break;
+		case 'i':
+			iterations = atoi(optarg);
+			break;
+		case 'p':
+			pages = atoi(optarg);
+			break;
+		case 's':
+			stages = atoi(optarg);
+			if (stages >= WORKERS_MAX)
+				usage();
+			break;
+		case 't':
+			halting = TRUE;
+			break;
+		case 'w':
+			consumer_fnp = &reader_writer_fn;
+			break;
+		case 'v':
+			verbosity = atoi(optarg);
+			break;
+		case '?':
+		case 'h':
+		default:
+			usage();
+		}
+	}
+	argc -= optind; argv += optind;
+	if (argc > 0)
+		sets = atoi(*argv);
+
+	if (cache_config)
+		auto_config(pages, &buffers, &sets);
+
+	pthread_mutex_init(&funnel, NULL);
+	pthread_cond_init(&barrier, NULL);
+
+	/*
+ 	 * Fire up the worker threads.
+	 */
+	threads = sets * stages;
+	mutter("Launching %d set%s of %d threads with %saffinity, "
+			"consumer reads%s data\n",
+		sets, s_if_plural(sets), stages, affinity? "": "no ",
+		(consumer_fnp == &reader_writer_fn)? " and writes" : "");
+	if (pages < 256)
+		mutter("  %dkB bytes per buffer, ", pages * 4);
+	else
+		mutter("  %dMB bytes per buffer, ", pages / 256);
+	mutter("%d buffer%s per set ",
+		buffers, s_if_plural(buffers));
+	if (buffers * pages < 256)
+		mutter("(total %dkB)\n", buffers * pages * 4);
+	else
+		mutter("(total %dMB)\n", buffers * pages / 256);
+	mutter("  processing %d buffer%s...\n",
+		iterations, s_if_plural(iterations));
+	line_info = (line_info_t *) malloc(sets * sizeof(line_info_t));
+	stage_info = (stage_info_t *) malloc(sets * stages * sizeof(stage_info_t));
+	for (i = 0; i < sets; i++) {
+		work_t	*work_array;
+
+		lp = &line_info[i];
+
+		lp->setnum = i + 1;
+		lp->isize = pages * 4096 / sizeof(int);
+		lp->data = (int *) malloc(buffers * pages * 4096);
+
+		/* Set up the queue for the workers of this thread set: */
+		for (j = 0; j < stages; j++) {
+			sp = &stage_info[(i*stages) + j];
+			sp->stagenum = j;
+			sp->set = lp;
+			lp->stage[j] = sp;
+			pthread_mutex_init(&sp->bufq.mtx, NULL);
+			pthread_cond_init(&sp->bufq.cnd, NULL);
+			TAILQ_INIT(&sp->bufq.queue);
+			sp->bufq.waiters = FALSE;
+		}
+
+		/*
+		 * Take a second pass through the stages
+		 * to define what the workers are and to interconnect their input/outputs
+		 */
+		for (j = 0; j < stages; j++) {
+			sp = lp->stage[j];
+			if (j == 0) {
+				sp->fn = producer_fnp;
+				sp->name = "producer";
+			} else {
+				sp->fn = consumer_fnp;
+				sp->name = "consumer";
+			}
+			sp->input = &lp->stage[j]->bufq;
+			sp->output = &lp->stage[(j + 1) % stages]->bufq;
+		}
+
+		/* Set up the buffers on the first worker of the set. */
+		work_array = (work_t *)  malloc(buffers * sizeof(work_t));
+		for (j = 0; j < buffers; j++) {
+			work_array[j].data = lp->data + (lp->isize * j);	
+			TAILQ_INSERT_TAIL(&lp->stage[0]->bufq.queue, &work_array[j], link);
+			DBG("  empty work item %p for set %d data %p\n",
+				&work_array[j], i, work_array[j].data);
+		}
+
+		/* Create this set of threads */
+		for (j = 0; j < stages; j++) {
+			if (ret = pthread_create(&lp->stage[j]->thread, NULL,
+					&manager_fn,
+					(void *) lp->stage[j]))
+			err(1, "pthread_create %d,%d", i, j);
+		}
+	}
+
+	/*
+	 * We sit back anf wait for the slave to finish.
+	 */
+	for (i = 0; i < sets; i++) {
+		lp = &line_info[i];
+		for (j = 0; j < stages; j++) {
+			if(ret = pthread_join(lp->stage[j]->thread, (void **)&status))
+			    err(1, "pthread_join %d,%d", i, j);
+			DBG("Thread %d,%d status %d\n", i, j, status);
+		}
+	}
+
+	/*
+	 * See how long the work took.
+	 */
+	timer = mach_absolute_time() - timer;
+	timer = timer / 1000000ULL;
+	printf("%d.%03d seconds elapsed.\n",
+		(int) (timer/1000ULL), (int) (timer % 1000ULL));
+
+	return 0;
+}
diff --git a/tools/tests/affinity/tags.c b/tools/tests/affinity/tags.c
new file mode 100644
index 000000000..e0872087b
--- /dev/null
+++ b/tools/tests/affinity/tags.c
@@ -0,0 +1,152 @@
+#include <AvailabilityMacros.h>
+#ifdef AVAILABLE_MAC_OS_X_VERSION_10_5_AND_LATER
+#include </System/Library/Frameworks/System.framework/PrivateHeaders/mach/thread_policy.h>
+#endif
+#include <mach/mach.h>
+#include <mach/mach_traps.h>
+#include <mach/mach_error.h>
+#include <mach/mach_time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <err.h>
+
+int		verbosity = 1;
+
+#define DBG(x...) do {				\
+	if (verbosity > 1) {			\
+		printf(x);			\
+	}					\
+} while (0)
+
+#define mutter(x...) do {			\
+	if (verbosity > 0) {			\
+		printf(x);			\
+	}					\
+} while (0)
+
+#define s_if_plural(x)	(((x) > 1) ? "s" : "")
+
+static void
+usage()
+{
+	fprintf(stderr,
+		"usage: tags [-i]   interactive/input\n"
+		"            [-v V] verbosity level 0..2 (1)\n"
+		"            [-h]   help info\n"
+		"            pid    process id of target task\n"
+	);
+	exit(1);
+}
+
+void
+thread_tag_set(thread_t thread, int tag)
+{
+	kern_return_t			ret;
+	thread_affinity_policy_data_t	policy;
+
+	policy.affinity_tag = tag;
+	ret = thread_policy_set(
+			thread, THREAD_AFFINITY_POLICY,
+			(thread_policy_t) &policy,
+			THREAD_AFFINITY_POLICY_COUNT);
+	if (ret != KERN_SUCCESS) {
+		printf("thread_policy_set(1) returned %d\n", ret);
+		exit(1);
+	}
+}
+
+int
+thread_tag_get(thread_t thread)
+{
+	kern_return_t			ret;
+	boolean_t			get_default = FALSE;
+	thread_affinity_policy_data_t	policy;
+	mach_msg_type_number_t		count = THREAD_AFFINITY_POLICY_COUNT;
+
+	ret = thread_policy_get(
+			thread, THREAD_AFFINITY_POLICY,
+			(thread_policy_t) &policy, &count, &get_default);
+	if (ret != KERN_SUCCESS) {
+		printf("thread_policy_set(1) returned %d\n", ret);
+		exit(1);
+	}
+
+	return policy.affinity_tag;
+}
+
+char			input[81];
+int
+main(int argc, char *argv[])
+{
+	kern_return_t		ret;
+	mach_port_name_t	port;
+	int			pid;
+	int			c;
+	thread_act_t		*thread_array;
+	mach_msg_type_number_t	num_threads;
+	int			i;
+	boolean_t		interactive = FALSE;
+	int			tag;
+
+	if (geteuid() != 0) {
+		printf("Must be run as root\n");
+		exit(1);
+	}
+
+	/* Do switch parsing: */
+	while ((c = getopt (argc, argv, "hiv:")) != -1) {
+		switch (c) {
+		case 'i':
+			interactive = TRUE;
+			break;
+		case 'v':
+			verbosity = atoi(optarg);
+			break;
+		case 'h':
+		case '?':
+		default:
+			usage();
+		}
+	}
+	argc -= optind; argv += optind;
+	if (argc > 0)
+		pid = atoi(*argv);
+
+	ret = task_for_pid(mach_task_self(), pid, &port);
+	if (ret != KERN_SUCCESS)
+		err(1, "task_for_pid(,%d,) returned %d", pid, ret);	
+
+	mutter("task %p\n", port);
+	ret = task_threads(port, &thread_array, &num_threads);
+	if (ret != KERN_SUCCESS)
+		err(1, "task_threads() returned %d", pid, ret);	
+ 
+	for (i = 0; i < num_threads; i++) {
+		printf(" %d: thread 0x%08x tag %d\n",
+			i, thread_array[i], thread_tag_get(thread_array[i]));
+	}
+
+	while (interactive) {
+		printf("Enter new tag or <return> to skip or ^D to quit\n");
+		for (i = 0; i < num_threads; i++) {
+			tag =  thread_tag_get(thread_array[i]);
+			printf(" %d: thread 0x%08x tag %d: ",
+				i, thread_array[i], tag);
+			fflush(stdout);
+			(void) fgets(input, 20, stdin);
+			if (feof(stdin)) {
+				printf("\n");
+				interactive = FALSE;
+				break;
+			}
+			if (strlen(input) > 1) {
+				tag = atoi(input);
+				thread_tag_set(thread_array[i], tag);
+			}
+		}
+	}
+
+	return 0;
+}
diff --git a/tools/tests/xnu_quick_test/README b/tools/tests/xnu_quick_test/README
new file mode 100644
index 000000000..06f204bc7
--- /dev/null
+++ b/tools/tests/xnu_quick_test/README
@@ -0,0 +1,178 @@
+xnu_quick_test - this tool will do a quick test of every (well, to be
+honest most) system calls we support in xnu.
+
+WARNING - this is not meant to be a full regression test of all the
+system calls.  The intent is to have a quick test of each system call that
+can be run very easily and quickly when a new kerenl is built.
+
+This tool is meant to grow as we find xnu problems that could have be
+caught before we submit to a build train.  So please add more tests and
+make the existing ones better.  Some of the original tests are nothing
+more than place holders and quite lame.  Just keep in mind that the tool
+should run as fast as possible.  If it gets too slow then most people
+will stop running it.
+
+LP64 testing tip - when adding or modifying tests, keep in mind the
+variants in the LP64 world.  If xnu gets passed a structure the varies in
+size between 32 and 64-bit processes, try to test that a field in the 
+sructure contains valid data.  For example if we know foo structure
+looks like:
+struct foo {
+             int             an_int;
+             long    a_long;
+             int             another_int;
+}
+And if we know what another_int should contain then test for the known
+value since it's offset will vary depending on whether the calling process
+is 32 or 64 bits.
+
+NOTE - we have several workarounds and test exceptions for some
+outstanding bugs in xnu.  All the workarounds are marked with "todo" and
+some comments noting the radar number of the offending bug.  Do a seach
+for "todo" in the source files for this project to locate which tests have
+known failures.   And please tag any new exceptions you find with "todo"
+in the comment and the radar number of the bug.
+
+To build a fat binary, export ARCH="ppc ppc64 i386 x86_64". This will work
+for any architectures that Apple gcc recognizes.
+
+Added three defines which you can use at the compile line to build variants.
+DEBUG
+       turn on additional printfs
+CONFORMANCE_TESTS_IN_XNU 
+       when conformance tests are in xnu, set this to 1
+TEST_SYSTEM_CALLS
+       test system calls (doesn't compile; a different bug)
+by default, all three are set to 0, i.e. disabled.  To build, export
+MORECFLAGS with the values you want set, e.g. 
+       export MORECFLAGS="-D DEBUG=1 -D CONFORMANCE_TESTS_IN_XNU=1"
+
+todo:
+-- have folks with area expertise fix lame tests
+   (most of the networking related tests are pretty lame)
+-- mach system calls support
+
+
+USAGE:  xnu_quick_test -target TARGET_PATH 
+
+         -f[ailures] MAX_FAILS_ALLOWED   # number of test cases that may fail before we give up.  defaults to 0  
+         -l[ist]                         # list all the tests this tool performs   
+         -r[un] 1, 3, 10 - 19            # run specific tests.  enter individual test numbers and/or range of numbers.  use -list to list tests.   
+         -s[kip]                         # skip setuid tests   
+         -t[arget] TARGET_PATH           # path to directory where tool will create test files.  defaults to "/tmp/"   
+
+examples:  
+--- Place all test files and directories at the root of volume "test_vol" --- 
+xnu_quick_test -t /Volumes/test_vol/ 
+ 
+--- Run the tool for tests 10 thru 15, test 18 and test 20 --- 
+xnu_quick_test -r 10-15, 18, 20 
+
+
+--- example of running the tool to list all the tests it currently supports ---
+xnu_quick_test -l
+List of all tests this tool performs... 
+ 1         syscall 
+ 2         fork, wait4, exit 
+ 3         fsync, ftruncate, lseek, pread, pwrite, read, readv, truncate, write, writev 
+ 4         close, fpathconf, fstat, open, pathconf 
+ 5         link, stat, unlink 
+ 6         chdir, fchdir 
+ 7         access, chmod, fchmod 
+ 8         chown, fchown, lchown, lstat, readlink, symlink 
+ 9         fstatfs, getattrlist, getfsstat, statfs 
+ 10        getpid, getppid, pipe 
+ 11        getauid, gettid, getuid, geteuid, issetugid, setauid, seteuid, settid, settid_with_pid, setuid 
+ 12        mkdir, rmdir, umask 
+ 13        mknod, sync 
+ 14        fsync, getsockopt, poll, select, setsockopt, socketpair 
+ 15        accept, bind, connect, getpeername, getsockname, listen, socket, recvmsg, sendmsg, sendto 
+ 16        chflags, fchflags 
+ 17        kill, vfork, execve 
+ 18        getegid, getgid, getgroups, setegid, setgid, setgroups 
+ 19        dup, dup2, getdtablesize 
+ 20        getrusage, profil 
+ 21        getitimer, setitimer, sigaction, sigpending, sigprocmask, sigsuspend, sigwait 
+ 22        acct 
+ 23        ioctl 
+ 24        chroot 
+ 25        madvise, mincore, minherit, mlock, mlock, mmap, mprotect, msync, munmap 
+ 26        getpgrp, getpgid, getsid, setpgid, setpgrp, setsid 
+ 27        fcntl 
+ 28        getlogin, setlogin 
+ 29        getpriority, setpriority 
+ 30        futimes, gettimeofday, settimeofday, utimes 
+ 31        rename, stat 
+ 32        flock 
+ 33        mkfifo, read, write 
+ 34        quotactl 
+ 35        getrlimit, setrlimit 
+ 36        getattrlist, getdirentries, getdirentriesattr, setattrlist 
+ 37        exchangedata 
+ 38        searchfs 
+ 39        sem_close, sem_open, sem_post, sem_trywait, sem_unlink, sem_wait 
+ 40        semctl, semget, semop 
+ 41        shm_open, shm_unlink 
+ 42        shmat, shmctl, shmdt, shmget 
+ 43        fgetxattr, flistxattr, fremovexattr, fsetxattr, getxattr, listxattr, removexattr, setxattr 
+ 44        aio_cancel, aio_error, aio_read, aio_return, aio_suspend, aio_write, fcntl, lio_listio 
+ 45        kevent, kqueue 
+ 46        msgctl, msgget, msgrcv, msgsnd 
+
+
+--- example of a full test run ---
+cottje% ./BUILD/dst/xnu_quick_test 
+created test directory at "/tmp/xnu_quick_test-1660251855" 
+Will allow 0 failures before testing is aborted 
+
+Begin testing - Tue Dec 13 15:56:50 2005
+ 
+test #1 - syscall 
+test #2 - fork, wait4, exit 
+test #3 - fsync, ftruncate, lseek, pread, pwrite, read, readv, truncate, write, writev 
+test #4 - close, fpathconf, fstat, open, pathconf 
+test #5 - link, stat, unlink 
+test #6 - chdir, fchdir 
+test #7 - access, chmod, fchmod 
+test #8 - chown, fchown, lchown, lstat, readlink, symlink 
+test #9 - fstatfs, getattrlist, getfsstat, statfs 
+test #10 - getpid, getppid, pipe 
+test #11 - getauid, gettid, getuid, geteuid, issetugid, setauid, seteuid, settid, settid_with_pid, setuid 
+test #12 - mkdir, rmdir, umask 
+test #13 - mknod, sync 
+test #14 - fsync, getsockopt, poll, select, setsockopt, socketpair 
+test #15 - accept, bind, connect, getpeername, getsockname, listen, socket, recvmsg, sendmsg, sendto 
+test #16 - chflags, fchflags 
+test #17 - kill, vfork, execve 
+test #18 - getegid, getgid, getgroups, setegid, setgid, setgroups 
+test #19 - dup, dup2, getdtablesize 
+test #20 - getrusage, profil 
+test #21 - ktrace 
+test #22 - getitimer, setitimer, sigaction, sigpending, sigprocmask, sigsuspend, sigwait 
+test #23 - acct 
+test #24 - ioctl 
+test #25 - chroot 
+test #26 - madvise, mincore, minherit, mlock, mlock, mmap, mprotect, msync, munmap 
+test #27 - getpgrp, getpgid, getsid, setpgid, setpgrp, setsid 
+test #28 - fcntl 
+test #29 - getlogin, setlogin 
+test #30 - getpriority, setpriority 
+test #31 - futimes, gettimeofday, settimeofday, utimes 
+test #32 - rename, stat 
+test #33 - flock 
+test #34 - mkfifo, read, write 
+test #35 - quotactl 
+test #36 - getrlimit, setrlimit 
+test #37 - getattrlist, getdirentries, getdirentriesattr, setattrlist 
+test #38 - exchangedata 
+test #39 - searchfs 
+test #40 - sem_close, sem_open, sem_post, sem_trywait, sem_unlink, sem_wait 
+test #41 - semctl, semget, semop 
+test #42 - shm_open, shm_unlink 
+test #43 - shmat, shmctl, shmdt, shmget 
+test #44 - fgetxattr, flistxattr, fremovexattr, fsetxattr, getxattr, listxattr, removexattr, setxattr 
+test #45 - aio_cancel, aio_error, aio_read, aio_return, aio_suspend, aio_write, fcntl, lio_listio 
+test #46 - kevent, kqueue 
+test #47 - msgctl, msgget, msgrcv, msgsnd 
+
+End testing - Tue Dec 13 15:57:08 2005
diff --git a/tools/tests/xnu_quick_test/helpers/arch.c b/tools/tests/xnu_quick_test/helpers/arch.c
new file mode 100644
index 000000000..fdf66fb53
--- /dev/null
+++ b/tools/tests/xnu_quick_test/helpers/arch.c
@@ -0,0 +1,24 @@
+#include <spawn.h>
+#include <mach/machine.h>
+
+/*
+ * Helper function for posix_spawn test: returns binary type as exit code.
+ */
+int main()
+{
+#if __ppc__
+	return CPU_TYPE_POWERPC;
+#endif /* __ppc__ */
+#if __ppc64__
+	return CPU_TYPE_POWERPC64;
+#endif /* __ppc64__ */
+#if __i386__
+	return CPU_TYPE_I386;
+#endif /* __i386__ */
+#if __x86_64__
+	return CPU_TYPE_X86_64;
+#endif /* __x86_64__ */
+	/* unrecognized type */
+	return -1;
+}
+
diff --git a/tools/tests/xnu_quick_test/helpers/launch.c b/tools/tests/xnu_quick_test/helpers/launch.c
new file mode 100644
index 000000000..5d5772bb6
--- /dev/null
+++ b/tools/tests/xnu_quick_test/helpers/launch.c
@@ -0,0 +1,118 @@
+/* 
+ *  Part of the execve tests. This program should not be compiled fat. xnu_quick_test
+ * will call the various single-architecture builds of this program as helpers to test
+ * the exec() transitions it cannot test itself.
+ *
+ * When running on a 64-bit machine (x86_64 or PPC64), the 32-bit version of 
+ * xnu_quick_test will fork and exec a 64-bit helper process that performs 
+ * the following tests.
+ * 1. 64 bit process forking() 64-bit child, child execing() 64-bit file(4GB pagezero)
+ * 2. 64 bit process forking() 64-bit child, child execing() 64-bit file (4KB pagezero)
+ * 3. 64 bit process forking() 64-bit child, child execing() 32-bit file
+ *
+ *  The 64-bit version of xnu_quick_test will fork and exec a 32-bit process 
+ * that performs the following tests.
+ * 4. 32 bit process forking() 32-bit child, child execing() 32-bit file
+ * 5. 32 bit process forking() 32-bit child, child execing() 64 bit file (4GB pagezero) 
+ * 6. 32 bit process forking() 32-bit child, child execing() 64 bit file (4KB pagezero)
+ */
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+
+extern int do_execve_test(char * path, char * argv[], void * envp, int killwait);
+extern int get_bits(void);
+
+
+int main(int argc, const char * argv[])
+{
+	int	my_err, my_status;
+	pid_t	my_pid, my_wait_pid;
+	char *	errmsg = NULL; 
+	char *	argvs[2] = {"", NULL};
+	int 	bits = get_bits();		/* Gets actual processor bit-ness. */
+
+#if defined(__i386__)
+	/* 
+	 * This is the helper binary for the x86_64 version of  xnu_quick_test. xnu_quick_test 
+	 * forks and execs this code to test exec()ing from a 32-bit binary.
+	 */
+	errmsg = "execve failed: from i386 forking and exec()ing i386 process.\n";
+	argvs[0] = "sleep-i386";
+	if (do_execve_test("helpers/sleep-i386", argvs, NULL, 0))	goto test_failed_exit;
+
+	errmsg = "execve failed: from i386 forking and exec()ing x86_64 process w/ 4G pagezero.\n";
+	argvs[0] = "sleep-x86_64-4G";
+	if (do_execve_test("helpers/sleep-x86_64-4G", argvs, NULL, 0))	goto test_failed_exit;
+
+	errmsg = "execve failed: from i386 forking and exec()ing x86_64 process w/ 4K pagezero.\n";
+	argvs[0] = "sleep-x86_64-4K";
+	if (do_execve_test("helpers/sleep-x86_64-4K", argvs, NULL, 0))	goto test_failed_exit;
+#endif
+
+
+#if defined(__x86_64__)
+	/* 
+	 * This is the helper binary for the i386 version of xnu_quick_test. xnu_quick_test 
+	 * forks and execs this code to test exec()ing from a 64-bit binary.
+	 */
+	errmsg = "execve failed: from x86_64 forking and exec()ing 64-bit x86_64 process w/ 4G pagezero.\n";
+	argvs[0] = "sleep-x86_64-4G";
+	if (do_execve_test("helpers/sleep-x86_64-4G", argvs, NULL, 1))		goto test_failed_exit;
+
+	errmsg = "execve failed: from x86_64 forking and exec()ing 64-bit x86_64 process w/ 4K Pagezero.\n";
+	argvs[0] = "sleep-x86_64-4K";
+	if (do_execve_test("helpers/sleep-x86_64-4K", argvs, NULL, 1))		goto test_failed_exit;
+
+	errmsg = "execve failed: from x64_64 forking and exec()ing 32-bit i386 process.\n";
+	argvs[0] = "sleep-i386";
+	if (do_execve_test("helpers/sleep-i386", argvs, NULL, 1))		goto test_failed_exit;
+#endif
+
+
+#if defined(__ppc__)
+	/* 
+	 * This is the helper binary for the PPC64 version of xnu_quick_test. xnu_quick_test 
+	 * forks and execs this code to test exec()ing from a 32-bit binary.
+	 */
+	errmsg = "execve failed: from ppc forking and exec()ing ppc process.\n";
+	argvs[0] = "sleep-ppc32";
+	if (do_execve_test("helpers/sleep-ppc32", argvs, NULL, 0))	goto test_failed_exit;
+
+	errmsg = "execve failed: from ppc forking and exec()ing ppc64 process w/ 4G pagezero.\n";
+	argvs[0] = "sleep-ppc64-4G";
+	if (do_execve_test("helpers/sleep-ppc64-4G", argvs, NULL, 0))	goto test_failed_exit;
+
+	errmsg = "execve failed: from ppc forking and exec()ing ppc64 process w/ 4K pagezero.\n";
+	argvs[0] = "sleep-ppc64-4K";
+	if (do_execve_test("helpers/sleep-ppc64-4K", argvs, NULL, 0))	goto test_failed_exit;
+#endif
+
+
+#if defined(__ppc64__)
+	/* 
+	 * This is the helper binary for the ppc version of xnu_quick_test. xnu_quick_test 
+	 * forks and execs this code to test exec()ing from a 64-bit binary.
+	 */
+	errmsg = "execve failed: from ppc64 forking and exec()ing 64-bit ppc process w/ 4G pagezero.\n";
+	argvs[0] = "sleep-ppc64-4G";
+	if (do_execve_test("helpers/sleep-ppc64-4G", argvs, NULL, 1))		goto test_failed_exit;
+
+	errmsg = "execve failed: from ppc64 forking and exec()ing 64-bit ppc process w/ 4K pagezero.\n";
+	argvs[0] = "sleep-ppc64-4K";
+	if (do_execve_test("helpers/sleep-ppc64-4K", argvs, NULL, 1))		goto test_failed_exit;
+
+	errmsg = "execve failed: from ppc64 forking and exec()ing 32 bit ppc process.\n";
+	argvs[0] = "sleep-ppc32";
+	if (do_execve_test("helpers/sleep-ppc32", argvs, NULL, 1))		goto test_failed_exit;	
+#endif
+
+	return 0;
+
+test_failed_exit:
+	if (errmsg)
+		printf("%s", errmsg);
+	return -1;
+}
+
diff --git a/tools/tests/xnu_quick_test/helpers/sleep.c b/tools/tests/xnu_quick_test/helpers/sleep.c
new file mode 100644
index 000000000..e74110f3e
--- /dev/null
+++ b/tools/tests/xnu_quick_test/helpers/sleep.c
@@ -0,0 +1,6 @@
+/* Helper binary for the execve tests. Added for PR-4607285 */
+#include <unistd.h>
+int main()
+{
+	sleep(120);
+}
diff --git a/tools/tests/xnu_quick_test/main.c b/tools/tests/xnu_quick_test/main.c
new file mode 100644
index 000000000..237d14cdc
--- /dev/null
+++ b/tools/tests/xnu_quick_test/main.c
@@ -0,0 +1,527 @@
+/* 
+ * xnu_quick_test - this tool will do a quick test of every (well, to be
+ * honest most) system calls we support in xnu.
+ *
+ * WARNING - this is not meant to be a full regression test of all the
+ * system calls.  The intent is to have a quick test of each system call that
+ * can be run very easily and quickly when a new kerenl is built.
+ *
+ * This tool is meant to grow as we find xnu problems that could have be
+ * caught before we submit to a build train.  So please add more tests and
+ * make the existing ones better.  Some of the original tests are nothing
+ * more than place holders and quite lame.  Just keep in mind that the tool
+ * should run as fast as possible.  If it gets too slow then most people
+ * will stop running it.
+ *
+ * LP64 testing tip - when adding or modifying tests, keep in mind the
+ * variants in the LP64 world.  If xnu gets passed a structure the varies in
+ * size between 32 and 64-bit processes, try to test that a field in the 
+ * sructure contains valid data.  For example if we know foo structure
+ * looks like:
+ * struct foo {
+ *		int		an_int;
+ *		long	a_long;
+ *		int		another_int;
+ * }
+ * And if we know what another_int should contain then test for the known
+ * value since it's offset will vary depending on whether the calling process
+ * is 32 or 64 bits.
+ *
+ * NOTE - we have several workarounds and test exceptions for some
+ * outstanding bugs in xnu.  All the workarounds are marked with "todo" and
+ * some comments noting the radar number of the offending bug.  Do a seach
+ * for "todo" in the source files for this project to locate which tests have
+ * known failures.   And please tag any new exceptions you find with "todo"
+ * in the comment and the radar number of the bug.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/mount.h>
+#include <sys/param.h>
+#include <sys/select.h>
+#include <sys/stat.h>
+#include <sys/syslimits.h>
+#include <sys/types.h>
+#include <sys/ucred.h>
+#include <sys/uio.h>
+#include <XILog/XILog.h>
+
+#include "tests.h"
+
+
+/* our table of tests to run  */
+struct test_entry   g_tests[] =
+{
+	{1, &syscall_test, NULL, "syscall"},
+	{1, &fork_wait4_exit_test, NULL, "fork, wait4, exit"},
+	{1, &read_write_test, NULL, "fsync, ftruncate, lseek, pread, pwrite, read, readv, truncate, write, writev"},
+	{1, &open_close_test, NULL, "close, fpathconf, fstat, open, pathconf"},
+	{1, &link_stat_unlink_test, NULL, "link, stat, unlink"},
+	{1, &chdir_fchdir_test, NULL, "chdir, fchdir"},
+	{1, &access_chmod_fchmod_test, NULL, "access, chmod, fchmod"},
+	{1, &chown_fchown_lchown_lstat_symlink_test, NULL, "chown, fchown, lchown, lstat, readlink, symlink"},
+	{1, &fs_stat_tests, NULL, "fstatfs, getattrlist, getfsstat, statfs, getfsstat64, statfs64, fstatfs64"},
+	{1, &getpid_getppid_pipe_test, NULL, "getpid, getppid, pipe"},
+	{1, &uid_tests, NULL, "getauid, gettid, getuid, geteuid, issetugid, setauid, seteuid, settid, settid_with_pid, setuid"},
+	{1, &mkdir_rmdir_umask_test, NULL, "mkdir, rmdir, umask"},
+	{1, &mknod_sync_test, NULL, "mknod, sync"},
+	{1, &socket2_tests, NULL, "fsync, getsockopt, poll, select, setsockopt, socketpair"},
+	{1, &socket_tests, NULL, "accept, bind, connect, getpeername, getsockname, listen, socket, recvmsg, sendmsg, sendto"},
+	{1, &chflags_fchflags_test, NULL, "chflags, fchflags"},
+	{1, &execve_kill_vfork_test, NULL, "kill, vfork, execve, posix_spawn"},
+	{1, &groups_test, NULL, "getegid, getgid, getgroups, setegid, setgid, setgroups"},
+	{1, &dup_test, NULL, "dup, dup2, getdtablesize"},
+	{1, &getrusage_profil_test, NULL, "getrusage, profil"},
+	{1, &signals_test, NULL, "getitimer, setitimer, sigaction, sigpending, sigprocmask, sigsuspend, sigwait"},
+	{1, &acct_test, NULL, "acct"},
+	{1, &ioctl_test, NULL, "ioctl"},
+	{1, &chroot_test, NULL, "chroot"},
+	{1, &memory_tests, NULL, "madvise, mincore, minherit, mlock, mlock, mmap, mprotect, msync, munmap"},
+	{1, &process_group_test, NULL, "getpgrp, getpgid, getsid, setpgid, setpgrp, setsid"},
+	{1, &fcntl_test, NULL, "fcntl"},
+	{1, &getlogin_setlogin_test, NULL, "getlogin, setlogin"},
+	{1, &getpriority_setpriority_test, NULL, "getpriority, setpriority"},
+	{1, &time_tests, NULL, "futimes, gettimeofday, settimeofday, utimes"},
+	{1, &rename_test, NULL, "rename, stat"},
+	{1, &locking_test, NULL, "flock"},
+	{1, &mkfifo_test, NULL, "mkfifo, read, write"},
+	{1, &quotactl_test, NULL, "quotactl"},
+	{1, &limit_tests, NULL, "getrlimit, setrlimit"},
+	{1, &directory_tests, NULL, "getattrlist, getdirentries, getdirentriesattr, setattrlist"},
+	{1, &exchangedata_test, NULL, "exchangedata"},
+	{1, &searchfs_test, NULL, "searchfs"},
+	{1, &sema2_tests, NULL, "sem_close, sem_open, sem_post, sem_trywait, sem_unlink, sem_wait"},
+	{1, &sema_tests, NULL, "semctl, semget, semop"},
+	{1, &bsd_shm_tests, NULL, "shm_open, shm_unlink"},
+	{1, &shm_tests, NULL, "shmat, shmctl, shmdt, shmget"},
+	{1, &xattr_tests, NULL, "fgetxattr, flistxattr, fremovexattr, fsetxattr, getxattr, listxattr, removexattr, setxattr"},
+	{1, &aio_tests, NULL, "aio_cancel, aio_error, aio_read, aio_return, aio_suspend, aio_write, fcntl, lio_listio"},
+	{1, &kqueue_tests, NULL, "kevent, kqueue"},
+	{1, &message_queue_tests, NULL, "msgctl, msgget, msgrcv, msgsnd"},
+	{0, NULL, NULL, "last one"}
+};
+
+static void create_target_directory( const char * the_targetp );
+static void list_all_tests( void );
+static void mark_tests_to_run( long my_start, long my_end );
+static int parse_tests_to_run( int argc, const char * argv[], int * indexp );
+static void usage( void );
+
+/* globals */
+long		g_max_failures = 0;
+int		g_skip_setuid_tests = 0;
+int		g_xilog_active = 0;
+const char *	g_cmd_namep;
+char		g_target_path[ PATH_MAX ];
+int		g_is_under_rosetta = 0;
+ 
+int main( int argc, const char * argv[] ) 
+{
+	int				my_tests_count, i;
+	int				err;
+	int				my_failures = 0;
+	int				list_the_tests = 0;
+	const char *	my_targetp;
+	time_t			my_start_time, my_end_time;
+	struct stat		my_stat_buf;
+	char			my_buffer[64];
+	/* vars for XILog */
+	XILogRef		logRef;
+	char			*logPath = "";
+	char			*config = NULL;
+	int				echo = 0;
+	int				xml = 0;
+	
+	sranddev( );				/* set up seed for our random name generator */
+	g_cmd_namep = argv[0];
+	
+	/* NOTE - code in create_target_directory will append '/' if it is necessary */
+	my_targetp = getenv("TMPDIR");
+	if ( my_targetp == NULL )
+		my_targetp = "/tmp";
+	
+	/* make sure our executable is owned by root and has set uid bit */
+	err = stat( g_cmd_namep, &my_stat_buf );
+	if ( err != 0 ) {
+		err = errno;
+		printf( "stat call on our executable failed -  \"%s\" \n", g_cmd_namep );
+		printf( " failed with error %d - \"%s\" \n", err, strerror( err) );
+		exit( -1 );
+	}
+	if ( my_stat_buf.st_uid != 0 || (my_stat_buf.st_mode & S_ISUID) == 0 ) {
+		printf( "executable file -  \"%s\" \n", g_cmd_namep );
+		printf( "does not have correct owner (must be root) or setuid bit is not set \n" );
+		exit( -1 );
+	}
+	
+	/* parse input parameters */
+	for ( i = 1; i < argc; i++ ) {
+		if ( strcmp( argv[i], "-u" ) == 0 ) {
+			usage( );
+		}
+		if ( strcmp( argv[i], "-t" ) == 0 ||
+			 strcmp( argv[i], "-target" ) == 0 ) {
+			if ( ++i >= argc ) {
+				printf( "invalid target parameter \n" );
+				usage( );
+			}
+			/* verify our target directory exists */
+			my_targetp = argv[i];
+			err = stat( my_targetp, &my_stat_buf );
+			if ( err != 0 || S_ISDIR(my_stat_buf.st_mode) == 0 ) {
+				printf( "invalid target path \n" );
+				if ( err != 0 ) {
+					printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+				}
+				usage( );
+			}
+			continue;
+		}
+		if ( strcmp( argv[i], "-f" ) == 0 ||
+			 strcmp( argv[i], "-failures" ) == 0 ) {
+			if ( ++i >= argc ) {
+				printf( "invalid failures parameter \n" );
+				usage( );
+			}
+
+			/* get our max number of failures */
+			g_max_failures = strtol( argv[i], NULL, 10 );
+			continue;
+		}
+		if ( strcmp( argv[i], "-l" ) == 0 ||
+			 strcmp( argv[i], "-list" ) == 0 ) {
+			/* list all the tests this tool will do.
+			 */
+			list_the_tests = 1;
+			continue;
+		}
+		if ( strcmp( argv[i], "-r" ) == 0 ||
+			 strcmp( argv[i], "-run" ) == 0 ) {
+			if ( ++i >= argc ) {
+				printf( "invalid run tests parameter \n" );
+				usage( );
+			}
+
+			/* get which tests to run */
+			if ( parse_tests_to_run( argc, argv, &i ) != 0 ) {
+				printf( "invalid run tests parameter \n" );
+				usage( );
+			}
+			continue;
+		}
+		if ( strcmp( argv[i], "-s" ) == 0 ||
+			 strcmp( argv[i], "-skip" ) == 0 ) {
+			/* set that want to skip the setuid related tests - this is useful for debgugging since since I can't  
+			 * get setuid tests to work while in gdb.
+			 */
+			g_skip_setuid_tests = 1;
+			continue;
+		}
+		if ( strcmp( argv[i], "-x" ) == 0 ||
+			 strcmp( argv[i], "-xilog" ) == 0 ) {
+			g_xilog_active = 1;
+			continue;
+		}
+
+		printf( "invalid argument \"%s\" \n", argv[i] );
+		usage( );
+	}
+
+	/* done parsing.
+	 */
+    
+#ifdef __ppc__
+	/* determine if we are running under Rosetta 
+	 */
+	{
+	    int val = 0;
+	    size_t size = sizeof val;
+	    if (sysctlbyname("sysctl.proc_native", &val, &size, NULL, 0) == -1)
+		g_is_under_rosetta = 0;
+	    else
+		g_is_under_rosetta = val ? 0 : 1;
+	}
+#endif
+    
+	if ( list_the_tests != 0 ) {
+		list_all_tests( );
+		return 0;
+	}
+
+	if (g_xilog_active == 1) {	
+		logRef = XILogOpenLogExtended( logPath, "xnu_quick_test", "com.apple.coreos", 
+										config, xml, echo, NULL, "ResultOwner", 
+										"com.apple.coreos", NULL );
+		if( logRef == NULL )  {
+			fprintf(stderr,"Couldn't create log: %s",logPath);
+			exit(-1);
+		}
+	}
+	
+	/* build a test target directory that we use as our path to create any test
+	 * files and directories.
+	 */
+	create_target_directory( my_targetp );
+	printf( "Will allow %d failures before testing is aborted \n", g_max_failures );
+	
+        if (g_is_under_rosetta) {
+                printf("Running under Rosetta.\n");
+        }
+        
+	my_start_time = time( NULL );
+	printf( "\nBegin testing - %s \n", ctime_r( &my_start_time, &my_buffer[0] ) );
+	
+	/* run each test that is marked to run in our table until we complete all of them or
+	 * hit the maximum number of failures.
+	 */
+	my_tests_count = (sizeof( g_tests ) / sizeof( g_tests[0] ));
+	for ( i = 0; i < (my_tests_count - 1); i++ ) {
+		int				my_err;
+		test_entryp		my_testp;
+
+		my_testp = &g_tests[i];
+		if ( my_testp->test_run_it == 0 || my_testp->test_routine == NULL )
+			continue;
+		if (g_xilog_active == 1) {	
+			XILogBeginTestCase( logRef, my_testp->test_infop, my_testp->test_infop );	
+			XILogMsg( "test #%d - %s \n", (i + 1), my_testp->test_infop );
+		}
+		printf( "test #%d - %s \n", (i + 1), my_testp->test_infop );
+		my_err = my_testp->test_routine( my_testp->test_input );
+		if ( my_err != 0 ) {
+			printf("\t--> FAILED \n");
+			if (g_xilog_active == 1) {	
+				XILogMsg("SysCall %s failed", my_testp->test_infop);
+				XILogErr("Result %d", my_err);
+			}
+			my_failures++;
+			if ( my_failures > g_max_failures ) {
+				if (g_xilog_active == 1) {	
+					XILogEndTestCase( logRef, kXILogTestPassOnErrorLevel );
+				}
+				printf( "\n too many failures - test aborted \n" );
+				goto exit_this_routine;
+			}
+		}
+		if (g_xilog_active == 1) {	
+			XILogEndTestCase(logRef, kXILogTestPassOnErrorLevel);
+		}
+	}
+	
+exit_this_routine:
+	my_end_time = time( NULL );
+	printf( "\nEnd testing - %s \n", ctime_r( &my_end_time, &my_buffer[0] ) );
+
+	/* clean up our test directory */
+	rmdir( &g_target_path[0] );	
+
+	if (g_xilog_active == 1) {	
+		XILogCloseLog(logRef);
+	}
+	
+    return 0;
+} /* main */
+
+
+/* 
+ * parse_tests_to_run - parse the -run argument parameters.  the run argument tells us which tests the user
+ * wants to run.  we accept ranges (example  1 - 44) and runs of tests (example 2, 33, 34, 100) or a mix of
+ * both (example  1, 44 - 100, 200, 250)
+ */
+static int parse_tests_to_run( int argc, const char * argv[], int * indexp )
+{
+	int				my_tests_count, is_range = 0, i;
+	const char *	my_ptr;
+	char *			my_temp_ptr;
+	long			my_first_test_number, my_last_test_number;
+	char			my_buffer[ 128 ];
+	
+	/* set tests table to not run any tests then go back and set the specific tests the caller asked for */
+	my_tests_count = (sizeof( g_tests ) / sizeof( g_tests[0] ));
+	for ( i = 0; i < (my_tests_count - 1); i++ ) {
+		g_tests[ i ].test_run_it = 0;
+	}
+	 
+	for ( i = *indexp; i < argc; i++ ) {
+		my_ptr = argv[ i ];
+		if ( strlen( my_ptr ) > 1 && *my_ptr == '-' && isalpha( *(my_ptr + 1) ) ) {
+			/* we have hit a new argument - need to make sure caller uses this argument on the next
+			 * pass through its parse loop (which will bump the index value so we want to be one less
+			 * than the actual index). 
+			 */
+			*indexp = (i - 1);
+			return 0;
+		}
+		
+		if ( strlen( my_ptr ) == 1 && *my_ptr == '-' ) {
+			/* we are dealing with a range of tests, for example:  33 - 44  */
+			is_range = 1;
+			continue;
+		}
+
+		if ( strlen( my_ptr ) > (sizeof( my_buffer ) - 1) ) {
+			printf( "-run argument has too many test parameters (max of %d characters) \n", sizeof( my_buffer ) );
+			return -1;
+		}
+		/* get a local copy of the parameter string to work with - break range into two strings */
+		strcpy( &my_buffer[0], my_ptr );
+
+		my_temp_ptr = strrchr( &my_buffer[0], '-' );
+		if ( my_temp_ptr != NULL ) {
+			/* we are dealing with a range of tests with no white space, for example:  33-44  or  33- 44  */
+			my_temp_ptr = strrchr( &my_buffer[0], '-' );
+			*my_temp_ptr = 0x00;
+			my_first_test_number = strtol( &my_buffer[0], NULL, 10 );
+			if ( *(my_temp_ptr + 1) == 0x00 ) {
+				/* we hit the case where the range indicator is at the end of one string, for example:  33-  */
+				is_range = 1;
+				continue;
+			}
+			my_last_test_number = strtol( (my_temp_ptr + 1), NULL, 10 );
+			if ( my_first_test_number < 1 || my_first_test_number > my_last_test_number ) {
+				printf( "-run argument has invalid range parmeters \n" );
+				return -1;
+			}
+			mark_tests_to_run( my_first_test_number, my_last_test_number );
+			is_range = 0;
+			continue;
+		}
+		
+		if ( is_range ) {
+			/* should be the second part of the test number range */
+			my_last_test_number = strtol( &my_buffer[0], NULL, 10 );
+			if ( my_first_test_number < 1 || my_first_test_number > my_last_test_number ) {
+				printf( "-run argument has invalid range parmeters \n" );
+				return -1;
+			}
+
+			mark_tests_to_run( my_first_test_number, my_last_test_number );
+			is_range = 0;
+			continue;
+		}
+		else {
+			my_first_test_number = strtol( &my_buffer[0], NULL, 10 );
+			if ( my_first_test_number < 1 ) {
+				printf( "-run argument has invalid test number parameter \n" );
+				return -1;
+			}
+			mark_tests_to_run( my_first_test_number, my_first_test_number );
+			continue;
+		}
+	}
+	
+	*indexp = i;
+	return 0;
+	
+} /* parse_tests_to_run */
+
+
+static void create_target_directory( const char * the_targetp )
+{
+	int			err;
+	
+	if ( strlen( the_targetp ) > (sizeof(g_target_path) - 1) ) {
+		printf( "target path too long - \"%s\" \n", the_targetp );
+		exit( 1 );
+	}
+	
+	for ( ;; ) {
+        int			my_rand;
+        char		my_name[64];
+		
+        my_rand = rand( );
+        sprintf( &my_name[0], "xnu_quick_test-%d", my_rand );
+        if ( (strlen( &my_name[0] ) + strlen( the_targetp ) + 2) > PATH_MAX ) {
+			printf( "target path plus our test directory name is too long: \n" );
+			printf( "target path - \"%s\"  \n", the_targetp );
+			printf( "test directory name - \"%s\"  \n", &my_name[0] );
+			exit( 1 );
+		}
+
+        /* append generated directory name onto our path */
+		g_target_path[0] = 0x00;
+        strcat( &g_target_path[0], the_targetp );
+		if ( g_target_path[ (strlen(the_targetp) - 1) ] != '/' ) {
+			strcat( &g_target_path[0], "/" );
+		}
+        strcat( &g_target_path[0], &my_name[0] );
+		
+		/* try to create the test directory */
+		err = mkdir( &g_target_path[0], (S_IRWXU | S_IRWXG | S_IROTH) );
+		if ( err == 0 ) {
+			break;
+		}
+		err = errno;
+		if ( EEXIST != err ) {
+			printf( "test directory creation failed - \"%s\" \n", &g_target_path[0] );
+			printf( "mkdir call failed with error %d - \"%s\" \n", errno, strerror( err) );
+			exit( 1 );
+		}
+	}
+	printf( "created test directory at \"%s\" \n", &g_target_path[0] );
+	
+} /* create_target_directory */
+
+
+static void list_all_tests( void )
+{
+	int		i, my_tests_count;
+	
+	my_tests_count = (sizeof( g_tests ) / sizeof( g_tests[0] ));
+	printf( "\nList of all tests this tool performs... \n" );
+
+	for ( i = 0; i < (my_tests_count - 1); i++ ) {
+		printf( " %d \t   %s \n", (i + 1), g_tests[ i ].test_infop );
+	}
+	
+	return;
+} /* list_all_tests */
+
+
+static void mark_tests_to_run( long my_start, long my_end )
+{
+	int			my_tests_count, i;
+
+	my_tests_count = (sizeof( g_tests ) / sizeof( g_tests[0] ));
+	my_end = (my_end < (my_tests_count - 1)) ? my_end : (my_tests_count - 1);
+	for ( i = (my_start - 1); i < my_end; i++ ) {
+		g_tests[ i ].test_run_it = 1;  /* run this test */
+	}
+	return;
+} /* mark_tests_to_run */
+
+
+static void usage( void )
+{
+	char *		my_ptr;
+	
+	/* skip past full path and just show the tool name */
+	my_ptr = strrchr( g_cmd_namep, '/' );
+	if ( my_ptr != NULL ) {
+		my_ptr++;
+	}
+	
+	printf( "\nUSAGE:  %s -target TARGET_PATH \n\n", (my_ptr != NULL) ? my_ptr : g_cmd_namep );
+	printf( "\t -f[ailures] MAX_FAILS_ALLOWED   # number of test cases that may fail before we give up.  defaults to 0  \n" );
+	printf( "\t -l[ist]                         # list all the tests this tool performs   \n" );
+	printf( "\t -r[un] 1, 3, 10 - 19            # run specific tests.  enter individual test numbers and/or range of numbers.  use -list to list tests.   \n" );
+	printf( "\t -s[kip]                         # skip setuid tests   \n" );
+	printf( "\t -t[arget] TARGET_PATH           # path to directory where tool will create test files.  defaults to \"/tmp/\"   \n" );
+	printf( "\t -x[ilog]                        # use XILog\n");
+	printf( "\nexamples:  \n" );
+	printf( "--- Place all test files and directories at the root of volume \"test_vol\" --- \n" );
+	printf( "%s -t /Volumes/test_vol/ \n", (my_ptr != NULL) ? my_ptr : g_cmd_namep );
+	printf( " \n" );
+	printf( "--- Run the tool for tests 10 thru 15, test 18 and test 20 --- \n" );
+	printf( "%s -r 10-15, 18, 20 \n", (my_ptr != NULL) ? my_ptr : g_cmd_namep );
+	printf( " \n" );
+	exit( 1 );
+
+} /* usage */
+
diff --git a/tools/tests/xnu_quick_test/makefile b/tools/tests/xnu_quick_test/makefile
new file mode 100644
index 000000000..8cdb458d0
--- /dev/null
+++ b/tools/tests/xnu_quick_test/makefile
@@ -0,0 +1,112 @@
+ifdef RC_BUILDIT
+DOING_BUILDIT=yes
+endif
+
+ifdef RC_OS
+DOING_BUILDIT=yes
+endif
+
+ifdef DOING_BUILDIT
+include $(MAKEFILEPATH)/CoreOS/ReleaseControl/Common.make
+MY_ARCH = $(patsubst %, -arch %, $(RC_ARCHS)) 
+install:: xnu_quick_test
+else
+	ifndef SRCROOT
+		SRCROOT=$(shell /bin/pwd)
+	endif
+	ifndef OBJROOT
+		OBJROOT=$(SRCROOT)/BUILD/obj
+	endif
+	ifndef DSTROOT
+		DSTROOT=$(SRCROOT)/BUILD/dst
+	endif
+	
+	ifndef ARCH
+		ARCH=i386 x86_64 ppc ppc64
+	endif
+	
+	ifdef ARCH
+		MY_ARCH = $(patsubst %, -arch %, $(ARCH)) # allows building multiple archs.
+	endif
+
+	CFLAGS += $(MY_ARCH)
+endif
+
+CFLAGS += -g -I /System/Library/Frameworks/System.framework/Versions/B/PrivateHeaders/ -F/AppleInternal/Library/Frameworks/ $(MORECFLAGS)
+LIBFLAGS = -I /System/Library/Frameworks/System.framework/Versions/B/PrivateHeaders/ -F/AppleInternal/Library/Frameworks/ -framework XILog
+#CFLAGS+= $(MY_ARCH) -g -D_POSIX_C_SOURCE=200112L
+
+MY_OBJECTS = $(OBJROOT)/main.o $(OBJROOT)/memory_tests.o $(OBJROOT)/misc.o \
+			 $(OBJROOT)/sema_tests.o $(OBJROOT)/shared_memory_tests.o \
+			 $(OBJROOT)/socket_tests.o $(OBJROOT)/tests.o $(OBJROOT)/xattr_tests.o
+
+
+xnu_quick_test : $(OBJROOT) $(DSTROOT) $(MY_OBJECTS) helpers
+	sudo rm -rf $(DSTROOT)/xnu_quick_test
+	cc $(MY_ARCH) $(LIBFLAGS) -o $(DSTROOT)/xnu_quick_test $(MY_OBJECTS)
+	sudo chown root $(DSTROOT)/xnu_quick_test
+	sudo chmod 4755 $(DSTROOT)/xnu_quick_test
+
+# The helper binaries are used to test exec()'ing between 64bit and 32bit. 
+# Creates test binaries with page zero sizes = 4KB and 4GB. Also creates 32-bit
+# helper processes for the 64-bit version of xnu_quick_test to test the conversion
+# from a 32-bit process to a 64-bit process.
+helpers : helpers/sleep.c helpers/launch.c helpers/arch.c helperdir $(OBJROOT)/misc.o
+	gcc -arch ppc                               helpers/sleep.c -o $(DSTROOT)/helpers/sleep-ppc32
+	gcc -arch i386                              helpers/sleep.c -o $(DSTROOT)/helpers/sleep-i386
+	gcc -arch x86_64 -pagezero_size 0x100000000 helpers/sleep.c -o $(DSTROOT)/helpers/sleep-x86_64-4G
+	gcc -arch x86_64 -pagezero_size 0x1000      helpers/sleep.c -o $(DSTROOT)/helpers/sleep-x86_64-4K
+	gcc -arch ppc64  -pagezero_size 0x100000000 helpers/sleep.c -o $(DSTROOT)/helpers/sleep-ppc64-4G
+	gcc -arch ppc64  -pagezero_size 0x1000      helpers/sleep.c -o $(DSTROOT)/helpers/sleep-ppc64-4K
+	gcc $(LIBFLAGS) -arch i386	$(OBJROOT)/misc.o helpers/launch.c -o $(DSTROOT)/helpers/launch-i386
+	gcc $(LIBFLAGS) -arch x86_64	$(OBJROOT)/misc.o helpers/launch.c -o $(DSTROOT)/helpers/launch-x86_64
+	gcc $(LIBFLAGS) -arch ppc	$(OBJROOT)/misc.o helpers/launch.c -o $(DSTROOT)/helpers/launch-ppc
+	gcc $(LIBFLAGS) -arch ppc64	$(OBJROOT)/misc.o helpers/launch.c -o $(DSTROOT)/helpers/launch-ppc64
+	gcc -arch ppc -arch ppc64 -arch i386 -arch x86_64 helpers/arch.c -o $(DSTROOT)/helpers/arch
+	
+helperdir :
+	mkdir -p $(DSTROOT)/helpers
+
+$(OBJROOT) :
+	mkdir -p $(OBJROOT);
+	
+$(DSTROOT) :
+	mkdir -p $(DSTROOT);
+
+INCLUDES = /Developer/SDKs/Purple/System/Library/Frameworks/System.framework/Versions/B/PrivateHeaders/
+	
+
+$(OBJROOT)/main.o : main.c tests.h
+	cc $(CFLAGS) -c main.c  -o $@
+	
+$(OBJROOT)/memory_tests.o : memory_tests.c tests.h
+	cc $(CFLAGS) -c memory_tests.c  -o $@
+
+# misc.o has to be built 4-way for the helpers to link
+$(OBJROOT)/misc.o : misc.c tests.h
+	cc -arch i386 -arch x86_64 -arch ppc -arch ppc64 $(CFLAGS) -c misc.c   -o $@
+	
+$(OBJROOT)/sema_tests.o : sema_tests.c tests.h
+	cc $(CFLAGS) -c sema_tests.c   -o $@
+	
+$(OBJROOT)/shared_memory_tests.o : shared_memory_tests.c tests.h
+	cc $(CFLAGS) -c shared_memory_tests.c   -o $@
+
+$(OBJROOT)/socket_tests.o : socket_tests.c tests.h
+	cc $(CFLAGS) -c socket_tests.c   -o $@
+
+$(OBJROOT)/tests.o : tests.c tests.h
+	cc $(CFLAGS) -c tests.c    -o $@
+
+$(OBJROOT)/xattr_tests.o : xattr_tests.c tests.h
+	cc $(CFLAGS) -c xattr_tests.c    -o $@
+
+
+ifndef DOING_BUILDIT
+.PHONY : clean
+clean :
+	sudo rm -f $(DSTROOT)/xnu_quick_test
+	sudo rm -f $(DSTROOT)/helpers/*
+	rm -f $(OBJROOT)/*.o
+endif
+
diff --git a/tools/tests/xnu_quick_test/memory_tests.c b/tools/tests/xnu_quick_test/memory_tests.c
new file mode 100644
index 000000000..952be5ab9
--- /dev/null
+++ b/tools/tests/xnu_quick_test/memory_tests.c
@@ -0,0 +1,255 @@
+/*
+ *  memory_tests.c.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 4/12/05.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+
+extern char  g_target_path[ PATH_MAX ];
+
+/*  **************************************************************************************************************
+ *	Test madvise, mincore, minherit, mlock, mlock, mmap, mprotect, msync, munmap system calls.
+ *	todo - see if Francois has better versions of these tests...
+ *  **************************************************************************************************************
+ */
+int memory_tests( void * the_argp )
+{
+	int			my_err;
+	int			my_page_size, my_status;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+	char *		my_bufp = NULL;
+	char *		my_addr = NULL;
+	char *		my_test_page_p = NULL;
+	ssize_t		my_result;
+	pid_t		my_pid, my_wait_pid;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_page_size = getpagesize( );
+	my_test_page_p = (char *) malloc( my_page_size );
+	if ( my_test_page_p == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_test_page_p = 0x00;
+	strcat( my_test_page_p, "parent data" );
+ 
+	/* test minherit - share a page with child, add to the string in child then 
+	 * check for modification after child terminates.
+	 */ 
+	my_err = minherit( my_test_page_p, my_page_size, VM_INHERIT_SHARE );
+	if ( my_err == -1 ) {
+		printf( "minherit failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process...
+		 */		
+		strcat( my_test_page_p, " child data" );
+
+		/* create a test file in page size chunks */
+		my_bufp = (char *) malloc( (my_page_size * 10) );
+		if ( my_bufp == NULL ) {
+			printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		memset( my_bufp, 'j', (my_page_size * 10) );
+		my_fd = open( my_pathp, O_RDWR, 0 );
+		if ( my_fd == -1 ) {
+			printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		
+		my_result = write( my_fd, my_bufp, (my_page_size * 10) );
+		if ( my_result == -1 ) {
+			printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		
+		/* map the file into memory */
+		my_addr = (char *) mmap( NULL, (my_page_size * 2), (PROT_READ | PROT_WRITE), (MAP_FILE | MAP_SHARED), my_fd, 0 );
+		if ( my_addr == (char *) -1 ) {
+			printf( "mmap call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		
+		/* make sure we got the right data mapped */
+		if ( *my_addr != 'j' || *(my_addr + my_page_size) != 'j' ) {
+			printf( "did not map in correct data \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		/* test madvise */
+		my_err = madvise( my_addr, (my_page_size * 2), MADV_WILLNEED );
+		if ( my_err == -1 ) {
+			printf( "madvise call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		/* test mincore, mlock, mlock */
+		my_err = mlock( my_addr, my_page_size );
+		if ( my_err == -1 ) {
+			printf( "mlock call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		my_err = mincore( my_addr, 1, my_bufp );	
+		if ( my_err == -1 ) {
+			printf( "mincore call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		/* page my_addr is in should be resident after mlock */
+		if ( (*my_bufp & MINCORE_INCORE) == 0 ) {
+			printf( "mincore call failed to find resident page \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+		
+		my_err = munlock( my_addr, my_page_size );
+		if ( my_err == -1 ) {
+			printf( "munlock call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		 
+		/* modify first page then use msync to push data out */
+		memset( my_addr, 'x', my_page_size );
+		my_err = msync( my_addr, my_page_size, (MS_SYNC | MS_INVALIDATE) );	
+		if ( my_err == -1 ) {
+			printf( "msync call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+					
+		/* verify that the file was updated */
+		lseek( my_fd, 0, SEEK_SET );	
+		bzero( (void *)my_bufp, my_page_size );
+		my_result = read( my_fd, my_bufp, my_page_size );
+		if ( my_result == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		if ( *my_bufp != 'x' ) {
+			printf( "msync did not flush correct data \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+		 
+		/* unmap our test page */
+		my_err = munmap( my_addr, (my_page_size * 2) );
+		if ( my_err == -1 ) {
+			printf( "munmap call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		my_addr = NULL;
+
+		/* map the file into memory again for mprotect test  */
+		my_addr = (char *) mmap( NULL, (my_page_size * 2), (PROT_READ | PROT_WRITE), (MAP_FILE | MAP_SHARED), my_fd, 0 );
+		if ( my_addr == (char *) -1 ) {
+			printf( "mmap call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		*my_addr = 'a';
+	 
+
+
+		/* test mprotect - change protection to only PROT_READ */
+		my_err = mprotect( my_addr, my_page_size, PROT_READ );
+		if ( my_err == -1 ) {
+			printf( "mprotect call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		*my_addr = 'z'; /* should cause SIGBUS signal (we look for this at child termination within the parent) */
+	
+
+		 
+		my_err = 0;
+exit_child:
+		exit( my_err );
+	}
+
+	
+	/* parent process -
+	 * we should get SIGBUS exit when child tries to write to read only memory 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}
+
+	if ( WIFSIGNALED( my_status ) && WTERMSIG( my_status ) != SIGBUS ) {
+		printf( "wait4 returned wrong signal status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+
+	/* make sure shared page got modified in child */
+	if ( strcmp( my_test_page_p, "parent data child data" ) != 0 ) {
+		printf( "minherit did not work correctly - shared page looks wrong \n" );
+		goto test_failed_exit;
+	}
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	 if ( my_test_page_p != NULL ) {
+		free( my_test_page_p );
+	 }
+	return( my_err );
+}
+
diff --git a/tools/tests/xnu_quick_test/misc.c b/tools/tests/xnu_quick_test/misc.c
new file mode 100644
index 000000000..b274ddfd1
--- /dev/null
+++ b/tools/tests/xnu_quick_test/misc.c
@@ -0,0 +1,366 @@
+
+#include "tests.h"
+
+/*
+ * create_random_name - creates a file with a random / unique name in the given directory.
+ * when do_open is true we create a file else we generaate a name that does not exist in the
+ * given directory (we do not create anything when do_open is 0).
+ * WARNING - caller provides enough space in path buffer for longest possible name.
+ * WARNING - assumes caller has appended a trailing '/' on the path passed to us.
+ * RAND_MAX is currently 2147483647 (ten characters plus one for a slash)
+ */
+int create_random_name( char *the_pathp, int do_open ) {
+	int		i, my_err;
+	int		my_fd = -1;
+	
+    for ( i = 0; i < 1; i++ ) {
+        int			my_rand;
+        char		*myp;
+        char		my_name[32];
+        
+        my_rand = rand( );
+        sprintf( &my_name[0], "%d", my_rand );
+        if ( (strlen( &my_name[0] ) + strlen( the_pathp ) + 2) > PATH_MAX ) {
+            printf( "%s - path to test file greater than PATH_MAX \n", __FUNCTION__ );
+            return( -1 );
+        }
+
+        // append generated file name onto our path
+        myp = strrchr( the_pathp, '/' );
+        *(myp + 1) = 0x00;
+        strcat( the_pathp, &my_name[0] );
+		if ( do_open ) {
+			/* create a file with this name */
+			my_fd = open( the_pathp, (O_RDWR | O_CREAT | O_EXCL),
+							(S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) );
+			if ( my_fd == -1 ) {
+				if ( errno != EEXIST ) {
+					printf( "%s - open failed with errno %d - %s \n",
+							__FUNCTION__, errno, strerror( errno ) );
+					return( -1 );
+				}
+				// name already exists, try another
+				i--;
+				continue;
+			}
+		}
+		else {
+			/* make sure the name is unique */
+			struct stat		my_sb;
+			my_err = stat( the_pathp, &my_sb );
+			if ( my_err != 0 ) {
+				if ( errno == ENOENT ) {
+					break;
+				}
+				else {
+					printf( "%s - open failed with errno %d - %s \n",
+							__FUNCTION__, errno, strerror( errno ) );
+					return( -1 );
+				}
+			}
+			/* name already exists, try another */
+			i--;
+			continue;
+		}
+    }
+	
+	if ( my_fd != -1 )
+		close( my_fd );
+	
+	return( 0 );
+	
+} /* create_random_name */
+
+/*
+ * create_file_with_name - create a file in the given target directory using the given name.
+ * If an existing file or directory is present use the value of remove_existing to determine if the
+ * object is to be deleted.
+ * returns 0 if file could be created, 1 if file exists, 2 if directory exists, else -1 
+ * NOTE - will fail if a directory is present with the given name and it is not empty.
+ */
+int create_file_with_name( char *the_target_dirp, char *the_namep, int remove_existing ) {
+	int				create_test_file, my_err, my_result;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	struct stat		my_sb;
+	
+	create_test_file = 0;
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto failure_exit;
+	}
+	strcpy( my_pathp, the_target_dirp );
+	strcat( my_pathp, the_namep );
+
+	/* make sure the name is unique */
+	my_result = 0;
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		create_test_file = 1;
+		if ( errno != ENOENT ) {
+			goto failure_exit;
+		}
+	}
+	else {
+		/* name already exists */
+		if ( S_ISDIR( my_sb.st_mode ) ) {
+			my_result = 2; /* tell caller directory exists with target name */
+			if ( remove_existing ) {
+				my_err = rmdir( my_pathp );
+				if ( my_err == -1 ) {
+					printf( "rmdir failed with error %d - \"%s\" \n", errno, strerror( errno) );
+					goto failure_exit;
+				}
+				create_test_file = 1;
+			}
+		}
+		else {
+			my_result = 1; /* tell caller file exists with target name */
+			if ( remove_existing ) {
+				my_err = unlink( my_pathp );
+				if ( my_err == -1 ) {
+					printf( "unlink failed with error %d - \"%s\" \n", errno, strerror( errno) );
+					goto failure_exit;
+				}
+				create_test_file = 1;
+			}
+		}
+	}
+	
+	if ( create_test_file ) {
+		/* create a file with this name */
+		my_fd = open( my_pathp, (O_RDWR | O_CREAT | O_EXCL),
+						(S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) );
+		if ( my_fd == -1 ) {
+			printf( "open failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto failure_exit;
+		}
+		close( my_fd );
+	} 
+	goto routine_exit;
+
+failure_exit:	
+	my_result = -1;
+routine_exit:
+	if ( my_pathp != NULL ) {
+		if ( my_result == -1 && create_test_file ) {
+			remove( my_pathp );	
+		}
+		free( my_pathp );
+	 }
+	
+	return( my_result );
+	
+} /* create_file_with_name */
+
+
+
+
+/*
+ * This function is needed by both xnu_quick_test proper and the execve() helper
+ * program. It forks a child process and then exec()s an image on that child.
+ * Path, argv, and envp are fed directly to the execve() call.
+ * Parameter killwait decides how long to wait before killing the child.
+ */
+int do_execve_test(char * path, char * argv[], void * envp, int killwait)
+{
+	int	my_err = 0, my_status;
+	pid_t	my_pid, my_wait_pid;
+
+#if DEBUG
+	printf("do_execve_test(path = %s)\n", path);
+	printf("CWD= %s\n", getwd(NULL));
+	fflush(stdout);
+#endif
+	/* vfork then execve sleep system command (which we will kill from the parent process) */
+	my_pid = vfork();
+	if (my_pid == -1) {
+		printf( "vfork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - use execve to start one of the customized helper
+		 * binaries, which just sleep for 120 seconds. Let our parent kill us.
+		 */
+
+		my_err = execve(path, argv, envp);
+		if ( my_err != 0 ) { /* TODO: execve() on x86_64 inca returns weird error codes, see rdar://4655612 */
+			printf( "execve call failed with return value: %d, errno: %d - \"%s\"; path: %s \n",
+				my_err, errno, strerror( errno), path );
+			fflush(stdout);
+			exit(-2);
+		}
+
+		/* should never get here */
+		printf("Execve failed and it was not caught by our test\n");
+		return(-1);
+	}
+	/* 
+	 * parent process - let's kill our sleeping child
+	 */     
+	sleep(killwait);
+	my_err = kill( my_pid, SIGKILL );
+	if ( my_err == -1 ) {
+		printf( "kill call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* wait for child to exit */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}       
+
+	if ( WIFSIGNALED( my_status ) && WTERMSIG( my_status ) != SIGKILL ) {
+		printf( "wait4 returned wrong signal status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = 1;
+
+test_passed_exit:
+	return( my_err );
+} /* do_execve_test */
+
+/*
+ * Helper function for posix_spawn test
+ * 	arch: target architecture to spawn for
+ */
+int do_spawn_test(int arch, int shouldfail)
+{
+	int my_err, my_pid, my_status;
+	size_t my_size;
+	posix_spawnattr_t attr;
+
+	char * args[] = {"helpers/arch", NULL};
+
+	my_err = posix_spawnattr_init(&attr);
+	if (my_err != 0) {
+		printf("posix_spawnattr_init failed\n");
+		goto done;
+	}
+
+	/* set spawn to only succeed for arch 'arch' */
+	my_err = posix_spawnattr_setbinpref_np(&attr, 1, &arch, &my_size);
+	if (my_err != 0 || my_size != 1) {
+		printf("posix_spawnattr_setbinpref_np failed\n");
+		goto done;
+	}
+
+	/* spawn off child process */
+	my_err = posix_spawn(&my_pid, args[0], NULL, &attr, args, NULL);
+	if (shouldfail) {
+		if( my_err == 0) {
+			printf("posix_spawn should have failed on arch %d\n", arch);
+			goto done;
+		}
+		my_err = 0;
+	} else {
+		/* child should exit with return code == arch */
+		if (my_err != 0) {
+			printf("posix_spawn failed with errno %d - %s\n", errno, strerror(errno));
+			goto done;
+		}
+
+		my_err = wait4(my_pid, &my_status, 0, NULL);
+		if (my_err == -1) {
+			printf("wait4 failed with errno %d - %s\n", errno, strerror(errno));
+			goto done;
+		}
+		my_err = 0;
+
+		if (WEXITSTATUS(my_status) != (arch & 0xff)) {
+			printf("child exited with status %d (expected %d)\n", 
+					(WEXITSTATUS(my_status)), 
+					(arch & 0xff));
+			my_err = -1;
+			goto done;
+		}
+	}
+
+done:
+	return my_err;
+}
+
+/*
+ * Uses sysctlbyname to determine the cpu type. Currently, XNU classifies G5 as a 
+ * 32-bit CPU, so this shouldn't be used to determine whether or not a CPU
+ * is 64-bit.
+ */
+int get_architecture()
+{
+	int rval = -1;
+	size_t length = 0;
+	int my_err, buf;
+	char *errmsg = NULL;
+
+	errmsg = "sysctlbyname() failed when getting hw.cputype";
+	if (my_err = sysctlbyname("hw.cputype", NULL, &length, NULL, 0)) goto finished;	/* get length of data */
+	if (length != sizeof(buf))					 goto finished;
+	if (my_err = sysctlbyname("hw.cputype", &buf, &length, NULL, 0)) goto finished; /* copy data */
+	switch (buf) {
+	case CPU_TYPE_X86:
+	case CPU_TYPE_X86_64:
+		rval = INTEL;
+		break;
+	case CPU_TYPE_POWERPC:
+	case CPU_TYPE_POWERPC64:
+		rval = POWERPC;
+		break;
+	}
+
+finished:
+	if (rval == -1 && errmsg)
+		printf("%s", errmsg);
+
+	return rval;
+}
+
+
+/*
+ * Gets the bit'ed-ness of the current host. Returns either 32 or 64.
+ * This get the hardware capability, but does not tell us whether this
+ * binary is executing in 64 bit or 32 bit mode. Check sizeof long
+ * or pointer to determine that.
+ */
+int get_bits()
+{
+	int  my_err, buf;
+	size_t len = 0;
+	int rval = 32;	/*
+			 * On 32-bit systems the sysctls 64bitops and x86_64 don't 
+			 * even exists, so if we don't find them then we assume 
+			 * a 32-bit system.
+			 */
+
+	/* Check for PPC 64 */
+	if ((my_err = sysctlbyname("hw.optional.64bitops", NULL, &len, NULL, 0)))	goto x86_64check; /* Request size */
+	if (len > sizeof(buf))								goto x86_64check;
+	if ((my_err = sysctlbyname("hw.optional.64bitops", &buf, &len, NULL, 0)))	goto x86_64check; /* Copy value out from kernel */
+	if (buf == 1) rval = 64;
+	goto finished;
+
+x86_64check:
+	/* Check for x86_64 */
+	if ((my_err = sysctlbyname("hw.optional.x86_64", NULL, &len, NULL, 0)))	goto finished; /* Request size */
+	if (len > sizeof(buf))							goto finished;
+	if ((my_err = sysctlbyname("hw.optional.x86_64", &buf, &len, NULL, 0)))	goto finished; /* Copy value out from kernel */
+	if (buf == 1) rval = 64;
+
+finished:
+	return rval;
+}
+
diff --git a/tools/tests/xnu_quick_test/sema_tests.c b/tools/tests/xnu_quick_test/sema_tests.c
new file mode 100644
index 000000000..ccb154971
--- /dev/null
+++ b/tools/tests/xnu_quick_test/sema_tests.c
@@ -0,0 +1,165 @@
+/*
+ *  sema_tests.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 6/2/2005.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+#include <sys/sem.h> 
+#include <semaphore.h>
+
+/*  **************************************************************************************************************
+ *	Test semctl, semget, semop system calls.
+ *  **************************************************************************************************************
+ */
+int sema_tests( void * the_argp ) 
+{
+	int				my_err, i;
+	int				my_sem_id = -1;
+ 	union semun		my_sem_union;
+
+	srand( (unsigned int)getpid() );
+	my_sem_id = semget( (key_t)1234, 1, (0666 | IPC_CREAT) );
+	if ( my_sem_id == -1 ) {
+		printf( "semget failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+#if 1 // todo - remove this once 4149385 is fixed
+	/* workaround for bug in the xnu implementation of semctl */
+	if ( sizeof( long ) == 8 ) {
+		my_sem_union.array = (void *)1;
+	}
+	else
+#endif
+ 	my_sem_union.val = 1;
+	my_err = semctl( my_sem_id, 0, SETVAL, my_sem_union );
+	if ( my_sem_id == -1 ) {
+		printf( "semget failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	for ( i = 0; i < 10000; i++ ) {
+		struct sembuf		my_sembuf;
+
+		my_sembuf.sem_num = 0;
+		my_sembuf.sem_op  = -1;
+		my_sembuf.sem_flg = SEM_UNDO;
+	
+		my_err = semop( my_sem_id, &my_sembuf, 1 );
+		if ( my_err == -1 ) {
+			printf( "semop failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+
+        my_err = semctl( my_sem_id, 0, GETVAL, 0 );
+		if ( my_err == -1 ) {
+			printf( "semctl failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+		if ( my_err != 0 ) {
+			printf( "semctl(getval) returned %d. it should be 0 (locked) here \n", my_err );
+			goto test_failed_exit;
+        }
+
+		my_sembuf.sem_num = 0;
+		my_sembuf.sem_op  = 1;
+		my_sembuf.sem_flg = SEM_UNDO;
+		
+		my_err = semop( my_sem_id, &my_sembuf, 1 );
+		if ( my_err == -1 ) {
+			printf( "semop failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+	}
+
+	my_err = semctl( my_sem_id, 0, IPC_RMID, my_sem_union );
+	if ( my_err == -1 ) {
+		printf( "semctl (IPC_RMID) failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_sem_id = -1;
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_sem_id != -1 ) {
+		semctl( my_sem_id, 0, IPC_RMID, my_sem_union );
+	}
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test sem_close, sem_open, sem_post, sem_trywait, sem_unlink, sem_wait system calls.
+ *  **************************************************************************************************************
+ */
+int sema2_tests( void * the_argp ) 
+{
+	int				my_err;
+	sem_t *			my_sem_t = (sem_t *)SEM_FAILED;
+	char			my_sema_name[ 64 ];
+	
+	/* get a semaphore (initialized as locked) */
+	sprintf( &my_sema_name[0], "sema_testing_%d", getpid( ) );
+	my_sem_t = sem_open( &my_sema_name[0], (O_CREAT | O_EXCL), (S_IRUSR | S_IWUSR), 0 );
+	if ( my_sem_t == (sem_t*)SEM_FAILED ) {
+		printf( "sem_open failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* get the lock - should return EAGAIN (EWOULDBLOCK) */
+	my_err = sem_trywait( my_sem_t );
+	if ( my_err == -1 ) {
+		my_err = errno;
+		if ( my_err != EAGAIN ) {
+			printf( "sem_trywait failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+			goto test_failed_exit;
+		} 
+	}
+
+	/* unlock our semaphore */
+	my_err = sem_post( my_sem_t );
+	if ( my_err == -1 ) {
+		printf( "sem_post failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* get the lock again */
+	my_err = sem_wait( my_sem_t );
+	if ( my_err == -1 ) {
+		printf( "sem_wait failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = sem_unlink( &my_sema_name[0] );
+	if ( my_err == -1 ) {
+		printf( "sem_unlink failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = sem_close( my_sem_t );
+	if ( my_err == -1 ) {
+		printf( "sem_close failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_sem_t = (sem_t *)SEM_FAILED;
+
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_sem_t != (sem_t *)SEM_FAILED ) {
+		sem_close( my_sem_t );
+	}
+	return( my_err );
+}
diff --git a/tools/tests/xnu_quick_test/shared_memory_tests.c b/tools/tests/xnu_quick_test/shared_memory_tests.c
new file mode 100644
index 000000000..362c777ed
--- /dev/null
+++ b/tools/tests/xnu_quick_test/shared_memory_tests.c
@@ -0,0 +1,137 @@
+/*
+ *  shared_memory_tests.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 6/2/2005.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+#include <sys/ipc.h>
+#include <sys/mman.h>
+#include <sys/shm.h>
+
+extern char  g_target_path[ PATH_MAX ];
+
+
+/*  **************************************************************************************************************
+ *	Test shmat, shmctl, shmdt, shmget system calls.
+ *  **************************************************************************************************************
+ */
+int shm_tests( void * the_argp )
+{
+	int					my_err;
+	int					my_shm_id;
+	void *				my_shm_addr = NULL;
+	struct shmid_ds		my_shmid_ds;
+
+	my_shm_id = shmget( IPC_PRIVATE, 4096, (IPC_CREAT | IPC_R | IPC_W) );
+	if ( my_shm_id == -1 ) {
+		printf( "shmget failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_shm_addr = shmat( my_shm_id, NULL, SHM_RND );
+	if ( my_shm_addr == (void *) -1 ) {
+		my_shm_addr = NULL;
+		printf( "shmat failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* try writing to the shared segment */
+	*((char *) my_shm_addr) = 'A';
+
+	my_err = shmctl( my_shm_id, IPC_STAT, &my_shmid_ds );
+	if ( my_err == -1 ) {
+		printf( "shmctl failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_shmid_ds.shm_segsz != 4096 ) {
+		printf( "shmctl failed get correct shared segment size \n" );
+		goto test_failed_exit;
+	}
+	if ( getpid( ) != my_shmid_ds.shm_cpid ) {
+		printf( "shmctl failed get correct creator pid \n" );
+		goto test_failed_exit;
+	}
+
+	my_err = shmdt( my_shm_addr );
+	if ( my_err == -1 ) {
+		printf( "shmdt failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_shm_addr = NULL;
+	 
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_shm_addr != NULL ) {
+		shmdt( my_shm_addr );
+	}
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test BSD shared memory system calls.
+ *  **************************************************************************************************************
+ */
+int bsd_shm_tests( void * the_argp )
+{
+	int			my_err, i;
+	int			my_fd = -1;
+	char *		my_addr = NULL;
+	char		my_name[ 64 ];
+
+	for ( i = 0; i < 100; i++ ) {
+		sprintf( &my_name[0], "bsd_shm_tests_%d", i );
+		my_fd = shm_open( &my_name[0], (O_RDWR | O_CREAT | O_EXCL), S_IRWXU );
+		if ( my_fd != -1 ) 
+			break;
+		my_err = errno;
+		if ( my_err != EEXIST ) {
+			printf( "shm_open failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+			goto test_failed_exit;
+		}
+	}
+	if ( my_fd == -1 ) {
+		printf( "shm_open failed to open a shared memory object with name \"%s\" \n", &my_name[0] );
+		goto test_failed_exit;
+	}
+	
+	/* grow shared memory object */
+	my_err = ftruncate( my_fd, 4096 );		
+	if ( my_err == -1 ) {
+		printf( "ftruncate call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = shm_unlink( &my_name[0] );
+	if ( my_err == -1 ) {
+		printf( "shm_unlink failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_addr = (char *) mmap( NULL, 4096, (PROT_READ | PROT_WRITE), (MAP_FILE | MAP_SHARED), my_fd, 0 );
+	if ( my_addr == (char *) -1 ) {
+		printf( "mmap call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+ 	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	return( my_err );
+}
+
diff --git a/tools/tests/xnu_quick_test/socket_tests.c b/tools/tests/xnu_quick_test/socket_tests.c
new file mode 100644
index 000000000..b76652b4d
--- /dev/null
+++ b/tools/tests/xnu_quick_test/socket_tests.c
@@ -0,0 +1,529 @@
+/*
+ *  socket_tests.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 4/12/05.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+#include <poll.h>
+
+extern char  g_target_path[ PATH_MAX ];
+
+/*  **************************************************************************************************************
+ *	Test accept, bind, connect, listen, socket, recvmsg, sendmsg, recvfrom, sendto, getpeername, getsockname
+ *  system calls.
+ *  WARNING - I don't do networking - this should get a good look from a networking stud.
+ *  **************************************************************************************************************
+ */
+int socket_tests( void * the_argp )
+{
+	int				my_err, my_status, my_len;
+	pid_t			my_pid, my_wait_pid;
+	int				my_socket_fd = -1;
+	int				my_accepted_socket = -1;
+	char *			my_parent_pathp = NULL;
+	char *			my_child_pathp = NULL;
+	socklen_t		my_accept_len;
+	struct sockaddr *my_sockaddr;
+	ssize_t			my_result;
+	off_t			my_current_offset;
+	char			my_parent_socket_name[sizeof(struct sockaddr) + 64];
+	char			my_child_socket_name[sizeof(struct sockaddr) + 64];
+	char			my_accept_buffer[sizeof(struct sockaddr) + 64];
+
+	/* generate 2 names for binding to the sockets (one socket in the parent and one in the child) */
+	my_parent_pathp = (char *) malloc( 128 );
+	if ( my_parent_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_child_pathp = (char *) malloc( 128 );
+	if ( my_child_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_parent_pathp = 0x00;
+	strcat( my_parent_pathp, "/tmp/" );
+
+	/* get a unique name for our testing */
+	my_err = create_random_name( my_parent_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	strcpy( my_child_pathp, my_parent_pathp );
+	strcat( my_parent_pathp, "p" ); /* append 'p' to mean "parent" */
+	strcat( my_child_pathp, "c" ); /* append 'c' to mean "child" */
+
+	memset( &my_parent_socket_name[0], 0, sizeof(my_parent_socket_name) );
+	memset( &my_child_socket_name[0], 0, sizeof(my_child_socket_name) );
+
+	/* use unique names we generated in /tmp/  */
+	my_sockaddr = (struct sockaddr *) &my_parent_socket_name[0];
+	my_len = sizeof(*my_sockaddr) - sizeof(my_sockaddr->sa_data) + strlen(my_parent_pathp);
+	my_sockaddr->sa_len = my_len;
+	my_sockaddr->sa_family = AF_UNIX;
+	strcpy( &my_sockaddr->sa_data[0], my_parent_pathp );
+
+	my_sockaddr = (struct sockaddr *) &my_child_socket_name[0];
+	my_len = sizeof(*my_sockaddr) - sizeof(my_sockaddr->sa_data) + strlen(my_child_pathp);
+	my_sockaddr->sa_len = my_len;
+	my_sockaddr->sa_family = AF_UNIX;
+	strcpy( &my_sockaddr->sa_data[0], my_child_pathp );
+
+	/* set up socket for parent side */
+	my_socket_fd = socket( AF_UNIX, SOCK_STREAM, 0 );
+	if ( my_socket_fd == -1 ) {
+		printf( "socket call in parent failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_sockaddr = (struct sockaddr *) &my_parent_socket_name[0];
+	my_err = bind( my_socket_fd, my_sockaddr, my_sockaddr->sa_len );
+	if ( my_err == -1 ) {
+		printf( "bind call in child failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* test getsockname */
+	my_sockaddr = (struct sockaddr *) &my_accept_buffer[0];
+	my_accept_len = sizeof(my_accept_buffer);
+	my_err = getsockname( my_socket_fd, my_sockaddr, &my_accept_len );
+	if ( my_err == -1 ) {
+		printf( "getsockname call in child failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_sockaddr->sa_family != SOCK_STREAM ) {
+		printf( "getsockname test failed - did not get correct socket name data \n" );
+		goto test_failed_exit;
+	}
+	
+	/* make sure we can't seek on a socket */
+	my_current_offset = lseek( my_socket_fd, 0, SEEK_CUR );
+	if ( my_current_offset != -1 ) {
+		printf( "lseek on socket should fail but did not \n" );
+		goto test_failed_exit;
+	}
+
+	/*
+	 * spin off a child process that we communicate with via sockets.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - open a socket and use it to talk to our parent.
+		 */
+		int					my_child_fd = -1;
+		struct msghdr		my_msghdr;
+		struct iovec		my_iov;
+		char				my_buffer[128];
+
+		my_child_fd = socket( AF_UNIX, SOCK_STREAM, 0 );
+		if ( my_child_fd == -1 ) {
+			printf( "socket call in child failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		my_sockaddr = (struct sockaddr *) &my_child_socket_name[0];
+		my_err = bind( my_child_fd, my_sockaddr, my_sockaddr->sa_len );
+		if ( my_err == -1 ) {
+			close( my_child_fd );
+			printf( "bind call in child failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		sleep(2);
+
+		/* connect to socket in our parent */
+		my_sockaddr = (struct sockaddr *) &my_parent_socket_name[0];
+		my_err = connect( my_child_fd, my_sockaddr, my_sockaddr->sa_len );
+		if ( my_err == -1 ) {
+			close( my_child_fd );
+			printf( "connect call in child failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+	/* get some data from the child via socket and test socket peer data */
+	{
+		socklen_t			my_buffer_len;
+		struct sockaddr *	my_sockaddr;
+		char				my_parent_buffer[256];
+
+		my_sockaddr = (struct sockaddr *) &my_parent_buffer[0];
+		my_buffer_len = sizeof(my_parent_buffer);
+		my_err = getpeername( my_child_fd, my_sockaddr, &my_buffer_len );
+		if ( my_err == -1 ) {
+			printf( "getpeername call in parent failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+
+		/* test results - should be sa_family == SOCK_STREAM and name should match my_child_pathp */
+		if ( my_sockaddr->sa_family != SOCK_STREAM ) {
+			printf( "getpeername test failed - did not get correct peer data \n" );
+			goto test_failed_exit;
+		}
+	}
+
+		my_buffer[0] = 'j';
+		my_iov.iov_base = &my_buffer[0];
+		my_iov.iov_len = 1;
+		
+		my_sockaddr = (struct sockaddr *) &my_parent_socket_name[0];
+		my_msghdr.msg_name = my_sockaddr;
+		my_msghdr.msg_namelen = my_sockaddr->sa_len;
+		my_msghdr.msg_iov = &my_iov;
+		my_msghdr.msg_iovlen = 1;
+		my_msghdr.msg_control = NULL;
+		my_msghdr.msg_controllen = 0;
+		my_msghdr.msg_flags = 0;
+
+		my_result = sendmsg( my_child_fd, &my_msghdr, 0 );
+		if ( my_result == -1 ) {
+			printf( "sendmsg failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			close( my_child_fd );
+			exit( -1 );
+		}
+		
+#if 1
+		/* get data from our parent */
+		my_result = recvfrom( my_child_fd, &my_buffer[0], 1, 
+							  MSG_WAITALL, NULL, NULL );
+		if ( my_result == -1 ) {
+			printf( "recvfrom failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			close( my_child_fd );
+			exit( -1 );
+		}
+		
+		/* verify that we got the correct message from our child */
+		if ( my_buffer[0] != 'e' ) {
+			printf( "test failed - did not get correct data from child \n" );
+			close( my_child_fd );
+			exit( -1 );
+		}
+#endif
+		
+		/* tell parent we're done */
+		my_result = write( my_child_fd, "all done", 8 );
+		if ( my_result == -1 ) {
+			close( my_child_fd );
+			exit( -1 );
+		}
+
+		close( my_child_fd );
+		exit(0);
+	}
+	
+	/* 
+	 * parent process - listen for connection requests
+	 */
+	my_err = listen( my_socket_fd, 10 );
+	if ( my_err == -1 ) {
+		printf( "listen call in parent failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* accept connection from child */
+	my_sockaddr = (struct sockaddr *) &my_accept_buffer[0];
+	my_accepted_socket = accept( my_socket_fd, my_sockaddr, &my_accept_len );
+	if ( my_accepted_socket == -1 ) {
+		printf( "accept call in parent failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}	 
+			
+	/* get some data from the child via socket and test socket peer data */
+	{
+		//socklen_t			my_buffer_len;
+		struct msghdr		my_msghdr;
+		struct iovec		my_iov;
+		char				my_parent_buffer[128];
+
+		my_parent_buffer[0] = 'x';
+		my_iov.iov_base = &my_parent_buffer[0];
+		my_iov.iov_len = 1;
+		
+		my_msghdr.msg_name = &my_accept_buffer[0];
+		my_msghdr.msg_namelen = my_accept_len;
+		my_msghdr.msg_iov = &my_iov;
+		my_msghdr.msg_iovlen = 1;
+		my_msghdr.msg_control = NULL;
+		my_msghdr.msg_controllen = 0;
+		my_msghdr.msg_flags = 0;
+		
+		my_result = recvmsg( my_accepted_socket, &my_msghdr, MSG_WAITALL );
+		if ( my_result == -1 ) {
+			printf( "recvmsg failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+		
+		/* verify that we got the correct message from our child */
+		if ( my_parent_buffer[0] != 'j' ) {
+			printf( "test failed - did not get correct data from child \n" );
+			goto test_failed_exit;
+		}
+
+#if 1
+		/* now send some data to our child */
+		my_parent_buffer[0] = 'e';
+		my_sockaddr = (struct sockaddr *) &my_child_socket_name[0];
+		my_result = sendto( my_accepted_socket, &my_parent_buffer[0], 1, 0, my_sockaddr, 
+							my_sockaddr->sa_len );
+		if ( my_result == -1 ) {
+			printf( "sendto failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+#endif
+
+		/* see if child is done */
+		bzero( (void *)&my_parent_buffer[0], sizeof(my_parent_buffer) );
+		my_result = read( my_accepted_socket, &my_parent_buffer[0], sizeof(my_parent_buffer) );
+		if ( my_result == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+		if ( strcmp( "all done", &my_parent_buffer[0] ) != 0 ) {
+			printf( "read wrong message from child \n" );
+			goto test_failed_exit;
+		}
+	}
+
+	/* wait for child to exit */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+			
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_socket_fd != -1 )
+		close( my_socket_fd );
+	if ( my_accepted_socket != -1 )
+		close( my_accepted_socket );
+	if ( my_parent_pathp != NULL ) {
+		remove( my_parent_pathp );	
+		free( my_parent_pathp );
+	 }
+	if ( my_child_pathp != NULL ) {
+		remove( my_child_pathp );	
+		free( my_child_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test fsync, getsockopt, poll, select, setsockopt, socketpair system calls.
+ *  **************************************************************************************************************
+ */
+int socket2_tests( void * the_argp )
+{
+	int					my_err, my_status;
+	int					my_sockets[ 2 ] = {-1, -1};
+	pid_t				my_pid, my_wait_pid;
+	ssize_t				my_count;
+	socklen_t			my_socklen;
+	struct timeval *	my_tvp;
+	struct timeval		my_orig_tv;
+	char				my_buffer[ 32 ];
+
+	my_err = socketpair( AF_UNIX, SOCK_STREAM, 0, &my_sockets[0] );
+	if ( my_err == -1 ) {
+		printf( "socketpair failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* test getsockopt and setsockopt */
+	my_socklen = sizeof( my_buffer );
+	my_err = getsockopt( my_sockets[0], SOL_SOCKET, SO_TYPE, &my_buffer[0], &my_socklen);
+	if ( my_err == -1 ) {
+		printf( "getsockopt - SO_TYPE - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( SOCK_STREAM != *((int *)&my_buffer[0]) ) {
+		printf( "getsockopt returned incorrect socket type \n" );
+		goto test_failed_exit;
+	}
+
+	/* get and set receive timeout */
+	my_socklen = sizeof( my_buffer );
+	my_err = getsockopt( my_sockets[0], SOL_SOCKET, SO_RCVTIMEO, &my_buffer[0], &my_socklen);
+	if ( my_err == -1 ) {
+		printf( "getsockopt - SO_RCVTIMEO - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_tvp = (struct timeval *) &my_buffer[0];
+	my_orig_tv.tv_sec = my_tvp->tv_sec;
+	my_orig_tv.tv_usec = my_tvp->tv_usec;
+ 
+	my_tvp->tv_sec += 60;
+ 	my_err = setsockopt( my_sockets[0], SOL_SOCKET, SO_RCVTIMEO, &my_buffer[0], sizeof(struct timeval) );
+	if ( my_err == -1 ) {
+		printf( "setsockopt - SO_RCVTIMEO - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* verify we set it */
+	my_socklen = sizeof( my_buffer );
+	my_err = getsockopt( my_sockets[0], SOL_SOCKET, SO_RCVTIMEO, &my_buffer[0], &my_socklen);
+	if ( my_err == -1 ) {
+		printf( "getsockopt - SO_RCVTIMEO - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_tvp = (struct timeval *) &my_buffer[0];
+	if ( my_tvp->tv_sec != (my_orig_tv.tv_sec + 60) || my_tvp->tv_usec != my_orig_tv.tv_usec ) {
+		printf( "setsockopt - SO_RCVTIMEO - did not set correct timeval \n" );
+		goto test_failed_exit;
+	}
+	
+	/* set back to original receive timeout */
+ 	my_err = setsockopt( my_sockets[0], SOL_SOCKET, SO_RCVTIMEO, &my_orig_tv, sizeof(struct timeval) );
+	if ( my_err == -1 ) {
+		printf( "setsockopt - SO_RCVTIMEO - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* test fsync - should fail when used with a socket fd */
+	errno = 0;
+	my_err = fsync( my_sockets[0] );
+	if ( my_err == -1 && errno != ENOTSUP ) {
+		printf( "fsync failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	else if ( my_err != -1 ) {
+		printf( "fsync should have failed with errno ENOTSUP \n" );
+		goto test_failed_exit;
+	}
+	 
+	/*
+	 * spin off a child process that we will talk to via our socketpair.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - tell parent we are ready to go.
+		 */
+		char			my_buffer[ 32 ];
+		struct pollfd	my_pollfd;
+
+		my_count = write( my_sockets[1], "r", 1 );
+		if ( my_count == -1 ) {
+			printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		
+		/* test select by using it to wait for message from parent */
+		for ( ;; ) {
+			fd_set			my_read_set;
+			struct timeval	my_timeout;
+			
+			FD_ZERO( &my_read_set );
+			FD_SET( my_sockets[1], &my_read_set );
+			timerclear( &my_timeout );
+			my_timeout.tv_sec = 1;
+			
+			/* check to see if we are done, if no message is ready after a second
+			 * return and try again... 
+			 */
+			my_err = select( (my_sockets[1] + 1), &my_read_set, NULL, NULL, &my_timeout );
+			if ( my_err == -1 ) {
+				printf( "select call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+				exit( -1 );
+			}
+			else if ( my_err > 0 ) {
+				/* we're done */
+				break;
+			}
+		}
+		
+		/* test poll too */
+		my_pollfd.fd = my_sockets[1];
+		my_pollfd.events = (POLLIN | POLLPRI);
+		my_pollfd.revents = 0;
+		my_err = poll( &my_pollfd, 1, 500 );
+		if ( my_err == -1 ) {
+			printf( "poll call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		/* should be ready for read */
+		if ( (my_pollfd.revents & POLLIN) == 0 ) {
+			printf( "poll should have returned ready for read \n" );
+			exit( -1 );
+		}
+		
+		my_count = read( my_sockets[1], &my_buffer[0], sizeof(my_buffer) );
+		if ( my_count == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( my_buffer[0] != 'd' ) {
+			printf( "read call on socket failed to get \"all done\" message \n" );
+			exit( -1 );
+		}
+	 
+		exit(0);
+	}
+	
+	/* 
+	 * parent process - wait for child to spin up
+	 */
+	my_count = read( my_sockets[0], &my_buffer[0], sizeof(my_buffer) );
+	if ( my_count == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_buffer[0] != 'r' ) {
+		printf( "read call on socket failed to get \"ready to go message\" \n" );
+		goto test_failed_exit;
+	}
+
+	/* tell child we're done */
+	write( my_sockets[0], "d", 1 );
+
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		printf( "wait4 returned wrong exit status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_sockets[0] != -1 )
+		close( my_sockets[0] );
+	if ( my_sockets[1] != -1 )
+		close( my_sockets[1] );
+	return( my_err );
+}
+
diff --git a/tools/tests/xnu_quick_test/tests.c b/tools/tests/xnu_quick_test/tests.c
new file mode 100644
index 000000000..ad0db0e63
--- /dev/null
+++ b/tools/tests/xnu_quick_test/tests.c
@@ -0,0 +1,5128 @@
+/*
+ *  tests.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 3/25/05.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+#include <sys/event.h>		/* for kqueue tests */
+#include <sys/ipc.h>		/* for message queue tests */
+#include <sys/msg.h>		/* for message queue tests */
+#include <sys/syscall.h>	/* for get / settid */
+#include <sys/sysctl.h>		/* for determining hw */
+#include <AvailabilityMacros.h>	/* for determination of Mac OS X version (tiger, leopard, etc.) */
+#include <libkern/OSByteOrder.h> /* for OSSwap32() */
+
+extern char		g_target_path[ PATH_MAX ];
+extern int		g_skip_setuid_tests;
+extern int		g_is_under_rosetta;
+
+ 
+
+#if TEST_SYSTEM_CALLS /* system calls to do */
+	"reboot",             /* 55 = reboot */
+	"revoke",             /* 56 = revoke */
+	"sbrk",               /* 69 = sbrk */
+	"sstk",               /* 70 = sstk */
+	"mount",              /* 167 = mount */
+	"unmount",            /* 159 = unmount */
+	"undelete",           /* 205 = undelete */
+	"watchevent",         /* 231 = watchevent */
+	"waitevent",          /* 232 = waitevent */
+	"modwatch",           /* 233 = modwatch */
+	"fsctl",              /* 242 = fsctl */
+	"initgroups",         /* 243 = initgroups */
+	"semsys",             /* 251 = semsys */
+	"semconfig",          /* 257 = semconfig */
+	"msgsys",             /* 252 = msgsys */
+	"shmsys",             /* 253 = shmsys */
+	"load_shared_file",   /* 296 = load_shared_file */
+	"reset_shared_file",  /* 297 = reset_shared_file */
+	"new_system_shared_regions",  /* 298 = new_system_shared_regions */
+	"shared_region_map_file_np",  /* 299 = shared_region_map_file_np */
+	"shared_region_make_private_np",  /* 300 = shared_region_make_private_np */
+	"__pthread_kill",     /* 328 = __pthread_kill */
+	"pthread_sigmask",    /* 329 = pthread_sigmask */
+	"__disable_threadsignal",  /* 331 = __disable_threadsignal */
+	"__pthread_markcancel",  /* 332 = __pthread_markcancel */
+	"__pthread_canceled",  /* 333 = __pthread_canceled */
+	"__semwait_signal",   /* 334 = __semwait_signal */
+	"audit",              /* 350 = audit */
+	"auditon",            /* 351 = auditon */
+	"getaudit",           /* 355 = getaudit */
+	"setaudit",           /* 356 = setaudit */
+	"getaudit_addr",      /* 357 = getaudit_addr */
+	"setaudit_addr",      /* 358 = setaudit_addr */
+	"auditctl",           /* 359 = auditctl */
+#endif
+
+/*  **************************************************************************************************************
+ *	Test the syscall system call.
+ *  **************************************************************************************************************
+ */
+int syscall_test( void * the_argp )
+{
+	int			my_err;
+	int			my_fd = -1;
+	char *		my_pathp;
+	
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcpy( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	/* use an indirect system call to open our test file.
+	 * I picked open since it uses a path pointer which grows to 64 bits in an LP64 environment.
+	 */
+	my_fd = syscall( SYS_open, my_pathp, (O_RDWR | O_EXCL), 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test fork wait4, and exit system calls.
+ *  **************************************************************************************************************
+ */
+int fork_wait4_exit_test( void * the_argp )
+{
+	int				my_err, my_status;
+    pid_t			my_pid, my_wait_pid;
+	struct rusage	my_usage;
+
+	/* spin off another process */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		return( -1 );
+	}
+	else if ( my_pid == 0 ) {
+		struct stat		my_sb;
+		
+		/* child process does very little then exits */
+		my_err = stat( &g_target_path[0], &my_sb );
+		if ( my_err != 0 ) {
+			printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			printf( "\t path we stated \"%s\" \n", &g_target_path[0] );
+			exit( -1 );
+		}
+		exit( 44 );
+	}
+	
+	/* parent process waits for child to exit */
+	my_wait_pid = wait4( my_pid, &my_status, 0, &my_usage );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		return( -1 );
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		return( -1 );
+	}
+
+	/* kind of just guessing on these values so if this fails we should take a closer 
+	 * look at the returned rusage structure. 
+	 */
+	if ( my_usage.ru_utime.tv_sec > 1 || my_usage.ru_stime.tv_sec > 1 ||
+		 my_usage.ru_majflt > 1000 || my_usage.ru_msgsnd > 100 ) {
+		printf( "wait4 returned an odd looking rusage structure \n" );
+		return( -1 );
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) == 44 ) {
+	}
+	else {
+		printf( "wait4 returned wrong exit status - 0x%02X \n", my_status );
+		return( -1 );
+	}
+	 
+	return( 0 );
+}
+
+/*  **************************************************************************************************************
+ *	Test fsync, ftruncate, lseek, pread, pwrite, read, readv, truncate, write, writev system calls.
+ *  **************************************************************************************************************
+ */
+int read_write_test( void * the_argp )
+{
+	int				my_fd = -1;
+	int				my_err;
+	char *			my_pathp = NULL;
+	char *			my_bufp = NULL;
+	ssize_t			my_result;
+	off_t			my_current_offset;
+	struct iovec	my_iovs[2];
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_bufp = (char *) malloc( MY_BUFFER_SIZE );
+	if ( my_bufp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+
+	/* should get EOF since the file is empty at this point */
+	my_result = read( my_fd, my_bufp, 10);
+	if ( my_result == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 0 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "read call failed - should have read 0 bytes on empty file - read %lld \n", my_result );
+		}
+		else {
+			printf( "read call failed - should have read 0 bytes on empty file - read %d \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+
+	/* this write should fail since we opened for read only */
+	my_result = write( my_fd, my_bufp, 10 );
+	my_err = errno;
+	if ( my_result != -1 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "write should have failed for read only fd -  %lld \n", my_result );
+		}
+		else {
+			printf( "write should have failed for read only fd -  %d \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+	if ( my_err != EBADF ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "should have failed with EBADF error %d \n", EBADF );
+		goto test_failed_exit;
+	}
+	
+	/* now really write some data */
+	close( my_fd );
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	memset( my_bufp, 'j', MY_BUFFER_SIZE );
+	my_result = write( my_fd, my_bufp, MY_BUFFER_SIZE );
+	if ( my_result == -1 ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != MY_BUFFER_SIZE ) {
+		printf( "write failed to write out all the data \n" );
+		goto test_failed_exit;
+	}
+	
+	/* push data to disk */
+	my_err = fsync( my_fd );
+	if ( my_err == -1 ) {
+		printf( "fsync failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* now verify the write worked OK using readv */
+	lseek( my_fd, 0, SEEK_SET );	
+	bzero( (void *)my_bufp, MY_BUFFER_SIZE );
+	my_iovs[0].iov_base = my_bufp;
+	my_iovs[0].iov_len = 16;
+	my_iovs[1].iov_base = (my_bufp + MY_BUFFER_SIZE - 16) ;
+	my_iovs[1].iov_len = 16;
+
+	my_result = readv( my_fd, &my_iovs[0], 2 );
+	if ( my_result == -1 ) {
+		printf( "readv call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 32 ) {
+		printf( "readv failed to get all the data - asked for %d got back %d\n", MY_BUFFER_SIZE, my_result );
+		goto test_failed_exit;
+	}
+	if ( *my_bufp != 'j' || *(my_bufp + (MY_BUFFER_SIZE - 1)) != 'j' ) {
+		printf( "readv failed to get correct data \n" );
+		goto test_failed_exit;
+	}
+
+	/* test ftruncate */
+	my_err = ftruncate( my_fd, 0 );		
+	if ( my_err == -1 ) {
+		printf( "ftruncate call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = fstat( my_fd, &my_sb );	
+	if ( my_err == -1 ) {
+		printf( "fstat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_size != 0 ) {
+		printf( "ftruncate call failed - file size is wrong \n" );
+		goto test_failed_exit;
+	}
+	
+	/* test writev */
+	lseek( my_fd, 0, SEEK_SET );	
+	memset( my_bufp, 'z', MY_BUFFER_SIZE );
+	my_iovs[0].iov_base = my_bufp;
+	my_iovs[0].iov_len = 8;
+	my_iovs[1].iov_base = (my_bufp + MY_BUFFER_SIZE - 8) ;
+	my_iovs[1].iov_len = 8;
+	my_result = writev( my_fd, &my_iovs[0], 2 );
+	if ( my_result == -1 ) {
+		printf( "writev call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 16 ) {
+		printf( "writev failed to get all the data - asked for %d got back %d\n", MY_BUFFER_SIZE, my_result );
+		goto test_failed_exit;
+	}
+
+	/* now verify the writev worked OK */
+	lseek( my_fd, 0, SEEK_SET );	
+	bzero( (void *)my_bufp, MY_BUFFER_SIZE );
+	my_iovs[0].iov_base = my_bufp;
+	my_iovs[0].iov_len = 8;
+	my_iovs[1].iov_base = (my_bufp + MY_BUFFER_SIZE - 8) ;
+	my_iovs[1].iov_len = 8;
+
+	my_result = readv( my_fd, &my_iovs[0], 2 );
+	if ( my_result == -1 ) {
+		printf( "readv call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 16 ) {
+		printf( "readv failed to get all the data - asked for %d got back %d\n", MY_BUFFER_SIZE, my_result );
+		goto test_failed_exit;
+	}
+	if ( *my_bufp != 'z' || *(my_bufp + (MY_BUFFER_SIZE - 1)) != 'z' ) {
+		printf( "readv failed to get correct data \n" );
+		goto test_failed_exit;
+	}
+
+	/* test pread and pwrite */
+	my_current_offset = lseek( my_fd, 0, SEEK_CUR );
+	if ( my_current_offset == -1 ) {
+		printf( "lseek call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	 
+	my_result =  pwrite( my_fd, "jer", 3, my_current_offset );
+	if ( my_result == -1 ) {
+		printf( "pwrite call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 3 ) {
+		printf( "pwrite failed to write all the data \n" );
+		goto test_failed_exit;
+	}
+	
+	/* make sure file position did not advance */
+	if ( my_current_offset != lseek( my_fd, 0, SEEK_CUR ) ) {
+		printf( "pwrite advanced file positiion \n" );
+		goto test_failed_exit;
+	}
+	 
+	bzero( (void *)my_bufp, MY_BUFFER_SIZE );
+	my_result =  pread( my_fd, my_bufp, 3, my_current_offset );
+	if ( my_result == -1 ) {
+		printf( "pread call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 3 ) {
+		printf( "pread failed to write all the data \n" );
+		goto test_failed_exit;
+	}
+
+	/* make sure file position did not advance */
+	if ( my_current_offset != lseek( my_fd, 0, SEEK_CUR ) ) {
+		printf( "pread advanced file positiion \n" );
+		goto test_failed_exit;
+	}
+	
+	/* make sure pread and pwrite transferred correct data */
+	if ( strcmp( my_bufp, "jer" ) != 0 ) {
+		printf( "pread or pwrite failed to read / write correct data \n" );
+		goto test_failed_exit;
+	}
+
+	/* test truncate */
+	my_err = truncate( my_pathp, 0 );		
+	if ( my_err == -1 ) {
+		printf( "truncate call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = stat( my_pathp, &my_sb );	
+	if ( my_err == -1 ) {
+		printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_size != 0 ) {
+		printf( "truncate call failed - file size is wrong \n" );
+		goto test_failed_exit;
+	}
+				
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	if ( my_bufp != NULL )
+		free( my_bufp );
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test close, fpathconf, fstat, open, pathconf system calls.
+ *  **************************************************************************************************************
+ */
+int open_close_test( void * the_argp )
+{
+	int			my_err;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+	ssize_t		my_result;
+	long		my_pconf_result;
+	struct stat	my_sb;
+	char		my_buffer[32];
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/*  test O_WRONLY case */
+	my_fd = open( my_pathp, O_WRONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+
+	/* test pathconf and fpathconf */
+	my_pconf_result = pathconf( my_pathp, _PC_PATH_MAX );
+	if ( my_pconf_result == -1 ) {
+		printf( "pathconf - _PC_PATH_MAX - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}	
+//	printf( "_PC_PATH_MAX %ld \n", my_pconf_result );
+	/* results look OK? */
+	if ( my_pconf_result < PATH_MAX ) {
+		printf( "pathconf - _PC_PATH_MAX - looks like wrong resutls \n" );
+		goto test_failed_exit;
+	} 
+
+	my_pconf_result = fpathconf( my_fd, _PC_NAME_MAX );
+	if ( my_pconf_result == -1 ) {
+		printf( "fpathconf - _PC_PATH_MAX - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}	
+//	printf( "_PC_NAME_MAX %ld \n", my_pconf_result );
+	/* results look OK? */
+	if ( my_pconf_result < 6 ) {
+		printf( "fpathconf - _PC_NAME_MAX - looks like wrong resutls \n" );
+		goto test_failed_exit;
+	} 
+
+	/* write some data then try to read it */
+	my_result = write( my_fd, "kat", 3 );
+	my_err = errno;
+	if ( my_result != 3 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %lld \n", my_result );
+		}
+		else {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %ld \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+	
+	/* Try to read - this should fail since we opened file with O_WRONLY */
+	my_result = read( my_fd, &my_buffer[0], sizeof(my_buffer) );
+	my_err = errno;
+	if ( my_result != -1 ) {
+		printf( "read call should have failed with errno 9 (EBADF) \n" );
+		goto test_failed_exit;
+	}
+	else if ( my_err != EBADF ) {
+		printf( "read call should have failed with errno 9 (EBADF).  actually failed with %d - \"%s\" \n", my_err, strerror( my_err) );
+		goto test_failed_exit;
+	}
+
+	close( my_fd );
+
+	/*  test O_TRUNC and O_APPEND case */
+	my_fd = open( my_pathp, (O_RDWR | O_TRUNC | O_APPEND), 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+
+	my_result = read( my_fd, &my_buffer[0], sizeof(my_buffer) );
+	if ( my_result == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 0 ) {
+		printf( "read failed - should have read 0 bytes. \n" );
+		goto test_failed_exit;
+	}
+
+	my_result = write( my_fd, "kat", 3 );
+	my_err = errno;
+	if ( my_result != 3 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %lld \n", my_result );
+		}
+		else {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %ld \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+
+	/* add some more data to the test file - this should be appended */
+	lseek( my_fd, 0, SEEK_SET );
+	my_result = write( my_fd, "zzz", 3 );
+	my_err = errno;
+	if ( my_result != 3 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %lld \n", my_result );
+		}
+		else {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %ld \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+			
+	/* now verify the writes */
+	bzero( (void *)&my_buffer[0], sizeof(my_buffer) );
+	lseek( my_fd, 0, SEEK_SET );
+	my_result = read( my_fd, &my_buffer[0], sizeof(my_buffer) );
+	if ( my_result == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_buffer[0] != 'k' || my_buffer[5] != 'z' ) {
+		printf( "read failed to get correct data \n" );
+		goto test_failed_exit;
+	}
+
+	/* test fstat */
+	my_err = fstat( my_fd, &my_sb );	
+	if ( my_err == -1 ) {
+		printf( "fstat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_size != 6 ) {
+		printf( "fstat call failed - st_size is wrong \n" );
+		goto test_failed_exit;
+	}
+	if ( !S_ISREG( my_sb.st_mode ) ) {
+		printf( "fstat call failed - st_mode does not indicate regular file \n" );
+		goto test_failed_exit;
+	}
+	 
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test link, stat and unlink system calls.
+ *  **************************************************************************************************************
+ */
+int link_stat_unlink_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	char *			my_path2p = NULL;
+	nlink_t			my_link_count;
+	ssize_t			my_result;
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_path2p = (char *) malloc( PATH_MAX );
+	if ( my_path2p == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	*my_path2p = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* now create a name for the link file */
+	strcat( my_path2p, my_pathp );
+	strcat( my_path2p, "link" );
+	
+	/* get the current link count */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_link_count = my_sb.st_nlink;
+	
+	/* check file size (should be 0) */
+	if ( my_sb.st_size != 0 ) {
+		printf( "stat structure looks bogus for test file \"%s\" \n", my_pathp );
+		printf( "st_size is not 0 \n" );
+		goto test_failed_exit;
+	}
+
+	/* change file size */
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	my_result = write( my_fd, "kat", 3 );
+	my_err = errno;
+	if ( my_result != 3 ) {
+		if ( sizeof( ssize_t ) > sizeof( int ) ) {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %lld \n", my_result );
+		}
+		else {
+			printf( "write failed.  should have written 3 bytes actually wrote -  %ld \n", my_result );
+		}
+		goto test_failed_exit;
+	}
+	close( my_fd );
+	my_fd = -1;
+	
+	/* now link another file to our test file and recheck link count */
+	my_err = link( my_pathp, my_path2p );
+	if ( my_err != 0 ) {
+		printf( "link call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( (my_link_count + 1) != my_sb.st_nlink ) {
+		printf( "stat structure looks bogus for test file \"%s\" \n", my_pathp );
+		printf( "incorrect st_nlink \n" );
+		goto test_failed_exit;
+	}
+	
+	/* check file size (should be 3) */
+	if ( my_sb.st_size != 3 ) {
+		printf( "stat structure looks bogus for test file \"%s\" \n", my_pathp );
+		printf( "st_size is not 3 \n" );
+		goto test_failed_exit;
+	}
+	
+	/* now make sure unlink works OK */
+	my_err = unlink( my_path2p );
+	if ( my_err != 0 ) {
+		printf( "unlink call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_link_count != my_sb.st_nlink ) {
+		printf( "stat structure looks bogus for test file \"%s\" \n", my_pathp );
+		printf( "incorrect st_nlink \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	}
+	if ( my_path2p != NULL ) {
+		remove( my_path2p );	
+		free( my_path2p );
+	}
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test chdir and fchdir system calls.
+ *  **************************************************************************************************************
+ */
+int chdir_fchdir_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	char *			my_file_namep;
+	struct stat		my_sb;
+	struct stat		my_sb2;
+
+	char *cwd = getwd(NULL);	/* Save current working directory so we can restore later */
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* test by doing a stat on the test file using a full path and a partial path.
+	 * get full path first.
+	 */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* now do the chdir to our test directory and then do the stat relative to that location */
+	my_err = chdir( &g_target_path[0] );
+	if ( my_err != 0 ) {
+		printf( "chdir call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_file_namep = strrchr( my_pathp, '/' );
+	my_file_namep++;
+	my_err = stat( my_file_namep, &my_sb2 );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* both stat buffers should contain the same data since they should be referencing the same
+	 * file.
+	 */
+	if ( my_sb.st_ino != my_sb2.st_ino || my_sb.st_size != my_sb2.st_size ||
+		 my_sb.st_mtimespec.tv_sec != my_sb2.st_mtimespec.tv_sec ||
+		 my_sb.st_mtimespec.tv_nsec != my_sb2.st_mtimespec.tv_nsec  ) {
+		printf( "chdir call appears to have failed.  stat buffer contents do not match! \n" );
+		goto test_failed_exit;
+	}
+	
+	/* now change our current directory to "/" and use fchdir to get back to our test directory */
+	my_err = chdir( "/" );
+	if ( my_err != 0 ) {
+		printf( "chdir call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* we should not find our test file at the root of the volume */
+	my_err = stat( my_file_namep, &my_sb2 );
+	if ( my_err == 0 ) {
+		printf( "chdir to root volume has failed \n" );
+		goto test_failed_exit;
+	}
+
+	/* get a file descriptor to the test directory for use with fchdir */
+	my_fd = open( &g_target_path[0], O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t we attempted to open -> \"%s\" \n", &g_target_path[0] );
+		goto test_failed_exit;
+	}
+	
+	my_err = fchdir( my_fd );
+	if ( my_err == -1 ) {
+		printf( "fchdir call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	my_err = stat( my_file_namep, &my_sb2 );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* both stat buffers should contain the same data since they should be referencing the same
+	 * file.
+	 */
+	if ( my_sb.st_ino != my_sb2.st_ino || my_sb.st_size != my_sb2.st_size ||
+		 my_sb.st_mtimespec.tv_sec != my_sb2.st_mtimespec.tv_sec ||
+		 my_sb.st_mtimespec.tv_nsec != my_sb2.st_mtimespec.tv_nsec  ) {
+		printf( "chdir call appears to have failed.  stat buffer contents do not match! \n" );
+		goto test_failed_exit;
+	}
+
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	if ( chdir(cwd) != 0)	/* Changes back to original directory, don't screw up the env. */
+		my_err = -1;
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test access, chmod and fchmod system calls.
+ *  **************************************************************************************************************
+ */
+int access_chmod_fchmod_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* test chmod */
+	my_err = chmod( my_pathp, S_IRWXU );
+	if ( my_err == -1 ) {
+		printf( "chmod call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_err = chmod( my_pathp, (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP) );
+	if ( my_err == -1 ) {
+		printf( "chmod call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* test access - this should fail */
+	my_err = access( my_pathp, (X_OK) );
+	if ( my_err == 0 ) {
+		printf( "access call should have failed, but did not. \n" );
+		goto test_failed_exit;
+	}
+	else if ( my_err == -1  ) {
+		/* special case when running as root - we get back EPERM when running as root */
+		my_err = errno;
+		if ( (getuid( ) == 0 && my_err != EPERM) || (getuid( ) != 0 && my_err != EACCES) ) {
+			printf( "access failed with errno %d - %s. \n", my_err, strerror( my_err ) );
+			goto test_failed_exit;
+		}
+	}
+
+	/* verify correct modes are set */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	if ( (my_sb.st_mode & (S_IRWXO | S_IXGRP)) != 0 ||
+		 (my_sb.st_mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)) == 0 ) {
+		printf( "chmod call appears to have failed.  stat shows incorrect values in st_mode! \n" );
+		goto test_failed_exit;
+	}
+
+	/* test fchmod */
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t we attempted to open -> \"%s\" \n", &g_target_path[0] );
+		goto test_failed_exit;
+	}
+
+	my_err = fchmod( my_fd, S_IRWXU );
+	if ( my_err == -1 ) {
+		printf( "fchmod call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* verify correct modes are set */
+	if ( (my_sb.st_mode & (S_IRWXG | S_IRWXO)) != 0 ||
+		 (my_sb.st_mode & (S_IRWXU)) == 0 ) {
+		printf( "fchmod call appears to have failed.  stat shows incorrect values in st_mode! \n" );
+		goto test_failed_exit;
+	}
+		
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test chown, fchown, lchown, lstat, readlink, symlink system calls.
+ *  **************************************************************************************************************
+ */
+int chown_fchown_lchown_lstat_symlink_test( void * the_argp )
+{
+	int				my_err, my_group_count, i;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	char *			my_link_pathp = NULL;
+	uid_t			my_orig_uid;
+	gid_t			my_orig_gid, my_new_gid1 = 0, my_new_gid2 = 0;
+	ssize_t			my_result;
+	struct stat		my_sb;
+	gid_t			my_groups[ NGROUPS_MAX ];
+	char			my_buffer[ 64 ];
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_link_pathp = (char *) malloc( PATH_MAX );
+	if ( my_link_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_link_pathp = 0x00;
+	strcat( my_link_pathp, &g_target_path[0] );
+	strcat( my_link_pathp, "/" );
+
+	/* get a test file name for the link */
+	my_err = create_random_name( my_link_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* set up by getting a list of groups */
+	my_group_count = getgroups( NGROUPS_MAX, &my_groups[0] );
+	if ( my_group_count == -1 || my_group_count < 1 ) {
+		printf( "getgroups call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* now change group owner to something other than current value */
+	my_orig_gid = my_sb.st_gid;
+	my_orig_uid = my_sb.st_uid;
+	for ( i = 0; i < my_group_count; i++ ) {
+		if ( my_orig_gid != my_groups[ i ] ) {
+			if ( my_new_gid1 == 0 ) {
+				my_new_gid1 = my_groups[ i ];
+			}
+			else {
+				my_new_gid2 = my_groups[ i ];
+				break;
+			}
+		}
+	}
+	if ( i >= my_group_count ) {
+		printf( "not enough groups to choose from.  st_gid is the same as current groups! \n" );
+		goto test_failed_exit;
+	}
+		
+	my_err = chown( my_pathp, my_orig_uid, my_new_gid1 );
+	if ( my_err != 0 ) {
+		printf( "chown call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* make sure the group owner was changed */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_gid == my_orig_gid ) {
+		printf( "chown call failed.  st_gid is not correct! \n" );
+		goto test_failed_exit;
+	}
+	
+	/* change group owner back using fchown */
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t we attempted to open -> \"%s\" \n", &g_target_path[0] );
+		goto test_failed_exit;
+	}
+
+	my_err = fchown( my_fd, my_orig_uid, my_new_gid2 );
+	if ( my_err != 0 ) {
+		printf( "fchown call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* make sure the group owner was changed back to the original value */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_gid == my_new_gid1 ) {
+		printf( "fchown call failed.  st_gid is not correct! \n" );
+		goto test_failed_exit;
+	}
+
+	/* create a link file and test lchown */
+	my_err = symlink( my_pathp, my_link_pathp );
+	if ( my_err != 0 ) {
+		printf( "symlink call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	 
+	my_err = lstat( my_link_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "lstat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* now change group owner to something other than current value */
+	my_orig_gid = my_sb.st_gid;
+	my_orig_uid = my_sb.st_uid;
+	my_err = lchown( my_link_pathp, my_orig_uid, my_new_gid1 );
+	if ( my_err != 0 ) {
+		printf( "lchown call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* make sure the group owner was changed to new value */
+	my_err = lstat( my_link_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "lstat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_sb.st_gid == my_new_gid2 ) {
+		printf( "lchown call failed.  st_gid is not correct! \n" );
+		goto test_failed_exit;
+	}
+
+	/* make sure we can read the symlink file */
+	my_result = readlink( my_link_pathp, &my_buffer[0], sizeof(my_buffer) );
+	if ( my_result == -1 ) {
+		printf( "readlink call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	/* make sure we read some data */
+	if ( my_result < 1 ) {
+		printf( "readlink failed to read any data. \n" );
+		goto test_failed_exit;
+	}
+
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	if ( my_link_pathp != NULL ) {
+		unlink( my_link_pathp );	
+		free( my_link_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test fstatfs, getattrlist, getfsstat, statfs, getfsstat64, statfs64, fstatfs64 system calls.
+ *  **************************************************************************************************************
+ */
+ 
+#pragma pack(4)
+struct vol_attr_buf {
+	u_int32_t	length;
+	off_t   	volume_size;
+	u_int32_t	io_blksize;
+};
+#pragma pack()
+typedef struct vol_attr_buf vol_attr_buf;
+
+int fs_stat_tests( void * the_argp )
+{
+	int					my_err, my_count, i;
+	int					my_buffer_size, my_buffer64_size;
+	int					my_fd = -1;
+	int					is_ufs = 0;
+	void *				my_bufferp = NULL;
+	void *				my_buffer64p = NULL;
+	struct statfs *		my_statfsp;
+	struct statfs64 *	my_statfs64p;
+	long				my_io_size;
+	fsid_t				my_fsid;
+	struct attrlist		my_attrlist;
+	vol_attr_buf		my_attr_buf;
+
+	my_buffer_size = (sizeof(struct statfs) * 10);
+	my_buffer64_size = (sizeof(struct statfs64) * 10);
+	my_bufferp = malloc( my_buffer_size );
+	if ( my_bufferp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_buffer64p = malloc( my_buffer64_size );
+	if ( my_buffer64p == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_statfsp = (struct statfs *) my_bufferp;
+	my_err = statfs( "/", my_statfsp );
+	if ( my_err == -1 ) {
+		printf( "statfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( memcmp( &my_statfsp->f_fstypename[0], "ufs", 3 ) == 0 ) {
+		is_ufs = 1;
+	}
+	
+	my_count = getfsstat( (struct statfs *)my_bufferp, my_buffer_size, MNT_NOWAIT );
+	if ( my_count == -1 ) {
+		printf( "getfsstat call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* validate results */
+	my_statfsp = (struct statfs *) my_bufferp;
+	for ( i = 0; i < my_count; i++, my_statfsp++ ) {
+		if ( memcmp( &my_statfsp->f_fstypename[0], "hfs", 3 ) == 0 ||
+			 memcmp( &my_statfsp->f_fstypename[0], "ufs", 3 ) == 0 ||
+			 memcmp( &my_statfsp->f_fstypename[0], "devfs", 5 ) == 0 ||
+			 memcmp( &my_statfsp->f_fstypename[0], "volfs", 5 ) == 0 ) {
+			/* found a valid entry */
+			break;
+		}
+	}
+	if ( i >= my_count ) {
+		printf( "getfsstat call failed.  could not find valid f_fstypename! \n" );
+		goto test_failed_exit;
+	}
+
+	/* now try statfs64 */
+	my_statfs64p = (struct statfs64 *) my_buffer64p;
+	my_err = statfs64( "/", my_statfs64p );
+	if ( my_err == -1 ) {
+		printf( "statfs64 call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_statfs64p->f_fsid.val[0] != my_statfsp->f_fsid.val[0] ||
+		 my_statfs64p->f_fsid.val[1] != my_statfsp->f_fsid.val[1] ) {
+		printf( "statfs64 call failed.  wrong f_fsid! \n" );
+		goto test_failed_exit;
+	}
+	
+	my_count = getfsstat64( (struct statfs64 *)my_buffer64p, my_buffer64_size, MNT_NOWAIT );
+	if ( my_count == -1 ) {
+		printf( "getfsstat64 call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* validate results */
+	my_statfs64p = (struct statfs64 *) my_buffer64p;
+	for ( i = 0; i < my_count; i++, my_statfs64p++ ) {
+		if ( memcmp( &my_statfs64p->f_fstypename[0], "hfs", 3 ) == 0 ||
+			 memcmp( &my_statfs64p->f_fstypename[0], "ufs", 3 ) == 0 ||
+			 memcmp( &my_statfs64p->f_fstypename[0], "devfs", 5 ) == 0 ||
+			 memcmp( &my_statfs64p->f_fstypename[0], "volfs", 5 ) == 0 ) {
+			/* found a valid entry */
+			break;
+		}
+	}
+	if ( i >= my_count ) {
+		printf( "getfsstat64 call failed.  could not find valid f_fstypename! \n" );
+		goto test_failed_exit;
+	}
+
+	/* set up to validate results via multiple sources.  we use getattrlist to get volume
+	 * related attributes to verify against results from fstatfs and statfs - but only if
+	 * we are not targeting ufs volume since it doesn't support getattr calls
+	 */
+	if ( is_ufs == 0 ) {
+		memset( &my_attrlist, 0, sizeof(my_attrlist) );
+		my_attrlist.bitmapcount = ATTR_BIT_MAP_COUNT;
+		my_attrlist.volattr = (ATTR_VOL_SIZE | ATTR_VOL_IOBLOCKSIZE);
+		my_err = getattrlist( "/", &my_attrlist, &my_attr_buf, sizeof(my_attr_buf), 0 );
+		if ( my_err != 0 ) {
+			printf( "getattrlist call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+			goto test_failed_exit;
+		}
+	}
+	
+	/* open kernel to use as test file for fstatfs */
+ 	my_fd = open( "/mach_kernel", O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* testing fstatfs64 */
+	my_statfs64p = (struct statfs64 *) my_buffer64p;
+	my_err = fstatfs64( my_fd, my_statfs64p );
+	if ( my_err == -1 ) {
+		printf( "fstatfs64 call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* validate results - assumes we only boot from hfs or ufs */
+	if ( !(memcmp( &my_statfs64p->f_fstypename[0], "hfs", 3 ) == 0 ||
+		   memcmp( &my_statfs64p->f_fstypename[0], "ufs", 3 ) == 0) ) {
+		printf( "fstatfs64 call failed.  could not find valid f_fstypename! \n" );
+		goto test_failed_exit;
+	}
+	
+	/* testing fstatfs */
+	my_statfsp = (struct statfs *) my_bufferp;
+	my_err = fstatfs( my_fd, my_statfsp );
+	if ( my_err == -1 ) {
+		printf( "fstatfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* validate resutls */
+	if ( !(memcmp( &my_statfsp->f_fstypename[0], "hfs", 3 ) == 0 ||
+		   memcmp( &my_statfsp->f_fstypename[0], "ufs", 3 ) == 0) ) {
+		printf( "fstatfs call failed.  could not find valid f_fstypename! \n" );
+		goto test_failed_exit;
+	}
+	my_io_size = my_statfsp->f_iosize;
+	my_fsid = my_statfsp->f_fsid;
+	if ( is_ufs == 0 && my_statfsp->f_iosize != my_attr_buf.io_blksize ) {
+		printf( "fstatfs and getattrlist results do not match for volume block size  \n" );
+		goto test_failed_exit;
+	} 
+
+	/* try again with statfs */
+	my_err = statfs( "/mach_kernel", my_statfsp );
+	if ( my_err == -1 ) {
+		printf( "statfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* validate resutls */
+	if ( my_io_size != my_statfsp->f_iosize || my_fsid.val[0] != my_statfsp->f_fsid.val[0] ||
+		 my_fsid.val[1] != my_statfsp->f_fsid.val[1] ) {
+		printf( "statfs call failed.  wrong f_iosize or f_fsid! \n" );
+		goto test_failed_exit;
+	}
+	if ( is_ufs == 0 && my_statfsp->f_iosize != my_attr_buf.io_blksize ) {
+		printf( "statfs and getattrlist results do not match for volume block size  \n" );
+		goto test_failed_exit;
+	} 
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_bufferp != NULL ) {
+		free( my_bufferp );
+	 }
+	 if ( my_buffer64p != NULL ) {
+		free( my_buffer64p );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getpid, getppid, and pipe system calls.
+ *  **************************************************************************************************************
+ */
+int getpid_getppid_pipe_test( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+	ssize_t		my_count;
+	int			my_fildes[2] = {-1, -1};
+	off_t		my_current_offset;
+	char		my_pid_string[64];
+
+	my_err = pipe( &my_fildes[0] );
+	if ( my_err != 0 ) {
+		printf( "pipe call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure we can't seek on a pipe */
+	my_current_offset = lseek( my_fildes[0], 0, SEEK_CUR );
+	if ( my_current_offset != -1 ) {
+		printf( "lseek on pipe should fail but did not \n" );
+		goto test_failed_exit;
+	}
+	 
+	/* fork here and use pipe to communicate */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	else if ( my_pid == 0 ) {
+		/* child process */
+		unsigned long	my_ppid;
+		char			my_buffer[64];
+		
+		close( my_fildes[1] ); /* close write end of pipe */
+		my_fildes[1] = -1;
+		
+		/* get the parent's pid using getppid and from the parent (using getpid in porent) */
+		my_count = read( my_fildes[0], &my_buffer[0], sizeof(my_buffer) );
+		if ( my_count == -1 ) {
+			printf( "read from pipe failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+			exit(-1);
+		}
+		
+		/* parent wrote (to our pipe) its pid as character string */
+		my_ppid = strtoul( &my_buffer[0], NULL, 10 );
+		if ( my_ppid == 0 ) {
+			printf( "strtoul failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+			exit(-1);
+		}
+
+		if ( getppid( ) != my_ppid ) {
+			printf( "getppid failed.  pid we got from parent does not match getppid result. \n" );
+			exit(-1);
+		}
+		exit(0);
+	}
+	
+	/* parent process - get our pid using getpid and send it to child for verification */
+	close( my_fildes[0] ); /* close read end of pipe */
+	my_fildes[0] = -1;
+	
+	sprintf( &my_pid_string[0], "%d\n", getpid( ) );
+
+	my_count = write( my_fildes[1], &my_pid_string[0], sizeof(my_pid_string) );
+	if ( my_count == -1 ) {
+		printf( "write to pipe failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait for child to exit */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		printf( "wait4 returned wrong exit status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fildes[0] != -1 )
+		close( my_fildes[0] );
+	if ( my_fildes[1] != -1 )
+		close( my_fildes[1] );
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test getauid, gettid, getuid, geteuid, issetugid, setauid, seteuid, settid, settid_with_pid, setuid system calls.
+ *  **************************************************************************************************************
+ */
+int uid_tests( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	/* test issetugid - should return 1 when not root and 0 when root */
+	my_err = issetugid( );
+	if ( getuid( ) == 0 ) {
+		if ( my_err == 1 ) {
+			printf( "issetugid should return false \n" );
+			goto test_failed_exit;
+		}
+	}
+	else {
+		if ( my_err == 0 ) {
+			printf( "issetugid should return true \n" );
+			goto test_failed_exit;
+		}
+	}
+
+	/*
+	 * fork here and do the setuid work in the child 
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	else if ( my_pid == 0 ) {
+		/* 
+		 * child process 
+		 */
+		uid_t		my_ruid, my_euid;
+		uid_t		my_uid, my_temp_uid;
+		gid_t		my_gid, my_temp_gid;
+		au_id_t		my_au_id, my_temp_au_id;
+		
+		my_ruid = getuid( );
+		my_euid = geteuid( );
+		if ( my_ruid == my_euid ) {
+			exit( 0 );
+		}
+
+		/* Test getauid, gettid, setauid, settid, settid_with_pid */
+		/* get our current uid and gid for comparison later */
+		my_uid = getuid( );
+		my_gid = getgid( );
+
+		my_err = syscall( SYS_settid, 4444, 5555 );
+		//my_err = settid( 4444, 5555 );
+		if (my_err != 0) {
+			printf( "settid call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		my_err = syscall( SYS_gettid, &my_temp_uid, &my_temp_gid );
+		//my_err = gettid( &my_temp_uid, &my_temp_gid );
+		if (my_err != 0) {
+			printf( "gettid call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if (my_temp_uid != 4444) {
+			printf("get / settid test failed - wrong uid was set - %d \n", my_temp_uid);
+			exit( -1 );
+		}
+		if (my_temp_gid != 5555) {
+			printf("get / settid test failed - wrong gid was set - %d \n", my_temp_gid);
+			exit( -1 );
+		}
+
+		/* resume original identity */
+		my_err = syscall( SYS_settid, KAUTH_UID_NONE, KAUTH_GID_NONE );
+		//my_err = settid( KAUTH_UID_NONE, KAUTH_GID_NONE );
+		if (my_err != 0) {
+			printf( "settid revert - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		/* values should be returned to original settings */
+		my_temp_uid = getuid( );
+		if (my_temp_uid == 4444) {
+			printf("test failed - wrong uid was set - %d \n", my_temp_uid);
+			exit( -1 );
+		}
+		my_temp_gid = getgid( );
+		if (my_temp_gid == 5555) {
+			printf("test failed - wrong gid was set - %d \n", my_temp_gid);
+			exit( -1 );
+		}
+
+		/*
+		 * Assume the identity of our parent.
+		 */
+		my_err = syscall( SYS_settid_with_pid, getppid( ), 1 );
+		//my_err = settid_with_pid, my_target_pid, 1 );
+		if (my_err != 0) {
+			printf( "settid_with_pid assume - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		/*
+		 * Resume our identity.
+		 */
+		my_err = syscall( SYS_settid_with_pid, 0, 0 );
+		//my_err = settid_with_pid( my_target_pid, 0 );
+		if (my_err != 0) {
+			printf( "settid_with_pid resume - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		
+		/*
+		 * test to make sure setauid doesn't cause audit info to get lost from 
+		 * the credential.
+		 */
+		my_err = getauid( &my_au_id );
+		if (my_err != 0) {
+			printf( "getauid - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		//printf("current au_id is %d \n", my_au_id);
+		
+		my_temp_au_id = 442344;
+		my_err = setauid( &my_temp_au_id );
+		if (my_err != 0) {
+			printf( "setauid - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		my_temp_au_id = 0;
+		my_err = getauid( &my_temp_au_id );
+		if (my_err != 0) {
+			printf( "getauid - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		//printf("new au_id is %d \n", my_temp_au_id);
+
+		if (my_temp_au_id != 442344) {
+			printf("test failed - wrong au_id was set - %d \n", my_temp_au_id);
+			exit( -1 );
+		}
+		
+		my_err = setauid( &my_au_id );
+		if (my_err != 0) {
+			printf( "setauid - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		my_temp_au_id = 0;
+		my_err = getauid( &my_temp_au_id );
+		if (my_err != 0) {
+			printf( "getauid - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		if (my_temp_au_id != my_au_id) {
+			printf("test failed - wrong au_id was set - %d \n", my_temp_au_id);
+			exit( -1 );
+		}
+	
+		/* change real uid and effective uid to current euid */
+		my_err = setuid( my_euid );
+		if ( my_err == -1 ) {
+			printf( "setuid call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( getuid( ) != my_euid ) {
+			printf( "setuid call failed to set the real uid \n" );
+			exit( -1 );
+		}
+
+		/* change effective uid to current euid - really a NOP */
+		my_err = seteuid( my_euid );
+		if ( my_err == -1 ) {
+			printf( "seteuid call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( geteuid( ) != my_euid ) {
+			printf( "seteuid call failed to set the original euid \n" );
+			exit( -1 );
+		}
+
+		/* change real uid and effective uid to original real uid */
+		my_err = setuid( my_ruid );
+		if ( my_err == -1 ) {
+			printf( "setuid call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( getuid( ) != my_ruid ) {
+			printf( "setuid call failed to set the real uid \n" );
+			exit( -1 );
+		}
+
+		exit(0);
+	}
+	
+	/* 
+	 * parent process - 
+	 * wait for child to exit 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		printf( "wait4 returned wrong exit status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test mknod, sync system calls.
+ *  **************************************************************************************************************
+ */
+int mknod_sync_test( void * the_argp )
+{
+	int			my_err;
+	char *		my_pathp = NULL;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, "/dev/" );
+
+	/* get a unique name for our test file */
+	my_err = create_random_name( my_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_err = mknod( my_pathp, (S_IFCHR | S_IRWXU), 0 );	
+	if ( my_err == -1 ) {
+		printf( "mknod failed with errno %d - %s \n", errno, strerror( errno ) );
+		printf( "path \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	/* not really sure what to do with sync call test */
+	sync( );
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test chflags, fchflags system calls.
+ *  **************************************************************************************************************
+ */
+int chflags_fchflags_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd = -1;
+	u_int			my_flags;
+	char *			my_pathp = NULL;
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* make test file unchangable */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_flags = (my_sb.st_flags | UF_IMMUTABLE);
+	my_err = chflags( my_pathp, my_flags );
+	if ( my_err != 0 ) {
+		printf( "chflags call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* should fail with EPERM since we cannot change the file now */
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 && errno != EPERM ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "open failed with wrong error - should be EPERM \n" );
+		goto test_failed_exit;
+	}
+	
+	/* this open should work OK */
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_flags = (my_sb.st_flags & ~UF_IMMUTABLE);
+	my_err = fchflags( my_fd, my_flags );
+	if ( my_err != 0 ) {
+		printf( "chflags call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	close( my_fd );
+	my_fd = -1;
+	
+	/* should now work */
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test kill, vfork, execve system calls.
+ *  **************************************************************************************************************
+ */
+/*  There are many new exec() situations to test now that 64-bit is in. These extra tests are in response to 
+ * rdar://4606399 and rdar://4607285. It should cover every permutation of the following variables.
+ * 
+ *  - Current Process "Bitness": 			64 or 32
+ *  - exec()'ed process "bitness": 			64 or 32
+ *  	(if 64 bit, size of page zero:)			(4GB or 4KB)
+ *  - Parent Process "Bitness":				64 or 32
+
+ *  Test to make sure certain inheritance properties of fork()'ed children
+ * are correctly set.
+ *  1. 64 bit process forking() 64-bit child, child execing() 64-bit file (4GB pagezero)
+ *  2. 64 bit process forking() 64-bit child, child execing() 64-bit file (4KB pagezero)
+ *  3. 64 bit process forking() 64-bit child, child execing() 32-bit file
+ *  4. 32 bit process forking() 32-bit child, child execing() 32-bit file
+ *  5. 32 bit process forking() 32-bit child, child execing() 64 bit file (4GB pagezero) 
+ *  6. 32 bit process forking() 32-bit child, child execing() 64 bit file (4KB pagezero)
+ *
+ */
+
+
+int execve_kill_vfork_test( void * the_argp )
+{
+	int	my_err, my_status;
+	pid_t	my_pid, my_wait_pid;
+	char *	errmsg = NULL; 
+	char * argvs[2] = {"", NULL};
+	int bits = get_bits();		/* Gets actual processor bit-ness. */
+
+
+	if (bits != 32 && bits != 64) {
+		printf("Determination of processor bit-ness failed, get_bits() returned %d.\n", get_bits());
+		return(-1);
+	}
+
+	if (get_architecture() == -1) {
+		errmsg = "get_architecture() could not determine the CPU architecture.\n";
+		goto test_failed_exit;
+	}
+	if (get_architecture() == INTEL) {
+		if	(bits == 64 && sizeof(long) == 8) {
+			/*
+			 * Running on x86_64 hardware and running in 64-bit mode.
+			 * Check cases 1, 2, 3 and fork a child to check 4, 5, 6. 
+			 */ 
+			errmsg = "execve failed: from x86_64 forking and exec()ing 64-bit x86_64 process w/ 4G pagezero.\n";
+			argvs[0] = "sleep-x86_64-4G";
+			if (do_execve_test("helpers/sleep-x86_64-4G", argvs, NULL, 1))		goto test_failed_exit;
+
+			errmsg = "execve failed: from x86_64 forking and exec()ing 64-bit x86_64 process w/ 4K Pagezero.\n";
+			argvs[0] = "sleep-x86_64-4K";
+			if (do_execve_test("helpers/sleep-x86_64-4K", argvs, NULL, 1))		goto test_failed_exit;
+
+			errmsg = "execve failed: from x64_64 forking and exec()ing 32-bit i386 process.\n";
+			argvs[0] = "sleep-i386";
+			if (do_execve_test("helpers/sleep-i386", argvs, NULL, 1))		goto test_failed_exit;
+
+			/* Fork off a helper process and load a 32-bit program in it to test 32->64 bit exec(). */
+			errmsg = "execve failed to exec the helper process.\n";
+			argvs[0] = "launch-i386";
+			if (do_execve_test("helpers/launch-i386", argvs, NULL, 1) != 0)		goto test_failed_exit;
+
+			/* Test posix_spawn for ppc64 (should fail), i386, x86_64, and ppc (should succeed) */
+			errmsg = NULL;
+			if (do_spawn_test(CPU_TYPE_POWERPC64, 1))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_I386, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_X86_64, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+
+		}
+		else if (bits == 64 && sizeof(long) == 4) {
+			/*
+			 * Running on x86_64 hardware, but actually running in 32-bit mode.
+			 * Check cases 4, 5, 6 and fork a child to check 1, 2, 3.
+			 */
+			errmsg = "execve failed: from i386 forking and exec()ing i386 process.\n";
+			argvs[0] = "sleep-i386";
+			if (do_execve_test("helpers/sleep-i386", argvs, NULL, 0))		goto test_failed_exit;
+
+			errmsg = "execve failed: from i386 forking and exec()ing x86_64 process w/ 4G pagezero.\n";
+			argvs[0] = "sleep-x86_64-4G";
+			if (do_execve_test("helpers/sleep-x86_64-4G", argvs, NULL, 0))		goto test_failed_exit;
+
+			errmsg = "execve failed: from i386 forking and exec()ing x86_64 process w/ 4K pagezero.\n";
+			argvs[0] = "sleep-x86_64-4K";
+			if (do_execve_test("helpers/sleep-x86_64-4K", argvs, NULL, 0))		goto test_failed_exit;
+
+			/* Fork off a helper process and load a 64-bit program in it to test 64->32 bit exec(). */
+			errmsg = "execve failed to exec the helper process.\n";
+			argvs[0] = "launch-x86_64";
+			if (do_execve_test("helpers/launch-x86_64", argvs, NULL, 1) != 0)	goto test_failed_exit;
+
+			/* Test posix_spawn for ppc64 (should fail), i386, x86_64, and ppc (should succeed) */
+			errmsg = NULL;
+			if (do_spawn_test(CPU_TYPE_POWERPC64, 1))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_I386, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_X86_64, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+		}
+		else if (bits == 32) {
+			/* Running on i386 hardware. Check cases 4. */
+			errmsg = "execve failed: from i386 forking and exec()ing 32-bit i386 process.\n";
+			argvs[0] = "sleep-i386";
+			if (do_execve_test("helpers/sleep-i386", argvs, NULL, 1)) 		goto test_failed_exit;
+
+			/* Test posix_spawn for x86_64 (should fail), i386, and ppc (should succeed) */
+			errmsg = NULL;
+			if (do_spawn_test(CPU_TYPE_X86_64, 1))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_I386, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+		}
+	}
+	else if (get_architecture() == POWERPC) {
+		if	(bits == 64 && sizeof(long) == 8) {
+			/*
+			 * Running on PPC64 hardware and running in 64-bit mode.
+			 * Check cases 1, 2, 3 and fork a child to check 4, 5, 6. 
+			 */ 
+			errmsg = "execve failed: from ppc64 forking and exec()ing 64-bit ppc process w/ 4G pagezero.\n";
+			argvs[0] = "sleep-ppc64-4G";
+			if (do_execve_test("helpers/sleep-ppc64-4G", argvs, NULL, 1))		goto test_failed_exit;
+
+			errmsg = "execve failed: from ppc64 forking and exec()ing 64-bit ppc process w/ 4K pagezero.\n";
+			argvs[0] = "sleep-ppc64-4K";
+			if (do_execve_test("helpers/sleep-ppc64-4K", argvs, NULL, 1))		goto test_failed_exit;
+
+			errmsg = "execve failed: from ppc64 forking and exec()ing 32 bit ppc process.\n";
+			argvs[0] = "sleep-ppc32";
+			if (do_execve_test("helpers/sleep-ppc32", argvs, NULL, 1))		goto test_failed_exit;	
+
+			/* Fork off a helper process and load a 32-bit program in it to test 32->64 bit exec(). */
+			errmsg = "execve failed to exec the helper process.\n";
+			argvs[0] = "launch-ppc";
+			if (do_execve_test("helpers/launch-ppc", argvs, NULL, 1) != 0)		goto test_failed_exit;
+			
+			/* Test posix_spawn for i386 (should fail), ppc64, and ppc (should succeed) */
+			errmsg = NULL;
+			if (do_spawn_test(CPU_TYPE_I386, 1))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC64, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+
+		}
+		else if	(bits == 64 && sizeof(long) == 4) {
+			/*
+			 * Running on PPC64 hardware, but actually running in 32-bit mode.
+			 * Check cases 4, 5, 6 and fork a child to check 1, 2, 3. 
+			 */ 
+			errmsg = "execve failed: from ppc forking and exec()ing ppc process.\n";
+			argvs[0] = "sleep-ppc32";
+			if (do_execve_test("helpers/sleep-ppc32", argvs, NULL, 0))	goto test_failed_exit;
+
+			errmsg = "execve failed: from ppc forking and exec()ing ppc64 process w/ 4G pagezero.\n";
+			argvs[0] = "sleep-ppc64-4G";
+			if (do_execve_test("helpers/sleep-ppc64-4G", argvs, NULL, 0))	goto test_failed_exit;
+
+			errmsg = "execve failed: from ppc forking and exec()ing ppc64 process w/ 4K pagezero.\n";
+			argvs[0] = "sleep-ppc64-4K";
+			if (do_execve_test("helpers/sleep-ppc64-4K", argvs, NULL, 0))	goto test_failed_exit;
+
+			/* Fork off a helper process and load a 64-bit program in it to test 64->32 bit exec(). */
+			errmsg = "execve failed to exec the helper process.\n";
+			argvs[0] = "launch-ppc";
+			if (do_execve_test("helpers/launch-ppc64", argvs, NULL, 1) != 0)		goto test_failed_exit;
+
+			/* Test posix_spawn for i386 (should fail), ppc64, and ppc (should succeed) */
+			errmsg = NULL;
+			if (do_spawn_test(CPU_TYPE_I386, 1))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC64, 0))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+		}
+		else if (bits == 32) {
+			/* Running on ppc hardware. Check cases 4. */
+			errmsg = "execve failed: from ppc forking and exec()ing 32 bit ppc process.\n";
+			argvs[0] = "sleep-ppc32";
+			if (do_execve_test("helpers/sleep-ppc32", argvs, NULL, 1))		goto test_failed_exit;	
+			/* Test posix_spawn for i386 (should fail) and ppc (should succeed) */
+			errmsg = NULL;
+			 /* when under Rosetta, this process is CPU_TYPE_POWERPC, but the system should be able to run CPU_TYPE_I386 binaries */
+			if (do_spawn_test(CPU_TYPE_I386, (g_is_under_rosetta ? 0 : 1)))
+				goto test_failed_exit;
+			if (do_spawn_test(CPU_TYPE_POWERPC, 0))
+				goto test_failed_exit;
+		}
+	}
+	else {
+		/* Just in case someone decides we need more architectures in the future */
+		printf("get_architecture() returned unknown architecture");
+		return(-1);
+	}	
+
+	return 0;
+
+test_failed_exit:
+	if (errmsg)
+		printf("%s", errmsg);
+	return -1;
+}
+
+
+/*  **************************************************************************************************************
+ *	Test getegid, getgid, getgroups, setegid, setgid, setgroups system calls.
+ *  **************************************************************************************************************
+ */
+int groups_test( void * the_argp )
+{
+	int			my_err, i;
+	int			my_group_count, my_orig_group_count;
+	gid_t		my_real_gid;
+	gid_t		my_effective_gid;
+	gid_t		my_removed_gid;
+	gid_t		my_new_gid;
+	gid_t		my_groups[ NGROUPS_MAX ];
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_real_gid = getgid( );
+	my_effective_gid = getegid( );
+
+	/* start by getting list of groups the current user belongs to */
+	my_orig_group_count = getgroups( NGROUPS_MAX, &my_groups[0] );
+	if ( my_orig_group_count == -1 || my_orig_group_count < 1 ) {
+		printf( "getgroups call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure real and effective gids are correct */
+	for ( i = 0; i < my_orig_group_count; i++ ) {
+		if ( my_groups[i] == my_real_gid )
+			break;
+	}
+	if ( i >= my_orig_group_count ) {
+		printf( "getgid or getgroups call failed.  could not find real gid in list of groups. \n" );
+		goto test_failed_exit;
+	}
+	for ( i = 0; i < my_orig_group_count; i++ ) {
+		if ( my_groups[i] == my_effective_gid )
+			break;
+	}
+	if ( i >= my_orig_group_count ) {
+		printf( "getegid or getgroups call failed.  could not find effective gid in list of groups. \n" );
+		goto test_failed_exit;
+	}
+		
+	/* remove the last group */
+	my_removed_gid = my_groups[ (my_orig_group_count - 1) ];
+	my_err = setgroups( (my_orig_group_count - 1), &my_groups[0] );
+	if ( my_err == -1 ) {
+		printf( "setgroups call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_group_count = getgroups( NGROUPS_MAX, &my_groups[0] );
+	if ( my_group_count == -1 || my_group_count < 1 ) {
+		printf( "getgroups call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure setgroups dropped one */
+	if ( my_orig_group_count <= my_group_count ) {
+		printf( "setgroups call failed.  current group count is too high. \n" );
+		goto test_failed_exit;
+	}
+	
+	/* now put removed gid back */
+	my_groups[ (my_orig_group_count - 1) ] = my_removed_gid;
+	my_err = setgroups( my_orig_group_count, &my_groups[0] );
+	if ( my_err == -1 ) {
+		printf( "setgroups call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* find a group to change real and effective gid to then do it */
+	my_new_gid = -1;
+	for ( i = 0; i < my_orig_group_count; i++ ) {
+		if ( my_groups[i] == my_effective_gid || my_groups[i] == my_real_gid )
+			continue;
+		my_new_gid = my_groups[i];
+	}
+	
+	if ( my_new_gid == -1 ) {
+		printf( "could not find a gid to switch to. \n" );
+		goto test_failed_exit;
+	}
+	
+	/* test setegid */
+	my_err = setegid( my_new_gid );
+	if ( my_err == -1 ) {
+		printf( "setegid call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	/* verify it changed */
+	if ( getegid( ) != my_new_gid ) {
+		printf( "setegid failed to change the effective gid. \n" );
+		goto test_failed_exit;
+	}
+	/* change it back to original value */
+	my_err = setegid( my_effective_gid );
+	if ( my_err == -1 ) {
+		printf( "setegid call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* test setgid */
+	my_err = setgid( my_new_gid );
+	if ( my_err == -1 ) {
+		printf( "setgid call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	/* verify it changed */
+	if ( getgid( ) != my_new_gid ) {
+		printf( "setgid failed to change the real gid. \n" );
+		goto test_failed_exit;
+	}
+	/* change it back to original value */
+	my_err = setgid( my_real_gid );
+	if ( my_err == -1 ) {
+		printf( "setegid call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+		   
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test dup, dup2, getdtablesize system calls.
+ *  **************************************************************************************************************
+ */
+int dup_test( void * the_argp )
+{
+	int			my_err;
+	int			my_fd = -1;
+	int			my_newfd = -1;
+	int			my_table_size, my_loop_counter = 0;
+	char *		my_pathp = NULL;
+	ssize_t		my_count;
+	char		my_buffer[64];
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* test dup, dup2, getdtablesize */
+	my_table_size = getdtablesize( );
+	if ( my_table_size < 20 ) {
+		printf( "getdtablesize should return at least 20, returned %d \n", my_table_size );
+		goto test_failed_exit;
+	}
+
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_newfd = dup( my_fd );
+	if ( my_newfd == -1 ) {
+		printf( "dup call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+redo:
+	/* now write somne data to the orginal and new fd */
+	/* make sure test file is empty */
+	my_err = ftruncate( my_fd, 0 );		
+	if ( my_err == -1 ) {
+		printf( "ftruncate call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	lseek( my_fd, 0, SEEK_SET );
+	my_count = write( my_fd, "aa", 2 );
+	if ( my_count == -1 ) {
+		printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	my_count = write( my_newfd, "xx", 2 );
+	if ( my_count == -1 ) {
+		printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* now read it back and make sure data is correct */
+	lseek( my_fd, 0, SEEK_SET );
+	my_count = read( my_fd, &my_buffer[0], sizeof(my_buffer) );
+	if ( my_count == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_buffer[0] != 'a' || my_buffer[1] != 'a' || my_buffer[2] != 'x' || my_buffer[3] != 'x' ) {
+		printf( "wrong data in test file. \n" );
+		goto test_failed_exit;
+	}
+	
+	bzero( &my_buffer[0], sizeof(my_buffer) );
+	lseek( my_newfd, 0, SEEK_SET );
+	my_count = read( my_newfd, &my_buffer[0], sizeof(my_buffer) );
+	if ( my_count == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_buffer[0] != 'a' || my_buffer[1] != 'a' || my_buffer[2] != 'x' || my_buffer[3] != 'x' ) {
+		printf( "wrong data in test file. \n" );
+		goto test_failed_exit;
+	}
+
+	/* we do the above tests twice - once for dup and once for dup2 */
+	if ( my_loop_counter < 1 ) {
+		my_loop_counter++;
+		close( my_newfd );
+     
+		my_err = dup2( my_fd, my_newfd );
+		if ( my_err == -1 ) {
+			printf( "dup2 call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+		
+		goto redo;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_newfd != -1 )
+		close( my_newfd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test profil, getrusage system calls.
+ *  todo - how do we really test profil is working?
+ *  **************************************************************************************************************
+ */
+int getrusage_profil_test( void * the_argp )
+{
+	int				my_err;
+	char *			my_bufferp = NULL;
+	struct rusage	my_rusage;
+
+	my_bufferp = (char *) malloc( (1024 * 1000) );
+	if ( my_bufferp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	bzero( (void *)my_bufferp, (1024 * 1000) );
+	
+	/* turn on profiling */
+	my_err = profil( my_bufferp, (1024 * 1000), 0, 32768 );	
+	if ( my_err == -1 ) {
+		printf( "profil failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = getrusage( RUSAGE_SELF, &my_rusage );	
+	if ( my_err == -1 ) {
+		printf( "getrusage failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* do a sanity check on the getrusage results */
+	if ( my_rusage.ru_msgrcv > 1000 || my_rusage.ru_msgrcv < 0 ) {
+		printf( "getrusage seems to report wrong data - ru_msgrcv looks odd. \n" );
+		goto test_failed_exit;
+	}
+	if ( my_rusage.ru_nsignals > 1000 || my_rusage.ru_nsignals < 0 ) {
+		printf( "getrusage seems to report wrong data - ru_nsignals looks odd. \n" );
+		goto test_failed_exit;
+	}
+			
+	/* turn off profiling (scale value of 0 turns off profiling) */
+	my_err = profil( my_bufferp, (1024 * 1000), 0, 0 );	
+	if ( my_err == -1 ) {
+		printf( "profil failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_bufferp != NULL ) {
+		free( my_bufferp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getitimer, setitimer, sigaction, sigpending, sigprocmask, sigsuspend, sigwait system calls.
+ *  **************************************************************************************************************
+ */
+
+int		alarm_global = 0;
+void test_alarm_handler( int the_arg );
+void test_alarm_handler( int the_arg )
+{	
+	alarm_global = 4;
+	//printf( "test_alarm_handler - got here \n" );
+	if ( the_arg == 0 ) {
+	}
+	return;
+}
+
+void test_signal_handler( int the_arg );
+void test_signal_handler( int the_arg )
+{	
+	//printf( "test_signal_handler - got here \n" );
+	if ( the_arg == 0 ) {
+	}
+	return;
+}
+
+int signals_test( void * the_argp )
+{
+	int			my_err, my_status;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+	pid_t		my_pid, my_wait_pid;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/*
+	 * spin off a child process that we will use for signal related testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - test signal related system calls.
+		 */
+		//int					my_counter;
+		int					my_signal;
+		sigset_t			my_sigset;
+		struct sigaction	my_sigaction;
+#ifdef MAC_OS_X_VERSION_10_5
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5
+		/* If this is Leopard. To allow compiling for Inca x86_64 this definition cannot 
+		 * be included. But it is needed to compile on Leopard.
+		 */
+		struct __darwin_sigaltstack	my_sigaltstack;
+#endif
+#else
+		struct sigaltstack	my_sigaltstack;
+#endif
+		struct itimerval    my_timer;
+
+
+		/* test getting the current signal stack context */
+		my_err = sigaltstack( NULL, &my_sigaltstack );
+		if ( my_err == -1 ) {
+			printf( "sigaction failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		if ( (my_sigaltstack.ss_flags & SS_DISABLE) == 0 ) {
+			printf( "sigaction must have failed - SS_DISABLE is cleared \n" );
+			exit( -1 );
+		}
+				
+		/* set up to catch SIGUSR1 */
+		my_sigaction.sa_handler = test_signal_handler;
+		my_sigaction.sa_flags = SA_RESTART;
+		my_sigaction.sa_mask = 0;
+
+		my_err = sigaction( SIGUSR1, &my_sigaction, NULL );
+		if ( my_err == -1 ) {
+			printf( "sigaction failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+	 		
+		/* now suspend until signal SIGUSR1 is sent */ 
+		sigemptyset( &my_sigset );
+		my_err = sigsuspend( &my_sigset );
+		if ( my_err == -1 ) {
+			if ( errno != EINTR ) {
+				printf( "sigsuspend should have returned with errno EINTR \n" );
+				exit( -1 );
+			}
+		}
+					
+		/* block SIGUSR1 */
+		sigemptyset( &my_sigset );
+		sigaddset( &my_sigset, SIGUSR1 );
+		if ( sigismember( &my_sigset, SIGUSR1 ) == 0 ) {
+			printf( "sigaddset call failed to add SIGUSR1 to signal set \n" );
+			exit( -1 );
+		}
+		my_err = sigprocmask( SIG_BLOCK, &my_sigset, NULL );
+		if ( my_err == -1 ) {
+			printf( "sigprocmask failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		
+		/* make sure we are blocking SIGUSR1 */
+		sigemptyset( &my_sigset );
+		my_err = sigprocmask( 0, NULL, &my_sigset );
+		if ( my_err == -1 ) {
+			printf( "sigprocmask failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		if ( sigismember( &my_sigset, SIGUSR1 ) == 0 ) {
+			printf( "sigaddset call failed to add SIGUSR1 to signal set \n" );
+			exit( -1 );
+		}
+
+		/* our parent will send a 2nd SIGUSR1 signal which we should now see getting
+		 * blocked.
+		 */
+		sigemptyset( &my_sigset );
+		sigaddset( &my_sigset, SIGUSR1 );
+		my_err = sigwait( &my_sigset, &my_signal );
+		if ( my_err == -1 ) {
+			printf( "sigwait failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		//printf( "%s - %d - signal 0x%02X %d \n", __FUNCTION__, __LINE__, my_signal, my_signal );
+		if ( my_signal != SIGUSR1 ) {
+			printf( "sigwait failed to catch a pending SIGUSR1 signal. \n" );
+			exit( -1 );
+		}
+	 	 
+		/* now unblock SIGUSR1 */
+		sigfillset( &my_sigset );
+		sigdelset( &my_sigset, SIGUSR1 );
+		my_err = sigprocmask( SIG_UNBLOCK, &my_sigset, NULL );
+		if ( my_err == -1 ) {
+			printf( "sigprocmask failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		if ( sigismember( &my_sigset, SIGUSR1 ) != 0 ) {
+			printf( "sigprocmask call failed to unblock SIGUSR1 \n" );
+			exit( -1 );
+		}
+		
+		/* test get / setitimer */
+		timerclear( &my_timer.it_interval );
+		timerclear( &my_timer.it_value );
+		my_err = setitimer( ITIMER_VIRTUAL, &my_timer, NULL );
+		if ( my_err == -1 ) {
+			printf( "setitimer - ITIMER_VIRTUAL - failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		my_err = setitimer( ITIMER_PROF, &my_timer, NULL );
+		if ( my_err == -1 ) {
+			printf( "setitimer - ITIMER_PROF - failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+
+		/* set up to catch SIGALRM */
+		alarm_global = 0;
+		my_sigaction.sa_handler = test_alarm_handler;
+		my_sigaction.sa_flags = SA_RESTART;
+		my_sigaction.sa_mask = 0;
+
+		my_err = sigaction( SIGALRM, &my_sigaction, NULL );
+		if ( my_err == -1 ) {
+			printf( "sigaction - SIGALRM - failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		
+		/* set timer for half a second */
+		my_timer.it_value.tv_usec = (1000000 / 2);
+		my_err = setitimer( ITIMER_REAL, &my_timer, NULL );
+		if ( my_err == -1 ) {
+			printf( "setitimer - ITIMER_REAL - failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+	 		
+		/* now suspend until signal SIGALRM is sent */ 
+		sigfillset( &my_sigset );
+		sigdelset( &my_sigset, SIGALRM );
+		my_err = sigsuspend( &my_sigset );
+		if ( my_err == -1 ) {
+			if ( errno != EINTR ) {
+				printf( "sigsuspend should have returned with errno EINTR \n" );
+				exit( -1 );
+			}
+		}
+		if ( alarm_global != 4 ) {
+			printf( "setitimer test failed - did not catch SIGALRM \n" );
+			exit( -1 );
+		}
+
+		/* make sure ITIMER_REAL is now clear */
+		my_timer.it_value.tv_sec = 44;
+		my_timer.it_value.tv_usec = 44;
+		my_err = getitimer( ITIMER_REAL, &my_timer );
+		if ( my_err == -1 ) {
+			printf( "getitimer - ITIMER_REAL - failed with errno %d - %s \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+		if ( timerisset( &my_timer.it_value ) || timerisset( &my_timer.it_interval ) ) {
+			printf( "ITIMER_REAL is set, but should not be \n" );
+			exit( -1 );
+		}
+		
+		exit(0);
+	}
+	
+	/* 
+	 * parent process - let child set up to suspend then signal it with SIGUSR1
+	 */
+	sleep( 1 );
+	my_err = kill( my_pid, SIGUSR1 );
+	if ( my_err == -1 ) {
+		printf( "kill call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	 
+	/* send 2nd signal to suspended child - which should be blocking SIGUSR1 signals */
+	sleep( 1 );
+	my_err = kill( my_pid, SIGUSR1 );
+	if ( my_err == -1 ) {
+		printf( "kill call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	 
+	/* wait for child to exit */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getlogin, setlogin system calls.
+ *  **************************************************************************************************************
+ */
+int getlogin_setlogin_test( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - do getlogin and setlogin testing.
+		 */
+		char *		my_namep;
+		int			my_len;
+		char *		my_new_namep = NULL;
+		
+		my_namep = getlogin( );
+		if ( my_namep == NULL ) {
+			printf( "getlogin returned NULL name pointer \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+		my_len = strlen( my_namep ) + 4;
+
+		my_new_namep = (char *) malloc( my_len );
+		if ( my_new_namep == NULL ) {
+			printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+		bzero( (void *)my_new_namep, my_len );
+		strcat( my_new_namep, my_namep );
+		strcat( my_new_namep, "2" );
+
+		/* set new name */
+		my_err = setlogin( my_new_namep );
+		if ( my_err == -1 ) {
+			printf( "setlogin failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_err = -1;
+			goto exit_child;
+		}
+
+		/* make sure we set new name */
+		my_namep = getlogin( );
+		if ( my_namep == NULL ) {
+			printf( "getlogin returned NULL name pointer \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+		if ( memcmp( my_namep, my_new_namep, strlen( my_new_namep ) ) != 0 ) {
+			printf( "setlogin failed to set the new name \n" );
+			my_err = -1;
+			goto exit_child;
+		}
+	 
+		my_err = 0;
+exit_child:
+		if ( my_new_namep != NULL ) {
+			free( my_new_namep );
+		}
+		exit( my_err );
+	}
+	
+	/* parent process -
+	 * wait for child to exit 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test acct system call.
+ *  **************************************************************************************************************
+ */
+int acct_test( void * the_argp )
+{
+	int				my_err, my_status;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+	struct acct *	my_acctp;
+	pid_t			my_pid, my_wait_pid;
+	ssize_t			my_count;
+	char			my_buffer[ (sizeof(struct acct) + 32) ];
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* enable process accounting */
+	my_err =  acct( my_pathp );	
+	if ( my_err == -1 ) {
+		printf( "acct failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - do a little work then exit.
+		 */
+		my_err = execv( "/usr/bin/true", (char **) 0 );
+		exit( 0 );
+	}
+	
+	/* parent process -
+	 * wait for child to exit 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+
+	/* diable process accounting */
+	my_err =  acct( NULL );	
+	if ( my_err == -1 ) {
+		printf( "acct failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* now verify that there is accounting info in the log file */
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	lseek( my_fd, 0, SEEK_SET );
+	bzero( (void *)&my_buffer[0], sizeof(my_buffer) );
+	my_count = read( my_fd, &my_buffer[0], sizeof(struct acct) );
+	if ( my_count == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_acctp = (struct acct *) &my_buffer[0];
+	/* first letters in ac_comm should match the name of the executable */
+	if ( getuid( ) != my_acctp->ac_uid || getgid( ) != my_acctp->ac_gid ||
+			my_acctp->ac_comm[0] != 't' || my_acctp->ac_comm[1] != 'r' ) {
+		if (g_is_under_rosetta) {
+			// on x86 systems, data written by kernel to accounting info file is little endian; 
+                        // but Rosetta processes expects it to be big endian; so swap the uid for our test
+			if ( getuid( ) != OSSwapInt32(my_acctp->ac_uid) || 
+					getgid( ) != OSSwapInt32(my_acctp->ac_gid) ||
+					my_acctp->ac_comm[0] != 't' || 
+					my_acctp->ac_comm[1] != 'r' ) {
+				printf( "accounting data does not look correct under Rosetta.\n" );
+			}
+			else {
+				// is cool under Rosetta 
+				my_err = 0;
+				goto test_passed_exit;
+			}
+		}
+		else {
+			printf( "accounting data does not look correct \n" );
+		}
+		goto test_failed_exit;
+	}
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test ioctl system calls.
+ *  **************************************************************************************************************
+ */
+int ioctl_test( void * the_argp )
+{
+	int					my_err, my_result;
+	int					my_fd = -1;
+	struct statfs *		my_infop;
+	char *				my_ptr;
+    int					my_blksize;
+    long long			my_block_count;
+	char				my_name[ 128 ];
+
+	my_result = getmntinfo( &my_infop, MNT_NOWAIT );
+	if ( my_result < 1 ) {
+		printf( "getmntinfo failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* make this a raw device */
+	strcpy( &my_name[0], &my_infop->f_mntfromname[0] );
+	if ( (my_ptr = strrchr( &my_name[0], '/' )) != 0 ) {
+		if ( my_ptr[1] != 'r' ) {
+			my_ptr[ strlen( my_ptr ) ] = 0x00;
+			memmove( &my_ptr[2], &my_ptr[1], (strlen( &my_ptr[1] ) + 1) );
+			my_ptr[1] = 'r';
+		}
+	}
+
+	my_fd = open(&my_name[0], O_RDONLY );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+    /* obtain the size of the media (in blocks) */
+	my_err = ioctl( my_fd, DKIOCGETBLOCKCOUNT, &my_block_count );
+	if ( my_err == -1 ) {
+		printf( "ioctl DKIOCGETBLOCKCOUNT failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+    /* obtain the block size of the media */
+	my_err = ioctl( my_fd, DKIOCGETBLOCKSIZE, &my_blksize );
+	if ( my_err == -1 ) {
+		printf( "ioctl DKIOCGETBLOCKSIZE failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	//printf( "my_block_count %qd my_blksize %d \n", my_block_count, my_blksize );
+
+	/* make sure the returned data looks somewhat valid */
+	if ( my_blksize < 0 || my_blksize > (1024 * 1000) ) {
+		printf( "ioctl appears to have returned incorrect block size data \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test mkdir, rmdir, umask system calls.
+ *  **************************************************************************************************************
+ */
+int mkdir_rmdir_umask_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd = -1;
+	int				did_umask = 0;
+	char *			my_pathp = NULL;
+	mode_t			my_orig_mask;
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* get a unique name to use with mkdir */
+	my_err = create_random_name( my_pathp, 0 );
+	if ( my_err != 0 ) {
+		printf( "create_random_name failed with error %d\n", my_err );
+		goto test_failed_exit;
+	}
+	
+	/* set umask to clear WX for other and group and clear X for user */
+	my_orig_mask = umask( (S_IXUSR | S_IWGRP | S_IXGRP | S_IWOTH | S_IXOTH) );	
+	did_umask = 1;
+
+	/* create a directory with RWX for user, group, other (which should be limited by umask) */
+	my_err = mkdir( my_pathp, (S_IRWXU | S_IRWXG | S_IRWXO) );
+	if ( my_err == -1 ) {
+		printf( "mkdir failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* verify results - (S_IXUSR | S_IWGRP | S_IXGRP | S_IWOTH | S_IXOTH) should be clear*/
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( (my_sb.st_mode & (S_IXUSR | S_IWGRP | S_IXGRP | S_IWOTH | S_IXOTH)) != 0 ) {
+		printf( "umask did not limit modes as it should have \n" );
+		goto test_failed_exit;
+	}
+	
+	/* get rid of our test directory */
+	my_err = rmdir( my_pathp );
+	if ( my_err == -1 ) {
+		printf( "rmdir failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		rmdir( my_pathp );	
+		free( my_pathp );
+	 }
+	 if ( did_umask != 0 ) {
+		umask( my_orig_mask );	
+	 }
+
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test chroot system call.
+ *  **************************************************************************************************************
+ */
+int chroot_test( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+	char *		my_pathp = NULL;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf("\t skipping this test \n");
+		my_err = 0;
+		goto test_passed_exit;
+	}
+		
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* get a unique name for our test directory */
+	my_err = create_random_name( my_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	/* create a test directory */
+	my_err = mkdir( my_pathp, (S_IRWXU | S_IRWXG | S_IRWXO) );
+	if ( my_err == -1 ) {
+		printf( "mkdir failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process - do getlogin and setlogin testing.
+		 */
+		struct stat		my_sb;
+
+		/* change our root to our new test directory */
+		my_err = chroot( my_pathp );	 
+		if ( my_err != 0 ) {
+			printf( "chroot failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		
+		/* verify root directory is now an empty directory */
+		my_err = stat( "/", &my_sb );
+		if ( my_err != 0 ) {
+			printf( "stat call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( my_sb.st_nlink > 2 ) {
+			printf( "root dir should be emnpty! \n" );
+			exit( -1 );
+		}
+		exit( 0 );
+	}
+	
+	/* parent process -
+	 * wait for child to exit 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		printf( "bad exit status\n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_pathp != NULL ) {
+		my_err = rmdir( my_pathp );
+		if ( my_err != 0 ) {
+			printf( "rmdir failed with error %d - \"%s\" path %p\n", errno, strerror( errno), my_pathp );
+		}
+		free( my_pathp );
+	}
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getpgrp, getpgid, getsid, setpgid, setpgrp, setsid system calls.
+ *  **************************************************************************************************************
+ */
+int process_group_test( void * the_argp )
+{
+	int		my_err = 0, i = 0;
+	pid_t		my_session_id, my_pid, my_process_group;
+
+	/* get current session ID, pgid, and pid */
+	my_session_id = getsid( 0 );
+	if ( my_session_id == -1 ) {
+		printf( "getsid call failed with error %d - \"%s\" \n", 
+				errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_pid = getpid( );
+	my_process_group = getpgrp( );
+	 
+	
+	/* test getpgrp and getpgid - they should return the same results when 0 is passed to getpgid */
+	if ( my_process_group != getpgid( 0 ) ) {
+		printf( "getpgrp and getpgid did not return the same process group ID \n" );
+		goto test_failed_exit;
+	}
+
+	if ( my_pid == my_process_group ) {
+		/* we are process group leader */
+		my_err = setsid( );
+		if ( my_err == 0  || errno != EPERM ) {
+			printf( "setsid call should have failed with EPERM\n" );
+			goto test_failed_exit;
+		}
+	} else {
+		/* we are not process group leader: try creating new session */
+		my_err = setsid( );
+		if ( my_err == -1 ) {
+			printf( "setsid call failed with error %d - \"%s\" \n",
+					errno, strerror( errno ) );
+			goto test_failed_exit;
+		}
+
+		if ( my_process_group == getpgid( 0 ) ) {
+			printf( "process group was not reset \n" );
+			goto test_failed_exit;
+		}
+	}
+	
+	/* find an unused process group ID */
+	for ( i = 10000; i < 1000000; i++ ) {
+		my_process_group = getpgid( i );
+		if ( my_process_group == -1 ) {
+			break;
+		}
+	}
+
+	/* this should fail */
+	my_err = setpgid( 0, my_process_group );
+	if ( my_err != -1 ) {
+		printf( "setpgid should have failed, but did not \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test fcntl system calls.
+ *  **************************************************************************************************************
+ */
+int fcntl_test( void * the_argp )
+{
+	int			my_err, my_result, my_tmep;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* open our test file and use fcntl to get / set file descriptor flags */
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_result = fcntl( my_fd, F_GETFD, 0 );
+	if ( my_result == -1 ) {
+		printf( "fcntl - F_GETFD - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_tmep = (my_result & FD_CLOEXEC);
+	if ( my_tmep ) {
+		/* FD_CLOEXEC is on, let's turn it off */
+		my_result = fcntl( my_fd, F_SETFD, 0 );
+	}
+	else {
+		/* FD_CLOEXEC is off, let's turn it on */
+		my_result = fcntl( my_fd, F_SETFD, 1 );
+	}
+	if ( my_result == -1 ) {
+		printf( "fcntl - F_SETFD - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* now check to see if it is set correctly */
+	my_result = fcntl( my_fd, F_GETFD, 0 );
+	if ( my_result == -1 ) {
+		printf( "fcntl - F_GETFD - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_tmep == (my_result & 0x01) ) {
+		printf( "fcntl - F_SETFD failed to set FD_CLOEXEC correctly!!! \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getpriority, setpriority system calls.
+ *  **************************************************************************************************************
+ */
+int getpriority_setpriority_test( void * the_argp )
+{
+	int			my_err;
+	int			my_priority;
+	int			my_new_priority;
+
+	/* getpriority returns scheduling priority so -1 is a valid value */
+	errno = 0;
+	my_priority = getpriority( PRIO_PROCESS, 0 );
+	if ( my_priority == -1 && errno != 0 ) {
+		printf( "getpriority - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* change scheduling priority */
+	my_new_priority = (my_priority == PRIO_MIN) ? (my_priority + 10) : (PRIO_MIN);
+	my_err = setpriority( PRIO_PROCESS, 0, my_new_priority );
+	if ( my_err == -1 ) {
+		printf( "setpriority - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* verify change */
+	errno = 0;
+	my_priority = getpriority( PRIO_PROCESS, 0 );
+	if ( my_priority == -1 && errno != 0 ) {
+		printf( "getpriority - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	if ( my_priority != my_new_priority ) {
+		printf( "setpriority - failed to set correct scheduling priority \n" );
+		goto test_failed_exit;
+	}
+	
+	/* reset scheduling priority */
+	my_err = setpriority( PRIO_PROCESS, 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "setpriority - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test futimes, gettimeofday, settimeofday, utimes system calls.
+ *  **************************************************************************************************************
+ */
+int time_tests( void * the_argp )
+{
+	int					my_err;
+	int					my_fd = -1;
+	char *				my_pathp = NULL;
+	struct timeval		my_orig_time;
+	struct timeval		my_temp_time;
+	struct timeval		my_utimes[4];
+	struct timezone		my_tz;
+	struct stat			my_sb;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf( "\t skipping this test \n" );
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	my_err = gettimeofday( &my_orig_time, &my_tz );
+	if ( my_err == -1 ) {
+		printf( "gettimeofday - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	//printf( "tv_sec %d tv_usec %ld \n", my_orig_time.tv_sec, my_orig_time.tv_usec );
+	
+	my_temp_time = my_orig_time;
+	my_temp_time.tv_sec -= 60;
+	my_err = settimeofday( &my_temp_time, NULL );
+	if ( my_err == -1 ) {
+		printf( "settimeofday - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = gettimeofday( &my_temp_time, NULL );
+	if ( my_err == -1 ) {
+		printf( "gettimeofday - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	//printf( "tv_sec %d tv_usec %ld \n", my_temp_time.tv_sec, my_temp_time.tv_usec );
+	if ( my_orig_time.tv_sec <= my_temp_time.tv_sec ) {
+		printf( "settimeofday did not set correct time \n" );
+		goto test_failed_exit;
+	}
+
+	/* set time back to original value plus 1 second */
+	my_temp_time = my_orig_time;
+	my_temp_time.tv_sec += 1;
+	my_err = settimeofday( &my_temp_time, NULL );
+	if ( my_err == -1 ) {
+		printf( "settimeofday - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* test utimes and futimes - get current access and mod times then change them */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	TIMESPEC_TO_TIMEVAL( &my_utimes[0], &my_sb.st_atimespec );
+	TIMESPEC_TO_TIMEVAL( &my_utimes[1], &my_sb.st_mtimespec );
+	my_utimes[0].tv_sec -= 120;		/* make access time 2 minutes older */ 
+	my_utimes[1].tv_sec -= 120;		/* make mod time 2 minutes older */ 
+	
+	my_err = utimes( my_pathp, &my_utimes[0] );
+	if ( my_err == -1 ) {
+		printf( "utimes - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* make sure the correct times are set */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	TIMESPEC_TO_TIMEVAL( &my_utimes[2], &my_sb.st_atimespec );
+	TIMESPEC_TO_TIMEVAL( &my_utimes[3], &my_sb.st_mtimespec );
+	if ( my_utimes[0].tv_sec != my_utimes[2].tv_sec ||
+		 my_utimes[1].tv_sec != my_utimes[3].tv_sec ) {
+		printf( "utimes failed to set access and mod times \n" );
+		goto test_failed_exit;
+	}
+	
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_utimes[0].tv_sec -= 120;  /* make access time 2 minutes older */ 
+	my_utimes[1].tv_sec -= 120;  /* make mod time 2 minutes older */ 
+	my_err = futimes( my_fd, &my_utimes[0] );
+	if ( my_err == -1 ) {
+		printf( "futimes - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* make sure the correct times are set */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	TIMESPEC_TO_TIMEVAL( &my_utimes[2], &my_sb.st_atimespec );
+	TIMESPEC_TO_TIMEVAL( &my_utimes[3], &my_sb.st_mtimespec );
+	if ( my_utimes[0].tv_sec != my_utimes[2].tv_sec ||
+		 my_utimes[1].tv_sec != my_utimes[3].tv_sec ) {
+		printf( "futimes failed to set access and mod times \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test rename, stat system calls.
+ *  **************************************************************************************************************
+ */
+int rename_test( void * the_argp )
+{
+	int				my_err;
+	char *			my_pathp = NULL;
+	char *			my_new_pathp = NULL;
+	ino_t			my_file_id;
+	struct stat		my_sb;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_new_pathp = (char *) malloc( PATH_MAX );
+	if ( my_new_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_new_pathp = 0x00;
+	strcat( my_new_pathp, &g_target_path[0] );
+	strcat( my_new_pathp, "/" );
+
+	/* get a unique name for our rename test */
+	my_err = create_random_name( my_new_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+		
+	/* save file ID for later use */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_file_id = my_sb.st_ino;
+	
+	/* test rename */
+	my_err = rename( my_pathp, my_new_pathp );
+	if ( my_err == -1 ) {
+		printf( "rename - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+		
+	/* make sure old name is no longer there */
+	my_err = stat( my_pathp, &my_sb );
+	if ( my_err == 0 ) {
+		printf( "rename call failed - found old name \n" );
+		goto test_failed_exit;
+	}
+		
+	/* make sure new name is there and is correct file id */
+	my_err = stat( my_new_pathp, &my_sb );
+	if ( my_err != 0 ) {
+		printf( "stat - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_file_id != my_sb.st_ino ) {
+		printf( "rename failed - wrong file id \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	if ( my_new_pathp != NULL ) {
+		remove( my_new_pathp );	
+		free( my_new_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test locking system calls.
+ *  **************************************************************************************************************
+ */
+int locking_test( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* test flock */
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err =  flock( my_fd, LOCK_EX );
+	if ( my_err == -1 ) {
+		printf( "flock - LOCK_EX - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process.
+		 */
+		int			my_child_fd = -1;
+		int			my_child_err;
+		
+		my_child_fd = open( my_pathp, O_RDWR, 0 );
+		if ( my_child_fd == -1 ) {
+			printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_child_err = -1;
+			goto child_exit;
+		}
+
+		my_err =  flock( my_child_fd, (LOCK_EX | LOCK_NB) );
+		if ( my_err == -1 ) {
+			if ( errno != EWOULDBLOCK ) {
+				printf( "flock call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+				my_child_err = -1;
+				goto child_exit;
+			}
+		}
+		else {
+			printf( "flock call should have failed with EWOULDBLOCK err \n" );
+			my_child_err = -1;
+			goto child_exit;
+		}
+		my_child_err = 0;
+child_exit:
+		if ( my_child_fd != -1 )
+			close( my_child_fd );
+		exit( my_child_err );
+	}
+
+	/* parent process -
+	 * wait for child to exit 
+	 */
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_err =  flock( my_fd, LOCK_UN );
+	if ( my_err == -1 ) {
+		printf( "flock - LOCK_UN - failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test mkfifo system calls.
+ *  **************************************************************************************************************
+ */
+int mkfifo_test( void * the_argp )
+{
+	int			my_err, my_status;
+	pid_t		my_pid, my_wait_pid;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+	ssize_t		my_result;
+	off_t		my_current_offset;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* get unique name for our fifo */
+	my_err = create_random_name( my_pathp, 0 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+
+	my_err = mkfifo( my_pathp, (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) );
+	if ( my_err != 0 ) {
+		printf( "mkfifo failed with errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/*
+	 * spin off a child process that we will use for testing.   
+	 */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_pid == 0 ) {
+		/* 
+		 * child process.
+		 */
+		int			my_child_fd = -1;
+		int			my_child_err;
+		char		my_buffer[64];
+		
+		/* open read end of fifo */
+		my_child_fd = open( my_pathp, O_RDWR, 0 );
+		if ( my_child_fd == -1 ) {
+			printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_child_err = -1;
+			goto child_exit;
+		}
+
+		/* read message from parent */
+		bzero( (void *)&my_buffer[0], sizeof(my_buffer) );
+		my_result = read( my_child_fd, &my_buffer[0], sizeof(my_buffer) );
+		if ( my_result == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			my_child_err = -1;
+			goto child_exit;
+		}
+		if ( strcmp( "parent to child", &my_buffer[0] ) != 0 ) {
+			printf( "read wrong message from parent \n" );
+			my_child_err = -1;
+			goto child_exit;
+		}
+
+		my_child_err = 0;
+child_exit:
+		if ( my_child_fd != -1 )
+			close( my_child_fd );
+		exit( my_child_err );
+	}
+
+	/* parent process - open write end of fifo
+	 */
+	my_fd = open( my_pathp, O_WRONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure we can't seek on a fifo */
+	my_current_offset = lseek( my_fd, 0, SEEK_CUR );
+	if ( my_current_offset != -1 ) {
+		printf( "lseek on fifo should fail but did not \n" );
+		goto test_failed_exit;
+	}
+
+	my_result = write( my_fd, "parent to child", 15 );
+	if ( my_result == -1 ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test quotactl system calls.
+ *  **************************************************************************************************************
+ */
+int quotactl_test( void * the_argp )
+{
+	int				my_err;
+	int				is_quotas_on = 0;
+	struct dqblk	my_quota_blk;
+
+	if ( g_skip_setuid_tests != 0 ) {
+		printf( "\t skipping this test \n" );
+		my_err = 0;
+		goto test_passed_exit;
+	}
+	
+	/* start off by checking the status of quotas on the boot volume */
+	my_err = quotactl( "/mach_kernel", QCMD(Q_QUOTASTAT, USRQUOTA), 0, (caddr_t)&is_quotas_on );
+	if ( my_err == -1 ) {
+		printf( "quotactl - Q_QUOTASTAT - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	if ( is_quotas_on == 0 ) {
+		/* quotas are off */
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_err = quotactl( "/mach_kernel", QCMD(Q_GETQUOTA, USRQUOTA), getuid(), (caddr_t)&my_quota_blk );
+	if ( my_err == -1 ) {
+		printf( "quotactl - Q_GETQUOTA - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getrlimit, setrlimit system calls.
+ *  **************************************************************************************************************
+ */
+int limit_tests( void * the_argp )
+{
+	int				my_err;
+	struct rlimit	my_current_rlimit;
+	struct rlimit	my_rlimit;
+
+ 	my_err = getrlimit( RLIMIT_NOFILE, &my_current_rlimit );
+	if ( my_err == -1 ) {
+		printf( "getrlimit - failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_current_rlimit.rlim_cur != RLIM_INFINITY ) {
+		if ( my_current_rlimit.rlim_cur != my_current_rlimit.rlim_max )
+			my_current_rlimit.rlim_cur += 1;
+		else
+			my_current_rlimit.rlim_cur -= 1;
+		my_rlimit.rlim_cur = my_current_rlimit.rlim_cur;
+		my_rlimit.rlim_max = my_current_rlimit.rlim_max;
+		my_err = setrlimit( RLIMIT_NOFILE, &my_rlimit );
+		if ( my_err == -1 ) {
+			printf( "setrlimit - failed with errno %d - %s \n", errno, strerror( errno ) );
+			goto test_failed_exit;
+		}
+		
+		/* verify that we set a new limit */
+		bzero( (void *) &my_rlimit, sizeof( my_rlimit ) );
+		my_err = getrlimit( RLIMIT_NOFILE, &my_rlimit );
+		if ( my_err == -1 ) {
+			printf( "getrlimit - failed with errno %d - %s \n", errno, strerror( errno ) );
+			goto test_failed_exit;
+		}
+		if ( my_rlimit.rlim_cur != my_current_rlimit.rlim_cur ) {
+			printf( "failed to get/set new RLIMIT_NOFILE soft limit \n" );
+			printf( "soft limits - current %lld should be %lld \n", my_rlimit.rlim_cur, my_current_rlimit.rlim_cur );
+			goto test_failed_exit;
+		}
+#if CONFORMANCE_CHANGES_IN_XNU // can't do this check until conformance changes get into xnu 
+		printf( "hard limits - current %lld should be %lld \n", my_rlimit.rlim_max, my_current_rlimit.rlim_max );
+		if ( my_rlimit.rlim_max != my_current_rlimit.rlim_max ) {
+			printf( "failed to get/set new RLIMIT_NOFILE hard limit \n" );
+			goto test_failed_exit;
+		}
+#endif
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test getattrlist, getdirentries, getdirentriesattr, setattrlist system calls.
+ *  **************************************************************************************************************
+ */
+struct test_attr_buf {
+	uint32_t			length;
+	fsobj_type_t		obj_type;
+	fsobj_id_t			obj_id;
+	struct timespec   	backup_time;
+};
+	
+typedef struct test_attr_buf test_attr_buf;
+
+int directory_tests( void * the_argp )
+{
+	int					my_err, done, found_it, i;
+	int					my_fd = -1;
+	int					is_ufs = 0;
+	char *				my_pathp = NULL;
+	char *				my_bufp = NULL;
+	char *				my_file_namep;
+#ifdef __LP64__
+	unsigned int		my_base;
+	unsigned int		my_count;
+	unsigned int		my_new_state;
+#else
+	unsigned long		my_base;
+	unsigned long		my_count;
+	unsigned long		my_new_state;
+#endif
+	fsobj_id_t			my_obj_id;
+	struct timespec		my_new_backup_time;
+	struct attrlist		my_attrlist;
+	test_attr_buf		my_attr_buf[4];
+	struct statfs 		my_statfs_buf;
+
+	/* need to know type of file system */
+	my_err = statfs( &g_target_path[0], &my_statfs_buf );
+	if ( my_err == -1 ) {
+		printf( "statfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( memcmp( &my_statfs_buf.f_fstypename[0], "ufs", 3 ) == 0 ) {
+		is_ufs = 1;
+	}
+
+	my_bufp = (char *) malloc( (1024 * 5) );
+	if ( my_bufp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* get pointer to just the file name */
+	my_file_namep = strrchr( my_pathp, '/' );
+	my_file_namep++;
+	
+	/* check out the  test directory */
+	my_fd = open( &g_target_path[0], (O_RDONLY), 0 );
+	if ( my_fd == -1 ) {
+		printf( "open failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	done = found_it = 0;
+	while ( done == 0 ) {
+		int					my_result, i;
+		struct dirent *		my_dirent_p;
+		
+		my_result = getdirentries( my_fd, my_bufp, (1024 * 5), &my_base );
+		if ( my_result <= 0 )
+			break;
+		for ( i = 0; i < my_result; ) {
+			my_dirent_p = (struct dirent *) (my_bufp + i);
+#if DEBUG
+			printf( "d_ino %d d_reclen %d d_type %d d_namlen %d \"%s\" \n", 
+					 my_dirent_p->d_ino, my_dirent_p->d_reclen, my_dirent_p->d_type,
+					 my_dirent_p->d_namlen, &my_dirent_p->d_name[0] );
+#endif
+
+			i += my_dirent_p->d_reclen;
+			/* validate results by looking for our test file */
+			if ( my_dirent_p->d_type == DT_REG && my_dirent_p->d_ino != 0 &&
+				 strlen( my_file_namep ) == my_dirent_p->d_namlen &&
+				 memcmp( &my_dirent_p->d_name[0], my_file_namep, my_dirent_p->d_namlen ) == 0 ) {
+				done = found_it = 1;
+				break;
+			}
+		}
+	}
+	if ( found_it == 0 ) {
+		printf( "getdirentries failed to find test file. \n" );
+		goto test_failed_exit;
+	}
+
+	/* test get/setattrlist */
+	memset( &my_attrlist, 0, sizeof(my_attrlist) );
+	my_attrlist.bitmapcount = ATTR_BIT_MAP_COUNT;
+	my_attrlist.commonattr = (ATTR_CMN_OBJTYPE | ATTR_CMN_OBJID | ATTR_CMN_BKUPTIME); 
+	my_err = getattrlist( my_pathp, &my_attrlist, &my_attr_buf[0], sizeof(my_attr_buf[0]), 0 );
+	if ( my_err != 0 ) {
+		if ( errno == ENOTSUP && is_ufs ) {
+			/* getattr calls not supported on ufs */
+			my_err = 0;
+			goto test_passed_exit;
+		}
+		printf( "getattrlist call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	/* validate returned data */
+	if ( my_attr_buf[0].obj_type != VREG ) {
+		printf( "getattrlist returned incorrect obj_type data. \n" );
+		goto test_failed_exit;
+	}
+	
+	/* set new backup time */
+	my_obj_id = my_attr_buf[0].obj_id;
+	my_new_backup_time = my_attr_buf[0].backup_time;
+	my_new_backup_time.tv_sec += 60;
+	my_attr_buf[0].backup_time.tv_sec = my_new_backup_time.tv_sec;
+	my_attrlist.commonattr = (ATTR_CMN_BKUPTIME); 
+	my_err = setattrlist( my_pathp, &my_attrlist, &my_attr_buf[0].backup_time, sizeof(my_attr_buf[0].backup_time), 0 );
+	if ( my_err != 0 ) {
+		printf( "setattrlist call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* validate setattrlist using getdirentriesattr */
+	close( my_fd );
+	my_fd = open( &g_target_path[0], (O_RDONLY), 0 );
+	if ( my_fd == -1 ) {
+		printf( "open failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	memset( &my_attrlist, 0, sizeof(my_attrlist) );
+	memset( &my_attr_buf, 0, sizeof(my_attr_buf) );
+	my_attrlist.bitmapcount = ATTR_BIT_MAP_COUNT;
+	my_attrlist.commonattr = (ATTR_CMN_OBJTYPE | ATTR_CMN_OBJID | ATTR_CMN_BKUPTIME); 
+	my_count = 4;
+	my_base = 0;
+	my_err = getdirentriesattr( my_fd, &my_attrlist, &my_attr_buf[0], sizeof(my_attr_buf), &my_count,
+								&my_base, &my_new_state, 0 );
+	if ( my_err < 0 ) {
+		printf( "getdirentriesattr call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	found_it = 0;
+	for ( i = 0; i < my_count; i++ ) {
+		if ( my_attr_buf[i].obj_id.fid_objno == my_obj_id.fid_objno &&
+			 my_attr_buf[i].obj_id.fid_generation == my_obj_id.fid_generation ) {
+			found_it = 1;
+			if ( my_attr_buf[i].backup_time.tv_sec !=  my_new_backup_time.tv_sec ) {
+				printf( "setattrlist failed to set backup time. \n" );
+				goto test_failed_exit;
+			}
+		}
+	}
+	if ( found_it == 0 ) {
+		printf( "getdirentriesattr failed to find test file. \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	if ( my_bufp != NULL ) {
+		free( my_bufp );
+	 }
+	return( my_err );
+}
+
+/*  **************************************************************************************************************
+ *	Test exchangedata system calls.
+ *  **************************************************************************************************************
+ */
+int exchangedata_test( void * the_argp )
+{
+	int				my_err;
+	int				my_fd1 = -1;
+	int				my_fd2 = -1;
+	char *			my_file1_pathp = NULL;
+	char *			my_file2_pathp = NULL;
+	ssize_t			my_result;
+	char			my_buffer[16];
+	struct statfs	my_statfs_buf;
+
+	/* need to know type of file system */
+	my_err = statfs( &g_target_path[0], &my_statfs_buf );
+	if ( my_err == -1 ) {
+		printf( "statfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( memcmp( &my_statfs_buf.f_fstypename[0], "ufs", 3 ) == 0 ) {
+		/* ufs does not support exchangedata */
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_file1_pathp = (char *) malloc( PATH_MAX );
+	if ( my_file1_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_file1_pathp = 0x00;
+	strcat( my_file1_pathp, &g_target_path[0] );
+	strcat( my_file1_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_file1_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	my_fd1 = open( my_file1_pathp, O_RDWR, 0 );
+	if ( my_fd1 == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_result = write( my_fd1, "11111111", 8 );
+	if ( my_result == -1 ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_file2_pathp = (char *) malloc( PATH_MAX );
+	if ( my_file2_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_file2_pathp = 0x00;
+	strcat( my_file2_pathp, &g_target_path[0] );
+	strcat( my_file2_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_file2_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	my_fd2 = open( my_file2_pathp, O_RDWR, 0 );
+	if ( my_fd2 == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	my_result = write( my_fd2, "22222222", 8 );
+	if ( my_result == -1 ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	close(my_fd1);
+	my_fd1 = -1;
+	close(my_fd2);
+	my_fd2 = -1;
+	
+	/* test exchangedata */
+	my_err = exchangedata( my_file1_pathp, my_file2_pathp, 0 );
+	if ( my_err == -1 ) {
+		printf( "exchangedata failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* now validate exchange */
+	my_fd1 = open( my_file1_pathp, O_RDONLY, 0 );
+	if ( my_fd1 == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	bzero( (void *)&my_buffer[0], sizeof(my_buffer) );
+	my_result = read( my_fd1, &my_buffer[0], 8 );
+	if ( my_result == -1 ) {
+		printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	if ( memcmp( &my_buffer[0], "22222222", 8 ) != 0 ) {
+		printf( "exchangedata failed - incorrect data in file \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd1 != -1 )
+		close( my_fd1 );
+	if ( my_file1_pathp != NULL ) {
+		remove( my_file1_pathp );	
+		free( my_file1_pathp );
+	 }
+	if ( my_fd2 != -1 )
+		close( my_fd2 );
+	if ( my_file2_pathp != NULL ) {
+		remove( my_file2_pathp );	
+		free( my_file2_pathp );
+	 }
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test searchfs system calls.
+ *  **************************************************************************************************************
+ */
+
+struct packed_name_attr {
+    u_int32_t	            size;	/* Of the remaining fields */
+    struct attrreference	ref;	/* Offset/length of name itself */
+    char 			        name[  PATH_MAX ];
+};
+
+struct packed_attr_ref {
+    u_int32_t    		    size;	/* Of the remaining fields */
+    struct attrreference	ref;	/* Offset/length of attr itself */
+};
+
+struct packed_result {
+    u_int32_t	        size;		/* Including size field itself */
+    attrreference_t     obj_name;
+    struct fsobj_id	    obj_id;
+    struct timespec     obj_create_time;
+    char                room_for_name[ 64 ];
+};
+typedef struct packed_result packed_result;
+typedef struct packed_result * packed_result_p;
+
+#define MAX_MATCHES	10
+
+int searchfs_test( void * the_argp )
+{
+	int						my_err, my_items_found = 0;
+	char *					my_pathp = NULL;
+    unsigned long			my_matches;
+    unsigned long			my_search_options;
+    struct fssearchblock	my_search_blk;
+    struct attrlist			my_return_list;
+    struct searchstate		my_search_state;
+    struct packed_name_attr	my_info1;
+    struct packed_attr_ref	my_info2;
+    packed_result			my_result_buffer[ MAX_MATCHES ];
+	struct statfs			my_statfs_buf;
+
+	/* need to know type of file system */
+	my_err = statfs( &g_target_path[0], &my_statfs_buf );
+	if ( my_err == -1 ) {
+		printf( "statfs call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( memcmp( &my_statfs_buf.f_fstypename[0], "ufs", 3 ) == 0 ) {
+		/* ufs does not support exchangedata */
+		my_err = 0;
+		goto test_passed_exit;
+	}
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create test files */
+	my_err = create_file_with_name( my_pathp, "foo", 0 );
+	if ( my_err < 0 ) {
+		printf( "failed to create a test file name in \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	my_err = create_file_with_name( my_pathp, "foobar", 0 );
+	if ( my_err < 0 ) {
+		printf( "failed to create a test file name in \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	my_err = create_file_with_name( my_pathp, "foofoo", 0 );
+	if ( my_err < 0 ) {
+		printf( "failed to create a test file name in \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	my_err = create_file_with_name( my_pathp, "xxxfoo", 0 );
+	if ( my_err < 0 ) {
+		printf( "failed to create a test file name in \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+	
+	/* search target volume for all file system objects with "foo" in the name */
+    /* Set up the attributes we're searching on. */
+    my_search_blk.searchattrs.bitmapcount = ATTR_BIT_MAP_COUNT;
+    my_search_blk.searchattrs.reserved = 0;
+    my_search_blk.searchattrs.commonattr = ATTR_CMN_NAME;
+    my_search_blk.searchattrs.volattr = 0;
+    my_search_blk.searchattrs.dirattr = 0;
+    my_search_blk.searchattrs.fileattr = 0;
+    my_search_blk.searchattrs.forkattr = 0;
+    
+    /* Set up the attributes we want for all returned matches. */
+    /* Why is returnattrs a pointer instead of an embedded struct? */
+    my_search_blk.returnattrs = &my_return_list;
+    my_return_list.bitmapcount = ATTR_BIT_MAP_COUNT;
+    my_return_list.reserved = 0;
+    my_return_list.commonattr = ATTR_CMN_NAME | ATTR_CMN_OBJID | ATTR_CMN_CRTIME;
+    my_return_list.volattr = 0;
+    my_return_list.dirattr = 0;
+    my_return_list.fileattr = 0;
+    my_return_list.forkattr = 0;
+    
+    /* Allocate a buffer for returned matches */
+    my_search_blk.returnbuffer = my_result_buffer;
+    my_search_blk.returnbuffersize = sizeof(my_result_buffer);
+    
+    /* Pack the searchparams1 into a buffer */
+    /* NOTE: A name appears only in searchparams1 */
+    strcpy( my_info1.name, "foo" );
+    my_info1.ref.attr_dataoffset = sizeof(struct attrreference);
+    my_info1.ref.attr_length = strlen(my_info1.name) + 1;
+    my_info1.size = sizeof(struct attrreference) + my_info1.ref.attr_length;
+    my_search_blk.searchparams1 = &my_info1;
+    my_search_blk.sizeofsearchparams1 = my_info1.size + sizeof(u_int32_t);
+    
+    /* Pack the searchparams2 into a buffer */
+    my_info2.size = sizeof(struct attrreference);
+    my_info2.ref.attr_dataoffset = sizeof(struct attrreference);
+    my_info2.ref.attr_length = 0;
+    my_search_blk.searchparams2 = &my_info2;
+    my_search_blk.sizeofsearchparams2 = sizeof(my_info2);
+    
+    /* Maximum number of matches we want */
+    my_search_blk.maxmatches = MAX_MATCHES;
+    
+    /* Maximum time to search, per call */
+    my_search_blk.timelimit.tv_sec = 1;
+    my_search_blk.timelimit.tv_usec = 0;
+    
+    my_search_options = (SRCHFS_START | SRCHFS_MATCHPARTIALNAMES |
+						 SRCHFS_MATCHFILES | SRCHFS_MATCHDIRS);
+	do {
+		char *  my_end_ptr;
+		char *	my_ptr;
+		int		i;
+		
+		my_err = searchfs( my_pathp, &my_search_blk, &my_matches, 0, my_search_options, &my_search_state );
+        if ( my_err == -1 )
+            my_err = errno;
+        if ( (my_err == 0 || my_err == EAGAIN) && my_matches > 0 ) {
+            /* Unpack the results */
+          //  printf("my_matches %d \n", my_matches);
+            my_ptr = (char *) &my_result_buffer[0];
+            my_end_ptr = (my_ptr + sizeof(my_result_buffer));
+            for ( i = 0; i < my_matches; ++i ) {
+                packed_result_p		my_result_p = (packed_result_p) my_ptr;
+				char *				my_name_p;
+				
+				/* see if we foound all our test files */
+				my_name_p = (((char *)(&my_result_p->obj_name)) + my_result_p->obj_name.attr_dataoffset);
+				if ( memcmp( my_name_p, "foo", 3 ) == 0 ||
+					 memcmp( my_name_p, "foobar", 6 ) == 0 ||
+					 memcmp( my_name_p, "foofoo", 6 ) == 0 ||
+					 memcmp( my_name_p, "xxxfoo", 6 ) == 0 ) {
+					my_items_found++;
+				}
+#if DEBUG
+                printf("obj_name \"%.*s\" \n", 
+                    (int) my_result_p->obj_name.attr_length,
+                    (((char *)(&my_result_p->obj_name)) + 
+                     my_result_p->obj_name.attr_dataoffset));
+                printf("size %d fid_objno %d fid_generation %d tv_sec 0x%02LX \n", 
+                    my_result_p->size, my_result_p->obj_id.fid_objno, 
+                    my_result_p->obj_id.fid_generation, 
+                    my_result_p->obj_create_time.tv_sec);
+#endif				
+                my_ptr = (my_ptr + my_result_p->size);
+                if (my_ptr > my_end_ptr)
+                    break;
+            }
+        }
+	if ( !(my_err == 0 || my_err == EAGAIN) ) {
+		printf( "searchfs failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+	}
+	my_search_options &= ~SRCHFS_START;
+    } while ( my_err == EAGAIN );
+
+	if ( my_items_found < 4 ) {
+		printf( "searchfs failed to find all test files \n" );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_pathp != NULL ) {
+		char *   my_ptr = (my_pathp + strlen( my_pathp ));
+		strcat( my_pathp, "foo" );
+		remove( my_pathp );	
+		*my_ptr = 0x00;
+		strcat( my_pathp, "foobar" );
+		remove( my_pathp );	
+		*my_ptr = 0x00;
+		strcat( my_pathp, "foofoo" ); 
+		remove( my_pathp );	
+		*my_ptr = 0x00;
+		strcat( my_pathp, "xxxfoo" );
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+
+#define  AIO_TESTS_BUFFER_SIZE  (1024 * 4000)
+#define  AIO_TESTS_OUR_COUNT  5
+/*  **************************************************************************************************************
+ *	Test aio_error, aio_read, aio_return, aio_suspend, aio_write, fcntl system calls.
+ *  **************************************************************************************************************
+ */
+int aio_tests( void * the_argp )
+{
+	int					my_err, i;
+	char *				my_pathp;
+	struct aiocb *		my_aiocbp;
+	ssize_t				my_result;
+	struct timespec		my_timeout;
+	int					my_fd_list[ AIO_TESTS_OUR_COUNT ];
+	char *				my_buffers[ AIO_TESTS_OUR_COUNT ];
+	struct aiocb *		my_aiocb_list[ AIO_TESTS_OUR_COUNT ];
+	struct aiocb		my_aiocbs[ AIO_TESTS_OUR_COUNT ];
+	char *				my_file_paths[ AIO_TESTS_OUR_COUNT ];
+
+	/* set up to have the ability to fire off up to AIO_TESTS_OUR_COUNT async IOs at once */
+	memset( &my_fd_list[0], 0xFF, sizeof( my_fd_list ) );
+	memset( &my_buffers[0], 0x00, sizeof( my_buffers ) );
+	memset( &my_aiocb_list[0], 0x00, sizeof( my_aiocb_list ) );
+	memset( &my_file_paths[0], 0x00, sizeof( my_file_paths ) );
+	for ( i = 0; i < AIO_TESTS_OUR_COUNT; i++ ) {
+		my_buffers[ i ] = malloc( AIO_TESTS_BUFFER_SIZE );
+		if ( my_buffers[ i ] == NULL ) {
+			printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+
+		my_file_paths[ i ] = malloc( PATH_MAX );
+		if ( my_file_paths[ i ] == NULL ) {
+			printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+		my_pathp = my_file_paths[ i ];
+		*my_pathp = 0x00;
+		strcat( my_pathp, &g_target_path[0] );
+		strcat( my_pathp, "/" );
+
+		/* create a test file */
+		my_err = create_random_name( my_pathp, 1 );
+		if ( my_err != 0 ) {
+			goto test_failed_exit;
+		}
+		my_fd_list[ i ] = open( my_pathp, O_RDWR, 0 );
+		if ( my_fd_list[ i ] <= 0 ) {
+			printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			goto test_failed_exit;
+		}
+
+		my_aiocbp = &my_aiocbs[ i ];
+		my_aiocb_list[ i ] = my_aiocbp;
+		memset( my_aiocbp, 0x00, sizeof( *my_aiocbp ) );
+		my_aiocbp->aio_fildes = my_fd_list[ i ];
+		my_aiocbp->aio_buf = (char *) my_buffers[ i ];
+		my_aiocbp->aio_nbytes = 1024;
+		my_aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE; // no signals at completion;
+		my_aiocbp->aio_sigevent.sigev_signo = 0;
+	}
+
+	/* test direct IO (F_NOCACHE) and aio_write */
+	my_err = fcntl( my_fd_list[ 0 ], F_NOCACHE, 1 );
+	if ( my_err != 0 ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	my_aiocbp = &my_aiocbs[ 0 ];
+    my_aiocbp->aio_fildes = my_fd_list[ 0 ];
+	my_aiocbp->aio_offset = 0;
+	my_aiocbp->aio_buf = my_buffers[ 0 ];
+    my_aiocbp->aio_nbytes = AIO_TESTS_BUFFER_SIZE;
+    my_aiocbp->aio_reqprio = 0;
+    my_aiocbp->aio_sigevent.sigev_notify = 0;
+    my_aiocbp->aio_sigevent.sigev_signo = 0;
+    my_aiocbp->aio_sigevent.sigev_value.sival_int = 0;
+    my_aiocbp->aio_sigevent.sigev_notify_function = NULL;
+    my_aiocbp->aio_sigevent.sigev_notify_attributes = NULL;
+    my_aiocbp->aio_lio_opcode = 0;
+	
+	/* write some data */
+	memset( my_buffers[ 0 ], 'j', AIO_TESTS_BUFFER_SIZE );
+    my_err = aio_write( my_aiocbp );
+	if ( my_err != 0 ) {
+		printf( "aio_write failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+		goto test_failed_exit;
+	}
+    
+    while ( 1 ) {
+        my_err = aio_error( my_aiocbp );
+        if ( my_err == EINPROGRESS ) {
+            /* wait for IO to complete */
+            sleep( 1 );
+            continue;
+        }
+        else if ( my_err == 0 ) {
+            ssize_t		my_result;
+            my_result = aio_return( my_aiocbp );
+            break;
+        }
+        else {
+			printf( "aio_error failed with error %d - \"%s\" \n", my_err, strerror( my_err ) );
+			goto test_failed_exit;
+        }
+    } /* while loop */
+
+	/* read some data */
+	memset( my_buffers[ 0 ], 'x', AIO_TESTS_BUFFER_SIZE );
+    my_err = aio_read( my_aiocbp );
+
+    while ( 1 ) {
+        my_err = aio_error( my_aiocbp );
+        if ( my_err == EINPROGRESS ) {
+            /* wait for IO to complete */
+            sleep( 1 );
+            continue;
+        }
+        else if ( my_err == 0 ) {
+            ssize_t		my_result;
+            my_result = aio_return( my_aiocbp );
+			
+			if ( *(my_buffers[ 0 ]) != 'j' || *(my_buffers[ 0 ] + AIO_TESTS_BUFFER_SIZE - 1) != 'j' ) {
+				printf( "aio_read or aio_write failed - wrong data read \n" );
+				goto test_failed_exit;
+			}
+            break;
+        }
+        else {
+			printf( "aio_read failed with error %d - \"%s\" \n", my_err, strerror( my_err ) );
+			goto test_failed_exit;
+        }
+    } /* while loop */
+
+	/* test aio_fsync */
+	close( my_fd_list[ 0 ] );
+	my_fd_list[ 0 ] = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd_list[ 0 ] == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_aiocbp = &my_aiocbs[ 0 ];
+    my_aiocbp->aio_fildes = my_fd_list[ 0 ];
+    my_aiocbp->aio_offset = 0;
+    my_aiocbp->aio_buf = my_buffers[ 0 ];
+    my_aiocbp->aio_nbytes = 1024;
+    my_aiocbp->aio_reqprio = 0;
+    my_aiocbp->aio_sigevent.sigev_notify = 0;
+    my_aiocbp->aio_sigevent.sigev_signo = 0;
+    my_aiocbp->aio_sigevent.sigev_value.sival_int = 0;
+    my_aiocbp->aio_sigevent.sigev_notify_function = NULL;
+    my_aiocbp->aio_sigevent.sigev_notify_attributes = NULL;
+    my_aiocbp->aio_lio_opcode = 0;
+	
+	/* write some data */
+	memset( my_buffers[ 0 ], 'e', 1024 );
+    my_err = aio_write( my_aiocbp );
+	if ( my_err != 0 ) {
+		printf( "aio_write failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+		goto test_failed_exit;
+	}
+    while ( 1 ) {
+        my_err = aio_error( my_aiocbp );
+        if ( my_err == EINPROGRESS ) {
+            /* wait for IO to complete */
+            sleep( 1 );
+            continue;
+        }
+        else if ( my_err == 0 ) {
+            ssize_t		my_result;
+            my_result = aio_return( my_aiocbp );
+            break;
+        }
+        else {
+			printf( "aio_error failed with error %d - \"%s\" \n", my_err, strerror( my_err ) );
+			goto test_failed_exit;
+        }
+    } /* while loop */
+
+	my_err = aio_fsync( O_SYNC, my_aiocbp );
+	if ( my_err != 0 ) {
+		printf( "aio_fsync failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+		goto test_failed_exit;
+	}
+    while ( 1 ) {
+        my_err = aio_error( my_aiocbp );
+        if ( my_err == EINPROGRESS ) {
+            /* wait for IO to complete */
+            sleep( 1 );
+            continue;
+        }
+        else if ( my_err == 0 ) {
+			aio_return( my_aiocbp );
+            break;
+        }
+        else {
+			printf( "aio_error failed with error %d - \"%s\" \n", my_err, strerror( my_err ) );
+			goto test_failed_exit;
+        }
+    } /* while loop */
+
+	/* validate write */
+	memset( my_buffers[ 0 ], 0x20, 16 );
+	lseek( my_fd_list[ 0 ], 0, SEEK_SET );	
+	my_result = read( my_fd_list[ 0 ], my_buffers[ 0 ], 16);
+	if ( my_result == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( *(my_buffers[ 0 ]) != 'e' || *(my_buffers[ 0 ] + 16 - 1) != 'e' ) {
+		printf( "aio_fsync or aio_write failed - wrong data read \n" );
+		goto test_failed_exit;
+	}
+
+	/* test aio_suspend and lio_listio */
+	for ( i = 0; i < AIO_TESTS_OUR_COUNT; i++ ) {
+		memset( my_buffers[ i ], 'a', AIO_TESTS_BUFFER_SIZE );
+		my_aiocbp = &my_aiocbs[ i ];
+		my_aiocbp->aio_nbytes = AIO_TESTS_BUFFER_SIZE;
+		my_aiocbp->aio_lio_opcode = LIO_WRITE;
+	}
+    my_err = lio_listio( LIO_NOWAIT, my_aiocb_list, AIO_TESTS_OUR_COUNT, NULL );
+	if ( my_err != 0 ) {
+		printf( "lio_listio call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_timeout.tv_sec = 1;
+	my_timeout.tv_nsec = 0;
+	my_err = aio_suspend( (const struct aiocb *const*) my_aiocb_list, AIO_TESTS_OUR_COUNT, &my_timeout );
+	if ( my_err != 0 ) {
+		printf( "aio_suspend call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* test aio_cancel */
+	for ( i = 0; i < AIO_TESTS_OUR_COUNT; i++ ) {
+		my_aiocbp = &my_aiocbs[ i ];
+		my_err = aio_cancel( my_aiocbp->aio_fildes, my_aiocbp );
+		if ( my_err != AIO_ALLDONE && my_err != AIO_CANCELED && my_err != AIO_NOTCANCELED ) {
+			printf( "aio_cancel failed with error %d - \"%s\" \n", my_err, strerror( my_err) );
+			goto test_failed_exit;
+		}
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	for ( i = 0; i < AIO_TESTS_OUR_COUNT; i++ ) {
+		if ( my_fd_list[ i ] != -1 ) {
+			close( my_fd_list[ i ] );
+			my_fd_list[ i ] = -1;
+		}
+		if ( my_file_paths[ i ] != NULL ) {
+			remove( my_file_paths[ i ] );	
+			free( my_file_paths[ i ] );
+			my_file_paths[ i ] = NULL;
+		}
+		if ( my_buffers[ i ] != NULL ) {
+			free( my_buffers[ i ] );
+			my_buffers[ i ] = NULL;
+		}
+	}
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test kevent, kqueue system calls.
+ *  **************************************************************************************************************
+ */
+int kqueue_tests( void * the_argp )
+{
+	int				my_err, my_status;
+	int				my_kqueue = -1;
+	int				my_fd = -1;
+	char *			my_pathp = NULL;
+    pid_t			my_pid, my_wait_pid;
+	size_t			my_count;
+	int				my_sockets[ 2 ] = {-1, -1};
+	struct kevent	my_kevent;
+	struct timespec	my_timeout;
+	char			my_buffer[ 16 ];
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	my_fd = open( my_pathp, O_RDWR, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	my_err = socketpair( AF_UNIX, SOCK_STREAM, 0, &my_sockets[0] );
+	if ( my_err == -1 ) {
+		printf( "socketpair failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* fork here and use pipe to communicate */
+	my_pid = fork( );
+	if ( my_pid == -1 ) {
+		printf( "fork failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	else if ( my_pid == 0 ) {
+		/* 
+		 * child process - tell parent we are ready to go.
+		 */
+		my_count = write( my_sockets[1], "r", 1 );
+		if ( my_count == -1 ) {
+			printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+			exit( -1 );
+		}
+
+		my_count = read( my_sockets[1], &my_buffer[0], 1 );
+		if ( my_count == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( my_buffer[0] != 'g' ) {
+			printf( "read call on socket failed to get \"all done\" message \n" );
+			exit( -1 );
+		}
+
+		/* now do some work that will trigger events our parent will track */
+		my_count = write( my_fd, "11111111", 8 );
+		if ( my_count == -1 ) {
+			printf( "write call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+	
+		my_err = unlink( my_pathp );
+		if ( my_err == -1 ) {
+			printf( "unlink failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+
+		/* wait for parent to tell us to exit */
+		my_count = read( my_sockets[1], &my_buffer[0], 1 );
+		if ( my_count == -1 ) {
+			printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+			exit( -1 );
+		}
+		if ( my_buffer[0] != 'e' ) {
+			printf( "read call on socket failed to get \"all done\" message \n" );
+			exit( -1 );
+		}
+		exit(0);
+	}
+	
+	/* parent process - wait for child to spin up */
+	my_count = read( my_sockets[0], &my_buffer[0], sizeof(my_buffer) );
+	if ( my_count == -1 ) {
+		printf( "read call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_buffer[0] != 'r' ) {
+		printf( "read call on socket failed to get \"ready to go message\" \n" );
+		goto test_failed_exit;
+	}
+
+	/* set up a kqueue and register for some events */
+	my_kqueue = kqueue( );
+	if ( my_kqueue == -1 ) {
+		printf( "kqueue call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* look for our test file to get unlinked or written to */
+	EV_SET( &my_kevent, my_fd, EVFILT_VNODE, (EV_ADD | EV_CLEAR), (NOTE_DELETE | NOTE_WRITE), 0, 0 );
+
+	my_timeout.tv_sec = 0;
+	my_timeout.tv_nsec = 0;
+	my_err = kevent( my_kqueue, &my_kevent, 1, NULL, 0, &my_timeout);
+	if ( my_err == -1 ) {
+		printf( "kevent call to register events failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* tell child to get to work */
+	my_count = write( my_sockets[0], "g", 1 );
+	if ( my_count == -1 ) {
+		printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* go get vnode events */
+	EV_SET( &my_kevent, my_fd, EVFILT_VNODE, (EV_CLEAR), 0, 0, 0 );
+	my_err = kevent( my_kqueue, NULL, 0, &my_kevent, 1, NULL );
+	if ( my_err == -1 ) {
+		printf( "kevent call to get vnode events failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_err == 0 ) {
+		printf( "kevent call to get vnode events did not return any when it should have \n" );
+		goto test_failed_exit;
+	}
+	if ( (my_kevent.fflags & (NOTE_DELETE | NOTE_WRITE)) == 0 ) {
+		printf( "kevent call to get vnode events did not return NOTE_DELETE or NOTE_WRITE \n" );
+		printf( "fflags 0x%02X \n", my_kevent.fflags );
+		goto test_failed_exit;
+	}
+
+	/* tell child to get to exit */
+	my_count = write( my_sockets[0], "e", 1 );
+	if ( my_count == -1 ) {
+		printf( "write call failed.  got errno %d - %s. \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_wait_pid = wait4( my_pid, &my_status, 0, NULL );
+	if ( my_wait_pid == -1 ) {
+		printf( "wait4 failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	/* wait4 should return our child's pid when it exits */
+	if ( my_wait_pid != my_pid ) {
+		printf( "wait4 did not return child pid - returned %d should be %d \n", my_wait_pid, my_pid );
+		goto test_failed_exit;
+	}
+
+	if ( WIFEXITED( my_status ) && WEXITSTATUS( my_status ) != 0 ) {
+		printf( "wait4 returned wrong exit status - 0x%02X \n", my_status );
+		goto test_failed_exit;
+	}
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_sockets[0] != -1 )
+		close( my_sockets[0] );
+	if ( my_sockets[1] != -1 )
+		close( my_sockets[1] );
+	if ( my_kqueue != -1 )
+		close( my_kqueue );
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+
+/*  **************************************************************************************************************
+ *	Test msgctl, msgget, msgrcv, msgsnd system calls. 
+ *  **************************************************************************************************************
+ */
+int message_queue_tests( void * the_argp )
+{
+	int					my_err;
+	int					my_msg_queue_id = -1;
+	ssize_t				my_result;
+	struct msqid_ds		my_msq_ds;
+	struct testing_msq_message {
+		long	msq_type;
+		char	msq_buffer[ 32 ];
+	}					my_msg;
+
+	/* get a message queue established for our use */
+	my_msg_queue_id = msgget( IPC_PRIVATE, (IPC_CREAT | IPC_EXCL | IPC_R | IPC_W) );
+	if ( my_msg_queue_id == -1 ) {
+		printf( "msgget failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	
+	/* get some stats on our message queue */
+	my_err = msgctl( my_msg_queue_id, IPC_STAT, &my_msq_ds );
+	if ( my_err == -1 ) {
+		printf( "msgctl failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_msq_ds.msg_perm.cuid != geteuid( ) ) {
+		printf( "msgctl IPC_STAT failed to get correct creator uid \n" );
+		goto test_failed_exit;
+	}
+	if ( (my_msq_ds.msg_perm.mode & (IPC_R | IPC_W)) == 0 ) {
+		printf( "msgctl IPC_STAT failed to get correct mode \n" );
+		goto test_failed_exit;
+	}
+	
+	/* put a message into our queue */
+	my_msg.msq_type = 1;
+	strcpy( &my_msg.msq_buffer[ 0 ], "testing 1, 2, 3" );
+	my_err = msgsnd( my_msg_queue_id, &my_msg, sizeof( my_msg.msq_buffer ), 0 );
+	if ( my_err == -1 ) {
+		printf( "msgsnd failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+
+	my_err = msgctl( my_msg_queue_id, IPC_STAT, &my_msq_ds );
+	if ( my_err == -1 ) {
+		printf( "msgctl failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_msq_ds.msg_qnum != 1 ) {
+		printf( "msgctl IPC_STAT failed to get correct number of messages on the queue \n" );
+		goto test_failed_exit;
+	}
+
+	/* pull message off the queue */
+	bzero( (void *)&my_msg, sizeof( my_msg ) );
+	my_result = msgrcv( my_msg_queue_id, &my_msg, sizeof( my_msg.msq_buffer ), 0, 0 );
+	if ( my_result == -1 ) {
+		printf( "msgrcv failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_result != sizeof( my_msg.msq_buffer ) ) {
+		printf( "msgrcv failed to return the correct number of bytes in our buffer \n" );
+		goto test_failed_exit;
+	}
+	if ( strcmp( &my_msg.msq_buffer[ 0 ], "testing 1, 2, 3" ) != 0 ) {
+		printf( "msgrcv failed to get the correct message \n" );
+		goto test_failed_exit;
+	}
+
+	my_err = msgctl( my_msg_queue_id, IPC_STAT, &my_msq_ds );
+	if ( my_err == -1 ) {
+		printf( "msgctl failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	if ( my_msq_ds.msg_qnum != 0 ) {
+		printf( "msgctl IPC_STAT failed to get correct number of messages on the queue \n" );
+		goto test_failed_exit;
+	}
+
+	/* tear down the message queue */
+	my_err = msgctl( my_msg_queue_id, IPC_RMID, NULL );
+	if ( my_err == -1 ) {
+		printf( "msgctl IPC_RMID failed with errno %d - %s \n", errno, strerror( errno ) );
+		goto test_failed_exit;
+	}
+	my_msg_queue_id = -1;
+
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_msg_queue_id != -1 ) {
+		msgctl( my_msg_queue_id, IPC_RMID, NULL );
+	}
+	return( my_err );
+}
+
+
+#if TEST_SYSTEM_CALLS 
+
+/*  **************************************************************************************************************
+ *	Test xxxxxxxxx system calls.
+ *  **************************************************************************************************************
+ */
+int sample_test( void * the_argp )
+{
+	int			my_err;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* add your test code here... */
+	
+	
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
+#endif
diff --git a/tools/tests/xnu_quick_test/tests.h b/tools/tests/xnu_quick_test/tests.h
new file mode 100644
index 000000000..6505466ea
--- /dev/null
+++ b/tools/tests/xnu_quick_test/tests.h
@@ -0,0 +1,121 @@
+#ifndef _TESTS_H_
+#define	_TESTS_H_
+
+#ifndef DEBUG
+#define DEBUG                          0
+#endif
+#ifndef CONFORMANCE_TESTS_IN_XNU
+#define CONFORMANCE_TESTS_IN_XNU       0
+#endif
+#ifndef TEST_SYSTEM_CALLS
+#define TEST_SYSTEM_CALLS              0
+#endif
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>		/* Install signal handlers*/
+#include <spawn.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <mach/machine.h>	/* Used to determine host properties */
+#include <mach/vm_inherit.h>
+#include <sys/acct.h>
+#include <sys/aio.h>
+#include <sys/attr.h>
+#include <sys/dirent.h>
+#include <sys/disk.h>
+#include <sys/uio.h>
+#include <sys/kauth.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/param.h>
+#include <sys/ptrace.h>
+#include <sys/quota.h>
+#include <sys/resource.h>
+#include <sys/select.h>
+#include <sys/signal.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/sysctl.h>		/* Used to determine host properties */
+#include <sys/syslimits.h>
+#include <sys/time.h>
+#include <sys/ttycom.h>
+#include <sys/types.h>
+#include <sys/ucred.h>
+#include <sys/vnode.h>
+#include <sys/wait.h>
+	 
+#define MY_BUFFER_SIZE (1024 * 10)
+#define POWERPC	238947
+#define INTEL	38947			/* 
+					 * Random values used by execve tests to 
+					 * determine architecture of machine.
+					 */
+
+typedef int (*test_rtn_t)(void *);
+
+int access_chmod_fchmod_test( void * the_argp );
+int acct_test( void * the_argp );
+int aio_tests( void * the_argp );
+int bsd_shm_tests( void * the_argp );
+int chdir_fchdir_test( void * the_argp );
+int chflags_fchflags_test( void * the_argp );
+int chroot_test( void * the_argp );
+int chown_fchown_lchown_lstat_symlink_test( void * the_argp );
+int create_file_with_name( char *the_pathp, char *the_namep, int remove_existing );
+int create_random_name( char *the_pathp, int do_open );
+int directory_tests( void * the_argp );
+int do_execve_test(char * path, char * argv[], void * envpi, int killwait);
+int dup_test( void * the_argp );
+int exchangedata_test( void * the_argp );
+int execve_kill_vfork_test( void * the_argp );
+int fcntl_test( void * the_argp );
+int fork_wait4_exit_test( void * the_argp );
+int fs_stat_tests( void * the_argp );
+int get_architecture(void);				/* Intel or PPC */
+int get_bits(void);					/* 64 or 32 */
+int getlogin_setlogin_test( void * the_argp );
+int getpid_getppid_pipe_test( void * the_argp );
+int getpriority_setpriority_test( void * the_argp );
+int getrusage_profil_test( void * the_argp );
+int groups_test( void * the_argp );
+int ioctl_test( void * the_argp );
+int kqueue_tests( void * the_argp );
+int limit_tests( void * the_argp );
+int link_stat_unlink_test( void * the_argp );
+int locking_test( void * the_argp );
+int memory_tests( void * the_argp );
+int message_queue_tests( void * the_argp );
+int mkdir_rmdir_umask_test( void * the_argp );
+int mkfifo_test( void * the_argp );
+int mknod_sync_test( void * the_argp );
+int open_close_test( void * the_argp );
+int process_group_test( void * the_argp );
+int quotactl_test( void * the_argp );
+int read_write_test( void * the_argp );
+int rename_test( void * the_argp );
+int searchfs_test( void * the_argp );
+int sema_tests( void * the_argp );
+int sema2_tests( void * the_argp );
+int shm_tests( void * the_argp );
+int signals_test( void * the_argp );
+int socket_tests( void * the_argp );
+int socket2_tests( void * the_argp );
+int syscall_test( void * the_argp );
+int time_tests( void * the_argp );
+int uid_tests( void * the_argp );
+int xattr_tests( void * the_argp );
+
+struct test_entry 
+{
+	int				test_run_it;		/* 0 means do not run this test, else run it */
+	test_rtn_t		test_routine;		/* routine to call */
+	void *			test_input;			/* optional input to test_routine */ 
+	char *			test_infop;			/* information about what is tested */ 
+};
+typedef struct test_entry * test_entryp;
+
+#endif /* !_TESTS_H_ */
diff --git a/tools/tests/xnu_quick_test/xattr_tests.c b/tools/tests/xnu_quick_test/xattr_tests.c
new file mode 100644
index 000000000..bcd83802b
--- /dev/null
+++ b/tools/tests/xnu_quick_test/xattr_tests.c
@@ -0,0 +1,164 @@
+/*
+ *  xattr_tests.c
+ *  xnu_quick_test
+ *
+ *  Created by Jerry Cottingham on 6/2/2005.
+ *  Copyright 2005 Apple Computer Inc. All rights reserved.
+ *
+ */
+
+#include "tests.h"
+#include <sys/xattr.h>
+
+extern char  g_target_path[ PATH_MAX ];
+
+#define XATTR_TEST_NAME "com.apple.xattr_test"
+
+/*  **************************************************************************************************************
+ *	Test xattr system calls.
+ *  **************************************************************************************************************
+ */
+int xattr_tests( void * the_argp )
+{
+	int			my_err;
+	int			my_fd = -1;
+	char *		my_pathp = NULL;
+	ssize_t		my_result;
+	char		my_buffer[ 64 ];
+	char		my_xattr_data[ ] = "xattr_foo";
+
+	my_pathp = (char *) malloc( PATH_MAX );
+	if ( my_pathp == NULL ) {
+		printf( "malloc failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	*my_pathp = 0x00;
+	strcat( my_pathp, &g_target_path[0] );
+	strcat( my_pathp, "/" );
+
+	/* create a test file */
+	my_err = create_random_name( my_pathp, 1 );
+	if ( my_err != 0 ) {
+		goto test_failed_exit;
+	}
+	
+	/* use setxattr to add an attribute to our test file */
+	my_err = setxattr( my_pathp, XATTR_TEST_NAME, &my_xattr_data[0], sizeof(my_xattr_data), 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "setxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure it is there using listxattr and getxattr */
+	my_result = listxattr( my_pathp, NULL, 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "listxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != (strlen( XATTR_TEST_NAME ) + 1) ) {
+		printf( "listxattr did not get the attribute name length \n" );
+		goto test_failed_exit;
+	}
+
+	memset( &my_buffer[0], 0x00, sizeof( my_buffer ) );
+	my_result = getxattr( my_pathp, XATTR_TEST_NAME, &my_buffer[0], sizeof(my_buffer), 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "getxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != (strlen( &my_xattr_data[0] ) + 1) ||
+		 strcmp( &my_buffer[0], &my_xattr_data[0] ) != 0 ) {
+		printf( "getxattr did not get the correct attribute data \n" );
+		goto test_failed_exit;
+	}
+
+	/* use removexattr to remove an attribute to our test file */
+	my_err = removexattr( my_pathp, XATTR_TEST_NAME, 0 );
+	if ( my_err == -1 ) {
+		printf( "removexattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* make sure it is gone */
+	my_result = listxattr( my_pathp, NULL, 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "listxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 0 ) {
+		printf( "removexattr did not remove our test attribute \n" );
+		goto test_failed_exit;
+	}
+
+	/* repeat tests using file descriptor versions of the xattr system calls */
+	my_fd = open( my_pathp, O_RDONLY, 0 );
+	if ( my_fd == -1 ) {
+		printf( "open call failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		printf( "\t file we attempted to open -> \"%s\" \n", my_pathp );
+		goto test_failed_exit;
+	}
+
+	/* use fsetxattr to add an attribute to our test file */
+	my_err = fsetxattr( my_fd, XATTR_TEST_NAME, &my_xattr_data[0], sizeof(my_xattr_data), 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "fsetxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	
+	/* make sure it is there using flistxattr and fgetxattr */
+	my_result = flistxattr( my_fd, NULL, 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "flistxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != (strlen( XATTR_TEST_NAME ) + 1) ) {
+		printf( "flistxattr did not get the attribute name length \n" );
+		goto test_failed_exit;
+	}
+
+	memset( &my_buffer[0], 0x00, sizeof( my_buffer ) );
+	my_result = fgetxattr( my_fd, XATTR_TEST_NAME, &my_buffer[0], sizeof(my_buffer), 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "fgetxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != (strlen( &my_xattr_data[0] ) + 1) ||
+		 strcmp( &my_buffer[0], &my_xattr_data[0] ) != 0 ) {
+		printf( "fgetxattr did not get the correct attribute data \n" );
+		goto test_failed_exit;
+	}
+
+	/* use fremovexattr to remove an attribute to our test file */
+	my_err = fremovexattr( my_fd, XATTR_TEST_NAME, 0 );
+	if ( my_err == -1 ) {
+		printf( "fremovexattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+
+	/* make sure it is gone */
+	my_result = flistxattr( my_fd, NULL, 0, 0 );
+	if ( my_err == -1 ) {
+		printf( "flistxattr failed with error %d - \"%s\" \n", errno, strerror( errno) );
+		goto test_failed_exit;
+	}
+	if ( my_result != 0 ) {
+		printf( "fremovexattr did not remove our test attribute \n" );
+		goto test_failed_exit;
+	}
+	 
+	my_err = 0;
+	goto test_passed_exit;
+
+test_failed_exit:
+	my_err = -1;
+	
+test_passed_exit:
+	if ( my_fd != -1 )
+		close( my_fd );
+	if ( my_pathp != NULL ) {
+		remove( my_pathp );	
+		free( my_pathp );
+	 }
+	return( my_err );
+}
+
-- 
2.50.0

